From 1eaaded0eea36e59e3d20f94efb6e7e0df053aea Mon Sep 17 00:00:00 2001
From: Javier <jrzaurin@gmail.com>
Date: Sat, 5 Oct 2024 22:24:43 +0100
Subject: [PATCH 01/24] bayesian functionalities updated to support MPS backend

---
 .../_base_contrastive_denoising_trainer.py     |  3 ++-
 .../_base_encoder_decoder_trainer.py           |  3 ++-
 pytorch_widedeep/tab2vec.py                    | 14 +++++++++++---
 .../training/_base_bayesian_trainer.py         |  3 ++-
 pytorch_widedeep/training/bayesian_trainer.py  | 18 ++++++++++++++----
 pytorch_widedeep/training/trainer.py           |  3 ++-
 .../training/trainer_from_folder.py            |  5 +++--
 pytorch_widedeep/utils/general_utils.py        | 11 +++++++++++
 .../test_b_miscellaneous.py                    |  6 ++++--
 .../test_bayes_tab2vec/test_b_t2v.py           |  5 +++--
 .../test_mc_attn_layers.py                     |  3 ++-
 tests/test_tab2vec/test_t2v.py                 |  4 ++--
 12 files changed, 58 insertions(+), 20 deletions(-)

diff --git a/pytorch_widedeep/self_supervised_training/_base_contrastive_denoising_trainer.py b/pytorch_widedeep/self_supervised_training/_base_contrastive_denoising_trainer.py
index 95706b50..11c9b835 100644
--- a/pytorch_widedeep/self_supervised_training/_base_contrastive_denoising_trainer.py
+++ b/pytorch_widedeep/self_supervised_training/_base_contrastive_denoising_trainer.py
@@ -27,6 +27,7 @@
     CallbackContainer,
     LRShedulerCallback,
 )
+from pytorch_widedeep.utils.general_utils import setup_device
 from pytorch_widedeep.models.tabular.self_supervised import ContrastiveDenoisingModel
 from pytorch_widedeep.preprocessing.tab_preprocessor import TabPreprocessor
 
@@ -270,7 +271,7 @@ def _set_device_and_num_workers(**kwargs):
             if sys.platform == "darwin" and sys.version_info.minor > 7
             else os.cpu_count()
         )
-        default_device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+        default_device = setup_device()
         device = kwargs.get("device", default_device)
         num_workers = kwargs.get("num_workers", default_num_workers)
         return device, num_workers
diff --git a/pytorch_widedeep/self_supervised_training/_base_encoder_decoder_trainer.py b/pytorch_widedeep/self_supervised_training/_base_encoder_decoder_trainer.py
index fe0aa4a8..97491b4f 100644
--- a/pytorch_widedeep/self_supervised_training/_base_encoder_decoder_trainer.py
+++ b/pytorch_widedeep/self_supervised_training/_base_encoder_decoder_trainer.py
@@ -25,6 +25,7 @@
     CallbackContainer,
     LRShedulerCallback,
 )
+from pytorch_widedeep.utils.general_utils import setup_device
 from pytorch_widedeep.models.tabular.self_supervised import EncoderDecoderModel
 
 
@@ -245,7 +246,7 @@ def _set_device_and_num_workers(**kwargs):
             if sys.platform == "darwin" and sys.version_info.minor > 7
             else os.cpu_count()
         )
-        default_device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+        default_device = setup_device()
         device = kwargs.get("device", default_device)
         num_workers = kwargs.get("num_workers", default_num_workers)
         return device, num_workers
diff --git a/pytorch_widedeep/tab2vec.py b/pytorch_widedeep/tab2vec.py
index af3d9273..5ba2ab48 100644
--- a/pytorch_widedeep/tab2vec.py
+++ b/pytorch_widedeep/tab2vec.py
@@ -10,8 +10,6 @@
 from pytorch_widedeep.bayesian_models import BayesianWide, BayesianTabMlp
 from pytorch_widedeep.bayesian_models._base_bayesian_model import BaseBayesianModel
 
-device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
-
 
 class Tab2Vec:
     r"""Class to transform an input dataframe into vectorized form.
@@ -87,6 +85,7 @@ def __init__(
         self,
         tab_preprocessor: TabPreprocessor,
         model: Union[WideDeep, BayesianWide, BayesianTabMlp],
+        device: str | torch.device,
         return_dataframe: bool = False,
         verbose: bool = False,
     ):
@@ -94,6 +93,11 @@ def __init__(
 
         self._check_inputs(tab_preprocessor, model, verbose)
 
+        if isinstance(device, str):
+            self.device = torch.device(device)
+        else:
+            self.device = device
+
         self.tab_preprocessor = tab_preprocessor
         self.return_dataframe = return_dataframe
         self.verbose = verbose
@@ -158,7 +162,11 @@ def transform(
         """
 
         X_tab = self.tab_preprocessor.transform(df)
-        X = torch.from_numpy(X_tab.astype("float")).to(device)
+
+        if self.device.type == "mps":
+            X = torch.from_numpy(X_tab.astype("float32")).to(self.device)
+        else:
+            X = torch.from_numpy(X_tab.astype("float")).to(self.device)
 
         with torch.no_grad():
             if self.is_tab_transformer:
diff --git a/pytorch_widedeep/training/_base_bayesian_trainer.py b/pytorch_widedeep/training/_base_bayesian_trainer.py
index 02993ef9..fa508531 100644
--- a/pytorch_widedeep/training/_base_bayesian_trainer.py
+++ b/pytorch_widedeep/training/_base_bayesian_trainer.py
@@ -25,6 +25,7 @@
     CallbackContainer,
     LRShedulerCallback,
 )
+from pytorch_widedeep.utils.general_utils import setup_device
 from pytorch_widedeep.training._trainer_utils import bayesian_alias_to_loss
 from pytorch_widedeep.bayesian_models._base_bayesian_model import BaseBayesianModel
 
@@ -235,7 +236,7 @@ def _set_device_and_num_workers(**kwargs):
             if sys.platform == "darwin" and sys.version_info.minor > 7
             else os.cpu_count()
         )
-        default_device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+        default_device = setup_device()
         device = kwargs.get("device", default_device)
         num_workers = kwargs.get("num_workers", default_num_workers)
         return device, num_workers
diff --git a/pytorch_widedeep/training/bayesian_trainer.py b/pytorch_widedeep/training/bayesian_trainer.py
index 88c426fe..65020c23 100644
--- a/pytorch_widedeep/training/bayesian_trainer.py
+++ b/pytorch_widedeep/training/bayesian_trainer.py
@@ -86,7 +86,8 @@ class BayesianTrainer(BaseBayesianTrainer):
         Other infrequently used arguments that can also be passed as kwargs are:
 
         - **device**: `str`<br/>
-            string indicating the device. One of _'cpu'_ or _'gpu'_
+            string indicating the device. One of _'cpu'_, _'gpu'_ or 'mps' if
+            run on a Mac with Apple silicon or AMD GPU(s)
 
         - **num_workers**: `int`<br/>
             number of workers to be used internally by the data loaders
@@ -396,7 +397,10 @@ def _train_step(
     ):
         self.model.train()
 
-        X = X_tab.to(self.device)
+        try:
+            X = X_tab.to(self.device)
+        except TypeError:
+            X = X_tab.to(self.device, dtype=torch.float32)
         y = target.view(-1, 1).float() if self.objective != "multiclass" else target
         y = y.to(self.device)
 
@@ -424,7 +428,10 @@ def _eval_step(
     ):
         self.model.eval()
         with torch.no_grad():
-            X = X_tab.to(self.device)
+            try:
+                X = X_tab.to(self.device)
+            except TypeError:
+                X = X_tab.to(self.device, dtype=torch.float32)
             y = target.view(-1, 1).float() if self.objective != "multiclass" else target
             y = y.to(self.device)
 
@@ -479,7 +486,10 @@ def _predict(  # noqa: C901
                 for _, Xl in zip(tt, test_loader):
                     tt.set_description("predict")
 
-                    X = Xl[0].to(self.device)
+                    try:
+                        X = Xl[0].to(self.device)
+                    except TypeError:
+                        X = Xl[0].to(self.device, dtype=torch.float32)
 
                     if return_samples:
                         preds = torch.stack([self.model(X) for _ in range(n_samples)])
diff --git a/pytorch_widedeep/training/trainer.py b/pytorch_widedeep/training/trainer.py
index f4dd1a91..873a11b4 100644
--- a/pytorch_widedeep/training/trainer.py
+++ b/pytorch_widedeep/training/trainer.py
@@ -151,7 +151,8 @@ class Trainer(BaseTrainer):
         Other infrequently used arguments that can also be passed as kwargs are:
 
         - **device**: `str`<br/>
-            string indicating the device. One of _'cpu'_ or _'gpu'_
+            string indicating the device. One of _'cpu'_, _'gpu'_ or 'mps' if
+            run on a Mac with Apple silicon or AMD GPU(s)
 
         - **num_workers**: `int`<br/>
             number of workers to be used internally by the data loaders
diff --git a/pytorch_widedeep/training/trainer_from_folder.py b/pytorch_widedeep/training/trainer_from_folder.py
index e2159b71..0641f3d8 100644
--- a/pytorch_widedeep/training/trainer_from_folder.py
+++ b/pytorch_widedeep/training/trainer_from_folder.py
@@ -156,8 +156,9 @@ class TrainerFromFolder(BaseTrainer):
      **kwargs: dict
          Other infrequently used arguments that can also be passed as kwargs are:
 
-         - **device**: `str`<br/>
-             string indicating the device. One of _'cpu'_ or _'gpu'_
+        - **device**: `str`<br/>
+            string indicating the device. One of _'cpu'_, _'gpu'_ or 'mps' if
+            run on a Mac with Apple silicon or AMD GPU(s)
 
          - **num_workers**: `int`<br/>
              number of workers to be used internally by the data loaders
diff --git a/pytorch_widedeep/utils/general_utils.py b/pytorch_widedeep/utils/general_utils.py
index 6db4b4b3..cac80e84 100644
--- a/pytorch_widedeep/utils/general_utils.py
+++ b/pytorch_widedeep/utils/general_utils.py
@@ -1,5 +1,16 @@
 from functools import wraps
 
+import torch
+
+
+def setup_device():
+    if torch.cuda.is_available():
+        return torch.device("cuda")
+    elif torch.backends.mps.is_available():
+        return torch.device("mps")
+    else:
+        return torch.device("cpu")
+
 
 def alias(original_name: str, alternative_names: list):
     def decorator(func):
diff --git a/tests/test_bayesian_models/test_bayes_model_functioning/test_b_miscellaneous.py b/tests/test_bayesian_models/test_bayes_model_functioning/test_b_miscellaneous.py
index 5b472a21..9e0c6629 100644
--- a/tests/test_bayesian_models/test_bayes_model_functioning/test_b_miscellaneous.py
+++ b/tests/test_bayesian_models/test_bayes_model_functioning/test_b_miscellaneous.py
@@ -66,7 +66,8 @@ def test_save_and_load(model):
     )
 
     new_model = torch.load(
-        "tests/test_bayesian_models/test_bayes_model_functioning/model_dir/bayesian_model.pt"
+        "tests/test_bayesian_models/test_bayes_model_functioning/model_dir/bayesian_model.pt",
+        weights_only=False,
     )
 
     if model.__class__.__name__ == "BayesianWide":
@@ -113,7 +114,8 @@ def test_save_and_load_dict(model_name):
 
     btrainer2.model.load_state_dict(
         torch.load(
-            "tests/test_bayesian_models/test_bayes_model_functioning/model_dir/bayesian_model.pt"
+            "tests/test_bayesian_models/test_bayes_model_functioning/model_dir/bayesian_model.pt",
+            weights_only=False,
         )
     )
 
diff --git a/tests/test_bayesian_models/test_bayes_tab2vec/test_b_t2v.py b/tests/test_bayesian_models/test_bayes_tab2vec/test_b_t2v.py
index 17d71f9f..ee82b931 100644
--- a/tests/test_bayesian_models/test_bayes_tab2vec/test_b_t2v.py
+++ b/tests/test_bayesian_models/test_bayes_tab2vec/test_b_t2v.py
@@ -2,15 +2,15 @@
 from random import choices
 
 import numpy as np
-import torch
 import pandas as pd
 import pytest
 
 from pytorch_widedeep import Tab2Vec
 from pytorch_widedeep.preprocessing import TabPreprocessor
 from pytorch_widedeep.bayesian_models import BayesianTabMlp
+from pytorch_widedeep.utils.general_utils import setup_device
 
-device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+device = setup_device()
 
 colnames = list(string.ascii_lowercase)[:4] + ["target"]
 cat_col1_vals = ["a", "b", "c"]
@@ -55,6 +55,7 @@ def test_bayesian_mlp_models(return_dataframe, embed_continuous):
     t2v = Tab2Vec(
         tab_preprocessor=tab_preprocessor,
         model=model,
+        device=device,
         return_dataframe=return_dataframe,
     )
     t2v_out, _ = t2v.transform(df_t2v, target_col="target")
diff --git a/tests/test_model_components/test_mc_attn_layers.py b/tests/test_model_components/test_mc_attn_layers.py
index e5106993..ead4b5fc 100644
--- a/tests/test_model_components/test_mc_attn_layers.py
+++ b/tests/test_model_components/test_mc_attn_layers.py
@@ -4,6 +4,7 @@
 import torch
 import pytest
 
+from pytorch_widedeep.utils.general_utils import setup_device
 from pytorch_widedeep.models.tabular.transformers._attention_layers import (
     MultiHeadedAttention,
 )
@@ -12,7 +13,7 @@
 
 input_dim = 128
 n_heads = 4
-device = "cuda" if torch.cuda.is_available() else "cpu"
+device = setup_device()
 
 standard_attn = (
     MultiHeadedAttention(
diff --git a/tests/test_tab2vec/test_t2v.py b/tests/test_tab2vec/test_t2v.py
index a466aadf..5a92b9e7 100644
--- a/tests/test_tab2vec/test_t2v.py
+++ b/tests/test_tab2vec/test_t2v.py
@@ -2,7 +2,6 @@
 from random import choices
 
 import numpy as np
-import torch
 import pandas as pd
 import pytest
 
@@ -21,8 +20,9 @@
     ContextAttentionMLP,
 )
 from pytorch_widedeep.preprocessing import TabPreprocessor
+from pytorch_widedeep.utils.general_utils import setup_device
 
-device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+device = setup_device()
 
 colnames = list(string.ascii_lowercase)[:4] + ["target"]
 cat_col1_vals = ["a", "b", "c"]

From 69a1fab263e6fd070239d1435a0c375610c3bfcd Mon Sep 17 00:00:00 2001
From: Javier <jrzaurin@gmail.com>
Date: Tue, 8 Oct 2024 18:10:26 +0100
Subject: [PATCH 02/24] Support for mps backend added. All unit test passing.
 Need to run some example scripts

---
 pytorch_widedeep/losses.py                    | 26 ++++++++--
 .../training/_base_bayesian_trainer.py        |  8 ++--
 pytorch_widedeep/training/_base_trainer.py    | 11 +++--
 .../training/_feature_importance.py           |  8 ++--
 pytorch_widedeep/training/_finetune.py        | 15 +++---
 pytorch_widedeep/training/bayesian_trainer.py | 16 ++-----
 pytorch_widedeep/training/trainer.py          | 18 +++----
 .../training/trainer_from_folder.py           | 18 +++----
 pytorch_widedeep/utils/general_utils.py       | 47 +++++++++++++++++--
 .../test_finetune/test_finetuning_routines.py | 16 +++----
 tests/test_hf_integration/test_models.py      |  1 +
 tests/test_tab2vec/test_t2v.py                |  4 ++
 12 files changed, 118 insertions(+), 70 deletions(-)

diff --git a/pytorch_widedeep/losses.py b/pytorch_widedeep/losses.py
index 2674f1ed..5f5a5769 100644
--- a/pytorch_widedeep/losses.py
+++ b/pytorch_widedeep/losses.py
@@ -272,8 +272,7 @@ def forward(self, input: Tensor, target: Tensor) -> Tensor:
         else:
             num_class = input_prob.size(1)
         binary_target = torch.eye(num_class)[target.squeeze().cpu().long()]
-        if use_cuda:
-            binary_target = binary_target.cuda()
+        binary_target = binary_target.to(input.device)
         binary_target = binary_target.contiguous()
         weight = self._get_weight(input_prob, binary_target)
 
@@ -430,9 +429,7 @@ def forward(self, input: Tensor, target: Tensor) -> Tensor:
         # when using max the two input tensors (input and other) have to be of
         # the same type
         max_input = F.softplus(input[..., 2:])
-        max_other = torch.sqrt(torch.Tensor([torch.finfo(torch.double).eps])).type(
-            max_input.type()
-        )
+        max_other = self.get_eps(max_input)
         scale = torch.max(max_input, max_other)
         safe_labels = positive * target + (1 - positive) * torch.ones_like(target)
 
@@ -446,6 +443,25 @@ def forward(self, input: Tensor, target: Tensor) -> Tensor:
 
         return torch.mean(classification_loss + regression_loss)
 
+    @staticmethod
+    def get_eps(max_input: Tensor) -> Tensor:
+        if max_input.device.type == "mps":
+            # For MPS, use float32 and then convert to the input type
+            eps = torch.finfo(torch.float32).eps
+            max_other = (
+                torch.sqrt(torch.tensor([eps], device="cpu"))
+                .to(max_input.device)
+                .to(max_input.dtype)
+            )
+        else:
+            # For other devices, use the original approach
+            eps = torch.finfo(torch.double).eps
+            max_other = (
+                torch.sqrt(torch.tensor([eps])).to(max_input.device).to(max_input.dtype)
+            )
+
+        return max_other
+
 
 class L1Loss(nn.Module):
     r"""L1 loss"""
diff --git a/pytorch_widedeep/training/_base_bayesian_trainer.py b/pytorch_widedeep/training/_base_bayesian_trainer.py
index fa508531..bbdf8255 100644
--- a/pytorch_widedeep/training/_base_bayesian_trainer.py
+++ b/pytorch_widedeep/training/_base_bayesian_trainer.py
@@ -12,6 +12,7 @@
 from pytorch_widedeep.wdtypes import (
     Any,
     List,
+    Tuple,
     Union,
     Module,
     Optional,
@@ -25,7 +26,7 @@
     CallbackContainer,
     LRShedulerCallback,
 )
-from pytorch_widedeep.utils.general_utils import setup_device
+from pytorch_widedeep.utils.general_utils import setup_device, to_device_model
 from pytorch_widedeep.training._trainer_utils import bayesian_alias_to_loss
 from pytorch_widedeep.bayesian_models._base_bayesian_model import BaseBayesianModel
 
@@ -58,8 +59,7 @@ def __init__(
         self.device, self.num_workers = self._set_device_and_num_workers(**kwargs)
 
         self.early_stop = False
-        self.model = model
-        self.model.to(self.device)
+        self.model = to_device_model(model, self.device)
 
         self.verbose = verbose
         self.seed = seed
@@ -230,7 +230,7 @@ def _set_callbacks_and_metrics(self, callbacks: Any, metrics: Any):
         self.callback_container.set_trainer(self)
 
     @staticmethod
-    def _set_device_and_num_workers(**kwargs):
+    def _set_device_and_num_workers(**kwargs) -> Tuple[str, int]:
         default_num_workers = (
             0
             if sys.platform == "darwin" and sys.version_info.minor > 7
diff --git a/pytorch_widedeep/training/_base_trainer.py b/pytorch_widedeep/training/_base_trainer.py
index 74a57202..45cb2e9e 100644
--- a/pytorch_widedeep/training/_base_trainer.py
+++ b/pytorch_widedeep/training/_base_trainer.py
@@ -15,6 +15,7 @@
     Any,
     Dict,
     List,
+    Tuple,
     Union,
     Module,
     Optional,
@@ -31,6 +32,7 @@
     LRShedulerCallback,
 )
 from pytorch_widedeep.initializers import Initializer, MultipleInitializer
+from pytorch_widedeep.utils.general_utils import setup_device, to_device_model
 from pytorch_widedeep.training._trainer_utils import alias_to_loss
 from pytorch_widedeep.training._multiple_optimizer import MultipleOptimizer
 from pytorch_widedeep.training._multiple_transforms import MultipleTransforms
@@ -74,10 +76,9 @@ def __init__(
         self.verbose = verbose
         self.seed = seed
 
-        self.model = model
+        self.model = to_device_model(model, self.device)
         if self.model.is_tabnet:
             self.lambda_sparse = kwargs.get("lambda_sparse", 1e-3)
-        self.model.to(self.device)
         self.model.wd_device = self.device
 
         self.objective = objective
@@ -406,7 +407,7 @@ def _check_inputs(
             )
 
     @staticmethod
-    def _set_device_and_num_workers(**kwargs):
+    def _set_device_and_num_workers(**kwargs) -> Tuple[str, int]:
         # Important note for Mac users: Since python 3.8, the multiprocessing
         # library start method changed from 'fork' to 'spawn'. This affects the
         # data-loaders, which will not run in parallel.
@@ -415,9 +416,9 @@ def _set_device_and_num_workers(**kwargs):
             if sys.platform == "darwin" and sys.version_info.minor > 7
             else os.cpu_count()
         )
-        default_device = "cuda" if torch.cuda.is_available() else "cpu"
-        device = kwargs.get("device", default_device)
         num_workers = kwargs.get("num_workers", default_num_workers)
+        default_device = setup_device()
+        device = kwargs.get("device", default_device)
         return device, num_workers
 
     def __repr__(self) -> str:  # noqa: C901
diff --git a/pytorch_widedeep/training/_feature_importance.py b/pytorch_widedeep/training/_feature_importance.py
index 79a37f19..8bf9588b 100644
--- a/pytorch_widedeep/training/_feature_importance.py
+++ b/pytorch_widedeep/training/_feature_importance.py
@@ -24,7 +24,7 @@
     SelfAttentionMLP,
     ContextAttentionMLP,
 )
-from pytorch_widedeep.utils.general_utils import alias
+from pytorch_widedeep.utils.general_utils import alias, to_device
 from pytorch_widedeep.training._wd_dataset import WideDeepDataset
 from pytorch_widedeep.models.tabular.tabnet._utils import create_explain_matrix
 
@@ -136,7 +136,7 @@ def _sample_data(self, loader: DataLoader) -> Tensor:
         batches = []
         for i, (data, _) in enumerate(loader):
             if i < n_iterations:
-                batches.append(data["deeptabular"].to(self.device))
+                batches.append(to_device(data["deeptabular"], self.device))
             else:
                 break
 
@@ -307,7 +307,7 @@ def explain(
 
         m_explain_l = []
         for batch_nb, data in enumerate(loader):
-            X = data["deeptabular"].to(self.device)
+            X = to_device(data["deeptabular"], self.device)
             M_explain, masks = model_backbone.forward_masks(X)  # type: ignore[operator]
             m_explain_l.append(
                 csc_matrix.dot(M_explain.cpu().detach().numpy(), reducing_matrix)
@@ -363,7 +363,7 @@ def explain(
 
         batch_feat_imp: Any = []
         for _, data in enumerate(loader):
-            X = data["deeptabular"].to(self.device)
+            X = to_device(data["deeptabular"], self.device)
             _ = model.deeptabular(X)
 
             feat_imp, _ = self._feature_importance(model)
diff --git a/pytorch_widedeep/training/_finetune.py b/pytorch_widedeep/training/_finetune.py
index 90579179..5567fc42 100644
--- a/pytorch_widedeep/training/_finetune.py
+++ b/pytorch_widedeep/training/_finetune.py
@@ -16,10 +16,9 @@
     DataLoader,
     LRScheduler,
 )
+from pytorch_widedeep.utils.general_utils import to_device, setup_device
 from pytorch_widedeep.models._base_wd_model_component import BaseWDModelComponent
 
-use_cuda = torch.cuda.is_available()
-
 WDModel = Union[nn.Module, BaseWDModelComponent]
 
 
@@ -67,6 +66,8 @@ def __init__(
         self.method = method
         self.verbose = verbose
 
+        self.device = setup_device()
+
     def finetune_all(
         self,
         model: Union[WDModel, nn.ModuleList],
@@ -432,20 +433,16 @@ def _train(  # noqa: C901
                         data, target = packed_data
 
                     if idx is not None:
-                        X = (
-                            data[model_name][idx].cuda()
-                            if use_cuda
-                            else data[model_name][idx]
-                        )
+                        X = to_device(data[model_name][idx], self.device)
                     else:
-                        X = data[model_name].cuda() if use_cuda else data[model_name]
+                        X = to_device(data[model_name], self.device)
 
                     y = (
                         target.view(-1, 1).float()
                         if self.method not in ["multiclass", "qregression"]
                         else target
                     )
-                    y = y.cuda() if use_cuda else y
+                    y = to_device(y, self.device)
 
                     optimizer.zero_grad()
                     y_pred = model(X)
diff --git a/pytorch_widedeep/training/bayesian_trainer.py b/pytorch_widedeep/training/bayesian_trainer.py
index 65020c23..3c314934 100644
--- a/pytorch_widedeep/training/bayesian_trainer.py
+++ b/pytorch_widedeep/training/bayesian_trainer.py
@@ -20,7 +20,7 @@
     LRScheduler,
 )
 from pytorch_widedeep.callbacks import Callback
-from pytorch_widedeep.utils.general_utils import alias
+from pytorch_widedeep.utils.general_utils import alias, to_device
 from pytorch_widedeep.training._trainer_utils import (
     save_epoch_logs,
     print_loss_and_metric,
@@ -397,12 +397,9 @@ def _train_step(
     ):
         self.model.train()
 
-        try:
-            X = X_tab.to(self.device)
-        except TypeError:
-            X = X_tab.to(self.device, dtype=torch.float32)
+        X = to_device(X_tab, self.device)
         y = target.view(-1, 1).float() if self.objective != "multiclass" else target
-        y = y.to(self.device)
+        y = to_device(y, self.device)
 
         self.optimizer.zero_grad()
         y_pred, loss = self.model.sample_elbo(X, y, self.loss_fn, n_samples, n_batches)  # type: ignore[arg-type]
@@ -428,12 +425,9 @@ def _eval_step(
     ):
         self.model.eval()
         with torch.no_grad():
-            try:
-                X = X_tab.to(self.device)
-            except TypeError:
-                X = X_tab.to(self.device, dtype=torch.float32)
+            X = to_device(X_tab, self.device)
             y = target.view(-1, 1).float() if self.objective != "multiclass" else target
-            y = y.to(self.device)
+            y = to_device(y, self.device)
 
             y_pred, loss = self.model.sample_elbo(
                 X,  # type: ignore[arg-type]
diff --git a/pytorch_widedeep/training/trainer.py b/pytorch_widedeep/training/trainer.py
index 873a11b4..8ca7d459 100644
--- a/pytorch_widedeep/training/trainer.py
+++ b/pytorch_widedeep/training/trainer.py
@@ -26,7 +26,7 @@
 from pytorch_widedeep.callbacks import Callback
 from pytorch_widedeep.initializers import Initializer
 from pytorch_widedeep.training._finetune import FineTune
-from pytorch_widedeep.utils.general_utils import alias
+from pytorch_widedeep.utils.general_utils import alias, to_device
 from pytorch_widedeep.training._wd_dataset import WideDeepDataset
 from pytorch_widedeep.training._base_trainer import BaseTrainer
 from pytorch_widedeep.training._trainer_utils import (
@@ -914,15 +914,15 @@ def _train_step(
         X: Dict[str, Union[Tensor, List[Tensor]]] = {}
         for k, v in data.items():
             if isinstance(v, list):
-                X[k] = [i.to(self.device) for i in v]
+                X[k] = [to_device(i, self.device) for i in v]
             else:
-                X[k] = v.to(self.device)
+                X[k] = to_device(v, self.device)
         y = (
             target.view(-1, 1).float()
             if self.method not in ["multiclass", "qregression", "multitarget"]
             else target
         )
-        y = y.to(self.device)
+        y = to_device(y, self.device)
 
         self.optimizer.zero_grad()
 
@@ -954,15 +954,15 @@ def _eval_step(
             X: Dict[str, Union[Tensor, List[Tensor]]] = {}
             for k, v in data.items():
                 if isinstance(v, list):
-                    X[k] = [i.to(self.device) for i in v]
+                    X[k] = [to_device(i, self.device) for i in v]
                 else:
-                    X[k] = v.to(self.device)
+                    X[k] = to_device(v, self.device)
             y = (
                 target.view(-1, 1).float()
                 if self.method not in ["multiclass", "qregression", "multitarget"]
                 else target
             )
-            y = y.to(self.device)
+            y = to_device(y, self.device)
 
             y_pred = self.model(X)
             if self.model.is_tabnet:
@@ -1061,9 +1061,9 @@ def _predict(  # type: ignore[override, return]  # noqa: C901
                             X: Dict[str, Union[Tensor, List[Tensor]]] = {}
                             for k, v in data.items():
                                 if isinstance(v, list):
-                                    X[k] = [i.to(self.device) for i in v]
+                                    X[k] = [to_device(i, self.device) for i in v]
                                 else:
-                                    X[k] = v.to(self.device)
+                                    X[k] = to_device(v, self.device)
                             preds = (
                                 self.model(X)
                                 if not self.model.is_tabnet
diff --git a/pytorch_widedeep/training/trainer_from_folder.py b/pytorch_widedeep/training/trainer_from_folder.py
index 0641f3d8..1ebc2629 100644
--- a/pytorch_widedeep/training/trainer_from_folder.py
+++ b/pytorch_widedeep/training/trainer_from_folder.py
@@ -23,7 +23,7 @@
 from pytorch_widedeep.callbacks import Callback
 from pytorch_widedeep.initializers import Initializer
 from pytorch_widedeep.training._finetune import FineTune
-from pytorch_widedeep.utils.general_utils import alias
+from pytorch_widedeep.utils.general_utils import alias, to_device
 from pytorch_widedeep.training._wd_dataset import WideDeepDataset
 from pytorch_widedeep.training._base_trainer import BaseTrainer
 from pytorch_widedeep.training._trainer_utils import (
@@ -525,15 +525,15 @@ def _train_step(
         X: Dict[str, Union[Tensor, List[Tensor]]] = {}
         for k, v in data.items():
             if isinstance(v, list):
-                X[k] = [i.to(self.device) for i in v]
+                X[k] = [to_device(i, self.device) for i in v]
             else:
-                X[k] = v.to(self.device)
+                X[k] = to_device(v, self.device)
         y = (
             target.view(-1, 1).float()
             if self.method not in ["multiclass", "qregression"]
             else target
         )
-        y = y.to(self.device)
+        y = to_device(y, self.device)
 
         self.optimizer.zero_grad()
 
@@ -560,15 +560,15 @@ def _eval_step(self, data: Dict[str, Tensor], target: Tensor, batch_idx: int):
             X: Dict[str, Union[Tensor, List[Tensor]]] = {}
             for k, v in data.items():
                 if isinstance(v, list):
-                    X[k] = [i.to(self.device) for i in v]
+                    X[k] = [to_device(i, self.device) for i in v]
                 else:
-                    X[k] = v.to(self.device)
+                    X[k] = to_device(v, self.device)
             y = (
                 target.view(-1, 1).float()
                 if self.method not in ["multiclass", "qregression"]
                 else target
             )
-            y = y.to(self.device)
+            y = to_device(y, self.device)
 
             y_pred = self.model(X)
             if self.model.is_tabnet:  # pragma: no cover
@@ -670,9 +670,9 @@ def _predict(  # noqa: C901
                             X: Dict[str, Union[Tensor, List[Tensor]]] = {}
                             for k, v in data.items():
                                 if isinstance(v, list):
-                                    X[k] = [i.to(self.device) for i in v]
+                                    X[k] = [to_device(i, self.device) for i in v]
                                 else:
-                                    X[k] = v.to(self.device)
+                                    X[k] = to_device(v, self.device)
                             preds = (
                                 self.model(X)
                                 if not self.model.is_tabnet
diff --git a/pytorch_widedeep/utils/general_utils.py b/pytorch_widedeep/utils/general_utils.py
index cac80e84..781439ef 100644
--- a/pytorch_widedeep/utils/general_utils.py
+++ b/pytorch_widedeep/utils/general_utils.py
@@ -1,15 +1,54 @@
 from functools import wraps
 
 import torch
+from torch import Tensor
 
 
-def setup_device():
+def setup_device() -> str:
     if torch.cuda.is_available():
-        return torch.device("cuda")
+        return "cuda"
     elif torch.backends.mps.is_available():
-        return torch.device("mps")
+        return "mps"
     else:
-        return torch.device("cpu")
+        return "cpu"
+
+
+def to_device(X: Tensor, device: str) -> Tensor:
+    # Adjustmet in case the backend is mps which does not support float64
+    if device == "mps" and X.dtype == torch.float64:
+        X = X.float()
+    return X.to(device)
+
+
+def to_device_model(model, device: str):  # noqa: C901
+    # insistent transformation since it some cases overall approaches such as
+    # model.to('mps') do not work
+
+    if device in ["cpu", "cuda"]:
+        return model.to(device)
+
+    if device == "mps":
+
+        try:
+            return model.to(device)
+        except (RuntimeError, TypeError):
+
+            def convert(t):
+                if isinstance(t, torch.Tensor):
+                    return t.float().to(device)
+                return t
+
+            model.apply(lambda module: module._apply(convert))
+
+            for param in model.parameters():
+                if param.device.type != "mps":
+                    param.data = param.data.float().to(device)
+
+            for buffer_name, buffer in model.named_buffers():
+                if buffer.device.type != "mps":
+                    model._buffers[buffer_name] = buffer.float().to(device)
+
+    return model
 
 
 def alias(original_name: str, alternative_names: list):
diff --git a/tests/test_finetune/test_finetuning_routines.py b/tests/test_finetune/test_finetuning_routines.py
index b64eacc5..7e03f8c8 100644
--- a/tests/test_finetune/test_finetuning_routines.py
+++ b/tests/test_finetune/test_finetuning_routines.py
@@ -1,7 +1,6 @@
 import string
 
 import numpy as np
-import torch
 import pytest
 import torch.nn.functional as F
 from torch import nn
@@ -10,10 +9,11 @@
 from pytorch_widedeep.models import Wide, TabMlp
 from pytorch_widedeep.metrics import Accuracy, MultipleMetrics
 from pytorch_widedeep.training._finetune import FineTune
+from pytorch_widedeep.utils.general_utils import setup_device, to_device_model
 from pytorch_widedeep.models.image._layers import conv_layer
 from pytorch_widedeep.training._wd_dataset import WideDeepDataset
 
-use_cuda = torch.cuda.is_available()
+device = setup_device()
 
 
 # Define a series of simple models to quickly test the FineTune class
@@ -87,8 +87,7 @@ def loss_fn(y_pred, y_true):
 
 # wide/linear
 wide = Wide(np.unique(X_wide).size, 1)
-if use_cuda:
-    wide.cuda()
+wide = to_device_model(wide, device)
 
 # tabular
 tab_mlp = TabMlp(
@@ -99,18 +98,15 @@ def loss_fn(y_pred, y_true):
     mlp_dropout=0.2,
 )
 tab_mlp = nn.Sequential(tab_mlp, nn.Linear(8, 1))  # type: ignore[assignment]
-if use_cuda:
-    tab_mlp.cuda()
+tab_mlp = to_device_model(tab_mlp, device)
 
 # text
 deeptext = TextModeTestClass()
-if use_cuda:
-    deeptext.cuda()
+deeptext = to_device_model(deeptext, device)
 
 # image
 deepimage = ImageModeTestClass()
-if use_cuda:
-    deepimage.cuda()
+ddeimage = to_device_model(deepimage, device)
 
 # Define the loader
 mmset = WideDeepDataset(X_wide, X_tab, X_text, X_image, target)
diff --git a/tests/test_hf_integration/test_models.py b/tests/test_hf_integration/test_models.py
index 847731d8..0c86039e 100644
--- a/tests/test_hf_integration/test_models.py
+++ b/tests/test_hf_integration/test_models.py
@@ -161,6 +161,7 @@ def test_full_training_process(model_name):
         model,
         objective="binary",
         verbose=0,
+        # device="cpu",
     )
 
     trainer.fit(
diff --git a/tests/test_tab2vec/test_t2v.py b/tests/test_tab2vec/test_t2v.py
index 5a92b9e7..cc9b2746 100644
--- a/tests/test_tab2vec/test_t2v.py
+++ b/tests/test_tab2vec/test_t2v.py
@@ -85,6 +85,7 @@ def test_non_transformer_models(deeptabular, return_dataframe):
     # Let's assume the model is trained
     t2v = Tab2Vec(
         tab_preprocessor=tab_preprocessor,
+        device=device,
         model=model,
         return_dataframe=return_dataframe,
     )
@@ -163,6 +164,7 @@ def test_tab_transformer_models(
     t2v = Tab2Vec(
         tab_preprocessor=tab_preprocessor,
         model=model,
+        device=device,
     )
     x_vec = t2v.transform(df_t2v)
 
@@ -229,6 +231,7 @@ def test_attentive_mlp(
     t2v = Tab2Vec(
         tab_preprocessor=tab_preprocessor,
         model=model,
+        device=device,
     )
     x_vec = t2v.transform(df_t2v)
 
@@ -305,6 +308,7 @@ def test_transformer_family_models(
         tab_preprocessor=tab_preprocessor,
         model=model,
         return_dataframe=return_dataframe,
+        device=device,
     )
     x_vec = t2v.transform(df_t2v)
 

From 73894d4a45de727c47005a1b0a7a9252ece8a9b1 Mon Sep 17 00:00:00 2001
From: Javier <jrzaurin@gmail.com>
Date: Fri, 11 Oct 2024 18:25:58 +0100
Subject: [PATCH 03/24] The 1st version of the DINPreprocessor seems to work.
 Need to test it

---
 pytorch_widedeep/models/rec/din.py            |   2 +-
 pytorch_widedeep/preprocessing/__init__.py    |   1 +
 .../preprocessing/din_preprocessor.py         | 451 ++++++++++++++++++
 3 files changed, 453 insertions(+), 1 deletion(-)
 create mode 100644 pytorch_widedeep/preprocessing/din_preprocessor.py

diff --git a/pytorch_widedeep/models/rec/din.py b/pytorch_widedeep/models/rec/din.py
index 1eba4f97..293f249e 100644
--- a/pytorch_widedeep/models/rec/din.py
+++ b/pytorch_widedeep/models/rec/din.py
@@ -242,8 +242,8 @@ def __init__(
         self,
         *,
         column_idx: Dict[str, int],
-        target_item_col: str,
         user_behavior_confiq: Tuple[List[str], int, int],
+        target_item_col: str = "target_item",
         action_seq_config: Optional[Tuple[List[str], int]] = None,
         other_seq_cols_confiq: Optional[List[Tuple[List[str], int, int]]] = None,
         attention_unit_activation: Literal["prelu", "dice"] = "prelu",
diff --git a/pytorch_widedeep/preprocessing/__init__.py b/pytorch_widedeep/preprocessing/__init__.py
index 59f8b787..b4c5bcee 100644
--- a/pytorch_widedeep/preprocessing/__init__.py
+++ b/pytorch_widedeep/preprocessing/__init__.py
@@ -5,6 +5,7 @@
 from pytorch_widedeep.preprocessing.tab_preprocessor import (
     TabPreprocessor,
     ChunkTabPreprocessor,
+    embed_sz_rule,
 )
 from pytorch_widedeep.preprocessing.text_preprocessor import (
     TextPreprocessor,
diff --git a/pytorch_widedeep/preprocessing/din_preprocessor.py b/pytorch_widedeep/preprocessing/din_preprocessor.py
new file mode 100644
index 00000000..04655a82
--- /dev/null
+++ b/pytorch_widedeep/preprocessing/din_preprocessor.py
@@ -0,0 +1,451 @@
+import re
+from typing import Dict, List, Tuple, Union, Literal, Optional
+
+import numpy as np
+import pandas as pd
+
+from pytorch_widedeep import Trainer
+from pytorch_widedeep.models import WideDeep
+from pytorch_widedeep.metrics import Accuracy
+from pytorch_widedeep.datasets import load_movielens100k
+from pytorch_widedeep.preprocessing import TabPreprocessor, embed_sz_rule
+from pytorch_widedeep.models.rec.din import DeepInterestNetwork
+from pytorch_widedeep.utils.text_utils import pad_sequences
+from pytorch_widedeep.utils.deeptabular_utils import LabelEncoder
+from pytorch_widedeep.preprocessing.base_preprocessor import (
+    BasePreprocessor,
+    check_is_fitted,
+)
+
+
+class DINPreprocessor(BasePreprocessor):
+    def __init__(
+        self,
+        *,
+        user_id_col: str,
+        target_col: str,
+        item_embed_col: Union[str, Tuple[str, int]],
+        max_seq_length: int,
+        action_col: Optional[str] = None,
+        other_seq_embed_cols: Optional[List[str] | List[Tuple[str, int]]] = None,
+        cat_embed_cols: Optional[Union[List[str], List[Tuple[str, int]]]] = None,
+        continuous_cols: Optional[List[str]] = None,
+        quantization_setup: Optional[
+            Union[int, Dict[str, Union[int, List[float]]]]
+        ] = None,
+        cols_to_scale: Optional[Union[List[str], str]] = None,
+        auto_embed_dim: bool = True,
+        embedding_rule: Literal["google", "fastai_old", "fastai_new"] = "fastai_new",
+        default_embed_dim: int = 16,
+        verbose: int = 1,
+        scale: bool = False,
+        already_standard: Optional[List[str]] = None,
+        **kwargs,
+    ):
+
+        self.user_id_col = user_id_col
+        self.item_embed_col = item_embed_col
+        self.max_seq_length = max_seq_length
+        self.target_col = target_col if target_col is not None else "target"
+        self.action_col = action_col
+        self.other_seq_embed_cols = other_seq_embed_cols
+        self.cat_embed_cols = cat_embed_cols
+        self.continuous_cols = continuous_cols
+        self.quantization_setup = quantization_setup
+        self.cols_to_scale = cols_to_scale
+        self.auto_embed_dim = auto_embed_dim
+        self.embedding_rule = embedding_rule
+        self.default_embed_dim = default_embed_dim
+        self.verbose = verbose
+        self.scale = scale
+        self.already_standard = already_standard
+        self.kwargs = kwargs
+
+        self.has_standard_tab_data = False
+        if self.cat_embed_cols or self.continuous_cols:
+            self.has_standard_tab_data = True
+            self.tab_preprocessor = TabPreprocessor(
+                cat_embed_cols=self.cat_embed_cols,
+                continuous_cols=self.continuous_cols,
+                quantization_setup=self.quantization_setup,
+                cols_to_scale=self.cols_to_scale,
+                auto_embed_dim=self.auto_embed_dim,
+                embedding_rule=self.embedding_rule,
+                default_embed_dim=self.default_embed_dim,
+                verbose=self.verbose,
+                scale=self.scale,
+                already_standard=self.already_standard,
+                **self.kwargs,
+            )
+
+        self.is_fitted = False
+
+    def fit(self, df: pd.DataFrame):
+
+        if self.has_standard_tab_data:
+            self.tab_preprocessor.fit(df)
+            self.din_columns_idx = {
+                col: i for i, col in enumerate(self.tab_preprocessor.column_idx.keys())
+            }
+        else:
+            self.din_columns_idx = {}
+
+        self.item_le = LabelEncoder(
+            columns_to_encode=[
+                (
+                    self.item_embed_col[0]
+                    if isinstance(self.item_embed_col, tuple)
+                    else self.item_embed_col
+                )
+            ]
+        )
+        self.item_le.fit(df)
+        self.n_items = max(self.item_le.encoding_dict[self.item_embed_col[0]].values())
+        user_behaviour_embed_size = (
+            self.item_embed_col[1]
+            if isinstance(self.item_embed_col, tuple)
+            else embed_sz_rule(self.n_items)
+        )
+        self.user_behaviour_config: Tuple[List[str], int, int] = (
+            [f"item_{i+1}" for i in range(self.max_seq_length)],
+            self.n_items,
+            user_behaviour_embed_size,
+        )
+
+        _current_len = len(self.din_columns_idx)
+        self.din_columns_idx.update(
+            {f"item_{i+1}": i + _current_len for i in range(self.max_seq_length)}
+        )
+        self.din_columns_idx.update(
+            {
+                "target_item": len(self.din_columns_idx),
+            }
+        )
+
+        if self.action_col is not None:
+            self.action_le = LabelEncoder(columns_to_encode=[self.action_col])
+            self.action_le.fit(df)
+
+            self.n_actions = len(self.action_le.encoding_dict[self.action_col])
+            self.action_seq_config: Tuple[List[str], int] = (
+                [f"action_{i+1}" for i in range(self.max_seq_length)],
+                self.n_actions,
+            )
+
+            _current_len = len(self.din_columns_idx)
+            self.din_columns_idx.update(
+                {f"action_{i+1}": i + _current_len for i in range(self.max_seq_length)}
+            )
+
+        if self.other_seq_embed_cols is not None:
+            self.other_seq_le = LabelEncoder(
+                columns_to_encode=[
+                    col[0] if isinstance(col, tuple) else col
+                    for col in self.other_seq_embed_cols
+                ]
+            )
+            self.other_seq_le.fit(df)
+
+            other_seq_cols = [
+                col[0] if isinstance(col, tuple) else col
+                for col in self.other_seq_embed_cols
+            ]
+            self.n_other_seq_cols: Dict[str, int] = {
+                col: max(self.other_seq_le.encoding_dict[col].values())
+                for col in other_seq_cols
+            }
+
+            if isinstance(self.other_seq_embed_cols[0], tuple):
+                other_seq_embed_sizes: Dict[str, int] = {
+                    tp[0]: tp[1] for tp in self.other_seq_embed_cols  # type: ignore[misc]
+                }
+            else:
+                other_seq_embed_sizes = {
+                    col: embed_sz_rule(self.n_other_seq_cols[col])
+                    for col in other_seq_cols
+                }
+
+            self.other_seq_config: List[Tuple[List[str], int, int]] = [
+                (
+                    [f"{col}_{i+1}" for i in range(self.max_seq_length)],
+                    self.n_other_seq_cols[col],
+                    other_seq_embed_sizes[col],
+                )
+                for col in other_seq_cols
+            ]
+
+            _current_len = len(self.din_columns_idx)
+            for col in other_seq_cols:
+                self.din_columns_idx.update(
+                    {
+                        f"{col}_{i+1}": i + _current_len
+                        for i in range(self.max_seq_length)
+                    }
+                )
+                self.din_columns_idx.update(
+                    {f"target_{col}": len(self.din_columns_idx)}
+                )
+                _current_len = len(self.din_columns_idx)
+
+        self.is_fitted = True
+
+        return self
+
+    def transform(self, df: pd.DataFrame) -> Tuple[np.ndarray, np.ndarray]:
+
+        _df = self._pre_transform(df)
+
+        df_w_sequences = self._build_sequences(_df)
+
+        user_behaviour_seq = df_w_sequences["items_sequence"].tolist()
+        X_user_behaviour = np.vstack(
+            [
+                pad_sequences(seq, self.max_seq_length, pad_first=False, pad_idx=0)
+                for seq in user_behaviour_seq
+            ]
+        )
+
+        X_target_item = np.array(df_w_sequences["target_item"].tolist()).reshape(-1, 1)
+
+        X_all = np.concatenate([X_user_behaviour, X_target_item], axis=1)
+
+        if self.has_standard_tab_data:
+            X_tab = self.tab_preprocessor.transform(df_w_sequences)
+            X_all = np.concatenate([X_tab, X_all], axis=1)
+
+        if self.action_col is not None:
+            action_seq = df_w_sequences["actions_sequence"].tolist()
+            X_actions = np.vstack(
+                [
+                    pad_sequences(seq, self.max_seq_length, pad_first=False, pad_idx=0)
+                    for seq in action_seq
+                ]
+            )
+            X_all = np.concatenate([X_all, X_actions], axis=1)
+
+        if self.other_seq_embed_cols is not None:
+            other_seq_cols = [
+                col[0] if isinstance(col, tuple) else col
+                for col in self.other_seq_embed_cols
+            ]
+            other_seq = {
+                col: df_w_sequences[f"{col}_sequence"].tolist()
+                for col in other_seq_cols
+            }
+            other_seq_target = {
+                col: df_w_sequences[f"target_{col}"].tolist() for col in other_seq_cols
+            }
+            X_other_seq_arrays: List[np.ndarray] = []
+            # to obsessively make sure that the order of the columns is the
+            # same
+            for col in other_seq_cols:
+                X_other_seq_arrays.append(
+                    np.vstack(
+                        [
+                            pad_sequences(
+                                s, self.max_seq_length, pad_first=False, pad_idx=0
+                            )
+                            for s in other_seq[col]
+                        ]
+                    )
+                )
+                X_other_seq_arrays.append(
+                    np.array(other_seq_target[col]).reshape(-1, 1)
+                )
+
+            X_all = np.concatenate([X_all] + X_other_seq_arrays, axis=1)
+
+        assert len(self.din_columns_idx) == X_all.shape[1], (
+            f"Something went wrong. The number of columns in the final array "
+            f"({X_all.shape[1]}) is different from the number of columns in "
+            f"self.din_columns_idx ({len(self.din_columns_idx)})"
+        )
+
+        return X_all, np.array(df_w_sequences["target"].tolist())
+
+    def fit_transform(self, df: pd.DataFrame) -> Tuple[np.ndarray, np.ndarray]:
+        self.fit(df)
+        return self.transform(df)
+
+    def _pre_transform(self, df: pd.DataFrame) -> pd.DataFrame:
+
+        check_is_fitted(self, attributes=["din_columns_idx"])
+
+        df = self.item_le.transform(df)
+
+        if self.action_col is not None:
+            df = self.action_le.transform(df)
+
+        if self.other_seq_embed_cols is not None:
+            df = self.other_seq_le.transform(df)
+
+        return df
+
+    def _build_sequences(self, df: pd.DataFrame) -> pd.DataFrame:
+
+        res_df = (
+            df.groupby(self.user_id_col)
+            .apply(self._group_sequences)
+            .reset_index(drop=True)
+        )
+
+        return res_df
+
+    def _group_sequences(self, group: pd.DataFrame) -> pd.DataFrame:
+
+        item_col = (
+            self.item_embed_col[0]
+            if isinstance(self.item_embed_col, tuple)
+            else self.item_embed_col
+        )
+        items = group[item_col].tolist()
+
+        targets = group[self.target_col].tolist()
+        drop_cols = [item_col, self.target_col]
+
+        sequences: List[Dict[str, str | List[int]]] = []
+        for i in range(len(items) - self.max_seq_length):
+            item_sequences = items[i : i + self.max_seq_length]
+            target_item = items[i + self.max_seq_length]
+            target = targets[i + self.max_seq_length]
+
+            sequence = {
+                "user_id": group.name,
+                "items_sequence": item_sequences,
+                "target_item": target_item,
+            }
+
+            if self.action_col is not None:
+                if self.action_col != self.target_col:
+                    actions = group[self.action_col].tolist()
+                    action_sequences = actions[i : i + self.max_seq_length]
+                    drop_cols.append(self.action_col)
+                else:
+                    target -= (
+                        1  # the 'transform' method adds 1 as it saves 0 for padding
+                    )
+                    action_sequences = targets[i : i + self.max_seq_length]
+
+                sequence["target"] = target
+                sequence["actions_sequence"] = action_sequences
+
+            if self.other_seq_embed_cols is not None:
+                other_seq_cols = [
+                    col[0] if isinstance(col, tuple) else col
+                    for col in self.other_seq_embed_cols
+                ]
+                drop_cols += other_seq_cols
+                other_seqs: Dict[str, List[int]] = {
+                    col: group[col].tolist() for col in other_seq_cols
+                }
+                other_seqs_sequences = {
+                    col: other_seqs[col][i : i + self.max_seq_length]
+                    for col in other_seq_cols
+                }
+
+                for col in other_seq_cols:
+                    sequence[f"{col}_sequence"] = other_seqs_sequences[col]
+                    sequence[f"target_{col}"] = other_seqs[col][i + self.max_seq_length]
+
+            sequences.append(sequence)
+
+        seq_df = pd.DataFrame(sequences)
+
+        non_seq_cols = group.drop_duplicates(["user_id"]).drop(drop_cols, axis=1)
+
+        return pd.merge(seq_df, non_seq_cols, on="user_id")
+
+    def inverse_transform(self, X: np.ndarray) -> pd.DataFrame:
+        # trasform mutates the df and is complex to revert the transformation
+        raise NotImplementedError(
+            "inverse_transform is not implemented for this preprocessor"
+        )
+
+
+if __name__ == "__main__":
+
+    def clean_genre_list(genre_list):
+        return "_".join(
+            sorted([re.sub(r"[^a-z0-9]", "", genre.lower()) for genre in genre_list])
+        )
+
+    data, users, items = load_movielens100k(as_frame=True)
+
+    list_of_genres = [
+        "unknown",
+        "Action",
+        "Adventure",
+        "Animation",
+        "Children's",
+        "Comedy",
+        "Crime",
+        "Documentary",
+        "Drama",
+        "Fantasy",
+        "Film-Noir",
+        "Horror",
+        "Musical",
+        "Mystery",
+        "Romance",
+        "Sci-Fi",
+        "Thriller",
+        "War",
+        "Western",
+    ]
+
+    assert (
+        isinstance(items, pd.DataFrame)
+        and isinstance(data, pd.DataFrame)
+        and isinstance(users, pd.DataFrame)
+    )
+    items["genre_list"] = items[list_of_genres].apply(
+        lambda x: [genre for genre in list_of_genres if x[genre] == 1], axis=1
+    )
+
+    items["genre_list"] = items["genre_list"].apply(clean_genre_list)
+
+    df = pd.merge(data, items[["movie_id", "genre_list"]], on="movie_id")
+    df = pd.merge(
+        df,
+        users[["user_id", "age", "gender", "occupation"]],
+        on="user_id",
+    )
+
+    df["rating"] = df["rating"].apply(lambda x: 1 if x >= 4 else 0)
+
+    df = df.sort_values(by=["timestamp"]).reset_index(drop=True)
+    df = df.drop("timestamp", axis=1)
+
+    din_preprocessor = DINPreprocessor(
+        user_id_col="user_id",
+        target_col="rating",
+        item_embed_col=("movie_id", 16),
+        max_seq_length=5,
+        action_col="rating",
+        other_seq_embed_cols=[("genre_list", 16)],
+        cat_embed_cols=["user_id", "age", "gender", "occupation"],
+    )
+
+    X, y = din_preprocessor.fit_transform(df)
+
+    din = DeepInterestNetwork(
+        column_idx=din_preprocessor.din_columns_idx,
+        user_behavior_confiq=din_preprocessor.user_behaviour_config,
+        action_seq_config=din_preprocessor.action_seq_config,
+        other_seq_cols_confiq=din_preprocessor.other_seq_config,
+        cat_embed_input=din_preprocessor.tab_preprocessor.cat_embed_input,  # type: ignore[attr-defined]
+        mlp_hidden_dims=[128, 64],
+    )
+
+    # And from here on, everything is standard
+    model = WideDeep(deeptabular=din)
+
+    trainer = Trainer(model=model, objective="binary", metrics=[Accuracy()])
+
+    # in the real world you would have to split the data into train, val and test
+    trainer.fit(
+        X_tab=X,
+        target=y,
+        n_epochs=5,
+        batch_size=512,
+    )

From 45fce4d207ecb70b339d4b9b4d04be8504af92b8 Mon Sep 17 00:00:00 2001
From: Javier <jrzaurin@gmail.com>
Date: Mon, 14 Oct 2024 09:55:33 +0100
Subject: [PATCH 04/24] first test added for the din preprocessor

---
 pytorch_widedeep/preprocessing/__init__.py    |   1 +
 .../preprocessing/din_preprocessor.py         | 768 ++++++++++--------
 .../interactions_data.csv                     | 113 +++
 .../generate_interactions_data.py             | 194 +++++
 .../test_data_utils/test_din_preprocessor.py  |  87 ++
 5 files changed, 842 insertions(+), 321 deletions(-)
 create mode 100644 tests/test_data_utils/data_for_rec_preprocessor/interactions_data.csv
 create mode 100644 tests/test_data_utils/generate_interactions_data.py
 create mode 100644 tests/test_data_utils/test_din_preprocessor.py

diff --git a/pytorch_widedeep/preprocessing/__init__.py b/pytorch_widedeep/preprocessing/__init__.py
index b4c5bcee..f52a5e73 100644
--- a/pytorch_widedeep/preprocessing/__init__.py
+++ b/pytorch_widedeep/preprocessing/__init__.py
@@ -2,6 +2,7 @@
     HFPreprocessor,
     ChunkHFPreprocessor,
 )
+from pytorch_widedeep.preprocessing.din_preprocessor import DINPreprocessor
 from pytorch_widedeep.preprocessing.tab_preprocessor import (
     TabPreprocessor,
     ChunkTabPreprocessor,
diff --git a/pytorch_widedeep/preprocessing/din_preprocessor.py b/pytorch_widedeep/preprocessing/din_preprocessor.py
index 04655a82..d28e6926 100644
--- a/pytorch_widedeep/preprocessing/din_preprocessor.py
+++ b/pytorch_widedeep/preprocessing/din_preprocessor.py
@@ -1,17 +1,22 @@
-import re
+# import re
+import warnings
 from typing import Dict, List, Tuple, Union, Literal, Optional
 
 import numpy as np
 import pandas as pd
 
-from pytorch_widedeep import Trainer
-from pytorch_widedeep.models import WideDeep
-from pytorch_widedeep.metrics import Accuracy
-from pytorch_widedeep.datasets import load_movielens100k
-from pytorch_widedeep.preprocessing import TabPreprocessor, embed_sz_rule
-from pytorch_widedeep.models.rec.din import DeepInterestNetwork
+# from pytorch_widedeep.models import WideDeep
+# from pytorch_widedeep.metrics import Accuracy
+# from pytorch_widedeep.datasets import load_movielens100k
+# from pytorch_widedeep.training import Trainer
+# from pytorch_widedeep.models.rec.din import DeepInterestNetwork
 from pytorch_widedeep.utils.text_utils import pad_sequences
+from pytorch_widedeep.utils.general_utils import alias
 from pytorch_widedeep.utils.deeptabular_utils import LabelEncoder
+from pytorch_widedeep.preprocessing.tab_preprocessor import (
+    TabPreprocessor,
+    embed_sz_rule,
+)
 from pytorch_widedeep.preprocessing.base_preprocessor import (
     BasePreprocessor,
     check_is_fitted,
@@ -19,15 +24,94 @@
 
 
 class DINPreprocessor(BasePreprocessor):
+    """
+    Preprocessor for Deep Interest Network (DIN) models.
+
+    This preprocessor handles the preparation of data for DIN models,
+    including sequence building, label encoding, and handling of various
+    types of input columns (categorical, continuous, and sequential).
+
+    Parameters:
+    -----------
+    user_id_col : str
+        Name of the column containing user IDs.
+    item_embed_col : Union[str, Tuple[str, int]]
+        Name of the column containing item IDs to be embedded, or a tuple of
+        (column_name, embedding_dim).
+    target_col : str
+        Name of the column containing the target variable.
+    max_seq_length : int
+        Maximum length of sequences to be created.
+    action_col : Optional[str], default=None
+        Name of the column containing user actions (if applicable).
+    other_seq_embed_cols : Optional[List[str] | List[Tuple[str, int]]], default=None
+        List of other columns to be treated as sequences.
+    cat_embed_cols : Optional[Union[List[str], List[Tuple[str, int]]]], default=None
+        List of categorical columns to be represented by embeddings.
+    continuous_cols: List, default = None
+        List with the name of the continuous cols.
+    quantization_setup: int or Dict[str, Union[int, List[float]]], default=None
+        Continuous columns can be turned into categorical via `pd.cut`.
+    cols_to_scale: List or str, default = None,
+        List with the names of the columns that will be standardized via
+        sklearn's `StandardScaler`.
+    auto_embed_dim: bool, default = True
+        Boolean indicating whether the embedding dimensions will be
+        automatically defined via rule of thumb.
+    embedding_rule: str, default = 'fastai_new'
+        Rule of thumb for embedding size.
+    default_embed_dim: int, default=16
+        Default dimension for the embeddings.
+    verbose : int, default=1
+        Verbosity level.
+    scale: bool, default = False
+        Boolean indicating whether or not to scale/standardize continuous cols.
+    already_standard: List, default = None
+        List with the name of the continuous cols that do not need to be
+        scaled/standardized.
+    **kwargs :
+        Additional keyword arguments to be passed to the TabPreprocessor.
+
+    Attributes:
+    -----------
+    is_fitted : bool
+        Whether the preprocessor has been fitted.
+    has_standard_tab_data : bool
+        Whether the data includes standard tabular data.
+    tab_preprocessor : TabPreprocessor
+        Preprocessor for standard tabular data.
+    din_columns_idx : Dict[str, int]
+        Dictionary mapping column names to their indices in the processed data.
+    item_le : LabelEncoder
+        Label encoder for item IDs.
+    n_items : int
+        Number of unique items.
+    user_behaviour_config : Tuple[List[str], int, int]
+        Configuration for user behavior sequences.
+    action_le : LabelEncoder
+        Label encoder for action column (if applicable).
+    n_actions : int
+        Number of unique actions (if applicable).
+    action_seq_config : Tuple[List[str], int]
+        Configuration for action sequences (if applicable).
+    other_seq_le : LabelEncoder
+        Label encoder for other sequence columns (if applicable).
+    n_other_seq_cols : Dict[str, int]
+        Number of unique values in each other sequence column.
+    other_seq_config : List[Tuple[List[str], int, int]]
+        Configuration for other sequence columns.
+    """
+
+    @alias("item_embed_col", ["item_id_col"])
     def __init__(
         self,
         *,
         user_id_col: str,
-        target_col: str,
         item_embed_col: Union[str, Tuple[str, int]],
+        target_col: str,
         max_seq_length: int,
         action_col: Optional[str] = None,
-        other_seq_embed_cols: Optional[List[str] | List[Tuple[str, int]]] = None,
+        other_seq_embed_cols: Optional[Union[List[str], List[Tuple[str, int]]]] = None,
         cat_embed_cols: Optional[Union[List[str], List[Tuple[str, int]]]] = None,
         continuous_cols: Optional[List[str]] = None,
         quantization_setup: Optional[
@@ -42,7 +126,6 @@ def __init__(
         already_standard: Optional[List[str]] = None,
         **kwargs,
     ):
-
         self.user_id_col = user_id_col
         self.item_embed_col = item_embed_col
         self.max_seq_length = max_seq_length
@@ -61,9 +144,8 @@ def __init__(
         self.already_standard = already_standard
         self.kwargs = kwargs
 
-        self.has_standard_tab_data = False
-        if self.cat_embed_cols or self.continuous_cols:
-            self.has_standard_tab_data = True
+        self.has_standard_tab_data = bool(self.cat_embed_cols or self.continuous_cols)
+        if self.has_standard_tab_data:
             self.tab_preprocessor = TabPreprocessor(
                 cat_embed_cols=self.cat_embed_cols,
                 continuous_cols=self.continuous_cols,
@@ -80,8 +162,7 @@ def __init__(
 
         self.is_fitted = False
 
-    def fit(self, df: pd.DataFrame):
-
+    def fit(self, df: pd.DataFrame) -> "DINPreprocessor":
         if self.has_standard_tab_data:
             self.tab_preprocessor.fit(df)
             self.din_columns_idx = {
@@ -90,362 +171,407 @@ def fit(self, df: pd.DataFrame):
         else:
             self.din_columns_idx = {}
 
-        self.item_le = LabelEncoder(
-            columns_to_encode=[
-                (
-                    self.item_embed_col[0]
-                    if isinstance(self.item_embed_col, tuple)
-                    else self.item_embed_col
-                )
-            ]
+        self._fit_label_encoders(df)
+        self.is_fitted = True
+        return self
+
+    def transform(self, df: pd.DataFrame) -> Tuple[np.ndarray, np.ndarray]:
+        _df = self._pre_transform(df)
+        df_w_sequences = self._build_sequences(_df)
+        X_all = self._concatenate_features(df_w_sequences)
+        return X_all, np.array(df_w_sequences["target"].tolist())
+
+    def fit_transform(self, df: pd.DataFrame) -> Tuple[np.ndarray, np.ndarray]:
+        self.fit(df)
+        return self.transform(df)
+
+    def inverse_transform(self, X: np.ndarray) -> pd.DataFrame:
+        raise NotImplementedError(
+            "inverse_transform is not implemented for this preprocessor"
         )
+
+    def _fit_label_encoders(self, df: pd.DataFrame) -> None:
+        self.item_le = LabelEncoder(columns_to_encode=[self._get_item_col()])
         self.item_le.fit(df)
-        self.n_items = max(self.item_le.encoding_dict[self.item_embed_col[0]].values())
-        user_behaviour_embed_size = (
-            self.item_embed_col[1]
-            if isinstance(self.item_embed_col, tuple)
-            else embed_sz_rule(self.n_items)
+        self.n_items = max(self.item_le.encoding_dict[self._get_item_col()].values())
+        user_behaviour_embed_size = self._get_embed_size(
+            self.item_embed_col, self.n_items
         )
-        self.user_behaviour_config: Tuple[List[str], int, int] = (
+        self.user_behaviour_config = (
             [f"item_{i+1}" for i in range(self.max_seq_length)],
             self.n_items,
             user_behaviour_embed_size,
         )
 
-        _current_len = len(self.din_columns_idx)
-        self.din_columns_idx.update(
-            {f"item_{i+1}": i + _current_len for i in range(self.max_seq_length)}
-        )
-        self.din_columns_idx.update(
-            {
-                "target_item": len(self.din_columns_idx),
-            }
+        self._update_din_columns_idx("item", self.max_seq_length)
+        self.din_columns_idx["target_item"] = len(self.din_columns_idx)
+
+        if self.action_col:
+            self._fit_action_label_encoder(df)
+        if self.other_seq_embed_cols:
+            self._fit_other_seq_label_encoders(df)
+
+    def _fit_action_label_encoder(self, df: pd.DataFrame) -> None:
+        self.action_le = LabelEncoder(columns_to_encode=[self.action_col])
+        self.action_le.fit(df)
+        self.n_actions = len(self.action_le.encoding_dict[self.action_col])
+        self.action_seq_config = (
+            [f"action_{i+1}" for i in range(self.max_seq_length)],
+            self.n_actions,
         )
+        self._update_din_columns_idx("action", self.max_seq_length)
+
+    def _other_seq_cols_float_warning(self, df: pd.DataFrame) -> None:
+        other_seq_cols = [
+            col[0] if isinstance(col, tuple) else col
+            for col in self.other_seq_embed_cols
+        ]
+        for col in other_seq_cols:
+            if df[col].dtype == float:
+                warnings.warn(
+                    f"{col} is a float column. It will be converted to integers. "
+                    "If this is not what you want, please convert it beforehand.",
+                    UserWarning,
+                )
 
-        if self.action_col is not None:
-            self.action_le = LabelEncoder(columns_to_encode=[self.action_col])
-            self.action_le.fit(df)
-
-            self.n_actions = len(self.action_le.encoding_dict[self.action_col])
-            self.action_seq_config: Tuple[List[str], int] = (
-                [f"action_{i+1}" for i in range(self.max_seq_length)],
-                self.n_actions,
-            )
-
-            _current_len = len(self.din_columns_idx)
-            self.din_columns_idx.update(
-                {f"action_{i+1}": i + _current_len for i in range(self.max_seq_length)}
-            )
-
-        if self.other_seq_embed_cols is not None:
-            self.other_seq_le = LabelEncoder(
-                columns_to_encode=[
-                    col[0] if isinstance(col, tuple) else col
-                    for col in self.other_seq_embed_cols
-                ]
-            )
-            self.other_seq_le.fit(df)
-
-            other_seq_cols = [
+    def _convert_other_seq_cols_to_int(self, df: pd.DataFrame) -> None:
+        other_seq_cols = [
+            col[0] if isinstance(col, tuple) else col
+            for col in self.other_seq_embed_cols
+        ]
+        for col in other_seq_cols:
+            if df[col].dtype == float:
+                df[col] = df[col].astype(int)
+
+    def _fit_other_seq_label_encoders(self, df: pd.DataFrame) -> None:
+        self._other_seq_cols_float_warning(df)
+        self._convert_other_seq_cols_to_int(df)
+        self.other_seq_le = LabelEncoder(
+            columns_to_encode=[
                 col[0] if isinstance(col, tuple) else col
                 for col in self.other_seq_embed_cols
             ]
-            self.n_other_seq_cols: Dict[str, int] = {
-                col: max(self.other_seq_le.encoding_dict[col].values())
-                for col in other_seq_cols
-            }
-
-            if isinstance(self.other_seq_embed_cols[0], tuple):
-                other_seq_embed_sizes: Dict[str, int] = {
-                    tp[0]: tp[1] for tp in self.other_seq_embed_cols  # type: ignore[misc]
-                }
-            else:
-                other_seq_embed_sizes = {
-                    col: embed_sz_rule(self.n_other_seq_cols[col])
-                    for col in other_seq_cols
-                }
-
-            self.other_seq_config: List[Tuple[List[str], int, int]] = [
-                (
-                    [f"{col}_{i+1}" for i in range(self.max_seq_length)],
-                    self.n_other_seq_cols[col],
-                    other_seq_embed_sizes[col],
-                )
-                for col in other_seq_cols
-            ]
-
-            _current_len = len(self.din_columns_idx)
-            for col in other_seq_cols:
-                self.din_columns_idx.update(
-                    {
-                        f"{col}_{i+1}": i + _current_len
-                        for i in range(self.max_seq_length)
-                    }
-                )
-                self.din_columns_idx.update(
-                    {f"target_{col}": len(self.din_columns_idx)}
-                )
-                _current_len = len(self.din_columns_idx)
-
-        self.is_fitted = True
-
-        return self
-
-    def transform(self, df: pd.DataFrame) -> Tuple[np.ndarray, np.ndarray]:
-
-        _df = self._pre_transform(df)
-
-        df_w_sequences = self._build_sequences(_df)
-
-        user_behaviour_seq = df_w_sequences["items_sequence"].tolist()
-        X_user_behaviour = np.vstack(
-            [
-                pad_sequences(seq, self.max_seq_length, pad_first=False, pad_idx=0)
-                for seq in user_behaviour_seq
-            ]
         )
-
-        X_target_item = np.array(df_w_sequences["target_item"].tolist()).reshape(-1, 1)
-
-        X_all = np.concatenate([X_user_behaviour, X_target_item], axis=1)
-
-        if self.has_standard_tab_data:
-            X_tab = self.tab_preprocessor.transform(df_w_sequences)
-            X_all = np.concatenate([X_tab, X_all], axis=1)
-
-        if self.action_col is not None:
-            action_seq = df_w_sequences["actions_sequence"].tolist()
-            X_actions = np.vstack(
-                [
-                    pad_sequences(seq, self.max_seq_length, pad_first=False, pad_idx=0)
-                    for seq in action_seq
-                ]
+        self.other_seq_le.fit(df)
+        other_seq_cols = [
+            col[0] if isinstance(col, tuple) else col
+            for col in self.other_seq_embed_cols
+        ]
+        self.n_other_seq_cols = {
+            col: max(self.other_seq_le.encoding_dict[col].values())
+            for col in other_seq_cols
+        }
+        other_seq_embed_sizes = {
+            col: self._get_embed_size(col, self.n_other_seq_cols[col])
+            for col in other_seq_cols
+        }
+        self.other_seq_config = [
+            (
+                self._get_seq_col_names(col),
+                self.n_other_seq_cols[col],
+                other_seq_embed_sizes[col],
             )
-            X_all = np.concatenate([X_all, X_actions], axis=1)
+            for col in other_seq_cols
+        ]
+        self._update_din_columns_idx_for_other_seq(other_seq_cols)
 
-        if self.other_seq_embed_cols is not None:
-            other_seq_cols = [
-                col[0] if isinstance(col, tuple) else col
-                for col in self.other_seq_embed_cols
-            ]
-            other_seq = {
-                col: df_w_sequences[f"{col}_sequence"].tolist()
-                for col in other_seq_cols
-            }
-            other_seq_target = {
-                col: df_w_sequences[f"target_{col}"].tolist() for col in other_seq_cols
-            }
-            X_other_seq_arrays: List[np.ndarray] = []
-            # to obsessively make sure that the order of the columns is the
-            # same
-            for col in other_seq_cols:
-                X_other_seq_arrays.append(
-                    np.vstack(
-                        [
-                            pad_sequences(
-                                s, self.max_seq_length, pad_first=False, pad_idx=0
-                            )
-                            for s in other_seq[col]
-                        ]
-                    )
-                )
-                X_other_seq_arrays.append(
-                    np.array(other_seq_target[col]).reshape(-1, 1)
-                )
+    def _get_item_col(self) -> str:
+        return (
+            self.item_embed_col[0]
+            if isinstance(self.item_embed_col, tuple)
+            else self.item_embed_col
+        )
 
-            X_all = np.concatenate([X_all] + X_other_seq_arrays, axis=1)
+    def _get_embed_size(self, col: Union[str, Tuple[str, int]], n_unique: int) -> int:
+        return col[1] if isinstance(col, tuple) else embed_sz_rule(n_unique)
 
-        assert len(self.din_columns_idx) == X_all.shape[1], (
-            f"Something went wrong. The number of columns in the final array "
-            f"({X_all.shape[1]}) is different from the number of columns in "
-            f"self.din_columns_idx ({len(self.din_columns_idx)})"
-        )
+    def _get_seq_col_names(self, col: str) -> List[str]:
+        return [f"{col}_{i+1}" for i in range(self.max_seq_length)]
 
-        return X_all, np.array(df_w_sequences["target"].tolist())
+    def _update_din_columns_idx(self, prefix: str, length: int) -> None:
+        current_len = len(self.din_columns_idx)
+        self.din_columns_idx.update(
+            {f"{prefix}_{i+1}": i + current_len for i in range(length)}
+        )
 
-    def fit_transform(self, df: pd.DataFrame) -> Tuple[np.ndarray, np.ndarray]:
-        self.fit(df)
-        return self.transform(df)
+    def _update_din_columns_idx_for_other_seq(self, other_seq_cols: List[str]) -> None:
+        current_len = len(self.din_columns_idx)
+        for col in other_seq_cols:
+            self.din_columns_idx.update(
+                {f"{col}_{i+1}": i + current_len for i in range(self.max_seq_length)}
+            )
+            self.din_columns_idx[f"target_{col}"] = len(self.din_columns_idx)
+            current_len = len(self.din_columns_idx)
 
     def _pre_transform(self, df: pd.DataFrame) -> pd.DataFrame:
-
         check_is_fitted(self, attributes=["din_columns_idx"])
-
         df = self.item_le.transform(df)
-
-        if self.action_col is not None:
+        if self.action_col:
             df = self.action_le.transform(df)
-
-        if self.other_seq_embed_cols is not None:
+        if self.other_seq_embed_cols:
             df = self.other_seq_le.transform(df)
-
         return df
 
     def _build_sequences(self, df: pd.DataFrame) -> pd.DataFrame:
-
-        res_df = (
+        # TO DO: do something to avoid warning here
+        return (
             df.groupby(self.user_id_col)
             .apply(self._group_sequences)
             .reset_index(drop=True)
         )
 
-        return res_df
-
     def _group_sequences(self, group: pd.DataFrame) -> pd.DataFrame:
-
-        item_col = (
-            self.item_embed_col[0]
-            if isinstance(self.item_embed_col, tuple)
-            else self.item_embed_col
-        )
+        item_col = self._get_item_col()
         items = group[item_col].tolist()
-
         targets = group[self.target_col].tolist()
         drop_cols = [item_col, self.target_col]
 
-        sequences: List[Dict[str, str | List[int]]] = []
-        for i in range(len(items) - self.max_seq_length):
-            item_sequences = items[i : i + self.max_seq_length]
-            target_item = items[i + self.max_seq_length]
-            target = targets[i + self.max_seq_length]
+        # sequences cannot be built with user, item pairs with only one
+        # interaction
+        if len(items) <= 1:
+            return pd.DataFrame()
 
-            sequence = {
-                "user_id": group.name,
-                "items_sequence": item_sequences,
-                "target_item": target_item,
-            }
+        sequences = [
+            self._create_sequence(group, items, targets, i, drop_cols)
+            for i in range(max(1, len(items) - self.max_seq_length))
+        ]
 
-            if self.action_col is not None:
-                if self.action_col != self.target_col:
-                    actions = group[self.action_col].tolist()
-                    action_sequences = actions[i : i + self.max_seq_length]
-                    drop_cols.append(self.action_col)
-                else:
-                    target -= (
-                        1  # the 'transform' method adds 1 as it saves 0 for padding
-                    )
-                    action_sequences = targets[i : i + self.max_seq_length]
-
-                sequence["target"] = target
-                sequence["actions_sequence"] = action_sequences
-
-            if self.other_seq_embed_cols is not None:
-                other_seq_cols = [
-                    col[0] if isinstance(col, tuple) else col
-                    for col in self.other_seq_embed_cols
-                ]
-                drop_cols += other_seq_cols
-                other_seqs: Dict[str, List[int]] = {
-                    col: group[col].tolist() for col in other_seq_cols
-                }
-                other_seqs_sequences = {
-                    col: other_seqs[col][i : i + self.max_seq_length]
-                    for col in other_seq_cols
-                }
+        seq_df = pd.DataFrame(sequences)
+        non_seq_cols = group.drop_duplicates([self.user_id_col]).drop(drop_cols, axis=1)
+        return pd.merge(seq_df, non_seq_cols, on=self.user_id_col)
 
-                for col in other_seq_cols:
-                    sequence[f"{col}_sequence"] = other_seqs_sequences[col]
-                    sequence[f"target_{col}"] = other_seqs[col][i + self.max_seq_length]
+    def _create_sequence(
+        self,
+        group: pd.DataFrame,
+        items: List[int],
+        targets: List[int],
+        i: int,
+        drop_cols: List[str],
+    ) -> Dict[str, Union[str, List[int]]]:
+        end_idx = min(i + self.max_seq_length, len(items) - 1)
+        item_sequences = items[i:end_idx]
+        target_item = items[end_idx]
+        target = targets[end_idx]
+
+        sequence = {
+            self.user_id_col: group.name,
+            "items_sequence": item_sequences,
+            "target_item": target_item,
+        }
+
+        if self.action_col:
+            sequence.update(
+                self._create_action_sequence(group, targets, i, target, drop_cols)
+            )
+        if self.other_seq_embed_cols:
+            sequence.update(self._create_other_seq_sequence(group, i, drop_cols))
 
-            sequences.append(sequence)
+        return sequence
 
-        seq_df = pd.DataFrame(sequences)
+    def _create_action_sequence(
+        self,
+        group: pd.DataFrame,
+        targets: List[int],
+        i: int,
+        target: int,
+        drop_cols: List[str],
+    ) -> Dict[str, Union[int, List[int]]]:
+        if self.action_col != self.target_col:
+            actions = group[self.action_col].tolist()
+            action_sequences = (
+                actions[i : i + self.max_seq_length]
+                if i + self.max_seq_length < len(actions)
+                else actions[i:-1]
+            )
+            drop_cols.append(self.action_col)
+        else:
+            target -= 1  # the 'transform' method adds 1 as it saves 0 for padding
+            action_sequences = (
+                targets[i : i + self.max_seq_length]
+                if i + self.max_seq_length < len(targets)
+                else targets[i:-1]
+            )
+
+        return {"target": target, "actions_sequence": action_sequences}
+
+    def _create_other_seq_sequence(
+        self, group: pd.DataFrame, i: int, drop_cols: List[str]
+    ) -> Dict[str, Union[int, List[int]]]:
+        other_seq_cols = [
+            col[0] if isinstance(col, tuple) else col
+            for col in self.other_seq_embed_cols
+        ]
+        drop_cols += other_seq_cols
+        other_seqs = {col: group[col].tolist() for col in other_seq_cols}
+
+        other_seqs_sequences: Dict[str, Union[int, List[int]]] = {}
+        for col in other_seq_cols:
+            other_seqs_sequences[col] = (
+                other_seqs[col][i : i + self.max_seq_length]
+                if i + self.max_seq_length < len(other_seqs[col])
+                else other_seqs[col][i:-1]
+            )
 
-        non_seq_cols = group.drop_duplicates(["user_id"]).drop(drop_cols, axis=1)
+        sequence: Dict[str, Union[int, List[int]]] = {}
+        for col in other_seq_cols:
+            sequence[f"{col}_sequence"] = other_seqs_sequences[col]
+            sequence[f"target_{col}"] = (
+                other_seqs[col][i + self.max_seq_length]
+                if i + self.max_seq_length < len(other_seqs[col])
+                else other_seqs[col][-1]
+            )
 
-        return pd.merge(seq_df, non_seq_cols, on="user_id")
+        return sequence
 
-    def inverse_transform(self, X: np.ndarray) -> pd.DataFrame:
-        # trasform mutates the df and is complex to revert the transformation
-        raise NotImplementedError(
-            "inverse_transform is not implemented for this preprocessor"
+    def _concatenate_features(self, df_w_sequences: pd.DataFrame) -> np.ndarray:
+        user_behaviour_seq = df_w_sequences["items_sequence"].tolist()
+        X_user_behaviour = np.vstack(
+            [
+                pad_sequences(seq, self.max_seq_length, pad_idx=0)
+                for seq in user_behaviour_seq
+            ]
         )
+        X_target_item = np.array(df_w_sequences["target_item"].tolist()).reshape(-1, 1)
+        X_all = np.concatenate([X_user_behaviour, X_target_item], axis=1)
 
+        if self.has_standard_tab_data:
+            X_tab = self.tab_preprocessor.transform(df_w_sequences)
+            X_all = np.concatenate([X_tab, X_all], axis=1)
 
-if __name__ == "__main__":
+        if self.action_col:
+            action_seq = df_w_sequences["actions_sequence"].tolist()
+            X_actions = np.vstack(
+                [
+                    pad_sequences(seq, self.max_seq_length, pad_idx=0)
+                    for seq in action_seq
+                ]
+            )
+            X_all = np.concatenate([X_all, X_actions], axis=1)
+
+        if self.other_seq_embed_cols:
+            X_all = self._concatenate_other_seq_features(df_w_sequences, X_all)
 
-    def clean_genre_list(genre_list):
-        return "_".join(
-            sorted([re.sub(r"[^a-z0-9]", "", genre.lower()) for genre in genre_list])
+        assert len(self.din_columns_idx) == X_all.shape[1], (
+            f"Something went wrong. The number of columns in the final array "
+            f"({X_all.shape[1]}) is different from the number of columns in "
+            f"self.din_columns_idx ({len(self.din_columns_idx)})"
         )
 
-    data, users, items = load_movielens100k(as_frame=True)
-
-    list_of_genres = [
-        "unknown",
-        "Action",
-        "Adventure",
-        "Animation",
-        "Children's",
-        "Comedy",
-        "Crime",
-        "Documentary",
-        "Drama",
-        "Fantasy",
-        "Film-Noir",
-        "Horror",
-        "Musical",
-        "Mystery",
-        "Romance",
-        "Sci-Fi",
-        "Thriller",
-        "War",
-        "Western",
-    ]
-
-    assert (
-        isinstance(items, pd.DataFrame)
-        and isinstance(data, pd.DataFrame)
-        and isinstance(users, pd.DataFrame)
-    )
-    items["genre_list"] = items[list_of_genres].apply(
-        lambda x: [genre for genre in list_of_genres if x[genre] == 1], axis=1
-    )
-
-    items["genre_list"] = items["genre_list"].apply(clean_genre_list)
-
-    df = pd.merge(data, items[["movie_id", "genre_list"]], on="movie_id")
-    df = pd.merge(
-        df,
-        users[["user_id", "age", "gender", "occupation"]],
-        on="user_id",
-    )
-
-    df["rating"] = df["rating"].apply(lambda x: 1 if x >= 4 else 0)
-
-    df = df.sort_values(by=["timestamp"]).reset_index(drop=True)
-    df = df.drop("timestamp", axis=1)
-
-    din_preprocessor = DINPreprocessor(
-        user_id_col="user_id",
-        target_col="rating",
-        item_embed_col=("movie_id", 16),
-        max_seq_length=5,
-        action_col="rating",
-        other_seq_embed_cols=[("genre_list", 16)],
-        cat_embed_cols=["user_id", "age", "gender", "occupation"],
-    )
-
-    X, y = din_preprocessor.fit_transform(df)
-
-    din = DeepInterestNetwork(
-        column_idx=din_preprocessor.din_columns_idx,
-        user_behavior_confiq=din_preprocessor.user_behaviour_config,
-        action_seq_config=din_preprocessor.action_seq_config,
-        other_seq_cols_confiq=din_preprocessor.other_seq_config,
-        cat_embed_input=din_preprocessor.tab_preprocessor.cat_embed_input,  # type: ignore[attr-defined]
-        mlp_hidden_dims=[128, 64],
-    )
-
-    # And from here on, everything is standard
-    model = WideDeep(deeptabular=din)
-
-    trainer = Trainer(model=model, objective="binary", metrics=[Accuracy()])
-
-    # in the real world you would have to split the data into train, val and test
-    trainer.fit(
-        X_tab=X,
-        target=y,
-        n_epochs=5,
-        batch_size=512,
-    )
+        return X_all
+
+    def _concatenate_other_seq_features(
+        self, df_w_sequences: pd.DataFrame, X_all: np.ndarray
+    ) -> np.ndarray:
+        other_seq_cols = [
+            col[0] if isinstance(col, tuple) else col
+            for col in self.other_seq_embed_cols
+        ]
+        other_seq = {
+            col: df_w_sequences[f"{col}_sequence"].tolist() for col in other_seq_cols
+        }
+        other_seq_target = {
+            col: df_w_sequences[f"target_{col}"].tolist() for col in other_seq_cols
+        }
+        X_other_seq_arrays = []
+
+        for col in other_seq_cols:
+            X_other_seq_arrays.append(
+                np.vstack(
+                    [
+                        pad_sequences(s, self.max_seq_length, pad_idx=0)
+                        for s in other_seq[col]
+                    ]
+                )
+            )
+            X_other_seq_arrays.append(np.array(other_seq_target[col]).reshape(-1, 1))
+
+        return np.concatenate([X_all] + X_other_seq_arrays, axis=1)
+
+
+# if __name__ == "__main__":
+
+#     def clean_genre_list(genre_list):
+#         return "_".join(
+#             sorted([re.sub(r"[^a-z0-9]", "", genre.lower()) for genre in genre_list])
+#         )
+
+#     data, users, items = load_movielens100k(as_frame=True)
+
+#     list_of_genres = [
+#         "unknown",
+#         "Action",
+#         "Adventure",
+#         "Animation",
+#         "Children's",
+#         "Comedy",
+#         "Crime",
+#         "Documentary",
+#         "Drama",
+#         "Fantasy",
+#         "Film-Noir",
+#         "Horror",
+#         "Musical",
+#         "Mystery",
+#         "Romance",
+#         "Sci-Fi",
+#         "Thriller",
+#         "War",
+#         "Western",
+#     ]
+
+#     assert (
+#         isinstance(items, pd.DataFrame)
+#         and isinstance(data, pd.DataFrame)
+#         and isinstance(users, pd.DataFrame)
+#     )
+#     items["genre_list"] = items[list_of_genres].apply(
+#         lambda x: [genre for genre in list_of_genres if x[genre] == 1], axis=1
+#     )
+
+#     items["genre_list"] = items["genre_list"].apply(clean_genre_list)
+
+#     df = pd.merge(data, items[["movie_id", "genre_list"]], on="movie_id")
+#     df = pd.merge(
+#         df,
+#         users[["user_id", "age", "gender", "occupation"]],
+#         on="user_id",
+#     )
+
+#     df["rating"] = df["rating"].apply(lambda x: 1 if x >= 4 else 0)
+
+#     df = df.sort_values(by=["timestamp"]).reset_index(drop=True)
+#     df = df.drop("timestamp", axis=1)
+
+#     din_preprocessor = DINPreprocessor(
+#         user_id_col="user_id",
+#         target_col="rating",
+#         item_embed_col=("movie_id", 16),
+#         max_seq_length=5,
+#         action_col="rating",
+#         other_seq_embed_cols=[("genre_list", 16)],
+#         cat_embed_cols=["user_id", "age", "gender", "occupation"],
+#     )
+
+#     X, y = din_preprocessor.fit_transform(df)
+
+#     din = DeepInterestNetwork(
+#         column_idx=din_preprocessor.din_columns_idx,
+#         user_behavior_confiq=din_preprocessor.user_behaviour_config,
+#         action_seq_config=din_preprocessor.action_seq_config,
+#         other_seq_cols_confiq=din_preprocessor.other_seq_config,
+#         cat_embed_input=din_preprocessor.tab_preprocessor.cat_embed_input,  # type: ignore[attr-defined]
+#         mlp_hidden_dims=[128, 64],
+#     )
+
+#     # And from here on, everything is standard
+#     model = WideDeep(deeptabular=din)
+
+#     trainer = Trainer(model=model, objective="binary", metrics=[Accuracy()])
+
+#     # in the real world you would have to split the data into train, val and test
+#     trainer.fit(
+#         X_tab=X,
+#         target=y,
+#         n_epochs=5,
+#         batch_size=512,
+#     )
diff --git a/tests/test_data_utils/data_for_rec_preprocessor/interactions_data.csv b/tests/test_data_utils/data_for_rec_preprocessor/interactions_data.csv
new file mode 100644
index 00000000..e7587df7
--- /dev/null
+++ b/tests/test_data_utils/data_for_rec_preprocessor/interactions_data.csv
@@ -0,0 +1,113 @@
+user_id,item_id,age,gender,height,weight,price,category,timestamp,interaction
+5,2,25,M,182.5,99.6,65.05,C,2023-01-02,1
+3,1,32,M,166.7,96.9,65.57,B,2023-01-02,0
+5,6,25,M,182.5,99.6,45.99,A,2023-01-04,0
+3,7,32,M,166.7,96.9,14.2,C,2023-01-05,1
+3,8,32,M,166.7,96.9,97.64,C,2023-01-09,0
+4,2,60,F,157.1,50.0,65.05,C,2023-01-13,1
+2,6,46,M,173.0,86.1,45.99,A,2023-01-15,1
+5,1,25,M,182.5,99.6,65.57,B,2023-01-17,0
+5,7,25,M,182.5,99.6,14.2,C,2023-01-22,0
+5,6,25,M,182.5,99.6,45.99,A,2023-01-27,0
+4,2,60,F,157.1,50.0,65.05,C,2023-01-28,1
+4,8,60,F,157.1,50.0,97.64,C,2023-01-29,1
+3,2,32,M,166.7,96.9,65.05,C,2023-02-02,1
+3,6,32,M,166.7,96.9,45.99,A,2023-02-04,1
+3,9,32,M,166.7,96.9,30.95,A,2023-02-06,0
+2,7,46,M,173.0,86.1,14.2,C,2023-02-10,1
+3,4,32,M,166.7,96.9,12.08,A,2023-02-11,1
+4,2,60,F,157.1,50.0,65.05,C,2023-02-13,1
+4,10,60,F,157.1,50.0,18.15,A,2023-02-14,1
+5,10,25,M,182.5,99.6,18.15,A,2023-02-20,0
+5,2,25,M,182.5,99.6,65.05,C,2023-02-21,1
+1,2,56,M,155.0,52.8,65.05,C,2023-02-22,0
+3,1,32,M,166.7,96.9,65.57,B,2023-03-01,0
+2,6,46,M,173.0,86.1,45.99,A,2023-03-06,1
+4,2,60,F,157.1,50.0,65.05,C,2023-03-07,1
+2,5,46,M,173.0,86.1,57.23,C,2023-03-22,1
+2,9,46,M,173.0,86.1,30.95,A,2023-04-08,0
+3,5,32,M,166.7,96.9,57.23,C,2023-04-09,1
+3,1,32,M,166.7,96.9,65.57,B,2023-04-09,0
+3,3,32,M,166.7,96.9,10.64,C,2023-04-09,0
+4,1,60,F,157.1,50.0,65.57,B,2023-04-09,0
+4,5,60,F,157.1,50.0,57.23,C,2023-04-14,0
+4,4,60,F,157.1,50.0,12.08,A,2023-04-22,0
+3,3,32,M,166.7,96.9,10.64,C,2023-04-22,0
+4,4,60,F,157.1,50.0,12.08,A,2023-04-23,0
+5,9,25,M,182.5,99.6,30.95,A,2023-04-23,1
+5,3,25,M,182.5,99.6,10.64,C,2023-04-23,1
+3,1,32,M,166.7,96.9,65.57,B,2023-04-26,0
+1,1,56,M,155.0,52.8,65.57,B,2023-05-06,0
+5,6,25,M,182.5,99.6,45.99,A,2023-05-08,0
+2,8,46,M,173.0,86.1,97.64,C,2023-05-08,0
+3,5,32,M,166.7,96.9,57.23,C,2023-05-11,1
+4,6,60,F,157.1,50.0,45.99,A,2023-05-14,1
+5,7,25,M,182.5,99.6,14.2,C,2023-05-15,0
+3,2,32,M,166.7,96.9,65.05,C,2023-05-15,1
+2,3,46,M,173.0,86.1,10.64,C,2023-05-16,1
+5,8,25,M,182.5,99.6,97.64,C,2023-05-17,1
+4,9,60,F,157.1,50.0,30.95,A,2023-05-23,1
+3,9,32,M,166.7,96.9,30.95,A,2023-05-24,0
+2,4,46,M,173.0,86.1,12.08,A,2023-05-27,0
+2,8,46,M,173.0,86.1,97.64,C,2023-05-31,0
+3,8,32,M,166.7,96.9,97.64,C,2023-06-01,0
+5,5,25,M,182.5,99.6,57.23,C,2023-06-01,1
+5,8,25,M,182.5,99.6,97.64,C,2023-06-09,1
+5,7,25,M,182.5,99.6,14.2,C,2023-06-10,0
+4,7,60,F,157.1,50.0,14.2,C,2023-06-18,0
+5,1,25,M,182.5,99.6,65.57,B,2023-06-25,0
+5,2,25,M,182.5,99.6,65.05,C,2023-06-29,1
+5,2,25,M,182.5,99.6,65.05,C,2023-07-06,1
+2,8,46,M,173.0,86.1,97.64,C,2023-07-06,0
+2,7,46,M,173.0,86.1,14.2,C,2023-07-07,1
+2,7,46,M,173.0,86.1,14.2,C,2023-07-09,1
+5,6,25,M,182.5,99.6,45.99,A,2023-07-11,0
+2,1,46,M,173.0,86.1,65.57,B,2023-07-20,0
+2,10,46,M,173.0,86.1,18.15,A,2023-07-21,1
+2,8,46,M,173.0,86.1,97.64,C,2023-07-22,0
+1,10,56,M,155.0,52.8,18.15,A,2023-07-25,1
+3,6,32,M,166.7,96.9,45.99,A,2023-07-26,1
+4,4,60,F,157.1,50.0,12.08,A,2023-07-27,0
+2,6,46,M,173.0,86.1,45.99,A,2023-07-27,1
+2,10,46,M,173.0,86.1,18.15,A,2023-08-05,1
+1,4,56,M,155.0,52.8,12.08,A,2023-08-06,0
+5,3,25,M,182.5,99.6,10.64,C,2023-08-08,1
+5,7,25,M,182.5,99.6,14.2,C,2023-08-08,0
+5,9,25,M,182.5,99.6,30.95,A,2023-08-12,1
+5,9,25,M,182.5,99.6,30.95,A,2023-08-13,1
+3,7,32,M,166.7,96.9,14.2,C,2023-08-15,1
+5,8,25,M,182.5,99.6,97.64,C,2023-08-22,1
+2,4,46,M,173.0,86.1,12.08,A,2023-08-28,0
+4,8,60,F,157.1,50.0,97.64,C,2023-08-31,1
+2,8,46,M,173.0,86.1,97.64,C,2023-09-09,0
+4,5,60,F,157.1,50.0,57.23,C,2023-09-11,0
+3,4,32,M,166.7,96.9,12.08,A,2023-09-12,1
+2,4,46,M,173.0,86.1,12.08,A,2023-09-12,0
+5,2,25,M,182.5,99.6,65.05,C,2023-09-16,1
+2,9,46,M,173.0,86.1,30.95,A,2023-09-16,0
+1,5,56,M,155.0,52.8,57.23,C,2023-09-21,1
+2,1,46,M,173.0,86.1,65.57,B,2023-09-24,0
+2,7,46,M,173.0,86.1,14.2,C,2023-09-27,1
+5,4,25,M,182.5,99.6,12.08,A,2023-10-04,0
+2,6,46,M,173.0,86.1,45.99,A,2023-10-07,1
+5,1,25,M,182.5,99.6,65.57,B,2023-10-11,0
+4,2,60,F,157.1,50.0,65.05,C,2023-10-13,1
+5,7,25,M,182.5,99.6,14.2,C,2023-10-16,0
+5,5,25,M,182.5,99.6,57.23,C,2023-10-22,1
+3,8,32,M,166.7,96.9,97.64,C,2023-10-31,0
+3,1,32,M,166.7,96.9,65.57,B,2023-10-31,0
+5,10,25,M,182.5,99.6,18.15,A,2023-10-31,0
+4,3,60,F,157.1,50.0,10.64,C,2023-11-06,0
+5,1,25,M,182.5,99.6,65.57,B,2023-11-13,0
+5,2,25,M,182.5,99.6,65.05,C,2023-11-14,1
+2,3,46,M,173.0,86.1,10.64,C,2023-11-23,1
+3,7,32,M,166.7,96.9,14.2,C,2023-11-24,1
+4,3,60,F,157.1,50.0,10.64,C,2023-12-04,0
+2,6,46,M,173.0,86.1,45.99,A,2023-12-04,1
+1,9,56,M,155.0,52.8,30.95,A,2023-12-06,1
+2,3,46,M,173.0,86.1,10.64,C,2023-12-10,1
+2,4,46,M,173.0,86.1,12.08,A,2023-12-12,0
+4,1,60,F,157.1,50.0,65.57,B,2023-12-15,0
+2,8,46,M,173.0,86.1,97.64,C,2023-12-18,0
+3,2,32,M,166.7,96.9,65.05,C,2023-12-26,1
+2,2,46,M,173.0,86.1,65.05,C,2023-12-26,0
diff --git a/tests/test_data_utils/generate_interactions_data.py b/tests/test_data_utils/generate_interactions_data.py
new file mode 100644
index 00000000..df128e37
--- /dev/null
+++ b/tests/test_data_utils/generate_interactions_data.py
@@ -0,0 +1,194 @@
+import os
+from pathlib import Path
+from datetime import datetime, timedelta
+
+import numpy as np
+import pandas as pd
+
+full_path = os.path.realpath(__file__)
+path = os.path.split(full_path)[0]
+
+save_dir = Path(path) / "data_for_rec_preprocessor"
+
+
+def generate_sample_data(n_users=5, n_items=10, seed=42, return_df=False):
+    """
+    Generate a sample dataset for recommendation system testing.
+
+    Parameters:
+    -----------
+    - n_users: int, number of users to generate (default 5)
+    - n_items: int, number of items to generate (default 10)
+    - seed: int, random seed for reproducibility (default 42)
+    """
+    np.random.seed(seed)
+
+    # Generate user data
+    users = pd.DataFrame(
+        {
+            "user_id": range(1, n_users + 1),
+            "age": np.random.randint(18, 65, n_users),
+            "gender": np.random.choice(["M", "F"], n_users),
+            "height": np.random.uniform(150, 200, n_users).round(1),
+            "weight": np.random.uniform(50, 100, n_users).round(1),
+        }
+    )
+
+    # Generate item data
+    items = pd.DataFrame(
+        {
+            "item_id": range(1, n_items + 1),
+            "price": np.random.uniform(10, 100, n_items).round(2),
+            "category": np.random.choice(["A", "B", "C"], n_items),
+        }
+    )
+
+    # Generate positive interactions
+    positive_interactions = []
+    start_date = datetime(2023, 1, 1)
+
+    for user_id in users["user_id"]:
+        # Randomly select 5 items for this user to interact with
+        user_items = np.random.choice(items["item_id"], 5, replace=False)
+
+        if user_id == 1:
+            n_interactions = 3
+        else:
+            n_interactions = np.random.randint(4, 21)
+
+        for _ in range(n_interactions):
+            item_id = np.random.choice(user_items)
+            timestamp = start_date + timedelta(days=np.random.randint(0, 365))
+            positive_interactions.append(
+                {
+                    "user_id": user_id,
+                    "item_id": item_id,
+                    "timestamp": timestamp,
+                    "interaction": 1,  # Positive interaction
+                }
+            )
+
+    positive_df = pd.DataFrame(positive_interactions)
+
+    # Generate negative interactions
+    negative_interactions = []
+    for user_id in users["user_id"]:
+        positive_items = positive_df[positive_df["user_id"] == user_id][
+            "item_id"
+        ].unique()
+        negative_items = items[~items["item_id"].isin(positive_items)]["item_id"]
+
+        n_negative = len(positive_df[positive_df["user_id"] == user_id])
+
+        for _ in range(n_negative):
+            item_id = np.random.choice(negative_items)
+            timestamp = start_date + timedelta(days=np.random.randint(0, 365))
+            negative_interactions.append(
+                {
+                    "user_id": user_id,
+                    "item_id": item_id,
+                    "timestamp": timestamp,
+                    "interaction": 0,  # Negative interaction
+                }
+            )
+
+    negative_df = pd.DataFrame(negative_interactions)
+
+    # Combine positive and negative interactions
+    interactions_df = pd.concat([positive_df, negative_df], ignore_index=True)
+
+    # Merge all data
+    final_df = interactions_df.merge(users, on="user_id").merge(items, on="item_id")
+
+    # Sort by timestamp
+    final_df = final_df.sort_values("timestamp").reset_index(drop=True)
+
+    # Reorder columns
+    column_order = [
+        "user_id",
+        "item_id",
+        "age",
+        "gender",
+        "height",
+        "weight",
+        "price",
+        "category",
+        "timestamp",
+        "interaction",
+    ]
+    final_df = final_df[column_order]
+
+    if not save_dir.exists():
+        save_dir.mkdir(parents=True)
+
+    if not return_df:
+        final_df.to_csv(save_dir / "interactions_data.csv", index=False)
+    else:
+        return final_df
+
+
+def split_by_timestamp(df, train_ratio=0.8, val_ratio=0.1, test_ratio=0.1):
+    """
+    Split the dataframe into train, validation, and test sets based on timestamp.
+
+    Parameters:
+    - df: pd.DataFrame, the input dataframe
+    - train_ratio: float, ratio of data for training (default 0.8)
+    - val_ratio: float, ratio of data for validation (default 0.1)
+    - test_ratio: float, ratio of data for testing (default 0.1)
+
+    Returns:
+    - train_df, val_df, test_df: pd.DataFrame, the split dataframes
+    """
+    assert np.isclose(train_ratio + val_ratio + test_ratio, 1.0), "Ratios must sum to 1"
+
+    df_sorted = df.sort_values("timestamp")
+    total_rows = len(df_sorted)
+    train_rows = int(total_rows * train_ratio)
+    val_rows = int(total_rows * val_ratio)
+
+    train_df = df_sorted.iloc[:train_rows]
+    val_df = df_sorted.iloc[train_rows : train_rows + val_rows]
+    test_df = df_sorted.iloc[train_rows + val_rows :]
+
+    return train_df, val_df, test_df
+
+
+def split_by_last_interactions(df):
+    """
+    Split the dataframe into train, validation, and test sets based on the last interactions.
+    Train: all interactions but last two
+    Validation: second to last interaction
+    Test: last interaction
+
+    Parameters:
+    - df: pd.DataFrame, the input dataframe
+
+    Returns:
+    - train_df, val_df, test_df: pd.DataFrame, the split dataframes
+    """
+    df_sorted = df.sort_values(["user_id", "timestamp"])
+
+    # Get the last two interactions for each user
+    last_two = df_sorted.groupby("user_id").tail(2)
+
+    # Split the last two interactions into validation and test
+    test_df = last_two.groupby("user_id").last().reset_index()
+    val_df = last_two.groupby("user_id").first().reset_index()
+
+    # Remove the last two interactions from the original dataframe to create the train set
+    train_df = df_sorted[~df_sorted.index.isin(last_two.index)].reset_index(drop=True)
+
+    return train_df, val_df, test_df
+
+
+if __name__ == "__main__":
+    generate_sample_data()
+
+    # df = generate_sample_data(return_df=True)
+
+    # # Split the data by timestamp
+    # train_df, val_df, test_df = split_by_timestamp(df)
+
+    # # Split the data by last interactions
+    # train_df2, val_df2, test_df2 = split_by_last_interactions(df)
diff --git a/tests/test_data_utils/test_din_preprocessor.py b/tests/test_data_utils/test_din_preprocessor.py
new file mode 100644
index 00000000..36523b77
--- /dev/null
+++ b/tests/test_data_utils/test_din_preprocessor.py
@@ -0,0 +1,87 @@
+import os
+from pathlib import Path
+
+import numpy as np
+import pandas as pd
+
+from pytorch_widedeep.preprocessing import DINPreprocessor
+
+full_path = os.path.realpath(__file__)
+path = os.path.split(full_path)[0]
+
+save_dir = Path(path) / "data_for_rec_preprocessor"
+
+df_interactions = pd.read_csv(save_dir / "interactions_data.csv")
+
+df_interactions = df_interactions.sort_values(by=["user_id", "timestamp"]).reset_index(
+    drop=True
+)
+
+cat_embed_cols = ["user_id", "age", "gender"]
+continuous_cols = ["height", "weight"]
+
+
+def user_behaviour_well_encoded(
+    input_df: pd.DataFrame,
+    din_preprocessor: DINPreprocessor,
+    X: np.ndarray,
+    max_seq_length: int,
+):
+
+    user_id: int = 5
+
+    encoded_user_id = (  # type: ignore
+        din_preprocessor.tab_preprocessor.label_encoder.encoding_dict["user_id"][
+            user_id
+        ]
+    )
+
+    user_items = (
+        input_df[input_df["user_id"] == user_id]
+        .groupby("user_id")["item_id"]
+        .agg(list)
+        .values[0]
+    )[:max_seq_length]
+
+    encoded_items = [
+        din_preprocessor.item_le.encoding_dict["item_id"][item] for item in user_items
+    ]
+
+    rows = np.where(
+        X[:, din_preprocessor.din_columns_idx["user_id"]] == encoded_user_id
+    )[0]
+    X_user_id = X[rows][:1]
+
+    item_seq_cols = din_preprocessor.user_behaviour_config[0]
+    item_seq_cols_idx = [din_preprocessor.din_columns_idx[col] for col in item_seq_cols]
+
+    X_user_items = list(X_user_id[:, item_seq_cols_idx].astype(int)[0])
+
+    return X_user_items == encoded_items
+
+
+def test_din_preprocessor():
+
+    din_preprocessor = DINPreprocessor(
+        user_id_col="user_id",
+        item_embed_col="item_id",
+        target_col="interaction",
+        action_col="interaction",
+        other_seq_embed_cols=["category", "price"],
+        cat_embed_cols=cat_embed_cols,
+        continuous_cols=continuous_cols,
+        cols_to_scale=continuous_cols,
+        max_seq_length=5,
+    )
+
+    X, y = din_preprocessor.fit_transform(df_interactions)
+    # 5 items + 5 actions + 2 * 5 other_seq_embed_cols + (1 target item + 1
+    # target category + 1 target price) + 5 continuous_cols and
+    # cat_embed_cols
+    expected_n_cols = 5 + 5 + (2 * 5) + 3 + 5
+    assert X.shape[1] == expected_n_cols
+    assert user_behaviour_well_encoded(df_interactions, din_preprocessor, X, 5)
+
+
+if __name__ == "__main__":
+    test_din_preprocessor()

From ecc6ea0532cf9a1cb97abee134f1b72cf8921d74 Mon Sep 17 00:00:00 2001
From: Javier <jrzaurin@gmail.com>
Date: Mon, 14 Oct 2024 20:34:07 +0100
Subject: [PATCH 05/24] ready to test the whole thing. I would have to add an
 extra text combining the preprocessor and the model

---
 .../preprocessing/din_preprocessor.py         |   8 +-
 .../generate_interactions_data.py             |   2 +-
 .../interactions_data.csv                     |   0
 .../test_data_utils/test_din_preprocessor.py  |  87 ----------
 .../test_du_din_preprocessor.py               | 162 ++++++++++++++++++
 5 files changed, 169 insertions(+), 90 deletions(-)
 rename tests/test_data_utils/{data_for_rec_preprocessor => interactions_df_for_rec_preprocessor}/interactions_data.csv (100%)
 delete mode 100644 tests/test_data_utils/test_din_preprocessor.py
 create mode 100644 tests/test_data_utils/test_du_din_preprocessor.py

diff --git a/pytorch_widedeep/preprocessing/din_preprocessor.py b/pytorch_widedeep/preprocessing/din_preprocessor.py
index d28e6926..5819875f 100644
--- a/pytorch_widedeep/preprocessing/din_preprocessor.py
+++ b/pytorch_widedeep/preprocessing/din_preprocessor.py
@@ -179,7 +179,7 @@ def transform(self, df: pd.DataFrame) -> Tuple[np.ndarray, np.ndarray]:
         _df = self._pre_transform(df)
         df_w_sequences = self._build_sequences(_df)
         X_all = self._concatenate_features(df_w_sequences)
-        return X_all, np.array(df_w_sequences["target"].tolist())
+        return X_all, np.array(df_w_sequences[self.target_col].tolist())
 
     def fit_transform(self, df: pd.DataFrame) -> Tuple[np.ndarray, np.ndarray]:
         self.fit(df)
@@ -363,6 +363,10 @@ def _create_sequence(
             sequence.update(
                 self._create_action_sequence(group, targets, i, target, drop_cols)
             )
+        else:
+            sequence[self.target_col] = target
+            drop_cols.append(self.target_col)
+
         if self.other_seq_embed_cols:
             sequence.update(self._create_other_seq_sequence(group, i, drop_cols))
 
@@ -392,7 +396,7 @@ def _create_action_sequence(
                 else targets[i:-1]
             )
 
-        return {"target": target, "actions_sequence": action_sequences}
+        return {self.target_col: target, "actions_sequence": action_sequences}
 
     def _create_other_seq_sequence(
         self, group: pd.DataFrame, i: int, drop_cols: List[str]
diff --git a/tests/test_data_utils/generate_interactions_data.py b/tests/test_data_utils/generate_interactions_data.py
index df128e37..7df32497 100644
--- a/tests/test_data_utils/generate_interactions_data.py
+++ b/tests/test_data_utils/generate_interactions_data.py
@@ -8,7 +8,7 @@
 full_path = os.path.realpath(__file__)
 path = os.path.split(full_path)[0]
 
-save_dir = Path(path) / "data_for_rec_preprocessor"
+save_dir = Path(path) / "interactions_df_for_rec_preprocessor"
 
 
 def generate_sample_data(n_users=5, n_items=10, seed=42, return_df=False):
diff --git a/tests/test_data_utils/data_for_rec_preprocessor/interactions_data.csv b/tests/test_data_utils/interactions_df_for_rec_preprocessor/interactions_data.csv
similarity index 100%
rename from tests/test_data_utils/data_for_rec_preprocessor/interactions_data.csv
rename to tests/test_data_utils/interactions_df_for_rec_preprocessor/interactions_data.csv
diff --git a/tests/test_data_utils/test_din_preprocessor.py b/tests/test_data_utils/test_din_preprocessor.py
deleted file mode 100644
index 36523b77..00000000
--- a/tests/test_data_utils/test_din_preprocessor.py
+++ /dev/null
@@ -1,87 +0,0 @@
-import os
-from pathlib import Path
-
-import numpy as np
-import pandas as pd
-
-from pytorch_widedeep.preprocessing import DINPreprocessor
-
-full_path = os.path.realpath(__file__)
-path = os.path.split(full_path)[0]
-
-save_dir = Path(path) / "data_for_rec_preprocessor"
-
-df_interactions = pd.read_csv(save_dir / "interactions_data.csv")
-
-df_interactions = df_interactions.sort_values(by=["user_id", "timestamp"]).reset_index(
-    drop=True
-)
-
-cat_embed_cols = ["user_id", "age", "gender"]
-continuous_cols = ["height", "weight"]
-
-
-def user_behaviour_well_encoded(
-    input_df: pd.DataFrame,
-    din_preprocessor: DINPreprocessor,
-    X: np.ndarray,
-    max_seq_length: int,
-):
-
-    user_id: int = 5
-
-    encoded_user_id = (  # type: ignore
-        din_preprocessor.tab_preprocessor.label_encoder.encoding_dict["user_id"][
-            user_id
-        ]
-    )
-
-    user_items = (
-        input_df[input_df["user_id"] == user_id]
-        .groupby("user_id")["item_id"]
-        .agg(list)
-        .values[0]
-    )[:max_seq_length]
-
-    encoded_items = [
-        din_preprocessor.item_le.encoding_dict["item_id"][item] for item in user_items
-    ]
-
-    rows = np.where(
-        X[:, din_preprocessor.din_columns_idx["user_id"]] == encoded_user_id
-    )[0]
-    X_user_id = X[rows][:1]
-
-    item_seq_cols = din_preprocessor.user_behaviour_config[0]
-    item_seq_cols_idx = [din_preprocessor.din_columns_idx[col] for col in item_seq_cols]
-
-    X_user_items = list(X_user_id[:, item_seq_cols_idx].astype(int)[0])
-
-    return X_user_items == encoded_items
-
-
-def test_din_preprocessor():
-
-    din_preprocessor = DINPreprocessor(
-        user_id_col="user_id",
-        item_embed_col="item_id",
-        target_col="interaction",
-        action_col="interaction",
-        other_seq_embed_cols=["category", "price"],
-        cat_embed_cols=cat_embed_cols,
-        continuous_cols=continuous_cols,
-        cols_to_scale=continuous_cols,
-        max_seq_length=5,
-    )
-
-    X, y = din_preprocessor.fit_transform(df_interactions)
-    # 5 items + 5 actions + 2 * 5 other_seq_embed_cols + (1 target item + 1
-    # target category + 1 target price) + 5 continuous_cols and
-    # cat_embed_cols
-    expected_n_cols = 5 + 5 + (2 * 5) + 3 + 5
-    assert X.shape[1] == expected_n_cols
-    assert user_behaviour_well_encoded(df_interactions, din_preprocessor, X, 5)
-
-
-if __name__ == "__main__":
-    test_din_preprocessor()
diff --git a/tests/test_data_utils/test_du_din_preprocessor.py b/tests/test_data_utils/test_du_din_preprocessor.py
new file mode 100644
index 00000000..40ea8dfd
--- /dev/null
+++ b/tests/test_data_utils/test_du_din_preprocessor.py
@@ -0,0 +1,162 @@
+import os
+from pathlib import Path
+
+import numpy as np
+import pandas as pd
+import pytest
+
+from pytorch_widedeep.preprocessing import DINPreprocessor
+
+full_path = os.path.realpath(__file__)
+path = os.path.split(full_path)[0]
+
+save_dir = Path(path) / "interactions_df_for_rec_preprocessor"
+
+df_interactions = pd.read_csv(save_dir / "interactions_data.csv")
+
+df_interactions = df_interactions.sort_values(by=["user_id", "timestamp"]).reset_index(
+    drop=True
+)
+
+cat_embed_cols = ["user_id", "age", "gender"]
+continuous_cols = ["height", "weight"]
+
+
+def user_behaviour_well_encoded(
+    input_df: pd.DataFrame,
+    din_preprocessor: DINPreprocessor,
+    X: np.ndarray,
+    max_seq_length: int,
+):
+
+    user_id: int = 5
+
+    encoded_user_id = (  # type: ignore
+        din_preprocessor.tab_preprocessor.label_encoder.encoding_dict["user_id"][
+            user_id
+        ]
+    )
+
+    user_items = (
+        input_df[input_df["user_id"] == user_id]
+        .groupby("user_id")["item_id"]
+        .agg(list)
+        .values[0]
+    )[:max_seq_length]
+
+    encoded_items = [
+        din_preprocessor.item_le.encoding_dict["item_id"][item] for item in user_items
+    ]
+
+    rows = np.where(
+        X[:, din_preprocessor.din_columns_idx["user_id"]] == encoded_user_id
+    )[0]
+    X_user_id = X[rows][:1]
+
+    item_seq_cols = din_preprocessor.user_behaviour_config[0]
+    item_seq_cols_idx = [din_preprocessor.din_columns_idx[col] for col in item_seq_cols]
+
+    X_user_items = list(X_user_id[:, item_seq_cols_idx].astype(int)[0])
+
+    return X_user_items == encoded_items
+
+
+def test_din_preprocessor():
+
+    din_preprocessor = DINPreprocessor(
+        user_id_col="user_id",
+        item_embed_col="item_id",
+        target_col="interaction",
+        action_col="interaction",
+        other_seq_embed_cols=["category", "price"],
+        cat_embed_cols=cat_embed_cols,
+        continuous_cols=continuous_cols,
+        cols_to_scale=continuous_cols,
+        max_seq_length=5,
+    )
+
+    X, y = din_preprocessor.fit_transform(df_interactions)
+    # 5 items + 5 actions + 2 * 5 other_seq_embed_cols + (1 target item + 1
+    # target category + 1 target price) + 5 continuous_cols and
+    # cat_embed_cols
+    expected_n_cols = 5 + 5 + (2 * 5) + 3 + 5
+    assert X.shape[1] == expected_n_cols
+    assert din_preprocessor.is_fitted
+    assert user_behaviour_well_encoded(df_interactions, din_preprocessor, X, 5)
+
+
+@pytest.mark.parametrize(
+    "with_cat_embed_cols, with_continuous_cols, with_action_col, with_other_seq_embed_cols",
+    [
+        (True, True, True, True),
+        (True, True, True, False),
+        (True, True, False, True),
+        (True, True, False, False),
+        (True, False, True, True),
+        (True, False, True, False),
+        (True, False, False, True),
+        (True, False, False, False),
+        (False, True, True, True),
+        (False, True, True, False),
+        (False, True, False, True),
+        (False, True, False, False),
+        (False, False, True, True),
+        (False, False, True, False),
+        (False, False, False, True),
+        (False, False, False, False),
+    ],
+)
+def test_din_preprocessor_diff_input_params(
+    with_cat_embed_cols,
+    with_continuous_cols,
+    with_action_col,
+    with_other_seq_embed_cols,
+):
+
+    max_seq_length = 5
+
+    din_preprocessor = DINPreprocessor(
+        user_id_col="user_id",
+        item_embed_col="item_id",
+        target_col="interaction",
+        action_col="interaction" if with_action_col else None,
+        other_seq_embed_cols=(
+            ["category", "price"] if with_other_seq_embed_cols else None
+        ),
+        cat_embed_cols=cat_embed_cols if with_cat_embed_cols else None,
+        continuous_cols=continuous_cols if with_continuous_cols else None,
+        cols_to_scale=continuous_cols if with_continuous_cols else None,
+        max_seq_length=max_seq_length,
+    )
+
+    X, y = din_preprocessor.fit_transform(df_interactions)
+
+    expected_n_cols = max_seq_length + 1
+
+    assert din_preprocessor.user_behaviour_config[0] == [
+        f"item_{i+1}" for i in range(max_seq_length)
+    ]
+
+    if with_action_col:
+        expected_n_cols += 5
+        assert din_preprocessor.action_seq_config[0] == [
+            f"action_{i+1}" for i in range(max_seq_length)
+        ]
+
+    if with_other_seq_embed_cols:
+        for i, col in enumerate(["category", "price"]):
+            expected_n_cols += 5 + 1
+            assert din_preprocessor.other_seq_config[i][0] == [
+                f"{col}_{i+1}" for i in range(max_seq_length)
+            ]
+
+    if with_cat_embed_cols:
+        expected_n_cols += len(cat_embed_cols)
+
+    if with_continuous_cols:
+        expected_n_cols += len(continuous_cols)
+
+    assert din_preprocessor.is_fitted
+    assert X.shape[1] == expected_n_cols
+    if with_cat_embed_cols:
+        assert user_behaviour_well_encoded(df_interactions, din_preprocessor, X, 5)

From 112d8c317b804fe74c041de3521a8cc11bf2a627 Mon Sep 17 00:00:00 2001
From: Javier <jrzaurin@gmail.com>
Date: Mon, 14 Oct 2024 21:34:50 +0100
Subject: [PATCH 06/24] Added test for the full process. Ready to test the
 whole library

---
 pytorch_widedeep/models/rec/din.py            |  3 +-
 .../test_du_din_preprocessor.py               | 42 +++++++++++++++++++
 tests/test_rec/test_din.py                    |  2 +-
 3 files changed, 45 insertions(+), 2 deletions(-)

diff --git a/pytorch_widedeep/models/rec/din.py b/pytorch_widedeep/models/rec/din.py
index 293f249e..571717a1 100644
--- a/pytorch_widedeep/models/rec/din.py
+++ b/pytorch_widedeep/models/rec/din.py
@@ -405,7 +405,8 @@ def forward(self, X: Tensor) -> Tensor:
                 [
                     self.other_seq_cols_embed[col]._get_embeddings(X_other_seq[col])
                     for col in self.other_seq_cols_indexes.keys()
-                ]
+                ],
+                dim=-1,
             ).sum(1)
             deep_out = torch.cat([deep_out, other_seq_embed], dim=1)
 
diff --git a/tests/test_data_utils/test_du_din_preprocessor.py b/tests/test_data_utils/test_du_din_preprocessor.py
index 40ea8dfd..b56cd04d 100644
--- a/tests/test_data_utils/test_du_din_preprocessor.py
+++ b/tests/test_data_utils/test_du_din_preprocessor.py
@@ -5,6 +5,9 @@
 import pandas as pd
 import pytest
 
+from pytorch_widedeep import Trainer
+from pytorch_widedeep.models import WideDeep
+from pytorch_widedeep.models.rec import DeepInterestNetwork
 from pytorch_widedeep.preprocessing import DINPreprocessor
 
 full_path = os.path.realpath(__file__)
@@ -160,3 +163,42 @@ def test_din_preprocessor_diff_input_params(
     assert X.shape[1] == expected_n_cols
     if with_cat_embed_cols:
         assert user_behaviour_well_encoded(df_interactions, din_preprocessor, X, 5)
+
+
+def test_din_full_process_w_processor():
+
+    din_preprocessor = DINPreprocessor(
+        user_id_col="user_id",
+        item_embed_col="item_id",
+        target_col="interaction",
+        action_col="interaction",
+        other_seq_embed_cols=["category", "price"],
+        cat_embed_cols=cat_embed_cols,
+        continuous_cols=continuous_cols,
+        cols_to_scale=continuous_cols,
+        max_seq_length=5,
+    )
+
+    X, y = din_preprocessor.fit_transform(df_interactions)
+
+    din = DeepInterestNetwork(
+        column_idx=din_preprocessor.din_columns_idx,
+        user_behavior_confiq=din_preprocessor.user_behaviour_config,
+        target_item_col="target_item",
+        action_seq_config=din_preprocessor.action_seq_config,
+        other_seq_cols_confiq=din_preprocessor.other_seq_config,
+        cat_embed_input=din_preprocessor.tab_preprocessor.cat_embed_input,  # type: ignore
+        continuous_cols=din_preprocessor.tab_preprocessor.continuous_cols,
+        mlp_hidden_dims=[16, 8],
+    )
+
+    model = WideDeep(deeptabular=din)
+
+    trainer = Trainer(model, objective="binary", verbose=0)
+
+    trainer.fit(X_tab=X, target=y, n_epochs=2)
+
+    preds = trainer.predict(X_tab=X)
+
+    assert preds.shape[0] == X.shape[0]
+    assert trainer.history is not None and "train_loss" in trainer.history
diff --git a/tests/test_rec/test_din.py b/tests/test_rec/test_din.py
index 92724791..dcdbde86 100644
--- a/tests/test_rec/test_din.py
+++ b/tests/test_rec/test_din.py
@@ -79,8 +79,8 @@ def test_din_with_params(
 
     din = DeepInterestNetwork(
         column_idx=column_idx,
-        target_item_col="target_item",
         user_behavior_confiq=item_seq_config,
+        target_item_col="target_item",
         action_seq_config=item_purchased_seq_config,
         other_seq_cols_confiq=item_category_seq_config,
         cat_embed_input=cat_embed_input,

From adc77d7bf87fe80b94e4041058b29d3264c533ae Mon Sep 17 00:00:00 2001
From: Javier <jrzaurin@gmail.com>
Date: Tue, 15 Oct 2024 10:42:08 +0100
Subject: [PATCH 07/24] All test passed. Need to increase coverage a bit

---
 .../sources/pytorch-widedeep/preprocessing.md |   2 +
 pytorch_widedeep/models/rec/din.py            |   7 +-
 .../preprocessing/din_preprocessor.py         | 124 ++++--------------
 pytorch_widedeep/tab2vec.py                   |   2 +-
 4 files changed, 34 insertions(+), 101 deletions(-)

diff --git a/mkdocs/sources/pytorch-widedeep/preprocessing.md b/mkdocs/sources/pytorch-widedeep/preprocessing.md
index ab608846..520dd914 100644
--- a/mkdocs/sources/pytorch-widedeep/preprocessing.md
+++ b/mkdocs/sources/pytorch-widedeep/preprocessing.md
@@ -21,6 +21,8 @@ available: one for the case when no Hugging Face model is used
 
 ::: pytorch_widedeep.preprocessing.image_preprocessor.ImagePreprocessor
 
+::: pytorch_widedeep.preprocessing.din_preprocessor.DINPreprocessor
+
 
 ## Chunked versions
 
diff --git a/pytorch_widedeep/models/rec/din.py b/pytorch_widedeep/models/rec/din.py
index 571717a1..3c42a975 100644
--- a/pytorch_widedeep/models/rec/din.py
+++ b/pytorch_widedeep/models/rec/din.py
@@ -23,10 +23,9 @@ class DeepInterestNetwork(BaseWDModelComponent):
     sequential columns and will be treated as standard tabular data.
 
     This model requires some specific data preparation that allows for quite a
-    lot of flexibility. Therefore, this library does not currently include a
-    preprocessor designed specifically for this model. Please, see the
-    example 'movielens_din.py' in the examples folder to understand the data
-    preparation process.
+    lot of flexibility. Therefore, I have included a preprocessor
+    (`DINPreprocessor`) in the preprocessing module that will take care of
+    the data preparation.
 
     Parameters
     ----------
diff --git a/pytorch_widedeep/preprocessing/din_preprocessor.py b/pytorch_widedeep/preprocessing/din_preprocessor.py
index 5819875f..f66bbdf7 100644
--- a/pytorch_widedeep/preprocessing/din_preprocessor.py
+++ b/pytorch_widedeep/preprocessing/din_preprocessor.py
@@ -5,11 +5,6 @@
 import numpy as np
 import pandas as pd
 
-# from pytorch_widedeep.models import WideDeep
-# from pytorch_widedeep.metrics import Accuracy
-# from pytorch_widedeep.datasets import load_movielens100k
-# from pytorch_widedeep.training import Trainer
-# from pytorch_widedeep.models.rec.din import DeepInterestNetwork
 from pytorch_widedeep.utils.text_utils import pad_sequences
 from pytorch_widedeep.utils.general_utils import alias
 from pytorch_widedeep.utils.deeptabular_utils import LabelEncoder
@@ -100,6 +95,32 @@ class DINPreprocessor(BasePreprocessor):
         Number of unique values in each other sequence column.
     other_seq_config : List[Tuple[List[str], int, int]]
         Configuration for other sequence columns.
+
+    Examples:
+    ---------
+    >>> import pandas as pd
+    >>> from pytorch_widedeep.preprocessing import DINPreprocessor
+    >>> data = {
+    ...    'user_id': [1, 1, 1, 2, 2, 2, 3, 3, 3],
+    ...    'item_id': [101, 102, 103, 101, 103, 104, 102, 103, 104],
+    ...    'timestamp': [1, 2, 3, 1, 2, 3, 1, 2, 3],
+    ...    'category': ['A', 'B', 'A', 'B', 'A', 'C', 'B', 'A', 'C'],
+    ...    'price': [10.5, 15.0, 12.0, 10.5, 12.0, 20.0, 15.0, 12.0, 20.0],
+    ...    'rating': [0, 1, 0, 1, 0, 1, 0, 1, 0]
+    ... }
+    >>> df = pd.DataFrame(data)
+    >>> din_preprocessor = DINPreprocessor(
+    ...     user_id_col='user_id',
+    ...     item_embed_col='item_id',
+    ...     target_col='rating',
+    ...     max_seq_length=2,
+    ...     action_col='rating',
+    ...     other_seq_embed_cols=['category'],
+    ...     cat_embed_cols=['user_id'],
+    ...     continuous_cols=['price'],
+    ...     cols_to_scale=['price']
+    ... )
+    >>> X, y = din_preprocessor.fit_transform(df)
     """
 
     @alias("item_embed_col", ["item_id_col"])
@@ -313,9 +334,9 @@ def _pre_transform(self, df: pd.DataFrame) -> pd.DataFrame:
         return df
 
     def _build_sequences(self, df: pd.DataFrame) -> pd.DataFrame:
-        # TO DO: do something to avoid warning here
+        # df.columns.tolist() -> to avoid annoying pandas warning
         return (
-            df.groupby(self.user_id_col)
+            df.groupby(self.user_id_col)[df.columns.tolist()]  # type: ignore[index]
             .apply(self._group_sequences)
             .reset_index(drop=True)
         )
@@ -490,92 +511,3 @@ def _concatenate_other_seq_features(
             X_other_seq_arrays.append(np.array(other_seq_target[col]).reshape(-1, 1))
 
         return np.concatenate([X_all] + X_other_seq_arrays, axis=1)
-
-
-# if __name__ == "__main__":
-
-#     def clean_genre_list(genre_list):
-#         return "_".join(
-#             sorted([re.sub(r"[^a-z0-9]", "", genre.lower()) for genre in genre_list])
-#         )
-
-#     data, users, items = load_movielens100k(as_frame=True)
-
-#     list_of_genres = [
-#         "unknown",
-#         "Action",
-#         "Adventure",
-#         "Animation",
-#         "Children's",
-#         "Comedy",
-#         "Crime",
-#         "Documentary",
-#         "Drama",
-#         "Fantasy",
-#         "Film-Noir",
-#         "Horror",
-#         "Musical",
-#         "Mystery",
-#         "Romance",
-#         "Sci-Fi",
-#         "Thriller",
-#         "War",
-#         "Western",
-#     ]
-
-#     assert (
-#         isinstance(items, pd.DataFrame)
-#         and isinstance(data, pd.DataFrame)
-#         and isinstance(users, pd.DataFrame)
-#     )
-#     items["genre_list"] = items[list_of_genres].apply(
-#         lambda x: [genre for genre in list_of_genres if x[genre] == 1], axis=1
-#     )
-
-#     items["genre_list"] = items["genre_list"].apply(clean_genre_list)
-
-#     df = pd.merge(data, items[["movie_id", "genre_list"]], on="movie_id")
-#     df = pd.merge(
-#         df,
-#         users[["user_id", "age", "gender", "occupation"]],
-#         on="user_id",
-#     )
-
-#     df["rating"] = df["rating"].apply(lambda x: 1 if x >= 4 else 0)
-
-#     df = df.sort_values(by=["timestamp"]).reset_index(drop=True)
-#     df = df.drop("timestamp", axis=1)
-
-#     din_preprocessor = DINPreprocessor(
-#         user_id_col="user_id",
-#         target_col="rating",
-#         item_embed_col=("movie_id", 16),
-#         max_seq_length=5,
-#         action_col="rating",
-#         other_seq_embed_cols=[("genre_list", 16)],
-#         cat_embed_cols=["user_id", "age", "gender", "occupation"],
-#     )
-
-#     X, y = din_preprocessor.fit_transform(df)
-
-#     din = DeepInterestNetwork(
-#         column_idx=din_preprocessor.din_columns_idx,
-#         user_behavior_confiq=din_preprocessor.user_behaviour_config,
-#         action_seq_config=din_preprocessor.action_seq_config,
-#         other_seq_cols_confiq=din_preprocessor.other_seq_config,
-#         cat_embed_input=din_preprocessor.tab_preprocessor.cat_embed_input,  # type: ignore[attr-defined]
-#         mlp_hidden_dims=[128, 64],
-#     )
-
-#     # And from here on, everything is standard
-#     model = WideDeep(deeptabular=din)
-
-#     trainer = Trainer(model=model, objective="binary", metrics=[Accuracy()])
-
-#     # in the real world you would have to split the data into train, val and test
-#     trainer.fit(
-#         X_tab=X,
-#         target=y,
-#         n_epochs=5,
-#         batch_size=512,
-#     )
diff --git a/pytorch_widedeep/tab2vec.py b/pytorch_widedeep/tab2vec.py
index 5ba2ab48..c10ddb56 100644
--- a/pytorch_widedeep/tab2vec.py
+++ b/pytorch_widedeep/tab2vec.py
@@ -77,7 +77,7 @@ class Tab2Vec:
     >>> # ...train the model...
     >>>
     >>> # vectorise the dataframe
-    >>> t2v = Tab2Vec(tab_preprocessor, model)
+    >>> t2v = Tab2Vec(tab_preprocessor, model, device="cpu")
     >>> X_vec = t2v.transform(df_t2v)
     """
 

From f076867711b34d065b250fb8223adde29c0aa35a Mon Sep 17 00:00:00 2001
From: Javier <jrzaurin@gmail.com>
Date: Fri, 18 Oct 2024 23:02:58 +0100
Subject: [PATCH 08/24] Added tests for the new metrics. Still need to test the
 full functioning with the Trainer

---
 pytorch_widedeep/metrics.py                | 417 ++++++++++++++++++++-
 pytorch_widedeep/training/_base_trainer.py |  33 +-
 pytorch_widedeep/training/trainer.py       |  45 ++-
 tests/test_metrics/test_ranking_metrics.py | 188 ++++++++++
 4 files changed, 661 insertions(+), 22 deletions(-)
 create mode 100644 tests/test_metrics/test_ranking_metrics.py

diff --git a/pytorch_widedeep/metrics.py b/pytorch_widedeep/metrics.py
index a2947380..e4b871ed 100644
--- a/pytorch_widedeep/metrics.py
+++ b/pytorch_widedeep/metrics.py
@@ -2,7 +2,8 @@
 import torch
 from torchmetrics import Metric as TorchMetric
 
-from pytorch_widedeep.wdtypes import Dict, List, Union, Tensor
+from pytorch_widedeep.wdtypes import Dict, List, Union, Tensor, Optional
+from pytorch_widedeep.utils.general_utils import alias
 
 
 class Metric(object):
@@ -394,3 +395,417 @@ def __call__(self, y_pred: Tensor, y_true: Tensor) -> np.ndarray:
         y_true_avg = self.y_true_sum / self.num_examples
         self.denominator += ((y_true - y_true_avg) ** 2).sum().item()
         return np.array((1 - (self.numerator / self.denominator)))
+
+
+def reshape_1d_to_2d(tensor: Tensor, n_columns: int) -> Tensor:
+    if tensor.dim() != 1:
+        raise ValueError("Input tensor must be 1-dimensional")
+    if tensor.size(0) % n_columns != 0:
+        raise ValueError(
+            f"Tensor length ({tensor.size(0)}) must be divisible by n_columns ({n_columns})"
+        )
+    n_rows = tensor.size(0) // n_columns
+    return tensor.reshape(n_rows, n_columns)
+
+
+class NDCG_at_k(Metric):
+    r"""
+    Normalized Discounted Cumulative Gain (NDCG) at k.
+
+    Parameters
+    ----------
+    n_cols: int, default = 10
+        Number of columns in the input tensors. This parameter is neccessary
+        because the input tensors are reshaped to 2D tensors. n_cols is the
+        number of columns in the reshaped tensor. Alias for this parameter
+        are: 'n_items', 'n_items_per_query',
+        'n_items_per_id', 'n_items_per_user'
+    k: int, Optional, default = None
+        Number of top items to consider. It must be less than or equal to n_cols.
+        If is None, k will be equal to n_cols.
+    eps: float, default = 1e-8
+        Small value to avoid division by zero.
+
+    Examples
+    --------
+    >>> import torch
+    >>> from pytorch_widedeep.metrics import NDCG_at_k
+    >>>
+    >>> ndcg = NDCG_at_k(k=10)
+    >>> y_pred = torch.rand(100, 10)
+    >>> y_true = torch.randint(100, 10)
+    >>> score = ndcg(y_pred, y_true)
+    """
+
+    @alias(
+        "n_cols", ["n_items", "n_items_per_query", "n_items_per_id", "n_items_per_user"]
+    )
+    def __init__(self, n_cols: int = 10, k: Optional[int] = None, eps: float = 1e-8):
+        super(NDCG_at_k, self).__init__()
+
+        if k is not None and k > n_cols:
+            raise ValueError(
+                f"k must be less than or equal to n_cols. Got k: {k}, n_cols: {n_cols}"
+            )
+
+        self.n_cols = n_cols
+        self.k = k if k is not None else n_cols
+        self.eps = eps
+        self._name = f"ndcg@{k}"
+        self.reset()
+
+    def reset(self):
+        self.sum_ndcg = 0.0
+        self.count = 0
+
+    def __call__(self, y_pred: Tensor, y_true: Tensor) -> np.ndarray:
+        device = y_pred.device
+
+        y_pred_reshaped = reshape_1d_to_2d(y_pred, self.n_cols)
+        y_true_reshaped = reshape_1d_to_2d(y_true, self.n_cols)
+
+        batch_size = y_true_reshaped.shape[0]
+
+        _, top_k_indices = torch.topk(y_pred_reshaped, self.k, dim=1)
+        top_k_relevance = y_true_reshaped.gather(1, top_k_indices)
+        discounts = 1.0 / torch.log2(
+            torch.arange(2, top_k_relevance.shape[1] + 2, device=device)
+        )
+
+        dcg = (torch.pow(2, top_k_relevance) - 1) * discounts.unsqueeze(0)
+        dcg = dcg.sum(dim=1)
+
+        sorted_relevance, _ = torch.sort(y_true_reshaped, dim=1, descending=True)
+        ideal_relevance = sorted_relevance[:, : self.k]
+
+        idcg = (torch.pow(2, ideal_relevance) - 1) * discounts[
+            : ideal_relevance.shape[1]
+        ].unsqueeze(0)
+
+        idcg = idcg.sum(dim=1)
+        ndcg = dcg / (idcg + self.eps)
+
+        self.sum_ndcg += ndcg.sum().item()
+        self.count += batch_size
+
+        return np.array(self.sum_ndcg / max(self.count, 1))
+
+
+class BinaryNDCG_at_k(Metric):
+    r"""
+    Normalized Discounted Cumulative Gain (NDCG) at k for binary relevance.
+
+    Parameters
+    ----------
+    n_cols: int, default = 10
+        Number of columns in the input tensors. This parameter is neccessary
+        because the input tensors are reshaped to 2D tensors. n_cols is the
+        number of columns in the reshaped tensor. Alias for this parameter
+        are: 'n_items', 'n_items_per_query',
+        'n_items_per_id', 'n_items_per_user'
+    k: int, Optional, default = None
+        Number of top items to consider. It must be less than or equal to n_cols.
+        If is None, k will be equal to n_cols.
+    eps: float, default = 1e-8
+        Small value to avoid division by zero.
+
+    Examples
+    --------
+    >>> import torch
+    >>> from pytorch_widedeep.metrics import BinaryNDCG_at_k
+    >>>
+    >>> ndcg = BinaryNDCG_at_k(k=10)
+    >>> y_pred = torch.rand(100, 10)
+    >>> y_true = torch.randint(0, 2, (100, 10))
+    >>> score = ndcg(y_pred, y_true)
+    """
+
+    @alias(
+        "n_cols", ["n_items", "n_items_per_query", "n_items_per_id", "n_items_per_user"]
+    )
+    def __init__(self, n_cols: int = 10, k: Optional[int] = None, eps: float = 1e-8):
+        super(BinaryNDCG_at_k, self).__init__()
+
+        if k is not None and k > n_cols:
+            raise ValueError(
+                f"k must be less than or equal to n_cols. Got k: {k}, n_cols: {n_cols}"
+            )
+
+        self.n_cols = n_cols
+        self.k = k if k is not None else n_cols
+        self.eps = eps
+        self._name = f"binary_ndcg@{k}"
+        self.reset()
+
+    def reset(self):
+        self.sum_ndcg = 0.0
+        self.count = 0
+
+    def __call__(self, y_pred: Tensor, y_true: Tensor) -> np.ndarray:
+        device = y_pred.device
+
+        y_pred_reshaped = reshape_1d_to_2d(y_pred, self.n_cols)
+        y_true_reshaped = reshape_1d_to_2d(y_true, self.n_cols)
+
+        batch_size = y_pred_reshaped.shape[0]
+
+        _, top_k_indices = torch.topk(y_pred_reshaped, self.k, dim=1)
+        top_k_mask = torch.zeros_like(y_pred_reshaped, dtype=torch.bool).scatter_(
+            1, top_k_indices, 1
+        )
+
+        _discounts = 1.0 / torch.log2(
+            torch.arange(2, self.k + 2, device=device).float()
+        )
+        expanded_discounts = _discounts.repeat(1, batch_size)
+        discounts = torch.zeros_like(top_k_mask, dtype=torch.float)
+        discounts[top_k_mask] = expanded_discounts
+
+        dcg = (y_true_reshaped * top_k_mask * discounts).sum(dim=1)
+        n_relevant = torch.minimum(
+            y_true_reshaped.sum(dim=1), torch.tensor(self.k, device=device)
+        ).int()
+        ideal_discounts = 1.0 / torch.log2(
+            torch.arange(2, y_true_reshaped.shape[1] + 2, device=device).float()
+        )
+
+        idcg = torch.zeros(batch_size, device=device)
+        for i in range(batch_size):
+            idcg[i] = ideal_discounts[: n_relevant[i]].sum()
+
+        ndcg = dcg / (idcg + self.eps)
+
+        self.sum_ndcg += ndcg.sum().item()
+        self.count += batch_size
+
+        return np.array(self.sum_ndcg / max(self.count, 1))
+
+
+class MAP_at_k(Metric):
+    r"""
+    Mean Average Precision (MAP) at k.
+
+    Parameters
+    ----------
+    n_cols: int, default = 10
+        Number of columns in the input tensors. This parameter is neccessary
+        because the input tensors are reshaped to 2D tensors. n_cols is the
+        number of columns in the reshaped tensor. Alias for this parameter
+        are: 'n_items', 'n_items_per_query', 'n_items_per_id', 'n_items_per_user'
+    k: int, Optional, default = None
+        Number of top items to consider. It must be less than or equal to n_cols.
+        If is None, k will be equal to n_cols.
+
+    Examples
+    --------
+    >>> import torch
+    >>> from pytorch_widedeep.metrics import MAP_at_k
+    >>>
+    >>> map_at_k = MAP_at_k(k=10)
+    >>> y_pred = torch.rand(100, 10)
+    >>> y_true = torch.randint(0, 2, (100, 10))
+    >>> score = map_at_k(y_pred, y_true)
+    """
+
+    def __init__(self, n_cols: int = 10, k: Optional[int] = None):
+        super(MAP_at_k, self).__init__()
+
+        if k is not None and k > n_cols:
+            raise ValueError(
+                f"k must be less than or equal to n_cols. Got k: {k}, n_cols: {n_cols}"
+            )
+
+        self.n_cols = n_cols
+        self.k = k if k is not None else n_cols
+        self._name = f"map@{k}"
+        self.reset()
+
+    def reset(self):
+        self.sum_avg_precision = 0.0
+        self.count = 0
+
+    def __call__(self, y_pred: Tensor, y_true: Tensor) -> np.ndarray:
+
+        y_pred_reshaped = reshape_1d_to_2d(y_pred, self.n_cols)
+        y_true_reshaped = reshape_1d_to_2d(y_true, self.n_cols)
+
+        batch_size = y_pred_reshaped.shape[0]
+        _, top_k_indices = torch.topk(y_pred_reshaped, self.k, dim=1)
+        batch_relevance = y_true_reshaped.gather(1, top_k_indices)
+        cumsum_relevance = torch.cumsum(batch_relevance, dim=1)
+        precision_at_i = cumsum_relevance / torch.arange(
+            1, self.k + 1, device=y_pred_reshaped.device
+        ).float().unsqueeze(0)
+        avg_precision = (precision_at_i * batch_relevance).sum(dim=1) / torch.clamp(
+            y_true_reshaped.sum(dim=1), min=1
+        )
+        self.sum_avg_precision += avg_precision.sum().item()
+        self.count += batch_size
+        return np.array(self.sum_avg_precision / max(self.count, 1))
+
+
+class HitRatio_at_k(Metric):
+    r"""
+    Hit Ratio (HR) at k.
+
+    Parameters
+    ----------
+    n_cols: int, default = 10
+        Number of columns in the input tensors. This parameter is neccessary
+        because the input tensors are reshaped to 2D tensors. n_cols is the
+        number of columns in the reshaped tensor. Alias for this parameter
+        are: 'n_items', 'n_items_per_query', 'n_items_per_id', 'n_items_per_user'
+    k: int, Optional, default = None
+        Number of top items to consider. It must be less than or equal to n_cols.
+        If is None, k will be equal to n_cols.
+
+    Examples
+    --------
+    >>> import torch
+    >>> from pytorch_widedeep.metrics import HitRatio_at_k
+    >>>
+    >>> hr_at_k = HitRatio_at_k(k=10)
+    >>> y_pred = torch.rand(100, 10)
+    >>> y_true = torch.randint(0, 2, (100, 10))
+    >>> score = hr_at_k(y_pred, y_true
+    """
+
+    def __init__(self, n_cols: int = 10, k: Optional[int] = None):
+        super(HitRatio_at_k, self).__init__()
+
+        if k is not None and k > n_cols:
+            raise ValueError(
+                f"k must be less than or equal to n_cols. Got k: {k}, n_cols: {n_cols}"
+            )
+
+        self.n_cols = n_cols
+        self.k = k if k is not None else n_cols
+        self._name = f"hr@{k}"
+        self.reset()
+
+    def reset(self):
+        self.sum_hr = 0.0
+        self.count = 0
+
+    def __call__(self, y_pred: Tensor, y_true: Tensor) -> np.ndarray:
+        y_pred_reshaped = reshape_1d_to_2d(y_pred, self.n_cols)
+        y_true_reshaped = reshape_1d_to_2d(y_true, self.n_cols)
+        batch_size = y_pred_reshaped.shape[0]
+        _, top_k_indices = torch.topk(y_pred_reshaped, self.k, dim=1)
+        batch_relevance = y_true_reshaped.gather(1, top_k_indices)
+        hit = (batch_relevance.sum(dim=1) > 0).float()
+        self.sum_hr += hit.sum().item()
+        self.count += batch_size
+        return np.array(self.sum_hr / max(self.count, 1))
+
+
+class Precision_at_k(Metric):
+    r"""
+    Precision at k.
+
+    Parameters
+    ----------
+    n_cols: int, default = 10
+        Number of columns in the input tensors. This parameter is neccessary
+        because the input tensors are reshaped to 2D tensors. n_cols is the
+        number of columns in the reshaped tensor. Alias for this parameter
+        are: 'n_items', 'n_items_per_query',
+        'n_items_per_id', 'n_items_per
+    k: int, Optional, default = None
+        Number of top items to consider. It must be less than or equal to n_cols.
+        If is None, k will be equal to n_cols.
+
+    Examples
+    --------
+    >>> import torch
+    >>> from pytorch_widedeep.metrics import Precision_at_k
+    >>>
+    >>> prec_at_k = Precision_at_k(k=10)
+    >>> y_pred = torch.rand(100, 10)
+    >>> y_true = torch.randint(0, 2, (100, 10))
+    >>> score = prec_at_k(y_pred, y_true)
+    """
+
+    def __init__(self, n_cols: int = 10, k: Optional[int] = None):
+        super(Precision_at_k, self).__init__()
+
+        if k is not None and k > n_cols:
+            raise ValueError(
+                f"k must be less than or equal to n_cols. Got k: {k}, n_cols: {n_cols}"
+            )
+
+        self.n_cols = n_cols
+        self.k = k if k is not None else n_cols
+        self._name = f"precision@{k}"
+        self.reset()
+
+    def reset(self):
+        self.sum_precision = 0.0
+        self.count = 0
+
+    def __call__(self, y_pred: Tensor, y_true: Tensor) -> np.ndarray:
+        y_pred_reshaped = reshape_1d_to_2d(y_pred, self.n_cols)
+        y_true_reshaped = reshape_1d_to_2d(y_true, self.n_cols)
+        batch_size = y_pred_reshaped.shape[0]
+        _, top_k_indices = torch.topk(y_pred_reshaped, self.k, dim=1)
+        batch_relevance = y_true_reshaped.gather(1, top_k_indices)
+        precision = batch_relevance.sum(dim=1) / self.k
+        self.sum_precision += precision.sum().item()
+        self.count += batch_size
+        return np.array(self.sum_precision / max(self.count, 1))
+
+
+class Recall_at_k(Metric):
+    r"""
+    Recall at k.
+
+    Parameters
+    ----------
+    k: int, default = 10
+        Number of top items to consider.
+
+    Examples
+    --------
+    >>> import torch
+    >>> from pytorch_widedeep.metrics import Recall_at_k
+    >>>
+    >>> rec_at_k = Recall_at_k(k=10)
+    >>> y_pred = torch.rand(100, 10)
+    >>> y_true = torch.randint(0, 2, (100, 10))
+    >>> score = rec_at_k(y_pred, y_true)
+    """
+
+    def __init__(self, n_cols: int = 10, k: Optional[int] = None):
+        super(Recall_at_k, self).__init__()
+
+        if k is not None and k > n_cols:
+            raise ValueError(
+                f"k must be less than or equal to n_cols. Got k: {k}, n_cols: {n_cols}"
+            )
+
+        self.n_cols = n_cols
+        self.k = k if k is not None else n_cols
+        self._name = f"recall@{k}"
+        self.reset()
+
+    def reset(self):
+        self.sum_recall = 0.0
+        self.count = 0
+
+    def __call__(self, y_pred: Tensor, y_true: Tensor) -> np.ndarray:
+        y_pred_reshaped = reshape_1d_to_2d(y_pred, self.n_cols)
+        y_true_reshaped = reshape_1d_to_2d(y_true, self.n_cols)
+        batch_size = y_pred_reshaped.shape[0]
+        _, top_k_indices = torch.topk(y_pred_reshaped, self.k, dim=1)
+        batch_relevance = y_true_reshaped.gather(1, top_k_indices)
+        recall = batch_relevance.sum(dim=1) / torch.clamp(
+            y_true_reshaped.sum(dim=1), min=1
+        )
+        self.sum_recall += recall.sum().item()
+        self.count += batch_size
+        return np.array(self.sum_recall / max(self.count, 1))
+
+
+RankingMetrics = Union[
+    BinaryNDCG_at_k, MAP_at_k, HitRatio_at_k, Precision_at_k, Recall_at_k
+]
diff --git a/pytorch_widedeep/training/_base_trainer.py b/pytorch_widedeep/training/_base_trainer.py
index 45cb2e9e..21d8985b 100644
--- a/pytorch_widedeep/training/_base_trainer.py
+++ b/pytorch_widedeep/training/_base_trainer.py
@@ -10,7 +10,7 @@
 from torchmetrics import Metric as TorchMetric
 from torch.optim.lr_scheduler import ReduceLROnPlateau
 
-from pytorch_widedeep.metrics import Metric, MultipleMetrics
+from pytorch_widedeep.metrics import Metric, RankingMetrics, MultipleMetrics
 from pytorch_widedeep.wdtypes import (
     Any,
     Dict,
@@ -285,13 +285,14 @@ def _set_transforms(transforms):
         else:
             return None
 
-    # this needs type fixing to adjust for the fact that the main class can
-    # take an 'object', a non-instastiated Class, so, should be something like:
-    # callbacks: Optional[List[Union[object, Callback]]] in all places
+    # TO DO: this needs type fixing to adjust for the fact that the main class
+    # can take an 'object', a non-instastiated Class, so, should be something
+    # like: callbacks: Optional[List[Union[object, Callback]]] in all places
     def _set_callbacks_and_metrics(
         self,
         callbacks: Any,
-        metrics: Any,
+        metrics: Any,  # Union[List[Metric], List[TorchMetric]],
+        eval_metrics: Optional[Any] = None,  # Union[List[Metric], List[TorchMetric]],
     ):
         self.callbacks: List = [History(), LRShedulerCallback()]
         if callbacks is not None:
@@ -301,9 +302,31 @@ def _set_callbacks_and_metrics(
                 self.callbacks.append(callback)
         if metrics is not None:
             self.metric = MultipleMetrics(metrics)
+            # assert that no ranking metric is used during training assertion
+            # here so all metrics are instanciated
+            assert not any(
+                [isinstance(m, RankingMetrics) for m in self.metric._metrics]  # type: ignore[arg-type, misc]
+            ), "Currently, ranking metrics are not supported during training"
+
             self.callbacks += [MetricCallback(self.metric)]
         else:
             self.metric = None
+        if eval_metrics is not None:
+            self.eval_metric = MultipleMetrics(eval_metrics)
+            # assert that if any of the metrics is a ranking metric, all metrics
+            # must be ranking metrics
+            if any(
+                [isinstance(m, RankingMetrics) for m in self.eval_metric._metrics]  # type: ignore[arg-type, misc]
+            ):
+                assert all(
+                    [isinstance(m, RankingMetrics) for m in self.eval_metric._metrics]  # type: ignore[arg-type, misc]
+                ), (
+                    "All eval metrics must be ranking metrics if any of the eval"
+                    " metrics is a ranking metric"
+                )
+            self.callbacks += [MetricCallback(self.eval_metric)]
+        else:
+            self.eval_metric = None
         self.callback_container = CallbackContainer(self.callbacks)
         self.callback_container.set_model(self.model)
         self.callback_container.set_trainer(self)
diff --git a/pytorch_widedeep/training/trainer.py b/pytorch_widedeep/training/trainer.py
index 8ca7d459..cff69bfd 100644
--- a/pytorch_widedeep/training/trainer.py
+++ b/pytorch_widedeep/training/trainer.py
@@ -228,6 +228,7 @@ class Trainer(BaseTrainer):
         "objective",
         ["loss_function", "loss_fn", "loss", "cost_function", "cost_fn", "cost"],
     )
+    @alias("metrics", ["train_metrics"])
     def __init__(
         self,
         model: WideDeep,
@@ -245,6 +246,7 @@ def __init__(
         transforms: Optional[List[Transforms]] = None,
         callbacks: Optional[List[Callback]] = None,
         metrics: Optional[Union[List[Metric], List[TorchMetric]]] = None,
+        eval_metrics: Optional[Union[List[Metric], List[TorchMetric]]] = None,
         verbose: int = 1,
         seed: int = 1,
         **kwargs,
@@ -259,6 +261,7 @@ def __init__(
             transforms=transforms,
             callbacks=callbacks,
             metrics=metrics,
+            eval_metrics=eval_metrics,
             verbose=verbose,
             seed=seed,
             **kwargs,
@@ -600,7 +603,7 @@ def predict_uncertainty(  # type: ignore[return]
         X_img: Optional[Union[np.ndarray, List[np.ndarray]]] = None,
         X_test: Optional[Dict[str, Union[np.ndarray, List[np.ndarray]]]] = None,
         batch_size: Optional[int] = None,
-        uncertainty_granularity=1000,
+        uncertainty_granularity: int = 1000,
     ) -> np.ndarray:
         r"""Returns the predicted ucnertainty of the model for the test dataset
         using a Monte Carlo method during which dropout layers are activated
@@ -930,10 +933,10 @@ def _train_step(
 
         if self.model.is_tabnet:
             loss = self.loss_fn(y_pred[0], y) - self.lambda_sparse * y_pred[1]
-            score = self._get_score(y_pred[0], y)
+            score = self._get_score(y_pred[0], y, is_train=True)
         else:
             loss = self.loss_fn(y_pred, y)
-            score = self._get_score(y_pred, y)
+            score = self._get_score(y_pred, y, is_train=True)
 
         loss.backward()
         self.optimizer.step()
@@ -967,9 +970,9 @@ def _eval_step(
             y_pred = self.model(X)
             if self.model.is_tabnet:
                 loss = self.loss_fn(y_pred[0], y) - self.lambda_sparse * y_pred[1]
-                score = self._get_score(y_pred[0], y)
+                score = self._get_score(y_pred[0], y, is_train=False)
             else:
-                score = self._get_score(y_pred, y)
+                score = self._get_score(y_pred, y, is_train=False)
                 loss = self.loss_fn(y_pred, y)
 
             self.valid_running_loss += loss.item()
@@ -978,20 +981,30 @@ def _eval_step(
         self.model.train()
         return score, avg_loss
 
-    def _get_score(self, y_pred, y):
-        if self.metric is not None:
+    def _get_score(
+        self, y_pred: Tensor, y: Tensor, is_train: bool
+    ) -> Optional[Dict[str, float]]:
+
+        score = None
+        metric = None
+
+        if hasattr(self, "metric") and not hasattr(self, "eval_metric"):
+            metric = self.metric
+        elif hasattr(self, "metric") and hasattr(self, "eval_metric"):
+            metric = self.metric if is_train else self.eval_metric
+        elif not hasattr(self, "metric") and hasattr(self, "eval_metric"):
+            metric = None if is_train else self.eval_metric
+
+        if metric is not None:
             if self.method == "regression":
-                score = self.metric(y_pred, y)
+                score = metric(y_pred, y)
             if self.method == "binary":
-                score = self.metric(torch.sigmoid(y_pred), y)
+                score = metric(torch.sigmoid(y_pred), y)
             if self.method == "qregression":
-                score = self.metric(y_pred, y)
+                score = metric(y_pred, y)
             if self.method == "multiclass":
-                score = self.metric(F.softmax(y_pred, dim=1), y)
-            # TO DO: handle multitarget
-            return score
-        else:
-            return None
+                score = metric(F.softmax(y_pred, dim=1), y)
+        return score
 
     def _predict(  # type: ignore[override, return]  # noqa: C901
         self,
@@ -1001,7 +1014,7 @@ def _predict(  # type: ignore[override, return]  # noqa: C901
         X_img: Optional[Union[np.ndarray, List[np.ndarray]]] = None,
         X_test: Optional[Dict[str, Union[np.ndarray, List[np.ndarray]]]] = None,
         batch_size: Optional[int] = None,
-        uncertainty_granularity=1000,
+        uncertainty_granularity: int = 1000,
         uncertainty: bool = False,
     ) -> List:
         r"""Private method to avoid code repetition in predict and
diff --git a/tests/test_metrics/test_ranking_metrics.py b/tests/test_metrics/test_ranking_metrics.py
new file mode 100644
index 00000000..b08ecb4a
--- /dev/null
+++ b/tests/test_metrics/test_ranking_metrics.py
@@ -0,0 +1,188 @@
+# Thanks claude 3.5 for the test cases.
+import numpy as np
+import torch
+import pytest
+
+from pytorch_widedeep.metrics import (
+    MAP_at_k,
+    NDCG_at_k,
+    Recall_at_k,
+    HitRatio_at_k,
+    Precision_at_k,
+    BinaryNDCG_at_k,
+)
+
+
+@pytest.fixture
+def setup_data():
+    y_pred = torch.tensor(
+        [
+            [0.7, 0.9, 0.5, 0.8, 0.6],
+            [0.3, 0.1, 0.5, 0.2, 0.4],
+        ]
+    )
+    y_true = torch.tensor(
+        [
+            [0, 1, 0, 1, 1],
+            [1, 1, 0, 0, 0],
+        ]
+    )
+    return y_pred, y_true
+
+
+@pytest.fixture
+def setup_data_ndcg():
+    y_pred = torch.tensor(
+        [
+            [0.7, 0.9, 0.5, 0.8, 0.6],
+            [0.3, 0.1, 0.5, 0.2, 0.4],
+        ]
+    )
+    # Using relevance scores from 0 to 3
+    y_true = torch.tensor(
+        [
+            [1, 3, 1, 2, 0],
+            [3, 1, 0, 2, 1],
+        ]
+    )
+    return y_pred, y_true
+
+
+def test_binary_ndcg_at_k(setup_data):
+    y_pred, y_true = setup_data
+    binary_ndcg = BinaryNDCG_at_k(n_cols=5, k=3)
+    result = binary_ndcg(y_pred.flatten(), y_true.flatten())
+    expected = np.array(0.5719)
+    np.testing.assert_almost_equal(result, expected, decimal=4)
+
+
+def test_map_at_k(setup_data):
+    y_pred, y_true = setup_data
+    map_at_k = MAP_at_k(n_cols=5, k=3)
+    result = map_at_k(y_pred.flatten(), y_true.flatten())
+    expected = np.array(0.4166)
+    np.testing.assert_almost_equal(result, expected, decimal=4)
+
+
+def test_hit_ratio_at_k(setup_data):
+    y_pred, y_true = setup_data
+    hr_at_k = HitRatio_at_k(n_cols=5, k=3)
+    result = hr_at_k(y_pred.flatten(), y_true.flatten())
+    expected = np.array(1.0)
+    np.testing.assert_almost_equal(result, expected, decimal=4)
+
+
+def test_precision_at_k(setup_data):
+    y_pred, y_true = setup_data
+    prec_at_k = Precision_at_k(n_cols=5, k=3)
+    result = prec_at_k(y_pred.flatten(), y_true.flatten())
+    expected = np.array(0.5)
+    np.testing.assert_almost_equal(result, expected, decimal=4)
+
+
+def test_recall_at_k(setup_data):
+    y_pred, y_true = setup_data
+    rec_at_k = Recall_at_k(n_cols=5, k=3)
+    result = rec_at_k(y_pred.flatten(), y_true.flatten())
+    expected = np.array(0.5833)
+    np.testing.assert_almost_equal(result, expected, decimal=4)
+
+
+def test_edge_cases_all_relevant_items():
+    # Test with all relevant items
+    y_pred = torch.tensor([0.9, 0.8, 0.7, 0.6, 0.5])
+    y_true = torch.tensor([1, 1, 1, 1, 1])
+
+    ndcg = NDCG_at_k(n_cols=5, k=3)
+    assert ndcg(y_pred, y_true) == 1.0
+
+    binary_ndcg = BinaryNDCG_at_k(n_cols=5, k=3)
+    assert binary_ndcg(y_pred, y_true) == 1.0
+
+    map_at_k = MAP_at_k(n_cols=5, k=3)
+    assert np.isclose(map_at_k(y_pred, y_true), 0.6)
+
+    hr_at_k = HitRatio_at_k(n_cols=5, k=3)
+    assert hr_at_k(y_pred, y_true) == 1.0
+
+    prec_at_k = Precision_at_k(n_cols=5, k=3)
+    assert prec_at_k(y_pred, y_true) == 1.0
+
+    rec_at_k = Recall_at_k(n_cols=5, k=3)
+    assert np.isclose(rec_at_k(y_pred, y_true), 0.6)
+
+
+def test_edge_cases_no_relevant_items():
+
+    # Test with no relevant items
+    y_pred = torch.tensor([0.9, 0.8, 0.7, 0.6, 0.5])
+    y_true = torch.tensor([0, 0, 0, 0, 0])
+
+    ndcg = NDCG_at_k(n_cols=5, k=3)
+    assert ndcg(y_pred, y_true) == 0.0
+
+    binary_ndcg = BinaryNDCG_at_k(n_cols=5, k=3)
+    assert binary_ndcg(y_pred, y_true) == 0.0
+
+    map_at_k = MAP_at_k(n_cols=5, k=3)
+    assert map_at_k(y_pred, y_true) == 0.0
+
+    hr_at_k = HitRatio_at_k(n_cols=5, k=3)
+    assert hr_at_k(y_pred, y_true) == 0.0
+
+    prec_at_k = Precision_at_k(n_cols=5, k=3)
+    assert prec_at_k(y_pred, y_true) == 0.0
+
+    rec_at_k = Recall_at_k(n_cols=5, k=3)
+    assert rec_at_k(y_pred, y_true) == 0.0
+
+
+def test_k_greater_than_n_cols():
+    with pytest.raises(ValueError):
+        NDCG_at_k(n_cols=5, k=10)
+    with pytest.raises(ValueError):
+        BinaryNDCG_at_k(n_cols=5, k=10)
+    with pytest.raises(ValueError):
+        MAP_at_k(n_cols=5, k=10)
+    with pytest.raises(ValueError):
+        HitRatio_at_k(n_cols=5, k=10)
+    with pytest.raises(ValueError):
+        Precision_at_k(n_cols=5, k=10)
+    with pytest.raises(ValueError):
+        Recall_at_k(n_cols=5, k=10)
+
+
+def test_ndcg_at_k(setup_data_ndcg):
+    y_pred, y_true = setup_data_ndcg
+    ndcg = NDCG_at_k(n_cols=5, k=3)
+    result = ndcg(y_pred.flatten(), y_true.flatten())
+    expected = np.array(0.7198)
+    np.testing.assert_almost_equal(result, expected, decimal=4)
+
+
+def test_ndcg_at_k_edge_cases():
+    # Test with non-decreasing ranking
+    y_pred = torch.tensor([0.5, 0.6, 0.7, 0.8, 0.9])
+    y_true = torch.tensor([0, 1, 2, 3, 4])
+
+    ndcg = NDCG_at_k(n_cols=5, k=3)
+    result = ndcg(y_pred, y_true)
+    assert result == 1.0
+
+    # Test with non-increasing ranking
+    ndcg = NDCG_at_k(n_cols=5, k=3)
+    y_pred = torch.tensor([0.9, 0.8, 0.7, 0.6, 0.5])
+    result = ndcg(y_pred, y_true)
+    expected = np.array(0.1019)
+    np.testing.assert_almost_equal(result, expected, decimal=4)
+
+    # Test with all zero relevance
+    ndcg = NDCG_at_k(n_cols=5, k=3)
+    y_true = torch.tensor([0, 0, 0, 0, 0])
+    result = ndcg(y_pred, y_true)
+    assert result == 0.0
+
+
+def test_ndcg_at_k_k_greater_than_n_cols():
+    with pytest.raises(ValueError):
+        NDCG_at_k(n_cols=5, k=10)

From d32397a36c2049ba0817f477bb1a9f0a000f4f39 Mon Sep 17 00:00:00 2001
From: Javier <jrzaurin@gmail.com>
Date: Sat, 19 Oct 2024 16:15:48 +0100
Subject: [PATCH 09/24] Added explanation to the metrics and a bit of
 refactoring

---
 pytorch_widedeep/metrics.py                |  76 ++++----
 pytorch_widedeep/training/_base_trainer.py |  35 ++--
 pytorch_widedeep/training/trainer.py       | 192 ++++++++++++++-------
 tests/test_metrics/test_ranking_metrics.py |  42 +++++
 4 files changed, 224 insertions(+), 121 deletions(-)

diff --git a/pytorch_widedeep/metrics.py b/pytorch_widedeep/metrics.py
index e4b871ed..5286952b 100644
--- a/pytorch_widedeep/metrics.py
+++ b/pytorch_widedeep/metrics.py
@@ -461,13 +461,13 @@ def reset(self):
     def __call__(self, y_pred: Tensor, y_true: Tensor) -> np.ndarray:
         device = y_pred.device
 
-        y_pred_reshaped = reshape_1d_to_2d(y_pred, self.n_cols)
-        y_true_reshaped = reshape_1d_to_2d(y_true, self.n_cols)
+        y_pred_2d = reshape_1d_to_2d(y_pred, self.n_cols)
+        y_true_2d = reshape_1d_to_2d(y_true, self.n_cols)
 
-        batch_size = y_true_reshaped.shape[0]
+        batch_size = y_true_2d.shape[0]
 
-        _, top_k_indices = torch.topk(y_pred_reshaped, self.k, dim=1)
-        top_k_relevance = y_true_reshaped.gather(1, top_k_indices)
+        _, top_k_indices = torch.topk(y_pred_2d, self.k, dim=1)
+        top_k_relevance = y_true_2d.gather(1, top_k_indices)
         discounts = 1.0 / torch.log2(
             torch.arange(2, top_k_relevance.shape[1] + 2, device=device)
         )
@@ -475,7 +475,7 @@ def __call__(self, y_pred: Tensor, y_true: Tensor) -> np.ndarray:
         dcg = (torch.pow(2, top_k_relevance) - 1) * discounts.unsqueeze(0)
         dcg = dcg.sum(dim=1)
 
-        sorted_relevance, _ = torch.sort(y_true_reshaped, dim=1, descending=True)
+        sorted_relevance, _ = torch.sort(y_true_2d, dim=1, descending=True)
         ideal_relevance = sorted_relevance[:, : self.k]
 
         idcg = (torch.pow(2, ideal_relevance) - 1) * discounts[
@@ -544,13 +544,13 @@ def reset(self):
     def __call__(self, y_pred: Tensor, y_true: Tensor) -> np.ndarray:
         device = y_pred.device
 
-        y_pred_reshaped = reshape_1d_to_2d(y_pred, self.n_cols)
-        y_true_reshaped = reshape_1d_to_2d(y_true, self.n_cols)
+        y_pred_2d = reshape_1d_to_2d(y_pred, self.n_cols)
+        y_true_2d = reshape_1d_to_2d(y_true, self.n_cols)
 
-        batch_size = y_pred_reshaped.shape[0]
+        batch_size = y_pred_2d.shape[0]
 
-        _, top_k_indices = torch.topk(y_pred_reshaped, self.k, dim=1)
-        top_k_mask = torch.zeros_like(y_pred_reshaped, dtype=torch.bool).scatter_(
+        _, top_k_indices = torch.topk(y_pred_2d, self.k, dim=1)
+        top_k_mask = torch.zeros_like(y_pred_2d, dtype=torch.bool).scatter_(
             1, top_k_indices, 1
         )
 
@@ -561,12 +561,12 @@ def __call__(self, y_pred: Tensor, y_true: Tensor) -> np.ndarray:
         discounts = torch.zeros_like(top_k_mask, dtype=torch.float)
         discounts[top_k_mask] = expanded_discounts
 
-        dcg = (y_true_reshaped * top_k_mask * discounts).sum(dim=1)
+        dcg = (y_true_2d * top_k_mask * discounts).sum(dim=1)
         n_relevant = torch.minimum(
-            y_true_reshaped.sum(dim=1), torch.tensor(self.k, device=device)
+            y_true_2d.sum(dim=1), torch.tensor(self.k, device=device)
         ).int()
         ideal_discounts = 1.0 / torch.log2(
-            torch.arange(2, y_true_reshaped.shape[1] + 2, device=device).float()
+            torch.arange(2, y_true_2d.shape[1] + 2, device=device).float()
         )
 
         idcg = torch.zeros(batch_size, device=device)
@@ -626,18 +626,18 @@ def reset(self):
 
     def __call__(self, y_pred: Tensor, y_true: Tensor) -> np.ndarray:
 
-        y_pred_reshaped = reshape_1d_to_2d(y_pred, self.n_cols)
-        y_true_reshaped = reshape_1d_to_2d(y_true, self.n_cols)
+        y_pred_2d = reshape_1d_to_2d(y_pred, self.n_cols)
+        y_true_2d = reshape_1d_to_2d(y_true, self.n_cols)
 
-        batch_size = y_pred_reshaped.shape[0]
-        _, top_k_indices = torch.topk(y_pred_reshaped, self.k, dim=1)
-        batch_relevance = y_true_reshaped.gather(1, top_k_indices)
+        batch_size = y_pred_2d.shape[0]
+        _, top_k_indices = torch.topk(y_pred_2d, self.k, dim=1)
+        batch_relevance = y_true_2d.gather(1, top_k_indices)
         cumsum_relevance = torch.cumsum(batch_relevance, dim=1)
         precision_at_i = cumsum_relevance / torch.arange(
-            1, self.k + 1, device=y_pred_reshaped.device
+            1, self.k + 1, device=y_pred_2d.device
         ).float().unsqueeze(0)
         avg_precision = (precision_at_i * batch_relevance).sum(dim=1) / torch.clamp(
-            y_true_reshaped.sum(dim=1), min=1
+            y_true_2d.sum(dim=1), min=1
         )
         self.sum_avg_precision += avg_precision.sum().item()
         self.count += batch_size
@@ -688,11 +688,11 @@ def reset(self):
         self.count = 0
 
     def __call__(self, y_pred: Tensor, y_true: Tensor) -> np.ndarray:
-        y_pred_reshaped = reshape_1d_to_2d(y_pred, self.n_cols)
-        y_true_reshaped = reshape_1d_to_2d(y_true, self.n_cols)
-        batch_size = y_pred_reshaped.shape[0]
-        _, top_k_indices = torch.topk(y_pred_reshaped, self.k, dim=1)
-        batch_relevance = y_true_reshaped.gather(1, top_k_indices)
+        y_pred_2d = reshape_1d_to_2d(y_pred, self.n_cols)
+        y_true_2d = reshape_1d_to_2d(y_true, self.n_cols)
+        batch_size = y_pred_2d.shape[0]
+        _, top_k_indices = torch.topk(y_pred_2d, self.k, dim=1)
+        batch_relevance = y_true_2d.gather(1, top_k_indices)
         hit = (batch_relevance.sum(dim=1) > 0).float()
         self.sum_hr += hit.sum().item()
         self.count += batch_size
@@ -744,11 +744,11 @@ def reset(self):
         self.count = 0
 
     def __call__(self, y_pred: Tensor, y_true: Tensor) -> np.ndarray:
-        y_pred_reshaped = reshape_1d_to_2d(y_pred, self.n_cols)
-        y_true_reshaped = reshape_1d_to_2d(y_true, self.n_cols)
-        batch_size = y_pred_reshaped.shape[0]
-        _, top_k_indices = torch.topk(y_pred_reshaped, self.k, dim=1)
-        batch_relevance = y_true_reshaped.gather(1, top_k_indices)
+        y_pred_2d = reshape_1d_to_2d(y_pred, self.n_cols)
+        y_true_2d = reshape_1d_to_2d(y_true, self.n_cols)
+        batch_size = y_pred_2d.shape[0]
+        _, top_k_indices = torch.topk(y_pred_2d, self.k, dim=1)
+        batch_relevance = y_true_2d.gather(1, top_k_indices)
         precision = batch_relevance.sum(dim=1) / self.k
         self.sum_precision += precision.sum().item()
         self.count += batch_size
@@ -793,14 +793,12 @@ def reset(self):
         self.count = 0
 
     def __call__(self, y_pred: Tensor, y_true: Tensor) -> np.ndarray:
-        y_pred_reshaped = reshape_1d_to_2d(y_pred, self.n_cols)
-        y_true_reshaped = reshape_1d_to_2d(y_true, self.n_cols)
-        batch_size = y_pred_reshaped.shape[0]
-        _, top_k_indices = torch.topk(y_pred_reshaped, self.k, dim=1)
-        batch_relevance = y_true_reshaped.gather(1, top_k_indices)
-        recall = batch_relevance.sum(dim=1) / torch.clamp(
-            y_true_reshaped.sum(dim=1), min=1
-        )
+        y_pred_2d = reshape_1d_to_2d(y_pred, self.n_cols)
+        y_true_2d = reshape_1d_to_2d(y_true, self.n_cols)
+        batch_size = y_pred_2d.shape[0]
+        _, top_k_indices = torch.topk(y_pred_2d, self.k, dim=1)
+        batch_relevance = y_true_2d.gather(1, top_k_indices)
+        recall = batch_relevance.sum(dim=1) / torch.clamp(y_true_2d.sum(dim=1), min=1)
         self.sum_recall += recall.sum().item()
         self.count += batch_size
         return np.array(self.sum_recall / max(self.count, 1))
diff --git a/pytorch_widedeep/training/_base_trainer.py b/pytorch_widedeep/training/_base_trainer.py
index 21d8985b..bf2c3875 100644
--- a/pytorch_widedeep/training/_base_trainer.py
+++ b/pytorch_widedeep/training/_base_trainer.py
@@ -302,27 +302,32 @@ def _set_callbacks_and_metrics(
                 self.callbacks.append(callback)
         if metrics is not None:
             self.metric = MultipleMetrics(metrics)
-            # assert that no ranking metric is used during training assertion
-            # here so all metrics are instanciated
-            assert not any(
-                [isinstance(m, RankingMetrics) for m in self.metric._metrics]  # type: ignore[arg-type, misc]
-            ), "Currently, ranking metrics are not supported during training"
-
+            if (
+                any(
+                    [isinstance(m, RankingMetrics) for m in self.metric._metrics]  # type: ignore[arg-type, misc]
+                )
+                and self.verbose
+            ):
+                UserWarning(
+                    "There are ranking metrics in the 'metrics' list. The implementation "
+                    "in this library requires that all query or user ids must have the "
+                    "same number of entries or items."
+                )
             self.callbacks += [MetricCallback(self.metric)]
         else:
             self.metric = None
         if eval_metrics is not None:
             self.eval_metric = MultipleMetrics(eval_metrics)
-            # assert that if any of the metrics is a ranking metric, all metrics
-            # must be ranking metrics
-            if any(
-                [isinstance(m, RankingMetrics) for m in self.eval_metric._metrics]  # type: ignore[arg-type, misc]
-            ):
-                assert all(
+            if (
+                any(
                     [isinstance(m, RankingMetrics) for m in self.eval_metric._metrics]  # type: ignore[arg-type, misc]
-                ), (
-                    "All eval metrics must be ranking metrics if any of the eval"
-                    " metrics is a ranking metric"
+                )
+                and self.verbose
+            ):
+                UserWarning(
+                    "There are ranking metrics in the 'eval_metric' list. The implementation "
+                    "in this library requires that all query or user ids must have the "
+                    "same number of entries or items."
                 )
             self.callbacks += [MetricCallback(self.eval_metric)]
         else:
diff --git a/pytorch_widedeep/training/trainer.py b/pytorch_widedeep/training/trainer.py
index cff69bfd..7400409b 100644
--- a/pytorch_widedeep/training/trainer.py
+++ b/pytorch_widedeep/training/trainer.py
@@ -12,8 +12,10 @@
 from pytorch_widedeep.losses import ZILNLoss
 from pytorch_widedeep.metrics import Metric
 from pytorch_widedeep.wdtypes import (
+    Any,
     Dict,
     List,
+    Tuple,
     Union,
     Tensor,
     Literal,
@@ -268,6 +270,8 @@ def __init__(
         )
 
     @alias("finetune", ["warmup"])
+    @alias("train_dataloader", ["custom_dataloader"])
+    @alias("eval_dataloader", ["custom_eval_dataloader"])
     def fit(  # noqa: C901
         self,
         X_wide: Optional[np.ndarray] = None,
@@ -281,7 +285,8 @@ def fit(  # noqa: C901
         n_epochs: int = 1,
         validation_freq: int = 1,
         batch_size: int = 32,
-        custom_dataloader: Optional[DataLoader] = None,
+        train_dataloader: Optional[DataLoader] = None,
+        eval_dataloader: Optional[DataLoader] = None,
         feature_importance_sample_size: Optional[int] = None,
         finetune: bool = False,
         **kwargs,
@@ -418,11 +423,8 @@ def fit(  # noqa: C901
         [Examples](https://github.com/jrzaurin/pytorch-widedeep/tree/master/examples)
         folder in the repo
         """
-
         dataloader_args, finetune_args = self._extract_kwargs(kwargs)
-
         self.batch_size = batch_size
-
         train_set, eval_set = wd_train_val_split(
             self.seed,
             self.method,  # type: ignore
@@ -436,35 +438,18 @@ def fit(  # noqa: C901
             target,
             self.transforms,
         )
-        if custom_dataloader is not None:
-            # make sure is callable (and HAS to be an subclass of DataLoader)
-            assert isinstance(custom_dataloader, type)
-            train_loader = custom_dataloader(  # type: ignore[misc]
-                dataset=train_set,
-                batch_size=batch_size,
-                num_workers=self.num_workers,
-                **dataloader_args,
-            )
-        else:
-            train_loader = DataLoader(
-                dataset=train_set,
-                batch_size=batch_size,
-                num_workers=self.num_workers,
-                **dataloader_args,
-            )
-        train_steps = len(train_loader)
-        if eval_set is not None:
-            eval_loader = DataLoader(
-                dataset=eval_set,
-                batch_size=batch_size,
-                num_workers=self.num_workers,
-                shuffle=False,
-            )
-            eval_steps = len(eval_loader)
+        train_loader, eval_loader = self._get_dataloaders(
+            train_dataloader,
+            eval_dataloader,
+            train_set,
+            eval_set,
+            batch_size,
+            dataloader_args,
+        )
 
         if finetune:
             self.with_finetuning: bool = True
-            self._finetune(train_loader, **finetune_args)
+            self._do_finetune(train_loader, **finetune_args)
             if self.verbose:
                 print(
                     "Fine-tuning (or warmup) of individual components completed. "
@@ -474,54 +459,32 @@ def fit(  # noqa: C901
             self.with_finetuning = False
 
         self.callback_container.on_train_begin(
-            {"batch_size": batch_size, "train_steps": train_steps, "n_epochs": n_epochs}
+            {
+                "batch_size": batch_size,
+                "train_steps": len(train_loader),
+                "n_epochs": n_epochs,
+            }
         )
         for epoch in range(n_epochs):
-            epoch_logs: Dict[str, float] = {}
-            self.callback_container.on_epoch_begin(epoch, logs=epoch_logs)
-
-            self.train_running_loss = 0.0
-            with trange(train_steps, disable=self.verbose != 1) as t:
-                for batch_idx, (data, targett) in zip(t, train_loader):
-                    t.set_description("epoch %i" % (epoch + 1))
-                    train_score, train_loss = self._train_step(data, targett, batch_idx)
-                    print_loss_and_metric(t, train_loss, train_score)
-                    self.callback_container.on_batch_end(batch=batch_idx)
-            epoch_logs = save_epoch_logs(epoch_logs, train_loss, train_score, "train")
-
-            on_epoch_end_metric = None
-            if eval_set is not None and epoch % validation_freq == (
+            epoch_logs = self._train_epoch(train_loader, epoch)
+            if eval_loader is not None and epoch % validation_freq == (
                 validation_freq - 1
             ):
-                self.callback_container.on_eval_begin()
-                self.valid_running_loss = 0.0
-                with trange(eval_steps, disable=self.verbose != 1) as v:
-                    for i, (data, targett) in zip(v, eval_loader):
-                        v.set_description("valid")
-                        val_score, val_loss = self._eval_step(data, targett, i)
-                        print_loss_and_metric(v, val_loss, val_score)
-                epoch_logs = save_epoch_logs(epoch_logs, val_loss, val_score, "val")
-
-                if self.reducelronplateau:
-                    if self.reducelronplateau_criterion == "loss":
-                        on_epoch_end_metric = val_loss
-                    else:
-                        on_epoch_end_metric = val_score[
-                            self.reducelronplateau_criterion
-                        ]
+                epoch_logs, on_epoch_end_metric = self._eval_epoch(
+                    eval_loader, epoch_logs
+                )
             else:
+                on_epoch_end_metric = None
                 if self.reducelronplateau:
                     raise NotImplementedError(
                         "ReduceLROnPlateau scheduler can be used only with validation data."
                     )
             self.callback_container.on_epoch_end(epoch, epoch_logs, on_epoch_end_metric)
-
             if self.early_stop:
                 # self.callback_container.on_train_end(epoch_logs)
                 break
 
         self.callback_container.on_train_end(epoch_logs)
-
         if feature_importance_sample_size is not None:
             self.feature_importance = FeatureImportance(
                 self.device, feature_importance_sample_size
@@ -817,9 +780,55 @@ def save(
             with open(Path(path) / "feature_importance.json", "w") as fi:
                 json.dump(self.feature_importance, fi)
 
+    def _get_dataloaders(
+        self,
+        train_dataloader: Optional[DataLoader],
+        eval_dataloader: Optional[DataLoader],
+        train_set: WideDeepDataset,
+        eval_set: Optional[WideDeepDataset],
+        batch_size: int,
+        dataloader_args: Dict[str, Any],
+    ) -> Tuple[DataLoader, Optional[DataLoader]]:
+        if train_dataloader is not None:
+            # make sure is callable (and HAS to be an subclass of DataLoader)
+            assert isinstance(train_dataloader, type)
+            train_loader = train_dataloader(  # type: ignore[misc]
+                dataset=train_set,
+                batch_size=batch_size,
+                num_workers=self.num_workers,
+                **dataloader_args,
+            )
+        else:
+            train_loader = DataLoader(
+                dataset=train_set,
+                batch_size=batch_size,
+                num_workers=self.num_workers,
+                **dataloader_args,
+            )
+
+        eval_loader = None
+        if eval_set is not None:
+            if eval_dataloader is not None:
+                assert isinstance(eval_dataloader, type)
+                eval_loader = eval_dataloader(  # type: ignore[misc]
+                    dataset=eval_set,
+                    batch_size=batch_size,
+                    num_workers=self.num_workers,
+                    shuffle=False,
+                )
+            else:
+                eval_loader = DataLoader(
+                    dataset=eval_set,
+                    batch_size=batch_size,
+                    num_workers=self.num_workers,
+                    shuffle=False,
+                )
+
+        return train_loader, eval_loader
+
     @alias("n_epochs", ["finetune_epochs", "warmup_epochs"])
     @alias("max_lr", ["finetune_max_lr", "warmup_max_lr"])
-    def _finetune(
+    def _do_finetune(
         self,
         loader: DataLoader,
         n_epochs: int = 5,
@@ -905,6 +914,27 @@ def _finetune(
                     self.model.deepimage, "deepimage", loader, n_epochs, max_lr
                 )
 
+    def _train_epoch(
+        self,
+        train_loader: DataLoader,
+        epoch: int,
+    ):
+        epoch_logs: Dict[str, float] = {}
+        self.callback_container.on_epoch_begin(epoch, logs=epoch_logs)
+        self.train_running_loss = 0.0
+
+        train_steps = len(train_loader)
+        with trange(train_steps, disable=self.verbose != 1) as t:
+            for batch_idx, (data, targett) in zip(t, train_loader):
+                t.set_description("epoch %i" % (epoch + 1))
+                train_score, train_loss = self._train_step(data, targett, batch_idx)
+                print_loss_and_metric(t, train_loss, train_score)
+                self.callback_container.on_batch_end(batch=batch_idx)
+
+        epoch_logs = save_epoch_logs(epoch_logs, train_loss, train_score, "train")
+
+        return epoch_logs
+
     def _train_step(
         self,
         data: Dict[str, Union[Tensor, List[Tensor]]],
@@ -946,6 +976,33 @@ def _train_step(
 
         return score, avg_loss
 
+    def _eval_epoch(
+        self,
+        eval_loader: DataLoader,
+        epoch_logs: Dict[str, float],
+    ) -> Tuple[Dict[str, float], Optional[float]]:
+        self.callback_container.on_eval_begin()
+        self.valid_running_loss = 0.0
+
+        eval_steps = len(eval_loader)
+        with trange(eval_steps, disable=self.verbose != 1) as v:
+            for i, (data, targett) in zip(v, eval_loader):
+                v.set_description("valid")
+                val_score, val_loss = self._eval_step(data, targett, i)
+                print_loss_and_metric(v, val_loss, val_score)
+
+        epoch_logs = save_epoch_logs(epoch_logs, val_loss, val_score, "val")
+
+        if self.reducelronplateau:
+            if self.reducelronplateau_criterion == "loss":
+                on_epoch_end_metric = val_loss
+            else:
+                on_epoch_end_metric = val_score[self.reducelronplateau_criterion]
+        else:
+            on_epoch_end_metric = None
+
+        return epoch_logs, on_epoch_end_metric
+
     def _eval_step(
         self,
         data: Dict[str, Union[Tensor, List[Tensor]]],
@@ -988,11 +1045,11 @@ def _get_score(
         score = None
         metric = None
 
-        if hasattr(self, "metric") and not hasattr(self, "eval_metric"):
+        if self.metric and not self.eval_metric:
             metric = self.metric
-        elif hasattr(self, "metric") and hasattr(self, "eval_metric"):
+        elif self.metric and self.eval_metric:
             metric = self.metric if is_train else self.eval_metric
-        elif not hasattr(self, "metric") and hasattr(self, "eval_metric"):
+        elif not self.metric and self.eval_metric:
             metric = None if is_train else self.eval_metric
 
         if metric is not None:
@@ -1097,6 +1154,7 @@ def _predict(  # type: ignore[override, return]  # noqa: C901
 
     @staticmethod
     def _predict_ziln(preds: Tensor) -> Tensor:
+        # Legacy implementation. It will be removed in future versions
         """Calculates predicted mean of zero inflated lognormal logits.
 
         Adjusted implementaion of `code
diff --git a/tests/test_metrics/test_ranking_metrics.py b/tests/test_metrics/test_ranking_metrics.py
index b08ecb4a..b221658e 100644
--- a/tests/test_metrics/test_ranking_metrics.py
+++ b/tests/test_metrics/test_ranking_metrics.py
@@ -55,6 +55,15 @@ def test_binary_ndcg_at_k(setup_data):
     expected = np.array(0.5719)
     np.testing.assert_almost_equal(result, expected, decimal=4)
 
+    # Explaining the expected value (@k=3):
+    # DCG_row1 = 0*1 + 1*0.6309 + 1*0.5 = 1.1309 -> (0.7 -> 0, 0.9 -> 1, 0.8 -> 1)
+    # DCG_row2 = 1*1 + 0*0.6309 + 0*0.5 = 1.0 -> (0.3 -> 1, 0.5 -> 0, 0.4 -> 0)
+    # IDCG_row1 = 1*1 + 1*0.6309 + 1*0.5 = 2.1309 (3 relevant items)
+    # IDCG_row2 = 1*1 + 1*0.6309 = 1.6309 (2 relevant items)
+    # NDCG_row1 = DCG_row1 / IDCG_row1 = 1.1309 / 2.1309 = 0.5309
+    # NDCG_row2 = DCG_row2 / IDCG_row2 = 1.0 / 1.6309 = 0.6129
+    # binary_NDCG = (NDCG_row1 + NDCG_row2) / 2 = (0.5309 + 0.6129) / 2 = 0.5719
+
 
 def test_map_at_k(setup_data):
     y_pred, y_true = setup_data
@@ -63,6 +72,12 @@ def test_map_at_k(setup_data):
     expected = np.array(0.4166)
     np.testing.assert_almost_equal(result, expected, decimal=4)
 
+    # Explaining the expected value (@k=3):
+    # batch relevance for row top 3 predictions: [[1, 1, 0], [0, 0, 1]]
+    # AP_row1 = (1/1 + 2/2 + 0/3) / 3 = 0.6666 -> [(1/1) * 1 + (2/2) * 1 + (2/3) * 0] / 3
+    # AP_row2 = (0/1 + 0/2 + 1/3) / 2 = 0.1666 -> [(0/1) * 0 + (0/2) * 0 + (1/3) * 1] / 2
+    # MAP = (AP_row1 + AP_row2) / 2 = (0.6666 + 0.1666) / 2 = 0.4166
+
 
 def test_hit_ratio_at_k(setup_data):
     y_pred, y_true = setup_data
@@ -79,6 +94,14 @@ def test_precision_at_k(setup_data):
     expected = np.array(0.5)
     np.testing.assert_almost_equal(result, expected, decimal=4)
 
+    # Explaining the expected value (@k=3):
+    # batch relevance for row top 3 predictions: [[1, 1, 0], [0, 0, 1]]
+    # Precision_row1 = 2 / 3 = 0.666
+    # Precision_row2 = 1 / 3 = 0.333
+    # Precision = (Precision_row1 + Precision_row2) / 2 = (0.666 + 0.333) / 2 = 0.5
+    # caveat: if k is higher than the number of relevant items for a given row
+    # this metric will never be 1.0
+
 
 def test_recall_at_k(setup_data):
     y_pred, y_true = setup_data
@@ -87,6 +110,12 @@ def test_recall_at_k(setup_data):
     expected = np.array(0.5833)
     np.testing.assert_almost_equal(result, expected, decimal=4)
 
+    # Explaining the expected value (@k=3):
+    # batch relevance for row top 3 predictions: [[1, 1, 0], [0, 0, 1]]
+    # Recall_row1 = 2 / 3 = 0.666
+    # Recall_row2 = 1 / 2 = 0.5
+    # Recall = (Recall_row1 + Recall_row2) / 2 = (0.666 + 0.5) / 2 = 0.5833
+
 
 def test_edge_cases_all_relevant_items():
     # Test with all relevant items
@@ -159,6 +188,19 @@ def test_ndcg_at_k(setup_data_ndcg):
     expected = np.array(0.7198)
     np.testing.assert_almost_equal(result, expected, decimal=4)
 
+    # Explaining the expected value (@k=3):
+    # top 3 predictions for row 1: [0.9, 0.8, 0.7] -> [3, 2, 1]
+    # top 3 predictions for row 2: [0.5, 0.4, 0.3] -> [0, 1, 3]
+    # DCG = sum[(2^rel - 1) / log2(rank + 1)]
+    # DCG_row1 = 2^3 - 1 / log2(1+1) + 2^2 - 1 / log2(2+1) + 2^1 - 1 / log2(3+1) = 7.0 + 1.8928 + 0.5000 = 9.3928
+    # DCG_row2 = 2^0 - 1 / log2(1+1) + 2^1 - 1 / log2(2+1) + 2^3 - 1 / log2(3+1) = 0.0 + 0.6309 + 3.5 = 4.1309
+    # IDCG = sum[(2^rel - 1) / log2(rank + 1)] for the ideal ranking at k=3 [[3, 2, 1], [3, 2, 1]]
+    # IDCG_row1 = 2^3 - 1 / log2(1+1) + 2^2 - 1 / log2(2+1) + 2^1 - 1 / log2(3+1) = 7.0 + 1.8928 + 0.5000 = 9.3928
+    # IDCG_row2 = IDCG_row1 = 9.3928
+    # NDGC_row1 = DCG_row1 / IDCG_row1 = 9.3928 / 9.3928 = 1.0
+    # NDGC_row2 = DCG_row2 / IDCG_row2 = 4.1309 / 9.3928 = 0.4398
+    # NDCG = (NDGC_row1 + NDGC_row2) / 2 = (1.0 + 0.4398) / 2 = 0.7198
+
 
 def test_ndcg_at_k_edge_cases():
     # Test with non-decreasing ranking

From f091194bf69be95558be667ea337f2ff518f1bda Mon Sep 17 00:00:00 2001
From: Javier <jrzaurin@gmail.com>
Date: Sat, 19 Oct 2024 23:11:08 +0100
Subject: [PATCH 10/24] First integration test run. I will have to adjust all
 examples containing the  class

---
 pytest.ini                                    |   2 +
 pytorch_widedeep/dataloaders.py               |  79 +++++++--
 pytorch_widedeep/metrics.py                   |  66 +++++---
 pytorch_widedeep/training/trainer.py          |  67 +++-----
 .../test_ranking_metrics_integration.py       | 153 ++++++++++++++++++
 5 files changed, 285 insertions(+), 82 deletions(-)
 create mode 100644 pytest.ini
 create mode 100644 tests/test_rec/test_ranking_metrics_integration.py

diff --git a/pytest.ini b/pytest.ini
new file mode 100644
index 00000000..c1fa8785
--- /dev/null
+++ b/pytest.ini
@@ -0,0 +1,2 @@
+[pytest]
+addopts = -p no:warnings
\ No newline at end of file
diff --git a/pytorch_widedeep/dataloaders.py b/pytorch_widedeep/dataloaders.py
index 3c17bc5c..db53ace0 100644
--- a/pytorch_widedeep/dataloaders.py
+++ b/pytorch_widedeep/dataloaders.py
@@ -1,4 +1,4 @@
-from typing import Tuple
+from typing import Tuple, Optional
 
 import numpy as np
 from torch.utils.data import DataLoader, WeightedRandomSampler
@@ -6,6 +6,45 @@
 from pytorch_widedeep.training._wd_dataset import WideDeepDataset
 
 
+class DatasetAlreadySetError(Exception):
+    """Exception raised when attempting to set a dataset that has already been set."""
+
+    def __init__(self, message="Dataset has already been set and cannot be changed."):
+        self.message = message
+        super().__init__(self.message)
+
+
+class CustomDataLoader(DataLoader):
+    r"""
+    Wrapper around the `torch.utils.data.DataLoader` class that allows
+    to set the dataset after the class has been instantiated.
+    """
+
+    def __init__(self, dataset: Optional[WideDeepDataset] = None, *args, **kwargs):
+        self.dataset_set = dataset is not None
+        if self.dataset_set:
+            super().__init__(dataset, *args, **kwargs)
+        else:
+            self.args = args
+            self.kwargs = kwargs
+
+    def set_dataset(self, dataset: WideDeepDataset):
+        if self.dataset_set:
+            raise DatasetAlreadySetError()
+
+        self.dataset_set = True
+        super().__init__(dataset, *self.args, **self.kwargs)
+
+    def __iter__(self):
+        if not self.dataset_set:
+            raise ValueError(
+                "Dataset has not been set. Use set_dataset method to set a dataset."
+            )
+        return super().__iter__()
+
+
+# From here on is legacy code and there are better ways to do it. It will be
+# removed in the next version.
 def get_class_weights(dataset: WideDeepDataset) -> Tuple[np.ndarray, int, int]:
     """Helper function to get weights of classes in the imbalanced dataset.
 
@@ -37,7 +76,7 @@ def get_class_weights(dataset: WideDeepDataset) -> Tuple[np.ndarray, int, int]:
     return weights, minor_class_count, num_classes
 
 
-class DataLoaderImbalanced(DataLoader):
+class DataLoaderImbalanced(CustomDataLoader):
     r"""Class to load and shuffle batches with adjusted weights for imbalanced
     datasets. If the classes do not begin from 0 remapping is necessary. See
     [here](https://towardsdatascience.com/pytorch-tabular-multiclass-classification-9f8211a123ab).
@@ -46,13 +85,11 @@ class DataLoaderImbalanced(DataLoader):
     ----------
     dataset: `WideDeepDataset`
         see `pytorch_widedeep.training._wd_dataset`
-    batch_size: int
-        size of batch
-    num_workers: int
-        number of workers
 
     Other Parameters
     ----------------
+    *args: Any
+        Positional arguments to be passed to the parent CustomDataLoader.
     **kwargs: Dict
         This can include any parameter that can be passed to the _'standard'_
         pytorch
@@ -69,23 +106,31 @@ class DataLoaderImbalanced(DataLoader):
         $$
     """
 
-    def __init__(
-        self, dataset: WideDeepDataset, batch_size: int, num_workers: int, **kwargs
-    ):
+    def __init__(self, dataset: Optional[WideDeepDataset] = None, *args, **kwargs):
+        self.args = args
+        self.kwargs = kwargs
+
+        if dataset is not None:
+            self._setup_sampler(dataset)
+            super().__init__(dataset, *args, sampler=self.sampler, **kwargs)
+        else:
+            super().__init__()
+
+    def set_dataset(self, dataset: WideDeepDataset):
+        sampler = self._setup_sampler(dataset)
+        # update the kwargs with the new sampler
+        self.kwargs["sampler"] = sampler
+        super().set_dataset(dataset)
+
+    def _setup_sampler(self, dataset: WideDeepDataset) -> WeightedRandomSampler:
         assert dataset.Y is not None, (
             "The 'dataset' instance of WideDeepDataset must contain a "
             "target array 'Y'"
         )
 
-        if "oversample_mul" in kwargs:
-            oversample_mul = kwargs["oversample_mul"]
-            del kwargs["oversample_mul"]
-        else:
-            oversample_mul = 1
+        oversample_mul = self.kwargs.pop("oversample_mul", 1)
         weights, minor_cls_cnt, num_clss = get_class_weights(dataset)
         num_samples = int(minor_cls_cnt * num_clss * oversample_mul)
         samples_weight = list(np.array([weights[i] for i in dataset.Y]))
         sampler = WeightedRandomSampler(samples_weight, num_samples, replacement=True)
-        super().__init__(
-            dataset, batch_size, num_workers=num_workers, sampler=sampler, **kwargs
-        )
+        return sampler
diff --git a/pytorch_widedeep/metrics.py b/pytorch_widedeep/metrics.py
index 5286952b..c708ff9b 100644
--- a/pytorch_widedeep/metrics.py
+++ b/pytorch_widedeep/metrics.py
@@ -397,15 +397,25 @@ def __call__(self, y_pred: Tensor, y_true: Tensor) -> np.ndarray:
         return np.array((1 - (self.numerator / self.denominator)))
 
 
-def reshape_1d_to_2d(tensor: Tensor, n_columns: int) -> Tensor:
-    if tensor.dim() != 1:
-        raise ValueError("Input tensor must be 1-dimensional")
-    if tensor.size(0) % n_columns != 0:
+def reshape_to_2d(tensor: Tensor, n_columns: int) -> Tensor:
+    if tensor.dim() == 1:
+        if tensor.size(0) % n_columns != 0:
+            raise ValueError(
+                f"Tensor length ({tensor.size(0)}) must be divisible by n_columns ({n_columns})"
+            )
+        n_rows = tensor.size(0) // n_columns
+        return tensor.reshape(n_rows, n_columns)
+    elif tensor.dim() == 2 and tensor.size(1) == 1:
+        if tensor.size(0) % n_columns != 0:
+            raise ValueError(
+                f"Tensor length ({tensor.size(0)}) must be divisible by n_columns ({n_columns})"
+            )
+        n_rows = tensor.size(0) // n_columns
+        return tensor.reshape(n_rows, n_columns)
+    else:
         raise ValueError(
-            f"Tensor length ({tensor.size(0)}) must be divisible by n_columns ({n_columns})"
+            "Input tensor must be 1-dimensional or 2-dimensional with one column"
         )
-    n_rows = tensor.size(0) // n_columns
-    return tensor.reshape(n_rows, n_columns)
 
 
 class NDCG_at_k(Metric):
@@ -461,8 +471,8 @@ def reset(self):
     def __call__(self, y_pred: Tensor, y_true: Tensor) -> np.ndarray:
         device = y_pred.device
 
-        y_pred_2d = reshape_1d_to_2d(y_pred, self.n_cols)
-        y_true_2d = reshape_1d_to_2d(y_true, self.n_cols)
+        y_pred_2d = reshape_to_2d(y_pred, self.n_cols)
+        y_true_2d = reshape_to_2d(y_true, self.n_cols)
 
         batch_size = y_true_2d.shape[0]
 
@@ -544,8 +554,8 @@ def reset(self):
     def __call__(self, y_pred: Tensor, y_true: Tensor) -> np.ndarray:
         device = y_pred.device
 
-        y_pred_2d = reshape_1d_to_2d(y_pred, self.n_cols)
-        y_true_2d = reshape_1d_to_2d(y_true, self.n_cols)
+        y_pred_2d = reshape_to_2d(y_pred, self.n_cols)
+        y_true_2d = reshape_to_2d(y_true, self.n_cols)
 
         batch_size = y_pred_2d.shape[0]
 
@@ -607,6 +617,9 @@ class MAP_at_k(Metric):
     >>> score = map_at_k(y_pred, y_true)
     """
 
+    @alias(
+        "n_cols", ["n_items", "n_items_per_query", "n_items_per_id", "n_items_per_user"]
+    )
     def __init__(self, n_cols: int = 10, k: Optional[int] = None):
         super(MAP_at_k, self).__init__()
 
@@ -626,8 +639,8 @@ def reset(self):
 
     def __call__(self, y_pred: Tensor, y_true: Tensor) -> np.ndarray:
 
-        y_pred_2d = reshape_1d_to_2d(y_pred, self.n_cols)
-        y_true_2d = reshape_1d_to_2d(y_true, self.n_cols)
+        y_pred_2d = reshape_to_2d(y_pred, self.n_cols)
+        y_true_2d = reshape_to_2d(y_true, self.n_cols)
 
         batch_size = y_pred_2d.shape[0]
         _, top_k_indices = torch.topk(y_pred_2d, self.k, dim=1)
@@ -670,6 +683,9 @@ class HitRatio_at_k(Metric):
     >>> score = hr_at_k(y_pred, y_true
     """
 
+    @alias(
+        "n_cols", ["n_items", "n_items_per_query", "n_items_per_id", "n_items_per_user"]
+    )
     def __init__(self, n_cols: int = 10, k: Optional[int] = None):
         super(HitRatio_at_k, self).__init__()
 
@@ -688,8 +704,8 @@ def reset(self):
         self.count = 0
 
     def __call__(self, y_pred: Tensor, y_true: Tensor) -> np.ndarray:
-        y_pred_2d = reshape_1d_to_2d(y_pred, self.n_cols)
-        y_true_2d = reshape_1d_to_2d(y_true, self.n_cols)
+        y_pred_2d = reshape_to_2d(y_pred, self.n_cols)
+        y_true_2d = reshape_to_2d(y_true, self.n_cols)
         batch_size = y_pred_2d.shape[0]
         _, top_k_indices = torch.topk(y_pred_2d, self.k, dim=1)
         batch_relevance = y_true_2d.gather(1, top_k_indices)
@@ -726,6 +742,9 @@ class Precision_at_k(Metric):
     >>> score = prec_at_k(y_pred, y_true)
     """
 
+    @alias(
+        "n_cols", ["n_items", "n_items_per_query", "n_items_per_id", "n_items_per_user"]
+    )
     def __init__(self, n_cols: int = 10, k: Optional[int] = None):
         super(Precision_at_k, self).__init__()
 
@@ -744,8 +763,8 @@ def reset(self):
         self.count = 0
 
     def __call__(self, y_pred: Tensor, y_true: Tensor) -> np.ndarray:
-        y_pred_2d = reshape_1d_to_2d(y_pred, self.n_cols)
-        y_true_2d = reshape_1d_to_2d(y_true, self.n_cols)
+        y_pred_2d = reshape_to_2d(y_pred, self.n_cols)
+        y_true_2d = reshape_to_2d(y_true, self.n_cols)
         batch_size = y_pred_2d.shape[0]
         _, top_k_indices = torch.topk(y_pred_2d, self.k, dim=1)
         batch_relevance = y_true_2d.gather(1, top_k_indices)
@@ -761,6 +780,12 @@ class Recall_at_k(Metric):
 
     Parameters
     ----------
+    n_cols: int, default = 10
+        Number of columns in the input tensors. This parameter is neccessary
+        because the input tensors are reshaped to 2D tensors. n_cols is the
+        number of columns in the reshaped tensor. Alias for this parameter
+        are: 'n_items', 'n_items_per_query',
+        'n_items_per_id', 'n_items_per_user'
     k: int, default = 10
         Number of top items to consider.
 
@@ -775,6 +800,9 @@ class Recall_at_k(Metric):
     >>> score = rec_at_k(y_pred, y_true)
     """
 
+    @alias(
+        "n_cols", ["n_items", "n_items_per_query", "n_items_per_id", "n_items_per_user"]
+    )
     def __init__(self, n_cols: int = 10, k: Optional[int] = None):
         super(Recall_at_k, self).__init__()
 
@@ -793,8 +821,8 @@ def reset(self):
         self.count = 0
 
     def __call__(self, y_pred: Tensor, y_true: Tensor) -> np.ndarray:
-        y_pred_2d = reshape_1d_to_2d(y_pred, self.n_cols)
-        y_true_2d = reshape_1d_to_2d(y_true, self.n_cols)
+        y_pred_2d = reshape_to_2d(y_pred, self.n_cols)
+        y_true_2d = reshape_to_2d(y_true, self.n_cols)
         batch_size = y_pred_2d.shape[0]
         _, top_k_indices = torch.topk(y_pred_2d, self.k, dim=1)
         batch_relevance = y_true_2d.gather(1, top_k_indices)
diff --git a/pytorch_widedeep/training/trainer.py b/pytorch_widedeep/training/trainer.py
index 7400409b..bd871f3b 100644
--- a/pytorch_widedeep/training/trainer.py
+++ b/pytorch_widedeep/training/trainer.py
@@ -26,6 +26,7 @@
     LRScheduler,
 )
 from pytorch_widedeep.callbacks import Callback
+from pytorch_widedeep.dataloaders import CustomDataLoader, DatasetAlreadySetError
 from pytorch_widedeep.initializers import Initializer
 from pytorch_widedeep.training._finetune import FineTune
 from pytorch_widedeep.utils.general_utils import alias, to_device
@@ -285,8 +286,8 @@ def fit(  # noqa: C901
         n_epochs: int = 1,
         validation_freq: int = 1,
         batch_size: int = 32,
-        train_dataloader: Optional[DataLoader] = None,
-        eval_dataloader: Optional[DataLoader] = None,
+        train_dataloader: Optional[CustomDataLoader] = None,
+        eval_dataloader: Optional[CustomDataLoader] = None,
         feature_importance_sample_size: Optional[int] = None,
         finetune: bool = False,
         **kwargs,
@@ -438,13 +439,13 @@ def fit(  # noqa: C901
             target,
             self.transforms,
         )
-        train_loader, eval_loader = self._get_dataloaders(
-            train_dataloader,
-            eval_dataloader,
-            train_set,
-            eval_set,
-            batch_size,
-            dataloader_args,
+        train_loader = self._set_dataloader(
+            train_dataloader, train_set, batch_size, dataloader_args
+        )
+        eval_loader = (
+            self._set_dataloader(eval_dataloader, eval_set, batch_size, dataloader_args)
+            if eval_set is not None
+            else None
         )
 
         if finetune:
@@ -780,51 +781,25 @@ def save(
             with open(Path(path) / "feature_importance.json", "w") as fi:
                 json.dump(self.feature_importance, fi)
 
-    def _get_dataloaders(
+    def _set_dataloader(
         self,
-        train_dataloader: Optional[DataLoader],
-        eval_dataloader: Optional[DataLoader],
-        train_set: WideDeepDataset,
-        eval_set: Optional[WideDeepDataset],
+        dataloader: Optional[CustomDataLoader],
+        dataset: WideDeepDataset,
         batch_size: int,
         dataloader_args: Dict[str, Any],
-    ) -> Tuple[DataLoader, Optional[DataLoader]]:
-        if train_dataloader is not None:
-            # make sure is callable (and HAS to be an subclass of DataLoader)
-            assert isinstance(train_dataloader, type)
-            train_loader = train_dataloader(  # type: ignore[misc]
-                dataset=train_set,
-                batch_size=batch_size,
-                num_workers=self.num_workers,
-                **dataloader_args,
-            )
+    ) -> DataLoader | CustomDataLoader:
+        if dataloader is not None:
+            dataloader.set_dataset(dataset)
+            return dataloader
         else:
-            train_loader = DataLoader(
-                dataset=train_set,
+            # var name 'loader' to avoid reassigment and type errors
+            loader = DataLoader(
+                dataset=dataset,
                 batch_size=batch_size,
                 num_workers=self.num_workers,
                 **dataloader_args,
             )
-
-        eval_loader = None
-        if eval_set is not None:
-            if eval_dataloader is not None:
-                assert isinstance(eval_dataloader, type)
-                eval_loader = eval_dataloader(  # type: ignore[misc]
-                    dataset=eval_set,
-                    batch_size=batch_size,
-                    num_workers=self.num_workers,
-                    shuffle=False,
-                )
-            else:
-                eval_loader = DataLoader(
-                    dataset=eval_set,
-                    batch_size=batch_size,
-                    num_workers=self.num_workers,
-                    shuffle=False,
-                )
-
-        return train_loader, eval_loader
+        return loader
 
     @alias("n_epochs", ["finetune_epochs", "warmup_epochs"])
     @alias("max_lr", ["finetune_max_lr", "warmup_max_lr"])
diff --git a/tests/test_rec/test_ranking_metrics_integration.py b/tests/test_rec/test_ranking_metrics_integration.py
new file mode 100644
index 00000000..1239ae54
--- /dev/null
+++ b/tests/test_rec/test_ranking_metrics_integration.py
@@ -0,0 +1,153 @@
+# silence DeprecationWarning
+import warnings
+from typing import Tuple
+
+import numpy as np
+import pandas as pd
+import pytest
+
+from pytorch_widedeep.models import TabMlp, WideDeep
+from pytorch_widedeep.metrics import (
+    F1Score,
+    Accuracy,
+    MAP_at_k,
+    NDCG_at_k,
+    Recall_at_k,
+    HitRatio_at_k,
+    Precision_at_k,
+    BinaryNDCG_at_k,
+)
+from pytorch_widedeep.training import Trainer
+from pytorch_widedeep.preprocessing import TabPreprocessor
+from pytorch_widedeep.training._wd_dataset import WideDeepDataset
+
+
+def generate_user_item_interactions(
+    n_users: int = 10,
+    n_interactions_per_user: int = 10,
+    n_items: int = 5,
+    random_seed: int = 42,
+) -> pd.DataFrame:
+    np.random.seed(random_seed)
+
+    user_ids = np.repeat(range(1, n_users + 1), n_interactions_per_user)
+    item_ids = np.random.randint(1, n_items + 1, size=n_users * n_interactions_per_user)
+    user_categories = np.random.choice(
+        ["A", "B", "C"], size=n_users * n_interactions_per_user
+    )
+    item_categories = np.random.choice(
+        ["X", "Y", "Z"], size=n_users * n_interactions_per_user
+    )
+    binary_target = np.random.randint(0, 2, size=n_users * n_interactions_per_user)
+    categorical_target = np.random.randint(0, 3, size=n_users * n_interactions_per_user)
+
+    df = pd.DataFrame(
+        {
+            "user_id": user_ids,
+            "item_id": item_ids,
+            "user_category": user_categories,
+            "item_category": item_categories,
+            "liked": binary_target,
+            "rating": categorical_target,
+        }
+    )
+
+    return df
+
+
+def split_train_validation(
+    df: pd.DataFrame, validation_interactions_per_user: int
+) -> Tuple[pd.DataFrame, pd.DataFrame]:
+    grouped = (
+        df.groupby("user_id")
+        .apply(lambda x: x.sample(frac=1, random_state=42))
+        .reset_index(drop=True)
+    )
+
+    train_df = (
+        grouped.groupby("user_id")
+        .apply(lambda x: x.iloc[:-validation_interactions_per_user])
+        .reset_index(drop=True)
+    )
+    val_df = (
+        grouped.groupby("user_id")
+        .apply(lambda x: x.iloc[-validation_interactions_per_user:])
+        .reset_index(drop=True)
+    )
+
+    return train_df, val_df
+
+
+@pytest.fixture
+def user_item_data(request):
+    validation_interactions_per_user = request.param.get(
+        "validation_interactions_per_user", 5
+    )
+    df = generate_user_item_interactions()
+    train_df, val_df = split_train_validation(df, validation_interactions_per_user)
+    return train_df, val_df
+
+
+# strategy 1:
+# 	* ranking metrics for both training and validation sets.
+# 	* Same number of items per user in both sets
+
+
+@pytest.mark.parametrize(
+    "user_item_data", [{"validation_interactions_per_user": 5}], indirect=True
+)
+@pytest.mark.parametrize(
+    "metric",
+    [
+        MAP_at_k(n_cols=5, k=3),
+        NDCG_at_k(n_cols=5, k=3),
+        Recall_at_k(n_cols=5, k=3),
+        HitRatio_at_k(n_cols=5, k=3),
+        Precision_at_k(n_cols=5, k=3),
+        BinaryNDCG_at_k(n_cols=5, k=3),
+    ],
+)
+def test_binary_classification_strategy_1(user_item_data, metric):
+
+    train_df, val_df = user_item_data
+
+    categorical_cols = ["user_id", "item_id", "user_category", "item_category"]
+
+    target_col = "liked"
+
+    tab_preprocessor = TabPreprocessor(
+        embed_cols=categorical_cols,
+        for_transformer=False,
+    )
+
+    tab_preprocessor.fit(train_df)
+
+    X_train = tab_preprocessor.transform(train_df)
+    X_val = tab_preprocessor.transform(val_df)
+
+    tab_mlp = TabMlp(
+        column_idx=tab_preprocessor.column_idx,
+        cat_embed_input=tab_preprocessor.cat_embed_input,  # type: ignore[arg-type]
+    )
+    model = WideDeep(deeptabular=tab_mlp)
+
+    trainer = Trainer(model, objective="binary", metrics=[metric])
+
+    trainer.fit(
+        X_train={"X_tab": X_train, "target": train_df[target_col].values},
+        X_val={"X_tab": X_val, "target": val_df[target_col].values},
+        n_epochs=1,
+        batch_size=2 * 5,
+    )
+
+    # predict on validation, this is just a test...
+    preds = trainer.predict(X_tab=X_val)
+
+    assert preds.shape[0] == X_val.shape[0]
+    assert (
+        trainer.history is not None
+        and "train_loss" in trainer.history
+        and "val_loss" in trainer.history
+        and f"train_{metric._name}" in trainer.history
+        and f"val_{metric._name}" in trainer.history
+    )

From 62db3ca23cc540dfec6de0afba583afd0d441942 Mon Sep 17 00:00:00 2001
From: Javier <jrzaurin@gmail.com>
Date: Mon, 21 Oct 2024 09:57:14 +0100
Subject: [PATCH 11/24] All test passed. I need to update examples for the
 Imbalanced Data Loader. Then review all the docs and cleans/rmove the docs
 folder

---
 .gitignore                                    |  11 +-
 .isort.cfg                                    |   4 -
 .travis.yml                                   |  34 --
 code_style.sh                                 |   8 -
 pytest.ini                                    |   2 -
 pytorch_widedeep/metrics.py                   |  10 +
 pytorch_widedeep/training/_base_trainer.py    |   3 +-
 pytorch_widedeep/training/trainer.py          |   2 +-
 .../training/trainer_from_folder.py           | 330 ++---------------
 .../test_fit_methods.py                       |   4 +-
 .../test_ranking_metrics_integration.py       | 339 +++++++++++++++++-
 11 files changed, 382 insertions(+), 365 deletions(-)
 delete mode 100644 .isort.cfg
 delete mode 100644 .travis.yml
 delete mode 100755 code_style.sh
 delete mode 100644 pytest.ini

diff --git a/.gitignore b/.gitignore
index 06c4cc68..710f3c01 100644
--- a/.gitignore
+++ b/.gitignore
@@ -60,4 +60,13 @@ checkpoints
 
 # wnb
 wandb/
-wandb_api.key
\ No newline at end of file
+wandb_api.key
+
+# pytest
+pytest.ini
+
+# code style
+code_style.sh
+
+# isort
+.isort.cfg
diff --git a/.isort.cfg b/.isort.cfg
deleted file mode 100644
index 023dbdb4..00000000
--- a/.isort.cfg
+++ /dev/null
@@ -1,4 +0,0 @@
-[settings]
-profile=black
-multi_line_output=3
-length_sort=1
\ No newline at end of file
diff --git a/.travis.yml b/.travis.yml
deleted file mode 100644
index 75550745..00000000
--- a/.travis.yml
+++ /dev/null
@@ -1,34 +0,0 @@
-dist: xenial
-language: python
-python:
-  - "3.7.9"
-  - "3.8"
-  - "3.9"
-matrix:
-  fast_finish: true
-  include:
-    - name: "Code Style (Black/Flake8)"
-      install:
-        - pip install black
-        - pip install flake8
-      script:
-        # Black code style
-        - black --check --diff pytorch_widedeep tests examples setup.py
-        # Stop the build if there are Python syntax errors or undefined names
-        - flake8 . --count --select=E901,E999,F821,F822,F823 --ignore=E266 --show-source --statistics
-        # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
-        - flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --ignore=E203,E266,E501,E722,F401,F403,F405,F811,W503,C901 --statistics
-      after_success: skip
-install:
-  - pip install --upgrade pip
-  - pip install pytest-cov
-  - pip install codecov
-  - pip install .
-script:
-  - pytest --doctest-modules pytorch_widedeep --cov-report html --cov-report term --disable-pytest-warnings --cov=pytorch_widedeep tests/
-  # - pytest tests
-after_success:
-  # Combine coverage reports from Travis jobs
-  - coverage combine
-  # Send reports to Codecov. It automatically handles merging coverage results
-  - codecov
\ No newline at end of file
diff --git a/code_style.sh b/code_style.sh
deleted file mode 100755
index 52eb2d64..00000000
--- a/code_style.sh
+++ /dev/null
@@ -1,8 +0,0 @@
-# sort imports
-isort --quiet . pytorch_widedeep tests examples setup.py
-# Black code style
-black . pytorch_widedeep tests examples setup.py
-# flake8 standards
-flake8 . --max-complexity=10 --max-line-length=127 --ignore=E203,E266,E501,E722,E721,F401,F403,F405,W503,C901,F811
-# mypy
-mypy pytorch_widedeep --ignore-missing-imports --no-strict-optional
\ No newline at end of file
diff --git a/pytest.ini b/pytest.ini
deleted file mode 100644
index c1fa8785..00000000
--- a/pytest.ini
+++ /dev/null
@@ -1,2 +0,0 @@
-[pytest]
-addopts = -p no:warnings
\ No newline at end of file
diff --git a/pytorch_widedeep/metrics.py b/pytorch_widedeep/metrics.py
index c708ff9b..772f4c57 100644
--- a/pytorch_widedeep/metrics.py
+++ b/pytorch_widedeep/metrics.py
@@ -469,8 +469,18 @@ def reset(self):
         self.count = 0
 
     def __call__(self, y_pred: Tensor, y_true: Tensor) -> np.ndarray:
+        # NDGC@k is supposed to be used when the output reflects interest
+        # scores, i.e, could be used in a regression or a multiclass problem.
+        # If regression y_pred will be a float tensor, if multiclass, y_pred
+        # will be a float tensor with the output of a softmax activation
+        # function and we need to turn it into a 1D tensor with the class.
+        # Finally, for binary problems, please use BinaryNDCG_at_k
         device = y_pred.device
 
+        if y_pred.ndim > 1 and y_pred.size(1) > 1:
+            # multiclass
+            y_pred = y_pred.topk(1, 1)[1]
+
         y_pred_2d = reshape_to_2d(y_pred, self.n_cols)
         y_true_2d = reshape_to_2d(y_true, self.n_cols)
 
diff --git a/pytorch_widedeep/training/_base_trainer.py b/pytorch_widedeep/training/_base_trainer.py
index bf2c3875..2f15a3bb 100644
--- a/pytorch_widedeep/training/_base_trainer.py
+++ b/pytorch_widedeep/training/_base_trainer.py
@@ -63,6 +63,7 @@ def __init__(
         transforms: Optional[List[Transforms]],
         callbacks: Optional[List[Callback]],
         metrics: Optional[Union[List[Metric], List[TorchMetric]]],
+        eval_metrics: Optional[Union[List[Metric], List[TorchMetric]]],
         verbose: int,
         seed: int,
         **kwargs,
@@ -90,7 +91,7 @@ def __init__(
         self.optimizer = self._set_optimizer(optimizers)
         self.lr_scheduler = self._set_lr_scheduler(lr_schedulers, **kwargs)
         self.transforms = self._set_transforms(transforms)
-        self._set_callbacks_and_metrics(callbacks, metrics)
+        self._set_callbacks_and_metrics(callbacks, metrics, eval_metrics)
 
     @abstractmethod
     def fit(self, **kwargs):
diff --git a/pytorch_widedeep/training/trainer.py b/pytorch_widedeep/training/trainer.py
index bd871f3b..e259268d 100644
--- a/pytorch_widedeep/training/trainer.py
+++ b/pytorch_widedeep/training/trainer.py
@@ -26,7 +26,7 @@
     LRScheduler,
 )
 from pytorch_widedeep.callbacks import Callback
-from pytorch_widedeep.dataloaders import CustomDataLoader, DatasetAlreadySetError
+from pytorch_widedeep.dataloaders import CustomDataLoader
 from pytorch_widedeep.initializers import Initializer
 from pytorch_widedeep.training._finetune import FineTune
 from pytorch_widedeep.utils.general_utils import alias, to_device
diff --git a/pytorch_widedeep/training/trainer_from_folder.py b/pytorch_widedeep/training/trainer_from_folder.py
index 1ebc2629..fac5a925 100644
--- a/pytorch_widedeep/training/trainer_from_folder.py
+++ b/pytorch_widedeep/training/trainer_from_folder.py
@@ -13,7 +13,6 @@
     List,
     Union,
     Tensor,
-    Literal,
     Optional,
     WideDeep,
     Optimizer,
@@ -22,30 +21,12 @@
 )
 from pytorch_widedeep.callbacks import Callback
 from pytorch_widedeep.initializers import Initializer
-from pytorch_widedeep.training._finetune import FineTune
+from pytorch_widedeep.training.trainer import Trainer
 from pytorch_widedeep.utils.general_utils import alias, to_device
 from pytorch_widedeep.training._wd_dataset import WideDeepDataset
-from pytorch_widedeep.training._base_trainer import BaseTrainer
-from pytorch_widedeep.training._trainer_utils import (
-    save_epoch_logs,
-    print_loss_and_metric,
-)
-
-# Observation 1: I am annoyed by sublime highlighting an override issue with
-# the abstractmethods. There is no override issue. The Signature of
-# the 'predict' method is compatible with the supertype. Buy for whatever
-# issue sublime highlights this as an error (not vscode and is not returned
-# as an error when running mypy). I am ignoring it
 
-# There is a lot of code repetition between this class and the 'Trainer'
-# class (and in consquence a lot of ignore methods for test coverage). Maybe
-# in the future I decided to merge the two of them and offer the ability to
-# laod from folder based on the input parameters. For now, I'll leave it like
-# this, separated, since it is the easiest and most manageable(i.e. easier to
-# debug) implementation
 
-
-class TrainerFromFolder(BaseTrainer):
+class TrainerFromFolder(Trainer):
     r"""Class to set the of attributes that will be used during the
      training process.
 
@@ -187,6 +168,7 @@ class TrainerFromFolder(BaseTrainer):
         "objective",
         ["loss_function", "loss_fn", "loss", "cost_function", "cost_fn", "cost"],
     )
+    @alias("metrics", ["train_metrics"])
     def __init__(
         self,
         model: WideDeep,
@@ -204,6 +186,7 @@ def __init__(
         transforms: Optional[List[Transforms]] = None,
         callbacks: Optional[List[Callback]] = None,
         metrics: Optional[Union[List[Metric], List[TorchMetric]]] = None,
+        eval_metrics: Optional[Union[List[Metric], List[TorchMetric]]] = None,
         verbose: int = 1,
         seed: int = 1,
         **kwargs,
@@ -218,13 +201,19 @@ def __init__(
             transforms=transforms,
             callbacks=callbacks,
             metrics=metrics,
+            eval_metrics=eval_metrics,
             verbose=verbose,
             seed=seed,
             **kwargs,
         )
 
+        if self.method == "multitarget":
+            raise NotImplementedError(
+                "Training from folder is not supported for multitarget models"
+            )
+
     @alias("finetune", ["warmup"])
-    def fit(  # noqa: C901
+    def fit(  # type: ignore[override]  # noqa: C901
         self,
         train_loader: DataLoader,
         eval_loader: Optional[DataLoader] = None,
@@ -233,68 +222,44 @@ def fit(  # noqa: C901
         finetune: bool = False,
         **kwargs,
     ):
-        finetune_args = self._extract_kwargs(kwargs)
 
-        train_steps = len(train_loader)
+        # There will never be dataloader_args when using 'TrainingFromFolder'
+        # as the loaders are passed as arguments
+        _, finetune_args = self._extract_kwargs(kwargs)
 
         if finetune:
-            self._finetune(train_loader, **finetune_args)
+            self.with_finetuning: bool = True
+            self._do_finetune(train_loader, **finetune_args)
             if self.verbose:
                 print(
                     "Fine-tuning (or warmup) of individual components completed. "
                     "Training the whole model for {} epochs".format(n_epochs)
                 )
+        else:
+            self.with_finetuning = False
 
         self.callback_container.on_train_begin(
             {
                 "batch_size": train_loader.batch_size,
-                "train_steps": train_steps,
+                "train_steps": len(train_loader),
                 "n_epochs": n_epochs,
             }
         )
         for epoch in range(n_epochs):
-            epoch_logs: Dict[str, float] = {}
-            self.callback_container.on_epoch_begin(epoch, logs=epoch_logs)
-
-            self.train_running_loss = 0.0
-            with trange(train_steps, disable=self.verbose != 1) as t:
-                for batch_idx, (data, targett) in zip(t, train_loader):
-                    t.set_description("epoch %i" % (epoch + 1))
-                    train_score, train_loss = self._train_step(
-                        data, targett, batch_idx, epoch
-                    )
-                    print_loss_and_metric(t, train_loss, train_score)
-                    self.callback_container.on_batch_end(batch=batch_idx)
-            epoch_logs = save_epoch_logs(epoch_logs, train_loss, train_score, "train")
-
-            on_epoch_end_metric = None
+            epoch_logs = self._train_epoch(train_loader, epoch)
             if eval_loader is not None and epoch % validation_freq == (
                 validation_freq - 1
             ):
-                eval_steps = len(eval_loader)
-                self.callback_container.on_eval_begin()
-                self.valid_running_loss = 0.0
-                with trange(eval_steps, disable=self.verbose != 1) as v:
-                    for i, (data, targett) in zip(v, eval_loader):
-                        v.set_description("valid")
-                        val_score, val_loss = self._eval_step(data, targett, i)
-                        print_loss_and_metric(v, val_loss, val_score)
-                epoch_logs = save_epoch_logs(epoch_logs, val_loss, val_score, "val")
-
-                if self.reducelronplateau:  # pragma: no cover
-                    if self.reducelronplateau_criterion == "loss":
-                        on_epoch_end_metric = val_loss
-                    else:
-                        on_epoch_end_metric = val_score[
-                            self.reducelronplateau_criterion
-                        ]
+                epoch_logs, on_epoch_end_metric = self._eval_epoch(
+                    eval_loader, epoch_logs
+                )
             else:
+                on_epoch_end_metric = None
                 if self.reducelronplateau:
                     raise NotImplementedError(
                         "ReduceLROnPlateau scheduler can be used only with validation data."
                     )
             self.callback_container.on_epoch_end(epoch, epoch_logs, on_epoch_end_metric)
-
             if self.early_stop:
                 # self.callback_container.on_train_end(epoch_logs)
                 break
@@ -327,7 +292,7 @@ def predict(  # type: ignore[override, return]
             preds = np.vstack(preds_l)
             return np.argmax(preds, 1)
 
-    def predict_uncertainty(  # type: ignore[return]
+    def predict_uncertainty(  # type: ignore[override, return]
         self,
         X_wide: Optional[np.ndarray] = None,
         X_tab: Optional[np.ndarray] = None,
@@ -402,202 +367,7 @@ def predict_proba(  # type: ignore[override, return]
         if self.method == "multiclass":
             return np.vstack(preds_l)
 
-    def save(
-        self,
-        path: str,
-        save_state_dict: bool = False,
-        save_optimizer: bool = False,
-        model_filename: str = "wd_model.pt",
-    ):  # pragma: no cover
-        """
-        Parameters
-        ----------
-        path: str
-            path to the directory where the model and the feature importance
-            attribute will be saved.
-        save_state_dict: bool, default = False
-            Boolean indicating whether to save directly the model
-            (and optimizer) or the model's (and optimizer's) state
-            dictionary
-        save_optimizer: bool, default = False
-            Boolean indicating whether to save the optimizer
-        model_filename: str, Optional, default = "wd_model.pt"
-            filename where the model weights will be store
-        """
-        self._save_history(path)
-
-        self._save_model_and_optimizer(
-            path, save_state_dict, save_optimizer, model_filename
-        )
-
-    @alias("n_epochs", ["finetune_epochs", "warmup_epochs"])
-    @alias("max_lr", ["finetune_max_lr", "warmup_max_lr"])
-    def _finetune(
-        self,
-        loader: DataLoader,
-        n_epochs: int = 5,
-        max_lr: float = 0.01,
-        routine: Literal["howard", "felbo"] = "howard",
-        deeptabular_gradual: bool = False,
-        deeptabular_layers: Optional[List[nn.Module]] = None,
-        deeptabular_max_lr: float = 0.01,
-        deeptext_gradual: bool = False,
-        deeptext_layers: Optional[List[nn.Module]] = None,
-        deeptext_max_lr: float = 0.01,
-        deepimage_gradual: bool = False,
-        deepimage_layers: Optional[List[nn.Module]] = None,
-        deepimage_max_lr: float = 0.01,
-    ):
-        r"""
-        Simple wrap-up to individually fine-tune model components
-        """
-        if self.model.deephead is not None:
-            raise ValueError(
-                "Currently warming up is only supported without a fully connected 'DeepHead'"
-            )
-
-        finetuner = FineTune(self.loss_fn, self.metric, self.method, self.verbose)  # type: ignore[arg-type]
-        if self.model.wide:
-            finetuner.finetune_all(self.model.wide, "wide", loader, n_epochs, max_lr)
-
-        if self.model.deeptabular:
-            if deeptabular_gradual:
-                assert (
-                    deeptabular_layers is not None
-                ), "deeptabular_layers must be passed if deeptabular_gradual=True"
-                finetuner.finetune_gradual(
-                    self.model.deeptabular,
-                    "deeptabular",
-                    loader,
-                    deeptabular_max_lr,
-                    deeptabular_layers,
-                    routine,
-                )
-            else:
-                finetuner.finetune_all(
-                    self.model.deeptabular, "deeptabular", loader, n_epochs, max_lr
-                )
-
-        if self.model.deeptext:
-            if deeptext_gradual:
-                assert (
-                    deeptext_layers is not None
-                ), "deeptext_layers must be passed if deeptabular_gradual=True"
-                finetuner.finetune_gradual(
-                    self.model.deeptext,
-                    "deeptext",
-                    loader,
-                    deeptext_max_lr,
-                    deeptext_layers,
-                    routine,
-                )
-            else:
-                finetuner.finetune_all(
-                    self.model.deeptext, "deeptext", loader, n_epochs, max_lr
-                )
-
-        if self.model.deepimage:
-            if deepimage_gradual:
-                assert (
-                    deepimage_layers is not None
-                ), "deepimage_layers must be passed if deeptabular_gradual=True"
-                finetuner.finetune_gradual(
-                    self.model.deepimage,
-                    "deepimage",
-                    loader,
-                    deepimage_max_lr,
-                    deepimage_layers,
-                    routine,
-                )
-            else:
-                finetuner.finetune_all(
-                    self.model.deepimage, "deepimage", loader, n_epochs, max_lr
-                )
-
-    def _train_step(
-        self,
-        data: Dict[str, Union[Tensor, List[Tensor]]],
-        target: Tensor,
-        batch_idx: int,
-        epoch: int,
-    ):
-        self.model.train()
-        X: Dict[str, Union[Tensor, List[Tensor]]] = {}
-        for k, v in data.items():
-            if isinstance(v, list):
-                X[k] = [to_device(i, self.device) for i in v]
-            else:
-                X[k] = to_device(v, self.device)
-        y = (
-            target.view(-1, 1).float()
-            if self.method not in ["multiclass", "qregression"]
-            else target
-        )
-        y = to_device(y, self.device)
-
-        self.optimizer.zero_grad()
-
-        y_pred = self.model(X)
-
-        if self.model.is_tabnet:  # pragma: no cover
-            loss = self.loss_fn(y_pred[0], y) - self.lambda_sparse * y_pred[1]
-            score = self._get_score(y_pred[0], y)
-        else:
-            loss = self.loss_fn(y_pred, y)
-            score = self._get_score(y_pred, y)
-
-        loss.backward()
-        self.optimizer.step()
-
-        self.train_running_loss += loss.item()
-        avg_loss = self.train_running_loss / (batch_idx + 1)
-
-        return score, avg_loss
-
-    def _eval_step(self, data: Dict[str, Tensor], target: Tensor, batch_idx: int):
-        self.model.eval()
-        with torch.no_grad():
-            X: Dict[str, Union[Tensor, List[Tensor]]] = {}
-            for k, v in data.items():
-                if isinstance(v, list):
-                    X[k] = [to_device(i, self.device) for i in v]
-                else:
-                    X[k] = to_device(v, self.device)
-            y = (
-                target.view(-1, 1).float()
-                if self.method not in ["multiclass", "qregression"]
-                else target
-            )
-            y = to_device(y, self.device)
-
-            y_pred = self.model(X)
-            if self.model.is_tabnet:  # pragma: no cover
-                loss = self.loss_fn(y_pred[0], y) - self.lambda_sparse * y_pred[1]
-                score = self._get_score(y_pred[0], y)
-            else:
-                score = self._get_score(y_pred, y)
-                loss = self.loss_fn(y_pred, y)
-
-            self.valid_running_loss += loss.item()
-            avg_loss = self.valid_running_loss / (batch_idx + 1)
-
-        return score, avg_loss
-
-    def _get_score(self, y_pred, y):  # pragma: no cover
-        if self.metric is not None:
-            if self.method == "regression":
-                score = self.metric(y_pred, y)
-            if self.method == "binary":
-                score = self.metric(torch.sigmoid(y_pred), y)
-            if self.method == "qregression":
-                score = self.metric(y_pred, y)
-            if self.method == "multiclass":
-                score = self.metric(F.softmax(y_pred, dim=1), y)
-            return score
-        else:
-            return None
-
-    def _predict(  # noqa: C901
+    def _predict(  # type: ignore[override]  # noqa: C901
         self,
         X_wide: Optional[np.ndarray] = None,
         X_tab: Optional[np.ndarray] = None,
@@ -606,7 +376,7 @@ def _predict(  # noqa: C901
         X_test: Optional[Dict[str, Union[np.ndarray, List[np.ndarray]]]] = None,
         test_loader: Optional[DataLoader] = None,
         batch_size: Optional[int] = None,
-        uncertainty_granularity=1000,
+        uncertainty_granularity: int = 1000,
         uncertainty: bool = False,
     ) -> List:
         r"""Private method to avoid code repetition in predict and
@@ -690,49 +460,3 @@ def _predict(  # noqa: C901
                             preds_l.append(preds)
         self.model.train()
         return preds_l
-
-    @staticmethod
-    def _predict_ziln(preds: Tensor) -> Tensor:  # pragma: no cover
-        """Calculates predicted mean of zero inflated lognormal logits.
-
-        Adjusted implementaion of `code
-        <https://github.com/google/lifetime_value/blob/master/lifetime_value/zero_inflated_lognormal.py>`
-
-        Arguments:
-            preds: [batch_size, 3] tensor of logits.
-        Returns:
-            ziln_preds: [batch_size, 1] tensor of predicted mean.
-        """
-        positive_probs = torch.sigmoid(preds[..., :1])
-        loc = preds[..., 1:2]
-        scale = F.softplus(preds[..., 2:])
-        ziln_preds = positive_probs * torch.exp(loc + 0.5 * torch.square(scale))
-        return ziln_preds
-
-    @staticmethod
-    def _extract_kwargs(kwargs):
-        finetune_params = [
-            "n_epochs",
-            "finetune_epochs",
-            "warmup_epochs",
-            "max_lr",
-            "finetune_max_lr",
-            "warmup_max_lr",
-            "routine",
-            "deeptabular_gradual",
-            "deeptabular_layers",
-            "deeptabular_max_lr",
-            "deeptext_gradual",
-            "deeptext_layers",
-            "deeptext_max_lr",
-            "deepimage_gradual",
-            "deepimage_layers",
-            "deepimage_max_lr",
-        ]
-
-        finetune_args = {}
-        for k, v in kwargs.items():
-            if k in finetune_params:
-                finetune_args[k] = v
-
-        return finetune_args
diff --git a/tests/test_model_functioning/test_fit_methods.py b/tests/test_model_functioning/test_fit_methods.py
index f873f75d..a1188c02 100644
--- a/tests/test_model_functioning/test_fit_methods.py
+++ b/tests/test_model_functioning/test_fit_methods.py
@@ -286,12 +286,12 @@ def test_custom_dataloader():
     )
     model = WideDeep(wide=wide, deeptabular=deeptabular)
     trainer = Trainer(model, loss="binary", verbose=0)
+
     trainer.fit(
         X_wide=X_wide,
         X_tab=X_tab,
         target=target_binary_imbalanced,
-        batch_size=16,
-        custom_dataloader=DataLoaderImbalanced,
+        custom_dataloader=DataLoaderImbalanced(batch_size=16),
     )
     # simply checking that runs with DataLoaderImbalanced
     assert "train_loss" in trainer.history.keys()
diff --git a/tests/test_rec/test_ranking_metrics_integration.py b/tests/test_rec/test_ranking_metrics_integration.py
index 1239ae54..81f8881b 100644
--- a/tests/test_rec/test_ranking_metrics_integration.py
+++ b/tests/test_rec/test_ranking_metrics_integration.py
@@ -1,5 +1,3 @@
-# silence DeprecationWarning
-import warnings
 from typing import Tuple
 
 import numpy as np
@@ -18,8 +16,8 @@
     BinaryNDCG_at_k,
 )
 from pytorch_widedeep.training import Trainer
+from pytorch_widedeep.dataloaders import CustomDataLoader
 from pytorch_widedeep.preprocessing import TabPreprocessor
-from pytorch_widedeep.training._wd_dataset import WideDeepDataset
 
 
 def generate_user_item_interactions(
@@ -89,8 +87,8 @@ def user_item_data(request):
 
 
 # strategy 1:
-# 	* ranking metrics for both training and validation sets.
-# 	* Same number of items per user in both sets
+#   * Same ranking metrics for both training and validation sets.
+#   * Same number of items per user in both sets
 
 
 @pytest.mark.parametrize(
@@ -100,11 +98,14 @@ def user_item_data(request):
     "metric",
     [
         MAP_at_k(n_cols=5, k=3),
-        NDCG_at_k(n_cols=5, k=3),
         Recall_at_k(n_cols=5, k=3),
         HitRatio_at_k(n_cols=5, k=3),
         Precision_at_k(n_cols=5, k=3),
         BinaryNDCG_at_k(n_cols=5, k=3),
+        [Accuracy(), MAP_at_k(n_cols=5, k=3)],
+        [F1Score(), MAP_at_k(n_cols=5, k=3)],
+        [Accuracy(), BinaryNDCG_at_k(n_cols=5, k=3)],
+        [F1Score(), BinaryNDCG_at_k(n_cols=5, k=3)],
     ],
 )
 def test_binary_classification_strategy_1(user_item_data, metric):
@@ -131,13 +132,17 @@ def test_binary_classification_strategy_1(user_item_data, metric):
     )
     model = WideDeep(deeptabular=tab_mlp)
 
-    trainer = Trainer(model, objective="binary", metrics=[metric])
+    if isinstance(metric, list):
+        trainer = Trainer(model, objective="binary", metrics=metric)
+    else:
+        trainer = Trainer(model, objective="binary", metrics=[metric])
 
     trainer.fit(
         X_train={"X_tab": X_train, "target": train_df[target_col].values},
         X_val={"X_tab": X_val, "target": val_df[target_col].values},
         n_epochs=1,
         batch_size=2 * 5,
+        verbose=0,
     )
 
     # predict on validation, this is just a test...
@@ -148,6 +153,322 @@ def test_binary_classification_strategy_1(user_item_data, metric):
         trainer.history is not None
         and "train_loss" in trainer.history
         and "val_loss" in trainer.history
-        and f"train_{metric._name}" in trainer.history
-        and f"val_{metric._name}" in trainer.history
     )
+    if not isinstance(metric, list):
+        metric = [metric]
+    for m in metric:
+        assert f"train_{m._name}" in trainer.history
+        assert f"val_{m._name}" in trainer.history
+
+
+@pytest.mark.parametrize(
+    "user_item_data", [{"validation_interactions_per_user": 5}], indirect=True
+)
+def test_multiclass_classification_strategy_1(user_item_data):
+
+    train_df, val_df = user_item_data
+
+    categorical_cols = ["user_id", "item_id", "user_category", "item_category"]
+
+    target_col = "rating"
+
+    tab_preprocessor = TabPreprocessor(
+        embed_cols=categorical_cols,
+        for_transformer=False,
+    )
+
+    tab_preprocessor.fit(train_df)
+
+    X_train = tab_preprocessor.transform(train_df)
+    X_val = tab_preprocessor.transform(val_df)
+
+    tab_mlp = TabMlp(
+        column_idx=tab_preprocessor.column_idx,
+        cat_embed_input=tab_preprocessor.cat_embed_input,  # type: ignore[arg-type]
+    )
+    model = WideDeep(deeptabular=tab_mlp, pred_dim=3)
+
+    trainer = Trainer(
+        model,
+        objective="multiclass",
+        metrics=[Accuracy(), NDCG_at_k(n_cols=5, k=3)],
+        verbose=0,
+    )
+
+    trainer.fit(
+        X_train={"X_tab": X_train, "target": train_df[target_col].values},
+        X_val={"X_tab": X_val, "target": val_df[target_col].values},
+        n_epochs=1,
+        batch_size=2 * 5,  # 2 * n_items
+    )
+
+    # predict on validation, this is just a test...
+    preds = trainer.predict(X_tab=X_val)
+
+    assert preds.shape[0] == X_val.shape[0]
+    assert (
+        trainer.history is not None
+        and "train_loss" in trainer.history
+        and "val_loss" in trainer.history
+    )
+
+
+# strategy 2:
+#   * Diff metrics for training and validation sets.
+#   * Same number of items per user in both sets
+
+
+@pytest.mark.parametrize(
+    "user_item_data", [{"validation_interactions_per_user": 5}], indirect=True
+)
+@pytest.mark.parametrize("target_col", ["liked", "rating"])
+def test_strategy_2(user_item_data, target_col):
+
+    train_df, val_df = user_item_data
+
+    categorical_cols = ["user_id", "item_id", "user_category", "item_category"]
+
+    tab_preprocessor = TabPreprocessor(
+        embed_cols=categorical_cols,
+        for_transformer=False,
+    )
+
+    tab_preprocessor.fit(train_df)
+
+    X_train = tab_preprocessor.transform(train_df)
+    X_val = tab_preprocessor.transform(val_df)
+
+    tab_mlp = TabMlp(
+        column_idx=tab_preprocessor.column_idx,
+        cat_embed_input=tab_preprocessor.cat_embed_input,  # type: ignore[arg-type]
+    )
+    model = WideDeep(deeptabular=tab_mlp, pred_dim=1 if target_col == "liked" else 3)
+
+    eval_metrics = (
+        [BinaryNDCG_at_k(n_cols=5, k=3)]
+        if target_col == "liked"
+        else [NDCG_at_k(n_cols=5, k=3)]
+    )
+    trainer = Trainer(
+        model,
+        objective="binary" if target_col == "liked" else "multiclass",
+        metrics=[Accuracy()],
+        eval_metrics=eval_metrics,
+        verbose=0,
+    )
+
+    trainer.fit(
+        X_train={"X_tab": X_train, "target": train_df[target_col].values},
+        X_val={"X_tab": X_val, "target": val_df[target_col].values},
+        n_epochs=1,
+        batch_size=2 * 5,
+    )
+
+    # predict on validation, this is just a test...
+    preds = trainer.predict(X_tab=X_val)
+
+    assert preds.shape[0] == X_val.shape[0]
+    assert (
+        trainer.history is not None
+        and "train_loss" in trainer.history
+        and "val_loss" in trainer.history
+        and "train_acc" in trainer.history
+        and "val_binary_ndcg@3" in trainer.history
+        if target_col == "liked"
+        else "val_ndcg@3" in trainer.history
+    )
+
+
+# strategy 3:
+#   * Diff number of items per user in both sets which implies CustomDataLoaders
+#   * everything else
+
+
+@pytest.mark.parametrize(
+    "user_item_data", [{"validation_interactions_per_user": 6}], indirect=True
+)
+@pytest.mark.parametrize(
+    "metrics",
+    [
+        [[BinaryNDCG_at_k(n_cols=4, k=3)], [BinaryNDCG_at_k(n_cols=6, k=3)]],
+        [[Accuracy(), F1Score()], [F1Score(), BinaryNDCG_at_k(n_cols=6, k=3)]],
+        [[Accuracy(), F1Score()], [F1Score(), MAP_at_k(n_cols=6, k=3)]],
+        [[Accuracy(), F1Score()], [F1Score(), Precision_at_k(n_cols=6, k=3)]],
+        [[Accuracy(), F1Score()], [F1Score(), Recall_at_k(n_cols=6, k=3)]],
+        [[Accuracy(), F1Score()], [F1Score(), HitRatio_at_k(n_cols=6, k=3)]],
+    ],
+)
+@pytest.mark.parametrize("with_train_data_loader", [True, False])
+def test_binary_classification_strategy_3(
+    user_item_data, metrics, with_train_data_loader
+):
+
+    train_metrics, val_metrics = metrics[0], metrics[1]
+
+    train_df, val_df = user_item_data
+
+    categorical_cols = ["user_id", "item_id", "user_category", "item_category"]
+
+    target_col = "liked"
+
+    tab_preprocessor = TabPreprocessor(
+        embed_cols=categorical_cols,
+        for_transformer=False,
+    )
+
+    tab_preprocessor.fit(train_df)
+
+    X_train = tab_preprocessor.transform(train_df)
+    X_val = tab_preprocessor.transform(val_df)
+
+    tab_mlp = TabMlp(
+        column_idx=tab_preprocessor.column_idx,
+        cat_embed_input=tab_preprocessor.cat_embed_input,  # type: ignore[arg-type]
+    )
+    model = WideDeep(deeptabular=tab_mlp)
+
+    trainer = Trainer(
+        model,
+        objective="binary",
+        metrics=train_metrics,
+        eval_metrics=val_metrics,
+    )
+
+    if with_train_data_loader:
+        train_dl = CustomDataLoader(
+            batch_size=2 * 4, shuffle=False  # in case there is a ranking metric
+        )
+    else:
+        train_dl = None
+
+    valid_dl = CustomDataLoader(
+        batch_size=2 * 6,
+        shuffle=False,
+    )
+
+    trainer.fit(
+        X_train={"X_tab": X_train, "target": train_df[target_col].values},
+        X_val={"X_tab": X_val, "target": val_df[target_col].values},
+        n_epochs=1,
+        batch_size=2
+        * 4,  # it only applies to the training set. Will be ignored if train_dl is not None
+        train_dataloader=train_dl,
+        eval_dataloader=valid_dl,
+    )
+
+    train_metric_names = [m._name for m in train_metrics]
+    val_metric_names = [m._name for m in val_metrics]
+
+    # predict on validation, this is just a test...
+    preds = trainer.predict(X_tab=X_val)
+
+    assert valid_dl.dataset.X_tab.shape[0] == X_val.shape[0] == 60
+
+    if with_train_data_loader:
+        assert train_dl.dataset.X_tab.shape[0] == X_train.shape[0] == 40
+
+    assert preds.shape[0] == X_val.shape[0]
+    assert (
+        trainer.history is not None
+        and "train_loss" in trainer.history
+        and "val_loss" in trainer.history
+    )
+
+    for train_metric_name in train_metric_names:
+        assert f"train_{train_metric_name}" in trainer.history
+
+    for val_metric_name in val_metric_names:
+        assert f"val_{val_metric_name}" in trainer.history
+
+
+@pytest.mark.parametrize(
+    "user_item_data", [{"validation_interactions_per_user": 6}], indirect=True
+)
+@pytest.mark.parametrize(
+    "metrics",
+    [
+        [[NDCG_at_k(n_cols=4, k=3)], [NDCG_at_k(n_cols=6, k=3)]],
+        [[Accuracy(), F1Score()], [F1Score(), NDCG_at_k(n_cols=6, k=3)]],
+    ],
+)
+@pytest.mark.parametrize("with_train_data_loader", [True, False])
+def test_multiclass_classification_strategy_3(
+    user_item_data, metrics, with_train_data_loader
+):
+
+    train_metrics, val_metrics = metrics[0], metrics[1]
+
+    train_df, val_df = user_item_data
+
+    categorical_cols = ["user_id", "item_id", "user_category", "item_category"]
+
+    target_col = "rating"
+
+    tab_preprocessor = TabPreprocessor(
+        embed_cols=categorical_cols,
+        for_transformer=False,
+    )
+
+    tab_preprocessor.fit(train_df)
+
+    X_train = tab_preprocessor.transform(train_df)
+    X_val = tab_preprocessor.transform(val_df)
+
+    tab_mlp = TabMlp(
+        column_idx=tab_preprocessor.column_idx,
+        cat_embed_input=tab_preprocessor.cat_embed_input,  # type: ignore[arg-type]
+    )
+    model = WideDeep(deeptabular=tab_mlp, pred_dim=3)
+
+    trainer = Trainer(
+        model,
+        objective="multiclass",
+        metrics=train_metrics,
+        eval_metrics=val_metrics,
+    )
+
+    if with_train_data_loader:
+        train_dl = CustomDataLoader(
+            batch_size=2 * 4, shuffle=False  # in case there is a ranking metric
+        )
+    else:
+        train_dl = None
+
+    valid_dl = CustomDataLoader(
+        batch_size=2 * 6,
+        shuffle=False,
+    )
+
+    trainer.fit(
+        X_train={"X_tab": X_train, "target": train_df[target_col].values},
+        X_val={"X_tab": X_val, "target": val_df[target_col].values},
+        n_epochs=1,
+        batch_size=2
+        * 4,  # it only applies to the training set. Will be ignored if train_dl is not None
+        train_dataloader=train_dl,
+        eval_dataloader=valid_dl,
+    )
+
+    train_metric_names = [m._name for m in train_metrics]
+    val_metric_names = [m._name for m in val_metrics]
+
+    # predict on validation, this is just a test...
+    preds = trainer.predict(X_tab=X_val)
+
+    assert valid_dl.dataset.X_tab.shape[0] == X_val.shape[0] == 60
+
+    if with_train_data_loader:
+        assert train_dl.dataset.X_tab.shape[0] == X_train.shape[0] == 40
+
+    assert preds.shape[0] == X_val.shape[0]
+    assert (
+        trainer.history is not None
+        and "train_loss" in trainer.history
+        and "val_loss" in trainer.history
+    )
+
+    for train_metric_name in train_metric_names:
+        assert f"train_{train_metric_name}" in trainer.history
+
+    for val_metric_name in val_metric_names:
+        assert f"val_{val_metric_name}" in trainer.history

From 808a7bbbedbc6908ca8c61966ce5e0488c391e6b Mon Sep 17 00:00:00 2001
From: Javier <jrzaurin@gmail.com>
Date: Mon, 21 Oct 2024 10:33:48 +0100
Subject: [PATCH 12/24] remove unnecessary, legacy docs folder and modified
 figures path in README

---
 mkdocs/site/index.html                        |   3 +-
 mkdocs/site/objects.inv                       | Bin 1996 -> 2007 bytes
 .../pytorch-widedeep/bayesian_trainer.html    |  33 +-
 mkdocs/site/pytorch-widedeep/dataloaders.html |  87 ++--
 mkdocs/site/pytorch-widedeep/losses.html      | 432 +++++++++--------
 mkdocs/site/pytorch-widedeep/metrics.html     |  72 +--
 .../site/pytorch-widedeep/preprocessing.html  | 297 ++++++++++++
 mkdocs/site/pytorch-widedeep/preprocessing.md |   2 +
 mkdocs/site/pytorch-widedeep/tab2vec.html     |  73 +--
 mkdocs/site/pytorch-widedeep/trainer.html     | 435 ++++++++----------
 mkdocs/site/search/search_index.json          |   2 +-
 mkdocs/site/sitemap.xml                       |  88 ++--
 mkdocs/site/sitemap.xml.gz                    | Bin 796 -> 795 bytes
 13 files changed, 869 insertions(+), 655 deletions(-)

diff --git a/mkdocs/site/index.html b/mkdocs/site/index.html
index 1f68a10a..666a04ab 100644
--- a/mkdocs/site/index.html
+++ b/mkdocs/site/index.html
@@ -2382,8 +2382,7 @@ <h4 id="apa">APA<a class="headerlink" href="#apa" title="Permanent link">&para;<
       
         <a href="mailto:jrzaurin@gmail.com">Javier</a>, 
         <a href="mailto:mulinka.pavol@gmail.com">Pavol Mulinka</a>, 
-        <a href="mailto:javierrodriguezzaurin@javiers-macbook-pro.local">Javier Rodriguez Zaurin</a>, 
-        <a href="mailto:not.committed.yet">Not Committed Yet</a>
+        <a href="mailto:javierrodriguezzaurin@javiers-macbook-pro.local">Javier Rodriguez Zaurin</a>
     </nav>
   </span>
 
diff --git a/mkdocs/site/objects.inv b/mkdocs/site/objects.inv
index 2818ee0861c603228b2f2d166adb8ebdc801c0d6..2208df9aa6290a7b647eaabf35faa393dd5d954b 100644
GIT binary patch
delta 1881
zcmV-f2d4PU57!TnlYdQ)d&?x_N~#=J)=te<?S+y>V`~F~l|(+~*RLeNU)V-CYB84t
zWa#&4b+-TsyL?xLpzHXRC6EA=(UQD_WQ50gkw7LRMV3`Y#I`+S9Cr83nbEgq?{B^H
zseJ#FW<<(QU-hOn|G72)<<R_E<=O9leER%bIT$3*b`H{?+JA-?<cTaFveFX@C0Vv4
z^jSvViOAj;swhjwS7)tn&vCIko!>F3KYjOT^TD`vvj+NH7L2P8TTX}~SwRvIGP>2<
zQ@eeiFG)r?h2*Tj?#ZVm%n@-aiabt>OdlJuoUEY!yVrhiS|++-V)xHVcZ?3uZ<DV1
zzxSC=v5^T(wSR>`fH!6T?7OcU0*JpRvB@0?s^j{tKH;|h3<E?;kel&j@qMZrdW_pM
z2AYi0Lc9`Tt0%2ERjlA2v3?7@U{a4x-gx&c`o{gvsUD`pE!cnFn7d|ugHH-QW_sh@
zSVO^_$J4KVQCo{hxDOu7ZQmpk@En17j)Z(yEj?D+#($b`%zK^|vm1ZCx?|5bs>kZW
zDJ!4ZiIkNOi>Xh>DGKCY72%3y;0<)wpl?{OV-nV2H1`_!Iac7GUI7y3FYY_`e53wz
zKb?8wY>jz_jPoi}OcAjH6+M{kC%b)Pcv;FQF#^84IHfj+l>n`Wrlc$TQ0bZ{zMafL
z2}WgfO@C=62t7ZA&>rk=F4E+n`U3B6ck*>X3jv-e8*APnzkwoNIX4D<qdp3t%%qkl
z(v3aepz|xA9oks)4hd!|%on3I<_)o)rmUyLUg{A2aEQD)Ev7|Y796;`Od%>NRaPoa
zSZ>6KQo@f`eMVYY5j$f!sbtSAjyILiUPYxS{(lA9p9_Npb<IXgA{mX9AYAIJb#A8U
zw%cbrRIMS_7R*F=m4Ta@*-z6vGLvu<23c~g?+;)?s6K`Lw3oIALf4D~j&!3OM2~SR
zcFz^)T2?bquZ4K_uJhJ7_Wex>$RUV-?euc9s$&;(dKYnq=jSOI*Za!L70sd$Ex4V&
z)_(=APu3E4spzI|y>1&HrjJNo4`PV6N4x@Ct(EjUHeCjVHB#=<NXXIyl!a~z6d(un
zkfCwG2@#ZbVbx(l)u+%1zBP*lNG|FN2iFgUMzYz|s&l&)UJ<=_&uHUzv;N--H>xwV
z4nC@d-nB<nkvnyJA1R|$?cHNeo?P!T1Apj;8lc^Q(6eEeL=whTw79;XJ%xzGj@&36
zzpEd#pH1zhM5^{Ni!3L4ZxXBo&gqX5iL*e#LsNpFz+P&Q$95-FV2-Pg7M2q@!Gp3l
zFF}@pFB<0wMhngvB(wmTm$}x}YQ^{}B^2U4sC8Pu4H?}{?6j5d*bEZygax4|1b^qU
z)p>zf??%_{-h4L0@V`eAJsCF)^@O3(|7Nx@-R(-@Cb$)qiqL0B;{Co})U83H>`V|Y
z=<Y~Z0W$lsZD?QO^ws~RfKn9HthDl#nYN9$Erp0~?`MxY`)5`6O}R?y5o}NKW?k{;
zyGdZxkp-i~K8*HF>7qyjA5A)CYJV)<?PYWn0><?nrG$|wei8LplDy22Qn3Ca#2$zJ
zCIoW5yU2#fLnjVo2K!9a8RN~f6>bbVy7jArKOu<N2>u;N;*GB^%5mtdI%Ehm#`bbx
z>zmccBju%=vGcE^{7Z-;roEGP96aD1icqQFPx1>9px$QKC$w%An#VQdf`4#S#;Xcf
zfRYHsNEQbK8KwKj?D=Eb*rR|0MWFG>fy6t3ed$D2C5RU$Au%p8ZcZ_;1PLO|)X?VZ
zpikz*p4-nugBoOG(=SHUn(pfh&q({<G##N&jslF;8+03>IB6qPf2^ULvgqmpNI!`j
z#L?rC8O`Y}&JgeGX*LnpGk<+z<9AOn;Cgvn9L=pU0@(v{#8>@5cl83)JjY%A0DAu2
z;pW)Wb$?*Z<!0a_qwd$%;pTmsZsW4~)ftE3fBTa)P56e}cj<Q@GCHvmMng*J;wY9R
zgrFoZfT81(0>jNYRoZZ?jsd>hvRgbqU|(AWq0Eb5?+7X`Nix7}Cx4$#jvH65$>=^N
zeFUtNe*XEa#D^G<B{KqNE8~OpNO|B9H)0>+?i2(9uGKh4+1hof8b=0~A~}$Csfl6g
zy(Mr+;F$3PDO?i#1c_TdU&iB*zNe^8T_pUm?J5i*bg8F-NT<rj{2GC@-<@oKO0I61
zk|ku#gT>jq7JK-Yu782k7d{tBmpdCgW%maSB!<D0iAWp+=ChFq(0jnVpibDfr)Bh1
zJOl4tLa+6@XMaF1YGz~O=C=m=-NlhND^B6`e-xA~19i34il_jhAh3t-Bbq<S3x^SZ
z0KCdUFiJ9=U!k>47PIs{xGZ*G*RMO(+t;!Ls_btpIIjWLx>CDVJSgt0Qn~qkss4hQ
z1@1u3cEwQrv@8Af=CWd05<|X(B+<8VGR}z-?9EMc$IbxB6(~FMOH7fHB(a45D$sYv
Ts6x_)E*=`LV`BdUpl4e@;*O~J

delta 1870
zcmV-U2eJ6q56lmclYb@0y=9WPk}4;awNtZId!b~}*xG<#C6SN$^(zVR7q$_OTFfN@
z8Tx%%-7P@EE<aQu=qi3=2_yhzv>+cK8R2nWB#_BSkwujev1!j3hwXiHX7s(;`&;jP
zDL?+C8IkhKh2FI0zqaPT?VDezJp28RFJFHv2ZQ9<)<ODH+keo4Jd?#kR(e9AB+C|r
zzR2j7i0osoin3&UdD8mk92dLO*&UPm(+`g}AB<bqE1)lB!MOUg<%B4b6(j*6qZ_?F
zwcGdkf@FkKNKOiDpL|@x91*9Y$m6ug^sy1k$r9?nJMH(TWuhA<cK@Vw$LIk4HtCxG
zJD=$k8=1gVTYm@ycvtq%uKT(nfcRSyo7|D0I<DX76K?9yFhHaPxgJmEw^QBFW89uG
z&}5Vr;*AJfJ!!?MVg>(<^&8*?lX`UW#=B$DH|{N`dYBS7VE=hz?wa)tJ}LB=>5X@7
z4Fz)+PcQtUwic0aA3T=Zu1O@|83OSP3Hh#Cc&xOwHGkij_dG2gul@DvmObC79;*wd
ztbArCQdU09r#=~{D3E_uge#VTH_&Z^zG1zNNmzr?%xm0dSb=|j14x*^xNq6>jr!01
z^wAq<Ys@oboL8A*iijns=)r71+wE(^%Th*(5%A^3F|`>i1!z4qC0*J3N>@Db?PLy0
zFe;-fN`EUs=;<kh_F#8&ktPS#7kGEOlZ!bm1bCusta*q04vKi?TpRR_`XqodlUklg
z*Y<pa&Mtj+Xk*PgB$%l%UyRn6H^h3HvYrxqtwZ#~A@b(5m=<|iaNz1Zg{Y`hS*bW-
zxe+5u2|rr(8EIuj?2P54l0CCH-c&++6_uj+7k_AfE({jbH5)C6WHeTSaH+4>xtXHd
zY@h5<wSrh%FcaZb25xF*KTY$<Ou|hVWXYAjKY$6L`V{ulUfLcAT`>+g(v5NuJ;trr
zJy)P>Sv`V!E5wU;o%hDE+jk`(hamp7)631Oj$O>@UBnrlpT}fe?<>z&G>byC;CA|2
z7k{`uSxeZZqU*Z#s%?CjJ|THMh#}e>@dj+UQqu3(bRHDeNV!cTAxrmA=DI0RfE?6A
zhQ<XaL{Qp=Rfh#tpF$(})+`nvxu`E3Tt5^V$!1fl&g@orMfBc1p^e+k`hP3js6L`~
z@Ifu~u05!V+^L)UNExMS=N@zN?0T0OKz~2f0PPNho(<b1k}$5K`PKd7bBIW6$&J$S
z+xkKK+0<T2q-r0t$YP@RCc#SJl>R7@I0+OyG$jZM?6n4YYIi~f=CJx`VL5^mJSaQ!
z5@ZqhqH&sFwBVdTLJN?2nQK)omy9n{LLuIPTBY^dkkQ@5PFwko%^>kkSP*(baDOga
zomYtUZgkb|%^n{y{O^%OPsTMvJz;3{znd*gce_%!32sHDBJ>54c(<?Tb!(6)I}?Np
zx;qk9fXseu8`_sRef57SpcDl)E3JHKhLcT0A)@P5#b53wPg`v#h4-wTsXDS?l-MW3
zzWG%YY2c$tr%a8dySb)~LcqAby?>K1GQ}^V9!rv!8Bz+?-%8lwuwP3+u6GyN5P9gt
zfy`hZa5`hWdC<X)K}WZKkMAc05gWn3JxRRr)kQfDomKk`fyUTg4s3n1G<cxAbTfAT
zb(DVzQN*-&(vE`%yh9Nx_4`SFAp+Ff4Ewm$twQtcgj^7g%6L`b3Q!WE7=Ov)fFPrE
z|Cl|0NE>?;aG(e@9yySBC$MjT$f^YK$|NMlMaIo3=KUN&q?xB#e>d~leAsjQxo=Q|
ztZn+mh+5Nqec>5t_iLXc^vOYhv3i4U0u(20gzAqqlv5U6o&o76k%KsTI5ML-y}=pc
zeLc-4;(De}to`mO23#)>i+`iJHAWzNKo0n-ALuS$ftu&I%O5~bzsTDhd%W)Vthw9_
zoMqJg+B)34Pt$E&);|H`F#K<RKc)%aaQiO(?n6c=R>EjVDP0`Jf`ky1<P|V<TvA}T
zIj2e+Zq+fs=UaA*=O^rIt00tlE9(P6#U)7wnC;}#(Q)I_H5uK<q<_zVb<)p2pOyF&
z<Dq0m;A~}lupTH6JmN;|Q{0_`K)|&crzl&yE>+{m;8G-evMx0-Oue@R_6Zy_ejtTQ
zf*&Dq%jffW9Mbm~)v=3&Kela!A%rgVG!W@n`Iw&ukoJp=?eDtPEmN|9ta-3FdDmhO
z|JF5d{KDrV>2ha-r+@7Jpn=3Ncrp=*W59ei5&?P-n773V+w`=Io{JaYy-Vn|Uia)z
z=ta$JY}|bDzu#RPd9&gaPX9+i$s$l!Tdjx+APNF|=su$PlRR@6@dv=G90a2z)A<!z
z+hj3IKZ46*_jUccQ@wp9OQ6dB;DPfRV6D4t#e?G3DwUhx=SS+#nOWct<ZM?A)la+9
zUvDldmL)Od3rG@u8z<wOD8b&{G`H*wkX(YYBfrEHDM=Db_^$$eXN)Q&ZRp~m;W{Mt
IKYYqBWcp32B>(^b

diff --git a/mkdocs/site/pytorch-widedeep/bayesian_trainer.html b/mkdocs/site/pytorch-widedeep/bayesian_trainer.html
index 51e79f11..2b758b4b 100644
--- a/mkdocs/site/pytorch-widedeep/bayesian_trainer.html
+++ b/mkdocs/site/pytorch-widedeep/bayesian_trainer.html
@@ -1755,7 +1755,8 @@ <h2 id="pytorch_widedeep.training.BayesianTrainer" class="doc doc-heading">
 <ul>
 <li>
 <p><strong>device</strong>: <code>str</code><br/>
-    string indicating the device. One of <em>'cpu'</em> or <em>'gpu'</em></p>
+    string indicating the device. One of <em>'cpu'</em>, <em>'gpu'</em> or 'mps' if
+    run on a Mac with Apple silicon or AMD GPU(s)</p>
 </li>
 <li>
 <p><strong>num_workers</strong>: <code>int</code><br/>
@@ -1794,8 +1795,7 @@ <h2 id="pytorch_widedeep.training.BayesianTrainer" class="doc doc-heading">
 
                   <details class="quote">
                     <summary>Source code in <code>pytorch_widedeep/training/bayesian_trainer.py</code></summary>
-                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">111</span>
-<span class="normal">112</span>
+                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">112</span>
 <span class="normal">113</span>
 <span class="normal">114</span>
 <span class="normal">115</span>
@@ -1822,7 +1822,8 @@ <h2 id="pytorch_widedeep.training.BayesianTrainer" class="doc doc-heading">
 <span class="normal">136</span>
 <span class="normal">137</span>
 <span class="normal">138</span>
-<span class="normal">139</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="nd">@alias</span><span class="p">(</span>  <span class="c1"># noqa: C901</span>
+<span class="normal">139</span>
+<span class="normal">140</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="nd">@alias</span><span class="p">(</span>  <span class="c1"># noqa: C901</span>
     <span class="s2">&quot;objective&quot;</span><span class="p">,</span>
     <span class="p">[</span><span class="s2">&quot;loss_function&quot;</span><span class="p">,</span> <span class="s2">&quot;loss_fn&quot;</span><span class="p">,</span> <span class="s2">&quot;loss&quot;</span><span class="p">,</span> <span class="s2">&quot;cost_function&quot;</span><span class="p">,</span> <span class="s2">&quot;cost_fn&quot;</span><span class="p">,</span> <span class="s2">&quot;cost&quot;</span><span class="p">],</span>
 <span class="p">)</span>
@@ -1998,8 +1999,7 @@ <h3 id="pytorch_widedeep.training.BayesianTrainer.fit" class="doc doc-heading">
 
             <details class="quote">
               <summary>Source code in <code>pytorch_widedeep/training/bayesian_trainer.py</code></summary>
-              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">141</span>
-<span class="normal">142</span>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">142</span>
 <span class="normal">143</span>
 <span class="normal">144</span>
 <span class="normal">145</span>
@@ -2112,7 +2112,8 @@ <h3 id="pytorch_widedeep.training.BayesianTrainer.fit" class="doc doc-heading">
 <span class="normal">252</span>
 <span class="normal">253</span>
 <span class="normal">254</span>
-<span class="normal">255</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">fit</span><span class="p">(</span>  <span class="c1"># noqa: C901</span>
+<span class="normal">255</span>
+<span class="normal">256</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">fit</span><span class="p">(</span>  <span class="c1"># noqa: C901</span>
     <span class="bp">self</span><span class="p">,</span>
     <span class="n">X_tab</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">,</span>
     <span class="n">target</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">,</span>
@@ -2308,8 +2309,7 @@ <h3 id="pytorch_widedeep.training.BayesianTrainer.predict" class="doc doc-headin
 
             <details class="quote">
               <summary>Source code in <code>pytorch_widedeep/training/bayesian_trainer.py</code></summary>
-              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">257</span>
-<span class="normal">258</span>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">258</span>
 <span class="normal">259</span>
 <span class="normal">260</span>
 <span class="normal">261</span>
@@ -2344,7 +2344,8 @@ <h3 id="pytorch_widedeep.training.BayesianTrainer.predict" class="doc doc-headin
 <span class="normal">290</span>
 <span class="normal">291</span>
 <span class="normal">292</span>
-<span class="normal">293</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">predict</span><span class="p">(</span>  <span class="c1"># type: ignore[return]</span>
+<span class="normal">293</span>
+<span class="normal">294</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">predict</span><span class="p">(</span>  <span class="c1"># type: ignore[return]</span>
     <span class="bp">self</span><span class="p">,</span>
     <span class="n">X_tab</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">,</span>
     <span class="n">n_samples</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">5</span><span class="p">,</span>
@@ -2462,8 +2463,7 @@ <h3 id="pytorch_widedeep.training.BayesianTrainer.predict_proba" class="doc doc-
 
             <details class="quote">
               <summary>Source code in <code>pytorch_widedeep/training/bayesian_trainer.py</code></summary>
-              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">295</span>
-<span class="normal">296</span>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">296</span>
 <span class="normal">297</span>
 <span class="normal">298</span>
 <span class="normal">299</span>
@@ -2505,7 +2505,8 @@ <h3 id="pytorch_widedeep.training.BayesianTrainer.predict_proba" class="doc doc-
 <span class="normal">335</span>
 <span class="normal">336</span>
 <span class="normal">337</span>
-<span class="normal">338</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">predict_proba</span><span class="p">(</span>  <span class="c1"># type: ignore[return]</span>
+<span class="normal">338</span>
+<span class="normal">339</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">predict_proba</span><span class="p">(</span>  <span class="c1"># type: ignore[return]</span>
     <span class="bp">self</span><span class="p">,</span>
     <span class="n">X_tab</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">,</span>
     <span class="n">n_samples</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">5</span><span class="p">,</span>
@@ -2618,8 +2619,7 @@ <h3 id="pytorch_widedeep.training.BayesianTrainer.save" class="doc doc-heading">
 
             <details class="quote">
               <summary>Source code in <code>pytorch_widedeep/training/bayesian_trainer.py</code></summary>
-              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">340</span>
-<span class="normal">341</span>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">341</span>
 <span class="normal">342</span>
 <span class="normal">343</span>
 <span class="normal">344</span>
@@ -2665,7 +2665,8 @@ <h3 id="pytorch_widedeep.training.BayesianTrainer.save" class="doc doc-heading">
 <span class="normal">384</span>
 <span class="normal">385</span>
 <span class="normal">386</span>
-<span class="normal">387</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">save</span><span class="p">(</span>
+<span class="normal">387</span>
+<span class="normal">388</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">save</span><span class="p">(</span>
     <span class="bp">self</span><span class="p">,</span>
     <span class="n">path</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
     <span class="n">save_state_dict</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
diff --git a/mkdocs/site/pytorch-widedeep/dataloaders.html b/mkdocs/site/pytorch-widedeep/dataloaders.html
index 45f846f4..3453782e 100644
--- a/mkdocs/site/pytorch-widedeep/dataloaders.html
+++ b/mkdocs/site/pytorch-widedeep/dataloaders.html
@@ -1610,14 +1610,12 @@ <h2 id="pytorch_widedeep.dataloaders.DataLoaderImbalanced" class="doc doc-headin
 
 
 <a href="#pytorch_widedeep.dataloaders.DataLoaderImbalanced" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="nf">DataLoaderImbalanced</span><span class="p">(</span>
-    <span class="n">dataset</span><span class="p">,</span> <span class="n">batch_size</span><span class="p">,</span> <span class="n">num_workers</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span>
-<span class="p">)</span>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">DataLoaderImbalanced</span><span class="p">(</span><span class="n">dataset</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
 </code></pre></div>
 
     <div class="doc doc-contents first">
             <p class="doc doc-class-bases">
-              Bases: <code><span title="torch.utils.data.DataLoader">DataLoader</span></code></p>
+              Bases: <code><span title="pytorch_widedeep.dataloaders.CustomDataLoader">CustomDataLoader</span></code></p>
 
 
         <p>Class to load and shuffle batches with adjusted weights for imbalanced
@@ -1629,33 +1627,26 @@ <h2 id="pytorch_widedeep.dataloaders.DataLoaderImbalanced" class="doc doc-headin
     <ul>
         <li class="doc-section-item field-body">
           <b><code>dataset</code></b>
-              (<code><span title="pytorch_widedeep.training._wd_dataset.WideDeepDataset">WideDeepDataset</span></code>)
+              (<code><span title="typing.Optional">Optional</span>[<span title="pytorch_widedeep.training._wd_dataset.WideDeepDataset">WideDeepDataset</span>]</code>, default:
+                  <code>None</code>
+)
           –
           <div class="doc-md-description">
             <p>see <code>pytorch_widedeep.training._wd_dataset</code></p>
           </div>
         </li>
-        <li class="doc-section-item field-body">
-          <b><code>batch_size</code></b>
-              (<code>int</code>)
-          –
-          <div class="doc-md-description">
-            <p>size of batch</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>num_workers</code></b>
-              (<code>int</code>)
-          –
-          <div class="doc-md-description">
-            <p>number of workers</p>
-          </div>
-        </li>
     </ul>
 
 
 <p><span class="doc-section-title">Other Parameters:</span></p>
     <ul>
+        <li class="doc-section-item field-body">
+          <b><code>*args</code></b>
+          –
+          <div class="doc-md-description">
+            <p>Positional arguments to be passed to the parent CustomDataLoader.</p>
+          </div>
+        </li>
         <li class="doc-section-item field-body">
           <b><code>**kwargs</code></b>
           –
@@ -1677,45 +1668,23 @@ <h2 id="pytorch_widedeep.dataloaders.DataLoaderImbalanced" class="doc doc-headin
 
                   <details class="quote">
                     <summary>Source code in <code>pytorch_widedeep/dataloaders.py</code></summary>
-                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">72</span>
-<span class="normal">73</span>
-<span class="normal">74</span>
-<span class="normal">75</span>
-<span class="normal">76</span>
-<span class="normal">77</span>
-<span class="normal">78</span>
-<span class="normal">79</span>
-<span class="normal">80</span>
-<span class="normal">81</span>
-<span class="normal">82</span>
-<span class="normal">83</span>
-<span class="normal">84</span>
-<span class="normal">85</span>
-<span class="normal">86</span>
-<span class="normal">87</span>
-<span class="normal">88</span>
-<span class="normal">89</span>
-<span class="normal">90</span>
-<span class="normal">91</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
-    <span class="bp">self</span><span class="p">,</span> <span class="n">dataset</span><span class="p">:</span> <span class="n">WideDeepDataset</span><span class="p">,</span> <span class="n">batch_size</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span> <span class="n">num_workers</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span>
-<span class="p">):</span>
-    <span class="k">assert</span> <span class="n">dataset</span><span class="o">.</span><span class="n">Y</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">,</span> <span class="p">(</span>
-        <span class="s2">&quot;The &#39;dataset&#39; instance of WideDeepDataset must contain a &quot;</span>
-        <span class="s2">&quot;target array &#39;Y&#39;&quot;</span>
-    <span class="p">)</span>
-
-    <span class="k">if</span> <span class="s2">&quot;oversample_mul&quot;</span> <span class="ow">in</span> <span class="n">kwargs</span><span class="p">:</span>
-        <span class="n">oversample_mul</span> <span class="o">=</span> <span class="n">kwargs</span><span class="p">[</span><span class="s2">&quot;oversample_mul&quot;</span><span class="p">]</span>
-        <span class="k">del</span> <span class="n">kwargs</span><span class="p">[</span><span class="s2">&quot;oversample_mul&quot;</span><span class="p">]</span>
+                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">109</span>
+<span class="normal">110</span>
+<span class="normal">111</span>
+<span class="normal">112</span>
+<span class="normal">113</span>
+<span class="normal">114</span>
+<span class="normal">115</span>
+<span class="normal">116</span>
+<span class="normal">117</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">dataset</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">WideDeepDataset</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">args</span> <span class="o">=</span> <span class="n">args</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">kwargs</span> <span class="o">=</span> <span class="n">kwargs</span>
+
+    <span class="k">if</span> <span class="n">dataset</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">_setup_sampler</span><span class="p">(</span><span class="n">dataset</span><span class="p">)</span>
+        <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">dataset</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="n">sampler</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">sampler</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
     <span class="k">else</span><span class="p">:</span>
-        <span class="n">oversample_mul</span> <span class="o">=</span> <span class="mi">1</span>
-    <span class="n">weights</span><span class="p">,</span> <span class="n">minor_cls_cnt</span><span class="p">,</span> <span class="n">num_clss</span> <span class="o">=</span> <span class="n">get_class_weights</span><span class="p">(</span><span class="n">dataset</span><span class="p">)</span>
-    <span class="n">num_samples</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">minor_cls_cnt</span> <span class="o">*</span> <span class="n">num_clss</span> <span class="o">*</span> <span class="n">oversample_mul</span><span class="p">)</span>
-    <span class="n">samples_weight</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">([</span><span class="n">weights</span><span class="p">[</span><span class="n">i</span><span class="p">]</span> <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="n">dataset</span><span class="o">.</span><span class="n">Y</span><span class="p">]))</span>
-    <span class="n">sampler</span> <span class="o">=</span> <span class="n">WeightedRandomSampler</span><span class="p">(</span><span class="n">samples_weight</span><span class="p">,</span> <span class="n">num_samples</span><span class="p">,</span> <span class="n">replacement</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
-    <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span>
-        <span class="n">dataset</span><span class="p">,</span> <span class="n">batch_size</span><span class="p">,</span> <span class="n">num_workers</span><span class="o">=</span><span class="n">num_workers</span><span class="p">,</span> <span class="n">sampler</span><span class="o">=</span><span class="n">sampler</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span>
-    <span class="p">)</span>
+        <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
 </code></pre></div></td></tr></table></div>
                   </details>
 
diff --git a/mkdocs/site/pytorch-widedeep/losses.html b/mkdocs/site/pytorch-widedeep/losses.html
index 39ad8311..91b430e8 100644
--- a/mkdocs/site/pytorch-widedeep/losses.html
+++ b/mkdocs/site/pytorch-widedeep/losses.html
@@ -2756,8 +2756,7 @@ <h3 id="pytorch_widedeep.losses.FocalLoss.forward" class="doc doc-heading">
 <span class="normal">278</span>
 <span class="normal">279</span>
 <span class="normal">280</span>
-<span class="normal">281</span>
-<span class="normal">282</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">forward</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="nb">input</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">,</span> <span class="n">target</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tensor</span><span class="p">:</span>
+<span class="normal">281</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">forward</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="nb">input</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">,</span> <span class="n">target</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tensor</span><span class="p">:</span>
 <span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;</span>
 <span class="sd">    Parameters</span>
 <span class="sd">    ----------</span>
@@ -2789,8 +2788,7 @@ <h3 id="pytorch_widedeep.losses.FocalLoss.forward" class="doc doc-heading">
     <span class="k">else</span><span class="p">:</span>
         <span class="n">num_class</span> <span class="o">=</span> <span class="n">input_prob</span><span class="o">.</span><span class="n">size</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span>
     <span class="n">binary_target</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">eye</span><span class="p">(</span><span class="n">num_class</span><span class="p">)[</span><span class="n">target</span><span class="o">.</span><span class="n">squeeze</span><span class="p">()</span><span class="o">.</span><span class="n">cpu</span><span class="p">()</span><span class="o">.</span><span class="n">long</span><span class="p">()]</span>
-    <span class="k">if</span> <span class="n">use_cuda</span><span class="p">:</span>
-        <span class="n">binary_target</span> <span class="o">=</span> <span class="n">binary_target</span><span class="o">.</span><span class="n">cuda</span><span class="p">()</span>
+    <span class="n">binary_target</span> <span class="o">=</span> <span class="n">binary_target</span><span class="o">.</span><span class="n">to</span><span class="p">(</span><span class="nb">input</span><span class="o">.</span><span class="n">device</span><span class="p">)</span>
     <span class="n">binary_target</span> <span class="o">=</span> <span class="n">binary_target</span><span class="o">.</span><span class="n">contiguous</span><span class="p">()</span>
     <span class="n">weight</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_weight</span><span class="p">(</span><span class="n">input_prob</span><span class="p">,</span> <span class="n">binary_target</span><span class="p">)</span>
 
@@ -2834,8 +2832,8 @@ <h2 id="pytorch_widedeep.losses.BayesianSELoss" class="doc doc-heading">
 
                   <details class="quote">
                     <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
-                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">311</span>
-<span class="normal">312</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">310</span>
+<span class="normal">311</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
     <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
 </code></pre></div></td></tr></table></div>
                   </details>
@@ -2898,7 +2896,8 @@ <h3 id="pytorch_widedeep.losses.BayesianSELoss.forward" class="doc doc-heading">
 
             <details class="quote">
               <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
-              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">314</span>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">313</span>
+<span class="normal">314</span>
 <span class="normal">315</span>
 <span class="normal">316</span>
 <span class="normal">317</span>
@@ -2914,8 +2913,7 @@ <h3 id="pytorch_widedeep.losses.BayesianSELoss.forward" class="doc doc-heading">
 <span class="normal">327</span>
 <span class="normal">328</span>
 <span class="normal">329</span>
-<span class="normal">330</span>
-<span class="normal">331</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">forward</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="nb">input</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">,</span> <span class="n">target</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tensor</span><span class="p">:</span>
+<span class="normal">330</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">forward</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="nb">input</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">,</span> <span class="n">target</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tensor</span><span class="p">:</span>
 <span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;</span>
 <span class="sd">    Parameters</span>
 <span class="sd">    ----------</span>
@@ -2971,8 +2969,8 @@ <h2 id="pytorch_widedeep.losses.TweedieLoss" class="doc doc-heading">
 
                   <details class="quote">
                     <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
-                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">343</span>
-<span class="normal">344</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">342</span>
+<span class="normal">343</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
     <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
 </code></pre></div></td></tr></table></div>
                   </details>
@@ -3047,7 +3045,8 @@ <h3 id="pytorch_widedeep.losses.TweedieLoss.forward" class="doc doc-heading">
 
             <details class="quote">
               <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
-              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">346</span>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">345</span>
+<span class="normal">346</span>
 <span class="normal">347</span>
 <span class="normal">348</span>
 <span class="normal">349</span>
@@ -3082,8 +3081,7 @@ <h3 id="pytorch_widedeep.losses.TweedieLoss.forward" class="doc doc-heading">
 <span class="normal">378</span>
 <span class="normal">379</span>
 <span class="normal">380</span>
-<span class="normal">381</span>
-<span class="normal">382</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">forward</span><span class="p">(</span>
+<span class="normal">381</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">forward</span><span class="p">(</span>
     <span class="bp">self</span><span class="p">,</span>
     <span class="nb">input</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">,</span>
     <span class="n">target</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">,</span>
@@ -3158,8 +3156,8 @@ <h2 id="pytorch_widedeep.losses.ZILNLoss" class="doc doc-heading">
 
                   <details class="quote">
                     <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
-                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">393</span>
-<span class="normal">394</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">392</span>
+<span class="normal">393</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
     <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
 </code></pre></div></td></tr></table></div>
                   </details>
@@ -3223,7 +3221,8 @@ <h3 id="pytorch_widedeep.losses.ZILNLoss.forward" class="doc doc-heading">
 
             <details class="quote">
               <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
-              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">396</span>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">395</span>
+<span class="normal">396</span>
 <span class="normal">397</span>
 <span class="normal">398</span>
 <span class="normal">399</span>
@@ -3271,10 +3270,7 @@ <h3 id="pytorch_widedeep.losses.ZILNLoss.forward" class="doc doc-heading">
 <span class="normal">441</span>
 <span class="normal">442</span>
 <span class="normal">443</span>
-<span class="normal">444</span>
-<span class="normal">445</span>
-<span class="normal">446</span>
-<span class="normal">447</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">forward</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="nb">input</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">,</span> <span class="n">target</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tensor</span><span class="p">:</span>
+<span class="normal">444</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">forward</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="nb">input</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">,</span> <span class="n">target</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tensor</span><span class="p">:</span>
 <span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;</span>
 <span class="sd">    Parameters</span>
 <span class="sd">    ----------</span>
@@ -3311,9 +3307,7 @@ <h3 id="pytorch_widedeep.losses.ZILNLoss.forward" class="doc doc-heading">
     <span class="c1"># when using max the two input tensors (input and other) have to be of</span>
     <span class="c1"># the same type</span>
     <span class="n">max_input</span> <span class="o">=</span> <span class="n">F</span><span class="o">.</span><span class="n">softplus</span><span class="p">(</span><span class="nb">input</span><span class="p">[</span><span class="o">...</span><span class="p">,</span> <span class="mi">2</span><span class="p">:])</span>
-    <span class="n">max_other</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">sqrt</span><span class="p">(</span><span class="n">torch</span><span class="o">.</span><span class="n">Tensor</span><span class="p">([</span><span class="n">torch</span><span class="o">.</span><span class="n">finfo</span><span class="p">(</span><span class="n">torch</span><span class="o">.</span><span class="n">double</span><span class="p">)</span><span class="o">.</span><span class="n">eps</span><span class="p">]))</span><span class="o">.</span><span class="n">type</span><span class="p">(</span>
-        <span class="n">max_input</span><span class="o">.</span><span class="n">type</span><span class="p">()</span>
-    <span class="p">)</span>
+    <span class="n">max_other</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_eps</span><span class="p">(</span><span class="n">max_input</span><span class="p">)</span>
     <span class="n">scale</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">max</span><span class="p">(</span><span class="n">max_input</span><span class="p">,</span> <span class="n">max_other</span><span class="p">)</span>
     <span class="n">safe_labels</span> <span class="o">=</span> <span class="n">positive</span> <span class="o">*</span> <span class="n">target</span> <span class="o">+</span> <span class="p">(</span><span class="mi">1</span> <span class="o">-</span> <span class="n">positive</span><span class="p">)</span> <span class="o">*</span> <span class="n">torch</span><span class="o">.</span><span class="n">ones_like</span><span class="p">(</span><span class="n">target</span><span class="p">)</span>
 
@@ -3361,8 +3355,8 @@ <h2 id="pytorch_widedeep.losses.L1Loss" class="doc doc-heading">
 
                   <details class="quote">
                     <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
-                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">456</span>
-<span class="normal">457</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">472</span>
+<span class="normal">473</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
     <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
 </code></pre></div></td></tr></table></div>
                   </details>
@@ -3427,27 +3421,27 @@ <h3 id="pytorch_widedeep.losses.L1Loss.forward" class="doc doc-heading">
 
             <details class="quote">
               <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
-              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">459</span>
-<span class="normal">460</span>
-<span class="normal">461</span>
-<span class="normal">462</span>
-<span class="normal">463</span>
-<span class="normal">464</span>
-<span class="normal">465</span>
-<span class="normal">466</span>
-<span class="normal">467</span>
-<span class="normal">468</span>
-<span class="normal">469</span>
-<span class="normal">470</span>
-<span class="normal">471</span>
-<span class="normal">472</span>
-<span class="normal">473</span>
-<span class="normal">474</span>
-<span class="normal">475</span>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">475</span>
 <span class="normal">476</span>
 <span class="normal">477</span>
 <span class="normal">478</span>
-<span class="normal">479</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">forward</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="nb">input</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">,</span> <span class="n">target</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tensor</span><span class="p">:</span>
+<span class="normal">479</span>
+<span class="normal">480</span>
+<span class="normal">481</span>
+<span class="normal">482</span>
+<span class="normal">483</span>
+<span class="normal">484</span>
+<span class="normal">485</span>
+<span class="normal">486</span>
+<span class="normal">487</span>
+<span class="normal">488</span>
+<span class="normal">489</span>
+<span class="normal">490</span>
+<span class="normal">491</span>
+<span class="normal">492</span>
+<span class="normal">493</span>
+<span class="normal">494</span>
+<span class="normal">495</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">forward</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="nb">input</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">,</span> <span class="n">target</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tensor</span><span class="p">:</span>
 <span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;</span>
 <span class="sd">    Parameters</span>
 <span class="sd">    ----------</span>
@@ -3541,16 +3535,16 @@ <h2 id="pytorch_widedeep.losses.FocalR_L1Loss" class="doc doc-heading">
 
                   <details class="quote">
                     <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
-                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">499</span>
-<span class="normal">500</span>
-<span class="normal">501</span>
-<span class="normal">502</span>
-<span class="normal">503</span>
-<span class="normal">504</span>
-<span class="normal">505</span>
-<span class="normal">506</span>
-<span class="normal">507</span>
-<span class="normal">508</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
+                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">515</span>
+<span class="normal">516</span>
+<span class="normal">517</span>
+<span class="normal">518</span>
+<span class="normal">519</span>
+<span class="normal">520</span>
+<span class="normal">521</span>
+<span class="normal">522</span>
+<span class="normal">523</span>
+<span class="normal">524</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
     <span class="bp">self</span><span class="p">,</span>
     <span class="n">beta</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.2</span><span class="p">,</span>
     <span class="n">gamma</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">1.0</span><span class="p">,</span>
@@ -3623,23 +3617,7 @@ <h3 id="pytorch_widedeep.losses.FocalR_L1Loss.forward" class="doc doc-heading">
 
             <details class="quote">
               <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
-              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">510</span>
-<span class="normal">511</span>
-<span class="normal">512</span>
-<span class="normal">513</span>
-<span class="normal">514</span>
-<span class="normal">515</span>
-<span class="normal">516</span>
-<span class="normal">517</span>
-<span class="normal">518</span>
-<span class="normal">519</span>
-<span class="normal">520</span>
-<span class="normal">521</span>
-<span class="normal">522</span>
-<span class="normal">523</span>
-<span class="normal">524</span>
-<span class="normal">525</span>
-<span class="normal">526</span>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">526</span>
 <span class="normal">527</span>
 <span class="normal">528</span>
 <span class="normal">529</span>
@@ -3657,7 +3635,23 @@ <h3 id="pytorch_widedeep.losses.FocalR_L1Loss.forward" class="doc doc-heading">
 <span class="normal">541</span>
 <span class="normal">542</span>
 <span class="normal">543</span>
-<span class="normal">544</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">forward</span><span class="p">(</span>
+<span class="normal">544</span>
+<span class="normal">545</span>
+<span class="normal">546</span>
+<span class="normal">547</span>
+<span class="normal">548</span>
+<span class="normal">549</span>
+<span class="normal">550</span>
+<span class="normal">551</span>
+<span class="normal">552</span>
+<span class="normal">553</span>
+<span class="normal">554</span>
+<span class="normal">555</span>
+<span class="normal">556</span>
+<span class="normal">557</span>
+<span class="normal">558</span>
+<span class="normal">559</span>
+<span class="normal">560</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">forward</span><span class="p">(</span>
     <span class="bp">self</span><span class="p">,</span>
     <span class="nb">input</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">,</span>
     <span class="n">target</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">,</span>
@@ -3767,16 +3761,16 @@ <h2 id="pytorch_widedeep.losses.FocalR_MSELoss" class="doc doc-heading">
 
                   <details class="quote">
                     <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
-                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">564</span>
-<span class="normal">565</span>
-<span class="normal">566</span>
-<span class="normal">567</span>
-<span class="normal">568</span>
-<span class="normal">569</span>
-<span class="normal">570</span>
-<span class="normal">571</span>
-<span class="normal">572</span>
-<span class="normal">573</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
+                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">580</span>
+<span class="normal">581</span>
+<span class="normal">582</span>
+<span class="normal">583</span>
+<span class="normal">584</span>
+<span class="normal">585</span>
+<span class="normal">586</span>
+<span class="normal">587</span>
+<span class="normal">588</span>
+<span class="normal">589</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
     <span class="bp">self</span><span class="p">,</span>
     <span class="n">beta</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.2</span><span class="p">,</span>
     <span class="n">gamma</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">1.0</span><span class="p">,</span>
@@ -3849,23 +3843,7 @@ <h3 id="pytorch_widedeep.losses.FocalR_MSELoss.forward" class="doc doc-heading">
 
             <details class="quote">
               <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
-              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">575</span>
-<span class="normal">576</span>
-<span class="normal">577</span>
-<span class="normal">578</span>
-<span class="normal">579</span>
-<span class="normal">580</span>
-<span class="normal">581</span>
-<span class="normal">582</span>
-<span class="normal">583</span>
-<span class="normal">584</span>
-<span class="normal">585</span>
-<span class="normal">586</span>
-<span class="normal">587</span>
-<span class="normal">588</span>
-<span class="normal">589</span>
-<span class="normal">590</span>
-<span class="normal">591</span>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">591</span>
 <span class="normal">592</span>
 <span class="normal">593</span>
 <span class="normal">594</span>
@@ -3883,7 +3861,23 @@ <h3 id="pytorch_widedeep.losses.FocalR_MSELoss.forward" class="doc doc-heading">
 <span class="normal">606</span>
 <span class="normal">607</span>
 <span class="normal">608</span>
-<span class="normal">609</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">forward</span><span class="p">(</span>
+<span class="normal">609</span>
+<span class="normal">610</span>
+<span class="normal">611</span>
+<span class="normal">612</span>
+<span class="normal">613</span>
+<span class="normal">614</span>
+<span class="normal">615</span>
+<span class="normal">616</span>
+<span class="normal">617</span>
+<span class="normal">618</span>
+<span class="normal">619</span>
+<span class="normal">620</span>
+<span class="normal">621</span>
+<span class="normal">622</span>
+<span class="normal">623</span>
+<span class="normal">624</span>
+<span class="normal">625</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">forward</span><span class="p">(</span>
     <span class="bp">self</span><span class="p">,</span>
     <span class="nb">input</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">,</span>
     <span class="n">target</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">,</span>
@@ -3993,16 +3987,16 @@ <h2 id="pytorch_widedeep.losses.FocalR_RMSELoss" class="doc doc-heading">
 
                   <details class="quote">
                     <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
-                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">629</span>
-<span class="normal">630</span>
-<span class="normal">631</span>
-<span class="normal">632</span>
-<span class="normal">633</span>
-<span class="normal">634</span>
-<span class="normal">635</span>
-<span class="normal">636</span>
-<span class="normal">637</span>
-<span class="normal">638</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
+                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">645</span>
+<span class="normal">646</span>
+<span class="normal">647</span>
+<span class="normal">648</span>
+<span class="normal">649</span>
+<span class="normal">650</span>
+<span class="normal">651</span>
+<span class="normal">652</span>
+<span class="normal">653</span>
+<span class="normal">654</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
     <span class="bp">self</span><span class="p">,</span>
     <span class="n">beta</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.2</span><span class="p">,</span>
     <span class="n">gamma</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">1.0</span><span class="p">,</span>
@@ -4075,23 +4069,7 @@ <h3 id="pytorch_widedeep.losses.FocalR_RMSELoss.forward" class="doc doc-heading"
 
             <details class="quote">
               <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
-              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">640</span>
-<span class="normal">641</span>
-<span class="normal">642</span>
-<span class="normal">643</span>
-<span class="normal">644</span>
-<span class="normal">645</span>
-<span class="normal">646</span>
-<span class="normal">647</span>
-<span class="normal">648</span>
-<span class="normal">649</span>
-<span class="normal">650</span>
-<span class="normal">651</span>
-<span class="normal">652</span>
-<span class="normal">653</span>
-<span class="normal">654</span>
-<span class="normal">655</span>
-<span class="normal">656</span>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">656</span>
 <span class="normal">657</span>
 <span class="normal">658</span>
 <span class="normal">659</span>
@@ -4109,7 +4087,23 @@ <h3 id="pytorch_widedeep.losses.FocalR_RMSELoss.forward" class="doc doc-heading"
 <span class="normal">671</span>
 <span class="normal">672</span>
 <span class="normal">673</span>
-<span class="normal">674</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">forward</span><span class="p">(</span>
+<span class="normal">674</span>
+<span class="normal">675</span>
+<span class="normal">676</span>
+<span class="normal">677</span>
+<span class="normal">678</span>
+<span class="normal">679</span>
+<span class="normal">680</span>
+<span class="normal">681</span>
+<span class="normal">682</span>
+<span class="normal">683</span>
+<span class="normal">684</span>
+<span class="normal">685</span>
+<span class="normal">686</span>
+<span class="normal">687</span>
+<span class="normal">688</span>
+<span class="normal">689</span>
+<span class="normal">690</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">forward</span><span class="p">(</span>
     <span class="bp">self</span><span class="p">,</span>
     <span class="nb">input</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">,</span>
     <span class="n">target</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">,</span>
@@ -4180,9 +4174,9 @@ <h2 id="pytorch_widedeep.losses.HuberLoss" class="doc doc-heading">
 
                   <details class="quote">
                     <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
-                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">683</span>
-<span class="normal">684</span>
-<span class="normal">685</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">beta</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.2</span><span class="p">):</span>
+                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">699</span>
+<span class="normal">700</span>
+<span class="normal">701</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">beta</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.2</span><span class="p">):</span>
     <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
     <span class="bp">self</span><span class="o">.</span><span class="n">beta</span> <span class="o">=</span> <span class="n">beta</span>
 </code></pre></div></td></tr></table></div>
@@ -4248,23 +4242,7 @@ <h3 id="pytorch_widedeep.losses.HuberLoss.forward" class="doc doc-heading">
 
             <details class="quote">
               <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
-              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">687</span>
-<span class="normal">688</span>
-<span class="normal">689</span>
-<span class="normal">690</span>
-<span class="normal">691</span>
-<span class="normal">692</span>
-<span class="normal">693</span>
-<span class="normal">694</span>
-<span class="normal">695</span>
-<span class="normal">696</span>
-<span class="normal">697</span>
-<span class="normal">698</span>
-<span class="normal">699</span>
-<span class="normal">700</span>
-<span class="normal">701</span>
-<span class="normal">702</span>
-<span class="normal">703</span>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">703</span>
 <span class="normal">704</span>
 <span class="normal">705</span>
 <span class="normal">706</span>
@@ -4276,7 +4254,23 @@ <h3 id="pytorch_widedeep.losses.HuberLoss.forward" class="doc doc-heading">
 <span class="normal">712</span>
 <span class="normal">713</span>
 <span class="normal">714</span>
-<span class="normal">715</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">forward</span><span class="p">(</span>
+<span class="normal">715</span>
+<span class="normal">716</span>
+<span class="normal">717</span>
+<span class="normal">718</span>
+<span class="normal">719</span>
+<span class="normal">720</span>
+<span class="normal">721</span>
+<span class="normal">722</span>
+<span class="normal">723</span>
+<span class="normal">724</span>
+<span class="normal">725</span>
+<span class="normal">726</span>
+<span class="normal">727</span>
+<span class="normal">728</span>
+<span class="normal">729</span>
+<span class="normal">730</span>
+<span class="normal">731</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">forward</span><span class="p">(</span>
     <span class="bp">self</span><span class="p">,</span>
     <span class="nb">input</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">,</span>
     <span class="n">target</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">,</span>
@@ -4373,11 +4367,11 @@ <h2 id="pytorch_widedeep.losses.InfoNCELoss" class="doc doc-heading">
 
                   <details class="quote">
                     <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
-                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">740</span>
-<span class="normal">741</span>
-<span class="normal">742</span>
-<span class="normal">743</span>
-<span class="normal">744</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">temperature</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.1</span><span class="p">,</span> <span class="n">reduction</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;mean&quot;</span><span class="p">):</span>
+                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">756</span>
+<span class="normal">757</span>
+<span class="normal">758</span>
+<span class="normal">759</span>
+<span class="normal">760</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">temperature</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.1</span><span class="p">,</span> <span class="n">reduction</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;mean&quot;</span><span class="p">):</span>
     <span class="nb">super</span><span class="p">(</span><span class="n">InfoNCELoss</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
 
     <span class="bp">self</span><span class="o">.</span><span class="n">temperature</span> <span class="o">=</span> <span class="n">temperature</span>
@@ -4437,23 +4431,7 @@ <h3 id="pytorch_widedeep.losses.InfoNCELoss.forward" class="doc doc-heading">
 
             <details class="quote">
               <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
-              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">746</span>
-<span class="normal">747</span>
-<span class="normal">748</span>
-<span class="normal">749</span>
-<span class="normal">750</span>
-<span class="normal">751</span>
-<span class="normal">752</span>
-<span class="normal">753</span>
-<span class="normal">754</span>
-<span class="normal">755</span>
-<span class="normal">756</span>
-<span class="normal">757</span>
-<span class="normal">758</span>
-<span class="normal">759</span>
-<span class="normal">760</span>
-<span class="normal">761</span>
-<span class="normal">762</span>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">762</span>
 <span class="normal">763</span>
 <span class="normal">764</span>
 <span class="normal">765</span>
@@ -4468,7 +4446,23 @@ <h3 id="pytorch_widedeep.losses.InfoNCELoss.forward" class="doc doc-heading">
 <span class="normal">774</span>
 <span class="normal">775</span>
 <span class="normal">776</span>
-<span class="normal">777</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">forward</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">g_projs</span><span class="p">:</span> <span class="n">Tuple</span><span class="p">[</span><span class="n">Tensor</span><span class="p">,</span> <span class="n">Tensor</span><span class="p">])</span> <span class="o">-&gt;</span> <span class="n">Tensor</span><span class="p">:</span>
+<span class="normal">777</span>
+<span class="normal">778</span>
+<span class="normal">779</span>
+<span class="normal">780</span>
+<span class="normal">781</span>
+<span class="normal">782</span>
+<span class="normal">783</span>
+<span class="normal">784</span>
+<span class="normal">785</span>
+<span class="normal">786</span>
+<span class="normal">787</span>
+<span class="normal">788</span>
+<span class="normal">789</span>
+<span class="normal">790</span>
+<span class="normal">791</span>
+<span class="normal">792</span>
+<span class="normal">793</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">forward</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">g_projs</span><span class="p">:</span> <span class="n">Tuple</span><span class="p">[</span><span class="n">Tensor</span><span class="p">,</span> <span class="n">Tensor</span><span class="p">])</span> <span class="o">-&gt;</span> <span class="n">Tensor</span><span class="p">:</span>
 <span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;</span>
 <span class="sd">    Parameters</span>
 <span class="sd">    ----------</span>
@@ -4581,14 +4575,14 @@ <h2 id="pytorch_widedeep.losses.DenoisingLoss" class="doc doc-heading">
 
                   <details class="quote">
                     <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
-                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">804</span>
-<span class="normal">805</span>
-<span class="normal">806</span>
-<span class="normal">807</span>
-<span class="normal">808</span>
-<span class="normal">809</span>
-<span class="normal">810</span>
-<span class="normal">811</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
+                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">820</span>
+<span class="normal">821</span>
+<span class="normal">822</span>
+<span class="normal">823</span>
+<span class="normal">824</span>
+<span class="normal">825</span>
+<span class="normal">826</span>
+<span class="normal">827</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
     <span class="bp">self</span><span class="p">,</span> <span class="n">lambda_cat</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">1.0</span><span class="p">,</span> <span class="n">lambda_cont</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">1.0</span><span class="p">,</span> <span class="n">reduction</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;mean&quot;</span>
 <span class="p">):</span>
     <span class="nb">super</span><span class="p">(</span><span class="n">DenoisingLoss</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
@@ -4662,23 +4656,7 @@ <h3 id="pytorch_widedeep.losses.DenoisingLoss.forward" class="doc doc-heading">
 
             <details class="quote">
               <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
-              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">813</span>
-<span class="normal">814</span>
-<span class="normal">815</span>
-<span class="normal">816</span>
-<span class="normal">817</span>
-<span class="normal">818</span>
-<span class="normal">819</span>
-<span class="normal">820</span>
-<span class="normal">821</span>
-<span class="normal">822</span>
-<span class="normal">823</span>
-<span class="normal">824</span>
-<span class="normal">825</span>
-<span class="normal">826</span>
-<span class="normal">827</span>
-<span class="normal">828</span>
-<span class="normal">829</span>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">829</span>
 <span class="normal">830</span>
 <span class="normal">831</span>
 <span class="normal">832</span>
@@ -4704,7 +4682,23 @@ <h3 id="pytorch_widedeep.losses.DenoisingLoss.forward" class="doc doc-heading">
 <span class="normal">852</span>
 <span class="normal">853</span>
 <span class="normal">854</span>
-<span class="normal">855</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">forward</span><span class="p">(</span>
+<span class="normal">855</span>
+<span class="normal">856</span>
+<span class="normal">857</span>
+<span class="normal">858</span>
+<span class="normal">859</span>
+<span class="normal">860</span>
+<span class="normal">861</span>
+<span class="normal">862</span>
+<span class="normal">863</span>
+<span class="normal">864</span>
+<span class="normal">865</span>
+<span class="normal">866</span>
+<span class="normal">867</span>
+<span class="normal">868</span>
+<span class="normal">869</span>
+<span class="normal">870</span>
+<span class="normal">871</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">forward</span><span class="p">(</span>
     <span class="bp">self</span><span class="p">,</span>
     <span class="n">x_cat_and_cat_</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span>
         <span class="n">Union</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="n">Tuple</span><span class="p">[</span><span class="n">Tensor</span><span class="p">,</span> <span class="n">Tensor</span><span class="p">]],</span> <span class="n">Tuple</span><span class="p">[</span><span class="n">Tensor</span><span class="p">,</span> <span class="n">Tensor</span><span class="p">]]</span>
@@ -4805,9 +4799,9 @@ <h2 id="pytorch_widedeep.losses.EncoderDecoderLoss" class="doc doc-heading">
 
                   <details class="quote">
                     <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
-                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">911</span>
-<span class="normal">912</span>
-<span class="normal">913</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">eps</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">1e-9</span><span class="p">):</span>
+                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">927</span>
+<span class="normal">928</span>
+<span class="normal">929</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">eps</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">1e-9</span><span class="p">):</span>
     <span class="nb">super</span><span class="p">(</span><span class="n">EncoderDecoderLoss</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
     <span class="bp">self</span><span class="o">.</span><span class="n">eps</span> <span class="o">=</span> <span class="n">eps</span>
 </code></pre></div></td></tr></table></div>
@@ -4882,23 +4876,7 @@ <h3 id="pytorch_widedeep.losses.EncoderDecoderLoss.forward" class="doc doc-headi
 
             <details class="quote">
               <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
-              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">915</span>
-<span class="normal">916</span>
-<span class="normal">917</span>
-<span class="normal">918</span>
-<span class="normal">919</span>
-<span class="normal">920</span>
-<span class="normal">921</span>
-<span class="normal">922</span>
-<span class="normal">923</span>
-<span class="normal">924</span>
-<span class="normal">925</span>
-<span class="normal">926</span>
-<span class="normal">927</span>
-<span class="normal">928</span>
-<span class="normal">929</span>
-<span class="normal">930</span>
-<span class="normal">931</span>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">931</span>
 <span class="normal">932</span>
 <span class="normal">933</span>
 <span class="normal">934</span>
@@ -4921,7 +4899,23 @@ <h3 id="pytorch_widedeep.losses.EncoderDecoderLoss.forward" class="doc doc-headi
 <span class="normal">951</span>
 <span class="normal">952</span>
 <span class="normal">953</span>
-<span class="normal">954</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">forward</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">x_true</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">,</span> <span class="n">x_pred</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">,</span> <span class="n">mask</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tensor</span><span class="p">:</span>
+<span class="normal">954</span>
+<span class="normal">955</span>
+<span class="normal">956</span>
+<span class="normal">957</span>
+<span class="normal">958</span>
+<span class="normal">959</span>
+<span class="normal">960</span>
+<span class="normal">961</span>
+<span class="normal">962</span>
+<span class="normal">963</span>
+<span class="normal">964</span>
+<span class="normal">965</span>
+<span class="normal">966</span>
+<span class="normal">967</span>
+<span class="normal">968</span>
+<span class="normal">969</span>
+<span class="normal">970</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">forward</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">x_true</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">,</span> <span class="n">x_pred</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">,</span> <span class="n">mask</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tensor</span><span class="p">:</span>
 <span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;</span>
 <span class="sd">    Parameters</span>
 <span class="sd">    ----------</span>
diff --git a/mkdocs/site/pytorch-widedeep/metrics.html b/mkdocs/site/pytorch-widedeep/metrics.html
index 349df8a8..2835b15b 100644
--- a/mkdocs/site/pytorch-widedeep/metrics.html
+++ b/mkdocs/site/pytorch-widedeep/metrics.html
@@ -1721,13 +1721,13 @@ <h2 id="pytorch_widedeep.metrics.Accuracy" class="doc doc-heading">
 
                   <details class="quote">
                     <summary>Source code in <code>pytorch_widedeep/metrics.py</code></summary>
-                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">75</span>
-<span class="normal">76</span>
+                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">76</span>
 <span class="normal">77</span>
 <span class="normal">78</span>
 <span class="normal">79</span>
 <span class="normal">80</span>
-<span class="normal">81</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">top_k</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1</span><span class="p">):</span>
+<span class="normal">81</span>
+<span class="normal">82</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">top_k</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1</span><span class="p">):</span>
     <span class="nb">super</span><span class="p">(</span><span class="n">Accuracy</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
 
     <span class="bp">self</span><span class="o">.</span><span class="n">top_k</span> <span class="o">=</span> <span class="n">top_k</span>
@@ -1766,12 +1766,12 @@ <h3 id="pytorch_widedeep.metrics.Accuracy.reset" class="doc doc-heading">
 
             <details class="quote">
               <summary>Source code in <code>pytorch_widedeep/metrics.py</code></summary>
-              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">83</span>
-<span class="normal">84</span>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">84</span>
 <span class="normal">85</span>
 <span class="normal">86</span>
 <span class="normal">87</span>
-<span class="normal">88</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">reset</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+<span class="normal">88</span>
+<span class="normal">89</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">reset</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
 <span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
 <span class="sd">    resets counters to 0</span>
 <span class="sd">    &quot;&quot;&quot;</span>
@@ -1847,14 +1847,14 @@ <h2 id="pytorch_widedeep.metrics.Precision" class="doc doc-heading">
 
                   <details class="quote">
                     <summary>Source code in <code>pytorch_widedeep/metrics.py</code></summary>
-                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">134</span>
-<span class="normal">135</span>
+                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">135</span>
 <span class="normal">136</span>
 <span class="normal">137</span>
 <span class="normal">138</span>
 <span class="normal">139</span>
 <span class="normal">140</span>
-<span class="normal">141</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">average</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">):</span>
+<span class="normal">141</span>
+<span class="normal">142</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">average</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">):</span>
     <span class="nb">super</span><span class="p">(</span><span class="n">Precision</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
 
     <span class="bp">self</span><span class="o">.</span><span class="n">average</span> <span class="o">=</span> <span class="n">average</span>
@@ -1894,12 +1894,12 @@ <h3 id="pytorch_widedeep.metrics.Precision.reset" class="doc doc-heading">
 
             <details class="quote">
               <summary>Source code in <code>pytorch_widedeep/metrics.py</code></summary>
-              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">143</span>
-<span class="normal">144</span>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">144</span>
 <span class="normal">145</span>
 <span class="normal">146</span>
 <span class="normal">147</span>
-<span class="normal">148</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">reset</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+<span class="normal">148</span>
+<span class="normal">149</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">reset</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
 <span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
 <span class="sd">    resets counters to 0</span>
 <span class="sd">    &quot;&quot;&quot;</span>
@@ -1975,14 +1975,14 @@ <h2 id="pytorch_widedeep.metrics.Recall" class="doc doc-heading">
 
                   <details class="quote">
                     <summary>Source code in <code>pytorch_widedeep/metrics.py</code></summary>
-                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">200</span>
-<span class="normal">201</span>
+                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">201</span>
 <span class="normal">202</span>
 <span class="normal">203</span>
 <span class="normal">204</span>
 <span class="normal">205</span>
 <span class="normal">206</span>
-<span class="normal">207</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">average</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">):</span>
+<span class="normal">207</span>
+<span class="normal">208</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">average</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">):</span>
     <span class="nb">super</span><span class="p">(</span><span class="n">Recall</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
 
     <span class="bp">self</span><span class="o">.</span><span class="n">average</span> <span class="o">=</span> <span class="n">average</span>
@@ -2022,12 +2022,12 @@ <h3 id="pytorch_widedeep.metrics.Recall.reset" class="doc doc-heading">
 
             <details class="quote">
               <summary>Source code in <code>pytorch_widedeep/metrics.py</code></summary>
-              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">209</span>
-<span class="normal">210</span>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">210</span>
 <span class="normal">211</span>
 <span class="normal">212</span>
 <span class="normal">213</span>
-<span class="normal">214</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">reset</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+<span class="normal">214</span>
+<span class="normal">215</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">reset</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
 <span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
 <span class="sd">    resets counters to 0</span>
 <span class="sd">    &quot;&quot;&quot;</span>
@@ -2114,15 +2114,15 @@ <h2 id="pytorch_widedeep.metrics.FBetaScore" class="doc doc-heading">
 
                   <details class="quote">
                     <summary>Source code in <code>pytorch_widedeep/metrics.py</code></summary>
-                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">272</span>
-<span class="normal">273</span>
+                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">273</span>
 <span class="normal">274</span>
 <span class="normal">275</span>
 <span class="normal">276</span>
 <span class="normal">277</span>
 <span class="normal">278</span>
 <span class="normal">279</span>
-<span class="normal">280</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">beta</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span> <span class="n">average</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">):</span>
+<span class="normal">280</span>
+<span class="normal">281</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">beta</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span> <span class="n">average</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">):</span>
     <span class="nb">super</span><span class="p">(</span><span class="n">FBetaScore</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
 
     <span class="bp">self</span><span class="o">.</span><span class="n">beta</span> <span class="o">=</span> <span class="n">beta</span>
@@ -2163,12 +2163,12 @@ <h3 id="pytorch_widedeep.metrics.FBetaScore.reset" class="doc doc-heading">
 
             <details class="quote">
               <summary>Source code in <code>pytorch_widedeep/metrics.py</code></summary>
-              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">282</span>
-<span class="normal">283</span>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">283</span>
 <span class="normal">284</span>
 <span class="normal">285</span>
 <span class="normal">286</span>
-<span class="normal">287</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">reset</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+<span class="normal">287</span>
+<span class="normal">288</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">reset</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
 <span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
 <span class="sd">    resets precision and recall</span>
 <span class="sd">    &quot;&quot;&quot;</span>
@@ -2244,12 +2244,12 @@ <h2 id="pytorch_widedeep.metrics.F1Score" class="doc doc-heading">
 
                   <details class="quote">
                     <summary>Source code in <code>pytorch_widedeep/metrics.py</code></summary>
-                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">330</span>
-<span class="normal">331</span>
+                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">331</span>
 <span class="normal">332</span>
 <span class="normal">333</span>
 <span class="normal">334</span>
-<span class="normal">335</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">average</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">):</span>
+<span class="normal">335</span>
+<span class="normal">336</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">average</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">):</span>
     <span class="nb">super</span><span class="p">(</span><span class="n">F1Score</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
 
     <span class="bp">self</span><span class="o">.</span><span class="n">average</span> <span class="o">=</span> <span class="n">average</span>
@@ -2287,11 +2287,11 @@ <h3 id="pytorch_widedeep.metrics.F1Score.reset" class="doc doc-heading">
 
             <details class="quote">
               <summary>Source code in <code>pytorch_widedeep/metrics.py</code></summary>
-              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">337</span>
-<span class="normal">338</span>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">338</span>
 <span class="normal">339</span>
 <span class="normal">340</span>
-<span class="normal">341</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">reset</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+<span class="normal">341</span>
+<span class="normal">342</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">reset</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
 <span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
 <span class="sd">    resets counters to 0</span>
 <span class="sd">    &quot;&quot;&quot;</span>
@@ -2350,13 +2350,13 @@ <h2 id="pytorch_widedeep.metrics.R2Score" class="doc doc-heading">
 
                   <details class="quote">
                     <summary>Source code in <code>pytorch_widedeep/metrics.py</code></summary>
-                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">372</span>
-<span class="normal">373</span>
+                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">373</span>
 <span class="normal">374</span>
 <span class="normal">375</span>
 <span class="normal">376</span>
 <span class="normal">377</span>
-<span class="normal">378</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+<span class="normal">378</span>
+<span class="normal">379</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
     <span class="bp">self</span><span class="o">.</span><span class="n">numerator</span> <span class="o">=</span> <span class="mi">0</span>
     <span class="bp">self</span><span class="o">.</span><span class="n">denominator</span> <span class="o">=</span> <span class="mi">0</span>
     <span class="bp">self</span><span class="o">.</span><span class="n">num_examples</span> <span class="o">=</span> <span class="mi">0</span>
@@ -2395,14 +2395,14 @@ <h3 id="pytorch_widedeep.metrics.R2Score.reset" class="doc doc-heading">
 
             <details class="quote">
               <summary>Source code in <code>pytorch_widedeep/metrics.py</code></summary>
-              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">380</span>
-<span class="normal">381</span>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">381</span>
 <span class="normal">382</span>
 <span class="normal">383</span>
 <span class="normal">384</span>
 <span class="normal">385</span>
 <span class="normal">386</span>
-<span class="normal">387</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">reset</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+<span class="normal">387</span>
+<span class="normal">388</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">reset</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
 <span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
 <span class="sd">    resets counters to 0</span>
 <span class="sd">    &quot;&quot;&quot;</span>
diff --git a/mkdocs/site/pytorch-widedeep/preprocessing.html b/mkdocs/site/pytorch-widedeep/preprocessing.html
index 2e3ec7a4..5f013907 100644
--- a/mkdocs/site/pytorch-widedeep/preprocessing.html
+++ b/mkdocs/site/pytorch-widedeep/preprocessing.html
@@ -775,6 +775,15 @@
     </span>
   </a>
   
+</li>
+      
+        <li class="md-nav__item">
+  <a href="#pytorch_widedeep.preprocessing.din_preprocessor.DINPreprocessor" class="md-nav__link">
+    <span class="md-ellipsis">
+      DINPreprocessor
+    </span>
+  </a>
+  
 </li>
       
         <li class="md-nav__item">
@@ -5416,6 +5425,294 @@ <h3 id="pytorch_widedeep.preprocessing.image_preprocessor.ImagePreprocessor.fit_
 
 
 
+  </div>
+
+    </div>
+
+</div>
+
+<div class="doc doc-object doc-class">
+
+
+
+<h2 id="pytorch_widedeep.preprocessing.din_preprocessor.DINPreprocessor" class="doc doc-heading">
+            <span class="doc doc-object-name doc-class-name">DINPreprocessor</span>
+
+
+<a href="#pytorch_widedeep.preprocessing.din_preprocessor.DINPreprocessor" class="headerlink" title="Permanent link">&para;</a></h2>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">DINPreprocessor</span><span class="p">(</span>
+    <span class="o">*</span><span class="p">,</span>
+    <span class="n">user_id_col</span><span class="p">,</span>
+    <span class="n">item_embed_col</span><span class="p">,</span>
+    <span class="n">target_col</span><span class="p">,</span>
+    <span class="n">max_seq_length</span><span class="p">,</span>
+    <span class="n">action_col</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+    <span class="n">other_seq_embed_cols</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+    <span class="n">cat_embed_cols</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+    <span class="n">continuous_cols</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+    <span class="n">quantization_setup</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+    <span class="n">cols_to_scale</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+    <span class="n">auto_embed_dim</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
+    <span class="n">embedding_rule</span><span class="o">=</span><span class="s2">&quot;fastai_new&quot;</span><span class="p">,</span>
+    <span class="n">default_embed_dim</span><span class="o">=</span><span class="mi">16</span><span class="p">,</span>
+    <span class="n">verbose</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span>
+    <span class="n">scale</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
+    <span class="n">already_standard</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+    <span class="o">**</span><span class="n">kwargs</span>
+<span class="p">)</span>
+</code></pre></div>
+
+    <div class="doc doc-contents first">
+            <p class="doc doc-class-bases">
+              Bases: <code><span title="pytorch_widedeep.preprocessing.base_preprocessor.BasePreprocessor">BasePreprocessor</span></code></p>
+
+
+        <p>Preprocessor for Deep Interest Network (DIN) models.</p>
+<p>This preprocessor handles the preparation of data for DIN models,
+including sequence building, label encoding, and handling of various
+types of input columns (categorical, continuous, and sequential).</p>
+
+
+<details class="parameters:" open>
+  <summary></summary>
+  <p>user_id_col : str
+    Name of the column containing user IDs.
+item_embed_col : Union[str, Tuple[str, int]]
+    Name of the column containing item IDs to be embedded, or a tuple of
+    (column_name, embedding_dim).
+target_col : str
+    Name of the column containing the target variable.
+max_seq_length : int
+    Maximum length of sequences to be created.
+action_col : Optional[str], default=None
+    Name of the column containing user actions (if applicable).
+other_seq_embed_cols : Optional[List[str] | List[Tuple[str, int]]], default=None
+    List of other columns to be treated as sequences.
+cat_embed_cols : Optional[Union[List[str], List[Tuple[str, int]]]], default=None
+    List of categorical columns to be represented by embeddings.
+continuous_cols: List, default = None
+    List with the name of the continuous cols.
+quantization_setup: int or Dict[str, Union[int, List[float]]], default=None
+    Continuous columns can be turned into categorical via <code>pd.cut</code>.
+cols_to_scale: List or str, default = None,
+    List with the names of the columns that will be standardized via
+    sklearn's <code>StandardScaler</code>.
+auto_embed_dim: bool, default = True
+    Boolean indicating whether the embedding dimensions will be
+    automatically defined via rule of thumb.
+embedding_rule: str, default = 'fastai_new'
+    Rule of thumb for embedding size.
+default_embed_dim: int, default=16
+    Default dimension for the embeddings.
+verbose : int, default=1
+    Verbosity level.
+scale: bool, default = False
+    Boolean indicating whether or not to scale/standardize continuous cols.
+already_standard: List, default = None
+    List with the name of the continuous cols that do not need to be
+    scaled/standardized.
+**kwargs :
+    Additional keyword arguments to be passed to the TabPreprocessor.</p>
+</details>
+
+<details class="attributes:" open>
+  <summary></summary>
+  <p>is_fitted : bool
+    Whether the preprocessor has been fitted.
+has_standard_tab_data : bool
+    Whether the data includes standard tabular data.
+tab_preprocessor : TabPreprocessor
+    Preprocessor for standard tabular data.
+din_columns_idx : Dict[str, int]
+    Dictionary mapping column names to their indices in the processed data.
+item_le : LabelEncoder
+    Label encoder for item IDs.
+n_items : int
+    Number of unique items.
+user_behaviour_config : Tuple[List[str], int, int]
+    Configuration for user behavior sequences.
+action_le : LabelEncoder
+    Label encoder for action column (if applicable).
+n_actions : int
+    Number of unique actions (if applicable).
+action_seq_config : Tuple[List[str], int]
+    Configuration for action sequences (if applicable).
+other_seq_le : LabelEncoder
+    Label encoder for other sequence columns (if applicable).
+n_other_seq_cols : Dict[str, int]
+    Number of unique values in each other sequence column.
+other_seq_config : List[Tuple[List[str], int, int]]
+    Configuration for other sequence columns.</p>
+</details>
+
+<details class="examples:" open>
+  <summary></summary>
+  <blockquote>
+<blockquote>
+<blockquote>
+<p>import pandas as pd
+from pytorch_widedeep.preprocessing import DINPreprocessor
+data = {
+...    'user_id': [1, 1, 1, 2, 2, 2, 3, 3, 3],
+...    'item_id': [101, 102, 103, 101, 103, 104, 102, 103, 104],
+...    'timestamp': [1, 2, 3, 1, 2, 3, 1, 2, 3],
+...    'category': ['A', 'B', 'A', 'B', 'A', 'C', 'B', 'A', 'C'],
+...    'price': [10.5, 15.0, 12.0, 10.5, 12.0, 20.0, 15.0, 12.0, 20.0],
+...    'rating': [0, 1, 0, 1, 0, 1, 0, 1, 0]
+... }
+df = pd.DataFrame(data)
+din_preprocessor = DINPreprocessor(
+...     user_id_col='user_id',
+...     item_embed_col='item_id',
+...     target_col='rating',
+...     max_seq_length=2,
+...     action_col='rating',
+...     other_seq_embed_cols=['category'],
+...     cat_embed_cols=['user_id'],
+...     continuous_cols=['price'],
+...     cols_to_scale=['price']
+... )
+X, y = din_preprocessor.fit_transform(df)</p>
+</blockquote>
+</blockquote>
+</blockquote>
+</details>
+                  <details class="quote">
+                    <summary>Source code in <code>pytorch_widedeep/preprocessing/din_preprocessor.py</code></summary>
+                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">126</span>
+<span class="normal">127</span>
+<span class="normal">128</span>
+<span class="normal">129</span>
+<span class="normal">130</span>
+<span class="normal">131</span>
+<span class="normal">132</span>
+<span class="normal">133</span>
+<span class="normal">134</span>
+<span class="normal">135</span>
+<span class="normal">136</span>
+<span class="normal">137</span>
+<span class="normal">138</span>
+<span class="normal">139</span>
+<span class="normal">140</span>
+<span class="normal">141</span>
+<span class="normal">142</span>
+<span class="normal">143</span>
+<span class="normal">144</span>
+<span class="normal">145</span>
+<span class="normal">146</span>
+<span class="normal">147</span>
+<span class="normal">148</span>
+<span class="normal">149</span>
+<span class="normal">150</span>
+<span class="normal">151</span>
+<span class="normal">152</span>
+<span class="normal">153</span>
+<span class="normal">154</span>
+<span class="normal">155</span>
+<span class="normal">156</span>
+<span class="normal">157</span>
+<span class="normal">158</span>
+<span class="normal">159</span>
+<span class="normal">160</span>
+<span class="normal">161</span>
+<span class="normal">162</span>
+<span class="normal">163</span>
+<span class="normal">164</span>
+<span class="normal">165</span>
+<span class="normal">166</span>
+<span class="normal">167</span>
+<span class="normal">168</span>
+<span class="normal">169</span>
+<span class="normal">170</span>
+<span class="normal">171</span>
+<span class="normal">172</span>
+<span class="normal">173</span>
+<span class="normal">174</span>
+<span class="normal">175</span>
+<span class="normal">176</span>
+<span class="normal">177</span>
+<span class="normal">178</span>
+<span class="normal">179</span>
+<span class="normal">180</span>
+<span class="normal">181</span>
+<span class="normal">182</span>
+<span class="normal">183</span>
+<span class="normal">184</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="nd">@alias</span><span class="p">(</span><span class="s2">&quot;item_embed_col&quot;</span><span class="p">,</span> <span class="p">[</span><span class="s2">&quot;item_id_col&quot;</span><span class="p">])</span>
+<span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
+    <span class="bp">self</span><span class="p">,</span>
+    <span class="o">*</span><span class="p">,</span>
+    <span class="n">user_id_col</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
+    <span class="n">item_embed_col</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Tuple</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">]],</span>
+    <span class="n">target_col</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
+    <span class="n">max_seq_length</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span>
+    <span class="n">action_col</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+    <span class="n">other_seq_embed_cols</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Union</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">],</span> <span class="n">List</span><span class="p">[</span><span class="n">Tuple</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">]]]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+    <span class="n">cat_embed_cols</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Union</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">],</span> <span class="n">List</span><span class="p">[</span><span class="n">Tuple</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">]]]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+    <span class="n">continuous_cols</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+    <span class="n">quantization_setup</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span>
+        <span class="n">Union</span><span class="p">[</span><span class="nb">int</span><span class="p">,</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Union</span><span class="p">[</span><span class="nb">int</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="nb">float</span><span class="p">]]]]</span>
+    <span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+    <span class="n">cols_to_scale</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Union</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">],</span> <span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+    <span class="n">auto_embed_dim</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">,</span>
+    <span class="n">embedding_rule</span><span class="p">:</span> <span class="n">Literal</span><span class="p">[</span><span class="s2">&quot;google&quot;</span><span class="p">,</span> <span class="s2">&quot;fastai_old&quot;</span><span class="p">,</span> <span class="s2">&quot;fastai_new&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="s2">&quot;fastai_new&quot;</span><span class="p">,</span>
+    <span class="n">default_embed_dim</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">16</span><span class="p">,</span>
+    <span class="n">verbose</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1</span><span class="p">,</span>
+    <span class="n">scale</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
+    <span class="n">already_standard</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+    <span class="o">**</span><span class="n">kwargs</span><span class="p">,</span>
+<span class="p">):</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">user_id_col</span> <span class="o">=</span> <span class="n">user_id_col</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">item_embed_col</span> <span class="o">=</span> <span class="n">item_embed_col</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">max_seq_length</span> <span class="o">=</span> <span class="n">max_seq_length</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">target_col</span> <span class="o">=</span> <span class="n">target_col</span> <span class="k">if</span> <span class="n">target_col</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="k">else</span> <span class="s2">&quot;target&quot;</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">action_col</span> <span class="o">=</span> <span class="n">action_col</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">other_seq_embed_cols</span> <span class="o">=</span> <span class="n">other_seq_embed_cols</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">cat_embed_cols</span> <span class="o">=</span> <span class="n">cat_embed_cols</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">continuous_cols</span> <span class="o">=</span> <span class="n">continuous_cols</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">quantization_setup</span> <span class="o">=</span> <span class="n">quantization_setup</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">cols_to_scale</span> <span class="o">=</span> <span class="n">cols_to_scale</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">auto_embed_dim</span> <span class="o">=</span> <span class="n">auto_embed_dim</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">embedding_rule</span> <span class="o">=</span> <span class="n">embedding_rule</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">default_embed_dim</span> <span class="o">=</span> <span class="n">default_embed_dim</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">verbose</span> <span class="o">=</span> <span class="n">verbose</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">scale</span> <span class="o">=</span> <span class="n">scale</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">already_standard</span> <span class="o">=</span> <span class="n">already_standard</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">kwargs</span> <span class="o">=</span> <span class="n">kwargs</span>
+
+    <span class="bp">self</span><span class="o">.</span><span class="n">has_standard_tab_data</span> <span class="o">=</span> <span class="nb">bool</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">cat_embed_cols</span> <span class="ow">or</span> <span class="bp">self</span><span class="o">.</span><span class="n">continuous_cols</span><span class="p">)</span>
+    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">has_standard_tab_data</span><span class="p">:</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">tab_preprocessor</span> <span class="o">=</span> <span class="n">TabPreprocessor</span><span class="p">(</span>
+            <span class="n">cat_embed_cols</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">cat_embed_cols</span><span class="p">,</span>
+            <span class="n">continuous_cols</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">continuous_cols</span><span class="p">,</span>
+            <span class="n">quantization_setup</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">quantization_setup</span><span class="p">,</span>
+            <span class="n">cols_to_scale</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">cols_to_scale</span><span class="p">,</span>
+            <span class="n">auto_embed_dim</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">auto_embed_dim</span><span class="p">,</span>
+            <span class="n">embedding_rule</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">embedding_rule</span><span class="p">,</span>
+            <span class="n">default_embed_dim</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">default_embed_dim</span><span class="p">,</span>
+            <span class="n">verbose</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">verbose</span><span class="p">,</span>
+            <span class="n">scale</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">scale</span><span class="p">,</span>
+            <span class="n">already_standard</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">already_standard</span><span class="p">,</span>
+            <span class="o">**</span><span class="bp">self</span><span class="o">.</span><span class="n">kwargs</span><span class="p">,</span>
+        <span class="p">)</span>
+
+    <span class="bp">self</span><span class="o">.</span><span class="n">is_fitted</span> <span class="o">=</span> <span class="kc">False</span>
+</code></pre></div></td></tr></table></div>
+                  </details>
+
+
+
+  <div class="doc doc-children">
+
+
+
+
+
+
+
+
+
+
+
   </div>
 
     </div>
diff --git a/mkdocs/site/pytorch-widedeep/preprocessing.md b/mkdocs/site/pytorch-widedeep/preprocessing.md
index ab608846..520dd914 100644
--- a/mkdocs/site/pytorch-widedeep/preprocessing.md
+++ b/mkdocs/site/pytorch-widedeep/preprocessing.md
@@ -21,6 +21,8 @@ available: one for the case when no Hugging Face model is used
 
 ::: pytorch_widedeep.preprocessing.image_preprocessor.ImagePreprocessor
 
+::: pytorch_widedeep.preprocessing.din_preprocessor.DINPreprocessor
+
 
 ## Chunked versions
 
diff --git a/mkdocs/site/pytorch-widedeep/tab2vec.html b/mkdocs/site/pytorch-widedeep/tab2vec.html
index f33c81b7..e36e452a 100644
--- a/mkdocs/site/pytorch-widedeep/tab2vec.html
+++ b/mkdocs/site/pytorch-widedeep/tab2vec.html
@@ -1608,6 +1608,7 @@ <h2 id="pytorch_widedeep.tab2vec.Tab2Vec" class="doc doc-heading">
 <div class="doc-signature highlight"><pre><span></span><code><span class="nf">Tab2Vec</span><span class="p">(</span>
     <span class="n">tab_preprocessor</span><span class="p">,</span>
     <span class="n">model</span><span class="p">,</span>
+    <span class="n">device</span><span class="p">,</span>
     <span class="n">return_dataframe</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
     <span class="n">verbose</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
 <span class="p">)</span>
@@ -1709,13 +1710,15 @@ <h2 id="pytorch_widedeep.tab2vec.Tab2Vec" class="doc doc-heading">
 <span class="gp">&gt;&gt;&gt; </span><span class="c1"># ...train the model...</span>
 <span class="gp">&gt;&gt;&gt;</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="c1"># vectorise the dataframe</span>
-<span class="gp">&gt;&gt;&gt; </span><span class="n">t2v</span> <span class="o">=</span> <span class="n">Tab2Vec</span><span class="p">(</span><span class="n">tab_preprocessor</span><span class="p">,</span> <span class="n">model</span><span class="p">)</span>
+<span class="gp">&gt;&gt;&gt; </span><span class="n">t2v</span> <span class="o">=</span> <span class="n">Tab2Vec</span><span class="p">(</span><span class="n">tab_preprocessor</span><span class="p">,</span> <span class="n">model</span><span class="p">,</span> <span class="n">device</span><span class="o">=</span><span class="s2">&quot;cpu&quot;</span><span class="p">)</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="n">X_vec</span> <span class="o">=</span> <span class="n">t2v</span><span class="o">.</span><span class="n">transform</span><span class="p">(</span><span class="n">df_t2v</span><span class="p">)</span>
 </code></pre></div>
 
                   <details class="quote">
                     <summary>Source code in <code>pytorch_widedeep/tab2vec.py</code></summary>
-                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 86</span>
+                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 84</span>
+<span class="normal"> 85</span>
+<span class="normal"> 86</span>
 <span class="normal"> 87</span>
 <span class="normal"> 88</span>
 <span class="normal"> 89</span>
@@ -1732,10 +1735,15 @@ <h2 id="pytorch_widedeep.tab2vec.Tab2Vec" class="doc doc-heading">
 <span class="normal">100</span>
 <span class="normal">101</span>
 <span class="normal">102</span>
-<span class="normal">103</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
+<span class="normal">103</span>
+<span class="normal">104</span>
+<span class="normal">105</span>
+<span class="normal">106</span>
+<span class="normal">107</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
     <span class="bp">self</span><span class="p">,</span>
     <span class="n">tab_preprocessor</span><span class="p">:</span> <span class="n">TabPreprocessor</span><span class="p">,</span>
     <span class="n">model</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="n">WideDeep</span><span class="p">,</span> <span class="n">BayesianWide</span><span class="p">,</span> <span class="n">BayesianTabMlp</span><span class="p">],</span>
+    <span class="n">device</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="n">torch</span><span class="o">.</span><span class="n">device</span><span class="p">,</span>
     <span class="n">return_dataframe</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
     <span class="n">verbose</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
 <span class="p">):</span>
@@ -1743,6 +1751,11 @@ <h2 id="pytorch_widedeep.tab2vec.Tab2Vec" class="doc doc-heading">
 
     <span class="bp">self</span><span class="o">.</span><span class="n">_check_inputs</span><span class="p">(</span><span class="n">tab_preprocessor</span><span class="p">,</span> <span class="n">model</span><span class="p">,</span> <span class="n">verbose</span><span class="p">)</span>
 
+    <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">device</span><span class="p">,</span> <span class="nb">str</span><span class="p">):</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">device</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">device</span><span class="p">(</span><span class="n">device</span><span class="p">)</span>
+    <span class="k">else</span><span class="p">:</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">device</span> <span class="o">=</span> <span class="n">device</span>
+
     <span class="bp">self</span><span class="o">.</span><span class="n">tab_preprocessor</span> <span class="o">=</span> <span class="n">tab_preprocessor</span>
     <span class="bp">self</span><span class="o">.</span><span class="n">return_dataframe</span> <span class="o">=</span> <span class="n">return_dataframe</span>
     <span class="bp">self</span><span class="o">.</span><span class="n">verbose</span> <span class="o">=</span> <span class="n">verbose</span>
@@ -1822,11 +1835,7 @@ <h3 id="pytorch_widedeep.tab2vec.Tab2Vec.fit" class="doc doc-heading">
 
             <details class="quote">
               <summary>Source code in <code>pytorch_widedeep/tab2vec.py</code></summary>
-              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">105</span>
-<span class="normal">106</span>
-<span class="normal">107</span>
-<span class="normal">108</span>
-<span class="normal">109</span>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">109</span>
 <span class="normal">110</span>
 <span class="normal">111</span>
 <span class="normal">112</span>
@@ -1842,7 +1851,11 @@ <h3 id="pytorch_widedeep.tab2vec.Tab2Vec.fit" class="doc doc-heading">
 <span class="normal">122</span>
 <span class="normal">123</span>
 <span class="normal">124</span>
-<span class="normal">125</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">fit</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">,</span> <span class="n">target_col</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="s2">&quot;Tab2Vec&quot;</span><span class="p">:</span>
+<span class="normal">125</span>
+<span class="normal">126</span>
+<span class="normal">127</span>
+<span class="normal">128</span>
+<span class="normal">129</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">fit</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">,</span> <span class="n">target_col</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="s2">&quot;Tab2Vec&quot;</span><span class="p">:</span>
 <span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;This is an empty method i.e. Returns the unchanged object itself. Is</span>
 <span class="sd">    only included for consistency in case `Tab2Vec` is used as part of a</span>
 <span class="sd">    Pipeline</span>
@@ -1929,11 +1942,7 @@ <h3 id="pytorch_widedeep.tab2vec.Tab2Vec.transform" class="doc doc-heading">
 
             <details class="quote">
               <summary>Source code in <code>pytorch_widedeep/tab2vec.py</code></summary>
-              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">127</span>
-<span class="normal">128</span>
-<span class="normal">129</span>
-<span class="normal">130</span>
-<span class="normal">131</span>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">131</span>
 <span class="normal">132</span>
 <span class="normal">133</span>
 <span class="normal">134</span>
@@ -1993,7 +2002,15 @@ <h3 id="pytorch_widedeep.tab2vec.Tab2Vec.transform" class="doc doc-heading">
 <span class="normal">188</span>
 <span class="normal">189</span>
 <span class="normal">190</span>
-<span class="normal">191</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">transform</span><span class="p">(</span>
+<span class="normal">191</span>
+<span class="normal">192</span>
+<span class="normal">193</span>
+<span class="normal">194</span>
+<span class="normal">195</span>
+<span class="normal">196</span>
+<span class="normal">197</span>
+<span class="normal">198</span>
+<span class="normal">199</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">transform</span><span class="p">(</span>
     <span class="bp">self</span><span class="p">,</span>
     <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">,</span>
     <span class="n">target_col</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
@@ -2027,7 +2044,11 @@ <h3 id="pytorch_widedeep.tab2vec.Tab2Vec.transform" class="doc doc-heading">
 <span class="sd">    &quot;&quot;&quot;</span>
 
     <span class="n">X_tab</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">tab_preprocessor</span><span class="o">.</span><span class="n">transform</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
-    <span class="n">X</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">from_numpy</span><span class="p">(</span><span class="n">X_tab</span><span class="o">.</span><span class="n">astype</span><span class="p">(</span><span class="s2">&quot;float&quot;</span><span class="p">))</span><span class="o">.</span><span class="n">to</span><span class="p">(</span><span class="n">device</span><span class="p">)</span>
+
+    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">device</span><span class="o">.</span><span class="n">type</span> <span class="o">==</span> <span class="s2">&quot;mps&quot;</span><span class="p">:</span>
+        <span class="n">X</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">from_numpy</span><span class="p">(</span><span class="n">X_tab</span><span class="o">.</span><span class="n">astype</span><span class="p">(</span><span class="s2">&quot;float32&quot;</span><span class="p">))</span><span class="o">.</span><span class="n">to</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">device</span><span class="p">)</span>
+    <span class="k">else</span><span class="p">:</span>
+        <span class="n">X</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">from_numpy</span><span class="p">(</span><span class="n">X_tab</span><span class="o">.</span><span class="n">astype</span><span class="p">(</span><span class="s2">&quot;float&quot;</span><span class="p">))</span><span class="o">.</span><span class="n">to</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">device</span><span class="p">)</span>
 
     <span class="k">with</span> <span class="n">torch</span><span class="o">.</span><span class="n">no_grad</span><span class="p">():</span>
         <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">is_tab_transformer</span><span class="p">:</span>
@@ -2081,16 +2102,16 @@ <h3 id="pytorch_widedeep.tab2vec.Tab2Vec.fit_transform" class="doc doc-heading">
 
             <details class="quote">
               <summary>Source code in <code>pytorch_widedeep/tab2vec.py</code></summary>
-              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">193</span>
-<span class="normal">194</span>
-<span class="normal">195</span>
-<span class="normal">196</span>
-<span class="normal">197</span>
-<span class="normal">198</span>
-<span class="normal">199</span>
-<span class="normal">200</span>
-<span class="normal">201</span>
-<span class="normal">202</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">fit_transform</span><span class="p">(</span>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">201</span>
+<span class="normal">202</span>
+<span class="normal">203</span>
+<span class="normal">204</span>
+<span class="normal">205</span>
+<span class="normal">206</span>
+<span class="normal">207</span>
+<span class="normal">208</span>
+<span class="normal">209</span>
+<span class="normal">210</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">fit_transform</span><span class="p">(</span>
     <span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">,</span> <span class="n">target_col</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span>
 <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Union</span><span class="p">[</span>
     <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">,</span>
diff --git a/mkdocs/site/pytorch-widedeep/trainer.html b/mkdocs/site/pytorch-widedeep/trainer.html
index 666c35e8..e1da16a0 100644
--- a/mkdocs/site/pytorch-widedeep/trainer.html
+++ b/mkdocs/site/pytorch-widedeep/trainer.html
@@ -1622,6 +1622,7 @@ <h2 id="pytorch_widedeep.training.Trainer" class="doc doc-heading">
     <span class="n">transforms</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
     <span class="n">callbacks</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
     <span class="n">metrics</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+    <span class="n">eval_metrics</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
     <span class="n">verbose</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span>
     <span class="n">seed</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span>
     <span class="o">**</span><span class="n">kwargs</span>
@@ -1856,7 +1857,8 @@ <h2 id="pytorch_widedeep.training.Trainer" class="doc doc-heading">
 <ul>
 <li>
 <p><strong>device</strong>: <code>str</code><br/>
-    string indicating the device. One of <em>'cpu'</em> or <em>'gpu'</em></p>
+    string indicating the device. One of <em>'cpu'</em>, <em>'gpu'</em> or 'mps' if
+    run on a Mac with Apple silicon or AMD GPU(s)</p>
 </li>
 <li>
 <p><strong>num_workers</strong>: <code>int</code><br/>
@@ -1952,11 +1954,7 @@ <h2 id="pytorch_widedeep.training.Trainer" class="doc doc-heading">
 
                   <details class="quote">
                     <summary>Source code in <code>pytorch_widedeep/training/trainer.py</code></summary>
-                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">226</span>
-<span class="normal">227</span>
-<span class="normal">228</span>
-<span class="normal">229</span>
-<span class="normal">230</span>
+                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">230</span>
 <span class="normal">231</span>
 <span class="normal">232</span>
 <span class="normal">233</span>
@@ -1990,10 +1988,18 @@ <h2 id="pytorch_widedeep.training.Trainer" class="doc doc-heading">
 <span class="normal">261</span>
 <span class="normal">262</span>
 <span class="normal">263</span>
-<span class="normal">264</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="nd">@alias</span><span class="p">(</span>  <span class="c1"># noqa: C901</span>
+<span class="normal">264</span>
+<span class="normal">265</span>
+<span class="normal">266</span>
+<span class="normal">267</span>
+<span class="normal">268</span>
+<span class="normal">269</span>
+<span class="normal">270</span>
+<span class="normal">271</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="nd">@alias</span><span class="p">(</span>  <span class="c1"># noqa: C901</span>
     <span class="s2">&quot;objective&quot;</span><span class="p">,</span>
     <span class="p">[</span><span class="s2">&quot;loss_function&quot;</span><span class="p">,</span> <span class="s2">&quot;loss_fn&quot;</span><span class="p">,</span> <span class="s2">&quot;loss&quot;</span><span class="p">,</span> <span class="s2">&quot;cost_function&quot;</span><span class="p">,</span> <span class="s2">&quot;cost_fn&quot;</span><span class="p">,</span> <span class="s2">&quot;cost&quot;</span><span class="p">],</span>
 <span class="p">)</span>
+<span class="nd">@alias</span><span class="p">(</span><span class="s2">&quot;metrics&quot;</span><span class="p">,</span> <span class="p">[</span><span class="s2">&quot;train_metrics&quot;</span><span class="p">])</span>
 <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
     <span class="bp">self</span><span class="p">,</span>
     <span class="n">model</span><span class="p">:</span> <span class="n">WideDeep</span><span class="p">,</span>
@@ -2011,6 +2017,7 @@ <h2 id="pytorch_widedeep.training.Trainer" class="doc doc-heading">
     <span class="n">transforms</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="n">Transforms</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
     <span class="n">callbacks</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="n">Callback</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
     <span class="n">metrics</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Union</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="n">Metric</span><span class="p">],</span> <span class="n">List</span><span class="p">[</span><span class="n">TorchMetric</span><span class="p">]]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+    <span class="n">eval_metrics</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Union</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="n">Metric</span><span class="p">],</span> <span class="n">List</span><span class="p">[</span><span class="n">TorchMetric</span><span class="p">]]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
     <span class="n">verbose</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1</span><span class="p">,</span>
     <span class="n">seed</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1</span><span class="p">,</span>
     <span class="o">**</span><span class="n">kwargs</span><span class="p">,</span>
@@ -2025,6 +2032,7 @@ <h2 id="pytorch_widedeep.training.Trainer" class="doc doc-heading">
         <span class="n">transforms</span><span class="o">=</span><span class="n">transforms</span><span class="p">,</span>
         <span class="n">callbacks</span><span class="o">=</span><span class="n">callbacks</span><span class="p">,</span>
         <span class="n">metrics</span><span class="o">=</span><span class="n">metrics</span><span class="p">,</span>
+        <span class="n">eval_metrics</span><span class="o">=</span><span class="n">eval_metrics</span><span class="p">,</span>
         <span class="n">verbose</span><span class="o">=</span><span class="n">verbose</span><span class="p">,</span>
         <span class="n">seed</span><span class="o">=</span><span class="n">seed</span><span class="p">,</span>
         <span class="o">**</span><span class="n">kwargs</span><span class="p">,</span>
@@ -2064,7 +2072,8 @@ <h3 id="pytorch_widedeep.training.Trainer.fit" class="doc doc-heading">
     <span class="n">n_epochs</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span>
     <span class="n">validation_freq</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span>
     <span class="n">batch_size</span><span class="o">=</span><span class="mi">32</span><span class="p">,</span>
-    <span class="n">custom_dataloader</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+    <span class="n">train_dataloader</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+    <span class="n">eval_dataloader</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
     <span class="n">feature_importance_sample_size</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
     <span class="n">finetune</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
     <span class="o">**</span><span class="n">kwargs</span>
@@ -2211,9 +2220,6 @@ <h3 id="pytorch_widedeep.training.Trainer.fit" class="doc doc-heading">
         </li>
         <li class="doc-section-item field-body">
           <b><code>custom_dataloader</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="torch.utils.data.DataLoader">DataLoader</span>]</code>, default:
-                  <code>None</code>
-)
           –
           <div class="doc-md-description">
             <p>object of class <code>torch.utils.data.DataLoader</code>. Available
@@ -2322,14 +2328,7 @@ <h3 id="pytorch_widedeep.training.Trainer.fit" class="doc doc-heading">
 
             <details class="quote">
               <summary>Source code in <code>pytorch_widedeep/training/trainer.py</code></summary>
-              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">266</span>
-<span class="normal">267</span>
-<span class="normal">268</span>
-<span class="normal">269</span>
-<span class="normal">270</span>
-<span class="normal">271</span>
-<span class="normal">272</span>
-<span class="normal">273</span>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">273</span>
 <span class="normal">274</span>
 <span class="normal">275</span>
 <span class="normal">276</span>
@@ -2550,39 +2549,9 @@ <h3 id="pytorch_widedeep.training.Trainer.fit" class="doc doc-heading">
 <span class="normal">491</span>
 <span class="normal">492</span>
 <span class="normal">493</span>
-<span class="normal">494</span>
-<span class="normal">495</span>
-<span class="normal">496</span>
-<span class="normal">497</span>
-<span class="normal">498</span>
-<span class="normal">499</span>
-<span class="normal">500</span>
-<span class="normal">501</span>
-<span class="normal">502</span>
-<span class="normal">503</span>
-<span class="normal">504</span>
-<span class="normal">505</span>
-<span class="normal">506</span>
-<span class="normal">507</span>
-<span class="normal">508</span>
-<span class="normal">509</span>
-<span class="normal">510</span>
-<span class="normal">511</span>
-<span class="normal">512</span>
-<span class="normal">513</span>
-<span class="normal">514</span>
-<span class="normal">515</span>
-<span class="normal">516</span>
-<span class="normal">517</span>
-<span class="normal">518</span>
-<span class="normal">519</span>
-<span class="normal">520</span>
-<span class="normal">521</span>
-<span class="normal">522</span>
-<span class="normal">523</span>
-<span class="normal">524</span>
-<span class="normal">525</span>
-<span class="normal">526</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="nd">@alias</span><span class="p">(</span><span class="s2">&quot;finetune&quot;</span><span class="p">,</span> <span class="p">[</span><span class="s2">&quot;warmup&quot;</span><span class="p">])</span>
+<span class="normal">494</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="nd">@alias</span><span class="p">(</span><span class="s2">&quot;finetune&quot;</span><span class="p">,</span> <span class="p">[</span><span class="s2">&quot;warmup&quot;</span><span class="p">])</span>
+<span class="nd">@alias</span><span class="p">(</span><span class="s2">&quot;train_dataloader&quot;</span><span class="p">,</span> <span class="p">[</span><span class="s2">&quot;custom_dataloader&quot;</span><span class="p">])</span>
+<span class="nd">@alias</span><span class="p">(</span><span class="s2">&quot;eval_dataloader&quot;</span><span class="p">,</span> <span class="p">[</span><span class="s2">&quot;custom_eval_dataloader&quot;</span><span class="p">])</span>
 <span class="k">def</span> <span class="nf">fit</span><span class="p">(</span>  <span class="c1"># noqa: C901</span>
     <span class="bp">self</span><span class="p">,</span>
     <span class="n">X_wide</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
@@ -2596,7 +2565,8 @@ <h3 id="pytorch_widedeep.training.Trainer.fit" class="doc doc-heading">
     <span class="n">n_epochs</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1</span><span class="p">,</span>
     <span class="n">validation_freq</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1</span><span class="p">,</span>
     <span class="n">batch_size</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">32</span><span class="p">,</span>
-    <span class="n">custom_dataloader</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">DataLoader</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+    <span class="n">train_dataloader</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">CustomDataLoader</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+    <span class="n">eval_dataloader</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">CustomDataLoader</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
     <span class="n">feature_importance_sample_size</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
     <span class="n">finetune</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
     <span class="o">**</span><span class="n">kwargs</span><span class="p">,</span>
@@ -2733,11 +2703,8 @@ <h3 id="pytorch_widedeep.training.Trainer.fit" class="doc doc-heading">
 <span class="sd">    [Examples](https://github.com/jrzaurin/pytorch-widedeep/tree/master/examples)</span>
 <span class="sd">    folder in the repo</span>
 <span class="sd">    &quot;&quot;&quot;</span>
-
     <span class="n">dataloader_args</span><span class="p">,</span> <span class="n">finetune_args</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_extract_kwargs</span><span class="p">(</span><span class="n">kwargs</span><span class="p">)</span>
-
     <span class="bp">self</span><span class="o">.</span><span class="n">batch_size</span> <span class="o">=</span> <span class="n">batch_size</span>
-
     <span class="n">train_set</span><span class="p">,</span> <span class="n">eval_set</span> <span class="o">=</span> <span class="n">wd_train_val_split</span><span class="p">(</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">seed</span><span class="p">,</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">method</span><span class="p">,</span>  <span class="c1"># type: ignore</span>
@@ -2751,35 +2718,18 @@ <h3 id="pytorch_widedeep.training.Trainer.fit" class="doc doc-heading">
         <span class="n">target</span><span class="p">,</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">transforms</span><span class="p">,</span>
     <span class="p">)</span>
-    <span class="k">if</span> <span class="n">custom_dataloader</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
-        <span class="c1"># make sure is callable (and HAS to be an subclass of DataLoader)</span>
-        <span class="k">assert</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">custom_dataloader</span><span class="p">,</span> <span class="nb">type</span><span class="p">)</span>
-        <span class="n">train_loader</span> <span class="o">=</span> <span class="n">custom_dataloader</span><span class="p">(</span>  <span class="c1"># type: ignore[misc]</span>
-            <span class="n">dataset</span><span class="o">=</span><span class="n">train_set</span><span class="p">,</span>
-            <span class="n">batch_size</span><span class="o">=</span><span class="n">batch_size</span><span class="p">,</span>
-            <span class="n">num_workers</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">num_workers</span><span class="p">,</span>
-            <span class="o">**</span><span class="n">dataloader_args</span><span class="p">,</span>
-        <span class="p">)</span>
-    <span class="k">else</span><span class="p">:</span>
-        <span class="n">train_loader</span> <span class="o">=</span> <span class="n">DataLoader</span><span class="p">(</span>
-            <span class="n">dataset</span><span class="o">=</span><span class="n">train_set</span><span class="p">,</span>
-            <span class="n">batch_size</span><span class="o">=</span><span class="n">batch_size</span><span class="p">,</span>
-            <span class="n">num_workers</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">num_workers</span><span class="p">,</span>
-            <span class="o">**</span><span class="n">dataloader_args</span><span class="p">,</span>
-        <span class="p">)</span>
-    <span class="n">train_steps</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">train_loader</span><span class="p">)</span>
-    <span class="k">if</span> <span class="n">eval_set</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
-        <span class="n">eval_loader</span> <span class="o">=</span> <span class="n">DataLoader</span><span class="p">(</span>
-            <span class="n">dataset</span><span class="o">=</span><span class="n">eval_set</span><span class="p">,</span>
-            <span class="n">batch_size</span><span class="o">=</span><span class="n">batch_size</span><span class="p">,</span>
-            <span class="n">num_workers</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">num_workers</span><span class="p">,</span>
-            <span class="n">shuffle</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
-        <span class="p">)</span>
-        <span class="n">eval_steps</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">eval_loader</span><span class="p">)</span>
+    <span class="n">train_loader</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_set_dataloader</span><span class="p">(</span>
+        <span class="n">train_dataloader</span><span class="p">,</span> <span class="n">train_set</span><span class="p">,</span> <span class="n">batch_size</span><span class="p">,</span> <span class="n">dataloader_args</span>
+    <span class="p">)</span>
+    <span class="n">eval_loader</span> <span class="o">=</span> <span class="p">(</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">_set_dataloader</span><span class="p">(</span><span class="n">eval_dataloader</span><span class="p">,</span> <span class="n">eval_set</span><span class="p">,</span> <span class="n">batch_size</span><span class="p">,</span> <span class="n">dataloader_args</span><span class="p">)</span>
+        <span class="k">if</span> <span class="n">eval_set</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span>
+        <span class="k">else</span> <span class="kc">None</span>
+    <span class="p">)</span>
 
     <span class="k">if</span> <span class="n">finetune</span><span class="p">:</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">with_finetuning</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">_finetune</span><span class="p">(</span><span class="n">train_loader</span><span class="p">,</span> <span class="o">**</span><span class="n">finetune_args</span><span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">_do_finetune</span><span class="p">(</span><span class="n">train_loader</span><span class="p">,</span> <span class="o">**</span><span class="n">finetune_args</span><span class="p">)</span>
         <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">verbose</span><span class="p">:</span>
             <span class="nb">print</span><span class="p">(</span>
                 <span class="s2">&quot;Fine-tuning (or warmup) of individual components completed. &quot;</span>
@@ -2789,54 +2739,32 @@ <h3 id="pytorch_widedeep.training.Trainer.fit" class="doc doc-heading">
         <span class="bp">self</span><span class="o">.</span><span class="n">with_finetuning</span> <span class="o">=</span> <span class="kc">False</span>
 
     <span class="bp">self</span><span class="o">.</span><span class="n">callback_container</span><span class="o">.</span><span class="n">on_train_begin</span><span class="p">(</span>
-        <span class="p">{</span><span class="s2">&quot;batch_size&quot;</span><span class="p">:</span> <span class="n">batch_size</span><span class="p">,</span> <span class="s2">&quot;train_steps&quot;</span><span class="p">:</span> <span class="n">train_steps</span><span class="p">,</span> <span class="s2">&quot;n_epochs&quot;</span><span class="p">:</span> <span class="n">n_epochs</span><span class="p">}</span>
+        <span class="p">{</span>
+            <span class="s2">&quot;batch_size&quot;</span><span class="p">:</span> <span class="n">batch_size</span><span class="p">,</span>
+            <span class="s2">&quot;train_steps&quot;</span><span class="p">:</span> <span class="nb">len</span><span class="p">(</span><span class="n">train_loader</span><span class="p">),</span>
+            <span class="s2">&quot;n_epochs&quot;</span><span class="p">:</span> <span class="n">n_epochs</span><span class="p">,</span>
+        <span class="p">}</span>
     <span class="p">)</span>
     <span class="k">for</span> <span class="n">epoch</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">n_epochs</span><span class="p">):</span>
-        <span class="n">epoch_logs</span><span class="p">:</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="p">{}</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">callback_container</span><span class="o">.</span><span class="n">on_epoch_begin</span><span class="p">(</span><span class="n">epoch</span><span class="p">,</span> <span class="n">logs</span><span class="o">=</span><span class="n">epoch_logs</span><span class="p">)</span>
-
-        <span class="bp">self</span><span class="o">.</span><span class="n">train_running_loss</span> <span class="o">=</span> <span class="mf">0.0</span>
-        <span class="k">with</span> <span class="n">trange</span><span class="p">(</span><span class="n">train_steps</span><span class="p">,</span> <span class="n">disable</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">verbose</span> <span class="o">!=</span> <span class="mi">1</span><span class="p">)</span> <span class="k">as</span> <span class="n">t</span><span class="p">:</span>
-            <span class="k">for</span> <span class="n">batch_idx</span><span class="p">,</span> <span class="p">(</span><span class="n">data</span><span class="p">,</span> <span class="n">targett</span><span class="p">)</span> <span class="ow">in</span> <span class="nb">zip</span><span class="p">(</span><span class="n">t</span><span class="p">,</span> <span class="n">train_loader</span><span class="p">):</span>
-                <span class="n">t</span><span class="o">.</span><span class="n">set_description</span><span class="p">(</span><span class="s2">&quot;epoch </span><span class="si">%i</span><span class="s2">&quot;</span> <span class="o">%</span> <span class="p">(</span><span class="n">epoch</span> <span class="o">+</span> <span class="mi">1</span><span class="p">))</span>
-                <span class="n">train_score</span><span class="p">,</span> <span class="n">train_loss</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_train_step</span><span class="p">(</span><span class="n">data</span><span class="p">,</span> <span class="n">targett</span><span class="p">,</span> <span class="n">batch_idx</span><span class="p">)</span>
-                <span class="n">print_loss_and_metric</span><span class="p">(</span><span class="n">t</span><span class="p">,</span> <span class="n">train_loss</span><span class="p">,</span> <span class="n">train_score</span><span class="p">)</span>
-                <span class="bp">self</span><span class="o">.</span><span class="n">callback_container</span><span class="o">.</span><span class="n">on_batch_end</span><span class="p">(</span><span class="n">batch</span><span class="o">=</span><span class="n">batch_idx</span><span class="p">)</span>
-        <span class="n">epoch_logs</span> <span class="o">=</span> <span class="n">save_epoch_logs</span><span class="p">(</span><span class="n">epoch_logs</span><span class="p">,</span> <span class="n">train_loss</span><span class="p">,</span> <span class="n">train_score</span><span class="p">,</span> <span class="s2">&quot;train&quot;</span><span class="p">)</span>
-
-        <span class="n">on_epoch_end_metric</span> <span class="o">=</span> <span class="kc">None</span>
-        <span class="k">if</span> <span class="n">eval_set</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">and</span> <span class="n">epoch</span> <span class="o">%</span> <span class="n">validation_freq</span> <span class="o">==</span> <span class="p">(</span>
+        <span class="n">epoch_logs</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_train_epoch</span><span class="p">(</span><span class="n">train_loader</span><span class="p">,</span> <span class="n">epoch</span><span class="p">)</span>
+        <span class="k">if</span> <span class="n">eval_loader</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">and</span> <span class="n">epoch</span> <span class="o">%</span> <span class="n">validation_freq</span> <span class="o">==</span> <span class="p">(</span>
             <span class="n">validation_freq</span> <span class="o">-</span> <span class="mi">1</span>
         <span class="p">):</span>
-            <span class="bp">self</span><span class="o">.</span><span class="n">callback_container</span><span class="o">.</span><span class="n">on_eval_begin</span><span class="p">()</span>
-            <span class="bp">self</span><span class="o">.</span><span class="n">valid_running_loss</span> <span class="o">=</span> <span class="mf">0.0</span>
-            <span class="k">with</span> <span class="n">trange</span><span class="p">(</span><span class="n">eval_steps</span><span class="p">,</span> <span class="n">disable</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">verbose</span> <span class="o">!=</span> <span class="mi">1</span><span class="p">)</span> <span class="k">as</span> <span class="n">v</span><span class="p">:</span>
-                <span class="k">for</span> <span class="n">i</span><span class="p">,</span> <span class="p">(</span><span class="n">data</span><span class="p">,</span> <span class="n">targett</span><span class="p">)</span> <span class="ow">in</span> <span class="nb">zip</span><span class="p">(</span><span class="n">v</span><span class="p">,</span> <span class="n">eval_loader</span><span class="p">):</span>
-                    <span class="n">v</span><span class="o">.</span><span class="n">set_description</span><span class="p">(</span><span class="s2">&quot;valid&quot;</span><span class="p">)</span>
-                    <span class="n">val_score</span><span class="p">,</span> <span class="n">val_loss</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_eval_step</span><span class="p">(</span><span class="n">data</span><span class="p">,</span> <span class="n">targett</span><span class="p">,</span> <span class="n">i</span><span class="p">)</span>
-                    <span class="n">print_loss_and_metric</span><span class="p">(</span><span class="n">v</span><span class="p">,</span> <span class="n">val_loss</span><span class="p">,</span> <span class="n">val_score</span><span class="p">)</span>
-            <span class="n">epoch_logs</span> <span class="o">=</span> <span class="n">save_epoch_logs</span><span class="p">(</span><span class="n">epoch_logs</span><span class="p">,</span> <span class="n">val_loss</span><span class="p">,</span> <span class="n">val_score</span><span class="p">,</span> <span class="s2">&quot;val&quot;</span><span class="p">)</span>
-
-            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">reducelronplateau</span><span class="p">:</span>
-                <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">reducelronplateau_criterion</span> <span class="o">==</span> <span class="s2">&quot;loss&quot;</span><span class="p">:</span>
-                    <span class="n">on_epoch_end_metric</span> <span class="o">=</span> <span class="n">val_loss</span>
-                <span class="k">else</span><span class="p">:</span>
-                    <span class="n">on_epoch_end_metric</span> <span class="o">=</span> <span class="n">val_score</span><span class="p">[</span>
-                        <span class="bp">self</span><span class="o">.</span><span class="n">reducelronplateau_criterion</span>
-                    <span class="p">]</span>
+            <span class="n">epoch_logs</span><span class="p">,</span> <span class="n">on_epoch_end_metric</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_eval_epoch</span><span class="p">(</span>
+                <span class="n">eval_loader</span><span class="p">,</span> <span class="n">epoch_logs</span>
+            <span class="p">)</span>
         <span class="k">else</span><span class="p">:</span>
+            <span class="n">on_epoch_end_metric</span> <span class="o">=</span> <span class="kc">None</span>
             <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">reducelronplateau</span><span class="p">:</span>
                 <span class="k">raise</span> <span class="ne">NotImplementedError</span><span class="p">(</span>
                     <span class="s2">&quot;ReduceLROnPlateau scheduler can be used only with validation data.&quot;</span>
                 <span class="p">)</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">callback_container</span><span class="o">.</span><span class="n">on_epoch_end</span><span class="p">(</span><span class="n">epoch</span><span class="p">,</span> <span class="n">epoch_logs</span><span class="p">,</span> <span class="n">on_epoch_end_metric</span><span class="p">)</span>
-
         <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">early_stop</span><span class="p">:</span>
             <span class="c1"># self.callback_container.on_train_end(epoch_logs)</span>
             <span class="k">break</span>
 
     <span class="bp">self</span><span class="o">.</span><span class="n">callback_container</span><span class="o">.</span><span class="n">on_train_end</span><span class="p">(</span><span class="n">epoch_logs</span><span class="p">)</span>
-
     <span class="k">if</span> <span class="n">feature_importance_sample_size</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">feature_importance</span> <span class="o">=</span> <span class="n">FeatureImportance</span><span class="p">(</span>
             <span class="bp">self</span><span class="o">.</span><span class="n">device</span><span class="p">,</span> <span class="n">feature_importance_sample_size</span>
@@ -2961,7 +2889,39 @@ <h3 id="pytorch_widedeep.training.Trainer.predict" class="doc doc-heading">
 
             <details class="quote">
               <summary>Source code in <code>pytorch_widedeep/training/trainer.py</code></summary>
-              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">528</span>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">496</span>
+<span class="normal">497</span>
+<span class="normal">498</span>
+<span class="normal">499</span>
+<span class="normal">500</span>
+<span class="normal">501</span>
+<span class="normal">502</span>
+<span class="normal">503</span>
+<span class="normal">504</span>
+<span class="normal">505</span>
+<span class="normal">506</span>
+<span class="normal">507</span>
+<span class="normal">508</span>
+<span class="normal">509</span>
+<span class="normal">510</span>
+<span class="normal">511</span>
+<span class="normal">512</span>
+<span class="normal">513</span>
+<span class="normal">514</span>
+<span class="normal">515</span>
+<span class="normal">516</span>
+<span class="normal">517</span>
+<span class="normal">518</span>
+<span class="normal">519</span>
+<span class="normal">520</span>
+<span class="normal">521</span>
+<span class="normal">522</span>
+<span class="normal">523</span>
+<span class="normal">524</span>
+<span class="normal">525</span>
+<span class="normal">526</span>
+<span class="normal">527</span>
+<span class="normal">528</span>
 <span class="normal">529</span>
 <span class="normal">530</span>
 <span class="normal">531</span>
@@ -2993,39 +2953,7 @@ <h3 id="pytorch_widedeep.training.Trainer.predict" class="doc doc-heading">
 <span class="normal">557</span>
 <span class="normal">558</span>
 <span class="normal">559</span>
-<span class="normal">560</span>
-<span class="normal">561</span>
-<span class="normal">562</span>
-<span class="normal">563</span>
-<span class="normal">564</span>
-<span class="normal">565</span>
-<span class="normal">566</span>
-<span class="normal">567</span>
-<span class="normal">568</span>
-<span class="normal">569</span>
-<span class="normal">570</span>
-<span class="normal">571</span>
-<span class="normal">572</span>
-<span class="normal">573</span>
-<span class="normal">574</span>
-<span class="normal">575</span>
-<span class="normal">576</span>
-<span class="normal">577</span>
-<span class="normal">578</span>
-<span class="normal">579</span>
-<span class="normal">580</span>
-<span class="normal">581</span>
-<span class="normal">582</span>
-<span class="normal">583</span>
-<span class="normal">584</span>
-<span class="normal">585</span>
-<span class="normal">586</span>
-<span class="normal">587</span>
-<span class="normal">588</span>
-<span class="normal">589</span>
-<span class="normal">590</span>
-<span class="normal">591</span>
-<span class="normal">592</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">predict</span><span class="p">(</span>  <span class="c1"># type: ignore[override, return]</span>
+<span class="normal">560</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">predict</span><span class="p">(</span>  <span class="c1"># type: ignore[override, return]</span>
     <span class="bp">self</span><span class="p">,</span>
     <span class="n">X_wide</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
     <span class="n">X_tab</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Union</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">]]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
@@ -3198,6 +3126,9 @@ <h3 id="pytorch_widedeep.training.Trainer.predict_uncertainty" class="doc doc-he
         </li>
         <li class="doc-section-item field-body">
           <b><code>uncertainty_granularity</code></b>
+              (<code>int</code>, default:
+                  <code>1000</code>
+)
           –
           <div class="doc-md-description">
             <p>number of times the model does prediction for each sample</p>
@@ -3227,7 +3158,39 @@ <h3 id="pytorch_widedeep.training.Trainer.predict_uncertainty" class="doc doc-he
 
             <details class="quote">
               <summary>Source code in <code>pytorch_widedeep/training/trainer.py</code></summary>
-              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">594</span>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">562</span>
+<span class="normal">563</span>
+<span class="normal">564</span>
+<span class="normal">565</span>
+<span class="normal">566</span>
+<span class="normal">567</span>
+<span class="normal">568</span>
+<span class="normal">569</span>
+<span class="normal">570</span>
+<span class="normal">571</span>
+<span class="normal">572</span>
+<span class="normal">573</span>
+<span class="normal">574</span>
+<span class="normal">575</span>
+<span class="normal">576</span>
+<span class="normal">577</span>
+<span class="normal">578</span>
+<span class="normal">579</span>
+<span class="normal">580</span>
+<span class="normal">581</span>
+<span class="normal">582</span>
+<span class="normal">583</span>
+<span class="normal">584</span>
+<span class="normal">585</span>
+<span class="normal">586</span>
+<span class="normal">587</span>
+<span class="normal">588</span>
+<span class="normal">589</span>
+<span class="normal">590</span>
+<span class="normal">591</span>
+<span class="normal">592</span>
+<span class="normal">593</span>
+<span class="normal">594</span>
 <span class="normal">595</span>
 <span class="normal">596</span>
 <span class="normal">597</span>
@@ -3295,39 +3258,7 @@ <h3 id="pytorch_widedeep.training.Trainer.predict_uncertainty" class="doc doc-he
 <span class="normal">659</span>
 <span class="normal">660</span>
 <span class="normal">661</span>
-<span class="normal">662</span>
-<span class="normal">663</span>
-<span class="normal">664</span>
-<span class="normal">665</span>
-<span class="normal">666</span>
-<span class="normal">667</span>
-<span class="normal">668</span>
-<span class="normal">669</span>
-<span class="normal">670</span>
-<span class="normal">671</span>
-<span class="normal">672</span>
-<span class="normal">673</span>
-<span class="normal">674</span>
-<span class="normal">675</span>
-<span class="normal">676</span>
-<span class="normal">677</span>
-<span class="normal">678</span>
-<span class="normal">679</span>
-<span class="normal">680</span>
-<span class="normal">681</span>
-<span class="normal">682</span>
-<span class="normal">683</span>
-<span class="normal">684</span>
-<span class="normal">685</span>
-<span class="normal">686</span>
-<span class="normal">687</span>
-<span class="normal">688</span>
-<span class="normal">689</span>
-<span class="normal">690</span>
-<span class="normal">691</span>
-<span class="normal">692</span>
-<span class="normal">693</span>
-<span class="normal">694</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">predict_uncertainty</span><span class="p">(</span>  <span class="c1"># type: ignore[return]</span>
+<span class="normal">662</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">predict_uncertainty</span><span class="p">(</span>  <span class="c1"># type: ignore[return]</span>
     <span class="bp">self</span><span class="p">,</span>
     <span class="n">X_wide</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
     <span class="n">X_tab</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Union</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">]]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
@@ -3335,7 +3266,7 @@ <h3 id="pytorch_widedeep.training.Trainer.predict_uncertainty" class="doc doc-he
     <span class="n">X_img</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Union</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">]]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
     <span class="n">X_test</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Union</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">]]]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
     <span class="n">batch_size</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">uncertainty_granularity</span><span class="o">=</span><span class="mi">1000</span><span class="p">,</span>
+    <span class="n">uncertainty_granularity</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1000</span><span class="p">,</span>
 <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">:</span>
 <span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Returns the predicted ucnertainty of the model for the test dataset</span>
 <span class="sd">    using a Monte Carlo method during which dropout layers are activated</span>
@@ -3547,7 +3478,39 @@ <h3 id="pytorch_widedeep.training.Trainer.predict_proba" class="doc doc-heading"
 
             <details class="quote">
               <summary>Source code in <code>pytorch_widedeep/training/trainer.py</code></summary>
-              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">696</span>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">664</span>
+<span class="normal">665</span>
+<span class="normal">666</span>
+<span class="normal">667</span>
+<span class="normal">668</span>
+<span class="normal">669</span>
+<span class="normal">670</span>
+<span class="normal">671</span>
+<span class="normal">672</span>
+<span class="normal">673</span>
+<span class="normal">674</span>
+<span class="normal">675</span>
+<span class="normal">676</span>
+<span class="normal">677</span>
+<span class="normal">678</span>
+<span class="normal">679</span>
+<span class="normal">680</span>
+<span class="normal">681</span>
+<span class="normal">682</span>
+<span class="normal">683</span>
+<span class="normal">684</span>
+<span class="normal">685</span>
+<span class="normal">686</span>
+<span class="normal">687</span>
+<span class="normal">688</span>
+<span class="normal">689</span>
+<span class="normal">690</span>
+<span class="normal">691</span>
+<span class="normal">692</span>
+<span class="normal">693</span>
+<span class="normal">694</span>
+<span class="normal">695</span>
+<span class="normal">696</span>
 <span class="normal">697</span>
 <span class="normal">698</span>
 <span class="normal">699</span>
@@ -3570,39 +3533,7 @@ <h3 id="pytorch_widedeep.training.Trainer.predict_proba" class="doc doc-heading"
 <span class="normal">716</span>
 <span class="normal">717</span>
 <span class="normal">718</span>
-<span class="normal">719</span>
-<span class="normal">720</span>
-<span class="normal">721</span>
-<span class="normal">722</span>
-<span class="normal">723</span>
-<span class="normal">724</span>
-<span class="normal">725</span>
-<span class="normal">726</span>
-<span class="normal">727</span>
-<span class="normal">728</span>
-<span class="normal">729</span>
-<span class="normal">730</span>
-<span class="normal">731</span>
-<span class="normal">732</span>
-<span class="normal">733</span>
-<span class="normal">734</span>
-<span class="normal">735</span>
-<span class="normal">736</span>
-<span class="normal">737</span>
-<span class="normal">738</span>
-<span class="normal">739</span>
-<span class="normal">740</span>
-<span class="normal">741</span>
-<span class="normal">742</span>
-<span class="normal">743</span>
-<span class="normal">744</span>
-<span class="normal">745</span>
-<span class="normal">746</span>
-<span class="normal">747</span>
-<span class="normal">748</span>
-<span class="normal">749</span>
-<span class="normal">750</span>
-<span class="normal">751</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">predict_proba</span><span class="p">(</span>  <span class="c1"># type: ignore[override, return]  # noqa: C901</span>
+<span class="normal">719</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">predict_proba</span><span class="p">(</span>  <span class="c1"># type: ignore[override, return]  # noqa: C901</span>
     <span class="bp">self</span><span class="p">,</span>
     <span class="n">X_wide</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
     <span class="n">X_tab</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Union</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">]]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
@@ -3748,7 +3679,39 @@ <h3 id="pytorch_widedeep.training.Trainer.save" class="doc doc-heading">
 
             <details class="quote">
               <summary>Source code in <code>pytorch_widedeep/training/trainer.py</code></summary>
-              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">764</span>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">732</span>
+<span class="normal">733</span>
+<span class="normal">734</span>
+<span class="normal">735</span>
+<span class="normal">736</span>
+<span class="normal">737</span>
+<span class="normal">738</span>
+<span class="normal">739</span>
+<span class="normal">740</span>
+<span class="normal">741</span>
+<span class="normal">742</span>
+<span class="normal">743</span>
+<span class="normal">744</span>
+<span class="normal">745</span>
+<span class="normal">746</span>
+<span class="normal">747</span>
+<span class="normal">748</span>
+<span class="normal">749</span>
+<span class="normal">750</span>
+<span class="normal">751</span>
+<span class="normal">752</span>
+<span class="normal">753</span>
+<span class="normal">754</span>
+<span class="normal">755</span>
+<span class="normal">756</span>
+<span class="normal">757</span>
+<span class="normal">758</span>
+<span class="normal">759</span>
+<span class="normal">760</span>
+<span class="normal">761</span>
+<span class="normal">762</span>
+<span class="normal">763</span>
+<span class="normal">764</span>
 <span class="normal">765</span>
 <span class="normal">766</span>
 <span class="normal">767</span>
@@ -3766,39 +3729,7 @@ <h3 id="pytorch_widedeep.training.Trainer.save" class="doc doc-heading">
 <span class="normal">779</span>
 <span class="normal">780</span>
 <span class="normal">781</span>
-<span class="normal">782</span>
-<span class="normal">783</span>
-<span class="normal">784</span>
-<span class="normal">785</span>
-<span class="normal">786</span>
-<span class="normal">787</span>
-<span class="normal">788</span>
-<span class="normal">789</span>
-<span class="normal">790</span>
-<span class="normal">791</span>
-<span class="normal">792</span>
-<span class="normal">793</span>
-<span class="normal">794</span>
-<span class="normal">795</span>
-<span class="normal">796</span>
-<span class="normal">797</span>
-<span class="normal">798</span>
-<span class="normal">799</span>
-<span class="normal">800</span>
-<span class="normal">801</span>
-<span class="normal">802</span>
-<span class="normal">803</span>
-<span class="normal">804</span>
-<span class="normal">805</span>
-<span class="normal">806</span>
-<span class="normal">807</span>
-<span class="normal">808</span>
-<span class="normal">809</span>
-<span class="normal">810</span>
-<span class="normal">811</span>
-<span class="normal">812</span>
-<span class="normal">813</span>
-<span class="normal">814</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">save</span><span class="p">(</span>
+<span class="normal">782</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">save</span><span class="p">(</span>
     <span class="bp">self</span><span class="p">,</span>
     <span class="n">path</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
     <span class="n">save_state_dict</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
diff --git a/mkdocs/site/search/search_index.json b/mkdocs/site/search/search_index.json
index c3e78296..46740eda 100644
--- a/mkdocs/site/search/search_index.json
+++ b/mkdocs/site/search/search_index.json
@@ -1 +1 @@
-{"config":{"lang":["en"],"separator":"[\\s\\-]+","pipeline":["stopWordFilter"]},"docs":[{"location":"index.html","title":"Home","text":""},{"location":"index.html#pytorch-widedeep","title":"pytorch-widedeep","text":"<p>A flexible package for multimodal-deep-learning to combine tabular data with text and images using Wide and Deep models in Pytorch</p> <p>Documentation: https://pytorch-widedeep.readthedocs.io</p> <p>Companion posts and tutorials: infinitoml</p> <p>Experiments and comparison with <code>LightGBM</code>: TabularDL vs LightGBM</p> <p>Slack: if you want to contribute or just want to chat with us, join slack</p> <p>The content of this document is organized as follows:</p> <ul> <li>pytorch-widedeep<ul> <li>Introduction</li> <li>Architectures</li> <li>The <code>deeptabular</code> component</li> <li>The <code>rec</code> module</li> <li>Text and Images</li> <li>Acknowledgments</li> <li>License</li> <li>Cite<ul> <li>BibTex</li> <li>APA</li> </ul> </li> </ul> </li> </ul>"},{"location":"index.html#introduction","title":"Introduction","text":"<p><code>pytorch-widedeep</code> is based on Google's Wide and Deep Algorithm, adjusted for multi-modal datasets.</p> <p>In general terms, <code>pytorch-widedeep</code> is a package to use deep learning with tabular data. In particular, is intended to facilitate the combination of text and images with corresponding tabular data using wide and deep models. With that in mind there are a number of architectures that can be implemented with the library. The main components of those architectures are shown in the Figure below:</p> <p> </p> <p>In math terms, and following the notation in the paper, the expression for the architecture without a <code>deephead</code> component can be formulated as:</p> <p> </p> <p>Where \u03c3 is the sigmoid function, 'W' are the weight matrices applied to the wide model and to the final activations of the deep models, 'a' are these final activations, \u03c6(x) are the cross product transformations of the original features 'x', and , and 'b' is the bias term. In case you are wondering what are \"cross product transformations\", here is a quote taken directly from the paper: \"For binary features, a cross-product transformation (e.g., \u201cAND(gender=female, language=en)\u201d) is 1 if and only if the constituent features (\u201cgender=female\u201d and \u201clanguage=en\u201d) are all 1, and 0 otherwise\".</p> <p>It is perfectly possible to use custom models (and not necessarily those in the library) as long as the the custom models have a property called <code>output_dim</code> with the size of the last layer of activations, so that <code>WideDeep</code> can be constructed. Examples on how to use custom components can be found in the Examples folder and the section below.</p>"},{"location":"index.html#architectures","title":"Architectures","text":"<p>The <code>pytorch-widedeep</code> library offers a number of different architectures. In this section we will show some of them in their simplest form (i.e. with default param values in most cases) with their corresponding code snippets. Note that all the snippets below shoud run locally. For a more detailed explanation of the different components and their parameters, please refer to the documentation.</p> <p>For the examples below we will be using a toy dataset generated as follows:</p> <pre><code>import os\nimport random\n\nimport numpy as np\nimport pandas as pd\nfrom PIL import Image\nfrom faker import Faker\n\n\ndef create_and_save_random_image(image_number, size=(32, 32)):\n\n    if not os.path.exists(\"images\"):\n        os.makedirs(\"images\")\n\n    array = np.random.randint(0, 256, (size[0], size[1], 3), dtype=np.uint8)\n\n    image = Image.fromarray(array)\n\n    image_name = f\"image_{image_number}.png\"\n    image.save(os.path.join(\"images\", image_name))\n\n    return image_name\n\n\nfake = Faker()\n\ncities = [\"New York\", \"Los Angeles\", \"Chicago\", \"Houston\"]\nnames = [\"Alice\", \"Bob\", \"Charlie\", \"David\", \"Eva\"]\n\ndata = {\n    \"city\": [random.choice(cities) for _ in range(100)],\n    \"name\": [random.choice(names) for _ in range(100)],\n    \"age\": [random.uniform(18, 70) for _ in range(100)],\n    \"height\": [random.uniform(150, 200) for _ in range(100)],\n    \"sentence\": [fake.sentence() for _ in range(100)],\n    \"other_sentence\": [fake.sentence() for _ in range(100)],\n    \"image_name\": [create_and_save_random_image(i) for i in range(100)],\n    \"target\": [random.choice([0, 1]) for _ in range(100)],\n}\n\ndf = pd.DataFrame(data)\n</code></pre> <p>This will create a 100 rows dataframe and a dir in your local folder, called <code>images</code> with 100 random images (or images with just noise).</p> <p>Perhaps the simplest architecture would be just one component, <code>wide</code>, <code>deeptabular</code>, <code>deeptext</code> or <code>deepimage</code> on their own, which is also possible, but let's start the examples with a standard Wide and Deep architecture. From there, how to build a model comprised only of one component will be straightforward.</p> <p>Note that the examples shown below would be almost identical using any of the models available in the library. For example, <code>TabMlp</code> can be replaced by <code>TabResnet</code>, <code>TabNet</code>, <code>TabTransformer</code>, etc. Similarly, <code>BasicRNN</code> can be replaced by <code>AttentiveRNN</code>, <code>StackedAttentiveRNN</code>, or <code>HFModel</code> with their corresponding parameters and preprocessor in the case of the Hugging Face models.</p> <p>1. Wide and Tabular component (aka deeptabular)</p> <p> </p> <pre><code>from pytorch_widedeep.preprocessing import TabPreprocessor, WidePreprocessor\nfrom pytorch_widedeep.models import Wide, TabMlp, WideDeep\nfrom pytorch_widedeep.training import Trainer\n\n# Wide\nwide_cols = [\"city\"]\ncrossed_cols = [(\"city\", \"name\")]\nwide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols)\nX_wide = wide_preprocessor.fit_transform(df)\nwide = Wide(input_dim=np.unique(X_wide).shape[0])\n\n# Tabular\ntab_preprocessor = TabPreprocessor(\n    embed_cols=[\"city\", \"name\"], continuous_cols=[\"age\", \"height\"]\n)\nX_tab = tab_preprocessor.fit_transform(df)\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=tab_preprocessor.continuous_cols,\n    mlp_hidden_dims=[64, 32],\n)\n\n# WideDeep\nmodel = WideDeep(wide=wide, deeptabular=tab_mlp)\n\n# Train\ntrainer = Trainer(model, objective=\"binary\")\n\ntrainer.fit(\n    X_wide=X_wide,\n    X_tab=X_tab,\n    target=df[\"target\"].values,\n    n_epochs=1,\n    batch_size=32,\n)\n</code></pre> <p>2. Tabular and Text data</p> <p> </p> <pre><code>from pytorch_widedeep.preprocessing import TabPreprocessor, TextPreprocessor\nfrom pytorch_widedeep.models import TabMlp, BasicRNN, WideDeep\nfrom pytorch_widedeep.training import Trainer\n\n# Tabular\ntab_preprocessor = TabPreprocessor(\n    embed_cols=[\"city\", \"name\"], continuous_cols=[\"age\", \"height\"]\n)\nX_tab = tab_preprocessor.fit_transform(df)\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=tab_preprocessor.continuous_cols,\n    mlp_hidden_dims=[64, 32],\n)\n\n# Text\ntext_preprocessor = TextPreprocessor(\n    text_col=\"sentence\", maxlen=20, max_vocab=100, n_cpus=1\n)\nX_text = text_preprocessor.fit_transform(df)\nrnn = BasicRNN(\n    vocab_size=len(text_preprocessor.vocab.itos),\n    embed_dim=16,\n    hidden_dim=8,\n    n_layers=1,\n)\n\n# WideDeep\nmodel = WideDeep(deeptabular=tab_mlp, deeptext=rnn)\n\n# Train\ntrainer = Trainer(model, objective=\"binary\")\n\ntrainer.fit(\n    X_tab=X_tab,\n    X_text=X_text,\n    target=df[\"target\"].values,\n    n_epochs=1,\n    batch_size=32,\n)\n</code></pre> <p>3. Tabular and text with a FC head on top via the <code>head_hidden_dims</code> param   in <code>WideDeep</code></p> <p> </p> <pre><code>from pytorch_widedeep.preprocessing import TabPreprocessor, TextPreprocessor\nfrom pytorch_widedeep.models import TabMlp, BasicRNN, WideDeep\nfrom pytorch_widedeep.training import Trainer\n\n# Tabular\ntab_preprocessor = TabPreprocessor(\n    embed_cols=[\"city\", \"name\"], continuous_cols=[\"age\", \"height\"]\n)\nX_tab = tab_preprocessor.fit_transform(df)\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=tab_preprocessor.continuous_cols,\n    mlp_hidden_dims=[64, 32],\n)\n\n# Text\ntext_preprocessor = TextPreprocessor(\n    text_col=\"sentence\", maxlen=20, max_vocab=100, n_cpus=1\n)\nX_text = text_preprocessor.fit_transform(df)\nrnn = BasicRNN(\n    vocab_size=len(text_preprocessor.vocab.itos),\n    embed_dim=16,\n    hidden_dim=8,\n    n_layers=1,\n)\n\n# WideDeep\nmodel = WideDeep(deeptabular=tab_mlp, deeptext=rnn, head_hidden_dims=[32, 16])\n\n# Train\ntrainer = Trainer(model, objective=\"binary\")\n\ntrainer.fit(\n    X_tab=X_tab,\n    X_text=X_text,\n    target=df[\"target\"].values,\n    n_epochs=1,\n    batch_size=32,\n)\n</code></pre> <p>4. Tabular and multiple text columns that are passed directly to   <code>WideDeep</code></p> <p> </p> <pre><code>from pytorch_widedeep.preprocessing import TabPreprocessor, TextPreprocessor\nfrom pytorch_widedeep.models import TabMlp, BasicRNN, WideDeep\nfrom pytorch_widedeep.training import Trainer\n\n\n# Tabular\ntab_preprocessor = TabPreprocessor(\n    embed_cols=[\"city\", \"name\"], continuous_cols=[\"age\", \"height\"]\n)\nX_tab = tab_preprocessor.fit_transform(df)\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=tab_preprocessor.continuous_cols,\n    mlp_hidden_dims=[64, 32],\n)\n\n# Text\ntext_preprocessor_1 = TextPreprocessor(\n    text_col=\"sentence\", maxlen=20, max_vocab=100, n_cpus=1\n)\nX_text_1 = text_preprocessor_1.fit_transform(df)\ntext_preprocessor_2 = TextPreprocessor(\n    text_col=\"other_sentence\", maxlen=20, max_vocab=100, n_cpus=1\n)\nX_text_2 = text_preprocessor_2.fit_transform(df)\nrnn_1 = BasicRNN(\n    vocab_size=len(text_preprocessor_1.vocab.itos),\n    embed_dim=16,\n    hidden_dim=8,\n    n_layers=1,\n)\nrnn_2 = BasicRNN(\n    vocab_size=len(text_preprocessor_2.vocab.itos),\n    embed_dim=16,\n    hidden_dim=8,\n    n_layers=1,\n)\n\n# WideDeep\nmodel = WideDeep(deeptabular=tab_mlp, deeptext=[rnn_1, rnn_2])\n\n# Train\ntrainer = Trainer(model, objective=\"binary\")\n\ntrainer.fit(\n    X_tab=X_tab,\n    X_text=[X_text_1, X_text_2],\n    target=df[\"target\"].values,\n    n_epochs=1,\n    batch_size=32,\n)\n</code></pre> <p>5. Tabular data and multiple text columns that are fused via a the library's   <code>ModelFuser</code> class</p> <p> </p> <pre><code>from pytorch_widedeep.preprocessing import TabPreprocessor, TextPreprocessor\nfrom pytorch_widedeep.models import TabMlp, BasicRNN, WideDeep, ModelFuser\nfrom pytorch_widedeep import Trainer\n\n# Tabular\ntab_preprocessor = TabPreprocessor(\n    embed_cols=[\"city\", \"name\"], continuous_cols=[\"age\", \"height\"]\n)\nX_tab = tab_preprocessor.fit_transform(df)\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=tab_preprocessor.continuous_cols,\n    mlp_hidden_dims=[64, 32],\n)\n\n# Text\ntext_preprocessor_1 = TextPreprocessor(\n    text_col=\"sentence\", maxlen=20, max_vocab=100, n_cpus=1\n)\nX_text_1 = text_preprocessor_1.fit_transform(df)\ntext_preprocessor_2 = TextPreprocessor(\n    text_col=\"other_sentence\", maxlen=20, max_vocab=100, n_cpus=1\n)\nX_text_2 = text_preprocessor_2.fit_transform(df)\n\nrnn_1 = BasicRNN(\n    vocab_size=len(text_preprocessor_1.vocab.itos),\n    embed_dim=16,\n    hidden_dim=8,\n    n_layers=1,\n)\nrnn_2 = BasicRNN(\n    vocab_size=len(text_preprocessor_2.vocab.itos),\n    embed_dim=16,\n    hidden_dim=8,\n    n_layers=1,\n)\n\nmodels_fuser = ModelFuser(models=[rnn_1, rnn_2], fusion_method=\"mult\")\n\n# WideDeep\nmodel = WideDeep(deeptabular=tab_mlp, deeptext=models_fuser)\n\n# Train\ntrainer = Trainer(model, objective=\"binary\")\n\ntrainer.fit(\n    X_tab=X_tab,\n    X_text=[X_text_1, X_text_2],\n    target=df[\"target\"].values,\n    n_epochs=1,\n    batch_size=32,\n)\n</code></pre> <p>6. Tabular and multiple text columns, with an image column. The text columns   are fused via the library's <code>ModelFuser</code> and then all fused via the   deephead paramenter in <code>WideDeep</code> which is a custom <code>ModelFuser</code> coded by   the user</p> <p>This is perhaps the less elegant solution as it involves a custom component by the user and slicing the 'incoming' tensor. In the future, we will include a <code>TextAndImageModelFuser</code> to make this process more straightforward. Still, is not really complicated and it is a good example of how to use custom components in <code>pytorch-widedeep</code>.</p> <p>Note that the only requirement for the custom component is that it has a property called <code>output_dim</code> that returns the size of the last layer of activations. In other words, it does not need to inherit from <code>BaseWDModelComponent</code>. This base class simply checks the existence of such property and avoids some typing errors internally.</p> <p> </p> <pre><code>import torch\n\nfrom pytorch_widedeep.preprocessing import TabPreprocessor, TextPreprocessor, ImagePreprocessor\nfrom pytorch_widedeep.models import TabMlp, BasicRNN, WideDeep, ModelFuser, Vision\nfrom pytorch_widedeep.models._base_wd_model_component import BaseWDModelComponent\nfrom pytorch_widedeep import Trainer\n\n# Tabular\ntab_preprocessor = TabPreprocessor(\n    embed_cols=[\"city\", \"name\"], continuous_cols=[\"age\", \"height\"]\n)\nX_tab = tab_preprocessor.fit_transform(df)\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=tab_preprocessor.continuous_cols,\n    mlp_hidden_dims=[16, 8],\n)\n\n# Text\ntext_preprocessor_1 = TextPreprocessor(\n    text_col=\"sentence\", maxlen=20, max_vocab=100, n_cpus=1\n)\nX_text_1 = text_preprocessor_1.fit_transform(df)\ntext_preprocessor_2 = TextPreprocessor(\n    text_col=\"other_sentence\", maxlen=20, max_vocab=100, n_cpus=1\n)\nX_text_2 = text_preprocessor_2.fit_transform(df)\nrnn_1 = BasicRNN(\n    vocab_size=len(text_preprocessor_1.vocab.itos),\n    embed_dim=16,\n    hidden_dim=8,\n    n_layers=1,\n)\nrnn_2 = BasicRNN(\n    vocab_size=len(text_preprocessor_2.vocab.itos),\n    embed_dim=16,\n    hidden_dim=8,\n    n_layers=1,\n)\nmodels_fuser = ModelFuser(\n    models=[rnn_1, rnn_2],\n    fusion_method=\"mult\",\n)\n\n# Image\nimage_preprocessor = ImagePreprocessor(img_col=\"image_name\", img_path=\"images\")\nX_img = image_preprocessor.fit_transform(df)\nvision = Vision(pretrained_model_setup=\"resnet18\", head_hidden_dims=[16, 8])\n\n# deephead (custom model fuser)\nclass MyModelFuser(BaseWDModelComponent):\n    \"\"\"\n    Simply a Linear + Relu sequence on top of the text + images followed by a\n    Linear -&gt; Relu -&gt; Linear for the concatenation of tabular slice of the\n    tensor and the output of the text and image sequential model\n    \"\"\"\n    def __init__(\n        self,\n        tab_incoming_dim: int,\n        text_incoming_dim: int,\n        image_incoming_dim: int,\n        output_units: int,\n    ):\n\n        super(MyModelFuser, self).__init__()\n\n        self.tab_incoming_dim = tab_incoming_dim\n        self.text_incoming_dim = text_incoming_dim\n        self.image_incoming_dim = image_incoming_dim\n        self.output_units = output_units\n        self.text_and_image_fuser = torch.nn.Sequential(\n            torch.nn.Linear(text_incoming_dim + image_incoming_dim, output_units),\n            torch.nn.ReLU(),\n        )\n        self.out = torch.nn.Sequential(\n            torch.nn.Linear(output_units + tab_incoming_dim, output_units * 4),\n            torch.nn.ReLU(),\n            torch.nn.Linear(output_units * 4, output_units),\n        )\n\n    def forward(self, X: torch.Tensor) -&gt; torch.Tensor:\n        tab_slice = slice(0, self.tab_incoming_dim)\n        text_slice = slice(\n            self.tab_incoming_dim, self.tab_incoming_dim + self.text_incoming_dim\n        )\n        image_slice = slice(\n            self.tab_incoming_dim + self.text_incoming_dim,\n            self.tab_incoming_dim + self.text_incoming_dim + self.image_incoming_dim,\n        )\n        X_tab = X[:, tab_slice]\n        X_text = X[:, text_slice]\n        X_img = X[:, image_slice]\n        X_text_and_image = self.text_and_image_fuser(torch.cat([X_text, X_img], dim=1))\n        return self.out(torch.cat([X_tab, X_text_and_image], dim=1))\n\n    @property\n    def output_dim(self):\n        return self.output_units\n\ndeephead = MyModelFuser(\n    tab_incoming_dim=tab_mlp.output_dim,\n    text_incoming_dim=models_fuser.output_dim,\n    image_incoming_dim=vision.output_dim,\n    output_units=8,\n)\n\n# WideDeep\nmodel = WideDeep(\n    deeptabular=tab_mlp,\n    deeptext=models_fuser,\n    deepimage=vision,\n    deephead=deephead,\n)\n\n# Train\ntrainer = Trainer(model, objective=\"binary\")\n\ntrainer.fit(\n    X_tab=X_tab,\n    X_text=[X_text_1, X_text_2],\n    X_img=X_img,\n    target=df[\"target\"].values,\n    n_epochs=1,\n    batch_size=32,\n)\n</code></pre> <p>7. A two-tower model</p> <p>This is a popular model in the context of recommendation systems. Let's say we have a tabular dataset formed my triples (user features, item features, target). We can create a two-tower model where the user and item features are passed through two separate models and then \"fused\" via a dot product.</p> <p> </p> <pre><code>import numpy as np\nimport pandas as pd\n\nfrom pytorch_widedeep import Trainer\nfrom pytorch_widedeep.preprocessing import TabPreprocessor\nfrom pytorch_widedeep.models import TabMlp, WideDeep, ModelFuser\n\n# Let's create the interaction dataset\n# user_features dataframe\nnp.random.seed(42)\nuser_ids = np.arange(1, 101)\nages = np.random.randint(18, 60, size=100)\ngenders = np.random.choice([\"male\", \"female\"], size=100)\nlocations = np.random.choice([\"city_a\", \"city_b\", \"city_c\", \"city_d\"], size=100)\nuser_features = pd.DataFrame(\n    {\"id\": user_ids, \"age\": ages, \"gender\": genders, \"location\": locations}\n)\n\n# item_features dataframe\nitem_ids = np.arange(1, 101)\nprices = np.random.uniform(10, 500, size=100).round(2)\ncolors = np.random.choice([\"red\", \"blue\", \"green\", \"black\"], size=100)\ncategories = np.random.choice([\"electronics\", \"clothing\", \"home\", \"toys\"], size=100)\n\nitem_features = pd.DataFrame(\n    {\"id\": item_ids, \"price\": prices, \"color\": colors, \"category\": categories}\n)\n\n# Interactions dataframe\ninteraction_user_ids = np.random.choice(user_ids, size=1000)\ninteraction_item_ids = np.random.choice(item_ids, size=1000)\npurchased = np.random.choice([0, 1], size=1000, p=[0.7, 0.3])\ninteractions = pd.DataFrame(\n    {\n        \"user_id\": interaction_user_ids,\n        \"item_id\": interaction_item_ids,\n        \"purchased\": purchased,\n    }\n)\nuser_item_purchased = interactions.merge(\n    user_features, left_on=\"user_id\", right_on=\"id\"\n).merge(item_features, left_on=\"item_id\", right_on=\"id\")\n\n# Users\ntab_preprocessor_user = TabPreprocessor(\n    cat_embed_cols=[\"gender\", \"location\"],\n    continuous_cols=[\"age\"],\n)\nX_user = tab_preprocessor_user.fit_transform(user_item_purchased)\ntab_mlp_user = TabMlp(\n    column_idx=tab_preprocessor_user.column_idx,\n    cat_embed_input=tab_preprocessor_user.cat_embed_input,\n    continuous_cols=[\"age\"],\n    mlp_hidden_dims=[16, 8],\n    mlp_dropout=[0.2, 0.2],\n)\n\n# Items\ntab_preprocessor_item = TabPreprocessor(\n    cat_embed_cols=[\"color\", \"category\"],\n    continuous_cols=[\"price\"],\n)\nX_item = tab_preprocessor_item.fit_transform(user_item_purchased)\ntab_mlp_item = TabMlp(\n    column_idx=tab_preprocessor_item.column_idx,\n    cat_embed_input=tab_preprocessor_item.cat_embed_input,\n    continuous_cols=[\"price\"],\n    mlp_hidden_dims=[16, 8],\n    mlp_dropout=[0.2, 0.2],\n)\n\ntwo_tower_model = ModelFuser([tab_mlp_user, tab_mlp_item], fusion_method=\"dot\")\n\nmodel = WideDeep(deeptabular=two_tower_model)\n\ntrainer = Trainer(model, objective=\"binary\")\n\ntrainer.fit(\n    X_tab=[X_user, X_item],\n    target=interactions.purchased.values,\n    n_epochs=1,\n    batch_size=32,\n)\n</code></pre> <p>8. Tabular with a multi-target loss</p> <p>This one is \"a bonus\" to illustrate the use of multi-target losses, more than actually a different architecture.</p> <p> </p> <pre><code>from pytorch_widedeep.preprocessing import TabPreprocessor, TextPreprocessor, ImagePreprocessor\nfrom pytorch_widedeep.models import TabMlp, BasicRNN, WideDeep, ModelFuser, Vision\nfrom pytorch_widedeep.losses_multitarget import MultiTargetClassificationLoss\nfrom pytorch_widedeep.models._base_wd_model_component import BaseWDModelComponent\nfrom pytorch_widedeep import Trainer\n\n# let's add a second target to the dataframe\ndf[\"target2\"] = [random.choice([0, 1]) for _ in range(100)]\n\n# Tabular\ntab_preprocessor = TabPreprocessor(\n    embed_cols=[\"city\", \"name\"], continuous_cols=[\"age\", \"height\"]\n)\nX_tab = tab_preprocessor.fit_transform(df)\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=tab_preprocessor.continuous_cols,\n    mlp_hidden_dims=[64, 32],\n)\n\n# 'pred_dim=2' because we have two binary targets. For other types of targets,\n#  please, see the documentation\nmodel = WideDeep(deeptabular=tab_mlp, pred_dim=2).\n\nloss = MultiTargetClassificationLoss(binary_config=[0, 1], reduction=\"mean\")\n\n# When a multi-target loss is used, 'custom_loss_function' must not be None.\n# See the docs\ntrainer = Trainer(model, objective=\"multitarget\", custom_loss_function=loss)\n\ntrainer.fit(\n    X_tab=X_tab,\n    target=df[[\"target\", \"target2\"]].values,\n    n_epochs=1,\n    batch_size=32,\n)\n</code></pre>"},{"location":"index.html#the-deeptabular-component","title":"The <code>deeptabular</code> component","text":"<p>It is important to emphasize again that each individual component, <code>wide</code>, <code>deeptabular</code>, <code>deeptext</code> and <code>deepimage</code>, can be used independently and in isolation. For example, one could use only <code>wide</code>, which is in simply a linear model. In fact, one of the most interesting functionalities in<code>pytorch-widedeep</code> would be the use of the <code>deeptabular</code> component on its own, i.e. what one might normally refer as Deep Learning for Tabular Data. Currently, <code>pytorch-widedeep</code> offers the following different models for that component:</p> <ol> <li>Wide: a simple linear model where the nonlinearities are captured via cross-product transformations, as explained before.</li> <li>TabMlp: a simple MLP that receives embeddings representing the categorical features, concatenated with the continuous features, which can also be embedded.</li> <li>TabResnet: similar to the previous model but the embeddings are passed through a series of ResNet blocks built with dense layers.</li> <li>TabNet: details on TabNet can be found in TabNet: Attentive Interpretable Tabular Learning</li> </ol> <p>Two simpler attention based models that we call:</p> <ol> <li>ContextAttentionMLP: MLP with at attention mechanism \"on top\" that is based on     Hierarchical Attention Networks for Document Classification</li> <li>SelfAttentionMLP: MLP with an attention mechanism that is a simplified     version of a transformer block that we refer as \"query-key self-attention\".</li> </ol> <p>The <code>Tabformer</code> family, i.e. Transformers for Tabular data:</p> <ol> <li>TabTransformer: details on the TabTransformer can be found in TabTransformer: Tabular Data Modeling Using Contextual Embeddings.</li> <li>SAINT: Details on SAINT can be found in SAINT: Improved Neural Networks for Tabular Data via Row Attention and Contrastive Pre-Training.</li> <li>FT-Transformer: details on the FT-Transformer can be found in Revisiting Deep Learning Models for Tabular Data.</li> <li>TabFastFormer: adaptation of the FastFormer for tabular data. Details on the Fasformer can be found in FastFormers: Highly Efficient Transformer Models for Natural Language Understanding</li> <li>TabPerceiver: adaptation of the Perceiver for tabular data. Details on the Perceiver can be found in Perceiver: General Perception with Iterative Attention</li> </ol> <p>And probabilistic DL models for tabular data based on Weight Uncertainty in Neural Networks:</p> <ol> <li>BayesianWide: Probabilistic adaptation of the <code>Wide</code> model.</li> <li>BayesianTabMlp: Probabilistic adaptation of the <code>TabMlp</code> model</li> </ol> <p>Note that while there are scientific publications for the TabTransformer, SAINT and FT-Transformer, the TabFasfFormer and TabPerceiver are our own adaptation of those algorithms for tabular data.</p> <p>In addition, Self-Supervised pre-training can be used for all <code>deeptabular</code> models, with the exception of the <code>TabPerceiver</code>. Self-Supervised pre-training can be used via two methods or routines which we refer as: encoder-decoder method and constrastive-denoising method. Please, see the documentation and the examples for details on this functionality, and all other options in the library.</p>"},{"location":"index.html#the-rec-module","title":"The <code>rec</code> module","text":"<p>This module was introduced as an extension to the existing components in the library, addressing questions and issues related to recommendation systems. While still under active development, it currently includes a select number of powerful recommendation models.</p> <p>It's worth noting that this library already supported the implementation of various recommendation algorithms using existing components. For example, models like Wide and Deep, Two-Tower, or Neural Collaborative Filtering could be constructed using the library's core functionalities.</p> <p>The recommendation algorithms in the <code>rec</code> module are:</p> <ol> <li>DeepFM: A Factorization-Machine based Neural Network for CTR Prediction</li> <li>(Deep) Field Aware Factorization Machine (FFM): a Deep Learning version of the algorithm presented in Field-aware Factorization Machines in a Real-world Online Advertising System</li> <li>xDeepFM: Combining Explicit and Implicit Feature Interactions for Recommender Systems</li> <li>Deep Interest Network for Click-Through Rate Prediction</li> </ol> <p>These can all be used as the <code>deeptabular</code> component in the <code>WideDeep</code> model. See the examples for more details.</p>"},{"location":"index.html#text-and-images","title":"Text and Images","text":"<p>For the text component, <code>deeptext</code>, the library offers the following models:</p> <ol> <li>BasicRNN: a simple RNN 2. AttentiveRNN: a RNN with an attention mechanism based on the Hierarchical Attention Networks for DocumentClassification</li> <li>StackedAttentiveRNN: a stack of AttentiveRNNs</li> <li>HFModel: a wrapper around Hugging Face Transfomer-based models. At the moment only models from the families BERT, RoBERTa, DistilBERT, ALBERT and ELECTRA are supported. This is because this library is designed to address classification and regression tasks and these are the most 'popular' encoder-only models, which have proved to be those that work best for these tasks. If there is demand for other models, they will be included in the future.</li> </ol> <p>For the image component, <code>deepimage</code>, the library supports models from the following families: 'resnet', 'shufflenet', 'resnext', 'wide_resnet', 'regnet', 'densenet', 'mobilenetv3',  'mobilenetv2', 'mnasnet', 'efficientnet' and 'squeezenet'.  These are  offered via <code>torchvision</code> and wrapped up in the <code>Vision</code> class.</p>"},{"location":"index.html#acknowledgments","title":"Acknowledgments","text":"<p>This library takes from a series of other libraries, so I think it is just fair to mention them here in the README (specific mentions are also included in the code).</p> <p>The <code>Callbacks</code> and <code>Initializers</code> structure and code is inspired by the <code>torchsample</code> library, which in itself partially inspired by <code>Keras</code>.</p> <p>The <code>TextProcessor</code> class in this library uses the <code>fastai</code>'s <code>Tokenizer</code> and <code>Vocab</code>. The code at <code>utils.fastai_transforms</code> is a minor adaptation of their code so it functions within this library. To my experience their <code>Tokenizer</code> is the best in class.</p> <p>The <code>ImageProcessor</code> class in this library uses code from the fantastic Deep Learning for Computer Vision (DL4CV) book by Adrian Rosebrock.</p>"},{"location":"index.html#license","title":"License","text":"<p>This work is dual-licensed under Apache 2.0 and MIT (or any later version). You can choose between one of them if you use this work.</p> <p><code>SPDX-License-Identifier: Apache-2.0 AND MIT</code></p>"},{"location":"index.html#cite","title":"Cite","text":""},{"location":"index.html#bibtex","title":"BibTex","text":"<pre><code>@article{Zaurin_pytorch-widedeep_A_flexible_2023,\nauthor = {Zaurin, Javier Rodriguez and Mulinka, Pavol},\ndoi = {10.21105/joss.05027},\njournal = {Journal of Open Source Software},\nmonth = jun,\nnumber = {86},\npages = {5027},\ntitle = {{pytorch-widedeep: A flexible package for multimodal deep learning}},\nurl = {https://joss.theoj.org/papers/10.21105/joss.05027},\nvolume = {8},\nyear = {2023}\n}\n</code></pre>"},{"location":"index.html#apa","title":"APA","text":"<pre><code>Zaurin, J. R., &amp; Mulinka, P. (2023). pytorch-widedeep: A flexible package for\nmultimodal deep learning. Journal of Open Source Software, 8(86), 5027.\nhttps://doi.org/10.21105/joss.05027\n</code></pre>"},{"location":"contributing.html","title":"Contributing","text":"<p>Pytorch-widedeep is being developed and used by many active community members. Your help is very valuable to make it better for everyone.</p> <ul> <li>Check for the Roadmap or Open an issue to report problems or recommend new features and submit a draft pull requests, which will be changed to pull request after intial review</li> <li>Contribute to the tests to make it more reliable.</li> <li>Contribute to the documentation to make it clearer for everyone.</li> <li>Contribute to the examples to share your experience with other users.</li> <li>Join the dicussion on slack</li> </ul>"},{"location":"installation.html","title":"Installation","text":"<p>This section explains how to install <code>pytorch-widedeep</code>.</p> <p>For the latest stable release, execute:</p> <pre><code>pip install pytorch-widedeep\n</code></pre> <p>For the bleeding-edge version, execute:</p> <pre><code>pip install git+https://github.com/jrzaurin/pytorch-widedeep.git\n</code></pre> <p>For developer install</p> <pre><code># Clone the repository\ngit clone https://github.com/jrzaurin/pytorch-widedeep\ncd pytorch-widedeep\n\n# Install in dev mode\npip install -e .\n</code></pre>"},{"location":"installation.html#dependencies","title":"Dependencies","text":"<ul> <li>pandas&gt;=1.3.5</li> <li>numpy&gt;=1.21.6</li> <li>scipy&gt;=1.7.3,&lt;=1.12.0</li> <li>scikit-learn&gt;=1.0.2</li> <li>gensim</li> <li>spacy</li> <li>opencv-contrib-python</li> <li>imutils</li> <li>tqdm</li> <li>torch &gt;= 2.0.0</li> <li>torchvision &gt;= 0.15.0</li> <li>einops</li> <li>wrapt</li> <li>torchmetrics</li> <li>pyarrow</li> <li>fastparquet&gt;=0.8.1</li> <li>transformers</li> <li>sentence-transformers</li> <li>sentencepiece</li> </ul>"},{"location":"quick_start.html","title":"Quick Start","text":"<p>This is an example of a binary classification with the adult census dataset using a combination of a wide and deep model (in this case a so called <code>deeptabular</code> model) with defaults settings.</p> <pre><code>import numpy as np\nimport torch\nfrom sklearn.model_selection import train_test_split\n\nfrom pytorch_widedeep import Trainer\nfrom pytorch_widedeep.preprocessing import WidePreprocessor, TabPreprocessor\nfrom pytorch_widedeep.models import Wide, TabMlp, WideDeep\nfrom pytorch_widedeep.metrics import Accuracy\nfrom pytorch_widedeep.datasets import load_adult\n\n\ndf = load_adult(as_frame=True)\ndf[\"income_label\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int)\ndf.drop(\"income\", axis=1, inplace=True)\ndf_train, df_test = train_test_split(df, test_size=0.2, stratify=df.income_label)\n\n# Define the 'column set up'\nwide_cols = [\n    \"education\",\n    \"relationship\",\n    \"workclass\",\n    \"occupation\",\n    \"native-country\",\n    \"gender\",\n]\ncrossed_cols = [(\"education\", \"occupation\"), (\"native-country\", \"occupation\")]\n\ncat_embed_cols = [\n    \"workclass\",\n    \"education\",\n    \"marital-status\",\n    \"occupation\",\n    \"relationship\",\n    \"race\",\n    \"gender\",\n    \"capital-gain\",\n    \"capital-loss\",\n    \"native-country\",\n]\ncontinuous_cols = [\"age\", \"hours-per-week\"]\ntarget = \"income_label\"\ntarget = df_train[target].values\n\n# prepare the data\nwide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols)\nX_wide = wide_preprocessor.fit_transform(df_train)\n\ntab_preprocessor = TabPreprocessor(\n    cat_embed_cols=cat_embed_cols, continuous_cols=continuous_cols  # type: ignore[arg-type]\n)\nX_tab = tab_preprocessor.fit_transform(df_train)\n\n# build the model\nwide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1)\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=continuous_cols,\n)\nmodel = WideDeep(wide=wide, deeptabular=tab_mlp)\n\n# train and validate\ntrainer = Trainer(model, objective=\"binary\", metrics=[Accuracy])\ntrainer.fit(\n    X_wide=X_wide,\n    X_tab=X_tab,\n    target=target,\n    n_epochs=5,\n    batch_size=256,\n)\n\n# predict on test\nX_wide_te = wide_preprocessor.transform(df_test)\nX_tab_te = tab_preprocessor.transform(df_test)\npreds = trainer.predict(X_wide=X_wide_te, X_tab=X_tab_te)\n\n# Save and load\n\n# Option 1: this will also save training history and lr history if the\n# LRHistory callback is used\ntrainer.save(path=\"model_weights\", save_state_dict=True)\n\n# Option 2: save as any other torch model\ntorch.save(model.state_dict(), \"model_weights/wd_model.pt\")\n\n# From here in advance, Option 1 or 2 are the same. I assume the user has\n# prepared the data and defined the new model components:\n# 1. Build the model\nmodel_new = WideDeep(wide=wide, deeptabular=tab_mlp)\nmodel_new.load_state_dict(torch.load(\"model_weights/wd_model.pt\"))\n\n# 2. Instantiate the trainer\ntrainer_new = Trainer(model_new, objective=\"binary\")\n\n# 3. Either start the fit or directly predict\npreds = trainer_new.predict(X_wide=X_wide, X_tab=X_tab, batch_size=32)\n</code></pre>"},{"location":"examples/01_preprocessors_and_utils.html","title":"01_preprocessors_and_utils","text":"<p>For example</p> In\u00a0[1]: Copied! <pre>import numpy as np\nimport pandas as pd\nimport pytorch_widedeep as wd\n\nfrom pytorch_widedeep.datasets import load_adult\nfrom pytorch_widedeep.preprocessing import WidePreprocessor\n</pre> import numpy as np import pandas as pd import pytorch_widedeep as wd  from pytorch_widedeep.datasets import load_adult from pytorch_widedeep.preprocessing import WidePreprocessor <pre>/Users/javierrodriguezzaurin/.pyenv/versions/3.10.13/envs/widedeep310/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n  from .autonotebook import tqdm as notebook_tqdm\n</pre> In\u00a0[2]: Copied! <pre>df = load_adult(as_frame=True)\ndf.head()\n</pre> df = load_adult(as_frame=True) df.head() Out[2]: age workclass fnlwgt education educational-num marital-status occupation relationship race gender capital-gain capital-loss hours-per-week native-country income 0 25 Private 226802 11th 7 Never-married Machine-op-inspct Own-child Black Male 0 0 40 United-States &lt;=50K 1 38 Private 89814 HS-grad 9 Married-civ-spouse Farming-fishing Husband White Male 0 0 50 United-States &lt;=50K 2 28 Local-gov 336951 Assoc-acdm 12 Married-civ-spouse Protective-serv Husband White Male 0 0 40 United-States &gt;50K 3 44 Private 160323 Some-college 10 Married-civ-spouse Machine-op-inspct Husband Black Male 7688 0 40 United-States &gt;50K 4 18 ? 103497 Some-college 10 Never-married ? Own-child White Female 0 0 30 United-States &lt;=50K In\u00a0[3]: Copied! <pre>wide_cols = [\n    \"education\",\n    \"relationship\",\n    \"workclass\",\n    \"occupation\",\n    \"native-country\",\n    \"gender\",\n]\ncrossed_cols = [(\"education\", \"occupation\"), (\"native-country\", \"occupation\")]\n</pre> wide_cols = [     \"education\",     \"relationship\",     \"workclass\",     \"occupation\",     \"native-country\",     \"gender\", ] crossed_cols = [(\"education\", \"occupation\"), (\"native-country\", \"occupation\")] In\u00a0[4]: Copied! <pre>wide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols)\nX_wide = wide_preprocessor.fit_transform(df)\n# From here on, any new observation can be prepared by simply running `.transform`\n# new_X_wide = wide_preprocessor.transform(new_df)\n</pre> wide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols) X_wide = wide_preprocessor.fit_transform(df) # From here on, any new observation can be prepared by simply running `.transform` # new_X_wide = wide_preprocessor.transform(new_df) In\u00a0[5]: Copied! <pre>X_wide\n</pre> X_wide Out[5]: <pre>array([[  1,  17,  23, ...,  89,  91, 316],\n       [  2,  18,  23, ...,  89,  92, 317],\n       [  3,  18,  24, ...,  89,  93, 318],\n       ...,\n       [  2,  20,  23, ...,  90, 103, 323],\n       [  2,  17,  23, ...,  89, 103, 323],\n       [  2,  21,  29, ...,  90, 115, 324]])</pre> <p>Note that the label encoding starts from <code>1</code>. This is because it is convenient to leave <code>0</code> for padding, i.e. unknown categories. Let's take from example the first entry</p> In\u00a0[6]: Copied! <pre>X_wide[0]\n</pre> X_wide[0] Out[6]: <pre>array([  1,  17,  23,  32,  47,  89,  91, 316])</pre> In\u00a0[7]: Copied! <pre>wide_preprocessor.inverse_transform(X_wide[:1])\n</pre> wide_preprocessor.inverse_transform(X_wide[:1]) Out[7]: education relationship workclass occupation native-country gender education_occupation native-country_occupation 0 11th Own-child Private Machine-op-inspct United-States Male 11th-Machine-op-inspct United-States-Machine-op-inspct <p>As we can see, <code>wide_preprocessor</code> numerically encodes the <code>wide_cols</code> and the <code>crossed_cols</code>, which can be recovered using the method <code>inverse_transform</code>.</p> In\u00a0[8]: Copied! <pre>from pytorch_widedeep.preprocessing import TabPreprocessor\n</pre> from pytorch_widedeep.preprocessing import TabPreprocessor In\u00a0[9]: Copied! <pre># cat_embed_cols = [(column_name, embed_dim), ...]\ncat_embed_cols = [\n    (\"education\", 10),\n    (\"relationship\", 8),\n    (\"workclass\", 10),\n    (\"occupation\", 10),\n    (\"native-country\", 10),\n]\ncontinuous_cols = [\"age\", \"hours-per-week\"]\n</pre> # cat_embed_cols = [(column_name, embed_dim), ...] cat_embed_cols = [     (\"education\", 10),     (\"relationship\", 8),     (\"workclass\", 10),     (\"occupation\", 10),     (\"native-country\", 10), ] continuous_cols = [\"age\", \"hours-per-week\"] In\u00a0[10]: Copied! <pre>tab_preprocessor = TabPreprocessor(\n    cat_embed_cols=cat_embed_cols,\n    continuous_cols=continuous_cols,\n    cols_to_scale=[\"age\"],  # or scale=True or cols_to_scale=continuous_cols\n)\nX_tab = tab_preprocessor.fit_transform(df)\n# From here on, any new observation can be prepared by simply running `.transform`\n# new_X_deep = deep_preprocessor.transform(new_df)\n</pre> tab_preprocessor = TabPreprocessor(     cat_embed_cols=cat_embed_cols,     continuous_cols=continuous_cols,     cols_to_scale=[\"age\"],  # or scale=True or cols_to_scale=continuous_cols ) X_tab = tab_preprocessor.fit_transform(df) # From here on, any new observation can be prepared by simply running `.transform` # new_X_deep = deep_preprocessor.transform(new_df) In\u00a0[11]: Copied! <pre>X_tab\n</pre> X_tab Out[11]: <pre>array([[ 1.00000000e+00,  1.00000000e+00,  1.00000000e+00, ...,\n         1.00000000e+00, -9.95128932e-01,  4.00000000e+01],\n       [ 2.00000000e+00,  2.00000000e+00,  1.00000000e+00, ...,\n         1.00000000e+00, -4.69415091e-02,  5.00000000e+01],\n       [ 3.00000000e+00,  2.00000000e+00,  2.00000000e+00, ...,\n         1.00000000e+00, -7.76316450e-01,  4.00000000e+01],\n       ...,\n       [ 2.00000000e+00,  4.00000000e+00,  1.00000000e+00, ...,\n         1.00000000e+00,  1.41180837e+00,  4.00000000e+01],\n       [ 2.00000000e+00,  1.00000000e+00,  1.00000000e+00, ...,\n         1.00000000e+00, -1.21394141e+00,  2.00000000e+01],\n       [ 2.00000000e+00,  5.00000000e+00,  7.00000000e+00, ...,\n         1.00000000e+00,  9.74183408e-01,  4.00000000e+01]])</pre> <p>Note that the label encoding starts from <code>1</code>. This is because it is convenient to leave <code>0</code> for padding, i.e. unknown categories. Let's take from example the first entry</p> In\u00a0[12]: Copied! <pre>X_tab[0]\n</pre> X_tab[0] Out[12]: <pre>array([ 1.        ,  1.        ,  1.        ,  1.        ,  1.        ,\n       -0.99512893, 40.        ])</pre> In\u00a0[13]: Copied! <pre>tab_preprocessor.inverse_transform(X_tab[:1])\n</pre> tab_preprocessor.inverse_transform(X_tab[:1]) Out[13]: education relationship workclass occupation native-country age hours-per-week 0 11th Own-child Private Machine-op-inspct United-States 25.0 40.0 <p>The <code>TabPreprocessor</code> will have a series of useful attributes that can later be used when instantiating the different Tabular Models, such us for example, the column indexes (used to slice the tensors, internally in the models) or the categorical embeddings set up</p> In\u00a0[14]: Copied! <pre>tab_preprocessor.column_idx\n</pre> tab_preprocessor.column_idx Out[14]: <pre>{'education': 0,\n 'relationship': 1,\n 'workclass': 2,\n 'occupation': 3,\n 'native-country': 4,\n 'age': 5,\n 'hours-per-week': 6}</pre> In\u00a0[15]: Copied! <pre># column name, num unique, embedding dim\ntab_preprocessor.cat_embed_input\n</pre> # column name, num unique, embedding dim tab_preprocessor.cat_embed_input Out[15]: <pre>[('education', 16, 10),\n ('relationship', 6, 8),\n ('workclass', 9, 10),\n ('occupation', 15, 10),\n ('native-country', 42, 10)]</pre> <p>As I mentioned, there is more one can do, such as for example, quantize (or bucketize) the continuous cols. For this we could use the <code>quantization_setup</code> param. This parameter accepts a number of different inputs and uses <code>pd.cut</code> under the hood to quantize the continuous cols. For more info, please, read the docs. Let's use it here to quantize \"age\" and \"hours-per-week\" in 4 and 5 \"buckets\" respectively</p> In\u00a0[16]: Copied! <pre>quantization_setup = {\n    \"age\": 4,\n    \"hours-per-week\": 5,\n}  # you can also pass a list of floats with the boundaries if you wanted\nquant_tab_preprocessor = TabPreprocessor(\n    cat_embed_cols=cat_embed_cols,\n    continuous_cols=continuous_cols,\n    quantization_setup=quantization_setup,\n)\nqX_tab = quant_tab_preprocessor.fit_transform(df)\n</pre> quantization_setup = {     \"age\": 4,     \"hours-per-week\": 5, }  # you can also pass a list of floats with the boundaries if you wanted quant_tab_preprocessor = TabPreprocessor(     cat_embed_cols=cat_embed_cols,     continuous_cols=continuous_cols,     quantization_setup=quantization_setup, ) qX_tab = quant_tab_preprocessor.fit_transform(df) <pre>/Users/javierrodriguezzaurin/Projects/pytorch-widedeep/pytorch_widedeep/preprocessing/tab_preprocessor.py:358: UserWarning: Continuous columns will not be normalised\n  warnings.warn(\"Continuous columns will not be normalised\")\n</pre> In\u00a0[17]: Copied! <pre>qX_tab\n</pre> qX_tab Out[17]: <pre>array([[1, 1, 1, ..., 1, 1, 2],\n       [2, 2, 1, ..., 1, 2, 3],\n       [3, 2, 2, ..., 1, 1, 2],\n       ...,\n       [2, 4, 1, ..., 1, 3, 2],\n       [2, 1, 1, ..., 1, 1, 1],\n       [2, 5, 7, ..., 1, 2, 2]])</pre> <p>Note that the continuous columns that have been bucketised into quantiles are treated as any other categorical column</p> In\u00a0[18]: Copied! <pre>quant_tab_preprocessor.cat_embed_input\n</pre> quant_tab_preprocessor.cat_embed_input Out[18]: <pre>[('education', 16, 10),\n ('relationship', 6, 8),\n ('workclass', 9, 10),\n ('occupation', 15, 10),\n ('native-country', 42, 10),\n ('age', 4, 4),\n ('hours-per-week', 5, 4)]</pre> <p>Where the column 'age' has now 4 categories, which will be encoded using embeddings of 4 dims. Note that, as any other categorical columns, the categorical \"counter\" starts with 1. This is because all incoming values that are lower/higher than the existing lowest/highest value in the train (or already seen) dataset, will be encoded as 0.</p> In\u00a0[19]: Copied! <pre>np.unique(qX_tab[:, quant_tab_preprocessor.column_idx[\"age\"]])\n</pre> np.unique(qX_tab[:, quant_tab_preprocessor.column_idx[\"age\"]]) Out[19]: <pre>array([1, 2, 3, 4])</pre> <p>Finally, if we now wanted to <code>inverse_transform</code> the transformed array into the original dataframe, we could still do it, but the continuous, bucketized columns will be transformed back to the middle of their quantile/bucket range</p> In\u00a0[20]: Copied! <pre>df_decoded = quant_tab_preprocessor.inverse_transform(qX_tab)\n</pre> df_decoded = quant_tab_preprocessor.inverse_transform(qX_tab) <pre>Note that quantized cols will be turned into the mid point of the corresponding bin\n</pre> In\u00a0[21]: Copied! <pre>df.head(2)\n</pre> df.head(2) Out[21]: age workclass fnlwgt education educational-num marital-status occupation relationship race gender capital-gain capital-loss hours-per-week native-country income 0 25 Private 226802 11th 7 Never-married Machine-op-inspct Own-child Black Male 0 0 40 United-States &lt;=50K 1 38 Private 89814 HS-grad 9 Married-civ-spouse Farming-fishing Husband White Male 0 0 50 United-States &lt;=50K In\u00a0[22]: Copied! <pre>df_decoded.head(2)\n</pre> df_decoded.head(2) Out[22]: education relationship workclass occupation native-country age hours-per-week 0 11th Own-child Private Machine-op-inspct United-States 26.0885 30.4 1 HS-grad Husband Private Farming-fishing United-States 44.3750 50.0 <p>there is one final comment to make regarding to the <code>inverse_transform</code> functionality. As we mentioned before, the encoding <code>0</code> is reserved for values that fall outside the range covered by the data we used to run the <code>fit</code> method. For example</p> In\u00a0[23]: Copied! <pre>df.age.min(), df.age.max()\n</pre> df.age.min(), df.age.max() Out[23]: <pre>(17, 90)</pre> <p>All future age values outside that range will be encoded as 0 and decoded as <code>NaN</code></p> In\u00a0[24]: Copied! <pre>tmp_df = df.head(1).copy()\ntmp_df.loc[:, \"age\"] = 5\n</pre> tmp_df = df.head(1).copy() tmp_df.loc[:, \"age\"] = 5 In\u00a0[25]: Copied! <pre>tmp_df\n</pre> tmp_df Out[25]: age workclass fnlwgt education educational-num marital-status occupation relationship race gender capital-gain capital-loss hours-per-week native-country income 0 5 Private 226802 11th 7 Never-married Machine-op-inspct Own-child Black Male 0 0 40 United-States &lt;=50K In\u00a0[26]: Copied! <pre># quant_tab_preprocessor has already been fitted with a data that has an age range between 17 and 90\ntmp_qX_tab = quant_tab_preprocessor.transform(tmp_df)\n</pre> # quant_tab_preprocessor has already been fitted with a data that has an age range between 17 and 90 tmp_qX_tab = quant_tab_preprocessor.transform(tmp_df) In\u00a0[27]: Copied! <pre>tmp_qX_tab\n</pre> tmp_qX_tab Out[27]: <pre>array([[1, 1, 1, 1, 1, 0, 2]])</pre> In\u00a0[28]: Copied! <pre>quant_tab_preprocessor.inverse_transform(tmp_qX_tab)\n</pre> quant_tab_preprocessor.inverse_transform(tmp_qX_tab) <pre>Note that quantized cols will be turned into the mid point of the corresponding bin\n</pre> Out[28]: education relationship workclass occupation native-country age hours-per-week 0 11th Own-child Private Machine-op-inspct United-States NaN 30.4 In\u00a0[29]: Copied! <pre>from pytorch_widedeep.preprocessing import TextPreprocessor\n</pre> from pytorch_widedeep.preprocessing import TextPreprocessor In\u00a0[30]: Copied! <pre># The airbnb dataset, which you could get from here:\n# http://insideairbnb.com/get-the-data.html, is too big to be included in\n# our datasets module (when including images). Therefore, go there,\n# download it, and use the download_images.py script to get the images\n# and the airbnb_data_processing.py to process the data. We'll find\n# better datasets in the future ;). Note that here we are only using a\n# small sample to illustrate the use, so PLEASE ignore the results, just\n# focus on usage\ndf = pd.read_csv(\"../tmp_data/airbnb/airbnb_sample.csv\")\n</pre> # The airbnb dataset, which you could get from here: # http://insideairbnb.com/get-the-data.html, is too big to be included in # our datasets module (when including images). Therefore, go there, # download it, and use the download_images.py script to get the images # and the airbnb_data_processing.py to process the data. We'll find # better datasets in the future ;). Note that here we are only using a # small sample to illustrate the use, so PLEASE ignore the results, just # focus on usage df = pd.read_csv(\"../tmp_data/airbnb/airbnb_sample.csv\") In\u00a0[31]: Copied! <pre>texts = df.description.tolist()\ntexts[:2]\n</pre> texts = df.description.tolist() texts[:2] Out[31]: <pre>[\"My bright double bedroom with a large window has a relaxed feeling! It comfortably fits one or two and is centrally located just two blocks from Finsbury Park. Enjoy great restaurants in the area and easy access to easy transport tubes, trains and buses. Babies and children of all ages are welcome. Hello Everyone, I'm offering my lovely double bedroom in Finsbury Park area (zone 2) for let in a shared apartment.  You will share the apartment with me and it is fully furnished with a self catering kitchen. Two people can easily sleep well as the room has a queen size bed. I also have a travel cot for a baby for guest with small children.  I will require a deposit up front as a security gesture on both our parts and will be given back to you when you return the keys.  I trust anyone who will be responding to this add would treat my home with care and respect .  Best Wishes  Alina Guest will have access to the self catering kitchen and bathroom. There is the flat is equipped wifi internet,\",\n \"Lots of windows and light.  St Luke's Gardens are at the end of the block, and the river not too far the other way. Ten minutes walk if you go slowly. Buses to everywhere round the corner and shops, restaurants, pubs, the cinema and Waitrose . Bright Chelsea Apartment  This is a bright one bedroom ground floor apartment in an interesting listed building. There is one double bedroom and a living room/kitchen The apartment has a full  bathroom and the kitchen is fully equipped. Two wardrobes are available exclusively for guests and bedside tables and two long drawers. This sunny convenient compact flat is just around the corner from the Waitrose supermarket and all sorts of shops, cinemas, restaurants and pubs.  This is a lovely part of London. There is a fun farmers market in the King's Road at the weekend.  Buses to everywhere are just round the corner, and two underground stations are within ten minutes walk. There is a very nice pub round by St. Luke's gardens, 4 mins slow walk, the \"]</pre> In\u00a0[32]: Copied! <pre>text_preprocessor = TextPreprocessor(text_col=\"description\")\nX_text = text_preprocessor.fit_transform(df)\n# From here on, any new observation can be prepared by simply running `.transform`\n# new_X_text = text_preprocessor.transform(new_df)\n</pre> text_preprocessor = TextPreprocessor(text_col=\"description\") X_text = text_preprocessor.fit_transform(df) # From here on, any new observation can be prepared by simply running `.transform` # new_X_text = text_preprocessor.transform(new_df) <pre>The vocabulary contains 2192 tokens\n</pre> In\u00a0[33]: Copied! <pre>print(X_text[0])\n</pre> print(X_text[0]) <pre>[  29   48   37  367  818   17  910   17  177   15  122  349   53  879\n 1174  126  393   40  911    0   23  228   71  819    9   53   55 1380\n  225   11   18  308   18 1564   10  755    0  942  239   53   55    0\n   11   36 1013  277 1974   70   62   15 1475    9  943    5  251    5\n    0    5    0    5  177   53   37   75   11   10  294  726   32    9\n   42    5   25   12   10   22   12  136  100  145]\n</pre> In\u00a0[34]: Copied! <pre>from pytorch_widedeep.preprocessing import ImagePreprocessor\n</pre> from pytorch_widedeep.preprocessing import ImagePreprocessor In\u00a0[35]: Copied! <pre>image_preprocessor = wd.preprocessing.ImagePreprocessor(\n    img_col=\"id\", img_path=\"../tmp_data/airbnb/property_picture/\"\n)\nX_images = image_preprocessor.fit_transform(df)\n# From here on, any new observation can be prepared by simply running `.transform`\n# new_X_images = image_preprocessor.transform(new_df)\n</pre> image_preprocessor = wd.preprocessing.ImagePreprocessor(     img_col=\"id\", img_path=\"../tmp_data/airbnb/property_picture/\" ) X_images = image_preprocessor.fit_transform(df) # From here on, any new observation can be prepared by simply running `.transform` # new_X_images = image_preprocessor.transform(new_df) <pre>Reading Images from ../tmp_data/airbnb/property_picture/\nResizing\n</pre> <pre>100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 1001/1001 [00:01&lt;00:00, 667.89it/s]\n</pre> <pre>Computing normalisation metrics\n</pre> In\u00a0[36]: Copied! <pre>X_images[0].shape\n</pre> X_images[0].shape Out[36]: <pre>(224, 224, 3)</pre>"},{"location":"examples/01_preprocessors_and_utils.html#processors-and-utils","title":"Processors and Utils\u00b6","text":"<p>Description of the main tools and utilities that one needs to prepare the data for a <code>WideDeep</code> model constructor.</p>"},{"location":"examples/01_preprocessors_and_utils.html#the-preprocessing-module","title":"The <code>preprocessing</code>  module\u00b6","text":"<p>There are 4 preprocessors, corresponding to 4 main components of the <code>WideDeep</code> model. These are</p> <ul> <li><code>WidePreprocessor</code></li> <li><code>TabPreprocessor</code></li> <li><code>TextPreprocessor</code></li> <li><code>ImagePreprocessor</code></li> </ul> <p>Behind the scenes, these preprocessors use a series of helper funcions and classes that are in the <code>utils</code> module. If you were interested please go and have a look to the documentation</p>"},{"location":"examples/01_preprocessors_and_utils.html#1-widepreprocessor","title":"1. WidePreprocessor\u00b6","text":"<p>The <code>wide</code> component of the model is a linear model that in principle, could be implemented as a linear layer receiving the result of on one-hot encoding categorical columns. However, this is not memory efficient. Therefore, we implement a liner layer as an Embedding layer plus a bias. I will explain in a bit more detail later.</p> <p>With that in mind, <code>WidePreprocessor</code> simply encodes the categories numerically so that they are the indexes of the lookup table that is an Embedding layer.</p>"},{"location":"examples/01_preprocessors_and_utils.html#2-tabpreprocessor","title":"2. TabPreprocessor\u00b6","text":"<p>The <code>TabPreprocessor</code> has a lot of different functionalities. Let's explore some of them in detail. In its basic use, the <code>TabPreprocessor</code> simply label encodes the categorical columns and normalises the numerical ones (unless otherwised specified).</p>"},{"location":"examples/01_preprocessors_and_utils.html#3-textpreprocessor","title":"3. TextPreprocessor\u00b6","text":"<p>This preprocessor returns the tokenised, padded sequences that will be directly fed to the stack of LSTMs.</p>"},{"location":"examples/01_preprocessors_and_utils.html#4-imagepreprocessor","title":"4. ImagePreprocessor\u00b6","text":"<p><code>ImagePreprocessor</code> simply resizes the images, being aware of the aspect ratio.</p>"},{"location":"examples/02_model_components.html","title":"02_model_components","text":"In\u00a0[1]: Copied! <pre>import torch\nimport pandas as pd\nimport numpy as np\n\nfrom torch import nn\n</pre> import torch import pandas as pd import numpy as np  from torch import nn In\u00a0[2]: Copied! <pre>df = pd.DataFrame({\"color\": [\"r\", \"b\", \"g\"], \"size\": [\"s\", \"n\", \"l\"]})\ndf.head()\n</pre> df = pd.DataFrame({\"color\": [\"r\", \"b\", \"g\"], \"size\": [\"s\", \"n\", \"l\"]}) df.head() Out[2]: color size 0 r s 1 b n 2 g l <p>one hot encoded, the first observation would be</p> In\u00a0[3]: Copied! <pre>obs_0_oh = (np.array([1.0, 0.0, 0.0, 1.0, 0.0, 0.0])).astype(\"float32\")\n</pre> obs_0_oh = (np.array([1.0, 0.0, 0.0, 1.0, 0.0, 0.0])).astype(\"float32\") <p>if we simply numerically encode (label encode or <code>le</code>) the values:</p> In\u00a0[4]: Copied! <pre>obs_0_le = (np.array([0, 3])).astype(\"int64\")\n</pre> obs_0_le = (np.array([0, 3])).astype(\"int64\") <p>Note that in the functioning implementation of the package we start from 1, saving 0 for padding, i.e. unseen values.</p> <p>Now, let's see if the two implementations are equivalent</p> In\u00a0[5]: Copied! <pre># we have 6 different values. Let's assume we are performing a regression, so pred_dim = 1\nlin = nn.Linear(6, 1)\n</pre> # we have 6 different values. Let's assume we are performing a regression, so pred_dim = 1 lin = nn.Linear(6, 1) In\u00a0[6]: Copied! <pre>emb = nn.Embedding(6, 1)\nemb.weight = nn.Parameter(lin.weight.reshape_as(emb.weight))\n</pre> emb = nn.Embedding(6, 1) emb.weight = nn.Parameter(lin.weight.reshape_as(emb.weight)) In\u00a0[7]: Copied! <pre>lin(torch.tensor(obs_0_oh))\n</pre> lin(torch.tensor(obs_0_oh)) Out[7]: <pre>tensor([-0.5181], grad_fn=&lt;ViewBackward0&gt;)</pre> In\u00a0[8]: Copied! <pre>emb(torch.tensor(obs_0_le)).sum() + lin.bias\n</pre> emb(torch.tensor(obs_0_le)).sum() + lin.bias Out[8]: <pre>tensor([-0.5181], grad_fn=&lt;AddBackward0&gt;)</pre> <p>And this is precisely how the linear model <code>Wide</code> is implemented</p> In\u00a0[9]: Copied! <pre>from pytorch_widedeep.models import Wide\n</pre> from pytorch_widedeep.models import Wide <pre>/Users/javierrodriguezzaurin/.pyenv/versions/3.10.13/envs/widedeep310/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n  from .autonotebook import tqdm as notebook_tqdm\n</pre> In\u00a0[10]: Copied! <pre># ?Wide\n</pre> # ?Wide In\u00a0[11]: Copied! <pre>wide = Wide(input_dim=10, pred_dim=1)\nwide\n</pre> wide = Wide(input_dim=10, pred_dim=1) wide Out[11]: <pre>Wide(\n  (wide_linear): Embedding(11, 1, padding_idx=0)\n)</pre> <p>Note that even though the input dim is 10, the Embedding layer has 11 weights. Again, this is because we save <code>0</code> for padding, which is used for unseen values during the encoding process.</p> <p>As I mentioned, <code>deeptabular</code> has enough complexity on its own and it will be described in a separated notebook. Let's then jump to <code>deeptext</code>.</p> In\u00a0[12]: Copied! <pre>from pytorch_widedeep.preprocessing import TabPreprocessor\nfrom pytorch_widedeep.models import TabMlp\n</pre> from pytorch_widedeep.preprocessing import TabPreprocessor from pytorch_widedeep.models import TabMlp In\u00a0[13]: Copied! <pre>data = {\n    \"cat1\": np.random.choice([\"A\", \"B\", \"C\"], size=20),\n    \"cat2\": np.random.choice([\"X\", \"Y\"], size=20),\n    \"cont1\": np.random.rand(20),\n    \"cont2\": np.random.rand(20),\n}\n\ndf = pd.DataFrame(data)\n</pre> data = {     \"cat1\": np.random.choice([\"A\", \"B\", \"C\"], size=20),     \"cat2\": np.random.choice([\"X\", \"Y\"], size=20),     \"cont1\": np.random.rand(20),     \"cont2\": np.random.rand(20), }  df = pd.DataFrame(data) In\u00a0[14]: Copied! <pre>df.head()\n</pre> df.head() Out[14]: cat1 cat2 cont1 cont2 0 A Y 0.789347 0.561789 1 C X 0.050822 0.061538 2 A Y 0.863784 0.241967 3 C X 0.917848 0.644658 4 C Y 0.042328 0.417303 In\u00a0[15]: Copied! <pre># see the docs for details on all params/options\ntab_preprocessor = TabPreprocessor(\n    cat_embed_cols=[\"cat1\", \"cat2\"],\n    continuous_cols=[\"cont1\", \"cont2\"],\n    embedding_rule=\"fastai\",\n)\n</pre> # see the docs for details on all params/options tab_preprocessor = TabPreprocessor(     cat_embed_cols=[\"cat1\", \"cat2\"],     continuous_cols=[\"cont1\", \"cont2\"],     embedding_rule=\"fastai\", ) In\u00a0[16]: Copied! <pre>X_tab = tab_preprocessor.fit_transform(df)\n</pre> X_tab = tab_preprocessor.fit_transform(df) <pre>/Users/javierrodriguezzaurin/Projects/pytorch-widedeep/pytorch_widedeep/preprocessing/tab_preprocessor.py:358: UserWarning: Continuous columns will not be normalised\n  warnings.warn(\"Continuous columns will not be normalised\")\n</pre> In\u00a0[17]: Copied! <pre># toy example just to build a model.\ntabmlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=tab_preprocessor.continuous_cols,\n    embed_continuous_method=\"standard\",\n    cont_embed_dim=4,\n    mlp_hidden_dims=[8, 4],\n    mlp_linear_first=True,\n)\ntabmlp\n</pre> # toy example just to build a model. tabmlp = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     continuous_cols=tab_preprocessor.continuous_cols,     embed_continuous_method=\"standard\",     cont_embed_dim=4,     mlp_hidden_dims=[8, 4],     mlp_linear_first=True, ) tabmlp Out[17]: <pre>TabMlp(\n  (cat_embed): DiffSizeCatEmbeddings(\n    (embed_layers): ModuleDict(\n      (emb_layer_cat1): Embedding(4, 3, padding_idx=0)\n      (emb_layer_cat2): Embedding(3, 2, padding_idx=0)\n    )\n    (embedding_dropout): Dropout(p=0.0, inplace=False)\n  )\n  (cont_norm): Identity()\n  (cont_embed): ContEmbeddings(\n    INFO: [ContLinear = weight(n_cont_cols, embed_dim) + bias(n_cont_cols, embed_dim)]\n    (linear): ContLinear(n_cont_cols=2, embed_dim=4, embed_dropout=0.0)\n    (dropout): Dropout(p=0.0, inplace=False)\n  )\n  (encoder): MLP(\n    (mlp): Sequential(\n      (dense_layer_0): Sequential(\n        (0): Linear(in_features=13, out_features=8, bias=True)\n        (1): ReLU(inplace=True)\n        (2): Dropout(p=0.1, inplace=False)\n      )\n      (dense_layer_1): Sequential(\n        (0): Linear(in_features=8, out_features=4, bias=True)\n        (1): ReLU(inplace=True)\n        (2): Dropout(p=0.1, inplace=False)\n      )\n    )\n  )\n)</pre> <p>Lets describe a bit the model: first we have what we call a <code>DiffSizeCatEmbeddings</code>, where categorical columns with different number of unique categories will be encoded with embeddings of different dimensions. Then the continuous columns will not be normalised (the normalised layer is just the identity) and they will be embedded via a \"standard\" method, using a so-called <code>ContLinear</code> layer. This layer displays some <code>INFO</code> that tells us what it is (<code>ContLinear = weight(n_cont_cols, embed_dim) + bias(n_cont_cols, embed_dim)]</code>). There are two other options available to embed the continuous cols based on the paper On Embeddings for Numerical Features in Tabular Deep Learning. These are <code>PieceWise</code> and <code>Periodic</code> and all available via the <code>embed_continuous_method</code> param, which can adopt values <code>\"standard\", \"piecewise\"</code> and <code>\"periodic\"</code>. The embedded categorical and continuous columns will be then concatenated ($3 + 2 + (4 * 2) = 13$ input dims) and passed to an MLP.</p> In\u00a0[18]: Copied! <pre>from pytorch_widedeep.models import BasicRNN\n</pre> from pytorch_widedeep.models import BasicRNN In\u00a0[19]: Copied! <pre>basic_rnn = BasicRNN(vocab_size=4, hidden_dim=4, n_layers=1, padding_idx=0, embed_dim=4)\n</pre> basic_rnn = BasicRNN(vocab_size=4, hidden_dim=4, n_layers=1, padding_idx=0, embed_dim=4) <pre>/Users/javierrodriguezzaurin/.pyenv/versions/3.10.13/envs/widedeep310/lib/python3.10/site-packages/torch/nn/modules/rnn.py:82: UserWarning: dropout option adds dropout after all but last recurrent layer, so non-zero dropout expects num_layers greater than 1, but got dropout=0.1 and num_layers=1\n  warnings.warn(\"dropout option adds dropout after all but last \"\n</pre> In\u00a0[20]: Copied! <pre>basic_rnn\n</pre> basic_rnn Out[20]: <pre>BasicRNN(\n  (word_embed): Embedding(4, 4, padding_idx=0)\n  (rnn): LSTM(4, 4, batch_first=True, dropout=0.1)\n  (rnn_mlp): Identity()\n)</pre> <p>You could, if you wanted, add a Fully Connected Head (FC-Head) on top of it</p> In\u00a0[21]: Copied! <pre>from pytorch_widedeep.models import Vision\n</pre> from pytorch_widedeep.models import Vision In\u00a0[22]: Copied! <pre>resnet = Vision(pretrained_model_setup=\"resnet18\", n_trainable=0)\n</pre> resnet = Vision(pretrained_model_setup=\"resnet18\", n_trainable=0) In\u00a0[23]: Copied! <pre>resnet\n</pre> resnet Out[23]: <pre>Vision(\n  (features): Sequential(\n    (0): Conv2d(3, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False)\n    (1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n    (2): ReLU(inplace=True)\n    (3): MaxPool2d(kernel_size=3, stride=2, padding=1, dilation=1, ceil_mode=False)\n    (4): Sequential(\n      (0): BasicBlock(\n        (conv1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n        (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (relu): ReLU(inplace=True)\n        (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n        (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n      )\n      (1): BasicBlock(\n        (conv1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n        (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (relu): ReLU(inplace=True)\n        (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n        (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n      )\n    )\n    (5): Sequential(\n      (0): BasicBlock(\n        (conv1): Conv2d(64, 128, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n        (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (relu): ReLU(inplace=True)\n        (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n        (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (downsample): Sequential(\n          (0): Conv2d(64, 128, kernel_size=(1, 1), stride=(2, 2), bias=False)\n          (1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        )\n      )\n      (1): BasicBlock(\n        (conv1): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n        (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (relu): ReLU(inplace=True)\n        (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n        (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n      )\n    )\n    (6): Sequential(\n      (0): BasicBlock(\n        (conv1): Conv2d(128, 256, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n        (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (relu): ReLU(inplace=True)\n        (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n        (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (downsample): Sequential(\n          (0): Conv2d(128, 256, kernel_size=(1, 1), stride=(2, 2), bias=False)\n          (1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        )\n      )\n      (1): BasicBlock(\n        (conv1): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n        (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (relu): ReLU(inplace=True)\n        (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n        (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n      )\n    )\n    (7): Sequential(\n      (0): BasicBlock(\n        (conv1): Conv2d(256, 512, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n        (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (relu): ReLU(inplace=True)\n        (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n        (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (downsample): Sequential(\n          (0): Conv2d(256, 512, kernel_size=(1, 1), stride=(2, 2), bias=False)\n          (1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        )\n      )\n      (1): BasicBlock(\n        (conv1): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n        (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (relu): ReLU(inplace=True)\n        (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n        (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n      )\n    )\n    (8): AdaptiveAvgPool2d(output_size=(1, 1))\n  )\n)</pre>"},{"location":"examples/02_model_components.html#model-components","title":"Model Components\u00b6","text":"<p>The main components of a <code>WideDeep</code> (i.e. Multimodal) model are tabular data, text and images, which are feed into the model via so called <code>wide</code>, <code>deeptabular</code>, <code>deeptext</code> and <code>deepimage</code> model components</p>"},{"location":"examples/02_model_components.html#1-wide","title":"1. <code>wide</code>\u00b6","text":"<p>The <code>wide</code> component is a Linear layer \"plugged\" into the output neuron(s). Here, the non-linearities are captured via crossed columns. Crossed columns are, quoting directly the paper: \"For binary features, a cross-product transformation (e.g., \u201cAND(gender=female, language=en)\u201d) is 1 if and only if the constituent features (\u201cgender=female\u201d and \u201clanguage=en\u201d) are all 1, and 0 otherwise\".</p> <p>The only particularity of our implementation is that we have implemented the linear layer via an Embedding layer plus a bias. While the implementations are equivalent, the latter is faster and far more memory efficient, since we do not need to one hot encode the categorical features.</p> <p>Let's assume we the following dataset:</p>"},{"location":"examples/02_model_components.html#2-deeptabular","title":"2. <code>deeptabular</code>\u00b6","text":"<p>The <code>deeptabular</code> model alone is what normally would be referred as Deep Learning for tabular data. As mentioned a number of times throughout the library, each component can be used independently. Therefore, if you wanted to use any of the models below alone, it is perfectly possible. There are just a couple of simple requirement that will be covered in a later notebook.</p> <p>By the time of writing, there are a number of models available in <code>pytorch-widedeep</code> to do DL for tabular data. These are:</p> <ol> <li><code>TabMlp</code></li> <li><code>ContextAttentionMLP</code></li> <li><code>SelfAttentionMLP</code></li> <li><code>TabResnet</code></li> <li><code>Tabnet</code></li> <li><code>TabTransformer</code></li> <li><code>FT-Tabransformer</code></li> <li><code>SAINT</code></li> <li><code>TabFastFormer</code></li> <li><code>TabPerceiver</code></li> </ol> <p>Let's have a look to one of them. For more information on each of these models, please, have a look to the documentation</p>"},{"location":"examples/02_model_components.html#3-deeptext","title":"3. <code>deeptext</code>\u00b6","text":"<p>At the time of writing, <code>pytorch-widedeep</code> offers three models that can be passed to <code>WideDeep</code> as the <code>deeptext</code> component. These are:</p> <ol> <li>BasicRNN</li> <li>AttentiveRNN</li> <li>StackedAttentiveRNN</li> </ol> <p>For details on each of these models, please, have a look to the documentation of the package.</p> <p>We will soon integrate with Hugginface, but let me insist. It is perfectly possible to use custom models for each component, please, have a look to the corresponding notebook. In general, simply, build them and pass them as the corresponding parameters. Note that the custom models MUST return a last layer of activations (i.e. not the final prediction) so that  these activations are collected by <code>WideDeep</code> and combined accordingly. In  addition, the models MUST also contain an attribute <code>output_dim</code> with the size of these last layers of activations.</p> <p>Let's have a look to the <code>BasicRNN</code> model</p>"},{"location":"examples/02_model_components.html#4-deepimage","title":"4. <code>deepimage</code>\u00b6","text":"<p>At the time of writing <code>pytorch-widedeep</code> is integrated with torchvision via the <code>Vision</code> class. This means that the it is possible to use a variant of the following architectures:</p> <ol> <li>resnet</li> <li>shufflenet</li> <li>resnext</li> <li>wide_resnet</li> <li>regnet</li> <li>densenet</li> <li>mobilenet</li> <li>mnasnet</li> <li>efficientnet</li> <li>squeezenet</li> </ol> <p>The user can choose which layers will be trainable. Alternatively, in none of these architectures is useful, one could use a simple, fully trained CNN (please see the package documentation) or pass a custom model.</p> <p>let's have a look</p>"},{"location":"examples/03_binary_classification_with_defaults.html","title":"03_binary_classification_with_defaults","text":"In\u00a0[1]: Copied! <pre>import numpy as np\nimport pandas as pd\nimport torch\n\nfrom pytorch_widedeep.preprocessing import WidePreprocessor, TabPreprocessor\nfrom pytorch_widedeep.training import Trainer\nfrom pytorch_widedeep.models import Wide, TabMlp, WideDeep\nfrom pytorch_widedeep.metrics import Accuracy, Precision\nfrom pytorch_widedeep.datasets import load_adult\n</pre> import numpy as np import pandas as pd import torch  from pytorch_widedeep.preprocessing import WidePreprocessor, TabPreprocessor from pytorch_widedeep.training import Trainer from pytorch_widedeep.models import Wide, TabMlp, WideDeep from pytorch_widedeep.metrics import Accuracy, Precision from pytorch_widedeep.datasets import load_adult <pre>/Users/javierrodriguezzaurin/.pyenv/versions/3.10.13/envs/widedeep310/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n  from .autonotebook import tqdm as notebook_tqdm\n</pre> In\u00a0[2]: Copied! <pre>df = load_adult(as_frame=True)\ndf.head()\n</pre> df = load_adult(as_frame=True) df.head() Out[2]: age workclass fnlwgt education educational-num marital-status occupation relationship race gender capital-gain capital-loss hours-per-week native-country income 0 25 Private 226802 11th 7 Never-married Machine-op-inspct Own-child Black Male 0 0 40 United-States &lt;=50K 1 38 Private 89814 HS-grad 9 Married-civ-spouse Farming-fishing Husband White Male 0 0 50 United-States &lt;=50K 2 28 Local-gov 336951 Assoc-acdm 12 Married-civ-spouse Protective-serv Husband White Male 0 0 40 United-States &gt;50K 3 44 Private 160323 Some-college 10 Married-civ-spouse Machine-op-inspct Husband Black Male 7688 0 40 United-States &gt;50K 4 18 ? 103497 Some-college 10 Never-married ? Own-child White Female 0 0 30 United-States &lt;=50K In\u00a0[3]: Copied! <pre># For convenience, we'll replace '-' with '_'\ndf.columns = [c.replace(\"-\", \"_\") for c in df.columns]\n# binary target\ndf[\"income_label\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int)\ndf.drop(\"income\", axis=1, inplace=True)\ndf.head()\n</pre> # For convenience, we'll replace '-' with '_' df.columns = [c.replace(\"-\", \"_\") for c in df.columns] # binary target df[\"income_label\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int) df.drop(\"income\", axis=1, inplace=True) df.head() Out[3]: age workclass fnlwgt education educational_num marital_status occupation relationship race gender capital_gain capital_loss hours_per_week native_country income_label 0 25 Private 226802 11th 7 Never-married Machine-op-inspct Own-child Black Male 0 0 40 United-States 0 1 38 Private 89814 HS-grad 9 Married-civ-spouse Farming-fishing Husband White Male 0 0 50 United-States 0 2 28 Local-gov 336951 Assoc-acdm 12 Married-civ-spouse Protective-serv Husband White Male 0 0 40 United-States 1 3 44 Private 160323 Some-college 10 Married-civ-spouse Machine-op-inspct Husband Black Male 7688 0 40 United-States 1 4 18 ? 103497 Some-college 10 Never-married ? Own-child White Female 0 0 30 United-States 0 In\u00a0[4]: Copied! <pre>df.drop([\"fnlwgt\", \"educational_num\"], axis=1, inplace=True)\n</pre> df.drop([\"fnlwgt\", \"educational_num\"], axis=1, inplace=True) In\u00a0[5]: Copied! <pre># Define wide, crossed and deep tabular columns\nwide_cols = [\n    \"workclass\",\n    \"education\",\n    \"marital_status\",\n    \"occupation\",\n    \"relationship\",\n    \"race\",\n    \"gender\",\n    \"native_country\",\n]\ncrossed_cols = [(\"education\", \"occupation\"), (\"native_country\", \"occupation\")]\n</pre> # Define wide, crossed and deep tabular columns wide_cols = [     \"workclass\",     \"education\",     \"marital_status\",     \"occupation\",     \"relationship\",     \"race\",     \"gender\",     \"native_country\", ] crossed_cols = [(\"education\", \"occupation\"), (\"native_country\", \"occupation\")] In\u00a0[6]: Copied! <pre>cat_embed_cols = [\n    \"workclass\",\n    \"education\",\n    \"marital_status\",\n    \"occupation\",\n    \"relationship\",\n    \"race\",\n    \"gender\",\n    \"capital_gain\",\n    \"capital_loss\",\n    \"native_country\",\n]\ncontinuous_cols = [\"age\", \"hours_per_week\"]\n</pre> cat_embed_cols = [     \"workclass\",     \"education\",     \"marital_status\",     \"occupation\",     \"relationship\",     \"race\",     \"gender\",     \"capital_gain\",     \"capital_loss\",     \"native_country\", ] continuous_cols = [\"age\", \"hours_per_week\"] In\u00a0[7]: Copied! <pre># TARGET\ntarget_col = \"income_label\"\ntarget = df[target_col].values\n</pre> # TARGET target_col = \"income_label\" target = df[target_col].values <p>let's see what the preprocessors do</p> In\u00a0[8]: Copied! <pre># wide\nwide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols)\nX_wide = wide_preprocessor.fit_transform(df)\n</pre> # wide wide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols) X_wide = wide_preprocessor.fit_transform(df) In\u00a0[9]: Copied! <pre># # wide_preprocessor has an attribute called encoding_dict with the encoding dictionary\n# wide_preprocessor.encoding_dict\n</pre> # # wide_preprocessor has an attribute called encoding_dict with the encoding dictionary # wide_preprocessor.encoding_dict In\u00a0[10]: Copied! <pre># deeptabular\ntab_preprocessor = TabPreprocessor(\n    embed_cols=cat_embed_cols,\n    continuous_cols=continuous_cols,\n    cols_to_scale=continuous_cols,\n)\nX_tab = tab_preprocessor.fit_transform(df)\n</pre> # deeptabular tab_preprocessor = TabPreprocessor(     embed_cols=cat_embed_cols,     continuous_cols=continuous_cols,     cols_to_scale=continuous_cols, ) X_tab = tab_preprocessor.fit_transform(df) In\u00a0[11]: Copied! <pre># check the docs to understand the useful attributes that the tab_preprocessor has. For example,\n# as well as an encoding dictionary, tab_preprocessor has an attribute called cat_embed_input\n# that specifies the categortical columns that will be represented as embeddings, the number\n# of different categories per feature, and the dimension of the embeddings as defined by some\n# of the internal rules of thumb that the preprocessor has (have a look to the docs)\ntab_preprocessor.cat_embed_input\n</pre> # check the docs to understand the useful attributes that the tab_preprocessor has. For example, # as well as an encoding dictionary, tab_preprocessor has an attribute called cat_embed_input # that specifies the categortical columns that will be represented as embeddings, the number # of different categories per feature, and the dimension of the embeddings as defined by some # of the internal rules of thumb that the preprocessor has (have a look to the docs) tab_preprocessor.cat_embed_input Out[11]: <pre>[('workclass', 9, 5),\n ('education', 16, 8),\n ('marital_status', 7, 5),\n ('occupation', 15, 7),\n ('relationship', 6, 4),\n ('race', 5, 4),\n ('gender', 2, 2),\n ('capital_gain', 123, 24),\n ('capital_loss', 99, 21),\n ('native_country', 42, 13)]</pre> In\u00a0[12]: Copied! <pre>print(X_wide)\nprint(X_wide.shape)\n</pre> print(X_wide) print(X_wide.shape) <pre>[[  1  10  26 ...  61 103 328]\n [  1  11  27 ...  61 104 329]\n [  2  12  27 ...  61 105 330]\n ...\n [  1  11  28 ...  61 115 335]\n [  1  11  26 ...  61 115 335]\n [  7  11  27 ...  61 127 336]]\n(48842, 10)\n</pre> In\u00a0[13]: Copied! <pre>print(X_tab)\nprint(X_tab.shape)\n</pre> print(X_tab) print(X_tab.shape) <pre>[[ 1.          1.          1.         ...  1.         -0.99512893\n  -0.03408696]\n [ 1.          2.          2.         ...  1.         -0.04694151\n   0.77292975]\n [ 2.          3.          2.         ...  1.         -0.77631645\n  -0.03408696]\n ...\n [ 1.          2.          3.         ...  1.          1.41180837\n  -0.03408696]\n [ 1.          2.          1.         ...  1.         -1.21394141\n  -1.64812038]\n [ 7.          2.          2.         ...  1.          0.97418341\n  -0.03408696]]\n(48842, 12)\n</pre> In\u00a0[14]: Copied! <pre>wide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1)\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    cat_embed_dropout=0.1,\n    continuous_cols=continuous_cols,\n    mlp_hidden_dims=[400, 200],\n    mlp_dropout=0.5,\n    mlp_activation=\"leaky_relu\",\n)\n</pre> wide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1) tab_mlp = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     cat_embed_dropout=0.1,     continuous_cols=continuous_cols,     mlp_hidden_dims=[400, 200],     mlp_dropout=0.5,     mlp_activation=\"leaky_relu\", ) <p>Let's first find out how a linear model performs</p> In\u00a0[15]: Copied! <pre>wide\n</pre> wide Out[15]: <pre>Wide(\n  (wide_linear): Embedding(809, 1, padding_idx=0)\n)</pre> <p>Before being passed to the Trainer, the models need to be \"constructed\" with the <code>WideDeep</code> constructor class. For the particular case of the wide/linear model, not much really happens</p> In\u00a0[16]: Copied! <pre>lin_model = WideDeep(wide=wide)\n</pre> lin_model = WideDeep(wide=wide) In\u00a0[17]: Copied! <pre>lin_model\n</pre> lin_model Out[17]: <pre>WideDeep(\n  (wide): Wide(\n    (wide_linear): Embedding(809, 1, padding_idx=0)\n  )\n)</pre> In\u00a0[18]: Copied! <pre>lin_trainer = Trainer(\n    model=lin_model,\n    objective=\"binary\",\n    optimizers=torch.optim.AdamW(lin_model.parameters(), lr=0.01),\n    metrics=[Accuracy, Precision],\n)\n</pre> lin_trainer = Trainer(     model=lin_model,     objective=\"binary\",     optimizers=torch.optim.AdamW(lin_model.parameters(), lr=0.01),     metrics=[Accuracy, Precision], ) In\u00a0[19]: Copied! <pre>lin_trainer.fit(X_wide=X_wide, target=target, n_epochs=4, batch_size=128, val_split=0.2)\n</pre> lin_trainer.fit(X_wide=X_wide, target=target, n_epochs=4, batch_size=128, val_split=0.2) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 306/306 [00:02&lt;00:00, 109.04it/s, loss=0.426, metrics={'acc': 0.7983, 'prec': 0.6152}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 77/77 [00:00&lt;00:00, 102.46it/s, loss=0.366, metrics={'acc': 0.832, 'prec': 0.6916}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 306/306 [00:02&lt;00:00, 130.27it/s, loss=0.364, metrics={'acc': 0.8305, 'prec': 0.6933}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 77/77 [00:00&lt;00:00, 150.46it/s, loss=0.361, metrics={'acc': 0.8357, 'prec': 0.6982}]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 306/306 [00:02&lt;00:00, 133.19it/s, loss=0.359, metrics={'acc': 0.8329, 'prec': 0.6994}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 77/77 [00:00&lt;00:00, 145.75it/s, loss=0.361, metrics={'acc': 0.836, 'prec': 0.7009}]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 306/306 [00:02&lt;00:00, 130.91it/s, loss=0.358, metrics={'acc': 0.8333, 'prec': 0.7005}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 77/77 [00:00&lt;00:00, 155.08it/s, loss=0.361, metrics={'acc': 0.8364, 'prec': 0.702}]\n</pre> <p>Bear in mind that <code>wide</code> is a linear model where the non-linearities are captured via the crossed columns. For the crossed-columns to be effective one needs proper business knowledge. There is no magic formula to produce them</p> <p>Let's have a look to the tabular model by itself</p> In\u00a0[20]: Copied! <pre>tab_model = WideDeep(deeptabular=tab_mlp)\n</pre> tab_model = WideDeep(deeptabular=tab_mlp) In\u00a0[21]: Copied! <pre>tab_model\n</pre> tab_model Out[21]: <pre>WideDeep(\n  (deeptabular): Sequential(\n    (0): TabMlp(\n      (cat_embed): DiffSizeCatEmbeddings(\n        (embed_layers): ModuleDict(\n          (emb_layer_workclass): Embedding(10, 5, padding_idx=0)\n          (emb_layer_education): Embedding(17, 8, padding_idx=0)\n          (emb_layer_marital_status): Embedding(8, 5, padding_idx=0)\n          (emb_layer_occupation): Embedding(16, 7, padding_idx=0)\n          (emb_layer_relationship): Embedding(7, 4, padding_idx=0)\n          (emb_layer_race): Embedding(6, 4, padding_idx=0)\n          (emb_layer_gender): Embedding(3, 2, padding_idx=0)\n          (emb_layer_capital_gain): Embedding(124, 24, padding_idx=0)\n          (emb_layer_capital_loss): Embedding(100, 21, padding_idx=0)\n          (emb_layer_native_country): Embedding(43, 13, padding_idx=0)\n        )\n        (embedding_dropout): Dropout(p=0.1, inplace=False)\n      )\n      (cont_norm): Identity()\n      (encoder): MLP(\n        (mlp): Sequential(\n          (dense_layer_0): Sequential(\n            (0): Linear(in_features=95, out_features=400, bias=True)\n            (1): LeakyReLU(negative_slope=0.01, inplace=True)\n            (2): Dropout(p=0.5, inplace=False)\n          )\n          (dense_layer_1): Sequential(\n            (0): Linear(in_features=400, out_features=200, bias=True)\n            (1): LeakyReLU(negative_slope=0.01, inplace=True)\n            (2): Dropout(p=0.5, inplace=False)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=200, out_features=1, bias=True)\n  )\n)</pre> <p>You can see how the <code>WideDeep</code> class has added a final prediction layer that collects the activations from the last layer of the model and plugs them into the output neuron. If this was a multiclass classification problem, the prediction dimension (i.e. the size of that final layer) needs to be specified via the <code>pred_dim</code> when instantiating the <code>WideDeep</code> class, as we will see later</p> In\u00a0[22]: Copied! <pre>tab_trainer = Trainer(\n    model=tab_model,\n    objective=\"binary\",\n    optimizers=torch.optim.AdamW(tab_model.parameters(), lr=0.001),\n    metrics=[Accuracy, Precision],\n)\n</pre> tab_trainer = Trainer(     model=tab_model,     objective=\"binary\",     optimizers=torch.optim.AdamW(tab_model.parameters(), lr=0.001),     metrics=[Accuracy, Precision], ) In\u00a0[23]: Copied! <pre>tab_trainer.fit(X_tab=X_tab, target=target, n_epochs=4, batch_size=128, val_split=0.2)\n</pre> tab_trainer.fit(X_tab=X_tab, target=target, n_epochs=4, batch_size=128, val_split=0.2) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 306/306 [00:03&lt;00:00, 97.00it/s, loss=0.37, metrics={'acc': 0.8267, 'prec': 0.7037}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 77/77 [00:00&lt;00:00, 134.91it/s, loss=0.313, metrics={'acc': 0.8588, 'prec': 0.7577}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 306/306 [00:03&lt;00:00, 86.86it/s, loss=0.319, metrics={'acc': 0.8514, 'prec': 0.761}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 77/77 [00:01&lt;00:00, 73.13it/s, loss=0.296, metrics={'acc': 0.8675, 'prec': 0.7685}]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 306/306 [00:03&lt;00:00, 79.07it/s, loss=0.305, metrics={'acc': 0.8574, 'prec': 0.7646}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 77/77 [00:00&lt;00:00, 130.11it/s, loss=0.289, metrics={'acc': 0.8696, 'prec': 0.7765}]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 306/306 [00:03&lt;00:00, 87.39it/s, loss=0.296, metrics={'acc': 0.8622, 'prec': 0.7769}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 77/77 [00:00&lt;00:00, 90.63it/s, loss=0.285, metrics={'acc': 0.8697, 'prec': 0.7741}]\n</pre> <p>The best result I ever obtained with <code>LightGBM</code> on this dataset is 0.8782...so we are pretty close.</p> <p>Let's combine the <code>wide</code> and <code>tab_mlp</code> components see if it helps</p> In\u00a0[24]: Copied! <pre>wide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1)\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    cat_embed_dropout=0.1,\n    continuous_cols=continuous_cols,\n    mlp_hidden_dims=[400, 200],\n    mlp_dropout=0.5,\n    mlp_activation=\"leaky_relu\",\n)\nwd_model = WideDeep(wide=wide, deeptabular=tab_mlp)\n</pre> wide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1) tab_mlp = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     cat_embed_dropout=0.1,     continuous_cols=continuous_cols,     mlp_hidden_dims=[400, 200],     mlp_dropout=0.5,     mlp_activation=\"leaky_relu\", ) wd_model = WideDeep(wide=wide, deeptabular=tab_mlp) In\u00a0[25]: Copied! <pre>wd_trainer = Trainer(\n    model=wd_model,\n    objective=\"binary\",\n    optimizers=torch.optim.AdamW(wd_model.parameters(), lr=0.001),\n    metrics=[Accuracy, Precision],\n)\n</pre> wd_trainer = Trainer(     model=wd_model,     objective=\"binary\",     optimizers=torch.optim.AdamW(wd_model.parameters(), lr=0.001),     metrics=[Accuracy, Precision], ) In\u00a0[26]: Copied! <pre>wd_trainer.fit(\n    X_wide=X_wide, X_tab=X_tab, target=target, n_epochs=4, batch_size=128, val_split=0.2\n)\n</pre> wd_trainer.fit(     X_wide=X_wide, X_tab=X_tab, target=target, n_epochs=4, batch_size=128, val_split=0.2 ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 306/306 [00:03&lt;00:00, 77.48it/s, loss=0.418, metrics={'acc': 0.8047, 'prec': 0.6154}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 77/77 [00:00&lt;00:00, 110.51it/s, loss=0.321, metrics={'acc': 0.8521, 'prec': 0.7059}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 306/306 [00:03&lt;00:00, 82.70it/s, loss=0.333, metrics={'acc': 0.8428, 'prec': 0.7141}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 77/77 [00:00&lt;00:00, 112.52it/s, loss=0.299, metrics={'acc': 0.866, 'prec': 0.7447}]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 306/306 [00:04&lt;00:00, 74.34it/s, loss=0.312, metrics={'acc': 0.8533, 'prec': 0.7404}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 77/77 [00:00&lt;00:00, 89.86it/s, loss=0.29, metrics={'acc': 0.8683, 'prec': 0.7496}]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 306/306 [00:04&lt;00:00, 65.32it/s, loss=0.301, metrics={'acc': 0.8591, 'prec': 0.7542}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 77/77 [00:00&lt;00:00, 86.81it/s, loss=0.286, metrics={'acc': 0.8712, 'prec': 0.7552}]\n</pre> <p>For this particular case, the combination of both did not lead to better results that using just the tab_mlp model, when using only 4 epochs.</p> <p>Note that we have use a <code>TabMlp</code> model, but we could use any other model in the library using the same syntax</p> In\u00a0[27]: Copied! <pre>from pytorch_widedeep.models import TabTransformer\n</pre> from pytorch_widedeep.models import TabTransformer <p>The parameters for the <code>TabTransformer</code> are this</p> <pre><code>column_idx: Dict[str, int],\ncat_embed_input: Optional[List[Tuple[str, int]]] = None,\ncat_embed_dropout: Optional[float] = None,\nuse_cat_bias: Optional[bool] = None,\ncat_embed_activation: Optional[str] = None,\nshared_embed: Optional[bool] = None,\nadd_shared_embed: Optional[bool] = None,\nfrac_shared_embed: Optional[float] = None,\ncontinuous_cols: Optional[List[str]] = None,\ncont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\nembed_continuous: Optional[bool] = None,\nembed_continuous_method: Optional[Literal[\"standard\", \"piecewise\", \"periodic\"]] = None,\ncont_embed_dropout: Optional[float] = None,\ncont_embed_activation: Optional[str] = None,\nquantization_setup: Optional[Dict[str, List[float]]] = None,\nn_frequencies: Optional[int] = None,\nsigma: Optional[float] = None,\nshare_last_layer: Optional[bool] = None,\nfull_embed_dropout: Optional[bool] = None,\ninput_dim: int = 32,\nn_heads: int = 8,\nuse_qkv_bias: bool = False,\nn_blocks: int = 4,\nattn_dropout: float = 0.2,\nff_dropout: float = 0.1,\nff_factor: int = 4,\ntransformer_activation: str = \"gelu\",\nuse_linear_attention: bool = False,\nuse_flash_attention: bool = False,\nmlp_hidden_dims: Optional[List[int]] = None,\nmlp_activation: str = \"relu\",\nmlp_dropout: float = 0.1,\nmlp_batchnorm: bool = False,\nmlp_batchnorm_last: bool = False,\nmlp_linear_first: bool = True,\n</code></pre> <p>Please, see the documentation for details on each one of them, for now let's see how one could use a <code>TabTransformer</code> model in a few lines of code</p> In\u00a0[28]: Copied! <pre>tab_transformer = TabTransformer(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    cat_embed_dropout=0.1,\n    continuous_cols=continuous_cols,\n    embed_continuous_method=\"standard\",\n    cont_norm_layer=\"layernorm\",\n    cont_embed_dropout=0.2,\n    cont_embed_activation=\"leaky_relu\",\n    n_heads=4,\n    ff_dropout=0.2,\n    mlp_dropout=0.5,\n    mlp_activation=\"leaky_relu\",\n    mlp_linear_first=\"True\",\n)\n</pre> tab_transformer = TabTransformer(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     cat_embed_dropout=0.1,     continuous_cols=continuous_cols,     embed_continuous_method=\"standard\",     cont_norm_layer=\"layernorm\",     cont_embed_dropout=0.2,     cont_embed_activation=\"leaky_relu\",     n_heads=4,     ff_dropout=0.2,     mlp_dropout=0.5,     mlp_activation=\"leaky_relu\",     mlp_linear_first=\"True\", ) In\u00a0[29]: Copied! <pre>tab_model = WideDeep(deeptabular=tab_transformer)\n</pre> tab_model = WideDeep(deeptabular=tab_transformer) In\u00a0[30]: Copied! <pre>tab_model\n</pre> tab_model Out[30]: <pre>WideDeep(\n  (deeptabular): Sequential(\n    (0): TabTransformer(\n      (cat_embed): SameSizeCatEmbeddings(\n        (embed): Embedding(325, 32, padding_idx=0)\n        (dropout): Dropout(p=0.1, inplace=False)\n      )\n      (cont_norm): LayerNorm((2,), eps=1e-05, elementwise_affine=True)\n      (cont_embed): ContEmbeddings(\n        INFO: [ContLinear = weight(n_cont_cols, embed_dim) + bias(n_cont_cols, embed_dim)]\n        (linear): ContLinear(n_cont_cols=2, embed_dim=32, embed_dropout=0.2)\n        (activation_fn): LeakyReLU(negative_slope=0.01, inplace=True)\n        (dropout): Dropout(p=0.2, inplace=False)\n      )\n      (encoder): Sequential(\n        (transformer_block0): TransformerEncoder(\n          (attn): MultiHeadedAttention(\n            (dropout): Dropout(p=0.2, inplace=False)\n            (q_proj): Linear(in_features=32, out_features=32, bias=False)\n            (kv_proj): Linear(in_features=32, out_features=64, bias=False)\n            (out_proj): Linear(in_features=32, out_features=32, bias=False)\n          )\n          (ff): FeedForward(\n            (w_1): Linear(in_features=32, out_features=128, bias=True)\n            (w_2): Linear(in_features=128, out_features=32, bias=True)\n            (dropout): Dropout(p=0.2, inplace=False)\n            (activation): GELU(approximate='none')\n          )\n          (attn_addnorm): AddNorm(\n            (dropout): Dropout(p=0.2, inplace=False)\n            (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n          )\n          (ff_addnorm): AddNorm(\n            (dropout): Dropout(p=0.2, inplace=False)\n            (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n          )\n        )\n        (transformer_block1): TransformerEncoder(\n          (attn): MultiHeadedAttention(\n            (dropout): Dropout(p=0.2, inplace=False)\n            (q_proj): Linear(in_features=32, out_features=32, bias=False)\n            (kv_proj): Linear(in_features=32, out_features=64, bias=False)\n            (out_proj): Linear(in_features=32, out_features=32, bias=False)\n          )\n          (ff): FeedForward(\n            (w_1): Linear(in_features=32, out_features=128, bias=True)\n            (w_2): Linear(in_features=128, out_features=32, bias=True)\n            (dropout): Dropout(p=0.2, inplace=False)\n            (activation): GELU(approximate='none')\n          )\n          (attn_addnorm): AddNorm(\n            (dropout): Dropout(p=0.2, inplace=False)\n            (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n          )\n          (ff_addnorm): AddNorm(\n            (dropout): Dropout(p=0.2, inplace=False)\n            (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n          )\n        )\n        (transformer_block2): TransformerEncoder(\n          (attn): MultiHeadedAttention(\n            (dropout): Dropout(p=0.2, inplace=False)\n            (q_proj): Linear(in_features=32, out_features=32, bias=False)\n            (kv_proj): Linear(in_features=32, out_features=64, bias=False)\n            (out_proj): Linear(in_features=32, out_features=32, bias=False)\n          )\n          (ff): FeedForward(\n            (w_1): Linear(in_features=32, out_features=128, bias=True)\n            (w_2): Linear(in_features=128, out_features=32, bias=True)\n            (dropout): Dropout(p=0.2, inplace=False)\n            (activation): GELU(approximate='none')\n          )\n          (attn_addnorm): AddNorm(\n            (dropout): Dropout(p=0.2, inplace=False)\n            (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n          )\n          (ff_addnorm): AddNorm(\n            (dropout): Dropout(p=0.2, inplace=False)\n            (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n          )\n        )\n        (transformer_block3): TransformerEncoder(\n          (attn): MultiHeadedAttention(\n            (dropout): Dropout(p=0.2, inplace=False)\n            (q_proj): Linear(in_features=32, out_features=32, bias=False)\n            (kv_proj): Linear(in_features=32, out_features=64, bias=False)\n            (out_proj): Linear(in_features=32, out_features=32, bias=False)\n          )\n          (ff): FeedForward(\n            (w_1): Linear(in_features=32, out_features=128, bias=True)\n            (w_2): Linear(in_features=128, out_features=32, bias=True)\n            (dropout): Dropout(p=0.2, inplace=False)\n            (activation): GELU(approximate='none')\n          )\n          (attn_addnorm): AddNorm(\n            (dropout): Dropout(p=0.2, inplace=False)\n            (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n          )\n          (ff_addnorm): AddNorm(\n            (dropout): Dropout(p=0.2, inplace=False)\n            (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=384, out_features=1, bias=True)\n  )\n)</pre> In\u00a0[31]: Copied! <pre>tab_trainer = Trainer(\n    model=tab_model,\n    objective=\"binary\",\n    optimizers=torch.optim.AdamW(tab_model.parameters(), lr=0.001),\n    metrics=[Accuracy, Precision],\n)\n</pre> tab_trainer = Trainer(     model=tab_model,     objective=\"binary\",     optimizers=torch.optim.AdamW(tab_model.parameters(), lr=0.001),     metrics=[Accuracy, Precision], ) In\u00a0[32]: Copied! <pre>tab_trainer.fit(X_tab=X_tab, target=target, n_epochs=1, batch_size=128, val_split=0.2)\n</pre> tab_trainer.fit(X_tab=X_tab, target=target, n_epochs=1, batch_size=128, val_split=0.2) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 306/306 [00:11&lt;00:00, 27.57it/s, loss=0.359, metrics={'acc': 0.8334, 'prec': 0.7082}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 77/77 [00:01&lt;00:00, 57.89it/s, loss=0.33, metrics={'acc': 0.8536, 'prec': 0.7152}]\n</pre>"},{"location":"examples/03_binary_classification_with_defaults.html#simple-binary-classification-with-defaults","title":"Simple Binary Classification with defaults\u00b6","text":"<p>In this notebook we will train a Wide and Deep model and simply a \"Deep\" model using the well known adult dataset</p>"},{"location":"examples/03_binary_classification_with_defaults.html#preparing-the-data","title":"Preparing the data\u00b6","text":""},{"location":"examples/03_binary_classification_with_defaults.html#defining-the-model","title":"Defining the model\u00b6","text":""},{"location":"examples/04_regression_with_images_and_text.html","title":"04_regression_with_images_and_text","text":"In\u00a0[1]: Copied! <pre>import numpy as np\nimport pandas as pd\nimport os\nimport torch\nfrom torchvision.transforms import ToTensor, Normalize\n\nfrom pytorch_widedeep import Trainer\nfrom pytorch_widedeep.preprocessing import (\n    WidePreprocessor,\n    TabPreprocessor,\n    TextPreprocessor,\n    ImagePreprocessor,\n)\nfrom pytorch_widedeep.models import (\n    Wide,\n    TabMlp,\n    Vision,\n    BasicRNN,\n    WideDeep,\n)\nfrom pytorch_widedeep.losses import RMSELoss\nfrom pytorch_widedeep.initializers import *\nfrom pytorch_widedeep.callbacks import *\n</pre> import numpy as np import pandas as pd import os import torch from torchvision.transforms import ToTensor, Normalize  from pytorch_widedeep import Trainer from pytorch_widedeep.preprocessing import (     WidePreprocessor,     TabPreprocessor,     TextPreprocessor,     ImagePreprocessor, ) from pytorch_widedeep.models import (     Wide,     TabMlp,     Vision,     BasicRNN,     WideDeep, ) from pytorch_widedeep.losses import RMSELoss from pytorch_widedeep.initializers import * from pytorch_widedeep.callbacks import * <pre>/Users/javierrodriguezzaurin/.pyenv/versions/3.10.13/envs/widedeep310/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n  from .autonotebook import tqdm as notebook_tqdm\n</pre> In\u00a0[2]: Copied! <pre>df = pd.read_csv(\"../tmp_data/airbnb/airbnb_sample.csv\")\ndf.head()\n</pre> df = pd.read_csv(\"../tmp_data/airbnb/airbnb_sample.csv\") df.head() Out[2]: id host_id description host_listings_count host_identity_verified neighbourhood_cleansed latitude longitude is_location_exact property_type ... amenity_wide_entrance amenity_wide_entrance_for_guests amenity_wide_entryway amenity_wide_hallways amenity_wifi amenity_window_guards amenity_wine_cooler security_deposit extra_people yield 0 13913.jpg 54730 My bright double bedroom with a large window h... 4.0 f Islington 51.56802 -0.11121 t apartment ... 1 0 0 0 1 0 0 100.0 15.0 12.00 1 15400.jpg 60302 Lots of windows and light.  St Luke's Gardens ... 1.0 t Kensington and Chelsea 51.48796 -0.16898 t apartment ... 0 0 0 0 1 0 0 150.0 0.0 109.50 2 17402.jpg 67564 Open from June 2018 after a 3-year break, we a... 19.0 t Westminster 51.52098 -0.14002 t apartment ... 0 0 0 0 1 0 0 350.0 10.0 149.65 3 24328.jpg 41759 Artist house, bright high ceiling rooms, priva... 2.0 t Wandsworth 51.47298 -0.16376 t other ... 0 0 0 0 1 0 0 250.0 0.0 215.60 4 25023.jpg 102813 Large, all comforts, 2-bed flat; first floor; ... 1.0 f Wandsworth 51.44687 -0.21874 t apartment ... 0 0 0 0 1 0 0 250.0 11.0 79.35 <p>5 rows \u00d7 223 columns</p> In\u00a0[3]: Copied! <pre># There are a number of columns that are already binary. Therefore, no need to one hot encode them\ncrossed_cols = [(\"property_type\", \"room_type\")]\nalready_dummies = [c for c in df.columns if \"amenity\" in c] + [\"has_house_rules\"]\nwide_cols = [\n    \"is_location_exact\",\n    \"property_type\",\n    \"room_type\",\n    \"host_gender\",\n    \"instant_bookable\",\n] + already_dummies\n\ncat_embed_cols = [(c, 16) for c in df.columns if \"catg\" in c] + [\n    (\"neighbourhood_cleansed\", 64),\n    (\"cancellation_policy\", 16),\n]\ncontinuous_cols = [\"latitude\", \"longitude\", \"security_deposit\", \"extra_people\"]\n\n# text and image colnames\ntext_col = \"description\"\nimg_col = \"id\"\n\n# path to pretrained word embeddings and the images\nword_vectors_path = \"../tmp_data/glove.6B/glove.6B.100d.txt\"\nimg_path = \"../tmp_data/airbnb/property_picture\"\n\n# target\ntarget_col = \"yield\"\n</pre> # There are a number of columns that are already binary. Therefore, no need to one hot encode them crossed_cols = [(\"property_type\", \"room_type\")] already_dummies = [c for c in df.columns if \"amenity\" in c] + [\"has_house_rules\"] wide_cols = [     \"is_location_exact\",     \"property_type\",     \"room_type\",     \"host_gender\",     \"instant_bookable\", ] + already_dummies  cat_embed_cols = [(c, 16) for c in df.columns if \"catg\" in c] + [     (\"neighbourhood_cleansed\", 64),     (\"cancellation_policy\", 16), ] continuous_cols = [\"latitude\", \"longitude\", \"security_deposit\", \"extra_people\"]  # text and image colnames text_col = \"description\" img_col = \"id\"  # path to pretrained word embeddings and the images word_vectors_path = \"../tmp_data/glove.6B/glove.6B.100d.txt\" img_path = \"../tmp_data/airbnb/property_picture\"  # target target_col = \"yield\" In\u00a0[4]: Copied! <pre>target = df[target_col].values\n</pre> target = df[target_col].values In\u00a0[5]: Copied! <pre>wide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols)\nX_wide = wide_preprocessor.fit_transform(df)\n</pre> wide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols) X_wide = wide_preprocessor.fit_transform(df) In\u00a0[6]: Copied! <pre>tab_preprocessor = TabPreprocessor(\n    cat_embed_cols=cat_embed_cols,\n    continuous_cols=continuous_cols,\n)\nX_tab = tab_preprocessor.fit_transform(df)\n</pre> tab_preprocessor = TabPreprocessor(     cat_embed_cols=cat_embed_cols,     continuous_cols=continuous_cols, ) X_tab = tab_preprocessor.fit_transform(df) <pre>/Users/javierrodriguezzaurin/Projects/pytorch-widedeep/pytorch_widedeep/preprocessing/tab_preprocessor.py:358: UserWarning: Continuous columns will not be normalised\n  warnings.warn(\"Continuous columns will not be normalised\")\n</pre> In\u00a0[7]: Copied! <pre>text_preprocessor = TextPreprocessor(\n    word_vectors_path=word_vectors_path, text_col=text_col\n)\nX_text = text_preprocessor.fit_transform(df)\n</pre> text_preprocessor = TextPreprocessor(     word_vectors_path=word_vectors_path, text_col=text_col ) X_text = text_preprocessor.fit_transform(df) <pre>The vocabulary contains 2192 tokens\nIndexing word vectors...\nLoaded 400000 word vectors\nPreparing embeddings matrix...\n2175 words in the vocabulary had ../tmp_data/glove.6B/glove.6B.100d.txt vectors and appear more than 5 times\n</pre> In\u00a0[8]: Copied! <pre>image_processor = ImagePreprocessor(img_col=img_col, img_path=img_path)\nX_images = image_processor.fit_transform(df)\n</pre> image_processor = ImagePreprocessor(img_col=img_col, img_path=img_path) X_images = image_processor.fit_transform(df) <pre>Reading Images from ../tmp_data/airbnb/property_picture\nResizing\n</pre> <pre>100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 1001/1001 [00:01&lt;00:00, 638.00it/s]\n</pre> <pre>Computing normalisation metrics\n</pre> In\u00a0[9]: Copied! <pre># Linear model\nwide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1)\n\n# DeepDense: 2 Dense layers\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=continuous_cols,\n    mlp_hidden_dims=[128, 64],\n    mlp_dropout=0.1,\n)\n\n# DeepText: a stack of 2 LSTMs\nbasic_rnn = BasicRNN(\n    vocab_size=len(text_preprocessor.vocab.itos),\n    embed_matrix=text_preprocessor.embedding_matrix,\n    n_layers=2,\n    hidden_dim=64,\n    rnn_dropout=0.5,\n)\n\n# Pretrained Resnet 18\nresnet = Vision(pretrained_model_setup=\"resnet18\", n_trainable=4)\n</pre> # Linear model wide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1)  # DeepDense: 2 Dense layers tab_mlp = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     continuous_cols=continuous_cols,     mlp_hidden_dims=[128, 64],     mlp_dropout=0.1, )  # DeepText: a stack of 2 LSTMs basic_rnn = BasicRNN(     vocab_size=len(text_preprocessor.vocab.itos),     embed_matrix=text_preprocessor.embedding_matrix,     n_layers=2,     hidden_dim=64,     rnn_dropout=0.5, )  # Pretrained Resnet 18 resnet = Vision(pretrained_model_setup=\"resnet18\", n_trainable=4) <p>Combine them all with the \"collector\" class <code>WideDeep</code></p> In\u00a0[10]: Copied! <pre>model = WideDeep(\n    wide=wide,\n    deeptabular=tab_mlp,\n    deeptext=basic_rnn,\n    deepimage=resnet,\n    head_hidden_dims=[256, 128],\n)\n</pre> model = WideDeep(     wide=wide,     deeptabular=tab_mlp,     deeptext=basic_rnn,     deepimage=resnet,     head_hidden_dims=[256, 128], ) In\u00a0[11]: Copied! <pre>trainer = Trainer(model, objective=\"rmse\")\n</pre> trainer = Trainer(model, objective=\"rmse\") In\u00a0[12]: Copied! <pre>trainer.fit(\n    X_wide=X_wide,\n    X_tab=X_tab,\n    X_text=X_text,\n    X_img=X_images,\n    target=target,\n    n_epochs=1,\n    batch_size=32,\n    val_split=0.2,\n)\n</pre> trainer.fit(     X_wide=X_wide,     X_tab=X_tab,     X_text=X_text,     X_img=X_images,     target=target,     n_epochs=1,     batch_size=32,     val_split=0.2, ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 25/25 [00:19&lt;00:00,  1.28it/s, loss=115]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 7/7 [00:04&lt;00:00,  1.62it/s, loss=94.1]\n</pre> <p>Both, the Text and Image components allow FC-heads on their own (have a look to the documentation).</p> <p>Now let's go \"kaggle crazy\". Let's use different optimizers, initializers and schedulers for different components. Moreover, let's use a different learning rate for different parameter groups, for the <code>deeptabular</code> component</p> In\u00a0[13]: Copied! <pre>deep_params = []\nfor childname, child in model.named_children():\n    if childname == \"deeptabular\":\n        for n, p in child.named_parameters():\n            if \"embed_layer\" in n:\n                deep_params.append({\"params\": p, \"lr\": 1e-4})\n            else:\n                deep_params.append({\"params\": p, \"lr\": 1e-3})\n</pre> deep_params = [] for childname, child in model.named_children():     if childname == \"deeptabular\":         for n, p in child.named_parameters():             if \"embed_layer\" in n:                 deep_params.append({\"params\": p, \"lr\": 1e-4})             else:                 deep_params.append({\"params\": p, \"lr\": 1e-3}) In\u00a0[14]: Copied! <pre>wide_opt = torch.optim.Adam(model.wide.parameters(), lr=0.03)\ndeep_opt = torch.optim.Adam(deep_params)\ntext_opt = torch.optim.AdamW(model.deeptext.parameters())\nimg_opt = torch.optim.AdamW(model.deepimage.parameters())\nhead_opt = torch.optim.Adam(model.deephead.parameters())\n</pre> wide_opt = torch.optim.Adam(model.wide.parameters(), lr=0.03) deep_opt = torch.optim.Adam(deep_params) text_opt = torch.optim.AdamW(model.deeptext.parameters()) img_opt = torch.optim.AdamW(model.deepimage.parameters()) head_opt = torch.optim.Adam(model.deephead.parameters()) In\u00a0[15]: Copied! <pre>wide_sch = torch.optim.lr_scheduler.StepLR(wide_opt, step_size=5)\ndeep_sch = torch.optim.lr_scheduler.MultiStepLR(deep_opt, milestones=[3, 8])\ntext_sch = torch.optim.lr_scheduler.StepLR(text_opt, step_size=5)\nimg_sch = torch.optim.lr_scheduler.MultiStepLR(deep_opt, milestones=[3, 8])\nhead_sch = torch.optim.lr_scheduler.StepLR(head_opt, step_size=5)\n</pre> wide_sch = torch.optim.lr_scheduler.StepLR(wide_opt, step_size=5) deep_sch = torch.optim.lr_scheduler.MultiStepLR(deep_opt, milestones=[3, 8]) text_sch = torch.optim.lr_scheduler.StepLR(text_opt, step_size=5) img_sch = torch.optim.lr_scheduler.MultiStepLR(deep_opt, milestones=[3, 8]) head_sch = torch.optim.lr_scheduler.StepLR(head_opt, step_size=5) In\u00a0[16]: Copied! <pre># remember, one optimizer per model components, for lr_schedures and initializers is not neccesary\noptimizers = {\n    \"wide\": wide_opt,\n    \"deeptabular\": deep_opt,\n    \"deeptext\": text_opt,\n    \"deepimage\": img_opt,\n    \"deephead\": head_opt,\n}\nschedulers = {\n    \"wide\": wide_sch,\n    \"deeptabular\": deep_sch,\n    \"deeptext\": text_sch,\n    \"deepimage\": img_sch,\n    \"deephead\": head_sch,\n}\n\n# Now...we have used pretrained word embeddings, so you do not want to\n# initialise these  embeddings. However you might still want to initialise the\n# other layers in the DeepText component. No probs, you can do that with the\n# parameter pattern and your knowledge on regular  expressions. Here we are\n# telling to the KaimingNormal initializer to NOT touch the  parameters whose\n# name contains the string word_embed.\ninitializers = {\n    \"wide\": KaimingNormal,\n    \"deeptabular\": KaimingNormal,\n    \"deeptext\": KaimingNormal(pattern=r\"^(?!.*word_embed).*$\"),\n    \"deepimage\": KaimingNormal,\n}\n\nmean = [0.406, 0.456, 0.485]  # BGR\nstd = [0.225, 0.224, 0.229]  # BGR\ntransforms = [ToTensor, Normalize(mean=mean, std=std)]\ncallbacks = [\n    LRHistory(n_epochs=10),\n    EarlyStopping,\n    ModelCheckpoint(filepath=\"model_weights/wd_out\"),\n]\n</pre> # remember, one optimizer per model components, for lr_schedures and initializers is not neccesary optimizers = {     \"wide\": wide_opt,     \"deeptabular\": deep_opt,     \"deeptext\": text_opt,     \"deepimage\": img_opt,     \"deephead\": head_opt, } schedulers = {     \"wide\": wide_sch,     \"deeptabular\": deep_sch,     \"deeptext\": text_sch,     \"deepimage\": img_sch,     \"deephead\": head_sch, }  # Now...we have used pretrained word embeddings, so you do not want to # initialise these  embeddings. However you might still want to initialise the # other layers in the DeepText component. No probs, you can do that with the # parameter pattern and your knowledge on regular  expressions. Here we are # telling to the KaimingNormal initializer to NOT touch the  parameters whose # name contains the string word_embed. initializers = {     \"wide\": KaimingNormal,     \"deeptabular\": KaimingNormal,     \"deeptext\": KaimingNormal(pattern=r\"^(?!.*word_embed).*$\"),     \"deepimage\": KaimingNormal, }  mean = [0.406, 0.456, 0.485]  # BGR std = [0.225, 0.224, 0.229]  # BGR transforms = [ToTensor, Normalize(mean=mean, std=std)] callbacks = [     LRHistory(n_epochs=10),     EarlyStopping,     ModelCheckpoint(filepath=\"model_weights/wd_out\"), ] In\u00a0[17]: Copied! <pre>trainer = Trainer(\n    model,\n    objective=\"rmse\",\n    initializers=initializers,\n    optimizers=optimizers,\n    lr_schedulers=schedulers,\n    callbacks=callbacks,\n    transforms=transforms,\n)\n</pre> trainer = Trainer(     model,     objective=\"rmse\",     initializers=initializers,     optimizers=optimizers,     lr_schedulers=schedulers,     callbacks=callbacks,     transforms=transforms, ) <pre>/Users/javierrodriguezzaurin/Projects/pytorch-widedeep/pytorch_widedeep/initializers.py:34: UserWarning: No initializer found for deephead\n  warnings.warn(\n</pre> In\u00a0[18]: Copied! <pre>trainer.fit(\n    X_wide=X_wide,\n    X_tab=X_tab,\n    X_text=X_text,\n    X_img=X_images,\n    target=target,\n    n_epochs=1,\n    batch_size=32,\n    val_split=0.2,\n)\n</pre> trainer.fit(     X_wide=X_wide,     X_tab=X_tab,     X_text=X_text,     X_img=X_images,     target=target,     n_epochs=1,     batch_size=32,     val_split=0.2, ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 25/25 [00:19&lt;00:00,  1.25it/s, loss=101]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 7/7 [00:04&lt;00:00,  1.62it/s, loss=90.6]\n</pre> <pre>Model weights after training corresponds to the those of the final epoch which might not be the best performing weights. Use the 'ModelCheckpoint' Callback to restore the best epoch weights.\n</pre> <p>we have only run one epoch, but let's check that the LRHistory callback records the lr values for each group</p> In\u00a0[19]: Copied! <pre>trainer.lr_history\n</pre> trainer.lr_history Out[19]: <pre>{'lr_wide_0': [0.03, 0.03],\n 'lr_deeptabular_0': [0.0001, 0.0001],\n 'lr_deeptabular_1': [0.0001, 0.0001],\n 'lr_deeptabular_2': [0.0001, 0.0001],\n 'lr_deeptabular_3': [0.0001, 0.0001],\n 'lr_deeptabular_4': [0.0001, 0.0001],\n 'lr_deeptabular_5': [0.0001, 0.0001],\n 'lr_deeptabular_6': [0.0001, 0.0001],\n 'lr_deeptabular_7': [0.0001, 0.0001],\n 'lr_deeptabular_8': [0.0001, 0.0001],\n 'lr_deeptabular_9': [0.001, 0.001],\n 'lr_deeptabular_10': [0.001, 0.001],\n 'lr_deeptabular_11': [0.001, 0.001],\n 'lr_deeptabular_12': [0.001, 0.001],\n 'lr_deeptext_0': [0.001, 0.001],\n 'lr_deepimage_0': [0.001, 0.001],\n 'lr_deephead_0': [0.001, 0.001]}</pre>"},{"location":"examples/04_regression_with_images_and_text.html#regression-with-images-and-text","title":"Regression with Images and Text\u00b6","text":"<p>In this notebook we will go through a series of examples on how to combine all Wide &amp; Deep components.</p> <p>To that aim I will use the Airbnb listings dataset for London, which you can download from here. I use this dataset simply because it contains tabular data, images and text.</p> <p>I have taken a sample of 1000 listings to keep the data tractable in this notebook. Also, I have preprocessed the data and prepared it for this exercise. All preprocessing steps can be found in the notebook <code>airbnb_data_preprocessing.ipynb</code> in this <code>examples</code> folder.</p>"},{"location":"examples/04_regression_with_images_and_text.html#regression-with-the-defaults","title":"Regression with the defaults\u00b6","text":"<p>The set up</p>"},{"location":"examples/04_regression_with_images_and_text.html#prepare-the-data","title":"Prepare the data\u00b6","text":"<p>I will focus here on how to prepare the data and run the model. Check notebooks 1 and 2 to see what's going on behind the scences</p> <p>Preparing the data is rather simple</p>"},{"location":"examples/04_regression_with_images_and_text.html#build-the-model-components","title":"Build the model components\u00b6","text":""},{"location":"examples/04_regression_with_images_and_text.html#build-the-trainer-and-fit","title":"Build the trainer and fit\u00b6","text":""},{"location":"examples/05_save_and_load_model_and_artifacts.html","title":"05_save_and_load_model_and_artifacts","text":"In\u00a0[1]: Copied! <pre>import pickle\nimport numpy as np\nimport pandas as pd\nimport torch\nimport shutil\n\nfrom pytorch_widedeep.preprocessing import TabPreprocessor\nfrom pytorch_widedeep.training import Trainer\nfrom pytorch_widedeep.callbacks import EarlyStopping, ModelCheckpoint, LRHistory\nfrom pytorch_widedeep.models import TabMlp, WideDeep\nfrom pytorch_widedeep.metrics import Accuracy\nfrom pytorch_widedeep.datasets import load_adult\nfrom sklearn.model_selection import train_test_split\n</pre> import pickle import numpy as np import pandas as pd import torch import shutil  from pytorch_widedeep.preprocessing import TabPreprocessor from pytorch_widedeep.training import Trainer from pytorch_widedeep.callbacks import EarlyStopping, ModelCheckpoint, LRHistory from pytorch_widedeep.models import TabMlp, WideDeep from pytorch_widedeep.metrics import Accuracy from pytorch_widedeep.datasets import load_adult from sklearn.model_selection import train_test_split <pre>/Users/javierrodriguezzaurin/.pyenv/versions/3.10.13/envs/widedeep310/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n  from .autonotebook import tqdm as notebook_tqdm\n</pre> In\u00a0[2]: Copied! <pre>df = load_adult(as_frame=True)\ndf.head()\n</pre> df = load_adult(as_frame=True) df.head() Out[2]: age workclass fnlwgt education educational-num marital-status occupation relationship race gender capital-gain capital-loss hours-per-week native-country income 0 25 Private 226802 11th 7 Never-married Machine-op-inspct Own-child Black Male 0 0 40 United-States &lt;=50K 1 38 Private 89814 HS-grad 9 Married-civ-spouse Farming-fishing Husband White Male 0 0 50 United-States &lt;=50K 2 28 Local-gov 336951 Assoc-acdm 12 Married-civ-spouse Protective-serv Husband White Male 0 0 40 United-States &gt;50K 3 44 Private 160323 Some-college 10 Married-civ-spouse Machine-op-inspct Husband Black Male 7688 0 40 United-States &gt;50K 4 18 ? 103497 Some-college 10 Never-married ? Own-child White Female 0 0 30 United-States &lt;=50K In\u00a0[3]: Copied! <pre># For convenience, we'll replace '-' with '_'\ndf.columns = [c.replace(\"-\", \"_\") for c in df.columns]\n# binary target\ndf[\"target\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int)\ndf.drop(\"income\", axis=1, inplace=True)\ndf.head()\n</pre> # For convenience, we'll replace '-' with '_' df.columns = [c.replace(\"-\", \"_\") for c in df.columns] # binary target df[\"target\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int) df.drop(\"income\", axis=1, inplace=True) df.head() Out[3]: age workclass fnlwgt education educational_num marital_status occupation relationship race gender capital_gain capital_loss hours_per_week native_country target 0 25 Private 226802 11th 7 Never-married Machine-op-inspct Own-child Black Male 0 0 40 United-States 0 1 38 Private 89814 HS-grad 9 Married-civ-spouse Farming-fishing Husband White Male 0 0 50 United-States 0 2 28 Local-gov 336951 Assoc-acdm 12 Married-civ-spouse Protective-serv Husband White Male 0 0 40 United-States 1 3 44 Private 160323 Some-college 10 Married-civ-spouse Machine-op-inspct Husband Black Male 7688 0 40 United-States 1 4 18 ? 103497 Some-college 10 Never-married ? Own-child White Female 0 0 30 United-States 0 In\u00a0[4]: Copied! <pre>train, valid = train_test_split(df, test_size=0.2, stratify=df.target)\n# the test data will be used lately as if it was \"fresh\", new data coming after some time...\nvalid, test = train_test_split(valid, test_size=0.5, stratify=valid.target)\n</pre> train, valid = train_test_split(df, test_size=0.2, stratify=df.target) # the test data will be used lately as if it was \"fresh\", new data coming after some time... valid, test = train_test_split(valid, test_size=0.5, stratify=valid.target) In\u00a0[5]: Copied! <pre>print(f\"train shape: {train.shape}\")\nprint(f\"valid shape: {valid.shape}\")\nprint(f\"test shape: {test.shape}\")\n</pre> print(f\"train shape: {train.shape}\") print(f\"valid shape: {valid.shape}\") print(f\"test shape: {test.shape}\") <pre>train shape: (39073, 15)\nvalid shape: (4884, 15)\ntest shape: (4885, 15)\n</pre> In\u00a0[6]: Copied! <pre>cat_embed_cols = [\n    \"workclass\",\n    \"education\",\n    \"marital_status\",\n    \"occupation\",\n    \"relationship\",\n    \"race\",\n    \"gender\",\n    \"capital_gain\",\n    \"capital_loss\",\n    \"native_country\",\n]\ncontinuous_cols = [\"age\", \"hours_per_week\"]\n</pre> cat_embed_cols = [     \"workclass\",     \"education\",     \"marital_status\",     \"occupation\",     \"relationship\",     \"race\",     \"gender\",     \"capital_gain\",     \"capital_loss\",     \"native_country\", ] continuous_cols = [\"age\", \"hours_per_week\"] In\u00a0[7]: Copied! <pre>tab_preprocessor = TabPreprocessor(\n    embed_cols=cat_embed_cols,\n    continuous_cols=continuous_cols,\n)\nX_tab_train = tab_preprocessor.fit_transform(train)\ny_train = train.target.values\nX_tab_valid = tab_preprocessor.transform(valid)\ny_valid = valid.target.values\n</pre> tab_preprocessor = TabPreprocessor(     embed_cols=cat_embed_cols,     continuous_cols=continuous_cols, ) X_tab_train = tab_preprocessor.fit_transform(train) y_train = train.target.values X_tab_valid = tab_preprocessor.transform(valid) y_valid = valid.target.values <pre>/Users/javierrodriguezzaurin/Projects/pytorch-widedeep/pytorch_widedeep/preprocessing/tab_preprocessor.py:358: UserWarning: Continuous columns will not be normalised\n  warnings.warn(\"Continuous columns will not be normalised\")\n</pre> In\u00a0[8]: Copied! <pre>tab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    cat_embed_dropout=0.1,\n    continuous_cols=continuous_cols,\n    cont_norm_layer=\"layernorm\",\n    embed_continuous_method=\"standard\",\n    cont_embed_dim=8,\n    mlp_hidden_dims=[64, 32],\n    mlp_dropout=0.2,\n    mlp_activation=\"leaky_relu\",\n)\nmodel = WideDeep(deeptabular=tab_mlp)\n</pre> tab_mlp = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     cat_embed_dropout=0.1,     continuous_cols=continuous_cols,     cont_norm_layer=\"layernorm\",     embed_continuous_method=\"standard\",     cont_embed_dim=8,     mlp_hidden_dims=[64, 32],     mlp_dropout=0.2,     mlp_activation=\"leaky_relu\", ) model = WideDeep(deeptabular=tab_mlp) In\u00a0[9]: Copied! <pre>model\n</pre> model Out[9]: <pre>WideDeep(\n  (deeptabular): Sequential(\n    (0): TabMlp(\n      (cat_embed): DiffSizeCatEmbeddings(\n        (embed_layers): ModuleDict(\n          (emb_layer_workclass): Embedding(10, 5, padding_idx=0)\n          (emb_layer_education): Embedding(17, 8, padding_idx=0)\n          (emb_layer_marital_status): Embedding(8, 5, padding_idx=0)\n          (emb_layer_occupation): Embedding(16, 7, padding_idx=0)\n          (emb_layer_relationship): Embedding(7, 4, padding_idx=0)\n          (emb_layer_race): Embedding(6, 4, padding_idx=0)\n          (emb_layer_gender): Embedding(3, 2, padding_idx=0)\n          (emb_layer_capital_gain): Embedding(122, 23, padding_idx=0)\n          (emb_layer_capital_loss): Embedding(97, 21, padding_idx=0)\n          (emb_layer_native_country): Embedding(43, 13, padding_idx=0)\n        )\n        (embedding_dropout): Dropout(p=0.1, inplace=False)\n      )\n      (cont_norm): LayerNorm((2,), eps=1e-05, elementwise_affine=True)\n      (cont_embed): ContEmbeddings(\n        INFO: [ContLinear = weight(n_cont_cols, embed_dim) + bias(n_cont_cols, embed_dim)]\n        (linear): ContLinear(n_cont_cols=2, embed_dim=8, embed_dropout=0.0)\n        (dropout): Dropout(p=0.0, inplace=False)\n      )\n      (encoder): MLP(\n        (mlp): Sequential(\n          (dense_layer_0): Sequential(\n            (0): Linear(in_features=108, out_features=64, bias=True)\n            (1): LeakyReLU(negative_slope=0.01, inplace=True)\n            (2): Dropout(p=0.2, inplace=False)\n          )\n          (dense_layer_1): Sequential(\n            (0): Linear(in_features=64, out_features=32, bias=True)\n            (1): LeakyReLU(negative_slope=0.01, inplace=True)\n            (2): Dropout(p=0.2, inplace=False)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=32, out_features=1, bias=True)\n  )\n)</pre> In\u00a0[10]: Copied! <pre>early_stopping = EarlyStopping()\nmodel_checkpoint = ModelCheckpoint(\n    filepath=\"tmp_dir/adult_tabmlp_model\",\n    save_best_only=True,\n    verbose=1,\n    max_save=1,\n)\n\ntrainer = Trainer(\n    model,\n    objective=\"binary\",\n    callbacks=[early_stopping, model_checkpoint],\n    metrics=[Accuracy],\n)\n\ntrainer.fit(\n    X_train={\"X_tab\": X_tab_train, \"target\": y_train},\n    X_val={\"X_tab\": X_tab_valid, \"target\": y_valid},\n    n_epochs=4,\n    batch_size=256,\n)\n</pre> early_stopping = EarlyStopping() model_checkpoint = ModelCheckpoint(     filepath=\"tmp_dir/adult_tabmlp_model\",     save_best_only=True,     verbose=1,     max_save=1, )  trainer = Trainer(     model,     objective=\"binary\",     callbacks=[early_stopping, model_checkpoint],     metrics=[Accuracy], )  trainer.fit(     X_train={\"X_tab\": X_tab_train, \"target\": y_train},     X_val={\"X_tab\": X_tab_valid, \"target\": y_valid},     n_epochs=4,     batch_size=256, ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:02&lt;00:00, 76.25it/s, loss=0.452, metrics={'acc': 0.7867}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 20/20 [00:00&lt;00:00, 125.36it/s, loss=0.335, metrics={'acc': 0.8532}]\n</pre> <pre>\nEpoch 1: val_loss improved from inf to 0.33532 Saving model to tmp_dir/adult_tabmlp_model_1.p\n</pre> <pre>epoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 76.98it/s, loss=0.355, metrics={'acc': 0.8401}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 20/20 [00:00&lt;00:00, 106.51it/s, loss=0.303, metrics={'acc': 0.8665}]\n</pre> <pre>\nEpoch 2: val_loss improved from 0.33532 to 0.30273 Saving model to tmp_dir/adult_tabmlp_model_2.p\n</pre> <pre>epoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 82.71it/s, loss=0.332, metrics={'acc': 0.849}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 20/20 [00:00&lt;00:00, 107.80it/s, loss=0.288, metrics={'acc': 0.8757}]\n</pre> <pre>\nEpoch 3: val_loss improved from 0.30273 to 0.28791 Saving model to tmp_dir/adult_tabmlp_model_3.p\n</pre> <pre>epoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 79.02it/s, loss=0.32, metrics={'acc': 0.8541}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 20/20 [00:00&lt;00:00, 127.07it/s, loss=0.282, metrics={'acc': 0.8763}]</pre> <pre>\nEpoch 4: val_loss improved from 0.28791 to 0.28238 Saving model to tmp_dir/adult_tabmlp_model_4.p\nModel weights restored to best epoch: 4\n</pre> <pre>\n</pre> In\u00a0[11]: Copied! <pre>torch.save(model, \"tmp_dir/model_saved_option_1.pt\")\n</pre> torch.save(model, \"tmp_dir/model_saved_option_1.pt\") In\u00a0[12]: Copied! <pre>torch.save(model.state_dict(), \"tmp_dir/model_state_dict_saved_option_1.pt\")\n</pre> torch.save(model.state_dict(), \"tmp_dir/model_state_dict_saved_option_1.pt\") In\u00a0[13]: Copied! <pre>trainer.save(path=\"tmp_dir/\", model_filename=\"model_saved_option_2.pt\")\n</pre> trainer.save(path=\"tmp_dir/\", model_filename=\"model_saved_option_2.pt\") <p>or the state dict</p> In\u00a0[14]: Copied! <pre>trainer.save(\n    path=\"tmp_dir/\",\n    model_filename=\"model_state_dict_saved_option_2.pt\",\n    save_state_dict=True,\n)\n</pre> trainer.save(     path=\"tmp_dir/\",     model_filename=\"model_state_dict_saved_option_2.pt\",     save_state_dict=True, ) In\u00a0[15]: Copied! <pre>%%bash\n\nls tmp_dir/\n</pre> %%bash  ls tmp_dir/ <pre>adult_tabmlp_model_4.p\nhistory\nmodel_saved_option_1.pt\nmodel_saved_option_2.pt\nmodel_state_dict_saved_option_1.pt\nmodel_state_dict_saved_option_2.pt\n</pre> In\u00a0[16]: Copied! <pre>%%bash\n\nls tmp_dir/history/\n</pre> %%bash  ls tmp_dir/history/ <pre>train_eval_history.json\n</pre> <p>Note that since we have used the <code>ModelCheckpoint</code> Callback, <code>adult_tabmlp_model_2.p</code> is the model state dict of the model at epoch 2, i.e. same as <code>model_state_dict_saved_option_1.p</code> or <code>model_state_dict_saved_option_2.p</code>.</p> In\u00a0[17]: Copied! <pre>with open(\"tmp_dir/tab_preproc.pkl\", \"wb\") as dp:\n    pickle.dump(tab_preprocessor, dp)\n</pre> with open(\"tmp_dir/tab_preproc.pkl\", \"wb\") as dp:     pickle.dump(tab_preprocessor, dp) In\u00a0[18]: Copied! <pre>with open(\"tmp_dir/eary_stop.pkl\", \"wb\") as es:\n    pickle.dump(early_stopping, es)\n</pre> with open(\"tmp_dir/eary_stop.pkl\", \"wb\") as es:     pickle.dump(early_stopping, es) In\u00a0[19]: Copied! <pre>%%bash\n\nls tmp_dir/\n</pre> %%bash  ls tmp_dir/ <pre>adult_tabmlp_model_4.p\neary_stop.pkl\nhistory\nmodel_saved_option_1.pt\nmodel_saved_option_2.pt\nmodel_state_dict_saved_option_1.pt\nmodel_state_dict_saved_option_2.pt\ntab_preproc.pkl\n</pre> <p>And that is pretty much all you need to resume training or directly predict, let's see</p> In\u00a0[20]: Copied! <pre>test.head()\n</pre> test.head() Out[20]: age workclass fnlwgt education educational_num marital_status occupation relationship race gender capital_gain capital_loss hours_per_week native_country target 10103 43 Private 198282 HS-grad 9 Married-civ-spouse Craft-repair Husband White Male 0 0 40 United-States 1 31799 20 Private 228686 11th 7 Married-civ-spouse Other-service Husband White Male 0 0 40 United-States 0 19971 26 Private 291968 HS-grad 9 Married-civ-spouse Transport-moving Husband White Male 0 0 44 United-States 0 3039 48 Private 175958 Bachelors 13 Divorced Prof-specialty Not-in-family White Male 0 0 30 United-States 0 20725 18 Private 232024 11th 7 Never-married Machine-op-inspct Own-child White Male 0 0 55 United-States 0 In\u00a0[21]: Copied! <pre>with open(\"tmp_dir/tab_preproc.pkl\", \"rb\") as tp:\n    tab_preprocessor_new = pickle.load(tp)\n</pre> with open(\"tmp_dir/tab_preproc.pkl\", \"rb\") as tp:     tab_preprocessor_new = pickle.load(tp) In\u00a0[22]: Copied! <pre>X_test_tab = tab_preprocessor_new.transform(test)\ny_test = test.target\n</pre> X_test_tab = tab_preprocessor_new.transform(test) y_test = test.target In\u00a0[23]: Copied! <pre>tab_mlp_new = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    cat_embed_dropout=0.1,\n    continuous_cols=continuous_cols,\n    cont_norm_layer=\"layernorm\",\n    embed_continuous_method=\"standard\",\n    cont_embed_dim=8,\n    mlp_hidden_dims=[64, 32],\n    mlp_dropout=0.2,\n    mlp_activation=\"leaky_relu\",\n)\nnew_model = WideDeep(deeptabular=tab_mlp)\n</pre> tab_mlp_new = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     cat_embed_dropout=0.1,     continuous_cols=continuous_cols,     cont_norm_layer=\"layernorm\",     embed_continuous_method=\"standard\",     cont_embed_dim=8,     mlp_hidden_dims=[64, 32],     mlp_dropout=0.2,     mlp_activation=\"leaky_relu\", ) new_model = WideDeep(deeptabular=tab_mlp) In\u00a0[24]: Copied! <pre>new_model.load_state_dict(torch.load(\"tmp_dir/model_state_dict_saved_option_2.pt\"))\n</pre> new_model.load_state_dict(torch.load(\"tmp_dir/model_state_dict_saved_option_2.pt\")) Out[24]: <pre>&lt;All keys matched successfully&gt;</pre> In\u00a0[25]: Copied! <pre>trainer = Trainer(\n    model,\n    objective=\"binary\",\n)\n</pre> trainer = Trainer(     model,     objective=\"binary\", ) In\u00a0[26]: Copied! <pre>preds = trainer.predict(X_tab=X_test_tab, batch_size=32)\n</pre> preds = trainer.predict(X_tab=X_test_tab, batch_size=32) <pre>predict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:00&lt;00:00, 309.83it/s]\n</pre> In\u00a0[27]: Copied! <pre>from sklearn.metrics import accuracy_score\n</pre> from sklearn.metrics import accuracy_score In\u00a0[28]: Copied! <pre>accuracy_score(y_test, preds)\n</pre> accuracy_score(y_test, preds) Out[28]: <pre>0.8595701125895598</pre> In\u00a0[29]: Copied! <pre>shutil.rmtree(\"tmp_dir/\")\n</pre> shutil.rmtree(\"tmp_dir/\")"},{"location":"examples/05_save_and_load_model_and_artifacts.html#save-and-load-model-and-artifacts","title":"Save and load model and artifacts\u00b6","text":"<p>In this notebook I will show the different options to save and load a model, as well as some additional objects produced during training.</p> <p>On a given day, you train a model...</p>"},{"location":"examples/05_save_and_load_model_and_artifacts.html#save-model-option-1","title":"Save model: option 1\u00b6","text":"<p>save (and load) a model as you woud do with any other torch model</p>"},{"location":"examples/05_save_and_load_model_and_artifacts.html#save-model-option-2","title":"Save model: option 2\u00b6","text":"<p>use the <code>trainer</code>. The <code>trainer</code> will also save the training history and the learning rate history (if learning rate schedulers are used)</p>"},{"location":"examples/05_save_and_load_model_and_artifacts.html#save-preprocessors-and-callbacks","title":"Save preprocessors and callbacks\u00b6","text":"<p>...just pickle them</p>"},{"location":"examples/05_save_and_load_model_and_artifacts.html#run-new-experiment-prepare-new-dataset-load-model-and-predict","title":"Run New experiment: prepare new dataset, load model, and predict\u00b6","text":""},{"location":"examples/06_finetune_and_warmup.html","title":"06_finetune_and_warmup","text":"In\u00a0[1]: Copied! <pre>import numpy as np\nimport pandas as pd\nimport torch\n\nfrom pytorch_widedeep import Trainer\nfrom pytorch_widedeep.preprocessing import WidePreprocessor, TabPreprocessor\nfrom pytorch_widedeep.models import Wide, TabMlp, TabResnet, WideDeep\nfrom pytorch_widedeep.metrics import Accuracy\nfrom pytorch_widedeep.datasets import load_adult\n</pre> import numpy as np import pandas as pd import torch  from pytorch_widedeep import Trainer from pytorch_widedeep.preprocessing import WidePreprocessor, TabPreprocessor from pytorch_widedeep.models import Wide, TabMlp, TabResnet, WideDeep from pytorch_widedeep.metrics import Accuracy from pytorch_widedeep.datasets import load_adult <pre>/Users/javierrodriguezzaurin/.pyenv/versions/3.10.13/envs/widedeep310/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n  from .autonotebook import tqdm as notebook_tqdm\n</pre> In\u00a0[2]: Copied! <pre>df = load_adult(as_frame=True)\n# For convenience, we'll replace '-' with '_'\ndf.columns = [c.replace(\"-\", \"_\") for c in df.columns]\n# binary target\ndf[\"income_label\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int)\ndf.drop(\"income\", axis=1, inplace=True)\ndf.head()\n</pre> df = load_adult(as_frame=True) # For convenience, we'll replace '-' with '_' df.columns = [c.replace(\"-\", \"_\") for c in df.columns] # binary target df[\"income_label\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int) df.drop(\"income\", axis=1, inplace=True) df.head() Out[2]: age workclass fnlwgt education educational_num marital_status occupation relationship race gender capital_gain capital_loss hours_per_week native_country income_label 0 25 Private 226802 11th 7 Never-married Machine-op-inspct Own-child Black Male 0 0 40 United-States 0 1 38 Private 89814 HS-grad 9 Married-civ-spouse Farming-fishing Husband White Male 0 0 50 United-States 0 2 28 Local-gov 336951 Assoc-acdm 12 Married-civ-spouse Protective-serv Husband White Male 0 0 40 United-States 1 3 44 Private 160323 Some-college 10 Married-civ-spouse Machine-op-inspct Husband Black Male 7688 0 40 United-States 1 4 18 ? 103497 Some-college 10 Never-married ? Own-child White Female 0 0 30 United-States 0 In\u00a0[3]: Copied! <pre># Define wide, crossed and deep tabular columns\nwide_cols = [\n    \"workclass\",\n    \"education\",\n    \"marital_status\",\n    \"occupation\",\n    \"relationship\",\n    \"race\",\n    \"gender\",\n    \"native_country\",\n]\ncrossed_cols = [(\"education\", \"occupation\"), (\"native_country\", \"occupation\")]\ncat_embed_cols = [\n    \"workclass\",\n    \"education\",\n    \"marital_status\",\n    \"occupation\",\n    \"relationship\",\n    \"race\",\n    \"gender\",\n    \"capital_gain\",\n    \"capital_loss\",\n    \"native_country\",\n]\ncontinuous_cols = [\"age\", \"hours_per_week\"]\ntarget_col = \"income_label\"\ntarget = df[target_col].values\n</pre> # Define wide, crossed and deep tabular columns wide_cols = [     \"workclass\",     \"education\",     \"marital_status\",     \"occupation\",     \"relationship\",     \"race\",     \"gender\",     \"native_country\", ] crossed_cols = [(\"education\", \"occupation\"), (\"native_country\", \"occupation\")] cat_embed_cols = [     \"workclass\",     \"education\",     \"marital_status\",     \"occupation\",     \"relationship\",     \"race\",     \"gender\",     \"capital_gain\",     \"capital_loss\",     \"native_country\", ] continuous_cols = [\"age\", \"hours_per_week\"] target_col = \"income_label\" target = df[target_col].values In\u00a0[4]: Copied! <pre># TARGET\ntarget = df[target_col].values\n\n# WIDE\nwide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols)\nX_wide = wide_preprocessor.fit_transform(df)\n\n# DEEP\ntab_preprocessor = TabPreprocessor(\n    cat_embed_cols=cat_embed_cols, continuous_cols=continuous_cols\n)\nX_tab = tab_preprocessor.fit_transform(df)\n</pre> # TARGET target = df[target_col].values  # WIDE wide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols) X_wide = wide_preprocessor.fit_transform(df)  # DEEP tab_preprocessor = TabPreprocessor(     cat_embed_cols=cat_embed_cols, continuous_cols=continuous_cols ) X_tab = tab_preprocessor.fit_transform(df) <pre>/Users/javierrodriguezzaurin/Projects/pytorch-widedeep/pytorch_widedeep/preprocessing/tab_preprocessor.py:358: UserWarning: Continuous columns will not be normalised\n  warnings.warn(\"Continuous columns will not be normalised\")\n</pre> In\u00a0[5]: Copied! <pre>wide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1)\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    cat_embed_dropout=0.1,\n    continuous_cols=continuous_cols,\n    embed_continuous_method=\"standard\",\n    cont_embed_dim=8,\n    mlp_hidden_dims=[64, 32],\n    mlp_dropout=0.2,\n    mlp_activation=\"leaky_relu\",\n)\nmodel = WideDeep(wide=wide, deeptabular=tab_mlp)\n</pre> wide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1) tab_mlp = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     cat_embed_dropout=0.1,     continuous_cols=continuous_cols,     embed_continuous_method=\"standard\",     cont_embed_dim=8,     mlp_hidden_dims=[64, 32],     mlp_dropout=0.2,     mlp_activation=\"leaky_relu\", ) model = WideDeep(wide=wide, deeptabular=tab_mlp) In\u00a0[6]: Copied! <pre>trainer = Trainer(\n    model,\n    objective=\"binary\",\n    optimizers=torch.optim.Adam(model.parameters(), lr=0.01),\n    metrics=[Accuracy],\n)\n</pre> trainer = Trainer(     model,     objective=\"binary\",     optimizers=torch.optim.Adam(model.parameters(), lr=0.01),     metrics=[Accuracy], ) In\u00a0[7]: Copied! <pre>trainer.fit(\n    X_wide=X_wide, X_tab=X_tab, target=target, n_epochs=2, val_split=0.2, batch_size=256\n)\n</pre> trainer.fit(     X_wide=X_wide, X_tab=X_tab, target=target, n_epochs=2, val_split=0.2, batch_size=256 ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:02&lt;00:00, 74.26it/s, loss=0.399, metrics={'acc': 0.8163}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 39/39 [00:00&lt;00:00, 91.03it/s, loss=0.296, metrics={'acc': 0.8677}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 81.31it/s, loss=0.3, metrics={'acc': 0.8614}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 39/39 [00:00&lt;00:00, 106.45it/s, loss=0.285, metrics={'acc': 0.8721}]\n</pre> In\u00a0[8]: Copied! <pre>trainer.save(path=\"models_dir/\", save_state_dict=True, model_filename=\"model_1.pt\")\n</pre> trainer.save(path=\"models_dir/\", save_state_dict=True, model_filename=\"model_1.pt\") <p>Now time goes by...and we want to fine-tune the model to another, new dataset (for example, a dataset that is identical to the one you used to train the previous model but for another country).</p> <p>Here I will use the same dataset just for illustration purposes, but the flow would be identical to that new dataset</p> In\u00a0[9]: Copied! <pre>wide_1 = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1)\ntab_mlp_1 = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    cat_embed_dropout=0.1,\n    continuous_cols=continuous_cols,\n    embed_continuous_method=\"standard\",\n    cont_embed_dim=8,\n    mlp_hidden_dims=[64, 32],\n    mlp_dropout=0.2,\n    mlp_activation=\"leaky_relu\",\n)\nmodel_1 = WideDeep(wide=wide, deeptabular=tab_mlp)\n</pre> wide_1 = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1) tab_mlp_1 = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     cat_embed_dropout=0.1,     continuous_cols=continuous_cols,     embed_continuous_method=\"standard\",     cont_embed_dim=8,     mlp_hidden_dims=[64, 32],     mlp_dropout=0.2,     mlp_activation=\"leaky_relu\", ) model_1 = WideDeep(wide=wide, deeptabular=tab_mlp) In\u00a0[10]: Copied! <pre>model_1.load_state_dict(torch.load(\"models_dir/model_1.pt\"))\n</pre> model_1.load_state_dict(torch.load(\"models_dir/model_1.pt\")) Out[10]: <pre>&lt;All keys matched successfully&gt;</pre> In\u00a0[11]: Copied! <pre>trainer_1 = Trainer(model_1, objective=\"binary\", metrics=[Accuracy])\n</pre> trainer_1 = Trainer(model_1, objective=\"binary\", metrics=[Accuracy]) In\u00a0[12]: Copied! <pre>trainer_1.fit(\n    X_wide=X_wide,\n    X_tab=X_tab,\n    target=target,\n    n_epochs=2,\n    batch_size=256,\n    finetune=True,\n    finetune_epochs=2,\n)\n</pre> trainer_1.fit(     X_wide=X_wide,     X_tab=X_tab,     target=target,     n_epochs=2,     batch_size=256,     finetune=True,     finetune_epochs=2, ) <pre>Training wide for 2 epochs\n</pre> <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 191/191 [00:01&lt;00:00, 97.37it/s, loss=0.39, metrics={'acc': 0.8152}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 191/191 [00:01&lt;00:00, 104.04it/s, loss=0.359, metrics={'acc': 0.824}]\n</pre> <pre>Training deeptabular for 2 epochs\n</pre> <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 191/191 [00:02&lt;00:00, 83.83it/s, loss=0.297, metrics={'acc': 0.8365}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 191/191 [00:02&lt;00:00, 82.78it/s, loss=0.283, metrics={'acc': 0.8445}]\n</pre> <pre>Fine-tuning (or warmup) of individual components completed. Training the whole model for 2 epochs\n</pre> <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 191/191 [00:02&lt;00:00, 72.84it/s, loss=0.281, metrics={'acc': 0.8716}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 191/191 [00:02&lt;00:00, 77.46it/s, loss=0.273, metrics={'acc': 0.8744}]\n</pre> <p>Note that, as I describe above, in scenario 2, we can just use this to warm up models before they joined training begins:</p> In\u00a0[13]: Copied! <pre>wide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1)\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    cat_embed_dropout=0.1,\n    continuous_cols=continuous_cols,\n    embed_continuous_method=\"standard\",\n    cont_embed_dim=8,\n    mlp_hidden_dims=[64, 32],\n    mlp_dropout=0.2,\n    mlp_activation=\"leaky_relu\",\n)\nmodel = WideDeep(wide=wide, deeptabular=tab_mlp)\n</pre> wide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1) tab_mlp = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     cat_embed_dropout=0.1,     continuous_cols=continuous_cols,     embed_continuous_method=\"standard\",     cont_embed_dim=8,     mlp_hidden_dims=[64, 32],     mlp_dropout=0.2,     mlp_activation=\"leaky_relu\", ) model = WideDeep(wide=wide, deeptabular=tab_mlp) In\u00a0[14]: Copied! <pre>trainer_2 = Trainer(model, objective=\"binary\", metrics=[Accuracy])\n</pre> trainer_2 = Trainer(model, objective=\"binary\", metrics=[Accuracy]) In\u00a0[15]: Copied! <pre>trainer_2.fit(\n    X_wide=X_wide,\n    X_tab=X_tab,\n    target=target,\n    val_split=0.1,\n    warmup=True,\n    warmup_epochs=2,\n    n_epochs=2,\n    batch_size=256,\n)\n</pre> trainer_2.fit(     X_wide=X_wide,     X_tab=X_tab,     target=target,     val_split=0.1,     warmup=True,     warmup_epochs=2,     n_epochs=2,     batch_size=256, ) <pre>Training wide for 2 epochs\n</pre> <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:01&lt;00:00, 102.49it/s, loss=0.52, metrics={'acc': 0.7519}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:01&lt;00:00, 98.15it/s, loss=0.381, metrics={'acc': 0.7891}]\n</pre> <pre>Training deeptabular for 2 epochs\n</pre> <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:02&lt;00:00, 82.97it/s, loss=0.356, metrics={'acc': 0.8043}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:02&lt;00:00, 80.27it/s, loss=0.295, metrics={'acc': 0.8195}]\n</pre> <pre>Fine-tuning (or warmup) of individual components completed. Training the whole model for 2 epochs\n</pre> <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:02&lt;00:00, 77.27it/s, loss=0.291, metrics={'acc': 0.8667}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 20/20 [00:00&lt;00:00, 89.57it/s, loss=0.289, metrics={'acc': 0.8665}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:02&lt;00:00, 72.69it/s, loss=0.283, metrics={'acc': 0.8693}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 20/20 [00:00&lt;00:00, 91.81it/s, loss=0.284, metrics={'acc': 0.869}]\n</pre> <p>We need to explicitly indicate</p> <ol> <li><p>That we want fine-tune</p> </li> <li><p>The components that we want to individually fine-tune</p> </li> <li><p>In case of gradual fine-tuning, the routine (\"felbo\" or \"howard\")</p> </li> <li><p>The layers we want to fine-tune.</p> </li> </ol> <p>For example</p> In\u00a0[16]: Copied! <pre>wide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1)\ntab_resnet = TabResnet(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    cat_embed_dropout=0.1,\n    continuous_cols=continuous_cols,\n    blocks_dims=[200, 200, 200],\n)\nmodel = WideDeep(wide=wide, deeptabular=tab_resnet)\n</pre> wide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1) tab_resnet = TabResnet(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     cat_embed_dropout=0.1,     continuous_cols=continuous_cols,     blocks_dims=[200, 200, 200], ) model = WideDeep(wide=wide, deeptabular=tab_resnet) In\u00a0[17]: Copied! <pre>model\n</pre> model Out[17]: <pre>WideDeep(\n  (wide): Wide(\n    (wide_linear): Embedding(809, 1, padding_idx=0)\n  )\n  (deeptabular): Sequential(\n    (0): TabResnet(\n      (cat_embed): DiffSizeCatEmbeddings(\n        (embed_layers): ModuleDict(\n          (emb_layer_workclass): Embedding(10, 5, padding_idx=0)\n          (emb_layer_education): Embedding(17, 8, padding_idx=0)\n          (emb_layer_marital_status): Embedding(8, 5, padding_idx=0)\n          (emb_layer_occupation): Embedding(16, 7, padding_idx=0)\n          (emb_layer_relationship): Embedding(7, 4, padding_idx=0)\n          (emb_layer_race): Embedding(6, 4, padding_idx=0)\n          (emb_layer_gender): Embedding(3, 2, padding_idx=0)\n          (emb_layer_capital_gain): Embedding(124, 24, padding_idx=0)\n          (emb_layer_capital_loss): Embedding(100, 21, padding_idx=0)\n          (emb_layer_native_country): Embedding(43, 13, padding_idx=0)\n        )\n        (embedding_dropout): Dropout(p=0.1, inplace=False)\n      )\n      (cont_norm): Identity()\n      (encoder): DenseResnet(\n        (dense_resnet): Sequential(\n          (lin_inp): Linear(in_features=95, out_features=200, bias=False)\n          (bn_inp): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n          (block_0): BasicBlock(\n            (lin1): Linear(in_features=200, out_features=200, bias=False)\n            (bn1): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n            (leaky_relu): LeakyReLU(negative_slope=0.01, inplace=True)\n            (dp): Dropout(p=0.1, inplace=False)\n            (lin2): Linear(in_features=200, out_features=200, bias=False)\n            (bn2): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n          )\n          (block_1): BasicBlock(\n            (lin1): Linear(in_features=200, out_features=200, bias=False)\n            (bn1): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n            (leaky_relu): LeakyReLU(negative_slope=0.01, inplace=True)\n            (dp): Dropout(p=0.1, inplace=False)\n            (lin2): Linear(in_features=200, out_features=200, bias=False)\n            (bn2): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=200, out_features=1, bias=True)\n  )\n)</pre> <p>let's first train as usual</p> In\u00a0[18]: Copied! <pre>trainer_3 = Trainer(model, objective=\"binary\", metrics=[Accuracy])\n</pre> trainer_3 = Trainer(model, objective=\"binary\", metrics=[Accuracy]) In\u00a0[19]: Copied! <pre>trainer_3.fit(\n    X_wide=X_wide, X_tab=X_tab, target=target, val_split=0.1, n_epochs=2, batch_size=256\n)\n</pre> trainer_3.fit(     X_wide=X_wide, X_tab=X_tab, target=target, val_split=0.1, n_epochs=2, batch_size=256 ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:03&lt;00:00, 54.23it/s, loss=0.382, metrics={'acc': 0.8239}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 20/20 [00:00&lt;00:00, 84.72it/s, loss=0.331, metrics={'acc': 0.8526}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:03&lt;00:00, 54.35it/s, loss=0.33, metrics={'acc': 0.8465}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 20/20 [00:00&lt;00:00, 68.15it/s, loss=0.312, metrics={'acc': 0.8604}]\n</pre> In\u00a0[20]: Copied! <pre>trainer_3.save(path=\"models_dir\", save_state_dict=True, model_filename=\"model_3.pt\")\n</pre> trainer_3.save(path=\"models_dir\", save_state_dict=True, model_filename=\"model_3.pt\") <p>Now we are going to fine-tune the model components, and in the case of the <code>deeptabular</code> component, we will fine-tune the resnet-blocks and the linear layer but NOT the embeddings.</p> <p>For this, we need to access the model component's children: <code>deeptabular</code> $\\rightarrow$ <code>tab_resnet</code> $\\rightarrow$ <code>dense_resnet</code> $\\rightarrow$ <code>blocks</code></p> In\u00a0[21]: Copied! <pre>wide_3 = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1)\ntab_resnet_3 = TabResnet(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    cat_embed_dropout=0.1,\n    continuous_cols=continuous_cols,\n    blocks_dims=[200, 200, 200],\n)\nmodel_3 = WideDeep(wide=wide, deeptabular=tab_resnet)\n</pre> wide_3 = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1) tab_resnet_3 = TabResnet(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     cat_embed_dropout=0.1,     continuous_cols=continuous_cols,     blocks_dims=[200, 200, 200], ) model_3 = WideDeep(wide=wide, deeptabular=tab_resnet) In\u00a0[22]: Copied! <pre>model_3.load_state_dict(torch.load(\"models_dir/model_3.pt\"))\n</pre> model_3.load_state_dict(torch.load(\"models_dir/model_3.pt\")) Out[22]: <pre>&lt;All keys matched successfully&gt;</pre> In\u00a0[23]: Copied! <pre>model_3\n</pre> model_3 Out[23]: <pre>WideDeep(\n  (wide): Wide(\n    (wide_linear): Embedding(809, 1, padding_idx=0)\n  )\n  (deeptabular): Sequential(\n    (0): TabResnet(\n      (cat_embed): DiffSizeCatEmbeddings(\n        (embed_layers): ModuleDict(\n          (emb_layer_workclass): Embedding(10, 5, padding_idx=0)\n          (emb_layer_education): Embedding(17, 8, padding_idx=0)\n          (emb_layer_marital_status): Embedding(8, 5, padding_idx=0)\n          (emb_layer_occupation): Embedding(16, 7, padding_idx=0)\n          (emb_layer_relationship): Embedding(7, 4, padding_idx=0)\n          (emb_layer_race): Embedding(6, 4, padding_idx=0)\n          (emb_layer_gender): Embedding(3, 2, padding_idx=0)\n          (emb_layer_capital_gain): Embedding(124, 24, padding_idx=0)\n          (emb_layer_capital_loss): Embedding(100, 21, padding_idx=0)\n          (emb_layer_native_country): Embedding(43, 13, padding_idx=0)\n        )\n        (embedding_dropout): Dropout(p=0.1, inplace=False)\n      )\n      (cont_norm): Identity()\n      (encoder): DenseResnet(\n        (dense_resnet): Sequential(\n          (lin_inp): Linear(in_features=95, out_features=200, bias=False)\n          (bn_inp): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n          (block_0): BasicBlock(\n            (lin1): Linear(in_features=200, out_features=200, bias=False)\n            (bn1): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n            (leaky_relu): LeakyReLU(negative_slope=0.01, inplace=True)\n            (dp): Dropout(p=0.1, inplace=False)\n            (lin2): Linear(in_features=200, out_features=200, bias=False)\n            (bn2): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n          )\n          (block_1): BasicBlock(\n            (lin1): Linear(in_features=200, out_features=200, bias=False)\n            (bn1): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n            (leaky_relu): LeakyReLU(negative_slope=0.01, inplace=True)\n            (dp): Dropout(p=0.1, inplace=False)\n            (lin2): Linear(in_features=200, out_features=200, bias=False)\n            (bn2): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=200, out_features=1, bias=True)\n  )\n)</pre> In\u00a0[24]: Copied! <pre>tab_lin_layer = list(model_3.deeptabular.children())[1]\n</pre> tab_lin_layer = list(model_3.deeptabular.children())[1] In\u00a0[25]: Copied! <pre>tab_lin_layer\n</pre> tab_lin_layer Out[25]: <pre>Linear(in_features=200, out_features=1, bias=True)</pre> In\u00a0[26]: Copied! <pre>tab_deep_layers = []\nfor n1, c1 in model_3.deeptabular.named_children():\n    if (\n        n1 == \"0\"\n    ):  # 0 is the model component and 1 is always the prediction layer added by the `WideDeep` class\n        for n2, c2 in c1.named_children():\n            if n2 == \"encoder\":  # TabResnet\n                for _, c3 in c2.named_children():\n                    for n4, c4 in c3.named_children():  # dense_resnet\n                        if \"block\" in n4:\n                            tab_deep_layers.append((n4, c4))\n</pre> tab_deep_layers = [] for n1, c1 in model_3.deeptabular.named_children():     if (         n1 == \"0\"     ):  # 0 is the model component and 1 is always the prediction layer added by the `WideDeep` class         for n2, c2 in c1.named_children():             if n2 == \"encoder\":  # TabResnet                 for _, c3 in c2.named_children():                     for n4, c4 in c3.named_children():  # dense_resnet                         if \"block\" in n4:                             tab_deep_layers.append((n4, c4)) In\u00a0[27]: Copied! <pre>tab_deep_layers\n</pre> tab_deep_layers Out[27]: <pre>[('block_0',\n  BasicBlock(\n    (lin1): Linear(in_features=200, out_features=200, bias=False)\n    (bn1): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n    (leaky_relu): LeakyReLU(negative_slope=0.01, inplace=True)\n    (dp): Dropout(p=0.1, inplace=False)\n    (lin2): Linear(in_features=200, out_features=200, bias=False)\n    (bn2): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n  )),\n ('block_1',\n  BasicBlock(\n    (lin1): Linear(in_features=200, out_features=200, bias=False)\n    (bn1): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n    (leaky_relu): LeakyReLU(negative_slope=0.01, inplace=True)\n    (dp): Dropout(p=0.1, inplace=False)\n    (lin2): Linear(in_features=200, out_features=200, bias=False)\n    (bn2): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n  ))]</pre> <p>Now remember, we need to pass ONLY LAYERS (before I included the name for clarity) the layers in WARM UP ORDER, therefore:</p> In\u00a0[28]: Copied! <pre>tab_deep_layers = [el[1] for el in tab_deep_layers][::-1]\n</pre> tab_deep_layers = [el[1] for el in tab_deep_layers][::-1] In\u00a0[29]: Copied! <pre>tab_layers = [tab_lin_layer] + tab_deep_layers[::-1]\n</pre> tab_layers = [tab_lin_layer] + tab_deep_layers[::-1] In\u00a0[30]: Copied! <pre>tab_layers\n</pre> tab_layers Out[30]: <pre>[Linear(in_features=200, out_features=1, bias=True),\n BasicBlock(\n   (lin1): Linear(in_features=200, out_features=200, bias=False)\n   (bn1): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n   (leaky_relu): LeakyReLU(negative_slope=0.01, inplace=True)\n   (dp): Dropout(p=0.1, inplace=False)\n   (lin2): Linear(in_features=200, out_features=200, bias=False)\n   (bn2): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n ),\n BasicBlock(\n   (lin1): Linear(in_features=200, out_features=200, bias=False)\n   (bn1): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n   (leaky_relu): LeakyReLU(negative_slope=0.01, inplace=True)\n   (dp): Dropout(p=0.1, inplace=False)\n   (lin2): Linear(in_features=200, out_features=200, bias=False)\n   (bn2): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n )]</pre> <p>And now simply</p> In\u00a0[31]: Copied! <pre>trainer_4 = Trainer(model_3, objective=\"binary\", metrics=[Accuracy])\n</pre> trainer_4 = Trainer(model_3, objective=\"binary\", metrics=[Accuracy]) In\u00a0[32]: Copied! <pre>trainer_4.fit(\n    X_wide=X_wide,\n    X_tab=X_tab,\n    target=target,\n    val_split=0.1,\n    finetune=True,\n    finetune_epochs=2,\n    deeptabular_gradual=True,\n    deeptabular_layers=tab_layers,\n    deeptabular_max_lr=0.01,\n    n_epochs=2,\n    batch_size=256,\n)\n</pre> trainer_4.fit(     X_wide=X_wide,     X_tab=X_tab,     target=target,     val_split=0.1,     finetune=True,     finetune_epochs=2,     deeptabular_gradual=True,     deeptabular_layers=tab_layers,     deeptabular_max_lr=0.01,     n_epochs=2,     batch_size=256, ) <pre>Training wide for 2 epochs\n</pre> <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:01&lt;00:00, 95.17it/s, loss=0.504, metrics={'acc': 0.7523}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:01&lt;00:00, 99.83it/s, loss=0.384, metrics={'acc': 0.789}]\n</pre> <pre>Training deeptabular, layer 1 of 3\n</pre> <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:02&lt;00:00, 72.31it/s, loss=0.317, metrics={'acc': 0.8098}]\n</pre> <pre>Training deeptabular, layer 2 of 3\n</pre> <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:02&lt;00:00, 65.97it/s, loss=0.312, metrics={'acc': 0.8214}]\n</pre> <pre>Training deeptabular, layer 3 of 3\n</pre> <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:02&lt;00:00, 63.92it/s, loss=0.306, metrics={'acc': 0.8284}]\n</pre> <pre>Fine-tuning (or warmup) of individual components completed. Training the whole model for 2 epochs\n</pre> <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:03&lt;00:00, 57.26it/s, loss=0.292, metrics={'acc': 0.8664}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 20/20 [00:00&lt;00:00, 84.56it/s, loss=0.292, metrics={'acc': 0.8696}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:03&lt;00:00, 53.61it/s, loss=0.282, metrics={'acc': 0.8693}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 20/20 [00:00&lt;00:00, 80.59it/s, loss=0.289, metrics={'acc': 0.8719}]\n</pre> <p>Finally, there is one more use case I would like to consider. The case where we train only one component and we just want to fine-tune and stop the training afterwards, since there is no joined training. This is a simple as</p> In\u00a0[33]: Copied! <pre>tab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    cat_embed_dropout=0.1,\n    continuous_cols=continuous_cols,\n    mlp_hidden_dims=[64, 32],\n    mlp_dropout=0.2,\n    mlp_activation=\"leaky_relu\",\n)\nmodel = WideDeep(deeptabular=tab_mlp)\n</pre> tab_mlp = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     cat_embed_dropout=0.1,     continuous_cols=continuous_cols,     mlp_hidden_dims=[64, 32],     mlp_dropout=0.2,     mlp_activation=\"leaky_relu\", ) model = WideDeep(deeptabular=tab_mlp) In\u00a0[34]: Copied! <pre>trainer_5 = Trainer(\n    model,\n    objective=\"binary\",\n    optimizers=torch.optim.Adam(model.parameters(), lr=0.01),\n    metrics=[Accuracy],\n)\n</pre> trainer_5 = Trainer(     model,     objective=\"binary\",     optimizers=torch.optim.Adam(model.parameters(), lr=0.01),     metrics=[Accuracy], ) In\u00a0[35]: Copied! <pre>trainer_5.fit(\n    X_wide=X_wide, X_tab=X_tab, target=target, val_split=0.1, n_epochs=1, batch_size=256\n)\n</pre> trainer_5.fit(     X_wide=X_wide, X_tab=X_tab, target=target, val_split=0.1, n_epochs=1, batch_size=256 ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:02&lt;00:00, 73.69it/s, loss=0.365, metrics={'acc': 0.8331}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 20/20 [00:00&lt;00:00, 92.56it/s, loss=0.299, metrics={'acc': 0.8673}]\n</pre> In\u00a0[36]: Copied! <pre>trainer_5.save(path=\"models_dir\", save_state_dict=True, model_filename=\"model_5.pt\")\n</pre> trainer_5.save(path=\"models_dir\", save_state_dict=True, model_filename=\"model_5.pt\") In\u00a0[37]: Copied! <pre>tab_mlp_5 = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    cat_embed_dropout=0.1,\n    continuous_cols=continuous_cols,\n    mlp_hidden_dims=[64, 32],\n    mlp_dropout=0.2,\n    mlp_activation=\"leaky_relu\",\n)\nmodel_5 = WideDeep(deeptabular=tab_mlp_5)\n</pre> tab_mlp_5 = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     cat_embed_dropout=0.1,     continuous_cols=continuous_cols,     mlp_hidden_dims=[64, 32],     mlp_dropout=0.2,     mlp_activation=\"leaky_relu\", ) model_5 = WideDeep(deeptabular=tab_mlp_5) In\u00a0[38]: Copied! <pre>model_5.load_state_dict(torch.load(\"models_dir/model_5.pt\"))\n</pre> model_5.load_state_dict(torch.load(\"models_dir/model_5.pt\")) Out[38]: <pre>&lt;All keys matched successfully&gt;</pre> <p>...times go by...</p> In\u00a0[39]: Copied! <pre>trainer_6 = Trainer(\n    model_5,\n    objective=\"binary\",\n    optimizers=torch.optim.Adam(model.parameters(), lr=0.01),\n    metrics=[Accuracy],\n)\n</pre> trainer_6 = Trainer(     model_5,     objective=\"binary\",     optimizers=torch.optim.Adam(model.parameters(), lr=0.01),     metrics=[Accuracy], ) In\u00a0[40]: Copied! <pre>trainer_6.fit(\n    X_wide=X_wide,\n    X_tab=X_tab,\n    target=target,\n    val_split=0.1,\n    finetune=True,\n    finetune_epochs=2,\n    finetune_max_lr=0.01,\n    stop_after_finetuning=True,\n    batch_size=256,\n)\n</pre> trainer_6.fit(     X_wide=X_wide,     X_tab=X_tab,     target=target,     val_split=0.1,     finetune=True,     finetune_epochs=2,     finetune_max_lr=0.01,     stop_after_finetuning=True,     batch_size=256, ) <pre>Training deeptabular for 2 epochs\n</pre> <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:02&lt;00:00, 73.86it/s, loss=0.298, metrics={'acc': 0.8652}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:02&lt;00:00, 75.45it/s, loss=0.286, metrics={'acc': 0.8669}]\n</pre> <pre>Fine-tuning (or warmup) of individual components completed. Training the whole model for 1 epochs\n</pre> <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:02&lt;00:00, 76.29it/s, loss=0.282, metrics={'acc': 0.8698}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 20/20 [00:00&lt;00:00, 84.93it/s, loss=0.281, metrics={'acc': 0.8749}]\n</pre> In\u00a0[42]: Copied! <pre>import shutil\n\nshutil.rmtree(\"models_dir/\")\nshutil.rmtree(\"model_weights/\")\n</pre> import shutil  shutil.rmtree(\"models_dir/\") shutil.rmtree(\"model_weights/\") In\u00a0[\u00a0]: Copied! <pre>\n</pre>"},{"location":"examples/06_finetune_and_warmup.html#the-finetunewarm-up-option","title":"The FineTune/Warm Up option\u00b6","text":"<p>Let's place ourselves in two possible scenarios.</p> <ol> <li><p>Let's assume we have run a model and we want to just transfer the learnings (you know...transfer-learning) to another dataset, or simply we have received new data and we do not want to start the training of each component from scratch. Simply, we want to load the pre-trained weights and fine-tune.</p> </li> <li><p>We just want to \"warm up\" individual model components individually before the joined training begins.</p> </li> </ol> <p>This can be done with the <code>finetune</code> set of parameters. There are 3 fine-tuning routines:</p> <ol> <li>Fine-tune all trainable layers at once with a triangular one-cycle learning rate (referred as slanted triangular learning rates in Howard &amp; Ruder 2018)</li> <li>Gradual fine-tuning inspired by the work of Felbo et al., 2017</li> <li>Gradual fine-tuning based on the work of Howard &amp; Ruder 2018</li> </ol> <p>Currently fine-tunning is only supported without a fully connected head, i.e. if <code>deephead=None</code>. In addition, <code>Felbo</code> and <code>Howard</code> routines only applied, of course, to the <code>deeptabular</code>, <code>deeptext</code> and <code>deepimage</code> models. The <code>wide</code> component can also be fine-tuned, but only in an \"all at once\" mode.</p>"},{"location":"examples/06_finetune_and_warmup.html#fine-tune-or-warm-up-all-at-once","title":"Fine-tune or warm-up all at once\u00b6","text":"<p>Here, the model components will be trained for <code>finetune_epochs</code> using a triangular one-cycle learning rate (slanted triangular learning rate) ranging from <code>finetune_max_lr/10</code> to <code>finetune_max_lr</code> (default is 0.01). 10% of the training steps are used to increase the learning rate which then decreases for the remaining 90%.</p> <p>Here all trainable layers are fine-tuned.</p> <p>Let's have a look to one example.</p>"},{"location":"examples/06_finetune_and_warmup.html#fine-tune-gradually-the-felbo-and-the-howard-routines","title":"Fine-tune Gradually: The \"felbo\"  and the \"howard\" routines\u00b6","text":"<p>The Felbo routine can be illustrated as follows:</p> <p> </p> <p>Figure 1. The figure can be described as follows: fine-tune (or train) the last layer for one epoch using a one cycle triangular learning rate. Then fine-tune the next deeper layer for one epoch, with a learning rate that is a factor of 2.5 lower than the previous learning rate (the 2.5 factor is fixed) while freezing the already warmed up layer(s). Repeat untill all individual layers are warmed. Then warm one last epoch with all warmed layers trainable. The vanishing color gradient in the figure attempts to illustrate the decreasing learning rate.</p> <p>Note that this is not identical to the Fine-Tunning routine described in Felbo et al, 2017, this is why I used the word 'inspired'.</p> <p>The Howard routine can be illustrated as follows:</p> <p> </p> <p>Figure 2. The figure can be described as follows: fine-tune (or train) the last layer for one epoch using a one cycle triangular learning rate. Then fine-tune the next deeper layer for one epoch, with a learning rate that is a factor of 2.5 lower than the previous learning rate (the 2.5 factor is fixed) while keeping the already warmed up layer(s) trainable. Repeat. The vanishing color gradient in the figure attempts to illustrate the decreasing learning rate.</p> <p>Note that I write \"fine-tune (or train) the last layer for one epoch [...]\". However, in practice the user will have to specify the order of the layers to be fine-tuned. This is another reason why I wrote that the fine-tune routines I have implemented are inspired by the work of Felbo and Howard and not identical to their implemenations.</p> <p>The <code>felbo</code> and <code>howard</code> routines can be accessed with via the <code>fine-tune</code> parameters.</p>"},{"location":"examples/07_custom_components.html","title":"07_custom_components","text":"In\u00a0[1]: Copied! <pre>import numpy as np\nimport pandas as pd\nimport os\nimport torch\n\nfrom torch import Tensor\nfrom pytorch_widedeep import Trainer\nfrom pytorch_widedeep.preprocessing import (\n    WidePreprocessor,\n    TabPreprocessor,\n    TextPreprocessor,\n    ImagePreprocessor,\n)\nfrom pytorch_widedeep.models import (\n    Wide,\n    TabMlp,\n    Vision,\n    BasicRNN,\n    WideDeep,\n)\nfrom pytorch_widedeep.losses import RMSELoss\nfrom pytorch_widedeep.initializers import *\nfrom pytorch_widedeep.callbacks import *\nfrom pytorch_widedeep.datasets import load_adult\n</pre> import numpy as np import pandas as pd import os import torch  from torch import Tensor from pytorch_widedeep import Trainer from pytorch_widedeep.preprocessing import (     WidePreprocessor,     TabPreprocessor,     TextPreprocessor,     ImagePreprocessor, ) from pytorch_widedeep.models import (     Wide,     TabMlp,     Vision,     BasicRNN,     WideDeep, ) from pytorch_widedeep.losses import RMSELoss from pytorch_widedeep.initializers import * from pytorch_widedeep.callbacks import * from pytorch_widedeep.datasets import load_adult <pre>/Users/javierrodriguezzaurin/.pyenv/versions/3.10.13/envs/widedeep310/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n  from .autonotebook import tqdm as notebook_tqdm\n</pre> In\u00a0[2]: Copied! <pre>df = pd.read_csv(\"../tmp_data/airbnb/airbnb_sample.csv\")\ndf.head()\n</pre> df = pd.read_csv(\"../tmp_data/airbnb/airbnb_sample.csv\") df.head() Out[2]: id host_id description host_listings_count host_identity_verified neighbourhood_cleansed latitude longitude is_location_exact property_type ... amenity_wide_entrance amenity_wide_entrance_for_guests amenity_wide_entryway amenity_wide_hallways amenity_wifi amenity_window_guards amenity_wine_cooler security_deposit extra_people yield 0 13913.jpg 54730 My bright double bedroom with a large window h... 4.0 f Islington 51.56802 -0.11121 t apartment ... 1 0 0 0 1 0 0 100.0 15.0 12.00 1 15400.jpg 60302 Lots of windows and light.  St Luke's Gardens ... 1.0 t Kensington and Chelsea 51.48796 -0.16898 t apartment ... 0 0 0 0 1 0 0 150.0 0.0 109.50 2 17402.jpg 67564 Open from June 2018 after a 3-year break, we a... 19.0 t Westminster 51.52098 -0.14002 t apartment ... 0 0 0 0 1 0 0 350.0 10.0 149.65 3 24328.jpg 41759 Artist house, bright high ceiling rooms, priva... 2.0 t Wandsworth 51.47298 -0.16376 t other ... 0 0 0 0 1 0 0 250.0 0.0 215.60 4 25023.jpg 102813 Large, all comforts, 2-bed flat; first floor; ... 1.0 f Wandsworth 51.44687 -0.21874 t apartment ... 0 0 0 0 1 0 0 250.0 11.0 79.35 <p>5 rows \u00d7 223 columns</p> In\u00a0[3]: Copied! <pre># There are a number of columns that are already binary. Therefore, no need to one hot encode them\ncrossed_cols = [(\"property_type\", \"room_type\")]\nalready_dummies = [c for c in df.columns if \"amenity\" in c] + [\"has_house_rules\"]\nwide_cols = [\n    \"is_location_exact\",\n    \"property_type\",\n    \"room_type\",\n    \"host_gender\",\n    \"instant_bookable\",\n] + already_dummies\n\ncat_embed_cols = [(c, 16) for c in df.columns if \"catg\" in c] + [\n    (\"neighbourhood_cleansed\", 64),\n    (\"cancellation_policy\", 16),\n]\ncontinuous_cols = [\"latitude\", \"longitude\", \"security_deposit\", \"extra_people\"]\n# it does not make sense to standarised Latitude and Longitude\nalready_standard = [\"latitude\", \"longitude\"]\n\n# text and image colnames\ntext_col = \"description\"\nimg_col = \"id\"\n\n# path to pretrained word embeddings and the images\nword_vectors_path = \"../tmp_data/glove.6B/glove.6B.100d.txt\"\nimg_path = \"../tmp_data/airbnb/property_picture\"\n\n# target\ntarget_col = \"yield\"\n</pre> # There are a number of columns that are already binary. Therefore, no need to one hot encode them crossed_cols = [(\"property_type\", \"room_type\")] already_dummies = [c for c in df.columns if \"amenity\" in c] + [\"has_house_rules\"] wide_cols = [     \"is_location_exact\",     \"property_type\",     \"room_type\",     \"host_gender\",     \"instant_bookable\", ] + already_dummies  cat_embed_cols = [(c, 16) for c in df.columns if \"catg\" in c] + [     (\"neighbourhood_cleansed\", 64),     (\"cancellation_policy\", 16), ] continuous_cols = [\"latitude\", \"longitude\", \"security_deposit\", \"extra_people\"] # it does not make sense to standarised Latitude and Longitude already_standard = [\"latitude\", \"longitude\"]  # text and image colnames text_col = \"description\" img_col = \"id\"  # path to pretrained word embeddings and the images word_vectors_path = \"../tmp_data/glove.6B/glove.6B.100d.txt\" img_path = \"../tmp_data/airbnb/property_picture\"  # target target_col = \"yield\" In\u00a0[4]: Copied! <pre>target = df[target_col].values\n</pre> target = df[target_col].values In\u00a0[5]: Copied! <pre>wide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols)\nX_wide = wide_preprocessor.fit_transform(df)\n\ntab_preprocessor = TabPreprocessor(\n    embed_cols=cat_embed_cols, continuous_cols=continuous_cols\n)\nX_tab = tab_preprocessor.fit_transform(df)\n\ntext_preprocessor = TextPreprocessor(\n    word_vectors_path=word_vectors_path, text_col=text_col\n)\nX_text = text_preprocessor.fit_transform(df)\n\nimage_processor = ImagePreprocessor(img_col=img_col, img_path=img_path)\nX_images = image_processor.fit_transform(df)\n</pre> wide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols) X_wide = wide_preprocessor.fit_transform(df)  tab_preprocessor = TabPreprocessor(     embed_cols=cat_embed_cols, continuous_cols=continuous_cols ) X_tab = tab_preprocessor.fit_transform(df)  text_preprocessor = TextPreprocessor(     word_vectors_path=word_vectors_path, text_col=text_col ) X_text = text_preprocessor.fit_transform(df)  image_processor = ImagePreprocessor(img_col=img_col, img_path=img_path) X_images = image_processor.fit_transform(df) <pre>/Users/javierrodriguezzaurin/Projects/pytorch-widedeep/pytorch_widedeep/preprocessing/tab_preprocessor.py:358: UserWarning: Continuous columns will not be normalised\n  warnings.warn(\"Continuous columns will not be normalised\")\n</pre> <pre>The vocabulary contains 2192 tokens\nIndexing word vectors...\nLoaded 400000 word vectors\nPreparing embeddings matrix...\n2175 words in the vocabulary had ../tmp_data/glove.6B/glove.6B.100d.txt vectors and appear more than 5 times\nReading Images from ../tmp_data/airbnb/property_picture\nResizing\n</pre> <pre>100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 1001/1001 [00:02&lt;00:00, 497.80it/s]\n</pre> <pre>Computing normalisation metrics\n</pre> <p>Now we are ready to build a wide and deep model. Three of the four components we will use are included in this package, and they will be combined with a custom <code>deeptext</code> component. Then the fit process will run with a custom loss function.</p> <p>Let's have a look</p> In\u00a0[6]: Copied! <pre># Linear model\nwide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1)\n\n# DeepDense: 2 Dense layers\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    cat_embed_dropout=0.1,\n    continuous_cols=continuous_cols,\n    mlp_hidden_dims=[128, 64],\n    mlp_dropout=0.1,\n)\n\n# Pretrained Resnet 18\nresnet = Vision(pretrained_model_name=\"resnet18\", n_trainable=0)\n</pre> # Linear model wide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1)  # DeepDense: 2 Dense layers tab_mlp = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     cat_embed_dropout=0.1,     continuous_cols=continuous_cols,     mlp_hidden_dims=[128, 64],     mlp_dropout=0.1, )  # Pretrained Resnet 18 resnet = Vision(pretrained_model_name=\"resnet18\", n_trainable=0) In\u00a0[7]: Copied! <pre>class MyDeepText(nn.Module):\n    def __init__(self, vocab_size, padding_idx=1, embed_dim=100, hidden_dim=64):\n        super(MyDeepText, self).__init__()\n\n        # word/token embeddings\n        self.word_embed = nn.Embedding(vocab_size, embed_dim, padding_idx=padding_idx)\n\n        # stack of RNNs\n        self.rnn = nn.GRU(\n            embed_dim,\n            hidden_dim,\n            num_layers=2,\n            bidirectional=True,\n            batch_first=True,\n        )\n\n        # Remember, this MUST be defined. If not WideDeep will through an error\n        self.output_dim = hidden_dim * 2\n\n    def forward(self, X):\n        embed = self.word_embed(X.long())\n        o, h = self.rnn(embed)\n        return torch.cat((h[-2], h[-1]), dim=1)\n</pre> class MyDeepText(nn.Module):     def __init__(self, vocab_size, padding_idx=1, embed_dim=100, hidden_dim=64):         super(MyDeepText, self).__init__()          # word/token embeddings         self.word_embed = nn.Embedding(vocab_size, embed_dim, padding_idx=padding_idx)          # stack of RNNs         self.rnn = nn.GRU(             embed_dim,             hidden_dim,             num_layers=2,             bidirectional=True,             batch_first=True,         )          # Remember, this MUST be defined. If not WideDeep will through an error         self.output_dim = hidden_dim * 2      def forward(self, X):         embed = self.word_embed(X.long())         o, h = self.rnn(embed)         return torch.cat((h[-2], h[-1]), dim=1) In\u00a0[8]: Copied! <pre>mydeeptext = MyDeepText(vocab_size=len(text_preprocessor.vocab.itos))\n</pre> mydeeptext = MyDeepText(vocab_size=len(text_preprocessor.vocab.itos)) In\u00a0[9]: Copied! <pre>model = WideDeep(wide=wide, deeptabular=tab_mlp, deeptext=mydeeptext, deepimage=resnet)\n</pre> model = WideDeep(wide=wide, deeptabular=tab_mlp, deeptext=mydeeptext, deepimage=resnet) In\u00a0[10]: Copied! <pre>class RMSELoss(nn.Module):\n    def __init__(self):\n        \"\"\"root mean squared error\"\"\"\n        super().__init__()\n        self.mse = nn.MSELoss()\n\n    def forward(self, input: Tensor, target: Tensor) -&gt; Tensor:\n        return torch.sqrt(self.mse(input, target))\n</pre> class RMSELoss(nn.Module):     def __init__(self):         \"\"\"root mean squared error\"\"\"         super().__init__()         self.mse = nn.MSELoss()      def forward(self, input: Tensor, target: Tensor) -&gt; Tensor:         return torch.sqrt(self.mse(input, target)) <p>and now we just instantiate the <code>Trainer</code> as usual. Needless to say, but this runs with 1000 random observations, so loss and metric values are meaningless. This is just an example</p> In\u00a0[11]: Copied! <pre>trainer = Trainer(model, objective=\"regression\", custom_loss_function=RMSELoss())\n</pre> trainer = Trainer(model, objective=\"regression\", custom_loss_function=RMSELoss()) In\u00a0[12]: Copied! <pre>trainer.fit(\n    X_wide=X_wide,\n    X_tab=X_tab,\n    X_text=X_text,\n    X_img=X_images,\n    target=target,\n    n_epochs=1,\n    batch_size=32,\n    val_split=0.2,\n)\n</pre> trainer.fit(     X_wide=X_wide,     X_tab=X_tab,     X_text=X_text,     X_img=X_images,     target=target,     n_epochs=1,     batch_size=32,     val_split=0.2, ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 25/25 [00:23&lt;00:00,  1.07it/s, loss=126]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 7/7 [00:05&lt;00:00,  1.24it/s, loss=97.4]\n</pre> <p>In addition to model components and loss functions, we can also use custom callbacks or custom metrics. The former need to be of type <code>Callback</code> and the latter need to be of type <code>Metric</code>. See:</p> <pre>pytorch-widedeep.callbacks\n</pre> <p>and</p> <pre>pytorch-widedeep.metrics\n</pre> <p>For this example let me use the adult dataset. Again, we first prepare the data as usual</p> In\u00a0[13]: Copied! <pre>df = load_adult(as_frame=True)\ndf.head()\n</pre> df = load_adult(as_frame=True) df.head() Out[13]: age workclass fnlwgt education educational-num marital-status occupation relationship race gender capital-gain capital-loss hours-per-week native-country income 0 25 Private 226802 11th 7 Never-married Machine-op-inspct Own-child Black Male 0 0 40 United-States &lt;=50K 1 38 Private 89814 HS-grad 9 Married-civ-spouse Farming-fishing Husband White Male 0 0 50 United-States &lt;=50K 2 28 Local-gov 336951 Assoc-acdm 12 Married-civ-spouse Protective-serv Husband White Male 0 0 40 United-States &gt;50K 3 44 Private 160323 Some-college 10 Married-civ-spouse Machine-op-inspct Husband Black Male 7688 0 40 United-States &gt;50K 4 18 ? 103497 Some-college 10 Never-married ? Own-child White Female 0 0 30 United-States &lt;=50K In\u00a0[14]: Copied! <pre># For convenience, we'll replace '-' with '_'\ndf.columns = [c.replace(\"-\", \"_\") for c in df.columns]\n# binary target\ndf[\"income_label\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int)\ndf.drop(\"income\", axis=1, inplace=True)\ndf.head()\n</pre> # For convenience, we'll replace '-' with '_' df.columns = [c.replace(\"-\", \"_\") for c in df.columns] # binary target df[\"income_label\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int) df.drop(\"income\", axis=1, inplace=True) df.head() Out[14]: age workclass fnlwgt education educational_num marital_status occupation relationship race gender capital_gain capital_loss hours_per_week native_country income_label 0 25 Private 226802 11th 7 Never-married Machine-op-inspct Own-child Black Male 0 0 40 United-States 0 1 38 Private 89814 HS-grad 9 Married-civ-spouse Farming-fishing Husband White Male 0 0 50 United-States 0 2 28 Local-gov 336951 Assoc-acdm 12 Married-civ-spouse Protective-serv Husband White Male 0 0 40 United-States 1 3 44 Private 160323 Some-college 10 Married-civ-spouse Machine-op-inspct Husband Black Male 7688 0 40 United-States 1 4 18 ? 103497 Some-college 10 Never-married ? Own-child White Female 0 0 30 United-States 0 In\u00a0[15]: Copied! <pre># Define wide, crossed and deep tabular columns\nwide_cols = [\n    \"workclass\",\n    \"education\",\n    \"marital_status\",\n    \"occupation\",\n    \"relationship\",\n    \"race\",\n    \"gender\",\n    \"native_country\",\n]\ncrossed_cols = [(\"education\", \"occupation\"), (\"native_country\", \"occupation\")]\ncat_embed_cols = [\n    \"workclass\",\n    \"education\",\n    \"marital_status\",\n    \"occupation\",\n    \"relationship\",\n    \"race\",\n    \"gender\",\n    \"capital_gain\",\n    \"capital_loss\",\n    \"native_country\",\n]\ncontinuous_cols = [\"age\", \"hours_per_week\"]\ntarget_col = \"income_label\"\ntarget = df[target_col].values\n</pre> # Define wide, crossed and deep tabular columns wide_cols = [     \"workclass\",     \"education\",     \"marital_status\",     \"occupation\",     \"relationship\",     \"race\",     \"gender\",     \"native_country\", ] crossed_cols = [(\"education\", \"occupation\"), (\"native_country\", \"occupation\")] cat_embed_cols = [     \"workclass\",     \"education\",     \"marital_status\",     \"occupation\",     \"relationship\",     \"race\",     \"gender\",     \"capital_gain\",     \"capital_loss\",     \"native_country\", ] continuous_cols = [\"age\", \"hours_per_week\"] target_col = \"income_label\" target = df[target_col].values In\u00a0[16]: Copied! <pre># wide\nwide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols)\nX_wide = wide_preprocessor.fit_transform(df)\n\n# deeptabular\ntab_preprocessor = TabPreprocessor(\n    embed_cols=cat_embed_cols, continuous_cols=continuous_cols\n)\nX_tab = tab_preprocessor.fit_transform(df)\n</pre> # wide wide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols) X_wide = wide_preprocessor.fit_transform(df)  # deeptabular tab_preprocessor = TabPreprocessor(     embed_cols=cat_embed_cols, continuous_cols=continuous_cols ) X_tab = tab_preprocessor.fit_transform(df) <pre>/Users/javierrodriguezzaurin/Projects/pytorch-widedeep/pytorch_widedeep/preprocessing/tab_preprocessor.py:358: UserWarning: Continuous columns will not be normalised\n  warnings.warn(\"Continuous columns will not be normalised\")\n</pre> In\u00a0[17]: Copied! <pre>wide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1)\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=continuous_cols,\n    mlp_hidden_dims=[128, 64],\n    mlp_dropout=0.2,\n    mlp_activation=\"leaky_relu\",\n)\nmodel = WideDeep(wide=wide, deeptabular=tab_mlp)\n</pre> wide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1) tab_mlp = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     continuous_cols=continuous_cols,     mlp_hidden_dims=[128, 64],     mlp_dropout=0.2,     mlp_activation=\"leaky_relu\", ) model = WideDeep(wide=wide, deeptabular=tab_mlp) In\u00a0[18]: Copied! <pre>from pytorch_widedeep.metrics import Metric\n</pre> from pytorch_widedeep.metrics import Metric In\u00a0[19]: Copied! <pre>class Accuracy(Metric):\n    def __init__(self, top_k: int = 1):\n        super(Accuracy, self).__init__()\n\n        self.top_k = top_k\n        self.correct_count = 0\n        self.total_count = 0\n\n        # \u00a0metric name needs to be defined\n        self._name = \"acc\"\n\n    def reset(self):\n        self.correct_count = 0\n        self.total_count = 0\n\n    def __call__(self, y_pred: Tensor, y_true: Tensor) -&gt; np.ndarray:\n        num_classes = y_pred.size(1)\n\n        if num_classes == 1:\n            y_pred = y_pred.round()\n            y_true = y_true\n        elif num_classes &gt; 1:\n            y_pred = y_pred.topk(self.top_k, 1)[1]\n            y_true = y_true.view(-1, 1).expand_as(y_pred)\n\n        self.correct_count += y_pred.eq(y_true).sum().item()\n        self.total_count += len(y_pred)\n        accuracy = float(self.correct_count) / float(self.total_count)\n        return np.array(accuracy)\n</pre> class Accuracy(Metric):     def __init__(self, top_k: int = 1):         super(Accuracy, self).__init__()          self.top_k = top_k         self.correct_count = 0         self.total_count = 0          # \u00a0metric name needs to be defined         self._name = \"acc\"      def reset(self):         self.correct_count = 0         self.total_count = 0      def __call__(self, y_pred: Tensor, y_true: Tensor) -&gt; np.ndarray:         num_classes = y_pred.size(1)          if num_classes == 1:             y_pred = y_pred.round()             y_true = y_true         elif num_classes &gt; 1:             y_pred = y_pred.topk(self.top_k, 1)[1]             y_true = y_true.view(-1, 1).expand_as(y_pred)          self.correct_count += y_pred.eq(y_true).sum().item()         self.total_count += len(y_pred)         accuracy = float(self.correct_count) / float(self.total_count)         return np.array(accuracy) In\u00a0[20]: Copied! <pre># have a look to the class\nfrom pytorch_widedeep.callbacks import Callback\n</pre> # have a look to the class from pytorch_widedeep.callbacks import Callback In\u00a0[21]: Copied! <pre>class SillyCallback(Callback):\n    def on_train_begin(self, logs=None):\n        # recordings will be the trainer object attributes\n        self.trainer.silly_callback = {}\n\n        self.trainer.silly_callback[\"beginning\"] = []\n        self.trainer.silly_callback[\"end\"] = []\n\n    def on_epoch_begin(self, epoch, logs=None):\n        self.trainer.silly_callback[\"beginning\"].append(epoch + 1)\n\n    def on_epoch_end(self, epoch, logs=None, metric=None):\n        self.trainer.silly_callback[\"end\"].append(epoch + 1)\n</pre> class SillyCallback(Callback):     def on_train_begin(self, logs=None):         # recordings will be the trainer object attributes         self.trainer.silly_callback = {}          self.trainer.silly_callback[\"beginning\"] = []         self.trainer.silly_callback[\"end\"] = []      def on_epoch_begin(self, epoch, logs=None):         self.trainer.silly_callback[\"beginning\"].append(epoch + 1)      def on_epoch_end(self, epoch, logs=None, metric=None):         self.trainer.silly_callback[\"end\"].append(epoch + 1) <p>and now, as usual:</p> In\u00a0[22]: Copied! <pre>trainer = Trainer(\n    model, objective=\"binary\", metrics=[Accuracy], callbacks=[SillyCallback]\n)\n</pre> trainer = Trainer(     model, objective=\"binary\", metrics=[Accuracy], callbacks=[SillyCallback] ) In\u00a0[23]: Copied! <pre>trainer.fit(\n    X_wide=X_wide, X_tab=X_tab, target=target, n_epochs=5, batch_size=64, val_split=0.2\n)\n</pre> trainer.fit(     X_wide=X_wide, X_tab=X_tab, target=target, n_epochs=5, batch_size=64, val_split=0.2 ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 611/611 [00:06&lt;00:00, 94.39it/s, loss=0.411, metrics={'acc': 0.814}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 121.91it/s, loss=0.327, metrics={'acc': 0.8449}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 611/611 [00:07&lt;00:00, 85.39it/s, loss=0.324, metrics={'acc': 0.8495}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 88.68it/s, loss=0.298, metrics={'acc': 0.8612}]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 611/611 [00:08&lt;00:00, 74.35it/s, loss=0.302, metrics={'acc': 0.8593}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 100.51it/s, loss=0.29, metrics={'acc': 0.8665}]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 611/611 [00:08&lt;00:00, 73.83it/s, loss=0.292, metrics={'acc': 0.8637}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 105.98it/s, loss=0.286, metrics={'acc': 0.8695}]\nepoch 5: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 611/611 [00:08&lt;00:00, 72.15it/s, loss=0.286, metrics={'acc': 0.866}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 92.27it/s, loss=0.284, metrics={'acc': 0.8698}]\n</pre> In\u00a0[24]: Copied! <pre>trainer.silly_callback\n</pre> trainer.silly_callback Out[24]: <pre>{'beginning': [1, 2, 3, 4, 5], 'end': [1, 2, 3, 4, 5]}</pre>"},{"location":"examples/07_custom_components.html#custom-components","title":"Custom components\u00b6","text":"<p>As I mentioned earlier in the example notebooks, and also in the <code>README</code>, it is possible to customise almost every component in <code>pytorch-widedeep</code>.</p> <p>Let's now go through a couple of simple examples to illustrate how that could be done.</p> <p>First let's load and process the data \"as usual\", let's start with a regression and the airbnb dataset.</p>"},{"location":"examples/07_custom_components.html#custom-deeptext","title":"Custom <code>deeptext</code>\u00b6","text":"<p>Standard Pytorch model</p>"},{"location":"examples/07_custom_components.html#custom-loss-function","title":"Custom loss function\u00b6","text":"<p>Loss functions must simply inherit pytorch's <code>nn.Module</code>. For example, let's say we want to use <code>RMSE</code> (note that this is already available in the package, but I will pass it here as a custom loss for illustration purposes)</p>"},{"location":"examples/07_custom_components.html#custom-metric","title":"Custom metric\u00b6","text":"<p>Let's say we want to use our own accuracy metric (again, this is already available in the package, but I will pass it here as a custom loss for illustration purposes).</p> <p>This could be done as:</p>"},{"location":"examples/07_custom_components.html#custom-callback","title":"Custom Callback\u00b6","text":"<p>Let's code a callback that records the current epoch at the beginning and the end of each epoch (silly, but you know, this is just an example)</p>"},{"location":"examples/08_custom_dataLoader_imbalanced_dataset.html","title":"08_custom_dataLoader_imbalanced_dataset","text":"<ul> <li>In this notebook we will use the higly imbalanced Protein Homology Dataset from KDD cup 2004</li> </ul> <pre><code>* The first element of each line is a BLOCK ID that denotes to which native sequence this example belongs. There is a unique BLOCK ID for each native sequence. BLOCK IDs are integers running from 1 to 303 (one for each native sequence, i.e. for each query). BLOCK IDs were assigned before the blocks were split into the train and test sets, so they do not run consecutively in either file.\n* The second element of each line is an EXAMPLE ID that uniquely describes the example. You will need this EXAMPLE ID and the BLOCK ID when you submit results.\n* The third element is the class of the example. Proteins that are homologous to the native sequence are denoted by 1, non-homologous proteins (i.e. decoys) by 0. Test examples have a \"?\" in this position.\n* All following elements are feature values. There are 74 feature values in each line. The features describe the match (e.g. the score of a sequence alignment) between the native protein sequence and the sequence that is tested for homology.\n</code></pre> In\u00a0[1]: Copied! <pre>import numpy as np\nimport pandas as pd\nimport torch\nfrom torch.optim import SGD, lr_scheduler\n\nfrom pytorch_widedeep import Trainer\nfrom pytorch_widedeep.preprocessing import TabPreprocessor\nfrom pytorch_widedeep.models import TabMlp, WideDeep\nfrom pytorch_widedeep.dataloaders import DataLoaderImbalanced, DataLoaderDefault\nfrom torchmetrics import F1Score as F1_torchmetrics\nfrom torchmetrics import Accuracy as Accuracy_torchmetrics\nfrom torchmetrics import Precision as Precision_torchmetrics\nfrom torchmetrics import Recall as Recall_torchmetrics\nfrom pytorch_widedeep.metrics import Accuracy, Recall, Precision, F1Score, R2Score\nfrom pytorch_widedeep.initializers import XavierNormal\nfrom pytorch_widedeep.datasets import load_bio_kdd04\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import classification_report\n\nimport time\nimport datetime\n\nimport warnings\n\nwarnings.filterwarnings(\"ignore\", category=DeprecationWarning)\n\n# increase displayed columns in jupyter notebook\npd.set_option(\"display.max_columns\", 200)\npd.set_option(\"display.max_rows\", 300)\n</pre> import numpy as np import pandas as pd import torch from torch.optim import SGD, lr_scheduler  from pytorch_widedeep import Trainer from pytorch_widedeep.preprocessing import TabPreprocessor from pytorch_widedeep.models import TabMlp, WideDeep from pytorch_widedeep.dataloaders import DataLoaderImbalanced, DataLoaderDefault from torchmetrics import F1Score as F1_torchmetrics from torchmetrics import Accuracy as Accuracy_torchmetrics from torchmetrics import Precision as Precision_torchmetrics from torchmetrics import Recall as Recall_torchmetrics from pytorch_widedeep.metrics import Accuracy, Recall, Precision, F1Score, R2Score from pytorch_widedeep.initializers import XavierNormal from pytorch_widedeep.datasets import load_bio_kdd04  from sklearn.model_selection import train_test_split from sklearn.metrics import classification_report  import time import datetime  import warnings  warnings.filterwarnings(\"ignore\", category=DeprecationWarning)  # increase displayed columns in jupyter notebook pd.set_option(\"display.max_columns\", 200) pd.set_option(\"display.max_rows\", 300) <pre>/Users/javierrodriguezzaurin/.pyenv/versions/3.10.13/envs/widedeep310/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n  from .autonotebook import tqdm as notebook_tqdm\n</pre> In\u00a0[2]: Copied! <pre>df = load_bio_kdd04(as_frame=True)\n# drop columns we won't need in this example\ndf.drop(columns=[\"EXAMPLE_ID\", \"BLOCK_ID\"], inplace=True)\n\ndf_train, df_valid = train_test_split(\n    df, test_size=0.2, stratify=df[\"target\"], random_state=1\n)\ndf_valid, df_test = train_test_split(\n    df_valid, test_size=0.5, stratify=df_valid[\"target\"], random_state=1\n)\n\ncontinuous_cols = df.drop(columns=[\"target\"]).columns.values.tolist()\n</pre> df = load_bio_kdd04(as_frame=True) # drop columns we won't need in this example df.drop(columns=[\"EXAMPLE_ID\", \"BLOCK_ID\"], inplace=True)  df_train, df_valid = train_test_split(     df, test_size=0.2, stratify=df[\"target\"], random_state=1 ) df_valid, df_test = train_test_split(     df_valid, test_size=0.5, stratify=df_valid[\"target\"], random_state=1 )  continuous_cols = df.drop(columns=[\"target\"]).columns.values.tolist() In\u00a0[3]: Copied! <pre># deeptabular\ntab_preprocessor = TabPreprocessor(continuous_cols=continuous_cols, scale=True)\nX_tab_train = tab_preprocessor.fit_transform(df_train)\nX_tab_valid = tab_preprocessor.transform(df_valid)\nX_tab_test = tab_preprocessor.transform(df_test)\n\n# target\ny_train = df_train[\"target\"].values\ny_valid = df_valid[\"target\"].values\ny_test = df_test[\"target\"].values\n</pre> # deeptabular tab_preprocessor = TabPreprocessor(continuous_cols=continuous_cols, scale=True) X_tab_train = tab_preprocessor.fit_transform(df_train) X_tab_valid = tab_preprocessor.transform(df_valid) X_tab_test = tab_preprocessor.transform(df_test)  # target y_train = df_train[\"target\"].values y_valid = df_valid[\"target\"].values y_test = df_test[\"target\"].values In\u00a0[4]: Copied! <pre># Define the model\ninput_layer = len(tab_preprocessor.continuous_cols)\noutput_layer = 1\nhidden_layers = np.linspace(\n    input_layer * 2, output_layer, 5, endpoint=False, dtype=int\n).tolist()\n\ndeeptabular = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    continuous_cols=tab_preprocessor.continuous_cols,\n    mlp_hidden_dims=hidden_layers,\n)\nmodel = WideDeep(deeptabular=deeptabular)\nmodel\n</pre> # Define the model input_layer = len(tab_preprocessor.continuous_cols) output_layer = 1 hidden_layers = np.linspace(     input_layer * 2, output_layer, 5, endpoint=False, dtype=int ).tolist()  deeptabular = TabMlp(     column_idx=tab_preprocessor.column_idx,     continuous_cols=tab_preprocessor.continuous_cols,     mlp_hidden_dims=hidden_layers, ) model = WideDeep(deeptabular=deeptabular) model Out[4]: <pre>WideDeep(\n  (deeptabular): Sequential(\n    (0): TabMlp(\n      (cont_norm): Identity()\n      (encoder): MLP(\n        (mlp): Sequential(\n          (dense_layer_0): Sequential(\n            (0): Linear(in_features=74, out_features=148, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_1): Sequential(\n            (0): Linear(in_features=148, out_features=118, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_2): Sequential(\n            (0): Linear(in_features=118, out_features=89, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_3): Sequential(\n            (0): Linear(in_features=89, out_features=59, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_4): Sequential(\n            (0): Linear(in_features=59, out_features=30, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=30, out_features=1, bias=True)\n  )\n)</pre> In\u00a0[5]: Copied! <pre># Metrics from pytorch-widedeep\naccuracy = Accuracy(top_k=2)\nprecision = Precision(average=False)\n\n# # Metrics from torchmetrics\n# accuracy = Accuracy_torchmetrics(average=None, num_classes=1)\n# precision = Precision_torchmetrics(average=\"micro\", num_classes=1)\n\n# Optimizers\ndeep_opt = SGD(model.deeptabular.parameters(), lr=0.1)\n\n# LR Scheduler\ndeep_sch = lr_scheduler.StepLR(deep_opt, step_size=3)\n\ntrainer = Trainer(\n    model,\n    objective=\"binary\",\n    lr_schedulers={\"deeptabular\": deep_sch},\n    initializers={\"deeptabular\": XavierNormal},\n    optimizers={\"deeptabular\": deep_opt},\n    metrics=[accuracy, precision],\n    verbose=1,\n)\n</pre> # Metrics from pytorch-widedeep accuracy = Accuracy(top_k=2) precision = Precision(average=False)  # # Metrics from torchmetrics # accuracy = Accuracy_torchmetrics(average=None, num_classes=1) # precision = Precision_torchmetrics(average=\"micro\", num_classes=1)  # Optimizers deep_opt = SGD(model.deeptabular.parameters(), lr=0.1)  # LR Scheduler deep_sch = lr_scheduler.StepLR(deep_opt, step_size=3)  trainer = Trainer(     model,     objective=\"binary\",     lr_schedulers={\"deeptabular\": deep_sch},     initializers={\"deeptabular\": XavierNormal},     optimizers={\"deeptabular\": deep_opt},     metrics=[accuracy, precision],     verbose=1, ) In\u00a0[6]: Copied! <pre>start = time.time()\ntrainer.fit(\n    X_train={\"X_tab\": X_tab_train, \"target\": y_train},\n    X_val={\"X_tab\": X_tab_valid, \"target\": y_valid},\n    n_epochs=1,\n    batch_size=32,\n    custom_dataloader=DataLoaderImbalanced,\n    oversample_mul=5,\n)\nprint(\n    \"Training time[s]: {}\".format(\n        datetime.timedelta(seconds=round(time.time() - start))\n    )\n)\n\npd.DataFrame(trainer.history)\n\ndf_pred = trainer.predict(X_tab=X_tab_test)\nprint(classification_report(df_test[\"target\"].to_list(), df_pred))\nprint(\"Actual predicted values:\\n{}\".format(np.unique(df_pred, return_counts=True)))\n</pre> start = time.time() trainer.fit(     X_train={\"X_tab\": X_tab_train, \"target\": y_train},     X_val={\"X_tab\": X_tab_valid, \"target\": y_valid},     n_epochs=1,     batch_size=32,     custom_dataloader=DataLoaderImbalanced,     oversample_mul=5, ) print(     \"Training time[s]: {}\".format(         datetime.timedelta(seconds=round(time.time() - start))     ) )  pd.DataFrame(trainer.history)  df_pred = trainer.predict(X_tab=X_tab_test) print(classification_report(df_test[\"target\"].to_list(), df_pred)) print(\"Actual predicted values:\\n{}\".format(np.unique(df_pred, return_counts=True))) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 325/325 [00:02&lt;00:00, 153.99it/s, loss=0.163, metrics={'acc': 0.9363, 'prec': [0.9358]}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 456/456 [00:02&lt;00:00, 205.93it/s, loss=0.1, metrics={'acc': 0.9501, 'prec': [0.1447]}]\n</pre> <pre>Training time[s]: 0:00:04\n</pre> <pre>predict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 456/456 [00:01&lt;00:00, 368.16it/s]\n</pre> <pre>              precision    recall  f1-score   support\n\n           0       1.00      0.95      0.97     14446\n           1       0.15      0.95      0.25       130\n\n    accuracy                           0.95     14576\n   macro avg       0.57      0.95      0.61     14576\nweighted avg       0.99      0.95      0.97     14576\n\nActual predicted values:\n(array([0, 1]), array([13736,   840]))\n</pre>"},{"location":"examples/08_custom_dataLoader_imbalanced_dataset.html#custom-dataloader-for-imbalanced-dataset","title":"Custom DataLoader for Imbalanced dataset\u00b6","text":""},{"location":"examples/08_custom_dataLoader_imbalanced_dataset.html#initial-imports","title":"Initial imports\u00b6","text":""},{"location":"examples/09_extracting_embeddings.html","title":"09_extracting_embeddings","text":"In\u00a0[1]: Copied! <pre>import numpy as np\nimport pandas as pd\nimport torch\n\nfrom pytorch_widedeep.preprocessing import TabPreprocessor\nfrom pytorch_widedeep.training import Trainer\nfrom pytorch_widedeep.models import FTTransformer, WideDeep\nfrom pytorch_widedeep.metrics import Accuracy\nfrom pytorch_widedeep import Tab2Vec\nfrom pytorch_widedeep.datasets import load_adult\n</pre> import numpy as np import pandas as pd import torch  from pytorch_widedeep.preprocessing import TabPreprocessor from pytorch_widedeep.training import Trainer from pytorch_widedeep.models import FTTransformer, WideDeep from pytorch_widedeep.metrics import Accuracy from pytorch_widedeep import Tab2Vec from pytorch_widedeep.datasets import load_adult In\u00a0[2]: Copied! <pre>df = load_adult(as_frame=True)\ndf.head()\n</pre> df = load_adult(as_frame=True) df.head() Out[2]: age workclass fnlwgt education educational-num marital-status occupation relationship race gender capital-gain capital-loss hours-per-week native-country income 0 25 Private 226802 11th 7 Never-married Machine-op-inspct Own-child Black Male 0 0 40 United-States &lt;=50K 1 38 Private 89814 HS-grad 9 Married-civ-spouse Farming-fishing Husband White Male 0 0 50 United-States &lt;=50K 2 28 Local-gov 336951 Assoc-acdm 12 Married-civ-spouse Protective-serv Husband White Male 0 0 40 United-States &gt;50K 3 44 Private 160323 Some-college 10 Married-civ-spouse Machine-op-inspct Husband Black Male 7688 0 40 United-States &gt;50K 4 18 ? 103497 Some-college 10 Never-married ? Own-child White Female 0 0 30 United-States &lt;=50K In\u00a0[3]: Copied! <pre># For convenience, we'll replace '-' with '_'\ndf.columns = [c.replace(\"-\", \"_\") for c in df.columns]\n# binary target\ndf[\"target\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int)\ndf.drop([\"income\", \"educational_num\"], axis=1, inplace=True)\n\ndf.head()\n</pre> # For convenience, we'll replace '-' with '_' df.columns = [c.replace(\"-\", \"_\") for c in df.columns] # binary target df[\"target\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int) df.drop([\"income\", \"educational_num\"], axis=1, inplace=True)  df.head() Out[3]: age workclass fnlwgt education marital_status occupation relationship race gender capital_gain capital_loss hours_per_week native_country target 0 25 Private 226802 11th Never-married Machine-op-inspct Own-child Black Male 0 0 40 United-States 0 1 38 Private 89814 HS-grad Married-civ-spouse Farming-fishing Husband White Male 0 0 50 United-States 0 2 28 Local-gov 336951 Assoc-acdm Married-civ-spouse Protective-serv Husband White Male 0 0 40 United-States 1 3 44 Private 160323 Some-college Married-civ-spouse Machine-op-inspct Husband Black Male 7688 0 40 United-States 1 4 18 ? 103497 Some-college Never-married ? Own-child White Female 0 0 30 United-States 0 In\u00a0[4]: Copied! <pre>cat_cols, cont_cols = [], []\nfor col in df.columns:\n    # 50 is just a random number I choose here for this example\n    if df[col].dtype == \"O\" or df[col].nunique() &lt; 50 and col != \"target\":\n        cat_cols.append(col)\n    elif col != \"target\":\n        cont_cols.append(col)\ntarget_col = \"target\"\n</pre> cat_cols, cont_cols = [], [] for col in df.columns:     # 50 is just a random number I choose here for this example     if df[col].dtype == \"O\" or df[col].nunique() &lt; 50 and col != \"target\":         cat_cols.append(col)     elif col != \"target\":         cont_cols.append(col) target_col = \"target\" In\u00a0[5]: Copied! <pre>target = df[target_col].values\n\ntab_preprocessor = TabPreprocessor(\n    embed_cols=cat_cols, continuous_cols=cont_cols, for_transformer=True\n)\nX_tab = tab_preprocessor.fit_transform(df)\n</pre> target = df[target_col].values  tab_preprocessor = TabPreprocessor(     embed_cols=cat_cols, continuous_cols=cont_cols, for_transformer=True ) X_tab = tab_preprocessor.fit_transform(df) <pre>/Users/javierrodriguezzaurin/Projects/pytorch-widedeep/pytorch_widedeep/preprocessing/tab_preprocessor.py:358: UserWarning: Continuous columns will not be normalised\n  warnings.warn(\"Continuous columns will not be normalised\")\n</pre> In\u00a0[6]: Copied! <pre>ft_transformer = FTTransformer(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=tab_preprocessor.continuous_cols,\n    embed_continuous_method=\"standard\",\n    n_blocks=2,\n    n_heads=4,\n    input_dim=16,\n)\n</pre> ft_transformer = FTTransformer(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     continuous_cols=tab_preprocessor.continuous_cols,     embed_continuous_method=\"standard\",     n_blocks=2,     n_heads=4,     input_dim=16, ) <pre>/Users/javierrodriguezzaurin/Projects/pytorch-widedeep/pytorch_widedeep/utils/general_utils.py:12: DeprecationWarning: The 'embed_continuous' parameter is deprecated and will be removed in the next release. Please use 'embed_continuous_method' instead See the documentation for more details.\n  return func(*args, **kwargs)\n</pre> In\u00a0[7]: Copied! <pre>model = WideDeep(deeptabular=ft_transformer)\ntrainer = Trainer(model, objective=\"binary\", metrics=[Accuracy])\ntrainer.fit(X_tab=X_tab, target=target, n_epochs=1, batch_size=256, val_split=0.2)\n</pre> model = WideDeep(deeptabular=ft_transformer) trainer = Trainer(model, objective=\"binary\", metrics=[Accuracy]) trainer.fit(X_tab=X_tab, target=target, n_epochs=1, batch_size=256, val_split=0.2) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:03&lt;00:00, 41.47it/s, loss=221, metrics={'acc': 0.686}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 39/39 [00:00&lt;00:00, 86.24it/s, loss=9.28, metrics={'acc': 0.76}]\n</pre> In\u00a0[8]: Copied! <pre>t2v = Tab2Vec(model=model, tab_preprocessor=tab_preprocessor)\n</pre> t2v = Tab2Vec(model=model, tab_preprocessor=tab_preprocessor) In\u00a0[9]: Copied! <pre># assuming is a test set with target col\nX_vec, y = t2v.transform(df.sample(100), target_col=\"target\")\n</pre> # assuming is a test set with target col X_vec, y = t2v.transform(df.sample(100), target_col=\"target\") In\u00a0[10]: Copied! <pre># X vec is the dataframe turned into the embeddings\nX_vec.shape\n</pre> # X vec is the dataframe turned into the embeddings X_vec.shape Out[10]: <pre>(100, 208)</pre> <p><code>208 = input_dim (16) * n_cols (13)</code></p> In\u00a0[11]: Copied! <pre># ...or if we don't have target col\nX_vec = t2v.transform(df.sample(100))\n</pre> # ...or if we don't have target col X_vec = t2v.transform(df.sample(100))"},{"location":"examples/09_extracting_embeddings.html#extracting-embeddings","title":"Extracting embeddings\u00b6","text":"<p>This notebook is a simple guide to extracting learned feature embeddings using Tab2Vec</p>"},{"location":"examples/10_3rd_party_integration-RayTune_WnB.html","title":"10_3rd_party_integration-RayTune_WnB","text":"In\u00a0[11]: Copied! <pre>from typing import Optional, Dict\nimport os\n\nimport numpy as np\nimport pandas as pd\nimport torch\nimport wandb\nfrom torch.optim import SGD, lr_scheduler\n\nfrom pytorch_widedeep import Trainer\nfrom pytorch_widedeep.preprocessing import TabPreprocessor\nfrom pytorch_widedeep.models import TabMlp, WideDeep\nfrom torchmetrics import F1Score as F1_torchmetrics\nfrom torchmetrics import Accuracy as Accuracy_torchmetrics\nfrom torchmetrics import Precision as Precision_torchmetrics\nfrom torchmetrics import Recall as Recall_torchmetrics\nfrom pytorch_widedeep.metrics import Accuracy, Recall, Precision, F1Score, R2Score\nfrom pytorch_widedeep.initializers import XavierNormal\nfrom pytorch_widedeep.callbacks import (\n    EarlyStopping,\n    ModelCheckpoint,\n    Callback,\n)\nfrom pytorch_widedeep.datasets import load_bio_kdd04\n\nfrom sklearn.model_selection import train_test_split\nimport warnings\n\nwarnings.filterwarnings(\"ignore\", category=DeprecationWarning)\n\nfrom ray import tune\nfrom ray.tune.schedulers import AsyncHyperBandScheduler\nfrom ray.tune import JupyterNotebookReporter\nfrom ray.air.integrations.wandb import WandbLoggerCallback\n\n# from ray.tune.integration.wandb import wandb_mixin\n\nimport tracemalloc\n\ntracemalloc.start()\n\n# increase displayed columns in jupyter notebook\npd.set_option(\"display.max_columns\", 200)\npd.set_option(\"display.max_rows\", 300)\n</pre> from typing import Optional, Dict import os  import numpy as np import pandas as pd import torch import wandb from torch.optim import SGD, lr_scheduler  from pytorch_widedeep import Trainer from pytorch_widedeep.preprocessing import TabPreprocessor from pytorch_widedeep.models import TabMlp, WideDeep from torchmetrics import F1Score as F1_torchmetrics from torchmetrics import Accuracy as Accuracy_torchmetrics from torchmetrics import Precision as Precision_torchmetrics from torchmetrics import Recall as Recall_torchmetrics from pytorch_widedeep.metrics import Accuracy, Recall, Precision, F1Score, R2Score from pytorch_widedeep.initializers import XavierNormal from pytorch_widedeep.callbacks import (     EarlyStopping,     ModelCheckpoint,     Callback, ) from pytorch_widedeep.datasets import load_bio_kdd04  from sklearn.model_selection import train_test_split import warnings  warnings.filterwarnings(\"ignore\", category=DeprecationWarning)  from ray import tune from ray.tune.schedulers import AsyncHyperBandScheduler from ray.tune import JupyterNotebookReporter from ray.air.integrations.wandb import WandbLoggerCallback  # from ray.tune.integration.wandb import wandb_mixin  import tracemalloc  tracemalloc.start()  # increase displayed columns in jupyter notebook pd.set_option(\"display.max_columns\", 200) pd.set_option(\"display.max_rows\", 300) In\u00a0[12]: Copied! <pre>class RayTuneReporter(Callback):\n    r\"\"\"Callback that allows reporting history and lr_history values to RayTune\n    during Hyperparameter tuning\n\n    Callbacks are passed as input parameters to the ``Trainer`` class. See\n    :class:`pytorch_widedeep.trainer.Trainer`\n\n    For examples see the examples folder at:\n\n        .. code-block:: bash\n\n            /examples/12_HyperParameter_tuning_w_RayTune.ipynb\n    \"\"\"\n\n    def on_epoch_end(\n        self, epoch: int, logs: Optional[Dict] = None, metric: Optional[float] = None\n    ):\n        report_dict = {}\n        for k, v in self.trainer.history.items():\n            report_dict.update({k: v[-1]})\n        if hasattr(self.trainer, \"lr_history\"):\n            for k, v in self.trainer.lr_history.items():\n                report_dict.update({k: v[-1]})\n        tune.report(report_dict)\n\n\nclass WnBReportBest(Callback):\n    r\"\"\"Callback that allows reporting best performance of a run to WnB\n    during Hyperparameter tuning. It is an adjusted pytorch_widedeep.callbacks.ModelCheckpoint\n    with added WnB and removed checkpoint saving.\n\n    Callbacks are passed as input parameters to the ``Trainer`` class.\n\n    Parameters\n    ----------\n    wb: obj\n        Weights&amp;Biases API interface to report single best result usable for\n        comparisson of multiple paramater combinations by, for example,\n        `parallel coordinates\n        &lt;https://docs.wandb.ai/ref/app/features/panels/parallel-coordinates&gt;`_.\n        E.g W&amp;B summary report `wandb.run.summary[\"best\"]`.\n    monitor: str, default=\"loss\"\n        quantity to monitor. Typically `'val_loss'` or metric name\n        (e.g. `'val_acc'`)\n    mode: str, default=\"auto\"\n        If ``save_best_only=True``, the decision to overwrite the current save\n        file is made based on either the maximization or the minimization of\n        the monitored quantity. For `'acc'`, this should be `'max'`, for\n        `'loss'` this should be `'min'`, etc. In `'auto'` mode, the\n        direction is automatically inferred from the name of the monitored\n        quantity.\n\n    \"\"\"\n\n    def __init__(\n        self,\n        wb: object,\n        monitor: str = \"val_loss\",\n        mode: str = \"auto\",\n    ):\n        super(WnBReportBest, self).__init__()\n\n        self.monitor = monitor\n        self.mode = mode\n        self.wb = wb\n\n        if self.mode not in [\"auto\", \"min\", \"max\"]:\n            warnings.warn(\n                \"WnBReportBest mode %s is unknown, \"\n                \"fallback to auto mode.\" % (self.mode),\n                RuntimeWarning,\n            )\n            self.mode = \"auto\"\n        if self.mode == \"min\":\n            self.monitor_op = np.less\n            self.best = np.Inf\n        elif self.mode == \"max\":\n            self.monitor_op = np.greater  # type: ignore[assignment]\n            self.best = -np.Inf\n        else:\n            if self._is_metric(self.monitor):\n                self.monitor_op = np.greater  # type: ignore[assignment]\n                self.best = -np.Inf\n            else:\n                self.monitor_op = np.less\n                self.best = np.Inf\n\n    def on_epoch_end(  # noqa: C901\n        self, epoch: int, logs: Optional[Dict] = None, metric: Optional[float] = None\n    ):\n        logs = logs or {}\n        current = logs.get(self.monitor)\n        if current is not None:\n            if self.monitor_op(current, self.best):\n                self.wb.run.summary[\"best\"] = current  # type: ignore[attr-defined]\n                self.best = current\n                self.best_epoch = epoch\n\n    @staticmethod\n    def _is_metric(monitor: str):\n        \"copied from pytorch_widedeep.callbacks\"\n        if any([s in monitor for s in [\"acc\", \"prec\", \"rec\", \"fscore\", \"f1\", \"f2\"]]):\n            return True\n        else:\n            return False\n</pre> class RayTuneReporter(Callback):     r\"\"\"Callback that allows reporting history and lr_history values to RayTune     during Hyperparameter tuning      Callbacks are passed as input parameters to the ``Trainer`` class. See     :class:`pytorch_widedeep.trainer.Trainer`      For examples see the examples folder at:          .. code-block:: bash              /examples/12_HyperParameter_tuning_w_RayTune.ipynb     \"\"\"      def on_epoch_end(         self, epoch: int, logs: Optional[Dict] = None, metric: Optional[float] = None     ):         report_dict = {}         for k, v in self.trainer.history.items():             report_dict.update({k: v[-1]})         if hasattr(self.trainer, \"lr_history\"):             for k, v in self.trainer.lr_history.items():                 report_dict.update({k: v[-1]})         tune.report(report_dict)   class WnBReportBest(Callback):     r\"\"\"Callback that allows reporting best performance of a run to WnB     during Hyperparameter tuning. It is an adjusted pytorch_widedeep.callbacks.ModelCheckpoint     with added WnB and removed checkpoint saving.      Callbacks are passed as input parameters to the ``Trainer`` class.      Parameters     ----------     wb: obj         Weights&amp;Biases API interface to report single best result usable for         comparisson of multiple paramater combinations by, for example,         `parallel coordinates         `_.         E.g W&amp;B summary report `wandb.run.summary[\"best\"]`.     monitor: str, default=\"loss\"         quantity to monitor. Typically `'val_loss'` or metric name         (e.g. `'val_acc'`)     mode: str, default=\"auto\"         If ``save_best_only=True``, the decision to overwrite the current save         file is made based on either the maximization or the minimization of         the monitored quantity. For `'acc'`, this should be `'max'`, for         `'loss'` this should be `'min'`, etc. In `'auto'` mode, the         direction is automatically inferred from the name of the monitored         quantity.      \"\"\"      def __init__(         self,         wb: object,         monitor: str = \"val_loss\",         mode: str = \"auto\",     ):         super(WnBReportBest, self).__init__()          self.monitor = monitor         self.mode = mode         self.wb = wb          if self.mode not in [\"auto\", \"min\", \"max\"]:             warnings.warn(                 \"WnBReportBest mode %s is unknown, \"                 \"fallback to auto mode.\" % (self.mode),                 RuntimeWarning,             )             self.mode = \"auto\"         if self.mode == \"min\":             self.monitor_op = np.less             self.best = np.Inf         elif self.mode == \"max\":             self.monitor_op = np.greater  # type: ignore[assignment]             self.best = -np.Inf         else:             if self._is_metric(self.monitor):                 self.monitor_op = np.greater  # type: ignore[assignment]                 self.best = -np.Inf             else:                 self.monitor_op = np.less                 self.best = np.Inf      def on_epoch_end(  # noqa: C901         self, epoch: int, logs: Optional[Dict] = None, metric: Optional[float] = None     ):         logs = logs or {}         current = logs.get(self.monitor)         if current is not None:             if self.monitor_op(current, self.best):                 self.wb.run.summary[\"best\"] = current  # type: ignore[attr-defined]                 self.best = current                 self.best_epoch = epoch      @staticmethod     def _is_metric(monitor: str):         \"copied from pytorch_widedeep.callbacks\"         if any([s in monitor for s in [\"acc\", \"prec\", \"rec\", \"fscore\", \"f1\", \"f2\"]]):             return True         else:             return False In\u00a0[13]: Copied! <pre>df = load_bio_kdd04(as_frame=True)\ndf.head()\n</pre> df = load_bio_kdd04(as_frame=True) df.head() Out[13]: EXAMPLE_ID BLOCK_ID target 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 0 279 261532 0 52.0 32.69 0.30 2.5 20.0 1256.8 -0.89 0.33 11.0 -55.0 267.2 0.52 0.05 -2.36 49.6 252.0 0.43 1.16 -2.06 -33.0 -123.2 1.60 -0.49 -6.06 65.0 296.1 -0.28 -0.26 -3.83 -22.6 -170.0 3.06 -1.05 -3.29 22.9 286.3 0.12 2.58 4.08 -33.0 -178.9 1.88 0.53 -7.0 -44.0 1987.0 -5.41 0.95 -4.0 -57.0 722.9 -3.26 -0.55 -7.5 125.5 1547.2 -0.36 1.12 9.0 -37.0 72.5 0.47 0.74 -11.0 -8.0 1595.1 -1.64 2.83 -2.0 -50.0 445.2 -0.35 0.26 0.76 1 279 261533 0 58.0 33.33 0.00 16.5 9.5 608.1 0.50 0.07 20.5 -52.5 521.6 -1.08 0.58 -0.02 -3.2 103.6 -0.95 0.23 -2.87 -25.9 -52.2 -0.21 0.87 -1.81 10.4 62.0 -0.28 -0.04 1.48 -17.6 -198.3 3.43 2.84 5.87 -16.9 72.6 -0.31 2.79 2.71 -33.5 -11.6 -1.11 4.01 5.0 -57.0 666.3 1.13 4.38 5.0 -64.0 39.3 1.07 -0.16 32.5 100.0 1893.7 -2.80 -0.22 2.5 -28.5 45.0 0.58 0.41 -19.0 -6.0 762.9 0.29 0.82 -3.0 -35.0 140.3 1.16 0.39 0.73 2 279 261534 0 77.0 27.27 -0.91 6.0 58.5 1623.6 -1.40 0.02 -6.5 -48.0 621.0 -1.20 0.14 -0.20 73.6 609.1 -0.44 -0.58 -0.04 -23.0 -27.4 -0.72 -1.04 -1.09 91.1 635.6 -0.88 0.24 0.59 -18.7 -7.2 -0.60 -2.82 -0.71 52.4 504.1 0.89 -0.67 -9.30 -20.8 -25.7 -0.77 -0.85 0.0 -20.0 2259.0 -0.94 1.15 -4.0 -44.0 -22.7 0.94 -0.98 -19.0 105.0 1267.9 1.03 1.27 11.0 -39.5 82.3 0.47 -0.19 -10.0 7.0 1491.8 0.32 -1.29 0.0 -34.0 658.2 -0.76 0.26 0.24 3 279 261535 0 41.0 27.91 -0.35 3.0 46.0 1921.6 -1.36 -0.47 -32.0 -51.5 560.9 -0.29 -0.10 -1.11 124.3 791.6 0.00 0.39 -1.85 -21.7 -44.9 -0.21 0.02 0.89 133.9 797.8 -0.08 1.06 -0.26 -16.4 -74.1 0.97 -0.80 -0.41 66.9 955.3 -1.90 1.28 -6.65 -28.1 47.5 -1.91 1.42 1.0 -30.0 1846.7 0.76 1.10 -4.0 -52.0 -53.9 1.71 -0.22 -12.0 97.5 1969.8 -1.70 0.16 -1.0 -32.5 255.9 -0.46 1.57 10.0 6.0 2047.7 -0.98 1.53 0.0 -49.0 554.2 -0.83 0.39 0.73 4 279 261536 0 50.0 28.00 -1.32 -9.0 12.0 464.8 0.88 0.19 8.0 -51.5 98.1 1.09 -0.33 -2.16 -3.9 102.7 0.39 -1.22 -3.39 -15.2 -42.2 -1.18 -1.11 -3.55 8.9 141.3 -0.16 -0.43 -4.15 -12.9 -13.4 -1.32 -0.98 -3.69 8.8 136.1 -0.30 4.13 1.89 -13.0 -18.7 -1.37 -0.93 0.0 -1.0 810.1 -2.29 6.72 1.0 -23.0 -29.7 0.58 -1.10 -18.5 33.5 206.8 1.84 -0.13 4.0 -29.0 30.1 0.80 -0.24 5.0 -14.0 479.5 0.68 -0.59 2.0 -36.0 -6.9 2.02 0.14 -0.23 In\u00a0[14]: Copied! <pre># imbalance of the classes\ndf[\"target\"].value_counts()\n</pre> # imbalance of the classes df[\"target\"].value_counts() Out[14]: <pre>target\n0    144455\n1      1296\nName: count, dtype: int64</pre> In\u00a0[15]: Copied! <pre># drop columns we won't need in this example\ndf.drop(columns=[\"EXAMPLE_ID\", \"BLOCK_ID\"], inplace=True)\n</pre> # drop columns we won't need in this example df.drop(columns=[\"EXAMPLE_ID\", \"BLOCK_ID\"], inplace=True) In\u00a0[16]: Copied! <pre>df_train, df_valid = train_test_split(\n    df, test_size=0.2, stratify=df[\"target\"], random_state=1\n)\ndf_valid, df_test = train_test_split(\n    df_valid, test_size=0.5, stratify=df_valid[\"target\"], random_state=1\n)\n</pre> df_train, df_valid = train_test_split(     df, test_size=0.2, stratify=df[\"target\"], random_state=1 ) df_valid, df_test = train_test_split(     df_valid, test_size=0.5, stratify=df_valid[\"target\"], random_state=1 ) In\u00a0[17]: Copied! <pre>continuous_cols = df.drop(columns=[\"target\"]).columns.values.tolist()\n</pre> continuous_cols = df.drop(columns=[\"target\"]).columns.values.tolist() In\u00a0[18]: Copied! <pre># deeptabular\ntab_preprocessor = TabPreprocessor(continuous_cols=continuous_cols, scale=True)\nX_tab_train = tab_preprocessor.fit_transform(df_train)\nX_tab_valid = tab_preprocessor.transform(df_valid)\nX_tab_test = tab_preprocessor.transform(df_test)\n\n# target\ny_train = df_train[\"target\"].values\ny_valid = df_valid[\"target\"].values\ny_test = df_test[\"target\"].values\n</pre> # deeptabular tab_preprocessor = TabPreprocessor(continuous_cols=continuous_cols, scale=True) X_tab_train = tab_preprocessor.fit_transform(df_train) X_tab_valid = tab_preprocessor.transform(df_valid) X_tab_test = tab_preprocessor.transform(df_test)  # target y_train = df_train[\"target\"].values y_valid = df_valid[\"target\"].values y_test = df_test[\"target\"].values In\u00a0[19]: Copied! <pre>input_layer = len(tab_preprocessor.continuous_cols)\noutput_layer = 1\nhidden_layers = np.linspace(\n    input_layer * 2, output_layer, 5, endpoint=False, dtype=int\n).tolist()\n</pre> input_layer = len(tab_preprocessor.continuous_cols) output_layer = 1 hidden_layers = np.linspace(     input_layer * 2, output_layer, 5, endpoint=False, dtype=int ).tolist() In\u00a0[20]: Copied! <pre>deeptabular = TabMlp(\n    mlp_hidden_dims=hidden_layers,\n    column_idx=tab_preprocessor.column_idx,\n    continuous_cols=tab_preprocessor.continuous_cols,\n)\nmodel = WideDeep(deeptabular=deeptabular)\nmodel\n</pre> deeptabular = TabMlp(     mlp_hidden_dims=hidden_layers,     column_idx=tab_preprocessor.column_idx,     continuous_cols=tab_preprocessor.continuous_cols, ) model = WideDeep(deeptabular=deeptabular) model Out[20]: <pre>WideDeep(\n  (deeptabular): Sequential(\n    (0): TabMlp(\n      (cont_norm): Identity()\n      (encoder): MLP(\n        (mlp): Sequential(\n          (dense_layer_0): Sequential(\n            (0): Linear(in_features=74, out_features=148, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_1): Sequential(\n            (0): Linear(in_features=148, out_features=118, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_2): Sequential(\n            (0): Linear(in_features=118, out_features=89, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_3): Sequential(\n            (0): Linear(in_features=89, out_features=59, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_4): Sequential(\n            (0): Linear(in_features=59, out_features=30, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=30, out_features=1, bias=True)\n  )\n)</pre> In\u00a0[24]: Copied! <pre># Metrics from torchmetrics\naccuracy = Accuracy_torchmetrics(average=None, num_classes=1, task=\"binary\")\nprecision = Precision_torchmetrics(average=\"micro\", num_classes=1, task=\"binary\")\nf1 = F1_torchmetrics(average=None, num_classes=1, task=\"binary\")\nrecall = Recall_torchmetrics(average=None, num_classes=1, task=\"binary\")\n</pre> # Metrics from torchmetrics accuracy = Accuracy_torchmetrics(average=None, num_classes=1, task=\"binary\") precision = Precision_torchmetrics(average=\"micro\", num_classes=1, task=\"binary\") f1 = F1_torchmetrics(average=None, num_classes=1, task=\"binary\") recall = Recall_torchmetrics(average=None, num_classes=1, task=\"binary\") <p>Note:</p> <p>Following cells includes usage of both <code>RayTuneReporter</code> and <code>WnBReportBest</code> callbacks. In case you want to use just <code>RayTuneReporter</code>, remove following:</p> <ul> <li>wandb from config</li> <li><code>WandbLoggerCallback</code></li> <li><code>WnBReportBest</code></li> <li><code>@wandb_mixin</code> decorator</li> </ul> <p>We do not see strong reason to use WnB without RayTune for a single paramater combination run, but it is possible:</p> <ul> <li>option01: define paramaters in config only for a single value <code>tune.grid_search([1000])</code> (single value RayTune run)</li> <li>option02: define WnB callback that reports currnet validation/training loss, metrics, etc. at the end of batch, ie. do not report to WnB at <code>epoch_end</code> as in <code>WnBReportBest</code> but at the <code>on_batch_end</code>, see <code>pytorch_widedeep.callbacks.Callback</code></li> </ul> In\u00a0[26]: Copied! <pre>config = {\n    \"batch_size\": tune.grid_search([1000, 5000]),\n    \"wandb\": {\n        \"project\": \"test\",\n        # \"api_key_file\": os.getcwd() + \"/wandb_api.key\",\n        \"api_key\": \"WNB_API_KEY\",\n    },\n}\n\n# Optimizers\ndeep_opt = SGD(model.deeptabular.parameters(), lr=0.1)\n# LR Scheduler\ndeep_sch = lr_scheduler.StepLR(deep_opt, step_size=3)\n\n\n@wandb_mixin\ndef training_function(config, X_train, X_val):\n    early_stopping = EarlyStopping()\n    model_checkpoint = ModelCheckpoint(save_best_only=True)\n    # Hyperparameters\n    batch_size = config[\"batch_size\"]\n    trainer = Trainer(\n        model,\n        objective=\"binary_focal_loss\",\n        callbacks=[\n            RayTuneReporter,\n            WnBReportBest(wb=wandb),\n            early_stopping,\n            model_checkpoint,\n        ],\n        lr_schedulers={\"deeptabular\": deep_sch},\n        initializers={\"deeptabular\": XavierNormal},\n        optimizers={\"deeptabular\": deep_opt},\n        metrics=[accuracy, precision, recall, f1],\n        verbose=0,\n    )\n\n    trainer.fit(X_train=X_train, X_val=X_val, n_epochs=5, batch_size=batch_size)\n\n\nX_train = {\"X_tab\": X_tab_train, \"target\": y_train}\nX_val = {\"X_tab\": X_tab_valid, \"target\": y_valid}\n\nasha_scheduler = AsyncHyperBandScheduler(\n    time_attr=\"training_iteration\",\n    metric=\"_metric/val_loss\",\n    mode=\"min\",\n    max_t=100,\n    grace_period=10,\n    reduction_factor=3,\n    brackets=1,\n)\n\nanalysis = tune.run(\n    tune.with_parameters(training_function, X_train=X_train, X_val=X_val),\n    resources_per_trial={\"cpu\": 1, \"gpu\": 0},\n    progress_reporter=JupyterNotebookReporter(overwrite=True),\n    scheduler=asha_scheduler,\n    config=config,\n    callbacks=[\n        WandbLoggerCallback(\n            project=config[\"wandb\"][\"project\"],\n            # api_key_file=config[\"wandb\"][\"api_key_file\"],\n            api_key=config[\"wandb\"][\"api_key\"],\n            log_config=True,\n        )\n    ],\n)\n</pre> config = {     \"batch_size\": tune.grid_search([1000, 5000]),     \"wandb\": {         \"project\": \"test\",         # \"api_key_file\": os.getcwd() + \"/wandb_api.key\",         \"api_key\": \"WNB_API_KEY\",     }, }  # Optimizers deep_opt = SGD(model.deeptabular.parameters(), lr=0.1) # LR Scheduler deep_sch = lr_scheduler.StepLR(deep_opt, step_size=3)   @wandb_mixin def training_function(config, X_train, X_val):     early_stopping = EarlyStopping()     model_checkpoint = ModelCheckpoint(save_best_only=True)     # Hyperparameters     batch_size = config[\"batch_size\"]     trainer = Trainer(         model,         objective=\"binary_focal_loss\",         callbacks=[             RayTuneReporter,             WnBReportBest(wb=wandb),             early_stopping,             model_checkpoint,         ],         lr_schedulers={\"deeptabular\": deep_sch},         initializers={\"deeptabular\": XavierNormal},         optimizers={\"deeptabular\": deep_opt},         metrics=[accuracy, precision, recall, f1],         verbose=0,     )      trainer.fit(X_train=X_train, X_val=X_val, n_epochs=5, batch_size=batch_size)   X_train = {\"X_tab\": X_tab_train, \"target\": y_train} X_val = {\"X_tab\": X_tab_valid, \"target\": y_valid}  asha_scheduler = AsyncHyperBandScheduler(     time_attr=\"training_iteration\",     metric=\"_metric/val_loss\",     mode=\"min\",     max_t=100,     grace_period=10,     reduction_factor=3,     brackets=1, )  analysis = tune.run(     tune.with_parameters(training_function, X_train=X_train, X_val=X_val),     resources_per_trial={\"cpu\": 1, \"gpu\": 0},     progress_reporter=JupyterNotebookReporter(overwrite=True),     scheduler=asha_scheduler,     config=config,     callbacks=[         WandbLoggerCallback(             project=config[\"wandb\"][\"project\"],             # api_key_file=config[\"wandb\"][\"api_key_file\"],             api_key=config[\"wandb\"][\"api_key\"],             log_config=True,         )     ], ) <pre>/Users/javierrodriguezzaurin/.pyenv/versions/3.10.13/lib/python3.10/tempfile.py:860: ResourceWarning: Implicitly cleaning up &lt;TemporaryDirectory '/var/folders/_2/lrjn1qn54c758tdtktr1bvkc0000gn/T/tmp60pfyl1kwandb'&gt;\n  _warnings.warn(warn_message, ResourceWarning)\n/Users/javierrodriguezzaurin/.pyenv/versions/3.10.13/lib/python3.10/tempfile.py:860: ResourceWarning: Implicitly cleaning up &lt;TemporaryDirectory '/var/folders/_2/lrjn1qn54c758tdtktr1bvkc0000gn/T/tmpnjv2rg1wwandb-artifacts'&gt;\n  _warnings.warn(warn_message, ResourceWarning)\n/Users/javierrodriguezzaurin/.pyenv/versions/3.10.13/lib/python3.10/tempfile.py:860: ResourceWarning: Implicitly cleaning up &lt;TemporaryDirectory '/var/folders/_2/lrjn1qn54c758tdtktr1bvkc0000gn/T/tmpgebu5k1kwandb-media'&gt;\n  _warnings.warn(warn_message, ResourceWarning)\n/Users/javierrodriguezzaurin/.pyenv/versions/3.10.13/lib/python3.10/tempfile.py:860: ResourceWarning: Implicitly cleaning up &lt;TemporaryDirectory '/var/folders/_2/lrjn1qn54c758tdtktr1bvkc0000gn/T/tmpxy9y2yriwandb-media'&gt;\n  _warnings.warn(warn_message, ResourceWarning)\n</pre> In\u00a0[14]: Copied! <pre>analysis.results\n</pre> analysis.results Out[14]: <pre>{'fc9a8_00000': {'_metric': {'train_loss': 0.006297602537127896,\n   'train_Accuracy': 0.9925042986869812,\n   'train_Precision': 0.9939393997192383,\n   'train_Recall': 0.15814851224422455,\n   'train_F1Score': 0.2728785574436188,\n   'val_loss': 0.005045663565397263,\n   'val_Accuracy': 0.9946483969688416,\n   'val_Precision': 1.0,\n   'val_Recall': 0.39534884691238403,\n   'val_F1Score': 0.5666667222976685},\n  'time_this_iter_s': 2.388202428817749,\n  'done': True,\n  'timesteps_total': None,\n  'episodes_total': None,\n  'training_iteration': 5,\n  'trial_id': 'fc9a8_00000',\n  'experiment_id': 'baad1d4f3d924b48b9ece1b9f26c80cc',\n  'date': '2022-07-31_14-06-51',\n  'timestamp': 1659276411,\n  'time_total_s': 12.656474113464355,\n  'pid': 1813,\n  'hostname': 'jupyter-5uperpalo',\n  'node_ip': '10.32.44.172',\n  'config': {'batch_size': 1000},\n  'time_since_restore': 12.656474113464355,\n  'timesteps_since_restore': 0,\n  'iterations_since_restore': 5,\n  'warmup_time': 0.8006253242492676,\n  'experiment_tag': '0_batch_size=1000'},\n 'fc9a8_00001': {'_metric': {'train_loss': 0.02519632239515583,\n   'train_Accuracy': 0.9910891652107239,\n   'train_Precision': 0.25,\n   'train_Recall': 0.0009643201483413577,\n   'train_F1Score': 0.0019212296465411782,\n   'val_loss': 0.02578434906899929,\n   'val_Accuracy': 0.9911492466926575,\n   'val_Precision': 0.0,\n   'val_Recall': 0.0,\n   'val_F1Score': 0.0},\n  'time_this_iter_s': 4.113586902618408,\n  'done': True,\n  'timesteps_total': None,\n  'episodes_total': None,\n  'training_iteration': 5,\n  'trial_id': 'fc9a8_00001',\n  'experiment_id': 'f2e54a6a5780429fbf0db0746853347e',\n  'date': '2022-07-31_14-06-56',\n  'timestamp': 1659276416,\n  'time_total_s': 12.926990509033203,\n  'pid': 1962,\n  'hostname': 'jupyter-5uperpalo',\n  'node_ip': '10.32.44.172',\n  'config': {'batch_size': 5000},\n  'time_since_restore': 12.926990509033203,\n  'timesteps_since_restore': 0,\n  'iterations_since_restore': 5,\n  'warmup_time': 0.9253025054931641,\n  'experiment_tag': '1_batch_size=5000'}}</pre> <p>Using Weights and Biases logging you can create parallel coordinates graphs that map parametr combinations to the best(lowest) loss achieved during the training of the networks</p> <p></p> <p>local visualization of raytune reults using tensorboard</p> In\u00a0[23]: Copied! <pre>%load_ext tensorboard\n%tensorboard --logdir ~/ray_results\n</pre> %load_ext tensorboard %tensorboard --logdir ~/ray_results"},{"location":"examples/10_3rd_party_integration-RayTune_WnB.html#3rd-party-integration-raytune-weights-biases","title":"3rd party integration - RayTune, Weights &amp; Biases\u00b6","text":"<p>This notebook provides guideline for integration of external library functions in the model training process through <code>Callback</code> objects, a popular concept of using objects as arguments for other objects.</p> <p>[DISCLAIMER]</p> <p>We show integration of RayTune (a hyperparameter tuning framework) and Weights &amp; Biases (ML projects experiment tracking and versioning solution) in the <code>pytorch_widedeep</code> model training process. We did not include <code>RayTuneReporter</code> and <code>WnBReportBest</code> in the library code to minimize the dependencies on other libraries that are not directly included in the model design and training.</p>"},{"location":"examples/10_3rd_party_integration-RayTune_WnB.html#initial-imports","title":"Initial imports\u00b6","text":""},{"location":"examples/10_3rd_party_integration-RayTune_WnB.html#preparing-the-data","title":"Preparing the data\u00b6","text":""},{"location":"examples/10_3rd_party_integration-RayTune_WnB.html#define-the-model","title":"Define the model\u00b6","text":""},{"location":"examples/11_auc_multiclass.html","title":"11_auc_multiclass","text":"In\u00a0[1]: Copied! <pre>import numpy as np\nimport pandas as pd\nfrom torch.optim import SGD, lr_scheduler\n\nfrom pytorch_widedeep import Trainer\nfrom pytorch_widedeep.preprocessing import TabPreprocessor\nfrom pytorch_widedeep.models import TabMlp, WideDeep\nfrom torchmetrics import AUROC\nfrom pytorch_widedeep.initializers import XavierNormal\nfrom pytorch_widedeep.datasets import load_ecoli\nfrom pytorch_widedeep.utils import LabelEncoder\n\nfrom sklearn.model_selection import train_test_split\n\n# increase displayed columns in jupyter notebook\npd.set_option(\"display.max_columns\", 200)\npd.set_option(\"display.max_rows\", 300)\n</pre> import numpy as np import pandas as pd from torch.optim import SGD, lr_scheduler  from pytorch_widedeep import Trainer from pytorch_widedeep.preprocessing import TabPreprocessor from pytorch_widedeep.models import TabMlp, WideDeep from torchmetrics import AUROC from pytorch_widedeep.initializers import XavierNormal from pytorch_widedeep.datasets import load_ecoli from pytorch_widedeep.utils import LabelEncoder  from sklearn.model_selection import train_test_split  # increase displayed columns in jupyter notebook pd.set_option(\"display.max_columns\", 200) pd.set_option(\"display.max_rows\", 300) In\u00a0[2]: Copied! <pre>df = load_ecoli(as_frame=True)\ndf.head()\n</pre> df = load_ecoli(as_frame=True) df.head() Out[2]: SequenceName mcg gvh lip chg aac alm1 alm2 class 0 AAT_ECOLI 0.49 0.29 0.48 0.5 0.56 0.24 0.35 cp 1 ACEA_ECOLI 0.07 0.40 0.48 0.5 0.54 0.35 0.44 cp 2 ACEK_ECOLI 0.56 0.40 0.48 0.5 0.49 0.37 0.46 cp 3 ACKA_ECOLI 0.59 0.49 0.48 0.5 0.52 0.45 0.36 cp 4 ADI_ECOLI 0.23 0.32 0.48 0.5 0.55 0.25 0.35 cp In\u00a0[3]: Copied! <pre># imbalance of the classes\ndf[\"class\"].value_counts()\n</pre> # imbalance of the classes df[\"class\"].value_counts() Out[3]: <pre>class\ncp     143\nim      77\npp      52\nimU     35\nom      20\nomL      5\nimS      2\nimL      2\nName: count, dtype: int64</pre> In\u00a0[4]: Copied! <pre>df = df.loc[~df[\"class\"].isin([\"omL\", \"imS\", \"imL\"])]\ndf.reset_index(inplace=True, drop=True)\n</pre> df = df.loc[~df[\"class\"].isin([\"omL\", \"imS\", \"imL\"])] df.reset_index(inplace=True, drop=True) In\u00a0[5]: Copied! <pre>encoder = LabelEncoder([\"class\"])\ndf_enc = encoder.fit_transform(df)\ndf_enc[\"class\"] = df_enc[\"class\"] - 1\n</pre> encoder = LabelEncoder([\"class\"]) df_enc = encoder.fit_transform(df) df_enc[\"class\"] = df_enc[\"class\"] - 1 In\u00a0[6]: Copied! <pre># drop columns we won't need in this example\ndf_enc = df_enc.drop(columns=[\"SequenceName\"])\n</pre> # drop columns we won't need in this example df_enc = df_enc.drop(columns=[\"SequenceName\"]) In\u00a0[7]: Copied! <pre>df_train, df_valid = train_test_split(\n    df_enc, test_size=0.2, stratify=df_enc[\"class\"], random_state=1\n)\ndf_valid, df_test = train_test_split(\n    df_valid, test_size=0.5, stratify=df_valid[\"class\"], random_state=1\n)\n</pre> df_train, df_valid = train_test_split(     df_enc, test_size=0.2, stratify=df_enc[\"class\"], random_state=1 ) df_valid, df_test = train_test_split(     df_valid, test_size=0.5, stratify=df_valid[\"class\"], random_state=1 ) In\u00a0[8]: Copied! <pre>continuous_cols = df_enc.drop(columns=[\"class\"]).columns.values.tolist()\n</pre> continuous_cols = df_enc.drop(columns=[\"class\"]).columns.values.tolist() In\u00a0[9]: Copied! <pre># deeptabular\ntab_preprocessor = TabPreprocessor(continuous_cols=continuous_cols, scale=True)\nX_tab_train = tab_preprocessor.fit_transform(df_train)\nX_tab_valid = tab_preprocessor.transform(df_valid)\nX_tab_test = tab_preprocessor.transform(df_test)\n\n# target\ny_train = df_train[\"class\"].values\ny_valid = df_valid[\"class\"].values\ny_test = df_test[\"class\"].values\n\nX_train = {\"X_tab\": X_tab_train, \"target\": y_train}\nX_val = {\"X_tab\": X_tab_valid, \"target\": y_valid}\n</pre> # deeptabular tab_preprocessor = TabPreprocessor(continuous_cols=continuous_cols, scale=True) X_tab_train = tab_preprocessor.fit_transform(df_train) X_tab_valid = tab_preprocessor.transform(df_valid) X_tab_test = tab_preprocessor.transform(df_test)  # target y_train = df_train[\"class\"].values y_valid = df_valid[\"class\"].values y_test = df_test[\"class\"].values  X_train = {\"X_tab\": X_tab_train, \"target\": y_train} X_val = {\"X_tab\": X_tab_valid, \"target\": y_valid} <pre>/Users/javierrodriguezzaurin/Projects/pytorch-widedeep/pytorch_widedeep/preprocessing/tab_preprocessor.py:295: DeprecationWarning: 'scale' and 'already_standard' will be deprecated in the next release. Please use 'cols_to_scale' instead\n  self._check_inputs(cat_embed_cols)\n</pre> In\u00a0[10]: Copied! <pre>deeptabular = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    continuous_cols=tab_preprocessor.continuous_cols,\n)\nmodel = WideDeep(deeptabular=deeptabular, pred_dim=df_enc[\"class\"].nunique())\nmodel\n</pre> deeptabular = TabMlp(     column_idx=tab_preprocessor.column_idx,     continuous_cols=tab_preprocessor.continuous_cols, ) model = WideDeep(deeptabular=deeptabular, pred_dim=df_enc[\"class\"].nunique()) model Out[10]: <pre>WideDeep(\n  (deeptabular): Sequential(\n    (0): TabMlp(\n      (cont_norm): Identity()\n      (encoder): MLP(\n        (mlp): Sequential(\n          (dense_layer_0): Sequential(\n            (0): Linear(in_features=7, out_features=200, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_1): Sequential(\n            (0): Linear(in_features=200, out_features=100, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=100, out_features=5, bias=True)\n  )\n)</pre> In\u00a0[11]: Copied! <pre>auroc = AUROC(num_classes=df_enc[\"class\"].nunique(), task=\"multiclass\")\n</pre> auroc = AUROC(num_classes=df_enc[\"class\"].nunique(), task=\"multiclass\") In\u00a0[12]: Copied! <pre># Optimizers\ndeep_opt = SGD(model.deeptabular.parameters(), lr=0.1)\n# LR Scheduler\ndeep_sch = lr_scheduler.StepLR(deep_opt, step_size=3)\n# Hyperparameters\ntrainer = Trainer(\n    model,\n    objective=\"multiclass_focal_loss\",\n    lr_schedulers={\"deeptabular\": deep_sch},\n    initializers={\"deeptabular\": XavierNormal},\n    optimizers={\"deeptabular\": deep_opt},\n    metrics=[auroc],\n)\n\ntrainer.fit(X_train=X_train, X_val=X_val, n_epochs=5, batch_size=50)\n</pre> # Optimizers deep_opt = SGD(model.deeptabular.parameters(), lr=0.1) # LR Scheduler deep_sch = lr_scheduler.StepLR(deep_opt, step_size=3) # Hyperparameters trainer = Trainer(     model,     objective=\"multiclass_focal_loss\",     lr_schedulers={\"deeptabular\": deep_sch},     initializers={\"deeptabular\": XavierNormal},     optimizers={\"deeptabular\": deep_opt},     metrics=[auroc], )  trainer.fit(X_train=X_train, X_val=X_val, n_epochs=5, batch_size=50) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 6/6 [00:00&lt;00:00, 54.59it/s, loss=0.109, metrics={'MulticlassAUROC': 0.314}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 1/1 [00:00&lt;00:00, 98.35it/s, loss=0.105, metrics={'MulticlassAUROC': 0.2558}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 6/6 [00:00&lt;00:00, 91.55it/s, loss=0.105, metrics={'MulticlassAUROC': 0.3546}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 1/1 [00:00&lt;00:00, 111.68it/s, loss=0.101, metrics={'MulticlassAUROC': 0.2737}]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 6/6 [00:00&lt;00:00, 62.55it/s, loss=0.1, metrics={'MulticlassAUROC': 0.3795}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 1/1 [00:00&lt;00:00, 108.51it/s, loss=0.0966, metrics={'MulticlassAUROC': 0.3053}]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 6/6 [00:00&lt;00:00, 99.35it/s, loss=0.0965, metrics={'MulticlassAUROC': 0.3809}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 1/1 [00:00&lt;00:00, 117.73it/s, loss=0.0962, metrics={'MulticlassAUROC': 0.3089}]\nepoch 5: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 6/6 [00:00&lt;00:00, 110.56it/s, loss=0.0967, metrics={'MulticlassAUROC': 0.3509}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 1/1 [00:00&lt;00:00, 127.35it/s, loss=0.0958, metrics={'MulticlassAUROC': 0.3089}]\n</pre> In\u00a0[\u00a0]: Copied! <pre>\n</pre>"},{"location":"examples/11_auc_multiclass.html#auc-multiclass-computation","title":"AUC multiclass computation\u00b6","text":""},{"location":"examples/11_auc_multiclass.html#initial-imports","title":"Initial imports\u00b6","text":""},{"location":"examples/11_auc_multiclass.html#preparing-the-data","title":"Preparing the data\u00b6","text":""},{"location":"examples/11_auc_multiclass.html#define-the-model","title":"Define the model\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html","title":"12_ZILNLoss_origkeras_vs_pytorch_widedeep","text":"In\u00a0[1]: Copied! <pre># @title Copyright 2019 The Lifetime Value Authors.\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     https://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n# ============================================================================\n</pre> # @title Copyright 2019 The Lifetime Value Authors. # Licensed under the Apache License, Version 2.0 (the \"License\"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # #     https://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an \"AS IS\" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ Run in Google Colab View source on GitHub In\u00a0[3]: Copied! <pre>import os\n\nimport numpy as np\nimport pandas as pd\nfrom scipy import stats\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport tensorflow as tf\nimport tensorflow_probability as tfp\nfrom typing import Sequence\n\n# install and import ltv\n!pip install -q git+https://github.com/google/lifetime_value\nimport lifetime_value as ltv\n</pre> import os  import numpy as np import pandas as pd from scipy import stats import matplotlib.pyplot as plt import seaborn as sns import tensorflow as tf import tensorflow_probability as tfp from typing import Sequence  # install and import ltv !pip install -q git+https://github.com/google/lifetime_value import lifetime_value as ltv In\u00a0[\u00a0]: Copied! <pre>tfd = tfp.distributions\n%config InlineBackend.figure_format='retina'\nsns.set_style(\"whitegrid\")\n</pre> tfd = tfp.distributions %config InlineBackend.figure_format='retina' sns.set_style(\"whitegrid\") In\u00a0[\u00a0]: Copied! <pre>MODEL = \"dnn\"\nLOSS = \"ziln\"  # @param { isTemplate: true, type: 'string'} ['mse', 'ziln']\nLEARNING_RATE = 0.001  # @param { isTemplate: true}\nVERSION = 0  # @param { isTemplate: true, type: 'integer'}\nOUTPUT_CSV_FOLDER = \"/tmp/lifetime-value/kdd_cup_98/result\"  # @param { isTemplate: true, type: 'string'}\n</pre> MODEL = \"dnn\" LOSS = \"ziln\"  # @param { isTemplate: true, type: 'string'} ['mse', 'ziln'] LEARNING_RATE = 0.001  # @param { isTemplate: true} VERSION = 0  # @param { isTemplate: true, type: 'integer'} OUTPUT_CSV_FOLDER = \"/tmp/lifetime-value/kdd_cup_98/result\"  # @param { isTemplate: true, type: 'string'} <p>Download kdd_cup_98 data to /tmp/lifetime-value/kdd_cup_98</p> In\u00a0[\u00a0]: Copied! <pre>%%bash\nmkdir -p /tmp/lifetime-value/kdd_cup_98\nwget https://kdd.ics.uci.edu/databases/kddcup98/epsilon_mirror/cup98lrn.zip -P /tmp/lifetime-value/kdd_cup_98/\nwget https://kdd.ics.uci.edu/databases/kddcup98/epsilon_mirror/cup98val.zip -P /tmp/lifetime-value/kdd_cup_98/\nwget https://kdd.ics.uci.edu/databases/kddcup98/epsilon_mirror/valtargt.txt -P /tmp/lifetime-value/kdd_cup_98/\ncd /tmp/lifetime-value/kdd_cup_98/\nunzip cup98lrn.zip\nunzip cup98val.zip\n</pre> %%bash mkdir -p /tmp/lifetime-value/kdd_cup_98 wget https://kdd.ics.uci.edu/databases/kddcup98/epsilon_mirror/cup98lrn.zip -P /tmp/lifetime-value/kdd_cup_98/ wget https://kdd.ics.uci.edu/databases/kddcup98/epsilon_mirror/cup98val.zip -P /tmp/lifetime-value/kdd_cup_98/ wget https://kdd.ics.uci.edu/databases/kddcup98/epsilon_mirror/valtargt.txt -P /tmp/lifetime-value/kdd_cup_98/ cd /tmp/lifetime-value/kdd_cup_98/ unzip cup98lrn.zip unzip cup98val.zip In\u00a0[\u00a0]: Copied! <pre>df_train = pd.read_csv(\"/tmp/lifetime-value/kdd_cup_98/cup98LRN.txt\")\nnum_train = df_train.shape[0]\ndf_eval = pd.read_csv(\"/tmp/lifetime-value/kdd_cup_98/cup98VAL.txt\")\ndf_eval_target = pd.read_csv(\"/tmp/lifetime-value/kdd_cup_98/valtargt.txt\")\ndf_eval = df_eval.merge(df_eval_target, on=\"CONTROLN\")\n</pre> df_train = pd.read_csv(\"/tmp/lifetime-value/kdd_cup_98/cup98LRN.txt\") num_train = df_train.shape[0] df_eval = pd.read_csv(\"/tmp/lifetime-value/kdd_cup_98/cup98VAL.txt\") df_eval_target = pd.read_csv(\"/tmp/lifetime-value/kdd_cup_98/valtargt.txt\") df_eval = df_eval.merge(df_eval_target, on=\"CONTROLN\") In\u00a0[\u00a0]: Copied! <pre>df = pd.concat([df_train, df_eval], axis=0, sort=True)\n</pre> df = pd.concat([df_train, df_eval], axis=0, sort=True) In\u00a0[\u00a0]: Copied! <pre>y = df[\"TARGET_D\"][:num_train]\n</pre> y = df[\"TARGET_D\"][:num_train] In\u00a0[\u00a0]: Copied! <pre>def plot_hist_log_scale(y):\n    max_val = y.max() + 1.0\n    ax = pd.Series(y).hist(\n        figsize=(8, 5), bins=10 ** np.linspace(0.0, np.log10(max_val), 20)\n    )\n\n    plt.xlabel(\"Donation ($)\")\n    plt.ylabel(\"Count\")\n    # plt.title('Histogram of LTV')\n    plt.xticks(rotation=\"horizontal\")\n    plt.legend(loc=\"upper left\")\n    ax.set_xscale(\"log\")\n    ax.grid(False)\n    # Hide the right and top spines\n    ax.spines[\"right\"].set_visible(False)\n    ax.spines[\"top\"].set_visible(False)\n    # Only show ticks on the left and bottom spines\n    ax.yaxis.set_ticks_position(\"left\")\n    ax.xaxis.set_ticks_position(\"bottom\")\n    plt.show()\n\n    fig = ax.get_figure()\n    output_file = tf.io.gfile.GFile(\n        \"/tmp/lifetime-value/kdd_cup_98/histogram_kdd98_log_scale.pdf\", \"wb\"\n    )\n    fig.savefig(output_file, bbox_inches=\"tight\", format=\"pdf\")\n</pre> def plot_hist_log_scale(y):     max_val = y.max() + 1.0     ax = pd.Series(y).hist(         figsize=(8, 5), bins=10 ** np.linspace(0.0, np.log10(max_val), 20)     )      plt.xlabel(\"Donation ($)\")     plt.ylabel(\"Count\")     # plt.title('Histogram of LTV')     plt.xticks(rotation=\"horizontal\")     plt.legend(loc=\"upper left\")     ax.set_xscale(\"log\")     ax.grid(False)     # Hide the right and top spines     ax.spines[\"right\"].set_visible(False)     ax.spines[\"top\"].set_visible(False)     # Only show ticks on the left and bottom spines     ax.yaxis.set_ticks_position(\"left\")     ax.xaxis.set_ticks_position(\"bottom\")     plt.show()      fig = ax.get_figure()     output_file = tf.io.gfile.GFile(         \"/tmp/lifetime-value/kdd_cup_98/histogram_kdd98_log_scale.pdf\", \"wb\"     )     fig.savefig(output_file, bbox_inches=\"tight\", format=\"pdf\") In\u00a0[\u00a0]: Copied! <pre>plot_hist_log_scale(y[y &gt; 0])\n</pre> plot_hist_log_scale(y[y &gt; 0]) In\u00a0[\u00a0]: Copied! <pre>VOCAB_FEATURES = [\n    \"ODATEDW\",  # date of donor's first gift (YYMM)\n    \"OSOURCE\",  # donor acquisition mailing list\n    \"TCODE\",  # donor title code\n    \"STATE\",\n    \"ZIP\",\n    \"DOMAIN\",  # urbanicity level and socio-economic status of the neighborhood\n    \"CLUSTER\",  # socio-economic status\n    \"GENDER\",\n    \"MAXADATE\",  # date of the most recent promotion received\n    \"MINRDATE\",\n    \"LASTDATE\",\n    \"FISTDATE\",\n    \"RFA_2A\",\n]\n</pre> VOCAB_FEATURES = [     \"ODATEDW\",  # date of donor's first gift (YYMM)     \"OSOURCE\",  # donor acquisition mailing list     \"TCODE\",  # donor title code     \"STATE\",     \"ZIP\",     \"DOMAIN\",  # urbanicity level and socio-economic status of the neighborhood     \"CLUSTER\",  # socio-economic status     \"GENDER\",     \"MAXADATE\",  # date of the most recent promotion received     \"MINRDATE\",     \"LASTDATE\",     \"FISTDATE\",     \"RFA_2A\", ] In\u00a0[\u00a0]: Copied! <pre>df[\"ODATEDW\"] = df[\"ODATEDW\"].astype(\"str\")\ndf[\"TCODE\"] = df[\"TCODE\"].apply(lambda x: \"{:03d}\".format(x // 1000 if x &gt; 1000 else x))\ndf[\"ZIP\"] = df[\"ZIP\"].str.slice(0, 5)\ndf[\"MAXADATE\"] = df[\"MAXADATE\"].astype(\"str\")\ndf[\"MINRDATE\"] = df[\"MINRDATE\"].astype(\"str\")\ndf[\"LASTDATE\"] = df[\"LASTDATE\"].astype(\"str\")\ndf[\"FISTDATE\"] = df[\"FISTDATE\"].astype(\"str\")\n</pre> df[\"ODATEDW\"] = df[\"ODATEDW\"].astype(\"str\") df[\"TCODE\"] = df[\"TCODE\"].apply(lambda x: \"{:03d}\".format(x // 1000 if x &gt; 1000 else x)) df[\"ZIP\"] = df[\"ZIP\"].str.slice(0, 5) df[\"MAXADATE\"] = df[\"MAXADATE\"].astype(\"str\") df[\"MINRDATE\"] = df[\"MINRDATE\"].astype(\"str\") df[\"LASTDATE\"] = df[\"LASTDATE\"].astype(\"str\") df[\"FISTDATE\"] = df[\"FISTDATE\"].astype(\"str\") In\u00a0[\u00a0]: Copied! <pre>def label_encoding(y, frequency_threshold=100):\n    value_counts = pd.value_counts(y)\n    categories = value_counts[value_counts &gt;= frequency_threshold].index.to_numpy()\n    # 0 indicates the unknown category.\n    return pd.Categorical(y, categories=categories).codes + 1\n</pre> def label_encoding(y, frequency_threshold=100):     value_counts = pd.value_counts(y)     categories = value_counts[value_counts &gt;= frequency_threshold].index.to_numpy()     # 0 indicates the unknown category.     return pd.Categorical(y, categories=categories).codes + 1 In\u00a0[\u00a0]: Copied! <pre>for key in VOCAB_FEATURES:\n    df[key] = label_encoding(df[key])\n</pre> for key in VOCAB_FEATURES:     df[key] = label_encoding(df[key]) In\u00a0[\u00a0]: Copied! <pre>MAIL_ORDER_RESPONSES = [\n    \"MBCRAFT\",\n    \"MBGARDEN\",\n    \"MBBOOKS\",\n    \"MBCOLECT\",\n    \"MAGFAML\",\n    \"MAGFEM\",\n    \"MAGMALE\",\n    \"PUBGARDN\",\n    \"PUBCULIN\",\n    \"PUBHLTH\",\n    \"PUBDOITY\",\n    \"PUBNEWFN\",\n    \"PUBPHOTO\",\n    \"PUBOPP\",\n    \"RFA_2F\",\n]\n</pre> MAIL_ORDER_RESPONSES = [     \"MBCRAFT\",     \"MBGARDEN\",     \"MBBOOKS\",     \"MBCOLECT\",     \"MAGFAML\",     \"MAGFEM\",     \"MAGMALE\",     \"PUBGARDN\",     \"PUBCULIN\",     \"PUBHLTH\",     \"PUBDOITY\",     \"PUBNEWFN\",     \"PUBPHOTO\",     \"PUBOPP\",     \"RFA_2F\", ] In\u00a0[\u00a0]: Copied! <pre>INDICATOR_FEATURES = [\n    \"AGE\",  # age decile, 0 indicates unknown\n    \"NUMCHLD\",\n    \"INCOME\",\n    \"WEALTH1\",\n    \"HIT\",\n] + MAIL_ORDER_RESPONSES\n</pre> INDICATOR_FEATURES = [     \"AGE\",  # age decile, 0 indicates unknown     \"NUMCHLD\",     \"INCOME\",     \"WEALTH1\",     \"HIT\", ] + MAIL_ORDER_RESPONSES In\u00a0[\u00a0]: Copied! <pre>df[\"AGE\"] = pd.qcut(df[\"AGE\"].values, 10).codes + 1\ndf[\"NUMCHLD\"] = df[\"NUMCHLD\"].apply(lambda x: 0 if np.isnan(x) else int(x))\ndf[\"INCOME\"] = df[\"INCOME\"].apply(lambda x: 0 if np.isnan(x) else int(x))\ndf[\"WEALTH1\"] = df[\"WEALTH1\"].apply(lambda x: 0 if np.isnan(x) else int(x) + 1)\ndf[\"HIT\"] = pd.qcut(df[\"HIT\"].values, q=50, duplicates=\"drop\").codes\n\nfor col in MAIL_ORDER_RESPONSES:\n    df[col] = pd.qcut(df[col].values, q=20, duplicates=\"drop\").codes + 1\n</pre> df[\"AGE\"] = pd.qcut(df[\"AGE\"].values, 10).codes + 1 df[\"NUMCHLD\"] = df[\"NUMCHLD\"].apply(lambda x: 0 if np.isnan(x) else int(x)) df[\"INCOME\"] = df[\"INCOME\"].apply(lambda x: 0 if np.isnan(x) else int(x)) df[\"WEALTH1\"] = df[\"WEALTH1\"].apply(lambda x: 0 if np.isnan(x) else int(x) + 1) df[\"HIT\"] = pd.qcut(df[\"HIT\"].values, q=50, duplicates=\"drop\").codes  for col in MAIL_ORDER_RESPONSES:     df[col] = pd.qcut(df[col].values, q=20, duplicates=\"drop\").codes + 1 In\u00a0[\u00a0]: Copied! <pre>NUMERIC_FEATURES = [\n    # binary\n    \"MAILCODE\",  # bad address\n    \"NOEXCH\",  # do not exchange\n    \"RECINHSE\",  # donor has given to PVA's in house program\n    \"RECP3\",  # donor has given to PVA's P3 program\n    \"RECPGVG\",  # planned giving record\n    \"RECSWEEP\",  # sweepstakes record\n    \"HOMEOWNR\",  # home owner\n    \"CHILD03\",\n    \"CHILD07\",\n    \"CHILD12\",\n    \"CHILD18\",\n    # continuous\n    \"CARDPROM\",\n    \"NUMPROM\",\n    \"CARDPM12\",\n    \"NUMPRM12\",\n    \"RAMNTALL\",\n    \"NGIFTALL\",\n    \"MINRAMNT\",\n    \"MAXRAMNT\",\n    \"LASTGIFT\",\n    \"AVGGIFT\",\n]\n</pre> NUMERIC_FEATURES = [     # binary     \"MAILCODE\",  # bad address     \"NOEXCH\",  # do not exchange     \"RECINHSE\",  # donor has given to PVA's in house program     \"RECP3\",  # donor has given to PVA's P3 program     \"RECPGVG\",  # planned giving record     \"RECSWEEP\",  # sweepstakes record     \"HOMEOWNR\",  # home owner     \"CHILD03\",     \"CHILD07\",     \"CHILD12\",     \"CHILD18\",     # continuous     \"CARDPROM\",     \"NUMPROM\",     \"CARDPM12\",     \"NUMPRM12\",     \"RAMNTALL\",     \"NGIFTALL\",     \"MINRAMNT\",     \"MAXRAMNT\",     \"LASTGIFT\",     \"AVGGIFT\", ] In\u00a0[\u00a0]: Copied! <pre>df[\"MAILCODE\"] = (df[\"MAILCODE\"] == \"B\").astype(\"float32\")\ndf[\"PVASTATE\"] = df[\"PVASTATE\"].isin([\"P\", \"E\"]).astype(\"float32\")\ndf[\"NOEXCH\"] = df[\"NOEXCH\"].isin([\"X\", \"1\"]).astype(\"float32\")\ndf[\"RECINHSE\"] = (df[\"RECINHSE\"] == \"X\").astype(\"float32\")\ndf[\"RECP3\"] = (df[\"RECP3\"] == \"X\").astype(\"float32\")\ndf[\"RECPGVG\"] = (df[\"RECPGVG\"] == \"X\").astype(\"float32\")\ndf[\"RECSWEEP\"] = (df[\"RECSWEEP\"] == \"X\").astype(\"float32\")\ndf[\"HOMEOWNR\"] = (df[\"HOMEOWNR\"] == \"H\").astype(\"float32\")\ndf[\"CHILD03\"] = df[\"CHILD03\"].isin([\"M\", \"F\", \"B\"]).astype(\"float32\")\ndf[\"CHILD07\"] = df[\"CHILD07\"].isin([\"M\", \"F\", \"B\"]).astype(\"float32\")\ndf[\"CHILD12\"] = df[\"CHILD12\"].isin([\"M\", \"F\", \"B\"]).astype(\"float32\")\ndf[\"CHILD18\"] = df[\"CHILD18\"].isin([\"M\", \"F\", \"B\"]).astype(\"float32\")\n\ndf[\"CARDPROM\"] = df[\"CARDPROM\"] / 100\ndf[\"NUMPROM\"] = df[\"NUMPROM\"] / 100\ndf[\"CARDPM12\"] = df[\"CARDPM12\"] / 100\ndf[\"NUMPRM12\"] = df[\"NUMPRM12\"] / 100\ndf[\"RAMNTALL\"] = np.log1p(df[\"RAMNTALL\"])\ndf[\"NGIFTALL\"] = np.log1p(df[\"NGIFTALL\"])\ndf[\"MINRAMNT\"] = np.log1p(df[\"MINRAMNT\"])\ndf[\"MAXRAMNT\"] = np.log1p(df[\"MAXRAMNT\"])\ndf[\"LASTGIFT\"] = np.log1p(df[\"LASTGIFT\"])\ndf[\"AVGGIFT\"] = np.log1p(df[\"AVGGIFT\"])\n</pre> df[\"MAILCODE\"] = (df[\"MAILCODE\"] == \"B\").astype(\"float32\") df[\"PVASTATE\"] = df[\"PVASTATE\"].isin([\"P\", \"E\"]).astype(\"float32\") df[\"NOEXCH\"] = df[\"NOEXCH\"].isin([\"X\", \"1\"]).astype(\"float32\") df[\"RECINHSE\"] = (df[\"RECINHSE\"] == \"X\").astype(\"float32\") df[\"RECP3\"] = (df[\"RECP3\"] == \"X\").astype(\"float32\") df[\"RECPGVG\"] = (df[\"RECPGVG\"] == \"X\").astype(\"float32\") df[\"RECSWEEP\"] = (df[\"RECSWEEP\"] == \"X\").astype(\"float32\") df[\"HOMEOWNR\"] = (df[\"HOMEOWNR\"] == \"H\").astype(\"float32\") df[\"CHILD03\"] = df[\"CHILD03\"].isin([\"M\", \"F\", \"B\"]).astype(\"float32\") df[\"CHILD07\"] = df[\"CHILD07\"].isin([\"M\", \"F\", \"B\"]).astype(\"float32\") df[\"CHILD12\"] = df[\"CHILD12\"].isin([\"M\", \"F\", \"B\"]).astype(\"float32\") df[\"CHILD18\"] = df[\"CHILD18\"].isin([\"M\", \"F\", \"B\"]).astype(\"float32\")  df[\"CARDPROM\"] = df[\"CARDPROM\"] / 100 df[\"NUMPROM\"] = df[\"NUMPROM\"] / 100 df[\"CARDPM12\"] = df[\"CARDPM12\"] / 100 df[\"NUMPRM12\"] = df[\"NUMPRM12\"] / 100 df[\"RAMNTALL\"] = np.log1p(df[\"RAMNTALL\"]) df[\"NGIFTALL\"] = np.log1p(df[\"NGIFTALL\"]) df[\"MINRAMNT\"] = np.log1p(df[\"MINRAMNT\"]) df[\"MAXRAMNT\"] = np.log1p(df[\"MAXRAMNT\"]) df[\"LASTGIFT\"] = np.log1p(df[\"LASTGIFT\"]) df[\"AVGGIFT\"] = np.log1p(df[\"AVGGIFT\"]) In\u00a0[\u00a0]: Copied! <pre>CATEGORICAL_FEATURES = VOCAB_FEATURES + INDICATOR_FEATURES\nALL_FEATURES = CATEGORICAL_FEATURES + NUMERIC_FEATURES\n</pre> CATEGORICAL_FEATURES = VOCAB_FEATURES + INDICATOR_FEATURES ALL_FEATURES = CATEGORICAL_FEATURES + NUMERIC_FEATURES In\u00a0[\u00a0]: Copied! <pre>def dnn_split(df):\n    df_train = df.iloc[:num_train]\n    df_eval = df.iloc[num_train:]\n\n    def feature_dict(df):\n        features = {k: v.values for k, v in dict(df[CATEGORICAL_FEATURES]).items()}\n        features[\"numeric\"] = df[NUMERIC_FEATURES].astype(\"float32\").values\n        return features\n\n    x_train, y_train = (\n        feature_dict(df_train),\n        df_train[\"TARGET_D\"].astype(\"float32\").values,\n    )\n    x_eval, y_eval = feature_dict(df_eval), df_eval[\"TARGET_D\"].astype(\"float32\").values\n\n    return x_train, x_eval, y_train, y_eval\n</pre> def dnn_split(df):     df_train = df.iloc[:num_train]     df_eval = df.iloc[num_train:]      def feature_dict(df):         features = {k: v.values for k, v in dict(df[CATEGORICAL_FEATURES]).items()}         features[\"numeric\"] = df[NUMERIC_FEATURES].astype(\"float32\").values         return features      x_train, y_train = (         feature_dict(df_train),         df_train[\"TARGET_D\"].astype(\"float32\").values,     )     x_eval, y_eval = feature_dict(df_eval), df_eval[\"TARGET_D\"].astype(\"float32\").values      return x_train, x_eval, y_train, y_eval In\u00a0[\u00a0]: Copied! <pre>def embedding_dim(x):\n    return int(x**0.25) + 1\n\n\ndef embedding_layer(vocab_size):\n    return tf.keras.Sequential(\n        [\n            tf.keras.layers.Embedding(\n                input_dim=vocab_size,\n                output_dim=embedding_dim(vocab_size),\n                input_length=1,\n            ),\n            tf.keras.layers.Flatten(),\n        ]\n    )\n\n\ndef dnn_model(output_units):\n    numeric_input = tf.keras.layers.Input(\n        shape=(len(NUMERIC_FEATURES),), name=\"numeric\"\n    )\n\n    embedding_inputs = [\n        tf.keras.layers.Input(shape=(1,), name=key, dtype=np.int64)\n        for key in CATEGORICAL_FEATURES\n    ]\n\n    embedding_outputs = [\n        embedding_layer(vocab_size=df[key].max() + 1)(input)\n        for key, input in zip(CATEGORICAL_FEATURES, embedding_inputs)\n    ]\n\n    deep_input = tf.keras.layers.concatenate([numeric_input] + embedding_outputs)\n    deep_model = tf.keras.Sequential(\n        [\n            tf.keras.layers.Dense(128, activation=\"relu\"),\n            tf.keras.layers.Dense(128, activation=\"relu\"),\n            tf.keras.layers.Dense(64, activation=\"relu\"),\n            tf.keras.layers.Dense(64, activation=\"relu\"),\n            tf.keras.layers.Dense(units=output_units),\n        ]\n    )\n    return tf.keras.Model(\n        inputs=[numeric_input] + embedding_inputs, outputs=deep_model(deep_input)\n    )\n</pre> def embedding_dim(x):     return int(x**0.25) + 1   def embedding_layer(vocab_size):     return tf.keras.Sequential(         [             tf.keras.layers.Embedding(                 input_dim=vocab_size,                 output_dim=embedding_dim(vocab_size),                 input_length=1,             ),             tf.keras.layers.Flatten(),         ]     )   def dnn_model(output_units):     numeric_input = tf.keras.layers.Input(         shape=(len(NUMERIC_FEATURES),), name=\"numeric\"     )      embedding_inputs = [         tf.keras.layers.Input(shape=(1,), name=key, dtype=np.int64)         for key in CATEGORICAL_FEATURES     ]      embedding_outputs = [         embedding_layer(vocab_size=df[key].max() + 1)(input)         for key, input in zip(CATEGORICAL_FEATURES, embedding_inputs)     ]      deep_input = tf.keras.layers.concatenate([numeric_input] + embedding_outputs)     deep_model = tf.keras.Sequential(         [             tf.keras.layers.Dense(128, activation=\"relu\"),             tf.keras.layers.Dense(128, activation=\"relu\"),             tf.keras.layers.Dense(64, activation=\"relu\"),             tf.keras.layers.Dense(64, activation=\"relu\"),             tf.keras.layers.Dense(units=output_units),         ]     )     return tf.keras.Model(         inputs=[numeric_input] + embedding_inputs, outputs=deep_model(deep_input)     ) In\u00a0[\u00a0]: Copied! <pre>if LOSS == \"mse\":\n    loss = tf.keras.losses.MeanSquaredError()\n    output_units = 1\n\nif LOSS == \"ziln\":\n    loss = ltv.zero_inflated_lognormal_loss\n    output_units = 3\n</pre> if LOSS == \"mse\":     loss = tf.keras.losses.MeanSquaredError()     output_units = 1  if LOSS == \"ziln\":     loss = ltv.zero_inflated_lognormal_loss     output_units = 3 In\u00a0[\u00a0]: Copied! <pre>x_train, x_eval, y_train, y_eval = dnn_split(df)\nmodel = dnn_model(output_units)\n</pre> x_train, x_eval, y_train, y_eval = dnn_split(df) model = dnn_model(output_units) In\u00a0[\u00a0]: Copied! <pre>model.compile(optimizer=tf.keras.optimizers.Nadam(lr=LEARNING_RATE), loss=loss)\n</pre> model.compile(optimizer=tf.keras.optimizers.Nadam(lr=LEARNING_RATE), loss=loss) In\u00a0[\u00a0]: Copied! <pre>callbacks = [\n    tf.keras.callbacks.ReduceLROnPlateau(monitor=\"val_loss\", min_lr=1e-6),\n    tf.keras.callbacks.EarlyStopping(monitor=\"val_loss\", patience=10),\n]\n</pre> callbacks = [     tf.keras.callbacks.ReduceLROnPlateau(monitor=\"val_loss\", min_lr=1e-6),     tf.keras.callbacks.EarlyStopping(monitor=\"val_loss\", patience=10), ] In\u00a0[\u00a0]: Copied! <pre>history = model.fit(\n    x=x_train,\n    y=y_train,\n    batch_size=2048,\n    epochs=200,\n    verbose=2,\n    callbacks=callbacks,\n    validation_data=(x_eval, y_eval),\n).history\n</pre> history = model.fit(     x=x_train,     y=y_train,     batch_size=2048,     epochs=200,     verbose=2,     callbacks=callbacks,     validation_data=(x_eval, y_eval), ).history In\u00a0[\u00a0]: Copied! <pre>pd.DataFrame(history)[[\"loss\", \"val_loss\"]].plot();\n</pre> pd.DataFrame(history)[[\"loss\", \"val_loss\"]].plot(); In\u00a0[\u00a0]: Copied! <pre>if LOSS == \"mse\":\n    y_pred = model.predict(x=x_eval, batch_size=1024).flatten()\n\nif LOSS == \"ziln\":\n    logits = model.predict(x=x_eval, batch_size=1024)\n    y_pred = ltv.zero_inflated_lognormal_pred(logits).numpy().flatten()\n</pre> if LOSS == \"mse\":     y_pred = model.predict(x=x_eval, batch_size=1024).flatten()  if LOSS == \"ziln\":     logits = model.predict(x=x_eval, batch_size=1024)     y_pred = ltv.zero_inflated_lognormal_pred(logits).numpy().flatten() In\u00a0[\u00a0]: Copied! <pre>from pytorch_widedeep.preprocessing import TabPreprocessor\nfrom pytorch_widedeep.training import Trainer\nfrom pytorch_widedeep.models import TabMlp, WideDeep\nfrom torch.optim.lr_scheduler import ReduceLROnPlateau\nfrom pytorch_widedeep.callbacks import EarlyStopping\nfrom torch.optim import NAdam\n</pre> from pytorch_widedeep.preprocessing import TabPreprocessor from pytorch_widedeep.training import Trainer from pytorch_widedeep.models import TabMlp, WideDeep from torch.optim.lr_scheduler import ReduceLROnPlateau from pytorch_widedeep.callbacks import EarlyStopping from torch.optim import NAdam In\u00a0[\u00a0]: Copied! <pre># CATEGORICAL_FEATURES\nNUMERICAL_FEATURES = [\"num\" + str(i) for i in range(21)]\nx_train_pyt_num = pd.DataFrame(columns=NUMERICAL_FEATURES, data=x_train[\"numeric\"])\nx_train_pyt_cat = pd.DataFrame(\n    {key: value for key, value in x_train.items() if key not in [\"numeric\"]}\n)\n\nx_eval_pyt_num = pd.DataFrame(columns=NUMERICAL_FEATURES, data=x_eval[\"numeric\"])\nx_eval_pyt_cat = pd.DataFrame(\n    {key: value for key, value in x_eval.items() if key not in [\"numeric\"]}\n)\n</pre> # CATEGORICAL_FEATURES NUMERICAL_FEATURES = [\"num\" + str(i) for i in range(21)] x_train_pyt_num = pd.DataFrame(columns=NUMERICAL_FEATURES, data=x_train[\"numeric\"]) x_train_pyt_cat = pd.DataFrame(     {key: value for key, value in x_train.items() if key not in [\"numeric\"]} )  x_eval_pyt_num = pd.DataFrame(columns=NUMERICAL_FEATURES, data=x_eval[\"numeric\"]) x_eval_pyt_cat = pd.DataFrame(     {key: value for key, value in x_eval.items() if key not in [\"numeric\"]} ) In\u00a0[\u00a0]: Copied! <pre>x_train_pyt = pd.concat([x_train_pyt_num, x_train_pyt_cat], axis=1)\nx_eval_pyt = pd.concat([x_eval_pyt_num, x_eval_pyt_cat], axis=1)\n</pre> x_train_pyt = pd.concat([x_train_pyt_num, x_train_pyt_cat], axis=1) x_eval_pyt = pd.concat([x_eval_pyt_num, x_eval_pyt_cat], axis=1) In\u00a0[\u00a0]: Copied! <pre>embed_input = [\n    (u, int(x_train_pyt[u].nunique() ** 0.25) + 1) for u in CATEGORICAL_FEATURES\n]\n</pre> embed_input = [     (u, int(x_train_pyt[u].nunique() ** 0.25) + 1) for u in CATEGORICAL_FEATURES ] In\u00a0[\u00a0]: Copied! <pre># deeptabular\ntab_preprocessor = TabPreprocessor(\n    embed_cols=embed_input,\n    continuous_cols=NUMERICAL_FEATURES,\n    shared_embed=False,\n    scale=False,\n)\nX_tab_train = tab_preprocessor.fit_transform(x_train_pyt)\nX_tab_valid = tab_preprocessor.transform(x_eval_pyt)\nX_tab_test = tab_preprocessor.transform(x_eval_pyt)\n\n# target\ny_train = y_train\ny_valid = y_eval\ny_test = y_train\n\nX_train = {\"X_tab\": X_tab_train, \"target\": y_train}\nX_val = {\"X_tab\": X_tab_valid, \"target\": y_valid}\nX_test = {\"X_tab\": X_tab_test}\n\ndeeptabular = TabMlp(\n    mlp_hidden_dims=[128, 128, 64, 64],\n    column_idx=tab_preprocessor.column_idx,\n    embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=tab_preprocessor.continuous_cols,\n)\n\nmodel = WideDeep(deeptabular=deeptabular, pred_dim=3)\n\ndeep_opt = NAdam(model.deeptabular.parameters(), lr=LEARNING_RATE)\ncallbacks = [EarlyStopping()]\ndeep_sch = ReduceLROnPlateau(deep_opt, min_lr=1e-6)\n\nobjective = \"ziln\"\n\ntrainer = Trainer(\n    model,\n    callbacks=callbacks,\n    lr_schedulers={\"deeptabular\": deep_sch},\n    objective=objective,\n    optimizers={\"deeptabular\": deep_opt},\n)\n\ntrainer.fit(\n    X_train=X_train,\n    X_val=X_val,\n    n_epochs=200,\n    batch_size=2048,\n)\n\ny_pred_pytorch = trainer.predict(X_test=X_test)\n</pre> # deeptabular tab_preprocessor = TabPreprocessor(     embed_cols=embed_input,     continuous_cols=NUMERICAL_FEATURES,     shared_embed=False,     scale=False, ) X_tab_train = tab_preprocessor.fit_transform(x_train_pyt) X_tab_valid = tab_preprocessor.transform(x_eval_pyt) X_tab_test = tab_preprocessor.transform(x_eval_pyt)  # target y_train = y_train y_valid = y_eval y_test = y_train  X_train = {\"X_tab\": X_tab_train, \"target\": y_train} X_val = {\"X_tab\": X_tab_valid, \"target\": y_valid} X_test = {\"X_tab\": X_tab_test}  deeptabular = TabMlp(     mlp_hidden_dims=[128, 128, 64, 64],     column_idx=tab_preprocessor.column_idx,     embed_input=tab_preprocessor.cat_embed_input,     continuous_cols=tab_preprocessor.continuous_cols, )  model = WideDeep(deeptabular=deeptabular, pred_dim=3)  deep_opt = NAdam(model.deeptabular.parameters(), lr=LEARNING_RATE) callbacks = [EarlyStopping()] deep_sch = ReduceLROnPlateau(deep_opt, min_lr=1e-6)  objective = \"ziln\"  trainer = Trainer(     model,     callbacks=callbacks,     lr_schedulers={\"deeptabular\": deep_sch},     objective=objective,     optimizers={\"deeptabular\": deep_opt}, )  trainer.fit(     X_train=X_train,     X_val=X_val,     n_epochs=200,     batch_size=2048, )  y_pred_pytorch = trainer.predict(X_test=X_test) In\u00a0[\u00a0]: Copied! <pre>pd.DataFrame(trainer.history)[[\"train_loss\", \"val_loss\"]].plot();\n</pre> pd.DataFrame(trainer.history)[[\"train_loss\", \"val_loss\"]].plot(); In\u00a0[\u00a0]: Copied! <pre>from sklearn.metrics import mean_squared_error\n\nmean_squared_error(y_pred, y_pred_pytorch)\n</pre> from sklearn.metrics import mean_squared_error  mean_squared_error(y_pred, y_pred_pytorch) In\u00a0[\u00a0]: Copied! <pre>unit_costs = [0.4, 0.5, 0.6, 0.68, 0.7, 0.8, 0.9, 1.0]\n</pre> unit_costs = [0.4, 0.5, 0.6, 0.68, 0.7, 0.8, 0.9, 1.0] In\u00a0[\u00a0]: Copied! <pre>num_mailed = [np.sum(y_pred &gt; v) for v in unit_costs]\nnum_mailed\n</pre> num_mailed = [np.sum(y_pred &gt; v) for v in unit_costs] num_mailed In\u00a0[\u00a0]: Copied! <pre>baseline_total_profit = np.sum(y_eval - 0.68)\nbaseline_total_profit\n</pre> baseline_total_profit = np.sum(y_eval - 0.68) baseline_total_profit In\u00a0[\u00a0]: Copied! <pre>total_profits = [np.sum(y_eval[y_pred &gt; v] - v) for v in unit_costs]\ntotal_profits\n</pre> total_profits = [np.sum(y_eval[y_pred &gt; v] - v) for v in unit_costs] total_profits In\u00a0[\u00a0]: Copied! <pre>gain = pd.DataFrame(\n    {\n        \"lorenz\": ltv.cumulative_true(y_eval, y_eval),\n        \"baseline\": ltv.cumulative_true(y_eval, x_eval[\"numeric\"][:, 19]),\n        \"model\": ltv.cumulative_true(y_eval, y_pred),\n    }\n)\n</pre> gain = pd.DataFrame(     {         \"lorenz\": ltv.cumulative_true(y_eval, y_eval),         \"baseline\": ltv.cumulative_true(y_eval, x_eval[\"numeric\"][:, 19]),         \"model\": ltv.cumulative_true(y_eval, y_pred),     } ) In\u00a0[\u00a0]: Copied! <pre>num_customers = np.float32(gain.shape[0])\ngain[\"cumulative_customer\"] = (np.arange(num_customers) + 1.0) / num_customers\n</pre> num_customers = np.float32(gain.shape[0]) gain[\"cumulative_customer\"] = (np.arange(num_customers) + 1.0) / num_customers In\u00a0[\u00a0]: Copied! <pre>ax = gain[\n    [\n        \"cumulative_customer\",\n        \"lorenz\",\n        \"baseline\",\n        \"model\",\n    ]\n].plot(x=\"cumulative_customer\", figsize=(8, 5), legend=True)\n\nax.legend([\"Groundtruth\", \"Baseline\", \"Model\"], loc=\"lower right\")\n\nax.set_xlabel(\"Cumulative Fraction of Customers\")\nax.set_xticks(np.arange(0, 1.1, 0.1))\nax.set_xlim((0, 1.0))\n\nax.set_ylabel(\"Cumulative Fraction of Total Lifetime Value\")\nax.set_yticks(np.arange(0, 1.1, 0.1))\nax.set_ylim((0, 1.05))\nax.set_title(\"Gain Chart\");\n</pre> ax = gain[     [         \"cumulative_customer\",         \"lorenz\",         \"baseline\",         \"model\",     ] ].plot(x=\"cumulative_customer\", figsize=(8, 5), legend=True)  ax.legend([\"Groundtruth\", \"Baseline\", \"Model\"], loc=\"lower right\")  ax.set_xlabel(\"Cumulative Fraction of Customers\") ax.set_xticks(np.arange(0, 1.1, 0.1)) ax.set_xlim((0, 1.0))  ax.set_ylabel(\"Cumulative Fraction of Total Lifetime Value\") ax.set_yticks(np.arange(0, 1.1, 0.1)) ax.set_ylim((0, 1.05)) ax.set_title(\"Gain Chart\"); In\u00a0[\u00a0]: Copied! <pre>gini = ltv.gini_from_gain(gain[[\"lorenz\", \"baseline\", \"model\"]])\ngini\n</pre> gini = ltv.gini_from_gain(gain[[\"lorenz\", \"baseline\", \"model\"]]) gini In\u00a0[\u00a0]: Copied! <pre>df_decile = ltv.decile_stats(y_eval, y_pred)\ndf_decile\n</pre> df_decile = ltv.decile_stats(y_eval, y_pred) df_decile In\u00a0[\u00a0]: Copied! <pre>ax = df_decile[[\"label_mean\", \"pred_mean\"]].plot.bar(rot=0)\n\nax.set_title(\"Decile Chart\")\nax.set_xlabel(\"Prediction bucket\")\nax.set_ylabel(\"Average bucket value\")\nax.legend([\"Label\", \"Prediction\"], loc=\"upper left\");\n</pre> ax = df_decile[[\"label_mean\", \"pred_mean\"]].plot.bar(rot=0)  ax.set_title(\"Decile Chart\") ax.set_xlabel(\"Prediction bucket\") ax.set_ylabel(\"Average bucket value\") ax.legend([\"Label\", \"Prediction\"], loc=\"upper left\"); In\u00a0[\u00a0]: Copied! <pre>def spearmanr(x1: Sequence[float], x2: Sequence[float]) -&gt; float:\n    \"\"\"Calculates spearmanr rank correlation coefficient.\n\n    See https://docs.scipy.org/doc/scipy/reference/stats.html.\n\n    Args:\n      x1: 1D array_like.\n      x2: 1D array_like.\n\n    Returns:\n      correlation: float.\n    \"\"\"\n    return stats.spearmanr(x1, x2, nan_policy=\"raise\")[0]\n\n\nspearman_corr = spearmanr(y_eval, y_pred)\nspearman_corr\n</pre> def spearmanr(x1: Sequence[float], x2: Sequence[float]) -&gt; float:     \"\"\"Calculates spearmanr rank correlation coefficient.      See https://docs.scipy.org/doc/scipy/reference/stats.html.      Args:       x1: 1D array_like.       x2: 1D array_like.      Returns:       correlation: float.     \"\"\"     return stats.spearmanr(x1, x2, nan_policy=\"raise\")[0]   spearman_corr = spearmanr(y_eval, y_pred) spearman_corr In\u00a0[\u00a0]: Copied! <pre>df_metrics = pd.DataFrame(\n    {\n        \"model\": MODEL,\n        \"loss_function\": LOSS,\n        \"train_loss\": history[\"loss\"][-1],\n        \"eval_loss\": history[\"val_loss\"][-1],\n        \"label_positive\": np.mean(y_eval &gt; 0),\n        \"label_mean\": y_eval.mean(),\n        \"pred_mean\": y_pred.mean(),\n        \"decile_mape\": df_decile[\"decile_mape\"].mean(),\n        \"baseline_gini\": gini[\"normalized\"][1],\n        \"gini\": gini[\"normalized\"][2],\n        \"spearman_corr\": spearman_corr,\n    },\n    index=[VERSION],\n)\n</pre> df_metrics = pd.DataFrame(     {         \"model\": MODEL,         \"loss_function\": LOSS,         \"train_loss\": history[\"loss\"][-1],         \"eval_loss\": history[\"val_loss\"][-1],         \"label_positive\": np.mean(y_eval &gt; 0),         \"label_mean\": y_eval.mean(),         \"pred_mean\": y_pred.mean(),         \"decile_mape\": df_decile[\"decile_mape\"].mean(),         \"baseline_gini\": gini[\"normalized\"][1],         \"gini\": gini[\"normalized\"][2],         \"spearman_corr\": spearman_corr,     },     index=[VERSION], ) In\u00a0[\u00a0]: Copied! <pre>for unit_cost, total_profit in zip(unit_costs, total_profits):\n    df_metrics[\"total_profit_{:02d}\".format(int(unit_cost * 100))] = total_profit\n</pre> for unit_cost, total_profit in zip(unit_costs, total_profits):     df_metrics[\"total_profit_{:02d}\".format(int(unit_cost * 100))] = total_profit In\u00a0[\u00a0]: Copied! <pre>df_metrics.T\n</pre> df_metrics.T In\u00a0[\u00a0]: Copied! <pre>output_path = OUTPUT_CSV_FOLDER\n</pre> output_path = OUTPUT_CSV_FOLDER In\u00a0[\u00a0]: Copied! <pre>if not os.path.isdir(output_path):\n    os.makedirs(output_path)\n</pre> if not os.path.isdir(output_path):     os.makedirs(output_path) In\u00a0[\u00a0]: Copied! <pre>output_file = os.path.join(\n    output_path, \"{}_regression_{}_{}.csv\".format(MODEL, LOSS, VERSION)\n)\n</pre> output_file = os.path.join(     output_path, \"{}_regression_{}_{}.csv\".format(MODEL, LOSS, VERSION) ) In\u00a0[\u00a0]: Copied! <pre>df_metrics.to_csv(output_file, index=False)\n</pre> df_metrics.to_csv(output_file, index=False)"},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#zilnloss","title":"ZILNLoss\u00b6","text":"<p>[DISCLAIMER]</p> <p>Purpose of this notebook is to check if ZILNloss implemented originaly Keras give same results in pytorch-widedeep implemenatation</p>"},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#kdd-cup-98-ltv-prediction","title":"KDD Cup 98 LTV Prediction\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#configs","title":"Configs\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#load-data","title":"Load data\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#label-distribution","title":"Label distribution\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#preprocess-features","title":"Preprocess features\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#vocab","title":"Vocab\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#indicator","title":"Indicator\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#numeric","title":"Numeric\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#all","title":"All\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#traineval-split","title":"Train/eval split\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#model","title":"Model\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#loss","title":"Loss\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#train","title":"Train\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#eval","title":"Eval\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#pytorch-widedeep-approach","title":"Pytorch-widedeep approach\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#appendix","title":"Appendix\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#total-profit","title":"Total Profit\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#gini-coefficient","title":"Gini Coefficient\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#calibration","title":"Calibration\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#rank-correlation","title":"Rank Correlation\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#all-metrics-together","title":"All metrics together\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#save","title":"Save\u00b6","text":""},{"location":"examples/13_model_uncertainty_prediction.html","title":"13_model_uncertainty_prediction","text":"<ul> <li>In this notebook we will use the higly imbalanced Protein Homology Dataset from KDD cup 2004</li> </ul> <pre><code>* The first element of each line is a BLOCK ID that denotes to which native sequence this example belongs. There is a unique BLOCK ID for each native sequence. BLOCK IDs are integers running from 1 to 303 (one for each native sequence, i.e. for each query). BLOCK IDs were assigned before the blocks were split into the train and test sets, so they do not run consecutively in either file.\n* The second element of each line is an EXAMPLE ID that uniquely describes the example. You will need this EXAMPLE ID and the BLOCK ID when you submit results.\n* The third element is the class of the example. Proteins that are homologous to the native sequence are denoted by 1, non-homologous proteins (i.e. decoys) by 0. Test examples have a \"?\" in this position.\n* All following elements are feature values. There are 74 feature values in each line. The features describe the match (e.g. the score of a sequence alignment) between the native protein sequence and the sequence that is tested for homology.\n</code></pre> In\u00a0[1]: Copied! <pre>import numpy as np\nimport pandas as pd\nimport torch\nfrom torch.optim import SGD, lr_scheduler\n\nfrom pytorch_widedeep import Trainer\nfrom pytorch_widedeep.preprocessing import TabPreprocessor\nfrom pytorch_widedeep.models import TabMlp, WideDeep\nfrom pytorch_widedeep.dataloaders import DataLoaderImbalanced, DataLoaderDefault\nfrom torchmetrics import F1Score as F1_torchmetrics\nfrom torchmetrics import Accuracy as Accuracy_torchmetrics\nfrom torchmetrics import Precision as Precision_torchmetrics\nfrom torchmetrics import Recall as Recall_torchmetrics\nfrom pytorch_widedeep.metrics import Accuracy, Recall, Precision, F1Score, R2Score\nfrom pytorch_widedeep.initializers import XavierNormal\nfrom pytorch_widedeep.datasets import load_bio_kdd04\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import classification_report\n\nimport time\nimport datetime\n\nimport warnings\n\nwarnings.filterwarnings(\"ignore\", category=DeprecationWarning)\n\n# increase displayed columns in jupyter notebook\npd.set_option(\"display.max_columns\", 200)\npd.set_option(\"display.max_rows\", 300)\n</pre> import numpy as np import pandas as pd import torch from torch.optim import SGD, lr_scheduler  from pytorch_widedeep import Trainer from pytorch_widedeep.preprocessing import TabPreprocessor from pytorch_widedeep.models import TabMlp, WideDeep from pytorch_widedeep.dataloaders import DataLoaderImbalanced, DataLoaderDefault from torchmetrics import F1Score as F1_torchmetrics from torchmetrics import Accuracy as Accuracy_torchmetrics from torchmetrics import Precision as Precision_torchmetrics from torchmetrics import Recall as Recall_torchmetrics from pytorch_widedeep.metrics import Accuracy, Recall, Precision, F1Score, R2Score from pytorch_widedeep.initializers import XavierNormal from pytorch_widedeep.datasets import load_bio_kdd04  from sklearn.model_selection import train_test_split from sklearn.metrics import classification_report  import time import datetime  import warnings  warnings.filterwarnings(\"ignore\", category=DeprecationWarning)  # increase displayed columns in jupyter notebook pd.set_option(\"display.max_columns\", 200) pd.set_option(\"display.max_rows\", 300) In\u00a0[2]: Copied! <pre>df = load_bio_kdd04(as_frame=True)\ndf.head()\n</pre> df = load_bio_kdd04(as_frame=True) df.head() Out[2]: EXAMPLE_ID BLOCK_ID target 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 0 279 261532 0 52.0 32.69 0.30 2.5 20.0 1256.8 -0.89 0.33 11.0 -55.0 267.2 0.52 0.05 -2.36 49.6 252.0 0.43 1.16 -2.06 -33.0 -123.2 1.60 -0.49 -6.06 65.0 296.1 -0.28 -0.26 -3.83 -22.6 -170.0 3.06 -1.05 -3.29 22.9 286.3 0.12 2.58 4.08 -33.0 -178.9 1.88 0.53 -7.0 -44.0 1987.0 -5.41 0.95 -4.0 -57.0 722.9 -3.26 -0.55 -7.5 125.5 1547.2 -0.36 1.12 9.0 -37.0 72.5 0.47 0.74 -11.0 -8.0 1595.1 -1.64 2.83 -2.0 -50.0 445.2 -0.35 0.26 0.76 1 279 261533 0 58.0 33.33 0.00 16.5 9.5 608.1 0.50 0.07 20.5 -52.5 521.6 -1.08 0.58 -0.02 -3.2 103.6 -0.95 0.23 -2.87 -25.9 -52.2 -0.21 0.87 -1.81 10.4 62.0 -0.28 -0.04 1.48 -17.6 -198.3 3.43 2.84 5.87 -16.9 72.6 -0.31 2.79 2.71 -33.5 -11.6 -1.11 4.01 5.0 -57.0 666.3 1.13 4.38 5.0 -64.0 39.3 1.07 -0.16 32.5 100.0 1893.7 -2.80 -0.22 2.5 -28.5 45.0 0.58 0.41 -19.0 -6.0 762.9 0.29 0.82 -3.0 -35.0 140.3 1.16 0.39 0.73 2 279 261534 0 77.0 27.27 -0.91 6.0 58.5 1623.6 -1.40 0.02 -6.5 -48.0 621.0 -1.20 0.14 -0.20 73.6 609.1 -0.44 -0.58 -0.04 -23.0 -27.4 -0.72 -1.04 -1.09 91.1 635.6 -0.88 0.24 0.59 -18.7 -7.2 -0.60 -2.82 -0.71 52.4 504.1 0.89 -0.67 -9.30 -20.8 -25.7 -0.77 -0.85 0.0 -20.0 2259.0 -0.94 1.15 -4.0 -44.0 -22.7 0.94 -0.98 -19.0 105.0 1267.9 1.03 1.27 11.0 -39.5 82.3 0.47 -0.19 -10.0 7.0 1491.8 0.32 -1.29 0.0 -34.0 658.2 -0.76 0.26 0.24 3 279 261535 0 41.0 27.91 -0.35 3.0 46.0 1921.6 -1.36 -0.47 -32.0 -51.5 560.9 -0.29 -0.10 -1.11 124.3 791.6 0.00 0.39 -1.85 -21.7 -44.9 -0.21 0.02 0.89 133.9 797.8 -0.08 1.06 -0.26 -16.4 -74.1 0.97 -0.80 -0.41 66.9 955.3 -1.90 1.28 -6.65 -28.1 47.5 -1.91 1.42 1.0 -30.0 1846.7 0.76 1.10 -4.0 -52.0 -53.9 1.71 -0.22 -12.0 97.5 1969.8 -1.70 0.16 -1.0 -32.5 255.9 -0.46 1.57 10.0 6.0 2047.7 -0.98 1.53 0.0 -49.0 554.2 -0.83 0.39 0.73 4 279 261536 0 50.0 28.00 -1.32 -9.0 12.0 464.8 0.88 0.19 8.0 -51.5 98.1 1.09 -0.33 -2.16 -3.9 102.7 0.39 -1.22 -3.39 -15.2 -42.2 -1.18 -1.11 -3.55 8.9 141.3 -0.16 -0.43 -4.15 -12.9 -13.4 -1.32 -0.98 -3.69 8.8 136.1 -0.30 4.13 1.89 -13.0 -18.7 -1.37 -0.93 0.0 -1.0 810.1 -2.29 6.72 1.0 -23.0 -29.7 0.58 -1.10 -18.5 33.5 206.8 1.84 -0.13 4.0 -29.0 30.1 0.80 -0.24 5.0 -14.0 479.5 0.68 -0.59 2.0 -36.0 -6.9 2.02 0.14 -0.23 In\u00a0[3]: Copied! <pre># imbalance of the classes\ndf[\"target\"].value_counts()\n</pre> # imbalance of the classes df[\"target\"].value_counts() Out[3]: <pre>target\n0    144455\n1      1296\nName: count, dtype: int64</pre> In\u00a0[4]: Copied! <pre># drop columns we won't need in this example\ndf.drop(columns=[\"EXAMPLE_ID\", \"BLOCK_ID\"], inplace=True)\n</pre> # drop columns we won't need in this example df.drop(columns=[\"EXAMPLE_ID\", \"BLOCK_ID\"], inplace=True) In\u00a0[5]: Copied! <pre>df_train, df_valid = train_test_split(\n    df, test_size=0.2, stratify=df[\"target\"], random_state=1\n)\ndf_valid, df_test = train_test_split(\n    df_valid, test_size=0.5, stratify=df_valid[\"target\"], random_state=1\n)\n</pre> df_train, df_valid = train_test_split(     df, test_size=0.2, stratify=df[\"target\"], random_state=1 ) df_valid, df_test = train_test_split(     df_valid, test_size=0.5, stratify=df_valid[\"target\"], random_state=1 ) In\u00a0[6]: Copied! <pre>continuous_cols = df.drop(columns=[\"target\"]).columns.values.tolist()\n</pre> continuous_cols = df.drop(columns=[\"target\"]).columns.values.tolist() In\u00a0[7]: Copied! <pre># deeptabular\ntab_preprocessor = TabPreprocessor(continuous_cols=continuous_cols, scale=True)\nX_tab_train = tab_preprocessor.fit_transform(df_train)\nX_tab_valid = tab_preprocessor.transform(df_valid)\nX_tab_test = tab_preprocessor.transform(df_test)\n\n# target\ny_train = df_train[\"target\"].values\ny_valid = df_valid[\"target\"].values\ny_test = df_test[\"target\"].values\n</pre> # deeptabular tab_preprocessor = TabPreprocessor(continuous_cols=continuous_cols, scale=True) X_tab_train = tab_preprocessor.fit_transform(df_train) X_tab_valid = tab_preprocessor.transform(df_valid) X_tab_test = tab_preprocessor.transform(df_test)  # target y_train = df_train[\"target\"].values y_valid = df_valid[\"target\"].values y_test = df_test[\"target\"].values In\u00a0[8]: Copied! <pre>input_layer = len(tab_preprocessor.continuous_cols)\noutput_layer = 1\nhidden_layers = np.linspace(\n    input_layer * 2, output_layer, 5, endpoint=False, dtype=int\n).tolist()\n</pre> input_layer = len(tab_preprocessor.continuous_cols) output_layer = 1 hidden_layers = np.linspace(     input_layer * 2, output_layer, 5, endpoint=False, dtype=int ).tolist() In\u00a0[9]: Copied! <pre>deeptabular = TabMlp(\n    mlp_hidden_dims=hidden_layers,\n    column_idx=tab_preprocessor.column_idx,\n    continuous_cols=tab_preprocessor.continuous_cols,\n)\nmodel = WideDeep(deeptabular=deeptabular, pred_dim=1)\nmodel\n</pre> deeptabular = TabMlp(     mlp_hidden_dims=hidden_layers,     column_idx=tab_preprocessor.column_idx,     continuous_cols=tab_preprocessor.continuous_cols, ) model = WideDeep(deeptabular=deeptabular, pred_dim=1) model Out[9]: <pre>WideDeep(\n  (deeptabular): Sequential(\n    (0): TabMlp(\n      (cont_norm): Identity()\n      (encoder): MLP(\n        (mlp): Sequential(\n          (dense_layer_0): Sequential(\n            (0): Linear(in_features=74, out_features=148, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_1): Sequential(\n            (0): Linear(in_features=148, out_features=118, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_2): Sequential(\n            (0): Linear(in_features=118, out_features=89, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_3): Sequential(\n            (0): Linear(in_features=89, out_features=59, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_4): Sequential(\n            (0): Linear(in_features=59, out_features=30, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=30, out_features=1, bias=True)\n  )\n)</pre> In\u00a0[10]: Copied! <pre># # Metrics from torchmetrics\n# accuracy = Accuracy_torchmetrics(average=None, num_classes=1)\n# precision = Precision_torchmetrics(average=\"micro\", num_classes=1)\n# f1 = F1_torchmetrics(average=None, num_classes=1)\n# recall = Recall_torchmetrics(average=None, num_classes=1)\n</pre> # # Metrics from torchmetrics # accuracy = Accuracy_torchmetrics(average=None, num_classes=1) # precision = Precision_torchmetrics(average=\"micro\", num_classes=1) # f1 = F1_torchmetrics(average=None, num_classes=1) # recall = Recall_torchmetrics(average=None, num_classes=1) In\u00a0[11]: Copied! <pre># Metrics from pytorch-widedeep\naccuracy = Accuracy(top_k=2)\nprecision = Precision(average=False)\nrecall = Recall(average=True)\nf1 = F1Score(average=False)\n</pre> # Metrics from pytorch-widedeep accuracy = Accuracy(top_k=2) precision = Precision(average=False) recall = Recall(average=True) f1 = F1Score(average=False) In\u00a0[12]: Copied! <pre># Optimizers\ndeep_opt = SGD(model.deeptabular.parameters(), lr=0.1)\n# LR Scheduler\ndeep_sch = lr_scheduler.StepLR(deep_opt, step_size=3)\n\ntrainer = Trainer(\n    model,\n    objective=\"binary\",\n    lr_schedulers={\"deeptabular\": deep_sch},\n    initializers={\"deeptabular\": XavierNormal},\n    optimizers={\"deeptabular\": deep_opt},\n    metrics=[accuracy, precision, recall, f1],\n    verbose=1,\n)\n</pre> # Optimizers deep_opt = SGD(model.deeptabular.parameters(), lr=0.1) # LR Scheduler deep_sch = lr_scheduler.StepLR(deep_opt, step_size=3)  trainer = Trainer(     model,     objective=\"binary\",     lr_schedulers={\"deeptabular\": deep_sch},     initializers={\"deeptabular\": XavierNormal},     optimizers={\"deeptabular\": deep_opt},     metrics=[accuracy, precision, recall, f1],     verbose=1, ) In\u00a0[13]: Copied! <pre>start = time.time()\ntrainer.fit(\n    X_train={\"X_tab\": X_tab_train, \"target\": y_train},\n    X_val={\"X_tab\": X_tab_valid, \"target\": y_valid},\n    n_epochs=3,\n    batch_size=50,\n    custom_dataloader=DataLoaderImbalanced,\n    oversample_mul=5,\n)\nprint(\n    \"Training time[s]: {}\".format(\n        datetime.timedelta(seconds=round(time.time() - start))\n    )\n)\n</pre> start = time.time() trainer.fit(     X_train={\"X_tab\": X_tab_train, \"target\": y_train},     X_val={\"X_tab\": X_tab_valid, \"target\": y_valid},     n_epochs=3,     batch_size=50,     custom_dataloader=DataLoaderImbalanced,     oversample_mul=5, ) print(     \"Training time[s]: {}\".format(         datetime.timedelta(seconds=round(time.time() - start))     ) ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 208/208 [00:01&lt;00:00, 130.75it/s, loss=0.187, metrics={'acc': 0.9214, 'prec': [0.9149], 'rec': 0.9318, 'f1': [0.9233]}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 292/292 [00:01&lt;00:00, 173.71it/s, loss=0.106, metrics={'acc': 0.9499, 'prec': [0.1435], 'rec': 0.938, 'f1': [0.249]}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 208/208 [00:01&lt;00:00, 139.68it/s, loss=0.109, metrics={'acc': 0.9559, 'prec': [0.9537], 'rec': 0.9572, 'f1': [0.9554]}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 292/292 [00:01&lt;00:00, 177.53it/s, loss=0.0888, metrics={'acc': 0.9602, 'prec': [0.1755], 'rec': 0.9457, 'f1': [0.2961]}]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 208/208 [00:01&lt;00:00, 141.63it/s, loss=0.08, metrics={'acc': 0.9706, 'prec': [0.9648], 'rec': 0.9766, 'f1': [0.9707]}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 292/292 [00:01&lt;00:00, 165.36it/s, loss=0.0969, metrics={'acc': 0.9564, 'prec': [0.1636], 'rec': 0.9535, 'f1': [0.2792]}]</pre> <pre>Training time[s]: 0:00:10\n</pre> <pre>\n</pre> In\u00a0[14]: Copied! <pre>pd.DataFrame(trainer.history)\n</pre> pd.DataFrame(trainer.history) Out[14]: train_loss train_acc train_prec train_rec train_f1 val_loss val_acc val_prec val_rec val_f1 0 0.186707 0.921408 [0.9149412512779236] 0.931801 [0.9232940673828125] 0.106023 0.949914 [0.14353498816490173] 0.937984 [0.24897116422653198] 1 0.109498 0.955931 [0.9536514282226562] 0.957193 [0.9554190039634705] 0.088787 0.960206 [0.17553956806659698] 0.945736 [0.29611650109291077] 2 0.079979 0.970588 [0.9648183584213257] 0.976582 [0.9706646203994751] 0.096858 0.956432 [0.1635638326406479] 0.953488 [0.279228150844574] In\u00a0[15]: Copied! <pre>df_pred = trainer.predict(X_tab=X_tab_test)\nprint(classification_report(df_test[\"target\"].to_list(), df_pred))\nprint(\"Actual predicted values:\\n{}\".format(np.unique(df_pred, return_counts=True)))\n</pre> df_pred = trainer.predict(X_tab=X_tab_test) print(classification_report(df_test[\"target\"].to_list(), df_pred)) print(\"Actual predicted values:\\n{}\".format(np.unique(df_pred, return_counts=True))) <pre>predict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 292/292 [00:00&lt;00:00, 346.55it/s]\n</pre> <pre>              precision    recall  f1-score   support\n\n           0       1.00      0.96      0.98     14446\n           1       0.17      0.95      0.29       130\n\n    accuracy                           0.96     14576\n   macro avg       0.58      0.95      0.63     14576\nweighted avg       0.99      0.96      0.97     14576\n\nActual predicted values:\n(array([0, 1]), array([13845,   731]))\n</pre> In\u00a0[16]: Copied! <pre>df_pred_unc = trainer.predict_uncertainty(X_tab=X_tab_test, uncertainty_granularity=10)\nprint(classification_report(df_test[\"target\"].to_list(), df_pred))\nprint(\n    \"Actual predicted values:\\n{}\".format(\n        np.unique(df_pred_unc[:, -1], return_counts=True)\n    )\n)\n</pre> df_pred_unc = trainer.predict_uncertainty(X_tab=X_tab_test, uncertainty_granularity=10) print(classification_report(df_test[\"target\"].to_list(), df_pred)) print(     \"Actual predicted values:\\n{}\".format(         np.unique(df_pred_unc[:, -1], return_counts=True)     ) ) <pre>predict_UncertaintyIter: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 10/10 [00:03&lt;00:00,  3.25it/s]</pre> <pre>              precision    recall  f1-score   support\n\n           0       1.00      0.96      0.98     14446\n           1       0.17      0.95      0.29       130\n\n    accuracy                           0.96     14576\n   macro avg       0.58      0.95      0.63     14576\nweighted avg       0.99      0.96      0.97     14576\n\nActual predicted values:\n(array([0.]), array([14576]))\n</pre> <pre>\n</pre> In\u00a0[17]: Copied! <pre>df_pred_unc\n</pre> df_pred_unc Out[17]: <pre>array([[9.98401165e-01, 1.59881881e-03, 0.00000000e+00],\n       [9.99941409e-01, 5.85634953e-05, 0.00000000e+00],\n       [9.97351170e-01, 2.64881272e-03, 0.00000000e+00],\n       ...,\n       [9.99494374e-01, 5.05603210e-04, 0.00000000e+00],\n       [9.99981642e-01, 1.83574630e-05, 0.00000000e+00],\n       [9.99996483e-01, 3.52600046e-06, 0.00000000e+00]])</pre>"},{"location":"examples/13_model_uncertainty_prediction.html#model-uncertainty-prediction","title":"Model Uncertainty prediction\u00b6","text":"<p>Note:</p> <p>This notebook extends the \"Custom DataLoader for Imbalanced dataset\" notebook</p>"},{"location":"examples/13_model_uncertainty_prediction.html#initial-imports","title":"Initial imports\u00b6","text":""},{"location":"examples/13_model_uncertainty_prediction.html#preparing-the-data","title":"Preparing the data\u00b6","text":""},{"location":"examples/13_model_uncertainty_prediction.html#define-the-model","title":"Define the model\u00b6","text":""},{"location":"examples/13_model_uncertainty_prediction.html#normal-prediction","title":"\"Normal\" prediction\u00b6","text":""},{"location":"examples/13_model_uncertainty_prediction.html#prediction-using-uncertainty","title":"Prediction using uncertainty\u00b6","text":""},{"location":"examples/14_bayesian_models.html","title":"14_bayesian_models","text":"In\u00a0[1]: Copied! <pre>import numpy as np\nimport torch\nimport pandas as pd\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import accuracy_score\n\nfrom pytorch_widedeep.metrics import Accuracy\nfrom pytorch_widedeep.datasets import load_adult\nfrom pytorch_widedeep.callbacks import EarlyStopping, ModelCheckpoint\nfrom pytorch_widedeep.preprocessing import TabPreprocessor, WidePreprocessor\nfrom pytorch_widedeep.bayesian_models import BayesianWide, BayesianTabMlp\nfrom pytorch_widedeep.training.bayesian_trainer import BayesianTrainer\n</pre> import numpy as np import torch import pandas as pd  from sklearn.model_selection import train_test_split from sklearn.metrics import accuracy_score  from pytorch_widedeep.metrics import Accuracy from pytorch_widedeep.datasets import load_adult from pytorch_widedeep.callbacks import EarlyStopping, ModelCheckpoint from pytorch_widedeep.preprocessing import TabPreprocessor, WidePreprocessor from pytorch_widedeep.bayesian_models import BayesianWide, BayesianTabMlp from pytorch_widedeep.training.bayesian_trainer import BayesianTrainer <p>The first few things to do we know them very well, like with any other model described in any of the other notebooks</p> In\u00a0[2]: Copied! <pre>df = load_adult(as_frame=True)\ndf.columns = [c.replace(\"-\", \"_\") for c in df.columns]\ndf[\"age_buckets\"] = pd.cut(\n    df.age, bins=[16, 25, 30, 35, 40, 45, 50, 55, 60, 91], labels=np.arange(9)\n)\ndf[\"income_label\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int)\ndf.drop(\"income\", axis=1, inplace=True)\ndf.head()\n</pre> df = load_adult(as_frame=True) df.columns = [c.replace(\"-\", \"_\") for c in df.columns] df[\"age_buckets\"] = pd.cut(     df.age, bins=[16, 25, 30, 35, 40, 45, 50, 55, 60, 91], labels=np.arange(9) ) df[\"income_label\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int) df.drop(\"income\", axis=1, inplace=True) df.head() Out[2]: age workclass fnlwgt education educational_num marital_status occupation relationship race gender capital_gain capital_loss hours_per_week native_country age_buckets income_label 0 25 Private 226802 11th 7 Never-married Machine-op-inspct Own-child Black Male 0 0 40 United-States 0 0 1 38 Private 89814 HS-grad 9 Married-civ-spouse Farming-fishing Husband White Male 0 0 50 United-States 3 0 2 28 Local-gov 336951 Assoc-acdm 12 Married-civ-spouse Protective-serv Husband White Male 0 0 40 United-States 1 1 3 44 Private 160323 Some-college 10 Married-civ-spouse Machine-op-inspct Husband Black Male 7688 0 40 United-States 4 1 4 18 ? 103497 Some-college 10 Never-married ? Own-child White Female 0 0 30 United-States 0 0 In\u00a0[3]: Copied! <pre>train, test = train_test_split(df, test_size=0.2, stratify=df.income_label)\n</pre> train, test = train_test_split(df, test_size=0.2, stratify=df.income_label) In\u00a0[4]: Copied! <pre>wide_cols = [\n    \"age_buckets\",\n    \"education\",\n    \"relationship\",\n    \"workclass\",\n    \"occupation\",\n    \"native_country\",\n    \"gender\",\n]\ncrossed_cols = [(\"education\", \"occupation\"), (\"native_country\", \"occupation\")]\n\ncat_embed_cols = [\n    \"workclass\",\n    \"education\",\n    \"marital_status\",\n    \"occupation\",\n    \"relationship\",\n    \"race\",\n    \"gender\",\n    \"capital_gain\",\n    \"capital_loss\",\n    \"native_country\",\n]\ncontinuous_cols = [\"age\", \"hours_per_week\"]\n\ntarget = train[\"income_label\"].values\n</pre> wide_cols = [     \"age_buckets\",     \"education\",     \"relationship\",     \"workclass\",     \"occupation\",     \"native_country\",     \"gender\", ] crossed_cols = [(\"education\", \"occupation\"), (\"native_country\", \"occupation\")]  cat_embed_cols = [     \"workclass\",     \"education\",     \"marital_status\",     \"occupation\",     \"relationship\",     \"race\",     \"gender\",     \"capital_gain\",     \"capital_loss\",     \"native_country\", ] continuous_cols = [\"age\", \"hours_per_week\"]  target = train[\"income_label\"].values In\u00a0[5]: Copied! <pre>wide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols)\nX_tab = wide_preprocessor.fit_transform(train)\n</pre> wide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols) X_tab = wide_preprocessor.fit_transform(train) In\u00a0[6]: Copied! <pre>model = BayesianWide(\n    input_dim=np.unique(X_tab).shape[0],\n    prior_sigma_1=1.0,\n    prior_sigma_2=0.002,\n    prior_pi=0.8,\n    posterior_mu_init=0,\n    posterior_rho_init=-7.0,\n    pred_dim=1,  # here the models are NOT passed to a WideDeep constructor class so the output dim MUST be specified\n)\n</pre> model = BayesianWide(     input_dim=np.unique(X_tab).shape[0],     prior_sigma_1=1.0,     prior_sigma_2=0.002,     prior_pi=0.8,     posterior_mu_init=0,     posterior_rho_init=-7.0,     pred_dim=1,  # here the models are NOT passed to a WideDeep constructor class so the output dim MUST be specified ) In\u00a0[7]: Copied! <pre>trainer = BayesianTrainer(\n    model,\n    objective=\"binary\",\n    optimizer=torch.optim.Adam(model.parameters(), lr=0.01),\n    metrics=[Accuracy],\n)\n</pre> trainer = BayesianTrainer(     model,     objective=\"binary\",     optimizer=torch.optim.Adam(model.parameters(), lr=0.01),     metrics=[Accuracy], ) In\u00a0[8]: Copied! <pre>trainer.fit(\n    X_tab=X_tab,\n    target=target,\n    val_split=0.2,\n    n_epochs=2,\n    batch_size=256,\n)\n</pre> trainer.fit(     X_tab=X_tab,     target=target,     val_split=0.2,     n_epochs=2,     batch_size=256, ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 123/123 [00:00&lt;00:00, 124.32it/s, loss=163, metrics={'acc': 0.7813}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 31/31 [00:00&lt;00:00, 238.67it/s, loss=141, metrics={'acc': 0.8219}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 123/123 [00:00&lt;00:00, 132.81it/s, loss=140, metrics={'acc': 0.8285}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 31/31 [00:00&lt;00:00, 190.16it/s, loss=140, metrics={'acc': 0.8298}]\n</pre> In\u00a0[9]: Copied! <pre>tab_preprocessor = TabPreprocessor(\n    cat_embed_cols=cat_embed_cols, continuous_cols=continuous_cols\n)\nX_tab = tab_preprocessor.fit_transform(train)\n</pre> tab_preprocessor = TabPreprocessor(     cat_embed_cols=cat_embed_cols, continuous_cols=continuous_cols ) X_tab = tab_preprocessor.fit_transform(train) <pre>/Users/javierrodriguezzaurin/Projects/pytorch-widedeep/pytorch_widedeep/preprocessing/tab_preprocessor.py:358: UserWarning: Continuous columns will not be normalised\n  warnings.warn(\"Continuous columns will not be normalised\")\n</pre> In\u00a0[10]: Copied! <pre>model = BayesianTabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=continuous_cols,\n    #     embed_continuous_method = \"standard\",\n    #     cont_embed_activation=\"leaky_relu\",\n    #     cont_embed_dim = 8,\n    mlp_hidden_dims=[128, 64],\n    prior_sigma_1=1.0,\n    prior_sigma_2=0.002,\n    prior_pi=0.8,\n    posterior_mu_init=0,\n    posterior_rho_init=-7.0,\n    pred_dim=1,\n)\n</pre> model = BayesianTabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     continuous_cols=continuous_cols,     #     embed_continuous_method = \"standard\",     #     cont_embed_activation=\"leaky_relu\",     #     cont_embed_dim = 8,     mlp_hidden_dims=[128, 64],     prior_sigma_1=1.0,     prior_sigma_2=0.002,     prior_pi=0.8,     posterior_mu_init=0,     posterior_rho_init=-7.0,     pred_dim=1, ) In\u00a0[11]: Copied! <pre>trainer = BayesianTrainer(\n    model,\n    objective=\"binary\",\n    optimizer=torch.optim.Adam(model.parameters(), lr=0.01),\n    metrics=[Accuracy],\n)\n</pre> trainer = BayesianTrainer(     model,     objective=\"binary\",     optimizer=torch.optim.Adam(model.parameters(), lr=0.01),     metrics=[Accuracy], ) In\u00a0[12]: Copied! <pre>trainer.fit(\n    X_tab=X_tab,\n    target=target,\n    val_split=0.2,\n    n_epochs=2,\n    batch_size=256,\n)\n</pre> trainer.fit(     X_tab=X_tab,     target=target,     val_split=0.2,     n_epochs=2,     batch_size=256, ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 123/123 [00:04&lt;00:00, 28.74it/s, loss=2e+3, metrics={'acc': 0.8007}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 31/31 [00:00&lt;00:00, 136.89it/s, loss=1.75e+3, metrics={'acc': 0.8418}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 123/123 [00:04&lt;00:00, 29.41it/s, loss=1.73e+3, metrics={'acc': 0.8596}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 31/31 [00:00&lt;00:00, 143.87it/s, loss=1.71e+3, metrics={'acc': 0.8569}]\n</pre> <p>These models are powerful beyond the success metrics because they give us a sense of uncertainty as we predict. Let's have a look</p> In\u00a0[13]: Copied! <pre>X_tab_test = tab_preprocessor.transform(test)\n</pre> X_tab_test = tab_preprocessor.transform(test) In\u00a0[14]: Copied! <pre>preds = trainer.predict(X_tab_test, return_samples=True, n_samples=5)\n</pre> preds = trainer.predict(X_tab_test, return_samples=True, n_samples=5) <pre>predict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 39/39 [00:01&lt;00:00, 33.92it/s]\n</pre> In\u00a0[15]: Copied! <pre>preds.shape\n</pre> preds.shape Out[15]: <pre>(5, 9769)</pre> <p>as we can see the prediction have shape <code>(5, 9769)</code>, one set of predictions each time we have internally run predict (i.e. sample the network and predict, defined by the parameter <code>n_samples</code>). This gives us an idea of how certain the model is about a certain prediction.</p> <p>Similarly, we could obtain the probabilities</p> In\u00a0[16]: Copied! <pre>probs = trainer.predict_proba(X_tab_test, return_samples=True, n_samples=5)\n</pre> probs = trainer.predict_proba(X_tab_test, return_samples=True, n_samples=5) <pre>predict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 39/39 [00:01&lt;00:00, 32.79it/s]\n</pre> In\u00a0[17]: Copied! <pre>probs.shape\n</pre> probs.shape Out[17]: <pre>(5, 9769, 2)</pre> <p>And we could see how the model performs each time we sampled the network</p> In\u00a0[18]: Copied! <pre>for p in preds:\n    print(accuracy_score(p, test[\"income_label\"].values))\n</pre> for p in preds:     print(accuracy_score(p, test[\"income_label\"].values)) <pre>0.8559729757395844\n0.8564847988535162\n0.8567918927218753\n0.8562800696079435\n0.8558706111167981\n</pre> In\u00a0[\u00a0]: Copied! <pre>\n</pre>"},{"location":"examples/14_bayesian_models.html#the-bayesian-models","title":"The Bayesian Models\u00b6","text":"<p>Perhaps one of the most interesting functionality in the library is the access to full Bayesian models in almost exactly the same way one would use any of the other models in the library.</p> <p>Note however that the Bayesian models are ONLY available for tabular data and, at the moment, we do not support combining them to form a Wide and Deep model.</p> <p>The implementation in this library is based on the publication: Weight Uncertainty in Neural Networks, by Blundell et al., 2015. Code-wise, our implementation is inspired by a number of source:</p> <ol> <li>https://joshfeldman.net/WeightUncertainty/</li> <li>https://www.nitarshan.com/bayes-by-backprop/</li> <li>https://github.com/piEsposito/blitz-bayesian-deep-learning</li> <li>https://github.com/zackchase/mxnet-the-straight-dope/tree/master/chapter18_variational-methods-and-uncertainty</li> </ol> <p>The two Bayesian models available in the library are:</p> <ol> <li>BayesianWide: this is a linear model where the non-linearities are captured via crossed-columns</li> <li>BayesianMLP: this is a standard MLP that receives categorical embeddings and continuous cols (embedded or not) which are the passed through a series of dense layers. All parameters in the model are probabilistic.</li> </ol>"},{"location":"examples/14_bayesian_models.html#1-bayesianwide","title":"1. <code>BayesianWide</code>\u00b6","text":""},{"location":"examples/14_bayesian_models.html#2-bayesiantabmlp","title":"2. <code>BayesianTabMlp</code>\u00b6","text":""},{"location":"examples/15_Self_Supervised_Pretraning_pt1.html","title":"15 Self Supervised Pretraning pt1","text":"In\u00a0[1]: Copied! <pre>import torch\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.model_selection import train_test_split\n\nfrom pytorch_widedeep import Trainer\nfrom pytorch_widedeep.models import TabMlp, WideDeep\nfrom pytorch_widedeep.metrics import Accuracy\nfrom pytorch_widedeep.datasets import load_adult\nfrom pytorch_widedeep.preprocessing import TabPreprocessor\nfrom pytorch_widedeep.self_supervised_training import EncoderDecoderTrainer\n</pre> import torch from sklearn.metrics import accuracy_score from sklearn.model_selection import train_test_split  from pytorch_widedeep import Trainer from pytorch_widedeep.models import TabMlp, WideDeep from pytorch_widedeep.metrics import Accuracy from pytorch_widedeep.datasets import load_adult from pytorch_widedeep.preprocessing import TabPreprocessor from pytorch_widedeep.self_supervised_training import EncoderDecoderTrainer In\u00a0[2]: Copied! <pre>df = load_adult(as_frame=True)\ndf.columns = [c.replace(\"-\", \"_\") for c in df.columns]\ndf[\"income_label\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int)\ndf.drop(\"income\", axis=1, inplace=True)\n</pre> df = load_adult(as_frame=True) df.columns = [c.replace(\"-\", \"_\") for c in df.columns] df[\"income_label\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int) df.drop(\"income\", axis=1, inplace=True) In\u00a0[3]: Copied! <pre># one could chose to use a validation set for early stopping, hyperparam\n# optimization, etc. This is just an example, so we simply use train/test\n# split\ndf_tr, df_te = train_test_split(df, test_size=0.2, stratify=df.income_label)\n</pre> # one could chose to use a validation set for early stopping, hyperparam # optimization, etc. This is just an example, so we simply use train/test # split df_tr, df_te = train_test_split(df, test_size=0.2, stratify=df.income_label) In\u00a0[4]: Copied! <pre>df_tr.head(2)\n</pre> df_tr.head(2) Out[4]: age workclass fnlwgt education educational_num marital_status occupation relationship race gender capital_gain capital_loss hours_per_week native_country income_label 9042 26 Local-gov 250551 HS-grad 9 Married-civ-spouse Craft-repair Own-child Black Male 0 0 40 United-States 0 25322 50 Private 34832 Bachelors 13 Married-civ-spouse Tech-support Husband White Male 15024 0 40 United-States 1 In\u00a0[5]: Copied! <pre># As always, we need to define which cols will be represented as embeddings\n# and which one will be continuous features\ncat_embed_cols = [\n    \"workclass\",\n    \"education\",\n    \"marital_status\",\n    \"occupation\",\n    \"relationship\",\n    \"race\",\n    \"gender\",\n    \"capital_gain\",\n    \"capital_loss\",\n    \"native_country\",\n]\ncontinuous_cols = [\"age\", \"hours_per_week\"]\ntarget_col = \"income_label\"\n</pre> # As always, we need to define which cols will be represented as embeddings # and which one will be continuous features cat_embed_cols = [     \"workclass\",     \"education\",     \"marital_status\",     \"occupation\",     \"relationship\",     \"race\",     \"gender\",     \"capital_gain\",     \"capital_loss\",     \"native_country\", ] continuous_cols = [\"age\", \"hours_per_week\"] target_col = \"income_label\" In\u00a0[6]: Copied! <pre># We prepare the data to be passed to the model\ntab_preprocessor = TabPreprocessor(\n    cat_embed_cols=cat_embed_cols, continuous_cols=continuous_cols\n)\nX_tab = tab_preprocessor.fit_transform(df_tr)\ntarget = df_tr[target_col].values\n</pre> # We prepare the data to be passed to the model tab_preprocessor = TabPreprocessor(     cat_embed_cols=cat_embed_cols, continuous_cols=continuous_cols ) X_tab = tab_preprocessor.fit_transform(df_tr) target = df_tr[target_col].values <pre>/Users/javierrodriguezzaurin/Projects/pytorch-widedeep/pytorch_widedeep/preprocessing/tab_preprocessor.py:358: UserWarning: Continuous columns will not be normalised\n  warnings.warn(\"Continuous columns will not be normalised\")\n</pre> In\u00a0[7]: Copied! <pre>X_tab[:5]\n</pre> X_tab[:5] Out[7]: <pre>array([[ 1,  1,  1,  1,  1,  1,  1,  1,  1,  1, 26, 40],\n       [ 2,  2,  1,  2,  2,  2,  1,  2,  1,  1, 50, 40],\n       [ 2,  1,  1,  3,  2,  2,  1,  1,  2,  1, 39, 46],\n       [ 2,  3,  2,  4,  1,  2,  2,  1,  1,  1, 17, 10],\n       [ 3,  4,  2,  1,  1,  2,  1,  1,  1,  1, 32, 20]])</pre> In\u00a0[8]: Copied! <pre># We define a model that will act as the encoder in the encoder/decoder\n# architecture. This could be any of: TabMlp, TabResnet or TabNet\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=tab_preprocessor.continuous_cols,\n)\n</pre> # We define a model that will act as the encoder in the encoder/decoder # architecture. This could be any of: TabMlp, TabResnet or TabNet tab_mlp = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     continuous_cols=tab_preprocessor.continuous_cols, ) In\u00a0[9]: Copied! <pre>tab_mlp\n</pre> tab_mlp Out[9]: <pre>TabMlp(\n  (cat_embed): DiffSizeCatEmbeddings(\n    (embed_layers): ModuleDict(\n      (emb_layer_workclass): Embedding(10, 5, padding_idx=0)\n      (emb_layer_education): Embedding(17, 8, padding_idx=0)\n      (emb_layer_marital_status): Embedding(8, 5, padding_idx=0)\n      (emb_layer_occupation): Embedding(16, 7, padding_idx=0)\n      (emb_layer_relationship): Embedding(7, 4, padding_idx=0)\n      (emb_layer_race): Embedding(6, 4, padding_idx=0)\n      (emb_layer_gender): Embedding(3, 2, padding_idx=0)\n      (emb_layer_capital_gain): Embedding(124, 24, padding_idx=0)\n      (emb_layer_capital_loss): Embedding(98, 21, padding_idx=0)\n      (emb_layer_native_country): Embedding(42, 13, padding_idx=0)\n    )\n    (embedding_dropout): Dropout(p=0.0, inplace=False)\n  )\n  (cont_norm): Identity()\n  (encoder): MLP(\n    (mlp): Sequential(\n      (dense_layer_0): Sequential(\n        (0): Linear(in_features=95, out_features=200, bias=True)\n        (1): ReLU(inplace=True)\n        (2): Dropout(p=0.1, inplace=False)\n      )\n      (dense_layer_1): Sequential(\n        (0): Linear(in_features=200, out_features=100, bias=True)\n        (1): ReLU(inplace=True)\n        (2): Dropout(p=0.1, inplace=False)\n      )\n    )\n  )\n)</pre> In\u00a0[10]: Copied! <pre># If we do not pass a custom decoder, which is perfectly possible via the\n# decoder param,  the EncoderDecoderTrainer will automatically build a\n# decoder which will be the 'mirror' image of the encoder\nencoder_decoder_trainer = EncoderDecoderTrainer(encoder=tab_mlp)\n</pre> # If we do not pass a custom decoder, which is perfectly possible via the # decoder param,  the EncoderDecoderTrainer will automatically build a # decoder which will be the 'mirror' image of the encoder encoder_decoder_trainer = EncoderDecoderTrainer(encoder=tab_mlp) In\u00a0[11]: Copied! <pre># let's have a look to the encoder_decoder_model (aka ed_model)\nencoder_decoder_trainer.ed_model\n</pre> # let's have a look to the encoder_decoder_model (aka ed_model) encoder_decoder_trainer.ed_model Out[11]: <pre>EncoderDecoderModel(\n  (encoder): TabMlp(\n    (cat_embed): DiffSizeCatEmbeddings(\n      (embed_layers): ModuleDict(\n        (emb_layer_workclass): Embedding(10, 5, padding_idx=0)\n        (emb_layer_education): Embedding(17, 8, padding_idx=0)\n        (emb_layer_marital_status): Embedding(8, 5, padding_idx=0)\n        (emb_layer_occupation): Embedding(16, 7, padding_idx=0)\n        (emb_layer_relationship): Embedding(7, 4, padding_idx=0)\n        (emb_layer_race): Embedding(6, 4, padding_idx=0)\n        (emb_layer_gender): Embedding(3, 2, padding_idx=0)\n        (emb_layer_capital_gain): Embedding(124, 24, padding_idx=0)\n        (emb_layer_capital_loss): Embedding(98, 21, padding_idx=0)\n        (emb_layer_native_country): Embedding(42, 13, padding_idx=0)\n      )\n      (embedding_dropout): Dropout(p=0.0, inplace=False)\n    )\n    (cont_norm): Identity()\n    (encoder): MLP(\n      (mlp): Sequential(\n        (dense_layer_0): Sequential(\n          (0): Linear(in_features=95, out_features=200, bias=True)\n          (1): ReLU(inplace=True)\n          (2): Dropout(p=0.1, inplace=False)\n        )\n        (dense_layer_1): Sequential(\n          (0): Linear(in_features=200, out_features=100, bias=True)\n          (1): ReLU(inplace=True)\n          (2): Dropout(p=0.1, inplace=False)\n        )\n      )\n    )\n  )\n  (decoder): TabMlpDecoder(\n    (decoder): MLP(\n      (mlp): Sequential(\n        (dense_layer_0): Sequential(\n          (0): Linear(in_features=100, out_features=200, bias=True)\n          (1): ReLU(inplace=True)\n          (2): Dropout(p=0.1, inplace=False)\n        )\n        (dense_layer_1): Sequential(\n          (0): Linear(in_features=200, out_features=95, bias=True)\n          (1): ReLU(inplace=True)\n          (2): Dropout(p=0.1, inplace=False)\n        )\n      )\n    )\n  )\n  (masker): RandomObfuscator()\n)</pre> <p>Ignoring the <code>masker</code>, which just...well...masks, the <code>ed_model</code> consists of:</p> <ol> <li>An encoder model that is a <code>TabMlp</code> model that is in itself comprised by an Embedding layer (or rather a collection of them, referred as <code>cat_and_cont_embed</code>) and an encoder (a simple MLP, referred as <code>encoder</code>)</li> <li>A decoder which is just an \"inverted\" MLP (referred as <code>decoder</code>)</li> </ol> In\u00a0[12]: Copied! <pre># And we just...pretrain\nencoder_decoder_trainer.pretrain(X_tab, n_epochs=5, batch_size=256)\n</pre> # And we just...pretrain encoder_decoder_trainer.pretrain(X_tab, n_epochs=5, batch_size=256) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 82.90it/s, loss=4.07]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 89.87it/s, loss=3.09]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 92.86it/s, loss=2.53]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 91.24it/s, loss=2.09]\nepoch 5: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 91.38it/s, loss=1.78]\n</pre> <p>At this point we have two options, we could either save the model for later use or we could continue to supervised training. The latter is rather simple, after running:</p> <pre>encoder_decoder_trainer.pretrain(X_tab, n_epochs=5, batch_size=256)\n</pre> <p>you just have to</p> <pre>model = WideDeep(deeptabular=tab_mlp)\ntrainer = Trainer(model=model, objective=\"binary\", metrics=[Accuracy])\n\ntrainer.fit(X_tab=X_tab, target=target, n_epochs=5, batch_size=256)\n\n# And, you know...we get a test metric\nX_tab_te = tab_preprocessor.transform(df_te)\ntarget_te = df_te[target_col].values\n\npreds = trainer.predict(X_tab=X_tab_te)\ntest_acc = accuracy_score(target_te, preds)\n</pre> <p>Let's say that in any case, we are 'decent' scientists/people and we want to save the model:</p> In\u00a0[13]: Copied! <pre>encoder_decoder_trainer.save(\n    path=\"pretrained_weights\", model_filename=\"encoder_decoder_model.pt\"\n)\n</pre> encoder_decoder_trainer.save(     path=\"pretrained_weights\", model_filename=\"encoder_decoder_model.pt\" ) <p>some time has passed...</p> In\u00a0[14]: Copied! <pre>encoder_decoder_model = torch.load(\"pretrained_weights/encoder_decoder_model.pt\")\n</pre> encoder_decoder_model = torch.load(\"pretrained_weights/encoder_decoder_model.pt\") <p>Now, AND THIS IS IMPORTANT We have loaded the encoder AND the decoder. To proceed to the supervised training we ONLY need the encoder</p> In\u00a0[15]: Copied! <pre>pretrained_encoder = encoder_decoder_model.encoder\n</pre> pretrained_encoder = encoder_decoder_model.encoder In\u00a0[16]: Copied! <pre>pretrained_encoder\n</pre> pretrained_encoder Out[16]: <pre>TabMlp(\n  (cat_embed): DiffSizeCatEmbeddings(\n    (embed_layers): ModuleDict(\n      (emb_layer_workclass): Embedding(10, 5, padding_idx=0)\n      (emb_layer_education): Embedding(17, 8, padding_idx=0)\n      (emb_layer_marital_status): Embedding(8, 5, padding_idx=0)\n      (emb_layer_occupation): Embedding(16, 7, padding_idx=0)\n      (emb_layer_relationship): Embedding(7, 4, padding_idx=0)\n      (emb_layer_race): Embedding(6, 4, padding_idx=0)\n      (emb_layer_gender): Embedding(3, 2, padding_idx=0)\n      (emb_layer_capital_gain): Embedding(124, 24, padding_idx=0)\n      (emb_layer_capital_loss): Embedding(98, 21, padding_idx=0)\n      (emb_layer_native_country): Embedding(42, 13, padding_idx=0)\n    )\n    (embedding_dropout): Dropout(p=0.0, inplace=False)\n  )\n  (cont_norm): Identity()\n  (encoder): MLP(\n    (mlp): Sequential(\n      (dense_layer_0): Sequential(\n        (0): Linear(in_features=95, out_features=200, bias=True)\n        (1): ReLU(inplace=True)\n        (2): Dropout(p=0.1, inplace=False)\n      )\n      (dense_layer_1): Sequential(\n        (0): Linear(in_features=200, out_features=100, bias=True)\n        (1): ReLU(inplace=True)\n        (2): Dropout(p=0.1, inplace=False)\n      )\n    )\n  )\n)</pre> In\u00a0[17]: Copied! <pre># and as always, ANY supervised model in this library has to go throuth the WideDeep class:\nmodel = WideDeep(deeptabular=pretrained_encoder)\ntrainer = Trainer(model=model, objective=\"binary\", metrics=[Accuracy])\n\ntrainer.fit(X_tab=X_tab, target=target, n_epochs=5, batch_size=256)\n\nX_tab_te = tab_preprocessor.transform(df_te)\ntarget_te = df_te[target_col].values\n\npreds = trainer.predict(X_tab=X_tab_te)\ntest_acc = accuracy_score(target_te, preds)\nprint(test_acc)\n</pre> # and as always, ANY supervised model in this library has to go throuth the WideDeep class: model = WideDeep(deeptabular=pretrained_encoder) trainer = Trainer(model=model, objective=\"binary\", metrics=[Accuracy])  trainer.fit(X_tab=X_tab, target=target, n_epochs=5, batch_size=256)  X_tab_te = tab_preprocessor.transform(df_te) target_te = df_te[target_col].values  preds = trainer.predict(X_tab=X_tab_te) test_acc = accuracy_score(target_te, preds) print(test_acc) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 88.04it/s, loss=0.374, metrics={'acc': 0.8253}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 85.63it/s, loss=0.324, metrics={'acc': 0.8491}]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 87.56it/s, loss=0.301, metrics={'acc': 0.8608}]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:02&lt;00:00, 73.38it/s, loss=0.29, metrics={'acc': 0.8655}]\nepoch 5: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 78.68it/s, loss=0.284, metrics={'acc': 0.8686}]\npredict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 39/39 [00:00&lt;00:00, 173.02it/s]\n</pre> <pre>0.8730678677449074\n</pre> <p>As we mentioned before, we can also use a <code>TabResNet</code> or <code>TabNet</code> model and a custom decoder. Let's have a look:</p> In\u00a0[18]: Copied! <pre>from pytorch_widedeep.models import TabResnet as TabResnetEncoder, TabResnetDecoder\n</pre> from pytorch_widedeep.models import TabResnet as TabResnetEncoder, TabResnetDecoder In\u00a0[19]: Copied! <pre>resnet_encoder = TabResnetEncoder(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=continuous_cols,\n    blocks_dims=[200, 100, 100],\n)\n</pre> resnet_encoder = TabResnetEncoder(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     continuous_cols=continuous_cols,     blocks_dims=[200, 100, 100], ) <p>let's have a look to the model</p> In\u00a0[20]: Copied! <pre>resnet_encoder\n</pre> resnet_encoder Out[20]: <pre>TabResnet(\n  (cat_embed): DiffSizeCatEmbeddings(\n    (embed_layers): ModuleDict(\n      (emb_layer_workclass): Embedding(10, 5, padding_idx=0)\n      (emb_layer_education): Embedding(17, 8, padding_idx=0)\n      (emb_layer_marital_status): Embedding(8, 5, padding_idx=0)\n      (emb_layer_occupation): Embedding(16, 7, padding_idx=0)\n      (emb_layer_relationship): Embedding(7, 4, padding_idx=0)\n      (emb_layer_race): Embedding(6, 4, padding_idx=0)\n      (emb_layer_gender): Embedding(3, 2, padding_idx=0)\n      (emb_layer_capital_gain): Embedding(124, 24, padding_idx=0)\n      (emb_layer_capital_loss): Embedding(98, 21, padding_idx=0)\n      (emb_layer_native_country): Embedding(42, 13, padding_idx=0)\n    )\n    (embedding_dropout): Dropout(p=0.0, inplace=False)\n  )\n  (cont_norm): Identity()\n  (encoder): DenseResnet(\n    (dense_resnet): Sequential(\n      (lin_inp): Linear(in_features=95, out_features=200, bias=False)\n      (bn_inp): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n      (block_0): BasicBlock(\n        (resize): Sequential(\n          (0): Linear(in_features=200, out_features=100, bias=False)\n          (1): BatchNorm1d(100, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        )\n        (lin1): Linear(in_features=200, out_features=100, bias=False)\n        (bn1): BatchNorm1d(100, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (leaky_relu): LeakyReLU(negative_slope=0.01, inplace=True)\n        (dp): Dropout(p=0.1, inplace=False)\n        (lin2): Linear(in_features=100, out_features=100, bias=False)\n        (bn2): BatchNorm1d(100, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n      )\n      (block_1): BasicBlock(\n        (lin1): Linear(in_features=100, out_features=100, bias=False)\n        (bn1): BatchNorm1d(100, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (leaky_relu): LeakyReLU(negative_slope=0.01, inplace=True)\n        (dp): Dropout(p=0.1, inplace=False)\n        (lin2): Linear(in_features=100, out_features=100, bias=False)\n        (bn2): BatchNorm1d(100, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n      )\n    )\n  )\n)</pre> <p>As we can see, the tensor we are trying to reconstruct, the embeddings, is of size <code>94</code> (this number is stored in the attribute: <code>esnet_encoder.cat_and_cont_embed.output_dim</code>), with that information we could build or own decoder as:</p> In\u00a0[21]: Copied! <pre># for all possible params see the docs\nresnet_decoder = TabResnetDecoder(\n    embed_dim=resnet_encoder.cat_out_dim + resnet_encoder.cont_out_dim,\n    blocks_dims=[100, 100, 200],\n)\n</pre> # for all possible params see the docs resnet_decoder = TabResnetDecoder(     embed_dim=resnet_encoder.cat_out_dim + resnet_encoder.cont_out_dim,     blocks_dims=[100, 100, 200], ) In\u00a0[22]: Copied! <pre>resnet_decoder\n</pre> resnet_decoder Out[22]: <pre>TabResnetDecoder(\n  (decoder): DenseResnet(\n    (dense_resnet): Sequential(\n      (block_0): BasicBlock(\n        (lin1): Linear(in_features=100, out_features=100, bias=False)\n        (bn1): BatchNorm1d(100, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (leaky_relu): LeakyReLU(negative_slope=0.01, inplace=True)\n        (dp): Dropout(p=0.1, inplace=False)\n        (lin2): Linear(in_features=100, out_features=100, bias=False)\n        (bn2): BatchNorm1d(100, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n      )\n      (block_1): BasicBlock(\n        (resize): Sequential(\n          (0): Linear(in_features=100, out_features=200, bias=False)\n          (1): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        )\n        (lin1): Linear(in_features=100, out_features=200, bias=False)\n        (bn1): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (leaky_relu): LeakyReLU(negative_slope=0.01, inplace=True)\n        (dp): Dropout(p=0.1, inplace=False)\n        (lin2): Linear(in_features=200, out_features=200, bias=False)\n        (bn2): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n      )\n    )\n  )\n  (reconstruction_layer): Linear(in_features=200, out_features=95, bias=False)\n)</pre> <p>and now:</p> In\u00a0[23]: Copied! <pre>ec_trainer = EncoderDecoderTrainer(\n    encoder=resnet_encoder,\n    decoder=resnet_decoder,\n    masked_prob=0.2,\n)\nec_trainer.pretrain(X_tab, n_epochs=5, batch_size=256)\n</pre> ec_trainer = EncoderDecoderTrainer(     encoder=resnet_encoder,     decoder=resnet_decoder,     masked_prob=0.2, ) ec_trainer.pretrain(X_tab, n_epochs=5, batch_size=256) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:03&lt;00:00, 46.89it/s, loss=1.52]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:03&lt;00:00, 46.78it/s, loss=0.81]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:03&lt;00:00, 39.82it/s, loss=0.56]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:03&lt;00:00, 46.73it/s, loss=0.417]\nepoch 5: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:03&lt;00:00, 46.24it/s, loss=0.329]\n</pre> In\u00a0[24]: Copied! <pre># and as always, ANY supervised model in this library has to go throuth the WideDeep class:\nmodel = WideDeep(deeptabular=resnet_encoder)\ntrainer = Trainer(model=model, objective=\"binary\", metrics=[Accuracy])\n\ntrainer.fit(X_tab=X_tab, target=target, n_epochs=5, batch_size=256)\n\nX_tab_te = tab_preprocessor.transform(df_te)\ntarget_te = df_te[target_col].values\n\npreds = trainer.predict(X_tab=X_tab_te)\ntest_acc = accuracy_score(target_te, preds)\nprint(test_acc)\n</pre> # and as always, ANY supervised model in this library has to go throuth the WideDeep class: model = WideDeep(deeptabular=resnet_encoder) trainer = Trainer(model=model, objective=\"binary\", metrics=[Accuracy])  trainer.fit(X_tab=X_tab, target=target, n_epochs=5, batch_size=256)  X_tab_te = tab_preprocessor.transform(df_te) target_te = df_te[target_col].values  preds = trainer.predict(X_tab=X_tab_te) test_acc = accuracy_score(target_te, preds) print(test_acc) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:02&lt;00:00, 58.63it/s, loss=0.335, metrics={'acc': 0.8442}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:02&lt;00:00, 58.02it/s, loss=0.296, metrics={'acc': 0.864}]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:02&lt;00:00, 55.91it/s, loss=0.283, metrics={'acc': 0.8687}]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:02&lt;00:00, 55.00it/s, loss=0.276, metrics={'acc': 0.871}]\nepoch 5: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:02&lt;00:00, 51.95it/s, loss=0.272, metrics={'acc': 0.8732}]\npredict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 39/39 [00:00&lt;00:00, 120.15it/s]\n</pre> <pre>0.8725560446309756\n</pre> In\u00a0[\u00a0]: Copied! <pre>\n</pre>"},{"location":"examples/15_Self_Supervised_Pretraning_pt1.html#self-supervised-pretraining-for-tabular-data","title":"Self Supervised Pretraining for Tabular Data\u00b6","text":"<p>We have implemented two Self Supervised Pre-training routines that allow the user to pre-train all tabular models in the library with the exception of the TabPerceiver (which is a special monster).</p> <p>The two routines implemented are illustrated in the figures below. The 1st is from TabNet: Attentive Interpretable Tabular Learning and is designed for models that do not use transformer-based architectures, while the second is from SAINT: Improved Neural Networks for Tabular Data via Row Attention and Contrastive Pre-Training, and is designed for models that use transformer-based architectures.</p> <p> </p> <p>Fig 1. Figure 2 in their paper. I have included de original caption in case is useful, althought the Figure itself is pretty self explanatory</p> <p> </p> <p>Fig 2. Figure 1 in their paper. Here the caption is necessary \ud83d\ude0f</p> <p>It is beyond the scope of this notebook to explain in detail those implementations. Therefore, we strongly recommend the user to go and read the papers if this functionality is of interest to her/him.</p> <p>One thing is worth noticing however. As seen in Fig 1(the TabNet paper's Fig 2) the masking of the input features happens in the feature space. However, the implementation in this library is inspired by that at the dreamquark-ai repo, which is in itself inspired by the original implementation (by the way, at this point I will write it once again. All TabNet related things in this library are inspired when not directly based in the code in that repo, therefore, ALL CREDIT TO THE GUYS AT dreamquark-ai).</p> <p>In that implementation the masking happens in the embedding space, and currently does not mask the entire embedding (i.e. categorical feature). We decided to release as it is in this version and we will implement the exact same process described in the paper in future releases.</p> <p>Having said all of the above let's see how to use self supervision for tabular data with <code>pytorch-widedeep</code>. We will concentrate in this notebook on the 1st of the two approaches (the 'TabNet approach'). For details on the second approach please see <code>16_Self_Supervised_Pretraning_pt2</code>.</p>"},{"location":"examples/15_Self_Supervised_Pretraning_pt1.html#self-supervision-for-non-transformer-based-models","title":"Self Supervision for non-transformer-based models..\u00b6","text":"<p>...or in general, for models where the embeddigns can have all different dimensions. In this library, these are: <code>TabMlp</code>, <code>TabResNet</code> and <code>TabNet</code></p> <p>As shown in Figure, this is an encoder-encoder approach where we learn to predict values in the incoming data that have been masked. However, as I mentioned before, our implementation is a bit different, and the masking occurs in th embedding space.</p> <p>Nonetheless, the code below illustrates how to use this encoder-decoder approach with <code>pytorch-widedeep</code></p>"},{"location":"examples/15_Self_Supervised_Pretraning_pt2.html","title":"15 Self Supervised Pretraning pt2","text":"In\u00a0[1]: Copied! <pre>import torch\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.model_selection import train_test_split\n\nfrom pytorch_widedeep import Trainer\nfrom pytorch_widedeep.models import WideDeep, FTTransformer\nfrom pytorch_widedeep.metrics import Accuracy\nfrom pytorch_widedeep.datasets import load_adult\nfrom pytorch_widedeep.preprocessing import TabPreprocessor\nfrom pytorch_widedeep.self_supervised_training import (\n    ContrastiveDenoisingTrainer,\n)\n</pre> import torch from sklearn.metrics import accuracy_score from sklearn.model_selection import train_test_split  from pytorch_widedeep import Trainer from pytorch_widedeep.models import WideDeep, FTTransformer from pytorch_widedeep.metrics import Accuracy from pytorch_widedeep.datasets import load_adult from pytorch_widedeep.preprocessing import TabPreprocessor from pytorch_widedeep.self_supervised_training import (     ContrastiveDenoisingTrainer, ) In\u00a0[2]: Copied! <pre>df = load_adult(as_frame=True)\ndf.columns = [c.replace(\"-\", \"_\") for c in df.columns]\ndf[\"income_label\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int)\ndf.drop(\"income\", axis=1, inplace=True)\n\n# one could chose to use a validation set for early stopping, hyperparam\n# optimization, etc. This is just an example, so we simply use train/test\n# split\ndf_tr, df_te = train_test_split(df, test_size=0.2, stratify=df.income_label)\n\ncat_embed_cols = [\n    \"workclass\",\n    \"education\",\n    \"marital_status\",\n    \"occupation\",\n    \"relationship\",\n    \"race\",\n    \"gender\",\n    \"capital_gain\",\n    \"capital_loss\",\n    \"native_country\",\n]\ncontinuous_cols = [\"age\", \"hours_per_week\"]\ntarget_col = \"income_label\"\n\ntab_preprocessor = TabPreprocessor(\n    cat_embed_cols=cat_embed_cols,\n    continuous_cols=continuous_cols,\n    with_attention=True,\n    with_cls_token=True,  # this is optional\n)\nX_tab = tab_preprocessor.fit_transform(df_tr)\ntarget = df_tr[target_col].values\n</pre> df = load_adult(as_frame=True) df.columns = [c.replace(\"-\", \"_\") for c in df.columns] df[\"income_label\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int) df.drop(\"income\", axis=1, inplace=True)  # one could chose to use a validation set for early stopping, hyperparam # optimization, etc. This is just an example, so we simply use train/test # split df_tr, df_te = train_test_split(df, test_size=0.2, stratify=df.income_label)  cat_embed_cols = [     \"workclass\",     \"education\",     \"marital_status\",     \"occupation\",     \"relationship\",     \"race\",     \"gender\",     \"capital_gain\",     \"capital_loss\",     \"native_country\", ] continuous_cols = [\"age\", \"hours_per_week\"] target_col = \"income_label\"  tab_preprocessor = TabPreprocessor(     cat_embed_cols=cat_embed_cols,     continuous_cols=continuous_cols,     with_attention=True,     with_cls_token=True,  # this is optional ) X_tab = tab_preprocessor.fit_transform(df_tr) target = df_tr[target_col].values <pre>/Users/javierrodriguezzaurin/Projects/pytorch-widedeep/pytorch_widedeep/preprocessing/tab_preprocessor.py:358: UserWarning: Continuous columns will not be normalised\n  warnings.warn(\"Continuous columns will not be normalised\")\n</pre> In\u00a0[3]: Copied! <pre>ft_transformer = FTTransformer(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=tab_preprocessor.continuous_cols,\n    embed_continuous_method=\"standard\",\n    input_dim=32,\n    kv_compression_factor=0.5,\n    n_blocks=3,\n    n_heads=4,\n)\n</pre> ft_transformer = FTTransformer(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     continuous_cols=tab_preprocessor.continuous_cols,     embed_continuous_method=\"standard\",     input_dim=32,     kv_compression_factor=0.5,     n_blocks=3,     n_heads=4, ) In\u00a0[4]: Copied! <pre># for a full list of the params for the the ContrastiveDenoisingTrainer (which are many) please see the docs.\n# Note that using these params involves some knowledge of the routine and the architecture of the model used\ncontrastive_denoising_trainer = ContrastiveDenoisingTrainer(\n    model=ft_transformer,\n    preprocessor=tab_preprocessor,\n)\ncontrastive_denoising_trainer.pretrain(X_tab, n_epochs=5, batch_size=256)\n</pre> # for a full list of the params for the the ContrastiveDenoisingTrainer (which are many) please see the docs. # Note that using these params involves some knowledge of the routine and the architecture of the model used contrastive_denoising_trainer = ContrastiveDenoisingTrainer(     model=ft_transformer,     preprocessor=tab_preprocessor, ) contrastive_denoising_trainer.pretrain(X_tab, n_epochs=5, batch_size=256) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:13&lt;00:00, 11.73it/s, loss=579]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:12&lt;00:00, 12.56it/s, loss=143]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:12&lt;00:00, 12.49it/s, loss=141]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:11&lt;00:00, 12.77it/s, loss=138]\nepoch 5: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:11&lt;00:00, 13.29it/s, loss=137]\n</pre> In\u00a0[5]: Copied! <pre>contrastive_denoising_trainer.save(\n    path=\"pretrained_weights\", model_filename=\"contrastive_denoising_model.pt\"\n)\n</pre> contrastive_denoising_trainer.save(     path=\"pretrained_weights\", model_filename=\"contrastive_denoising_model.pt\" ) <p>some time has passed</p> In\u00a0[6]: Copied! <pre># some time has passed, we load the model with torch as usual:\ncontrastive_denoising_model = torch.load(\n    \"pretrained_weights/contrastive_denoising_model.pt\"\n)\n</pre> # some time has passed, we load the model with torch as usual: contrastive_denoising_model = torch.load(     \"pretrained_weights/contrastive_denoising_model.pt\" ) <p>NOW, AND THIS IS IMPORTANT! We have loaded the entire contrastive, denoising model. To proceed to the supervised training we ONLY need the attention-based model, which is the 'model' attribute of the trainer, let's have a look</p> In\u00a0[7]: Copied! <pre>contrastive_denoising_model.model\n</pre> contrastive_denoising_model.model Out[7]: <pre>FTTransformer(\n  (cat_embed): SameSizeCatEmbeddings(\n    (embed): Embedding(323, 32, padding_idx=0)\n    (dropout): Dropout(p=0.0, inplace=False)\n  )\n  (cont_norm): Identity()\n  (cont_embed): ContEmbeddings(\n    INFO: [ContLinear = weight(n_cont_cols, embed_dim) + bias(n_cont_cols, embed_dim)]\n    (linear): ContLinear(n_cont_cols=2, embed_dim=32, embed_dropout=0.0)\n    (dropout): Dropout(p=0.0, inplace=False)\n  )\n  (encoder): Sequential(\n    (fttransformer_block0): FTTransformerEncoder(\n      (attn): LinearAttentionLinformer(\n        (dropout): Dropout(p=0.2, inplace=False)\n        (qkv_proj): Linear(in_features=32, out_features=96, bias=False)\n        (out_proj): Linear(in_features=32, out_features=32, bias=False)\n      )\n      (ff): FeedForward(\n        (w_1): Linear(in_features=32, out_features=84, bias=True)\n        (w_2): Linear(in_features=42, out_features=32, bias=True)\n        (dropout): Dropout(p=0.1, inplace=False)\n        (activation): REGLU()\n      )\n      (attn_normadd): NormAdd(\n        (dropout): Dropout(p=0.2, inplace=False)\n        (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n      )\n      (ff_normadd): NormAdd(\n        (dropout): Dropout(p=0.1, inplace=False)\n        (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n      )\n    )\n    (fttransformer_block1): FTTransformerEncoder(\n      (attn): LinearAttentionLinformer(\n        (dropout): Dropout(p=0.2, inplace=False)\n        (qkv_proj): Linear(in_features=32, out_features=96, bias=False)\n        (out_proj): Linear(in_features=32, out_features=32, bias=False)\n      )\n      (ff): FeedForward(\n        (w_1): Linear(in_features=32, out_features=84, bias=True)\n        (w_2): Linear(in_features=42, out_features=32, bias=True)\n        (dropout): Dropout(p=0.1, inplace=False)\n        (activation): REGLU()\n      )\n      (attn_normadd): NormAdd(\n        (dropout): Dropout(p=0.2, inplace=False)\n        (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n      )\n      (ff_normadd): NormAdd(\n        (dropout): Dropout(p=0.1, inplace=False)\n        (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n      )\n    )\n    (fttransformer_block2): FTTransformerEncoder(\n      (attn): LinearAttentionLinformer(\n        (dropout): Dropout(p=0.2, inplace=False)\n        (qkv_proj): Linear(in_features=32, out_features=96, bias=False)\n        (out_proj): Linear(in_features=32, out_features=32, bias=False)\n      )\n      (ff): FeedForward(\n        (w_1): Linear(in_features=32, out_features=84, bias=True)\n        (w_2): Linear(in_features=42, out_features=32, bias=True)\n        (dropout): Dropout(p=0.1, inplace=False)\n        (activation): REGLU()\n      )\n      (attn_normadd): NormAdd(\n        (dropout): Dropout(p=0.2, inplace=False)\n        (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n      )\n      (ff_normadd): NormAdd(\n        (dropout): Dropout(p=0.1, inplace=False)\n        (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n      )\n    )\n  )\n)</pre> In\u00a0[8]: Copied! <pre>pretrained_model = contrastive_denoising_model.model\n</pre> pretrained_model = contrastive_denoising_model.model In\u00a0[9]: Copied! <pre># and as always, ANY supervised model in this library has to go throuth the WideDeep class:\nmodel = WideDeep(deeptabular=pretrained_model)\ntrainer = Trainer(model=model, objective=\"binary\", metrics=[Accuracy])\n\ntrainer.fit(X_tab=X_tab, target=target, n_epochs=5, batch_size=256)\n\n# And, you know...we get a test metric\nX_tab_te = tab_preprocessor.transform(df_te)\ntarget_te = df_te[target_col].values\n\npreds = trainer.predict(X_tab=X_tab_te)\ntest_acc = accuracy_score(target_te, preds)\nprint(test_acc)\n</pre> # and as always, ANY supervised model in this library has to go throuth the WideDeep class: model = WideDeep(deeptabular=pretrained_model) trainer = Trainer(model=model, objective=\"binary\", metrics=[Accuracy])  trainer.fit(X_tab=X_tab, target=target, n_epochs=5, batch_size=256)  # And, you know...we get a test metric X_tab_te = tab_preprocessor.transform(df_te) target_te = df_te[target_col].values  preds = trainer.predict(X_tab=X_tab_te) test_acc = accuracy_score(target_te, preds) print(test_acc) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:05&lt;00:00, 27.19it/s, loss=0.383, metrics={'acc': 0.8176}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:05&lt;00:00, 26.08it/s, loss=0.325, metrics={'acc': 0.8502}]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:05&lt;00:00, 26.56it/s, loss=0.306, metrics={'acc': 0.8601}]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:05&lt;00:00, 27.41it/s, loss=0.295, metrics={'acc': 0.8641}]\nepoch 5: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:06&lt;00:00, 24.70it/s, loss=0.289, metrics={'acc': 0.8656}]\npredict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 39/39 [00:00&lt;00:00, 97.26it/s]</pre> <pre>0.8695874705701709\n</pre> <pre>\n</pre> In\u00a0[\u00a0]: Copied! <pre>\n</pre>"},{"location":"examples/15_Self_Supervised_Pretraning_pt2.html#self-supervised-pretraining-for-tabular-data","title":"Self Supervised Pretraining for Tabular Data\u00b6","text":"<p>We have implemented two Self Supervised Pre-training routines that allow the user to pre-train all tabular models in the library with the exception of the TabPerceiver (which is a special monster).</p> <p>The two routines implemented are illustrated in the figures below. The 1st is from TabNet: Attentive Interpretable Tabular Learning and is designed for models that do not use transformer-based architectures, while the second is from SAINT: Improved Neural Networks for Tabular Data via Row Attention and Contrastive Pre-Training, and is designed for models that use transformer-based architectures.</p> <p> </p> <p>Fig 1. Figure 2 in their paper. I have included de original caption in case is useful, althought the Figure itself is pretty self explanatory</p> <p> </p> <p>Fig 2. Figure 1 in their paper. Here the caption is necessary \ud83d\ude0f</p> <p>It is beyond the scope of this notebook to explain in detail those implementations. Therefore, we strongly recommend the user to go and read the papers if this functionality is of interest to her/him.</p> <p>One thing is worth noticing however. As seen in Fig 1(the TabNet paper's Fig 2) the masking of the input features happens in the feature space. However, the implementation in this library is inspired by that at the dreamquark-ai repo, which is in itself inspired by the original implementation (by the way, at this point I will write it once again. All TabNet related things in this library are inspired when not directly based in the code in that repo, therefore, ALL CREDIT TO THE GUYS AT dreamquark-ai).</p> <p>In that implementation the masking happens in the embedding space, and currently does not mask the entire embedding (i.e. categorical feature). We decided to release as it is in this version and we will implement the exact same process described in the paper in future releases.</p> <p>Having said all of the above let's see how to use self supervision for tabular data with <code>pytorch-widedeep</code>. We will concentrate in this notebook on the 2nd of the two approaches (the 'SAINT approach'). For details on the 1st approach (the 'TabNet' approach) please see <code>16_Self_Supervised_Pretraning_pt1</code>.</p>"},{"location":"examples/15_Self_Supervised_Pretraning_pt2.html#self-supervision-transformer-based-models","title":"Self Supervision transformer-based models..\u00b6","text":"<p>...or in general, for models where the embeddigns have all the same dimensions. In this library, these are:</p> <ul> <li>TabTransformer</li> <li>FTTransformer</li> <li>SAINT</li> <li>TabFastFormer</li> </ul> <p>Note that there is one additional Transformer-based model, the <code>TabPerceiver</code>, however this is a \"particular\" model and at the moment we do not support self supervision for it, but it will come.</p> <p>Let see at one example using the <code>FTTransformer</code>.</p>"},{"location":"examples/16_Usign_a_custom_hugging_face_model.html","title":"16 Usign a custom hugging face model","text":"In\u00a0[1]: Copied! <pre>import numpy as np\nimport torch\nimport lightgbm as lgb\nfrom lightgbm import Dataset as lgbDataset\nfrom scipy.sparse import hstack, csr_matrix\nfrom sklearn.metrics import (\n    f1_score,\n    recall_score,\n    accuracy_score,\n    precision_score,\n    confusion_matrix,\n)\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.feature_extraction.text import TfidfVectorizer\n\nfrom torch import Tensor, nn\nfrom transformers import DistilBertModel, DistilBertTokenizer\nfrom pytorch_widedeep import Trainer\nfrom pytorch_widedeep.models import TabMlp, BasicRNN, WideDeep\nfrom pytorch_widedeep.metrics import F1Score, Accuracy\nfrom pytorch_widedeep.utils import Tokenizer, LabelEncoder\nfrom pytorch_widedeep.preprocessing import TextPreprocessor, TabPreprocessor\nfrom pytorch_widedeep.datasets import load_womens_ecommerce\nfrom pytorch_widedeep.utils.fastai_transforms import (\n    fix_html,\n    spec_add_spaces,\n    rm_useless_spaces,\n)\n</pre> import numpy as np import torch import lightgbm as lgb from lightgbm import Dataset as lgbDataset from scipy.sparse import hstack, csr_matrix from sklearn.metrics import (     f1_score,     recall_score,     accuracy_score,     precision_score,     confusion_matrix, ) from sklearn.model_selection import train_test_split from sklearn.feature_extraction.text import TfidfVectorizer  from torch import Tensor, nn from transformers import DistilBertModel, DistilBertTokenizer from pytorch_widedeep import Trainer from pytorch_widedeep.models import TabMlp, BasicRNN, WideDeep from pytorch_widedeep.metrics import F1Score, Accuracy from pytorch_widedeep.utils import Tokenizer, LabelEncoder from pytorch_widedeep.preprocessing import TextPreprocessor, TabPreprocessor from pytorch_widedeep.datasets import load_womens_ecommerce from pytorch_widedeep.utils.fastai_transforms import (     fix_html,     spec_add_spaces,     rm_useless_spaces, ) <p>Let's load the data and have a look:</p> In\u00a0[2]: Copied! <pre>df = load_womens_ecommerce(as_frame=True)\n\ndf.columns = [c.replace(\" \", \"_\").lower() for c in df.columns]\n\n# classes from [0,num_class)\ndf[\"rating\"] = (df[\"rating\"] - 1).astype(\"int64\")\n\n# group reviews with 1 and 2 scores into one class\ndf.loc[df.rating == 0, \"rating\"] = 1\n\n# and back again to [0,num_class)\ndf[\"rating\"] = (df[\"rating\"] - 1).astype(\"int64\")\n\n# drop short reviews\ndf = df[~df.review_text.isna()]\ndf[\"review_length\"] = df.review_text.apply(lambda x: len(x.split(\" \")))\ndf = df[df.review_length &gt;= 5]\ndf = df.drop(\"review_length\", axis=1).reset_index(drop=True)\n</pre> df = load_womens_ecommerce(as_frame=True)  df.columns = [c.replace(\" \", \"_\").lower() for c in df.columns]  # classes from [0,num_class) df[\"rating\"] = (df[\"rating\"] - 1).astype(\"int64\")  # group reviews with 1 and 2 scores into one class df.loc[df.rating == 0, \"rating\"] = 1  # and back again to [0,num_class) df[\"rating\"] = (df[\"rating\"] - 1).astype(\"int64\")  # drop short reviews df = df[~df.review_text.isna()] df[\"review_length\"] = df.review_text.apply(lambda x: len(x.split(\" \"))) df = df[df.review_length &gt;= 5] df = df.drop(\"review_length\", axis=1).reset_index(drop=True) In\u00a0[3]: Copied! <pre>df.head()\n</pre> df.head() Out[3]: clothing_id age title review_text rating recommended_ind positive_feedback_count division_name department_name class_name 0 767 33 None Absolutely wonderful - silky and sexy and comf... 2 1 0 Initmates Intimate Intimates 1 1080 34 None Love this dress!  it's sooo pretty.  i happene... 3 1 4 General Dresses Dresses 2 1077 60 Some major design flaws I had such high hopes for this dress and reall... 1 0 0 General Dresses Dresses 3 1049 50 My favorite buy! I love, love, love this jumpsuit. it's fun, fl... 3 1 0 General Petite Bottoms Pants 4 847 47 Flattering shirt This shirt is very flattering to all due to th... 3 1 6 General Tops Blouses <p>So, we will use the <code>review_text</code> column to predict the <code>rating</code>. Later on, we will try to combine it with some other columns (like <code>division_name</code> and <code>age</code>) see if these help.</p> <p>Let's first have a look to the distribution of ratings</p> In\u00a0[4]: Copied! <pre>df.rating.value_counts()\n</pre> df.rating.value_counts() Out[4]: <pre>rating\n3    12515\n2     4904\n1     2820\n0     2369\nName: count, dtype: int64</pre> <p>This shows that we could have perhaps grouped rating scores of 1, 2 and 3 into 1...but anyway, let's just move on with those 4 classes.</p> <p>We are not going to carry any hyperparameter optimization here, so, we will only need a train and a test set (i.e.  no need of a validation set for the example in this notebook)</p> In\u00a0[5]: Copied! <pre>train, test = train_test_split(df, train_size=0.8, random_state=1, stratify=df.rating)\n</pre> train, test = train_test_split(df, train_size=0.8, random_state=1, stratify=df.rating) <p>Let's see what we have to beat. What metrics would we obtain if we always predict the most common rating (3)?</p> In\u00a0[6]: Copied! <pre>most_common_pred = [train.rating.value_counts().index[0]] * len(test)\n\nmost_common_acc = accuracy_score(test.rating, most_common_pred)\nmost_common_f1 = f1_score(test.rating, most_common_pred, average=\"weighted\")\n</pre> most_common_pred = [train.rating.value_counts().index[0]] * len(test)  most_common_acc = accuracy_score(test.rating, most_common_pred) most_common_f1 = f1_score(test.rating, most_common_pred, average=\"weighted\") In\u00a0[7]: Copied! <pre>print(f\"Accuracy: {most_common_acc}. F1 Score: {most_common_f1}\")\n</pre> print(f\"Accuracy: {most_common_acc}. F1 Score: {most_common_f1}\") <pre>Accuracy: 0.553516143299425. F1 Score: 0.3944344218301668\n</pre> <p>ok, these are our \"baseline\" metrics.</p> <p>Let's start by using simply tf-idf + lightGBM</p> In\u00a0[8]: Copied! <pre># ?Tokenizer\n</pre> # ?Tokenizer In\u00a0[9]: Copied! <pre># this Tokenizer is part of our utils module but of course, any valid tokenizer can be used here.\n\n# When using notebooks there seems to be an issue related with multiprocessing (and sometimes tqdm)\n# that can only be solved by using only one CPU\ntok = Tokenizer(n_cpus=1)\ntok_reviews_tr = tok.process_all(train.review_text.tolist())\ntok_reviews_te = tok.process_all(test.review_text.tolist())\n</pre> # this Tokenizer is part of our utils module but of course, any valid tokenizer can be used here.  # When using notebooks there seems to be an issue related with multiprocessing (and sometimes tqdm) # that can only be solved by using only one CPU tok = Tokenizer(n_cpus=1) tok_reviews_tr = tok.process_all(train.review_text.tolist()) tok_reviews_te = tok.process_all(test.review_text.tolist()) In\u00a0[10]: Copied! <pre>vectorizer = TfidfVectorizer(\n    max_features=5000, preprocessor=lambda x: x, tokenizer=lambda x: x, min_df=5\n)\n\nX_text_tr = vectorizer.fit_transform(tok_reviews_tr)\nX_text_te = vectorizer.transform(tok_reviews_te)\n</pre> vectorizer = TfidfVectorizer(     max_features=5000, preprocessor=lambda x: x, tokenizer=lambda x: x, min_df=5 )  X_text_tr = vectorizer.fit_transform(tok_reviews_tr) X_text_te = vectorizer.transform(tok_reviews_te) <pre>/Users/javierrodriguezzaurin/.pyenv/versions/3.10.13/envs/widedeep310/lib/python3.10/site-packages/sklearn/feature_extraction/text.py:525: UserWarning: The parameter 'token_pattern' will not be used since 'tokenizer' is not None'\n  warnings.warn(\n</pre> In\u00a0[11]: Copied! <pre>X_text_tr\n</pre> X_text_tr Out[11]: <pre>&lt;18086x4566 sparse matrix of type '&lt;class 'numpy.float64'&gt;'\n\twith 884074 stored elements in Compressed Sparse Row format&gt;</pre> <p>We now move our matrices to lightGBM <code>Dataset</code> format</p> In\u00a0[12]: Copied! <pre>lgbtrain_text = lgbDataset(\n    X_text_tr,\n    train.rating.values,\n    free_raw_data=False,\n)\n\nlgbtest_text = lgbDataset(\n    X_text_te,\n    test.rating.values,\n    reference=lgbtrain_text,\n    free_raw_data=False,\n)\n</pre> lgbtrain_text = lgbDataset(     X_text_tr,     train.rating.values,     free_raw_data=False, )  lgbtest_text = lgbDataset(     X_text_te,     test.rating.values,     reference=lgbtrain_text,     free_raw_data=False, ) <p>and off we go. By the way, I think as we run the next cell, we should appreciate how fast lightGBM runs. Yes, the input is a sparse matrix, but still, trains on 18086x4566 in a matter of secs</p> In\u00a0[\u00a0]: Copied! <pre>lgb_text_model = lgb.train(\n    {\"objective\": \"multiclass\", \"num_classes\": 4},\n    lgbtrain_text,\n    valid_sets=[lgbtest_text, lgbtrain_text],\n    valid_names=[\"test\", \"train\"],\n)\n</pre> lgb_text_model = lgb.train(     {\"objective\": \"multiclass\", \"num_classes\": 4},     lgbtrain_text,     valid_sets=[lgbtest_text, lgbtrain_text],     valid_names=[\"test\", \"train\"], ) In\u00a0[14]: Copied! <pre>preds_text = lgb_text_model.predict(X_text_te)\npred_text_class = np.argmax(preds_text, 1)\n</pre> preds_text = lgb_text_model.predict(X_text_te) pred_text_class = np.argmax(preds_text, 1) In\u00a0[15]: Copied! <pre>acc_text = accuracy_score(lgbtest_text.label, pred_text_class)\nf1_text = f1_score(lgbtest_text.label, pred_text_class, average=\"weighted\")\ncm_text = confusion_matrix(lgbtest_text.label, pred_text_class)\n</pre> acc_text = accuracy_score(lgbtest_text.label, pred_text_class) f1_text = f1_score(lgbtest_text.label, pred_text_class, average=\"weighted\") cm_text = confusion_matrix(lgbtest_text.label, pred_text_class) In\u00a0[16]: Copied! <pre>print(f\"LightGBM Accuracy: {acc_text}. LightGBM F1 Score: {f1_text}\")\n</pre> print(f\"LightGBM Accuracy: {acc_text}. LightGBM F1 Score: {f1_text}\") <pre>LightGBM Accuracy: 0.6444051304732419. LightGBM F1 Score: 0.617154488246181\n</pre> In\u00a0[17]: Copied! <pre>print(f\"LightGBM Confusion Matrix: \\n {cm_text}\")\n</pre> print(f\"LightGBM Confusion Matrix: \\n {cm_text}\") <pre>LightGBM Confusion Matrix: \n [[ 199  135   61   79]\n [ 123  169  149  123]\n [  30   94  279  578]\n [  16   30  190 2267]]\n</pre> <p>Ok, so, with no hyperparameter optimization lightGBM gets an accuracy of 0.64 and a F1 score of 0.62. This is significantly better than predicting always the most popular.</p> <p>Let's see if in this implementation, some additional features, like <code>age</code> or <code>class_name</code> are of any help</p> In\u00a0[18]: Copied! <pre>tab_cols = [\n    \"age\",\n    \"division_name\",\n    \"department_name\",\n    \"class_name\",\n]\n\nfor tab_df in [train, test]:\n    for c in [\"division_name\", \"department_name\", \"class_name\"]:\n        tab_df[c] = tab_df[c].str.lower()\n        tab_df[c].fillna(\"missing\", inplace=True)\n</pre> tab_cols = [     \"age\",     \"division_name\",     \"department_name\",     \"class_name\", ]  for tab_df in [train, test]:     for c in [\"division_name\", \"department_name\", \"class_name\"]:         tab_df[c] = tab_df[c].str.lower()         tab_df[c].fillna(\"missing\", inplace=True) In\u00a0[19]: Copied! <pre># This is our LabelEncoder. A class that is designed to work with the models in this library but\n# can be used for general purposes\nle = LabelEncoder(columns_to_encode=[\"division_name\", \"department_name\", \"class_name\"])\ntrain_tab_le = le.fit_transform(train)\ntest_tab_le = le.transform(test)\n</pre> # This is our LabelEncoder. A class that is designed to work with the models in this library but # can be used for general purposes le = LabelEncoder(columns_to_encode=[\"division_name\", \"department_name\", \"class_name\"]) train_tab_le = le.fit_transform(train) test_tab_le = le.transform(test) In\u00a0[20]: Copied! <pre>train_tab_le.head()\n</pre> train_tab_le.head() Out[20]: clothing_id age title review_text rating recommended_ind positive_feedback_count division_name department_name class_name 4541 836 35 None Bought this on sale in my reg size- 10. im 5'9... 2 1 2 1 1 1 18573 1022 25 Look like \"mom jeans\" Maybe i just have the wrong body type for thes... 1 0 0 2 2 2 1058 815 39 Ig brought me here Love the way this top layers under my jackets ... 2 1 0 1 1 1 12132 984 47 Runs small especially the arms I love this jacket. it's the prettiest and mos... 3 1 0 1 3 3 20756 1051 42 True red, true beauty. These pants are gorgeous--the fabric has a sat... 3 1 0 2 2 4 <p>let's for example have a look to the encodings for the categorical feature <code>class_name</code></p> In\u00a0[21]: Copied! <pre>le.encoding_dict[\"class_name\"]\n</pre> le.encoding_dict[\"class_name\"] Out[21]: <pre>{'blouses': 1,\n 'jeans': 2,\n 'jackets': 3,\n 'pants': 4,\n 'knits': 5,\n 'dresses': 6,\n 'skirts': 7,\n 'sweaters': 8,\n 'fine gauge': 9,\n 'legwear': 10,\n 'lounge': 11,\n 'shorts': 12,\n 'outerwear': 13,\n 'intimates': 14,\n 'swim': 15,\n 'trend': 16,\n 'sleep': 17,\n 'layering': 18,\n 'missing': 19,\n 'casual bottoms': 20,\n 'chemises': 21}</pre> In\u00a0[22]: Copied! <pre># tabular training and test sets\nX_tab_tr = csr_matrix(train_tab_le[tab_cols].values)\nX_tab_te = csr_matrix(test_tab_le[tab_cols].values)\n\n# text + tabular training and test sets\nX_tab_text_tr = hstack((X_tab_tr, X_text_tr))\nX_tab_text_te = hstack((X_tab_te, X_text_te))\n</pre> # tabular training and test sets X_tab_tr = csr_matrix(train_tab_le[tab_cols].values) X_tab_te = csr_matrix(test_tab_le[tab_cols].values)  # text + tabular training and test sets X_tab_text_tr = hstack((X_tab_tr, X_text_tr)) X_tab_text_te = hstack((X_tab_te, X_text_te)) In\u00a0[23]: Copied! <pre>X_tab_tr\n</pre> X_tab_tr Out[23]: <pre>&lt;18086x4 sparse matrix of type '&lt;class 'numpy.int64'&gt;'\n\twith 72344 stored elements in Compressed Sparse Row format&gt;</pre> In\u00a0[24]: Copied! <pre>X_tab_text_tr\n</pre> X_tab_text_tr Out[24]: <pre>&lt;18086x4570 sparse matrix of type '&lt;class 'numpy.float64'&gt;'\n\twith 956418 stored elements in Compressed Sparse Row format&gt;</pre> In\u00a0[25]: Copied! <pre>lgbtrain_tab_text = lgbDataset(\n    X_tab_text_tr,\n    train.rating.values,\n    categorical_feature=[0, 1, 2, 3],\n    free_raw_data=False,\n)\n\nlgbtest_tab_text = lgbDataset(\n    X_tab_text_te,\n    test.rating.values,\n    reference=lgbtrain_tab_text,\n    free_raw_data=False,\n)\n</pre> lgbtrain_tab_text = lgbDataset(     X_tab_text_tr,     train.rating.values,     categorical_feature=[0, 1, 2, 3],     free_raw_data=False, )  lgbtest_tab_text = lgbDataset(     X_tab_text_te,     test.rating.values,     reference=lgbtrain_tab_text,     free_raw_data=False, ) In\u00a0[26]: Copied! <pre>lgb_tab_text_model = lgb.train(\n    {\"objective\": \"multiclass\", \"num_classes\": 4},\n    lgbtrain_tab_text,\n    valid_sets=[lgbtrain_tab_text, lgbtest_tab_text],\n    valid_names=[\"test\", \"train\"],\n    verbose_eval=False,\n)\n</pre> lgb_tab_text_model = lgb.train(     {\"objective\": \"multiclass\", \"num_classes\": 4},     lgbtrain_tab_text,     valid_sets=[lgbtrain_tab_text, lgbtest_tab_text],     valid_names=[\"test\", \"train\"],     verbose_eval=False, ) <pre>/opt/conda/envs/wd38/lib/python3.8/site-packages/lightgbm/basic.py:2065: UserWarning: Using categorical_feature in Dataset.\n  _log_warning('Using categorical_feature in Dataset.')\n/opt/conda/envs/wd38/lib/python3.8/site-packages/lightgbm/basic.py:2068: UserWarning: categorical_feature in Dataset is overridden.\nNew categorical_feature is [0, 1, 2, 3]\n  _log_warning('categorical_feature in Dataset is overridden.\\n'\n/opt/conda/envs/wd38/lib/python3.8/site-packages/lightgbm/engine.py:239: UserWarning: 'verbose_eval' argument is deprecated and will be removed in a future release of LightGBM. Pass 'log_evaluation()' callback via 'callbacks' argument instead.\n  _log_warning(\"'verbose_eval' argument is deprecated and will be removed in a future release of LightGBM. \"\n</pre> <pre>[LightGBM] [Warning] Auto-choosing col-wise multi-threading, the overhead of testing was 0.138280 seconds.\nYou can set `force_col_wise=true` to remove the overhead.\n[LightGBM] [Info] Total Bins 143432\n[LightGBM] [Info] Number of data points in the train set: 18086, number of used features: 2289\n[LightGBM] [Info] Start training from score -2.255919\n[LightGBM] [Info] Start training from score -2.081545\n[LightGBM] [Info] Start training from score -1.528281\n[LightGBM] [Info] Start training from score -0.591354\n</pre> <pre>/opt/conda/envs/wd38/lib/python3.8/site-packages/lightgbm/basic.py:1780: UserWarning: Overriding the parameters from Reference Dataset.\n  _log_warning('Overriding the parameters from Reference Dataset.')\n/opt/conda/envs/wd38/lib/python3.8/site-packages/lightgbm/basic.py:1513: UserWarning: categorical_column in param dict is overridden.\n  _log_warning(f'{cat_alias} in param dict is overridden.')\n</pre> In\u00a0[27]: Copied! <pre>preds_tab_text = lgb_tab_text_model.predict(X_tab_text_te)\npreds_tab_text_class = np.argmax(preds_tab_text, 1)\n\nacc_tab_text = accuracy_score(lgbtest_tab_text.label, preds_tab_text_class)\nf1_tab_text = f1_score(lgbtest_tab_text.label, preds_tab_text_class, average=\"weighted\")\ncm_tab_text = confusion_matrix(lgbtest_tab_text.label, preds_tab_text_class)\n</pre> preds_tab_text = lgb_tab_text_model.predict(X_tab_text_te) preds_tab_text_class = np.argmax(preds_tab_text, 1)  acc_tab_text = accuracy_score(lgbtest_tab_text.label, preds_tab_text_class) f1_tab_text = f1_score(lgbtest_tab_text.label, preds_tab_text_class, average=\"weighted\") cm_tab_text = confusion_matrix(lgbtest_tab_text.label, preds_tab_text_class) In\u00a0[28]: Copied! <pre>print(\n    f\"LightGBM text + tabular Accuracy: {acc_tab_text}. LightGBM text + tabular F1 Score: {f1_tab_text}\"\n)\n</pre> print(     f\"LightGBM text + tabular Accuracy: {acc_tab_text}. LightGBM text + tabular F1 Score: {f1_tab_text}\" ) <pre>LightGBM text + tabular Accuracy: 0.6382131800088456. LightGBM text + tabular F1 Score: 0.6080251307242649\n</pre> In\u00a0[29]: Copied! <pre>print(f\"LightGBM text + tabular Confusion Matrix:\\n {cm_tab_text}\")\n</pre> print(f\"LightGBM text + tabular Confusion Matrix:\\n {cm_tab_text}\") <pre>LightGBM text + tabular Confusion Matrix:\n [[ 193  123   68   90]\n [ 123  146  157  138]\n [  37   90  272  582]\n [  16   37  175 2275]]\n</pre> <p>So, in this set up, the addition tabular columns do not help performance.</p> In\u00a0[30]: Copied! <pre>text_preprocessor = TextPreprocessor(\n    text_col=\"review_text\", max_vocab=5000, min_freq=5, maxlen=90, n_cpus=1\n)\n\nwd_X_text_tr = text_preprocessor.fit_transform(train)\nwd_X_text_te = text_preprocessor.transform(test)\n</pre> text_preprocessor = TextPreprocessor(     text_col=\"review_text\", max_vocab=5000, min_freq=5, maxlen=90, n_cpus=1 )  wd_X_text_tr = text_preprocessor.fit_transform(train) wd_X_text_te = text_preprocessor.transform(test) <pre>The vocabulary contains 4328 tokens\n</pre> In\u00a0[31]: Copied! <pre>basic_rnn = BasicRNN(\n    vocab_size=len(text_preprocessor.vocab.itos),\n    embed_dim=300,\n    hidden_dim=64,\n    n_layers=3,\n    rnn_dropout=0.2,\n    head_hidden_dims=[32],\n)\n\n\nwd_text_model = WideDeep(deeptext=basic_rnn, pred_dim=4)\n</pre> basic_rnn = BasicRNN(     vocab_size=len(text_preprocessor.vocab.itos),     embed_dim=300,     hidden_dim=64,     n_layers=3,     rnn_dropout=0.2,     head_hidden_dims=[32], )   wd_text_model = WideDeep(deeptext=basic_rnn, pred_dim=4) In\u00a0[32]: Copied! <pre>wd_text_model\n</pre> wd_text_model Out[32]: <pre>WideDeep(\n  (deeptext): Sequential(\n    (0): BasicRNN(\n      (word_embed): Embedding(4328, 300, padding_idx=1)\n      (rnn): LSTM(300, 64, num_layers=3, batch_first=True, dropout=0.2)\n      (rnn_mlp): MLP(\n        (mlp): Sequential(\n          (dense_layer_0): Sequential(\n            (0): Linear(in_features=64, out_features=32, bias=True)\n            (1): ReLU(inplace=True)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=32, out_features=4, bias=True)\n  )\n)</pre> In\u00a0[33]: Copied! <pre>text_trainer = Trainer(\n    wd_text_model,\n    objective=\"multiclass\",\n    metrics=[Accuracy, F1Score(average=True)],\n    num_workers=0,  # As in the case of the tokenizer, in notebook I need to set this to 0 for the Trainer to work\n)\n</pre> text_trainer = Trainer(     wd_text_model,     objective=\"multiclass\",     metrics=[Accuracy, F1Score(average=True)],     num_workers=0,  # As in the case of the tokenizer, in notebook I need to set this to 0 for the Trainer to work ) In\u00a0[34]: Copied! <pre>text_trainer.fit(\n    X_text=wd_X_text_tr,\n    target=train.rating.values,\n    n_epochs=5,\n    batch_size=256,\n)\n</pre> text_trainer.fit(     X_text=wd_X_text_tr,     target=train.rating.values,     n_epochs=5,     batch_size=256, ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 71/71 [00:01&lt;00:00, 52.39it/s, loss=1.16, metrics={'acc': 0.5349, 'f1': 0.2011}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 71/71 [00:01&lt;00:00, 70.35it/s, loss=0.964, metrics={'acc': 0.5827, 'f1': 0.3005}]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 71/71 [00:01&lt;00:00, 70.33it/s, loss=0.845, metrics={'acc': 0.6252, 'f1': 0.4133}]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 71/71 [00:01&lt;00:00, 69.99it/s, loss=0.765, metrics={'acc': 0.6575, 'f1': 0.4875}]\nepoch 5: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 71/71 [00:01&lt;00:00, 69.55it/s, loss=0.709, metrics={'acc': 0.6879, 'f1': 0.5423}]\n</pre> In\u00a0[35]: Copied! <pre>wd_pred_text = text_trainer.predict_proba(X_text=wd_X_text_te)\nwd_pred_text_class = np.argmax(wd_pred_text, 1)\n</pre> wd_pred_text = text_trainer.predict_proba(X_text=wd_X_text_te) wd_pred_text_class = np.argmax(wd_pred_text, 1) <pre>predict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 18/18 [00:00&lt;00:00, 211.51it/s]\n</pre> In\u00a0[36]: Copied! <pre>wd_acc_text = accuracy_score(test.rating, wd_pred_text_class)\nwd_f1_text = f1_score(test.rating, wd_pred_text_class, average=\"weighted\")\nwd_cm_text = confusion_matrix(test.rating, wd_pred_text_class)\n</pre> wd_acc_text = accuracy_score(test.rating, wd_pred_text_class) wd_f1_text = f1_score(test.rating, wd_pred_text_class, average=\"weighted\") wd_cm_text = confusion_matrix(test.rating, wd_pred_text_class) In\u00a0[37]: Copied! <pre>print(f\"Basic RNN Accuracy: {wd_acc_text}. Basic RNN F1 Score: {wd_f1_text}\")\n</pre> print(f\"Basic RNN Accuracy: {wd_acc_text}. Basic RNN F1 Score: {wd_f1_text}\") <pre>Basic RNN Accuracy: 0.6076957098628926. Basic RNN F1 Score: 0.6017335854471788\n</pre> In\u00a0[38]: Copied! <pre>print(f\"Basic RNN Confusion Matrix:\\n {wd_cm_text}\")\n</pre> print(f\"Basic RNN Confusion Matrix:\\n {wd_cm_text}\") <pre>Basic RNN Confusion Matrix:\n [[ 327   76   62    9]\n [ 285  115  117   47]\n [ 131  122  315  413]\n [  42   69  401 1991]]\n</pre> <p>The performance is very similar to that of using simply tf-idf and lightgbm. Let see if adding tabular features helps when using <code>pytorch-widedeep</code></p> In\u00a0[39]: Copied! <pre># ?TabPreprocessor\n</pre> # ?TabPreprocessor In\u00a0[40]: Copied! <pre>tab_preprocessor = TabPreprocessor(cat_embed_cols=tab_cols)\n\nwd_X_tab_tr = tab_preprocessor.fit_transform(train)\nwd_X_tab_te = tab_preprocessor.transform(test)\n</pre> tab_preprocessor = TabPreprocessor(cat_embed_cols=tab_cols)  wd_X_tab_tr = tab_preprocessor.fit_transform(train) wd_X_tab_te = tab_preprocessor.transform(test) In\u00a0[41]: Copied! <pre># ?TabMlp\n</pre> # ?TabMlp In\u00a0[42]: Copied! <pre>tab_model = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    mlp_hidden_dims=[100, 50],\n)\n</pre> tab_model = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     mlp_hidden_dims=[100, 50], ) In\u00a0[43]: Copied! <pre>tab_model\n</pre> tab_model Out[43]: <pre>TabMlp(\n  (cat_and_cont_embed): DiffSizeCatAndContEmbeddings(\n    (cat_embed): DiffSizeCatEmbeddings(\n      (embed_layers): ModuleDict(\n        (emb_layer_age): Embedding(78, 18, padding_idx=0)\n        (emb_layer_division_name): Embedding(5, 3, padding_idx=0)\n        (emb_layer_department_name): Embedding(8, 5, padding_idx=0)\n        (emb_layer_class_name): Embedding(22, 9, padding_idx=0)\n      )\n      (embedding_dropout): Dropout(p=0.1, inplace=False)\n    )\n  )\n  (encoder): MLP(\n    (mlp): Sequential(\n      (dense_layer_0): Sequential(\n        (0): Dropout(p=0.1, inplace=False)\n        (1): Linear(in_features=35, out_features=100, bias=True)\n        (2): ReLU(inplace=True)\n      )\n      (dense_layer_1): Sequential(\n        (0): Dropout(p=0.1, inplace=False)\n        (1): Linear(in_features=100, out_features=50, bias=True)\n        (2): ReLU(inplace=True)\n      )\n    )\n  )\n)</pre> In\u00a0[44]: Copied! <pre>text_model = BasicRNN(\n    vocab_size=len(text_preprocessor.vocab.itos),\n    embed_dim=300,\n    hidden_dim=64,\n    n_layers=3,\n    rnn_dropout=0.2,\n    head_hidden_dims=[32],\n)\n</pre> text_model = BasicRNN(     vocab_size=len(text_preprocessor.vocab.itos),     embed_dim=300,     hidden_dim=64,     n_layers=3,     rnn_dropout=0.2,     head_hidden_dims=[32], ) In\u00a0[45]: Copied! <pre>wd_tab_and_text_model = WideDeep(deeptabular=tab_model, deeptext=text_model, pred_dim=4)\n</pre> wd_tab_and_text_model = WideDeep(deeptabular=tab_model, deeptext=text_model, pred_dim=4) In\u00a0[46]: Copied! <pre>wd_tab_and_text_model\n</pre> wd_tab_and_text_model Out[46]: <pre>WideDeep(\n  (deeptabular): Sequential(\n    (0): TabMlp(\n      (cat_and_cont_embed): DiffSizeCatAndContEmbeddings(\n        (cat_embed): DiffSizeCatEmbeddings(\n          (embed_layers): ModuleDict(\n            (emb_layer_age): Embedding(78, 18, padding_idx=0)\n            (emb_layer_division_name): Embedding(5, 3, padding_idx=0)\n            (emb_layer_department_name): Embedding(8, 5, padding_idx=0)\n            (emb_layer_class_name): Embedding(22, 9, padding_idx=0)\n          )\n          (embedding_dropout): Dropout(p=0.1, inplace=False)\n        )\n      )\n      (encoder): MLP(\n        (mlp): Sequential(\n          (dense_layer_0): Sequential(\n            (0): Dropout(p=0.1, inplace=False)\n            (1): Linear(in_features=35, out_features=100, bias=True)\n            (2): ReLU(inplace=True)\n          )\n          (dense_layer_1): Sequential(\n            (0): Dropout(p=0.1, inplace=False)\n            (1): Linear(in_features=100, out_features=50, bias=True)\n            (2): ReLU(inplace=True)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=50, out_features=4, bias=True)\n  )\n  (deeptext): Sequential(\n    (0): BasicRNN(\n      (word_embed): Embedding(4328, 300, padding_idx=1)\n      (rnn): LSTM(300, 64, num_layers=3, batch_first=True, dropout=0.2)\n      (rnn_mlp): MLP(\n        (mlp): Sequential(\n          (dense_layer_0): Sequential(\n            (0): Linear(in_features=64, out_features=32, bias=True)\n            (1): ReLU(inplace=True)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=32, out_features=4, bias=True)\n  )\n)</pre> In\u00a0[47]: Copied! <pre>tab_and_text_trainer = Trainer(\n    wd_tab_and_text_model,\n    objective=\"multiclass\",\n    metrics=[Accuracy, F1Score(average=True)],\n    num_workers=0,  # As in the case of the tokenizer, in notebook I need to set this to 0 for the Trainer to work\n)\n</pre> tab_and_text_trainer = Trainer(     wd_tab_and_text_model,     objective=\"multiclass\",     metrics=[Accuracy, F1Score(average=True)],     num_workers=0,  # As in the case of the tokenizer, in notebook I need to set this to 0 for the Trainer to work ) In\u00a0[48]: Copied! <pre>tab_and_text_trainer.fit(\n    X_tab=wd_X_tab_tr,\n    X_text=wd_X_text_tr,\n    target=train.rating.values,\n    n_epochs=5,\n    batch_size=256,\n)\n</pre> tab_and_text_trainer.fit(     X_tab=wd_X_tab_tr,     X_text=wd_X_text_tr,     target=train.rating.values,     n_epochs=5,     batch_size=256, ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 71/71 [00:01&lt;00:00, 52.04it/s, loss=1.13, metrics={'acc': 0.538, 'f1': 0.1911}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 71/71 [00:01&lt;00:00, 52.28it/s, loss=0.936, metrics={'acc': 0.5887, 'f1': 0.3507}]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 71/71 [00:01&lt;00:00, 52.26it/s, loss=0.825, metrics={'acc': 0.6394, 'f1': 0.4545}]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 71/71 [00:01&lt;00:00, 51.33it/s, loss=0.757, metrics={'acc': 0.6696, 'f1': 0.5214}]\nepoch 5: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 71/71 [00:01&lt;00:00, 50.39it/s, loss=0.702, metrics={'acc': 0.6963, 'f1': 0.5654}]\n</pre> In\u00a0[49]: Copied! <pre>wd_pred_tab_and_text = tab_and_text_trainer.predict_proba(\n    X_tab=wd_X_tab_te, X_text=wd_X_text_te\n)\nwd_pred_tab_and_text_class = np.argmax(wd_pred_tab_and_text, 1)\n</pre> wd_pred_tab_and_text = tab_and_text_trainer.predict_proba(     X_tab=wd_X_tab_te, X_text=wd_X_text_te ) wd_pred_tab_and_text_class = np.argmax(wd_pred_tab_and_text, 1) <pre>predict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 18/18 [00:00&lt;00:00, 136.94it/s]\n</pre> In\u00a0[50]: Copied! <pre>wd_acc_tab_and_text = accuracy_score(test.rating, wd_pred_tab_and_text_class)\nwd_f1_tab_and_text = f1_score(\n    test.rating, wd_pred_tab_and_text_class, average=\"weighted\"\n)\nwd_cm_tab_and_text = confusion_matrix(test.rating, wd_pred_tab_and_text_class)\n</pre> wd_acc_tab_and_text = accuracy_score(test.rating, wd_pred_tab_and_text_class) wd_f1_tab_and_text = f1_score(     test.rating, wd_pred_tab_and_text_class, average=\"weighted\" ) wd_cm_tab_and_text = confusion_matrix(test.rating, wd_pred_tab_and_text_class) In\u00a0[51]: Copied! <pre>print(\n    f\"Basic RNN + Tabular  Accuracy: {wd_acc_tab_and_text}. Basic RNN + TabularF1 Score: {wd_f1_tab_and_text}\"\n)\nprint(f\"Basic RNN + Tabular  Confusion Matrix:\\n {wd_cm_tab_and_text}\")\n</pre> print(     f\"Basic RNN + Tabular  Accuracy: {wd_acc_tab_and_text}. Basic RNN + TabularF1 Score: {wd_f1_tab_and_text}\" ) print(f\"Basic RNN + Tabular  Confusion Matrix:\\n {wd_cm_tab_and_text}\") <pre>Basic RNN + Tabular  Accuracy: 0.6333480760725343. Basic RNN + TabularF1 Score: 0.6332310089593208\nBasic RNN + Tabular  Confusion Matrix:\n [[ 267  132   65   10]\n [ 198  168  159   39]\n [  57  113  410  401]\n [  12   58  414 2019]]\n</pre> <p>We are going to \"manually\" code the Tokenizer and the model and see how they can be used as part of the process along with the <code>pytorch-widedeep</code> library.</p> <p>Tokenizer:</p> In\u00a0[52]: Copied! <pre>class BertTokenizer(object):\n    def __init__(\n        self,\n        pretrained_tokenizer=\"distilbert-base-uncased\",\n        do_lower_case=True,\n        max_length=90,\n    ):\n        super(BertTokenizer, self).__init__()\n        self.pretrained_tokenizer = pretrained_tokenizer\n        self.do_lower_case = do_lower_case\n        self.max_length = max_length\n\n    def fit(self, texts):\n        self.tokenizer = DistilBertTokenizer.from_pretrained(\n            self.pretrained_tokenizer, do_lower_case=self.do_lower_case\n        )\n\n        return self\n\n    def transform(self, texts):\n        input_ids = []\n        for text in texts:\n            encoded_sent = self.tokenizer.encode_plus(\n                text=self._pre_rules(text),\n                add_special_tokens=True,\n                max_length=self.max_length,\n                padding=\"max_length\",\n                truncation=True,\n            )\n\n            input_ids.append(encoded_sent.get(\"input_ids\"))\n        return np.stack(input_ids)\n\n    def fit_transform(self, texts):\n        return self.fit(texts).transform(texts)\n\n    @staticmethod\n    def _pre_rules(text):\n        return fix_html(rm_useless_spaces(spec_add_spaces(text)))\n</pre> class BertTokenizer(object):     def __init__(         self,         pretrained_tokenizer=\"distilbert-base-uncased\",         do_lower_case=True,         max_length=90,     ):         super(BertTokenizer, self).__init__()         self.pretrained_tokenizer = pretrained_tokenizer         self.do_lower_case = do_lower_case         self.max_length = max_length      def fit(self, texts):         self.tokenizer = DistilBertTokenizer.from_pretrained(             self.pretrained_tokenizer, do_lower_case=self.do_lower_case         )          return self      def transform(self, texts):         input_ids = []         for text in texts:             encoded_sent = self.tokenizer.encode_plus(                 text=self._pre_rules(text),                 add_special_tokens=True,                 max_length=self.max_length,                 padding=\"max_length\",                 truncation=True,             )              input_ids.append(encoded_sent.get(\"input_ids\"))         return np.stack(input_ids)      def fit_transform(self, texts):         return self.fit(texts).transform(texts)      @staticmethod     def _pre_rules(text):         return fix_html(rm_useless_spaces(spec_add_spaces(text))) <p>Model:</p> In\u00a0[53]: Copied! <pre>class BertModel(nn.Module):\n    def __init__(\n        self,\n        model_name: str = \"distilbert-base-uncased\",\n        freeze_bert: bool = False,\n    ):\n        super(BertModel, self).__init__()\n\n        self.bert = DistilBertModel.from_pretrained(\n            model_name,\n        )\n\n        if freeze_bert:\n            for param in self.bert.parameters():\n                param.requires_grad = False\n\n    def forward(self, X_inp: Tensor) -&gt; Tensor:\n        attn_mask = (X_inp != 0).type(torch.int8)\n        outputs = self.bert(input_ids=X_inp, attention_mask=attn_mask)\n        return outputs[0][:, 0, :]\n\n    @property\n    def output_dim(self) -&gt; int:\n        # This is THE ONLY requirement for any model to work with pytorch-widedeep. Must\n        # have a 'output_dim' property so the WideDeep class knows the incoming dims\n        # from the custom model. in this case, I hardcoded it\n        return 768\n</pre> class BertModel(nn.Module):     def __init__(         self,         model_name: str = \"distilbert-base-uncased\",         freeze_bert: bool = False,     ):         super(BertModel, self).__init__()          self.bert = DistilBertModel.from_pretrained(             model_name,         )          if freeze_bert:             for param in self.bert.parameters():                 param.requires_grad = False      def forward(self, X_inp: Tensor) -&gt; Tensor:         attn_mask = (X_inp != 0).type(torch.int8)         outputs = self.bert(input_ids=X_inp, attention_mask=attn_mask)         return outputs[0][:, 0, :]      @property     def output_dim(self) -&gt; int:         # This is THE ONLY requirement for any model to work with pytorch-widedeep. Must         # have a 'output_dim' property so the WideDeep class knows the incoming dims         # from the custom model. in this case, I hardcoded it         return 768 In\u00a0[54]: Copied! <pre>bert_tokenizer = BertTokenizer()\nX_bert_tr = bert_tokenizer.fit_transform(train[\"review_text\"].tolist())\nX_bert_te = bert_tokenizer.transform(test[\"review_text\"].tolist())\n</pre> bert_tokenizer = BertTokenizer() X_bert_tr = bert_tokenizer.fit_transform(train[\"review_text\"].tolist()) X_bert_te = bert_tokenizer.transform(test[\"review_text\"].tolist()) <p>As I mentioned a number of times in the documentation and examples, <code>pytorch-widedeep</code> is designed for flexibility. For any of the data modes (tabular, text and images) there are available components/models in the library. However, the user can choose to use any model they want with the only requirement that such model must have a <code>output_dim</code> property.</p> <p>With that in mind, the <code>BertModel</code> class defined above can be used by <code>pytorch-widedeep</code> as any other of the internal components. In other words, simply...pass it to the <code>WideDeep</code> class. In this case we are going to add a FC-head as part of the classifier.</p> In\u00a0[55]: Copied! <pre>bert_model = BertModel(freeze_bert=True)\nwd_bert_model = WideDeep(\n    deeptext=bert_model,\n    head_hidden_dims=[256, 128, 64],\n    pred_dim=4,\n)\n</pre> bert_model = BertModel(freeze_bert=True) wd_bert_model = WideDeep(     deeptext=bert_model,     head_hidden_dims=[256, 128, 64],     pred_dim=4, ) <pre>Some weights of the model checkpoint at distilbert-base-uncased were not used when initializing DistilBertModel: ['vocab_projector.bias', 'vocab_layer_norm.bias', 'vocab_transform.weight', 'vocab_transform.bias', 'vocab_projector.weight', 'vocab_layer_norm.weight']\n- This IS expected if you are initializing DistilBertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n- This IS NOT expected if you are initializing DistilBertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n</pre> In\u00a0[56]: Copied! <pre>wd_bert_model\n</pre> wd_bert_model Out[56]: <pre>WideDeep(\n  (deeptext): BertModel(\n    (bert): DistilBertModel(\n      (embeddings): Embeddings(\n        (word_embeddings): Embedding(30522, 768, padding_idx=0)\n        (position_embeddings): Embedding(512, 768)\n        (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n        (dropout): Dropout(p=0.1, inplace=False)\n      )\n      (transformer): Transformer(\n        (layer): ModuleList(\n          (0): TransformerBlock(\n            (attention): MultiHeadSelfAttention(\n              (dropout): Dropout(p=0.1, inplace=False)\n              (q_lin): Linear(in_features=768, out_features=768, bias=True)\n              (k_lin): Linear(in_features=768, out_features=768, bias=True)\n              (v_lin): Linear(in_features=768, out_features=768, bias=True)\n              (out_lin): Linear(in_features=768, out_features=768, bias=True)\n            )\n            (sa_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n            (ffn): FFN(\n              (dropout): Dropout(p=0.1, inplace=False)\n              (lin1): Linear(in_features=768, out_features=3072, bias=True)\n              (lin2): Linear(in_features=3072, out_features=768, bias=True)\n              (activation): GELUActivation()\n            )\n            (output_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n          )\n          (1): TransformerBlock(\n            (attention): MultiHeadSelfAttention(\n              (dropout): Dropout(p=0.1, inplace=False)\n              (q_lin): Linear(in_features=768, out_features=768, bias=True)\n              (k_lin): Linear(in_features=768, out_features=768, bias=True)\n              (v_lin): Linear(in_features=768, out_features=768, bias=True)\n              (out_lin): Linear(in_features=768, out_features=768, bias=True)\n            )\n            (sa_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n            (ffn): FFN(\n              (dropout): Dropout(p=0.1, inplace=False)\n              (lin1): Linear(in_features=768, out_features=3072, bias=True)\n              (lin2): Linear(in_features=3072, out_features=768, bias=True)\n              (activation): GELUActivation()\n            )\n            (output_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n          )\n          (2): TransformerBlock(\n            (attention): MultiHeadSelfAttention(\n              (dropout): Dropout(p=0.1, inplace=False)\n              (q_lin): Linear(in_features=768, out_features=768, bias=True)\n              (k_lin): Linear(in_features=768, out_features=768, bias=True)\n              (v_lin): Linear(in_features=768, out_features=768, bias=True)\n              (out_lin): Linear(in_features=768, out_features=768, bias=True)\n            )\n            (sa_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n            (ffn): FFN(\n              (dropout): Dropout(p=0.1, inplace=False)\n              (lin1): Linear(in_features=768, out_features=3072, bias=True)\n              (lin2): Linear(in_features=3072, out_features=768, bias=True)\n              (activation): GELUActivation()\n            )\n            (output_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n          )\n          (3): TransformerBlock(\n            (attention): MultiHeadSelfAttention(\n              (dropout): Dropout(p=0.1, inplace=False)\n              (q_lin): Linear(in_features=768, out_features=768, bias=True)\n              (k_lin): Linear(in_features=768, out_features=768, bias=True)\n              (v_lin): Linear(in_features=768, out_features=768, bias=True)\n              (out_lin): Linear(in_features=768, out_features=768, bias=True)\n            )\n            (sa_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n            (ffn): FFN(\n              (dropout): Dropout(p=0.1, inplace=False)\n              (lin1): Linear(in_features=768, out_features=3072, bias=True)\n              (lin2): Linear(in_features=3072, out_features=768, bias=True)\n              (activation): GELUActivation()\n            )\n            (output_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n          )\n          (4): TransformerBlock(\n            (attention): MultiHeadSelfAttention(\n              (dropout): Dropout(p=0.1, inplace=False)\n              (q_lin): Linear(in_features=768, out_features=768, bias=True)\n              (k_lin): Linear(in_features=768, out_features=768, bias=True)\n              (v_lin): Linear(in_features=768, out_features=768, bias=True)\n              (out_lin): Linear(in_features=768, out_features=768, bias=True)\n            )\n            (sa_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n            (ffn): FFN(\n              (dropout): Dropout(p=0.1, inplace=False)\n              (lin1): Linear(in_features=768, out_features=3072, bias=True)\n              (lin2): Linear(in_features=3072, out_features=768, bias=True)\n              (activation): GELUActivation()\n            )\n            (output_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n          )\n          (5): TransformerBlock(\n            (attention): MultiHeadSelfAttention(\n              (dropout): Dropout(p=0.1, inplace=False)\n              (q_lin): Linear(in_features=768, out_features=768, bias=True)\n              (k_lin): Linear(in_features=768, out_features=768, bias=True)\n              (v_lin): Linear(in_features=768, out_features=768, bias=True)\n              (out_lin): Linear(in_features=768, out_features=768, bias=True)\n            )\n            (sa_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n            (ffn): FFN(\n              (dropout): Dropout(p=0.1, inplace=False)\n              (lin1): Linear(in_features=768, out_features=3072, bias=True)\n              (lin2): Linear(in_features=3072, out_features=768, bias=True)\n              (activation): GELUActivation()\n            )\n            (output_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n          )\n        )\n      )\n    )\n  )\n  (deephead): Sequential(\n    (0): MLP(\n      (mlp): Sequential(\n        (dense_layer_0): Sequential(\n          (0): Linear(in_features=768, out_features=256, bias=True)\n          (1): ReLU(inplace=True)\n          (2): Dropout(p=0.1, inplace=False)\n        )\n        (dense_layer_1): Sequential(\n          (0): Linear(in_features=256, out_features=128, bias=True)\n          (1): ReLU(inplace=True)\n          (2): Dropout(p=0.1, inplace=False)\n        )\n        (dense_layer_2): Sequential(\n          (0): Linear(in_features=128, out_features=64, bias=True)\n          (1): ReLU(inplace=True)\n          (2): Dropout(p=0.1, inplace=False)\n        )\n      )\n    )\n    (1): Linear(in_features=64, out_features=4, bias=True)\n  )\n)</pre> In\u00a0[57]: Copied! <pre>wd_bert_trainer = Trainer(\n    wd_bert_model,\n    objective=\"multiclass\",\n    metrics=[Accuracy, F1Score(average=True)],\n    num_workers=0,  # As in the case of the tokenizer, in notebook I need to set this to 0 for the Trainer to work\n)\n\nwd_bert_trainer.fit(\n    X_text=X_bert_tr,\n    target=train.rating.values,\n    n_epochs=3,\n    batch_size=64,\n)\n</pre> wd_bert_trainer = Trainer(     wd_bert_model,     objective=\"multiclass\",     metrics=[Accuracy, F1Score(average=True)],     num_workers=0,  # As in the case of the tokenizer, in notebook I need to set this to 0 for the Trainer to work )  wd_bert_trainer.fit(     X_text=X_bert_tr,     target=train.rating.values,     n_epochs=3,     batch_size=64, ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 283/283 [00:14&lt;00:00, 19.68it/s, loss=0.968, metrics={'acc': 0.5879, 'f1': 0.3591}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 283/283 [00:14&lt;00:00, 19.63it/s, loss=0.884, metrics={'acc': 0.6178, 'f1': 0.4399}]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 283/283 [00:14&lt;00:00, 19.55it/s, loss=0.87, metrics={'acc': 0.6234, 'f1': 0.4527}]\n</pre> In\u00a0[58]: Copied! <pre>wd_bert_pred_text = wd_bert_trainer.predict_proba(X_text=X_bert_te)\nwd_bert_pred_text_class = np.argmax(wd_bert_pred_text, 1)\n</pre> wd_bert_pred_text = wd_bert_trainer.predict_proba(X_text=X_bert_te) wd_bert_pred_text_class = np.argmax(wd_bert_pred_text, 1) <pre>predict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 71/71 [00:03&lt;00:00, 21.97it/s]\n</pre> In\u00a0[59]: Copied! <pre>wd_bert_acc = accuracy_score(test.rating, wd_bert_pred_text_class)\nwd_bert_f1 = f1_score(test.rating, wd_bert_pred_text_class, average=\"weighted\")\nwd_bert_cm = confusion_matrix(test.rating, wd_bert_pred_text_class)\n</pre> wd_bert_acc = accuracy_score(test.rating, wd_bert_pred_text_class) wd_bert_f1 = f1_score(test.rating, wd_bert_pred_text_class, average=\"weighted\") wd_bert_cm = confusion_matrix(test.rating, wd_bert_pred_text_class) In\u00a0[60]: Copied! <pre>print(f\"Distilbert Accuracy: {wd_bert_acc}. Distilbert F1 Score: {wd_bert_f1}\")\nprint(f\"Distilbert Confusion Matrix:\\n {wd_bert_cm}\")\n</pre> print(f\"Distilbert Accuracy: {wd_bert_acc}. Distilbert F1 Score: {wd_bert_f1}\") print(f\"Distilbert Confusion Matrix:\\n {wd_bert_cm}\") <pre>Distilbert Accuracy: 0.6326846528084918. Distilbert F1 Score: 0.5796652991272998\nDistilbert Confusion Matrix:\n [[ 287   75   22   90]\n [ 197  136   62  169]\n [  68  119  123  671]\n [  40   64   84 2315]]\n</pre> <p>Now, adding a tabular model follows the exact same process as the one described in section 2.</p> In\u00a0[61]: Copied! <pre>tab_model = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    mlp_hidden_dims=[100, 50],\n)\n</pre> tab_model = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     mlp_hidden_dims=[100, 50], ) In\u00a0[62]: Copied! <pre>wd_tab_bert_model = WideDeep(\n    deeptabular=tab_model,\n    deeptext=bert_model,\n    head_hidden_dims=[256, 128, 64],\n    pred_dim=4,\n)\n</pre> wd_tab_bert_model = WideDeep(     deeptabular=tab_model,     deeptext=bert_model,     head_hidden_dims=[256, 128, 64],     pred_dim=4, ) In\u00a0[63]: Copied! <pre>wd_tab_bert_trainer = Trainer(\n    wd_tab_bert_model,\n    objective=\"multiclass\",\n    metrics=[Accuracy, F1Score(average=True)],\n    num_workers=0,  # As in the case of the tokenizer, in notebook I need to set this to 0 for the Trainer to work\n)\n</pre> wd_tab_bert_trainer = Trainer(     wd_tab_bert_model,     objective=\"multiclass\",     metrics=[Accuracy, F1Score(average=True)],     num_workers=0,  # As in the case of the tokenizer, in notebook I need to set this to 0 for the Trainer to work ) In\u00a0[64]: Copied! <pre>wd_tab_bert_trainer.fit(\n    X_tab=wd_X_tab_tr,\n    X_text=X_bert_tr,\n    target=train.rating.values,\n    n_epochs=3,\n    batch_size=64,\n)\n</pre> wd_tab_bert_trainer.fit(     X_tab=wd_X_tab_tr,     X_text=X_bert_tr,     target=train.rating.values,     n_epochs=3,     batch_size=64, ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 283/283 [00:15&lt;00:00, 18.15it/s, loss=0.974, metrics={'acc': 0.5838, 'f1': 0.3404}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 283/283 [00:15&lt;00:00, 18.38it/s, loss=0.885, metrics={'acc': 0.618, 'f1': 0.4378}]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 283/283 [00:15&lt;00:00, 18.40it/s, loss=0.868, metrics={'acc': 0.6252, 'f1': 0.4575}]\n</pre> In\u00a0[65]: Copied! <pre>wd_tab_bert_pred_text = wd_tab_bert_trainer.predict_proba(\n    X_tab=wd_X_tab_te, X_text=X_bert_te\n)\nwd_tab_bert_pred_text_class = np.argmax(wd_tab_bert_pred_text, 1)\n</pre> wd_tab_bert_pred_text = wd_tab_bert_trainer.predict_proba(     X_tab=wd_X_tab_te, X_text=X_bert_te ) wd_tab_bert_pred_text_class = np.argmax(wd_tab_bert_pred_text, 1) <pre>predict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 71/71 [00:03&lt;00:00, 21.32it/s]\n</pre> In\u00a0[66]: Copied! <pre>wd_tab_bert_acc = accuracy_score(test.rating, wd_tab_bert_pred_text_class)\nwd_tab_bert_f1 = f1_score(test.rating, wd_tab_bert_pred_text_class, average=\"weighted\")\nwd_tab_bert_cm = confusion_matrix(test.rating, wd_tab_bert_pred_text_class)\n</pre> wd_tab_bert_acc = accuracy_score(test.rating, wd_tab_bert_pred_text_class) wd_tab_bert_f1 = f1_score(test.rating, wd_tab_bert_pred_text_class, average=\"weighted\") wd_tab_bert_cm = confusion_matrix(test.rating, wd_tab_bert_pred_text_class) In\u00a0[67]: Copied! <pre>print(\n    f\"Distilbert + Tabular Accuracy: {wd_tab_bert_acc}. Distilbert+ Tabular F1 Score: {wd_tab_bert_f1}\"\n)\nprint(f\"Distilbert + Tabular Confusion Matrix:\\n {wd_tab_bert_cm}\")\n</pre> print(     f\"Distilbert + Tabular Accuracy: {wd_tab_bert_acc}. Distilbert+ Tabular F1 Score: {wd_tab_bert_f1}\" ) print(f\"Distilbert + Tabular Confusion Matrix:\\n {wd_tab_bert_cm}\") <pre>Distilbert + Tabular Accuracy: 0.6242812914639541. Distilbert+ Tabular F1 Score: 0.5508351761564895\nDistilbert + Tabular Confusion Matrix:\n [[ 297   56   11  110]\n [ 229   91   38  206]\n [  86   90   71  734]\n [  49   48   42 2364]]\n</pre>"},{"location":"examples/16_Usign_a_custom_hugging_face_model.html#using-a-hugginface-model","title":"Using a Hugginface model\u00b6","text":"<p>In this notebook we will show how to use an \"external\" Hugginface model along with any other model in the libray. In particular we will show how to combine it with a tabular DL model.</p> <p>Since we are here, we will also compare the performance of a few models on a text classification problem.</p> <p>The notebook will go as follows:</p> <ol> <li>Text classification using tf-idf + LightGBM</li> <li>Text classification using a basic RNN</li> <li>Text classification using Distilbert</li> </ol> <p>In all 3 cases we will add some tabular features to see if these help.</p> <p>In general, I would not pay much attention to the results since I have placed no effort in getting the best possible results (i.e. no hyperparameter optimization or trying different architectures, for example).</p> <p>Let's go</p>"},{"location":"examples/16_Usign_a_custom_hugging_face_model.html#1-text-classification-using-tf-idf-lightgbm","title":"1. Text classification using tf-idf + LightGBM\u00b6","text":""},{"location":"examples/16_Usign_a_custom_hugging_face_model.html#2-text-classification-using-pytorch-widedeeps-built-in-models-a-basic-rnn","title":"2. Text classification using pytorch-widedeep's built-in models (a basic RNN)\u00b6","text":"<p>Moving on now to fully using <code>pytorch-widedeep</code> in this dataset, let's have a look on how one could use a simple RNN to predict the ratings with the library.</p>"},{"location":"examples/16_Usign_a_custom_hugging_face_model.html#3-text-classification-using-a-hugginface-model-as-a-custom-model-in-pytorch-widedeeps","title":"3. Text classification using a Hugginface model as a custom model in pytorch-widedeep's\u00b6","text":""},{"location":"examples/17_feature_importance_via_attention_weights.html","title":"17 feature importance via attention weights","text":"In\u00a0[1]: Copied! <pre>import torch\n\nimport numpy as np\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import accuracy_score\n\n\nfrom pytorch_widedeep import Trainer\nfrom pytorch_widedeep.models import TabTransformer, ContextAttentionMLP, WideDeep\nfrom pytorch_widedeep.callbacks import EarlyStopping\nfrom pytorch_widedeep.metrics import Accuracy\nfrom pytorch_widedeep.datasets import load_adult\nfrom pytorch_widedeep.preprocessing import TabPreprocessor\n</pre> import torch  import numpy as np from sklearn.model_selection import train_test_split from sklearn.metrics import accuracy_score   from pytorch_widedeep import Trainer from pytorch_widedeep.models import TabTransformer, ContextAttentionMLP, WideDeep from pytorch_widedeep.callbacks import EarlyStopping from pytorch_widedeep.metrics import Accuracy from pytorch_widedeep.datasets import load_adult from pytorch_widedeep.preprocessing import TabPreprocessor In\u00a0[2]: Copied! <pre># use_cuda = torch.cuda.is_available()\ndf = load_adult(as_frame=True)\ndf.columns = [c.replace(\"-\", \"_\") for c in df.columns]\ndf[\"income_label\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int)\ndf.drop([\"income\", \"fnlwgt\", \"educational_num\"], axis=1, inplace=True)\ntarget_colname = \"income_label\"\n</pre> # use_cuda = torch.cuda.is_available() df = load_adult(as_frame=True) df.columns = [c.replace(\"-\", \"_\") for c in df.columns] df[\"income_label\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int) df.drop([\"income\", \"fnlwgt\", \"educational_num\"], axis=1, inplace=True) target_colname = \"income_label\" In\u00a0[3]: Copied! <pre>df.head()\n</pre> df.head() Out[3]: age workclass education marital_status occupation relationship race gender capital_gain capital_loss hours_per_week native_country income_label 0 25 Private 11th Never-married Machine-op-inspct Own-child Black Male 0 0 40 United-States 0 1 38 Private HS-grad Married-civ-spouse Farming-fishing Husband White Male 0 0 50 United-States 0 2 28 Local-gov Assoc-acdm Married-civ-spouse Protective-serv Husband White Male 0 0 40 United-States 1 3 44 Private Some-college Married-civ-spouse Machine-op-inspct Husband Black Male 7688 0 40 United-States 1 4 18 ? Some-college Never-married ? Own-child White Female 0 0 30 United-States 0 In\u00a0[4]: Copied! <pre>cat_embed_cols = []\nfor col in df.columns:\n    if df[col].dtype == \"O\" or df[col].nunique() &lt; 200 and col != target_colname:\n        cat_embed_cols.append(col)\n</pre> cat_embed_cols = [] for col in df.columns:     if df[col].dtype == \"O\" or df[col].nunique() &lt; 200 and col != target_colname:         cat_embed_cols.append(col) In\u00a0[5]: Copied! <pre># all cols will be categorical\nassert len(cat_embed_cols) == df.shape[1] - 1\n</pre> # all cols will be categorical assert len(cat_embed_cols) == df.shape[1] - 1 In\u00a0[6]: Copied! <pre>train, test = train_test_split(\n    df, test_size=0.1, random_state=1, stratify=df[[target_colname]]\n)\n</pre> train, test = train_test_split(     df, test_size=0.1, random_state=1, stratify=df[[target_colname]] ) In\u00a0[7]: Copied! <pre>tab_preprocessor = TabPreprocessor(cat_embed_cols=cat_embed_cols, with_attention=True)\n</pre> tab_preprocessor = TabPreprocessor(cat_embed_cols=cat_embed_cols, with_attention=True) In\u00a0[8]: Copied! <pre>X_tab_train = tab_preprocessor.fit_transform(train)\nX_tab_test = tab_preprocessor.transform(test)\ntarget = train[target_colname].values\n</pre> X_tab_train = tab_preprocessor.fit_transform(train) X_tab_test = tab_preprocessor.transform(test) target = train[target_colname].values In\u00a0[9]: Copied! <pre>tab_transformer = TabTransformer(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    embed_continuous_method=\"standard\",\n    input_dim=8,\n    n_heads=2,\n    n_blocks=1,\n    attn_dropout=0.1,\n    transformer_activation=\"relu\",\n)\n</pre> tab_transformer = TabTransformer(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     embed_continuous_method=\"standard\",     input_dim=8,     n_heads=2,     n_blocks=1,     attn_dropout=0.1,     transformer_activation=\"relu\", ) In\u00a0[10]: Copied! <pre>model = WideDeep(deeptabular=tab_transformer)\n</pre> model = WideDeep(deeptabular=tab_transformer) In\u00a0[11]: Copied! <pre>optimizer = torch.optim.Adam(model.parameters(), lr=0.01, weight_decay=0.0)\n</pre> optimizer = torch.optim.Adam(model.parameters(), lr=0.01, weight_decay=0.0) In\u00a0[12]: Copied! <pre>lr_scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(\n    optimizer,\n    threshold=0.001,\n    threshold_mode=\"abs\",\n    patience=10,\n)\n</pre> lr_scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(     optimizer,     threshold=0.001,     threshold_mode=\"abs\",     patience=10, ) In\u00a0[13]: Copied! <pre>early_stopping = EarlyStopping(\n    min_delta=0.001, patience=30, restore_best_weights=True, verbose=True\n)\n</pre> early_stopping = EarlyStopping(     min_delta=0.001, patience=30, restore_best_weights=True, verbose=True ) In\u00a0[14]: Copied! <pre>trainer = Trainer(\n    model,\n    objective=\"binary\",\n    optimizers=optimizer,\n    lr_schedulers=lr_scheduler,\n    reducelronplateau_criterion=\"loss\",\n    callbacks=[early_stopping],\n    metrics=[Accuracy],\n)\n</pre> trainer = Trainer(     model,     objective=\"binary\",     optimizers=optimizer,     lr_schedulers=lr_scheduler,     reducelronplateau_criterion=\"loss\",     callbacks=[early_stopping],     metrics=[Accuracy], ) <p>The feature importances will be computed after training, using a sample of the training dataset of size <code>feature_importance_sample_size</code></p> In\u00a0[15]: Copied! <pre>trainer.fit(\n    X_tab=X_tab_train,\n    target=target,\n    val_split=0.2,\n    n_epochs=100,\n    batch_size=128,\n    validation_freq=1,\n    feature_importance_sample_size=1000,\n)\n</pre> trainer.fit(     X_tab=X_tab_train,     target=target,     val_split=0.2,     n_epochs=100,     batch_size=128,     validation_freq=1,     feature_importance_sample_size=1000, ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 81.80it/s, loss=0.334, metrics={'acc': 0.847}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 111.34it/s, loss=0.294, metrics={'acc': 0.8669}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 83.02it/s, loss=0.293, metrics={'acc': 0.8656}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 124.03it/s, loss=0.283, metrics={'acc': 0.8678}]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 87.69it/s, loss=0.282, metrics={'acc': 0.8703}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 118.22it/s, loss=0.279, metrics={'acc': 0.8717}]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 79.24it/s, loss=0.277, metrics={'acc': 0.8718}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 115.29it/s, loss=0.277, metrics={'acc': 0.8731}]\nepoch 5: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 83.76it/s, loss=0.275, metrics={'acc': 0.8727}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 120.80it/s, loss=0.276, metrics={'acc': 0.8727}]\nepoch 6: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 82.78it/s, loss=0.273, metrics={'acc': 0.873}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 100.43it/s, loss=0.276, metrics={'acc': 0.871}]\nepoch 7: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 76.35it/s, loss=0.271, metrics={'acc': 0.8742}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 116.14it/s, loss=0.275, metrics={'acc': 0.8726}]\nepoch 8: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 74.29it/s, loss=0.271, metrics={'acc': 0.875}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 117.46it/s, loss=0.276, metrics={'acc': 0.8718}]\nepoch 9: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 77.10it/s, loss=0.27, metrics={'acc': 0.8761}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 105.49it/s, loss=0.275, metrics={'acc': 0.8728}]\nepoch 10: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 70.40it/s, loss=0.269, metrics={'acc': 0.8747}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 105.47it/s, loss=0.275, metrics={'acc': 0.8726}]\nepoch 11: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 72.83it/s, loss=0.268, metrics={'acc': 0.8742}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 105.03it/s, loss=0.274, metrics={'acc': 0.873}]\nepoch 12: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 71.86it/s, loss=0.267, metrics={'acc': 0.8743}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 106.61it/s, loss=0.274, metrics={'acc': 0.8734}]\nepoch 13: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 72.39it/s, loss=0.267, metrics={'acc': 0.876}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 108.05it/s, loss=0.275, metrics={'acc': 0.8717}]\nepoch 14: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 73.36it/s, loss=0.265, metrics={'acc': 0.8767}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 109.60it/s, loss=0.276, metrics={'acc': 0.8747}]\nepoch 15: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 73.34it/s, loss=0.264, metrics={'acc': 0.876}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 117.55it/s, loss=0.276, metrics={'acc': 0.8706}]\nepoch 16: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 75.35it/s, loss=0.264, metrics={'acc': 0.8777}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 122.08it/s, loss=0.275, metrics={'acc': 0.8753}]\nepoch 17: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 75.96it/s, loss=0.263, metrics={'acc': 0.877}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 120.83it/s, loss=0.277, metrics={'acc': 0.8739}]\nepoch 18: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 75.38it/s, loss=0.263, metrics={'acc': 0.8779}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 118.18it/s, loss=0.278, metrics={'acc': 0.8714}]\nepoch 19: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 75.61it/s, loss=0.261, metrics={'acc': 0.8784}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 119.19it/s, loss=0.278, metrics={'acc': 0.8712}]\nepoch 20: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 72.43it/s, loss=0.261, metrics={'acc': 0.8791}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 118.87it/s, loss=0.28, metrics={'acc': 0.873}]\nepoch 21: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 73.97it/s, loss=0.26, metrics={'acc': 0.8787}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 107.50it/s, loss=0.279, metrics={'acc': 0.8732}]\nepoch 22: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 71.76it/s, loss=0.253, metrics={'acc': 0.8816}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 108.11it/s, loss=0.279, metrics={'acc': 0.8707}]\nepoch 23: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 71.92it/s, loss=0.252, metrics={'acc': 0.8828}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 100.14it/s, loss=0.28, metrics={'acc': 0.8711}]\nepoch 24: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 75.04it/s, loss=0.252, metrics={'acc': 0.8829}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 98.36it/s, loss=0.28, metrics={'acc': 0.8708}]\nepoch 25: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 76.31it/s, loss=0.251, metrics={'acc': 0.883}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 122.97it/s, loss=0.281, metrics={'acc': 0.8709}]\nepoch 26: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 76.63it/s, loss=0.25, metrics={'acc': 0.8834}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 123.07it/s, loss=0.281, metrics={'acc': 0.8698}]\nepoch 27: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 78.37it/s, loss=0.251, metrics={'acc': 0.884}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 114.75it/s, loss=0.281, metrics={'acc': 0.87}]\nepoch 28: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 77.19it/s, loss=0.25, metrics={'acc': 0.883}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 117.99it/s, loss=0.282, metrics={'acc': 0.8699}]\nepoch 29: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 76.19it/s, loss=0.25, metrics={'acc': 0.8829}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 111.11it/s, loss=0.282, metrics={'acc': 0.8695}]\nepoch 30: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 77.57it/s, loss=0.249, metrics={'acc': 0.8839}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 115.64it/s, loss=0.283, metrics={'acc': 0.8689}]\nepoch 31: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 79.55it/s, loss=0.249, metrics={'acc': 0.8846}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 125.10it/s, loss=0.283, metrics={'acc': 0.869}]\nepoch 32: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 82.56it/s, loss=0.248, metrics={'acc': 0.8841}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 118.45it/s, loss=0.284, metrics={'acc': 0.8687}]\nepoch 33: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 81.06it/s, loss=0.248, metrics={'acc': 0.8848}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 129.26it/s, loss=0.284, metrics={'acc': 0.8689}]\nepoch 34: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 80.53it/s, loss=0.248, metrics={'acc': 0.8854}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 120.61it/s, loss=0.283, metrics={'acc': 0.869}]\nepoch 35: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 80.78it/s, loss=0.248, metrics={'acc': 0.8853}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 127.31it/s, loss=0.283, metrics={'acc': 0.8694}]\nepoch 36: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 82.51it/s, loss=0.248, metrics={'acc': 0.8863}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 125.94it/s, loss=0.283, metrics={'acc': 0.8693}]\nepoch 37: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 81.35it/s, loss=0.247, metrics={'acc': 0.8844}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 126.77it/s, loss=0.283, metrics={'acc': 0.8692}]\nepoch 38: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 80.62it/s, loss=0.248, metrics={'acc': 0.8837}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 116.62it/s, loss=0.283, metrics={'acc': 0.8692}]\nepoch 39: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 79.82it/s, loss=0.248, metrics={'acc': 0.8842}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 123.64it/s, loss=0.283, metrics={'acc': 0.8695}]\nepoch 40: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 76.86it/s, loss=0.247, metrics={'acc': 0.8855}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 120.34it/s, loss=0.283, metrics={'acc': 0.8692}]\n</pre> <pre>Best Epoch: 10. Best val_loss: 0.27451\nRestoring model weights from the end of the best epoch\n</pre> In\u00a0[16]: Copied! <pre>trainer.feature_importance\n</pre> trainer.feature_importance Out[16]: <pre>{'age': 0.09718182,\n 'workclass': 0.090637445,\n 'education': 0.08910798,\n 'marital_status': 0.08971319,\n 'occupation': 0.12546304,\n 'relationship': 0.086381145,\n 'race': 0.050686445,\n 'gender': 0.05116429,\n 'capital_gain': 0.08165918,\n 'capital_loss': 0.07702667,\n 'hours_per_week': 0.08205996,\n 'native_country': 0.07891885}</pre> In\u00a0[17]: Copied! <pre>preds = trainer.predict(X_tab=X_tab_test)\n</pre> preds = trainer.predict(X_tab=X_tab_test) <pre>predict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 39/39 [00:00&lt;00:00, 199.63it/s]\n</pre> In\u00a0[18]: Copied! <pre>accuracy_score(preds, test.income_label)\n</pre> accuracy_score(preds, test.income_label) Out[18]: <pre>0.8685772773797339</pre> In\u00a0[19]: Copied! <pre>test.reset_index(drop=True, inplace=True)\n</pre> test.reset_index(drop=True, inplace=True) In\u00a0[20]: Copied! <pre>test[test.income_label == 0].head(1)\n</pre> test[test.income_label == 0].head(1) Out[20]: age workclass education marital_status occupation relationship race gender capital_gain capital_loss hours_per_week native_country income_label 0 26 Private Some-college Never-married Exec-managerial Not-in-family White Male 0 0 60 United-States 0 In\u00a0[21]: Copied! <pre>test[test.income_label == 1].head(1)\n</pre> test[test.income_label == 1].head(1) Out[21]: age workclass education marital_status occupation relationship race gender capital_gain capital_loss hours_per_week native_country income_label 3 36 Local-gov Doctorate Married-civ-spouse Prof-specialty Husband White Male 0 1887 50 United-States 1 <p>To get the feature importance of a test dataset, simply use the <code>explain</code> method</p> In\u00a0[22]: Copied! <pre>feat_imp_per_sample = trainer.explain(X_tab_test, save_step_masks=False)\n</pre> feat_imp_per_sample = trainer.explain(X_tab_test, save_step_masks=False) In\u00a0[23]: Copied! <pre>list(test.iloc[0].index[np.argsort(-feat_imp_per_sample[0])])\n</pre> list(test.iloc[0].index[np.argsort(-feat_imp_per_sample[0])]) Out[23]: <pre>['hours_per_week',\n 'education',\n 'relationship',\n 'occupation',\n 'workclass',\n 'capital_gain',\n 'native_country',\n 'marital_status',\n 'capital_loss',\n 'age',\n 'race',\n 'gender']</pre> In\u00a0[24]: Copied! <pre>list(test.iloc[3].index[np.argsort(-feat_imp_per_sample[3])])\n</pre> list(test.iloc[3].index[np.argsort(-feat_imp_per_sample[3])]) Out[24]: <pre>['age',\n 'capital_loss',\n 'hours_per_week',\n 'marital_status',\n 'native_country',\n 'relationship',\n 'race',\n 'education',\n 'occupation',\n 'capital_gain',\n 'gender',\n 'workclass']</pre> <p>We could do the same with the <code>ContextAttentionMLP</code></p> In\u00a0[25]: Copied! <pre>context_attn_mlp = ContextAttentionMLP(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    cat_embed_dropout=0.0,\n    input_dim=16,\n    attn_dropout=0.1,\n    attn_activation=\"relu\",\n)\n</pre> context_attn_mlp = ContextAttentionMLP(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     cat_embed_dropout=0.0,     input_dim=16,     attn_dropout=0.1,     attn_activation=\"relu\", ) In\u00a0[26]: Copied! <pre>mlp_model = WideDeep(deeptabular=context_attn_mlp)\n</pre> mlp_model = WideDeep(deeptabular=context_attn_mlp) In\u00a0[27]: Copied! <pre>mlp_optimizer = torch.optim.Adam(mlp_model.parameters(), lr=0.01, weight_decay=0.0)\n</pre> mlp_optimizer = torch.optim.Adam(mlp_model.parameters(), lr=0.01, weight_decay=0.0) In\u00a0[28]: Copied! <pre>mlp_lr_scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(\n    mlp_optimizer,\n    threshold=0.001,\n    threshold_mode=\"abs\",\n    patience=10,\n)\n</pre> mlp_lr_scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(     mlp_optimizer,     threshold=0.001,     threshold_mode=\"abs\",     patience=10, ) In\u00a0[29]: Copied! <pre>mlp_early_stopping = EarlyStopping(\n    min_delta=0.001, patience=30, restore_best_weights=True, verbose=True\n)\n</pre> mlp_early_stopping = EarlyStopping(     min_delta=0.001, patience=30, restore_best_weights=True, verbose=True ) In\u00a0[30]: Copied! <pre>mlp_trainer = Trainer(\n    mlp_model,\n    objective=\"binary\",\n    optimizers=mlp_optimizer,\n    lr_schedulers=mlp_lr_scheduler,\n    reducelronplateau_criterion=\"loss\",\n    callbacks=[mlp_early_stopping],\n    metrics=[Accuracy],\n)\n</pre> mlp_trainer = Trainer(     mlp_model,     objective=\"binary\",     optimizers=mlp_optimizer,     lr_schedulers=mlp_lr_scheduler,     reducelronplateau_criterion=\"loss\",     callbacks=[mlp_early_stopping],     metrics=[Accuracy], ) In\u00a0[31]: Copied! <pre>mlp_trainer.fit(\n    X_tab=X_tab_train,\n    target=target,\n    val_split=0.2,\n    n_epochs=100,\n    batch_size=128,\n    validation_freq=1,\n    feature_importance_sample_size=1000,\n)\n</pre> mlp_trainer.fit(     X_tab=X_tab_train,     target=target,     val_split=0.2,     n_epochs=100,     batch_size=128,     validation_freq=1,     feature_importance_sample_size=1000, ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 73.11it/s, loss=0.405, metrics={'acc': 0.8094}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 119.26it/s, loss=0.309, metrics={'acc': 0.8583}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 71.70it/s, loss=0.332, metrics={'acc': 0.8447}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 116.36it/s, loss=0.293, metrics={'acc': 0.8646}]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 78.42it/s, loss=0.319, metrics={'acc': 0.8505}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 118.05it/s, loss=0.293, metrics={'acc': 0.8654}]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 79.00it/s, loss=0.312, metrics={'acc': 0.8554}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 116.49it/s, loss=0.291, metrics={'acc': 0.8661}]\nepoch 5: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 79.50it/s, loss=0.308, metrics={'acc': 0.8583}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 120.48it/s, loss=0.287, metrics={'acc': 0.8669}]\nepoch 6: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 80.84it/s, loss=0.303, metrics={'acc': 0.8605}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 128.70it/s, loss=0.288, metrics={'acc': 0.8673}]\nepoch 7: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 79.93it/s, loss=0.301, metrics={'acc': 0.8597}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 121.47it/s, loss=0.298, metrics={'acc': 0.8628}]\nepoch 8: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 80.56it/s, loss=0.3, metrics={'acc': 0.8592}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 119.84it/s, loss=0.281, metrics={'acc': 0.8718}]\nepoch 9: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 79.17it/s, loss=0.298, metrics={'acc': 0.8619}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 126.32it/s, loss=0.28, metrics={'acc': 0.8716}]\nepoch 10: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 82.13it/s, loss=0.297, metrics={'acc': 0.8615}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 120.50it/s, loss=0.281, metrics={'acc': 0.8718}]\nepoch 11: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 82.54it/s, loss=0.293, metrics={'acc': 0.8641}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 122.57it/s, loss=0.284, metrics={'acc': 0.867}]\nepoch 12: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 80.92it/s, loss=0.293, metrics={'acc': 0.863}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 126.42it/s, loss=0.282, metrics={'acc': 0.8701}]\nepoch 13: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 79.61it/s, loss=0.293, metrics={'acc': 0.8635}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 117.56it/s, loss=0.276, metrics={'acc': 0.8719}]\nepoch 14: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 74.92it/s, loss=0.29, metrics={'acc': 0.8633}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 112.06it/s, loss=0.286, metrics={'acc': 0.8669}]\nepoch 15: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 76.83it/s, loss=0.291, metrics={'acc': 0.865}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 112.88it/s, loss=0.282, metrics={'acc': 0.8677}]\nepoch 16: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 76.74it/s, loss=0.29, metrics={'acc': 0.8653}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 120.85it/s, loss=0.285, metrics={'acc': 0.8672}]\nepoch 17: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 75.99it/s, loss=0.29, metrics={'acc': 0.865}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 113.53it/s, loss=0.282, metrics={'acc': 0.8681}]\nepoch 18: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 71.22it/s, loss=0.288, metrics={'acc': 0.8651}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 112.89it/s, loss=0.288, metrics={'acc': 0.8676}]\nepoch 19: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 75.85it/s, loss=0.29, metrics={'acc': 0.8661}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 114.26it/s, loss=0.284, metrics={'acc': 0.8662}]\nepoch 20: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 75.80it/s, loss=0.289, metrics={'acc': 0.8661}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 119.44it/s, loss=0.281, metrics={'acc': 0.8703}]\nepoch 21: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 75.72it/s, loss=0.29, metrics={'acc': 0.8661}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 113.04it/s, loss=0.285, metrics={'acc': 0.8648}]\nepoch 22: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 74.86it/s, loss=0.289, metrics={'acc': 0.8656}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 111.75it/s, loss=0.282, metrics={'acc': 0.8666}]\nepoch 23: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 76.68it/s, loss=0.289, metrics={'acc': 0.8668}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 115.86it/s, loss=0.282, metrics={'acc': 0.8724}]\nepoch 24: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 75.93it/s, loss=0.288, metrics={'acc': 0.8653}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 114.69it/s, loss=0.285, metrics={'acc': 0.8656}]\nepoch 25: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 79.99it/s, loss=0.284, metrics={'acc': 0.8671}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 118.27it/s, loss=0.277, metrics={'acc': 0.8707}]\nepoch 26: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 79.86it/s, loss=0.282, metrics={'acc': 0.8686}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 123.94it/s, loss=0.276, metrics={'acc': 0.8712}]\nepoch 27: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 77.86it/s, loss=0.283, metrics={'acc': 0.8691}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 114.11it/s, loss=0.277, metrics={'acc': 0.8716}]\nepoch 28: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 77.43it/s, loss=0.281, metrics={'acc': 0.8696}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 114.64it/s, loss=0.277, metrics={'acc': 0.8712}]\nepoch 29: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 77.39it/s, loss=0.281, metrics={'acc': 0.8696}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 115.83it/s, loss=0.277, metrics={'acc': 0.872}]\nepoch 30: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 74.93it/s, loss=0.28, metrics={'acc': 0.8706}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 113.07it/s, loss=0.275, metrics={'acc': 0.8714}]\nepoch 31: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 77.35it/s, loss=0.281, metrics={'acc': 0.8697}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 112.68it/s, loss=0.276, metrics={'acc': 0.872}]\nepoch 32: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 76.72it/s, loss=0.28, metrics={'acc': 0.8693}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 122.50it/s, loss=0.276, metrics={'acc': 0.8709}]\nepoch 33: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 76.68it/s, loss=0.28, metrics={'acc': 0.8716}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 110.07it/s, loss=0.277, metrics={'acc': 0.8709}]\nepoch 34: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 73.58it/s, loss=0.279, metrics={'acc': 0.8704}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 111.07it/s, loss=0.274, metrics={'acc': 0.8719}]\nepoch 35: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 74.03it/s, loss=0.28, metrics={'acc': 0.8687}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 110.15it/s, loss=0.276, metrics={'acc': 0.871}]\nepoch 36: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 73.11it/s, loss=0.279, metrics={'acc': 0.8706}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 111.10it/s, loss=0.278, metrics={'acc': 0.8705}]\nepoch 37: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 74.30it/s, loss=0.279, metrics={'acc': 0.869}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 110.75it/s, loss=0.279, metrics={'acc': 0.8702}]\nepoch 38: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 72.34it/s, loss=0.28, metrics={'acc': 0.8691}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 117.79it/s, loss=0.277, metrics={'acc': 0.8698}]\nepoch 39: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 75.49it/s, loss=0.279, metrics={'acc': 0.8694}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 115.25it/s, loss=0.279, metrics={'acc': 0.87}]\nepoch 40: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 75.51it/s, loss=0.28, metrics={'acc': 0.8694}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 110.90it/s, loss=0.277, metrics={'acc': 0.8694}]\nepoch 41: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 75.35it/s, loss=0.278, metrics={'acc': 0.8716}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 115.74it/s, loss=0.28, metrics={'acc': 0.8675}]\nepoch 42: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 76.35it/s, loss=0.279, metrics={'acc': 0.8695}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 122.76it/s, loss=0.277, metrics={'acc': 0.8699}]\nepoch 43: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:04&lt;00:00, 66.14it/s, loss=0.279, metrics={'acc': 0.8681}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 106.20it/s, loss=0.277, metrics={'acc': 0.8714}]\nepoch 44: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 73.37it/s, loss=0.279, metrics={'acc': 0.8704}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 115.18it/s, loss=0.277, metrics={'acc': 0.8716}]\nepoch 45: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 76.23it/s, loss=0.278, metrics={'acc': 0.8702}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 124.83it/s, loss=0.278, metrics={'acc': 0.8707}]\nepoch 46: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 77.12it/s, loss=0.278, metrics={'acc': 0.8704}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 126.62it/s, loss=0.279, metrics={'acc': 0.8693}]\nepoch 47: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 78.55it/s, loss=0.276, metrics={'acc': 0.8713}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 117.99it/s, loss=0.279, metrics={'acc': 0.8691}]\nepoch 48: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 77.25it/s, loss=0.278, metrics={'acc': 0.8719}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 124.52it/s, loss=0.278, metrics={'acc': 0.8695}]\nepoch 49: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 78.35it/s, loss=0.277, metrics={'acc': 0.8721}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 119.82it/s, loss=0.279, metrics={'acc': 0.8691}]\nepoch 50: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 77.15it/s, loss=0.277, metrics={'acc': 0.8717}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 122.62it/s, loss=0.278, metrics={'acc': 0.8699}]\nepoch 51: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 78.55it/s, loss=0.277, metrics={'acc': 0.8713}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 117.63it/s, loss=0.278, metrics={'acc': 0.87}]\nepoch 52: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 77.04it/s, loss=0.276, metrics={'acc': 0.8721}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 126.39it/s, loss=0.278, metrics={'acc': 0.8697}]\nepoch 53: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 78.15it/s, loss=0.277, metrics={'acc': 0.8721}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 127.56it/s, loss=0.278, metrics={'acc': 0.8699}]\nepoch 54: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 77.41it/s, loss=0.277, metrics={'acc': 0.8711}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 118.95it/s, loss=0.278, metrics={'acc': 0.8698}]\nepoch 55: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 76.35it/s, loss=0.277, metrics={'acc': 0.8718}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 126.90it/s, loss=0.278, metrics={'acc': 0.8699}]\nepoch 56: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 77.83it/s, loss=0.277, metrics={'acc': 0.8707}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 115.13it/s, loss=0.279, metrics={'acc': 0.8691}]\nepoch 57: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 75.20it/s, loss=0.277, metrics={'acc': 0.8722}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 123.16it/s, loss=0.279, metrics={'acc': 0.8691}]\nepoch 58: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 73.33it/s, loss=0.276, metrics={'acc': 0.871}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 123.37it/s, loss=0.278, metrics={'acc': 0.8691}]\nepoch 59: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 78.41it/s, loss=0.277, metrics={'acc': 0.8714}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 125.17it/s, loss=0.278, metrics={'acc': 0.8695}]\nepoch 60: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 79.26it/s, loss=0.276, metrics={'acc': 0.8721}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 120.60it/s, loss=0.278, metrics={'acc': 0.869}]\nepoch 61: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 75.88it/s, loss=0.278, metrics={'acc': 0.8703}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 124.47it/s, loss=0.278, metrics={'acc': 0.8692}]\nepoch 62: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 77.13it/s, loss=0.276, metrics={'acc': 0.8711}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 125.80it/s, loss=0.278, metrics={'acc': 0.8691}]\nepoch 63: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 77.20it/s, loss=0.277, metrics={'acc': 0.8715}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 118.50it/s, loss=0.278, metrics={'acc': 0.8695}]\nepoch 64: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 78.11it/s, loss=0.276, metrics={'acc': 0.8719}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 114.52it/s, loss=0.278, metrics={'acc': 0.869}]\n</pre> <pre>Best Epoch: 34. Best val_loss: 0.27449\nRestoring model weights from the end of the best epoch\n</pre> In\u00a0[32]: Copied! <pre>mlp_trainer.feature_importance\n</pre> mlp_trainer.feature_importance Out[32]: <pre>{'age': 0.116632804,\n 'workclass': 0.050255153,\n 'education': 0.094621316,\n 'marital_status': 0.12328919,\n 'occupation': 0.107893184,\n 'relationship': 0.11747801,\n 'race': 0.054717205,\n 'gender': 0.07514235,\n 'capital_gain': 0.059732802,\n 'capital_loss': 0.06738944,\n 'hours_per_week': 0.0610674,\n 'native_country': 0.07178114}</pre> In\u00a0[33]: Copied! <pre>mlp_preds = mlp_trainer.predict(X_tab=X_tab_test)\n</pre> mlp_preds = mlp_trainer.predict(X_tab=X_tab_test) <pre>predict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 39/39 [00:00&lt;00:00, 212.38it/s]\n</pre> In\u00a0[34]: Copied! <pre>accuracy_score(mlp_preds, test.income_label)\n</pre> accuracy_score(mlp_preds, test.income_label) Out[34]: <pre>0.8726714431934494</pre>"},{"location":"examples/17_feature_importance_via_attention_weights.html#feature-importance-via-the-attention-weights","title":"Feature Importance via the attention weights\u00b6","text":"<p>I will start by saying that I consider this feature of the library purely experimental. First of all I think there are multiple ways one could address finding the features importances for these models. However, and more importantly, one has to bear in mind that even tree-based algorithms on the same dataset produce different feature importances. This is more \"dramatic\" if one uses different techniques, such as shap or feature permutation (see for example this and references therein). All this to say that, sometimes, feature importance is just a measure contained within the experiment run, and for the model used.</p> <p>With that in mind, each instantiation of a deep tabular model, that has millions of trainable parameters, will potentially produce a different set of feature importances, even if the model has the same architecture. Moreover, this effect will become more apparent if the dataset is relatively easy and there are dependent/related columns so that one could get to the same success metric with different parameters.</p> <p>In summary, feature importances are implemented in this librray for all attention-based models for tabular data, with the exception of the <code>TabPerceiver</code>. However this functionality has to be used and interpreted with care and consider of value within the 'universe' (or context) of the model with which these features were produced.</p> <p>Nonetheless, let's have a look to how one would access to the feature importances when using this library.</p>"},{"location":"examples/18_wide_and_deep_for_recsys_pt1.html","title":"18 wide and deep for recsys pt1","text":"<p>The goal of this, and the companion (part 2) notebooks is to illustrate how one could use this library in the context of recommendation systems. In particular, this notebook and the scripts at the <code>wide_deep_for_recsys</code> dir are a response to this issue. Therefore, we will use the Kaggle notebook referred in that issue here.</p> <p>In order to keep the length of the notebook tractable, we will split this exercise in 2. In this first notebook we will prepare the data in almost the exact same way as it is done in the Kaggle notebook and also show how one could use <code>pytorch-widedeep</code> to build a model almost identical to the one in that notebook.</p> <p>In a second notebook, we will show how one could use this library to implement other models, still following the same problem formulation.</p> In\u00a0[1]: Copied! <pre>from pathlib import Path\nimport warnings\n\nimport pandas as pd\nfrom sklearn.model_selection import train_test_split\n\nfrom pytorch_widedeep.datasets import load_movielens100k\n</pre> from pathlib import Path import warnings  import pandas as pd from sklearn.model_selection import train_test_split  from pytorch_widedeep.datasets import load_movielens100k In\u00a0[2]: Copied! <pre>warnings.filterwarnings(\"ignore\")\n</pre> warnings.filterwarnings(\"ignore\") In\u00a0[3]: Copied! <pre>save_path = Path(\"prepared_data\")\nif not save_path.exists():\n    save_path.mkdir(parents=True, exist_ok=True)\n</pre> save_path = Path(\"prepared_data\") if not save_path.exists():     save_path.mkdir(parents=True, exist_ok=True) In\u00a0[4]: Copied! <pre>data, users, items = load_movielens100k(as_frame=True)\n</pre> data, users, items = load_movielens100k(as_frame=True) In\u00a0[5]: Copied! <pre># Alternatively, as specified in the docs: 'The last 19 fields are the genres' so:\n# list_of_genres = items.columns.tolist()[-19:]\nlist_of_genres = [\n    \"unknown\",\n    \"Action\",\n    \"Adventure\",\n    \"Animation\",\n    \"Children's\",\n    \"Comedy\",\n    \"Crime\",\n    \"Documentary\",\n    \"Drama\",\n    \"Fantasy\",\n    \"Film-Noir\",\n    \"Horror\",\n    \"Musical\",\n    \"Mystery\",\n    \"Romance\",\n    \"Sci-Fi\",\n    \"Thriller\",\n    \"War\",\n    \"Western\",\n]\n</pre> # Alternatively, as specified in the docs: 'The last 19 fields are the genres' so: # list_of_genres = items.columns.tolist()[-19:] list_of_genres = [     \"unknown\",     \"Action\",     \"Adventure\",     \"Animation\",     \"Children's\",     \"Comedy\",     \"Crime\",     \"Documentary\",     \"Drama\",     \"Fantasy\",     \"Film-Noir\",     \"Horror\",     \"Musical\",     \"Mystery\",     \"Romance\",     \"Sci-Fi\",     \"Thriller\",     \"War\",     \"Western\", ] <p>Let's first start by loading the interactions, user and item data</p> In\u00a0[6]: Copied! <pre>data.head()\n</pre> data.head() Out[6]: user_id movie_id rating timestamp 0 196 242 3 881250949 1 186 302 3 891717742 2 22 377 1 878887116 3 244 51 2 880606923 4 166 346 1 886397596 In\u00a0[7]: Copied! <pre>users.head()\n</pre> users.head() Out[7]: user_id age gender occupation zip_code 0 1 24 M technician 85711 1 2 53 F other 94043 2 3 23 M writer 32067 3 4 24 M technician 43537 4 5 33 F other 15213 In\u00a0[8]: Copied! <pre>items.head()\n</pre> items.head() Out[8]: movie_id movie_title release_date video_release_date IMDb_URL unknown Action Adventure Animation Children's ... Fantasy Film-Noir Horror Musical Mystery Romance Sci-Fi Thriller War Western 0 1 Toy Story (1995) 01-Jan-1995 NaN http://us.imdb.com/M/title-exact?Toy%20Story%2... 0 0 0 1 1 ... 0 0 0 0 0 0 0 0 0 0 1 2 GoldenEye (1995) 01-Jan-1995 NaN http://us.imdb.com/M/title-exact?GoldenEye%20(... 0 1 1 0 0 ... 0 0 0 0 0 0 0 1 0 0 2 3 Four Rooms (1995) 01-Jan-1995 NaN http://us.imdb.com/M/title-exact?Four%20Rooms%... 0 0 0 0 0 ... 0 0 0 0 0 0 0 1 0 0 3 4 Get Shorty (1995) 01-Jan-1995 NaN http://us.imdb.com/M/title-exact?Get%20Shorty%... 0 1 0 0 0 ... 0 0 0 0 0 0 0 0 0 0 4 5 Copycat (1995) 01-Jan-1995 NaN http://us.imdb.com/M/title-exact?Copycat%20(1995) 0 0 0 0 0 ... 0 0 0 0 0 0 0 1 0 0 <p>5 rows \u00d7 24 columns</p> In\u00a0[9]: Copied! <pre># adding a column with the number of movies watched per user\ndataset = data.sort_values([\"user_id\", \"timestamp\"]).reset_index(drop=True)\ndataset[\"one\"] = 1\ndataset[\"num_watched\"] = dataset.groupby(\"user_id\")[\"one\"].cumsum()\ndataset.drop(\"one\", axis=1, inplace=True)\ndataset.head()\n</pre> # adding a column with the number of movies watched per user dataset = data.sort_values([\"user_id\", \"timestamp\"]).reset_index(drop=True) dataset[\"one\"] = 1 dataset[\"num_watched\"] = dataset.groupby(\"user_id\")[\"one\"].cumsum() dataset.drop(\"one\", axis=1, inplace=True) dataset.head() Out[9]: user_id movie_id rating timestamp num_watched 0 1 168 5 874965478 1 1 1 172 5 874965478 2 2 1 165 5 874965518 3 3 1 156 4 874965556 4 4 1 196 5 874965677 5 In\u00a0[10]: Copied! <pre># adding a column with the mean rating at a point in time per user\ndataset[\"mean_rate\"] = (\n    dataset.groupby(\"user_id\")[\"rating\"].cumsum() / dataset[\"num_watched\"]\n)\ndataset.head()\n</pre> # adding a column with the mean rating at a point in time per user dataset[\"mean_rate\"] = (     dataset.groupby(\"user_id\")[\"rating\"].cumsum() / dataset[\"num_watched\"] ) dataset.head() Out[10]: user_id movie_id rating timestamp num_watched mean_rate 0 1 168 5 874965478 1 5.00 1 1 172 5 874965478 2 5.00 2 1 165 5 874965518 3 5.00 3 1 156 4 874965556 4 4.75 4 1 196 5 874965677 5 4.80 In\u00a0[11]: Copied! <pre>dataset[\"target\"] = dataset.groupby(\"user_id\")[\"movie_id\"].shift(-1)\n</pre> dataset[\"target\"] = dataset.groupby(\"user_id\")[\"movie_id\"].shift(-1) <p>Following the same processing used by the author in the before-mentioned Kaggle notebook, we build sequences of previous movies watched</p> In\u00a0[12]: Copied! <pre># Here the author builds the sequences\ndataset[\"prev_movies\"] = dataset[\"movie_id\"].apply(lambda x: str(x))\ndataset[\"prev_movies\"] = (\n    dataset.groupby(\"user_id\")[\"prev_movies\"]\n    .apply(lambda x: (x + \" \").cumsum().str.strip())\n    .reset_index(drop=True)\n)\ndataset[\"prev_movies\"] = dataset[\"prev_movies\"].apply(lambda x: x.split())\ndataset.head()\n</pre> # Here the author builds the sequences dataset[\"prev_movies\"] = dataset[\"movie_id\"].apply(lambda x: str(x)) dataset[\"prev_movies\"] = (     dataset.groupby(\"user_id\")[\"prev_movies\"]     .apply(lambda x: (x + \" \").cumsum().str.strip())     .reset_index(drop=True) ) dataset[\"prev_movies\"] = dataset[\"prev_movies\"].apply(lambda x: x.split()) dataset.head() Out[12]: user_id movie_id rating timestamp num_watched mean_rate target prev_movies 0 1 168 5 874965478 1 5.00 172.0 [168] 1 1 172 5 874965478 2 5.00 165.0 [168, 172] 2 1 165 5 874965518 3 5.00 156.0 [168, 172, 165] 3 1 156 4 874965556 4 4.75 196.0 [168, 172, 165, 156] 4 1 196 5 874965677 5 4.80 166.0 [168, 172, 165, 156, 196] <p>And now we add a <code>genre_rate</code> as the mean of all movies rated for a given genre per user</p> In\u00a0[13]: Copied! <pre>dataset = dataset.merge(items[[\"movie_id\"] + list_of_genres], on=\"movie_id\", how=\"left\")\nfor genre in list_of_genres:\n    dataset[f\"{genre}_rate\"] = dataset[genre] * dataset[\"rating\"]\n    dataset[genre] = dataset.groupby(\"user_id\")[genre].cumsum()\n    dataset[f\"{genre}_rate\"] = (\n        dataset.groupby(\"user_id\")[f\"{genre}_rate\"].cumsum() / dataset[genre]\n    )\ndataset[list_of_genres] = dataset[list_of_genres].apply(\n    lambda x: x / dataset[\"num_watched\"]\n)\ndataset.head()\n</pre> dataset = dataset.merge(items[[\"movie_id\"] + list_of_genres], on=\"movie_id\", how=\"left\") for genre in list_of_genres:     dataset[f\"{genre}_rate\"] = dataset[genre] * dataset[\"rating\"]     dataset[genre] = dataset.groupby(\"user_id\")[genre].cumsum()     dataset[f\"{genre}_rate\"] = (         dataset.groupby(\"user_id\")[f\"{genre}_rate\"].cumsum() / dataset[genre]     ) dataset[list_of_genres] = dataset[list_of_genres].apply(     lambda x: x / dataset[\"num_watched\"] ) dataset.head() Out[13]: user_id movie_id rating timestamp num_watched mean_rate target prev_movies unknown Action ... Fantasy_rate Film-Noir_rate Horror_rate Musical_rate Mystery_rate Romance_rate Sci-Fi_rate Thriller_rate War_rate Western_rate 0 1 168 5 874965478 1 5.00 172.0 [168] 0.0 0.000000 ... NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN 1 1 172 5 874965478 2 5.00 165.0 [168, 172] 0.0 0.500000 ... NaN NaN NaN NaN NaN 5.0 5.0 NaN 5.0 NaN 2 1 165 5 874965518 3 5.00 156.0 [168, 172, 165] 0.0 0.333333 ... NaN NaN NaN NaN NaN 5.0 5.0 NaN 5.0 NaN 3 1 156 4 874965556 4 4.75 196.0 [168, 172, 165, 156] 0.0 0.250000 ... NaN NaN NaN NaN NaN 5.0 5.0 4.0 5.0 NaN 4 1 196 5 874965677 5 4.80 166.0 [168, 172, 165, 156, 196] 0.0 0.200000 ... NaN NaN NaN NaN NaN 5.0 5.0 4.0 5.0 NaN <p>5 rows \u00d7 46 columns</p> <p>Adding user features</p> In\u00a0[14]: Copied! <pre>dataset = dataset.merge(users, on=\"user_id\", how=\"left\")\ndataset.head()\n</pre> dataset = dataset.merge(users, on=\"user_id\", how=\"left\") dataset.head() Out[14]: user_id movie_id rating timestamp num_watched mean_rate target prev_movies unknown Action ... Mystery_rate Romance_rate Sci-Fi_rate Thriller_rate War_rate Western_rate age gender occupation zip_code 0 1 168 5 874965478 1 5.00 172.0 [168] 0.0 0.000000 ... NaN NaN NaN NaN NaN NaN 24 M technician 85711 1 1 172 5 874965478 2 5.00 165.0 [168, 172] 0.0 0.500000 ... NaN 5.0 5.0 NaN 5.0 NaN 24 M technician 85711 2 1 165 5 874965518 3 5.00 156.0 [168, 172, 165] 0.0 0.333333 ... NaN 5.0 5.0 NaN 5.0 NaN 24 M technician 85711 3 1 156 4 874965556 4 4.75 196.0 [168, 172, 165, 156] 0.0 0.250000 ... NaN 5.0 5.0 4.0 5.0 NaN 24 M technician 85711 4 1 196 5 874965677 5 4.80 166.0 [168, 172, 165, 156, 196] 0.0 0.200000 ... NaN 5.0 5.0 4.0 5.0 NaN 24 M technician 85711 <p>5 rows \u00d7 50 columns</p> <p>Again, we use the same settings as those in the Kaggle notebook, but <code>COLD_START_TRESH</code> is pretty aggressive</p> In\u00a0[15]: Copied! <pre>COLD_START_TRESH = 5\n\nfiltred_data = dataset[\n    (dataset[\"num_watched\"] &gt;= COLD_START_TRESH) &amp; ~(dataset[\"target\"].isna())\n].sort_values(\"timestamp\")\ntrain_data, _test_data = train_test_split(filtred_data, test_size=0.2, shuffle=False)\nvalid_data, test_data = train_test_split(_test_data, test_size=0.5, shuffle=False)\n</pre> COLD_START_TRESH = 5  filtred_data = dataset[     (dataset[\"num_watched\"] &gt;= COLD_START_TRESH) &amp; ~(dataset[\"target\"].isna()) ].sort_values(\"timestamp\") train_data, _test_data = train_test_split(filtred_data, test_size=0.2, shuffle=False) valid_data, test_data = train_test_split(_test_data, test_size=0.5, shuffle=False) In\u00a0[16]: Copied! <pre>cols_to_drop = [\n    # \"rating\",\n    \"timestamp\",\n    \"num_watched\",\n]\n\ndf_train = train_data.drop(cols_to_drop, axis=1)\ndf_valid = valid_data.drop(cols_to_drop, axis=1)\ndf_test = test_data.drop(cols_to_drop, axis=1)\n\ndf_train.to_pickle(save_path / \"df_train.pkl\")\ndf_valid.to_pickle(save_path / \"df_valid.pkl\")\ndf_test.to_pickle(save_path / \"df_test.pkl\")\n</pre> cols_to_drop = [     # \"rating\",     \"timestamp\",     \"num_watched\", ]  df_train = train_data.drop(cols_to_drop, axis=1) df_valid = valid_data.drop(cols_to_drop, axis=1) df_test = test_data.drop(cols_to_drop, axis=1)  df_train.to_pickle(save_path / \"df_train.pkl\") df_valid.to_pickle(save_path / \"df_valid.pkl\") df_test.to_pickle(save_path / \"df_test.pkl\") <p>Let's now build a model that is nearly identical to the one use in the Kaggle notebook</p> In\u00a0[17]: Copied! <pre>import numpy as np\nimport torch\nfrom torch import nn\nfrom scipy.sparse import coo_matrix\n\nfrom pytorch_widedeep import Trainer\nfrom pytorch_widedeep.models import TabMlp, BasicRNN, WideDeep\nfrom pytorch_widedeep.preprocessing import TabPreprocessor\n</pre> import numpy as np import torch from torch import nn from scipy.sparse import coo_matrix  from pytorch_widedeep import Trainer from pytorch_widedeep.models import TabMlp, BasicRNN, WideDeep from pytorch_widedeep.preprocessing import TabPreprocessor In\u00a0[18]: Copied! <pre>device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n\nsave_path = Path(\"prepared_data\")\n\nPAD_IDX = 0\n</pre> device = \"cuda\" if torch.cuda.is_available() else \"cpu\"  save_path = Path(\"prepared_data\")  PAD_IDX = 0 <p>Let's use some of the functions the author of the kaggle's notebook uses to prepare the data</p> In\u00a0[19]: Copied! <pre>def get_coo_indexes(lil):\n    rows = []\n    cols = []\n    for i, el in enumerate(lil):\n        if type(el) != list:\n            el = [el]\n        for j in el:\n            rows.append(i)\n            cols.append(j)\n    return rows, cols\n\n\ndef get_sparse_features(series, shape):\n    coo_indexes = get_coo_indexes(series.tolist())\n    sparse_df = coo_matrix(\n        (np.ones(len(coo_indexes[0])), (coo_indexes[0], coo_indexes[1])), shape=shape\n    )\n    return sparse_df\n\n\ndef sparse_to_idx(data, pad_idx=-1):\n    indexes = data.nonzero()\n    indexes_df = pd.DataFrame()\n    indexes_df[\"rows\"] = indexes[0]\n    indexes_df[\"cols\"] = indexes[1]\n    mdf = indexes_df.groupby(\"rows\").apply(lambda x: x[\"cols\"].tolist())\n    max_len = mdf.apply(lambda x: len(x)).max()\n    return mdf.apply(lambda x: pd.Series(x + [pad_idx] * (max_len - len(x)))).values\n</pre> def get_coo_indexes(lil):     rows = []     cols = []     for i, el in enumerate(lil):         if type(el) != list:             el = [el]         for j in el:             rows.append(i)             cols.append(j)     return rows, cols   def get_sparse_features(series, shape):     coo_indexes = get_coo_indexes(series.tolist())     sparse_df = coo_matrix(         (np.ones(len(coo_indexes[0])), (coo_indexes[0], coo_indexes[1])), shape=shape     )     return sparse_df   def sparse_to_idx(data, pad_idx=-1):     indexes = data.nonzero()     indexes_df = pd.DataFrame()     indexes_df[\"rows\"] = indexes[0]     indexes_df[\"cols\"] = indexes[1]     mdf = indexes_df.groupby(\"rows\").apply(lambda x: x[\"cols\"].tolist())     max_len = mdf.apply(lambda x: len(x)).max()     return mdf.apply(lambda x: pd.Series(x + [pad_idx] * (max_len - len(x)))).values <p>For the time being, we will not use a validation set for hyperparameter optimization, and we will simply concatenate the validation and the test set in one test set. I simply splitted the data into train/valid/test in case the reader wants to actually do hyperparameter optimization (and because I know in the future I will).</p> <p>There is also another caveat worth mentioning, related to the indexing of the movies. To build the matrices of movies watched, we use the entire dataset. A more realistic (and correct) approach would be to use ONLY the movies that appear in the training set and consider <code>unknown</code> or <code>unseen</code> those in the testing set that have not been seen during training. Nonetheless, this will not affect the purposes of this notebook, which is to illustrate how one could use <code>pytorch-widedeep</code> to build a recommendation algorithm. However, if one wanted to explore the performance of different algorithms in a \"proper\" way, these \"details\" need to be accounted for.</p> In\u00a0[20]: Copied! <pre>df_test = pd.concat([df_valid, df_test], ignore_index=True)\n</pre> df_test = pd.concat([df_valid, df_test], ignore_index=True) In\u00a0[21]: Copied! <pre>id_cols = [\"user_id\", \"movie_id\"]\nmax_movie_index = max(df_train.movie_id.max(), df_test.movie_id.max())\n</pre> id_cols = [\"user_id\", \"movie_id\"] max_movie_index = max(df_train.movie_id.max(), df_test.movie_id.max()) In\u00a0[22]: Copied! <pre>X_train = df_train.drop(id_cols + [\"rating\", \"prev_movies\", \"target\"], axis=1)\ny_train = np.array(df_train.target.values, dtype=\"int64\")\ntrain_movies_watched = get_sparse_features(\n    df_train[\"prev_movies\"], (len(df_train), max_movie_index + 1)\n)\n\nX_test = df_test.drop(id_cols + [\"rating\", \"prev_movies\", \"target\"], axis=1)\ny_test = np.array(df_test.target.values, dtype=\"int64\")\ntest_movies_watched = get_sparse_features(\n    df_test[\"prev_movies\"], (len(df_test), max_movie_index + 1)\n)\n</pre> X_train = df_train.drop(id_cols + [\"rating\", \"prev_movies\", \"target\"], axis=1) y_train = np.array(df_train.target.values, dtype=\"int64\") train_movies_watched = get_sparse_features(     df_train[\"prev_movies\"], (len(df_train), max_movie_index + 1) )  X_test = df_test.drop(id_cols + [\"rating\", \"prev_movies\", \"target\"], axis=1) y_test = np.array(df_test.target.values, dtype=\"int64\") test_movies_watched = get_sparse_features(     df_test[\"prev_movies\"], (len(df_test), max_movie_index + 1) ) <p>let's have a look to the information in each dataset</p> In\u00a0[23]: Copied! <pre>X_train.head()\n</pre> X_train.head() Out[23]: mean_rate unknown Action Adventure Animation Children's Comedy Crime Documentary Drama ... Mystery_rate Romance_rate Sci-Fi_rate Thriller_rate War_rate Western_rate age gender occupation zip_code 25423 4.000000 0.0 0.400000 0.200000 0.0 0.0 0.400000 0.0 0.0 0.200000 ... NaN 4.0 4.0 4.000000 4.0 NaN 21 M student 48823 25425 4.000000 0.0 0.285714 0.142857 0.0 0.0 0.428571 0.0 0.0 0.285714 ... NaN 4.0 4.0 4.000000 4.0 NaN 21 M student 48823 25424 4.000000 0.0 0.333333 0.166667 0.0 0.0 0.333333 0.0 0.0 0.333333 ... NaN 4.0 4.0 4.000000 4.0 NaN 21 M student 48823 25426 3.875000 0.0 0.250000 0.125000 0.0 0.0 0.375000 0.0 0.0 0.250000 ... NaN 4.0 4.0 3.666667 4.0 NaN 21 M student 48823 25427 3.888889 0.0 0.222222 0.111111 0.0 0.0 0.333333 0.0 0.0 0.333333 ... NaN 4.0 4.0 3.666667 4.0 NaN 21 M student 48823 <p>5 rows \u00d7 43 columns</p> In\u00a0[24]: Copied! <pre>y_train\n</pre> y_train Out[24]: <pre>array([772, 288, 108, ..., 183, 432, 509])</pre> In\u00a0[25]: Copied! <pre>train_movies_watched\n</pre> train_movies_watched Out[25]: <pre>&lt;76228x1683 sparse matrix of type '&lt;class 'numpy.float64'&gt;'\n\twith 7957390 stored elements in COOrdinate format&gt;</pre> In\u00a0[26]: Copied! <pre>sorted(df_train.prev_movies.tolist()[0])\n</pre> sorted(df_train.prev_movies.tolist()[0]) Out[26]: <pre>['173', '185', '255', '286', '298']</pre> In\u00a0[27]: Copied! <pre>np.where(train_movies_watched.todense()[0])\n</pre> np.where(train_movies_watched.todense()[0]) Out[27]: <pre>(array([0, 0, 0, 0, 0]), array([173, 185, 255, 286, 298]))</pre> <p>And from now on is when the specifics related to this library start to appear. The only component that is going to be a bit different is the so-called tabular component, referred as <code>continuous</code> in the notebook.</p> <p>In the case of <code>pytorch-widedeep</code> we have the <code>TabPreprocessor</code> that allows for a lot of flexibility as to how we would like to process the tabular component of this Wide and Deep model. In other words, here our tabular component is a bit more elaborated than that in the notebook, just a bit...</p> In\u00a0[28]: Copied! <pre>cat_cols = [\"gender\", \"occupation\", \"zip_code\"]\ncont_cols = [c for c in X_train if c not in cat_cols]\ntab_preprocessor = TabPreprocessor(\n    cat_embed_cols=cat_cols,\n    continuous_cols=cont_cols,\n)\n</pre> cat_cols = [\"gender\", \"occupation\", \"zip_code\"] cont_cols = [c for c in X_train if c not in cat_cols] tab_preprocessor = TabPreprocessor(     cat_embed_cols=cat_cols,     continuous_cols=cont_cols, ) In\u00a0[29]: Copied! <pre>X_train_tab = tab_preprocessor.fit_transform(X_train.fillna(0))\nX_test_tab = tab_preprocessor.transform(X_test.fillna(0))\n</pre> X_train_tab = tab_preprocessor.fit_transform(X_train.fillna(0)) X_test_tab = tab_preprocessor.transform(X_test.fillna(0)) <p>Now, in the notebook, the author moves the sparse matrices to sparse tensors and then turns them into dense tensors. In reality, this is not neccessary, one could feed sparse tensors to <code>nn.Linear</code> layers in pytorch. Nonetheless, this is not the most efficient implementation and is the reason why in our library the wide, linear component is implemented as an embedding layer.</p> <p>Nonetheless, to reproduce the notebook the best we can and because currently the <code>Wide</code> model in <code>pytorch-widedeep</code> is not designed to receive sparse tensors (we might consider implementing this functionality), we will turn the sparse COO matrices into dense arrays. We will then code a fairly simple, custom <code>Wide</code> component.</p> In\u00a0[30]: Copied! <pre>X_train_wide = np.array(train_movies_watched.todense())\nX_test_wide = np.array(test_movies_watched.todense())\n</pre> X_train_wide = np.array(train_movies_watched.todense()) X_test_wide = np.array(test_movies_watched.todense()) <p>Finally, the author of the notebook uses a simple <code>Embedding</code> layer to encode the sequences of movies watched, the <code>prev_movies</code> columns. In my opinion, there is an element of information redundancy here. This is because the wide and text components have implicitely the same information, but in different form. Moreover, both of the models used for these two components ignore the sequential element in the data. Nonetheless, we want to reproduce the Kaggle notebook as close as possible, AND as one can explore later (by simply performing simple ablation studies), the wide component seems to carry most of the predictive power.</p> In\u00a0[31]: Copied! <pre>X_train_text = sparse_to_idx(train_movies_watched, pad_idx=PAD_IDX)\nX_test_text = sparse_to_idx(test_movies_watched, pad_idx=PAD_IDX)\n</pre> X_train_text = sparse_to_idx(train_movies_watched, pad_idx=PAD_IDX) X_test_text = sparse_to_idx(test_movies_watched, pad_idx=PAD_IDX) <p>Let's now build the models</p> In\u00a0[32]: Copied! <pre>class Wide(nn.Module):\n    def __init__(self, input_dim: int, pred_dim: int):\n        super().__init__()\n\n        self.input_dim = input_dim\n        self.pred_dim = pred_dim\n\n        # When I coded the library I never though that someone would want to code\n        # their own wide component. However, if you do, the wide component must have\n        # a 'wide_linear' attribute. In other words, the linear layer must be\n        # called 'wide_linear'\n        self.wide_linear = nn.Linear(input_dim, pred_dim)\n\n    def forward(self, X):\n        out = self.wide_linear(X.type(torch.float32))\n        return out\n\n\nwide = Wide(X_train_wide.shape[1], max_movie_index + 1)\n</pre> class Wide(nn.Module):     def __init__(self, input_dim: int, pred_dim: int):         super().__init__()          self.input_dim = input_dim         self.pred_dim = pred_dim          # When I coded the library I never though that someone would want to code         # their own wide component. However, if you do, the wide component must have         # a 'wide_linear' attribute. In other words, the linear layer must be         # called 'wide_linear'         self.wide_linear = nn.Linear(input_dim, pred_dim)      def forward(self, X):         out = self.wide_linear(X.type(torch.float32))         return out   wide = Wide(X_train_wide.shape[1], max_movie_index + 1) In\u00a0[33]: Copied! <pre>wide\n</pre> wide Out[33]: <pre>Wide(\n  (wide_linear): Linear(in_features=1683, out_features=1683, bias=True)\n)</pre> In\u00a0[34]: Copied! <pre>class SimpleEmbed(nn.Module):\n    def __init__(self, vocab_size: int, embed_dim: int, pad_idx: int):\n        super().__init__()\n\n        self.vocab_size = vocab_size\n        self.embed_dim = embed_dim\n        self.pad_idx = pad_idx\n\n        # The sequences of movies watched are simply embedded in the Kaggle\n        # notebook. No RNN, Transformer or any model is used\n        self.embed = nn.Embedding(vocab_size, embed_dim, padding_idx=pad_idx)\n\n    def forward(self, X):\n        embed = self.embed(X)\n        embed_mean = torch.mean(embed, dim=1)\n        return embed_mean\n\n    @property\n    def output_dim(self) -&gt; int:\n        # All deep components in a custom 'pytorch-widedeep' model must have\n        # an output_dim property\n        return self.embed_dim\n\n\n#  In the notebook the author uses simply embeddings\nsimple_embed = SimpleEmbed(max_movie_index + 1, 16, 0)\n</pre> class SimpleEmbed(nn.Module):     def __init__(self, vocab_size: int, embed_dim: int, pad_idx: int):         super().__init__()          self.vocab_size = vocab_size         self.embed_dim = embed_dim         self.pad_idx = pad_idx          # The sequences of movies watched are simply embedded in the Kaggle         # notebook. No RNN, Transformer or any model is used         self.embed = nn.Embedding(vocab_size, embed_dim, padding_idx=pad_idx)      def forward(self, X):         embed = self.embed(X)         embed_mean = torch.mean(embed, dim=1)         return embed_mean      @property     def output_dim(self) -&gt; int:         # All deep components in a custom 'pytorch-widedeep' model must have         # an output_dim property         return self.embed_dim   #  In the notebook the author uses simply embeddings simple_embed = SimpleEmbed(max_movie_index + 1, 16, 0) In\u00a0[35]: Copied! <pre>simple_embed\n</pre> simple_embed Out[35]: <pre>SimpleEmbed(\n  (embed): Embedding(1683, 16, padding_idx=0)\n)</pre> <p>Maybe one would like to use an RNN to account for the sequence nature of the problem. If that was the case it would be as easy as:</p> In\u00a0[36]: Copied! <pre>basic_rnn = BasicRNN(\n    vocab_size=max_movie_index + 1,\n    embed_dim=16,\n    hidden_dim=32,\n    n_layers=2,\n    rnn_type=\"gru\",\n)\n</pre> basic_rnn = BasicRNN(     vocab_size=max_movie_index + 1,     embed_dim=16,     hidden_dim=32,     n_layers=2,     rnn_type=\"gru\", ) <p>And finally, the tabular component, which is the notebook is simply a stak of linear + Rely layers. In our case we have an embedding layer before the linear layers to encode categorial and numerical cols</p> In\u00a0[37]: Copied! <pre>tab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=tab_preprocessor.continuous_cols,\n    cont_norm_layer=None,\n    mlp_hidden_dims=[1024, 512, 256],\n    mlp_activation=\"relu\",\n)\n</pre> tab_mlp = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     continuous_cols=tab_preprocessor.continuous_cols,     cont_norm_layer=None,     mlp_hidden_dims=[1024, 512, 256],     mlp_activation=\"relu\", ) In\u00a0[38]: Copied! <pre>tab_mlp\n</pre> tab_mlp Out[38]: <pre>TabMlp(\n  (cat_embed): DiffSizeCatEmbeddings(\n    (embed_layers): ModuleDict(\n      (emb_layer_gender): Embedding(3, 2, padding_idx=0)\n      (emb_layer_occupation): Embedding(22, 9, padding_idx=0)\n      (emb_layer_zip_code): Embedding(648, 60, padding_idx=0)\n    )\n    (embedding_dropout): Dropout(p=0.0, inplace=False)\n  )\n  (cont_norm): Identity()\n  (encoder): MLP(\n    (mlp): Sequential(\n      (dense_layer_0): Sequential(\n        (0): Linear(in_features=111, out_features=1024, bias=True)\n        (1): ReLU(inplace=True)\n        (2): Dropout(p=0.1, inplace=False)\n      )\n      (dense_layer_1): Sequential(\n        (0): Linear(in_features=1024, out_features=512, bias=True)\n        (1): ReLU(inplace=True)\n        (2): Dropout(p=0.1, inplace=False)\n      )\n      (dense_layer_2): Sequential(\n        (0): Linear(in_features=512, out_features=256, bias=True)\n        (1): ReLU(inplace=True)\n        (2): Dropout(p=0.1, inplace=False)\n      )\n    )\n  )\n)</pre> <p>Finally, we simply wrap up all models with the <code>WideDeep</code> 'collector' class and we are ready to train.</p> In\u00a0[39]: Copied! <pre>wide_deep_model = WideDeep(\n    wide=wide, deeptabular=tab_mlp, deeptext=simple_embed, pred_dim=max_movie_index + 1\n)\n</pre> wide_deep_model = WideDeep(     wide=wide, deeptabular=tab_mlp, deeptext=simple_embed, pred_dim=max_movie_index + 1 ) In\u00a0[40]: Copied! <pre>wide_deep_model\n</pre> wide_deep_model Out[40]: <pre>WideDeep(\n  (wide): Wide(\n    (wide_linear): Linear(in_features=1683, out_features=1683, bias=True)\n  )\n  (deeptabular): Sequential(\n    (0): TabMlp(\n      (cat_embed): DiffSizeCatEmbeddings(\n        (embed_layers): ModuleDict(\n          (emb_layer_gender): Embedding(3, 2, padding_idx=0)\n          (emb_layer_occupation): Embedding(22, 9, padding_idx=0)\n          (emb_layer_zip_code): Embedding(648, 60, padding_idx=0)\n        )\n        (embedding_dropout): Dropout(p=0.0, inplace=False)\n      )\n      (cont_norm): Identity()\n      (encoder): MLP(\n        (mlp): Sequential(\n          (dense_layer_0): Sequential(\n            (0): Linear(in_features=111, out_features=1024, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_1): Sequential(\n            (0): Linear(in_features=1024, out_features=512, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_2): Sequential(\n            (0): Linear(in_features=512, out_features=256, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=256, out_features=1683, bias=True)\n  )\n  (deeptext): Sequential(\n    (0): SimpleEmbed(\n      (embed): Embedding(1683, 16, padding_idx=0)\n    )\n    (1): Linear(in_features=16, out_features=1683, bias=True)\n  )\n)</pre> <p>Note that the main difference between this wide and deep model and the Wide and Deep model in the Kaggle notebook is that in that notebook, the author concatenates the embedings and the tabular features, then passes this concatenation through a stack of linear + Relu layers with a final output dim of 256. Then concatenates this output with the binary features and connects this concatenation with the final linear layer (so the final weights are of dim (batch_size, 256 + 1683)). Our implementation follows the notation of the original paper and instead of concatenating the tabular, text and wide components and then connect them to the output neurons, we first compute their output, and then add it (see here: https://arxiv.org/pdf/1606.07792.pdf, their Eq 3). Note that this is effectively the same, with the caveat that while in one case one initialises a big weight matrix \"at once\", in our implementation we initialise different matrices for different components. Anyway, let's give it a go.</p> In\u00a0[41]: Copied! <pre>trainer = Trainer(\n    model=wide_deep_model,\n    objective=\"multiclass\",\n    custom_loss_function=nn.CrossEntropyLoss(ignore_index=PAD_IDX),\n    optimizers=torch.optim.Adam(wide_deep_model.parameters(), lr=1e-3),\n)\n</pre> trainer = Trainer(     model=wide_deep_model,     objective=\"multiclass\",     custom_loss_function=nn.CrossEntropyLoss(ignore_index=PAD_IDX),     optimizers=torch.optim.Adam(wide_deep_model.parameters(), lr=1e-3), ) In\u00a0[42]: Copied! <pre>trainer.fit(\n    X_train={\n        \"X_wide\": X_train_wide,\n        \"X_tab\": X_train_tab,\n        \"X_text\": X_train_text,\n        \"target\": y_train,\n    },\n    X_val={\n        \"X_wide\": X_test_wide,\n        \"X_tab\": X_test_tab,\n        \"X_text\": X_test_text,\n        \"target\": y_test,\n    },\n    n_epochs=5,\n    batch_size=512,\n    shuffle=False,\n)\n</pre> trainer.fit(     X_train={         \"X_wide\": X_train_wide,         \"X_tab\": X_train_tab,         \"X_text\": X_train_text,         \"target\": y_train,     },     X_val={         \"X_wide\": X_test_wide,         \"X_tab\": X_test_tab,         \"X_text\": X_test_text,         \"target\": y_test,     },     n_epochs=5,     batch_size=512,     shuffle=False, ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 149/149 [00:19&lt;00:00,  7.66it/s, loss=6.66]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 38/38 [00:02&lt;00:00, 18.75it/s, loss=6.6]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 149/149 [00:21&lt;00:00,  6.95it/s, loss=5.97]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 38/38 [00:01&lt;00:00, 21.03it/s, loss=6.52]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 149/149 [00:19&lt;00:00,  7.51it/s, loss=5.65]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 38/38 [00:01&lt;00:00, 20.16it/s, loss=6.53]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 149/149 [00:23&lt;00:00,  6.29it/s, loss=5.41]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 38/38 [00:02&lt;00:00, 13.97it/s, loss=6.57]\nepoch 5: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 149/149 [00:19&lt;00:00,  7.58it/s, loss=5.2]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 38/38 [00:02&lt;00:00, 18.82it/s, loss=6.63]\n</pre> <p>Now one could continue to the 'compare' metrics section of the Kaggle notebook. However, for the purposes of illustrating how one could use <code>pytorch-widedeep</code> to build recommendation algorithms we consider this notebook completed and move onto part 2</p>"},{"location":"examples/18_wide_and_deep_for_recsys_pt1.html#problem-formulation","title":"Problem formulation\u00b6","text":"<p>In this particular exercise the problem is formulated as predicting the next movie that will be watched (in consequence the last interactions will be discarded)</p>"},{"location":"examples/18_wide_and_deep_for_recsys_pt2.html","title":"18 wide and deep for recsys pt2","text":"<p>This is the second of the two notebooks where we aim to illustrate how one could use this library to build recommendation algorithms using the example in this Kaggle notebook as guidance. In the previous notebook we used <code>pytorch-widedeep</code> to build a model that replicated almost exactly that in the notebook. In this, shorter notebook we will show how one could use the library to explore other models, following the same problem formulation, this is: given a state of a user at a certain point in time having watched a series of movies, our goal is to predict which movie the user will watch next.</p> <p>Assuming that one has read (and run) the previous notebook, the required data will be stored in a local dir called <code>prepared_data</code>, so let's read it:</p> In\u00a0[1]: Copied! <pre>from pathlib import Path\n\nimport numpy as np\nimport torch\nimport pandas as pd\nfrom torch import nn\n\nfrom pytorch_widedeep import Trainer\nfrom pytorch_widedeep.utils import pad_sequences\nfrom pytorch_widedeep.models import TabMlp, WideDeep, Transformer\nfrom pytorch_widedeep.preprocessing import TabPreprocessor\n</pre> from pathlib import Path  import numpy as np import torch import pandas as pd from torch import nn  from pytorch_widedeep import Trainer from pytorch_widedeep.utils import pad_sequences from pytorch_widedeep.models import TabMlp, WideDeep, Transformer from pytorch_widedeep.preprocessing import TabPreprocessor In\u00a0[2]: Copied! <pre>save_path = Path(\"prepared_data\")\n\nPAD_IDX = 0\n\nid_cols = [\"user_id\", \"movie_id\"]\n\ndf_train = pd.read_pickle(save_path / \"df_train.pkl\")\ndf_valid = pd.read_pickle(save_path / \"df_valid.pkl\")\ndf_test = pd.read_pickle(save_path / \"df_test.pkl\")\n</pre> save_path = Path(\"prepared_data\")  PAD_IDX = 0  id_cols = [\"user_id\", \"movie_id\"]  df_train = pd.read_pickle(save_path / \"df_train.pkl\") df_valid = pd.read_pickle(save_path / \"df_valid.pkl\") df_test = pd.read_pickle(save_path / \"df_test.pkl\") <p>...remember that in the previous notebook we explained that we are not  going to use a validation set here (in a real-world example, or simply a more realistic example, one should always use it).</p> In\u00a0[3]: Copied! <pre>df_test = pd.concat([df_valid, df_test], ignore_index=True)\n</pre> df_test = pd.concat([df_valid, df_test], ignore_index=True) <p>Also remember that, in the previous notebook we discussed that the <code>'maxlen'</code> and <code>'max_movie_index'</code> parameters should be computed using only the train set. In particular, to properly do the tokenization, one would have to use ONLY train tokens and add a token for new 'unknown'/'unseen' movies in the test set. This can also be done with this library or manually, so I will leave it to the reader to implement that tokenzation appraoch.</p> In\u00a0[4]: Copied! <pre>maxlen = max(\n    df_train.prev_movies.apply(lambda x: len(x)).max(),\n    df_test.prev_movies.apply(lambda x: len(x)).max(),\n)\n\nmax_movie_index = max(df_train.movie_id.max(), df_test.movie_id.max())\n</pre> maxlen = max(     df_train.prev_movies.apply(lambda x: len(x)).max(),     df_test.prev_movies.apply(lambda x: len(x)).max(), )  max_movie_index = max(df_train.movie_id.max(), df_test.movie_id.max()) <p>From now one things are pretty simple, moreover bearing in mind that in this example we are not going to use a wide component since, in pple, one would believe that the information in that component is also 'carried' by the movie sequences (However in the previous notebook, if one performs ablation studies, these suggest that most of the prediction power comes from the linear, wide model).</p> <p>In the example here we are going to explore one (of many) possibilities. We are simply going to encode the triplet <code>(user, item, rating)</code> and use it as a <code>deeptabular</code> component and the sequences of previously watched movies as the <code>deeptext</code> component. For the <code>deeptext</code> component we are going to use a basic encoder-only transformer model.</p> <p>Let's start with the tabular data preparation</p> In\u00a0[5]: Copied! <pre>df_train_user_item = df_train[[\"user_id\", \"movie_id\", \"rating\"]]\ntrain_movies_sequences = df_train.prev_movies.apply(\n    lambda x: [int(el) for el in x]\n).to_list()\ny_train = df_train.target.values.astype(int)\n\ndf_test_user_item = df_train[[\"user_id\", \"movie_id\", \"rating\"]]\ntest_movies_sequences = df_test.prev_movies.apply(\n    lambda x: [int(el) for el in x]\n).to_list()\ny_test = df_test.target.values.astype(int)\n\ntab_preprocessor = tab_preprocessor = TabPreprocessor(\n    cat_embed_cols=[\"user_id\", \"movie_id\", \"rating\"],\n)\nX_train_tab = tab_preprocessor.fit_transform(df_train_user_item)\nX_test_tab = tab_preprocessor.transform(df_test_user_item)\n</pre> df_train_user_item = df_train[[\"user_id\", \"movie_id\", \"rating\"]] train_movies_sequences = df_train.prev_movies.apply(     lambda x: [int(el) for el in x] ).to_list() y_train = df_train.target.values.astype(int)  df_test_user_item = df_train[[\"user_id\", \"movie_id\", \"rating\"]] test_movies_sequences = df_test.prev_movies.apply(     lambda x: [int(el) for el in x] ).to_list() y_test = df_test.target.values.astype(int)  tab_preprocessor = tab_preprocessor = TabPreprocessor(     cat_embed_cols=[\"user_id\", \"movie_id\", \"rating\"], ) X_train_tab = tab_preprocessor.fit_transform(df_train_user_item) X_test_tab = tab_preprocessor.transform(df_test_user_item) <p>And not the text component, simply padding the sequences:</p> In\u00a0[6]: Copied! <pre>X_train_text = np.array(\n    [\n        pad_sequences(\n            s,\n            maxlen=maxlen,\n            pad_first=False,\n            pad_idx=PAD_IDX,\n        )\n        for s in train_movies_sequences\n    ]\n)\nX_test_text = np.array(\n    [\n        pad_sequences(\n            s,\n            maxlen=maxlen,\n            pad_first=False,\n            pad_idx=0,\n        )\n        for s in test_movies_sequences\n    ]\n)\n</pre> X_train_text = np.array(     [         pad_sequences(             s,             maxlen=maxlen,             pad_first=False,             pad_idx=PAD_IDX,         )         for s in train_movies_sequences     ] ) X_test_text = np.array(     [         pad_sequences(             s,             maxlen=maxlen,             pad_first=False,             pad_idx=0,         )         for s in test_movies_sequences     ] ) <p>We now define the model components and the wide and deep model.</p> In\u00a0[7]: Copied! <pre>tab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    mlp_hidden_dims=[1024, 512, 256],\n    mlp_activation=\"relu\",\n)\n\n# plenty of options here, see the docs\ntransformer = Transformer(\n    vocab_size=max_movie_index + 1,\n    embed_dim=32,\n    n_heads=2,\n    n_blocks=2,\n    seq_length=maxlen,\n)\n\nwide_deep_model = WideDeep(\n    deeptabular=tab_mlp, deeptext=transformer, pred_dim=max_movie_index + 1\n)\n</pre> tab_mlp = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     mlp_hidden_dims=[1024, 512, 256],     mlp_activation=\"relu\", )  # plenty of options here, see the docs transformer = Transformer(     vocab_size=max_movie_index + 1,     embed_dim=32,     n_heads=2,     n_blocks=2,     seq_length=maxlen, )  wide_deep_model = WideDeep(     deeptabular=tab_mlp, deeptext=transformer, pred_dim=max_movie_index + 1 ) In\u00a0[8]: Copied! <pre>wide_deep_model\n</pre> wide_deep_model Out[8]: <pre>WideDeep(\n  (deeptabular): Sequential(\n    (0): TabMlp(\n      (cat_embed): DiffSizeCatEmbeddings(\n        (embed_layers): ModuleDict(\n          (emb_layer_user_id): Embedding(749, 65, padding_idx=0)\n          (emb_layer_movie_id): Embedding(1612, 100, padding_idx=0)\n          (emb_layer_rating): Embedding(6, 4, padding_idx=0)\n        )\n        (embedding_dropout): Dropout(p=0.0, inplace=False)\n      )\n      (encoder): MLP(\n        (mlp): Sequential(\n          (dense_layer_0): Sequential(\n            (0): Linear(in_features=169, out_features=1024, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_1): Sequential(\n            (0): Linear(in_features=1024, out_features=512, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_2): Sequential(\n            (0): Linear(in_features=512, out_features=256, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=256, out_features=1683, bias=True)\n  )\n  (deeptext): Sequential(\n    (0): Transformer(\n      (embedding): Embedding(1683, 32, padding_idx=0)\n      (pos_encoder): PositionalEncoding(\n        (dropout): Dropout(p=0.1, inplace=False)\n      )\n      (encoder): Sequential(\n        (transformer_block0): TransformerEncoder(\n          (attn): MultiHeadedAttention(\n            (dropout): Dropout(p=0.1, inplace=False)\n            (q_proj): Linear(in_features=32, out_features=32, bias=False)\n            (kv_proj): Linear(in_features=32, out_features=64, bias=False)\n            (out_proj): Linear(in_features=32, out_features=32, bias=False)\n          )\n          (ff): FeedForward(\n            (w_1): Linear(in_features=32, out_features=128, bias=True)\n            (w_2): Linear(in_features=128, out_features=32, bias=True)\n            (dropout): Dropout(p=0.1, inplace=False)\n            (activation): GELU(approximate='none')\n          )\n          (attn_addnorm): AddNorm(\n            (dropout): Dropout(p=0.1, inplace=False)\n            (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n          )\n          (ff_addnorm): AddNorm(\n            (dropout): Dropout(p=0.1, inplace=False)\n            (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n          )\n        )\n        (transformer_block1): TransformerEncoder(\n          (attn): MultiHeadedAttention(\n            (dropout): Dropout(p=0.1, inplace=False)\n            (q_proj): Linear(in_features=32, out_features=32, bias=False)\n            (kv_proj): Linear(in_features=32, out_features=64, bias=False)\n            (out_proj): Linear(in_features=32, out_features=32, bias=False)\n          )\n          (ff): FeedForward(\n            (w_1): Linear(in_features=32, out_features=128, bias=True)\n            (w_2): Linear(in_features=128, out_features=32, bias=True)\n            (dropout): Dropout(p=0.1, inplace=False)\n            (activation): GELU(approximate='none')\n          )\n          (attn_addnorm): AddNorm(\n            (dropout): Dropout(p=0.1, inplace=False)\n            (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n          )\n          (ff_addnorm): AddNorm(\n            (dropout): Dropout(p=0.1, inplace=False)\n            (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=23552, out_features=1683, bias=True)\n  )\n)</pre> <p>And as in the previous notebook, let's train (you will need a GPU for this)</p> In\u00a0[\u00a0]: Copied! <pre>trainer = Trainer(\n    model=wide_deep_model,\n    objective=\"multiclass\",\n    custom_loss_function=nn.CrossEntropyLoss(ignore_index=PAD_IDX),\n    optimizers=torch.optim.Adam(wide_deep_model.parameters(), lr=1e-3),\n)\n\ntrainer.fit(\n    X_train={\n        \"X_tab\": X_train_tab,\n        \"X_text\": X_train_text,\n        \"target\": y_train,\n    },\n    X_val={\n        \"X_tab\": X_test_tab,\n        \"X_text\": X_test_text,\n        \"target\": y_test,\n    },\n    n_epochs=10,\n    batch_size=521,\n    shuffle=False,\n)\n</pre> trainer = Trainer(     model=wide_deep_model,     objective=\"multiclass\",     custom_loss_function=nn.CrossEntropyLoss(ignore_index=PAD_IDX),     optimizers=torch.optim.Adam(wide_deep_model.parameters(), lr=1e-3), )  trainer.fit(     X_train={         \"X_tab\": X_train_tab,         \"X_text\": X_train_text,         \"target\": y_train,     },     X_val={         \"X_tab\": X_test_tab,         \"X_text\": X_test_text,         \"target\": y_test,     },     n_epochs=10,     batch_size=521,     shuffle=False, ) <pre>epoch 1:   0%|                                                                                                         | 0/147 [00:34&lt;?, ?it/s]\n</pre> In\u00a0[\u00a0]: Copied! <pre>\n</pre>"},{"location":"examples/19_load_from_folder_functionality.html","title":"19 load from folder functionality","text":"<p>In this notebook I want to illustrate how one can use our <code>[...]FromFolder</code> functionalities along with the <code>[...]ChunkPreProcessors</code> in those cases where the dataset is too bit to fit in memory.</p> <p>These functionalities in the library have been designed for the following scenarop</p> In\u00a0[1]: Copied! <pre>import numpy as np\nimport torch\nimport pandas as pd\nfrom torch.utils.data import DataLoader\n\nfrom pytorch_widedeep.models import TabMlp, Vision, BasicRNN, WideDeep\nfrom pytorch_widedeep.training import TrainerFromFolder\nfrom pytorch_widedeep.callbacks import EarlyStopping, ModelCheckpoint\nfrom pytorch_widedeep.preprocessing import (\n    TabPreprocessor,\n    TextPreprocessor,\n    ImagePreprocessor,\n    ChunkTabPreprocessor,\n    ChunkTextPreprocessor,\n)\nfrom pytorch_widedeep.load_from_folder import (\n    TabFromFolder,\n    TextFromFolder,\n    ImageFromFolder,\n    WideDeepDatasetFromFolder,\n)\n</pre> import numpy as np import torch import pandas as pd from torch.utils.data import DataLoader  from pytorch_widedeep.models import TabMlp, Vision, BasicRNN, WideDeep from pytorch_widedeep.training import TrainerFromFolder from pytorch_widedeep.callbacks import EarlyStopping, ModelCheckpoint from pytorch_widedeep.preprocessing import (     TabPreprocessor,     TextPreprocessor,     ImagePreprocessor,     ChunkTabPreprocessor,     ChunkTextPreprocessor, ) from pytorch_widedeep.load_from_folder import (     TabFromFolder,     TextFromFolder,     ImageFromFolder,     WideDeepDatasetFromFolder, ) In\u00a0[2]: Copied! <pre># in my case, I place the data in a folder I call tmp_data, let's see how it looks\nairbnb_data = pd.read_csv(\"../tmp_data/airbnb/airbnb_sample.csv\")\nairbnb_data.sample(5)\n</pre> # in my case, I place the data in a folder I call tmp_data, let's see how it looks airbnb_data = pd.read_csv(\"../tmp_data/airbnb/airbnb_sample.csv\") airbnb_data.sample(5) Out[2]: id host_id description host_listings_count host_identity_verified neighbourhood_cleansed latitude longitude is_location_exact property_type ... amenity_wide_entrance amenity_wide_entrance_for_guests amenity_wide_entryway amenity_wide_hallways amenity_wifi amenity_window_guards amenity_wine_cooler security_deposit extra_people yield 39 53242.jpg 247650 A lovely big bright bedroom in a 2 bedroom fla... 2.0 t Lambeth 51.47075 -0.12913 t apartment ... 0 0 0 0 1 0 0 250.0 5.0 9.75 214 236716.jpg 1241070 We offer a warm welcome in our quiet double ro... 1.0 t Hackney 51.56593 -0.07482 t other ... 0 0 0 0 1 0 0 200.0 10.0 76.50 400 346523.jpg 1756532 Available for you to rent is a cozy studio in ... 2.0 t Kensington and Chelsea 51.48311 -0.18428 t other ... 0 0 0 0 1 0 0 0.0 50.0 180.90 512 389627.jpg 1949299 This gorgeous studio flat is situated in the v... 1.0 t Westminster 51.51838 -0.14238 f apartment ... 0 0 0 0 1 0 0 250.0 25.0 276.90 504 388767.jpg 1945165 If you want to experience London at it's best ... 2.0 f Camden 51.54293 -0.14073 t apartment ... 0 0 0 0 1 0 0 150.0 10.0 591.10 <p>5 rows \u00d7 223 columns</p> In\u00a0[3]: Copied! <pre># for example\nfrom IPython.display import Image\n\npil_img = Image(filename=\"../tmp_data/airbnb/property_picture/272908.jpg\")\ndisplay(pil_img)\n</pre> # for example from IPython.display import Image  pil_img = Image(filename=\"../tmp_data/airbnb/property_picture/272908.jpg\") display(pil_img) In\u00a0[4]: Copied! <pre># And the description for the property that that picture belongs to is:\nairbnb_data[airbnb_data.id == \"272908.jpg\"].description.tolist()\n</pre> # And the description for the property that that picture belongs to is: airbnb_data[airbnb_data.id == \"272908.jpg\"].description.tolist() Out[4]: <pre>[\"Bright, sunny beautiful room that will give you the perfect base to explore all of London. Come and explore one of London's best neighbourhoods - Herne Hill! As mentioned in (Website hidden by Airbnb)   (Website hidden by Airbnb)  WiFi availability with a fully stocked and clean uplifting home. Lovely sunny, airy and big double bedroom on a leafy south-London street.    Note: This room comes with a reserved Off-Street parking spot! The room is on the first floor and boasts an enormous Super King bed, gorgeous wooden floors, tall ceilings and large windows which let in the sunshine almost all day. (Yoga May or meditation cushion available on request) The flat is bright and airy and big! So lots of space for all.  Location wise you are only 10 minutes walk to either Herne Hill or West Dulwich stations, both of which will take you to Victoria and the city within minutes. You can also hop on a bus right outside the flat that will take you to Brixton tube station within 8 minutes where you \"]</pre> <p>Ok, so we have tabular data where one column is <code>description</code> and another <code>id</code>, points towards the images stored in disk. Now, remember the following, because this will appear a few times in the notebook: our \"reference dataset\" is the tabular data.</p> <p>Therefore, since I want to illustrate a \"semi-realistic\" case, if we need to split the data into training, validation and test datasets, these datasets needs to be separetely stored in disk. In my case I have done this and in the <code>tmp_data/airbnb</code> dir I have the following:</p> <pre>../tmp_data/airbnb\n\u251c\u2500\u2500 airbnb_sample.csv\n\u251c\u2500\u2500 airbnb_sample_eval.csv\n\u251c\u2500\u2500 airbnb_sample_test.csv\n\u251c\u2500\u2500 airbnb_sample_train.csv\n\u2514\u2500\u2500 property_picture\n</pre> <p>Where <code>airbnb_sample.csv</code> is the full sample (1001 rows) and the <code>train</code>, <code>eval</code> and <code>test</code> set is the corresponding splits. In a realistic example, the full sample would be the 'gigantic' dataset and the rest the corresponding splits. One has to do this 'offline', prior to start the coding.</p> <p>Also, one thing that one needs to know is the number of total observations/rows, as well as the splits. In our case the train size is 800, and the eval and test sizes are 100 and 101 respectively.</p> <p>With all that info, let's star</p> In\u00a0[5]: Copied! <pre># path to the tabular data and the splits\ndata_path = \"../tmp_data/airbnb/\"\ntrain_fname = \"airbnb_sample_train.csv\"\neval_fname = \"airbnb_sample_eval.csv\"\ntest_fname = \"airbnb_sample_test.csv\"\n\n# split sizes\ntrain_size = 800\neval_size = 100\ntest_size = 101\n\n# number of chunks for the Chunk Preprocessors\nchunksize = 100\nn_chunks = int(np.ceil(train_size / chunksize))\n\n# path to the image dataset and name of the image col\nimg_path = \"../tmp_data/airbnb/property_picture/\"\nimg_col = \"id\"\n\n# name of the text col\ntext_col = \"description\"\n\n# mane of the target\ntarget_col = \"yield\"\n\n# definition of the categorical and continuous cols for the TabPreprocessor\ncat_embed_cols = [\n    \"host_listings_count\",\n    \"neighbourhood_cleansed\",\n    \"is_location_exact\",\n    \"property_type\",\n    \"room_type\",\n    \"accommodates\",\n    \"bathrooms\",\n    \"bedrooms\",\n    \"beds\",\n    \"guests_included\",\n    \"minimum_nights\",\n    \"instant_bookable\",\n    \"cancellation_policy\",\n    \"has_house_rules\",\n    \"host_gender\",\n    \"accommodates_catg\",\n    \"guests_included_catg\",\n    \"minimum_nights_catg\",\n    \"host_listings_count_catg\",\n    \"bathrooms_catg\",\n    \"bedrooms_catg\",\n    \"beds_catg\",\n    \"security_deposit\",\n    \"extra_people\",\n]\ncont_cols = [\"latitude\", \"longitude\"]\n</pre> # path to the tabular data and the splits data_path = \"../tmp_data/airbnb/\" train_fname = \"airbnb_sample_train.csv\" eval_fname = \"airbnb_sample_eval.csv\" test_fname = \"airbnb_sample_test.csv\"  # split sizes train_size = 800 eval_size = 100 test_size = 101  # number of chunks for the Chunk Preprocessors chunksize = 100 n_chunks = int(np.ceil(train_size / chunksize))  # path to the image dataset and name of the image col img_path = \"../tmp_data/airbnb/property_picture/\" img_col = \"id\"  # name of the text col text_col = \"description\"  # mane of the target target_col = \"yield\"  # definition of the categorical and continuous cols for the TabPreprocessor cat_embed_cols = [     \"host_listings_count\",     \"neighbourhood_cleansed\",     \"is_location_exact\",     \"property_type\",     \"room_type\",     \"accommodates\",     \"bathrooms\",     \"bedrooms\",     \"beds\",     \"guests_included\",     \"minimum_nights\",     \"instant_bookable\",     \"cancellation_policy\",     \"has_house_rules\",     \"host_gender\",     \"accommodates_catg\",     \"guests_included_catg\",     \"minimum_nights_catg\",     \"host_listings_count_catg\",     \"bathrooms_catg\",     \"bedrooms_catg\",     \"beds_catg\",     \"security_deposit\",     \"extra_people\", ] cont_cols = [\"latitude\", \"longitude\"] In\u00a0[6]: Copied! <pre>tab_preprocessor = TabPreprocessor(\n    embed_cols=cat_embed_cols,\n    continuous_cols=cont_cols,\n    default_embed_dim=8,\n    verbose=0,\n)\n\ntext_preprocessor = TextPreprocessor(\n    text_col=text_col,\n    n_cpus=1,\n)\n\nimg_preprocessor = ImagePreprocessor(\n    img_col=img_col,\n    img_path=img_path,\n)\n</pre> tab_preprocessor = TabPreprocessor(     embed_cols=cat_embed_cols,     continuous_cols=cont_cols,     default_embed_dim=8,     verbose=0, )  text_preprocessor = TextPreprocessor(     text_col=text_col,     n_cpus=1, )  img_preprocessor = ImagePreprocessor(     img_col=img_col,     img_path=img_path, ) In\u00a0[7]: Copied! <pre>tab_preprocessor.fit(airbnb_data)\ntext_preprocessor.fit(airbnb_data)\nimg_preprocessor.fit(airbnb_data)\n</pre> tab_preprocessor.fit(airbnb_data) text_preprocessor.fit(airbnb_data) img_preprocessor.fit(airbnb_data) <pre>The vocabulary contains 2192 tokens\n</pre> Out[7]: <pre>ImagePreprocessor(img_col=id, img_path=../tmp_data/airbnb/property_picture/, width=224, height=224, verbose=1)</pre> In\u00a0[8]: Copied! <pre>chunk_tab_preprocessor = ChunkTabPreprocessor(\n    embed_cols=cat_embed_cols,\n    continuous_cols=cont_cols,\n    n_chunks=n_chunks,\n    default_embed_dim=8,\n    verbose=0,\n)\n\nchunk_text_preprocessor = ChunkTextPreprocessor(\n    n_chunks=n_chunks,\n    text_col=text_col,\n    n_cpus=1,\n    verbose=0,\n)\n\nfor i, chunk in enumerate(\n    pd.read_csv(\"/\".join([data_path, train_fname]), chunksize=chunksize)\n):\n    print(f\"chunk in loop: {i + 1}\")\n    chunk_tab_preprocessor.fit(chunk)\n    chunk_text_preprocessor.fit(chunk)\n</pre> chunk_tab_preprocessor = ChunkTabPreprocessor(     embed_cols=cat_embed_cols,     continuous_cols=cont_cols,     n_chunks=n_chunks,     default_embed_dim=8,     verbose=0, )  chunk_text_preprocessor = ChunkTextPreprocessor(     n_chunks=n_chunks,     text_col=text_col,     n_cpus=1,     verbose=0, )  for i, chunk in enumerate(     pd.read_csv(\"/\".join([data_path, train_fname]), chunksize=chunksize) ):     print(f\"chunk in loop: {i + 1}\")     chunk_tab_preprocessor.fit(chunk)     chunk_text_preprocessor.fit(chunk) <pre>chunk in loop: 1\nchunk in loop: 2\nchunk in loop: 3\nchunk in loop: 4\nchunk in loop: 5\nchunk in loop: 6\nchunk in loop: 7\nchunk in loop: 8\n</pre> In\u00a0[9]: Copied! <pre>train_tab_folder = TabFromFolder(\n    fname=train_fname,\n    directory=data_path,\n    target_col=target_col,\n    preprocessor=tab_preprocessor,\n    text_col=text_col,\n    img_col=img_col,\n)\n\n# Note how we can use the `train_tab_folder` as reference so we don't have to\n# define all parameters again\neval_tab_folder = TabFromFolder(fname=eval_fname, reference=train_tab_folder)\n\n# Note that for the test set we can ignore the target as no metric will be\n# computed by the `predict` method\ntest_tab_folder = TabFromFolder(\n    fname=test_fname, reference=train_tab_folder, ignore_target=True\n)\n</pre> train_tab_folder = TabFromFolder(     fname=train_fname,     directory=data_path,     target_col=target_col,     preprocessor=tab_preprocessor,     text_col=text_col,     img_col=img_col, )  # Note how we can use the `train_tab_folder` as reference so we don't have to # define all parameters again eval_tab_folder = TabFromFolder(fname=eval_fname, reference=train_tab_folder)  # Note that for the test set we can ignore the target as no metric will be # computed by the `predict` method test_tab_folder = TabFromFolder(     fname=test_fname, reference=train_tab_folder, ignore_target=True ) In\u00a0[10]: Copied! <pre># for the text and image datasets we do not need to specify eval or test loaders\ntext_folder = TextFromFolder(preprocessor=text_preprocessor)\nimg_folder = ImageFromFolder(preprocessor=img_preprocessor)\n</pre> # for the text and image datasets we do not need to specify eval or test loaders text_folder = TextFromFolder(preprocessor=text_preprocessor) img_folder = ImageFromFolder(preprocessor=img_preprocessor) In\u00a0[11]: Copied! <pre>train_dataset_folder = WideDeepDatasetFromFolder(\n    n_samples=train_size,\n    tab_from_folder=train_tab_folder,\n    text_from_folder=text_folder,\n    img_from_folder=img_folder,\n)\n\n# Note that the eval and test loaders only need their corresponding\n# `TabFromFolder` classes. The rest of the parameters can be defined\n# via a `reference` `TabFromFolder` class\neval_dataset_folder = WideDeepDatasetFromFolder(\n    n_samples=eval_size,\n    tab_from_folder=eval_tab_folder,\n    reference=train_dataset_folder,\n)\n\ntest_dataset_folder = WideDeepDatasetFromFolder(\n    n_samples=test_size,\n    tab_from_folder=test_tab_folder,\n    reference=train_dataset_folder,\n)\n</pre> train_dataset_folder = WideDeepDatasetFromFolder(     n_samples=train_size,     tab_from_folder=train_tab_folder,     text_from_folder=text_folder,     img_from_folder=img_folder, )  # Note that the eval and test loaders only need their corresponding # `TabFromFolder` classes. The rest of the parameters can be defined # via a `reference` `TabFromFolder` class eval_dataset_folder = WideDeepDatasetFromFolder(     n_samples=eval_size,     tab_from_folder=eval_tab_folder,     reference=train_dataset_folder, )  test_dataset_folder = WideDeepDatasetFromFolder(     n_samples=test_size,     tab_from_folder=test_tab_folder,     reference=train_dataset_folder, ) In\u00a0[12]: Copied! <pre>train_loader = DataLoader(train_dataset_folder, batch_size=16, num_workers=1)\neval_loader = DataLoader(eval_dataset_folder, batch_size=16, num_workers=1)\ntest_loader = DataLoader(test_dataset_folder, batch_size=16, num_workers=1)\n</pre> train_loader = DataLoader(train_dataset_folder, batch_size=16, num_workers=1) eval_loader = DataLoader(eval_dataset_folder, batch_size=16, num_workers=1) test_loader = DataLoader(test_dataset_folder, batch_size=16, num_workers=1) <p>And from here on is business as usual:</p> In\u00a0[13]: Copied! <pre># for example\nbasic_rnn = BasicRNN(\n    vocab_size=len(text_preprocessor.vocab.itos),\n    embed_dim=32,\n    hidden_dim=64,\n    n_layers=2,\n)\n\ndeepimage = Vision()\n\ndeepdense = TabMlp(\n    mlp_hidden_dims=[32, 16],\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=cont_cols,\n)\n\nmodel = WideDeep(\n    deeptabular=deepdense,\n    deeptext=basic_rnn,\n    deepimage=deepimage,\n)\n\nmodel\n</pre> # for example basic_rnn = BasicRNN(     vocab_size=len(text_preprocessor.vocab.itos),     embed_dim=32,     hidden_dim=64,     n_layers=2, )  deepimage = Vision()  deepdense = TabMlp(     mlp_hidden_dims=[32, 16],     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     continuous_cols=cont_cols, )  model = WideDeep(     deeptabular=deepdense,     deeptext=basic_rnn,     deepimage=deepimage, )  model Out[13]: <pre>WideDeep(\n  (deeptabular): Sequential(\n    (0): TabMlp(\n      (cat_embed): DiffSizeCatEmbeddings(\n        (embed_layers): ModuleDict(\n          (emb_layer_host_listings_count): Embedding(28, 10, padding_idx=0)\n          (emb_layer_neighbourhood_cleansed): Embedding(33, 11, padding_idx=0)\n          (emb_layer_is_location_exact): Embedding(3, 2, padding_idx=0)\n          (emb_layer_property_type): Embedding(4, 3, padding_idx=0)\n          (emb_layer_room_type): Embedding(4, 3, padding_idx=0)\n          (emb_layer_accommodates): Embedding(14, 7, padding_idx=0)\n          (emb_layer_bathrooms): Embedding(11, 6, padding_idx=0)\n          (emb_layer_bedrooms): Embedding(7, 4, padding_idx=0)\n          (emb_layer_beds): Embedding(11, 6, padding_idx=0)\n          (emb_layer_guests_included): Embedding(11, 6, padding_idx=0)\n          (emb_layer_minimum_nights): Embedding(25, 9, padding_idx=0)\n          (emb_layer_instant_bookable): Embedding(3, 2, padding_idx=0)\n          (emb_layer_cancellation_policy): Embedding(6, 4, padding_idx=0)\n          (emb_layer_has_house_rules): Embedding(3, 2, padding_idx=0)\n          (emb_layer_host_gender): Embedding(4, 3, padding_idx=0)\n          (emb_layer_accommodates_catg): Embedding(4, 3, padding_idx=0)\n          (emb_layer_guests_included_catg): Embedding(4, 3, padding_idx=0)\n          (emb_layer_minimum_nights_catg): Embedding(4, 3, padding_idx=0)\n          (emb_layer_host_listings_count_catg): Embedding(5, 3, padding_idx=0)\n          (emb_layer_bathrooms_catg): Embedding(4, 3, padding_idx=0)\n          (emb_layer_bedrooms_catg): Embedding(5, 3, padding_idx=0)\n          (emb_layer_beds_catg): Embedding(5, 3, padding_idx=0)\n          (emb_layer_security_deposit): Embedding(53, 15, padding_idx=0)\n          (emb_layer_extra_people): Embedding(39, 12, padding_idx=0)\n        )\n        (embedding_dropout): Dropout(p=0.0, inplace=False)\n      )\n      (cont_norm): Identity()\n      (encoder): MLP(\n        (mlp): Sequential(\n          (dense_layer_0): Sequential(\n            (0): Linear(in_features=128, out_features=32, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_1): Sequential(\n            (0): Linear(in_features=32, out_features=16, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=16, out_features=1, bias=True)\n  )\n  (deeptext): Sequential(\n    (0): BasicRNN(\n      (word_embed): Embedding(2192, 32, padding_idx=1)\n      (rnn): LSTM(32, 64, num_layers=2, batch_first=True, dropout=0.1)\n      (rnn_mlp): Identity()\n    )\n    (1): Linear(in_features=64, out_features=1, bias=True)\n  )\n  (deepimage): Sequential(\n    (0): Vision(\n      (features): Sequential(\n        (conv_layer_0): Sequential(\n          (0): Conv2d(3, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False)\n          (1): BatchNorm2d(64, eps=1e-05, momentum=0.01, affine=True, track_running_stats=True)\n          (2): LeakyReLU(negative_slope=0.1, inplace=True)\n          (maxpool): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n        )\n        (conv_layer_1): Sequential(\n          (0): Conv2d(64, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n          (1): BatchNorm2d(128, eps=1e-05, momentum=0.01, affine=True, track_running_stats=True)\n          (2): LeakyReLU(negative_slope=0.1, inplace=True)\n        )\n        (conv_layer_2): Sequential(\n          (0): Conv2d(128, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n          (1): BatchNorm2d(256, eps=1e-05, momentum=0.01, affine=True, track_running_stats=True)\n          (2): LeakyReLU(negative_slope=0.1, inplace=True)\n        )\n        (conv_layer_3): Sequential(\n          (0): Conv2d(256, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n          (1): BatchNorm2d(512, eps=1e-05, momentum=0.01, affine=True, track_running_stats=True)\n          (2): LeakyReLU(negative_slope=0.1, inplace=True)\n          (adaptiveavgpool): AdaptiveAvgPool2d(output_size=(1, 1))\n        )\n      )\n    )\n    (1): Linear(in_features=512, out_features=1, bias=True)\n  )\n)</pre> In\u00a0[14]: Copied! <pre>trainer = TrainerFromFolder(\n    model,\n    objective=\"regression\",\n)\n\ntrainer.fit(\n    train_loader=train_loader,\n    eval_loader=eval_loader,\n)\n</pre> trainer = TrainerFromFolder(     model,     objective=\"regression\", )  trainer.fit(     train_loader=train_loader,     eval_loader=eval_loader, ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 50/50 [03:41&lt;00:00,  4.42s/it, loss=1.64e+4]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 7/7 [00:23&lt;00:00,  3.30s/it, loss=6.27e+3]\n</pre> In\u00a0[15]: Copied! <pre>preds = trainer.predict(test_loader=test_loader)\n</pre> preds = trainer.predict(test_loader=test_loader) <pre>predict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 7/7 [00:22&lt;00:00,  3.26s/it]\n</pre> <p>Note that in the case of predict you could also choose to do this</p> In\u00a0[16]: Copied! <pre>df_test = pd.read_csv(\"/\".join([data_path, test_fname]))\n</pre> df_test = pd.read_csv(\"/\".join([data_path, test_fname])) In\u00a0[17]: Copied! <pre># if the images for the test set fit in memory\nX_tab_test = chunk_tab_preprocessor.transform(df_test)\nX_text_test = chunk_text_preprocessor.transform(df_test)\nX_img_test = img_preprocessor.transform(df_test)\n</pre> # if the images for the test set fit in memory X_tab_test = chunk_tab_preprocessor.transform(df_test) X_text_test = chunk_text_preprocessor.transform(df_test) X_img_test = img_preprocessor.transform(df_test) <pre>Reading Images from ../tmp_data/airbnb/property_picture/\nResizing\n</pre> <pre>100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 101/101 [00:00&lt;00:00, 708.23it/s]</pre> <pre>Computing normalisation metrics\n</pre> <pre>\n</pre> In\u00a0[18]: Copied! <pre>preds = trainer.predict(\n    X_tab=X_tab_test, X_text=X_text_test, X_img=X_img_test, batch_size=32\n)\n</pre> preds = trainer.predict(     X_tab=X_tab_test, X_text=X_text_test, X_img=X_img_test, batch_size=32 ) <pre>predict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 4/4 [00:03&lt;00:00,  1.14it/s]\n</pre>"},{"location":"examples/19_load_from_folder_functionality.html#scenario","title":"Scenario\u00b6","text":"<p>We have a tabular dataset combined with images and text and either some, or all these datasets do not fit in memory. Note that the tabular dataset MUST ALWAYS be present as it is considered the rerefence. This is, if we have an image dataset, the tabular dataset must contain a column that points to the image file names as stored in disk. Similarly, if we have a text dataset, then the tabular dataset must contain a column with the texts themselves or a the file names of the text files as stored in disk.</p> <p>If you only have text and/or images and not a tabular component, I would suggest using other libraries (such as hugginface probably).</p> <p>Within this scenario, they are two possible scenarios that we will cover here:</p> <ol> <li><p>The tabular data itsel fits in memory and is only the images that do not: in this case you could use the 'standard' <code>Preprocessors</code> in the library and off you go, move directly to the <code>[...]FromFolder</code> functionalities</p> </li> <li><p>The tabular data is also very large and does not fit in memory, so we have to process it in chuncks. For this second case I have created the so called <code>Chunk[...]Preprocessor</code> (Wide, Tab and Text).</p> </li> </ol> <p>Note that at the moment ONLY csv format is allowed for the tabular file. More formats will be supported in the future.</p> <p>Let's see a complete example to illustrate how each of these cases would be addressed with the new functionalities in the library. For this example we will use a sample of the airbnb dataset</p> <p>The airbnb dataset, which you could get from here, is too big to be included in our datasets module (when including images). Therefore, what I did was, go there, download it, and use the download_images.py script to get the images and the airbnb_data_processing.py to process the data. I did this ages ago and I believe the format of the dataset might be different now. Nonetheless, I will show samples of the dataset as we go through so you can extrapolate the content of this notebook to your particular problem.</p> <p>In the future we will find better datasets\ud83d\ude42. Finally, note that here we are only using a small sample to illustrate the use, so PLEASE ignore the results, just focus on usage.</p>"},{"location":"examples/19_load_from_folder_functionality.html#setting-variables-and-constants","title":"Setting variables and constants\u00b6","text":""},{"location":"examples/19_load_from_folder_functionality.html#step-1-the-preprocessors","title":"Step 1: the preprocessors\u00b6","text":""},{"location":"examples/19_load_from_folder_functionality.html#scenario-1-only-the-images-do-not-fit-in-disk","title":"Scenario 1: only the images do not fit in disk\u00b6","text":"<p>In this case we can prepare the data in the 'standard' way</p>"},{"location":"examples/19_load_from_folder_functionality.html#scenario-2-the-tabular-data-is-also-huge","title":"Scenario 2: the tabular data is also huge\u00b6","text":"<p>Then we need to prepare it in chunks. Note that, unfortunately, the tabular and text preprocessors need to see the whole dataset once. This is because to process tabular or text data we need to encode values. For those encodings to be consistent they need to have seen the whole dataset. Alternatively, one could code a solution with some streaming encoder for both datasets. However, such implementation is not trivial for this library (and in general). I also don't think that having to see the whole data once is such a big limitation. Let's see how is done.</p> <p>Note that I have not mentioned the image dataset. This is because the processing of the image dataset does not require any form of encoding and in consequence can be done 'on the fly'. Therefore, no <code>ChunkImgPreprocessor</code> processor is needed.</p>"},{"location":"examples/19_load_from_folder_functionality.html#step-2-the-fromfolder-classes","title":"Step 2: the <code>[...]FromFolder</code> classes\u00b6","text":"<p>Once we have the preprocessors, we need to instantiate the classes that will enable us to load the data from their respective folders. From now on I am going to proceed with the <code>chunk_tab_preprocessor</code>, <code>chunk_text_preprocessor</code> and <code>img_preprocessor</code>, but the code would be identical if instead of the first two preprocessors we decided to use the <code>tab_preprocessor</code> and <code>text_preprocessor</code>.</p> <p>Once more, our reference datasets are the tabular datasets, which we have splitted in train, eval and test prior to start the coding. Therefore, we will eventually need a loader for each split</p>"},{"location":"examples/19_load_from_folder_functionality.html#step-3-pytorch-datasets-and-dataloaders","title":"Step 3: pytorch datasets and dataloaders\u00b6","text":"<p>From here in advance, is all very 'standard' if you are familiar with pytorch. One needs to define a class that inherits from the <code>Dataset</code> class in pytorch. Then this will be passed to a <code>DataLoader</code> class and we are ready to train. Our <code>Dataset</code> child class is <code>WideDeepDatasetFromFolder</code>. This class will use the tabular dataset and the corresponding text and image columns to load the adequate data in the batches</p> <p>Let's do it</p>"},{"location":"examples/19_load_from_folder_functionality.html#step-4-define-the-model","title":"Step 4: define the model\u00b6","text":""},{"location":"examples/19_load_from_folder_functionality.html#step-5-fit-and-predict","title":"Step 5: fit and predict\u00b6","text":""},{"location":"examples/20_Using_huggingface_within_widedeep.html","title":"20 Using huggingface within widedeep","text":"<p>In this notebook we will show how to use Hugginface's tokenizers and models as they are integrated within the library. In notebook number 17 you can find examples on how to code your own, custom, Hugginface (hereafter HF) model and use it in combination of any other model in the library</p> In\u00a0[1]: Copied! <pre>import numpy as np\nimport pandas as pd\nfrom sklearn.metrics import f1_score, accuracy_score\nfrom sklearn.model_selection import train_test_split\n\nfrom pytorch_widedeep import Trainer\nfrom pytorch_widedeep.models import HFModel, WideDeep\nfrom pytorch_widedeep.metrics import F1Score, Accuracy\nfrom pytorch_widedeep.datasets import load_womens_ecommerce\nfrom pytorch_widedeep.preprocessing import HFPreprocessor\n</pre> import numpy as np import pandas as pd from sklearn.metrics import f1_score, accuracy_score from sklearn.model_selection import train_test_split  from pytorch_widedeep import Trainer from pytorch_widedeep.models import HFModel, WideDeep from pytorch_widedeep.metrics import F1Score, Accuracy from pytorch_widedeep.datasets import load_womens_ecommerce from pytorch_widedeep.preprocessing import HFPreprocessor <pre>/Users/javierrodriguezzaurin/.pyenv/versions/3.10.13/envs/widedeep310/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n  from .autonotebook import tqdm as notebook_tqdm\n</pre> In\u00a0[2]: Copied! <pre>df: pd.DataFrame = load_womens_ecommerce(as_frame=True)  # type: ignore\n</pre> df: pd.DataFrame = load_womens_ecommerce(as_frame=True)  # type: ignore In\u00a0[3]: Copied! <pre>df.shape\n</pre> df.shape Out[3]: <pre>(23486, 10)</pre> In\u00a0[4]: Copied! <pre>df.sample(3)\n</pre> df.sample(3) Out[4]: Clothing ID Age Title Review Text Rating Recommended IND Positive Feedback Count Division Name Department Name Class Name 7004 862 43 Cute and feminine Loved this sweater wrap and bought it in both ... 5 1 2 General Tops Knits 12508 975 66 Love it The linen fabric is elegantly thin feels and l... 5 1 3 General Jackets Jackets 10288 950 41 Perfect for fall This sweater is just as pictured. the fit is t... 5 1 0 General Tops Sweaters In\u00a0[5]: Copied! <pre># Let's do some mild preprocessing\ndf.columns = [c.replace(\" \", \"_\").lower() for c in df.columns]\n\n# classes from [0,num_class)\ndf[\"rating\"] = (df[\"rating\"] - 1).astype(\"int64\")\n\n# group reviews with 1 and 2 scores into one class\ndf.loc[df.rating == 0, \"rating\"] = 1\n\n# and back again to [0,num_class)\ndf[\"rating\"] = (df[\"rating\"] - 1).astype(\"int64\")\n</pre> # Let's do some mild preprocessing df.columns = [c.replace(\" \", \"_\").lower() for c in df.columns]  # classes from [0,num_class) df[\"rating\"] = (df[\"rating\"] - 1).astype(\"int64\")  # group reviews with 1 and 2 scores into one class df.loc[df.rating == 0, \"rating\"] = 1  # and back again to [0,num_class) df[\"rating\"] = (df[\"rating\"] - 1).astype(\"int64\") In\u00a0[6]: Copied! <pre># drop short reviews\ndf = df[~df.review_text.isna()]\ndf[\"review_length\"] = df.review_text.apply(lambda x: len(x.split(\" \")))\ndf = df[df.review_length &gt;= 5]\ndf = df.drop(\"review_length\", axis=1).reset_index(drop=True)\n</pre> # drop short reviews df = df[~df.review_text.isna()] df[\"review_length\"] = df.review_text.apply(lambda x: len(x.split(\" \"))) df = df[df.review_length &gt;= 5] df = df.drop(\"review_length\", axis=1).reset_index(drop=True) In\u00a0[7]: Copied! <pre>df.shape\n</pre> df.shape Out[7]: <pre>(22608, 10)</pre> In\u00a0[8]: Copied! <pre># if you run this on a CPU, you might want to subsample the dataset. With that in mind I am simply going to stratify-sample to the minimum category occurrence and then sample at random\n# If you run this on a GPU you can comment out the following two cells\ndf.rating.value_counts()\n</pre> # if you run this on a CPU, you might want to subsample the dataset. With that in mind I am simply going to stratify-sample to the minimum category occurrence and then sample at random # If you run this on a GPU you can comment out the following two cells df.rating.value_counts() Out[8]: <pre>rating\n3    12515\n2     4904\n1     2820\n0     2369\nName: count, dtype: int64</pre> In\u00a0[9]: Copied! <pre>df = (\n    df.groupby(\"rating\", group_keys=False)\n    .apply(lambda x: x.sample(min(len(x), 2369)))\n    .sample(1000)\n)\n</pre> df = (     df.groupby(\"rating\", group_keys=False)     .apply(lambda x: x.sample(min(len(x), 2369)))     .sample(1000) ) <pre>/var/folders/_2/lrjn1qn54c758tdtktr1bvkc0000gn/T/ipykernel_5886/895673206.py:3: DeprecationWarning: DataFrameGroupBy.apply operated on the grouping columns. This behavior is deprecated, and in a future version of pandas the grouping columns will be excluded from the operation. Either pass `include_groups=False` to exclude the groupings or explicitly select the grouping columns after groupby to silence this warning.\n  .apply(lambda x: x.sample(min(len(x), 2369)))\n</pre> In\u00a0[10]: Copied! <pre>train, test = train_test_split(df, train_size=0.8, random_state=1, stratify=df.rating)\n\n# possible model names currently supported in the library\nmodel_names = [\n    \"distilbert-base-uncased\",\n    \"bert-base-uncased\",\n    \"FacebookAI/roberta-base\",\n    \"albert-base-v2\",\n    \"google/electra-base-discriminator\",\n]\n\n# Let's choose one. The syntax is the same for all the models\nmodel_name = \"distilbert-base-uncased\"\n</pre> train, test = train_test_split(df, train_size=0.8, random_state=1, stratify=df.rating)  # possible model names currently supported in the library model_names = [     \"distilbert-base-uncased\",     \"bert-base-uncased\",     \"FacebookAI/roberta-base\",     \"albert-base-v2\",     \"google/electra-base-discriminator\", ]  # Let's choose one. The syntax is the same for all the models model_name = \"distilbert-base-uncased\" <p>Now we can use the <code>HFPreprocessor</code> class. As most things in this library, the integration with HF has been coded aiming for a flexible use. With this in mind, there are two ways one can use a <code>HFPreprocessor</code> class.</p> <ol> <li>Passing a <code>text_col</code> and <code>encode_params</code> as the class is instantiated and then using the <code>fit</code> and <code>transform</code> as with any other preprocessor in the library</li> <li>Without passing  <code>text_col</code> and <code>encode_params</code> as the class is instantiated and using the <code>encode</code> method of the <code>HFPreprocessor</code> which is simply a wrapper around the encode method of HF's tokenizers</li> </ol> <p>Let's have a look</p> In\u00a0[11]: Copied! <pre>tokenizer1 = HFPreprocessor(\n    model_name=model_name,\n    text_col=\"review_text\",\n    num_workers=1,\n    encode_params={\n        \"max_length\": 90,\n        \"padding\": \"max_length\",\n        \"truncation\": True,\n        \"add_special_tokens\": True,\n    },\n)\n\nX_text_tr1 = tokenizer1.fit_transform(train)\nX_text_te1 = tokenizer1.transform(test)\n</pre> tokenizer1 = HFPreprocessor(     model_name=model_name,     text_col=\"review_text\",     num_workers=1,     encode_params={         \"max_length\": 90,         \"padding\": \"max_length\",         \"truncation\": True,         \"add_special_tokens\": True,     }, )  X_text_tr1 = tokenizer1.fit_transform(train) X_text_te1 = tokenizer1.transform(test) In\u00a0[12]: Copied! <pre>tokenizer2 = HFPreprocessor(\n    model_name=model_name,\n    num_workers=1,\n)\n\nX_text_tr2 = tokenizer2.encode(\n    train.review_text.tolist(),\n    max_length=90,\n    padding=\"max_length\",\n    truncation=True,\n    add_special_tokens=True,\n)\nX_text_te2 = tokenizer2.encode(\n    test.review_text.tolist(),\n    max_length=90,\n    padding=\"max_length\",\n    truncation=True,\n    add_special_tokens=True,\n)\n</pre> tokenizer2 = HFPreprocessor(     model_name=model_name,     num_workers=1, )  X_text_tr2 = tokenizer2.encode(     train.review_text.tolist(),     max_length=90,     padding=\"max_length\",     truncation=True,     add_special_tokens=True, ) X_text_te2 = tokenizer2.encode(     test.review_text.tolist(),     max_length=90,     padding=\"max_length\",     truncation=True,     add_special_tokens=True, ) In\u00a0[13]: Copied! <pre>all(X_text_tr1[0] == X_text_tr2[0])\n</pre> all(X_text_tr1[0] == X_text_tr2[0]) Out[13]: <pre>True</pre> In\u00a0[14]: Copied! <pre># Now we define a model which is as easy as:\n# Note that this will instantiation will lead to NO parameter trainable in the HF model.\n# If you want to fine-tune the HF model, you can set the trainable parameters via the 'trainable_parameters' argument.\n# Alternatively, you can use a head (MLP) via the 'head'-related arguments (see the docs for more details)\nhf_model = HFModel(model_name=model_name)\n</pre> # Now we define a model which is as easy as: # Note that this will instantiation will lead to NO parameter trainable in the HF model. # If you want to fine-tune the HF model, you can set the trainable parameters via the 'trainable_parameters' argument. # Alternatively, you can use a head (MLP) via the 'head'-related arguments (see the docs for more details) hf_model = HFModel(model_name=model_name) In\u00a0[15]: Copied! <pre># And from here on is the same as any other WideDeep model\nmodel = WideDeep(\n    deeptext=hf_model,\n    pred_dim=4,\n)\n\ntrainer = Trainer(\n    model,\n    objective=\"multiclass\",\n    metrics=[Accuracy(), F1Score(average=True)],\n)\n\ntrainer.fit(\n    X_text=X_text_tr2,\n    target=train.rating.values,\n    n_epochs=1,\n    batch_size=64,\n)\n# If you run this on a CPU and you sampled the data, the metrics will not be better than a random guess. Remember, this is just a demo\n</pre> # And from here on is the same as any other WideDeep model model = WideDeep(     deeptext=hf_model,     pred_dim=4, )  trainer = Trainer(     model,     objective=\"multiclass\",     metrics=[Accuracy(), F1Score(average=True)], )  trainer.fit(     X_text=X_text_tr2,     target=train.rating.values,     n_epochs=1,     batch_size=64, ) # If you run this on a CPU and you sampled the data, the metrics will not be better than a random guess. Remember, this is just a demo <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 13/13 [02:06&lt;00:00,  9.75s/it, loss=3.2, metrics={'acc': 0.235, 'f1': 0.2336}]  \n</pre> In\u00a0[17]: Copied! <pre>preds_text = trainer.predict_proba(X_text=X_text_te2)\npred_text_class = np.argmax(preds_text, 1)\n\nacc_text = accuracy_score(test.rating, pred_text_class)\nf1_text = f1_score(test.rating, pred_text_class, average=\"weighted\")\nprint(f\"Accuracy: {acc_text:.4f}\")\nprint(f\"F1: {f1_text:.4f}\")\n</pre> preds_text = trainer.predict_proba(X_text=X_text_te2) pred_text_class = np.argmax(preds_text, 1)  acc_text = accuracy_score(test.rating, pred_text_class) f1_text = f1_score(test.rating, pred_text_class, average=\"weighted\") print(f\"Accuracy: {acc_text:.4f}\") print(f\"F1: {f1_text:.4f}\") <pre>predict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 4/4 [00:05&lt;00:00,  1.43s/it]</pre> <pre>Accuracy: 0.2500\nF1: 0.1000\n</pre> <pre>\n</pre>"},{"location":"pytorch-widedeep/bayesian_models.html","title":"The <code>bayesian models</code> module","text":"<p>This module contains the two Bayesian Models available in this library, namely the bayesian version of the <code>Wide</code> and <code>TabMlp</code> models, referred as <code>BayesianWide</code> and <code>BayesianTabMlp</code>. These models are very useful in scenarios where getting a measure of uncertainty is important.</p> <p>The models in this module are based on the publication: Weight Uncertainty in Neural Networks.</p>"},{"location":"pytorch-widedeep/bayesian_models.html#pytorch_widedeep.bayesian_models.tabular.bayesian_linear.bayesian_wide.BayesianWide","title":"BayesianWide","text":"<pre><code>BayesianWide(\n    input_dim,\n    pred_dim=1,\n    prior_sigma_1=1.0,\n    prior_sigma_2=0.002,\n    prior_pi=0.8,\n    posterior_mu_init=0.0,\n    posterior_rho_init=-7.0,\n)\n</code></pre> <p>               Bases: <code>BaseBayesianModel</code></p> <p>Defines a <code>Wide</code> model. This is a linear model where the non-linearlities are captured via crossed-columns</p> <p>Parameters:</p> <ul> <li> <code>input_dim</code>               (<code>int</code>)           \u2013            <p>size of the Embedding layer. <code>input_dim</code> is the summation of all the individual values for all the features that go through the wide component. For example, if the wide component receives 2 features with 5 individual values each, <code>input_dim = 10</code></p> </li> <li> <code>pred_dim</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>size of the ouput tensor containing the predictions</p> </li> <li> <code>prior_sigma_1</code>               (<code>float</code>, default:                   <code>1.0</code> )           \u2013            <p>The prior weight distribution is a scaled mixture of two Gaussian densities:</p> \\[    \\begin{aligned}    P(\\mathbf{w}) = \\prod_{i=j} \\pi N (\\mathbf{w}_j | 0, \\sigma_{1}^{2}) + (1 - \\pi) N (\\mathbf{w}_j | 0, \\sigma_{2}^{2})    \\end{aligned} \\] <p><code>prior_sigma_1</code> is the prior of the sigma parameter for the first of the two Gaussians that will be mixed to produce the prior weight distribution.</p> </li> <li> <code>prior_sigma_2</code>               (<code>float</code>, default:                   <code>0.002</code> )           \u2013            <p>Prior of the sigma parameter for the second of the two Gaussian distributions that will be mixed to produce the prior weight distribution</p> </li> <li> <code>prior_pi</code>               (<code>float</code>, default:                   <code>0.8</code> )           \u2013            <p>Scaling factor that will be used to mix the Gaussians to produce the prior weight distribution</p> </li> <li> <code>posterior_mu_init</code>               (<code>float</code>, default:                   <code>0.0</code> )           \u2013            <p>The posterior sample of the weights is defined as:</p> \\[    \\begin{aligned}    \\mathbf{w} &amp;= \\mu + log(1 + exp(\\rho))    \\end{aligned} \\] <p>where:</p> \\[    \\begin{aligned}    \\mathcal{N}(x\\vert \\mu, \\sigma) &amp;= \\frac{1}{\\sqrt{2\\pi}\\sigma}e^{-\\frac{(x-\\mu)^2}{2\\sigma^2}}\\\\    \\log{\\mathcal{N}(x\\vert \\mu, \\sigma)} &amp;= -\\log{\\sqrt{2\\pi}} -\\log{\\sigma} -\\frac{(x-\\mu)^2}{2\\sigma^2}\\\\    \\end{aligned} \\] <p>\\(\\mu\\) is initialised using a normal distributtion with mean <code>posterior_mu_init</code> and std equal to 0.1.</p> </li> <li> <code>posterior_rho_init</code>               (<code>float</code>, default:                   <code>-7.0</code> )           \u2013            <p>As in the case of \\(\\mu\\), \\(\\rho\\) is initialised using a normal distributtion with mean <code>posterior_rho_init</code> and std equal to 0.1.</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>bayesian_wide_linear</code>               (<code>Module</code>)           \u2013            <p>the linear layer that comprises the wide branch of the model</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.bayesian_models import BayesianWide\n&gt;&gt;&gt; X = torch.empty(4, 4).random_(6)\n&gt;&gt;&gt; wide = BayesianWide(input_dim=int(X.max().item()), pred_dim=1)\n&gt;&gt;&gt; out = wide(X)\n</code></pre> Source code in <code>pytorch_widedeep/bayesian_models/tabular/bayesian_linear/bayesian_wide.py</code> <pre><code>def __init__(\n    self,\n    input_dim: int,\n    pred_dim: int = 1,\n    prior_sigma_1: float = 1.0,\n    prior_sigma_2: float = 0.002,\n    prior_pi: float = 0.8,\n    posterior_mu_init: float = 0.0,\n    posterior_rho_init: float = -7.0,\n):\n    super(BayesianWide, self).__init__()\n    #  Embeddings: val + 1 because 0 is reserved for padding/unseen cateogories.\n    self.bayesian_wide_linear = bnn.BayesianEmbedding(\n        n_embed=input_dim + 1,\n        embed_dim=pred_dim,\n        padding_idx=0,\n        prior_sigma_1=prior_sigma_1,\n        prior_sigma_2=prior_sigma_2,\n        prior_pi=prior_pi,\n        posterior_mu_init=posterior_mu_init,\n        posterior_rho_init=posterior_rho_init,\n    )\n    self.bias = nn.Parameter(torch.zeros(pred_dim))\n</code></pre>"},{"location":"pytorch-widedeep/bayesian_models.html#pytorch_widedeep.bayesian_models.tabular.bayesian_mlp.bayesian_tab_mlp.BayesianTabMlp","title":"BayesianTabMlp","text":"<pre><code>BayesianTabMlp(\n    column_idx,\n    *,\n    cat_embed_input=None,\n    cat_embed_activation=None,\n    continuous_cols=None,\n    embed_continuous=None,\n    cont_embed_dim=None,\n    cont_embed_dropout=None,\n    cont_embed_activation=None,\n    use_cont_bias=None,\n    cont_norm_layer=None,\n    mlp_hidden_dims=[200, 100],\n    mlp_activation=\"leaky_relu\",\n    prior_sigma_1=1,\n    prior_sigma_2=0.002,\n    prior_pi=0.8,\n    posterior_mu_init=0.0,\n    posterior_rho_init=-7.0,\n    pred_dim=1\n)\n</code></pre> <p>               Bases: <code>BaseBayesianModel</code></p> <p>Defines a <code>BayesianTabMlp</code> model.</p> <p>This class combines embedding representations of the categorical features with numerical (aka continuous) features, embedded or not. These are then passed through a series of probabilistic dense layers (i.e. a MLP).</p> <p>Parameters:</p> <ul> <li> <code>column_idx</code>               (<code>Dict[str, int]</code>)           \u2013            <p>Dict containing the index of the columns that will be passed through the <code>TabMlp</code> model. Required to slice the tensors. e.g. {'education': 0, 'relationship': 1, 'workclass': 2, ...}</p> </li> <li> <code>cat_embed_input</code>               (<code>Optional[List[Tuple[str, int, int]]]</code>, default:                   <code>None</code> )           \u2013            <p>List of Tuples with the column name, number of unique values and embedding dimension. e.g. [(education, 11, 32), ...]</p> </li> <li> <code>cat_embed_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the categorical embeddings, if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> </li> <li> <code>continuous_cols</code>               (<code>Optional[List[str]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the name of the numeric (aka continuous) columns</p> </li> <li> <code>cont_norm_layer</code>               (<code>Optional[Literal[batchnorm, layernorm]]</code>, default:                   <code>None</code> )           \u2013            <p>Type of normalization layer applied to the continuous features. Options are: 'layernorm', 'batchnorm' or None.</p> </li> <li> <code>embed_continuous</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating if the continuous columns will be embedded (i.e. passed each through a linear layer with or without activation)</p> </li> <li> <code>cont_embed_dim</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>Size of the continuous embeddings</p> </li> <li> <code>cont_embed_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Dropout for the continuous embeddings</p> </li> <li> <code>use_cont_bias</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating if bias will be used for the continuous embeddings</p> </li> <li> <code>cont_embed_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the continuous embeddings if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> </li> <li> <code>mlp_hidden_dims</code>               (<code>List[int]</code>, default:                   <code>[200, 100]</code> )           \u2013            <p>List with the number of neurons per dense layer in the mlp.</p> </li> <li> <code>mlp_activation</code>               (<code>str</code>, default:                   <code>'leaky_relu'</code> )           \u2013            <p>Activation function for the dense layers of the MLP. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> </li> <li> <code>prior_sigma_1</code>               (<code>float</code>, default:                   <code>1</code> )           \u2013            <p>The prior weight distribution is a scaled mixture of two Gaussian densities:</p> \\[    \\begin{aligned}    P(\\mathbf{w}) = \\prod_{i=j} \\pi N (\\mathbf{w}_j | 0, \\sigma_{1}^{2}) + (1 - \\pi) N (\\mathbf{w}_j | 0, \\sigma_{2}^{2})    \\end{aligned} \\] <p><code>prior_sigma_1</code> is the prior of the sigma parameter for the first of the two Gaussians that will be mixed to produce the prior weight distribution.</p> </li> <li> <code>prior_sigma_2</code>               (<code>float</code>, default:                   <code>0.002</code> )           \u2013            <p>Prior of the sigma parameter for the second of the two Gaussian distributions that will be mixed to produce the prior weight distribution for each Bayesian linear and embedding layer</p> </li> <li> <code>prior_pi</code>               (<code>float</code>, default:                   <code>0.8</code> )           \u2013            <p>Scaling factor that will be used to mix the Gaussians to produce the prior weight distribution ffor each Bayesian linear and embedding layer</p> </li> <li> <code>posterior_mu_init</code>               (<code>float</code>, default:                   <code>0.0</code> )           \u2013            <p>The posterior sample of the weights is defined as:</p> <p>$$    \\begin{aligned}    \\mathbf{w} &amp;= \\mu + log(1 + exp(\\rho))    \\end{aligned} $$ where:</p> \\[    \\begin{aligned}    \\mathcal{N}(x\\vert \\mu, \\sigma) &amp;= \\frac{1}{\\sqrt{2\\pi}\\sigma}e^{-\\frac{(x-\\mu)^2}{2\\sigma^2}}\\\\    \\log{\\mathcal{N}(x\\vert \\mu, \\sigma)} &amp;= -\\log{\\sqrt{2\\pi}} -\\log{\\sigma} -\\frac{(x-\\mu)^2}{2\\sigma^2}\\\\    \\end{aligned} \\] <p>\\(\\mu\\) is initialised using a normal distributtion with mean <code>posterior_mu_init</code> and std equal to 0.1.</p> </li> <li> <code>posterior_rho_init</code>               (<code>float</code>, default:                   <code>-7.0</code> )           \u2013            <p>As in the case of \\(\\mu\\), \\(\\rho\\) is initialised using a normal distributtion with mean <code>posterior_rho_init</code> and std equal to 0.1.</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>bayesian_cat_and_cont_embed</code>               (<code>Module</code>)           \u2013            <p>This is the module that processes the categorical and continuous columns</p> </li> <li> <code>bayesian_tab_mlp</code>               (<code>Sequential</code>)           \u2013            <p>mlp model that will receive the concatenation of the embeddings and the continuous columns</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.bayesian_models import BayesianTabMlp\n&gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n&gt;&gt;&gt; colnames = ['a', 'b', 'c', 'd', 'e']\n&gt;&gt;&gt; cat_embed_input = [(u,i,j) for u,i,j in zip(colnames[:4], [4]*4, [8]*4)]\n&gt;&gt;&gt; column_idx = {k:v for v,k in enumerate(colnames)}\n&gt;&gt;&gt; model = BayesianTabMlp(mlp_hidden_dims=[8,4], column_idx=column_idx, cat_embed_input=cat_embed_input,\n... continuous_cols = ['e'])\n&gt;&gt;&gt; out = model(X_tab)\n</code></pre> Source code in <code>pytorch_widedeep/bayesian_models/tabular/bayesian_mlp/bayesian_tab_mlp.py</code> <pre><code>def __init__(\n    self,\n    column_idx: Dict[str, int],\n    *,\n    cat_embed_input: Optional[List[Tuple[str, int, int]]] = None,\n    cat_embed_activation: Optional[str] = None,\n    continuous_cols: Optional[List[str]] = None,\n    embed_continuous: Optional[bool] = None,\n    cont_embed_dim: Optional[int] = None,\n    cont_embed_dropout: Optional[float] = None,\n    cont_embed_activation: Optional[str] = None,\n    use_cont_bias: Optional[bool] = None,\n    cont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\n    mlp_hidden_dims: List[int] = [200, 100],\n    mlp_activation: str = \"leaky_relu\",\n    prior_sigma_1: float = 1,\n    prior_sigma_2: float = 0.002,\n    prior_pi: float = 0.8,\n    posterior_mu_init: float = 0.0,\n    posterior_rho_init: float = -7.0,\n    pred_dim=1,  # Bayesian models will require their own trainer and need the output layer\n):\n    super(BayesianTabMlp, self).__init__()\n\n    self.column_idx = column_idx\n    self.cat_embed_input = cat_embed_input\n    self.cat_embed_activation = cat_embed_activation\n\n    self.continuous_cols = continuous_cols\n    self.cont_norm_layer = cont_norm_layer\n    self.embed_continuous = embed_continuous\n    self.cont_embed_dim = cont_embed_dim\n    self.cont_embed_dropout = cont_embed_dropout\n    self.use_cont_bias = use_cont_bias\n    self.cont_embed_activation = cont_embed_activation\n\n    self.mlp_hidden_dims = mlp_hidden_dims\n    self.mlp_activation = mlp_activation\n\n    self.prior_sigma_1 = prior_sigma_1\n    self.prior_sigma_2 = prior_sigma_2\n    self.prior_pi = prior_pi\n    self.posterior_mu_init = posterior_mu_init\n    self.posterior_rho_init = posterior_rho_init\n\n    self.pred_dim = pred_dim\n\n    allowed_activations = [\"relu\", \"leaky_relu\", \"tanh\", \"gelu\"]\n    if self.mlp_activation not in allowed_activations:\n        raise ValueError(\n            \"Currently, only the following activation functions are supported \"\n            \"for the Bayesian MLP's dense layers: {}. Got '{}' instead\".format(\n                \", \".join(allowed_activations),\n                self.mlp_activation,\n            )\n        )\n\n    # Categorical\n    if self.cat_embed_input is not None:\n        self.cat_embed = BayesianDiffSizeCatEmbeddings(\n            column_idx=self.column_idx,\n            embed_input=self.cat_embed_input,\n            prior_sigma_1=self.prior_sigma_1,\n            prior_sigma_2=self.prior_sigma_2,\n            prior_pi=self.prior_pi,\n            posterior_mu_init=self.posterior_mu_init,\n            posterior_rho_init=self.posterior_rho_init,\n            activation_fn=self.cat_embed_activation,\n        )\n        self.cat_out_dim = int(np.sum([embed[2] for embed in self.cat_embed_input]))\n    else:\n        self.cat_out_dim = 0\n\n    # Continuous\n    if self.continuous_cols is not None:\n        self.cont_idx = [self.column_idx[col] for col in self.continuous_cols]\n        if cont_norm_layer == \"layernorm\":\n            self.cont_norm: NormLayers = nn.LayerNorm(len(self.continuous_cols))\n        elif cont_norm_layer == \"batchnorm\":\n            self.cont_norm = nn.BatchNorm1d(len(self.continuous_cols))\n        else:\n            self.cont_norm = nn.Identity()\n        if self.embed_continuous:\n            assert self.cont_embed_dim is not None, (\n                \"If 'embed_continuous' is True, 'cont_embed_dim' must be \"\n                \"provided\"\n            )\n            self.cont_embed = BayesianContEmbeddings(\n                n_cont_cols=len(self.continuous_cols),\n                embed_dim=self.cont_embed_dim,\n                prior_sigma_1=self.prior_sigma_1,\n                prior_sigma_2=self.prior_sigma_2,\n                prior_pi=self.prior_pi,\n                posterior_mu_init=self.posterior_mu_init,\n                posterior_rho_init=self.posterior_rho_init,\n                use_bias=(\n                    False if self.use_cont_bias is None else self.use_cont_bias\n                ),\n                activation_fn=self.cont_embed_activation,\n            )\n            self.cont_out_dim = len(self.continuous_cols) * self.cont_embed_dim\n        else:\n            self.cont_out_dim = len(self.continuous_cols)\n    else:\n        self.cont_out_dim = 0\n\n    self.output_dim = self.cat_out_dim + self.cont_out_dim\n\n    mlp_hidden_dims = [self.output_dim] + mlp_hidden_dims + [pred_dim]\n    self.bayesian_tab_mlp = BayesianMLP(\n        mlp_hidden_dims,\n        mlp_activation,\n        True,  # use_bias\n        prior_sigma_1,\n        prior_sigma_2,\n        prior_pi,\n        posterior_mu_init,\n        posterior_rho_init,\n    )\n</code></pre>"},{"location":"pytorch-widedeep/bayesian_trainer.html","title":"Training Deep Learning Probabilistic Models","text":""},{"location":"pytorch-widedeep/bayesian_trainer.html#pytorch_widedeep.training.BayesianTrainer","title":"BayesianTrainer","text":"<pre><code>BayesianTrainer(\n    model,\n    objective,\n    custom_loss_function=None,\n    optimizer=None,\n    lr_scheduler=None,\n    callbacks=None,\n    metrics=None,\n    verbose=1,\n    seed=1,\n    **kwargs\n)\n</code></pre> <p>               Bases: <code>BaseBayesianTrainer</code></p> <p>Class to set the of attributes that will be used during the training process.</p> <p>Both the Bayesian models and the Trainer in this repo are based on the paper: Weight Uncertainty in Neural Networks.</p> <p>Parameters:</p> <ul> <li> <code>model</code>               (<code>BaseBayesianModel</code>)           \u2013            <p>An object of class <code>BaseBayesianModel</code>. See the <code>Model Components</code> section here in the docs.</p> </li> <li> <code>objective</code>               (<code>str</code>)           \u2013            <p>Defines the objective, loss or cost function. Param aliases: <code>loss_function</code>, <code>loss_fn</code>, <code>loss</code>, <code>cost_function</code>, <code>cost_fn</code>, <code>cost</code> Possible values are: 'binary', 'multiclass', 'regression'</p> </li> <li> <code>custom_loss_function</code>               (<code>Optional[Module]</code>, default:                   <code>None</code> )           \u2013            <p>If none of the loss functions available suits the user, it is possible to pass a custom loss function. See for example <code>pytorch_widedeep.losses.FocalLoss</code> for the required structure of the object or the Examples folder in the repo.</p> </li> <li> <code>optimizer</code>               (<code>Optional[Optimizer]</code>, default:                   <code>None</code> )           \u2013            <p>An instance of Pytorch's <code>Optimizer</code> object(e.g. <code>torch.optim.Adam ()</code>). if no optimizer is passed it will default to <code>AdamW</code>.</p> </li> <li> <code>lr_scheduler</code>               (<code>Optional[LRScheduler]</code>, default:                   <code>None</code> )           \u2013            <p>An instance of Pytorch's <code>LRScheduler</code> object (e.g <code>torch.optim.lr_scheduler.StepLR(opt, step_size=5)</code>).</p> </li> <li> <code>callbacks</code>               (<code>Optional[List[Callback]]</code>, default:                   <code>None</code> )           \u2013            <p>List with <code>Callback</code> objects. The three callbacks available in <code>pytorch-widedeep</code> are: <code>LRHistory</code>, <code>ModelCheckpoint</code> and <code>EarlyStopping</code>. This can also be a custom callback. See <code>pytorch_widedeep.callbacks.Callback</code> or the Examples folder in the repo.</p> </li> <li> <code>metrics</code>               (<code>Optional[Union[List[Metric], List[Metric]]]</code>, default:                   <code>None</code> )           \u2013            <ul> <li>List of objects of type <code>Metric</code>. Metrics available are:   <code>Accuracy</code>, <code>Precision</code>, <code>Recall</code>, <code>FBetaScore</code>,   <code>F1Score</code> and <code>R2Score</code>. This can also be a custom metric as   long as it is an object of type <code>Metric</code>. See   <code>pytorch_widedeep.metrics.Metric</code> or the Examples folder in the repo</li> <li>List of objects of type <code>torchmetrics.Metric</code>. This can be any   metric from torchmetrics library Examples   classification-metrics&gt;<code>_. It can also be a torchmetric custom metric as   long as it is an object of type</code>Metric<code>.   See</code>the instructions</li> </ul> </li> <li> <code>verbose</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>Setting it to 0 will print nothing during training.</p> </li> <li> <code>seed</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>Random seed to be used internally for train_test_split</p> </li> </ul> <p>Other Parameters:</p> <ul> <li> <code>**kwargs</code>           \u2013            <p>Other infrequently used arguments that can also be passed as kwargs are:</p> <ul> <li> <p>device: <code>str</code>     string indicating the device. One of 'cpu' or 'gpu'</p> </li> <li> <p>num_workers: <code>int</code>     number of workers to be used internally by the data loaders</p> </li> <li> <p>class_weight: <code>List[float]</code>     This is the <code>weight</code> or <code>pos_weight</code> parameter in     <code>CrossEntropyLoss</code> and <code>BCEWithLogitsLoss</code>, depending on whether</p> </li> <li> <p>reducelronplateau_criterion: <code>str</code>     This sets the criterion that will be used by the lr scheduler to     take a step: One of 'loss' or 'metric'. The ReduceLROnPlateau     learning rate is a bit particular.</p> </li> </ul> </li> </ul> <p>Attributes:</p> <ul> <li> <code>cyclic_lr</code>               (<code>bool</code>)           \u2013            <p>Attribute that indicates if  the lr_scheduler is cyclic_lr (i.e. <code>CyclicLR</code> or <code>OneCycleLR</code>). See <code>Pytorch schedulers &lt;https://pytorch.org/docs/stable/optim.html&gt;</code>_.</p> </li> </ul> Source code in <code>pytorch_widedeep/training/bayesian_trainer.py</code> <pre><code>@alias(  # noqa: C901\n    \"objective\",\n    [\"loss_function\", \"loss_fn\", \"loss\", \"cost_function\", \"cost_fn\", \"cost\"],\n)\ndef __init__(\n    self,\n    model: BaseBayesianModel,\n    objective: str,\n    custom_loss_function: Optional[Module] = None,\n    optimizer: Optional[Optimizer] = None,\n    lr_scheduler: Optional[LRScheduler] = None,\n    callbacks: Optional[List[Callback]] = None,\n    metrics: Optional[Union[List[Metric], List[TorchMetric]]] = None,\n    verbose: int = 1,\n    seed: int = 1,\n    **kwargs,\n):\n    super().__init__(\n        model=model,\n        objective=objective,\n        custom_loss_function=custom_loss_function,\n        optimizer=optimizer,\n        lr_scheduler=lr_scheduler,\n        callbacks=callbacks,\n        metrics=metrics,\n        verbose=verbose,\n        seed=seed,\n        **kwargs,\n    )\n</code></pre>"},{"location":"pytorch-widedeep/bayesian_trainer.html#pytorch_widedeep.training.BayesianTrainer.fit","title":"fit","text":"<pre><code>fit(\n    X_tab,\n    target,\n    X_tab_val=None,\n    target_val=None,\n    val_split=None,\n    n_epochs=1,\n    validation_freq=1,\n    batch_size=32,\n    n_train_samples=2,\n    n_val_samples=2,\n)\n</code></pre> <p>Fit method.</p> <p>Parameters:</p> <ul> <li> <code>X_tab</code>               (<code>ndarray</code>)           \u2013            <p>tabular dataset</p> </li> <li> <code>target</code>               (<code>ndarray</code>)           \u2013            <p>target values</p> </li> <li> <code>X_tab_val</code>               (<code>Optional[ndarray]</code>, default:                   <code>None</code> )           \u2013            <p>validation data</p> </li> <li> <code>target_val</code>               (<code>Optional[ndarray]</code>, default:                   <code>None</code> )           \u2013            <p>validation target values</p> </li> <li> <code>val_split</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>An alterative to passing the validation set is to use a train/val split fraction via <code>val_split</code></p> </li> <li> <code>n_epochs</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>number of epochs</p> </li> <li> <code>validation_freq</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>epochs validation frequency</p> </li> <li> <code>batch_size</code>               (<code>int</code>, default:                   <code>32</code> )           \u2013            <p>batch size</p> </li> <li> <code>n_train_samples</code>               (<code>int</code>, default:                   <code>2</code> )           \u2013            <p>number of samples to average over during the training process. See Weight Uncertainty in Neural Networks for details.</p> </li> <li> <code>n_val_samples</code>               (<code>int</code>, default:                   <code>2</code> )           \u2013            <p>number of samples to average over during the validation process. See Weight Uncertainty in Neural Networks for details.</p> </li> </ul> Source code in <code>pytorch_widedeep/training/bayesian_trainer.py</code> <pre><code>def fit(  # noqa: C901\n    self,\n    X_tab: np.ndarray,\n    target: np.ndarray,\n    X_tab_val: Optional[np.ndarray] = None,\n    target_val: Optional[np.ndarray] = None,\n    val_split: Optional[float] = None,\n    n_epochs: int = 1,\n    validation_freq: int = 1,\n    batch_size: int = 32,\n    n_train_samples: int = 2,\n    n_val_samples: int = 2,\n):\n    r\"\"\"Fit method.\n\n    Parameters\n    ----------\n    X_tab: np.ndarray,\n        tabular dataset\n    target: np.ndarray\n        target values\n    X_tab_val: np.ndarray, Optional, default = None\n        validation data\n    target_val: np.ndarray, Optional, default = None\n        validation target values\n    val_split: float, Optional. default=None\n        An alterative to passing the validation set is to use a train/val\n        split fraction via `val_split`\n    n_epochs: int, default=1\n        number of epochs\n    validation_freq: int, default=1\n        epochs validation frequency\n    batch_size: int, default=32\n        batch size\n    n_train_samples: int, default=2\n        number of samples to average over during the training process.\n        See [Weight Uncertainty in Neural Networks](https://arxiv.org/pdf/1505.05424.pdf) for details.\n    n_val_samples: int, default=2\n        number of samples to average over during the validation process.\n        See [Weight Uncertainty in Neural Networks](https://arxiv.org/pdf/1505.05424.pdf) for details.\n    \"\"\"\n\n    self.batch_size = batch_size\n\n    train_set, eval_set = tabular_train_val_split(\n        self.seed, self.objective, X_tab, target, X_tab_val, target_val, val_split\n    )\n    train_loader = DataLoader(\n        dataset=train_set, batch_size=batch_size, num_workers=self.num_workers\n    )\n    train_steps = len(train_loader)\n\n    if eval_set is not None:\n        eval_loader = DataLoader(\n            dataset=eval_set,\n            batch_size=batch_size,\n            num_workers=self.num_workers,\n            shuffle=False,\n        )\n        eval_steps = len(eval_loader)\n\n    self.callback_container.on_train_begin(\n        {\n            \"batch_size\": batch_size,\n            \"train_steps\": train_steps,\n            \"n_epochs\": n_epochs,\n        }\n    )\n    for epoch in range(n_epochs):\n        epoch_logs: Dict[str, float] = {}\n        self.callback_container.on_epoch_begin(epoch, logs=epoch_logs)\n\n        self.train_running_loss = 0.0\n        with trange(train_steps, disable=self.verbose != 1) as t:\n            for batch_idx, (X, y) in zip(t, train_loader):\n                t.set_description(\"epoch %i\" % (epoch + 1))\n                train_score, train_loss = self._train_step(\n                    X, y, n_train_samples, train_steps, batch_idx\n                )\n                print_loss_and_metric(t, train_loss, train_score)\n                self.callback_container.on_batch_end(batch=batch_idx)\n        epoch_logs = save_epoch_logs(epoch_logs, train_loss, train_score, \"train\")\n\n        on_epoch_end_metric = None\n        if eval_set is not None and epoch % validation_freq == (\n            validation_freq - 1\n        ):\n            self.callback_container.on_eval_begin()\n            self.valid_running_loss = 0.0\n            with trange(eval_steps, disable=self.verbose != 1) as v:\n                for i, (X, y) in zip(v, eval_loader):\n                    v.set_description(\"valid\")\n                    val_score, val_loss = self._eval_step(\n                        X, y, n_val_samples, train_steps, i\n                    )\n                    print_loss_and_metric(v, val_loss, val_score)\n            epoch_logs = save_epoch_logs(epoch_logs, val_loss, val_score, \"val\")\n\n            if self.reducelronplateau:\n                if self.reducelronplateau_criterion == \"loss\":\n                    on_epoch_end_metric = val_loss\n                else:\n                    on_epoch_end_metric = val_score[\n                        self.reducelronplateau_criterion\n                    ]\n\n        self.callback_container.on_epoch_end(epoch, epoch_logs, on_epoch_end_metric)\n\n        if self.early_stop:\n            self.callback_container.on_train_end(epoch_logs)\n            break\n\n    self.callback_container.on_train_end(epoch_logs)\n    self._restore_best_weights()\n    self.model.train()\n</code></pre>"},{"location":"pytorch-widedeep/bayesian_trainer.html#pytorch_widedeep.training.BayesianTrainer.predict","title":"predict","text":"<pre><code>predict(\n    X_tab, n_samples=5, return_samples=False, batch_size=256\n)\n</code></pre> <p>Returns the predictions</p> <p>Parameters:</p> <ul> <li> <code>X_tab</code>               (<code>ndarray</code>)           \u2013            <p>tabular dataset</p> </li> <li> <code>n_samples</code>               (<code>int</code>, default:                   <code>5</code> )           \u2013            <p>number of samples that will be either returned or averaged to produce an overal prediction</p> </li> <li> <code>return_samples</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether the n samples will be averaged or directly returned</p> </li> <li> <code>batch_size</code>               (<code>int</code>, default:                   <code>256</code> )           \u2013            <p>batch size</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>np.ndarray:</code>           \u2013            <p>array with the predictions</p> </li> </ul> Source code in <code>pytorch_widedeep/training/bayesian_trainer.py</code> <pre><code>def predict(  # type: ignore[return]\n    self,\n    X_tab: np.ndarray,\n    n_samples: int = 5,\n    return_samples: bool = False,\n    batch_size: int = 256,\n) -&gt; np.ndarray:\n    r\"\"\"Returns the predictions\n\n    Parameters\n    ----------\n    X_tab: np.ndarray,\n        tabular dataset\n    n_samples: int, default=5\n        number of samples that will be either returned or averaged to\n        produce an overal prediction\n    return_samples: bool, default = False\n        Boolean indicating whether the n samples will be averaged or directly returned\n    batch_size: int, default = 256\n        batch size\n\n    Returns\n    -------\n    np.ndarray:\n        array with the predictions\n    \"\"\"\n\n    preds_l = self._predict(X_tab, n_samples, return_samples, batch_size)\n    preds = np.hstack(preds_l) if return_samples else np.vstack(preds_l)\n    axis = 2 if return_samples else 1\n\n    if self.objective == \"regression\":\n        return preds.squeeze(axis)\n    if self.objective == \"binary\":\n        return (preds.squeeze(axis) &gt; 0.5).astype(\"int\")\n    if self.objective == \"multiclass\":\n        return np.argmax(preds, axis)\n</code></pre>"},{"location":"pytorch-widedeep/bayesian_trainer.html#pytorch_widedeep.training.BayesianTrainer.predict_proba","title":"predict_proba","text":"<pre><code>predict_proba(\n    X_tab, n_samples=5, return_samples=False, batch_size=256\n)\n</code></pre> <p>Returns the predicted probabilities</p> <p>Parameters:</p> <ul> <li> <code>X_tab</code>               (<code>ndarray</code>)           \u2013            <p>tabular dataset</p> </li> <li> <code>n_samples</code>               (<code>int</code>, default:                   <code>5</code> )           \u2013            <p>number of samples that will be either returned or averaged to produce an overal prediction</p> </li> <li> <code>return_samples</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether the n samples will be averaged or directly returned</p> </li> <li> <code>batch_size</code>               (<code>int</code>, default:                   <code>256</code> )           \u2013            <p>batch size</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>ndarray</code>           \u2013            <p>array with the probabilities per class</p> </li> </ul> Source code in <code>pytorch_widedeep/training/bayesian_trainer.py</code> <pre><code>def predict_proba(  # type: ignore[return]\n    self,\n    X_tab: np.ndarray,\n    n_samples: int = 5,\n    return_samples: bool = False,\n    batch_size: int = 256,\n) -&gt; np.ndarray:\n    r\"\"\"Returns the predicted probabilities\n\n    Parameters\n    ----------\n    X_tab: np.ndarray,\n        tabular dataset\n    n_samples: int, default=5\n        number of samples that will be either returned or averaged to\n        produce an overal prediction\n    return_samples: bool, default = False\n        Boolean indicating whether the n samples will be averaged or directly returned\n    batch_size: int, default = 256\n        batch size\n\n    Returns\n    -------\n    np.ndarray\n        array with the probabilities per class\n    \"\"\"\n    preds_l = self._predict(X_tab, n_samples, return_samples, batch_size)\n    preds = np.hstack(preds_l) if return_samples else np.vstack(preds_l)\n\n    if self.objective == \"binary\":\n        if return_samples:\n            preds = preds.squeeze(2)\n            probs = np.zeros([n_samples, preds.shape[1], 2])\n            for i in range(n_samples):\n                probs[i, :, 0] = 1 - preds[i]\n                probs[i, :, 1] = preds[i]\n        else:\n            preds = preds.squeeze(1)\n            probs = np.zeros([preds.shape[0], 2])\n            probs[:, 0] = 1 - preds\n            probs[:, 1] = preds\n        return probs\n    if self.objective == \"multiclass\":\n        return preds\n</code></pre>"},{"location":"pytorch-widedeep/bayesian_trainer.html#pytorch_widedeep.training.BayesianTrainer.save","title":"save","text":"<pre><code>save(\n    path,\n    save_state_dict=False,\n    model_filename=\"bayesian_model.pt\",\n)\n</code></pre> <p>Saves the model, training and evaluation history to disk</p> <p>The <code>Trainer</code> class is built so that it 'just' trains a model. With that in mind, all the torch related parameters (such as optimizers or learning rate schedulers) have to be defined externally and then passed to the <code>Trainer</code>. As a result, the <code>Trainer</code> does not generate any attribute or additional data products that need to be saved other than the <code>model</code> object itself, which can be saved as any other torch model (e.g. <code>torch.save(model, path)</code>).</p> <p>Parameters:</p> <ul> <li> <code>path</code>               (<code>str</code>)           \u2013            <p>path to the directory where the model and the feature importance attribute will be saved.</p> </li> <li> <code>save_state_dict</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether to save directly the model or the model's state dictionary</p> </li> <li> <code>model_filename</code>               (<code>str</code>, default:                   <code>'bayesian_model.pt'</code> )           \u2013            <p>filename where the model weights will be store</p> </li> </ul> Source code in <code>pytorch_widedeep/training/bayesian_trainer.py</code> <pre><code>def save(\n    self,\n    path: str,\n    save_state_dict: bool = False,\n    model_filename: str = \"bayesian_model.pt\",\n):\n    r\"\"\"Saves the model, training and evaluation history to disk\n\n    The `Trainer` class is built so that it 'just' trains a model. With\n    that in mind, all the torch related parameters (such as optimizers or\n    learning rate schedulers) have to be defined externally and then\n    passed to the `Trainer`. As a result, the `Trainer` does not\n    generate any attribute or additional data products that need to be\n    saved other than the `model` object itself, which can be saved as\n    any other torch model (e.g. `torch.save(model, path)`).\n\n    Parameters\n    ----------\n    path: str\n        path to the directory where the model and the feature importance\n        attribute will be saved.\n    save_state_dict: bool, default = False\n        Boolean indicating whether to save directly the model or the\n        model's state dictionary\n    model_filename: str, Optional, default = \"wd_model.pt\"\n        filename where the model weights will be store\n    \"\"\"\n\n    save_dir = Path(path)\n    history_dir = save_dir / \"history\"\n    history_dir.mkdir(exist_ok=True, parents=True)\n\n    # the trainer is run with the History Callback by default\n    with open(history_dir / \"train_eval_history.json\", \"w\") as teh:\n        json.dump(self.history, teh)  # type: ignore[attr-defined]\n\n    has_lr_history = any(\n        [clbk.__class__.__name__ == \"LRHistory\" for clbk in self.callbacks]\n    )\n    if self.lr_scheduler is not None and has_lr_history:\n        with open(history_dir / \"lr_history.json\", \"w\") as lrh:\n            json.dump(self.lr_history, lrh)  # type: ignore[attr-defined]\n\n    model_path = save_dir / model_filename\n    if save_state_dict:\n        torch.save(self.model.state_dict(), model_path)\n    else:\n        torch.save(self.model, model_path)\n</code></pre>"},{"location":"pytorch-widedeep/callbacks.html","title":"Callbacks","text":"<p>Here are the 4 callbacks available to the user in <code>pytorch-widedepp</code>: <code>LRHistory</code>, <code>ModelCheckpoint</code>, <code>EarlyStopping</code> and <code>RayTuneReporter</code>.</p> <p> NOTE: other callbacks , like <code>History</code>, run always  by default. In particular, the <code>History</code> callback saves the metrics in the  <code>history</code> attribute of the <code>Trainer</code>.</p>"},{"location":"pytorch-widedeep/callbacks.html#pytorch_widedeep.callbacks.LRHistory","title":"LRHistory","text":"<pre><code>LRHistory(n_epochs)\n</code></pre> <p>               Bases: <code>Callback</code></p> <p>Saves the learning rates during training in the <code>lr_history</code> attribute of the <code>Trainer</code>.</p> <p>Callbacks are passed as input parameters to the <code>Trainer</code> class. See <code>pytorch_widedeep.trainer.Trainer</code></p> <p>Parameters:</p> <ul> <li> <code>n_epochs</code>               (<code>int</code>)           \u2013            <p>number of training epochs</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; from pytorch_widedeep.callbacks import LRHistory\n&gt;&gt;&gt; from pytorch_widedeep.models import TabMlp, Wide, WideDeep\n&gt;&gt;&gt; from pytorch_widedeep.training import Trainer\n&gt;&gt;&gt;\n&gt;&gt;&gt; embed_input = [(u, i, j) for u, i, j in zip([\"a\", \"b\", \"c\"][:4], [4] * 3, [8] * 3)]\n&gt;&gt;&gt; column_idx = {k: v for v, k in enumerate([\"a\", \"b\", \"c\"])}\n&gt;&gt;&gt; wide = Wide(10, 1)\n&gt;&gt;&gt; deep = TabMlp(mlp_hidden_dims=[8, 4], column_idx=column_idx, cat_embed_input=embed_input)\n&gt;&gt;&gt; model = WideDeep(wide, deep)\n&gt;&gt;&gt; trainer = Trainer(model, objective=\"regression\", callbacks=[LRHistory(n_epochs=10)])\n</code></pre> Source code in <code>pytorch_widedeep/callbacks.py</code> <pre><code>def __init__(self, n_epochs: int):\n    super(LRHistory, self).__init__()\n    self.n_epochs = n_epochs\n</code></pre>"},{"location":"pytorch-widedeep/callbacks.html#pytorch_widedeep.callbacks.ModelCheckpoint","title":"ModelCheckpoint","text":"<pre><code>ModelCheckpoint(\n    filepath=None,\n    monitor=\"val_loss\",\n    min_delta=0.0,\n    verbose=0,\n    save_best_only=False,\n    mode=\"auto\",\n    period=1,\n    max_save=-1,\n)\n</code></pre> <p>               Bases: <code>Callback</code></p> <p>Saves the model after every epoch.</p> <p>This class is almost identical to the corresponding keras class. Therefore, credit to the Keras Team.</p> <p>Callbacks are passed as input parameters to the <code>Trainer</code> class. See <code>pytorch_widedeep.trainer.Trainer</code></p> <p>Parameters:</p> <ul> <li> <code>filepath</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Full path to save the output weights. It must contain only the root of the filenames. Epoch number and <code>.pt</code> extension (for pytorch) will be added. e.g. <code>filepath=\"path/to/output_weights/weights_out\"</code> And the saved files in that directory will be named: 'weights_out_1.pt', 'weights_out_2.pt', .... If set to <code>None</code> the class just report best metric and best_epoch.</p> </li> <li> <code>monitor</code>               (<code>str</code>, default:                   <code>'val_loss'</code> )           \u2013            <p>quantity to monitor. Typically 'val_loss' or metric name (e.g. 'val_acc')</p> </li> <li> <code>min_delta</code>               (<code>float</code>, default:                   <code>0.0</code> )           \u2013            <p>minimum change in the monitored quantity to qualify as an improvement, i.e. an absolute change of less than min_delta, will count as no improvement.</p> </li> <li> <code>verbose</code>               (<code>int</code>, default:                   <code>0</code> )           \u2013            <p>verbosity mode</p> </li> <li> <code>save_best_only</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>the latest best model according to the quantity monitored will not be overwritten.</p> </li> <li> <code>mode</code>               (<code>str</code>, default:                   <code>'auto'</code> )           \u2013            <p>If <code>save_best_only=True</code>, the decision to overwrite the current save file is made based on either the maximization or the minimization of the monitored quantity. For 'acc', this should be 'max', for 'loss' this should be 'min', etc. In 'auto' mode, the direction is automatically inferred from the name of the monitored quantity.</p> </li> <li> <code>period</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>Interval (number of epochs) between checkpoints.</p> </li> <li> <code>max_save</code>               (<code>int</code>, default:                   <code>-1</code> )           \u2013            <p>Maximum number of outputs to save. If -1 will save all outputs</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>best</code>               (<code>float</code>)           \u2013            <p>best metric</p> </li> <li> <code>best_epoch</code>               (<code>int</code>)           \u2013            <p>best epoch</p> </li> <li> <code>best_state_dict</code>               (<code>dict</code>)           \u2013            <p>best model state dictionary. To restore model to its best state use <code>Trainer.model.load_state_dict (model_checkpoint.best_state_dict)</code> where <code>model_checkpoint</code> is an instance of the class <code>ModelCheckpoint</code>. See the Examples folder in the repo or the Examples section in this documentation for details</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; from pytorch_widedeep.callbacks import ModelCheckpoint\n&gt;&gt;&gt; from pytorch_widedeep.models import TabMlp, Wide, WideDeep\n&gt;&gt;&gt; from pytorch_widedeep.training import Trainer\n&gt;&gt;&gt;\n&gt;&gt;&gt; embed_input = [(u, i, j) for u, i, j in zip([\"a\", \"b\", \"c\"][:4], [4] * 3, [8] * 3)]\n&gt;&gt;&gt; column_idx = {k: v for v, k in enumerate([\"a\", \"b\", \"c\"])}\n&gt;&gt;&gt; wide = Wide(10, 1)\n&gt;&gt;&gt; deep = TabMlp(mlp_hidden_dims=[8, 4], column_idx=column_idx, cat_embed_input=embed_input)\n&gt;&gt;&gt; model = WideDeep(wide, deep)\n&gt;&gt;&gt; trainer = Trainer(model, objective=\"regression\", callbacks=[ModelCheckpoint(filepath='checkpoints/weights_out')])\n</code></pre> Source code in <code>pytorch_widedeep/callbacks.py</code> <pre><code>def __init__(\n    self,\n    filepath: Optional[str] = None,\n    monitor: str = \"val_loss\",\n    min_delta: float = 0.0,\n    verbose: int = 0,\n    save_best_only: bool = False,\n    mode: str = \"auto\",\n    period: int = 1,\n    max_save: int = -1,\n):\n    super(ModelCheckpoint, self).__init__()\n\n    self.filepath = filepath\n    self.monitor = monitor\n    self.min_delta = min_delta\n    self.verbose = verbose\n    self.save_best_only = save_best_only\n    self.mode = mode\n    self.period = period\n    self.max_save = max_save\n\n    self.epochs_since_last_save = 0\n\n    if self.filepath:\n        if len(self.filepath.split(\"/\")[:-1]) == 0:\n            raise ValueError(\n                \"'filepath' must be the full path to save the output weights,\"\n                \" including the root of the filenames. e.g. 'checkpoints/weights_out'\"\n            )\n\n        root_dir = (\"/\").join(self.filepath.split(\"/\")[:-1])\n        if not os.path.exists(root_dir):\n            os.makedirs(root_dir)\n\n    if self.max_save &gt; 0:\n        self.old_files: List[str] = []\n\n    if self.mode not in [\"auto\", \"min\", \"max\"]:\n        warnings.warn(\n            \"ModelCheckpoint mode %s is unknown, \"\n            \"fallback to auto mode.\" % (self.mode),\n            RuntimeWarning,\n        )\n        self.mode = \"auto\"\n    if self.mode == \"min\":\n        self.monitor_op = np.less\n        self.best = np.Inf\n    elif self.mode == \"max\":\n        self.monitor_op = np.greater  # type: ignore[assignment]\n        self.best = -np.Inf\n    else:\n        if _is_metric(self.monitor):\n            self.monitor_op = np.greater  # type: ignore[assignment]\n            self.best = -np.Inf\n        else:\n            self.monitor_op = np.less\n            self.best = np.Inf\n\n    if self.monitor_op == np.greater:\n        self.min_delta *= 1\n    else:\n        self.min_delta *= -1\n</code></pre>"},{"location":"pytorch-widedeep/callbacks.html#pytorch_widedeep.callbacks.EarlyStopping","title":"EarlyStopping","text":"<pre><code>EarlyStopping(\n    monitor=\"val_loss\",\n    min_delta=0.0,\n    patience=10,\n    verbose=0,\n    mode=\"auto\",\n    baseline=None,\n    restore_best_weights=False,\n)\n</code></pre> <p>               Bases: <code>Callback</code></p> <p>Stop training when a monitored quantity has stopped improving.</p> <p>This class is almost identical to the corresponding keras class. Therefore, credit to the Keras Team.</p> <p>Callbacks are passed as input parameters to the <code>Trainer</code> class. See <code>pytorch_widedeep.trainer.Trainer</code></p> <p>Parameters:</p> <ul> <li> <code>monitor</code>               (<code>str</code>, default:                   <code>'val_loss'</code> )           \u2013            <p>Quantity to monitor. Typically 'val_loss' or metric name (e.g. 'val_acc')</p> </li> <li> <code>min_delta</code>               (<code>float</code>, default:                   <code>0.0</code> )           \u2013            <p>minimum change in the monitored quantity to qualify as an improvement, i.e. an absolute change of less than min_delta, will count as no improvement.</p> </li> <li> <code>patience</code>               (<code>int</code>, default:                   <code>10</code> )           \u2013            <p>Number of epochs that produced the monitored quantity with no improvement after which training will be stopped.</p> </li> <li> <code>verbose</code>               (<code>int</code>, default:                   <code>0</code> )           \u2013            <p>verbosity mode.</p> </li> <li> <code>mode</code>               (<code>str</code>, default:                   <code>'auto'</code> )           \u2013            <p>one of {'auto', 'min', 'max'}. In 'min' mode, training will stop when the quantity monitored has stopped decreasing; in 'max' mode it will stop when the quantity monitored has stopped increasing; in 'auto' mode, the direction is automatically inferred from the name of the monitored quantity.</p> </li> <li> <code>baseline</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Baseline value for the monitored quantity to reach. Training will stop if the model does not show improvement over the baseline.</p> </li> <li> <code>restore_best_weights</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Whether to restore model weights from the epoch with the best value of the monitored quantity. If <code>False</code>, the model weights obtained at the last step of training are used.</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>best</code>               (<code>float</code>)           \u2013            <p>best metric</p> </li> <li> <code>stopped_epoch</code>               (<code>int</code>)           \u2013            <p>epoch when the training stopped</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; from pytorch_widedeep.callbacks import EarlyStopping\n&gt;&gt;&gt; from pytorch_widedeep.models import TabMlp, Wide, WideDeep\n&gt;&gt;&gt; from pytorch_widedeep.training import Trainer\n&gt;&gt;&gt;\n&gt;&gt;&gt; embed_input = [(u, i, j) for u, i, j in zip([\"a\", \"b\", \"c\"][:4], [4] * 3, [8] * 3)]\n&gt;&gt;&gt; column_idx = {k: v for v, k in enumerate([\"a\", \"b\", \"c\"])}\n&gt;&gt;&gt; wide = Wide(10, 1)\n&gt;&gt;&gt; deep = TabMlp(mlp_hidden_dims=[8, 4], column_idx=column_idx, cat_embed_input=embed_input)\n&gt;&gt;&gt; model = WideDeep(wide, deep)\n&gt;&gt;&gt; trainer = Trainer(model, objective=\"regression\", callbacks=[EarlyStopping(patience=10)])\n</code></pre> Source code in <code>pytorch_widedeep/callbacks.py</code> <pre><code>def __init__(\n    self,\n    monitor: str = \"val_loss\",\n    min_delta: float = 0.0,\n    patience: int = 10,\n    verbose: int = 0,\n    mode: str = \"auto\",\n    baseline: Optional[float] = None,\n    restore_best_weights: bool = False,\n):\n    super(EarlyStopping, self).__init__()\n\n    self.monitor = monitor\n    self.min_delta = min_delta\n    self.patience = patience\n    self.verbose = verbose\n    self.mode = mode\n    self.baseline = baseline\n    self.restore_best_weights = restore_best_weights\n\n    self.wait = 0\n    self.stopped_epoch = 0\n    self.state_dict = None\n\n    if self.mode not in [\"auto\", \"min\", \"max\"]:\n        warnings.warn(\n            \"EarlyStopping mode %s is unknown, \"\n            \"fallback to auto mode.\" % self.mode,\n            RuntimeWarning,\n        )\n        self.mode = \"auto\"\n\n    if self.mode == \"min\":\n        self.monitor_op = np.less\n    elif self.mode == \"max\":\n        self.monitor_op = np.greater  # type: ignore[assignment]\n    else:\n        if _is_metric(self.monitor):\n            self.monitor_op = np.greater  # type: ignore[assignment]\n        else:\n            self.monitor_op = np.less\n\n    if self.monitor_op == np.greater:\n        self.min_delta *= 1\n    else:\n        self.min_delta *= -1\n</code></pre>"},{"location":"pytorch-widedeep/dataloaders.html","title":"Dataloaders","text":"<p> NOTE: This module should contain custom dataloaders  that the user might want to implement. At the moment <code>pytorch-widedeep</code>  offers one custom dataloader, <code>DataLoaderImbalanced</code>.</p>"},{"location":"pytorch-widedeep/dataloaders.html#pytorch_widedeep.dataloaders.DataLoaderImbalanced","title":"DataLoaderImbalanced","text":"<pre><code>DataLoaderImbalanced(\n    dataset, batch_size, num_workers, **kwargs\n)\n</code></pre> <p>               Bases: <code>DataLoader</code></p> <p>Class to load and shuffle batches with adjusted weights for imbalanced datasets. If the classes do not begin from 0 remapping is necessary. See here.</p> <p>Parameters:</p> <ul> <li> <code>dataset</code>               (<code>WideDeepDataset</code>)           \u2013            <p>see <code>pytorch_widedeep.training._wd_dataset</code></p> </li> <li> <code>batch_size</code>               (<code>int</code>)           \u2013            <p>size of batch</p> </li> <li> <code>num_workers</code>               (<code>int</code>)           \u2013            <p>number of workers</p> </li> </ul> <p>Other Parameters:</p> <ul> <li> <code>**kwargs</code>           \u2013            <p>This can include any parameter that can be passed to the 'standard' pytorch DataLoader and that is not already explicitely passed to the class. In addition, the dictionary can also include the extra parameter <code>oversample_mul</code> which will multiply the number of samples of the minority class to be sampled by the <code>WeightedRandomSampler</code>.</p> <p>In other words, the <code>num_samples</code> param in <code>WeightedRandomSampler</code> will be defined as:</p> \\[ minority \\space class \\space count \\times number \\space of \\space classes \\times oversample\\_mul \\] </li> </ul> Source code in <code>pytorch_widedeep/dataloaders.py</code> <pre><code>def __init__(\n    self, dataset: WideDeepDataset, batch_size: int, num_workers: int, **kwargs\n):\n    assert dataset.Y is not None, (\n        \"The 'dataset' instance of WideDeepDataset must contain a \"\n        \"target array 'Y'\"\n    )\n\n    if \"oversample_mul\" in kwargs:\n        oversample_mul = kwargs[\"oversample_mul\"]\n        del kwargs[\"oversample_mul\"]\n    else:\n        oversample_mul = 1\n    weights, minor_cls_cnt, num_clss = get_class_weights(dataset)\n    num_samples = int(minor_cls_cnt * num_clss * oversample_mul)\n    samples_weight = list(np.array([weights[i] for i in dataset.Y]))\n    sampler = WeightedRandomSampler(samples_weight, num_samples, replacement=True)\n    super().__init__(\n        dataset, batch_size, num_workers=num_workers, sampler=sampler, **kwargs\n    )\n</code></pre>"},{"location":"pytorch-widedeep/load_from_folder.html","title":"The <code>load_from_folder</code> module","text":"<p>The <code>load_from_folder</code> module contains the classes that are necessary to load data from disk and these are inspired by the <code>ImageFolder</code> class in the <code>torchvision</code> library. This module is designed with one specific case in mind. Such case is the following: given a multi-modal dataset with tabular data, images and text, the images do not fit in memory, and therefore, they have to be loaded from disk. However, as any other functionality in this library, there is some flexibility and some additional cases can also be addressed using this module.</p> <p>For this module to be used, the datasets must be prepared in a certain way:</p> <ol> <li> <p>the tabular data must contain a column with the images names as stored in disk, including the extension (<code>.jpg</code>, <code>.png</code>, etc...).</p> </li> <li> <p>Regarding to the text dataset, the tabular data can contain a column with the texts themselves or the names of the files containing the texts as stored in disk.</p> </li> </ol> <p>The tabular data might or might not fit in disk itself. If it does not, please see the <code>ChunkPreprocessor</code> utilities at the[<code>preprocessing</code>] (preprocessing.md) module and the examples folder in the repo, which illustrate such case. Finally note that only <code>csv</code> format is currently supported in that case(more formats coming soon).</p>"},{"location":"pytorch-widedeep/load_from_folder.html#pytorch_widedeep.load_from_folder.tabular.tabular_from_folder.TabFromFolder","title":"TabFromFolder","text":"<pre><code>TabFromFolder(\n    fname,\n    directory=None,\n    target_col=None,\n    preprocessor=None,\n    text_col=None,\n    img_col=None,\n    ignore_target=False,\n    reference=None,\n    verbose=1,\n)\n</code></pre> <p>This class is used to load tabular data from disk. The current constrains are:</p> <ol> <li>The only file format supported right now is csv</li> <li>The csv file must contain headers</li> </ol> <p>For examples, please, see the examples folder in the repo.</p> <p>Parameters:</p> <ul> <li> <code>fname</code>               (<code>str</code>)           \u2013            <p>the name of the csv file</p> </li> <li> <code>directory</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>the path to the directory where the csv file is located. If None, a <code>TabFromFolder</code> reference object must be provided</p> </li> <li> <code>target_col</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>the name of the target column. If None, a <code>TabFromFolder</code> reference object must be provided</p> </li> <li> <code>preprocessor</code>               (<code>Optional[TabularPreprocessor]</code>, default:                   <code>None</code> )           \u2013            <p>a fitted <code>TabularPreprocessor</code> object. If None, a <code>TabFromFolder</code> reference object must be provided</p> </li> <li> <code>text_col</code>               (<code>Optional[Union[str, List[str]]]</code>, default:                   <code>None</code> )           \u2013            <p>the name of the column with the texts themselves or the names of the files that contain the text dataset. If None, either there is no text column or a <code>TabFromFolder</code> reference object must be provided</p> </li> <li> <code>img_col</code>               (<code>Optional[Union[str, List[str]]]</code>, default:                   <code>None</code> )           \u2013            <p>the name of the column with the the names of the images. If None, either there is no image column or a <code>TabFromFolder</code> reference object must be provided</p> </li> <li> <code>ignore_target</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>whether to ignore the target column. This is normally set to True when this class is used for a test dataset.</p> </li> <li> <code>reference</code>               (<code>Optional[Any]</code>, default:                   <code>None</code> )           \u2013            <p>a reference <code>TabFromFolder</code> object. If provided, the <code>TabFromFolder</code> object will be created using the attributes of the reference object. This is useful to instantiate a <code>TabFromFolder</code> object for evaluation or test purposes</p> </li> <li> <code>verbose</code>               (<code>Optional[int]</code>, default:                   <code>1</code> )           \u2013            <p>verbosity. If 0, no output will be printed during the process.</p> </li> </ul> Source code in <code>pytorch_widedeep/load_from_folder/tabular/tabular_from_folder.py</code> <pre><code>def __init__(\n    self,\n    fname: str,\n    directory: Optional[str] = None,\n    target_col: Optional[str] = None,\n    preprocessor: Optional[TabularPreprocessor] = None,\n    text_col: Optional[Union[str, List[str]]] = None,\n    img_col: Optional[Union[str, List[str]]] = None,\n    ignore_target: bool = False,\n    reference: Optional[Any] = None,  # is Type[\"TabFromFolder\"],\n    verbose: Optional[int] = 1,\n):\n    self.fname = fname\n    self.ignore_target = ignore_target\n    self.verbose = verbose\n\n    if reference is not None:\n        (\n            self.directory,\n            self.target_col,\n            self.preprocessor,\n            self.text_col,\n            self.img_col,\n        ) = self._set_from_reference(reference, preprocessor)\n    else:\n        assert (\n            directory is not None\n            and (target_col is not None and not ignore_target)\n            and preprocessor is not None\n        ), (\n            \"if no reference is provided, 'directory', 'target_col' and 'preprocessor' \"\n            \"must be provided\"\n        )\n\n        self.directory = directory\n        self.target_col = target_col\n        self.preprocessor = preprocessor\n        self.text_col = text_col\n        self.img_col = img_col\n\n    assert (\n        self.preprocessor.is_fitted\n    ), \"The preprocessor must be fitted before passing it to this class\"\n</code></pre>"},{"location":"pytorch-widedeep/load_from_folder.html#pytorch_widedeep.load_from_folder.tabular.tabular_from_folder.WideFromFolder","title":"WideFromFolder","text":"<pre><code>WideFromFolder(\n    fname,\n    directory=None,\n    target_col=None,\n    preprocessor=None,\n    text_col=None,\n    img_col=None,\n    ignore_target=False,\n    reference=None,\n    verbose=1,\n)\n</code></pre> <p>               Bases: <code>TabFromFolder</code></p> <p>This class is mostly identical to <code>TabFromFolder</code> but exists because we want to separate the treatment of the wide and the deep tabular components</p> <p>Parameters:</p> <ul> <li> <code>fname</code>               (<code>str</code>)           \u2013            <p>the name of the csv file</p> </li> <li> <code>directory</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>the path to the directory where the csv file is located. If None, a <code>WideFromFolder</code> reference object must be provided</p> </li> <li> <code>target_col</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>the name of the target column. If None, a <code>WideFromFolder</code> reference object must be provided</p> </li> <li> <code>preprocessor</code>               (<code>Optional[TabularPreprocessor]</code>, default:                   <code>None</code> )           \u2013            <p>a fitted <code>TabularPreprocessor</code> object. If None, a <code>WideFromFolder</code> reference object must be provided</p> </li> <li> <code>text_col</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>the name of the column with the texts themselves or the names of the files that contain the text dataset. If None, either there is no text column or a <code>WideFromFolder</code> reference object must be provided=</p> </li> <li> <code>img_col</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>the name of the column with the the names of the images. If None, either there is no image column or a <code>WideFromFolder</code> reference object must be provided</p> </li> <li> <code>ignore_target</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>whether to ignore the target column. This is normally used when this class is used for a test dataset.</p> </li> <li> <code>reference</code>               (<code>Optional[Any]</code>, default:                   <code>None</code> )           \u2013            <p>a reference <code>WideFromFolder</code> object. If provided, the <code>WideFromFolder</code> object will be created using the attributes of the reference object. This is useful to instantiate a <code>WideFromFolder</code> object for evaluation or test purposes</p> </li> <li> <code>verbose</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>verbosity. If 0, no output will be printed during the process.</p> </li> </ul> Source code in <code>pytorch_widedeep/load_from_folder/tabular/tabular_from_folder.py</code> <pre><code>def __init__(\n    self,\n    fname: str,\n    directory: Optional[str] = None,\n    target_col: Optional[str] = None,\n    preprocessor: Optional[TabularPreprocessor] = None,\n    text_col: Optional[str] = None,\n    img_col: Optional[str] = None,\n    ignore_target: bool = False,\n    reference: Optional[Any] = None,  # is Type[\"WideFromFolder\"],\n    verbose: int = 1,\n):\n    super(WideFromFolder, self).__init__(\n        fname=fname,\n        directory=directory,\n        target_col=target_col,\n        preprocessor=preprocessor,\n        text_col=text_col,\n        img_col=img_col,\n        reference=reference,\n        ignore_target=ignore_target,\n        verbose=verbose,\n    )\n</code></pre>"},{"location":"pytorch-widedeep/load_from_folder.html#pytorch_widedeep.load_from_folder.text.text_from_folder.TextFromFolder","title":"TextFromFolder","text":"<pre><code>TextFromFolder(preprocessor)\n</code></pre> <p>This class is used to load the text dataset (i.e. the text files) from a folder, or to retrieve the text given a texts column specified within the preprocessor object.</p> <p>For examples, please, see the examples folder in the repo.</p> <p>Parameters:</p> <ul> <li> <code>preprocessor</code>               (<code>Union[TextPreprocessor, ChunkTextPreprocessor, HFPreprocessor, ChunkHFPreprocessor, List[TextPreprocessor], List[ChunkTextPreprocessor], List[HFPreprocessor], List[ChunkHFPreprocessor]]</code>)           \u2013            <p>The preprocessor used to process the text. It must be fitted before using this class</p> </li> </ul> Source code in <code>pytorch_widedeep/load_from_folder/text/text_from_folder.py</code> <pre><code>def __init__(\n    self,\n    preprocessor: Union[\n        TextPreprocessor,\n        ChunkTextPreprocessor,\n        HFPreprocessor,\n        ChunkHFPreprocessor,\n        List[TextPreprocessor],\n        List[ChunkTextPreprocessor],\n        List[HFPreprocessor],\n        List[ChunkHFPreprocessor],\n    ],\n):\n    if isinstance(preprocessor, list):\n        for p in preprocessor:\n            assert (\n                p.is_fitted\n            ), \"All preprocessors must be fitted before using this class\"\n    else:\n        assert (\n            preprocessor.is_fitted\n        ), \"The preprocessor must be fitted before using this class\"\n\n    self.preprocessor = preprocessor\n</code></pre>"},{"location":"pytorch-widedeep/load_from_folder.html#pytorch_widedeep.load_from_folder.image.image_from_folder.ImageFromFolder","title":"ImageFromFolder","text":"<pre><code>ImageFromFolder(\n    directory=None,\n    preprocessor=None,\n    loader=default_loader,\n    extensions=None,\n    transforms=None,\n)\n</code></pre> <p>This class is used to load the image dataset from disk. It is inspired by the <code>ImageFolder</code> class at the <code>torchvision</code> library. Here, we have simply adapted to work within the context of a Wide and Deep multi-modal model.</p> <p>For examples, please, see the examples folder in the repo.</p> <p>Parameters:</p> <ul> <li> <code>directory</code>               (<code>Optional[Union[str, List[str]]]</code>, default:                   <code>None</code> )           \u2013            <p>the path to the directory where the images are located. If None, a preprocessor must be provided.</p> </li> <li> <code>preprocessor</code>               (<code>Optional[Union[ImagePreprocessor, List[ImagePreprocessor]]]</code>, default:                   <code>None</code> )           \u2013            <p>a fitted <code>ImagePreprocessor</code> object.</p> </li> <li> <code>loader</code>               (<code>Callable[[str], Any]</code>, default:                   <code>default_loader</code> )           \u2013            <p>a function to load a sample given its path.</p> </li> <li> <code>extensions</code>               (<code>Optional[Tuple[str, ...]]</code>, default:                   <code>None</code> )           \u2013            <p>a tuple with the allowed extensions. If None, IMG_EXTENSIONS will be used where IMG_EXTENSIONS =\".jpg\", \".jpeg\", \".png\", \".ppm\", \".bmp\", \".pgm\", \".tif\", \".tiff\", \".webp\"</p> </li> <li> <code>transforms</code>               (<code>Optional[Any]</code>, default:                   <code>None</code> )           \u2013            <p>a <code>torchvision.transforms</code> object. If None, this class will simply return an array representation of the PIL Image</p> </li> </ul> Source code in <code>pytorch_widedeep/load_from_folder/image/image_from_folder.py</code> <pre><code>def __init__(\n    self,\n    directory: Optional[Union[str, List[str]]] = None,\n    preprocessor: Optional[\n        Union[ImagePreprocessor, List[ImagePreprocessor]]\n    ] = None,\n    loader: Callable[[str], Any] = default_loader,\n    extensions: Optional[Tuple[str, ...]] = None,\n    transforms: Optional[Any] = None,\n) -&gt; None:\n    assert (\n        directory is not None or preprocessor is not None\n    ), \"Either a directory or an instance of ImagePreprocessor(s) must be provided\"\n\n    if directory is not None and preprocessor is not None:  # pragma: no cover\n        error_msg = (\n            \"If both 'directory' and 'preprocessor' are provided, the 'img_path' \"\n            \"attribute of the 'preprocessor' must be the same as the 'directory'\"\n        )\n        if isinstance(directory, list):\n            assert isinstance(preprocessor, list)\n            assert len(directory) == len(preprocessor)\n            for d, p in zip(directory, preprocessor):\n                assert d == p.img_path, error_msg\n        else:\n            assert isinstance(preprocessor, ImagePreprocessor)\n            assert directory == preprocessor.img_path, error_msg\n\n    if directory is not None:\n        self.directory = directory\n    else:\n        assert (\n            preprocessor is not None\n        ), \"Either a directory or an instance of ImagePreprocessor must be provided\"\n        if isinstance(preprocessor, list):\n            self.directory = [p.img_path for p in preprocessor]\n        else:\n            self.directory = preprocessor.img_path\n\n    self.preprocessor = preprocessor\n    self.loader = loader\n    self.extensions = extensions if extensions is not None else IMG_EXTENSIONS\n    self.transforms = transforms\n    if self.transforms:\n        self.transforms_names = [\n            tr.__class__.__name__ for tr in self.transforms.transforms\n        ]\n    else:\n        self.transforms_names = []\n\n        self.transpose = True\n</code></pre>"},{"location":"pytorch-widedeep/load_from_folder.html#pytorch_widedeep.load_from_folder.wd_dataset_from_folder.WideDeepDatasetFromFolder","title":"WideDeepDatasetFromFolder","text":"<pre><code>WideDeepDatasetFromFolder(\n    n_samples,\n    tab_from_folder=None,\n    wide_from_folder=None,\n    text_from_folder=None,\n    img_from_folder=None,\n    reference=None,\n)\n</code></pre> <p>               Bases: <code>Dataset</code></p> <p>This class is the Dataset counterpart of the <code>WideDeepDataset</code> class.</p> <p>Given a reference tabular dataset, with columns that indicate the path to the images and to the text files or the texts themselves, it will use the <code>[...]FromFolder</code> classes to load the data consistently from disk per batch.</p> <p>For examples, please, see the examples folder in the repo.</p> <p>Parameters:</p> <ul> <li> <code>n_samples</code>               (<code>int</code>)           \u2013            <p>Number of samples in the dataset</p> </li> <li> <code>tab_from_folder</code>               (<code>Optional[TabFromFolder]</code>, default:                   <code>None</code> )           \u2013            <p>Instance of the <code>TabFromFolder</code> class</p> </li> <li> <code>wide_from_folder</code>               (<code>Optional[WideFromFolder]</code>, default:                   <code>None</code> )           \u2013            <p>Instance of the <code>WideFromFolder</code> class</p> </li> <li> <code>text_from_folder</code>               (<code>Optional[TextFromFolder]</code>, default:                   <code>None</code> )           \u2013            <p>Instance of the <code>TextFromFolder</code> class</p> </li> <li> <code>img_from_folder</code>               (<code>Optional[ImageFromFolder]</code>, default:                   <code>None</code> )           \u2013            <p>Instance of the <code>ImageFromFolder</code> class</p> </li> <li> <code>reference</code>               (<code>Optional[Any]</code>, default:                   <code>None</code> )           \u2013            <p>If not None, the 'text_from_folder' and 'img_from_folder' objects will be retrieved from the reference class. This is useful when we want to use a <code>WideDeepDatasetFromFolder</code> class used for a train dataset as a reference for the validation and test datasets. In this case, the <code>text_from_folder</code> and <code>img_from_folder</code> objects will be the same for all three datasets, so there is no need to create a new instance for each dataset.</p> </li> </ul> Source code in <code>pytorch_widedeep/load_from_folder/wd_dataset_from_folder.py</code> <pre><code>def __init__(\n    self,\n    n_samples: int,\n    tab_from_folder: Optional[TabFromFolder] = None,\n    wide_from_folder: Optional[WideFromFolder] = None,\n    text_from_folder: Optional[TextFromFolder] = None,\n    img_from_folder: Optional[ImageFromFolder] = None,\n    reference: Optional[Any] = None,  # is Type[\"WideDeepDatasetFromFolder\"],\n):\n    super(WideDeepDatasetFromFolder, self).__init__()\n\n    if tab_from_folder is None and wide_from_folder is None:\n        raise ValueError(\n            \"Either 'tab_from_folder' or 'wide_from_folder' must be not None\"\n        )\n\n    if reference is not None:\n        assert (\n            img_from_folder is None and text_from_folder is None\n        ), \"If reference is not None, 'img_from_folder' and 'text_from_folder' left as None\"\n        self.text_from_folder, self.img_from_folder = self._get_from_reference(\n            reference\n        )\n    else:\n        assert (\n            text_from_folder is not None and img_from_folder is not None\n        ), \"If reference is None, 'img_from_folder' and 'text_from_folder' must be not None\"\n        self.text_from_folder = text_from_folder\n        self.img_from_folder = img_from_folder\n\n    self.n_samples = n_samples\n    self.tab_from_folder = tab_from_folder\n    self.wide_from_folder = wide_from_folder\n</code></pre>"},{"location":"pytorch-widedeep/losses.html","title":"Losses","text":"<p><code>pytorch-widedeep</code> accepts a number of losses and objectives that can be passed to the <code>Trainer</code> class via the parameter <code>objective</code> (see <code>pytorch-widedeep.training.Trainer</code>). For most cases the loss function that <code>pytorch-widedeep</code> will use internally is already implemented in Pytorch.</p> <p>In addition, <code>pytorch-widedeep</code> implements a series of  \"custom\" loss functions. These are described below for completion since, as mentioned before, they are used internally by the <code>Trainer</code>. Of course, onen could always use them on their own and can be imported as:</p> <p><code>from pytorch_widedeep.losses import FocalLoss</code></p> <p> NOTE:  Losses in this module expect the predictions  and ground truth to have the same dimensions for regression and binary  classification problems \\((N_{samples}, 1)\\). In the case of multiclass  classification problems the ground truth is expected to be a 1D tensor with  the corresponding classes. See Examples below</p>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.MSELoss","title":"MSELoss","text":"<pre><code>MSELoss()\n</code></pre> <p>               Bases: <code>Module</code></p> <p>Mean square error loss</p> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def __init__(self):\n    super().__init__()\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.MSELoss.forward","title":"forward","text":"<pre><code>forward(input, target)\n</code></pre> <p>Parameters:</p> <ul> <li> <code>input</code>               (<code>Tensor</code>)           \u2013            <p>Input tensor with predictions</p> </li> <li> <code>target</code>               (<code>Tensor</code>)           \u2013            <p>Target tensor with the actual values</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.losses import MSELoss\n&gt;&gt;&gt;\n&gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n&gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n&gt;&gt;&gt; loss = MSELoss()(input, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(\n    self,\n    input: Tensor,\n    target: Tensor,\n) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    input: Tensor\n        Input tensor with predictions\n    target: Tensor\n        Target tensor with the actual values\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.losses import MSELoss\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n    &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n    &gt;&gt;&gt; loss = MSELoss()(input, target)\n    \"\"\"\n    loss = (input - target) ** 2\n    return torch.mean(loss)\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.MSLELoss","title":"MSLELoss","text":"<pre><code>MSLELoss()\n</code></pre> <p>               Bases: <code>Module</code></p> <p>Mean square log error loss</p> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def __init__(self):\n    super().__init__()\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.MSLELoss.forward","title":"forward","text":"<pre><code>forward(input, target)\n</code></pre> <p>Parameters:</p> <ul> <li> <code>input</code>               (<code>Tensor</code>)           \u2013            <p>Input tensor with predictions (not probabilities)</p> </li> <li> <code>target</code>               (<code>Tensor</code>)           \u2013            <p>Target tensor with the actual classes</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.losses import MSLELoss\n&gt;&gt;&gt;\n&gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n&gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n&gt;&gt;&gt; loss = MSLELoss()(input, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(\n    self,\n    input: Tensor,\n    target: Tensor,\n) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    input: Tensor\n        Input tensor with predictions (not probabilities)\n    target: Tensor\n        Target tensor with the actual classes\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.losses import MSLELoss\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n    &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n    &gt;&gt;&gt; loss = MSLELoss()(input, target)\n    \"\"\"\n    assert (\n        input.min() &gt;= 0\n    ), \"\"\"All input values must be &gt;=0, if your model is predicting\n        values &lt;0 try to enforce positive values by activation function\n        on last layer with `trainer.enforce_positive_output=True`\"\"\"\n    assert target.min() &gt;= 0, \"All target values must be &gt;=0\"\n\n    loss = (torch.log(input + 1) - torch.log(target + 1)) ** 2\n    return torch.mean(loss)\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.RMSELoss","title":"RMSELoss","text":"<pre><code>RMSELoss()\n</code></pre> <p>               Bases: <code>Module</code></p> <p>Root mean square error loss</p> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def __init__(self):\n    super().__init__()\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.RMSELoss.forward","title":"forward","text":"<pre><code>forward(input, target)\n</code></pre> <p>Parameters:</p> <ul> <li> <code>input</code>               (<code>Tensor</code>)           \u2013            <p>Input tensor with predictions (not probabilities)</p> </li> <li> <code>target</code>               (<code>Tensor</code>)           \u2013            <p>Target tensor with the actual classes</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.losses import RMSELoss\n&gt;&gt;&gt;\n&gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n&gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n&gt;&gt;&gt; loss = RMSELoss()(input, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(self, input: Tensor, target: Tensor) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    input: Tensor\n        Input tensor with predictions (not probabilities)\n    target: Tensor\n        Target tensor with the actual classes\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.losses import RMSELoss\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n    &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n    &gt;&gt;&gt; loss = RMSELoss()(input, target)\n    \"\"\"\n    loss = (input - target) ** 2\n    return torch.sqrt(torch.mean(loss))\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.RMSLELoss","title":"RMSLELoss","text":"<pre><code>RMSLELoss()\n</code></pre> <p>               Bases: <code>Module</code></p> <p>Root mean square log error loss</p> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def __init__(self):\n    super().__init__()\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.RMSLELoss.forward","title":"forward","text":"<pre><code>forward(input, target)\n</code></pre> <p>Parameters:</p> <ul> <li> <code>input</code>               (<code>Tensor</code>)           \u2013            <p>Input tensor with predictions (not probabilities)</p> </li> <li> <code>target</code>               (<code>Tensor</code>)           \u2013            <p>Target tensor with the actual classes</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.losses import RMSLELoss\n&gt;&gt;&gt;\n&gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n&gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n&gt;&gt;&gt; loss = RMSLELoss()(input, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(self, input: Tensor, target: Tensor) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    input: Tensor\n        Input tensor with predictions (not probabilities)\n    target: Tensor\n        Target tensor with the actual classes\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.losses import RMSLELoss\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n    &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n    &gt;&gt;&gt; loss = RMSLELoss()(input, target)\n    \"\"\"\n    assert (\n        input.min() &gt;= 0\n    ), \"\"\"All input values must be &gt;=0, if your model is predicting\n        values &lt;0 try to enforce positive values by activation function\n        on last layer with `trainer.enforce_positive_output=True`\"\"\"\n    assert target.min() &gt;= 0, \"All target values must be &gt;=0\"\n\n    loss = (torch.log(input + 1) - torch.log(target + 1)) ** 2\n    return torch.sqrt(torch.mean(loss))\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.QuantileLoss","title":"QuantileLoss","text":"<pre><code>QuantileLoss(\n    quantiles=[0.02, 0.1, 0.25, 0.5, 0.75, 0.9, 0.98]\n)\n</code></pre> <p>               Bases: <code>Module</code></p> <p>Quantile loss defined as:</p> \\[ Loss = max(q \\times (y-y_{pred}), (1-q) \\times (y_{pred}-y)) \\] <p>All credits go to the implementation at pytorch-forecasting.</p> <p>Parameters:</p> <ul> <li> <code>quantiles</code>               (<code>List[float]</code>, default:                   <code>[0.02, 0.1, 0.25, 0.5, 0.75, 0.9, 0.98]</code> )           \u2013            <p>List of quantiles</p> </li> </ul> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def __init__(\n    self,\n    quantiles: List[float] = [0.02, 0.1, 0.25, 0.5, 0.75, 0.9, 0.98],\n):\n    super().__init__()\n    self.quantiles = quantiles\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.QuantileLoss.forward","title":"forward","text":"<pre><code>forward(input, target)\n</code></pre> <p>Parameters:</p> <ul> <li> <code>input</code>               (<code>Tensor</code>)           \u2013            <p>Input tensor with predictions</p> </li> <li> <code>target</code>               (<code>Tensor</code>)           \u2013            <p>Target tensor with the actual values</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt;\n&gt;&gt;&gt; from pytorch_widedeep.losses import QuantileLoss\n&gt;&gt;&gt;\n&gt;&gt;&gt; # REGRESSION\n&gt;&gt;&gt; target = torch.tensor([[0.6, 1.5]]).view(-1, 1)\n&gt;&gt;&gt; input = torch.tensor([[.1, .2,], [.4, .5]])\n&gt;&gt;&gt; qloss = QuantileLoss([0.25, 0.75])\n&gt;&gt;&gt; loss = qloss(input, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(self, input: Tensor, target: Tensor) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    input: Tensor\n        Input tensor with predictions\n    target: Tensor\n        Target tensor with the actual values\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; from pytorch_widedeep.losses import QuantileLoss\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; # REGRESSION\n    &gt;&gt;&gt; target = torch.tensor([[0.6, 1.5]]).view(-1, 1)\n    &gt;&gt;&gt; input = torch.tensor([[.1, .2,], [.4, .5]])\n    &gt;&gt;&gt; qloss = QuantileLoss([0.25, 0.75])\n    &gt;&gt;&gt; loss = qloss(input, target)\n    \"\"\"\n\n    assert input.shape == torch.Size([target.shape[0], len(self.quantiles)]), (\n        \"The input and target have inconsistent shape. The dimension of the prediction \"\n        \"of the model that is using QuantileLoss must be equal to number of quantiles, \"\n        f\"i.e. {len(self.quantiles)}.\"\n    )\n    target = target.view(-1, 1).float()\n    losses = []\n    for i, q in enumerate(self.quantiles):\n        errors = target - input[..., i]\n        losses.append(torch.max((q - 1) * errors, q * errors).unsqueeze(-1))\n\n    loss = torch.cat(losses, dim=2)\n\n    return torch.mean(loss)\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.FocalLoss","title":"FocalLoss","text":"<pre><code>FocalLoss(alpha=0.25, gamma=1.0)\n</code></pre> <p>               Bases: <code>Module</code></p> <p>Implementation of the Focal loss for both binary and multiclass classification:</p> \\[ FL(p_t) = \\alpha (1 - p_t)^{\\gamma} log(p_t) \\] <p>where, for a case of a binary classification problem</p> \\[ \\begin{equation} p_t= \\begin{cases}p, &amp; \\text{if $y=1$}.\\\\1-p, &amp; \\text{otherwise}. \\end{cases} \\end{equation} \\] <p>Parameters:</p> <ul> <li> <code>alpha</code>               (<code>float</code>, default:                   <code>0.25</code> )           \u2013            <p>Focal Loss <code>alpha</code> parameter</p> </li> <li> <code>gamma</code>               (<code>float</code>, default:                   <code>1.0</code> )           \u2013            <p>Focal Loss <code>gamma</code> parameter</p> </li> </ul> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def __init__(self, alpha: float = 0.25, gamma: float = 1.0):\n    super().__init__()\n    self.alpha = alpha\n    self.gamma = gamma\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.FocalLoss.forward","title":"forward","text":"<pre><code>forward(input, target)\n</code></pre> <p>Parameters:</p> <ul> <li> <code>input</code>               (<code>Tensor</code>)           \u2013            <p>Input tensor with predictions (not probabilities)</p> </li> <li> <code>target</code>               (<code>Tensor</code>)           \u2013            <p>Target tensor with the actual classes</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt;\n&gt;&gt;&gt; from pytorch_widedeep.losses import FocalLoss\n&gt;&gt;&gt;\n&gt;&gt;&gt; # BINARY\n&gt;&gt;&gt; target = torch.tensor([0, 1, 0, 1]).view(-1, 1)\n&gt;&gt;&gt; input = torch.tensor([[0.6, 0.7, 0.3, 0.8]]).t()\n&gt;&gt;&gt; loss = FocalLoss()(input, target)\n&gt;&gt;&gt;\n&gt;&gt;&gt; # MULTICLASS\n&gt;&gt;&gt; target = torch.tensor([1, 0, 2]).view(-1, 1)\n&gt;&gt;&gt; input = torch.tensor([[0.2, 0.5, 0.3], [0.8, 0.1, 0.1], [0.7, 0.2, 0.1]])\n&gt;&gt;&gt; loss = FocalLoss()(input, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(self, input: Tensor, target: Tensor) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    input: Tensor\n        Input tensor with predictions (not probabilities)\n    target: Tensor\n        Target tensor with the actual classes\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; from pytorch_widedeep.losses import FocalLoss\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; # BINARY\n    &gt;&gt;&gt; target = torch.tensor([0, 1, 0, 1]).view(-1, 1)\n    &gt;&gt;&gt; input = torch.tensor([[0.6, 0.7, 0.3, 0.8]]).t()\n    &gt;&gt;&gt; loss = FocalLoss()(input, target)\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; # MULTICLASS\n    &gt;&gt;&gt; target = torch.tensor([1, 0, 2]).view(-1, 1)\n    &gt;&gt;&gt; input = torch.tensor([[0.2, 0.5, 0.3], [0.8, 0.1, 0.1], [0.7, 0.2, 0.1]])\n    &gt;&gt;&gt; loss = FocalLoss()(input, target)\n    \"\"\"\n    input_prob = torch.sigmoid(input)\n    if input.size(1) == 1:\n        input_prob = torch.cat([1 - input_prob, input_prob], axis=1)  # type: ignore\n        num_class = 2\n    else:\n        num_class = input_prob.size(1)\n    binary_target = torch.eye(num_class)[target.squeeze().cpu().long()]\n    if use_cuda:\n        binary_target = binary_target.cuda()\n    binary_target = binary_target.contiguous()\n    weight = self._get_weight(input_prob, binary_target)\n\n    return F.binary_cross_entropy(\n        input_prob, binary_target, weight, reduction=\"mean\"\n    )\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.BayesianSELoss","title":"BayesianSELoss","text":"<pre><code>BayesianSELoss()\n</code></pre> <p>               Bases: <code>Module</code></p> <p>Squared Loss (log Gaussian) for the case of a regression as specified in the original publication Weight Uncertainty in Neural Networks.</p> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def __init__(self):\n    super().__init__()\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.BayesianSELoss.forward","title":"forward","text":"<pre><code>forward(input, target)\n</code></pre> <p>Parameters:</p> <ul> <li> <code>input</code>               (<code>Tensor</code>)           \u2013            <p>Input tensor with predictions (not probabilities)</p> </li> <li> <code>target</code>               (<code>Tensor</code>)           \u2013            <p>Target tensor with the actual classes</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.losses import BayesianSELoss\n&gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n&gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n&gt;&gt;&gt; loss = BayesianSELoss()(input, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(self, input: Tensor, target: Tensor) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    input: Tensor\n        Input tensor with predictions (not probabilities)\n    target: Tensor\n        Target tensor with the actual classes\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.losses import BayesianSELoss\n    &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n    &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n    &gt;&gt;&gt; loss = BayesianSELoss()(input, target)\n    \"\"\"\n    return (0.5 * (input - target) ** 2).sum()\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.TweedieLoss","title":"TweedieLoss","text":"<pre><code>TweedieLoss()\n</code></pre> <p>               Bases: <code>Module</code></p> <p>Tweedie loss for extremely unbalanced zero-inflated data</p> <p>All credits go to Wenbo Shi. See this post and the original publication for details.</p> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def __init__(self):\n    super().__init__()\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.TweedieLoss.forward","title":"forward","text":"<pre><code>forward(input, target, p=1.5)\n</code></pre> <p>Parameters:</p> <ul> <li> <code>input</code>               (<code>Tensor</code>)           \u2013            <p>Input tensor with predictions</p> </li> <li> <code>target</code>               (<code>Tensor</code>)           \u2013            <p>Target tensor with the actual values</p> </li> <li> <code>p</code>               (<code>float</code>, default:                   <code>1.5</code> )           \u2013            <p>the power to be used to compute the loss. See the original publication for details</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.losses import TweedieLoss\n&gt;&gt;&gt;\n&gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n&gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n&gt;&gt;&gt; loss = TweedieLoss()(input, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(\n    self,\n    input: Tensor,\n    target: Tensor,\n    p: float = 1.5,\n) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    input: Tensor\n        Input tensor with predictions\n    target: Tensor\n        Target tensor with the actual values\n    p: float, default = 1.5\n        the power to be used to compute the loss. See the original\n        publication for details\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.losses import TweedieLoss\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n    &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n    &gt;&gt;&gt; loss = TweedieLoss()(input, target)\n    \"\"\"\n\n    assert (\n        input.min() &gt; 0\n    ), \"\"\"All input values must be &gt;=0, if your model is predicting\n        values &lt;0 try to enforce positive values by activation function\n        on last layer with `trainer.enforce_positive_output=True`\"\"\"\n    assert target.min() &gt;= 0, \"All target values must be &gt;=0\"\n    loss = -target * torch.pow(input, 1 - p) / (1 - p) + torch.pow(input, 2 - p) / (\n        2 - p\n    )\n    return torch.mean(loss)\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.ZILNLoss","title":"ZILNLoss","text":"<pre><code>ZILNLoss()\n</code></pre> <p>               Bases: <code>Module</code></p> <p>Adjusted implementation of the Zero Inflated LogNormal Loss</p> <p>See A Deep Probabilistic Model for Customer Lifetime Value Prediction and the corresponding code.</p> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def __init__(self):\n    super().__init__()\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.ZILNLoss.forward","title":"forward","text":"<pre><code>forward(input, target)\n</code></pre> <p>Parameters:</p> <ul> <li> <code>input</code>               (<code>Tensor</code>)           \u2013            <p>Input tensor with predictions with spape (N,3), where N is the batch size</p> </li> <li> <code>target</code>               (<code>Tensor</code>)           \u2013            <p>Target tensor with the actual target values</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.losses import ZILNLoss\n&gt;&gt;&gt;\n&gt;&gt;&gt; target = torch.tensor([[0., 1.5]]).view(-1, 1)\n&gt;&gt;&gt; input = torch.tensor([[.1, .2, .3], [.4, .5, .6]])\n&gt;&gt;&gt; loss = ZILNLoss()(input, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(self, input: Tensor, target: Tensor) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    input: Tensor\n        Input tensor with predictions with spape (N,3), where N is the batch size\n    target: Tensor\n        Target tensor with the actual target values\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.losses import ZILNLoss\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; target = torch.tensor([[0., 1.5]]).view(-1, 1)\n    &gt;&gt;&gt; input = torch.tensor([[.1, .2, .3], [.4, .5, .6]])\n    &gt;&gt;&gt; loss = ZILNLoss()(input, target)\n    \"\"\"\n    positive = target &gt; 0\n    positive = positive.float()\n\n    assert input.shape == torch.Size([target.shape[0], 3]), (\n        \"Wrong shape of the 'input' tensor. The pred_dim of the \"\n        \"model that is using ZILNLoss must be equal to 3.\"\n    )\n\n    positive_input = input[..., :1]\n\n    classification_loss = F.binary_cross_entropy_with_logits(\n        positive_input, positive, reduction=\"none\"\n    ).flatten()\n\n    loc = input[..., 1:2]\n\n    # when using max the two input tensors (input and other) have to be of\n    # the same type\n    max_input = F.softplus(input[..., 2:])\n    max_other = torch.sqrt(torch.Tensor([torch.finfo(torch.double).eps])).type(\n        max_input.type()\n    )\n    scale = torch.max(max_input, max_other)\n    safe_labels = positive * target + (1 - positive) * torch.ones_like(target)\n\n    regression_loss = -torch.mean(\n        positive\n        * torch.distributions.log_normal.LogNormal(loc=loc, scale=scale).log_prob(\n            safe_labels\n        ),\n        dim=-1,\n    )\n\n    return torch.mean(classification_loss + regression_loss)\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.L1Loss","title":"L1Loss","text":"<pre><code>L1Loss()\n</code></pre> <p>               Bases: <code>Module</code></p> <p>L1 loss</p> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def __init__(self):\n    super().__init__()\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.L1Loss.forward","title":"forward","text":"<pre><code>forward(input, target)\n</code></pre> <p>Parameters:</p> <ul> <li> <code>input</code>               (<code>Tensor</code>)           \u2013            <p>Input tensor with predictions</p> </li> <li> <code>target</code>               (<code>Tensor</code>)           \u2013            <p>Target tensor with the actual values</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt;\n&gt;&gt;&gt; from pytorch_widedeep.losses import L1Loss\n&gt;&gt;&gt;\n&gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n&gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n&gt;&gt;&gt; loss = L1Loss()(input, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(self, input: Tensor, target: Tensor) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    input: Tensor\n        Input tensor with predictions\n    target: Tensor\n        Target tensor with the actual values\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; from pytorch_widedeep.losses import L1Loss\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n    &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n    &gt;&gt;&gt; loss = L1Loss()(input, target)\n    \"\"\"\n    loss = F.l1_loss(input, target, reduction=\"none\")\n    return torch.mean(loss)\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.FocalR_L1Loss","title":"FocalR_L1Loss","text":"<pre><code>FocalR_L1Loss(beta=0.2, gamma=1.0, activation_fn='sigmoid')\n</code></pre> <p>               Bases: <code>Module</code></p> <p>Focal-R L1 loss</p> <p>Based on Delving into Deep Imbalanced Regression.</p> <p>Parameters:</p> <ul> <li> <code>beta</code>               (<code>float</code>, default:                   <code>0.2</code> )           \u2013            <p>Focal Loss <code>beta</code> parameter in their implementation</p> </li> <li> <code>gamma</code>               (<code>float</code>, default:                   <code>1.0</code> )           \u2013            <p>Focal Loss <code>gamma</code> parameter</p> </li> <li> <code>activation_fn</code>               (<code>Literal[sigmoid, tanh]</code>, default:                   <code>'sigmoid'</code> )           \u2013            <p>Activation function to be used during the computation of the loss. Possible values are 'sigmoid' and 'tanh'. See the original publication for details.</p> </li> </ul> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def __init__(\n    self,\n    beta: float = 0.2,\n    gamma: float = 1.0,\n    activation_fn: Literal[\"sigmoid\", \"tanh\"] = \"sigmoid\",\n):\n    super().__init__()\n    self.beta = beta\n    self.gamma = gamma\n    self.activation_fn = activation_fn\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.FocalR_L1Loss.forward","title":"forward","text":"<pre><code>forward(input, target)\n</code></pre> <p>Parameters:</p> <ul> <li> <code>input</code>               (<code>Tensor</code>)           \u2013            <p>Input tensor with predictions (not probabilities)</p> </li> <li> <code>target</code>               (<code>Tensor</code>)           \u2013            <p>Target tensor with the actual classes</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt;\n&gt;&gt;&gt; from pytorch_widedeep.losses import FocalR_L1Loss\n&gt;&gt;&gt;\n&gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n&gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n&gt;&gt;&gt; loss = FocalR_L1Loss()(input, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(\n    self,\n    input: Tensor,\n    target: Tensor,\n) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    input: Tensor\n        Input tensor with predictions (not probabilities)\n    target: Tensor\n        Target tensor with the actual classes\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; from pytorch_widedeep.losses import FocalR_L1Loss\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n    &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n    &gt;&gt;&gt; loss = FocalR_L1Loss()(input, target)\n    \"\"\"\n    loss = F.l1_loss(input, target, reduction=\"none\")\n    if self.activation_fn == \"tanh\":\n        loss *= (torch.tanh(self.beta * torch.abs(input - target))) ** self.gamma\n    elif self.activation_fn == \"sigmoid\":\n        loss *= (\n            2 * torch.sigmoid(self.beta * torch.abs(input - target)) - 1\n        ) ** self.gamma\n    else:\n        ValueError(\n            \"Incorrect activation function value - must be in ['sigmoid', 'tanh']\"\n        )\n    return torch.mean(loss)\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.FocalR_MSELoss","title":"FocalR_MSELoss","text":"<pre><code>FocalR_MSELoss(\n    beta=0.2, gamma=1.0, activation_fn=\"sigmoid\"\n)\n</code></pre> <p>               Bases: <code>Module</code></p> <p>Focal-R MSE loss</p> <p>Based on Delving into Deep Imbalanced Regression.</p> <p>Parameters:</p> <ul> <li> <code>beta</code>               (<code>float</code>, default:                   <code>0.2</code> )           \u2013            <p>Focal Loss <code>beta</code> parameter in their implementation</p> </li> <li> <code>gamma</code>               (<code>float</code>, default:                   <code>1.0</code> )           \u2013            <p>Focal Loss <code>gamma</code> parameter</p> </li> <li> <code>activation_fn</code>               (<code>Literal[sigmoid, tanh]</code>, default:                   <code>'sigmoid'</code> )           \u2013            <p>Activation function to be used during the computation of the loss. Possible values are 'sigmoid' and 'tanh'. See the original publication for details.</p> </li> </ul> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def __init__(\n    self,\n    beta: float = 0.2,\n    gamma: float = 1.0,\n    activation_fn: Literal[\"sigmoid\", \"tanh\"] = \"sigmoid\",\n):\n    super().__init__()\n    self.beta = beta\n    self.gamma = gamma\n    self.activation_fn = activation_fn\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.FocalR_MSELoss.forward","title":"forward","text":"<pre><code>forward(input, target)\n</code></pre> <p>Parameters:</p> <ul> <li> <code>input</code>               (<code>Tensor</code>)           \u2013            <p>Input tensor with predictions (not probabilities)</p> </li> <li> <code>target</code>               (<code>Tensor</code>)           \u2013            <p>Target tensor with the actual classes</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt;\n&gt;&gt;&gt; from pytorch_widedeep.losses import FocalR_MSELoss\n&gt;&gt;&gt;\n&gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n&gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n&gt;&gt;&gt; loss = FocalR_MSELoss()(input, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(\n    self,\n    input: Tensor,\n    target: Tensor,\n) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    input: Tensor\n        Input tensor with predictions (not probabilities)\n    target: Tensor\n        Target tensor with the actual classes\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; from pytorch_widedeep.losses import FocalR_MSELoss\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n    &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n    &gt;&gt;&gt; loss = FocalR_MSELoss()(input, target)\n    \"\"\"\n    loss = (input - target) ** 2\n    if self.activation_fn == \"tanh\":\n        loss *= (torch.tanh(self.beta * torch.abs(input - target))) ** self.gamma\n    elif self.activation_fn == \"sigmoid\":\n        loss *= (\n            2 * torch.sigmoid(self.beta * torch.abs((input - target) ** 2)) - 1\n        ) ** self.gamma\n    else:\n        ValueError(\n            \"Incorrect activation function value - must be in ['sigmoid', 'tanh']\"\n        )\n    return torch.mean(loss)\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.FocalR_RMSELoss","title":"FocalR_RMSELoss","text":"<pre><code>FocalR_RMSELoss(\n    beta=0.2, gamma=1.0, activation_fn=\"sigmoid\"\n)\n</code></pre> <p>               Bases: <code>Module</code></p> <p>Focal-R RMSE loss</p> <p>Based on Delving into Deep Imbalanced Regression.</p> <p>Parameters:</p> <ul> <li> <code>beta</code>               (<code>float</code>, default:                   <code>0.2</code> )           \u2013            <p>Focal Loss <code>beta</code> parameter in their implementation</p> </li> <li> <code>gamma</code>               (<code>float</code>, default:                   <code>1.0</code> )           \u2013            <p>Focal Loss <code>gamma</code> parameter</p> </li> <li> <code>activation_fn</code>               (<code>Literal[sigmoid, tanh]</code>, default:                   <code>'sigmoid'</code> )           \u2013            <p>Activation function to be used during the computation of the loss. Possible values are 'sigmoid' and 'tanh'. See the original publication for details.</p> </li> </ul> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def __init__(\n    self,\n    beta: float = 0.2,\n    gamma: float = 1.0,\n    activation_fn: Literal[\"sigmoid\", \"tanh\"] = \"sigmoid\",\n):\n    super().__init__()\n    self.beta = beta\n    self.gamma = gamma\n    self.activation_fn = activation_fn\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.FocalR_RMSELoss.forward","title":"forward","text":"<pre><code>forward(input, target)\n</code></pre> <p>Parameters:</p> <ul> <li> <code>input</code>               (<code>Tensor</code>)           \u2013            <p>Input tensor with predictions (not probabilities)</p> </li> <li> <code>target</code>               (<code>Tensor</code>)           \u2013            <p>Target tensor with the actual classes</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt;\n&gt;&gt;&gt; from pytorch_widedeep.losses import FocalR_RMSELoss\n&gt;&gt;&gt;\n&gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n&gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n&gt;&gt;&gt; loss = FocalR_RMSELoss()(input, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(\n    self,\n    input: Tensor,\n    target: Tensor,\n) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    input: Tensor\n        Input tensor with predictions (not probabilities)\n    target: Tensor\n        Target tensor with the actual classes\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; from pytorch_widedeep.losses import FocalR_RMSELoss\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n    &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n    &gt;&gt;&gt; loss = FocalR_RMSELoss()(input, target)\n    \"\"\"\n    loss = (input - target) ** 2\n    if self.activation_fn == \"tanh\":\n        loss *= (torch.tanh(self.beta * torch.abs(input - target))) ** self.gamma\n    elif self.activation_fn == \"sigmoid\":\n        loss *= (\n            2 * torch.sigmoid(self.beta * torch.abs((input - target) ** 2)) - 1\n        ) ** self.gamma\n    else:\n        ValueError(\n            \"Incorrect activation function value - must be in ['sigmoid', 'tanh']\"\n        )\n    return torch.sqrt(torch.mean(loss))\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.HuberLoss","title":"HuberLoss","text":"<pre><code>HuberLoss(beta=0.2)\n</code></pre> <p>               Bases: <code>Module</code></p> <p>Hubbler Loss</p> <p>Based on Delving into Deep Imbalanced Regression.</p> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def __init__(self, beta: float = 0.2):\n    super().__init__()\n    self.beta = beta\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.HuberLoss.forward","title":"forward","text":"<pre><code>forward(input, target)\n</code></pre> <p>Parameters:</p> <ul> <li> <code>input</code>               (<code>Tensor</code>)           \u2013            <p>Input tensor with predictions (not probabilities)</p> </li> <li> <code>target</code>               (<code>Tensor</code>)           \u2013            <p>Target tensor with the actual classes</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt;\n&gt;&gt;&gt; from pytorch_widedeep.losses import HuberLoss\n&gt;&gt;&gt;\n&gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n&gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n&gt;&gt;&gt; loss = HuberLoss()(input, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(\n    self,\n    input: Tensor,\n    target: Tensor,\n) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    input: Tensor\n        Input tensor with predictions (not probabilities)\n    target: Tensor\n        Target tensor with the actual classes\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; from pytorch_widedeep.losses import HuberLoss\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n    &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n    &gt;&gt;&gt; loss = HuberLoss()(input, target)\n    \"\"\"\n    l1_loss = torch.abs(input - target)\n    cond = l1_loss &lt; self.beta\n    loss = torch.where(\n        cond, 0.5 * l1_loss**2 / self.beta, l1_loss - 0.5 * self.beta\n    )\n    return torch.mean(loss)\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.InfoNCELoss","title":"InfoNCELoss","text":"<pre><code>InfoNCELoss(temperature=0.1, reduction='mean')\n</code></pre> <p>               Bases: <code>Module</code></p> <p>InfoNCE Loss. Loss applied during the Contrastive Denoising Self Supervised Pre-training routine available in this library</p> <p> NOTE: This loss is in principle not exposed to  the user, as it is used internally in the library, but it is included  here for completion.</p> <p>See SAINT: Improved Neural Networks for Tabular Data via Row Attention and Contrastive Pre-Training and references therein</p> <p>Partially inspired by the code in this repo</p> <p>Parameters:</p> <ul> <li> <code>temperature</code>               (<code>float</code>, default:                   <code>0.1</code> )           \u2013            <p>The logits are divided by the temperature before computing the loss value</p> </li> <li> <code>reduction</code>               (<code>str</code>, default:                   <code>'mean'</code> )           \u2013            <p>Loss reduction method</p> </li> </ul> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def __init__(self, temperature: float = 0.1, reduction: str = \"mean\"):\n    super(InfoNCELoss, self).__init__()\n\n    self.temperature = temperature\n    self.reduction = reduction\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.InfoNCELoss.forward","title":"forward","text":"<pre><code>forward(g_projs)\n</code></pre> <p>Parameters:</p> <ul> <li> <code>g_projs</code>               (<code>Tuple[Tensor, Tensor]</code>)           \u2013            <p>Tuple with the two tensors corresponding to the output of the two projection heads, as described 'SAINT: Improved Neural Networks for Tabular Data via Row Attention and Contrastive Pre-Training'.</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.losses import InfoNCELoss\n&gt;&gt;&gt; g_projs = (torch.rand(3, 5, 16), torch.rand(3, 5, 16))\n&gt;&gt;&gt; loss = InfoNCELoss()\n&gt;&gt;&gt; res = loss(g_projs)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(self, g_projs: Tuple[Tensor, Tensor]) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    g_projs: Tuple\n        Tuple with the two tensors corresponding to the output of the two\n        projection heads, as described 'SAINT: Improved Neural Networks\n        for Tabular Data via Row Attention and Contrastive Pre-Training'.\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.losses import InfoNCELoss\n    &gt;&gt;&gt; g_projs = (torch.rand(3, 5, 16), torch.rand(3, 5, 16))\n    &gt;&gt;&gt; loss = InfoNCELoss()\n    &gt;&gt;&gt; res = loss(g_projs)\n    \"\"\"\n    z, z_ = g_projs[0], g_projs[1]\n\n    norm_z = F.normalize(z, dim=-1).flatten(1)\n    norm_z_ = F.normalize(z_, dim=-1).flatten(1)\n\n    logits = (norm_z @ norm_z_.t()) / self.temperature\n    logits_ = (norm_z_ @ norm_z.t()) / self.temperature\n\n    # the target/labels are the entries on the diagonal\n    target = torch.arange(len(norm_z), device=norm_z.device)\n\n    loss = F.cross_entropy(logits, target, reduction=self.reduction)\n    loss_ = F.cross_entropy(logits_, target, reduction=self.reduction)\n\n    return (loss + loss_) / 2.0\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.DenoisingLoss","title":"DenoisingLoss","text":"<pre><code>DenoisingLoss(\n    lambda_cat=1.0, lambda_cont=1.0, reduction=\"mean\"\n)\n</code></pre> <p>               Bases: <code>Module</code></p> <p>Denoising Loss. Loss applied during the Contrastive Denoising Self Supervised Pre-training routine available in this library</p> <p> NOTE: This loss is in principle not exposed to  the user, as it is used internally in the library, but it is included  here for completion.</p> <p>See SAINT: Improved Neural Networks for Tabular Data via Row Attention and Contrastive Pre-Training and references therein</p> <p>Parameters:</p> <ul> <li> <code>lambda_cat</code>               (<code>float</code>, default:                   <code>1.0</code> )           \u2013            <p>Multiplicative factor that will be applied to loss associated to the categorical features</p> </li> <li> <code>lambda_cont</code>               (<code>float</code>, default:                   <code>1.0</code> )           \u2013            <p>Multiplicative factor that will be applied to loss associated to the continuous features</p> </li> <li> <code>reduction</code>               (<code>str</code>, default:                   <code>'mean'</code> )           \u2013            <p>Loss reduction method</p> </li> </ul> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def __init__(\n    self, lambda_cat: float = 1.0, lambda_cont: float = 1.0, reduction: str = \"mean\"\n):\n    super(DenoisingLoss, self).__init__()\n\n    self.lambda_cat = lambda_cat\n    self.lambda_cont = lambda_cont\n    self.reduction = reduction\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.DenoisingLoss.forward","title":"forward","text":"<pre><code>forward(x_cat_and_cat_, x_cont_and_cont_)\n</code></pre> <p>Parameters:</p> <ul> <li> <code>x_cat_and_cat_</code>               (<code>Optional[Union[List[Tuple[Tensor, Tensor]], Tuple[Tensor, Tensor]]]</code>)           \u2013            <p>Tuple of tensors containing the raw input features and their encodings, referred in the SAINT paper as \\(x\\) and \\(x''\\) respectively. If one denoising MLP is used per categorical feature <code>x_cat_and_cat_</code> will be a list of tuples, one per categorical feature</p> </li> <li> <code>x_cont_and_cont_</code>               (<code>Optional[Union[List[Tuple[Tensor, Tensor]], Tuple[Tensor, Tensor]]]</code>)           \u2013            <p>same as <code>x_cat_and_cat_</code> but for continuous columns</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.losses import DenoisingLoss\n&gt;&gt;&gt; x_cat_and_cat_ = (torch.empty(3).random_(3).long(), torch.randn(3, 3))\n&gt;&gt;&gt; x_cont_and_cont_ = (torch.randn(3, 1), torch.randn(3, 1))\n&gt;&gt;&gt; loss = DenoisingLoss()\n&gt;&gt;&gt; res = loss(x_cat_and_cat_, x_cont_and_cont_)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(\n    self,\n    x_cat_and_cat_: Optional[\n        Union[List[Tuple[Tensor, Tensor]], Tuple[Tensor, Tensor]]\n    ],\n    x_cont_and_cont_: Optional[\n        Union[List[Tuple[Tensor, Tensor]], Tuple[Tensor, Tensor]]\n    ],\n) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    x_cat_and_cat_: tuple of Tensors or lists of tuples\n        Tuple of tensors containing the raw input features and their\n        encodings, referred in the SAINT paper as $x$ and $x''$\n        respectively. If one denoising MLP is used per categorical\n        feature `x_cat_and_cat_` will be a list of tuples, one per\n        categorical feature\n    x_cont_and_cont_: tuple of Tensors or lists of tuples\n        same as `x_cat_and_cat_` but for continuous columns\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.losses import DenoisingLoss\n    &gt;&gt;&gt; x_cat_and_cat_ = (torch.empty(3).random_(3).long(), torch.randn(3, 3))\n    &gt;&gt;&gt; x_cont_and_cont_ = (torch.randn(3, 1), torch.randn(3, 1))\n    &gt;&gt;&gt; loss = DenoisingLoss()\n    &gt;&gt;&gt; res = loss(x_cat_and_cat_, x_cont_and_cont_)\n    \"\"\"\n\n    loss_cat = (\n        self._compute_cat_loss(x_cat_and_cat_)\n        if x_cat_and_cat_ is not None\n        else torch.tensor(0.0)\n    )\n    loss_cont = (\n        self._compute_cont_loss(x_cont_and_cont_)\n        if x_cont_and_cont_ is not None\n        else torch.tensor(0.0)\n    )\n\n    return self.lambda_cat * loss_cat + self.lambda_cont * loss_cont\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.EncoderDecoderLoss","title":"EncoderDecoderLoss","text":"<pre><code>EncoderDecoderLoss(eps=1e-09)\n</code></pre> <p>               Bases: <code>Module</code></p> <p>'Standard' Encoder Decoder Loss. Loss applied during the Endoder-Decoder  Self-Supervised Pre-Training routine available in this library</p> <p> NOTE: This loss is in principle not exposed to  the user, as it is used internally in the library, but it is included  here for completion.</p> <p>The implementation of this lost is based on that at the tabnet repo, which is in itself an adaptation of that in the original paper TabNet: Attentive Interpretable Tabular Learning.</p> <p>Parameters:</p> <ul> <li> <code>eps</code>               (<code>float</code>, default:                   <code>1e-09</code> )           \u2013            <p>Simply a small number to avoid dividing by zero</p> </li> </ul> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def __init__(self, eps: float = 1e-9):\n    super(EncoderDecoderLoss, self).__init__()\n    self.eps = eps\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.EncoderDecoderLoss.forward","title":"forward","text":"<pre><code>forward(x_true, x_pred, mask)\n</code></pre> <p>Parameters:</p> <ul> <li> <code>x_true</code>               (<code>Tensor</code>)           \u2013            <p>Embeddings of the input data</p> </li> <li> <code>x_pred</code>               (<code>Tensor</code>)           \u2013            <p>Reconstructed embeddings</p> </li> <li> <code>mask</code>               (<code>Tensor</code>)           \u2013            <p>Mask with 1s indicated that the reconstruction, and therefore the loss, is based on those features.</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.losses import EncoderDecoderLoss\n&gt;&gt;&gt; x_true = torch.rand(3, 3)\n&gt;&gt;&gt; x_pred = torch.rand(3, 3)\n&gt;&gt;&gt; mask = torch.empty(3, 3).random_(2)\n&gt;&gt;&gt; loss = EncoderDecoderLoss()\n&gt;&gt;&gt; res = loss(x_true, x_pred, mask)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(self, x_true: Tensor, x_pred: Tensor, mask: Tensor) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    x_true: Tensor\n        Embeddings of the input data\n    x_pred: Tensor\n        Reconstructed embeddings\n    mask: Tensor\n        Mask with 1s indicated that the reconstruction, and therefore the\n        loss, is based on those features.\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.losses import EncoderDecoderLoss\n    &gt;&gt;&gt; x_true = torch.rand(3, 3)\n    &gt;&gt;&gt; x_pred = torch.rand(3, 3)\n    &gt;&gt;&gt; mask = torch.empty(3, 3).random_(2)\n    &gt;&gt;&gt; loss = EncoderDecoderLoss()\n    &gt;&gt;&gt; res = loss(x_true, x_pred, mask)\n    \"\"\"\n\n    errors = x_pred - x_true\n\n    reconstruction_errors = torch.mul(errors, mask) ** 2\n\n    x_true_means = torch.mean(x_true, dim=0)\n    x_true_means[x_true_means == 0] = 1\n\n    x_true_stds = torch.std(x_true, dim=0) ** 2\n    x_true_stds[x_true_stds == 0] = x_true_means[x_true_stds == 0]\n\n    features_loss = torch.matmul(reconstruction_errors, 1 / x_true_stds)\n    nb_reconstructed_variables = torch.sum(mask, dim=1)\n    features_loss_norm = features_loss / (nb_reconstructed_variables + self.eps)\n\n    loss = torch.mean(features_loss_norm)\n\n    return loss\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses_multitarget.MultiTargetRegressionLoss","title":"MultiTargetRegressionLoss","text":"<pre><code>MultiTargetRegressionLoss(weights=None, reduction='mean')\n</code></pre> <p>               Bases: <code>Module</code></p> <p>This class is a wrapper around the Pytorch MSELoss. It allows for multi-target regression problems. The user can provide a list of weights to apply to each target. The loss can be either the sum or the mean of the individual losses</p> <p>Parameters:</p> <ul> <li> <code>weights</code>               (<code>Optional[List[float]]</code>, default:                   <code>None</code> )           \u2013            <p>List of weights to apply to the loss associated to each target. The length of the list must match the number of targets. Alias: 'target_weights'</p> </li> <li> <code>reduction</code>               (<code>Literal[mean, sum]</code>, default:                   <code>'mean'</code> )           \u2013            <p>Specifies the reduction to apply to the loss associated to each target: 'mean' | 'sum'. Note that this is NOT the same as the reduction in the MSELoss. This reduction is applied after the loss for each target has been computed. Alias: 'target_reduction'</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.losses_multitarget import MultiTargetRegressionLoss\n&gt;&gt;&gt; input = torch.randn(3, 2)\n&gt;&gt;&gt; target = torch.randn(3, 2)\n&gt;&gt;&gt; loss = MultiTargetRegressionLoss(weights=[0.5, 0.5], reduction=\"mean\")\n&gt;&gt;&gt; output = loss(input, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses_multitarget.py</code> <pre><code>@alias(\"reduction\", [\"target_reduction\"])\n@alias(\"weights\", [\"target_weights\"])\ndef __init__(\n    self,\n    weights: Optional[List[float]] = None,\n    reduction: Literal[\"mean\", \"sum\"] = \"mean\",\n):\n    super(MultiTargetRegressionLoss, self).__init__()\n\n    self.weights = weights\n    self.reduction = reduction\n\n    if self.reduction not in [\"mean\", \"sum\"]:\n        raise ValueError(\"reduction must be either 'mean' or 'sum'\")\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses_multitarget.MultiTargetClassificationLoss","title":"MultiTargetClassificationLoss","text":"<pre><code>MultiTargetClassificationLoss(\n    binary_config=None,\n    multiclass_config=None,\n    weights=None,\n    reduction=\"mean\",\n    binary_trick=False,\n)\n</code></pre> <p>               Bases: <code>Module</code></p> <p>This class is a wrapper around the Pytorch binary_cross_entropy_with_logits and cross_entropy losses. It allows for multi-target classification problems. The user can provide a list of weights to apply to each target. The loss can be either the sum or the mean of the individual losses</p> <p>Parameters:</p> <ul> <li> <code>binary_config</code>               (<code>Optional[List[Union[int, Tuple[int, float]]]]</code>, default:                   <code>None</code> )           \u2013            <p>List of integers with the index of the target for binary classification or tuples with two elements: the index of the targets or binary classification and the positive weight for binary classification</p> </li> <li> <code>multiclass_config</code>               (<code>Optional[List[Union[Tuple[int, int], Tuple[int, int, List[float]]]]]</code>, default:                   <code>None</code> )           \u2013            <p>List of tuples with two or three elements: the index of the target and the number of classes for multiclass classification, or a tuple with the index of the target, the number of classes and a list of weights to apply to each class (i.e. the 'weight' parameter in the cross_entropy loss)</p> </li> <li> <code>weights</code>               (<code>Optional[List[float]]</code>, default:                   <code>None</code> )           \u2013            <p>List of weights to apply to the loss associated to each target. The length of the list must match the number of targets. Alias: 'target_weights'</p> </li> <li> <code>reduction</code>               (<code>Literal[mean, sum]</code>, default:                   <code>'mean'</code> )           \u2013            <p>Specifies the reduction to apply to the loss associated to each target: 'mean' | 'sum'. Note that this is NOT the same as the reduction in the cross_entropy loss or the binary_cross_entropy_with_logits. This reduction is applied after the loss for each target has been computed. Alias: 'target_reduction'</p> </li> <li> <code>binary_trick</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>If True, each target will be considered independently and the loss will be computed as binary_cross_entropy_with_logits. This is a faster implementation. Note that the 'weights' parameter is not compatible with binary_trick=True. Also note that if binary_trick=True, the 'binary_config' must be a list of integers and the 'multiclass_config' must be a list of tuples with two integers: the index of the target and the number of classes. Finally, if binary_trick=True, the binary targets must be the first targets in the target tensor.</p> <p> NOTE: When using the binary_trick, the binary targets are   considered as 2 classes. Therefore, the pred_dim parametere of the   WideDeep class should be adjusted accordingly (adding 2 to per   binary target). For example, in a problem with a binary target and   a 4 class multiclassification target, the pred_dim should be 6.</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.losses_multitarget import MultiTargetClassificationLoss\n&gt;&gt;&gt; input = torch.randn(5, 4)\n&gt;&gt;&gt; input_binary_trick = torch.randn(5, 5)\n&gt;&gt;&gt; target = torch.stack([torch.tensor([0, 1, 0, 1, 1]), torch.tensor([0, 1, 2, 0, 2])], 1)\n&gt;&gt;&gt; loss_1 = MultiTargetClassificationLoss(binary_config=[0], multiclass_config=[(1, 3)], reduction=\"mean\")\n&gt;&gt;&gt; output_1 = loss_1(input, target)\n&gt;&gt;&gt; loss_2 = MultiTargetClassificationLoss(binary_config=[(0, 0.5)], multiclass_config=[(1, 3, [1., 2., 3.])],\n... reduction=\"sum\", weights=[0.5, 0.5])\n&gt;&gt;&gt; output_2 = loss_2(input, target)\n&gt;&gt;&gt; loss_3 = MultiTargetClassificationLoss(binary_config=[0], multiclass_config=[(1, 3)], binary_trick=True)\n&gt;&gt;&gt; output_3 = loss_3(input_binary_trick, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses_multitarget.py</code> <pre><code>@alias(\"reduction\", [\"target_reduction\"])\n@alias(\"weights\", [\"target_weights\"])\ndef __init__(  # noqa: C901\n    self,\n    binary_config: Optional[List[Union[int, Tuple[int, float]]]] = None,\n    multiclass_config: Optional[\n        List[Union[Tuple[int, int], Tuple[int, int, List[float]]]]\n    ] = None,\n    weights: Optional[List[float]] = None,\n    reduction: Literal[\"mean\", \"sum\"] = \"mean\",\n    binary_trick: bool = False,\n):\n    super(MultiTargetClassificationLoss, self).__init__()\n\n    if reduction not in [\"mean\", \"sum\"]:\n        raise ValueError(\"reduction must be either 'mean' or 'sum'\")\n\n    self.binary_config = binary_config\n    self.multiclass_config = multiclass_config\n    self.weights = weights\n    self.reduction = reduction\n    self.binary_trick = binary_trick\n\n    if self.weights is not None:\n        if len(self.weights) != (\n            len(self.binary_config) if self.binary_config is not None else 0\n        ) + (\n            len(self.multiclass_config) if self.multiclass_config is not None else 0\n        ):\n            raise ValueError(\n                \"The number of weights must match the number of binary and multiclass targets\"\n            )\n\n    if self.binary_trick:\n        self._check_inputs_with_binary_trick()\n        self._binary_config: List[int] = binary_config  # type: ignore[assignment]\n        self._multiclass_config: List[Tuple[int, int]] = self.multiclass_config  # type: ignore[assignment]\n    else:\n        self.binary_config_with_pos_weights = (\n            (self._set_binary_config_without_binary_trick())\n            if self.binary_config is not None\n            else None\n        )\n        self.multiclass_config_with_weights = (\n            (self._set_multiclass_config_without_binary_trick())\n            if self.multiclass_config is not None\n            else None\n        )\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses_multitarget.MutilTargetRegressionAndClassificationLoss","title":"MutilTargetRegressionAndClassificationLoss","text":"<pre><code>MutilTargetRegressionAndClassificationLoss(\n    regression_config=[],\n    binary_config=None,\n    multiclass_config=None,\n    weights=None,\n    reduction=\"mean\",\n    binary_trick=False,\n)\n</code></pre> <p>               Bases: <code>Module</code></p> <p>This class is a wrapper around the MultiTargetRegressionLoss and the MultiTargetClassificationLoss. It allows for multi-target regression and classification problems. The user can provide a list of weights to apply to each target. The loss can be either the sum or the mean of the individual losses</p> <p>Parameters:</p> <ul> <li> <code>regression_config</code>               (<code>List[int]</code>, default:                   <code>[]</code> )           \u2013            <p>List of integers with the indices of the regression targets</p> </li> <li> <code>binary_config</code>               (<code>Optional[List[Union[int, Tuple[int, float]]]]</code>, default:                   <code>None</code> )           \u2013            <p>List of integers with the index of the target for binary classification or tuples with two elements: the index of the targets or binary classification and the positive weight for binary classification</p> </li> <li> <code>multiclass_config</code>               (<code>Optional[List[Union[Tuple[int, int], Tuple[int, int, List[float]]]]]</code>, default:                   <code>None</code> )           \u2013            <p>List of tuples with two or three elements: the index of the target and the number of classes for multiclass classification, or a tuple with the index of the target, the number of classes and a list of weights to apply to each class (i.e. the 'weight' parameter in the cross_entropy loss)</p> </li> <li> <code>weights</code>               (<code>Optional[List[float]]</code>, default:                   <code>None</code> )           \u2013            <p>List of weights to apply to the loss associated to each target. The length of the list must match the number of targets. Alias: 'target_weights'</p> </li> <li> <code>reduction</code>               (<code>Literal[mean, sum]</code>, default:                   <code>'mean'</code> )           \u2013            <p>Specifies the reduction to apply to the output: 'mean' | 'sum'. Note that this is NOT the same as the reduction in the cross_entropy loss, the binary_cross_entropy_with_logits or the MSELoss. This reduction is applied after each target has been computed. Alias: 'target_reduction'</p> </li> <li> <code>binary_trick</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>If True, each target will be considered independently and the loss will be computed as binary_cross_entropy_with_logits. This is a faster implementation. Note that the 'weights' parameter is not compatible with binary_trick=True. Also note that if binary_trick=True, the 'binary_config' must be a list of integers and the 'multiclass_config' must be a list of tuples with two integers: the index of the target and the number of classes. Finally, if binary_trick=True, the binary targets must be the first targets in the target tensor.</p> <p> NOTE: When using the binary_trick, the binary targets are   considered as 2 classes. Therefore, the pred_dim parametere of the   WideDeep class should be adjusted accordingly (adding 2 to per   binary target). For example, in a problem with a binary target and   a 4 class multiclassification target, the pred_dim should be 6.</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.losses_multitarget import MutilTargetRegressionAndClassificationLoss\n&gt;&gt;&gt; input = torch.randn(5, 5)\n&gt;&gt;&gt; target = torch.stack([torch.randn(5), torch.tensor([0, 1, 0, 1, 1]), torch.tensor([0, 1, 2, 0, 2])], 1)\n&gt;&gt;&gt; loss = MutilTargetRegressionAndClassificationLoss(regression_config=[0], binary_config=[2],\n... multiclass_config=[(2, 3)], reduction=\"mean\")\n&gt;&gt;&gt; output = loss(input, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses_multitarget.py</code> <pre><code>@alias(\"reduction\", [\"target_reduction\"])\n@alias(\"weights\", [\"target_weights\"])\ndef __init__(  # noqa: C901\n    self,\n    regression_config: List[int] = [],\n    binary_config: Optional[List[Union[int, Tuple[int, float]]]] = None,\n    multiclass_config: Optional[\n        List[Union[Tuple[int, int], Tuple[int, int, List[float]]]]\n    ] = None,\n    weights: Optional[List[float]] = None,\n    reduction: Literal[\"mean\", \"sum\"] = \"mean\",\n    binary_trick: bool = False,\n):\n\n    super(MutilTargetRegressionAndClassificationLoss, self).__init__()\n\n    self.regression_config = regression_config\n\n    assert binary_config is not None or multiclass_config is not None, (\n        \"Either binary_config or multiclass_config must be provided. \"\n        \"Otherwise, use the MultiTargetRegressionLoss\"\n    )\n\n    if binary_trick:\n        self._check_inputs_with_binary_trick(\n            regression_config, binary_config, multiclass_config\n        )\n\n    if weights is not None:\n        if len(weights) != (\n            len(regression_config)\n            + (len(binary_config) if binary_config is not None else 0)\n            + (len(multiclass_config) if multiclass_config is not None else 0)\n        ):\n            raise ValueError(\n                \"The number of weights must match the number of regression, binary and multiclass targets\"\n            )\n\n        self.weights_regression = self._prepare_weights_for_regression_targets(\n            weights, regression_config\n        )\n        self.weights_binary = self._prepare_weights_per_binary_targets(\n            weights, binary_config\n        )\n        self.weights_multiclass = self._prepare_weights_per_multiclass_targets(\n            weights, multiclass_config\n        )\n        self.weights = weights\n    else:\n        self.weights_regression = None\n        self.weights_binary = None\n        self.weights_multiclass = None\n\n    self.multi_target_regression_loss = MultiTargetRegressionLoss(\n        weights=self.weights_regression, reduction=reduction\n    )\n\n    self.multi_target_classification_loss = MultiTargetClassificationLoss(\n        binary_config=binary_config,\n        multiclass_config=multiclass_config,\n        weights=(\n            self.weights_binary + self.weights_multiclass\n            if self.weights_binary is not None\n            and self.weights_multiclass is not None\n            else (\n                self.weights_binary\n                if self.weights_binary is not None\n                else self.weights_multiclass\n            )\n        ),\n        reduction=reduction,\n        binary_trick=binary_trick,\n    )\n</code></pre>"},{"location":"pytorch-widedeep/metrics.html","title":"Metrics","text":"<p> NOTE: metrics in this module expect the predictions  and ground truth to have the same dimensions for regression and binary  classification problems: \\((N_{samples}, 1)\\). In the case of multiclass  classification problems the ground truth is expected to be a 1D tensor with  the corresponding classes. See Examples below</p> <p>We have added the possibility of using the metrics available at the torchmetrics library. Note that this library is still in its early versions and therefore this option should be used with caution. To use <code>torchmetrics</code> simply import them and use them as any of the <code>pytorch-widedeep</code> metrics described below.</p> <pre><code>from torchmetrics import Accuracy, Precision\n\naccuracy = Accuracy(average=None, num_classes=2)\nprecision = Precision(average='micro', num_classes=2)\n\ntrainer = Trainer(model, objective=\"binary\", metrics=[accuracy, precision])\n</code></pre> <p>A functioning example for <code>pytorch-widedeep</code> using <code>torchmetrics</code> can be found in the Examples folder</p> <p> NOTE: the forward method for all metrics in this  module takes two tensors, <code>y_pred</code> and <code>y_true</code> (in that order). Therefore,  we do not include the method in the documentation.</p>"},{"location":"pytorch-widedeep/metrics.html#pytorch_widedeep.metrics.Accuracy","title":"Accuracy","text":"<pre><code>Accuracy(top_k=1)\n</code></pre> <p>               Bases: <code>Metric</code></p> <p>Class to calculate the accuracy for both binary and categorical problems</p> <p>Parameters:</p> <ul> <li> <code>top_k</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>Accuracy will be computed using the top k most likely classes in multiclass problems</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt;\n&gt;&gt;&gt; from pytorch_widedeep.metrics import Accuracy\n&gt;&gt;&gt;\n&gt;&gt;&gt; acc = Accuracy()\n&gt;&gt;&gt; y_true = torch.tensor([0, 1, 0, 1]).view(-1, 1)\n&gt;&gt;&gt; y_pred = torch.tensor([[0.3, 0.2, 0.6, 0.7]]).view(-1, 1)\n&gt;&gt;&gt; acc(y_pred, y_true)\narray(0.5)\n&gt;&gt;&gt;\n&gt;&gt;&gt; acc = Accuracy(top_k=2)\n&gt;&gt;&gt; y_true = torch.tensor([0, 1, 2])\n&gt;&gt;&gt; y_pred = torch.tensor([[0.3, 0.5, 0.2], [0.1, 0.1, 0.8], [0.1, 0.5, 0.4]])\n&gt;&gt;&gt; acc(y_pred, y_true)\narray(0.66666667)\n</code></pre> Source code in <code>pytorch_widedeep/metrics.py</code> <pre><code>def __init__(self, top_k: int = 1):\n    super(Accuracy, self).__init__()\n\n    self.top_k = top_k\n    self.correct_count = 0\n    self.total_count = 0\n    self._name = \"acc\"\n</code></pre>"},{"location":"pytorch-widedeep/metrics.html#pytorch_widedeep.metrics.Accuracy.reset","title":"reset","text":"<pre><code>reset()\n</code></pre> <p>resets counters to 0</p> Source code in <code>pytorch_widedeep/metrics.py</code> <pre><code>def reset(self):\n    \"\"\"\n    resets counters to 0\n    \"\"\"\n    self.correct_count = 0\n    self.total_count = 0\n</code></pre>"},{"location":"pytorch-widedeep/metrics.html#pytorch_widedeep.metrics.Precision","title":"Precision","text":"<pre><code>Precision(average=True)\n</code></pre> <p>               Bases: <code>Metric</code></p> <p>Class to calculate the precision for both binary and categorical problems</p> <p>Parameters:</p> <ul> <li> <code>average</code>               (<code>bool</code>, default:                   <code>True</code> )           \u2013            <p>This applies only to multiclass problems. if <code>True</code> calculate precision for each label, and finds their unweighted mean.</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt;\n&gt;&gt;&gt; from pytorch_widedeep.metrics import Precision\n&gt;&gt;&gt;\n&gt;&gt;&gt; prec = Precision()\n&gt;&gt;&gt; y_true = torch.tensor([0, 1, 0, 1]).view(-1, 1)\n&gt;&gt;&gt; y_pred = torch.tensor([[0.3, 0.2, 0.6, 0.7]]).view(-1, 1)\n&gt;&gt;&gt; prec(y_pred, y_true)\narray(0.5)\n&gt;&gt;&gt;\n&gt;&gt;&gt; prec = Precision(average=True)\n&gt;&gt;&gt; y_true = torch.tensor([0, 1, 2])\n&gt;&gt;&gt; y_pred = torch.tensor([[0.7, 0.1, 0.2], [0.1, 0.1, 0.8], [0.1, 0.5, 0.4]])\n&gt;&gt;&gt; prec(y_pred, y_true)\narray(0.33333334)\n</code></pre> Source code in <code>pytorch_widedeep/metrics.py</code> <pre><code>def __init__(self, average: bool = True):\n    super(Precision, self).__init__()\n\n    self.average = average\n    self.true_positives = 0\n    self.all_positives = 0\n    self.eps = 1e-20\n    self._name = \"prec\"\n</code></pre>"},{"location":"pytorch-widedeep/metrics.html#pytorch_widedeep.metrics.Precision.reset","title":"reset","text":"<pre><code>reset()\n</code></pre> <p>resets counters to 0</p> Source code in <code>pytorch_widedeep/metrics.py</code> <pre><code>def reset(self):\n    \"\"\"\n    resets counters to 0\n    \"\"\"\n    self.true_positives = 0\n    self.all_positives = 0\n</code></pre>"},{"location":"pytorch-widedeep/metrics.html#pytorch_widedeep.metrics.Recall","title":"Recall","text":"<pre><code>Recall(average=True)\n</code></pre> <p>               Bases: <code>Metric</code></p> <p>Class to calculate the recall for both binary and categorical problems</p> <p>Parameters:</p> <ul> <li> <code>average</code>               (<code>bool</code>, default:                   <code>True</code> )           \u2013            <p>This applies only to multiclass problems. if <code>True</code> calculate recall for each label, and finds their unweighted mean.</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt;\n&gt;&gt;&gt; from pytorch_widedeep.metrics import Recall\n&gt;&gt;&gt;\n&gt;&gt;&gt; rec = Recall()\n&gt;&gt;&gt; y_true = torch.tensor([0, 1, 0, 1]).view(-1, 1)\n&gt;&gt;&gt; y_pred = torch.tensor([[0.3, 0.2, 0.6, 0.7]]).view(-1, 1)\n&gt;&gt;&gt; rec(y_pred, y_true)\narray(0.5)\n&gt;&gt;&gt;\n&gt;&gt;&gt; rec = Recall(average=True)\n&gt;&gt;&gt; y_true = torch.tensor([0, 1, 2])\n&gt;&gt;&gt; y_pred = torch.tensor([[0.7, 0.1, 0.2], [0.1, 0.1, 0.8], [0.1, 0.5, 0.4]])\n&gt;&gt;&gt; rec(y_pred, y_true)\narray(0.33333334)\n</code></pre> Source code in <code>pytorch_widedeep/metrics.py</code> <pre><code>def __init__(self, average: bool = True):\n    super(Recall, self).__init__()\n\n    self.average = average\n    self.true_positives = 0\n    self.actual_positives = 0\n    self.eps = 1e-20\n    self._name = \"rec\"\n</code></pre>"},{"location":"pytorch-widedeep/metrics.html#pytorch_widedeep.metrics.Recall.reset","title":"reset","text":"<pre><code>reset()\n</code></pre> <p>resets counters to 0</p> Source code in <code>pytorch_widedeep/metrics.py</code> <pre><code>def reset(self):\n    \"\"\"\n    resets counters to 0\n    \"\"\"\n    self.true_positives = 0\n    self.actual_positives = 0\n</code></pre>"},{"location":"pytorch-widedeep/metrics.html#pytorch_widedeep.metrics.FBetaScore","title":"FBetaScore","text":"<pre><code>FBetaScore(beta, average=True)\n</code></pre> <p>               Bases: <code>Metric</code></p> <p>Class to calculate the fbeta score for both binary and categorical problems</p> \\[ F_{\\beta} = ((1 + {\\beta}^2) * \\frac{(precision * recall)}{({\\beta}^2 * precision + recall)} \\] <p>Parameters:</p> <ul> <li> <code>beta</code>               (<code>int</code>)           \u2013            <p>Coefficient to control the balance between precision and recall</p> </li> <li> <code>average</code>               (<code>bool</code>, default:                   <code>True</code> )           \u2013            <p>This applies only to multiclass problems. if <code>True</code> calculate fbeta for each label, and find their unweighted mean.</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt;\n&gt;&gt;&gt; from pytorch_widedeep.metrics import FBetaScore\n&gt;&gt;&gt;\n&gt;&gt;&gt; fbeta = FBetaScore(beta=2)\n&gt;&gt;&gt; y_true = torch.tensor([0, 1, 0, 1]).view(-1, 1)\n&gt;&gt;&gt; y_pred = torch.tensor([[0.3, 0.2, 0.6, 0.7]]).view(-1, 1)\n&gt;&gt;&gt; fbeta(y_pred, y_true)\narray(0.5)\n&gt;&gt;&gt;\n&gt;&gt;&gt; fbeta = FBetaScore(beta=2)\n&gt;&gt;&gt; y_true = torch.tensor([0, 1, 2])\n&gt;&gt;&gt; y_pred = torch.tensor([[0.7, 0.1, 0.2], [0.1, 0.1, 0.8], [0.1, 0.5, 0.4]])\n&gt;&gt;&gt; fbeta(y_pred, y_true)\narray(0.33333334)\n</code></pre> Source code in <code>pytorch_widedeep/metrics.py</code> <pre><code>def __init__(self, beta: int, average: bool = True):\n    super(FBetaScore, self).__init__()\n\n    self.beta = beta\n    self.average = average\n    self.precision = Precision(average=False)\n    self.recall = Recall(average=False)\n    self.eps = 1e-20\n    self._name = \"\".join([\"f\", str(self.beta)])\n</code></pre>"},{"location":"pytorch-widedeep/metrics.html#pytorch_widedeep.metrics.FBetaScore.reset","title":"reset","text":"<pre><code>reset()\n</code></pre> <p>resets precision and recall</p> Source code in <code>pytorch_widedeep/metrics.py</code> <pre><code>def reset(self):\n    \"\"\"\n    resets precision and recall\n    \"\"\"\n    self.precision.reset()\n    self.recall.reset()\n</code></pre>"},{"location":"pytorch-widedeep/metrics.html#pytorch_widedeep.metrics.F1Score","title":"F1Score","text":"<pre><code>F1Score(average=True)\n</code></pre> <p>               Bases: <code>Metric</code></p> <p>Class to calculate the f1 score for both binary and categorical problems</p> <p>Parameters:</p> <ul> <li> <code>average</code>               (<code>bool</code>, default:                   <code>True</code> )           \u2013            <p>This applies only to multiclass problems. if <code>True</code> calculate f1 for each label, and find their unweighted mean.</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt;\n&gt;&gt;&gt; from pytorch_widedeep.metrics import F1Score\n&gt;&gt;&gt;\n&gt;&gt;&gt; f1 = F1Score()\n&gt;&gt;&gt; y_true = torch.tensor([0, 1, 0, 1]).view(-1, 1)\n&gt;&gt;&gt; y_pred = torch.tensor([[0.3, 0.2, 0.6, 0.7]]).view(-1, 1)\n&gt;&gt;&gt; f1(y_pred, y_true)\narray(0.5)\n&gt;&gt;&gt;\n&gt;&gt;&gt; f1 = F1Score()\n&gt;&gt;&gt; y_true = torch.tensor([0, 1, 2])\n&gt;&gt;&gt; y_pred = torch.tensor([[0.7, 0.1, 0.2], [0.1, 0.1, 0.8], [0.1, 0.5, 0.4]])\n&gt;&gt;&gt; f1(y_pred, y_true)\narray(0.33333334)\n</code></pre> Source code in <code>pytorch_widedeep/metrics.py</code> <pre><code>def __init__(self, average: bool = True):\n    super(F1Score, self).__init__()\n\n    self.average = average\n    self.f1 = FBetaScore(beta=1, average=self.average)\n    self._name = self.f1._name\n</code></pre>"},{"location":"pytorch-widedeep/metrics.html#pytorch_widedeep.metrics.F1Score.reset","title":"reset","text":"<pre><code>reset()\n</code></pre> <p>resets counters to 0</p> Source code in <code>pytorch_widedeep/metrics.py</code> <pre><code>def reset(self):\n    \"\"\"\n    resets counters to 0\n    \"\"\"\n    self.f1.reset()\n</code></pre>"},{"location":"pytorch-widedeep/metrics.html#pytorch_widedeep.metrics.R2Score","title":"R2Score","text":"<pre><code>R2Score()\n</code></pre> <p>               Bases: <code>Metric</code></p> <p>Calculates R-Squared, the coefficient of determination:</p> \\[ R^2 = 1 - \\frac{\\sum_{j=1}^n(y_j - \\hat{y_j})^2}{\\sum_{j=1}^n(y_j - \\bar{y})^2} \\] <p>where \\(\\hat{y_j}\\) is the ground truth, \\(y_j\\) is the predicted value and \\(\\bar{y}\\) is the mean of the ground truth.</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt;\n&gt;&gt;&gt; from pytorch_widedeep.metrics import R2Score\n&gt;&gt;&gt;\n&gt;&gt;&gt; r2 = R2Score()\n&gt;&gt;&gt; y_true = torch.tensor([3, -0.5, 2, 7]).view(-1, 1)\n&gt;&gt;&gt; y_pred = torch.tensor([2.5, 0.0, 2, 8]).view(-1, 1)\n&gt;&gt;&gt; r2(y_pred, y_true)\narray(0.94860814)\n</code></pre> Source code in <code>pytorch_widedeep/metrics.py</code> <pre><code>def __init__(self):\n    self.numerator = 0\n    self.denominator = 0\n    self.num_examples = 0\n    self.y_true_sum = 0\n\n    self._name = \"r2\"\n</code></pre>"},{"location":"pytorch-widedeep/metrics.html#pytorch_widedeep.metrics.R2Score.reset","title":"reset","text":"<pre><code>reset()\n</code></pre> <p>resets counters to 0</p> Source code in <code>pytorch_widedeep/metrics.py</code> <pre><code>def reset(self):\n    \"\"\"\n    resets counters to 0\n    \"\"\"\n    self.numerator = 0\n    self.denominator = 0\n    self.num_examples = 0\n    self.y_true_sum = 0\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html","title":"The <code>models</code> module","text":"<p>This module contains the models that can be used as the four main components that will comprise a Wide and Deep model (<code>wide</code>, <code>deeptabular</code>, <code>deeptext</code>, <code>deepimage</code>), as well as the <code>WideDeep</code> \"constructor\" class. Note that each of the four components can be used independently. It also contains all the documentation for the models that can be used for self-supervised pre-training with tabular data.</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.linear.wide.Wide","title":"Wide","text":"<pre><code>Wide(input_dim, pred_dim=1)\n</code></pre> <p>               Bases: <code>Module</code></p> <p>Defines a <code>Wide</code> (linear) model where the non-linearities are captured via the so-called crossed-columns. This can be used as the <code>wide</code> component of a Wide &amp; Deep model.</p> <p>Parameters:</p> <ul> <li> <code>input_dim</code>               (<code>int</code>)           \u2013            <p>size of the Linear layer (implemented via an Embedding layer). <code>input_dim</code> is the summation of all the individual values for all the features that go through the wide model. For example, if the wide model receives 2 features with 5 individual values each, <code>input_dim = 10</code></p> </li> <li> <code>pred_dim</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>size of the ouput tensor containing the predictions. Note that unlike all the other models, the wide model is connected directly to the output neuron(s) when used to build a Wide and Deep model. Therefore, it requires the <code>pred_dim</code> parameter.</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>wide_linear</code>               (<code>Module</code>)           \u2013            <p>the linear layer that comprises the wide branch of the model</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import Wide\n&gt;&gt;&gt; X = torch.empty(4, 4).random_(20)\n&gt;&gt;&gt; wide = Wide(input_dim=int(X.max().item()), pred_dim=1)\n&gt;&gt;&gt; out = wide(X)\n</code></pre> Source code in <code>pytorch_widedeep/models/tabular/linear/wide.py</code> <pre><code>@alias(\"pred_dim\", [\"pred_size\", \"num_class\"])\ndef __init__(self, input_dim: int, pred_dim: int = 1):\n    super(Wide, self).__init__()\n\n    self.input_dim = input_dim\n    self.pred_dim = pred_dim\n\n    # Embeddings: val + 1 because 0 is reserved for padding/unseen cateogories.\n    self.wide_linear = nn.Embedding(input_dim + 1, pred_dim, padding_idx=0)\n    # (Sum(Embedding) + bias) is equivalent to (OneHotVector + Linear)\n    self.bias = nn.Parameter(torch.zeros(pred_dim))\n    self._reset_parameters()\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.linear.wide.Wide.forward","title":"forward","text":"<pre><code>forward(X)\n</code></pre> <p>Forward pass. Simply connecting the Embedding layer with the ouput neuron(s)</p> Source code in <code>pytorch_widedeep/models/tabular/linear/wide.py</code> <pre><code>def forward(self, X: Tensor) -&gt; Tensor:\n    r\"\"\"Forward pass. Simply connecting the Embedding layer with the ouput\n    neuron(s)\"\"\"\n    out = self.wide_linear(X.long()).sum(dim=1) + self.bias\n    return out\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.mlp.tab_mlp.TabMlp","title":"TabMlp","text":"<pre><code>TabMlp(\n    column_idx,\n    *,\n    cat_embed_input=None,\n    cat_embed_dropout=None,\n    use_cat_bias=None,\n    cat_embed_activation=None,\n    continuous_cols=None,\n    cont_norm_layer=None,\n    embed_continuous=None,\n    embed_continuous_method=None,\n    cont_embed_dim=None,\n    cont_embed_dropout=None,\n    cont_embed_activation=None,\n    quantization_setup=None,\n    n_frequencies=None,\n    sigma=None,\n    share_last_layer=None,\n    full_embed_dropout=None,\n    mlp_hidden_dims=[200, 100],\n    mlp_activation=\"relu\",\n    mlp_dropout=0.1,\n    mlp_batchnorm=False,\n    mlp_batchnorm_last=False,\n    mlp_linear_first=True\n)\n</code></pre> <p>               Bases: <code>BaseTabularModelWithoutAttention</code></p> <p>Defines a <code>TabMlp</code> model that can be used as the <code>deeptabular</code> component of a Wide &amp; Deep model or independently by itself.</p> <p>This class combines embedding representations of the categorical features with numerical (aka continuous) features, embedded or not. These are then passed through a series of dense layers (i.e. a MLP).</p> <p>Most of the parameters for this class are <code>Optional</code> since the use of categorical or continuous is in fact optional (i.e. one can use categorical features only, continuous features only or both).</p> <p>Parameters:</p> <ul> <li> <code>column_idx</code>               (<code>Dict[str, int]</code>)           \u2013            <p>Dict containing the index of the columns that will be passed through the <code>TabMlp</code> model. Required to slice the tensors. e.g. {'education': 0, 'relationship': 1, 'workclass': 2, ...}.</p> </li> <li> <code>cat_embed_input</code>               (<code>Optional[List[Tuple[str, int, int]]]</code>, default:                   <code>None</code> )           \u2013            <p>List of Tuples with the column name, number of unique values and embedding dimension. e.g. [(education, 11, 32), ...]</p> </li> <li> <code>cat_embed_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Categorical embeddings dropout. If <code>None</code>, it will default to 0.</p> </li> <li> <code>use_cat_bias</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating if bias will be used for the categorical embeddings. If <code>None</code>, it will default to 'False'.</p> </li> <li> <code>cat_embed_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the categorical embeddings, if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> </li> <li> <code>continuous_cols</code>               (<code>Optional[List[str]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the name of the numeric (aka continuous) columns</p> </li> <li> <code>cont_norm_layer</code>               (<code>Optional[Literal[batchnorm, layernorm]]</code>, default:                   <code>None</code> )           \u2013            <p>Type of normalization layer applied to the continuous features. Options are: 'layernorm' and 'batchnorm'. if <code>None</code>, no normalization layer will be used.</p> </li> <li> <code>embed_continuous</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating if the continuous columns will be embedded using one of the available methods: 'standard', 'periodic' or 'piecewise'. If <code>None</code>, it will default to 'False'. NOTE: This parameter is deprecated and it  will be removed in future releases. Please, use the  <code>embed_continuous_method</code> parameter instead.</p> </li> <li> <code>embed_continuous_method</code>               (<code>Optional[Literal[standard, piecewise, periodic]]</code>, default:                   <code>None</code> )           \u2013            <p>Method to use to embed the continuous features. Options are: 'standard', 'periodic' or 'piecewise'. The 'standard' embedding method is based on the FT-Transformer implementation presented in the paper: Revisiting Deep Learning Models for Tabular Data. The 'periodic' and_'piecewise'_ methods were presented in the paper: On Embeddings for Numerical Features in Tabular Deep Learning. Please, read the papers for details.</p> </li> <li> <code>cont_embed_dim</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>Size of the continuous embeddings. If the continuous columns are embedded, <code>cont_embed_dim</code> must be passed.</p> </li> <li> <code>cont_embed_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p> </li> <li> <code>cont_embed_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the continuous embeddings if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported. If <code>None</code>, no activation function will be applied.</p> </li> <li> <code>quantization_setup</code>               (<code>Optional[Dict[str, List[float]]]</code>, default:                   <code>None</code> )           \u2013            <p>This parameter is used when the 'piecewise' method is used to embed the continuous cols. It is a dict where keys are the name of the continuous columns and values are lists with the boundaries for the quantization of the continuous_cols. See the examples for details. If If the 'piecewise' method is used, this parameter is required.</p> </li> <li> <code>n_frequencies</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>This is the so called 'k' in their paper On Embeddings for Numerical Features in Tabular Deep Learning, and is the number of 'frequencies' that will be used to represent each continuous column. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>sigma</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>This is the sigma parameter in the paper mentioned when describing the previous parameters and it is used to initialise the 'frequency weights'. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>share_last_layer</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>This parameter is not present in the before mentioned paper but it is implemented in the official repo. If <code>True</code> the linear layer that turns the frequencies into embeddings will be shared across the continuous columns. If <code>False</code> a different linear layer will be used for each continuous column. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>full_embed_dropout</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>If <code>True</code>, the full embedding corresponding to a column will be masked out/dropout. If <code>None</code>, it will default to <code>False</code>.</p> </li> <li> <code>mlp_hidden_dims</code>               (<code>List[int]</code>, default:                   <code>[200, 100]</code> )           \u2013            <p>List with the number of neurons per dense layer in the mlp.</p> </li> <li> <code>mlp_activation</code>               (<code>str</code>, default:                   <code>'relu'</code> )           \u2013            <p>Activation function for the dense layers of the MLP. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> </li> <li> <code>mlp_dropout</code>               (<code>Union[float, List[float]]</code>, default:                   <code>0.1</code> )           \u2013            <p>float or List of floats with the dropout between the dense layers. e.g: [0.5,0.5]</p> </li> <li> <code>mlp_batchnorm</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether or not batch normalization will be applied to the dense layers</p> </li> <li> <code>mlp_batchnorm_last</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether or not batch normalization will be applied to the last of the dense layers</p> </li> <li> <code>mlp_linear_first</code>               (<code>bool</code>, default:                   <code>True</code> )           \u2013            <p>Boolean indicating the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code></p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>encoder</code>               (<code>Module</code>)           \u2013            <p>mlp model that will receive the concatenation of the embeddings and the continuous columns</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import TabMlp\n&gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n&gt;&gt;&gt; colnames = [\"a\", \"b\", \"c\", \"d\", \"e\"]\n&gt;&gt;&gt; cat_embed_input = [(u, i, j) for u, i, j in zip(colnames[:4], [4] * 4, [8] * 4)]\n&gt;&gt;&gt; column_idx = {k: v for v, k in enumerate(colnames)}\n&gt;&gt;&gt; model = TabMlp(mlp_hidden_dims=[8, 4], column_idx=column_idx, cat_embed_input=cat_embed_input,\n... continuous_cols=[\"e\"])\n&gt;&gt;&gt; out = model(X_tab)\n</code></pre> Source code in <code>pytorch_widedeep/models/tabular/mlp/tab_mlp.py</code> <pre><code>def __init__(\n    self,\n    column_idx: Dict[str, int],\n    *,\n    cat_embed_input: Optional[List[Tuple[str, int, int]]] = None,\n    cat_embed_dropout: Optional[float] = None,\n    use_cat_bias: Optional[bool] = None,\n    cat_embed_activation: Optional[str] = None,\n    continuous_cols: Optional[List[str]] = None,\n    cont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\n    embed_continuous: Optional[bool] = None,\n    embed_continuous_method: Optional[\n        Literal[\"standard\", \"piecewise\", \"periodic\"]\n    ] = None,\n    cont_embed_dim: Optional[int] = None,\n    cont_embed_dropout: Optional[float] = None,\n    cont_embed_activation: Optional[str] = None,\n    quantization_setup: Optional[Dict[str, List[float]]] = None,\n    n_frequencies: Optional[int] = None,\n    sigma: Optional[float] = None,\n    share_last_layer: Optional[bool] = None,\n    full_embed_dropout: Optional[bool] = None,\n    mlp_hidden_dims: List[int] = [200, 100],\n    mlp_activation: str = \"relu\",\n    mlp_dropout: Union[float, List[float]] = 0.1,\n    mlp_batchnorm: bool = False,\n    mlp_batchnorm_last: bool = False,\n    mlp_linear_first: bool = True,\n):\n    super(TabMlp, self).__init__(\n        column_idx=column_idx,\n        cat_embed_input=cat_embed_input,\n        cat_embed_dropout=cat_embed_dropout,\n        use_cat_bias=use_cat_bias,\n        cat_embed_activation=cat_embed_activation,\n        continuous_cols=continuous_cols,\n        cont_norm_layer=cont_norm_layer,\n        embed_continuous=embed_continuous,\n        embed_continuous_method=embed_continuous_method,\n        cont_embed_dim=cont_embed_dim,\n        cont_embed_dropout=cont_embed_dropout,\n        cont_embed_activation=cont_embed_activation,\n        quantization_setup=quantization_setup,\n        n_frequencies=n_frequencies,\n        sigma=sigma,\n        share_last_layer=share_last_layer,\n        full_embed_dropout=full_embed_dropout,\n    )\n\n    self.mlp_hidden_dims = mlp_hidden_dims\n    self.mlp_activation = mlp_activation\n    self.mlp_dropout = mlp_dropout\n    self.mlp_batchnorm = mlp_batchnorm\n    self.mlp_batchnorm_last = mlp_batchnorm_last\n    self.mlp_linear_first = mlp_linear_first\n\n    # Embeddings are instantiated at the base model\n    # Mlp\n    mlp_input_dim = self.cat_out_dim + self.cont_out_dim\n    mlp_hidden_dims = [mlp_input_dim] + mlp_hidden_dims\n    self.encoder = MLP(\n        mlp_hidden_dims,\n        mlp_activation,\n        mlp_dropout,\n        mlp_batchnorm,\n        mlp_batchnorm_last,\n        mlp_linear_first,\n    )\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.mlp.tab_mlp.TabMlp.output_dim","title":"output_dim  <code>property</code>","text":"<pre><code>output_dim\n</code></pre> <p>The output dimension of the model. This is a required property neccesary to build the <code>WideDeep</code> class</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.mlp.tab_mlp.TabMlpDecoder","title":"TabMlpDecoder","text":"<pre><code>TabMlpDecoder(\n    embed_dim,\n    mlp_hidden_dims=[100, 200],\n    mlp_activation=\"relu\",\n    mlp_dropout=0.1,\n    mlp_batchnorm=False,\n    mlp_batchnorm_last=False,\n    mlp_linear_first=True,\n)\n</code></pre> <p>               Bases: <code>Module</code></p> <p>Companion decoder model for the <code>TabMlp</code> model (which can be considered an encoder itself).</p> <p>This class is designed to be used with the <code>EncoderDecoderTrainer</code> when using self-supervised pre-training (see the corresponding section in the docs). The <code>TabMlpDecoder</code> will receive the output from the MLP and 'reconstruct' the embeddings.</p> <p>Parameters:</p> <ul> <li> <code>embed_dim</code>               (<code>int</code>)           \u2013            <p>Size of the embeddings tensor that needs to be reconstructed.</p> </li> <li> <code>mlp_hidden_dims</code>               (<code>List[int]</code>, default:                   <code>[100, 200]</code> )           \u2013            <p>List with the number of neurons per dense layer in the mlp.</p> </li> <li> <code>mlp_activation</code>               (<code>str</code>, default:                   <code>'relu'</code> )           \u2013            <p>Activation function for the dense layers of the MLP. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> </li> <li> <code>mlp_dropout</code>               (<code>Union[float, List[float]]</code>, default:                   <code>0.1</code> )           \u2013            <p>float or List of floats with the dropout between the dense layers. e.g: [0.5,0.5]</p> </li> <li> <code>mlp_batchnorm</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether or not batch normalization will be applied to the dense layers</p> </li> <li> <code>mlp_batchnorm_last</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether or not batch normalization will be applied to the last of the dense layers</p> </li> <li> <code>mlp_linear_first</code>               (<code>bool</code>, default:                   <code>True</code> )           \u2013            <p>Boolean indicating the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code></p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>decoder</code>               (<code>Module</code>)           \u2013            <p>mlp model that will receive the output of the encoder</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import TabMlpDecoder\n&gt;&gt;&gt; x_inp = torch.rand(3, 8)\n&gt;&gt;&gt; decoder = TabMlpDecoder(embed_dim=32, mlp_hidden_dims=[8,16])\n&gt;&gt;&gt; res = decoder(x_inp)\n&gt;&gt;&gt; res.shape\ntorch.Size([3, 32])\n</code></pre> Source code in <code>pytorch_widedeep/models/tabular/mlp/tab_mlp.py</code> <pre><code>def __init__(\n    self,\n    embed_dim: int,\n    mlp_hidden_dims: List[int] = [100, 200],\n    mlp_activation: str = \"relu\",\n    mlp_dropout: Union[float, List[float]] = 0.1,\n    mlp_batchnorm: bool = False,\n    mlp_batchnorm_last: bool = False,\n    mlp_linear_first: bool = True,\n):\n    super(TabMlpDecoder, self).__init__()\n\n    self.embed_dim = embed_dim\n\n    self.mlp_hidden_dims = mlp_hidden_dims\n    self.mlp_activation = mlp_activation\n    self.mlp_dropout = mlp_dropout\n    self.mlp_batchnorm = mlp_batchnorm\n    self.mlp_batchnorm_last = mlp_batchnorm_last\n    self.mlp_linear_first = mlp_linear_first\n\n    self.decoder = MLP(\n        mlp_hidden_dims + [self.embed_dim],\n        mlp_activation,\n        mlp_dropout,\n        mlp_batchnorm,\n        mlp_batchnorm_last,\n        mlp_linear_first,\n    )\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.resnet.tab_resnet.TabResnet","title":"TabResnet","text":"<pre><code>TabResnet(\n    column_idx,\n    *,\n    cat_embed_input=None,\n    cat_embed_dropout=None,\n    use_cat_bias=None,\n    cat_embed_activation=None,\n    continuous_cols=None,\n    cont_norm_layer=None,\n    embed_continuous=None,\n    embed_continuous_method=None,\n    cont_embed_dim=None,\n    cont_embed_dropout=None,\n    cont_embed_activation=None,\n    quantization_setup=None,\n    n_frequencies=None,\n    sigma=None,\n    share_last_layer=None,\n    full_embed_dropout=None,\n    blocks_dims=[200, 100, 100],\n    blocks_dropout=0.1,\n    simplify_blocks=False,\n    mlp_hidden_dims=None,\n    mlp_activation=None,\n    mlp_dropout=None,\n    mlp_batchnorm=None,\n    mlp_batchnorm_last=None,\n    mlp_linear_first=None\n)\n</code></pre> <p>               Bases: <code>BaseTabularModelWithoutAttention</code></p> <p>Defines a <code>TabResnet</code> model that can be used as the <code>deeptabular</code> component of a Wide &amp; Deep model or independently by itself.</p> <p>This class combines embedding representations of the categorical features with numerical (aka continuous) features, embedded or not. These are then passed through a series of Resnet blocks. See <code>pytorch_widedeep.models.tab_resnet._layers</code> for details on the structure of each block.</p> <p>Most of the parameters for this class are <code>Optional</code> since the use of categorical or continuous is in fact optional (i.e. one can use categorical features only, continuous features only or both).</p> <p>Parameters:</p> <ul> <li> <code>column_idx</code>               (<code>Dict[str, int]</code>)           \u2013            <p>Dict containing the index of the columns that will be passed through the <code>TabMlp</code> model. Required to slice the tensors. e.g. {'education': 0, 'relationship': 1, 'workclass': 2, ...}.</p> </li> <li> <code>cat_embed_input</code>               (<code>Optional[List[Tuple[str, int, int]]]</code>, default:                   <code>None</code> )           \u2013            <p>List of Tuples with the column name, number of unique values and embedding dimension. e.g. [(education, 11, 32), ...]</p> </li> <li> <code>cat_embed_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Categorical embeddings dropout. If <code>None</code>, it will default to 0.</p> </li> <li> <code>use_cat_bias</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating if bias will be used for the categorical embeddings. If <code>None</code>, it will default to 'False'.</p> </li> <li> <code>cat_embed_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the categorical embeddings, if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> </li> <li> <code>continuous_cols</code>               (<code>Optional[List[str]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the name of the numeric (aka continuous) columns</p> </li> <li> <code>cont_norm_layer</code>               (<code>Optional[Literal[batchnorm, layernorm]]</code>, default:                   <code>None</code> )           \u2013            <p>Type of normalization layer applied to the continuous features. Options are: 'layernorm' and 'batchnorm'. if <code>None</code>, no normalization layer will be used.</p> </li> <li> <code>embed_continuous</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating if the continuous columns will be embedded using one of the available methods: 'standard', 'periodic' or 'piecewise'. If <code>None</code>, it will default to 'False'. NOTE: This parameter is deprecated and it  will be removed in future releases. Please, use the  <code>embed_continuous_method</code> parameter instead.</p> </li> <li> <code>embed_continuous_method</code>               (<code>Optional[Literal[standard, piecewise, periodic]]</code>, default:                   <code>None</code> )           \u2013            <p>Method to use to embed the continuous features. Options are: 'standard', 'periodic' or 'piecewise'. The 'standard' embedding method is based on the FT-Transformer implementation presented in the paper: Revisiting Deep Learning Models for Tabular Data. The 'periodic' and_'piecewise'_ methods were presented in the paper: On Embeddings for Numerical Features in Tabular Deep Learning. Please, read the papers for details.</p> </li> <li> <code>cont_embed_dim</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>Size of the continuous embeddings. If the continuous columns are embedded, <code>cont_embed_dim</code> must be passed.</p> </li> <li> <code>cont_embed_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p> </li> <li> <code>cont_embed_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the continuous embeddings if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported. If <code>None</code>, no activation function will be applied.</p> </li> <li> <code>quantization_setup</code>               (<code>Optional[Dict[str, List[float]]]</code>, default:                   <code>None</code> )           \u2013            <p>This parameter is used when the 'piecewise' method is used to embed the continuous cols. It is a dict where keys are the name of the continuous columns and values are lists with the boundaries for the quantization of the continuous_cols. See the examples for details. If If the 'piecewise' method is used, this parameter is required.</p> </li> <li> <code>n_frequencies</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>This is the so called 'k' in their paper On Embeddings for Numerical Features in Tabular Deep Learning, and is the number of 'frequencies' that will be used to represent each continuous column. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>sigma</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>This is the sigma parameter in the paper mentioned when describing the previous parameters and it is used to initialise the 'frequency weights'. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>share_last_layer</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>This parameter is not present in the before mentioned paper but it is implemented in the official repo. If <code>True</code> the linear layer that turns the frequencies into embeddings will be shared across the continuous columns. If <code>False</code> a different linear layer will be used for each continuous column. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>full_embed_dropout</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>If <code>True</code>, the full embedding corresponding to a column will be masked out/dropout. If <code>None</code>, it will default to <code>False</code>.</p> </li> <li> <code>blocks_dims</code>               (<code>List[int]</code>, default:                   <code>[200, 100, 100]</code> )           \u2013            <p>List of integers that define the input and output units of each block. For example: [200, 100, 100] will generate 2 blocks. The first will receive a tensor of size 200 and output a tensor of size 100, and the second will receive a tensor of size 100 and output a tensor of size 100. See <code>pytorch_widedeep.models.tab_resnet._layers</code> for details on the structure of each block.</p> </li> <li> <code>blocks_dropout</code>               (<code>float</code>, default:                   <code>0.1</code> )           \u2013            <p>Block's internal dropout.</p> </li> <li> <code>simplify_blocks</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating if the simplest possible residual blocks (<code>X -&gt; [ [LIN, BN, ACT]  + X ]</code>) will be used instead of a standard one (<code>X -&gt; [ [LIN1, BN1, ACT1] -&gt; [LIN2, BN2]  + X ]</code>).</p> </li> <li> <code>mlp_hidden_dims</code>               (<code>Optional[List[int]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the number of neurons per dense layer in the MLP. e.g: [64, 32]. If <code>None</code> the  output of the Resnet Blocks will be connected directly to the output neuron(s).</p> </li> <li> <code>mlp_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the dense layers of the MLP. Currently 'tanh', 'relu', 'leaky'_relu' and _'gelu' are supported. If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to 'relu'.</p> </li> <li> <code>mlp_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>float with the dropout between the dense layers of the MLP. If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to 0.0.</p> </li> <li> <code>mlp_batchnorm</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating whether or not batch normalization will be applied to the dense layers If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to False.</p> </li> <li> <code>mlp_batchnorm_last</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating whether or not batch normalization will be applied to the last of the dense layers If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to False.</p> </li> <li> <code>mlp_linear_first</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code> If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to <code>True</code>.</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>encoder</code>               (<code>Module</code>)           \u2013            <p>deep dense Resnet model that will receive the concatenation of the embeddings and the continuous columns</p> </li> <li> <code>mlp</code>               (<code>Module</code>)           \u2013            <p>if <code>mlp_hidden_dims</code> is <code>True</code>, this attribute will be an mlp model that will receive the results of the concatenation of the embeddings and the continuous columns -- if present --.</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import TabResnet\n&gt;&gt;&gt; X_deep = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n&gt;&gt;&gt; colnames = ['a', 'b', 'c', 'd', 'e']\n&gt;&gt;&gt; cat_embed_input = [(u,i,j) for u,i,j in zip(colnames[:4], [4]*4, [8]*4)]\n&gt;&gt;&gt; column_idx = {k:v for v,k in enumerate(colnames)}\n&gt;&gt;&gt; model = TabResnet(blocks_dims=[16,4], column_idx=column_idx, cat_embed_input=cat_embed_input,\n... continuous_cols = ['e'])\n&gt;&gt;&gt; out = model(X_deep)\n</code></pre> Source code in <code>pytorch_widedeep/models/tabular/resnet/tab_resnet.py</code> <pre><code>def __init__(\n    self,\n    column_idx: Dict[str, int],\n    *,\n    cat_embed_input: Optional[List[Tuple[str, int, int]]] = None,\n    cat_embed_dropout: Optional[float] = None,\n    use_cat_bias: Optional[bool] = None,\n    cat_embed_activation: Optional[str] = None,\n    continuous_cols: Optional[List[str]] = None,\n    cont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\n    embed_continuous: Optional[bool] = None,\n    embed_continuous_method: Optional[\n        Literal[\"standard\", \"piecewise\", \"periodic\"]\n    ] = None,\n    cont_embed_dim: Optional[int] = None,\n    cont_embed_dropout: Optional[float] = None,\n    cont_embed_activation: Optional[str] = None,\n    quantization_setup: Optional[Dict[str, List[float]]] = None,\n    n_frequencies: Optional[int] = None,\n    sigma: Optional[float] = None,\n    share_last_layer: Optional[bool] = None,\n    full_embed_dropout: Optional[bool] = None,\n    blocks_dims: List[int] = [200, 100, 100],\n    blocks_dropout: float = 0.1,\n    simplify_blocks: bool = False,\n    mlp_hidden_dims: Optional[List[int]] = None,\n    mlp_activation: Optional[str] = None,\n    mlp_dropout: Optional[float] = None,\n    mlp_batchnorm: Optional[bool] = None,\n    mlp_batchnorm_last: Optional[bool] = None,\n    mlp_linear_first: Optional[bool] = None,\n):\n    super(TabResnet, self).__init__(\n        column_idx=column_idx,\n        cat_embed_input=cat_embed_input,\n        cat_embed_dropout=cat_embed_dropout,\n        use_cat_bias=use_cat_bias,\n        cat_embed_activation=cat_embed_activation,\n        continuous_cols=continuous_cols,\n        cont_norm_layer=cont_norm_layer,\n        embed_continuous=embed_continuous,\n        embed_continuous_method=embed_continuous_method,\n        cont_embed_dim=cont_embed_dim,\n        cont_embed_dropout=cont_embed_dropout,\n        cont_embed_activation=cont_embed_activation,\n        quantization_setup=quantization_setup,\n        n_frequencies=n_frequencies,\n        sigma=sigma,\n        share_last_layer=share_last_layer,\n        full_embed_dropout=full_embed_dropout,\n    )\n\n    if len(blocks_dims) &lt; 2:\n        raise ValueError(\n            \"'blocks' must contain at least two elements, e.g. [256, 128]\"\n        )\n\n    self.blocks_dims = blocks_dims\n    self.blocks_dropout = blocks_dropout\n    self.simplify_blocks = simplify_blocks\n\n    self.mlp_hidden_dims = mlp_hidden_dims\n    self.mlp_activation = mlp_activation\n    self.mlp_dropout = mlp_dropout\n    self.mlp_batchnorm = mlp_batchnorm\n    self.mlp_batchnorm_last = mlp_batchnorm_last\n    self.mlp_linear_first = mlp_linear_first\n\n    # Embeddings are instantiated at the base model\n\n    # Resnet\n    dense_resnet_input_dim = self.cat_out_dim + self.cont_out_dim\n    self.encoder = DenseResnet(\n        dense_resnet_input_dim, blocks_dims, blocks_dropout, self.simplify_blocks\n    )\n\n    # Mlp: adding an MLP on top of the Resnet blocks is optional and\n    # therefore all related params are optional\n    if self.mlp_hidden_dims is not None:\n        self.mlp = MLP(\n            d_hidden=[self.blocks_dims[-1]] + self.mlp_hidden_dims,\n            activation=(\n                \"relu\" if self.mlp_activation is None else self.mlp_activation\n            ),\n            dropout=0.0 if self.mlp_dropout is None else self.mlp_dropout,\n            batchnorm=False if self.mlp_batchnorm is None else self.mlp_batchnorm,\n            batchnorm_last=(\n                False\n                if self.mlp_batchnorm_last is None\n                else self.mlp_batchnorm_last\n            ),\n            linear_first=(\n                True if self.mlp_linear_first is None else self.mlp_linear_first\n            ),\n        )\n    else:\n        self.mlp = None\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.resnet.tab_resnet.TabResnet.output_dim","title":"output_dim  <code>property</code>","text":"<pre><code>output_dim\n</code></pre> <p>The output dimension of the model. This is a required property neccesary to build the <code>WideDeep</code> class</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.resnet.tab_resnet.TabResnetDecoder","title":"TabResnetDecoder","text":"<pre><code>TabResnetDecoder(\n    embed_dim,\n    blocks_dims=[100, 100, 200],\n    blocks_dropout=0.1,\n    simplify_blocks=False,\n    mlp_hidden_dims=None,\n    mlp_activation=None,\n    mlp_dropout=None,\n    mlp_batchnorm=None,\n    mlp_batchnorm_last=None,\n    mlp_linear_first=None,\n)\n</code></pre> <p>               Bases: <code>Module</code></p> <p>Companion decoder model for the <code>TabResnet</code> model (which can be considered an encoder itself)</p> <p>This class is designed to be used with the <code>EncoderDecoderTrainer</code> when using self-supervised pre-training (see the corresponding section in the docs). This class will receive the output from the ResNet blocks or the MLP(if present) and 'reconstruct' the embeddings.</p> <p>Parameters:</p> <ul> <li> <code>embed_dim</code>               (<code>int</code>)           \u2013            <p>Size of the embeddings tensor to be reconstructed.</p> </li> <li> <code>blocks_dims</code>               (<code>List[int]</code>, default:                   <code>[100, 100, 200]</code> )           \u2013            <p>List of integers that define the input and output units of each block. For example: [200, 100, 100] will generate 2 blocks. The first will receive a tensor of size 200 and output a tensor of size 100, and the second will receive a tensor of size 100 and output a tensor of size 100. See <code>pytorch_widedeep.models.tab_resnet._layers</code> for details on the structure of each block.</p> </li> <li> <code>blocks_dropout</code>               (<code>float</code>, default:                   <code>0.1</code> )           \u2013            <p>Block's internal dropout.</p> </li> <li> <code>simplify_blocks</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating if the simplest possible residual blocks (<code>X -&gt; [ [LIN, BN, ACT]  + X ]</code>) will be used instead of a standard one (<code>X -&gt; [ [LIN1, BN1, ACT1] -&gt; [LIN2, BN2]  + X ]</code>).</p> </li> <li> <code>mlp_hidden_dims</code>               (<code>Optional[List[int]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the number of neurons per dense layer in the MLP. e.g: [64, 32]. If <code>None</code> the  output of the Resnet Blocks will be connected directly to the output neuron(s).</p> </li> <li> <code>mlp_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the dense layers of the MLP. Currently 'tanh', 'relu', 'leaky'_relu' and _'gelu' are supported. If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to 'relu'.</p> </li> <li> <code>mlp_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>float with the dropout between the dense layers of the MLP. If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to 0.0.</p> </li> <li> <code>mlp_batchnorm</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating whether or not batch normalization will be applied to the dense layers If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to False.</p> </li> <li> <code>mlp_batchnorm_last</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating whether or not batch normalization will be applied to the last of the dense layers If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to False.</p> </li> <li> <code>mlp_linear_first</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code> If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to <code>True</code>.</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>decoder</code>               (<code>Module</code>)           \u2013            <p>deep dense Resnet model that will receive the output of the encoder IF <code>mlp_hidden_dims</code> is None</p> </li> <li> <code>mlp</code>               (<code>Module</code>)           \u2013            <p>if <code>mlp_hidden_dims</code> is not None, the overall decoder will consist in an MLP that will receive the output of the encoder followed by the deep dense Resnet.</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import TabResnetDecoder\n&gt;&gt;&gt; x_inp = torch.rand(3, 8)\n&gt;&gt;&gt; decoder = TabResnetDecoder(embed_dim=32, blocks_dims=[8, 16, 16])\n&gt;&gt;&gt; res = decoder(x_inp)\n&gt;&gt;&gt; res.shape\ntorch.Size([3, 32])\n</code></pre> Source code in <code>pytorch_widedeep/models/tabular/resnet/tab_resnet.py</code> <pre><code>def __init__(\n    self,\n    embed_dim: int,\n    blocks_dims: List[int] = [100, 100, 200],\n    blocks_dropout: float = 0.1,\n    simplify_blocks: bool = False,\n    mlp_hidden_dims: Optional[List[int]] = None,\n    mlp_activation: Optional[str] = None,\n    mlp_dropout: Optional[float] = None,\n    mlp_batchnorm: Optional[bool] = None,\n    mlp_batchnorm_last: Optional[bool] = None,\n    mlp_linear_first: Optional[bool] = None,\n):\n    super(TabResnetDecoder, self).__init__()\n\n    if len(blocks_dims) &lt; 2:\n        raise ValueError(\n            \"'blocks' must contain at least two elements, e.g. [256, 128]\"\n        )\n\n    self.embed_dim = embed_dim\n\n    self.blocks_dims = blocks_dims\n    self.blocks_dropout = blocks_dropout\n    self.simplify_blocks = simplify_blocks\n\n    self.mlp_hidden_dims = mlp_hidden_dims\n    self.mlp_activation = mlp_activation\n    self.mlp_dropout = mlp_dropout\n    self.mlp_batchnorm = mlp_batchnorm\n    self.mlp_batchnorm_last = mlp_batchnorm_last\n    self.mlp_linear_first = mlp_linear_first\n\n    if self.mlp_hidden_dims is not None:\n        self.mlp = MLP(\n            d_hidden=[self.mlp_first_hidden_dim] + self.mlp_hidden_dims,\n            activation=(\n                \"relu\" if self.mlp_activation is None else self.mlp_activation\n            ),\n            dropout=0.0 if self.mlp_dropout is None else self.mlp_dropout,\n            batchnorm=False if self.mlp_batchnorm is None else self.mlp_batchnorm,\n            batchnorm_last=(\n                False\n                if self.mlp_batchnorm_last is None\n                else self.mlp_batchnorm_last\n            ),\n            linear_first=(\n                True if self.mlp_linear_first is None else self.mlp_linear_first\n            ),\n        )\n        self.decoder = DenseResnet(\n            self.mlp_hidden_dims[-1],\n            blocks_dims,\n            blocks_dropout,\n            self.simplify_blocks,\n        )\n    else:\n        self.mlp = None\n        self.decoder = DenseResnet(\n            blocks_dims[0], blocks_dims, blocks_dropout, self.simplify_blocks\n        )\n\n    self.reconstruction_layer = nn.Linear(blocks_dims[-1], embed_dim, bias=False)\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.tabnet.tab_net.TabNet","title":"TabNet","text":"<pre><code>TabNet(\n    column_idx,\n    *,\n    cat_embed_input=None,\n    cat_embed_dropout=None,\n    use_cat_bias=None,\n    cat_embed_activation=None,\n    continuous_cols=None,\n    cont_norm_layer=None,\n    embed_continuous=None,\n    embed_continuous_method=None,\n    cont_embed_dim=None,\n    cont_embed_dropout=None,\n    cont_embed_activation=None,\n    quantization_setup=None,\n    n_frequencies=None,\n    sigma=None,\n    share_last_layer=None,\n    full_embed_dropout=None,\n    n_steps=3,\n    step_dim=8,\n    attn_dim=8,\n    dropout=0.0,\n    n_glu_step_dependent=2,\n    n_glu_shared=2,\n    ghost_bn=True,\n    virtual_batch_size=128,\n    momentum=0.02,\n    gamma=1.3,\n    epsilon=1e-15,\n    mask_type=\"sparsemax\"\n)\n</code></pre> <p>               Bases: <code>BaseTabularModelWithoutAttention</code></p> <p>Defines a TabNet model that can be used as the <code>deeptabular</code> component of a Wide &amp; Deep model or independently by itself.</p> <p>The implementation in this library is fully based on that here by the dreamquark-ai team, simply adapted so that it can work within the <code>WideDeep</code> frame. Therefore, ALL CREDIT TO THE DREAMQUARK-AI TEAM.</p> <p>Parameters:</p> <ul> <li> <code>column_idx</code>               (<code>Dict[str, int]</code>)           \u2013            <p>Dict containing the index of the columns that will be passed through the <code>TabMlp</code> model. Required to slice the tensors. e.g. {'education': 0, 'relationship': 1, 'workclass': 2, ...}.</p> </li> <li> <code>cat_embed_input</code>               (<code>Optional[List[Tuple[str, int, int]]]</code>, default:                   <code>None</code> )           \u2013            <p>List of Tuples with the column name, number of unique values and embedding dimension. e.g. [(education, 11, 32), ...]</p> </li> <li> <code>cat_embed_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Categorical embeddings dropout. If <code>None</code>, it will default to 0.</p> </li> <li> <code>use_cat_bias</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating if bias will be used for the categorical embeddings. If <code>None</code>, it will default to 'False'.</p> </li> <li> <code>cat_embed_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the categorical embeddings, if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> </li> <li> <code>continuous_cols</code>               (<code>Optional[List[str]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the name of the numeric (aka continuous) columns</p> </li> <li> <code>cont_norm_layer</code>               (<code>Optional[Literal[batchnorm, layernorm]]</code>, default:                   <code>None</code> )           \u2013            <p>Type of normalization layer applied to the continuous features. Options are: 'layernorm' and 'batchnorm'. if <code>None</code>, no normalization layer will be used.</p> </li> <li> <code>embed_continuous</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating if the continuous columns will be embedded using one of the available methods: 'standard', 'periodic' or 'piecewise'. If <code>None</code>, it will default to 'False'. NOTE: This parameter is deprecated and it  will be removed in future releases. Please, use the  <code>embed_continuous_method</code> parameter instead.</p> </li> <li> <code>embed_continuous_method</code>               (<code>Optional[Literal[standard, piecewise, periodic]]</code>, default:                   <code>None</code> )           \u2013            <p>Method to use to embed the continuous features. Options are: 'standard', 'periodic' or 'piecewise'. The 'standard' embedding method is based on the FT-Transformer implementation presented in the paper: Revisiting Deep Learning Models for Tabular Data. The 'periodic' and_'piecewise'_ methods were presented in the paper: On Embeddings for Numerical Features in Tabular Deep Learning. Please, read the papers for details.</p> </li> <li> <code>cont_embed_dim</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>Size of the continuous embeddings. If the continuous columns are embedded, <code>cont_embed_dim</code> must be passed.</p> </li> <li> <code>cont_embed_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p> </li> <li> <code>cont_embed_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the continuous embeddings if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported. If <code>None</code>, no activation function will be applied.</p> </li> <li> <code>quantization_setup</code>               (<code>Optional[Dict[str, List[float]]]</code>, default:                   <code>None</code> )           \u2013            <p>This parameter is used when the 'piecewise' method is used to embed the continuous cols. It is a dict where keys are the name of the continuous columns and values are lists with the boundaries for the quantization of the continuous_cols. See the examples for details. If If the 'piecewise' method is used, this parameter is required.</p> </li> <li> <code>n_frequencies</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>This is the so called 'k' in their paper On Embeddings for Numerical Features in Tabular Deep Learning, and is the number of 'frequencies' that will be used to represent each continuous column. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>sigma</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>This is the sigma parameter in the paper mentioned when describing the previous parameters and it is used to initialise the 'frequency weights'. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>share_last_layer</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>This parameter is not present in the before mentioned paper but it is implemented in the official repo. If <code>True</code> the linear layer that turns the frequencies into embeddings will be shared across the continuous columns. If <code>False</code> a different linear layer will be used for each continuous column. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>full_embed_dropout</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>If <code>True</code>, the full embedding corresponding to a column will be masked out/dropout. If <code>None</code>, it will default to <code>False</code>.</p> </li> <li> <code>n_steps</code>               (<code>int</code>, default:                   <code>3</code> )           \u2013            <p>number of decision steps. For a better understanding of the function of <code>n_steps</code> and the upcoming parameters, please see the paper.</p> </li> <li> <code>step_dim</code>               (<code>int</code>, default:                   <code>8</code> )           \u2013            <p>Step's output dimension. This is the output dimension that <code>WideDeep</code> will collect and connect to the output neuron(s).</p> </li> <li> <code>attn_dim</code>               (<code>int</code>, default:                   <code>8</code> )           \u2013            <p>Attention dimension</p> </li> <li> <code>dropout</code>               (<code>float</code>, default:                   <code>0.0</code> )           \u2013            <p>GLU block's internal dropout</p> </li> <li> <code>n_glu_step_dependent</code>               (<code>int</code>, default:                   <code>2</code> )           \u2013            <p>number of GLU Blocks (<code>[FC -&gt; BN -&gt; GLU]</code>) that are step dependent</p> </li> <li> <code>n_glu_shared</code>               (<code>int</code>, default:                   <code>2</code> )           \u2013            <p>number of GLU Blocks (<code>[FC -&gt; BN -&gt; GLU]</code>) that will be shared across decision steps</p> </li> <li> <code>ghost_bn</code>               (<code>bool</code>, default:                   <code>True</code> )           \u2013            <p>Boolean indicating if Ghost Batch Normalization will be used.</p> </li> <li> <code>virtual_batch_size</code>               (<code>int</code>, default:                   <code>128</code> )           \u2013            <p>Batch size when using Ghost Batch Normalization</p> </li> <li> <code>momentum</code>               (<code>float</code>, default:                   <code>0.02</code> )           \u2013            <p>Ghost Batch Normalization's momentum. The dreamquark-ai advises for very low values. However high values are used in the original publication. During our tests higher values lead to better results</p> </li> <li> <code>gamma</code>               (<code>float</code>, default:                   <code>1.3</code> )           \u2013            <p>Relaxation parameter in the paper. When gamma = 1, a feature is enforced to be used only at one decision step. As gamma increases, more flexibility is provided to use a feature at multiple decision steps</p> </li> <li> <code>epsilon</code>               (<code>float</code>, default:                   <code>1e-15</code> )           \u2013            <p>Float to avoid log(0). Always keep low</p> </li> <li> <code>mask_type</code>               (<code>str</code>, default:                   <code>'sparsemax'</code> )           \u2013            <p>Mask function to use. Either 'sparsemax' or 'entmax'</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>encoder</code>               (<code>Module</code>)           \u2013            <p>the TabNet encoder. For details see the original publication.</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n&gt;&gt;&gt; colnames = [\"a\", \"b\", \"c\", \"d\", \"e\"]\n&gt;&gt;&gt; cat_embed_input = [(u, i, j) for u, i, j in zip(colnames[:4], [4] * 4, [8] * 4)]\n&gt;&gt;&gt; column_idx = {k: v for v, k in enumerate(colnames)}\n&gt;&gt;&gt; model = TabNet(column_idx=column_idx, cat_embed_input=cat_embed_input, continuous_cols=[\"e\"])\n&gt;&gt;&gt; out = model(X_tab)\n</code></pre> Source code in <code>pytorch_widedeep/models/tabular/tabnet/tab_net.py</code> <pre><code>def __init__(\n    self,\n    column_idx: Dict[str, int],\n    *,\n    cat_embed_input: Optional[List[Tuple[str, int, int]]] = None,\n    cat_embed_dropout: Optional[float] = None,\n    use_cat_bias: Optional[bool] = None,\n    cat_embed_activation: Optional[str] = None,\n    continuous_cols: Optional[List[str]] = None,\n    cont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\n    embed_continuous: Optional[bool] = None,\n    embed_continuous_method: Optional[\n        Literal[\"standard\", \"piecewise\", \"periodic\"]\n    ] = None,\n    cont_embed_dim: Optional[int] = None,\n    cont_embed_dropout: Optional[float] = None,\n    cont_embed_activation: Optional[str] = None,\n    quantization_setup: Optional[Dict[str, List[float]]] = None,\n    n_frequencies: Optional[int] = None,\n    sigma: Optional[float] = None,\n    share_last_layer: Optional[bool] = None,\n    full_embed_dropout: Optional[bool] = None,\n    n_steps: int = 3,\n    step_dim: int = 8,\n    attn_dim: int = 8,\n    dropout: float = 0.0,\n    n_glu_step_dependent: int = 2,\n    n_glu_shared: int = 2,\n    ghost_bn: bool = True,\n    virtual_batch_size: int = 128,\n    momentum: float = 0.02,\n    gamma: float = 1.3,\n    epsilon: float = 1e-15,\n    mask_type: str = \"sparsemax\",\n):\n    super(TabNet, self).__init__(\n        column_idx=column_idx,\n        cat_embed_input=cat_embed_input,\n        cat_embed_dropout=cat_embed_dropout,\n        use_cat_bias=use_cat_bias,\n        cat_embed_activation=cat_embed_activation,\n        continuous_cols=continuous_cols,\n        cont_norm_layer=cont_norm_layer,\n        embed_continuous=embed_continuous,\n        embed_continuous_method=embed_continuous_method,\n        cont_embed_dim=cont_embed_dim,\n        cont_embed_dropout=cont_embed_dropout,\n        cont_embed_activation=cont_embed_activation,\n        quantization_setup=quantization_setup,\n        n_frequencies=n_frequencies,\n        sigma=sigma,\n        share_last_layer=share_last_layer,\n        full_embed_dropout=full_embed_dropout,\n    )\n\n    self.n_steps = n_steps\n    self.step_dim = step_dim\n    self.attn_dim = attn_dim\n    self.dropout = dropout\n    self.n_glu_step_dependent = n_glu_step_dependent\n    self.n_glu_shared = n_glu_shared\n    self.ghost_bn = ghost_bn\n    self.virtual_batch_size = virtual_batch_size\n    self.momentum = momentum\n    self.gamma = gamma\n    self.epsilon = epsilon\n    self.mask_type = mask_type\n\n    # Embeddings are instantiated at the base model\n    self.embed_out_dim = self.cat_out_dim + self.cont_out_dim\n\n    # TabNet\n    self.encoder = TabNetEncoder(\n        self.embed_out_dim,\n        n_steps,\n        step_dim,\n        attn_dim,\n        dropout,\n        n_glu_step_dependent,\n        n_glu_shared,\n        ghost_bn,\n        virtual_batch_size,\n        momentum,\n        gamma,\n        epsilon,\n        mask_type,\n    )\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.tabnet.tab_net.TabNet.output_dim","title":"output_dim  <code>property</code>","text":"<pre><code>output_dim\n</code></pre> <p>The output dimension of the model. This is a required property neccesary to build the <code>WideDeep</code> class</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.tabnet.tab_net.TabNetDecoder","title":"TabNetDecoder","text":"<pre><code>TabNetDecoder(\n    embed_dim,\n    n_steps=3,\n    step_dim=8,\n    dropout=0.0,\n    n_glu_step_dependent=2,\n    n_glu_shared=2,\n    ghost_bn=True,\n    virtual_batch_size=128,\n    momentum=0.02,\n)\n</code></pre> <p>               Bases: <code>Module</code></p> <p>Companion decoder model for the <code>TabNet</code> model (which can be considered an encoder itself)</p> <p>This class is designed to be used with the <code>EncoderDecoderTrainer</code> when using self-supervised pre-training (see the corresponding section in the docs). This class will receive the output from the <code>TabNet</code> encoder (i.e. the output from the so called 'steps') and 'reconstruct' the embeddings.</p> <p>Parameters:</p> <ul> <li> <code>embed_dim</code>               (<code>int</code>)           \u2013            <p>Size of the embeddings tensor to be reconstructed.</p> </li> <li> <code>n_steps</code>               (<code>int</code>, default:                   <code>3</code> )           \u2013            <p>number of decision steps. For a better understanding of the function of <code>n_steps</code> and the upcoming parameters, please see the paper.</p> </li> <li> <code>step_dim</code>               (<code>int</code>, default:                   <code>8</code> )           \u2013            <p>Step's output dimension. This is the output dimension that <code>WideDeep</code> will collect and connect to the output neuron(s).</p> </li> <li> <code>dropout</code>               (<code>float</code>, default:                   <code>0.0</code> )           \u2013            <p>GLU block's internal dropout</p> </li> <li> <code>n_glu_step_dependent</code>               (<code>int</code>, default:                   <code>2</code> )           \u2013            <p>number of GLU Blocks (<code>[FC -&gt; BN -&gt; GLU]</code>) that are step dependent</p> </li> <li> <code>n_glu_shared</code>               (<code>int</code>, default:                   <code>2</code> )           \u2013            <p>number of GLU Blocks (<code>[FC -&gt; BN -&gt; GLU]</code>) that will be shared across decision steps</p> </li> <li> <code>ghost_bn</code>               (<code>bool</code>, default:                   <code>True</code> )           \u2013            <p>Boolean indicating if Ghost Batch Normalization will be used.</p> </li> <li> <code>virtual_batch_size</code>               (<code>int</code>, default:                   <code>128</code> )           \u2013            <p>Batch size when using Ghost Batch Normalization</p> </li> <li> <code>momentum</code>               (<code>float</code>, default:                   <code>0.02</code> )           \u2013            <p>Ghost Batch Normalization's momentum. The dreamquark-ai advises for very low values. However high values are used in the original publication. During our tests higher values lead to better results</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>decoder</code>               (<code>Module</code>)           \u2013            <p>decoder that will receive the output from the encoder's steps and will reconstruct the embeddings</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import TabNetDecoder\n&gt;&gt;&gt; x_inp = [torch.rand(3, 8), torch.rand(3, 8), torch.rand(3, 8)]\n&gt;&gt;&gt; decoder = TabNetDecoder(embed_dim=32, ghost_bn=False)\n&gt;&gt;&gt; res = decoder(x_inp)\n&gt;&gt;&gt; res.shape\ntorch.Size([3, 32])\n</code></pre> Source code in <code>pytorch_widedeep/models/tabular/tabnet/tab_net.py</code> <pre><code>def __init__(\n    self,\n    embed_dim: int,\n    n_steps: int = 3,\n    step_dim: int = 8,\n    dropout: float = 0.0,\n    n_glu_step_dependent: int = 2,\n    n_glu_shared: int = 2,\n    ghost_bn: bool = True,\n    virtual_batch_size: int = 128,\n    momentum: float = 0.02,\n):\n    super(TabNetDecoder, self).__init__()\n\n    self.n_steps = n_steps\n    self.step_dim = step_dim\n    self.dropout = dropout\n    self.n_glu_step_dependent = n_glu_step_dependent\n    self.n_glu_shared = n_glu_shared\n    self.ghost_bn = ghost_bn\n    self.virtual_batch_size = virtual_batch_size\n    self.momentum = momentum\n\n    shared_layers = nn.ModuleList()\n    for i in range(n_glu_shared):\n        if i == 0:\n            shared_layers.append(nn.Linear(step_dim, 2 * step_dim, bias=False))\n        else:\n            shared_layers.append(nn.Linear(step_dim, 2 * step_dim, bias=False))\n\n    self.decoder = nn.ModuleList()\n    for step in range(n_steps):\n        transformer = FeatTransformer(\n            step_dim,\n            step_dim,\n            dropout,\n            shared_layers,\n            n_glu_step_dependent,\n            ghost_bn,\n            virtual_batch_size,\n            momentum=momentum,\n        )\n        self.decoder.append(transformer)\n\n    self.reconstruction_layer = nn.Linear(step_dim, embed_dim, bias=False)\n    initialize_non_glu(self.reconstruction_layer, step_dim, embed_dim)\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.mlp.context_attention_mlp.ContextAttentionMLP","title":"ContextAttentionMLP","text":"<pre><code>ContextAttentionMLP(\n    column_idx,\n    *,\n    cat_embed_input=None,\n    cat_embed_dropout=None,\n    use_cat_bias=None,\n    cat_embed_activation=None,\n    shared_embed=None,\n    add_shared_embed=None,\n    frac_shared_embed=None,\n    continuous_cols=None,\n    cont_norm_layer=None,\n    embed_continuous_method=\"standard\",\n    cont_embed_dropout=None,\n    cont_embed_activation=None,\n    quantization_setup=None,\n    n_frequencies=None,\n    sigma=None,\n    share_last_layer=None,\n    full_embed_dropout=None,\n    input_dim=32,\n    attn_dropout=0.2,\n    with_addnorm=False,\n    attn_activation=\"leaky_relu\",\n    n_blocks=3\n)\n</code></pre> <p>               Bases: <code>BaseTabularModelWithAttention</code></p> <p>Defines a <code>ContextAttentionMLP</code> model that can be used as the <code>deeptabular</code> component of a Wide &amp; Deep model or independently by itself.</p> <p>This class combines embedding representations of the categorical features with numerical (aka continuous) features that are also embedded. These are then passed through a series of attention blocks. Each attention block is comprised by a <code>ContextAttentionEncoder</code>. Such encoder is in part inspired by the attention mechanism described in Hierarchical Attention Networks for Document Classification. See <code>pytorch_widedeep.models.tabular.mlp._attention_layers</code> for details.</p> <p>Most of the parameters for this class are <code>Optional</code> since the use of categorical or continuous is in fact optional (i.e. one can use categorical features only, continuous features only or both).</p> <p>Parameters:</p> <ul> <li> <code>column_idx</code>               (<code>Dict[str, int]</code>)           \u2013            <p>Dict containing the index of the columns that will be passed through the <code>TabMlp</code> model. Required to slice the tensors. e.g. {'education': 0, 'relationship': 1, 'workclass': 2, ...}.</p> </li> <li> <code>cat_embed_input</code>               (<code>Optional[List[Tuple[str, int]]]</code>, default:                   <code>None</code> )           \u2013            <p>List of Tuples with the column name and number of unique values and embedding dimension. e.g. [(education, 11), ...]</p> </li> <li> <code>cat_embed_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Categorical embeddings dropout. If <code>None</code>, it will default to 0.</p> </li> <li> <code>use_cat_bias</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating if bias will be used for the categorical embeddings. If <code>None</code>, it will default to 'False'.</p> </li> <li> <code>cat_embed_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the categorical embeddings, if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> </li> <li> <code>shared_embed</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating if the embeddings will be \"shared\". The idea behind <code>shared_embed</code> is described in the Appendix A in the TabTransformer paper: 'The goal of having column embedding is to enable the model to distinguish the classes in one column from those in the other columns'. In other words, the idea is to let the model learn which column is embedded at the time. See: <code>pytorch_widedeep.models.transformers._layers.SharedEmbeddings</code>.</p> </li> <li> <code>add_shared_embed</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>The two embedding sharing strategies are: 1) add the shared embeddings to the column embeddings or 2) to replace the first <code>frac_shared_embed</code> with the shared embeddings. See <code>pytorch_widedeep.models.embeddings_layers.SharedEmbeddings</code> If 'None' is passed, it will default to 'False'.</p> </li> <li> <code>frac_shared_embed</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>The fraction of embeddings that will be shared (if <code>add_shared_embed = False</code>) by all the different categories for one particular column. If 'None' is passed, it will default to 0.0.</p> </li> <li> <code>continuous_cols</code>               (<code>Optional[List[str]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the name of the numeric (aka continuous) columns</p> </li> <li> <code>cont_norm_layer</code>               (<code>Optional[Literal[batchnorm, layernorm]]</code>, default:                   <code>None</code> )           \u2013            <p>Type of normalization layer applied to the continuous features. Options are: 'layernorm' and 'batchnorm'. if <code>None</code>, no normalization layer will be used.</p> </li> <li> <code>embed_continuous_method</code>               (<code>Optional[Literal[standard, piecewise, periodic]]</code>, default:                   <code>'standard'</code> )           \u2013            <p>Method to use to embed the continuous features. Options are: 'standard', 'periodic' or 'piecewise'. The 'standard' embedding method is based on the FT-Transformer implementation presented in the paper: Revisiting Deep Learning Models for Tabular Data. The 'periodic' and_'piecewise'_ methods were presented in the paper: On Embeddings for Numerical Features in Tabular Deep Learning. Please, read the papers for details.</p> </li> <li> <code>cont_embed_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p> </li> <li> <code>cont_embed_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the continuous embeddings if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported. If <code>None</code>, no activation function will be applied.</p> </li> <li> <code>quantization_setup</code>               (<code>Optional[Dict[str, List[float]]]</code>, default:                   <code>None</code> )           \u2013            <p>This parameter is used when the 'piecewise' method is used to embed the continuous cols. It is a dict where keys are the name of the continuous columns and values are lists with the boundaries for the quantization of the continuous_cols. See the examples for details. If If the 'piecewise' method is used, this parameter is required.</p> </li> <li> <code>n_frequencies</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>This is the so called 'k' in their paper On Embeddings for Numerical Features in Tabular Deep Learning, and is the number of 'frequencies' that will be used to represent each continuous column. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>sigma</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>This is the sigma parameter in the paper mentioned when describing the previous parameters and it is used to initialise the 'frequency weights'. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>share_last_layer</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>This parameter is not present in the before mentioned paper but it is implemented in the official repo. If <code>True</code> the linear layer that turns the frequencies into embeddings will be shared across the continuous columns. If <code>False</code> a different linear layer will be used for each continuous column. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>full_embed_dropout</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>If <code>True</code>, the full embedding corresponding to a column will be masked out/dropout. If <code>None</code>, it will default to <code>False</code>.</p> </li> <li> <code>input_dim</code>               (<code>int</code>, default:                   <code>32</code> )           \u2013            <p>The so-called dimension of the model. Is the number of embeddings used to encode the categorical and/or continuous columns</p> </li> <li> <code>attn_dropout</code>               (<code>float</code>, default:                   <code>0.2</code> )           \u2013            <p>Dropout for each attention block</p> </li> <li> <code>with_addnorm</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating if residual connections will be used in the attention blocks</p> </li> <li> <code>attn_activation</code>               (<code>str</code>, default:                   <code>'leaky_relu'</code> )           \u2013            <p>String indicating the activation function to be applied to the dense layer in each attention encoder. 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported.</p> </li> <li> <code>n_blocks</code>               (<code>int</code>, default:                   <code>3</code> )           \u2013            <p>Number of attention blocks</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>encoder</code>               (<code>Module</code>)           \u2013            <p>Sequence of attention encoders.</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import ContextAttentionMLP\n&gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n&gt;&gt;&gt; colnames = ['a', 'b', 'c', 'd', 'e']\n&gt;&gt;&gt; cat_embed_input = [(u,i,j) for u,i,j in zip(colnames[:4], [4]*4, [8]*4)]\n&gt;&gt;&gt; column_idx = {k:v for v,k in enumerate(colnames)}\n&gt;&gt;&gt; model = ContextAttentionMLP(column_idx=column_idx, cat_embed_input=cat_embed_input, continuous_cols = ['e'])\n&gt;&gt;&gt; out = model(X_tab)\n</code></pre> Source code in <code>pytorch_widedeep/models/tabular/mlp/context_attention_mlp.py</code> <pre><code>def __init__(\n    self,\n    column_idx: Dict[str, int],\n    *,\n    cat_embed_input: Optional[List[Tuple[str, int]]] = None,\n    cat_embed_dropout: Optional[float] = None,\n    use_cat_bias: Optional[bool] = None,\n    cat_embed_activation: Optional[str] = None,\n    shared_embed: Optional[bool] = None,\n    add_shared_embed: Optional[bool] = None,\n    frac_shared_embed: Optional[float] = None,\n    continuous_cols: Optional[List[str]] = None,\n    cont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\n    embed_continuous_method: Optional[\n        Literal[\"standard\", \"piecewise\", \"periodic\"]\n    ] = \"standard\",\n    cont_embed_dropout: Optional[float] = None,\n    cont_embed_activation: Optional[str] = None,\n    quantization_setup: Optional[Dict[str, List[float]]] = None,\n    n_frequencies: Optional[int] = None,\n    sigma: Optional[float] = None,\n    share_last_layer: Optional[bool] = None,\n    full_embed_dropout: Optional[bool] = None,\n    input_dim: int = 32,\n    attn_dropout: float = 0.2,\n    with_addnorm: bool = False,\n    attn_activation: str = \"leaky_relu\",\n    n_blocks: int = 3,\n):\n    super(ContextAttentionMLP, self).__init__(\n        column_idx=column_idx,\n        cat_embed_input=cat_embed_input,\n        cat_embed_dropout=cat_embed_dropout,\n        use_cat_bias=use_cat_bias,\n        cat_embed_activation=cat_embed_activation,\n        shared_embed=shared_embed,\n        add_shared_embed=add_shared_embed,\n        frac_shared_embed=frac_shared_embed,\n        continuous_cols=continuous_cols,\n        cont_norm_layer=cont_norm_layer,\n        embed_continuous_method=embed_continuous_method,\n        cont_embed_dropout=cont_embed_dropout,\n        cont_embed_activation=cont_embed_activation,\n        input_dim=input_dim,\n        quantization_setup=quantization_setup,\n        n_frequencies=n_frequencies,\n        sigma=sigma,\n        share_last_layer=share_last_layer,\n        full_embed_dropout=full_embed_dropout,\n    )\n\n    self.attn_dropout = attn_dropout\n    self.with_addnorm = with_addnorm\n    self.attn_activation = attn_activation\n    self.n_blocks = n_blocks\n\n    self.with_cls_token = \"cls_token\" in column_idx\n    self.n_cat = len(cat_embed_input) if cat_embed_input is not None else 0\n    self.n_cont = len(continuous_cols) if continuous_cols is not None else 0\n\n    # Embeddings are instantiated at the base model\n    # Attention Blocks\n    self.encoder = nn.Sequential()\n    for i in range(n_blocks):\n        self.encoder.add_module(\n            \"attention_block\" + str(i),\n            ContextAttentionEncoder(\n                input_dim,\n                attn_dropout,\n                with_addnorm,\n                attn_activation,\n            ),\n        )\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.mlp.context_attention_mlp.ContextAttentionMLP.output_dim","title":"output_dim  <code>property</code>","text":"<pre><code>output_dim\n</code></pre> <p>The output dimension of the model. This is a required property neccesary to build the <code>WideDeep</code> class</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.mlp.context_attention_mlp.ContextAttentionMLP.attention_weights","title":"attention_weights  <code>property</code>","text":"<pre><code>attention_weights\n</code></pre> <p>List with the attention weights per block</p> <p>The shape of the attention weights is \\((N, F)\\), where \\(N\\) is the batch size and \\(F\\) is the number of features/columns in the dataset</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.mlp.self_attention_mlp.SelfAttentionMLP","title":"SelfAttentionMLP","text":"<pre><code>SelfAttentionMLP(\n    column_idx,\n    *,\n    cat_embed_input=None,\n    cat_embed_dropout=None,\n    use_cat_bias=None,\n    cat_embed_activation=None,\n    shared_embed=None,\n    add_shared_embed=None,\n    frac_shared_embed=None,\n    continuous_cols=None,\n    cont_norm_layer=None,\n    embed_continuous_method=\"standard\",\n    cont_embed_dropout=None,\n    cont_embed_activation=None,\n    quantization_setup=None,\n    n_frequencies=None,\n    sigma=None,\n    share_last_layer=None,\n    full_embed_dropout=None,\n    input_dim=32,\n    attn_dropout=0.2,\n    n_heads=8,\n    use_bias=False,\n    with_addnorm=False,\n    attn_activation=\"leaky_relu\",\n    n_blocks=3\n)\n</code></pre> <p>               Bases: <code>BaseTabularModelWithAttention</code></p> <p>Defines a <code>SelfAttentionMLP</code> model that can be used as the deeptabular component of a Wide &amp; Deep model or independently by itself.</p> <p>This class combines embedding representations of the categorical features with numerical (aka continuous) features that are also embedded. These are then passed through a series of attention blocks. Each attention block is comprised by what we would refer as a simplified <code>SelfAttentionEncoder</code>. See <code>pytorch_widedeep.models.tabular.mlp._attention_layers</code> for details. The reason to use a simplified version of self attention is because we observed that the 'standard' attention mechanism used in the TabTransformer has a notable tendency to overfit.</p> <p>In more detail, this model only uses Q and K (and not V). If we think about it as in terms of text (and intuitively), the Softmax(QK^T) is the attention mechanism that tells us how much, at each position in the input sentence, each word is represented or 'expressed'. We refer to that as 'attention weights'. These attention weighst are normally multiplied by a Value matrix to further strength the focus on the words that each word should be attending to (again, intuitively).</p> <p>In this implementation we skip this last multiplication and instead we multiply the attention weights directly by the input tensor. This is a simplification that we expect is beneficial in terms of avoiding overfitting for tabular data.</p> <p>Most of the parameters for this class are <code>Optional</code> since the use of categorical or continuous is in fact optional (i.e. one can use categorical features only, continuous features only or both).</p> <p>Parameters:</p> <ul> <li> <code>column_idx</code>               (<code>Dict[str, int]</code>)           \u2013            <p>Dict containing the index of the columns that will be passed through the <code>TabMlp</code> model. Required to slice the tensors. e.g. {'education': 0, 'relationship': 1, 'workclass': 2, ...}.</p> </li> <li> <code>cat_embed_input</code>               (<code>Optional[List[Tuple[str, int]]]</code>, default:                   <code>None</code> )           \u2013            <p>List of Tuples with the column name and number of unique values and embedding dimension. e.g. [(education, 11), ...]</p> </li> <li> <code>cat_embed_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Categorical embeddings dropout. If <code>None</code>, it will default to 0.</p> </li> <li> <code>use_cat_bias</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating if bias will be used for the categorical embeddings. If <code>None</code>, it will default to 'False'.</p> </li> <li> <code>cat_embed_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the categorical embeddings, if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> </li> <li> <code>shared_embed</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating if the embeddings will be \"shared\". The idea behind <code>shared_embed</code> is described in the Appendix A in the TabTransformer paper: 'The goal of having column embedding is to enable the model to distinguish the classes in one column from those in the other columns'. In other words, the idea is to let the model learn which column is embedded at the time. See: <code>pytorch_widedeep.models.transformers._layers.SharedEmbeddings</code>.</p> </li> <li> <code>add_shared_embed</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>The two embedding sharing strategies are: 1) add the shared embeddings to the column embeddings or 2) to replace the first <code>frac_shared_embed</code> with the shared embeddings. See <code>pytorch_widedeep.models.embeddings_layers.SharedEmbeddings</code> If 'None' is passed, it will default to 'False'.</p> </li> <li> <code>frac_shared_embed</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>The fraction of embeddings that will be shared (if <code>add_shared_embed = False</code>) by all the different categories for one particular column. If 'None' is passed, it will default to 0.0.</p> </li> <li> <code>continuous_cols</code>               (<code>Optional[List[str]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the name of the numeric (aka continuous) columns</p> </li> <li> <code>cont_norm_layer</code>               (<code>Optional[Literal[batchnorm, layernorm]]</code>, default:                   <code>None</code> )           \u2013            <p>Type of normalization layer applied to the continuous features. Options are: 'layernorm' and 'batchnorm'. if <code>None</code>, no normalization layer will be used.</p> </li> <li> <code>embed_continuous_method</code>               (<code>Optional[Literal[standard, piecewise, periodic]]</code>, default:                   <code>'standard'</code> )           \u2013            <p>Method to use to embed the continuous features. Options are: 'standard', 'periodic' or 'piecewise'. The 'standard' embedding method is based on the FT-Transformer implementation presented in the paper: Revisiting Deep Learning Models for Tabular Data. The 'periodic' and_'piecewise'_ methods were presented in the paper: On Embeddings for Numerical Features in Tabular Deep Learning. Please, read the papers for details.</p> </li> <li> <code>cont_embed_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p> </li> <li> <code>cont_embed_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the continuous embeddings if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported. If <code>None</code>, no activation function will be applied.</p> </li> <li> <code>quantization_setup</code>               (<code>Optional[Dict[str, List[float]]]</code>, default:                   <code>None</code> )           \u2013            <p>This parameter is used when the 'piecewise' method is used to embed the continuous cols. It is a dict where keys are the name of the continuous columns and values are lists with the boundaries for the quantization of the continuous_cols. See the examples for details. If If the 'piecewise' method is used, this parameter is required.</p> </li> <li> <code>n_frequencies</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>This is the so called 'k' in their paper On Embeddings for Numerical Features in Tabular Deep Learning, and is the number of 'frequencies' that will be used to represent each continuous column. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>sigma</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>This is the sigma parameter in the paper mentioned when describing the previous parameters and it is used to initialise the 'frequency weights'. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>share_last_layer</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>This parameter is not present in the before mentioned paper but it is implemented in the official repo. If <code>True</code> the linear layer that turns the frequencies into embeddings will be shared across the continuous columns. If <code>False</code> a different linear layer will be used for each continuous column. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>full_embed_dropout</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>If <code>True</code>, the full embedding corresponding to a column will be masked out/dropout. If <code>None</code>, it will default to <code>False</code>.</p> </li> <li> <code>input_dim</code>               (<code>int</code>, default:                   <code>32</code> )           \u2013            <p>The so-called dimension of the model. Is the number of embeddings used to encode the categorical and/or continuous columns</p> </li> <li> <code>attn_dropout</code>               (<code>float</code>, default:                   <code>0.2</code> )           \u2013            <p>Dropout for each attention block</p> </li> <li> <code>n_heads</code>               (<code>int</code>, default:                   <code>8</code> )           \u2013            <p>Number of attention heads per attention block.</p> </li> <li> <code>use_bias</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether or not to use bias in the Q, K projection layers.</p> </li> <li> <code>with_addnorm</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating if residual connections will be used in the attention blocks</p> </li> <li> <code>attn_activation</code>               (<code>str</code>, default:                   <code>'leaky_relu'</code> )           \u2013            <p>String indicating the activation function to be applied to the dense layer in each attention encoder. 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported.</p> </li> <li> <code>n_blocks</code>               (<code>int</code>, default:                   <code>3</code> )           \u2013            <p>Number of attention blocks</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>cat_and_cont_embed</code>               (<code>Module</code>)           \u2013            <p>This is the module that processes the categorical and continuous columns</p> </li> <li> <code>encoder</code>               (<code>Module</code>)           \u2013            <p>Sequence of attention encoders.</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import SelfAttentionMLP\n&gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n&gt;&gt;&gt; colnames = ['a', 'b', 'c', 'd', 'e']\n&gt;&gt;&gt; cat_embed_input = [(u,i,j) for u,i,j in zip(colnames[:4], [4]*4, [8]*4)]\n&gt;&gt;&gt; column_idx = {k:v for v,k in enumerate(colnames)}\n&gt;&gt;&gt; model = SelfAttentionMLP(column_idx=column_idx, cat_embed_input=cat_embed_input, continuous_cols = ['e'])\n&gt;&gt;&gt; out = model(X_tab)\n</code></pre> Source code in <code>pytorch_widedeep/models/tabular/mlp/self_attention_mlp.py</code> <pre><code>def __init__(\n    self,\n    column_idx: Dict[str, int],\n    *,\n    cat_embed_input: Optional[List[Tuple[str, int]]] = None,\n    cat_embed_dropout: Optional[float] = None,\n    use_cat_bias: Optional[bool] = None,\n    cat_embed_activation: Optional[str] = None,\n    shared_embed: Optional[bool] = None,\n    add_shared_embed: Optional[bool] = None,\n    frac_shared_embed: Optional[float] = None,\n    continuous_cols: Optional[List[str]] = None,\n    cont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\n    embed_continuous_method: Optional[\n        Literal[\"standard\", \"piecewise\", \"periodic\"]\n    ] = \"standard\",\n    cont_embed_dropout: Optional[float] = None,\n    cont_embed_activation: Optional[str] = None,\n    quantization_setup: Optional[Dict[str, List[float]]] = None,\n    n_frequencies: Optional[int] = None,\n    sigma: Optional[float] = None,\n    share_last_layer: Optional[bool] = None,\n    full_embed_dropout: Optional[bool] = None,\n    input_dim: int = 32,\n    attn_dropout: float = 0.2,\n    n_heads: int = 8,\n    use_bias: bool = False,\n    with_addnorm: bool = False,\n    attn_activation: str = \"leaky_relu\",\n    n_blocks: int = 3,\n):\n    super(SelfAttentionMLP, self).__init__(\n        column_idx=column_idx,\n        input_dim=input_dim,\n        cat_embed_input=cat_embed_input,\n        cat_embed_dropout=cat_embed_dropout,\n        use_cat_bias=use_cat_bias,\n        cat_embed_activation=cat_embed_activation,\n        shared_embed=shared_embed,\n        add_shared_embed=add_shared_embed,\n        frac_shared_embed=frac_shared_embed,\n        continuous_cols=continuous_cols,\n        cont_norm_layer=cont_norm_layer,\n        embed_continuous_method=embed_continuous_method,\n        cont_embed_dropout=cont_embed_dropout,\n        cont_embed_activation=cont_embed_activation,\n        quantization_setup=quantization_setup,\n        n_frequencies=n_frequencies,\n        sigma=sigma,\n        share_last_layer=share_last_layer,\n        full_embed_dropout=full_embed_dropout,\n    )\n\n    self.attn_dropout = attn_dropout\n    self.n_heads = n_heads\n    self.use_bias = use_bias\n    self.with_addnorm = with_addnorm\n    self.attn_activation = attn_activation\n    self.n_blocks = n_blocks\n\n    self.with_cls_token = \"cls_token\" in column_idx\n    self.n_cat = len(cat_embed_input) if cat_embed_input is not None else 0\n    self.n_cont = len(continuous_cols) if continuous_cols is not None else 0\n\n    # Embeddings are instantiated at the base model\n    # Attention Blocks\n    self.encoder = nn.Sequential()\n    for i in range(n_blocks):\n        self.encoder.add_module(\n            \"attention_block\" + str(i),\n            SelfAttentionEncoder(\n                input_dim,\n                attn_dropout,\n                use_bias,\n                n_heads,\n                with_addnorm,\n                attn_activation,\n            ),\n        )\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.mlp.self_attention_mlp.SelfAttentionMLP.output_dim","title":"output_dim  <code>property</code>","text":"<pre><code>output_dim\n</code></pre> <p>The output dimension of the model. This is a required property neccesary to build the WideDeep class</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.mlp.self_attention_mlp.SelfAttentionMLP.attention_weights","title":"attention_weights  <code>property</code>","text":"<pre><code>attention_weights\n</code></pre> <p>List with the attention weights per block</p> <p>The shape of the attention weights is \\((N, H, F, F)\\), where \\(N\\) is the batch size, \\(H\\) is the number of attention heads and \\(F\\) is the number of features/columns in the dataset</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.transformers.tab_transformer.TabTransformer","title":"TabTransformer","text":"<pre><code>TabTransformer(\n    column_idx,\n    *,\n    cat_embed_input=None,\n    cat_embed_dropout=None,\n    use_cat_bias=None,\n    cat_embed_activation=None,\n    shared_embed=None,\n    add_shared_embed=None,\n    frac_shared_embed=None,\n    continuous_cols=None,\n    cont_norm_layer=None,\n    embed_continuous=None,\n    embed_continuous_method=None,\n    cont_embed_dropout=None,\n    cont_embed_activation=None,\n    quantization_setup=None,\n    n_frequencies=None,\n    sigma=None,\n    share_last_layer=None,\n    full_embed_dropout=None,\n    input_dim=32,\n    n_heads=8,\n    use_qkv_bias=False,\n    n_blocks=4,\n    attn_dropout=0.2,\n    ff_dropout=0.1,\n    ff_factor=4,\n    transformer_activation=\"gelu\",\n    use_linear_attention=False,\n    use_flash_attention=False,\n    mlp_hidden_dims=None,\n    mlp_activation=\"relu\",\n    mlp_dropout=0.1,\n    mlp_batchnorm=False,\n    mlp_batchnorm_last=False,\n    mlp_linear_first=True\n)\n</code></pre> <p>               Bases: <code>BaseTabularModelWithAttention</code></p> <p>Defines our adptation of the TabTransformer model that can be used as the <code>deeptabular</code> component of a Wide &amp; Deep model or independently by itself.</p> <p>Most of the parameters for this class are <code>Optional</code> since the use of categorical or continuous is in fact optional (i.e. one can use categorical features only, continuous features only or both).</p> <p> NOTE: This is an enhanced adaptation of the model described in the paper. It can be considered as the flagship of our transformer family of models for tabular data and offers mutiple, additional features relative to the original publication(and some other models in the library)</p> <p>Parameters:</p> <ul> <li> <code>column_idx</code>               (<code>Dict[str, int]</code>)           \u2013            <p>Dict containing the index of the columns that will be passed through the <code>TabMlp</code> model. Required to slice the tensors. e.g. {'education': 0, 'relationship': 1, 'workclass': 2, ...}.</p> </li> <li> <code>cat_embed_input</code>               (<code>Optional[List[Tuple[str, int]]]</code>, default:                   <code>None</code> )           \u2013            <p>List of Tuples with the column name and number of unique values and embedding dimension. e.g. [(education, 11), ...]</p> </li> <li> <code>cat_embed_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Categorical embeddings dropout. If <code>None</code>, it will default to 0.</p> </li> <li> <code>use_cat_bias</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating if bias will be used for the categorical embeddings. If <code>None</code>, it will default to 'False'.</p> </li> <li> <code>cat_embed_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the categorical embeddings, if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> </li> <li> <code>shared_embed</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating if the embeddings will be \"shared\". The idea behind <code>shared_embed</code> is described in the Appendix A in the TabTransformer paper: 'The goal of having column embedding is to enable the model to distinguish the classes in one column from those in the other columns'. In other words, the idea is to let the model learn which column is embedded at the time. See: <code>pytorch_widedeep.models.transformers._layers.SharedEmbeddings</code>.</p> </li> <li> <code>add_shared_embed</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>The two embedding sharing strategies are: 1) add the shared embeddings to the column embeddings or 2) to replace the first <code>frac_shared_embed</code> with the shared embeddings. See <code>pytorch_widedeep.models.embeddings_layers.SharedEmbeddings</code> If 'None' is passed, it will default to 'False'.</p> </li> <li> <code>frac_shared_embed</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>The fraction of embeddings that will be shared (if <code>add_shared_embed = False</code>) by all the different categories for one particular column. If 'None' is passed, it will default to 0.0.</p> </li> <li> <code>continuous_cols</code>               (<code>Optional[List[str]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the name of the numeric (aka continuous) columns</p> </li> <li> <code>cont_norm_layer</code>               (<code>Optional[Literal[batchnorm, layernorm]]</code>, default:                   <code>None</code> )           \u2013            <p>Type of normalization layer applied to the continuous features. Options are: 'layernorm' and 'batchnorm'. if <code>None</code>, no normalization layer will be used.</p> </li> <li> <code>embed_continuous_method</code>               (<code>Optional[Literal[standard, piecewise, periodic]]</code>, default:                   <code>None</code> )           \u2013            <p>Method to use to embed the continuous features. Options are: 'standard', 'periodic' or 'piecewise'. The 'standard' embedding method is based on the FT-Transformer implementation presented in the paper: Revisiting Deep Learning Models for Tabular Data. The 'periodic' and_'piecewise'_ methods were presented in the paper: On Embeddings for Numerical Features in Tabular Deep Learning. Please, read the papers for details.</p> </li> <li> <code>cont_embed_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p> </li> <li> <code>cont_embed_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the continuous embeddings if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported. If <code>None</code>, no activation function will be applied.</p> </li> <li> <code>quantization_setup</code>               (<code>Optional[Dict[str, List[float]]]</code>, default:                   <code>None</code> )           \u2013            <p>This parameter is used when the 'piecewise' method is used to embed the continuous cols. It is a dict where keys are the name of the continuous columns and values are lists with the boundaries for the quantization of the continuous_cols. See the examples for details. If If the 'piecewise' method is used, this parameter is required.</p> </li> <li> <code>n_frequencies</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>This is the so called 'k' in their paper On Embeddings for Numerical Features in Tabular Deep Learning, and is the number of 'frequencies' that will be used to represent each continuous column. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>sigma</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>This is the sigma parameter in the paper mentioned when describing the previous parameters and it is used to initialise the 'frequency weights'. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>share_last_layer</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>This parameter is not present in the before mentioned paper but it is implemented in the official repo. If <code>True</code> the linear layer that turns the frequencies into embeddings will be shared across the continuous columns. If <code>False</code> a different linear layer will be used for each continuous column. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>full_embed_dropout</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>If <code>True</code>, the full embedding corresponding to a column will be masked out/dropout. If <code>None</code>, it will default to <code>False</code>.</p> </li> <li> <code>input_dim</code>               (<code>int</code>, default:                   <code>32</code> )           \u2013            <p>The so-called dimension of the model. Is the number of embeddings used to encode the categorical and/or continuous columns</p> </li> <li> <code>n_heads</code>               (<code>int</code>, default:                   <code>8</code> )           \u2013            <p>Number of attention heads per Transformer block</p> </li> <li> <code>use_qkv_bias</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether or not to use bias in the Q, K, and V projection layers.</p> </li> <li> <code>n_blocks</code>               (<code>int</code>, default:                   <code>4</code> )           \u2013            <p>Number of Transformer blocks</p> </li> <li> <code>attn_dropout</code>               (<code>float</code>, default:                   <code>0.2</code> )           \u2013            <p>Dropout that will be applied to the Multi-Head Attention layers</p> </li> <li> <code>ff_dropout</code>               (<code>float</code>, default:                   <code>0.1</code> )           \u2013            <p>Dropout that will be applied to the FeedForward network</p> </li> <li> <code>ff_factor</code>               (<code>int</code>, default:                   <code>4</code> )           \u2013            <p>Multiplicative factor applied to the first layer of the FF network in each Transformer block, This is normally set to 4.</p> </li> <li> <code>transformer_activation</code>               (<code>str</code>, default:                   <code>'gelu'</code> )           \u2013            <p>Transformer Encoder activation function. 'tanh', 'relu', 'leaky_relu', 'gelu', 'geglu' and 'reglu' are supported</p> </li> <li> <code>use_linear_attention</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating if Linear Attention (from Transformers are RNNs: Fast Autoregressive Transformers with Linear Attention) will be used. The inclusing of this mode of attention is inspired by this post, where the Uber team finds that this attention mechanism leads to the best results for their tabular data.</p> </li> <li> <code>use_flash_attention</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating if Flash Attention will be used. </p> </li> <li> <code>mlp_hidden_dims</code>               (<code>Optional[List[int]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the number of neurons per dense layer in the MLP. e.g: [64, 32]. If not provided no MLP on top of the final Transformer block will be used.</p> </li> <li> <code>mlp_activation</code>               (<code>str</code>, default:                   <code>'relu'</code> )           \u2013            <p>Activation function for the dense layers of the MLP. Currently 'tanh', 'relu', 'leaky'_relu' and _'gelu' are supported. If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to 'relu'.</p> </li> <li> <code>mlp_dropout</code>               (<code>float</code>, default:                   <code>0.1</code> )           \u2013            <p>float with the dropout between the dense layers of the MLP. If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to 0.0.</p> </li> <li> <code>mlp_batchnorm</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether or not batch normalization will be applied to the dense layers If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to False.</p> </li> <li> <code>mlp_batchnorm_last</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether or not batch normalization will be applied to the last of the dense layers If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to False.</p> </li> <li> <code>mlp_linear_first</code>               (<code>bool</code>, default:                   <code>True</code> )           \u2013            <p>Boolean indicating the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code> If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to <code>True</code>.</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>encoder</code>               (<code>Module</code>)           \u2013            <p>Sequence of Transformer blocks</p> </li> <li> <code>mlp</code>               (<code>Module</code>)           \u2013            <p>MLP component in the model</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import TabTransformer\n&gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n&gt;&gt;&gt; colnames = ['a', 'b', 'c', 'd', 'e']\n&gt;&gt;&gt; cat_embed_input = [(u,i) for u,i in zip(colnames[:4], [4]*4)]\n&gt;&gt;&gt; continuous_cols = ['e']\n&gt;&gt;&gt; column_idx = {k:v for v,k in enumerate(colnames)}\n&gt;&gt;&gt; model = TabTransformer(column_idx=column_idx, cat_embed_input=cat_embed_input, continuous_cols=continuous_cols)\n&gt;&gt;&gt; out = model(X_tab)\n</code></pre> Source code in <code>pytorch_widedeep/models/tabular/transformers/tab_transformer.py</code> <pre><code>def __init__(\n    self,\n    column_idx: Dict[str, int],\n    *,\n    cat_embed_input: Optional[List[Tuple[str, int]]] = None,\n    cat_embed_dropout: Optional[float] = None,\n    use_cat_bias: Optional[bool] = None,\n    cat_embed_activation: Optional[str] = None,\n    shared_embed: Optional[bool] = None,\n    add_shared_embed: Optional[bool] = None,\n    frac_shared_embed: Optional[float] = None,\n    continuous_cols: Optional[List[str]] = None,\n    cont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\n    embed_continuous: Optional[bool] = None,\n    embed_continuous_method: Optional[\n        Literal[\"standard\", \"piecewise\", \"periodic\"]\n    ] = None,\n    cont_embed_dropout: Optional[float] = None,\n    cont_embed_activation: Optional[str] = None,\n    quantization_setup: Optional[Dict[str, List[float]]] = None,\n    n_frequencies: Optional[int] = None,\n    sigma: Optional[float] = None,\n    share_last_layer: Optional[bool] = None,\n    full_embed_dropout: Optional[bool] = None,\n    input_dim: int = 32,\n    n_heads: int = 8,\n    use_qkv_bias: bool = False,\n    n_blocks: int = 4,\n    attn_dropout: float = 0.2,\n    ff_dropout: float = 0.1,\n    ff_factor: int = 4,\n    transformer_activation: str = \"gelu\",\n    use_linear_attention: bool = False,\n    use_flash_attention: bool = False,\n    mlp_hidden_dims: Optional[List[int]] = None,\n    mlp_activation: str = \"relu\",\n    mlp_dropout: float = 0.1,\n    mlp_batchnorm: bool = False,\n    mlp_batchnorm_last: bool = False,\n    mlp_linear_first: bool = True,\n):\n    super(TabTransformer, self).__init__(\n        column_idx=column_idx,\n        cat_embed_input=cat_embed_input,\n        cat_embed_dropout=cat_embed_dropout,\n        use_cat_bias=use_cat_bias,\n        cat_embed_activation=cat_embed_activation,\n        shared_embed=shared_embed,\n        add_shared_embed=add_shared_embed,\n        frac_shared_embed=frac_shared_embed,\n        continuous_cols=continuous_cols,\n        cont_norm_layer=cont_norm_layer,\n        # UGLY hack to be able to use the base class __init__ method\n        embed_continuous_method=(\n            \"standard\"\n            if embed_continuous_method is None\n            else embed_continuous_method\n        ),\n        cont_embed_dropout=cont_embed_dropout,\n        cont_embed_activation=cont_embed_activation,\n        quantization_setup=quantization_setup,\n        n_frequencies=n_frequencies,\n        sigma=sigma,\n        share_last_layer=share_last_layer,\n        input_dim=input_dim,\n        full_embed_dropout=full_embed_dropout,\n    )\n\n    # we overwrite the embed_continuous_method parameter that was set\n    # to 'standard' if embed_continuous_method is None. In other words,\n    # this will already have the right value if the user provided a value\n    # for embed_continuous_method, otherwise will be \"standard\" when it\n    # should be None\n    self.embed_continuous_method = embed_continuous_method\n\n    if embed_continuous is not None:\n        self.embed_continuous = embed_continuous\n        if embed_continuous and embed_continuous_method is None:\n            raise ValueError(\n                \"If 'embed_continuous' is True, 'embed_continuous_method' must be \"\n                \"one of 'standard', 'piecewise' or 'periodic'.\"\n            )\n    elif self.embed_continuous_method is not None:\n        self.embed_continuous = True\n    else:\n        self.embed_continuous = False\n\n    self.n_heads = n_heads\n    self.use_qkv_bias = use_qkv_bias\n    self.n_blocks = n_blocks\n    self.attn_dropout = attn_dropout\n    self.ff_dropout = ff_dropout\n    self.transformer_activation = transformer_activation\n    self.use_linear_attention = use_linear_attention\n    self.use_flash_attention = use_flash_attention\n    self.ff_factor = ff_factor\n\n    self.mlp_hidden_dims = mlp_hidden_dims\n    self.mlp_activation = mlp_activation\n    self.mlp_dropout = mlp_dropout\n    self.mlp_batchnorm = mlp_batchnorm\n    self.mlp_batchnorm_last = mlp_batchnorm_last\n    self.mlp_linear_first = mlp_linear_first\n\n    self.with_cls_token = \"cls_token\" in column_idx\n    self.n_cat = len(cat_embed_input) if cat_embed_input is not None else 0\n    self.n_cont = len(continuous_cols) if continuous_cols is not None else 0\n\n    if self.n_cont and not self.n_cat and not self.embed_continuous:\n        raise ValueError(\n            \"If only continuous features are used 'embed_continuous' must be set to 'True'\"\n        )\n\n    # Embeddings are instantiated at the base model\n    # Transformer blocks\n    self.encoder = nn.Sequential()\n    for i in range(n_blocks):\n        self.encoder.add_module(\n            \"transformer_block\" + str(i),\n            TransformerEncoder(\n                input_dim,\n                n_heads,\n                use_qkv_bias,\n                attn_dropout,\n                ff_dropout,\n                ff_factor,\n                transformer_activation,\n                use_linear_attention,\n                use_flash_attention,\n            ),\n        )\n\n    self.mlp_first_hidden_dim = self._mlp_first_hidden_dim()\n\n    if self.mlp_hidden_dims is not None:\n        self.mlp = MLP(\n            d_hidden=[self.mlp_first_hidden_dim] + self.mlp_hidden_dims,\n            activation=(\n                \"relu\" if self.mlp_activation is None else self.mlp_activation\n            ),\n            dropout=0.0 if self.mlp_dropout is None else self.mlp_dropout,\n            batchnorm=False if self.mlp_batchnorm is None else self.mlp_batchnorm,\n            batchnorm_last=(\n                False\n                if self.mlp_batchnorm_last is None\n                else self.mlp_batchnorm_last\n            ),\n            linear_first=(\n                False if self.mlp_linear_first is None else self.mlp_linear_first\n            ),\n        )\n    else:\n        self.mlp = None\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.transformers.tab_transformer.TabTransformer.output_dim","title":"output_dim  <code>property</code>","text":"<pre><code>output_dim\n</code></pre> <p>The output dimension of the model. This is a required property neccesary to build the <code>WideDeep</code> class</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.transformers.tab_transformer.TabTransformer.attention_weights","title":"attention_weights  <code>property</code>","text":"<pre><code>attention_weights\n</code></pre> <p>List with the attention weights per block</p> <p>The shape of the attention weights is \\((N, H, F, F)\\), where \\(N\\) is the batch size, \\(H\\) is the number of attention heads and \\(F\\) is the number of features/columns in the dataset</p> <p> NOTE: if flash attention or linear attention are used, no attention weights are saved during the training process and calling this property will throw a ValueError</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.transformers.saint.SAINT","title":"SAINT","text":"<pre><code>SAINT(\n    column_idx,\n    *,\n    cat_embed_input=None,\n    cat_embed_dropout=None,\n    use_cat_bias=None,\n    cat_embed_activation=None,\n    shared_embed=None,\n    add_shared_embed=None,\n    frac_shared_embed=None,\n    continuous_cols=None,\n    cont_norm_layer=None,\n    embed_continuous_method=\"standard\",\n    cont_embed_dropout=None,\n    cont_embed_activation=None,\n    quantization_setup=None,\n    n_frequencies=None,\n    sigma=None,\n    share_last_layer=None,\n    full_embed_dropout=None,\n    input_dim=32,\n    use_qkv_bias=False,\n    n_heads=8,\n    n_blocks=2,\n    attn_dropout=0.1,\n    ff_dropout=0.2,\n    ff_factor=4,\n    transformer_activation=\"gelu\",\n    mlp_hidden_dims=None,\n    mlp_activation=None,\n    mlp_dropout=None,\n    mlp_batchnorm=None,\n    mlp_batchnorm_last=None,\n    mlp_linear_first=None\n)\n</code></pre> <p>               Bases: <code>BaseTabularModelWithAttention</code></p> <p>Defines a SAINT model that can be used as the <code>deeptabular</code> component of a Wide &amp; Deep model or independently by itself.</p> <p>Most of the parameters for this class are <code>Optional</code> since the use of categorical or continuous is in fact optional (i.e. one can use categorical features only, continuous features only or both).</p> <p> NOTE: This is an slightly modified and enhanced  version of the model described in the paper,</p> <p>Parameters:</p> <ul> <li> <code>column_idx</code>               (<code>Dict[str, int]</code>)           \u2013            <p>Dict containing the index of the columns that will be passed through the <code>TabMlp</code> model. Required to slice the tensors. e.g. {'education': 0, 'relationship': 1, 'workclass': 2, ...}.</p> </li> <li> <code>cat_embed_input</code>               (<code>Optional[List[Tuple[str, int]]]</code>, default:                   <code>None</code> )           \u2013            <p>List of Tuples with the column name and number of unique values and embedding dimension. e.g. [(education, 11), ...]</p> </li> <li> <code>cat_embed_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Categorical embeddings dropout. If <code>None</code>, it will default to 0.</p> </li> <li> <code>use_cat_bias</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating if bias will be used for the categorical embeddings. If <code>None</code>, it will default to 'False'.</p> </li> <li> <code>cat_embed_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the categorical embeddings, if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> </li> <li> <code>shared_embed</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating if the embeddings will be \"shared\". The idea behind <code>shared_embed</code> is described in the Appendix A in the TabTransformer paper: 'The goal of having column embedding is to enable the model to distinguish the classes in one column from those in the other columns'. In other words, the idea is to let the model learn which column is embedded at the time. See: <code>pytorch_widedeep.models.transformers._layers.SharedEmbeddings</code>.</p> </li> <li> <code>add_shared_embed</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>The two embedding sharing strategies are: 1) add the shared embeddings to the column embeddings or 2) to replace the first <code>frac_shared_embed</code> with the shared embeddings. See <code>pytorch_widedeep.models.embeddings_layers.SharedEmbeddings</code> If 'None' is passed, it will default to 'False'.</p> </li> <li> <code>frac_shared_embed</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>The fraction of embeddings that will be shared (if <code>add_shared_embed = False</code>) by all the different categories for one particular column. If 'None' is passed, it will default to 0.0.</p> </li> <li> <code>continuous_cols</code>               (<code>Optional[List[str]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the name of the numeric (aka continuous) columns</p> </li> <li> <code>cont_norm_layer</code>               (<code>Optional[Literal[batchnorm, layernorm]]</code>, default:                   <code>None</code> )           \u2013            <p>Type of normalization layer applied to the continuous features. Options are: 'layernorm' and 'batchnorm'. if <code>None</code>, no normalization layer will be used.</p> </li> <li> <code>embed_continuous_method</code>               (<code>Optional[Literal[standard, piecewise, periodic]]</code>, default:                   <code>'standard'</code> )           \u2013            <p>Method to use to embed the continuous features. Options are: 'standard', 'periodic' or 'piecewise'. The 'standard' embedding method is based on the FT-Transformer implementation presented in the paper: Revisiting Deep Learning Models for Tabular Data. The 'periodic' and_'piecewise'_ methods were presented in the paper: On Embeddings for Numerical Features in Tabular Deep Learning. Please, read the papers for details.</p> </li> <li> <code>cont_embed_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p> </li> <li> <code>cont_embed_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the continuous embeddings if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported. If <code>None</code>, no activation function will be applied.</p> </li> <li> <code>quantization_setup</code>               (<code>Optional[Dict[str, List[float]]]</code>, default:                   <code>None</code> )           \u2013            <p>This parameter is used when the 'piecewise' method is used to embed the continuous cols. It is a dict where keys are the name of the continuous columns and values are lists with the boundaries for the quantization of the continuous_cols. See the examples for details. If If the 'piecewise' method is used, this parameter is required.</p> </li> <li> <code>n_frequencies</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>This is the so called 'k' in their paper On Embeddings for Numerical Features in Tabular Deep Learning, and is the number of 'frequencies' that will be used to represent each continuous column. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>sigma</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>This is the sigma parameter in the paper mentioned when describing the previous parameters and it is used to initialise the 'frequency weights'. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>share_last_layer</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>This parameter is not present in the before mentioned paper but it is implemented in the official repo. If <code>True</code> the linear layer that turns the frequencies into embeddings will be shared across the continuous columns. If <code>False</code> a different linear layer will be used for each continuous column. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>full_embed_dropout</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>If <code>True</code>, the full embedding corresponding to a column will be masked out/dropout. If <code>None</code>, it will default to <code>False</code>.</p> </li> <li> <code>input_dim</code>               (<code>int</code>, default:                   <code>32</code> )           \u2013            <p>The so-called dimension of the model. Is the number of embeddings used to encode the categorical and/or continuous columns</p> </li> <li> <code>n_heads</code>               (<code>int</code>, default:                   <code>8</code> )           \u2013            <p>Number of attention heads per Transformer block</p> </li> <li> <code>use_qkv_bias</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether or not to use bias in the Q, K, and V projection layers</p> </li> <li> <code>n_blocks</code>               (<code>int</code>, default:                   <code>2</code> )           \u2013            <p>Number of SAINT-Transformer blocks.</p> </li> <li> <code>attn_dropout</code>               (<code>float</code>, default:                   <code>0.1</code> )           \u2013            <p>Dropout that will be applied to the Multi-Head Attention column and row layers</p> </li> <li> <code>ff_dropout</code>               (<code>float</code>, default:                   <code>0.2</code> )           \u2013            <p>Dropout that will be applied to the FeedForward network</p> </li> <li> <code>ff_factor</code>               (<code>int</code>, default:                   <code>4</code> )           \u2013            <p>Multiplicative factor applied to the first layer of the FF network in each Transformer block, This is normally set to 4.</p> </li> <li> <code>transformer_activation</code>               (<code>str</code>, default:                   <code>'gelu'</code> )           \u2013            <p>Transformer Encoder activation function. 'tanh', 'relu', 'leaky_relu', 'gelu', 'geglu' and 'reglu' are supported</p> </li> <li> <code>mlp_hidden_dims</code>               (<code>Optional[List[int]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the number of neurons per dense layer in the MLP. e.g: [64, 32]. If not provided no MLP on top of the final Transformer block will be used.</p> </li> <li> <code>mlp_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the dense layers of the MLP. Currently 'tanh', 'relu', 'leaky'_relu' and _'gelu' are supported. If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to 'relu'.</p> </li> <li> <code>mlp_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>float with the dropout between the dense layers of the MLP. If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to 0.0.</p> </li> <li> <code>mlp_batchnorm</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating whether or not batch normalization will be applied to the dense layers If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to False.</p> </li> <li> <code>mlp_batchnorm_last</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating whether or not batch normalization will be applied to the last of the dense layers If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to False.</p> </li> <li> <code>mlp_linear_first</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code> If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to <code>True</code>.</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>encoder</code>               (<code>Module</code>)           \u2013            <p>Sequence of SAINT-Transformer blocks</p> </li> <li> <code>mlp</code>               (<code>Module</code>)           \u2013            <p>MLP component in the model</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import SAINT\n&gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n&gt;&gt;&gt; colnames = ['a', 'b', 'c', 'd', 'e']\n&gt;&gt;&gt; cat_embed_input = [(u,i) for u,i in zip(colnames[:4], [4]*4)]\n&gt;&gt;&gt; continuous_cols = ['e']\n&gt;&gt;&gt; column_idx = {k:v for v,k in enumerate(colnames)}\n&gt;&gt;&gt; model = SAINT(column_idx=column_idx, cat_embed_input=cat_embed_input, continuous_cols=continuous_cols)\n&gt;&gt;&gt; out = model(X_tab)\n</code></pre> Source code in <code>pytorch_widedeep/models/tabular/transformers/saint.py</code> <pre><code>def __init__(\n    self,\n    column_idx: Dict[str, int],\n    *,\n    cat_embed_input: Optional[List[Tuple[str, int]]] = None,\n    cat_embed_dropout: Optional[float] = None,\n    use_cat_bias: Optional[bool] = None,\n    cat_embed_activation: Optional[str] = None,\n    shared_embed: Optional[bool] = None,\n    add_shared_embed: Optional[bool] = None,\n    frac_shared_embed: Optional[float] = None,\n    continuous_cols: Optional[List[str]] = None,\n    cont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\n    embed_continuous_method: Optional[\n        Literal[\"standard\", \"piecewise\", \"periodic\"]\n    ] = \"standard\",\n    cont_embed_dropout: Optional[float] = None,\n    cont_embed_activation: Optional[str] = None,\n    quantization_setup: Optional[Dict[str, List[float]]] = None,\n    n_frequencies: Optional[int] = None,\n    sigma: Optional[float] = None,\n    share_last_layer: Optional[bool] = None,\n    full_embed_dropout: Optional[bool] = None,\n    input_dim: int = 32,\n    use_qkv_bias: bool = False,\n    n_heads: int = 8,\n    n_blocks: int = 2,\n    attn_dropout: float = 0.1,\n    ff_dropout: float = 0.2,\n    ff_factor: int = 4,\n    transformer_activation: str = \"gelu\",\n    mlp_hidden_dims: Optional[List[int]] = None,\n    mlp_activation: Optional[str] = None,\n    mlp_dropout: Optional[float] = None,\n    mlp_batchnorm: Optional[bool] = None,\n    mlp_batchnorm_last: Optional[bool] = None,\n    mlp_linear_first: Optional[bool] = None,\n):\n    super(SAINT, self).__init__(\n        column_idx=column_idx,\n        cat_embed_input=cat_embed_input,\n        cat_embed_dropout=cat_embed_dropout,\n        use_cat_bias=use_cat_bias,\n        cat_embed_activation=cat_embed_activation,\n        shared_embed=shared_embed,\n        add_shared_embed=add_shared_embed,\n        frac_shared_embed=frac_shared_embed,\n        continuous_cols=continuous_cols,\n        cont_norm_layer=cont_norm_layer,\n        embed_continuous_method=embed_continuous_method,\n        cont_embed_dropout=cont_embed_dropout,\n        cont_embed_activation=cont_embed_activation,\n        input_dim=input_dim,\n        quantization_setup=quantization_setup,\n        n_frequencies=n_frequencies,\n        sigma=sigma,\n        share_last_layer=share_last_layer,\n        full_embed_dropout=full_embed_dropout,\n    )\n\n    self.use_qkv_bias = use_qkv_bias\n    self.n_heads = n_heads\n    self.n_blocks = n_blocks\n    self.attn_dropout = attn_dropout\n    self.ff_dropout = ff_dropout\n    self.ff_factor = ff_factor\n    self.transformer_activation = transformer_activation\n\n    self.mlp_hidden_dims = mlp_hidden_dims\n    self.mlp_activation = mlp_activation\n    self.mlp_dropout = mlp_dropout\n    self.mlp_batchnorm = mlp_batchnorm\n    self.mlp_batchnorm_last = mlp_batchnorm_last\n    self.mlp_linear_first = mlp_linear_first\n\n    self.with_cls_token = \"cls_token\" in column_idx\n    self.n_cat = len(cat_embed_input) if cat_embed_input is not None else 0\n    self.n_cont = len(continuous_cols) if continuous_cols is not None else 0\n    self.n_feats = self.n_cat + self.n_cont\n\n    # Embeddings are instantiated at the base model\n    # Transformer blocks\n    self.encoder = nn.Sequential()\n    for i in range(n_blocks):\n        self.encoder.add_module(\n            \"saint_block\" + str(i),\n            SaintEncoder(\n                input_dim,\n                n_heads,\n                use_qkv_bias,\n                attn_dropout,\n                ff_dropout,\n                ff_factor,\n                transformer_activation,\n                self.n_feats,\n            ),\n        )\n\n    self.mlp_first_hidden_dim = (\n        self.input_dim if self.with_cls_token else (self.n_feats * self.input_dim)\n    )\n\n    # Mlp: adding an MLP on top of the Resnet blocks is optional and\n    # therefore all related params are optional\n    if self.mlp_hidden_dims is not None:\n        self.mlp = MLP(\n            d_hidden=[self.mlp_first_hidden_dim] + self.mlp_hidden_dims,\n            activation=(\n                \"relu\" if self.mlp_activation is None else self.mlp_activation\n            ),\n            dropout=0.0 if self.mlp_dropout is None else self.mlp_dropout,\n            batchnorm=False if self.mlp_batchnorm is None else self.mlp_batchnorm,\n            batchnorm_last=(\n                False\n                if self.mlp_batchnorm_last is None\n                else self.mlp_batchnorm_last\n            ),\n            linear_first=(\n                False if self.mlp_linear_first is None else self.mlp_linear_first\n            ),\n        )\n    else:\n        self.mlp = None\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.transformers.saint.SAINT.output_dim","title":"output_dim  <code>property</code>","text":"<pre><code>output_dim\n</code></pre> <p>The output dimension of the model. This is a required property neccesary to build the <code>WideDeep</code> class</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.transformers.saint.SAINT.attention_weights","title":"attention_weights  <code>property</code>","text":"<pre><code>attention_weights\n</code></pre> <p>List with the attention weights. Each element of the list is a tuple where the first and the second elements are the column and row attention weights respectively</p> <p>The shape of the attention weights is:</p> <ul> <li> <p>column attention: \\((N, H, F, F)\\)</p> </li> <li> <p>row attention: \\((1, H, N, N)\\)</p> </li> </ul> <p>where \\(N\\) is the batch size, \\(H\\) is the number of heads and \\(F\\) is the number of features/columns in the dataset</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.transformers.ft_transformer.FTTransformer","title":"FTTransformer","text":"<pre><code>FTTransformer(\n    column_idx,\n    *,\n    cat_embed_input=None,\n    cat_embed_dropout=None,\n    use_cat_bias=None,\n    cat_embed_activation=None,\n    shared_embed=None,\n    add_shared_embed=None,\n    frac_shared_embed=None,\n    continuous_cols=None,\n    cont_norm_layer=None,\n    embed_continuous_method=\"standard\",\n    cont_embed_dropout=None,\n    cont_embed_activation=None,\n    quantization_setup=None,\n    n_frequencies=None,\n    sigma=None,\n    share_last_layer=None,\n    full_embed_dropout=None,\n    input_dim=64,\n    kv_compression_factor=0.5,\n    kv_sharing=False,\n    use_qkv_bias=False,\n    n_heads=8,\n    n_blocks=4,\n    attn_dropout=0.2,\n    ff_dropout=0.1,\n    ff_factor=1.33,\n    transformer_activation=\"reglu\",\n    mlp_hidden_dims=None,\n    mlp_activation=None,\n    mlp_dropout=None,\n    mlp_batchnorm=None,\n    mlp_batchnorm_last=None,\n    mlp_linear_first=None\n)\n</code></pre> <p>               Bases: <code>BaseTabularModelWithAttention</code></p> <p>Defines a FTTransformer model that can be used as the <code>deeptabular</code> component of a Wide &amp; Deep model or independently by itself.</p> <p>Most of the parameters for this class are <code>Optional</code> since the use of categorical or continuous is in fact optional (i.e. one can use categorical features only, continuous features only or both).</p> <p>Parameters:</p> <ul> <li> <code>column_idx</code>               (<code>Dict[str, int]</code>)           \u2013            <p>Dict containing the index of the columns that will be passed through the <code>TabMlp</code> model. Required to slice the tensors. e.g. {'education': 0, 'relationship': 1, 'workclass': 2, ...}.</p> </li> <li> <code>cat_embed_input</code>               (<code>Optional[List[Tuple[str, int]]]</code>, default:                   <code>None</code> )           \u2013            <p>List of Tuples with the column name and number of unique values and embedding dimension. e.g. [(education, 11), ...]</p> </li> <li> <code>cat_embed_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Categorical embeddings dropout. If <code>None</code>, it will default to 0.</p> </li> <li> <code>use_cat_bias</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating if bias will be used for the categorical embeddings. If <code>None</code>, it will default to 'False'.</p> </li> <li> <code>cat_embed_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the categorical embeddings, if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> </li> <li> <code>shared_embed</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating if the embeddings will be \"shared\". The idea behind <code>shared_embed</code> is described in the Appendix A in the TabTransformer paper: 'The goal of having column embedding is to enable the model to distinguish the classes in one column from those in the other columns'. In other words, the idea is to let the model learn which column is embedded at the time. See: <code>pytorch_widedeep.models.transformers._layers.SharedEmbeddings</code>.</p> </li> <li> <code>add_shared_embed</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>The two embedding sharing strategies are: 1) add the shared embeddings to the column embeddings or 2) to replace the first <code>frac_shared_embed</code> with the shared embeddings. See <code>pytorch_widedeep.models.embeddings_layers.SharedEmbeddings</code> If 'None' is passed, it will default to 'False'.</p> </li> <li> <code>frac_shared_embed</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>The fraction of embeddings that will be shared (if <code>add_shared_embed = False</code>) by all the different categories for one particular column. If 'None' is passed, it will default to 0.0.</p> </li> <li> <code>continuous_cols</code>               (<code>Optional[List[str]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the name of the numeric (aka continuous) columns</p> </li> <li> <code>cont_norm_layer</code>               (<code>Optional[Literal[batchnorm, layernorm]]</code>, default:                   <code>None</code> )           \u2013            <p>Type of normalization layer applied to the continuous features. Options are: 'layernorm' and 'batchnorm'. if <code>None</code>, no normalization layer will be used.</p> </li> <li> <code>embed_continuous_method</code>               (<code>Optional[Literal[standard, piecewise, periodic]]</code>, default:                   <code>'standard'</code> )           \u2013            <p>Method to use to embed the continuous features. Options are: 'standard', 'periodic' or 'piecewise'. The 'standard' embedding method is based on the FT-Transformer implementation presented in the paper: Revisiting Deep Learning Models for Tabular Data. The 'periodic' and_'piecewise'_ methods were presented in the paper: On Embeddings for Numerical Features in Tabular Deep Learning. Please, read the papers for details.</p> </li> <li> <code>cont_embed_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p> </li> <li> <code>cont_embed_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the continuous embeddings if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported. If <code>None</code>, no activation function will be applied.</p> </li> <li> <code>quantization_setup</code>               (<code>Optional[Dict[str, List[float]]]</code>, default:                   <code>None</code> )           \u2013            <p>This parameter is used when the 'piecewise' method is used to embed the continuous cols. It is a dict where keys are the name of the continuous columns and values are lists with the boundaries for the quantization of the continuous_cols. See the examples for details. If If the 'piecewise' method is used, this parameter is required.</p> </li> <li> <code>n_frequencies</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>This is the so called 'k' in their paper On Embeddings for Numerical Features in Tabular Deep Learning, and is the number of 'frequencies' that will be used to represent each continuous column. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>sigma</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>This is the sigma parameter in the paper mentioned when describing the previous parameters and it is used to initialise the 'frequency weights'. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>share_last_layer</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>This parameter is not present in the before mentioned paper but it is implemented in the official repo. If <code>True</code> the linear layer that turns the frequencies into embeddings will be shared across the continuous columns. If <code>False</code> a different linear layer will be used for each continuous column. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>full_embed_dropout</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>If <code>True</code>, the full embedding corresponding to a column will be masked out/dropout. If <code>None</code>, it will default to <code>False</code>.</p> </li> <li> <code>input_dim</code>               (<code>int</code>, default:                   <code>64</code> )           \u2013            <p>The so-called dimension of the model. Is the number of embeddings used to encode the categorical and/or continuous columns.</p> </li> <li> <code>kv_compression_factor</code>               (<code>float</code>, default:                   <code>0.5</code> )           \u2013            <p>By default, the FTTransformer uses Linear Attention (See Linformer: Self-Attention with Linear Complexity ). The compression factor that will be used to reduce the input sequence length. If we denote the resulting sequence length as \\(k = int(kv_{compression \\space factor} \\times s)\\) where \\(s\\) is the input sequence length.</p> </li> <li> <code>kv_sharing</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating if the \\(E\\) and \\(F\\) projection matrices will share weights.  See Linformer: Self-Attention with Linear Complexity for details</p> </li> <li> <code>n_heads</code>               (<code>int</code>, default:                   <code>8</code> )           \u2013            <p>Number of attention heads per FTTransformer block</p> </li> <li> <code>use_qkv_bias</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether or not to use bias in the Q, K, and V projection layers</p> </li> <li> <code>n_blocks</code>               (<code>int</code>, default:                   <code>4</code> )           \u2013            <p>Number of FTTransformer blocks</p> </li> <li> <code>attn_dropout</code>               (<code>float</code>, default:                   <code>0.2</code> )           \u2013            <p>Dropout that will be applied to the Linear-Attention layers</p> </li> <li> <code>ff_dropout</code>               (<code>float</code>, default:                   <code>0.1</code> )           \u2013            <p>Dropout that will be applied to the FeedForward network</p> </li> <li> <code>ff_factor</code>               (<code>float</code>, default:                   <code>1.33</code> )           \u2013            <p>Multiplicative factor applied to the first layer of the FF network in each Transformer block, This is normally set to 4, but they use 4/3 in the paper.</p> </li> <li> <code>transformer_activation</code>               (<code>str</code>, default:                   <code>'reglu'</code> )           \u2013            <p>Transformer Encoder activation function. 'tanh', 'relu', 'leaky_relu', 'gelu', 'geglu' and 'reglu' are supported</p> </li> <li> <code>mlp_hidden_dims</code>               (<code>Optional[List[int]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the number of neurons per dense layer in the MLP. e.g: [64, 32]. If not provided no MLP on top of the final FTTransformer block will be used.</p> </li> <li> <code>mlp_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the dense layers of the MLP. Currently 'tanh', 'relu', 'leaky'_relu' and _'gelu' are supported. If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to 'relu'.</p> </li> <li> <code>mlp_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>float with the dropout between the dense layers of the MLP. If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to 0.0.</p> </li> <li> <code>mlp_batchnorm</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating whether or not batch normalization will be applied to the dense layers If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to False.</p> </li> <li> <code>mlp_batchnorm_last</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating whether or not batch normalization will be applied to the last of the dense layers If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to False.</p> </li> <li> <code>mlp_linear_first</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code> If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to <code>True</code>.</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>encoder</code>               (<code>Module</code>)           \u2013            <p>Sequence of FTTransformer blocks</p> </li> <li> <code>mlp</code>               (<code>Module</code>)           \u2013            <p>MLP component in the model</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import FTTransformer\n&gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n&gt;&gt;&gt; colnames = ['a', 'b', 'c', 'd', 'e']\n&gt;&gt;&gt; cat_embed_input = [(u,i) for u,i in zip(colnames[:4], [4]*4)]\n&gt;&gt;&gt; continuous_cols = ['e']\n&gt;&gt;&gt; column_idx = {k:v for v,k in enumerate(colnames)}\n&gt;&gt;&gt; model = FTTransformer(column_idx=column_idx, cat_embed_input=cat_embed_input, continuous_cols=continuous_cols)\n&gt;&gt;&gt; out = model(X_tab)\n</code></pre> Source code in <code>pytorch_widedeep/models/tabular/transformers/ft_transformer.py</code> <pre><code>def __init__(\n    self,\n    column_idx: Dict[str, int],\n    *,\n    cat_embed_input: Optional[List[Tuple[str, int]]] = None,\n    cat_embed_dropout: Optional[float] = None,\n    use_cat_bias: Optional[bool] = None,\n    cat_embed_activation: Optional[str] = None,\n    shared_embed: Optional[bool] = None,\n    add_shared_embed: Optional[bool] = None,\n    frac_shared_embed: Optional[float] = None,\n    continuous_cols: Optional[List[str]] = None,\n    cont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\n    embed_continuous_method: Optional[\n        Literal[\"standard\", \"piecewise\", \"periodic\"]\n    ] = \"standard\",\n    cont_embed_dropout: Optional[float] = None,\n    cont_embed_activation: Optional[str] = None,\n    quantization_setup: Optional[Dict[str, List[float]]] = None,\n    n_frequencies: Optional[int] = None,\n    sigma: Optional[float] = None,\n    share_last_layer: Optional[bool] = None,\n    full_embed_dropout: Optional[bool] = None,\n    input_dim: int = 64,\n    kv_compression_factor: float = 0.5,\n    kv_sharing: bool = False,\n    use_qkv_bias: bool = False,\n    n_heads: int = 8,\n    n_blocks: int = 4,\n    attn_dropout: float = 0.2,\n    ff_dropout: float = 0.1,\n    ff_factor: float = 1.33,\n    transformer_activation: str = \"reglu\",\n    mlp_hidden_dims: Optional[List[int]] = None,\n    mlp_activation: Optional[str] = None,\n    mlp_dropout: Optional[float] = None,\n    mlp_batchnorm: Optional[bool] = None,\n    mlp_batchnorm_last: Optional[bool] = None,\n    mlp_linear_first: Optional[bool] = None,\n):\n    super(FTTransformer, self).__init__(\n        column_idx=column_idx,\n        cat_embed_input=cat_embed_input,\n        cat_embed_dropout=cat_embed_dropout,\n        use_cat_bias=use_cat_bias,\n        cat_embed_activation=cat_embed_activation,\n        shared_embed=shared_embed,\n        add_shared_embed=add_shared_embed,\n        frac_shared_embed=frac_shared_embed,\n        continuous_cols=continuous_cols,\n        cont_norm_layer=cont_norm_layer,\n        embed_continuous_method=embed_continuous_method,\n        cont_embed_dropout=cont_embed_dropout,\n        cont_embed_activation=cont_embed_activation,\n        input_dim=input_dim,\n        quantization_setup=quantization_setup,\n        n_frequencies=n_frequencies,\n        sigma=sigma,\n        share_last_layer=share_last_layer,\n        full_embed_dropout=full_embed_dropout,\n    )\n\n    self.kv_compression_factor = kv_compression_factor\n    self.kv_sharing = kv_sharing\n    self.use_qkv_bias = use_qkv_bias\n    self.n_heads = n_heads\n    self.n_blocks = n_blocks\n    self.attn_dropout = attn_dropout\n    self.ff_dropout = ff_dropout\n    self.ff_factor = ff_factor\n    self.transformer_activation = transformer_activation\n\n    self.mlp_hidden_dims = mlp_hidden_dims\n    self.mlp_activation = mlp_activation\n    self.mlp_dropout = mlp_dropout\n    self.mlp_batchnorm = mlp_batchnorm\n    self.mlp_batchnorm_last = mlp_batchnorm_last\n    self.mlp_linear_first = mlp_linear_first\n\n    self.with_cls_token = \"cls_token\" in column_idx\n    self.n_cat = len(cat_embed_input) if cat_embed_input is not None else 0\n    self.n_cont = len(continuous_cols) if continuous_cols is not None else 0\n    self.n_feats = self.n_cat + self.n_cont\n\n    # Embeddings are instantiated at the base model\n    # Transformer blocks\n    is_first = True\n    self.encoder = nn.Sequential()\n    for i in range(n_blocks):\n        self.encoder.add_module(\n            \"fttransformer_block\" + str(i),\n            FTTransformerEncoder(\n                input_dim,\n                self.n_feats,\n                n_heads,\n                use_qkv_bias,\n                attn_dropout,\n                ff_dropout,\n                ff_factor,\n                kv_compression_factor,\n                kv_sharing,\n                transformer_activation,\n                is_first,\n            ),\n        )\n        is_first = False\n\n    self.mlp_first_hidden_dim = (\n        self.input_dim if self.with_cls_token else (self.n_feats * self.input_dim)\n    )\n\n    # Mlp: adding an MLP on top of the Resnet blocks is optional and\n    # therefore all related params are optional\n    if self.mlp_hidden_dims is not None:\n        self.mlp = MLP(\n            d_hidden=[self.mlp_first_hidden_dim] + self.mlp_hidden_dims,\n            activation=(\n                \"relu\" if self.mlp_activation is None else self.mlp_activation\n            ),\n            dropout=0.0 if self.mlp_dropout is None else self.mlp_dropout,\n            batchnorm=False if self.mlp_batchnorm is None else self.mlp_batchnorm,\n            batchnorm_last=(\n                False\n                if self.mlp_batchnorm_last is None\n                else self.mlp_batchnorm_last\n            ),\n            linear_first=(\n                False if self.mlp_linear_first is None else self.mlp_linear_first\n            ),\n        )\n    else:\n        self.mlp = None\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.transformers.ft_transformer.FTTransformer.output_dim","title":"output_dim  <code>property</code>","text":"<pre><code>output_dim\n</code></pre> <p>The output dimension of the model. This is a required property neccesary to build the <code>WideDeep</code> class</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.transformers.ft_transformer.FTTransformer.attention_weights","title":"attention_weights  <code>property</code>","text":"<pre><code>attention_weights\n</code></pre> <p>List with the attention weights per block</p> <p>The shape of the attention weights is: \\((N, H, F, k)\\), where \\(N\\) is the batch size, \\(H\\) is the number of attention heads, \\(F\\) is the number of features/columns and \\(k\\) is the reduced sequence length or dimension, i.e. \\(k = int(kv_{compression \\space factor} \\times s)\\)</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.transformers.tab_perceiver.TabPerceiver","title":"TabPerceiver","text":"<pre><code>TabPerceiver(\n    column_idx,\n    *,\n    cat_embed_input=None,\n    cat_embed_dropout=None,\n    use_cat_bias=None,\n    cat_embed_activation=None,\n    shared_embed=None,\n    add_shared_embed=None,\n    frac_shared_embed=None,\n    continuous_cols=None,\n    cont_norm_layer=None,\n    embed_continuous_method=\"standard\",\n    cont_embed_dropout=None,\n    cont_embed_activation=None,\n    quantization_setup=None,\n    n_frequencies=None,\n    sigma=None,\n    share_last_layer=None,\n    full_embed_dropout=None,\n    input_dim=32,\n    n_cross_attns=1,\n    n_cross_attn_heads=4,\n    n_latents=16,\n    latent_dim=128,\n    n_latent_heads=4,\n    n_latent_blocks=4,\n    n_perceiver_blocks=4,\n    share_weights=False,\n    attn_dropout=0.1,\n    ff_dropout=0.1,\n    ff_factor=4,\n    transformer_activation=\"geglu\",\n    mlp_hidden_dims=None,\n    mlp_activation=None,\n    mlp_dropout=None,\n    mlp_batchnorm=None,\n    mlp_batchnorm_last=None,\n    mlp_linear_first=None\n)\n</code></pre> <p>               Bases: <code>BaseTabularModelWithAttention</code></p> <p>Defines an adaptation of a Perceiver  that can be used as the <code>deeptabular</code> component of a Wide &amp; Deep model  or independently by itself.</p> <p>Most of the parameters for this class are <code>Optional</code> since the use of categorical or continuous is in fact optional (i.e. one can use categorical features only, continuous features only or both).</p> <p> NOTE: while there are scientific publications for  the <code>TabTransformer</code>, <code>SAINT</code> and <code>FTTransformer</code>, the <code>TabPerceiver</code>  and the <code>TabFastFormer</code> are our own adaptations of the  Perceiver and the  FastFormer for tabular data.</p> <p>Parameters:</p> <ul> <li> <code>column_idx</code>               (<code>Dict[str, int]</code>)           \u2013            <p>Dict containing the index of the columns that will be passed through the <code>TabMlp</code> model. Required to slice the tensors. e.g. {'education': 0, 'relationship': 1, 'workclass': 2, ...}.</p> </li> <li> <code>cat_embed_input</code>               (<code>Optional[List[Tuple[str, int]]]</code>, default:                   <code>None</code> )           \u2013            <p>List of Tuples with the column name and number of unique values and embedding dimension. e.g. [(education, 11), ...]</p> </li> <li> <code>cat_embed_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Categorical embeddings dropout. If <code>None</code>, it will default to 0.</p> </li> <li> <code>use_cat_bias</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating if bias will be used for the categorical embeddings. If <code>None</code>, it will default to 'False'.</p> </li> <li> <code>cat_embed_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the categorical embeddings, if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> </li> <li> <code>shared_embed</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating if the embeddings will be \"shared\". The idea behind <code>shared_embed</code> is described in the Appendix A in the TabTransformer paper: 'The goal of having column embedding is to enable the model to distinguish the classes in one column from those in the other columns'. In other words, the idea is to let the model learn which column is embedded at the time. See: <code>pytorch_widedeep.models.transformers._layers.SharedEmbeddings</code>.</p> </li> <li> <code>add_shared_embed</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>The two embedding sharing strategies are: 1) add the shared embeddings to the column embeddings or 2) to replace the first <code>frac_shared_embed</code> with the shared embeddings. See <code>pytorch_widedeep.models.embeddings_layers.SharedEmbeddings</code> If 'None' is passed, it will default to 'False'.</p> </li> <li> <code>frac_shared_embed</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>The fraction of embeddings that will be shared (if <code>add_shared_embed = False</code>) by all the different categories for one particular column. If 'None' is passed, it will default to 0.0.</p> </li> <li> <code>continuous_cols</code>               (<code>Optional[List[str]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the name of the numeric (aka continuous) columns</p> </li> <li> <code>cont_norm_layer</code>               (<code>Optional[Literal[batchnorm, layernorm]]</code>, default:                   <code>None</code> )           \u2013            <p>Type of normalization layer applied to the continuous features. Options are: 'layernorm' and 'batchnorm'. if <code>None</code>, no normalization layer will be used.</p> </li> <li> <code>embed_continuous_method</code>               (<code>Optional[Literal[standard, piecewise, periodic]]</code>, default:                   <code>'standard'</code> )           \u2013            <p>Method to use to embed the continuous features. Options are: 'standard', 'periodic' or 'piecewise'. The 'standard' embedding method is based on the FT-Transformer implementation presented in the paper: Revisiting Deep Learning Models for Tabular Data. The 'periodic' and_'piecewise'_ methods were presented in the paper: On Embeddings for Numerical Features in Tabular Deep Learning. Please, read the papers for details.</p> </li> <li> <code>cont_embed_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p> </li> <li> <code>cont_embed_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the continuous embeddings if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported. If <code>None</code>, no activation function will be applied.</p> </li> <li> <code>quantization_setup</code>               (<code>Optional[Dict[str, List[float]]]</code>, default:                   <code>None</code> )           \u2013            <p>This parameter is used when the 'piecewise' method is used to embed the continuous cols. It is a dict where keys are the name of the continuous columns and values are lists with the boundaries for the quantization of the continuous_cols. See the examples for details. If If the 'piecewise' method is used, this parameter is required.</p> </li> <li> <code>n_frequencies</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>This is the so called 'k' in their paper On Embeddings for Numerical Features in Tabular Deep Learning, and is the number of 'frequencies' that will be used to represent each continuous column. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>sigma</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>This is the sigma parameter in the paper mentioned when describing the previous parameters and it is used to initialise the 'frequency weights'. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>share_last_layer</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>This parameter is not present in the before mentioned paper but it is implemented in the official repo. If <code>True</code> the linear layer that turns the frequencies into embeddings will be shared across the continuous columns. If <code>False</code> a different linear layer will be used for each continuous column. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>full_embed_dropout</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>If <code>True</code>, the full embedding corresponding to a column will be masked out/dropout. If <code>None</code>, it will default to <code>False</code>.</p> </li> <li> <code>input_dim</code>               (<code>int</code>, default:                   <code>32</code> )           \u2013            <p>The so-called dimension of the model. Is the number of embeddings used to encode the categorical and/or continuous columns.</p> </li> <li> <code>n_cross_attns</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>Number of times each perceiver block will cross attend to the input data (i.e. number of cross attention components per perceiver block). This should normally be 1. However, in the paper they describe some architectures (normally computer vision-related problems) where the Perceiver attends multiple times to the input array. Therefore, maybe multiple cross attention to the input array is also useful in some cases for tabular data  .</p> </li> <li> <code>n_cross_attn_heads</code>               (<code>int</code>, default:                   <code>4</code> )           \u2013            <p>Number of attention heads for the cross attention component</p> </li> <li> <code>n_latents</code>               (<code>int</code>, default:                   <code>16</code> )           \u2013            <p>Number of latents. This is the \\(N\\) parameter in the paper. As indicated in the paper, this number should be significantly lower than \\(M\\) (the number of columns in the dataset). Setting \\(N\\) closer to \\(M\\) defies the main purpose of the Perceiver, which is to overcome the transformer quadratic bottleneck</p> </li> <li> <code>latent_dim</code>               (<code>int</code>, default:                   <code>128</code> )           \u2013            <p>Latent dimension.</p> </li> <li> <code>n_latent_heads</code>               (<code>int</code>, default:                   <code>4</code> )           \u2013            <p>Number of attention heads per Latent Transformer</p> </li> <li> <code>n_latent_blocks</code>               (<code>int</code>, default:                   <code>4</code> )           \u2013            <p>Number of transformer encoder blocks (normalised MHA + normalised FF) per Latent Transformer</p> </li> <li> <code>n_perceiver_blocks</code>               (<code>int</code>, default:                   <code>4</code> )           \u2013            <p>Number of Perceiver blocks defined as [Cross Attention + Latent Transformer]</p> </li> <li> <code>share_weights</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating if the weights will be shared between Perceiver blocks</p> </li> <li> <code>attn_dropout</code>               (<code>float</code>, default:                   <code>0.1</code> )           \u2013            <p>Dropout that will be applied to the Multi-Head Attention layers</p> </li> <li> <code>ff_dropout</code>               (<code>float</code>, default:                   <code>0.1</code> )           \u2013            <p>Dropout that will be applied to the FeedForward network</p> </li> <li> <code>ff_factor</code>               (<code>int</code>, default:                   <code>4</code> )           \u2013            <p>Multiplicative factor applied to the first layer of the FF network in each Transformer block, This is normally set to 4.</p> </li> <li> <code>transformer_activation</code>               (<code>str</code>, default:                   <code>'geglu'</code> )           \u2013            <p>Transformer Encoder activation function. 'tanh', 'relu', 'leaky_relu', 'gelu', 'geglu' and 'reglu' are supported</p> </li> <li> <code>mlp_hidden_dims</code>               (<code>Optional[List[int]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the number of neurons per dense layer in the MLP. e.g: [64, 32]. If not provided no MLP on top of the final Transformer block will be used.</p> </li> <li> <code>mlp_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the dense layers of the MLP. Currently 'tanh', 'relu', 'leaky'_relu' and _'gelu' are supported. If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to 'relu'.</p> </li> <li> <code>mlp_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>float with the dropout between the dense layers of the MLP. If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to 0.0.</p> </li> <li> <code>mlp_batchnorm</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating whether or not batch normalization will be applied to the dense layers If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to False.</p> </li> <li> <code>mlp_batchnorm_last</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating whether or not batch normalization will be applied to the last of the dense layers If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to False.</p> </li> <li> <code>mlp_linear_first</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code> If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to <code>True</code>.</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>encoder</code>               (<code>ModuleDict</code>)           \u2013            <p>ModuleDict with the Perceiver blocks</p> </li> <li> <code>latents</code>               (<code>Parameter</code>)           \u2013            <p>Latents that will be used for prediction</p> </li> <li> <code>mlp</code>               (<code>Module</code>)           \u2013            <p>MLP component in the model</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import TabPerceiver\n&gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n&gt;&gt;&gt; colnames = ['a', 'b', 'c', 'd', 'e']\n&gt;&gt;&gt; cat_embed_input = [(u,i) for u,i in zip(colnames[:4], [4]*4)]\n&gt;&gt;&gt; continuous_cols = ['e']\n&gt;&gt;&gt; column_idx = {k:v for v,k in enumerate(colnames)}\n&gt;&gt;&gt; model = TabPerceiver(column_idx=column_idx, cat_embed_input=cat_embed_input,\n... continuous_cols=continuous_cols, n_latents=2, latent_dim=16,\n... n_perceiver_blocks=2)\n&gt;&gt;&gt; out = model(X_tab)\n</code></pre> Source code in <code>pytorch_widedeep/models/tabular/transformers/tab_perceiver.py</code> <pre><code>def __init__(\n    self,\n    column_idx: Dict[str, int],\n    *,\n    cat_embed_input: Optional[List[Tuple[str, int]]] = None,\n    cat_embed_dropout: Optional[float] = None,\n    use_cat_bias: Optional[bool] = None,\n    cat_embed_activation: Optional[str] = None,\n    shared_embed: Optional[bool] = None,\n    add_shared_embed: Optional[bool] = None,\n    frac_shared_embed: Optional[float] = None,\n    continuous_cols: Optional[List[str]] = None,\n    cont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\n    embed_continuous_method: Optional[\n        Literal[\"standard\", \"piecewise\", \"periodic\"]\n    ] = \"standard\",\n    cont_embed_dropout: Optional[float] = None,\n    cont_embed_activation: Optional[str] = None,\n    quantization_setup: Optional[Dict[str, List[float]]] = None,\n    n_frequencies: Optional[int] = None,\n    sigma: Optional[float] = None,\n    share_last_layer: Optional[bool] = None,\n    full_embed_dropout: Optional[bool] = None,\n    input_dim: int = 32,\n    n_cross_attns: int = 1,\n    n_cross_attn_heads: int = 4,\n    n_latents: int = 16,\n    latent_dim: int = 128,\n    n_latent_heads: int = 4,\n    n_latent_blocks: int = 4,\n    n_perceiver_blocks: int = 4,\n    share_weights: bool = False,\n    attn_dropout: float = 0.1,\n    ff_dropout: float = 0.1,\n    ff_factor: int = 4,\n    transformer_activation: str = \"geglu\",\n    mlp_hidden_dims: Optional[List[int]] = None,\n    mlp_activation: Optional[str] = None,\n    mlp_dropout: Optional[float] = None,\n    mlp_batchnorm: Optional[bool] = None,\n    mlp_batchnorm_last: Optional[bool] = None,\n    mlp_linear_first: Optional[bool] = None,\n):\n    super(TabPerceiver, self).__init__(\n        column_idx=column_idx,\n        cat_embed_input=cat_embed_input,\n        cat_embed_dropout=cat_embed_dropout,\n        use_cat_bias=use_cat_bias,\n        cat_embed_activation=cat_embed_activation,\n        shared_embed=shared_embed,\n        add_shared_embed=add_shared_embed,\n        frac_shared_embed=frac_shared_embed,\n        continuous_cols=continuous_cols,\n        cont_norm_layer=cont_norm_layer,\n        embed_continuous_method=embed_continuous_method,\n        cont_embed_dropout=cont_embed_dropout,\n        cont_embed_activation=cont_embed_activation,\n        input_dim=input_dim,\n        quantization_setup=quantization_setup,\n        n_frequencies=n_frequencies,\n        sigma=sigma,\n        share_last_layer=share_last_layer,\n        full_embed_dropout=full_embed_dropout,\n    )\n\n    self.n_cross_attns = n_cross_attns\n    self.n_cross_attn_heads = n_cross_attn_heads\n    self.n_latents = n_latents\n    self.latent_dim = latent_dim\n    self.n_latent_heads = n_latent_heads\n    self.n_latent_blocks = n_latent_blocks\n    self.n_perceiver_blocks = n_perceiver_blocks\n    self.share_weights = share_weights\n    self.attn_dropout = attn_dropout\n    self.ff_dropout = ff_dropout\n    self.ff_factor = ff_factor\n    self.transformer_activation = transformer_activation\n\n    self.mlp_hidden_dims = mlp_hidden_dims\n    self.mlp_activation = mlp_activation\n    self.mlp_dropout = mlp_dropout\n    self.mlp_batchnorm = mlp_batchnorm\n    self.mlp_batchnorm_last = mlp_batchnorm_last\n    self.mlp_linear_first = mlp_linear_first\n\n    # Embeddings are instantiated at the base model\n    # Transformer blocks\n    self.latents = nn.init.trunc_normal_(\n        nn.Parameter(torch.empty(n_latents, latent_dim))\n    )\n\n    self.encoder = nn.ModuleDict()\n    first_perceiver_block = self._build_perceiver_block()\n    self.encoder[\"perceiver_block0\"] = first_perceiver_block\n\n    if share_weights:\n        for n in range(1, n_perceiver_blocks):\n            self.encoder[\"perceiver_block\" + str(n)] = first_perceiver_block\n    else:\n        for n in range(1, n_perceiver_blocks):\n            self.encoder[\"perceiver_block\" + str(n)] = self._build_perceiver_block()\n\n    self.mlp_first_hidden_dim = self.latent_dim\n\n    # Mlp\n    if self.mlp_hidden_dims is not None:\n        self.mlp = MLP(\n            d_hidden=[self.mlp_first_hidden_dim] + self.mlp_hidden_dims,\n            activation=(\n                \"relu\" if self.mlp_activation is None else self.mlp_activation\n            ),\n            dropout=0.0 if self.mlp_dropout is None else self.mlp_dropout,\n            batchnorm=False if self.mlp_batchnorm is None else self.mlp_batchnorm,\n            batchnorm_last=(\n                False\n                if self.mlp_batchnorm_last is None\n                else self.mlp_batchnorm_last\n            ),\n            linear_first=(\n                False if self.mlp_linear_first is None else self.mlp_linear_first\n            ),\n        )\n    else:\n        self.mlp = None\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.transformers.tab_perceiver.TabPerceiver.output_dim","title":"output_dim  <code>property</code>","text":"<pre><code>output_dim\n</code></pre> <p>The output dimension of the model. This is a required property neccesary to build the <code>WideDeep</code> class</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.transformers.tab_perceiver.TabPerceiver.attention_weights","title":"attention_weights  <code>property</code>","text":"<pre><code>attention_weights\n</code></pre> <p>List with the attention weights. If the weights are not shared between perceiver blocks each element of the list will be a list itself containing the Cross Attention and Latent Transformer attention weights respectively</p> <p>The shape of the attention weights is:</p> <ul> <li> <p>Cross Attention: \\((N, C, L, F)\\)</p> </li> <li> <p>Latent Attention: \\((N, T, L, L)\\)</p> </li> </ul> <p>WHere \\(N\\) is the batch size, \\(C\\) is the number of Cross Attention heads, \\(L\\) is the number of Latents, \\(F\\) is the number of features/columns in the dataset and \\(T\\) is the number of Latent Attention heads</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.transformers.tab_fastformer.TabFastFormer","title":"TabFastFormer","text":"<pre><code>TabFastFormer(\n    column_idx,\n    *,\n    cat_embed_input=None,\n    cat_embed_dropout=None,\n    use_cat_bias=None,\n    cat_embed_activation=None,\n    shared_embed=None,\n    add_shared_embed=None,\n    frac_shared_embed=None,\n    continuous_cols=None,\n    cont_norm_layer=None,\n    embed_continuous_method=\"standard\",\n    cont_embed_dropout=None,\n    cont_embed_activation=None,\n    quantization_setup=None,\n    n_frequencies=None,\n    sigma=None,\n    share_last_layer=None,\n    full_embed_dropout=None,\n    input_dim=32,\n    n_heads=8,\n    use_bias=False,\n    n_blocks=4,\n    attn_dropout=0.1,\n    ff_dropout=0.2,\n    ff_factor=4,\n    share_qv_weights=False,\n    share_weights=False,\n    transformer_activation=\"relu\",\n    mlp_hidden_dims=None,\n    mlp_activation=None,\n    mlp_dropout=None,\n    mlp_batchnorm=None,\n    mlp_batchnorm_last=None,\n    mlp_linear_first=None\n)\n</code></pre> <p>               Bases: <code>BaseTabularModelWithAttention</code></p> <p>Defines an adaptation of a FastFormer that can be used as the <code>deeptabular</code> component of a Wide &amp; Deep model or independently by itself.</p> <p>Most of the parameters for this class are <code>Optional</code> since the use of categorical or continuous is in fact optional (i.e. one can use categorical features only, continuous features only or both).</p> <p> NOTE: while there are scientific publications for  the <code>TabTransformer</code>, <code>SAINT</code> and <code>FTTransformer</code>, the <code>TabPerceiver</code>  and the <code>TabFastFormer</code> are our own adaptations of the  Perceiver and the  FastFormer for tabular data.</p> <p>Parameters:</p> <ul> <li> <code>column_idx</code>               (<code>Dict[str, int]</code>)           \u2013            <p>Dict containing the index of the columns that will be passed through the <code>TabMlp</code> model. Required to slice the tensors. e.g. {'education': 0, 'relationship': 1, 'workclass': 2, ...}.</p> </li> <li> <code>cat_embed_input</code>               (<code>Optional[List[Tuple[str, int]]]</code>, default:                   <code>None</code> )           \u2013            <p>List of Tuples with the column name and number of unique values and embedding dimension. e.g. [(education, 11), ...]</p> </li> <li> <code>cat_embed_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Categorical embeddings dropout. If <code>None</code>, it will default to 0.</p> </li> <li> <code>use_cat_bias</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating if bias will be used for the categorical embeddings. If <code>None</code>, it will default to 'False'.</p> </li> <li> <code>cat_embed_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the categorical embeddings, if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> </li> <li> <code>shared_embed</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating if the embeddings will be \"shared\". The idea behind <code>shared_embed</code> is described in the Appendix A in the TabTransformer paper: 'The goal of having column embedding is to enable the model to distinguish the classes in one column from those in the other columns'. In other words, the idea is to let the model learn which column is embedded at the time. See: <code>pytorch_widedeep.models.transformers._layers.SharedEmbeddings</code>.</p> </li> <li> <code>add_shared_embed</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>The two embedding sharing strategies are: 1) add the shared embeddings to the column embeddings or 2) to replace the first <code>frac_shared_embed</code> with the shared embeddings. See <code>pytorch_widedeep.models.embeddings_layers.SharedEmbeddings</code> If 'None' is passed, it will default to 'False'.</p> </li> <li> <code>frac_shared_embed</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>The fraction of embeddings that will be shared (if <code>add_shared_embed = False</code>) by all the different categories for one particular column. If 'None' is passed, it will default to 0.0.</p> </li> <li> <code>continuous_cols</code>               (<code>Optional[List[str]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the name of the numeric (aka continuous) columns</p> </li> <li> <code>cont_norm_layer</code>               (<code>Optional[Literal[batchnorm, layernorm]]</code>, default:                   <code>None</code> )           \u2013            <p>Type of normalization layer applied to the continuous features. Options are: 'layernorm' and 'batchnorm'. if <code>None</code>, no normalization layer will be used.</p> </li> <li> <code>embed_continuous_method</code>               (<code>Optional[Literal[standard, piecewise, periodic]]</code>, default:                   <code>'standard'</code> )           \u2013            <p>Method to use to embed the continuous features. Options are: 'standard', 'periodic' or 'piecewise'. The 'standard' embedding method is based on the FT-Transformer implementation presented in the paper: Revisiting Deep Learning Models for Tabular Data. The 'periodic' and_'piecewise'_ methods were presented in the paper: On Embeddings for Numerical Features in Tabular Deep Learning. Please, read the papers for details.</p> </li> <li> <code>cont_embed_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p> </li> <li> <code>cont_embed_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the continuous embeddings if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported. If <code>None</code>, no activation function will be applied.</p> </li> <li> <code>quantization_setup</code>               (<code>Optional[Dict[str, List[float]]]</code>, default:                   <code>None</code> )           \u2013            <p>This parameter is used when the 'piecewise' method is used to embed the continuous cols. It is a dict where keys are the name of the continuous columns and values are lists with the boundaries for the quantization of the continuous_cols. See the examples for details. If If the 'piecewise' method is used, this parameter is required.</p> </li> <li> <code>n_frequencies</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>This is the so called 'k' in their paper On Embeddings for Numerical Features in Tabular Deep Learning, and is the number of 'frequencies' that will be used to represent each continuous column. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>sigma</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>This is the sigma parameter in the paper mentioned when describing the previous parameters and it is used to initialise the 'frequency weights'. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>share_last_layer</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>This parameter is not present in the before mentioned paper but it is implemented in the official repo. If <code>True</code> the linear layer that turns the frequencies into embeddings will be shared across the continuous columns. If <code>False</code> a different linear layer will be used for each continuous column. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>full_embed_dropout</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>If <code>True</code>, the full embedding corresponding to a column will be masked out/dropout. If <code>None</code>, it will default to <code>False</code>.</p> </li> <li> <code>input_dim</code>               (<code>int</code>, default:                   <code>32</code> )           \u2013            <p>The so-called dimension of the model. Is the number of embeddings used to encode the categorical and/or continuous columns</p> </li> <li> <code>n_heads</code>               (<code>int</code>, default:                   <code>8</code> )           \u2013            <p>Number of attention heads per FastFormer block</p> </li> <li> <code>use_bias</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether or not to use bias in the Q, K, and V projection layers</p> </li> <li> <code>n_blocks</code>               (<code>int</code>, default:                   <code>4</code> )           \u2013            <p>Number of FastFormer blocks</p> </li> <li> <code>attn_dropout</code>               (<code>float</code>, default:                   <code>0.1</code> )           \u2013            <p>Dropout that will be applied to the Additive Attention layers</p> </li> <li> <code>ff_dropout</code>               (<code>float</code>, default:                   <code>0.2</code> )           \u2013            <p>Dropout that will be applied to the FeedForward network</p> </li> <li> <code>ff_factor</code>               (<code>int</code>, default:                   <code>4</code> )           \u2013            <p>Multiplicative factor applied to the first layer of the FF network in each Transformer block, This is normally set to 4.</p> </li> <li> <code>share_qv_weights</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Following the paper, this is a boolean indicating if the Value (\\(V\\)) and the Query (\\(Q\\)) transformation parameters will be shared.</p> </li> <li> <code>share_weights</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>In addition to sharing the \\(V\\) and \\(Q\\) transformation parameters, the parameters across different Fastformer layers can also be shared. Please, see <code>pytorch_widedeep/models/tabular/transformers/tab_fastformer.py</code> for details</p> </li> <li> <code>transformer_activation</code>               (<code>str</code>, default:                   <code>'relu'</code> )           \u2013            <p>Transformer Encoder activation function. 'tanh', 'relu', 'leaky_relu', 'gelu', 'geglu' and 'reglu' are supported</p> </li> <li> <code>mlp_hidden_dims</code>               (<code>Optional[List[int]]</code>, default:                   <code>None</code> )           \u2013            <p>MLP hidden dimensions. If not provided no MLP on top of the final FTTransformer block will be used</p> </li> <li> <code>mlp_hidden_dims</code>               (<code>Optional[List[int]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the number of neurons per dense layer in the MLP. e.g: [64, 32]. If not provided no MLP on top of the final Transformer block will be used.</p> </li> <li> <code>mlp_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the dense layers of the MLP. Currently 'tanh', 'relu', 'leaky'_relu' and _'gelu' are supported. If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to 'relu'.</p> </li> <li> <code>mlp_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>float with the dropout between the dense layers of the MLP. If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to 0.0.</p> </li> <li> <code>mlp_batchnorm</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating whether or not batch normalization will be applied to the dense layers If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to False.</p> </li> <li> <code>mlp_batchnorm_last</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating whether or not batch normalization will be applied to the last of the dense layers If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to False.</p> </li> <li> <code>mlp_linear_first</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code> If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to <code>True</code>.</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>encoder</code>               (<code>Module</code>)           \u2013            <p>Sequence of FasFormer blocks.</p> </li> <li> <code>mlp</code>               (<code>Module</code>)           \u2013            <p>MLP component in the model</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import TabFastFormer\n&gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n&gt;&gt;&gt; colnames = ['a', 'b', 'c', 'd', 'e']\n&gt;&gt;&gt; cat_embed_input = [(u,i) for u,i in zip(colnames[:4], [4]*4)]\n&gt;&gt;&gt; continuous_cols = ['e']\n&gt;&gt;&gt; column_idx = {k:v for v,k in enumerate(colnames)}\n&gt;&gt;&gt; model = TabFastFormer(column_idx=column_idx, cat_embed_input=cat_embed_input, continuous_cols=continuous_cols)\n&gt;&gt;&gt; out = model(X_tab)\n</code></pre> Source code in <code>pytorch_widedeep/models/tabular/transformers/tab_fastformer.py</code> <pre><code>def __init__(\n    self,\n    column_idx: Dict[str, int],\n    *,\n    cat_embed_input: Optional[List[Tuple[str, int]]] = None,\n    cat_embed_dropout: Optional[float] = None,\n    use_cat_bias: Optional[bool] = None,\n    cat_embed_activation: Optional[str] = None,\n    shared_embed: Optional[bool] = None,\n    add_shared_embed: Optional[bool] = None,\n    frac_shared_embed: Optional[float] = None,\n    continuous_cols: Optional[List[str]] = None,\n    cont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\n    embed_continuous_method: Optional[\n        Literal[\"standard\", \"piecewise\", \"periodic\"]\n    ] = \"standard\",\n    cont_embed_dropout: Optional[float] = None,\n    cont_embed_activation: Optional[str] = None,\n    quantization_setup: Optional[Dict[str, List[float]]] = None,\n    n_frequencies: Optional[int] = None,\n    sigma: Optional[float] = None,\n    share_last_layer: Optional[bool] = None,\n    full_embed_dropout: Optional[bool] = None,\n    input_dim: int = 32,\n    n_heads: int = 8,\n    use_bias: bool = False,\n    n_blocks: int = 4,\n    attn_dropout: float = 0.1,\n    ff_dropout: float = 0.2,\n    ff_factor: int = 4,\n    share_qv_weights: bool = False,\n    share_weights: bool = False,\n    transformer_activation: str = \"relu\",\n    mlp_hidden_dims: Optional[List[int]] = None,\n    mlp_activation: Optional[str] = None,\n    mlp_dropout: Optional[float] = None,\n    mlp_batchnorm: Optional[bool] = None,\n    mlp_batchnorm_last: Optional[bool] = None,\n    mlp_linear_first: Optional[bool] = None,\n):\n    super(TabFastFormer, self).__init__(\n        column_idx=column_idx,\n        cat_embed_input=cat_embed_input,\n        cat_embed_dropout=cat_embed_dropout,\n        use_cat_bias=use_cat_bias,\n        cat_embed_activation=cat_embed_activation,\n        shared_embed=shared_embed,\n        add_shared_embed=add_shared_embed,\n        frac_shared_embed=frac_shared_embed,\n        continuous_cols=continuous_cols,\n        cont_norm_layer=cont_norm_layer,\n        embed_continuous_method=embed_continuous_method,\n        cont_embed_dropout=cont_embed_dropout,\n        cont_embed_activation=cont_embed_activation,\n        input_dim=input_dim,\n        quantization_setup=quantization_setup,\n        n_frequencies=n_frequencies,\n        sigma=sigma,\n        share_last_layer=share_last_layer,\n        full_embed_dropout=full_embed_dropout,\n    )\n\n    self.n_heads = n_heads\n    self.use_bias = use_bias\n    self.n_blocks = n_blocks\n    self.attn_dropout = attn_dropout\n    self.ff_dropout = ff_dropout\n    self.ff_factor = ff_factor\n    self.share_qv_weights = share_qv_weights\n    self.share_weights = share_weights\n    self.transformer_activation = transformer_activation\n\n    self.mlp_hidden_dims = mlp_hidden_dims\n    self.mlp_activation = mlp_activation\n    self.mlp_dropout = mlp_dropout\n    self.mlp_batchnorm = mlp_batchnorm\n    self.mlp_batchnorm_last = mlp_batchnorm_last\n    self.mlp_linear_first = mlp_linear_first\n\n    self.with_cls_token = \"cls_token\" in column_idx\n    self.n_cat = len(cat_embed_input) if cat_embed_input is not None else 0\n    self.n_cont = len(continuous_cols) if continuous_cols is not None else 0\n    self.n_feats = self.n_cat + self.n_cont\n\n    # Embeddings are instantiated at the base model\n    # Transformer blocks\n    self.encoder = nn.Sequential()\n    first_fastformer_block = FastFormerEncoder(\n        input_dim,\n        n_heads,\n        use_bias,\n        attn_dropout,\n        ff_dropout,\n        ff_factor,\n        share_qv_weights,\n        transformer_activation,\n    )\n    self.encoder.add_module(\"fastformer_block0\", first_fastformer_block)\n    for i in range(1, n_blocks):\n        if share_weights:\n            self.encoder.add_module(\n                \"fastformer_block\" + str(i), first_fastformer_block\n            )\n        else:\n            self.encoder.add_module(\n                \"fastformer_block\" + str(i),\n                FastFormerEncoder(\n                    input_dim,\n                    n_heads,\n                    use_bias,\n                    attn_dropout,\n                    ff_dropout,\n                    ff_factor,\n                    share_qv_weights,\n                    transformer_activation,\n                ),\n            )\n\n    self.mlp_first_hidden_dim = (\n        self.input_dim if self.with_cls_token else (self.n_feats * self.input_dim)\n    )\n\n    # Mlp: adding an MLP on top of the Resnet blocks is optional and\n    # therefore all related params are optional\n    if self.mlp_hidden_dims is not None:\n        self.mlp = MLP(\n            d_hidden=[self.mlp_first_hidden_dim] + self.mlp_hidden_dims,\n            activation=(\n                \"relu\" if self.mlp_activation is None else self.mlp_activation\n            ),\n            dropout=0.0 if self.mlp_dropout is None else self.mlp_dropout,\n            batchnorm=False if self.mlp_batchnorm is None else self.mlp_batchnorm,\n            batchnorm_last=(\n                False\n                if self.mlp_batchnorm_last is None\n                else self.mlp_batchnorm_last\n            ),\n            linear_first=(\n                False if self.mlp_linear_first is None else self.mlp_linear_first\n            ),\n        )\n    else:\n        self.mlp = None\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.transformers.tab_fastformer.TabFastFormer.output_dim","title":"output_dim  <code>property</code>","text":"<pre><code>output_dim\n</code></pre> <p>The output dimension of the model. This is a required property neccesary to build the <code>WideDeep</code> class</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.transformers.tab_fastformer.TabFastFormer.attention_weights","title":"attention_weights  <code>property</code>","text":"<pre><code>attention_weights\n</code></pre> <p>List with the attention weights. Each element of the list is a tuple where the first and second elements are the \\(\\alpha\\) and \\(\\beta\\) attention weights in the paper.</p> <p>The shape of the attention weights is \\((N, H, F)\\) where \\(N\\) is the batch size, \\(H\\) is the number of attention heads and \\(F\\) is the number of features/columns in the dataset</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.text.rnns.basic_rnn.BasicRNN","title":"BasicRNN","text":"<pre><code>BasicRNN(\n    vocab_size,\n    embed_dim=None,\n    embed_matrix=None,\n    embed_trainable=True,\n    rnn_type=\"lstm\",\n    hidden_dim=64,\n    n_layers=3,\n    rnn_dropout=0.0,\n    bidirectional=False,\n    use_hidden_state=True,\n    padding_idx=1,\n    head_hidden_dims=None,\n    head_activation=\"relu\",\n    head_dropout=None,\n    head_batchnorm=False,\n    head_batchnorm_last=False,\n    head_linear_first=False,\n)\n</code></pre> <p>               Bases: <code>BaseWDModelComponent</code></p> <p>Standard text classifier/regressor comprised by a stack of RNNs (LSTMs or GRUs) that can be used as the <code>deeptext</code> component of a Wide &amp; Deep model or independently by itself.</p> <p>In addition, there is the option to add a Fully Connected (FC) set of dense layers on top of the stack of RNNs</p> <p>Parameters:</p> <ul> <li> <code>vocab_size</code>               (<code>int</code>)           \u2013            <p>Number of words in the vocabulary</p> </li> <li> <code>embed_dim</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>Dimension of the word embeddings if non-pretained word vectors are used</p> </li> <li> <code>embed_matrix</code>               (<code>Optional[ndarray]</code>, default:                   <code>None</code> )           \u2013            <p>Pretrained word embeddings</p> </li> <li> <code>embed_trainable</code>               (<code>bool</code>, default:                   <code>True</code> )           \u2013            <p>Boolean indicating if the pretrained embeddings are trainable</p> </li> <li> <code>rnn_type</code>               (<code>Literal[lstm, gru]</code>, default:                   <code>'lstm'</code> )           \u2013            <p>String indicating the type of RNN to use. One of 'lstm' or 'gru'</p> </li> <li> <code>hidden_dim</code>               (<code>int</code>, default:                   <code>64</code> )           \u2013            <p>Hidden dim of the RNN</p> </li> <li> <code>n_layers</code>               (<code>int</code>, default:                   <code>3</code> )           \u2013            <p>Number of recurrent layers</p> </li> <li> <code>rnn_dropout</code>               (<code>float</code>, default:                   <code>0.0</code> )           \u2013            <p>Dropout for each RNN layer except the last layer</p> </li> <li> <code>bidirectional</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether the staked RNNs are bidirectional</p> </li> <li> <code>use_hidden_state</code>               (<code>bool</code>, default:                   <code>True</code> )           \u2013            <p>Boolean indicating whether to use the final hidden state or the RNN's output as predicting features. Typically the former is used.</p> </li> <li> <code>padding_idx</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>index of the padding token in the padded-tokenised sequences. The <code>TextPreprocessor</code> class within this library uses fastai's tokenizer where the token index 0 is reserved for the 'unknown' word token. Therefore, the default value is set to 1.</p> </li> <li> <code>head_hidden_dims</code>               (<code>Optional[List[int]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the sizes of the dense layers in the head e.g: [128, 64]</p> </li> <li> <code>head_activation</code>               (<code>str</code>, default:                   <code>'relu'</code> )           \u2013            <p>Activation function for the dense layers in the head. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> </li> <li> <code>head_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Dropout of the dense layers in the head</p> </li> <li> <code>head_batchnorm</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether or not to include batch normalization in the dense layers that form the 'rnn_mlp'</p> </li> <li> <code>head_batchnorm_last</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether or not to apply batch normalization to the last of the dense layers in the head</p> </li> <li> <code>head_linear_first</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code></p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>word_embed</code>               (<code>Module</code>)           \u2013            <p>word embedding matrix</p> </li> <li> <code>rnn</code>               (<code>Module</code>)           \u2013            <p>Stack of RNNs</p> </li> <li> <code>rnn_mlp</code>               (<code>Module</code>)           \u2013            <p>Stack of dense layers on top of the RNN. This will only exists if <code>head_layers_dim</code> is not None</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import BasicRNN\n&gt;&gt;&gt; X_text = torch.cat((torch.zeros([5,1]), torch.empty(5, 4).random_(1,4)), axis=1)\n&gt;&gt;&gt; model = BasicRNN(vocab_size=4, hidden_dim=4, n_layers=2, padding_idx=0, embed_dim=4)\n&gt;&gt;&gt; out = model(X_text)\n</code></pre> Source code in <code>pytorch_widedeep/models/text/rnns/basic_rnn.py</code> <pre><code>def __init__(\n    self,\n    vocab_size: int,\n    embed_dim: Optional[int] = None,\n    embed_matrix: Optional[np.ndarray] = None,\n    embed_trainable: bool = True,\n    rnn_type: Literal[\"lstm\", \"gru\"] = \"lstm\",\n    hidden_dim: int = 64,\n    n_layers: int = 3,\n    rnn_dropout: float = 0.0,\n    bidirectional: bool = False,\n    use_hidden_state: bool = True,\n    padding_idx: int = 1,\n    head_hidden_dims: Optional[List[int]] = None,\n    head_activation: str = \"relu\",\n    head_dropout: Optional[float] = None,\n    head_batchnorm: bool = False,\n    head_batchnorm_last: bool = False,\n    head_linear_first: bool = False,\n):\n    super(BasicRNN, self).__init__()\n\n    if embed_dim is None and embed_matrix is None:\n        raise ValueError(\n            \"If no 'embed_matrix' is passed, the embedding dimension must\"\n            \"be specified with 'embed_dim'\"\n        )\n\n    if rnn_type.lower() not in [\"lstm\", \"gru\"]:\n        raise ValueError(\n            f\"'rnn_type' must be 'lstm' or 'gru', got {rnn_type} instead\"\n        )\n\n    if (\n        embed_dim is not None\n        and embed_matrix is not None\n        and not embed_dim == embed_matrix.shape[1]\n    ):\n        warnings.warn(\n            \"the input embedding dimension {} and the dimension of the \"\n            \"pretrained embeddings {} do not match. The pretrained embeddings \"\n            \"dimension ({}) will be used\".format(\n                embed_dim, embed_matrix.shape[1], embed_matrix.shape[1]\n            ),\n            UserWarning,\n        )\n\n    self.vocab_size = vocab_size\n    self.embed_trainable = embed_trainable\n    self.embed_dim = embed_dim\n\n    self.rnn_type = rnn_type\n    self.hidden_dim = hidden_dim\n    self.n_layers = n_layers\n    self.rnn_dropout = rnn_dropout\n    self.bidirectional = bidirectional\n    self.use_hidden_state = use_hidden_state\n    self.padding_idx = padding_idx\n\n    self.head_hidden_dims = head_hidden_dims\n    self.head_activation = head_activation\n    self.head_dropout = head_dropout\n    self.head_batchnorm = head_batchnorm\n    self.head_batchnorm_last = head_batchnorm_last\n    self.head_linear_first = head_linear_first\n\n    # Embeddings\n    if embed_matrix is not None:\n        self.word_embed, self.embed_dim = self._set_embeddings(embed_matrix)\n    else:\n        assert self.embed_dim is not None\n        self.word_embed = nn.Embedding(\n            self.vocab_size, self.embed_dim, padding_idx=self.padding_idx\n        )\n\n    # RNN\n    rnn_params = {\n        \"input_size\": self.embed_dim,\n        \"hidden_size\": hidden_dim,\n        \"num_layers\": n_layers,\n        \"bidirectional\": bidirectional,\n        \"dropout\": rnn_dropout,\n        \"batch_first\": True,\n    }\n    if self.rnn_type.lower() == \"lstm\":\n        self.rnn: Union[nn.LSTM, nn.GRU] = nn.LSTM(**rnn_params)\n    elif self.rnn_type.lower() == \"gru\":\n        self.rnn = nn.GRU(**rnn_params)\n    else:\n        raise ValueError(\n            f\"'rnn_type' must be 'lstm' or 'gru', got {self.rnn_type} instead\"\n        )\n\n    self.rnn_output_dim = hidden_dim * 2 if bidirectional else hidden_dim\n\n    # FC-Head (Mlp)\n    if self.head_hidden_dims is not None:\n        head_hidden_dims = [self.rnn_output_dim] + self.head_hidden_dims\n        self.rnn_mlp: Union[MLP, nn.Identity] = MLP(\n            head_hidden_dims,\n            head_activation,\n            head_dropout,\n            head_batchnorm,\n            head_batchnorm_last,\n            head_linear_first,\n        )\n    else:\n        # simple hack to add readability in the forward pass\n        self.rnn_mlp = nn.Identity()\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.text.rnns.basic_rnn.BasicRNN.output_dim","title":"output_dim  <code>property</code>","text":"<pre><code>output_dim\n</code></pre> <p>The output dimension of the model. This is a required property neccesary to build the <code>WideDeep</code> class</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.text.rnns.attentive_rnn.AttentiveRNN","title":"AttentiveRNN","text":"<pre><code>AttentiveRNN(\n    vocab_size,\n    embed_dim=None,\n    embed_matrix=None,\n    embed_trainable=True,\n    rnn_type=\"lstm\",\n    hidden_dim=64,\n    n_layers=3,\n    rnn_dropout=0.1,\n    bidirectional=False,\n    use_hidden_state=True,\n    padding_idx=1,\n    attn_concatenate=True,\n    attn_dropout=0.1,\n    head_hidden_dims=None,\n    head_activation=\"relu\",\n    head_dropout=None,\n    head_batchnorm=False,\n    head_batchnorm_last=False,\n    head_linear_first=False,\n)\n</code></pre> <p>               Bases: <code>BasicRNN</code></p> <p>Text classifier/regressor comprised by a stack of RNNs (LSTMs or GRUs) plus an attention layer. This model can be used as the <code>deeptext</code> component of a Wide &amp; Deep model or independently by itself.</p> <p>In addition, there is the option to add a Fully Connected (FC) set of dense layers on top of attention layer</p> <p>Parameters:</p> <ul> <li> <code>vocab_size</code>               (<code>int</code>)           \u2013            <p>Number of words in the vocabulary</p> </li> <li> <code>embed_dim</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>Dimension of the word embeddings if non-pretained word vectors are used</p> </li> <li> <code>embed_matrix</code>               (<code>Optional[ndarray]</code>, default:                   <code>None</code> )           \u2013            <p>Pretrained word embeddings</p> </li> <li> <code>embed_trainable</code>               (<code>bool</code>, default:                   <code>True</code> )           \u2013            <p>Boolean indicating if the pretrained embeddings are trainable</p> </li> <li> <code>rnn_type</code>               (<code>Literal[lstm, gru]</code>, default:                   <code>'lstm'</code> )           \u2013            <p>String indicating the type of RNN to use. One of 'lstm' or 'gru'</p> </li> <li> <code>hidden_dim</code>               (<code>int</code>, default:                   <code>64</code> )           \u2013            <p>Hidden dim of the RNN</p> </li> <li> <code>n_layers</code>               (<code>int</code>, default:                   <code>3</code> )           \u2013            <p>Number of recurrent layers</p> </li> <li> <code>rnn_dropout</code>               (<code>float</code>, default:                   <code>0.1</code> )           \u2013            <p>Dropout for each RNN layer except the last layer</p> </li> <li> <code>bidirectional</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether the staked RNNs are bidirectional</p> </li> <li> <code>use_hidden_state</code>               (<code>bool</code>, default:                   <code>True</code> )           \u2013            <p>Boolean indicating whether to use the final hidden state or the RNN's output as predicting features. Typically the former is used.</p> </li> <li> <code>padding_idx</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>index of the padding token in the padded-tokenised sequences. The <code>TextPreprocessor</code> class within this library uses fastai's tokenizer where the token index 0 is reserved for the 'unknown' word token. Therefore, the default value is set to 1.</p> </li> <li> <code>attn_concatenate</code>               (<code>bool</code>, default:                   <code>True</code> )           \u2013            <p>Boolean indicating if the input to the attention mechanism will be the output of the RNN or the output of the RNN concatenated with the last hidden state.</p> </li> <li> <code>attn_dropout</code>               (<code>float</code>, default:                   <code>0.1</code> )           \u2013            <p>Internal dropout for the attention mechanism</p> </li> <li> <code>head_hidden_dims</code>               (<code>Optional[List[int]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the sizes of the dense layers in the head e.g: [128, 64]</p> </li> <li> <code>head_activation</code>               (<code>str</code>, default:                   <code>'relu'</code> )           \u2013            <p>Activation function for the dense layers in the head. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> </li> <li> <code>head_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Dropout of the dense layers in the head</p> </li> <li> <code>head_batchnorm</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether or not to include batch normalization in the dense layers that form the 'rnn_mlp'</p> </li> <li> <code>head_batchnorm_last</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether or not to apply batch normalization to the last of the dense layers in the head</p> </li> <li> <code>head_linear_first</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code></p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>word_embed</code>               (<code>Module</code>)           \u2013            <p>word embedding matrix</p> </li> <li> <code>rnn</code>               (<code>Module</code>)           \u2013            <p>Stack of RNNs</p> </li> <li> <code>rnn_mlp</code>               (<code>Module</code>)           \u2013            <p>Stack of dense layers on top of the RNN. This will only exists if <code>head_layers_dim</code> is not <code>None</code></p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import AttentiveRNN\n&gt;&gt;&gt; X_text = torch.cat((torch.zeros([5,1]), torch.empty(5, 4).random_(1,4)), axis=1)\n&gt;&gt;&gt; model = AttentiveRNN(vocab_size=4, hidden_dim=4, n_layers=2, padding_idx=0, embed_dim=4)\n&gt;&gt;&gt; out = model(X_text)\n</code></pre> Source code in <code>pytorch_widedeep/models/text/rnns/attentive_rnn.py</code> <pre><code>def __init__(\n    self,\n    vocab_size: int,\n    embed_dim: Optional[int] = None,\n    embed_matrix: Optional[np.ndarray] = None,\n    embed_trainable: bool = True,\n    rnn_type: Literal[\"lstm\", \"gru\"] = \"lstm\",\n    hidden_dim: int = 64,\n    n_layers: int = 3,\n    rnn_dropout: float = 0.1,\n    bidirectional: bool = False,\n    use_hidden_state: bool = True,\n    padding_idx: int = 1,\n    attn_concatenate: bool = True,\n    attn_dropout: float = 0.1,\n    head_hidden_dims: Optional[List[int]] = None,\n    head_activation: str = \"relu\",\n    head_dropout: Optional[float] = None,\n    head_batchnorm: bool = False,\n    head_batchnorm_last: bool = False,\n    head_linear_first: bool = False,\n):\n    super(AttentiveRNN, self).__init__(\n        vocab_size=vocab_size,\n        embed_dim=embed_dim,\n        embed_matrix=embed_matrix,\n        embed_trainable=embed_trainable,\n        rnn_type=rnn_type,\n        hidden_dim=hidden_dim,\n        n_layers=n_layers,\n        rnn_dropout=rnn_dropout,\n        bidirectional=bidirectional,\n        use_hidden_state=use_hidden_state,\n        padding_idx=padding_idx,\n        head_hidden_dims=head_hidden_dims,\n        head_activation=head_activation,\n        head_dropout=head_dropout,\n        head_batchnorm=head_batchnorm,\n        head_batchnorm_last=head_batchnorm_last,\n        head_linear_first=head_linear_first,\n    )\n\n    # Embeddings and RNN defined in the BasicRNN inherited class\n\n    # Attention\n    self.attn_concatenate = attn_concatenate\n    self.attn_dropout = attn_dropout\n\n    if bidirectional and attn_concatenate:\n        self.rnn_output_dim = hidden_dim * 4\n    elif bidirectional or attn_concatenate:\n        self.rnn_output_dim = hidden_dim * 2\n    else:\n        self.rnn_output_dim = hidden_dim\n    self.attn = ContextAttention(\n        self.rnn_output_dim, attn_dropout, sum_along_seq=True\n    )\n\n    # FC-Head (Mlp)\n    if self.head_hidden_dims is not None:\n        head_hidden_dims = [self.rnn_output_dim] + self.head_hidden_dims\n        self.rnn_mlp = MLP(\n            head_hidden_dims,\n            head_activation,\n            head_dropout,\n            head_batchnorm,\n            head_batchnorm_last,\n            head_linear_first,\n        )\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.text.rnns.attentive_rnn.AttentiveRNN.attention_weights","title":"attention_weights  <code>property</code>","text":"<pre><code>attention_weights\n</code></pre> <p>List with the attention weights</p> <p>The shape of the attention weights is \\((N, S)\\), where \\(N\\) is the batch size and \\(S\\) is the length of the sequence</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.text.rnns.stacked_attentive_rnn.StackedAttentiveRNN","title":"StackedAttentiveRNN","text":"<pre><code>StackedAttentiveRNN(\n    vocab_size,\n    embed_dim=None,\n    embed_matrix=None,\n    embed_trainable=True,\n    rnn_type=\"lstm\",\n    hidden_dim=64,\n    bidirectional=False,\n    padding_idx=1,\n    n_blocks=3,\n    attn_concatenate=False,\n    attn_dropout=0.1,\n    with_addnorm=False,\n    head_hidden_dims=None,\n    head_activation=\"relu\",\n    head_dropout=None,\n    head_batchnorm=False,\n    head_batchnorm_last=False,\n    head_linear_first=False,\n)\n</code></pre> <p>               Bases: <code>BaseWDModelComponent</code></p> <p>Text classifier/regressor comprised by a stack of blocks: <code>[RNN + Attention]</code>. This can be used as the <code>deeptext</code> component of a Wide &amp; Deep model or independently by itself.</p> <p>In addition, there is the option to add a Fully Connected (FC) set of dense layers on top of the attentiob blocks</p> <p>Parameters:</p> <ul> <li> <code>vocab_size</code>               (<code>int</code>)           \u2013            <p>Number of words in the vocabulary</p> </li> <li> <code>embed_dim</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>Dimension of the word embeddings if non-pretained word vectors are used</p> </li> <li> <code>embed_matrix</code>               (<code>Optional[ndarray]</code>, default:                   <code>None</code> )           \u2013            <p>Pretrained word embeddings</p> </li> <li> <code>embed_trainable</code>               (<code>bool</code>, default:                   <code>True</code> )           \u2013            <p>Boolean indicating if the pretrained embeddings are trainable</p> </li> <li> <code>rnn_type</code>               (<code>Literal[lstm, gru]</code>, default:                   <code>'lstm'</code> )           \u2013            <p>String indicating the type of RNN to use. One of 'lstm' or 'gru'</p> </li> <li> <code>hidden_dim</code>               (<code>int</code>, default:                   <code>64</code> )           \u2013            <p>Hidden dim of the RNN</p> </li> <li> <code>bidirectional</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether the staked RNNs are bidirectional</p> </li> <li> <code>padding_idx</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>index of the padding token in the padded-tokenised sequences. The <code>TextPreprocessor</code> class within this library uses fastai's tokenizer where the token index 0 is reserved for the 'unknown' word token. Therefore, the default value is set to 1.</p> </li> <li> <code>n_blocks</code>               (<code>int</code>, default:                   <code>3</code> )           \u2013            <p>Number of attention blocks. Each block is comprised by an RNN and a Context Attention Encoder</p> </li> <li> <code>attn_concatenate</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating if the input to the attention mechanism will be the output of the RNN or the output of the RNN concatenated with the last hidden state or simply</p> </li> <li> <code>attn_dropout</code>               (<code>float</code>, default:                   <code>0.1</code> )           \u2013            <p>Internal dropout for the attention mechanism</p> </li> <li> <code>with_addnorm</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating if the output of each block will be added to the input and normalised</p> </li> <li> <code>head_hidden_dims</code>               (<code>Optional[List[int]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the sizes of the dense layers in the head e.g: [128, 64]</p> </li> <li> <code>head_activation</code>               (<code>str</code>, default:                   <code>'relu'</code> )           \u2013            <p>Activation function for the dense layers in the head. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> </li> <li> <code>head_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Dropout of the dense layers in the head</p> </li> <li> <code>head_batchnorm</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether or not to include batch normalization in the dense layers that form the 'rnn_mlp'</p> </li> <li> <code>head_batchnorm_last</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether or not to apply batch normalization to the last of the dense layers in the head</p> </li> <li> <code>head_linear_first</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code></p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>word_embed</code>               (<code>Module</code>)           \u2013            <p>word embedding matrix</p> </li> <li> <code>rnn</code>               (<code>Module</code>)           \u2013            <p>Stack of RNNs</p> </li> <li> <code>rnn_mlp</code>               (<code>Module</code>)           \u2013            <p>Stack of dense layers on top of the RNN. This will only exists if <code>head_layers_dim</code> is not <code>None</code></p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import StackedAttentiveRNN\n&gt;&gt;&gt; X_text = torch.cat((torch.zeros([5,1]), torch.empty(5, 4).random_(1,4)), axis=1)\n&gt;&gt;&gt; model = StackedAttentiveRNN(vocab_size=4, hidden_dim=4, padding_idx=0, embed_dim=4)\n&gt;&gt;&gt; out = model(X_text)\n</code></pre> Source code in <code>pytorch_widedeep/models/text/rnns/stacked_attentive_rnn.py</code> <pre><code>def __init__(\n    self,\n    vocab_size: int,\n    embed_dim: Optional[int] = None,\n    embed_matrix: Optional[np.ndarray] = None,\n    embed_trainable: bool = True,\n    rnn_type: Literal[\"lstm\", \"gru\"] = \"lstm\",\n    hidden_dim: int = 64,\n    bidirectional: bool = False,\n    padding_idx: int = 1,\n    n_blocks: int = 3,\n    attn_concatenate: bool = False,\n    attn_dropout: float = 0.1,\n    with_addnorm: bool = False,\n    head_hidden_dims: Optional[List[int]] = None,\n    head_activation: str = \"relu\",\n    head_dropout: Optional[float] = None,\n    head_batchnorm: bool = False,\n    head_batchnorm_last: bool = False,\n    head_linear_first: bool = False,\n):\n    super(StackedAttentiveRNN, self).__init__()\n\n    if (\n        embed_dim is not None\n        and embed_matrix is not None\n        and not embed_dim == embed_matrix.shape[1]\n    ):\n        warnings.warn(\n            \"the input embedding dimension {} and the dimension of the \"\n            \"pretrained embeddings {} do not match. The pretrained embeddings \"\n            \"dimension ({}) will be used\".format(\n                embed_dim, embed_matrix.shape[1], embed_matrix.shape[1]\n            ),\n            UserWarning,\n        )\n\n    if rnn_type.lower() not in [\"lstm\", \"gru\"]:\n        raise ValueError(\n            f\"'rnn_type' must be 'lstm' or 'gru', got {rnn_type} instead\"\n        )\n\n    self.vocab_size = vocab_size\n    self.embed_trainable = embed_trainable\n    self.embed_dim = embed_dim\n\n    self.rnn_type = rnn_type\n    self.hidden_dim = hidden_dim\n    self.bidirectional = bidirectional\n    self.padding_idx = padding_idx\n\n    self.n_blocks = n_blocks\n    self.attn_concatenate = attn_concatenate\n    self.attn_dropout = attn_dropout\n    self.with_addnorm = with_addnorm\n\n    self.head_hidden_dims = head_hidden_dims\n    self.head_activation = head_activation\n    self.head_dropout = head_dropout\n    self.head_batchnorm = head_batchnorm\n    self.head_batchnorm_last = head_batchnorm_last\n    self.head_linear_first = head_linear_first\n\n    # Embeddings\n    self.word_embed, self.embed_dim = self._set_embeddings(embed_matrix)\n\n    # Linear Projection: if embed_dim is different that the input of the\n    # attention blocks we add a linear projection\n    if bidirectional and attn_concatenate:\n        self.rnn_output_dim = hidden_dim * 4\n    elif bidirectional or attn_concatenate:\n        self.rnn_output_dim = hidden_dim * 2\n    else:\n        self.rnn_output_dim = hidden_dim\n\n    if self.rnn_output_dim != self.embed_dim:\n        self.embed_proj: Union[nn.Linear, nn.Identity] = nn.Linear(\n            self.embed_dim, self.rnn_output_dim\n        )\n    else:\n        self.embed_proj = nn.Identity()\n\n    # RNN\n    rnn_params = {\n        \"input_size\": self.rnn_output_dim,\n        \"hidden_size\": hidden_dim,\n        \"bidirectional\": bidirectional,\n        \"batch_first\": True,\n    }\n    if self.rnn_type.lower() == \"lstm\":\n        self.rnn: Union[nn.LSTM, nn.GRU] = nn.LSTM(**rnn_params)\n    elif self.rnn_type.lower() == \"gru\":\n        self.rnn = nn.GRU(**rnn_params)\n\n    # FC-Head (Mlp)\n    self.attention_blks = nn.ModuleList()\n    for i in range(n_blocks):\n        self.attention_blks.append(\n            ContextAttentionEncoder(\n                self.rnn,\n                self.rnn_output_dim,\n                attn_dropout,\n                attn_concatenate,\n                with_addnorm=with_addnorm if i != n_blocks - 1 else False,\n                sum_along_seq=i == n_blocks - 1,\n            )\n        )\n\n    # Mlp\n    if self.head_hidden_dims is not None:\n        head_hidden_dims = [self.rnn_output_dim] + self.head_hidden_dims\n        self.rnn_mlp: Union[MLP, nn.Identity] = MLP(\n            head_hidden_dims,\n            head_activation,\n            head_dropout,\n            head_batchnorm,\n            head_batchnorm_last,\n            head_linear_first,\n        )\n    else:\n        # simple hack to add readability in the forward pass\n        self.rnn_mlp = nn.Identity()\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.text.rnns.stacked_attentive_rnn.StackedAttentiveRNN.output_dim","title":"output_dim  <code>property</code>","text":"<pre><code>output_dim\n</code></pre> <p>The output dimension of the model. This is a required property neccesary to build the <code>WideDeep</code> class</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.text.rnns.stacked_attentive_rnn.StackedAttentiveRNN.attention_weights","title":"attention_weights  <code>property</code>","text":"<pre><code>attention_weights\n</code></pre> <p>List with the attention weights per block</p> <p>The shape of the attention weights is \\((N, S)\\) Where \\(N\\) is the batch size and \\(S\\) is the length of the sequence</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.text.huggingface_transformers.hf_model.HFModel","title":"HFModel","text":"<pre><code>HFModel(\n    model_name,\n    use_cls_token=True,\n    trainable_parameters=None,\n    head_hidden_dims=None,\n    head_activation=\"relu\",\n    head_dropout=None,\n    head_batchnorm=False,\n    head_batchnorm_last=False,\n    head_linear_first=False,\n    verbose=False,\n    **kwargs\n)\n</code></pre> <p>               Bases: <code>BaseWDModelComponent</code></p> <p>This class is a wrapper around the Hugging Face transformers library. It can be used as the text component of a Wide &amp; Deep model or independently by itself.</p> <p>At the moment only models from the families BERT, RoBERTa, DistilBERT, ALBERT and ELECTRA are supported. This is because this library is designed to address classification and regression tasks and these are the most 'popular' encoder-only models, which have proved to be those that work best for these tasks.</p> <p>Parameters:</p> <ul> <li> <code>model_name</code>               (<code>str</code>)           \u2013            <p>The model name from the transformers library e.g. 'bert-base-uncased'. Currently supported models are those from the families: BERT, RoBERTa, DistilBERT, ALBERT and ELECTRA.</p> </li> <li> <code>use_cls_token</code>               (<code>bool</code>, default:                   <code>True</code> )           \u2013            <p>Boolean indicating whether to use the [CLS] token or the mean of the sequence of hidden states as the sentence embedding</p> </li> <li> <code>trainable_parameters</code>               (<code>Optional[List[str]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the names of the model parameters that will be trained. If None, none of the parameters will be trainable</p> </li> <li> <code>head_hidden_dims</code>               (<code>Optional[List[int]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the sizes of the dense layers in the head e.g: [128, 64]</p> </li> <li> <code>head_activation</code>               (<code>str</code>, default:                   <code>'relu'</code> )           \u2013            <p>Activation function for the dense layers in the head. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> </li> <li> <code>head_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Dropout of the dense layers in the head</p> </li> <li> <code>head_batchnorm</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether or not to include batch normalization in the dense layers that form the head</p> </li> <li> <code>head_batchnorm_last</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether or not to apply batch normalization to the last of the dense layers in the head</p> </li> <li> <code>head_linear_first</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code></p> </li> <li> <code>verbose</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>If True, it will print information about the model</p> </li> <li> <code>**kwargs</code>           \u2013            <p>Additional kwargs to be passed to the model</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>head</code>               (<code>Module</code>)           \u2013            <p>Stack of dense layers on top of the transformer. This will only exists if <code>head_layers_dim</code> is not None</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import HFModel\n&gt;&gt;&gt; X_text = torch.cat((torch.zeros([5,1]), torch.empty(5, 4).random_(1,4)), axis=1).long()\n&gt;&gt;&gt; model = HFModel(model_name='bert-base-uncased')\n&gt;&gt;&gt; out = model(X_text)\n</code></pre> Source code in <code>pytorch_widedeep/models/text/huggingface_transformers/hf_model.py</code> <pre><code>@alias(\"use_cls_token\", [\"use_special_token\"])\ndef __init__(\n    self,\n    model_name: str,\n    use_cls_token: bool = True,\n    trainable_parameters: Optional[List[str]] = None,\n    head_hidden_dims: Optional[List[int]] = None,\n    head_activation: str = \"relu\",\n    head_dropout: Optional[float] = None,\n    head_batchnorm: bool = False,\n    head_batchnorm_last: bool = False,\n    head_linear_first: bool = False,\n    verbose: bool = False,\n    **kwargs,\n):\n    super().__init__()\n\n    # TO DO: add warning regarging ELECTRA as ELECTRA does not have a cls\n    # token.  Research what happens with ELECTRA\n    self.model_name = model_name\n    self.use_cls_token = use_cls_token\n    self.trainable_parameters = trainable_parameters\n    self.head_hidden_dims = head_hidden_dims\n    self.head_activation = head_activation\n    self.head_dropout = head_dropout\n    self.head_batchnorm = head_batchnorm\n    self.head_batchnorm_last = head_batchnorm_last\n    self.head_linear_first = head_linear_first\n    self.verbose = verbose\n    self.kwargs = kwargs\n\n    if self.verbose and self.use_cls_token:\n        warnings.warn(\n            \"The model will use the [CLS] token. Make sure the tokenizer \"\n            \"was run with add_special_tokens=True\",\n            UserWarning,\n        )\n\n    self.model_class = get_model_class(model_name)\n\n    self.config, self.model = get_config_and_model(self.model_name)\n\n    self.output_attention_weights = kwargs.get(\"output_attentions\", False)\n\n    if self.trainable_parameters is not None:\n        for n, p in self.model.named_parameters():\n            p.requires_grad = any([tl in n for tl in self.trainable_parameters])\n\n    # FC-Head (Mlp). Note that the FC head will always be trainable\n    if self.head_hidden_dims is not None:\n        head_hidden_dims = [self.config.hidden_size] + self.head_hidden_dims\n        self.head = MLP(\n            head_hidden_dims,\n            head_activation,\n            head_dropout,\n            head_batchnorm,\n            head_batchnorm_last,\n            head_linear_first,\n        )\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.text.huggingface_transformers.hf_model.HFModel.attention_weight","title":"attention_weight  <code>property</code>","text":"<pre><code>attention_weight\n</code></pre> <p>Returns the attention weights if the model was created with the output_attention_weights=True argument. If not, it will raise an AttributeError.</p> <p>The shape of the attention weights is \\((N, H, F, F)\\), where \\(N\\) is the batch size, \\(H\\) is the number of attention heads and \\(F\\) is the sequence length.</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.image.vision.Vision","title":"Vision","text":"<pre><code>Vision(\n    pretrained_model_setup=None,\n    n_trainable=None,\n    trainable_params=None,\n    channel_sizes=[64, 128, 256, 512],\n    kernel_sizes=[7, 3, 3, 3],\n    strides=[2, 1, 1, 1],\n    head_hidden_dims=None,\n    head_activation=\"relu\",\n    head_dropout=0.1,\n    head_batchnorm=False,\n    head_batchnorm_last=False,\n    head_linear_first=False,\n)\n</code></pre> <p>               Bases: <code>BaseWDModelComponent</code></p> <p>Defines a standard image classifier/regressor using a pretrained network or a sequence of convolution layers that can be used as the <code>deepimage</code> component of a Wide &amp; Deep model or independently by itself.</p> <p> NOTE: this class represents the integration  between <code>pytorch-widedeep</code> and <code>torchvision</code>. New architectures will be  available as they are added to <code>torchvision</code>. In a distant future we aim  to bring transformer-based architectures as well. However, simple  CNN-based architectures (and even MLP-based) seem to produce SoTA  results. For the time being, we describe below the options available  through this class</p> <p>Parameters:</p> <ul> <li> <code>pretrained_model_setup</code>               (<code>Union[str, Dict[str, Union[str, WeightsEnum]]]</code>, default:                   <code>None</code> )           \u2013            <p>Name of the pretrained model. Should be a variant of the following architectures: 'resnet', 'shufflenet', 'resnext', 'wide_resnet', 'regnet', 'densenet', 'mobilenetv3', 'mobilenetv2', 'mnasnet', 'efficientnet' and 'squeezenet'. if <code>pretrained_model_setup = None</code> a basic, fully trainable CNN will be used. Alternatively, since Torchvision 0.13 one can use pretrained models with different weigths. Therefore, <code>pretrained_model_setup</code> can also be dictionary with the name of the model and the weights (e.g. <code>{'resnet50': ResNet50_Weights.DEFAULT}</code> or <code>{'resnet50': \"IMAGENET1K_V2\"}</code>).  Aliased as <code>pretrained_model_name</code>.</p> </li> <li> <code>n_trainable</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>Number of trainable layers starting from the layer closer to the output neuron(s). Note that this number DOES NOT take into account the so-called 'head' which is ALWAYS trainable. If <code>trainable_params</code> is not None this parameter will be ignored</p> </li> <li> <code>trainable_params</code>               (<code>Optional[List[str]]</code>, default:                   <code>None</code> )           \u2013            <p>List of strings containing the names (or substring within the name) of the parameters that will be trained. For example, if we use a 'resnet18' pretrained model and we set <code>trainable_params = ['layer4']</code> only the parameters of 'layer4' of the network (and the head, as mentioned before) will be trained. Note that setting this or the previous parameter involves some knowledge of the architecture used.</p> </li> <li> <code>channel_sizes</code>               (<code>List[int]</code>, default:                   <code>[64, 128, 256, 512]</code> )           \u2013            <p>List of integers with the channel sizes of a CNN in case we choose not to use a pretrained model</p> </li> <li> <code>kernel_sizes</code>               (<code>Union[int, List[int]]</code>, default:                   <code>[7, 3, 3, 3]</code> )           \u2013            <p>List of integers with the kernel sizes of a CNN in case we choose not to use a pretrained model. Must be of length equal to <code>len(channel_sizes) - 1</code>.</p> </li> <li> <code>strides</code>               (<code>Union[int, List[int]]</code>, default:                   <code>[2, 1, 1, 1]</code> )           \u2013            <p>List of integers with the stride sizes of a CNN in case we choose not to use a pretrained model. Must be of length equal to <code>len(channel_sizes) - 1</code>.</p> </li> <li> <code>head_hidden_dims</code>               (<code>Optional[List[int]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the number of neurons per dense layer in the head. e.g: [64,32]</p> </li> <li> <code>head_activation</code>               (<code>str</code>, default:                   <code>'relu'</code> )           \u2013            <p>Activation function for the dense layers in the head. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> </li> <li> <code>head_dropout</code>               (<code>Union[float, List[float]]</code>, default:                   <code>0.1</code> )           \u2013            <p>float indicating the dropout between the dense layers.</p> </li> <li> <code>head_batchnorm</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether or not batch normalization will be applied to the dense layers</p> </li> <li> <code>head_batchnorm_last</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether or not batch normalization will be applied to the last of the dense layers</p> </li> <li> <code>head_linear_first</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code></p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>features</code>               (<code>Module</code>)           \u2013            <p>The pretrained model or Standard CNN plus the optional head</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import Vision\n&gt;&gt;&gt; X_img = torch.rand((2,3,224,224))\n&gt;&gt;&gt; model = Vision(channel_sizes=[64, 128], kernel_sizes = [3, 3], strides=[1, 1], head_hidden_dims=[32, 8])\n&gt;&gt;&gt; out = model(X_img)\n</code></pre> Source code in <code>pytorch_widedeep/models/image/vision.py</code> <pre><code>@alias(\"pretrained_model_setup\", [\"pretrained_model_name\"])\ndef __init__(\n    self,\n    pretrained_model_setup: Union[str, Dict[str, Union[str, WeightsEnum]]] = None,\n    n_trainable: Optional[int] = None,\n    trainable_params: Optional[List[str]] = None,\n    channel_sizes: List[int] = [64, 128, 256, 512],\n    kernel_sizes: Union[int, List[int]] = [7, 3, 3, 3],\n    strides: Union[int, List[int]] = [2, 1, 1, 1],\n    head_hidden_dims: Optional[List[int]] = None,\n    head_activation: str = \"relu\",\n    head_dropout: Union[float, List[float]] = 0.1,\n    head_batchnorm: bool = False,\n    head_batchnorm_last: bool = False,\n    head_linear_first: bool = False,\n):\n    super(Vision, self).__init__()\n\n    self._check_pretrained_model_setup(\n        pretrained_model_setup, n_trainable, trainable_params\n    )\n\n    self.pretrained_model_setup = pretrained_model_setup\n    self.n_trainable = n_trainable\n    self.trainable_params = trainable_params\n    self.channel_sizes = channel_sizes\n    self.kernel_sizes = kernel_sizes\n    self.strides = strides\n    self.head_hidden_dims = head_hidden_dims\n    self.head_activation = head_activation\n    self.head_dropout = head_dropout\n    self.head_batchnorm = head_batchnorm\n    self.head_batchnorm_last = head_batchnorm_last\n    self.head_linear_first = head_linear_first\n\n    self.features, self.backbone_output_dim = self._get_features()\n\n    if pretrained_model_setup is not None:\n        self._freeze(self.features)\n\n    if self.head_hidden_dims is not None:\n        head_hidden_dims = [self.backbone_output_dim] + self.head_hidden_dims\n        self.vision_mlp = MLP(\n            head_hidden_dims,\n            self.head_activation,\n            self.head_dropout,\n            self.head_batchnorm,\n            self.head_batchnorm_last,\n            self.head_linear_first,\n        )\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.image.vision.Vision.output_dim","title":"output_dim  <code>property</code>","text":"<pre><code>output_dim\n</code></pre> <p>The output dimension of the model. This is a required property neccesary to build the <code>WideDeep</code> class</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.model_fusion.ModelFuser","title":"ModelFuser","text":"<pre><code>ModelFuser(\n    models,\n    *,\n    fusion_method,\n    projection_method=None,\n    custom_head=None,\n    head_hidden_dims=None,\n    head_activation=None,\n    head_dropout=None,\n    head_batchnorm=None,\n    head_batchnorm_last=None,\n    head_linear_first=None\n)\n</code></pre> <p>               Bases: <code>BaseWDModelComponent</code></p> <p>This class is a wrapper around a list of models that are associated to the different text and/or image columns (and datasets) The class is designed to 'fuse' the models using a variety of methods.</p> <p>Parameters:</p> <ul> <li> <code>models</code>               (<code>List[BaseWDModelComponent]</code>)           \u2013            <p>List of models whose outputs will be fused</p> </li> <li> <code>fusion_method</code>               (<code>Union[Literal[concatenate, mean, max, sum, mult, dot, head], List[Literal[concatenate, mean, max, sum, mult, head]]]</code>)           \u2013            <p>Method to fuse the output of the models. It can be one of ['concatenate', 'mean', 'max', 'sum', 'mult', 'dot', 'head'] or a list of those, but 'dot'. If a list is provided the output of the models will be fused using all the methods in the list and the final output will be the concatenation of the outputs of each method</p> </li> <li> <code>projection_method</code>               (<code>Optional[Literal[min, max, mean]]</code>, default:                   <code>None</code> )           \u2013            <p>If the fusion_method is not 'concatenate', this parameter will determine how to project the output of the models to a common dimension. It can be one of ['min', 'max', 'mean']. Default is None</p> </li> <li> <code>custom_head</code>               (<code>Optional[Union[BaseWDModelComponent, Module]]</code>, default:                   <code>None</code> )           \u2013            <p>Custom head to be used to fuse the output of the models. If provided, this will take precedence over head_hidden_dims. Also, if provided, 'projection_method' will be ignored.</p> </li> <li> <code>head_hidden_dims</code>               (<code>Optional[List[int]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the number of neurons per layer in the custom head. If custom_head is provided, this parameter will be ignored</p> </li> <li> <code>head_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function to be used in the custom head. Default is None</p> </li> <li> <code>head_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Dropout to be used in the custom head. Default is None</p> </li> <li> <code>head_batchnorm</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Whether to use batchnorm in the custom head. Default is None</p> </li> <li> <code>head_batchnorm_last</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Whether or not batch normalization will be applied to the last of the dense layers</p> </li> <li> <code>head_linear_first</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code></p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>head</code>               (<code>Module or BaseWDModelComponent</code>)           \u2013            <p>Custom head to be used to fuse the output of the models. If custom_head is provided, this will take precedence over head_hidden_dims</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; from pytorch_widedeep.preprocessing import TextPreprocessor\n&gt;&gt;&gt; from pytorch_widedeep.models import BasicRNN, ModelFuser\n&gt;&gt;&gt; import torch\n&gt;&gt;&gt; import pandas as pd\n&gt;&gt;&gt;\n&gt;&gt;&gt; df = pd.DataFrame({'text_col1': ['hello world', 'this is a test'],\n... 'text_col2': ['goodbye world', 'this is another test']})\n&gt;&gt;&gt; text_preprocessor_1 = TextPreprocessor(\n...     text_col=\"text_col1\",\n...     max_vocab=10,\n...     min_freq=1,\n...     maxlen=5,\n...     n_cpus=1,\n...     verbose=0)\n&gt;&gt;&gt; text_preprocessor_2 = TextPreprocessor(\n...     text_col=\"text_col2\",\n...     max_vocab=10,\n...     min_freq=1,\n...     maxlen=5,\n...     n_cpus=1,\n...     verbose=0)\n&gt;&gt;&gt; X_text1 = text_preprocessor_1.fit_transform(df)\n&gt;&gt;&gt; X_text2 = text_preprocessor_2.fit_transform(df)\n&gt;&gt;&gt; X_text1_tnsr = torch.from_numpy(X_text1)\n&gt;&gt;&gt; X_text2_tnsr = torch.from_numpy(X_text2)\n&gt;&gt;&gt; rnn1 = BasicRNN(\n...     vocab_size=len(text_preprocessor_1.vocab.itos),\n...     embed_dim=4,\n...     hidden_dim=4,\n...     n_layers=1,\n...     bidirectional=False)\n&gt;&gt;&gt; rnn2 = BasicRNN(\n...     vocab_size=len(text_preprocessor_2.vocab.itos),\n...     embed_dim=4,\n...     hidden_dim=4,\n...     n_layers=1,\n...     bidirectional=False)\n&gt;&gt;&gt; fused_model = ModelFuser(models=[rnn1, rnn2], fusion_method='concatenate')\n&gt;&gt;&gt; out = fused_model([X_text1_tnsr, X_text2_tnsr])\n</code></pre> Source code in <code>pytorch_widedeep/models/model_fusion.py</code> <pre><code>def __init__(\n    self,\n    models: List[BaseWDModelComponent],\n    *,\n    fusion_method: Union[\n        Literal[\n            \"concatenate\",\n            \"mean\",\n            \"max\",\n            \"sum\",\n            \"mult\",\n            \"dot\",\n            \"head\",\n        ],\n        List[Literal[\"concatenate\", \"mean\", \"max\", \"sum\", \"mult\", \"head\"]],\n    ],\n    projection_method: Optional[Literal[\"min\", \"max\", \"mean\"]] = None,\n    custom_head: Optional[Union[BaseWDModelComponent, nn.Module]] = None,\n    head_hidden_dims: Optional[List[int]] = None,\n    head_activation: Optional[str] = None,\n    head_dropout: Optional[float] = None,\n    head_batchnorm: Optional[bool] = None,\n    head_batchnorm_last: Optional[bool] = None,\n    head_linear_first: Optional[bool] = None,\n) -&gt; None:\n    super(ModelFuser, self).__init__()\n\n    self.models = nn.ModuleList(models)\n    self.fusion_method = fusion_method\n    self.projection_method = projection_method\n\n    self.all_output_dim_equal = all(\n        model.output_dim == self.models[0].output_dim for model in self.models\n    )\n\n    self.check_input_parameters()\n\n    if self.fusion_method == \"head\":\n        assert (\n            head_hidden_dims is not None or custom_head is not None\n        ), \"When using 'head' as fusion_method, either head_hidden_dims or custom_head must be provided\"\n        if custom_head is not None:\n            # custom_head takes precedence over head_hidden_dims (in case\n            # both are provided)\n            assert hasattr(\n                custom_head, \"output_dim\"\n            ), \"custom_head must have an 'output_dim' property\"\n            self.head: Union[BaseWDModelComponent, nn.Module] = custom_head\n        else:\n            assert head_hidden_dims is not None\n            self.head_hidden_dims = head_hidden_dims\n            self.head_activation = head_activation\n            self.head_dropout = head_dropout\n            self.head_batchnorm = head_batchnorm\n            self.head_batchnorm_last = head_batchnorm_last\n            self.head_linear_first = head_linear_first\n\n            self.head = MLP(\n                d_hidden=[sum([model.output_dim for model in self.models])]\n                + self.head_hidden_dims,\n                activation=(\n                    \"relu\" if self.head_activation is None else self.head_activation\n                ),\n                dropout=0.0 if self.head_dropout is None else self.head_dropout,\n                batchnorm=(\n                    False if self.head_batchnorm is None else self.head_batchnorm\n                ),\n                batchnorm_last=(\n                    False\n                    if self.head_batchnorm_last is None\n                    else self.head_batchnorm_last\n                ),\n                linear_first=(\n                    True\n                    if self.head_linear_first is None\n                    else self.head_linear_first\n                ),\n            )\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.model_fusion.ModelFuser.output_dim","title":"output_dim  <code>property</code>","text":"<pre><code>output_dim\n</code></pre> <p>Returns the output dimension of the model.</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.model_fusion.ModelFuser.project","title":"project","text":"<pre><code>project(X)\n</code></pre> <p>Projects the output of the models to a common dimension.</p> Source code in <code>pytorch_widedeep/models/model_fusion.py</code> <pre><code>def project(self, X: List[Tensor]) -&gt; List[Tensor]:\n    r\"\"\"Projects the output of the models to a common dimension.\"\"\"\n\n    if self.all_output_dim_equal and self.projection_method is None:\n        return X\n\n    output_dims = [model.output_dim for model in self.models]\n\n    if self.projection_method == \"min\":\n        proj_dim = min(output_dims)\n        idx = output_dims.index(proj_dim)\n    elif self.projection_method == \"max\":\n        proj_dim = max(output_dims)\n        idx = output_dims.index(proj_dim)\n    elif self.projection_method == \"mean\":\n        proj_dim = int(sum(output_dims) / len(output_dims))\n        idx = None\n    else:\n        raise ValueError(\"projection_method must be one of ['min', 'max', 'mean']\")\n\n    x_proj: List[Tensor] = []\n    for i, x in enumerate(X):\n        if i == idx:\n            x_proj.append(x)\n        else:\n            x_proj.append(\n                nn.Linear(output_dims[i], proj_dim, bias=False, device=x.device)(x)\n            )\n\n    return x_proj\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.wide_deep.WideDeep","title":"WideDeep","text":"<pre><code>WideDeep(\n    wide=None,\n    deeptabular=None,\n    deeptext=None,\n    deepimage=None,\n    deephead=None,\n    head_hidden_dims=None,\n    head_activation=\"relu\",\n    head_dropout=0.1,\n    head_batchnorm=False,\n    head_batchnorm_last=False,\n    head_linear_first=True,\n    enforce_positive=False,\n    enforce_positive_activation=\"softplus\",\n    pred_dim=1,\n)\n</code></pre> <p>               Bases: <code>Module</code></p> <p>Main collector class that combines all <code>wide</code>, <code>deeptabular</code> <code>deeptext</code> and <code>deepimage</code> models.</p> <p>Note that all models described so far in this library must be passed to the <code>WideDeep</code> class once constructed. This is because the models output the last layer before the prediction layer. Such prediction layer is added by the <code>WideDeep</code> class as it collects the components for every data mode.</p> <p>There are two options to combine these models that correspond to the two main architectures that <code>pytorch-widedeep</code> can build.</p> <ul> <li> <p>Directly connecting the output of the model components to an ouput neuron(s).</p> </li> <li> <p>Adding a <code>Fully-Connected Head</code> (FC-Head) on top of the deep models.   This FC-Head will combine the output form the <code>deeptabular</code>, <code>deeptext</code> and   <code>deepimage</code> and will be then connected to the output neuron(s).</p> </li> </ul> <p>Parameters:</p> <ul> <li> <code>wide</code>               (<code>Optional[Module]</code>, default:                   <code>None</code> )           \u2013            <p><code>Wide</code> model. This is a linear model where the non-linearities are captured via crossed-columns.</p> </li> <li> <code>deeptabular</code>               (<code>Optional[Union[BaseWDModelComponent, List[BaseWDModelComponent]]]</code>, default:                   <code>None</code> )           \u2013            <p>Currently this library implements a number of possible architectures for the <code>deeptabular</code> component. See the documenation of the package. Note that <code>deeptabular</code> can be a list of models. This is useful when using multiple tabular inputs (e.g. for example in the context of a two-tower model for recommendation systems)</p> </li> <li> <code>deeptext</code>               (<code>Optional[Union[BaseWDModelComponent, List[BaseWDModelComponent]]]</code>, default:                   <code>None</code> )           \u2013            <p>Currently this library implements a number of possible architectures for the <code>deeptext</code> component. See the documenation of the package. Note that <code>deeptext</code> can be a list of models. This is useful when using multiple text inputs.</p> </li> <li> <code>deepimage</code>               (<code>Optional[Union[BaseWDModelComponent, List[BaseWDModelComponent]]]</code>, default:                   <code>None</code> )           \u2013            <p>Currently this library uses <code>torchvision</code> and implements a number of possible architectures for the <code>deepimage</code> component. See the documenation of the package. Note that <code>deepimage</code> can be a list of models. This is useful when using multiple image inputs.</p> </li> <li> <code>deephead</code>               (<code>Optional[BaseWDModelComponent]</code>, default:                   <code>None</code> )           \u2013            <p>Alternatively, the user can pass a custom model that will receive the output of the deep component. If <code>deephead</code> is not None all the previous fc-head parameters will be ignored</p> </li> <li> <code>head_hidden_dims</code>               (<code>Optional[List[int]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the sizes of the dense layers in the head e.g: [128, 64]</p> </li> <li> <code>head_activation</code>               (<code>str</code>, default:                   <code>'relu'</code> )           \u2013            <p>Activation function for the dense layers in the head. Currently <code>'tanh'</code>, <code>'relu'</code>, <code>'leaky_relu'</code> and <code>'gelu'</code> are supported</p> </li> <li> <code>head_dropout</code>               (<code>float</code>, default:                   <code>0.1</code> )           \u2013            <p>Dropout of the dense layers in the head</p> </li> <li> <code>head_batchnorm</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether or not to include batch normalization in the dense layers that form the <code>'rnn_mlp'</code></p> </li> <li> <code>head_batchnorm_last</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether or not to apply batch normalization to the last of the dense layers in the head</p> </li> <li> <code>head_linear_first</code>               (<code>bool</code>, default:                   <code>True</code> )           \u2013            <p>Boolean indicating whether the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code></p> </li> <li> <code>enforce_positive</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating if the output from the final layer must be positive. This is important if you are using loss functions with non-negative input restrictions, e.g. RMSLE, or if you know your predictions are bounded in between 0 and inf</p> </li> <li> <code>enforce_positive_activation</code>               (<code>str</code>, default:                   <code>'softplus'</code> )           \u2013            <p>Activation function to enforce that the final layer has a positive output. <code>'softplus'</code> or <code>'relu'</code> are supported.</p> </li> <li> <code>pred_dim</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>Size of the final wide and deep output layer containing the predictions. <code>1</code> for regression and binary classification or number of classes for multiclass classification.</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; from pytorch_widedeep.models import TabResnet, Vision, BasicRNN, Wide, WideDeep\n&gt;&gt;&gt; embed_input = [(u, i, j) for u, i, j in zip([\"a\", \"b\", \"c\"][:4], [4] * 3, [8] * 3)]\n&gt;&gt;&gt; column_idx = {k: v for v, k in enumerate([\"a\", \"b\", \"c\"])}\n&gt;&gt;&gt; wide = Wide(10, 1)\n&gt;&gt;&gt; deeptabular = TabResnet(blocks_dims=[8, 4], column_idx=column_idx, cat_embed_input=embed_input)\n&gt;&gt;&gt; deeptext = BasicRNN(vocab_size=10, embed_dim=4, padding_idx=0)\n&gt;&gt;&gt; deepimage = Vision()\n&gt;&gt;&gt; model = WideDeep(wide=wide, deeptabular=deeptabular, deeptext=deeptext, deepimage=deepimage)\n</code></pre> <p> NOTE: It is possible to use custom components to  build Wide &amp; Deep models. Simply, build them and pass them as the  corresponding parameters. Note that the custom models MUST return a last  layer of activations(i.e. not the final prediction) so that  these  activations are collected by <code>WideDeep</code> and combined accordingly. In  addition, the models MUST also contain an attribute <code>output_dim</code> with  the size of these last layers of activations. See for example  <code>pytorch_widedeep.models.tab_mlp.TabMlp</code></p> Source code in <code>pytorch_widedeep/models/wide_deep.py</code> <pre><code>@alias(  # noqa: C901\n    \"pred_dim\",\n    [\"num_class\", \"pred_size\"],\n)\ndef __init__(\n    self,\n    wide: Optional[nn.Module] = None,\n    deeptabular: Optional[\n        Union[BaseWDModelComponent, List[BaseWDModelComponent]]\n    ] = None,\n    deeptext: Optional[\n        Union[BaseWDModelComponent, List[BaseWDModelComponent]]\n    ] = None,\n    deepimage: Optional[\n        Union[BaseWDModelComponent, List[BaseWDModelComponent]]\n    ] = None,\n    deephead: Optional[BaseWDModelComponent] = None,\n    head_hidden_dims: Optional[List[int]] = None,\n    head_activation: str = \"relu\",\n    head_dropout: float = 0.1,\n    head_batchnorm: bool = False,\n    head_batchnorm_last: bool = False,\n    head_linear_first: bool = True,\n    enforce_positive: bool = False,\n    enforce_positive_activation: str = \"softplus\",\n    pred_dim: int = 1,\n):\n    super(WideDeep, self).__init__()\n\n    self._check_inputs(\n        wide,\n        deeptabular,\n        deeptext,\n        deepimage,\n        deephead,\n        head_hidden_dims,\n        pred_dim,\n    )\n\n    # this attribute will be eventually over-written by the Trainer's\n    # device. Acts here as a 'placeholder'.\n    self.wd_device: Optional[str] = None\n\n    # required as attribute just in case we pass a deephead\n    self.pred_dim = pred_dim\n\n    self.enforce_positive = enforce_positive\n\n    # better to set this attribute already here\n    if isinstance(deeptabular, list):\n        self.is_tabnet = False\n    else:\n        self.is_tabnet = deeptabular.__class__.__name__ == \"TabNet\"\n\n    # The main 5 components of the wide and deep assemble: wide,\n    # deeptabular, deeptext, deepimage and deephead\n    self.with_deephead = deephead is not None or head_hidden_dims is not None\n    if deephead is None and head_hidden_dims is not None:\n        self.deephead = self._build_deephead(\n            deeptabular,\n            deeptext,\n            deepimage,\n            head_hidden_dims,\n            head_activation,\n            head_dropout,\n            head_batchnorm,\n            head_batchnorm_last,\n            head_linear_first,\n        )\n    elif deephead is not None:\n        self.deephead = nn.Sequential(\n            deephead, nn.Linear(deephead.output_dim, self.pred_dim)\n        )\n    else:\n        # for consistency with other components we default to None\n        self.deephead = None\n\n    self.wide = wide\n    self.deeptabular, self.deeptext, self.deepimage = self._set_model_components(\n        deeptabular, deeptext, deepimage\n    )\n\n    if self.enforce_positive:\n        self.enf_pos = get_activation_fn(enforce_positive_activation)\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html","title":"The <code>preprocessing</code> module","text":"<p>This module contains the classes that are used to prepare the data before being passed to the models. There is one Preprocessor per data mode or model component (<code>wide</code>, <code>deeptabular</code>, <code>deepimage</code> and <code>deeptext</code>) with the exception of the <code>deeptext</code> component. In this case, two processors are available: one for the case when no Hugging Face model is used (<code>TextPreprocessor</code>) and another one when a Hugging Face model is used (<code>HFPreprocessor</code>).</p>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.wide_preprocessor.WidePreprocessor","title":"WidePreprocessor","text":"<pre><code>WidePreprocessor(wide_cols, crossed_cols=None)\n</code></pre> <p>               Bases: <code>BasePreprocessor</code></p> <p>Preprocessor to prepare the wide input dataset</p> <p>This Preprocessor prepares the data for the wide, linear component. This linear model is implemented via an Embedding layer that is connected to the output neuron. <code>WidePreprocessor</code> numerically encodes all the unique values of all categorical columns <code>wide_cols + crossed_cols</code>. See the Example below.</p> <p>Parameters:</p> <ul> <li> <code>wide_cols</code>               (<code>List[str]</code>)           \u2013            <p>List of strings with the name of the columns that will label encoded and passed through the <code>wide</code> component</p> </li> <li> <code>crossed_cols</code>               (<code>Optional[List[Tuple[str, str]]]</code>, default:                   <code>None</code> )           \u2013            <p>List of Tuples with the name of the columns that will be <code>'crossed'</code> and then label encoded. e.g. [('education', 'occupation'), ...]. For binary features, a cross-product transformation is 1 if and only if the constituent features are all 1, and 0 otherwise.</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>wide_crossed_cols</code>               (<code>List</code>)           \u2013            <p>List with the names of all columns that will be label encoded</p> </li> <li> <code>encoding_dict</code>               (<code>Dict</code>)           \u2013            <p>Dictionary where the keys are the result of pasting <code>colname + '_' + column value</code> and the values are the corresponding mapped integer.</p> </li> <li> <code>inverse_encoding_dict</code>               (<code>Dict</code>)           \u2013            <p>the inverse encoding dictionary</p> </li> <li> <code>wide_dim</code>               (<code>int</code>)           \u2013            <p>Dimension of the wide model (i.e. dim of the linear layer)</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import pandas as pd\n&gt;&gt;&gt; from pytorch_widedeep.preprocessing import WidePreprocessor\n&gt;&gt;&gt; df = pd.DataFrame({'color': ['r', 'b', 'g'], 'size': ['s', 'n', 'l']})\n&gt;&gt;&gt; wide_cols = ['color']\n&gt;&gt;&gt; crossed_cols = [('color', 'size')]\n&gt;&gt;&gt; wide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols)\n&gt;&gt;&gt; X_wide = wide_preprocessor.fit_transform(df)\n&gt;&gt;&gt; X_wide\narray([[1, 4],\n       [2, 5],\n       [3, 6]])\n&gt;&gt;&gt; wide_preprocessor.encoding_dict\n{'color_r': 1, 'color_b': 2, 'color_g': 3, 'color_size_r-s': 4, 'color_size_b-n': 5, 'color_size_g-l': 6}\n&gt;&gt;&gt; wide_preprocessor.inverse_transform(X_wide)\n  color color_size\n0     r        r-s\n1     b        b-n\n2     g        g-l\n</code></pre> Source code in <code>pytorch_widedeep/preprocessing/wide_preprocessor.py</code> <pre><code>def __init__(\n    self, wide_cols: List[str], crossed_cols: Optional[List[Tuple[str, str]]] = None\n):\n    super(WidePreprocessor, self).__init__()\n\n    self.wide_cols = wide_cols\n    self.crossed_cols = crossed_cols\n\n    self.is_fitted = False\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.wide_preprocessor.WidePreprocessor.fit","title":"fit","text":"<pre><code>fit(df)\n</code></pre> <p>Fits the Preprocessor and creates required attributes</p> <p>Parameters:</p> <ul> <li> <code>df</code>               (<code>DataFrame</code>)           \u2013            <p>Input pandas dataframe</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>WidePreprocessor</code>           \u2013            <p><code>WidePreprocessor</code> fitted object</p> </li> </ul> Source code in <code>pytorch_widedeep/preprocessing/wide_preprocessor.py</code> <pre><code>def fit(self, df: pd.DataFrame) -&gt; \"WidePreprocessor\":\n    r\"\"\"Fits the Preprocessor and creates required attributes\n\n    Parameters\n    ----------\n    df: pd.DataFrame\n        Input pandas dataframe\n\n    Returns\n    -------\n    WidePreprocessor\n        `WidePreprocessor` fitted object\n    \"\"\"\n    df_wide = self._prepare_wide(df)\n    self.wide_crossed_cols = df_wide.columns.tolist()\n    glob_feature_list = self._make_global_feature_list(\n        df_wide[self.wide_crossed_cols]\n    )\n    # leave 0 for padding/\"unseen\" categories\n    self.encoding_dict = {v: i + 1 for i, v in enumerate(glob_feature_list)}\n    self.wide_dim = len(self.encoding_dict)\n    self.inverse_encoding_dict = {k: v for v, k in self.encoding_dict.items()}\n    self.inverse_encoding_dict[0] = \"unseen\"\n\n    self.is_fitted = True\n\n    return self\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.wide_preprocessor.WidePreprocessor.transform","title":"transform","text":"<pre><code>transform(df)\n</code></pre> <p>Parameters:</p> <ul> <li> <code>df</code>               (<code>DataFrame</code>)           \u2013            <p>Input pandas dataframe</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>ndarray</code>           \u2013            <p>transformed input dataframe</p> </li> </ul> Source code in <code>pytorch_widedeep/preprocessing/wide_preprocessor.py</code> <pre><code>def transform(self, df: pd.DataFrame) -&gt; np.ndarray:\n    r\"\"\"\n    Parameters\n    ----------\n    df: pd.DataFrame\n        Input pandas dataframe\n\n    Returns\n    -------\n    np.ndarray\n        transformed input dataframe\n    \"\"\"\n    check_is_fitted(self, attributes=[\"encoding_dict\"])\n    df_wide = self._prepare_wide(df)\n    encoded = np.zeros([len(df_wide), len(self.wide_crossed_cols)])\n    for col_i, col in enumerate(self.wide_crossed_cols):\n        encoded[:, col_i] = df_wide[col].apply(\n            lambda x: (\n                self.encoding_dict[col + \"_\" + str(x)]\n                if col + \"_\" + str(x) in self.encoding_dict\n                else 0\n            )\n        )\n    return encoded.astype(\"int64\")\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.wide_preprocessor.WidePreprocessor.inverse_transform","title":"inverse_transform","text":"<pre><code>inverse_transform(encoded)\n</code></pre> <p>Takes as input the output from the <code>transform</code> method and it will return the original values.</p> <p>Parameters:</p> <ul> <li> <code>encoded</code>               (<code>ndarray</code>)           \u2013            <p>numpy array with the encoded values that are the output from the <code>transform</code> method</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>DataFrame</code>           \u2013            <p>Pandas dataframe with the original values</p> </li> </ul> Source code in <code>pytorch_widedeep/preprocessing/wide_preprocessor.py</code> <pre><code>def inverse_transform(self, encoded: np.ndarray) -&gt; pd.DataFrame:\n    r\"\"\"Takes as input the output from the `transform` method and it will\n    return the original values.\n\n    Parameters\n    ----------\n    encoded: np.ndarray\n        numpy array with the encoded values that are the output from the\n        `transform` method\n\n    Returns\n    -------\n    pd.DataFrame\n        Pandas dataframe with the original values\n    \"\"\"\n    decoded = pd.DataFrame(encoded, columns=self.wide_crossed_cols)\n\n    if pd.__version__ &gt;= \"2.1.0\":\n        decoded = decoded.map(lambda x: self.inverse_encoding_dict[x])\n    else:\n        decoded = decoded.applymap(lambda x: self.inverse_encoding_dict[x])\n\n    for col in decoded.columns:\n        rm_str = \"\".join([col, \"_\"])\n        decoded[col] = decoded[col].apply(lambda x: x.replace(rm_str, \"\"))\n    return decoded\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.wide_preprocessor.WidePreprocessor.fit_transform","title":"fit_transform","text":"<pre><code>fit_transform(df)\n</code></pre> <p>Combines <code>fit</code> and <code>transform</code></p> <p>Parameters:</p> <ul> <li> <code>df</code>               (<code>DataFrame</code>)           \u2013            <p>Input pandas dataframe</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>ndarray</code>           \u2013            <p>transformed input dataframe</p> </li> </ul> Source code in <code>pytorch_widedeep/preprocessing/wide_preprocessor.py</code> <pre><code>def fit_transform(self, df: pd.DataFrame) -&gt; np.ndarray:\n    \"\"\"Combines `fit` and `transform`\n\n    Parameters\n    ----------\n    df: pd.DataFrame\n        Input pandas dataframe\n\n    Returns\n    -------\n    np.ndarray\n        transformed input dataframe\n    \"\"\"\n    return self.fit(df).transform(df)\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor","title":"TabPreprocessor","text":"<pre><code>TabPreprocessor(\n    cat_embed_cols=None,\n    continuous_cols=None,\n    quantization_setup=None,\n    cols_to_scale=None,\n    auto_embed_dim=True,\n    embedding_rule=\"fastai_new\",\n    default_embed_dim=16,\n    with_attention=False,\n    with_cls_token=False,\n    shared_embed=False,\n    verbose=1,\n    *,\n    scale=False,\n    already_standard=None,\n    **kwargs\n)\n</code></pre> <p>               Bases: <code>BasePreprocessor</code></p> <p>Preprocessor to prepare the <code>deeptabular</code> component input dataset</p> <p>Parameters:</p> <ul> <li> <code>cat_embed_cols</code>               (<code>Optional[Union[List[str], List[Tuple[str, int]]]]</code>, default:                   <code>None</code> )           \u2013            <p>List containing the name of the categorical columns that will be represented by embeddings (e.g. ['education', 'relationship', ...]) or a Tuple with the name and the embedding dimension (e.g.: [ ('education',32), ('relationship',16), ...])</p> </li> <li> <code>continuous_cols</code>               (<code>Optional[List[str]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the name of the continuous cols</p> </li> <li> <code>quantization_setup</code>               (<code>Optional[Union[int, Dict[str, Union[int, List[float]]]]]</code>, default:                   <code>None</code> )           \u2013            <p>Continuous columns can be turned into categorical via <code>pd.cut</code>. If <code>quantization_setup</code> is an <code>int</code>, all continuous columns will be quantized using this value as the number of bins. Alternatively, a dictionary where the keys are the column names to quantize and the values are the either integers indicating the number of bins or a list of scalars indicating the bin edges can also be used.</p> </li> <li> <code>cols_to_scale</code>               (<code>Optional[Union[List[str], str]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the names of the columns that will be standarised via sklearn's <code>StandardScaler</code>. It can also be the string <code>'all'</code> in which case all the continuous cols will be scaled.</p> </li> <li> <code>auto_embed_dim</code>               (<code>bool</code>, default:                   <code>True</code> )           \u2013            <p>Boolean indicating whether the embedding dimensions will be automatically defined via rule of thumb. See <code>embedding_rule</code> below.</p> </li> <li> <code>embedding_rule</code>               (<code>Literal[google, fastai_old, fastai_new]</code>, default:                   <code>'fastai_new'</code> )           \u2013            <p>If <code>auto_embed_dim=True</code>, this is the choice of embedding rule of thumb. Choices are:</p> <ul> <li> <p>fastai_new: \\(min(600, round(1.6 \\times n_{cat}^{0.56}))\\)</p> </li> <li> <p>fastai_old: \\(min(50, (n_{cat}//{2})+1)\\)</p> </li> <li> <p>google: \\(min(600, round(n_{cat}^{0.24}))\\)</p> </li> </ul> </li> <li> <code>default_embed_dim</code>               (<code>int</code>, default:                   <code>16</code> )           \u2013            <p>Dimension for the embeddings if the embedding dimension is not provided in the <code>cat_embed_cols</code> parameter and <code>auto_embed_dim</code> is set to <code>False</code>.</p> </li> <li> <code>with_attention</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether the preprocessed data will be passed to an attention-based model (more precisely a model where all embeddings must have the same dimensions). If <code>True</code>, the param <code>cat_embed_cols</code> must just be a list containing just the categorical column names: e.g. ['education', 'relationship', ...]. This is because they will all be  encoded using embeddings of the same dim, which will be specified  later when the model is defined.  Param alias:  <code>for_transformer</code></p> </li> <li> <code>with_cls_token</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating if a <code>'[CLS]'</code> token will be added to the dataset when using attention-based models. The final hidden state corresponding to this token is used as the aggregated representation for classification and regression tasks. If not, the categorical and/or continuous embeddings will be concatenated before being passed to the final MLP (if present).</p> </li> <li> <code>shared_embed</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating if the embeddings will be \"shared\" when using attention-based models. The idea behind <code>shared_embed</code> is described in the Appendix A in the TabTransformer paper: 'The goal of having column embedding is to enable the model to distinguish the classes in one column from those in the other columns'. In other words, the idea is to let the model learn which column is embedded at the time. See: <code>pytorch_widedeep.models.transformers._layers.SharedEmbeddings</code>.</p> </li> <li> <code>verbose</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            </li> <li> <code>scale</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p> note: this arg will be removed in upcoming  releases. Please use <code>cols_to_scale</code> instead.  Bool indicating  whether or not to scale/standarise continuous cols. It is important  to emphasize that all the DL models for tabular data in the library  also include the possibility of normalising the input continuous  features via a <code>BatchNorm</code> or a <code>LayerNorm</code>.  Param alias:  <code>scale_cont_cols</code>.</p> </li> <li> <code>already_standard</code>               (<code>Optional[List[str]]</code>, default:                   <code>None</code> )           \u2013            <p> note: this arg will be removed in upcoming  releases. Please use <code>cols_to_scale</code> instead.  List with the  name of the continuous cols that do not need to be  scaled/standarised.</p> </li> </ul> <p>Other Parameters:</p> <ul> <li> <code>**kwargs</code>           \u2013            <p><code>pd.cut</code> and <code>StandardScaler</code> related args</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>embed_dim</code>               (<code>Dict</code>)           \u2013            <p>Dictionary where keys are the embed cols and values are the embedding dimensions. If <code>with_attention</code> is set to <code>True</code> this attribute is not generated during the <code>fit</code> process</p> </li> <li> <code>label_encoder</code>               (<code>LabelEncoder</code>)           \u2013            <p>see <code>pytorch_widedeep.utils.dense_utils.LabelEncder</code></p> </li> <li> <code>cat_embed_input</code>               (<code>List</code>)           \u2013            <p>List of Tuples with the column name, number of individual values for that column and, If <code>with_attention</code> is set to <code>False</code>, the corresponding embeddings dim, e.g. [('education', 16, 10), ('relationship', 6, 8), ...].</p> </li> <li> <code>standardize_cols</code>               (<code>List</code>)           \u2013            <p>List of the columns that will be standarized</p> </li> <li> <code>scaler</code>               (<code>StandardScaler</code>)           \u2013            <p>an instance of <code>sklearn.preprocessing.StandardScaler</code></p> </li> <li> <code>column_idx</code>               (<code>Dict</code>)           \u2013            <p>Dictionary where keys are column names and values are column indexes. This is neccesary to slice tensors</p> </li> <li> <code>quantizer</code>               (<code>Quantizer</code>)           \u2013            <p>an instance of <code>Quantizer</code></p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import pandas as pd\n&gt;&gt;&gt; import numpy as np\n&gt;&gt;&gt; from pytorch_widedeep.preprocessing import TabPreprocessor\n&gt;&gt;&gt; df = pd.DataFrame({'color': ['r', 'b', 'g'], 'size': ['s', 'n', 'l'], 'age': [25, 40, 55]})\n&gt;&gt;&gt; cat_embed_cols = [('color',5), ('size',5)]\n&gt;&gt;&gt; cont_cols = ['age']\n&gt;&gt;&gt; deep_preprocessor = TabPreprocessor(cat_embed_cols=cat_embed_cols, continuous_cols=cont_cols)\n&gt;&gt;&gt; X_tab = deep_preprocessor.fit_transform(df)\n&gt;&gt;&gt; deep_preprocessor.cat_embed_cols\n[('color', 5), ('size', 5)]\n&gt;&gt;&gt; deep_preprocessor.column_idx\n{'color': 0, 'size': 1, 'age': 2}\n&gt;&gt;&gt; cont_df = pd.DataFrame({\"col1\": np.random.rand(10), \"col2\": np.random.rand(10) + 1})\n&gt;&gt;&gt; cont_cols = [\"col1\", \"col2\"]\n&gt;&gt;&gt; tab_preprocessor = TabPreprocessor(continuous_cols=cont_cols, quantization_setup=3)\n&gt;&gt;&gt; ft_cont_df = tab_preprocessor.fit_transform(cont_df)\n&gt;&gt;&gt; # or...\n&gt;&gt;&gt; quantization_setup = {'col1': [0., 0.4, 1.], 'col2': [1., 1.4, 2.]}\n&gt;&gt;&gt; tab_preprocessor2 = TabPreprocessor(continuous_cols=cont_cols, quantization_setup=quantization_setup)\n&gt;&gt;&gt; ft_cont_df2 = tab_preprocessor2.fit_transform(cont_df)\n</code></pre> Source code in <code>pytorch_widedeep/preprocessing/tab_preprocessor.py</code> <pre><code>@alias(\"with_attention\", [\"for_transformer\", \"for_matrix_factorization\", \"for_mf\"])\n@alias(\"cat_embed_cols\", [\"embed_cols\"])\n@alias(\"scale\", [\"scale_cont_cols\"])\n@alias(\"quantization_setup\", [\"cols_and_bins\"])\ndef __init__(\n    self,\n    cat_embed_cols: Optional[Union[List[str], List[Tuple[str, int]]]] = None,\n    continuous_cols: Optional[List[str]] = None,\n    quantization_setup: Optional[\n        Union[int, Dict[str, Union[int, List[float]]]]\n    ] = None,\n    cols_to_scale: Optional[Union[List[str], str]] = None,\n    auto_embed_dim: bool = True,\n    embedding_rule: Literal[\"google\", \"fastai_old\", \"fastai_new\"] = \"fastai_new\",\n    default_embed_dim: int = 16,\n    with_attention: bool = False,\n    with_cls_token: bool = False,\n    shared_embed: bool = False,\n    verbose: int = 1,\n    *,\n    scale: bool = False,\n    already_standard: Optional[List[str]] = None,\n    **kwargs,\n):\n    super(TabPreprocessor, self).__init__()\n\n    self.continuous_cols = continuous_cols\n    self.quantization_setup = quantization_setup\n    self.cols_to_scale = cols_to_scale\n    self.scale = scale\n    self.already_standard = already_standard\n    self.auto_embed_dim = auto_embed_dim\n    self.embedding_rule = embedding_rule\n    self.default_embed_dim = default_embed_dim\n    self.with_attention = with_attention\n    self.with_cls_token = with_cls_token\n    self.shared_embed = shared_embed\n    self.verbose = verbose\n\n    self.quant_args = {\n        k: v for k, v in kwargs.items() if k in pd.cut.__code__.co_varnames\n    }\n    self.scale_args = {\n        k: v for k, v in kwargs.items() if k in StandardScaler().get_params()\n    }\n\n    self._check_inputs(cat_embed_cols)\n\n    if with_cls_token:\n        self.cat_embed_cols = (\n            [\"cls_token\"] + cat_embed_cols  # type: ignore[operator]\n            if cat_embed_cols is not None\n            else [\"cls_token\"]\n        )\n    else:\n        self.cat_embed_cols = cat_embed_cols  # type: ignore[assignment]\n\n    self.is_fitted = False\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor.fit","title":"fit","text":"<pre><code>fit(df)\n</code></pre> <p>Fits the Preprocessor and creates required attributes</p> <p>Parameters:</p> <ul> <li> <code>df</code>               (<code>DataFrame</code>)           \u2013            <p>Input pandas dataframe</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>TabPreprocessor</code>           \u2013            <p><code>TabPreprocessor</code> fitted object</p> </li> </ul> Source code in <code>pytorch_widedeep/preprocessing/tab_preprocessor.py</code> <pre><code>def fit(self, df: pd.DataFrame) -&gt; BasePreprocessor:  # noqa: C901\n    \"\"\"Fits the Preprocessor and creates required attributes\n\n    Parameters\n    ----------\n    df: pd.DataFrame\n        Input pandas dataframe\n\n    Returns\n    -------\n    TabPreprocessor\n        `TabPreprocessor` fitted object\n    \"\"\"\n\n    df_adj = self._insert_cls_token(df) if self.with_cls_token else df.copy()\n\n    self.column_idx: Dict[str, int] = {}\n\n    # Categorical embeddings logic\n    if self.cat_embed_cols is not None or self.quantization_setup is not None:\n        self.cat_embed_input: List[Union[Tuple[str, int], Tuple[str, int, int]]] = (\n            []\n        )\n\n    if self.cat_embed_cols is not None:\n        df_cat, cat_embed_dim = self._prepare_categorical(df_adj)\n\n        self.label_encoder = LabelEncoder(\n            columns_to_encode=df_cat.columns.tolist(),\n            shared_embed=self.shared_embed,\n            with_attention=self.with_attention,\n        )\n        self.label_encoder.fit(df_cat)\n\n        for k, v in self.label_encoder.encoding_dict.items():\n            if self.with_attention:\n                self.cat_embed_input.append((k, len(v)))\n            else:\n                self.cat_embed_input.append((k, len(v), cat_embed_dim[k]))\n\n        self.column_idx.update({k: v for v, k in enumerate(df_cat.columns)})\n\n    # Continuous columns logic\n    if self.continuous_cols is not None:\n        df_cont, cont_embed_dim = self._prepare_continuous(df_adj)\n\n        # Standardization logic\n        if self.standardize_cols is not None:\n            self.scaler = StandardScaler(**self.scale_args).fit(\n                df_cont[self.standardize_cols].values\n            )\n        elif self.verbose:\n            warnings.warn(\"Continuous columns will not be normalised\")\n\n        # Quantization logic\n        if self.cols_and_bins is not None:\n            # we do not run 'Quantizer.fit' here since in the wild case\n            # someone wants standardization and quantization for the same\n            # columns, the Quantizer will run on the scaled data\n            self.quantizer = Quantizer(self.cols_and_bins, **self.quant_args)\n\n            if self.with_attention:\n                for col, n_cat, _ in cont_embed_dim:\n                    self.cat_embed_input.append((col, n_cat))\n            else:\n                self.cat_embed_input.extend(cont_embed_dim)\n\n        self.column_idx.update(\n            {k: v + len(self.column_idx) for v, k in enumerate(df_cont)}\n        )\n\n    self.is_fitted = True\n\n    return self\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor.transform","title":"transform","text":"<pre><code>transform(df)\n</code></pre> <p>Returns the processed <code>dataframe</code> as a np.ndarray</p> <p>Parameters:</p> <ul> <li> <code>df</code>               (<code>DataFrame</code>)           \u2013            <p>Input pandas dataframe</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>ndarray</code>           \u2013            <p>transformed input dataframe</p> </li> </ul> Source code in <code>pytorch_widedeep/preprocessing/tab_preprocessor.py</code> <pre><code>def transform(self, df: pd.DataFrame) -&gt; np.ndarray:  # noqa: C901\n    \"\"\"Returns the processed `dataframe` as a np.ndarray\n\n    Parameters\n    ----------\n    df: pd.DataFrame\n        Input pandas dataframe\n\n    Returns\n    -------\n    np.ndarray\n        transformed input dataframe\n    \"\"\"\n    check_is_fitted(self, condition=self.is_fitted)\n\n    df_adj = self._insert_cls_token(df) if self.with_cls_token else df.copy()\n\n    if self.cat_embed_cols is not None:\n        df_cat = df_adj[self.cat_cols]\n        df_cat = self.label_encoder.transform(df_cat)\n    if self.continuous_cols is not None:\n        df_cont = df_adj[self.continuous_cols]\n        # Standardization logic\n        if self.standardize_cols:\n            df_cont[self.standardize_cols] = self.scaler.transform(\n                df_cont[self.standardize_cols].values\n            )\n        # Quantization logic\n        if self.cols_and_bins is not None:\n            # Adjustment so I don't have to override the method\n            # in 'ChunkTabPreprocessor'\n            if self.quantizer.is_fitted:\n                df_cont = self.quantizer.transform(df_cont)\n            else:\n                df_cont = self.quantizer.fit_transform(df_cont)\n    try:\n        df_deep = pd.concat([df_cat, df_cont], axis=1)\n    except NameError:\n        try:\n            df_deep = df_cat.copy()\n        except NameError:\n            df_deep = df_cont.copy()\n\n    return df_deep.values\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor.inverse_transform","title":"inverse_transform","text":"<pre><code>inverse_transform(encoded)\n</code></pre> <p>Takes as input the output from the <code>transform</code> method and it will return the original values.</p> <p>Parameters:</p> <ul> <li> <code>encoded</code>               (<code>ndarray</code>)           \u2013            <p>array with the output of the <code>transform</code> method</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>DataFrame</code>           \u2013            <p>Pandas dataframe with the original values</p> </li> </ul> Source code in <code>pytorch_widedeep/preprocessing/tab_preprocessor.py</code> <pre><code>def inverse_transform(self, encoded: np.ndarray) -&gt; pd.DataFrame:  # noqa: C901\n    r\"\"\"Takes as input the output from the `transform` method and it will\n    return the original values.\n\n    Parameters\n    ----------\n    encoded: np.ndarray\n        array with the output of the `transform` method\n\n    Returns\n    -------\n    pd.DataFrame\n        Pandas dataframe with the original values\n    \"\"\"\n    decoded = pd.DataFrame(encoded, columns=list(self.column_idx.keys()))\n    # embeddings back to original category\n    if self.cat_embed_cols is not None:\n        decoded = self.label_encoder.inverse_transform(decoded)\n    if self.continuous_cols is not None:\n        # quantized cols to the mid point\n        if self.cols_and_bins is not None:\n            if self.verbose:\n                print(\n                    \"Note that quantized cols will be turned into the mid point of \"\n                    \"the corresponding bin\"\n                )\n            for k, v in self.quantizer.inversed_bins.items():\n                decoded[k] = decoded[k].map(v)\n        # continuous_cols back to non-standarised\n        try:\n            decoded[self.standardize_cols] = self.scaler.inverse_transform(\n                decoded[self.standardize_cols]\n            )\n        except Exception:  # KeyError:\n            pass\n\n    if \"cls_token\" in decoded.columns:\n        decoded.drop(\"cls_token\", axis=1, inplace=True)\n\n    return decoded\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor.fit_transform","title":"fit_transform","text":"<pre><code>fit_transform(df)\n</code></pre> <p>Combines <code>fit</code> and <code>transform</code></p> <p>Parameters:</p> <ul> <li> <code>df</code>               (<code>DataFrame</code>)           \u2013            <p>Input pandas dataframe</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>ndarray</code>           \u2013            <p>transformed input dataframe</p> </li> </ul> Source code in <code>pytorch_widedeep/preprocessing/tab_preprocessor.py</code> <pre><code>def fit_transform(self, df: pd.DataFrame) -&gt; np.ndarray:\n    \"\"\"Combines `fit` and `transform`\n\n    Parameters\n    ----------\n    df: pd.DataFrame\n        Input pandas dataframe\n\n    Returns\n    -------\n    np.ndarray\n        transformed input dataframe\n    \"\"\"\n    return self.fit(df).transform(df)\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.tab_preprocessor.Quantizer","title":"Quantizer","text":"<pre><code>Quantizer(quantization_setup, **kwargs)\n</code></pre> <p>Helper class to perform the quantization of continuous columns. It is included in this docs for completion, since depending on the value of the parameter <code>'quantization_setup'</code> of the <code>TabPreprocessor</code> class, that class might have an attribute of type <code>Quantizer</code>. However, this class is designed to always run internally within the <code>TabPreprocessor</code> class.</p> <p>Parameters:</p> <ul> <li> <code>quantization_setup</code>               (<code>Dict[str, Union[int, List[float]]]</code>)           \u2013            <p>Dictionary where the keys are the column names to quantize and the values are the either integers indicating the number of bins or a list of scalars indicating the bin edges.</p> </li> </ul> Source code in <code>pytorch_widedeep/preprocessing/tab_preprocessor.py</code> <pre><code>def __init__(\n    self,\n    quantization_setup: Dict[str, Union[int, List[float]]],\n    **kwargs,\n):\n    self.quantization_setup = quantization_setup\n    self.quant_args = kwargs\n\n    self.is_fitted = False\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor","title":"TextPreprocessor","text":"<pre><code>TextPreprocessor(\n    text_col,\n    max_vocab=30000,\n    min_freq=5,\n    maxlen=80,\n    pad_first=True,\n    pad_idx=1,\n    already_processed=False,\n    word_vectors_path=None,\n    n_cpus=None,\n    verbose=1,\n)\n</code></pre> <p>               Bases: <code>BasePreprocessor</code></p> <p>Preprocessor to prepare the <code>deeptext</code> input dataset</p> <p>Parameters:</p> <ul> <li> <code>text_col</code>               (<code>str</code>)           \u2013            <p>column in the input dataframe containing the texts</p> </li> <li> <code>max_vocab</code>               (<code>int</code>, default:                   <code>30000</code> )           \u2013            <p>Maximum number of tokens in the vocabulary</p> </li> <li> <code>min_freq</code>               (<code>int</code>, default:                   <code>5</code> )           \u2013            <p>Minimum frequency for a token to be part of the vocabulary</p> </li> <li> <code>maxlen</code>               (<code>int</code>, default:                   <code>80</code> )           \u2013            <p>Maximum length of the tokenized sequences</p> </li> <li> <code>pad_first</code>               (<code>bool</code>, default:                   <code>True</code> )           \u2013            <p>Indicates whether the padding index will be added at the beginning or the end of the sequences</p> </li> <li> <code>pad_idx</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>padding index. Fastai's Tokenizer leaves 0 for the 'unknown' token.</p> </li> <li> <code>already_processed</code>               (<code>Optional[bool]</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating if the sequence of elements is already processed or prepared. If this is the case, this Preprocessor will simply tokenize and pad the sequence. </p> <pre><code>Param aliases: `not_text`. &lt;br/&gt;\n</code></pre> <p>This parameter is thought for those cases where the input sequences are already fully processed or are directly not text (e.g. IDs)</p> </li> <li> <code>word_vectors_path</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Path to the pretrained word vectors</p> </li> <li> <code>n_cpus</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>number of CPUs to used during the tokenization process</p> </li> <li> <code>verbose</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>Enable verbose output.</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>vocab</code>               (<code>Vocab</code>)           \u2013            <p>an instance of <code>pytorch_widedeep.utils.fastai_transforms.Vocab</code></p> </li> <li> <code>embedding_matrix</code>               (<code>ndarray</code>)           \u2013            <p>Array with the pretrained embeddings</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import pandas as pd\n&gt;&gt;&gt; from pytorch_widedeep.preprocessing import TextPreprocessor\n&gt;&gt;&gt; df_train = pd.DataFrame({'text_column': [\"life is like a box of chocolates\",\n... \"You never know what you're gonna get\"]})\n&gt;&gt;&gt; text_preprocessor = TextPreprocessor(text_col='text_column', max_vocab=25, min_freq=1, maxlen=10)\n&gt;&gt;&gt; text_preprocessor.fit_transform(df_train)\nThe vocabulary contains 24 tokens\narray([[ 1,  1,  1,  1, 10, 11, 12, 13, 14, 15],\n       [ 5,  9, 16, 17, 18,  9, 19, 20, 21, 22]], dtype=int32)\n&gt;&gt;&gt; df_te = pd.DataFrame({'text_column': ['you never know what is in the box']})\n&gt;&gt;&gt; text_preprocessor.transform(df_te)\narray([[ 1,  1,  9, 16, 17, 18, 11,  0,  0, 13]], dtype=int32)\n</code></pre> Source code in <code>pytorch_widedeep/preprocessing/text_preprocessor.py</code> <pre><code>@alias(\"already_processed\", [\"not_text\"])\ndef __init__(\n    self,\n    text_col: str,\n    max_vocab: int = 30000,\n    min_freq: int = 5,\n    maxlen: int = 80,\n    pad_first: bool = True,\n    pad_idx: int = 1,\n    already_processed: Optional[bool] = False,\n    word_vectors_path: Optional[str] = None,\n    n_cpus: Optional[int] = None,\n    verbose: int = 1,\n):\n    super(TextPreprocessor, self).__init__()\n\n    self.text_col = text_col\n    self.max_vocab = max_vocab\n    self.min_freq = min_freq\n    self.maxlen = maxlen\n    self.pad_first = pad_first\n    self.pad_idx = pad_idx\n    self.already_processed = already_processed\n    self.word_vectors_path = word_vectors_path\n    self.verbose = verbose\n    self.n_cpus = n_cpus if n_cpus is not None else os.cpu_count()\n\n    self.is_fitted = False\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor.fit","title":"fit","text":"<pre><code>fit(df)\n</code></pre> <p>Builds the vocabulary</p> <p>Parameters:</p> <ul> <li> <code>df</code>               (<code>DataFrame</code>)           \u2013            <p>Input pandas dataframe</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>TextPreprocessor</code>           \u2013            <p><code>TextPreprocessor</code> fitted object</p> </li> </ul> Source code in <code>pytorch_widedeep/preprocessing/text_preprocessor.py</code> <pre><code>def fit(self, df: pd.DataFrame) -&gt; BasePreprocessor:\n    \"\"\"Builds the vocabulary\n\n    Parameters\n    ----------\n    df: pd.DataFrame\n        Input pandas dataframe\n\n    Returns\n    -------\n    TextPreprocessor\n        `TextPreprocessor` fitted object\n    \"\"\"\n    texts = self._read_texts(df)\n\n    tokens = get_texts(texts, self.already_processed, self.n_cpus)\n\n    self.vocab: TVocab = Vocab(\n        max_vocab=self.max_vocab,\n        min_freq=self.min_freq,\n        pad_idx=self.pad_idx,\n    ).fit(\n        tokens,\n    )\n\n    if self.verbose:\n        print(\"The vocabulary contains {} tokens\".format(len(self.vocab.stoi)))\n    if self.word_vectors_path is not None:\n        self.embedding_matrix = build_embeddings_matrix(\n            self.vocab, self.word_vectors_path, self.min_freq\n        )\n\n    self.is_fitted = True\n\n    return self\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor.transform","title":"transform","text":"<pre><code>transform(df)\n</code></pre> <p>Returns the padded, 'numericalised' sequences</p> <p>Parameters:</p> <ul> <li> <code>df</code>               (<code>DataFrame</code>)           \u2013            <p>Input pandas dataframe</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>ndarray</code>           \u2013            <p>Padded, 'numericalised' sequences</p> </li> </ul> Source code in <code>pytorch_widedeep/preprocessing/text_preprocessor.py</code> <pre><code>def transform(self, df: pd.DataFrame) -&gt; np.ndarray:\n    \"\"\"Returns the padded, _'numericalised'_ sequences\n\n    Parameters\n    ----------\n    df: pd.DataFrame\n        Input pandas dataframe\n\n    Returns\n    -------\n    np.ndarray\n        Padded, _'numericalised'_ sequences\n    \"\"\"\n    check_is_fitted(self, attributes=[\"vocab\"])\n    texts = self._read_texts(df)\n    tokens = get_texts(texts, self.already_processed, self.n_cpus)\n    return self._pad_sequences(tokens)\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor.transform_sample","title":"transform_sample","text":"<pre><code>transform_sample(text)\n</code></pre> <p>Returns the padded, 'numericalised' sequence</p> <p>Parameters:</p> <ul> <li> <code>text</code>               (<code>str</code>)           \u2013            <p>text to be tokenized and padded</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>ndarray</code>           \u2013            <p>Padded, 'numericalised' sequence</p> </li> </ul> Source code in <code>pytorch_widedeep/preprocessing/text_preprocessor.py</code> <pre><code>def transform_sample(self, text: str) -&gt; np.ndarray:\n    \"\"\"Returns the padded, _'numericalised'_ sequence\n\n    Parameters\n    ----------\n    text: str\n        text to be tokenized and padded\n\n    Returns\n    -------\n    np.ndarray\n        Padded, _'numericalised'_ sequence\n    \"\"\"\n    check_is_fitted(self, attributes=[\"vocab\"])\n    tokens = get_texts([text], self.already_processed, self.n_cpus)\n    return self._pad_sequences(tokens)[0]\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor.fit_transform","title":"fit_transform","text":"<pre><code>fit_transform(df)\n</code></pre> <p>Combines <code>fit</code> and <code>transform</code></p> <p>Parameters:</p> <ul> <li> <code>df</code>               (<code>DataFrame</code>)           \u2013            <p>Input pandas dataframe</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>ndarray</code>           \u2013            <p>Padded, 'numericalised' sequences</p> </li> </ul> Source code in <code>pytorch_widedeep/preprocessing/text_preprocessor.py</code> <pre><code>def fit_transform(self, df: pd.DataFrame) -&gt; np.ndarray:\n    \"\"\"Combines `fit` and `transform`\n\n    Parameters\n    ----------\n    df: pd.DataFrame\n        Input pandas dataframe\n\n    Returns\n    -------\n    np.ndarray\n        Padded, _'numericalised'_ sequences\n    \"\"\"\n    return self.fit(df).transform(df)\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor.inverse_transform","title":"inverse_transform","text":"<pre><code>inverse_transform(padded_seq)\n</code></pre> <p>Returns the original text plus the added 'special' tokens</p> <p>Parameters:</p> <ul> <li> <code>padded_seq</code>               (<code>ndarray</code>)           \u2013            <p>array with the output of the <code>transform</code> method</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>DataFrame</code>           \u2013            <p>Pandas dataframe with the original text plus the added 'special' tokens</p> </li> </ul> Source code in <code>pytorch_widedeep/preprocessing/text_preprocessor.py</code> <pre><code>def inverse_transform(self, padded_seq: np.ndarray) -&gt; pd.DataFrame:\n    \"\"\"Returns the original text plus the added 'special' tokens\n\n    Parameters\n    ----------\n    padded_seq: np.ndarray\n        array with the output of the `transform` method\n\n    Returns\n    -------\n    pd.DataFrame\n        Pandas dataframe with the original text plus the added 'special' tokens\n    \"\"\"\n    texts = [self.vocab.inverse_transform(num) for num in padded_seq]\n    return pd.DataFrame({self.text_col: texts})\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor","title":"HFPreprocessor","text":"<pre><code>HFPreprocessor(\n    model_name,\n    *,\n    use_fast_tokenizer=False,\n    text_col=None,\n    root_dir=None,\n    num_workers=None,\n    preprocessing_rules=None,\n    tokenizer_params=None,\n    encode_params=None,\n    **kwargs\n)\n</code></pre> <p>               Bases: <code>BasePreprocessor</code></p> <p>Text processor to prepare the <code>deeptext</code> input dataset that is a wrapper around HuggingFace's tokenizers.</p> <p>Following the main phylosophy of the <code>pytorch-widedeep</code> library, this class is designed to be as flexible as possible. Therefore, it is coded so that the user can use it as one would use any HuggingFace tokenizers, or following the API call 'protocol' of the rest of the library.</p> <p>Parameters:</p> <ul> <li> <code>model_name</code>               (<code>str</code>)           \u2013            <p>The model name from the transformers library e.g. 'bert-base-uncased'. Currently supported models are those from the families: BERT, RoBERTa, DistilBERT, ALBERT and ELECTRA.</p> </li> <li> <code>use_fast_tokenizer</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Whether to use the fast tokenizer from HuggingFace or not</p> </li> <li> <code>text_col</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>The column in the input dataframe containing the text data. If this tokenizer is used via the <code>fit</code> and <code>transform</code> methods, this argument is mandatory. If the tokenizer is used via the <code>encode</code> method, this argument is not needed since the input text is passed directly to the <code>encode</code> method.</p> </li> <li> <code>num_workers</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>Number of workers to use when preprocessing the text data. If not None, and <code>use_fast_tokenizer</code> is False, the text data will be preprocessed in parallel using the number of workers specified. If <code>use_fast_tokenizer</code> is True, this argument is ignored.</p> </li> <li> <code>preprocessing_rules</code>               (<code>Optional[List[Callable[[str], str]]]</code>, default:                   <code>None</code> )           \u2013            <p>A list of functions to be applied to the text data before encoding. This can be useful to clean the text data before encoding. For example, removing html tags, special characters, etc.</p> </li> <li> <code>tokenizer_params</code>               (<code>Optional[Dict[str, Any]]</code>, default:                   <code>None</code> )           \u2013            <p>Additional parameters to be passed to the HuggingFace's <code>PreTrainedTokenizer</code>. Parameters to the <code>PreTrainedTokenizer</code> can also be passed via the <code>**kwargs</code> argument</p> </li> <li> <code>encode_params</code>               (<code>Optional[Dict[str, Any]]</code>, default:                   <code>None</code> )           \u2013            <p>Additional parameters to be passed to the <code>batch_encode_plus</code> method of the HuggingFace's <code>PreTrainedTokenizer</code>. If the <code>fit</code> and <code>transform</code> methods are used, the <code>encode_params</code> dict parameter is mandatory. If the <code>encode</code> method is used, this parameter is not needed since the input text is passed directly to the <code>encode</code> method.</p> </li> <li> <code>**kwargs</code>           \u2013            <p>Additional kwargs to be passed to the model, in particular to the <code>PreTrainedTokenizer</code> class.</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>is_fitted</code>               (<code>bool</code>)           \u2013            <p>Boolean indicating if the preprocessor has been fitted. This is a HuggingFacea tokenizer, so it is always considered fitted and this attribute is manually set to True internally. This parameter exists for consistency with the rest of the library and because is needed for some functionality in the library.</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import pandas as pd\n&gt;&gt;&gt; from pytorch_widedeep.preprocessing import HFPreprocessor\n&gt;&gt;&gt; df = pd.DataFrame({\"text\": [\"this is the first text\", \"this is the second text\"]})\n&gt;&gt;&gt; hf_processor_1 = HFPreprocessor(model_name=\"bert-base-uncased\", text_col=\"text\")\n&gt;&gt;&gt; X_text_1 = hf_processor_1.fit_transform(df)\n&gt;&gt;&gt; texts = [\"this is a new text\", \"this is another text\"]\n&gt;&gt;&gt; hf_processor_2 = HFPreprocessor(model_name=\"bert-base-uncased\")\n&gt;&gt;&gt; X_text_2 = hf_processor_2.encode(texts, max_length=10, padding=\"max_length\", truncation=True)\n</code></pre> Source code in <code>pytorch_widedeep/preprocessing/hf_preprocessor.py</code> <pre><code>def __init__(\n    self,\n    model_name: str,\n    *,\n    use_fast_tokenizer: bool = False,\n    text_col: Optional[str] = None,\n    root_dir: Optional[str] = None,\n    num_workers: Optional[int] = None,\n    preprocessing_rules: Optional[List[Callable[[str], str]]] = None,\n    tokenizer_params: Optional[Dict[str, Any]] = None,\n    encode_params: Optional[Dict[str, Any]] = None,\n    **kwargs,\n):\n    self.model_name = model_name\n    self.use_fast_tokenizer = use_fast_tokenizer\n    self.text_col = text_col\n    self.root_dir = root_dir\n    self.num_workers = num_workers\n    self.preprocessing_rules = preprocessing_rules\n    self.tokenizer_params = tokenizer_params if tokenizer_params is not None else {}\n    self.encode_params = encode_params if encode_params is not None else {}\n\n    self._multiprocessing = (\n        num_workers is not None and num_workers &gt; 1 and not use_fast_tokenizer\n    )\n\n    if kwargs:\n        self.tokenizer_params.update(kwargs)\n\n    self.tokenizer = get_tokenizer(\n        model_name=self.model_name,\n        use_fast_tokenizer=self.use_fast_tokenizer,\n        **self.tokenizer_params,\n    )\n\n    # A HuggingFace tokenizer is already trained, since we need this\n    # attribute elsewhere in the library, we simply set it to True\n    self.is_fitted = True\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.encode","title":"encode","text":"<pre><code>encode(texts, **kwargs)\n</code></pre> <p>Encodes a list of texts. The method is a wrapper around the <code>batch_encode_plus</code> method of the HuggingFace's tokenizer.</p> <p>if 'use_fast_tokenizer' is True, the method will use the <code>batch_encode_plus</code></p> <p>Parameters:</p> <ul> <li> <code>texts</code>               (<code>List[str]</code>)           \u2013            <p>List of texts to be encoded</p> </li> <li> <code>**kwargs</code>           \u2013            <p>Additional parameters to be passed to the <code>batch_encode_plus</code> method of the HuggingFace's tokenizer. If the 'encode_params' dict was passed when instantiating the class, that dictionaly will be updated with the kwargs passed here.</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>array</code>           \u2013            <p>The encoded texts</p> </li> </ul> Source code in <code>pytorch_widedeep/preprocessing/hf_preprocessor.py</code> <pre><code>def encode(self, texts: List[str], **kwargs) -&gt; npt.NDArray[np.int64]:\n    \"\"\"\n    Encodes a list of texts. The method is a wrapper around the\n    `batch_encode_plus` method of the HuggingFace's tokenizer.\n\n    if 'use_fast_tokenizer' is True, the method will use the `batch_encode_plus`\n\n    Parameters\n    ----------\n    texts: List[str]\n        List of texts to be encoded\n    **kwargs\n        Additional parameters to be passed to the `batch_encode_plus` method\n        of the HuggingFace's tokenizer. If the 'encode_params' dict was passed\n        when instantiating the class, that dictionaly will be updated with\n        the kwargs passed here.\n\n    Returns\n    -------\n    np.array\n        The encoded texts\n    \"\"\"\n    if kwargs:\n        self.encode_params.update(kwargs)\n\n    if self.preprocessing_rules:\n        if self._multiprocessing:\n            texts = self._process_text_parallel(texts)\n        else:\n            texts = [self._preprocess_text(text) for text in texts]\n\n    if self._multiprocessing:\n        input_ids = self._encode_paralell(texts, **self.encode_params)\n    else:\n        encoded_texts = self.tokenizer.batch_encode_plus(\n            texts,\n            **self.encode_params,\n        )\n        input_ids = encoded_texts.get(\"input_ids\")\n\n    self.is_fitted = True\n\n    try:\n        output = np.array(input_ids)\n    except ValueError:\n        warnings.warn(\n            \"Padding and Truncating parameters were not passed and all input arrays \"\n            \"do not have the same shape. Padding to the longest sequence. \"\n            \"Padding will be done with the index of the pad token for the model\",\n            UserWarning,\n        )\n        max_len = max([len(ids) for ids in input_ids])\n        output = np.array(\n            [\n                np.pad(ids, (self.tokenizer.pad_token_id, max_len - len(ids)))\n                for ids in input_ids\n            ]\n        )\n\n    return output\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.decode","title":"decode","text":"<pre><code>decode(input_ids, skip_special_tokens)\n</code></pre> <p>Decodes a list of input_ids. The method is a wrapper around the <code>convert_ids_to_tokens</code> and <code>convert_tokens_to_string</code> methods of the HuggingFace's tokenizer.</p> <p>Parameters:</p> <ul> <li> <code>input_ids</code>               (<code>NDArray[int64]</code>)           \u2013            <p>The input_ids to be decoded</p> </li> <li> <code>skip_special_tokens</code>               (<code>bool</code>)           \u2013            <p>Whether to skip the special tokens or not</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>List[str]</code>           \u2013            <p>The decoded texts</p> </li> </ul> Source code in <code>pytorch_widedeep/preprocessing/hf_preprocessor.py</code> <pre><code>def decode(\n    self, input_ids: npt.NDArray[np.int64], skip_special_tokens: bool\n) -&gt; List[str]:\n    \"\"\"\n    Decodes a list of input_ids. The method is a wrapper around the\n    `convert_ids_to_tokens` and `convert_tokens_to_string` methods of the\n    HuggingFace's tokenizer.\n\n    Parameters\n    ----------\n    input_ids: npt.NDArray[np.int64]\n        The input_ids to be decoded\n    skip_special_tokens: bool\n        Whether to skip the special tokens or not\n\n    Returns\n    -------\n    List[str]\n        The decoded texts\n    \"\"\"\n    texts = [\n        self.tokenizer.convert_tokens_to_string(\n            self.tokenizer.convert_ids_to_tokens(input_ids[i], skip_special_tokens)\n        )\n        for i in range(input_ids.shape[0])\n    ]\n    return texts\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.fit","title":"fit","text":"<pre><code>fit(df)\n</code></pre> <p>This method is included for consistency with the rest of the library in general and with the <code>BasePreprocessor</code> in particular. HuggingFace's tokenizers and models are already trained. Therefore, the 'fit' method here does nothing other than checking that the 'text_col' parameter is not <code>None</code>.</p> <p>Parameters:</p> <ul> <li> <code>df</code>               (<code>DataFrame</code>)           \u2013            <p>The dataframe containing the text data in the column specified by the 'text_col' parameter</p> </li> </ul> Source code in <code>pytorch_widedeep/preprocessing/hf_preprocessor.py</code> <pre><code>def fit(self, df: pd.DataFrame) -&gt; \"HFPreprocessor\":\n    \"\"\"\n    This method is included for consistency with the rest of the library\n    in general and with the `BasePreprocessor` in particular. HuggingFace's\n    tokenizers and models are already trained. Therefore, the 'fit' method\n    here does nothing other than checking that the 'text_col' parameter is\n    not `None`.\n\n    Parameters\n    ----------\n    df: pd.DataFrame\n        The dataframe containing the text data in the column specified by\n        the 'text_col' parameter\n    \"\"\"\n    if self.text_col is None:\n        raise ValueError(\n            \"'text_col' is None. Please specify the column name containing the text data\"\n            \" if you want to use the 'fit' method\"\n        )\n    return self\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.transform","title":"transform","text":"<pre><code>transform(df)\n</code></pre> <p>Encodes the text data in the input dataframe. This method simply calls the <code>encode</code> method under the hood. Similar to the <code>fit</code> method, this method is included for consistency with the rest of the library in general and with the <code>BasePreprocessor</code> in particular.</p> <p>Parameters:</p> <ul> <li> <code>df</code>               (<code>DataFrame</code>)           \u2013            <p>The dataframe containing the text data in the column specified by the 'text_col' parameter</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>array</code>           \u2013            <p>The encoded texts</p> </li> </ul> Source code in <code>pytorch_widedeep/preprocessing/hf_preprocessor.py</code> <pre><code>def transform(self, df: pd.DataFrame) -&gt; npt.NDArray[np.int64]:\n    \"\"\"\n    Encodes the text data in the input dataframe. This method simply\n    calls the `encode` method under the hood. Similar to the `fit` method,\n    this method is included for consistency with the rest of the library\n    in general and with the `BasePreprocessor` in particular.\n\n    Parameters\n    ----------\n    df: pd.DataFrame\n        The dataframe containing the text data in the column specified by\n        the 'text_col' parameter\n\n    Returns\n    -------\n    np.array\n        The encoded texts\n    \"\"\"\n    if self.text_col is None:\n        raise ValueError(\n            \"'text_col' is None. Please specify the column name containing the text data\"\n            \" if you want to use the 'fit' method\"\n        )\n\n    texts = self._read_texts(df, self.root_dir)\n\n    return self.encode(texts)\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.transform_sample","title":"transform_sample","text":"<pre><code>transform_sample(text)\n</code></pre> <p>Encodes a single text sample.</p> <p>Parameters:</p> <ul> <li> <code>text</code>               (<code>str</code>)           \u2013            <p>The text sample to be encoded</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>array</code>           \u2013            <p>The encoded text</p> </li> </ul> Source code in <code>pytorch_widedeep/preprocessing/hf_preprocessor.py</code> <pre><code>def transform_sample(self, text: str) -&gt; npt.NDArray[np.int64]:\n    \"\"\"\n    Encodes a single text sample.\n\n    Parameters\n    ----------\n    text: str\n        The text sample to be encoded\n\n    Returns\n    -------\n    np.array\n        The encoded text\n    \"\"\"\n\n    if not self.is_fitted:\n        raise ValueError(\n            \"The `encode` (or `fit`) method must be called before calling `transform_sample`\"\n        )\n    return self.encode([text])[0]\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.fit_transform","title":"fit_transform","text":"<pre><code>fit_transform(df)\n</code></pre> <p>Encodes the text data in the input dataframe.</p> <p>Parameters:</p> <ul> <li> <code>df</code>               (<code>DataFrame</code>)           \u2013            <p>The dataframe containing the text data in the column specified by the 'text_col' parameter</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>array</code>           \u2013            <p>The encoded texts</p> </li> </ul> Source code in <code>pytorch_widedeep/preprocessing/hf_preprocessor.py</code> <pre><code>def fit_transform(self, df: pd.DataFrame) -&gt; npt.NDArray[np.int64]:\n    \"\"\"\n    Encodes the text data in the input dataframe.\n\n    Parameters\n    ----------\n    df: pd.DataFrame\n        The dataframe containing the text data in the column specified by\n        the 'text_col' parameter\n\n    Returns\n    -------\n    np.array\n        The encoded texts\n    \"\"\"\n    return self.fit(df).transform(df)\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.inverse_transform","title":"inverse_transform","text":"<pre><code>inverse_transform(input_ids, skip_special_tokens)\n</code></pre> <p>Decodes a list of input_ids. The method simply calls the <code>decode</code> method under the hood.</p> <p>Parameters:</p> <ul> <li> <code>input_ids</code>               (<code>NDArray[int64]</code>)           \u2013            <p>The input_ids to be decoded</p> </li> <li> <code>skip_special_tokens</code>               (<code>bool</code>)           \u2013            <p>Whether to skip the special tokens or not</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>List[str]</code>           \u2013            <p>The decoded texts</p> </li> </ul> Source code in <code>pytorch_widedeep/preprocessing/hf_preprocessor.py</code> <pre><code>def inverse_transform(\n    self, input_ids: npt.NDArray[np.int64], skip_special_tokens: bool\n) -&gt; List[str]:\n    \"\"\"\n    Decodes a list of input_ids. The method simply calls the `decode` method\n    under the hood.\n\n    Parameters\n    ----------\n    input_ids: npt.NDArray[np.int64]\n        The input_ids to be decoded\n    skip_special_tokens: bool\n        Whether to skip the special tokens or not\n\n    Returns\n    -------\n    List[str]\n        The decoded texts\n    \"\"\"\n    return self.decode(input_ids, skip_special_tokens)\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.image_preprocessor.ImagePreprocessor","title":"ImagePreprocessor","text":"<pre><code>ImagePreprocessor(\n    img_col, img_path, width=224, height=224, verbose=1\n)\n</code></pre> <p>               Bases: <code>BasePreprocessor</code></p> <p>Preprocessor to prepare the <code>deepimage</code> input dataset.</p> <p>The Preprocessing consists simply on resizing according to their aspect ratio</p> <p>Parameters:</p> <ul> <li> <code>img_col</code>               (<code>str</code>)           \u2013            <p>name of the column with the images filenames</p> </li> <li> <code>img_path</code>               (<code>str</code>)           \u2013            <p>path to the dicrectory where the images are stored</p> </li> <li> <code>width</code>               (<code>int</code>, default:                   <code>224</code> )           \u2013            <p>width of the resulting processed image.</p> </li> <li> <code>height</code>               (<code>int</code>, default:                   <code>224</code> )           \u2013            <p>width of the resulting processed image.</p> </li> <li> <code>verbose</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>Enable verbose output.</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>aap</code>               (<code>AspectAwarePreprocessor</code>)           \u2013            <p>an instance of <code>pytorch_widedeep.utils.image_utils.AspectAwarePreprocessor</code></p> </li> <li> <code>spp</code>               (<code>SimplePreprocessor</code>)           \u2013            <p>an instance of <code>pytorch_widedeep.utils.image_utils.SimplePreprocessor</code></p> </li> <li> <code>normalise_metrics</code>               (<code>Dict</code>)           \u2013            <p>Dict containing the normalisation metrics of the image dataset, i.e. mean and std for the R, G and B channels</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import pandas as pd\n&gt;&gt;&gt;\n&gt;&gt;&gt; from pytorch_widedeep.preprocessing import ImagePreprocessor\n&gt;&gt;&gt;\n&gt;&gt;&gt; path_to_image1 = 'tests/test_data_utils/images/galaxy1.png'\n&gt;&gt;&gt; path_to_image2 = 'tests/test_data_utils/images/galaxy2.png'\n&gt;&gt;&gt;\n&gt;&gt;&gt; df_train = pd.DataFrame({'images_column': [path_to_image1]})\n&gt;&gt;&gt; df_test = pd.DataFrame({'images_column': [path_to_image2]})\n&gt;&gt;&gt; img_preprocessor = ImagePreprocessor(img_col='images_column', img_path='.', verbose=0)\n&gt;&gt;&gt; resized_images = img_preprocessor.fit_transform(df_train)\n&gt;&gt;&gt; new_resized_images = img_preprocessor.transform(df_train)\n</code></pre> <p> NOTE: Normalising metrics will only be computed when the <code>fit_transform</code> method is run. Running <code>transform</code> only will not change the computed metrics and running <code>fit</code> only simply instantiates the resizing functions.</p> Source code in <code>pytorch_widedeep/preprocessing/image_preprocessor.py</code> <pre><code>def __init__(\n    self,\n    img_col: str,\n    img_path: str,\n    width: int = 224,\n    height: int = 224,\n    verbose: int = 1,\n):\n    super(ImagePreprocessor, self).__init__()\n\n    self.img_col = img_col\n    self.img_path = img_path\n    self.width = width\n    self.height = height\n    self.verbose = verbose\n\n    self.aap = AspectAwarePreprocessor(self.width, self.height)\n    self.spp = SimplePreprocessor(self.width, self.height)\n\n    self.compute_normalising_computed = False\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.image_preprocessor.ImagePreprocessor.transform","title":"transform","text":"<pre><code>transform(df)\n</code></pre> <p>Resizes the images to the input height and width.</p> <p>Parameters:</p> <ul> <li> <code>df</code>               (<code>DataFrame</code>)           \u2013            <p>Input pandas dataframe with the <code>img_col</code></p> </li> </ul> <p>Returns:</p> <ul> <li> <code>ndarray</code>           \u2013            <p>Resized images to the input height and width</p> </li> </ul> Source code in <code>pytorch_widedeep/preprocessing/image_preprocessor.py</code> <pre><code>def transform(self, df: pd.DataFrame) -&gt; np.ndarray:\n    \"\"\"Resizes the images to the input height and width.\n\n\n    Parameters\n    ----------\n    df: pd.DataFrame\n        Input pandas dataframe with the `img_col`\n\n    Returns\n    -------\n    np.ndarray\n        Resized images to the input height and width\n    \"\"\"\n    image_list = df[self.img_col].tolist()\n    if self.verbose:\n        print(\"Reading Images from {}\".format(self.img_path))\n    imgs = [cv2.imread(\"/\".join([self.img_path, img])) for img in image_list]\n\n    # finding images with different height and width\n    aspect = [(im.shape[0], im.shape[1]) for im in imgs]\n    aspect_r = [a[0] / a[1] for a in aspect]\n    diff_idx = [i for i, r in enumerate(aspect_r) if r != 1.0]\n\n    if self.verbose:\n        print(\"Resizing\")\n    resized_imgs = []\n    for i, img in tqdm(enumerate(imgs), total=len(imgs), disable=self.verbose != 1):\n        if i in diff_idx:\n            resized_imgs.append(self.aap.preprocess(img))\n        else:\n            # if aspect ratio is 1:1, no need for AspectAwarePreprocessor\n            resized_imgs.append(self.spp.preprocess(img))\n\n    if not self.compute_normalising_computed:\n        if self.verbose:\n            print(\"Computing normalisation metrics\")\n        # mean and std deviation will only be computed when the fit method\n        # is called\n        mean_R, mean_G, mean_B = [], [], []\n        std_R, std_G, std_B = [], [], []\n        for rsz_img in resized_imgs:\n            (mean_b, mean_g, mean_r), (std_b, std_g, std_r) = cv2.meanStdDev(\n                rsz_img\n            )\n            mean_R.append(mean_r)\n            mean_G.append(mean_g)\n            mean_B.append(mean_b)\n            std_R.append(std_r)\n            std_G.append(std_g)\n            std_B.append(std_b)\n        self.normalise_metrics = dict(\n            mean={\n                \"R\": np.mean(mean_R) / 255.0,\n                \"G\": np.mean(mean_G) / 255.0,\n                \"B\": np.mean(mean_B) / 255.0,\n            },\n            std={\n                \"R\": np.mean(std_R) / 255.0,\n                \"G\": np.mean(std_G) / 255.0,\n                \"B\": np.mean(std_B) / 255.0,\n            },\n        )\n        self.compute_normalising_computed = True\n    return np.asarray(resized_imgs)\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.image_preprocessor.ImagePreprocessor.fit_transform","title":"fit_transform","text":"<pre><code>fit_transform(df)\n</code></pre> <p>Combines <code>fit</code> and <code>transform</code></p> <p>Parameters:</p> <ul> <li> <code>df</code>               (<code>DataFrame</code>)           \u2013            <p>Input pandas dataframe</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>ndarray</code>           \u2013            <p>Resized images to the input height and width</p> </li> </ul> Source code in <code>pytorch_widedeep/preprocessing/image_preprocessor.py</code> <pre><code>def fit_transform(self, df: pd.DataFrame) -&gt; np.ndarray:\n    \"\"\"Combines `fit` and `transform`\n\n    Parameters\n    ----------\n    df: pd.DataFrame\n        Input pandas dataframe\n\n    Returns\n    -------\n    np.ndarray\n        Resized images to the input height and width\n    \"\"\"\n    return self.fit(df).transform(df)\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#chunked-versions","title":"Chunked versions","text":"<p>Chunked versions of the preprocessors are also available. These are useful when the data is too big to fit in memory. See also the <code>load_from_folder</code> module in the library and the corresponding section here in the documentation.</p> <p>Note that there is not a <code>ChunkImagePreprocessor</code>. This is because the processing of the images will occur inside the <code>ImageFromFolder</code> class in the <code>load_from_folder</code> module.</p>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.wide_preprocessor.ChunkWidePreprocessor","title":"ChunkWidePreprocessor","text":"<pre><code>ChunkWidePreprocessor(\n    wide_cols, n_chunks, crossed_cols=None\n)\n</code></pre> <p>               Bases: <code>WidePreprocessor</code></p> <p>Preprocessor to prepare the wide input dataset</p> <p>This Preprocessor prepares the data for the wide, linear component. This linear model is implemented via an Embedding layer that is connected to the output neuron. <code>ChunkWidePreprocessor</code> numerically encodes all the unique values of all categorical columns <code>wide_cols + crossed_cols</code>. See the Example below.</p> <p>Parameters:</p> <ul> <li> <code>wide_cols</code>               (<code>List[str]</code>)           \u2013            <p>List of strings with the name of the columns that will label encoded and passed through the <code>wide</code> component</p> </li> <li> <code>crossed_cols</code>               (<code>Optional[List[Tuple[str, str]]]</code>, default:                   <code>None</code> )           \u2013            <p>List of Tuples with the name of the columns that will be <code>'crossed'</code> and then label encoded. e.g. [('education', 'occupation'), ...]. For binary features, a cross-product transformation is 1 if and only if the constituent features are all 1, and 0 otherwise.</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>wide_crossed_cols</code>               (<code>List</code>)           \u2013            <p>List with the names of all columns that will be label encoded</p> </li> <li> <code>encoding_dict</code>               (<code>Dict</code>)           \u2013            <p>Dictionary where the keys are the result of pasting <code>colname + '_' + column value</code> and the values are the corresponding mapped integer.</p> </li> <li> <code>inverse_encoding_dict</code>               (<code>Dict</code>)           \u2013            <p>the inverse encoding dictionary</p> </li> <li> <code>wide_dim</code>               (<code>int</code>)           \u2013            <p>Dimension of the wide model (i.e. dim of the linear layer)</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import pandas as pd\n&gt;&gt;&gt; from pytorch_widedeep.preprocessing import ChunkWidePreprocessor\n&gt;&gt;&gt; chunk = pd.DataFrame({'color': ['r', 'b', 'g'], 'size': ['s', 'n', 'l']})\n&gt;&gt;&gt; wide_cols = ['color']\n&gt;&gt;&gt; crossed_cols = [('color', 'size')]\n&gt;&gt;&gt; chunk_wide_preprocessor = ChunkWidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols,\n... n_chunks=1)\n&gt;&gt;&gt; X_wide = chunk_wide_preprocessor.fit_transform(chunk)\n</code></pre> Source code in <code>pytorch_widedeep/preprocessing/wide_preprocessor.py</code> <pre><code>def __init__(\n    self,\n    wide_cols: List[str],\n    n_chunks: int,\n    crossed_cols: Optional[List[Tuple[str, str]]] = None,\n):\n    super(ChunkWidePreprocessor, self).__init__(wide_cols, crossed_cols)\n\n    self.n_chunks = n_chunks\n\n    self.chunk_counter = 0\n\n    self.is_fitted = False\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.wide_preprocessor.ChunkWidePreprocessor.partial_fit","title":"partial_fit","text":"<pre><code>partial_fit(chunk)\n</code></pre> <p>Fits the Preprocessor and creates required attributes</p> <p>Parameters:</p> <ul> <li> <code>chunk</code>               (<code>DataFrame</code>)           \u2013            <p>Input pandas dataframe</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>ChunkWidePreprocessor</code>           \u2013            <p><code>ChunkWidePreprocessor</code> fitted object</p> </li> </ul> Source code in <code>pytorch_widedeep/preprocessing/wide_preprocessor.py</code> <pre><code>def partial_fit(self, chunk: pd.DataFrame) -&gt; \"ChunkWidePreprocessor\":\n    r\"\"\"Fits the Preprocessor and creates required attributes\n\n    Parameters\n    ----------\n    chunk: pd.DataFrame\n        Input pandas dataframe\n\n    Returns\n    -------\n    ChunkWidePreprocessor\n        `ChunkWidePreprocessor` fitted object\n    \"\"\"\n    df_wide = self._prepare_wide(chunk)\n    self.wide_crossed_cols = df_wide.columns.tolist()\n\n    if self.chunk_counter == 0:\n        self.glob_feature_set = set(\n            self._make_global_feature_list(df_wide[self.wide_crossed_cols])\n        )\n    else:\n        self.glob_feature_set.update(\n            self._make_global_feature_list(df_wide[self.wide_crossed_cols])\n        )\n\n    self.chunk_counter += 1\n\n    if self.chunk_counter == self.n_chunks:\n        self.encoding_dict = {v: i + 1 for i, v in enumerate(self.glob_feature_set)}\n        self.wide_dim = len(self.encoding_dict)\n        self.inverse_encoding_dict = {k: v for v, k in self.encoding_dict.items()}\n        self.inverse_encoding_dict[0] = \"unseen\"\n\n        self.is_fitted = True\n\n    return self\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.wide_preprocessor.ChunkWidePreprocessor.fit","title":"fit","text":"<pre><code>fit(df)\n</code></pre> <p>Runs <code>partial_fit</code>. This is just to override the fit method in the base class. This class is not designed or thought to run fit</p> Source code in <code>pytorch_widedeep/preprocessing/wide_preprocessor.py</code> <pre><code>def fit(self, df: pd.DataFrame) -&gt; \"ChunkWidePreprocessor\":\n    \"\"\"\n    Runs `partial_fit`. This is just to override the fit method in the base\n    class. This class is not designed or thought to run fit\n    \"\"\"\n    return self.partial_fit(df)\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.tab_preprocessor.ChunkTabPreprocessor","title":"ChunkTabPreprocessor","text":"<pre><code>ChunkTabPreprocessor(\n    n_chunks,\n    cat_embed_cols=None,\n    continuous_cols=None,\n    cols_and_bins=None,\n    cols_to_scale=None,\n    default_embed_dim=16,\n    with_attention=False,\n    with_cls_token=False,\n    shared_embed=False,\n    verbose=1,\n    *,\n    scale=False,\n    already_standard=None,\n    **kwargs\n)\n</code></pre> <p>               Bases: <code>TabPreprocessor</code></p> <p>Preprocessor to prepare the <code>deeptabular</code> component input dataset</p> <p>Parameters:</p> <ul> <li> <code>n_chunks</code>               (<code>int</code>)           \u2013            <p>Number of chunks that the tabular dataset is divided by.</p> </li> <li> <code>cat_embed_cols</code>               (<code>Optional[Union[List[str], List[Tuple[str, int]]]]</code>, default:                   <code>None</code> )           \u2013            <p>List containing the name of the categorical columns that will be represented by embeddings (e.g. ['education', 'relationship', ...]) or a Tuple with the name and the embedding dimension (e.g.: [ ('education',32), ('relationship',16), ...])</p> </li> <li> <code>continuous_cols</code>               (<code>Optional[List[str]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the name of the continuous cols</p> </li> <li> <code>cols_and_bins</code>               (<code>Optional[Dict[str, List[float]]]</code>, default:                   <code>None</code> )           \u2013            <p>Continuous columns can be turned into categorical via <code>pd.cut</code>. 'cols_and_bins' is dictionary where the keys are the column names to quantize and the values are a list of scalars indicating the bin edges.</p> </li> <li> <code>cols_to_scale</code>               (<code>Optional[Union[List[str], str]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the names of the columns that will be standarised via sklearn's <code>StandardScaler</code></p> </li> <li> <code>default_embed_dim</code>               (<code>int</code>, default:                   <code>16</code> )           \u2013            <p>Dimension for the embeddings if the embed_dim is not provided in the <code>cat_embed_cols</code> parameter and <code>auto_embed_dim</code> is set to <code>False</code>.</p> </li> <li> <code>with_attention</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether the preprocessed data will be passed to an attention-based model (more precisely a model where all embeddings must have the same dimensions). If <code>True</code>, the param <code>cat_embed_cols</code> must just be a list containing just the categorical column names: e.g. ['education', 'relationship', ...]. This is because they will all be  encoded using embeddings of the same dim, which will be specified  later when the model is defined.  Param alias:  <code>for_transformer</code></p> </li> <li> <code>with_cls_token</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating if a <code>'[CLS]'</code> token will be added to the dataset when using attention-based models. The final hidden state corresponding to this token is used as the aggregated representation for classification and regression tasks. If not, the categorical (and continuous embeddings if present) will be concatenated before being passed to the final MLP (if present).</p> </li> <li> <code>shared_embed</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating if the embeddings will be \"shared\" when using attention-based models. The idea behind <code>shared_embed</code> is described in the Appendix A in the TabTransformer paper: 'The goal of having column embedding is to enable the model to distinguish the classes in one column from those in the other columns'. In other words, the idea is to let the model learn which column is embedded at the time. See: <code>pytorch_widedeep.models.transformers._layers.SharedEmbeddings</code>.</p> </li> <li> <code>verbose</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            </li> <li> <code>scale</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p> note: this arg will be removed in upcoming  releases. Please use <code>cols_to_scale</code> instead.  Bool indicating  whether or not to scale/standarise continuous cols. It is important  to emphasize that all the DL models for tabular data in the library  also include the possibility of normalising the input continuous  features via a <code>BatchNorm</code> or a <code>LayerNorm</code>.  Param alias:  <code>scale_cont_cols</code>.</p> </li> <li> <code>already_standard</code>               (<code>Optional[List[str]]</code>, default:                   <code>None</code> )           \u2013            <p> note: this arg will be removed in upcoming  releases. Please use <code>cols_to_scale</code> instead.  List with the  name of the continuous cols that do not need to be  scaled/standarised.</p> </li> </ul> <p>Other Parameters:</p> <ul> <li> <code>**kwargs</code>           \u2013            <p><code>pd.cut</code> and <code>StandardScaler</code> related args</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>embed_dim</code>               (<code>Dict</code>)           \u2013            <p>Dictionary where keys are the embed cols and values are the embedding dimensions. If <code>with_attention</code> is set to <code>True</code> this attribute is not generated during the <code>fit</code> process</p> </li> <li> <code>label_encoder</code>               (<code>LabelEncoder</code>)           \u2013            <p>see <code>pytorch_widedeep.utils.dense_utils.LabelEncder</code></p> </li> <li> <code>cat_embed_input</code>               (<code>List</code>)           \u2013            <p>List of Tuples with the column name, number of individual values for that column and, If <code>with_attention</code> is set to <code>False</code>, the corresponding embeddings dim, e.g. [('education', 16, 10), ('relationship', 6, 8), ...].</p> </li> <li> <code>standardize_cols</code>               (<code>List</code>)           \u2013            <p>List of the columns that will be standarized</p> </li> <li> <code>scaler</code>               (<code>StandardScaler</code>)           \u2013            <p>an instance of <code>sklearn.preprocessing.StandardScaler</code> if 'cols_to_scale' is not None or 'scale' is 'True'</p> </li> <li> <code>column_idx</code>               (<code>Dict</code>)           \u2013            <p>Dictionary where keys are column names and values are column indexes. This is neccesary to slice tensors</p> </li> <li> <code>quantizer</code>               (<code>Quantizer</code>)           \u2013            <p>an instance of <code>Quantizer</code></p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import pandas as pd\n&gt;&gt;&gt; import numpy as np\n&gt;&gt;&gt; from pytorch_widedeep.preprocessing import ChunkTabPreprocessor\n&gt;&gt;&gt; np.random.seed(42)\n&gt;&gt;&gt; chunk_df = pd.DataFrame({'cat_col': np.random.choice(['A', 'B', 'C'], size=8),\n... 'cont_col': np.random.uniform(1, 100, size=8)})\n&gt;&gt;&gt; cat_embed_cols = [('cat_col',4)]\n&gt;&gt;&gt; cont_cols = ['cont_col']\n&gt;&gt;&gt; tab_preprocessor = ChunkTabPreprocessor(\n... n_chunks=1, cat_embed_cols=cat_embed_cols, continuous_cols=cont_cols\n... )\n&gt;&gt;&gt; X_tab = tab_preprocessor.fit_transform(chunk_df)\n&gt;&gt;&gt; tab_preprocessor.cat_embed_cols\n[('cat_col', 4)]\n&gt;&gt;&gt; tab_preprocessor.column_idx\n{'cat_col': 0, 'cont_col': 1}\n</code></pre> Source code in <code>pytorch_widedeep/preprocessing/tab_preprocessor.py</code> <pre><code>@alias(\"with_attention\", [\"for_transformer\"])\n@alias(\"cat_embed_cols\", [\"embed_cols\"])\n@alias(\"scale\", [\"scale_cont_cols\"])\n@alias(\"cols_and_bins\", [\"quantization_setup\"])\ndef __init__(\n    self,\n    n_chunks: int,\n    cat_embed_cols: Optional[Union[List[str], List[Tuple[str, int]]]] = None,\n    continuous_cols: Optional[List[str]] = None,\n    cols_and_bins: Optional[Dict[str, List[float]]] = None,\n    cols_to_scale: Optional[Union[List[str], str]] = None,\n    default_embed_dim: int = 16,\n    with_attention: bool = False,\n    with_cls_token: bool = False,\n    shared_embed: bool = False,\n    verbose: int = 1,\n    *,\n    scale: bool = False,\n    already_standard: Optional[List[str]] = None,\n    **kwargs,\n):\n    super(ChunkTabPreprocessor, self).__init__(\n        cat_embed_cols=cat_embed_cols,\n        continuous_cols=continuous_cols,\n        quantization_setup=None,\n        cols_to_scale=cols_to_scale,\n        auto_embed_dim=False,\n        embedding_rule=\"google\",  # does not matter, irrelevant\n        default_embed_dim=default_embed_dim,\n        with_attention=with_attention,\n        with_cls_token=with_cls_token,\n        shared_embed=shared_embed,\n        verbose=verbose,\n        scale=scale,\n        already_standard=already_standard,\n        **kwargs,\n    )\n\n    self.n_chunks = n_chunks\n    self.chunk_counter = 0\n\n    self.cols_and_bins = cols_and_bins  # type: ignore[assignment]\n    if self.cols_and_bins is not None:\n        self.quantizer = Quantizer(self.cols_and_bins, **self.quant_args)\n\n    self.embed_prepared = False\n    self.continuous_prepared = False\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.text_preprocessor.ChunkTextPreprocessor","title":"ChunkTextPreprocessor","text":"<pre><code>ChunkTextPreprocessor(\n    text_col,\n    n_chunks,\n    root_dir=None,\n    max_vocab=30000,\n    min_freq=5,\n    maxlen=80,\n    pad_first=True,\n    pad_idx=1,\n    already_processed=False,\n    word_vectors_path=None,\n    n_cpus=None,\n    verbose=1,\n)\n</code></pre> <p>               Bases: <code>TextPreprocessor</code></p> <p>Preprocessor to prepare the <code>deeptext</code> input dataset</p> <p>Parameters:</p> <ul> <li> <code>text_col</code>               (<code>str</code>)           \u2013            <p>column in the input dataframe containing either the texts or the filenames where the text documents are stored</p> </li> <li> <code>n_chunks</code>               (<code>int</code>)           \u2013            <p>Number of chunks that the text dataset is divided by.</p> </li> <li> <code>root_dir</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>If 'text_col' contains the filenames with the text documents, this is the path to the directory where those documents are stored.</p> </li> <li> <code>max_vocab</code>               (<code>int</code>, default:                   <code>30000</code> )           \u2013            <p>Maximum number of tokens in the vocabulary</p> </li> <li> <code>min_freq</code>               (<code>int</code>, default:                   <code>5</code> )           \u2013            <p>Minimum frequency for a token to be part of the vocabulary</p> </li> <li> <code>maxlen</code>               (<code>int</code>, default:                   <code>80</code> )           \u2013            <p>Maximum length of the tokenized sequences</p> </li> <li> <code>pad_first</code>               (<code>bool</code>, default:                   <code>True</code> )           \u2013            <p>Indicates whether the padding index will be added at the beginning or the end of the sequences</p> </li> <li> <code>pad_idx</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>padding index. Fastai's Tokenizer leaves 0 for the 'unknown' token.</p> </li> <li> <code>word_vectors_path</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Path to the pretrained word vectors</p> </li> <li> <code>n_cpus</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>number of CPUs to used during the tokenization process</p> </li> <li> <code>verbose</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>Enable verbose output.</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>vocab</code>               (<code>Vocab</code>)           \u2013            <p>an instance of <code>pytorch_widedeep.utils.fastai_transforms.ChunkVocab</code></p> </li> <li> <code>embedding_matrix</code>               (<code>ndarray</code>)           \u2013            <p>Array with the pretrained embeddings if <code>word_vectors_path</code> is not None</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import pandas as pd\n&gt;&gt;&gt; from pytorch_widedeep.preprocessing import ChunkTextPreprocessor\n&gt;&gt;&gt; chunk_df = pd.DataFrame({'text_column': [\"life is like a box of chocolates\",\n... \"You never know what you're gonna get\"]})\n&gt;&gt;&gt; chunk_text_preprocessor = ChunkTextPreprocessor(text_col='text_column', n_chunks=1,\n... max_vocab=25, min_freq=1, maxlen=10, verbose=0, n_cpus=1)\n&gt;&gt;&gt; processed_chunk = chunk_text_preprocessor.fit_transform(chunk_df)\n</code></pre> Source code in <code>pytorch_widedeep/preprocessing/text_preprocessor.py</code> <pre><code>def __init__(\n    self,\n    text_col: str,\n    n_chunks: int,\n    root_dir: Optional[str] = None,\n    max_vocab: int = 30000,\n    min_freq: int = 5,\n    maxlen: int = 80,\n    pad_first: bool = True,\n    pad_idx: int = 1,\n    already_processed: Optional[bool] = False,\n    word_vectors_path: Optional[str] = None,\n    n_cpus: Optional[int] = None,\n    verbose: int = 1,\n):\n    super(ChunkTextPreprocessor, self).__init__(\n        text_col=text_col,\n        max_vocab=max_vocab,\n        min_freq=min_freq,\n        maxlen=maxlen,\n        pad_first=pad_first,\n        pad_idx=pad_idx,\n        already_processed=already_processed,\n        word_vectors_path=word_vectors_path,\n        n_cpus=n_cpus,\n        verbose=verbose,\n    )\n\n    self.n_chunks = n_chunks\n    self.root_dir = root_dir\n\n    self.chunk_counter = 0\n\n    self.is_fitted = False\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.hf_preprocessor.ChunkHFPreprocessor","title":"ChunkHFPreprocessor","text":"<pre><code>ChunkHFPreprocessor(\n    model_name,\n    *,\n    text_col,\n    root_dir=None,\n    use_fast_tokenizer=True,\n    num_workers=None,\n    preprocessing_rules=None,\n    tokenizer_params=None,\n    encode_params=None\n)\n</code></pre> <p>               Bases: <code>HFPreprocessor</code></p> <p>Text processor to prepare the <code>deeptext</code> input dataset that is a wrapper around HuggingFace's tokenizers.</p> <p>Hugginface Tokenizer's are already 'trained'. Therefore, unlike the <code>ChunkTextPreprocessor</code> this is mostly identical to the <code>HFPreprocessor</code> with the only difference that the class needs a 'text_col' parameter to be passed. Also the parameter <code>encode_params</code> is not really optional when using this class. It must be passed containing at least the 'max_length' encoding parameter. This is because we need to ensure that  all sequences have the same length when encoding in chunks.</p> <p>Parameters:</p> <ul> <li> <code>model_name</code>               (<code>str</code>)           \u2013            <p>The model name from the transformers library e.g. 'bert-base-uncased'. Currently supported models are those from the families: BERT, RoBERTa, DistilBERT, ALBERT and ELECTRA.</p> </li> <li> <code>text_col</code>               (<code>str</code>)           \u2013            <p>The column in the input dataframe containing the text data. When using the <code>ChunkHFPreprocessor</code> the <code>text_col</code> parameter is mandatory.</p> </li> <li> <code>root_dir</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>The root directory where the text files are located. This is only needed if the text data is stored in text files. If the text data is stored in a column in the input dataframe, this parameter is not needed.</p> </li> <li> <code>use_fast_tokenizer</code>               (<code>bool</code>, default:                   <code>True</code> )           \u2013            <p>Whether to use the fast tokenizer from HuggingFace or not</p> </li> <li> <code>num_workers</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>Number of workers to use when preprocessing the text data. If not None, and <code>use_fast_tokenizer</code> is False, the text data will be preprocessed in parallel using the number of workers specified. If <code>use_fast_tokenizer</code> is True, this argument is ignored.</p> </li> <li> <code>preprocessing_rules</code>               (<code>Optional[List[Callable[[str], str]]]</code>, default:                   <code>None</code> )           \u2013            <p>A list of functions to be applied to the text data before encoding. This can be useful to clean the text data before encoding. For example, removing html tags, special characters, etc.</p> </li> <li> <code>tokenizer_params</code>               (<code>Optional[Dict[str, Any]]</code>, default:                   <code>None</code> )           \u2013            <p>Additional parameters to be passed to the HuggingFace's <code>PreTrainedTokenizer</code>.</p> </li> <li> <code>encode_params</code>               (<code>Optional[Dict[str, Any]]</code>, default:                   <code>None</code> )           \u2013            <p>Additional parameters to be passed to the <code>batch_encode_plus</code> method of the HuggingFace's <code>PreTrainedTokenizer</code>. In the case of the <code>ChunkHFPreprocessor</code>, this parameter is not really <code>Optional</code>. It must be passed containing at least the 'max_length' encoding parameter</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>is_fitted</code>               (<code>bool</code>)           \u2013            <p>Boolean indicating if the preprocessor has been fitted. This is a HuggingFacea tokenizer, so it is always considered fitted and this attribute is manually set to True internally. This parameter exists for consistency with the rest of the library and because is needed for some functionality in the library.</p> </li> </ul> Source code in <code>pytorch_widedeep/preprocessing/hf_preprocessor.py</code> <pre><code>def __init__(\n    self,\n    model_name: str,\n    *,\n    text_col: str,\n    root_dir: Optional[str] = None,\n    use_fast_tokenizer: bool = True,\n    num_workers: Optional[int] = None,\n    preprocessing_rules: Optional[List[Callable[[str], str]]] = None,\n    tokenizer_params: Optional[Dict[str, Any]] = None,\n    encode_params: Optional[Dict[str, Any]] = None,\n):\n    super().__init__(\n        model_name=model_name,\n        use_fast_tokenizer=use_fast_tokenizer,\n        text_col=text_col,\n        num_workers=num_workers,\n        preprocessing_rules=preprocessing_rules,\n        tokenizer_params=tokenizer_params,\n        encode_params=encode_params,\n    )\n\n    self.root_dir = root_dir\n\n    # when using in chunks encode_params is not really optional. I will\n    # review types in due time\n    if self.encode_params is None:\n        raise ValueError(\n            \"The 'encode_params' dict must be passed to the ChunkHFTokenizer \"\n            \"containing at least the 'max_length' encoding parameter\"\n        )\n\n    if \"padding\" not in self.encode_params or not self.encode_params[\"padding\"]:\n        self.encode_params[\"padding\"] = True\n\n    if (\n        \"truncation\" not in self.encode_params\n        or not self.encode_params[\"truncation\"]\n    ):\n        self.encode_params[\"truncation\"] = True\n</code></pre>"},{"location":"pytorch-widedeep/self_supervised_pretraining.html","title":"Self Supervised Pre-training for tabular data","text":"<p>In this library we have implemented two methods or routines that allow the user to use self-suerpvised pre-training for all tabular models in the library with the exception of the <code>TabPerceiver</code> (this is a particular model and self-supervised pre-training requires some adjustments that will be implemented in future versions). Please see the examples folder in the repo or the examples section in the docs for details on how to use self-supervised pre-training with this library.</p> <p>The two routines implemented are illustrated in the figures below. The first is from TabNet: Attentive Interpretable Tabular Learning. It is a 'standard' encoder-decoder architecture and and is designed here for models that do not use transformer-based architectures (or when the embeddings can all have different dimensions). The second is from SAINT: Improved Neural Networks for Tabular Data via Row Attention and Contrastive Pre-Training, it is based on Contrastive and Denoising learning and is designed for models that use transformer-based architectures (or when the embeddings all need to have the same dimension):</p> <p> </p> <p>Figure 1. Figure 2 in their paper. The caption of the original paper is included in case it is useful.</p> <p> </p> <p>Figure 2. Figure 1 in their paper. The caption of the original paper is included in case it is useful.</p> <p>Note that the self-supervised pre-trainers described below focus, of course, on the self-supervised pre-training phase, i.e. the left side in Figure 1 and the upper part in Figure 2. When combined with the <code>Trainer</code> described earlier in the documenation, one can reproduce the full process illustrated in the figures above.</p> <p>Also Note that it is beyond the scope of this docs to explain in detail these routines. In addition, to fully utilise the self-supervised trainers implemented in this library a minimum understanding of the processes as described in the papers is required. Therefore, we strongly encourage the users to have a look to the papers.</p>"},{"location":"pytorch-widedeep/self_supervised_pretraining.html#pytorch_widedeep.self_supervised_training.EncoderDecoderTrainer","title":"EncoderDecoderTrainer","text":"<pre><code>EncoderDecoderTrainer(\n    encoder,\n    decoder=None,\n    masked_prob=0.2,\n    optimizer=None,\n    lr_scheduler=None,\n    callbacks=None,\n    verbose=1,\n    seed=1,\n    **kwargs\n)\n</code></pre> <p>               Bases: <code>BaseEncoderDecoderTrainer</code></p> <p>This class implements an Encoder-Decoder self-supervised 'routine' inspired by TabNet: Attentive Interpretable Tabular Learning. See Figure 1 above.</p> <p>Parameters:</p> <ul> <li> <code>encoder</code>               (<code>ModelWithoutAttention</code>)           \u2013            <p>An instance of a <code>TabMlp</code>, <code>TabResNet</code> or <code>TabNet</code> model</p> </li> <li> <code>decoder</code>               (<code>Optional[DecoderWithoutAttention]</code>, default:                   <code>None</code> )           \u2013            <p>An instance of  a <code>TabMlpDecoder</code>, <code>TabResNetDecoder</code> or <code>TabNetDecoder</code> model. if <code>None</code> the decoder will be automatically built as a 'simetric' model to the Encoder</p> </li> <li> <code>masked_prob</code>               (<code>float</code>, default:                   <code>0.2</code> )           \u2013            <p>Indicates the fraction of elements in the embedding tensor that will be masked and hence used for reconstruction</p> </li> <li> <code>optimizer</code>               (<code>Optional[Optimizer]</code>, default:                   <code>None</code> )           \u2013            <p>An instance of Pytorch's <code>Optimizer</code> object (e.g. <code>torch.optim.Adam ()</code>). if no optimizer is passed it will default to <code>AdamW</code>.</p> </li> <li> <code>lr_scheduler</code>               (<code>Optional[LRScheduler]</code>, default:                   <code>None</code> )           \u2013            <p>An instance of Pytorch's <code>LRScheduler</code> object (e.g <code>torch.optim.lr_scheduler.StepLR(opt, step_size=5)</code>).</p> </li> <li> <code>callbacks</code>               (<code>Optional[List[Callback]]</code>, default:                   <code>None</code> )           \u2013            <p>List with <code>Callback</code> objects. The three callbacks available in <code>pytorch-widedeep</code> are: <code>LRHistory</code>, <code>ModelCheckpoint</code> and <code>EarlyStopping</code>. This can also be a custom callback. See <code>pytorch_widedeep.callbacks.Callback</code> or the Examples folder in the repo.</p> </li> <li> <code>verbose</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>Setting it to 0 will print nothing during training.</p> </li> <li> <code>seed</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>Random seed to be used internally for train_test_split</p> </li> </ul> <p>Other Parameters:</p> <ul> <li> <code>**kwargs</code>           \u2013            <p>Other infrequently used arguments that can also be passed as kwargs are:</p> <ul> <li> <p>device: <code>str</code>     string indicating the device. One of 'cpu' or 'gpu'</p> </li> <li> <p>num_workers: <code>int</code>     number of workers to be used internally by the data loaders</p> </li> <li> <p>reducelronplateau_criterion: <code>str</code>     This sets the criterion that will be used by the lr scheduler to     take a step: One of 'loss' or 'metric'. The ReduceLROnPlateau     learning rate is a bit particular.</p> </li> </ul> </li> </ul> Source code in <code>pytorch_widedeep/self_supervised_training/encoder_decoder_trainer.py</code> <pre><code>def __init__(\n    self,\n    encoder: ModelWithoutAttention,\n    decoder: Optional[DecoderWithoutAttention] = None,\n    masked_prob: float = 0.2,\n    optimizer: Optional[Optimizer] = None,\n    lr_scheduler: Optional[LRScheduler] = None,\n    callbacks: Optional[List[Callback]] = None,\n    verbose: int = 1,\n    seed: int = 1,\n    **kwargs,\n):\n    super().__init__(\n        encoder=encoder,\n        decoder=decoder,\n        masked_prob=masked_prob,\n        optimizer=optimizer,\n        lr_scheduler=lr_scheduler,\n        callbacks=callbacks,\n        verbose=verbose,\n        seed=seed,\n        **kwargs,\n    )\n</code></pre>"},{"location":"pytorch-widedeep/self_supervised_pretraining.html#pytorch_widedeep.self_supervised_training.EncoderDecoderTrainer.pretrain","title":"pretrain","text":"<pre><code>pretrain(\n    X_tab,\n    X_tab_val=None,\n    val_split=None,\n    validation_freq=1,\n    n_epochs=1,\n    batch_size=32,\n)\n</code></pre> <p>Pretrain method. Can also be called using <code>.fit(&lt;same_args&gt;)</code></p> <p>Parameters:</p> <ul> <li> <code>X_tab</code>               (<code>ndarray</code>)           \u2013            <p>tabular dataset</p> </li> <li> <code>X_tab_val</code>               (<code>Optional[ndarray]</code>, default:                   <code>None</code> )           \u2013            <p>validation data</p> </li> <li> <code>val_split</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>An alterative to passing the validation set is to use a train/val split fraction via <code>val_split</code></p> </li> <li> <code>validation_freq</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>epochs validation frequency</p> </li> <li> <code>n_epochs</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>number of epochs</p> </li> <li> <code>batch_size</code>               (<code>int</code>, default:                   <code>32</code> )           \u2013            <p>batch size</p> </li> </ul> Source code in <code>pytorch_widedeep/self_supervised_training/encoder_decoder_trainer.py</code> <pre><code>def pretrain(\n    self,\n    X_tab: np.ndarray,\n    X_tab_val: Optional[np.ndarray] = None,\n    val_split: Optional[float] = None,\n    validation_freq: int = 1,\n    n_epochs: int = 1,\n    batch_size: int = 32,\n):\n    r\"\"\"Pretrain method. Can also be called using `.fit(&lt;same_args&gt;)`\n\n    Parameters\n    ----------\n    X_tab: np.ndarray,\n        tabular dataset\n    X_tab_val: np.ndarray, Optional, default = None\n        validation data\n    val_split: float, Optional. default=None\n        An alterative to passing the validation set is to use a train/val\n        split fraction via `val_split`\n    validation_freq: int, default=1\n        epochs validation frequency\n    n_epochs: int, default=1\n        number of epochs\n    batch_size: int, default=32\n        batch size\n    \"\"\"\n\n    self.batch_size = batch_size\n\n    train_set, eval_set = self._train_eval_split(X_tab, X_tab_val, val_split)\n    train_loader = DataLoader(\n        dataset=train_set, batch_size=batch_size, num_workers=self.num_workers\n    )\n    train_steps = len(train_loader)\n    if eval_set is not None:\n        eval_loader = DataLoader(\n            dataset=eval_set,\n            batch_size=batch_size,\n            num_workers=self.num_workers,\n            shuffle=False,\n        )\n        eval_steps = len(eval_loader)\n\n    self.callback_container.on_train_begin(\n        {\n            \"batch_size\": batch_size,\n            \"train_steps\": train_steps,\n            \"n_epochs\": n_epochs,\n        }\n    )\n    for epoch in range(n_epochs):\n        epoch_logs: Dict[str, float] = {}\n        self.callback_container.on_epoch_begin(epoch, logs=epoch_logs)\n\n        self.train_running_loss = 0.0\n        with trange(train_steps, disable=self.verbose != 1) as t:\n            for batch_idx, X in zip(t, train_loader):\n                t.set_description(\"epoch %i\" % (epoch + 1))\n                train_loss = self._train_step(X[0], batch_idx)\n                self.callback_container.on_batch_end(batch=batch_idx)\n                print_loss_and_metric(t, train_loss)\n\n        epoch_logs = save_epoch_logs(epoch_logs, train_loss, None, \"train\")\n\n        on_epoch_end_metric = None\n        if eval_set is not None and epoch % validation_freq == (\n            validation_freq - 1\n        ):\n            self.callback_container.on_eval_begin()\n            self.valid_running_loss = 0.0\n            with trange(eval_steps, disable=self.verbose != 1) as v:\n                for batch_idx, X in zip(v, eval_loader):\n                    v.set_description(\"valid\")\n                    val_loss = self._eval_step(X[0], batch_idx)\n                    print_loss_and_metric(v, val_loss)\n            epoch_logs = save_epoch_logs(epoch_logs, val_loss, None, \"val\")\n            on_epoch_end_metric = val_loss\n        else:\n            if self.reducelronplateau:\n                raise NotImplementedError(\n                    \"ReduceLROnPlateau scheduler can be used only with validation data.\"\n                )\n\n        self.callback_container.on_epoch_end(epoch, epoch_logs, on_epoch_end_metric)\n\n        if self.early_stop:\n            self.callback_container.on_train_end(epoch_logs)\n            break\n\n    self.callback_container.on_train_end(epoch_logs)\n    self._restore_best_weights()\n    self.ed_model.train()\n</code></pre>"},{"location":"pytorch-widedeep/self_supervised_pretraining.html#pytorch_widedeep.self_supervised_training.ContrastiveDenoisingTrainer","title":"ContrastiveDenoisingTrainer","text":"<pre><code>ContrastiveDenoisingTrainer(\n    model,\n    preprocessor,\n    optimizer=None,\n    lr_scheduler=None,\n    callbacks=None,\n    loss_type=\"both\",\n    projection_head1_dims=None,\n    projection_head2_dims=None,\n    projection_heads_activation=\"relu\",\n    cat_mlp_type=\"multiple\",\n    cont_mlp_type=\"multiple\",\n    denoise_mlps_activation=\"relu\",\n    verbose=1,\n    seed=1,\n    **kwargs\n)\n</code></pre> <p>               Bases: <code>BaseContrastiveDenoisingTrainer</code></p> <p>This class trains a Contrastive, Denoising Self Supervised 'routine' that is based on the one described in SAINT: Improved Neural Networks for Tabular Data via Row Attention and Contrastive Pre-Training, their Figure 1.</p> <p>Parameters:</p> <ul> <li> <code>model</code>               (<code>ModelWithAttention</code>)           \u2013            <p>An instance of a <code>TabTransformer</code>, <code>SAINT</code>, <code>FTTransformer</code>, <code>TabFastFormer</code>, <code>TabPerceiver</code>, <code>ContextAttentionMLP</code> and <code>SelfAttentionMLP</code>.</p> </li> <li> <code>preprocessor</code>               (<code>TabPreprocessor</code>)           \u2013            <p>A fitted <code>TabPreprocessor</code> object. See <code>pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor</code></p> </li> <li> <code>optimizer</code>               (<code>Optional[Optimizer]</code>, default:                   <code>None</code> )           \u2013            <p>An instance of Pytorch's <code>Optimizer</code> object (e.g. <code>torch.optim.Adam ()</code>). if no optimizer is passed it will default to <code>AdamW</code>.</p> </li> <li> <code>lr_scheduler</code>               (<code>Optional[LRScheduler]</code>, default:                   <code>None</code> )           \u2013            <p>An instance of Pytorch's <code>LRScheduler</code> object (e.g <code>torch.optim.lr_scheduler.StepLR(opt, step_size=5)</code>).</p> </li> <li> <code>callbacks</code>               (<code>Optional[List[Callback]]</code>, default:                   <code>None</code> )           \u2013            <p>List with <code>Callback</code> objects. The three callbacks available in <code>pytorch-widedeep</code> are: <code>LRHistory</code>, <code>ModelCheckpoint</code> and <code>EarlyStopping</code>. This can also be a custom callback. See <code>pytorch_widedeep.callbacks.Callback</code> or the Examples folder in the repo.</p> </li> <li> <code>loss_type</code>               (<code>Literal[contrastive, denoising, both]</code>, default:                   <code>'both'</code> )           \u2013            <p>One of 'contrastive', 'denoising' or 'both'. See SAINT: Improved Neural Networks for Tabular Data via Row Attention and Contrastive Pre-Training, their figure (1) and their equation (5).</p> </li> <li> <code>projection_head1_dims</code>               (<code>Optional[List[int]]</code>, default:                   <code>None</code> )           \u2013            <p>The projection heads are simply MLPs. This parameter is a list of integers with the dimensions of the MLP hidden layers. See the paper for details. Note that setting up this parameter requires some knowledge of the architecture one is using. For example, if we are representing the features with embeddings of dim 32 (i.e. the so called dimension of the model is 32), then the first dimension of the projection head must be 32 (e.g. [32, 16])</p> </li> <li> <code>projection_head2_dims</code>               (<code>Optional[List[int]]</code>, default:                   <code>None</code> )           \u2013            <p>Same as 'projection_head1_dims' for the second head</p> </li> <li> <code>projection_heads_activation</code>               (<code>str</code>, default:                   <code>'relu'</code> )           \u2013            <p>Activation function for the projection heads</p> </li> <li> <code>cat_mlp_type</code>               (<code>Literal[single, multiple]</code>, default:                   <code>'multiple'</code> )           \u2013            <p>If 'denoising' loss is used, one can choose two types of 'stacked' MLPs to process the output from the transformer-based encoder that receives 'corrupted' (cut-mixed and mixed-up) features. These are 'single' or 'multiple'. The former approach will apply a single MLP to all the categorical features while the latter will use one MLP per categorical feature</p> </li> <li> <code>cont_mlp_type</code>               (<code>Literal[single, multiple]</code>, default:                   <code>'multiple'</code> )           \u2013            <p>Same as 'cat_mlp_type' but for the continuous features</p> </li> <li> <code>denoise_mlps_activation</code>               (<code>str</code>, default:                   <code>'relu'</code> )           \u2013            <p>activation function for the so called 'denoising mlps'.</p> </li> <li> <code>verbose</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>Setting it to 0 will print nothing during training.</p> </li> <li> <code>seed</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>Random seed to be used internally for train_test_split</p> </li> </ul> <p>Other Parameters:</p> <ul> <li> <code>**kwargs</code>           \u2013            <p>Other infrequently used arguments that can also be passed as kwargs are:</p> <ul> <li> <p>device: <code>str</code>     string indicating the device. One of 'cpu' or 'gpu'</p> </li> <li> <p>num_workers: <code>int</code>     number of workers to be used internally by the data loaders</p> </li> <li> <p>reducelronplateau_criterion: <code>str</code>     This sets the criterion that will be used by the lr scheduler to     take a step: One of 'loss' or 'metric'. The ReduceLROnPlateau     learning rate is a bit particular.</p> </li> </ul> </li> </ul> Source code in <code>pytorch_widedeep/self_supervised_training/contrastive_denoising_trainer.py</code> <pre><code>def __init__(\n    self,\n    model: ModelWithAttention,\n    preprocessor: TabPreprocessor,\n    optimizer: Optional[Optimizer] = None,\n    lr_scheduler: Optional[LRScheduler] = None,\n    callbacks: Optional[List[Callback]] = None,\n    loss_type: Literal[\"contrastive\", \"denoising\", \"both\"] = \"both\",\n    projection_head1_dims: Optional[List[int]] = None,\n    projection_head2_dims: Optional[List[int]] = None,\n    projection_heads_activation: str = \"relu\",\n    cat_mlp_type: Literal[\"single\", \"multiple\"] = \"multiple\",\n    cont_mlp_type: Literal[\"single\", \"multiple\"] = \"multiple\",\n    denoise_mlps_activation: str = \"relu\",\n    verbose: int = 1,\n    seed: int = 1,\n    **kwargs,\n):\n    super().__init__(\n        model=model,\n        preprocessor=preprocessor,\n        loss_type=loss_type,\n        optimizer=optimizer,\n        lr_scheduler=lr_scheduler,\n        callbacks=callbacks,\n        projection_head1_dims=projection_head1_dims,\n        projection_head2_dims=projection_head2_dims,\n        projection_heads_activation=projection_heads_activation,\n        cat_mlp_type=cat_mlp_type,\n        cont_mlp_type=cont_mlp_type,\n        denoise_mlps_activation=denoise_mlps_activation,\n        verbose=verbose,\n        seed=seed,\n        **kwargs,\n    )\n</code></pre>"},{"location":"pytorch-widedeep/self_supervised_pretraining.html#pytorch_widedeep.self_supervised_training.ContrastiveDenoisingTrainer.pretrain","title":"pretrain","text":"<pre><code>pretrain(\n    X_tab,\n    X_tab_val=None,\n    val_split=None,\n    validation_freq=1,\n    n_epochs=1,\n    batch_size=32,\n)\n</code></pre> <p>Pretrain method. Can also be called using <code>.fit(&lt;same_args&gt;)</code></p> <p>Parameters:</p> <ul> <li> <code>X_tab</code>               (<code>ndarray</code>)           \u2013            <p>tabular dataset</p> </li> <li> <code>X_tab_val</code>               (<code>Optional[ndarray]</code>, default:                   <code>None</code> )           \u2013            <p>validation data. Note that, although it is possible to use contrastive-denoising training with a validation set, such set must include feature values that are all seen in the training set in the case of the categorical columns. This is because the values of the columns themselves will be used as targets when computing the loss. Therefore, if a new category is present in the validation set that was not seen in training this will effectively be like trying to predict a new, never seen category (and Pytorch will throw an error)</p> </li> <li> <code>val_split</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>An alterative to passing the validation set is to use a train/val split fraction via <code>val_split</code></p> </li> <li> <code>validation_freq</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>epochs validation frequency</p> </li> <li> <code>n_epochs</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>number of epochs</p> </li> <li> <code>batch_size</code>               (<code>int</code>, default:                   <code>32</code> )           \u2013            <p>batch size</p> </li> </ul> Source code in <code>pytorch_widedeep/self_supervised_training/contrastive_denoising_trainer.py</code> <pre><code>def pretrain(\n    self,\n    X_tab: np.ndarray,\n    X_tab_val: Optional[np.ndarray] = None,\n    val_split: Optional[float] = None,\n    validation_freq: int = 1,\n    n_epochs: int = 1,\n    batch_size: int = 32,\n):\n    r\"\"\"Pretrain method. Can also be called using `.fit(&lt;same_args&gt;)`\n\n    Parameters\n    ----------\n    X_tab: np.ndarray,\n        tabular dataset\n    X_tab_val: np.ndarray, Optional, default = None\n        validation data. Note that, although it is possible to use\n        contrastive-denoising training with a validation set, such set\n        must include feature values that are _all_ seen in the training\n        set in the case of the categorical columns. This is because the\n        values of the columns themselves will be used as targets when\n        computing the loss. Therefore, if a new category is present in\n        the validation set that was not seen in training this will\n        effectively be like trying to predict a new, never seen category\n        (and Pytorch will throw an error)\n    val_split: float, Optional. default=None\n        An alterative to passing the validation set is to use a train/val\n        split fraction via `val_split`\n    validation_freq: int, default=1\n        epochs validation frequency\n    n_epochs: int, default=1\n        number of epochs\n    batch_size: int, default=32\n        batch size\n    \"\"\"\n\n    self.batch_size = batch_size\n\n    train_set, eval_set = self._train_eval_split(X_tab, X_tab_val, val_split)\n    train_loader = DataLoader(\n        dataset=train_set, batch_size=batch_size, num_workers=self.num_workers\n    )\n    train_steps = len(train_loader)\n    if eval_set is not None:\n        eval_loader = DataLoader(\n            dataset=eval_set,\n            batch_size=batch_size,\n            num_workers=self.num_workers,\n            shuffle=False,\n        )\n        eval_steps = len(eval_loader)\n\n    self.callback_container.on_train_begin(\n        {\n            \"batch_size\": batch_size,\n            \"train_steps\": train_steps,\n            \"n_epochs\": n_epochs,\n        }\n    )\n    for epoch in range(n_epochs):\n        epoch_logs: Dict[str, float] = {}\n        self.callback_container.on_epoch_begin(epoch, logs=epoch_logs)\n\n        self.train_running_loss = 0.0\n        with trange(train_steps, disable=self.verbose != 1) as t:\n            for batch_idx, X in zip(t, train_loader):\n                t.set_description(\"epoch %i\" % (epoch + 1))\n                train_loss = self._train_step(X[0], batch_idx)\n                self.callback_container.on_batch_end(batch=batch_idx)\n                print_loss_and_metric(t, train_loss)\n\n        epoch_logs = save_epoch_logs(epoch_logs, train_loss, None, \"train\")\n\n        on_epoch_end_metric = None\n        if eval_set is not None and epoch % validation_freq == (\n            validation_freq - 1\n        ):\n            self.callback_container.on_eval_begin()\n            self.valid_running_loss = 0.0\n            with trange(eval_steps, disable=self.verbose != 1) as v:\n                for batch_idx, X in zip(v, eval_loader):\n                    v.set_description(\"valid\")\n                    val_loss = self._eval_step(X[0], batch_idx)\n                    print_loss_and_metric(v, val_loss)\n            epoch_logs = save_epoch_logs(epoch_logs, val_loss, None, \"val\")\n            on_epoch_end_metric = val_loss\n        else:\n            if self.reducelronplateau:\n                raise NotImplementedError(\n                    \"ReduceLROnPlateau scheduler can be used only with validation data.\"\n                )\n\n        self.callback_container.on_epoch_end(epoch, epoch_logs, on_epoch_end_metric)\n\n        if self.early_stop:\n            self.callback_container.on_train_end(epoch_logs)\n            break\n\n    self.callback_container.on_train_end(epoch_logs)\n    self._restore_best_weights()\n    self.cd_model.train()\n</code></pre>"},{"location":"pytorch-widedeep/tab2vec.html","title":"Tab2Vec","text":""},{"location":"pytorch-widedeep/tab2vec.html#pytorch_widedeep.tab2vec.Tab2Vec","title":"Tab2Vec","text":"<pre><code>Tab2Vec(\n    tab_preprocessor,\n    model,\n    return_dataframe=False,\n    verbose=False,\n)\n</code></pre> <p>Class to transform an input dataframe into vectorized form.</p> <p>This class will take an input dataframe in the form of the dataframe used for training, and it will turn it into a vectorised form based on the processing applied by the model to the categorical and continuous columns.</p> <p> NOTE: Currently this class is only implemented  for the deeptabular component. Therefore, if the input dataframe has a  text column or a column with the path to images, these will be ignored.  We will be adding these functionalities in future versions</p> <p>Parameters:</p> <ul> <li> <code>model</code>               (<code>Union[WideDeep, BayesianWide, BayesianTabMlp]</code>)           \u2013            <p><code>WideDeep</code>, <code>BayesianWide</code> or <code>BayesianTabMlp</code> model. Must be trained.</p> </li> <li> <code>tab_preprocessor</code>               (<code>TabPreprocessor</code>)           \u2013            <p><code>TabPreprocessor</code> object. Must be fitted.</p> </li> <li> <code>return_dataframe</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating of the returned object(s) will be array(s) or pandas dataframe(s)</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>vectorizer</code>               (<code>Module</code>)           \u2013            <p>Torch module with the categorical and continuous encoding process</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import string\n&gt;&gt;&gt; from random import choices\n&gt;&gt;&gt; import numpy as np\n&gt;&gt;&gt; import pandas as pd\n&gt;&gt;&gt; from pytorch_widedeep import Tab2Vec\n&gt;&gt;&gt; from pytorch_widedeep.models import TabMlp, WideDeep\n&gt;&gt;&gt; from pytorch_widedeep.preprocessing import TabPreprocessor\n&gt;&gt;&gt;\n&gt;&gt;&gt; colnames = list(string.ascii_lowercase)[:4]\n&gt;&gt;&gt; cat_col1_vals = [\"a\", \"b\", \"c\"]\n&gt;&gt;&gt; cat_col2_vals = [\"d\", \"e\", \"f\"]\n&gt;&gt;&gt;\n&gt;&gt;&gt; # Create the toy input dataframe and a toy dataframe to be vectorised\n&gt;&gt;&gt; cat_inp = [np.array(choices(c, k=5)) for c in [cat_col1_vals, cat_col2_vals]]\n&gt;&gt;&gt; cont_inp = [np.round(np.random.rand(5), 2) for _ in range(2)]\n&gt;&gt;&gt; df_inp = pd.DataFrame(np.vstack(cat_inp + cont_inp).transpose(), columns=colnames)\n&gt;&gt;&gt; cat_t2v = [np.array(choices(c, k=5)) for c in [cat_col1_vals, cat_col2_vals]]\n&gt;&gt;&gt; cont_t2v = [np.round(np.random.rand(5), 2) for _ in range(2)]\n&gt;&gt;&gt; df_t2v = pd.DataFrame(np.vstack(cat_t2v + cont_t2v).transpose(), columns=colnames)\n&gt;&gt;&gt;\n&gt;&gt;&gt; # fit the TabPreprocessor\n&gt;&gt;&gt; embed_cols = [(\"a\", 2), (\"b\", 4)]\n&gt;&gt;&gt; cont_cols = [\"c\", \"d\"]\n&gt;&gt;&gt; tab_preprocessor = TabPreprocessor(cat_embed_cols=embed_cols, continuous_cols=cont_cols)\n&gt;&gt;&gt; X_tab = tab_preprocessor.fit_transform(df_inp)\n&gt;&gt;&gt;\n&gt;&gt;&gt; # define the model (and let's assume we train it)\n&gt;&gt;&gt; tabmlp = TabMlp(\n... column_idx=tab_preprocessor.column_idx,\n... cat_embed_input=tab_preprocessor.cat_embed_input,\n... continuous_cols=tab_preprocessor.continuous_cols,\n... mlp_hidden_dims=[8, 4])\n&gt;&gt;&gt; model = WideDeep(deeptabular=tabmlp)\n&gt;&gt;&gt; # ...train the model...\n&gt;&gt;&gt;\n&gt;&gt;&gt; # vectorise the dataframe\n&gt;&gt;&gt; t2v = Tab2Vec(tab_preprocessor, model)\n&gt;&gt;&gt; X_vec = t2v.transform(df_t2v)\n</code></pre> Source code in <code>pytorch_widedeep/tab2vec.py</code> <pre><code>def __init__(\n    self,\n    tab_preprocessor: TabPreprocessor,\n    model: Union[WideDeep, BayesianWide, BayesianTabMlp],\n    return_dataframe: bool = False,\n    verbose: bool = False,\n):\n    super(Tab2Vec, self).__init__()\n\n    self._check_inputs(tab_preprocessor, model, verbose)\n\n    self.tab_preprocessor = tab_preprocessor\n    self.return_dataframe = return_dataframe\n    self.verbose = verbose\n\n    self.vectorizer = self._set_vectorizer(model)\n\n    self._set_dim_attributes(tab_preprocessor, model)\n</code></pre>"},{"location":"pytorch-widedeep/tab2vec.html#pytorch_widedeep.tab2vec.Tab2Vec.fit","title":"fit","text":"<pre><code>fit(df, target_col=None)\n</code></pre> <p>This is an empty method i.e. Returns the unchanged object itself. Is only included for consistency in case <code>Tab2Vec</code> is used as part of a Pipeline</p> <p>Parameters:</p> <ul> <li> <code>df</code>               (<code>DataFrame</code>)           \u2013            <p>DataFrame to be vectorised, i.e. the categorical and continuous columns will be encoded based on the processing applied within the model</p> </li> <li> <code>target_col</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Column name of the target_col variable. If <code>None</code> only the array of predictors will be returned</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>Tab2Vec</code>           \u2013            </li> </ul> Source code in <code>pytorch_widedeep/tab2vec.py</code> <pre><code>def fit(self, df: pd.DataFrame, target_col: Optional[str] = None) -&gt; \"Tab2Vec\":\n    r\"\"\"This is an empty method i.e. Returns the unchanged object itself. Is\n    only included for consistency in case `Tab2Vec` is used as part of a\n    Pipeline\n\n    Parameters\n    ----------\n    df: pd.DataFrame\n        DataFrame to be vectorised, i.e. the categorical and continuous\n        columns will be encoded based on the processing applied within\n        the model\n    target_col: str, Optional\n        Column name of the target_col variable. If `None` only the array of\n        predictors will be returned\n\n    Returns\n    -------\n    Tab2Vec\n    \"\"\"\n\n    return self\n</code></pre>"},{"location":"pytorch-widedeep/tab2vec.html#pytorch_widedeep.tab2vec.Tab2Vec.transform","title":"transform","text":"<pre><code>transform(df, target_col=None)\n</code></pre> <p>Transforms the input dataframe into vectorized form. If a target column name is passed the target values will be returned separately in their corresponding type (np.ndarray or pd.DataFrame)</p> <p>Parameters:</p> <ul> <li> <code>df</code>               (<code>DataFrame</code>)           \u2013            <p>DataFrame to be vectorised, i.e. the categorical and continuous columns will be encoded based on the processing applied within the model</p> </li> <li> <code>target_col</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Column name of the target_col variable. If <code>None</code> only the array of predictors will be returned</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>Union[np.ndarray, Tuple[np.ndarray, np.ndarray], pd.DataFrame, Tuple[pd.DataFrame, pd.Series]</code>           \u2013            <p>Returns eiter a numpy array with the vectorised values, or a Tuple of numpy arrays with the vectorised values and the target. The same applies to dataframes in case we choose to set <code>return_dataframe = True</code></p> </li> </ul> Source code in <code>pytorch_widedeep/tab2vec.py</code> <pre><code>def transform(\n    self,\n    df: pd.DataFrame,\n    target_col: Optional[str] = None,\n) -&gt; Union[\n    np.ndarray,\n    Tuple[np.ndarray, np.ndarray],\n    pd.DataFrame,\n    Tuple[pd.DataFrame, pd.Series],\n]:\n    r\"\"\"Transforms the input dataframe into vectorized form. If a target\n    column name is passed the target values will be returned separately\n    in their corresponding type (np.ndarray or pd.DataFrame)\n\n    Parameters\n    ----------\n    df: pd.DataFrame\n        DataFrame to be vectorised, i.e. the categorical and continuous\n        columns will be encoded based on the processing applied within\n        the model\n    target_col: str, Optional\n        Column name of the target_col variable. If `None` only the array of\n        predictors will be returned\n\n    Returns\n    -------\n    Union[np.ndarray, Tuple[np.ndarray, np.ndarray], pd.DataFrame, Tuple[pd.DataFrame, pd.Series]\n        Returns eiter a numpy array with the vectorised values, or a Tuple\n        of numpy arrays with the vectorised values and the target. The\n        same applies to dataframes in case we choose to set\n        `return_dataframe = True`\n    \"\"\"\n\n    X_tab = self.tab_preprocessor.transform(df)\n    X = torch.from_numpy(X_tab.astype(\"float\")).to(device)\n\n    with torch.no_grad():\n        if self.is_tab_transformer:\n            x_vec, x_cont_not_embed = self.vectorizer(X)\n        else:\n            x_vec = self.vectorizer(X)\n            x_cont_not_embed = None\n\n    if self.tab_preprocessor.with_cls_token:\n        x_vec = x_vec[:, 1:, :]\n\n    if self.tab_preprocessor.with_attention:\n        x_vec = einops.rearrange(x_vec, \"s c e -&gt; s (c e)\")\n\n    if x_cont_not_embed is not None:\n        x_vec = torch.cat([x_vec, x_cont_not_embed], 1).detach().cpu().numpy()\n    else:\n        x_vec = x_vec.detach().cpu().numpy()\n\n    if self.return_dataframe:\n        new_colnames = self._new_colnames()\n        if target_col:\n            return pd.DataFrame(data=x_vec, columns=new_colnames), df[[target_col]]\n        else:\n            return pd.DataFrame(data=x_vec, columns=new_colnames)\n    else:\n        if target_col:\n            return x_vec, df[target_col].values\n        else:\n            return x_vec\n</code></pre>"},{"location":"pytorch-widedeep/tab2vec.html#pytorch_widedeep.tab2vec.Tab2Vec.fit_transform","title":"fit_transform","text":"<pre><code>fit_transform(df, target_col=None)\n</code></pre> <p>Combines <code>fit</code> and <code>transform</code></p> Source code in <code>pytorch_widedeep/tab2vec.py</code> <pre><code>def fit_transform(\n    self, df: pd.DataFrame, target_col: Optional[str] = None\n) -&gt; Union[\n    np.ndarray,\n    Tuple[np.ndarray, np.ndarray],\n    pd.DataFrame,\n    Tuple[pd.DataFrame, pd.Series],\n]:\n    r\"\"\"Combines `fit` and `transform`\"\"\"\n    return self.fit(df, target_col).transform(df, target_col)\n</code></pre>"},{"location":"pytorch-widedeep/the_rec_module.html","title":"The <code>rec</code> module","text":"<p>This module contains models are that specifically designed for recommendation systems. While the rest of the models can be accessed from the <code>pytorch_widedeep.models</code> module, models in this module need to be specifically imported from the <code>rec</code> module, e.g.:</p> <pre><code>from pytorch_widedeep.models.rec import DeepFactorizationMachine\n</code></pre> <p>The list of models here is not meant to be exhaustive, but it includes some common architectures such as factorization machines, field aware factorization machines or extreme factorization machines. More models will be added in the future.</p>"},{"location":"pytorch-widedeep/the_rec_module.html#pytorch_widedeep.models.rec.deepfm.DeepFactorizationMachine","title":"DeepFactorizationMachine","text":"<pre><code>DeepFactorizationMachine(\n    *,\n    column_idx,\n    num_factors,\n    reduce_sum=True,\n    cat_embed_input=None,\n    cat_embed_dropout=None,\n    use_cat_bias=None,\n    cat_embed_activation=None,\n    continuous_cols=None,\n    cont_norm_layer=None,\n    embed_continuous=None,\n    embed_continuous_method=\"piecewise\",\n    cont_embed_dropout=None,\n    cont_embed_activation=None,\n    quantization_setup=None,\n    n_frequencies=None,\n    sigma=None,\n    share_last_layer=None,\n    full_embed_dropout=None,\n    mlp_hidden_dims=None,\n    mlp_activation=None,\n    mlp_dropout=None,\n    mlp_batchnorm=None,\n    mlp_batchnorm_last=None,\n    mlp_linear_first=None\n)\n</code></pre> <p>               Bases: <code>BaseTabularModelWithAttention</code></p> <p>Deep Factorization Machine (DeepFM) for recommendation systems, which is an adaptation of 'Factorization Machines' by Steffen Rendle. Presented in 'DeepFM: A Factorization-Machine based Neural Network for CTR Prediction' by Huifeng Guo, Ruiming Tang, Yunming Yey, Zhenguo Li, Xiuqiang He. 2017.</p> <p>The implementation in this library takes advantage of all the functionalities available to encode categorical and continuous features. The model can be used with only the factorization machine</p> <p>Note that this class implements only the 'Deep' component of the model described in the paper. The linear component is not implemented 'internally' and, if one wants to include it, it can be easily added using the 'wide' (aka linear) component available in this library. See the examples in the examples folder.</p> <p>Parameters:</p> <ul> <li> <code>column_idx</code>               (<code>Dict[str, int]</code>)           \u2013            <p>Dictionary mapping column names to their corresponding index.</p> </li> <li> <code>num_factors</code>               (<code>int</code>)           \u2013            <p>Number of factors for the factorization machine.</p> </li> <li> <code>reduce_sum</code>               (<code>bool</code>, default:                   <code>True</code> )           \u2013            <p>Whether to reduce the sum in the factorization machine output.</p> </li> <li> <code>cat_embed_input</code>               (<code>Optional[List[Tuple[str, int]]]</code>, default:                   <code>None</code> )           \u2013            <p>List of tuples with categorical column names and number of unique values.</p> </li> <li> <code>cat_embed_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Categorical embeddings dropout. If <code>None</code>, it will default to 0.</p> </li> <li> <code>use_cat_bias</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating if bias will be used for the categorical embeddings. If <code>None</code>, it will default to 'False'.</p> </li> <li> <code>cat_embed_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the categorical embeddings, if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> </li> <li> <code>continuous_cols</code>               (<code>Optional[List[str]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the name of the numeric (aka continuous) columns</p> </li> <li> <code>cont_norm_layer</code>               (<code>Optional[Literal['batchnorm', 'layernorm']]</code>, default:                   <code>None</code> )           \u2013            <p>Type of normalization layer applied to the continuous features. Options are: 'layernorm' and 'batchnorm'. if <code>None</code>, no normalization layer will be used.</p> </li> <li> <code>embed_continuous_method</code>               (<code>Optional[Literal['piecewise', 'periodic']]</code>, default:                   <code>\"piecewise\"</code> )           \u2013            <p>Method to use to embed the continuous features. Options are: 'standard', 'periodic' or 'piecewise'. The 'standard' embedding method is based on the FT-Transformer implementation presented in the paper: Revisiting Deep Learning Models for Tabular Data. The 'periodic' and_'piecewise'_ methods were presented in the paper: On Embeddings for Numerical Features in Tabular Deep Learning. Please, read the papers for details.</p> </li> <li> <code>cont_embed_dim</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>Size of the continuous embeddings. If the continuous columns are embedded, <code>cont_embed_dim</code> must be passed.</p> </li> <li> <code>cont_embed_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p> </li> <li> <code>cont_embed_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the continuous embeddings if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported. If <code>None</code>, no activation function will be applied.</p> </li> <li> <code>quantization_setup</code>               (<code>Optional[Dict[str, List[float]]]</code>, default:                   <code>None</code> )           \u2013            <p>This parameter is used when the 'piecewise' method is used to embed the continuous cols. It is a dict where keys are the name of the continuous columns and values are lists with the boundaries for the quantization of the continuous_cols. See the examples for details. If If the 'piecewise' method is used, this parameter is required.</p> </li> <li> <code>n_frequencies</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>This is the so called 'k' in their paper On Embeddings for Numerical Features in Tabular Deep Learning, and is the number of 'frequencies' that will be used to represent each continuous column. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>sigma</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>This is the sigma parameter in the paper mentioned when describing the previous parameters and it is used to initialise the 'frequency weights'. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>share_last_layer</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>This parameter is not present in the before mentioned paper but it is implemented in the official repo. If <code>True</code> the linear layer that turns the frequencies into embeddings will be shared across the continuous columns. If <code>False</code> a different linear layer will be used for each continuous column. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>full_embed_dropout</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>If <code>True</code>, the full embedding corresponding to a column will be masked out/dropout. If <code>None</code>, it will default to <code>False</code>.</p> </li> <li> <code>mlp_hidden_dims</code>               (<code>Optional[List[int]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the number of neurons per dense layer in the mlp.</p> </li> <li> <code>mlp_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the dense layers of the MLP. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> </li> <li> <code>mlp_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>float or List of floats with the dropout between the dense layers. e.g: [0.5,0.5]</p> </li> <li> <code>mlp_batchnorm</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating whether or not batch normalization will be applied to the dense layers</p> </li> <li> <code>mlp_batchnorm_last</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating whether or not batch normalization will be applied to the last of the dense layers</p> </li> <li> <code>mlp_linear_first</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code></p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>mlp</code>               (<code>MLP</code>)           \u2013            <p>MLP component of the model if the mlp_hidden_dims parameter is not None. If None, the model will only return the output of the factorization machine.</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; from typing import Dict, List, Tuple\n&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from torch import Tensor\n&gt;&gt;&gt; from pytorch_widedeep.models.rec import DeepFactorizationMachine\n&gt;&gt;&gt; X = torch.randint(0, 10, (16, 2))\n&gt;&gt;&gt; column_idx: Dict[str, int] = {\"col1\": 0, \"col2\": 1}\n&gt;&gt;&gt; cat_embed_input: List[Tuple[str, int]] = [(\"col1\", 10), (\"col2\", 10)]\n&gt;&gt;&gt; fm = DeepFactorizationMachine(\n...     column_idx=column_idx,\n...     num_factors=8,\n...     cat_embed_input=cat_embed_input,\n...     mlp_hidden_dims=[16, 8]\n... )\n&gt;&gt;&gt; out = fm(X)\n</code></pre> Source code in <code>pytorch_widedeep/models/rec/deepfm.py</code> <pre><code>def __init__(\n    self,\n    *,\n    column_idx: Dict[str, int],\n    num_factors: int,\n    reduce_sum: bool = True,\n    cat_embed_input: Optional[List[Tuple[str, int]]] = None,\n    cat_embed_dropout: Optional[float] = None,\n    use_cat_bias: Optional[bool] = None,\n    cat_embed_activation: Optional[str] = None,\n    continuous_cols: Optional[List[str]] = None,\n    cont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\n    embed_continuous: Optional[bool] = None,\n    embed_continuous_method: Optional[\n        Literal[\"piecewise\", \"periodic\"]\n    ] = \"piecewise\",\n    cont_embed_dropout: Optional[float] = None,\n    cont_embed_activation: Optional[str] = None,\n    quantization_setup: Optional[Dict[str, List[float]]] = None,\n    n_frequencies: Optional[int] = None,\n    sigma: Optional[float] = None,\n    share_last_layer: Optional[bool] = None,\n    full_embed_dropout: Optional[bool] = None,\n    mlp_hidden_dims: Optional[List[int]] = None,\n    mlp_activation: Optional[str] = None,\n    mlp_dropout: Optional[float] = None,\n    mlp_batchnorm: Optional[bool] = None,\n    mlp_batchnorm_last: Optional[bool] = None,\n    mlp_linear_first: Optional[bool] = None,\n):\n    super(DeepFactorizationMachine, self).__init__(\n        column_idx=column_idx,\n        input_dim=num_factors,\n        cat_embed_input=cat_embed_input,\n        cat_embed_dropout=cat_embed_dropout,\n        use_cat_bias=use_cat_bias,\n        cat_embed_activation=cat_embed_activation,\n        shared_embed=False,\n        add_shared_embed=None,\n        frac_shared_embed=None,\n        continuous_cols=continuous_cols,\n        cont_norm_layer=cont_norm_layer,\n        embed_continuous_method=embed_continuous_method,\n        cont_embed_dropout=cont_embed_dropout,\n        cont_embed_activation=cont_embed_activation,\n        quantization_setup=quantization_setup,\n        n_frequencies=n_frequencies,\n        sigma=sigma,\n        share_last_layer=share_last_layer,\n        full_embed_dropout=full_embed_dropout,\n    )\n\n    self.reduce_sum = reduce_sum\n\n    self.mlp_hidden_dims = mlp_hidden_dims\n    self.mlp_activation = mlp_activation\n    self.mlp_dropout = mlp_dropout\n    self.mlp_batchnorm = mlp_batchnorm\n    self.mlp_batchnorm_last = mlp_batchnorm_last\n    self.mlp_linear_first = mlp_linear_first\n\n    if self.mlp_hidden_dims is not None:\n\n        if self.mlp_hidden_dims[-1] != self.input_dim:\n            d_hidden = (\n                [self.input_dim * len(self.column_idx)]\n                + self.mlp_hidden_dims\n                + [self.input_dim]\n            )\n        else:\n            d_hidden = [\n                self.input_dim * len(self.column_idx)\n            ] + self.mlp_hidden_dims\n\n        self.mlp = MLP(\n            d_hidden=d_hidden,\n            activation=(\n                \"relu\" if self.mlp_activation is None else self.mlp_activation\n            ),\n            dropout=0.0 if self.mlp_dropout is None else self.mlp_dropout,\n            batchnorm=False if self.mlp_batchnorm is None else self.mlp_batchnorm,\n            batchnorm_last=(\n                False\n                if self.mlp_batchnorm_last is None\n                else self.mlp_batchnorm_last\n            ),\n            linear_first=(\n                False if self.mlp_linear_first is None else self.mlp_linear_first\n            ),\n        )\n    else:\n        self.mlp = None\n</code></pre>"},{"location":"pytorch-widedeep/the_rec_module.html#pytorch_widedeep.models.rec.deepffm.DeepFieldAwareFactorizationMachine","title":"DeepFieldAwareFactorizationMachine","text":"<pre><code>DeepFieldAwareFactorizationMachine(\n    *,\n    column_idx,\n    num_factors,\n    cat_embed_input,\n    reduce_sum=True,\n    cat_embed_dropout=None,\n    use_cat_bias=None,\n    cat_embed_activation=None,\n    mlp_hidden_dims=None,\n    mlp_activation=None,\n    mlp_dropout=None,\n    mlp_batchnorm=None,\n    mlp_batchnorm_last=None,\n    mlp_linear_first=None\n)\n</code></pre> <p>               Bases: <code>BaseTabularModelWithAttention</code></p> <p>Deep Field Aware Factorization Machine (DeepFFM) for recommendation systems. Adaptation of the paper 'Field-aware Factorization Machines in a Real-world Online Advertising System', Juan et al. 2017.</p> <p>This class implements only the 'Deep' component of the model described in the paper. The linear component is not implemented 'internally' and, if one wants to include it, it can be easily added using the 'wide'/linear component in this library. See the examples in the examples folder.</p> <p>Note that in this case, only categorical features are accepted. This is because the embeddings of each feature will be learned using all other features. Therefore these embeddings have to be all of the same nature. This does not occur if we mix categorical and continuous features.</p> <p>Parameters:</p> <ul> <li> <code>column_idx</code>               (<code>Dict[str, int]</code>)           \u2013            <p>Dictionary mapping column names to their corresponding index.</p> </li> <li> <code>num_factors</code>               (<code>int</code>)           \u2013            <p>Number of factors for the factorization machine.</p> </li> <li> <code>reduce_sum</code>               (<code>bool</code>, default:                   <code>True</code> )           \u2013            <p>Whether to reduce the sum in the factorization machine output.</p> </li> <li> <code>cat_embed_input</code>               (<code>Optional[List[Tuple[str, int]]]</code>, default:                   <code>None</code> )           \u2013            <p>List of tuples with categorical column names and number of unique values.</p> </li> <li> <code>cat_embed_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Categorical embeddings dropout. If <code>None</code>, it will default to 0.</p> </li> <li> <code>use_cat_bias</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating if bias will be used for the categorical embeddings. If <code>None</code>, it will default to 'False'.</p> </li> <li> <code>cat_embed_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the categorical embeddings, if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> </li> <li> <code>mlp_hidden_dims</code>               (<code>Optional[List[int]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the number of neurons per dense layer in the mlp.</p> </li> <li> <code>mlp_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the dense layers of the MLP. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> </li> <li> <code>mlp_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>float or List of floats with the dropout between the dense layers. e.g: [0.5,0.5]</p> </li> <li> <code>mlp_batchnorm</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating whether or not batch normalization will be applied to the dense layers</p> </li> <li> <code>mlp_batchnorm_last</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating whether or not batch normalization will be applied to the last of the dense layers</p> </li> <li> <code>mlp_linear_first</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code></p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>n_features</code>               (<code>int</code>)           \u2013            <p>Number of unique features/columns</p> </li> <li> <code>n_tokens</code>               (<code>int</code>)           \u2013            <p>Number of unique values (tokens) in the full dataset (corpus)</p> </li> <li> <code>encoders</code>               (<code>ModuleList</code>)           \u2013            <p>List of <code>BaseTabularModelWithAttention</code> instances. One per categorical column</p> </li> <li> <code>mlp</code>               (<code>Module</code>)           \u2013            <p>Multi-layer perceptron. If <code>None</code> the output will be the output of the factorization machine (i.e. the sum of the interactions)</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from torch import Tensor\n&gt;&gt;&gt; from typing import Dict, List, Tuple\n&gt;&gt;&gt; from pytorch_widedeep.models.rec import DeepFieldAwareFactorizationMachine\n&gt;&gt;&gt; X = torch.randint(0, 10, (16, 2))\n&gt;&gt;&gt; column_idx: Dict[str, int] = {\"col1\": 0, \"col2\": 1}\n&gt;&gt;&gt; cat_embed_input: List[Tuple[str, int]] = [(\"col1\", 10), (\"col2\", 10)]\n&gt;&gt;&gt; ffm = DeepFieldAwareFactorizationMachine(\n...     column_idx=column_idx,\n...     num_factors=4,\n...     cat_embed_input=cat_embed_input,\n...     mlp_hidden_dims=[16, 8]\n... )\n&gt;&gt;&gt; output = ffm(X)\n</code></pre> Source code in <code>pytorch_widedeep/models/rec/deepffm.py</code> <pre><code>def __init__(\n    self,\n    *,\n    column_idx: Dict[str, int],\n    num_factors: int,\n    cat_embed_input: List[Tuple[str, int]],\n    reduce_sum: bool = True,\n    cat_embed_dropout: Optional[float] = None,\n    use_cat_bias: Optional[bool] = None,\n    cat_embed_activation: Optional[str] = None,\n    mlp_hidden_dims: Optional[List[int]] = None,\n    mlp_activation: Optional[str] = None,\n    mlp_dropout: Optional[float] = None,\n    mlp_batchnorm: Optional[bool] = None,\n    mlp_batchnorm_last: Optional[bool] = None,\n    mlp_linear_first: Optional[bool] = None,\n):\n    super(DeepFieldAwareFactorizationMachine, self).__init__(\n        column_idx=column_idx,\n        input_dim=num_factors,\n        cat_embed_input=cat_embed_input,\n        cat_embed_dropout=cat_embed_dropout,\n        use_cat_bias=use_cat_bias,\n        cat_embed_activation=cat_embed_activation,\n        shared_embed=False,\n        add_shared_embed=None,\n        frac_shared_embed=None,\n        continuous_cols=None,\n        cont_norm_layer=None,\n        embed_continuous_method=None,\n        cont_embed_dropout=None,\n        cont_embed_activation=None,\n        quantization_setup=None,\n        n_frequencies=None,\n        sigma=None,\n        share_last_layer=None,\n        full_embed_dropout=None,\n    )\n\n    self.reduce_sum = reduce_sum\n\n    self.mlp_hidden_dims = mlp_hidden_dims\n    self.mlp_activation = mlp_activation\n    self.mlp_dropout = mlp_dropout\n    self.mlp_batchnorm = mlp_batchnorm\n    self.mlp_batchnorm_last = mlp_batchnorm_last\n    self.mlp_linear_first = mlp_linear_first\n\n    self.n_features = len(self.column_idx)\n    self.n_tokens = sum([ei[1] for ei in cat_embed_input])\n\n    self.encoders = nn.ModuleList(\n        [\n            BaseTabularModelWithAttention(**config)\n            for config in self._get_encoder_configs()\n        ]\n    )\n\n    if self.mlp_hidden_dims is not None:\n        d_hidden = [\n            self.n_features * (self.n_features - 1) // 2 * num_factors\n        ] + self.mlp_hidden_dims\n        self.mlp = MLP(\n            d_hidden=d_hidden,\n            activation=(\n                \"relu\" if self.mlp_activation is None else self.mlp_activation\n            ),\n            dropout=0.0 if self.mlp_dropout is None else self.mlp_dropout,\n            batchnorm=False if self.mlp_batchnorm is None else self.mlp_batchnorm,\n            batchnorm_last=(\n                False\n                if self.mlp_batchnorm_last is None\n                else self.mlp_batchnorm_last\n            ),\n            linear_first=(\n                False if self.mlp_linear_first is None else self.mlp_linear_first\n            ),\n        )\n    else:\n        self.mlp = None\n</code></pre>"},{"location":"pytorch-widedeep/the_rec_module.html#pytorch_widedeep.models.rec.xdeepfm.ExtremeDeepFactorizationMachine","title":"ExtremeDeepFactorizationMachine","text":"<pre><code>ExtremeDeepFactorizationMachine(\n    *,\n    column_idx,\n    input_dim,\n    reduce_sum=True,\n    cin_layer_dims,\n    cat_embed_input,\n    cat_embed_dropout=None,\n    use_cat_bias=None,\n    cat_embed_activation=None,\n    continuous_cols=None,\n    cont_norm_layer=None,\n    embed_continuous_method=\"piecewise\",\n    cont_embed_dropout=None,\n    cont_embed_activation=None,\n    quantization_setup=None,\n    n_frequencies=None,\n    sigma=None,\n    share_last_layer=None,\n    full_embed_dropout=None,\n    mlp_hidden_dims=None,\n    mlp_activation=None,\n    mlp_dropout=None,\n    mlp_batchnorm=None,\n    mlp_batchnorm_last=None,\n    mlp_linear_first=None\n)\n</code></pre> <p>               Bases: <code>BaseTabularModelWithAttention</code></p> <p>Adaptation of 'xDeepFM implementation: xDeepFM: Combining Explicit and Implicit Feature Interactions for Recommender Systems' by Jianxun Lian, Xiaohuan Zhou, Fuzheng Zhang, Zhongxia Chen, Xing Xie, Guangzhong Sun and Enhong Chen, 2018</p> <p>The implementation in this library takes advantage of all the functionalities available to encode categorical and continuous features. The model can be used with only the factorization machine</p> <p>Note that this class implements only the 'Deep' component of the model described in the paper. The linear component is not implemented 'internally' and, if one wants to include it, it can be easily added using the 'wide'/linear component in this library. See the examples in the examples folder.</p> <p>Parameters:</p> <ul> <li> <code>column_idx</code>               (<code>Dict[str, int]</code>)           \u2013            <p>Dictionary mapping column names to their corresponding index.</p> </li> <li> <code>input_dim</code>               (<code>int</code>)           \u2013            <p>Embedding input dimensions</p> </li> <li> <code>reduce_sum</code>               (<code>bool</code>, default:                   <code>True</code> )           \u2013            <p>Whether to reduce the sum in the factorization machine output.</p> </li> <li> <code>cin_layer_dims</code>               (<code>List[int]</code>)           \u2013            <p>List with the number of units per CIN layer. e.g: [128, 64]</p> </li> <li> <code>cat_embed_input</code>               (<code>Optional[List[Tuple[str, int]]]</code>, default:                   <code>None</code> )           \u2013            <p>List of tuples with categorical column names and number of unique values.</p> </li> <li> <code>cat_embed_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Categorical embeddings dropout. If <code>None</code>, it will default to 0.</p> </li> <li> <code>use_cat_bias</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating if bias will be used for the categorical embeddings. If <code>None</code>, it will default to 'False'.</p> </li> <li> <code>cat_embed_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the categorical embeddings, if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> </li> <li> <code>continuous_cols</code>               (<code>Optional[List[str]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the name of the numeric (aka continuous) columns</p> </li> <li> <code>cont_norm_layer</code>               (<code>Optional[Literal['batchnorm', 'layernorm']]</code>, default:                   <code>None</code> )           \u2013            <p>Type of normalization layer applied to the continuous features. Options are: 'layernorm' and 'batchnorm'. if <code>None</code>, no normalization layer will be used.</p> </li> <li> <code>embed_continuous_method</code>               (<code>Optional[Literal['piecewise', 'periodic']]</code>, default:                   <code>\"piecewise\"</code> )           \u2013            <p>Method to use to embed the continuous features. Options are: 'standard', 'periodic' or 'piecewise'. The 'standard' embedding method is based on the FT-Transformer implementation presented in the paper: Revisiting Deep Learning Models for Tabular Data. The 'periodic' and_'piecewise'_ methods were presented in the paper: On Embeddings for Numerical Features in Tabular Deep Learning. Please, read the papers for details.</p> </li> <li> <code>cont_embed_dim</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>Size of the continuous embeddings. If the continuous columns are embedded, <code>cont_embed_dim</code> must be passed.</p> </li> <li> <code>cont_embed_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p> </li> <li> <code>cont_embed_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the continuous embeddings if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported. If <code>None</code>, no activation function will be applied.</p> </li> <li> <code>quantization_setup</code>               (<code>Optional[Dict[str, List[float]]]</code>, default:                   <code>None</code> )           \u2013            <p>This parameter is used when the 'piecewise' method is used to embed the continuous cols. It is a dict where keys are the name of the continuous columns and values are lists with the boundaries for the quantization of the continuous_cols. See the examples for details. If If the 'piecewise' method is used, this parameter is required.</p> </li> <li> <code>n_frequencies</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>This is the so called 'k' in their paper On Embeddings for Numerical Features in Tabular Deep Learning, and is the number of 'frequencies' that will be used to represent each continuous column. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>sigma</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>This is the sigma parameter in the paper mentioned when describing the previous parameters and it is used to initialise the 'frequency weights'. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>share_last_layer</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>This parameter is not present in the before mentioned paper but it is implemented in the official repo. If <code>True</code> the linear layer that turns the frequencies into embeddings will be shared across the continuous columns. If <code>False</code> a different linear layer will be used for each continuous column. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>full_embed_dropout</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>If <code>True</code>, the full embedding corresponding to a column will be masked out/dropout. If <code>None</code>, it will default to <code>False</code>.</p> </li> <li> <code>mlp_hidden_dims</code>               (<code>Optional[List[int]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the number of neurons per dense layer in the mlp.</p> </li> <li> <code>mlp_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the dense layers of the MLP. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> </li> <li> <code>mlp_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>float or List of floats with the dropout between the dense layers. e.g: [0.5,0.5]</p> </li> <li> <code>mlp_batchnorm</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating whether or not batch normalization will be applied to the dense layers</p> </li> <li> <code>mlp_batchnorm_last</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating whether or not batch normalization will be applied to the last of the dense layers</p> </li> <li> <code>mlp_linear_first</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code></p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>n_features</code>               (<code>int</code>)           \u2013            <p>Number of unique features/columns</p> </li> <li> <code>cin</code>               (<code>CompressedInteractionNetwork</code>)           \u2013            <p>Instance of the <code>CompressedInteractionNetwork</code> class</p> </li> <li> <code>mlp</code>               (<code>MLP</code>)           \u2013            <p>Instance of the <code>MLP</code> class if <code>mlp_hidden_dims</code> is not None. If None, the model will return directly the output of the <code>CIN</code></p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models.rec import ExtremeDeepFactorizationMachine\n&gt;&gt;&gt; X_tab = torch.randint(0, 10, (16, 2))\n&gt;&gt;&gt; column_idx = {\"col1\": 0, \"col2\": 1}\n&gt;&gt;&gt; cat_embed_input = [(\"col1\", 10), (\"col2\", 10)]\n&gt;&gt;&gt; xdeepfm = ExtremeDeepFactorizationMachine(\n...     column_idx=column_idx,\n...     input_dim=4,\n...     cin_layer_dims=[8, 16],\n...     cat_embed_input=cat_embed_input,\n...     mlp_hidden_dims=[16, 8]\n... )\n&gt;&gt;&gt; output = xdeepfm(X_tab)\n</code></pre> Source code in <code>pytorch_widedeep/models/rec/xdeepfm.py</code> <pre><code>def __init__(\n    self,\n    *,\n    column_idx: Dict[str, int],\n    input_dim: int,\n    reduce_sum: bool = True,\n    cin_layer_dims: List[int],\n    cat_embed_input: List[Tuple[str, int]],\n    cat_embed_dropout: Optional[float] = None,\n    use_cat_bias: Optional[bool] = None,\n    cat_embed_activation: Optional[str] = None,\n    continuous_cols: Optional[List[str]] = None,\n    cont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\n    embed_continuous_method: Optional[\n        Literal[\"piecewise\", \"periodic\"]\n    ] = \"piecewise\",\n    cont_embed_dropout: Optional[float] = None,\n    cont_embed_activation: Optional[str] = None,\n    quantization_setup: Optional[Dict[str, List[float]]] = None,\n    n_frequencies: Optional[int] = None,\n    sigma: Optional[float] = None,\n    share_last_layer: Optional[bool] = None,\n    full_embed_dropout: Optional[bool] = None,\n    mlp_hidden_dims: Optional[List[int]] = None,\n    mlp_activation: Optional[str] = None,\n    mlp_dropout: Optional[float] = None,\n    mlp_batchnorm: Optional[bool] = None,\n    mlp_batchnorm_last: Optional[bool] = None,\n    mlp_linear_first: Optional[bool] = None,\n):\n    super(ExtremeDeepFactorizationMachine, self).__init__(\n        column_idx=column_idx,\n        input_dim=input_dim,\n        cat_embed_input=cat_embed_input,\n        cat_embed_dropout=cat_embed_dropout,\n        use_cat_bias=use_cat_bias,\n        cat_embed_activation=cat_embed_activation,\n        shared_embed=False,\n        add_shared_embed=None,\n        frac_shared_embed=None,\n        continuous_cols=continuous_cols,\n        cont_norm_layer=cont_norm_layer,\n        embed_continuous_method=embed_continuous_method,\n        cont_embed_dropout=cont_embed_dropout,\n        cont_embed_activation=cont_embed_activation,\n        quantization_setup=quantization_setup,\n        n_frequencies=n_frequencies,\n        sigma=sigma,\n        share_last_layer=share_last_layer,\n        full_embed_dropout=full_embed_dropout,\n    )\n\n    self.mlp_hidden_dims = mlp_hidden_dims\n    self.mlp_activation = mlp_activation\n    self.mlp_dropout = mlp_dropout\n    self.mlp_batchnorm = mlp_batchnorm\n    self.mlp_batchnorm_last = mlp_batchnorm_last\n    self.mlp_linear_first = mlp_linear_first\n\n    self.reduce_sum = reduce_sum\n    self.cin_layer_dims = cin_layer_dims\n\n    self.n_features = len(self.column_idx)\n\n    self.cin = CompressedInteractionNetwork(\n        num_cols=self.n_features, cin_layer_dims=self.cin_layer_dims\n    )\n\n    if self.mlp_hidden_dims is not None:\n        if (\n            self.mlp_hidden_dims[-1] != sum(self.cin_layer_dims)\n            and not self.reduce_sum\n        ):\n            d_hidden = (\n                [sum(self.cin_layer_dims)]\n                + self.mlp_hidden_dims\n                + [sum(self.cin_layer_dims)]\n            )\n        else:\n            d_hidden = [sum(self.cin_layer_dims)] + self.mlp_hidden_dims\n\n        self.mlp = MLP(\n            d_hidden=d_hidden,\n            activation=(\n                \"relu\" if self.mlp_activation is None else self.mlp_activation\n            ),\n            dropout=0.0 if self.mlp_dropout is None else self.mlp_dropout,\n            batchnorm=False if self.mlp_batchnorm is None else self.mlp_batchnorm,\n            batchnorm_last=(\n                False\n                if self.mlp_batchnorm_last is None\n                else self.mlp_batchnorm_last\n            ),\n            linear_first=(\n                False if self.mlp_linear_first is None else self.mlp_linear_first\n            ),\n        )\n    else:\n        self.mlp = None\n</code></pre>"},{"location":"pytorch-widedeep/the_rec_module.html#pytorch_widedeep.models.rec.din.DeepInterestNetwork","title":"DeepInterestNetwork","text":"<pre><code>DeepInterestNetwork(\n    *,\n    column_idx,\n    target_item_col,\n    user_behavior_confiq,\n    action_seq_config=None,\n    other_seq_cols_confiq=None,\n    attention_unit_activation=\"prelu\",\n    cat_embed_input=None,\n    cat_embed_dropout=None,\n    use_cat_bias=None,\n    cat_embed_activation=None,\n    continuous_cols=None,\n    cont_norm_layer=None,\n    embed_continuous=None,\n    embed_continuous_method=None,\n    cont_embed_dim=None,\n    cont_embed_dropout=None,\n    cont_embed_activation=None,\n    quantization_setup=None,\n    n_frequencies=None,\n    sigma=None,\n    share_last_layer=None,\n    full_embed_dropout=None,\n    mlp_hidden_dims=None,\n    mlp_activation=None,\n    mlp_dropout=None,\n    mlp_batchnorm=None,\n    mlp_batchnorm_last=None,\n    mlp_linear_first=None\n)\n</code></pre> <p>               Bases: <code>BaseWDModelComponent</code></p> <p>Adaptation of the Deep Interest Network (DIN) for recommendation systems as described in the paper: 'Deep Interest Network for Click-Through Rate Prediction' by Guorui Zhou et al. 2018.</p> <p>Note that all the categorical- and continuous-related parameters refer to the categorical and continuous columns that are not part of the sequential columns and will be treated as standard tabular data.</p> <p>This model requires some specific data preparation that allows for quite a lot of flexibility. Therefore, this library does not currently include a preprocessor designed specifically for this model. Please, see the example 'movielens_din.py' in the examples folder to understand the data preparation process.</p> <p>Parameters:</p> <ul> <li> <code>column_idx</code>               (<code>Dict[str, int]</code>)           \u2013            <p>Dictionary mapping column names to their corresponding index.</p> </li> <li> <code>target_item_col</code>               (<code>str</code>)           \u2013            <p>Name of the target item column.</p> </li> <li> <code>user_behavior_confiq</code>               (<code>Tuple[List[str], int, int]</code>)           \u2013            <p>Configuration for user behavior sequence columns. Tuple containing: - List of column names that correspond to the user behavior sequence - Number of unique feature values (n_tokens) - Embedding dimension Example: <code>([\"item_1\", \"item_2\", \"item_3\"], 5, 8)</code></p> </li> <li> <code>action_seq_config</code>               (<code>Optional[Tuple[List[str], int]]</code>, default:                   <code>None</code> )           \u2013            <p>Configuration for a so-called action sequence columns (for example a rating, or purchased/not-purchased, etc). Tuple containing: - List of column names - Number of unique feature values (n_tokens) This action will always be learned as a 1d embedding and will be combined with the user behaviour. For example, imagine that the action is purchased/not-purchased. then per item in the user behaviour sequence there will be a binary action to learn 0/1. Such action will be represented by a float number that will multiply the corresponding item embedding in the user behaviour sequence. Example: <code>([\"rating_1\", \"rating_2\", \"rating_3\"], 5)</code> Internally, the embedding dimension will be set to 1</p> </li> <li> <code>other_seq_cols_confiq</code>               (<code>Optional[List[Tuple[List[str], int, int]]]</code>, default:                   <code>None</code> )           \u2013            <p>Configuration for other sequential columns. List of tuples containing: - List of column names that correspond to the sequential column - Number of unique feature values (n_tokens) - Embedding dimension Example: <code>[([\"seq1_col1\", \"seq1_col2\"], 5, 8), ([\"seq2_col1\", \"seq2_col2\"], 5, 8)]</code></p> </li> <li> <code>attention_unit_activation</code>               (<code>Literal['prelu', 'dice']</code>, default:                   <code>\"prelu\"</code> )           \u2013            <p>Activation function to use in the attention unit.</p> </li> <li> <code>cat_embed_input</code>               (<code>Optional[List[Tuple[str, int, int]]]</code>, default:                   <code>None</code> )           \u2013            <p>Configuration for other columns. List of tuples containing: - Column name - Number of unique feature values (n_tokens) - Embedding dimension</p> <p>Note: From here in advance the remaining parameters are related to the categorical and continuous columns that are not part of the sequential columns and will be treated as standard tabular data.</p> </li> <li> <code>cat_embed_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Categorical embeddings dropout. If <code>None</code>, it will default to 0.</p> </li> <li> <code>use_cat_bias</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating if bias will be used for the categorical embeddings. If <code>None</code>, it will default to 'False'.</p> </li> <li> <code>cat_embed_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the categorical embeddings, if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> </li> <li> <code>continuous_cols</code>               (<code>Optional[List[str]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the name of the numeric (aka continuous) columns</p> </li> <li> <code>cont_norm_layer</code>               (<code>Optional[Literal['batchnorm', 'layernorm']]</code>, default:                   <code>None</code> )           \u2013            <p>Type of normalization layer applied to the continuous features. Options are: 'layernorm' and 'batchnorm'. if <code>None</code>, no normalization layer will be used.</p> </li> <li> <code>embed_continuous</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating if the continuous columns will be embedded using one of the available methods: 'standard', 'periodic' or 'piecewise'. If <code>None</code>, it will default to 'False'. NOTE: This parameter is deprecated and it  will be removed in future releases. Please, use the  <code>embed_continuous_method</code> parameter instead.</p> </li> <li> <code>embed_continuous_method</code>               (<code>Optional[Literal['piecewise', 'periodic']]</code>, default:                   <code>\"piecewise\"</code> )           \u2013            <p>Method to use to embed the continuous features. Options are: 'standard', 'periodic' or 'piecewise'. The 'standard' embedding method is based on the FT-Transformer implementation presented in the paper: Revisiting Deep Learning Models for Tabular Data. The 'periodic' and_'piecewise'_ methods were presented in the paper: On Embeddings for Numerical Features in Tabular Deep Learning. Please, read the papers for details.</p> </li> <li> <code>cont_embed_dim</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>Size of the continuous embeddings. If the continuous columns are embedded, <code>cont_embed_dim</code> must be passed.</p> </li> <li> <code>cont_embed_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p> </li> <li> <code>cont_embed_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the continuous embeddings if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported. If <code>None</code>, no activation function will be applied.</p> </li> <li> <code>quantization_setup</code>               (<code>Optional[Dict[str, List[float]]]</code>, default:                   <code>None</code> )           \u2013            <p>This parameter is used when the 'piecewise' method is used to embed the continuous cols. It is a dict where keys are the name of the continuous columns and values are lists with the boundaries for the quantization of the continuous_cols. See the examples for details. If If the 'piecewise' method is used, this parameter is required.</p> </li> <li> <code>n_frequencies</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>This is the so called 'k' in their paper On Embeddings for Numerical Features in Tabular Deep Learning, and is the number of 'frequencies' that will be used to represent each continuous column. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>sigma</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>This is the sigma parameter in the paper mentioned when describing the previous parameters and it is used to initialise the 'frequency weights'. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>share_last_layer</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>This parameter is not present in the before mentioned paper but it is implemented in the official repo. If <code>True</code> the linear layer that turns the frequencies into embeddings will be shared across the continuous columns. If <code>False</code> a different linear layer will be used for each continuous column. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>full_embed_dropout</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>If <code>True</code>, the full embedding corresponding to a column will be masked out/dropout. If <code>None</code>, it will default to <code>False</code>.</p> </li> <li> <code>mlp_hidden_dims</code>               (<code>Optional[List[int]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the number of neurons per dense layer in the mlp.</p> </li> <li> <code>mlp_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the dense layers of the MLP. Currently 'tanh', 'relu', 'leaky_relu', 'gelu' and 'preglu' are   supported</p> </li> <li> <code>mlp_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>float or List of floats with the dropout between the dense layers. e.g: [0.5,0.5]</p> </li> <li> <code>mlp_batchnorm</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating whether or not batch normalization will be applied to the dense layers</p> </li> <li> <code>mlp_batchnorm_last</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating whether or not batch normalization will be applied to the last of the dense layers</p> </li> <li> <code>mlp_linear_first</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code></p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>user_behavior_indexes</code>               (<code>List[int]</code>)           \u2013            <p>List with the indexes of the user behavior columns</p> </li> <li> <code>user_behavior_embed</code>               (<code>BaseTabularModelWithAttention</code>)           \u2013            <p>Embedding layer for the user</p> </li> <li> <code>action_seq_indexes</code>               (<code>List[int]</code>)           \u2013            <p>List with the indexes of the rating sequence columns if the action_seq_config parameter is not None</p> </li> <li> <code>action_embed</code>               (<code>BaseTabularModelWithAttention</code>)           \u2013            <p>Embedding layer for the rating sequence columns if the action_seq_config parameter is not None</p> </li> <li> <code>other_seq_cols_indexes</code>               (<code>Dict[str, List[int]]</code>)           \u2013            <p>Dictionary with the indexes of the other sequential columns if the other_seq_cols_confiq parameter is not None</p> </li> <li> <code>other_seq_cols_embed</code>               (<code>ModuleDict</code>)           \u2013            <p>Dictionary with the embedding layers for the other sequential columns if the other_seq_cols_confiq parameter is not None</p> </li> <li> <code>other_cols_idx</code>               (<code>List[int]</code>)           \u2013            <p>List with the indexes of the other columns if the other_cols_config parameter is not None</p> </li> <li> <code>other_col_embed</code>               (<code>BaseTabularModel</code>)           \u2013            <p>Embedding layer for the other columns if the other_cols_config parameter is not None</p> </li> <li> <code>mlp</code>               (<code>Optional[MLP]</code>)           \u2013            <p>MLP component of the model. If None, no MLP will be used. This should almost always be not None.</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; import numpy as np\n&gt;&gt;&gt; from torch import Tensor\n&gt;&gt;&gt; from typing import Dict, List, Tuple\n&gt;&gt;&gt; from pytorch_widedeep.models.rec import DeepInterestNetwork\n&gt;&gt;&gt; np_seed = np.random.seed(42)\n&gt;&gt;&gt; torch_seed = torch.manual_seed(42)\n&gt;&gt;&gt; num_users = 10\n&gt;&gt;&gt; num_items = 5\n&gt;&gt;&gt; num_contexts = 3\n&gt;&gt;&gt; seq_length = 3\n&gt;&gt;&gt; num_samples = 10\n&gt;&gt;&gt; user_ids = np.random.randint(0, num_users, num_samples)\n&gt;&gt;&gt; target_item_ids = np.random.randint(0, num_items, num_samples)\n&gt;&gt;&gt; context_ids = np.random.randint(0, num_contexts, num_samples)\n&gt;&gt;&gt; user_behavior = np.array(\n...     [\n...         np.random.choice(num_items, seq_length, replace=False)\n...         for _ in range(num_samples)\n...     ]\n... )\n&gt;&gt;&gt; X_arr = np.column_stack((user_ids, target_item_ids, context_ids, user_behavior))\n&gt;&gt;&gt; X = torch.tensor(X_arr, dtype=torch.long)\n&gt;&gt;&gt; column_idx: Dict[str, int] = {\n...     \"user_id\": 0,\n...     \"target_item\": 1,\n...     \"context\": 2,\n...     \"item_1\": 3,\n...     \"item_2\": 4,\n...     \"item_3\": 5,\n... }\n&gt;&gt;&gt; user_behavior_config: Tuple[List[str], int, int] = (\n...     [\"item_1\", \"item_2\", \"item_3\"],\n...     num_items,\n...     8,\n... )\n&gt;&gt;&gt; cat_embed_input: List[Tuple[str, int, int]] = [\n...     (\"user_id\", num_users, 8),\n...     (\"context\", num_contexts, 4),\n... ]\n&gt;&gt;&gt; model = DeepInterestNetwork(\n...     column_idx=column_idx,\n...     target_item_col=\"target_item\",\n...     user_behavior_confiq=user_behavior_config,\n...     cat_embed_input=cat_embed_input,\n...     mlp_hidden_dims=[16, 8],\n... )\n&gt;&gt;&gt; output = model(X)\n</code></pre> Source code in <code>pytorch_widedeep/models/rec/din.py</code> <pre><code>def __init__(\n    self,\n    *,\n    column_idx: Dict[str, int],\n    target_item_col: str,\n    user_behavior_confiq: Tuple[List[str], int, int],\n    action_seq_config: Optional[Tuple[List[str], int]] = None,\n    other_seq_cols_confiq: Optional[List[Tuple[List[str], int, int]]] = None,\n    attention_unit_activation: Literal[\"prelu\", \"dice\"] = \"prelu\",\n    cat_embed_input: Optional[List[Tuple[str, int, int]]] = None,\n    cat_embed_dropout: Optional[float] = None,\n    use_cat_bias: Optional[bool] = None,\n    cat_embed_activation: Optional[str] = None,\n    continuous_cols: Optional[List[str]] = None,\n    cont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\n    embed_continuous: Optional[bool] = None,\n    embed_continuous_method: Optional[\n        Literal[\"standard\", \"piecewise\", \"periodic\"]\n    ] = None,\n    cont_embed_dim: Optional[int] = None,\n    cont_embed_dropout: Optional[float] = None,\n    cont_embed_activation: Optional[str] = None,\n    quantization_setup: Optional[Dict[str, List[float]]] = None,\n    n_frequencies: Optional[int] = None,\n    sigma: Optional[float] = None,\n    share_last_layer: Optional[bool] = None,\n    full_embed_dropout: Optional[bool] = None,\n    mlp_hidden_dims: Optional[List[int]] = None,\n    mlp_activation: Optional[str] = None,\n    mlp_dropout: Optional[float] = None,\n    mlp_batchnorm: Optional[bool] = None,\n    mlp_batchnorm_last: Optional[bool] = None,\n    mlp_linear_first: Optional[bool] = None,\n):\n    super(DeepInterestNetwork, self).__init__()\n\n    self.column_idx = {\n        k: v for k, v in sorted(column_idx.items(), key=lambda x: x[1])\n    }\n\n    self.column_idx = column_idx\n    self.target_item_col = target_item_col\n    self.user_behavior_confiq = user_behavior_confiq\n    self.action_seq_config = (\n        (action_seq_config[0], action_seq_config[1], 1)\n        if action_seq_config is not None\n        else None\n    )\n    self.other_seq_cols_confiq = other_seq_cols_confiq\n    self.cat_embed_input = cat_embed_input\n    self.attention_unit_activation = attention_unit_activation\n\n    self.cat_embed_dropout = cat_embed_dropout\n    self.use_cat_bias = use_cat_bias\n    self.cat_embed_activation = cat_embed_activation\n    self.continuous_cols = continuous_cols\n    self.cont_norm_layer = cont_norm_layer\n    self.embed_continuous = embed_continuous\n    self.embed_continuous_method = embed_continuous_method\n    self.cont_embed_dim = cont_embed_dim\n    self.cont_embed_dropout = cont_embed_dropout\n    self.cont_embed_activation = cont_embed_activation\n    self.quantization_setup = quantization_setup\n    self.n_frequencies = n_frequencies\n    self.sigma = sigma\n    self.share_last_layer = share_last_layer\n    self.full_embed_dropout = full_embed_dropout\n\n    self.mlp_hidden_dims = mlp_hidden_dims\n    self.mlp_activation = mlp_activation\n    self.mlp_dropout = mlp_dropout\n    self.mlp_batchnorm = mlp_batchnorm\n    self.mlp_batchnorm_last = mlp_batchnorm_last\n    self.mlp_linear_first = mlp_linear_first\n\n    self.target_item_idx = self.column_idx[target_item_col]\n\n    self.user_behavior_indexes, self.user_behavior_embed = (\n        self.set_user_behavior_indexes_and_embed(user_behavior_confiq)\n    )\n    self.user_behavior_dim = user_behavior_confiq[2]\n\n    if self.action_seq_config is not None:\n        self.action_seq_indexes, self.action_embed = (\n            self._set_rating_indexes_and_embed(self.action_seq_config)\n        )\n    else:\n        self.action_embed = None\n\n    if self.other_seq_cols_confiq is not None:\n        (\n            self.other_seq_cols_indexes,\n            self.other_seq_cols_embed,\n            self.other_seq_dim,\n        ) = self._set_other_seq_cols_indexes_embed_and_dim(\n            self.other_seq_cols_confiq\n        )\n    else:\n        self.other_seq_cols_embed = None\n        self.other_seq_dim = 0\n\n    if self.cat_embed_input is not None or self.continuous_cols is not None:\n        self.other_cols_idx, self.other_col_embed, self.other_cols_dim = (\n            self._set_other_cols_idx_embed_and_dim(\n                self.cat_embed_input, self.continuous_cols\n            )\n        )\n    else:\n        self.other_col_embed = None\n        self.other_cols_dim = 0\n\n    self.attention = ActivationUnit(\n        user_behavior_confiq[2], attention_unit_activation\n    )\n\n    if self.mlp_hidden_dims is not None:\n        mlp_input_dim = (\n            self.user_behavior_dim * 2 + self.other_seq_dim + self.other_cols_dim\n        )\n        self.mlp = MLP(\n            d_hidden=[mlp_input_dim] + self.mlp_hidden_dims,\n            activation=(\n                \"relu\" if self.mlp_activation is None else self.mlp_activation\n            ),\n            dropout=0.0 if self.mlp_dropout is None else self.mlp_dropout,\n            batchnorm=False if self.mlp_batchnorm is None else self.mlp_batchnorm,\n            batchnorm_last=(\n                False\n                if self.mlp_batchnorm_last is None\n                else self.mlp_batchnorm_last\n            ),\n            linear_first=(\n                False if self.mlp_linear_first is None else self.mlp_linear_first\n            ),\n        )\n    else:\n        self.mlp = None\n</code></pre>"},{"location":"pytorch-widedeep/the_rec_module.html#pytorch_widedeep.models.rec.basic_transformer.Transformer","title":"Transformer","text":"<pre><code>Transformer(\n    vocab_size,\n    seq_length,\n    input_dim,\n    n_heads,\n    n_blocks,\n    attn_dropout=0.1,\n    ff_dropout=0.1,\n    ff_factor=4,\n    activation=\"gelu\",\n    use_linear_attention=False,\n    use_flash_attention=False,\n    padding_idx=0,\n    with_cls_token=False,\n    *,\n    with_pos_encoding=True,\n    pos_encoding_dropout=0.1,\n    pos_encoder=None\n)\n</code></pre> <p>               Bases: <code>Module</code></p> <p>Basic Encoder-Only Transformer Model for sequence classification/regression. As all other models in the library this model can be used as the <code>deeptext</code> component of a Wide &amp; Deep model or independently by itself.</p> <p> NOTE: This model is introduced in the context of recommendation systems and thought for sequences of any nature (e.g. items). It can, of course, still be used for text.</p> <p>Parameters:</p> <ul> <li> <code>vocab_size</code>               (<code>int</code>)           \u2013            <p>Number of words in the vocabulary</p> </li> <li> <code>input_dim</code>               (<code>int</code>)           \u2013            <p>Dimension of the token embeddings</p> <p>Param aliases: <code>embed_dim</code>, <code>d_model</code>. </p> </li> <li> <code>seq_length</code>               (<code>int</code>)           \u2013            <p>Input sequence length</p> </li> <li> <code>n_heads</code>               (<code>int</code>)           \u2013            <p>Number of attention heads per Transformer block</p> </li> <li> <code>n_blocks</code>               (<code>int</code>)           \u2013            <p>Number of Transformer blocks</p> </li> <li> <code>attn_dropout</code>               (<code>float</code>, default:                   <code>0.1</code> )           \u2013            <p>Dropout that will be applied to the Multi-Head Attention layers</p> </li> <li> <code>ff_dropout</code>               (<code>float</code>, default:                   <code>0.1</code> )           \u2013            <p>Dropout that will be applied to the FeedForward network</p> </li> <li> <code>ff_factor</code>               (<code>int</code>, default:                   <code>4</code> )           \u2013            <p>Multiplicative factor applied to the first layer of the FF network in each Transformer block, This is normally set to 4.</p> </li> <li> <code>activation</code>               (<code>str</code>, default:                   <code>'gelu'</code> )           \u2013            <p>Transformer Encoder activation function. 'tanh', 'relu', 'leaky_relu', 'gelu', 'geglu' and 'reglu' are supported</p> </li> <li> <code>padding_idx</code>               (<code>int</code>, default:                   <code>0</code> )           \u2013            <p>index of the padding token in the padded-tokenised sequences.</p> </li> <li> <code>with_cls_token</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating if a <code>'[CLS]'</code> token is included in the tokenized sequences. If present, the final hidden state corresponding to this token is used as the aggregated representation for classification and regression tasks. NOTE: if included in the tokenized sequences it must be inserted as the first token in the sequences.</p> </li> <li> <code>with_pos_encoding</code>               (<code>bool</code>, default:                   <code>True</code> )           \u2013            <p>Boolean indicating if positional encoding will be used</p> </li> <li> <code>pos_encoding_dropout</code>               (<code>float</code>, default:                   <code>0.1</code> )           \u2013            <p>Positional encoding dropout</p> </li> <li> <code>pos_encoder</code>               (<code>Optional[Module]</code>, default:                   <code>None</code> )           \u2013            <p>This model uses by default a standard positional encoding approach. However, any custom positional encoder can also be used and pass to the Transformer model via the 'pos_encoder' parameter</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>embedding</code>               (<code>Module</code>)           \u2013            <p>Standard token embedding layer</p> </li> <li> <code>pos_encoder</code>               (<code>Module</code>)           \u2013            <p>Positional Encoder</p> </li> <li> <code>encoder</code>               (<code>Module</code>)           \u2013            <p>Sequence of Transformer blocks</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models.rec import Transformer\n&gt;&gt;&gt; X = torch.cat((torch.zeros([5,1]), torch.empty(5, 4).random_(1,4)), axis=1)\n&gt;&gt;&gt; model = Transformer(vocab_size=4, seq_length=5, input_dim=8, n_heads=1, n_blocks=1)\n&gt;&gt;&gt; out = model(X)\n</code></pre> Source code in <code>pytorch_widedeep/models/rec/basic_transformer.py</code> <pre><code>@alias(\"input_dim\", [\"embed_dim\", \"d_model\"])\n@alias(\"seq_length\", [\"max_length\", \"maxlen\"])\ndef __init__(\n    self,\n    vocab_size: int,\n    seq_length: int,\n    input_dim: int,\n    n_heads: int,\n    n_blocks: int,\n    attn_dropout: float = 0.1,\n    ff_dropout: float = 0.1,\n    ff_factor: int = 4,\n    activation: str = \"gelu\",\n    use_linear_attention: bool = False,\n    use_flash_attention: bool = False,\n    padding_idx: int = 0,\n    with_cls_token: bool = False,\n    *,  # from here on pos encoding args\n    with_pos_encoding: bool = True,\n    pos_encoding_dropout: float = 0.1,\n    pos_encoder: Optional[nn.Module] = None,\n):\n    super().__init__()\n\n    self.input_dim = input_dim\n    self.seq_length = seq_length\n    self.n_heads = n_heads\n    self.n_blocks = n_blocks\n    self.attn_dropout = attn_dropout\n    self.ff_dropout = ff_dropout\n    self.ff_factor = ff_factor\n    self.activation = activation\n    self.use_linear_attention = use_linear_attention\n    self.use_flash_attention = use_flash_attention\n    self.padding_idx = padding_idx\n    self.with_cls_token = with_cls_token\n    self.with_pos_encoding = with_pos_encoding\n    self.pos_encoding_dropout = pos_encoding_dropout\n\n    self.embedding = nn.Embedding(\n        vocab_size, input_dim, padding_idx=self.padding_idx\n    )\n\n    if with_pos_encoding:\n        if pos_encoder is not None:\n            self.pos_encoder: Union[nn.Module, nn.Identity, PositionalEncoding] = (\n                pos_encoder\n            )\n        else:\n            self.pos_encoder = PositionalEncoding(\n                input_dim, pos_encoding_dropout, seq_length\n            )\n    else:\n        self.pos_encoder = nn.Identity()\n\n    self.encoder = nn.Sequential()\n    for i in range(n_blocks):\n        self.encoder.add_module(\n            \"transformer_block\" + str(i),\n            TransformerEncoder(\n                input_dim,\n                n_heads,\n                False,  # use_qkv_bias\n                attn_dropout,\n                ff_dropout,\n                ff_factor,\n                activation,\n                use_linear_attention,\n                use_flash_attention,\n            ),\n        )\n</code></pre>"},{"location":"pytorch-widedeep/trainer.html","title":"Training multimodal Deep Learning Models","text":"<p>Here is the documentation for the <code>Trainer</code> class, that will do all the heavy lifting.</p> <p>Trainer is also available from <code>pytorch-widedeep</code> directly, for example, one could do:</p> <pre><code>    from pytorch-widedeep.training import Trainer\n</code></pre> <p>or also:</p> <pre><code>    from pytorch-widedeep import Trainer\n</code></pre>"},{"location":"pytorch-widedeep/trainer.html#pytorch_widedeep.training.Trainer","title":"Trainer","text":"<pre><code>Trainer(\n    model,\n    objective,\n    custom_loss_function=None,\n    optimizers=None,\n    lr_schedulers=None,\n    initializers=None,\n    transforms=None,\n    callbacks=None,\n    metrics=None,\n    verbose=1,\n    seed=1,\n    **kwargs\n)\n</code></pre> <p>               Bases: <code>BaseTrainer</code></p> <p>Class to set the of attributes that will be used during the training process.</p> <p>Parameters:</p> <ul> <li> <code>model</code>               (<code>WideDeep</code>)           \u2013            <p>An object of class <code>WideDeep</code></p> </li> <li> <code>objective</code>               (<code>str</code>)           \u2013            <p>Defines the objective, loss or cost function. </p> <p>Param aliases: <code>loss_function</code>, <code>loss_fn</code>, <code>loss</code>, <code>cost_function</code>, <code>cost_fn</code>, <code>cost</code>. </p> <p>Possible values are:</p> <ul> <li> <p><code>binary</code>, aliases: <code>logistic</code>, <code>binary_logloss</code>, <code>binary_cross_entropy</code></p> </li> <li> <p><code>binary_focal_loss</code></p> </li> <li> <p><code>multiclass</code>, aliases: <code>multi_logloss</code>, <code>cross_entropy</code>, <code>categorical_cross_entropy</code>,</p> </li> <li> <p><code>multiclass_focal_loss</code></p> </li> <li> <p><code>regression</code>, aliases: <code>mse</code>, <code>l2</code>, <code>mean_squared_error</code></p> </li> <li> <p><code>mean_absolute_error</code>, aliases: <code>mae</code>, <code>l1</code></p> </li> <li> <p><code>mean_squared_log_error</code>, aliases: <code>msle</code></p> </li> <li> <p><code>root_mean_squared_error</code>, aliases:  <code>rmse</code></p> </li> <li> <p><code>root_mean_squared_log_error</code>, aliases: <code>rmsle</code></p> </li> <li> <p><code>zero_inflated_lognormal</code>, aliases: <code>ziln</code></p> </li> <li> <p><code>quantile</code></p> </li> <li> <p><code>tweedie</code></p> </li> <li> <p><code>multitarget</code>, aliases: <code>multi_target</code></p> </li> </ul> <p>NOTE: For <code>multitarget</code> a custom loss function must be passed</p> </li> <li> <code>custom_loss_function</code>               (<code>Optional[Module]</code>, default:                   <code>None</code> )           \u2013            <p>It is possible to pass a custom loss function. See for example <code>pytorch_widedeep.losses.FocalLoss</code> for the required structure of the object or the Examples section in this documentation or in the repo. Note that if <code>custom_loss_function</code> is not <code>None</code>, <code>objective</code> must be 'binary', 'multiclass' or 'regression', consistent with the loss function</p> </li> <li> <code>optimizers</code>               (<code>Optional[Union[Optimizer, Dict[str, Union[Optimizer, List[Optimizer]]]]]</code>, default:                   <code>None</code> )           \u2013            <ul> <li>An instance of Pytorch's <code>Optimizer</code> object   (e.g. <code>torch.optim.Adam()</code>) or</li> <li>a dictionary where there keys are the model components (i.e.   'wide', 'deeptabular', 'deeptext', 'deepimage'   and/or 'deephead')  and the values are the corresponding   optimizers or list of optimizers if multiple models are used for   the given data mode (e.g. two text columns/models for the deeptext   component). If multiple optimizers are used the   dictionary MUST contain an optimizer per model component.</li> </ul> <p>if no optimizers are passed it will default to <code>Adam</code> for all model components</p> </li> <li> <code>lr_schedulers</code>               (<code>Optional[Union[LRScheduler, Dict[str, Union[LRScheduler, List[LRScheduler]]]]]</code>, default:                   <code>None</code> )           \u2013            <ul> <li>An instance of Pytorch's <code>LRScheduler</code> object (e.g   <code>torch.optim.lr_scheduler.StepLR(opt, step_size=5)</code>) or</li> <li>a dictionary where there keys are the model componenst (i.e. 'wide',   'deeptabular', 'deeptext', 'deepimage' and/or 'deephead') and the   values are the corresponding learning rate schedulers or list of     learning rate schedulers if multiple models are used for the given     data mode (e.g. two text columns/models for the deeptext component).</li> </ul> </li> <li> <code>initializers</code>               (<code>Optional[Union[Initializer, Dict[str, Union[Initializer, List[Initializer]]]]]</code>, default:                   <code>None</code> )           \u2013            <ul> <li>An instance of an <code>Initializer</code> object see <code>pytorch-widedeep.initializers</code> or</li> <li>a dictionary where there keys are the model components (i.e. 'wide',   'deeptabular', 'deeptext', 'deepimage' and/or 'deephead')   and the values are the corresponding initializers or list of     initializers if multiple models are used for the given data mode (e.g.     two text columns/models for the deeptext component).</li> </ul> </li> <li> <code>transforms</code>               (<code>Optional[List[Transforms]]</code>, default:                   <code>None</code> )           \u2013            <p>List with <code>torchvision.transforms</code> to be applied to the image component of the model (i.e. <code>deepimage</code>) See torchvision transforms.</p> </li> <li> <code>callbacks</code>               (<code>Optional[List[Callback]]</code>, default:                   <code>None</code> )           \u2013            <p>List with <code>Callback</code> objects. The three callbacks available in <code>pytorch-widedeep</code> are: <code>LRHistory</code>, <code>ModelCheckpoint</code> and <code>EarlyStopping</code>. The <code>History</code> and the <code>LRShedulerCallback</code> callbacks are used by default. This can also be a custom callback as long as the object of type <code>Callback</code>. See <code>pytorch_widedeep.callbacks.Callback</code> or the examples folder in the repo.</p> </li> <li> <code>metrics</code>               (<code>Optional[Union[List[Metric], List[Metric]]]</code>, default:                   <code>None</code> )           \u2013            <ul> <li>List of objects of type <code>Metric</code>. Metrics available are:   <code>Accuracy</code>, <code>Precision</code>, <code>Recall</code>, <code>FBetaScore</code>,   <code>F1Score</code> and <code>R2Score</code>. This can also be a custom metric as long   as it is an object of type <code>Metric</code>. See   <code>pytorch_widedeep.metrics.Metric</code> or the examples folder in the   repo</li> <li>List of objects of type <code>torchmetrics.Metric</code>. This can be any   metric from torchmetrics library   Examples.   This can also be a custom metric as long as   it is an object of type <code>Metric</code>. See   the instructions.</li> </ul> </li> <li> <code>verbose</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>Verbosity level. If set to 0 nothing will be printed during training</p> </li> <li> <code>seed</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>Random seed to be used internally for train/test split</p> </li> </ul> <p>Other Parameters:</p> <ul> <li> <code>**kwargs</code>           \u2013            <p>Other infrequently used arguments that can also be passed as kwargs are:</p> <ul> <li> <p>device: <code>str</code>     string indicating the device. One of 'cpu' or 'gpu'</p> </li> <li> <p>num_workers: <code>int</code>     number of workers to be used internally by the data loaders</p> </li> <li> <p>lambda_sparse: <code>float</code>     lambda sparse parameter in case the <code>deeptabular</code> component is <code>TabNet</code></p> </li> <li> <p>class_weight: <code>List[float]</code>     This is the <code>weight</code> or <code>pos_weight</code> parameter in     <code>CrossEntropyLoss</code> and <code>BCEWithLogitsLoss</code>, depending on whether</p> </li> <li>reducelronplateau_criterion: <code>str</code>     This sets the criterion that will be used by the lr scheduler to     take a step: One of 'loss' or 'metric'. The ReduceLROnPlateau     learning rate is a bit particular.</li> </ul> </li> </ul> <p>Attributes:</p> <ul> <li> <code>cyclic_lr</code>               (<code>bool</code>)           \u2013            <p>Attribute that indicates if any of the lr_schedulers is cyclic_lr (i.e. <code>CyclicLR</code> or <code>OneCycleLR</code>). See Pytorch schedulers.</p> </li> <li> <code>feature_importance</code>               (<code>dict</code>)           \u2013            <p>dict where the keys are the column names and the values are the corresponding feature importances. This attribute will only exist if the <code>deeptabular</code> component is a Tabnet model.</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from torchvision.transforms import ToTensor\n&gt;&gt;&gt;\n&gt;&gt;&gt; # wide deep imports\n&gt;&gt;&gt; from pytorch_widedeep.callbacks import EarlyStopping, LRHistory\n&gt;&gt;&gt; from pytorch_widedeep.initializers import KaimingNormal, KaimingUniform, Normal, Uniform\n&gt;&gt;&gt; from pytorch_widedeep.models import TabResnet, Vision, BasicRNN, Wide, WideDeep\n&gt;&gt;&gt; from pytorch_widedeep import Trainer\n&gt;&gt;&gt;\n&gt;&gt;&gt; embed_input = [(u, i, j) for u, i, j in zip([\"a\", \"b\", \"c\"][:4], [4] * 3, [8] * 3)]\n&gt;&gt;&gt; column_idx = {k: v for v, k in enumerate([\"a\", \"b\", \"c\"])}\n&gt;&gt;&gt; wide = Wide(10, 1)\n&gt;&gt;&gt;\n&gt;&gt;&gt; # build the model\n&gt;&gt;&gt; deeptabular = TabResnet(blocks_dims=[8, 4], column_idx=column_idx, cat_embed_input=embed_input)\n&gt;&gt;&gt; deeptext = BasicRNN(vocab_size=10, embed_dim=4, padding_idx=0)\n&gt;&gt;&gt; deepimage = Vision()\n&gt;&gt;&gt; model = WideDeep(wide=wide, deeptabular=deeptabular, deeptext=deeptext, deepimage=deepimage)\n&gt;&gt;&gt;\n&gt;&gt;&gt; # set optimizers and schedulers\n&gt;&gt;&gt; wide_opt = torch.optim.Adam(model.wide.parameters())\n&gt;&gt;&gt; deep_opt = torch.optim.AdamW(model.deeptabular.parameters())\n&gt;&gt;&gt; text_opt = torch.optim.Adam(model.deeptext.parameters())\n&gt;&gt;&gt; img_opt = torch.optim.AdamW(model.deepimage.parameters())\n&gt;&gt;&gt;\n&gt;&gt;&gt; wide_sch = torch.optim.lr_scheduler.StepLR(wide_opt, step_size=5)\n&gt;&gt;&gt; deep_sch = torch.optim.lr_scheduler.StepLR(deep_opt, step_size=3)\n&gt;&gt;&gt; text_sch = torch.optim.lr_scheduler.StepLR(text_opt, step_size=5)\n&gt;&gt;&gt; img_sch = torch.optim.lr_scheduler.StepLR(img_opt, step_size=3)\n&gt;&gt;&gt;\n&gt;&gt;&gt; optimizers = {\"wide\": wide_opt, \"deeptabular\": deep_opt, \"deeptext\": text_opt, \"deepimage\": img_opt}\n&gt;&gt;&gt; schedulers = {\"wide\": wide_sch, \"deeptabular\": deep_sch, \"deeptext\": text_sch, \"deepimage\": img_sch}\n&gt;&gt;&gt;\n&gt;&gt;&gt; # set initializers and callbacks\n&gt;&gt;&gt; initializers = {\"wide\": Uniform, \"deeptabular\": Normal, \"deeptext\": KaimingNormal, \"deepimage\": KaimingUniform}\n&gt;&gt;&gt; transforms = [ToTensor]\n&gt;&gt;&gt; callbacks = [LRHistory(n_epochs=4), EarlyStopping]\n&gt;&gt;&gt;\n&gt;&gt;&gt; # set the trainer\n&gt;&gt;&gt; trainer = Trainer(model, objective=\"regression\", initializers=initializers, optimizers=optimizers,\n... lr_schedulers=schedulers, callbacks=callbacks, transforms=transforms)\n</code></pre> Source code in <code>pytorch_widedeep/training/trainer.py</code> <pre><code>@alias(  # noqa: C901\n    \"objective\",\n    [\"loss_function\", \"loss_fn\", \"loss\", \"cost_function\", \"cost_fn\", \"cost\"],\n)\ndef __init__(\n    self,\n    model: WideDeep,\n    objective: str,\n    custom_loss_function: Optional[nn.Module] = None,\n    optimizers: Optional[\n        Union[Optimizer, Dict[str, Union[Optimizer, List[Optimizer]]]]\n    ] = None,\n    lr_schedulers: Optional[\n        Union[LRScheduler, Dict[str, Union[LRScheduler, List[LRScheduler]]]]\n    ] = None,\n    initializers: Optional[\n        Union[Initializer, Dict[str, Union[Initializer, List[Initializer]]]]\n    ] = None,\n    transforms: Optional[List[Transforms]] = None,\n    callbacks: Optional[List[Callback]] = None,\n    metrics: Optional[Union[List[Metric], List[TorchMetric]]] = None,\n    verbose: int = 1,\n    seed: int = 1,\n    **kwargs,\n):\n    super().__init__(\n        model=model,\n        objective=objective,\n        custom_loss_function=custom_loss_function,\n        optimizers=optimizers,\n        lr_schedulers=lr_schedulers,\n        initializers=initializers,\n        transforms=transforms,\n        callbacks=callbacks,\n        metrics=metrics,\n        verbose=verbose,\n        seed=seed,\n        **kwargs,\n    )\n</code></pre>"},{"location":"pytorch-widedeep/trainer.html#pytorch_widedeep.training.Trainer.fit","title":"fit","text":"<pre><code>fit(\n    X_wide=None,\n    X_tab=None,\n    X_text=None,\n    X_img=None,\n    X_train=None,\n    X_val=None,\n    val_split=None,\n    target=None,\n    n_epochs=1,\n    validation_freq=1,\n    batch_size=32,\n    custom_dataloader=None,\n    feature_importance_sample_size=None,\n    finetune=False,\n    **kwargs\n)\n</code></pre> <p>Fit method.</p> <p>The input datasets can be passed either directly via numpy arrays (<code>X_wide</code>, <code>X_tab</code>, <code>X_text</code> or <code>X_img</code>) or alternatively, in dictionaries (<code>X_train</code> or <code>X_val</code>).</p> <p>Parameters:</p> <ul> <li> <code>X_wide</code>               (<code>Optional[ndarray]</code>, default:                   <code>None</code> )           \u2013            <p>Input for the <code>wide</code> model component. See <code>pytorch_widedeep.preprocessing.WidePreprocessor</code></p> </li> <li> <code>X_tab</code>               (<code>Optional[Union[ndarray, List[ndarray]]]</code>, default:                   <code>None</code> )           \u2013            <p>Input for the <code>deeptabular</code> model component. See <code>pytorch_widedeep.preprocessing.TabPreprocessor</code>. If multiple tabular models are used for different columns, this should be a list of numpy arrays</p> </li> <li> <code>X_text</code>               (<code>Optional[Union[ndarray, List[ndarray]]]</code>, default:                   <code>None</code> )           \u2013            <p>Input for the <code>deeptext</code> model component. See <code>pytorch_widedeep.preprocessing.TextPreprocessor</code>. If multiple text columns/models are used, this should be a list of numpy arrays</p> </li> <li> <code>X_img</code>               (<code>Optional[Union[ndarray, List[ndarray]]]</code>, default:                   <code>None</code> )           \u2013            <p>Input for the <code>deepimage</code> model component. See <code>pytorch_widedeep.preprocessing.ImagePreprocessor</code>. If multiple image columns/models are used, this should be a list of numpy arrays</p> </li> <li> <code>X_train</code>               (<code>Optional[Dict[str, Union[ndarray, List[ndarray]]]]</code>, default:                   <code>None</code> )           \u2013            <p>The training dataset can also be passed in a dictionary. Keys are 'X_wide', 'X_tab', 'X_text', 'X_img' and 'target'. Values are the corresponding matrices. Note that of multiple text or image columns/models are used, the corresponding values should be lists of numpy arrays</p> </li> <li> <code>X_val</code>               (<code>Optional[Dict[str, Union[ndarray, List[ndarray]]]]</code>, default:                   <code>None</code> )           \u2013            <p>The validation dataset can also be passed in a dictionary. Keys are 'X_wide', 'X_tab', 'X_text', 'X_img' and 'target'. Values are the corresponding matrices. Note that of multiple text or image columns/models are used, the corresponding values should be lists of numpy arrays</p> </li> <li> <code>val_split</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>train/val split fraction</p> </li> <li> <code>target</code>               (<code>Optional[ndarray]</code>, default:                   <code>None</code> )           \u2013            <p>target values</p> </li> <li> <code>n_epochs</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>number of epochs</p> </li> <li> <code>validation_freq</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>epochs validation frequency</p> </li> <li> <code>batch_size</code>               (<code>int</code>, default:                   <code>32</code> )           \u2013            <p>batch size</p> </li> <li> <code>custom_dataloader</code>               (<code>Optional[DataLoader]</code>, default:                   <code>None</code> )           \u2013            <p>object of class <code>torch.utils.data.DataLoader</code>. Available predefined dataloaders are in <code>pytorch-widedeep.dataloaders</code>.If <code>None</code>, a standard torch <code>DataLoader</code> is used.</p> </li> <li> <code>finetune</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>fine-tune individual model components. This functionality can also be used to 'warm-up' (and hence the alias <code>warmup</code>) individual components before the joined training starts, and hence its alias. See the Examples folder in the repo for more details</p> <p><code>pytorch_widedeep</code> implements 3 fine-tune routines.</p> <ul> <li>fine-tune all trainable layers at once. This routine is   inspired by the work of Howard &amp; Sebastian Ruder 2018 in their   ULMfit paper. Using a   Slanted Triangular learing (see   Leslie N. Smith paper ) ,   the process is the following: i) the learning rate will   gradually increase for 10% of the training steps from max_lr/10   to max_lr. ii) It will then gradually decrease to max_lr/10   for the remaining 90% of the steps. The optimizer used in the   process is <code>Adam</code>.</li> </ul> <p>and two gradual fine-tune routines, where only certain layers are trained at a time.</p> <ul> <li>The so called <code>Felbo</code> gradual fine-tune rourine, based on the the   Felbo et al., 2017 DeepEmoji paper.</li> <li>The <code>Howard</code> routine based on the work of Howard &amp; Sebastian Ruder 2018 in their   ULMfit paper.</li> </ul> <p>For details on how these routines work, please see the Examples section in this documentation and the Examples folder in the repo.  Param Alias: <code>warmup</code></p> </li> </ul> <p>Other Parameters:</p> <ul> <li> <code>**kwargs</code>           \u2013            <p>Other keyword arguments are:</p> <ul> <li> <p>DataLoader related parameters:     For example,  <code>sampler</code>, <code>batch_sampler</code>, <code>collate_fn</code>, etc.     Please, see the pytorch     DataLoader docs     for details.</p> </li> <li> <p>Finetune related parameters:     see the source code at <code>pytorch_widedeep._finetune</code>. Namely, these are:</p> <ul> <li><code>finetune_epochs</code> (<code>int</code>):     number of epochs use for fine tuning</li> <li><code>finetune_max_lr</code> (<code>float</code>):    max lr during fine tuning</li> <li><code>routine</code> (<code>str</code>):    one of 'howard' or 'felbo'</li> <li><code>deeptabular_gradual</code> (<code>bool</code>):    boolean indicating if the <code>deeptabular</code> component will be fine tuned gradually</li> <li><code>deeptabular_layers</code> (<code>Optional[Union[List[nn.Module], List[List[nn.Module]]]]</code>):    List of pytorch modules indicating the layers of the    <code>deeptabular</code> that will be fine tuned</li> <li><code>deeptabular_max_lr</code> (<code>Union[float, List[float]]</code>):    max lr for the <code>deeptabular</code> componet during fine tuning</li> <li><code>deeptext_gradual</code> (<code>bool</code>):    same as <code>deeptabular_gradual</code> but for the <code>deeptext</code> component</li> <li><code>deeptext_layers</code> (<code>Optional[Union[List[nn.Module], List[List[nn.Module]]]]</code>):    same as <code>deeptabular_gradual</code> but for the <code>deeptext</code> component.    If there are multiple text columns/models, this should be a list of lists</li> <li><code>deeptext_max_lr</code> (<code>Union[float, List[float]]</code>):    same as <code>deeptabular_gradual</code> but for the <code>deeptext</code> component    If there are multiple text columns/models, this should be a list of floats</li> <li><code>deepimage_gradual</code> (<code>bool</code>):    same as <code>deeptext_layers</code> but for the <code>deepimage</code> component</li> <li><code>deepimage_layers</code> (<code>Optional[Union[List[nn.Module], List[List[nn.Module]]]]</code>):    same as <code>deeptext_layers</code> but for the <code>deepimage</code> component</li> <li><code>deepimage_max_lr</code> (<code>Union[float, List[float]]</code>):     same as <code>deeptext_layers</code> but for the <code>deepimage</code> component</li> </ul> </li> </ul> </li> </ul> <p>Examples:</p> <p>For a series of comprehensive examples on how to use the <code>fit</code> method, please see the Examples folder in the repo</p> Source code in <code>pytorch_widedeep/training/trainer.py</code> <pre><code>@alias(\"finetune\", [\"warmup\"])\ndef fit(  # noqa: C901\n    self,\n    X_wide: Optional[np.ndarray] = None,\n    X_tab: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n    X_text: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n    X_img: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n    X_train: Optional[Dict[str, Union[np.ndarray, List[np.ndarray]]]] = None,\n    X_val: Optional[Dict[str, Union[np.ndarray, List[np.ndarray]]]] = None,\n    val_split: Optional[float] = None,\n    target: Optional[np.ndarray] = None,\n    n_epochs: int = 1,\n    validation_freq: int = 1,\n    batch_size: int = 32,\n    custom_dataloader: Optional[DataLoader] = None,\n    feature_importance_sample_size: Optional[int] = None,\n    finetune: bool = False,\n    **kwargs,\n):\n    r\"\"\"Fit method.\n\n    The input datasets can be passed either directly via numpy arrays\n    (`X_wide`, `X_tab`, `X_text` or `X_img`) or alternatively, in\n    dictionaries (`X_train` or `X_val`).\n\n    Parameters\n    ----------\n    X_wide: np.ndarray, Optional. default=None\n        Input for the `wide` model component.\n        See `pytorch_widedeep.preprocessing.WidePreprocessor`\n    X_tab: np.ndarray, Optional. default=None\n        Input for the `deeptabular` model component.\n        See `pytorch_widedeep.preprocessing.TabPreprocessor`. If multiple\n        tabular models are used for different columns, this should be a\n        list of numpy arrays\n    X_text: Union[np.ndarray, List[np.ndarray]], Optional. default=None\n        Input for the `deeptext` model component.\n        See `pytorch_widedeep.preprocessing.TextPreprocessor`.\n        If multiple text columns/models are used, this should be a list of\n        numpy arrays\n    X_img: np.ndarray, Optional. default=None\n        Input for the `deepimage` model component.\n        See `pytorch_widedeep.preprocessing.ImagePreprocessor`.\n        If multiple image columns/models are used, this should be a list of\n        numpy arrays\n    X_train: Dict, Optional. default=None\n        The training dataset can also be passed in a dictionary. Keys are\n        _'X_wide'_, _'X_tab'_, _'X_text'_, _'X_img'_ and _'target'_. Values\n        are the corresponding matrices. Note that of multiple text or image\n        columns/models are used, the corresponding values should be lists\n        of numpy arrays\n    X_val: Dict, Optional. default=None\n        The validation dataset can also be passed in a dictionary. Keys\n        are _'X_wide'_, _'X_tab'_, _'X_text'_, _'X_img'_ and _'target'_.\n        Values are the corresponding matrices. Note that of multiple text\n        or image columns/models are used, the corresponding values should\n        be lists of numpy arrays\n    val_split: float, Optional. default=None\n        train/val split fraction\n    target: np.ndarray, Optional. default=None\n        target values\n    n_epochs: int, default=1\n        number of epochs\n    validation_freq: int, default=1\n        epochs validation frequency\n    batch_size: int, default=32\n        batch size\n    custom_dataloader: `DataLoader`, Optional, default=None\n        object of class `torch.utils.data.DataLoader`. Available\n        predefined dataloaders are in `pytorch-widedeep.dataloaders`.If\n        `None`, a standard torch `DataLoader` is used.\n    finetune: bool, default=False\n        fine-tune individual model components. This functionality can also\n        be used to 'warm-up' (and hence the alias `warmup`) individual\n        components before the joined training starts, and hence its\n        alias. See the Examples folder in the repo for more details\n\n        `pytorch_widedeep` implements 3 fine-tune routines.\n\n        - fine-tune all trainable layers at once. This routine is\n          inspired by the work of Howard &amp; Sebastian Ruder 2018 in their\n          [ULMfit paper](https://arxiv.org/abs/1801.06146). Using a\n          Slanted Triangular learing (see\n          [Leslie N. Smith paper](https://arxiv.org/pdf/1506.01186.pdf) ) ,\n          the process is the following: *i*) the learning rate will\n          gradually increase for 10% of the training steps from max_lr/10\n          to max_lr. *ii*) It will then gradually decrease to max_lr/10\n          for the remaining 90% of the steps. The optimizer used in the\n          process is `Adam`.\n\n        and two gradual fine-tune routines, where only certain layers are\n        trained at a time.\n\n        - The so called `Felbo` gradual fine-tune rourine, based on the the\n          Felbo et al., 2017 [DeepEmoji paper](https://arxiv.org/abs/1708.00524).\n        - The `Howard` routine based on the work of Howard &amp; Sebastian Ruder 2018 in their\n          [ULMfit paper](https://arxiv.org/abs/1801.06146&gt;).\n\n        For details on how these routines work, please see the Examples\n        section in this documentation and the Examples folder in the repo. &lt;br/&gt;\n        Param Alias: `warmup`\n\n    Other Parameters\n    ----------------\n    **kwargs:\n        Other keyword arguments are:\n\n        - **DataLoader related parameters**:&lt;br/&gt;\n            For example,  `sampler`, `batch_sampler`, `collate_fn`, etc.\n            Please, see the pytorch\n            [DataLoader docs](https://pytorch.org/docs/stable/data.html#torch.utils.data.DataLoader)\n            for details.\n\n        - **Finetune related parameters**:&lt;br/&gt;\n            see the source code at `pytorch_widedeep._finetune`. Namely, these are:\n\n            - `finetune_epochs` (`int`):\n                number of epochs use for fine tuning\n            - `finetune_max_lr` (`float`):\n               max lr during fine tuning\n            - `routine` (`str`):\n               one of _'howard'_ or _'felbo'_\n            - `deeptabular_gradual` (`bool`):\n               boolean indicating if the `deeptabular` component will be fine tuned gradually\n            - `deeptabular_layers` (`Optional[Union[List[nn.Module], List[List[nn.Module]]]]`):\n               List of pytorch modules indicating the layers of the\n               `deeptabular` that will be fine tuned\n            - `deeptabular_max_lr` (`Union[float, List[float]]`):\n               max lr for the `deeptabular` componet during fine tuning\n            - `deeptext_gradual` (`bool`):\n               same as `deeptabular_gradual` but for the `deeptext` component\n            - `deeptext_layers` (`Optional[Union[List[nn.Module], List[List[nn.Module]]]]`):\n               same as `deeptabular_gradual` but for the `deeptext` component.\n               If there are multiple text columns/models, this should be a list of lists\n            - `deeptext_max_lr` (`Union[float, List[float]]`):\n               same as `deeptabular_gradual` but for the `deeptext` component\n               If there are multiple text columns/models, this should be a list of floats\n            - `deepimage_gradual` (`bool`):\n               same as `deeptext_layers` but for the `deepimage` component\n            - `deepimage_layers` (`Optional[Union[List[nn.Module], List[List[nn.Module]]]]`):\n               same as `deeptext_layers` but for the `deepimage` component\n            - `deepimage_max_lr` (`Union[float, List[float]]`):\n                same as `deeptext_layers` but for the `deepimage` component\n\n    Examples\n    --------\n\n    For a series of comprehensive examples on how to use the `fit` method, please see the\n    [Examples](https://github.com/jrzaurin/pytorch-widedeep/tree/master/examples)\n    folder in the repo\n    \"\"\"\n\n    dataloader_args, finetune_args = self._extract_kwargs(kwargs)\n\n    self.batch_size = batch_size\n\n    train_set, eval_set = wd_train_val_split(\n        self.seed,\n        self.method,  # type: ignore\n        X_wide,\n        X_tab,\n        X_text,\n        X_img,\n        X_train,\n        X_val,\n        val_split,\n        target,\n        self.transforms,\n    )\n    if custom_dataloader is not None:\n        # make sure is callable (and HAS to be an subclass of DataLoader)\n        assert isinstance(custom_dataloader, type)\n        train_loader = custom_dataloader(  # type: ignore[misc]\n            dataset=train_set,\n            batch_size=batch_size,\n            num_workers=self.num_workers,\n            **dataloader_args,\n        )\n    else:\n        train_loader = DataLoader(\n            dataset=train_set,\n            batch_size=batch_size,\n            num_workers=self.num_workers,\n            **dataloader_args,\n        )\n    train_steps = len(train_loader)\n    if eval_set is not None:\n        eval_loader = DataLoader(\n            dataset=eval_set,\n            batch_size=batch_size,\n            num_workers=self.num_workers,\n            shuffle=False,\n        )\n        eval_steps = len(eval_loader)\n\n    if finetune:\n        self.with_finetuning: bool = True\n        self._finetune(train_loader, **finetune_args)\n        if self.verbose:\n            print(\n                \"Fine-tuning (or warmup) of individual components completed. \"\n                \"Training the whole model for {} epochs\".format(n_epochs)\n            )\n    else:\n        self.with_finetuning = False\n\n    self.callback_container.on_train_begin(\n        {\"batch_size\": batch_size, \"train_steps\": train_steps, \"n_epochs\": n_epochs}\n    )\n    for epoch in range(n_epochs):\n        epoch_logs: Dict[str, float] = {}\n        self.callback_container.on_epoch_begin(epoch, logs=epoch_logs)\n\n        self.train_running_loss = 0.0\n        with trange(train_steps, disable=self.verbose != 1) as t:\n            for batch_idx, (data, targett) in zip(t, train_loader):\n                t.set_description(\"epoch %i\" % (epoch + 1))\n                train_score, train_loss = self._train_step(data, targett, batch_idx)\n                print_loss_and_metric(t, train_loss, train_score)\n                self.callback_container.on_batch_end(batch=batch_idx)\n        epoch_logs = save_epoch_logs(epoch_logs, train_loss, train_score, \"train\")\n\n        on_epoch_end_metric = None\n        if eval_set is not None and epoch % validation_freq == (\n            validation_freq - 1\n        ):\n            self.callback_container.on_eval_begin()\n            self.valid_running_loss = 0.0\n            with trange(eval_steps, disable=self.verbose != 1) as v:\n                for i, (data, targett) in zip(v, eval_loader):\n                    v.set_description(\"valid\")\n                    val_score, val_loss = self._eval_step(data, targett, i)\n                    print_loss_and_metric(v, val_loss, val_score)\n            epoch_logs = save_epoch_logs(epoch_logs, val_loss, val_score, \"val\")\n\n            if self.reducelronplateau:\n                if self.reducelronplateau_criterion == \"loss\":\n                    on_epoch_end_metric = val_loss\n                else:\n                    on_epoch_end_metric = val_score[\n                        self.reducelronplateau_criterion\n                    ]\n        else:\n            if self.reducelronplateau:\n                raise NotImplementedError(\n                    \"ReduceLROnPlateau scheduler can be used only with validation data.\"\n                )\n        self.callback_container.on_epoch_end(epoch, epoch_logs, on_epoch_end_metric)\n\n        if self.early_stop:\n            # self.callback_container.on_train_end(epoch_logs)\n            break\n\n    self.callback_container.on_train_end(epoch_logs)\n\n    if feature_importance_sample_size is not None:\n        self.feature_importance = FeatureImportance(\n            self.device, feature_importance_sample_size\n        ).feature_importance(train_loader, self.model)\n    self._restore_best_weights()\n    self.model.train()\n</code></pre>"},{"location":"pytorch-widedeep/trainer.html#pytorch_widedeep.training.Trainer.predict","title":"predict","text":"<pre><code>predict(\n    X_wide=None,\n    X_tab=None,\n    X_text=None,\n    X_img=None,\n    X_test=None,\n    batch_size=None,\n)\n</code></pre> <p>Returns the predictions</p> <p>The input datasets can be passed either directly via numpy arrays (<code>X_wide</code>, <code>X_tab</code>, <code>X_text</code> or <code>X_img</code>) or alternatively, in a dictionary (<code>X_test</code>)</p> <p>Parameters:</p> <ul> <li> <code>X_wide</code>               (<code>Optional[ndarray]</code>, default:                   <code>None</code> )           \u2013            <p>Input for the <code>wide</code> model component. See <code>pytorch_widedeep.preprocessing.WidePreprocessor</code></p> </li> <li> <code>X_tab</code>               (<code>Optional[Union[ndarray, List[ndarray]]]</code>, default:                   <code>None</code> )           \u2013            <p>Input for the <code>deeptabular</code> model component. See <code>pytorch_widedeep.preprocessing.TabPreprocessor</code></p> </li> <li> <code>X_text</code>               (<code>Optional[Union[ndarray, List[ndarray]]]</code>, default:                   <code>None</code> )           \u2013            <p>Input for the <code>deeptext</code> model component. See <code>pytorch_widedeep.preprocessing.TextPreprocessor</code></p> </li> <li> <code>X_img</code>               (<code>Optional[Union[ndarray, List[ndarray]]]</code>, default:                   <code>None</code> )           \u2013            <p>Input for the <code>deepimage</code> model component. See <code>pytorch_widedeep.preprocessing.ImagePreprocessor</code></p> </li> <li> <code>X_test</code>               (<code>Optional[Dict[str, Union[ndarray, List[ndarray]]]]</code>, default:                   <code>None</code> )           \u2013            <p>The test dataset can also be passed in a dictionary. Keys are <code>X_wide</code>, 'X_tab', 'X_text', 'X_img' and 'target'. Values are the corresponding matrices.</p> </li> <li> <code>batch_size</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>If a trainer is used to predict after having trained a model, the <code>batch_size</code> needs to be defined as it will not be defined as the <code>Trainer</code> is instantiated</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>np.ndarray:</code>           \u2013            <p>array with the predictions</p> </li> </ul> Source code in <code>pytorch_widedeep/training/trainer.py</code> <pre><code>def predict(  # type: ignore[override, return]\n    self,\n    X_wide: Optional[np.ndarray] = None,\n    X_tab: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n    X_text: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n    X_img: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n    X_test: Optional[Dict[str, Union[np.ndarray, List[np.ndarray]]]] = None,\n    batch_size: Optional[int] = None,\n) -&gt; np.ndarray:\n    r\"\"\"Returns the predictions\n\n    The input datasets can be passed either directly via numpy arrays\n    (`X_wide`, `X_tab`, `X_text` or `X_img`) or alternatively, in\n    a dictionary (`X_test`)\n\n\n    Parameters\n    ----------\n    X_wide: np.ndarray, Optional. default=None\n        Input for the `wide` model component.\n        See `pytorch_widedeep.preprocessing.WidePreprocessor`\n    X_tab: np.ndarray or List[np.ndarray], Optional. default=None\n        Input for the `deeptabular` model component.\n        See `pytorch_widedeep.preprocessing.TabPreprocessor`\n    X_text: np.ndarray or List[np.ndarray], Optional. default=None\n        Input for the `deeptext` model component.\n        See `pytorch_widedeep.preprocessing.TextPreprocessor`\n    X_img: np.ndarray or List[np.ndarray], Optional. default=None\n        Input for the `deepimage` model component.\n        See `pytorch_widedeep.preprocessing.ImagePreprocessor`\n    X_test: Dict, Optional. default=None\n        The test dataset can also be passed in a dictionary. Keys are\n        `X_wide`, _'X_tab'_, _'X_text'_, _'X_img'_ and _'target'_. Values\n        are the corresponding matrices.\n    batch_size: int, default = 256\n        If a trainer is used to predict after having trained a model, the\n        `batch_size` needs to be defined as it will not be defined as\n        the `Trainer` is instantiated\n\n    Returns\n    -------\n    np.ndarray:\n        array with the predictions\n    \"\"\"\n    preds_l = self._predict(X_wide, X_tab, X_text, X_img, X_test, batch_size)\n    if self.method == \"regression\":\n        return np.vstack(preds_l).squeeze(1)\n    if self.method == \"binary\":\n        preds = np.vstack(preds_l).squeeze(1)\n        return (preds &gt; 0.5).astype(\"int\")\n    if self.method == \"qregression\":\n        return np.vstack(preds_l)\n    if self.method == \"multiclass\":\n        preds = np.vstack(preds_l)\n        return np.argmax(preds, 1)  # type: ignore[return-value]\n    if self.method == \"multitarget\":\n        if self.loss_fn.__class__.__name__ in [\n            \"MultiTargetClassificationLoss\",\n            \"MutilTargetRegressionAndClassificationLoss\",\n        ]:\n            raise ValueError(\n                \"MultiTargetClassificationLoss and MutilTargetRegressionAndClassificationLoss \"\n                \"are not supported by predict method. Please use predict_proba method instead.\"\n            )\n        return np.vstack(preds_l)\n</code></pre>"},{"location":"pytorch-widedeep/trainer.html#pytorch_widedeep.training.Trainer.predict_uncertainty","title":"predict_uncertainty","text":"<pre><code>predict_uncertainty(\n    X_wide=None,\n    X_tab=None,\n    X_text=None,\n    X_img=None,\n    X_test=None,\n    batch_size=None,\n    uncertainty_granularity=1000,\n)\n</code></pre> <p>Returns the predicted ucnertainty of the model for the test dataset using a Monte Carlo method during which dropout layers are activated in the evaluation/prediction phase and each sample is predicted N times (<code>uncertainty_granularity</code> times).</p> <p>This is based on Dropout as a Bayesian Approximation: Representing Model Uncertainty in Deep Learning.</p> <p>Parameters:</p> <ul> <li> <code>X_wide</code>               (<code>Optional[ndarray]</code>, default:                   <code>None</code> )           \u2013            <p>Input for the <code>wide</code> model component. See <code>pytorch_widedeep.preprocessing.WidePreprocessor</code></p> </li> <li> <code>X_tab</code>               (<code>Optional[Union[ndarray, List[ndarray]]]</code>, default:                   <code>None</code> )           \u2013            <p>Input for the <code>deeptabular</code> model component. See <code>pytorch_widedeep.preprocessing.TabPreprocessor</code></p> </li> <li> <code>X_text</code>               (<code>Optional[Union[ndarray, List[ndarray]]]</code>, default:                   <code>None</code> )           \u2013            <p>Input for the <code>deeptext</code> model component. See <code>pytorch_widedeep.preprocessing.TextPreprocessor</code></p> </li> <li> <code>X_img</code>               (<code>Optional[Union[ndarray, List[ndarray]]]</code>, default:                   <code>None</code> )           \u2013            <p>Input for the <code>deepimage</code> model component. See <code>pytorch_widedeep.preprocessing.ImagePreprocessor</code></p> </li> <li> <code>X_test</code>               (<code>Optional[Dict[str, Union[ndarray, List[ndarray]]]]</code>, default:                   <code>None</code> )           \u2013            <p>The test dataset can also be passed in a dictionary. Keys are 'X_wide', 'X_tab', 'X_text', 'X_img' and 'target'. Values are the corresponding matrices.</p> </li> <li> <code>batch_size</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>If a trainer is used to predict after having trained a model, the <code>batch_size</code> needs to be defined as it will not be defined as the <code>Trainer</code> is instantiated</p> </li> <li> <code>uncertainty_granularity</code>           \u2013            <p>number of times the model does prediction for each sample</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>np.ndarray:</code>           \u2013            <ul> <li>if <code>method = regression</code>, it will return an array with <code>(max, min, mean, stdev)</code>   values for each sample.</li> <li>if <code>method = binary</code> it will return an array with   <code>(mean_cls_0_prob, mean_cls_1_prob, predicted_cls)</code> for each sample.</li> <li>if <code>method = multiclass</code> it will return an array with   <code>(mean_cls_0_prob, mean_cls_1_prob, mean_cls_2_prob, ... , predicted_cls)</code>   values for each sample.</li> </ul> </li> </ul> Source code in <code>pytorch_widedeep/training/trainer.py</code> <pre><code>def predict_uncertainty(  # type: ignore[return]\n    self,\n    X_wide: Optional[np.ndarray] = None,\n    X_tab: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n    X_text: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n    X_img: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n    X_test: Optional[Dict[str, Union[np.ndarray, List[np.ndarray]]]] = None,\n    batch_size: Optional[int] = None,\n    uncertainty_granularity=1000,\n) -&gt; np.ndarray:\n    r\"\"\"Returns the predicted ucnertainty of the model for the test dataset\n    using a Monte Carlo method during which dropout layers are activated\n    in the evaluation/prediction phase and each sample is predicted N\n    times (`uncertainty_granularity` times).\n\n    This is based on\n    [Dropout as a Bayesian Approximation: Representing\n    Model Uncertainty in Deep Learning](https://arxiv.org/abs/1506.02142?context=stat).\n\n    Parameters\n    ----------\n    X_wide: np.ndarray, Optional. default=None\n        Input for the `wide` model component.\n        See `pytorch_widedeep.preprocessing.WidePreprocessor`\n    X_tab: np.ndarray or List[np.ndarray], Optional. default=None\n        Input for the `deeptabular` model component.\n        See `pytorch_widedeep.preprocessing.TabPreprocessor`\n    X_text: np.ndarray or List[np.ndarray], Optional. default=None\n        Input for the `deeptext` model component.\n        See `pytorch_widedeep.preprocessing.TextPreprocessor`\n    X_img: np.ndarray or List[np.ndarray], Optional. default=None\n        Input for the `deepimage` model component.\n        See `pytorch_widedeep.preprocessing.ImagePreprocessor`\n    X_test: Dict, Optional. default=None\n        The test dataset can also be passed in a dictionary. Keys are\n        _'X_wide'_, _'X_tab'_, _'X_text'_, _'X_img'_ and _'target'_. Values\n        are the corresponding matrices.\n    batch_size: int, default = 256\n        If a trainer is used to predict after having trained a model, the\n        `batch_size` needs to be defined as it will not be defined as\n        the `Trainer` is instantiated\n    uncertainty_granularity: int default = 1000\n        number of times the model does prediction for each sample\n\n    Returns\n    -------\n    np.ndarray:\n        - if `method = regression`, it will return an array with `(max, min, mean, stdev)`\n          values for each sample.\n        - if `method = binary` it will return an array with\n          `(mean_cls_0_prob, mean_cls_1_prob, predicted_cls)` for each sample.\n        - if `method = multiclass` it will return an array with\n          `(mean_cls_0_prob, mean_cls_1_prob, mean_cls_2_prob, ... , predicted_cls)`\n          values for each sample.\n\n    \"\"\"\n    preds_l = self._predict(\n        X_wide,\n        X_tab,\n        X_text,\n        X_img,\n        X_test,\n        batch_size,\n        uncertainty_granularity,\n        uncertainty=True,\n    )\n    preds = np.vstack(preds_l)\n    samples_num = int(preds.shape[0] / uncertainty_granularity)\n    if self.method == \"regression\":\n        preds = preds.squeeze(1)\n        preds = preds.reshape((uncertainty_granularity, samples_num))\n        return np.array(\n            (\n                preds.max(axis=0),\n                preds.min(axis=0),\n                preds.mean(axis=0),\n                preds.std(axis=0),\n            )\n        ).T\n    if self.method == \"qregression\":\n        raise ValueError(\n            \"Currently predict_uncertainty is not supported for qregression method\"\n        )\n    if self.method == \"binary\":\n        preds = preds.squeeze(1)\n        preds = preds.reshape((uncertainty_granularity, samples_num))\n        preds = preds.mean(axis=0)\n        probs = np.zeros([preds.shape[0], 3])\n        probs[:, 0] = 1 - preds\n        probs[:, 1] = preds\n        return probs\n    if self.method == \"multiclass\":\n        preds = preds.reshape(uncertainty_granularity, samples_num, preds.shape[1])\n        preds = preds.mean(axis=0)\n        preds = np.hstack((preds, np.vstack(np.argmax(preds, 1))))\n        return preds\n\n    if self.method == \"multitarget\":\n        raise ValueError(\n            \"Currently predict_uncertainty is not supported for multitarget method\"\n        )\n</code></pre>"},{"location":"pytorch-widedeep/trainer.html#pytorch_widedeep.training.Trainer.predict_proba","title":"predict_proba","text":"<pre><code>predict_proba(\n    X_wide=None,\n    X_tab=None,\n    X_text=None,\n    X_img=None,\n    X_test=None,\n    batch_size=None,\n)\n</code></pre> <p>Returns the predicted probabilities for the test dataset for  binary and multiclass methods</p> <p>The input datasets can be passed either directly via numpy arrays (<code>X_wide</code>, <code>X_tab</code>, <code>X_text</code> or <code>X_img</code>) or alternatively, in a dictionary (<code>X_test</code>)</p> <p>Parameters:</p> <ul> <li> <code>X_wide</code>               (<code>Optional[ndarray]</code>, default:                   <code>None</code> )           \u2013            <p>Input for the <code>wide</code> model component. See <code>pytorch_widedeep.preprocessing.WidePreprocessor</code></p> </li> <li> <code>X_tab</code>               (<code>Optional[Union[ndarray, List[ndarray]]]</code>, default:                   <code>None</code> )           \u2013            <p>Input for the <code>deeptabular</code> model component. See <code>pytorch_widedeep.preprocessing.TabPreprocessor</code></p> </li> <li> <code>X_text</code>               (<code>Optional[Union[ndarray, List[ndarray]]]</code>, default:                   <code>None</code> )           \u2013            <p>Input for the <code>deeptext</code> model component. See <code>pytorch_widedeep.preprocessing.TextPreprocessor</code></p> </li> <li> <code>X_img</code>               (<code>Optional[Union[ndarray, List[ndarray]]]</code>, default:                   <code>None</code> )           \u2013            <p>Input for the <code>deepimage</code> model component. See <code>pytorch_widedeep.preprocessing.ImagePreprocessor</code></p> </li> <li> <code>X_test</code>               (<code>Optional[Dict[str, Union[ndarray, List[ndarray]]]]</code>, default:                   <code>None</code> )           \u2013            <p>The test dataset can also be passed in a dictionary. Keys are <code>X_wide</code>, 'X_tab', 'X_text', 'X_img' and 'target'. Values are the corresponding matrices.</p> </li> <li> <code>batch_size</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>If a trainer is used to predict after having trained a model, the <code>batch_size</code> needs to be defined as it will not be defined as the <code>Trainer</code> is instantiated</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>ndarray</code>           \u2013            <p>array with the probabilities per class</p> </li> </ul> Source code in <code>pytorch_widedeep/training/trainer.py</code> <pre><code>def predict_proba(  # type: ignore[override, return]  # noqa: C901\n    self,\n    X_wide: Optional[np.ndarray] = None,\n    X_tab: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n    X_text: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n    X_img: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n    X_test: Optional[Dict[str, Union[np.ndarray, List[np.ndarray]]]] = None,\n    batch_size: Optional[int] = None,\n) -&gt; np.ndarray:\n    r\"\"\"Returns the predicted probabilities for the test dataset for  binary\n    and multiclass methods\n\n    The input datasets can be passed either directly via numpy arrays\n    (`X_wide`, `X_tab`, `X_text` or `X_img`) or alternatively, in\n    a dictionary (`X_test`)\n\n    Parameters\n    ----------\n    X_wide: np.ndarray, Optional. default=None\n        Input for the `wide` model component.\n        See `pytorch_widedeep.preprocessing.WidePreprocessor`\n    X_tab: np.ndarray or List[np.ndarray], Optional. default=None\n        Input for the `deeptabular` model component.\n        See `pytorch_widedeep.preprocessing.TabPreprocessor`\n    X_text: np.ndarray or List[np.ndarray], Optional. default=None\n        Input for the `deeptext` model component.\n        See `pytorch_widedeep.preprocessing.TextPreprocessor`\n    X_img: np.ndarray or List[np.ndarray], Optional. default=None\n        Input for the `deepimage` model component.\n        See `pytorch_widedeep.preprocessing.ImagePreprocessor`\n    X_test: Dict, Optional. default=None\n        The test dataset can also be passed in a dictionary. Keys are\n        `X_wide`, _'X_tab'_, _'X_text'_, _'X_img'_ and _'target'_. Values\n        are the corresponding matrices.\n    batch_size: int, default = 256\n        If a trainer is used to predict after having trained a model, the\n        `batch_size` needs to be defined as it will not be defined as\n        the `Trainer` is instantiated\n\n    Returns\n    -------\n    np.ndarray\n        array with the probabilities per class\n    \"\"\"\n\n    preds_l = self._predict(X_wide, X_tab, X_text, X_img, X_test, batch_size)\n    if self.method == \"binary\":\n        preds = np.vstack(preds_l).squeeze(1)\n        probs = np.zeros([preds.shape[0], 2])\n        probs[:, 0] = 1 - preds\n        probs[:, 1] = preds\n        return probs\n    if self.method == \"multiclass\":\n        return np.vstack(preds_l)\n    if self.method == \"multitarget\":\n        return np.vstack(preds_l)\n</code></pre>"},{"location":"pytorch-widedeep/trainer.html#pytorch_widedeep.training.Trainer.save","title":"save","text":"<pre><code>save(\n    path,\n    save_state_dict=False,\n    save_optimizer=False,\n    model_filename=\"wd_model.pt\",\n)\n</code></pre> <p>Saves the model, training and evaluation history, and the <code>feature_importance</code> attribute (if the <code>deeptabular</code> component is a Tabnet model) to disk</p> <p>The <code>Trainer</code> class is built so that it 'just' trains a model. With that in mind, all the torch related parameters (such as optimizers, learning rate schedulers, initializers, etc) have to be defined externally and then passed to the <code>Trainer</code>. As a result, the <code>Trainer</code> does not generate any attribute or additional data products that need to be saved other than the <code>model</code> object itself, which can be saved as any other torch model (e.g. <code>torch.save(model, path)</code>).</p> <p>The exception is Tabnet. If the <code>deeptabular</code> component is a Tabnet model, an attribute (a dict) called <code>feature_importance</code> will be created at the end of the training process. Therefore, a <code>save</code> method was created that will save the feature importance dictionary to a json file and, since we are here, the model weights, training history and learning rate history.</p> <p>Parameters:</p> <ul> <li> <code>path</code>               (<code>str</code>)           \u2013            <p>path to the directory where the model and the feature importance attribute will be saved.</p> </li> <li> <code>save_state_dict</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether to save directly the model (and optimizer) or the model's (and optimizer's) state dictionary</p> </li> <li> <code>save_optimizer</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether to save the optimizer</p> </li> <li> <code>model_filename</code>               (<code>str</code>, default:                   <code>'wd_model.pt'</code> )           \u2013            <p>filename where the model weights will be store</p> </li> </ul> Source code in <code>pytorch_widedeep/training/trainer.py</code> <pre><code>def save(\n    self,\n    path: str,\n    save_state_dict: bool = False,\n    save_optimizer: bool = False,\n    model_filename: str = \"wd_model.pt\",\n):\n    r\"\"\"Saves the model, training and evaluation history, and the\n    `feature_importance` attribute (if the `deeptabular` component is a\n    Tabnet model) to disk\n\n    The `Trainer` class is built so that it 'just' trains a model. With\n    that in mind, all the torch related parameters (such as optimizers,\n    learning rate schedulers, initializers, etc) have to be defined\n    externally and then passed to the `Trainer`. As a result, the\n    `Trainer` does not generate any attribute or additional data\n    products that need to be saved other than the `model` object itself,\n    which can be saved as any other torch model (e.g. `torch.save(model,\n    path)`).\n\n    The exception is Tabnet. If the `deeptabular` component is a Tabnet\n    model, an attribute (a dict) called `feature_importance` will be\n    created at the end of the training process. Therefore, a `save`\n    method was created that will save the feature importance dictionary\n    to a json file and, since we are here, the model weights, training\n    history and learning rate history.\n\n    Parameters\n    ----------\n    path: str\n        path to the directory where the model and the feature importance\n        attribute will be saved.\n    save_state_dict: bool, default = False\n        Boolean indicating whether to save directly the model\n        (and optimizer) or the model's (and optimizer's) state\n        dictionary\n    save_optimizer: bool, default = False\n        Boolean indicating whether to save the optimizer\n    model_filename: str, Optional, default = \"wd_model.pt\"\n        filename where the model weights will be store\n    \"\"\"\n\n    self._save_history(path)\n\n    self._save_model_and_optimizer(\n        path, save_state_dict, save_optimizer, model_filename\n    )\n\n    if self.model.is_tabnet:\n        with open(Path(path) / \"feature_importance.json\", \"w\") as fi:\n            json.dump(self.feature_importance, fi)\n</code></pre>"},{"location":"pytorch-widedeep/utils/index.html","title":"The <code>utils</code> module","text":"<p>These are a series of utilities that might be useful for a number of preprocessing tasks, even not directly related to <code>pytorch-widedeep</code>. All the classes and functions discussed here are available directly from the <code>utils</code> module. For example, the <code>LabelEncoder</code> within the <code>deeptabular_utils</code> submodule can be imported as:</p> <pre><code>from pytorch_widedeep.utils import LabelEncoder\n</code></pre> <p>These are classes and functions that are internally used in the library. We include them here in case the user finds them useful for other purposes.</p>"},{"location":"pytorch-widedeep/utils/deeptabular_utils.html","title":"deeptabular utils","text":""},{"location":"pytorch-widedeep/utils/deeptabular_utils.html#pytorch_widedeep.utils.deeptabular_utils.LabelEncoder","title":"LabelEncoder","text":"<pre><code>LabelEncoder(\n    columns_to_encode=None,\n    with_attention=False,\n    shared_embed=False,\n)\n</code></pre> <p>Label Encode categorical values for multiple columns at once</p> <p> NOTE: LabelEncoder reserves 0 for <code>unseen</code> new categories. This is convenient when defining the embedding layers, since we can just set padding idx to 0.</p> <p>Parameters:</p> <ul> <li> <code>columns_to_encode</code>               (<code>Optional[List[str]]</code>, default:                   <code>None</code> )           \u2013            <p>List of strings containing the names of the columns to encode. If <code>None</code> all columns of type <code>object</code> in the dataframe will be label encoded.</p> </li> <li> <code>with_attention</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether the preprocessed data will be passed to an attention-based model. Aliased as <code>for_transformer</code>.</p> </li> <li> <code>shared_embed</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating if the embeddings will be \"shared\" when using attention-based models. The idea behind <code>shared_embed</code> is described in the Appendix A in the TabTransformer paper: 'The goal of having column embedding is to enable the model to distinguish the classes in one column from those in the other columns'. In other words, the idea is to let the model learn which column is embedded at the time. See: <code>pytorch_widedeep.models.transformers._layers.SharedEmbeddings</code>.</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>encoding_dict</code>               (<code>Dict</code>)           \u2013            <p>Dictionary containing the encoding mappings in the format, e.g. :  <code>{'colname1': {'cat1': 1, 'cat2': 2, ...}, 'colname2': {'cat1': 1, 'cat2': 2, ...}, ...}</code></p> </li> <li> <code>inverse_encoding_dict</code>               (<code>Dict</code>)           \u2013            <p>Dictionary containing the inverse encoding mappings in the format, e.g. :  <code>{'colname1': {1: 'cat1', 2: 'cat2', ...}, 'colname2': {1: 'cat1', 2: 'cat2', ...}, ...}</code></p> </li> </ul> Source code in <code>pytorch_widedeep/utils/deeptabular_utils.py</code> <pre><code>@alias(\"with_attention\", [\"for_transformer\"])\ndef __init__(\n    self,\n    columns_to_encode: Optional[List[str]] = None,\n    with_attention: bool = False,\n    shared_embed: bool = False,\n):\n    self.columns_to_encode = columns_to_encode\n\n    self.shared_embed = shared_embed\n    self.with_attention = with_attention\n\n    self.reset_embed_idx = not self.with_attention or self.shared_embed\n</code></pre>"},{"location":"pytorch-widedeep/utils/deeptabular_utils.html#pytorch_widedeep.utils.deeptabular_utils.LabelEncoder.partial_fit","title":"partial_fit","text":"<pre><code>partial_fit(df)\n</code></pre> <p>Main method. Creates encoding attributes.</p> <p>Returns:</p> <ul> <li> <code>LabelEncoder</code>           \u2013            <p><code>LabelEncoder</code> fitted object</p> </li> </ul> Source code in <code>pytorch_widedeep/utils/deeptabular_utils.py</code> <pre><code>def partial_fit(self, df: pd.DataFrame) -&gt; \"LabelEncoder\":  # noqa: C901\n    \"\"\"Main method. Creates encoding attributes.\n\n    Returns\n    -------\n    LabelEncoder\n        `LabelEncoder` fitted object\n    \"\"\"\n    # here df is a chunk of the data. this is meant to be run when the\n    # data is large and we pass a chunk at a time. Therefore, we do not\n    # copy the input chunk as mutating a chunk is ok\n    if self.columns_to_encode is None:\n        self.columns_to_encode = list(df.select_dtypes(include=[\"object\"]).columns)\n    else:\n        # sanity check to make sure all categorical columns are in an adequate\n        # format\n        for col in self.columns_to_encode:\n            df[col] = df[col].astype(\"O\")\n\n    unique_column_vals: Dict[str, List[str]] = {}\n    for c in self.columns_to_encode:\n        unique_column_vals[c] = df[c].unique().tolist()\n\n    if not hasattr(self, \"encoding_dict\"):\n        # we run the method 'partial_fit' for the 1st time\n        self.encoding_dict: Dict[str, Dict[str, int]] = {}\n        if \"cls_token\" in unique_column_vals and self.shared_embed:\n            self.encoding_dict[\"cls_token\"] = {\"[CLS]\": 0}\n            del unique_column_vals[\"cls_token\"]\n\n        # leave 0 for padding/\"unseen\" categories. Also we need an\n        # attribute to keep track of the encoding in case we use\n        # attention and we do not re-start the index/counter\n        self.cum_idx: int = 1\n        for k, v in unique_column_vals.items():\n            self.encoding_dict[k] = {o: i + self.cum_idx for i, o in enumerate(v)}\n            self.cum_idx = 1 if self.reset_embed_idx else self.cum_idx + len(v)\n    else:\n        # the 'partial_fit' method has already run.\n        # \"cls_token\" will have been added already\n        if \"cls_token\" in unique_column_vals and self.shared_embed:\n            del unique_column_vals[\"cls_token\"]\n\n        # Classes in the new df/chunk of the dataset that have not been seen\n        # before\n        unseen_classes: Dict[str, List[str]] = {}\n        for c in self.columns_to_encode:\n            unseen_classes[c] = list(\n                np.setdiff1d(\n                    unique_column_vals[c], list(self.encoding_dict[c].keys())\n                )\n            )\n\n        # leave 0 for padding/\"unseen\" categories\n        for k, v in unique_column_vals.items():\n            # if we use attention we need to start encoding from the\n            # last 'overall' encoding index. Otherwise, we use the max\n            # encoding index per categorical col\n            _idx = (\n                max(self.encoding_dict[k].values()) + 1\n                if self.reset_embed_idx\n                else self.cum_idx\n            )\n            if len(unseen_classes[k]) != 0:\n                for i, o in enumerate(unseen_classes[k]):\n                    if o not in self.encoding_dict[k]:\n                        self.encoding_dict[k][o] = i + _idx\n                # if self.reset_embed_idx is True it will be 1 anyway\n                self.cum_idx = (\n                    1\n                    if self.reset_embed_idx\n                    else self.cum_idx + len(unseen_classes[k])\n                )\n\n    return self\n</code></pre>"},{"location":"pytorch-widedeep/utils/deeptabular_utils.html#pytorch_widedeep.utils.deeptabular_utils.LabelEncoder.fit","title":"fit","text":"<pre><code>fit(df)\n</code></pre> <p>Simply runs the <code>partial_fit</code> method when the data fits in memory</p> <p>Returns:</p> <ul> <li> <code>LabelEncoder</code>           \u2013            <p><code>LabelEncoder</code> fitted object</p> </li> </ul> Source code in <code>pytorch_widedeep/utils/deeptabular_utils.py</code> <pre><code>def fit(self, df: pd.DataFrame) -&gt; \"LabelEncoder\":\n    \"\"\"Simply runs the `partial_fit` method when the data fits in memory\n\n    Returns\n    -------\n    LabelEncoder\n        `LabelEncoder` fitted object\n    \"\"\"\n    # this is meant to be run when the data fits in memory and therefore,\n    # we do not want to mutate the original df, so we copy it\n    self.partial_fit(df.copy())\n\n    self.inverse_encoding_dict = self.create_inverse_encoding_dict()\n\n    return self\n</code></pre>"},{"location":"pytorch-widedeep/utils/deeptabular_utils.html#pytorch_widedeep.utils.deeptabular_utils.LabelEncoder.transform","title":"transform","text":"<pre><code>transform(df)\n</code></pre> <p>Label Encoded the categories in <code>columns_to_encode</code></p> <p>Returns:</p> <ul> <li> <code>DataFrame</code>           \u2013            <p>label-encoded dataframe</p> </li> </ul> Source code in <code>pytorch_widedeep/utils/deeptabular_utils.py</code> <pre><code>def transform(self, df: pd.DataFrame) -&gt; pd.DataFrame:\n    \"\"\"Label Encoded the categories in `columns_to_encode`\n\n    Returns\n    -------\n    pd.DataFrame\n        label-encoded dataframe\n    \"\"\"\n    try:\n        self.encoding_dict\n    except AttributeError:\n        raise NotFittedError(\n            \"This LabelEncoder instance is not fitted yet. \"\n            \"Call 'fit' with appropriate arguments before using this LabelEncoder.\"\n        )\n\n    df_inp = df.copy()\n    # sanity check to make sure all categorical columns are in an adequate\n    # format\n    for col in self.columns_to_encode:  # type: ignore\n        df_inp[col] = df_inp[col].astype(\"O\")\n\n    for k, v in self.encoding_dict.items():\n        df_inp[k] = df_inp[k].apply(lambda x: v[x] if x in v.keys() else 0)\n\n    return df_inp\n</code></pre>"},{"location":"pytorch-widedeep/utils/deeptabular_utils.html#pytorch_widedeep.utils.deeptabular_utils.LabelEncoder.fit_transform","title":"fit_transform","text":"<pre><code>fit_transform(df)\n</code></pre> <p>Combines <code>fit</code> and <code>transform</code></p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import pandas as pd\n&gt;&gt;&gt; from pytorch_widedeep.utils import LabelEncoder\n&gt;&gt;&gt; df = pd.DataFrame({'col1': [1,2,3], 'col2': ['me', 'you', 'him']})\n&gt;&gt;&gt; columns_to_encode = ['col2']\n&gt;&gt;&gt; encoder = LabelEncoder(columns_to_encode)\n&gt;&gt;&gt; encoder.fit_transform(df)\n   col1  col2\n0     1     1\n1     2     2\n2     3     3\n&gt;&gt;&gt; encoder.encoding_dict\n{'col2': {'me': 1, 'you': 2, 'him': 3}}\n</code></pre> <p>Returns:</p> <ul> <li> <code>DataFrame</code>           \u2013            <p>label-encoded dataframe</p> </li> </ul> Source code in <code>pytorch_widedeep/utils/deeptabular_utils.py</code> <pre><code>def fit_transform(self, df: pd.DataFrame) -&gt; pd.DataFrame:\n    \"\"\"Combines `fit` and `transform`\n\n    Examples\n    --------\n\n    &gt;&gt;&gt; import pandas as pd\n    &gt;&gt;&gt; from pytorch_widedeep.utils import LabelEncoder\n    &gt;&gt;&gt; df = pd.DataFrame({'col1': [1,2,3], 'col2': ['me', 'you', 'him']})\n    &gt;&gt;&gt; columns_to_encode = ['col2']\n    &gt;&gt;&gt; encoder = LabelEncoder(columns_to_encode)\n    &gt;&gt;&gt; encoder.fit_transform(df)\n       col1  col2\n    0     1     1\n    1     2     2\n    2     3     3\n    &gt;&gt;&gt; encoder.encoding_dict\n    {'col2': {'me': 1, 'you': 2, 'him': 3}}\n\n    Returns\n    -------\n    pd.DataFrame\n        label-encoded dataframe\n    \"\"\"\n    return self.fit(df).transform(df)\n</code></pre>"},{"location":"pytorch-widedeep/utils/deeptabular_utils.html#pytorch_widedeep.utils.deeptabular_utils.LabelEncoder.inverse_transform","title":"inverse_transform","text":"<pre><code>inverse_transform(df)\n</code></pre> <p>Returns the original categories</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import pandas as pd\n&gt;&gt;&gt; from pytorch_widedeep.utils import LabelEncoder\n&gt;&gt;&gt; df = pd.DataFrame({'col1': [1,2,3], 'col2': ['me', 'you', 'him']})\n&gt;&gt;&gt; columns_to_encode = ['col2']\n&gt;&gt;&gt; encoder = LabelEncoder(columns_to_encode)\n&gt;&gt;&gt; df_enc = encoder.fit_transform(df)\n&gt;&gt;&gt; encoder.inverse_transform(df_enc)\n   col1 col2\n0     1   me\n1     2  you\n2     3  him\n</code></pre> <p>Returns:</p> <ul> <li> <code>DataFrame</code>           \u2013            <p>DataFrame with original categories</p> </li> </ul> Source code in <code>pytorch_widedeep/utils/deeptabular_utils.py</code> <pre><code>def inverse_transform(self, df: pd.DataFrame) -&gt; pd.DataFrame:\n    \"\"\"Returns the original categories\n\n    Examples\n    --------\n\n    &gt;&gt;&gt; import pandas as pd\n    &gt;&gt;&gt; from pytorch_widedeep.utils import LabelEncoder\n    &gt;&gt;&gt; df = pd.DataFrame({'col1': [1,2,3], 'col2': ['me', 'you', 'him']})\n    &gt;&gt;&gt; columns_to_encode = ['col2']\n    &gt;&gt;&gt; encoder = LabelEncoder(columns_to_encode)\n    &gt;&gt;&gt; df_enc = encoder.fit_transform(df)\n    &gt;&gt;&gt; encoder.inverse_transform(df_enc)\n       col1 col2\n    0     1   me\n    1     2  you\n    2     3  him\n\n    Returns\n    -------\n    pd.DataFrame\n        DataFrame with original categories\n    \"\"\"\n\n    if not hasattr(self, \"inverse_encoding_dict\"):\n        self.inverse_encoding_dict = self.create_inverse_encoding_dict()\n\n    for k, v in self.inverse_encoding_dict.items():\n        df[k] = df[k].apply(lambda x: v[x])\n\n    return df\n</code></pre>"},{"location":"pytorch-widedeep/utils/fastai_transforms.html","title":"Fastai transforms","text":"<p>I directly copied and pasted part of the <code>transforms.py</code> module from the <code>fastai</code> library (from an old version). The reason to do such a thing is because <code>pytorch_widedeep</code> only needs the <code>Tokenizer</code> and the <code>Vocab</code> classes there. This way I avoid extra dependencies. Credit for all the code in the <code>fastai_transforms</code> module in this <code>pytorch-widedeep</code> package goes to Jeremy Howard and the <code>fastai</code> team. I only include the documentation here for completion, but I strongly advise the user to read the <code>fastai</code> documentation.</p>"},{"location":"pytorch-widedeep/utils/fastai_transforms.html#pytorch_widedeep.utils.fastai_transforms.Tokenizer","title":"Tokenizer","text":"<pre><code>Tokenizer(\n    tok_func=SpacyTokenizer,\n    lang=\"en\",\n    pre_rules=None,\n    post_rules=None,\n    special_cases=None,\n    n_cpus=None,\n)\n</code></pre> <p>Class to combine a series of rules and a tokenizer function to tokenize text with multiprocessing.</p> <p>Setting some of the parameters of this class require perhaps some familiarity with the source code.</p> <p>Parameters:</p> <ul> <li> <code>tok_func</code>               (<code>Callable</code>, default:                   <code>SpacyTokenizer</code> )           \u2013            <p>Tokenizer Object. See <code>pytorch_widedeep.utils.fastai_transforms.SpacyTokenizer</code></p> </li> <li> <code>lang</code>               (<code>str</code>, default:                   <code>'en'</code> )           \u2013            <p>Text's Language</p> </li> <li> <code>pre_rules</code>               (<code>Optional[ListRules]</code>, default:                   <code>None</code> )           \u2013            <p>Custom type: <code>Collection[Callable[[str], str]]</code>. These are <code>Callable</code> objects that will be applied to the text (str) directly as <code>rule(tok)</code> before being tokenized.</p> </li> <li> <code>post_rules</code>               (<code>Optional[ListRules]</code>, default:                   <code>None</code> )           \u2013            <p>Custom type: <code>Collection[Callable[[str], str]]</code>. These are <code>Callable</code> objects that will be applied to the tokens as <code>rule(tokens)</code> after the text has been tokenized.</p> </li> <li> <code>special_cases</code>               (<code>Optional[Collection[str]]</code>, default:                   <code>None</code> )           \u2013            <p>special cases to be added to the tokenizer via <code>Spacy</code>'s <code>add_special_case</code> method</p> </li> <li> <code>n_cpus</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>number of CPUs to used during the tokenization process</p> </li> </ul> Source code in <code>pytorch_widedeep/utils/fastai_transforms.py</code> <pre><code>def __init__(\n    self,\n    tok_func: Callable = SpacyTokenizer,\n    lang: str = \"en\",\n    pre_rules: Optional[ListRules] = None,\n    post_rules: Optional[ListRules] = None,\n    special_cases: Optional[Collection[str]] = None,\n    n_cpus: Optional[int] = None,\n):\n    self.tok_func, self.lang, self.special_cases = tok_func, lang, special_cases\n    self.pre_rules = ifnone(pre_rules, defaults.text_pre_rules)\n    self.post_rules = ifnone(post_rules, defaults.text_post_rules)\n    self.special_cases = (\n        special_cases if special_cases is not None else defaults.text_spec_tok\n    )\n    self.n_cpus = ifnone(n_cpus, defaults.cpus)\n</code></pre>"},{"location":"pytorch-widedeep/utils/fastai_transforms.html#pytorch_widedeep.utils.fastai_transforms.Tokenizer.process_text","title":"process_text","text":"<pre><code>process_text(t, tok)\n</code></pre> <p>Process and tokenize one text <code>t</code> with tokenizer <code>tok</code>.</p> <p>Parameters:</p> <ul> <li> <code>t</code>               (<code>str</code>)           \u2013            <p>text to be processed and tokenized</p> </li> <li> <code>tok</code>               (<code>BaseTokenizer</code>)           \u2013            <p>Instance of <code>BaseTokenizer</code>. See <code>pytorch_widedeep.utils.fastai_transforms.BaseTokenizer</code></p> </li> </ul> <p>Returns:</p> <ul> <li> <code>List[str]</code>           \u2013            <p>List of tokens</p> </li> </ul> Source code in <code>pytorch_widedeep/utils/fastai_transforms.py</code> <pre><code>def process_text(self, t: str, tok: BaseTokenizer) -&gt; List[str]:\n    r\"\"\"Process and tokenize one text ``t`` with tokenizer ``tok``.\n\n    Parameters\n    ----------\n    t: str\n        text to be processed and tokenized\n    tok: ``BaseTokenizer``\n        Instance of `BaseTokenizer`. See\n        `pytorch_widedeep.utils.fastai_transforms.BaseTokenizer`\n\n    Returns\n    -------\n    List[str]\n        List of tokens\n    \"\"\"\n    for rule in self.pre_rules:\n        t = rule(t)\n    toks = tok.tokenizer(t)\n    for rule in self.post_rules:\n        toks = rule(toks)\n    return toks\n</code></pre>"},{"location":"pytorch-widedeep/utils/fastai_transforms.html#pytorch_widedeep.utils.fastai_transforms.Tokenizer.process_all","title":"process_all","text":"<pre><code>process_all(texts)\n</code></pre> <p>Process a list of texts. Parallel execution of <code>process_text</code>.</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; from pytorch_widedeep.utils import Tokenizer\n&gt;&gt;&gt; texts = ['Machine learning is great', 'but building stuff is even better']\n&gt;&gt;&gt; tok = Tokenizer()\n&gt;&gt;&gt; tok.process_all(texts)\n[['xxmaj', 'machine', 'learning', 'is', 'great'], ['but', 'building', 'stuff', 'is', 'even', 'better']]\n</code></pre> <p> NOTE: Note the token <code>TK_MAJ</code> (<code>xxmaj</code>), used to indicate the next word begins with a capital in the original text. For more details of special tokens please see the <code>fastai</code> docs.</p> <p>Returns:</p> <ul> <li> <code>List[List[str]]</code>           \u2013            <p>List containing lists of tokens. One list per \"document\"</p> </li> </ul> Source code in <code>pytorch_widedeep/utils/fastai_transforms.py</code> <pre><code>def process_all(self, texts: Collection[str]) -&gt; List[List[str]]:\n    r\"\"\"Process a list of texts. Parallel execution of ``process_text``.\n\n    Examples\n    --------\n    &gt;&gt;&gt; from pytorch_widedeep.utils import Tokenizer\n    &gt;&gt;&gt; texts = ['Machine learning is great', 'but building stuff is even better']\n    &gt;&gt;&gt; tok = Tokenizer()\n    &gt;&gt;&gt; tok.process_all(texts)\n    [['xxmaj', 'machine', 'learning', 'is', 'great'], ['but', 'building', 'stuff', 'is', 'even', 'better']]\n\n    :information_source: **NOTE**:\n    Note the token ``TK_MAJ`` (`xxmaj`), used to indicate the\n    next word begins with a capital in the original text. For more\n    details of special tokens please see the [``fastai`` docs](https://docs.fast.ai/text.core.html#Tokenizing).\n\n    Returns\n    -------\n    List[List[str]]\n        List containing lists of tokens. One list per \"_document_\"\n\n    \"\"\"\n\n    if self.n_cpus &lt;= 1:\n        return self._process_all_1(texts)\n    with ProcessPoolExecutor(self.n_cpus) as e:\n        return sum(\n            e.map(self._process_all_1, partition_by_cores(texts, self.n_cpus)), []\n        )\n</code></pre>"},{"location":"pytorch-widedeep/utils/fastai_transforms.html#pytorch_widedeep.utils.fastai_transforms.Vocab","title":"Vocab","text":"<pre><code>Vocab(\n    max_vocab, min_freq, pad_idx=None, special_cases=None\n)\n</code></pre> <p>Contains the correspondence between numbers and tokens.</p> <p>Parameters:</p> <ul> <li> <code>max_vocab</code>               (<code>int</code>)           \u2013            <p>maximum vocabulary size</p> </li> <li> <code>min_freq</code>               (<code>int</code>)           \u2013            <p>minimum frequency for a token to be considereds</p> </li> <li> <code>pad_idx</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>padding index. If <code>None</code>, Fastai's Tokenizer leaves the 0 index for the unknown token ('xxunk') and defaults to 1 for the padding token ('xxpad').</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>itos</code>               (<code>Collection</code>)           \u2013            <p><code>index to str</code>. Collection of strings that are the tokens of the vocabulary</p> </li> <li> <code>stoi</code>               (<code>defaultdict</code>)           \u2013            <p><code>str to index</code>. Dictionary containing the tokens of the vocabulary and their corresponding index</p> </li> </ul> Source code in <code>pytorch_widedeep/utils/fastai_transforms.py</code> <pre><code>def __init__(\n    self,\n    max_vocab: int,\n    min_freq: int,\n    pad_idx: Optional[int] = None,\n    special_cases: Optional[Collection[str]] = None,\n):\n    self.max_vocab = max_vocab\n    self.min_freq = min_freq\n    self.pad_idx = pad_idx\n    self.special_cases = (\n        special_cases if special_cases is not None else defaults.text_spec_tok\n    )\n</code></pre>"},{"location":"pytorch-widedeep/utils/fastai_transforms.html#pytorch_widedeep.utils.fastai_transforms.Vocab.create","title":"create","text":"<pre><code>create(tokens)\n</code></pre> <p>Create a vocabulary object from a set of tokens.</p> <p>Parameters:</p> <ul> <li> <code>tokens</code>               (<code>Tokens</code>)           \u2013            <p>Custom type: <code>Collection[Collection[str]]</code>  see <code>pytorch_widedeep.wdtypes</code>. Collection of collection of strings (e.g. list of tokenized sentences)</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; from pytorch_widedeep.utils import Tokenizer, Vocab\n&gt;&gt;&gt; texts = ['Machine learning is great', 'but building stuff is even better']\n&gt;&gt;&gt; tokens = Tokenizer().process_all(texts)\n&gt;&gt;&gt; vocab = Vocab(max_vocab=18, min_freq=1).create(tokens)\n&gt;&gt;&gt; vocab.numericalize(['machine', 'learning', 'is', 'great'])\n[10, 11, 9, 12]\n&gt;&gt;&gt; vocab.textify([10, 11, 9, 12])\n'machine learning is great'\n</code></pre> <p> NOTE: Note the many special tokens that <code>fastai</code>'s' tokenizer adds. These are particularly useful when building Language models and/or in classification/Regression tasks. Please see the <code>fastai</code> docs.</p> <p>Returns:</p> <ul> <li> <code>Vocab</code>           \u2013            <p>An instance of a <code>Vocab</code> object</p> </li> </ul> Source code in <code>pytorch_widedeep/utils/fastai_transforms.py</code> <pre><code>def create(\n    self,\n    tokens: Tokens,\n) -&gt; \"Vocab\":\n    r\"\"\"Create a vocabulary object from a set of tokens.\n\n    Parameters\n    ----------\n    tokens: Tokens\n        Custom type: ``Collection[Collection[str]]``  see\n        `pytorch_widedeep.wdtypes`. Collection of collection of\n        strings (e.g. list of tokenized sentences)\n\n    Examples\n    --------\n    &gt;&gt;&gt; from pytorch_widedeep.utils import Tokenizer, Vocab\n    &gt;&gt;&gt; texts = ['Machine learning is great', 'but building stuff is even better']\n    &gt;&gt;&gt; tokens = Tokenizer().process_all(texts)\n    &gt;&gt;&gt; vocab = Vocab(max_vocab=18, min_freq=1).create(tokens)\n    &gt;&gt;&gt; vocab.numericalize(['machine', 'learning', 'is', 'great'])\n    [10, 11, 9, 12]\n    &gt;&gt;&gt; vocab.textify([10, 11, 9, 12])\n    'machine learning is great'\n\n    :information_source: **NOTE**:\n    Note the many special tokens that ``fastai``'s' tokenizer adds. These\n    are particularly useful when building Language models and/or in\n    classification/Regression tasks. Please see the [``fastai`` docs](https://docs.fast.ai/text.core.html#Tokenizing).\n\n    Returns\n    -------\n    Vocab\n        An instance of a `Vocab` object\n    \"\"\"\n\n    freq = Counter(p for o in tokens for p in o)\n    itos = [o for o, c in freq.most_common(self.max_vocab) if c &gt;= self.min_freq]\n    for o in reversed(self.special_cases):\n        if o in itos:\n            itos.remove(o)\n        itos.insert(0, o)\n\n    if self.pad_idx is not None and self.pad_idx != 1:\n        itos.remove(PAD)\n        itos.insert(self.pad_idx, PAD)\n        # get the new 'xxunk' index\n        xxunk_idx = np.where([el == \"xxunk\" for el in itos])[0][0]\n    else:\n        xxunk_idx = 0\n\n    itos = itos[: self.max_vocab]\n    if (\n        len(itos) &lt; self.max_vocab\n    ):  # Make sure vocab size is a multiple of 8 for fast mixed precision training\n        while len(itos) % 8 != 0:\n            itos.append(\"xxfake\")\n\n    self.itos = itos\n    self.stoi = defaultdict(\n        lambda: xxunk_idx, {v: k for k, v in enumerate(self.itos)}\n    )\n\n    return self\n</code></pre>"},{"location":"pytorch-widedeep/utils/fastai_transforms.html#pytorch_widedeep.utils.fastai_transforms.Vocab.fit","title":"fit","text":"<pre><code>fit(tokens)\n</code></pre> <p>Calls the <code>create</code> method. I simply want to honor fast ai naming, but for consistency with the rest of the library I am including a fit method</p> Source code in <code>pytorch_widedeep/utils/fastai_transforms.py</code> <pre><code>def fit(\n    self,\n    tokens: Tokens,\n) -&gt; \"Vocab\":\n    \"\"\"\n    Calls the `create` method. I simply want to honor fast ai naming, but\n    for consistency with the rest of the library I am including a fit method\n    \"\"\"\n    return self.create(tokens)\n</code></pre>"},{"location":"pytorch-widedeep/utils/fastai_transforms.html#pytorch_widedeep.utils.fastai_transforms.Vocab.numericalize","title":"numericalize","text":"<pre><code>numericalize(t)\n</code></pre> <p>Convert a list of tokens <code>t</code> to their ids.</p> <p>Returns:</p> <ul> <li> <code>List[int]</code>           \u2013            <p>List of 'numericalsed' tokens</p> </li> </ul> Source code in <code>pytorch_widedeep/utils/fastai_transforms.py</code> <pre><code>def numericalize(self, t: Collection[str]) -&gt; List[int]:\n    \"\"\"Convert a list of tokens ``t`` to their ids.\n\n    Returns\n    -------\n    List[int]\n        List of '_numericalsed_' tokens\n    \"\"\"\n    return [self.stoi[w] for w in t]\n</code></pre>"},{"location":"pytorch-widedeep/utils/fastai_transforms.html#pytorch_widedeep.utils.fastai_transforms.Vocab.transform","title":"transform","text":"<pre><code>transform(t)\n</code></pre> <p>Calls the <code>numericalize</code> method. I simply want to honor fast ai naming, but for consistency with the rest of the library I am including a transform method</p> Source code in <code>pytorch_widedeep/utils/fastai_transforms.py</code> <pre><code>def transform(self, t: Collection[str]) -&gt; List[int]:\n    \"\"\"\n    Calls the `numericalize` method. I simply want to honor fast ai naming,\n    but for consistency with the rest of the library I am including a\n    transform method\n    \"\"\"\n    return self.numericalize(t)\n</code></pre>"},{"location":"pytorch-widedeep/utils/fastai_transforms.html#pytorch_widedeep.utils.fastai_transforms.Vocab.textify","title":"textify","text":"<pre><code>textify(nums, sep=' ')\n</code></pre> <p>Convert a list of <code>nums</code> (or indexes) to their tokens.</p> <p>Returns:</p> <ul> <li> <code>List[str]</code>           \u2013            <p>List of tokens</p> </li> </ul> Source code in <code>pytorch_widedeep/utils/fastai_transforms.py</code> <pre><code>def textify(self, nums: Collection[int], sep=\" \") -&gt; Union[str, List[str]]:\n    \"\"\"Convert a list of ``nums`` (or indexes) to their tokens.\n\n    Returns\n    -------\n    List[str]\n        List of tokens\n    \"\"\"\n    return (\n        sep.join([self.itos[i] for i in nums])\n        if sep is not None\n        else [self.itos[i] for i in nums]\n    )\n</code></pre>"},{"location":"pytorch-widedeep/utils/fastai_transforms.html#pytorch_widedeep.utils.fastai_transforms.Vocab.inverse_transform","title":"inverse_transform","text":"<pre><code>inverse_transform(nums, sep=' ')\n</code></pre> <p>Calls the <code>textify</code> method. I simply want to honor fast ai naming, but for consistency with the rest of the library I am including an inverse_transform method</p> Source code in <code>pytorch_widedeep/utils/fastai_transforms.py</code> <pre><code>def inverse_transform(\n    self, nums: Collection[int], sep=\" \"\n) -&gt; Union[str, List[str]]:\n    \"\"\"\n    Calls the `textify` method. I simply want to honor fast ai naming, but\n    for consistency with the rest of the library I am including an\n    inverse_transform method\n    \"\"\"\n    # I simply want to honor fast ai naming, but for consistency with the\n    # rest of the library I am including an inverse_transform method\n    return self.textify(nums, sep)\n</code></pre>"},{"location":"pytorch-widedeep/utils/image_utils.html","title":"Image utils","text":"<p>SimplePreprocessor and AspectAwarePreprocessor are directly taked from the great series of Books `Deep Learning for Computer Vision by Adrian. Therefore, all credit for the code in the <code>image_utils</code> module goes to Adrian Rosebrock.</p>"},{"location":"pytorch-widedeep/utils/image_utils.html#pytorch_widedeep.utils.image_utils.AspectAwarePreprocessor","title":"AspectAwarePreprocessor","text":"<pre><code>AspectAwarePreprocessor(\n    width, height, inter=cv2.INTER_AREA\n)\n</code></pre> <p>Class to resize an image to a certain width and height taking into account the image aspect ratio</p> <p>Parameters:</p> <ul> <li> <code>width</code>               (<code>int</code>)           \u2013            <p>output width</p> </li> <li> <code>height</code>               (<code>int</code>)           \u2013            <p>output height</p> </li> <li> <code>inter</code>           \u2013            <p><code>opencv</code> interpolation method. See <code>opencv</code> <code>InterpolationFlags</code>.</p> </li> </ul> Source code in <code>pytorch_widedeep/utils/image_utils.py</code> <pre><code>def __init__(self, width: int, height: int, inter=cv2.INTER_AREA):\n    self.width = width\n    self.height = height\n    self.inter = inter\n</code></pre>"},{"location":"pytorch-widedeep/utils/image_utils.html#pytorch_widedeep.utils.image_utils.AspectAwarePreprocessor.preprocess","title":"preprocess","text":"<pre><code>preprocess(image)\n</code></pre> <p>Returns the resized input image taking into account the image aspect ratio</p> <p>Parameters:</p> <ul> <li> <code>image</code>               (<code>ndarray</code>)           \u2013            <p>Input image to be resized</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import cv2\n&gt;&gt;&gt; from pytorch_widedeep.utils import AspectAwarePreprocessor\n&gt;&gt;&gt; img = cv2.imread(\"tests/test_data_utils/images/galaxy1.png\")\n&gt;&gt;&gt; img.shape\n(694, 890, 3)\n&gt;&gt;&gt; app = AspectAwarePreprocessor(width=224, height=224)\n&gt;&gt;&gt; resized_img = app.preprocess(img)\n&gt;&gt;&gt; resized_img.shape\n(224, 224, 3)\n</code></pre> <p>Returns:</p> <ul> <li> <code>ndarray</code>           \u2013            <p>Resized image according to its original image aspect ratio</p> </li> </ul> Source code in <code>pytorch_widedeep/utils/image_utils.py</code> <pre><code>def preprocess(self, image: np.ndarray) -&gt; np.ndarray:\n    r\"\"\"Returns the resized input image taking into account the image aspect ratio\n\n    Parameters\n    ----------\n    image: np.ndarray\n        Input image to be resized\n\n    Examples\n    --------\n    &gt;&gt;&gt; import cv2\n    &gt;&gt;&gt; from pytorch_widedeep.utils import AspectAwarePreprocessor\n    &gt;&gt;&gt; img = cv2.imread(\"tests/test_data_utils/images/galaxy1.png\")\n    &gt;&gt;&gt; img.shape\n    (694, 890, 3)\n    &gt;&gt;&gt; app = AspectAwarePreprocessor(width=224, height=224)\n    &gt;&gt;&gt; resized_img = app.preprocess(img)\n    &gt;&gt;&gt; resized_img.shape\n    (224, 224, 3)\n\n    Returns\n    -------\n    np.ndarray\n        Resized image according to its original image aspect ratio\n    \"\"\"\n    (h, w) = image.shape[:2]\n    dW = 0\n    dH = 0\n\n    if w &lt; h:\n        image = imutils.resize(image, width=self.width, inter=self.inter)\n        dH = int((image.shape[0] - self.height) / 2.0)\n    else:\n        image = imutils.resize(image, height=self.height, inter=self.inter)\n        dW = int((image.shape[1] - self.width) / 2.0)\n\n    (h, w) = image.shape[:2]\n    image = image[dH : h - dH, dW : w - dW]\n\n    resized_image = cv2.resize(\n        image, (self.width, self.height), interpolation=self.inter\n    )\n\n    return resized_image\n</code></pre>"},{"location":"pytorch-widedeep/utils/image_utils.html#pytorch_widedeep.utils.image_utils.SimplePreprocessor","title":"SimplePreprocessor","text":"<pre><code>SimplePreprocessor(width, height, inter=cv2.INTER_AREA)\n</code></pre> <p>Class to resize an image to a certain width and height</p> <p>Parameters:</p> <ul> <li> <code>width</code>               (<code>int</code>)           \u2013            <p>output width</p> </li> <li> <code>height</code>               (<code>int</code>)           \u2013            <p>output height</p> </li> <li> <code>inter</code>           \u2013            <p><code>opencv</code> interpolation method. See <code>opencv</code> <code>InterpolationFlags</code>.</p> </li> </ul> Source code in <code>pytorch_widedeep/utils/image_utils.py</code> <pre><code>def __init__(self, width: int, height: int, inter=cv2.INTER_AREA):\n    self.width = width\n    self.height = height\n    self.inter = inter\n</code></pre>"},{"location":"pytorch-widedeep/utils/image_utils.html#pytorch_widedeep.utils.image_utils.SimplePreprocessor.preprocess","title":"preprocess","text":"<pre><code>preprocess(image)\n</code></pre> <p>Returns the resized input image</p> <p>Parameters:</p> <ul> <li> <code>image</code>               (<code>ndarray</code>)           \u2013            <p>Input image to be resized</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>ndarray</code>           \u2013            <p>Resized image</p> </li> </ul> Source code in <code>pytorch_widedeep/utils/image_utils.py</code> <pre><code>def preprocess(self, image: np.ndarray) -&gt; np.ndarray:\n    r\"\"\"Returns the resized input image\n\n    Parameters\n    ----------\n    image: np.ndarray\n        Input image to be resized\n\n    Returns\n    -------\n    np.ndarray\n        Resized image\n\n    \"\"\"\n    resized_image = cv2.resize(\n        image, (self.width, self.height), interpolation=self.inter\n    )\n\n    return resized_image\n</code></pre>"},{"location":"pytorch-widedeep/utils/text_utils.html","title":"Text utils","text":"<p>Collection of helper function that facilitate processing text.</p>"},{"location":"pytorch-widedeep/utils/text_utils.html#pytorch_widedeep.utils.text_utils.simple_preprocess","title":"simple_preprocess","text":"<pre><code>simple_preprocess(\n    doc, lower=False, deacc=False, min_len=2, max_len=15\n)\n</code></pre> <p>This is <code>Gensim</code>'s <code>simple_preprocess</code> with a <code>lower</code> param to indicate wether or not to lower case all the token in the doc</p> <p>For more information see: <code>Gensim</code> utils module</p> <p>Parameters:</p> <ul> <li> <code>doc</code>               (<code>str</code>)           \u2013            <p>Input document.</p> </li> <li> <code>lower</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Lower case tokens in the input doc</p> </li> <li> <code>deacc</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Remove accent marks from tokens using <code>Gensim</code>'s <code>deaccent</code></p> </li> <li> <code>min_len</code>               (<code>int</code>, default:                   <code>2</code> )           \u2013            <p>Minimum length of token (inclusive). Shorter tokens are discarded.</p> </li> <li> <code>max_len</code>               (<code>int</code>, default:                   <code>15</code> )           \u2013            <p>Maximum length of token in result (inclusive). Longer tokens are discarded.</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; from pytorch_widedeep.utils import simple_preprocess\n&gt;&gt;&gt; simple_preprocess('Machine learning is great')\n['Machine', 'learning', 'is', 'great']\n</code></pre> <p>Returns:</p> <ul> <li> <code>List[str]</code>           \u2013            <p>List with the processed tokens</p> </li> </ul> Source code in <code>pytorch_widedeep/utils/text_utils.py</code> <pre><code>def simple_preprocess(\n    doc: str,\n    lower: bool = False,\n    deacc: bool = False,\n    min_len: int = 2,\n    max_len: int = 15,\n) -&gt; List[str]:\n    r\"\"\"\n    This is `Gensim`'s `simple_preprocess` with a `lower` param to\n    indicate wether or not to lower case all the token in the doc\n\n    For more information see: `Gensim` [utils module](https://radimrehurek.com/gensim/utils.html)\n\n    Parameters\n    ----------\n    doc: str\n        Input document.\n    lower: bool, default = False\n        Lower case tokens in the input doc\n    deacc: bool, default = False\n        Remove accent marks from tokens using `Gensim`'s `deaccent`\n    min_len: int, default = 2\n        Minimum length of token (inclusive). Shorter tokens are discarded.\n    max_len: int, default = 15\n        Maximum length of token in result (inclusive). Longer tokens are discarded.\n\n    Examples\n    --------\n    &gt;&gt;&gt; from pytorch_widedeep.utils import simple_preprocess\n    &gt;&gt;&gt; simple_preprocess('Machine learning is great')\n    ['Machine', 'learning', 'is', 'great']\n\n    Returns\n    -------\n    List[str]\n        List with the processed tokens\n    \"\"\"\n    tokens = [\n        token\n        for token in tokenize(doc, lower=lower, deacc=deacc, errors=\"ignore\")\n        if min_len &lt;= len(token) &lt;= max_len and not token.startswith(\"_\")\n    ]\n    return tokens\n</code></pre>"},{"location":"pytorch-widedeep/utils/text_utils.html#pytorch_widedeep.utils.text_utils.get_texts","title":"get_texts","text":"<pre><code>get_texts(texts, already_processed=False, n_cpus=None)\n</code></pre> <p>Tokenization using <code>Fastai</code>'s <code>Tokenizer</code> because it does a series of very convenients things during the tokenization process</p> <p>See <code>pytorch_widedeep.utils.fastai_utils.Tokenizer</code></p> <p>Parameters:</p> <ul> <li> <code>texts</code>               (<code>List[str]</code>)           \u2013            <p>List of str with the texts (or documents). One str per document</p> </li> <li> <code>already_processed</code>               (<code>Optional[bool]</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating if the text is already processed and we simply want to tokenize it. This parameter is thought for those cases where the input sequences might not be text (but IDs, or anything else) and we just want to tokenize it</p> </li> <li> <code>n_cpus</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>number of CPUs to used during the tokenization process</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; from pytorch_widedeep.utils import get_texts\n&gt;&gt;&gt; texts = ['Machine learning is great', 'but building stuff is even better']\n&gt;&gt;&gt; get_texts(texts)\n[['xxmaj', 'machine', 'learning', 'is', 'great'], ['but', 'building', 'stuff', 'is', 'even', 'better']]\n</code></pre> <p>Returns:</p> <ul> <li> <code>List[List[str]]</code>           \u2013            <p>List of lists, one list per 'document' containing its corresponding tokens</p> </li> <li> <code>information_source: **NOTE**:</code>           \u2013            </li> <li> <code>`get_texts` uses `pytorch_widedeep.utils.fastai_transforms.Tokenizer`.</code>           \u2013            </li> <li> <code>Such tokenizer uses a series of convenient processing steps, including</code>           \u2013            </li> <li> <code>the  addition of some special tokens, such as `TK_MAJ` (`xxmaj`), used to</code>           \u2013            </li> <li> <code>indicate the next word begins with a capital in the original text. For more</code>           \u2013            </li> <li> <code>details of special tokens please see the [`fastai` `docs](https://docs.fast.ai/text.core.html#Tokenizing)</code>           \u2013            </li> </ul> Source code in <code>pytorch_widedeep/utils/text_utils.py</code> <pre><code>def get_texts(\n    texts: List[str],\n    already_processed: Optional[bool] = False,\n    n_cpus: Optional[int] = None,\n) -&gt; List[List[str]]:\n    r\"\"\"Tokenization using `Fastai`'s `Tokenizer` because it does a\n    series of very convenients things during the tokenization process\n\n    See `pytorch_widedeep.utils.fastai_utils.Tokenizer`\n\n    Parameters\n    ----------\n    texts: List\n        List of str with the texts (or documents). One str per document\n    already_processed: bool, Optional, default = False\n        Boolean indicating if the text is already processed and we simply want\n        to tokenize it. This parameter is thought for those cases where the\n        input sequences might not be text (but IDs, or anything else) and we\n        just want to tokenize it\n    n_cpus: int, Optional, default = None\n        number of CPUs to used during the tokenization process\n\n    Examples\n    --------\n    &gt;&gt;&gt; from pytorch_widedeep.utils import get_texts\n    &gt;&gt;&gt; texts = ['Machine learning is great', 'but building stuff is even better']\n    &gt;&gt;&gt; get_texts(texts)\n    [['xxmaj', 'machine', 'learning', 'is', 'great'], ['but', 'building', 'stuff', 'is', 'even', 'better']]\n\n    Returns\n    -------\n    List[List[str]]\n        List of lists, one list per '_document_' containing its corresponding tokens\n\n    :information_source: **NOTE**:\n    `get_texts` uses `pytorch_widedeep.utils.fastai_transforms.Tokenizer`.\n    Such tokenizer uses a series of convenient processing steps, including\n    the  addition of some special tokens, such as `TK_MAJ` (`xxmaj`), used to\n    indicate the next word begins with a capital in the original text. For more\n    details of special tokens please see the [`fastai` `docs](https://docs.fast.ai/text.core.html#Tokenizing)\n    \"\"\"\n\n    num_cpus = n_cpus if n_cpus is not None else os.cpu_count()\n\n    if not already_processed:\n        processed_texts = [\" \".join(simple_preprocess(t)) for t in texts]\n    else:\n        processed_texts = texts\n    tok = Tokenizer(n_cpus=num_cpus).process_all(processed_texts)\n    return tok\n</code></pre>"},{"location":"pytorch-widedeep/utils/text_utils.html#pytorch_widedeep.utils.text_utils.pad_sequences","title":"pad_sequences","text":"<pre><code>pad_sequences(seq, maxlen, pad_first=True, pad_idx=1)\n</code></pre> <p>Given a List of tokenized and <code>numericalised</code> sequences it will return padded sequences according to the input parameters.</p> <p>Parameters:</p> <ul> <li> <code>seq</code>               (<code>List[int]</code>)           \u2013            <p>List of int with the <code>numericalised</code> tokens</p> </li> <li> <code>maxlen</code>               (<code>int</code>)           \u2013            <p>Maximum length of the padded sequences</p> </li> <li> <code>pad_first</code>               (<code>bool</code>, default:                   <code>True</code> )           \u2013            <p>Indicates whether the padding index will be added at the beginning or the end of the sequences</p> </li> <li> <code>pad_idx</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>padding index. Fastai's Tokenizer leaves 0 for the 'unknown' token.</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; from pytorch_widedeep.utils import pad_sequences\n&gt;&gt;&gt; seq = [1,2,3]\n&gt;&gt;&gt; pad_sequences(seq, maxlen=5, pad_idx=0)\narray([0, 0, 1, 2, 3], dtype=int32)\n</code></pre> <p>Returns:</p> <ul> <li> <code>ndarray</code>           \u2013            <p>numpy array with the padded sequences</p> </li> </ul> Source code in <code>pytorch_widedeep/utils/text_utils.py</code> <pre><code>def pad_sequences(\n    seq: List[int], maxlen: int, pad_first: bool = True, pad_idx: int = 1\n) -&gt; np.ndarray:\n    r\"\"\"\n    Given a List of tokenized and `numericalised` sequences it will return\n    padded sequences according to the input parameters.\n\n    Parameters\n    ----------\n    seq: List\n        List of int with the `numericalised` tokens\n    maxlen: int\n        Maximum length of the padded sequences\n    pad_first: bool,  default = True\n        Indicates whether the padding index will be added at the beginning or the\n        end of the sequences\n    pad_idx: int, default = 1\n        padding index. Fastai's Tokenizer leaves 0 for the 'unknown' token.\n\n    Examples\n    --------\n    &gt;&gt;&gt; from pytorch_widedeep.utils import pad_sequences\n    &gt;&gt;&gt; seq = [1,2,3]\n    &gt;&gt;&gt; pad_sequences(seq, maxlen=5, pad_idx=0)\n    array([0, 0, 1, 2, 3], dtype=int32)\n\n    Returns\n    -------\n    np.ndarray\n        numpy array with the padded sequences\n    \"\"\"\n    if len(seq) == 0:\n        return np.zeros(maxlen, dtype=\"int32\") + pad_idx\n    elif len(seq) &gt;= maxlen:\n        res = np.array(seq[-maxlen:]).astype(\"int32\")\n        return res\n    else:\n        res = np.zeros(maxlen, dtype=\"int32\") + pad_idx\n        if pad_first:\n            res[-len(seq) :] = seq\n        else:\n            res[: len(seq) :] = seq\n        return res\n</code></pre>"},{"location":"pytorch-widedeep/utils/text_utils.html#pytorch_widedeep.utils.text_utils.build_embeddings_matrix","title":"build_embeddings_matrix","text":"<pre><code>build_embeddings_matrix(\n    vocab, word_vectors_path, min_freq, verbose=1\n)\n</code></pre> <p>Build the embedding matrix using pretrained word vectors.</p> <p>Returns pretrained word embeddings. If a word in our vocabulary is not among the pretrained embeddings it will be assigned the mean pretrained word-embeddings vector</p> <p>Parameters:</p> <ul> <li> <code>vocab</code>               (<code>Union[Vocab, ChunkVocab]</code>)           \u2013            <p>see <code>pytorch_widedeep.utils.fastai_utils.Vocab</code></p> </li> <li> <code>word_vectors_path</code>               (<code>str</code>)           \u2013            <p>path to the pretrained word embeddings</p> </li> <li> <code>min_freq</code>               (<code>int</code>)           \u2013            <p>minimum frequency required for a word to be in the vocabulary</p> </li> <li> <code>verbose</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>level of verbosity. Set to 0 for no verbosity</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>ndarray</code>           \u2013            <p>Pretrained word embeddings</p> </li> </ul> Source code in <code>pytorch_widedeep/utils/text_utils.py</code> <pre><code>def build_embeddings_matrix(\n    vocab: Union[Vocab, ChunkVocab],\n    word_vectors_path: str,\n    min_freq: int,\n    verbose: int = 1,\n) -&gt; np.ndarray:  # pragma: no cover\n    r\"\"\"Build the embedding matrix using pretrained word vectors.\n\n    Returns pretrained word embeddings. If a word in our vocabulary is not\n    among the pretrained embeddings it will be assigned the mean pretrained\n    word-embeddings vector\n\n    Parameters\n    ----------\n    vocab: Vocab\n        see `pytorch_widedeep.utils.fastai_utils.Vocab`\n    word_vectors_path: str\n        path to the pretrained word embeddings\n    min_freq: int\n        minimum frequency required for a word to be in the vocabulary\n    verbose: int,  default=1\n        level of verbosity. Set to 0 for no verbosity\n\n    Returns\n    -------\n    np.ndarray\n        Pretrained word embeddings\n    \"\"\"\n    if not os.path.isfile(word_vectors_path):\n        raise FileNotFoundError(\"{} not found\".format(word_vectors_path))\n    if verbose:\n        print(\"Indexing word vectors...\")\n\n    embeddings_index = {}\n    f = open(word_vectors_path)\n    for line in f:\n        values = line.split()\n        word = values[0]\n        coefs = np.asarray(values[1:], dtype=\"float32\")\n        embeddings_index[word] = coefs\n    f.close()\n\n    if verbose:\n        print(\"Loaded {} word vectors\".format(len(embeddings_index)))\n        print(\"Preparing embeddings matrix...\")\n\n    mean_word_vector = np.mean(list(embeddings_index.values()), axis=0)  # type: ignore[arg-type]\n    embedding_dim = len(list(embeddings_index.values())[0])\n    num_words = len(vocab.itos)\n    embedding_matrix = np.zeros((num_words, embedding_dim))\n    found_words = 0\n    for i, word in enumerate(vocab.itos):\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[i] = embedding_vector\n            found_words += 1\n        else:\n            embedding_matrix[i] = mean_word_vector\n\n    if verbose:\n        print(\n            \"{} words in the vocabulary had {} vectors and appear more than {} times\".format(\n                found_words, word_vectors_path, min_freq\n            )\n        )\n\n    return embedding_matrix.astype(\"float32\")\n</code></pre>"}]}
\ No newline at end of file
+{"config":{"lang":["en"],"separator":"[\\s\\-]+","pipeline":["stopWordFilter"]},"docs":[{"location":"index.html","title":"Home","text":""},{"location":"index.html#pytorch-widedeep","title":"pytorch-widedeep","text":"<p>A flexible package for multimodal-deep-learning to combine tabular data with text and images using Wide and Deep models in Pytorch</p> <p>Documentation: https://pytorch-widedeep.readthedocs.io</p> <p>Companion posts and tutorials: infinitoml</p> <p>Experiments and comparison with <code>LightGBM</code>: TabularDL vs LightGBM</p> <p>Slack: if you want to contribute or just want to chat with us, join slack</p> <p>The content of this document is organized as follows:</p> <ul> <li>pytorch-widedeep<ul> <li>Introduction</li> <li>Architectures</li> <li>The <code>deeptabular</code> component</li> <li>The <code>rec</code> module</li> <li>Text and Images</li> <li>Acknowledgments</li> <li>License</li> <li>Cite<ul> <li>BibTex</li> <li>APA</li> </ul> </li> </ul> </li> </ul>"},{"location":"index.html#introduction","title":"Introduction","text":"<p><code>pytorch-widedeep</code> is based on Google's Wide and Deep Algorithm, adjusted for multi-modal datasets.</p> <p>In general terms, <code>pytorch-widedeep</code> is a package to use deep learning with tabular data. In particular, is intended to facilitate the combination of text and images with corresponding tabular data using wide and deep models. With that in mind there are a number of architectures that can be implemented with the library. The main components of those architectures are shown in the Figure below:</p> <p> </p> <p>In math terms, and following the notation in the paper, the expression for the architecture without a <code>deephead</code> component can be formulated as:</p> <p> </p> <p>Where \u03c3 is the sigmoid function, 'W' are the weight matrices applied to the wide model and to the final activations of the deep models, 'a' are these final activations, \u03c6(x) are the cross product transformations of the original features 'x', and , and 'b' is the bias term. In case you are wondering what are \"cross product transformations\", here is a quote taken directly from the paper: \"For binary features, a cross-product transformation (e.g., \u201cAND(gender=female, language=en)\u201d) is 1 if and only if the constituent features (\u201cgender=female\u201d and \u201clanguage=en\u201d) are all 1, and 0 otherwise\".</p> <p>It is perfectly possible to use custom models (and not necessarily those in the library) as long as the the custom models have a property called <code>output_dim</code> with the size of the last layer of activations, so that <code>WideDeep</code> can be constructed. Examples on how to use custom components can be found in the Examples folder and the section below.</p>"},{"location":"index.html#architectures","title":"Architectures","text":"<p>The <code>pytorch-widedeep</code> library offers a number of different architectures. In this section we will show some of them in their simplest form (i.e. with default param values in most cases) with their corresponding code snippets. Note that all the snippets below shoud run locally. For a more detailed explanation of the different components and their parameters, please refer to the documentation.</p> <p>For the examples below we will be using a toy dataset generated as follows:</p> <pre><code>import os\nimport random\n\nimport numpy as np\nimport pandas as pd\nfrom PIL import Image\nfrom faker import Faker\n\n\ndef create_and_save_random_image(image_number, size=(32, 32)):\n\n    if not os.path.exists(\"images\"):\n        os.makedirs(\"images\")\n\n    array = np.random.randint(0, 256, (size[0], size[1], 3), dtype=np.uint8)\n\n    image = Image.fromarray(array)\n\n    image_name = f\"image_{image_number}.png\"\n    image.save(os.path.join(\"images\", image_name))\n\n    return image_name\n\n\nfake = Faker()\n\ncities = [\"New York\", \"Los Angeles\", \"Chicago\", \"Houston\"]\nnames = [\"Alice\", \"Bob\", \"Charlie\", \"David\", \"Eva\"]\n\ndata = {\n    \"city\": [random.choice(cities) for _ in range(100)],\n    \"name\": [random.choice(names) for _ in range(100)],\n    \"age\": [random.uniform(18, 70) for _ in range(100)],\n    \"height\": [random.uniform(150, 200) for _ in range(100)],\n    \"sentence\": [fake.sentence() for _ in range(100)],\n    \"other_sentence\": [fake.sentence() for _ in range(100)],\n    \"image_name\": [create_and_save_random_image(i) for i in range(100)],\n    \"target\": [random.choice([0, 1]) for _ in range(100)],\n}\n\ndf = pd.DataFrame(data)\n</code></pre> <p>This will create a 100 rows dataframe and a dir in your local folder, called <code>images</code> with 100 random images (or images with just noise).</p> <p>Perhaps the simplest architecture would be just one component, <code>wide</code>, <code>deeptabular</code>, <code>deeptext</code> or <code>deepimage</code> on their own, which is also possible, but let's start the examples with a standard Wide and Deep architecture. From there, how to build a model comprised only of one component will be straightforward.</p> <p>Note that the examples shown below would be almost identical using any of the models available in the library. For example, <code>TabMlp</code> can be replaced by <code>TabResnet</code>, <code>TabNet</code>, <code>TabTransformer</code>, etc. Similarly, <code>BasicRNN</code> can be replaced by <code>AttentiveRNN</code>, <code>StackedAttentiveRNN</code>, or <code>HFModel</code> with their corresponding parameters and preprocessor in the case of the Hugging Face models.</p> <p>1. Wide and Tabular component (aka deeptabular)</p> <p> </p> <pre><code>from pytorch_widedeep.preprocessing import TabPreprocessor, WidePreprocessor\nfrom pytorch_widedeep.models import Wide, TabMlp, WideDeep\nfrom pytorch_widedeep.training import Trainer\n\n# Wide\nwide_cols = [\"city\"]\ncrossed_cols = [(\"city\", \"name\")]\nwide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols)\nX_wide = wide_preprocessor.fit_transform(df)\nwide = Wide(input_dim=np.unique(X_wide).shape[0])\n\n# Tabular\ntab_preprocessor = TabPreprocessor(\n    embed_cols=[\"city\", \"name\"], continuous_cols=[\"age\", \"height\"]\n)\nX_tab = tab_preprocessor.fit_transform(df)\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=tab_preprocessor.continuous_cols,\n    mlp_hidden_dims=[64, 32],\n)\n\n# WideDeep\nmodel = WideDeep(wide=wide, deeptabular=tab_mlp)\n\n# Train\ntrainer = Trainer(model, objective=\"binary\")\n\ntrainer.fit(\n    X_wide=X_wide,\n    X_tab=X_tab,\n    target=df[\"target\"].values,\n    n_epochs=1,\n    batch_size=32,\n)\n</code></pre> <p>2. Tabular and Text data</p> <p> </p> <pre><code>from pytorch_widedeep.preprocessing import TabPreprocessor, TextPreprocessor\nfrom pytorch_widedeep.models import TabMlp, BasicRNN, WideDeep\nfrom pytorch_widedeep.training import Trainer\n\n# Tabular\ntab_preprocessor = TabPreprocessor(\n    embed_cols=[\"city\", \"name\"], continuous_cols=[\"age\", \"height\"]\n)\nX_tab = tab_preprocessor.fit_transform(df)\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=tab_preprocessor.continuous_cols,\n    mlp_hidden_dims=[64, 32],\n)\n\n# Text\ntext_preprocessor = TextPreprocessor(\n    text_col=\"sentence\", maxlen=20, max_vocab=100, n_cpus=1\n)\nX_text = text_preprocessor.fit_transform(df)\nrnn = BasicRNN(\n    vocab_size=len(text_preprocessor.vocab.itos),\n    embed_dim=16,\n    hidden_dim=8,\n    n_layers=1,\n)\n\n# WideDeep\nmodel = WideDeep(deeptabular=tab_mlp, deeptext=rnn)\n\n# Train\ntrainer = Trainer(model, objective=\"binary\")\n\ntrainer.fit(\n    X_tab=X_tab,\n    X_text=X_text,\n    target=df[\"target\"].values,\n    n_epochs=1,\n    batch_size=32,\n)\n</code></pre> <p>3. Tabular and text with a FC head on top via the <code>head_hidden_dims</code> param   in <code>WideDeep</code></p> <p> </p> <pre><code>from pytorch_widedeep.preprocessing import TabPreprocessor, TextPreprocessor\nfrom pytorch_widedeep.models import TabMlp, BasicRNN, WideDeep\nfrom pytorch_widedeep.training import Trainer\n\n# Tabular\ntab_preprocessor = TabPreprocessor(\n    embed_cols=[\"city\", \"name\"], continuous_cols=[\"age\", \"height\"]\n)\nX_tab = tab_preprocessor.fit_transform(df)\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=tab_preprocessor.continuous_cols,\n    mlp_hidden_dims=[64, 32],\n)\n\n# Text\ntext_preprocessor = TextPreprocessor(\n    text_col=\"sentence\", maxlen=20, max_vocab=100, n_cpus=1\n)\nX_text = text_preprocessor.fit_transform(df)\nrnn = BasicRNN(\n    vocab_size=len(text_preprocessor.vocab.itos),\n    embed_dim=16,\n    hidden_dim=8,\n    n_layers=1,\n)\n\n# WideDeep\nmodel = WideDeep(deeptabular=tab_mlp, deeptext=rnn, head_hidden_dims=[32, 16])\n\n# Train\ntrainer = Trainer(model, objective=\"binary\")\n\ntrainer.fit(\n    X_tab=X_tab,\n    X_text=X_text,\n    target=df[\"target\"].values,\n    n_epochs=1,\n    batch_size=32,\n)\n</code></pre> <p>4. Tabular and multiple text columns that are passed directly to   <code>WideDeep</code></p> <p> </p> <pre><code>from pytorch_widedeep.preprocessing import TabPreprocessor, TextPreprocessor\nfrom pytorch_widedeep.models import TabMlp, BasicRNN, WideDeep\nfrom pytorch_widedeep.training import Trainer\n\n\n# Tabular\ntab_preprocessor = TabPreprocessor(\n    embed_cols=[\"city\", \"name\"], continuous_cols=[\"age\", \"height\"]\n)\nX_tab = tab_preprocessor.fit_transform(df)\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=tab_preprocessor.continuous_cols,\n    mlp_hidden_dims=[64, 32],\n)\n\n# Text\ntext_preprocessor_1 = TextPreprocessor(\n    text_col=\"sentence\", maxlen=20, max_vocab=100, n_cpus=1\n)\nX_text_1 = text_preprocessor_1.fit_transform(df)\ntext_preprocessor_2 = TextPreprocessor(\n    text_col=\"other_sentence\", maxlen=20, max_vocab=100, n_cpus=1\n)\nX_text_2 = text_preprocessor_2.fit_transform(df)\nrnn_1 = BasicRNN(\n    vocab_size=len(text_preprocessor_1.vocab.itos),\n    embed_dim=16,\n    hidden_dim=8,\n    n_layers=1,\n)\nrnn_2 = BasicRNN(\n    vocab_size=len(text_preprocessor_2.vocab.itos),\n    embed_dim=16,\n    hidden_dim=8,\n    n_layers=1,\n)\n\n# WideDeep\nmodel = WideDeep(deeptabular=tab_mlp, deeptext=[rnn_1, rnn_2])\n\n# Train\ntrainer = Trainer(model, objective=\"binary\")\n\ntrainer.fit(\n    X_tab=X_tab,\n    X_text=[X_text_1, X_text_2],\n    target=df[\"target\"].values,\n    n_epochs=1,\n    batch_size=32,\n)\n</code></pre> <p>5. Tabular data and multiple text columns that are fused via a the library's   <code>ModelFuser</code> class</p> <p> </p> <pre><code>from pytorch_widedeep.preprocessing import TabPreprocessor, TextPreprocessor\nfrom pytorch_widedeep.models import TabMlp, BasicRNN, WideDeep, ModelFuser\nfrom pytorch_widedeep import Trainer\n\n# Tabular\ntab_preprocessor = TabPreprocessor(\n    embed_cols=[\"city\", \"name\"], continuous_cols=[\"age\", \"height\"]\n)\nX_tab = tab_preprocessor.fit_transform(df)\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=tab_preprocessor.continuous_cols,\n    mlp_hidden_dims=[64, 32],\n)\n\n# Text\ntext_preprocessor_1 = TextPreprocessor(\n    text_col=\"sentence\", maxlen=20, max_vocab=100, n_cpus=1\n)\nX_text_1 = text_preprocessor_1.fit_transform(df)\ntext_preprocessor_2 = TextPreprocessor(\n    text_col=\"other_sentence\", maxlen=20, max_vocab=100, n_cpus=1\n)\nX_text_2 = text_preprocessor_2.fit_transform(df)\n\nrnn_1 = BasicRNN(\n    vocab_size=len(text_preprocessor_1.vocab.itos),\n    embed_dim=16,\n    hidden_dim=8,\n    n_layers=1,\n)\nrnn_2 = BasicRNN(\n    vocab_size=len(text_preprocessor_2.vocab.itos),\n    embed_dim=16,\n    hidden_dim=8,\n    n_layers=1,\n)\n\nmodels_fuser = ModelFuser(models=[rnn_1, rnn_2], fusion_method=\"mult\")\n\n# WideDeep\nmodel = WideDeep(deeptabular=tab_mlp, deeptext=models_fuser)\n\n# Train\ntrainer = Trainer(model, objective=\"binary\")\n\ntrainer.fit(\n    X_tab=X_tab,\n    X_text=[X_text_1, X_text_2],\n    target=df[\"target\"].values,\n    n_epochs=1,\n    batch_size=32,\n)\n</code></pre> <p>6. Tabular and multiple text columns, with an image column. The text columns   are fused via the library's <code>ModelFuser</code> and then all fused via the   deephead paramenter in <code>WideDeep</code> which is a custom <code>ModelFuser</code> coded by   the user</p> <p>This is perhaps the less elegant solution as it involves a custom component by the user and slicing the 'incoming' tensor. In the future, we will include a <code>TextAndImageModelFuser</code> to make this process more straightforward. Still, is not really complicated and it is a good example of how to use custom components in <code>pytorch-widedeep</code>.</p> <p>Note that the only requirement for the custom component is that it has a property called <code>output_dim</code> that returns the size of the last layer of activations. In other words, it does not need to inherit from <code>BaseWDModelComponent</code>. This base class simply checks the existence of such property and avoids some typing errors internally.</p> <p> </p> <pre><code>import torch\n\nfrom pytorch_widedeep.preprocessing import TabPreprocessor, TextPreprocessor, ImagePreprocessor\nfrom pytorch_widedeep.models import TabMlp, BasicRNN, WideDeep, ModelFuser, Vision\nfrom pytorch_widedeep.models._base_wd_model_component import BaseWDModelComponent\nfrom pytorch_widedeep import Trainer\n\n# Tabular\ntab_preprocessor = TabPreprocessor(\n    embed_cols=[\"city\", \"name\"], continuous_cols=[\"age\", \"height\"]\n)\nX_tab = tab_preprocessor.fit_transform(df)\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=tab_preprocessor.continuous_cols,\n    mlp_hidden_dims=[16, 8],\n)\n\n# Text\ntext_preprocessor_1 = TextPreprocessor(\n    text_col=\"sentence\", maxlen=20, max_vocab=100, n_cpus=1\n)\nX_text_1 = text_preprocessor_1.fit_transform(df)\ntext_preprocessor_2 = TextPreprocessor(\n    text_col=\"other_sentence\", maxlen=20, max_vocab=100, n_cpus=1\n)\nX_text_2 = text_preprocessor_2.fit_transform(df)\nrnn_1 = BasicRNN(\n    vocab_size=len(text_preprocessor_1.vocab.itos),\n    embed_dim=16,\n    hidden_dim=8,\n    n_layers=1,\n)\nrnn_2 = BasicRNN(\n    vocab_size=len(text_preprocessor_2.vocab.itos),\n    embed_dim=16,\n    hidden_dim=8,\n    n_layers=1,\n)\nmodels_fuser = ModelFuser(\n    models=[rnn_1, rnn_2],\n    fusion_method=\"mult\",\n)\n\n# Image\nimage_preprocessor = ImagePreprocessor(img_col=\"image_name\", img_path=\"images\")\nX_img = image_preprocessor.fit_transform(df)\nvision = Vision(pretrained_model_setup=\"resnet18\", head_hidden_dims=[16, 8])\n\n# deephead (custom model fuser)\nclass MyModelFuser(BaseWDModelComponent):\n    \"\"\"\n    Simply a Linear + Relu sequence on top of the text + images followed by a\n    Linear -&gt; Relu -&gt; Linear for the concatenation of tabular slice of the\n    tensor and the output of the text and image sequential model\n    \"\"\"\n    def __init__(\n        self,\n        tab_incoming_dim: int,\n        text_incoming_dim: int,\n        image_incoming_dim: int,\n        output_units: int,\n    ):\n\n        super(MyModelFuser, self).__init__()\n\n        self.tab_incoming_dim = tab_incoming_dim\n        self.text_incoming_dim = text_incoming_dim\n        self.image_incoming_dim = image_incoming_dim\n        self.output_units = output_units\n        self.text_and_image_fuser = torch.nn.Sequential(\n            torch.nn.Linear(text_incoming_dim + image_incoming_dim, output_units),\n            torch.nn.ReLU(),\n        )\n        self.out = torch.nn.Sequential(\n            torch.nn.Linear(output_units + tab_incoming_dim, output_units * 4),\n            torch.nn.ReLU(),\n            torch.nn.Linear(output_units * 4, output_units),\n        )\n\n    def forward(self, X: torch.Tensor) -&gt; torch.Tensor:\n        tab_slice = slice(0, self.tab_incoming_dim)\n        text_slice = slice(\n            self.tab_incoming_dim, self.tab_incoming_dim + self.text_incoming_dim\n        )\n        image_slice = slice(\n            self.tab_incoming_dim + self.text_incoming_dim,\n            self.tab_incoming_dim + self.text_incoming_dim + self.image_incoming_dim,\n        )\n        X_tab = X[:, tab_slice]\n        X_text = X[:, text_slice]\n        X_img = X[:, image_slice]\n        X_text_and_image = self.text_and_image_fuser(torch.cat([X_text, X_img], dim=1))\n        return self.out(torch.cat([X_tab, X_text_and_image], dim=1))\n\n    @property\n    def output_dim(self):\n        return self.output_units\n\ndeephead = MyModelFuser(\n    tab_incoming_dim=tab_mlp.output_dim,\n    text_incoming_dim=models_fuser.output_dim,\n    image_incoming_dim=vision.output_dim,\n    output_units=8,\n)\n\n# WideDeep\nmodel = WideDeep(\n    deeptabular=tab_mlp,\n    deeptext=models_fuser,\n    deepimage=vision,\n    deephead=deephead,\n)\n\n# Train\ntrainer = Trainer(model, objective=\"binary\")\n\ntrainer.fit(\n    X_tab=X_tab,\n    X_text=[X_text_1, X_text_2],\n    X_img=X_img,\n    target=df[\"target\"].values,\n    n_epochs=1,\n    batch_size=32,\n)\n</code></pre> <p>7. A two-tower model</p> <p>This is a popular model in the context of recommendation systems. Let's say we have a tabular dataset formed my triples (user features, item features, target). We can create a two-tower model where the user and item features are passed through two separate models and then \"fused\" via a dot product.</p> <p> </p> <pre><code>import numpy as np\nimport pandas as pd\n\nfrom pytorch_widedeep import Trainer\nfrom pytorch_widedeep.preprocessing import TabPreprocessor\nfrom pytorch_widedeep.models import TabMlp, WideDeep, ModelFuser\n\n# Let's create the interaction dataset\n# user_features dataframe\nnp.random.seed(42)\nuser_ids = np.arange(1, 101)\nages = np.random.randint(18, 60, size=100)\ngenders = np.random.choice([\"male\", \"female\"], size=100)\nlocations = np.random.choice([\"city_a\", \"city_b\", \"city_c\", \"city_d\"], size=100)\nuser_features = pd.DataFrame(\n    {\"id\": user_ids, \"age\": ages, \"gender\": genders, \"location\": locations}\n)\n\n# item_features dataframe\nitem_ids = np.arange(1, 101)\nprices = np.random.uniform(10, 500, size=100).round(2)\ncolors = np.random.choice([\"red\", \"blue\", \"green\", \"black\"], size=100)\ncategories = np.random.choice([\"electronics\", \"clothing\", \"home\", \"toys\"], size=100)\n\nitem_features = pd.DataFrame(\n    {\"id\": item_ids, \"price\": prices, \"color\": colors, \"category\": categories}\n)\n\n# Interactions dataframe\ninteraction_user_ids = np.random.choice(user_ids, size=1000)\ninteraction_item_ids = np.random.choice(item_ids, size=1000)\npurchased = np.random.choice([0, 1], size=1000, p=[0.7, 0.3])\ninteractions = pd.DataFrame(\n    {\n        \"user_id\": interaction_user_ids,\n        \"item_id\": interaction_item_ids,\n        \"purchased\": purchased,\n    }\n)\nuser_item_purchased = interactions.merge(\n    user_features, left_on=\"user_id\", right_on=\"id\"\n).merge(item_features, left_on=\"item_id\", right_on=\"id\")\n\n# Users\ntab_preprocessor_user = TabPreprocessor(\n    cat_embed_cols=[\"gender\", \"location\"],\n    continuous_cols=[\"age\"],\n)\nX_user = tab_preprocessor_user.fit_transform(user_item_purchased)\ntab_mlp_user = TabMlp(\n    column_idx=tab_preprocessor_user.column_idx,\n    cat_embed_input=tab_preprocessor_user.cat_embed_input,\n    continuous_cols=[\"age\"],\n    mlp_hidden_dims=[16, 8],\n    mlp_dropout=[0.2, 0.2],\n)\n\n# Items\ntab_preprocessor_item = TabPreprocessor(\n    cat_embed_cols=[\"color\", \"category\"],\n    continuous_cols=[\"price\"],\n)\nX_item = tab_preprocessor_item.fit_transform(user_item_purchased)\ntab_mlp_item = TabMlp(\n    column_idx=tab_preprocessor_item.column_idx,\n    cat_embed_input=tab_preprocessor_item.cat_embed_input,\n    continuous_cols=[\"price\"],\n    mlp_hidden_dims=[16, 8],\n    mlp_dropout=[0.2, 0.2],\n)\n\ntwo_tower_model = ModelFuser([tab_mlp_user, tab_mlp_item], fusion_method=\"dot\")\n\nmodel = WideDeep(deeptabular=two_tower_model)\n\ntrainer = Trainer(model, objective=\"binary\")\n\ntrainer.fit(\n    X_tab=[X_user, X_item],\n    target=interactions.purchased.values,\n    n_epochs=1,\n    batch_size=32,\n)\n</code></pre> <p>8. Tabular with a multi-target loss</p> <p>This one is \"a bonus\" to illustrate the use of multi-target losses, more than actually a different architecture.</p> <p> </p> <pre><code>from pytorch_widedeep.preprocessing import TabPreprocessor, TextPreprocessor, ImagePreprocessor\nfrom pytorch_widedeep.models import TabMlp, BasicRNN, WideDeep, ModelFuser, Vision\nfrom pytorch_widedeep.losses_multitarget import MultiTargetClassificationLoss\nfrom pytorch_widedeep.models._base_wd_model_component import BaseWDModelComponent\nfrom pytorch_widedeep import Trainer\n\n# let's add a second target to the dataframe\ndf[\"target2\"] = [random.choice([0, 1]) for _ in range(100)]\n\n# Tabular\ntab_preprocessor = TabPreprocessor(\n    embed_cols=[\"city\", \"name\"], continuous_cols=[\"age\", \"height\"]\n)\nX_tab = tab_preprocessor.fit_transform(df)\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=tab_preprocessor.continuous_cols,\n    mlp_hidden_dims=[64, 32],\n)\n\n# 'pred_dim=2' because we have two binary targets. For other types of targets,\n#  please, see the documentation\nmodel = WideDeep(deeptabular=tab_mlp, pred_dim=2).\n\nloss = MultiTargetClassificationLoss(binary_config=[0, 1], reduction=\"mean\")\n\n# When a multi-target loss is used, 'custom_loss_function' must not be None.\n# See the docs\ntrainer = Trainer(model, objective=\"multitarget\", custom_loss_function=loss)\n\ntrainer.fit(\n    X_tab=X_tab,\n    target=df[[\"target\", \"target2\"]].values,\n    n_epochs=1,\n    batch_size=32,\n)\n</code></pre>"},{"location":"index.html#the-deeptabular-component","title":"The <code>deeptabular</code> component","text":"<p>It is important to emphasize again that each individual component, <code>wide</code>, <code>deeptabular</code>, <code>deeptext</code> and <code>deepimage</code>, can be used independently and in isolation. For example, one could use only <code>wide</code>, which is in simply a linear model. In fact, one of the most interesting functionalities in<code>pytorch-widedeep</code> would be the use of the <code>deeptabular</code> component on its own, i.e. what one might normally refer as Deep Learning for Tabular Data. Currently, <code>pytorch-widedeep</code> offers the following different models for that component:</p> <ol> <li>Wide: a simple linear model where the nonlinearities are captured via cross-product transformations, as explained before.</li> <li>TabMlp: a simple MLP that receives embeddings representing the categorical features, concatenated with the continuous features, which can also be embedded.</li> <li>TabResnet: similar to the previous model but the embeddings are passed through a series of ResNet blocks built with dense layers.</li> <li>TabNet: details on TabNet can be found in TabNet: Attentive Interpretable Tabular Learning</li> </ol> <p>Two simpler attention based models that we call:</p> <ol> <li>ContextAttentionMLP: MLP with at attention mechanism \"on top\" that is based on     Hierarchical Attention Networks for Document Classification</li> <li>SelfAttentionMLP: MLP with an attention mechanism that is a simplified     version of a transformer block that we refer as \"query-key self-attention\".</li> </ol> <p>The <code>Tabformer</code> family, i.e. Transformers for Tabular data:</p> <ol> <li>TabTransformer: details on the TabTransformer can be found in TabTransformer: Tabular Data Modeling Using Contextual Embeddings.</li> <li>SAINT: Details on SAINT can be found in SAINT: Improved Neural Networks for Tabular Data via Row Attention and Contrastive Pre-Training.</li> <li>FT-Transformer: details on the FT-Transformer can be found in Revisiting Deep Learning Models for Tabular Data.</li> <li>TabFastFormer: adaptation of the FastFormer for tabular data. Details on the Fasformer can be found in FastFormers: Highly Efficient Transformer Models for Natural Language Understanding</li> <li>TabPerceiver: adaptation of the Perceiver for tabular data. Details on the Perceiver can be found in Perceiver: General Perception with Iterative Attention</li> </ol> <p>And probabilistic DL models for tabular data based on Weight Uncertainty in Neural Networks:</p> <ol> <li>BayesianWide: Probabilistic adaptation of the <code>Wide</code> model.</li> <li>BayesianTabMlp: Probabilistic adaptation of the <code>TabMlp</code> model</li> </ol> <p>Note that while there are scientific publications for the TabTransformer, SAINT and FT-Transformer, the TabFasfFormer and TabPerceiver are our own adaptation of those algorithms for tabular data.</p> <p>In addition, Self-Supervised pre-training can be used for all <code>deeptabular</code> models, with the exception of the <code>TabPerceiver</code>. Self-Supervised pre-training can be used via two methods or routines which we refer as: encoder-decoder method and constrastive-denoising method. Please, see the documentation and the examples for details on this functionality, and all other options in the library.</p>"},{"location":"index.html#the-rec-module","title":"The <code>rec</code> module","text":"<p>This module was introduced as an extension to the existing components in the library, addressing questions and issues related to recommendation systems. While still under active development, it currently includes a select number of powerful recommendation models.</p> <p>It's worth noting that this library already supported the implementation of various recommendation algorithms using existing components. For example, models like Wide and Deep, Two-Tower, or Neural Collaborative Filtering could be constructed using the library's core functionalities.</p> <p>The recommendation algorithms in the <code>rec</code> module are:</p> <ol> <li>DeepFM: A Factorization-Machine based Neural Network for CTR Prediction</li> <li>(Deep) Field Aware Factorization Machine (FFM): a Deep Learning version of the algorithm presented in Field-aware Factorization Machines in a Real-world Online Advertising System</li> <li>xDeepFM: Combining Explicit and Implicit Feature Interactions for Recommender Systems</li> <li>Deep Interest Network for Click-Through Rate Prediction</li> </ol> <p>These can all be used as the <code>deeptabular</code> component in the <code>WideDeep</code> model. See the examples for more details.</p>"},{"location":"index.html#text-and-images","title":"Text and Images","text":"<p>For the text component, <code>deeptext</code>, the library offers the following models:</p> <ol> <li>BasicRNN: a simple RNN 2. AttentiveRNN: a RNN with an attention mechanism based on the Hierarchical Attention Networks for DocumentClassification</li> <li>StackedAttentiveRNN: a stack of AttentiveRNNs</li> <li>HFModel: a wrapper around Hugging Face Transfomer-based models. At the moment only models from the families BERT, RoBERTa, DistilBERT, ALBERT and ELECTRA are supported. This is because this library is designed to address classification and regression tasks and these are the most 'popular' encoder-only models, which have proved to be those that work best for these tasks. If there is demand for other models, they will be included in the future.</li> </ol> <p>For the image component, <code>deepimage</code>, the library supports models from the following families: 'resnet', 'shufflenet', 'resnext', 'wide_resnet', 'regnet', 'densenet', 'mobilenetv3',  'mobilenetv2', 'mnasnet', 'efficientnet' and 'squeezenet'.  These are  offered via <code>torchvision</code> and wrapped up in the <code>Vision</code> class.</p>"},{"location":"index.html#acknowledgments","title":"Acknowledgments","text":"<p>This library takes from a series of other libraries, so I think it is just fair to mention them here in the README (specific mentions are also included in the code).</p> <p>The <code>Callbacks</code> and <code>Initializers</code> structure and code is inspired by the <code>torchsample</code> library, which in itself partially inspired by <code>Keras</code>.</p> <p>The <code>TextProcessor</code> class in this library uses the <code>fastai</code>'s <code>Tokenizer</code> and <code>Vocab</code>. The code at <code>utils.fastai_transforms</code> is a minor adaptation of their code so it functions within this library. To my experience their <code>Tokenizer</code> is the best in class.</p> <p>The <code>ImageProcessor</code> class in this library uses code from the fantastic Deep Learning for Computer Vision (DL4CV) book by Adrian Rosebrock.</p>"},{"location":"index.html#license","title":"License","text":"<p>This work is dual-licensed under Apache 2.0 and MIT (or any later version). You can choose between one of them if you use this work.</p> <p><code>SPDX-License-Identifier: Apache-2.0 AND MIT</code></p>"},{"location":"index.html#cite","title":"Cite","text":""},{"location":"index.html#bibtex","title":"BibTex","text":"<pre><code>@article{Zaurin_pytorch-widedeep_A_flexible_2023,\nauthor = {Zaurin, Javier Rodriguez and Mulinka, Pavol},\ndoi = {10.21105/joss.05027},\njournal = {Journal of Open Source Software},\nmonth = jun,\nnumber = {86},\npages = {5027},\ntitle = {{pytorch-widedeep: A flexible package for multimodal deep learning}},\nurl = {https://joss.theoj.org/papers/10.21105/joss.05027},\nvolume = {8},\nyear = {2023}\n}\n</code></pre>"},{"location":"index.html#apa","title":"APA","text":"<pre><code>Zaurin, J. R., &amp; Mulinka, P. (2023). pytorch-widedeep: A flexible package for\nmultimodal deep learning. Journal of Open Source Software, 8(86), 5027.\nhttps://doi.org/10.21105/joss.05027\n</code></pre>"},{"location":"contributing.html","title":"Contributing","text":"<p>Pytorch-widedeep is being developed and used by many active community members. Your help is very valuable to make it better for everyone.</p> <ul> <li>Check for the Roadmap or Open an issue to report problems or recommend new features and submit a draft pull requests, which will be changed to pull request after intial review</li> <li>Contribute to the tests to make it more reliable.</li> <li>Contribute to the documentation to make it clearer for everyone.</li> <li>Contribute to the examples to share your experience with other users.</li> <li>Join the dicussion on slack</li> </ul>"},{"location":"installation.html","title":"Installation","text":"<p>This section explains how to install <code>pytorch-widedeep</code>.</p> <p>For the latest stable release, execute:</p> <pre><code>pip install pytorch-widedeep\n</code></pre> <p>For the bleeding-edge version, execute:</p> <pre><code>pip install git+https://github.com/jrzaurin/pytorch-widedeep.git\n</code></pre> <p>For developer install</p> <pre><code># Clone the repository\ngit clone https://github.com/jrzaurin/pytorch-widedeep\ncd pytorch-widedeep\n\n# Install in dev mode\npip install -e .\n</code></pre>"},{"location":"installation.html#dependencies","title":"Dependencies","text":"<ul> <li>pandas&gt;=1.3.5</li> <li>numpy&gt;=1.21.6</li> <li>scipy&gt;=1.7.3,&lt;=1.12.0</li> <li>scikit-learn&gt;=1.0.2</li> <li>gensim</li> <li>spacy</li> <li>opencv-contrib-python</li> <li>imutils</li> <li>tqdm</li> <li>torch &gt;= 2.0.0</li> <li>torchvision &gt;= 0.15.0</li> <li>einops</li> <li>wrapt</li> <li>torchmetrics</li> <li>pyarrow</li> <li>fastparquet&gt;=0.8.1</li> <li>transformers</li> <li>sentence-transformers</li> <li>sentencepiece</li> </ul>"},{"location":"quick_start.html","title":"Quick Start","text":"<p>This is an example of a binary classification with the adult census dataset using a combination of a wide and deep model (in this case a so called <code>deeptabular</code> model) with defaults settings.</p> <pre><code>import numpy as np\nimport torch\nfrom sklearn.model_selection import train_test_split\n\nfrom pytorch_widedeep import Trainer\nfrom pytorch_widedeep.preprocessing import WidePreprocessor, TabPreprocessor\nfrom pytorch_widedeep.models import Wide, TabMlp, WideDeep\nfrom pytorch_widedeep.metrics import Accuracy\nfrom pytorch_widedeep.datasets import load_adult\n\n\ndf = load_adult(as_frame=True)\ndf[\"income_label\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int)\ndf.drop(\"income\", axis=1, inplace=True)\ndf_train, df_test = train_test_split(df, test_size=0.2, stratify=df.income_label)\n\n# Define the 'column set up'\nwide_cols = [\n    \"education\",\n    \"relationship\",\n    \"workclass\",\n    \"occupation\",\n    \"native-country\",\n    \"gender\",\n]\ncrossed_cols = [(\"education\", \"occupation\"), (\"native-country\", \"occupation\")]\n\ncat_embed_cols = [\n    \"workclass\",\n    \"education\",\n    \"marital-status\",\n    \"occupation\",\n    \"relationship\",\n    \"race\",\n    \"gender\",\n    \"capital-gain\",\n    \"capital-loss\",\n    \"native-country\",\n]\ncontinuous_cols = [\"age\", \"hours-per-week\"]\ntarget = \"income_label\"\ntarget = df_train[target].values\n\n# prepare the data\nwide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols)\nX_wide = wide_preprocessor.fit_transform(df_train)\n\ntab_preprocessor = TabPreprocessor(\n    cat_embed_cols=cat_embed_cols, continuous_cols=continuous_cols  # type: ignore[arg-type]\n)\nX_tab = tab_preprocessor.fit_transform(df_train)\n\n# build the model\nwide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1)\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=continuous_cols,\n)\nmodel = WideDeep(wide=wide, deeptabular=tab_mlp)\n\n# train and validate\ntrainer = Trainer(model, objective=\"binary\", metrics=[Accuracy])\ntrainer.fit(\n    X_wide=X_wide,\n    X_tab=X_tab,\n    target=target,\n    n_epochs=5,\n    batch_size=256,\n)\n\n# predict on test\nX_wide_te = wide_preprocessor.transform(df_test)\nX_tab_te = tab_preprocessor.transform(df_test)\npreds = trainer.predict(X_wide=X_wide_te, X_tab=X_tab_te)\n\n# Save and load\n\n# Option 1: this will also save training history and lr history if the\n# LRHistory callback is used\ntrainer.save(path=\"model_weights\", save_state_dict=True)\n\n# Option 2: save as any other torch model\ntorch.save(model.state_dict(), \"model_weights/wd_model.pt\")\n\n# From here in advance, Option 1 or 2 are the same. I assume the user has\n# prepared the data and defined the new model components:\n# 1. Build the model\nmodel_new = WideDeep(wide=wide, deeptabular=tab_mlp)\nmodel_new.load_state_dict(torch.load(\"model_weights/wd_model.pt\"))\n\n# 2. Instantiate the trainer\ntrainer_new = Trainer(model_new, objective=\"binary\")\n\n# 3. Either start the fit or directly predict\npreds = trainer_new.predict(X_wide=X_wide, X_tab=X_tab, batch_size=32)\n</code></pre>"},{"location":"examples/01_preprocessors_and_utils.html","title":"01_preprocessors_and_utils","text":"<p>For example</p> In\u00a0[1]: Copied! <pre>import numpy as np\nimport pandas as pd\nimport pytorch_widedeep as wd\n\nfrom pytorch_widedeep.datasets import load_adult\nfrom pytorch_widedeep.preprocessing import WidePreprocessor\n</pre> import numpy as np import pandas as pd import pytorch_widedeep as wd  from pytorch_widedeep.datasets import load_adult from pytorch_widedeep.preprocessing import WidePreprocessor <pre>/Users/javierrodriguezzaurin/.pyenv/versions/3.10.13/envs/widedeep310/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n  from .autonotebook import tqdm as notebook_tqdm\n</pre> In\u00a0[2]: Copied! <pre>df = load_adult(as_frame=True)\ndf.head()\n</pre> df = load_adult(as_frame=True) df.head() Out[2]: age workclass fnlwgt education educational-num marital-status occupation relationship race gender capital-gain capital-loss hours-per-week native-country income 0 25 Private 226802 11th 7 Never-married Machine-op-inspct Own-child Black Male 0 0 40 United-States &lt;=50K 1 38 Private 89814 HS-grad 9 Married-civ-spouse Farming-fishing Husband White Male 0 0 50 United-States &lt;=50K 2 28 Local-gov 336951 Assoc-acdm 12 Married-civ-spouse Protective-serv Husband White Male 0 0 40 United-States &gt;50K 3 44 Private 160323 Some-college 10 Married-civ-spouse Machine-op-inspct Husband Black Male 7688 0 40 United-States &gt;50K 4 18 ? 103497 Some-college 10 Never-married ? Own-child White Female 0 0 30 United-States &lt;=50K In\u00a0[3]: Copied! <pre>wide_cols = [\n    \"education\",\n    \"relationship\",\n    \"workclass\",\n    \"occupation\",\n    \"native-country\",\n    \"gender\",\n]\ncrossed_cols = [(\"education\", \"occupation\"), (\"native-country\", \"occupation\")]\n</pre> wide_cols = [     \"education\",     \"relationship\",     \"workclass\",     \"occupation\",     \"native-country\",     \"gender\", ] crossed_cols = [(\"education\", \"occupation\"), (\"native-country\", \"occupation\")] In\u00a0[4]: Copied! <pre>wide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols)\nX_wide = wide_preprocessor.fit_transform(df)\n# From here on, any new observation can be prepared by simply running `.transform`\n# new_X_wide = wide_preprocessor.transform(new_df)\n</pre> wide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols) X_wide = wide_preprocessor.fit_transform(df) # From here on, any new observation can be prepared by simply running `.transform` # new_X_wide = wide_preprocessor.transform(new_df) In\u00a0[5]: Copied! <pre>X_wide\n</pre> X_wide Out[5]: <pre>array([[  1,  17,  23, ...,  89,  91, 316],\n       [  2,  18,  23, ...,  89,  92, 317],\n       [  3,  18,  24, ...,  89,  93, 318],\n       ...,\n       [  2,  20,  23, ...,  90, 103, 323],\n       [  2,  17,  23, ...,  89, 103, 323],\n       [  2,  21,  29, ...,  90, 115, 324]])</pre> <p>Note that the label encoding starts from <code>1</code>. This is because it is convenient to leave <code>0</code> for padding, i.e. unknown categories. Let's take from example the first entry</p> In\u00a0[6]: Copied! <pre>X_wide[0]\n</pre> X_wide[0] Out[6]: <pre>array([  1,  17,  23,  32,  47,  89,  91, 316])</pre> In\u00a0[7]: Copied! <pre>wide_preprocessor.inverse_transform(X_wide[:1])\n</pre> wide_preprocessor.inverse_transform(X_wide[:1]) Out[7]: education relationship workclass occupation native-country gender education_occupation native-country_occupation 0 11th Own-child Private Machine-op-inspct United-States Male 11th-Machine-op-inspct United-States-Machine-op-inspct <p>As we can see, <code>wide_preprocessor</code> numerically encodes the <code>wide_cols</code> and the <code>crossed_cols</code>, which can be recovered using the method <code>inverse_transform</code>.</p> In\u00a0[8]: Copied! <pre>from pytorch_widedeep.preprocessing import TabPreprocessor\n</pre> from pytorch_widedeep.preprocessing import TabPreprocessor In\u00a0[9]: Copied! <pre># cat_embed_cols = [(column_name, embed_dim), ...]\ncat_embed_cols = [\n    (\"education\", 10),\n    (\"relationship\", 8),\n    (\"workclass\", 10),\n    (\"occupation\", 10),\n    (\"native-country\", 10),\n]\ncontinuous_cols = [\"age\", \"hours-per-week\"]\n</pre> # cat_embed_cols = [(column_name, embed_dim), ...] cat_embed_cols = [     (\"education\", 10),     (\"relationship\", 8),     (\"workclass\", 10),     (\"occupation\", 10),     (\"native-country\", 10), ] continuous_cols = [\"age\", \"hours-per-week\"] In\u00a0[10]: Copied! <pre>tab_preprocessor = TabPreprocessor(\n    cat_embed_cols=cat_embed_cols,\n    continuous_cols=continuous_cols,\n    cols_to_scale=[\"age\"],  # or scale=True or cols_to_scale=continuous_cols\n)\nX_tab = tab_preprocessor.fit_transform(df)\n# From here on, any new observation can be prepared by simply running `.transform`\n# new_X_deep = deep_preprocessor.transform(new_df)\n</pre> tab_preprocessor = TabPreprocessor(     cat_embed_cols=cat_embed_cols,     continuous_cols=continuous_cols,     cols_to_scale=[\"age\"],  # or scale=True or cols_to_scale=continuous_cols ) X_tab = tab_preprocessor.fit_transform(df) # From here on, any new observation can be prepared by simply running `.transform` # new_X_deep = deep_preprocessor.transform(new_df) In\u00a0[11]: Copied! <pre>X_tab\n</pre> X_tab Out[11]: <pre>array([[ 1.00000000e+00,  1.00000000e+00,  1.00000000e+00, ...,\n         1.00000000e+00, -9.95128932e-01,  4.00000000e+01],\n       [ 2.00000000e+00,  2.00000000e+00,  1.00000000e+00, ...,\n         1.00000000e+00, -4.69415091e-02,  5.00000000e+01],\n       [ 3.00000000e+00,  2.00000000e+00,  2.00000000e+00, ...,\n         1.00000000e+00, -7.76316450e-01,  4.00000000e+01],\n       ...,\n       [ 2.00000000e+00,  4.00000000e+00,  1.00000000e+00, ...,\n         1.00000000e+00,  1.41180837e+00,  4.00000000e+01],\n       [ 2.00000000e+00,  1.00000000e+00,  1.00000000e+00, ...,\n         1.00000000e+00, -1.21394141e+00,  2.00000000e+01],\n       [ 2.00000000e+00,  5.00000000e+00,  7.00000000e+00, ...,\n         1.00000000e+00,  9.74183408e-01,  4.00000000e+01]])</pre> <p>Note that the label encoding starts from <code>1</code>. This is because it is convenient to leave <code>0</code> for padding, i.e. unknown categories. Let's take from example the first entry</p> In\u00a0[12]: Copied! <pre>X_tab[0]\n</pre> X_tab[0] Out[12]: <pre>array([ 1.        ,  1.        ,  1.        ,  1.        ,  1.        ,\n       -0.99512893, 40.        ])</pre> In\u00a0[13]: Copied! <pre>tab_preprocessor.inverse_transform(X_tab[:1])\n</pre> tab_preprocessor.inverse_transform(X_tab[:1]) Out[13]: education relationship workclass occupation native-country age hours-per-week 0 11th Own-child Private Machine-op-inspct United-States 25.0 40.0 <p>The <code>TabPreprocessor</code> will have a series of useful attributes that can later be used when instantiating the different Tabular Models, such us for example, the column indexes (used to slice the tensors, internally in the models) or the categorical embeddings set up</p> In\u00a0[14]: Copied! <pre>tab_preprocessor.column_idx\n</pre> tab_preprocessor.column_idx Out[14]: <pre>{'education': 0,\n 'relationship': 1,\n 'workclass': 2,\n 'occupation': 3,\n 'native-country': 4,\n 'age': 5,\n 'hours-per-week': 6}</pre> In\u00a0[15]: Copied! <pre># column name, num unique, embedding dim\ntab_preprocessor.cat_embed_input\n</pre> # column name, num unique, embedding dim tab_preprocessor.cat_embed_input Out[15]: <pre>[('education', 16, 10),\n ('relationship', 6, 8),\n ('workclass', 9, 10),\n ('occupation', 15, 10),\n ('native-country', 42, 10)]</pre> <p>As I mentioned, there is more one can do, such as for example, quantize (or bucketize) the continuous cols. For this we could use the <code>quantization_setup</code> param. This parameter accepts a number of different inputs and uses <code>pd.cut</code> under the hood to quantize the continuous cols. For more info, please, read the docs. Let's use it here to quantize \"age\" and \"hours-per-week\" in 4 and 5 \"buckets\" respectively</p> In\u00a0[16]: Copied! <pre>quantization_setup = {\n    \"age\": 4,\n    \"hours-per-week\": 5,\n}  # you can also pass a list of floats with the boundaries if you wanted\nquant_tab_preprocessor = TabPreprocessor(\n    cat_embed_cols=cat_embed_cols,\n    continuous_cols=continuous_cols,\n    quantization_setup=quantization_setup,\n)\nqX_tab = quant_tab_preprocessor.fit_transform(df)\n</pre> quantization_setup = {     \"age\": 4,     \"hours-per-week\": 5, }  # you can also pass a list of floats with the boundaries if you wanted quant_tab_preprocessor = TabPreprocessor(     cat_embed_cols=cat_embed_cols,     continuous_cols=continuous_cols,     quantization_setup=quantization_setup, ) qX_tab = quant_tab_preprocessor.fit_transform(df) <pre>/Users/javierrodriguezzaurin/Projects/pytorch-widedeep/pytorch_widedeep/preprocessing/tab_preprocessor.py:358: UserWarning: Continuous columns will not be normalised\n  warnings.warn(\"Continuous columns will not be normalised\")\n</pre> In\u00a0[17]: Copied! <pre>qX_tab\n</pre> qX_tab Out[17]: <pre>array([[1, 1, 1, ..., 1, 1, 2],\n       [2, 2, 1, ..., 1, 2, 3],\n       [3, 2, 2, ..., 1, 1, 2],\n       ...,\n       [2, 4, 1, ..., 1, 3, 2],\n       [2, 1, 1, ..., 1, 1, 1],\n       [2, 5, 7, ..., 1, 2, 2]])</pre> <p>Note that the continuous columns that have been bucketised into quantiles are treated as any other categorical column</p> In\u00a0[18]: Copied! <pre>quant_tab_preprocessor.cat_embed_input\n</pre> quant_tab_preprocessor.cat_embed_input Out[18]: <pre>[('education', 16, 10),\n ('relationship', 6, 8),\n ('workclass', 9, 10),\n ('occupation', 15, 10),\n ('native-country', 42, 10),\n ('age', 4, 4),\n ('hours-per-week', 5, 4)]</pre> <p>Where the column 'age' has now 4 categories, which will be encoded using embeddings of 4 dims. Note that, as any other categorical columns, the categorical \"counter\" starts with 1. This is because all incoming values that are lower/higher than the existing lowest/highest value in the train (or already seen) dataset, will be encoded as 0.</p> In\u00a0[19]: Copied! <pre>np.unique(qX_tab[:, quant_tab_preprocessor.column_idx[\"age\"]])\n</pre> np.unique(qX_tab[:, quant_tab_preprocessor.column_idx[\"age\"]]) Out[19]: <pre>array([1, 2, 3, 4])</pre> <p>Finally, if we now wanted to <code>inverse_transform</code> the transformed array into the original dataframe, we could still do it, but the continuous, bucketized columns will be transformed back to the middle of their quantile/bucket range</p> In\u00a0[20]: Copied! <pre>df_decoded = quant_tab_preprocessor.inverse_transform(qX_tab)\n</pre> df_decoded = quant_tab_preprocessor.inverse_transform(qX_tab) <pre>Note that quantized cols will be turned into the mid point of the corresponding bin\n</pre> In\u00a0[21]: Copied! <pre>df.head(2)\n</pre> df.head(2) Out[21]: age workclass fnlwgt education educational-num marital-status occupation relationship race gender capital-gain capital-loss hours-per-week native-country income 0 25 Private 226802 11th 7 Never-married Machine-op-inspct Own-child Black Male 0 0 40 United-States &lt;=50K 1 38 Private 89814 HS-grad 9 Married-civ-spouse Farming-fishing Husband White Male 0 0 50 United-States &lt;=50K In\u00a0[22]: Copied! <pre>df_decoded.head(2)\n</pre> df_decoded.head(2) Out[22]: education relationship workclass occupation native-country age hours-per-week 0 11th Own-child Private Machine-op-inspct United-States 26.0885 30.4 1 HS-grad Husband Private Farming-fishing United-States 44.3750 50.0 <p>there is one final comment to make regarding to the <code>inverse_transform</code> functionality. As we mentioned before, the encoding <code>0</code> is reserved for values that fall outside the range covered by the data we used to run the <code>fit</code> method. For example</p> In\u00a0[23]: Copied! <pre>df.age.min(), df.age.max()\n</pre> df.age.min(), df.age.max() Out[23]: <pre>(17, 90)</pre> <p>All future age values outside that range will be encoded as 0 and decoded as <code>NaN</code></p> In\u00a0[24]: Copied! <pre>tmp_df = df.head(1).copy()\ntmp_df.loc[:, \"age\"] = 5\n</pre> tmp_df = df.head(1).copy() tmp_df.loc[:, \"age\"] = 5 In\u00a0[25]: Copied! <pre>tmp_df\n</pre> tmp_df Out[25]: age workclass fnlwgt education educational-num marital-status occupation relationship race gender capital-gain capital-loss hours-per-week native-country income 0 5 Private 226802 11th 7 Never-married Machine-op-inspct Own-child Black Male 0 0 40 United-States &lt;=50K In\u00a0[26]: Copied! <pre># quant_tab_preprocessor has already been fitted with a data that has an age range between 17 and 90\ntmp_qX_tab = quant_tab_preprocessor.transform(tmp_df)\n</pre> # quant_tab_preprocessor has already been fitted with a data that has an age range between 17 and 90 tmp_qX_tab = quant_tab_preprocessor.transform(tmp_df) In\u00a0[27]: Copied! <pre>tmp_qX_tab\n</pre> tmp_qX_tab Out[27]: <pre>array([[1, 1, 1, 1, 1, 0, 2]])</pre> In\u00a0[28]: Copied! <pre>quant_tab_preprocessor.inverse_transform(tmp_qX_tab)\n</pre> quant_tab_preprocessor.inverse_transform(tmp_qX_tab) <pre>Note that quantized cols will be turned into the mid point of the corresponding bin\n</pre> Out[28]: education relationship workclass occupation native-country age hours-per-week 0 11th Own-child Private Machine-op-inspct United-States NaN 30.4 In\u00a0[29]: Copied! <pre>from pytorch_widedeep.preprocessing import TextPreprocessor\n</pre> from pytorch_widedeep.preprocessing import TextPreprocessor In\u00a0[30]: Copied! <pre># The airbnb dataset, which you could get from here:\n# http://insideairbnb.com/get-the-data.html, is too big to be included in\n# our datasets module (when including images). Therefore, go there,\n# download it, and use the download_images.py script to get the images\n# and the airbnb_data_processing.py to process the data. We'll find\n# better datasets in the future ;). Note that here we are only using a\n# small sample to illustrate the use, so PLEASE ignore the results, just\n# focus on usage\ndf = pd.read_csv(\"../tmp_data/airbnb/airbnb_sample.csv\")\n</pre> # The airbnb dataset, which you could get from here: # http://insideairbnb.com/get-the-data.html, is too big to be included in # our datasets module (when including images). Therefore, go there, # download it, and use the download_images.py script to get the images # and the airbnb_data_processing.py to process the data. We'll find # better datasets in the future ;). Note that here we are only using a # small sample to illustrate the use, so PLEASE ignore the results, just # focus on usage df = pd.read_csv(\"../tmp_data/airbnb/airbnb_sample.csv\") In\u00a0[31]: Copied! <pre>texts = df.description.tolist()\ntexts[:2]\n</pre> texts = df.description.tolist() texts[:2] Out[31]: <pre>[\"My bright double bedroom with a large window has a relaxed feeling! It comfortably fits one or two and is centrally located just two blocks from Finsbury Park. Enjoy great restaurants in the area and easy access to easy transport tubes, trains and buses. Babies and children of all ages are welcome. Hello Everyone, I'm offering my lovely double bedroom in Finsbury Park area (zone 2) for let in a shared apartment.  You will share the apartment with me and it is fully furnished with a self catering kitchen. Two people can easily sleep well as the room has a queen size bed. I also have a travel cot for a baby for guest with small children.  I will require a deposit up front as a security gesture on both our parts and will be given back to you when you return the keys.  I trust anyone who will be responding to this add would treat my home with care and respect .  Best Wishes  Alina Guest will have access to the self catering kitchen and bathroom. There is the flat is equipped wifi internet,\",\n \"Lots of windows and light.  St Luke's Gardens are at the end of the block, and the river not too far the other way. Ten minutes walk if you go slowly. Buses to everywhere round the corner and shops, restaurants, pubs, the cinema and Waitrose . Bright Chelsea Apartment  This is a bright one bedroom ground floor apartment in an interesting listed building. There is one double bedroom and a living room/kitchen The apartment has a full  bathroom and the kitchen is fully equipped. Two wardrobes are available exclusively for guests and bedside tables and two long drawers. This sunny convenient compact flat is just around the corner from the Waitrose supermarket and all sorts of shops, cinemas, restaurants and pubs.  This is a lovely part of London. There is a fun farmers market in the King's Road at the weekend.  Buses to everywhere are just round the corner, and two underground stations are within ten minutes walk. There is a very nice pub round by St. Luke's gardens, 4 mins slow walk, the \"]</pre> In\u00a0[32]: Copied! <pre>text_preprocessor = TextPreprocessor(text_col=\"description\")\nX_text = text_preprocessor.fit_transform(df)\n# From here on, any new observation can be prepared by simply running `.transform`\n# new_X_text = text_preprocessor.transform(new_df)\n</pre> text_preprocessor = TextPreprocessor(text_col=\"description\") X_text = text_preprocessor.fit_transform(df) # From here on, any new observation can be prepared by simply running `.transform` # new_X_text = text_preprocessor.transform(new_df) <pre>The vocabulary contains 2192 tokens\n</pre> In\u00a0[33]: Copied! <pre>print(X_text[0])\n</pre> print(X_text[0]) <pre>[  29   48   37  367  818   17  910   17  177   15  122  349   53  879\n 1174  126  393   40  911    0   23  228   71  819    9   53   55 1380\n  225   11   18  308   18 1564   10  755    0  942  239   53   55    0\n   11   36 1013  277 1974   70   62   15 1475    9  943    5  251    5\n    0    5    0    5  177   53   37   75   11   10  294  726   32    9\n   42    5   25   12   10   22   12  136  100  145]\n</pre> In\u00a0[34]: Copied! <pre>from pytorch_widedeep.preprocessing import ImagePreprocessor\n</pre> from pytorch_widedeep.preprocessing import ImagePreprocessor In\u00a0[35]: Copied! <pre>image_preprocessor = wd.preprocessing.ImagePreprocessor(\n    img_col=\"id\", img_path=\"../tmp_data/airbnb/property_picture/\"\n)\nX_images = image_preprocessor.fit_transform(df)\n# From here on, any new observation can be prepared by simply running `.transform`\n# new_X_images = image_preprocessor.transform(new_df)\n</pre> image_preprocessor = wd.preprocessing.ImagePreprocessor(     img_col=\"id\", img_path=\"../tmp_data/airbnb/property_picture/\" ) X_images = image_preprocessor.fit_transform(df) # From here on, any new observation can be prepared by simply running `.transform` # new_X_images = image_preprocessor.transform(new_df) <pre>Reading Images from ../tmp_data/airbnb/property_picture/\nResizing\n</pre> <pre>100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 1001/1001 [00:01&lt;00:00, 667.89it/s]\n</pre> <pre>Computing normalisation metrics\n</pre> In\u00a0[36]: Copied! <pre>X_images[0].shape\n</pre> X_images[0].shape Out[36]: <pre>(224, 224, 3)</pre>"},{"location":"examples/01_preprocessors_and_utils.html#processors-and-utils","title":"Processors and Utils\u00b6","text":"<p>Description of the main tools and utilities that one needs to prepare the data for a <code>WideDeep</code> model constructor.</p>"},{"location":"examples/01_preprocessors_and_utils.html#the-preprocessing-module","title":"The <code>preprocessing</code>  module\u00b6","text":"<p>There are 4 preprocessors, corresponding to 4 main components of the <code>WideDeep</code> model. These are</p> <ul> <li><code>WidePreprocessor</code></li> <li><code>TabPreprocessor</code></li> <li><code>TextPreprocessor</code></li> <li><code>ImagePreprocessor</code></li> </ul> <p>Behind the scenes, these preprocessors use a series of helper funcions and classes that are in the <code>utils</code> module. If you were interested please go and have a look to the documentation</p>"},{"location":"examples/01_preprocessors_and_utils.html#1-widepreprocessor","title":"1. WidePreprocessor\u00b6","text":"<p>The <code>wide</code> component of the model is a linear model that in principle, could be implemented as a linear layer receiving the result of on one-hot encoding categorical columns. However, this is not memory efficient. Therefore, we implement a liner layer as an Embedding layer plus a bias. I will explain in a bit more detail later.</p> <p>With that in mind, <code>WidePreprocessor</code> simply encodes the categories numerically so that they are the indexes of the lookup table that is an Embedding layer.</p>"},{"location":"examples/01_preprocessors_and_utils.html#2-tabpreprocessor","title":"2. TabPreprocessor\u00b6","text":"<p>The <code>TabPreprocessor</code> has a lot of different functionalities. Let's explore some of them in detail. In its basic use, the <code>TabPreprocessor</code> simply label encodes the categorical columns and normalises the numerical ones (unless otherwised specified).</p>"},{"location":"examples/01_preprocessors_and_utils.html#3-textpreprocessor","title":"3. TextPreprocessor\u00b6","text":"<p>This preprocessor returns the tokenised, padded sequences that will be directly fed to the stack of LSTMs.</p>"},{"location":"examples/01_preprocessors_and_utils.html#4-imagepreprocessor","title":"4. ImagePreprocessor\u00b6","text":"<p><code>ImagePreprocessor</code> simply resizes the images, being aware of the aspect ratio.</p>"},{"location":"examples/02_model_components.html","title":"02_model_components","text":"In\u00a0[1]: Copied! <pre>import torch\nimport pandas as pd\nimport numpy as np\n\nfrom torch import nn\n</pre> import torch import pandas as pd import numpy as np  from torch import nn In\u00a0[2]: Copied! <pre>df = pd.DataFrame({\"color\": [\"r\", \"b\", \"g\"], \"size\": [\"s\", \"n\", \"l\"]})\ndf.head()\n</pre> df = pd.DataFrame({\"color\": [\"r\", \"b\", \"g\"], \"size\": [\"s\", \"n\", \"l\"]}) df.head() Out[2]: color size 0 r s 1 b n 2 g l <p>one hot encoded, the first observation would be</p> In\u00a0[3]: Copied! <pre>obs_0_oh = (np.array([1.0, 0.0, 0.0, 1.0, 0.0, 0.0])).astype(\"float32\")\n</pre> obs_0_oh = (np.array([1.0, 0.0, 0.0, 1.0, 0.0, 0.0])).astype(\"float32\") <p>if we simply numerically encode (label encode or <code>le</code>) the values:</p> In\u00a0[4]: Copied! <pre>obs_0_le = (np.array([0, 3])).astype(\"int64\")\n</pre> obs_0_le = (np.array([0, 3])).astype(\"int64\") <p>Note that in the functioning implementation of the package we start from 1, saving 0 for padding, i.e. unseen values.</p> <p>Now, let's see if the two implementations are equivalent</p> In\u00a0[5]: Copied! <pre># we have 6 different values. Let's assume we are performing a regression, so pred_dim = 1\nlin = nn.Linear(6, 1)\n</pre> # we have 6 different values. Let's assume we are performing a regression, so pred_dim = 1 lin = nn.Linear(6, 1) In\u00a0[6]: Copied! <pre>emb = nn.Embedding(6, 1)\nemb.weight = nn.Parameter(lin.weight.reshape_as(emb.weight))\n</pre> emb = nn.Embedding(6, 1) emb.weight = nn.Parameter(lin.weight.reshape_as(emb.weight)) In\u00a0[7]: Copied! <pre>lin(torch.tensor(obs_0_oh))\n</pre> lin(torch.tensor(obs_0_oh)) Out[7]: <pre>tensor([-0.5181], grad_fn=&lt;ViewBackward0&gt;)</pre> In\u00a0[8]: Copied! <pre>emb(torch.tensor(obs_0_le)).sum() + lin.bias\n</pre> emb(torch.tensor(obs_0_le)).sum() + lin.bias Out[8]: <pre>tensor([-0.5181], grad_fn=&lt;AddBackward0&gt;)</pre> <p>And this is precisely how the linear model <code>Wide</code> is implemented</p> In\u00a0[9]: Copied! <pre>from pytorch_widedeep.models import Wide\n</pre> from pytorch_widedeep.models import Wide <pre>/Users/javierrodriguezzaurin/.pyenv/versions/3.10.13/envs/widedeep310/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n  from .autonotebook import tqdm as notebook_tqdm\n</pre> In\u00a0[10]: Copied! <pre># ?Wide\n</pre> # ?Wide In\u00a0[11]: Copied! <pre>wide = Wide(input_dim=10, pred_dim=1)\nwide\n</pre> wide = Wide(input_dim=10, pred_dim=1) wide Out[11]: <pre>Wide(\n  (wide_linear): Embedding(11, 1, padding_idx=0)\n)</pre> <p>Note that even though the input dim is 10, the Embedding layer has 11 weights. Again, this is because we save <code>0</code> for padding, which is used for unseen values during the encoding process.</p> <p>As I mentioned, <code>deeptabular</code> has enough complexity on its own and it will be described in a separated notebook. Let's then jump to <code>deeptext</code>.</p> In\u00a0[12]: Copied! <pre>from pytorch_widedeep.preprocessing import TabPreprocessor\nfrom pytorch_widedeep.models import TabMlp\n</pre> from pytorch_widedeep.preprocessing import TabPreprocessor from pytorch_widedeep.models import TabMlp In\u00a0[13]: Copied! <pre>data = {\n    \"cat1\": np.random.choice([\"A\", \"B\", \"C\"], size=20),\n    \"cat2\": np.random.choice([\"X\", \"Y\"], size=20),\n    \"cont1\": np.random.rand(20),\n    \"cont2\": np.random.rand(20),\n}\n\ndf = pd.DataFrame(data)\n</pre> data = {     \"cat1\": np.random.choice([\"A\", \"B\", \"C\"], size=20),     \"cat2\": np.random.choice([\"X\", \"Y\"], size=20),     \"cont1\": np.random.rand(20),     \"cont2\": np.random.rand(20), }  df = pd.DataFrame(data) In\u00a0[14]: Copied! <pre>df.head()\n</pre> df.head() Out[14]: cat1 cat2 cont1 cont2 0 A Y 0.789347 0.561789 1 C X 0.050822 0.061538 2 A Y 0.863784 0.241967 3 C X 0.917848 0.644658 4 C Y 0.042328 0.417303 In\u00a0[15]: Copied! <pre># see the docs for details on all params/options\ntab_preprocessor = TabPreprocessor(\n    cat_embed_cols=[\"cat1\", \"cat2\"],\n    continuous_cols=[\"cont1\", \"cont2\"],\n    embedding_rule=\"fastai\",\n)\n</pre> # see the docs for details on all params/options tab_preprocessor = TabPreprocessor(     cat_embed_cols=[\"cat1\", \"cat2\"],     continuous_cols=[\"cont1\", \"cont2\"],     embedding_rule=\"fastai\", ) In\u00a0[16]: Copied! <pre>X_tab = tab_preprocessor.fit_transform(df)\n</pre> X_tab = tab_preprocessor.fit_transform(df) <pre>/Users/javierrodriguezzaurin/Projects/pytorch-widedeep/pytorch_widedeep/preprocessing/tab_preprocessor.py:358: UserWarning: Continuous columns will not be normalised\n  warnings.warn(\"Continuous columns will not be normalised\")\n</pre> In\u00a0[17]: Copied! <pre># toy example just to build a model.\ntabmlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=tab_preprocessor.continuous_cols,\n    embed_continuous_method=\"standard\",\n    cont_embed_dim=4,\n    mlp_hidden_dims=[8, 4],\n    mlp_linear_first=True,\n)\ntabmlp\n</pre> # toy example just to build a model. tabmlp = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     continuous_cols=tab_preprocessor.continuous_cols,     embed_continuous_method=\"standard\",     cont_embed_dim=4,     mlp_hidden_dims=[8, 4],     mlp_linear_first=True, ) tabmlp Out[17]: <pre>TabMlp(\n  (cat_embed): DiffSizeCatEmbeddings(\n    (embed_layers): ModuleDict(\n      (emb_layer_cat1): Embedding(4, 3, padding_idx=0)\n      (emb_layer_cat2): Embedding(3, 2, padding_idx=0)\n    )\n    (embedding_dropout): Dropout(p=0.0, inplace=False)\n  )\n  (cont_norm): Identity()\n  (cont_embed): ContEmbeddings(\n    INFO: [ContLinear = weight(n_cont_cols, embed_dim) + bias(n_cont_cols, embed_dim)]\n    (linear): ContLinear(n_cont_cols=2, embed_dim=4, embed_dropout=0.0)\n    (dropout): Dropout(p=0.0, inplace=False)\n  )\n  (encoder): MLP(\n    (mlp): Sequential(\n      (dense_layer_0): Sequential(\n        (0): Linear(in_features=13, out_features=8, bias=True)\n        (1): ReLU(inplace=True)\n        (2): Dropout(p=0.1, inplace=False)\n      )\n      (dense_layer_1): Sequential(\n        (0): Linear(in_features=8, out_features=4, bias=True)\n        (1): ReLU(inplace=True)\n        (2): Dropout(p=0.1, inplace=False)\n      )\n    )\n  )\n)</pre> <p>Lets describe a bit the model: first we have what we call a <code>DiffSizeCatEmbeddings</code>, where categorical columns with different number of unique categories will be encoded with embeddings of different dimensions. Then the continuous columns will not be normalised (the normalised layer is just the identity) and they will be embedded via a \"standard\" method, using a so-called <code>ContLinear</code> layer. This layer displays some <code>INFO</code> that tells us what it is (<code>ContLinear = weight(n_cont_cols, embed_dim) + bias(n_cont_cols, embed_dim)]</code>). There are two other options available to embed the continuous cols based on the paper On Embeddings for Numerical Features in Tabular Deep Learning. These are <code>PieceWise</code> and <code>Periodic</code> and all available via the <code>embed_continuous_method</code> param, which can adopt values <code>\"standard\", \"piecewise\"</code> and <code>\"periodic\"</code>. The embedded categorical and continuous columns will be then concatenated ($3 + 2 + (4 * 2) = 13$ input dims) and passed to an MLP.</p> In\u00a0[18]: Copied! <pre>from pytorch_widedeep.models import BasicRNN\n</pre> from pytorch_widedeep.models import BasicRNN In\u00a0[19]: Copied! <pre>basic_rnn = BasicRNN(vocab_size=4, hidden_dim=4, n_layers=1, padding_idx=0, embed_dim=4)\n</pre> basic_rnn = BasicRNN(vocab_size=4, hidden_dim=4, n_layers=1, padding_idx=0, embed_dim=4) <pre>/Users/javierrodriguezzaurin/.pyenv/versions/3.10.13/envs/widedeep310/lib/python3.10/site-packages/torch/nn/modules/rnn.py:82: UserWarning: dropout option adds dropout after all but last recurrent layer, so non-zero dropout expects num_layers greater than 1, but got dropout=0.1 and num_layers=1\n  warnings.warn(\"dropout option adds dropout after all but last \"\n</pre> In\u00a0[20]: Copied! <pre>basic_rnn\n</pre> basic_rnn Out[20]: <pre>BasicRNN(\n  (word_embed): Embedding(4, 4, padding_idx=0)\n  (rnn): LSTM(4, 4, batch_first=True, dropout=0.1)\n  (rnn_mlp): Identity()\n)</pre> <p>You could, if you wanted, add a Fully Connected Head (FC-Head) on top of it</p> In\u00a0[21]: Copied! <pre>from pytorch_widedeep.models import Vision\n</pre> from pytorch_widedeep.models import Vision In\u00a0[22]: Copied! <pre>resnet = Vision(pretrained_model_setup=\"resnet18\", n_trainable=0)\n</pre> resnet = Vision(pretrained_model_setup=\"resnet18\", n_trainable=0) In\u00a0[23]: Copied! <pre>resnet\n</pre> resnet Out[23]: <pre>Vision(\n  (features): Sequential(\n    (0): Conv2d(3, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False)\n    (1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n    (2): ReLU(inplace=True)\n    (3): MaxPool2d(kernel_size=3, stride=2, padding=1, dilation=1, ceil_mode=False)\n    (4): Sequential(\n      (0): BasicBlock(\n        (conv1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n        (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (relu): ReLU(inplace=True)\n        (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n        (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n      )\n      (1): BasicBlock(\n        (conv1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n        (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (relu): ReLU(inplace=True)\n        (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n        (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n      )\n    )\n    (5): Sequential(\n      (0): BasicBlock(\n        (conv1): Conv2d(64, 128, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n        (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (relu): ReLU(inplace=True)\n        (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n        (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (downsample): Sequential(\n          (0): Conv2d(64, 128, kernel_size=(1, 1), stride=(2, 2), bias=False)\n          (1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        )\n      )\n      (1): BasicBlock(\n        (conv1): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n        (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (relu): ReLU(inplace=True)\n        (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n        (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n      )\n    )\n    (6): Sequential(\n      (0): BasicBlock(\n        (conv1): Conv2d(128, 256, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n        (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (relu): ReLU(inplace=True)\n        (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n        (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (downsample): Sequential(\n          (0): Conv2d(128, 256, kernel_size=(1, 1), stride=(2, 2), bias=False)\n          (1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        )\n      )\n      (1): BasicBlock(\n        (conv1): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n        (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (relu): ReLU(inplace=True)\n        (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n        (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n      )\n    )\n    (7): Sequential(\n      (0): BasicBlock(\n        (conv1): Conv2d(256, 512, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n        (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (relu): ReLU(inplace=True)\n        (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n        (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (downsample): Sequential(\n          (0): Conv2d(256, 512, kernel_size=(1, 1), stride=(2, 2), bias=False)\n          (1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        )\n      )\n      (1): BasicBlock(\n        (conv1): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n        (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (relu): ReLU(inplace=True)\n        (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n        (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n      )\n    )\n    (8): AdaptiveAvgPool2d(output_size=(1, 1))\n  )\n)</pre>"},{"location":"examples/02_model_components.html#model-components","title":"Model Components\u00b6","text":"<p>The main components of a <code>WideDeep</code> (i.e. Multimodal) model are tabular data, text and images, which are feed into the model via so called <code>wide</code>, <code>deeptabular</code>, <code>deeptext</code> and <code>deepimage</code> model components</p>"},{"location":"examples/02_model_components.html#1-wide","title":"1. <code>wide</code>\u00b6","text":"<p>The <code>wide</code> component is a Linear layer \"plugged\" into the output neuron(s). Here, the non-linearities are captured via crossed columns. Crossed columns are, quoting directly the paper: \"For binary features, a cross-product transformation (e.g., \u201cAND(gender=female, language=en)\u201d) is 1 if and only if the constituent features (\u201cgender=female\u201d and \u201clanguage=en\u201d) are all 1, and 0 otherwise\".</p> <p>The only particularity of our implementation is that we have implemented the linear layer via an Embedding layer plus a bias. While the implementations are equivalent, the latter is faster and far more memory efficient, since we do not need to one hot encode the categorical features.</p> <p>Let's assume we the following dataset:</p>"},{"location":"examples/02_model_components.html#2-deeptabular","title":"2. <code>deeptabular</code>\u00b6","text":"<p>The <code>deeptabular</code> model alone is what normally would be referred as Deep Learning for tabular data. As mentioned a number of times throughout the library, each component can be used independently. Therefore, if you wanted to use any of the models below alone, it is perfectly possible. There are just a couple of simple requirement that will be covered in a later notebook.</p> <p>By the time of writing, there are a number of models available in <code>pytorch-widedeep</code> to do DL for tabular data. These are:</p> <ol> <li><code>TabMlp</code></li> <li><code>ContextAttentionMLP</code></li> <li><code>SelfAttentionMLP</code></li> <li><code>TabResnet</code></li> <li><code>Tabnet</code></li> <li><code>TabTransformer</code></li> <li><code>FT-Tabransformer</code></li> <li><code>SAINT</code></li> <li><code>TabFastFormer</code></li> <li><code>TabPerceiver</code></li> </ol> <p>Let's have a look to one of them. For more information on each of these models, please, have a look to the documentation</p>"},{"location":"examples/02_model_components.html#3-deeptext","title":"3. <code>deeptext</code>\u00b6","text":"<p>At the time of writing, <code>pytorch-widedeep</code> offers three models that can be passed to <code>WideDeep</code> as the <code>deeptext</code> component. These are:</p> <ol> <li>BasicRNN</li> <li>AttentiveRNN</li> <li>StackedAttentiveRNN</li> </ol> <p>For details on each of these models, please, have a look to the documentation of the package.</p> <p>We will soon integrate with Hugginface, but let me insist. It is perfectly possible to use custom models for each component, please, have a look to the corresponding notebook. In general, simply, build them and pass them as the corresponding parameters. Note that the custom models MUST return a last layer of activations (i.e. not the final prediction) so that  these activations are collected by <code>WideDeep</code> and combined accordingly. In  addition, the models MUST also contain an attribute <code>output_dim</code> with the size of these last layers of activations.</p> <p>Let's have a look to the <code>BasicRNN</code> model</p>"},{"location":"examples/02_model_components.html#4-deepimage","title":"4. <code>deepimage</code>\u00b6","text":"<p>At the time of writing <code>pytorch-widedeep</code> is integrated with torchvision via the <code>Vision</code> class. This means that the it is possible to use a variant of the following architectures:</p> <ol> <li>resnet</li> <li>shufflenet</li> <li>resnext</li> <li>wide_resnet</li> <li>regnet</li> <li>densenet</li> <li>mobilenet</li> <li>mnasnet</li> <li>efficientnet</li> <li>squeezenet</li> </ol> <p>The user can choose which layers will be trainable. Alternatively, in none of these architectures is useful, one could use a simple, fully trained CNN (please see the package documentation) or pass a custom model.</p> <p>let's have a look</p>"},{"location":"examples/03_binary_classification_with_defaults.html","title":"03_binary_classification_with_defaults","text":"In\u00a0[1]: Copied! <pre>import numpy as np\nimport pandas as pd\nimport torch\n\nfrom pytorch_widedeep.preprocessing import WidePreprocessor, TabPreprocessor\nfrom pytorch_widedeep.training import Trainer\nfrom pytorch_widedeep.models import Wide, TabMlp, WideDeep\nfrom pytorch_widedeep.metrics import Accuracy, Precision\nfrom pytorch_widedeep.datasets import load_adult\n</pre> import numpy as np import pandas as pd import torch  from pytorch_widedeep.preprocessing import WidePreprocessor, TabPreprocessor from pytorch_widedeep.training import Trainer from pytorch_widedeep.models import Wide, TabMlp, WideDeep from pytorch_widedeep.metrics import Accuracy, Precision from pytorch_widedeep.datasets import load_adult <pre>/Users/javierrodriguezzaurin/.pyenv/versions/3.10.13/envs/widedeep310/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n  from .autonotebook import tqdm as notebook_tqdm\n</pre> In\u00a0[2]: Copied! <pre>df = load_adult(as_frame=True)\ndf.head()\n</pre> df = load_adult(as_frame=True) df.head() Out[2]: age workclass fnlwgt education educational-num marital-status occupation relationship race gender capital-gain capital-loss hours-per-week native-country income 0 25 Private 226802 11th 7 Never-married Machine-op-inspct Own-child Black Male 0 0 40 United-States &lt;=50K 1 38 Private 89814 HS-grad 9 Married-civ-spouse Farming-fishing Husband White Male 0 0 50 United-States &lt;=50K 2 28 Local-gov 336951 Assoc-acdm 12 Married-civ-spouse Protective-serv Husband White Male 0 0 40 United-States &gt;50K 3 44 Private 160323 Some-college 10 Married-civ-spouse Machine-op-inspct Husband Black Male 7688 0 40 United-States &gt;50K 4 18 ? 103497 Some-college 10 Never-married ? Own-child White Female 0 0 30 United-States &lt;=50K In\u00a0[3]: Copied! <pre># For convenience, we'll replace '-' with '_'\ndf.columns = [c.replace(\"-\", \"_\") for c in df.columns]\n# binary target\ndf[\"income_label\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int)\ndf.drop(\"income\", axis=1, inplace=True)\ndf.head()\n</pre> # For convenience, we'll replace '-' with '_' df.columns = [c.replace(\"-\", \"_\") for c in df.columns] # binary target df[\"income_label\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int) df.drop(\"income\", axis=1, inplace=True) df.head() Out[3]: age workclass fnlwgt education educational_num marital_status occupation relationship race gender capital_gain capital_loss hours_per_week native_country income_label 0 25 Private 226802 11th 7 Never-married Machine-op-inspct Own-child Black Male 0 0 40 United-States 0 1 38 Private 89814 HS-grad 9 Married-civ-spouse Farming-fishing Husband White Male 0 0 50 United-States 0 2 28 Local-gov 336951 Assoc-acdm 12 Married-civ-spouse Protective-serv Husband White Male 0 0 40 United-States 1 3 44 Private 160323 Some-college 10 Married-civ-spouse Machine-op-inspct Husband Black Male 7688 0 40 United-States 1 4 18 ? 103497 Some-college 10 Never-married ? Own-child White Female 0 0 30 United-States 0 In\u00a0[4]: Copied! <pre>df.drop([\"fnlwgt\", \"educational_num\"], axis=1, inplace=True)\n</pre> df.drop([\"fnlwgt\", \"educational_num\"], axis=1, inplace=True) In\u00a0[5]: Copied! <pre># Define wide, crossed and deep tabular columns\nwide_cols = [\n    \"workclass\",\n    \"education\",\n    \"marital_status\",\n    \"occupation\",\n    \"relationship\",\n    \"race\",\n    \"gender\",\n    \"native_country\",\n]\ncrossed_cols = [(\"education\", \"occupation\"), (\"native_country\", \"occupation\")]\n</pre> # Define wide, crossed and deep tabular columns wide_cols = [     \"workclass\",     \"education\",     \"marital_status\",     \"occupation\",     \"relationship\",     \"race\",     \"gender\",     \"native_country\", ] crossed_cols = [(\"education\", \"occupation\"), (\"native_country\", \"occupation\")] In\u00a0[6]: Copied! <pre>cat_embed_cols = [\n    \"workclass\",\n    \"education\",\n    \"marital_status\",\n    \"occupation\",\n    \"relationship\",\n    \"race\",\n    \"gender\",\n    \"capital_gain\",\n    \"capital_loss\",\n    \"native_country\",\n]\ncontinuous_cols = [\"age\", \"hours_per_week\"]\n</pre> cat_embed_cols = [     \"workclass\",     \"education\",     \"marital_status\",     \"occupation\",     \"relationship\",     \"race\",     \"gender\",     \"capital_gain\",     \"capital_loss\",     \"native_country\", ] continuous_cols = [\"age\", \"hours_per_week\"] In\u00a0[7]: Copied! <pre># TARGET\ntarget_col = \"income_label\"\ntarget = df[target_col].values\n</pre> # TARGET target_col = \"income_label\" target = df[target_col].values <p>let's see what the preprocessors do</p> In\u00a0[8]: Copied! <pre># wide\nwide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols)\nX_wide = wide_preprocessor.fit_transform(df)\n</pre> # wide wide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols) X_wide = wide_preprocessor.fit_transform(df) In\u00a0[9]: Copied! <pre># # wide_preprocessor has an attribute called encoding_dict with the encoding dictionary\n# wide_preprocessor.encoding_dict\n</pre> # # wide_preprocessor has an attribute called encoding_dict with the encoding dictionary # wide_preprocessor.encoding_dict In\u00a0[10]: Copied! <pre># deeptabular\ntab_preprocessor = TabPreprocessor(\n    embed_cols=cat_embed_cols,\n    continuous_cols=continuous_cols,\n    cols_to_scale=continuous_cols,\n)\nX_tab = tab_preprocessor.fit_transform(df)\n</pre> # deeptabular tab_preprocessor = TabPreprocessor(     embed_cols=cat_embed_cols,     continuous_cols=continuous_cols,     cols_to_scale=continuous_cols, ) X_tab = tab_preprocessor.fit_transform(df) In\u00a0[11]: Copied! <pre># check the docs to understand the useful attributes that the tab_preprocessor has. For example,\n# as well as an encoding dictionary, tab_preprocessor has an attribute called cat_embed_input\n# that specifies the categortical columns that will be represented as embeddings, the number\n# of different categories per feature, and the dimension of the embeddings as defined by some\n# of the internal rules of thumb that the preprocessor has (have a look to the docs)\ntab_preprocessor.cat_embed_input\n</pre> # check the docs to understand the useful attributes that the tab_preprocessor has. For example, # as well as an encoding dictionary, tab_preprocessor has an attribute called cat_embed_input # that specifies the categortical columns that will be represented as embeddings, the number # of different categories per feature, and the dimension of the embeddings as defined by some # of the internal rules of thumb that the preprocessor has (have a look to the docs) tab_preprocessor.cat_embed_input Out[11]: <pre>[('workclass', 9, 5),\n ('education', 16, 8),\n ('marital_status', 7, 5),\n ('occupation', 15, 7),\n ('relationship', 6, 4),\n ('race', 5, 4),\n ('gender', 2, 2),\n ('capital_gain', 123, 24),\n ('capital_loss', 99, 21),\n ('native_country', 42, 13)]</pre> In\u00a0[12]: Copied! <pre>print(X_wide)\nprint(X_wide.shape)\n</pre> print(X_wide) print(X_wide.shape) <pre>[[  1  10  26 ...  61 103 328]\n [  1  11  27 ...  61 104 329]\n [  2  12  27 ...  61 105 330]\n ...\n [  1  11  28 ...  61 115 335]\n [  1  11  26 ...  61 115 335]\n [  7  11  27 ...  61 127 336]]\n(48842, 10)\n</pre> In\u00a0[13]: Copied! <pre>print(X_tab)\nprint(X_tab.shape)\n</pre> print(X_tab) print(X_tab.shape) <pre>[[ 1.          1.          1.         ...  1.         -0.99512893\n  -0.03408696]\n [ 1.          2.          2.         ...  1.         -0.04694151\n   0.77292975]\n [ 2.          3.          2.         ...  1.         -0.77631645\n  -0.03408696]\n ...\n [ 1.          2.          3.         ...  1.          1.41180837\n  -0.03408696]\n [ 1.          2.          1.         ...  1.         -1.21394141\n  -1.64812038]\n [ 7.          2.          2.         ...  1.          0.97418341\n  -0.03408696]]\n(48842, 12)\n</pre> In\u00a0[14]: Copied! <pre>wide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1)\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    cat_embed_dropout=0.1,\n    continuous_cols=continuous_cols,\n    mlp_hidden_dims=[400, 200],\n    mlp_dropout=0.5,\n    mlp_activation=\"leaky_relu\",\n)\n</pre> wide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1) tab_mlp = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     cat_embed_dropout=0.1,     continuous_cols=continuous_cols,     mlp_hidden_dims=[400, 200],     mlp_dropout=0.5,     mlp_activation=\"leaky_relu\", ) <p>Let's first find out how a linear model performs</p> In\u00a0[15]: Copied! <pre>wide\n</pre> wide Out[15]: <pre>Wide(\n  (wide_linear): Embedding(809, 1, padding_idx=0)\n)</pre> <p>Before being passed to the Trainer, the models need to be \"constructed\" with the <code>WideDeep</code> constructor class. For the particular case of the wide/linear model, not much really happens</p> In\u00a0[16]: Copied! <pre>lin_model = WideDeep(wide=wide)\n</pre> lin_model = WideDeep(wide=wide) In\u00a0[17]: Copied! <pre>lin_model\n</pre> lin_model Out[17]: <pre>WideDeep(\n  (wide): Wide(\n    (wide_linear): Embedding(809, 1, padding_idx=0)\n  )\n)</pre> In\u00a0[18]: Copied! <pre>lin_trainer = Trainer(\n    model=lin_model,\n    objective=\"binary\",\n    optimizers=torch.optim.AdamW(lin_model.parameters(), lr=0.01),\n    metrics=[Accuracy, Precision],\n)\n</pre> lin_trainer = Trainer(     model=lin_model,     objective=\"binary\",     optimizers=torch.optim.AdamW(lin_model.parameters(), lr=0.01),     metrics=[Accuracy, Precision], ) In\u00a0[19]: Copied! <pre>lin_trainer.fit(X_wide=X_wide, target=target, n_epochs=4, batch_size=128, val_split=0.2)\n</pre> lin_trainer.fit(X_wide=X_wide, target=target, n_epochs=4, batch_size=128, val_split=0.2) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 306/306 [00:02&lt;00:00, 109.04it/s, loss=0.426, metrics={'acc': 0.7983, 'prec': 0.6152}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 77/77 [00:00&lt;00:00, 102.46it/s, loss=0.366, metrics={'acc': 0.832, 'prec': 0.6916}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 306/306 [00:02&lt;00:00, 130.27it/s, loss=0.364, metrics={'acc': 0.8305, 'prec': 0.6933}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 77/77 [00:00&lt;00:00, 150.46it/s, loss=0.361, metrics={'acc': 0.8357, 'prec': 0.6982}]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 306/306 [00:02&lt;00:00, 133.19it/s, loss=0.359, metrics={'acc': 0.8329, 'prec': 0.6994}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 77/77 [00:00&lt;00:00, 145.75it/s, loss=0.361, metrics={'acc': 0.836, 'prec': 0.7009}]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 306/306 [00:02&lt;00:00, 130.91it/s, loss=0.358, metrics={'acc': 0.8333, 'prec': 0.7005}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 77/77 [00:00&lt;00:00, 155.08it/s, loss=0.361, metrics={'acc': 0.8364, 'prec': 0.702}]\n</pre> <p>Bear in mind that <code>wide</code> is a linear model where the non-linearities are captured via the crossed columns. For the crossed-columns to be effective one needs proper business knowledge. There is no magic formula to produce them</p> <p>Let's have a look to the tabular model by itself</p> In\u00a0[20]: Copied! <pre>tab_model = WideDeep(deeptabular=tab_mlp)\n</pre> tab_model = WideDeep(deeptabular=tab_mlp) In\u00a0[21]: Copied! <pre>tab_model\n</pre> tab_model Out[21]: <pre>WideDeep(\n  (deeptabular): Sequential(\n    (0): TabMlp(\n      (cat_embed): DiffSizeCatEmbeddings(\n        (embed_layers): ModuleDict(\n          (emb_layer_workclass): Embedding(10, 5, padding_idx=0)\n          (emb_layer_education): Embedding(17, 8, padding_idx=0)\n          (emb_layer_marital_status): Embedding(8, 5, padding_idx=0)\n          (emb_layer_occupation): Embedding(16, 7, padding_idx=0)\n          (emb_layer_relationship): Embedding(7, 4, padding_idx=0)\n          (emb_layer_race): Embedding(6, 4, padding_idx=0)\n          (emb_layer_gender): Embedding(3, 2, padding_idx=0)\n          (emb_layer_capital_gain): Embedding(124, 24, padding_idx=0)\n          (emb_layer_capital_loss): Embedding(100, 21, padding_idx=0)\n          (emb_layer_native_country): Embedding(43, 13, padding_idx=0)\n        )\n        (embedding_dropout): Dropout(p=0.1, inplace=False)\n      )\n      (cont_norm): Identity()\n      (encoder): MLP(\n        (mlp): Sequential(\n          (dense_layer_0): Sequential(\n            (0): Linear(in_features=95, out_features=400, bias=True)\n            (1): LeakyReLU(negative_slope=0.01, inplace=True)\n            (2): Dropout(p=0.5, inplace=False)\n          )\n          (dense_layer_1): Sequential(\n            (0): Linear(in_features=400, out_features=200, bias=True)\n            (1): LeakyReLU(negative_slope=0.01, inplace=True)\n            (2): Dropout(p=0.5, inplace=False)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=200, out_features=1, bias=True)\n  )\n)</pre> <p>You can see how the <code>WideDeep</code> class has added a final prediction layer that collects the activations from the last layer of the model and plugs them into the output neuron. If this was a multiclass classification problem, the prediction dimension (i.e. the size of that final layer) needs to be specified via the <code>pred_dim</code> when instantiating the <code>WideDeep</code> class, as we will see later</p> In\u00a0[22]: Copied! <pre>tab_trainer = Trainer(\n    model=tab_model,\n    objective=\"binary\",\n    optimizers=torch.optim.AdamW(tab_model.parameters(), lr=0.001),\n    metrics=[Accuracy, Precision],\n)\n</pre> tab_trainer = Trainer(     model=tab_model,     objective=\"binary\",     optimizers=torch.optim.AdamW(tab_model.parameters(), lr=0.001),     metrics=[Accuracy, Precision], ) In\u00a0[23]: Copied! <pre>tab_trainer.fit(X_tab=X_tab, target=target, n_epochs=4, batch_size=128, val_split=0.2)\n</pre> tab_trainer.fit(X_tab=X_tab, target=target, n_epochs=4, batch_size=128, val_split=0.2) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 306/306 [00:03&lt;00:00, 97.00it/s, loss=0.37, metrics={'acc': 0.8267, 'prec': 0.7037}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 77/77 [00:00&lt;00:00, 134.91it/s, loss=0.313, metrics={'acc': 0.8588, 'prec': 0.7577}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 306/306 [00:03&lt;00:00, 86.86it/s, loss=0.319, metrics={'acc': 0.8514, 'prec': 0.761}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 77/77 [00:01&lt;00:00, 73.13it/s, loss=0.296, metrics={'acc': 0.8675, 'prec': 0.7685}]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 306/306 [00:03&lt;00:00, 79.07it/s, loss=0.305, metrics={'acc': 0.8574, 'prec': 0.7646}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 77/77 [00:00&lt;00:00, 130.11it/s, loss=0.289, metrics={'acc': 0.8696, 'prec': 0.7765}]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 306/306 [00:03&lt;00:00, 87.39it/s, loss=0.296, metrics={'acc': 0.8622, 'prec': 0.7769}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 77/77 [00:00&lt;00:00, 90.63it/s, loss=0.285, metrics={'acc': 0.8697, 'prec': 0.7741}]\n</pre> <p>The best result I ever obtained with <code>LightGBM</code> on this dataset is 0.8782...so we are pretty close.</p> <p>Let's combine the <code>wide</code> and <code>tab_mlp</code> components see if it helps</p> In\u00a0[24]: Copied! <pre>wide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1)\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    cat_embed_dropout=0.1,\n    continuous_cols=continuous_cols,\n    mlp_hidden_dims=[400, 200],\n    mlp_dropout=0.5,\n    mlp_activation=\"leaky_relu\",\n)\nwd_model = WideDeep(wide=wide, deeptabular=tab_mlp)\n</pre> wide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1) tab_mlp = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     cat_embed_dropout=0.1,     continuous_cols=continuous_cols,     mlp_hidden_dims=[400, 200],     mlp_dropout=0.5,     mlp_activation=\"leaky_relu\", ) wd_model = WideDeep(wide=wide, deeptabular=tab_mlp) In\u00a0[25]: Copied! <pre>wd_trainer = Trainer(\n    model=wd_model,\n    objective=\"binary\",\n    optimizers=torch.optim.AdamW(wd_model.parameters(), lr=0.001),\n    metrics=[Accuracy, Precision],\n)\n</pre> wd_trainer = Trainer(     model=wd_model,     objective=\"binary\",     optimizers=torch.optim.AdamW(wd_model.parameters(), lr=0.001),     metrics=[Accuracy, Precision], ) In\u00a0[26]: Copied! <pre>wd_trainer.fit(\n    X_wide=X_wide, X_tab=X_tab, target=target, n_epochs=4, batch_size=128, val_split=0.2\n)\n</pre> wd_trainer.fit(     X_wide=X_wide, X_tab=X_tab, target=target, n_epochs=4, batch_size=128, val_split=0.2 ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 306/306 [00:03&lt;00:00, 77.48it/s, loss=0.418, metrics={'acc': 0.8047, 'prec': 0.6154}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 77/77 [00:00&lt;00:00, 110.51it/s, loss=0.321, metrics={'acc': 0.8521, 'prec': 0.7059}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 306/306 [00:03&lt;00:00, 82.70it/s, loss=0.333, metrics={'acc': 0.8428, 'prec': 0.7141}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 77/77 [00:00&lt;00:00, 112.52it/s, loss=0.299, metrics={'acc': 0.866, 'prec': 0.7447}]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 306/306 [00:04&lt;00:00, 74.34it/s, loss=0.312, metrics={'acc': 0.8533, 'prec': 0.7404}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 77/77 [00:00&lt;00:00, 89.86it/s, loss=0.29, metrics={'acc': 0.8683, 'prec': 0.7496}]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 306/306 [00:04&lt;00:00, 65.32it/s, loss=0.301, metrics={'acc': 0.8591, 'prec': 0.7542}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 77/77 [00:00&lt;00:00, 86.81it/s, loss=0.286, metrics={'acc': 0.8712, 'prec': 0.7552}]\n</pre> <p>For this particular case, the combination of both did not lead to better results that using just the tab_mlp model, when using only 4 epochs.</p> <p>Note that we have use a <code>TabMlp</code> model, but we could use any other model in the library using the same syntax</p> In\u00a0[27]: Copied! <pre>from pytorch_widedeep.models import TabTransformer\n</pre> from pytorch_widedeep.models import TabTransformer <p>The parameters for the <code>TabTransformer</code> are this</p> <pre><code>column_idx: Dict[str, int],\ncat_embed_input: Optional[List[Tuple[str, int]]] = None,\ncat_embed_dropout: Optional[float] = None,\nuse_cat_bias: Optional[bool] = None,\ncat_embed_activation: Optional[str] = None,\nshared_embed: Optional[bool] = None,\nadd_shared_embed: Optional[bool] = None,\nfrac_shared_embed: Optional[float] = None,\ncontinuous_cols: Optional[List[str]] = None,\ncont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\nembed_continuous: Optional[bool] = None,\nembed_continuous_method: Optional[Literal[\"standard\", \"piecewise\", \"periodic\"]] = None,\ncont_embed_dropout: Optional[float] = None,\ncont_embed_activation: Optional[str] = None,\nquantization_setup: Optional[Dict[str, List[float]]] = None,\nn_frequencies: Optional[int] = None,\nsigma: Optional[float] = None,\nshare_last_layer: Optional[bool] = None,\nfull_embed_dropout: Optional[bool] = None,\ninput_dim: int = 32,\nn_heads: int = 8,\nuse_qkv_bias: bool = False,\nn_blocks: int = 4,\nattn_dropout: float = 0.2,\nff_dropout: float = 0.1,\nff_factor: int = 4,\ntransformer_activation: str = \"gelu\",\nuse_linear_attention: bool = False,\nuse_flash_attention: bool = False,\nmlp_hidden_dims: Optional[List[int]] = None,\nmlp_activation: str = \"relu\",\nmlp_dropout: float = 0.1,\nmlp_batchnorm: bool = False,\nmlp_batchnorm_last: bool = False,\nmlp_linear_first: bool = True,\n</code></pre> <p>Please, see the documentation for details on each one of them, for now let's see how one could use a <code>TabTransformer</code> model in a few lines of code</p> In\u00a0[28]: Copied! <pre>tab_transformer = TabTransformer(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    cat_embed_dropout=0.1,\n    continuous_cols=continuous_cols,\n    embed_continuous_method=\"standard\",\n    cont_norm_layer=\"layernorm\",\n    cont_embed_dropout=0.2,\n    cont_embed_activation=\"leaky_relu\",\n    n_heads=4,\n    ff_dropout=0.2,\n    mlp_dropout=0.5,\n    mlp_activation=\"leaky_relu\",\n    mlp_linear_first=\"True\",\n)\n</pre> tab_transformer = TabTransformer(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     cat_embed_dropout=0.1,     continuous_cols=continuous_cols,     embed_continuous_method=\"standard\",     cont_norm_layer=\"layernorm\",     cont_embed_dropout=0.2,     cont_embed_activation=\"leaky_relu\",     n_heads=4,     ff_dropout=0.2,     mlp_dropout=0.5,     mlp_activation=\"leaky_relu\",     mlp_linear_first=\"True\", ) In\u00a0[29]: Copied! <pre>tab_model = WideDeep(deeptabular=tab_transformer)\n</pre> tab_model = WideDeep(deeptabular=tab_transformer) In\u00a0[30]: Copied! <pre>tab_model\n</pre> tab_model Out[30]: <pre>WideDeep(\n  (deeptabular): Sequential(\n    (0): TabTransformer(\n      (cat_embed): SameSizeCatEmbeddings(\n        (embed): Embedding(325, 32, padding_idx=0)\n        (dropout): Dropout(p=0.1, inplace=False)\n      )\n      (cont_norm): LayerNorm((2,), eps=1e-05, elementwise_affine=True)\n      (cont_embed): ContEmbeddings(\n        INFO: [ContLinear = weight(n_cont_cols, embed_dim) + bias(n_cont_cols, embed_dim)]\n        (linear): ContLinear(n_cont_cols=2, embed_dim=32, embed_dropout=0.2)\n        (activation_fn): LeakyReLU(negative_slope=0.01, inplace=True)\n        (dropout): Dropout(p=0.2, inplace=False)\n      )\n      (encoder): Sequential(\n        (transformer_block0): TransformerEncoder(\n          (attn): MultiHeadedAttention(\n            (dropout): Dropout(p=0.2, inplace=False)\n            (q_proj): Linear(in_features=32, out_features=32, bias=False)\n            (kv_proj): Linear(in_features=32, out_features=64, bias=False)\n            (out_proj): Linear(in_features=32, out_features=32, bias=False)\n          )\n          (ff): FeedForward(\n            (w_1): Linear(in_features=32, out_features=128, bias=True)\n            (w_2): Linear(in_features=128, out_features=32, bias=True)\n            (dropout): Dropout(p=0.2, inplace=False)\n            (activation): GELU(approximate='none')\n          )\n          (attn_addnorm): AddNorm(\n            (dropout): Dropout(p=0.2, inplace=False)\n            (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n          )\n          (ff_addnorm): AddNorm(\n            (dropout): Dropout(p=0.2, inplace=False)\n            (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n          )\n        )\n        (transformer_block1): TransformerEncoder(\n          (attn): MultiHeadedAttention(\n            (dropout): Dropout(p=0.2, inplace=False)\n            (q_proj): Linear(in_features=32, out_features=32, bias=False)\n            (kv_proj): Linear(in_features=32, out_features=64, bias=False)\n            (out_proj): Linear(in_features=32, out_features=32, bias=False)\n          )\n          (ff): FeedForward(\n            (w_1): Linear(in_features=32, out_features=128, bias=True)\n            (w_2): Linear(in_features=128, out_features=32, bias=True)\n            (dropout): Dropout(p=0.2, inplace=False)\n            (activation): GELU(approximate='none')\n          )\n          (attn_addnorm): AddNorm(\n            (dropout): Dropout(p=0.2, inplace=False)\n            (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n          )\n          (ff_addnorm): AddNorm(\n            (dropout): Dropout(p=0.2, inplace=False)\n            (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n          )\n        )\n        (transformer_block2): TransformerEncoder(\n          (attn): MultiHeadedAttention(\n            (dropout): Dropout(p=0.2, inplace=False)\n            (q_proj): Linear(in_features=32, out_features=32, bias=False)\n            (kv_proj): Linear(in_features=32, out_features=64, bias=False)\n            (out_proj): Linear(in_features=32, out_features=32, bias=False)\n          )\n          (ff): FeedForward(\n            (w_1): Linear(in_features=32, out_features=128, bias=True)\n            (w_2): Linear(in_features=128, out_features=32, bias=True)\n            (dropout): Dropout(p=0.2, inplace=False)\n            (activation): GELU(approximate='none')\n          )\n          (attn_addnorm): AddNorm(\n            (dropout): Dropout(p=0.2, inplace=False)\n            (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n          )\n          (ff_addnorm): AddNorm(\n            (dropout): Dropout(p=0.2, inplace=False)\n            (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n          )\n        )\n        (transformer_block3): TransformerEncoder(\n          (attn): MultiHeadedAttention(\n            (dropout): Dropout(p=0.2, inplace=False)\n            (q_proj): Linear(in_features=32, out_features=32, bias=False)\n            (kv_proj): Linear(in_features=32, out_features=64, bias=False)\n            (out_proj): Linear(in_features=32, out_features=32, bias=False)\n          )\n          (ff): FeedForward(\n            (w_1): Linear(in_features=32, out_features=128, bias=True)\n            (w_2): Linear(in_features=128, out_features=32, bias=True)\n            (dropout): Dropout(p=0.2, inplace=False)\n            (activation): GELU(approximate='none')\n          )\n          (attn_addnorm): AddNorm(\n            (dropout): Dropout(p=0.2, inplace=False)\n            (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n          )\n          (ff_addnorm): AddNorm(\n            (dropout): Dropout(p=0.2, inplace=False)\n            (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=384, out_features=1, bias=True)\n  )\n)</pre> In\u00a0[31]: Copied! <pre>tab_trainer = Trainer(\n    model=tab_model,\n    objective=\"binary\",\n    optimizers=torch.optim.AdamW(tab_model.parameters(), lr=0.001),\n    metrics=[Accuracy, Precision],\n)\n</pre> tab_trainer = Trainer(     model=tab_model,     objective=\"binary\",     optimizers=torch.optim.AdamW(tab_model.parameters(), lr=0.001),     metrics=[Accuracy, Precision], ) In\u00a0[32]: Copied! <pre>tab_trainer.fit(X_tab=X_tab, target=target, n_epochs=1, batch_size=128, val_split=0.2)\n</pre> tab_trainer.fit(X_tab=X_tab, target=target, n_epochs=1, batch_size=128, val_split=0.2) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 306/306 [00:11&lt;00:00, 27.57it/s, loss=0.359, metrics={'acc': 0.8334, 'prec': 0.7082}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 77/77 [00:01&lt;00:00, 57.89it/s, loss=0.33, metrics={'acc': 0.8536, 'prec': 0.7152}]\n</pre>"},{"location":"examples/03_binary_classification_with_defaults.html#simple-binary-classification-with-defaults","title":"Simple Binary Classification with defaults\u00b6","text":"<p>In this notebook we will train a Wide and Deep model and simply a \"Deep\" model using the well known adult dataset</p>"},{"location":"examples/03_binary_classification_with_defaults.html#preparing-the-data","title":"Preparing the data\u00b6","text":""},{"location":"examples/03_binary_classification_with_defaults.html#defining-the-model","title":"Defining the model\u00b6","text":""},{"location":"examples/04_regression_with_images_and_text.html","title":"04_regression_with_images_and_text","text":"In\u00a0[1]: Copied! <pre>import numpy as np\nimport pandas as pd\nimport os\nimport torch\nfrom torchvision.transforms import ToTensor, Normalize\n\nfrom pytorch_widedeep import Trainer\nfrom pytorch_widedeep.preprocessing import (\n    WidePreprocessor,\n    TabPreprocessor,\n    TextPreprocessor,\n    ImagePreprocessor,\n)\nfrom pytorch_widedeep.models import (\n    Wide,\n    TabMlp,\n    Vision,\n    BasicRNN,\n    WideDeep,\n)\nfrom pytorch_widedeep.losses import RMSELoss\nfrom pytorch_widedeep.initializers import *\nfrom pytorch_widedeep.callbacks import *\n</pre> import numpy as np import pandas as pd import os import torch from torchvision.transforms import ToTensor, Normalize  from pytorch_widedeep import Trainer from pytorch_widedeep.preprocessing import (     WidePreprocessor,     TabPreprocessor,     TextPreprocessor,     ImagePreprocessor, ) from pytorch_widedeep.models import (     Wide,     TabMlp,     Vision,     BasicRNN,     WideDeep, ) from pytorch_widedeep.losses import RMSELoss from pytorch_widedeep.initializers import * from pytorch_widedeep.callbacks import * <pre>/Users/javierrodriguezzaurin/.pyenv/versions/3.10.13/envs/widedeep310/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n  from .autonotebook import tqdm as notebook_tqdm\n</pre> In\u00a0[2]: Copied! <pre>df = pd.read_csv(\"../tmp_data/airbnb/airbnb_sample.csv\")\ndf.head()\n</pre> df = pd.read_csv(\"../tmp_data/airbnb/airbnb_sample.csv\") df.head() Out[2]: id host_id description host_listings_count host_identity_verified neighbourhood_cleansed latitude longitude is_location_exact property_type ... amenity_wide_entrance amenity_wide_entrance_for_guests amenity_wide_entryway amenity_wide_hallways amenity_wifi amenity_window_guards amenity_wine_cooler security_deposit extra_people yield 0 13913.jpg 54730 My bright double bedroom with a large window h... 4.0 f Islington 51.56802 -0.11121 t apartment ... 1 0 0 0 1 0 0 100.0 15.0 12.00 1 15400.jpg 60302 Lots of windows and light.  St Luke's Gardens ... 1.0 t Kensington and Chelsea 51.48796 -0.16898 t apartment ... 0 0 0 0 1 0 0 150.0 0.0 109.50 2 17402.jpg 67564 Open from June 2018 after a 3-year break, we a... 19.0 t Westminster 51.52098 -0.14002 t apartment ... 0 0 0 0 1 0 0 350.0 10.0 149.65 3 24328.jpg 41759 Artist house, bright high ceiling rooms, priva... 2.0 t Wandsworth 51.47298 -0.16376 t other ... 0 0 0 0 1 0 0 250.0 0.0 215.60 4 25023.jpg 102813 Large, all comforts, 2-bed flat; first floor; ... 1.0 f Wandsworth 51.44687 -0.21874 t apartment ... 0 0 0 0 1 0 0 250.0 11.0 79.35 <p>5 rows \u00d7 223 columns</p> In\u00a0[3]: Copied! <pre># There are a number of columns that are already binary. Therefore, no need to one hot encode them\ncrossed_cols = [(\"property_type\", \"room_type\")]\nalready_dummies = [c for c in df.columns if \"amenity\" in c] + [\"has_house_rules\"]\nwide_cols = [\n    \"is_location_exact\",\n    \"property_type\",\n    \"room_type\",\n    \"host_gender\",\n    \"instant_bookable\",\n] + already_dummies\n\ncat_embed_cols = [(c, 16) for c in df.columns if \"catg\" in c] + [\n    (\"neighbourhood_cleansed\", 64),\n    (\"cancellation_policy\", 16),\n]\ncontinuous_cols = [\"latitude\", \"longitude\", \"security_deposit\", \"extra_people\"]\n\n# text and image colnames\ntext_col = \"description\"\nimg_col = \"id\"\n\n# path to pretrained word embeddings and the images\nword_vectors_path = \"../tmp_data/glove.6B/glove.6B.100d.txt\"\nimg_path = \"../tmp_data/airbnb/property_picture\"\n\n# target\ntarget_col = \"yield\"\n</pre> # There are a number of columns that are already binary. Therefore, no need to one hot encode them crossed_cols = [(\"property_type\", \"room_type\")] already_dummies = [c for c in df.columns if \"amenity\" in c] + [\"has_house_rules\"] wide_cols = [     \"is_location_exact\",     \"property_type\",     \"room_type\",     \"host_gender\",     \"instant_bookable\", ] + already_dummies  cat_embed_cols = [(c, 16) for c in df.columns if \"catg\" in c] + [     (\"neighbourhood_cleansed\", 64),     (\"cancellation_policy\", 16), ] continuous_cols = [\"latitude\", \"longitude\", \"security_deposit\", \"extra_people\"]  # text and image colnames text_col = \"description\" img_col = \"id\"  # path to pretrained word embeddings and the images word_vectors_path = \"../tmp_data/glove.6B/glove.6B.100d.txt\" img_path = \"../tmp_data/airbnb/property_picture\"  # target target_col = \"yield\" In\u00a0[4]: Copied! <pre>target = df[target_col].values\n</pre> target = df[target_col].values In\u00a0[5]: Copied! <pre>wide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols)\nX_wide = wide_preprocessor.fit_transform(df)\n</pre> wide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols) X_wide = wide_preprocessor.fit_transform(df) In\u00a0[6]: Copied! <pre>tab_preprocessor = TabPreprocessor(\n    cat_embed_cols=cat_embed_cols,\n    continuous_cols=continuous_cols,\n)\nX_tab = tab_preprocessor.fit_transform(df)\n</pre> tab_preprocessor = TabPreprocessor(     cat_embed_cols=cat_embed_cols,     continuous_cols=continuous_cols, ) X_tab = tab_preprocessor.fit_transform(df) <pre>/Users/javierrodriguezzaurin/Projects/pytorch-widedeep/pytorch_widedeep/preprocessing/tab_preprocessor.py:358: UserWarning: Continuous columns will not be normalised\n  warnings.warn(\"Continuous columns will not be normalised\")\n</pre> In\u00a0[7]: Copied! <pre>text_preprocessor = TextPreprocessor(\n    word_vectors_path=word_vectors_path, text_col=text_col\n)\nX_text = text_preprocessor.fit_transform(df)\n</pre> text_preprocessor = TextPreprocessor(     word_vectors_path=word_vectors_path, text_col=text_col ) X_text = text_preprocessor.fit_transform(df) <pre>The vocabulary contains 2192 tokens\nIndexing word vectors...\nLoaded 400000 word vectors\nPreparing embeddings matrix...\n2175 words in the vocabulary had ../tmp_data/glove.6B/glove.6B.100d.txt vectors and appear more than 5 times\n</pre> In\u00a0[8]: Copied! <pre>image_processor = ImagePreprocessor(img_col=img_col, img_path=img_path)\nX_images = image_processor.fit_transform(df)\n</pre> image_processor = ImagePreprocessor(img_col=img_col, img_path=img_path) X_images = image_processor.fit_transform(df) <pre>Reading Images from ../tmp_data/airbnb/property_picture\nResizing\n</pre> <pre>100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 1001/1001 [00:01&lt;00:00, 638.00it/s]\n</pre> <pre>Computing normalisation metrics\n</pre> In\u00a0[9]: Copied! <pre># Linear model\nwide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1)\n\n# DeepDense: 2 Dense layers\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=continuous_cols,\n    mlp_hidden_dims=[128, 64],\n    mlp_dropout=0.1,\n)\n\n# DeepText: a stack of 2 LSTMs\nbasic_rnn = BasicRNN(\n    vocab_size=len(text_preprocessor.vocab.itos),\n    embed_matrix=text_preprocessor.embedding_matrix,\n    n_layers=2,\n    hidden_dim=64,\n    rnn_dropout=0.5,\n)\n\n# Pretrained Resnet 18\nresnet = Vision(pretrained_model_setup=\"resnet18\", n_trainable=4)\n</pre> # Linear model wide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1)  # DeepDense: 2 Dense layers tab_mlp = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     continuous_cols=continuous_cols,     mlp_hidden_dims=[128, 64],     mlp_dropout=0.1, )  # DeepText: a stack of 2 LSTMs basic_rnn = BasicRNN(     vocab_size=len(text_preprocessor.vocab.itos),     embed_matrix=text_preprocessor.embedding_matrix,     n_layers=2,     hidden_dim=64,     rnn_dropout=0.5, )  # Pretrained Resnet 18 resnet = Vision(pretrained_model_setup=\"resnet18\", n_trainable=4) <p>Combine them all with the \"collector\" class <code>WideDeep</code></p> In\u00a0[10]: Copied! <pre>model = WideDeep(\n    wide=wide,\n    deeptabular=tab_mlp,\n    deeptext=basic_rnn,\n    deepimage=resnet,\n    head_hidden_dims=[256, 128],\n)\n</pre> model = WideDeep(     wide=wide,     deeptabular=tab_mlp,     deeptext=basic_rnn,     deepimage=resnet,     head_hidden_dims=[256, 128], ) In\u00a0[11]: Copied! <pre>trainer = Trainer(model, objective=\"rmse\")\n</pre> trainer = Trainer(model, objective=\"rmse\") In\u00a0[12]: Copied! <pre>trainer.fit(\n    X_wide=X_wide,\n    X_tab=X_tab,\n    X_text=X_text,\n    X_img=X_images,\n    target=target,\n    n_epochs=1,\n    batch_size=32,\n    val_split=0.2,\n)\n</pre> trainer.fit(     X_wide=X_wide,     X_tab=X_tab,     X_text=X_text,     X_img=X_images,     target=target,     n_epochs=1,     batch_size=32,     val_split=0.2, ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 25/25 [00:19&lt;00:00,  1.28it/s, loss=115]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 7/7 [00:04&lt;00:00,  1.62it/s, loss=94.1]\n</pre> <p>Both, the Text and Image components allow FC-heads on their own (have a look to the documentation).</p> <p>Now let's go \"kaggle crazy\". Let's use different optimizers, initializers and schedulers for different components. Moreover, let's use a different learning rate for different parameter groups, for the <code>deeptabular</code> component</p> In\u00a0[13]: Copied! <pre>deep_params = []\nfor childname, child in model.named_children():\n    if childname == \"deeptabular\":\n        for n, p in child.named_parameters():\n            if \"embed_layer\" in n:\n                deep_params.append({\"params\": p, \"lr\": 1e-4})\n            else:\n                deep_params.append({\"params\": p, \"lr\": 1e-3})\n</pre> deep_params = [] for childname, child in model.named_children():     if childname == \"deeptabular\":         for n, p in child.named_parameters():             if \"embed_layer\" in n:                 deep_params.append({\"params\": p, \"lr\": 1e-4})             else:                 deep_params.append({\"params\": p, \"lr\": 1e-3}) In\u00a0[14]: Copied! <pre>wide_opt = torch.optim.Adam(model.wide.parameters(), lr=0.03)\ndeep_opt = torch.optim.Adam(deep_params)\ntext_opt = torch.optim.AdamW(model.deeptext.parameters())\nimg_opt = torch.optim.AdamW(model.deepimage.parameters())\nhead_opt = torch.optim.Adam(model.deephead.parameters())\n</pre> wide_opt = torch.optim.Adam(model.wide.parameters(), lr=0.03) deep_opt = torch.optim.Adam(deep_params) text_opt = torch.optim.AdamW(model.deeptext.parameters()) img_opt = torch.optim.AdamW(model.deepimage.parameters()) head_opt = torch.optim.Adam(model.deephead.parameters()) In\u00a0[15]: Copied! <pre>wide_sch = torch.optim.lr_scheduler.StepLR(wide_opt, step_size=5)\ndeep_sch = torch.optim.lr_scheduler.MultiStepLR(deep_opt, milestones=[3, 8])\ntext_sch = torch.optim.lr_scheduler.StepLR(text_opt, step_size=5)\nimg_sch = torch.optim.lr_scheduler.MultiStepLR(deep_opt, milestones=[3, 8])\nhead_sch = torch.optim.lr_scheduler.StepLR(head_opt, step_size=5)\n</pre> wide_sch = torch.optim.lr_scheduler.StepLR(wide_opt, step_size=5) deep_sch = torch.optim.lr_scheduler.MultiStepLR(deep_opt, milestones=[3, 8]) text_sch = torch.optim.lr_scheduler.StepLR(text_opt, step_size=5) img_sch = torch.optim.lr_scheduler.MultiStepLR(deep_opt, milestones=[3, 8]) head_sch = torch.optim.lr_scheduler.StepLR(head_opt, step_size=5) In\u00a0[16]: Copied! <pre># remember, one optimizer per model components, for lr_schedures and initializers is not neccesary\noptimizers = {\n    \"wide\": wide_opt,\n    \"deeptabular\": deep_opt,\n    \"deeptext\": text_opt,\n    \"deepimage\": img_opt,\n    \"deephead\": head_opt,\n}\nschedulers = {\n    \"wide\": wide_sch,\n    \"deeptabular\": deep_sch,\n    \"deeptext\": text_sch,\n    \"deepimage\": img_sch,\n    \"deephead\": head_sch,\n}\n\n# Now...we have used pretrained word embeddings, so you do not want to\n# initialise these  embeddings. However you might still want to initialise the\n# other layers in the DeepText component. No probs, you can do that with the\n# parameter pattern and your knowledge on regular  expressions. Here we are\n# telling to the KaimingNormal initializer to NOT touch the  parameters whose\n# name contains the string word_embed.\ninitializers = {\n    \"wide\": KaimingNormal,\n    \"deeptabular\": KaimingNormal,\n    \"deeptext\": KaimingNormal(pattern=r\"^(?!.*word_embed).*$\"),\n    \"deepimage\": KaimingNormal,\n}\n\nmean = [0.406, 0.456, 0.485]  # BGR\nstd = [0.225, 0.224, 0.229]  # BGR\ntransforms = [ToTensor, Normalize(mean=mean, std=std)]\ncallbacks = [\n    LRHistory(n_epochs=10),\n    EarlyStopping,\n    ModelCheckpoint(filepath=\"model_weights/wd_out\"),\n]\n</pre> # remember, one optimizer per model components, for lr_schedures and initializers is not neccesary optimizers = {     \"wide\": wide_opt,     \"deeptabular\": deep_opt,     \"deeptext\": text_opt,     \"deepimage\": img_opt,     \"deephead\": head_opt, } schedulers = {     \"wide\": wide_sch,     \"deeptabular\": deep_sch,     \"deeptext\": text_sch,     \"deepimage\": img_sch,     \"deephead\": head_sch, }  # Now...we have used pretrained word embeddings, so you do not want to # initialise these  embeddings. However you might still want to initialise the # other layers in the DeepText component. No probs, you can do that with the # parameter pattern and your knowledge on regular  expressions. Here we are # telling to the KaimingNormal initializer to NOT touch the  parameters whose # name contains the string word_embed. initializers = {     \"wide\": KaimingNormal,     \"deeptabular\": KaimingNormal,     \"deeptext\": KaimingNormal(pattern=r\"^(?!.*word_embed).*$\"),     \"deepimage\": KaimingNormal, }  mean = [0.406, 0.456, 0.485]  # BGR std = [0.225, 0.224, 0.229]  # BGR transforms = [ToTensor, Normalize(mean=mean, std=std)] callbacks = [     LRHistory(n_epochs=10),     EarlyStopping,     ModelCheckpoint(filepath=\"model_weights/wd_out\"), ] In\u00a0[17]: Copied! <pre>trainer = Trainer(\n    model,\n    objective=\"rmse\",\n    initializers=initializers,\n    optimizers=optimizers,\n    lr_schedulers=schedulers,\n    callbacks=callbacks,\n    transforms=transforms,\n)\n</pre> trainer = Trainer(     model,     objective=\"rmse\",     initializers=initializers,     optimizers=optimizers,     lr_schedulers=schedulers,     callbacks=callbacks,     transforms=transforms, ) <pre>/Users/javierrodriguezzaurin/Projects/pytorch-widedeep/pytorch_widedeep/initializers.py:34: UserWarning: No initializer found for deephead\n  warnings.warn(\n</pre> In\u00a0[18]: Copied! <pre>trainer.fit(\n    X_wide=X_wide,\n    X_tab=X_tab,\n    X_text=X_text,\n    X_img=X_images,\n    target=target,\n    n_epochs=1,\n    batch_size=32,\n    val_split=0.2,\n)\n</pre> trainer.fit(     X_wide=X_wide,     X_tab=X_tab,     X_text=X_text,     X_img=X_images,     target=target,     n_epochs=1,     batch_size=32,     val_split=0.2, ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 25/25 [00:19&lt;00:00,  1.25it/s, loss=101]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 7/7 [00:04&lt;00:00,  1.62it/s, loss=90.6]\n</pre> <pre>Model weights after training corresponds to the those of the final epoch which might not be the best performing weights. Use the 'ModelCheckpoint' Callback to restore the best epoch weights.\n</pre> <p>we have only run one epoch, but let's check that the LRHistory callback records the lr values for each group</p> In\u00a0[19]: Copied! <pre>trainer.lr_history\n</pre> trainer.lr_history Out[19]: <pre>{'lr_wide_0': [0.03, 0.03],\n 'lr_deeptabular_0': [0.0001, 0.0001],\n 'lr_deeptabular_1': [0.0001, 0.0001],\n 'lr_deeptabular_2': [0.0001, 0.0001],\n 'lr_deeptabular_3': [0.0001, 0.0001],\n 'lr_deeptabular_4': [0.0001, 0.0001],\n 'lr_deeptabular_5': [0.0001, 0.0001],\n 'lr_deeptabular_6': [0.0001, 0.0001],\n 'lr_deeptabular_7': [0.0001, 0.0001],\n 'lr_deeptabular_8': [0.0001, 0.0001],\n 'lr_deeptabular_9': [0.001, 0.001],\n 'lr_deeptabular_10': [0.001, 0.001],\n 'lr_deeptabular_11': [0.001, 0.001],\n 'lr_deeptabular_12': [0.001, 0.001],\n 'lr_deeptext_0': [0.001, 0.001],\n 'lr_deepimage_0': [0.001, 0.001],\n 'lr_deephead_0': [0.001, 0.001]}</pre>"},{"location":"examples/04_regression_with_images_and_text.html#regression-with-images-and-text","title":"Regression with Images and Text\u00b6","text":"<p>In this notebook we will go through a series of examples on how to combine all Wide &amp; Deep components.</p> <p>To that aim I will use the Airbnb listings dataset for London, which you can download from here. I use this dataset simply because it contains tabular data, images and text.</p> <p>I have taken a sample of 1000 listings to keep the data tractable in this notebook. Also, I have preprocessed the data and prepared it for this exercise. All preprocessing steps can be found in the notebook <code>airbnb_data_preprocessing.ipynb</code> in this <code>examples</code> folder.</p>"},{"location":"examples/04_regression_with_images_and_text.html#regression-with-the-defaults","title":"Regression with the defaults\u00b6","text":"<p>The set up</p>"},{"location":"examples/04_regression_with_images_and_text.html#prepare-the-data","title":"Prepare the data\u00b6","text":"<p>I will focus here on how to prepare the data and run the model. Check notebooks 1 and 2 to see what's going on behind the scences</p> <p>Preparing the data is rather simple</p>"},{"location":"examples/04_regression_with_images_and_text.html#build-the-model-components","title":"Build the model components\u00b6","text":""},{"location":"examples/04_regression_with_images_and_text.html#build-the-trainer-and-fit","title":"Build the trainer and fit\u00b6","text":""},{"location":"examples/05_save_and_load_model_and_artifacts.html","title":"05_save_and_load_model_and_artifacts","text":"In\u00a0[1]: Copied! <pre>import pickle\nimport numpy as np\nimport pandas as pd\nimport torch\nimport shutil\n\nfrom pytorch_widedeep.preprocessing import TabPreprocessor\nfrom pytorch_widedeep.training import Trainer\nfrom pytorch_widedeep.callbacks import EarlyStopping, ModelCheckpoint, LRHistory\nfrom pytorch_widedeep.models import TabMlp, WideDeep\nfrom pytorch_widedeep.metrics import Accuracy\nfrom pytorch_widedeep.datasets import load_adult\nfrom sklearn.model_selection import train_test_split\n</pre> import pickle import numpy as np import pandas as pd import torch import shutil  from pytorch_widedeep.preprocessing import TabPreprocessor from pytorch_widedeep.training import Trainer from pytorch_widedeep.callbacks import EarlyStopping, ModelCheckpoint, LRHistory from pytorch_widedeep.models import TabMlp, WideDeep from pytorch_widedeep.metrics import Accuracy from pytorch_widedeep.datasets import load_adult from sklearn.model_selection import train_test_split <pre>/Users/javierrodriguezzaurin/.pyenv/versions/3.10.13/envs/widedeep310/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n  from .autonotebook import tqdm as notebook_tqdm\n</pre> In\u00a0[2]: Copied! <pre>df = load_adult(as_frame=True)\ndf.head()\n</pre> df = load_adult(as_frame=True) df.head() Out[2]: age workclass fnlwgt education educational-num marital-status occupation relationship race gender capital-gain capital-loss hours-per-week native-country income 0 25 Private 226802 11th 7 Never-married Machine-op-inspct Own-child Black Male 0 0 40 United-States &lt;=50K 1 38 Private 89814 HS-grad 9 Married-civ-spouse Farming-fishing Husband White Male 0 0 50 United-States &lt;=50K 2 28 Local-gov 336951 Assoc-acdm 12 Married-civ-spouse Protective-serv Husband White Male 0 0 40 United-States &gt;50K 3 44 Private 160323 Some-college 10 Married-civ-spouse Machine-op-inspct Husband Black Male 7688 0 40 United-States &gt;50K 4 18 ? 103497 Some-college 10 Never-married ? Own-child White Female 0 0 30 United-States &lt;=50K In\u00a0[3]: Copied! <pre># For convenience, we'll replace '-' with '_'\ndf.columns = [c.replace(\"-\", \"_\") for c in df.columns]\n# binary target\ndf[\"target\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int)\ndf.drop(\"income\", axis=1, inplace=True)\ndf.head()\n</pre> # For convenience, we'll replace '-' with '_' df.columns = [c.replace(\"-\", \"_\") for c in df.columns] # binary target df[\"target\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int) df.drop(\"income\", axis=1, inplace=True) df.head() Out[3]: age workclass fnlwgt education educational_num marital_status occupation relationship race gender capital_gain capital_loss hours_per_week native_country target 0 25 Private 226802 11th 7 Never-married Machine-op-inspct Own-child Black Male 0 0 40 United-States 0 1 38 Private 89814 HS-grad 9 Married-civ-spouse Farming-fishing Husband White Male 0 0 50 United-States 0 2 28 Local-gov 336951 Assoc-acdm 12 Married-civ-spouse Protective-serv Husband White Male 0 0 40 United-States 1 3 44 Private 160323 Some-college 10 Married-civ-spouse Machine-op-inspct Husband Black Male 7688 0 40 United-States 1 4 18 ? 103497 Some-college 10 Never-married ? Own-child White Female 0 0 30 United-States 0 In\u00a0[4]: Copied! <pre>train, valid = train_test_split(df, test_size=0.2, stratify=df.target)\n# the test data will be used lately as if it was \"fresh\", new data coming after some time...\nvalid, test = train_test_split(valid, test_size=0.5, stratify=valid.target)\n</pre> train, valid = train_test_split(df, test_size=0.2, stratify=df.target) # the test data will be used lately as if it was \"fresh\", new data coming after some time... valid, test = train_test_split(valid, test_size=0.5, stratify=valid.target) In\u00a0[5]: Copied! <pre>print(f\"train shape: {train.shape}\")\nprint(f\"valid shape: {valid.shape}\")\nprint(f\"test shape: {test.shape}\")\n</pre> print(f\"train shape: {train.shape}\") print(f\"valid shape: {valid.shape}\") print(f\"test shape: {test.shape}\") <pre>train shape: (39073, 15)\nvalid shape: (4884, 15)\ntest shape: (4885, 15)\n</pre> In\u00a0[6]: Copied! <pre>cat_embed_cols = [\n    \"workclass\",\n    \"education\",\n    \"marital_status\",\n    \"occupation\",\n    \"relationship\",\n    \"race\",\n    \"gender\",\n    \"capital_gain\",\n    \"capital_loss\",\n    \"native_country\",\n]\ncontinuous_cols = [\"age\", \"hours_per_week\"]\n</pre> cat_embed_cols = [     \"workclass\",     \"education\",     \"marital_status\",     \"occupation\",     \"relationship\",     \"race\",     \"gender\",     \"capital_gain\",     \"capital_loss\",     \"native_country\", ] continuous_cols = [\"age\", \"hours_per_week\"] In\u00a0[7]: Copied! <pre>tab_preprocessor = TabPreprocessor(\n    embed_cols=cat_embed_cols,\n    continuous_cols=continuous_cols,\n)\nX_tab_train = tab_preprocessor.fit_transform(train)\ny_train = train.target.values\nX_tab_valid = tab_preprocessor.transform(valid)\ny_valid = valid.target.values\n</pre> tab_preprocessor = TabPreprocessor(     embed_cols=cat_embed_cols,     continuous_cols=continuous_cols, ) X_tab_train = tab_preprocessor.fit_transform(train) y_train = train.target.values X_tab_valid = tab_preprocessor.transform(valid) y_valid = valid.target.values <pre>/Users/javierrodriguezzaurin/Projects/pytorch-widedeep/pytorch_widedeep/preprocessing/tab_preprocessor.py:358: UserWarning: Continuous columns will not be normalised\n  warnings.warn(\"Continuous columns will not be normalised\")\n</pre> In\u00a0[8]: Copied! <pre>tab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    cat_embed_dropout=0.1,\n    continuous_cols=continuous_cols,\n    cont_norm_layer=\"layernorm\",\n    embed_continuous_method=\"standard\",\n    cont_embed_dim=8,\n    mlp_hidden_dims=[64, 32],\n    mlp_dropout=0.2,\n    mlp_activation=\"leaky_relu\",\n)\nmodel = WideDeep(deeptabular=tab_mlp)\n</pre> tab_mlp = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     cat_embed_dropout=0.1,     continuous_cols=continuous_cols,     cont_norm_layer=\"layernorm\",     embed_continuous_method=\"standard\",     cont_embed_dim=8,     mlp_hidden_dims=[64, 32],     mlp_dropout=0.2,     mlp_activation=\"leaky_relu\", ) model = WideDeep(deeptabular=tab_mlp) In\u00a0[9]: Copied! <pre>model\n</pre> model Out[9]: <pre>WideDeep(\n  (deeptabular): Sequential(\n    (0): TabMlp(\n      (cat_embed): DiffSizeCatEmbeddings(\n        (embed_layers): ModuleDict(\n          (emb_layer_workclass): Embedding(10, 5, padding_idx=0)\n          (emb_layer_education): Embedding(17, 8, padding_idx=0)\n          (emb_layer_marital_status): Embedding(8, 5, padding_idx=0)\n          (emb_layer_occupation): Embedding(16, 7, padding_idx=0)\n          (emb_layer_relationship): Embedding(7, 4, padding_idx=0)\n          (emb_layer_race): Embedding(6, 4, padding_idx=0)\n          (emb_layer_gender): Embedding(3, 2, padding_idx=0)\n          (emb_layer_capital_gain): Embedding(122, 23, padding_idx=0)\n          (emb_layer_capital_loss): Embedding(97, 21, padding_idx=0)\n          (emb_layer_native_country): Embedding(43, 13, padding_idx=0)\n        )\n        (embedding_dropout): Dropout(p=0.1, inplace=False)\n      )\n      (cont_norm): LayerNorm((2,), eps=1e-05, elementwise_affine=True)\n      (cont_embed): ContEmbeddings(\n        INFO: [ContLinear = weight(n_cont_cols, embed_dim) + bias(n_cont_cols, embed_dim)]\n        (linear): ContLinear(n_cont_cols=2, embed_dim=8, embed_dropout=0.0)\n        (dropout): Dropout(p=0.0, inplace=False)\n      )\n      (encoder): MLP(\n        (mlp): Sequential(\n          (dense_layer_0): Sequential(\n            (0): Linear(in_features=108, out_features=64, bias=True)\n            (1): LeakyReLU(negative_slope=0.01, inplace=True)\n            (2): Dropout(p=0.2, inplace=False)\n          )\n          (dense_layer_1): Sequential(\n            (0): Linear(in_features=64, out_features=32, bias=True)\n            (1): LeakyReLU(negative_slope=0.01, inplace=True)\n            (2): Dropout(p=0.2, inplace=False)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=32, out_features=1, bias=True)\n  )\n)</pre> In\u00a0[10]: Copied! <pre>early_stopping = EarlyStopping()\nmodel_checkpoint = ModelCheckpoint(\n    filepath=\"tmp_dir/adult_tabmlp_model\",\n    save_best_only=True,\n    verbose=1,\n    max_save=1,\n)\n\ntrainer = Trainer(\n    model,\n    objective=\"binary\",\n    callbacks=[early_stopping, model_checkpoint],\n    metrics=[Accuracy],\n)\n\ntrainer.fit(\n    X_train={\"X_tab\": X_tab_train, \"target\": y_train},\n    X_val={\"X_tab\": X_tab_valid, \"target\": y_valid},\n    n_epochs=4,\n    batch_size=256,\n)\n</pre> early_stopping = EarlyStopping() model_checkpoint = ModelCheckpoint(     filepath=\"tmp_dir/adult_tabmlp_model\",     save_best_only=True,     verbose=1,     max_save=1, )  trainer = Trainer(     model,     objective=\"binary\",     callbacks=[early_stopping, model_checkpoint],     metrics=[Accuracy], )  trainer.fit(     X_train={\"X_tab\": X_tab_train, \"target\": y_train},     X_val={\"X_tab\": X_tab_valid, \"target\": y_valid},     n_epochs=4,     batch_size=256, ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:02&lt;00:00, 76.25it/s, loss=0.452, metrics={'acc': 0.7867}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 20/20 [00:00&lt;00:00, 125.36it/s, loss=0.335, metrics={'acc': 0.8532}]\n</pre> <pre>\nEpoch 1: val_loss improved from inf to 0.33532 Saving model to tmp_dir/adult_tabmlp_model_1.p\n</pre> <pre>epoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 76.98it/s, loss=0.355, metrics={'acc': 0.8401}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 20/20 [00:00&lt;00:00, 106.51it/s, loss=0.303, metrics={'acc': 0.8665}]\n</pre> <pre>\nEpoch 2: val_loss improved from 0.33532 to 0.30273 Saving model to tmp_dir/adult_tabmlp_model_2.p\n</pre> <pre>epoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 82.71it/s, loss=0.332, metrics={'acc': 0.849}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 20/20 [00:00&lt;00:00, 107.80it/s, loss=0.288, metrics={'acc': 0.8757}]\n</pre> <pre>\nEpoch 3: val_loss improved from 0.30273 to 0.28791 Saving model to tmp_dir/adult_tabmlp_model_3.p\n</pre> <pre>epoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 79.02it/s, loss=0.32, metrics={'acc': 0.8541}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 20/20 [00:00&lt;00:00, 127.07it/s, loss=0.282, metrics={'acc': 0.8763}]</pre> <pre>\nEpoch 4: val_loss improved from 0.28791 to 0.28238 Saving model to tmp_dir/adult_tabmlp_model_4.p\nModel weights restored to best epoch: 4\n</pre> <pre>\n</pre> In\u00a0[11]: Copied! <pre>torch.save(model, \"tmp_dir/model_saved_option_1.pt\")\n</pre> torch.save(model, \"tmp_dir/model_saved_option_1.pt\") In\u00a0[12]: Copied! <pre>torch.save(model.state_dict(), \"tmp_dir/model_state_dict_saved_option_1.pt\")\n</pre> torch.save(model.state_dict(), \"tmp_dir/model_state_dict_saved_option_1.pt\") In\u00a0[13]: Copied! <pre>trainer.save(path=\"tmp_dir/\", model_filename=\"model_saved_option_2.pt\")\n</pre> trainer.save(path=\"tmp_dir/\", model_filename=\"model_saved_option_2.pt\") <p>or the state dict</p> In\u00a0[14]: Copied! <pre>trainer.save(\n    path=\"tmp_dir/\",\n    model_filename=\"model_state_dict_saved_option_2.pt\",\n    save_state_dict=True,\n)\n</pre> trainer.save(     path=\"tmp_dir/\",     model_filename=\"model_state_dict_saved_option_2.pt\",     save_state_dict=True, ) In\u00a0[15]: Copied! <pre>%%bash\n\nls tmp_dir/\n</pre> %%bash  ls tmp_dir/ <pre>adult_tabmlp_model_4.p\nhistory\nmodel_saved_option_1.pt\nmodel_saved_option_2.pt\nmodel_state_dict_saved_option_1.pt\nmodel_state_dict_saved_option_2.pt\n</pre> In\u00a0[16]: Copied! <pre>%%bash\n\nls tmp_dir/history/\n</pre> %%bash  ls tmp_dir/history/ <pre>train_eval_history.json\n</pre> <p>Note that since we have used the <code>ModelCheckpoint</code> Callback, <code>adult_tabmlp_model_2.p</code> is the model state dict of the model at epoch 2, i.e. same as <code>model_state_dict_saved_option_1.p</code> or <code>model_state_dict_saved_option_2.p</code>.</p> In\u00a0[17]: Copied! <pre>with open(\"tmp_dir/tab_preproc.pkl\", \"wb\") as dp:\n    pickle.dump(tab_preprocessor, dp)\n</pre> with open(\"tmp_dir/tab_preproc.pkl\", \"wb\") as dp:     pickle.dump(tab_preprocessor, dp) In\u00a0[18]: Copied! <pre>with open(\"tmp_dir/eary_stop.pkl\", \"wb\") as es:\n    pickle.dump(early_stopping, es)\n</pre> with open(\"tmp_dir/eary_stop.pkl\", \"wb\") as es:     pickle.dump(early_stopping, es) In\u00a0[19]: Copied! <pre>%%bash\n\nls tmp_dir/\n</pre> %%bash  ls tmp_dir/ <pre>adult_tabmlp_model_4.p\neary_stop.pkl\nhistory\nmodel_saved_option_1.pt\nmodel_saved_option_2.pt\nmodel_state_dict_saved_option_1.pt\nmodel_state_dict_saved_option_2.pt\ntab_preproc.pkl\n</pre> <p>And that is pretty much all you need to resume training or directly predict, let's see</p> In\u00a0[20]: Copied! <pre>test.head()\n</pre> test.head() Out[20]: age workclass fnlwgt education educational_num marital_status occupation relationship race gender capital_gain capital_loss hours_per_week native_country target 10103 43 Private 198282 HS-grad 9 Married-civ-spouse Craft-repair Husband White Male 0 0 40 United-States 1 31799 20 Private 228686 11th 7 Married-civ-spouse Other-service Husband White Male 0 0 40 United-States 0 19971 26 Private 291968 HS-grad 9 Married-civ-spouse Transport-moving Husband White Male 0 0 44 United-States 0 3039 48 Private 175958 Bachelors 13 Divorced Prof-specialty Not-in-family White Male 0 0 30 United-States 0 20725 18 Private 232024 11th 7 Never-married Machine-op-inspct Own-child White Male 0 0 55 United-States 0 In\u00a0[21]: Copied! <pre>with open(\"tmp_dir/tab_preproc.pkl\", \"rb\") as tp:\n    tab_preprocessor_new = pickle.load(tp)\n</pre> with open(\"tmp_dir/tab_preproc.pkl\", \"rb\") as tp:     tab_preprocessor_new = pickle.load(tp) In\u00a0[22]: Copied! <pre>X_test_tab = tab_preprocessor_new.transform(test)\ny_test = test.target\n</pre> X_test_tab = tab_preprocessor_new.transform(test) y_test = test.target In\u00a0[23]: Copied! <pre>tab_mlp_new = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    cat_embed_dropout=0.1,\n    continuous_cols=continuous_cols,\n    cont_norm_layer=\"layernorm\",\n    embed_continuous_method=\"standard\",\n    cont_embed_dim=8,\n    mlp_hidden_dims=[64, 32],\n    mlp_dropout=0.2,\n    mlp_activation=\"leaky_relu\",\n)\nnew_model = WideDeep(deeptabular=tab_mlp)\n</pre> tab_mlp_new = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     cat_embed_dropout=0.1,     continuous_cols=continuous_cols,     cont_norm_layer=\"layernorm\",     embed_continuous_method=\"standard\",     cont_embed_dim=8,     mlp_hidden_dims=[64, 32],     mlp_dropout=0.2,     mlp_activation=\"leaky_relu\", ) new_model = WideDeep(deeptabular=tab_mlp) In\u00a0[24]: Copied! <pre>new_model.load_state_dict(torch.load(\"tmp_dir/model_state_dict_saved_option_2.pt\"))\n</pre> new_model.load_state_dict(torch.load(\"tmp_dir/model_state_dict_saved_option_2.pt\")) Out[24]: <pre>&lt;All keys matched successfully&gt;</pre> In\u00a0[25]: Copied! <pre>trainer = Trainer(\n    model,\n    objective=\"binary\",\n)\n</pre> trainer = Trainer(     model,     objective=\"binary\", ) In\u00a0[26]: Copied! <pre>preds = trainer.predict(X_tab=X_test_tab, batch_size=32)\n</pre> preds = trainer.predict(X_tab=X_test_tab, batch_size=32) <pre>predict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:00&lt;00:00, 309.83it/s]\n</pre> In\u00a0[27]: Copied! <pre>from sklearn.metrics import accuracy_score\n</pre> from sklearn.metrics import accuracy_score In\u00a0[28]: Copied! <pre>accuracy_score(y_test, preds)\n</pre> accuracy_score(y_test, preds) Out[28]: <pre>0.8595701125895598</pre> In\u00a0[29]: Copied! <pre>shutil.rmtree(\"tmp_dir/\")\n</pre> shutil.rmtree(\"tmp_dir/\")"},{"location":"examples/05_save_and_load_model_and_artifacts.html#save-and-load-model-and-artifacts","title":"Save and load model and artifacts\u00b6","text":"<p>In this notebook I will show the different options to save and load a model, as well as some additional objects produced during training.</p> <p>On a given day, you train a model...</p>"},{"location":"examples/05_save_and_load_model_and_artifacts.html#save-model-option-1","title":"Save model: option 1\u00b6","text":"<p>save (and load) a model as you woud do with any other torch model</p>"},{"location":"examples/05_save_and_load_model_and_artifacts.html#save-model-option-2","title":"Save model: option 2\u00b6","text":"<p>use the <code>trainer</code>. The <code>trainer</code> will also save the training history and the learning rate history (if learning rate schedulers are used)</p>"},{"location":"examples/05_save_and_load_model_and_artifacts.html#save-preprocessors-and-callbacks","title":"Save preprocessors and callbacks\u00b6","text":"<p>...just pickle them</p>"},{"location":"examples/05_save_and_load_model_and_artifacts.html#run-new-experiment-prepare-new-dataset-load-model-and-predict","title":"Run New experiment: prepare new dataset, load model, and predict\u00b6","text":""},{"location":"examples/06_finetune_and_warmup.html","title":"06_finetune_and_warmup","text":"In\u00a0[1]: Copied! <pre>import numpy as np\nimport pandas as pd\nimport torch\n\nfrom pytorch_widedeep import Trainer\nfrom pytorch_widedeep.preprocessing import WidePreprocessor, TabPreprocessor\nfrom pytorch_widedeep.models import Wide, TabMlp, TabResnet, WideDeep\nfrom pytorch_widedeep.metrics import Accuracy\nfrom pytorch_widedeep.datasets import load_adult\n</pre> import numpy as np import pandas as pd import torch  from pytorch_widedeep import Trainer from pytorch_widedeep.preprocessing import WidePreprocessor, TabPreprocessor from pytorch_widedeep.models import Wide, TabMlp, TabResnet, WideDeep from pytorch_widedeep.metrics import Accuracy from pytorch_widedeep.datasets import load_adult <pre>/Users/javierrodriguezzaurin/.pyenv/versions/3.10.13/envs/widedeep310/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n  from .autonotebook import tqdm as notebook_tqdm\n</pre> In\u00a0[2]: Copied! <pre>df = load_adult(as_frame=True)\n# For convenience, we'll replace '-' with '_'\ndf.columns = [c.replace(\"-\", \"_\") for c in df.columns]\n# binary target\ndf[\"income_label\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int)\ndf.drop(\"income\", axis=1, inplace=True)\ndf.head()\n</pre> df = load_adult(as_frame=True) # For convenience, we'll replace '-' with '_' df.columns = [c.replace(\"-\", \"_\") for c in df.columns] # binary target df[\"income_label\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int) df.drop(\"income\", axis=1, inplace=True) df.head() Out[2]: age workclass fnlwgt education educational_num marital_status occupation relationship race gender capital_gain capital_loss hours_per_week native_country income_label 0 25 Private 226802 11th 7 Never-married Machine-op-inspct Own-child Black Male 0 0 40 United-States 0 1 38 Private 89814 HS-grad 9 Married-civ-spouse Farming-fishing Husband White Male 0 0 50 United-States 0 2 28 Local-gov 336951 Assoc-acdm 12 Married-civ-spouse Protective-serv Husband White Male 0 0 40 United-States 1 3 44 Private 160323 Some-college 10 Married-civ-spouse Machine-op-inspct Husband Black Male 7688 0 40 United-States 1 4 18 ? 103497 Some-college 10 Never-married ? Own-child White Female 0 0 30 United-States 0 In\u00a0[3]: Copied! <pre># Define wide, crossed and deep tabular columns\nwide_cols = [\n    \"workclass\",\n    \"education\",\n    \"marital_status\",\n    \"occupation\",\n    \"relationship\",\n    \"race\",\n    \"gender\",\n    \"native_country\",\n]\ncrossed_cols = [(\"education\", \"occupation\"), (\"native_country\", \"occupation\")]\ncat_embed_cols = [\n    \"workclass\",\n    \"education\",\n    \"marital_status\",\n    \"occupation\",\n    \"relationship\",\n    \"race\",\n    \"gender\",\n    \"capital_gain\",\n    \"capital_loss\",\n    \"native_country\",\n]\ncontinuous_cols = [\"age\", \"hours_per_week\"]\ntarget_col = \"income_label\"\ntarget = df[target_col].values\n</pre> # Define wide, crossed and deep tabular columns wide_cols = [     \"workclass\",     \"education\",     \"marital_status\",     \"occupation\",     \"relationship\",     \"race\",     \"gender\",     \"native_country\", ] crossed_cols = [(\"education\", \"occupation\"), (\"native_country\", \"occupation\")] cat_embed_cols = [     \"workclass\",     \"education\",     \"marital_status\",     \"occupation\",     \"relationship\",     \"race\",     \"gender\",     \"capital_gain\",     \"capital_loss\",     \"native_country\", ] continuous_cols = [\"age\", \"hours_per_week\"] target_col = \"income_label\" target = df[target_col].values In\u00a0[4]: Copied! <pre># TARGET\ntarget = df[target_col].values\n\n# WIDE\nwide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols)\nX_wide = wide_preprocessor.fit_transform(df)\n\n# DEEP\ntab_preprocessor = TabPreprocessor(\n    cat_embed_cols=cat_embed_cols, continuous_cols=continuous_cols\n)\nX_tab = tab_preprocessor.fit_transform(df)\n</pre> # TARGET target = df[target_col].values  # WIDE wide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols) X_wide = wide_preprocessor.fit_transform(df)  # DEEP tab_preprocessor = TabPreprocessor(     cat_embed_cols=cat_embed_cols, continuous_cols=continuous_cols ) X_tab = tab_preprocessor.fit_transform(df) <pre>/Users/javierrodriguezzaurin/Projects/pytorch-widedeep/pytorch_widedeep/preprocessing/tab_preprocessor.py:358: UserWarning: Continuous columns will not be normalised\n  warnings.warn(\"Continuous columns will not be normalised\")\n</pre> In\u00a0[5]: Copied! <pre>wide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1)\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    cat_embed_dropout=0.1,\n    continuous_cols=continuous_cols,\n    embed_continuous_method=\"standard\",\n    cont_embed_dim=8,\n    mlp_hidden_dims=[64, 32],\n    mlp_dropout=0.2,\n    mlp_activation=\"leaky_relu\",\n)\nmodel = WideDeep(wide=wide, deeptabular=tab_mlp)\n</pre> wide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1) tab_mlp = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     cat_embed_dropout=0.1,     continuous_cols=continuous_cols,     embed_continuous_method=\"standard\",     cont_embed_dim=8,     mlp_hidden_dims=[64, 32],     mlp_dropout=0.2,     mlp_activation=\"leaky_relu\", ) model = WideDeep(wide=wide, deeptabular=tab_mlp) In\u00a0[6]: Copied! <pre>trainer = Trainer(\n    model,\n    objective=\"binary\",\n    optimizers=torch.optim.Adam(model.parameters(), lr=0.01),\n    metrics=[Accuracy],\n)\n</pre> trainer = Trainer(     model,     objective=\"binary\",     optimizers=torch.optim.Adam(model.parameters(), lr=0.01),     metrics=[Accuracy], ) In\u00a0[7]: Copied! <pre>trainer.fit(\n    X_wide=X_wide, X_tab=X_tab, target=target, n_epochs=2, val_split=0.2, batch_size=256\n)\n</pre> trainer.fit(     X_wide=X_wide, X_tab=X_tab, target=target, n_epochs=2, val_split=0.2, batch_size=256 ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:02&lt;00:00, 74.26it/s, loss=0.399, metrics={'acc': 0.8163}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 39/39 [00:00&lt;00:00, 91.03it/s, loss=0.296, metrics={'acc': 0.8677}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 81.31it/s, loss=0.3, metrics={'acc': 0.8614}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 39/39 [00:00&lt;00:00, 106.45it/s, loss=0.285, metrics={'acc': 0.8721}]\n</pre> In\u00a0[8]: Copied! <pre>trainer.save(path=\"models_dir/\", save_state_dict=True, model_filename=\"model_1.pt\")\n</pre> trainer.save(path=\"models_dir/\", save_state_dict=True, model_filename=\"model_1.pt\") <p>Now time goes by...and we want to fine-tune the model to another, new dataset (for example, a dataset that is identical to the one you used to train the previous model but for another country).</p> <p>Here I will use the same dataset just for illustration purposes, but the flow would be identical to that new dataset</p> In\u00a0[9]: Copied! <pre>wide_1 = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1)\ntab_mlp_1 = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    cat_embed_dropout=0.1,\n    continuous_cols=continuous_cols,\n    embed_continuous_method=\"standard\",\n    cont_embed_dim=8,\n    mlp_hidden_dims=[64, 32],\n    mlp_dropout=0.2,\n    mlp_activation=\"leaky_relu\",\n)\nmodel_1 = WideDeep(wide=wide, deeptabular=tab_mlp)\n</pre> wide_1 = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1) tab_mlp_1 = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     cat_embed_dropout=0.1,     continuous_cols=continuous_cols,     embed_continuous_method=\"standard\",     cont_embed_dim=8,     mlp_hidden_dims=[64, 32],     mlp_dropout=0.2,     mlp_activation=\"leaky_relu\", ) model_1 = WideDeep(wide=wide, deeptabular=tab_mlp) In\u00a0[10]: Copied! <pre>model_1.load_state_dict(torch.load(\"models_dir/model_1.pt\"))\n</pre> model_1.load_state_dict(torch.load(\"models_dir/model_1.pt\")) Out[10]: <pre>&lt;All keys matched successfully&gt;</pre> In\u00a0[11]: Copied! <pre>trainer_1 = Trainer(model_1, objective=\"binary\", metrics=[Accuracy])\n</pre> trainer_1 = Trainer(model_1, objective=\"binary\", metrics=[Accuracy]) In\u00a0[12]: Copied! <pre>trainer_1.fit(\n    X_wide=X_wide,\n    X_tab=X_tab,\n    target=target,\n    n_epochs=2,\n    batch_size=256,\n    finetune=True,\n    finetune_epochs=2,\n)\n</pre> trainer_1.fit(     X_wide=X_wide,     X_tab=X_tab,     target=target,     n_epochs=2,     batch_size=256,     finetune=True,     finetune_epochs=2, ) <pre>Training wide for 2 epochs\n</pre> <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 191/191 [00:01&lt;00:00, 97.37it/s, loss=0.39, metrics={'acc': 0.8152}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 191/191 [00:01&lt;00:00, 104.04it/s, loss=0.359, metrics={'acc': 0.824}]\n</pre> <pre>Training deeptabular for 2 epochs\n</pre> <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 191/191 [00:02&lt;00:00, 83.83it/s, loss=0.297, metrics={'acc': 0.8365}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 191/191 [00:02&lt;00:00, 82.78it/s, loss=0.283, metrics={'acc': 0.8445}]\n</pre> <pre>Fine-tuning (or warmup) of individual components completed. Training the whole model for 2 epochs\n</pre> <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 191/191 [00:02&lt;00:00, 72.84it/s, loss=0.281, metrics={'acc': 0.8716}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 191/191 [00:02&lt;00:00, 77.46it/s, loss=0.273, metrics={'acc': 0.8744}]\n</pre> <p>Note that, as I describe above, in scenario 2, we can just use this to warm up models before they joined training begins:</p> In\u00a0[13]: Copied! <pre>wide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1)\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    cat_embed_dropout=0.1,\n    continuous_cols=continuous_cols,\n    embed_continuous_method=\"standard\",\n    cont_embed_dim=8,\n    mlp_hidden_dims=[64, 32],\n    mlp_dropout=0.2,\n    mlp_activation=\"leaky_relu\",\n)\nmodel = WideDeep(wide=wide, deeptabular=tab_mlp)\n</pre> wide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1) tab_mlp = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     cat_embed_dropout=0.1,     continuous_cols=continuous_cols,     embed_continuous_method=\"standard\",     cont_embed_dim=8,     mlp_hidden_dims=[64, 32],     mlp_dropout=0.2,     mlp_activation=\"leaky_relu\", ) model = WideDeep(wide=wide, deeptabular=tab_mlp) In\u00a0[14]: Copied! <pre>trainer_2 = Trainer(model, objective=\"binary\", metrics=[Accuracy])\n</pre> trainer_2 = Trainer(model, objective=\"binary\", metrics=[Accuracy]) In\u00a0[15]: Copied! <pre>trainer_2.fit(\n    X_wide=X_wide,\n    X_tab=X_tab,\n    target=target,\n    val_split=0.1,\n    warmup=True,\n    warmup_epochs=2,\n    n_epochs=2,\n    batch_size=256,\n)\n</pre> trainer_2.fit(     X_wide=X_wide,     X_tab=X_tab,     target=target,     val_split=0.1,     warmup=True,     warmup_epochs=2,     n_epochs=2,     batch_size=256, ) <pre>Training wide for 2 epochs\n</pre> <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:01&lt;00:00, 102.49it/s, loss=0.52, metrics={'acc': 0.7519}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:01&lt;00:00, 98.15it/s, loss=0.381, metrics={'acc': 0.7891}]\n</pre> <pre>Training deeptabular for 2 epochs\n</pre> <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:02&lt;00:00, 82.97it/s, loss=0.356, metrics={'acc': 0.8043}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:02&lt;00:00, 80.27it/s, loss=0.295, metrics={'acc': 0.8195}]\n</pre> <pre>Fine-tuning (or warmup) of individual components completed. Training the whole model for 2 epochs\n</pre> <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:02&lt;00:00, 77.27it/s, loss=0.291, metrics={'acc': 0.8667}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 20/20 [00:00&lt;00:00, 89.57it/s, loss=0.289, metrics={'acc': 0.8665}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:02&lt;00:00, 72.69it/s, loss=0.283, metrics={'acc': 0.8693}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 20/20 [00:00&lt;00:00, 91.81it/s, loss=0.284, metrics={'acc': 0.869}]\n</pre> <p>We need to explicitly indicate</p> <ol> <li><p>That we want fine-tune</p> </li> <li><p>The components that we want to individually fine-tune</p> </li> <li><p>In case of gradual fine-tuning, the routine (\"felbo\" or \"howard\")</p> </li> <li><p>The layers we want to fine-tune.</p> </li> </ol> <p>For example</p> In\u00a0[16]: Copied! <pre>wide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1)\ntab_resnet = TabResnet(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    cat_embed_dropout=0.1,\n    continuous_cols=continuous_cols,\n    blocks_dims=[200, 200, 200],\n)\nmodel = WideDeep(wide=wide, deeptabular=tab_resnet)\n</pre> wide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1) tab_resnet = TabResnet(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     cat_embed_dropout=0.1,     continuous_cols=continuous_cols,     blocks_dims=[200, 200, 200], ) model = WideDeep(wide=wide, deeptabular=tab_resnet) In\u00a0[17]: Copied! <pre>model\n</pre> model Out[17]: <pre>WideDeep(\n  (wide): Wide(\n    (wide_linear): Embedding(809, 1, padding_idx=0)\n  )\n  (deeptabular): Sequential(\n    (0): TabResnet(\n      (cat_embed): DiffSizeCatEmbeddings(\n        (embed_layers): ModuleDict(\n          (emb_layer_workclass): Embedding(10, 5, padding_idx=0)\n          (emb_layer_education): Embedding(17, 8, padding_idx=0)\n          (emb_layer_marital_status): Embedding(8, 5, padding_idx=0)\n          (emb_layer_occupation): Embedding(16, 7, padding_idx=0)\n          (emb_layer_relationship): Embedding(7, 4, padding_idx=0)\n          (emb_layer_race): Embedding(6, 4, padding_idx=0)\n          (emb_layer_gender): Embedding(3, 2, padding_idx=0)\n          (emb_layer_capital_gain): Embedding(124, 24, padding_idx=0)\n          (emb_layer_capital_loss): Embedding(100, 21, padding_idx=0)\n          (emb_layer_native_country): Embedding(43, 13, padding_idx=0)\n        )\n        (embedding_dropout): Dropout(p=0.1, inplace=False)\n      )\n      (cont_norm): Identity()\n      (encoder): DenseResnet(\n        (dense_resnet): Sequential(\n          (lin_inp): Linear(in_features=95, out_features=200, bias=False)\n          (bn_inp): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n          (block_0): BasicBlock(\n            (lin1): Linear(in_features=200, out_features=200, bias=False)\n            (bn1): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n            (leaky_relu): LeakyReLU(negative_slope=0.01, inplace=True)\n            (dp): Dropout(p=0.1, inplace=False)\n            (lin2): Linear(in_features=200, out_features=200, bias=False)\n            (bn2): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n          )\n          (block_1): BasicBlock(\n            (lin1): Linear(in_features=200, out_features=200, bias=False)\n            (bn1): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n            (leaky_relu): LeakyReLU(negative_slope=0.01, inplace=True)\n            (dp): Dropout(p=0.1, inplace=False)\n            (lin2): Linear(in_features=200, out_features=200, bias=False)\n            (bn2): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=200, out_features=1, bias=True)\n  )\n)</pre> <p>let's first train as usual</p> In\u00a0[18]: Copied! <pre>trainer_3 = Trainer(model, objective=\"binary\", metrics=[Accuracy])\n</pre> trainer_3 = Trainer(model, objective=\"binary\", metrics=[Accuracy]) In\u00a0[19]: Copied! <pre>trainer_3.fit(\n    X_wide=X_wide, X_tab=X_tab, target=target, val_split=0.1, n_epochs=2, batch_size=256\n)\n</pre> trainer_3.fit(     X_wide=X_wide, X_tab=X_tab, target=target, val_split=0.1, n_epochs=2, batch_size=256 ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:03&lt;00:00, 54.23it/s, loss=0.382, metrics={'acc': 0.8239}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 20/20 [00:00&lt;00:00, 84.72it/s, loss=0.331, metrics={'acc': 0.8526}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:03&lt;00:00, 54.35it/s, loss=0.33, metrics={'acc': 0.8465}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 20/20 [00:00&lt;00:00, 68.15it/s, loss=0.312, metrics={'acc': 0.8604}]\n</pre> In\u00a0[20]: Copied! <pre>trainer_3.save(path=\"models_dir\", save_state_dict=True, model_filename=\"model_3.pt\")\n</pre> trainer_3.save(path=\"models_dir\", save_state_dict=True, model_filename=\"model_3.pt\") <p>Now we are going to fine-tune the model components, and in the case of the <code>deeptabular</code> component, we will fine-tune the resnet-blocks and the linear layer but NOT the embeddings.</p> <p>For this, we need to access the model component's children: <code>deeptabular</code> $\\rightarrow$ <code>tab_resnet</code> $\\rightarrow$ <code>dense_resnet</code> $\\rightarrow$ <code>blocks</code></p> In\u00a0[21]: Copied! <pre>wide_3 = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1)\ntab_resnet_3 = TabResnet(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    cat_embed_dropout=0.1,\n    continuous_cols=continuous_cols,\n    blocks_dims=[200, 200, 200],\n)\nmodel_3 = WideDeep(wide=wide, deeptabular=tab_resnet)\n</pre> wide_3 = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1) tab_resnet_3 = TabResnet(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     cat_embed_dropout=0.1,     continuous_cols=continuous_cols,     blocks_dims=[200, 200, 200], ) model_3 = WideDeep(wide=wide, deeptabular=tab_resnet) In\u00a0[22]: Copied! <pre>model_3.load_state_dict(torch.load(\"models_dir/model_3.pt\"))\n</pre> model_3.load_state_dict(torch.load(\"models_dir/model_3.pt\")) Out[22]: <pre>&lt;All keys matched successfully&gt;</pre> In\u00a0[23]: Copied! <pre>model_3\n</pre> model_3 Out[23]: <pre>WideDeep(\n  (wide): Wide(\n    (wide_linear): Embedding(809, 1, padding_idx=0)\n  )\n  (deeptabular): Sequential(\n    (0): TabResnet(\n      (cat_embed): DiffSizeCatEmbeddings(\n        (embed_layers): ModuleDict(\n          (emb_layer_workclass): Embedding(10, 5, padding_idx=0)\n          (emb_layer_education): Embedding(17, 8, padding_idx=0)\n          (emb_layer_marital_status): Embedding(8, 5, padding_idx=0)\n          (emb_layer_occupation): Embedding(16, 7, padding_idx=0)\n          (emb_layer_relationship): Embedding(7, 4, padding_idx=0)\n          (emb_layer_race): Embedding(6, 4, padding_idx=0)\n          (emb_layer_gender): Embedding(3, 2, padding_idx=0)\n          (emb_layer_capital_gain): Embedding(124, 24, padding_idx=0)\n          (emb_layer_capital_loss): Embedding(100, 21, padding_idx=0)\n          (emb_layer_native_country): Embedding(43, 13, padding_idx=0)\n        )\n        (embedding_dropout): Dropout(p=0.1, inplace=False)\n      )\n      (cont_norm): Identity()\n      (encoder): DenseResnet(\n        (dense_resnet): Sequential(\n          (lin_inp): Linear(in_features=95, out_features=200, bias=False)\n          (bn_inp): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n          (block_0): BasicBlock(\n            (lin1): Linear(in_features=200, out_features=200, bias=False)\n            (bn1): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n            (leaky_relu): LeakyReLU(negative_slope=0.01, inplace=True)\n            (dp): Dropout(p=0.1, inplace=False)\n            (lin2): Linear(in_features=200, out_features=200, bias=False)\n            (bn2): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n          )\n          (block_1): BasicBlock(\n            (lin1): Linear(in_features=200, out_features=200, bias=False)\n            (bn1): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n            (leaky_relu): LeakyReLU(negative_slope=0.01, inplace=True)\n            (dp): Dropout(p=0.1, inplace=False)\n            (lin2): Linear(in_features=200, out_features=200, bias=False)\n            (bn2): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=200, out_features=1, bias=True)\n  )\n)</pre> In\u00a0[24]: Copied! <pre>tab_lin_layer = list(model_3.deeptabular.children())[1]\n</pre> tab_lin_layer = list(model_3.deeptabular.children())[1] In\u00a0[25]: Copied! <pre>tab_lin_layer\n</pre> tab_lin_layer Out[25]: <pre>Linear(in_features=200, out_features=1, bias=True)</pre> In\u00a0[26]: Copied! <pre>tab_deep_layers = []\nfor n1, c1 in model_3.deeptabular.named_children():\n    if (\n        n1 == \"0\"\n    ):  # 0 is the model component and 1 is always the prediction layer added by the `WideDeep` class\n        for n2, c2 in c1.named_children():\n            if n2 == \"encoder\":  # TabResnet\n                for _, c3 in c2.named_children():\n                    for n4, c4 in c3.named_children():  # dense_resnet\n                        if \"block\" in n4:\n                            tab_deep_layers.append((n4, c4))\n</pre> tab_deep_layers = [] for n1, c1 in model_3.deeptabular.named_children():     if (         n1 == \"0\"     ):  # 0 is the model component and 1 is always the prediction layer added by the `WideDeep` class         for n2, c2 in c1.named_children():             if n2 == \"encoder\":  # TabResnet                 for _, c3 in c2.named_children():                     for n4, c4 in c3.named_children():  # dense_resnet                         if \"block\" in n4:                             tab_deep_layers.append((n4, c4)) In\u00a0[27]: Copied! <pre>tab_deep_layers\n</pre> tab_deep_layers Out[27]: <pre>[('block_0',\n  BasicBlock(\n    (lin1): Linear(in_features=200, out_features=200, bias=False)\n    (bn1): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n    (leaky_relu): LeakyReLU(negative_slope=0.01, inplace=True)\n    (dp): Dropout(p=0.1, inplace=False)\n    (lin2): Linear(in_features=200, out_features=200, bias=False)\n    (bn2): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n  )),\n ('block_1',\n  BasicBlock(\n    (lin1): Linear(in_features=200, out_features=200, bias=False)\n    (bn1): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n    (leaky_relu): LeakyReLU(negative_slope=0.01, inplace=True)\n    (dp): Dropout(p=0.1, inplace=False)\n    (lin2): Linear(in_features=200, out_features=200, bias=False)\n    (bn2): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n  ))]</pre> <p>Now remember, we need to pass ONLY LAYERS (before I included the name for clarity) the layers in WARM UP ORDER, therefore:</p> In\u00a0[28]: Copied! <pre>tab_deep_layers = [el[1] for el in tab_deep_layers][::-1]\n</pre> tab_deep_layers = [el[1] for el in tab_deep_layers][::-1] In\u00a0[29]: Copied! <pre>tab_layers = [tab_lin_layer] + tab_deep_layers[::-1]\n</pre> tab_layers = [tab_lin_layer] + tab_deep_layers[::-1] In\u00a0[30]: Copied! <pre>tab_layers\n</pre> tab_layers Out[30]: <pre>[Linear(in_features=200, out_features=1, bias=True),\n BasicBlock(\n   (lin1): Linear(in_features=200, out_features=200, bias=False)\n   (bn1): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n   (leaky_relu): LeakyReLU(negative_slope=0.01, inplace=True)\n   (dp): Dropout(p=0.1, inplace=False)\n   (lin2): Linear(in_features=200, out_features=200, bias=False)\n   (bn2): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n ),\n BasicBlock(\n   (lin1): Linear(in_features=200, out_features=200, bias=False)\n   (bn1): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n   (leaky_relu): LeakyReLU(negative_slope=0.01, inplace=True)\n   (dp): Dropout(p=0.1, inplace=False)\n   (lin2): Linear(in_features=200, out_features=200, bias=False)\n   (bn2): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n )]</pre> <p>And now simply</p> In\u00a0[31]: Copied! <pre>trainer_4 = Trainer(model_3, objective=\"binary\", metrics=[Accuracy])\n</pre> trainer_4 = Trainer(model_3, objective=\"binary\", metrics=[Accuracy]) In\u00a0[32]: Copied! <pre>trainer_4.fit(\n    X_wide=X_wide,\n    X_tab=X_tab,\n    target=target,\n    val_split=0.1,\n    finetune=True,\n    finetune_epochs=2,\n    deeptabular_gradual=True,\n    deeptabular_layers=tab_layers,\n    deeptabular_max_lr=0.01,\n    n_epochs=2,\n    batch_size=256,\n)\n</pre> trainer_4.fit(     X_wide=X_wide,     X_tab=X_tab,     target=target,     val_split=0.1,     finetune=True,     finetune_epochs=2,     deeptabular_gradual=True,     deeptabular_layers=tab_layers,     deeptabular_max_lr=0.01,     n_epochs=2,     batch_size=256, ) <pre>Training wide for 2 epochs\n</pre> <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:01&lt;00:00, 95.17it/s, loss=0.504, metrics={'acc': 0.7523}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:01&lt;00:00, 99.83it/s, loss=0.384, metrics={'acc': 0.789}]\n</pre> <pre>Training deeptabular, layer 1 of 3\n</pre> <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:02&lt;00:00, 72.31it/s, loss=0.317, metrics={'acc': 0.8098}]\n</pre> <pre>Training deeptabular, layer 2 of 3\n</pre> <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:02&lt;00:00, 65.97it/s, loss=0.312, metrics={'acc': 0.8214}]\n</pre> <pre>Training deeptabular, layer 3 of 3\n</pre> <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:02&lt;00:00, 63.92it/s, loss=0.306, metrics={'acc': 0.8284}]\n</pre> <pre>Fine-tuning (or warmup) of individual components completed. Training the whole model for 2 epochs\n</pre> <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:03&lt;00:00, 57.26it/s, loss=0.292, metrics={'acc': 0.8664}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 20/20 [00:00&lt;00:00, 84.56it/s, loss=0.292, metrics={'acc': 0.8696}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:03&lt;00:00, 53.61it/s, loss=0.282, metrics={'acc': 0.8693}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 20/20 [00:00&lt;00:00, 80.59it/s, loss=0.289, metrics={'acc': 0.8719}]\n</pre> <p>Finally, there is one more use case I would like to consider. The case where we train only one component and we just want to fine-tune and stop the training afterwards, since there is no joined training. This is a simple as</p> In\u00a0[33]: Copied! <pre>tab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    cat_embed_dropout=0.1,\n    continuous_cols=continuous_cols,\n    mlp_hidden_dims=[64, 32],\n    mlp_dropout=0.2,\n    mlp_activation=\"leaky_relu\",\n)\nmodel = WideDeep(deeptabular=tab_mlp)\n</pre> tab_mlp = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     cat_embed_dropout=0.1,     continuous_cols=continuous_cols,     mlp_hidden_dims=[64, 32],     mlp_dropout=0.2,     mlp_activation=\"leaky_relu\", ) model = WideDeep(deeptabular=tab_mlp) In\u00a0[34]: Copied! <pre>trainer_5 = Trainer(\n    model,\n    objective=\"binary\",\n    optimizers=torch.optim.Adam(model.parameters(), lr=0.01),\n    metrics=[Accuracy],\n)\n</pre> trainer_5 = Trainer(     model,     objective=\"binary\",     optimizers=torch.optim.Adam(model.parameters(), lr=0.01),     metrics=[Accuracy], ) In\u00a0[35]: Copied! <pre>trainer_5.fit(\n    X_wide=X_wide, X_tab=X_tab, target=target, val_split=0.1, n_epochs=1, batch_size=256\n)\n</pre> trainer_5.fit(     X_wide=X_wide, X_tab=X_tab, target=target, val_split=0.1, n_epochs=1, batch_size=256 ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:02&lt;00:00, 73.69it/s, loss=0.365, metrics={'acc': 0.8331}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 20/20 [00:00&lt;00:00, 92.56it/s, loss=0.299, metrics={'acc': 0.8673}]\n</pre> In\u00a0[36]: Copied! <pre>trainer_5.save(path=\"models_dir\", save_state_dict=True, model_filename=\"model_5.pt\")\n</pre> trainer_5.save(path=\"models_dir\", save_state_dict=True, model_filename=\"model_5.pt\") In\u00a0[37]: Copied! <pre>tab_mlp_5 = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    cat_embed_dropout=0.1,\n    continuous_cols=continuous_cols,\n    mlp_hidden_dims=[64, 32],\n    mlp_dropout=0.2,\n    mlp_activation=\"leaky_relu\",\n)\nmodel_5 = WideDeep(deeptabular=tab_mlp_5)\n</pre> tab_mlp_5 = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     cat_embed_dropout=0.1,     continuous_cols=continuous_cols,     mlp_hidden_dims=[64, 32],     mlp_dropout=0.2,     mlp_activation=\"leaky_relu\", ) model_5 = WideDeep(deeptabular=tab_mlp_5) In\u00a0[38]: Copied! <pre>model_5.load_state_dict(torch.load(\"models_dir/model_5.pt\"))\n</pre> model_5.load_state_dict(torch.load(\"models_dir/model_5.pt\")) Out[38]: <pre>&lt;All keys matched successfully&gt;</pre> <p>...times go by...</p> In\u00a0[39]: Copied! <pre>trainer_6 = Trainer(\n    model_5,\n    objective=\"binary\",\n    optimizers=torch.optim.Adam(model.parameters(), lr=0.01),\n    metrics=[Accuracy],\n)\n</pre> trainer_6 = Trainer(     model_5,     objective=\"binary\",     optimizers=torch.optim.Adam(model.parameters(), lr=0.01),     metrics=[Accuracy], ) In\u00a0[40]: Copied! <pre>trainer_6.fit(\n    X_wide=X_wide,\n    X_tab=X_tab,\n    target=target,\n    val_split=0.1,\n    finetune=True,\n    finetune_epochs=2,\n    finetune_max_lr=0.01,\n    stop_after_finetuning=True,\n    batch_size=256,\n)\n</pre> trainer_6.fit(     X_wide=X_wide,     X_tab=X_tab,     target=target,     val_split=0.1,     finetune=True,     finetune_epochs=2,     finetune_max_lr=0.01,     stop_after_finetuning=True,     batch_size=256, ) <pre>Training deeptabular for 2 epochs\n</pre> <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:02&lt;00:00, 73.86it/s, loss=0.298, metrics={'acc': 0.8652}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:02&lt;00:00, 75.45it/s, loss=0.286, metrics={'acc': 0.8669}]\n</pre> <pre>Fine-tuning (or warmup) of individual components completed. Training the whole model for 1 epochs\n</pre> <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:02&lt;00:00, 76.29it/s, loss=0.282, metrics={'acc': 0.8698}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 20/20 [00:00&lt;00:00, 84.93it/s, loss=0.281, metrics={'acc': 0.8749}]\n</pre> In\u00a0[42]: Copied! <pre>import shutil\n\nshutil.rmtree(\"models_dir/\")\nshutil.rmtree(\"model_weights/\")\n</pre> import shutil  shutil.rmtree(\"models_dir/\") shutil.rmtree(\"model_weights/\") In\u00a0[\u00a0]: Copied! <pre>\n</pre>"},{"location":"examples/06_finetune_and_warmup.html#the-finetunewarm-up-option","title":"The FineTune/Warm Up option\u00b6","text":"<p>Let's place ourselves in two possible scenarios.</p> <ol> <li><p>Let's assume we have run a model and we want to just transfer the learnings (you know...transfer-learning) to another dataset, or simply we have received new data and we do not want to start the training of each component from scratch. Simply, we want to load the pre-trained weights and fine-tune.</p> </li> <li><p>We just want to \"warm up\" individual model components individually before the joined training begins.</p> </li> </ol> <p>This can be done with the <code>finetune</code> set of parameters. There are 3 fine-tuning routines:</p> <ol> <li>Fine-tune all trainable layers at once with a triangular one-cycle learning rate (referred as slanted triangular learning rates in Howard &amp; Ruder 2018)</li> <li>Gradual fine-tuning inspired by the work of Felbo et al., 2017</li> <li>Gradual fine-tuning based on the work of Howard &amp; Ruder 2018</li> </ol> <p>Currently fine-tunning is only supported without a fully connected head, i.e. if <code>deephead=None</code>. In addition, <code>Felbo</code> and <code>Howard</code> routines only applied, of course, to the <code>deeptabular</code>, <code>deeptext</code> and <code>deepimage</code> models. The <code>wide</code> component can also be fine-tuned, but only in an \"all at once\" mode.</p>"},{"location":"examples/06_finetune_and_warmup.html#fine-tune-or-warm-up-all-at-once","title":"Fine-tune or warm-up all at once\u00b6","text":"<p>Here, the model components will be trained for <code>finetune_epochs</code> using a triangular one-cycle learning rate (slanted triangular learning rate) ranging from <code>finetune_max_lr/10</code> to <code>finetune_max_lr</code> (default is 0.01). 10% of the training steps are used to increase the learning rate which then decreases for the remaining 90%.</p> <p>Here all trainable layers are fine-tuned.</p> <p>Let's have a look to one example.</p>"},{"location":"examples/06_finetune_and_warmup.html#fine-tune-gradually-the-felbo-and-the-howard-routines","title":"Fine-tune Gradually: The \"felbo\"  and the \"howard\" routines\u00b6","text":"<p>The Felbo routine can be illustrated as follows:</p> <p> </p> <p>Figure 1. The figure can be described as follows: fine-tune (or train) the last layer for one epoch using a one cycle triangular learning rate. Then fine-tune the next deeper layer for one epoch, with a learning rate that is a factor of 2.5 lower than the previous learning rate (the 2.5 factor is fixed) while freezing the already warmed up layer(s). Repeat untill all individual layers are warmed. Then warm one last epoch with all warmed layers trainable. The vanishing color gradient in the figure attempts to illustrate the decreasing learning rate.</p> <p>Note that this is not identical to the Fine-Tunning routine described in Felbo et al, 2017, this is why I used the word 'inspired'.</p> <p>The Howard routine can be illustrated as follows:</p> <p> </p> <p>Figure 2. The figure can be described as follows: fine-tune (or train) the last layer for one epoch using a one cycle triangular learning rate. Then fine-tune the next deeper layer for one epoch, with a learning rate that is a factor of 2.5 lower than the previous learning rate (the 2.5 factor is fixed) while keeping the already warmed up layer(s) trainable. Repeat. The vanishing color gradient in the figure attempts to illustrate the decreasing learning rate.</p> <p>Note that I write \"fine-tune (or train) the last layer for one epoch [...]\". However, in practice the user will have to specify the order of the layers to be fine-tuned. This is another reason why I wrote that the fine-tune routines I have implemented are inspired by the work of Felbo and Howard and not identical to their implemenations.</p> <p>The <code>felbo</code> and <code>howard</code> routines can be accessed with via the <code>fine-tune</code> parameters.</p>"},{"location":"examples/07_custom_components.html","title":"07_custom_components","text":"In\u00a0[1]: Copied! <pre>import numpy as np\nimport pandas as pd\nimport os\nimport torch\n\nfrom torch import Tensor\nfrom pytorch_widedeep import Trainer\nfrom pytorch_widedeep.preprocessing import (\n    WidePreprocessor,\n    TabPreprocessor,\n    TextPreprocessor,\n    ImagePreprocessor,\n)\nfrom pytorch_widedeep.models import (\n    Wide,\n    TabMlp,\n    Vision,\n    BasicRNN,\n    WideDeep,\n)\nfrom pytorch_widedeep.losses import RMSELoss\nfrom pytorch_widedeep.initializers import *\nfrom pytorch_widedeep.callbacks import *\nfrom pytorch_widedeep.datasets import load_adult\n</pre> import numpy as np import pandas as pd import os import torch  from torch import Tensor from pytorch_widedeep import Trainer from pytorch_widedeep.preprocessing import (     WidePreprocessor,     TabPreprocessor,     TextPreprocessor,     ImagePreprocessor, ) from pytorch_widedeep.models import (     Wide,     TabMlp,     Vision,     BasicRNN,     WideDeep, ) from pytorch_widedeep.losses import RMSELoss from pytorch_widedeep.initializers import * from pytorch_widedeep.callbacks import * from pytorch_widedeep.datasets import load_adult <pre>/Users/javierrodriguezzaurin/.pyenv/versions/3.10.13/envs/widedeep310/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n  from .autonotebook import tqdm as notebook_tqdm\n</pre> In\u00a0[2]: Copied! <pre>df = pd.read_csv(\"../tmp_data/airbnb/airbnb_sample.csv\")\ndf.head()\n</pre> df = pd.read_csv(\"../tmp_data/airbnb/airbnb_sample.csv\") df.head() Out[2]: id host_id description host_listings_count host_identity_verified neighbourhood_cleansed latitude longitude is_location_exact property_type ... amenity_wide_entrance amenity_wide_entrance_for_guests amenity_wide_entryway amenity_wide_hallways amenity_wifi amenity_window_guards amenity_wine_cooler security_deposit extra_people yield 0 13913.jpg 54730 My bright double bedroom with a large window h... 4.0 f Islington 51.56802 -0.11121 t apartment ... 1 0 0 0 1 0 0 100.0 15.0 12.00 1 15400.jpg 60302 Lots of windows and light.  St Luke's Gardens ... 1.0 t Kensington and Chelsea 51.48796 -0.16898 t apartment ... 0 0 0 0 1 0 0 150.0 0.0 109.50 2 17402.jpg 67564 Open from June 2018 after a 3-year break, we a... 19.0 t Westminster 51.52098 -0.14002 t apartment ... 0 0 0 0 1 0 0 350.0 10.0 149.65 3 24328.jpg 41759 Artist house, bright high ceiling rooms, priva... 2.0 t Wandsworth 51.47298 -0.16376 t other ... 0 0 0 0 1 0 0 250.0 0.0 215.60 4 25023.jpg 102813 Large, all comforts, 2-bed flat; first floor; ... 1.0 f Wandsworth 51.44687 -0.21874 t apartment ... 0 0 0 0 1 0 0 250.0 11.0 79.35 <p>5 rows \u00d7 223 columns</p> In\u00a0[3]: Copied! <pre># There are a number of columns that are already binary. Therefore, no need to one hot encode them\ncrossed_cols = [(\"property_type\", \"room_type\")]\nalready_dummies = [c for c in df.columns if \"amenity\" in c] + [\"has_house_rules\"]\nwide_cols = [\n    \"is_location_exact\",\n    \"property_type\",\n    \"room_type\",\n    \"host_gender\",\n    \"instant_bookable\",\n] + already_dummies\n\ncat_embed_cols = [(c, 16) for c in df.columns if \"catg\" in c] + [\n    (\"neighbourhood_cleansed\", 64),\n    (\"cancellation_policy\", 16),\n]\ncontinuous_cols = [\"latitude\", \"longitude\", \"security_deposit\", \"extra_people\"]\n# it does not make sense to standarised Latitude and Longitude\nalready_standard = [\"latitude\", \"longitude\"]\n\n# text and image colnames\ntext_col = \"description\"\nimg_col = \"id\"\n\n# path to pretrained word embeddings and the images\nword_vectors_path = \"../tmp_data/glove.6B/glove.6B.100d.txt\"\nimg_path = \"../tmp_data/airbnb/property_picture\"\n\n# target\ntarget_col = \"yield\"\n</pre> # There are a number of columns that are already binary. Therefore, no need to one hot encode them crossed_cols = [(\"property_type\", \"room_type\")] already_dummies = [c for c in df.columns if \"amenity\" in c] + [\"has_house_rules\"] wide_cols = [     \"is_location_exact\",     \"property_type\",     \"room_type\",     \"host_gender\",     \"instant_bookable\", ] + already_dummies  cat_embed_cols = [(c, 16) for c in df.columns if \"catg\" in c] + [     (\"neighbourhood_cleansed\", 64),     (\"cancellation_policy\", 16), ] continuous_cols = [\"latitude\", \"longitude\", \"security_deposit\", \"extra_people\"] # it does not make sense to standarised Latitude and Longitude already_standard = [\"latitude\", \"longitude\"]  # text and image colnames text_col = \"description\" img_col = \"id\"  # path to pretrained word embeddings and the images word_vectors_path = \"../tmp_data/glove.6B/glove.6B.100d.txt\" img_path = \"../tmp_data/airbnb/property_picture\"  # target target_col = \"yield\" In\u00a0[4]: Copied! <pre>target = df[target_col].values\n</pre> target = df[target_col].values In\u00a0[5]: Copied! <pre>wide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols)\nX_wide = wide_preprocessor.fit_transform(df)\n\ntab_preprocessor = TabPreprocessor(\n    embed_cols=cat_embed_cols, continuous_cols=continuous_cols\n)\nX_tab = tab_preprocessor.fit_transform(df)\n\ntext_preprocessor = TextPreprocessor(\n    word_vectors_path=word_vectors_path, text_col=text_col\n)\nX_text = text_preprocessor.fit_transform(df)\n\nimage_processor = ImagePreprocessor(img_col=img_col, img_path=img_path)\nX_images = image_processor.fit_transform(df)\n</pre> wide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols) X_wide = wide_preprocessor.fit_transform(df)  tab_preprocessor = TabPreprocessor(     embed_cols=cat_embed_cols, continuous_cols=continuous_cols ) X_tab = tab_preprocessor.fit_transform(df)  text_preprocessor = TextPreprocessor(     word_vectors_path=word_vectors_path, text_col=text_col ) X_text = text_preprocessor.fit_transform(df)  image_processor = ImagePreprocessor(img_col=img_col, img_path=img_path) X_images = image_processor.fit_transform(df) <pre>/Users/javierrodriguezzaurin/Projects/pytorch-widedeep/pytorch_widedeep/preprocessing/tab_preprocessor.py:358: UserWarning: Continuous columns will not be normalised\n  warnings.warn(\"Continuous columns will not be normalised\")\n</pre> <pre>The vocabulary contains 2192 tokens\nIndexing word vectors...\nLoaded 400000 word vectors\nPreparing embeddings matrix...\n2175 words in the vocabulary had ../tmp_data/glove.6B/glove.6B.100d.txt vectors and appear more than 5 times\nReading Images from ../tmp_data/airbnb/property_picture\nResizing\n</pre> <pre>100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 1001/1001 [00:02&lt;00:00, 497.80it/s]\n</pre> <pre>Computing normalisation metrics\n</pre> <p>Now we are ready to build a wide and deep model. Three of the four components we will use are included in this package, and they will be combined with a custom <code>deeptext</code> component. Then the fit process will run with a custom loss function.</p> <p>Let's have a look</p> In\u00a0[6]: Copied! <pre># Linear model\nwide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1)\n\n# DeepDense: 2 Dense layers\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    cat_embed_dropout=0.1,\n    continuous_cols=continuous_cols,\n    mlp_hidden_dims=[128, 64],\n    mlp_dropout=0.1,\n)\n\n# Pretrained Resnet 18\nresnet = Vision(pretrained_model_name=\"resnet18\", n_trainable=0)\n</pre> # Linear model wide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1)  # DeepDense: 2 Dense layers tab_mlp = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     cat_embed_dropout=0.1,     continuous_cols=continuous_cols,     mlp_hidden_dims=[128, 64],     mlp_dropout=0.1, )  # Pretrained Resnet 18 resnet = Vision(pretrained_model_name=\"resnet18\", n_trainable=0) In\u00a0[7]: Copied! <pre>class MyDeepText(nn.Module):\n    def __init__(self, vocab_size, padding_idx=1, embed_dim=100, hidden_dim=64):\n        super(MyDeepText, self).__init__()\n\n        # word/token embeddings\n        self.word_embed = nn.Embedding(vocab_size, embed_dim, padding_idx=padding_idx)\n\n        # stack of RNNs\n        self.rnn = nn.GRU(\n            embed_dim,\n            hidden_dim,\n            num_layers=2,\n            bidirectional=True,\n            batch_first=True,\n        )\n\n        # Remember, this MUST be defined. If not WideDeep will through an error\n        self.output_dim = hidden_dim * 2\n\n    def forward(self, X):\n        embed = self.word_embed(X.long())\n        o, h = self.rnn(embed)\n        return torch.cat((h[-2], h[-1]), dim=1)\n</pre> class MyDeepText(nn.Module):     def __init__(self, vocab_size, padding_idx=1, embed_dim=100, hidden_dim=64):         super(MyDeepText, self).__init__()          # word/token embeddings         self.word_embed = nn.Embedding(vocab_size, embed_dim, padding_idx=padding_idx)          # stack of RNNs         self.rnn = nn.GRU(             embed_dim,             hidden_dim,             num_layers=2,             bidirectional=True,             batch_first=True,         )          # Remember, this MUST be defined. If not WideDeep will through an error         self.output_dim = hidden_dim * 2      def forward(self, X):         embed = self.word_embed(X.long())         o, h = self.rnn(embed)         return torch.cat((h[-2], h[-1]), dim=1) In\u00a0[8]: Copied! <pre>mydeeptext = MyDeepText(vocab_size=len(text_preprocessor.vocab.itos))\n</pre> mydeeptext = MyDeepText(vocab_size=len(text_preprocessor.vocab.itos)) In\u00a0[9]: Copied! <pre>model = WideDeep(wide=wide, deeptabular=tab_mlp, deeptext=mydeeptext, deepimage=resnet)\n</pre> model = WideDeep(wide=wide, deeptabular=tab_mlp, deeptext=mydeeptext, deepimage=resnet) In\u00a0[10]: Copied! <pre>class RMSELoss(nn.Module):\n    def __init__(self):\n        \"\"\"root mean squared error\"\"\"\n        super().__init__()\n        self.mse = nn.MSELoss()\n\n    def forward(self, input: Tensor, target: Tensor) -&gt; Tensor:\n        return torch.sqrt(self.mse(input, target))\n</pre> class RMSELoss(nn.Module):     def __init__(self):         \"\"\"root mean squared error\"\"\"         super().__init__()         self.mse = nn.MSELoss()      def forward(self, input: Tensor, target: Tensor) -&gt; Tensor:         return torch.sqrt(self.mse(input, target)) <p>and now we just instantiate the <code>Trainer</code> as usual. Needless to say, but this runs with 1000 random observations, so loss and metric values are meaningless. This is just an example</p> In\u00a0[11]: Copied! <pre>trainer = Trainer(model, objective=\"regression\", custom_loss_function=RMSELoss())\n</pre> trainer = Trainer(model, objective=\"regression\", custom_loss_function=RMSELoss()) In\u00a0[12]: Copied! <pre>trainer.fit(\n    X_wide=X_wide,\n    X_tab=X_tab,\n    X_text=X_text,\n    X_img=X_images,\n    target=target,\n    n_epochs=1,\n    batch_size=32,\n    val_split=0.2,\n)\n</pre> trainer.fit(     X_wide=X_wide,     X_tab=X_tab,     X_text=X_text,     X_img=X_images,     target=target,     n_epochs=1,     batch_size=32,     val_split=0.2, ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 25/25 [00:23&lt;00:00,  1.07it/s, loss=126]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 7/7 [00:05&lt;00:00,  1.24it/s, loss=97.4]\n</pre> <p>In addition to model components and loss functions, we can also use custom callbacks or custom metrics. The former need to be of type <code>Callback</code> and the latter need to be of type <code>Metric</code>. See:</p> <pre>pytorch-widedeep.callbacks\n</pre> <p>and</p> <pre>pytorch-widedeep.metrics\n</pre> <p>For this example let me use the adult dataset. Again, we first prepare the data as usual</p> In\u00a0[13]: Copied! <pre>df = load_adult(as_frame=True)\ndf.head()\n</pre> df = load_adult(as_frame=True) df.head() Out[13]: age workclass fnlwgt education educational-num marital-status occupation relationship race gender capital-gain capital-loss hours-per-week native-country income 0 25 Private 226802 11th 7 Never-married Machine-op-inspct Own-child Black Male 0 0 40 United-States &lt;=50K 1 38 Private 89814 HS-grad 9 Married-civ-spouse Farming-fishing Husband White Male 0 0 50 United-States &lt;=50K 2 28 Local-gov 336951 Assoc-acdm 12 Married-civ-spouse Protective-serv Husband White Male 0 0 40 United-States &gt;50K 3 44 Private 160323 Some-college 10 Married-civ-spouse Machine-op-inspct Husband Black Male 7688 0 40 United-States &gt;50K 4 18 ? 103497 Some-college 10 Never-married ? Own-child White Female 0 0 30 United-States &lt;=50K In\u00a0[14]: Copied! <pre># For convenience, we'll replace '-' with '_'\ndf.columns = [c.replace(\"-\", \"_\") for c in df.columns]\n# binary target\ndf[\"income_label\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int)\ndf.drop(\"income\", axis=1, inplace=True)\ndf.head()\n</pre> # For convenience, we'll replace '-' with '_' df.columns = [c.replace(\"-\", \"_\") for c in df.columns] # binary target df[\"income_label\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int) df.drop(\"income\", axis=1, inplace=True) df.head() Out[14]: age workclass fnlwgt education educational_num marital_status occupation relationship race gender capital_gain capital_loss hours_per_week native_country income_label 0 25 Private 226802 11th 7 Never-married Machine-op-inspct Own-child Black Male 0 0 40 United-States 0 1 38 Private 89814 HS-grad 9 Married-civ-spouse Farming-fishing Husband White Male 0 0 50 United-States 0 2 28 Local-gov 336951 Assoc-acdm 12 Married-civ-spouse Protective-serv Husband White Male 0 0 40 United-States 1 3 44 Private 160323 Some-college 10 Married-civ-spouse Machine-op-inspct Husband Black Male 7688 0 40 United-States 1 4 18 ? 103497 Some-college 10 Never-married ? Own-child White Female 0 0 30 United-States 0 In\u00a0[15]: Copied! <pre># Define wide, crossed and deep tabular columns\nwide_cols = [\n    \"workclass\",\n    \"education\",\n    \"marital_status\",\n    \"occupation\",\n    \"relationship\",\n    \"race\",\n    \"gender\",\n    \"native_country\",\n]\ncrossed_cols = [(\"education\", \"occupation\"), (\"native_country\", \"occupation\")]\ncat_embed_cols = [\n    \"workclass\",\n    \"education\",\n    \"marital_status\",\n    \"occupation\",\n    \"relationship\",\n    \"race\",\n    \"gender\",\n    \"capital_gain\",\n    \"capital_loss\",\n    \"native_country\",\n]\ncontinuous_cols = [\"age\", \"hours_per_week\"]\ntarget_col = \"income_label\"\ntarget = df[target_col].values\n</pre> # Define wide, crossed and deep tabular columns wide_cols = [     \"workclass\",     \"education\",     \"marital_status\",     \"occupation\",     \"relationship\",     \"race\",     \"gender\",     \"native_country\", ] crossed_cols = [(\"education\", \"occupation\"), (\"native_country\", \"occupation\")] cat_embed_cols = [     \"workclass\",     \"education\",     \"marital_status\",     \"occupation\",     \"relationship\",     \"race\",     \"gender\",     \"capital_gain\",     \"capital_loss\",     \"native_country\", ] continuous_cols = [\"age\", \"hours_per_week\"] target_col = \"income_label\" target = df[target_col].values In\u00a0[16]: Copied! <pre># wide\nwide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols)\nX_wide = wide_preprocessor.fit_transform(df)\n\n# deeptabular\ntab_preprocessor = TabPreprocessor(\n    embed_cols=cat_embed_cols, continuous_cols=continuous_cols\n)\nX_tab = tab_preprocessor.fit_transform(df)\n</pre> # wide wide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols) X_wide = wide_preprocessor.fit_transform(df)  # deeptabular tab_preprocessor = TabPreprocessor(     embed_cols=cat_embed_cols, continuous_cols=continuous_cols ) X_tab = tab_preprocessor.fit_transform(df) <pre>/Users/javierrodriguezzaurin/Projects/pytorch-widedeep/pytorch_widedeep/preprocessing/tab_preprocessor.py:358: UserWarning: Continuous columns will not be normalised\n  warnings.warn(\"Continuous columns will not be normalised\")\n</pre> In\u00a0[17]: Copied! <pre>wide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1)\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=continuous_cols,\n    mlp_hidden_dims=[128, 64],\n    mlp_dropout=0.2,\n    mlp_activation=\"leaky_relu\",\n)\nmodel = WideDeep(wide=wide, deeptabular=tab_mlp)\n</pre> wide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1) tab_mlp = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     continuous_cols=continuous_cols,     mlp_hidden_dims=[128, 64],     mlp_dropout=0.2,     mlp_activation=\"leaky_relu\", ) model = WideDeep(wide=wide, deeptabular=tab_mlp) In\u00a0[18]: Copied! <pre>from pytorch_widedeep.metrics import Metric\n</pre> from pytorch_widedeep.metrics import Metric In\u00a0[19]: Copied! <pre>class Accuracy(Metric):\n    def __init__(self, top_k: int = 1):\n        super(Accuracy, self).__init__()\n\n        self.top_k = top_k\n        self.correct_count = 0\n        self.total_count = 0\n\n        # \u00a0metric name needs to be defined\n        self._name = \"acc\"\n\n    def reset(self):\n        self.correct_count = 0\n        self.total_count = 0\n\n    def __call__(self, y_pred: Tensor, y_true: Tensor) -&gt; np.ndarray:\n        num_classes = y_pred.size(1)\n\n        if num_classes == 1:\n            y_pred = y_pred.round()\n            y_true = y_true\n        elif num_classes &gt; 1:\n            y_pred = y_pred.topk(self.top_k, 1)[1]\n            y_true = y_true.view(-1, 1).expand_as(y_pred)\n\n        self.correct_count += y_pred.eq(y_true).sum().item()\n        self.total_count += len(y_pred)\n        accuracy = float(self.correct_count) / float(self.total_count)\n        return np.array(accuracy)\n</pre> class Accuracy(Metric):     def __init__(self, top_k: int = 1):         super(Accuracy, self).__init__()          self.top_k = top_k         self.correct_count = 0         self.total_count = 0          # \u00a0metric name needs to be defined         self._name = \"acc\"      def reset(self):         self.correct_count = 0         self.total_count = 0      def __call__(self, y_pred: Tensor, y_true: Tensor) -&gt; np.ndarray:         num_classes = y_pred.size(1)          if num_classes == 1:             y_pred = y_pred.round()             y_true = y_true         elif num_classes &gt; 1:             y_pred = y_pred.topk(self.top_k, 1)[1]             y_true = y_true.view(-1, 1).expand_as(y_pred)          self.correct_count += y_pred.eq(y_true).sum().item()         self.total_count += len(y_pred)         accuracy = float(self.correct_count) / float(self.total_count)         return np.array(accuracy) In\u00a0[20]: Copied! <pre># have a look to the class\nfrom pytorch_widedeep.callbacks import Callback\n</pre> # have a look to the class from pytorch_widedeep.callbacks import Callback In\u00a0[21]: Copied! <pre>class SillyCallback(Callback):\n    def on_train_begin(self, logs=None):\n        # recordings will be the trainer object attributes\n        self.trainer.silly_callback = {}\n\n        self.trainer.silly_callback[\"beginning\"] = []\n        self.trainer.silly_callback[\"end\"] = []\n\n    def on_epoch_begin(self, epoch, logs=None):\n        self.trainer.silly_callback[\"beginning\"].append(epoch + 1)\n\n    def on_epoch_end(self, epoch, logs=None, metric=None):\n        self.trainer.silly_callback[\"end\"].append(epoch + 1)\n</pre> class SillyCallback(Callback):     def on_train_begin(self, logs=None):         # recordings will be the trainer object attributes         self.trainer.silly_callback = {}          self.trainer.silly_callback[\"beginning\"] = []         self.trainer.silly_callback[\"end\"] = []      def on_epoch_begin(self, epoch, logs=None):         self.trainer.silly_callback[\"beginning\"].append(epoch + 1)      def on_epoch_end(self, epoch, logs=None, metric=None):         self.trainer.silly_callback[\"end\"].append(epoch + 1) <p>and now, as usual:</p> In\u00a0[22]: Copied! <pre>trainer = Trainer(\n    model, objective=\"binary\", metrics=[Accuracy], callbacks=[SillyCallback]\n)\n</pre> trainer = Trainer(     model, objective=\"binary\", metrics=[Accuracy], callbacks=[SillyCallback] ) In\u00a0[23]: Copied! <pre>trainer.fit(\n    X_wide=X_wide, X_tab=X_tab, target=target, n_epochs=5, batch_size=64, val_split=0.2\n)\n</pre> trainer.fit(     X_wide=X_wide, X_tab=X_tab, target=target, n_epochs=5, batch_size=64, val_split=0.2 ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 611/611 [00:06&lt;00:00, 94.39it/s, loss=0.411, metrics={'acc': 0.814}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 121.91it/s, loss=0.327, metrics={'acc': 0.8449}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 611/611 [00:07&lt;00:00, 85.39it/s, loss=0.324, metrics={'acc': 0.8495}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 88.68it/s, loss=0.298, metrics={'acc': 0.8612}]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 611/611 [00:08&lt;00:00, 74.35it/s, loss=0.302, metrics={'acc': 0.8593}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 100.51it/s, loss=0.29, metrics={'acc': 0.8665}]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 611/611 [00:08&lt;00:00, 73.83it/s, loss=0.292, metrics={'acc': 0.8637}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 105.98it/s, loss=0.286, metrics={'acc': 0.8695}]\nepoch 5: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 611/611 [00:08&lt;00:00, 72.15it/s, loss=0.286, metrics={'acc': 0.866}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 92.27it/s, loss=0.284, metrics={'acc': 0.8698}]\n</pre> In\u00a0[24]: Copied! <pre>trainer.silly_callback\n</pre> trainer.silly_callback Out[24]: <pre>{'beginning': [1, 2, 3, 4, 5], 'end': [1, 2, 3, 4, 5]}</pre>"},{"location":"examples/07_custom_components.html#custom-components","title":"Custom components\u00b6","text":"<p>As I mentioned earlier in the example notebooks, and also in the <code>README</code>, it is possible to customise almost every component in <code>pytorch-widedeep</code>.</p> <p>Let's now go through a couple of simple examples to illustrate how that could be done.</p> <p>First let's load and process the data \"as usual\", let's start with a regression and the airbnb dataset.</p>"},{"location":"examples/07_custom_components.html#custom-deeptext","title":"Custom <code>deeptext</code>\u00b6","text":"<p>Standard Pytorch model</p>"},{"location":"examples/07_custom_components.html#custom-loss-function","title":"Custom loss function\u00b6","text":"<p>Loss functions must simply inherit pytorch's <code>nn.Module</code>. For example, let's say we want to use <code>RMSE</code> (note that this is already available in the package, but I will pass it here as a custom loss for illustration purposes)</p>"},{"location":"examples/07_custom_components.html#custom-metric","title":"Custom metric\u00b6","text":"<p>Let's say we want to use our own accuracy metric (again, this is already available in the package, but I will pass it here as a custom loss for illustration purposes).</p> <p>This could be done as:</p>"},{"location":"examples/07_custom_components.html#custom-callback","title":"Custom Callback\u00b6","text":"<p>Let's code a callback that records the current epoch at the beginning and the end of each epoch (silly, but you know, this is just an example)</p>"},{"location":"examples/08_custom_dataLoader_imbalanced_dataset.html","title":"08_custom_dataLoader_imbalanced_dataset","text":"<ul> <li>In this notebook we will use the higly imbalanced Protein Homology Dataset from KDD cup 2004</li> </ul> <pre><code>* The first element of each line is a BLOCK ID that denotes to which native sequence this example belongs. There is a unique BLOCK ID for each native sequence. BLOCK IDs are integers running from 1 to 303 (one for each native sequence, i.e. for each query). BLOCK IDs were assigned before the blocks were split into the train and test sets, so they do not run consecutively in either file.\n* The second element of each line is an EXAMPLE ID that uniquely describes the example. You will need this EXAMPLE ID and the BLOCK ID when you submit results.\n* The third element is the class of the example. Proteins that are homologous to the native sequence are denoted by 1, non-homologous proteins (i.e. decoys) by 0. Test examples have a \"?\" in this position.\n* All following elements are feature values. There are 74 feature values in each line. The features describe the match (e.g. the score of a sequence alignment) between the native protein sequence and the sequence that is tested for homology.\n</code></pre> In\u00a0[1]: Copied! <pre>import numpy as np\nimport pandas as pd\nimport torch\nfrom torch.optim import SGD, lr_scheduler\n\nfrom pytorch_widedeep import Trainer\nfrom pytorch_widedeep.preprocessing import TabPreprocessor\nfrom pytorch_widedeep.models import TabMlp, WideDeep\nfrom pytorch_widedeep.dataloaders import DataLoaderImbalanced, DataLoaderDefault\nfrom torchmetrics import F1Score as F1_torchmetrics\nfrom torchmetrics import Accuracy as Accuracy_torchmetrics\nfrom torchmetrics import Precision as Precision_torchmetrics\nfrom torchmetrics import Recall as Recall_torchmetrics\nfrom pytorch_widedeep.metrics import Accuracy, Recall, Precision, F1Score, R2Score\nfrom pytorch_widedeep.initializers import XavierNormal\nfrom pytorch_widedeep.datasets import load_bio_kdd04\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import classification_report\n\nimport time\nimport datetime\n\nimport warnings\n\nwarnings.filterwarnings(\"ignore\", category=DeprecationWarning)\n\n# increase displayed columns in jupyter notebook\npd.set_option(\"display.max_columns\", 200)\npd.set_option(\"display.max_rows\", 300)\n</pre> import numpy as np import pandas as pd import torch from torch.optim import SGD, lr_scheduler  from pytorch_widedeep import Trainer from pytorch_widedeep.preprocessing import TabPreprocessor from pytorch_widedeep.models import TabMlp, WideDeep from pytorch_widedeep.dataloaders import DataLoaderImbalanced, DataLoaderDefault from torchmetrics import F1Score as F1_torchmetrics from torchmetrics import Accuracy as Accuracy_torchmetrics from torchmetrics import Precision as Precision_torchmetrics from torchmetrics import Recall as Recall_torchmetrics from pytorch_widedeep.metrics import Accuracy, Recall, Precision, F1Score, R2Score from pytorch_widedeep.initializers import XavierNormal from pytorch_widedeep.datasets import load_bio_kdd04  from sklearn.model_selection import train_test_split from sklearn.metrics import classification_report  import time import datetime  import warnings  warnings.filterwarnings(\"ignore\", category=DeprecationWarning)  # increase displayed columns in jupyter notebook pd.set_option(\"display.max_columns\", 200) pd.set_option(\"display.max_rows\", 300) <pre>/Users/javierrodriguezzaurin/.pyenv/versions/3.10.13/envs/widedeep310/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n  from .autonotebook import tqdm as notebook_tqdm\n</pre> In\u00a0[2]: Copied! <pre>df = load_bio_kdd04(as_frame=True)\n# drop columns we won't need in this example\ndf.drop(columns=[\"EXAMPLE_ID\", \"BLOCK_ID\"], inplace=True)\n\ndf_train, df_valid = train_test_split(\n    df, test_size=0.2, stratify=df[\"target\"], random_state=1\n)\ndf_valid, df_test = train_test_split(\n    df_valid, test_size=0.5, stratify=df_valid[\"target\"], random_state=1\n)\n\ncontinuous_cols = df.drop(columns=[\"target\"]).columns.values.tolist()\n</pre> df = load_bio_kdd04(as_frame=True) # drop columns we won't need in this example df.drop(columns=[\"EXAMPLE_ID\", \"BLOCK_ID\"], inplace=True)  df_train, df_valid = train_test_split(     df, test_size=0.2, stratify=df[\"target\"], random_state=1 ) df_valid, df_test = train_test_split(     df_valid, test_size=0.5, stratify=df_valid[\"target\"], random_state=1 )  continuous_cols = df.drop(columns=[\"target\"]).columns.values.tolist() In\u00a0[3]: Copied! <pre># deeptabular\ntab_preprocessor = TabPreprocessor(continuous_cols=continuous_cols, scale=True)\nX_tab_train = tab_preprocessor.fit_transform(df_train)\nX_tab_valid = tab_preprocessor.transform(df_valid)\nX_tab_test = tab_preprocessor.transform(df_test)\n\n# target\ny_train = df_train[\"target\"].values\ny_valid = df_valid[\"target\"].values\ny_test = df_test[\"target\"].values\n</pre> # deeptabular tab_preprocessor = TabPreprocessor(continuous_cols=continuous_cols, scale=True) X_tab_train = tab_preprocessor.fit_transform(df_train) X_tab_valid = tab_preprocessor.transform(df_valid) X_tab_test = tab_preprocessor.transform(df_test)  # target y_train = df_train[\"target\"].values y_valid = df_valid[\"target\"].values y_test = df_test[\"target\"].values In\u00a0[4]: Copied! <pre># Define the model\ninput_layer = len(tab_preprocessor.continuous_cols)\noutput_layer = 1\nhidden_layers = np.linspace(\n    input_layer * 2, output_layer, 5, endpoint=False, dtype=int\n).tolist()\n\ndeeptabular = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    continuous_cols=tab_preprocessor.continuous_cols,\n    mlp_hidden_dims=hidden_layers,\n)\nmodel = WideDeep(deeptabular=deeptabular)\nmodel\n</pre> # Define the model input_layer = len(tab_preprocessor.continuous_cols) output_layer = 1 hidden_layers = np.linspace(     input_layer * 2, output_layer, 5, endpoint=False, dtype=int ).tolist()  deeptabular = TabMlp(     column_idx=tab_preprocessor.column_idx,     continuous_cols=tab_preprocessor.continuous_cols,     mlp_hidden_dims=hidden_layers, ) model = WideDeep(deeptabular=deeptabular) model Out[4]: <pre>WideDeep(\n  (deeptabular): Sequential(\n    (0): TabMlp(\n      (cont_norm): Identity()\n      (encoder): MLP(\n        (mlp): Sequential(\n          (dense_layer_0): Sequential(\n            (0): Linear(in_features=74, out_features=148, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_1): Sequential(\n            (0): Linear(in_features=148, out_features=118, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_2): Sequential(\n            (0): Linear(in_features=118, out_features=89, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_3): Sequential(\n            (0): Linear(in_features=89, out_features=59, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_4): Sequential(\n            (0): Linear(in_features=59, out_features=30, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=30, out_features=1, bias=True)\n  )\n)</pre> In\u00a0[5]: Copied! <pre># Metrics from pytorch-widedeep\naccuracy = Accuracy(top_k=2)\nprecision = Precision(average=False)\n\n# # Metrics from torchmetrics\n# accuracy = Accuracy_torchmetrics(average=None, num_classes=1)\n# precision = Precision_torchmetrics(average=\"micro\", num_classes=1)\n\n# Optimizers\ndeep_opt = SGD(model.deeptabular.parameters(), lr=0.1)\n\n# LR Scheduler\ndeep_sch = lr_scheduler.StepLR(deep_opt, step_size=3)\n\ntrainer = Trainer(\n    model,\n    objective=\"binary\",\n    lr_schedulers={\"deeptabular\": deep_sch},\n    initializers={\"deeptabular\": XavierNormal},\n    optimizers={\"deeptabular\": deep_opt},\n    metrics=[accuracy, precision],\n    verbose=1,\n)\n</pre> # Metrics from pytorch-widedeep accuracy = Accuracy(top_k=2) precision = Precision(average=False)  # # Metrics from torchmetrics # accuracy = Accuracy_torchmetrics(average=None, num_classes=1) # precision = Precision_torchmetrics(average=\"micro\", num_classes=1)  # Optimizers deep_opt = SGD(model.deeptabular.parameters(), lr=0.1)  # LR Scheduler deep_sch = lr_scheduler.StepLR(deep_opt, step_size=3)  trainer = Trainer(     model,     objective=\"binary\",     lr_schedulers={\"deeptabular\": deep_sch},     initializers={\"deeptabular\": XavierNormal},     optimizers={\"deeptabular\": deep_opt},     metrics=[accuracy, precision],     verbose=1, ) In\u00a0[6]: Copied! <pre>start = time.time()\ntrainer.fit(\n    X_train={\"X_tab\": X_tab_train, \"target\": y_train},\n    X_val={\"X_tab\": X_tab_valid, \"target\": y_valid},\n    n_epochs=1,\n    batch_size=32,\n    custom_dataloader=DataLoaderImbalanced,\n    oversample_mul=5,\n)\nprint(\n    \"Training time[s]: {}\".format(\n        datetime.timedelta(seconds=round(time.time() - start))\n    )\n)\n\npd.DataFrame(trainer.history)\n\ndf_pred = trainer.predict(X_tab=X_tab_test)\nprint(classification_report(df_test[\"target\"].to_list(), df_pred))\nprint(\"Actual predicted values:\\n{}\".format(np.unique(df_pred, return_counts=True)))\n</pre> start = time.time() trainer.fit(     X_train={\"X_tab\": X_tab_train, \"target\": y_train},     X_val={\"X_tab\": X_tab_valid, \"target\": y_valid},     n_epochs=1,     batch_size=32,     custom_dataloader=DataLoaderImbalanced,     oversample_mul=5, ) print(     \"Training time[s]: {}\".format(         datetime.timedelta(seconds=round(time.time() - start))     ) )  pd.DataFrame(trainer.history)  df_pred = trainer.predict(X_tab=X_tab_test) print(classification_report(df_test[\"target\"].to_list(), df_pred)) print(\"Actual predicted values:\\n{}\".format(np.unique(df_pred, return_counts=True))) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 325/325 [00:02&lt;00:00, 153.99it/s, loss=0.163, metrics={'acc': 0.9363, 'prec': [0.9358]}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 456/456 [00:02&lt;00:00, 205.93it/s, loss=0.1, metrics={'acc': 0.9501, 'prec': [0.1447]}]\n</pre> <pre>Training time[s]: 0:00:04\n</pre> <pre>predict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 456/456 [00:01&lt;00:00, 368.16it/s]\n</pre> <pre>              precision    recall  f1-score   support\n\n           0       1.00      0.95      0.97     14446\n           1       0.15      0.95      0.25       130\n\n    accuracy                           0.95     14576\n   macro avg       0.57      0.95      0.61     14576\nweighted avg       0.99      0.95      0.97     14576\n\nActual predicted values:\n(array([0, 1]), array([13736,   840]))\n</pre>"},{"location":"examples/08_custom_dataLoader_imbalanced_dataset.html#custom-dataloader-for-imbalanced-dataset","title":"Custom DataLoader for Imbalanced dataset\u00b6","text":""},{"location":"examples/08_custom_dataLoader_imbalanced_dataset.html#initial-imports","title":"Initial imports\u00b6","text":""},{"location":"examples/09_extracting_embeddings.html","title":"09_extracting_embeddings","text":"In\u00a0[1]: Copied! <pre>import numpy as np\nimport pandas as pd\nimport torch\n\nfrom pytorch_widedeep.preprocessing import TabPreprocessor\nfrom pytorch_widedeep.training import Trainer\nfrom pytorch_widedeep.models import FTTransformer, WideDeep\nfrom pytorch_widedeep.metrics import Accuracy\nfrom pytorch_widedeep import Tab2Vec\nfrom pytorch_widedeep.datasets import load_adult\n</pre> import numpy as np import pandas as pd import torch  from pytorch_widedeep.preprocessing import TabPreprocessor from pytorch_widedeep.training import Trainer from pytorch_widedeep.models import FTTransformer, WideDeep from pytorch_widedeep.metrics import Accuracy from pytorch_widedeep import Tab2Vec from pytorch_widedeep.datasets import load_adult In\u00a0[2]: Copied! <pre>df = load_adult(as_frame=True)\ndf.head()\n</pre> df = load_adult(as_frame=True) df.head() Out[2]: age workclass fnlwgt education educational-num marital-status occupation relationship race gender capital-gain capital-loss hours-per-week native-country income 0 25 Private 226802 11th 7 Never-married Machine-op-inspct Own-child Black Male 0 0 40 United-States &lt;=50K 1 38 Private 89814 HS-grad 9 Married-civ-spouse Farming-fishing Husband White Male 0 0 50 United-States &lt;=50K 2 28 Local-gov 336951 Assoc-acdm 12 Married-civ-spouse Protective-serv Husband White Male 0 0 40 United-States &gt;50K 3 44 Private 160323 Some-college 10 Married-civ-spouse Machine-op-inspct Husband Black Male 7688 0 40 United-States &gt;50K 4 18 ? 103497 Some-college 10 Never-married ? Own-child White Female 0 0 30 United-States &lt;=50K In\u00a0[3]: Copied! <pre># For convenience, we'll replace '-' with '_'\ndf.columns = [c.replace(\"-\", \"_\") for c in df.columns]\n# binary target\ndf[\"target\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int)\ndf.drop([\"income\", \"educational_num\"], axis=1, inplace=True)\n\ndf.head()\n</pre> # For convenience, we'll replace '-' with '_' df.columns = [c.replace(\"-\", \"_\") for c in df.columns] # binary target df[\"target\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int) df.drop([\"income\", \"educational_num\"], axis=1, inplace=True)  df.head() Out[3]: age workclass fnlwgt education marital_status occupation relationship race gender capital_gain capital_loss hours_per_week native_country target 0 25 Private 226802 11th Never-married Machine-op-inspct Own-child Black Male 0 0 40 United-States 0 1 38 Private 89814 HS-grad Married-civ-spouse Farming-fishing Husband White Male 0 0 50 United-States 0 2 28 Local-gov 336951 Assoc-acdm Married-civ-spouse Protective-serv Husband White Male 0 0 40 United-States 1 3 44 Private 160323 Some-college Married-civ-spouse Machine-op-inspct Husband Black Male 7688 0 40 United-States 1 4 18 ? 103497 Some-college Never-married ? Own-child White Female 0 0 30 United-States 0 In\u00a0[4]: Copied! <pre>cat_cols, cont_cols = [], []\nfor col in df.columns:\n    # 50 is just a random number I choose here for this example\n    if df[col].dtype == \"O\" or df[col].nunique() &lt; 50 and col != \"target\":\n        cat_cols.append(col)\n    elif col != \"target\":\n        cont_cols.append(col)\ntarget_col = \"target\"\n</pre> cat_cols, cont_cols = [], [] for col in df.columns:     # 50 is just a random number I choose here for this example     if df[col].dtype == \"O\" or df[col].nunique() &lt; 50 and col != \"target\":         cat_cols.append(col)     elif col != \"target\":         cont_cols.append(col) target_col = \"target\" In\u00a0[5]: Copied! <pre>target = df[target_col].values\n\ntab_preprocessor = TabPreprocessor(\n    embed_cols=cat_cols, continuous_cols=cont_cols, for_transformer=True\n)\nX_tab = tab_preprocessor.fit_transform(df)\n</pre> target = df[target_col].values  tab_preprocessor = TabPreprocessor(     embed_cols=cat_cols, continuous_cols=cont_cols, for_transformer=True ) X_tab = tab_preprocessor.fit_transform(df) <pre>/Users/javierrodriguezzaurin/Projects/pytorch-widedeep/pytorch_widedeep/preprocessing/tab_preprocessor.py:358: UserWarning: Continuous columns will not be normalised\n  warnings.warn(\"Continuous columns will not be normalised\")\n</pre> In\u00a0[6]: Copied! <pre>ft_transformer = FTTransformer(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=tab_preprocessor.continuous_cols,\n    embed_continuous_method=\"standard\",\n    n_blocks=2,\n    n_heads=4,\n    input_dim=16,\n)\n</pre> ft_transformer = FTTransformer(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     continuous_cols=tab_preprocessor.continuous_cols,     embed_continuous_method=\"standard\",     n_blocks=2,     n_heads=4,     input_dim=16, ) <pre>/Users/javierrodriguezzaurin/Projects/pytorch-widedeep/pytorch_widedeep/utils/general_utils.py:12: DeprecationWarning: The 'embed_continuous' parameter is deprecated and will be removed in the next release. Please use 'embed_continuous_method' instead See the documentation for more details.\n  return func(*args, **kwargs)\n</pre> In\u00a0[7]: Copied! <pre>model = WideDeep(deeptabular=ft_transformer)\ntrainer = Trainer(model, objective=\"binary\", metrics=[Accuracy])\ntrainer.fit(X_tab=X_tab, target=target, n_epochs=1, batch_size=256, val_split=0.2)\n</pre> model = WideDeep(deeptabular=ft_transformer) trainer = Trainer(model, objective=\"binary\", metrics=[Accuracy]) trainer.fit(X_tab=X_tab, target=target, n_epochs=1, batch_size=256, val_split=0.2) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:03&lt;00:00, 41.47it/s, loss=221, metrics={'acc': 0.686}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 39/39 [00:00&lt;00:00, 86.24it/s, loss=9.28, metrics={'acc': 0.76}]\n</pre> In\u00a0[8]: Copied! <pre>t2v = Tab2Vec(model=model, tab_preprocessor=tab_preprocessor)\n</pre> t2v = Tab2Vec(model=model, tab_preprocessor=tab_preprocessor) In\u00a0[9]: Copied! <pre># assuming is a test set with target col\nX_vec, y = t2v.transform(df.sample(100), target_col=\"target\")\n</pre> # assuming is a test set with target col X_vec, y = t2v.transform(df.sample(100), target_col=\"target\") In\u00a0[10]: Copied! <pre># X vec is the dataframe turned into the embeddings\nX_vec.shape\n</pre> # X vec is the dataframe turned into the embeddings X_vec.shape Out[10]: <pre>(100, 208)</pre> <p><code>208 = input_dim (16) * n_cols (13)</code></p> In\u00a0[11]: Copied! <pre># ...or if we don't have target col\nX_vec = t2v.transform(df.sample(100))\n</pre> # ...or if we don't have target col X_vec = t2v.transform(df.sample(100))"},{"location":"examples/09_extracting_embeddings.html#extracting-embeddings","title":"Extracting embeddings\u00b6","text":"<p>This notebook is a simple guide to extracting learned feature embeddings using Tab2Vec</p>"},{"location":"examples/10_3rd_party_integration-RayTune_WnB.html","title":"10_3rd_party_integration-RayTune_WnB","text":"In\u00a0[11]: Copied! <pre>from typing import Optional, Dict\nimport os\n\nimport numpy as np\nimport pandas as pd\nimport torch\nimport wandb\nfrom torch.optim import SGD, lr_scheduler\n\nfrom pytorch_widedeep import Trainer\nfrom pytorch_widedeep.preprocessing import TabPreprocessor\nfrom pytorch_widedeep.models import TabMlp, WideDeep\nfrom torchmetrics import F1Score as F1_torchmetrics\nfrom torchmetrics import Accuracy as Accuracy_torchmetrics\nfrom torchmetrics import Precision as Precision_torchmetrics\nfrom torchmetrics import Recall as Recall_torchmetrics\nfrom pytorch_widedeep.metrics import Accuracy, Recall, Precision, F1Score, R2Score\nfrom pytorch_widedeep.initializers import XavierNormal\nfrom pytorch_widedeep.callbacks import (\n    EarlyStopping,\n    ModelCheckpoint,\n    Callback,\n)\nfrom pytorch_widedeep.datasets import load_bio_kdd04\n\nfrom sklearn.model_selection import train_test_split\nimport warnings\n\nwarnings.filterwarnings(\"ignore\", category=DeprecationWarning)\n\nfrom ray import tune\nfrom ray.tune.schedulers import AsyncHyperBandScheduler\nfrom ray.tune import JupyterNotebookReporter\nfrom ray.air.integrations.wandb import WandbLoggerCallback\n\n# from ray.tune.integration.wandb import wandb_mixin\n\nimport tracemalloc\n\ntracemalloc.start()\n\n# increase displayed columns in jupyter notebook\npd.set_option(\"display.max_columns\", 200)\npd.set_option(\"display.max_rows\", 300)\n</pre> from typing import Optional, Dict import os  import numpy as np import pandas as pd import torch import wandb from torch.optim import SGD, lr_scheduler  from pytorch_widedeep import Trainer from pytorch_widedeep.preprocessing import TabPreprocessor from pytorch_widedeep.models import TabMlp, WideDeep from torchmetrics import F1Score as F1_torchmetrics from torchmetrics import Accuracy as Accuracy_torchmetrics from torchmetrics import Precision as Precision_torchmetrics from torchmetrics import Recall as Recall_torchmetrics from pytorch_widedeep.metrics import Accuracy, Recall, Precision, F1Score, R2Score from pytorch_widedeep.initializers import XavierNormal from pytorch_widedeep.callbacks import (     EarlyStopping,     ModelCheckpoint,     Callback, ) from pytorch_widedeep.datasets import load_bio_kdd04  from sklearn.model_selection import train_test_split import warnings  warnings.filterwarnings(\"ignore\", category=DeprecationWarning)  from ray import tune from ray.tune.schedulers import AsyncHyperBandScheduler from ray.tune import JupyterNotebookReporter from ray.air.integrations.wandb import WandbLoggerCallback  # from ray.tune.integration.wandb import wandb_mixin  import tracemalloc  tracemalloc.start()  # increase displayed columns in jupyter notebook pd.set_option(\"display.max_columns\", 200) pd.set_option(\"display.max_rows\", 300) In\u00a0[12]: Copied! <pre>class RayTuneReporter(Callback):\n    r\"\"\"Callback that allows reporting history and lr_history values to RayTune\n    during Hyperparameter tuning\n\n    Callbacks are passed as input parameters to the ``Trainer`` class. See\n    :class:`pytorch_widedeep.trainer.Trainer`\n\n    For examples see the examples folder at:\n\n        .. code-block:: bash\n\n            /examples/12_HyperParameter_tuning_w_RayTune.ipynb\n    \"\"\"\n\n    def on_epoch_end(\n        self, epoch: int, logs: Optional[Dict] = None, metric: Optional[float] = None\n    ):\n        report_dict = {}\n        for k, v in self.trainer.history.items():\n            report_dict.update({k: v[-1]})\n        if hasattr(self.trainer, \"lr_history\"):\n            for k, v in self.trainer.lr_history.items():\n                report_dict.update({k: v[-1]})\n        tune.report(report_dict)\n\n\nclass WnBReportBest(Callback):\n    r\"\"\"Callback that allows reporting best performance of a run to WnB\n    during Hyperparameter tuning. It is an adjusted pytorch_widedeep.callbacks.ModelCheckpoint\n    with added WnB and removed checkpoint saving.\n\n    Callbacks are passed as input parameters to the ``Trainer`` class.\n\n    Parameters\n    ----------\n    wb: obj\n        Weights&amp;Biases API interface to report single best result usable for\n        comparisson of multiple paramater combinations by, for example,\n        `parallel coordinates\n        &lt;https://docs.wandb.ai/ref/app/features/panels/parallel-coordinates&gt;`_.\n        E.g W&amp;B summary report `wandb.run.summary[\"best\"]`.\n    monitor: str, default=\"loss\"\n        quantity to monitor. Typically `'val_loss'` or metric name\n        (e.g. `'val_acc'`)\n    mode: str, default=\"auto\"\n        If ``save_best_only=True``, the decision to overwrite the current save\n        file is made based on either the maximization or the minimization of\n        the monitored quantity. For `'acc'`, this should be `'max'`, for\n        `'loss'` this should be `'min'`, etc. In `'auto'` mode, the\n        direction is automatically inferred from the name of the monitored\n        quantity.\n\n    \"\"\"\n\n    def __init__(\n        self,\n        wb: object,\n        monitor: str = \"val_loss\",\n        mode: str = \"auto\",\n    ):\n        super(WnBReportBest, self).__init__()\n\n        self.monitor = monitor\n        self.mode = mode\n        self.wb = wb\n\n        if self.mode not in [\"auto\", \"min\", \"max\"]:\n            warnings.warn(\n                \"WnBReportBest mode %s is unknown, \"\n                \"fallback to auto mode.\" % (self.mode),\n                RuntimeWarning,\n            )\n            self.mode = \"auto\"\n        if self.mode == \"min\":\n            self.monitor_op = np.less\n            self.best = np.Inf\n        elif self.mode == \"max\":\n            self.monitor_op = np.greater  # type: ignore[assignment]\n            self.best = -np.Inf\n        else:\n            if self._is_metric(self.monitor):\n                self.monitor_op = np.greater  # type: ignore[assignment]\n                self.best = -np.Inf\n            else:\n                self.monitor_op = np.less\n                self.best = np.Inf\n\n    def on_epoch_end(  # noqa: C901\n        self, epoch: int, logs: Optional[Dict] = None, metric: Optional[float] = None\n    ):\n        logs = logs or {}\n        current = logs.get(self.monitor)\n        if current is not None:\n            if self.monitor_op(current, self.best):\n                self.wb.run.summary[\"best\"] = current  # type: ignore[attr-defined]\n                self.best = current\n                self.best_epoch = epoch\n\n    @staticmethod\n    def _is_metric(monitor: str):\n        \"copied from pytorch_widedeep.callbacks\"\n        if any([s in monitor for s in [\"acc\", \"prec\", \"rec\", \"fscore\", \"f1\", \"f2\"]]):\n            return True\n        else:\n            return False\n</pre> class RayTuneReporter(Callback):     r\"\"\"Callback that allows reporting history and lr_history values to RayTune     during Hyperparameter tuning      Callbacks are passed as input parameters to the ``Trainer`` class. See     :class:`pytorch_widedeep.trainer.Trainer`      For examples see the examples folder at:          .. code-block:: bash              /examples/12_HyperParameter_tuning_w_RayTune.ipynb     \"\"\"      def on_epoch_end(         self, epoch: int, logs: Optional[Dict] = None, metric: Optional[float] = None     ):         report_dict = {}         for k, v in self.trainer.history.items():             report_dict.update({k: v[-1]})         if hasattr(self.trainer, \"lr_history\"):             for k, v in self.trainer.lr_history.items():                 report_dict.update({k: v[-1]})         tune.report(report_dict)   class WnBReportBest(Callback):     r\"\"\"Callback that allows reporting best performance of a run to WnB     during Hyperparameter tuning. It is an adjusted pytorch_widedeep.callbacks.ModelCheckpoint     with added WnB and removed checkpoint saving.      Callbacks are passed as input parameters to the ``Trainer`` class.      Parameters     ----------     wb: obj         Weights&amp;Biases API interface to report single best result usable for         comparisson of multiple paramater combinations by, for example,         `parallel coordinates         `_.         E.g W&amp;B summary report `wandb.run.summary[\"best\"]`.     monitor: str, default=\"loss\"         quantity to monitor. Typically `'val_loss'` or metric name         (e.g. `'val_acc'`)     mode: str, default=\"auto\"         If ``save_best_only=True``, the decision to overwrite the current save         file is made based on either the maximization or the minimization of         the monitored quantity. For `'acc'`, this should be `'max'`, for         `'loss'` this should be `'min'`, etc. In `'auto'` mode, the         direction is automatically inferred from the name of the monitored         quantity.      \"\"\"      def __init__(         self,         wb: object,         monitor: str = \"val_loss\",         mode: str = \"auto\",     ):         super(WnBReportBest, self).__init__()          self.monitor = monitor         self.mode = mode         self.wb = wb          if self.mode not in [\"auto\", \"min\", \"max\"]:             warnings.warn(                 \"WnBReportBest mode %s is unknown, \"                 \"fallback to auto mode.\" % (self.mode),                 RuntimeWarning,             )             self.mode = \"auto\"         if self.mode == \"min\":             self.monitor_op = np.less             self.best = np.Inf         elif self.mode == \"max\":             self.monitor_op = np.greater  # type: ignore[assignment]             self.best = -np.Inf         else:             if self._is_metric(self.monitor):                 self.monitor_op = np.greater  # type: ignore[assignment]                 self.best = -np.Inf             else:                 self.monitor_op = np.less                 self.best = np.Inf      def on_epoch_end(  # noqa: C901         self, epoch: int, logs: Optional[Dict] = None, metric: Optional[float] = None     ):         logs = logs or {}         current = logs.get(self.monitor)         if current is not None:             if self.monitor_op(current, self.best):                 self.wb.run.summary[\"best\"] = current  # type: ignore[attr-defined]                 self.best = current                 self.best_epoch = epoch      @staticmethod     def _is_metric(monitor: str):         \"copied from pytorch_widedeep.callbacks\"         if any([s in monitor for s in [\"acc\", \"prec\", \"rec\", \"fscore\", \"f1\", \"f2\"]]):             return True         else:             return False In\u00a0[13]: Copied! <pre>df = load_bio_kdd04(as_frame=True)\ndf.head()\n</pre> df = load_bio_kdd04(as_frame=True) df.head() Out[13]: EXAMPLE_ID BLOCK_ID target 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 0 279 261532 0 52.0 32.69 0.30 2.5 20.0 1256.8 -0.89 0.33 11.0 -55.0 267.2 0.52 0.05 -2.36 49.6 252.0 0.43 1.16 -2.06 -33.0 -123.2 1.60 -0.49 -6.06 65.0 296.1 -0.28 -0.26 -3.83 -22.6 -170.0 3.06 -1.05 -3.29 22.9 286.3 0.12 2.58 4.08 -33.0 -178.9 1.88 0.53 -7.0 -44.0 1987.0 -5.41 0.95 -4.0 -57.0 722.9 -3.26 -0.55 -7.5 125.5 1547.2 -0.36 1.12 9.0 -37.0 72.5 0.47 0.74 -11.0 -8.0 1595.1 -1.64 2.83 -2.0 -50.0 445.2 -0.35 0.26 0.76 1 279 261533 0 58.0 33.33 0.00 16.5 9.5 608.1 0.50 0.07 20.5 -52.5 521.6 -1.08 0.58 -0.02 -3.2 103.6 -0.95 0.23 -2.87 -25.9 -52.2 -0.21 0.87 -1.81 10.4 62.0 -0.28 -0.04 1.48 -17.6 -198.3 3.43 2.84 5.87 -16.9 72.6 -0.31 2.79 2.71 -33.5 -11.6 -1.11 4.01 5.0 -57.0 666.3 1.13 4.38 5.0 -64.0 39.3 1.07 -0.16 32.5 100.0 1893.7 -2.80 -0.22 2.5 -28.5 45.0 0.58 0.41 -19.0 -6.0 762.9 0.29 0.82 -3.0 -35.0 140.3 1.16 0.39 0.73 2 279 261534 0 77.0 27.27 -0.91 6.0 58.5 1623.6 -1.40 0.02 -6.5 -48.0 621.0 -1.20 0.14 -0.20 73.6 609.1 -0.44 -0.58 -0.04 -23.0 -27.4 -0.72 -1.04 -1.09 91.1 635.6 -0.88 0.24 0.59 -18.7 -7.2 -0.60 -2.82 -0.71 52.4 504.1 0.89 -0.67 -9.30 -20.8 -25.7 -0.77 -0.85 0.0 -20.0 2259.0 -0.94 1.15 -4.0 -44.0 -22.7 0.94 -0.98 -19.0 105.0 1267.9 1.03 1.27 11.0 -39.5 82.3 0.47 -0.19 -10.0 7.0 1491.8 0.32 -1.29 0.0 -34.0 658.2 -0.76 0.26 0.24 3 279 261535 0 41.0 27.91 -0.35 3.0 46.0 1921.6 -1.36 -0.47 -32.0 -51.5 560.9 -0.29 -0.10 -1.11 124.3 791.6 0.00 0.39 -1.85 -21.7 -44.9 -0.21 0.02 0.89 133.9 797.8 -0.08 1.06 -0.26 -16.4 -74.1 0.97 -0.80 -0.41 66.9 955.3 -1.90 1.28 -6.65 -28.1 47.5 -1.91 1.42 1.0 -30.0 1846.7 0.76 1.10 -4.0 -52.0 -53.9 1.71 -0.22 -12.0 97.5 1969.8 -1.70 0.16 -1.0 -32.5 255.9 -0.46 1.57 10.0 6.0 2047.7 -0.98 1.53 0.0 -49.0 554.2 -0.83 0.39 0.73 4 279 261536 0 50.0 28.00 -1.32 -9.0 12.0 464.8 0.88 0.19 8.0 -51.5 98.1 1.09 -0.33 -2.16 -3.9 102.7 0.39 -1.22 -3.39 -15.2 -42.2 -1.18 -1.11 -3.55 8.9 141.3 -0.16 -0.43 -4.15 -12.9 -13.4 -1.32 -0.98 -3.69 8.8 136.1 -0.30 4.13 1.89 -13.0 -18.7 -1.37 -0.93 0.0 -1.0 810.1 -2.29 6.72 1.0 -23.0 -29.7 0.58 -1.10 -18.5 33.5 206.8 1.84 -0.13 4.0 -29.0 30.1 0.80 -0.24 5.0 -14.0 479.5 0.68 -0.59 2.0 -36.0 -6.9 2.02 0.14 -0.23 In\u00a0[14]: Copied! <pre># imbalance of the classes\ndf[\"target\"].value_counts()\n</pre> # imbalance of the classes df[\"target\"].value_counts() Out[14]: <pre>target\n0    144455\n1      1296\nName: count, dtype: int64</pre> In\u00a0[15]: Copied! <pre># drop columns we won't need in this example\ndf.drop(columns=[\"EXAMPLE_ID\", \"BLOCK_ID\"], inplace=True)\n</pre> # drop columns we won't need in this example df.drop(columns=[\"EXAMPLE_ID\", \"BLOCK_ID\"], inplace=True) In\u00a0[16]: Copied! <pre>df_train, df_valid = train_test_split(\n    df, test_size=0.2, stratify=df[\"target\"], random_state=1\n)\ndf_valid, df_test = train_test_split(\n    df_valid, test_size=0.5, stratify=df_valid[\"target\"], random_state=1\n)\n</pre> df_train, df_valid = train_test_split(     df, test_size=0.2, stratify=df[\"target\"], random_state=1 ) df_valid, df_test = train_test_split(     df_valid, test_size=0.5, stratify=df_valid[\"target\"], random_state=1 ) In\u00a0[17]: Copied! <pre>continuous_cols = df.drop(columns=[\"target\"]).columns.values.tolist()\n</pre> continuous_cols = df.drop(columns=[\"target\"]).columns.values.tolist() In\u00a0[18]: Copied! <pre># deeptabular\ntab_preprocessor = TabPreprocessor(continuous_cols=continuous_cols, scale=True)\nX_tab_train = tab_preprocessor.fit_transform(df_train)\nX_tab_valid = tab_preprocessor.transform(df_valid)\nX_tab_test = tab_preprocessor.transform(df_test)\n\n# target\ny_train = df_train[\"target\"].values\ny_valid = df_valid[\"target\"].values\ny_test = df_test[\"target\"].values\n</pre> # deeptabular tab_preprocessor = TabPreprocessor(continuous_cols=continuous_cols, scale=True) X_tab_train = tab_preprocessor.fit_transform(df_train) X_tab_valid = tab_preprocessor.transform(df_valid) X_tab_test = tab_preprocessor.transform(df_test)  # target y_train = df_train[\"target\"].values y_valid = df_valid[\"target\"].values y_test = df_test[\"target\"].values In\u00a0[19]: Copied! <pre>input_layer = len(tab_preprocessor.continuous_cols)\noutput_layer = 1\nhidden_layers = np.linspace(\n    input_layer * 2, output_layer, 5, endpoint=False, dtype=int\n).tolist()\n</pre> input_layer = len(tab_preprocessor.continuous_cols) output_layer = 1 hidden_layers = np.linspace(     input_layer * 2, output_layer, 5, endpoint=False, dtype=int ).tolist() In\u00a0[20]: Copied! <pre>deeptabular = TabMlp(\n    mlp_hidden_dims=hidden_layers,\n    column_idx=tab_preprocessor.column_idx,\n    continuous_cols=tab_preprocessor.continuous_cols,\n)\nmodel = WideDeep(deeptabular=deeptabular)\nmodel\n</pre> deeptabular = TabMlp(     mlp_hidden_dims=hidden_layers,     column_idx=tab_preprocessor.column_idx,     continuous_cols=tab_preprocessor.continuous_cols, ) model = WideDeep(deeptabular=deeptabular) model Out[20]: <pre>WideDeep(\n  (deeptabular): Sequential(\n    (0): TabMlp(\n      (cont_norm): Identity()\n      (encoder): MLP(\n        (mlp): Sequential(\n          (dense_layer_0): Sequential(\n            (0): Linear(in_features=74, out_features=148, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_1): Sequential(\n            (0): Linear(in_features=148, out_features=118, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_2): Sequential(\n            (0): Linear(in_features=118, out_features=89, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_3): Sequential(\n            (0): Linear(in_features=89, out_features=59, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_4): Sequential(\n            (0): Linear(in_features=59, out_features=30, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=30, out_features=1, bias=True)\n  )\n)</pre> In\u00a0[24]: Copied! <pre># Metrics from torchmetrics\naccuracy = Accuracy_torchmetrics(average=None, num_classes=1, task=\"binary\")\nprecision = Precision_torchmetrics(average=\"micro\", num_classes=1, task=\"binary\")\nf1 = F1_torchmetrics(average=None, num_classes=1, task=\"binary\")\nrecall = Recall_torchmetrics(average=None, num_classes=1, task=\"binary\")\n</pre> # Metrics from torchmetrics accuracy = Accuracy_torchmetrics(average=None, num_classes=1, task=\"binary\") precision = Precision_torchmetrics(average=\"micro\", num_classes=1, task=\"binary\") f1 = F1_torchmetrics(average=None, num_classes=1, task=\"binary\") recall = Recall_torchmetrics(average=None, num_classes=1, task=\"binary\") <p>Note:</p> <p>Following cells includes usage of both <code>RayTuneReporter</code> and <code>WnBReportBest</code> callbacks. In case you want to use just <code>RayTuneReporter</code>, remove following:</p> <ul> <li>wandb from config</li> <li><code>WandbLoggerCallback</code></li> <li><code>WnBReportBest</code></li> <li><code>@wandb_mixin</code> decorator</li> </ul> <p>We do not see strong reason to use WnB without RayTune for a single paramater combination run, but it is possible:</p> <ul> <li>option01: define paramaters in config only for a single value <code>tune.grid_search([1000])</code> (single value RayTune run)</li> <li>option02: define WnB callback that reports currnet validation/training loss, metrics, etc. at the end of batch, ie. do not report to WnB at <code>epoch_end</code> as in <code>WnBReportBest</code> but at the <code>on_batch_end</code>, see <code>pytorch_widedeep.callbacks.Callback</code></li> </ul> In\u00a0[26]: Copied! <pre>config = {\n    \"batch_size\": tune.grid_search([1000, 5000]),\n    \"wandb\": {\n        \"project\": \"test\",\n        # \"api_key_file\": os.getcwd() + \"/wandb_api.key\",\n        \"api_key\": \"WNB_API_KEY\",\n    },\n}\n\n# Optimizers\ndeep_opt = SGD(model.deeptabular.parameters(), lr=0.1)\n# LR Scheduler\ndeep_sch = lr_scheduler.StepLR(deep_opt, step_size=3)\n\n\n@wandb_mixin\ndef training_function(config, X_train, X_val):\n    early_stopping = EarlyStopping()\n    model_checkpoint = ModelCheckpoint(save_best_only=True)\n    # Hyperparameters\n    batch_size = config[\"batch_size\"]\n    trainer = Trainer(\n        model,\n        objective=\"binary_focal_loss\",\n        callbacks=[\n            RayTuneReporter,\n            WnBReportBest(wb=wandb),\n            early_stopping,\n            model_checkpoint,\n        ],\n        lr_schedulers={\"deeptabular\": deep_sch},\n        initializers={\"deeptabular\": XavierNormal},\n        optimizers={\"deeptabular\": deep_opt},\n        metrics=[accuracy, precision, recall, f1],\n        verbose=0,\n    )\n\n    trainer.fit(X_train=X_train, X_val=X_val, n_epochs=5, batch_size=batch_size)\n\n\nX_train = {\"X_tab\": X_tab_train, \"target\": y_train}\nX_val = {\"X_tab\": X_tab_valid, \"target\": y_valid}\n\nasha_scheduler = AsyncHyperBandScheduler(\n    time_attr=\"training_iteration\",\n    metric=\"_metric/val_loss\",\n    mode=\"min\",\n    max_t=100,\n    grace_period=10,\n    reduction_factor=3,\n    brackets=1,\n)\n\nanalysis = tune.run(\n    tune.with_parameters(training_function, X_train=X_train, X_val=X_val),\n    resources_per_trial={\"cpu\": 1, \"gpu\": 0},\n    progress_reporter=JupyterNotebookReporter(overwrite=True),\n    scheduler=asha_scheduler,\n    config=config,\n    callbacks=[\n        WandbLoggerCallback(\n            project=config[\"wandb\"][\"project\"],\n            # api_key_file=config[\"wandb\"][\"api_key_file\"],\n            api_key=config[\"wandb\"][\"api_key\"],\n            log_config=True,\n        )\n    ],\n)\n</pre> config = {     \"batch_size\": tune.grid_search([1000, 5000]),     \"wandb\": {         \"project\": \"test\",         # \"api_key_file\": os.getcwd() + \"/wandb_api.key\",         \"api_key\": \"WNB_API_KEY\",     }, }  # Optimizers deep_opt = SGD(model.deeptabular.parameters(), lr=0.1) # LR Scheduler deep_sch = lr_scheduler.StepLR(deep_opt, step_size=3)   @wandb_mixin def training_function(config, X_train, X_val):     early_stopping = EarlyStopping()     model_checkpoint = ModelCheckpoint(save_best_only=True)     # Hyperparameters     batch_size = config[\"batch_size\"]     trainer = Trainer(         model,         objective=\"binary_focal_loss\",         callbacks=[             RayTuneReporter,             WnBReportBest(wb=wandb),             early_stopping,             model_checkpoint,         ],         lr_schedulers={\"deeptabular\": deep_sch},         initializers={\"deeptabular\": XavierNormal},         optimizers={\"deeptabular\": deep_opt},         metrics=[accuracy, precision, recall, f1],         verbose=0,     )      trainer.fit(X_train=X_train, X_val=X_val, n_epochs=5, batch_size=batch_size)   X_train = {\"X_tab\": X_tab_train, \"target\": y_train} X_val = {\"X_tab\": X_tab_valid, \"target\": y_valid}  asha_scheduler = AsyncHyperBandScheduler(     time_attr=\"training_iteration\",     metric=\"_metric/val_loss\",     mode=\"min\",     max_t=100,     grace_period=10,     reduction_factor=3,     brackets=1, )  analysis = tune.run(     tune.with_parameters(training_function, X_train=X_train, X_val=X_val),     resources_per_trial={\"cpu\": 1, \"gpu\": 0},     progress_reporter=JupyterNotebookReporter(overwrite=True),     scheduler=asha_scheduler,     config=config,     callbacks=[         WandbLoggerCallback(             project=config[\"wandb\"][\"project\"],             # api_key_file=config[\"wandb\"][\"api_key_file\"],             api_key=config[\"wandb\"][\"api_key\"],             log_config=True,         )     ], ) <pre>/Users/javierrodriguezzaurin/.pyenv/versions/3.10.13/lib/python3.10/tempfile.py:860: ResourceWarning: Implicitly cleaning up &lt;TemporaryDirectory '/var/folders/_2/lrjn1qn54c758tdtktr1bvkc0000gn/T/tmp60pfyl1kwandb'&gt;\n  _warnings.warn(warn_message, ResourceWarning)\n/Users/javierrodriguezzaurin/.pyenv/versions/3.10.13/lib/python3.10/tempfile.py:860: ResourceWarning: Implicitly cleaning up &lt;TemporaryDirectory '/var/folders/_2/lrjn1qn54c758tdtktr1bvkc0000gn/T/tmpnjv2rg1wwandb-artifacts'&gt;\n  _warnings.warn(warn_message, ResourceWarning)\n/Users/javierrodriguezzaurin/.pyenv/versions/3.10.13/lib/python3.10/tempfile.py:860: ResourceWarning: Implicitly cleaning up &lt;TemporaryDirectory '/var/folders/_2/lrjn1qn54c758tdtktr1bvkc0000gn/T/tmpgebu5k1kwandb-media'&gt;\n  _warnings.warn(warn_message, ResourceWarning)\n/Users/javierrodriguezzaurin/.pyenv/versions/3.10.13/lib/python3.10/tempfile.py:860: ResourceWarning: Implicitly cleaning up &lt;TemporaryDirectory '/var/folders/_2/lrjn1qn54c758tdtktr1bvkc0000gn/T/tmpxy9y2yriwandb-media'&gt;\n  _warnings.warn(warn_message, ResourceWarning)\n</pre> In\u00a0[14]: Copied! <pre>analysis.results\n</pre> analysis.results Out[14]: <pre>{'fc9a8_00000': {'_metric': {'train_loss': 0.006297602537127896,\n   'train_Accuracy': 0.9925042986869812,\n   'train_Precision': 0.9939393997192383,\n   'train_Recall': 0.15814851224422455,\n   'train_F1Score': 0.2728785574436188,\n   'val_loss': 0.005045663565397263,\n   'val_Accuracy': 0.9946483969688416,\n   'val_Precision': 1.0,\n   'val_Recall': 0.39534884691238403,\n   'val_F1Score': 0.5666667222976685},\n  'time_this_iter_s': 2.388202428817749,\n  'done': True,\n  'timesteps_total': None,\n  'episodes_total': None,\n  'training_iteration': 5,\n  'trial_id': 'fc9a8_00000',\n  'experiment_id': 'baad1d4f3d924b48b9ece1b9f26c80cc',\n  'date': '2022-07-31_14-06-51',\n  'timestamp': 1659276411,\n  'time_total_s': 12.656474113464355,\n  'pid': 1813,\n  'hostname': 'jupyter-5uperpalo',\n  'node_ip': '10.32.44.172',\n  'config': {'batch_size': 1000},\n  'time_since_restore': 12.656474113464355,\n  'timesteps_since_restore': 0,\n  'iterations_since_restore': 5,\n  'warmup_time': 0.8006253242492676,\n  'experiment_tag': '0_batch_size=1000'},\n 'fc9a8_00001': {'_metric': {'train_loss': 0.02519632239515583,\n   'train_Accuracy': 0.9910891652107239,\n   'train_Precision': 0.25,\n   'train_Recall': 0.0009643201483413577,\n   'train_F1Score': 0.0019212296465411782,\n   'val_loss': 0.02578434906899929,\n   'val_Accuracy': 0.9911492466926575,\n   'val_Precision': 0.0,\n   'val_Recall': 0.0,\n   'val_F1Score': 0.0},\n  'time_this_iter_s': 4.113586902618408,\n  'done': True,\n  'timesteps_total': None,\n  'episodes_total': None,\n  'training_iteration': 5,\n  'trial_id': 'fc9a8_00001',\n  'experiment_id': 'f2e54a6a5780429fbf0db0746853347e',\n  'date': '2022-07-31_14-06-56',\n  'timestamp': 1659276416,\n  'time_total_s': 12.926990509033203,\n  'pid': 1962,\n  'hostname': 'jupyter-5uperpalo',\n  'node_ip': '10.32.44.172',\n  'config': {'batch_size': 5000},\n  'time_since_restore': 12.926990509033203,\n  'timesteps_since_restore': 0,\n  'iterations_since_restore': 5,\n  'warmup_time': 0.9253025054931641,\n  'experiment_tag': '1_batch_size=5000'}}</pre> <p>Using Weights and Biases logging you can create parallel coordinates graphs that map parametr combinations to the best(lowest) loss achieved during the training of the networks</p> <p></p> <p>local visualization of raytune reults using tensorboard</p> In\u00a0[23]: Copied! <pre>%load_ext tensorboard\n%tensorboard --logdir ~/ray_results\n</pre> %load_ext tensorboard %tensorboard --logdir ~/ray_results"},{"location":"examples/10_3rd_party_integration-RayTune_WnB.html#3rd-party-integration-raytune-weights-biases","title":"3rd party integration - RayTune, Weights &amp; Biases\u00b6","text":"<p>This notebook provides guideline for integration of external library functions in the model training process through <code>Callback</code> objects, a popular concept of using objects as arguments for other objects.</p> <p>[DISCLAIMER]</p> <p>We show integration of RayTune (a hyperparameter tuning framework) and Weights &amp; Biases (ML projects experiment tracking and versioning solution) in the <code>pytorch_widedeep</code> model training process. We did not include <code>RayTuneReporter</code> and <code>WnBReportBest</code> in the library code to minimize the dependencies on other libraries that are not directly included in the model design and training.</p>"},{"location":"examples/10_3rd_party_integration-RayTune_WnB.html#initial-imports","title":"Initial imports\u00b6","text":""},{"location":"examples/10_3rd_party_integration-RayTune_WnB.html#preparing-the-data","title":"Preparing the data\u00b6","text":""},{"location":"examples/10_3rd_party_integration-RayTune_WnB.html#define-the-model","title":"Define the model\u00b6","text":""},{"location":"examples/11_auc_multiclass.html","title":"11_auc_multiclass","text":"In\u00a0[1]: Copied! <pre>import numpy as np\nimport pandas as pd\nfrom torch.optim import SGD, lr_scheduler\n\nfrom pytorch_widedeep import Trainer\nfrom pytorch_widedeep.preprocessing import TabPreprocessor\nfrom pytorch_widedeep.models import TabMlp, WideDeep\nfrom torchmetrics import AUROC\nfrom pytorch_widedeep.initializers import XavierNormal\nfrom pytorch_widedeep.datasets import load_ecoli\nfrom pytorch_widedeep.utils import LabelEncoder\n\nfrom sklearn.model_selection import train_test_split\n\n# increase displayed columns in jupyter notebook\npd.set_option(\"display.max_columns\", 200)\npd.set_option(\"display.max_rows\", 300)\n</pre> import numpy as np import pandas as pd from torch.optim import SGD, lr_scheduler  from pytorch_widedeep import Trainer from pytorch_widedeep.preprocessing import TabPreprocessor from pytorch_widedeep.models import TabMlp, WideDeep from torchmetrics import AUROC from pytorch_widedeep.initializers import XavierNormal from pytorch_widedeep.datasets import load_ecoli from pytorch_widedeep.utils import LabelEncoder  from sklearn.model_selection import train_test_split  # increase displayed columns in jupyter notebook pd.set_option(\"display.max_columns\", 200) pd.set_option(\"display.max_rows\", 300) In\u00a0[2]: Copied! <pre>df = load_ecoli(as_frame=True)\ndf.head()\n</pre> df = load_ecoli(as_frame=True) df.head() Out[2]: SequenceName mcg gvh lip chg aac alm1 alm2 class 0 AAT_ECOLI 0.49 0.29 0.48 0.5 0.56 0.24 0.35 cp 1 ACEA_ECOLI 0.07 0.40 0.48 0.5 0.54 0.35 0.44 cp 2 ACEK_ECOLI 0.56 0.40 0.48 0.5 0.49 0.37 0.46 cp 3 ACKA_ECOLI 0.59 0.49 0.48 0.5 0.52 0.45 0.36 cp 4 ADI_ECOLI 0.23 0.32 0.48 0.5 0.55 0.25 0.35 cp In\u00a0[3]: Copied! <pre># imbalance of the classes\ndf[\"class\"].value_counts()\n</pre> # imbalance of the classes df[\"class\"].value_counts() Out[3]: <pre>class\ncp     143\nim      77\npp      52\nimU     35\nom      20\nomL      5\nimS      2\nimL      2\nName: count, dtype: int64</pre> In\u00a0[4]: Copied! <pre>df = df.loc[~df[\"class\"].isin([\"omL\", \"imS\", \"imL\"])]\ndf.reset_index(inplace=True, drop=True)\n</pre> df = df.loc[~df[\"class\"].isin([\"omL\", \"imS\", \"imL\"])] df.reset_index(inplace=True, drop=True) In\u00a0[5]: Copied! <pre>encoder = LabelEncoder([\"class\"])\ndf_enc = encoder.fit_transform(df)\ndf_enc[\"class\"] = df_enc[\"class\"] - 1\n</pre> encoder = LabelEncoder([\"class\"]) df_enc = encoder.fit_transform(df) df_enc[\"class\"] = df_enc[\"class\"] - 1 In\u00a0[6]: Copied! <pre># drop columns we won't need in this example\ndf_enc = df_enc.drop(columns=[\"SequenceName\"])\n</pre> # drop columns we won't need in this example df_enc = df_enc.drop(columns=[\"SequenceName\"]) In\u00a0[7]: Copied! <pre>df_train, df_valid = train_test_split(\n    df_enc, test_size=0.2, stratify=df_enc[\"class\"], random_state=1\n)\ndf_valid, df_test = train_test_split(\n    df_valid, test_size=0.5, stratify=df_valid[\"class\"], random_state=1\n)\n</pre> df_train, df_valid = train_test_split(     df_enc, test_size=0.2, stratify=df_enc[\"class\"], random_state=1 ) df_valid, df_test = train_test_split(     df_valid, test_size=0.5, stratify=df_valid[\"class\"], random_state=1 ) In\u00a0[8]: Copied! <pre>continuous_cols = df_enc.drop(columns=[\"class\"]).columns.values.tolist()\n</pre> continuous_cols = df_enc.drop(columns=[\"class\"]).columns.values.tolist() In\u00a0[9]: Copied! <pre># deeptabular\ntab_preprocessor = TabPreprocessor(continuous_cols=continuous_cols, scale=True)\nX_tab_train = tab_preprocessor.fit_transform(df_train)\nX_tab_valid = tab_preprocessor.transform(df_valid)\nX_tab_test = tab_preprocessor.transform(df_test)\n\n# target\ny_train = df_train[\"class\"].values\ny_valid = df_valid[\"class\"].values\ny_test = df_test[\"class\"].values\n\nX_train = {\"X_tab\": X_tab_train, \"target\": y_train}\nX_val = {\"X_tab\": X_tab_valid, \"target\": y_valid}\n</pre> # deeptabular tab_preprocessor = TabPreprocessor(continuous_cols=continuous_cols, scale=True) X_tab_train = tab_preprocessor.fit_transform(df_train) X_tab_valid = tab_preprocessor.transform(df_valid) X_tab_test = tab_preprocessor.transform(df_test)  # target y_train = df_train[\"class\"].values y_valid = df_valid[\"class\"].values y_test = df_test[\"class\"].values  X_train = {\"X_tab\": X_tab_train, \"target\": y_train} X_val = {\"X_tab\": X_tab_valid, \"target\": y_valid} <pre>/Users/javierrodriguezzaurin/Projects/pytorch-widedeep/pytorch_widedeep/preprocessing/tab_preprocessor.py:295: DeprecationWarning: 'scale' and 'already_standard' will be deprecated in the next release. Please use 'cols_to_scale' instead\n  self._check_inputs(cat_embed_cols)\n</pre> In\u00a0[10]: Copied! <pre>deeptabular = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    continuous_cols=tab_preprocessor.continuous_cols,\n)\nmodel = WideDeep(deeptabular=deeptabular, pred_dim=df_enc[\"class\"].nunique())\nmodel\n</pre> deeptabular = TabMlp(     column_idx=tab_preprocessor.column_idx,     continuous_cols=tab_preprocessor.continuous_cols, ) model = WideDeep(deeptabular=deeptabular, pred_dim=df_enc[\"class\"].nunique()) model Out[10]: <pre>WideDeep(\n  (deeptabular): Sequential(\n    (0): TabMlp(\n      (cont_norm): Identity()\n      (encoder): MLP(\n        (mlp): Sequential(\n          (dense_layer_0): Sequential(\n            (0): Linear(in_features=7, out_features=200, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_1): Sequential(\n            (0): Linear(in_features=200, out_features=100, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=100, out_features=5, bias=True)\n  )\n)</pre> In\u00a0[11]: Copied! <pre>auroc = AUROC(num_classes=df_enc[\"class\"].nunique(), task=\"multiclass\")\n</pre> auroc = AUROC(num_classes=df_enc[\"class\"].nunique(), task=\"multiclass\") In\u00a0[12]: Copied! <pre># Optimizers\ndeep_opt = SGD(model.deeptabular.parameters(), lr=0.1)\n# LR Scheduler\ndeep_sch = lr_scheduler.StepLR(deep_opt, step_size=3)\n# Hyperparameters\ntrainer = Trainer(\n    model,\n    objective=\"multiclass_focal_loss\",\n    lr_schedulers={\"deeptabular\": deep_sch},\n    initializers={\"deeptabular\": XavierNormal},\n    optimizers={\"deeptabular\": deep_opt},\n    metrics=[auroc],\n)\n\ntrainer.fit(X_train=X_train, X_val=X_val, n_epochs=5, batch_size=50)\n</pre> # Optimizers deep_opt = SGD(model.deeptabular.parameters(), lr=0.1) # LR Scheduler deep_sch = lr_scheduler.StepLR(deep_opt, step_size=3) # Hyperparameters trainer = Trainer(     model,     objective=\"multiclass_focal_loss\",     lr_schedulers={\"deeptabular\": deep_sch},     initializers={\"deeptabular\": XavierNormal},     optimizers={\"deeptabular\": deep_opt},     metrics=[auroc], )  trainer.fit(X_train=X_train, X_val=X_val, n_epochs=5, batch_size=50) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 6/6 [00:00&lt;00:00, 54.59it/s, loss=0.109, metrics={'MulticlassAUROC': 0.314}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 1/1 [00:00&lt;00:00, 98.35it/s, loss=0.105, metrics={'MulticlassAUROC': 0.2558}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 6/6 [00:00&lt;00:00, 91.55it/s, loss=0.105, metrics={'MulticlassAUROC': 0.3546}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 1/1 [00:00&lt;00:00, 111.68it/s, loss=0.101, metrics={'MulticlassAUROC': 0.2737}]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 6/6 [00:00&lt;00:00, 62.55it/s, loss=0.1, metrics={'MulticlassAUROC': 0.3795}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 1/1 [00:00&lt;00:00, 108.51it/s, loss=0.0966, metrics={'MulticlassAUROC': 0.3053}]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 6/6 [00:00&lt;00:00, 99.35it/s, loss=0.0965, metrics={'MulticlassAUROC': 0.3809}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 1/1 [00:00&lt;00:00, 117.73it/s, loss=0.0962, metrics={'MulticlassAUROC': 0.3089}]\nepoch 5: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 6/6 [00:00&lt;00:00, 110.56it/s, loss=0.0967, metrics={'MulticlassAUROC': 0.3509}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 1/1 [00:00&lt;00:00, 127.35it/s, loss=0.0958, metrics={'MulticlassAUROC': 0.3089}]\n</pre> In\u00a0[\u00a0]: Copied! <pre>\n</pre>"},{"location":"examples/11_auc_multiclass.html#auc-multiclass-computation","title":"AUC multiclass computation\u00b6","text":""},{"location":"examples/11_auc_multiclass.html#initial-imports","title":"Initial imports\u00b6","text":""},{"location":"examples/11_auc_multiclass.html#preparing-the-data","title":"Preparing the data\u00b6","text":""},{"location":"examples/11_auc_multiclass.html#define-the-model","title":"Define the model\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html","title":"12_ZILNLoss_origkeras_vs_pytorch_widedeep","text":"In\u00a0[1]: Copied! <pre># @title Copyright 2019 The Lifetime Value Authors.\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     https://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n# ============================================================================\n</pre> # @title Copyright 2019 The Lifetime Value Authors. # Licensed under the Apache License, Version 2.0 (the \"License\"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # #     https://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an \"AS IS\" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ Run in Google Colab View source on GitHub In\u00a0[3]: Copied! <pre>import os\n\nimport numpy as np\nimport pandas as pd\nfrom scipy import stats\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport tensorflow as tf\nimport tensorflow_probability as tfp\nfrom typing import Sequence\n\n# install and import ltv\n!pip install -q git+https://github.com/google/lifetime_value\nimport lifetime_value as ltv\n</pre> import os  import numpy as np import pandas as pd from scipy import stats import matplotlib.pyplot as plt import seaborn as sns import tensorflow as tf import tensorflow_probability as tfp from typing import Sequence  # install and import ltv !pip install -q git+https://github.com/google/lifetime_value import lifetime_value as ltv In\u00a0[\u00a0]: Copied! <pre>tfd = tfp.distributions\n%config InlineBackend.figure_format='retina'\nsns.set_style(\"whitegrid\")\n</pre> tfd = tfp.distributions %config InlineBackend.figure_format='retina' sns.set_style(\"whitegrid\") In\u00a0[\u00a0]: Copied! <pre>MODEL = \"dnn\"\nLOSS = \"ziln\"  # @param { isTemplate: true, type: 'string'} ['mse', 'ziln']\nLEARNING_RATE = 0.001  # @param { isTemplate: true}\nVERSION = 0  # @param { isTemplate: true, type: 'integer'}\nOUTPUT_CSV_FOLDER = \"/tmp/lifetime-value/kdd_cup_98/result\"  # @param { isTemplate: true, type: 'string'}\n</pre> MODEL = \"dnn\" LOSS = \"ziln\"  # @param { isTemplate: true, type: 'string'} ['mse', 'ziln'] LEARNING_RATE = 0.001  # @param { isTemplate: true} VERSION = 0  # @param { isTemplate: true, type: 'integer'} OUTPUT_CSV_FOLDER = \"/tmp/lifetime-value/kdd_cup_98/result\"  # @param { isTemplate: true, type: 'string'} <p>Download kdd_cup_98 data to /tmp/lifetime-value/kdd_cup_98</p> In\u00a0[\u00a0]: Copied! <pre>%%bash\nmkdir -p /tmp/lifetime-value/kdd_cup_98\nwget https://kdd.ics.uci.edu/databases/kddcup98/epsilon_mirror/cup98lrn.zip -P /tmp/lifetime-value/kdd_cup_98/\nwget https://kdd.ics.uci.edu/databases/kddcup98/epsilon_mirror/cup98val.zip -P /tmp/lifetime-value/kdd_cup_98/\nwget https://kdd.ics.uci.edu/databases/kddcup98/epsilon_mirror/valtargt.txt -P /tmp/lifetime-value/kdd_cup_98/\ncd /tmp/lifetime-value/kdd_cup_98/\nunzip cup98lrn.zip\nunzip cup98val.zip\n</pre> %%bash mkdir -p /tmp/lifetime-value/kdd_cup_98 wget https://kdd.ics.uci.edu/databases/kddcup98/epsilon_mirror/cup98lrn.zip -P /tmp/lifetime-value/kdd_cup_98/ wget https://kdd.ics.uci.edu/databases/kddcup98/epsilon_mirror/cup98val.zip -P /tmp/lifetime-value/kdd_cup_98/ wget https://kdd.ics.uci.edu/databases/kddcup98/epsilon_mirror/valtargt.txt -P /tmp/lifetime-value/kdd_cup_98/ cd /tmp/lifetime-value/kdd_cup_98/ unzip cup98lrn.zip unzip cup98val.zip In\u00a0[\u00a0]: Copied! <pre>df_train = pd.read_csv(\"/tmp/lifetime-value/kdd_cup_98/cup98LRN.txt\")\nnum_train = df_train.shape[0]\ndf_eval = pd.read_csv(\"/tmp/lifetime-value/kdd_cup_98/cup98VAL.txt\")\ndf_eval_target = pd.read_csv(\"/tmp/lifetime-value/kdd_cup_98/valtargt.txt\")\ndf_eval = df_eval.merge(df_eval_target, on=\"CONTROLN\")\n</pre> df_train = pd.read_csv(\"/tmp/lifetime-value/kdd_cup_98/cup98LRN.txt\") num_train = df_train.shape[0] df_eval = pd.read_csv(\"/tmp/lifetime-value/kdd_cup_98/cup98VAL.txt\") df_eval_target = pd.read_csv(\"/tmp/lifetime-value/kdd_cup_98/valtargt.txt\") df_eval = df_eval.merge(df_eval_target, on=\"CONTROLN\") In\u00a0[\u00a0]: Copied! <pre>df = pd.concat([df_train, df_eval], axis=0, sort=True)\n</pre> df = pd.concat([df_train, df_eval], axis=0, sort=True) In\u00a0[\u00a0]: Copied! <pre>y = df[\"TARGET_D\"][:num_train]\n</pre> y = df[\"TARGET_D\"][:num_train] In\u00a0[\u00a0]: Copied! <pre>def plot_hist_log_scale(y):\n    max_val = y.max() + 1.0\n    ax = pd.Series(y).hist(\n        figsize=(8, 5), bins=10 ** np.linspace(0.0, np.log10(max_val), 20)\n    )\n\n    plt.xlabel(\"Donation ($)\")\n    plt.ylabel(\"Count\")\n    # plt.title('Histogram of LTV')\n    plt.xticks(rotation=\"horizontal\")\n    plt.legend(loc=\"upper left\")\n    ax.set_xscale(\"log\")\n    ax.grid(False)\n    # Hide the right and top spines\n    ax.spines[\"right\"].set_visible(False)\n    ax.spines[\"top\"].set_visible(False)\n    # Only show ticks on the left and bottom spines\n    ax.yaxis.set_ticks_position(\"left\")\n    ax.xaxis.set_ticks_position(\"bottom\")\n    plt.show()\n\n    fig = ax.get_figure()\n    output_file = tf.io.gfile.GFile(\n        \"/tmp/lifetime-value/kdd_cup_98/histogram_kdd98_log_scale.pdf\", \"wb\"\n    )\n    fig.savefig(output_file, bbox_inches=\"tight\", format=\"pdf\")\n</pre> def plot_hist_log_scale(y):     max_val = y.max() + 1.0     ax = pd.Series(y).hist(         figsize=(8, 5), bins=10 ** np.linspace(0.0, np.log10(max_val), 20)     )      plt.xlabel(\"Donation ($)\")     plt.ylabel(\"Count\")     # plt.title('Histogram of LTV')     plt.xticks(rotation=\"horizontal\")     plt.legend(loc=\"upper left\")     ax.set_xscale(\"log\")     ax.grid(False)     # Hide the right and top spines     ax.spines[\"right\"].set_visible(False)     ax.spines[\"top\"].set_visible(False)     # Only show ticks on the left and bottom spines     ax.yaxis.set_ticks_position(\"left\")     ax.xaxis.set_ticks_position(\"bottom\")     plt.show()      fig = ax.get_figure()     output_file = tf.io.gfile.GFile(         \"/tmp/lifetime-value/kdd_cup_98/histogram_kdd98_log_scale.pdf\", \"wb\"     )     fig.savefig(output_file, bbox_inches=\"tight\", format=\"pdf\") In\u00a0[\u00a0]: Copied! <pre>plot_hist_log_scale(y[y &gt; 0])\n</pre> plot_hist_log_scale(y[y &gt; 0]) In\u00a0[\u00a0]: Copied! <pre>VOCAB_FEATURES = [\n    \"ODATEDW\",  # date of donor's first gift (YYMM)\n    \"OSOURCE\",  # donor acquisition mailing list\n    \"TCODE\",  # donor title code\n    \"STATE\",\n    \"ZIP\",\n    \"DOMAIN\",  # urbanicity level and socio-economic status of the neighborhood\n    \"CLUSTER\",  # socio-economic status\n    \"GENDER\",\n    \"MAXADATE\",  # date of the most recent promotion received\n    \"MINRDATE\",\n    \"LASTDATE\",\n    \"FISTDATE\",\n    \"RFA_2A\",\n]\n</pre> VOCAB_FEATURES = [     \"ODATEDW\",  # date of donor's first gift (YYMM)     \"OSOURCE\",  # donor acquisition mailing list     \"TCODE\",  # donor title code     \"STATE\",     \"ZIP\",     \"DOMAIN\",  # urbanicity level and socio-economic status of the neighborhood     \"CLUSTER\",  # socio-economic status     \"GENDER\",     \"MAXADATE\",  # date of the most recent promotion received     \"MINRDATE\",     \"LASTDATE\",     \"FISTDATE\",     \"RFA_2A\", ] In\u00a0[\u00a0]: Copied! <pre>df[\"ODATEDW\"] = df[\"ODATEDW\"].astype(\"str\")\ndf[\"TCODE\"] = df[\"TCODE\"].apply(lambda x: \"{:03d}\".format(x // 1000 if x &gt; 1000 else x))\ndf[\"ZIP\"] = df[\"ZIP\"].str.slice(0, 5)\ndf[\"MAXADATE\"] = df[\"MAXADATE\"].astype(\"str\")\ndf[\"MINRDATE\"] = df[\"MINRDATE\"].astype(\"str\")\ndf[\"LASTDATE\"] = df[\"LASTDATE\"].astype(\"str\")\ndf[\"FISTDATE\"] = df[\"FISTDATE\"].astype(\"str\")\n</pre> df[\"ODATEDW\"] = df[\"ODATEDW\"].astype(\"str\") df[\"TCODE\"] = df[\"TCODE\"].apply(lambda x: \"{:03d}\".format(x // 1000 if x &gt; 1000 else x)) df[\"ZIP\"] = df[\"ZIP\"].str.slice(0, 5) df[\"MAXADATE\"] = df[\"MAXADATE\"].astype(\"str\") df[\"MINRDATE\"] = df[\"MINRDATE\"].astype(\"str\") df[\"LASTDATE\"] = df[\"LASTDATE\"].astype(\"str\") df[\"FISTDATE\"] = df[\"FISTDATE\"].astype(\"str\") In\u00a0[\u00a0]: Copied! <pre>def label_encoding(y, frequency_threshold=100):\n    value_counts = pd.value_counts(y)\n    categories = value_counts[value_counts &gt;= frequency_threshold].index.to_numpy()\n    # 0 indicates the unknown category.\n    return pd.Categorical(y, categories=categories).codes + 1\n</pre> def label_encoding(y, frequency_threshold=100):     value_counts = pd.value_counts(y)     categories = value_counts[value_counts &gt;= frequency_threshold].index.to_numpy()     # 0 indicates the unknown category.     return pd.Categorical(y, categories=categories).codes + 1 In\u00a0[\u00a0]: Copied! <pre>for key in VOCAB_FEATURES:\n    df[key] = label_encoding(df[key])\n</pre> for key in VOCAB_FEATURES:     df[key] = label_encoding(df[key]) In\u00a0[\u00a0]: Copied! <pre>MAIL_ORDER_RESPONSES = [\n    \"MBCRAFT\",\n    \"MBGARDEN\",\n    \"MBBOOKS\",\n    \"MBCOLECT\",\n    \"MAGFAML\",\n    \"MAGFEM\",\n    \"MAGMALE\",\n    \"PUBGARDN\",\n    \"PUBCULIN\",\n    \"PUBHLTH\",\n    \"PUBDOITY\",\n    \"PUBNEWFN\",\n    \"PUBPHOTO\",\n    \"PUBOPP\",\n    \"RFA_2F\",\n]\n</pre> MAIL_ORDER_RESPONSES = [     \"MBCRAFT\",     \"MBGARDEN\",     \"MBBOOKS\",     \"MBCOLECT\",     \"MAGFAML\",     \"MAGFEM\",     \"MAGMALE\",     \"PUBGARDN\",     \"PUBCULIN\",     \"PUBHLTH\",     \"PUBDOITY\",     \"PUBNEWFN\",     \"PUBPHOTO\",     \"PUBOPP\",     \"RFA_2F\", ] In\u00a0[\u00a0]: Copied! <pre>INDICATOR_FEATURES = [\n    \"AGE\",  # age decile, 0 indicates unknown\n    \"NUMCHLD\",\n    \"INCOME\",\n    \"WEALTH1\",\n    \"HIT\",\n] + MAIL_ORDER_RESPONSES\n</pre> INDICATOR_FEATURES = [     \"AGE\",  # age decile, 0 indicates unknown     \"NUMCHLD\",     \"INCOME\",     \"WEALTH1\",     \"HIT\", ] + MAIL_ORDER_RESPONSES In\u00a0[\u00a0]: Copied! <pre>df[\"AGE\"] = pd.qcut(df[\"AGE\"].values, 10).codes + 1\ndf[\"NUMCHLD\"] = df[\"NUMCHLD\"].apply(lambda x: 0 if np.isnan(x) else int(x))\ndf[\"INCOME\"] = df[\"INCOME\"].apply(lambda x: 0 if np.isnan(x) else int(x))\ndf[\"WEALTH1\"] = df[\"WEALTH1\"].apply(lambda x: 0 if np.isnan(x) else int(x) + 1)\ndf[\"HIT\"] = pd.qcut(df[\"HIT\"].values, q=50, duplicates=\"drop\").codes\n\nfor col in MAIL_ORDER_RESPONSES:\n    df[col] = pd.qcut(df[col].values, q=20, duplicates=\"drop\").codes + 1\n</pre> df[\"AGE\"] = pd.qcut(df[\"AGE\"].values, 10).codes + 1 df[\"NUMCHLD\"] = df[\"NUMCHLD\"].apply(lambda x: 0 if np.isnan(x) else int(x)) df[\"INCOME\"] = df[\"INCOME\"].apply(lambda x: 0 if np.isnan(x) else int(x)) df[\"WEALTH1\"] = df[\"WEALTH1\"].apply(lambda x: 0 if np.isnan(x) else int(x) + 1) df[\"HIT\"] = pd.qcut(df[\"HIT\"].values, q=50, duplicates=\"drop\").codes  for col in MAIL_ORDER_RESPONSES:     df[col] = pd.qcut(df[col].values, q=20, duplicates=\"drop\").codes + 1 In\u00a0[\u00a0]: Copied! <pre>NUMERIC_FEATURES = [\n    # binary\n    \"MAILCODE\",  # bad address\n    \"NOEXCH\",  # do not exchange\n    \"RECINHSE\",  # donor has given to PVA's in house program\n    \"RECP3\",  # donor has given to PVA's P3 program\n    \"RECPGVG\",  # planned giving record\n    \"RECSWEEP\",  # sweepstakes record\n    \"HOMEOWNR\",  # home owner\n    \"CHILD03\",\n    \"CHILD07\",\n    \"CHILD12\",\n    \"CHILD18\",\n    # continuous\n    \"CARDPROM\",\n    \"NUMPROM\",\n    \"CARDPM12\",\n    \"NUMPRM12\",\n    \"RAMNTALL\",\n    \"NGIFTALL\",\n    \"MINRAMNT\",\n    \"MAXRAMNT\",\n    \"LASTGIFT\",\n    \"AVGGIFT\",\n]\n</pre> NUMERIC_FEATURES = [     # binary     \"MAILCODE\",  # bad address     \"NOEXCH\",  # do not exchange     \"RECINHSE\",  # donor has given to PVA's in house program     \"RECP3\",  # donor has given to PVA's P3 program     \"RECPGVG\",  # planned giving record     \"RECSWEEP\",  # sweepstakes record     \"HOMEOWNR\",  # home owner     \"CHILD03\",     \"CHILD07\",     \"CHILD12\",     \"CHILD18\",     # continuous     \"CARDPROM\",     \"NUMPROM\",     \"CARDPM12\",     \"NUMPRM12\",     \"RAMNTALL\",     \"NGIFTALL\",     \"MINRAMNT\",     \"MAXRAMNT\",     \"LASTGIFT\",     \"AVGGIFT\", ] In\u00a0[\u00a0]: Copied! <pre>df[\"MAILCODE\"] = (df[\"MAILCODE\"] == \"B\").astype(\"float32\")\ndf[\"PVASTATE\"] = df[\"PVASTATE\"].isin([\"P\", \"E\"]).astype(\"float32\")\ndf[\"NOEXCH\"] = df[\"NOEXCH\"].isin([\"X\", \"1\"]).astype(\"float32\")\ndf[\"RECINHSE\"] = (df[\"RECINHSE\"] == \"X\").astype(\"float32\")\ndf[\"RECP3\"] = (df[\"RECP3\"] == \"X\").astype(\"float32\")\ndf[\"RECPGVG\"] = (df[\"RECPGVG\"] == \"X\").astype(\"float32\")\ndf[\"RECSWEEP\"] = (df[\"RECSWEEP\"] == \"X\").astype(\"float32\")\ndf[\"HOMEOWNR\"] = (df[\"HOMEOWNR\"] == \"H\").astype(\"float32\")\ndf[\"CHILD03\"] = df[\"CHILD03\"].isin([\"M\", \"F\", \"B\"]).astype(\"float32\")\ndf[\"CHILD07\"] = df[\"CHILD07\"].isin([\"M\", \"F\", \"B\"]).astype(\"float32\")\ndf[\"CHILD12\"] = df[\"CHILD12\"].isin([\"M\", \"F\", \"B\"]).astype(\"float32\")\ndf[\"CHILD18\"] = df[\"CHILD18\"].isin([\"M\", \"F\", \"B\"]).astype(\"float32\")\n\ndf[\"CARDPROM\"] = df[\"CARDPROM\"] / 100\ndf[\"NUMPROM\"] = df[\"NUMPROM\"] / 100\ndf[\"CARDPM12\"] = df[\"CARDPM12\"] / 100\ndf[\"NUMPRM12\"] = df[\"NUMPRM12\"] / 100\ndf[\"RAMNTALL\"] = np.log1p(df[\"RAMNTALL\"])\ndf[\"NGIFTALL\"] = np.log1p(df[\"NGIFTALL\"])\ndf[\"MINRAMNT\"] = np.log1p(df[\"MINRAMNT\"])\ndf[\"MAXRAMNT\"] = np.log1p(df[\"MAXRAMNT\"])\ndf[\"LASTGIFT\"] = np.log1p(df[\"LASTGIFT\"])\ndf[\"AVGGIFT\"] = np.log1p(df[\"AVGGIFT\"])\n</pre> df[\"MAILCODE\"] = (df[\"MAILCODE\"] == \"B\").astype(\"float32\") df[\"PVASTATE\"] = df[\"PVASTATE\"].isin([\"P\", \"E\"]).astype(\"float32\") df[\"NOEXCH\"] = df[\"NOEXCH\"].isin([\"X\", \"1\"]).astype(\"float32\") df[\"RECINHSE\"] = (df[\"RECINHSE\"] == \"X\").astype(\"float32\") df[\"RECP3\"] = (df[\"RECP3\"] == \"X\").astype(\"float32\") df[\"RECPGVG\"] = (df[\"RECPGVG\"] == \"X\").astype(\"float32\") df[\"RECSWEEP\"] = (df[\"RECSWEEP\"] == \"X\").astype(\"float32\") df[\"HOMEOWNR\"] = (df[\"HOMEOWNR\"] == \"H\").astype(\"float32\") df[\"CHILD03\"] = df[\"CHILD03\"].isin([\"M\", \"F\", \"B\"]).astype(\"float32\") df[\"CHILD07\"] = df[\"CHILD07\"].isin([\"M\", \"F\", \"B\"]).astype(\"float32\") df[\"CHILD12\"] = df[\"CHILD12\"].isin([\"M\", \"F\", \"B\"]).astype(\"float32\") df[\"CHILD18\"] = df[\"CHILD18\"].isin([\"M\", \"F\", \"B\"]).astype(\"float32\")  df[\"CARDPROM\"] = df[\"CARDPROM\"] / 100 df[\"NUMPROM\"] = df[\"NUMPROM\"] / 100 df[\"CARDPM12\"] = df[\"CARDPM12\"] / 100 df[\"NUMPRM12\"] = df[\"NUMPRM12\"] / 100 df[\"RAMNTALL\"] = np.log1p(df[\"RAMNTALL\"]) df[\"NGIFTALL\"] = np.log1p(df[\"NGIFTALL\"]) df[\"MINRAMNT\"] = np.log1p(df[\"MINRAMNT\"]) df[\"MAXRAMNT\"] = np.log1p(df[\"MAXRAMNT\"]) df[\"LASTGIFT\"] = np.log1p(df[\"LASTGIFT\"]) df[\"AVGGIFT\"] = np.log1p(df[\"AVGGIFT\"]) In\u00a0[\u00a0]: Copied! <pre>CATEGORICAL_FEATURES = VOCAB_FEATURES + INDICATOR_FEATURES\nALL_FEATURES = CATEGORICAL_FEATURES + NUMERIC_FEATURES\n</pre> CATEGORICAL_FEATURES = VOCAB_FEATURES + INDICATOR_FEATURES ALL_FEATURES = CATEGORICAL_FEATURES + NUMERIC_FEATURES In\u00a0[\u00a0]: Copied! <pre>def dnn_split(df):\n    df_train = df.iloc[:num_train]\n    df_eval = df.iloc[num_train:]\n\n    def feature_dict(df):\n        features = {k: v.values for k, v in dict(df[CATEGORICAL_FEATURES]).items()}\n        features[\"numeric\"] = df[NUMERIC_FEATURES].astype(\"float32\").values\n        return features\n\n    x_train, y_train = (\n        feature_dict(df_train),\n        df_train[\"TARGET_D\"].astype(\"float32\").values,\n    )\n    x_eval, y_eval = feature_dict(df_eval), df_eval[\"TARGET_D\"].astype(\"float32\").values\n\n    return x_train, x_eval, y_train, y_eval\n</pre> def dnn_split(df):     df_train = df.iloc[:num_train]     df_eval = df.iloc[num_train:]      def feature_dict(df):         features = {k: v.values for k, v in dict(df[CATEGORICAL_FEATURES]).items()}         features[\"numeric\"] = df[NUMERIC_FEATURES].astype(\"float32\").values         return features      x_train, y_train = (         feature_dict(df_train),         df_train[\"TARGET_D\"].astype(\"float32\").values,     )     x_eval, y_eval = feature_dict(df_eval), df_eval[\"TARGET_D\"].astype(\"float32\").values      return x_train, x_eval, y_train, y_eval In\u00a0[\u00a0]: Copied! <pre>def embedding_dim(x):\n    return int(x**0.25) + 1\n\n\ndef embedding_layer(vocab_size):\n    return tf.keras.Sequential(\n        [\n            tf.keras.layers.Embedding(\n                input_dim=vocab_size,\n                output_dim=embedding_dim(vocab_size),\n                input_length=1,\n            ),\n            tf.keras.layers.Flatten(),\n        ]\n    )\n\n\ndef dnn_model(output_units):\n    numeric_input = tf.keras.layers.Input(\n        shape=(len(NUMERIC_FEATURES),), name=\"numeric\"\n    )\n\n    embedding_inputs = [\n        tf.keras.layers.Input(shape=(1,), name=key, dtype=np.int64)\n        for key in CATEGORICAL_FEATURES\n    ]\n\n    embedding_outputs = [\n        embedding_layer(vocab_size=df[key].max() + 1)(input)\n        for key, input in zip(CATEGORICAL_FEATURES, embedding_inputs)\n    ]\n\n    deep_input = tf.keras.layers.concatenate([numeric_input] + embedding_outputs)\n    deep_model = tf.keras.Sequential(\n        [\n            tf.keras.layers.Dense(128, activation=\"relu\"),\n            tf.keras.layers.Dense(128, activation=\"relu\"),\n            tf.keras.layers.Dense(64, activation=\"relu\"),\n            tf.keras.layers.Dense(64, activation=\"relu\"),\n            tf.keras.layers.Dense(units=output_units),\n        ]\n    )\n    return tf.keras.Model(\n        inputs=[numeric_input] + embedding_inputs, outputs=deep_model(deep_input)\n    )\n</pre> def embedding_dim(x):     return int(x**0.25) + 1   def embedding_layer(vocab_size):     return tf.keras.Sequential(         [             tf.keras.layers.Embedding(                 input_dim=vocab_size,                 output_dim=embedding_dim(vocab_size),                 input_length=1,             ),             tf.keras.layers.Flatten(),         ]     )   def dnn_model(output_units):     numeric_input = tf.keras.layers.Input(         shape=(len(NUMERIC_FEATURES),), name=\"numeric\"     )      embedding_inputs = [         tf.keras.layers.Input(shape=(1,), name=key, dtype=np.int64)         for key in CATEGORICAL_FEATURES     ]      embedding_outputs = [         embedding_layer(vocab_size=df[key].max() + 1)(input)         for key, input in zip(CATEGORICAL_FEATURES, embedding_inputs)     ]      deep_input = tf.keras.layers.concatenate([numeric_input] + embedding_outputs)     deep_model = tf.keras.Sequential(         [             tf.keras.layers.Dense(128, activation=\"relu\"),             tf.keras.layers.Dense(128, activation=\"relu\"),             tf.keras.layers.Dense(64, activation=\"relu\"),             tf.keras.layers.Dense(64, activation=\"relu\"),             tf.keras.layers.Dense(units=output_units),         ]     )     return tf.keras.Model(         inputs=[numeric_input] + embedding_inputs, outputs=deep_model(deep_input)     ) In\u00a0[\u00a0]: Copied! <pre>if LOSS == \"mse\":\n    loss = tf.keras.losses.MeanSquaredError()\n    output_units = 1\n\nif LOSS == \"ziln\":\n    loss = ltv.zero_inflated_lognormal_loss\n    output_units = 3\n</pre> if LOSS == \"mse\":     loss = tf.keras.losses.MeanSquaredError()     output_units = 1  if LOSS == \"ziln\":     loss = ltv.zero_inflated_lognormal_loss     output_units = 3 In\u00a0[\u00a0]: Copied! <pre>x_train, x_eval, y_train, y_eval = dnn_split(df)\nmodel = dnn_model(output_units)\n</pre> x_train, x_eval, y_train, y_eval = dnn_split(df) model = dnn_model(output_units) In\u00a0[\u00a0]: Copied! <pre>model.compile(optimizer=tf.keras.optimizers.Nadam(lr=LEARNING_RATE), loss=loss)\n</pre> model.compile(optimizer=tf.keras.optimizers.Nadam(lr=LEARNING_RATE), loss=loss) In\u00a0[\u00a0]: Copied! <pre>callbacks = [\n    tf.keras.callbacks.ReduceLROnPlateau(monitor=\"val_loss\", min_lr=1e-6),\n    tf.keras.callbacks.EarlyStopping(monitor=\"val_loss\", patience=10),\n]\n</pre> callbacks = [     tf.keras.callbacks.ReduceLROnPlateau(monitor=\"val_loss\", min_lr=1e-6),     tf.keras.callbacks.EarlyStopping(monitor=\"val_loss\", patience=10), ] In\u00a0[\u00a0]: Copied! <pre>history = model.fit(\n    x=x_train,\n    y=y_train,\n    batch_size=2048,\n    epochs=200,\n    verbose=2,\n    callbacks=callbacks,\n    validation_data=(x_eval, y_eval),\n).history\n</pre> history = model.fit(     x=x_train,     y=y_train,     batch_size=2048,     epochs=200,     verbose=2,     callbacks=callbacks,     validation_data=(x_eval, y_eval), ).history In\u00a0[\u00a0]: Copied! <pre>pd.DataFrame(history)[[\"loss\", \"val_loss\"]].plot();\n</pre> pd.DataFrame(history)[[\"loss\", \"val_loss\"]].plot(); In\u00a0[\u00a0]: Copied! <pre>if LOSS == \"mse\":\n    y_pred = model.predict(x=x_eval, batch_size=1024).flatten()\n\nif LOSS == \"ziln\":\n    logits = model.predict(x=x_eval, batch_size=1024)\n    y_pred = ltv.zero_inflated_lognormal_pred(logits).numpy().flatten()\n</pre> if LOSS == \"mse\":     y_pred = model.predict(x=x_eval, batch_size=1024).flatten()  if LOSS == \"ziln\":     logits = model.predict(x=x_eval, batch_size=1024)     y_pred = ltv.zero_inflated_lognormal_pred(logits).numpy().flatten() In\u00a0[\u00a0]: Copied! <pre>from pytorch_widedeep.preprocessing import TabPreprocessor\nfrom pytorch_widedeep.training import Trainer\nfrom pytorch_widedeep.models import TabMlp, WideDeep\nfrom torch.optim.lr_scheduler import ReduceLROnPlateau\nfrom pytorch_widedeep.callbacks import EarlyStopping\nfrom torch.optim import NAdam\n</pre> from pytorch_widedeep.preprocessing import TabPreprocessor from pytorch_widedeep.training import Trainer from pytorch_widedeep.models import TabMlp, WideDeep from torch.optim.lr_scheduler import ReduceLROnPlateau from pytorch_widedeep.callbacks import EarlyStopping from torch.optim import NAdam In\u00a0[\u00a0]: Copied! <pre># CATEGORICAL_FEATURES\nNUMERICAL_FEATURES = [\"num\" + str(i) for i in range(21)]\nx_train_pyt_num = pd.DataFrame(columns=NUMERICAL_FEATURES, data=x_train[\"numeric\"])\nx_train_pyt_cat = pd.DataFrame(\n    {key: value for key, value in x_train.items() if key not in [\"numeric\"]}\n)\n\nx_eval_pyt_num = pd.DataFrame(columns=NUMERICAL_FEATURES, data=x_eval[\"numeric\"])\nx_eval_pyt_cat = pd.DataFrame(\n    {key: value for key, value in x_eval.items() if key not in [\"numeric\"]}\n)\n</pre> # CATEGORICAL_FEATURES NUMERICAL_FEATURES = [\"num\" + str(i) for i in range(21)] x_train_pyt_num = pd.DataFrame(columns=NUMERICAL_FEATURES, data=x_train[\"numeric\"]) x_train_pyt_cat = pd.DataFrame(     {key: value for key, value in x_train.items() if key not in [\"numeric\"]} )  x_eval_pyt_num = pd.DataFrame(columns=NUMERICAL_FEATURES, data=x_eval[\"numeric\"]) x_eval_pyt_cat = pd.DataFrame(     {key: value for key, value in x_eval.items() if key not in [\"numeric\"]} ) In\u00a0[\u00a0]: Copied! <pre>x_train_pyt = pd.concat([x_train_pyt_num, x_train_pyt_cat], axis=1)\nx_eval_pyt = pd.concat([x_eval_pyt_num, x_eval_pyt_cat], axis=1)\n</pre> x_train_pyt = pd.concat([x_train_pyt_num, x_train_pyt_cat], axis=1) x_eval_pyt = pd.concat([x_eval_pyt_num, x_eval_pyt_cat], axis=1) In\u00a0[\u00a0]: Copied! <pre>embed_input = [\n    (u, int(x_train_pyt[u].nunique() ** 0.25) + 1) for u in CATEGORICAL_FEATURES\n]\n</pre> embed_input = [     (u, int(x_train_pyt[u].nunique() ** 0.25) + 1) for u in CATEGORICAL_FEATURES ] In\u00a0[\u00a0]: Copied! <pre># deeptabular\ntab_preprocessor = TabPreprocessor(\n    embed_cols=embed_input,\n    continuous_cols=NUMERICAL_FEATURES,\n    shared_embed=False,\n    scale=False,\n)\nX_tab_train = tab_preprocessor.fit_transform(x_train_pyt)\nX_tab_valid = tab_preprocessor.transform(x_eval_pyt)\nX_tab_test = tab_preprocessor.transform(x_eval_pyt)\n\n# target\ny_train = y_train\ny_valid = y_eval\ny_test = y_train\n\nX_train = {\"X_tab\": X_tab_train, \"target\": y_train}\nX_val = {\"X_tab\": X_tab_valid, \"target\": y_valid}\nX_test = {\"X_tab\": X_tab_test}\n\ndeeptabular = TabMlp(\n    mlp_hidden_dims=[128, 128, 64, 64],\n    column_idx=tab_preprocessor.column_idx,\n    embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=tab_preprocessor.continuous_cols,\n)\n\nmodel = WideDeep(deeptabular=deeptabular, pred_dim=3)\n\ndeep_opt = NAdam(model.deeptabular.parameters(), lr=LEARNING_RATE)\ncallbacks = [EarlyStopping()]\ndeep_sch = ReduceLROnPlateau(deep_opt, min_lr=1e-6)\n\nobjective = \"ziln\"\n\ntrainer = Trainer(\n    model,\n    callbacks=callbacks,\n    lr_schedulers={\"deeptabular\": deep_sch},\n    objective=objective,\n    optimizers={\"deeptabular\": deep_opt},\n)\n\ntrainer.fit(\n    X_train=X_train,\n    X_val=X_val,\n    n_epochs=200,\n    batch_size=2048,\n)\n\ny_pred_pytorch = trainer.predict(X_test=X_test)\n</pre> # deeptabular tab_preprocessor = TabPreprocessor(     embed_cols=embed_input,     continuous_cols=NUMERICAL_FEATURES,     shared_embed=False,     scale=False, ) X_tab_train = tab_preprocessor.fit_transform(x_train_pyt) X_tab_valid = tab_preprocessor.transform(x_eval_pyt) X_tab_test = tab_preprocessor.transform(x_eval_pyt)  # target y_train = y_train y_valid = y_eval y_test = y_train  X_train = {\"X_tab\": X_tab_train, \"target\": y_train} X_val = {\"X_tab\": X_tab_valid, \"target\": y_valid} X_test = {\"X_tab\": X_tab_test}  deeptabular = TabMlp(     mlp_hidden_dims=[128, 128, 64, 64],     column_idx=tab_preprocessor.column_idx,     embed_input=tab_preprocessor.cat_embed_input,     continuous_cols=tab_preprocessor.continuous_cols, )  model = WideDeep(deeptabular=deeptabular, pred_dim=3)  deep_opt = NAdam(model.deeptabular.parameters(), lr=LEARNING_RATE) callbacks = [EarlyStopping()] deep_sch = ReduceLROnPlateau(deep_opt, min_lr=1e-6)  objective = \"ziln\"  trainer = Trainer(     model,     callbacks=callbacks,     lr_schedulers={\"deeptabular\": deep_sch},     objective=objective,     optimizers={\"deeptabular\": deep_opt}, )  trainer.fit(     X_train=X_train,     X_val=X_val,     n_epochs=200,     batch_size=2048, )  y_pred_pytorch = trainer.predict(X_test=X_test) In\u00a0[\u00a0]: Copied! <pre>pd.DataFrame(trainer.history)[[\"train_loss\", \"val_loss\"]].plot();\n</pre> pd.DataFrame(trainer.history)[[\"train_loss\", \"val_loss\"]].plot(); In\u00a0[\u00a0]: Copied! <pre>from sklearn.metrics import mean_squared_error\n\nmean_squared_error(y_pred, y_pred_pytorch)\n</pre> from sklearn.metrics import mean_squared_error  mean_squared_error(y_pred, y_pred_pytorch) In\u00a0[\u00a0]: Copied! <pre>unit_costs = [0.4, 0.5, 0.6, 0.68, 0.7, 0.8, 0.9, 1.0]\n</pre> unit_costs = [0.4, 0.5, 0.6, 0.68, 0.7, 0.8, 0.9, 1.0] In\u00a0[\u00a0]: Copied! <pre>num_mailed = [np.sum(y_pred &gt; v) for v in unit_costs]\nnum_mailed\n</pre> num_mailed = [np.sum(y_pred &gt; v) for v in unit_costs] num_mailed In\u00a0[\u00a0]: Copied! <pre>baseline_total_profit = np.sum(y_eval - 0.68)\nbaseline_total_profit\n</pre> baseline_total_profit = np.sum(y_eval - 0.68) baseline_total_profit In\u00a0[\u00a0]: Copied! <pre>total_profits = [np.sum(y_eval[y_pred &gt; v] - v) for v in unit_costs]\ntotal_profits\n</pre> total_profits = [np.sum(y_eval[y_pred &gt; v] - v) for v in unit_costs] total_profits In\u00a0[\u00a0]: Copied! <pre>gain = pd.DataFrame(\n    {\n        \"lorenz\": ltv.cumulative_true(y_eval, y_eval),\n        \"baseline\": ltv.cumulative_true(y_eval, x_eval[\"numeric\"][:, 19]),\n        \"model\": ltv.cumulative_true(y_eval, y_pred),\n    }\n)\n</pre> gain = pd.DataFrame(     {         \"lorenz\": ltv.cumulative_true(y_eval, y_eval),         \"baseline\": ltv.cumulative_true(y_eval, x_eval[\"numeric\"][:, 19]),         \"model\": ltv.cumulative_true(y_eval, y_pred),     } ) In\u00a0[\u00a0]: Copied! <pre>num_customers = np.float32(gain.shape[0])\ngain[\"cumulative_customer\"] = (np.arange(num_customers) + 1.0) / num_customers\n</pre> num_customers = np.float32(gain.shape[0]) gain[\"cumulative_customer\"] = (np.arange(num_customers) + 1.0) / num_customers In\u00a0[\u00a0]: Copied! <pre>ax = gain[\n    [\n        \"cumulative_customer\",\n        \"lorenz\",\n        \"baseline\",\n        \"model\",\n    ]\n].plot(x=\"cumulative_customer\", figsize=(8, 5), legend=True)\n\nax.legend([\"Groundtruth\", \"Baseline\", \"Model\"], loc=\"lower right\")\n\nax.set_xlabel(\"Cumulative Fraction of Customers\")\nax.set_xticks(np.arange(0, 1.1, 0.1))\nax.set_xlim((0, 1.0))\n\nax.set_ylabel(\"Cumulative Fraction of Total Lifetime Value\")\nax.set_yticks(np.arange(0, 1.1, 0.1))\nax.set_ylim((0, 1.05))\nax.set_title(\"Gain Chart\");\n</pre> ax = gain[     [         \"cumulative_customer\",         \"lorenz\",         \"baseline\",         \"model\",     ] ].plot(x=\"cumulative_customer\", figsize=(8, 5), legend=True)  ax.legend([\"Groundtruth\", \"Baseline\", \"Model\"], loc=\"lower right\")  ax.set_xlabel(\"Cumulative Fraction of Customers\") ax.set_xticks(np.arange(0, 1.1, 0.1)) ax.set_xlim((0, 1.0))  ax.set_ylabel(\"Cumulative Fraction of Total Lifetime Value\") ax.set_yticks(np.arange(0, 1.1, 0.1)) ax.set_ylim((0, 1.05)) ax.set_title(\"Gain Chart\"); In\u00a0[\u00a0]: Copied! <pre>gini = ltv.gini_from_gain(gain[[\"lorenz\", \"baseline\", \"model\"]])\ngini\n</pre> gini = ltv.gini_from_gain(gain[[\"lorenz\", \"baseline\", \"model\"]]) gini In\u00a0[\u00a0]: Copied! <pre>df_decile = ltv.decile_stats(y_eval, y_pred)\ndf_decile\n</pre> df_decile = ltv.decile_stats(y_eval, y_pred) df_decile In\u00a0[\u00a0]: Copied! <pre>ax = df_decile[[\"label_mean\", \"pred_mean\"]].plot.bar(rot=0)\n\nax.set_title(\"Decile Chart\")\nax.set_xlabel(\"Prediction bucket\")\nax.set_ylabel(\"Average bucket value\")\nax.legend([\"Label\", \"Prediction\"], loc=\"upper left\");\n</pre> ax = df_decile[[\"label_mean\", \"pred_mean\"]].plot.bar(rot=0)  ax.set_title(\"Decile Chart\") ax.set_xlabel(\"Prediction bucket\") ax.set_ylabel(\"Average bucket value\") ax.legend([\"Label\", \"Prediction\"], loc=\"upper left\"); In\u00a0[\u00a0]: Copied! <pre>def spearmanr(x1: Sequence[float], x2: Sequence[float]) -&gt; float:\n    \"\"\"Calculates spearmanr rank correlation coefficient.\n\n    See https://docs.scipy.org/doc/scipy/reference/stats.html.\n\n    Args:\n      x1: 1D array_like.\n      x2: 1D array_like.\n\n    Returns:\n      correlation: float.\n    \"\"\"\n    return stats.spearmanr(x1, x2, nan_policy=\"raise\")[0]\n\n\nspearman_corr = spearmanr(y_eval, y_pred)\nspearman_corr\n</pre> def spearmanr(x1: Sequence[float], x2: Sequence[float]) -&gt; float:     \"\"\"Calculates spearmanr rank correlation coefficient.      See https://docs.scipy.org/doc/scipy/reference/stats.html.      Args:       x1: 1D array_like.       x2: 1D array_like.      Returns:       correlation: float.     \"\"\"     return stats.spearmanr(x1, x2, nan_policy=\"raise\")[0]   spearman_corr = spearmanr(y_eval, y_pred) spearman_corr In\u00a0[\u00a0]: Copied! <pre>df_metrics = pd.DataFrame(\n    {\n        \"model\": MODEL,\n        \"loss_function\": LOSS,\n        \"train_loss\": history[\"loss\"][-1],\n        \"eval_loss\": history[\"val_loss\"][-1],\n        \"label_positive\": np.mean(y_eval &gt; 0),\n        \"label_mean\": y_eval.mean(),\n        \"pred_mean\": y_pred.mean(),\n        \"decile_mape\": df_decile[\"decile_mape\"].mean(),\n        \"baseline_gini\": gini[\"normalized\"][1],\n        \"gini\": gini[\"normalized\"][2],\n        \"spearman_corr\": spearman_corr,\n    },\n    index=[VERSION],\n)\n</pre> df_metrics = pd.DataFrame(     {         \"model\": MODEL,         \"loss_function\": LOSS,         \"train_loss\": history[\"loss\"][-1],         \"eval_loss\": history[\"val_loss\"][-1],         \"label_positive\": np.mean(y_eval &gt; 0),         \"label_mean\": y_eval.mean(),         \"pred_mean\": y_pred.mean(),         \"decile_mape\": df_decile[\"decile_mape\"].mean(),         \"baseline_gini\": gini[\"normalized\"][1],         \"gini\": gini[\"normalized\"][2],         \"spearman_corr\": spearman_corr,     },     index=[VERSION], ) In\u00a0[\u00a0]: Copied! <pre>for unit_cost, total_profit in zip(unit_costs, total_profits):\n    df_metrics[\"total_profit_{:02d}\".format(int(unit_cost * 100))] = total_profit\n</pre> for unit_cost, total_profit in zip(unit_costs, total_profits):     df_metrics[\"total_profit_{:02d}\".format(int(unit_cost * 100))] = total_profit In\u00a0[\u00a0]: Copied! <pre>df_metrics.T\n</pre> df_metrics.T In\u00a0[\u00a0]: Copied! <pre>output_path = OUTPUT_CSV_FOLDER\n</pre> output_path = OUTPUT_CSV_FOLDER In\u00a0[\u00a0]: Copied! <pre>if not os.path.isdir(output_path):\n    os.makedirs(output_path)\n</pre> if not os.path.isdir(output_path):     os.makedirs(output_path) In\u00a0[\u00a0]: Copied! <pre>output_file = os.path.join(\n    output_path, \"{}_regression_{}_{}.csv\".format(MODEL, LOSS, VERSION)\n)\n</pre> output_file = os.path.join(     output_path, \"{}_regression_{}_{}.csv\".format(MODEL, LOSS, VERSION) ) In\u00a0[\u00a0]: Copied! <pre>df_metrics.to_csv(output_file, index=False)\n</pre> df_metrics.to_csv(output_file, index=False)"},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#zilnloss","title":"ZILNLoss\u00b6","text":"<p>[DISCLAIMER]</p> <p>Purpose of this notebook is to check if ZILNloss implemented originaly Keras give same results in pytorch-widedeep implemenatation</p>"},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#kdd-cup-98-ltv-prediction","title":"KDD Cup 98 LTV Prediction\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#configs","title":"Configs\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#load-data","title":"Load data\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#label-distribution","title":"Label distribution\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#preprocess-features","title":"Preprocess features\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#vocab","title":"Vocab\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#indicator","title":"Indicator\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#numeric","title":"Numeric\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#all","title":"All\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#traineval-split","title":"Train/eval split\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#model","title":"Model\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#loss","title":"Loss\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#train","title":"Train\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#eval","title":"Eval\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#pytorch-widedeep-approach","title":"Pytorch-widedeep approach\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#appendix","title":"Appendix\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#total-profit","title":"Total Profit\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#gini-coefficient","title":"Gini Coefficient\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#calibration","title":"Calibration\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#rank-correlation","title":"Rank Correlation\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#all-metrics-together","title":"All metrics together\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#save","title":"Save\u00b6","text":""},{"location":"examples/13_model_uncertainty_prediction.html","title":"13_model_uncertainty_prediction","text":"<ul> <li>In this notebook we will use the higly imbalanced Protein Homology Dataset from KDD cup 2004</li> </ul> <pre><code>* The first element of each line is a BLOCK ID that denotes to which native sequence this example belongs. There is a unique BLOCK ID for each native sequence. BLOCK IDs are integers running from 1 to 303 (one for each native sequence, i.e. for each query). BLOCK IDs were assigned before the blocks were split into the train and test sets, so they do not run consecutively in either file.\n* The second element of each line is an EXAMPLE ID that uniquely describes the example. You will need this EXAMPLE ID and the BLOCK ID when you submit results.\n* The third element is the class of the example. Proteins that are homologous to the native sequence are denoted by 1, non-homologous proteins (i.e. decoys) by 0. Test examples have a \"?\" in this position.\n* All following elements are feature values. There are 74 feature values in each line. The features describe the match (e.g. the score of a sequence alignment) between the native protein sequence and the sequence that is tested for homology.\n</code></pre> In\u00a0[1]: Copied! <pre>import numpy as np\nimport pandas as pd\nimport torch\nfrom torch.optim import SGD, lr_scheduler\n\nfrom pytorch_widedeep import Trainer\nfrom pytorch_widedeep.preprocessing import TabPreprocessor\nfrom pytorch_widedeep.models import TabMlp, WideDeep\nfrom pytorch_widedeep.dataloaders import DataLoaderImbalanced, DataLoaderDefault\nfrom torchmetrics import F1Score as F1_torchmetrics\nfrom torchmetrics import Accuracy as Accuracy_torchmetrics\nfrom torchmetrics import Precision as Precision_torchmetrics\nfrom torchmetrics import Recall as Recall_torchmetrics\nfrom pytorch_widedeep.metrics import Accuracy, Recall, Precision, F1Score, R2Score\nfrom pytorch_widedeep.initializers import XavierNormal\nfrom pytorch_widedeep.datasets import load_bio_kdd04\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import classification_report\n\nimport time\nimport datetime\n\nimport warnings\n\nwarnings.filterwarnings(\"ignore\", category=DeprecationWarning)\n\n# increase displayed columns in jupyter notebook\npd.set_option(\"display.max_columns\", 200)\npd.set_option(\"display.max_rows\", 300)\n</pre> import numpy as np import pandas as pd import torch from torch.optim import SGD, lr_scheduler  from pytorch_widedeep import Trainer from pytorch_widedeep.preprocessing import TabPreprocessor from pytorch_widedeep.models import TabMlp, WideDeep from pytorch_widedeep.dataloaders import DataLoaderImbalanced, DataLoaderDefault from torchmetrics import F1Score as F1_torchmetrics from torchmetrics import Accuracy as Accuracy_torchmetrics from torchmetrics import Precision as Precision_torchmetrics from torchmetrics import Recall as Recall_torchmetrics from pytorch_widedeep.metrics import Accuracy, Recall, Precision, F1Score, R2Score from pytorch_widedeep.initializers import XavierNormal from pytorch_widedeep.datasets import load_bio_kdd04  from sklearn.model_selection import train_test_split from sklearn.metrics import classification_report  import time import datetime  import warnings  warnings.filterwarnings(\"ignore\", category=DeprecationWarning)  # increase displayed columns in jupyter notebook pd.set_option(\"display.max_columns\", 200) pd.set_option(\"display.max_rows\", 300) In\u00a0[2]: Copied! <pre>df = load_bio_kdd04(as_frame=True)\ndf.head()\n</pre> df = load_bio_kdd04(as_frame=True) df.head() Out[2]: EXAMPLE_ID BLOCK_ID target 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 0 279 261532 0 52.0 32.69 0.30 2.5 20.0 1256.8 -0.89 0.33 11.0 -55.0 267.2 0.52 0.05 -2.36 49.6 252.0 0.43 1.16 -2.06 -33.0 -123.2 1.60 -0.49 -6.06 65.0 296.1 -0.28 -0.26 -3.83 -22.6 -170.0 3.06 -1.05 -3.29 22.9 286.3 0.12 2.58 4.08 -33.0 -178.9 1.88 0.53 -7.0 -44.0 1987.0 -5.41 0.95 -4.0 -57.0 722.9 -3.26 -0.55 -7.5 125.5 1547.2 -0.36 1.12 9.0 -37.0 72.5 0.47 0.74 -11.0 -8.0 1595.1 -1.64 2.83 -2.0 -50.0 445.2 -0.35 0.26 0.76 1 279 261533 0 58.0 33.33 0.00 16.5 9.5 608.1 0.50 0.07 20.5 -52.5 521.6 -1.08 0.58 -0.02 -3.2 103.6 -0.95 0.23 -2.87 -25.9 -52.2 -0.21 0.87 -1.81 10.4 62.0 -0.28 -0.04 1.48 -17.6 -198.3 3.43 2.84 5.87 -16.9 72.6 -0.31 2.79 2.71 -33.5 -11.6 -1.11 4.01 5.0 -57.0 666.3 1.13 4.38 5.0 -64.0 39.3 1.07 -0.16 32.5 100.0 1893.7 -2.80 -0.22 2.5 -28.5 45.0 0.58 0.41 -19.0 -6.0 762.9 0.29 0.82 -3.0 -35.0 140.3 1.16 0.39 0.73 2 279 261534 0 77.0 27.27 -0.91 6.0 58.5 1623.6 -1.40 0.02 -6.5 -48.0 621.0 -1.20 0.14 -0.20 73.6 609.1 -0.44 -0.58 -0.04 -23.0 -27.4 -0.72 -1.04 -1.09 91.1 635.6 -0.88 0.24 0.59 -18.7 -7.2 -0.60 -2.82 -0.71 52.4 504.1 0.89 -0.67 -9.30 -20.8 -25.7 -0.77 -0.85 0.0 -20.0 2259.0 -0.94 1.15 -4.0 -44.0 -22.7 0.94 -0.98 -19.0 105.0 1267.9 1.03 1.27 11.0 -39.5 82.3 0.47 -0.19 -10.0 7.0 1491.8 0.32 -1.29 0.0 -34.0 658.2 -0.76 0.26 0.24 3 279 261535 0 41.0 27.91 -0.35 3.0 46.0 1921.6 -1.36 -0.47 -32.0 -51.5 560.9 -0.29 -0.10 -1.11 124.3 791.6 0.00 0.39 -1.85 -21.7 -44.9 -0.21 0.02 0.89 133.9 797.8 -0.08 1.06 -0.26 -16.4 -74.1 0.97 -0.80 -0.41 66.9 955.3 -1.90 1.28 -6.65 -28.1 47.5 -1.91 1.42 1.0 -30.0 1846.7 0.76 1.10 -4.0 -52.0 -53.9 1.71 -0.22 -12.0 97.5 1969.8 -1.70 0.16 -1.0 -32.5 255.9 -0.46 1.57 10.0 6.0 2047.7 -0.98 1.53 0.0 -49.0 554.2 -0.83 0.39 0.73 4 279 261536 0 50.0 28.00 -1.32 -9.0 12.0 464.8 0.88 0.19 8.0 -51.5 98.1 1.09 -0.33 -2.16 -3.9 102.7 0.39 -1.22 -3.39 -15.2 -42.2 -1.18 -1.11 -3.55 8.9 141.3 -0.16 -0.43 -4.15 -12.9 -13.4 -1.32 -0.98 -3.69 8.8 136.1 -0.30 4.13 1.89 -13.0 -18.7 -1.37 -0.93 0.0 -1.0 810.1 -2.29 6.72 1.0 -23.0 -29.7 0.58 -1.10 -18.5 33.5 206.8 1.84 -0.13 4.0 -29.0 30.1 0.80 -0.24 5.0 -14.0 479.5 0.68 -0.59 2.0 -36.0 -6.9 2.02 0.14 -0.23 In\u00a0[3]: Copied! <pre># imbalance of the classes\ndf[\"target\"].value_counts()\n</pre> # imbalance of the classes df[\"target\"].value_counts() Out[3]: <pre>target\n0    144455\n1      1296\nName: count, dtype: int64</pre> In\u00a0[4]: Copied! <pre># drop columns we won't need in this example\ndf.drop(columns=[\"EXAMPLE_ID\", \"BLOCK_ID\"], inplace=True)\n</pre> # drop columns we won't need in this example df.drop(columns=[\"EXAMPLE_ID\", \"BLOCK_ID\"], inplace=True) In\u00a0[5]: Copied! <pre>df_train, df_valid = train_test_split(\n    df, test_size=0.2, stratify=df[\"target\"], random_state=1\n)\ndf_valid, df_test = train_test_split(\n    df_valid, test_size=0.5, stratify=df_valid[\"target\"], random_state=1\n)\n</pre> df_train, df_valid = train_test_split(     df, test_size=0.2, stratify=df[\"target\"], random_state=1 ) df_valid, df_test = train_test_split(     df_valid, test_size=0.5, stratify=df_valid[\"target\"], random_state=1 ) In\u00a0[6]: Copied! <pre>continuous_cols = df.drop(columns=[\"target\"]).columns.values.tolist()\n</pre> continuous_cols = df.drop(columns=[\"target\"]).columns.values.tolist() In\u00a0[7]: Copied! <pre># deeptabular\ntab_preprocessor = TabPreprocessor(continuous_cols=continuous_cols, scale=True)\nX_tab_train = tab_preprocessor.fit_transform(df_train)\nX_tab_valid = tab_preprocessor.transform(df_valid)\nX_tab_test = tab_preprocessor.transform(df_test)\n\n# target\ny_train = df_train[\"target\"].values\ny_valid = df_valid[\"target\"].values\ny_test = df_test[\"target\"].values\n</pre> # deeptabular tab_preprocessor = TabPreprocessor(continuous_cols=continuous_cols, scale=True) X_tab_train = tab_preprocessor.fit_transform(df_train) X_tab_valid = tab_preprocessor.transform(df_valid) X_tab_test = tab_preprocessor.transform(df_test)  # target y_train = df_train[\"target\"].values y_valid = df_valid[\"target\"].values y_test = df_test[\"target\"].values In\u00a0[8]: Copied! <pre>input_layer = len(tab_preprocessor.continuous_cols)\noutput_layer = 1\nhidden_layers = np.linspace(\n    input_layer * 2, output_layer, 5, endpoint=False, dtype=int\n).tolist()\n</pre> input_layer = len(tab_preprocessor.continuous_cols) output_layer = 1 hidden_layers = np.linspace(     input_layer * 2, output_layer, 5, endpoint=False, dtype=int ).tolist() In\u00a0[9]: Copied! <pre>deeptabular = TabMlp(\n    mlp_hidden_dims=hidden_layers,\n    column_idx=tab_preprocessor.column_idx,\n    continuous_cols=tab_preprocessor.continuous_cols,\n)\nmodel = WideDeep(deeptabular=deeptabular, pred_dim=1)\nmodel\n</pre> deeptabular = TabMlp(     mlp_hidden_dims=hidden_layers,     column_idx=tab_preprocessor.column_idx,     continuous_cols=tab_preprocessor.continuous_cols, ) model = WideDeep(deeptabular=deeptabular, pred_dim=1) model Out[9]: <pre>WideDeep(\n  (deeptabular): Sequential(\n    (0): TabMlp(\n      (cont_norm): Identity()\n      (encoder): MLP(\n        (mlp): Sequential(\n          (dense_layer_0): Sequential(\n            (0): Linear(in_features=74, out_features=148, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_1): Sequential(\n            (0): Linear(in_features=148, out_features=118, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_2): Sequential(\n            (0): Linear(in_features=118, out_features=89, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_3): Sequential(\n            (0): Linear(in_features=89, out_features=59, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_4): Sequential(\n            (0): Linear(in_features=59, out_features=30, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=30, out_features=1, bias=True)\n  )\n)</pre> In\u00a0[10]: Copied! <pre># # Metrics from torchmetrics\n# accuracy = Accuracy_torchmetrics(average=None, num_classes=1)\n# precision = Precision_torchmetrics(average=\"micro\", num_classes=1)\n# f1 = F1_torchmetrics(average=None, num_classes=1)\n# recall = Recall_torchmetrics(average=None, num_classes=1)\n</pre> # # Metrics from torchmetrics # accuracy = Accuracy_torchmetrics(average=None, num_classes=1) # precision = Precision_torchmetrics(average=\"micro\", num_classes=1) # f1 = F1_torchmetrics(average=None, num_classes=1) # recall = Recall_torchmetrics(average=None, num_classes=1) In\u00a0[11]: Copied! <pre># Metrics from pytorch-widedeep\naccuracy = Accuracy(top_k=2)\nprecision = Precision(average=False)\nrecall = Recall(average=True)\nf1 = F1Score(average=False)\n</pre> # Metrics from pytorch-widedeep accuracy = Accuracy(top_k=2) precision = Precision(average=False) recall = Recall(average=True) f1 = F1Score(average=False) In\u00a0[12]: Copied! <pre># Optimizers\ndeep_opt = SGD(model.deeptabular.parameters(), lr=0.1)\n# LR Scheduler\ndeep_sch = lr_scheduler.StepLR(deep_opt, step_size=3)\n\ntrainer = Trainer(\n    model,\n    objective=\"binary\",\n    lr_schedulers={\"deeptabular\": deep_sch},\n    initializers={\"deeptabular\": XavierNormal},\n    optimizers={\"deeptabular\": deep_opt},\n    metrics=[accuracy, precision, recall, f1],\n    verbose=1,\n)\n</pre> # Optimizers deep_opt = SGD(model.deeptabular.parameters(), lr=0.1) # LR Scheduler deep_sch = lr_scheduler.StepLR(deep_opt, step_size=3)  trainer = Trainer(     model,     objective=\"binary\",     lr_schedulers={\"deeptabular\": deep_sch},     initializers={\"deeptabular\": XavierNormal},     optimizers={\"deeptabular\": deep_opt},     metrics=[accuracy, precision, recall, f1],     verbose=1, ) In\u00a0[13]: Copied! <pre>start = time.time()\ntrainer.fit(\n    X_train={\"X_tab\": X_tab_train, \"target\": y_train},\n    X_val={\"X_tab\": X_tab_valid, \"target\": y_valid},\n    n_epochs=3,\n    batch_size=50,\n    custom_dataloader=DataLoaderImbalanced,\n    oversample_mul=5,\n)\nprint(\n    \"Training time[s]: {}\".format(\n        datetime.timedelta(seconds=round(time.time() - start))\n    )\n)\n</pre> start = time.time() trainer.fit(     X_train={\"X_tab\": X_tab_train, \"target\": y_train},     X_val={\"X_tab\": X_tab_valid, \"target\": y_valid},     n_epochs=3,     batch_size=50,     custom_dataloader=DataLoaderImbalanced,     oversample_mul=5, ) print(     \"Training time[s]: {}\".format(         datetime.timedelta(seconds=round(time.time() - start))     ) ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 208/208 [00:01&lt;00:00, 130.75it/s, loss=0.187, metrics={'acc': 0.9214, 'prec': [0.9149], 'rec': 0.9318, 'f1': [0.9233]}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 292/292 [00:01&lt;00:00, 173.71it/s, loss=0.106, metrics={'acc': 0.9499, 'prec': [0.1435], 'rec': 0.938, 'f1': [0.249]}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 208/208 [00:01&lt;00:00, 139.68it/s, loss=0.109, metrics={'acc': 0.9559, 'prec': [0.9537], 'rec': 0.9572, 'f1': [0.9554]}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 292/292 [00:01&lt;00:00, 177.53it/s, loss=0.0888, metrics={'acc': 0.9602, 'prec': [0.1755], 'rec': 0.9457, 'f1': [0.2961]}]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 208/208 [00:01&lt;00:00, 141.63it/s, loss=0.08, metrics={'acc': 0.9706, 'prec': [0.9648], 'rec': 0.9766, 'f1': [0.9707]}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 292/292 [00:01&lt;00:00, 165.36it/s, loss=0.0969, metrics={'acc': 0.9564, 'prec': [0.1636], 'rec': 0.9535, 'f1': [0.2792]}]</pre> <pre>Training time[s]: 0:00:10\n</pre> <pre>\n</pre> In\u00a0[14]: Copied! <pre>pd.DataFrame(trainer.history)\n</pre> pd.DataFrame(trainer.history) Out[14]: train_loss train_acc train_prec train_rec train_f1 val_loss val_acc val_prec val_rec val_f1 0 0.186707 0.921408 [0.9149412512779236] 0.931801 [0.9232940673828125] 0.106023 0.949914 [0.14353498816490173] 0.937984 [0.24897116422653198] 1 0.109498 0.955931 [0.9536514282226562] 0.957193 [0.9554190039634705] 0.088787 0.960206 [0.17553956806659698] 0.945736 [0.29611650109291077] 2 0.079979 0.970588 [0.9648183584213257] 0.976582 [0.9706646203994751] 0.096858 0.956432 [0.1635638326406479] 0.953488 [0.279228150844574] In\u00a0[15]: Copied! <pre>df_pred = trainer.predict(X_tab=X_tab_test)\nprint(classification_report(df_test[\"target\"].to_list(), df_pred))\nprint(\"Actual predicted values:\\n{}\".format(np.unique(df_pred, return_counts=True)))\n</pre> df_pred = trainer.predict(X_tab=X_tab_test) print(classification_report(df_test[\"target\"].to_list(), df_pred)) print(\"Actual predicted values:\\n{}\".format(np.unique(df_pred, return_counts=True))) <pre>predict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 292/292 [00:00&lt;00:00, 346.55it/s]\n</pre> <pre>              precision    recall  f1-score   support\n\n           0       1.00      0.96      0.98     14446\n           1       0.17      0.95      0.29       130\n\n    accuracy                           0.96     14576\n   macro avg       0.58      0.95      0.63     14576\nweighted avg       0.99      0.96      0.97     14576\n\nActual predicted values:\n(array([0, 1]), array([13845,   731]))\n</pre> In\u00a0[16]: Copied! <pre>df_pred_unc = trainer.predict_uncertainty(X_tab=X_tab_test, uncertainty_granularity=10)\nprint(classification_report(df_test[\"target\"].to_list(), df_pred))\nprint(\n    \"Actual predicted values:\\n{}\".format(\n        np.unique(df_pred_unc[:, -1], return_counts=True)\n    )\n)\n</pre> df_pred_unc = trainer.predict_uncertainty(X_tab=X_tab_test, uncertainty_granularity=10) print(classification_report(df_test[\"target\"].to_list(), df_pred)) print(     \"Actual predicted values:\\n{}\".format(         np.unique(df_pred_unc[:, -1], return_counts=True)     ) ) <pre>predict_UncertaintyIter: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 10/10 [00:03&lt;00:00,  3.25it/s]</pre> <pre>              precision    recall  f1-score   support\n\n           0       1.00      0.96      0.98     14446\n           1       0.17      0.95      0.29       130\n\n    accuracy                           0.96     14576\n   macro avg       0.58      0.95      0.63     14576\nweighted avg       0.99      0.96      0.97     14576\n\nActual predicted values:\n(array([0.]), array([14576]))\n</pre> <pre>\n</pre> In\u00a0[17]: Copied! <pre>df_pred_unc\n</pre> df_pred_unc Out[17]: <pre>array([[9.98401165e-01, 1.59881881e-03, 0.00000000e+00],\n       [9.99941409e-01, 5.85634953e-05, 0.00000000e+00],\n       [9.97351170e-01, 2.64881272e-03, 0.00000000e+00],\n       ...,\n       [9.99494374e-01, 5.05603210e-04, 0.00000000e+00],\n       [9.99981642e-01, 1.83574630e-05, 0.00000000e+00],\n       [9.99996483e-01, 3.52600046e-06, 0.00000000e+00]])</pre>"},{"location":"examples/13_model_uncertainty_prediction.html#model-uncertainty-prediction","title":"Model Uncertainty prediction\u00b6","text":"<p>Note:</p> <p>This notebook extends the \"Custom DataLoader for Imbalanced dataset\" notebook</p>"},{"location":"examples/13_model_uncertainty_prediction.html#initial-imports","title":"Initial imports\u00b6","text":""},{"location":"examples/13_model_uncertainty_prediction.html#preparing-the-data","title":"Preparing the data\u00b6","text":""},{"location":"examples/13_model_uncertainty_prediction.html#define-the-model","title":"Define the model\u00b6","text":""},{"location":"examples/13_model_uncertainty_prediction.html#normal-prediction","title":"\"Normal\" prediction\u00b6","text":""},{"location":"examples/13_model_uncertainty_prediction.html#prediction-using-uncertainty","title":"Prediction using uncertainty\u00b6","text":""},{"location":"examples/14_bayesian_models.html","title":"14_bayesian_models","text":"In\u00a0[1]: Copied! <pre>import numpy as np\nimport torch\nimport pandas as pd\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import accuracy_score\n\nfrom pytorch_widedeep.metrics import Accuracy\nfrom pytorch_widedeep.datasets import load_adult\nfrom pytorch_widedeep.callbacks import EarlyStopping, ModelCheckpoint\nfrom pytorch_widedeep.preprocessing import TabPreprocessor, WidePreprocessor\nfrom pytorch_widedeep.bayesian_models import BayesianWide, BayesianTabMlp\nfrom pytorch_widedeep.training.bayesian_trainer import BayesianTrainer\n</pre> import numpy as np import torch import pandas as pd  from sklearn.model_selection import train_test_split from sklearn.metrics import accuracy_score  from pytorch_widedeep.metrics import Accuracy from pytorch_widedeep.datasets import load_adult from pytorch_widedeep.callbacks import EarlyStopping, ModelCheckpoint from pytorch_widedeep.preprocessing import TabPreprocessor, WidePreprocessor from pytorch_widedeep.bayesian_models import BayesianWide, BayesianTabMlp from pytorch_widedeep.training.bayesian_trainer import BayesianTrainer <p>The first few things to do we know them very well, like with any other model described in any of the other notebooks</p> In\u00a0[2]: Copied! <pre>df = load_adult(as_frame=True)\ndf.columns = [c.replace(\"-\", \"_\") for c in df.columns]\ndf[\"age_buckets\"] = pd.cut(\n    df.age, bins=[16, 25, 30, 35, 40, 45, 50, 55, 60, 91], labels=np.arange(9)\n)\ndf[\"income_label\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int)\ndf.drop(\"income\", axis=1, inplace=True)\ndf.head()\n</pre> df = load_adult(as_frame=True) df.columns = [c.replace(\"-\", \"_\") for c in df.columns] df[\"age_buckets\"] = pd.cut(     df.age, bins=[16, 25, 30, 35, 40, 45, 50, 55, 60, 91], labels=np.arange(9) ) df[\"income_label\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int) df.drop(\"income\", axis=1, inplace=True) df.head() Out[2]: age workclass fnlwgt education educational_num marital_status occupation relationship race gender capital_gain capital_loss hours_per_week native_country age_buckets income_label 0 25 Private 226802 11th 7 Never-married Machine-op-inspct Own-child Black Male 0 0 40 United-States 0 0 1 38 Private 89814 HS-grad 9 Married-civ-spouse Farming-fishing Husband White Male 0 0 50 United-States 3 0 2 28 Local-gov 336951 Assoc-acdm 12 Married-civ-spouse Protective-serv Husband White Male 0 0 40 United-States 1 1 3 44 Private 160323 Some-college 10 Married-civ-spouse Machine-op-inspct Husband Black Male 7688 0 40 United-States 4 1 4 18 ? 103497 Some-college 10 Never-married ? Own-child White Female 0 0 30 United-States 0 0 In\u00a0[3]: Copied! <pre>train, test = train_test_split(df, test_size=0.2, stratify=df.income_label)\n</pre> train, test = train_test_split(df, test_size=0.2, stratify=df.income_label) In\u00a0[4]: Copied! <pre>wide_cols = [\n    \"age_buckets\",\n    \"education\",\n    \"relationship\",\n    \"workclass\",\n    \"occupation\",\n    \"native_country\",\n    \"gender\",\n]\ncrossed_cols = [(\"education\", \"occupation\"), (\"native_country\", \"occupation\")]\n\ncat_embed_cols = [\n    \"workclass\",\n    \"education\",\n    \"marital_status\",\n    \"occupation\",\n    \"relationship\",\n    \"race\",\n    \"gender\",\n    \"capital_gain\",\n    \"capital_loss\",\n    \"native_country\",\n]\ncontinuous_cols = [\"age\", \"hours_per_week\"]\n\ntarget = train[\"income_label\"].values\n</pre> wide_cols = [     \"age_buckets\",     \"education\",     \"relationship\",     \"workclass\",     \"occupation\",     \"native_country\",     \"gender\", ] crossed_cols = [(\"education\", \"occupation\"), (\"native_country\", \"occupation\")]  cat_embed_cols = [     \"workclass\",     \"education\",     \"marital_status\",     \"occupation\",     \"relationship\",     \"race\",     \"gender\",     \"capital_gain\",     \"capital_loss\",     \"native_country\", ] continuous_cols = [\"age\", \"hours_per_week\"]  target = train[\"income_label\"].values In\u00a0[5]: Copied! <pre>wide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols)\nX_tab = wide_preprocessor.fit_transform(train)\n</pre> wide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols) X_tab = wide_preprocessor.fit_transform(train) In\u00a0[6]: Copied! <pre>model = BayesianWide(\n    input_dim=np.unique(X_tab).shape[0],\n    prior_sigma_1=1.0,\n    prior_sigma_2=0.002,\n    prior_pi=0.8,\n    posterior_mu_init=0,\n    posterior_rho_init=-7.0,\n    pred_dim=1,  # here the models are NOT passed to a WideDeep constructor class so the output dim MUST be specified\n)\n</pre> model = BayesianWide(     input_dim=np.unique(X_tab).shape[0],     prior_sigma_1=1.0,     prior_sigma_2=0.002,     prior_pi=0.8,     posterior_mu_init=0,     posterior_rho_init=-7.0,     pred_dim=1,  # here the models are NOT passed to a WideDeep constructor class so the output dim MUST be specified ) In\u00a0[7]: Copied! <pre>trainer = BayesianTrainer(\n    model,\n    objective=\"binary\",\n    optimizer=torch.optim.Adam(model.parameters(), lr=0.01),\n    metrics=[Accuracy],\n)\n</pre> trainer = BayesianTrainer(     model,     objective=\"binary\",     optimizer=torch.optim.Adam(model.parameters(), lr=0.01),     metrics=[Accuracy], ) In\u00a0[8]: Copied! <pre>trainer.fit(\n    X_tab=X_tab,\n    target=target,\n    val_split=0.2,\n    n_epochs=2,\n    batch_size=256,\n)\n</pre> trainer.fit(     X_tab=X_tab,     target=target,     val_split=0.2,     n_epochs=2,     batch_size=256, ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 123/123 [00:00&lt;00:00, 124.32it/s, loss=163, metrics={'acc': 0.7813}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 31/31 [00:00&lt;00:00, 238.67it/s, loss=141, metrics={'acc': 0.8219}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 123/123 [00:00&lt;00:00, 132.81it/s, loss=140, metrics={'acc': 0.8285}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 31/31 [00:00&lt;00:00, 190.16it/s, loss=140, metrics={'acc': 0.8298}]\n</pre> In\u00a0[9]: Copied! <pre>tab_preprocessor = TabPreprocessor(\n    cat_embed_cols=cat_embed_cols, continuous_cols=continuous_cols\n)\nX_tab = tab_preprocessor.fit_transform(train)\n</pre> tab_preprocessor = TabPreprocessor(     cat_embed_cols=cat_embed_cols, continuous_cols=continuous_cols ) X_tab = tab_preprocessor.fit_transform(train) <pre>/Users/javierrodriguezzaurin/Projects/pytorch-widedeep/pytorch_widedeep/preprocessing/tab_preprocessor.py:358: UserWarning: Continuous columns will not be normalised\n  warnings.warn(\"Continuous columns will not be normalised\")\n</pre> In\u00a0[10]: Copied! <pre>model = BayesianTabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=continuous_cols,\n    #     embed_continuous_method = \"standard\",\n    #     cont_embed_activation=\"leaky_relu\",\n    #     cont_embed_dim = 8,\n    mlp_hidden_dims=[128, 64],\n    prior_sigma_1=1.0,\n    prior_sigma_2=0.002,\n    prior_pi=0.8,\n    posterior_mu_init=0,\n    posterior_rho_init=-7.0,\n    pred_dim=1,\n)\n</pre> model = BayesianTabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     continuous_cols=continuous_cols,     #     embed_continuous_method = \"standard\",     #     cont_embed_activation=\"leaky_relu\",     #     cont_embed_dim = 8,     mlp_hidden_dims=[128, 64],     prior_sigma_1=1.0,     prior_sigma_2=0.002,     prior_pi=0.8,     posterior_mu_init=0,     posterior_rho_init=-7.0,     pred_dim=1, ) In\u00a0[11]: Copied! <pre>trainer = BayesianTrainer(\n    model,\n    objective=\"binary\",\n    optimizer=torch.optim.Adam(model.parameters(), lr=0.01),\n    metrics=[Accuracy],\n)\n</pre> trainer = BayesianTrainer(     model,     objective=\"binary\",     optimizer=torch.optim.Adam(model.parameters(), lr=0.01),     metrics=[Accuracy], ) In\u00a0[12]: Copied! <pre>trainer.fit(\n    X_tab=X_tab,\n    target=target,\n    val_split=0.2,\n    n_epochs=2,\n    batch_size=256,\n)\n</pre> trainer.fit(     X_tab=X_tab,     target=target,     val_split=0.2,     n_epochs=2,     batch_size=256, ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 123/123 [00:04&lt;00:00, 28.74it/s, loss=2e+3, metrics={'acc': 0.8007}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 31/31 [00:00&lt;00:00, 136.89it/s, loss=1.75e+3, metrics={'acc': 0.8418}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 123/123 [00:04&lt;00:00, 29.41it/s, loss=1.73e+3, metrics={'acc': 0.8596}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 31/31 [00:00&lt;00:00, 143.87it/s, loss=1.71e+3, metrics={'acc': 0.8569}]\n</pre> <p>These models are powerful beyond the success metrics because they give us a sense of uncertainty as we predict. Let's have a look</p> In\u00a0[13]: Copied! <pre>X_tab_test = tab_preprocessor.transform(test)\n</pre> X_tab_test = tab_preprocessor.transform(test) In\u00a0[14]: Copied! <pre>preds = trainer.predict(X_tab_test, return_samples=True, n_samples=5)\n</pre> preds = trainer.predict(X_tab_test, return_samples=True, n_samples=5) <pre>predict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 39/39 [00:01&lt;00:00, 33.92it/s]\n</pre> In\u00a0[15]: Copied! <pre>preds.shape\n</pre> preds.shape Out[15]: <pre>(5, 9769)</pre> <p>as we can see the prediction have shape <code>(5, 9769)</code>, one set of predictions each time we have internally run predict (i.e. sample the network and predict, defined by the parameter <code>n_samples</code>). This gives us an idea of how certain the model is about a certain prediction.</p> <p>Similarly, we could obtain the probabilities</p> In\u00a0[16]: Copied! <pre>probs = trainer.predict_proba(X_tab_test, return_samples=True, n_samples=5)\n</pre> probs = trainer.predict_proba(X_tab_test, return_samples=True, n_samples=5) <pre>predict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 39/39 [00:01&lt;00:00, 32.79it/s]\n</pre> In\u00a0[17]: Copied! <pre>probs.shape\n</pre> probs.shape Out[17]: <pre>(5, 9769, 2)</pre> <p>And we could see how the model performs each time we sampled the network</p> In\u00a0[18]: Copied! <pre>for p in preds:\n    print(accuracy_score(p, test[\"income_label\"].values))\n</pre> for p in preds:     print(accuracy_score(p, test[\"income_label\"].values)) <pre>0.8559729757395844\n0.8564847988535162\n0.8567918927218753\n0.8562800696079435\n0.8558706111167981\n</pre> In\u00a0[\u00a0]: Copied! <pre>\n</pre>"},{"location":"examples/14_bayesian_models.html#the-bayesian-models","title":"The Bayesian Models\u00b6","text":"<p>Perhaps one of the most interesting functionality in the library is the access to full Bayesian models in almost exactly the same way one would use any of the other models in the library.</p> <p>Note however that the Bayesian models are ONLY available for tabular data and, at the moment, we do not support combining them to form a Wide and Deep model.</p> <p>The implementation in this library is based on the publication: Weight Uncertainty in Neural Networks, by Blundell et al., 2015. Code-wise, our implementation is inspired by a number of source:</p> <ol> <li>https://joshfeldman.net/WeightUncertainty/</li> <li>https://www.nitarshan.com/bayes-by-backprop/</li> <li>https://github.com/piEsposito/blitz-bayesian-deep-learning</li> <li>https://github.com/zackchase/mxnet-the-straight-dope/tree/master/chapter18_variational-methods-and-uncertainty</li> </ol> <p>The two Bayesian models available in the library are:</p> <ol> <li>BayesianWide: this is a linear model where the non-linearities are captured via crossed-columns</li> <li>BayesianMLP: this is a standard MLP that receives categorical embeddings and continuous cols (embedded or not) which are the passed through a series of dense layers. All parameters in the model are probabilistic.</li> </ol>"},{"location":"examples/14_bayesian_models.html#1-bayesianwide","title":"1. <code>BayesianWide</code>\u00b6","text":""},{"location":"examples/14_bayesian_models.html#2-bayesiantabmlp","title":"2. <code>BayesianTabMlp</code>\u00b6","text":""},{"location":"examples/15_Self_Supervised_Pretraning_pt1.html","title":"15 Self Supervised Pretraning pt1","text":"In\u00a0[1]: Copied! <pre>import torch\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.model_selection import train_test_split\n\nfrom pytorch_widedeep import Trainer\nfrom pytorch_widedeep.models import TabMlp, WideDeep\nfrom pytorch_widedeep.metrics import Accuracy\nfrom pytorch_widedeep.datasets import load_adult\nfrom pytorch_widedeep.preprocessing import TabPreprocessor\nfrom pytorch_widedeep.self_supervised_training import EncoderDecoderTrainer\n</pre> import torch from sklearn.metrics import accuracy_score from sklearn.model_selection import train_test_split  from pytorch_widedeep import Trainer from pytorch_widedeep.models import TabMlp, WideDeep from pytorch_widedeep.metrics import Accuracy from pytorch_widedeep.datasets import load_adult from pytorch_widedeep.preprocessing import TabPreprocessor from pytorch_widedeep.self_supervised_training import EncoderDecoderTrainer In\u00a0[2]: Copied! <pre>df = load_adult(as_frame=True)\ndf.columns = [c.replace(\"-\", \"_\") for c in df.columns]\ndf[\"income_label\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int)\ndf.drop(\"income\", axis=1, inplace=True)\n</pre> df = load_adult(as_frame=True) df.columns = [c.replace(\"-\", \"_\") for c in df.columns] df[\"income_label\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int) df.drop(\"income\", axis=1, inplace=True) In\u00a0[3]: Copied! <pre># one could chose to use a validation set for early stopping, hyperparam\n# optimization, etc. This is just an example, so we simply use train/test\n# split\ndf_tr, df_te = train_test_split(df, test_size=0.2, stratify=df.income_label)\n</pre> # one could chose to use a validation set for early stopping, hyperparam # optimization, etc. This is just an example, so we simply use train/test # split df_tr, df_te = train_test_split(df, test_size=0.2, stratify=df.income_label) In\u00a0[4]: Copied! <pre>df_tr.head(2)\n</pre> df_tr.head(2) Out[4]: age workclass fnlwgt education educational_num marital_status occupation relationship race gender capital_gain capital_loss hours_per_week native_country income_label 9042 26 Local-gov 250551 HS-grad 9 Married-civ-spouse Craft-repair Own-child Black Male 0 0 40 United-States 0 25322 50 Private 34832 Bachelors 13 Married-civ-spouse Tech-support Husband White Male 15024 0 40 United-States 1 In\u00a0[5]: Copied! <pre># As always, we need to define which cols will be represented as embeddings\n# and which one will be continuous features\ncat_embed_cols = [\n    \"workclass\",\n    \"education\",\n    \"marital_status\",\n    \"occupation\",\n    \"relationship\",\n    \"race\",\n    \"gender\",\n    \"capital_gain\",\n    \"capital_loss\",\n    \"native_country\",\n]\ncontinuous_cols = [\"age\", \"hours_per_week\"]\ntarget_col = \"income_label\"\n</pre> # As always, we need to define which cols will be represented as embeddings # and which one will be continuous features cat_embed_cols = [     \"workclass\",     \"education\",     \"marital_status\",     \"occupation\",     \"relationship\",     \"race\",     \"gender\",     \"capital_gain\",     \"capital_loss\",     \"native_country\", ] continuous_cols = [\"age\", \"hours_per_week\"] target_col = \"income_label\" In\u00a0[6]: Copied! <pre># We prepare the data to be passed to the model\ntab_preprocessor = TabPreprocessor(\n    cat_embed_cols=cat_embed_cols, continuous_cols=continuous_cols\n)\nX_tab = tab_preprocessor.fit_transform(df_tr)\ntarget = df_tr[target_col].values\n</pre> # We prepare the data to be passed to the model tab_preprocessor = TabPreprocessor(     cat_embed_cols=cat_embed_cols, continuous_cols=continuous_cols ) X_tab = tab_preprocessor.fit_transform(df_tr) target = df_tr[target_col].values <pre>/Users/javierrodriguezzaurin/Projects/pytorch-widedeep/pytorch_widedeep/preprocessing/tab_preprocessor.py:358: UserWarning: Continuous columns will not be normalised\n  warnings.warn(\"Continuous columns will not be normalised\")\n</pre> In\u00a0[7]: Copied! <pre>X_tab[:5]\n</pre> X_tab[:5] Out[7]: <pre>array([[ 1,  1,  1,  1,  1,  1,  1,  1,  1,  1, 26, 40],\n       [ 2,  2,  1,  2,  2,  2,  1,  2,  1,  1, 50, 40],\n       [ 2,  1,  1,  3,  2,  2,  1,  1,  2,  1, 39, 46],\n       [ 2,  3,  2,  4,  1,  2,  2,  1,  1,  1, 17, 10],\n       [ 3,  4,  2,  1,  1,  2,  1,  1,  1,  1, 32, 20]])</pre> In\u00a0[8]: Copied! <pre># We define a model that will act as the encoder in the encoder/decoder\n# architecture. This could be any of: TabMlp, TabResnet or TabNet\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=tab_preprocessor.continuous_cols,\n)\n</pre> # We define a model that will act as the encoder in the encoder/decoder # architecture. This could be any of: TabMlp, TabResnet or TabNet tab_mlp = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     continuous_cols=tab_preprocessor.continuous_cols, ) In\u00a0[9]: Copied! <pre>tab_mlp\n</pre> tab_mlp Out[9]: <pre>TabMlp(\n  (cat_embed): DiffSizeCatEmbeddings(\n    (embed_layers): ModuleDict(\n      (emb_layer_workclass): Embedding(10, 5, padding_idx=0)\n      (emb_layer_education): Embedding(17, 8, padding_idx=0)\n      (emb_layer_marital_status): Embedding(8, 5, padding_idx=0)\n      (emb_layer_occupation): Embedding(16, 7, padding_idx=0)\n      (emb_layer_relationship): Embedding(7, 4, padding_idx=0)\n      (emb_layer_race): Embedding(6, 4, padding_idx=0)\n      (emb_layer_gender): Embedding(3, 2, padding_idx=0)\n      (emb_layer_capital_gain): Embedding(124, 24, padding_idx=0)\n      (emb_layer_capital_loss): Embedding(98, 21, padding_idx=0)\n      (emb_layer_native_country): Embedding(42, 13, padding_idx=0)\n    )\n    (embedding_dropout): Dropout(p=0.0, inplace=False)\n  )\n  (cont_norm): Identity()\n  (encoder): MLP(\n    (mlp): Sequential(\n      (dense_layer_0): Sequential(\n        (0): Linear(in_features=95, out_features=200, bias=True)\n        (1): ReLU(inplace=True)\n        (2): Dropout(p=0.1, inplace=False)\n      )\n      (dense_layer_1): Sequential(\n        (0): Linear(in_features=200, out_features=100, bias=True)\n        (1): ReLU(inplace=True)\n        (2): Dropout(p=0.1, inplace=False)\n      )\n    )\n  )\n)</pre> In\u00a0[10]: Copied! <pre># If we do not pass a custom decoder, which is perfectly possible via the\n# decoder param,  the EncoderDecoderTrainer will automatically build a\n# decoder which will be the 'mirror' image of the encoder\nencoder_decoder_trainer = EncoderDecoderTrainer(encoder=tab_mlp)\n</pre> # If we do not pass a custom decoder, which is perfectly possible via the # decoder param,  the EncoderDecoderTrainer will automatically build a # decoder which will be the 'mirror' image of the encoder encoder_decoder_trainer = EncoderDecoderTrainer(encoder=tab_mlp) In\u00a0[11]: Copied! <pre># let's have a look to the encoder_decoder_model (aka ed_model)\nencoder_decoder_trainer.ed_model\n</pre> # let's have a look to the encoder_decoder_model (aka ed_model) encoder_decoder_trainer.ed_model Out[11]: <pre>EncoderDecoderModel(\n  (encoder): TabMlp(\n    (cat_embed): DiffSizeCatEmbeddings(\n      (embed_layers): ModuleDict(\n        (emb_layer_workclass): Embedding(10, 5, padding_idx=0)\n        (emb_layer_education): Embedding(17, 8, padding_idx=0)\n        (emb_layer_marital_status): Embedding(8, 5, padding_idx=0)\n        (emb_layer_occupation): Embedding(16, 7, padding_idx=0)\n        (emb_layer_relationship): Embedding(7, 4, padding_idx=0)\n        (emb_layer_race): Embedding(6, 4, padding_idx=0)\n        (emb_layer_gender): Embedding(3, 2, padding_idx=0)\n        (emb_layer_capital_gain): Embedding(124, 24, padding_idx=0)\n        (emb_layer_capital_loss): Embedding(98, 21, padding_idx=0)\n        (emb_layer_native_country): Embedding(42, 13, padding_idx=0)\n      )\n      (embedding_dropout): Dropout(p=0.0, inplace=False)\n    )\n    (cont_norm): Identity()\n    (encoder): MLP(\n      (mlp): Sequential(\n        (dense_layer_0): Sequential(\n          (0): Linear(in_features=95, out_features=200, bias=True)\n          (1): ReLU(inplace=True)\n          (2): Dropout(p=0.1, inplace=False)\n        )\n        (dense_layer_1): Sequential(\n          (0): Linear(in_features=200, out_features=100, bias=True)\n          (1): ReLU(inplace=True)\n          (2): Dropout(p=0.1, inplace=False)\n        )\n      )\n    )\n  )\n  (decoder): TabMlpDecoder(\n    (decoder): MLP(\n      (mlp): Sequential(\n        (dense_layer_0): Sequential(\n          (0): Linear(in_features=100, out_features=200, bias=True)\n          (1): ReLU(inplace=True)\n          (2): Dropout(p=0.1, inplace=False)\n        )\n        (dense_layer_1): Sequential(\n          (0): Linear(in_features=200, out_features=95, bias=True)\n          (1): ReLU(inplace=True)\n          (2): Dropout(p=0.1, inplace=False)\n        )\n      )\n    )\n  )\n  (masker): RandomObfuscator()\n)</pre> <p>Ignoring the <code>masker</code>, which just...well...masks, the <code>ed_model</code> consists of:</p> <ol> <li>An encoder model that is a <code>TabMlp</code> model that is in itself comprised by an Embedding layer (or rather a collection of them, referred as <code>cat_and_cont_embed</code>) and an encoder (a simple MLP, referred as <code>encoder</code>)</li> <li>A decoder which is just an \"inverted\" MLP (referred as <code>decoder</code>)</li> </ol> In\u00a0[12]: Copied! <pre># And we just...pretrain\nencoder_decoder_trainer.pretrain(X_tab, n_epochs=5, batch_size=256)\n</pre> # And we just...pretrain encoder_decoder_trainer.pretrain(X_tab, n_epochs=5, batch_size=256) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 82.90it/s, loss=4.07]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 89.87it/s, loss=3.09]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 92.86it/s, loss=2.53]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 91.24it/s, loss=2.09]\nepoch 5: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 91.38it/s, loss=1.78]\n</pre> <p>At this point we have two options, we could either save the model for later use or we could continue to supervised training. The latter is rather simple, after running:</p> <pre>encoder_decoder_trainer.pretrain(X_tab, n_epochs=5, batch_size=256)\n</pre> <p>you just have to</p> <pre>model = WideDeep(deeptabular=tab_mlp)\ntrainer = Trainer(model=model, objective=\"binary\", metrics=[Accuracy])\n\ntrainer.fit(X_tab=X_tab, target=target, n_epochs=5, batch_size=256)\n\n# And, you know...we get a test metric\nX_tab_te = tab_preprocessor.transform(df_te)\ntarget_te = df_te[target_col].values\n\npreds = trainer.predict(X_tab=X_tab_te)\ntest_acc = accuracy_score(target_te, preds)\n</pre> <p>Let's say that in any case, we are 'decent' scientists/people and we want to save the model:</p> In\u00a0[13]: Copied! <pre>encoder_decoder_trainer.save(\n    path=\"pretrained_weights\", model_filename=\"encoder_decoder_model.pt\"\n)\n</pre> encoder_decoder_trainer.save(     path=\"pretrained_weights\", model_filename=\"encoder_decoder_model.pt\" ) <p>some time has passed...</p> In\u00a0[14]: Copied! <pre>encoder_decoder_model = torch.load(\"pretrained_weights/encoder_decoder_model.pt\")\n</pre> encoder_decoder_model = torch.load(\"pretrained_weights/encoder_decoder_model.pt\") <p>Now, AND THIS IS IMPORTANT We have loaded the encoder AND the decoder. To proceed to the supervised training we ONLY need the encoder</p> In\u00a0[15]: Copied! <pre>pretrained_encoder = encoder_decoder_model.encoder\n</pre> pretrained_encoder = encoder_decoder_model.encoder In\u00a0[16]: Copied! <pre>pretrained_encoder\n</pre> pretrained_encoder Out[16]: <pre>TabMlp(\n  (cat_embed): DiffSizeCatEmbeddings(\n    (embed_layers): ModuleDict(\n      (emb_layer_workclass): Embedding(10, 5, padding_idx=0)\n      (emb_layer_education): Embedding(17, 8, padding_idx=0)\n      (emb_layer_marital_status): Embedding(8, 5, padding_idx=0)\n      (emb_layer_occupation): Embedding(16, 7, padding_idx=0)\n      (emb_layer_relationship): Embedding(7, 4, padding_idx=0)\n      (emb_layer_race): Embedding(6, 4, padding_idx=0)\n      (emb_layer_gender): Embedding(3, 2, padding_idx=0)\n      (emb_layer_capital_gain): Embedding(124, 24, padding_idx=0)\n      (emb_layer_capital_loss): Embedding(98, 21, padding_idx=0)\n      (emb_layer_native_country): Embedding(42, 13, padding_idx=0)\n    )\n    (embedding_dropout): Dropout(p=0.0, inplace=False)\n  )\n  (cont_norm): Identity()\n  (encoder): MLP(\n    (mlp): Sequential(\n      (dense_layer_0): Sequential(\n        (0): Linear(in_features=95, out_features=200, bias=True)\n        (1): ReLU(inplace=True)\n        (2): Dropout(p=0.1, inplace=False)\n      )\n      (dense_layer_1): Sequential(\n        (0): Linear(in_features=200, out_features=100, bias=True)\n        (1): ReLU(inplace=True)\n        (2): Dropout(p=0.1, inplace=False)\n      )\n    )\n  )\n)</pre> In\u00a0[17]: Copied! <pre># and as always, ANY supervised model in this library has to go throuth the WideDeep class:\nmodel = WideDeep(deeptabular=pretrained_encoder)\ntrainer = Trainer(model=model, objective=\"binary\", metrics=[Accuracy])\n\ntrainer.fit(X_tab=X_tab, target=target, n_epochs=5, batch_size=256)\n\nX_tab_te = tab_preprocessor.transform(df_te)\ntarget_te = df_te[target_col].values\n\npreds = trainer.predict(X_tab=X_tab_te)\ntest_acc = accuracy_score(target_te, preds)\nprint(test_acc)\n</pre> # and as always, ANY supervised model in this library has to go throuth the WideDeep class: model = WideDeep(deeptabular=pretrained_encoder) trainer = Trainer(model=model, objective=\"binary\", metrics=[Accuracy])  trainer.fit(X_tab=X_tab, target=target, n_epochs=5, batch_size=256)  X_tab_te = tab_preprocessor.transform(df_te) target_te = df_te[target_col].values  preds = trainer.predict(X_tab=X_tab_te) test_acc = accuracy_score(target_te, preds) print(test_acc) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 88.04it/s, loss=0.374, metrics={'acc': 0.8253}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 85.63it/s, loss=0.324, metrics={'acc': 0.8491}]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 87.56it/s, loss=0.301, metrics={'acc': 0.8608}]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:02&lt;00:00, 73.38it/s, loss=0.29, metrics={'acc': 0.8655}]\nepoch 5: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 78.68it/s, loss=0.284, metrics={'acc': 0.8686}]\npredict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 39/39 [00:00&lt;00:00, 173.02it/s]\n</pre> <pre>0.8730678677449074\n</pre> <p>As we mentioned before, we can also use a <code>TabResNet</code> or <code>TabNet</code> model and a custom decoder. Let's have a look:</p> In\u00a0[18]: Copied! <pre>from pytorch_widedeep.models import TabResnet as TabResnetEncoder, TabResnetDecoder\n</pre> from pytorch_widedeep.models import TabResnet as TabResnetEncoder, TabResnetDecoder In\u00a0[19]: Copied! <pre>resnet_encoder = TabResnetEncoder(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=continuous_cols,\n    blocks_dims=[200, 100, 100],\n)\n</pre> resnet_encoder = TabResnetEncoder(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     continuous_cols=continuous_cols,     blocks_dims=[200, 100, 100], ) <p>let's have a look to the model</p> In\u00a0[20]: Copied! <pre>resnet_encoder\n</pre> resnet_encoder Out[20]: <pre>TabResnet(\n  (cat_embed): DiffSizeCatEmbeddings(\n    (embed_layers): ModuleDict(\n      (emb_layer_workclass): Embedding(10, 5, padding_idx=0)\n      (emb_layer_education): Embedding(17, 8, padding_idx=0)\n      (emb_layer_marital_status): Embedding(8, 5, padding_idx=0)\n      (emb_layer_occupation): Embedding(16, 7, padding_idx=0)\n      (emb_layer_relationship): Embedding(7, 4, padding_idx=0)\n      (emb_layer_race): Embedding(6, 4, padding_idx=0)\n      (emb_layer_gender): Embedding(3, 2, padding_idx=0)\n      (emb_layer_capital_gain): Embedding(124, 24, padding_idx=0)\n      (emb_layer_capital_loss): Embedding(98, 21, padding_idx=0)\n      (emb_layer_native_country): Embedding(42, 13, padding_idx=0)\n    )\n    (embedding_dropout): Dropout(p=0.0, inplace=False)\n  )\n  (cont_norm): Identity()\n  (encoder): DenseResnet(\n    (dense_resnet): Sequential(\n      (lin_inp): Linear(in_features=95, out_features=200, bias=False)\n      (bn_inp): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n      (block_0): BasicBlock(\n        (resize): Sequential(\n          (0): Linear(in_features=200, out_features=100, bias=False)\n          (1): BatchNorm1d(100, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        )\n        (lin1): Linear(in_features=200, out_features=100, bias=False)\n        (bn1): BatchNorm1d(100, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (leaky_relu): LeakyReLU(negative_slope=0.01, inplace=True)\n        (dp): Dropout(p=0.1, inplace=False)\n        (lin2): Linear(in_features=100, out_features=100, bias=False)\n        (bn2): BatchNorm1d(100, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n      )\n      (block_1): BasicBlock(\n        (lin1): Linear(in_features=100, out_features=100, bias=False)\n        (bn1): BatchNorm1d(100, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (leaky_relu): LeakyReLU(negative_slope=0.01, inplace=True)\n        (dp): Dropout(p=0.1, inplace=False)\n        (lin2): Linear(in_features=100, out_features=100, bias=False)\n        (bn2): BatchNorm1d(100, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n      )\n    )\n  )\n)</pre> <p>As we can see, the tensor we are trying to reconstruct, the embeddings, is of size <code>94</code> (this number is stored in the attribute: <code>esnet_encoder.cat_and_cont_embed.output_dim</code>), with that information we could build or own decoder as:</p> In\u00a0[21]: Copied! <pre># for all possible params see the docs\nresnet_decoder = TabResnetDecoder(\n    embed_dim=resnet_encoder.cat_out_dim + resnet_encoder.cont_out_dim,\n    blocks_dims=[100, 100, 200],\n)\n</pre> # for all possible params see the docs resnet_decoder = TabResnetDecoder(     embed_dim=resnet_encoder.cat_out_dim + resnet_encoder.cont_out_dim,     blocks_dims=[100, 100, 200], ) In\u00a0[22]: Copied! <pre>resnet_decoder\n</pre> resnet_decoder Out[22]: <pre>TabResnetDecoder(\n  (decoder): DenseResnet(\n    (dense_resnet): Sequential(\n      (block_0): BasicBlock(\n        (lin1): Linear(in_features=100, out_features=100, bias=False)\n        (bn1): BatchNorm1d(100, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (leaky_relu): LeakyReLU(negative_slope=0.01, inplace=True)\n        (dp): Dropout(p=0.1, inplace=False)\n        (lin2): Linear(in_features=100, out_features=100, bias=False)\n        (bn2): BatchNorm1d(100, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n      )\n      (block_1): BasicBlock(\n        (resize): Sequential(\n          (0): Linear(in_features=100, out_features=200, bias=False)\n          (1): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        )\n        (lin1): Linear(in_features=100, out_features=200, bias=False)\n        (bn1): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (leaky_relu): LeakyReLU(negative_slope=0.01, inplace=True)\n        (dp): Dropout(p=0.1, inplace=False)\n        (lin2): Linear(in_features=200, out_features=200, bias=False)\n        (bn2): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n      )\n    )\n  )\n  (reconstruction_layer): Linear(in_features=200, out_features=95, bias=False)\n)</pre> <p>and now:</p> In\u00a0[23]: Copied! <pre>ec_trainer = EncoderDecoderTrainer(\n    encoder=resnet_encoder,\n    decoder=resnet_decoder,\n    masked_prob=0.2,\n)\nec_trainer.pretrain(X_tab, n_epochs=5, batch_size=256)\n</pre> ec_trainer = EncoderDecoderTrainer(     encoder=resnet_encoder,     decoder=resnet_decoder,     masked_prob=0.2, ) ec_trainer.pretrain(X_tab, n_epochs=5, batch_size=256) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:03&lt;00:00, 46.89it/s, loss=1.52]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:03&lt;00:00, 46.78it/s, loss=0.81]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:03&lt;00:00, 39.82it/s, loss=0.56]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:03&lt;00:00, 46.73it/s, loss=0.417]\nepoch 5: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:03&lt;00:00, 46.24it/s, loss=0.329]\n</pre> In\u00a0[24]: Copied! <pre># and as always, ANY supervised model in this library has to go throuth the WideDeep class:\nmodel = WideDeep(deeptabular=resnet_encoder)\ntrainer = Trainer(model=model, objective=\"binary\", metrics=[Accuracy])\n\ntrainer.fit(X_tab=X_tab, target=target, n_epochs=5, batch_size=256)\n\nX_tab_te = tab_preprocessor.transform(df_te)\ntarget_te = df_te[target_col].values\n\npreds = trainer.predict(X_tab=X_tab_te)\ntest_acc = accuracy_score(target_te, preds)\nprint(test_acc)\n</pre> # and as always, ANY supervised model in this library has to go throuth the WideDeep class: model = WideDeep(deeptabular=resnet_encoder) trainer = Trainer(model=model, objective=\"binary\", metrics=[Accuracy])  trainer.fit(X_tab=X_tab, target=target, n_epochs=5, batch_size=256)  X_tab_te = tab_preprocessor.transform(df_te) target_te = df_te[target_col].values  preds = trainer.predict(X_tab=X_tab_te) test_acc = accuracy_score(target_te, preds) print(test_acc) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:02&lt;00:00, 58.63it/s, loss=0.335, metrics={'acc': 0.8442}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:02&lt;00:00, 58.02it/s, loss=0.296, metrics={'acc': 0.864}]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:02&lt;00:00, 55.91it/s, loss=0.283, metrics={'acc': 0.8687}]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:02&lt;00:00, 55.00it/s, loss=0.276, metrics={'acc': 0.871}]\nepoch 5: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:02&lt;00:00, 51.95it/s, loss=0.272, metrics={'acc': 0.8732}]\npredict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 39/39 [00:00&lt;00:00, 120.15it/s]\n</pre> <pre>0.8725560446309756\n</pre> In\u00a0[\u00a0]: Copied! <pre>\n</pre>"},{"location":"examples/15_Self_Supervised_Pretraning_pt1.html#self-supervised-pretraining-for-tabular-data","title":"Self Supervised Pretraining for Tabular Data\u00b6","text":"<p>We have implemented two Self Supervised Pre-training routines that allow the user to pre-train all tabular models in the library with the exception of the TabPerceiver (which is a special monster).</p> <p>The two routines implemented are illustrated in the figures below. The 1st is from TabNet: Attentive Interpretable Tabular Learning and is designed for models that do not use transformer-based architectures, while the second is from SAINT: Improved Neural Networks for Tabular Data via Row Attention and Contrastive Pre-Training, and is designed for models that use transformer-based architectures.</p> <p> </p> <p>Fig 1. Figure 2 in their paper. I have included de original caption in case is useful, althought the Figure itself is pretty self explanatory</p> <p> </p> <p>Fig 2. Figure 1 in their paper. Here the caption is necessary \ud83d\ude0f</p> <p>It is beyond the scope of this notebook to explain in detail those implementations. Therefore, we strongly recommend the user to go and read the papers if this functionality is of interest to her/him.</p> <p>One thing is worth noticing however. As seen in Fig 1(the TabNet paper's Fig 2) the masking of the input features happens in the feature space. However, the implementation in this library is inspired by that at the dreamquark-ai repo, which is in itself inspired by the original implementation (by the way, at this point I will write it once again. All TabNet related things in this library are inspired when not directly based in the code in that repo, therefore, ALL CREDIT TO THE GUYS AT dreamquark-ai).</p> <p>In that implementation the masking happens in the embedding space, and currently does not mask the entire embedding (i.e. categorical feature). We decided to release as it is in this version and we will implement the exact same process described in the paper in future releases.</p> <p>Having said all of the above let's see how to use self supervision for tabular data with <code>pytorch-widedeep</code>. We will concentrate in this notebook on the 1st of the two approaches (the 'TabNet approach'). For details on the second approach please see <code>16_Self_Supervised_Pretraning_pt2</code>.</p>"},{"location":"examples/15_Self_Supervised_Pretraning_pt1.html#self-supervision-for-non-transformer-based-models","title":"Self Supervision for non-transformer-based models..\u00b6","text":"<p>...or in general, for models where the embeddigns can have all different dimensions. In this library, these are: <code>TabMlp</code>, <code>TabResNet</code> and <code>TabNet</code></p> <p>As shown in Figure, this is an encoder-encoder approach where we learn to predict values in the incoming data that have been masked. However, as I mentioned before, our implementation is a bit different, and the masking occurs in th embedding space.</p> <p>Nonetheless, the code below illustrates how to use this encoder-decoder approach with <code>pytorch-widedeep</code></p>"},{"location":"examples/15_Self_Supervised_Pretraning_pt2.html","title":"15 Self Supervised Pretraning pt2","text":"In\u00a0[1]: Copied! <pre>import torch\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.model_selection import train_test_split\n\nfrom pytorch_widedeep import Trainer\nfrom pytorch_widedeep.models import WideDeep, FTTransformer\nfrom pytorch_widedeep.metrics import Accuracy\nfrom pytorch_widedeep.datasets import load_adult\nfrom pytorch_widedeep.preprocessing import TabPreprocessor\nfrom pytorch_widedeep.self_supervised_training import (\n    ContrastiveDenoisingTrainer,\n)\n</pre> import torch from sklearn.metrics import accuracy_score from sklearn.model_selection import train_test_split  from pytorch_widedeep import Trainer from pytorch_widedeep.models import WideDeep, FTTransformer from pytorch_widedeep.metrics import Accuracy from pytorch_widedeep.datasets import load_adult from pytorch_widedeep.preprocessing import TabPreprocessor from pytorch_widedeep.self_supervised_training import (     ContrastiveDenoisingTrainer, ) In\u00a0[2]: Copied! <pre>df = load_adult(as_frame=True)\ndf.columns = [c.replace(\"-\", \"_\") for c in df.columns]\ndf[\"income_label\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int)\ndf.drop(\"income\", axis=1, inplace=True)\n\n# one could chose to use a validation set for early stopping, hyperparam\n# optimization, etc. This is just an example, so we simply use train/test\n# split\ndf_tr, df_te = train_test_split(df, test_size=0.2, stratify=df.income_label)\n\ncat_embed_cols = [\n    \"workclass\",\n    \"education\",\n    \"marital_status\",\n    \"occupation\",\n    \"relationship\",\n    \"race\",\n    \"gender\",\n    \"capital_gain\",\n    \"capital_loss\",\n    \"native_country\",\n]\ncontinuous_cols = [\"age\", \"hours_per_week\"]\ntarget_col = \"income_label\"\n\ntab_preprocessor = TabPreprocessor(\n    cat_embed_cols=cat_embed_cols,\n    continuous_cols=continuous_cols,\n    with_attention=True,\n    with_cls_token=True,  # this is optional\n)\nX_tab = tab_preprocessor.fit_transform(df_tr)\ntarget = df_tr[target_col].values\n</pre> df = load_adult(as_frame=True) df.columns = [c.replace(\"-\", \"_\") for c in df.columns] df[\"income_label\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int) df.drop(\"income\", axis=1, inplace=True)  # one could chose to use a validation set for early stopping, hyperparam # optimization, etc. This is just an example, so we simply use train/test # split df_tr, df_te = train_test_split(df, test_size=0.2, stratify=df.income_label)  cat_embed_cols = [     \"workclass\",     \"education\",     \"marital_status\",     \"occupation\",     \"relationship\",     \"race\",     \"gender\",     \"capital_gain\",     \"capital_loss\",     \"native_country\", ] continuous_cols = [\"age\", \"hours_per_week\"] target_col = \"income_label\"  tab_preprocessor = TabPreprocessor(     cat_embed_cols=cat_embed_cols,     continuous_cols=continuous_cols,     with_attention=True,     with_cls_token=True,  # this is optional ) X_tab = tab_preprocessor.fit_transform(df_tr) target = df_tr[target_col].values <pre>/Users/javierrodriguezzaurin/Projects/pytorch-widedeep/pytorch_widedeep/preprocessing/tab_preprocessor.py:358: UserWarning: Continuous columns will not be normalised\n  warnings.warn(\"Continuous columns will not be normalised\")\n</pre> In\u00a0[3]: Copied! <pre>ft_transformer = FTTransformer(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=tab_preprocessor.continuous_cols,\n    embed_continuous_method=\"standard\",\n    input_dim=32,\n    kv_compression_factor=0.5,\n    n_blocks=3,\n    n_heads=4,\n)\n</pre> ft_transformer = FTTransformer(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     continuous_cols=tab_preprocessor.continuous_cols,     embed_continuous_method=\"standard\",     input_dim=32,     kv_compression_factor=0.5,     n_blocks=3,     n_heads=4, ) In\u00a0[4]: Copied! <pre># for a full list of the params for the the ContrastiveDenoisingTrainer (which are many) please see the docs.\n# Note that using these params involves some knowledge of the routine and the architecture of the model used\ncontrastive_denoising_trainer = ContrastiveDenoisingTrainer(\n    model=ft_transformer,\n    preprocessor=tab_preprocessor,\n)\ncontrastive_denoising_trainer.pretrain(X_tab, n_epochs=5, batch_size=256)\n</pre> # for a full list of the params for the the ContrastiveDenoisingTrainer (which are many) please see the docs. # Note that using these params involves some knowledge of the routine and the architecture of the model used contrastive_denoising_trainer = ContrastiveDenoisingTrainer(     model=ft_transformer,     preprocessor=tab_preprocessor, ) contrastive_denoising_trainer.pretrain(X_tab, n_epochs=5, batch_size=256) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:13&lt;00:00, 11.73it/s, loss=579]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:12&lt;00:00, 12.56it/s, loss=143]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:12&lt;00:00, 12.49it/s, loss=141]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:11&lt;00:00, 12.77it/s, loss=138]\nepoch 5: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:11&lt;00:00, 13.29it/s, loss=137]\n</pre> In\u00a0[5]: Copied! <pre>contrastive_denoising_trainer.save(\n    path=\"pretrained_weights\", model_filename=\"contrastive_denoising_model.pt\"\n)\n</pre> contrastive_denoising_trainer.save(     path=\"pretrained_weights\", model_filename=\"contrastive_denoising_model.pt\" ) <p>some time has passed</p> In\u00a0[6]: Copied! <pre># some time has passed, we load the model with torch as usual:\ncontrastive_denoising_model = torch.load(\n    \"pretrained_weights/contrastive_denoising_model.pt\"\n)\n</pre> # some time has passed, we load the model with torch as usual: contrastive_denoising_model = torch.load(     \"pretrained_weights/contrastive_denoising_model.pt\" ) <p>NOW, AND THIS IS IMPORTANT! We have loaded the entire contrastive, denoising model. To proceed to the supervised training we ONLY need the attention-based model, which is the 'model' attribute of the trainer, let's have a look</p> In\u00a0[7]: Copied! <pre>contrastive_denoising_model.model\n</pre> contrastive_denoising_model.model Out[7]: <pre>FTTransformer(\n  (cat_embed): SameSizeCatEmbeddings(\n    (embed): Embedding(323, 32, padding_idx=0)\n    (dropout): Dropout(p=0.0, inplace=False)\n  )\n  (cont_norm): Identity()\n  (cont_embed): ContEmbeddings(\n    INFO: [ContLinear = weight(n_cont_cols, embed_dim) + bias(n_cont_cols, embed_dim)]\n    (linear): ContLinear(n_cont_cols=2, embed_dim=32, embed_dropout=0.0)\n    (dropout): Dropout(p=0.0, inplace=False)\n  )\n  (encoder): Sequential(\n    (fttransformer_block0): FTTransformerEncoder(\n      (attn): LinearAttentionLinformer(\n        (dropout): Dropout(p=0.2, inplace=False)\n        (qkv_proj): Linear(in_features=32, out_features=96, bias=False)\n        (out_proj): Linear(in_features=32, out_features=32, bias=False)\n      )\n      (ff): FeedForward(\n        (w_1): Linear(in_features=32, out_features=84, bias=True)\n        (w_2): Linear(in_features=42, out_features=32, bias=True)\n        (dropout): Dropout(p=0.1, inplace=False)\n        (activation): REGLU()\n      )\n      (attn_normadd): NormAdd(\n        (dropout): Dropout(p=0.2, inplace=False)\n        (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n      )\n      (ff_normadd): NormAdd(\n        (dropout): Dropout(p=0.1, inplace=False)\n        (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n      )\n    )\n    (fttransformer_block1): FTTransformerEncoder(\n      (attn): LinearAttentionLinformer(\n        (dropout): Dropout(p=0.2, inplace=False)\n        (qkv_proj): Linear(in_features=32, out_features=96, bias=False)\n        (out_proj): Linear(in_features=32, out_features=32, bias=False)\n      )\n      (ff): FeedForward(\n        (w_1): Linear(in_features=32, out_features=84, bias=True)\n        (w_2): Linear(in_features=42, out_features=32, bias=True)\n        (dropout): Dropout(p=0.1, inplace=False)\n        (activation): REGLU()\n      )\n      (attn_normadd): NormAdd(\n        (dropout): Dropout(p=0.2, inplace=False)\n        (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n      )\n      (ff_normadd): NormAdd(\n        (dropout): Dropout(p=0.1, inplace=False)\n        (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n      )\n    )\n    (fttransformer_block2): FTTransformerEncoder(\n      (attn): LinearAttentionLinformer(\n        (dropout): Dropout(p=0.2, inplace=False)\n        (qkv_proj): Linear(in_features=32, out_features=96, bias=False)\n        (out_proj): Linear(in_features=32, out_features=32, bias=False)\n      )\n      (ff): FeedForward(\n        (w_1): Linear(in_features=32, out_features=84, bias=True)\n        (w_2): Linear(in_features=42, out_features=32, bias=True)\n        (dropout): Dropout(p=0.1, inplace=False)\n        (activation): REGLU()\n      )\n      (attn_normadd): NormAdd(\n        (dropout): Dropout(p=0.2, inplace=False)\n        (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n      )\n      (ff_normadd): NormAdd(\n        (dropout): Dropout(p=0.1, inplace=False)\n        (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n      )\n    )\n  )\n)</pre> In\u00a0[8]: Copied! <pre>pretrained_model = contrastive_denoising_model.model\n</pre> pretrained_model = contrastive_denoising_model.model In\u00a0[9]: Copied! <pre># and as always, ANY supervised model in this library has to go throuth the WideDeep class:\nmodel = WideDeep(deeptabular=pretrained_model)\ntrainer = Trainer(model=model, objective=\"binary\", metrics=[Accuracy])\n\ntrainer.fit(X_tab=X_tab, target=target, n_epochs=5, batch_size=256)\n\n# And, you know...we get a test metric\nX_tab_te = tab_preprocessor.transform(df_te)\ntarget_te = df_te[target_col].values\n\npreds = trainer.predict(X_tab=X_tab_te)\ntest_acc = accuracy_score(target_te, preds)\nprint(test_acc)\n</pre> # and as always, ANY supervised model in this library has to go throuth the WideDeep class: model = WideDeep(deeptabular=pretrained_model) trainer = Trainer(model=model, objective=\"binary\", metrics=[Accuracy])  trainer.fit(X_tab=X_tab, target=target, n_epochs=5, batch_size=256)  # And, you know...we get a test metric X_tab_te = tab_preprocessor.transform(df_te) target_te = df_te[target_col].values  preds = trainer.predict(X_tab=X_tab_te) test_acc = accuracy_score(target_te, preds) print(test_acc) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:05&lt;00:00, 27.19it/s, loss=0.383, metrics={'acc': 0.8176}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:05&lt;00:00, 26.08it/s, loss=0.325, metrics={'acc': 0.8502}]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:05&lt;00:00, 26.56it/s, loss=0.306, metrics={'acc': 0.8601}]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:05&lt;00:00, 27.41it/s, loss=0.295, metrics={'acc': 0.8641}]\nepoch 5: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:06&lt;00:00, 24.70it/s, loss=0.289, metrics={'acc': 0.8656}]\npredict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 39/39 [00:00&lt;00:00, 97.26it/s]</pre> <pre>0.8695874705701709\n</pre> <pre>\n</pre> In\u00a0[\u00a0]: Copied! <pre>\n</pre>"},{"location":"examples/15_Self_Supervised_Pretraning_pt2.html#self-supervised-pretraining-for-tabular-data","title":"Self Supervised Pretraining for Tabular Data\u00b6","text":"<p>We have implemented two Self Supervised Pre-training routines that allow the user to pre-train all tabular models in the library with the exception of the TabPerceiver (which is a special monster).</p> <p>The two routines implemented are illustrated in the figures below. The 1st is from TabNet: Attentive Interpretable Tabular Learning and is designed for models that do not use transformer-based architectures, while the second is from SAINT: Improved Neural Networks for Tabular Data via Row Attention and Contrastive Pre-Training, and is designed for models that use transformer-based architectures.</p> <p> </p> <p>Fig 1. Figure 2 in their paper. I have included de original caption in case is useful, althought the Figure itself is pretty self explanatory</p> <p> </p> <p>Fig 2. Figure 1 in their paper. Here the caption is necessary \ud83d\ude0f</p> <p>It is beyond the scope of this notebook to explain in detail those implementations. Therefore, we strongly recommend the user to go and read the papers if this functionality is of interest to her/him.</p> <p>One thing is worth noticing however. As seen in Fig 1(the TabNet paper's Fig 2) the masking of the input features happens in the feature space. However, the implementation in this library is inspired by that at the dreamquark-ai repo, which is in itself inspired by the original implementation (by the way, at this point I will write it once again. All TabNet related things in this library are inspired when not directly based in the code in that repo, therefore, ALL CREDIT TO THE GUYS AT dreamquark-ai).</p> <p>In that implementation the masking happens in the embedding space, and currently does not mask the entire embedding (i.e. categorical feature). We decided to release as it is in this version and we will implement the exact same process described in the paper in future releases.</p> <p>Having said all of the above let's see how to use self supervision for tabular data with <code>pytorch-widedeep</code>. We will concentrate in this notebook on the 2nd of the two approaches (the 'SAINT approach'). For details on the 1st approach (the 'TabNet' approach) please see <code>16_Self_Supervised_Pretraning_pt1</code>.</p>"},{"location":"examples/15_Self_Supervised_Pretraning_pt2.html#self-supervision-transformer-based-models","title":"Self Supervision transformer-based models..\u00b6","text":"<p>...or in general, for models where the embeddigns have all the same dimensions. In this library, these are:</p> <ul> <li>TabTransformer</li> <li>FTTransformer</li> <li>SAINT</li> <li>TabFastFormer</li> </ul> <p>Note that there is one additional Transformer-based model, the <code>TabPerceiver</code>, however this is a \"particular\" model and at the moment we do not support self supervision for it, but it will come.</p> <p>Let see at one example using the <code>FTTransformer</code>.</p>"},{"location":"examples/16_Usign_a_custom_hugging_face_model.html","title":"16 Usign a custom hugging face model","text":"In\u00a0[1]: Copied! <pre>import numpy as np\nimport torch\nimport lightgbm as lgb\nfrom lightgbm import Dataset as lgbDataset\nfrom scipy.sparse import hstack, csr_matrix\nfrom sklearn.metrics import (\n    f1_score,\n    recall_score,\n    accuracy_score,\n    precision_score,\n    confusion_matrix,\n)\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.feature_extraction.text import TfidfVectorizer\n\nfrom torch import Tensor, nn\nfrom transformers import DistilBertModel, DistilBertTokenizer\nfrom pytorch_widedeep import Trainer\nfrom pytorch_widedeep.models import TabMlp, BasicRNN, WideDeep\nfrom pytorch_widedeep.metrics import F1Score, Accuracy\nfrom pytorch_widedeep.utils import Tokenizer, LabelEncoder\nfrom pytorch_widedeep.preprocessing import TextPreprocessor, TabPreprocessor\nfrom pytorch_widedeep.datasets import load_womens_ecommerce\nfrom pytorch_widedeep.utils.fastai_transforms import (\n    fix_html,\n    spec_add_spaces,\n    rm_useless_spaces,\n)\n</pre> import numpy as np import torch import lightgbm as lgb from lightgbm import Dataset as lgbDataset from scipy.sparse import hstack, csr_matrix from sklearn.metrics import (     f1_score,     recall_score,     accuracy_score,     precision_score,     confusion_matrix, ) from sklearn.model_selection import train_test_split from sklearn.feature_extraction.text import TfidfVectorizer  from torch import Tensor, nn from transformers import DistilBertModel, DistilBertTokenizer from pytorch_widedeep import Trainer from pytorch_widedeep.models import TabMlp, BasicRNN, WideDeep from pytorch_widedeep.metrics import F1Score, Accuracy from pytorch_widedeep.utils import Tokenizer, LabelEncoder from pytorch_widedeep.preprocessing import TextPreprocessor, TabPreprocessor from pytorch_widedeep.datasets import load_womens_ecommerce from pytorch_widedeep.utils.fastai_transforms import (     fix_html,     spec_add_spaces,     rm_useless_spaces, ) <p>Let's load the data and have a look:</p> In\u00a0[2]: Copied! <pre>df = load_womens_ecommerce(as_frame=True)\n\ndf.columns = [c.replace(\" \", \"_\").lower() for c in df.columns]\n\n# classes from [0,num_class)\ndf[\"rating\"] = (df[\"rating\"] - 1).astype(\"int64\")\n\n# group reviews with 1 and 2 scores into one class\ndf.loc[df.rating == 0, \"rating\"] = 1\n\n# and back again to [0,num_class)\ndf[\"rating\"] = (df[\"rating\"] - 1).astype(\"int64\")\n\n# drop short reviews\ndf = df[~df.review_text.isna()]\ndf[\"review_length\"] = df.review_text.apply(lambda x: len(x.split(\" \")))\ndf = df[df.review_length &gt;= 5]\ndf = df.drop(\"review_length\", axis=1).reset_index(drop=True)\n</pre> df = load_womens_ecommerce(as_frame=True)  df.columns = [c.replace(\" \", \"_\").lower() for c in df.columns]  # classes from [0,num_class) df[\"rating\"] = (df[\"rating\"] - 1).astype(\"int64\")  # group reviews with 1 and 2 scores into one class df.loc[df.rating == 0, \"rating\"] = 1  # and back again to [0,num_class) df[\"rating\"] = (df[\"rating\"] - 1).astype(\"int64\")  # drop short reviews df = df[~df.review_text.isna()] df[\"review_length\"] = df.review_text.apply(lambda x: len(x.split(\" \"))) df = df[df.review_length &gt;= 5] df = df.drop(\"review_length\", axis=1).reset_index(drop=True) In\u00a0[3]: Copied! <pre>df.head()\n</pre> df.head() Out[3]: clothing_id age title review_text rating recommended_ind positive_feedback_count division_name department_name class_name 0 767 33 None Absolutely wonderful - silky and sexy and comf... 2 1 0 Initmates Intimate Intimates 1 1080 34 None Love this dress!  it's sooo pretty.  i happene... 3 1 4 General Dresses Dresses 2 1077 60 Some major design flaws I had such high hopes for this dress and reall... 1 0 0 General Dresses Dresses 3 1049 50 My favorite buy! I love, love, love this jumpsuit. it's fun, fl... 3 1 0 General Petite Bottoms Pants 4 847 47 Flattering shirt This shirt is very flattering to all due to th... 3 1 6 General Tops Blouses <p>So, we will use the <code>review_text</code> column to predict the <code>rating</code>. Later on, we will try to combine it with some other columns (like <code>division_name</code> and <code>age</code>) see if these help.</p> <p>Let's first have a look to the distribution of ratings</p> In\u00a0[4]: Copied! <pre>df.rating.value_counts()\n</pre> df.rating.value_counts() Out[4]: <pre>rating\n3    12515\n2     4904\n1     2820\n0     2369\nName: count, dtype: int64</pre> <p>This shows that we could have perhaps grouped rating scores of 1, 2 and 3 into 1...but anyway, let's just move on with those 4 classes.</p> <p>We are not going to carry any hyperparameter optimization here, so, we will only need a train and a test set (i.e.  no need of a validation set for the example in this notebook)</p> In\u00a0[5]: Copied! <pre>train, test = train_test_split(df, train_size=0.8, random_state=1, stratify=df.rating)\n</pre> train, test = train_test_split(df, train_size=0.8, random_state=1, stratify=df.rating) <p>Let's see what we have to beat. What metrics would we obtain if we always predict the most common rating (3)?</p> In\u00a0[6]: Copied! <pre>most_common_pred = [train.rating.value_counts().index[0]] * len(test)\n\nmost_common_acc = accuracy_score(test.rating, most_common_pred)\nmost_common_f1 = f1_score(test.rating, most_common_pred, average=\"weighted\")\n</pre> most_common_pred = [train.rating.value_counts().index[0]] * len(test)  most_common_acc = accuracy_score(test.rating, most_common_pred) most_common_f1 = f1_score(test.rating, most_common_pred, average=\"weighted\") In\u00a0[7]: Copied! <pre>print(f\"Accuracy: {most_common_acc}. F1 Score: {most_common_f1}\")\n</pre> print(f\"Accuracy: {most_common_acc}. F1 Score: {most_common_f1}\") <pre>Accuracy: 0.553516143299425. F1 Score: 0.3944344218301668\n</pre> <p>ok, these are our \"baseline\" metrics.</p> <p>Let's start by using simply tf-idf + lightGBM</p> In\u00a0[8]: Copied! <pre># ?Tokenizer\n</pre> # ?Tokenizer In\u00a0[9]: Copied! <pre># this Tokenizer is part of our utils module but of course, any valid tokenizer can be used here.\n\n# When using notebooks there seems to be an issue related with multiprocessing (and sometimes tqdm)\n# that can only be solved by using only one CPU\ntok = Tokenizer(n_cpus=1)\ntok_reviews_tr = tok.process_all(train.review_text.tolist())\ntok_reviews_te = tok.process_all(test.review_text.tolist())\n</pre> # this Tokenizer is part of our utils module but of course, any valid tokenizer can be used here.  # When using notebooks there seems to be an issue related with multiprocessing (and sometimes tqdm) # that can only be solved by using only one CPU tok = Tokenizer(n_cpus=1) tok_reviews_tr = tok.process_all(train.review_text.tolist()) tok_reviews_te = tok.process_all(test.review_text.tolist()) In\u00a0[10]: Copied! <pre>vectorizer = TfidfVectorizer(\n    max_features=5000, preprocessor=lambda x: x, tokenizer=lambda x: x, min_df=5\n)\n\nX_text_tr = vectorizer.fit_transform(tok_reviews_tr)\nX_text_te = vectorizer.transform(tok_reviews_te)\n</pre> vectorizer = TfidfVectorizer(     max_features=5000, preprocessor=lambda x: x, tokenizer=lambda x: x, min_df=5 )  X_text_tr = vectorizer.fit_transform(tok_reviews_tr) X_text_te = vectorizer.transform(tok_reviews_te) <pre>/Users/javierrodriguezzaurin/.pyenv/versions/3.10.13/envs/widedeep310/lib/python3.10/site-packages/sklearn/feature_extraction/text.py:525: UserWarning: The parameter 'token_pattern' will not be used since 'tokenizer' is not None'\n  warnings.warn(\n</pre> In\u00a0[11]: Copied! <pre>X_text_tr\n</pre> X_text_tr Out[11]: <pre>&lt;18086x4566 sparse matrix of type '&lt;class 'numpy.float64'&gt;'\n\twith 884074 stored elements in Compressed Sparse Row format&gt;</pre> <p>We now move our matrices to lightGBM <code>Dataset</code> format</p> In\u00a0[12]: Copied! <pre>lgbtrain_text = lgbDataset(\n    X_text_tr,\n    train.rating.values,\n    free_raw_data=False,\n)\n\nlgbtest_text = lgbDataset(\n    X_text_te,\n    test.rating.values,\n    reference=lgbtrain_text,\n    free_raw_data=False,\n)\n</pre> lgbtrain_text = lgbDataset(     X_text_tr,     train.rating.values,     free_raw_data=False, )  lgbtest_text = lgbDataset(     X_text_te,     test.rating.values,     reference=lgbtrain_text,     free_raw_data=False, ) <p>and off we go. By the way, I think as we run the next cell, we should appreciate how fast lightGBM runs. Yes, the input is a sparse matrix, but still, trains on 18086x4566 in a matter of secs</p> In\u00a0[\u00a0]: Copied! <pre>lgb_text_model = lgb.train(\n    {\"objective\": \"multiclass\", \"num_classes\": 4},\n    lgbtrain_text,\n    valid_sets=[lgbtest_text, lgbtrain_text],\n    valid_names=[\"test\", \"train\"],\n)\n</pre> lgb_text_model = lgb.train(     {\"objective\": \"multiclass\", \"num_classes\": 4},     lgbtrain_text,     valid_sets=[lgbtest_text, lgbtrain_text],     valid_names=[\"test\", \"train\"], ) In\u00a0[14]: Copied! <pre>preds_text = lgb_text_model.predict(X_text_te)\npred_text_class = np.argmax(preds_text, 1)\n</pre> preds_text = lgb_text_model.predict(X_text_te) pred_text_class = np.argmax(preds_text, 1) In\u00a0[15]: Copied! <pre>acc_text = accuracy_score(lgbtest_text.label, pred_text_class)\nf1_text = f1_score(lgbtest_text.label, pred_text_class, average=\"weighted\")\ncm_text = confusion_matrix(lgbtest_text.label, pred_text_class)\n</pre> acc_text = accuracy_score(lgbtest_text.label, pred_text_class) f1_text = f1_score(lgbtest_text.label, pred_text_class, average=\"weighted\") cm_text = confusion_matrix(lgbtest_text.label, pred_text_class) In\u00a0[16]: Copied! <pre>print(f\"LightGBM Accuracy: {acc_text}. LightGBM F1 Score: {f1_text}\")\n</pre> print(f\"LightGBM Accuracy: {acc_text}. LightGBM F1 Score: {f1_text}\") <pre>LightGBM Accuracy: 0.6444051304732419. LightGBM F1 Score: 0.617154488246181\n</pre> In\u00a0[17]: Copied! <pre>print(f\"LightGBM Confusion Matrix: \\n {cm_text}\")\n</pre> print(f\"LightGBM Confusion Matrix: \\n {cm_text}\") <pre>LightGBM Confusion Matrix: \n [[ 199  135   61   79]\n [ 123  169  149  123]\n [  30   94  279  578]\n [  16   30  190 2267]]\n</pre> <p>Ok, so, with no hyperparameter optimization lightGBM gets an accuracy of 0.64 and a F1 score of 0.62. This is significantly better than predicting always the most popular.</p> <p>Let's see if in this implementation, some additional features, like <code>age</code> or <code>class_name</code> are of any help</p> In\u00a0[18]: Copied! <pre>tab_cols = [\n    \"age\",\n    \"division_name\",\n    \"department_name\",\n    \"class_name\",\n]\n\nfor tab_df in [train, test]:\n    for c in [\"division_name\", \"department_name\", \"class_name\"]:\n        tab_df[c] = tab_df[c].str.lower()\n        tab_df[c].fillna(\"missing\", inplace=True)\n</pre> tab_cols = [     \"age\",     \"division_name\",     \"department_name\",     \"class_name\", ]  for tab_df in [train, test]:     for c in [\"division_name\", \"department_name\", \"class_name\"]:         tab_df[c] = tab_df[c].str.lower()         tab_df[c].fillna(\"missing\", inplace=True) In\u00a0[19]: Copied! <pre># This is our LabelEncoder. A class that is designed to work with the models in this library but\n# can be used for general purposes\nle = LabelEncoder(columns_to_encode=[\"division_name\", \"department_name\", \"class_name\"])\ntrain_tab_le = le.fit_transform(train)\ntest_tab_le = le.transform(test)\n</pre> # This is our LabelEncoder. A class that is designed to work with the models in this library but # can be used for general purposes le = LabelEncoder(columns_to_encode=[\"division_name\", \"department_name\", \"class_name\"]) train_tab_le = le.fit_transform(train) test_tab_le = le.transform(test) In\u00a0[20]: Copied! <pre>train_tab_le.head()\n</pre> train_tab_le.head() Out[20]: clothing_id age title review_text rating recommended_ind positive_feedback_count division_name department_name class_name 4541 836 35 None Bought this on sale in my reg size- 10. im 5'9... 2 1 2 1 1 1 18573 1022 25 Look like \"mom jeans\" Maybe i just have the wrong body type for thes... 1 0 0 2 2 2 1058 815 39 Ig brought me here Love the way this top layers under my jackets ... 2 1 0 1 1 1 12132 984 47 Runs small especially the arms I love this jacket. it's the prettiest and mos... 3 1 0 1 3 3 20756 1051 42 True red, true beauty. These pants are gorgeous--the fabric has a sat... 3 1 0 2 2 4 <p>let's for example have a look to the encodings for the categorical feature <code>class_name</code></p> In\u00a0[21]: Copied! <pre>le.encoding_dict[\"class_name\"]\n</pre> le.encoding_dict[\"class_name\"] Out[21]: <pre>{'blouses': 1,\n 'jeans': 2,\n 'jackets': 3,\n 'pants': 4,\n 'knits': 5,\n 'dresses': 6,\n 'skirts': 7,\n 'sweaters': 8,\n 'fine gauge': 9,\n 'legwear': 10,\n 'lounge': 11,\n 'shorts': 12,\n 'outerwear': 13,\n 'intimates': 14,\n 'swim': 15,\n 'trend': 16,\n 'sleep': 17,\n 'layering': 18,\n 'missing': 19,\n 'casual bottoms': 20,\n 'chemises': 21}</pre> In\u00a0[22]: Copied! <pre># tabular training and test sets\nX_tab_tr = csr_matrix(train_tab_le[tab_cols].values)\nX_tab_te = csr_matrix(test_tab_le[tab_cols].values)\n\n# text + tabular training and test sets\nX_tab_text_tr = hstack((X_tab_tr, X_text_tr))\nX_tab_text_te = hstack((X_tab_te, X_text_te))\n</pre> # tabular training and test sets X_tab_tr = csr_matrix(train_tab_le[tab_cols].values) X_tab_te = csr_matrix(test_tab_le[tab_cols].values)  # text + tabular training and test sets X_tab_text_tr = hstack((X_tab_tr, X_text_tr)) X_tab_text_te = hstack((X_tab_te, X_text_te)) In\u00a0[23]: Copied! <pre>X_tab_tr\n</pre> X_tab_tr Out[23]: <pre>&lt;18086x4 sparse matrix of type '&lt;class 'numpy.int64'&gt;'\n\twith 72344 stored elements in Compressed Sparse Row format&gt;</pre> In\u00a0[24]: Copied! <pre>X_tab_text_tr\n</pre> X_tab_text_tr Out[24]: <pre>&lt;18086x4570 sparse matrix of type '&lt;class 'numpy.float64'&gt;'\n\twith 956418 stored elements in Compressed Sparse Row format&gt;</pre> In\u00a0[25]: Copied! <pre>lgbtrain_tab_text = lgbDataset(\n    X_tab_text_tr,\n    train.rating.values,\n    categorical_feature=[0, 1, 2, 3],\n    free_raw_data=False,\n)\n\nlgbtest_tab_text = lgbDataset(\n    X_tab_text_te,\n    test.rating.values,\n    reference=lgbtrain_tab_text,\n    free_raw_data=False,\n)\n</pre> lgbtrain_tab_text = lgbDataset(     X_tab_text_tr,     train.rating.values,     categorical_feature=[0, 1, 2, 3],     free_raw_data=False, )  lgbtest_tab_text = lgbDataset(     X_tab_text_te,     test.rating.values,     reference=lgbtrain_tab_text,     free_raw_data=False, ) In\u00a0[26]: Copied! <pre>lgb_tab_text_model = lgb.train(\n    {\"objective\": \"multiclass\", \"num_classes\": 4},\n    lgbtrain_tab_text,\n    valid_sets=[lgbtrain_tab_text, lgbtest_tab_text],\n    valid_names=[\"test\", \"train\"],\n    verbose_eval=False,\n)\n</pre> lgb_tab_text_model = lgb.train(     {\"objective\": \"multiclass\", \"num_classes\": 4},     lgbtrain_tab_text,     valid_sets=[lgbtrain_tab_text, lgbtest_tab_text],     valid_names=[\"test\", \"train\"],     verbose_eval=False, ) <pre>/opt/conda/envs/wd38/lib/python3.8/site-packages/lightgbm/basic.py:2065: UserWarning: Using categorical_feature in Dataset.\n  _log_warning('Using categorical_feature in Dataset.')\n/opt/conda/envs/wd38/lib/python3.8/site-packages/lightgbm/basic.py:2068: UserWarning: categorical_feature in Dataset is overridden.\nNew categorical_feature is [0, 1, 2, 3]\n  _log_warning('categorical_feature in Dataset is overridden.\\n'\n/opt/conda/envs/wd38/lib/python3.8/site-packages/lightgbm/engine.py:239: UserWarning: 'verbose_eval' argument is deprecated and will be removed in a future release of LightGBM. Pass 'log_evaluation()' callback via 'callbacks' argument instead.\n  _log_warning(\"'verbose_eval' argument is deprecated and will be removed in a future release of LightGBM. \"\n</pre> <pre>[LightGBM] [Warning] Auto-choosing col-wise multi-threading, the overhead of testing was 0.138280 seconds.\nYou can set `force_col_wise=true` to remove the overhead.\n[LightGBM] [Info] Total Bins 143432\n[LightGBM] [Info] Number of data points in the train set: 18086, number of used features: 2289\n[LightGBM] [Info] Start training from score -2.255919\n[LightGBM] [Info] Start training from score -2.081545\n[LightGBM] [Info] Start training from score -1.528281\n[LightGBM] [Info] Start training from score -0.591354\n</pre> <pre>/opt/conda/envs/wd38/lib/python3.8/site-packages/lightgbm/basic.py:1780: UserWarning: Overriding the parameters from Reference Dataset.\n  _log_warning('Overriding the parameters from Reference Dataset.')\n/opt/conda/envs/wd38/lib/python3.8/site-packages/lightgbm/basic.py:1513: UserWarning: categorical_column in param dict is overridden.\n  _log_warning(f'{cat_alias} in param dict is overridden.')\n</pre> In\u00a0[27]: Copied! <pre>preds_tab_text = lgb_tab_text_model.predict(X_tab_text_te)\npreds_tab_text_class = np.argmax(preds_tab_text, 1)\n\nacc_tab_text = accuracy_score(lgbtest_tab_text.label, preds_tab_text_class)\nf1_tab_text = f1_score(lgbtest_tab_text.label, preds_tab_text_class, average=\"weighted\")\ncm_tab_text = confusion_matrix(lgbtest_tab_text.label, preds_tab_text_class)\n</pre> preds_tab_text = lgb_tab_text_model.predict(X_tab_text_te) preds_tab_text_class = np.argmax(preds_tab_text, 1)  acc_tab_text = accuracy_score(lgbtest_tab_text.label, preds_tab_text_class) f1_tab_text = f1_score(lgbtest_tab_text.label, preds_tab_text_class, average=\"weighted\") cm_tab_text = confusion_matrix(lgbtest_tab_text.label, preds_tab_text_class) In\u00a0[28]: Copied! <pre>print(\n    f\"LightGBM text + tabular Accuracy: {acc_tab_text}. LightGBM text + tabular F1 Score: {f1_tab_text}\"\n)\n</pre> print(     f\"LightGBM text + tabular Accuracy: {acc_tab_text}. LightGBM text + tabular F1 Score: {f1_tab_text}\" ) <pre>LightGBM text + tabular Accuracy: 0.6382131800088456. LightGBM text + tabular F1 Score: 0.6080251307242649\n</pre> In\u00a0[29]: Copied! <pre>print(f\"LightGBM text + tabular Confusion Matrix:\\n {cm_tab_text}\")\n</pre> print(f\"LightGBM text + tabular Confusion Matrix:\\n {cm_tab_text}\") <pre>LightGBM text + tabular Confusion Matrix:\n [[ 193  123   68   90]\n [ 123  146  157  138]\n [  37   90  272  582]\n [  16   37  175 2275]]\n</pre> <p>So, in this set up, the addition tabular columns do not help performance.</p> In\u00a0[30]: Copied! <pre>text_preprocessor = TextPreprocessor(\n    text_col=\"review_text\", max_vocab=5000, min_freq=5, maxlen=90, n_cpus=1\n)\n\nwd_X_text_tr = text_preprocessor.fit_transform(train)\nwd_X_text_te = text_preprocessor.transform(test)\n</pre> text_preprocessor = TextPreprocessor(     text_col=\"review_text\", max_vocab=5000, min_freq=5, maxlen=90, n_cpus=1 )  wd_X_text_tr = text_preprocessor.fit_transform(train) wd_X_text_te = text_preprocessor.transform(test) <pre>The vocabulary contains 4328 tokens\n</pre> In\u00a0[31]: Copied! <pre>basic_rnn = BasicRNN(\n    vocab_size=len(text_preprocessor.vocab.itos),\n    embed_dim=300,\n    hidden_dim=64,\n    n_layers=3,\n    rnn_dropout=0.2,\n    head_hidden_dims=[32],\n)\n\n\nwd_text_model = WideDeep(deeptext=basic_rnn, pred_dim=4)\n</pre> basic_rnn = BasicRNN(     vocab_size=len(text_preprocessor.vocab.itos),     embed_dim=300,     hidden_dim=64,     n_layers=3,     rnn_dropout=0.2,     head_hidden_dims=[32], )   wd_text_model = WideDeep(deeptext=basic_rnn, pred_dim=4) In\u00a0[32]: Copied! <pre>wd_text_model\n</pre> wd_text_model Out[32]: <pre>WideDeep(\n  (deeptext): Sequential(\n    (0): BasicRNN(\n      (word_embed): Embedding(4328, 300, padding_idx=1)\n      (rnn): LSTM(300, 64, num_layers=3, batch_first=True, dropout=0.2)\n      (rnn_mlp): MLP(\n        (mlp): Sequential(\n          (dense_layer_0): Sequential(\n            (0): Linear(in_features=64, out_features=32, bias=True)\n            (1): ReLU(inplace=True)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=32, out_features=4, bias=True)\n  )\n)</pre> In\u00a0[33]: Copied! <pre>text_trainer = Trainer(\n    wd_text_model,\n    objective=\"multiclass\",\n    metrics=[Accuracy, F1Score(average=True)],\n    num_workers=0,  # As in the case of the tokenizer, in notebook I need to set this to 0 for the Trainer to work\n)\n</pre> text_trainer = Trainer(     wd_text_model,     objective=\"multiclass\",     metrics=[Accuracy, F1Score(average=True)],     num_workers=0,  # As in the case of the tokenizer, in notebook I need to set this to 0 for the Trainer to work ) In\u00a0[34]: Copied! <pre>text_trainer.fit(\n    X_text=wd_X_text_tr,\n    target=train.rating.values,\n    n_epochs=5,\n    batch_size=256,\n)\n</pre> text_trainer.fit(     X_text=wd_X_text_tr,     target=train.rating.values,     n_epochs=5,     batch_size=256, ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 71/71 [00:01&lt;00:00, 52.39it/s, loss=1.16, metrics={'acc': 0.5349, 'f1': 0.2011}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 71/71 [00:01&lt;00:00, 70.35it/s, loss=0.964, metrics={'acc': 0.5827, 'f1': 0.3005}]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 71/71 [00:01&lt;00:00, 70.33it/s, loss=0.845, metrics={'acc': 0.6252, 'f1': 0.4133}]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 71/71 [00:01&lt;00:00, 69.99it/s, loss=0.765, metrics={'acc': 0.6575, 'f1': 0.4875}]\nepoch 5: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 71/71 [00:01&lt;00:00, 69.55it/s, loss=0.709, metrics={'acc': 0.6879, 'f1': 0.5423}]\n</pre> In\u00a0[35]: Copied! <pre>wd_pred_text = text_trainer.predict_proba(X_text=wd_X_text_te)\nwd_pred_text_class = np.argmax(wd_pred_text, 1)\n</pre> wd_pred_text = text_trainer.predict_proba(X_text=wd_X_text_te) wd_pred_text_class = np.argmax(wd_pred_text, 1) <pre>predict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 18/18 [00:00&lt;00:00, 211.51it/s]\n</pre> In\u00a0[36]: Copied! <pre>wd_acc_text = accuracy_score(test.rating, wd_pred_text_class)\nwd_f1_text = f1_score(test.rating, wd_pred_text_class, average=\"weighted\")\nwd_cm_text = confusion_matrix(test.rating, wd_pred_text_class)\n</pre> wd_acc_text = accuracy_score(test.rating, wd_pred_text_class) wd_f1_text = f1_score(test.rating, wd_pred_text_class, average=\"weighted\") wd_cm_text = confusion_matrix(test.rating, wd_pred_text_class) In\u00a0[37]: Copied! <pre>print(f\"Basic RNN Accuracy: {wd_acc_text}. Basic RNN F1 Score: {wd_f1_text}\")\n</pre> print(f\"Basic RNN Accuracy: {wd_acc_text}. Basic RNN F1 Score: {wd_f1_text}\") <pre>Basic RNN Accuracy: 0.6076957098628926. Basic RNN F1 Score: 0.6017335854471788\n</pre> In\u00a0[38]: Copied! <pre>print(f\"Basic RNN Confusion Matrix:\\n {wd_cm_text}\")\n</pre> print(f\"Basic RNN Confusion Matrix:\\n {wd_cm_text}\") <pre>Basic RNN Confusion Matrix:\n [[ 327   76   62    9]\n [ 285  115  117   47]\n [ 131  122  315  413]\n [  42   69  401 1991]]\n</pre> <p>The performance is very similar to that of using simply tf-idf and lightgbm. Let see if adding tabular features helps when using <code>pytorch-widedeep</code></p> In\u00a0[39]: Copied! <pre># ?TabPreprocessor\n</pre> # ?TabPreprocessor In\u00a0[40]: Copied! <pre>tab_preprocessor = TabPreprocessor(cat_embed_cols=tab_cols)\n\nwd_X_tab_tr = tab_preprocessor.fit_transform(train)\nwd_X_tab_te = tab_preprocessor.transform(test)\n</pre> tab_preprocessor = TabPreprocessor(cat_embed_cols=tab_cols)  wd_X_tab_tr = tab_preprocessor.fit_transform(train) wd_X_tab_te = tab_preprocessor.transform(test) In\u00a0[41]: Copied! <pre># ?TabMlp\n</pre> # ?TabMlp In\u00a0[42]: Copied! <pre>tab_model = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    mlp_hidden_dims=[100, 50],\n)\n</pre> tab_model = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     mlp_hidden_dims=[100, 50], ) In\u00a0[43]: Copied! <pre>tab_model\n</pre> tab_model Out[43]: <pre>TabMlp(\n  (cat_and_cont_embed): DiffSizeCatAndContEmbeddings(\n    (cat_embed): DiffSizeCatEmbeddings(\n      (embed_layers): ModuleDict(\n        (emb_layer_age): Embedding(78, 18, padding_idx=0)\n        (emb_layer_division_name): Embedding(5, 3, padding_idx=0)\n        (emb_layer_department_name): Embedding(8, 5, padding_idx=0)\n        (emb_layer_class_name): Embedding(22, 9, padding_idx=0)\n      )\n      (embedding_dropout): Dropout(p=0.1, inplace=False)\n    )\n  )\n  (encoder): MLP(\n    (mlp): Sequential(\n      (dense_layer_0): Sequential(\n        (0): Dropout(p=0.1, inplace=False)\n        (1): Linear(in_features=35, out_features=100, bias=True)\n        (2): ReLU(inplace=True)\n      )\n      (dense_layer_1): Sequential(\n        (0): Dropout(p=0.1, inplace=False)\n        (1): Linear(in_features=100, out_features=50, bias=True)\n        (2): ReLU(inplace=True)\n      )\n    )\n  )\n)</pre> In\u00a0[44]: Copied! <pre>text_model = BasicRNN(\n    vocab_size=len(text_preprocessor.vocab.itos),\n    embed_dim=300,\n    hidden_dim=64,\n    n_layers=3,\n    rnn_dropout=0.2,\n    head_hidden_dims=[32],\n)\n</pre> text_model = BasicRNN(     vocab_size=len(text_preprocessor.vocab.itos),     embed_dim=300,     hidden_dim=64,     n_layers=3,     rnn_dropout=0.2,     head_hidden_dims=[32], ) In\u00a0[45]: Copied! <pre>wd_tab_and_text_model = WideDeep(deeptabular=tab_model, deeptext=text_model, pred_dim=4)\n</pre> wd_tab_and_text_model = WideDeep(deeptabular=tab_model, deeptext=text_model, pred_dim=4) In\u00a0[46]: Copied! <pre>wd_tab_and_text_model\n</pre> wd_tab_and_text_model Out[46]: <pre>WideDeep(\n  (deeptabular): Sequential(\n    (0): TabMlp(\n      (cat_and_cont_embed): DiffSizeCatAndContEmbeddings(\n        (cat_embed): DiffSizeCatEmbeddings(\n          (embed_layers): ModuleDict(\n            (emb_layer_age): Embedding(78, 18, padding_idx=0)\n            (emb_layer_division_name): Embedding(5, 3, padding_idx=0)\n            (emb_layer_department_name): Embedding(8, 5, padding_idx=0)\n            (emb_layer_class_name): Embedding(22, 9, padding_idx=0)\n          )\n          (embedding_dropout): Dropout(p=0.1, inplace=False)\n        )\n      )\n      (encoder): MLP(\n        (mlp): Sequential(\n          (dense_layer_0): Sequential(\n            (0): Dropout(p=0.1, inplace=False)\n            (1): Linear(in_features=35, out_features=100, bias=True)\n            (2): ReLU(inplace=True)\n          )\n          (dense_layer_1): Sequential(\n            (0): Dropout(p=0.1, inplace=False)\n            (1): Linear(in_features=100, out_features=50, bias=True)\n            (2): ReLU(inplace=True)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=50, out_features=4, bias=True)\n  )\n  (deeptext): Sequential(\n    (0): BasicRNN(\n      (word_embed): Embedding(4328, 300, padding_idx=1)\n      (rnn): LSTM(300, 64, num_layers=3, batch_first=True, dropout=0.2)\n      (rnn_mlp): MLP(\n        (mlp): Sequential(\n          (dense_layer_0): Sequential(\n            (0): Linear(in_features=64, out_features=32, bias=True)\n            (1): ReLU(inplace=True)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=32, out_features=4, bias=True)\n  )\n)</pre> In\u00a0[47]: Copied! <pre>tab_and_text_trainer = Trainer(\n    wd_tab_and_text_model,\n    objective=\"multiclass\",\n    metrics=[Accuracy, F1Score(average=True)],\n    num_workers=0,  # As in the case of the tokenizer, in notebook I need to set this to 0 for the Trainer to work\n)\n</pre> tab_and_text_trainer = Trainer(     wd_tab_and_text_model,     objective=\"multiclass\",     metrics=[Accuracy, F1Score(average=True)],     num_workers=0,  # As in the case of the tokenizer, in notebook I need to set this to 0 for the Trainer to work ) In\u00a0[48]: Copied! <pre>tab_and_text_trainer.fit(\n    X_tab=wd_X_tab_tr,\n    X_text=wd_X_text_tr,\n    target=train.rating.values,\n    n_epochs=5,\n    batch_size=256,\n)\n</pre> tab_and_text_trainer.fit(     X_tab=wd_X_tab_tr,     X_text=wd_X_text_tr,     target=train.rating.values,     n_epochs=5,     batch_size=256, ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 71/71 [00:01&lt;00:00, 52.04it/s, loss=1.13, metrics={'acc': 0.538, 'f1': 0.1911}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 71/71 [00:01&lt;00:00, 52.28it/s, loss=0.936, metrics={'acc': 0.5887, 'f1': 0.3507}]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 71/71 [00:01&lt;00:00, 52.26it/s, loss=0.825, metrics={'acc': 0.6394, 'f1': 0.4545}]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 71/71 [00:01&lt;00:00, 51.33it/s, loss=0.757, metrics={'acc': 0.6696, 'f1': 0.5214}]\nepoch 5: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 71/71 [00:01&lt;00:00, 50.39it/s, loss=0.702, metrics={'acc': 0.6963, 'f1': 0.5654}]\n</pre> In\u00a0[49]: Copied! <pre>wd_pred_tab_and_text = tab_and_text_trainer.predict_proba(\n    X_tab=wd_X_tab_te, X_text=wd_X_text_te\n)\nwd_pred_tab_and_text_class = np.argmax(wd_pred_tab_and_text, 1)\n</pre> wd_pred_tab_and_text = tab_and_text_trainer.predict_proba(     X_tab=wd_X_tab_te, X_text=wd_X_text_te ) wd_pred_tab_and_text_class = np.argmax(wd_pred_tab_and_text, 1) <pre>predict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 18/18 [00:00&lt;00:00, 136.94it/s]\n</pre> In\u00a0[50]: Copied! <pre>wd_acc_tab_and_text = accuracy_score(test.rating, wd_pred_tab_and_text_class)\nwd_f1_tab_and_text = f1_score(\n    test.rating, wd_pred_tab_and_text_class, average=\"weighted\"\n)\nwd_cm_tab_and_text = confusion_matrix(test.rating, wd_pred_tab_and_text_class)\n</pre> wd_acc_tab_and_text = accuracy_score(test.rating, wd_pred_tab_and_text_class) wd_f1_tab_and_text = f1_score(     test.rating, wd_pred_tab_and_text_class, average=\"weighted\" ) wd_cm_tab_and_text = confusion_matrix(test.rating, wd_pred_tab_and_text_class) In\u00a0[51]: Copied! <pre>print(\n    f\"Basic RNN + Tabular  Accuracy: {wd_acc_tab_and_text}. Basic RNN + TabularF1 Score: {wd_f1_tab_and_text}\"\n)\nprint(f\"Basic RNN + Tabular  Confusion Matrix:\\n {wd_cm_tab_and_text}\")\n</pre> print(     f\"Basic RNN + Tabular  Accuracy: {wd_acc_tab_and_text}. Basic RNN + TabularF1 Score: {wd_f1_tab_and_text}\" ) print(f\"Basic RNN + Tabular  Confusion Matrix:\\n {wd_cm_tab_and_text}\") <pre>Basic RNN + Tabular  Accuracy: 0.6333480760725343. Basic RNN + TabularF1 Score: 0.6332310089593208\nBasic RNN + Tabular  Confusion Matrix:\n [[ 267  132   65   10]\n [ 198  168  159   39]\n [  57  113  410  401]\n [  12   58  414 2019]]\n</pre> <p>We are going to \"manually\" code the Tokenizer and the model and see how they can be used as part of the process along with the <code>pytorch-widedeep</code> library.</p> <p>Tokenizer:</p> In\u00a0[52]: Copied! <pre>class BertTokenizer(object):\n    def __init__(\n        self,\n        pretrained_tokenizer=\"distilbert-base-uncased\",\n        do_lower_case=True,\n        max_length=90,\n    ):\n        super(BertTokenizer, self).__init__()\n        self.pretrained_tokenizer = pretrained_tokenizer\n        self.do_lower_case = do_lower_case\n        self.max_length = max_length\n\n    def fit(self, texts):\n        self.tokenizer = DistilBertTokenizer.from_pretrained(\n            self.pretrained_tokenizer, do_lower_case=self.do_lower_case\n        )\n\n        return self\n\n    def transform(self, texts):\n        input_ids = []\n        for text in texts:\n            encoded_sent = self.tokenizer.encode_plus(\n                text=self._pre_rules(text),\n                add_special_tokens=True,\n                max_length=self.max_length,\n                padding=\"max_length\",\n                truncation=True,\n            )\n\n            input_ids.append(encoded_sent.get(\"input_ids\"))\n        return np.stack(input_ids)\n\n    def fit_transform(self, texts):\n        return self.fit(texts).transform(texts)\n\n    @staticmethod\n    def _pre_rules(text):\n        return fix_html(rm_useless_spaces(spec_add_spaces(text)))\n</pre> class BertTokenizer(object):     def __init__(         self,         pretrained_tokenizer=\"distilbert-base-uncased\",         do_lower_case=True,         max_length=90,     ):         super(BertTokenizer, self).__init__()         self.pretrained_tokenizer = pretrained_tokenizer         self.do_lower_case = do_lower_case         self.max_length = max_length      def fit(self, texts):         self.tokenizer = DistilBertTokenizer.from_pretrained(             self.pretrained_tokenizer, do_lower_case=self.do_lower_case         )          return self      def transform(self, texts):         input_ids = []         for text in texts:             encoded_sent = self.tokenizer.encode_plus(                 text=self._pre_rules(text),                 add_special_tokens=True,                 max_length=self.max_length,                 padding=\"max_length\",                 truncation=True,             )              input_ids.append(encoded_sent.get(\"input_ids\"))         return np.stack(input_ids)      def fit_transform(self, texts):         return self.fit(texts).transform(texts)      @staticmethod     def _pre_rules(text):         return fix_html(rm_useless_spaces(spec_add_spaces(text))) <p>Model:</p> In\u00a0[53]: Copied! <pre>class BertModel(nn.Module):\n    def __init__(\n        self,\n        model_name: str = \"distilbert-base-uncased\",\n        freeze_bert: bool = False,\n    ):\n        super(BertModel, self).__init__()\n\n        self.bert = DistilBertModel.from_pretrained(\n            model_name,\n        )\n\n        if freeze_bert:\n            for param in self.bert.parameters():\n                param.requires_grad = False\n\n    def forward(self, X_inp: Tensor) -&gt; Tensor:\n        attn_mask = (X_inp != 0).type(torch.int8)\n        outputs = self.bert(input_ids=X_inp, attention_mask=attn_mask)\n        return outputs[0][:, 0, :]\n\n    @property\n    def output_dim(self) -&gt; int:\n        # This is THE ONLY requirement for any model to work with pytorch-widedeep. Must\n        # have a 'output_dim' property so the WideDeep class knows the incoming dims\n        # from the custom model. in this case, I hardcoded it\n        return 768\n</pre> class BertModel(nn.Module):     def __init__(         self,         model_name: str = \"distilbert-base-uncased\",         freeze_bert: bool = False,     ):         super(BertModel, self).__init__()          self.bert = DistilBertModel.from_pretrained(             model_name,         )          if freeze_bert:             for param in self.bert.parameters():                 param.requires_grad = False      def forward(self, X_inp: Tensor) -&gt; Tensor:         attn_mask = (X_inp != 0).type(torch.int8)         outputs = self.bert(input_ids=X_inp, attention_mask=attn_mask)         return outputs[0][:, 0, :]      @property     def output_dim(self) -&gt; int:         # This is THE ONLY requirement for any model to work with pytorch-widedeep. Must         # have a 'output_dim' property so the WideDeep class knows the incoming dims         # from the custom model. in this case, I hardcoded it         return 768 In\u00a0[54]: Copied! <pre>bert_tokenizer = BertTokenizer()\nX_bert_tr = bert_tokenizer.fit_transform(train[\"review_text\"].tolist())\nX_bert_te = bert_tokenizer.transform(test[\"review_text\"].tolist())\n</pre> bert_tokenizer = BertTokenizer() X_bert_tr = bert_tokenizer.fit_transform(train[\"review_text\"].tolist()) X_bert_te = bert_tokenizer.transform(test[\"review_text\"].tolist()) <p>As I mentioned a number of times in the documentation and examples, <code>pytorch-widedeep</code> is designed for flexibility. For any of the data modes (tabular, text and images) there are available components/models in the library. However, the user can choose to use any model they want with the only requirement that such model must have a <code>output_dim</code> property.</p> <p>With that in mind, the <code>BertModel</code> class defined above can be used by <code>pytorch-widedeep</code> as any other of the internal components. In other words, simply...pass it to the <code>WideDeep</code> class. In this case we are going to add a FC-head as part of the classifier.</p> In\u00a0[55]: Copied! <pre>bert_model = BertModel(freeze_bert=True)\nwd_bert_model = WideDeep(\n    deeptext=bert_model,\n    head_hidden_dims=[256, 128, 64],\n    pred_dim=4,\n)\n</pre> bert_model = BertModel(freeze_bert=True) wd_bert_model = WideDeep(     deeptext=bert_model,     head_hidden_dims=[256, 128, 64],     pred_dim=4, ) <pre>Some weights of the model checkpoint at distilbert-base-uncased were not used when initializing DistilBertModel: ['vocab_projector.bias', 'vocab_layer_norm.bias', 'vocab_transform.weight', 'vocab_transform.bias', 'vocab_projector.weight', 'vocab_layer_norm.weight']\n- This IS expected if you are initializing DistilBertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n- This IS NOT expected if you are initializing DistilBertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n</pre> In\u00a0[56]: Copied! <pre>wd_bert_model\n</pre> wd_bert_model Out[56]: <pre>WideDeep(\n  (deeptext): BertModel(\n    (bert): DistilBertModel(\n      (embeddings): Embeddings(\n        (word_embeddings): Embedding(30522, 768, padding_idx=0)\n        (position_embeddings): Embedding(512, 768)\n        (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n        (dropout): Dropout(p=0.1, inplace=False)\n      )\n      (transformer): Transformer(\n        (layer): ModuleList(\n          (0): TransformerBlock(\n            (attention): MultiHeadSelfAttention(\n              (dropout): Dropout(p=0.1, inplace=False)\n              (q_lin): Linear(in_features=768, out_features=768, bias=True)\n              (k_lin): Linear(in_features=768, out_features=768, bias=True)\n              (v_lin): Linear(in_features=768, out_features=768, bias=True)\n              (out_lin): Linear(in_features=768, out_features=768, bias=True)\n            )\n            (sa_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n            (ffn): FFN(\n              (dropout): Dropout(p=0.1, inplace=False)\n              (lin1): Linear(in_features=768, out_features=3072, bias=True)\n              (lin2): Linear(in_features=3072, out_features=768, bias=True)\n              (activation): GELUActivation()\n            )\n            (output_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n          )\n          (1): TransformerBlock(\n            (attention): MultiHeadSelfAttention(\n              (dropout): Dropout(p=0.1, inplace=False)\n              (q_lin): Linear(in_features=768, out_features=768, bias=True)\n              (k_lin): Linear(in_features=768, out_features=768, bias=True)\n              (v_lin): Linear(in_features=768, out_features=768, bias=True)\n              (out_lin): Linear(in_features=768, out_features=768, bias=True)\n            )\n            (sa_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n            (ffn): FFN(\n              (dropout): Dropout(p=0.1, inplace=False)\n              (lin1): Linear(in_features=768, out_features=3072, bias=True)\n              (lin2): Linear(in_features=3072, out_features=768, bias=True)\n              (activation): GELUActivation()\n            )\n            (output_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n          )\n          (2): TransformerBlock(\n            (attention): MultiHeadSelfAttention(\n              (dropout): Dropout(p=0.1, inplace=False)\n              (q_lin): Linear(in_features=768, out_features=768, bias=True)\n              (k_lin): Linear(in_features=768, out_features=768, bias=True)\n              (v_lin): Linear(in_features=768, out_features=768, bias=True)\n              (out_lin): Linear(in_features=768, out_features=768, bias=True)\n            )\n            (sa_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n            (ffn): FFN(\n              (dropout): Dropout(p=0.1, inplace=False)\n              (lin1): Linear(in_features=768, out_features=3072, bias=True)\n              (lin2): Linear(in_features=3072, out_features=768, bias=True)\n              (activation): GELUActivation()\n            )\n            (output_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n          )\n          (3): TransformerBlock(\n            (attention): MultiHeadSelfAttention(\n              (dropout): Dropout(p=0.1, inplace=False)\n              (q_lin): Linear(in_features=768, out_features=768, bias=True)\n              (k_lin): Linear(in_features=768, out_features=768, bias=True)\n              (v_lin): Linear(in_features=768, out_features=768, bias=True)\n              (out_lin): Linear(in_features=768, out_features=768, bias=True)\n            )\n            (sa_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n            (ffn): FFN(\n              (dropout): Dropout(p=0.1, inplace=False)\n              (lin1): Linear(in_features=768, out_features=3072, bias=True)\n              (lin2): Linear(in_features=3072, out_features=768, bias=True)\n              (activation): GELUActivation()\n            )\n            (output_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n          )\n          (4): TransformerBlock(\n            (attention): MultiHeadSelfAttention(\n              (dropout): Dropout(p=0.1, inplace=False)\n              (q_lin): Linear(in_features=768, out_features=768, bias=True)\n              (k_lin): Linear(in_features=768, out_features=768, bias=True)\n              (v_lin): Linear(in_features=768, out_features=768, bias=True)\n              (out_lin): Linear(in_features=768, out_features=768, bias=True)\n            )\n            (sa_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n            (ffn): FFN(\n              (dropout): Dropout(p=0.1, inplace=False)\n              (lin1): Linear(in_features=768, out_features=3072, bias=True)\n              (lin2): Linear(in_features=3072, out_features=768, bias=True)\n              (activation): GELUActivation()\n            )\n            (output_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n          )\n          (5): TransformerBlock(\n            (attention): MultiHeadSelfAttention(\n              (dropout): Dropout(p=0.1, inplace=False)\n              (q_lin): Linear(in_features=768, out_features=768, bias=True)\n              (k_lin): Linear(in_features=768, out_features=768, bias=True)\n              (v_lin): Linear(in_features=768, out_features=768, bias=True)\n              (out_lin): Linear(in_features=768, out_features=768, bias=True)\n            )\n            (sa_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n            (ffn): FFN(\n              (dropout): Dropout(p=0.1, inplace=False)\n              (lin1): Linear(in_features=768, out_features=3072, bias=True)\n              (lin2): Linear(in_features=3072, out_features=768, bias=True)\n              (activation): GELUActivation()\n            )\n            (output_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n          )\n        )\n      )\n    )\n  )\n  (deephead): Sequential(\n    (0): MLP(\n      (mlp): Sequential(\n        (dense_layer_0): Sequential(\n          (0): Linear(in_features=768, out_features=256, bias=True)\n          (1): ReLU(inplace=True)\n          (2): Dropout(p=0.1, inplace=False)\n        )\n        (dense_layer_1): Sequential(\n          (0): Linear(in_features=256, out_features=128, bias=True)\n          (1): ReLU(inplace=True)\n          (2): Dropout(p=0.1, inplace=False)\n        )\n        (dense_layer_2): Sequential(\n          (0): Linear(in_features=128, out_features=64, bias=True)\n          (1): ReLU(inplace=True)\n          (2): Dropout(p=0.1, inplace=False)\n        )\n      )\n    )\n    (1): Linear(in_features=64, out_features=4, bias=True)\n  )\n)</pre> In\u00a0[57]: Copied! <pre>wd_bert_trainer = Trainer(\n    wd_bert_model,\n    objective=\"multiclass\",\n    metrics=[Accuracy, F1Score(average=True)],\n    num_workers=0,  # As in the case of the tokenizer, in notebook I need to set this to 0 for the Trainer to work\n)\n\nwd_bert_trainer.fit(\n    X_text=X_bert_tr,\n    target=train.rating.values,\n    n_epochs=3,\n    batch_size=64,\n)\n</pre> wd_bert_trainer = Trainer(     wd_bert_model,     objective=\"multiclass\",     metrics=[Accuracy, F1Score(average=True)],     num_workers=0,  # As in the case of the tokenizer, in notebook I need to set this to 0 for the Trainer to work )  wd_bert_trainer.fit(     X_text=X_bert_tr,     target=train.rating.values,     n_epochs=3,     batch_size=64, ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 283/283 [00:14&lt;00:00, 19.68it/s, loss=0.968, metrics={'acc': 0.5879, 'f1': 0.3591}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 283/283 [00:14&lt;00:00, 19.63it/s, loss=0.884, metrics={'acc': 0.6178, 'f1': 0.4399}]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 283/283 [00:14&lt;00:00, 19.55it/s, loss=0.87, metrics={'acc': 0.6234, 'f1': 0.4527}]\n</pre> In\u00a0[58]: Copied! <pre>wd_bert_pred_text = wd_bert_trainer.predict_proba(X_text=X_bert_te)\nwd_bert_pred_text_class = np.argmax(wd_bert_pred_text, 1)\n</pre> wd_bert_pred_text = wd_bert_trainer.predict_proba(X_text=X_bert_te) wd_bert_pred_text_class = np.argmax(wd_bert_pred_text, 1) <pre>predict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 71/71 [00:03&lt;00:00, 21.97it/s]\n</pre> In\u00a0[59]: Copied! <pre>wd_bert_acc = accuracy_score(test.rating, wd_bert_pred_text_class)\nwd_bert_f1 = f1_score(test.rating, wd_bert_pred_text_class, average=\"weighted\")\nwd_bert_cm = confusion_matrix(test.rating, wd_bert_pred_text_class)\n</pre> wd_bert_acc = accuracy_score(test.rating, wd_bert_pred_text_class) wd_bert_f1 = f1_score(test.rating, wd_bert_pred_text_class, average=\"weighted\") wd_bert_cm = confusion_matrix(test.rating, wd_bert_pred_text_class) In\u00a0[60]: Copied! <pre>print(f\"Distilbert Accuracy: {wd_bert_acc}. Distilbert F1 Score: {wd_bert_f1}\")\nprint(f\"Distilbert Confusion Matrix:\\n {wd_bert_cm}\")\n</pre> print(f\"Distilbert Accuracy: {wd_bert_acc}. Distilbert F1 Score: {wd_bert_f1}\") print(f\"Distilbert Confusion Matrix:\\n {wd_bert_cm}\") <pre>Distilbert Accuracy: 0.6326846528084918. Distilbert F1 Score: 0.5796652991272998\nDistilbert Confusion Matrix:\n [[ 287   75   22   90]\n [ 197  136   62  169]\n [  68  119  123  671]\n [  40   64   84 2315]]\n</pre> <p>Now, adding a tabular model follows the exact same process as the one described in section 2.</p> In\u00a0[61]: Copied! <pre>tab_model = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    mlp_hidden_dims=[100, 50],\n)\n</pre> tab_model = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     mlp_hidden_dims=[100, 50], ) In\u00a0[62]: Copied! <pre>wd_tab_bert_model = WideDeep(\n    deeptabular=tab_model,\n    deeptext=bert_model,\n    head_hidden_dims=[256, 128, 64],\n    pred_dim=4,\n)\n</pre> wd_tab_bert_model = WideDeep(     deeptabular=tab_model,     deeptext=bert_model,     head_hidden_dims=[256, 128, 64],     pred_dim=4, ) In\u00a0[63]: Copied! <pre>wd_tab_bert_trainer = Trainer(\n    wd_tab_bert_model,\n    objective=\"multiclass\",\n    metrics=[Accuracy, F1Score(average=True)],\n    num_workers=0,  # As in the case of the tokenizer, in notebook I need to set this to 0 for the Trainer to work\n)\n</pre> wd_tab_bert_trainer = Trainer(     wd_tab_bert_model,     objective=\"multiclass\",     metrics=[Accuracy, F1Score(average=True)],     num_workers=0,  # As in the case of the tokenizer, in notebook I need to set this to 0 for the Trainer to work ) In\u00a0[64]: Copied! <pre>wd_tab_bert_trainer.fit(\n    X_tab=wd_X_tab_tr,\n    X_text=X_bert_tr,\n    target=train.rating.values,\n    n_epochs=3,\n    batch_size=64,\n)\n</pre> wd_tab_bert_trainer.fit(     X_tab=wd_X_tab_tr,     X_text=X_bert_tr,     target=train.rating.values,     n_epochs=3,     batch_size=64, ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 283/283 [00:15&lt;00:00, 18.15it/s, loss=0.974, metrics={'acc': 0.5838, 'f1': 0.3404}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 283/283 [00:15&lt;00:00, 18.38it/s, loss=0.885, metrics={'acc': 0.618, 'f1': 0.4378}]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 283/283 [00:15&lt;00:00, 18.40it/s, loss=0.868, metrics={'acc': 0.6252, 'f1': 0.4575}]\n</pre> In\u00a0[65]: Copied! <pre>wd_tab_bert_pred_text = wd_tab_bert_trainer.predict_proba(\n    X_tab=wd_X_tab_te, X_text=X_bert_te\n)\nwd_tab_bert_pred_text_class = np.argmax(wd_tab_bert_pred_text, 1)\n</pre> wd_tab_bert_pred_text = wd_tab_bert_trainer.predict_proba(     X_tab=wd_X_tab_te, X_text=X_bert_te ) wd_tab_bert_pred_text_class = np.argmax(wd_tab_bert_pred_text, 1) <pre>predict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 71/71 [00:03&lt;00:00, 21.32it/s]\n</pre> In\u00a0[66]: Copied! <pre>wd_tab_bert_acc = accuracy_score(test.rating, wd_tab_bert_pred_text_class)\nwd_tab_bert_f1 = f1_score(test.rating, wd_tab_bert_pred_text_class, average=\"weighted\")\nwd_tab_bert_cm = confusion_matrix(test.rating, wd_tab_bert_pred_text_class)\n</pre> wd_tab_bert_acc = accuracy_score(test.rating, wd_tab_bert_pred_text_class) wd_tab_bert_f1 = f1_score(test.rating, wd_tab_bert_pred_text_class, average=\"weighted\") wd_tab_bert_cm = confusion_matrix(test.rating, wd_tab_bert_pred_text_class) In\u00a0[67]: Copied! <pre>print(\n    f\"Distilbert + Tabular Accuracy: {wd_tab_bert_acc}. Distilbert+ Tabular F1 Score: {wd_tab_bert_f1}\"\n)\nprint(f\"Distilbert + Tabular Confusion Matrix:\\n {wd_tab_bert_cm}\")\n</pre> print(     f\"Distilbert + Tabular Accuracy: {wd_tab_bert_acc}. Distilbert+ Tabular F1 Score: {wd_tab_bert_f1}\" ) print(f\"Distilbert + Tabular Confusion Matrix:\\n {wd_tab_bert_cm}\") <pre>Distilbert + Tabular Accuracy: 0.6242812914639541. Distilbert+ Tabular F1 Score: 0.5508351761564895\nDistilbert + Tabular Confusion Matrix:\n [[ 297   56   11  110]\n [ 229   91   38  206]\n [  86   90   71  734]\n [  49   48   42 2364]]\n</pre>"},{"location":"examples/16_Usign_a_custom_hugging_face_model.html#using-a-hugginface-model","title":"Using a Hugginface model\u00b6","text":"<p>In this notebook we will show how to use an \"external\" Hugginface model along with any other model in the libray. In particular we will show how to combine it with a tabular DL model.</p> <p>Since we are here, we will also compare the performance of a few models on a text classification problem.</p> <p>The notebook will go as follows:</p> <ol> <li>Text classification using tf-idf + LightGBM</li> <li>Text classification using a basic RNN</li> <li>Text classification using Distilbert</li> </ol> <p>In all 3 cases we will add some tabular features to see if these help.</p> <p>In general, I would not pay much attention to the results since I have placed no effort in getting the best possible results (i.e. no hyperparameter optimization or trying different architectures, for example).</p> <p>Let's go</p>"},{"location":"examples/16_Usign_a_custom_hugging_face_model.html#1-text-classification-using-tf-idf-lightgbm","title":"1. Text classification using tf-idf + LightGBM\u00b6","text":""},{"location":"examples/16_Usign_a_custom_hugging_face_model.html#2-text-classification-using-pytorch-widedeeps-built-in-models-a-basic-rnn","title":"2. Text classification using pytorch-widedeep's built-in models (a basic RNN)\u00b6","text":"<p>Moving on now to fully using <code>pytorch-widedeep</code> in this dataset, let's have a look on how one could use a simple RNN to predict the ratings with the library.</p>"},{"location":"examples/16_Usign_a_custom_hugging_face_model.html#3-text-classification-using-a-hugginface-model-as-a-custom-model-in-pytorch-widedeeps","title":"3. Text classification using a Hugginface model as a custom model in pytorch-widedeep's\u00b6","text":""},{"location":"examples/17_feature_importance_via_attention_weights.html","title":"17 feature importance via attention weights","text":"In\u00a0[1]: Copied! <pre>import torch\n\nimport numpy as np\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import accuracy_score\n\n\nfrom pytorch_widedeep import Trainer\nfrom pytorch_widedeep.models import TabTransformer, ContextAttentionMLP, WideDeep\nfrom pytorch_widedeep.callbacks import EarlyStopping\nfrom pytorch_widedeep.metrics import Accuracy\nfrom pytorch_widedeep.datasets import load_adult\nfrom pytorch_widedeep.preprocessing import TabPreprocessor\n</pre> import torch  import numpy as np from sklearn.model_selection import train_test_split from sklearn.metrics import accuracy_score   from pytorch_widedeep import Trainer from pytorch_widedeep.models import TabTransformer, ContextAttentionMLP, WideDeep from pytorch_widedeep.callbacks import EarlyStopping from pytorch_widedeep.metrics import Accuracy from pytorch_widedeep.datasets import load_adult from pytorch_widedeep.preprocessing import TabPreprocessor In\u00a0[2]: Copied! <pre># use_cuda = torch.cuda.is_available()\ndf = load_adult(as_frame=True)\ndf.columns = [c.replace(\"-\", \"_\") for c in df.columns]\ndf[\"income_label\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int)\ndf.drop([\"income\", \"fnlwgt\", \"educational_num\"], axis=1, inplace=True)\ntarget_colname = \"income_label\"\n</pre> # use_cuda = torch.cuda.is_available() df = load_adult(as_frame=True) df.columns = [c.replace(\"-\", \"_\") for c in df.columns] df[\"income_label\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int) df.drop([\"income\", \"fnlwgt\", \"educational_num\"], axis=1, inplace=True) target_colname = \"income_label\" In\u00a0[3]: Copied! <pre>df.head()\n</pre> df.head() Out[3]: age workclass education marital_status occupation relationship race gender capital_gain capital_loss hours_per_week native_country income_label 0 25 Private 11th Never-married Machine-op-inspct Own-child Black Male 0 0 40 United-States 0 1 38 Private HS-grad Married-civ-spouse Farming-fishing Husband White Male 0 0 50 United-States 0 2 28 Local-gov Assoc-acdm Married-civ-spouse Protective-serv Husband White Male 0 0 40 United-States 1 3 44 Private Some-college Married-civ-spouse Machine-op-inspct Husband Black Male 7688 0 40 United-States 1 4 18 ? Some-college Never-married ? Own-child White Female 0 0 30 United-States 0 In\u00a0[4]: Copied! <pre>cat_embed_cols = []\nfor col in df.columns:\n    if df[col].dtype == \"O\" or df[col].nunique() &lt; 200 and col != target_colname:\n        cat_embed_cols.append(col)\n</pre> cat_embed_cols = [] for col in df.columns:     if df[col].dtype == \"O\" or df[col].nunique() &lt; 200 and col != target_colname:         cat_embed_cols.append(col) In\u00a0[5]: Copied! <pre># all cols will be categorical\nassert len(cat_embed_cols) == df.shape[1] - 1\n</pre> # all cols will be categorical assert len(cat_embed_cols) == df.shape[1] - 1 In\u00a0[6]: Copied! <pre>train, test = train_test_split(\n    df, test_size=0.1, random_state=1, stratify=df[[target_colname]]\n)\n</pre> train, test = train_test_split(     df, test_size=0.1, random_state=1, stratify=df[[target_colname]] ) In\u00a0[7]: Copied! <pre>tab_preprocessor = TabPreprocessor(cat_embed_cols=cat_embed_cols, with_attention=True)\n</pre> tab_preprocessor = TabPreprocessor(cat_embed_cols=cat_embed_cols, with_attention=True) In\u00a0[8]: Copied! <pre>X_tab_train = tab_preprocessor.fit_transform(train)\nX_tab_test = tab_preprocessor.transform(test)\ntarget = train[target_colname].values\n</pre> X_tab_train = tab_preprocessor.fit_transform(train) X_tab_test = tab_preprocessor.transform(test) target = train[target_colname].values In\u00a0[9]: Copied! <pre>tab_transformer = TabTransformer(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    embed_continuous_method=\"standard\",\n    input_dim=8,\n    n_heads=2,\n    n_blocks=1,\n    attn_dropout=0.1,\n    transformer_activation=\"relu\",\n)\n</pre> tab_transformer = TabTransformer(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     embed_continuous_method=\"standard\",     input_dim=8,     n_heads=2,     n_blocks=1,     attn_dropout=0.1,     transformer_activation=\"relu\", ) In\u00a0[10]: Copied! <pre>model = WideDeep(deeptabular=tab_transformer)\n</pre> model = WideDeep(deeptabular=tab_transformer) In\u00a0[11]: Copied! <pre>optimizer = torch.optim.Adam(model.parameters(), lr=0.01, weight_decay=0.0)\n</pre> optimizer = torch.optim.Adam(model.parameters(), lr=0.01, weight_decay=0.0) In\u00a0[12]: Copied! <pre>lr_scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(\n    optimizer,\n    threshold=0.001,\n    threshold_mode=\"abs\",\n    patience=10,\n)\n</pre> lr_scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(     optimizer,     threshold=0.001,     threshold_mode=\"abs\",     patience=10, ) In\u00a0[13]: Copied! <pre>early_stopping = EarlyStopping(\n    min_delta=0.001, patience=30, restore_best_weights=True, verbose=True\n)\n</pre> early_stopping = EarlyStopping(     min_delta=0.001, patience=30, restore_best_weights=True, verbose=True ) In\u00a0[14]: Copied! <pre>trainer = Trainer(\n    model,\n    objective=\"binary\",\n    optimizers=optimizer,\n    lr_schedulers=lr_scheduler,\n    reducelronplateau_criterion=\"loss\",\n    callbacks=[early_stopping],\n    metrics=[Accuracy],\n)\n</pre> trainer = Trainer(     model,     objective=\"binary\",     optimizers=optimizer,     lr_schedulers=lr_scheduler,     reducelronplateau_criterion=\"loss\",     callbacks=[early_stopping],     metrics=[Accuracy], ) <p>The feature importances will be computed after training, using a sample of the training dataset of size <code>feature_importance_sample_size</code></p> In\u00a0[15]: Copied! <pre>trainer.fit(\n    X_tab=X_tab_train,\n    target=target,\n    val_split=0.2,\n    n_epochs=100,\n    batch_size=128,\n    validation_freq=1,\n    feature_importance_sample_size=1000,\n)\n</pre> trainer.fit(     X_tab=X_tab_train,     target=target,     val_split=0.2,     n_epochs=100,     batch_size=128,     validation_freq=1,     feature_importance_sample_size=1000, ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 81.80it/s, loss=0.334, metrics={'acc': 0.847}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 111.34it/s, loss=0.294, metrics={'acc': 0.8669}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 83.02it/s, loss=0.293, metrics={'acc': 0.8656}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 124.03it/s, loss=0.283, metrics={'acc': 0.8678}]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 87.69it/s, loss=0.282, metrics={'acc': 0.8703}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 118.22it/s, loss=0.279, metrics={'acc': 0.8717}]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 79.24it/s, loss=0.277, metrics={'acc': 0.8718}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 115.29it/s, loss=0.277, metrics={'acc': 0.8731}]\nepoch 5: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 83.76it/s, loss=0.275, metrics={'acc': 0.8727}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 120.80it/s, loss=0.276, metrics={'acc': 0.8727}]\nepoch 6: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 82.78it/s, loss=0.273, metrics={'acc': 0.873}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 100.43it/s, loss=0.276, metrics={'acc': 0.871}]\nepoch 7: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 76.35it/s, loss=0.271, metrics={'acc': 0.8742}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 116.14it/s, loss=0.275, metrics={'acc': 0.8726}]\nepoch 8: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 74.29it/s, loss=0.271, metrics={'acc': 0.875}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 117.46it/s, loss=0.276, metrics={'acc': 0.8718}]\nepoch 9: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 77.10it/s, loss=0.27, metrics={'acc': 0.8761}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 105.49it/s, loss=0.275, metrics={'acc': 0.8728}]\nepoch 10: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 70.40it/s, loss=0.269, metrics={'acc': 0.8747}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 105.47it/s, loss=0.275, metrics={'acc': 0.8726}]\nepoch 11: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 72.83it/s, loss=0.268, metrics={'acc': 0.8742}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 105.03it/s, loss=0.274, metrics={'acc': 0.873}]\nepoch 12: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 71.86it/s, loss=0.267, metrics={'acc': 0.8743}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 106.61it/s, loss=0.274, metrics={'acc': 0.8734}]\nepoch 13: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 72.39it/s, loss=0.267, metrics={'acc': 0.876}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 108.05it/s, loss=0.275, metrics={'acc': 0.8717}]\nepoch 14: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 73.36it/s, loss=0.265, metrics={'acc': 0.8767}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 109.60it/s, loss=0.276, metrics={'acc': 0.8747}]\nepoch 15: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 73.34it/s, loss=0.264, metrics={'acc': 0.876}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 117.55it/s, loss=0.276, metrics={'acc': 0.8706}]\nepoch 16: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 75.35it/s, loss=0.264, metrics={'acc': 0.8777}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 122.08it/s, loss=0.275, metrics={'acc': 0.8753}]\nepoch 17: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 75.96it/s, loss=0.263, metrics={'acc': 0.877}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 120.83it/s, loss=0.277, metrics={'acc': 0.8739}]\nepoch 18: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 75.38it/s, loss=0.263, metrics={'acc': 0.8779}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 118.18it/s, loss=0.278, metrics={'acc': 0.8714}]\nepoch 19: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 75.61it/s, loss=0.261, metrics={'acc': 0.8784}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 119.19it/s, loss=0.278, metrics={'acc': 0.8712}]\nepoch 20: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 72.43it/s, loss=0.261, metrics={'acc': 0.8791}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 118.87it/s, loss=0.28, metrics={'acc': 0.873}]\nepoch 21: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 73.97it/s, loss=0.26, metrics={'acc': 0.8787}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 107.50it/s, loss=0.279, metrics={'acc': 0.8732}]\nepoch 22: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 71.76it/s, loss=0.253, metrics={'acc': 0.8816}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 108.11it/s, loss=0.279, metrics={'acc': 0.8707}]\nepoch 23: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 71.92it/s, loss=0.252, metrics={'acc': 0.8828}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 100.14it/s, loss=0.28, metrics={'acc': 0.8711}]\nepoch 24: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 75.04it/s, loss=0.252, metrics={'acc': 0.8829}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 98.36it/s, loss=0.28, metrics={'acc': 0.8708}]\nepoch 25: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 76.31it/s, loss=0.251, metrics={'acc': 0.883}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 122.97it/s, loss=0.281, metrics={'acc': 0.8709}]\nepoch 26: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 76.63it/s, loss=0.25, metrics={'acc': 0.8834}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 123.07it/s, loss=0.281, metrics={'acc': 0.8698}]\nepoch 27: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 78.37it/s, loss=0.251, metrics={'acc': 0.884}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 114.75it/s, loss=0.281, metrics={'acc': 0.87}]\nepoch 28: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 77.19it/s, loss=0.25, metrics={'acc': 0.883}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 117.99it/s, loss=0.282, metrics={'acc': 0.8699}]\nepoch 29: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 76.19it/s, loss=0.25, metrics={'acc': 0.8829}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 111.11it/s, loss=0.282, metrics={'acc': 0.8695}]\nepoch 30: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 77.57it/s, loss=0.249, metrics={'acc': 0.8839}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 115.64it/s, loss=0.283, metrics={'acc': 0.8689}]\nepoch 31: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 79.55it/s, loss=0.249, metrics={'acc': 0.8846}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 125.10it/s, loss=0.283, metrics={'acc': 0.869}]\nepoch 32: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 82.56it/s, loss=0.248, metrics={'acc': 0.8841}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 118.45it/s, loss=0.284, metrics={'acc': 0.8687}]\nepoch 33: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 81.06it/s, loss=0.248, metrics={'acc': 0.8848}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 129.26it/s, loss=0.284, metrics={'acc': 0.8689}]\nepoch 34: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 80.53it/s, loss=0.248, metrics={'acc': 0.8854}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 120.61it/s, loss=0.283, metrics={'acc': 0.869}]\nepoch 35: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 80.78it/s, loss=0.248, metrics={'acc': 0.8853}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 127.31it/s, loss=0.283, metrics={'acc': 0.8694}]\nepoch 36: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 82.51it/s, loss=0.248, metrics={'acc': 0.8863}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 125.94it/s, loss=0.283, metrics={'acc': 0.8693}]\nepoch 37: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 81.35it/s, loss=0.247, metrics={'acc': 0.8844}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 126.77it/s, loss=0.283, metrics={'acc': 0.8692}]\nepoch 38: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 80.62it/s, loss=0.248, metrics={'acc': 0.8837}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 116.62it/s, loss=0.283, metrics={'acc': 0.8692}]\nepoch 39: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 79.82it/s, loss=0.248, metrics={'acc': 0.8842}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 123.64it/s, loss=0.283, metrics={'acc': 0.8695}]\nepoch 40: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 76.86it/s, loss=0.247, metrics={'acc': 0.8855}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 120.34it/s, loss=0.283, metrics={'acc': 0.8692}]\n</pre> <pre>Best Epoch: 10. Best val_loss: 0.27451\nRestoring model weights from the end of the best epoch\n</pre> In\u00a0[16]: Copied! <pre>trainer.feature_importance\n</pre> trainer.feature_importance Out[16]: <pre>{'age': 0.09718182,\n 'workclass': 0.090637445,\n 'education': 0.08910798,\n 'marital_status': 0.08971319,\n 'occupation': 0.12546304,\n 'relationship': 0.086381145,\n 'race': 0.050686445,\n 'gender': 0.05116429,\n 'capital_gain': 0.08165918,\n 'capital_loss': 0.07702667,\n 'hours_per_week': 0.08205996,\n 'native_country': 0.07891885}</pre> In\u00a0[17]: Copied! <pre>preds = trainer.predict(X_tab=X_tab_test)\n</pre> preds = trainer.predict(X_tab=X_tab_test) <pre>predict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 39/39 [00:00&lt;00:00, 199.63it/s]\n</pre> In\u00a0[18]: Copied! <pre>accuracy_score(preds, test.income_label)\n</pre> accuracy_score(preds, test.income_label) Out[18]: <pre>0.8685772773797339</pre> In\u00a0[19]: Copied! <pre>test.reset_index(drop=True, inplace=True)\n</pre> test.reset_index(drop=True, inplace=True) In\u00a0[20]: Copied! <pre>test[test.income_label == 0].head(1)\n</pre> test[test.income_label == 0].head(1) Out[20]: age workclass education marital_status occupation relationship race gender capital_gain capital_loss hours_per_week native_country income_label 0 26 Private Some-college Never-married Exec-managerial Not-in-family White Male 0 0 60 United-States 0 In\u00a0[21]: Copied! <pre>test[test.income_label == 1].head(1)\n</pre> test[test.income_label == 1].head(1) Out[21]: age workclass education marital_status occupation relationship race gender capital_gain capital_loss hours_per_week native_country income_label 3 36 Local-gov Doctorate Married-civ-spouse Prof-specialty Husband White Male 0 1887 50 United-States 1 <p>To get the feature importance of a test dataset, simply use the <code>explain</code> method</p> In\u00a0[22]: Copied! <pre>feat_imp_per_sample = trainer.explain(X_tab_test, save_step_masks=False)\n</pre> feat_imp_per_sample = trainer.explain(X_tab_test, save_step_masks=False) In\u00a0[23]: Copied! <pre>list(test.iloc[0].index[np.argsort(-feat_imp_per_sample[0])])\n</pre> list(test.iloc[0].index[np.argsort(-feat_imp_per_sample[0])]) Out[23]: <pre>['hours_per_week',\n 'education',\n 'relationship',\n 'occupation',\n 'workclass',\n 'capital_gain',\n 'native_country',\n 'marital_status',\n 'capital_loss',\n 'age',\n 'race',\n 'gender']</pre> In\u00a0[24]: Copied! <pre>list(test.iloc[3].index[np.argsort(-feat_imp_per_sample[3])])\n</pre> list(test.iloc[3].index[np.argsort(-feat_imp_per_sample[3])]) Out[24]: <pre>['age',\n 'capital_loss',\n 'hours_per_week',\n 'marital_status',\n 'native_country',\n 'relationship',\n 'race',\n 'education',\n 'occupation',\n 'capital_gain',\n 'gender',\n 'workclass']</pre> <p>We could do the same with the <code>ContextAttentionMLP</code></p> In\u00a0[25]: Copied! <pre>context_attn_mlp = ContextAttentionMLP(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    cat_embed_dropout=0.0,\n    input_dim=16,\n    attn_dropout=0.1,\n    attn_activation=\"relu\",\n)\n</pre> context_attn_mlp = ContextAttentionMLP(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     cat_embed_dropout=0.0,     input_dim=16,     attn_dropout=0.1,     attn_activation=\"relu\", ) In\u00a0[26]: Copied! <pre>mlp_model = WideDeep(deeptabular=context_attn_mlp)\n</pre> mlp_model = WideDeep(deeptabular=context_attn_mlp) In\u00a0[27]: Copied! <pre>mlp_optimizer = torch.optim.Adam(mlp_model.parameters(), lr=0.01, weight_decay=0.0)\n</pre> mlp_optimizer = torch.optim.Adam(mlp_model.parameters(), lr=0.01, weight_decay=0.0) In\u00a0[28]: Copied! <pre>mlp_lr_scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(\n    mlp_optimizer,\n    threshold=0.001,\n    threshold_mode=\"abs\",\n    patience=10,\n)\n</pre> mlp_lr_scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(     mlp_optimizer,     threshold=0.001,     threshold_mode=\"abs\",     patience=10, ) In\u00a0[29]: Copied! <pre>mlp_early_stopping = EarlyStopping(\n    min_delta=0.001, patience=30, restore_best_weights=True, verbose=True\n)\n</pre> mlp_early_stopping = EarlyStopping(     min_delta=0.001, patience=30, restore_best_weights=True, verbose=True ) In\u00a0[30]: Copied! <pre>mlp_trainer = Trainer(\n    mlp_model,\n    objective=\"binary\",\n    optimizers=mlp_optimizer,\n    lr_schedulers=mlp_lr_scheduler,\n    reducelronplateau_criterion=\"loss\",\n    callbacks=[mlp_early_stopping],\n    metrics=[Accuracy],\n)\n</pre> mlp_trainer = Trainer(     mlp_model,     objective=\"binary\",     optimizers=mlp_optimizer,     lr_schedulers=mlp_lr_scheduler,     reducelronplateau_criterion=\"loss\",     callbacks=[mlp_early_stopping],     metrics=[Accuracy], ) In\u00a0[31]: Copied! <pre>mlp_trainer.fit(\n    X_tab=X_tab_train,\n    target=target,\n    val_split=0.2,\n    n_epochs=100,\n    batch_size=128,\n    validation_freq=1,\n    feature_importance_sample_size=1000,\n)\n</pre> mlp_trainer.fit(     X_tab=X_tab_train,     target=target,     val_split=0.2,     n_epochs=100,     batch_size=128,     validation_freq=1,     feature_importance_sample_size=1000, ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 73.11it/s, loss=0.405, metrics={'acc': 0.8094}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 119.26it/s, loss=0.309, metrics={'acc': 0.8583}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 71.70it/s, loss=0.332, metrics={'acc': 0.8447}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 116.36it/s, loss=0.293, metrics={'acc': 0.8646}]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 78.42it/s, loss=0.319, metrics={'acc': 0.8505}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 118.05it/s, loss=0.293, metrics={'acc': 0.8654}]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 79.00it/s, loss=0.312, metrics={'acc': 0.8554}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 116.49it/s, loss=0.291, metrics={'acc': 0.8661}]\nepoch 5: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 79.50it/s, loss=0.308, metrics={'acc': 0.8583}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 120.48it/s, loss=0.287, metrics={'acc': 0.8669}]\nepoch 6: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 80.84it/s, loss=0.303, metrics={'acc': 0.8605}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 128.70it/s, loss=0.288, metrics={'acc': 0.8673}]\nepoch 7: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 79.93it/s, loss=0.301, metrics={'acc': 0.8597}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 121.47it/s, loss=0.298, metrics={'acc': 0.8628}]\nepoch 8: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 80.56it/s, loss=0.3, metrics={'acc': 0.8592}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 119.84it/s, loss=0.281, metrics={'acc': 0.8718}]\nepoch 9: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 79.17it/s, loss=0.298, metrics={'acc': 0.8619}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 126.32it/s, loss=0.28, metrics={'acc': 0.8716}]\nepoch 10: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 82.13it/s, loss=0.297, metrics={'acc': 0.8615}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 120.50it/s, loss=0.281, metrics={'acc': 0.8718}]\nepoch 11: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 82.54it/s, loss=0.293, metrics={'acc': 0.8641}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 122.57it/s, loss=0.284, metrics={'acc': 0.867}]\nepoch 12: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 80.92it/s, loss=0.293, metrics={'acc': 0.863}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 126.42it/s, loss=0.282, metrics={'acc': 0.8701}]\nepoch 13: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 79.61it/s, loss=0.293, metrics={'acc': 0.8635}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 117.56it/s, loss=0.276, metrics={'acc': 0.8719}]\nepoch 14: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 74.92it/s, loss=0.29, metrics={'acc': 0.8633}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 112.06it/s, loss=0.286, metrics={'acc': 0.8669}]\nepoch 15: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 76.83it/s, loss=0.291, metrics={'acc': 0.865}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 112.88it/s, loss=0.282, metrics={'acc': 0.8677}]\nepoch 16: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 76.74it/s, loss=0.29, metrics={'acc': 0.8653}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 120.85it/s, loss=0.285, metrics={'acc': 0.8672}]\nepoch 17: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 75.99it/s, loss=0.29, metrics={'acc': 0.865}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 113.53it/s, loss=0.282, metrics={'acc': 0.8681}]\nepoch 18: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 71.22it/s, loss=0.288, metrics={'acc': 0.8651}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 112.89it/s, loss=0.288, metrics={'acc': 0.8676}]\nepoch 19: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 75.85it/s, loss=0.29, metrics={'acc': 0.8661}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 114.26it/s, loss=0.284, metrics={'acc': 0.8662}]\nepoch 20: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 75.80it/s, loss=0.289, metrics={'acc': 0.8661}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 119.44it/s, loss=0.281, metrics={'acc': 0.8703}]\nepoch 21: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 75.72it/s, loss=0.29, metrics={'acc': 0.8661}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 113.04it/s, loss=0.285, metrics={'acc': 0.8648}]\nepoch 22: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 74.86it/s, loss=0.289, metrics={'acc': 0.8656}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 111.75it/s, loss=0.282, metrics={'acc': 0.8666}]\nepoch 23: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 76.68it/s, loss=0.289, metrics={'acc': 0.8668}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 115.86it/s, loss=0.282, metrics={'acc': 0.8724}]\nepoch 24: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 75.93it/s, loss=0.288, metrics={'acc': 0.8653}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 114.69it/s, loss=0.285, metrics={'acc': 0.8656}]\nepoch 25: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 79.99it/s, loss=0.284, metrics={'acc': 0.8671}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 118.27it/s, loss=0.277, metrics={'acc': 0.8707}]\nepoch 26: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 79.86it/s, loss=0.282, metrics={'acc': 0.8686}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 123.94it/s, loss=0.276, metrics={'acc': 0.8712}]\nepoch 27: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 77.86it/s, loss=0.283, metrics={'acc': 0.8691}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 114.11it/s, loss=0.277, metrics={'acc': 0.8716}]\nepoch 28: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 77.43it/s, loss=0.281, metrics={'acc': 0.8696}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 114.64it/s, loss=0.277, metrics={'acc': 0.8712}]\nepoch 29: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 77.39it/s, loss=0.281, metrics={'acc': 0.8696}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 115.83it/s, loss=0.277, metrics={'acc': 0.872}]\nepoch 30: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 74.93it/s, loss=0.28, metrics={'acc': 0.8706}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 113.07it/s, loss=0.275, metrics={'acc': 0.8714}]\nepoch 31: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 77.35it/s, loss=0.281, metrics={'acc': 0.8697}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 112.68it/s, loss=0.276, metrics={'acc': 0.872}]\nepoch 32: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 76.72it/s, loss=0.28, metrics={'acc': 0.8693}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 122.50it/s, loss=0.276, metrics={'acc': 0.8709}]\nepoch 33: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 76.68it/s, loss=0.28, metrics={'acc': 0.8716}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 110.07it/s, loss=0.277, metrics={'acc': 0.8709}]\nepoch 34: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 73.58it/s, loss=0.279, metrics={'acc': 0.8704}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 111.07it/s, loss=0.274, metrics={'acc': 0.8719}]\nepoch 35: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 74.03it/s, loss=0.28, metrics={'acc': 0.8687}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 110.15it/s, loss=0.276, metrics={'acc': 0.871}]\nepoch 36: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 73.11it/s, loss=0.279, metrics={'acc': 0.8706}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 111.10it/s, loss=0.278, metrics={'acc': 0.8705}]\nepoch 37: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 74.30it/s, loss=0.279, metrics={'acc': 0.869}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 110.75it/s, loss=0.279, metrics={'acc': 0.8702}]\nepoch 38: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 72.34it/s, loss=0.28, metrics={'acc': 0.8691}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 117.79it/s, loss=0.277, metrics={'acc': 0.8698}]\nepoch 39: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 75.49it/s, loss=0.279, metrics={'acc': 0.8694}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 115.25it/s, loss=0.279, metrics={'acc': 0.87}]\nepoch 40: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 75.51it/s, loss=0.28, metrics={'acc': 0.8694}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 110.90it/s, loss=0.277, metrics={'acc': 0.8694}]\nepoch 41: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 75.35it/s, loss=0.278, metrics={'acc': 0.8716}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 115.74it/s, loss=0.28, metrics={'acc': 0.8675}]\nepoch 42: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 76.35it/s, loss=0.279, metrics={'acc': 0.8695}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 122.76it/s, loss=0.277, metrics={'acc': 0.8699}]\nepoch 43: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:04&lt;00:00, 66.14it/s, loss=0.279, metrics={'acc': 0.8681}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 106.20it/s, loss=0.277, metrics={'acc': 0.8714}]\nepoch 44: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 73.37it/s, loss=0.279, metrics={'acc': 0.8704}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 115.18it/s, loss=0.277, metrics={'acc': 0.8716}]\nepoch 45: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 76.23it/s, loss=0.278, metrics={'acc': 0.8702}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 124.83it/s, loss=0.278, metrics={'acc': 0.8707}]\nepoch 46: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 77.12it/s, loss=0.278, metrics={'acc': 0.8704}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 126.62it/s, loss=0.279, metrics={'acc': 0.8693}]\nepoch 47: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 78.55it/s, loss=0.276, metrics={'acc': 0.8713}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 117.99it/s, loss=0.279, metrics={'acc': 0.8691}]\nepoch 48: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 77.25it/s, loss=0.278, metrics={'acc': 0.8719}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 124.52it/s, loss=0.278, metrics={'acc': 0.8695}]\nepoch 49: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 78.35it/s, loss=0.277, metrics={'acc': 0.8721}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 119.82it/s, loss=0.279, metrics={'acc': 0.8691}]\nepoch 50: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 77.15it/s, loss=0.277, metrics={'acc': 0.8717}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 122.62it/s, loss=0.278, metrics={'acc': 0.8699}]\nepoch 51: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 78.55it/s, loss=0.277, metrics={'acc': 0.8713}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 117.63it/s, loss=0.278, metrics={'acc': 0.87}]\nepoch 52: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 77.04it/s, loss=0.276, metrics={'acc': 0.8721}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 126.39it/s, loss=0.278, metrics={'acc': 0.8697}]\nepoch 53: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 78.15it/s, loss=0.277, metrics={'acc': 0.8721}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 127.56it/s, loss=0.278, metrics={'acc': 0.8699}]\nepoch 54: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 77.41it/s, loss=0.277, metrics={'acc': 0.8711}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 118.95it/s, loss=0.278, metrics={'acc': 0.8698}]\nepoch 55: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 76.35it/s, loss=0.277, metrics={'acc': 0.8718}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 126.90it/s, loss=0.278, metrics={'acc': 0.8699}]\nepoch 56: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 77.83it/s, loss=0.277, metrics={'acc': 0.8707}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 115.13it/s, loss=0.279, metrics={'acc': 0.8691}]\nepoch 57: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 75.20it/s, loss=0.277, metrics={'acc': 0.8722}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 123.16it/s, loss=0.279, metrics={'acc': 0.8691}]\nepoch 58: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 73.33it/s, loss=0.276, metrics={'acc': 0.871}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 123.37it/s, loss=0.278, metrics={'acc': 0.8691}]\nepoch 59: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 78.41it/s, loss=0.277, metrics={'acc': 0.8714}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 125.17it/s, loss=0.278, metrics={'acc': 0.8695}]\nepoch 60: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 79.26it/s, loss=0.276, metrics={'acc': 0.8721}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 120.60it/s, loss=0.278, metrics={'acc': 0.869}]\nepoch 61: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 75.88it/s, loss=0.278, metrics={'acc': 0.8703}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 124.47it/s, loss=0.278, metrics={'acc': 0.8692}]\nepoch 62: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 77.13it/s, loss=0.276, metrics={'acc': 0.8711}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 125.80it/s, loss=0.278, metrics={'acc': 0.8691}]\nepoch 63: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 77.20it/s, loss=0.277, metrics={'acc': 0.8715}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 118.50it/s, loss=0.278, metrics={'acc': 0.8695}]\nepoch 64: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 78.11it/s, loss=0.276, metrics={'acc': 0.8719}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 114.52it/s, loss=0.278, metrics={'acc': 0.869}]\n</pre> <pre>Best Epoch: 34. Best val_loss: 0.27449\nRestoring model weights from the end of the best epoch\n</pre> In\u00a0[32]: Copied! <pre>mlp_trainer.feature_importance\n</pre> mlp_trainer.feature_importance Out[32]: <pre>{'age': 0.116632804,\n 'workclass': 0.050255153,\n 'education': 0.094621316,\n 'marital_status': 0.12328919,\n 'occupation': 0.107893184,\n 'relationship': 0.11747801,\n 'race': 0.054717205,\n 'gender': 0.07514235,\n 'capital_gain': 0.059732802,\n 'capital_loss': 0.06738944,\n 'hours_per_week': 0.0610674,\n 'native_country': 0.07178114}</pre> In\u00a0[33]: Copied! <pre>mlp_preds = mlp_trainer.predict(X_tab=X_tab_test)\n</pre> mlp_preds = mlp_trainer.predict(X_tab=X_tab_test) <pre>predict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 39/39 [00:00&lt;00:00, 212.38it/s]\n</pre> In\u00a0[34]: Copied! <pre>accuracy_score(mlp_preds, test.income_label)\n</pre> accuracy_score(mlp_preds, test.income_label) Out[34]: <pre>0.8726714431934494</pre>"},{"location":"examples/17_feature_importance_via_attention_weights.html#feature-importance-via-the-attention-weights","title":"Feature Importance via the attention weights\u00b6","text":"<p>I will start by saying that I consider this feature of the library purely experimental. First of all I think there are multiple ways one could address finding the features importances for these models. However, and more importantly, one has to bear in mind that even tree-based algorithms on the same dataset produce different feature importances. This is more \"dramatic\" if one uses different techniques, such as shap or feature permutation (see for example this and references therein). All this to say that, sometimes, feature importance is just a measure contained within the experiment run, and for the model used.</p> <p>With that in mind, each instantiation of a deep tabular model, that has millions of trainable parameters, will potentially produce a different set of feature importances, even if the model has the same architecture. Moreover, this effect will become more apparent if the dataset is relatively easy and there are dependent/related columns so that one could get to the same success metric with different parameters.</p> <p>In summary, feature importances are implemented in this librray for all attention-based models for tabular data, with the exception of the <code>TabPerceiver</code>. However this functionality has to be used and interpreted with care and consider of value within the 'universe' (or context) of the model with which these features were produced.</p> <p>Nonetheless, let's have a look to how one would access to the feature importances when using this library.</p>"},{"location":"examples/18_wide_and_deep_for_recsys_pt1.html","title":"18 wide and deep for recsys pt1","text":"<p>The goal of this, and the companion (part 2) notebooks is to illustrate how one could use this library in the context of recommendation systems. In particular, this notebook and the scripts at the <code>wide_deep_for_recsys</code> dir are a response to this issue. Therefore, we will use the Kaggle notebook referred in that issue here.</p> <p>In order to keep the length of the notebook tractable, we will split this exercise in 2. In this first notebook we will prepare the data in almost the exact same way as it is done in the Kaggle notebook and also show how one could use <code>pytorch-widedeep</code> to build a model almost identical to the one in that notebook.</p> <p>In a second notebook, we will show how one could use this library to implement other models, still following the same problem formulation.</p> In\u00a0[1]: Copied! <pre>from pathlib import Path\nimport warnings\n\nimport pandas as pd\nfrom sklearn.model_selection import train_test_split\n\nfrom pytorch_widedeep.datasets import load_movielens100k\n</pre> from pathlib import Path import warnings  import pandas as pd from sklearn.model_selection import train_test_split  from pytorch_widedeep.datasets import load_movielens100k In\u00a0[2]: Copied! <pre>warnings.filterwarnings(\"ignore\")\n</pre> warnings.filterwarnings(\"ignore\") In\u00a0[3]: Copied! <pre>save_path = Path(\"prepared_data\")\nif not save_path.exists():\n    save_path.mkdir(parents=True, exist_ok=True)\n</pre> save_path = Path(\"prepared_data\") if not save_path.exists():     save_path.mkdir(parents=True, exist_ok=True) In\u00a0[4]: Copied! <pre>data, users, items = load_movielens100k(as_frame=True)\n</pre> data, users, items = load_movielens100k(as_frame=True) In\u00a0[5]: Copied! <pre># Alternatively, as specified in the docs: 'The last 19 fields are the genres' so:\n# list_of_genres = items.columns.tolist()[-19:]\nlist_of_genres = [\n    \"unknown\",\n    \"Action\",\n    \"Adventure\",\n    \"Animation\",\n    \"Children's\",\n    \"Comedy\",\n    \"Crime\",\n    \"Documentary\",\n    \"Drama\",\n    \"Fantasy\",\n    \"Film-Noir\",\n    \"Horror\",\n    \"Musical\",\n    \"Mystery\",\n    \"Romance\",\n    \"Sci-Fi\",\n    \"Thriller\",\n    \"War\",\n    \"Western\",\n]\n</pre> # Alternatively, as specified in the docs: 'The last 19 fields are the genres' so: # list_of_genres = items.columns.tolist()[-19:] list_of_genres = [     \"unknown\",     \"Action\",     \"Adventure\",     \"Animation\",     \"Children's\",     \"Comedy\",     \"Crime\",     \"Documentary\",     \"Drama\",     \"Fantasy\",     \"Film-Noir\",     \"Horror\",     \"Musical\",     \"Mystery\",     \"Romance\",     \"Sci-Fi\",     \"Thriller\",     \"War\",     \"Western\", ] <p>Let's first start by loading the interactions, user and item data</p> In\u00a0[6]: Copied! <pre>data.head()\n</pre> data.head() Out[6]: user_id movie_id rating timestamp 0 196 242 3 881250949 1 186 302 3 891717742 2 22 377 1 878887116 3 244 51 2 880606923 4 166 346 1 886397596 In\u00a0[7]: Copied! <pre>users.head()\n</pre> users.head() Out[7]: user_id age gender occupation zip_code 0 1 24 M technician 85711 1 2 53 F other 94043 2 3 23 M writer 32067 3 4 24 M technician 43537 4 5 33 F other 15213 In\u00a0[8]: Copied! <pre>items.head()\n</pre> items.head() Out[8]: movie_id movie_title release_date video_release_date IMDb_URL unknown Action Adventure Animation Children's ... Fantasy Film-Noir Horror Musical Mystery Romance Sci-Fi Thriller War Western 0 1 Toy Story (1995) 01-Jan-1995 NaN http://us.imdb.com/M/title-exact?Toy%20Story%2... 0 0 0 1 1 ... 0 0 0 0 0 0 0 0 0 0 1 2 GoldenEye (1995) 01-Jan-1995 NaN http://us.imdb.com/M/title-exact?GoldenEye%20(... 0 1 1 0 0 ... 0 0 0 0 0 0 0 1 0 0 2 3 Four Rooms (1995) 01-Jan-1995 NaN http://us.imdb.com/M/title-exact?Four%20Rooms%... 0 0 0 0 0 ... 0 0 0 0 0 0 0 1 0 0 3 4 Get Shorty (1995) 01-Jan-1995 NaN http://us.imdb.com/M/title-exact?Get%20Shorty%... 0 1 0 0 0 ... 0 0 0 0 0 0 0 0 0 0 4 5 Copycat (1995) 01-Jan-1995 NaN http://us.imdb.com/M/title-exact?Copycat%20(1995) 0 0 0 0 0 ... 0 0 0 0 0 0 0 1 0 0 <p>5 rows \u00d7 24 columns</p> In\u00a0[9]: Copied! <pre># adding a column with the number of movies watched per user\ndataset = data.sort_values([\"user_id\", \"timestamp\"]).reset_index(drop=True)\ndataset[\"one\"] = 1\ndataset[\"num_watched\"] = dataset.groupby(\"user_id\")[\"one\"].cumsum()\ndataset.drop(\"one\", axis=1, inplace=True)\ndataset.head()\n</pre> # adding a column with the number of movies watched per user dataset = data.sort_values([\"user_id\", \"timestamp\"]).reset_index(drop=True) dataset[\"one\"] = 1 dataset[\"num_watched\"] = dataset.groupby(\"user_id\")[\"one\"].cumsum() dataset.drop(\"one\", axis=1, inplace=True) dataset.head() Out[9]: user_id movie_id rating timestamp num_watched 0 1 168 5 874965478 1 1 1 172 5 874965478 2 2 1 165 5 874965518 3 3 1 156 4 874965556 4 4 1 196 5 874965677 5 In\u00a0[10]: Copied! <pre># adding a column with the mean rating at a point in time per user\ndataset[\"mean_rate\"] = (\n    dataset.groupby(\"user_id\")[\"rating\"].cumsum() / dataset[\"num_watched\"]\n)\ndataset.head()\n</pre> # adding a column with the mean rating at a point in time per user dataset[\"mean_rate\"] = (     dataset.groupby(\"user_id\")[\"rating\"].cumsum() / dataset[\"num_watched\"] ) dataset.head() Out[10]: user_id movie_id rating timestamp num_watched mean_rate 0 1 168 5 874965478 1 5.00 1 1 172 5 874965478 2 5.00 2 1 165 5 874965518 3 5.00 3 1 156 4 874965556 4 4.75 4 1 196 5 874965677 5 4.80 In\u00a0[11]: Copied! <pre>dataset[\"target\"] = dataset.groupby(\"user_id\")[\"movie_id\"].shift(-1)\n</pre> dataset[\"target\"] = dataset.groupby(\"user_id\")[\"movie_id\"].shift(-1) <p>Following the same processing used by the author in the before-mentioned Kaggle notebook, we build sequences of previous movies watched</p> In\u00a0[12]: Copied! <pre># Here the author builds the sequences\ndataset[\"prev_movies\"] = dataset[\"movie_id\"].apply(lambda x: str(x))\ndataset[\"prev_movies\"] = (\n    dataset.groupby(\"user_id\")[\"prev_movies\"]\n    .apply(lambda x: (x + \" \").cumsum().str.strip())\n    .reset_index(drop=True)\n)\ndataset[\"prev_movies\"] = dataset[\"prev_movies\"].apply(lambda x: x.split())\ndataset.head()\n</pre> # Here the author builds the sequences dataset[\"prev_movies\"] = dataset[\"movie_id\"].apply(lambda x: str(x)) dataset[\"prev_movies\"] = (     dataset.groupby(\"user_id\")[\"prev_movies\"]     .apply(lambda x: (x + \" \").cumsum().str.strip())     .reset_index(drop=True) ) dataset[\"prev_movies\"] = dataset[\"prev_movies\"].apply(lambda x: x.split()) dataset.head() Out[12]: user_id movie_id rating timestamp num_watched mean_rate target prev_movies 0 1 168 5 874965478 1 5.00 172.0 [168] 1 1 172 5 874965478 2 5.00 165.0 [168, 172] 2 1 165 5 874965518 3 5.00 156.0 [168, 172, 165] 3 1 156 4 874965556 4 4.75 196.0 [168, 172, 165, 156] 4 1 196 5 874965677 5 4.80 166.0 [168, 172, 165, 156, 196] <p>And now we add a <code>genre_rate</code> as the mean of all movies rated for a given genre per user</p> In\u00a0[13]: Copied! <pre>dataset = dataset.merge(items[[\"movie_id\"] + list_of_genres], on=\"movie_id\", how=\"left\")\nfor genre in list_of_genres:\n    dataset[f\"{genre}_rate\"] = dataset[genre] * dataset[\"rating\"]\n    dataset[genre] = dataset.groupby(\"user_id\")[genre].cumsum()\n    dataset[f\"{genre}_rate\"] = (\n        dataset.groupby(\"user_id\")[f\"{genre}_rate\"].cumsum() / dataset[genre]\n    )\ndataset[list_of_genres] = dataset[list_of_genres].apply(\n    lambda x: x / dataset[\"num_watched\"]\n)\ndataset.head()\n</pre> dataset = dataset.merge(items[[\"movie_id\"] + list_of_genres], on=\"movie_id\", how=\"left\") for genre in list_of_genres:     dataset[f\"{genre}_rate\"] = dataset[genre] * dataset[\"rating\"]     dataset[genre] = dataset.groupby(\"user_id\")[genre].cumsum()     dataset[f\"{genre}_rate\"] = (         dataset.groupby(\"user_id\")[f\"{genre}_rate\"].cumsum() / dataset[genre]     ) dataset[list_of_genres] = dataset[list_of_genres].apply(     lambda x: x / dataset[\"num_watched\"] ) dataset.head() Out[13]: user_id movie_id rating timestamp num_watched mean_rate target prev_movies unknown Action ... Fantasy_rate Film-Noir_rate Horror_rate Musical_rate Mystery_rate Romance_rate Sci-Fi_rate Thriller_rate War_rate Western_rate 0 1 168 5 874965478 1 5.00 172.0 [168] 0.0 0.000000 ... NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN 1 1 172 5 874965478 2 5.00 165.0 [168, 172] 0.0 0.500000 ... NaN NaN NaN NaN NaN 5.0 5.0 NaN 5.0 NaN 2 1 165 5 874965518 3 5.00 156.0 [168, 172, 165] 0.0 0.333333 ... NaN NaN NaN NaN NaN 5.0 5.0 NaN 5.0 NaN 3 1 156 4 874965556 4 4.75 196.0 [168, 172, 165, 156] 0.0 0.250000 ... NaN NaN NaN NaN NaN 5.0 5.0 4.0 5.0 NaN 4 1 196 5 874965677 5 4.80 166.0 [168, 172, 165, 156, 196] 0.0 0.200000 ... NaN NaN NaN NaN NaN 5.0 5.0 4.0 5.0 NaN <p>5 rows \u00d7 46 columns</p> <p>Adding user features</p> In\u00a0[14]: Copied! <pre>dataset = dataset.merge(users, on=\"user_id\", how=\"left\")\ndataset.head()\n</pre> dataset = dataset.merge(users, on=\"user_id\", how=\"left\") dataset.head() Out[14]: user_id movie_id rating timestamp num_watched mean_rate target prev_movies unknown Action ... Mystery_rate Romance_rate Sci-Fi_rate Thriller_rate War_rate Western_rate age gender occupation zip_code 0 1 168 5 874965478 1 5.00 172.0 [168] 0.0 0.000000 ... NaN NaN NaN NaN NaN NaN 24 M technician 85711 1 1 172 5 874965478 2 5.00 165.0 [168, 172] 0.0 0.500000 ... NaN 5.0 5.0 NaN 5.0 NaN 24 M technician 85711 2 1 165 5 874965518 3 5.00 156.0 [168, 172, 165] 0.0 0.333333 ... NaN 5.0 5.0 NaN 5.0 NaN 24 M technician 85711 3 1 156 4 874965556 4 4.75 196.0 [168, 172, 165, 156] 0.0 0.250000 ... NaN 5.0 5.0 4.0 5.0 NaN 24 M technician 85711 4 1 196 5 874965677 5 4.80 166.0 [168, 172, 165, 156, 196] 0.0 0.200000 ... NaN 5.0 5.0 4.0 5.0 NaN 24 M technician 85711 <p>5 rows \u00d7 50 columns</p> <p>Again, we use the same settings as those in the Kaggle notebook, but <code>COLD_START_TRESH</code> is pretty aggressive</p> In\u00a0[15]: Copied! <pre>COLD_START_TRESH = 5\n\nfiltred_data = dataset[\n    (dataset[\"num_watched\"] &gt;= COLD_START_TRESH) &amp; ~(dataset[\"target\"].isna())\n].sort_values(\"timestamp\")\ntrain_data, _test_data = train_test_split(filtred_data, test_size=0.2, shuffle=False)\nvalid_data, test_data = train_test_split(_test_data, test_size=0.5, shuffle=False)\n</pre> COLD_START_TRESH = 5  filtred_data = dataset[     (dataset[\"num_watched\"] &gt;= COLD_START_TRESH) &amp; ~(dataset[\"target\"].isna()) ].sort_values(\"timestamp\") train_data, _test_data = train_test_split(filtred_data, test_size=0.2, shuffle=False) valid_data, test_data = train_test_split(_test_data, test_size=0.5, shuffle=False) In\u00a0[16]: Copied! <pre>cols_to_drop = [\n    # \"rating\",\n    \"timestamp\",\n    \"num_watched\",\n]\n\ndf_train = train_data.drop(cols_to_drop, axis=1)\ndf_valid = valid_data.drop(cols_to_drop, axis=1)\ndf_test = test_data.drop(cols_to_drop, axis=1)\n\ndf_train.to_pickle(save_path / \"df_train.pkl\")\ndf_valid.to_pickle(save_path / \"df_valid.pkl\")\ndf_test.to_pickle(save_path / \"df_test.pkl\")\n</pre> cols_to_drop = [     # \"rating\",     \"timestamp\",     \"num_watched\", ]  df_train = train_data.drop(cols_to_drop, axis=1) df_valid = valid_data.drop(cols_to_drop, axis=1) df_test = test_data.drop(cols_to_drop, axis=1)  df_train.to_pickle(save_path / \"df_train.pkl\") df_valid.to_pickle(save_path / \"df_valid.pkl\") df_test.to_pickle(save_path / \"df_test.pkl\") <p>Let's now build a model that is nearly identical to the one use in the Kaggle notebook</p> In\u00a0[17]: Copied! <pre>import numpy as np\nimport torch\nfrom torch import nn\nfrom scipy.sparse import coo_matrix\n\nfrom pytorch_widedeep import Trainer\nfrom pytorch_widedeep.models import TabMlp, BasicRNN, WideDeep\nfrom pytorch_widedeep.preprocessing import TabPreprocessor\n</pre> import numpy as np import torch from torch import nn from scipy.sparse import coo_matrix  from pytorch_widedeep import Trainer from pytorch_widedeep.models import TabMlp, BasicRNN, WideDeep from pytorch_widedeep.preprocessing import TabPreprocessor In\u00a0[18]: Copied! <pre>device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n\nsave_path = Path(\"prepared_data\")\n\nPAD_IDX = 0\n</pre> device = \"cuda\" if torch.cuda.is_available() else \"cpu\"  save_path = Path(\"prepared_data\")  PAD_IDX = 0 <p>Let's use some of the functions the author of the kaggle's notebook uses to prepare the data</p> In\u00a0[19]: Copied! <pre>def get_coo_indexes(lil):\n    rows = []\n    cols = []\n    for i, el in enumerate(lil):\n        if type(el) != list:\n            el = [el]\n        for j in el:\n            rows.append(i)\n            cols.append(j)\n    return rows, cols\n\n\ndef get_sparse_features(series, shape):\n    coo_indexes = get_coo_indexes(series.tolist())\n    sparse_df = coo_matrix(\n        (np.ones(len(coo_indexes[0])), (coo_indexes[0], coo_indexes[1])), shape=shape\n    )\n    return sparse_df\n\n\ndef sparse_to_idx(data, pad_idx=-1):\n    indexes = data.nonzero()\n    indexes_df = pd.DataFrame()\n    indexes_df[\"rows\"] = indexes[0]\n    indexes_df[\"cols\"] = indexes[1]\n    mdf = indexes_df.groupby(\"rows\").apply(lambda x: x[\"cols\"].tolist())\n    max_len = mdf.apply(lambda x: len(x)).max()\n    return mdf.apply(lambda x: pd.Series(x + [pad_idx] * (max_len - len(x)))).values\n</pre> def get_coo_indexes(lil):     rows = []     cols = []     for i, el in enumerate(lil):         if type(el) != list:             el = [el]         for j in el:             rows.append(i)             cols.append(j)     return rows, cols   def get_sparse_features(series, shape):     coo_indexes = get_coo_indexes(series.tolist())     sparse_df = coo_matrix(         (np.ones(len(coo_indexes[0])), (coo_indexes[0], coo_indexes[1])), shape=shape     )     return sparse_df   def sparse_to_idx(data, pad_idx=-1):     indexes = data.nonzero()     indexes_df = pd.DataFrame()     indexes_df[\"rows\"] = indexes[0]     indexes_df[\"cols\"] = indexes[1]     mdf = indexes_df.groupby(\"rows\").apply(lambda x: x[\"cols\"].tolist())     max_len = mdf.apply(lambda x: len(x)).max()     return mdf.apply(lambda x: pd.Series(x + [pad_idx] * (max_len - len(x)))).values <p>For the time being, we will not use a validation set for hyperparameter optimization, and we will simply concatenate the validation and the test set in one test set. I simply splitted the data into train/valid/test in case the reader wants to actually do hyperparameter optimization (and because I know in the future I will).</p> <p>There is also another caveat worth mentioning, related to the indexing of the movies. To build the matrices of movies watched, we use the entire dataset. A more realistic (and correct) approach would be to use ONLY the movies that appear in the training set and consider <code>unknown</code> or <code>unseen</code> those in the testing set that have not been seen during training. Nonetheless, this will not affect the purposes of this notebook, which is to illustrate how one could use <code>pytorch-widedeep</code> to build a recommendation algorithm. However, if one wanted to explore the performance of different algorithms in a \"proper\" way, these \"details\" need to be accounted for.</p> In\u00a0[20]: Copied! <pre>df_test = pd.concat([df_valid, df_test], ignore_index=True)\n</pre> df_test = pd.concat([df_valid, df_test], ignore_index=True) In\u00a0[21]: Copied! <pre>id_cols = [\"user_id\", \"movie_id\"]\nmax_movie_index = max(df_train.movie_id.max(), df_test.movie_id.max())\n</pre> id_cols = [\"user_id\", \"movie_id\"] max_movie_index = max(df_train.movie_id.max(), df_test.movie_id.max()) In\u00a0[22]: Copied! <pre>X_train = df_train.drop(id_cols + [\"rating\", \"prev_movies\", \"target\"], axis=1)\ny_train = np.array(df_train.target.values, dtype=\"int64\")\ntrain_movies_watched = get_sparse_features(\n    df_train[\"prev_movies\"], (len(df_train), max_movie_index + 1)\n)\n\nX_test = df_test.drop(id_cols + [\"rating\", \"prev_movies\", \"target\"], axis=1)\ny_test = np.array(df_test.target.values, dtype=\"int64\")\ntest_movies_watched = get_sparse_features(\n    df_test[\"prev_movies\"], (len(df_test), max_movie_index + 1)\n)\n</pre> X_train = df_train.drop(id_cols + [\"rating\", \"prev_movies\", \"target\"], axis=1) y_train = np.array(df_train.target.values, dtype=\"int64\") train_movies_watched = get_sparse_features(     df_train[\"prev_movies\"], (len(df_train), max_movie_index + 1) )  X_test = df_test.drop(id_cols + [\"rating\", \"prev_movies\", \"target\"], axis=1) y_test = np.array(df_test.target.values, dtype=\"int64\") test_movies_watched = get_sparse_features(     df_test[\"prev_movies\"], (len(df_test), max_movie_index + 1) ) <p>let's have a look to the information in each dataset</p> In\u00a0[23]: Copied! <pre>X_train.head()\n</pre> X_train.head() Out[23]: mean_rate unknown Action Adventure Animation Children's Comedy Crime Documentary Drama ... Mystery_rate Romance_rate Sci-Fi_rate Thriller_rate War_rate Western_rate age gender occupation zip_code 25423 4.000000 0.0 0.400000 0.200000 0.0 0.0 0.400000 0.0 0.0 0.200000 ... NaN 4.0 4.0 4.000000 4.0 NaN 21 M student 48823 25425 4.000000 0.0 0.285714 0.142857 0.0 0.0 0.428571 0.0 0.0 0.285714 ... NaN 4.0 4.0 4.000000 4.0 NaN 21 M student 48823 25424 4.000000 0.0 0.333333 0.166667 0.0 0.0 0.333333 0.0 0.0 0.333333 ... NaN 4.0 4.0 4.000000 4.0 NaN 21 M student 48823 25426 3.875000 0.0 0.250000 0.125000 0.0 0.0 0.375000 0.0 0.0 0.250000 ... NaN 4.0 4.0 3.666667 4.0 NaN 21 M student 48823 25427 3.888889 0.0 0.222222 0.111111 0.0 0.0 0.333333 0.0 0.0 0.333333 ... NaN 4.0 4.0 3.666667 4.0 NaN 21 M student 48823 <p>5 rows \u00d7 43 columns</p> In\u00a0[24]: Copied! <pre>y_train\n</pre> y_train Out[24]: <pre>array([772, 288, 108, ..., 183, 432, 509])</pre> In\u00a0[25]: Copied! <pre>train_movies_watched\n</pre> train_movies_watched Out[25]: <pre>&lt;76228x1683 sparse matrix of type '&lt;class 'numpy.float64'&gt;'\n\twith 7957390 stored elements in COOrdinate format&gt;</pre> In\u00a0[26]: Copied! <pre>sorted(df_train.prev_movies.tolist()[0])\n</pre> sorted(df_train.prev_movies.tolist()[0]) Out[26]: <pre>['173', '185', '255', '286', '298']</pre> In\u00a0[27]: Copied! <pre>np.where(train_movies_watched.todense()[0])\n</pre> np.where(train_movies_watched.todense()[0]) Out[27]: <pre>(array([0, 0, 0, 0, 0]), array([173, 185, 255, 286, 298]))</pre> <p>And from now on is when the specifics related to this library start to appear. The only component that is going to be a bit different is the so-called tabular component, referred as <code>continuous</code> in the notebook.</p> <p>In the case of <code>pytorch-widedeep</code> we have the <code>TabPreprocessor</code> that allows for a lot of flexibility as to how we would like to process the tabular component of this Wide and Deep model. In other words, here our tabular component is a bit more elaborated than that in the notebook, just a bit...</p> In\u00a0[28]: Copied! <pre>cat_cols = [\"gender\", \"occupation\", \"zip_code\"]\ncont_cols = [c for c in X_train if c not in cat_cols]\ntab_preprocessor = TabPreprocessor(\n    cat_embed_cols=cat_cols,\n    continuous_cols=cont_cols,\n)\n</pre> cat_cols = [\"gender\", \"occupation\", \"zip_code\"] cont_cols = [c for c in X_train if c not in cat_cols] tab_preprocessor = TabPreprocessor(     cat_embed_cols=cat_cols,     continuous_cols=cont_cols, ) In\u00a0[29]: Copied! <pre>X_train_tab = tab_preprocessor.fit_transform(X_train.fillna(0))\nX_test_tab = tab_preprocessor.transform(X_test.fillna(0))\n</pre> X_train_tab = tab_preprocessor.fit_transform(X_train.fillna(0)) X_test_tab = tab_preprocessor.transform(X_test.fillna(0)) <p>Now, in the notebook, the author moves the sparse matrices to sparse tensors and then turns them into dense tensors. In reality, this is not neccessary, one could feed sparse tensors to <code>nn.Linear</code> layers in pytorch. Nonetheless, this is not the most efficient implementation and is the reason why in our library the wide, linear component is implemented as an embedding layer.</p> <p>Nonetheless, to reproduce the notebook the best we can and because currently the <code>Wide</code> model in <code>pytorch-widedeep</code> is not designed to receive sparse tensors (we might consider implementing this functionality), we will turn the sparse COO matrices into dense arrays. We will then code a fairly simple, custom <code>Wide</code> component.</p> In\u00a0[30]: Copied! <pre>X_train_wide = np.array(train_movies_watched.todense())\nX_test_wide = np.array(test_movies_watched.todense())\n</pre> X_train_wide = np.array(train_movies_watched.todense()) X_test_wide = np.array(test_movies_watched.todense()) <p>Finally, the author of the notebook uses a simple <code>Embedding</code> layer to encode the sequences of movies watched, the <code>prev_movies</code> columns. In my opinion, there is an element of information redundancy here. This is because the wide and text components have implicitely the same information, but in different form. Moreover, both of the models used for these two components ignore the sequential element in the data. Nonetheless, we want to reproduce the Kaggle notebook as close as possible, AND as one can explore later (by simply performing simple ablation studies), the wide component seems to carry most of the predictive power.</p> In\u00a0[31]: Copied! <pre>X_train_text = sparse_to_idx(train_movies_watched, pad_idx=PAD_IDX)\nX_test_text = sparse_to_idx(test_movies_watched, pad_idx=PAD_IDX)\n</pre> X_train_text = sparse_to_idx(train_movies_watched, pad_idx=PAD_IDX) X_test_text = sparse_to_idx(test_movies_watched, pad_idx=PAD_IDX) <p>Let's now build the models</p> In\u00a0[32]: Copied! <pre>class Wide(nn.Module):\n    def __init__(self, input_dim: int, pred_dim: int):\n        super().__init__()\n\n        self.input_dim = input_dim\n        self.pred_dim = pred_dim\n\n        # When I coded the library I never though that someone would want to code\n        # their own wide component. However, if you do, the wide component must have\n        # a 'wide_linear' attribute. In other words, the linear layer must be\n        # called 'wide_linear'\n        self.wide_linear = nn.Linear(input_dim, pred_dim)\n\n    def forward(self, X):\n        out = self.wide_linear(X.type(torch.float32))\n        return out\n\n\nwide = Wide(X_train_wide.shape[1], max_movie_index + 1)\n</pre> class Wide(nn.Module):     def __init__(self, input_dim: int, pred_dim: int):         super().__init__()          self.input_dim = input_dim         self.pred_dim = pred_dim          # When I coded the library I never though that someone would want to code         # their own wide component. However, if you do, the wide component must have         # a 'wide_linear' attribute. In other words, the linear layer must be         # called 'wide_linear'         self.wide_linear = nn.Linear(input_dim, pred_dim)      def forward(self, X):         out = self.wide_linear(X.type(torch.float32))         return out   wide = Wide(X_train_wide.shape[1], max_movie_index + 1) In\u00a0[33]: Copied! <pre>wide\n</pre> wide Out[33]: <pre>Wide(\n  (wide_linear): Linear(in_features=1683, out_features=1683, bias=True)\n)</pre> In\u00a0[34]: Copied! <pre>class SimpleEmbed(nn.Module):\n    def __init__(self, vocab_size: int, embed_dim: int, pad_idx: int):\n        super().__init__()\n\n        self.vocab_size = vocab_size\n        self.embed_dim = embed_dim\n        self.pad_idx = pad_idx\n\n        # The sequences of movies watched are simply embedded in the Kaggle\n        # notebook. No RNN, Transformer or any model is used\n        self.embed = nn.Embedding(vocab_size, embed_dim, padding_idx=pad_idx)\n\n    def forward(self, X):\n        embed = self.embed(X)\n        embed_mean = torch.mean(embed, dim=1)\n        return embed_mean\n\n    @property\n    def output_dim(self) -&gt; int:\n        # All deep components in a custom 'pytorch-widedeep' model must have\n        # an output_dim property\n        return self.embed_dim\n\n\n#  In the notebook the author uses simply embeddings\nsimple_embed = SimpleEmbed(max_movie_index + 1, 16, 0)\n</pre> class SimpleEmbed(nn.Module):     def __init__(self, vocab_size: int, embed_dim: int, pad_idx: int):         super().__init__()          self.vocab_size = vocab_size         self.embed_dim = embed_dim         self.pad_idx = pad_idx          # The sequences of movies watched are simply embedded in the Kaggle         # notebook. No RNN, Transformer or any model is used         self.embed = nn.Embedding(vocab_size, embed_dim, padding_idx=pad_idx)      def forward(self, X):         embed = self.embed(X)         embed_mean = torch.mean(embed, dim=1)         return embed_mean      @property     def output_dim(self) -&gt; int:         # All deep components in a custom 'pytorch-widedeep' model must have         # an output_dim property         return self.embed_dim   #  In the notebook the author uses simply embeddings simple_embed = SimpleEmbed(max_movie_index + 1, 16, 0) In\u00a0[35]: Copied! <pre>simple_embed\n</pre> simple_embed Out[35]: <pre>SimpleEmbed(\n  (embed): Embedding(1683, 16, padding_idx=0)\n)</pre> <p>Maybe one would like to use an RNN to account for the sequence nature of the problem. If that was the case it would be as easy as:</p> In\u00a0[36]: Copied! <pre>basic_rnn = BasicRNN(\n    vocab_size=max_movie_index + 1,\n    embed_dim=16,\n    hidden_dim=32,\n    n_layers=2,\n    rnn_type=\"gru\",\n)\n</pre> basic_rnn = BasicRNN(     vocab_size=max_movie_index + 1,     embed_dim=16,     hidden_dim=32,     n_layers=2,     rnn_type=\"gru\", ) <p>And finally, the tabular component, which is the notebook is simply a stak of linear + Rely layers. In our case we have an embedding layer before the linear layers to encode categorial and numerical cols</p> In\u00a0[37]: Copied! <pre>tab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=tab_preprocessor.continuous_cols,\n    cont_norm_layer=None,\n    mlp_hidden_dims=[1024, 512, 256],\n    mlp_activation=\"relu\",\n)\n</pre> tab_mlp = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     continuous_cols=tab_preprocessor.continuous_cols,     cont_norm_layer=None,     mlp_hidden_dims=[1024, 512, 256],     mlp_activation=\"relu\", ) In\u00a0[38]: Copied! <pre>tab_mlp\n</pre> tab_mlp Out[38]: <pre>TabMlp(\n  (cat_embed): DiffSizeCatEmbeddings(\n    (embed_layers): ModuleDict(\n      (emb_layer_gender): Embedding(3, 2, padding_idx=0)\n      (emb_layer_occupation): Embedding(22, 9, padding_idx=0)\n      (emb_layer_zip_code): Embedding(648, 60, padding_idx=0)\n    )\n    (embedding_dropout): Dropout(p=0.0, inplace=False)\n  )\n  (cont_norm): Identity()\n  (encoder): MLP(\n    (mlp): Sequential(\n      (dense_layer_0): Sequential(\n        (0): Linear(in_features=111, out_features=1024, bias=True)\n        (1): ReLU(inplace=True)\n        (2): Dropout(p=0.1, inplace=False)\n      )\n      (dense_layer_1): Sequential(\n        (0): Linear(in_features=1024, out_features=512, bias=True)\n        (1): ReLU(inplace=True)\n        (2): Dropout(p=0.1, inplace=False)\n      )\n      (dense_layer_2): Sequential(\n        (0): Linear(in_features=512, out_features=256, bias=True)\n        (1): ReLU(inplace=True)\n        (2): Dropout(p=0.1, inplace=False)\n      )\n    )\n  )\n)</pre> <p>Finally, we simply wrap up all models with the <code>WideDeep</code> 'collector' class and we are ready to train.</p> In\u00a0[39]: Copied! <pre>wide_deep_model = WideDeep(\n    wide=wide, deeptabular=tab_mlp, deeptext=simple_embed, pred_dim=max_movie_index + 1\n)\n</pre> wide_deep_model = WideDeep(     wide=wide, deeptabular=tab_mlp, deeptext=simple_embed, pred_dim=max_movie_index + 1 ) In\u00a0[40]: Copied! <pre>wide_deep_model\n</pre> wide_deep_model Out[40]: <pre>WideDeep(\n  (wide): Wide(\n    (wide_linear): Linear(in_features=1683, out_features=1683, bias=True)\n  )\n  (deeptabular): Sequential(\n    (0): TabMlp(\n      (cat_embed): DiffSizeCatEmbeddings(\n        (embed_layers): ModuleDict(\n          (emb_layer_gender): Embedding(3, 2, padding_idx=0)\n          (emb_layer_occupation): Embedding(22, 9, padding_idx=0)\n          (emb_layer_zip_code): Embedding(648, 60, padding_idx=0)\n        )\n        (embedding_dropout): Dropout(p=0.0, inplace=False)\n      )\n      (cont_norm): Identity()\n      (encoder): MLP(\n        (mlp): Sequential(\n          (dense_layer_0): Sequential(\n            (0): Linear(in_features=111, out_features=1024, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_1): Sequential(\n            (0): Linear(in_features=1024, out_features=512, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_2): Sequential(\n            (0): Linear(in_features=512, out_features=256, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=256, out_features=1683, bias=True)\n  )\n  (deeptext): Sequential(\n    (0): SimpleEmbed(\n      (embed): Embedding(1683, 16, padding_idx=0)\n    )\n    (1): Linear(in_features=16, out_features=1683, bias=True)\n  )\n)</pre> <p>Note that the main difference between this wide and deep model and the Wide and Deep model in the Kaggle notebook is that in that notebook, the author concatenates the embedings and the tabular features, then passes this concatenation through a stack of linear + Relu layers with a final output dim of 256. Then concatenates this output with the binary features and connects this concatenation with the final linear layer (so the final weights are of dim (batch_size, 256 + 1683)). Our implementation follows the notation of the original paper and instead of concatenating the tabular, text and wide components and then connect them to the output neurons, we first compute their output, and then add it (see here: https://arxiv.org/pdf/1606.07792.pdf, their Eq 3). Note that this is effectively the same, with the caveat that while in one case one initialises a big weight matrix \"at once\", in our implementation we initialise different matrices for different components. Anyway, let's give it a go.</p> In\u00a0[41]: Copied! <pre>trainer = Trainer(\n    model=wide_deep_model,\n    objective=\"multiclass\",\n    custom_loss_function=nn.CrossEntropyLoss(ignore_index=PAD_IDX),\n    optimizers=torch.optim.Adam(wide_deep_model.parameters(), lr=1e-3),\n)\n</pre> trainer = Trainer(     model=wide_deep_model,     objective=\"multiclass\",     custom_loss_function=nn.CrossEntropyLoss(ignore_index=PAD_IDX),     optimizers=torch.optim.Adam(wide_deep_model.parameters(), lr=1e-3), ) In\u00a0[42]: Copied! <pre>trainer.fit(\n    X_train={\n        \"X_wide\": X_train_wide,\n        \"X_tab\": X_train_tab,\n        \"X_text\": X_train_text,\n        \"target\": y_train,\n    },\n    X_val={\n        \"X_wide\": X_test_wide,\n        \"X_tab\": X_test_tab,\n        \"X_text\": X_test_text,\n        \"target\": y_test,\n    },\n    n_epochs=5,\n    batch_size=512,\n    shuffle=False,\n)\n</pre> trainer.fit(     X_train={         \"X_wide\": X_train_wide,         \"X_tab\": X_train_tab,         \"X_text\": X_train_text,         \"target\": y_train,     },     X_val={         \"X_wide\": X_test_wide,         \"X_tab\": X_test_tab,         \"X_text\": X_test_text,         \"target\": y_test,     },     n_epochs=5,     batch_size=512,     shuffle=False, ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 149/149 [00:19&lt;00:00,  7.66it/s, loss=6.66]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 38/38 [00:02&lt;00:00, 18.75it/s, loss=6.6]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 149/149 [00:21&lt;00:00,  6.95it/s, loss=5.97]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 38/38 [00:01&lt;00:00, 21.03it/s, loss=6.52]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 149/149 [00:19&lt;00:00,  7.51it/s, loss=5.65]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 38/38 [00:01&lt;00:00, 20.16it/s, loss=6.53]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 149/149 [00:23&lt;00:00,  6.29it/s, loss=5.41]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 38/38 [00:02&lt;00:00, 13.97it/s, loss=6.57]\nepoch 5: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 149/149 [00:19&lt;00:00,  7.58it/s, loss=5.2]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 38/38 [00:02&lt;00:00, 18.82it/s, loss=6.63]\n</pre> <p>Now one could continue to the 'compare' metrics section of the Kaggle notebook. However, for the purposes of illustrating how one could use <code>pytorch-widedeep</code> to build recommendation algorithms we consider this notebook completed and move onto part 2</p>"},{"location":"examples/18_wide_and_deep_for_recsys_pt1.html#problem-formulation","title":"Problem formulation\u00b6","text":"<p>In this particular exercise the problem is formulated as predicting the next movie that will be watched (in consequence the last interactions will be discarded)</p>"},{"location":"examples/18_wide_and_deep_for_recsys_pt2.html","title":"18 wide and deep for recsys pt2","text":"<p>This is the second of the two notebooks where we aim to illustrate how one could use this library to build recommendation algorithms using the example in this Kaggle notebook as guidance. In the previous notebook we used <code>pytorch-widedeep</code> to build a model that replicated almost exactly that in the notebook. In this, shorter notebook we will show how one could use the library to explore other models, following the same problem formulation, this is: given a state of a user at a certain point in time having watched a series of movies, our goal is to predict which movie the user will watch next.</p> <p>Assuming that one has read (and run) the previous notebook, the required data will be stored in a local dir called <code>prepared_data</code>, so let's read it:</p> In\u00a0[1]: Copied! <pre>from pathlib import Path\n\nimport numpy as np\nimport torch\nimport pandas as pd\nfrom torch import nn\n\nfrom pytorch_widedeep import Trainer\nfrom pytorch_widedeep.utils import pad_sequences\nfrom pytorch_widedeep.models import TabMlp, WideDeep, Transformer\nfrom pytorch_widedeep.preprocessing import TabPreprocessor\n</pre> from pathlib import Path  import numpy as np import torch import pandas as pd from torch import nn  from pytorch_widedeep import Trainer from pytorch_widedeep.utils import pad_sequences from pytorch_widedeep.models import TabMlp, WideDeep, Transformer from pytorch_widedeep.preprocessing import TabPreprocessor In\u00a0[2]: Copied! <pre>save_path = Path(\"prepared_data\")\n\nPAD_IDX = 0\n\nid_cols = [\"user_id\", \"movie_id\"]\n\ndf_train = pd.read_pickle(save_path / \"df_train.pkl\")\ndf_valid = pd.read_pickle(save_path / \"df_valid.pkl\")\ndf_test = pd.read_pickle(save_path / \"df_test.pkl\")\n</pre> save_path = Path(\"prepared_data\")  PAD_IDX = 0  id_cols = [\"user_id\", \"movie_id\"]  df_train = pd.read_pickle(save_path / \"df_train.pkl\") df_valid = pd.read_pickle(save_path / \"df_valid.pkl\") df_test = pd.read_pickle(save_path / \"df_test.pkl\") <p>...remember that in the previous notebook we explained that we are not  going to use a validation set here (in a real-world example, or simply a more realistic example, one should always use it).</p> In\u00a0[3]: Copied! <pre>df_test = pd.concat([df_valid, df_test], ignore_index=True)\n</pre> df_test = pd.concat([df_valid, df_test], ignore_index=True) <p>Also remember that, in the previous notebook we discussed that the <code>'maxlen'</code> and <code>'max_movie_index'</code> parameters should be computed using only the train set. In particular, to properly do the tokenization, one would have to use ONLY train tokens and add a token for new 'unknown'/'unseen' movies in the test set. This can also be done with this library or manually, so I will leave it to the reader to implement that tokenzation appraoch.</p> In\u00a0[4]: Copied! <pre>maxlen = max(\n    df_train.prev_movies.apply(lambda x: len(x)).max(),\n    df_test.prev_movies.apply(lambda x: len(x)).max(),\n)\n\nmax_movie_index = max(df_train.movie_id.max(), df_test.movie_id.max())\n</pre> maxlen = max(     df_train.prev_movies.apply(lambda x: len(x)).max(),     df_test.prev_movies.apply(lambda x: len(x)).max(), )  max_movie_index = max(df_train.movie_id.max(), df_test.movie_id.max()) <p>From now one things are pretty simple, moreover bearing in mind that in this example we are not going to use a wide component since, in pple, one would believe that the information in that component is also 'carried' by the movie sequences (However in the previous notebook, if one performs ablation studies, these suggest that most of the prediction power comes from the linear, wide model).</p> <p>In the example here we are going to explore one (of many) possibilities. We are simply going to encode the triplet <code>(user, item, rating)</code> and use it as a <code>deeptabular</code> component and the sequences of previously watched movies as the <code>deeptext</code> component. For the <code>deeptext</code> component we are going to use a basic encoder-only transformer model.</p> <p>Let's start with the tabular data preparation</p> In\u00a0[5]: Copied! <pre>df_train_user_item = df_train[[\"user_id\", \"movie_id\", \"rating\"]]\ntrain_movies_sequences = df_train.prev_movies.apply(\n    lambda x: [int(el) for el in x]\n).to_list()\ny_train = df_train.target.values.astype(int)\n\ndf_test_user_item = df_train[[\"user_id\", \"movie_id\", \"rating\"]]\ntest_movies_sequences = df_test.prev_movies.apply(\n    lambda x: [int(el) for el in x]\n).to_list()\ny_test = df_test.target.values.astype(int)\n\ntab_preprocessor = tab_preprocessor = TabPreprocessor(\n    cat_embed_cols=[\"user_id\", \"movie_id\", \"rating\"],\n)\nX_train_tab = tab_preprocessor.fit_transform(df_train_user_item)\nX_test_tab = tab_preprocessor.transform(df_test_user_item)\n</pre> df_train_user_item = df_train[[\"user_id\", \"movie_id\", \"rating\"]] train_movies_sequences = df_train.prev_movies.apply(     lambda x: [int(el) for el in x] ).to_list() y_train = df_train.target.values.astype(int)  df_test_user_item = df_train[[\"user_id\", \"movie_id\", \"rating\"]] test_movies_sequences = df_test.prev_movies.apply(     lambda x: [int(el) for el in x] ).to_list() y_test = df_test.target.values.astype(int)  tab_preprocessor = tab_preprocessor = TabPreprocessor(     cat_embed_cols=[\"user_id\", \"movie_id\", \"rating\"], ) X_train_tab = tab_preprocessor.fit_transform(df_train_user_item) X_test_tab = tab_preprocessor.transform(df_test_user_item) <p>And not the text component, simply padding the sequences:</p> In\u00a0[6]: Copied! <pre>X_train_text = np.array(\n    [\n        pad_sequences(\n            s,\n            maxlen=maxlen,\n            pad_first=False,\n            pad_idx=PAD_IDX,\n        )\n        for s in train_movies_sequences\n    ]\n)\nX_test_text = np.array(\n    [\n        pad_sequences(\n            s,\n            maxlen=maxlen,\n            pad_first=False,\n            pad_idx=0,\n        )\n        for s in test_movies_sequences\n    ]\n)\n</pre> X_train_text = np.array(     [         pad_sequences(             s,             maxlen=maxlen,             pad_first=False,             pad_idx=PAD_IDX,         )         for s in train_movies_sequences     ] ) X_test_text = np.array(     [         pad_sequences(             s,             maxlen=maxlen,             pad_first=False,             pad_idx=0,         )         for s in test_movies_sequences     ] ) <p>We now define the model components and the wide and deep model.</p> In\u00a0[7]: Copied! <pre>tab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    mlp_hidden_dims=[1024, 512, 256],\n    mlp_activation=\"relu\",\n)\n\n# plenty of options here, see the docs\ntransformer = Transformer(\n    vocab_size=max_movie_index + 1,\n    embed_dim=32,\n    n_heads=2,\n    n_blocks=2,\n    seq_length=maxlen,\n)\n\nwide_deep_model = WideDeep(\n    deeptabular=tab_mlp, deeptext=transformer, pred_dim=max_movie_index + 1\n)\n</pre> tab_mlp = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     mlp_hidden_dims=[1024, 512, 256],     mlp_activation=\"relu\", )  # plenty of options here, see the docs transformer = Transformer(     vocab_size=max_movie_index + 1,     embed_dim=32,     n_heads=2,     n_blocks=2,     seq_length=maxlen, )  wide_deep_model = WideDeep(     deeptabular=tab_mlp, deeptext=transformer, pred_dim=max_movie_index + 1 ) In\u00a0[8]: Copied! <pre>wide_deep_model\n</pre> wide_deep_model Out[8]: <pre>WideDeep(\n  (deeptabular): Sequential(\n    (0): TabMlp(\n      (cat_embed): DiffSizeCatEmbeddings(\n        (embed_layers): ModuleDict(\n          (emb_layer_user_id): Embedding(749, 65, padding_idx=0)\n          (emb_layer_movie_id): Embedding(1612, 100, padding_idx=0)\n          (emb_layer_rating): Embedding(6, 4, padding_idx=0)\n        )\n        (embedding_dropout): Dropout(p=0.0, inplace=False)\n      )\n      (encoder): MLP(\n        (mlp): Sequential(\n          (dense_layer_0): Sequential(\n            (0): Linear(in_features=169, out_features=1024, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_1): Sequential(\n            (0): Linear(in_features=1024, out_features=512, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_2): Sequential(\n            (0): Linear(in_features=512, out_features=256, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=256, out_features=1683, bias=True)\n  )\n  (deeptext): Sequential(\n    (0): Transformer(\n      (embedding): Embedding(1683, 32, padding_idx=0)\n      (pos_encoder): PositionalEncoding(\n        (dropout): Dropout(p=0.1, inplace=False)\n      )\n      (encoder): Sequential(\n        (transformer_block0): TransformerEncoder(\n          (attn): MultiHeadedAttention(\n            (dropout): Dropout(p=0.1, inplace=False)\n            (q_proj): Linear(in_features=32, out_features=32, bias=False)\n            (kv_proj): Linear(in_features=32, out_features=64, bias=False)\n            (out_proj): Linear(in_features=32, out_features=32, bias=False)\n          )\n          (ff): FeedForward(\n            (w_1): Linear(in_features=32, out_features=128, bias=True)\n            (w_2): Linear(in_features=128, out_features=32, bias=True)\n            (dropout): Dropout(p=0.1, inplace=False)\n            (activation): GELU(approximate='none')\n          )\n          (attn_addnorm): AddNorm(\n            (dropout): Dropout(p=0.1, inplace=False)\n            (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n          )\n          (ff_addnorm): AddNorm(\n            (dropout): Dropout(p=0.1, inplace=False)\n            (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n          )\n        )\n        (transformer_block1): TransformerEncoder(\n          (attn): MultiHeadedAttention(\n            (dropout): Dropout(p=0.1, inplace=False)\n            (q_proj): Linear(in_features=32, out_features=32, bias=False)\n            (kv_proj): Linear(in_features=32, out_features=64, bias=False)\n            (out_proj): Linear(in_features=32, out_features=32, bias=False)\n          )\n          (ff): FeedForward(\n            (w_1): Linear(in_features=32, out_features=128, bias=True)\n            (w_2): Linear(in_features=128, out_features=32, bias=True)\n            (dropout): Dropout(p=0.1, inplace=False)\n            (activation): GELU(approximate='none')\n          )\n          (attn_addnorm): AddNorm(\n            (dropout): Dropout(p=0.1, inplace=False)\n            (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n          )\n          (ff_addnorm): AddNorm(\n            (dropout): Dropout(p=0.1, inplace=False)\n            (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=23552, out_features=1683, bias=True)\n  )\n)</pre> <p>And as in the previous notebook, let's train (you will need a GPU for this)</p> In\u00a0[\u00a0]: Copied! <pre>trainer = Trainer(\n    model=wide_deep_model,\n    objective=\"multiclass\",\n    custom_loss_function=nn.CrossEntropyLoss(ignore_index=PAD_IDX),\n    optimizers=torch.optim.Adam(wide_deep_model.parameters(), lr=1e-3),\n)\n\ntrainer.fit(\n    X_train={\n        \"X_tab\": X_train_tab,\n        \"X_text\": X_train_text,\n        \"target\": y_train,\n    },\n    X_val={\n        \"X_tab\": X_test_tab,\n        \"X_text\": X_test_text,\n        \"target\": y_test,\n    },\n    n_epochs=10,\n    batch_size=521,\n    shuffle=False,\n)\n</pre> trainer = Trainer(     model=wide_deep_model,     objective=\"multiclass\",     custom_loss_function=nn.CrossEntropyLoss(ignore_index=PAD_IDX),     optimizers=torch.optim.Adam(wide_deep_model.parameters(), lr=1e-3), )  trainer.fit(     X_train={         \"X_tab\": X_train_tab,         \"X_text\": X_train_text,         \"target\": y_train,     },     X_val={         \"X_tab\": X_test_tab,         \"X_text\": X_test_text,         \"target\": y_test,     },     n_epochs=10,     batch_size=521,     shuffle=False, ) <pre>epoch 1:   0%|                                                                                                         | 0/147 [00:34&lt;?, ?it/s]\n</pre> In\u00a0[\u00a0]: Copied! <pre>\n</pre>"},{"location":"examples/19_load_from_folder_functionality.html","title":"19 load from folder functionality","text":"<p>In this notebook I want to illustrate how one can use our <code>[...]FromFolder</code> functionalities along with the <code>[...]ChunkPreProcessors</code> in those cases where the dataset is too bit to fit in memory.</p> <p>These functionalities in the library have been designed for the following scenarop</p> In\u00a0[1]: Copied! <pre>import numpy as np\nimport torch\nimport pandas as pd\nfrom torch.utils.data import DataLoader\n\nfrom pytorch_widedeep.models import TabMlp, Vision, BasicRNN, WideDeep\nfrom pytorch_widedeep.training import TrainerFromFolder\nfrom pytorch_widedeep.callbacks import EarlyStopping, ModelCheckpoint\nfrom pytorch_widedeep.preprocessing import (\n    TabPreprocessor,\n    TextPreprocessor,\n    ImagePreprocessor,\n    ChunkTabPreprocessor,\n    ChunkTextPreprocessor,\n)\nfrom pytorch_widedeep.load_from_folder import (\n    TabFromFolder,\n    TextFromFolder,\n    ImageFromFolder,\n    WideDeepDatasetFromFolder,\n)\n</pre> import numpy as np import torch import pandas as pd from torch.utils.data import DataLoader  from pytorch_widedeep.models import TabMlp, Vision, BasicRNN, WideDeep from pytorch_widedeep.training import TrainerFromFolder from pytorch_widedeep.callbacks import EarlyStopping, ModelCheckpoint from pytorch_widedeep.preprocessing import (     TabPreprocessor,     TextPreprocessor,     ImagePreprocessor,     ChunkTabPreprocessor,     ChunkTextPreprocessor, ) from pytorch_widedeep.load_from_folder import (     TabFromFolder,     TextFromFolder,     ImageFromFolder,     WideDeepDatasetFromFolder, ) In\u00a0[2]: Copied! <pre># in my case, I place the data in a folder I call tmp_data, let's see how it looks\nairbnb_data = pd.read_csv(\"../tmp_data/airbnb/airbnb_sample.csv\")\nairbnb_data.sample(5)\n</pre> # in my case, I place the data in a folder I call tmp_data, let's see how it looks airbnb_data = pd.read_csv(\"../tmp_data/airbnb/airbnb_sample.csv\") airbnb_data.sample(5) Out[2]: id host_id description host_listings_count host_identity_verified neighbourhood_cleansed latitude longitude is_location_exact property_type ... amenity_wide_entrance amenity_wide_entrance_for_guests amenity_wide_entryway amenity_wide_hallways amenity_wifi amenity_window_guards amenity_wine_cooler security_deposit extra_people yield 39 53242.jpg 247650 A lovely big bright bedroom in a 2 bedroom fla... 2.0 t Lambeth 51.47075 -0.12913 t apartment ... 0 0 0 0 1 0 0 250.0 5.0 9.75 214 236716.jpg 1241070 We offer a warm welcome in our quiet double ro... 1.0 t Hackney 51.56593 -0.07482 t other ... 0 0 0 0 1 0 0 200.0 10.0 76.50 400 346523.jpg 1756532 Available for you to rent is a cozy studio in ... 2.0 t Kensington and Chelsea 51.48311 -0.18428 t other ... 0 0 0 0 1 0 0 0.0 50.0 180.90 512 389627.jpg 1949299 This gorgeous studio flat is situated in the v... 1.0 t Westminster 51.51838 -0.14238 f apartment ... 0 0 0 0 1 0 0 250.0 25.0 276.90 504 388767.jpg 1945165 If you want to experience London at it's best ... 2.0 f Camden 51.54293 -0.14073 t apartment ... 0 0 0 0 1 0 0 150.0 10.0 591.10 <p>5 rows \u00d7 223 columns</p> In\u00a0[3]: Copied! <pre># for example\nfrom IPython.display import Image\n\npil_img = Image(filename=\"../tmp_data/airbnb/property_picture/272908.jpg\")\ndisplay(pil_img)\n</pre> # for example from IPython.display import Image  pil_img = Image(filename=\"../tmp_data/airbnb/property_picture/272908.jpg\") display(pil_img) In\u00a0[4]: Copied! <pre># And the description for the property that that picture belongs to is:\nairbnb_data[airbnb_data.id == \"272908.jpg\"].description.tolist()\n</pre> # And the description for the property that that picture belongs to is: airbnb_data[airbnb_data.id == \"272908.jpg\"].description.tolist() Out[4]: <pre>[\"Bright, sunny beautiful room that will give you the perfect base to explore all of London. Come and explore one of London's best neighbourhoods - Herne Hill! As mentioned in (Website hidden by Airbnb)   (Website hidden by Airbnb)  WiFi availability with a fully stocked and clean uplifting home. Lovely sunny, airy and big double bedroom on a leafy south-London street.    Note: This room comes with a reserved Off-Street parking spot! The room is on the first floor and boasts an enormous Super King bed, gorgeous wooden floors, tall ceilings and large windows which let in the sunshine almost all day. (Yoga May or meditation cushion available on request) The flat is bright and airy and big! So lots of space for all.  Location wise you are only 10 minutes walk to either Herne Hill or West Dulwich stations, both of which will take you to Victoria and the city within minutes. You can also hop on a bus right outside the flat that will take you to Brixton tube station within 8 minutes where you \"]</pre> <p>Ok, so we have tabular data where one column is <code>description</code> and another <code>id</code>, points towards the images stored in disk. Now, remember the following, because this will appear a few times in the notebook: our \"reference dataset\" is the tabular data.</p> <p>Therefore, since I want to illustrate a \"semi-realistic\" case, if we need to split the data into training, validation and test datasets, these datasets needs to be separetely stored in disk. In my case I have done this and in the <code>tmp_data/airbnb</code> dir I have the following:</p> <pre>../tmp_data/airbnb\n\u251c\u2500\u2500 airbnb_sample.csv\n\u251c\u2500\u2500 airbnb_sample_eval.csv\n\u251c\u2500\u2500 airbnb_sample_test.csv\n\u251c\u2500\u2500 airbnb_sample_train.csv\n\u2514\u2500\u2500 property_picture\n</pre> <p>Where <code>airbnb_sample.csv</code> is the full sample (1001 rows) and the <code>train</code>, <code>eval</code> and <code>test</code> set is the corresponding splits. In a realistic example, the full sample would be the 'gigantic' dataset and the rest the corresponding splits. One has to do this 'offline', prior to start the coding.</p> <p>Also, one thing that one needs to know is the number of total observations/rows, as well as the splits. In our case the train size is 800, and the eval and test sizes are 100 and 101 respectively.</p> <p>With all that info, let's star</p> In\u00a0[5]: Copied! <pre># path to the tabular data and the splits\ndata_path = \"../tmp_data/airbnb/\"\ntrain_fname = \"airbnb_sample_train.csv\"\neval_fname = \"airbnb_sample_eval.csv\"\ntest_fname = \"airbnb_sample_test.csv\"\n\n# split sizes\ntrain_size = 800\neval_size = 100\ntest_size = 101\n\n# number of chunks for the Chunk Preprocessors\nchunksize = 100\nn_chunks = int(np.ceil(train_size / chunksize))\n\n# path to the image dataset and name of the image col\nimg_path = \"../tmp_data/airbnb/property_picture/\"\nimg_col = \"id\"\n\n# name of the text col\ntext_col = \"description\"\n\n# mane of the target\ntarget_col = \"yield\"\n\n# definition of the categorical and continuous cols for the TabPreprocessor\ncat_embed_cols = [\n    \"host_listings_count\",\n    \"neighbourhood_cleansed\",\n    \"is_location_exact\",\n    \"property_type\",\n    \"room_type\",\n    \"accommodates\",\n    \"bathrooms\",\n    \"bedrooms\",\n    \"beds\",\n    \"guests_included\",\n    \"minimum_nights\",\n    \"instant_bookable\",\n    \"cancellation_policy\",\n    \"has_house_rules\",\n    \"host_gender\",\n    \"accommodates_catg\",\n    \"guests_included_catg\",\n    \"minimum_nights_catg\",\n    \"host_listings_count_catg\",\n    \"bathrooms_catg\",\n    \"bedrooms_catg\",\n    \"beds_catg\",\n    \"security_deposit\",\n    \"extra_people\",\n]\ncont_cols = [\"latitude\", \"longitude\"]\n</pre> # path to the tabular data and the splits data_path = \"../tmp_data/airbnb/\" train_fname = \"airbnb_sample_train.csv\" eval_fname = \"airbnb_sample_eval.csv\" test_fname = \"airbnb_sample_test.csv\"  # split sizes train_size = 800 eval_size = 100 test_size = 101  # number of chunks for the Chunk Preprocessors chunksize = 100 n_chunks = int(np.ceil(train_size / chunksize))  # path to the image dataset and name of the image col img_path = \"../tmp_data/airbnb/property_picture/\" img_col = \"id\"  # name of the text col text_col = \"description\"  # mane of the target target_col = \"yield\"  # definition of the categorical and continuous cols for the TabPreprocessor cat_embed_cols = [     \"host_listings_count\",     \"neighbourhood_cleansed\",     \"is_location_exact\",     \"property_type\",     \"room_type\",     \"accommodates\",     \"bathrooms\",     \"bedrooms\",     \"beds\",     \"guests_included\",     \"minimum_nights\",     \"instant_bookable\",     \"cancellation_policy\",     \"has_house_rules\",     \"host_gender\",     \"accommodates_catg\",     \"guests_included_catg\",     \"minimum_nights_catg\",     \"host_listings_count_catg\",     \"bathrooms_catg\",     \"bedrooms_catg\",     \"beds_catg\",     \"security_deposit\",     \"extra_people\", ] cont_cols = [\"latitude\", \"longitude\"] In\u00a0[6]: Copied! <pre>tab_preprocessor = TabPreprocessor(\n    embed_cols=cat_embed_cols,\n    continuous_cols=cont_cols,\n    default_embed_dim=8,\n    verbose=0,\n)\n\ntext_preprocessor = TextPreprocessor(\n    text_col=text_col,\n    n_cpus=1,\n)\n\nimg_preprocessor = ImagePreprocessor(\n    img_col=img_col,\n    img_path=img_path,\n)\n</pre> tab_preprocessor = TabPreprocessor(     embed_cols=cat_embed_cols,     continuous_cols=cont_cols,     default_embed_dim=8,     verbose=0, )  text_preprocessor = TextPreprocessor(     text_col=text_col,     n_cpus=1, )  img_preprocessor = ImagePreprocessor(     img_col=img_col,     img_path=img_path, ) In\u00a0[7]: Copied! <pre>tab_preprocessor.fit(airbnb_data)\ntext_preprocessor.fit(airbnb_data)\nimg_preprocessor.fit(airbnb_data)\n</pre> tab_preprocessor.fit(airbnb_data) text_preprocessor.fit(airbnb_data) img_preprocessor.fit(airbnb_data) <pre>The vocabulary contains 2192 tokens\n</pre> Out[7]: <pre>ImagePreprocessor(img_col=id, img_path=../tmp_data/airbnb/property_picture/, width=224, height=224, verbose=1)</pre> In\u00a0[8]: Copied! <pre>chunk_tab_preprocessor = ChunkTabPreprocessor(\n    embed_cols=cat_embed_cols,\n    continuous_cols=cont_cols,\n    n_chunks=n_chunks,\n    default_embed_dim=8,\n    verbose=0,\n)\n\nchunk_text_preprocessor = ChunkTextPreprocessor(\n    n_chunks=n_chunks,\n    text_col=text_col,\n    n_cpus=1,\n    verbose=0,\n)\n\nfor i, chunk in enumerate(\n    pd.read_csv(\"/\".join([data_path, train_fname]), chunksize=chunksize)\n):\n    print(f\"chunk in loop: {i + 1}\")\n    chunk_tab_preprocessor.fit(chunk)\n    chunk_text_preprocessor.fit(chunk)\n</pre> chunk_tab_preprocessor = ChunkTabPreprocessor(     embed_cols=cat_embed_cols,     continuous_cols=cont_cols,     n_chunks=n_chunks,     default_embed_dim=8,     verbose=0, )  chunk_text_preprocessor = ChunkTextPreprocessor(     n_chunks=n_chunks,     text_col=text_col,     n_cpus=1,     verbose=0, )  for i, chunk in enumerate(     pd.read_csv(\"/\".join([data_path, train_fname]), chunksize=chunksize) ):     print(f\"chunk in loop: {i + 1}\")     chunk_tab_preprocessor.fit(chunk)     chunk_text_preprocessor.fit(chunk) <pre>chunk in loop: 1\nchunk in loop: 2\nchunk in loop: 3\nchunk in loop: 4\nchunk in loop: 5\nchunk in loop: 6\nchunk in loop: 7\nchunk in loop: 8\n</pre> In\u00a0[9]: Copied! <pre>train_tab_folder = TabFromFolder(\n    fname=train_fname,\n    directory=data_path,\n    target_col=target_col,\n    preprocessor=tab_preprocessor,\n    text_col=text_col,\n    img_col=img_col,\n)\n\n# Note how we can use the `train_tab_folder` as reference so we don't have to\n# define all parameters again\neval_tab_folder = TabFromFolder(fname=eval_fname, reference=train_tab_folder)\n\n# Note that for the test set we can ignore the target as no metric will be\n# computed by the `predict` method\ntest_tab_folder = TabFromFolder(\n    fname=test_fname, reference=train_tab_folder, ignore_target=True\n)\n</pre> train_tab_folder = TabFromFolder(     fname=train_fname,     directory=data_path,     target_col=target_col,     preprocessor=tab_preprocessor,     text_col=text_col,     img_col=img_col, )  # Note how we can use the `train_tab_folder` as reference so we don't have to # define all parameters again eval_tab_folder = TabFromFolder(fname=eval_fname, reference=train_tab_folder)  # Note that for the test set we can ignore the target as no metric will be # computed by the `predict` method test_tab_folder = TabFromFolder(     fname=test_fname, reference=train_tab_folder, ignore_target=True ) In\u00a0[10]: Copied! <pre># for the text and image datasets we do not need to specify eval or test loaders\ntext_folder = TextFromFolder(preprocessor=text_preprocessor)\nimg_folder = ImageFromFolder(preprocessor=img_preprocessor)\n</pre> # for the text and image datasets we do not need to specify eval or test loaders text_folder = TextFromFolder(preprocessor=text_preprocessor) img_folder = ImageFromFolder(preprocessor=img_preprocessor) In\u00a0[11]: Copied! <pre>train_dataset_folder = WideDeepDatasetFromFolder(\n    n_samples=train_size,\n    tab_from_folder=train_tab_folder,\n    text_from_folder=text_folder,\n    img_from_folder=img_folder,\n)\n\n# Note that the eval and test loaders only need their corresponding\n# `TabFromFolder` classes. The rest of the parameters can be defined\n# via a `reference` `TabFromFolder` class\neval_dataset_folder = WideDeepDatasetFromFolder(\n    n_samples=eval_size,\n    tab_from_folder=eval_tab_folder,\n    reference=train_dataset_folder,\n)\n\ntest_dataset_folder = WideDeepDatasetFromFolder(\n    n_samples=test_size,\n    tab_from_folder=test_tab_folder,\n    reference=train_dataset_folder,\n)\n</pre> train_dataset_folder = WideDeepDatasetFromFolder(     n_samples=train_size,     tab_from_folder=train_tab_folder,     text_from_folder=text_folder,     img_from_folder=img_folder, )  # Note that the eval and test loaders only need their corresponding # `TabFromFolder` classes. The rest of the parameters can be defined # via a `reference` `TabFromFolder` class eval_dataset_folder = WideDeepDatasetFromFolder(     n_samples=eval_size,     tab_from_folder=eval_tab_folder,     reference=train_dataset_folder, )  test_dataset_folder = WideDeepDatasetFromFolder(     n_samples=test_size,     tab_from_folder=test_tab_folder,     reference=train_dataset_folder, ) In\u00a0[12]: Copied! <pre>train_loader = DataLoader(train_dataset_folder, batch_size=16, num_workers=1)\neval_loader = DataLoader(eval_dataset_folder, batch_size=16, num_workers=1)\ntest_loader = DataLoader(test_dataset_folder, batch_size=16, num_workers=1)\n</pre> train_loader = DataLoader(train_dataset_folder, batch_size=16, num_workers=1) eval_loader = DataLoader(eval_dataset_folder, batch_size=16, num_workers=1) test_loader = DataLoader(test_dataset_folder, batch_size=16, num_workers=1) <p>And from here on is business as usual:</p> In\u00a0[13]: Copied! <pre># for example\nbasic_rnn = BasicRNN(\n    vocab_size=len(text_preprocessor.vocab.itos),\n    embed_dim=32,\n    hidden_dim=64,\n    n_layers=2,\n)\n\ndeepimage = Vision()\n\ndeepdense = TabMlp(\n    mlp_hidden_dims=[32, 16],\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=cont_cols,\n)\n\nmodel = WideDeep(\n    deeptabular=deepdense,\n    deeptext=basic_rnn,\n    deepimage=deepimage,\n)\n\nmodel\n</pre> # for example basic_rnn = BasicRNN(     vocab_size=len(text_preprocessor.vocab.itos),     embed_dim=32,     hidden_dim=64,     n_layers=2, )  deepimage = Vision()  deepdense = TabMlp(     mlp_hidden_dims=[32, 16],     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     continuous_cols=cont_cols, )  model = WideDeep(     deeptabular=deepdense,     deeptext=basic_rnn,     deepimage=deepimage, )  model Out[13]: <pre>WideDeep(\n  (deeptabular): Sequential(\n    (0): TabMlp(\n      (cat_embed): DiffSizeCatEmbeddings(\n        (embed_layers): ModuleDict(\n          (emb_layer_host_listings_count): Embedding(28, 10, padding_idx=0)\n          (emb_layer_neighbourhood_cleansed): Embedding(33, 11, padding_idx=0)\n          (emb_layer_is_location_exact): Embedding(3, 2, padding_idx=0)\n          (emb_layer_property_type): Embedding(4, 3, padding_idx=0)\n          (emb_layer_room_type): Embedding(4, 3, padding_idx=0)\n          (emb_layer_accommodates): Embedding(14, 7, padding_idx=0)\n          (emb_layer_bathrooms): Embedding(11, 6, padding_idx=0)\n          (emb_layer_bedrooms): Embedding(7, 4, padding_idx=0)\n          (emb_layer_beds): Embedding(11, 6, padding_idx=0)\n          (emb_layer_guests_included): Embedding(11, 6, padding_idx=0)\n          (emb_layer_minimum_nights): Embedding(25, 9, padding_idx=0)\n          (emb_layer_instant_bookable): Embedding(3, 2, padding_idx=0)\n          (emb_layer_cancellation_policy): Embedding(6, 4, padding_idx=0)\n          (emb_layer_has_house_rules): Embedding(3, 2, padding_idx=0)\n          (emb_layer_host_gender): Embedding(4, 3, padding_idx=0)\n          (emb_layer_accommodates_catg): Embedding(4, 3, padding_idx=0)\n          (emb_layer_guests_included_catg): Embedding(4, 3, padding_idx=0)\n          (emb_layer_minimum_nights_catg): Embedding(4, 3, padding_idx=0)\n          (emb_layer_host_listings_count_catg): Embedding(5, 3, padding_idx=0)\n          (emb_layer_bathrooms_catg): Embedding(4, 3, padding_idx=0)\n          (emb_layer_bedrooms_catg): Embedding(5, 3, padding_idx=0)\n          (emb_layer_beds_catg): Embedding(5, 3, padding_idx=0)\n          (emb_layer_security_deposit): Embedding(53, 15, padding_idx=0)\n          (emb_layer_extra_people): Embedding(39, 12, padding_idx=0)\n        )\n        (embedding_dropout): Dropout(p=0.0, inplace=False)\n      )\n      (cont_norm): Identity()\n      (encoder): MLP(\n        (mlp): Sequential(\n          (dense_layer_0): Sequential(\n            (0): Linear(in_features=128, out_features=32, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_1): Sequential(\n            (0): Linear(in_features=32, out_features=16, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=16, out_features=1, bias=True)\n  )\n  (deeptext): Sequential(\n    (0): BasicRNN(\n      (word_embed): Embedding(2192, 32, padding_idx=1)\n      (rnn): LSTM(32, 64, num_layers=2, batch_first=True, dropout=0.1)\n      (rnn_mlp): Identity()\n    )\n    (1): Linear(in_features=64, out_features=1, bias=True)\n  )\n  (deepimage): Sequential(\n    (0): Vision(\n      (features): Sequential(\n        (conv_layer_0): Sequential(\n          (0): Conv2d(3, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False)\n          (1): BatchNorm2d(64, eps=1e-05, momentum=0.01, affine=True, track_running_stats=True)\n          (2): LeakyReLU(negative_slope=0.1, inplace=True)\n          (maxpool): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n        )\n        (conv_layer_1): Sequential(\n          (0): Conv2d(64, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n          (1): BatchNorm2d(128, eps=1e-05, momentum=0.01, affine=True, track_running_stats=True)\n          (2): LeakyReLU(negative_slope=0.1, inplace=True)\n        )\n        (conv_layer_2): Sequential(\n          (0): Conv2d(128, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n          (1): BatchNorm2d(256, eps=1e-05, momentum=0.01, affine=True, track_running_stats=True)\n          (2): LeakyReLU(negative_slope=0.1, inplace=True)\n        )\n        (conv_layer_3): Sequential(\n          (0): Conv2d(256, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n          (1): BatchNorm2d(512, eps=1e-05, momentum=0.01, affine=True, track_running_stats=True)\n          (2): LeakyReLU(negative_slope=0.1, inplace=True)\n          (adaptiveavgpool): AdaptiveAvgPool2d(output_size=(1, 1))\n        )\n      )\n    )\n    (1): Linear(in_features=512, out_features=1, bias=True)\n  )\n)</pre> In\u00a0[14]: Copied! <pre>trainer = TrainerFromFolder(\n    model,\n    objective=\"regression\",\n)\n\ntrainer.fit(\n    train_loader=train_loader,\n    eval_loader=eval_loader,\n)\n</pre> trainer = TrainerFromFolder(     model,     objective=\"regression\", )  trainer.fit(     train_loader=train_loader,     eval_loader=eval_loader, ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 50/50 [03:41&lt;00:00,  4.42s/it, loss=1.64e+4]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 7/7 [00:23&lt;00:00,  3.30s/it, loss=6.27e+3]\n</pre> In\u00a0[15]: Copied! <pre>preds = trainer.predict(test_loader=test_loader)\n</pre> preds = trainer.predict(test_loader=test_loader) <pre>predict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 7/7 [00:22&lt;00:00,  3.26s/it]\n</pre> <p>Note that in the case of predict you could also choose to do this</p> In\u00a0[16]: Copied! <pre>df_test = pd.read_csv(\"/\".join([data_path, test_fname]))\n</pre> df_test = pd.read_csv(\"/\".join([data_path, test_fname])) In\u00a0[17]: Copied! <pre># if the images for the test set fit in memory\nX_tab_test = chunk_tab_preprocessor.transform(df_test)\nX_text_test = chunk_text_preprocessor.transform(df_test)\nX_img_test = img_preprocessor.transform(df_test)\n</pre> # if the images for the test set fit in memory X_tab_test = chunk_tab_preprocessor.transform(df_test) X_text_test = chunk_text_preprocessor.transform(df_test) X_img_test = img_preprocessor.transform(df_test) <pre>Reading Images from ../tmp_data/airbnb/property_picture/\nResizing\n</pre> <pre>100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 101/101 [00:00&lt;00:00, 708.23it/s]</pre> <pre>Computing normalisation metrics\n</pre> <pre>\n</pre> In\u00a0[18]: Copied! <pre>preds = trainer.predict(\n    X_tab=X_tab_test, X_text=X_text_test, X_img=X_img_test, batch_size=32\n)\n</pre> preds = trainer.predict(     X_tab=X_tab_test, X_text=X_text_test, X_img=X_img_test, batch_size=32 ) <pre>predict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 4/4 [00:03&lt;00:00,  1.14it/s]\n</pre>"},{"location":"examples/19_load_from_folder_functionality.html#scenario","title":"Scenario\u00b6","text":"<p>We have a tabular dataset combined with images and text and either some, or all these datasets do not fit in memory. Note that the tabular dataset MUST ALWAYS be present as it is considered the rerefence. This is, if we have an image dataset, the tabular dataset must contain a column that points to the image file names as stored in disk. Similarly, if we have a text dataset, then the tabular dataset must contain a column with the texts themselves or a the file names of the text files as stored in disk.</p> <p>If you only have text and/or images and not a tabular component, I would suggest using other libraries (such as hugginface probably).</p> <p>Within this scenario, they are two possible scenarios that we will cover here:</p> <ol> <li><p>The tabular data itsel fits in memory and is only the images that do not: in this case you could use the 'standard' <code>Preprocessors</code> in the library and off you go, move directly to the <code>[...]FromFolder</code> functionalities</p> </li> <li><p>The tabular data is also very large and does not fit in memory, so we have to process it in chuncks. For this second case I have created the so called <code>Chunk[...]Preprocessor</code> (Wide, Tab and Text).</p> </li> </ol> <p>Note that at the moment ONLY csv format is allowed for the tabular file. More formats will be supported in the future.</p> <p>Let's see a complete example to illustrate how each of these cases would be addressed with the new functionalities in the library. For this example we will use a sample of the airbnb dataset</p> <p>The airbnb dataset, which you could get from here, is too big to be included in our datasets module (when including images). Therefore, what I did was, go there, download it, and use the download_images.py script to get the images and the airbnb_data_processing.py to process the data. I did this ages ago and I believe the format of the dataset might be different now. Nonetheless, I will show samples of the dataset as we go through so you can extrapolate the content of this notebook to your particular problem.</p> <p>In the future we will find better datasets\ud83d\ude42. Finally, note that here we are only using a small sample to illustrate the use, so PLEASE ignore the results, just focus on usage.</p>"},{"location":"examples/19_load_from_folder_functionality.html#setting-variables-and-constants","title":"Setting variables and constants\u00b6","text":""},{"location":"examples/19_load_from_folder_functionality.html#step-1-the-preprocessors","title":"Step 1: the preprocessors\u00b6","text":""},{"location":"examples/19_load_from_folder_functionality.html#scenario-1-only-the-images-do-not-fit-in-disk","title":"Scenario 1: only the images do not fit in disk\u00b6","text":"<p>In this case we can prepare the data in the 'standard' way</p>"},{"location":"examples/19_load_from_folder_functionality.html#scenario-2-the-tabular-data-is-also-huge","title":"Scenario 2: the tabular data is also huge\u00b6","text":"<p>Then we need to prepare it in chunks. Note that, unfortunately, the tabular and text preprocessors need to see the whole dataset once. This is because to process tabular or text data we need to encode values. For those encodings to be consistent they need to have seen the whole dataset. Alternatively, one could code a solution with some streaming encoder for both datasets. However, such implementation is not trivial for this library (and in general). I also don't think that having to see the whole data once is such a big limitation. Let's see how is done.</p> <p>Note that I have not mentioned the image dataset. This is because the processing of the image dataset does not require any form of encoding and in consequence can be done 'on the fly'. Therefore, no <code>ChunkImgPreprocessor</code> processor is needed.</p>"},{"location":"examples/19_load_from_folder_functionality.html#step-2-the-fromfolder-classes","title":"Step 2: the <code>[...]FromFolder</code> classes\u00b6","text":"<p>Once we have the preprocessors, we need to instantiate the classes that will enable us to load the data from their respective folders. From now on I am going to proceed with the <code>chunk_tab_preprocessor</code>, <code>chunk_text_preprocessor</code> and <code>img_preprocessor</code>, but the code would be identical if instead of the first two preprocessors we decided to use the <code>tab_preprocessor</code> and <code>text_preprocessor</code>.</p> <p>Once more, our reference datasets are the tabular datasets, which we have splitted in train, eval and test prior to start the coding. Therefore, we will eventually need a loader for each split</p>"},{"location":"examples/19_load_from_folder_functionality.html#step-3-pytorch-datasets-and-dataloaders","title":"Step 3: pytorch datasets and dataloaders\u00b6","text":"<p>From here in advance, is all very 'standard' if you are familiar with pytorch. One needs to define a class that inherits from the <code>Dataset</code> class in pytorch. Then this will be passed to a <code>DataLoader</code> class and we are ready to train. Our <code>Dataset</code> child class is <code>WideDeepDatasetFromFolder</code>. This class will use the tabular dataset and the corresponding text and image columns to load the adequate data in the batches</p> <p>Let's do it</p>"},{"location":"examples/19_load_from_folder_functionality.html#step-4-define-the-model","title":"Step 4: define the model\u00b6","text":""},{"location":"examples/19_load_from_folder_functionality.html#step-5-fit-and-predict","title":"Step 5: fit and predict\u00b6","text":""},{"location":"examples/20_Using_huggingface_within_widedeep.html","title":"20 Using huggingface within widedeep","text":"<p>In this notebook we will show how to use Hugginface's tokenizers and models as they are integrated within the library. In notebook number 17 you can find examples on how to code your own, custom, Hugginface (hereafter HF) model and use it in combination of any other model in the library</p> In\u00a0[1]: Copied! <pre>import numpy as np\nimport pandas as pd\nfrom sklearn.metrics import f1_score, accuracy_score\nfrom sklearn.model_selection import train_test_split\n\nfrom pytorch_widedeep import Trainer\nfrom pytorch_widedeep.models import HFModel, WideDeep\nfrom pytorch_widedeep.metrics import F1Score, Accuracy\nfrom pytorch_widedeep.datasets import load_womens_ecommerce\nfrom pytorch_widedeep.preprocessing import HFPreprocessor\n</pre> import numpy as np import pandas as pd from sklearn.metrics import f1_score, accuracy_score from sklearn.model_selection import train_test_split  from pytorch_widedeep import Trainer from pytorch_widedeep.models import HFModel, WideDeep from pytorch_widedeep.metrics import F1Score, Accuracy from pytorch_widedeep.datasets import load_womens_ecommerce from pytorch_widedeep.preprocessing import HFPreprocessor <pre>/Users/javierrodriguezzaurin/.pyenv/versions/3.10.13/envs/widedeep310/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n  from .autonotebook import tqdm as notebook_tqdm\n</pre> In\u00a0[2]: Copied! <pre>df: pd.DataFrame = load_womens_ecommerce(as_frame=True)  # type: ignore\n</pre> df: pd.DataFrame = load_womens_ecommerce(as_frame=True)  # type: ignore In\u00a0[3]: Copied! <pre>df.shape\n</pre> df.shape Out[3]: <pre>(23486, 10)</pre> In\u00a0[4]: Copied! <pre>df.sample(3)\n</pre> df.sample(3) Out[4]: Clothing ID Age Title Review Text Rating Recommended IND Positive Feedback Count Division Name Department Name Class Name 7004 862 43 Cute and feminine Loved this sweater wrap and bought it in both ... 5 1 2 General Tops Knits 12508 975 66 Love it The linen fabric is elegantly thin feels and l... 5 1 3 General Jackets Jackets 10288 950 41 Perfect for fall This sweater is just as pictured. the fit is t... 5 1 0 General Tops Sweaters In\u00a0[5]: Copied! <pre># Let's do some mild preprocessing\ndf.columns = [c.replace(\" \", \"_\").lower() for c in df.columns]\n\n# classes from [0,num_class)\ndf[\"rating\"] = (df[\"rating\"] - 1).astype(\"int64\")\n\n# group reviews with 1 and 2 scores into one class\ndf.loc[df.rating == 0, \"rating\"] = 1\n\n# and back again to [0,num_class)\ndf[\"rating\"] = (df[\"rating\"] - 1).astype(\"int64\")\n</pre> # Let's do some mild preprocessing df.columns = [c.replace(\" \", \"_\").lower() for c in df.columns]  # classes from [0,num_class) df[\"rating\"] = (df[\"rating\"] - 1).astype(\"int64\")  # group reviews with 1 and 2 scores into one class df.loc[df.rating == 0, \"rating\"] = 1  # and back again to [0,num_class) df[\"rating\"] = (df[\"rating\"] - 1).astype(\"int64\") In\u00a0[6]: Copied! <pre># drop short reviews\ndf = df[~df.review_text.isna()]\ndf[\"review_length\"] = df.review_text.apply(lambda x: len(x.split(\" \")))\ndf = df[df.review_length &gt;= 5]\ndf = df.drop(\"review_length\", axis=1).reset_index(drop=True)\n</pre> # drop short reviews df = df[~df.review_text.isna()] df[\"review_length\"] = df.review_text.apply(lambda x: len(x.split(\" \"))) df = df[df.review_length &gt;= 5] df = df.drop(\"review_length\", axis=1).reset_index(drop=True) In\u00a0[7]: Copied! <pre>df.shape\n</pre> df.shape Out[7]: <pre>(22608, 10)</pre> In\u00a0[8]: Copied! <pre># if you run this on a CPU, you might want to subsample the dataset. With that in mind I am simply going to stratify-sample to the minimum category occurrence and then sample at random\n# If you run this on a GPU you can comment out the following two cells\ndf.rating.value_counts()\n</pre> # if you run this on a CPU, you might want to subsample the dataset. With that in mind I am simply going to stratify-sample to the minimum category occurrence and then sample at random # If you run this on a GPU you can comment out the following two cells df.rating.value_counts() Out[8]: <pre>rating\n3    12515\n2     4904\n1     2820\n0     2369\nName: count, dtype: int64</pre> In\u00a0[9]: Copied! <pre>df = (\n    df.groupby(\"rating\", group_keys=False)\n    .apply(lambda x: x.sample(min(len(x), 2369)))\n    .sample(1000)\n)\n</pre> df = (     df.groupby(\"rating\", group_keys=False)     .apply(lambda x: x.sample(min(len(x), 2369)))     .sample(1000) ) <pre>/var/folders/_2/lrjn1qn54c758tdtktr1bvkc0000gn/T/ipykernel_5886/895673206.py:3: DeprecationWarning: DataFrameGroupBy.apply operated on the grouping columns. This behavior is deprecated, and in a future version of pandas the grouping columns will be excluded from the operation. Either pass `include_groups=False` to exclude the groupings or explicitly select the grouping columns after groupby to silence this warning.\n  .apply(lambda x: x.sample(min(len(x), 2369)))\n</pre> In\u00a0[10]: Copied! <pre>train, test = train_test_split(df, train_size=0.8, random_state=1, stratify=df.rating)\n\n# possible model names currently supported in the library\nmodel_names = [\n    \"distilbert-base-uncased\",\n    \"bert-base-uncased\",\n    \"FacebookAI/roberta-base\",\n    \"albert-base-v2\",\n    \"google/electra-base-discriminator\",\n]\n\n# Let's choose one. The syntax is the same for all the models\nmodel_name = \"distilbert-base-uncased\"\n</pre> train, test = train_test_split(df, train_size=0.8, random_state=1, stratify=df.rating)  # possible model names currently supported in the library model_names = [     \"distilbert-base-uncased\",     \"bert-base-uncased\",     \"FacebookAI/roberta-base\",     \"albert-base-v2\",     \"google/electra-base-discriminator\", ]  # Let's choose one. The syntax is the same for all the models model_name = \"distilbert-base-uncased\" <p>Now we can use the <code>HFPreprocessor</code> class. As most things in this library, the integration with HF has been coded aiming for a flexible use. With this in mind, there are two ways one can use a <code>HFPreprocessor</code> class.</p> <ol> <li>Passing a <code>text_col</code> and <code>encode_params</code> as the class is instantiated and then using the <code>fit</code> and <code>transform</code> as with any other preprocessor in the library</li> <li>Without passing  <code>text_col</code> and <code>encode_params</code> as the class is instantiated and using the <code>encode</code> method of the <code>HFPreprocessor</code> which is simply a wrapper around the encode method of HF's tokenizers</li> </ol> <p>Let's have a look</p> In\u00a0[11]: Copied! <pre>tokenizer1 = HFPreprocessor(\n    model_name=model_name,\n    text_col=\"review_text\",\n    num_workers=1,\n    encode_params={\n        \"max_length\": 90,\n        \"padding\": \"max_length\",\n        \"truncation\": True,\n        \"add_special_tokens\": True,\n    },\n)\n\nX_text_tr1 = tokenizer1.fit_transform(train)\nX_text_te1 = tokenizer1.transform(test)\n</pre> tokenizer1 = HFPreprocessor(     model_name=model_name,     text_col=\"review_text\",     num_workers=1,     encode_params={         \"max_length\": 90,         \"padding\": \"max_length\",         \"truncation\": True,         \"add_special_tokens\": True,     }, )  X_text_tr1 = tokenizer1.fit_transform(train) X_text_te1 = tokenizer1.transform(test) In\u00a0[12]: Copied! <pre>tokenizer2 = HFPreprocessor(\n    model_name=model_name,\n    num_workers=1,\n)\n\nX_text_tr2 = tokenizer2.encode(\n    train.review_text.tolist(),\n    max_length=90,\n    padding=\"max_length\",\n    truncation=True,\n    add_special_tokens=True,\n)\nX_text_te2 = tokenizer2.encode(\n    test.review_text.tolist(),\n    max_length=90,\n    padding=\"max_length\",\n    truncation=True,\n    add_special_tokens=True,\n)\n</pre> tokenizer2 = HFPreprocessor(     model_name=model_name,     num_workers=1, )  X_text_tr2 = tokenizer2.encode(     train.review_text.tolist(),     max_length=90,     padding=\"max_length\",     truncation=True,     add_special_tokens=True, ) X_text_te2 = tokenizer2.encode(     test.review_text.tolist(),     max_length=90,     padding=\"max_length\",     truncation=True,     add_special_tokens=True, ) In\u00a0[13]: Copied! <pre>all(X_text_tr1[0] == X_text_tr2[0])\n</pre> all(X_text_tr1[0] == X_text_tr2[0]) Out[13]: <pre>True</pre> In\u00a0[14]: Copied! <pre># Now we define a model which is as easy as:\n# Note that this will instantiation will lead to NO parameter trainable in the HF model.\n# If you want to fine-tune the HF model, you can set the trainable parameters via the 'trainable_parameters' argument.\n# Alternatively, you can use a head (MLP) via the 'head'-related arguments (see the docs for more details)\nhf_model = HFModel(model_name=model_name)\n</pre> # Now we define a model which is as easy as: # Note that this will instantiation will lead to NO parameter trainable in the HF model. # If you want to fine-tune the HF model, you can set the trainable parameters via the 'trainable_parameters' argument. # Alternatively, you can use a head (MLP) via the 'head'-related arguments (see the docs for more details) hf_model = HFModel(model_name=model_name) In\u00a0[15]: Copied! <pre># And from here on is the same as any other WideDeep model\nmodel = WideDeep(\n    deeptext=hf_model,\n    pred_dim=4,\n)\n\ntrainer = Trainer(\n    model,\n    objective=\"multiclass\",\n    metrics=[Accuracy(), F1Score(average=True)],\n)\n\ntrainer.fit(\n    X_text=X_text_tr2,\n    target=train.rating.values,\n    n_epochs=1,\n    batch_size=64,\n)\n# If you run this on a CPU and you sampled the data, the metrics will not be better than a random guess. Remember, this is just a demo\n</pre> # And from here on is the same as any other WideDeep model model = WideDeep(     deeptext=hf_model,     pred_dim=4, )  trainer = Trainer(     model,     objective=\"multiclass\",     metrics=[Accuracy(), F1Score(average=True)], )  trainer.fit(     X_text=X_text_tr2,     target=train.rating.values,     n_epochs=1,     batch_size=64, ) # If you run this on a CPU and you sampled the data, the metrics will not be better than a random guess. Remember, this is just a demo <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 13/13 [02:06&lt;00:00,  9.75s/it, loss=3.2, metrics={'acc': 0.235, 'f1': 0.2336}]  \n</pre> In\u00a0[17]: Copied! <pre>preds_text = trainer.predict_proba(X_text=X_text_te2)\npred_text_class = np.argmax(preds_text, 1)\n\nacc_text = accuracy_score(test.rating, pred_text_class)\nf1_text = f1_score(test.rating, pred_text_class, average=\"weighted\")\nprint(f\"Accuracy: {acc_text:.4f}\")\nprint(f\"F1: {f1_text:.4f}\")\n</pre> preds_text = trainer.predict_proba(X_text=X_text_te2) pred_text_class = np.argmax(preds_text, 1)  acc_text = accuracy_score(test.rating, pred_text_class) f1_text = f1_score(test.rating, pred_text_class, average=\"weighted\") print(f\"Accuracy: {acc_text:.4f}\") print(f\"F1: {f1_text:.4f}\") <pre>predict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 4/4 [00:05&lt;00:00,  1.43s/it]</pre> <pre>Accuracy: 0.2500\nF1: 0.1000\n</pre> <pre>\n</pre>"},{"location":"pytorch-widedeep/bayesian_models.html","title":"The <code>bayesian models</code> module","text":"<p>This module contains the two Bayesian Models available in this library, namely the bayesian version of the <code>Wide</code> and <code>TabMlp</code> models, referred as <code>BayesianWide</code> and <code>BayesianTabMlp</code>. These models are very useful in scenarios where getting a measure of uncertainty is important.</p> <p>The models in this module are based on the publication: Weight Uncertainty in Neural Networks.</p>"},{"location":"pytorch-widedeep/bayesian_models.html#pytorch_widedeep.bayesian_models.tabular.bayesian_linear.bayesian_wide.BayesianWide","title":"BayesianWide","text":"<pre><code>BayesianWide(\n    input_dim,\n    pred_dim=1,\n    prior_sigma_1=1.0,\n    prior_sigma_2=0.002,\n    prior_pi=0.8,\n    posterior_mu_init=0.0,\n    posterior_rho_init=-7.0,\n)\n</code></pre> <p>               Bases: <code>BaseBayesianModel</code></p> <p>Defines a <code>Wide</code> model. This is a linear model where the non-linearlities are captured via crossed-columns</p> <p>Parameters:</p> <ul> <li> <code>input_dim</code>               (<code>int</code>)           \u2013            <p>size of the Embedding layer. <code>input_dim</code> is the summation of all the individual values for all the features that go through the wide component. For example, if the wide component receives 2 features with 5 individual values each, <code>input_dim = 10</code></p> </li> <li> <code>pred_dim</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>size of the ouput tensor containing the predictions</p> </li> <li> <code>prior_sigma_1</code>               (<code>float</code>, default:                   <code>1.0</code> )           \u2013            <p>The prior weight distribution is a scaled mixture of two Gaussian densities:</p> \\[    \\begin{aligned}    P(\\mathbf{w}) = \\prod_{i=j} \\pi N (\\mathbf{w}_j | 0, \\sigma_{1}^{2}) + (1 - \\pi) N (\\mathbf{w}_j | 0, \\sigma_{2}^{2})    \\end{aligned} \\] <p><code>prior_sigma_1</code> is the prior of the sigma parameter for the first of the two Gaussians that will be mixed to produce the prior weight distribution.</p> </li> <li> <code>prior_sigma_2</code>               (<code>float</code>, default:                   <code>0.002</code> )           \u2013            <p>Prior of the sigma parameter for the second of the two Gaussian distributions that will be mixed to produce the prior weight distribution</p> </li> <li> <code>prior_pi</code>               (<code>float</code>, default:                   <code>0.8</code> )           \u2013            <p>Scaling factor that will be used to mix the Gaussians to produce the prior weight distribution</p> </li> <li> <code>posterior_mu_init</code>               (<code>float</code>, default:                   <code>0.0</code> )           \u2013            <p>The posterior sample of the weights is defined as:</p> \\[    \\begin{aligned}    \\mathbf{w} &amp;= \\mu + log(1 + exp(\\rho))    \\end{aligned} \\] <p>where:</p> \\[    \\begin{aligned}    \\mathcal{N}(x\\vert \\mu, \\sigma) &amp;= \\frac{1}{\\sqrt{2\\pi}\\sigma}e^{-\\frac{(x-\\mu)^2}{2\\sigma^2}}\\\\    \\log{\\mathcal{N}(x\\vert \\mu, \\sigma)} &amp;= -\\log{\\sqrt{2\\pi}} -\\log{\\sigma} -\\frac{(x-\\mu)^2}{2\\sigma^2}\\\\    \\end{aligned} \\] <p>\\(\\mu\\) is initialised using a normal distributtion with mean <code>posterior_mu_init</code> and std equal to 0.1.</p> </li> <li> <code>posterior_rho_init</code>               (<code>float</code>, default:                   <code>-7.0</code> )           \u2013            <p>As in the case of \\(\\mu\\), \\(\\rho\\) is initialised using a normal distributtion with mean <code>posterior_rho_init</code> and std equal to 0.1.</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>bayesian_wide_linear</code>               (<code>Module</code>)           \u2013            <p>the linear layer that comprises the wide branch of the model</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.bayesian_models import BayesianWide\n&gt;&gt;&gt; X = torch.empty(4, 4).random_(6)\n&gt;&gt;&gt; wide = BayesianWide(input_dim=int(X.max().item()), pred_dim=1)\n&gt;&gt;&gt; out = wide(X)\n</code></pre> Source code in <code>pytorch_widedeep/bayesian_models/tabular/bayesian_linear/bayesian_wide.py</code> <pre><code>def __init__(\n    self,\n    input_dim: int,\n    pred_dim: int = 1,\n    prior_sigma_1: float = 1.0,\n    prior_sigma_2: float = 0.002,\n    prior_pi: float = 0.8,\n    posterior_mu_init: float = 0.0,\n    posterior_rho_init: float = -7.0,\n):\n    super(BayesianWide, self).__init__()\n    #  Embeddings: val + 1 because 0 is reserved for padding/unseen cateogories.\n    self.bayesian_wide_linear = bnn.BayesianEmbedding(\n        n_embed=input_dim + 1,\n        embed_dim=pred_dim,\n        padding_idx=0,\n        prior_sigma_1=prior_sigma_1,\n        prior_sigma_2=prior_sigma_2,\n        prior_pi=prior_pi,\n        posterior_mu_init=posterior_mu_init,\n        posterior_rho_init=posterior_rho_init,\n    )\n    self.bias = nn.Parameter(torch.zeros(pred_dim))\n</code></pre>"},{"location":"pytorch-widedeep/bayesian_models.html#pytorch_widedeep.bayesian_models.tabular.bayesian_mlp.bayesian_tab_mlp.BayesianTabMlp","title":"BayesianTabMlp","text":"<pre><code>BayesianTabMlp(\n    column_idx,\n    *,\n    cat_embed_input=None,\n    cat_embed_activation=None,\n    continuous_cols=None,\n    embed_continuous=None,\n    cont_embed_dim=None,\n    cont_embed_dropout=None,\n    cont_embed_activation=None,\n    use_cont_bias=None,\n    cont_norm_layer=None,\n    mlp_hidden_dims=[200, 100],\n    mlp_activation=\"leaky_relu\",\n    prior_sigma_1=1,\n    prior_sigma_2=0.002,\n    prior_pi=0.8,\n    posterior_mu_init=0.0,\n    posterior_rho_init=-7.0,\n    pred_dim=1\n)\n</code></pre> <p>               Bases: <code>BaseBayesianModel</code></p> <p>Defines a <code>BayesianTabMlp</code> model.</p> <p>This class combines embedding representations of the categorical features with numerical (aka continuous) features, embedded or not. These are then passed through a series of probabilistic dense layers (i.e. a MLP).</p> <p>Parameters:</p> <ul> <li> <code>column_idx</code>               (<code>Dict[str, int]</code>)           \u2013            <p>Dict containing the index of the columns that will be passed through the <code>TabMlp</code> model. Required to slice the tensors. e.g. {'education': 0, 'relationship': 1, 'workclass': 2, ...}</p> </li> <li> <code>cat_embed_input</code>               (<code>Optional[List[Tuple[str, int, int]]]</code>, default:                   <code>None</code> )           \u2013            <p>List of Tuples with the column name, number of unique values and embedding dimension. e.g. [(education, 11, 32), ...]</p> </li> <li> <code>cat_embed_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the categorical embeddings, if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> </li> <li> <code>continuous_cols</code>               (<code>Optional[List[str]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the name of the numeric (aka continuous) columns</p> </li> <li> <code>cont_norm_layer</code>               (<code>Optional[Literal[batchnorm, layernorm]]</code>, default:                   <code>None</code> )           \u2013            <p>Type of normalization layer applied to the continuous features. Options are: 'layernorm', 'batchnorm' or None.</p> </li> <li> <code>embed_continuous</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating if the continuous columns will be embedded (i.e. passed each through a linear layer with or without activation)</p> </li> <li> <code>cont_embed_dim</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>Size of the continuous embeddings</p> </li> <li> <code>cont_embed_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Dropout for the continuous embeddings</p> </li> <li> <code>use_cont_bias</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating if bias will be used for the continuous embeddings</p> </li> <li> <code>cont_embed_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the continuous embeddings if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> </li> <li> <code>mlp_hidden_dims</code>               (<code>List[int]</code>, default:                   <code>[200, 100]</code> )           \u2013            <p>List with the number of neurons per dense layer in the mlp.</p> </li> <li> <code>mlp_activation</code>               (<code>str</code>, default:                   <code>'leaky_relu'</code> )           \u2013            <p>Activation function for the dense layers of the MLP. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> </li> <li> <code>prior_sigma_1</code>               (<code>float</code>, default:                   <code>1</code> )           \u2013            <p>The prior weight distribution is a scaled mixture of two Gaussian densities:</p> \\[    \\begin{aligned}    P(\\mathbf{w}) = \\prod_{i=j} \\pi N (\\mathbf{w}_j | 0, \\sigma_{1}^{2}) + (1 - \\pi) N (\\mathbf{w}_j | 0, \\sigma_{2}^{2})    \\end{aligned} \\] <p><code>prior_sigma_1</code> is the prior of the sigma parameter for the first of the two Gaussians that will be mixed to produce the prior weight distribution.</p> </li> <li> <code>prior_sigma_2</code>               (<code>float</code>, default:                   <code>0.002</code> )           \u2013            <p>Prior of the sigma parameter for the second of the two Gaussian distributions that will be mixed to produce the prior weight distribution for each Bayesian linear and embedding layer</p> </li> <li> <code>prior_pi</code>               (<code>float</code>, default:                   <code>0.8</code> )           \u2013            <p>Scaling factor that will be used to mix the Gaussians to produce the prior weight distribution ffor each Bayesian linear and embedding layer</p> </li> <li> <code>posterior_mu_init</code>               (<code>float</code>, default:                   <code>0.0</code> )           \u2013            <p>The posterior sample of the weights is defined as:</p> <p>$$    \\begin{aligned}    \\mathbf{w} &amp;= \\mu + log(1 + exp(\\rho))    \\end{aligned} $$ where:</p> \\[    \\begin{aligned}    \\mathcal{N}(x\\vert \\mu, \\sigma) &amp;= \\frac{1}{\\sqrt{2\\pi}\\sigma}e^{-\\frac{(x-\\mu)^2}{2\\sigma^2}}\\\\    \\log{\\mathcal{N}(x\\vert \\mu, \\sigma)} &amp;= -\\log{\\sqrt{2\\pi}} -\\log{\\sigma} -\\frac{(x-\\mu)^2}{2\\sigma^2}\\\\    \\end{aligned} \\] <p>\\(\\mu\\) is initialised using a normal distributtion with mean <code>posterior_mu_init</code> and std equal to 0.1.</p> </li> <li> <code>posterior_rho_init</code>               (<code>float</code>, default:                   <code>-7.0</code> )           \u2013            <p>As in the case of \\(\\mu\\), \\(\\rho\\) is initialised using a normal distributtion with mean <code>posterior_rho_init</code> and std equal to 0.1.</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>bayesian_cat_and_cont_embed</code>               (<code>Module</code>)           \u2013            <p>This is the module that processes the categorical and continuous columns</p> </li> <li> <code>bayesian_tab_mlp</code>               (<code>Sequential</code>)           \u2013            <p>mlp model that will receive the concatenation of the embeddings and the continuous columns</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.bayesian_models import BayesianTabMlp\n&gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n&gt;&gt;&gt; colnames = ['a', 'b', 'c', 'd', 'e']\n&gt;&gt;&gt; cat_embed_input = [(u,i,j) for u,i,j in zip(colnames[:4], [4]*4, [8]*4)]\n&gt;&gt;&gt; column_idx = {k:v for v,k in enumerate(colnames)}\n&gt;&gt;&gt; model = BayesianTabMlp(mlp_hidden_dims=[8,4], column_idx=column_idx, cat_embed_input=cat_embed_input,\n... continuous_cols = ['e'])\n&gt;&gt;&gt; out = model(X_tab)\n</code></pre> Source code in <code>pytorch_widedeep/bayesian_models/tabular/bayesian_mlp/bayesian_tab_mlp.py</code> <pre><code>def __init__(\n    self,\n    column_idx: Dict[str, int],\n    *,\n    cat_embed_input: Optional[List[Tuple[str, int, int]]] = None,\n    cat_embed_activation: Optional[str] = None,\n    continuous_cols: Optional[List[str]] = None,\n    embed_continuous: Optional[bool] = None,\n    cont_embed_dim: Optional[int] = None,\n    cont_embed_dropout: Optional[float] = None,\n    cont_embed_activation: Optional[str] = None,\n    use_cont_bias: Optional[bool] = None,\n    cont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\n    mlp_hidden_dims: List[int] = [200, 100],\n    mlp_activation: str = \"leaky_relu\",\n    prior_sigma_1: float = 1,\n    prior_sigma_2: float = 0.002,\n    prior_pi: float = 0.8,\n    posterior_mu_init: float = 0.0,\n    posterior_rho_init: float = -7.0,\n    pred_dim=1,  # Bayesian models will require their own trainer and need the output layer\n):\n    super(BayesianTabMlp, self).__init__()\n\n    self.column_idx = column_idx\n    self.cat_embed_input = cat_embed_input\n    self.cat_embed_activation = cat_embed_activation\n\n    self.continuous_cols = continuous_cols\n    self.cont_norm_layer = cont_norm_layer\n    self.embed_continuous = embed_continuous\n    self.cont_embed_dim = cont_embed_dim\n    self.cont_embed_dropout = cont_embed_dropout\n    self.use_cont_bias = use_cont_bias\n    self.cont_embed_activation = cont_embed_activation\n\n    self.mlp_hidden_dims = mlp_hidden_dims\n    self.mlp_activation = mlp_activation\n\n    self.prior_sigma_1 = prior_sigma_1\n    self.prior_sigma_2 = prior_sigma_2\n    self.prior_pi = prior_pi\n    self.posterior_mu_init = posterior_mu_init\n    self.posterior_rho_init = posterior_rho_init\n\n    self.pred_dim = pred_dim\n\n    allowed_activations = [\"relu\", \"leaky_relu\", \"tanh\", \"gelu\"]\n    if self.mlp_activation not in allowed_activations:\n        raise ValueError(\n            \"Currently, only the following activation functions are supported \"\n            \"for the Bayesian MLP's dense layers: {}. Got '{}' instead\".format(\n                \", \".join(allowed_activations),\n                self.mlp_activation,\n            )\n        )\n\n    # Categorical\n    if self.cat_embed_input is not None:\n        self.cat_embed = BayesianDiffSizeCatEmbeddings(\n            column_idx=self.column_idx,\n            embed_input=self.cat_embed_input,\n            prior_sigma_1=self.prior_sigma_1,\n            prior_sigma_2=self.prior_sigma_2,\n            prior_pi=self.prior_pi,\n            posterior_mu_init=self.posterior_mu_init,\n            posterior_rho_init=self.posterior_rho_init,\n            activation_fn=self.cat_embed_activation,\n        )\n        self.cat_out_dim = int(np.sum([embed[2] for embed in self.cat_embed_input]))\n    else:\n        self.cat_out_dim = 0\n\n    # Continuous\n    if self.continuous_cols is not None:\n        self.cont_idx = [self.column_idx[col] for col in self.continuous_cols]\n        if cont_norm_layer == \"layernorm\":\n            self.cont_norm: NormLayers = nn.LayerNorm(len(self.continuous_cols))\n        elif cont_norm_layer == \"batchnorm\":\n            self.cont_norm = nn.BatchNorm1d(len(self.continuous_cols))\n        else:\n            self.cont_norm = nn.Identity()\n        if self.embed_continuous:\n            assert self.cont_embed_dim is not None, (\n                \"If 'embed_continuous' is True, 'cont_embed_dim' must be \"\n                \"provided\"\n            )\n            self.cont_embed = BayesianContEmbeddings(\n                n_cont_cols=len(self.continuous_cols),\n                embed_dim=self.cont_embed_dim,\n                prior_sigma_1=self.prior_sigma_1,\n                prior_sigma_2=self.prior_sigma_2,\n                prior_pi=self.prior_pi,\n                posterior_mu_init=self.posterior_mu_init,\n                posterior_rho_init=self.posterior_rho_init,\n                use_bias=(\n                    False if self.use_cont_bias is None else self.use_cont_bias\n                ),\n                activation_fn=self.cont_embed_activation,\n            )\n            self.cont_out_dim = len(self.continuous_cols) * self.cont_embed_dim\n        else:\n            self.cont_out_dim = len(self.continuous_cols)\n    else:\n        self.cont_out_dim = 0\n\n    self.output_dim = self.cat_out_dim + self.cont_out_dim\n\n    mlp_hidden_dims = [self.output_dim] + mlp_hidden_dims + [pred_dim]\n    self.bayesian_tab_mlp = BayesianMLP(\n        mlp_hidden_dims,\n        mlp_activation,\n        True,  # use_bias\n        prior_sigma_1,\n        prior_sigma_2,\n        prior_pi,\n        posterior_mu_init,\n        posterior_rho_init,\n    )\n</code></pre>"},{"location":"pytorch-widedeep/bayesian_trainer.html","title":"Training Deep Learning Probabilistic Models","text":""},{"location":"pytorch-widedeep/bayesian_trainer.html#pytorch_widedeep.training.BayesianTrainer","title":"BayesianTrainer","text":"<pre><code>BayesianTrainer(\n    model,\n    objective,\n    custom_loss_function=None,\n    optimizer=None,\n    lr_scheduler=None,\n    callbacks=None,\n    metrics=None,\n    verbose=1,\n    seed=1,\n    **kwargs\n)\n</code></pre> <p>               Bases: <code>BaseBayesianTrainer</code></p> <p>Class to set the of attributes that will be used during the training process.</p> <p>Both the Bayesian models and the Trainer in this repo are based on the paper: Weight Uncertainty in Neural Networks.</p> <p>Parameters:</p> <ul> <li> <code>model</code>               (<code>BaseBayesianModel</code>)           \u2013            <p>An object of class <code>BaseBayesianModel</code>. See the <code>Model Components</code> section here in the docs.</p> </li> <li> <code>objective</code>               (<code>str</code>)           \u2013            <p>Defines the objective, loss or cost function. Param aliases: <code>loss_function</code>, <code>loss_fn</code>, <code>loss</code>, <code>cost_function</code>, <code>cost_fn</code>, <code>cost</code> Possible values are: 'binary', 'multiclass', 'regression'</p> </li> <li> <code>custom_loss_function</code>               (<code>Optional[Module]</code>, default:                   <code>None</code> )           \u2013            <p>If none of the loss functions available suits the user, it is possible to pass a custom loss function. See for example <code>pytorch_widedeep.losses.FocalLoss</code> for the required structure of the object or the Examples folder in the repo.</p> </li> <li> <code>optimizer</code>               (<code>Optional[Optimizer]</code>, default:                   <code>None</code> )           \u2013            <p>An instance of Pytorch's <code>Optimizer</code> object(e.g. <code>torch.optim.Adam ()</code>). if no optimizer is passed it will default to <code>AdamW</code>.</p> </li> <li> <code>lr_scheduler</code>               (<code>Optional[LRScheduler]</code>, default:                   <code>None</code> )           \u2013            <p>An instance of Pytorch's <code>LRScheduler</code> object (e.g <code>torch.optim.lr_scheduler.StepLR(opt, step_size=5)</code>).</p> </li> <li> <code>callbacks</code>               (<code>Optional[List[Callback]]</code>, default:                   <code>None</code> )           \u2013            <p>List with <code>Callback</code> objects. The three callbacks available in <code>pytorch-widedeep</code> are: <code>LRHistory</code>, <code>ModelCheckpoint</code> and <code>EarlyStopping</code>. This can also be a custom callback. See <code>pytorch_widedeep.callbacks.Callback</code> or the Examples folder in the repo.</p> </li> <li> <code>metrics</code>               (<code>Optional[Union[List[Metric], List[Metric]]]</code>, default:                   <code>None</code> )           \u2013            <ul> <li>List of objects of type <code>Metric</code>. Metrics available are:   <code>Accuracy</code>, <code>Precision</code>, <code>Recall</code>, <code>FBetaScore</code>,   <code>F1Score</code> and <code>R2Score</code>. This can also be a custom metric as   long as it is an object of type <code>Metric</code>. See   <code>pytorch_widedeep.metrics.Metric</code> or the Examples folder in the repo</li> <li>List of objects of type <code>torchmetrics.Metric</code>. This can be any   metric from torchmetrics library Examples   classification-metrics&gt;<code>_. It can also be a torchmetric custom metric as   long as it is an object of type</code>Metric<code>.   See</code>the instructions</li> </ul> </li> <li> <code>verbose</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>Setting it to 0 will print nothing during training.</p> </li> <li> <code>seed</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>Random seed to be used internally for train_test_split</p> </li> </ul> <p>Other Parameters:</p> <ul> <li> <code>**kwargs</code>           \u2013            <p>Other infrequently used arguments that can also be passed as kwargs are:</p> <ul> <li> <p>device: <code>str</code>     string indicating the device. One of 'cpu', 'gpu' or 'mps' if     run on a Mac with Apple silicon or AMD GPU(s)</p> </li> <li> <p>num_workers: <code>int</code>     number of workers to be used internally by the data loaders</p> </li> <li> <p>class_weight: <code>List[float]</code>     This is the <code>weight</code> or <code>pos_weight</code> parameter in     <code>CrossEntropyLoss</code> and <code>BCEWithLogitsLoss</code>, depending on whether</p> </li> <li> <p>reducelronplateau_criterion: <code>str</code>     This sets the criterion that will be used by the lr scheduler to     take a step: One of 'loss' or 'metric'. The ReduceLROnPlateau     learning rate is a bit particular.</p> </li> </ul> </li> </ul> <p>Attributes:</p> <ul> <li> <code>cyclic_lr</code>               (<code>bool</code>)           \u2013            <p>Attribute that indicates if  the lr_scheduler is cyclic_lr (i.e. <code>CyclicLR</code> or <code>OneCycleLR</code>). See <code>Pytorch schedulers &lt;https://pytorch.org/docs/stable/optim.html&gt;</code>_.</p> </li> </ul> Source code in <code>pytorch_widedeep/training/bayesian_trainer.py</code> <pre><code>@alias(  # noqa: C901\n    \"objective\",\n    [\"loss_function\", \"loss_fn\", \"loss\", \"cost_function\", \"cost_fn\", \"cost\"],\n)\ndef __init__(\n    self,\n    model: BaseBayesianModel,\n    objective: str,\n    custom_loss_function: Optional[Module] = None,\n    optimizer: Optional[Optimizer] = None,\n    lr_scheduler: Optional[LRScheduler] = None,\n    callbacks: Optional[List[Callback]] = None,\n    metrics: Optional[Union[List[Metric], List[TorchMetric]]] = None,\n    verbose: int = 1,\n    seed: int = 1,\n    **kwargs,\n):\n    super().__init__(\n        model=model,\n        objective=objective,\n        custom_loss_function=custom_loss_function,\n        optimizer=optimizer,\n        lr_scheduler=lr_scheduler,\n        callbacks=callbacks,\n        metrics=metrics,\n        verbose=verbose,\n        seed=seed,\n        **kwargs,\n    )\n</code></pre>"},{"location":"pytorch-widedeep/bayesian_trainer.html#pytorch_widedeep.training.BayesianTrainer.fit","title":"fit","text":"<pre><code>fit(\n    X_tab,\n    target,\n    X_tab_val=None,\n    target_val=None,\n    val_split=None,\n    n_epochs=1,\n    validation_freq=1,\n    batch_size=32,\n    n_train_samples=2,\n    n_val_samples=2,\n)\n</code></pre> <p>Fit method.</p> <p>Parameters:</p> <ul> <li> <code>X_tab</code>               (<code>ndarray</code>)           \u2013            <p>tabular dataset</p> </li> <li> <code>target</code>               (<code>ndarray</code>)           \u2013            <p>target values</p> </li> <li> <code>X_tab_val</code>               (<code>Optional[ndarray]</code>, default:                   <code>None</code> )           \u2013            <p>validation data</p> </li> <li> <code>target_val</code>               (<code>Optional[ndarray]</code>, default:                   <code>None</code> )           \u2013            <p>validation target values</p> </li> <li> <code>val_split</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>An alterative to passing the validation set is to use a train/val split fraction via <code>val_split</code></p> </li> <li> <code>n_epochs</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>number of epochs</p> </li> <li> <code>validation_freq</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>epochs validation frequency</p> </li> <li> <code>batch_size</code>               (<code>int</code>, default:                   <code>32</code> )           \u2013            <p>batch size</p> </li> <li> <code>n_train_samples</code>               (<code>int</code>, default:                   <code>2</code> )           \u2013            <p>number of samples to average over during the training process. See Weight Uncertainty in Neural Networks for details.</p> </li> <li> <code>n_val_samples</code>               (<code>int</code>, default:                   <code>2</code> )           \u2013            <p>number of samples to average over during the validation process. See Weight Uncertainty in Neural Networks for details.</p> </li> </ul> Source code in <code>pytorch_widedeep/training/bayesian_trainer.py</code> <pre><code>def fit(  # noqa: C901\n    self,\n    X_tab: np.ndarray,\n    target: np.ndarray,\n    X_tab_val: Optional[np.ndarray] = None,\n    target_val: Optional[np.ndarray] = None,\n    val_split: Optional[float] = None,\n    n_epochs: int = 1,\n    validation_freq: int = 1,\n    batch_size: int = 32,\n    n_train_samples: int = 2,\n    n_val_samples: int = 2,\n):\n    r\"\"\"Fit method.\n\n    Parameters\n    ----------\n    X_tab: np.ndarray,\n        tabular dataset\n    target: np.ndarray\n        target values\n    X_tab_val: np.ndarray, Optional, default = None\n        validation data\n    target_val: np.ndarray, Optional, default = None\n        validation target values\n    val_split: float, Optional. default=None\n        An alterative to passing the validation set is to use a train/val\n        split fraction via `val_split`\n    n_epochs: int, default=1\n        number of epochs\n    validation_freq: int, default=1\n        epochs validation frequency\n    batch_size: int, default=32\n        batch size\n    n_train_samples: int, default=2\n        number of samples to average over during the training process.\n        See [Weight Uncertainty in Neural Networks](https://arxiv.org/pdf/1505.05424.pdf) for details.\n    n_val_samples: int, default=2\n        number of samples to average over during the validation process.\n        See [Weight Uncertainty in Neural Networks](https://arxiv.org/pdf/1505.05424.pdf) for details.\n    \"\"\"\n\n    self.batch_size = batch_size\n\n    train_set, eval_set = tabular_train_val_split(\n        self.seed, self.objective, X_tab, target, X_tab_val, target_val, val_split\n    )\n    train_loader = DataLoader(\n        dataset=train_set, batch_size=batch_size, num_workers=self.num_workers\n    )\n    train_steps = len(train_loader)\n\n    if eval_set is not None:\n        eval_loader = DataLoader(\n            dataset=eval_set,\n            batch_size=batch_size,\n            num_workers=self.num_workers,\n            shuffle=False,\n        )\n        eval_steps = len(eval_loader)\n\n    self.callback_container.on_train_begin(\n        {\n            \"batch_size\": batch_size,\n            \"train_steps\": train_steps,\n            \"n_epochs\": n_epochs,\n        }\n    )\n    for epoch in range(n_epochs):\n        epoch_logs: Dict[str, float] = {}\n        self.callback_container.on_epoch_begin(epoch, logs=epoch_logs)\n\n        self.train_running_loss = 0.0\n        with trange(train_steps, disable=self.verbose != 1) as t:\n            for batch_idx, (X, y) in zip(t, train_loader):\n                t.set_description(\"epoch %i\" % (epoch + 1))\n                train_score, train_loss = self._train_step(\n                    X, y, n_train_samples, train_steps, batch_idx\n                )\n                print_loss_and_metric(t, train_loss, train_score)\n                self.callback_container.on_batch_end(batch=batch_idx)\n        epoch_logs = save_epoch_logs(epoch_logs, train_loss, train_score, \"train\")\n\n        on_epoch_end_metric = None\n        if eval_set is not None and epoch % validation_freq == (\n            validation_freq - 1\n        ):\n            self.callback_container.on_eval_begin()\n            self.valid_running_loss = 0.0\n            with trange(eval_steps, disable=self.verbose != 1) as v:\n                for i, (X, y) in zip(v, eval_loader):\n                    v.set_description(\"valid\")\n                    val_score, val_loss = self._eval_step(\n                        X, y, n_val_samples, train_steps, i\n                    )\n                    print_loss_and_metric(v, val_loss, val_score)\n            epoch_logs = save_epoch_logs(epoch_logs, val_loss, val_score, \"val\")\n\n            if self.reducelronplateau:\n                if self.reducelronplateau_criterion == \"loss\":\n                    on_epoch_end_metric = val_loss\n                else:\n                    on_epoch_end_metric = val_score[\n                        self.reducelronplateau_criterion\n                    ]\n\n        self.callback_container.on_epoch_end(epoch, epoch_logs, on_epoch_end_metric)\n\n        if self.early_stop:\n            self.callback_container.on_train_end(epoch_logs)\n            break\n\n    self.callback_container.on_train_end(epoch_logs)\n    self._restore_best_weights()\n    self.model.train()\n</code></pre>"},{"location":"pytorch-widedeep/bayesian_trainer.html#pytorch_widedeep.training.BayesianTrainer.predict","title":"predict","text":"<pre><code>predict(\n    X_tab, n_samples=5, return_samples=False, batch_size=256\n)\n</code></pre> <p>Returns the predictions</p> <p>Parameters:</p> <ul> <li> <code>X_tab</code>               (<code>ndarray</code>)           \u2013            <p>tabular dataset</p> </li> <li> <code>n_samples</code>               (<code>int</code>, default:                   <code>5</code> )           \u2013            <p>number of samples that will be either returned or averaged to produce an overal prediction</p> </li> <li> <code>return_samples</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether the n samples will be averaged or directly returned</p> </li> <li> <code>batch_size</code>               (<code>int</code>, default:                   <code>256</code> )           \u2013            <p>batch size</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>np.ndarray:</code>           \u2013            <p>array with the predictions</p> </li> </ul> Source code in <code>pytorch_widedeep/training/bayesian_trainer.py</code> <pre><code>def predict(  # type: ignore[return]\n    self,\n    X_tab: np.ndarray,\n    n_samples: int = 5,\n    return_samples: bool = False,\n    batch_size: int = 256,\n) -&gt; np.ndarray:\n    r\"\"\"Returns the predictions\n\n    Parameters\n    ----------\n    X_tab: np.ndarray,\n        tabular dataset\n    n_samples: int, default=5\n        number of samples that will be either returned or averaged to\n        produce an overal prediction\n    return_samples: bool, default = False\n        Boolean indicating whether the n samples will be averaged or directly returned\n    batch_size: int, default = 256\n        batch size\n\n    Returns\n    -------\n    np.ndarray:\n        array with the predictions\n    \"\"\"\n\n    preds_l = self._predict(X_tab, n_samples, return_samples, batch_size)\n    preds = np.hstack(preds_l) if return_samples else np.vstack(preds_l)\n    axis = 2 if return_samples else 1\n\n    if self.objective == \"regression\":\n        return preds.squeeze(axis)\n    if self.objective == \"binary\":\n        return (preds.squeeze(axis) &gt; 0.5).astype(\"int\")\n    if self.objective == \"multiclass\":\n        return np.argmax(preds, axis)\n</code></pre>"},{"location":"pytorch-widedeep/bayesian_trainer.html#pytorch_widedeep.training.BayesianTrainer.predict_proba","title":"predict_proba","text":"<pre><code>predict_proba(\n    X_tab, n_samples=5, return_samples=False, batch_size=256\n)\n</code></pre> <p>Returns the predicted probabilities</p> <p>Parameters:</p> <ul> <li> <code>X_tab</code>               (<code>ndarray</code>)           \u2013            <p>tabular dataset</p> </li> <li> <code>n_samples</code>               (<code>int</code>, default:                   <code>5</code> )           \u2013            <p>number of samples that will be either returned or averaged to produce an overal prediction</p> </li> <li> <code>return_samples</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether the n samples will be averaged or directly returned</p> </li> <li> <code>batch_size</code>               (<code>int</code>, default:                   <code>256</code> )           \u2013            <p>batch size</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>ndarray</code>           \u2013            <p>array with the probabilities per class</p> </li> </ul> Source code in <code>pytorch_widedeep/training/bayesian_trainer.py</code> <pre><code>def predict_proba(  # type: ignore[return]\n    self,\n    X_tab: np.ndarray,\n    n_samples: int = 5,\n    return_samples: bool = False,\n    batch_size: int = 256,\n) -&gt; np.ndarray:\n    r\"\"\"Returns the predicted probabilities\n\n    Parameters\n    ----------\n    X_tab: np.ndarray,\n        tabular dataset\n    n_samples: int, default=5\n        number of samples that will be either returned or averaged to\n        produce an overal prediction\n    return_samples: bool, default = False\n        Boolean indicating whether the n samples will be averaged or directly returned\n    batch_size: int, default = 256\n        batch size\n\n    Returns\n    -------\n    np.ndarray\n        array with the probabilities per class\n    \"\"\"\n    preds_l = self._predict(X_tab, n_samples, return_samples, batch_size)\n    preds = np.hstack(preds_l) if return_samples else np.vstack(preds_l)\n\n    if self.objective == \"binary\":\n        if return_samples:\n            preds = preds.squeeze(2)\n            probs = np.zeros([n_samples, preds.shape[1], 2])\n            for i in range(n_samples):\n                probs[i, :, 0] = 1 - preds[i]\n                probs[i, :, 1] = preds[i]\n        else:\n            preds = preds.squeeze(1)\n            probs = np.zeros([preds.shape[0], 2])\n            probs[:, 0] = 1 - preds\n            probs[:, 1] = preds\n        return probs\n    if self.objective == \"multiclass\":\n        return preds\n</code></pre>"},{"location":"pytorch-widedeep/bayesian_trainer.html#pytorch_widedeep.training.BayesianTrainer.save","title":"save","text":"<pre><code>save(\n    path,\n    save_state_dict=False,\n    model_filename=\"bayesian_model.pt\",\n)\n</code></pre> <p>Saves the model, training and evaluation history to disk</p> <p>The <code>Trainer</code> class is built so that it 'just' trains a model. With that in mind, all the torch related parameters (such as optimizers or learning rate schedulers) have to be defined externally and then passed to the <code>Trainer</code>. As a result, the <code>Trainer</code> does not generate any attribute or additional data products that need to be saved other than the <code>model</code> object itself, which can be saved as any other torch model (e.g. <code>torch.save(model, path)</code>).</p> <p>Parameters:</p> <ul> <li> <code>path</code>               (<code>str</code>)           \u2013            <p>path to the directory where the model and the feature importance attribute will be saved.</p> </li> <li> <code>save_state_dict</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether to save directly the model or the model's state dictionary</p> </li> <li> <code>model_filename</code>               (<code>str</code>, default:                   <code>'bayesian_model.pt'</code> )           \u2013            <p>filename where the model weights will be store</p> </li> </ul> Source code in <code>pytorch_widedeep/training/bayesian_trainer.py</code> <pre><code>def save(\n    self,\n    path: str,\n    save_state_dict: bool = False,\n    model_filename: str = \"bayesian_model.pt\",\n):\n    r\"\"\"Saves the model, training and evaluation history to disk\n\n    The `Trainer` class is built so that it 'just' trains a model. With\n    that in mind, all the torch related parameters (such as optimizers or\n    learning rate schedulers) have to be defined externally and then\n    passed to the `Trainer`. As a result, the `Trainer` does not\n    generate any attribute or additional data products that need to be\n    saved other than the `model` object itself, which can be saved as\n    any other torch model (e.g. `torch.save(model, path)`).\n\n    Parameters\n    ----------\n    path: str\n        path to the directory where the model and the feature importance\n        attribute will be saved.\n    save_state_dict: bool, default = False\n        Boolean indicating whether to save directly the model or the\n        model's state dictionary\n    model_filename: str, Optional, default = \"wd_model.pt\"\n        filename where the model weights will be store\n    \"\"\"\n\n    save_dir = Path(path)\n    history_dir = save_dir / \"history\"\n    history_dir.mkdir(exist_ok=True, parents=True)\n\n    # the trainer is run with the History Callback by default\n    with open(history_dir / \"train_eval_history.json\", \"w\") as teh:\n        json.dump(self.history, teh)  # type: ignore[attr-defined]\n\n    has_lr_history = any(\n        [clbk.__class__.__name__ == \"LRHistory\" for clbk in self.callbacks]\n    )\n    if self.lr_scheduler is not None and has_lr_history:\n        with open(history_dir / \"lr_history.json\", \"w\") as lrh:\n            json.dump(self.lr_history, lrh)  # type: ignore[attr-defined]\n\n    model_path = save_dir / model_filename\n    if save_state_dict:\n        torch.save(self.model.state_dict(), model_path)\n    else:\n        torch.save(self.model, model_path)\n</code></pre>"},{"location":"pytorch-widedeep/callbacks.html","title":"Callbacks","text":"<p>Here are the 4 callbacks available to the user in <code>pytorch-widedepp</code>: <code>LRHistory</code>, <code>ModelCheckpoint</code>, <code>EarlyStopping</code> and <code>RayTuneReporter</code>.</p> <p> NOTE: other callbacks , like <code>History</code>, run always  by default. In particular, the <code>History</code> callback saves the metrics in the  <code>history</code> attribute of the <code>Trainer</code>.</p>"},{"location":"pytorch-widedeep/callbacks.html#pytorch_widedeep.callbacks.LRHistory","title":"LRHistory","text":"<pre><code>LRHistory(n_epochs)\n</code></pre> <p>               Bases: <code>Callback</code></p> <p>Saves the learning rates during training in the <code>lr_history</code> attribute of the <code>Trainer</code>.</p> <p>Callbacks are passed as input parameters to the <code>Trainer</code> class. See <code>pytorch_widedeep.trainer.Trainer</code></p> <p>Parameters:</p> <ul> <li> <code>n_epochs</code>               (<code>int</code>)           \u2013            <p>number of training epochs</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; from pytorch_widedeep.callbacks import LRHistory\n&gt;&gt;&gt; from pytorch_widedeep.models import TabMlp, Wide, WideDeep\n&gt;&gt;&gt; from pytorch_widedeep.training import Trainer\n&gt;&gt;&gt;\n&gt;&gt;&gt; embed_input = [(u, i, j) for u, i, j in zip([\"a\", \"b\", \"c\"][:4], [4] * 3, [8] * 3)]\n&gt;&gt;&gt; column_idx = {k: v for v, k in enumerate([\"a\", \"b\", \"c\"])}\n&gt;&gt;&gt; wide = Wide(10, 1)\n&gt;&gt;&gt; deep = TabMlp(mlp_hidden_dims=[8, 4], column_idx=column_idx, cat_embed_input=embed_input)\n&gt;&gt;&gt; model = WideDeep(wide, deep)\n&gt;&gt;&gt; trainer = Trainer(model, objective=\"regression\", callbacks=[LRHistory(n_epochs=10)])\n</code></pre> Source code in <code>pytorch_widedeep/callbacks.py</code> <pre><code>def __init__(self, n_epochs: int):\n    super(LRHistory, self).__init__()\n    self.n_epochs = n_epochs\n</code></pre>"},{"location":"pytorch-widedeep/callbacks.html#pytorch_widedeep.callbacks.ModelCheckpoint","title":"ModelCheckpoint","text":"<pre><code>ModelCheckpoint(\n    filepath=None,\n    monitor=\"val_loss\",\n    min_delta=0.0,\n    verbose=0,\n    save_best_only=False,\n    mode=\"auto\",\n    period=1,\n    max_save=-1,\n)\n</code></pre> <p>               Bases: <code>Callback</code></p> <p>Saves the model after every epoch.</p> <p>This class is almost identical to the corresponding keras class. Therefore, credit to the Keras Team.</p> <p>Callbacks are passed as input parameters to the <code>Trainer</code> class. See <code>pytorch_widedeep.trainer.Trainer</code></p> <p>Parameters:</p> <ul> <li> <code>filepath</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Full path to save the output weights. It must contain only the root of the filenames. Epoch number and <code>.pt</code> extension (for pytorch) will be added. e.g. <code>filepath=\"path/to/output_weights/weights_out\"</code> And the saved files in that directory will be named: 'weights_out_1.pt', 'weights_out_2.pt', .... If set to <code>None</code> the class just report best metric and best_epoch.</p> </li> <li> <code>monitor</code>               (<code>str</code>, default:                   <code>'val_loss'</code> )           \u2013            <p>quantity to monitor. Typically 'val_loss' or metric name (e.g. 'val_acc')</p> </li> <li> <code>min_delta</code>               (<code>float</code>, default:                   <code>0.0</code> )           \u2013            <p>minimum change in the monitored quantity to qualify as an improvement, i.e. an absolute change of less than min_delta, will count as no improvement.</p> </li> <li> <code>verbose</code>               (<code>int</code>, default:                   <code>0</code> )           \u2013            <p>verbosity mode</p> </li> <li> <code>save_best_only</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>the latest best model according to the quantity monitored will not be overwritten.</p> </li> <li> <code>mode</code>               (<code>str</code>, default:                   <code>'auto'</code> )           \u2013            <p>If <code>save_best_only=True</code>, the decision to overwrite the current save file is made based on either the maximization or the minimization of the monitored quantity. For 'acc', this should be 'max', for 'loss' this should be 'min', etc. In 'auto' mode, the direction is automatically inferred from the name of the monitored quantity.</p> </li> <li> <code>period</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>Interval (number of epochs) between checkpoints.</p> </li> <li> <code>max_save</code>               (<code>int</code>, default:                   <code>-1</code> )           \u2013            <p>Maximum number of outputs to save. If -1 will save all outputs</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>best</code>               (<code>float</code>)           \u2013            <p>best metric</p> </li> <li> <code>best_epoch</code>               (<code>int</code>)           \u2013            <p>best epoch</p> </li> <li> <code>best_state_dict</code>               (<code>dict</code>)           \u2013            <p>best model state dictionary. To restore model to its best state use <code>Trainer.model.load_state_dict (model_checkpoint.best_state_dict)</code> where <code>model_checkpoint</code> is an instance of the class <code>ModelCheckpoint</code>. See the Examples folder in the repo or the Examples section in this documentation for details</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; from pytorch_widedeep.callbacks import ModelCheckpoint\n&gt;&gt;&gt; from pytorch_widedeep.models import TabMlp, Wide, WideDeep\n&gt;&gt;&gt; from pytorch_widedeep.training import Trainer\n&gt;&gt;&gt;\n&gt;&gt;&gt; embed_input = [(u, i, j) for u, i, j in zip([\"a\", \"b\", \"c\"][:4], [4] * 3, [8] * 3)]\n&gt;&gt;&gt; column_idx = {k: v for v, k in enumerate([\"a\", \"b\", \"c\"])}\n&gt;&gt;&gt; wide = Wide(10, 1)\n&gt;&gt;&gt; deep = TabMlp(mlp_hidden_dims=[8, 4], column_idx=column_idx, cat_embed_input=embed_input)\n&gt;&gt;&gt; model = WideDeep(wide, deep)\n&gt;&gt;&gt; trainer = Trainer(model, objective=\"regression\", callbacks=[ModelCheckpoint(filepath='checkpoints/weights_out')])\n</code></pre> Source code in <code>pytorch_widedeep/callbacks.py</code> <pre><code>def __init__(\n    self,\n    filepath: Optional[str] = None,\n    monitor: str = \"val_loss\",\n    min_delta: float = 0.0,\n    verbose: int = 0,\n    save_best_only: bool = False,\n    mode: str = \"auto\",\n    period: int = 1,\n    max_save: int = -1,\n):\n    super(ModelCheckpoint, self).__init__()\n\n    self.filepath = filepath\n    self.monitor = monitor\n    self.min_delta = min_delta\n    self.verbose = verbose\n    self.save_best_only = save_best_only\n    self.mode = mode\n    self.period = period\n    self.max_save = max_save\n\n    self.epochs_since_last_save = 0\n\n    if self.filepath:\n        if len(self.filepath.split(\"/\")[:-1]) == 0:\n            raise ValueError(\n                \"'filepath' must be the full path to save the output weights,\"\n                \" including the root of the filenames. e.g. 'checkpoints/weights_out'\"\n            )\n\n        root_dir = (\"/\").join(self.filepath.split(\"/\")[:-1])\n        if not os.path.exists(root_dir):\n            os.makedirs(root_dir)\n\n    if self.max_save &gt; 0:\n        self.old_files: List[str] = []\n\n    if self.mode not in [\"auto\", \"min\", \"max\"]:\n        warnings.warn(\n            \"ModelCheckpoint mode %s is unknown, \"\n            \"fallback to auto mode.\" % (self.mode),\n            RuntimeWarning,\n        )\n        self.mode = \"auto\"\n    if self.mode == \"min\":\n        self.monitor_op = np.less\n        self.best = np.Inf\n    elif self.mode == \"max\":\n        self.monitor_op = np.greater  # type: ignore[assignment]\n        self.best = -np.Inf\n    else:\n        if _is_metric(self.monitor):\n            self.monitor_op = np.greater  # type: ignore[assignment]\n            self.best = -np.Inf\n        else:\n            self.monitor_op = np.less\n            self.best = np.Inf\n\n    if self.monitor_op == np.greater:\n        self.min_delta *= 1\n    else:\n        self.min_delta *= -1\n</code></pre>"},{"location":"pytorch-widedeep/callbacks.html#pytorch_widedeep.callbacks.EarlyStopping","title":"EarlyStopping","text":"<pre><code>EarlyStopping(\n    monitor=\"val_loss\",\n    min_delta=0.0,\n    patience=10,\n    verbose=0,\n    mode=\"auto\",\n    baseline=None,\n    restore_best_weights=False,\n)\n</code></pre> <p>               Bases: <code>Callback</code></p> <p>Stop training when a monitored quantity has stopped improving.</p> <p>This class is almost identical to the corresponding keras class. Therefore, credit to the Keras Team.</p> <p>Callbacks are passed as input parameters to the <code>Trainer</code> class. See <code>pytorch_widedeep.trainer.Trainer</code></p> <p>Parameters:</p> <ul> <li> <code>monitor</code>               (<code>str</code>, default:                   <code>'val_loss'</code> )           \u2013            <p>Quantity to monitor. Typically 'val_loss' or metric name (e.g. 'val_acc')</p> </li> <li> <code>min_delta</code>               (<code>float</code>, default:                   <code>0.0</code> )           \u2013            <p>minimum change in the monitored quantity to qualify as an improvement, i.e. an absolute change of less than min_delta, will count as no improvement.</p> </li> <li> <code>patience</code>               (<code>int</code>, default:                   <code>10</code> )           \u2013            <p>Number of epochs that produced the monitored quantity with no improvement after which training will be stopped.</p> </li> <li> <code>verbose</code>               (<code>int</code>, default:                   <code>0</code> )           \u2013            <p>verbosity mode.</p> </li> <li> <code>mode</code>               (<code>str</code>, default:                   <code>'auto'</code> )           \u2013            <p>one of {'auto', 'min', 'max'}. In 'min' mode, training will stop when the quantity monitored has stopped decreasing; in 'max' mode it will stop when the quantity monitored has stopped increasing; in 'auto' mode, the direction is automatically inferred from the name of the monitored quantity.</p> </li> <li> <code>baseline</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Baseline value for the monitored quantity to reach. Training will stop if the model does not show improvement over the baseline.</p> </li> <li> <code>restore_best_weights</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Whether to restore model weights from the epoch with the best value of the monitored quantity. If <code>False</code>, the model weights obtained at the last step of training are used.</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>best</code>               (<code>float</code>)           \u2013            <p>best metric</p> </li> <li> <code>stopped_epoch</code>               (<code>int</code>)           \u2013            <p>epoch when the training stopped</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; from pytorch_widedeep.callbacks import EarlyStopping\n&gt;&gt;&gt; from pytorch_widedeep.models import TabMlp, Wide, WideDeep\n&gt;&gt;&gt; from pytorch_widedeep.training import Trainer\n&gt;&gt;&gt;\n&gt;&gt;&gt; embed_input = [(u, i, j) for u, i, j in zip([\"a\", \"b\", \"c\"][:4], [4] * 3, [8] * 3)]\n&gt;&gt;&gt; column_idx = {k: v for v, k in enumerate([\"a\", \"b\", \"c\"])}\n&gt;&gt;&gt; wide = Wide(10, 1)\n&gt;&gt;&gt; deep = TabMlp(mlp_hidden_dims=[8, 4], column_idx=column_idx, cat_embed_input=embed_input)\n&gt;&gt;&gt; model = WideDeep(wide, deep)\n&gt;&gt;&gt; trainer = Trainer(model, objective=\"regression\", callbacks=[EarlyStopping(patience=10)])\n</code></pre> Source code in <code>pytorch_widedeep/callbacks.py</code> <pre><code>def __init__(\n    self,\n    monitor: str = \"val_loss\",\n    min_delta: float = 0.0,\n    patience: int = 10,\n    verbose: int = 0,\n    mode: str = \"auto\",\n    baseline: Optional[float] = None,\n    restore_best_weights: bool = False,\n):\n    super(EarlyStopping, self).__init__()\n\n    self.monitor = monitor\n    self.min_delta = min_delta\n    self.patience = patience\n    self.verbose = verbose\n    self.mode = mode\n    self.baseline = baseline\n    self.restore_best_weights = restore_best_weights\n\n    self.wait = 0\n    self.stopped_epoch = 0\n    self.state_dict = None\n\n    if self.mode not in [\"auto\", \"min\", \"max\"]:\n        warnings.warn(\n            \"EarlyStopping mode %s is unknown, \"\n            \"fallback to auto mode.\" % self.mode,\n            RuntimeWarning,\n        )\n        self.mode = \"auto\"\n\n    if self.mode == \"min\":\n        self.monitor_op = np.less\n    elif self.mode == \"max\":\n        self.monitor_op = np.greater  # type: ignore[assignment]\n    else:\n        if _is_metric(self.monitor):\n            self.monitor_op = np.greater  # type: ignore[assignment]\n        else:\n            self.monitor_op = np.less\n\n    if self.monitor_op == np.greater:\n        self.min_delta *= 1\n    else:\n        self.min_delta *= -1\n</code></pre>"},{"location":"pytorch-widedeep/dataloaders.html","title":"Dataloaders","text":"<p> NOTE: This module should contain custom dataloaders  that the user might want to implement. At the moment <code>pytorch-widedeep</code>  offers one custom dataloader, <code>DataLoaderImbalanced</code>.</p>"},{"location":"pytorch-widedeep/dataloaders.html#pytorch_widedeep.dataloaders.DataLoaderImbalanced","title":"DataLoaderImbalanced","text":"<pre><code>DataLoaderImbalanced(dataset=None, *args, **kwargs)\n</code></pre> <p>               Bases: <code>CustomDataLoader</code></p> <p>Class to load and shuffle batches with adjusted weights for imbalanced datasets. If the classes do not begin from 0 remapping is necessary. See here.</p> <p>Parameters:</p> <ul> <li> <code>dataset</code>               (<code>Optional[WideDeepDataset]</code>, default:                   <code>None</code> )           \u2013            <p>see <code>pytorch_widedeep.training._wd_dataset</code></p> </li> </ul> <p>Other Parameters:</p> <ul> <li> <code>*args</code>           \u2013            <p>Positional arguments to be passed to the parent CustomDataLoader.</p> </li> <li> <code>**kwargs</code>           \u2013            <p>This can include any parameter that can be passed to the 'standard' pytorch DataLoader and that is not already explicitely passed to the class. In addition, the dictionary can also include the extra parameter <code>oversample_mul</code> which will multiply the number of samples of the minority class to be sampled by the <code>WeightedRandomSampler</code>.</p> <p>In other words, the <code>num_samples</code> param in <code>WeightedRandomSampler</code> will be defined as:</p> \\[ minority \\space class \\space count \\times number \\space of \\space classes \\times oversample\\_mul \\] </li> </ul> Source code in <code>pytorch_widedeep/dataloaders.py</code> <pre><code>def __init__(self, dataset: Optional[WideDeepDataset] = None, *args, **kwargs):\n    self.args = args\n    self.kwargs = kwargs\n\n    if dataset is not None:\n        self._setup_sampler(dataset)\n        super().__init__(dataset, *args, sampler=self.sampler, **kwargs)\n    else:\n        super().__init__()\n</code></pre>"},{"location":"pytorch-widedeep/load_from_folder.html","title":"The <code>load_from_folder</code> module","text":"<p>The <code>load_from_folder</code> module contains the classes that are necessary to load data from disk and these are inspired by the <code>ImageFolder</code> class in the <code>torchvision</code> library. This module is designed with one specific case in mind. Such case is the following: given a multi-modal dataset with tabular data, images and text, the images do not fit in memory, and therefore, they have to be loaded from disk. However, as any other functionality in this library, there is some flexibility and some additional cases can also be addressed using this module.</p> <p>For this module to be used, the datasets must be prepared in a certain way:</p> <ol> <li> <p>the tabular data must contain a column with the images names as stored in disk, including the extension (<code>.jpg</code>, <code>.png</code>, etc...).</p> </li> <li> <p>Regarding to the text dataset, the tabular data can contain a column with the texts themselves or the names of the files containing the texts as stored in disk.</p> </li> </ol> <p>The tabular data might or might not fit in disk itself. If it does not, please see the <code>ChunkPreprocessor</code> utilities at the[<code>preprocessing</code>] (preprocessing.md) module and the examples folder in the repo, which illustrate such case. Finally note that only <code>csv</code> format is currently supported in that case(more formats coming soon).</p>"},{"location":"pytorch-widedeep/load_from_folder.html#pytorch_widedeep.load_from_folder.tabular.tabular_from_folder.TabFromFolder","title":"TabFromFolder","text":"<pre><code>TabFromFolder(\n    fname,\n    directory=None,\n    target_col=None,\n    preprocessor=None,\n    text_col=None,\n    img_col=None,\n    ignore_target=False,\n    reference=None,\n    verbose=1,\n)\n</code></pre> <p>This class is used to load tabular data from disk. The current constrains are:</p> <ol> <li>The only file format supported right now is csv</li> <li>The csv file must contain headers</li> </ol> <p>For examples, please, see the examples folder in the repo.</p> <p>Parameters:</p> <ul> <li> <code>fname</code>               (<code>str</code>)           \u2013            <p>the name of the csv file</p> </li> <li> <code>directory</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>the path to the directory where the csv file is located. If None, a <code>TabFromFolder</code> reference object must be provided</p> </li> <li> <code>target_col</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>the name of the target column. If None, a <code>TabFromFolder</code> reference object must be provided</p> </li> <li> <code>preprocessor</code>               (<code>Optional[TabularPreprocessor]</code>, default:                   <code>None</code> )           \u2013            <p>a fitted <code>TabularPreprocessor</code> object. If None, a <code>TabFromFolder</code> reference object must be provided</p> </li> <li> <code>text_col</code>               (<code>Optional[Union[str, List[str]]]</code>, default:                   <code>None</code> )           \u2013            <p>the name of the column with the texts themselves or the names of the files that contain the text dataset. If None, either there is no text column or a <code>TabFromFolder</code> reference object must be provided</p> </li> <li> <code>img_col</code>               (<code>Optional[Union[str, List[str]]]</code>, default:                   <code>None</code> )           \u2013            <p>the name of the column with the the names of the images. If None, either there is no image column or a <code>TabFromFolder</code> reference object must be provided</p> </li> <li> <code>ignore_target</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>whether to ignore the target column. This is normally set to True when this class is used for a test dataset.</p> </li> <li> <code>reference</code>               (<code>Optional[Any]</code>, default:                   <code>None</code> )           \u2013            <p>a reference <code>TabFromFolder</code> object. If provided, the <code>TabFromFolder</code> object will be created using the attributes of the reference object. This is useful to instantiate a <code>TabFromFolder</code> object for evaluation or test purposes</p> </li> <li> <code>verbose</code>               (<code>Optional[int]</code>, default:                   <code>1</code> )           \u2013            <p>verbosity. If 0, no output will be printed during the process.</p> </li> </ul> Source code in <code>pytorch_widedeep/load_from_folder/tabular/tabular_from_folder.py</code> <pre><code>def __init__(\n    self,\n    fname: str,\n    directory: Optional[str] = None,\n    target_col: Optional[str] = None,\n    preprocessor: Optional[TabularPreprocessor] = None,\n    text_col: Optional[Union[str, List[str]]] = None,\n    img_col: Optional[Union[str, List[str]]] = None,\n    ignore_target: bool = False,\n    reference: Optional[Any] = None,  # is Type[\"TabFromFolder\"],\n    verbose: Optional[int] = 1,\n):\n    self.fname = fname\n    self.ignore_target = ignore_target\n    self.verbose = verbose\n\n    if reference is not None:\n        (\n            self.directory,\n            self.target_col,\n            self.preprocessor,\n            self.text_col,\n            self.img_col,\n        ) = self._set_from_reference(reference, preprocessor)\n    else:\n        assert (\n            directory is not None\n            and (target_col is not None and not ignore_target)\n            and preprocessor is not None\n        ), (\n            \"if no reference is provided, 'directory', 'target_col' and 'preprocessor' \"\n            \"must be provided\"\n        )\n\n        self.directory = directory\n        self.target_col = target_col\n        self.preprocessor = preprocessor\n        self.text_col = text_col\n        self.img_col = img_col\n\n    assert (\n        self.preprocessor.is_fitted\n    ), \"The preprocessor must be fitted before passing it to this class\"\n</code></pre>"},{"location":"pytorch-widedeep/load_from_folder.html#pytorch_widedeep.load_from_folder.tabular.tabular_from_folder.WideFromFolder","title":"WideFromFolder","text":"<pre><code>WideFromFolder(\n    fname,\n    directory=None,\n    target_col=None,\n    preprocessor=None,\n    text_col=None,\n    img_col=None,\n    ignore_target=False,\n    reference=None,\n    verbose=1,\n)\n</code></pre> <p>               Bases: <code>TabFromFolder</code></p> <p>This class is mostly identical to <code>TabFromFolder</code> but exists because we want to separate the treatment of the wide and the deep tabular components</p> <p>Parameters:</p> <ul> <li> <code>fname</code>               (<code>str</code>)           \u2013            <p>the name of the csv file</p> </li> <li> <code>directory</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>the path to the directory where the csv file is located. If None, a <code>WideFromFolder</code> reference object must be provided</p> </li> <li> <code>target_col</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>the name of the target column. If None, a <code>WideFromFolder</code> reference object must be provided</p> </li> <li> <code>preprocessor</code>               (<code>Optional[TabularPreprocessor]</code>, default:                   <code>None</code> )           \u2013            <p>a fitted <code>TabularPreprocessor</code> object. If None, a <code>WideFromFolder</code> reference object must be provided</p> </li> <li> <code>text_col</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>the name of the column with the texts themselves or the names of the files that contain the text dataset. If None, either there is no text column or a <code>WideFromFolder</code> reference object must be provided=</p> </li> <li> <code>img_col</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>the name of the column with the the names of the images. If None, either there is no image column or a <code>WideFromFolder</code> reference object must be provided</p> </li> <li> <code>ignore_target</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>whether to ignore the target column. This is normally used when this class is used for a test dataset.</p> </li> <li> <code>reference</code>               (<code>Optional[Any]</code>, default:                   <code>None</code> )           \u2013            <p>a reference <code>WideFromFolder</code> object. If provided, the <code>WideFromFolder</code> object will be created using the attributes of the reference object. This is useful to instantiate a <code>WideFromFolder</code> object for evaluation or test purposes</p> </li> <li> <code>verbose</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>verbosity. If 0, no output will be printed during the process.</p> </li> </ul> Source code in <code>pytorch_widedeep/load_from_folder/tabular/tabular_from_folder.py</code> <pre><code>def __init__(\n    self,\n    fname: str,\n    directory: Optional[str] = None,\n    target_col: Optional[str] = None,\n    preprocessor: Optional[TabularPreprocessor] = None,\n    text_col: Optional[str] = None,\n    img_col: Optional[str] = None,\n    ignore_target: bool = False,\n    reference: Optional[Any] = None,  # is Type[\"WideFromFolder\"],\n    verbose: int = 1,\n):\n    super(WideFromFolder, self).__init__(\n        fname=fname,\n        directory=directory,\n        target_col=target_col,\n        preprocessor=preprocessor,\n        text_col=text_col,\n        img_col=img_col,\n        reference=reference,\n        ignore_target=ignore_target,\n        verbose=verbose,\n    )\n</code></pre>"},{"location":"pytorch-widedeep/load_from_folder.html#pytorch_widedeep.load_from_folder.text.text_from_folder.TextFromFolder","title":"TextFromFolder","text":"<pre><code>TextFromFolder(preprocessor)\n</code></pre> <p>This class is used to load the text dataset (i.e. the text files) from a folder, or to retrieve the text given a texts column specified within the preprocessor object.</p> <p>For examples, please, see the examples folder in the repo.</p> <p>Parameters:</p> <ul> <li> <code>preprocessor</code>               (<code>Union[TextPreprocessor, ChunkTextPreprocessor, HFPreprocessor, ChunkHFPreprocessor, List[TextPreprocessor], List[ChunkTextPreprocessor], List[HFPreprocessor], List[ChunkHFPreprocessor]]</code>)           \u2013            <p>The preprocessor used to process the text. It must be fitted before using this class</p> </li> </ul> Source code in <code>pytorch_widedeep/load_from_folder/text/text_from_folder.py</code> <pre><code>def __init__(\n    self,\n    preprocessor: Union[\n        TextPreprocessor,\n        ChunkTextPreprocessor,\n        HFPreprocessor,\n        ChunkHFPreprocessor,\n        List[TextPreprocessor],\n        List[ChunkTextPreprocessor],\n        List[HFPreprocessor],\n        List[ChunkHFPreprocessor],\n    ],\n):\n    if isinstance(preprocessor, list):\n        for p in preprocessor:\n            assert (\n                p.is_fitted\n            ), \"All preprocessors must be fitted before using this class\"\n    else:\n        assert (\n            preprocessor.is_fitted\n        ), \"The preprocessor must be fitted before using this class\"\n\n    self.preprocessor = preprocessor\n</code></pre>"},{"location":"pytorch-widedeep/load_from_folder.html#pytorch_widedeep.load_from_folder.image.image_from_folder.ImageFromFolder","title":"ImageFromFolder","text":"<pre><code>ImageFromFolder(\n    directory=None,\n    preprocessor=None,\n    loader=default_loader,\n    extensions=None,\n    transforms=None,\n)\n</code></pre> <p>This class is used to load the image dataset from disk. It is inspired by the <code>ImageFolder</code> class at the <code>torchvision</code> library. Here, we have simply adapted to work within the context of a Wide and Deep multi-modal model.</p> <p>For examples, please, see the examples folder in the repo.</p> <p>Parameters:</p> <ul> <li> <code>directory</code>               (<code>Optional[Union[str, List[str]]]</code>, default:                   <code>None</code> )           \u2013            <p>the path to the directory where the images are located. If None, a preprocessor must be provided.</p> </li> <li> <code>preprocessor</code>               (<code>Optional[Union[ImagePreprocessor, List[ImagePreprocessor]]]</code>, default:                   <code>None</code> )           \u2013            <p>a fitted <code>ImagePreprocessor</code> object.</p> </li> <li> <code>loader</code>               (<code>Callable[[str], Any]</code>, default:                   <code>default_loader</code> )           \u2013            <p>a function to load a sample given its path.</p> </li> <li> <code>extensions</code>               (<code>Optional[Tuple[str, ...]]</code>, default:                   <code>None</code> )           \u2013            <p>a tuple with the allowed extensions. If None, IMG_EXTENSIONS will be used where IMG_EXTENSIONS =\".jpg\", \".jpeg\", \".png\", \".ppm\", \".bmp\", \".pgm\", \".tif\", \".tiff\", \".webp\"</p> </li> <li> <code>transforms</code>               (<code>Optional[Any]</code>, default:                   <code>None</code> )           \u2013            <p>a <code>torchvision.transforms</code> object. If None, this class will simply return an array representation of the PIL Image</p> </li> </ul> Source code in <code>pytorch_widedeep/load_from_folder/image/image_from_folder.py</code> <pre><code>def __init__(\n    self,\n    directory: Optional[Union[str, List[str]]] = None,\n    preprocessor: Optional[\n        Union[ImagePreprocessor, List[ImagePreprocessor]]\n    ] = None,\n    loader: Callable[[str], Any] = default_loader,\n    extensions: Optional[Tuple[str, ...]] = None,\n    transforms: Optional[Any] = None,\n) -&gt; None:\n    assert (\n        directory is not None or preprocessor is not None\n    ), \"Either a directory or an instance of ImagePreprocessor(s) must be provided\"\n\n    if directory is not None and preprocessor is not None:  # pragma: no cover\n        error_msg = (\n            \"If both 'directory' and 'preprocessor' are provided, the 'img_path' \"\n            \"attribute of the 'preprocessor' must be the same as the 'directory'\"\n        )\n        if isinstance(directory, list):\n            assert isinstance(preprocessor, list)\n            assert len(directory) == len(preprocessor)\n            for d, p in zip(directory, preprocessor):\n                assert d == p.img_path, error_msg\n        else:\n            assert isinstance(preprocessor, ImagePreprocessor)\n            assert directory == preprocessor.img_path, error_msg\n\n    if directory is not None:\n        self.directory = directory\n    else:\n        assert (\n            preprocessor is not None\n        ), \"Either a directory or an instance of ImagePreprocessor must be provided\"\n        if isinstance(preprocessor, list):\n            self.directory = [p.img_path for p in preprocessor]\n        else:\n            self.directory = preprocessor.img_path\n\n    self.preprocessor = preprocessor\n    self.loader = loader\n    self.extensions = extensions if extensions is not None else IMG_EXTENSIONS\n    self.transforms = transforms\n    if self.transforms:\n        self.transforms_names = [\n            tr.__class__.__name__ for tr in self.transforms.transforms\n        ]\n    else:\n        self.transforms_names = []\n\n        self.transpose = True\n</code></pre>"},{"location":"pytorch-widedeep/load_from_folder.html#pytorch_widedeep.load_from_folder.wd_dataset_from_folder.WideDeepDatasetFromFolder","title":"WideDeepDatasetFromFolder","text":"<pre><code>WideDeepDatasetFromFolder(\n    n_samples,\n    tab_from_folder=None,\n    wide_from_folder=None,\n    text_from_folder=None,\n    img_from_folder=None,\n    reference=None,\n)\n</code></pre> <p>               Bases: <code>Dataset</code></p> <p>This class is the Dataset counterpart of the <code>WideDeepDataset</code> class.</p> <p>Given a reference tabular dataset, with columns that indicate the path to the images and to the text files or the texts themselves, it will use the <code>[...]FromFolder</code> classes to load the data consistently from disk per batch.</p> <p>For examples, please, see the examples folder in the repo.</p> <p>Parameters:</p> <ul> <li> <code>n_samples</code>               (<code>int</code>)           \u2013            <p>Number of samples in the dataset</p> </li> <li> <code>tab_from_folder</code>               (<code>Optional[TabFromFolder]</code>, default:                   <code>None</code> )           \u2013            <p>Instance of the <code>TabFromFolder</code> class</p> </li> <li> <code>wide_from_folder</code>               (<code>Optional[WideFromFolder]</code>, default:                   <code>None</code> )           \u2013            <p>Instance of the <code>WideFromFolder</code> class</p> </li> <li> <code>text_from_folder</code>               (<code>Optional[TextFromFolder]</code>, default:                   <code>None</code> )           \u2013            <p>Instance of the <code>TextFromFolder</code> class</p> </li> <li> <code>img_from_folder</code>               (<code>Optional[ImageFromFolder]</code>, default:                   <code>None</code> )           \u2013            <p>Instance of the <code>ImageFromFolder</code> class</p> </li> <li> <code>reference</code>               (<code>Optional[Any]</code>, default:                   <code>None</code> )           \u2013            <p>If not None, the 'text_from_folder' and 'img_from_folder' objects will be retrieved from the reference class. This is useful when we want to use a <code>WideDeepDatasetFromFolder</code> class used for a train dataset as a reference for the validation and test datasets. In this case, the <code>text_from_folder</code> and <code>img_from_folder</code> objects will be the same for all three datasets, so there is no need to create a new instance for each dataset.</p> </li> </ul> Source code in <code>pytorch_widedeep/load_from_folder/wd_dataset_from_folder.py</code> <pre><code>def __init__(\n    self,\n    n_samples: int,\n    tab_from_folder: Optional[TabFromFolder] = None,\n    wide_from_folder: Optional[WideFromFolder] = None,\n    text_from_folder: Optional[TextFromFolder] = None,\n    img_from_folder: Optional[ImageFromFolder] = None,\n    reference: Optional[Any] = None,  # is Type[\"WideDeepDatasetFromFolder\"],\n):\n    super(WideDeepDatasetFromFolder, self).__init__()\n\n    if tab_from_folder is None and wide_from_folder is None:\n        raise ValueError(\n            \"Either 'tab_from_folder' or 'wide_from_folder' must be not None\"\n        )\n\n    if reference is not None:\n        assert (\n            img_from_folder is None and text_from_folder is None\n        ), \"If reference is not None, 'img_from_folder' and 'text_from_folder' left as None\"\n        self.text_from_folder, self.img_from_folder = self._get_from_reference(\n            reference\n        )\n    else:\n        assert (\n            text_from_folder is not None and img_from_folder is not None\n        ), \"If reference is None, 'img_from_folder' and 'text_from_folder' must be not None\"\n        self.text_from_folder = text_from_folder\n        self.img_from_folder = img_from_folder\n\n    self.n_samples = n_samples\n    self.tab_from_folder = tab_from_folder\n    self.wide_from_folder = wide_from_folder\n</code></pre>"},{"location":"pytorch-widedeep/losses.html","title":"Losses","text":"<p><code>pytorch-widedeep</code> accepts a number of losses and objectives that can be passed to the <code>Trainer</code> class via the parameter <code>objective</code> (see <code>pytorch-widedeep.training.Trainer</code>). For most cases the loss function that <code>pytorch-widedeep</code> will use internally is already implemented in Pytorch.</p> <p>In addition, <code>pytorch-widedeep</code> implements a series of  \"custom\" loss functions. These are described below for completion since, as mentioned before, they are used internally by the <code>Trainer</code>. Of course, onen could always use them on their own and can be imported as:</p> <p><code>from pytorch_widedeep.losses import FocalLoss</code></p> <p> NOTE:  Losses in this module expect the predictions  and ground truth to have the same dimensions for regression and binary  classification problems \\((N_{samples}, 1)\\). In the case of multiclass  classification problems the ground truth is expected to be a 1D tensor with  the corresponding classes. See Examples below</p>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.MSELoss","title":"MSELoss","text":"<pre><code>MSELoss()\n</code></pre> <p>               Bases: <code>Module</code></p> <p>Mean square error loss</p> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def __init__(self):\n    super().__init__()\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.MSELoss.forward","title":"forward","text":"<pre><code>forward(input, target)\n</code></pre> <p>Parameters:</p> <ul> <li> <code>input</code>               (<code>Tensor</code>)           \u2013            <p>Input tensor with predictions</p> </li> <li> <code>target</code>               (<code>Tensor</code>)           \u2013            <p>Target tensor with the actual values</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.losses import MSELoss\n&gt;&gt;&gt;\n&gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n&gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n&gt;&gt;&gt; loss = MSELoss()(input, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(\n    self,\n    input: Tensor,\n    target: Tensor,\n) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    input: Tensor\n        Input tensor with predictions\n    target: Tensor\n        Target tensor with the actual values\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.losses import MSELoss\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n    &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n    &gt;&gt;&gt; loss = MSELoss()(input, target)\n    \"\"\"\n    loss = (input - target) ** 2\n    return torch.mean(loss)\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.MSLELoss","title":"MSLELoss","text":"<pre><code>MSLELoss()\n</code></pre> <p>               Bases: <code>Module</code></p> <p>Mean square log error loss</p> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def __init__(self):\n    super().__init__()\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.MSLELoss.forward","title":"forward","text":"<pre><code>forward(input, target)\n</code></pre> <p>Parameters:</p> <ul> <li> <code>input</code>               (<code>Tensor</code>)           \u2013            <p>Input tensor with predictions (not probabilities)</p> </li> <li> <code>target</code>               (<code>Tensor</code>)           \u2013            <p>Target tensor with the actual classes</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.losses import MSLELoss\n&gt;&gt;&gt;\n&gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n&gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n&gt;&gt;&gt; loss = MSLELoss()(input, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(\n    self,\n    input: Tensor,\n    target: Tensor,\n) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    input: Tensor\n        Input tensor with predictions (not probabilities)\n    target: Tensor\n        Target tensor with the actual classes\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.losses import MSLELoss\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n    &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n    &gt;&gt;&gt; loss = MSLELoss()(input, target)\n    \"\"\"\n    assert (\n        input.min() &gt;= 0\n    ), \"\"\"All input values must be &gt;=0, if your model is predicting\n        values &lt;0 try to enforce positive values by activation function\n        on last layer with `trainer.enforce_positive_output=True`\"\"\"\n    assert target.min() &gt;= 0, \"All target values must be &gt;=0\"\n\n    loss = (torch.log(input + 1) - torch.log(target + 1)) ** 2\n    return torch.mean(loss)\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.RMSELoss","title":"RMSELoss","text":"<pre><code>RMSELoss()\n</code></pre> <p>               Bases: <code>Module</code></p> <p>Root mean square error loss</p> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def __init__(self):\n    super().__init__()\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.RMSELoss.forward","title":"forward","text":"<pre><code>forward(input, target)\n</code></pre> <p>Parameters:</p> <ul> <li> <code>input</code>               (<code>Tensor</code>)           \u2013            <p>Input tensor with predictions (not probabilities)</p> </li> <li> <code>target</code>               (<code>Tensor</code>)           \u2013            <p>Target tensor with the actual classes</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.losses import RMSELoss\n&gt;&gt;&gt;\n&gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n&gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n&gt;&gt;&gt; loss = RMSELoss()(input, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(self, input: Tensor, target: Tensor) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    input: Tensor\n        Input tensor with predictions (not probabilities)\n    target: Tensor\n        Target tensor with the actual classes\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.losses import RMSELoss\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n    &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n    &gt;&gt;&gt; loss = RMSELoss()(input, target)\n    \"\"\"\n    loss = (input - target) ** 2\n    return torch.sqrt(torch.mean(loss))\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.RMSLELoss","title":"RMSLELoss","text":"<pre><code>RMSLELoss()\n</code></pre> <p>               Bases: <code>Module</code></p> <p>Root mean square log error loss</p> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def __init__(self):\n    super().__init__()\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.RMSLELoss.forward","title":"forward","text":"<pre><code>forward(input, target)\n</code></pre> <p>Parameters:</p> <ul> <li> <code>input</code>               (<code>Tensor</code>)           \u2013            <p>Input tensor with predictions (not probabilities)</p> </li> <li> <code>target</code>               (<code>Tensor</code>)           \u2013            <p>Target tensor with the actual classes</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.losses import RMSLELoss\n&gt;&gt;&gt;\n&gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n&gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n&gt;&gt;&gt; loss = RMSLELoss()(input, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(self, input: Tensor, target: Tensor) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    input: Tensor\n        Input tensor with predictions (not probabilities)\n    target: Tensor\n        Target tensor with the actual classes\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.losses import RMSLELoss\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n    &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n    &gt;&gt;&gt; loss = RMSLELoss()(input, target)\n    \"\"\"\n    assert (\n        input.min() &gt;= 0\n    ), \"\"\"All input values must be &gt;=0, if your model is predicting\n        values &lt;0 try to enforce positive values by activation function\n        on last layer with `trainer.enforce_positive_output=True`\"\"\"\n    assert target.min() &gt;= 0, \"All target values must be &gt;=0\"\n\n    loss = (torch.log(input + 1) - torch.log(target + 1)) ** 2\n    return torch.sqrt(torch.mean(loss))\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.QuantileLoss","title":"QuantileLoss","text":"<pre><code>QuantileLoss(\n    quantiles=[0.02, 0.1, 0.25, 0.5, 0.75, 0.9, 0.98]\n)\n</code></pre> <p>               Bases: <code>Module</code></p> <p>Quantile loss defined as:</p> \\[ Loss = max(q \\times (y-y_{pred}), (1-q) \\times (y_{pred}-y)) \\] <p>All credits go to the implementation at pytorch-forecasting.</p> <p>Parameters:</p> <ul> <li> <code>quantiles</code>               (<code>List[float]</code>, default:                   <code>[0.02, 0.1, 0.25, 0.5, 0.75, 0.9, 0.98]</code> )           \u2013            <p>List of quantiles</p> </li> </ul> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def __init__(\n    self,\n    quantiles: List[float] = [0.02, 0.1, 0.25, 0.5, 0.75, 0.9, 0.98],\n):\n    super().__init__()\n    self.quantiles = quantiles\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.QuantileLoss.forward","title":"forward","text":"<pre><code>forward(input, target)\n</code></pre> <p>Parameters:</p> <ul> <li> <code>input</code>               (<code>Tensor</code>)           \u2013            <p>Input tensor with predictions</p> </li> <li> <code>target</code>               (<code>Tensor</code>)           \u2013            <p>Target tensor with the actual values</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt;\n&gt;&gt;&gt; from pytorch_widedeep.losses import QuantileLoss\n&gt;&gt;&gt;\n&gt;&gt;&gt; # REGRESSION\n&gt;&gt;&gt; target = torch.tensor([[0.6, 1.5]]).view(-1, 1)\n&gt;&gt;&gt; input = torch.tensor([[.1, .2,], [.4, .5]])\n&gt;&gt;&gt; qloss = QuantileLoss([0.25, 0.75])\n&gt;&gt;&gt; loss = qloss(input, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(self, input: Tensor, target: Tensor) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    input: Tensor\n        Input tensor with predictions\n    target: Tensor\n        Target tensor with the actual values\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; from pytorch_widedeep.losses import QuantileLoss\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; # REGRESSION\n    &gt;&gt;&gt; target = torch.tensor([[0.6, 1.5]]).view(-1, 1)\n    &gt;&gt;&gt; input = torch.tensor([[.1, .2,], [.4, .5]])\n    &gt;&gt;&gt; qloss = QuantileLoss([0.25, 0.75])\n    &gt;&gt;&gt; loss = qloss(input, target)\n    \"\"\"\n\n    assert input.shape == torch.Size([target.shape[0], len(self.quantiles)]), (\n        \"The input and target have inconsistent shape. The dimension of the prediction \"\n        \"of the model that is using QuantileLoss must be equal to number of quantiles, \"\n        f\"i.e. {len(self.quantiles)}.\"\n    )\n    target = target.view(-1, 1).float()\n    losses = []\n    for i, q in enumerate(self.quantiles):\n        errors = target - input[..., i]\n        losses.append(torch.max((q - 1) * errors, q * errors).unsqueeze(-1))\n\n    loss = torch.cat(losses, dim=2)\n\n    return torch.mean(loss)\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.FocalLoss","title":"FocalLoss","text":"<pre><code>FocalLoss(alpha=0.25, gamma=1.0)\n</code></pre> <p>               Bases: <code>Module</code></p> <p>Implementation of the Focal loss for both binary and multiclass classification:</p> \\[ FL(p_t) = \\alpha (1 - p_t)^{\\gamma} log(p_t) \\] <p>where, for a case of a binary classification problem</p> \\[ \\begin{equation} p_t= \\begin{cases}p, &amp; \\text{if $y=1$}.\\\\1-p, &amp; \\text{otherwise}. \\end{cases} \\end{equation} \\] <p>Parameters:</p> <ul> <li> <code>alpha</code>               (<code>float</code>, default:                   <code>0.25</code> )           \u2013            <p>Focal Loss <code>alpha</code> parameter</p> </li> <li> <code>gamma</code>               (<code>float</code>, default:                   <code>1.0</code> )           \u2013            <p>Focal Loss <code>gamma</code> parameter</p> </li> </ul> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def __init__(self, alpha: float = 0.25, gamma: float = 1.0):\n    super().__init__()\n    self.alpha = alpha\n    self.gamma = gamma\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.FocalLoss.forward","title":"forward","text":"<pre><code>forward(input, target)\n</code></pre> <p>Parameters:</p> <ul> <li> <code>input</code>               (<code>Tensor</code>)           \u2013            <p>Input tensor with predictions (not probabilities)</p> </li> <li> <code>target</code>               (<code>Tensor</code>)           \u2013            <p>Target tensor with the actual classes</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt;\n&gt;&gt;&gt; from pytorch_widedeep.losses import FocalLoss\n&gt;&gt;&gt;\n&gt;&gt;&gt; # BINARY\n&gt;&gt;&gt; target = torch.tensor([0, 1, 0, 1]).view(-1, 1)\n&gt;&gt;&gt; input = torch.tensor([[0.6, 0.7, 0.3, 0.8]]).t()\n&gt;&gt;&gt; loss = FocalLoss()(input, target)\n&gt;&gt;&gt;\n&gt;&gt;&gt; # MULTICLASS\n&gt;&gt;&gt; target = torch.tensor([1, 0, 2]).view(-1, 1)\n&gt;&gt;&gt; input = torch.tensor([[0.2, 0.5, 0.3], [0.8, 0.1, 0.1], [0.7, 0.2, 0.1]])\n&gt;&gt;&gt; loss = FocalLoss()(input, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(self, input: Tensor, target: Tensor) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    input: Tensor\n        Input tensor with predictions (not probabilities)\n    target: Tensor\n        Target tensor with the actual classes\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; from pytorch_widedeep.losses import FocalLoss\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; # BINARY\n    &gt;&gt;&gt; target = torch.tensor([0, 1, 0, 1]).view(-1, 1)\n    &gt;&gt;&gt; input = torch.tensor([[0.6, 0.7, 0.3, 0.8]]).t()\n    &gt;&gt;&gt; loss = FocalLoss()(input, target)\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; # MULTICLASS\n    &gt;&gt;&gt; target = torch.tensor([1, 0, 2]).view(-1, 1)\n    &gt;&gt;&gt; input = torch.tensor([[0.2, 0.5, 0.3], [0.8, 0.1, 0.1], [0.7, 0.2, 0.1]])\n    &gt;&gt;&gt; loss = FocalLoss()(input, target)\n    \"\"\"\n    input_prob = torch.sigmoid(input)\n    if input.size(1) == 1:\n        input_prob = torch.cat([1 - input_prob, input_prob], axis=1)  # type: ignore\n        num_class = 2\n    else:\n        num_class = input_prob.size(1)\n    binary_target = torch.eye(num_class)[target.squeeze().cpu().long()]\n    binary_target = binary_target.to(input.device)\n    binary_target = binary_target.contiguous()\n    weight = self._get_weight(input_prob, binary_target)\n\n    return F.binary_cross_entropy(\n        input_prob, binary_target, weight, reduction=\"mean\"\n    )\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.BayesianSELoss","title":"BayesianSELoss","text":"<pre><code>BayesianSELoss()\n</code></pre> <p>               Bases: <code>Module</code></p> <p>Squared Loss (log Gaussian) for the case of a regression as specified in the original publication Weight Uncertainty in Neural Networks.</p> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def __init__(self):\n    super().__init__()\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.BayesianSELoss.forward","title":"forward","text":"<pre><code>forward(input, target)\n</code></pre> <p>Parameters:</p> <ul> <li> <code>input</code>               (<code>Tensor</code>)           \u2013            <p>Input tensor with predictions (not probabilities)</p> </li> <li> <code>target</code>               (<code>Tensor</code>)           \u2013            <p>Target tensor with the actual classes</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.losses import BayesianSELoss\n&gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n&gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n&gt;&gt;&gt; loss = BayesianSELoss()(input, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(self, input: Tensor, target: Tensor) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    input: Tensor\n        Input tensor with predictions (not probabilities)\n    target: Tensor\n        Target tensor with the actual classes\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.losses import BayesianSELoss\n    &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n    &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n    &gt;&gt;&gt; loss = BayesianSELoss()(input, target)\n    \"\"\"\n    return (0.5 * (input - target) ** 2).sum()\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.TweedieLoss","title":"TweedieLoss","text":"<pre><code>TweedieLoss()\n</code></pre> <p>               Bases: <code>Module</code></p> <p>Tweedie loss for extremely unbalanced zero-inflated data</p> <p>All credits go to Wenbo Shi. See this post and the original publication for details.</p> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def __init__(self):\n    super().__init__()\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.TweedieLoss.forward","title":"forward","text":"<pre><code>forward(input, target, p=1.5)\n</code></pre> <p>Parameters:</p> <ul> <li> <code>input</code>               (<code>Tensor</code>)           \u2013            <p>Input tensor with predictions</p> </li> <li> <code>target</code>               (<code>Tensor</code>)           \u2013            <p>Target tensor with the actual values</p> </li> <li> <code>p</code>               (<code>float</code>, default:                   <code>1.5</code> )           \u2013            <p>the power to be used to compute the loss. See the original publication for details</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.losses import TweedieLoss\n&gt;&gt;&gt;\n&gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n&gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n&gt;&gt;&gt; loss = TweedieLoss()(input, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(\n    self,\n    input: Tensor,\n    target: Tensor,\n    p: float = 1.5,\n) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    input: Tensor\n        Input tensor with predictions\n    target: Tensor\n        Target tensor with the actual values\n    p: float, default = 1.5\n        the power to be used to compute the loss. See the original\n        publication for details\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.losses import TweedieLoss\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n    &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n    &gt;&gt;&gt; loss = TweedieLoss()(input, target)\n    \"\"\"\n\n    assert (\n        input.min() &gt; 0\n    ), \"\"\"All input values must be &gt;=0, if your model is predicting\n        values &lt;0 try to enforce positive values by activation function\n        on last layer with `trainer.enforce_positive_output=True`\"\"\"\n    assert target.min() &gt;= 0, \"All target values must be &gt;=0\"\n    loss = -target * torch.pow(input, 1 - p) / (1 - p) + torch.pow(input, 2 - p) / (\n        2 - p\n    )\n    return torch.mean(loss)\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.ZILNLoss","title":"ZILNLoss","text":"<pre><code>ZILNLoss()\n</code></pre> <p>               Bases: <code>Module</code></p> <p>Adjusted implementation of the Zero Inflated LogNormal Loss</p> <p>See A Deep Probabilistic Model for Customer Lifetime Value Prediction and the corresponding code.</p> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def __init__(self):\n    super().__init__()\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.ZILNLoss.forward","title":"forward","text":"<pre><code>forward(input, target)\n</code></pre> <p>Parameters:</p> <ul> <li> <code>input</code>               (<code>Tensor</code>)           \u2013            <p>Input tensor with predictions with spape (N,3), where N is the batch size</p> </li> <li> <code>target</code>               (<code>Tensor</code>)           \u2013            <p>Target tensor with the actual target values</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.losses import ZILNLoss\n&gt;&gt;&gt;\n&gt;&gt;&gt; target = torch.tensor([[0., 1.5]]).view(-1, 1)\n&gt;&gt;&gt; input = torch.tensor([[.1, .2, .3], [.4, .5, .6]])\n&gt;&gt;&gt; loss = ZILNLoss()(input, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(self, input: Tensor, target: Tensor) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    input: Tensor\n        Input tensor with predictions with spape (N,3), where N is the batch size\n    target: Tensor\n        Target tensor with the actual target values\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.losses import ZILNLoss\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; target = torch.tensor([[0., 1.5]]).view(-1, 1)\n    &gt;&gt;&gt; input = torch.tensor([[.1, .2, .3], [.4, .5, .6]])\n    &gt;&gt;&gt; loss = ZILNLoss()(input, target)\n    \"\"\"\n    positive = target &gt; 0\n    positive = positive.float()\n\n    assert input.shape == torch.Size([target.shape[0], 3]), (\n        \"Wrong shape of the 'input' tensor. The pred_dim of the \"\n        \"model that is using ZILNLoss must be equal to 3.\"\n    )\n\n    positive_input = input[..., :1]\n\n    classification_loss = F.binary_cross_entropy_with_logits(\n        positive_input, positive, reduction=\"none\"\n    ).flatten()\n\n    loc = input[..., 1:2]\n\n    # when using max the two input tensors (input and other) have to be of\n    # the same type\n    max_input = F.softplus(input[..., 2:])\n    max_other = self.get_eps(max_input)\n    scale = torch.max(max_input, max_other)\n    safe_labels = positive * target + (1 - positive) * torch.ones_like(target)\n\n    regression_loss = -torch.mean(\n        positive\n        * torch.distributions.log_normal.LogNormal(loc=loc, scale=scale).log_prob(\n            safe_labels\n        ),\n        dim=-1,\n    )\n\n    return torch.mean(classification_loss + regression_loss)\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.L1Loss","title":"L1Loss","text":"<pre><code>L1Loss()\n</code></pre> <p>               Bases: <code>Module</code></p> <p>L1 loss</p> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def __init__(self):\n    super().__init__()\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.L1Loss.forward","title":"forward","text":"<pre><code>forward(input, target)\n</code></pre> <p>Parameters:</p> <ul> <li> <code>input</code>               (<code>Tensor</code>)           \u2013            <p>Input tensor with predictions</p> </li> <li> <code>target</code>               (<code>Tensor</code>)           \u2013            <p>Target tensor with the actual values</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt;\n&gt;&gt;&gt; from pytorch_widedeep.losses import L1Loss\n&gt;&gt;&gt;\n&gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n&gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n&gt;&gt;&gt; loss = L1Loss()(input, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(self, input: Tensor, target: Tensor) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    input: Tensor\n        Input tensor with predictions\n    target: Tensor\n        Target tensor with the actual values\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; from pytorch_widedeep.losses import L1Loss\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n    &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n    &gt;&gt;&gt; loss = L1Loss()(input, target)\n    \"\"\"\n    loss = F.l1_loss(input, target, reduction=\"none\")\n    return torch.mean(loss)\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.FocalR_L1Loss","title":"FocalR_L1Loss","text":"<pre><code>FocalR_L1Loss(beta=0.2, gamma=1.0, activation_fn='sigmoid')\n</code></pre> <p>               Bases: <code>Module</code></p> <p>Focal-R L1 loss</p> <p>Based on Delving into Deep Imbalanced Regression.</p> <p>Parameters:</p> <ul> <li> <code>beta</code>               (<code>float</code>, default:                   <code>0.2</code> )           \u2013            <p>Focal Loss <code>beta</code> parameter in their implementation</p> </li> <li> <code>gamma</code>               (<code>float</code>, default:                   <code>1.0</code> )           \u2013            <p>Focal Loss <code>gamma</code> parameter</p> </li> <li> <code>activation_fn</code>               (<code>Literal[sigmoid, tanh]</code>, default:                   <code>'sigmoid'</code> )           \u2013            <p>Activation function to be used during the computation of the loss. Possible values are 'sigmoid' and 'tanh'. See the original publication for details.</p> </li> </ul> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def __init__(\n    self,\n    beta: float = 0.2,\n    gamma: float = 1.0,\n    activation_fn: Literal[\"sigmoid\", \"tanh\"] = \"sigmoid\",\n):\n    super().__init__()\n    self.beta = beta\n    self.gamma = gamma\n    self.activation_fn = activation_fn\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.FocalR_L1Loss.forward","title":"forward","text":"<pre><code>forward(input, target)\n</code></pre> <p>Parameters:</p> <ul> <li> <code>input</code>               (<code>Tensor</code>)           \u2013            <p>Input tensor with predictions (not probabilities)</p> </li> <li> <code>target</code>               (<code>Tensor</code>)           \u2013            <p>Target tensor with the actual classes</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt;\n&gt;&gt;&gt; from pytorch_widedeep.losses import FocalR_L1Loss\n&gt;&gt;&gt;\n&gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n&gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n&gt;&gt;&gt; loss = FocalR_L1Loss()(input, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(\n    self,\n    input: Tensor,\n    target: Tensor,\n) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    input: Tensor\n        Input tensor with predictions (not probabilities)\n    target: Tensor\n        Target tensor with the actual classes\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; from pytorch_widedeep.losses import FocalR_L1Loss\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n    &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n    &gt;&gt;&gt; loss = FocalR_L1Loss()(input, target)\n    \"\"\"\n    loss = F.l1_loss(input, target, reduction=\"none\")\n    if self.activation_fn == \"tanh\":\n        loss *= (torch.tanh(self.beta * torch.abs(input - target))) ** self.gamma\n    elif self.activation_fn == \"sigmoid\":\n        loss *= (\n            2 * torch.sigmoid(self.beta * torch.abs(input - target)) - 1\n        ) ** self.gamma\n    else:\n        ValueError(\n            \"Incorrect activation function value - must be in ['sigmoid', 'tanh']\"\n        )\n    return torch.mean(loss)\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.FocalR_MSELoss","title":"FocalR_MSELoss","text":"<pre><code>FocalR_MSELoss(\n    beta=0.2, gamma=1.0, activation_fn=\"sigmoid\"\n)\n</code></pre> <p>               Bases: <code>Module</code></p> <p>Focal-R MSE loss</p> <p>Based on Delving into Deep Imbalanced Regression.</p> <p>Parameters:</p> <ul> <li> <code>beta</code>               (<code>float</code>, default:                   <code>0.2</code> )           \u2013            <p>Focal Loss <code>beta</code> parameter in their implementation</p> </li> <li> <code>gamma</code>               (<code>float</code>, default:                   <code>1.0</code> )           \u2013            <p>Focal Loss <code>gamma</code> parameter</p> </li> <li> <code>activation_fn</code>               (<code>Literal[sigmoid, tanh]</code>, default:                   <code>'sigmoid'</code> )           \u2013            <p>Activation function to be used during the computation of the loss. Possible values are 'sigmoid' and 'tanh'. See the original publication for details.</p> </li> </ul> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def __init__(\n    self,\n    beta: float = 0.2,\n    gamma: float = 1.0,\n    activation_fn: Literal[\"sigmoid\", \"tanh\"] = \"sigmoid\",\n):\n    super().__init__()\n    self.beta = beta\n    self.gamma = gamma\n    self.activation_fn = activation_fn\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.FocalR_MSELoss.forward","title":"forward","text":"<pre><code>forward(input, target)\n</code></pre> <p>Parameters:</p> <ul> <li> <code>input</code>               (<code>Tensor</code>)           \u2013            <p>Input tensor with predictions (not probabilities)</p> </li> <li> <code>target</code>               (<code>Tensor</code>)           \u2013            <p>Target tensor with the actual classes</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt;\n&gt;&gt;&gt; from pytorch_widedeep.losses import FocalR_MSELoss\n&gt;&gt;&gt;\n&gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n&gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n&gt;&gt;&gt; loss = FocalR_MSELoss()(input, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(\n    self,\n    input: Tensor,\n    target: Tensor,\n) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    input: Tensor\n        Input tensor with predictions (not probabilities)\n    target: Tensor\n        Target tensor with the actual classes\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; from pytorch_widedeep.losses import FocalR_MSELoss\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n    &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n    &gt;&gt;&gt; loss = FocalR_MSELoss()(input, target)\n    \"\"\"\n    loss = (input - target) ** 2\n    if self.activation_fn == \"tanh\":\n        loss *= (torch.tanh(self.beta * torch.abs(input - target))) ** self.gamma\n    elif self.activation_fn == \"sigmoid\":\n        loss *= (\n            2 * torch.sigmoid(self.beta * torch.abs((input - target) ** 2)) - 1\n        ) ** self.gamma\n    else:\n        ValueError(\n            \"Incorrect activation function value - must be in ['sigmoid', 'tanh']\"\n        )\n    return torch.mean(loss)\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.FocalR_RMSELoss","title":"FocalR_RMSELoss","text":"<pre><code>FocalR_RMSELoss(\n    beta=0.2, gamma=1.0, activation_fn=\"sigmoid\"\n)\n</code></pre> <p>               Bases: <code>Module</code></p> <p>Focal-R RMSE loss</p> <p>Based on Delving into Deep Imbalanced Regression.</p> <p>Parameters:</p> <ul> <li> <code>beta</code>               (<code>float</code>, default:                   <code>0.2</code> )           \u2013            <p>Focal Loss <code>beta</code> parameter in their implementation</p> </li> <li> <code>gamma</code>               (<code>float</code>, default:                   <code>1.0</code> )           \u2013            <p>Focal Loss <code>gamma</code> parameter</p> </li> <li> <code>activation_fn</code>               (<code>Literal[sigmoid, tanh]</code>, default:                   <code>'sigmoid'</code> )           \u2013            <p>Activation function to be used during the computation of the loss. Possible values are 'sigmoid' and 'tanh'. See the original publication for details.</p> </li> </ul> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def __init__(\n    self,\n    beta: float = 0.2,\n    gamma: float = 1.0,\n    activation_fn: Literal[\"sigmoid\", \"tanh\"] = \"sigmoid\",\n):\n    super().__init__()\n    self.beta = beta\n    self.gamma = gamma\n    self.activation_fn = activation_fn\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.FocalR_RMSELoss.forward","title":"forward","text":"<pre><code>forward(input, target)\n</code></pre> <p>Parameters:</p> <ul> <li> <code>input</code>               (<code>Tensor</code>)           \u2013            <p>Input tensor with predictions (not probabilities)</p> </li> <li> <code>target</code>               (<code>Tensor</code>)           \u2013            <p>Target tensor with the actual classes</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt;\n&gt;&gt;&gt; from pytorch_widedeep.losses import FocalR_RMSELoss\n&gt;&gt;&gt;\n&gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n&gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n&gt;&gt;&gt; loss = FocalR_RMSELoss()(input, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(\n    self,\n    input: Tensor,\n    target: Tensor,\n) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    input: Tensor\n        Input tensor with predictions (not probabilities)\n    target: Tensor\n        Target tensor with the actual classes\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; from pytorch_widedeep.losses import FocalR_RMSELoss\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n    &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n    &gt;&gt;&gt; loss = FocalR_RMSELoss()(input, target)\n    \"\"\"\n    loss = (input - target) ** 2\n    if self.activation_fn == \"tanh\":\n        loss *= (torch.tanh(self.beta * torch.abs(input - target))) ** self.gamma\n    elif self.activation_fn == \"sigmoid\":\n        loss *= (\n            2 * torch.sigmoid(self.beta * torch.abs((input - target) ** 2)) - 1\n        ) ** self.gamma\n    else:\n        ValueError(\n            \"Incorrect activation function value - must be in ['sigmoid', 'tanh']\"\n        )\n    return torch.sqrt(torch.mean(loss))\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.HuberLoss","title":"HuberLoss","text":"<pre><code>HuberLoss(beta=0.2)\n</code></pre> <p>               Bases: <code>Module</code></p> <p>Hubbler Loss</p> <p>Based on Delving into Deep Imbalanced Regression.</p> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def __init__(self, beta: float = 0.2):\n    super().__init__()\n    self.beta = beta\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.HuberLoss.forward","title":"forward","text":"<pre><code>forward(input, target)\n</code></pre> <p>Parameters:</p> <ul> <li> <code>input</code>               (<code>Tensor</code>)           \u2013            <p>Input tensor with predictions (not probabilities)</p> </li> <li> <code>target</code>               (<code>Tensor</code>)           \u2013            <p>Target tensor with the actual classes</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt;\n&gt;&gt;&gt; from pytorch_widedeep.losses import HuberLoss\n&gt;&gt;&gt;\n&gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n&gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n&gt;&gt;&gt; loss = HuberLoss()(input, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(\n    self,\n    input: Tensor,\n    target: Tensor,\n) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    input: Tensor\n        Input tensor with predictions (not probabilities)\n    target: Tensor\n        Target tensor with the actual classes\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; from pytorch_widedeep.losses import HuberLoss\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n    &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n    &gt;&gt;&gt; loss = HuberLoss()(input, target)\n    \"\"\"\n    l1_loss = torch.abs(input - target)\n    cond = l1_loss &lt; self.beta\n    loss = torch.where(\n        cond, 0.5 * l1_loss**2 / self.beta, l1_loss - 0.5 * self.beta\n    )\n    return torch.mean(loss)\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.InfoNCELoss","title":"InfoNCELoss","text":"<pre><code>InfoNCELoss(temperature=0.1, reduction='mean')\n</code></pre> <p>               Bases: <code>Module</code></p> <p>InfoNCE Loss. Loss applied during the Contrastive Denoising Self Supervised Pre-training routine available in this library</p> <p> NOTE: This loss is in principle not exposed to  the user, as it is used internally in the library, but it is included  here for completion.</p> <p>See SAINT: Improved Neural Networks for Tabular Data via Row Attention and Contrastive Pre-Training and references therein</p> <p>Partially inspired by the code in this repo</p> <p>Parameters:</p> <ul> <li> <code>temperature</code>               (<code>float</code>, default:                   <code>0.1</code> )           \u2013            <p>The logits are divided by the temperature before computing the loss value</p> </li> <li> <code>reduction</code>               (<code>str</code>, default:                   <code>'mean'</code> )           \u2013            <p>Loss reduction method</p> </li> </ul> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def __init__(self, temperature: float = 0.1, reduction: str = \"mean\"):\n    super(InfoNCELoss, self).__init__()\n\n    self.temperature = temperature\n    self.reduction = reduction\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.InfoNCELoss.forward","title":"forward","text":"<pre><code>forward(g_projs)\n</code></pre> <p>Parameters:</p> <ul> <li> <code>g_projs</code>               (<code>Tuple[Tensor, Tensor]</code>)           \u2013            <p>Tuple with the two tensors corresponding to the output of the two projection heads, as described 'SAINT: Improved Neural Networks for Tabular Data via Row Attention and Contrastive Pre-Training'.</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.losses import InfoNCELoss\n&gt;&gt;&gt; g_projs = (torch.rand(3, 5, 16), torch.rand(3, 5, 16))\n&gt;&gt;&gt; loss = InfoNCELoss()\n&gt;&gt;&gt; res = loss(g_projs)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(self, g_projs: Tuple[Tensor, Tensor]) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    g_projs: Tuple\n        Tuple with the two tensors corresponding to the output of the two\n        projection heads, as described 'SAINT: Improved Neural Networks\n        for Tabular Data via Row Attention and Contrastive Pre-Training'.\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.losses import InfoNCELoss\n    &gt;&gt;&gt; g_projs = (torch.rand(3, 5, 16), torch.rand(3, 5, 16))\n    &gt;&gt;&gt; loss = InfoNCELoss()\n    &gt;&gt;&gt; res = loss(g_projs)\n    \"\"\"\n    z, z_ = g_projs[0], g_projs[1]\n\n    norm_z = F.normalize(z, dim=-1).flatten(1)\n    norm_z_ = F.normalize(z_, dim=-1).flatten(1)\n\n    logits = (norm_z @ norm_z_.t()) / self.temperature\n    logits_ = (norm_z_ @ norm_z.t()) / self.temperature\n\n    # the target/labels are the entries on the diagonal\n    target = torch.arange(len(norm_z), device=norm_z.device)\n\n    loss = F.cross_entropy(logits, target, reduction=self.reduction)\n    loss_ = F.cross_entropy(logits_, target, reduction=self.reduction)\n\n    return (loss + loss_) / 2.0\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.DenoisingLoss","title":"DenoisingLoss","text":"<pre><code>DenoisingLoss(\n    lambda_cat=1.0, lambda_cont=1.0, reduction=\"mean\"\n)\n</code></pre> <p>               Bases: <code>Module</code></p> <p>Denoising Loss. Loss applied during the Contrastive Denoising Self Supervised Pre-training routine available in this library</p> <p> NOTE: This loss is in principle not exposed to  the user, as it is used internally in the library, but it is included  here for completion.</p> <p>See SAINT: Improved Neural Networks for Tabular Data via Row Attention and Contrastive Pre-Training and references therein</p> <p>Parameters:</p> <ul> <li> <code>lambda_cat</code>               (<code>float</code>, default:                   <code>1.0</code> )           \u2013            <p>Multiplicative factor that will be applied to loss associated to the categorical features</p> </li> <li> <code>lambda_cont</code>               (<code>float</code>, default:                   <code>1.0</code> )           \u2013            <p>Multiplicative factor that will be applied to loss associated to the continuous features</p> </li> <li> <code>reduction</code>               (<code>str</code>, default:                   <code>'mean'</code> )           \u2013            <p>Loss reduction method</p> </li> </ul> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def __init__(\n    self, lambda_cat: float = 1.0, lambda_cont: float = 1.0, reduction: str = \"mean\"\n):\n    super(DenoisingLoss, self).__init__()\n\n    self.lambda_cat = lambda_cat\n    self.lambda_cont = lambda_cont\n    self.reduction = reduction\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.DenoisingLoss.forward","title":"forward","text":"<pre><code>forward(x_cat_and_cat_, x_cont_and_cont_)\n</code></pre> <p>Parameters:</p> <ul> <li> <code>x_cat_and_cat_</code>               (<code>Optional[Union[List[Tuple[Tensor, Tensor]], Tuple[Tensor, Tensor]]]</code>)           \u2013            <p>Tuple of tensors containing the raw input features and their encodings, referred in the SAINT paper as \\(x\\) and \\(x''\\) respectively. If one denoising MLP is used per categorical feature <code>x_cat_and_cat_</code> will be a list of tuples, one per categorical feature</p> </li> <li> <code>x_cont_and_cont_</code>               (<code>Optional[Union[List[Tuple[Tensor, Tensor]], Tuple[Tensor, Tensor]]]</code>)           \u2013            <p>same as <code>x_cat_and_cat_</code> but for continuous columns</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.losses import DenoisingLoss\n&gt;&gt;&gt; x_cat_and_cat_ = (torch.empty(3).random_(3).long(), torch.randn(3, 3))\n&gt;&gt;&gt; x_cont_and_cont_ = (torch.randn(3, 1), torch.randn(3, 1))\n&gt;&gt;&gt; loss = DenoisingLoss()\n&gt;&gt;&gt; res = loss(x_cat_and_cat_, x_cont_and_cont_)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(\n    self,\n    x_cat_and_cat_: Optional[\n        Union[List[Tuple[Tensor, Tensor]], Tuple[Tensor, Tensor]]\n    ],\n    x_cont_and_cont_: Optional[\n        Union[List[Tuple[Tensor, Tensor]], Tuple[Tensor, Tensor]]\n    ],\n) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    x_cat_and_cat_: tuple of Tensors or lists of tuples\n        Tuple of tensors containing the raw input features and their\n        encodings, referred in the SAINT paper as $x$ and $x''$\n        respectively. If one denoising MLP is used per categorical\n        feature `x_cat_and_cat_` will be a list of tuples, one per\n        categorical feature\n    x_cont_and_cont_: tuple of Tensors or lists of tuples\n        same as `x_cat_and_cat_` but for continuous columns\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.losses import DenoisingLoss\n    &gt;&gt;&gt; x_cat_and_cat_ = (torch.empty(3).random_(3).long(), torch.randn(3, 3))\n    &gt;&gt;&gt; x_cont_and_cont_ = (torch.randn(3, 1), torch.randn(3, 1))\n    &gt;&gt;&gt; loss = DenoisingLoss()\n    &gt;&gt;&gt; res = loss(x_cat_and_cat_, x_cont_and_cont_)\n    \"\"\"\n\n    loss_cat = (\n        self._compute_cat_loss(x_cat_and_cat_)\n        if x_cat_and_cat_ is not None\n        else torch.tensor(0.0)\n    )\n    loss_cont = (\n        self._compute_cont_loss(x_cont_and_cont_)\n        if x_cont_and_cont_ is not None\n        else torch.tensor(0.0)\n    )\n\n    return self.lambda_cat * loss_cat + self.lambda_cont * loss_cont\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.EncoderDecoderLoss","title":"EncoderDecoderLoss","text":"<pre><code>EncoderDecoderLoss(eps=1e-09)\n</code></pre> <p>               Bases: <code>Module</code></p> <p>'Standard' Encoder Decoder Loss. Loss applied during the Endoder-Decoder  Self-Supervised Pre-Training routine available in this library</p> <p> NOTE: This loss is in principle not exposed to  the user, as it is used internally in the library, but it is included  here for completion.</p> <p>The implementation of this lost is based on that at the tabnet repo, which is in itself an adaptation of that in the original paper TabNet: Attentive Interpretable Tabular Learning.</p> <p>Parameters:</p> <ul> <li> <code>eps</code>               (<code>float</code>, default:                   <code>1e-09</code> )           \u2013            <p>Simply a small number to avoid dividing by zero</p> </li> </ul> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def __init__(self, eps: float = 1e-9):\n    super(EncoderDecoderLoss, self).__init__()\n    self.eps = eps\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.EncoderDecoderLoss.forward","title":"forward","text":"<pre><code>forward(x_true, x_pred, mask)\n</code></pre> <p>Parameters:</p> <ul> <li> <code>x_true</code>               (<code>Tensor</code>)           \u2013            <p>Embeddings of the input data</p> </li> <li> <code>x_pred</code>               (<code>Tensor</code>)           \u2013            <p>Reconstructed embeddings</p> </li> <li> <code>mask</code>               (<code>Tensor</code>)           \u2013            <p>Mask with 1s indicated that the reconstruction, and therefore the loss, is based on those features.</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.losses import EncoderDecoderLoss\n&gt;&gt;&gt; x_true = torch.rand(3, 3)\n&gt;&gt;&gt; x_pred = torch.rand(3, 3)\n&gt;&gt;&gt; mask = torch.empty(3, 3).random_(2)\n&gt;&gt;&gt; loss = EncoderDecoderLoss()\n&gt;&gt;&gt; res = loss(x_true, x_pred, mask)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(self, x_true: Tensor, x_pred: Tensor, mask: Tensor) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    x_true: Tensor\n        Embeddings of the input data\n    x_pred: Tensor\n        Reconstructed embeddings\n    mask: Tensor\n        Mask with 1s indicated that the reconstruction, and therefore the\n        loss, is based on those features.\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.losses import EncoderDecoderLoss\n    &gt;&gt;&gt; x_true = torch.rand(3, 3)\n    &gt;&gt;&gt; x_pred = torch.rand(3, 3)\n    &gt;&gt;&gt; mask = torch.empty(3, 3).random_(2)\n    &gt;&gt;&gt; loss = EncoderDecoderLoss()\n    &gt;&gt;&gt; res = loss(x_true, x_pred, mask)\n    \"\"\"\n\n    errors = x_pred - x_true\n\n    reconstruction_errors = torch.mul(errors, mask) ** 2\n\n    x_true_means = torch.mean(x_true, dim=0)\n    x_true_means[x_true_means == 0] = 1\n\n    x_true_stds = torch.std(x_true, dim=0) ** 2\n    x_true_stds[x_true_stds == 0] = x_true_means[x_true_stds == 0]\n\n    features_loss = torch.matmul(reconstruction_errors, 1 / x_true_stds)\n    nb_reconstructed_variables = torch.sum(mask, dim=1)\n    features_loss_norm = features_loss / (nb_reconstructed_variables + self.eps)\n\n    loss = torch.mean(features_loss_norm)\n\n    return loss\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses_multitarget.MultiTargetRegressionLoss","title":"MultiTargetRegressionLoss","text":"<pre><code>MultiTargetRegressionLoss(weights=None, reduction='mean')\n</code></pre> <p>               Bases: <code>Module</code></p> <p>This class is a wrapper around the Pytorch MSELoss. It allows for multi-target regression problems. The user can provide a list of weights to apply to each target. The loss can be either the sum or the mean of the individual losses</p> <p>Parameters:</p> <ul> <li> <code>weights</code>               (<code>Optional[List[float]]</code>, default:                   <code>None</code> )           \u2013            <p>List of weights to apply to the loss associated to each target. The length of the list must match the number of targets. Alias: 'target_weights'</p> </li> <li> <code>reduction</code>               (<code>Literal[mean, sum]</code>, default:                   <code>'mean'</code> )           \u2013            <p>Specifies the reduction to apply to the loss associated to each target: 'mean' | 'sum'. Note that this is NOT the same as the reduction in the MSELoss. This reduction is applied after the loss for each target has been computed. Alias: 'target_reduction'</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.losses_multitarget import MultiTargetRegressionLoss\n&gt;&gt;&gt; input = torch.randn(3, 2)\n&gt;&gt;&gt; target = torch.randn(3, 2)\n&gt;&gt;&gt; loss = MultiTargetRegressionLoss(weights=[0.5, 0.5], reduction=\"mean\")\n&gt;&gt;&gt; output = loss(input, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses_multitarget.py</code> <pre><code>@alias(\"reduction\", [\"target_reduction\"])\n@alias(\"weights\", [\"target_weights\"])\ndef __init__(\n    self,\n    weights: Optional[List[float]] = None,\n    reduction: Literal[\"mean\", \"sum\"] = \"mean\",\n):\n    super(MultiTargetRegressionLoss, self).__init__()\n\n    self.weights = weights\n    self.reduction = reduction\n\n    if self.reduction not in [\"mean\", \"sum\"]:\n        raise ValueError(\"reduction must be either 'mean' or 'sum'\")\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses_multitarget.MultiTargetClassificationLoss","title":"MultiTargetClassificationLoss","text":"<pre><code>MultiTargetClassificationLoss(\n    binary_config=None,\n    multiclass_config=None,\n    weights=None,\n    reduction=\"mean\",\n    binary_trick=False,\n)\n</code></pre> <p>               Bases: <code>Module</code></p> <p>This class is a wrapper around the Pytorch binary_cross_entropy_with_logits and cross_entropy losses. It allows for multi-target classification problems. The user can provide a list of weights to apply to each target. The loss can be either the sum or the mean of the individual losses</p> <p>Parameters:</p> <ul> <li> <code>binary_config</code>               (<code>Optional[List[Union[int, Tuple[int, float]]]]</code>, default:                   <code>None</code> )           \u2013            <p>List of integers with the index of the target for binary classification or tuples with two elements: the index of the targets or binary classification and the positive weight for binary classification</p> </li> <li> <code>multiclass_config</code>               (<code>Optional[List[Union[Tuple[int, int], Tuple[int, int, List[float]]]]]</code>, default:                   <code>None</code> )           \u2013            <p>List of tuples with two or three elements: the index of the target and the number of classes for multiclass classification, or a tuple with the index of the target, the number of classes and a list of weights to apply to each class (i.e. the 'weight' parameter in the cross_entropy loss)</p> </li> <li> <code>weights</code>               (<code>Optional[List[float]]</code>, default:                   <code>None</code> )           \u2013            <p>List of weights to apply to the loss associated to each target. The length of the list must match the number of targets. Alias: 'target_weights'</p> </li> <li> <code>reduction</code>               (<code>Literal[mean, sum]</code>, default:                   <code>'mean'</code> )           \u2013            <p>Specifies the reduction to apply to the loss associated to each target: 'mean' | 'sum'. Note that this is NOT the same as the reduction in the cross_entropy loss or the binary_cross_entropy_with_logits. This reduction is applied after the loss for each target has been computed. Alias: 'target_reduction'</p> </li> <li> <code>binary_trick</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>If True, each target will be considered independently and the loss will be computed as binary_cross_entropy_with_logits. This is a faster implementation. Note that the 'weights' parameter is not compatible with binary_trick=True. Also note that if binary_trick=True, the 'binary_config' must be a list of integers and the 'multiclass_config' must be a list of tuples with two integers: the index of the target and the number of classes. Finally, if binary_trick=True, the binary targets must be the first targets in the target tensor.</p> <p> NOTE: When using the binary_trick, the binary targets are   considered as 2 classes. Therefore, the pred_dim parametere of the   WideDeep class should be adjusted accordingly (adding 2 to per   binary target). For example, in a problem with a binary target and   a 4 class multiclassification target, the pred_dim should be 6.</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.losses_multitarget import MultiTargetClassificationLoss\n&gt;&gt;&gt; input = torch.randn(5, 4)\n&gt;&gt;&gt; input_binary_trick = torch.randn(5, 5)\n&gt;&gt;&gt; target = torch.stack([torch.tensor([0, 1, 0, 1, 1]), torch.tensor([0, 1, 2, 0, 2])], 1)\n&gt;&gt;&gt; loss_1 = MultiTargetClassificationLoss(binary_config=[0], multiclass_config=[(1, 3)], reduction=\"mean\")\n&gt;&gt;&gt; output_1 = loss_1(input, target)\n&gt;&gt;&gt; loss_2 = MultiTargetClassificationLoss(binary_config=[(0, 0.5)], multiclass_config=[(1, 3, [1., 2., 3.])],\n... reduction=\"sum\", weights=[0.5, 0.5])\n&gt;&gt;&gt; output_2 = loss_2(input, target)\n&gt;&gt;&gt; loss_3 = MultiTargetClassificationLoss(binary_config=[0], multiclass_config=[(1, 3)], binary_trick=True)\n&gt;&gt;&gt; output_3 = loss_3(input_binary_trick, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses_multitarget.py</code> <pre><code>@alias(\"reduction\", [\"target_reduction\"])\n@alias(\"weights\", [\"target_weights\"])\ndef __init__(  # noqa: C901\n    self,\n    binary_config: Optional[List[Union[int, Tuple[int, float]]]] = None,\n    multiclass_config: Optional[\n        List[Union[Tuple[int, int], Tuple[int, int, List[float]]]]\n    ] = None,\n    weights: Optional[List[float]] = None,\n    reduction: Literal[\"mean\", \"sum\"] = \"mean\",\n    binary_trick: bool = False,\n):\n    super(MultiTargetClassificationLoss, self).__init__()\n\n    if reduction not in [\"mean\", \"sum\"]:\n        raise ValueError(\"reduction must be either 'mean' or 'sum'\")\n\n    self.binary_config = binary_config\n    self.multiclass_config = multiclass_config\n    self.weights = weights\n    self.reduction = reduction\n    self.binary_trick = binary_trick\n\n    if self.weights is not None:\n        if len(self.weights) != (\n            len(self.binary_config) if self.binary_config is not None else 0\n        ) + (\n            len(self.multiclass_config) if self.multiclass_config is not None else 0\n        ):\n            raise ValueError(\n                \"The number of weights must match the number of binary and multiclass targets\"\n            )\n\n    if self.binary_trick:\n        self._check_inputs_with_binary_trick()\n        self._binary_config: List[int] = binary_config  # type: ignore[assignment]\n        self._multiclass_config: List[Tuple[int, int]] = self.multiclass_config  # type: ignore[assignment]\n    else:\n        self.binary_config_with_pos_weights = (\n            (self._set_binary_config_without_binary_trick())\n            if self.binary_config is not None\n            else None\n        )\n        self.multiclass_config_with_weights = (\n            (self._set_multiclass_config_without_binary_trick())\n            if self.multiclass_config is not None\n            else None\n        )\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses_multitarget.MutilTargetRegressionAndClassificationLoss","title":"MutilTargetRegressionAndClassificationLoss","text":"<pre><code>MutilTargetRegressionAndClassificationLoss(\n    regression_config=[],\n    binary_config=None,\n    multiclass_config=None,\n    weights=None,\n    reduction=\"mean\",\n    binary_trick=False,\n)\n</code></pre> <p>               Bases: <code>Module</code></p> <p>This class is a wrapper around the MultiTargetRegressionLoss and the MultiTargetClassificationLoss. It allows for multi-target regression and classification problems. The user can provide a list of weights to apply to each target. The loss can be either the sum or the mean of the individual losses</p> <p>Parameters:</p> <ul> <li> <code>regression_config</code>               (<code>List[int]</code>, default:                   <code>[]</code> )           \u2013            <p>List of integers with the indices of the regression targets</p> </li> <li> <code>binary_config</code>               (<code>Optional[List[Union[int, Tuple[int, float]]]]</code>, default:                   <code>None</code> )           \u2013            <p>List of integers with the index of the target for binary classification or tuples with two elements: the index of the targets or binary classification and the positive weight for binary classification</p> </li> <li> <code>multiclass_config</code>               (<code>Optional[List[Union[Tuple[int, int], Tuple[int, int, List[float]]]]]</code>, default:                   <code>None</code> )           \u2013            <p>List of tuples with two or three elements: the index of the target and the number of classes for multiclass classification, or a tuple with the index of the target, the number of classes and a list of weights to apply to each class (i.e. the 'weight' parameter in the cross_entropy loss)</p> </li> <li> <code>weights</code>               (<code>Optional[List[float]]</code>, default:                   <code>None</code> )           \u2013            <p>List of weights to apply to the loss associated to each target. The length of the list must match the number of targets. Alias: 'target_weights'</p> </li> <li> <code>reduction</code>               (<code>Literal[mean, sum]</code>, default:                   <code>'mean'</code> )           \u2013            <p>Specifies the reduction to apply to the output: 'mean' | 'sum'. Note that this is NOT the same as the reduction in the cross_entropy loss, the binary_cross_entropy_with_logits or the MSELoss. This reduction is applied after each target has been computed. Alias: 'target_reduction'</p> </li> <li> <code>binary_trick</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>If True, each target will be considered independently and the loss will be computed as binary_cross_entropy_with_logits. This is a faster implementation. Note that the 'weights' parameter is not compatible with binary_trick=True. Also note that if binary_trick=True, the 'binary_config' must be a list of integers and the 'multiclass_config' must be a list of tuples with two integers: the index of the target and the number of classes. Finally, if binary_trick=True, the binary targets must be the first targets in the target tensor.</p> <p> NOTE: When using the binary_trick, the binary targets are   considered as 2 classes. Therefore, the pred_dim parametere of the   WideDeep class should be adjusted accordingly (adding 2 to per   binary target). For example, in a problem with a binary target and   a 4 class multiclassification target, the pred_dim should be 6.</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.losses_multitarget import MutilTargetRegressionAndClassificationLoss\n&gt;&gt;&gt; input = torch.randn(5, 5)\n&gt;&gt;&gt; target = torch.stack([torch.randn(5), torch.tensor([0, 1, 0, 1, 1]), torch.tensor([0, 1, 2, 0, 2])], 1)\n&gt;&gt;&gt; loss = MutilTargetRegressionAndClassificationLoss(regression_config=[0], binary_config=[2],\n... multiclass_config=[(2, 3)], reduction=\"mean\")\n&gt;&gt;&gt; output = loss(input, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses_multitarget.py</code> <pre><code>@alias(\"reduction\", [\"target_reduction\"])\n@alias(\"weights\", [\"target_weights\"])\ndef __init__(  # noqa: C901\n    self,\n    regression_config: List[int] = [],\n    binary_config: Optional[List[Union[int, Tuple[int, float]]]] = None,\n    multiclass_config: Optional[\n        List[Union[Tuple[int, int], Tuple[int, int, List[float]]]]\n    ] = None,\n    weights: Optional[List[float]] = None,\n    reduction: Literal[\"mean\", \"sum\"] = \"mean\",\n    binary_trick: bool = False,\n):\n\n    super(MutilTargetRegressionAndClassificationLoss, self).__init__()\n\n    self.regression_config = regression_config\n\n    assert binary_config is not None or multiclass_config is not None, (\n        \"Either binary_config or multiclass_config must be provided. \"\n        \"Otherwise, use the MultiTargetRegressionLoss\"\n    )\n\n    if binary_trick:\n        self._check_inputs_with_binary_trick(\n            regression_config, binary_config, multiclass_config\n        )\n\n    if weights is not None:\n        if len(weights) != (\n            len(regression_config)\n            + (len(binary_config) if binary_config is not None else 0)\n            + (len(multiclass_config) if multiclass_config is not None else 0)\n        ):\n            raise ValueError(\n                \"The number of weights must match the number of regression, binary and multiclass targets\"\n            )\n\n        self.weights_regression = self._prepare_weights_for_regression_targets(\n            weights, regression_config\n        )\n        self.weights_binary = self._prepare_weights_per_binary_targets(\n            weights, binary_config\n        )\n        self.weights_multiclass = self._prepare_weights_per_multiclass_targets(\n            weights, multiclass_config\n        )\n        self.weights = weights\n    else:\n        self.weights_regression = None\n        self.weights_binary = None\n        self.weights_multiclass = None\n\n    self.multi_target_regression_loss = MultiTargetRegressionLoss(\n        weights=self.weights_regression, reduction=reduction\n    )\n\n    self.multi_target_classification_loss = MultiTargetClassificationLoss(\n        binary_config=binary_config,\n        multiclass_config=multiclass_config,\n        weights=(\n            self.weights_binary + self.weights_multiclass\n            if self.weights_binary is not None\n            and self.weights_multiclass is not None\n            else (\n                self.weights_binary\n                if self.weights_binary is not None\n                else self.weights_multiclass\n            )\n        ),\n        reduction=reduction,\n        binary_trick=binary_trick,\n    )\n</code></pre>"},{"location":"pytorch-widedeep/metrics.html","title":"Metrics","text":"<p> NOTE: metrics in this module expect the predictions  and ground truth to have the same dimensions for regression and binary  classification problems: \\((N_{samples}, 1)\\). In the case of multiclass  classification problems the ground truth is expected to be a 1D tensor with  the corresponding classes. See Examples below</p> <p>We have added the possibility of using the metrics available at the torchmetrics library. Note that this library is still in its early versions and therefore this option should be used with caution. To use <code>torchmetrics</code> simply import them and use them as any of the <code>pytorch-widedeep</code> metrics described below.</p> <pre><code>from torchmetrics import Accuracy, Precision\n\naccuracy = Accuracy(average=None, num_classes=2)\nprecision = Precision(average='micro', num_classes=2)\n\ntrainer = Trainer(model, objective=\"binary\", metrics=[accuracy, precision])\n</code></pre> <p>A functioning example for <code>pytorch-widedeep</code> using <code>torchmetrics</code> can be found in the Examples folder</p> <p> NOTE: the forward method for all metrics in this  module takes two tensors, <code>y_pred</code> and <code>y_true</code> (in that order). Therefore,  we do not include the method in the documentation.</p>"},{"location":"pytorch-widedeep/metrics.html#pytorch_widedeep.metrics.Accuracy","title":"Accuracy","text":"<pre><code>Accuracy(top_k=1)\n</code></pre> <p>               Bases: <code>Metric</code></p> <p>Class to calculate the accuracy for both binary and categorical problems</p> <p>Parameters:</p> <ul> <li> <code>top_k</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>Accuracy will be computed using the top k most likely classes in multiclass problems</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt;\n&gt;&gt;&gt; from pytorch_widedeep.metrics import Accuracy\n&gt;&gt;&gt;\n&gt;&gt;&gt; acc = Accuracy()\n&gt;&gt;&gt; y_true = torch.tensor([0, 1, 0, 1]).view(-1, 1)\n&gt;&gt;&gt; y_pred = torch.tensor([[0.3, 0.2, 0.6, 0.7]]).view(-1, 1)\n&gt;&gt;&gt; acc(y_pred, y_true)\narray(0.5)\n&gt;&gt;&gt;\n&gt;&gt;&gt; acc = Accuracy(top_k=2)\n&gt;&gt;&gt; y_true = torch.tensor([0, 1, 2])\n&gt;&gt;&gt; y_pred = torch.tensor([[0.3, 0.5, 0.2], [0.1, 0.1, 0.8], [0.1, 0.5, 0.4]])\n&gt;&gt;&gt; acc(y_pred, y_true)\narray(0.66666667)\n</code></pre> Source code in <code>pytorch_widedeep/metrics.py</code> <pre><code>def __init__(self, top_k: int = 1):\n    super(Accuracy, self).__init__()\n\n    self.top_k = top_k\n    self.correct_count = 0\n    self.total_count = 0\n    self._name = \"acc\"\n</code></pre>"},{"location":"pytorch-widedeep/metrics.html#pytorch_widedeep.metrics.Accuracy.reset","title":"reset","text":"<pre><code>reset()\n</code></pre> <p>resets counters to 0</p> Source code in <code>pytorch_widedeep/metrics.py</code> <pre><code>def reset(self):\n    \"\"\"\n    resets counters to 0\n    \"\"\"\n    self.correct_count = 0\n    self.total_count = 0\n</code></pre>"},{"location":"pytorch-widedeep/metrics.html#pytorch_widedeep.metrics.Precision","title":"Precision","text":"<pre><code>Precision(average=True)\n</code></pre> <p>               Bases: <code>Metric</code></p> <p>Class to calculate the precision for both binary and categorical problems</p> <p>Parameters:</p> <ul> <li> <code>average</code>               (<code>bool</code>, default:                   <code>True</code> )           \u2013            <p>This applies only to multiclass problems. if <code>True</code> calculate precision for each label, and finds their unweighted mean.</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt;\n&gt;&gt;&gt; from pytorch_widedeep.metrics import Precision\n&gt;&gt;&gt;\n&gt;&gt;&gt; prec = Precision()\n&gt;&gt;&gt; y_true = torch.tensor([0, 1, 0, 1]).view(-1, 1)\n&gt;&gt;&gt; y_pred = torch.tensor([[0.3, 0.2, 0.6, 0.7]]).view(-1, 1)\n&gt;&gt;&gt; prec(y_pred, y_true)\narray(0.5)\n&gt;&gt;&gt;\n&gt;&gt;&gt; prec = Precision(average=True)\n&gt;&gt;&gt; y_true = torch.tensor([0, 1, 2])\n&gt;&gt;&gt; y_pred = torch.tensor([[0.7, 0.1, 0.2], [0.1, 0.1, 0.8], [0.1, 0.5, 0.4]])\n&gt;&gt;&gt; prec(y_pred, y_true)\narray(0.33333334)\n</code></pre> Source code in <code>pytorch_widedeep/metrics.py</code> <pre><code>def __init__(self, average: bool = True):\n    super(Precision, self).__init__()\n\n    self.average = average\n    self.true_positives = 0\n    self.all_positives = 0\n    self.eps = 1e-20\n    self._name = \"prec\"\n</code></pre>"},{"location":"pytorch-widedeep/metrics.html#pytorch_widedeep.metrics.Precision.reset","title":"reset","text":"<pre><code>reset()\n</code></pre> <p>resets counters to 0</p> Source code in <code>pytorch_widedeep/metrics.py</code> <pre><code>def reset(self):\n    \"\"\"\n    resets counters to 0\n    \"\"\"\n    self.true_positives = 0\n    self.all_positives = 0\n</code></pre>"},{"location":"pytorch-widedeep/metrics.html#pytorch_widedeep.metrics.Recall","title":"Recall","text":"<pre><code>Recall(average=True)\n</code></pre> <p>               Bases: <code>Metric</code></p> <p>Class to calculate the recall for both binary and categorical problems</p> <p>Parameters:</p> <ul> <li> <code>average</code>               (<code>bool</code>, default:                   <code>True</code> )           \u2013            <p>This applies only to multiclass problems. if <code>True</code> calculate recall for each label, and finds their unweighted mean.</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt;\n&gt;&gt;&gt; from pytorch_widedeep.metrics import Recall\n&gt;&gt;&gt;\n&gt;&gt;&gt; rec = Recall()\n&gt;&gt;&gt; y_true = torch.tensor([0, 1, 0, 1]).view(-1, 1)\n&gt;&gt;&gt; y_pred = torch.tensor([[0.3, 0.2, 0.6, 0.7]]).view(-1, 1)\n&gt;&gt;&gt; rec(y_pred, y_true)\narray(0.5)\n&gt;&gt;&gt;\n&gt;&gt;&gt; rec = Recall(average=True)\n&gt;&gt;&gt; y_true = torch.tensor([0, 1, 2])\n&gt;&gt;&gt; y_pred = torch.tensor([[0.7, 0.1, 0.2], [0.1, 0.1, 0.8], [0.1, 0.5, 0.4]])\n&gt;&gt;&gt; rec(y_pred, y_true)\narray(0.33333334)\n</code></pre> Source code in <code>pytorch_widedeep/metrics.py</code> <pre><code>def __init__(self, average: bool = True):\n    super(Recall, self).__init__()\n\n    self.average = average\n    self.true_positives = 0\n    self.actual_positives = 0\n    self.eps = 1e-20\n    self._name = \"rec\"\n</code></pre>"},{"location":"pytorch-widedeep/metrics.html#pytorch_widedeep.metrics.Recall.reset","title":"reset","text":"<pre><code>reset()\n</code></pre> <p>resets counters to 0</p> Source code in <code>pytorch_widedeep/metrics.py</code> <pre><code>def reset(self):\n    \"\"\"\n    resets counters to 0\n    \"\"\"\n    self.true_positives = 0\n    self.actual_positives = 0\n</code></pre>"},{"location":"pytorch-widedeep/metrics.html#pytorch_widedeep.metrics.FBetaScore","title":"FBetaScore","text":"<pre><code>FBetaScore(beta, average=True)\n</code></pre> <p>               Bases: <code>Metric</code></p> <p>Class to calculate the fbeta score for both binary and categorical problems</p> \\[ F_{\\beta} = ((1 + {\\beta}^2) * \\frac{(precision * recall)}{({\\beta}^2 * precision + recall)} \\] <p>Parameters:</p> <ul> <li> <code>beta</code>               (<code>int</code>)           \u2013            <p>Coefficient to control the balance between precision and recall</p> </li> <li> <code>average</code>               (<code>bool</code>, default:                   <code>True</code> )           \u2013            <p>This applies only to multiclass problems. if <code>True</code> calculate fbeta for each label, and find their unweighted mean.</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt;\n&gt;&gt;&gt; from pytorch_widedeep.metrics import FBetaScore\n&gt;&gt;&gt;\n&gt;&gt;&gt; fbeta = FBetaScore(beta=2)\n&gt;&gt;&gt; y_true = torch.tensor([0, 1, 0, 1]).view(-1, 1)\n&gt;&gt;&gt; y_pred = torch.tensor([[0.3, 0.2, 0.6, 0.7]]).view(-1, 1)\n&gt;&gt;&gt; fbeta(y_pred, y_true)\narray(0.5)\n&gt;&gt;&gt;\n&gt;&gt;&gt; fbeta = FBetaScore(beta=2)\n&gt;&gt;&gt; y_true = torch.tensor([0, 1, 2])\n&gt;&gt;&gt; y_pred = torch.tensor([[0.7, 0.1, 0.2], [0.1, 0.1, 0.8], [0.1, 0.5, 0.4]])\n&gt;&gt;&gt; fbeta(y_pred, y_true)\narray(0.33333334)\n</code></pre> Source code in <code>pytorch_widedeep/metrics.py</code> <pre><code>def __init__(self, beta: int, average: bool = True):\n    super(FBetaScore, self).__init__()\n\n    self.beta = beta\n    self.average = average\n    self.precision = Precision(average=False)\n    self.recall = Recall(average=False)\n    self.eps = 1e-20\n    self._name = \"\".join([\"f\", str(self.beta)])\n</code></pre>"},{"location":"pytorch-widedeep/metrics.html#pytorch_widedeep.metrics.FBetaScore.reset","title":"reset","text":"<pre><code>reset()\n</code></pre> <p>resets precision and recall</p> Source code in <code>pytorch_widedeep/metrics.py</code> <pre><code>def reset(self):\n    \"\"\"\n    resets precision and recall\n    \"\"\"\n    self.precision.reset()\n    self.recall.reset()\n</code></pre>"},{"location":"pytorch-widedeep/metrics.html#pytorch_widedeep.metrics.F1Score","title":"F1Score","text":"<pre><code>F1Score(average=True)\n</code></pre> <p>               Bases: <code>Metric</code></p> <p>Class to calculate the f1 score for both binary and categorical problems</p> <p>Parameters:</p> <ul> <li> <code>average</code>               (<code>bool</code>, default:                   <code>True</code> )           \u2013            <p>This applies only to multiclass problems. if <code>True</code> calculate f1 for each label, and find their unweighted mean.</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt;\n&gt;&gt;&gt; from pytorch_widedeep.metrics import F1Score\n&gt;&gt;&gt;\n&gt;&gt;&gt; f1 = F1Score()\n&gt;&gt;&gt; y_true = torch.tensor([0, 1, 0, 1]).view(-1, 1)\n&gt;&gt;&gt; y_pred = torch.tensor([[0.3, 0.2, 0.6, 0.7]]).view(-1, 1)\n&gt;&gt;&gt; f1(y_pred, y_true)\narray(0.5)\n&gt;&gt;&gt;\n&gt;&gt;&gt; f1 = F1Score()\n&gt;&gt;&gt; y_true = torch.tensor([0, 1, 2])\n&gt;&gt;&gt; y_pred = torch.tensor([[0.7, 0.1, 0.2], [0.1, 0.1, 0.8], [0.1, 0.5, 0.4]])\n&gt;&gt;&gt; f1(y_pred, y_true)\narray(0.33333334)\n</code></pre> Source code in <code>pytorch_widedeep/metrics.py</code> <pre><code>def __init__(self, average: bool = True):\n    super(F1Score, self).__init__()\n\n    self.average = average\n    self.f1 = FBetaScore(beta=1, average=self.average)\n    self._name = self.f1._name\n</code></pre>"},{"location":"pytorch-widedeep/metrics.html#pytorch_widedeep.metrics.F1Score.reset","title":"reset","text":"<pre><code>reset()\n</code></pre> <p>resets counters to 0</p> Source code in <code>pytorch_widedeep/metrics.py</code> <pre><code>def reset(self):\n    \"\"\"\n    resets counters to 0\n    \"\"\"\n    self.f1.reset()\n</code></pre>"},{"location":"pytorch-widedeep/metrics.html#pytorch_widedeep.metrics.R2Score","title":"R2Score","text":"<pre><code>R2Score()\n</code></pre> <p>               Bases: <code>Metric</code></p> <p>Calculates R-Squared, the coefficient of determination:</p> \\[ R^2 = 1 - \\frac{\\sum_{j=1}^n(y_j - \\hat{y_j})^2}{\\sum_{j=1}^n(y_j - \\bar{y})^2} \\] <p>where \\(\\hat{y_j}\\) is the ground truth, \\(y_j\\) is the predicted value and \\(\\bar{y}\\) is the mean of the ground truth.</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt;\n&gt;&gt;&gt; from pytorch_widedeep.metrics import R2Score\n&gt;&gt;&gt;\n&gt;&gt;&gt; r2 = R2Score()\n&gt;&gt;&gt; y_true = torch.tensor([3, -0.5, 2, 7]).view(-1, 1)\n&gt;&gt;&gt; y_pred = torch.tensor([2.5, 0.0, 2, 8]).view(-1, 1)\n&gt;&gt;&gt; r2(y_pred, y_true)\narray(0.94860814)\n</code></pre> Source code in <code>pytorch_widedeep/metrics.py</code> <pre><code>def __init__(self):\n    self.numerator = 0\n    self.denominator = 0\n    self.num_examples = 0\n    self.y_true_sum = 0\n\n    self._name = \"r2\"\n</code></pre>"},{"location":"pytorch-widedeep/metrics.html#pytorch_widedeep.metrics.R2Score.reset","title":"reset","text":"<pre><code>reset()\n</code></pre> <p>resets counters to 0</p> Source code in <code>pytorch_widedeep/metrics.py</code> <pre><code>def reset(self):\n    \"\"\"\n    resets counters to 0\n    \"\"\"\n    self.numerator = 0\n    self.denominator = 0\n    self.num_examples = 0\n    self.y_true_sum = 0\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html","title":"The <code>models</code> module","text":"<p>This module contains the models that can be used as the four main components that will comprise a Wide and Deep model (<code>wide</code>, <code>deeptabular</code>, <code>deeptext</code>, <code>deepimage</code>), as well as the <code>WideDeep</code> \"constructor\" class. Note that each of the four components can be used independently. It also contains all the documentation for the models that can be used for self-supervised pre-training with tabular data.</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.linear.wide.Wide","title":"Wide","text":"<pre><code>Wide(input_dim, pred_dim=1)\n</code></pre> <p>               Bases: <code>Module</code></p> <p>Defines a <code>Wide</code> (linear) model where the non-linearities are captured via the so-called crossed-columns. This can be used as the <code>wide</code> component of a Wide &amp; Deep model.</p> <p>Parameters:</p> <ul> <li> <code>input_dim</code>               (<code>int</code>)           \u2013            <p>size of the Linear layer (implemented via an Embedding layer). <code>input_dim</code> is the summation of all the individual values for all the features that go through the wide model. For example, if the wide model receives 2 features with 5 individual values each, <code>input_dim = 10</code></p> </li> <li> <code>pred_dim</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>size of the ouput tensor containing the predictions. Note that unlike all the other models, the wide model is connected directly to the output neuron(s) when used to build a Wide and Deep model. Therefore, it requires the <code>pred_dim</code> parameter.</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>wide_linear</code>               (<code>Module</code>)           \u2013            <p>the linear layer that comprises the wide branch of the model</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import Wide\n&gt;&gt;&gt; X = torch.empty(4, 4).random_(20)\n&gt;&gt;&gt; wide = Wide(input_dim=int(X.max().item()), pred_dim=1)\n&gt;&gt;&gt; out = wide(X)\n</code></pre> Source code in <code>pytorch_widedeep/models/tabular/linear/wide.py</code> <pre><code>@alias(\"pred_dim\", [\"pred_size\", \"num_class\"])\ndef __init__(self, input_dim: int, pred_dim: int = 1):\n    super(Wide, self).__init__()\n\n    self.input_dim = input_dim\n    self.pred_dim = pred_dim\n\n    # Embeddings: val + 1 because 0 is reserved for padding/unseen cateogories.\n    self.wide_linear = nn.Embedding(input_dim + 1, pred_dim, padding_idx=0)\n    # (Sum(Embedding) + bias) is equivalent to (OneHotVector + Linear)\n    self.bias = nn.Parameter(torch.zeros(pred_dim))\n    self._reset_parameters()\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.linear.wide.Wide.forward","title":"forward","text":"<pre><code>forward(X)\n</code></pre> <p>Forward pass. Simply connecting the Embedding layer with the ouput neuron(s)</p> Source code in <code>pytorch_widedeep/models/tabular/linear/wide.py</code> <pre><code>def forward(self, X: Tensor) -&gt; Tensor:\n    r\"\"\"Forward pass. Simply connecting the Embedding layer with the ouput\n    neuron(s)\"\"\"\n    out = self.wide_linear(X.long()).sum(dim=1) + self.bias\n    return out\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.mlp.tab_mlp.TabMlp","title":"TabMlp","text":"<pre><code>TabMlp(\n    column_idx,\n    *,\n    cat_embed_input=None,\n    cat_embed_dropout=None,\n    use_cat_bias=None,\n    cat_embed_activation=None,\n    continuous_cols=None,\n    cont_norm_layer=None,\n    embed_continuous=None,\n    embed_continuous_method=None,\n    cont_embed_dim=None,\n    cont_embed_dropout=None,\n    cont_embed_activation=None,\n    quantization_setup=None,\n    n_frequencies=None,\n    sigma=None,\n    share_last_layer=None,\n    full_embed_dropout=None,\n    mlp_hidden_dims=[200, 100],\n    mlp_activation=\"relu\",\n    mlp_dropout=0.1,\n    mlp_batchnorm=False,\n    mlp_batchnorm_last=False,\n    mlp_linear_first=True\n)\n</code></pre> <p>               Bases: <code>BaseTabularModelWithoutAttention</code></p> <p>Defines a <code>TabMlp</code> model that can be used as the <code>deeptabular</code> component of a Wide &amp; Deep model or independently by itself.</p> <p>This class combines embedding representations of the categorical features with numerical (aka continuous) features, embedded or not. These are then passed through a series of dense layers (i.e. a MLP).</p> <p>Most of the parameters for this class are <code>Optional</code> since the use of categorical or continuous is in fact optional (i.e. one can use categorical features only, continuous features only or both).</p> <p>Parameters:</p> <ul> <li> <code>column_idx</code>               (<code>Dict[str, int]</code>)           \u2013            <p>Dict containing the index of the columns that will be passed through the <code>TabMlp</code> model. Required to slice the tensors. e.g. {'education': 0, 'relationship': 1, 'workclass': 2, ...}.</p> </li> <li> <code>cat_embed_input</code>               (<code>Optional[List[Tuple[str, int, int]]]</code>, default:                   <code>None</code> )           \u2013            <p>List of Tuples with the column name, number of unique values and embedding dimension. e.g. [(education, 11, 32), ...]</p> </li> <li> <code>cat_embed_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Categorical embeddings dropout. If <code>None</code>, it will default to 0.</p> </li> <li> <code>use_cat_bias</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating if bias will be used for the categorical embeddings. If <code>None</code>, it will default to 'False'.</p> </li> <li> <code>cat_embed_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the categorical embeddings, if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> </li> <li> <code>continuous_cols</code>               (<code>Optional[List[str]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the name of the numeric (aka continuous) columns</p> </li> <li> <code>cont_norm_layer</code>               (<code>Optional[Literal[batchnorm, layernorm]]</code>, default:                   <code>None</code> )           \u2013            <p>Type of normalization layer applied to the continuous features. Options are: 'layernorm' and 'batchnorm'. if <code>None</code>, no normalization layer will be used.</p> </li> <li> <code>embed_continuous</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating if the continuous columns will be embedded using one of the available methods: 'standard', 'periodic' or 'piecewise'. If <code>None</code>, it will default to 'False'. NOTE: This parameter is deprecated and it  will be removed in future releases. Please, use the  <code>embed_continuous_method</code> parameter instead.</p> </li> <li> <code>embed_continuous_method</code>               (<code>Optional[Literal[standard, piecewise, periodic]]</code>, default:                   <code>None</code> )           \u2013            <p>Method to use to embed the continuous features. Options are: 'standard', 'periodic' or 'piecewise'. The 'standard' embedding method is based on the FT-Transformer implementation presented in the paper: Revisiting Deep Learning Models for Tabular Data. The 'periodic' and_'piecewise'_ methods were presented in the paper: On Embeddings for Numerical Features in Tabular Deep Learning. Please, read the papers for details.</p> </li> <li> <code>cont_embed_dim</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>Size of the continuous embeddings. If the continuous columns are embedded, <code>cont_embed_dim</code> must be passed.</p> </li> <li> <code>cont_embed_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p> </li> <li> <code>cont_embed_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the continuous embeddings if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported. If <code>None</code>, no activation function will be applied.</p> </li> <li> <code>quantization_setup</code>               (<code>Optional[Dict[str, List[float]]]</code>, default:                   <code>None</code> )           \u2013            <p>This parameter is used when the 'piecewise' method is used to embed the continuous cols. It is a dict where keys are the name of the continuous columns and values are lists with the boundaries for the quantization of the continuous_cols. See the examples for details. If If the 'piecewise' method is used, this parameter is required.</p> </li> <li> <code>n_frequencies</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>This is the so called 'k' in their paper On Embeddings for Numerical Features in Tabular Deep Learning, and is the number of 'frequencies' that will be used to represent each continuous column. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>sigma</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>This is the sigma parameter in the paper mentioned when describing the previous parameters and it is used to initialise the 'frequency weights'. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>share_last_layer</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>This parameter is not present in the before mentioned paper but it is implemented in the official repo. If <code>True</code> the linear layer that turns the frequencies into embeddings will be shared across the continuous columns. If <code>False</code> a different linear layer will be used for each continuous column. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>full_embed_dropout</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>If <code>True</code>, the full embedding corresponding to a column will be masked out/dropout. If <code>None</code>, it will default to <code>False</code>.</p> </li> <li> <code>mlp_hidden_dims</code>               (<code>List[int]</code>, default:                   <code>[200, 100]</code> )           \u2013            <p>List with the number of neurons per dense layer in the mlp.</p> </li> <li> <code>mlp_activation</code>               (<code>str</code>, default:                   <code>'relu'</code> )           \u2013            <p>Activation function for the dense layers of the MLP. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> </li> <li> <code>mlp_dropout</code>               (<code>Union[float, List[float]]</code>, default:                   <code>0.1</code> )           \u2013            <p>float or List of floats with the dropout between the dense layers. e.g: [0.5,0.5]</p> </li> <li> <code>mlp_batchnorm</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether or not batch normalization will be applied to the dense layers</p> </li> <li> <code>mlp_batchnorm_last</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether or not batch normalization will be applied to the last of the dense layers</p> </li> <li> <code>mlp_linear_first</code>               (<code>bool</code>, default:                   <code>True</code> )           \u2013            <p>Boolean indicating the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code></p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>encoder</code>               (<code>Module</code>)           \u2013            <p>mlp model that will receive the concatenation of the embeddings and the continuous columns</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import TabMlp\n&gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n&gt;&gt;&gt; colnames = [\"a\", \"b\", \"c\", \"d\", \"e\"]\n&gt;&gt;&gt; cat_embed_input = [(u, i, j) for u, i, j in zip(colnames[:4], [4] * 4, [8] * 4)]\n&gt;&gt;&gt; column_idx = {k: v for v, k in enumerate(colnames)}\n&gt;&gt;&gt; model = TabMlp(mlp_hidden_dims=[8, 4], column_idx=column_idx, cat_embed_input=cat_embed_input,\n... continuous_cols=[\"e\"])\n&gt;&gt;&gt; out = model(X_tab)\n</code></pre> Source code in <code>pytorch_widedeep/models/tabular/mlp/tab_mlp.py</code> <pre><code>def __init__(\n    self,\n    column_idx: Dict[str, int],\n    *,\n    cat_embed_input: Optional[List[Tuple[str, int, int]]] = None,\n    cat_embed_dropout: Optional[float] = None,\n    use_cat_bias: Optional[bool] = None,\n    cat_embed_activation: Optional[str] = None,\n    continuous_cols: Optional[List[str]] = None,\n    cont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\n    embed_continuous: Optional[bool] = None,\n    embed_continuous_method: Optional[\n        Literal[\"standard\", \"piecewise\", \"periodic\"]\n    ] = None,\n    cont_embed_dim: Optional[int] = None,\n    cont_embed_dropout: Optional[float] = None,\n    cont_embed_activation: Optional[str] = None,\n    quantization_setup: Optional[Dict[str, List[float]]] = None,\n    n_frequencies: Optional[int] = None,\n    sigma: Optional[float] = None,\n    share_last_layer: Optional[bool] = None,\n    full_embed_dropout: Optional[bool] = None,\n    mlp_hidden_dims: List[int] = [200, 100],\n    mlp_activation: str = \"relu\",\n    mlp_dropout: Union[float, List[float]] = 0.1,\n    mlp_batchnorm: bool = False,\n    mlp_batchnorm_last: bool = False,\n    mlp_linear_first: bool = True,\n):\n    super(TabMlp, self).__init__(\n        column_idx=column_idx,\n        cat_embed_input=cat_embed_input,\n        cat_embed_dropout=cat_embed_dropout,\n        use_cat_bias=use_cat_bias,\n        cat_embed_activation=cat_embed_activation,\n        continuous_cols=continuous_cols,\n        cont_norm_layer=cont_norm_layer,\n        embed_continuous=embed_continuous,\n        embed_continuous_method=embed_continuous_method,\n        cont_embed_dim=cont_embed_dim,\n        cont_embed_dropout=cont_embed_dropout,\n        cont_embed_activation=cont_embed_activation,\n        quantization_setup=quantization_setup,\n        n_frequencies=n_frequencies,\n        sigma=sigma,\n        share_last_layer=share_last_layer,\n        full_embed_dropout=full_embed_dropout,\n    )\n\n    self.mlp_hidden_dims = mlp_hidden_dims\n    self.mlp_activation = mlp_activation\n    self.mlp_dropout = mlp_dropout\n    self.mlp_batchnorm = mlp_batchnorm\n    self.mlp_batchnorm_last = mlp_batchnorm_last\n    self.mlp_linear_first = mlp_linear_first\n\n    # Embeddings are instantiated at the base model\n    # Mlp\n    mlp_input_dim = self.cat_out_dim + self.cont_out_dim\n    mlp_hidden_dims = [mlp_input_dim] + mlp_hidden_dims\n    self.encoder = MLP(\n        mlp_hidden_dims,\n        mlp_activation,\n        mlp_dropout,\n        mlp_batchnorm,\n        mlp_batchnorm_last,\n        mlp_linear_first,\n    )\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.mlp.tab_mlp.TabMlp.output_dim","title":"output_dim  <code>property</code>","text":"<pre><code>output_dim\n</code></pre> <p>The output dimension of the model. This is a required property neccesary to build the <code>WideDeep</code> class</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.mlp.tab_mlp.TabMlpDecoder","title":"TabMlpDecoder","text":"<pre><code>TabMlpDecoder(\n    embed_dim,\n    mlp_hidden_dims=[100, 200],\n    mlp_activation=\"relu\",\n    mlp_dropout=0.1,\n    mlp_batchnorm=False,\n    mlp_batchnorm_last=False,\n    mlp_linear_first=True,\n)\n</code></pre> <p>               Bases: <code>Module</code></p> <p>Companion decoder model for the <code>TabMlp</code> model (which can be considered an encoder itself).</p> <p>This class is designed to be used with the <code>EncoderDecoderTrainer</code> when using self-supervised pre-training (see the corresponding section in the docs). The <code>TabMlpDecoder</code> will receive the output from the MLP and 'reconstruct' the embeddings.</p> <p>Parameters:</p> <ul> <li> <code>embed_dim</code>               (<code>int</code>)           \u2013            <p>Size of the embeddings tensor that needs to be reconstructed.</p> </li> <li> <code>mlp_hidden_dims</code>               (<code>List[int]</code>, default:                   <code>[100, 200]</code> )           \u2013            <p>List with the number of neurons per dense layer in the mlp.</p> </li> <li> <code>mlp_activation</code>               (<code>str</code>, default:                   <code>'relu'</code> )           \u2013            <p>Activation function for the dense layers of the MLP. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> </li> <li> <code>mlp_dropout</code>               (<code>Union[float, List[float]]</code>, default:                   <code>0.1</code> )           \u2013            <p>float or List of floats with the dropout between the dense layers. e.g: [0.5,0.5]</p> </li> <li> <code>mlp_batchnorm</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether or not batch normalization will be applied to the dense layers</p> </li> <li> <code>mlp_batchnorm_last</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether or not batch normalization will be applied to the last of the dense layers</p> </li> <li> <code>mlp_linear_first</code>               (<code>bool</code>, default:                   <code>True</code> )           \u2013            <p>Boolean indicating the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code></p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>decoder</code>               (<code>Module</code>)           \u2013            <p>mlp model that will receive the output of the encoder</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import TabMlpDecoder\n&gt;&gt;&gt; x_inp = torch.rand(3, 8)\n&gt;&gt;&gt; decoder = TabMlpDecoder(embed_dim=32, mlp_hidden_dims=[8,16])\n&gt;&gt;&gt; res = decoder(x_inp)\n&gt;&gt;&gt; res.shape\ntorch.Size([3, 32])\n</code></pre> Source code in <code>pytorch_widedeep/models/tabular/mlp/tab_mlp.py</code> <pre><code>def __init__(\n    self,\n    embed_dim: int,\n    mlp_hidden_dims: List[int] = [100, 200],\n    mlp_activation: str = \"relu\",\n    mlp_dropout: Union[float, List[float]] = 0.1,\n    mlp_batchnorm: bool = False,\n    mlp_batchnorm_last: bool = False,\n    mlp_linear_first: bool = True,\n):\n    super(TabMlpDecoder, self).__init__()\n\n    self.embed_dim = embed_dim\n\n    self.mlp_hidden_dims = mlp_hidden_dims\n    self.mlp_activation = mlp_activation\n    self.mlp_dropout = mlp_dropout\n    self.mlp_batchnorm = mlp_batchnorm\n    self.mlp_batchnorm_last = mlp_batchnorm_last\n    self.mlp_linear_first = mlp_linear_first\n\n    self.decoder = MLP(\n        mlp_hidden_dims + [self.embed_dim],\n        mlp_activation,\n        mlp_dropout,\n        mlp_batchnorm,\n        mlp_batchnorm_last,\n        mlp_linear_first,\n    )\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.resnet.tab_resnet.TabResnet","title":"TabResnet","text":"<pre><code>TabResnet(\n    column_idx,\n    *,\n    cat_embed_input=None,\n    cat_embed_dropout=None,\n    use_cat_bias=None,\n    cat_embed_activation=None,\n    continuous_cols=None,\n    cont_norm_layer=None,\n    embed_continuous=None,\n    embed_continuous_method=None,\n    cont_embed_dim=None,\n    cont_embed_dropout=None,\n    cont_embed_activation=None,\n    quantization_setup=None,\n    n_frequencies=None,\n    sigma=None,\n    share_last_layer=None,\n    full_embed_dropout=None,\n    blocks_dims=[200, 100, 100],\n    blocks_dropout=0.1,\n    simplify_blocks=False,\n    mlp_hidden_dims=None,\n    mlp_activation=None,\n    mlp_dropout=None,\n    mlp_batchnorm=None,\n    mlp_batchnorm_last=None,\n    mlp_linear_first=None\n)\n</code></pre> <p>               Bases: <code>BaseTabularModelWithoutAttention</code></p> <p>Defines a <code>TabResnet</code> model that can be used as the <code>deeptabular</code> component of a Wide &amp; Deep model or independently by itself.</p> <p>This class combines embedding representations of the categorical features with numerical (aka continuous) features, embedded or not. These are then passed through a series of Resnet blocks. See <code>pytorch_widedeep.models.tab_resnet._layers</code> for details on the structure of each block.</p> <p>Most of the parameters for this class are <code>Optional</code> since the use of categorical or continuous is in fact optional (i.e. one can use categorical features only, continuous features only or both).</p> <p>Parameters:</p> <ul> <li> <code>column_idx</code>               (<code>Dict[str, int]</code>)           \u2013            <p>Dict containing the index of the columns that will be passed through the <code>TabMlp</code> model. Required to slice the tensors. e.g. {'education': 0, 'relationship': 1, 'workclass': 2, ...}.</p> </li> <li> <code>cat_embed_input</code>               (<code>Optional[List[Tuple[str, int, int]]]</code>, default:                   <code>None</code> )           \u2013            <p>List of Tuples with the column name, number of unique values and embedding dimension. e.g. [(education, 11, 32), ...]</p> </li> <li> <code>cat_embed_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Categorical embeddings dropout. If <code>None</code>, it will default to 0.</p> </li> <li> <code>use_cat_bias</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating if bias will be used for the categorical embeddings. If <code>None</code>, it will default to 'False'.</p> </li> <li> <code>cat_embed_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the categorical embeddings, if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> </li> <li> <code>continuous_cols</code>               (<code>Optional[List[str]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the name of the numeric (aka continuous) columns</p> </li> <li> <code>cont_norm_layer</code>               (<code>Optional[Literal[batchnorm, layernorm]]</code>, default:                   <code>None</code> )           \u2013            <p>Type of normalization layer applied to the continuous features. Options are: 'layernorm' and 'batchnorm'. if <code>None</code>, no normalization layer will be used.</p> </li> <li> <code>embed_continuous</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating if the continuous columns will be embedded using one of the available methods: 'standard', 'periodic' or 'piecewise'. If <code>None</code>, it will default to 'False'. NOTE: This parameter is deprecated and it  will be removed in future releases. Please, use the  <code>embed_continuous_method</code> parameter instead.</p> </li> <li> <code>embed_continuous_method</code>               (<code>Optional[Literal[standard, piecewise, periodic]]</code>, default:                   <code>None</code> )           \u2013            <p>Method to use to embed the continuous features. Options are: 'standard', 'periodic' or 'piecewise'. The 'standard' embedding method is based on the FT-Transformer implementation presented in the paper: Revisiting Deep Learning Models for Tabular Data. The 'periodic' and_'piecewise'_ methods were presented in the paper: On Embeddings for Numerical Features in Tabular Deep Learning. Please, read the papers for details.</p> </li> <li> <code>cont_embed_dim</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>Size of the continuous embeddings. If the continuous columns are embedded, <code>cont_embed_dim</code> must be passed.</p> </li> <li> <code>cont_embed_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p> </li> <li> <code>cont_embed_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the continuous embeddings if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported. If <code>None</code>, no activation function will be applied.</p> </li> <li> <code>quantization_setup</code>               (<code>Optional[Dict[str, List[float]]]</code>, default:                   <code>None</code> )           \u2013            <p>This parameter is used when the 'piecewise' method is used to embed the continuous cols. It is a dict where keys are the name of the continuous columns and values are lists with the boundaries for the quantization of the continuous_cols. See the examples for details. If If the 'piecewise' method is used, this parameter is required.</p> </li> <li> <code>n_frequencies</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>This is the so called 'k' in their paper On Embeddings for Numerical Features in Tabular Deep Learning, and is the number of 'frequencies' that will be used to represent each continuous column. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>sigma</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>This is the sigma parameter in the paper mentioned when describing the previous parameters and it is used to initialise the 'frequency weights'. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>share_last_layer</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>This parameter is not present in the before mentioned paper but it is implemented in the official repo. If <code>True</code> the linear layer that turns the frequencies into embeddings will be shared across the continuous columns. If <code>False</code> a different linear layer will be used for each continuous column. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>full_embed_dropout</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>If <code>True</code>, the full embedding corresponding to a column will be masked out/dropout. If <code>None</code>, it will default to <code>False</code>.</p> </li> <li> <code>blocks_dims</code>               (<code>List[int]</code>, default:                   <code>[200, 100, 100]</code> )           \u2013            <p>List of integers that define the input and output units of each block. For example: [200, 100, 100] will generate 2 blocks. The first will receive a tensor of size 200 and output a tensor of size 100, and the second will receive a tensor of size 100 and output a tensor of size 100. See <code>pytorch_widedeep.models.tab_resnet._layers</code> for details on the structure of each block.</p> </li> <li> <code>blocks_dropout</code>               (<code>float</code>, default:                   <code>0.1</code> )           \u2013            <p>Block's internal dropout.</p> </li> <li> <code>simplify_blocks</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating if the simplest possible residual blocks (<code>X -&gt; [ [LIN, BN, ACT]  + X ]</code>) will be used instead of a standard one (<code>X -&gt; [ [LIN1, BN1, ACT1] -&gt; [LIN2, BN2]  + X ]</code>).</p> </li> <li> <code>mlp_hidden_dims</code>               (<code>Optional[List[int]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the number of neurons per dense layer in the MLP. e.g: [64, 32]. If <code>None</code> the  output of the Resnet Blocks will be connected directly to the output neuron(s).</p> </li> <li> <code>mlp_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the dense layers of the MLP. Currently 'tanh', 'relu', 'leaky'_relu' and _'gelu' are supported. If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to 'relu'.</p> </li> <li> <code>mlp_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>float with the dropout between the dense layers of the MLP. If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to 0.0.</p> </li> <li> <code>mlp_batchnorm</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating whether or not batch normalization will be applied to the dense layers If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to False.</p> </li> <li> <code>mlp_batchnorm_last</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating whether or not batch normalization will be applied to the last of the dense layers If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to False.</p> </li> <li> <code>mlp_linear_first</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code> If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to <code>True</code>.</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>encoder</code>               (<code>Module</code>)           \u2013            <p>deep dense Resnet model that will receive the concatenation of the embeddings and the continuous columns</p> </li> <li> <code>mlp</code>               (<code>Module</code>)           \u2013            <p>if <code>mlp_hidden_dims</code> is <code>True</code>, this attribute will be an mlp model that will receive the results of the concatenation of the embeddings and the continuous columns -- if present --.</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import TabResnet\n&gt;&gt;&gt; X_deep = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n&gt;&gt;&gt; colnames = ['a', 'b', 'c', 'd', 'e']\n&gt;&gt;&gt; cat_embed_input = [(u,i,j) for u,i,j in zip(colnames[:4], [4]*4, [8]*4)]\n&gt;&gt;&gt; column_idx = {k:v for v,k in enumerate(colnames)}\n&gt;&gt;&gt; model = TabResnet(blocks_dims=[16,4], column_idx=column_idx, cat_embed_input=cat_embed_input,\n... continuous_cols = ['e'])\n&gt;&gt;&gt; out = model(X_deep)\n</code></pre> Source code in <code>pytorch_widedeep/models/tabular/resnet/tab_resnet.py</code> <pre><code>def __init__(\n    self,\n    column_idx: Dict[str, int],\n    *,\n    cat_embed_input: Optional[List[Tuple[str, int, int]]] = None,\n    cat_embed_dropout: Optional[float] = None,\n    use_cat_bias: Optional[bool] = None,\n    cat_embed_activation: Optional[str] = None,\n    continuous_cols: Optional[List[str]] = None,\n    cont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\n    embed_continuous: Optional[bool] = None,\n    embed_continuous_method: Optional[\n        Literal[\"standard\", \"piecewise\", \"periodic\"]\n    ] = None,\n    cont_embed_dim: Optional[int] = None,\n    cont_embed_dropout: Optional[float] = None,\n    cont_embed_activation: Optional[str] = None,\n    quantization_setup: Optional[Dict[str, List[float]]] = None,\n    n_frequencies: Optional[int] = None,\n    sigma: Optional[float] = None,\n    share_last_layer: Optional[bool] = None,\n    full_embed_dropout: Optional[bool] = None,\n    blocks_dims: List[int] = [200, 100, 100],\n    blocks_dropout: float = 0.1,\n    simplify_blocks: bool = False,\n    mlp_hidden_dims: Optional[List[int]] = None,\n    mlp_activation: Optional[str] = None,\n    mlp_dropout: Optional[float] = None,\n    mlp_batchnorm: Optional[bool] = None,\n    mlp_batchnorm_last: Optional[bool] = None,\n    mlp_linear_first: Optional[bool] = None,\n):\n    super(TabResnet, self).__init__(\n        column_idx=column_idx,\n        cat_embed_input=cat_embed_input,\n        cat_embed_dropout=cat_embed_dropout,\n        use_cat_bias=use_cat_bias,\n        cat_embed_activation=cat_embed_activation,\n        continuous_cols=continuous_cols,\n        cont_norm_layer=cont_norm_layer,\n        embed_continuous=embed_continuous,\n        embed_continuous_method=embed_continuous_method,\n        cont_embed_dim=cont_embed_dim,\n        cont_embed_dropout=cont_embed_dropout,\n        cont_embed_activation=cont_embed_activation,\n        quantization_setup=quantization_setup,\n        n_frequencies=n_frequencies,\n        sigma=sigma,\n        share_last_layer=share_last_layer,\n        full_embed_dropout=full_embed_dropout,\n    )\n\n    if len(blocks_dims) &lt; 2:\n        raise ValueError(\n            \"'blocks' must contain at least two elements, e.g. [256, 128]\"\n        )\n\n    self.blocks_dims = blocks_dims\n    self.blocks_dropout = blocks_dropout\n    self.simplify_blocks = simplify_blocks\n\n    self.mlp_hidden_dims = mlp_hidden_dims\n    self.mlp_activation = mlp_activation\n    self.mlp_dropout = mlp_dropout\n    self.mlp_batchnorm = mlp_batchnorm\n    self.mlp_batchnorm_last = mlp_batchnorm_last\n    self.mlp_linear_first = mlp_linear_first\n\n    # Embeddings are instantiated at the base model\n\n    # Resnet\n    dense_resnet_input_dim = self.cat_out_dim + self.cont_out_dim\n    self.encoder = DenseResnet(\n        dense_resnet_input_dim, blocks_dims, blocks_dropout, self.simplify_blocks\n    )\n\n    # Mlp: adding an MLP on top of the Resnet blocks is optional and\n    # therefore all related params are optional\n    if self.mlp_hidden_dims is not None:\n        self.mlp = MLP(\n            d_hidden=[self.blocks_dims[-1]] + self.mlp_hidden_dims,\n            activation=(\n                \"relu\" if self.mlp_activation is None else self.mlp_activation\n            ),\n            dropout=0.0 if self.mlp_dropout is None else self.mlp_dropout,\n            batchnorm=False if self.mlp_batchnorm is None else self.mlp_batchnorm,\n            batchnorm_last=(\n                False\n                if self.mlp_batchnorm_last is None\n                else self.mlp_batchnorm_last\n            ),\n            linear_first=(\n                True if self.mlp_linear_first is None else self.mlp_linear_first\n            ),\n        )\n    else:\n        self.mlp = None\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.resnet.tab_resnet.TabResnet.output_dim","title":"output_dim  <code>property</code>","text":"<pre><code>output_dim\n</code></pre> <p>The output dimension of the model. This is a required property neccesary to build the <code>WideDeep</code> class</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.resnet.tab_resnet.TabResnetDecoder","title":"TabResnetDecoder","text":"<pre><code>TabResnetDecoder(\n    embed_dim,\n    blocks_dims=[100, 100, 200],\n    blocks_dropout=0.1,\n    simplify_blocks=False,\n    mlp_hidden_dims=None,\n    mlp_activation=None,\n    mlp_dropout=None,\n    mlp_batchnorm=None,\n    mlp_batchnorm_last=None,\n    mlp_linear_first=None,\n)\n</code></pre> <p>               Bases: <code>Module</code></p> <p>Companion decoder model for the <code>TabResnet</code> model (which can be considered an encoder itself)</p> <p>This class is designed to be used with the <code>EncoderDecoderTrainer</code> when using self-supervised pre-training (see the corresponding section in the docs). This class will receive the output from the ResNet blocks or the MLP(if present) and 'reconstruct' the embeddings.</p> <p>Parameters:</p> <ul> <li> <code>embed_dim</code>               (<code>int</code>)           \u2013            <p>Size of the embeddings tensor to be reconstructed.</p> </li> <li> <code>blocks_dims</code>               (<code>List[int]</code>, default:                   <code>[100, 100, 200]</code> )           \u2013            <p>List of integers that define the input and output units of each block. For example: [200, 100, 100] will generate 2 blocks. The first will receive a tensor of size 200 and output a tensor of size 100, and the second will receive a tensor of size 100 and output a tensor of size 100. See <code>pytorch_widedeep.models.tab_resnet._layers</code> for details on the structure of each block.</p> </li> <li> <code>blocks_dropout</code>               (<code>float</code>, default:                   <code>0.1</code> )           \u2013            <p>Block's internal dropout.</p> </li> <li> <code>simplify_blocks</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating if the simplest possible residual blocks (<code>X -&gt; [ [LIN, BN, ACT]  + X ]</code>) will be used instead of a standard one (<code>X -&gt; [ [LIN1, BN1, ACT1] -&gt; [LIN2, BN2]  + X ]</code>).</p> </li> <li> <code>mlp_hidden_dims</code>               (<code>Optional[List[int]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the number of neurons per dense layer in the MLP. e.g: [64, 32]. If <code>None</code> the  output of the Resnet Blocks will be connected directly to the output neuron(s).</p> </li> <li> <code>mlp_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the dense layers of the MLP. Currently 'tanh', 'relu', 'leaky'_relu' and _'gelu' are supported. If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to 'relu'.</p> </li> <li> <code>mlp_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>float with the dropout between the dense layers of the MLP. If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to 0.0.</p> </li> <li> <code>mlp_batchnorm</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating whether or not batch normalization will be applied to the dense layers If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to False.</p> </li> <li> <code>mlp_batchnorm_last</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating whether or not batch normalization will be applied to the last of the dense layers If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to False.</p> </li> <li> <code>mlp_linear_first</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code> If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to <code>True</code>.</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>decoder</code>               (<code>Module</code>)           \u2013            <p>deep dense Resnet model that will receive the output of the encoder IF <code>mlp_hidden_dims</code> is None</p> </li> <li> <code>mlp</code>               (<code>Module</code>)           \u2013            <p>if <code>mlp_hidden_dims</code> is not None, the overall decoder will consist in an MLP that will receive the output of the encoder followed by the deep dense Resnet.</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import TabResnetDecoder\n&gt;&gt;&gt; x_inp = torch.rand(3, 8)\n&gt;&gt;&gt; decoder = TabResnetDecoder(embed_dim=32, blocks_dims=[8, 16, 16])\n&gt;&gt;&gt; res = decoder(x_inp)\n&gt;&gt;&gt; res.shape\ntorch.Size([3, 32])\n</code></pre> Source code in <code>pytorch_widedeep/models/tabular/resnet/tab_resnet.py</code> <pre><code>def __init__(\n    self,\n    embed_dim: int,\n    blocks_dims: List[int] = [100, 100, 200],\n    blocks_dropout: float = 0.1,\n    simplify_blocks: bool = False,\n    mlp_hidden_dims: Optional[List[int]] = None,\n    mlp_activation: Optional[str] = None,\n    mlp_dropout: Optional[float] = None,\n    mlp_batchnorm: Optional[bool] = None,\n    mlp_batchnorm_last: Optional[bool] = None,\n    mlp_linear_first: Optional[bool] = None,\n):\n    super(TabResnetDecoder, self).__init__()\n\n    if len(blocks_dims) &lt; 2:\n        raise ValueError(\n            \"'blocks' must contain at least two elements, e.g. [256, 128]\"\n        )\n\n    self.embed_dim = embed_dim\n\n    self.blocks_dims = blocks_dims\n    self.blocks_dropout = blocks_dropout\n    self.simplify_blocks = simplify_blocks\n\n    self.mlp_hidden_dims = mlp_hidden_dims\n    self.mlp_activation = mlp_activation\n    self.mlp_dropout = mlp_dropout\n    self.mlp_batchnorm = mlp_batchnorm\n    self.mlp_batchnorm_last = mlp_batchnorm_last\n    self.mlp_linear_first = mlp_linear_first\n\n    if self.mlp_hidden_dims is not None:\n        self.mlp = MLP(\n            d_hidden=[self.mlp_first_hidden_dim] + self.mlp_hidden_dims,\n            activation=(\n                \"relu\" if self.mlp_activation is None else self.mlp_activation\n            ),\n            dropout=0.0 if self.mlp_dropout is None else self.mlp_dropout,\n            batchnorm=False if self.mlp_batchnorm is None else self.mlp_batchnorm,\n            batchnorm_last=(\n                False\n                if self.mlp_batchnorm_last is None\n                else self.mlp_batchnorm_last\n            ),\n            linear_first=(\n                True if self.mlp_linear_first is None else self.mlp_linear_first\n            ),\n        )\n        self.decoder = DenseResnet(\n            self.mlp_hidden_dims[-1],\n            blocks_dims,\n            blocks_dropout,\n            self.simplify_blocks,\n        )\n    else:\n        self.mlp = None\n        self.decoder = DenseResnet(\n            blocks_dims[0], blocks_dims, blocks_dropout, self.simplify_blocks\n        )\n\n    self.reconstruction_layer = nn.Linear(blocks_dims[-1], embed_dim, bias=False)\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.tabnet.tab_net.TabNet","title":"TabNet","text":"<pre><code>TabNet(\n    column_idx,\n    *,\n    cat_embed_input=None,\n    cat_embed_dropout=None,\n    use_cat_bias=None,\n    cat_embed_activation=None,\n    continuous_cols=None,\n    cont_norm_layer=None,\n    embed_continuous=None,\n    embed_continuous_method=None,\n    cont_embed_dim=None,\n    cont_embed_dropout=None,\n    cont_embed_activation=None,\n    quantization_setup=None,\n    n_frequencies=None,\n    sigma=None,\n    share_last_layer=None,\n    full_embed_dropout=None,\n    n_steps=3,\n    step_dim=8,\n    attn_dim=8,\n    dropout=0.0,\n    n_glu_step_dependent=2,\n    n_glu_shared=2,\n    ghost_bn=True,\n    virtual_batch_size=128,\n    momentum=0.02,\n    gamma=1.3,\n    epsilon=1e-15,\n    mask_type=\"sparsemax\"\n)\n</code></pre> <p>               Bases: <code>BaseTabularModelWithoutAttention</code></p> <p>Defines a TabNet model that can be used as the <code>deeptabular</code> component of a Wide &amp; Deep model or independently by itself.</p> <p>The implementation in this library is fully based on that here by the dreamquark-ai team, simply adapted so that it can work within the <code>WideDeep</code> frame. Therefore, ALL CREDIT TO THE DREAMQUARK-AI TEAM.</p> <p>Parameters:</p> <ul> <li> <code>column_idx</code>               (<code>Dict[str, int]</code>)           \u2013            <p>Dict containing the index of the columns that will be passed through the <code>TabMlp</code> model. Required to slice the tensors. e.g. {'education': 0, 'relationship': 1, 'workclass': 2, ...}.</p> </li> <li> <code>cat_embed_input</code>               (<code>Optional[List[Tuple[str, int, int]]]</code>, default:                   <code>None</code> )           \u2013            <p>List of Tuples with the column name, number of unique values and embedding dimension. e.g. [(education, 11, 32), ...]</p> </li> <li> <code>cat_embed_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Categorical embeddings dropout. If <code>None</code>, it will default to 0.</p> </li> <li> <code>use_cat_bias</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating if bias will be used for the categorical embeddings. If <code>None</code>, it will default to 'False'.</p> </li> <li> <code>cat_embed_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the categorical embeddings, if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> </li> <li> <code>continuous_cols</code>               (<code>Optional[List[str]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the name of the numeric (aka continuous) columns</p> </li> <li> <code>cont_norm_layer</code>               (<code>Optional[Literal[batchnorm, layernorm]]</code>, default:                   <code>None</code> )           \u2013            <p>Type of normalization layer applied to the continuous features. Options are: 'layernorm' and 'batchnorm'. if <code>None</code>, no normalization layer will be used.</p> </li> <li> <code>embed_continuous</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating if the continuous columns will be embedded using one of the available methods: 'standard', 'periodic' or 'piecewise'. If <code>None</code>, it will default to 'False'. NOTE: This parameter is deprecated and it  will be removed in future releases. Please, use the  <code>embed_continuous_method</code> parameter instead.</p> </li> <li> <code>embed_continuous_method</code>               (<code>Optional[Literal[standard, piecewise, periodic]]</code>, default:                   <code>None</code> )           \u2013            <p>Method to use to embed the continuous features. Options are: 'standard', 'periodic' or 'piecewise'. The 'standard' embedding method is based on the FT-Transformer implementation presented in the paper: Revisiting Deep Learning Models for Tabular Data. The 'periodic' and_'piecewise'_ methods were presented in the paper: On Embeddings for Numerical Features in Tabular Deep Learning. Please, read the papers for details.</p> </li> <li> <code>cont_embed_dim</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>Size of the continuous embeddings. If the continuous columns are embedded, <code>cont_embed_dim</code> must be passed.</p> </li> <li> <code>cont_embed_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p> </li> <li> <code>cont_embed_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the continuous embeddings if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported. If <code>None</code>, no activation function will be applied.</p> </li> <li> <code>quantization_setup</code>               (<code>Optional[Dict[str, List[float]]]</code>, default:                   <code>None</code> )           \u2013            <p>This parameter is used when the 'piecewise' method is used to embed the continuous cols. It is a dict where keys are the name of the continuous columns and values are lists with the boundaries for the quantization of the continuous_cols. See the examples for details. If If the 'piecewise' method is used, this parameter is required.</p> </li> <li> <code>n_frequencies</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>This is the so called 'k' in their paper On Embeddings for Numerical Features in Tabular Deep Learning, and is the number of 'frequencies' that will be used to represent each continuous column. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>sigma</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>This is the sigma parameter in the paper mentioned when describing the previous parameters and it is used to initialise the 'frequency weights'. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>share_last_layer</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>This parameter is not present in the before mentioned paper but it is implemented in the official repo. If <code>True</code> the linear layer that turns the frequencies into embeddings will be shared across the continuous columns. If <code>False</code> a different linear layer will be used for each continuous column. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>full_embed_dropout</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>If <code>True</code>, the full embedding corresponding to a column will be masked out/dropout. If <code>None</code>, it will default to <code>False</code>.</p> </li> <li> <code>n_steps</code>               (<code>int</code>, default:                   <code>3</code> )           \u2013            <p>number of decision steps. For a better understanding of the function of <code>n_steps</code> and the upcoming parameters, please see the paper.</p> </li> <li> <code>step_dim</code>               (<code>int</code>, default:                   <code>8</code> )           \u2013            <p>Step's output dimension. This is the output dimension that <code>WideDeep</code> will collect and connect to the output neuron(s).</p> </li> <li> <code>attn_dim</code>               (<code>int</code>, default:                   <code>8</code> )           \u2013            <p>Attention dimension</p> </li> <li> <code>dropout</code>               (<code>float</code>, default:                   <code>0.0</code> )           \u2013            <p>GLU block's internal dropout</p> </li> <li> <code>n_glu_step_dependent</code>               (<code>int</code>, default:                   <code>2</code> )           \u2013            <p>number of GLU Blocks (<code>[FC -&gt; BN -&gt; GLU]</code>) that are step dependent</p> </li> <li> <code>n_glu_shared</code>               (<code>int</code>, default:                   <code>2</code> )           \u2013            <p>number of GLU Blocks (<code>[FC -&gt; BN -&gt; GLU]</code>) that will be shared across decision steps</p> </li> <li> <code>ghost_bn</code>               (<code>bool</code>, default:                   <code>True</code> )           \u2013            <p>Boolean indicating if Ghost Batch Normalization will be used.</p> </li> <li> <code>virtual_batch_size</code>               (<code>int</code>, default:                   <code>128</code> )           \u2013            <p>Batch size when using Ghost Batch Normalization</p> </li> <li> <code>momentum</code>               (<code>float</code>, default:                   <code>0.02</code> )           \u2013            <p>Ghost Batch Normalization's momentum. The dreamquark-ai advises for very low values. However high values are used in the original publication. During our tests higher values lead to better results</p> </li> <li> <code>gamma</code>               (<code>float</code>, default:                   <code>1.3</code> )           \u2013            <p>Relaxation parameter in the paper. When gamma = 1, a feature is enforced to be used only at one decision step. As gamma increases, more flexibility is provided to use a feature at multiple decision steps</p> </li> <li> <code>epsilon</code>               (<code>float</code>, default:                   <code>1e-15</code> )           \u2013            <p>Float to avoid log(0). Always keep low</p> </li> <li> <code>mask_type</code>               (<code>str</code>, default:                   <code>'sparsemax'</code> )           \u2013            <p>Mask function to use. Either 'sparsemax' or 'entmax'</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>encoder</code>               (<code>Module</code>)           \u2013            <p>the TabNet encoder. For details see the original publication.</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n&gt;&gt;&gt; colnames = [\"a\", \"b\", \"c\", \"d\", \"e\"]\n&gt;&gt;&gt; cat_embed_input = [(u, i, j) for u, i, j in zip(colnames[:4], [4] * 4, [8] * 4)]\n&gt;&gt;&gt; column_idx = {k: v for v, k in enumerate(colnames)}\n&gt;&gt;&gt; model = TabNet(column_idx=column_idx, cat_embed_input=cat_embed_input, continuous_cols=[\"e\"])\n&gt;&gt;&gt; out = model(X_tab)\n</code></pre> Source code in <code>pytorch_widedeep/models/tabular/tabnet/tab_net.py</code> <pre><code>def __init__(\n    self,\n    column_idx: Dict[str, int],\n    *,\n    cat_embed_input: Optional[List[Tuple[str, int, int]]] = None,\n    cat_embed_dropout: Optional[float] = None,\n    use_cat_bias: Optional[bool] = None,\n    cat_embed_activation: Optional[str] = None,\n    continuous_cols: Optional[List[str]] = None,\n    cont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\n    embed_continuous: Optional[bool] = None,\n    embed_continuous_method: Optional[\n        Literal[\"standard\", \"piecewise\", \"periodic\"]\n    ] = None,\n    cont_embed_dim: Optional[int] = None,\n    cont_embed_dropout: Optional[float] = None,\n    cont_embed_activation: Optional[str] = None,\n    quantization_setup: Optional[Dict[str, List[float]]] = None,\n    n_frequencies: Optional[int] = None,\n    sigma: Optional[float] = None,\n    share_last_layer: Optional[bool] = None,\n    full_embed_dropout: Optional[bool] = None,\n    n_steps: int = 3,\n    step_dim: int = 8,\n    attn_dim: int = 8,\n    dropout: float = 0.0,\n    n_glu_step_dependent: int = 2,\n    n_glu_shared: int = 2,\n    ghost_bn: bool = True,\n    virtual_batch_size: int = 128,\n    momentum: float = 0.02,\n    gamma: float = 1.3,\n    epsilon: float = 1e-15,\n    mask_type: str = \"sparsemax\",\n):\n    super(TabNet, self).__init__(\n        column_idx=column_idx,\n        cat_embed_input=cat_embed_input,\n        cat_embed_dropout=cat_embed_dropout,\n        use_cat_bias=use_cat_bias,\n        cat_embed_activation=cat_embed_activation,\n        continuous_cols=continuous_cols,\n        cont_norm_layer=cont_norm_layer,\n        embed_continuous=embed_continuous,\n        embed_continuous_method=embed_continuous_method,\n        cont_embed_dim=cont_embed_dim,\n        cont_embed_dropout=cont_embed_dropout,\n        cont_embed_activation=cont_embed_activation,\n        quantization_setup=quantization_setup,\n        n_frequencies=n_frequencies,\n        sigma=sigma,\n        share_last_layer=share_last_layer,\n        full_embed_dropout=full_embed_dropout,\n    )\n\n    self.n_steps = n_steps\n    self.step_dim = step_dim\n    self.attn_dim = attn_dim\n    self.dropout = dropout\n    self.n_glu_step_dependent = n_glu_step_dependent\n    self.n_glu_shared = n_glu_shared\n    self.ghost_bn = ghost_bn\n    self.virtual_batch_size = virtual_batch_size\n    self.momentum = momentum\n    self.gamma = gamma\n    self.epsilon = epsilon\n    self.mask_type = mask_type\n\n    # Embeddings are instantiated at the base model\n    self.embed_out_dim = self.cat_out_dim + self.cont_out_dim\n\n    # TabNet\n    self.encoder = TabNetEncoder(\n        self.embed_out_dim,\n        n_steps,\n        step_dim,\n        attn_dim,\n        dropout,\n        n_glu_step_dependent,\n        n_glu_shared,\n        ghost_bn,\n        virtual_batch_size,\n        momentum,\n        gamma,\n        epsilon,\n        mask_type,\n    )\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.tabnet.tab_net.TabNet.output_dim","title":"output_dim  <code>property</code>","text":"<pre><code>output_dim\n</code></pre> <p>The output dimension of the model. This is a required property neccesary to build the <code>WideDeep</code> class</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.tabnet.tab_net.TabNetDecoder","title":"TabNetDecoder","text":"<pre><code>TabNetDecoder(\n    embed_dim,\n    n_steps=3,\n    step_dim=8,\n    dropout=0.0,\n    n_glu_step_dependent=2,\n    n_glu_shared=2,\n    ghost_bn=True,\n    virtual_batch_size=128,\n    momentum=0.02,\n)\n</code></pre> <p>               Bases: <code>Module</code></p> <p>Companion decoder model for the <code>TabNet</code> model (which can be considered an encoder itself)</p> <p>This class is designed to be used with the <code>EncoderDecoderTrainer</code> when using self-supervised pre-training (see the corresponding section in the docs). This class will receive the output from the <code>TabNet</code> encoder (i.e. the output from the so called 'steps') and 'reconstruct' the embeddings.</p> <p>Parameters:</p> <ul> <li> <code>embed_dim</code>               (<code>int</code>)           \u2013            <p>Size of the embeddings tensor to be reconstructed.</p> </li> <li> <code>n_steps</code>               (<code>int</code>, default:                   <code>3</code> )           \u2013            <p>number of decision steps. For a better understanding of the function of <code>n_steps</code> and the upcoming parameters, please see the paper.</p> </li> <li> <code>step_dim</code>               (<code>int</code>, default:                   <code>8</code> )           \u2013            <p>Step's output dimension. This is the output dimension that <code>WideDeep</code> will collect and connect to the output neuron(s).</p> </li> <li> <code>dropout</code>               (<code>float</code>, default:                   <code>0.0</code> )           \u2013            <p>GLU block's internal dropout</p> </li> <li> <code>n_glu_step_dependent</code>               (<code>int</code>, default:                   <code>2</code> )           \u2013            <p>number of GLU Blocks (<code>[FC -&gt; BN -&gt; GLU]</code>) that are step dependent</p> </li> <li> <code>n_glu_shared</code>               (<code>int</code>, default:                   <code>2</code> )           \u2013            <p>number of GLU Blocks (<code>[FC -&gt; BN -&gt; GLU]</code>) that will be shared across decision steps</p> </li> <li> <code>ghost_bn</code>               (<code>bool</code>, default:                   <code>True</code> )           \u2013            <p>Boolean indicating if Ghost Batch Normalization will be used.</p> </li> <li> <code>virtual_batch_size</code>               (<code>int</code>, default:                   <code>128</code> )           \u2013            <p>Batch size when using Ghost Batch Normalization</p> </li> <li> <code>momentum</code>               (<code>float</code>, default:                   <code>0.02</code> )           \u2013            <p>Ghost Batch Normalization's momentum. The dreamquark-ai advises for very low values. However high values are used in the original publication. During our tests higher values lead to better results</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>decoder</code>               (<code>Module</code>)           \u2013            <p>decoder that will receive the output from the encoder's steps and will reconstruct the embeddings</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import TabNetDecoder\n&gt;&gt;&gt; x_inp = [torch.rand(3, 8), torch.rand(3, 8), torch.rand(3, 8)]\n&gt;&gt;&gt; decoder = TabNetDecoder(embed_dim=32, ghost_bn=False)\n&gt;&gt;&gt; res = decoder(x_inp)\n&gt;&gt;&gt; res.shape\ntorch.Size([3, 32])\n</code></pre> Source code in <code>pytorch_widedeep/models/tabular/tabnet/tab_net.py</code> <pre><code>def __init__(\n    self,\n    embed_dim: int,\n    n_steps: int = 3,\n    step_dim: int = 8,\n    dropout: float = 0.0,\n    n_glu_step_dependent: int = 2,\n    n_glu_shared: int = 2,\n    ghost_bn: bool = True,\n    virtual_batch_size: int = 128,\n    momentum: float = 0.02,\n):\n    super(TabNetDecoder, self).__init__()\n\n    self.n_steps = n_steps\n    self.step_dim = step_dim\n    self.dropout = dropout\n    self.n_glu_step_dependent = n_glu_step_dependent\n    self.n_glu_shared = n_glu_shared\n    self.ghost_bn = ghost_bn\n    self.virtual_batch_size = virtual_batch_size\n    self.momentum = momentum\n\n    shared_layers = nn.ModuleList()\n    for i in range(n_glu_shared):\n        if i == 0:\n            shared_layers.append(nn.Linear(step_dim, 2 * step_dim, bias=False))\n        else:\n            shared_layers.append(nn.Linear(step_dim, 2 * step_dim, bias=False))\n\n    self.decoder = nn.ModuleList()\n    for step in range(n_steps):\n        transformer = FeatTransformer(\n            step_dim,\n            step_dim,\n            dropout,\n            shared_layers,\n            n_glu_step_dependent,\n            ghost_bn,\n            virtual_batch_size,\n            momentum=momentum,\n        )\n        self.decoder.append(transformer)\n\n    self.reconstruction_layer = nn.Linear(step_dim, embed_dim, bias=False)\n    initialize_non_glu(self.reconstruction_layer, step_dim, embed_dim)\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.mlp.context_attention_mlp.ContextAttentionMLP","title":"ContextAttentionMLP","text":"<pre><code>ContextAttentionMLP(\n    column_idx,\n    *,\n    cat_embed_input=None,\n    cat_embed_dropout=None,\n    use_cat_bias=None,\n    cat_embed_activation=None,\n    shared_embed=None,\n    add_shared_embed=None,\n    frac_shared_embed=None,\n    continuous_cols=None,\n    cont_norm_layer=None,\n    embed_continuous_method=\"standard\",\n    cont_embed_dropout=None,\n    cont_embed_activation=None,\n    quantization_setup=None,\n    n_frequencies=None,\n    sigma=None,\n    share_last_layer=None,\n    full_embed_dropout=None,\n    input_dim=32,\n    attn_dropout=0.2,\n    with_addnorm=False,\n    attn_activation=\"leaky_relu\",\n    n_blocks=3\n)\n</code></pre> <p>               Bases: <code>BaseTabularModelWithAttention</code></p> <p>Defines a <code>ContextAttentionMLP</code> model that can be used as the <code>deeptabular</code> component of a Wide &amp; Deep model or independently by itself.</p> <p>This class combines embedding representations of the categorical features with numerical (aka continuous) features that are also embedded. These are then passed through a series of attention blocks. Each attention block is comprised by a <code>ContextAttentionEncoder</code>. Such encoder is in part inspired by the attention mechanism described in Hierarchical Attention Networks for Document Classification. See <code>pytorch_widedeep.models.tabular.mlp._attention_layers</code> for details.</p> <p>Most of the parameters for this class are <code>Optional</code> since the use of categorical or continuous is in fact optional (i.e. one can use categorical features only, continuous features only or both).</p> <p>Parameters:</p> <ul> <li> <code>column_idx</code>               (<code>Dict[str, int]</code>)           \u2013            <p>Dict containing the index of the columns that will be passed through the <code>TabMlp</code> model. Required to slice the tensors. e.g. {'education': 0, 'relationship': 1, 'workclass': 2, ...}.</p> </li> <li> <code>cat_embed_input</code>               (<code>Optional[List[Tuple[str, int]]]</code>, default:                   <code>None</code> )           \u2013            <p>List of Tuples with the column name and number of unique values and embedding dimension. e.g. [(education, 11), ...]</p> </li> <li> <code>cat_embed_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Categorical embeddings dropout. If <code>None</code>, it will default to 0.</p> </li> <li> <code>use_cat_bias</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating if bias will be used for the categorical embeddings. If <code>None</code>, it will default to 'False'.</p> </li> <li> <code>cat_embed_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the categorical embeddings, if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> </li> <li> <code>shared_embed</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating if the embeddings will be \"shared\". The idea behind <code>shared_embed</code> is described in the Appendix A in the TabTransformer paper: 'The goal of having column embedding is to enable the model to distinguish the classes in one column from those in the other columns'. In other words, the idea is to let the model learn which column is embedded at the time. See: <code>pytorch_widedeep.models.transformers._layers.SharedEmbeddings</code>.</p> </li> <li> <code>add_shared_embed</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>The two embedding sharing strategies are: 1) add the shared embeddings to the column embeddings or 2) to replace the first <code>frac_shared_embed</code> with the shared embeddings. See <code>pytorch_widedeep.models.embeddings_layers.SharedEmbeddings</code> If 'None' is passed, it will default to 'False'.</p> </li> <li> <code>frac_shared_embed</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>The fraction of embeddings that will be shared (if <code>add_shared_embed = False</code>) by all the different categories for one particular column. If 'None' is passed, it will default to 0.0.</p> </li> <li> <code>continuous_cols</code>               (<code>Optional[List[str]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the name of the numeric (aka continuous) columns</p> </li> <li> <code>cont_norm_layer</code>               (<code>Optional[Literal[batchnorm, layernorm]]</code>, default:                   <code>None</code> )           \u2013            <p>Type of normalization layer applied to the continuous features. Options are: 'layernorm' and 'batchnorm'. if <code>None</code>, no normalization layer will be used.</p> </li> <li> <code>embed_continuous_method</code>               (<code>Optional[Literal[standard, piecewise, periodic]]</code>, default:                   <code>'standard'</code> )           \u2013            <p>Method to use to embed the continuous features. Options are: 'standard', 'periodic' or 'piecewise'. The 'standard' embedding method is based on the FT-Transformer implementation presented in the paper: Revisiting Deep Learning Models for Tabular Data. The 'periodic' and_'piecewise'_ methods were presented in the paper: On Embeddings for Numerical Features in Tabular Deep Learning. Please, read the papers for details.</p> </li> <li> <code>cont_embed_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p> </li> <li> <code>cont_embed_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the continuous embeddings if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported. If <code>None</code>, no activation function will be applied.</p> </li> <li> <code>quantization_setup</code>               (<code>Optional[Dict[str, List[float]]]</code>, default:                   <code>None</code> )           \u2013            <p>This parameter is used when the 'piecewise' method is used to embed the continuous cols. It is a dict where keys are the name of the continuous columns and values are lists with the boundaries for the quantization of the continuous_cols. See the examples for details. If If the 'piecewise' method is used, this parameter is required.</p> </li> <li> <code>n_frequencies</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>This is the so called 'k' in their paper On Embeddings for Numerical Features in Tabular Deep Learning, and is the number of 'frequencies' that will be used to represent each continuous column. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>sigma</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>This is the sigma parameter in the paper mentioned when describing the previous parameters and it is used to initialise the 'frequency weights'. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>share_last_layer</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>This parameter is not present in the before mentioned paper but it is implemented in the official repo. If <code>True</code> the linear layer that turns the frequencies into embeddings will be shared across the continuous columns. If <code>False</code> a different linear layer will be used for each continuous column. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>full_embed_dropout</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>If <code>True</code>, the full embedding corresponding to a column will be masked out/dropout. If <code>None</code>, it will default to <code>False</code>.</p> </li> <li> <code>input_dim</code>               (<code>int</code>, default:                   <code>32</code> )           \u2013            <p>The so-called dimension of the model. Is the number of embeddings used to encode the categorical and/or continuous columns</p> </li> <li> <code>attn_dropout</code>               (<code>float</code>, default:                   <code>0.2</code> )           \u2013            <p>Dropout for each attention block</p> </li> <li> <code>with_addnorm</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating if residual connections will be used in the attention blocks</p> </li> <li> <code>attn_activation</code>               (<code>str</code>, default:                   <code>'leaky_relu'</code> )           \u2013            <p>String indicating the activation function to be applied to the dense layer in each attention encoder. 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported.</p> </li> <li> <code>n_blocks</code>               (<code>int</code>, default:                   <code>3</code> )           \u2013            <p>Number of attention blocks</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>encoder</code>               (<code>Module</code>)           \u2013            <p>Sequence of attention encoders.</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import ContextAttentionMLP\n&gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n&gt;&gt;&gt; colnames = ['a', 'b', 'c', 'd', 'e']\n&gt;&gt;&gt; cat_embed_input = [(u,i,j) for u,i,j in zip(colnames[:4], [4]*4, [8]*4)]\n&gt;&gt;&gt; column_idx = {k:v for v,k in enumerate(colnames)}\n&gt;&gt;&gt; model = ContextAttentionMLP(column_idx=column_idx, cat_embed_input=cat_embed_input, continuous_cols = ['e'])\n&gt;&gt;&gt; out = model(X_tab)\n</code></pre> Source code in <code>pytorch_widedeep/models/tabular/mlp/context_attention_mlp.py</code> <pre><code>def __init__(\n    self,\n    column_idx: Dict[str, int],\n    *,\n    cat_embed_input: Optional[List[Tuple[str, int]]] = None,\n    cat_embed_dropout: Optional[float] = None,\n    use_cat_bias: Optional[bool] = None,\n    cat_embed_activation: Optional[str] = None,\n    shared_embed: Optional[bool] = None,\n    add_shared_embed: Optional[bool] = None,\n    frac_shared_embed: Optional[float] = None,\n    continuous_cols: Optional[List[str]] = None,\n    cont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\n    embed_continuous_method: Optional[\n        Literal[\"standard\", \"piecewise\", \"periodic\"]\n    ] = \"standard\",\n    cont_embed_dropout: Optional[float] = None,\n    cont_embed_activation: Optional[str] = None,\n    quantization_setup: Optional[Dict[str, List[float]]] = None,\n    n_frequencies: Optional[int] = None,\n    sigma: Optional[float] = None,\n    share_last_layer: Optional[bool] = None,\n    full_embed_dropout: Optional[bool] = None,\n    input_dim: int = 32,\n    attn_dropout: float = 0.2,\n    with_addnorm: bool = False,\n    attn_activation: str = \"leaky_relu\",\n    n_blocks: int = 3,\n):\n    super(ContextAttentionMLP, self).__init__(\n        column_idx=column_idx,\n        cat_embed_input=cat_embed_input,\n        cat_embed_dropout=cat_embed_dropout,\n        use_cat_bias=use_cat_bias,\n        cat_embed_activation=cat_embed_activation,\n        shared_embed=shared_embed,\n        add_shared_embed=add_shared_embed,\n        frac_shared_embed=frac_shared_embed,\n        continuous_cols=continuous_cols,\n        cont_norm_layer=cont_norm_layer,\n        embed_continuous_method=embed_continuous_method,\n        cont_embed_dropout=cont_embed_dropout,\n        cont_embed_activation=cont_embed_activation,\n        input_dim=input_dim,\n        quantization_setup=quantization_setup,\n        n_frequencies=n_frequencies,\n        sigma=sigma,\n        share_last_layer=share_last_layer,\n        full_embed_dropout=full_embed_dropout,\n    )\n\n    self.attn_dropout = attn_dropout\n    self.with_addnorm = with_addnorm\n    self.attn_activation = attn_activation\n    self.n_blocks = n_blocks\n\n    self.with_cls_token = \"cls_token\" in column_idx\n    self.n_cat = len(cat_embed_input) if cat_embed_input is not None else 0\n    self.n_cont = len(continuous_cols) if continuous_cols is not None else 0\n\n    # Embeddings are instantiated at the base model\n    # Attention Blocks\n    self.encoder = nn.Sequential()\n    for i in range(n_blocks):\n        self.encoder.add_module(\n            \"attention_block\" + str(i),\n            ContextAttentionEncoder(\n                input_dim,\n                attn_dropout,\n                with_addnorm,\n                attn_activation,\n            ),\n        )\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.mlp.context_attention_mlp.ContextAttentionMLP.output_dim","title":"output_dim  <code>property</code>","text":"<pre><code>output_dim\n</code></pre> <p>The output dimension of the model. This is a required property neccesary to build the <code>WideDeep</code> class</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.mlp.context_attention_mlp.ContextAttentionMLP.attention_weights","title":"attention_weights  <code>property</code>","text":"<pre><code>attention_weights\n</code></pre> <p>List with the attention weights per block</p> <p>The shape of the attention weights is \\((N, F)\\), where \\(N\\) is the batch size and \\(F\\) is the number of features/columns in the dataset</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.mlp.self_attention_mlp.SelfAttentionMLP","title":"SelfAttentionMLP","text":"<pre><code>SelfAttentionMLP(\n    column_idx,\n    *,\n    cat_embed_input=None,\n    cat_embed_dropout=None,\n    use_cat_bias=None,\n    cat_embed_activation=None,\n    shared_embed=None,\n    add_shared_embed=None,\n    frac_shared_embed=None,\n    continuous_cols=None,\n    cont_norm_layer=None,\n    embed_continuous_method=\"standard\",\n    cont_embed_dropout=None,\n    cont_embed_activation=None,\n    quantization_setup=None,\n    n_frequencies=None,\n    sigma=None,\n    share_last_layer=None,\n    full_embed_dropout=None,\n    input_dim=32,\n    attn_dropout=0.2,\n    n_heads=8,\n    use_bias=False,\n    with_addnorm=False,\n    attn_activation=\"leaky_relu\",\n    n_blocks=3\n)\n</code></pre> <p>               Bases: <code>BaseTabularModelWithAttention</code></p> <p>Defines a <code>SelfAttentionMLP</code> model that can be used as the deeptabular component of a Wide &amp; Deep model or independently by itself.</p> <p>This class combines embedding representations of the categorical features with numerical (aka continuous) features that are also embedded. These are then passed through a series of attention blocks. Each attention block is comprised by what we would refer as a simplified <code>SelfAttentionEncoder</code>. See <code>pytorch_widedeep.models.tabular.mlp._attention_layers</code> for details. The reason to use a simplified version of self attention is because we observed that the 'standard' attention mechanism used in the TabTransformer has a notable tendency to overfit.</p> <p>In more detail, this model only uses Q and K (and not V). If we think about it as in terms of text (and intuitively), the Softmax(QK^T) is the attention mechanism that tells us how much, at each position in the input sentence, each word is represented or 'expressed'. We refer to that as 'attention weights'. These attention weighst are normally multiplied by a Value matrix to further strength the focus on the words that each word should be attending to (again, intuitively).</p> <p>In this implementation we skip this last multiplication and instead we multiply the attention weights directly by the input tensor. This is a simplification that we expect is beneficial in terms of avoiding overfitting for tabular data.</p> <p>Most of the parameters for this class are <code>Optional</code> since the use of categorical or continuous is in fact optional (i.e. one can use categorical features only, continuous features only or both).</p> <p>Parameters:</p> <ul> <li> <code>column_idx</code>               (<code>Dict[str, int]</code>)           \u2013            <p>Dict containing the index of the columns that will be passed through the <code>TabMlp</code> model. Required to slice the tensors. e.g. {'education': 0, 'relationship': 1, 'workclass': 2, ...}.</p> </li> <li> <code>cat_embed_input</code>               (<code>Optional[List[Tuple[str, int]]]</code>, default:                   <code>None</code> )           \u2013            <p>List of Tuples with the column name and number of unique values and embedding dimension. e.g. [(education, 11), ...]</p> </li> <li> <code>cat_embed_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Categorical embeddings dropout. If <code>None</code>, it will default to 0.</p> </li> <li> <code>use_cat_bias</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating if bias will be used for the categorical embeddings. If <code>None</code>, it will default to 'False'.</p> </li> <li> <code>cat_embed_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the categorical embeddings, if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> </li> <li> <code>shared_embed</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating if the embeddings will be \"shared\". The idea behind <code>shared_embed</code> is described in the Appendix A in the TabTransformer paper: 'The goal of having column embedding is to enable the model to distinguish the classes in one column from those in the other columns'. In other words, the idea is to let the model learn which column is embedded at the time. See: <code>pytorch_widedeep.models.transformers._layers.SharedEmbeddings</code>.</p> </li> <li> <code>add_shared_embed</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>The two embedding sharing strategies are: 1) add the shared embeddings to the column embeddings or 2) to replace the first <code>frac_shared_embed</code> with the shared embeddings. See <code>pytorch_widedeep.models.embeddings_layers.SharedEmbeddings</code> If 'None' is passed, it will default to 'False'.</p> </li> <li> <code>frac_shared_embed</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>The fraction of embeddings that will be shared (if <code>add_shared_embed = False</code>) by all the different categories for one particular column. If 'None' is passed, it will default to 0.0.</p> </li> <li> <code>continuous_cols</code>               (<code>Optional[List[str]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the name of the numeric (aka continuous) columns</p> </li> <li> <code>cont_norm_layer</code>               (<code>Optional[Literal[batchnorm, layernorm]]</code>, default:                   <code>None</code> )           \u2013            <p>Type of normalization layer applied to the continuous features. Options are: 'layernorm' and 'batchnorm'. if <code>None</code>, no normalization layer will be used.</p> </li> <li> <code>embed_continuous_method</code>               (<code>Optional[Literal[standard, piecewise, periodic]]</code>, default:                   <code>'standard'</code> )           \u2013            <p>Method to use to embed the continuous features. Options are: 'standard', 'periodic' or 'piecewise'. The 'standard' embedding method is based on the FT-Transformer implementation presented in the paper: Revisiting Deep Learning Models for Tabular Data. The 'periodic' and_'piecewise'_ methods were presented in the paper: On Embeddings for Numerical Features in Tabular Deep Learning. Please, read the papers for details.</p> </li> <li> <code>cont_embed_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p> </li> <li> <code>cont_embed_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the continuous embeddings if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported. If <code>None</code>, no activation function will be applied.</p> </li> <li> <code>quantization_setup</code>               (<code>Optional[Dict[str, List[float]]]</code>, default:                   <code>None</code> )           \u2013            <p>This parameter is used when the 'piecewise' method is used to embed the continuous cols. It is a dict where keys are the name of the continuous columns and values are lists with the boundaries for the quantization of the continuous_cols. See the examples for details. If If the 'piecewise' method is used, this parameter is required.</p> </li> <li> <code>n_frequencies</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>This is the so called 'k' in their paper On Embeddings for Numerical Features in Tabular Deep Learning, and is the number of 'frequencies' that will be used to represent each continuous column. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>sigma</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>This is the sigma parameter in the paper mentioned when describing the previous parameters and it is used to initialise the 'frequency weights'. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>share_last_layer</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>This parameter is not present in the before mentioned paper but it is implemented in the official repo. If <code>True</code> the linear layer that turns the frequencies into embeddings will be shared across the continuous columns. If <code>False</code> a different linear layer will be used for each continuous column. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>full_embed_dropout</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>If <code>True</code>, the full embedding corresponding to a column will be masked out/dropout. If <code>None</code>, it will default to <code>False</code>.</p> </li> <li> <code>input_dim</code>               (<code>int</code>, default:                   <code>32</code> )           \u2013            <p>The so-called dimension of the model. Is the number of embeddings used to encode the categorical and/or continuous columns</p> </li> <li> <code>attn_dropout</code>               (<code>float</code>, default:                   <code>0.2</code> )           \u2013            <p>Dropout for each attention block</p> </li> <li> <code>n_heads</code>               (<code>int</code>, default:                   <code>8</code> )           \u2013            <p>Number of attention heads per attention block.</p> </li> <li> <code>use_bias</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether or not to use bias in the Q, K projection layers.</p> </li> <li> <code>with_addnorm</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating if residual connections will be used in the attention blocks</p> </li> <li> <code>attn_activation</code>               (<code>str</code>, default:                   <code>'leaky_relu'</code> )           \u2013            <p>String indicating the activation function to be applied to the dense layer in each attention encoder. 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported.</p> </li> <li> <code>n_blocks</code>               (<code>int</code>, default:                   <code>3</code> )           \u2013            <p>Number of attention blocks</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>cat_and_cont_embed</code>               (<code>Module</code>)           \u2013            <p>This is the module that processes the categorical and continuous columns</p> </li> <li> <code>encoder</code>               (<code>Module</code>)           \u2013            <p>Sequence of attention encoders.</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import SelfAttentionMLP\n&gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n&gt;&gt;&gt; colnames = ['a', 'b', 'c', 'd', 'e']\n&gt;&gt;&gt; cat_embed_input = [(u,i,j) for u,i,j in zip(colnames[:4], [4]*4, [8]*4)]\n&gt;&gt;&gt; column_idx = {k:v for v,k in enumerate(colnames)}\n&gt;&gt;&gt; model = SelfAttentionMLP(column_idx=column_idx, cat_embed_input=cat_embed_input, continuous_cols = ['e'])\n&gt;&gt;&gt; out = model(X_tab)\n</code></pre> Source code in <code>pytorch_widedeep/models/tabular/mlp/self_attention_mlp.py</code> <pre><code>def __init__(\n    self,\n    column_idx: Dict[str, int],\n    *,\n    cat_embed_input: Optional[List[Tuple[str, int]]] = None,\n    cat_embed_dropout: Optional[float] = None,\n    use_cat_bias: Optional[bool] = None,\n    cat_embed_activation: Optional[str] = None,\n    shared_embed: Optional[bool] = None,\n    add_shared_embed: Optional[bool] = None,\n    frac_shared_embed: Optional[float] = None,\n    continuous_cols: Optional[List[str]] = None,\n    cont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\n    embed_continuous_method: Optional[\n        Literal[\"standard\", \"piecewise\", \"periodic\"]\n    ] = \"standard\",\n    cont_embed_dropout: Optional[float] = None,\n    cont_embed_activation: Optional[str] = None,\n    quantization_setup: Optional[Dict[str, List[float]]] = None,\n    n_frequencies: Optional[int] = None,\n    sigma: Optional[float] = None,\n    share_last_layer: Optional[bool] = None,\n    full_embed_dropout: Optional[bool] = None,\n    input_dim: int = 32,\n    attn_dropout: float = 0.2,\n    n_heads: int = 8,\n    use_bias: bool = False,\n    with_addnorm: bool = False,\n    attn_activation: str = \"leaky_relu\",\n    n_blocks: int = 3,\n):\n    super(SelfAttentionMLP, self).__init__(\n        column_idx=column_idx,\n        input_dim=input_dim,\n        cat_embed_input=cat_embed_input,\n        cat_embed_dropout=cat_embed_dropout,\n        use_cat_bias=use_cat_bias,\n        cat_embed_activation=cat_embed_activation,\n        shared_embed=shared_embed,\n        add_shared_embed=add_shared_embed,\n        frac_shared_embed=frac_shared_embed,\n        continuous_cols=continuous_cols,\n        cont_norm_layer=cont_norm_layer,\n        embed_continuous_method=embed_continuous_method,\n        cont_embed_dropout=cont_embed_dropout,\n        cont_embed_activation=cont_embed_activation,\n        quantization_setup=quantization_setup,\n        n_frequencies=n_frequencies,\n        sigma=sigma,\n        share_last_layer=share_last_layer,\n        full_embed_dropout=full_embed_dropout,\n    )\n\n    self.attn_dropout = attn_dropout\n    self.n_heads = n_heads\n    self.use_bias = use_bias\n    self.with_addnorm = with_addnorm\n    self.attn_activation = attn_activation\n    self.n_blocks = n_blocks\n\n    self.with_cls_token = \"cls_token\" in column_idx\n    self.n_cat = len(cat_embed_input) if cat_embed_input is not None else 0\n    self.n_cont = len(continuous_cols) if continuous_cols is not None else 0\n\n    # Embeddings are instantiated at the base model\n    # Attention Blocks\n    self.encoder = nn.Sequential()\n    for i in range(n_blocks):\n        self.encoder.add_module(\n            \"attention_block\" + str(i),\n            SelfAttentionEncoder(\n                input_dim,\n                attn_dropout,\n                use_bias,\n                n_heads,\n                with_addnorm,\n                attn_activation,\n            ),\n        )\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.mlp.self_attention_mlp.SelfAttentionMLP.output_dim","title":"output_dim  <code>property</code>","text":"<pre><code>output_dim\n</code></pre> <p>The output dimension of the model. This is a required property neccesary to build the WideDeep class</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.mlp.self_attention_mlp.SelfAttentionMLP.attention_weights","title":"attention_weights  <code>property</code>","text":"<pre><code>attention_weights\n</code></pre> <p>List with the attention weights per block</p> <p>The shape of the attention weights is \\((N, H, F, F)\\), where \\(N\\) is the batch size, \\(H\\) is the number of attention heads and \\(F\\) is the number of features/columns in the dataset</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.transformers.tab_transformer.TabTransformer","title":"TabTransformer","text":"<pre><code>TabTransformer(\n    column_idx,\n    *,\n    cat_embed_input=None,\n    cat_embed_dropout=None,\n    use_cat_bias=None,\n    cat_embed_activation=None,\n    shared_embed=None,\n    add_shared_embed=None,\n    frac_shared_embed=None,\n    continuous_cols=None,\n    cont_norm_layer=None,\n    embed_continuous=None,\n    embed_continuous_method=None,\n    cont_embed_dropout=None,\n    cont_embed_activation=None,\n    quantization_setup=None,\n    n_frequencies=None,\n    sigma=None,\n    share_last_layer=None,\n    full_embed_dropout=None,\n    input_dim=32,\n    n_heads=8,\n    use_qkv_bias=False,\n    n_blocks=4,\n    attn_dropout=0.2,\n    ff_dropout=0.1,\n    ff_factor=4,\n    transformer_activation=\"gelu\",\n    use_linear_attention=False,\n    use_flash_attention=False,\n    mlp_hidden_dims=None,\n    mlp_activation=\"relu\",\n    mlp_dropout=0.1,\n    mlp_batchnorm=False,\n    mlp_batchnorm_last=False,\n    mlp_linear_first=True\n)\n</code></pre> <p>               Bases: <code>BaseTabularModelWithAttention</code></p> <p>Defines our adptation of the TabTransformer model that can be used as the <code>deeptabular</code> component of a Wide &amp; Deep model or independently by itself.</p> <p>Most of the parameters for this class are <code>Optional</code> since the use of categorical or continuous is in fact optional (i.e. one can use categorical features only, continuous features only or both).</p> <p> NOTE: This is an enhanced adaptation of the model described in the paper. It can be considered as the flagship of our transformer family of models for tabular data and offers mutiple, additional features relative to the original publication(and some other models in the library)</p> <p>Parameters:</p> <ul> <li> <code>column_idx</code>               (<code>Dict[str, int]</code>)           \u2013            <p>Dict containing the index of the columns that will be passed through the <code>TabMlp</code> model. Required to slice the tensors. e.g. {'education': 0, 'relationship': 1, 'workclass': 2, ...}.</p> </li> <li> <code>cat_embed_input</code>               (<code>Optional[List[Tuple[str, int]]]</code>, default:                   <code>None</code> )           \u2013            <p>List of Tuples with the column name and number of unique values and embedding dimension. e.g. [(education, 11), ...]</p> </li> <li> <code>cat_embed_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Categorical embeddings dropout. If <code>None</code>, it will default to 0.</p> </li> <li> <code>use_cat_bias</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating if bias will be used for the categorical embeddings. If <code>None</code>, it will default to 'False'.</p> </li> <li> <code>cat_embed_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the categorical embeddings, if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> </li> <li> <code>shared_embed</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating if the embeddings will be \"shared\". The idea behind <code>shared_embed</code> is described in the Appendix A in the TabTransformer paper: 'The goal of having column embedding is to enable the model to distinguish the classes in one column from those in the other columns'. In other words, the idea is to let the model learn which column is embedded at the time. See: <code>pytorch_widedeep.models.transformers._layers.SharedEmbeddings</code>.</p> </li> <li> <code>add_shared_embed</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>The two embedding sharing strategies are: 1) add the shared embeddings to the column embeddings or 2) to replace the first <code>frac_shared_embed</code> with the shared embeddings. See <code>pytorch_widedeep.models.embeddings_layers.SharedEmbeddings</code> If 'None' is passed, it will default to 'False'.</p> </li> <li> <code>frac_shared_embed</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>The fraction of embeddings that will be shared (if <code>add_shared_embed = False</code>) by all the different categories for one particular column. If 'None' is passed, it will default to 0.0.</p> </li> <li> <code>continuous_cols</code>               (<code>Optional[List[str]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the name of the numeric (aka continuous) columns</p> </li> <li> <code>cont_norm_layer</code>               (<code>Optional[Literal[batchnorm, layernorm]]</code>, default:                   <code>None</code> )           \u2013            <p>Type of normalization layer applied to the continuous features. Options are: 'layernorm' and 'batchnorm'. if <code>None</code>, no normalization layer will be used.</p> </li> <li> <code>embed_continuous_method</code>               (<code>Optional[Literal[standard, piecewise, periodic]]</code>, default:                   <code>None</code> )           \u2013            <p>Method to use to embed the continuous features. Options are: 'standard', 'periodic' or 'piecewise'. The 'standard' embedding method is based on the FT-Transformer implementation presented in the paper: Revisiting Deep Learning Models for Tabular Data. The 'periodic' and_'piecewise'_ methods were presented in the paper: On Embeddings for Numerical Features in Tabular Deep Learning. Please, read the papers for details.</p> </li> <li> <code>cont_embed_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p> </li> <li> <code>cont_embed_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the continuous embeddings if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported. If <code>None</code>, no activation function will be applied.</p> </li> <li> <code>quantization_setup</code>               (<code>Optional[Dict[str, List[float]]]</code>, default:                   <code>None</code> )           \u2013            <p>This parameter is used when the 'piecewise' method is used to embed the continuous cols. It is a dict where keys are the name of the continuous columns and values are lists with the boundaries for the quantization of the continuous_cols. See the examples for details. If If the 'piecewise' method is used, this parameter is required.</p> </li> <li> <code>n_frequencies</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>This is the so called 'k' in their paper On Embeddings for Numerical Features in Tabular Deep Learning, and is the number of 'frequencies' that will be used to represent each continuous column. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>sigma</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>This is the sigma parameter in the paper mentioned when describing the previous parameters and it is used to initialise the 'frequency weights'. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>share_last_layer</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>This parameter is not present in the before mentioned paper but it is implemented in the official repo. If <code>True</code> the linear layer that turns the frequencies into embeddings will be shared across the continuous columns. If <code>False</code> a different linear layer will be used for each continuous column. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>full_embed_dropout</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>If <code>True</code>, the full embedding corresponding to a column will be masked out/dropout. If <code>None</code>, it will default to <code>False</code>.</p> </li> <li> <code>input_dim</code>               (<code>int</code>, default:                   <code>32</code> )           \u2013            <p>The so-called dimension of the model. Is the number of embeddings used to encode the categorical and/or continuous columns</p> </li> <li> <code>n_heads</code>               (<code>int</code>, default:                   <code>8</code> )           \u2013            <p>Number of attention heads per Transformer block</p> </li> <li> <code>use_qkv_bias</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether or not to use bias in the Q, K, and V projection layers.</p> </li> <li> <code>n_blocks</code>               (<code>int</code>, default:                   <code>4</code> )           \u2013            <p>Number of Transformer blocks</p> </li> <li> <code>attn_dropout</code>               (<code>float</code>, default:                   <code>0.2</code> )           \u2013            <p>Dropout that will be applied to the Multi-Head Attention layers</p> </li> <li> <code>ff_dropout</code>               (<code>float</code>, default:                   <code>0.1</code> )           \u2013            <p>Dropout that will be applied to the FeedForward network</p> </li> <li> <code>ff_factor</code>               (<code>int</code>, default:                   <code>4</code> )           \u2013            <p>Multiplicative factor applied to the first layer of the FF network in each Transformer block, This is normally set to 4.</p> </li> <li> <code>transformer_activation</code>               (<code>str</code>, default:                   <code>'gelu'</code> )           \u2013            <p>Transformer Encoder activation function. 'tanh', 'relu', 'leaky_relu', 'gelu', 'geglu' and 'reglu' are supported</p> </li> <li> <code>use_linear_attention</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating if Linear Attention (from Transformers are RNNs: Fast Autoregressive Transformers with Linear Attention) will be used. The inclusing of this mode of attention is inspired by this post, where the Uber team finds that this attention mechanism leads to the best results for their tabular data.</p> </li> <li> <code>use_flash_attention</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating if Flash Attention will be used. </p> </li> <li> <code>mlp_hidden_dims</code>               (<code>Optional[List[int]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the number of neurons per dense layer in the MLP. e.g: [64, 32]. If not provided no MLP on top of the final Transformer block will be used.</p> </li> <li> <code>mlp_activation</code>               (<code>str</code>, default:                   <code>'relu'</code> )           \u2013            <p>Activation function for the dense layers of the MLP. Currently 'tanh', 'relu', 'leaky'_relu' and _'gelu' are supported. If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to 'relu'.</p> </li> <li> <code>mlp_dropout</code>               (<code>float</code>, default:                   <code>0.1</code> )           \u2013            <p>float with the dropout between the dense layers of the MLP. If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to 0.0.</p> </li> <li> <code>mlp_batchnorm</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether or not batch normalization will be applied to the dense layers If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to False.</p> </li> <li> <code>mlp_batchnorm_last</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether or not batch normalization will be applied to the last of the dense layers If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to False.</p> </li> <li> <code>mlp_linear_first</code>               (<code>bool</code>, default:                   <code>True</code> )           \u2013            <p>Boolean indicating the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code> If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to <code>True</code>.</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>encoder</code>               (<code>Module</code>)           \u2013            <p>Sequence of Transformer blocks</p> </li> <li> <code>mlp</code>               (<code>Module</code>)           \u2013            <p>MLP component in the model</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import TabTransformer\n&gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n&gt;&gt;&gt; colnames = ['a', 'b', 'c', 'd', 'e']\n&gt;&gt;&gt; cat_embed_input = [(u,i) for u,i in zip(colnames[:4], [4]*4)]\n&gt;&gt;&gt; continuous_cols = ['e']\n&gt;&gt;&gt; column_idx = {k:v for v,k in enumerate(colnames)}\n&gt;&gt;&gt; model = TabTransformer(column_idx=column_idx, cat_embed_input=cat_embed_input, continuous_cols=continuous_cols)\n&gt;&gt;&gt; out = model(X_tab)\n</code></pre> Source code in <code>pytorch_widedeep/models/tabular/transformers/tab_transformer.py</code> <pre><code>def __init__(\n    self,\n    column_idx: Dict[str, int],\n    *,\n    cat_embed_input: Optional[List[Tuple[str, int]]] = None,\n    cat_embed_dropout: Optional[float] = None,\n    use_cat_bias: Optional[bool] = None,\n    cat_embed_activation: Optional[str] = None,\n    shared_embed: Optional[bool] = None,\n    add_shared_embed: Optional[bool] = None,\n    frac_shared_embed: Optional[float] = None,\n    continuous_cols: Optional[List[str]] = None,\n    cont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\n    embed_continuous: Optional[bool] = None,\n    embed_continuous_method: Optional[\n        Literal[\"standard\", \"piecewise\", \"periodic\"]\n    ] = None,\n    cont_embed_dropout: Optional[float] = None,\n    cont_embed_activation: Optional[str] = None,\n    quantization_setup: Optional[Dict[str, List[float]]] = None,\n    n_frequencies: Optional[int] = None,\n    sigma: Optional[float] = None,\n    share_last_layer: Optional[bool] = None,\n    full_embed_dropout: Optional[bool] = None,\n    input_dim: int = 32,\n    n_heads: int = 8,\n    use_qkv_bias: bool = False,\n    n_blocks: int = 4,\n    attn_dropout: float = 0.2,\n    ff_dropout: float = 0.1,\n    ff_factor: int = 4,\n    transformer_activation: str = \"gelu\",\n    use_linear_attention: bool = False,\n    use_flash_attention: bool = False,\n    mlp_hidden_dims: Optional[List[int]] = None,\n    mlp_activation: str = \"relu\",\n    mlp_dropout: float = 0.1,\n    mlp_batchnorm: bool = False,\n    mlp_batchnorm_last: bool = False,\n    mlp_linear_first: bool = True,\n):\n    super(TabTransformer, self).__init__(\n        column_idx=column_idx,\n        cat_embed_input=cat_embed_input,\n        cat_embed_dropout=cat_embed_dropout,\n        use_cat_bias=use_cat_bias,\n        cat_embed_activation=cat_embed_activation,\n        shared_embed=shared_embed,\n        add_shared_embed=add_shared_embed,\n        frac_shared_embed=frac_shared_embed,\n        continuous_cols=continuous_cols,\n        cont_norm_layer=cont_norm_layer,\n        # UGLY hack to be able to use the base class __init__ method\n        embed_continuous_method=(\n            \"standard\"\n            if embed_continuous_method is None\n            else embed_continuous_method\n        ),\n        cont_embed_dropout=cont_embed_dropout,\n        cont_embed_activation=cont_embed_activation,\n        quantization_setup=quantization_setup,\n        n_frequencies=n_frequencies,\n        sigma=sigma,\n        share_last_layer=share_last_layer,\n        input_dim=input_dim,\n        full_embed_dropout=full_embed_dropout,\n    )\n\n    # we overwrite the embed_continuous_method parameter that was set\n    # to 'standard' if embed_continuous_method is None. In other words,\n    # this will already have the right value if the user provided a value\n    # for embed_continuous_method, otherwise will be \"standard\" when it\n    # should be None\n    self.embed_continuous_method = embed_continuous_method\n\n    if embed_continuous is not None:\n        self.embed_continuous = embed_continuous\n        if embed_continuous and embed_continuous_method is None:\n            raise ValueError(\n                \"If 'embed_continuous' is True, 'embed_continuous_method' must be \"\n                \"one of 'standard', 'piecewise' or 'periodic'.\"\n            )\n    elif self.embed_continuous_method is not None:\n        self.embed_continuous = True\n    else:\n        self.embed_continuous = False\n\n    self.n_heads = n_heads\n    self.use_qkv_bias = use_qkv_bias\n    self.n_blocks = n_blocks\n    self.attn_dropout = attn_dropout\n    self.ff_dropout = ff_dropout\n    self.transformer_activation = transformer_activation\n    self.use_linear_attention = use_linear_attention\n    self.use_flash_attention = use_flash_attention\n    self.ff_factor = ff_factor\n\n    self.mlp_hidden_dims = mlp_hidden_dims\n    self.mlp_activation = mlp_activation\n    self.mlp_dropout = mlp_dropout\n    self.mlp_batchnorm = mlp_batchnorm\n    self.mlp_batchnorm_last = mlp_batchnorm_last\n    self.mlp_linear_first = mlp_linear_first\n\n    self.with_cls_token = \"cls_token\" in column_idx\n    self.n_cat = len(cat_embed_input) if cat_embed_input is not None else 0\n    self.n_cont = len(continuous_cols) if continuous_cols is not None else 0\n\n    if self.n_cont and not self.n_cat and not self.embed_continuous:\n        raise ValueError(\n            \"If only continuous features are used 'embed_continuous' must be set to 'True'\"\n        )\n\n    # Embeddings are instantiated at the base model\n    # Transformer blocks\n    self.encoder = nn.Sequential()\n    for i in range(n_blocks):\n        self.encoder.add_module(\n            \"transformer_block\" + str(i),\n            TransformerEncoder(\n                input_dim,\n                n_heads,\n                use_qkv_bias,\n                attn_dropout,\n                ff_dropout,\n                ff_factor,\n                transformer_activation,\n                use_linear_attention,\n                use_flash_attention,\n            ),\n        )\n\n    self.mlp_first_hidden_dim = self._mlp_first_hidden_dim()\n\n    if self.mlp_hidden_dims is not None:\n        self.mlp = MLP(\n            d_hidden=[self.mlp_first_hidden_dim] + self.mlp_hidden_dims,\n            activation=(\n                \"relu\" if self.mlp_activation is None else self.mlp_activation\n            ),\n            dropout=0.0 if self.mlp_dropout is None else self.mlp_dropout,\n            batchnorm=False if self.mlp_batchnorm is None else self.mlp_batchnorm,\n            batchnorm_last=(\n                False\n                if self.mlp_batchnorm_last is None\n                else self.mlp_batchnorm_last\n            ),\n            linear_first=(\n                False if self.mlp_linear_first is None else self.mlp_linear_first\n            ),\n        )\n    else:\n        self.mlp = None\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.transformers.tab_transformer.TabTransformer.output_dim","title":"output_dim  <code>property</code>","text":"<pre><code>output_dim\n</code></pre> <p>The output dimension of the model. This is a required property neccesary to build the <code>WideDeep</code> class</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.transformers.tab_transformer.TabTransformer.attention_weights","title":"attention_weights  <code>property</code>","text":"<pre><code>attention_weights\n</code></pre> <p>List with the attention weights per block</p> <p>The shape of the attention weights is \\((N, H, F, F)\\), where \\(N\\) is the batch size, \\(H\\) is the number of attention heads and \\(F\\) is the number of features/columns in the dataset</p> <p> NOTE: if flash attention or linear attention are used, no attention weights are saved during the training process and calling this property will throw a ValueError</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.transformers.saint.SAINT","title":"SAINT","text":"<pre><code>SAINT(\n    column_idx,\n    *,\n    cat_embed_input=None,\n    cat_embed_dropout=None,\n    use_cat_bias=None,\n    cat_embed_activation=None,\n    shared_embed=None,\n    add_shared_embed=None,\n    frac_shared_embed=None,\n    continuous_cols=None,\n    cont_norm_layer=None,\n    embed_continuous_method=\"standard\",\n    cont_embed_dropout=None,\n    cont_embed_activation=None,\n    quantization_setup=None,\n    n_frequencies=None,\n    sigma=None,\n    share_last_layer=None,\n    full_embed_dropout=None,\n    input_dim=32,\n    use_qkv_bias=False,\n    n_heads=8,\n    n_blocks=2,\n    attn_dropout=0.1,\n    ff_dropout=0.2,\n    ff_factor=4,\n    transformer_activation=\"gelu\",\n    mlp_hidden_dims=None,\n    mlp_activation=None,\n    mlp_dropout=None,\n    mlp_batchnorm=None,\n    mlp_batchnorm_last=None,\n    mlp_linear_first=None\n)\n</code></pre> <p>               Bases: <code>BaseTabularModelWithAttention</code></p> <p>Defines a SAINT model that can be used as the <code>deeptabular</code> component of a Wide &amp; Deep model or independently by itself.</p> <p>Most of the parameters for this class are <code>Optional</code> since the use of categorical or continuous is in fact optional (i.e. one can use categorical features only, continuous features only or both).</p> <p> NOTE: This is an slightly modified and enhanced  version of the model described in the paper,</p> <p>Parameters:</p> <ul> <li> <code>column_idx</code>               (<code>Dict[str, int]</code>)           \u2013            <p>Dict containing the index of the columns that will be passed through the <code>TabMlp</code> model. Required to slice the tensors. e.g. {'education': 0, 'relationship': 1, 'workclass': 2, ...}.</p> </li> <li> <code>cat_embed_input</code>               (<code>Optional[List[Tuple[str, int]]]</code>, default:                   <code>None</code> )           \u2013            <p>List of Tuples with the column name and number of unique values and embedding dimension. e.g. [(education, 11), ...]</p> </li> <li> <code>cat_embed_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Categorical embeddings dropout. If <code>None</code>, it will default to 0.</p> </li> <li> <code>use_cat_bias</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating if bias will be used for the categorical embeddings. If <code>None</code>, it will default to 'False'.</p> </li> <li> <code>cat_embed_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the categorical embeddings, if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> </li> <li> <code>shared_embed</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating if the embeddings will be \"shared\". The idea behind <code>shared_embed</code> is described in the Appendix A in the TabTransformer paper: 'The goal of having column embedding is to enable the model to distinguish the classes in one column from those in the other columns'. In other words, the idea is to let the model learn which column is embedded at the time. See: <code>pytorch_widedeep.models.transformers._layers.SharedEmbeddings</code>.</p> </li> <li> <code>add_shared_embed</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>The two embedding sharing strategies are: 1) add the shared embeddings to the column embeddings or 2) to replace the first <code>frac_shared_embed</code> with the shared embeddings. See <code>pytorch_widedeep.models.embeddings_layers.SharedEmbeddings</code> If 'None' is passed, it will default to 'False'.</p> </li> <li> <code>frac_shared_embed</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>The fraction of embeddings that will be shared (if <code>add_shared_embed = False</code>) by all the different categories for one particular column. If 'None' is passed, it will default to 0.0.</p> </li> <li> <code>continuous_cols</code>               (<code>Optional[List[str]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the name of the numeric (aka continuous) columns</p> </li> <li> <code>cont_norm_layer</code>               (<code>Optional[Literal[batchnorm, layernorm]]</code>, default:                   <code>None</code> )           \u2013            <p>Type of normalization layer applied to the continuous features. Options are: 'layernorm' and 'batchnorm'. if <code>None</code>, no normalization layer will be used.</p> </li> <li> <code>embed_continuous_method</code>               (<code>Optional[Literal[standard, piecewise, periodic]]</code>, default:                   <code>'standard'</code> )           \u2013            <p>Method to use to embed the continuous features. Options are: 'standard', 'periodic' or 'piecewise'. The 'standard' embedding method is based on the FT-Transformer implementation presented in the paper: Revisiting Deep Learning Models for Tabular Data. The 'periodic' and_'piecewise'_ methods were presented in the paper: On Embeddings for Numerical Features in Tabular Deep Learning. Please, read the papers for details.</p> </li> <li> <code>cont_embed_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p> </li> <li> <code>cont_embed_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the continuous embeddings if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported. If <code>None</code>, no activation function will be applied.</p> </li> <li> <code>quantization_setup</code>               (<code>Optional[Dict[str, List[float]]]</code>, default:                   <code>None</code> )           \u2013            <p>This parameter is used when the 'piecewise' method is used to embed the continuous cols. It is a dict where keys are the name of the continuous columns and values are lists with the boundaries for the quantization of the continuous_cols. See the examples for details. If If the 'piecewise' method is used, this parameter is required.</p> </li> <li> <code>n_frequencies</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>This is the so called 'k' in their paper On Embeddings for Numerical Features in Tabular Deep Learning, and is the number of 'frequencies' that will be used to represent each continuous column. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>sigma</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>This is the sigma parameter in the paper mentioned when describing the previous parameters and it is used to initialise the 'frequency weights'. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>share_last_layer</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>This parameter is not present in the before mentioned paper but it is implemented in the official repo. If <code>True</code> the linear layer that turns the frequencies into embeddings will be shared across the continuous columns. If <code>False</code> a different linear layer will be used for each continuous column. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>full_embed_dropout</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>If <code>True</code>, the full embedding corresponding to a column will be masked out/dropout. If <code>None</code>, it will default to <code>False</code>.</p> </li> <li> <code>input_dim</code>               (<code>int</code>, default:                   <code>32</code> )           \u2013            <p>The so-called dimension of the model. Is the number of embeddings used to encode the categorical and/or continuous columns</p> </li> <li> <code>n_heads</code>               (<code>int</code>, default:                   <code>8</code> )           \u2013            <p>Number of attention heads per Transformer block</p> </li> <li> <code>use_qkv_bias</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether or not to use bias in the Q, K, and V projection layers</p> </li> <li> <code>n_blocks</code>               (<code>int</code>, default:                   <code>2</code> )           \u2013            <p>Number of SAINT-Transformer blocks.</p> </li> <li> <code>attn_dropout</code>               (<code>float</code>, default:                   <code>0.1</code> )           \u2013            <p>Dropout that will be applied to the Multi-Head Attention column and row layers</p> </li> <li> <code>ff_dropout</code>               (<code>float</code>, default:                   <code>0.2</code> )           \u2013            <p>Dropout that will be applied to the FeedForward network</p> </li> <li> <code>ff_factor</code>               (<code>int</code>, default:                   <code>4</code> )           \u2013            <p>Multiplicative factor applied to the first layer of the FF network in each Transformer block, This is normally set to 4.</p> </li> <li> <code>transformer_activation</code>               (<code>str</code>, default:                   <code>'gelu'</code> )           \u2013            <p>Transformer Encoder activation function. 'tanh', 'relu', 'leaky_relu', 'gelu', 'geglu' and 'reglu' are supported</p> </li> <li> <code>mlp_hidden_dims</code>               (<code>Optional[List[int]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the number of neurons per dense layer in the MLP. e.g: [64, 32]. If not provided no MLP on top of the final Transformer block will be used.</p> </li> <li> <code>mlp_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the dense layers of the MLP. Currently 'tanh', 'relu', 'leaky'_relu' and _'gelu' are supported. If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to 'relu'.</p> </li> <li> <code>mlp_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>float with the dropout between the dense layers of the MLP. If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to 0.0.</p> </li> <li> <code>mlp_batchnorm</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating whether or not batch normalization will be applied to the dense layers If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to False.</p> </li> <li> <code>mlp_batchnorm_last</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating whether or not batch normalization will be applied to the last of the dense layers If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to False.</p> </li> <li> <code>mlp_linear_first</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code> If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to <code>True</code>.</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>encoder</code>               (<code>Module</code>)           \u2013            <p>Sequence of SAINT-Transformer blocks</p> </li> <li> <code>mlp</code>               (<code>Module</code>)           \u2013            <p>MLP component in the model</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import SAINT\n&gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n&gt;&gt;&gt; colnames = ['a', 'b', 'c', 'd', 'e']\n&gt;&gt;&gt; cat_embed_input = [(u,i) for u,i in zip(colnames[:4], [4]*4)]\n&gt;&gt;&gt; continuous_cols = ['e']\n&gt;&gt;&gt; column_idx = {k:v for v,k in enumerate(colnames)}\n&gt;&gt;&gt; model = SAINT(column_idx=column_idx, cat_embed_input=cat_embed_input, continuous_cols=continuous_cols)\n&gt;&gt;&gt; out = model(X_tab)\n</code></pre> Source code in <code>pytorch_widedeep/models/tabular/transformers/saint.py</code> <pre><code>def __init__(\n    self,\n    column_idx: Dict[str, int],\n    *,\n    cat_embed_input: Optional[List[Tuple[str, int]]] = None,\n    cat_embed_dropout: Optional[float] = None,\n    use_cat_bias: Optional[bool] = None,\n    cat_embed_activation: Optional[str] = None,\n    shared_embed: Optional[bool] = None,\n    add_shared_embed: Optional[bool] = None,\n    frac_shared_embed: Optional[float] = None,\n    continuous_cols: Optional[List[str]] = None,\n    cont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\n    embed_continuous_method: Optional[\n        Literal[\"standard\", \"piecewise\", \"periodic\"]\n    ] = \"standard\",\n    cont_embed_dropout: Optional[float] = None,\n    cont_embed_activation: Optional[str] = None,\n    quantization_setup: Optional[Dict[str, List[float]]] = None,\n    n_frequencies: Optional[int] = None,\n    sigma: Optional[float] = None,\n    share_last_layer: Optional[bool] = None,\n    full_embed_dropout: Optional[bool] = None,\n    input_dim: int = 32,\n    use_qkv_bias: bool = False,\n    n_heads: int = 8,\n    n_blocks: int = 2,\n    attn_dropout: float = 0.1,\n    ff_dropout: float = 0.2,\n    ff_factor: int = 4,\n    transformer_activation: str = \"gelu\",\n    mlp_hidden_dims: Optional[List[int]] = None,\n    mlp_activation: Optional[str] = None,\n    mlp_dropout: Optional[float] = None,\n    mlp_batchnorm: Optional[bool] = None,\n    mlp_batchnorm_last: Optional[bool] = None,\n    mlp_linear_first: Optional[bool] = None,\n):\n    super(SAINT, self).__init__(\n        column_idx=column_idx,\n        cat_embed_input=cat_embed_input,\n        cat_embed_dropout=cat_embed_dropout,\n        use_cat_bias=use_cat_bias,\n        cat_embed_activation=cat_embed_activation,\n        shared_embed=shared_embed,\n        add_shared_embed=add_shared_embed,\n        frac_shared_embed=frac_shared_embed,\n        continuous_cols=continuous_cols,\n        cont_norm_layer=cont_norm_layer,\n        embed_continuous_method=embed_continuous_method,\n        cont_embed_dropout=cont_embed_dropout,\n        cont_embed_activation=cont_embed_activation,\n        input_dim=input_dim,\n        quantization_setup=quantization_setup,\n        n_frequencies=n_frequencies,\n        sigma=sigma,\n        share_last_layer=share_last_layer,\n        full_embed_dropout=full_embed_dropout,\n    )\n\n    self.use_qkv_bias = use_qkv_bias\n    self.n_heads = n_heads\n    self.n_blocks = n_blocks\n    self.attn_dropout = attn_dropout\n    self.ff_dropout = ff_dropout\n    self.ff_factor = ff_factor\n    self.transformer_activation = transformer_activation\n\n    self.mlp_hidden_dims = mlp_hidden_dims\n    self.mlp_activation = mlp_activation\n    self.mlp_dropout = mlp_dropout\n    self.mlp_batchnorm = mlp_batchnorm\n    self.mlp_batchnorm_last = mlp_batchnorm_last\n    self.mlp_linear_first = mlp_linear_first\n\n    self.with_cls_token = \"cls_token\" in column_idx\n    self.n_cat = len(cat_embed_input) if cat_embed_input is not None else 0\n    self.n_cont = len(continuous_cols) if continuous_cols is not None else 0\n    self.n_feats = self.n_cat + self.n_cont\n\n    # Embeddings are instantiated at the base model\n    # Transformer blocks\n    self.encoder = nn.Sequential()\n    for i in range(n_blocks):\n        self.encoder.add_module(\n            \"saint_block\" + str(i),\n            SaintEncoder(\n                input_dim,\n                n_heads,\n                use_qkv_bias,\n                attn_dropout,\n                ff_dropout,\n                ff_factor,\n                transformer_activation,\n                self.n_feats,\n            ),\n        )\n\n    self.mlp_first_hidden_dim = (\n        self.input_dim if self.with_cls_token else (self.n_feats * self.input_dim)\n    )\n\n    # Mlp: adding an MLP on top of the Resnet blocks is optional and\n    # therefore all related params are optional\n    if self.mlp_hidden_dims is not None:\n        self.mlp = MLP(\n            d_hidden=[self.mlp_first_hidden_dim] + self.mlp_hidden_dims,\n            activation=(\n                \"relu\" if self.mlp_activation is None else self.mlp_activation\n            ),\n            dropout=0.0 if self.mlp_dropout is None else self.mlp_dropout,\n            batchnorm=False if self.mlp_batchnorm is None else self.mlp_batchnorm,\n            batchnorm_last=(\n                False\n                if self.mlp_batchnorm_last is None\n                else self.mlp_batchnorm_last\n            ),\n            linear_first=(\n                False if self.mlp_linear_first is None else self.mlp_linear_first\n            ),\n        )\n    else:\n        self.mlp = None\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.transformers.saint.SAINT.output_dim","title":"output_dim  <code>property</code>","text":"<pre><code>output_dim\n</code></pre> <p>The output dimension of the model. This is a required property neccesary to build the <code>WideDeep</code> class</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.transformers.saint.SAINT.attention_weights","title":"attention_weights  <code>property</code>","text":"<pre><code>attention_weights\n</code></pre> <p>List with the attention weights. Each element of the list is a tuple where the first and the second elements are the column and row attention weights respectively</p> <p>The shape of the attention weights is:</p> <ul> <li> <p>column attention: \\((N, H, F, F)\\)</p> </li> <li> <p>row attention: \\((1, H, N, N)\\)</p> </li> </ul> <p>where \\(N\\) is the batch size, \\(H\\) is the number of heads and \\(F\\) is the number of features/columns in the dataset</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.transformers.ft_transformer.FTTransformer","title":"FTTransformer","text":"<pre><code>FTTransformer(\n    column_idx,\n    *,\n    cat_embed_input=None,\n    cat_embed_dropout=None,\n    use_cat_bias=None,\n    cat_embed_activation=None,\n    shared_embed=None,\n    add_shared_embed=None,\n    frac_shared_embed=None,\n    continuous_cols=None,\n    cont_norm_layer=None,\n    embed_continuous_method=\"standard\",\n    cont_embed_dropout=None,\n    cont_embed_activation=None,\n    quantization_setup=None,\n    n_frequencies=None,\n    sigma=None,\n    share_last_layer=None,\n    full_embed_dropout=None,\n    input_dim=64,\n    kv_compression_factor=0.5,\n    kv_sharing=False,\n    use_qkv_bias=False,\n    n_heads=8,\n    n_blocks=4,\n    attn_dropout=0.2,\n    ff_dropout=0.1,\n    ff_factor=1.33,\n    transformer_activation=\"reglu\",\n    mlp_hidden_dims=None,\n    mlp_activation=None,\n    mlp_dropout=None,\n    mlp_batchnorm=None,\n    mlp_batchnorm_last=None,\n    mlp_linear_first=None\n)\n</code></pre> <p>               Bases: <code>BaseTabularModelWithAttention</code></p> <p>Defines a FTTransformer model that can be used as the <code>deeptabular</code> component of a Wide &amp; Deep model or independently by itself.</p> <p>Most of the parameters for this class are <code>Optional</code> since the use of categorical or continuous is in fact optional (i.e. one can use categorical features only, continuous features only or both).</p> <p>Parameters:</p> <ul> <li> <code>column_idx</code>               (<code>Dict[str, int]</code>)           \u2013            <p>Dict containing the index of the columns that will be passed through the <code>TabMlp</code> model. Required to slice the tensors. e.g. {'education': 0, 'relationship': 1, 'workclass': 2, ...}.</p> </li> <li> <code>cat_embed_input</code>               (<code>Optional[List[Tuple[str, int]]]</code>, default:                   <code>None</code> )           \u2013            <p>List of Tuples with the column name and number of unique values and embedding dimension. e.g. [(education, 11), ...]</p> </li> <li> <code>cat_embed_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Categorical embeddings dropout. If <code>None</code>, it will default to 0.</p> </li> <li> <code>use_cat_bias</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating if bias will be used for the categorical embeddings. If <code>None</code>, it will default to 'False'.</p> </li> <li> <code>cat_embed_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the categorical embeddings, if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> </li> <li> <code>shared_embed</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating if the embeddings will be \"shared\". The idea behind <code>shared_embed</code> is described in the Appendix A in the TabTransformer paper: 'The goal of having column embedding is to enable the model to distinguish the classes in one column from those in the other columns'. In other words, the idea is to let the model learn which column is embedded at the time. See: <code>pytorch_widedeep.models.transformers._layers.SharedEmbeddings</code>.</p> </li> <li> <code>add_shared_embed</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>The two embedding sharing strategies are: 1) add the shared embeddings to the column embeddings or 2) to replace the first <code>frac_shared_embed</code> with the shared embeddings. See <code>pytorch_widedeep.models.embeddings_layers.SharedEmbeddings</code> If 'None' is passed, it will default to 'False'.</p> </li> <li> <code>frac_shared_embed</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>The fraction of embeddings that will be shared (if <code>add_shared_embed = False</code>) by all the different categories for one particular column. If 'None' is passed, it will default to 0.0.</p> </li> <li> <code>continuous_cols</code>               (<code>Optional[List[str]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the name of the numeric (aka continuous) columns</p> </li> <li> <code>cont_norm_layer</code>               (<code>Optional[Literal[batchnorm, layernorm]]</code>, default:                   <code>None</code> )           \u2013            <p>Type of normalization layer applied to the continuous features. Options are: 'layernorm' and 'batchnorm'. if <code>None</code>, no normalization layer will be used.</p> </li> <li> <code>embed_continuous_method</code>               (<code>Optional[Literal[standard, piecewise, periodic]]</code>, default:                   <code>'standard'</code> )           \u2013            <p>Method to use to embed the continuous features. Options are: 'standard', 'periodic' or 'piecewise'. The 'standard' embedding method is based on the FT-Transformer implementation presented in the paper: Revisiting Deep Learning Models for Tabular Data. The 'periodic' and_'piecewise'_ methods were presented in the paper: On Embeddings for Numerical Features in Tabular Deep Learning. Please, read the papers for details.</p> </li> <li> <code>cont_embed_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p> </li> <li> <code>cont_embed_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the continuous embeddings if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported. If <code>None</code>, no activation function will be applied.</p> </li> <li> <code>quantization_setup</code>               (<code>Optional[Dict[str, List[float]]]</code>, default:                   <code>None</code> )           \u2013            <p>This parameter is used when the 'piecewise' method is used to embed the continuous cols. It is a dict where keys are the name of the continuous columns and values are lists with the boundaries for the quantization of the continuous_cols. See the examples for details. If If the 'piecewise' method is used, this parameter is required.</p> </li> <li> <code>n_frequencies</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>This is the so called 'k' in their paper On Embeddings for Numerical Features in Tabular Deep Learning, and is the number of 'frequencies' that will be used to represent each continuous column. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>sigma</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>This is the sigma parameter in the paper mentioned when describing the previous parameters and it is used to initialise the 'frequency weights'. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>share_last_layer</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>This parameter is not present in the before mentioned paper but it is implemented in the official repo. If <code>True</code> the linear layer that turns the frequencies into embeddings will be shared across the continuous columns. If <code>False</code> a different linear layer will be used for each continuous column. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>full_embed_dropout</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>If <code>True</code>, the full embedding corresponding to a column will be masked out/dropout. If <code>None</code>, it will default to <code>False</code>.</p> </li> <li> <code>input_dim</code>               (<code>int</code>, default:                   <code>64</code> )           \u2013            <p>The so-called dimension of the model. Is the number of embeddings used to encode the categorical and/or continuous columns.</p> </li> <li> <code>kv_compression_factor</code>               (<code>float</code>, default:                   <code>0.5</code> )           \u2013            <p>By default, the FTTransformer uses Linear Attention (See Linformer: Self-Attention with Linear Complexity ). The compression factor that will be used to reduce the input sequence length. If we denote the resulting sequence length as \\(k = int(kv_{compression \\space factor} \\times s)\\) where \\(s\\) is the input sequence length.</p> </li> <li> <code>kv_sharing</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating if the \\(E\\) and \\(F\\) projection matrices will share weights.  See Linformer: Self-Attention with Linear Complexity for details</p> </li> <li> <code>n_heads</code>               (<code>int</code>, default:                   <code>8</code> )           \u2013            <p>Number of attention heads per FTTransformer block</p> </li> <li> <code>use_qkv_bias</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether or not to use bias in the Q, K, and V projection layers</p> </li> <li> <code>n_blocks</code>               (<code>int</code>, default:                   <code>4</code> )           \u2013            <p>Number of FTTransformer blocks</p> </li> <li> <code>attn_dropout</code>               (<code>float</code>, default:                   <code>0.2</code> )           \u2013            <p>Dropout that will be applied to the Linear-Attention layers</p> </li> <li> <code>ff_dropout</code>               (<code>float</code>, default:                   <code>0.1</code> )           \u2013            <p>Dropout that will be applied to the FeedForward network</p> </li> <li> <code>ff_factor</code>               (<code>float</code>, default:                   <code>1.33</code> )           \u2013            <p>Multiplicative factor applied to the first layer of the FF network in each Transformer block, This is normally set to 4, but they use 4/3 in the paper.</p> </li> <li> <code>transformer_activation</code>               (<code>str</code>, default:                   <code>'reglu'</code> )           \u2013            <p>Transformer Encoder activation function. 'tanh', 'relu', 'leaky_relu', 'gelu', 'geglu' and 'reglu' are supported</p> </li> <li> <code>mlp_hidden_dims</code>               (<code>Optional[List[int]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the number of neurons per dense layer in the MLP. e.g: [64, 32]. If not provided no MLP on top of the final FTTransformer block will be used.</p> </li> <li> <code>mlp_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the dense layers of the MLP. Currently 'tanh', 'relu', 'leaky'_relu' and _'gelu' are supported. If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to 'relu'.</p> </li> <li> <code>mlp_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>float with the dropout between the dense layers of the MLP. If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to 0.0.</p> </li> <li> <code>mlp_batchnorm</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating whether or not batch normalization will be applied to the dense layers If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to False.</p> </li> <li> <code>mlp_batchnorm_last</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating whether or not batch normalization will be applied to the last of the dense layers If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to False.</p> </li> <li> <code>mlp_linear_first</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code> If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to <code>True</code>.</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>encoder</code>               (<code>Module</code>)           \u2013            <p>Sequence of FTTransformer blocks</p> </li> <li> <code>mlp</code>               (<code>Module</code>)           \u2013            <p>MLP component in the model</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import FTTransformer\n&gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n&gt;&gt;&gt; colnames = ['a', 'b', 'c', 'd', 'e']\n&gt;&gt;&gt; cat_embed_input = [(u,i) for u,i in zip(colnames[:4], [4]*4)]\n&gt;&gt;&gt; continuous_cols = ['e']\n&gt;&gt;&gt; column_idx = {k:v for v,k in enumerate(colnames)}\n&gt;&gt;&gt; model = FTTransformer(column_idx=column_idx, cat_embed_input=cat_embed_input, continuous_cols=continuous_cols)\n&gt;&gt;&gt; out = model(X_tab)\n</code></pre> Source code in <code>pytorch_widedeep/models/tabular/transformers/ft_transformer.py</code> <pre><code>def __init__(\n    self,\n    column_idx: Dict[str, int],\n    *,\n    cat_embed_input: Optional[List[Tuple[str, int]]] = None,\n    cat_embed_dropout: Optional[float] = None,\n    use_cat_bias: Optional[bool] = None,\n    cat_embed_activation: Optional[str] = None,\n    shared_embed: Optional[bool] = None,\n    add_shared_embed: Optional[bool] = None,\n    frac_shared_embed: Optional[float] = None,\n    continuous_cols: Optional[List[str]] = None,\n    cont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\n    embed_continuous_method: Optional[\n        Literal[\"standard\", \"piecewise\", \"periodic\"]\n    ] = \"standard\",\n    cont_embed_dropout: Optional[float] = None,\n    cont_embed_activation: Optional[str] = None,\n    quantization_setup: Optional[Dict[str, List[float]]] = None,\n    n_frequencies: Optional[int] = None,\n    sigma: Optional[float] = None,\n    share_last_layer: Optional[bool] = None,\n    full_embed_dropout: Optional[bool] = None,\n    input_dim: int = 64,\n    kv_compression_factor: float = 0.5,\n    kv_sharing: bool = False,\n    use_qkv_bias: bool = False,\n    n_heads: int = 8,\n    n_blocks: int = 4,\n    attn_dropout: float = 0.2,\n    ff_dropout: float = 0.1,\n    ff_factor: float = 1.33,\n    transformer_activation: str = \"reglu\",\n    mlp_hidden_dims: Optional[List[int]] = None,\n    mlp_activation: Optional[str] = None,\n    mlp_dropout: Optional[float] = None,\n    mlp_batchnorm: Optional[bool] = None,\n    mlp_batchnorm_last: Optional[bool] = None,\n    mlp_linear_first: Optional[bool] = None,\n):\n    super(FTTransformer, self).__init__(\n        column_idx=column_idx,\n        cat_embed_input=cat_embed_input,\n        cat_embed_dropout=cat_embed_dropout,\n        use_cat_bias=use_cat_bias,\n        cat_embed_activation=cat_embed_activation,\n        shared_embed=shared_embed,\n        add_shared_embed=add_shared_embed,\n        frac_shared_embed=frac_shared_embed,\n        continuous_cols=continuous_cols,\n        cont_norm_layer=cont_norm_layer,\n        embed_continuous_method=embed_continuous_method,\n        cont_embed_dropout=cont_embed_dropout,\n        cont_embed_activation=cont_embed_activation,\n        input_dim=input_dim,\n        quantization_setup=quantization_setup,\n        n_frequencies=n_frequencies,\n        sigma=sigma,\n        share_last_layer=share_last_layer,\n        full_embed_dropout=full_embed_dropout,\n    )\n\n    self.kv_compression_factor = kv_compression_factor\n    self.kv_sharing = kv_sharing\n    self.use_qkv_bias = use_qkv_bias\n    self.n_heads = n_heads\n    self.n_blocks = n_blocks\n    self.attn_dropout = attn_dropout\n    self.ff_dropout = ff_dropout\n    self.ff_factor = ff_factor\n    self.transformer_activation = transformer_activation\n\n    self.mlp_hidden_dims = mlp_hidden_dims\n    self.mlp_activation = mlp_activation\n    self.mlp_dropout = mlp_dropout\n    self.mlp_batchnorm = mlp_batchnorm\n    self.mlp_batchnorm_last = mlp_batchnorm_last\n    self.mlp_linear_first = mlp_linear_first\n\n    self.with_cls_token = \"cls_token\" in column_idx\n    self.n_cat = len(cat_embed_input) if cat_embed_input is not None else 0\n    self.n_cont = len(continuous_cols) if continuous_cols is not None else 0\n    self.n_feats = self.n_cat + self.n_cont\n\n    # Embeddings are instantiated at the base model\n    # Transformer blocks\n    is_first = True\n    self.encoder = nn.Sequential()\n    for i in range(n_blocks):\n        self.encoder.add_module(\n            \"fttransformer_block\" + str(i),\n            FTTransformerEncoder(\n                input_dim,\n                self.n_feats,\n                n_heads,\n                use_qkv_bias,\n                attn_dropout,\n                ff_dropout,\n                ff_factor,\n                kv_compression_factor,\n                kv_sharing,\n                transformer_activation,\n                is_first,\n            ),\n        )\n        is_first = False\n\n    self.mlp_first_hidden_dim = (\n        self.input_dim if self.with_cls_token else (self.n_feats * self.input_dim)\n    )\n\n    # Mlp: adding an MLP on top of the Resnet blocks is optional and\n    # therefore all related params are optional\n    if self.mlp_hidden_dims is not None:\n        self.mlp = MLP(\n            d_hidden=[self.mlp_first_hidden_dim] + self.mlp_hidden_dims,\n            activation=(\n                \"relu\" if self.mlp_activation is None else self.mlp_activation\n            ),\n            dropout=0.0 if self.mlp_dropout is None else self.mlp_dropout,\n            batchnorm=False if self.mlp_batchnorm is None else self.mlp_batchnorm,\n            batchnorm_last=(\n                False\n                if self.mlp_batchnorm_last is None\n                else self.mlp_batchnorm_last\n            ),\n            linear_first=(\n                False if self.mlp_linear_first is None else self.mlp_linear_first\n            ),\n        )\n    else:\n        self.mlp = None\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.transformers.ft_transformer.FTTransformer.output_dim","title":"output_dim  <code>property</code>","text":"<pre><code>output_dim\n</code></pre> <p>The output dimension of the model. This is a required property neccesary to build the <code>WideDeep</code> class</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.transformers.ft_transformer.FTTransformer.attention_weights","title":"attention_weights  <code>property</code>","text":"<pre><code>attention_weights\n</code></pre> <p>List with the attention weights per block</p> <p>The shape of the attention weights is: \\((N, H, F, k)\\), where \\(N\\) is the batch size, \\(H\\) is the number of attention heads, \\(F\\) is the number of features/columns and \\(k\\) is the reduced sequence length or dimension, i.e. \\(k = int(kv_{compression \\space factor} \\times s)\\)</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.transformers.tab_perceiver.TabPerceiver","title":"TabPerceiver","text":"<pre><code>TabPerceiver(\n    column_idx,\n    *,\n    cat_embed_input=None,\n    cat_embed_dropout=None,\n    use_cat_bias=None,\n    cat_embed_activation=None,\n    shared_embed=None,\n    add_shared_embed=None,\n    frac_shared_embed=None,\n    continuous_cols=None,\n    cont_norm_layer=None,\n    embed_continuous_method=\"standard\",\n    cont_embed_dropout=None,\n    cont_embed_activation=None,\n    quantization_setup=None,\n    n_frequencies=None,\n    sigma=None,\n    share_last_layer=None,\n    full_embed_dropout=None,\n    input_dim=32,\n    n_cross_attns=1,\n    n_cross_attn_heads=4,\n    n_latents=16,\n    latent_dim=128,\n    n_latent_heads=4,\n    n_latent_blocks=4,\n    n_perceiver_blocks=4,\n    share_weights=False,\n    attn_dropout=0.1,\n    ff_dropout=0.1,\n    ff_factor=4,\n    transformer_activation=\"geglu\",\n    mlp_hidden_dims=None,\n    mlp_activation=None,\n    mlp_dropout=None,\n    mlp_batchnorm=None,\n    mlp_batchnorm_last=None,\n    mlp_linear_first=None\n)\n</code></pre> <p>               Bases: <code>BaseTabularModelWithAttention</code></p> <p>Defines an adaptation of a Perceiver  that can be used as the <code>deeptabular</code> component of a Wide &amp; Deep model  or independently by itself.</p> <p>Most of the parameters for this class are <code>Optional</code> since the use of categorical or continuous is in fact optional (i.e. one can use categorical features only, continuous features only or both).</p> <p> NOTE: while there are scientific publications for  the <code>TabTransformer</code>, <code>SAINT</code> and <code>FTTransformer</code>, the <code>TabPerceiver</code>  and the <code>TabFastFormer</code> are our own adaptations of the  Perceiver and the  FastFormer for tabular data.</p> <p>Parameters:</p> <ul> <li> <code>column_idx</code>               (<code>Dict[str, int]</code>)           \u2013            <p>Dict containing the index of the columns that will be passed through the <code>TabMlp</code> model. Required to slice the tensors. e.g. {'education': 0, 'relationship': 1, 'workclass': 2, ...}.</p> </li> <li> <code>cat_embed_input</code>               (<code>Optional[List[Tuple[str, int]]]</code>, default:                   <code>None</code> )           \u2013            <p>List of Tuples with the column name and number of unique values and embedding dimension. e.g. [(education, 11), ...]</p> </li> <li> <code>cat_embed_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Categorical embeddings dropout. If <code>None</code>, it will default to 0.</p> </li> <li> <code>use_cat_bias</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating if bias will be used for the categorical embeddings. If <code>None</code>, it will default to 'False'.</p> </li> <li> <code>cat_embed_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the categorical embeddings, if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> </li> <li> <code>shared_embed</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating if the embeddings will be \"shared\". The idea behind <code>shared_embed</code> is described in the Appendix A in the TabTransformer paper: 'The goal of having column embedding is to enable the model to distinguish the classes in one column from those in the other columns'. In other words, the idea is to let the model learn which column is embedded at the time. See: <code>pytorch_widedeep.models.transformers._layers.SharedEmbeddings</code>.</p> </li> <li> <code>add_shared_embed</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>The two embedding sharing strategies are: 1) add the shared embeddings to the column embeddings or 2) to replace the first <code>frac_shared_embed</code> with the shared embeddings. See <code>pytorch_widedeep.models.embeddings_layers.SharedEmbeddings</code> If 'None' is passed, it will default to 'False'.</p> </li> <li> <code>frac_shared_embed</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>The fraction of embeddings that will be shared (if <code>add_shared_embed = False</code>) by all the different categories for one particular column. If 'None' is passed, it will default to 0.0.</p> </li> <li> <code>continuous_cols</code>               (<code>Optional[List[str]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the name of the numeric (aka continuous) columns</p> </li> <li> <code>cont_norm_layer</code>               (<code>Optional[Literal[batchnorm, layernorm]]</code>, default:                   <code>None</code> )           \u2013            <p>Type of normalization layer applied to the continuous features. Options are: 'layernorm' and 'batchnorm'. if <code>None</code>, no normalization layer will be used.</p> </li> <li> <code>embed_continuous_method</code>               (<code>Optional[Literal[standard, piecewise, periodic]]</code>, default:                   <code>'standard'</code> )           \u2013            <p>Method to use to embed the continuous features. Options are: 'standard', 'periodic' or 'piecewise'. The 'standard' embedding method is based on the FT-Transformer implementation presented in the paper: Revisiting Deep Learning Models for Tabular Data. The 'periodic' and_'piecewise'_ methods were presented in the paper: On Embeddings for Numerical Features in Tabular Deep Learning. Please, read the papers for details.</p> </li> <li> <code>cont_embed_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p> </li> <li> <code>cont_embed_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the continuous embeddings if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported. If <code>None</code>, no activation function will be applied.</p> </li> <li> <code>quantization_setup</code>               (<code>Optional[Dict[str, List[float]]]</code>, default:                   <code>None</code> )           \u2013            <p>This parameter is used when the 'piecewise' method is used to embed the continuous cols. It is a dict where keys are the name of the continuous columns and values are lists with the boundaries for the quantization of the continuous_cols. See the examples for details. If If the 'piecewise' method is used, this parameter is required.</p> </li> <li> <code>n_frequencies</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>This is the so called 'k' in their paper On Embeddings for Numerical Features in Tabular Deep Learning, and is the number of 'frequencies' that will be used to represent each continuous column. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>sigma</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>This is the sigma parameter in the paper mentioned when describing the previous parameters and it is used to initialise the 'frequency weights'. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>share_last_layer</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>This parameter is not present in the before mentioned paper but it is implemented in the official repo. If <code>True</code> the linear layer that turns the frequencies into embeddings will be shared across the continuous columns. If <code>False</code> a different linear layer will be used for each continuous column. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>full_embed_dropout</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>If <code>True</code>, the full embedding corresponding to a column will be masked out/dropout. If <code>None</code>, it will default to <code>False</code>.</p> </li> <li> <code>input_dim</code>               (<code>int</code>, default:                   <code>32</code> )           \u2013            <p>The so-called dimension of the model. Is the number of embeddings used to encode the categorical and/or continuous columns.</p> </li> <li> <code>n_cross_attns</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>Number of times each perceiver block will cross attend to the input data (i.e. number of cross attention components per perceiver block). This should normally be 1. However, in the paper they describe some architectures (normally computer vision-related problems) where the Perceiver attends multiple times to the input array. Therefore, maybe multiple cross attention to the input array is also useful in some cases for tabular data  .</p> </li> <li> <code>n_cross_attn_heads</code>               (<code>int</code>, default:                   <code>4</code> )           \u2013            <p>Number of attention heads for the cross attention component</p> </li> <li> <code>n_latents</code>               (<code>int</code>, default:                   <code>16</code> )           \u2013            <p>Number of latents. This is the \\(N\\) parameter in the paper. As indicated in the paper, this number should be significantly lower than \\(M\\) (the number of columns in the dataset). Setting \\(N\\) closer to \\(M\\) defies the main purpose of the Perceiver, which is to overcome the transformer quadratic bottleneck</p> </li> <li> <code>latent_dim</code>               (<code>int</code>, default:                   <code>128</code> )           \u2013            <p>Latent dimension.</p> </li> <li> <code>n_latent_heads</code>               (<code>int</code>, default:                   <code>4</code> )           \u2013            <p>Number of attention heads per Latent Transformer</p> </li> <li> <code>n_latent_blocks</code>               (<code>int</code>, default:                   <code>4</code> )           \u2013            <p>Number of transformer encoder blocks (normalised MHA + normalised FF) per Latent Transformer</p> </li> <li> <code>n_perceiver_blocks</code>               (<code>int</code>, default:                   <code>4</code> )           \u2013            <p>Number of Perceiver blocks defined as [Cross Attention + Latent Transformer]</p> </li> <li> <code>share_weights</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating if the weights will be shared between Perceiver blocks</p> </li> <li> <code>attn_dropout</code>               (<code>float</code>, default:                   <code>0.1</code> )           \u2013            <p>Dropout that will be applied to the Multi-Head Attention layers</p> </li> <li> <code>ff_dropout</code>               (<code>float</code>, default:                   <code>0.1</code> )           \u2013            <p>Dropout that will be applied to the FeedForward network</p> </li> <li> <code>ff_factor</code>               (<code>int</code>, default:                   <code>4</code> )           \u2013            <p>Multiplicative factor applied to the first layer of the FF network in each Transformer block, This is normally set to 4.</p> </li> <li> <code>transformer_activation</code>               (<code>str</code>, default:                   <code>'geglu'</code> )           \u2013            <p>Transformer Encoder activation function. 'tanh', 'relu', 'leaky_relu', 'gelu', 'geglu' and 'reglu' are supported</p> </li> <li> <code>mlp_hidden_dims</code>               (<code>Optional[List[int]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the number of neurons per dense layer in the MLP. e.g: [64, 32]. If not provided no MLP on top of the final Transformer block will be used.</p> </li> <li> <code>mlp_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the dense layers of the MLP. Currently 'tanh', 'relu', 'leaky'_relu' and _'gelu' are supported. If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to 'relu'.</p> </li> <li> <code>mlp_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>float with the dropout between the dense layers of the MLP. If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to 0.0.</p> </li> <li> <code>mlp_batchnorm</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating whether or not batch normalization will be applied to the dense layers If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to False.</p> </li> <li> <code>mlp_batchnorm_last</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating whether or not batch normalization will be applied to the last of the dense layers If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to False.</p> </li> <li> <code>mlp_linear_first</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code> If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to <code>True</code>.</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>encoder</code>               (<code>ModuleDict</code>)           \u2013            <p>ModuleDict with the Perceiver blocks</p> </li> <li> <code>latents</code>               (<code>Parameter</code>)           \u2013            <p>Latents that will be used for prediction</p> </li> <li> <code>mlp</code>               (<code>Module</code>)           \u2013            <p>MLP component in the model</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import TabPerceiver\n&gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n&gt;&gt;&gt; colnames = ['a', 'b', 'c', 'd', 'e']\n&gt;&gt;&gt; cat_embed_input = [(u,i) for u,i in zip(colnames[:4], [4]*4)]\n&gt;&gt;&gt; continuous_cols = ['e']\n&gt;&gt;&gt; column_idx = {k:v for v,k in enumerate(colnames)}\n&gt;&gt;&gt; model = TabPerceiver(column_idx=column_idx, cat_embed_input=cat_embed_input,\n... continuous_cols=continuous_cols, n_latents=2, latent_dim=16,\n... n_perceiver_blocks=2)\n&gt;&gt;&gt; out = model(X_tab)\n</code></pre> Source code in <code>pytorch_widedeep/models/tabular/transformers/tab_perceiver.py</code> <pre><code>def __init__(\n    self,\n    column_idx: Dict[str, int],\n    *,\n    cat_embed_input: Optional[List[Tuple[str, int]]] = None,\n    cat_embed_dropout: Optional[float] = None,\n    use_cat_bias: Optional[bool] = None,\n    cat_embed_activation: Optional[str] = None,\n    shared_embed: Optional[bool] = None,\n    add_shared_embed: Optional[bool] = None,\n    frac_shared_embed: Optional[float] = None,\n    continuous_cols: Optional[List[str]] = None,\n    cont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\n    embed_continuous_method: Optional[\n        Literal[\"standard\", \"piecewise\", \"periodic\"]\n    ] = \"standard\",\n    cont_embed_dropout: Optional[float] = None,\n    cont_embed_activation: Optional[str] = None,\n    quantization_setup: Optional[Dict[str, List[float]]] = None,\n    n_frequencies: Optional[int] = None,\n    sigma: Optional[float] = None,\n    share_last_layer: Optional[bool] = None,\n    full_embed_dropout: Optional[bool] = None,\n    input_dim: int = 32,\n    n_cross_attns: int = 1,\n    n_cross_attn_heads: int = 4,\n    n_latents: int = 16,\n    latent_dim: int = 128,\n    n_latent_heads: int = 4,\n    n_latent_blocks: int = 4,\n    n_perceiver_blocks: int = 4,\n    share_weights: bool = False,\n    attn_dropout: float = 0.1,\n    ff_dropout: float = 0.1,\n    ff_factor: int = 4,\n    transformer_activation: str = \"geglu\",\n    mlp_hidden_dims: Optional[List[int]] = None,\n    mlp_activation: Optional[str] = None,\n    mlp_dropout: Optional[float] = None,\n    mlp_batchnorm: Optional[bool] = None,\n    mlp_batchnorm_last: Optional[bool] = None,\n    mlp_linear_first: Optional[bool] = None,\n):\n    super(TabPerceiver, self).__init__(\n        column_idx=column_idx,\n        cat_embed_input=cat_embed_input,\n        cat_embed_dropout=cat_embed_dropout,\n        use_cat_bias=use_cat_bias,\n        cat_embed_activation=cat_embed_activation,\n        shared_embed=shared_embed,\n        add_shared_embed=add_shared_embed,\n        frac_shared_embed=frac_shared_embed,\n        continuous_cols=continuous_cols,\n        cont_norm_layer=cont_norm_layer,\n        embed_continuous_method=embed_continuous_method,\n        cont_embed_dropout=cont_embed_dropout,\n        cont_embed_activation=cont_embed_activation,\n        input_dim=input_dim,\n        quantization_setup=quantization_setup,\n        n_frequencies=n_frequencies,\n        sigma=sigma,\n        share_last_layer=share_last_layer,\n        full_embed_dropout=full_embed_dropout,\n    )\n\n    self.n_cross_attns = n_cross_attns\n    self.n_cross_attn_heads = n_cross_attn_heads\n    self.n_latents = n_latents\n    self.latent_dim = latent_dim\n    self.n_latent_heads = n_latent_heads\n    self.n_latent_blocks = n_latent_blocks\n    self.n_perceiver_blocks = n_perceiver_blocks\n    self.share_weights = share_weights\n    self.attn_dropout = attn_dropout\n    self.ff_dropout = ff_dropout\n    self.ff_factor = ff_factor\n    self.transformer_activation = transformer_activation\n\n    self.mlp_hidden_dims = mlp_hidden_dims\n    self.mlp_activation = mlp_activation\n    self.mlp_dropout = mlp_dropout\n    self.mlp_batchnorm = mlp_batchnorm\n    self.mlp_batchnorm_last = mlp_batchnorm_last\n    self.mlp_linear_first = mlp_linear_first\n\n    # Embeddings are instantiated at the base model\n    # Transformer blocks\n    self.latents = nn.init.trunc_normal_(\n        nn.Parameter(torch.empty(n_latents, latent_dim))\n    )\n\n    self.encoder = nn.ModuleDict()\n    first_perceiver_block = self._build_perceiver_block()\n    self.encoder[\"perceiver_block0\"] = first_perceiver_block\n\n    if share_weights:\n        for n in range(1, n_perceiver_blocks):\n            self.encoder[\"perceiver_block\" + str(n)] = first_perceiver_block\n    else:\n        for n in range(1, n_perceiver_blocks):\n            self.encoder[\"perceiver_block\" + str(n)] = self._build_perceiver_block()\n\n    self.mlp_first_hidden_dim = self.latent_dim\n\n    # Mlp\n    if self.mlp_hidden_dims is not None:\n        self.mlp = MLP(\n            d_hidden=[self.mlp_first_hidden_dim] + self.mlp_hidden_dims,\n            activation=(\n                \"relu\" if self.mlp_activation is None else self.mlp_activation\n            ),\n            dropout=0.0 if self.mlp_dropout is None else self.mlp_dropout,\n            batchnorm=False if self.mlp_batchnorm is None else self.mlp_batchnorm,\n            batchnorm_last=(\n                False\n                if self.mlp_batchnorm_last is None\n                else self.mlp_batchnorm_last\n            ),\n            linear_first=(\n                False if self.mlp_linear_first is None else self.mlp_linear_first\n            ),\n        )\n    else:\n        self.mlp = None\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.transformers.tab_perceiver.TabPerceiver.output_dim","title":"output_dim  <code>property</code>","text":"<pre><code>output_dim\n</code></pre> <p>The output dimension of the model. This is a required property neccesary to build the <code>WideDeep</code> class</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.transformers.tab_perceiver.TabPerceiver.attention_weights","title":"attention_weights  <code>property</code>","text":"<pre><code>attention_weights\n</code></pre> <p>List with the attention weights. If the weights are not shared between perceiver blocks each element of the list will be a list itself containing the Cross Attention and Latent Transformer attention weights respectively</p> <p>The shape of the attention weights is:</p> <ul> <li> <p>Cross Attention: \\((N, C, L, F)\\)</p> </li> <li> <p>Latent Attention: \\((N, T, L, L)\\)</p> </li> </ul> <p>WHere \\(N\\) is the batch size, \\(C\\) is the number of Cross Attention heads, \\(L\\) is the number of Latents, \\(F\\) is the number of features/columns in the dataset and \\(T\\) is the number of Latent Attention heads</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.transformers.tab_fastformer.TabFastFormer","title":"TabFastFormer","text":"<pre><code>TabFastFormer(\n    column_idx,\n    *,\n    cat_embed_input=None,\n    cat_embed_dropout=None,\n    use_cat_bias=None,\n    cat_embed_activation=None,\n    shared_embed=None,\n    add_shared_embed=None,\n    frac_shared_embed=None,\n    continuous_cols=None,\n    cont_norm_layer=None,\n    embed_continuous_method=\"standard\",\n    cont_embed_dropout=None,\n    cont_embed_activation=None,\n    quantization_setup=None,\n    n_frequencies=None,\n    sigma=None,\n    share_last_layer=None,\n    full_embed_dropout=None,\n    input_dim=32,\n    n_heads=8,\n    use_bias=False,\n    n_blocks=4,\n    attn_dropout=0.1,\n    ff_dropout=0.2,\n    ff_factor=4,\n    share_qv_weights=False,\n    share_weights=False,\n    transformer_activation=\"relu\",\n    mlp_hidden_dims=None,\n    mlp_activation=None,\n    mlp_dropout=None,\n    mlp_batchnorm=None,\n    mlp_batchnorm_last=None,\n    mlp_linear_first=None\n)\n</code></pre> <p>               Bases: <code>BaseTabularModelWithAttention</code></p> <p>Defines an adaptation of a FastFormer that can be used as the <code>deeptabular</code> component of a Wide &amp; Deep model or independently by itself.</p> <p>Most of the parameters for this class are <code>Optional</code> since the use of categorical or continuous is in fact optional (i.e. one can use categorical features only, continuous features only or both).</p> <p> NOTE: while there are scientific publications for  the <code>TabTransformer</code>, <code>SAINT</code> and <code>FTTransformer</code>, the <code>TabPerceiver</code>  and the <code>TabFastFormer</code> are our own adaptations of the  Perceiver and the  FastFormer for tabular data.</p> <p>Parameters:</p> <ul> <li> <code>column_idx</code>               (<code>Dict[str, int]</code>)           \u2013            <p>Dict containing the index of the columns that will be passed through the <code>TabMlp</code> model. Required to slice the tensors. e.g. {'education': 0, 'relationship': 1, 'workclass': 2, ...}.</p> </li> <li> <code>cat_embed_input</code>               (<code>Optional[List[Tuple[str, int]]]</code>, default:                   <code>None</code> )           \u2013            <p>List of Tuples with the column name and number of unique values and embedding dimension. e.g. [(education, 11), ...]</p> </li> <li> <code>cat_embed_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Categorical embeddings dropout. If <code>None</code>, it will default to 0.</p> </li> <li> <code>use_cat_bias</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating if bias will be used for the categorical embeddings. If <code>None</code>, it will default to 'False'.</p> </li> <li> <code>cat_embed_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the categorical embeddings, if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> </li> <li> <code>shared_embed</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating if the embeddings will be \"shared\". The idea behind <code>shared_embed</code> is described in the Appendix A in the TabTransformer paper: 'The goal of having column embedding is to enable the model to distinguish the classes in one column from those in the other columns'. In other words, the idea is to let the model learn which column is embedded at the time. See: <code>pytorch_widedeep.models.transformers._layers.SharedEmbeddings</code>.</p> </li> <li> <code>add_shared_embed</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>The two embedding sharing strategies are: 1) add the shared embeddings to the column embeddings or 2) to replace the first <code>frac_shared_embed</code> with the shared embeddings. See <code>pytorch_widedeep.models.embeddings_layers.SharedEmbeddings</code> If 'None' is passed, it will default to 'False'.</p> </li> <li> <code>frac_shared_embed</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>The fraction of embeddings that will be shared (if <code>add_shared_embed = False</code>) by all the different categories for one particular column. If 'None' is passed, it will default to 0.0.</p> </li> <li> <code>continuous_cols</code>               (<code>Optional[List[str]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the name of the numeric (aka continuous) columns</p> </li> <li> <code>cont_norm_layer</code>               (<code>Optional[Literal[batchnorm, layernorm]]</code>, default:                   <code>None</code> )           \u2013            <p>Type of normalization layer applied to the continuous features. Options are: 'layernorm' and 'batchnorm'. if <code>None</code>, no normalization layer will be used.</p> </li> <li> <code>embed_continuous_method</code>               (<code>Optional[Literal[standard, piecewise, periodic]]</code>, default:                   <code>'standard'</code> )           \u2013            <p>Method to use to embed the continuous features. Options are: 'standard', 'periodic' or 'piecewise'. The 'standard' embedding method is based on the FT-Transformer implementation presented in the paper: Revisiting Deep Learning Models for Tabular Data. The 'periodic' and_'piecewise'_ methods were presented in the paper: On Embeddings for Numerical Features in Tabular Deep Learning. Please, read the papers for details.</p> </li> <li> <code>cont_embed_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p> </li> <li> <code>cont_embed_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the continuous embeddings if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported. If <code>None</code>, no activation function will be applied.</p> </li> <li> <code>quantization_setup</code>               (<code>Optional[Dict[str, List[float]]]</code>, default:                   <code>None</code> )           \u2013            <p>This parameter is used when the 'piecewise' method is used to embed the continuous cols. It is a dict where keys are the name of the continuous columns and values are lists with the boundaries for the quantization of the continuous_cols. See the examples for details. If If the 'piecewise' method is used, this parameter is required.</p> </li> <li> <code>n_frequencies</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>This is the so called 'k' in their paper On Embeddings for Numerical Features in Tabular Deep Learning, and is the number of 'frequencies' that will be used to represent each continuous column. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>sigma</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>This is the sigma parameter in the paper mentioned when describing the previous parameters and it is used to initialise the 'frequency weights'. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>share_last_layer</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>This parameter is not present in the before mentioned paper but it is implemented in the official repo. If <code>True</code> the linear layer that turns the frequencies into embeddings will be shared across the continuous columns. If <code>False</code> a different linear layer will be used for each continuous column. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>full_embed_dropout</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>If <code>True</code>, the full embedding corresponding to a column will be masked out/dropout. If <code>None</code>, it will default to <code>False</code>.</p> </li> <li> <code>input_dim</code>               (<code>int</code>, default:                   <code>32</code> )           \u2013            <p>The so-called dimension of the model. Is the number of embeddings used to encode the categorical and/or continuous columns</p> </li> <li> <code>n_heads</code>               (<code>int</code>, default:                   <code>8</code> )           \u2013            <p>Number of attention heads per FastFormer block</p> </li> <li> <code>use_bias</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether or not to use bias in the Q, K, and V projection layers</p> </li> <li> <code>n_blocks</code>               (<code>int</code>, default:                   <code>4</code> )           \u2013            <p>Number of FastFormer blocks</p> </li> <li> <code>attn_dropout</code>               (<code>float</code>, default:                   <code>0.1</code> )           \u2013            <p>Dropout that will be applied to the Additive Attention layers</p> </li> <li> <code>ff_dropout</code>               (<code>float</code>, default:                   <code>0.2</code> )           \u2013            <p>Dropout that will be applied to the FeedForward network</p> </li> <li> <code>ff_factor</code>               (<code>int</code>, default:                   <code>4</code> )           \u2013            <p>Multiplicative factor applied to the first layer of the FF network in each Transformer block, This is normally set to 4.</p> </li> <li> <code>share_qv_weights</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Following the paper, this is a boolean indicating if the Value (\\(V\\)) and the Query (\\(Q\\)) transformation parameters will be shared.</p> </li> <li> <code>share_weights</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>In addition to sharing the \\(V\\) and \\(Q\\) transformation parameters, the parameters across different Fastformer layers can also be shared. Please, see <code>pytorch_widedeep/models/tabular/transformers/tab_fastformer.py</code> for details</p> </li> <li> <code>transformer_activation</code>               (<code>str</code>, default:                   <code>'relu'</code> )           \u2013            <p>Transformer Encoder activation function. 'tanh', 'relu', 'leaky_relu', 'gelu', 'geglu' and 'reglu' are supported</p> </li> <li> <code>mlp_hidden_dims</code>               (<code>Optional[List[int]]</code>, default:                   <code>None</code> )           \u2013            <p>MLP hidden dimensions. If not provided no MLP on top of the final FTTransformer block will be used</p> </li> <li> <code>mlp_hidden_dims</code>               (<code>Optional[List[int]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the number of neurons per dense layer in the MLP. e.g: [64, 32]. If not provided no MLP on top of the final Transformer block will be used.</p> </li> <li> <code>mlp_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the dense layers of the MLP. Currently 'tanh', 'relu', 'leaky'_relu' and _'gelu' are supported. If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to 'relu'.</p> </li> <li> <code>mlp_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>float with the dropout between the dense layers of the MLP. If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to 0.0.</p> </li> <li> <code>mlp_batchnorm</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating whether or not batch normalization will be applied to the dense layers If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to False.</p> </li> <li> <code>mlp_batchnorm_last</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating whether or not batch normalization will be applied to the last of the dense layers If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to False.</p> </li> <li> <code>mlp_linear_first</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code> If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to <code>True</code>.</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>encoder</code>               (<code>Module</code>)           \u2013            <p>Sequence of FasFormer blocks.</p> </li> <li> <code>mlp</code>               (<code>Module</code>)           \u2013            <p>MLP component in the model</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import TabFastFormer\n&gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n&gt;&gt;&gt; colnames = ['a', 'b', 'c', 'd', 'e']\n&gt;&gt;&gt; cat_embed_input = [(u,i) for u,i in zip(colnames[:4], [4]*4)]\n&gt;&gt;&gt; continuous_cols = ['e']\n&gt;&gt;&gt; column_idx = {k:v for v,k in enumerate(colnames)}\n&gt;&gt;&gt; model = TabFastFormer(column_idx=column_idx, cat_embed_input=cat_embed_input, continuous_cols=continuous_cols)\n&gt;&gt;&gt; out = model(X_tab)\n</code></pre> Source code in <code>pytorch_widedeep/models/tabular/transformers/tab_fastformer.py</code> <pre><code>def __init__(\n    self,\n    column_idx: Dict[str, int],\n    *,\n    cat_embed_input: Optional[List[Tuple[str, int]]] = None,\n    cat_embed_dropout: Optional[float] = None,\n    use_cat_bias: Optional[bool] = None,\n    cat_embed_activation: Optional[str] = None,\n    shared_embed: Optional[bool] = None,\n    add_shared_embed: Optional[bool] = None,\n    frac_shared_embed: Optional[float] = None,\n    continuous_cols: Optional[List[str]] = None,\n    cont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\n    embed_continuous_method: Optional[\n        Literal[\"standard\", \"piecewise\", \"periodic\"]\n    ] = \"standard\",\n    cont_embed_dropout: Optional[float] = None,\n    cont_embed_activation: Optional[str] = None,\n    quantization_setup: Optional[Dict[str, List[float]]] = None,\n    n_frequencies: Optional[int] = None,\n    sigma: Optional[float] = None,\n    share_last_layer: Optional[bool] = None,\n    full_embed_dropout: Optional[bool] = None,\n    input_dim: int = 32,\n    n_heads: int = 8,\n    use_bias: bool = False,\n    n_blocks: int = 4,\n    attn_dropout: float = 0.1,\n    ff_dropout: float = 0.2,\n    ff_factor: int = 4,\n    share_qv_weights: bool = False,\n    share_weights: bool = False,\n    transformer_activation: str = \"relu\",\n    mlp_hidden_dims: Optional[List[int]] = None,\n    mlp_activation: Optional[str] = None,\n    mlp_dropout: Optional[float] = None,\n    mlp_batchnorm: Optional[bool] = None,\n    mlp_batchnorm_last: Optional[bool] = None,\n    mlp_linear_first: Optional[bool] = None,\n):\n    super(TabFastFormer, self).__init__(\n        column_idx=column_idx,\n        cat_embed_input=cat_embed_input,\n        cat_embed_dropout=cat_embed_dropout,\n        use_cat_bias=use_cat_bias,\n        cat_embed_activation=cat_embed_activation,\n        shared_embed=shared_embed,\n        add_shared_embed=add_shared_embed,\n        frac_shared_embed=frac_shared_embed,\n        continuous_cols=continuous_cols,\n        cont_norm_layer=cont_norm_layer,\n        embed_continuous_method=embed_continuous_method,\n        cont_embed_dropout=cont_embed_dropout,\n        cont_embed_activation=cont_embed_activation,\n        input_dim=input_dim,\n        quantization_setup=quantization_setup,\n        n_frequencies=n_frequencies,\n        sigma=sigma,\n        share_last_layer=share_last_layer,\n        full_embed_dropout=full_embed_dropout,\n    )\n\n    self.n_heads = n_heads\n    self.use_bias = use_bias\n    self.n_blocks = n_blocks\n    self.attn_dropout = attn_dropout\n    self.ff_dropout = ff_dropout\n    self.ff_factor = ff_factor\n    self.share_qv_weights = share_qv_weights\n    self.share_weights = share_weights\n    self.transformer_activation = transformer_activation\n\n    self.mlp_hidden_dims = mlp_hidden_dims\n    self.mlp_activation = mlp_activation\n    self.mlp_dropout = mlp_dropout\n    self.mlp_batchnorm = mlp_batchnorm\n    self.mlp_batchnorm_last = mlp_batchnorm_last\n    self.mlp_linear_first = mlp_linear_first\n\n    self.with_cls_token = \"cls_token\" in column_idx\n    self.n_cat = len(cat_embed_input) if cat_embed_input is not None else 0\n    self.n_cont = len(continuous_cols) if continuous_cols is not None else 0\n    self.n_feats = self.n_cat + self.n_cont\n\n    # Embeddings are instantiated at the base model\n    # Transformer blocks\n    self.encoder = nn.Sequential()\n    first_fastformer_block = FastFormerEncoder(\n        input_dim,\n        n_heads,\n        use_bias,\n        attn_dropout,\n        ff_dropout,\n        ff_factor,\n        share_qv_weights,\n        transformer_activation,\n    )\n    self.encoder.add_module(\"fastformer_block0\", first_fastformer_block)\n    for i in range(1, n_blocks):\n        if share_weights:\n            self.encoder.add_module(\n                \"fastformer_block\" + str(i), first_fastformer_block\n            )\n        else:\n            self.encoder.add_module(\n                \"fastformer_block\" + str(i),\n                FastFormerEncoder(\n                    input_dim,\n                    n_heads,\n                    use_bias,\n                    attn_dropout,\n                    ff_dropout,\n                    ff_factor,\n                    share_qv_weights,\n                    transformer_activation,\n                ),\n            )\n\n    self.mlp_first_hidden_dim = (\n        self.input_dim if self.with_cls_token else (self.n_feats * self.input_dim)\n    )\n\n    # Mlp: adding an MLP on top of the Resnet blocks is optional and\n    # therefore all related params are optional\n    if self.mlp_hidden_dims is not None:\n        self.mlp = MLP(\n            d_hidden=[self.mlp_first_hidden_dim] + self.mlp_hidden_dims,\n            activation=(\n                \"relu\" if self.mlp_activation is None else self.mlp_activation\n            ),\n            dropout=0.0 if self.mlp_dropout is None else self.mlp_dropout,\n            batchnorm=False if self.mlp_batchnorm is None else self.mlp_batchnorm,\n            batchnorm_last=(\n                False\n                if self.mlp_batchnorm_last is None\n                else self.mlp_batchnorm_last\n            ),\n            linear_first=(\n                False if self.mlp_linear_first is None else self.mlp_linear_first\n            ),\n        )\n    else:\n        self.mlp = None\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.transformers.tab_fastformer.TabFastFormer.output_dim","title":"output_dim  <code>property</code>","text":"<pre><code>output_dim\n</code></pre> <p>The output dimension of the model. This is a required property neccesary to build the <code>WideDeep</code> class</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.transformers.tab_fastformer.TabFastFormer.attention_weights","title":"attention_weights  <code>property</code>","text":"<pre><code>attention_weights\n</code></pre> <p>List with the attention weights. Each element of the list is a tuple where the first and second elements are the \\(\\alpha\\) and \\(\\beta\\) attention weights in the paper.</p> <p>The shape of the attention weights is \\((N, H, F)\\) where \\(N\\) is the batch size, \\(H\\) is the number of attention heads and \\(F\\) is the number of features/columns in the dataset</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.text.rnns.basic_rnn.BasicRNN","title":"BasicRNN","text":"<pre><code>BasicRNN(\n    vocab_size,\n    embed_dim=None,\n    embed_matrix=None,\n    embed_trainable=True,\n    rnn_type=\"lstm\",\n    hidden_dim=64,\n    n_layers=3,\n    rnn_dropout=0.0,\n    bidirectional=False,\n    use_hidden_state=True,\n    padding_idx=1,\n    head_hidden_dims=None,\n    head_activation=\"relu\",\n    head_dropout=None,\n    head_batchnorm=False,\n    head_batchnorm_last=False,\n    head_linear_first=False,\n)\n</code></pre> <p>               Bases: <code>BaseWDModelComponent</code></p> <p>Standard text classifier/regressor comprised by a stack of RNNs (LSTMs or GRUs) that can be used as the <code>deeptext</code> component of a Wide &amp; Deep model or independently by itself.</p> <p>In addition, there is the option to add a Fully Connected (FC) set of dense layers on top of the stack of RNNs</p> <p>Parameters:</p> <ul> <li> <code>vocab_size</code>               (<code>int</code>)           \u2013            <p>Number of words in the vocabulary</p> </li> <li> <code>embed_dim</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>Dimension of the word embeddings if non-pretained word vectors are used</p> </li> <li> <code>embed_matrix</code>               (<code>Optional[ndarray]</code>, default:                   <code>None</code> )           \u2013            <p>Pretrained word embeddings</p> </li> <li> <code>embed_trainable</code>               (<code>bool</code>, default:                   <code>True</code> )           \u2013            <p>Boolean indicating if the pretrained embeddings are trainable</p> </li> <li> <code>rnn_type</code>               (<code>Literal[lstm, gru]</code>, default:                   <code>'lstm'</code> )           \u2013            <p>String indicating the type of RNN to use. One of 'lstm' or 'gru'</p> </li> <li> <code>hidden_dim</code>               (<code>int</code>, default:                   <code>64</code> )           \u2013            <p>Hidden dim of the RNN</p> </li> <li> <code>n_layers</code>               (<code>int</code>, default:                   <code>3</code> )           \u2013            <p>Number of recurrent layers</p> </li> <li> <code>rnn_dropout</code>               (<code>float</code>, default:                   <code>0.0</code> )           \u2013            <p>Dropout for each RNN layer except the last layer</p> </li> <li> <code>bidirectional</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether the staked RNNs are bidirectional</p> </li> <li> <code>use_hidden_state</code>               (<code>bool</code>, default:                   <code>True</code> )           \u2013            <p>Boolean indicating whether to use the final hidden state or the RNN's output as predicting features. Typically the former is used.</p> </li> <li> <code>padding_idx</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>index of the padding token in the padded-tokenised sequences. The <code>TextPreprocessor</code> class within this library uses fastai's tokenizer where the token index 0 is reserved for the 'unknown' word token. Therefore, the default value is set to 1.</p> </li> <li> <code>head_hidden_dims</code>               (<code>Optional[List[int]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the sizes of the dense layers in the head e.g: [128, 64]</p> </li> <li> <code>head_activation</code>               (<code>str</code>, default:                   <code>'relu'</code> )           \u2013            <p>Activation function for the dense layers in the head. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> </li> <li> <code>head_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Dropout of the dense layers in the head</p> </li> <li> <code>head_batchnorm</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether or not to include batch normalization in the dense layers that form the 'rnn_mlp'</p> </li> <li> <code>head_batchnorm_last</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether or not to apply batch normalization to the last of the dense layers in the head</p> </li> <li> <code>head_linear_first</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code></p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>word_embed</code>               (<code>Module</code>)           \u2013            <p>word embedding matrix</p> </li> <li> <code>rnn</code>               (<code>Module</code>)           \u2013            <p>Stack of RNNs</p> </li> <li> <code>rnn_mlp</code>               (<code>Module</code>)           \u2013            <p>Stack of dense layers on top of the RNN. This will only exists if <code>head_layers_dim</code> is not None</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import BasicRNN\n&gt;&gt;&gt; X_text = torch.cat((torch.zeros([5,1]), torch.empty(5, 4).random_(1,4)), axis=1)\n&gt;&gt;&gt; model = BasicRNN(vocab_size=4, hidden_dim=4, n_layers=2, padding_idx=0, embed_dim=4)\n&gt;&gt;&gt; out = model(X_text)\n</code></pre> Source code in <code>pytorch_widedeep/models/text/rnns/basic_rnn.py</code> <pre><code>def __init__(\n    self,\n    vocab_size: int,\n    embed_dim: Optional[int] = None,\n    embed_matrix: Optional[np.ndarray] = None,\n    embed_trainable: bool = True,\n    rnn_type: Literal[\"lstm\", \"gru\"] = \"lstm\",\n    hidden_dim: int = 64,\n    n_layers: int = 3,\n    rnn_dropout: float = 0.0,\n    bidirectional: bool = False,\n    use_hidden_state: bool = True,\n    padding_idx: int = 1,\n    head_hidden_dims: Optional[List[int]] = None,\n    head_activation: str = \"relu\",\n    head_dropout: Optional[float] = None,\n    head_batchnorm: bool = False,\n    head_batchnorm_last: bool = False,\n    head_linear_first: bool = False,\n):\n    super(BasicRNN, self).__init__()\n\n    if embed_dim is None and embed_matrix is None:\n        raise ValueError(\n            \"If no 'embed_matrix' is passed, the embedding dimension must\"\n            \"be specified with 'embed_dim'\"\n        )\n\n    if rnn_type.lower() not in [\"lstm\", \"gru\"]:\n        raise ValueError(\n            f\"'rnn_type' must be 'lstm' or 'gru', got {rnn_type} instead\"\n        )\n\n    if (\n        embed_dim is not None\n        and embed_matrix is not None\n        and not embed_dim == embed_matrix.shape[1]\n    ):\n        warnings.warn(\n            \"the input embedding dimension {} and the dimension of the \"\n            \"pretrained embeddings {} do not match. The pretrained embeddings \"\n            \"dimension ({}) will be used\".format(\n                embed_dim, embed_matrix.shape[1], embed_matrix.shape[1]\n            ),\n            UserWarning,\n        )\n\n    self.vocab_size = vocab_size\n    self.embed_trainable = embed_trainable\n    self.embed_dim = embed_dim\n\n    self.rnn_type = rnn_type\n    self.hidden_dim = hidden_dim\n    self.n_layers = n_layers\n    self.rnn_dropout = rnn_dropout\n    self.bidirectional = bidirectional\n    self.use_hidden_state = use_hidden_state\n    self.padding_idx = padding_idx\n\n    self.head_hidden_dims = head_hidden_dims\n    self.head_activation = head_activation\n    self.head_dropout = head_dropout\n    self.head_batchnorm = head_batchnorm\n    self.head_batchnorm_last = head_batchnorm_last\n    self.head_linear_first = head_linear_first\n\n    # Embeddings\n    if embed_matrix is not None:\n        self.word_embed, self.embed_dim = self._set_embeddings(embed_matrix)\n    else:\n        assert self.embed_dim is not None\n        self.word_embed = nn.Embedding(\n            self.vocab_size, self.embed_dim, padding_idx=self.padding_idx\n        )\n\n    # RNN\n    rnn_params = {\n        \"input_size\": self.embed_dim,\n        \"hidden_size\": hidden_dim,\n        \"num_layers\": n_layers,\n        \"bidirectional\": bidirectional,\n        \"dropout\": rnn_dropout,\n        \"batch_first\": True,\n    }\n    if self.rnn_type.lower() == \"lstm\":\n        self.rnn: Union[nn.LSTM, nn.GRU] = nn.LSTM(**rnn_params)\n    elif self.rnn_type.lower() == \"gru\":\n        self.rnn = nn.GRU(**rnn_params)\n    else:\n        raise ValueError(\n            f\"'rnn_type' must be 'lstm' or 'gru', got {self.rnn_type} instead\"\n        )\n\n    self.rnn_output_dim = hidden_dim * 2 if bidirectional else hidden_dim\n\n    # FC-Head (Mlp)\n    if self.head_hidden_dims is not None:\n        head_hidden_dims = [self.rnn_output_dim] + self.head_hidden_dims\n        self.rnn_mlp: Union[MLP, nn.Identity] = MLP(\n            head_hidden_dims,\n            head_activation,\n            head_dropout,\n            head_batchnorm,\n            head_batchnorm_last,\n            head_linear_first,\n        )\n    else:\n        # simple hack to add readability in the forward pass\n        self.rnn_mlp = nn.Identity()\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.text.rnns.basic_rnn.BasicRNN.output_dim","title":"output_dim  <code>property</code>","text":"<pre><code>output_dim\n</code></pre> <p>The output dimension of the model. This is a required property neccesary to build the <code>WideDeep</code> class</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.text.rnns.attentive_rnn.AttentiveRNN","title":"AttentiveRNN","text":"<pre><code>AttentiveRNN(\n    vocab_size,\n    embed_dim=None,\n    embed_matrix=None,\n    embed_trainable=True,\n    rnn_type=\"lstm\",\n    hidden_dim=64,\n    n_layers=3,\n    rnn_dropout=0.1,\n    bidirectional=False,\n    use_hidden_state=True,\n    padding_idx=1,\n    attn_concatenate=True,\n    attn_dropout=0.1,\n    head_hidden_dims=None,\n    head_activation=\"relu\",\n    head_dropout=None,\n    head_batchnorm=False,\n    head_batchnorm_last=False,\n    head_linear_first=False,\n)\n</code></pre> <p>               Bases: <code>BasicRNN</code></p> <p>Text classifier/regressor comprised by a stack of RNNs (LSTMs or GRUs) plus an attention layer. This model can be used as the <code>deeptext</code> component of a Wide &amp; Deep model or independently by itself.</p> <p>In addition, there is the option to add a Fully Connected (FC) set of dense layers on top of attention layer</p> <p>Parameters:</p> <ul> <li> <code>vocab_size</code>               (<code>int</code>)           \u2013            <p>Number of words in the vocabulary</p> </li> <li> <code>embed_dim</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>Dimension of the word embeddings if non-pretained word vectors are used</p> </li> <li> <code>embed_matrix</code>               (<code>Optional[ndarray]</code>, default:                   <code>None</code> )           \u2013            <p>Pretrained word embeddings</p> </li> <li> <code>embed_trainable</code>               (<code>bool</code>, default:                   <code>True</code> )           \u2013            <p>Boolean indicating if the pretrained embeddings are trainable</p> </li> <li> <code>rnn_type</code>               (<code>Literal[lstm, gru]</code>, default:                   <code>'lstm'</code> )           \u2013            <p>String indicating the type of RNN to use. One of 'lstm' or 'gru'</p> </li> <li> <code>hidden_dim</code>               (<code>int</code>, default:                   <code>64</code> )           \u2013            <p>Hidden dim of the RNN</p> </li> <li> <code>n_layers</code>               (<code>int</code>, default:                   <code>3</code> )           \u2013            <p>Number of recurrent layers</p> </li> <li> <code>rnn_dropout</code>               (<code>float</code>, default:                   <code>0.1</code> )           \u2013            <p>Dropout for each RNN layer except the last layer</p> </li> <li> <code>bidirectional</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether the staked RNNs are bidirectional</p> </li> <li> <code>use_hidden_state</code>               (<code>bool</code>, default:                   <code>True</code> )           \u2013            <p>Boolean indicating whether to use the final hidden state or the RNN's output as predicting features. Typically the former is used.</p> </li> <li> <code>padding_idx</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>index of the padding token in the padded-tokenised sequences. The <code>TextPreprocessor</code> class within this library uses fastai's tokenizer where the token index 0 is reserved for the 'unknown' word token. Therefore, the default value is set to 1.</p> </li> <li> <code>attn_concatenate</code>               (<code>bool</code>, default:                   <code>True</code> )           \u2013            <p>Boolean indicating if the input to the attention mechanism will be the output of the RNN or the output of the RNN concatenated with the last hidden state.</p> </li> <li> <code>attn_dropout</code>               (<code>float</code>, default:                   <code>0.1</code> )           \u2013            <p>Internal dropout for the attention mechanism</p> </li> <li> <code>head_hidden_dims</code>               (<code>Optional[List[int]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the sizes of the dense layers in the head e.g: [128, 64]</p> </li> <li> <code>head_activation</code>               (<code>str</code>, default:                   <code>'relu'</code> )           \u2013            <p>Activation function for the dense layers in the head. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> </li> <li> <code>head_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Dropout of the dense layers in the head</p> </li> <li> <code>head_batchnorm</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether or not to include batch normalization in the dense layers that form the 'rnn_mlp'</p> </li> <li> <code>head_batchnorm_last</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether or not to apply batch normalization to the last of the dense layers in the head</p> </li> <li> <code>head_linear_first</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code></p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>word_embed</code>               (<code>Module</code>)           \u2013            <p>word embedding matrix</p> </li> <li> <code>rnn</code>               (<code>Module</code>)           \u2013            <p>Stack of RNNs</p> </li> <li> <code>rnn_mlp</code>               (<code>Module</code>)           \u2013            <p>Stack of dense layers on top of the RNN. This will only exists if <code>head_layers_dim</code> is not <code>None</code></p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import AttentiveRNN\n&gt;&gt;&gt; X_text = torch.cat((torch.zeros([5,1]), torch.empty(5, 4).random_(1,4)), axis=1)\n&gt;&gt;&gt; model = AttentiveRNN(vocab_size=4, hidden_dim=4, n_layers=2, padding_idx=0, embed_dim=4)\n&gt;&gt;&gt; out = model(X_text)\n</code></pre> Source code in <code>pytorch_widedeep/models/text/rnns/attentive_rnn.py</code> <pre><code>def __init__(\n    self,\n    vocab_size: int,\n    embed_dim: Optional[int] = None,\n    embed_matrix: Optional[np.ndarray] = None,\n    embed_trainable: bool = True,\n    rnn_type: Literal[\"lstm\", \"gru\"] = \"lstm\",\n    hidden_dim: int = 64,\n    n_layers: int = 3,\n    rnn_dropout: float = 0.1,\n    bidirectional: bool = False,\n    use_hidden_state: bool = True,\n    padding_idx: int = 1,\n    attn_concatenate: bool = True,\n    attn_dropout: float = 0.1,\n    head_hidden_dims: Optional[List[int]] = None,\n    head_activation: str = \"relu\",\n    head_dropout: Optional[float] = None,\n    head_batchnorm: bool = False,\n    head_batchnorm_last: bool = False,\n    head_linear_first: bool = False,\n):\n    super(AttentiveRNN, self).__init__(\n        vocab_size=vocab_size,\n        embed_dim=embed_dim,\n        embed_matrix=embed_matrix,\n        embed_trainable=embed_trainable,\n        rnn_type=rnn_type,\n        hidden_dim=hidden_dim,\n        n_layers=n_layers,\n        rnn_dropout=rnn_dropout,\n        bidirectional=bidirectional,\n        use_hidden_state=use_hidden_state,\n        padding_idx=padding_idx,\n        head_hidden_dims=head_hidden_dims,\n        head_activation=head_activation,\n        head_dropout=head_dropout,\n        head_batchnorm=head_batchnorm,\n        head_batchnorm_last=head_batchnorm_last,\n        head_linear_first=head_linear_first,\n    )\n\n    # Embeddings and RNN defined in the BasicRNN inherited class\n\n    # Attention\n    self.attn_concatenate = attn_concatenate\n    self.attn_dropout = attn_dropout\n\n    if bidirectional and attn_concatenate:\n        self.rnn_output_dim = hidden_dim * 4\n    elif bidirectional or attn_concatenate:\n        self.rnn_output_dim = hidden_dim * 2\n    else:\n        self.rnn_output_dim = hidden_dim\n    self.attn = ContextAttention(\n        self.rnn_output_dim, attn_dropout, sum_along_seq=True\n    )\n\n    # FC-Head (Mlp)\n    if self.head_hidden_dims is not None:\n        head_hidden_dims = [self.rnn_output_dim] + self.head_hidden_dims\n        self.rnn_mlp = MLP(\n            head_hidden_dims,\n            head_activation,\n            head_dropout,\n            head_batchnorm,\n            head_batchnorm_last,\n            head_linear_first,\n        )\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.text.rnns.attentive_rnn.AttentiveRNN.attention_weights","title":"attention_weights  <code>property</code>","text":"<pre><code>attention_weights\n</code></pre> <p>List with the attention weights</p> <p>The shape of the attention weights is \\((N, S)\\), where \\(N\\) is the batch size and \\(S\\) is the length of the sequence</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.text.rnns.stacked_attentive_rnn.StackedAttentiveRNN","title":"StackedAttentiveRNN","text":"<pre><code>StackedAttentiveRNN(\n    vocab_size,\n    embed_dim=None,\n    embed_matrix=None,\n    embed_trainable=True,\n    rnn_type=\"lstm\",\n    hidden_dim=64,\n    bidirectional=False,\n    padding_idx=1,\n    n_blocks=3,\n    attn_concatenate=False,\n    attn_dropout=0.1,\n    with_addnorm=False,\n    head_hidden_dims=None,\n    head_activation=\"relu\",\n    head_dropout=None,\n    head_batchnorm=False,\n    head_batchnorm_last=False,\n    head_linear_first=False,\n)\n</code></pre> <p>               Bases: <code>BaseWDModelComponent</code></p> <p>Text classifier/regressor comprised by a stack of blocks: <code>[RNN + Attention]</code>. This can be used as the <code>deeptext</code> component of a Wide &amp; Deep model or independently by itself.</p> <p>In addition, there is the option to add a Fully Connected (FC) set of dense layers on top of the attentiob blocks</p> <p>Parameters:</p> <ul> <li> <code>vocab_size</code>               (<code>int</code>)           \u2013            <p>Number of words in the vocabulary</p> </li> <li> <code>embed_dim</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>Dimension of the word embeddings if non-pretained word vectors are used</p> </li> <li> <code>embed_matrix</code>               (<code>Optional[ndarray]</code>, default:                   <code>None</code> )           \u2013            <p>Pretrained word embeddings</p> </li> <li> <code>embed_trainable</code>               (<code>bool</code>, default:                   <code>True</code> )           \u2013            <p>Boolean indicating if the pretrained embeddings are trainable</p> </li> <li> <code>rnn_type</code>               (<code>Literal[lstm, gru]</code>, default:                   <code>'lstm'</code> )           \u2013            <p>String indicating the type of RNN to use. One of 'lstm' or 'gru'</p> </li> <li> <code>hidden_dim</code>               (<code>int</code>, default:                   <code>64</code> )           \u2013            <p>Hidden dim of the RNN</p> </li> <li> <code>bidirectional</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether the staked RNNs are bidirectional</p> </li> <li> <code>padding_idx</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>index of the padding token in the padded-tokenised sequences. The <code>TextPreprocessor</code> class within this library uses fastai's tokenizer where the token index 0 is reserved for the 'unknown' word token. Therefore, the default value is set to 1.</p> </li> <li> <code>n_blocks</code>               (<code>int</code>, default:                   <code>3</code> )           \u2013            <p>Number of attention blocks. Each block is comprised by an RNN and a Context Attention Encoder</p> </li> <li> <code>attn_concatenate</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating if the input to the attention mechanism will be the output of the RNN or the output of the RNN concatenated with the last hidden state or simply</p> </li> <li> <code>attn_dropout</code>               (<code>float</code>, default:                   <code>0.1</code> )           \u2013            <p>Internal dropout for the attention mechanism</p> </li> <li> <code>with_addnorm</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating if the output of each block will be added to the input and normalised</p> </li> <li> <code>head_hidden_dims</code>               (<code>Optional[List[int]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the sizes of the dense layers in the head e.g: [128, 64]</p> </li> <li> <code>head_activation</code>               (<code>str</code>, default:                   <code>'relu'</code> )           \u2013            <p>Activation function for the dense layers in the head. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> </li> <li> <code>head_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Dropout of the dense layers in the head</p> </li> <li> <code>head_batchnorm</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether or not to include batch normalization in the dense layers that form the 'rnn_mlp'</p> </li> <li> <code>head_batchnorm_last</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether or not to apply batch normalization to the last of the dense layers in the head</p> </li> <li> <code>head_linear_first</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code></p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>word_embed</code>               (<code>Module</code>)           \u2013            <p>word embedding matrix</p> </li> <li> <code>rnn</code>               (<code>Module</code>)           \u2013            <p>Stack of RNNs</p> </li> <li> <code>rnn_mlp</code>               (<code>Module</code>)           \u2013            <p>Stack of dense layers on top of the RNN. This will only exists if <code>head_layers_dim</code> is not <code>None</code></p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import StackedAttentiveRNN\n&gt;&gt;&gt; X_text = torch.cat((torch.zeros([5,1]), torch.empty(5, 4).random_(1,4)), axis=1)\n&gt;&gt;&gt; model = StackedAttentiveRNN(vocab_size=4, hidden_dim=4, padding_idx=0, embed_dim=4)\n&gt;&gt;&gt; out = model(X_text)\n</code></pre> Source code in <code>pytorch_widedeep/models/text/rnns/stacked_attentive_rnn.py</code> <pre><code>def __init__(\n    self,\n    vocab_size: int,\n    embed_dim: Optional[int] = None,\n    embed_matrix: Optional[np.ndarray] = None,\n    embed_trainable: bool = True,\n    rnn_type: Literal[\"lstm\", \"gru\"] = \"lstm\",\n    hidden_dim: int = 64,\n    bidirectional: bool = False,\n    padding_idx: int = 1,\n    n_blocks: int = 3,\n    attn_concatenate: bool = False,\n    attn_dropout: float = 0.1,\n    with_addnorm: bool = False,\n    head_hidden_dims: Optional[List[int]] = None,\n    head_activation: str = \"relu\",\n    head_dropout: Optional[float] = None,\n    head_batchnorm: bool = False,\n    head_batchnorm_last: bool = False,\n    head_linear_first: bool = False,\n):\n    super(StackedAttentiveRNN, self).__init__()\n\n    if (\n        embed_dim is not None\n        and embed_matrix is not None\n        and not embed_dim == embed_matrix.shape[1]\n    ):\n        warnings.warn(\n            \"the input embedding dimension {} and the dimension of the \"\n            \"pretrained embeddings {} do not match. The pretrained embeddings \"\n            \"dimension ({}) will be used\".format(\n                embed_dim, embed_matrix.shape[1], embed_matrix.shape[1]\n            ),\n            UserWarning,\n        )\n\n    if rnn_type.lower() not in [\"lstm\", \"gru\"]:\n        raise ValueError(\n            f\"'rnn_type' must be 'lstm' or 'gru', got {rnn_type} instead\"\n        )\n\n    self.vocab_size = vocab_size\n    self.embed_trainable = embed_trainable\n    self.embed_dim = embed_dim\n\n    self.rnn_type = rnn_type\n    self.hidden_dim = hidden_dim\n    self.bidirectional = bidirectional\n    self.padding_idx = padding_idx\n\n    self.n_blocks = n_blocks\n    self.attn_concatenate = attn_concatenate\n    self.attn_dropout = attn_dropout\n    self.with_addnorm = with_addnorm\n\n    self.head_hidden_dims = head_hidden_dims\n    self.head_activation = head_activation\n    self.head_dropout = head_dropout\n    self.head_batchnorm = head_batchnorm\n    self.head_batchnorm_last = head_batchnorm_last\n    self.head_linear_first = head_linear_first\n\n    # Embeddings\n    self.word_embed, self.embed_dim = self._set_embeddings(embed_matrix)\n\n    # Linear Projection: if embed_dim is different that the input of the\n    # attention blocks we add a linear projection\n    if bidirectional and attn_concatenate:\n        self.rnn_output_dim = hidden_dim * 4\n    elif bidirectional or attn_concatenate:\n        self.rnn_output_dim = hidden_dim * 2\n    else:\n        self.rnn_output_dim = hidden_dim\n\n    if self.rnn_output_dim != self.embed_dim:\n        self.embed_proj: Union[nn.Linear, nn.Identity] = nn.Linear(\n            self.embed_dim, self.rnn_output_dim\n        )\n    else:\n        self.embed_proj = nn.Identity()\n\n    # RNN\n    rnn_params = {\n        \"input_size\": self.rnn_output_dim,\n        \"hidden_size\": hidden_dim,\n        \"bidirectional\": bidirectional,\n        \"batch_first\": True,\n    }\n    if self.rnn_type.lower() == \"lstm\":\n        self.rnn: Union[nn.LSTM, nn.GRU] = nn.LSTM(**rnn_params)\n    elif self.rnn_type.lower() == \"gru\":\n        self.rnn = nn.GRU(**rnn_params)\n\n    # FC-Head (Mlp)\n    self.attention_blks = nn.ModuleList()\n    for i in range(n_blocks):\n        self.attention_blks.append(\n            ContextAttentionEncoder(\n                self.rnn,\n                self.rnn_output_dim,\n                attn_dropout,\n                attn_concatenate,\n                with_addnorm=with_addnorm if i != n_blocks - 1 else False,\n                sum_along_seq=i == n_blocks - 1,\n            )\n        )\n\n    # Mlp\n    if self.head_hidden_dims is not None:\n        head_hidden_dims = [self.rnn_output_dim] + self.head_hidden_dims\n        self.rnn_mlp: Union[MLP, nn.Identity] = MLP(\n            head_hidden_dims,\n            head_activation,\n            head_dropout,\n            head_batchnorm,\n            head_batchnorm_last,\n            head_linear_first,\n        )\n    else:\n        # simple hack to add readability in the forward pass\n        self.rnn_mlp = nn.Identity()\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.text.rnns.stacked_attentive_rnn.StackedAttentiveRNN.output_dim","title":"output_dim  <code>property</code>","text":"<pre><code>output_dim\n</code></pre> <p>The output dimension of the model. This is a required property neccesary to build the <code>WideDeep</code> class</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.text.rnns.stacked_attentive_rnn.StackedAttentiveRNN.attention_weights","title":"attention_weights  <code>property</code>","text":"<pre><code>attention_weights\n</code></pre> <p>List with the attention weights per block</p> <p>The shape of the attention weights is \\((N, S)\\) Where \\(N\\) is the batch size and \\(S\\) is the length of the sequence</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.text.huggingface_transformers.hf_model.HFModel","title":"HFModel","text":"<pre><code>HFModel(\n    model_name,\n    use_cls_token=True,\n    trainable_parameters=None,\n    head_hidden_dims=None,\n    head_activation=\"relu\",\n    head_dropout=None,\n    head_batchnorm=False,\n    head_batchnorm_last=False,\n    head_linear_first=False,\n    verbose=False,\n    **kwargs\n)\n</code></pre> <p>               Bases: <code>BaseWDModelComponent</code></p> <p>This class is a wrapper around the Hugging Face transformers library. It can be used as the text component of a Wide &amp; Deep model or independently by itself.</p> <p>At the moment only models from the families BERT, RoBERTa, DistilBERT, ALBERT and ELECTRA are supported. This is because this library is designed to address classification and regression tasks and these are the most 'popular' encoder-only models, which have proved to be those that work best for these tasks.</p> <p>Parameters:</p> <ul> <li> <code>model_name</code>               (<code>str</code>)           \u2013            <p>The model name from the transformers library e.g. 'bert-base-uncased'. Currently supported models are those from the families: BERT, RoBERTa, DistilBERT, ALBERT and ELECTRA.</p> </li> <li> <code>use_cls_token</code>               (<code>bool</code>, default:                   <code>True</code> )           \u2013            <p>Boolean indicating whether to use the [CLS] token or the mean of the sequence of hidden states as the sentence embedding</p> </li> <li> <code>trainable_parameters</code>               (<code>Optional[List[str]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the names of the model parameters that will be trained. If None, none of the parameters will be trainable</p> </li> <li> <code>head_hidden_dims</code>               (<code>Optional[List[int]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the sizes of the dense layers in the head e.g: [128, 64]</p> </li> <li> <code>head_activation</code>               (<code>str</code>, default:                   <code>'relu'</code> )           \u2013            <p>Activation function for the dense layers in the head. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> </li> <li> <code>head_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Dropout of the dense layers in the head</p> </li> <li> <code>head_batchnorm</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether or not to include batch normalization in the dense layers that form the head</p> </li> <li> <code>head_batchnorm_last</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether or not to apply batch normalization to the last of the dense layers in the head</p> </li> <li> <code>head_linear_first</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code></p> </li> <li> <code>verbose</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>If True, it will print information about the model</p> </li> <li> <code>**kwargs</code>           \u2013            <p>Additional kwargs to be passed to the model</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>head</code>               (<code>Module</code>)           \u2013            <p>Stack of dense layers on top of the transformer. This will only exists if <code>head_layers_dim</code> is not None</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import HFModel\n&gt;&gt;&gt; X_text = torch.cat((torch.zeros([5,1]), torch.empty(5, 4).random_(1,4)), axis=1).long()\n&gt;&gt;&gt; model = HFModel(model_name='bert-base-uncased')\n&gt;&gt;&gt; out = model(X_text)\n</code></pre> Source code in <code>pytorch_widedeep/models/text/huggingface_transformers/hf_model.py</code> <pre><code>@alias(\"use_cls_token\", [\"use_special_token\"])\ndef __init__(\n    self,\n    model_name: str,\n    use_cls_token: bool = True,\n    trainable_parameters: Optional[List[str]] = None,\n    head_hidden_dims: Optional[List[int]] = None,\n    head_activation: str = \"relu\",\n    head_dropout: Optional[float] = None,\n    head_batchnorm: bool = False,\n    head_batchnorm_last: bool = False,\n    head_linear_first: bool = False,\n    verbose: bool = False,\n    **kwargs,\n):\n    super().__init__()\n\n    # TO DO: add warning regarging ELECTRA as ELECTRA does not have a cls\n    # token.  Research what happens with ELECTRA\n    self.model_name = model_name\n    self.use_cls_token = use_cls_token\n    self.trainable_parameters = trainable_parameters\n    self.head_hidden_dims = head_hidden_dims\n    self.head_activation = head_activation\n    self.head_dropout = head_dropout\n    self.head_batchnorm = head_batchnorm\n    self.head_batchnorm_last = head_batchnorm_last\n    self.head_linear_first = head_linear_first\n    self.verbose = verbose\n    self.kwargs = kwargs\n\n    if self.verbose and self.use_cls_token:\n        warnings.warn(\n            \"The model will use the [CLS] token. Make sure the tokenizer \"\n            \"was run with add_special_tokens=True\",\n            UserWarning,\n        )\n\n    self.model_class = get_model_class(model_name)\n\n    self.config, self.model = get_config_and_model(self.model_name)\n\n    self.output_attention_weights = kwargs.get(\"output_attentions\", False)\n\n    if self.trainable_parameters is not None:\n        for n, p in self.model.named_parameters():\n            p.requires_grad = any([tl in n for tl in self.trainable_parameters])\n\n    # FC-Head (Mlp). Note that the FC head will always be trainable\n    if self.head_hidden_dims is not None:\n        head_hidden_dims = [self.config.hidden_size] + self.head_hidden_dims\n        self.head = MLP(\n            head_hidden_dims,\n            head_activation,\n            head_dropout,\n            head_batchnorm,\n            head_batchnorm_last,\n            head_linear_first,\n        )\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.text.huggingface_transformers.hf_model.HFModel.attention_weight","title":"attention_weight  <code>property</code>","text":"<pre><code>attention_weight\n</code></pre> <p>Returns the attention weights if the model was created with the output_attention_weights=True argument. If not, it will raise an AttributeError.</p> <p>The shape of the attention weights is \\((N, H, F, F)\\), where \\(N\\) is the batch size, \\(H\\) is the number of attention heads and \\(F\\) is the sequence length.</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.image.vision.Vision","title":"Vision","text":"<pre><code>Vision(\n    pretrained_model_setup=None,\n    n_trainable=None,\n    trainable_params=None,\n    channel_sizes=[64, 128, 256, 512],\n    kernel_sizes=[7, 3, 3, 3],\n    strides=[2, 1, 1, 1],\n    head_hidden_dims=None,\n    head_activation=\"relu\",\n    head_dropout=0.1,\n    head_batchnorm=False,\n    head_batchnorm_last=False,\n    head_linear_first=False,\n)\n</code></pre> <p>               Bases: <code>BaseWDModelComponent</code></p> <p>Defines a standard image classifier/regressor using a pretrained network or a sequence of convolution layers that can be used as the <code>deepimage</code> component of a Wide &amp; Deep model or independently by itself.</p> <p> NOTE: this class represents the integration  between <code>pytorch-widedeep</code> and <code>torchvision</code>. New architectures will be  available as they are added to <code>torchvision</code>. In a distant future we aim  to bring transformer-based architectures as well. However, simple  CNN-based architectures (and even MLP-based) seem to produce SoTA  results. For the time being, we describe below the options available  through this class</p> <p>Parameters:</p> <ul> <li> <code>pretrained_model_setup</code>               (<code>Union[str, Dict[str, Union[str, WeightsEnum]]]</code>, default:                   <code>None</code> )           \u2013            <p>Name of the pretrained model. Should be a variant of the following architectures: 'resnet', 'shufflenet', 'resnext', 'wide_resnet', 'regnet', 'densenet', 'mobilenetv3', 'mobilenetv2', 'mnasnet', 'efficientnet' and 'squeezenet'. if <code>pretrained_model_setup = None</code> a basic, fully trainable CNN will be used. Alternatively, since Torchvision 0.13 one can use pretrained models with different weigths. Therefore, <code>pretrained_model_setup</code> can also be dictionary with the name of the model and the weights (e.g. <code>{'resnet50': ResNet50_Weights.DEFAULT}</code> or <code>{'resnet50': \"IMAGENET1K_V2\"}</code>).  Aliased as <code>pretrained_model_name</code>.</p> </li> <li> <code>n_trainable</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>Number of trainable layers starting from the layer closer to the output neuron(s). Note that this number DOES NOT take into account the so-called 'head' which is ALWAYS trainable. If <code>trainable_params</code> is not None this parameter will be ignored</p> </li> <li> <code>trainable_params</code>               (<code>Optional[List[str]]</code>, default:                   <code>None</code> )           \u2013            <p>List of strings containing the names (or substring within the name) of the parameters that will be trained. For example, if we use a 'resnet18' pretrained model and we set <code>trainable_params = ['layer4']</code> only the parameters of 'layer4' of the network (and the head, as mentioned before) will be trained. Note that setting this or the previous parameter involves some knowledge of the architecture used.</p> </li> <li> <code>channel_sizes</code>               (<code>List[int]</code>, default:                   <code>[64, 128, 256, 512]</code> )           \u2013            <p>List of integers with the channel sizes of a CNN in case we choose not to use a pretrained model</p> </li> <li> <code>kernel_sizes</code>               (<code>Union[int, List[int]]</code>, default:                   <code>[7, 3, 3, 3]</code> )           \u2013            <p>List of integers with the kernel sizes of a CNN in case we choose not to use a pretrained model. Must be of length equal to <code>len(channel_sizes) - 1</code>.</p> </li> <li> <code>strides</code>               (<code>Union[int, List[int]]</code>, default:                   <code>[2, 1, 1, 1]</code> )           \u2013            <p>List of integers with the stride sizes of a CNN in case we choose not to use a pretrained model. Must be of length equal to <code>len(channel_sizes) - 1</code>.</p> </li> <li> <code>head_hidden_dims</code>               (<code>Optional[List[int]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the number of neurons per dense layer in the head. e.g: [64,32]</p> </li> <li> <code>head_activation</code>               (<code>str</code>, default:                   <code>'relu'</code> )           \u2013            <p>Activation function for the dense layers in the head. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> </li> <li> <code>head_dropout</code>               (<code>Union[float, List[float]]</code>, default:                   <code>0.1</code> )           \u2013            <p>float indicating the dropout between the dense layers.</p> </li> <li> <code>head_batchnorm</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether or not batch normalization will be applied to the dense layers</p> </li> <li> <code>head_batchnorm_last</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether or not batch normalization will be applied to the last of the dense layers</p> </li> <li> <code>head_linear_first</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code></p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>features</code>               (<code>Module</code>)           \u2013            <p>The pretrained model or Standard CNN plus the optional head</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import Vision\n&gt;&gt;&gt; X_img = torch.rand((2,3,224,224))\n&gt;&gt;&gt; model = Vision(channel_sizes=[64, 128], kernel_sizes = [3, 3], strides=[1, 1], head_hidden_dims=[32, 8])\n&gt;&gt;&gt; out = model(X_img)\n</code></pre> Source code in <code>pytorch_widedeep/models/image/vision.py</code> <pre><code>@alias(\"pretrained_model_setup\", [\"pretrained_model_name\"])\ndef __init__(\n    self,\n    pretrained_model_setup: Union[str, Dict[str, Union[str, WeightsEnum]]] = None,\n    n_trainable: Optional[int] = None,\n    trainable_params: Optional[List[str]] = None,\n    channel_sizes: List[int] = [64, 128, 256, 512],\n    kernel_sizes: Union[int, List[int]] = [7, 3, 3, 3],\n    strides: Union[int, List[int]] = [2, 1, 1, 1],\n    head_hidden_dims: Optional[List[int]] = None,\n    head_activation: str = \"relu\",\n    head_dropout: Union[float, List[float]] = 0.1,\n    head_batchnorm: bool = False,\n    head_batchnorm_last: bool = False,\n    head_linear_first: bool = False,\n):\n    super(Vision, self).__init__()\n\n    self._check_pretrained_model_setup(\n        pretrained_model_setup, n_trainable, trainable_params\n    )\n\n    self.pretrained_model_setup = pretrained_model_setup\n    self.n_trainable = n_trainable\n    self.trainable_params = trainable_params\n    self.channel_sizes = channel_sizes\n    self.kernel_sizes = kernel_sizes\n    self.strides = strides\n    self.head_hidden_dims = head_hidden_dims\n    self.head_activation = head_activation\n    self.head_dropout = head_dropout\n    self.head_batchnorm = head_batchnorm\n    self.head_batchnorm_last = head_batchnorm_last\n    self.head_linear_first = head_linear_first\n\n    self.features, self.backbone_output_dim = self._get_features()\n\n    if pretrained_model_setup is not None:\n        self._freeze(self.features)\n\n    if self.head_hidden_dims is not None:\n        head_hidden_dims = [self.backbone_output_dim] + self.head_hidden_dims\n        self.vision_mlp = MLP(\n            head_hidden_dims,\n            self.head_activation,\n            self.head_dropout,\n            self.head_batchnorm,\n            self.head_batchnorm_last,\n            self.head_linear_first,\n        )\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.image.vision.Vision.output_dim","title":"output_dim  <code>property</code>","text":"<pre><code>output_dim\n</code></pre> <p>The output dimension of the model. This is a required property neccesary to build the <code>WideDeep</code> class</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.model_fusion.ModelFuser","title":"ModelFuser","text":"<pre><code>ModelFuser(\n    models,\n    *,\n    fusion_method,\n    projection_method=None,\n    custom_head=None,\n    head_hidden_dims=None,\n    head_activation=None,\n    head_dropout=None,\n    head_batchnorm=None,\n    head_batchnorm_last=None,\n    head_linear_first=None\n)\n</code></pre> <p>               Bases: <code>BaseWDModelComponent</code></p> <p>This class is a wrapper around a list of models that are associated to the different text and/or image columns (and datasets) The class is designed to 'fuse' the models using a variety of methods.</p> <p>Parameters:</p> <ul> <li> <code>models</code>               (<code>List[BaseWDModelComponent]</code>)           \u2013            <p>List of models whose outputs will be fused</p> </li> <li> <code>fusion_method</code>               (<code>Union[Literal[concatenate, mean, max, sum, mult, dot, head], List[Literal[concatenate, mean, max, sum, mult, head]]]</code>)           \u2013            <p>Method to fuse the output of the models. It can be one of ['concatenate', 'mean', 'max', 'sum', 'mult', 'dot', 'head'] or a list of those, but 'dot'. If a list is provided the output of the models will be fused using all the methods in the list and the final output will be the concatenation of the outputs of each method</p> </li> <li> <code>projection_method</code>               (<code>Optional[Literal[min, max, mean]]</code>, default:                   <code>None</code> )           \u2013            <p>If the fusion_method is not 'concatenate', this parameter will determine how to project the output of the models to a common dimension. It can be one of ['min', 'max', 'mean']. Default is None</p> </li> <li> <code>custom_head</code>               (<code>Optional[Union[BaseWDModelComponent, Module]]</code>, default:                   <code>None</code> )           \u2013            <p>Custom head to be used to fuse the output of the models. If provided, this will take precedence over head_hidden_dims. Also, if provided, 'projection_method' will be ignored.</p> </li> <li> <code>head_hidden_dims</code>               (<code>Optional[List[int]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the number of neurons per layer in the custom head. If custom_head is provided, this parameter will be ignored</p> </li> <li> <code>head_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function to be used in the custom head. Default is None</p> </li> <li> <code>head_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Dropout to be used in the custom head. Default is None</p> </li> <li> <code>head_batchnorm</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Whether to use batchnorm in the custom head. Default is None</p> </li> <li> <code>head_batchnorm_last</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Whether or not batch normalization will be applied to the last of the dense layers</p> </li> <li> <code>head_linear_first</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code></p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>head</code>               (<code>Module or BaseWDModelComponent</code>)           \u2013            <p>Custom head to be used to fuse the output of the models. If custom_head is provided, this will take precedence over head_hidden_dims</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; from pytorch_widedeep.preprocessing import TextPreprocessor\n&gt;&gt;&gt; from pytorch_widedeep.models import BasicRNN, ModelFuser\n&gt;&gt;&gt; import torch\n&gt;&gt;&gt; import pandas as pd\n&gt;&gt;&gt;\n&gt;&gt;&gt; df = pd.DataFrame({'text_col1': ['hello world', 'this is a test'],\n... 'text_col2': ['goodbye world', 'this is another test']})\n&gt;&gt;&gt; text_preprocessor_1 = TextPreprocessor(\n...     text_col=\"text_col1\",\n...     max_vocab=10,\n...     min_freq=1,\n...     maxlen=5,\n...     n_cpus=1,\n...     verbose=0)\n&gt;&gt;&gt; text_preprocessor_2 = TextPreprocessor(\n...     text_col=\"text_col2\",\n...     max_vocab=10,\n...     min_freq=1,\n...     maxlen=5,\n...     n_cpus=1,\n...     verbose=0)\n&gt;&gt;&gt; X_text1 = text_preprocessor_1.fit_transform(df)\n&gt;&gt;&gt; X_text2 = text_preprocessor_2.fit_transform(df)\n&gt;&gt;&gt; X_text1_tnsr = torch.from_numpy(X_text1)\n&gt;&gt;&gt; X_text2_tnsr = torch.from_numpy(X_text2)\n&gt;&gt;&gt; rnn1 = BasicRNN(\n...     vocab_size=len(text_preprocessor_1.vocab.itos),\n...     embed_dim=4,\n...     hidden_dim=4,\n...     n_layers=1,\n...     bidirectional=False)\n&gt;&gt;&gt; rnn2 = BasicRNN(\n...     vocab_size=len(text_preprocessor_2.vocab.itos),\n...     embed_dim=4,\n...     hidden_dim=4,\n...     n_layers=1,\n...     bidirectional=False)\n&gt;&gt;&gt; fused_model = ModelFuser(models=[rnn1, rnn2], fusion_method='concatenate')\n&gt;&gt;&gt; out = fused_model([X_text1_tnsr, X_text2_tnsr])\n</code></pre> Source code in <code>pytorch_widedeep/models/model_fusion.py</code> <pre><code>def __init__(\n    self,\n    models: List[BaseWDModelComponent],\n    *,\n    fusion_method: Union[\n        Literal[\n            \"concatenate\",\n            \"mean\",\n            \"max\",\n            \"sum\",\n            \"mult\",\n            \"dot\",\n            \"head\",\n        ],\n        List[Literal[\"concatenate\", \"mean\", \"max\", \"sum\", \"mult\", \"head\"]],\n    ],\n    projection_method: Optional[Literal[\"min\", \"max\", \"mean\"]] = None,\n    custom_head: Optional[Union[BaseWDModelComponent, nn.Module]] = None,\n    head_hidden_dims: Optional[List[int]] = None,\n    head_activation: Optional[str] = None,\n    head_dropout: Optional[float] = None,\n    head_batchnorm: Optional[bool] = None,\n    head_batchnorm_last: Optional[bool] = None,\n    head_linear_first: Optional[bool] = None,\n) -&gt; None:\n    super(ModelFuser, self).__init__()\n\n    self.models = nn.ModuleList(models)\n    self.fusion_method = fusion_method\n    self.projection_method = projection_method\n\n    self.all_output_dim_equal = all(\n        model.output_dim == self.models[0].output_dim for model in self.models\n    )\n\n    self.check_input_parameters()\n\n    if self.fusion_method == \"head\":\n        assert (\n            head_hidden_dims is not None or custom_head is not None\n        ), \"When using 'head' as fusion_method, either head_hidden_dims or custom_head must be provided\"\n        if custom_head is not None:\n            # custom_head takes precedence over head_hidden_dims (in case\n            # both are provided)\n            assert hasattr(\n                custom_head, \"output_dim\"\n            ), \"custom_head must have an 'output_dim' property\"\n            self.head: Union[BaseWDModelComponent, nn.Module] = custom_head\n        else:\n            assert head_hidden_dims is not None\n            self.head_hidden_dims = head_hidden_dims\n            self.head_activation = head_activation\n            self.head_dropout = head_dropout\n            self.head_batchnorm = head_batchnorm\n            self.head_batchnorm_last = head_batchnorm_last\n            self.head_linear_first = head_linear_first\n\n            self.head = MLP(\n                d_hidden=[sum([model.output_dim for model in self.models])]\n                + self.head_hidden_dims,\n                activation=(\n                    \"relu\" if self.head_activation is None else self.head_activation\n                ),\n                dropout=0.0 if self.head_dropout is None else self.head_dropout,\n                batchnorm=(\n                    False if self.head_batchnorm is None else self.head_batchnorm\n                ),\n                batchnorm_last=(\n                    False\n                    if self.head_batchnorm_last is None\n                    else self.head_batchnorm_last\n                ),\n                linear_first=(\n                    True\n                    if self.head_linear_first is None\n                    else self.head_linear_first\n                ),\n            )\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.model_fusion.ModelFuser.output_dim","title":"output_dim  <code>property</code>","text":"<pre><code>output_dim\n</code></pre> <p>Returns the output dimension of the model.</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.model_fusion.ModelFuser.project","title":"project","text":"<pre><code>project(X)\n</code></pre> <p>Projects the output of the models to a common dimension.</p> Source code in <code>pytorch_widedeep/models/model_fusion.py</code> <pre><code>def project(self, X: List[Tensor]) -&gt; List[Tensor]:\n    r\"\"\"Projects the output of the models to a common dimension.\"\"\"\n\n    if self.all_output_dim_equal and self.projection_method is None:\n        return X\n\n    output_dims = [model.output_dim for model in self.models]\n\n    if self.projection_method == \"min\":\n        proj_dim = min(output_dims)\n        idx = output_dims.index(proj_dim)\n    elif self.projection_method == \"max\":\n        proj_dim = max(output_dims)\n        idx = output_dims.index(proj_dim)\n    elif self.projection_method == \"mean\":\n        proj_dim = int(sum(output_dims) / len(output_dims))\n        idx = None\n    else:\n        raise ValueError(\"projection_method must be one of ['min', 'max', 'mean']\")\n\n    x_proj: List[Tensor] = []\n    for i, x in enumerate(X):\n        if i == idx:\n            x_proj.append(x)\n        else:\n            x_proj.append(\n                nn.Linear(output_dims[i], proj_dim, bias=False, device=x.device)(x)\n            )\n\n    return x_proj\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.wide_deep.WideDeep","title":"WideDeep","text":"<pre><code>WideDeep(\n    wide=None,\n    deeptabular=None,\n    deeptext=None,\n    deepimage=None,\n    deephead=None,\n    head_hidden_dims=None,\n    head_activation=\"relu\",\n    head_dropout=0.1,\n    head_batchnorm=False,\n    head_batchnorm_last=False,\n    head_linear_first=True,\n    enforce_positive=False,\n    enforce_positive_activation=\"softplus\",\n    pred_dim=1,\n)\n</code></pre> <p>               Bases: <code>Module</code></p> <p>Main collector class that combines all <code>wide</code>, <code>deeptabular</code> <code>deeptext</code> and <code>deepimage</code> models.</p> <p>Note that all models described so far in this library must be passed to the <code>WideDeep</code> class once constructed. This is because the models output the last layer before the prediction layer. Such prediction layer is added by the <code>WideDeep</code> class as it collects the components for every data mode.</p> <p>There are two options to combine these models that correspond to the two main architectures that <code>pytorch-widedeep</code> can build.</p> <ul> <li> <p>Directly connecting the output of the model components to an ouput neuron(s).</p> </li> <li> <p>Adding a <code>Fully-Connected Head</code> (FC-Head) on top of the deep models.   This FC-Head will combine the output form the <code>deeptabular</code>, <code>deeptext</code> and   <code>deepimage</code> and will be then connected to the output neuron(s).</p> </li> </ul> <p>Parameters:</p> <ul> <li> <code>wide</code>               (<code>Optional[Module]</code>, default:                   <code>None</code> )           \u2013            <p><code>Wide</code> model. This is a linear model where the non-linearities are captured via crossed-columns.</p> </li> <li> <code>deeptabular</code>               (<code>Optional[Union[BaseWDModelComponent, List[BaseWDModelComponent]]]</code>, default:                   <code>None</code> )           \u2013            <p>Currently this library implements a number of possible architectures for the <code>deeptabular</code> component. See the documenation of the package. Note that <code>deeptabular</code> can be a list of models. This is useful when using multiple tabular inputs (e.g. for example in the context of a two-tower model for recommendation systems)</p> </li> <li> <code>deeptext</code>               (<code>Optional[Union[BaseWDModelComponent, List[BaseWDModelComponent]]]</code>, default:                   <code>None</code> )           \u2013            <p>Currently this library implements a number of possible architectures for the <code>deeptext</code> component. See the documenation of the package. Note that <code>deeptext</code> can be a list of models. This is useful when using multiple text inputs.</p> </li> <li> <code>deepimage</code>               (<code>Optional[Union[BaseWDModelComponent, List[BaseWDModelComponent]]]</code>, default:                   <code>None</code> )           \u2013            <p>Currently this library uses <code>torchvision</code> and implements a number of possible architectures for the <code>deepimage</code> component. See the documenation of the package. Note that <code>deepimage</code> can be a list of models. This is useful when using multiple image inputs.</p> </li> <li> <code>deephead</code>               (<code>Optional[BaseWDModelComponent]</code>, default:                   <code>None</code> )           \u2013            <p>Alternatively, the user can pass a custom model that will receive the output of the deep component. If <code>deephead</code> is not None all the previous fc-head parameters will be ignored</p> </li> <li> <code>head_hidden_dims</code>               (<code>Optional[List[int]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the sizes of the dense layers in the head e.g: [128, 64]</p> </li> <li> <code>head_activation</code>               (<code>str</code>, default:                   <code>'relu'</code> )           \u2013            <p>Activation function for the dense layers in the head. Currently <code>'tanh'</code>, <code>'relu'</code>, <code>'leaky_relu'</code> and <code>'gelu'</code> are supported</p> </li> <li> <code>head_dropout</code>               (<code>float</code>, default:                   <code>0.1</code> )           \u2013            <p>Dropout of the dense layers in the head</p> </li> <li> <code>head_batchnorm</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether or not to include batch normalization in the dense layers that form the <code>'rnn_mlp'</code></p> </li> <li> <code>head_batchnorm_last</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether or not to apply batch normalization to the last of the dense layers in the head</p> </li> <li> <code>head_linear_first</code>               (<code>bool</code>, default:                   <code>True</code> )           \u2013            <p>Boolean indicating whether the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code></p> </li> <li> <code>enforce_positive</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating if the output from the final layer must be positive. This is important if you are using loss functions with non-negative input restrictions, e.g. RMSLE, or if you know your predictions are bounded in between 0 and inf</p> </li> <li> <code>enforce_positive_activation</code>               (<code>str</code>, default:                   <code>'softplus'</code> )           \u2013            <p>Activation function to enforce that the final layer has a positive output. <code>'softplus'</code> or <code>'relu'</code> are supported.</p> </li> <li> <code>pred_dim</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>Size of the final wide and deep output layer containing the predictions. <code>1</code> for regression and binary classification or number of classes for multiclass classification.</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; from pytorch_widedeep.models import TabResnet, Vision, BasicRNN, Wide, WideDeep\n&gt;&gt;&gt; embed_input = [(u, i, j) for u, i, j in zip([\"a\", \"b\", \"c\"][:4], [4] * 3, [8] * 3)]\n&gt;&gt;&gt; column_idx = {k: v for v, k in enumerate([\"a\", \"b\", \"c\"])}\n&gt;&gt;&gt; wide = Wide(10, 1)\n&gt;&gt;&gt; deeptabular = TabResnet(blocks_dims=[8, 4], column_idx=column_idx, cat_embed_input=embed_input)\n&gt;&gt;&gt; deeptext = BasicRNN(vocab_size=10, embed_dim=4, padding_idx=0)\n&gt;&gt;&gt; deepimage = Vision()\n&gt;&gt;&gt; model = WideDeep(wide=wide, deeptabular=deeptabular, deeptext=deeptext, deepimage=deepimage)\n</code></pre> <p> NOTE: It is possible to use custom components to  build Wide &amp; Deep models. Simply, build them and pass them as the  corresponding parameters. Note that the custom models MUST return a last  layer of activations(i.e. not the final prediction) so that  these  activations are collected by <code>WideDeep</code> and combined accordingly. In  addition, the models MUST also contain an attribute <code>output_dim</code> with  the size of these last layers of activations. See for example  <code>pytorch_widedeep.models.tab_mlp.TabMlp</code></p> Source code in <code>pytorch_widedeep/models/wide_deep.py</code> <pre><code>@alias(  # noqa: C901\n    \"pred_dim\",\n    [\"num_class\", \"pred_size\"],\n)\ndef __init__(\n    self,\n    wide: Optional[nn.Module] = None,\n    deeptabular: Optional[\n        Union[BaseWDModelComponent, List[BaseWDModelComponent]]\n    ] = None,\n    deeptext: Optional[\n        Union[BaseWDModelComponent, List[BaseWDModelComponent]]\n    ] = None,\n    deepimage: Optional[\n        Union[BaseWDModelComponent, List[BaseWDModelComponent]]\n    ] = None,\n    deephead: Optional[BaseWDModelComponent] = None,\n    head_hidden_dims: Optional[List[int]] = None,\n    head_activation: str = \"relu\",\n    head_dropout: float = 0.1,\n    head_batchnorm: bool = False,\n    head_batchnorm_last: bool = False,\n    head_linear_first: bool = True,\n    enforce_positive: bool = False,\n    enforce_positive_activation: str = \"softplus\",\n    pred_dim: int = 1,\n):\n    super(WideDeep, self).__init__()\n\n    self._check_inputs(\n        wide,\n        deeptabular,\n        deeptext,\n        deepimage,\n        deephead,\n        head_hidden_dims,\n        pred_dim,\n    )\n\n    # this attribute will be eventually over-written by the Trainer's\n    # device. Acts here as a 'placeholder'.\n    self.wd_device: Optional[str] = None\n\n    # required as attribute just in case we pass a deephead\n    self.pred_dim = pred_dim\n\n    self.enforce_positive = enforce_positive\n\n    # better to set this attribute already here\n    if isinstance(deeptabular, list):\n        self.is_tabnet = False\n    else:\n        self.is_tabnet = deeptabular.__class__.__name__ == \"TabNet\"\n\n    # The main 5 components of the wide and deep assemble: wide,\n    # deeptabular, deeptext, deepimage and deephead\n    self.with_deephead = deephead is not None or head_hidden_dims is not None\n    if deephead is None and head_hidden_dims is not None:\n        self.deephead = self._build_deephead(\n            deeptabular,\n            deeptext,\n            deepimage,\n            head_hidden_dims,\n            head_activation,\n            head_dropout,\n            head_batchnorm,\n            head_batchnorm_last,\n            head_linear_first,\n        )\n    elif deephead is not None:\n        self.deephead = nn.Sequential(\n            deephead, nn.Linear(deephead.output_dim, self.pred_dim)\n        )\n    else:\n        # for consistency with other components we default to None\n        self.deephead = None\n\n    self.wide = wide\n    self.deeptabular, self.deeptext, self.deepimage = self._set_model_components(\n        deeptabular, deeptext, deepimage\n    )\n\n    if self.enforce_positive:\n        self.enf_pos = get_activation_fn(enforce_positive_activation)\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html","title":"The <code>preprocessing</code> module","text":"<p>This module contains the classes that are used to prepare the data before being passed to the models. There is one Preprocessor per data mode or model component (<code>wide</code>, <code>deeptabular</code>, <code>deepimage</code> and <code>deeptext</code>) with the exception of the <code>deeptext</code> component. In this case, two processors are available: one for the case when no Hugging Face model is used (<code>TextPreprocessor</code>) and another one when a Hugging Face model is used (<code>HFPreprocessor</code>).</p>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.wide_preprocessor.WidePreprocessor","title":"WidePreprocessor","text":"<pre><code>WidePreprocessor(wide_cols, crossed_cols=None)\n</code></pre> <p>               Bases: <code>BasePreprocessor</code></p> <p>Preprocessor to prepare the wide input dataset</p> <p>This Preprocessor prepares the data for the wide, linear component. This linear model is implemented via an Embedding layer that is connected to the output neuron. <code>WidePreprocessor</code> numerically encodes all the unique values of all categorical columns <code>wide_cols + crossed_cols</code>. See the Example below.</p> <p>Parameters:</p> <ul> <li> <code>wide_cols</code>               (<code>List[str]</code>)           \u2013            <p>List of strings with the name of the columns that will label encoded and passed through the <code>wide</code> component</p> </li> <li> <code>crossed_cols</code>               (<code>Optional[List[Tuple[str, str]]]</code>, default:                   <code>None</code> )           \u2013            <p>List of Tuples with the name of the columns that will be <code>'crossed'</code> and then label encoded. e.g. [('education', 'occupation'), ...]. For binary features, a cross-product transformation is 1 if and only if the constituent features are all 1, and 0 otherwise.</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>wide_crossed_cols</code>               (<code>List</code>)           \u2013            <p>List with the names of all columns that will be label encoded</p> </li> <li> <code>encoding_dict</code>               (<code>Dict</code>)           \u2013            <p>Dictionary where the keys are the result of pasting <code>colname + '_' + column value</code> and the values are the corresponding mapped integer.</p> </li> <li> <code>inverse_encoding_dict</code>               (<code>Dict</code>)           \u2013            <p>the inverse encoding dictionary</p> </li> <li> <code>wide_dim</code>               (<code>int</code>)           \u2013            <p>Dimension of the wide model (i.e. dim of the linear layer)</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import pandas as pd\n&gt;&gt;&gt; from pytorch_widedeep.preprocessing import WidePreprocessor\n&gt;&gt;&gt; df = pd.DataFrame({'color': ['r', 'b', 'g'], 'size': ['s', 'n', 'l']})\n&gt;&gt;&gt; wide_cols = ['color']\n&gt;&gt;&gt; crossed_cols = [('color', 'size')]\n&gt;&gt;&gt; wide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols)\n&gt;&gt;&gt; X_wide = wide_preprocessor.fit_transform(df)\n&gt;&gt;&gt; X_wide\narray([[1, 4],\n       [2, 5],\n       [3, 6]])\n&gt;&gt;&gt; wide_preprocessor.encoding_dict\n{'color_r': 1, 'color_b': 2, 'color_g': 3, 'color_size_r-s': 4, 'color_size_b-n': 5, 'color_size_g-l': 6}\n&gt;&gt;&gt; wide_preprocessor.inverse_transform(X_wide)\n  color color_size\n0     r        r-s\n1     b        b-n\n2     g        g-l\n</code></pre> Source code in <code>pytorch_widedeep/preprocessing/wide_preprocessor.py</code> <pre><code>def __init__(\n    self, wide_cols: List[str], crossed_cols: Optional[List[Tuple[str, str]]] = None\n):\n    super(WidePreprocessor, self).__init__()\n\n    self.wide_cols = wide_cols\n    self.crossed_cols = crossed_cols\n\n    self.is_fitted = False\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.wide_preprocessor.WidePreprocessor.fit","title":"fit","text":"<pre><code>fit(df)\n</code></pre> <p>Fits the Preprocessor and creates required attributes</p> <p>Parameters:</p> <ul> <li> <code>df</code>               (<code>DataFrame</code>)           \u2013            <p>Input pandas dataframe</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>WidePreprocessor</code>           \u2013            <p><code>WidePreprocessor</code> fitted object</p> </li> </ul> Source code in <code>pytorch_widedeep/preprocessing/wide_preprocessor.py</code> <pre><code>def fit(self, df: pd.DataFrame) -&gt; \"WidePreprocessor\":\n    r\"\"\"Fits the Preprocessor and creates required attributes\n\n    Parameters\n    ----------\n    df: pd.DataFrame\n        Input pandas dataframe\n\n    Returns\n    -------\n    WidePreprocessor\n        `WidePreprocessor` fitted object\n    \"\"\"\n    df_wide = self._prepare_wide(df)\n    self.wide_crossed_cols = df_wide.columns.tolist()\n    glob_feature_list = self._make_global_feature_list(\n        df_wide[self.wide_crossed_cols]\n    )\n    # leave 0 for padding/\"unseen\" categories\n    self.encoding_dict = {v: i + 1 for i, v in enumerate(glob_feature_list)}\n    self.wide_dim = len(self.encoding_dict)\n    self.inverse_encoding_dict = {k: v for v, k in self.encoding_dict.items()}\n    self.inverse_encoding_dict[0] = \"unseen\"\n\n    self.is_fitted = True\n\n    return self\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.wide_preprocessor.WidePreprocessor.transform","title":"transform","text":"<pre><code>transform(df)\n</code></pre> <p>Parameters:</p> <ul> <li> <code>df</code>               (<code>DataFrame</code>)           \u2013            <p>Input pandas dataframe</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>ndarray</code>           \u2013            <p>transformed input dataframe</p> </li> </ul> Source code in <code>pytorch_widedeep/preprocessing/wide_preprocessor.py</code> <pre><code>def transform(self, df: pd.DataFrame) -&gt; np.ndarray:\n    r\"\"\"\n    Parameters\n    ----------\n    df: pd.DataFrame\n        Input pandas dataframe\n\n    Returns\n    -------\n    np.ndarray\n        transformed input dataframe\n    \"\"\"\n    check_is_fitted(self, attributes=[\"encoding_dict\"])\n    df_wide = self._prepare_wide(df)\n    encoded = np.zeros([len(df_wide), len(self.wide_crossed_cols)])\n    for col_i, col in enumerate(self.wide_crossed_cols):\n        encoded[:, col_i] = df_wide[col].apply(\n            lambda x: (\n                self.encoding_dict[col + \"_\" + str(x)]\n                if col + \"_\" + str(x) in self.encoding_dict\n                else 0\n            )\n        )\n    return encoded.astype(\"int64\")\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.wide_preprocessor.WidePreprocessor.inverse_transform","title":"inverse_transform","text":"<pre><code>inverse_transform(encoded)\n</code></pre> <p>Takes as input the output from the <code>transform</code> method and it will return the original values.</p> <p>Parameters:</p> <ul> <li> <code>encoded</code>               (<code>ndarray</code>)           \u2013            <p>numpy array with the encoded values that are the output from the <code>transform</code> method</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>DataFrame</code>           \u2013            <p>Pandas dataframe with the original values</p> </li> </ul> Source code in <code>pytorch_widedeep/preprocessing/wide_preprocessor.py</code> <pre><code>def inverse_transform(self, encoded: np.ndarray) -&gt; pd.DataFrame:\n    r\"\"\"Takes as input the output from the `transform` method and it will\n    return the original values.\n\n    Parameters\n    ----------\n    encoded: np.ndarray\n        numpy array with the encoded values that are the output from the\n        `transform` method\n\n    Returns\n    -------\n    pd.DataFrame\n        Pandas dataframe with the original values\n    \"\"\"\n    decoded = pd.DataFrame(encoded, columns=self.wide_crossed_cols)\n\n    if pd.__version__ &gt;= \"2.1.0\":\n        decoded = decoded.map(lambda x: self.inverse_encoding_dict[x])\n    else:\n        decoded = decoded.applymap(lambda x: self.inverse_encoding_dict[x])\n\n    for col in decoded.columns:\n        rm_str = \"\".join([col, \"_\"])\n        decoded[col] = decoded[col].apply(lambda x: x.replace(rm_str, \"\"))\n    return decoded\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.wide_preprocessor.WidePreprocessor.fit_transform","title":"fit_transform","text":"<pre><code>fit_transform(df)\n</code></pre> <p>Combines <code>fit</code> and <code>transform</code></p> <p>Parameters:</p> <ul> <li> <code>df</code>               (<code>DataFrame</code>)           \u2013            <p>Input pandas dataframe</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>ndarray</code>           \u2013            <p>transformed input dataframe</p> </li> </ul> Source code in <code>pytorch_widedeep/preprocessing/wide_preprocessor.py</code> <pre><code>def fit_transform(self, df: pd.DataFrame) -&gt; np.ndarray:\n    \"\"\"Combines `fit` and `transform`\n\n    Parameters\n    ----------\n    df: pd.DataFrame\n        Input pandas dataframe\n\n    Returns\n    -------\n    np.ndarray\n        transformed input dataframe\n    \"\"\"\n    return self.fit(df).transform(df)\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor","title":"TabPreprocessor","text":"<pre><code>TabPreprocessor(\n    cat_embed_cols=None,\n    continuous_cols=None,\n    quantization_setup=None,\n    cols_to_scale=None,\n    auto_embed_dim=True,\n    embedding_rule=\"fastai_new\",\n    default_embed_dim=16,\n    with_attention=False,\n    with_cls_token=False,\n    shared_embed=False,\n    verbose=1,\n    *,\n    scale=False,\n    already_standard=None,\n    **kwargs\n)\n</code></pre> <p>               Bases: <code>BasePreprocessor</code></p> <p>Preprocessor to prepare the <code>deeptabular</code> component input dataset</p> <p>Parameters:</p> <ul> <li> <code>cat_embed_cols</code>               (<code>Optional[Union[List[str], List[Tuple[str, int]]]]</code>, default:                   <code>None</code> )           \u2013            <p>List containing the name of the categorical columns that will be represented by embeddings (e.g. ['education', 'relationship', ...]) or a Tuple with the name and the embedding dimension (e.g.: [ ('education',32), ('relationship',16), ...])</p> </li> <li> <code>continuous_cols</code>               (<code>Optional[List[str]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the name of the continuous cols</p> </li> <li> <code>quantization_setup</code>               (<code>Optional[Union[int, Dict[str, Union[int, List[float]]]]]</code>, default:                   <code>None</code> )           \u2013            <p>Continuous columns can be turned into categorical via <code>pd.cut</code>. If <code>quantization_setup</code> is an <code>int</code>, all continuous columns will be quantized using this value as the number of bins. Alternatively, a dictionary where the keys are the column names to quantize and the values are the either integers indicating the number of bins or a list of scalars indicating the bin edges can also be used.</p> </li> <li> <code>cols_to_scale</code>               (<code>Optional[Union[List[str], str]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the names of the columns that will be standarised via sklearn's <code>StandardScaler</code>. It can also be the string <code>'all'</code> in which case all the continuous cols will be scaled.</p> </li> <li> <code>auto_embed_dim</code>               (<code>bool</code>, default:                   <code>True</code> )           \u2013            <p>Boolean indicating whether the embedding dimensions will be automatically defined via rule of thumb. See <code>embedding_rule</code> below.</p> </li> <li> <code>embedding_rule</code>               (<code>Literal[google, fastai_old, fastai_new]</code>, default:                   <code>'fastai_new'</code> )           \u2013            <p>If <code>auto_embed_dim=True</code>, this is the choice of embedding rule of thumb. Choices are:</p> <ul> <li> <p>fastai_new: \\(min(600, round(1.6 \\times n_{cat}^{0.56}))\\)</p> </li> <li> <p>fastai_old: \\(min(50, (n_{cat}//{2})+1)\\)</p> </li> <li> <p>google: \\(min(600, round(n_{cat}^{0.24}))\\)</p> </li> </ul> </li> <li> <code>default_embed_dim</code>               (<code>int</code>, default:                   <code>16</code> )           \u2013            <p>Dimension for the embeddings if the embedding dimension is not provided in the <code>cat_embed_cols</code> parameter and <code>auto_embed_dim</code> is set to <code>False</code>.</p> </li> <li> <code>with_attention</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether the preprocessed data will be passed to an attention-based model (more precisely a model where all embeddings must have the same dimensions). If <code>True</code>, the param <code>cat_embed_cols</code> must just be a list containing just the categorical column names: e.g. ['education', 'relationship', ...]. This is because they will all be  encoded using embeddings of the same dim, which will be specified  later when the model is defined.  Param alias:  <code>for_transformer</code></p> </li> <li> <code>with_cls_token</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating if a <code>'[CLS]'</code> token will be added to the dataset when using attention-based models. The final hidden state corresponding to this token is used as the aggregated representation for classification and regression tasks. If not, the categorical and/or continuous embeddings will be concatenated before being passed to the final MLP (if present).</p> </li> <li> <code>shared_embed</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating if the embeddings will be \"shared\" when using attention-based models. The idea behind <code>shared_embed</code> is described in the Appendix A in the TabTransformer paper: 'The goal of having column embedding is to enable the model to distinguish the classes in one column from those in the other columns'. In other words, the idea is to let the model learn which column is embedded at the time. See: <code>pytorch_widedeep.models.transformers._layers.SharedEmbeddings</code>.</p> </li> <li> <code>verbose</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            </li> <li> <code>scale</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p> note: this arg will be removed in upcoming  releases. Please use <code>cols_to_scale</code> instead.  Bool indicating  whether or not to scale/standarise continuous cols. It is important  to emphasize that all the DL models for tabular data in the library  also include the possibility of normalising the input continuous  features via a <code>BatchNorm</code> or a <code>LayerNorm</code>.  Param alias:  <code>scale_cont_cols</code>.</p> </li> <li> <code>already_standard</code>               (<code>Optional[List[str]]</code>, default:                   <code>None</code> )           \u2013            <p> note: this arg will be removed in upcoming  releases. Please use <code>cols_to_scale</code> instead.  List with the  name of the continuous cols that do not need to be  scaled/standarised.</p> </li> </ul> <p>Other Parameters:</p> <ul> <li> <code>**kwargs</code>           \u2013            <p><code>pd.cut</code> and <code>StandardScaler</code> related args</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>embed_dim</code>               (<code>Dict</code>)           \u2013            <p>Dictionary where keys are the embed cols and values are the embedding dimensions. If <code>with_attention</code> is set to <code>True</code> this attribute is not generated during the <code>fit</code> process</p> </li> <li> <code>label_encoder</code>               (<code>LabelEncoder</code>)           \u2013            <p>see <code>pytorch_widedeep.utils.dense_utils.LabelEncder</code></p> </li> <li> <code>cat_embed_input</code>               (<code>List</code>)           \u2013            <p>List of Tuples with the column name, number of individual values for that column and, If <code>with_attention</code> is set to <code>False</code>, the corresponding embeddings dim, e.g. [('education', 16, 10), ('relationship', 6, 8), ...].</p> </li> <li> <code>standardize_cols</code>               (<code>List</code>)           \u2013            <p>List of the columns that will be standarized</p> </li> <li> <code>scaler</code>               (<code>StandardScaler</code>)           \u2013            <p>an instance of <code>sklearn.preprocessing.StandardScaler</code></p> </li> <li> <code>column_idx</code>               (<code>Dict</code>)           \u2013            <p>Dictionary where keys are column names and values are column indexes. This is neccesary to slice tensors</p> </li> <li> <code>quantizer</code>               (<code>Quantizer</code>)           \u2013            <p>an instance of <code>Quantizer</code></p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import pandas as pd\n&gt;&gt;&gt; import numpy as np\n&gt;&gt;&gt; from pytorch_widedeep.preprocessing import TabPreprocessor\n&gt;&gt;&gt; df = pd.DataFrame({'color': ['r', 'b', 'g'], 'size': ['s', 'n', 'l'], 'age': [25, 40, 55]})\n&gt;&gt;&gt; cat_embed_cols = [('color',5), ('size',5)]\n&gt;&gt;&gt; cont_cols = ['age']\n&gt;&gt;&gt; deep_preprocessor = TabPreprocessor(cat_embed_cols=cat_embed_cols, continuous_cols=cont_cols)\n&gt;&gt;&gt; X_tab = deep_preprocessor.fit_transform(df)\n&gt;&gt;&gt; deep_preprocessor.cat_embed_cols\n[('color', 5), ('size', 5)]\n&gt;&gt;&gt; deep_preprocessor.column_idx\n{'color': 0, 'size': 1, 'age': 2}\n&gt;&gt;&gt; cont_df = pd.DataFrame({\"col1\": np.random.rand(10), \"col2\": np.random.rand(10) + 1})\n&gt;&gt;&gt; cont_cols = [\"col1\", \"col2\"]\n&gt;&gt;&gt; tab_preprocessor = TabPreprocessor(continuous_cols=cont_cols, quantization_setup=3)\n&gt;&gt;&gt; ft_cont_df = tab_preprocessor.fit_transform(cont_df)\n&gt;&gt;&gt; # or...\n&gt;&gt;&gt; quantization_setup = {'col1': [0., 0.4, 1.], 'col2': [1., 1.4, 2.]}\n&gt;&gt;&gt; tab_preprocessor2 = TabPreprocessor(continuous_cols=cont_cols, quantization_setup=quantization_setup)\n&gt;&gt;&gt; ft_cont_df2 = tab_preprocessor2.fit_transform(cont_df)\n</code></pre> Source code in <code>pytorch_widedeep/preprocessing/tab_preprocessor.py</code> <pre><code>@alias(\"with_attention\", [\"for_transformer\", \"for_matrix_factorization\", \"for_mf\"])\n@alias(\"cat_embed_cols\", [\"embed_cols\"])\n@alias(\"scale\", [\"scale_cont_cols\"])\n@alias(\"quantization_setup\", [\"cols_and_bins\"])\ndef __init__(\n    self,\n    cat_embed_cols: Optional[Union[List[str], List[Tuple[str, int]]]] = None,\n    continuous_cols: Optional[List[str]] = None,\n    quantization_setup: Optional[\n        Union[int, Dict[str, Union[int, List[float]]]]\n    ] = None,\n    cols_to_scale: Optional[Union[List[str], str]] = None,\n    auto_embed_dim: bool = True,\n    embedding_rule: Literal[\"google\", \"fastai_old\", \"fastai_new\"] = \"fastai_new\",\n    default_embed_dim: int = 16,\n    with_attention: bool = False,\n    with_cls_token: bool = False,\n    shared_embed: bool = False,\n    verbose: int = 1,\n    *,\n    scale: bool = False,\n    already_standard: Optional[List[str]] = None,\n    **kwargs,\n):\n    super(TabPreprocessor, self).__init__()\n\n    self.continuous_cols = continuous_cols\n    self.quantization_setup = quantization_setup\n    self.cols_to_scale = cols_to_scale\n    self.scale = scale\n    self.already_standard = already_standard\n    self.auto_embed_dim = auto_embed_dim\n    self.embedding_rule = embedding_rule\n    self.default_embed_dim = default_embed_dim\n    self.with_attention = with_attention\n    self.with_cls_token = with_cls_token\n    self.shared_embed = shared_embed\n    self.verbose = verbose\n\n    self.quant_args = {\n        k: v for k, v in kwargs.items() if k in pd.cut.__code__.co_varnames\n    }\n    self.scale_args = {\n        k: v for k, v in kwargs.items() if k in StandardScaler().get_params()\n    }\n\n    self._check_inputs(cat_embed_cols)\n\n    if with_cls_token:\n        self.cat_embed_cols = (\n            [\"cls_token\"] + cat_embed_cols  # type: ignore[operator]\n            if cat_embed_cols is not None\n            else [\"cls_token\"]\n        )\n    else:\n        self.cat_embed_cols = cat_embed_cols  # type: ignore[assignment]\n\n    self.is_fitted = False\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor.fit","title":"fit","text":"<pre><code>fit(df)\n</code></pre> <p>Fits the Preprocessor and creates required attributes</p> <p>Parameters:</p> <ul> <li> <code>df</code>               (<code>DataFrame</code>)           \u2013            <p>Input pandas dataframe</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>TabPreprocessor</code>           \u2013            <p><code>TabPreprocessor</code> fitted object</p> </li> </ul> Source code in <code>pytorch_widedeep/preprocessing/tab_preprocessor.py</code> <pre><code>def fit(self, df: pd.DataFrame) -&gt; BasePreprocessor:  # noqa: C901\n    \"\"\"Fits the Preprocessor and creates required attributes\n\n    Parameters\n    ----------\n    df: pd.DataFrame\n        Input pandas dataframe\n\n    Returns\n    -------\n    TabPreprocessor\n        `TabPreprocessor` fitted object\n    \"\"\"\n\n    df_adj = self._insert_cls_token(df) if self.with_cls_token else df.copy()\n\n    self.column_idx: Dict[str, int] = {}\n\n    # Categorical embeddings logic\n    if self.cat_embed_cols is not None or self.quantization_setup is not None:\n        self.cat_embed_input: List[Union[Tuple[str, int], Tuple[str, int, int]]] = (\n            []\n        )\n\n    if self.cat_embed_cols is not None:\n        df_cat, cat_embed_dim = self._prepare_categorical(df_adj)\n\n        self.label_encoder = LabelEncoder(\n            columns_to_encode=df_cat.columns.tolist(),\n            shared_embed=self.shared_embed,\n            with_attention=self.with_attention,\n        )\n        self.label_encoder.fit(df_cat)\n\n        for k, v in self.label_encoder.encoding_dict.items():\n            if self.with_attention:\n                self.cat_embed_input.append((k, len(v)))\n            else:\n                self.cat_embed_input.append((k, len(v), cat_embed_dim[k]))\n\n        self.column_idx.update({k: v for v, k in enumerate(df_cat.columns)})\n\n    # Continuous columns logic\n    if self.continuous_cols is not None:\n        df_cont, cont_embed_dim = self._prepare_continuous(df_adj)\n\n        # Standardization logic\n        if self.standardize_cols is not None:\n            self.scaler = StandardScaler(**self.scale_args).fit(\n                df_cont[self.standardize_cols].values\n            )\n        elif self.verbose:\n            warnings.warn(\"Continuous columns will not be normalised\")\n\n        # Quantization logic\n        if self.cols_and_bins is not None:\n            # we do not run 'Quantizer.fit' here since in the wild case\n            # someone wants standardization and quantization for the same\n            # columns, the Quantizer will run on the scaled data\n            self.quantizer = Quantizer(self.cols_and_bins, **self.quant_args)\n\n            if self.with_attention:\n                for col, n_cat, _ in cont_embed_dim:\n                    self.cat_embed_input.append((col, n_cat))\n            else:\n                self.cat_embed_input.extend(cont_embed_dim)\n\n        self.column_idx.update(\n            {k: v + len(self.column_idx) for v, k in enumerate(df_cont)}\n        )\n\n    self.is_fitted = True\n\n    return self\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor.transform","title":"transform","text":"<pre><code>transform(df)\n</code></pre> <p>Returns the processed <code>dataframe</code> as a np.ndarray</p> <p>Parameters:</p> <ul> <li> <code>df</code>               (<code>DataFrame</code>)           \u2013            <p>Input pandas dataframe</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>ndarray</code>           \u2013            <p>transformed input dataframe</p> </li> </ul> Source code in <code>pytorch_widedeep/preprocessing/tab_preprocessor.py</code> <pre><code>def transform(self, df: pd.DataFrame) -&gt; np.ndarray:  # noqa: C901\n    \"\"\"Returns the processed `dataframe` as a np.ndarray\n\n    Parameters\n    ----------\n    df: pd.DataFrame\n        Input pandas dataframe\n\n    Returns\n    -------\n    np.ndarray\n        transformed input dataframe\n    \"\"\"\n    check_is_fitted(self, condition=self.is_fitted)\n\n    df_adj = self._insert_cls_token(df) if self.with_cls_token else df.copy()\n\n    if self.cat_embed_cols is not None:\n        df_cat = df_adj[self.cat_cols]\n        df_cat = self.label_encoder.transform(df_cat)\n    if self.continuous_cols is not None:\n        df_cont = df_adj[self.continuous_cols]\n        # Standardization logic\n        if self.standardize_cols:\n            df_cont[self.standardize_cols] = self.scaler.transform(\n                df_cont[self.standardize_cols].values\n            )\n        # Quantization logic\n        if self.cols_and_bins is not None:\n            # Adjustment so I don't have to override the method\n            # in 'ChunkTabPreprocessor'\n            if self.quantizer.is_fitted:\n                df_cont = self.quantizer.transform(df_cont)\n            else:\n                df_cont = self.quantizer.fit_transform(df_cont)\n    try:\n        df_deep = pd.concat([df_cat, df_cont], axis=1)\n    except NameError:\n        try:\n            df_deep = df_cat.copy()\n        except NameError:\n            df_deep = df_cont.copy()\n\n    return df_deep.values\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor.inverse_transform","title":"inverse_transform","text":"<pre><code>inverse_transform(encoded)\n</code></pre> <p>Takes as input the output from the <code>transform</code> method and it will return the original values.</p> <p>Parameters:</p> <ul> <li> <code>encoded</code>               (<code>ndarray</code>)           \u2013            <p>array with the output of the <code>transform</code> method</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>DataFrame</code>           \u2013            <p>Pandas dataframe with the original values</p> </li> </ul> Source code in <code>pytorch_widedeep/preprocessing/tab_preprocessor.py</code> <pre><code>def inverse_transform(self, encoded: np.ndarray) -&gt; pd.DataFrame:  # noqa: C901\n    r\"\"\"Takes as input the output from the `transform` method and it will\n    return the original values.\n\n    Parameters\n    ----------\n    encoded: np.ndarray\n        array with the output of the `transform` method\n\n    Returns\n    -------\n    pd.DataFrame\n        Pandas dataframe with the original values\n    \"\"\"\n    decoded = pd.DataFrame(encoded, columns=list(self.column_idx.keys()))\n    # embeddings back to original category\n    if self.cat_embed_cols is not None:\n        decoded = self.label_encoder.inverse_transform(decoded)\n    if self.continuous_cols is not None:\n        # quantized cols to the mid point\n        if self.cols_and_bins is not None:\n            if self.verbose:\n                print(\n                    \"Note that quantized cols will be turned into the mid point of \"\n                    \"the corresponding bin\"\n                )\n            for k, v in self.quantizer.inversed_bins.items():\n                decoded[k] = decoded[k].map(v)\n        # continuous_cols back to non-standarised\n        try:\n            decoded[self.standardize_cols] = self.scaler.inverse_transform(\n                decoded[self.standardize_cols]\n            )\n        except Exception:  # KeyError:\n            pass\n\n    if \"cls_token\" in decoded.columns:\n        decoded.drop(\"cls_token\", axis=1, inplace=True)\n\n    return decoded\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor.fit_transform","title":"fit_transform","text":"<pre><code>fit_transform(df)\n</code></pre> <p>Combines <code>fit</code> and <code>transform</code></p> <p>Parameters:</p> <ul> <li> <code>df</code>               (<code>DataFrame</code>)           \u2013            <p>Input pandas dataframe</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>ndarray</code>           \u2013            <p>transformed input dataframe</p> </li> </ul> Source code in <code>pytorch_widedeep/preprocessing/tab_preprocessor.py</code> <pre><code>def fit_transform(self, df: pd.DataFrame) -&gt; np.ndarray:\n    \"\"\"Combines `fit` and `transform`\n\n    Parameters\n    ----------\n    df: pd.DataFrame\n        Input pandas dataframe\n\n    Returns\n    -------\n    np.ndarray\n        transformed input dataframe\n    \"\"\"\n    return self.fit(df).transform(df)\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.tab_preprocessor.Quantizer","title":"Quantizer","text":"<pre><code>Quantizer(quantization_setup, **kwargs)\n</code></pre> <p>Helper class to perform the quantization of continuous columns. It is included in this docs for completion, since depending on the value of the parameter <code>'quantization_setup'</code> of the <code>TabPreprocessor</code> class, that class might have an attribute of type <code>Quantizer</code>. However, this class is designed to always run internally within the <code>TabPreprocessor</code> class.</p> <p>Parameters:</p> <ul> <li> <code>quantization_setup</code>               (<code>Dict[str, Union[int, List[float]]]</code>)           \u2013            <p>Dictionary where the keys are the column names to quantize and the values are the either integers indicating the number of bins or a list of scalars indicating the bin edges.</p> </li> </ul> Source code in <code>pytorch_widedeep/preprocessing/tab_preprocessor.py</code> <pre><code>def __init__(\n    self,\n    quantization_setup: Dict[str, Union[int, List[float]]],\n    **kwargs,\n):\n    self.quantization_setup = quantization_setup\n    self.quant_args = kwargs\n\n    self.is_fitted = False\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor","title":"TextPreprocessor","text":"<pre><code>TextPreprocessor(\n    text_col,\n    max_vocab=30000,\n    min_freq=5,\n    maxlen=80,\n    pad_first=True,\n    pad_idx=1,\n    already_processed=False,\n    word_vectors_path=None,\n    n_cpus=None,\n    verbose=1,\n)\n</code></pre> <p>               Bases: <code>BasePreprocessor</code></p> <p>Preprocessor to prepare the <code>deeptext</code> input dataset</p> <p>Parameters:</p> <ul> <li> <code>text_col</code>               (<code>str</code>)           \u2013            <p>column in the input dataframe containing the texts</p> </li> <li> <code>max_vocab</code>               (<code>int</code>, default:                   <code>30000</code> )           \u2013            <p>Maximum number of tokens in the vocabulary</p> </li> <li> <code>min_freq</code>               (<code>int</code>, default:                   <code>5</code> )           \u2013            <p>Minimum frequency for a token to be part of the vocabulary</p> </li> <li> <code>maxlen</code>               (<code>int</code>, default:                   <code>80</code> )           \u2013            <p>Maximum length of the tokenized sequences</p> </li> <li> <code>pad_first</code>               (<code>bool</code>, default:                   <code>True</code> )           \u2013            <p>Indicates whether the padding index will be added at the beginning or the end of the sequences</p> </li> <li> <code>pad_idx</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>padding index. Fastai's Tokenizer leaves 0 for the 'unknown' token.</p> </li> <li> <code>already_processed</code>               (<code>Optional[bool]</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating if the sequence of elements is already processed or prepared. If this is the case, this Preprocessor will simply tokenize and pad the sequence. </p> <pre><code>Param aliases: `not_text`. &lt;br/&gt;\n</code></pre> <p>This parameter is thought for those cases where the input sequences are already fully processed or are directly not text (e.g. IDs)</p> </li> <li> <code>word_vectors_path</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Path to the pretrained word vectors</p> </li> <li> <code>n_cpus</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>number of CPUs to used during the tokenization process</p> </li> <li> <code>verbose</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>Enable verbose output.</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>vocab</code>               (<code>Vocab</code>)           \u2013            <p>an instance of <code>pytorch_widedeep.utils.fastai_transforms.Vocab</code></p> </li> <li> <code>embedding_matrix</code>               (<code>ndarray</code>)           \u2013            <p>Array with the pretrained embeddings</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import pandas as pd\n&gt;&gt;&gt; from pytorch_widedeep.preprocessing import TextPreprocessor\n&gt;&gt;&gt; df_train = pd.DataFrame({'text_column': [\"life is like a box of chocolates\",\n... \"You never know what you're gonna get\"]})\n&gt;&gt;&gt; text_preprocessor = TextPreprocessor(text_col='text_column', max_vocab=25, min_freq=1, maxlen=10)\n&gt;&gt;&gt; text_preprocessor.fit_transform(df_train)\nThe vocabulary contains 24 tokens\narray([[ 1,  1,  1,  1, 10, 11, 12, 13, 14, 15],\n       [ 5,  9, 16, 17, 18,  9, 19, 20, 21, 22]], dtype=int32)\n&gt;&gt;&gt; df_te = pd.DataFrame({'text_column': ['you never know what is in the box']})\n&gt;&gt;&gt; text_preprocessor.transform(df_te)\narray([[ 1,  1,  9, 16, 17, 18, 11,  0,  0, 13]], dtype=int32)\n</code></pre> Source code in <code>pytorch_widedeep/preprocessing/text_preprocessor.py</code> <pre><code>@alias(\"already_processed\", [\"not_text\"])\ndef __init__(\n    self,\n    text_col: str,\n    max_vocab: int = 30000,\n    min_freq: int = 5,\n    maxlen: int = 80,\n    pad_first: bool = True,\n    pad_idx: int = 1,\n    already_processed: Optional[bool] = False,\n    word_vectors_path: Optional[str] = None,\n    n_cpus: Optional[int] = None,\n    verbose: int = 1,\n):\n    super(TextPreprocessor, self).__init__()\n\n    self.text_col = text_col\n    self.max_vocab = max_vocab\n    self.min_freq = min_freq\n    self.maxlen = maxlen\n    self.pad_first = pad_first\n    self.pad_idx = pad_idx\n    self.already_processed = already_processed\n    self.word_vectors_path = word_vectors_path\n    self.verbose = verbose\n    self.n_cpus = n_cpus if n_cpus is not None else os.cpu_count()\n\n    self.is_fitted = False\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor.fit","title":"fit","text":"<pre><code>fit(df)\n</code></pre> <p>Builds the vocabulary</p> <p>Parameters:</p> <ul> <li> <code>df</code>               (<code>DataFrame</code>)           \u2013            <p>Input pandas dataframe</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>TextPreprocessor</code>           \u2013            <p><code>TextPreprocessor</code> fitted object</p> </li> </ul> Source code in <code>pytorch_widedeep/preprocessing/text_preprocessor.py</code> <pre><code>def fit(self, df: pd.DataFrame) -&gt; BasePreprocessor:\n    \"\"\"Builds the vocabulary\n\n    Parameters\n    ----------\n    df: pd.DataFrame\n        Input pandas dataframe\n\n    Returns\n    -------\n    TextPreprocessor\n        `TextPreprocessor` fitted object\n    \"\"\"\n    texts = self._read_texts(df)\n\n    tokens = get_texts(texts, self.already_processed, self.n_cpus)\n\n    self.vocab: TVocab = Vocab(\n        max_vocab=self.max_vocab,\n        min_freq=self.min_freq,\n        pad_idx=self.pad_idx,\n    ).fit(\n        tokens,\n    )\n\n    if self.verbose:\n        print(\"The vocabulary contains {} tokens\".format(len(self.vocab.stoi)))\n    if self.word_vectors_path is not None:\n        self.embedding_matrix = build_embeddings_matrix(\n            self.vocab, self.word_vectors_path, self.min_freq\n        )\n\n    self.is_fitted = True\n\n    return self\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor.transform","title":"transform","text":"<pre><code>transform(df)\n</code></pre> <p>Returns the padded, 'numericalised' sequences</p> <p>Parameters:</p> <ul> <li> <code>df</code>               (<code>DataFrame</code>)           \u2013            <p>Input pandas dataframe</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>ndarray</code>           \u2013            <p>Padded, 'numericalised' sequences</p> </li> </ul> Source code in <code>pytorch_widedeep/preprocessing/text_preprocessor.py</code> <pre><code>def transform(self, df: pd.DataFrame) -&gt; np.ndarray:\n    \"\"\"Returns the padded, _'numericalised'_ sequences\n\n    Parameters\n    ----------\n    df: pd.DataFrame\n        Input pandas dataframe\n\n    Returns\n    -------\n    np.ndarray\n        Padded, _'numericalised'_ sequences\n    \"\"\"\n    check_is_fitted(self, attributes=[\"vocab\"])\n    texts = self._read_texts(df)\n    tokens = get_texts(texts, self.already_processed, self.n_cpus)\n    return self._pad_sequences(tokens)\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor.transform_sample","title":"transform_sample","text":"<pre><code>transform_sample(text)\n</code></pre> <p>Returns the padded, 'numericalised' sequence</p> <p>Parameters:</p> <ul> <li> <code>text</code>               (<code>str</code>)           \u2013            <p>text to be tokenized and padded</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>ndarray</code>           \u2013            <p>Padded, 'numericalised' sequence</p> </li> </ul> Source code in <code>pytorch_widedeep/preprocessing/text_preprocessor.py</code> <pre><code>def transform_sample(self, text: str) -&gt; np.ndarray:\n    \"\"\"Returns the padded, _'numericalised'_ sequence\n\n    Parameters\n    ----------\n    text: str\n        text to be tokenized and padded\n\n    Returns\n    -------\n    np.ndarray\n        Padded, _'numericalised'_ sequence\n    \"\"\"\n    check_is_fitted(self, attributes=[\"vocab\"])\n    tokens = get_texts([text], self.already_processed, self.n_cpus)\n    return self._pad_sequences(tokens)[0]\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor.fit_transform","title":"fit_transform","text":"<pre><code>fit_transform(df)\n</code></pre> <p>Combines <code>fit</code> and <code>transform</code></p> <p>Parameters:</p> <ul> <li> <code>df</code>               (<code>DataFrame</code>)           \u2013            <p>Input pandas dataframe</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>ndarray</code>           \u2013            <p>Padded, 'numericalised' sequences</p> </li> </ul> Source code in <code>pytorch_widedeep/preprocessing/text_preprocessor.py</code> <pre><code>def fit_transform(self, df: pd.DataFrame) -&gt; np.ndarray:\n    \"\"\"Combines `fit` and `transform`\n\n    Parameters\n    ----------\n    df: pd.DataFrame\n        Input pandas dataframe\n\n    Returns\n    -------\n    np.ndarray\n        Padded, _'numericalised'_ sequences\n    \"\"\"\n    return self.fit(df).transform(df)\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor.inverse_transform","title":"inverse_transform","text":"<pre><code>inverse_transform(padded_seq)\n</code></pre> <p>Returns the original text plus the added 'special' tokens</p> <p>Parameters:</p> <ul> <li> <code>padded_seq</code>               (<code>ndarray</code>)           \u2013            <p>array with the output of the <code>transform</code> method</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>DataFrame</code>           \u2013            <p>Pandas dataframe with the original text plus the added 'special' tokens</p> </li> </ul> Source code in <code>pytorch_widedeep/preprocessing/text_preprocessor.py</code> <pre><code>def inverse_transform(self, padded_seq: np.ndarray) -&gt; pd.DataFrame:\n    \"\"\"Returns the original text plus the added 'special' tokens\n\n    Parameters\n    ----------\n    padded_seq: np.ndarray\n        array with the output of the `transform` method\n\n    Returns\n    -------\n    pd.DataFrame\n        Pandas dataframe with the original text plus the added 'special' tokens\n    \"\"\"\n    texts = [self.vocab.inverse_transform(num) for num in padded_seq]\n    return pd.DataFrame({self.text_col: texts})\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor","title":"HFPreprocessor","text":"<pre><code>HFPreprocessor(\n    model_name,\n    *,\n    use_fast_tokenizer=False,\n    text_col=None,\n    root_dir=None,\n    num_workers=None,\n    preprocessing_rules=None,\n    tokenizer_params=None,\n    encode_params=None,\n    **kwargs\n)\n</code></pre> <p>               Bases: <code>BasePreprocessor</code></p> <p>Text processor to prepare the <code>deeptext</code> input dataset that is a wrapper around HuggingFace's tokenizers.</p> <p>Following the main phylosophy of the <code>pytorch-widedeep</code> library, this class is designed to be as flexible as possible. Therefore, it is coded so that the user can use it as one would use any HuggingFace tokenizers, or following the API call 'protocol' of the rest of the library.</p> <p>Parameters:</p> <ul> <li> <code>model_name</code>               (<code>str</code>)           \u2013            <p>The model name from the transformers library e.g. 'bert-base-uncased'. Currently supported models are those from the families: BERT, RoBERTa, DistilBERT, ALBERT and ELECTRA.</p> </li> <li> <code>use_fast_tokenizer</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Whether to use the fast tokenizer from HuggingFace or not</p> </li> <li> <code>text_col</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>The column in the input dataframe containing the text data. If this tokenizer is used via the <code>fit</code> and <code>transform</code> methods, this argument is mandatory. If the tokenizer is used via the <code>encode</code> method, this argument is not needed since the input text is passed directly to the <code>encode</code> method.</p> </li> <li> <code>num_workers</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>Number of workers to use when preprocessing the text data. If not None, and <code>use_fast_tokenizer</code> is False, the text data will be preprocessed in parallel using the number of workers specified. If <code>use_fast_tokenizer</code> is True, this argument is ignored.</p> </li> <li> <code>preprocessing_rules</code>               (<code>Optional[List[Callable[[str], str]]]</code>, default:                   <code>None</code> )           \u2013            <p>A list of functions to be applied to the text data before encoding. This can be useful to clean the text data before encoding. For example, removing html tags, special characters, etc.</p> </li> <li> <code>tokenizer_params</code>               (<code>Optional[Dict[str, Any]]</code>, default:                   <code>None</code> )           \u2013            <p>Additional parameters to be passed to the HuggingFace's <code>PreTrainedTokenizer</code>. Parameters to the <code>PreTrainedTokenizer</code> can also be passed via the <code>**kwargs</code> argument</p> </li> <li> <code>encode_params</code>               (<code>Optional[Dict[str, Any]]</code>, default:                   <code>None</code> )           \u2013            <p>Additional parameters to be passed to the <code>batch_encode_plus</code> method of the HuggingFace's <code>PreTrainedTokenizer</code>. If the <code>fit</code> and <code>transform</code> methods are used, the <code>encode_params</code> dict parameter is mandatory. If the <code>encode</code> method is used, this parameter is not needed since the input text is passed directly to the <code>encode</code> method.</p> </li> <li> <code>**kwargs</code>           \u2013            <p>Additional kwargs to be passed to the model, in particular to the <code>PreTrainedTokenizer</code> class.</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>is_fitted</code>               (<code>bool</code>)           \u2013            <p>Boolean indicating if the preprocessor has been fitted. This is a HuggingFacea tokenizer, so it is always considered fitted and this attribute is manually set to True internally. This parameter exists for consistency with the rest of the library and because is needed for some functionality in the library.</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import pandas as pd\n&gt;&gt;&gt; from pytorch_widedeep.preprocessing import HFPreprocessor\n&gt;&gt;&gt; df = pd.DataFrame({\"text\": [\"this is the first text\", \"this is the second text\"]})\n&gt;&gt;&gt; hf_processor_1 = HFPreprocessor(model_name=\"bert-base-uncased\", text_col=\"text\")\n&gt;&gt;&gt; X_text_1 = hf_processor_1.fit_transform(df)\n&gt;&gt;&gt; texts = [\"this is a new text\", \"this is another text\"]\n&gt;&gt;&gt; hf_processor_2 = HFPreprocessor(model_name=\"bert-base-uncased\")\n&gt;&gt;&gt; X_text_2 = hf_processor_2.encode(texts, max_length=10, padding=\"max_length\", truncation=True)\n</code></pre> Source code in <code>pytorch_widedeep/preprocessing/hf_preprocessor.py</code> <pre><code>def __init__(\n    self,\n    model_name: str,\n    *,\n    use_fast_tokenizer: bool = False,\n    text_col: Optional[str] = None,\n    root_dir: Optional[str] = None,\n    num_workers: Optional[int] = None,\n    preprocessing_rules: Optional[List[Callable[[str], str]]] = None,\n    tokenizer_params: Optional[Dict[str, Any]] = None,\n    encode_params: Optional[Dict[str, Any]] = None,\n    **kwargs,\n):\n    self.model_name = model_name\n    self.use_fast_tokenizer = use_fast_tokenizer\n    self.text_col = text_col\n    self.root_dir = root_dir\n    self.num_workers = num_workers\n    self.preprocessing_rules = preprocessing_rules\n    self.tokenizer_params = tokenizer_params if tokenizer_params is not None else {}\n    self.encode_params = encode_params if encode_params is not None else {}\n\n    self._multiprocessing = (\n        num_workers is not None and num_workers &gt; 1 and not use_fast_tokenizer\n    )\n\n    if kwargs:\n        self.tokenizer_params.update(kwargs)\n\n    self.tokenizer = get_tokenizer(\n        model_name=self.model_name,\n        use_fast_tokenizer=self.use_fast_tokenizer,\n        **self.tokenizer_params,\n    )\n\n    # A HuggingFace tokenizer is already trained, since we need this\n    # attribute elsewhere in the library, we simply set it to True\n    self.is_fitted = True\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.encode","title":"encode","text":"<pre><code>encode(texts, **kwargs)\n</code></pre> <p>Encodes a list of texts. The method is a wrapper around the <code>batch_encode_plus</code> method of the HuggingFace's tokenizer.</p> <p>if 'use_fast_tokenizer' is True, the method will use the <code>batch_encode_plus</code></p> <p>Parameters:</p> <ul> <li> <code>texts</code>               (<code>List[str]</code>)           \u2013            <p>List of texts to be encoded</p> </li> <li> <code>**kwargs</code>           \u2013            <p>Additional parameters to be passed to the <code>batch_encode_plus</code> method of the HuggingFace's tokenizer. If the 'encode_params' dict was passed when instantiating the class, that dictionaly will be updated with the kwargs passed here.</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>array</code>           \u2013            <p>The encoded texts</p> </li> </ul> Source code in <code>pytorch_widedeep/preprocessing/hf_preprocessor.py</code> <pre><code>def encode(self, texts: List[str], **kwargs) -&gt; npt.NDArray[np.int64]:\n    \"\"\"\n    Encodes a list of texts. The method is a wrapper around the\n    `batch_encode_plus` method of the HuggingFace's tokenizer.\n\n    if 'use_fast_tokenizer' is True, the method will use the `batch_encode_plus`\n\n    Parameters\n    ----------\n    texts: List[str]\n        List of texts to be encoded\n    **kwargs\n        Additional parameters to be passed to the `batch_encode_plus` method\n        of the HuggingFace's tokenizer. If the 'encode_params' dict was passed\n        when instantiating the class, that dictionaly will be updated with\n        the kwargs passed here.\n\n    Returns\n    -------\n    np.array\n        The encoded texts\n    \"\"\"\n    if kwargs:\n        self.encode_params.update(kwargs)\n\n    if self.preprocessing_rules:\n        if self._multiprocessing:\n            texts = self._process_text_parallel(texts)\n        else:\n            texts = [self._preprocess_text(text) for text in texts]\n\n    if self._multiprocessing:\n        input_ids = self._encode_paralell(texts, **self.encode_params)\n    else:\n        encoded_texts = self.tokenizer.batch_encode_plus(\n            texts,\n            **self.encode_params,\n        )\n        input_ids = encoded_texts.get(\"input_ids\")\n\n    self.is_fitted = True\n\n    try:\n        output = np.array(input_ids)\n    except ValueError:\n        warnings.warn(\n            \"Padding and Truncating parameters were not passed and all input arrays \"\n            \"do not have the same shape. Padding to the longest sequence. \"\n            \"Padding will be done with the index of the pad token for the model\",\n            UserWarning,\n        )\n        max_len = max([len(ids) for ids in input_ids])\n        output = np.array(\n            [\n                np.pad(ids, (self.tokenizer.pad_token_id, max_len - len(ids)))\n                for ids in input_ids\n            ]\n        )\n\n    return output\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.decode","title":"decode","text":"<pre><code>decode(input_ids, skip_special_tokens)\n</code></pre> <p>Decodes a list of input_ids. The method is a wrapper around the <code>convert_ids_to_tokens</code> and <code>convert_tokens_to_string</code> methods of the HuggingFace's tokenizer.</p> <p>Parameters:</p> <ul> <li> <code>input_ids</code>               (<code>NDArray[int64]</code>)           \u2013            <p>The input_ids to be decoded</p> </li> <li> <code>skip_special_tokens</code>               (<code>bool</code>)           \u2013            <p>Whether to skip the special tokens or not</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>List[str]</code>           \u2013            <p>The decoded texts</p> </li> </ul> Source code in <code>pytorch_widedeep/preprocessing/hf_preprocessor.py</code> <pre><code>def decode(\n    self, input_ids: npt.NDArray[np.int64], skip_special_tokens: bool\n) -&gt; List[str]:\n    \"\"\"\n    Decodes a list of input_ids. The method is a wrapper around the\n    `convert_ids_to_tokens` and `convert_tokens_to_string` methods of the\n    HuggingFace's tokenizer.\n\n    Parameters\n    ----------\n    input_ids: npt.NDArray[np.int64]\n        The input_ids to be decoded\n    skip_special_tokens: bool\n        Whether to skip the special tokens or not\n\n    Returns\n    -------\n    List[str]\n        The decoded texts\n    \"\"\"\n    texts = [\n        self.tokenizer.convert_tokens_to_string(\n            self.tokenizer.convert_ids_to_tokens(input_ids[i], skip_special_tokens)\n        )\n        for i in range(input_ids.shape[0])\n    ]\n    return texts\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.fit","title":"fit","text":"<pre><code>fit(df)\n</code></pre> <p>This method is included for consistency with the rest of the library in general and with the <code>BasePreprocessor</code> in particular. HuggingFace's tokenizers and models are already trained. Therefore, the 'fit' method here does nothing other than checking that the 'text_col' parameter is not <code>None</code>.</p> <p>Parameters:</p> <ul> <li> <code>df</code>               (<code>DataFrame</code>)           \u2013            <p>The dataframe containing the text data in the column specified by the 'text_col' parameter</p> </li> </ul> Source code in <code>pytorch_widedeep/preprocessing/hf_preprocessor.py</code> <pre><code>def fit(self, df: pd.DataFrame) -&gt; \"HFPreprocessor\":\n    \"\"\"\n    This method is included for consistency with the rest of the library\n    in general and with the `BasePreprocessor` in particular. HuggingFace's\n    tokenizers and models are already trained. Therefore, the 'fit' method\n    here does nothing other than checking that the 'text_col' parameter is\n    not `None`.\n\n    Parameters\n    ----------\n    df: pd.DataFrame\n        The dataframe containing the text data in the column specified by\n        the 'text_col' parameter\n    \"\"\"\n    if self.text_col is None:\n        raise ValueError(\n            \"'text_col' is None. Please specify the column name containing the text data\"\n            \" if you want to use the 'fit' method\"\n        )\n    return self\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.transform","title":"transform","text":"<pre><code>transform(df)\n</code></pre> <p>Encodes the text data in the input dataframe. This method simply calls the <code>encode</code> method under the hood. Similar to the <code>fit</code> method, this method is included for consistency with the rest of the library in general and with the <code>BasePreprocessor</code> in particular.</p> <p>Parameters:</p> <ul> <li> <code>df</code>               (<code>DataFrame</code>)           \u2013            <p>The dataframe containing the text data in the column specified by the 'text_col' parameter</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>array</code>           \u2013            <p>The encoded texts</p> </li> </ul> Source code in <code>pytorch_widedeep/preprocessing/hf_preprocessor.py</code> <pre><code>def transform(self, df: pd.DataFrame) -&gt; npt.NDArray[np.int64]:\n    \"\"\"\n    Encodes the text data in the input dataframe. This method simply\n    calls the `encode` method under the hood. Similar to the `fit` method,\n    this method is included for consistency with the rest of the library\n    in general and with the `BasePreprocessor` in particular.\n\n    Parameters\n    ----------\n    df: pd.DataFrame\n        The dataframe containing the text data in the column specified by\n        the 'text_col' parameter\n\n    Returns\n    -------\n    np.array\n        The encoded texts\n    \"\"\"\n    if self.text_col is None:\n        raise ValueError(\n            \"'text_col' is None. Please specify the column name containing the text data\"\n            \" if you want to use the 'fit' method\"\n        )\n\n    texts = self._read_texts(df, self.root_dir)\n\n    return self.encode(texts)\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.transform_sample","title":"transform_sample","text":"<pre><code>transform_sample(text)\n</code></pre> <p>Encodes a single text sample.</p> <p>Parameters:</p> <ul> <li> <code>text</code>               (<code>str</code>)           \u2013            <p>The text sample to be encoded</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>array</code>           \u2013            <p>The encoded text</p> </li> </ul> Source code in <code>pytorch_widedeep/preprocessing/hf_preprocessor.py</code> <pre><code>def transform_sample(self, text: str) -&gt; npt.NDArray[np.int64]:\n    \"\"\"\n    Encodes a single text sample.\n\n    Parameters\n    ----------\n    text: str\n        The text sample to be encoded\n\n    Returns\n    -------\n    np.array\n        The encoded text\n    \"\"\"\n\n    if not self.is_fitted:\n        raise ValueError(\n            \"The `encode` (or `fit`) method must be called before calling `transform_sample`\"\n        )\n    return self.encode([text])[0]\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.fit_transform","title":"fit_transform","text":"<pre><code>fit_transform(df)\n</code></pre> <p>Encodes the text data in the input dataframe.</p> <p>Parameters:</p> <ul> <li> <code>df</code>               (<code>DataFrame</code>)           \u2013            <p>The dataframe containing the text data in the column specified by the 'text_col' parameter</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>array</code>           \u2013            <p>The encoded texts</p> </li> </ul> Source code in <code>pytorch_widedeep/preprocessing/hf_preprocessor.py</code> <pre><code>def fit_transform(self, df: pd.DataFrame) -&gt; npt.NDArray[np.int64]:\n    \"\"\"\n    Encodes the text data in the input dataframe.\n\n    Parameters\n    ----------\n    df: pd.DataFrame\n        The dataframe containing the text data in the column specified by\n        the 'text_col' parameter\n\n    Returns\n    -------\n    np.array\n        The encoded texts\n    \"\"\"\n    return self.fit(df).transform(df)\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.inverse_transform","title":"inverse_transform","text":"<pre><code>inverse_transform(input_ids, skip_special_tokens)\n</code></pre> <p>Decodes a list of input_ids. The method simply calls the <code>decode</code> method under the hood.</p> <p>Parameters:</p> <ul> <li> <code>input_ids</code>               (<code>NDArray[int64]</code>)           \u2013            <p>The input_ids to be decoded</p> </li> <li> <code>skip_special_tokens</code>               (<code>bool</code>)           \u2013            <p>Whether to skip the special tokens or not</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>List[str]</code>           \u2013            <p>The decoded texts</p> </li> </ul> Source code in <code>pytorch_widedeep/preprocessing/hf_preprocessor.py</code> <pre><code>def inverse_transform(\n    self, input_ids: npt.NDArray[np.int64], skip_special_tokens: bool\n) -&gt; List[str]:\n    \"\"\"\n    Decodes a list of input_ids. The method simply calls the `decode` method\n    under the hood.\n\n    Parameters\n    ----------\n    input_ids: npt.NDArray[np.int64]\n        The input_ids to be decoded\n    skip_special_tokens: bool\n        Whether to skip the special tokens or not\n\n    Returns\n    -------\n    List[str]\n        The decoded texts\n    \"\"\"\n    return self.decode(input_ids, skip_special_tokens)\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.image_preprocessor.ImagePreprocessor","title":"ImagePreprocessor","text":"<pre><code>ImagePreprocessor(\n    img_col, img_path, width=224, height=224, verbose=1\n)\n</code></pre> <p>               Bases: <code>BasePreprocessor</code></p> <p>Preprocessor to prepare the <code>deepimage</code> input dataset.</p> <p>The Preprocessing consists simply on resizing according to their aspect ratio</p> <p>Parameters:</p> <ul> <li> <code>img_col</code>               (<code>str</code>)           \u2013            <p>name of the column with the images filenames</p> </li> <li> <code>img_path</code>               (<code>str</code>)           \u2013            <p>path to the dicrectory where the images are stored</p> </li> <li> <code>width</code>               (<code>int</code>, default:                   <code>224</code> )           \u2013            <p>width of the resulting processed image.</p> </li> <li> <code>height</code>               (<code>int</code>, default:                   <code>224</code> )           \u2013            <p>width of the resulting processed image.</p> </li> <li> <code>verbose</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>Enable verbose output.</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>aap</code>               (<code>AspectAwarePreprocessor</code>)           \u2013            <p>an instance of <code>pytorch_widedeep.utils.image_utils.AspectAwarePreprocessor</code></p> </li> <li> <code>spp</code>               (<code>SimplePreprocessor</code>)           \u2013            <p>an instance of <code>pytorch_widedeep.utils.image_utils.SimplePreprocessor</code></p> </li> <li> <code>normalise_metrics</code>               (<code>Dict</code>)           \u2013            <p>Dict containing the normalisation metrics of the image dataset, i.e. mean and std for the R, G and B channels</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import pandas as pd\n&gt;&gt;&gt;\n&gt;&gt;&gt; from pytorch_widedeep.preprocessing import ImagePreprocessor\n&gt;&gt;&gt;\n&gt;&gt;&gt; path_to_image1 = 'tests/test_data_utils/images/galaxy1.png'\n&gt;&gt;&gt; path_to_image2 = 'tests/test_data_utils/images/galaxy2.png'\n&gt;&gt;&gt;\n&gt;&gt;&gt; df_train = pd.DataFrame({'images_column': [path_to_image1]})\n&gt;&gt;&gt; df_test = pd.DataFrame({'images_column': [path_to_image2]})\n&gt;&gt;&gt; img_preprocessor = ImagePreprocessor(img_col='images_column', img_path='.', verbose=0)\n&gt;&gt;&gt; resized_images = img_preprocessor.fit_transform(df_train)\n&gt;&gt;&gt; new_resized_images = img_preprocessor.transform(df_train)\n</code></pre> <p> NOTE: Normalising metrics will only be computed when the <code>fit_transform</code> method is run. Running <code>transform</code> only will not change the computed metrics and running <code>fit</code> only simply instantiates the resizing functions.</p> Source code in <code>pytorch_widedeep/preprocessing/image_preprocessor.py</code> <pre><code>def __init__(\n    self,\n    img_col: str,\n    img_path: str,\n    width: int = 224,\n    height: int = 224,\n    verbose: int = 1,\n):\n    super(ImagePreprocessor, self).__init__()\n\n    self.img_col = img_col\n    self.img_path = img_path\n    self.width = width\n    self.height = height\n    self.verbose = verbose\n\n    self.aap = AspectAwarePreprocessor(self.width, self.height)\n    self.spp = SimplePreprocessor(self.width, self.height)\n\n    self.compute_normalising_computed = False\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.image_preprocessor.ImagePreprocessor.transform","title":"transform","text":"<pre><code>transform(df)\n</code></pre> <p>Resizes the images to the input height and width.</p> <p>Parameters:</p> <ul> <li> <code>df</code>               (<code>DataFrame</code>)           \u2013            <p>Input pandas dataframe with the <code>img_col</code></p> </li> </ul> <p>Returns:</p> <ul> <li> <code>ndarray</code>           \u2013            <p>Resized images to the input height and width</p> </li> </ul> Source code in <code>pytorch_widedeep/preprocessing/image_preprocessor.py</code> <pre><code>def transform(self, df: pd.DataFrame) -&gt; np.ndarray:\n    \"\"\"Resizes the images to the input height and width.\n\n\n    Parameters\n    ----------\n    df: pd.DataFrame\n        Input pandas dataframe with the `img_col`\n\n    Returns\n    -------\n    np.ndarray\n        Resized images to the input height and width\n    \"\"\"\n    image_list = df[self.img_col].tolist()\n    if self.verbose:\n        print(\"Reading Images from {}\".format(self.img_path))\n    imgs = [cv2.imread(\"/\".join([self.img_path, img])) for img in image_list]\n\n    # finding images with different height and width\n    aspect = [(im.shape[0], im.shape[1]) for im in imgs]\n    aspect_r = [a[0] / a[1] for a in aspect]\n    diff_idx = [i for i, r in enumerate(aspect_r) if r != 1.0]\n\n    if self.verbose:\n        print(\"Resizing\")\n    resized_imgs = []\n    for i, img in tqdm(enumerate(imgs), total=len(imgs), disable=self.verbose != 1):\n        if i in diff_idx:\n            resized_imgs.append(self.aap.preprocess(img))\n        else:\n            # if aspect ratio is 1:1, no need for AspectAwarePreprocessor\n            resized_imgs.append(self.spp.preprocess(img))\n\n    if not self.compute_normalising_computed:\n        if self.verbose:\n            print(\"Computing normalisation metrics\")\n        # mean and std deviation will only be computed when the fit method\n        # is called\n        mean_R, mean_G, mean_B = [], [], []\n        std_R, std_G, std_B = [], [], []\n        for rsz_img in resized_imgs:\n            (mean_b, mean_g, mean_r), (std_b, std_g, std_r) = cv2.meanStdDev(\n                rsz_img\n            )\n            mean_R.append(mean_r)\n            mean_G.append(mean_g)\n            mean_B.append(mean_b)\n            std_R.append(std_r)\n            std_G.append(std_g)\n            std_B.append(std_b)\n        self.normalise_metrics = dict(\n            mean={\n                \"R\": np.mean(mean_R) / 255.0,\n                \"G\": np.mean(mean_G) / 255.0,\n                \"B\": np.mean(mean_B) / 255.0,\n            },\n            std={\n                \"R\": np.mean(std_R) / 255.0,\n                \"G\": np.mean(std_G) / 255.0,\n                \"B\": np.mean(std_B) / 255.0,\n            },\n        )\n        self.compute_normalising_computed = True\n    return np.asarray(resized_imgs)\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.image_preprocessor.ImagePreprocessor.fit_transform","title":"fit_transform","text":"<pre><code>fit_transform(df)\n</code></pre> <p>Combines <code>fit</code> and <code>transform</code></p> <p>Parameters:</p> <ul> <li> <code>df</code>               (<code>DataFrame</code>)           \u2013            <p>Input pandas dataframe</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>ndarray</code>           \u2013            <p>Resized images to the input height and width</p> </li> </ul> Source code in <code>pytorch_widedeep/preprocessing/image_preprocessor.py</code> <pre><code>def fit_transform(self, df: pd.DataFrame) -&gt; np.ndarray:\n    \"\"\"Combines `fit` and `transform`\n\n    Parameters\n    ----------\n    df: pd.DataFrame\n        Input pandas dataframe\n\n    Returns\n    -------\n    np.ndarray\n        Resized images to the input height and width\n    \"\"\"\n    return self.fit(df).transform(df)\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.din_preprocessor.DINPreprocessor","title":"DINPreprocessor","text":"<pre><code>DINPreprocessor(\n    *,\n    user_id_col,\n    item_embed_col,\n    target_col,\n    max_seq_length,\n    action_col=None,\n    other_seq_embed_cols=None,\n    cat_embed_cols=None,\n    continuous_cols=None,\n    quantization_setup=None,\n    cols_to_scale=None,\n    auto_embed_dim=True,\n    embedding_rule=\"fastai_new\",\n    default_embed_dim=16,\n    verbose=1,\n    scale=False,\n    already_standard=None,\n    **kwargs\n)\n</code></pre> <p>               Bases: <code>BasePreprocessor</code></p> <p>Preprocessor for Deep Interest Network (DIN) models.</p> <p>This preprocessor handles the preparation of data for DIN models, including sequence building, label encoding, and handling of various types of input columns (categorical, continuous, and sequential).</p> <p>user_id_col : str     Name of the column containing user IDs. item_embed_col : Union[str, Tuple[str, int]]     Name of the column containing item IDs to be embedded, or a tuple of     (column_name, embedding_dim). target_col : str     Name of the column containing the target variable. max_seq_length : int     Maximum length of sequences to be created. action_col : Optional[str], default=None     Name of the column containing user actions (if applicable). other_seq_embed_cols : Optional[List[str] | List[Tuple[str, int]]], default=None     List of other columns to be treated as sequences. cat_embed_cols : Optional[Union[List[str], List[Tuple[str, int]]]], default=None     List of categorical columns to be represented by embeddings. continuous_cols: List, default = None     List with the name of the continuous cols. quantization_setup: int or Dict[str, Union[int, List[float]]], default=None     Continuous columns can be turned into categorical via <code>pd.cut</code>. cols_to_scale: List or str, default = None,     List with the names of the columns that will be standardized via     sklearn's <code>StandardScaler</code>. auto_embed_dim: bool, default = True     Boolean indicating whether the embedding dimensions will be     automatically defined via rule of thumb. embedding_rule: str, default = 'fastai_new'     Rule of thumb for embedding size. default_embed_dim: int, default=16     Default dimension for the embeddings. verbose : int, default=1     Verbosity level. scale: bool, default = False     Boolean indicating whether or not to scale/standardize continuous cols. already_standard: List, default = None     List with the name of the continuous cols that do not need to be     scaled/standardized. **kwargs :     Additional keyword arguments to be passed to the TabPreprocessor.</p> <p>is_fitted : bool     Whether the preprocessor has been fitted. has_standard_tab_data : bool     Whether the data includes standard tabular data. tab_preprocessor : TabPreprocessor     Preprocessor for standard tabular data. din_columns_idx : Dict[str, int]     Dictionary mapping column names to their indices in the processed data. item_le : LabelEncoder     Label encoder for item IDs. n_items : int     Number of unique items. user_behaviour_config : Tuple[List[str], int, int]     Configuration for user behavior sequences. action_le : LabelEncoder     Label encoder for action column (if applicable). n_actions : int     Number of unique actions (if applicable). action_seq_config : Tuple[List[str], int]     Configuration for action sequences (if applicable). other_seq_le : LabelEncoder     Label encoder for other sequence columns (if applicable). n_other_seq_cols : Dict[str, int]     Number of unique values in each other sequence column. other_seq_config : List[Tuple[List[str], int, int]]     Configuration for other sequence columns.</p> <p>import pandas as pd from pytorch_widedeep.preprocessing import DINPreprocessor data = { ...    'user_id': [1, 1, 1, 2, 2, 2, 3, 3, 3], ...    'item_id': [101, 102, 103, 101, 103, 104, 102, 103, 104], ...    'timestamp': [1, 2, 3, 1, 2, 3, 1, 2, 3], ...    'category': ['A', 'B', 'A', 'B', 'A', 'C', 'B', 'A', 'C'], ...    'price': [10.5, 15.0, 12.0, 10.5, 12.0, 20.0, 15.0, 12.0, 20.0], ...    'rating': [0, 1, 0, 1, 0, 1, 0, 1, 0] ... } df = pd.DataFrame(data) din_preprocessor = DINPreprocessor( ...     user_id_col='user_id', ...     item_embed_col='item_id', ...     target_col='rating', ...     max_seq_length=2, ...     action_col='rating', ...     other_seq_embed_cols=['category'], ...     cat_embed_cols=['user_id'], ...     continuous_cols=['price'], ...     cols_to_scale=['price'] ... ) X, y = din_preprocessor.fit_transform(df)</p> Source code in <code>pytorch_widedeep/preprocessing/din_preprocessor.py</code> <pre><code>@alias(\"item_embed_col\", [\"item_id_col\"])\ndef __init__(\n    self,\n    *,\n    user_id_col: str,\n    item_embed_col: Union[str, Tuple[str, int]],\n    target_col: str,\n    max_seq_length: int,\n    action_col: Optional[str] = None,\n    other_seq_embed_cols: Optional[Union[List[str], List[Tuple[str, int]]]] = None,\n    cat_embed_cols: Optional[Union[List[str], List[Tuple[str, int]]]] = None,\n    continuous_cols: Optional[List[str]] = None,\n    quantization_setup: Optional[\n        Union[int, Dict[str, Union[int, List[float]]]]\n    ] = None,\n    cols_to_scale: Optional[Union[List[str], str]] = None,\n    auto_embed_dim: bool = True,\n    embedding_rule: Literal[\"google\", \"fastai_old\", \"fastai_new\"] = \"fastai_new\",\n    default_embed_dim: int = 16,\n    verbose: int = 1,\n    scale: bool = False,\n    already_standard: Optional[List[str]] = None,\n    **kwargs,\n):\n    self.user_id_col = user_id_col\n    self.item_embed_col = item_embed_col\n    self.max_seq_length = max_seq_length\n    self.target_col = target_col if target_col is not None else \"target\"\n    self.action_col = action_col\n    self.other_seq_embed_cols = other_seq_embed_cols\n    self.cat_embed_cols = cat_embed_cols\n    self.continuous_cols = continuous_cols\n    self.quantization_setup = quantization_setup\n    self.cols_to_scale = cols_to_scale\n    self.auto_embed_dim = auto_embed_dim\n    self.embedding_rule = embedding_rule\n    self.default_embed_dim = default_embed_dim\n    self.verbose = verbose\n    self.scale = scale\n    self.already_standard = already_standard\n    self.kwargs = kwargs\n\n    self.has_standard_tab_data = bool(self.cat_embed_cols or self.continuous_cols)\n    if self.has_standard_tab_data:\n        self.tab_preprocessor = TabPreprocessor(\n            cat_embed_cols=self.cat_embed_cols,\n            continuous_cols=self.continuous_cols,\n            quantization_setup=self.quantization_setup,\n            cols_to_scale=self.cols_to_scale,\n            auto_embed_dim=self.auto_embed_dim,\n            embedding_rule=self.embedding_rule,\n            default_embed_dim=self.default_embed_dim,\n            verbose=self.verbose,\n            scale=self.scale,\n            already_standard=self.already_standard,\n            **self.kwargs,\n        )\n\n    self.is_fitted = False\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#chunked-versions","title":"Chunked versions","text":"<p>Chunked versions of the preprocessors are also available. These are useful when the data is too big to fit in memory. See also the <code>load_from_folder</code> module in the library and the corresponding section here in the documentation.</p> <p>Note that there is not a <code>ChunkImagePreprocessor</code>. This is because the processing of the images will occur inside the <code>ImageFromFolder</code> class in the <code>load_from_folder</code> module.</p>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.wide_preprocessor.ChunkWidePreprocessor","title":"ChunkWidePreprocessor","text":"<pre><code>ChunkWidePreprocessor(\n    wide_cols, n_chunks, crossed_cols=None\n)\n</code></pre> <p>               Bases: <code>WidePreprocessor</code></p> <p>Preprocessor to prepare the wide input dataset</p> <p>This Preprocessor prepares the data for the wide, linear component. This linear model is implemented via an Embedding layer that is connected to the output neuron. <code>ChunkWidePreprocessor</code> numerically encodes all the unique values of all categorical columns <code>wide_cols + crossed_cols</code>. See the Example below.</p> <p>Parameters:</p> <ul> <li> <code>wide_cols</code>               (<code>List[str]</code>)           \u2013            <p>List of strings with the name of the columns that will label encoded and passed through the <code>wide</code> component</p> </li> <li> <code>crossed_cols</code>               (<code>Optional[List[Tuple[str, str]]]</code>, default:                   <code>None</code> )           \u2013            <p>List of Tuples with the name of the columns that will be <code>'crossed'</code> and then label encoded. e.g. [('education', 'occupation'), ...]. For binary features, a cross-product transformation is 1 if and only if the constituent features are all 1, and 0 otherwise.</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>wide_crossed_cols</code>               (<code>List</code>)           \u2013            <p>List with the names of all columns that will be label encoded</p> </li> <li> <code>encoding_dict</code>               (<code>Dict</code>)           \u2013            <p>Dictionary where the keys are the result of pasting <code>colname + '_' + column value</code> and the values are the corresponding mapped integer.</p> </li> <li> <code>inverse_encoding_dict</code>               (<code>Dict</code>)           \u2013            <p>the inverse encoding dictionary</p> </li> <li> <code>wide_dim</code>               (<code>int</code>)           \u2013            <p>Dimension of the wide model (i.e. dim of the linear layer)</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import pandas as pd\n&gt;&gt;&gt; from pytorch_widedeep.preprocessing import ChunkWidePreprocessor\n&gt;&gt;&gt; chunk = pd.DataFrame({'color': ['r', 'b', 'g'], 'size': ['s', 'n', 'l']})\n&gt;&gt;&gt; wide_cols = ['color']\n&gt;&gt;&gt; crossed_cols = [('color', 'size')]\n&gt;&gt;&gt; chunk_wide_preprocessor = ChunkWidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols,\n... n_chunks=1)\n&gt;&gt;&gt; X_wide = chunk_wide_preprocessor.fit_transform(chunk)\n</code></pre> Source code in <code>pytorch_widedeep/preprocessing/wide_preprocessor.py</code> <pre><code>def __init__(\n    self,\n    wide_cols: List[str],\n    n_chunks: int,\n    crossed_cols: Optional[List[Tuple[str, str]]] = None,\n):\n    super(ChunkWidePreprocessor, self).__init__(wide_cols, crossed_cols)\n\n    self.n_chunks = n_chunks\n\n    self.chunk_counter = 0\n\n    self.is_fitted = False\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.wide_preprocessor.ChunkWidePreprocessor.partial_fit","title":"partial_fit","text":"<pre><code>partial_fit(chunk)\n</code></pre> <p>Fits the Preprocessor and creates required attributes</p> <p>Parameters:</p> <ul> <li> <code>chunk</code>               (<code>DataFrame</code>)           \u2013            <p>Input pandas dataframe</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>ChunkWidePreprocessor</code>           \u2013            <p><code>ChunkWidePreprocessor</code> fitted object</p> </li> </ul> Source code in <code>pytorch_widedeep/preprocessing/wide_preprocessor.py</code> <pre><code>def partial_fit(self, chunk: pd.DataFrame) -&gt; \"ChunkWidePreprocessor\":\n    r\"\"\"Fits the Preprocessor and creates required attributes\n\n    Parameters\n    ----------\n    chunk: pd.DataFrame\n        Input pandas dataframe\n\n    Returns\n    -------\n    ChunkWidePreprocessor\n        `ChunkWidePreprocessor` fitted object\n    \"\"\"\n    df_wide = self._prepare_wide(chunk)\n    self.wide_crossed_cols = df_wide.columns.tolist()\n\n    if self.chunk_counter == 0:\n        self.glob_feature_set = set(\n            self._make_global_feature_list(df_wide[self.wide_crossed_cols])\n        )\n    else:\n        self.glob_feature_set.update(\n            self._make_global_feature_list(df_wide[self.wide_crossed_cols])\n        )\n\n    self.chunk_counter += 1\n\n    if self.chunk_counter == self.n_chunks:\n        self.encoding_dict = {v: i + 1 for i, v in enumerate(self.glob_feature_set)}\n        self.wide_dim = len(self.encoding_dict)\n        self.inverse_encoding_dict = {k: v for v, k in self.encoding_dict.items()}\n        self.inverse_encoding_dict[0] = \"unseen\"\n\n        self.is_fitted = True\n\n    return self\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.wide_preprocessor.ChunkWidePreprocessor.fit","title":"fit","text":"<pre><code>fit(df)\n</code></pre> <p>Runs <code>partial_fit</code>. This is just to override the fit method in the base class. This class is not designed or thought to run fit</p> Source code in <code>pytorch_widedeep/preprocessing/wide_preprocessor.py</code> <pre><code>def fit(self, df: pd.DataFrame) -&gt; \"ChunkWidePreprocessor\":\n    \"\"\"\n    Runs `partial_fit`. This is just to override the fit method in the base\n    class. This class is not designed or thought to run fit\n    \"\"\"\n    return self.partial_fit(df)\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.tab_preprocessor.ChunkTabPreprocessor","title":"ChunkTabPreprocessor","text":"<pre><code>ChunkTabPreprocessor(\n    n_chunks,\n    cat_embed_cols=None,\n    continuous_cols=None,\n    cols_and_bins=None,\n    cols_to_scale=None,\n    default_embed_dim=16,\n    with_attention=False,\n    with_cls_token=False,\n    shared_embed=False,\n    verbose=1,\n    *,\n    scale=False,\n    already_standard=None,\n    **kwargs\n)\n</code></pre> <p>               Bases: <code>TabPreprocessor</code></p> <p>Preprocessor to prepare the <code>deeptabular</code> component input dataset</p> <p>Parameters:</p> <ul> <li> <code>n_chunks</code>               (<code>int</code>)           \u2013            <p>Number of chunks that the tabular dataset is divided by.</p> </li> <li> <code>cat_embed_cols</code>               (<code>Optional[Union[List[str], List[Tuple[str, int]]]]</code>, default:                   <code>None</code> )           \u2013            <p>List containing the name of the categorical columns that will be represented by embeddings (e.g. ['education', 'relationship', ...]) or a Tuple with the name and the embedding dimension (e.g.: [ ('education',32), ('relationship',16), ...])</p> </li> <li> <code>continuous_cols</code>               (<code>Optional[List[str]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the name of the continuous cols</p> </li> <li> <code>cols_and_bins</code>               (<code>Optional[Dict[str, List[float]]]</code>, default:                   <code>None</code> )           \u2013            <p>Continuous columns can be turned into categorical via <code>pd.cut</code>. 'cols_and_bins' is dictionary where the keys are the column names to quantize and the values are a list of scalars indicating the bin edges.</p> </li> <li> <code>cols_to_scale</code>               (<code>Optional[Union[List[str], str]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the names of the columns that will be standarised via sklearn's <code>StandardScaler</code></p> </li> <li> <code>default_embed_dim</code>               (<code>int</code>, default:                   <code>16</code> )           \u2013            <p>Dimension for the embeddings if the embed_dim is not provided in the <code>cat_embed_cols</code> parameter and <code>auto_embed_dim</code> is set to <code>False</code>.</p> </li> <li> <code>with_attention</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether the preprocessed data will be passed to an attention-based model (more precisely a model where all embeddings must have the same dimensions). If <code>True</code>, the param <code>cat_embed_cols</code> must just be a list containing just the categorical column names: e.g. ['education', 'relationship', ...]. This is because they will all be  encoded using embeddings of the same dim, which will be specified  later when the model is defined.  Param alias:  <code>for_transformer</code></p> </li> <li> <code>with_cls_token</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating if a <code>'[CLS]'</code> token will be added to the dataset when using attention-based models. The final hidden state corresponding to this token is used as the aggregated representation for classification and regression tasks. If not, the categorical (and continuous embeddings if present) will be concatenated before being passed to the final MLP (if present).</p> </li> <li> <code>shared_embed</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating if the embeddings will be \"shared\" when using attention-based models. The idea behind <code>shared_embed</code> is described in the Appendix A in the TabTransformer paper: 'The goal of having column embedding is to enable the model to distinguish the classes in one column from those in the other columns'. In other words, the idea is to let the model learn which column is embedded at the time. See: <code>pytorch_widedeep.models.transformers._layers.SharedEmbeddings</code>.</p> </li> <li> <code>verbose</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            </li> <li> <code>scale</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p> note: this arg will be removed in upcoming  releases. Please use <code>cols_to_scale</code> instead.  Bool indicating  whether or not to scale/standarise continuous cols. It is important  to emphasize that all the DL models for tabular data in the library  also include the possibility of normalising the input continuous  features via a <code>BatchNorm</code> or a <code>LayerNorm</code>.  Param alias:  <code>scale_cont_cols</code>.</p> </li> <li> <code>already_standard</code>               (<code>Optional[List[str]]</code>, default:                   <code>None</code> )           \u2013            <p> note: this arg will be removed in upcoming  releases. Please use <code>cols_to_scale</code> instead.  List with the  name of the continuous cols that do not need to be  scaled/standarised.</p> </li> </ul> <p>Other Parameters:</p> <ul> <li> <code>**kwargs</code>           \u2013            <p><code>pd.cut</code> and <code>StandardScaler</code> related args</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>embed_dim</code>               (<code>Dict</code>)           \u2013            <p>Dictionary where keys are the embed cols and values are the embedding dimensions. If <code>with_attention</code> is set to <code>True</code> this attribute is not generated during the <code>fit</code> process</p> </li> <li> <code>label_encoder</code>               (<code>LabelEncoder</code>)           \u2013            <p>see <code>pytorch_widedeep.utils.dense_utils.LabelEncder</code></p> </li> <li> <code>cat_embed_input</code>               (<code>List</code>)           \u2013            <p>List of Tuples with the column name, number of individual values for that column and, If <code>with_attention</code> is set to <code>False</code>, the corresponding embeddings dim, e.g. [('education', 16, 10), ('relationship', 6, 8), ...].</p> </li> <li> <code>standardize_cols</code>               (<code>List</code>)           \u2013            <p>List of the columns that will be standarized</p> </li> <li> <code>scaler</code>               (<code>StandardScaler</code>)           \u2013            <p>an instance of <code>sklearn.preprocessing.StandardScaler</code> if 'cols_to_scale' is not None or 'scale' is 'True'</p> </li> <li> <code>column_idx</code>               (<code>Dict</code>)           \u2013            <p>Dictionary where keys are column names and values are column indexes. This is neccesary to slice tensors</p> </li> <li> <code>quantizer</code>               (<code>Quantizer</code>)           \u2013            <p>an instance of <code>Quantizer</code></p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import pandas as pd\n&gt;&gt;&gt; import numpy as np\n&gt;&gt;&gt; from pytorch_widedeep.preprocessing import ChunkTabPreprocessor\n&gt;&gt;&gt; np.random.seed(42)\n&gt;&gt;&gt; chunk_df = pd.DataFrame({'cat_col': np.random.choice(['A', 'B', 'C'], size=8),\n... 'cont_col': np.random.uniform(1, 100, size=8)})\n&gt;&gt;&gt; cat_embed_cols = [('cat_col',4)]\n&gt;&gt;&gt; cont_cols = ['cont_col']\n&gt;&gt;&gt; tab_preprocessor = ChunkTabPreprocessor(\n... n_chunks=1, cat_embed_cols=cat_embed_cols, continuous_cols=cont_cols\n... )\n&gt;&gt;&gt; X_tab = tab_preprocessor.fit_transform(chunk_df)\n&gt;&gt;&gt; tab_preprocessor.cat_embed_cols\n[('cat_col', 4)]\n&gt;&gt;&gt; tab_preprocessor.column_idx\n{'cat_col': 0, 'cont_col': 1}\n</code></pre> Source code in <code>pytorch_widedeep/preprocessing/tab_preprocessor.py</code> <pre><code>@alias(\"with_attention\", [\"for_transformer\"])\n@alias(\"cat_embed_cols\", [\"embed_cols\"])\n@alias(\"scale\", [\"scale_cont_cols\"])\n@alias(\"cols_and_bins\", [\"quantization_setup\"])\ndef __init__(\n    self,\n    n_chunks: int,\n    cat_embed_cols: Optional[Union[List[str], List[Tuple[str, int]]]] = None,\n    continuous_cols: Optional[List[str]] = None,\n    cols_and_bins: Optional[Dict[str, List[float]]] = None,\n    cols_to_scale: Optional[Union[List[str], str]] = None,\n    default_embed_dim: int = 16,\n    with_attention: bool = False,\n    with_cls_token: bool = False,\n    shared_embed: bool = False,\n    verbose: int = 1,\n    *,\n    scale: bool = False,\n    already_standard: Optional[List[str]] = None,\n    **kwargs,\n):\n    super(ChunkTabPreprocessor, self).__init__(\n        cat_embed_cols=cat_embed_cols,\n        continuous_cols=continuous_cols,\n        quantization_setup=None,\n        cols_to_scale=cols_to_scale,\n        auto_embed_dim=False,\n        embedding_rule=\"google\",  # does not matter, irrelevant\n        default_embed_dim=default_embed_dim,\n        with_attention=with_attention,\n        with_cls_token=with_cls_token,\n        shared_embed=shared_embed,\n        verbose=verbose,\n        scale=scale,\n        already_standard=already_standard,\n        **kwargs,\n    )\n\n    self.n_chunks = n_chunks\n    self.chunk_counter = 0\n\n    self.cols_and_bins = cols_and_bins  # type: ignore[assignment]\n    if self.cols_and_bins is not None:\n        self.quantizer = Quantizer(self.cols_and_bins, **self.quant_args)\n\n    self.embed_prepared = False\n    self.continuous_prepared = False\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.text_preprocessor.ChunkTextPreprocessor","title":"ChunkTextPreprocessor","text":"<pre><code>ChunkTextPreprocessor(\n    text_col,\n    n_chunks,\n    root_dir=None,\n    max_vocab=30000,\n    min_freq=5,\n    maxlen=80,\n    pad_first=True,\n    pad_idx=1,\n    already_processed=False,\n    word_vectors_path=None,\n    n_cpus=None,\n    verbose=1,\n)\n</code></pre> <p>               Bases: <code>TextPreprocessor</code></p> <p>Preprocessor to prepare the <code>deeptext</code> input dataset</p> <p>Parameters:</p> <ul> <li> <code>text_col</code>               (<code>str</code>)           \u2013            <p>column in the input dataframe containing either the texts or the filenames where the text documents are stored</p> </li> <li> <code>n_chunks</code>               (<code>int</code>)           \u2013            <p>Number of chunks that the text dataset is divided by.</p> </li> <li> <code>root_dir</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>If 'text_col' contains the filenames with the text documents, this is the path to the directory where those documents are stored.</p> </li> <li> <code>max_vocab</code>               (<code>int</code>, default:                   <code>30000</code> )           \u2013            <p>Maximum number of tokens in the vocabulary</p> </li> <li> <code>min_freq</code>               (<code>int</code>, default:                   <code>5</code> )           \u2013            <p>Minimum frequency for a token to be part of the vocabulary</p> </li> <li> <code>maxlen</code>               (<code>int</code>, default:                   <code>80</code> )           \u2013            <p>Maximum length of the tokenized sequences</p> </li> <li> <code>pad_first</code>               (<code>bool</code>, default:                   <code>True</code> )           \u2013            <p>Indicates whether the padding index will be added at the beginning or the end of the sequences</p> </li> <li> <code>pad_idx</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>padding index. Fastai's Tokenizer leaves 0 for the 'unknown' token.</p> </li> <li> <code>word_vectors_path</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Path to the pretrained word vectors</p> </li> <li> <code>n_cpus</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>number of CPUs to used during the tokenization process</p> </li> <li> <code>verbose</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>Enable verbose output.</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>vocab</code>               (<code>Vocab</code>)           \u2013            <p>an instance of <code>pytorch_widedeep.utils.fastai_transforms.ChunkVocab</code></p> </li> <li> <code>embedding_matrix</code>               (<code>ndarray</code>)           \u2013            <p>Array with the pretrained embeddings if <code>word_vectors_path</code> is not None</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import pandas as pd\n&gt;&gt;&gt; from pytorch_widedeep.preprocessing import ChunkTextPreprocessor\n&gt;&gt;&gt; chunk_df = pd.DataFrame({'text_column': [\"life is like a box of chocolates\",\n... \"You never know what you're gonna get\"]})\n&gt;&gt;&gt; chunk_text_preprocessor = ChunkTextPreprocessor(text_col='text_column', n_chunks=1,\n... max_vocab=25, min_freq=1, maxlen=10, verbose=0, n_cpus=1)\n&gt;&gt;&gt; processed_chunk = chunk_text_preprocessor.fit_transform(chunk_df)\n</code></pre> Source code in <code>pytorch_widedeep/preprocessing/text_preprocessor.py</code> <pre><code>def __init__(\n    self,\n    text_col: str,\n    n_chunks: int,\n    root_dir: Optional[str] = None,\n    max_vocab: int = 30000,\n    min_freq: int = 5,\n    maxlen: int = 80,\n    pad_first: bool = True,\n    pad_idx: int = 1,\n    already_processed: Optional[bool] = False,\n    word_vectors_path: Optional[str] = None,\n    n_cpus: Optional[int] = None,\n    verbose: int = 1,\n):\n    super(ChunkTextPreprocessor, self).__init__(\n        text_col=text_col,\n        max_vocab=max_vocab,\n        min_freq=min_freq,\n        maxlen=maxlen,\n        pad_first=pad_first,\n        pad_idx=pad_idx,\n        already_processed=already_processed,\n        word_vectors_path=word_vectors_path,\n        n_cpus=n_cpus,\n        verbose=verbose,\n    )\n\n    self.n_chunks = n_chunks\n    self.root_dir = root_dir\n\n    self.chunk_counter = 0\n\n    self.is_fitted = False\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.hf_preprocessor.ChunkHFPreprocessor","title":"ChunkHFPreprocessor","text":"<pre><code>ChunkHFPreprocessor(\n    model_name,\n    *,\n    text_col,\n    root_dir=None,\n    use_fast_tokenizer=True,\n    num_workers=None,\n    preprocessing_rules=None,\n    tokenizer_params=None,\n    encode_params=None\n)\n</code></pre> <p>               Bases: <code>HFPreprocessor</code></p> <p>Text processor to prepare the <code>deeptext</code> input dataset that is a wrapper around HuggingFace's tokenizers.</p> <p>Hugginface Tokenizer's are already 'trained'. Therefore, unlike the <code>ChunkTextPreprocessor</code> this is mostly identical to the <code>HFPreprocessor</code> with the only difference that the class needs a 'text_col' parameter to be passed. Also the parameter <code>encode_params</code> is not really optional when using this class. It must be passed containing at least the 'max_length' encoding parameter. This is because we need to ensure that  all sequences have the same length when encoding in chunks.</p> <p>Parameters:</p> <ul> <li> <code>model_name</code>               (<code>str</code>)           \u2013            <p>The model name from the transformers library e.g. 'bert-base-uncased'. Currently supported models are those from the families: BERT, RoBERTa, DistilBERT, ALBERT and ELECTRA.</p> </li> <li> <code>text_col</code>               (<code>str</code>)           \u2013            <p>The column in the input dataframe containing the text data. When using the <code>ChunkHFPreprocessor</code> the <code>text_col</code> parameter is mandatory.</p> </li> <li> <code>root_dir</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>The root directory where the text files are located. This is only needed if the text data is stored in text files. If the text data is stored in a column in the input dataframe, this parameter is not needed.</p> </li> <li> <code>use_fast_tokenizer</code>               (<code>bool</code>, default:                   <code>True</code> )           \u2013            <p>Whether to use the fast tokenizer from HuggingFace or not</p> </li> <li> <code>num_workers</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>Number of workers to use when preprocessing the text data. If not None, and <code>use_fast_tokenizer</code> is False, the text data will be preprocessed in parallel using the number of workers specified. If <code>use_fast_tokenizer</code> is True, this argument is ignored.</p> </li> <li> <code>preprocessing_rules</code>               (<code>Optional[List[Callable[[str], str]]]</code>, default:                   <code>None</code> )           \u2013            <p>A list of functions to be applied to the text data before encoding. This can be useful to clean the text data before encoding. For example, removing html tags, special characters, etc.</p> </li> <li> <code>tokenizer_params</code>               (<code>Optional[Dict[str, Any]]</code>, default:                   <code>None</code> )           \u2013            <p>Additional parameters to be passed to the HuggingFace's <code>PreTrainedTokenizer</code>.</p> </li> <li> <code>encode_params</code>               (<code>Optional[Dict[str, Any]]</code>, default:                   <code>None</code> )           \u2013            <p>Additional parameters to be passed to the <code>batch_encode_plus</code> method of the HuggingFace's <code>PreTrainedTokenizer</code>. In the case of the <code>ChunkHFPreprocessor</code>, this parameter is not really <code>Optional</code>. It must be passed containing at least the 'max_length' encoding parameter</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>is_fitted</code>               (<code>bool</code>)           \u2013            <p>Boolean indicating if the preprocessor has been fitted. This is a HuggingFacea tokenizer, so it is always considered fitted and this attribute is manually set to True internally. This parameter exists for consistency with the rest of the library and because is needed for some functionality in the library.</p> </li> </ul> Source code in <code>pytorch_widedeep/preprocessing/hf_preprocessor.py</code> <pre><code>def __init__(\n    self,\n    model_name: str,\n    *,\n    text_col: str,\n    root_dir: Optional[str] = None,\n    use_fast_tokenizer: bool = True,\n    num_workers: Optional[int] = None,\n    preprocessing_rules: Optional[List[Callable[[str], str]]] = None,\n    tokenizer_params: Optional[Dict[str, Any]] = None,\n    encode_params: Optional[Dict[str, Any]] = None,\n):\n    super().__init__(\n        model_name=model_name,\n        use_fast_tokenizer=use_fast_tokenizer,\n        text_col=text_col,\n        num_workers=num_workers,\n        preprocessing_rules=preprocessing_rules,\n        tokenizer_params=tokenizer_params,\n        encode_params=encode_params,\n    )\n\n    self.root_dir = root_dir\n\n    # when using in chunks encode_params is not really optional. I will\n    # review types in due time\n    if self.encode_params is None:\n        raise ValueError(\n            \"The 'encode_params' dict must be passed to the ChunkHFTokenizer \"\n            \"containing at least the 'max_length' encoding parameter\"\n        )\n\n    if \"padding\" not in self.encode_params or not self.encode_params[\"padding\"]:\n        self.encode_params[\"padding\"] = True\n\n    if (\n        \"truncation\" not in self.encode_params\n        or not self.encode_params[\"truncation\"]\n    ):\n        self.encode_params[\"truncation\"] = True\n</code></pre>"},{"location":"pytorch-widedeep/self_supervised_pretraining.html","title":"Self Supervised Pre-training for tabular data","text":"<p>In this library we have implemented two methods or routines that allow the user to use self-suerpvised pre-training for all tabular models in the library with the exception of the <code>TabPerceiver</code> (this is a particular model and self-supervised pre-training requires some adjustments that will be implemented in future versions). Please see the examples folder in the repo or the examples section in the docs for details on how to use self-supervised pre-training with this library.</p> <p>The two routines implemented are illustrated in the figures below. The first is from TabNet: Attentive Interpretable Tabular Learning. It is a 'standard' encoder-decoder architecture and and is designed here for models that do not use transformer-based architectures (or when the embeddings can all have different dimensions). The second is from SAINT: Improved Neural Networks for Tabular Data via Row Attention and Contrastive Pre-Training, it is based on Contrastive and Denoising learning and is designed for models that use transformer-based architectures (or when the embeddings all need to have the same dimension):</p> <p> </p> <p>Figure 1. Figure 2 in their paper. The caption of the original paper is included in case it is useful.</p> <p> </p> <p>Figure 2. Figure 1 in their paper. The caption of the original paper is included in case it is useful.</p> <p>Note that the self-supervised pre-trainers described below focus, of course, on the self-supervised pre-training phase, i.e. the left side in Figure 1 and the upper part in Figure 2. When combined with the <code>Trainer</code> described earlier in the documenation, one can reproduce the full process illustrated in the figures above.</p> <p>Also Note that it is beyond the scope of this docs to explain in detail these routines. In addition, to fully utilise the self-supervised trainers implemented in this library a minimum understanding of the processes as described in the papers is required. Therefore, we strongly encourage the users to have a look to the papers.</p>"},{"location":"pytorch-widedeep/self_supervised_pretraining.html#pytorch_widedeep.self_supervised_training.EncoderDecoderTrainer","title":"EncoderDecoderTrainer","text":"<pre><code>EncoderDecoderTrainer(\n    encoder,\n    decoder=None,\n    masked_prob=0.2,\n    optimizer=None,\n    lr_scheduler=None,\n    callbacks=None,\n    verbose=1,\n    seed=1,\n    **kwargs\n)\n</code></pre> <p>               Bases: <code>BaseEncoderDecoderTrainer</code></p> <p>This class implements an Encoder-Decoder self-supervised 'routine' inspired by TabNet: Attentive Interpretable Tabular Learning. See Figure 1 above.</p> <p>Parameters:</p> <ul> <li> <code>encoder</code>               (<code>ModelWithoutAttention</code>)           \u2013            <p>An instance of a <code>TabMlp</code>, <code>TabResNet</code> or <code>TabNet</code> model</p> </li> <li> <code>decoder</code>               (<code>Optional[DecoderWithoutAttention]</code>, default:                   <code>None</code> )           \u2013            <p>An instance of  a <code>TabMlpDecoder</code>, <code>TabResNetDecoder</code> or <code>TabNetDecoder</code> model. if <code>None</code> the decoder will be automatically built as a 'simetric' model to the Encoder</p> </li> <li> <code>masked_prob</code>               (<code>float</code>, default:                   <code>0.2</code> )           \u2013            <p>Indicates the fraction of elements in the embedding tensor that will be masked and hence used for reconstruction</p> </li> <li> <code>optimizer</code>               (<code>Optional[Optimizer]</code>, default:                   <code>None</code> )           \u2013            <p>An instance of Pytorch's <code>Optimizer</code> object (e.g. <code>torch.optim.Adam ()</code>). if no optimizer is passed it will default to <code>AdamW</code>.</p> </li> <li> <code>lr_scheduler</code>               (<code>Optional[LRScheduler]</code>, default:                   <code>None</code> )           \u2013            <p>An instance of Pytorch's <code>LRScheduler</code> object (e.g <code>torch.optim.lr_scheduler.StepLR(opt, step_size=5)</code>).</p> </li> <li> <code>callbacks</code>               (<code>Optional[List[Callback]]</code>, default:                   <code>None</code> )           \u2013            <p>List with <code>Callback</code> objects. The three callbacks available in <code>pytorch-widedeep</code> are: <code>LRHistory</code>, <code>ModelCheckpoint</code> and <code>EarlyStopping</code>. This can also be a custom callback. See <code>pytorch_widedeep.callbacks.Callback</code> or the Examples folder in the repo.</p> </li> <li> <code>verbose</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>Setting it to 0 will print nothing during training.</p> </li> <li> <code>seed</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>Random seed to be used internally for train_test_split</p> </li> </ul> <p>Other Parameters:</p> <ul> <li> <code>**kwargs</code>           \u2013            <p>Other infrequently used arguments that can also be passed as kwargs are:</p> <ul> <li> <p>device: <code>str</code>     string indicating the device. One of 'cpu' or 'gpu'</p> </li> <li> <p>num_workers: <code>int</code>     number of workers to be used internally by the data loaders</p> </li> <li> <p>reducelronplateau_criterion: <code>str</code>     This sets the criterion that will be used by the lr scheduler to     take a step: One of 'loss' or 'metric'. The ReduceLROnPlateau     learning rate is a bit particular.</p> </li> </ul> </li> </ul> Source code in <code>pytorch_widedeep/self_supervised_training/encoder_decoder_trainer.py</code> <pre><code>def __init__(\n    self,\n    encoder: ModelWithoutAttention,\n    decoder: Optional[DecoderWithoutAttention] = None,\n    masked_prob: float = 0.2,\n    optimizer: Optional[Optimizer] = None,\n    lr_scheduler: Optional[LRScheduler] = None,\n    callbacks: Optional[List[Callback]] = None,\n    verbose: int = 1,\n    seed: int = 1,\n    **kwargs,\n):\n    super().__init__(\n        encoder=encoder,\n        decoder=decoder,\n        masked_prob=masked_prob,\n        optimizer=optimizer,\n        lr_scheduler=lr_scheduler,\n        callbacks=callbacks,\n        verbose=verbose,\n        seed=seed,\n        **kwargs,\n    )\n</code></pre>"},{"location":"pytorch-widedeep/self_supervised_pretraining.html#pytorch_widedeep.self_supervised_training.EncoderDecoderTrainer.pretrain","title":"pretrain","text":"<pre><code>pretrain(\n    X_tab,\n    X_tab_val=None,\n    val_split=None,\n    validation_freq=1,\n    n_epochs=1,\n    batch_size=32,\n)\n</code></pre> <p>Pretrain method. Can also be called using <code>.fit(&lt;same_args&gt;)</code></p> <p>Parameters:</p> <ul> <li> <code>X_tab</code>               (<code>ndarray</code>)           \u2013            <p>tabular dataset</p> </li> <li> <code>X_tab_val</code>               (<code>Optional[ndarray]</code>, default:                   <code>None</code> )           \u2013            <p>validation data</p> </li> <li> <code>val_split</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>An alterative to passing the validation set is to use a train/val split fraction via <code>val_split</code></p> </li> <li> <code>validation_freq</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>epochs validation frequency</p> </li> <li> <code>n_epochs</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>number of epochs</p> </li> <li> <code>batch_size</code>               (<code>int</code>, default:                   <code>32</code> )           \u2013            <p>batch size</p> </li> </ul> Source code in <code>pytorch_widedeep/self_supervised_training/encoder_decoder_trainer.py</code> <pre><code>def pretrain(\n    self,\n    X_tab: np.ndarray,\n    X_tab_val: Optional[np.ndarray] = None,\n    val_split: Optional[float] = None,\n    validation_freq: int = 1,\n    n_epochs: int = 1,\n    batch_size: int = 32,\n):\n    r\"\"\"Pretrain method. Can also be called using `.fit(&lt;same_args&gt;)`\n\n    Parameters\n    ----------\n    X_tab: np.ndarray,\n        tabular dataset\n    X_tab_val: np.ndarray, Optional, default = None\n        validation data\n    val_split: float, Optional. default=None\n        An alterative to passing the validation set is to use a train/val\n        split fraction via `val_split`\n    validation_freq: int, default=1\n        epochs validation frequency\n    n_epochs: int, default=1\n        number of epochs\n    batch_size: int, default=32\n        batch size\n    \"\"\"\n\n    self.batch_size = batch_size\n\n    train_set, eval_set = self._train_eval_split(X_tab, X_tab_val, val_split)\n    train_loader = DataLoader(\n        dataset=train_set, batch_size=batch_size, num_workers=self.num_workers\n    )\n    train_steps = len(train_loader)\n    if eval_set is not None:\n        eval_loader = DataLoader(\n            dataset=eval_set,\n            batch_size=batch_size,\n            num_workers=self.num_workers,\n            shuffle=False,\n        )\n        eval_steps = len(eval_loader)\n\n    self.callback_container.on_train_begin(\n        {\n            \"batch_size\": batch_size,\n            \"train_steps\": train_steps,\n            \"n_epochs\": n_epochs,\n        }\n    )\n    for epoch in range(n_epochs):\n        epoch_logs: Dict[str, float] = {}\n        self.callback_container.on_epoch_begin(epoch, logs=epoch_logs)\n\n        self.train_running_loss = 0.0\n        with trange(train_steps, disable=self.verbose != 1) as t:\n            for batch_idx, X in zip(t, train_loader):\n                t.set_description(\"epoch %i\" % (epoch + 1))\n                train_loss = self._train_step(X[0], batch_idx)\n                self.callback_container.on_batch_end(batch=batch_idx)\n                print_loss_and_metric(t, train_loss)\n\n        epoch_logs = save_epoch_logs(epoch_logs, train_loss, None, \"train\")\n\n        on_epoch_end_metric = None\n        if eval_set is not None and epoch % validation_freq == (\n            validation_freq - 1\n        ):\n            self.callback_container.on_eval_begin()\n            self.valid_running_loss = 0.0\n            with trange(eval_steps, disable=self.verbose != 1) as v:\n                for batch_idx, X in zip(v, eval_loader):\n                    v.set_description(\"valid\")\n                    val_loss = self._eval_step(X[0], batch_idx)\n                    print_loss_and_metric(v, val_loss)\n            epoch_logs = save_epoch_logs(epoch_logs, val_loss, None, \"val\")\n            on_epoch_end_metric = val_loss\n        else:\n            if self.reducelronplateau:\n                raise NotImplementedError(\n                    \"ReduceLROnPlateau scheduler can be used only with validation data.\"\n                )\n\n        self.callback_container.on_epoch_end(epoch, epoch_logs, on_epoch_end_metric)\n\n        if self.early_stop:\n            self.callback_container.on_train_end(epoch_logs)\n            break\n\n    self.callback_container.on_train_end(epoch_logs)\n    self._restore_best_weights()\n    self.ed_model.train()\n</code></pre>"},{"location":"pytorch-widedeep/self_supervised_pretraining.html#pytorch_widedeep.self_supervised_training.ContrastiveDenoisingTrainer","title":"ContrastiveDenoisingTrainer","text":"<pre><code>ContrastiveDenoisingTrainer(\n    model,\n    preprocessor,\n    optimizer=None,\n    lr_scheduler=None,\n    callbacks=None,\n    loss_type=\"both\",\n    projection_head1_dims=None,\n    projection_head2_dims=None,\n    projection_heads_activation=\"relu\",\n    cat_mlp_type=\"multiple\",\n    cont_mlp_type=\"multiple\",\n    denoise_mlps_activation=\"relu\",\n    verbose=1,\n    seed=1,\n    **kwargs\n)\n</code></pre> <p>               Bases: <code>BaseContrastiveDenoisingTrainer</code></p> <p>This class trains a Contrastive, Denoising Self Supervised 'routine' that is based on the one described in SAINT: Improved Neural Networks for Tabular Data via Row Attention and Contrastive Pre-Training, their Figure 1.</p> <p>Parameters:</p> <ul> <li> <code>model</code>               (<code>ModelWithAttention</code>)           \u2013            <p>An instance of a <code>TabTransformer</code>, <code>SAINT</code>, <code>FTTransformer</code>, <code>TabFastFormer</code>, <code>TabPerceiver</code>, <code>ContextAttentionMLP</code> and <code>SelfAttentionMLP</code>.</p> </li> <li> <code>preprocessor</code>               (<code>TabPreprocessor</code>)           \u2013            <p>A fitted <code>TabPreprocessor</code> object. See <code>pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor</code></p> </li> <li> <code>optimizer</code>               (<code>Optional[Optimizer]</code>, default:                   <code>None</code> )           \u2013            <p>An instance of Pytorch's <code>Optimizer</code> object (e.g. <code>torch.optim.Adam ()</code>). if no optimizer is passed it will default to <code>AdamW</code>.</p> </li> <li> <code>lr_scheduler</code>               (<code>Optional[LRScheduler]</code>, default:                   <code>None</code> )           \u2013            <p>An instance of Pytorch's <code>LRScheduler</code> object (e.g <code>torch.optim.lr_scheduler.StepLR(opt, step_size=5)</code>).</p> </li> <li> <code>callbacks</code>               (<code>Optional[List[Callback]]</code>, default:                   <code>None</code> )           \u2013            <p>List with <code>Callback</code> objects. The three callbacks available in <code>pytorch-widedeep</code> are: <code>LRHistory</code>, <code>ModelCheckpoint</code> and <code>EarlyStopping</code>. This can also be a custom callback. See <code>pytorch_widedeep.callbacks.Callback</code> or the Examples folder in the repo.</p> </li> <li> <code>loss_type</code>               (<code>Literal[contrastive, denoising, both]</code>, default:                   <code>'both'</code> )           \u2013            <p>One of 'contrastive', 'denoising' or 'both'. See SAINT: Improved Neural Networks for Tabular Data via Row Attention and Contrastive Pre-Training, their figure (1) and their equation (5).</p> </li> <li> <code>projection_head1_dims</code>               (<code>Optional[List[int]]</code>, default:                   <code>None</code> )           \u2013            <p>The projection heads are simply MLPs. This parameter is a list of integers with the dimensions of the MLP hidden layers. See the paper for details. Note that setting up this parameter requires some knowledge of the architecture one is using. For example, if we are representing the features with embeddings of dim 32 (i.e. the so called dimension of the model is 32), then the first dimension of the projection head must be 32 (e.g. [32, 16])</p> </li> <li> <code>projection_head2_dims</code>               (<code>Optional[List[int]]</code>, default:                   <code>None</code> )           \u2013            <p>Same as 'projection_head1_dims' for the second head</p> </li> <li> <code>projection_heads_activation</code>               (<code>str</code>, default:                   <code>'relu'</code> )           \u2013            <p>Activation function for the projection heads</p> </li> <li> <code>cat_mlp_type</code>               (<code>Literal[single, multiple]</code>, default:                   <code>'multiple'</code> )           \u2013            <p>If 'denoising' loss is used, one can choose two types of 'stacked' MLPs to process the output from the transformer-based encoder that receives 'corrupted' (cut-mixed and mixed-up) features. These are 'single' or 'multiple'. The former approach will apply a single MLP to all the categorical features while the latter will use one MLP per categorical feature</p> </li> <li> <code>cont_mlp_type</code>               (<code>Literal[single, multiple]</code>, default:                   <code>'multiple'</code> )           \u2013            <p>Same as 'cat_mlp_type' but for the continuous features</p> </li> <li> <code>denoise_mlps_activation</code>               (<code>str</code>, default:                   <code>'relu'</code> )           \u2013            <p>activation function for the so called 'denoising mlps'.</p> </li> <li> <code>verbose</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>Setting it to 0 will print nothing during training.</p> </li> <li> <code>seed</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>Random seed to be used internally for train_test_split</p> </li> </ul> <p>Other Parameters:</p> <ul> <li> <code>**kwargs</code>           \u2013            <p>Other infrequently used arguments that can also be passed as kwargs are:</p> <ul> <li> <p>device: <code>str</code>     string indicating the device. One of 'cpu' or 'gpu'</p> </li> <li> <p>num_workers: <code>int</code>     number of workers to be used internally by the data loaders</p> </li> <li> <p>reducelronplateau_criterion: <code>str</code>     This sets the criterion that will be used by the lr scheduler to     take a step: One of 'loss' or 'metric'. The ReduceLROnPlateau     learning rate is a bit particular.</p> </li> </ul> </li> </ul> Source code in <code>pytorch_widedeep/self_supervised_training/contrastive_denoising_trainer.py</code> <pre><code>def __init__(\n    self,\n    model: ModelWithAttention,\n    preprocessor: TabPreprocessor,\n    optimizer: Optional[Optimizer] = None,\n    lr_scheduler: Optional[LRScheduler] = None,\n    callbacks: Optional[List[Callback]] = None,\n    loss_type: Literal[\"contrastive\", \"denoising\", \"both\"] = \"both\",\n    projection_head1_dims: Optional[List[int]] = None,\n    projection_head2_dims: Optional[List[int]] = None,\n    projection_heads_activation: str = \"relu\",\n    cat_mlp_type: Literal[\"single\", \"multiple\"] = \"multiple\",\n    cont_mlp_type: Literal[\"single\", \"multiple\"] = \"multiple\",\n    denoise_mlps_activation: str = \"relu\",\n    verbose: int = 1,\n    seed: int = 1,\n    **kwargs,\n):\n    super().__init__(\n        model=model,\n        preprocessor=preprocessor,\n        loss_type=loss_type,\n        optimizer=optimizer,\n        lr_scheduler=lr_scheduler,\n        callbacks=callbacks,\n        projection_head1_dims=projection_head1_dims,\n        projection_head2_dims=projection_head2_dims,\n        projection_heads_activation=projection_heads_activation,\n        cat_mlp_type=cat_mlp_type,\n        cont_mlp_type=cont_mlp_type,\n        denoise_mlps_activation=denoise_mlps_activation,\n        verbose=verbose,\n        seed=seed,\n        **kwargs,\n    )\n</code></pre>"},{"location":"pytorch-widedeep/self_supervised_pretraining.html#pytorch_widedeep.self_supervised_training.ContrastiveDenoisingTrainer.pretrain","title":"pretrain","text":"<pre><code>pretrain(\n    X_tab,\n    X_tab_val=None,\n    val_split=None,\n    validation_freq=1,\n    n_epochs=1,\n    batch_size=32,\n)\n</code></pre> <p>Pretrain method. Can also be called using <code>.fit(&lt;same_args&gt;)</code></p> <p>Parameters:</p> <ul> <li> <code>X_tab</code>               (<code>ndarray</code>)           \u2013            <p>tabular dataset</p> </li> <li> <code>X_tab_val</code>               (<code>Optional[ndarray]</code>, default:                   <code>None</code> )           \u2013            <p>validation data. Note that, although it is possible to use contrastive-denoising training with a validation set, such set must include feature values that are all seen in the training set in the case of the categorical columns. This is because the values of the columns themselves will be used as targets when computing the loss. Therefore, if a new category is present in the validation set that was not seen in training this will effectively be like trying to predict a new, never seen category (and Pytorch will throw an error)</p> </li> <li> <code>val_split</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>An alterative to passing the validation set is to use a train/val split fraction via <code>val_split</code></p> </li> <li> <code>validation_freq</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>epochs validation frequency</p> </li> <li> <code>n_epochs</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>number of epochs</p> </li> <li> <code>batch_size</code>               (<code>int</code>, default:                   <code>32</code> )           \u2013            <p>batch size</p> </li> </ul> Source code in <code>pytorch_widedeep/self_supervised_training/contrastive_denoising_trainer.py</code> <pre><code>def pretrain(\n    self,\n    X_tab: np.ndarray,\n    X_tab_val: Optional[np.ndarray] = None,\n    val_split: Optional[float] = None,\n    validation_freq: int = 1,\n    n_epochs: int = 1,\n    batch_size: int = 32,\n):\n    r\"\"\"Pretrain method. Can also be called using `.fit(&lt;same_args&gt;)`\n\n    Parameters\n    ----------\n    X_tab: np.ndarray,\n        tabular dataset\n    X_tab_val: np.ndarray, Optional, default = None\n        validation data. Note that, although it is possible to use\n        contrastive-denoising training with a validation set, such set\n        must include feature values that are _all_ seen in the training\n        set in the case of the categorical columns. This is because the\n        values of the columns themselves will be used as targets when\n        computing the loss. Therefore, if a new category is present in\n        the validation set that was not seen in training this will\n        effectively be like trying to predict a new, never seen category\n        (and Pytorch will throw an error)\n    val_split: float, Optional. default=None\n        An alterative to passing the validation set is to use a train/val\n        split fraction via `val_split`\n    validation_freq: int, default=1\n        epochs validation frequency\n    n_epochs: int, default=1\n        number of epochs\n    batch_size: int, default=32\n        batch size\n    \"\"\"\n\n    self.batch_size = batch_size\n\n    train_set, eval_set = self._train_eval_split(X_tab, X_tab_val, val_split)\n    train_loader = DataLoader(\n        dataset=train_set, batch_size=batch_size, num_workers=self.num_workers\n    )\n    train_steps = len(train_loader)\n    if eval_set is not None:\n        eval_loader = DataLoader(\n            dataset=eval_set,\n            batch_size=batch_size,\n            num_workers=self.num_workers,\n            shuffle=False,\n        )\n        eval_steps = len(eval_loader)\n\n    self.callback_container.on_train_begin(\n        {\n            \"batch_size\": batch_size,\n            \"train_steps\": train_steps,\n            \"n_epochs\": n_epochs,\n        }\n    )\n    for epoch in range(n_epochs):\n        epoch_logs: Dict[str, float] = {}\n        self.callback_container.on_epoch_begin(epoch, logs=epoch_logs)\n\n        self.train_running_loss = 0.0\n        with trange(train_steps, disable=self.verbose != 1) as t:\n            for batch_idx, X in zip(t, train_loader):\n                t.set_description(\"epoch %i\" % (epoch + 1))\n                train_loss = self._train_step(X[0], batch_idx)\n                self.callback_container.on_batch_end(batch=batch_idx)\n                print_loss_and_metric(t, train_loss)\n\n        epoch_logs = save_epoch_logs(epoch_logs, train_loss, None, \"train\")\n\n        on_epoch_end_metric = None\n        if eval_set is not None and epoch % validation_freq == (\n            validation_freq - 1\n        ):\n            self.callback_container.on_eval_begin()\n            self.valid_running_loss = 0.0\n            with trange(eval_steps, disable=self.verbose != 1) as v:\n                for batch_idx, X in zip(v, eval_loader):\n                    v.set_description(\"valid\")\n                    val_loss = self._eval_step(X[0], batch_idx)\n                    print_loss_and_metric(v, val_loss)\n            epoch_logs = save_epoch_logs(epoch_logs, val_loss, None, \"val\")\n            on_epoch_end_metric = val_loss\n        else:\n            if self.reducelronplateau:\n                raise NotImplementedError(\n                    \"ReduceLROnPlateau scheduler can be used only with validation data.\"\n                )\n\n        self.callback_container.on_epoch_end(epoch, epoch_logs, on_epoch_end_metric)\n\n        if self.early_stop:\n            self.callback_container.on_train_end(epoch_logs)\n            break\n\n    self.callback_container.on_train_end(epoch_logs)\n    self._restore_best_weights()\n    self.cd_model.train()\n</code></pre>"},{"location":"pytorch-widedeep/tab2vec.html","title":"Tab2Vec","text":""},{"location":"pytorch-widedeep/tab2vec.html#pytorch_widedeep.tab2vec.Tab2Vec","title":"Tab2Vec","text":"<pre><code>Tab2Vec(\n    tab_preprocessor,\n    model,\n    device,\n    return_dataframe=False,\n    verbose=False,\n)\n</code></pre> <p>Class to transform an input dataframe into vectorized form.</p> <p>This class will take an input dataframe in the form of the dataframe used for training, and it will turn it into a vectorised form based on the processing applied by the model to the categorical and continuous columns.</p> <p> NOTE: Currently this class is only implemented  for the deeptabular component. Therefore, if the input dataframe has a  text column or a column with the path to images, these will be ignored.  We will be adding these functionalities in future versions</p> <p>Parameters:</p> <ul> <li> <code>model</code>               (<code>Union[WideDeep, BayesianWide, BayesianTabMlp]</code>)           \u2013            <p><code>WideDeep</code>, <code>BayesianWide</code> or <code>BayesianTabMlp</code> model. Must be trained.</p> </li> <li> <code>tab_preprocessor</code>               (<code>TabPreprocessor</code>)           \u2013            <p><code>TabPreprocessor</code> object. Must be fitted.</p> </li> <li> <code>return_dataframe</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating of the returned object(s) will be array(s) or pandas dataframe(s)</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>vectorizer</code>               (<code>Module</code>)           \u2013            <p>Torch module with the categorical and continuous encoding process</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import string\n&gt;&gt;&gt; from random import choices\n&gt;&gt;&gt; import numpy as np\n&gt;&gt;&gt; import pandas as pd\n&gt;&gt;&gt; from pytorch_widedeep import Tab2Vec\n&gt;&gt;&gt; from pytorch_widedeep.models import TabMlp, WideDeep\n&gt;&gt;&gt; from pytorch_widedeep.preprocessing import TabPreprocessor\n&gt;&gt;&gt;\n&gt;&gt;&gt; colnames = list(string.ascii_lowercase)[:4]\n&gt;&gt;&gt; cat_col1_vals = [\"a\", \"b\", \"c\"]\n&gt;&gt;&gt; cat_col2_vals = [\"d\", \"e\", \"f\"]\n&gt;&gt;&gt;\n&gt;&gt;&gt; # Create the toy input dataframe and a toy dataframe to be vectorised\n&gt;&gt;&gt; cat_inp = [np.array(choices(c, k=5)) for c in [cat_col1_vals, cat_col2_vals]]\n&gt;&gt;&gt; cont_inp = [np.round(np.random.rand(5), 2) for _ in range(2)]\n&gt;&gt;&gt; df_inp = pd.DataFrame(np.vstack(cat_inp + cont_inp).transpose(), columns=colnames)\n&gt;&gt;&gt; cat_t2v = [np.array(choices(c, k=5)) for c in [cat_col1_vals, cat_col2_vals]]\n&gt;&gt;&gt; cont_t2v = [np.round(np.random.rand(5), 2) for _ in range(2)]\n&gt;&gt;&gt; df_t2v = pd.DataFrame(np.vstack(cat_t2v + cont_t2v).transpose(), columns=colnames)\n&gt;&gt;&gt;\n&gt;&gt;&gt; # fit the TabPreprocessor\n&gt;&gt;&gt; embed_cols = [(\"a\", 2), (\"b\", 4)]\n&gt;&gt;&gt; cont_cols = [\"c\", \"d\"]\n&gt;&gt;&gt; tab_preprocessor = TabPreprocessor(cat_embed_cols=embed_cols, continuous_cols=cont_cols)\n&gt;&gt;&gt; X_tab = tab_preprocessor.fit_transform(df_inp)\n&gt;&gt;&gt;\n&gt;&gt;&gt; # define the model (and let's assume we train it)\n&gt;&gt;&gt; tabmlp = TabMlp(\n... column_idx=tab_preprocessor.column_idx,\n... cat_embed_input=tab_preprocessor.cat_embed_input,\n... continuous_cols=tab_preprocessor.continuous_cols,\n... mlp_hidden_dims=[8, 4])\n&gt;&gt;&gt; model = WideDeep(deeptabular=tabmlp)\n&gt;&gt;&gt; # ...train the model...\n&gt;&gt;&gt;\n&gt;&gt;&gt; # vectorise the dataframe\n&gt;&gt;&gt; t2v = Tab2Vec(tab_preprocessor, model, device=\"cpu\")\n&gt;&gt;&gt; X_vec = t2v.transform(df_t2v)\n</code></pre> Source code in <code>pytorch_widedeep/tab2vec.py</code> <pre><code>def __init__(\n    self,\n    tab_preprocessor: TabPreprocessor,\n    model: Union[WideDeep, BayesianWide, BayesianTabMlp],\n    device: str | torch.device,\n    return_dataframe: bool = False,\n    verbose: bool = False,\n):\n    super(Tab2Vec, self).__init__()\n\n    self._check_inputs(tab_preprocessor, model, verbose)\n\n    if isinstance(device, str):\n        self.device = torch.device(device)\n    else:\n        self.device = device\n\n    self.tab_preprocessor = tab_preprocessor\n    self.return_dataframe = return_dataframe\n    self.verbose = verbose\n\n    self.vectorizer = self._set_vectorizer(model)\n\n    self._set_dim_attributes(tab_preprocessor, model)\n</code></pre>"},{"location":"pytorch-widedeep/tab2vec.html#pytorch_widedeep.tab2vec.Tab2Vec.fit","title":"fit","text":"<pre><code>fit(df, target_col=None)\n</code></pre> <p>This is an empty method i.e. Returns the unchanged object itself. Is only included for consistency in case <code>Tab2Vec</code> is used as part of a Pipeline</p> <p>Parameters:</p> <ul> <li> <code>df</code>               (<code>DataFrame</code>)           \u2013            <p>DataFrame to be vectorised, i.e. the categorical and continuous columns will be encoded based on the processing applied within the model</p> </li> <li> <code>target_col</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Column name of the target_col variable. If <code>None</code> only the array of predictors will be returned</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>Tab2Vec</code>           \u2013            </li> </ul> Source code in <code>pytorch_widedeep/tab2vec.py</code> <pre><code>def fit(self, df: pd.DataFrame, target_col: Optional[str] = None) -&gt; \"Tab2Vec\":\n    r\"\"\"This is an empty method i.e. Returns the unchanged object itself. Is\n    only included for consistency in case `Tab2Vec` is used as part of a\n    Pipeline\n\n    Parameters\n    ----------\n    df: pd.DataFrame\n        DataFrame to be vectorised, i.e. the categorical and continuous\n        columns will be encoded based on the processing applied within\n        the model\n    target_col: str, Optional\n        Column name of the target_col variable. If `None` only the array of\n        predictors will be returned\n\n    Returns\n    -------\n    Tab2Vec\n    \"\"\"\n\n    return self\n</code></pre>"},{"location":"pytorch-widedeep/tab2vec.html#pytorch_widedeep.tab2vec.Tab2Vec.transform","title":"transform","text":"<pre><code>transform(df, target_col=None)\n</code></pre> <p>Transforms the input dataframe into vectorized form. If a target column name is passed the target values will be returned separately in their corresponding type (np.ndarray or pd.DataFrame)</p> <p>Parameters:</p> <ul> <li> <code>df</code>               (<code>DataFrame</code>)           \u2013            <p>DataFrame to be vectorised, i.e. the categorical and continuous columns will be encoded based on the processing applied within the model</p> </li> <li> <code>target_col</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Column name of the target_col variable. If <code>None</code> only the array of predictors will be returned</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>Union[np.ndarray, Tuple[np.ndarray, np.ndarray], pd.DataFrame, Tuple[pd.DataFrame, pd.Series]</code>           \u2013            <p>Returns eiter a numpy array with the vectorised values, or a Tuple of numpy arrays with the vectorised values and the target. The same applies to dataframes in case we choose to set <code>return_dataframe = True</code></p> </li> </ul> Source code in <code>pytorch_widedeep/tab2vec.py</code> <pre><code>def transform(\n    self,\n    df: pd.DataFrame,\n    target_col: Optional[str] = None,\n) -&gt; Union[\n    np.ndarray,\n    Tuple[np.ndarray, np.ndarray],\n    pd.DataFrame,\n    Tuple[pd.DataFrame, pd.Series],\n]:\n    r\"\"\"Transforms the input dataframe into vectorized form. If a target\n    column name is passed the target values will be returned separately\n    in their corresponding type (np.ndarray or pd.DataFrame)\n\n    Parameters\n    ----------\n    df: pd.DataFrame\n        DataFrame to be vectorised, i.e. the categorical and continuous\n        columns will be encoded based on the processing applied within\n        the model\n    target_col: str, Optional\n        Column name of the target_col variable. If `None` only the array of\n        predictors will be returned\n\n    Returns\n    -------\n    Union[np.ndarray, Tuple[np.ndarray, np.ndarray], pd.DataFrame, Tuple[pd.DataFrame, pd.Series]\n        Returns eiter a numpy array with the vectorised values, or a Tuple\n        of numpy arrays with the vectorised values and the target. The\n        same applies to dataframes in case we choose to set\n        `return_dataframe = True`\n    \"\"\"\n\n    X_tab = self.tab_preprocessor.transform(df)\n\n    if self.device.type == \"mps\":\n        X = torch.from_numpy(X_tab.astype(\"float32\")).to(self.device)\n    else:\n        X = torch.from_numpy(X_tab.astype(\"float\")).to(self.device)\n\n    with torch.no_grad():\n        if self.is_tab_transformer:\n            x_vec, x_cont_not_embed = self.vectorizer(X)\n        else:\n            x_vec = self.vectorizer(X)\n            x_cont_not_embed = None\n\n    if self.tab_preprocessor.with_cls_token:\n        x_vec = x_vec[:, 1:, :]\n\n    if self.tab_preprocessor.with_attention:\n        x_vec = einops.rearrange(x_vec, \"s c e -&gt; s (c e)\")\n\n    if x_cont_not_embed is not None:\n        x_vec = torch.cat([x_vec, x_cont_not_embed], 1).detach().cpu().numpy()\n    else:\n        x_vec = x_vec.detach().cpu().numpy()\n\n    if self.return_dataframe:\n        new_colnames = self._new_colnames()\n        if target_col:\n            return pd.DataFrame(data=x_vec, columns=new_colnames), df[[target_col]]\n        else:\n            return pd.DataFrame(data=x_vec, columns=new_colnames)\n    else:\n        if target_col:\n            return x_vec, df[target_col].values\n        else:\n            return x_vec\n</code></pre>"},{"location":"pytorch-widedeep/tab2vec.html#pytorch_widedeep.tab2vec.Tab2Vec.fit_transform","title":"fit_transform","text":"<pre><code>fit_transform(df, target_col=None)\n</code></pre> <p>Combines <code>fit</code> and <code>transform</code></p> Source code in <code>pytorch_widedeep/tab2vec.py</code> <pre><code>def fit_transform(\n    self, df: pd.DataFrame, target_col: Optional[str] = None\n) -&gt; Union[\n    np.ndarray,\n    Tuple[np.ndarray, np.ndarray],\n    pd.DataFrame,\n    Tuple[pd.DataFrame, pd.Series],\n]:\n    r\"\"\"Combines `fit` and `transform`\"\"\"\n    return self.fit(df, target_col).transform(df, target_col)\n</code></pre>"},{"location":"pytorch-widedeep/the_rec_module.html","title":"The <code>rec</code> module","text":"<p>This module contains models are that specifically designed for recommendation systems. While the rest of the models can be accessed from the <code>pytorch_widedeep.models</code> module, models in this module need to be specifically imported from the <code>rec</code> module, e.g.:</p> <pre><code>from pytorch_widedeep.models.rec import DeepFactorizationMachine\n</code></pre> <p>The list of models here is not meant to be exhaustive, but it includes some common architectures such as factorization machines, field aware factorization machines or extreme factorization machines. More models will be added in the future.</p>"},{"location":"pytorch-widedeep/the_rec_module.html#pytorch_widedeep.models.rec.deepfm.DeepFactorizationMachine","title":"DeepFactorizationMachine","text":"<pre><code>DeepFactorizationMachine(\n    *,\n    column_idx,\n    num_factors,\n    reduce_sum=True,\n    cat_embed_input=None,\n    cat_embed_dropout=None,\n    use_cat_bias=None,\n    cat_embed_activation=None,\n    continuous_cols=None,\n    cont_norm_layer=None,\n    embed_continuous=None,\n    embed_continuous_method=\"piecewise\",\n    cont_embed_dropout=None,\n    cont_embed_activation=None,\n    quantization_setup=None,\n    n_frequencies=None,\n    sigma=None,\n    share_last_layer=None,\n    full_embed_dropout=None,\n    mlp_hidden_dims=None,\n    mlp_activation=None,\n    mlp_dropout=None,\n    mlp_batchnorm=None,\n    mlp_batchnorm_last=None,\n    mlp_linear_first=None\n)\n</code></pre> <p>               Bases: <code>BaseTabularModelWithAttention</code></p> <p>Deep Factorization Machine (DeepFM) for recommendation systems, which is an adaptation of 'Factorization Machines' by Steffen Rendle. Presented in 'DeepFM: A Factorization-Machine based Neural Network for CTR Prediction' by Huifeng Guo, Ruiming Tang, Yunming Yey, Zhenguo Li, Xiuqiang He. 2017.</p> <p>The implementation in this library takes advantage of all the functionalities available to encode categorical and continuous features. The model can be used with only the factorization machine</p> <p>Note that this class implements only the 'Deep' component of the model described in the paper. The linear component is not implemented 'internally' and, if one wants to include it, it can be easily added using the 'wide' (aka linear) component available in this library. See the examples in the examples folder.</p> <p>Parameters:</p> <ul> <li> <code>column_idx</code>               (<code>Dict[str, int]</code>)           \u2013            <p>Dictionary mapping column names to their corresponding index.</p> </li> <li> <code>num_factors</code>               (<code>int</code>)           \u2013            <p>Number of factors for the factorization machine.</p> </li> <li> <code>reduce_sum</code>               (<code>bool</code>, default:                   <code>True</code> )           \u2013            <p>Whether to reduce the sum in the factorization machine output.</p> </li> <li> <code>cat_embed_input</code>               (<code>Optional[List[Tuple[str, int]]]</code>, default:                   <code>None</code> )           \u2013            <p>List of tuples with categorical column names and number of unique values.</p> </li> <li> <code>cat_embed_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Categorical embeddings dropout. If <code>None</code>, it will default to 0.</p> </li> <li> <code>use_cat_bias</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating if bias will be used for the categorical embeddings. If <code>None</code>, it will default to 'False'.</p> </li> <li> <code>cat_embed_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the categorical embeddings, if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> </li> <li> <code>continuous_cols</code>               (<code>Optional[List[str]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the name of the numeric (aka continuous) columns</p> </li> <li> <code>cont_norm_layer</code>               (<code>Optional[Literal['batchnorm', 'layernorm']]</code>, default:                   <code>None</code> )           \u2013            <p>Type of normalization layer applied to the continuous features. Options are: 'layernorm' and 'batchnorm'. if <code>None</code>, no normalization layer will be used.</p> </li> <li> <code>embed_continuous_method</code>               (<code>Optional[Literal['piecewise', 'periodic']]</code>, default:                   <code>\"piecewise\"</code> )           \u2013            <p>Method to use to embed the continuous features. Options are: 'standard', 'periodic' or 'piecewise'. The 'standard' embedding method is based on the FT-Transformer implementation presented in the paper: Revisiting Deep Learning Models for Tabular Data. The 'periodic' and_'piecewise'_ methods were presented in the paper: On Embeddings for Numerical Features in Tabular Deep Learning. Please, read the papers for details.</p> </li> <li> <code>cont_embed_dim</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>Size of the continuous embeddings. If the continuous columns are embedded, <code>cont_embed_dim</code> must be passed.</p> </li> <li> <code>cont_embed_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p> </li> <li> <code>cont_embed_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the continuous embeddings if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported. If <code>None</code>, no activation function will be applied.</p> </li> <li> <code>quantization_setup</code>               (<code>Optional[Dict[str, List[float]]]</code>, default:                   <code>None</code> )           \u2013            <p>This parameter is used when the 'piecewise' method is used to embed the continuous cols. It is a dict where keys are the name of the continuous columns and values are lists with the boundaries for the quantization of the continuous_cols. See the examples for details. If If the 'piecewise' method is used, this parameter is required.</p> </li> <li> <code>n_frequencies</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>This is the so called 'k' in their paper On Embeddings for Numerical Features in Tabular Deep Learning, and is the number of 'frequencies' that will be used to represent each continuous column. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>sigma</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>This is the sigma parameter in the paper mentioned when describing the previous parameters and it is used to initialise the 'frequency weights'. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>share_last_layer</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>This parameter is not present in the before mentioned paper but it is implemented in the official repo. If <code>True</code> the linear layer that turns the frequencies into embeddings will be shared across the continuous columns. If <code>False</code> a different linear layer will be used for each continuous column. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>full_embed_dropout</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>If <code>True</code>, the full embedding corresponding to a column will be masked out/dropout. If <code>None</code>, it will default to <code>False</code>.</p> </li> <li> <code>mlp_hidden_dims</code>               (<code>Optional[List[int]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the number of neurons per dense layer in the mlp.</p> </li> <li> <code>mlp_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the dense layers of the MLP. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> </li> <li> <code>mlp_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>float or List of floats with the dropout between the dense layers. e.g: [0.5,0.5]</p> </li> <li> <code>mlp_batchnorm</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating whether or not batch normalization will be applied to the dense layers</p> </li> <li> <code>mlp_batchnorm_last</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating whether or not batch normalization will be applied to the last of the dense layers</p> </li> <li> <code>mlp_linear_first</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code></p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>mlp</code>               (<code>MLP</code>)           \u2013            <p>MLP component of the model if the mlp_hidden_dims parameter is not None. If None, the model will only return the output of the factorization machine.</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; from typing import Dict, List, Tuple\n&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from torch import Tensor\n&gt;&gt;&gt; from pytorch_widedeep.models.rec import DeepFactorizationMachine\n&gt;&gt;&gt; X = torch.randint(0, 10, (16, 2))\n&gt;&gt;&gt; column_idx: Dict[str, int] = {\"col1\": 0, \"col2\": 1}\n&gt;&gt;&gt; cat_embed_input: List[Tuple[str, int]] = [(\"col1\", 10), (\"col2\", 10)]\n&gt;&gt;&gt; fm = DeepFactorizationMachine(\n...     column_idx=column_idx,\n...     num_factors=8,\n...     cat_embed_input=cat_embed_input,\n...     mlp_hidden_dims=[16, 8]\n... )\n&gt;&gt;&gt; out = fm(X)\n</code></pre> Source code in <code>pytorch_widedeep/models/rec/deepfm.py</code> <pre><code>def __init__(\n    self,\n    *,\n    column_idx: Dict[str, int],\n    num_factors: int,\n    reduce_sum: bool = True,\n    cat_embed_input: Optional[List[Tuple[str, int]]] = None,\n    cat_embed_dropout: Optional[float] = None,\n    use_cat_bias: Optional[bool] = None,\n    cat_embed_activation: Optional[str] = None,\n    continuous_cols: Optional[List[str]] = None,\n    cont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\n    embed_continuous: Optional[bool] = None,\n    embed_continuous_method: Optional[\n        Literal[\"piecewise\", \"periodic\"]\n    ] = \"piecewise\",\n    cont_embed_dropout: Optional[float] = None,\n    cont_embed_activation: Optional[str] = None,\n    quantization_setup: Optional[Dict[str, List[float]]] = None,\n    n_frequencies: Optional[int] = None,\n    sigma: Optional[float] = None,\n    share_last_layer: Optional[bool] = None,\n    full_embed_dropout: Optional[bool] = None,\n    mlp_hidden_dims: Optional[List[int]] = None,\n    mlp_activation: Optional[str] = None,\n    mlp_dropout: Optional[float] = None,\n    mlp_batchnorm: Optional[bool] = None,\n    mlp_batchnorm_last: Optional[bool] = None,\n    mlp_linear_first: Optional[bool] = None,\n):\n    super(DeepFactorizationMachine, self).__init__(\n        column_idx=column_idx,\n        input_dim=num_factors,\n        cat_embed_input=cat_embed_input,\n        cat_embed_dropout=cat_embed_dropout,\n        use_cat_bias=use_cat_bias,\n        cat_embed_activation=cat_embed_activation,\n        shared_embed=False,\n        add_shared_embed=None,\n        frac_shared_embed=None,\n        continuous_cols=continuous_cols,\n        cont_norm_layer=cont_norm_layer,\n        embed_continuous_method=embed_continuous_method,\n        cont_embed_dropout=cont_embed_dropout,\n        cont_embed_activation=cont_embed_activation,\n        quantization_setup=quantization_setup,\n        n_frequencies=n_frequencies,\n        sigma=sigma,\n        share_last_layer=share_last_layer,\n        full_embed_dropout=full_embed_dropout,\n    )\n\n    self.reduce_sum = reduce_sum\n\n    self.mlp_hidden_dims = mlp_hidden_dims\n    self.mlp_activation = mlp_activation\n    self.mlp_dropout = mlp_dropout\n    self.mlp_batchnorm = mlp_batchnorm\n    self.mlp_batchnorm_last = mlp_batchnorm_last\n    self.mlp_linear_first = mlp_linear_first\n\n    if self.mlp_hidden_dims is not None:\n\n        if self.mlp_hidden_dims[-1] != self.input_dim:\n            d_hidden = (\n                [self.input_dim * len(self.column_idx)]\n                + self.mlp_hidden_dims\n                + [self.input_dim]\n            )\n        else:\n            d_hidden = [\n                self.input_dim * len(self.column_idx)\n            ] + self.mlp_hidden_dims\n\n        self.mlp = MLP(\n            d_hidden=d_hidden,\n            activation=(\n                \"relu\" if self.mlp_activation is None else self.mlp_activation\n            ),\n            dropout=0.0 if self.mlp_dropout is None else self.mlp_dropout,\n            batchnorm=False if self.mlp_batchnorm is None else self.mlp_batchnorm,\n            batchnorm_last=(\n                False\n                if self.mlp_batchnorm_last is None\n                else self.mlp_batchnorm_last\n            ),\n            linear_first=(\n                False if self.mlp_linear_first is None else self.mlp_linear_first\n            ),\n        )\n    else:\n        self.mlp = None\n</code></pre>"},{"location":"pytorch-widedeep/the_rec_module.html#pytorch_widedeep.models.rec.deepffm.DeepFieldAwareFactorizationMachine","title":"DeepFieldAwareFactorizationMachine","text":"<pre><code>DeepFieldAwareFactorizationMachine(\n    *,\n    column_idx,\n    num_factors,\n    cat_embed_input,\n    reduce_sum=True,\n    cat_embed_dropout=None,\n    use_cat_bias=None,\n    cat_embed_activation=None,\n    mlp_hidden_dims=None,\n    mlp_activation=None,\n    mlp_dropout=None,\n    mlp_batchnorm=None,\n    mlp_batchnorm_last=None,\n    mlp_linear_first=None\n)\n</code></pre> <p>               Bases: <code>BaseTabularModelWithAttention</code></p> <p>Deep Field Aware Factorization Machine (DeepFFM) for recommendation systems. Adaptation of the paper 'Field-aware Factorization Machines in a Real-world Online Advertising System', Juan et al. 2017.</p> <p>This class implements only the 'Deep' component of the model described in the paper. The linear component is not implemented 'internally' and, if one wants to include it, it can be easily added using the 'wide'/linear component in this library. See the examples in the examples folder.</p> <p>Note that in this case, only categorical features are accepted. This is because the embeddings of each feature will be learned using all other features. Therefore these embeddings have to be all of the same nature. This does not occur if we mix categorical and continuous features.</p> <p>Parameters:</p> <ul> <li> <code>column_idx</code>               (<code>Dict[str, int]</code>)           \u2013            <p>Dictionary mapping column names to their corresponding index.</p> </li> <li> <code>num_factors</code>               (<code>int</code>)           \u2013            <p>Number of factors for the factorization machine.</p> </li> <li> <code>reduce_sum</code>               (<code>bool</code>, default:                   <code>True</code> )           \u2013            <p>Whether to reduce the sum in the factorization machine output.</p> </li> <li> <code>cat_embed_input</code>               (<code>Optional[List[Tuple[str, int]]]</code>, default:                   <code>None</code> )           \u2013            <p>List of tuples with categorical column names and number of unique values.</p> </li> <li> <code>cat_embed_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Categorical embeddings dropout. If <code>None</code>, it will default to 0.</p> </li> <li> <code>use_cat_bias</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating if bias will be used for the categorical embeddings. If <code>None</code>, it will default to 'False'.</p> </li> <li> <code>cat_embed_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the categorical embeddings, if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> </li> <li> <code>mlp_hidden_dims</code>               (<code>Optional[List[int]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the number of neurons per dense layer in the mlp.</p> </li> <li> <code>mlp_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the dense layers of the MLP. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> </li> <li> <code>mlp_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>float or List of floats with the dropout between the dense layers. e.g: [0.5,0.5]</p> </li> <li> <code>mlp_batchnorm</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating whether or not batch normalization will be applied to the dense layers</p> </li> <li> <code>mlp_batchnorm_last</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating whether or not batch normalization will be applied to the last of the dense layers</p> </li> <li> <code>mlp_linear_first</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code></p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>n_features</code>               (<code>int</code>)           \u2013            <p>Number of unique features/columns</p> </li> <li> <code>n_tokens</code>               (<code>int</code>)           \u2013            <p>Number of unique values (tokens) in the full dataset (corpus)</p> </li> <li> <code>encoders</code>               (<code>ModuleList</code>)           \u2013            <p>List of <code>BaseTabularModelWithAttention</code> instances. One per categorical column</p> </li> <li> <code>mlp</code>               (<code>Module</code>)           \u2013            <p>Multi-layer perceptron. If <code>None</code> the output will be the output of the factorization machine (i.e. the sum of the interactions)</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from torch import Tensor\n&gt;&gt;&gt; from typing import Dict, List, Tuple\n&gt;&gt;&gt; from pytorch_widedeep.models.rec import DeepFieldAwareFactorizationMachine\n&gt;&gt;&gt; X = torch.randint(0, 10, (16, 2))\n&gt;&gt;&gt; column_idx: Dict[str, int] = {\"col1\": 0, \"col2\": 1}\n&gt;&gt;&gt; cat_embed_input: List[Tuple[str, int]] = [(\"col1\", 10), (\"col2\", 10)]\n&gt;&gt;&gt; ffm = DeepFieldAwareFactorizationMachine(\n...     column_idx=column_idx,\n...     num_factors=4,\n...     cat_embed_input=cat_embed_input,\n...     mlp_hidden_dims=[16, 8]\n... )\n&gt;&gt;&gt; output = ffm(X)\n</code></pre> Source code in <code>pytorch_widedeep/models/rec/deepffm.py</code> <pre><code>def __init__(\n    self,\n    *,\n    column_idx: Dict[str, int],\n    num_factors: int,\n    cat_embed_input: List[Tuple[str, int]],\n    reduce_sum: bool = True,\n    cat_embed_dropout: Optional[float] = None,\n    use_cat_bias: Optional[bool] = None,\n    cat_embed_activation: Optional[str] = None,\n    mlp_hidden_dims: Optional[List[int]] = None,\n    mlp_activation: Optional[str] = None,\n    mlp_dropout: Optional[float] = None,\n    mlp_batchnorm: Optional[bool] = None,\n    mlp_batchnorm_last: Optional[bool] = None,\n    mlp_linear_first: Optional[bool] = None,\n):\n    super(DeepFieldAwareFactorizationMachine, self).__init__(\n        column_idx=column_idx,\n        input_dim=num_factors,\n        cat_embed_input=cat_embed_input,\n        cat_embed_dropout=cat_embed_dropout,\n        use_cat_bias=use_cat_bias,\n        cat_embed_activation=cat_embed_activation,\n        shared_embed=False,\n        add_shared_embed=None,\n        frac_shared_embed=None,\n        continuous_cols=None,\n        cont_norm_layer=None,\n        embed_continuous_method=None,\n        cont_embed_dropout=None,\n        cont_embed_activation=None,\n        quantization_setup=None,\n        n_frequencies=None,\n        sigma=None,\n        share_last_layer=None,\n        full_embed_dropout=None,\n    )\n\n    self.reduce_sum = reduce_sum\n\n    self.mlp_hidden_dims = mlp_hidden_dims\n    self.mlp_activation = mlp_activation\n    self.mlp_dropout = mlp_dropout\n    self.mlp_batchnorm = mlp_batchnorm\n    self.mlp_batchnorm_last = mlp_batchnorm_last\n    self.mlp_linear_first = mlp_linear_first\n\n    self.n_features = len(self.column_idx)\n    self.n_tokens = sum([ei[1] for ei in cat_embed_input])\n\n    self.encoders = nn.ModuleList(\n        [\n            BaseTabularModelWithAttention(**config)\n            for config in self._get_encoder_configs()\n        ]\n    )\n\n    if self.mlp_hidden_dims is not None:\n        d_hidden = [\n            self.n_features * (self.n_features - 1) // 2 * num_factors\n        ] + self.mlp_hidden_dims\n        self.mlp = MLP(\n            d_hidden=d_hidden,\n            activation=(\n                \"relu\" if self.mlp_activation is None else self.mlp_activation\n            ),\n            dropout=0.0 if self.mlp_dropout is None else self.mlp_dropout,\n            batchnorm=False if self.mlp_batchnorm is None else self.mlp_batchnorm,\n            batchnorm_last=(\n                False\n                if self.mlp_batchnorm_last is None\n                else self.mlp_batchnorm_last\n            ),\n            linear_first=(\n                False if self.mlp_linear_first is None else self.mlp_linear_first\n            ),\n        )\n    else:\n        self.mlp = None\n</code></pre>"},{"location":"pytorch-widedeep/the_rec_module.html#pytorch_widedeep.models.rec.xdeepfm.ExtremeDeepFactorizationMachine","title":"ExtremeDeepFactorizationMachine","text":"<pre><code>ExtremeDeepFactorizationMachine(\n    *,\n    column_idx,\n    input_dim,\n    reduce_sum=True,\n    cin_layer_dims,\n    cat_embed_input,\n    cat_embed_dropout=None,\n    use_cat_bias=None,\n    cat_embed_activation=None,\n    continuous_cols=None,\n    cont_norm_layer=None,\n    embed_continuous_method=\"piecewise\",\n    cont_embed_dropout=None,\n    cont_embed_activation=None,\n    quantization_setup=None,\n    n_frequencies=None,\n    sigma=None,\n    share_last_layer=None,\n    full_embed_dropout=None,\n    mlp_hidden_dims=None,\n    mlp_activation=None,\n    mlp_dropout=None,\n    mlp_batchnorm=None,\n    mlp_batchnorm_last=None,\n    mlp_linear_first=None\n)\n</code></pre> <p>               Bases: <code>BaseTabularModelWithAttention</code></p> <p>Adaptation of 'xDeepFM implementation: xDeepFM: Combining Explicit and Implicit Feature Interactions for Recommender Systems' by Jianxun Lian, Xiaohuan Zhou, Fuzheng Zhang, Zhongxia Chen, Xing Xie, Guangzhong Sun and Enhong Chen, 2018</p> <p>The implementation in this library takes advantage of all the functionalities available to encode categorical and continuous features. The model can be used with only the factorization machine</p> <p>Note that this class implements only the 'Deep' component of the model described in the paper. The linear component is not implemented 'internally' and, if one wants to include it, it can be easily added using the 'wide'/linear component in this library. See the examples in the examples folder.</p> <p>Parameters:</p> <ul> <li> <code>column_idx</code>               (<code>Dict[str, int]</code>)           \u2013            <p>Dictionary mapping column names to their corresponding index.</p> </li> <li> <code>input_dim</code>               (<code>int</code>)           \u2013            <p>Embedding input dimensions</p> </li> <li> <code>reduce_sum</code>               (<code>bool</code>, default:                   <code>True</code> )           \u2013            <p>Whether to reduce the sum in the factorization machine output.</p> </li> <li> <code>cin_layer_dims</code>               (<code>List[int]</code>)           \u2013            <p>List with the number of units per CIN layer. e.g: [128, 64]</p> </li> <li> <code>cat_embed_input</code>               (<code>Optional[List[Tuple[str, int]]]</code>, default:                   <code>None</code> )           \u2013            <p>List of tuples with categorical column names and number of unique values.</p> </li> <li> <code>cat_embed_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Categorical embeddings dropout. If <code>None</code>, it will default to 0.</p> </li> <li> <code>use_cat_bias</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating if bias will be used for the categorical embeddings. If <code>None</code>, it will default to 'False'.</p> </li> <li> <code>cat_embed_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the categorical embeddings, if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> </li> <li> <code>continuous_cols</code>               (<code>Optional[List[str]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the name of the numeric (aka continuous) columns</p> </li> <li> <code>cont_norm_layer</code>               (<code>Optional[Literal['batchnorm', 'layernorm']]</code>, default:                   <code>None</code> )           \u2013            <p>Type of normalization layer applied to the continuous features. Options are: 'layernorm' and 'batchnorm'. if <code>None</code>, no normalization layer will be used.</p> </li> <li> <code>embed_continuous_method</code>               (<code>Optional[Literal['piecewise', 'periodic']]</code>, default:                   <code>\"piecewise\"</code> )           \u2013            <p>Method to use to embed the continuous features. Options are: 'standard', 'periodic' or 'piecewise'. The 'standard' embedding method is based on the FT-Transformer implementation presented in the paper: Revisiting Deep Learning Models for Tabular Data. The 'periodic' and_'piecewise'_ methods were presented in the paper: On Embeddings for Numerical Features in Tabular Deep Learning. Please, read the papers for details.</p> </li> <li> <code>cont_embed_dim</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>Size of the continuous embeddings. If the continuous columns are embedded, <code>cont_embed_dim</code> must be passed.</p> </li> <li> <code>cont_embed_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p> </li> <li> <code>cont_embed_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the continuous embeddings if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported. If <code>None</code>, no activation function will be applied.</p> </li> <li> <code>quantization_setup</code>               (<code>Optional[Dict[str, List[float]]]</code>, default:                   <code>None</code> )           \u2013            <p>This parameter is used when the 'piecewise' method is used to embed the continuous cols. It is a dict where keys are the name of the continuous columns and values are lists with the boundaries for the quantization of the continuous_cols. See the examples for details. If If the 'piecewise' method is used, this parameter is required.</p> </li> <li> <code>n_frequencies</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>This is the so called 'k' in their paper On Embeddings for Numerical Features in Tabular Deep Learning, and is the number of 'frequencies' that will be used to represent each continuous column. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>sigma</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>This is the sigma parameter in the paper mentioned when describing the previous parameters and it is used to initialise the 'frequency weights'. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>share_last_layer</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>This parameter is not present in the before mentioned paper but it is implemented in the official repo. If <code>True</code> the linear layer that turns the frequencies into embeddings will be shared across the continuous columns. If <code>False</code> a different linear layer will be used for each continuous column. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>full_embed_dropout</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>If <code>True</code>, the full embedding corresponding to a column will be masked out/dropout. If <code>None</code>, it will default to <code>False</code>.</p> </li> <li> <code>mlp_hidden_dims</code>               (<code>Optional[List[int]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the number of neurons per dense layer in the mlp.</p> </li> <li> <code>mlp_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the dense layers of the MLP. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> </li> <li> <code>mlp_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>float or List of floats with the dropout between the dense layers. e.g: [0.5,0.5]</p> </li> <li> <code>mlp_batchnorm</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating whether or not batch normalization will be applied to the dense layers</p> </li> <li> <code>mlp_batchnorm_last</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating whether or not batch normalization will be applied to the last of the dense layers</p> </li> <li> <code>mlp_linear_first</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code></p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>n_features</code>               (<code>int</code>)           \u2013            <p>Number of unique features/columns</p> </li> <li> <code>cin</code>               (<code>CompressedInteractionNetwork</code>)           \u2013            <p>Instance of the <code>CompressedInteractionNetwork</code> class</p> </li> <li> <code>mlp</code>               (<code>MLP</code>)           \u2013            <p>Instance of the <code>MLP</code> class if <code>mlp_hidden_dims</code> is not None. If None, the model will return directly the output of the <code>CIN</code></p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models.rec import ExtremeDeepFactorizationMachine\n&gt;&gt;&gt; X_tab = torch.randint(0, 10, (16, 2))\n&gt;&gt;&gt; column_idx = {\"col1\": 0, \"col2\": 1}\n&gt;&gt;&gt; cat_embed_input = [(\"col1\", 10), (\"col2\", 10)]\n&gt;&gt;&gt; xdeepfm = ExtremeDeepFactorizationMachine(\n...     column_idx=column_idx,\n...     input_dim=4,\n...     cin_layer_dims=[8, 16],\n...     cat_embed_input=cat_embed_input,\n...     mlp_hidden_dims=[16, 8]\n... )\n&gt;&gt;&gt; output = xdeepfm(X_tab)\n</code></pre> Source code in <code>pytorch_widedeep/models/rec/xdeepfm.py</code> <pre><code>def __init__(\n    self,\n    *,\n    column_idx: Dict[str, int],\n    input_dim: int,\n    reduce_sum: bool = True,\n    cin_layer_dims: List[int],\n    cat_embed_input: List[Tuple[str, int]],\n    cat_embed_dropout: Optional[float] = None,\n    use_cat_bias: Optional[bool] = None,\n    cat_embed_activation: Optional[str] = None,\n    continuous_cols: Optional[List[str]] = None,\n    cont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\n    embed_continuous_method: Optional[\n        Literal[\"piecewise\", \"periodic\"]\n    ] = \"piecewise\",\n    cont_embed_dropout: Optional[float] = None,\n    cont_embed_activation: Optional[str] = None,\n    quantization_setup: Optional[Dict[str, List[float]]] = None,\n    n_frequencies: Optional[int] = None,\n    sigma: Optional[float] = None,\n    share_last_layer: Optional[bool] = None,\n    full_embed_dropout: Optional[bool] = None,\n    mlp_hidden_dims: Optional[List[int]] = None,\n    mlp_activation: Optional[str] = None,\n    mlp_dropout: Optional[float] = None,\n    mlp_batchnorm: Optional[bool] = None,\n    mlp_batchnorm_last: Optional[bool] = None,\n    mlp_linear_first: Optional[bool] = None,\n):\n    super(ExtremeDeepFactorizationMachine, self).__init__(\n        column_idx=column_idx,\n        input_dim=input_dim,\n        cat_embed_input=cat_embed_input,\n        cat_embed_dropout=cat_embed_dropout,\n        use_cat_bias=use_cat_bias,\n        cat_embed_activation=cat_embed_activation,\n        shared_embed=False,\n        add_shared_embed=None,\n        frac_shared_embed=None,\n        continuous_cols=continuous_cols,\n        cont_norm_layer=cont_norm_layer,\n        embed_continuous_method=embed_continuous_method,\n        cont_embed_dropout=cont_embed_dropout,\n        cont_embed_activation=cont_embed_activation,\n        quantization_setup=quantization_setup,\n        n_frequencies=n_frequencies,\n        sigma=sigma,\n        share_last_layer=share_last_layer,\n        full_embed_dropout=full_embed_dropout,\n    )\n\n    self.mlp_hidden_dims = mlp_hidden_dims\n    self.mlp_activation = mlp_activation\n    self.mlp_dropout = mlp_dropout\n    self.mlp_batchnorm = mlp_batchnorm\n    self.mlp_batchnorm_last = mlp_batchnorm_last\n    self.mlp_linear_first = mlp_linear_first\n\n    self.reduce_sum = reduce_sum\n    self.cin_layer_dims = cin_layer_dims\n\n    self.n_features = len(self.column_idx)\n\n    self.cin = CompressedInteractionNetwork(\n        num_cols=self.n_features, cin_layer_dims=self.cin_layer_dims\n    )\n\n    if self.mlp_hidden_dims is not None:\n        if (\n            self.mlp_hidden_dims[-1] != sum(self.cin_layer_dims)\n            and not self.reduce_sum\n        ):\n            d_hidden = (\n                [sum(self.cin_layer_dims)]\n                + self.mlp_hidden_dims\n                + [sum(self.cin_layer_dims)]\n            )\n        else:\n            d_hidden = [sum(self.cin_layer_dims)] + self.mlp_hidden_dims\n\n        self.mlp = MLP(\n            d_hidden=d_hidden,\n            activation=(\n                \"relu\" if self.mlp_activation is None else self.mlp_activation\n            ),\n            dropout=0.0 if self.mlp_dropout is None else self.mlp_dropout,\n            batchnorm=False if self.mlp_batchnorm is None else self.mlp_batchnorm,\n            batchnorm_last=(\n                False\n                if self.mlp_batchnorm_last is None\n                else self.mlp_batchnorm_last\n            ),\n            linear_first=(\n                False if self.mlp_linear_first is None else self.mlp_linear_first\n            ),\n        )\n    else:\n        self.mlp = None\n</code></pre>"},{"location":"pytorch-widedeep/the_rec_module.html#pytorch_widedeep.models.rec.din.DeepInterestNetwork","title":"DeepInterestNetwork","text":"<pre><code>DeepInterestNetwork(\n    *,\n    column_idx,\n    user_behavior_confiq,\n    target_item_col=\"target_item\",\n    action_seq_config=None,\n    other_seq_cols_confiq=None,\n    attention_unit_activation=\"prelu\",\n    cat_embed_input=None,\n    cat_embed_dropout=None,\n    use_cat_bias=None,\n    cat_embed_activation=None,\n    continuous_cols=None,\n    cont_norm_layer=None,\n    embed_continuous=None,\n    embed_continuous_method=None,\n    cont_embed_dim=None,\n    cont_embed_dropout=None,\n    cont_embed_activation=None,\n    quantization_setup=None,\n    n_frequencies=None,\n    sigma=None,\n    share_last_layer=None,\n    full_embed_dropout=None,\n    mlp_hidden_dims=None,\n    mlp_activation=None,\n    mlp_dropout=None,\n    mlp_batchnorm=None,\n    mlp_batchnorm_last=None,\n    mlp_linear_first=None\n)\n</code></pre> <p>               Bases: <code>BaseWDModelComponent</code></p> <p>Adaptation of the Deep Interest Network (DIN) for recommendation systems as described in the paper: 'Deep Interest Network for Click-Through Rate Prediction' by Guorui Zhou et al. 2018.</p> <p>Note that all the categorical- and continuous-related parameters refer to the categorical and continuous columns that are not part of the sequential columns and will be treated as standard tabular data.</p> <p>This model requires some specific data preparation that allows for quite a lot of flexibility. Therefore, I have included a preprocessor (<code>DINPreprocessor</code>) in the preprocessing module that will take care of the data preparation.</p> <p>Parameters:</p> <ul> <li> <code>column_idx</code>               (<code>Dict[str, int]</code>)           \u2013            <p>Dictionary mapping column names to their corresponding index.</p> </li> <li> <code>target_item_col</code>               (<code>str</code>, default:                   <code>'target_item'</code> )           \u2013            <p>Name of the target item column.</p> </li> <li> <code>user_behavior_confiq</code>               (<code>Tuple[List[str], int, int]</code>)           \u2013            <p>Configuration for user behavior sequence columns. Tuple containing: - List of column names that correspond to the user behavior sequence - Number of unique feature values (n_tokens) - Embedding dimension Example: <code>([\"item_1\", \"item_2\", \"item_3\"], 5, 8)</code></p> </li> <li> <code>action_seq_config</code>               (<code>Optional[Tuple[List[str], int]]</code>, default:                   <code>None</code> )           \u2013            <p>Configuration for a so-called action sequence columns (for example a rating, or purchased/not-purchased, etc). Tuple containing: - List of column names - Number of unique feature values (n_tokens) This action will always be learned as a 1d embedding and will be combined with the user behaviour. For example, imagine that the action is purchased/not-purchased. then per item in the user behaviour sequence there will be a binary action to learn 0/1. Such action will be represented by a float number that will multiply the corresponding item embedding in the user behaviour sequence. Example: <code>([\"rating_1\", \"rating_2\", \"rating_3\"], 5)</code> Internally, the embedding dimension will be set to 1</p> </li> <li> <code>other_seq_cols_confiq</code>               (<code>Optional[List[Tuple[List[str], int, int]]]</code>, default:                   <code>None</code> )           \u2013            <p>Configuration for other sequential columns. List of tuples containing: - List of column names that correspond to the sequential column - Number of unique feature values (n_tokens) - Embedding dimension Example: <code>[([\"seq1_col1\", \"seq1_col2\"], 5, 8), ([\"seq2_col1\", \"seq2_col2\"], 5, 8)]</code></p> </li> <li> <code>attention_unit_activation</code>               (<code>Literal['prelu', 'dice']</code>, default:                   <code>\"prelu\"</code> )           \u2013            <p>Activation function to use in the attention unit.</p> </li> <li> <code>cat_embed_input</code>               (<code>Optional[List[Tuple[str, int, int]]]</code>, default:                   <code>None</code> )           \u2013            <p>Configuration for other columns. List of tuples containing: - Column name - Number of unique feature values (n_tokens) - Embedding dimension</p> <p>Note: From here in advance the remaining parameters are related to the categorical and continuous columns that are not part of the sequential columns and will be treated as standard tabular data.</p> </li> <li> <code>cat_embed_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Categorical embeddings dropout. If <code>None</code>, it will default to 0.</p> </li> <li> <code>use_cat_bias</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating if bias will be used for the categorical embeddings. If <code>None</code>, it will default to 'False'.</p> </li> <li> <code>cat_embed_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the categorical embeddings, if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> </li> <li> <code>continuous_cols</code>               (<code>Optional[List[str]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the name of the numeric (aka continuous) columns</p> </li> <li> <code>cont_norm_layer</code>               (<code>Optional[Literal['batchnorm', 'layernorm']]</code>, default:                   <code>None</code> )           \u2013            <p>Type of normalization layer applied to the continuous features. Options are: 'layernorm' and 'batchnorm'. if <code>None</code>, no normalization layer will be used.</p> </li> <li> <code>embed_continuous</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating if the continuous columns will be embedded using one of the available methods: 'standard', 'periodic' or 'piecewise'. If <code>None</code>, it will default to 'False'. NOTE: This parameter is deprecated and it  will be removed in future releases. Please, use the  <code>embed_continuous_method</code> parameter instead.</p> </li> <li> <code>embed_continuous_method</code>               (<code>Optional[Literal['piecewise', 'periodic']]</code>, default:                   <code>\"piecewise\"</code> )           \u2013            <p>Method to use to embed the continuous features. Options are: 'standard', 'periodic' or 'piecewise'. The 'standard' embedding method is based on the FT-Transformer implementation presented in the paper: Revisiting Deep Learning Models for Tabular Data. The 'periodic' and_'piecewise'_ methods were presented in the paper: On Embeddings for Numerical Features in Tabular Deep Learning. Please, read the papers for details.</p> </li> <li> <code>cont_embed_dim</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>Size of the continuous embeddings. If the continuous columns are embedded, <code>cont_embed_dim</code> must be passed.</p> </li> <li> <code>cont_embed_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p> </li> <li> <code>cont_embed_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the continuous embeddings if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported. If <code>None</code>, no activation function will be applied.</p> </li> <li> <code>quantization_setup</code>               (<code>Optional[Dict[str, List[float]]]</code>, default:                   <code>None</code> )           \u2013            <p>This parameter is used when the 'piecewise' method is used to embed the continuous cols. It is a dict where keys are the name of the continuous columns and values are lists with the boundaries for the quantization of the continuous_cols. See the examples for details. If If the 'piecewise' method is used, this parameter is required.</p> </li> <li> <code>n_frequencies</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>This is the so called 'k' in their paper On Embeddings for Numerical Features in Tabular Deep Learning, and is the number of 'frequencies' that will be used to represent each continuous column. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>sigma</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>This is the sigma parameter in the paper mentioned when describing the previous parameters and it is used to initialise the 'frequency weights'. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>share_last_layer</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>This parameter is not present in the before mentioned paper but it is implemented in the official repo. If <code>True</code> the linear layer that turns the frequencies into embeddings will be shared across the continuous columns. If <code>False</code> a different linear layer will be used for each continuous column. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>full_embed_dropout</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>If <code>True</code>, the full embedding corresponding to a column will be masked out/dropout. If <code>None</code>, it will default to <code>False</code>.</p> </li> <li> <code>mlp_hidden_dims</code>               (<code>Optional[List[int]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the number of neurons per dense layer in the mlp.</p> </li> <li> <code>mlp_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the dense layers of the MLP. Currently 'tanh', 'relu', 'leaky_relu', 'gelu' and 'preglu' are   supported</p> </li> <li> <code>mlp_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>float or List of floats with the dropout between the dense layers. e.g: [0.5,0.5]</p> </li> <li> <code>mlp_batchnorm</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating whether or not batch normalization will be applied to the dense layers</p> </li> <li> <code>mlp_batchnorm_last</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating whether or not batch normalization will be applied to the last of the dense layers</p> </li> <li> <code>mlp_linear_first</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code></p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>user_behavior_indexes</code>               (<code>List[int]</code>)           \u2013            <p>List with the indexes of the user behavior columns</p> </li> <li> <code>user_behavior_embed</code>               (<code>BaseTabularModelWithAttention</code>)           \u2013            <p>Embedding layer for the user</p> </li> <li> <code>action_seq_indexes</code>               (<code>List[int]</code>)           \u2013            <p>List with the indexes of the rating sequence columns if the action_seq_config parameter is not None</p> </li> <li> <code>action_embed</code>               (<code>BaseTabularModelWithAttention</code>)           \u2013            <p>Embedding layer for the rating sequence columns if the action_seq_config parameter is not None</p> </li> <li> <code>other_seq_cols_indexes</code>               (<code>Dict[str, List[int]]</code>)           \u2013            <p>Dictionary with the indexes of the other sequential columns if the other_seq_cols_confiq parameter is not None</p> </li> <li> <code>other_seq_cols_embed</code>               (<code>ModuleDict</code>)           \u2013            <p>Dictionary with the embedding layers for the other sequential columns if the other_seq_cols_confiq parameter is not None</p> </li> <li> <code>other_cols_idx</code>               (<code>List[int]</code>)           \u2013            <p>List with the indexes of the other columns if the other_cols_config parameter is not None</p> </li> <li> <code>other_col_embed</code>               (<code>BaseTabularModel</code>)           \u2013            <p>Embedding layer for the other columns if the other_cols_config parameter is not None</p> </li> <li> <code>mlp</code>               (<code>Optional[MLP]</code>)           \u2013            <p>MLP component of the model. If None, no MLP will be used. This should almost always be not None.</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; import numpy as np\n&gt;&gt;&gt; from torch import Tensor\n&gt;&gt;&gt; from typing import Dict, List, Tuple\n&gt;&gt;&gt; from pytorch_widedeep.models.rec import DeepInterestNetwork\n&gt;&gt;&gt; np_seed = np.random.seed(42)\n&gt;&gt;&gt; torch_seed = torch.manual_seed(42)\n&gt;&gt;&gt; num_users = 10\n&gt;&gt;&gt; num_items = 5\n&gt;&gt;&gt; num_contexts = 3\n&gt;&gt;&gt; seq_length = 3\n&gt;&gt;&gt; num_samples = 10\n&gt;&gt;&gt; user_ids = np.random.randint(0, num_users, num_samples)\n&gt;&gt;&gt; target_item_ids = np.random.randint(0, num_items, num_samples)\n&gt;&gt;&gt; context_ids = np.random.randint(0, num_contexts, num_samples)\n&gt;&gt;&gt; user_behavior = np.array(\n...     [\n...         np.random.choice(num_items, seq_length, replace=False)\n...         for _ in range(num_samples)\n...     ]\n... )\n&gt;&gt;&gt; X_arr = np.column_stack((user_ids, target_item_ids, context_ids, user_behavior))\n&gt;&gt;&gt; X = torch.tensor(X_arr, dtype=torch.long)\n&gt;&gt;&gt; column_idx: Dict[str, int] = {\n...     \"user_id\": 0,\n...     \"target_item\": 1,\n...     \"context\": 2,\n...     \"item_1\": 3,\n...     \"item_2\": 4,\n...     \"item_3\": 5,\n... }\n&gt;&gt;&gt; user_behavior_config: Tuple[List[str], int, int] = (\n...     [\"item_1\", \"item_2\", \"item_3\"],\n...     num_items,\n...     8,\n... )\n&gt;&gt;&gt; cat_embed_input: List[Tuple[str, int, int]] = [\n...     (\"user_id\", num_users, 8),\n...     (\"context\", num_contexts, 4),\n... ]\n&gt;&gt;&gt; model = DeepInterestNetwork(\n...     column_idx=column_idx,\n...     target_item_col=\"target_item\",\n...     user_behavior_confiq=user_behavior_config,\n...     cat_embed_input=cat_embed_input,\n...     mlp_hidden_dims=[16, 8],\n... )\n&gt;&gt;&gt; output = model(X)\n</code></pre> Source code in <code>pytorch_widedeep/models/rec/din.py</code> <pre><code>def __init__(\n    self,\n    *,\n    column_idx: Dict[str, int],\n    user_behavior_confiq: Tuple[List[str], int, int],\n    target_item_col: str = \"target_item\",\n    action_seq_config: Optional[Tuple[List[str], int]] = None,\n    other_seq_cols_confiq: Optional[List[Tuple[List[str], int, int]]] = None,\n    attention_unit_activation: Literal[\"prelu\", \"dice\"] = \"prelu\",\n    cat_embed_input: Optional[List[Tuple[str, int, int]]] = None,\n    cat_embed_dropout: Optional[float] = None,\n    use_cat_bias: Optional[bool] = None,\n    cat_embed_activation: Optional[str] = None,\n    continuous_cols: Optional[List[str]] = None,\n    cont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\n    embed_continuous: Optional[bool] = None,\n    embed_continuous_method: Optional[\n        Literal[\"standard\", \"piecewise\", \"periodic\"]\n    ] = None,\n    cont_embed_dim: Optional[int] = None,\n    cont_embed_dropout: Optional[float] = None,\n    cont_embed_activation: Optional[str] = None,\n    quantization_setup: Optional[Dict[str, List[float]]] = None,\n    n_frequencies: Optional[int] = None,\n    sigma: Optional[float] = None,\n    share_last_layer: Optional[bool] = None,\n    full_embed_dropout: Optional[bool] = None,\n    mlp_hidden_dims: Optional[List[int]] = None,\n    mlp_activation: Optional[str] = None,\n    mlp_dropout: Optional[float] = None,\n    mlp_batchnorm: Optional[bool] = None,\n    mlp_batchnorm_last: Optional[bool] = None,\n    mlp_linear_first: Optional[bool] = None,\n):\n    super(DeepInterestNetwork, self).__init__()\n\n    self.column_idx = {\n        k: v for k, v in sorted(column_idx.items(), key=lambda x: x[1])\n    }\n\n    self.column_idx = column_idx\n    self.target_item_col = target_item_col\n    self.user_behavior_confiq = user_behavior_confiq\n    self.action_seq_config = (\n        (action_seq_config[0], action_seq_config[1], 1)\n        if action_seq_config is not None\n        else None\n    )\n    self.other_seq_cols_confiq = other_seq_cols_confiq\n    self.cat_embed_input = cat_embed_input\n    self.attention_unit_activation = attention_unit_activation\n\n    self.cat_embed_dropout = cat_embed_dropout\n    self.use_cat_bias = use_cat_bias\n    self.cat_embed_activation = cat_embed_activation\n    self.continuous_cols = continuous_cols\n    self.cont_norm_layer = cont_norm_layer\n    self.embed_continuous = embed_continuous\n    self.embed_continuous_method = embed_continuous_method\n    self.cont_embed_dim = cont_embed_dim\n    self.cont_embed_dropout = cont_embed_dropout\n    self.cont_embed_activation = cont_embed_activation\n    self.quantization_setup = quantization_setup\n    self.n_frequencies = n_frequencies\n    self.sigma = sigma\n    self.share_last_layer = share_last_layer\n    self.full_embed_dropout = full_embed_dropout\n\n    self.mlp_hidden_dims = mlp_hidden_dims\n    self.mlp_activation = mlp_activation\n    self.mlp_dropout = mlp_dropout\n    self.mlp_batchnorm = mlp_batchnorm\n    self.mlp_batchnorm_last = mlp_batchnorm_last\n    self.mlp_linear_first = mlp_linear_first\n\n    self.target_item_idx = self.column_idx[target_item_col]\n\n    self.user_behavior_indexes, self.user_behavior_embed = (\n        self.set_user_behavior_indexes_and_embed(user_behavior_confiq)\n    )\n    self.user_behavior_dim = user_behavior_confiq[2]\n\n    if self.action_seq_config is not None:\n        self.action_seq_indexes, self.action_embed = (\n            self._set_rating_indexes_and_embed(self.action_seq_config)\n        )\n    else:\n        self.action_embed = None\n\n    if self.other_seq_cols_confiq is not None:\n        (\n            self.other_seq_cols_indexes,\n            self.other_seq_cols_embed,\n            self.other_seq_dim,\n        ) = self._set_other_seq_cols_indexes_embed_and_dim(\n            self.other_seq_cols_confiq\n        )\n    else:\n        self.other_seq_cols_embed = None\n        self.other_seq_dim = 0\n\n    if self.cat_embed_input is not None or self.continuous_cols is not None:\n        self.other_cols_idx, self.other_col_embed, self.other_cols_dim = (\n            self._set_other_cols_idx_embed_and_dim(\n                self.cat_embed_input, self.continuous_cols\n            )\n        )\n    else:\n        self.other_col_embed = None\n        self.other_cols_dim = 0\n\n    self.attention = ActivationUnit(\n        user_behavior_confiq[2], attention_unit_activation\n    )\n\n    if self.mlp_hidden_dims is not None:\n        mlp_input_dim = (\n            self.user_behavior_dim * 2 + self.other_seq_dim + self.other_cols_dim\n        )\n        self.mlp = MLP(\n            d_hidden=[mlp_input_dim] + self.mlp_hidden_dims,\n            activation=(\n                \"relu\" if self.mlp_activation is None else self.mlp_activation\n            ),\n            dropout=0.0 if self.mlp_dropout is None else self.mlp_dropout,\n            batchnorm=False if self.mlp_batchnorm is None else self.mlp_batchnorm,\n            batchnorm_last=(\n                False\n                if self.mlp_batchnorm_last is None\n                else self.mlp_batchnorm_last\n            ),\n            linear_first=(\n                False if self.mlp_linear_first is None else self.mlp_linear_first\n            ),\n        )\n    else:\n        self.mlp = None\n</code></pre>"},{"location":"pytorch-widedeep/the_rec_module.html#pytorch_widedeep.models.rec.basic_transformer.Transformer","title":"Transformer","text":"<pre><code>Transformer(\n    vocab_size,\n    seq_length,\n    input_dim,\n    n_heads,\n    n_blocks,\n    attn_dropout=0.1,\n    ff_dropout=0.1,\n    ff_factor=4,\n    activation=\"gelu\",\n    use_linear_attention=False,\n    use_flash_attention=False,\n    padding_idx=0,\n    with_cls_token=False,\n    *,\n    with_pos_encoding=True,\n    pos_encoding_dropout=0.1,\n    pos_encoder=None\n)\n</code></pre> <p>               Bases: <code>Module</code></p> <p>Basic Encoder-Only Transformer Model for sequence classification/regression. As all other models in the library this model can be used as the <code>deeptext</code> component of a Wide &amp; Deep model or independently by itself.</p> <p> NOTE: This model is introduced in the context of recommendation systems and thought for sequences of any nature (e.g. items). It can, of course, still be used for text.</p> <p>Parameters:</p> <ul> <li> <code>vocab_size</code>               (<code>int</code>)           \u2013            <p>Number of words in the vocabulary</p> </li> <li> <code>input_dim</code>               (<code>int</code>)           \u2013            <p>Dimension of the token embeddings</p> <p>Param aliases: <code>embed_dim</code>, <code>d_model</code>. </p> </li> <li> <code>seq_length</code>               (<code>int</code>)           \u2013            <p>Input sequence length</p> </li> <li> <code>n_heads</code>               (<code>int</code>)           \u2013            <p>Number of attention heads per Transformer block</p> </li> <li> <code>n_blocks</code>               (<code>int</code>)           \u2013            <p>Number of Transformer blocks</p> </li> <li> <code>attn_dropout</code>               (<code>float</code>, default:                   <code>0.1</code> )           \u2013            <p>Dropout that will be applied to the Multi-Head Attention layers</p> </li> <li> <code>ff_dropout</code>               (<code>float</code>, default:                   <code>0.1</code> )           \u2013            <p>Dropout that will be applied to the FeedForward network</p> </li> <li> <code>ff_factor</code>               (<code>int</code>, default:                   <code>4</code> )           \u2013            <p>Multiplicative factor applied to the first layer of the FF network in each Transformer block, This is normally set to 4.</p> </li> <li> <code>activation</code>               (<code>str</code>, default:                   <code>'gelu'</code> )           \u2013            <p>Transformer Encoder activation function. 'tanh', 'relu', 'leaky_relu', 'gelu', 'geglu' and 'reglu' are supported</p> </li> <li> <code>padding_idx</code>               (<code>int</code>, default:                   <code>0</code> )           \u2013            <p>index of the padding token in the padded-tokenised sequences.</p> </li> <li> <code>with_cls_token</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating if a <code>'[CLS]'</code> token is included in the tokenized sequences. If present, the final hidden state corresponding to this token is used as the aggregated representation for classification and regression tasks. NOTE: if included in the tokenized sequences it must be inserted as the first token in the sequences.</p> </li> <li> <code>with_pos_encoding</code>               (<code>bool</code>, default:                   <code>True</code> )           \u2013            <p>Boolean indicating if positional encoding will be used</p> </li> <li> <code>pos_encoding_dropout</code>               (<code>float</code>, default:                   <code>0.1</code> )           \u2013            <p>Positional encoding dropout</p> </li> <li> <code>pos_encoder</code>               (<code>Optional[Module]</code>, default:                   <code>None</code> )           \u2013            <p>This model uses by default a standard positional encoding approach. However, any custom positional encoder can also be used and pass to the Transformer model via the 'pos_encoder' parameter</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>embedding</code>               (<code>Module</code>)           \u2013            <p>Standard token embedding layer</p> </li> <li> <code>pos_encoder</code>               (<code>Module</code>)           \u2013            <p>Positional Encoder</p> </li> <li> <code>encoder</code>               (<code>Module</code>)           \u2013            <p>Sequence of Transformer blocks</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models.rec import Transformer\n&gt;&gt;&gt; X = torch.cat((torch.zeros([5,1]), torch.empty(5, 4).random_(1,4)), axis=1)\n&gt;&gt;&gt; model = Transformer(vocab_size=4, seq_length=5, input_dim=8, n_heads=1, n_blocks=1)\n&gt;&gt;&gt; out = model(X)\n</code></pre> Source code in <code>pytorch_widedeep/models/rec/basic_transformer.py</code> <pre><code>@alias(\"input_dim\", [\"embed_dim\", \"d_model\"])\n@alias(\"seq_length\", [\"max_length\", \"maxlen\"])\ndef __init__(\n    self,\n    vocab_size: int,\n    seq_length: int,\n    input_dim: int,\n    n_heads: int,\n    n_blocks: int,\n    attn_dropout: float = 0.1,\n    ff_dropout: float = 0.1,\n    ff_factor: int = 4,\n    activation: str = \"gelu\",\n    use_linear_attention: bool = False,\n    use_flash_attention: bool = False,\n    padding_idx: int = 0,\n    with_cls_token: bool = False,\n    *,  # from here on pos encoding args\n    with_pos_encoding: bool = True,\n    pos_encoding_dropout: float = 0.1,\n    pos_encoder: Optional[nn.Module] = None,\n):\n    super().__init__()\n\n    self.input_dim = input_dim\n    self.seq_length = seq_length\n    self.n_heads = n_heads\n    self.n_blocks = n_blocks\n    self.attn_dropout = attn_dropout\n    self.ff_dropout = ff_dropout\n    self.ff_factor = ff_factor\n    self.activation = activation\n    self.use_linear_attention = use_linear_attention\n    self.use_flash_attention = use_flash_attention\n    self.padding_idx = padding_idx\n    self.with_cls_token = with_cls_token\n    self.with_pos_encoding = with_pos_encoding\n    self.pos_encoding_dropout = pos_encoding_dropout\n\n    self.embedding = nn.Embedding(\n        vocab_size, input_dim, padding_idx=self.padding_idx\n    )\n\n    if with_pos_encoding:\n        if pos_encoder is not None:\n            self.pos_encoder: Union[nn.Module, nn.Identity, PositionalEncoding] = (\n                pos_encoder\n            )\n        else:\n            self.pos_encoder = PositionalEncoding(\n                input_dim, pos_encoding_dropout, seq_length\n            )\n    else:\n        self.pos_encoder = nn.Identity()\n\n    self.encoder = nn.Sequential()\n    for i in range(n_blocks):\n        self.encoder.add_module(\n            \"transformer_block\" + str(i),\n            TransformerEncoder(\n                input_dim,\n                n_heads,\n                False,  # use_qkv_bias\n                attn_dropout,\n                ff_dropout,\n                ff_factor,\n                activation,\n                use_linear_attention,\n                use_flash_attention,\n            ),\n        )\n</code></pre>"},{"location":"pytorch-widedeep/trainer.html","title":"Training multimodal Deep Learning Models","text":"<p>Here is the documentation for the <code>Trainer</code> class, that will do all the heavy lifting.</p> <p>Trainer is also available from <code>pytorch-widedeep</code> directly, for example, one could do:</p> <pre><code>    from pytorch-widedeep.training import Trainer\n</code></pre> <p>or also:</p> <pre><code>    from pytorch-widedeep import Trainer\n</code></pre>"},{"location":"pytorch-widedeep/trainer.html#pytorch_widedeep.training.Trainer","title":"Trainer","text":"<pre><code>Trainer(\n    model,\n    objective,\n    custom_loss_function=None,\n    optimizers=None,\n    lr_schedulers=None,\n    initializers=None,\n    transforms=None,\n    callbacks=None,\n    metrics=None,\n    eval_metrics=None,\n    verbose=1,\n    seed=1,\n    **kwargs\n)\n</code></pre> <p>               Bases: <code>BaseTrainer</code></p> <p>Class to set the of attributes that will be used during the training process.</p> <p>Parameters:</p> <ul> <li> <code>model</code>               (<code>WideDeep</code>)           \u2013            <p>An object of class <code>WideDeep</code></p> </li> <li> <code>objective</code>               (<code>str</code>)           \u2013            <p>Defines the objective, loss or cost function. </p> <p>Param aliases: <code>loss_function</code>, <code>loss_fn</code>, <code>loss</code>, <code>cost_function</code>, <code>cost_fn</code>, <code>cost</code>. </p> <p>Possible values are:</p> <ul> <li> <p><code>binary</code>, aliases: <code>logistic</code>, <code>binary_logloss</code>, <code>binary_cross_entropy</code></p> </li> <li> <p><code>binary_focal_loss</code></p> </li> <li> <p><code>multiclass</code>, aliases: <code>multi_logloss</code>, <code>cross_entropy</code>, <code>categorical_cross_entropy</code>,</p> </li> <li> <p><code>multiclass_focal_loss</code></p> </li> <li> <p><code>regression</code>, aliases: <code>mse</code>, <code>l2</code>, <code>mean_squared_error</code></p> </li> <li> <p><code>mean_absolute_error</code>, aliases: <code>mae</code>, <code>l1</code></p> </li> <li> <p><code>mean_squared_log_error</code>, aliases: <code>msle</code></p> </li> <li> <p><code>root_mean_squared_error</code>, aliases:  <code>rmse</code></p> </li> <li> <p><code>root_mean_squared_log_error</code>, aliases: <code>rmsle</code></p> </li> <li> <p><code>zero_inflated_lognormal</code>, aliases: <code>ziln</code></p> </li> <li> <p><code>quantile</code></p> </li> <li> <p><code>tweedie</code></p> </li> <li> <p><code>multitarget</code>, aliases: <code>multi_target</code></p> </li> </ul> <p>NOTE: For <code>multitarget</code> a custom loss function must be passed</p> </li> <li> <code>custom_loss_function</code>               (<code>Optional[Module]</code>, default:                   <code>None</code> )           \u2013            <p>It is possible to pass a custom loss function. See for example <code>pytorch_widedeep.losses.FocalLoss</code> for the required structure of the object or the Examples section in this documentation or in the repo. Note that if <code>custom_loss_function</code> is not <code>None</code>, <code>objective</code> must be 'binary', 'multiclass' or 'regression', consistent with the loss function</p> </li> <li> <code>optimizers</code>               (<code>Optional[Union[Optimizer, Dict[str, Union[Optimizer, List[Optimizer]]]]]</code>, default:                   <code>None</code> )           \u2013            <ul> <li>An instance of Pytorch's <code>Optimizer</code> object   (e.g. <code>torch.optim.Adam()</code>) or</li> <li>a dictionary where there keys are the model components (i.e.   'wide', 'deeptabular', 'deeptext', 'deepimage'   and/or 'deephead')  and the values are the corresponding   optimizers or list of optimizers if multiple models are used for   the given data mode (e.g. two text columns/models for the deeptext   component). If multiple optimizers are used the   dictionary MUST contain an optimizer per model component.</li> </ul> <p>if no optimizers are passed it will default to <code>Adam</code> for all model components</p> </li> <li> <code>lr_schedulers</code>               (<code>Optional[Union[LRScheduler, Dict[str, Union[LRScheduler, List[LRScheduler]]]]]</code>, default:                   <code>None</code> )           \u2013            <ul> <li>An instance of Pytorch's <code>LRScheduler</code> object (e.g   <code>torch.optim.lr_scheduler.StepLR(opt, step_size=5)</code>) or</li> <li>a dictionary where there keys are the model componenst (i.e. 'wide',   'deeptabular', 'deeptext', 'deepimage' and/or 'deephead') and the   values are the corresponding learning rate schedulers or list of     learning rate schedulers if multiple models are used for the given     data mode (e.g. two text columns/models for the deeptext component).</li> </ul> </li> <li> <code>initializers</code>               (<code>Optional[Union[Initializer, Dict[str, Union[Initializer, List[Initializer]]]]]</code>, default:                   <code>None</code> )           \u2013            <ul> <li>An instance of an <code>Initializer</code> object see <code>pytorch-widedeep.initializers</code> or</li> <li>a dictionary where there keys are the model components (i.e. 'wide',   'deeptabular', 'deeptext', 'deepimage' and/or 'deephead')   and the values are the corresponding initializers or list of     initializers if multiple models are used for the given data mode (e.g.     two text columns/models for the deeptext component).</li> </ul> </li> <li> <code>transforms</code>               (<code>Optional[List[Transforms]]</code>, default:                   <code>None</code> )           \u2013            <p>List with <code>torchvision.transforms</code> to be applied to the image component of the model (i.e. <code>deepimage</code>) See torchvision transforms.</p> </li> <li> <code>callbacks</code>               (<code>Optional[List[Callback]]</code>, default:                   <code>None</code> )           \u2013            <p>List with <code>Callback</code> objects. The three callbacks available in <code>pytorch-widedeep</code> are: <code>LRHistory</code>, <code>ModelCheckpoint</code> and <code>EarlyStopping</code>. The <code>History</code> and the <code>LRShedulerCallback</code> callbacks are used by default. This can also be a custom callback as long as the object of type <code>Callback</code>. See <code>pytorch_widedeep.callbacks.Callback</code> or the examples folder in the repo.</p> </li> <li> <code>metrics</code>               (<code>Optional[Union[List[Metric], List[Metric]]]</code>, default:                   <code>None</code> )           \u2013            <ul> <li>List of objects of type <code>Metric</code>. Metrics available are:   <code>Accuracy</code>, <code>Precision</code>, <code>Recall</code>, <code>FBetaScore</code>,   <code>F1Score</code> and <code>R2Score</code>. This can also be a custom metric as long   as it is an object of type <code>Metric</code>. See   <code>pytorch_widedeep.metrics.Metric</code> or the examples folder in the   repo</li> <li>List of objects of type <code>torchmetrics.Metric</code>. This can be any   metric from torchmetrics library   Examples.   This can also be a custom metric as long as   it is an object of type <code>Metric</code>. See   the instructions.</li> </ul> </li> <li> <code>verbose</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>Verbosity level. If set to 0 nothing will be printed during training</p> </li> <li> <code>seed</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>Random seed to be used internally for train/test split</p> </li> </ul> <p>Other Parameters:</p> <ul> <li> <code>**kwargs</code>           \u2013            <p>Other infrequently used arguments that can also be passed as kwargs are:</p> <ul> <li> <p>device: <code>str</code>     string indicating the device. One of 'cpu', 'gpu' or 'mps' if     run on a Mac with Apple silicon or AMD GPU(s)</p> </li> <li> <p>num_workers: <code>int</code>     number of workers to be used internally by the data loaders</p> </li> <li> <p>lambda_sparse: <code>float</code>     lambda sparse parameter in case the <code>deeptabular</code> component is <code>TabNet</code></p> </li> <li> <p>class_weight: <code>List[float]</code>     This is the <code>weight</code> or <code>pos_weight</code> parameter in     <code>CrossEntropyLoss</code> and <code>BCEWithLogitsLoss</code>, depending on whether</p> </li> <li>reducelronplateau_criterion: <code>str</code>     This sets the criterion that will be used by the lr scheduler to     take a step: One of 'loss' or 'metric'. The ReduceLROnPlateau     learning rate is a bit particular.</li> </ul> </li> </ul> <p>Attributes:</p> <ul> <li> <code>cyclic_lr</code>               (<code>bool</code>)           \u2013            <p>Attribute that indicates if any of the lr_schedulers is cyclic_lr (i.e. <code>CyclicLR</code> or <code>OneCycleLR</code>). See Pytorch schedulers.</p> </li> <li> <code>feature_importance</code>               (<code>dict</code>)           \u2013            <p>dict where the keys are the column names and the values are the corresponding feature importances. This attribute will only exist if the <code>deeptabular</code> component is a Tabnet model.</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from torchvision.transforms import ToTensor\n&gt;&gt;&gt;\n&gt;&gt;&gt; # wide deep imports\n&gt;&gt;&gt; from pytorch_widedeep.callbacks import EarlyStopping, LRHistory\n&gt;&gt;&gt; from pytorch_widedeep.initializers import KaimingNormal, KaimingUniform, Normal, Uniform\n&gt;&gt;&gt; from pytorch_widedeep.models import TabResnet, Vision, BasicRNN, Wide, WideDeep\n&gt;&gt;&gt; from pytorch_widedeep import Trainer\n&gt;&gt;&gt;\n&gt;&gt;&gt; embed_input = [(u, i, j) for u, i, j in zip([\"a\", \"b\", \"c\"][:4], [4] * 3, [8] * 3)]\n&gt;&gt;&gt; column_idx = {k: v for v, k in enumerate([\"a\", \"b\", \"c\"])}\n&gt;&gt;&gt; wide = Wide(10, 1)\n&gt;&gt;&gt;\n&gt;&gt;&gt; # build the model\n&gt;&gt;&gt; deeptabular = TabResnet(blocks_dims=[8, 4], column_idx=column_idx, cat_embed_input=embed_input)\n&gt;&gt;&gt; deeptext = BasicRNN(vocab_size=10, embed_dim=4, padding_idx=0)\n&gt;&gt;&gt; deepimage = Vision()\n&gt;&gt;&gt; model = WideDeep(wide=wide, deeptabular=deeptabular, deeptext=deeptext, deepimage=deepimage)\n&gt;&gt;&gt;\n&gt;&gt;&gt; # set optimizers and schedulers\n&gt;&gt;&gt; wide_opt = torch.optim.Adam(model.wide.parameters())\n&gt;&gt;&gt; deep_opt = torch.optim.AdamW(model.deeptabular.parameters())\n&gt;&gt;&gt; text_opt = torch.optim.Adam(model.deeptext.parameters())\n&gt;&gt;&gt; img_opt = torch.optim.AdamW(model.deepimage.parameters())\n&gt;&gt;&gt;\n&gt;&gt;&gt; wide_sch = torch.optim.lr_scheduler.StepLR(wide_opt, step_size=5)\n&gt;&gt;&gt; deep_sch = torch.optim.lr_scheduler.StepLR(deep_opt, step_size=3)\n&gt;&gt;&gt; text_sch = torch.optim.lr_scheduler.StepLR(text_opt, step_size=5)\n&gt;&gt;&gt; img_sch = torch.optim.lr_scheduler.StepLR(img_opt, step_size=3)\n&gt;&gt;&gt;\n&gt;&gt;&gt; optimizers = {\"wide\": wide_opt, \"deeptabular\": deep_opt, \"deeptext\": text_opt, \"deepimage\": img_opt}\n&gt;&gt;&gt; schedulers = {\"wide\": wide_sch, \"deeptabular\": deep_sch, \"deeptext\": text_sch, \"deepimage\": img_sch}\n&gt;&gt;&gt;\n&gt;&gt;&gt; # set initializers and callbacks\n&gt;&gt;&gt; initializers = {\"wide\": Uniform, \"deeptabular\": Normal, \"deeptext\": KaimingNormal, \"deepimage\": KaimingUniform}\n&gt;&gt;&gt; transforms = [ToTensor]\n&gt;&gt;&gt; callbacks = [LRHistory(n_epochs=4), EarlyStopping]\n&gt;&gt;&gt;\n&gt;&gt;&gt; # set the trainer\n&gt;&gt;&gt; trainer = Trainer(model, objective=\"regression\", initializers=initializers, optimizers=optimizers,\n... lr_schedulers=schedulers, callbacks=callbacks, transforms=transforms)\n</code></pre> Source code in <code>pytorch_widedeep/training/trainer.py</code> <pre><code>@alias(  # noqa: C901\n    \"objective\",\n    [\"loss_function\", \"loss_fn\", \"loss\", \"cost_function\", \"cost_fn\", \"cost\"],\n)\n@alias(\"metrics\", [\"train_metrics\"])\ndef __init__(\n    self,\n    model: WideDeep,\n    objective: str,\n    custom_loss_function: Optional[nn.Module] = None,\n    optimizers: Optional[\n        Union[Optimizer, Dict[str, Union[Optimizer, List[Optimizer]]]]\n    ] = None,\n    lr_schedulers: Optional[\n        Union[LRScheduler, Dict[str, Union[LRScheduler, List[LRScheduler]]]]\n    ] = None,\n    initializers: Optional[\n        Union[Initializer, Dict[str, Union[Initializer, List[Initializer]]]]\n    ] = None,\n    transforms: Optional[List[Transforms]] = None,\n    callbacks: Optional[List[Callback]] = None,\n    metrics: Optional[Union[List[Metric], List[TorchMetric]]] = None,\n    eval_metrics: Optional[Union[List[Metric], List[TorchMetric]]] = None,\n    verbose: int = 1,\n    seed: int = 1,\n    **kwargs,\n):\n    super().__init__(\n        model=model,\n        objective=objective,\n        custom_loss_function=custom_loss_function,\n        optimizers=optimizers,\n        lr_schedulers=lr_schedulers,\n        initializers=initializers,\n        transforms=transforms,\n        callbacks=callbacks,\n        metrics=metrics,\n        eval_metrics=eval_metrics,\n        verbose=verbose,\n        seed=seed,\n        **kwargs,\n    )\n</code></pre>"},{"location":"pytorch-widedeep/trainer.html#pytorch_widedeep.training.Trainer.fit","title":"fit","text":"<pre><code>fit(\n    X_wide=None,\n    X_tab=None,\n    X_text=None,\n    X_img=None,\n    X_train=None,\n    X_val=None,\n    val_split=None,\n    target=None,\n    n_epochs=1,\n    validation_freq=1,\n    batch_size=32,\n    train_dataloader=None,\n    eval_dataloader=None,\n    feature_importance_sample_size=None,\n    finetune=False,\n    **kwargs\n)\n</code></pre> <p>Fit method.</p> <p>The input datasets can be passed either directly via numpy arrays (<code>X_wide</code>, <code>X_tab</code>, <code>X_text</code> or <code>X_img</code>) or alternatively, in dictionaries (<code>X_train</code> or <code>X_val</code>).</p> <p>Parameters:</p> <ul> <li> <code>X_wide</code>               (<code>Optional[ndarray]</code>, default:                   <code>None</code> )           \u2013            <p>Input for the <code>wide</code> model component. See <code>pytorch_widedeep.preprocessing.WidePreprocessor</code></p> </li> <li> <code>X_tab</code>               (<code>Optional[Union[ndarray, List[ndarray]]]</code>, default:                   <code>None</code> )           \u2013            <p>Input for the <code>deeptabular</code> model component. See <code>pytorch_widedeep.preprocessing.TabPreprocessor</code>. If multiple tabular models are used for different columns, this should be a list of numpy arrays</p> </li> <li> <code>X_text</code>               (<code>Optional[Union[ndarray, List[ndarray]]]</code>, default:                   <code>None</code> )           \u2013            <p>Input for the <code>deeptext</code> model component. See <code>pytorch_widedeep.preprocessing.TextPreprocessor</code>. If multiple text columns/models are used, this should be a list of numpy arrays</p> </li> <li> <code>X_img</code>               (<code>Optional[Union[ndarray, List[ndarray]]]</code>, default:                   <code>None</code> )           \u2013            <p>Input for the <code>deepimage</code> model component. See <code>pytorch_widedeep.preprocessing.ImagePreprocessor</code>. If multiple image columns/models are used, this should be a list of numpy arrays</p> </li> <li> <code>X_train</code>               (<code>Optional[Dict[str, Union[ndarray, List[ndarray]]]]</code>, default:                   <code>None</code> )           \u2013            <p>The training dataset can also be passed in a dictionary. Keys are 'X_wide', 'X_tab', 'X_text', 'X_img' and 'target'. Values are the corresponding matrices. Note that of multiple text or image columns/models are used, the corresponding values should be lists of numpy arrays</p> </li> <li> <code>X_val</code>               (<code>Optional[Dict[str, Union[ndarray, List[ndarray]]]]</code>, default:                   <code>None</code> )           \u2013            <p>The validation dataset can also be passed in a dictionary. Keys are 'X_wide', 'X_tab', 'X_text', 'X_img' and 'target'. Values are the corresponding matrices. Note that of multiple text or image columns/models are used, the corresponding values should be lists of numpy arrays</p> </li> <li> <code>val_split</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>train/val split fraction</p> </li> <li> <code>target</code>               (<code>Optional[ndarray]</code>, default:                   <code>None</code> )           \u2013            <p>target values</p> </li> <li> <code>n_epochs</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>number of epochs</p> </li> <li> <code>validation_freq</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>epochs validation frequency</p> </li> <li> <code>batch_size</code>               (<code>int</code>, default:                   <code>32</code> )           \u2013            <p>batch size</p> </li> <li> <code>custom_dataloader</code>           \u2013            <p>object of class <code>torch.utils.data.DataLoader</code>. Available predefined dataloaders are in <code>pytorch-widedeep.dataloaders</code>.If <code>None</code>, a standard torch <code>DataLoader</code> is used.</p> </li> <li> <code>finetune</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>fine-tune individual model components. This functionality can also be used to 'warm-up' (and hence the alias <code>warmup</code>) individual components before the joined training starts, and hence its alias. See the Examples folder in the repo for more details</p> <p><code>pytorch_widedeep</code> implements 3 fine-tune routines.</p> <ul> <li>fine-tune all trainable layers at once. This routine is   inspired by the work of Howard &amp; Sebastian Ruder 2018 in their   ULMfit paper. Using a   Slanted Triangular learing (see   Leslie N. Smith paper ) ,   the process is the following: i) the learning rate will   gradually increase for 10% of the training steps from max_lr/10   to max_lr. ii) It will then gradually decrease to max_lr/10   for the remaining 90% of the steps. The optimizer used in the   process is <code>Adam</code>.</li> </ul> <p>and two gradual fine-tune routines, where only certain layers are trained at a time.</p> <ul> <li>The so called <code>Felbo</code> gradual fine-tune rourine, based on the the   Felbo et al., 2017 DeepEmoji paper.</li> <li>The <code>Howard</code> routine based on the work of Howard &amp; Sebastian Ruder 2018 in their   ULMfit paper.</li> </ul> <p>For details on how these routines work, please see the Examples section in this documentation and the Examples folder in the repo.  Param Alias: <code>warmup</code></p> </li> </ul> <p>Other Parameters:</p> <ul> <li> <code>**kwargs</code>           \u2013            <p>Other keyword arguments are:</p> <ul> <li> <p>DataLoader related parameters:     For example,  <code>sampler</code>, <code>batch_sampler</code>, <code>collate_fn</code>, etc.     Please, see the pytorch     DataLoader docs     for details.</p> </li> <li> <p>Finetune related parameters:     see the source code at <code>pytorch_widedeep._finetune</code>. Namely, these are:</p> <ul> <li><code>finetune_epochs</code> (<code>int</code>):     number of epochs use for fine tuning</li> <li><code>finetune_max_lr</code> (<code>float</code>):    max lr during fine tuning</li> <li><code>routine</code> (<code>str</code>):    one of 'howard' or 'felbo'</li> <li><code>deeptabular_gradual</code> (<code>bool</code>):    boolean indicating if the <code>deeptabular</code> component will be fine tuned gradually</li> <li><code>deeptabular_layers</code> (<code>Optional[Union[List[nn.Module], List[List[nn.Module]]]]</code>):    List of pytorch modules indicating the layers of the    <code>deeptabular</code> that will be fine tuned</li> <li><code>deeptabular_max_lr</code> (<code>Union[float, List[float]]</code>):    max lr for the <code>deeptabular</code> componet during fine tuning</li> <li><code>deeptext_gradual</code> (<code>bool</code>):    same as <code>deeptabular_gradual</code> but for the <code>deeptext</code> component</li> <li><code>deeptext_layers</code> (<code>Optional[Union[List[nn.Module], List[List[nn.Module]]]]</code>):    same as <code>deeptabular_gradual</code> but for the <code>deeptext</code> component.    If there are multiple text columns/models, this should be a list of lists</li> <li><code>deeptext_max_lr</code> (<code>Union[float, List[float]]</code>):    same as <code>deeptabular_gradual</code> but for the <code>deeptext</code> component    If there are multiple text columns/models, this should be a list of floats</li> <li><code>deepimage_gradual</code> (<code>bool</code>):    same as <code>deeptext_layers</code> but for the <code>deepimage</code> component</li> <li><code>deepimage_layers</code> (<code>Optional[Union[List[nn.Module], List[List[nn.Module]]]]</code>):    same as <code>deeptext_layers</code> but for the <code>deepimage</code> component</li> <li><code>deepimage_max_lr</code> (<code>Union[float, List[float]]</code>):     same as <code>deeptext_layers</code> but for the <code>deepimage</code> component</li> </ul> </li> </ul> </li> </ul> <p>Examples:</p> <p>For a series of comprehensive examples on how to use the <code>fit</code> method, please see the Examples folder in the repo</p> Source code in <code>pytorch_widedeep/training/trainer.py</code> <pre><code>@alias(\"finetune\", [\"warmup\"])\n@alias(\"train_dataloader\", [\"custom_dataloader\"])\n@alias(\"eval_dataloader\", [\"custom_eval_dataloader\"])\ndef fit(  # noqa: C901\n    self,\n    X_wide: Optional[np.ndarray] = None,\n    X_tab: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n    X_text: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n    X_img: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n    X_train: Optional[Dict[str, Union[np.ndarray, List[np.ndarray]]]] = None,\n    X_val: Optional[Dict[str, Union[np.ndarray, List[np.ndarray]]]] = None,\n    val_split: Optional[float] = None,\n    target: Optional[np.ndarray] = None,\n    n_epochs: int = 1,\n    validation_freq: int = 1,\n    batch_size: int = 32,\n    train_dataloader: Optional[CustomDataLoader] = None,\n    eval_dataloader: Optional[CustomDataLoader] = None,\n    feature_importance_sample_size: Optional[int] = None,\n    finetune: bool = False,\n    **kwargs,\n):\n    r\"\"\"Fit method.\n\n    The input datasets can be passed either directly via numpy arrays\n    (`X_wide`, `X_tab`, `X_text` or `X_img`) or alternatively, in\n    dictionaries (`X_train` or `X_val`).\n\n    Parameters\n    ----------\n    X_wide: np.ndarray, Optional. default=None\n        Input for the `wide` model component.\n        See `pytorch_widedeep.preprocessing.WidePreprocessor`\n    X_tab: np.ndarray, Optional. default=None\n        Input for the `deeptabular` model component.\n        See `pytorch_widedeep.preprocessing.TabPreprocessor`. If multiple\n        tabular models are used for different columns, this should be a\n        list of numpy arrays\n    X_text: Union[np.ndarray, List[np.ndarray]], Optional. default=None\n        Input for the `deeptext` model component.\n        See `pytorch_widedeep.preprocessing.TextPreprocessor`.\n        If multiple text columns/models are used, this should be a list of\n        numpy arrays\n    X_img: np.ndarray, Optional. default=None\n        Input for the `deepimage` model component.\n        See `pytorch_widedeep.preprocessing.ImagePreprocessor`.\n        If multiple image columns/models are used, this should be a list of\n        numpy arrays\n    X_train: Dict, Optional. default=None\n        The training dataset can also be passed in a dictionary. Keys are\n        _'X_wide'_, _'X_tab'_, _'X_text'_, _'X_img'_ and _'target'_. Values\n        are the corresponding matrices. Note that of multiple text or image\n        columns/models are used, the corresponding values should be lists\n        of numpy arrays\n    X_val: Dict, Optional. default=None\n        The validation dataset can also be passed in a dictionary. Keys\n        are _'X_wide'_, _'X_tab'_, _'X_text'_, _'X_img'_ and _'target'_.\n        Values are the corresponding matrices. Note that of multiple text\n        or image columns/models are used, the corresponding values should\n        be lists of numpy arrays\n    val_split: float, Optional. default=None\n        train/val split fraction\n    target: np.ndarray, Optional. default=None\n        target values\n    n_epochs: int, default=1\n        number of epochs\n    validation_freq: int, default=1\n        epochs validation frequency\n    batch_size: int, default=32\n        batch size\n    custom_dataloader: `DataLoader`, Optional, default=None\n        object of class `torch.utils.data.DataLoader`. Available\n        predefined dataloaders are in `pytorch-widedeep.dataloaders`.If\n        `None`, a standard torch `DataLoader` is used.\n    finetune: bool, default=False\n        fine-tune individual model components. This functionality can also\n        be used to 'warm-up' (and hence the alias `warmup`) individual\n        components before the joined training starts, and hence its\n        alias. See the Examples folder in the repo for more details\n\n        `pytorch_widedeep` implements 3 fine-tune routines.\n\n        - fine-tune all trainable layers at once. This routine is\n          inspired by the work of Howard &amp; Sebastian Ruder 2018 in their\n          [ULMfit paper](https://arxiv.org/abs/1801.06146). Using a\n          Slanted Triangular learing (see\n          [Leslie N. Smith paper](https://arxiv.org/pdf/1506.01186.pdf) ) ,\n          the process is the following: *i*) the learning rate will\n          gradually increase for 10% of the training steps from max_lr/10\n          to max_lr. *ii*) It will then gradually decrease to max_lr/10\n          for the remaining 90% of the steps. The optimizer used in the\n          process is `Adam`.\n\n        and two gradual fine-tune routines, where only certain layers are\n        trained at a time.\n\n        - The so called `Felbo` gradual fine-tune rourine, based on the the\n          Felbo et al., 2017 [DeepEmoji paper](https://arxiv.org/abs/1708.00524).\n        - The `Howard` routine based on the work of Howard &amp; Sebastian Ruder 2018 in their\n          [ULMfit paper](https://arxiv.org/abs/1801.06146&gt;).\n\n        For details on how these routines work, please see the Examples\n        section in this documentation and the Examples folder in the repo. &lt;br/&gt;\n        Param Alias: `warmup`\n\n    Other Parameters\n    ----------------\n    **kwargs:\n        Other keyword arguments are:\n\n        - **DataLoader related parameters**:&lt;br/&gt;\n            For example,  `sampler`, `batch_sampler`, `collate_fn`, etc.\n            Please, see the pytorch\n            [DataLoader docs](https://pytorch.org/docs/stable/data.html#torch.utils.data.DataLoader)\n            for details.\n\n        - **Finetune related parameters**:&lt;br/&gt;\n            see the source code at `pytorch_widedeep._finetune`. Namely, these are:\n\n            - `finetune_epochs` (`int`):\n                number of epochs use for fine tuning\n            - `finetune_max_lr` (`float`):\n               max lr during fine tuning\n            - `routine` (`str`):\n               one of _'howard'_ or _'felbo'_\n            - `deeptabular_gradual` (`bool`):\n               boolean indicating if the `deeptabular` component will be fine tuned gradually\n            - `deeptabular_layers` (`Optional[Union[List[nn.Module], List[List[nn.Module]]]]`):\n               List of pytorch modules indicating the layers of the\n               `deeptabular` that will be fine tuned\n            - `deeptabular_max_lr` (`Union[float, List[float]]`):\n               max lr for the `deeptabular` componet during fine tuning\n            - `deeptext_gradual` (`bool`):\n               same as `deeptabular_gradual` but for the `deeptext` component\n            - `deeptext_layers` (`Optional[Union[List[nn.Module], List[List[nn.Module]]]]`):\n               same as `deeptabular_gradual` but for the `deeptext` component.\n               If there are multiple text columns/models, this should be a list of lists\n            - `deeptext_max_lr` (`Union[float, List[float]]`):\n               same as `deeptabular_gradual` but for the `deeptext` component\n               If there are multiple text columns/models, this should be a list of floats\n            - `deepimage_gradual` (`bool`):\n               same as `deeptext_layers` but for the `deepimage` component\n            - `deepimage_layers` (`Optional[Union[List[nn.Module], List[List[nn.Module]]]]`):\n               same as `deeptext_layers` but for the `deepimage` component\n            - `deepimage_max_lr` (`Union[float, List[float]]`):\n                same as `deeptext_layers` but for the `deepimage` component\n\n    Examples\n    --------\n\n    For a series of comprehensive examples on how to use the `fit` method, please see the\n    [Examples](https://github.com/jrzaurin/pytorch-widedeep/tree/master/examples)\n    folder in the repo\n    \"\"\"\n    dataloader_args, finetune_args = self._extract_kwargs(kwargs)\n    self.batch_size = batch_size\n    train_set, eval_set = wd_train_val_split(\n        self.seed,\n        self.method,  # type: ignore\n        X_wide,\n        X_tab,\n        X_text,\n        X_img,\n        X_train,\n        X_val,\n        val_split,\n        target,\n        self.transforms,\n    )\n    train_loader = self._set_dataloader(\n        train_dataloader, train_set, batch_size, dataloader_args\n    )\n    eval_loader = (\n        self._set_dataloader(eval_dataloader, eval_set, batch_size, dataloader_args)\n        if eval_set is not None\n        else None\n    )\n\n    if finetune:\n        self.with_finetuning: bool = True\n        self._do_finetune(train_loader, **finetune_args)\n        if self.verbose:\n            print(\n                \"Fine-tuning (or warmup) of individual components completed. \"\n                \"Training the whole model for {} epochs\".format(n_epochs)\n            )\n    else:\n        self.with_finetuning = False\n\n    self.callback_container.on_train_begin(\n        {\n            \"batch_size\": batch_size,\n            \"train_steps\": len(train_loader),\n            \"n_epochs\": n_epochs,\n        }\n    )\n    for epoch in range(n_epochs):\n        epoch_logs = self._train_epoch(train_loader, epoch)\n        if eval_loader is not None and epoch % validation_freq == (\n            validation_freq - 1\n        ):\n            epoch_logs, on_epoch_end_metric = self._eval_epoch(\n                eval_loader, epoch_logs\n            )\n        else:\n            on_epoch_end_metric = None\n            if self.reducelronplateau:\n                raise NotImplementedError(\n                    \"ReduceLROnPlateau scheduler can be used only with validation data.\"\n                )\n        self.callback_container.on_epoch_end(epoch, epoch_logs, on_epoch_end_metric)\n        if self.early_stop:\n            # self.callback_container.on_train_end(epoch_logs)\n            break\n\n    self.callback_container.on_train_end(epoch_logs)\n    if feature_importance_sample_size is not None:\n        self.feature_importance = FeatureImportance(\n            self.device, feature_importance_sample_size\n        ).feature_importance(train_loader, self.model)\n    self._restore_best_weights()\n    self.model.train()\n</code></pre>"},{"location":"pytorch-widedeep/trainer.html#pytorch_widedeep.training.Trainer.predict","title":"predict","text":"<pre><code>predict(\n    X_wide=None,\n    X_tab=None,\n    X_text=None,\n    X_img=None,\n    X_test=None,\n    batch_size=None,\n)\n</code></pre> <p>Returns the predictions</p> <p>The input datasets can be passed either directly via numpy arrays (<code>X_wide</code>, <code>X_tab</code>, <code>X_text</code> or <code>X_img</code>) or alternatively, in a dictionary (<code>X_test</code>)</p> <p>Parameters:</p> <ul> <li> <code>X_wide</code>               (<code>Optional[ndarray]</code>, default:                   <code>None</code> )           \u2013            <p>Input for the <code>wide</code> model component. See <code>pytorch_widedeep.preprocessing.WidePreprocessor</code></p> </li> <li> <code>X_tab</code>               (<code>Optional[Union[ndarray, List[ndarray]]]</code>, default:                   <code>None</code> )           \u2013            <p>Input for the <code>deeptabular</code> model component. See <code>pytorch_widedeep.preprocessing.TabPreprocessor</code></p> </li> <li> <code>X_text</code>               (<code>Optional[Union[ndarray, List[ndarray]]]</code>, default:                   <code>None</code> )           \u2013            <p>Input for the <code>deeptext</code> model component. See <code>pytorch_widedeep.preprocessing.TextPreprocessor</code></p> </li> <li> <code>X_img</code>               (<code>Optional[Union[ndarray, List[ndarray]]]</code>, default:                   <code>None</code> )           \u2013            <p>Input for the <code>deepimage</code> model component. See <code>pytorch_widedeep.preprocessing.ImagePreprocessor</code></p> </li> <li> <code>X_test</code>               (<code>Optional[Dict[str, Union[ndarray, List[ndarray]]]]</code>, default:                   <code>None</code> )           \u2013            <p>The test dataset can also be passed in a dictionary. Keys are <code>X_wide</code>, 'X_tab', 'X_text', 'X_img' and 'target'. Values are the corresponding matrices.</p> </li> <li> <code>batch_size</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>If a trainer is used to predict after having trained a model, the <code>batch_size</code> needs to be defined as it will not be defined as the <code>Trainer</code> is instantiated</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>np.ndarray:</code>           \u2013            <p>array with the predictions</p> </li> </ul> Source code in <code>pytorch_widedeep/training/trainer.py</code> <pre><code>def predict(  # type: ignore[override, return]\n    self,\n    X_wide: Optional[np.ndarray] = None,\n    X_tab: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n    X_text: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n    X_img: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n    X_test: Optional[Dict[str, Union[np.ndarray, List[np.ndarray]]]] = None,\n    batch_size: Optional[int] = None,\n) -&gt; np.ndarray:\n    r\"\"\"Returns the predictions\n\n    The input datasets can be passed either directly via numpy arrays\n    (`X_wide`, `X_tab`, `X_text` or `X_img`) or alternatively, in\n    a dictionary (`X_test`)\n\n\n    Parameters\n    ----------\n    X_wide: np.ndarray, Optional. default=None\n        Input for the `wide` model component.\n        See `pytorch_widedeep.preprocessing.WidePreprocessor`\n    X_tab: np.ndarray or List[np.ndarray], Optional. default=None\n        Input for the `deeptabular` model component.\n        See `pytorch_widedeep.preprocessing.TabPreprocessor`\n    X_text: np.ndarray or List[np.ndarray], Optional. default=None\n        Input for the `deeptext` model component.\n        See `pytorch_widedeep.preprocessing.TextPreprocessor`\n    X_img: np.ndarray or List[np.ndarray], Optional. default=None\n        Input for the `deepimage` model component.\n        See `pytorch_widedeep.preprocessing.ImagePreprocessor`\n    X_test: Dict, Optional. default=None\n        The test dataset can also be passed in a dictionary. Keys are\n        `X_wide`, _'X_tab'_, _'X_text'_, _'X_img'_ and _'target'_. Values\n        are the corresponding matrices.\n    batch_size: int, default = 256\n        If a trainer is used to predict after having trained a model, the\n        `batch_size` needs to be defined as it will not be defined as\n        the `Trainer` is instantiated\n\n    Returns\n    -------\n    np.ndarray:\n        array with the predictions\n    \"\"\"\n    preds_l = self._predict(X_wide, X_tab, X_text, X_img, X_test, batch_size)\n    if self.method == \"regression\":\n        return np.vstack(preds_l).squeeze(1)\n    if self.method == \"binary\":\n        preds = np.vstack(preds_l).squeeze(1)\n        return (preds &gt; 0.5).astype(\"int\")\n    if self.method == \"qregression\":\n        return np.vstack(preds_l)\n    if self.method == \"multiclass\":\n        preds = np.vstack(preds_l)\n        return np.argmax(preds, 1)  # type: ignore[return-value]\n    if self.method == \"multitarget\":\n        if self.loss_fn.__class__.__name__ in [\n            \"MultiTargetClassificationLoss\",\n            \"MutilTargetRegressionAndClassificationLoss\",\n        ]:\n            raise ValueError(\n                \"MultiTargetClassificationLoss and MutilTargetRegressionAndClassificationLoss \"\n                \"are not supported by predict method. Please use predict_proba method instead.\"\n            )\n        return np.vstack(preds_l)\n</code></pre>"},{"location":"pytorch-widedeep/trainer.html#pytorch_widedeep.training.Trainer.predict_uncertainty","title":"predict_uncertainty","text":"<pre><code>predict_uncertainty(\n    X_wide=None,\n    X_tab=None,\n    X_text=None,\n    X_img=None,\n    X_test=None,\n    batch_size=None,\n    uncertainty_granularity=1000,\n)\n</code></pre> <p>Returns the predicted ucnertainty of the model for the test dataset using a Monte Carlo method during which dropout layers are activated in the evaluation/prediction phase and each sample is predicted N times (<code>uncertainty_granularity</code> times).</p> <p>This is based on Dropout as a Bayesian Approximation: Representing Model Uncertainty in Deep Learning.</p> <p>Parameters:</p> <ul> <li> <code>X_wide</code>               (<code>Optional[ndarray]</code>, default:                   <code>None</code> )           \u2013            <p>Input for the <code>wide</code> model component. See <code>pytorch_widedeep.preprocessing.WidePreprocessor</code></p> </li> <li> <code>X_tab</code>               (<code>Optional[Union[ndarray, List[ndarray]]]</code>, default:                   <code>None</code> )           \u2013            <p>Input for the <code>deeptabular</code> model component. See <code>pytorch_widedeep.preprocessing.TabPreprocessor</code></p> </li> <li> <code>X_text</code>               (<code>Optional[Union[ndarray, List[ndarray]]]</code>, default:                   <code>None</code> )           \u2013            <p>Input for the <code>deeptext</code> model component. See <code>pytorch_widedeep.preprocessing.TextPreprocessor</code></p> </li> <li> <code>X_img</code>               (<code>Optional[Union[ndarray, List[ndarray]]]</code>, default:                   <code>None</code> )           \u2013            <p>Input for the <code>deepimage</code> model component. See <code>pytorch_widedeep.preprocessing.ImagePreprocessor</code></p> </li> <li> <code>X_test</code>               (<code>Optional[Dict[str, Union[ndarray, List[ndarray]]]]</code>, default:                   <code>None</code> )           \u2013            <p>The test dataset can also be passed in a dictionary. Keys are 'X_wide', 'X_tab', 'X_text', 'X_img' and 'target'. Values are the corresponding matrices.</p> </li> <li> <code>batch_size</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>If a trainer is used to predict after having trained a model, the <code>batch_size</code> needs to be defined as it will not be defined as the <code>Trainer</code> is instantiated</p> </li> <li> <code>uncertainty_granularity</code>               (<code>int</code>, default:                   <code>1000</code> )           \u2013            <p>number of times the model does prediction for each sample</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>np.ndarray:</code>           \u2013            <ul> <li>if <code>method = regression</code>, it will return an array with <code>(max, min, mean, stdev)</code>   values for each sample.</li> <li>if <code>method = binary</code> it will return an array with   <code>(mean_cls_0_prob, mean_cls_1_prob, predicted_cls)</code> for each sample.</li> <li>if <code>method = multiclass</code> it will return an array with   <code>(mean_cls_0_prob, mean_cls_1_prob, mean_cls_2_prob, ... , predicted_cls)</code>   values for each sample.</li> </ul> </li> </ul> Source code in <code>pytorch_widedeep/training/trainer.py</code> <pre><code>def predict_uncertainty(  # type: ignore[return]\n    self,\n    X_wide: Optional[np.ndarray] = None,\n    X_tab: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n    X_text: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n    X_img: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n    X_test: Optional[Dict[str, Union[np.ndarray, List[np.ndarray]]]] = None,\n    batch_size: Optional[int] = None,\n    uncertainty_granularity: int = 1000,\n) -&gt; np.ndarray:\n    r\"\"\"Returns the predicted ucnertainty of the model for the test dataset\n    using a Monte Carlo method during which dropout layers are activated\n    in the evaluation/prediction phase and each sample is predicted N\n    times (`uncertainty_granularity` times).\n\n    This is based on\n    [Dropout as a Bayesian Approximation: Representing\n    Model Uncertainty in Deep Learning](https://arxiv.org/abs/1506.02142?context=stat).\n\n    Parameters\n    ----------\n    X_wide: np.ndarray, Optional. default=None\n        Input for the `wide` model component.\n        See `pytorch_widedeep.preprocessing.WidePreprocessor`\n    X_tab: np.ndarray or List[np.ndarray], Optional. default=None\n        Input for the `deeptabular` model component.\n        See `pytorch_widedeep.preprocessing.TabPreprocessor`\n    X_text: np.ndarray or List[np.ndarray], Optional. default=None\n        Input for the `deeptext` model component.\n        See `pytorch_widedeep.preprocessing.TextPreprocessor`\n    X_img: np.ndarray or List[np.ndarray], Optional. default=None\n        Input for the `deepimage` model component.\n        See `pytorch_widedeep.preprocessing.ImagePreprocessor`\n    X_test: Dict, Optional. default=None\n        The test dataset can also be passed in a dictionary. Keys are\n        _'X_wide'_, _'X_tab'_, _'X_text'_, _'X_img'_ and _'target'_. Values\n        are the corresponding matrices.\n    batch_size: int, default = 256\n        If a trainer is used to predict after having trained a model, the\n        `batch_size` needs to be defined as it will not be defined as\n        the `Trainer` is instantiated\n    uncertainty_granularity: int default = 1000\n        number of times the model does prediction for each sample\n\n    Returns\n    -------\n    np.ndarray:\n        - if `method = regression`, it will return an array with `(max, min, mean, stdev)`\n          values for each sample.\n        - if `method = binary` it will return an array with\n          `(mean_cls_0_prob, mean_cls_1_prob, predicted_cls)` for each sample.\n        - if `method = multiclass` it will return an array with\n          `(mean_cls_0_prob, mean_cls_1_prob, mean_cls_2_prob, ... , predicted_cls)`\n          values for each sample.\n\n    \"\"\"\n    preds_l = self._predict(\n        X_wide,\n        X_tab,\n        X_text,\n        X_img,\n        X_test,\n        batch_size,\n        uncertainty_granularity,\n        uncertainty=True,\n    )\n    preds = np.vstack(preds_l)\n    samples_num = int(preds.shape[0] / uncertainty_granularity)\n    if self.method == \"regression\":\n        preds = preds.squeeze(1)\n        preds = preds.reshape((uncertainty_granularity, samples_num))\n        return np.array(\n            (\n                preds.max(axis=0),\n                preds.min(axis=0),\n                preds.mean(axis=0),\n                preds.std(axis=0),\n            )\n        ).T\n    if self.method == \"qregression\":\n        raise ValueError(\n            \"Currently predict_uncertainty is not supported for qregression method\"\n        )\n    if self.method == \"binary\":\n        preds = preds.squeeze(1)\n        preds = preds.reshape((uncertainty_granularity, samples_num))\n        preds = preds.mean(axis=0)\n        probs = np.zeros([preds.shape[0], 3])\n        probs[:, 0] = 1 - preds\n        probs[:, 1] = preds\n        return probs\n    if self.method == \"multiclass\":\n        preds = preds.reshape(uncertainty_granularity, samples_num, preds.shape[1])\n        preds = preds.mean(axis=0)\n        preds = np.hstack((preds, np.vstack(np.argmax(preds, 1))))\n        return preds\n\n    if self.method == \"multitarget\":\n        raise ValueError(\n            \"Currently predict_uncertainty is not supported for multitarget method\"\n        )\n</code></pre>"},{"location":"pytorch-widedeep/trainer.html#pytorch_widedeep.training.Trainer.predict_proba","title":"predict_proba","text":"<pre><code>predict_proba(\n    X_wide=None,\n    X_tab=None,\n    X_text=None,\n    X_img=None,\n    X_test=None,\n    batch_size=None,\n)\n</code></pre> <p>Returns the predicted probabilities for the test dataset for  binary and multiclass methods</p> <p>The input datasets can be passed either directly via numpy arrays (<code>X_wide</code>, <code>X_tab</code>, <code>X_text</code> or <code>X_img</code>) or alternatively, in a dictionary (<code>X_test</code>)</p> <p>Parameters:</p> <ul> <li> <code>X_wide</code>               (<code>Optional[ndarray]</code>, default:                   <code>None</code> )           \u2013            <p>Input for the <code>wide</code> model component. See <code>pytorch_widedeep.preprocessing.WidePreprocessor</code></p> </li> <li> <code>X_tab</code>               (<code>Optional[Union[ndarray, List[ndarray]]]</code>, default:                   <code>None</code> )           \u2013            <p>Input for the <code>deeptabular</code> model component. See <code>pytorch_widedeep.preprocessing.TabPreprocessor</code></p> </li> <li> <code>X_text</code>               (<code>Optional[Union[ndarray, List[ndarray]]]</code>, default:                   <code>None</code> )           \u2013            <p>Input for the <code>deeptext</code> model component. See <code>pytorch_widedeep.preprocessing.TextPreprocessor</code></p> </li> <li> <code>X_img</code>               (<code>Optional[Union[ndarray, List[ndarray]]]</code>, default:                   <code>None</code> )           \u2013            <p>Input for the <code>deepimage</code> model component. See <code>pytorch_widedeep.preprocessing.ImagePreprocessor</code></p> </li> <li> <code>X_test</code>               (<code>Optional[Dict[str, Union[ndarray, List[ndarray]]]]</code>, default:                   <code>None</code> )           \u2013            <p>The test dataset can also be passed in a dictionary. Keys are <code>X_wide</code>, 'X_tab', 'X_text', 'X_img' and 'target'. Values are the corresponding matrices.</p> </li> <li> <code>batch_size</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>If a trainer is used to predict after having trained a model, the <code>batch_size</code> needs to be defined as it will not be defined as the <code>Trainer</code> is instantiated</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>ndarray</code>           \u2013            <p>array with the probabilities per class</p> </li> </ul> Source code in <code>pytorch_widedeep/training/trainer.py</code> <pre><code>def predict_proba(  # type: ignore[override, return]  # noqa: C901\n    self,\n    X_wide: Optional[np.ndarray] = None,\n    X_tab: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n    X_text: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n    X_img: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n    X_test: Optional[Dict[str, Union[np.ndarray, List[np.ndarray]]]] = None,\n    batch_size: Optional[int] = None,\n) -&gt; np.ndarray:\n    r\"\"\"Returns the predicted probabilities for the test dataset for  binary\n    and multiclass methods\n\n    The input datasets can be passed either directly via numpy arrays\n    (`X_wide`, `X_tab`, `X_text` or `X_img`) or alternatively, in\n    a dictionary (`X_test`)\n\n    Parameters\n    ----------\n    X_wide: np.ndarray, Optional. default=None\n        Input for the `wide` model component.\n        See `pytorch_widedeep.preprocessing.WidePreprocessor`\n    X_tab: np.ndarray or List[np.ndarray], Optional. default=None\n        Input for the `deeptabular` model component.\n        See `pytorch_widedeep.preprocessing.TabPreprocessor`\n    X_text: np.ndarray or List[np.ndarray], Optional. default=None\n        Input for the `deeptext` model component.\n        See `pytorch_widedeep.preprocessing.TextPreprocessor`\n    X_img: np.ndarray or List[np.ndarray], Optional. default=None\n        Input for the `deepimage` model component.\n        See `pytorch_widedeep.preprocessing.ImagePreprocessor`\n    X_test: Dict, Optional. default=None\n        The test dataset can also be passed in a dictionary. Keys are\n        `X_wide`, _'X_tab'_, _'X_text'_, _'X_img'_ and _'target'_. Values\n        are the corresponding matrices.\n    batch_size: int, default = 256\n        If a trainer is used to predict after having trained a model, the\n        `batch_size` needs to be defined as it will not be defined as\n        the `Trainer` is instantiated\n\n    Returns\n    -------\n    np.ndarray\n        array with the probabilities per class\n    \"\"\"\n\n    preds_l = self._predict(X_wide, X_tab, X_text, X_img, X_test, batch_size)\n    if self.method == \"binary\":\n        preds = np.vstack(preds_l).squeeze(1)\n        probs = np.zeros([preds.shape[0], 2])\n        probs[:, 0] = 1 - preds\n        probs[:, 1] = preds\n        return probs\n    if self.method == \"multiclass\":\n        return np.vstack(preds_l)\n    if self.method == \"multitarget\":\n        return np.vstack(preds_l)\n</code></pre>"},{"location":"pytorch-widedeep/trainer.html#pytorch_widedeep.training.Trainer.save","title":"save","text":"<pre><code>save(\n    path,\n    save_state_dict=False,\n    save_optimizer=False,\n    model_filename=\"wd_model.pt\",\n)\n</code></pre> <p>Saves the model, training and evaluation history, and the <code>feature_importance</code> attribute (if the <code>deeptabular</code> component is a Tabnet model) to disk</p> <p>The <code>Trainer</code> class is built so that it 'just' trains a model. With that in mind, all the torch related parameters (such as optimizers, learning rate schedulers, initializers, etc) have to be defined externally and then passed to the <code>Trainer</code>. As a result, the <code>Trainer</code> does not generate any attribute or additional data products that need to be saved other than the <code>model</code> object itself, which can be saved as any other torch model (e.g. <code>torch.save(model, path)</code>).</p> <p>The exception is Tabnet. If the <code>deeptabular</code> component is a Tabnet model, an attribute (a dict) called <code>feature_importance</code> will be created at the end of the training process. Therefore, a <code>save</code> method was created that will save the feature importance dictionary to a json file and, since we are here, the model weights, training history and learning rate history.</p> <p>Parameters:</p> <ul> <li> <code>path</code>               (<code>str</code>)           \u2013            <p>path to the directory where the model and the feature importance attribute will be saved.</p> </li> <li> <code>save_state_dict</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether to save directly the model (and optimizer) or the model's (and optimizer's) state dictionary</p> </li> <li> <code>save_optimizer</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether to save the optimizer</p> </li> <li> <code>model_filename</code>               (<code>str</code>, default:                   <code>'wd_model.pt'</code> )           \u2013            <p>filename where the model weights will be store</p> </li> </ul> Source code in <code>pytorch_widedeep/training/trainer.py</code> <pre><code>def save(\n    self,\n    path: str,\n    save_state_dict: bool = False,\n    save_optimizer: bool = False,\n    model_filename: str = \"wd_model.pt\",\n):\n    r\"\"\"Saves the model, training and evaluation history, and the\n    `feature_importance` attribute (if the `deeptabular` component is a\n    Tabnet model) to disk\n\n    The `Trainer` class is built so that it 'just' trains a model. With\n    that in mind, all the torch related parameters (such as optimizers,\n    learning rate schedulers, initializers, etc) have to be defined\n    externally and then passed to the `Trainer`. As a result, the\n    `Trainer` does not generate any attribute or additional data\n    products that need to be saved other than the `model` object itself,\n    which can be saved as any other torch model (e.g. `torch.save(model,\n    path)`).\n\n    The exception is Tabnet. If the `deeptabular` component is a Tabnet\n    model, an attribute (a dict) called `feature_importance` will be\n    created at the end of the training process. Therefore, a `save`\n    method was created that will save the feature importance dictionary\n    to a json file and, since we are here, the model weights, training\n    history and learning rate history.\n\n    Parameters\n    ----------\n    path: str\n        path to the directory where the model and the feature importance\n        attribute will be saved.\n    save_state_dict: bool, default = False\n        Boolean indicating whether to save directly the model\n        (and optimizer) or the model's (and optimizer's) state\n        dictionary\n    save_optimizer: bool, default = False\n        Boolean indicating whether to save the optimizer\n    model_filename: str, Optional, default = \"wd_model.pt\"\n        filename where the model weights will be store\n    \"\"\"\n\n    self._save_history(path)\n\n    self._save_model_and_optimizer(\n        path, save_state_dict, save_optimizer, model_filename\n    )\n\n    if self.model.is_tabnet:\n        with open(Path(path) / \"feature_importance.json\", \"w\") as fi:\n            json.dump(self.feature_importance, fi)\n</code></pre>"},{"location":"pytorch-widedeep/utils/index.html","title":"The <code>utils</code> module","text":"<p>These are a series of utilities that might be useful for a number of preprocessing tasks, even not directly related to <code>pytorch-widedeep</code>. All the classes and functions discussed here are available directly from the <code>utils</code> module. For example, the <code>LabelEncoder</code> within the <code>deeptabular_utils</code> submodule can be imported as:</p> <pre><code>from pytorch_widedeep.utils import LabelEncoder\n</code></pre> <p>These are classes and functions that are internally used in the library. We include them here in case the user finds them useful for other purposes.</p>"},{"location":"pytorch-widedeep/utils/deeptabular_utils.html","title":"deeptabular utils","text":""},{"location":"pytorch-widedeep/utils/deeptabular_utils.html#pytorch_widedeep.utils.deeptabular_utils.LabelEncoder","title":"LabelEncoder","text":"<pre><code>LabelEncoder(\n    columns_to_encode=None,\n    with_attention=False,\n    shared_embed=False,\n)\n</code></pre> <p>Label Encode categorical values for multiple columns at once</p> <p> NOTE: LabelEncoder reserves 0 for <code>unseen</code> new categories. This is convenient when defining the embedding layers, since we can just set padding idx to 0.</p> <p>Parameters:</p> <ul> <li> <code>columns_to_encode</code>               (<code>Optional[List[str]]</code>, default:                   <code>None</code> )           \u2013            <p>List of strings containing the names of the columns to encode. If <code>None</code> all columns of type <code>object</code> in the dataframe will be label encoded.</p> </li> <li> <code>with_attention</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether the preprocessed data will be passed to an attention-based model. Aliased as <code>for_transformer</code>.</p> </li> <li> <code>shared_embed</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating if the embeddings will be \"shared\" when using attention-based models. The idea behind <code>shared_embed</code> is described in the Appendix A in the TabTransformer paper: 'The goal of having column embedding is to enable the model to distinguish the classes in one column from those in the other columns'. In other words, the idea is to let the model learn which column is embedded at the time. See: <code>pytorch_widedeep.models.transformers._layers.SharedEmbeddings</code>.</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>encoding_dict</code>               (<code>Dict</code>)           \u2013            <p>Dictionary containing the encoding mappings in the format, e.g. :  <code>{'colname1': {'cat1': 1, 'cat2': 2, ...}, 'colname2': {'cat1': 1, 'cat2': 2, ...}, ...}</code></p> </li> <li> <code>inverse_encoding_dict</code>               (<code>Dict</code>)           \u2013            <p>Dictionary containing the inverse encoding mappings in the format, e.g. :  <code>{'colname1': {1: 'cat1', 2: 'cat2', ...}, 'colname2': {1: 'cat1', 2: 'cat2', ...}, ...}</code></p> </li> </ul> Source code in <code>pytorch_widedeep/utils/deeptabular_utils.py</code> <pre><code>@alias(\"with_attention\", [\"for_transformer\"])\ndef __init__(\n    self,\n    columns_to_encode: Optional[List[str]] = None,\n    with_attention: bool = False,\n    shared_embed: bool = False,\n):\n    self.columns_to_encode = columns_to_encode\n\n    self.shared_embed = shared_embed\n    self.with_attention = with_attention\n\n    self.reset_embed_idx = not self.with_attention or self.shared_embed\n</code></pre>"},{"location":"pytorch-widedeep/utils/deeptabular_utils.html#pytorch_widedeep.utils.deeptabular_utils.LabelEncoder.partial_fit","title":"partial_fit","text":"<pre><code>partial_fit(df)\n</code></pre> <p>Main method. Creates encoding attributes.</p> <p>Returns:</p> <ul> <li> <code>LabelEncoder</code>           \u2013            <p><code>LabelEncoder</code> fitted object</p> </li> </ul> Source code in <code>pytorch_widedeep/utils/deeptabular_utils.py</code> <pre><code>def partial_fit(self, df: pd.DataFrame) -&gt; \"LabelEncoder\":  # noqa: C901\n    \"\"\"Main method. Creates encoding attributes.\n\n    Returns\n    -------\n    LabelEncoder\n        `LabelEncoder` fitted object\n    \"\"\"\n    # here df is a chunk of the data. this is meant to be run when the\n    # data is large and we pass a chunk at a time. Therefore, we do not\n    # copy the input chunk as mutating a chunk is ok\n    if self.columns_to_encode is None:\n        self.columns_to_encode = list(df.select_dtypes(include=[\"object\"]).columns)\n    else:\n        # sanity check to make sure all categorical columns are in an adequate\n        # format\n        for col in self.columns_to_encode:\n            df[col] = df[col].astype(\"O\")\n\n    unique_column_vals: Dict[str, List[str]] = {}\n    for c in self.columns_to_encode:\n        unique_column_vals[c] = df[c].unique().tolist()\n\n    if not hasattr(self, \"encoding_dict\"):\n        # we run the method 'partial_fit' for the 1st time\n        self.encoding_dict: Dict[str, Dict[str, int]] = {}\n        if \"cls_token\" in unique_column_vals and self.shared_embed:\n            self.encoding_dict[\"cls_token\"] = {\"[CLS]\": 0}\n            del unique_column_vals[\"cls_token\"]\n\n        # leave 0 for padding/\"unseen\" categories. Also we need an\n        # attribute to keep track of the encoding in case we use\n        # attention and we do not re-start the index/counter\n        self.cum_idx: int = 1\n        for k, v in unique_column_vals.items():\n            self.encoding_dict[k] = {o: i + self.cum_idx for i, o in enumerate(v)}\n            self.cum_idx = 1 if self.reset_embed_idx else self.cum_idx + len(v)\n    else:\n        # the 'partial_fit' method has already run.\n        # \"cls_token\" will have been added already\n        if \"cls_token\" in unique_column_vals and self.shared_embed:\n            del unique_column_vals[\"cls_token\"]\n\n        # Classes in the new df/chunk of the dataset that have not been seen\n        # before\n        unseen_classes: Dict[str, List[str]] = {}\n        for c in self.columns_to_encode:\n            unseen_classes[c] = list(\n                np.setdiff1d(\n                    unique_column_vals[c], list(self.encoding_dict[c].keys())\n                )\n            )\n\n        # leave 0 for padding/\"unseen\" categories\n        for k, v in unique_column_vals.items():\n            # if we use attention we need to start encoding from the\n            # last 'overall' encoding index. Otherwise, we use the max\n            # encoding index per categorical col\n            _idx = (\n                max(self.encoding_dict[k].values()) + 1\n                if self.reset_embed_idx\n                else self.cum_idx\n            )\n            if len(unseen_classes[k]) != 0:\n                for i, o in enumerate(unseen_classes[k]):\n                    if o not in self.encoding_dict[k]:\n                        self.encoding_dict[k][o] = i + _idx\n                # if self.reset_embed_idx is True it will be 1 anyway\n                self.cum_idx = (\n                    1\n                    if self.reset_embed_idx\n                    else self.cum_idx + len(unseen_classes[k])\n                )\n\n    return self\n</code></pre>"},{"location":"pytorch-widedeep/utils/deeptabular_utils.html#pytorch_widedeep.utils.deeptabular_utils.LabelEncoder.fit","title":"fit","text":"<pre><code>fit(df)\n</code></pre> <p>Simply runs the <code>partial_fit</code> method when the data fits in memory</p> <p>Returns:</p> <ul> <li> <code>LabelEncoder</code>           \u2013            <p><code>LabelEncoder</code> fitted object</p> </li> </ul> Source code in <code>pytorch_widedeep/utils/deeptabular_utils.py</code> <pre><code>def fit(self, df: pd.DataFrame) -&gt; \"LabelEncoder\":\n    \"\"\"Simply runs the `partial_fit` method when the data fits in memory\n\n    Returns\n    -------\n    LabelEncoder\n        `LabelEncoder` fitted object\n    \"\"\"\n    # this is meant to be run when the data fits in memory and therefore,\n    # we do not want to mutate the original df, so we copy it\n    self.partial_fit(df.copy())\n\n    self.inverse_encoding_dict = self.create_inverse_encoding_dict()\n\n    return self\n</code></pre>"},{"location":"pytorch-widedeep/utils/deeptabular_utils.html#pytorch_widedeep.utils.deeptabular_utils.LabelEncoder.transform","title":"transform","text":"<pre><code>transform(df)\n</code></pre> <p>Label Encoded the categories in <code>columns_to_encode</code></p> <p>Returns:</p> <ul> <li> <code>DataFrame</code>           \u2013            <p>label-encoded dataframe</p> </li> </ul> Source code in <code>pytorch_widedeep/utils/deeptabular_utils.py</code> <pre><code>def transform(self, df: pd.DataFrame) -&gt; pd.DataFrame:\n    \"\"\"Label Encoded the categories in `columns_to_encode`\n\n    Returns\n    -------\n    pd.DataFrame\n        label-encoded dataframe\n    \"\"\"\n    try:\n        self.encoding_dict\n    except AttributeError:\n        raise NotFittedError(\n            \"This LabelEncoder instance is not fitted yet. \"\n            \"Call 'fit' with appropriate arguments before using this LabelEncoder.\"\n        )\n\n    df_inp = df.copy()\n    # sanity check to make sure all categorical columns are in an adequate\n    # format\n    for col in self.columns_to_encode:  # type: ignore\n        df_inp[col] = df_inp[col].astype(\"O\")\n\n    for k, v in self.encoding_dict.items():\n        df_inp[k] = df_inp[k].apply(lambda x: v[x] if x in v.keys() else 0)\n\n    return df_inp\n</code></pre>"},{"location":"pytorch-widedeep/utils/deeptabular_utils.html#pytorch_widedeep.utils.deeptabular_utils.LabelEncoder.fit_transform","title":"fit_transform","text":"<pre><code>fit_transform(df)\n</code></pre> <p>Combines <code>fit</code> and <code>transform</code></p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import pandas as pd\n&gt;&gt;&gt; from pytorch_widedeep.utils import LabelEncoder\n&gt;&gt;&gt; df = pd.DataFrame({'col1': [1,2,3], 'col2': ['me', 'you', 'him']})\n&gt;&gt;&gt; columns_to_encode = ['col2']\n&gt;&gt;&gt; encoder = LabelEncoder(columns_to_encode)\n&gt;&gt;&gt; encoder.fit_transform(df)\n   col1  col2\n0     1     1\n1     2     2\n2     3     3\n&gt;&gt;&gt; encoder.encoding_dict\n{'col2': {'me': 1, 'you': 2, 'him': 3}}\n</code></pre> <p>Returns:</p> <ul> <li> <code>DataFrame</code>           \u2013            <p>label-encoded dataframe</p> </li> </ul> Source code in <code>pytorch_widedeep/utils/deeptabular_utils.py</code> <pre><code>def fit_transform(self, df: pd.DataFrame) -&gt; pd.DataFrame:\n    \"\"\"Combines `fit` and `transform`\n\n    Examples\n    --------\n\n    &gt;&gt;&gt; import pandas as pd\n    &gt;&gt;&gt; from pytorch_widedeep.utils import LabelEncoder\n    &gt;&gt;&gt; df = pd.DataFrame({'col1': [1,2,3], 'col2': ['me', 'you', 'him']})\n    &gt;&gt;&gt; columns_to_encode = ['col2']\n    &gt;&gt;&gt; encoder = LabelEncoder(columns_to_encode)\n    &gt;&gt;&gt; encoder.fit_transform(df)\n       col1  col2\n    0     1     1\n    1     2     2\n    2     3     3\n    &gt;&gt;&gt; encoder.encoding_dict\n    {'col2': {'me': 1, 'you': 2, 'him': 3}}\n\n    Returns\n    -------\n    pd.DataFrame\n        label-encoded dataframe\n    \"\"\"\n    return self.fit(df).transform(df)\n</code></pre>"},{"location":"pytorch-widedeep/utils/deeptabular_utils.html#pytorch_widedeep.utils.deeptabular_utils.LabelEncoder.inverse_transform","title":"inverse_transform","text":"<pre><code>inverse_transform(df)\n</code></pre> <p>Returns the original categories</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import pandas as pd\n&gt;&gt;&gt; from pytorch_widedeep.utils import LabelEncoder\n&gt;&gt;&gt; df = pd.DataFrame({'col1': [1,2,3], 'col2': ['me', 'you', 'him']})\n&gt;&gt;&gt; columns_to_encode = ['col2']\n&gt;&gt;&gt; encoder = LabelEncoder(columns_to_encode)\n&gt;&gt;&gt; df_enc = encoder.fit_transform(df)\n&gt;&gt;&gt; encoder.inverse_transform(df_enc)\n   col1 col2\n0     1   me\n1     2  you\n2     3  him\n</code></pre> <p>Returns:</p> <ul> <li> <code>DataFrame</code>           \u2013            <p>DataFrame with original categories</p> </li> </ul> Source code in <code>pytorch_widedeep/utils/deeptabular_utils.py</code> <pre><code>def inverse_transform(self, df: pd.DataFrame) -&gt; pd.DataFrame:\n    \"\"\"Returns the original categories\n\n    Examples\n    --------\n\n    &gt;&gt;&gt; import pandas as pd\n    &gt;&gt;&gt; from pytorch_widedeep.utils import LabelEncoder\n    &gt;&gt;&gt; df = pd.DataFrame({'col1': [1,2,3], 'col2': ['me', 'you', 'him']})\n    &gt;&gt;&gt; columns_to_encode = ['col2']\n    &gt;&gt;&gt; encoder = LabelEncoder(columns_to_encode)\n    &gt;&gt;&gt; df_enc = encoder.fit_transform(df)\n    &gt;&gt;&gt; encoder.inverse_transform(df_enc)\n       col1 col2\n    0     1   me\n    1     2  you\n    2     3  him\n\n    Returns\n    -------\n    pd.DataFrame\n        DataFrame with original categories\n    \"\"\"\n\n    if not hasattr(self, \"inverse_encoding_dict\"):\n        self.inverse_encoding_dict = self.create_inverse_encoding_dict()\n\n    for k, v in self.inverse_encoding_dict.items():\n        df[k] = df[k].apply(lambda x: v[x])\n\n    return df\n</code></pre>"},{"location":"pytorch-widedeep/utils/fastai_transforms.html","title":"Fastai transforms","text":"<p>I directly copied and pasted part of the <code>transforms.py</code> module from the <code>fastai</code> library (from an old version). The reason to do such a thing is because <code>pytorch_widedeep</code> only needs the <code>Tokenizer</code> and the <code>Vocab</code> classes there. This way I avoid extra dependencies. Credit for all the code in the <code>fastai_transforms</code> module in this <code>pytorch-widedeep</code> package goes to Jeremy Howard and the <code>fastai</code> team. I only include the documentation here for completion, but I strongly advise the user to read the <code>fastai</code> documentation.</p>"},{"location":"pytorch-widedeep/utils/fastai_transforms.html#pytorch_widedeep.utils.fastai_transforms.Tokenizer","title":"Tokenizer","text":"<pre><code>Tokenizer(\n    tok_func=SpacyTokenizer,\n    lang=\"en\",\n    pre_rules=None,\n    post_rules=None,\n    special_cases=None,\n    n_cpus=None,\n)\n</code></pre> <p>Class to combine a series of rules and a tokenizer function to tokenize text with multiprocessing.</p> <p>Setting some of the parameters of this class require perhaps some familiarity with the source code.</p> <p>Parameters:</p> <ul> <li> <code>tok_func</code>               (<code>Callable</code>, default:                   <code>SpacyTokenizer</code> )           \u2013            <p>Tokenizer Object. See <code>pytorch_widedeep.utils.fastai_transforms.SpacyTokenizer</code></p> </li> <li> <code>lang</code>               (<code>str</code>, default:                   <code>'en'</code> )           \u2013            <p>Text's Language</p> </li> <li> <code>pre_rules</code>               (<code>Optional[ListRules]</code>, default:                   <code>None</code> )           \u2013            <p>Custom type: <code>Collection[Callable[[str], str]]</code>. These are <code>Callable</code> objects that will be applied to the text (str) directly as <code>rule(tok)</code> before being tokenized.</p> </li> <li> <code>post_rules</code>               (<code>Optional[ListRules]</code>, default:                   <code>None</code> )           \u2013            <p>Custom type: <code>Collection[Callable[[str], str]]</code>. These are <code>Callable</code> objects that will be applied to the tokens as <code>rule(tokens)</code> after the text has been tokenized.</p> </li> <li> <code>special_cases</code>               (<code>Optional[Collection[str]]</code>, default:                   <code>None</code> )           \u2013            <p>special cases to be added to the tokenizer via <code>Spacy</code>'s <code>add_special_case</code> method</p> </li> <li> <code>n_cpus</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>number of CPUs to used during the tokenization process</p> </li> </ul> Source code in <code>pytorch_widedeep/utils/fastai_transforms.py</code> <pre><code>def __init__(\n    self,\n    tok_func: Callable = SpacyTokenizer,\n    lang: str = \"en\",\n    pre_rules: Optional[ListRules] = None,\n    post_rules: Optional[ListRules] = None,\n    special_cases: Optional[Collection[str]] = None,\n    n_cpus: Optional[int] = None,\n):\n    self.tok_func, self.lang, self.special_cases = tok_func, lang, special_cases\n    self.pre_rules = ifnone(pre_rules, defaults.text_pre_rules)\n    self.post_rules = ifnone(post_rules, defaults.text_post_rules)\n    self.special_cases = (\n        special_cases if special_cases is not None else defaults.text_spec_tok\n    )\n    self.n_cpus = ifnone(n_cpus, defaults.cpus)\n</code></pre>"},{"location":"pytorch-widedeep/utils/fastai_transforms.html#pytorch_widedeep.utils.fastai_transforms.Tokenizer.process_text","title":"process_text","text":"<pre><code>process_text(t, tok)\n</code></pre> <p>Process and tokenize one text <code>t</code> with tokenizer <code>tok</code>.</p> <p>Parameters:</p> <ul> <li> <code>t</code>               (<code>str</code>)           \u2013            <p>text to be processed and tokenized</p> </li> <li> <code>tok</code>               (<code>BaseTokenizer</code>)           \u2013            <p>Instance of <code>BaseTokenizer</code>. See <code>pytorch_widedeep.utils.fastai_transforms.BaseTokenizer</code></p> </li> </ul> <p>Returns:</p> <ul> <li> <code>List[str]</code>           \u2013            <p>List of tokens</p> </li> </ul> Source code in <code>pytorch_widedeep/utils/fastai_transforms.py</code> <pre><code>def process_text(self, t: str, tok: BaseTokenizer) -&gt; List[str]:\n    r\"\"\"Process and tokenize one text ``t`` with tokenizer ``tok``.\n\n    Parameters\n    ----------\n    t: str\n        text to be processed and tokenized\n    tok: ``BaseTokenizer``\n        Instance of `BaseTokenizer`. See\n        `pytorch_widedeep.utils.fastai_transforms.BaseTokenizer`\n\n    Returns\n    -------\n    List[str]\n        List of tokens\n    \"\"\"\n    for rule in self.pre_rules:\n        t = rule(t)\n    toks = tok.tokenizer(t)\n    for rule in self.post_rules:\n        toks = rule(toks)\n    return toks\n</code></pre>"},{"location":"pytorch-widedeep/utils/fastai_transforms.html#pytorch_widedeep.utils.fastai_transforms.Tokenizer.process_all","title":"process_all","text":"<pre><code>process_all(texts)\n</code></pre> <p>Process a list of texts. Parallel execution of <code>process_text</code>.</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; from pytorch_widedeep.utils import Tokenizer\n&gt;&gt;&gt; texts = ['Machine learning is great', 'but building stuff is even better']\n&gt;&gt;&gt; tok = Tokenizer()\n&gt;&gt;&gt; tok.process_all(texts)\n[['xxmaj', 'machine', 'learning', 'is', 'great'], ['but', 'building', 'stuff', 'is', 'even', 'better']]\n</code></pre> <p> NOTE: Note the token <code>TK_MAJ</code> (<code>xxmaj</code>), used to indicate the next word begins with a capital in the original text. For more details of special tokens please see the <code>fastai</code> docs.</p> <p>Returns:</p> <ul> <li> <code>List[List[str]]</code>           \u2013            <p>List containing lists of tokens. One list per \"document\"</p> </li> </ul> Source code in <code>pytorch_widedeep/utils/fastai_transforms.py</code> <pre><code>def process_all(self, texts: Collection[str]) -&gt; List[List[str]]:\n    r\"\"\"Process a list of texts. Parallel execution of ``process_text``.\n\n    Examples\n    --------\n    &gt;&gt;&gt; from pytorch_widedeep.utils import Tokenizer\n    &gt;&gt;&gt; texts = ['Machine learning is great', 'but building stuff is even better']\n    &gt;&gt;&gt; tok = Tokenizer()\n    &gt;&gt;&gt; tok.process_all(texts)\n    [['xxmaj', 'machine', 'learning', 'is', 'great'], ['but', 'building', 'stuff', 'is', 'even', 'better']]\n\n    :information_source: **NOTE**:\n    Note the token ``TK_MAJ`` (`xxmaj`), used to indicate the\n    next word begins with a capital in the original text. For more\n    details of special tokens please see the [``fastai`` docs](https://docs.fast.ai/text.core.html#Tokenizing).\n\n    Returns\n    -------\n    List[List[str]]\n        List containing lists of tokens. One list per \"_document_\"\n\n    \"\"\"\n\n    if self.n_cpus &lt;= 1:\n        return self._process_all_1(texts)\n    with ProcessPoolExecutor(self.n_cpus) as e:\n        return sum(\n            e.map(self._process_all_1, partition_by_cores(texts, self.n_cpus)), []\n        )\n</code></pre>"},{"location":"pytorch-widedeep/utils/fastai_transforms.html#pytorch_widedeep.utils.fastai_transforms.Vocab","title":"Vocab","text":"<pre><code>Vocab(\n    max_vocab, min_freq, pad_idx=None, special_cases=None\n)\n</code></pre> <p>Contains the correspondence between numbers and tokens.</p> <p>Parameters:</p> <ul> <li> <code>max_vocab</code>               (<code>int</code>)           \u2013            <p>maximum vocabulary size</p> </li> <li> <code>min_freq</code>               (<code>int</code>)           \u2013            <p>minimum frequency for a token to be considereds</p> </li> <li> <code>pad_idx</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>padding index. If <code>None</code>, Fastai's Tokenizer leaves the 0 index for the unknown token ('xxunk') and defaults to 1 for the padding token ('xxpad').</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>itos</code>               (<code>Collection</code>)           \u2013            <p><code>index to str</code>. Collection of strings that are the tokens of the vocabulary</p> </li> <li> <code>stoi</code>               (<code>defaultdict</code>)           \u2013            <p><code>str to index</code>. Dictionary containing the tokens of the vocabulary and their corresponding index</p> </li> </ul> Source code in <code>pytorch_widedeep/utils/fastai_transforms.py</code> <pre><code>def __init__(\n    self,\n    max_vocab: int,\n    min_freq: int,\n    pad_idx: Optional[int] = None,\n    special_cases: Optional[Collection[str]] = None,\n):\n    self.max_vocab = max_vocab\n    self.min_freq = min_freq\n    self.pad_idx = pad_idx\n    self.special_cases = (\n        special_cases if special_cases is not None else defaults.text_spec_tok\n    )\n</code></pre>"},{"location":"pytorch-widedeep/utils/fastai_transforms.html#pytorch_widedeep.utils.fastai_transforms.Vocab.create","title":"create","text":"<pre><code>create(tokens)\n</code></pre> <p>Create a vocabulary object from a set of tokens.</p> <p>Parameters:</p> <ul> <li> <code>tokens</code>               (<code>Tokens</code>)           \u2013            <p>Custom type: <code>Collection[Collection[str]]</code>  see <code>pytorch_widedeep.wdtypes</code>. Collection of collection of strings (e.g. list of tokenized sentences)</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; from pytorch_widedeep.utils import Tokenizer, Vocab\n&gt;&gt;&gt; texts = ['Machine learning is great', 'but building stuff is even better']\n&gt;&gt;&gt; tokens = Tokenizer().process_all(texts)\n&gt;&gt;&gt; vocab = Vocab(max_vocab=18, min_freq=1).create(tokens)\n&gt;&gt;&gt; vocab.numericalize(['machine', 'learning', 'is', 'great'])\n[10, 11, 9, 12]\n&gt;&gt;&gt; vocab.textify([10, 11, 9, 12])\n'machine learning is great'\n</code></pre> <p> NOTE: Note the many special tokens that <code>fastai</code>'s' tokenizer adds. These are particularly useful when building Language models and/or in classification/Regression tasks. Please see the <code>fastai</code> docs.</p> <p>Returns:</p> <ul> <li> <code>Vocab</code>           \u2013            <p>An instance of a <code>Vocab</code> object</p> </li> </ul> Source code in <code>pytorch_widedeep/utils/fastai_transforms.py</code> <pre><code>def create(\n    self,\n    tokens: Tokens,\n) -&gt; \"Vocab\":\n    r\"\"\"Create a vocabulary object from a set of tokens.\n\n    Parameters\n    ----------\n    tokens: Tokens\n        Custom type: ``Collection[Collection[str]]``  see\n        `pytorch_widedeep.wdtypes`. Collection of collection of\n        strings (e.g. list of tokenized sentences)\n\n    Examples\n    --------\n    &gt;&gt;&gt; from pytorch_widedeep.utils import Tokenizer, Vocab\n    &gt;&gt;&gt; texts = ['Machine learning is great', 'but building stuff is even better']\n    &gt;&gt;&gt; tokens = Tokenizer().process_all(texts)\n    &gt;&gt;&gt; vocab = Vocab(max_vocab=18, min_freq=1).create(tokens)\n    &gt;&gt;&gt; vocab.numericalize(['machine', 'learning', 'is', 'great'])\n    [10, 11, 9, 12]\n    &gt;&gt;&gt; vocab.textify([10, 11, 9, 12])\n    'machine learning is great'\n\n    :information_source: **NOTE**:\n    Note the many special tokens that ``fastai``'s' tokenizer adds. These\n    are particularly useful when building Language models and/or in\n    classification/Regression tasks. Please see the [``fastai`` docs](https://docs.fast.ai/text.core.html#Tokenizing).\n\n    Returns\n    -------\n    Vocab\n        An instance of a `Vocab` object\n    \"\"\"\n\n    freq = Counter(p for o in tokens for p in o)\n    itos = [o for o, c in freq.most_common(self.max_vocab) if c &gt;= self.min_freq]\n    for o in reversed(self.special_cases):\n        if o in itos:\n            itos.remove(o)\n        itos.insert(0, o)\n\n    if self.pad_idx is not None and self.pad_idx != 1:\n        itos.remove(PAD)\n        itos.insert(self.pad_idx, PAD)\n        # get the new 'xxunk' index\n        xxunk_idx = np.where([el == \"xxunk\" for el in itos])[0][0]\n    else:\n        xxunk_idx = 0\n\n    itos = itos[: self.max_vocab]\n    if (\n        len(itos) &lt; self.max_vocab\n    ):  # Make sure vocab size is a multiple of 8 for fast mixed precision training\n        while len(itos) % 8 != 0:\n            itos.append(\"xxfake\")\n\n    self.itos = itos\n    self.stoi = defaultdict(\n        lambda: xxunk_idx, {v: k for k, v in enumerate(self.itos)}\n    )\n\n    return self\n</code></pre>"},{"location":"pytorch-widedeep/utils/fastai_transforms.html#pytorch_widedeep.utils.fastai_transforms.Vocab.fit","title":"fit","text":"<pre><code>fit(tokens)\n</code></pre> <p>Calls the <code>create</code> method. I simply want to honor fast ai naming, but for consistency with the rest of the library I am including a fit method</p> Source code in <code>pytorch_widedeep/utils/fastai_transforms.py</code> <pre><code>def fit(\n    self,\n    tokens: Tokens,\n) -&gt; \"Vocab\":\n    \"\"\"\n    Calls the `create` method. I simply want to honor fast ai naming, but\n    for consistency with the rest of the library I am including a fit method\n    \"\"\"\n    return self.create(tokens)\n</code></pre>"},{"location":"pytorch-widedeep/utils/fastai_transforms.html#pytorch_widedeep.utils.fastai_transforms.Vocab.numericalize","title":"numericalize","text":"<pre><code>numericalize(t)\n</code></pre> <p>Convert a list of tokens <code>t</code> to their ids.</p> <p>Returns:</p> <ul> <li> <code>List[int]</code>           \u2013            <p>List of 'numericalsed' tokens</p> </li> </ul> Source code in <code>pytorch_widedeep/utils/fastai_transforms.py</code> <pre><code>def numericalize(self, t: Collection[str]) -&gt; List[int]:\n    \"\"\"Convert a list of tokens ``t`` to their ids.\n\n    Returns\n    -------\n    List[int]\n        List of '_numericalsed_' tokens\n    \"\"\"\n    return [self.stoi[w] for w in t]\n</code></pre>"},{"location":"pytorch-widedeep/utils/fastai_transforms.html#pytorch_widedeep.utils.fastai_transforms.Vocab.transform","title":"transform","text":"<pre><code>transform(t)\n</code></pre> <p>Calls the <code>numericalize</code> method. I simply want to honor fast ai naming, but for consistency with the rest of the library I am including a transform method</p> Source code in <code>pytorch_widedeep/utils/fastai_transforms.py</code> <pre><code>def transform(self, t: Collection[str]) -&gt; List[int]:\n    \"\"\"\n    Calls the `numericalize` method. I simply want to honor fast ai naming,\n    but for consistency with the rest of the library I am including a\n    transform method\n    \"\"\"\n    return self.numericalize(t)\n</code></pre>"},{"location":"pytorch-widedeep/utils/fastai_transforms.html#pytorch_widedeep.utils.fastai_transforms.Vocab.textify","title":"textify","text":"<pre><code>textify(nums, sep=' ')\n</code></pre> <p>Convert a list of <code>nums</code> (or indexes) to their tokens.</p> <p>Returns:</p> <ul> <li> <code>List[str]</code>           \u2013            <p>List of tokens</p> </li> </ul> Source code in <code>pytorch_widedeep/utils/fastai_transforms.py</code> <pre><code>def textify(self, nums: Collection[int], sep=\" \") -&gt; Union[str, List[str]]:\n    \"\"\"Convert a list of ``nums`` (or indexes) to their tokens.\n\n    Returns\n    -------\n    List[str]\n        List of tokens\n    \"\"\"\n    return (\n        sep.join([self.itos[i] for i in nums])\n        if sep is not None\n        else [self.itos[i] for i in nums]\n    )\n</code></pre>"},{"location":"pytorch-widedeep/utils/fastai_transforms.html#pytorch_widedeep.utils.fastai_transforms.Vocab.inverse_transform","title":"inverse_transform","text":"<pre><code>inverse_transform(nums, sep=' ')\n</code></pre> <p>Calls the <code>textify</code> method. I simply want to honor fast ai naming, but for consistency with the rest of the library I am including an inverse_transform method</p> Source code in <code>pytorch_widedeep/utils/fastai_transforms.py</code> <pre><code>def inverse_transform(\n    self, nums: Collection[int], sep=\" \"\n) -&gt; Union[str, List[str]]:\n    \"\"\"\n    Calls the `textify` method. I simply want to honor fast ai naming, but\n    for consistency with the rest of the library I am including an\n    inverse_transform method\n    \"\"\"\n    # I simply want to honor fast ai naming, but for consistency with the\n    # rest of the library I am including an inverse_transform method\n    return self.textify(nums, sep)\n</code></pre>"},{"location":"pytorch-widedeep/utils/image_utils.html","title":"Image utils","text":"<p>SimplePreprocessor and AspectAwarePreprocessor are directly taked from the great series of Books `Deep Learning for Computer Vision by Adrian. Therefore, all credit for the code in the <code>image_utils</code> module goes to Adrian Rosebrock.</p>"},{"location":"pytorch-widedeep/utils/image_utils.html#pytorch_widedeep.utils.image_utils.AspectAwarePreprocessor","title":"AspectAwarePreprocessor","text":"<pre><code>AspectAwarePreprocessor(\n    width, height, inter=cv2.INTER_AREA\n)\n</code></pre> <p>Class to resize an image to a certain width and height taking into account the image aspect ratio</p> <p>Parameters:</p> <ul> <li> <code>width</code>               (<code>int</code>)           \u2013            <p>output width</p> </li> <li> <code>height</code>               (<code>int</code>)           \u2013            <p>output height</p> </li> <li> <code>inter</code>           \u2013            <p><code>opencv</code> interpolation method. See <code>opencv</code> <code>InterpolationFlags</code>.</p> </li> </ul> Source code in <code>pytorch_widedeep/utils/image_utils.py</code> <pre><code>def __init__(self, width: int, height: int, inter=cv2.INTER_AREA):\n    self.width = width\n    self.height = height\n    self.inter = inter\n</code></pre>"},{"location":"pytorch-widedeep/utils/image_utils.html#pytorch_widedeep.utils.image_utils.AspectAwarePreprocessor.preprocess","title":"preprocess","text":"<pre><code>preprocess(image)\n</code></pre> <p>Returns the resized input image taking into account the image aspect ratio</p> <p>Parameters:</p> <ul> <li> <code>image</code>               (<code>ndarray</code>)           \u2013            <p>Input image to be resized</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import cv2\n&gt;&gt;&gt; from pytorch_widedeep.utils import AspectAwarePreprocessor\n&gt;&gt;&gt; img = cv2.imread(\"tests/test_data_utils/images/galaxy1.png\")\n&gt;&gt;&gt; img.shape\n(694, 890, 3)\n&gt;&gt;&gt; app = AspectAwarePreprocessor(width=224, height=224)\n&gt;&gt;&gt; resized_img = app.preprocess(img)\n&gt;&gt;&gt; resized_img.shape\n(224, 224, 3)\n</code></pre> <p>Returns:</p> <ul> <li> <code>ndarray</code>           \u2013            <p>Resized image according to its original image aspect ratio</p> </li> </ul> Source code in <code>pytorch_widedeep/utils/image_utils.py</code> <pre><code>def preprocess(self, image: np.ndarray) -&gt; np.ndarray:\n    r\"\"\"Returns the resized input image taking into account the image aspect ratio\n\n    Parameters\n    ----------\n    image: np.ndarray\n        Input image to be resized\n\n    Examples\n    --------\n    &gt;&gt;&gt; import cv2\n    &gt;&gt;&gt; from pytorch_widedeep.utils import AspectAwarePreprocessor\n    &gt;&gt;&gt; img = cv2.imread(\"tests/test_data_utils/images/galaxy1.png\")\n    &gt;&gt;&gt; img.shape\n    (694, 890, 3)\n    &gt;&gt;&gt; app = AspectAwarePreprocessor(width=224, height=224)\n    &gt;&gt;&gt; resized_img = app.preprocess(img)\n    &gt;&gt;&gt; resized_img.shape\n    (224, 224, 3)\n\n    Returns\n    -------\n    np.ndarray\n        Resized image according to its original image aspect ratio\n    \"\"\"\n    (h, w) = image.shape[:2]\n    dW = 0\n    dH = 0\n\n    if w &lt; h:\n        image = imutils.resize(image, width=self.width, inter=self.inter)\n        dH = int((image.shape[0] - self.height) / 2.0)\n    else:\n        image = imutils.resize(image, height=self.height, inter=self.inter)\n        dW = int((image.shape[1] - self.width) / 2.0)\n\n    (h, w) = image.shape[:2]\n    image = image[dH : h - dH, dW : w - dW]\n\n    resized_image = cv2.resize(\n        image, (self.width, self.height), interpolation=self.inter\n    )\n\n    return resized_image\n</code></pre>"},{"location":"pytorch-widedeep/utils/image_utils.html#pytorch_widedeep.utils.image_utils.SimplePreprocessor","title":"SimplePreprocessor","text":"<pre><code>SimplePreprocessor(width, height, inter=cv2.INTER_AREA)\n</code></pre> <p>Class to resize an image to a certain width and height</p> <p>Parameters:</p> <ul> <li> <code>width</code>               (<code>int</code>)           \u2013            <p>output width</p> </li> <li> <code>height</code>               (<code>int</code>)           \u2013            <p>output height</p> </li> <li> <code>inter</code>           \u2013            <p><code>opencv</code> interpolation method. See <code>opencv</code> <code>InterpolationFlags</code>.</p> </li> </ul> Source code in <code>pytorch_widedeep/utils/image_utils.py</code> <pre><code>def __init__(self, width: int, height: int, inter=cv2.INTER_AREA):\n    self.width = width\n    self.height = height\n    self.inter = inter\n</code></pre>"},{"location":"pytorch-widedeep/utils/image_utils.html#pytorch_widedeep.utils.image_utils.SimplePreprocessor.preprocess","title":"preprocess","text":"<pre><code>preprocess(image)\n</code></pre> <p>Returns the resized input image</p> <p>Parameters:</p> <ul> <li> <code>image</code>               (<code>ndarray</code>)           \u2013            <p>Input image to be resized</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>ndarray</code>           \u2013            <p>Resized image</p> </li> </ul> Source code in <code>pytorch_widedeep/utils/image_utils.py</code> <pre><code>def preprocess(self, image: np.ndarray) -&gt; np.ndarray:\n    r\"\"\"Returns the resized input image\n\n    Parameters\n    ----------\n    image: np.ndarray\n        Input image to be resized\n\n    Returns\n    -------\n    np.ndarray\n        Resized image\n\n    \"\"\"\n    resized_image = cv2.resize(\n        image, (self.width, self.height), interpolation=self.inter\n    )\n\n    return resized_image\n</code></pre>"},{"location":"pytorch-widedeep/utils/text_utils.html","title":"Text utils","text":"<p>Collection of helper function that facilitate processing text.</p>"},{"location":"pytorch-widedeep/utils/text_utils.html#pytorch_widedeep.utils.text_utils.simple_preprocess","title":"simple_preprocess","text":"<pre><code>simple_preprocess(\n    doc, lower=False, deacc=False, min_len=2, max_len=15\n)\n</code></pre> <p>This is <code>Gensim</code>'s <code>simple_preprocess</code> with a <code>lower</code> param to indicate wether or not to lower case all the token in the doc</p> <p>For more information see: <code>Gensim</code> utils module</p> <p>Parameters:</p> <ul> <li> <code>doc</code>               (<code>str</code>)           \u2013            <p>Input document.</p> </li> <li> <code>lower</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Lower case tokens in the input doc</p> </li> <li> <code>deacc</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Remove accent marks from tokens using <code>Gensim</code>'s <code>deaccent</code></p> </li> <li> <code>min_len</code>               (<code>int</code>, default:                   <code>2</code> )           \u2013            <p>Minimum length of token (inclusive). Shorter tokens are discarded.</p> </li> <li> <code>max_len</code>               (<code>int</code>, default:                   <code>15</code> )           \u2013            <p>Maximum length of token in result (inclusive). Longer tokens are discarded.</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; from pytorch_widedeep.utils import simple_preprocess\n&gt;&gt;&gt; simple_preprocess('Machine learning is great')\n['Machine', 'learning', 'is', 'great']\n</code></pre> <p>Returns:</p> <ul> <li> <code>List[str]</code>           \u2013            <p>List with the processed tokens</p> </li> </ul> Source code in <code>pytorch_widedeep/utils/text_utils.py</code> <pre><code>def simple_preprocess(\n    doc: str,\n    lower: bool = False,\n    deacc: bool = False,\n    min_len: int = 2,\n    max_len: int = 15,\n) -&gt; List[str]:\n    r\"\"\"\n    This is `Gensim`'s `simple_preprocess` with a `lower` param to\n    indicate wether or not to lower case all the token in the doc\n\n    For more information see: `Gensim` [utils module](https://radimrehurek.com/gensim/utils.html)\n\n    Parameters\n    ----------\n    doc: str\n        Input document.\n    lower: bool, default = False\n        Lower case tokens in the input doc\n    deacc: bool, default = False\n        Remove accent marks from tokens using `Gensim`'s `deaccent`\n    min_len: int, default = 2\n        Minimum length of token (inclusive). Shorter tokens are discarded.\n    max_len: int, default = 15\n        Maximum length of token in result (inclusive). Longer tokens are discarded.\n\n    Examples\n    --------\n    &gt;&gt;&gt; from pytorch_widedeep.utils import simple_preprocess\n    &gt;&gt;&gt; simple_preprocess('Machine learning is great')\n    ['Machine', 'learning', 'is', 'great']\n\n    Returns\n    -------\n    List[str]\n        List with the processed tokens\n    \"\"\"\n    tokens = [\n        token\n        for token in tokenize(doc, lower=lower, deacc=deacc, errors=\"ignore\")\n        if min_len &lt;= len(token) &lt;= max_len and not token.startswith(\"_\")\n    ]\n    return tokens\n</code></pre>"},{"location":"pytorch-widedeep/utils/text_utils.html#pytorch_widedeep.utils.text_utils.get_texts","title":"get_texts","text":"<pre><code>get_texts(texts, already_processed=False, n_cpus=None)\n</code></pre> <p>Tokenization using <code>Fastai</code>'s <code>Tokenizer</code> because it does a series of very convenients things during the tokenization process</p> <p>See <code>pytorch_widedeep.utils.fastai_utils.Tokenizer</code></p> <p>Parameters:</p> <ul> <li> <code>texts</code>               (<code>List[str]</code>)           \u2013            <p>List of str with the texts (or documents). One str per document</p> </li> <li> <code>already_processed</code>               (<code>Optional[bool]</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating if the text is already processed and we simply want to tokenize it. This parameter is thought for those cases where the input sequences might not be text (but IDs, or anything else) and we just want to tokenize it</p> </li> <li> <code>n_cpus</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>number of CPUs to used during the tokenization process</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; from pytorch_widedeep.utils import get_texts\n&gt;&gt;&gt; texts = ['Machine learning is great', 'but building stuff is even better']\n&gt;&gt;&gt; get_texts(texts)\n[['xxmaj', 'machine', 'learning', 'is', 'great'], ['but', 'building', 'stuff', 'is', 'even', 'better']]\n</code></pre> <p>Returns:</p> <ul> <li> <code>List[List[str]]</code>           \u2013            <p>List of lists, one list per 'document' containing its corresponding tokens</p> </li> <li> <code>information_source: **NOTE**:</code>           \u2013            </li> <li> <code>`get_texts` uses `pytorch_widedeep.utils.fastai_transforms.Tokenizer`.</code>           \u2013            </li> <li> <code>Such tokenizer uses a series of convenient processing steps, including</code>           \u2013            </li> <li> <code>the  addition of some special tokens, such as `TK_MAJ` (`xxmaj`), used to</code>           \u2013            </li> <li> <code>indicate the next word begins with a capital in the original text. For more</code>           \u2013            </li> <li> <code>details of special tokens please see the [`fastai` `docs](https://docs.fast.ai/text.core.html#Tokenizing)</code>           \u2013            </li> </ul> Source code in <code>pytorch_widedeep/utils/text_utils.py</code> <pre><code>def get_texts(\n    texts: List[str],\n    already_processed: Optional[bool] = False,\n    n_cpus: Optional[int] = None,\n) -&gt; List[List[str]]:\n    r\"\"\"Tokenization using `Fastai`'s `Tokenizer` because it does a\n    series of very convenients things during the tokenization process\n\n    See `pytorch_widedeep.utils.fastai_utils.Tokenizer`\n\n    Parameters\n    ----------\n    texts: List\n        List of str with the texts (or documents). One str per document\n    already_processed: bool, Optional, default = False\n        Boolean indicating if the text is already processed and we simply want\n        to tokenize it. This parameter is thought for those cases where the\n        input sequences might not be text (but IDs, or anything else) and we\n        just want to tokenize it\n    n_cpus: int, Optional, default = None\n        number of CPUs to used during the tokenization process\n\n    Examples\n    --------\n    &gt;&gt;&gt; from pytorch_widedeep.utils import get_texts\n    &gt;&gt;&gt; texts = ['Machine learning is great', 'but building stuff is even better']\n    &gt;&gt;&gt; get_texts(texts)\n    [['xxmaj', 'machine', 'learning', 'is', 'great'], ['but', 'building', 'stuff', 'is', 'even', 'better']]\n\n    Returns\n    -------\n    List[List[str]]\n        List of lists, one list per '_document_' containing its corresponding tokens\n\n    :information_source: **NOTE**:\n    `get_texts` uses `pytorch_widedeep.utils.fastai_transforms.Tokenizer`.\n    Such tokenizer uses a series of convenient processing steps, including\n    the  addition of some special tokens, such as `TK_MAJ` (`xxmaj`), used to\n    indicate the next word begins with a capital in the original text. For more\n    details of special tokens please see the [`fastai` `docs](https://docs.fast.ai/text.core.html#Tokenizing)\n    \"\"\"\n\n    num_cpus = n_cpus if n_cpus is not None else os.cpu_count()\n\n    if not already_processed:\n        processed_texts = [\" \".join(simple_preprocess(t)) for t in texts]\n    else:\n        processed_texts = texts\n    tok = Tokenizer(n_cpus=num_cpus).process_all(processed_texts)\n    return tok\n</code></pre>"},{"location":"pytorch-widedeep/utils/text_utils.html#pytorch_widedeep.utils.text_utils.pad_sequences","title":"pad_sequences","text":"<pre><code>pad_sequences(seq, maxlen, pad_first=True, pad_idx=1)\n</code></pre> <p>Given a List of tokenized and <code>numericalised</code> sequences it will return padded sequences according to the input parameters.</p> <p>Parameters:</p> <ul> <li> <code>seq</code>               (<code>List[int]</code>)           \u2013            <p>List of int with the <code>numericalised</code> tokens</p> </li> <li> <code>maxlen</code>               (<code>int</code>)           \u2013            <p>Maximum length of the padded sequences</p> </li> <li> <code>pad_first</code>               (<code>bool</code>, default:                   <code>True</code> )           \u2013            <p>Indicates whether the padding index will be added at the beginning or the end of the sequences</p> </li> <li> <code>pad_idx</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>padding index. Fastai's Tokenizer leaves 0 for the 'unknown' token.</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; from pytorch_widedeep.utils import pad_sequences\n&gt;&gt;&gt; seq = [1,2,3]\n&gt;&gt;&gt; pad_sequences(seq, maxlen=5, pad_idx=0)\narray([0, 0, 1, 2, 3], dtype=int32)\n</code></pre> <p>Returns:</p> <ul> <li> <code>ndarray</code>           \u2013            <p>numpy array with the padded sequences</p> </li> </ul> Source code in <code>pytorch_widedeep/utils/text_utils.py</code> <pre><code>def pad_sequences(\n    seq: List[int], maxlen: int, pad_first: bool = True, pad_idx: int = 1\n) -&gt; np.ndarray:\n    r\"\"\"\n    Given a List of tokenized and `numericalised` sequences it will return\n    padded sequences according to the input parameters.\n\n    Parameters\n    ----------\n    seq: List\n        List of int with the `numericalised` tokens\n    maxlen: int\n        Maximum length of the padded sequences\n    pad_first: bool,  default = True\n        Indicates whether the padding index will be added at the beginning or the\n        end of the sequences\n    pad_idx: int, default = 1\n        padding index. Fastai's Tokenizer leaves 0 for the 'unknown' token.\n\n    Examples\n    --------\n    &gt;&gt;&gt; from pytorch_widedeep.utils import pad_sequences\n    &gt;&gt;&gt; seq = [1,2,3]\n    &gt;&gt;&gt; pad_sequences(seq, maxlen=5, pad_idx=0)\n    array([0, 0, 1, 2, 3], dtype=int32)\n\n    Returns\n    -------\n    np.ndarray\n        numpy array with the padded sequences\n    \"\"\"\n    if len(seq) == 0:\n        return np.zeros(maxlen, dtype=\"int32\") + pad_idx\n    elif len(seq) &gt;= maxlen:\n        res = np.array(seq[-maxlen:]).astype(\"int32\")\n        return res\n    else:\n        res = np.zeros(maxlen, dtype=\"int32\") + pad_idx\n        if pad_first:\n            res[-len(seq) :] = seq\n        else:\n            res[: len(seq) :] = seq\n        return res\n</code></pre>"},{"location":"pytorch-widedeep/utils/text_utils.html#pytorch_widedeep.utils.text_utils.build_embeddings_matrix","title":"build_embeddings_matrix","text":"<pre><code>build_embeddings_matrix(\n    vocab, word_vectors_path, min_freq, verbose=1\n)\n</code></pre> <p>Build the embedding matrix using pretrained word vectors.</p> <p>Returns pretrained word embeddings. If a word in our vocabulary is not among the pretrained embeddings it will be assigned the mean pretrained word-embeddings vector</p> <p>Parameters:</p> <ul> <li> <code>vocab</code>               (<code>Union[Vocab, ChunkVocab]</code>)           \u2013            <p>see <code>pytorch_widedeep.utils.fastai_utils.Vocab</code></p> </li> <li> <code>word_vectors_path</code>               (<code>str</code>)           \u2013            <p>path to the pretrained word embeddings</p> </li> <li> <code>min_freq</code>               (<code>int</code>)           \u2013            <p>minimum frequency required for a word to be in the vocabulary</p> </li> <li> <code>verbose</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>level of verbosity. Set to 0 for no verbosity</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>ndarray</code>           \u2013            <p>Pretrained word embeddings</p> </li> </ul> Source code in <code>pytorch_widedeep/utils/text_utils.py</code> <pre><code>def build_embeddings_matrix(\n    vocab: Union[Vocab, ChunkVocab],\n    word_vectors_path: str,\n    min_freq: int,\n    verbose: int = 1,\n) -&gt; np.ndarray:  # pragma: no cover\n    r\"\"\"Build the embedding matrix using pretrained word vectors.\n\n    Returns pretrained word embeddings. If a word in our vocabulary is not\n    among the pretrained embeddings it will be assigned the mean pretrained\n    word-embeddings vector\n\n    Parameters\n    ----------\n    vocab: Vocab\n        see `pytorch_widedeep.utils.fastai_utils.Vocab`\n    word_vectors_path: str\n        path to the pretrained word embeddings\n    min_freq: int\n        minimum frequency required for a word to be in the vocabulary\n    verbose: int,  default=1\n        level of verbosity. Set to 0 for no verbosity\n\n    Returns\n    -------\n    np.ndarray\n        Pretrained word embeddings\n    \"\"\"\n    if not os.path.isfile(word_vectors_path):\n        raise FileNotFoundError(\"{} not found\".format(word_vectors_path))\n    if verbose:\n        print(\"Indexing word vectors...\")\n\n    embeddings_index = {}\n    f = open(word_vectors_path)\n    for line in f:\n        values = line.split()\n        word = values[0]\n        coefs = np.asarray(values[1:], dtype=\"float32\")\n        embeddings_index[word] = coefs\n    f.close()\n\n    if verbose:\n        print(\"Loaded {} word vectors\".format(len(embeddings_index)))\n        print(\"Preparing embeddings matrix...\")\n\n    mean_word_vector = np.mean(list(embeddings_index.values()), axis=0)  # type: ignore[arg-type]\n    embedding_dim = len(list(embeddings_index.values())[0])\n    num_words = len(vocab.itos)\n    embedding_matrix = np.zeros((num_words, embedding_dim))\n    found_words = 0\n    for i, word in enumerate(vocab.itos):\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[i] = embedding_vector\n            found_words += 1\n        else:\n            embedding_matrix[i] = mean_word_vector\n\n    if verbose:\n        print(\n            \"{} words in the vocabulary had {} vectors and appear more than {} times\".format(\n                found_words, word_vectors_path, min_freq\n            )\n        )\n\n    return embedding_matrix.astype(\"float32\")\n</code></pre>"}]}
\ No newline at end of file
diff --git a/mkdocs/site/sitemap.xml b/mkdocs/site/sitemap.xml
index f81fce34..5a8ffe99 100644
--- a/mkdocs/site/sitemap.xml
+++ b/mkdocs/site/sitemap.xml
@@ -2,178 +2,178 @@
 <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/index.html</loc>
-         <lastmod>2024-09-22</lastmod>
+         <lastmod>2024-10-21</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/contributing.html</loc>
-         <lastmod>2024-09-22</lastmod>
+         <lastmod>2024-10-21</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/installation.html</loc>
-         <lastmod>2024-09-22</lastmod>
+         <lastmod>2024-10-21</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/quick_start.html</loc>
-         <lastmod>2024-09-22</lastmod>
+         <lastmod>2024-10-21</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/examples/01_preprocessors_and_utils.html</loc>
-         <lastmod>2024-09-22</lastmod>
+         <lastmod>2024-10-21</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/examples/02_model_components.html</loc>
-         <lastmod>2024-09-22</lastmod>
+         <lastmod>2024-10-21</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/examples/03_binary_classification_with_defaults.html</loc>
-         <lastmod>2024-09-22</lastmod>
+         <lastmod>2024-10-21</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/examples/04_regression_with_images_and_text.html</loc>
-         <lastmod>2024-09-22</lastmod>
+         <lastmod>2024-10-21</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/examples/05_save_and_load_model_and_artifacts.html</loc>
-         <lastmod>2024-09-22</lastmod>
+         <lastmod>2024-10-21</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/examples/06_finetune_and_warmup.html</loc>
-         <lastmod>2024-09-22</lastmod>
+         <lastmod>2024-10-21</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/examples/07_custom_components.html</loc>
-         <lastmod>2024-09-22</lastmod>
+         <lastmod>2024-10-21</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/examples/08_custom_dataLoader_imbalanced_dataset.html</loc>
-         <lastmod>2024-09-22</lastmod>
+         <lastmod>2024-10-21</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/examples/09_extracting_embeddings.html</loc>
-         <lastmod>2024-09-22</lastmod>
+         <lastmod>2024-10-21</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/examples/10_3rd_party_integration-RayTune_WnB.html</loc>
-         <lastmod>2024-09-22</lastmod>
+         <lastmod>2024-10-21</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/examples/11_auc_multiclass.html</loc>
-         <lastmod>2024-09-22</lastmod>
+         <lastmod>2024-10-21</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html</loc>
-         <lastmod>2024-09-22</lastmod>
+         <lastmod>2024-10-21</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/examples/13_model_uncertainty_prediction.html</loc>
-         <lastmod>2024-09-22</lastmod>
+         <lastmod>2024-10-21</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/examples/14_bayesian_models.html</loc>
-         <lastmod>2024-09-22</lastmod>
+         <lastmod>2024-10-21</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/examples/15_Self_Supervised_Pretraning_pt1.html</loc>
-         <lastmod>2024-09-22</lastmod>
+         <lastmod>2024-10-21</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/examples/15_Self_Supervised_Pretraning_pt2.html</loc>
-         <lastmod>2024-09-22</lastmod>
+         <lastmod>2024-10-21</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/examples/16_Usign_a_custom_hugging_face_model.html</loc>
-         <lastmod>2024-09-22</lastmod>
+         <lastmod>2024-10-21</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/examples/17_feature_importance_via_attention_weights.html</loc>
-         <lastmod>2024-09-22</lastmod>
+         <lastmod>2024-10-21</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/examples/18_wide_and_deep_for_recsys_pt1.html</loc>
-         <lastmod>2024-09-22</lastmod>
+         <lastmod>2024-10-21</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/examples/18_wide_and_deep_for_recsys_pt2.html</loc>
-         <lastmod>2024-09-22</lastmod>
+         <lastmod>2024-10-21</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/examples/19_load_from_folder_functionality.html</loc>
-         <lastmod>2024-09-22</lastmod>
+         <lastmod>2024-10-21</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/examples/20_Using_huggingface_within_widedeep.html</loc>
-         <lastmod>2024-09-22</lastmod>
+         <lastmod>2024-10-21</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/pytorch-widedeep/bayesian_models.html</loc>
-         <lastmod>2024-09-22</lastmod>
+         <lastmod>2024-10-21</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/pytorch-widedeep/bayesian_trainer.html</loc>
-         <lastmod>2024-09-22</lastmod>
+         <lastmod>2024-10-21</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/pytorch-widedeep/callbacks.html</loc>
-         <lastmod>2024-09-22</lastmod>
+         <lastmod>2024-10-21</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/pytorch-widedeep/dataloaders.html</loc>
-         <lastmod>2024-09-22</lastmod>
+         <lastmod>2024-10-21</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/pytorch-widedeep/load_from_folder.html</loc>
-         <lastmod>2024-09-22</lastmod>
+         <lastmod>2024-10-21</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/pytorch-widedeep/losses.html</loc>
-         <lastmod>2024-09-22</lastmod>
+         <lastmod>2024-10-21</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/pytorch-widedeep/metrics.html</loc>
-         <lastmod>2024-09-22</lastmod>
+         <lastmod>2024-10-21</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/pytorch-widedeep/model_components.html</loc>
-         <lastmod>2024-09-22</lastmod>
+         <lastmod>2024-10-21</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/pytorch-widedeep/preprocessing.html</loc>
-         <lastmod>2024-09-22</lastmod>
+         <lastmod>2024-10-21</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/pytorch-widedeep/self_supervised_pretraining.html</loc>
-         <lastmod>2024-09-22</lastmod>
+         <lastmod>2024-10-21</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/pytorch-widedeep/tab2vec.html</loc>
-         <lastmod>2024-09-22</lastmod>
+         <lastmod>2024-10-21</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/pytorch-widedeep/the_rec_module.html</loc>
-         <lastmod>2024-09-22</lastmod>
+         <lastmod>2024-10-21</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/pytorch-widedeep/trainer.html</loc>
-         <lastmod>2024-09-22</lastmod>
+         <lastmod>2024-10-21</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/pytorch-widedeep/utils/index.html</loc>
-         <lastmod>2024-09-22</lastmod>
+         <lastmod>2024-10-21</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/pytorch-widedeep/utils/deeptabular_utils.html</loc>
-         <lastmod>2024-09-22</lastmod>
+         <lastmod>2024-10-21</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/pytorch-widedeep/utils/fastai_transforms.html</loc>
-         <lastmod>2024-09-22</lastmod>
+         <lastmod>2024-10-21</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/pytorch-widedeep/utils/image_utils.html</loc>
-         <lastmod>2024-09-22</lastmod>
+         <lastmod>2024-10-21</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/pytorch-widedeep/utils/text_utils.html</loc>
-         <lastmod>2024-09-22</lastmod>
+         <lastmod>2024-10-21</lastmod>
     </url>
 </urlset>
\ No newline at end of file
diff --git a/mkdocs/site/sitemap.xml.gz b/mkdocs/site/sitemap.xml.gz
index eb4fa3c7a0cdc377ce90efccf1fc13b1170abe60..fc00092d75d415151a7962dbe6542540569ce4f3 100644
GIT binary patch
literal 795
zcmV+$1LXW4iwFpSnH6UO|8r?{Wo=<_E_iKh0M(gIZ`&{ohVT6qMed1{^m}MiV21$%
zieXq=4A_NWWZD)gOP(kv&A%Vzr0vk6yKF!>Ut%kc^g!fGzECc&A6s|0qR2M*tI=#S
z9UYQ4p|rlf8r|Q08lR7@7YCPpbWG_`UiN%7YEtSx6vcYIp0G{SV#kvZ>w--q4_xd9
zi|OQIv^Y4FD^IY!8?H+i%p%XiGEKKhA)02qwk4HB-6Rs0si86$p4d=WU((~GNv*pq
z^6!412068%T+FBQ<MC`dp3mf!-Pm^B;(6h}K`_CmXqSDGZP_o0^_<W-OcJmClKkwg
zc>tk_X}=)!h;8SHi|GuyNL>tum_uYhUxLKUu@=BN$lT<>gtiNwd{RSp1WW5NZotSo
zY%6Pq6oa)*4U|-2@ASYPL!>%Nke>-{i#2Tnp6F3cLY@HQiUuqfu-rvIKZ<H>g+|Yw
zo<e0krQUCG*BINr(*kz}rsot|g~B<1LMt)hjRcvZWaAPYdPC(fDxFgc?FEQBqU4=4
zGN5)zW$yG^Hgh(GqgX;G;@E)oNkl$Wq477|+~v&t?muclodNa+S}BWmD4ANg=J4b5
z&6k_N3?bV3fg&=j7<Mzj^9)LOk9I||7rDfQ67@~);-xkFw>U49$FRf=u|>bdRTJ(L
zxFuJ?ZQoH`S(b|PHIgXYXMuNV_MdRhwQ!!oJ=@v?K9%*Ruj|Yx&6>8c*MfQm6=CWl
zNx$iWXh#$TD~o_BNrxP~5!t#?(<^7^L-iT_MSc;g5XH$E-ms=r{UfECM7`KL$|_2Z
zR>5W4QArKWVMk}vMhWM9ngu2L+39GIQMT;XD``cqzb{l(`uFLic#2_<%J2+2w?y-x
zZQHUBcY{0EM)q39>hPFJ+n%*z9jsADrRr?w4c^Py*xpYu->`8uhx}p=yTKe<ua7k0
za=s#?jjEySQF6=b9chD8*{OzuVGVySyhE06CE0q1QNtnjJCv0yg*KZJ&tjBYos`4z
Zu;N|JXV(7$Z}?zJi(iqXm8{qs000Y=kO2Sy

literal 796
zcmV+%1LOQ3iwFn+UhifC|8r?{Wo=<_E_iKh0M(gMYuhjo$KUrUO5PJEZo9UlO<@m%
zK^bG~80_V^$Y)zrmOP!DG~a$FCv8Vd_p(85eu=F((huGJ@BS#4*Y}+}T2o{j{MBSJ
zn@^6&n^0NbTur{+ewdz5u9t5vhv=Bnk-Y5rYSO0EzblH(W;0`(sKcITAvOh@Rvx(6
z4;J&;#bo*BNUl7=?ryv;T`<c$3(GY9Hic-~>Bd%65%sf3Sf!S#V0dOjVSPpSvo>|^
zvdF*teH!G{g=$&O%aiH+Vp^8+%6@FOZt=MA-yoRaQ?#og$+jGp#ClHX943j^VM%@t
z*4%;6#B^8?y2q|}#Kn98eWX4HL(CyEpszq;=2#102{Jc1Frn*%C!f@i9mC3cj9W0W
z4%^zAF~wkGQwtT<I5<79ClINL668lh+hIeyfG4_FlaQytxTX=y1+4bb&yS)STcgpl
zr*EOQp3>lVxEqY!&})G^12b?6okHQ9KcH2Z@J516QL=G`4!xmj9F@+gh4unO9Z~X5
z8W~WxqAGWKEt|QR!*Q&j7jbOC`XnMBtI+fdZf|pDe)sRSpe_Ih1D%vbJC;l>TqXSY
zc=PEdFhhv8xub{-Yli&{@Hm4K-s62y97HZLp+tR~yLe^I;VsS+<q52COKj2aaMgtS
z6uy$H;p@;-Tw9ik^Er|z+-HIJY4M+MmRdO9!Z)^!2Ye{&?a(xtQJOXFVy^}D3~ItO
zM3R2f2holw2-X$>Q<4rjdLyz;tEN{j&d2IA`iuM`)FFzKGrVO@srpAsHHmt$bCh+I
z9Ib=PwxgCBn!}FHrmYgra-Ibx`q}Gflu@?q)+=d6&%ZBJRr=TIq<D&9kIL{2I=4b|
zr)}G^4|k(G*GBeS#_I5xN!y-vVjZkeN2Tg)=LO!=+1SBPF<-E8Hi!IV4*Ss@TCa~Z
z;i_Dd(MHu$_9(e!4UV+Isq9qa!LWuu7hWODw~}mw!>Hj9`xVMsmO`7&h-Wd%oleT}
acv$f+<}>Smfj53IrR6W5N|8s{8vp=Y^OuwW


From 5e45588f9e638d546a99aa74ad1e9d2f0ebf8e48 Mon Sep 17 00:00:00 2001
From: Javier <jrzaurin@gmail.com>
Date: Mon, 21 Oct 2024 10:34:04 +0100
Subject: [PATCH 13/24] remove unnecessary, legacy docs folder and modified
 figures path in README

---
 README.md                               |  22 +-
 docs/Makefile                           |  21 --
 docs/_static/custom.css                 |  45 ----
 docs/_static/img/widedeep_logo_docs.ico | Bin 102115 -> 0 bytes
 docs/bayesian_models.rst                |  15 --
 docs/callbacks.rst                      |  29 ---
 docs/conf.py                            | 297 ------------------------
 docs/dataloaders.rst                    |  10 -
 docs/examples.rst                       |  20 --
 docs/figures/arch_1.png                 | Bin 58023 -> 0 bytes
 docs/figures/arch_2.png                 | Bin 51944 -> 0 bytes
 docs/figures/arch_3.png                 | Bin 61499 -> 0 bytes
 docs/figures/arch_4.png                 | Bin 74000 -> 0 bytes
 docs/figures/arch_5.png                 | Bin 76885 -> 0 bytes
 docs/figures/arch_6.png                 | Bin 123034 -> 0 bytes
 docs/figures/arch_7.png                 | Bin 57736 -> 0 bytes
 docs/figures/arch_8.png                 | Bin 33393 -> 0 bytes
 docs/figures/architecture_1_math.png    | Bin 47788 -> 0 bytes
 docs/figures/architecture_2_math.png    | Bin 16848 -> 0 bytes
 docs/figures/widedeep_arch.png          | Bin 157281 -> 0 bytes
 docs/figures/widedeep_arch_new.png      | Bin 96559 -> 0 bytes
 docs/figures/widedeep_logo.png          | Bin 85253 -> 0 bytes
 docs/index.rst                          | 160 -------------
 docs/installation.rst                   |  44 ----
 docs/losses.rst                         |  65 ------
 docs/make.bat                           |  35 ---
 docs/metrics.rst                        |  51 ----
 docs/model_components.rst               |  71 ------
 docs/preprocessing.rst                  |  22 --
 docs/quick_start.rst                    | 134 -----------
 docs/requirements.txt                   |  20 --
 docs/tab2vec.rst                        |   7 -
 docs/trainer.rst                        |  23 --
 docs/utils/deeptabular_utils.rst        |   6 -
 docs/utils/fastai_transforms.rst        |  19 --
 docs/utils/image_utils.rst              |  20 --
 docs/utils/index.rst                    |  23 --
 docs/utils/text_utils.rst               |  15 --
 38 files changed, 11 insertions(+), 1163 deletions(-)
 delete mode 100644 docs/Makefile
 delete mode 100644 docs/_static/custom.css
 delete mode 100644 docs/_static/img/widedeep_logo_docs.ico
 delete mode 100644 docs/bayesian_models.rst
 delete mode 100644 docs/callbacks.rst
 delete mode 100644 docs/conf.py
 delete mode 100644 docs/dataloaders.rst
 delete mode 100644 docs/examples.rst
 delete mode 100644 docs/figures/arch_1.png
 delete mode 100644 docs/figures/arch_2.png
 delete mode 100644 docs/figures/arch_3.png
 delete mode 100644 docs/figures/arch_4.png
 delete mode 100644 docs/figures/arch_5.png
 delete mode 100644 docs/figures/arch_6.png
 delete mode 100644 docs/figures/arch_7.png
 delete mode 100644 docs/figures/arch_8.png
 delete mode 100644 docs/figures/architecture_1_math.png
 delete mode 100644 docs/figures/architecture_2_math.png
 delete mode 100644 docs/figures/widedeep_arch.png
 delete mode 100644 docs/figures/widedeep_arch_new.png
 delete mode 100644 docs/figures/widedeep_logo.png
 delete mode 100644 docs/index.rst
 delete mode 100644 docs/installation.rst
 delete mode 100644 docs/losses.rst
 delete mode 100644 docs/make.bat
 delete mode 100644 docs/metrics.rst
 delete mode 100644 docs/model_components.rst
 delete mode 100644 docs/preprocessing.rst
 delete mode 100644 docs/quick_start.rst
 delete mode 100644 docs/requirements.txt
 delete mode 100644 docs/tab2vec.rst
 delete mode 100644 docs/trainer.rst
 delete mode 100644 docs/utils/deeptabular_utils.rst
 delete mode 100644 docs/utils/fastai_transforms.rst
 delete mode 100644 docs/utils/image_utils.rst
 delete mode 100644 docs/utils/index.rst
 delete mode 100644 docs/utils/text_utils.rst

diff --git a/README.md b/README.md
index 2db68794..30ca9537 100644
--- a/README.md
+++ b/README.md
@@ -1,6 +1,6 @@
 
 <p align="center">
-  <img width="300" src="docs/figures/widedeep_logo.png">
+  <img width="300" src="mkdocs/sources/docs/figures/widedeep_logo.png">
 </p>
 
 [![PyPI version](https://badge.fury.io/py/pytorch-widedeep.svg)](https://pypi.org/project/pytorch-widedeep/)
@@ -60,7 +60,7 @@ Figure below:
 
 
 <p align="center">
-  <img width="750" src="docs/figures/widedeep_arch_new.png">
+  <img width="750" src="mkdocs/sources/docs/figures/widedeep_arch_new.png">
 </p>
 
 
@@ -69,7 +69,7 @@ In math terms, and following the notation in the
 without a ``deephead`` component can be formulated as:
 
 <p align="center">
-  <img width="500" src="docs/figures/architecture_1_math.png">
+  <img width="500" src="mkdocs/sources/docs/figures/architecture_1_math.png">
 </p>
 
 
@@ -164,7 +164,7 @@ Face models.
 
 
 <p align="center">
-  <img width="400" src="docs/figures/arch_1.png">
+  <img width="400" src="mkdocs/sources/docs/figures/arch_1.png">
 </p>
 
 
@@ -210,7 +210,7 @@ trainer.fit(
 **2. Tabular and Text data**
 
 <p align="center">
-  <img width="400" src="docs/figures/arch_2.png">
+  <img width="400" src="mkdocs/sources/docs/figures/arch_2.png">
 </p>
 
 
@@ -262,7 +262,7 @@ trainer.fit(
   in `WideDeep`**
 
 <p align="center">
-  <img width="400" src="docs/figures/arch_3.png">
+  <img width="400" src="mkdocs/sources/docs/figures/arch_3.png">
 </p>
 
 ```python
@@ -313,7 +313,7 @@ trainer.fit(
   `WideDeep`**
 
 <p align="center">
-  <img width="500" src="docs/figures/arch_4.png">
+  <img width="500" src="mkdocs/sources/docs/figures/arch_4.png">
 </p>
 
 ```python
@@ -375,7 +375,7 @@ trainer.fit(
   `ModelFuser` class**
 
 <p align="center">
-    <img width="500" src="docs/figures/arch_5.png">
+    <img width="500" src="mkdocs/sources/docs/figures/arch_5.png">
 </p>
 
 ```python
@@ -454,7 +454,7 @@ property and avoids some typing errors internally.
 
 
 <p align="center">
-    <img width="600" src="docs/figures/arch_6.png">
+    <img width="600" src="mkdocs/sources/docs/figures/arch_6.png">
 </p>
 
 
@@ -596,7 +596,7 @@ target). We can create a two-tower model where the user and item features are
 passed through two separate models and then "fused" via a dot product.
 
 <p align="center">
-  <img width="350" src="docs/figures/arch_7.png">
+  <img width="350" src="mkdocs/sources/docs/figures/arch_7.png">
 </p>
 
 
@@ -692,7 +692,7 @@ This one is "a bonus" to illustrate the use of multi-target losses, more than
 actually a different architecture.
 
 <p align="center">
-  <img width="200" src="docs/figures/arch_8.png">
+  <img width="200" src="mkdocs/sources/docs/figures/arch_8.png">
 </p>
 
 
diff --git a/docs/Makefile b/docs/Makefile
deleted file mode 100644
index c1d05ae8..00000000
--- a/docs/Makefile
+++ /dev/null
@@ -1,21 +0,0 @@
-# Minimal makefile for Sphinx documentation
-#
-
-# You can set these variables from the command line, and also
-# from the environment for the first two.
-SPHINXOPTS    ?=
-SPHINXPROJ    = pytorch_widedeep
-SPHINXBUILD   ?= sphinx-build
-SOURCEDIR     = .
-BUILDDIR      = _build
-
-# Put it first so that "make" without argument is like "make help".
-help:
-	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
-
-.PHONY: help Makefile
-
-# Catch-all target: route all unknown targets to Sphinx using the new
-# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
-%: Makefile
-	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
diff --git a/docs/_static/custom.css b/docs/_static/custom.css
deleted file mode 100644
index cbe8279e..00000000
--- a/docs/_static/custom.css
+++ /dev/null
@@ -1,45 +0,0 @@
-body a {
-  font-weight: bold;
-}
-
-.math {
-    text-align: left;
-}
-
-.eqno {
-    float: right;
-}
-
-div.ethical-rtd {
-  /* hide ads */
-  display: none;
-}
-
-.rst-content dl:not(.docutils) dl dt strong {
-  padding-left: 6pt;
-}
-
-.rst-content dl:not(.docutils) dl dt span {
-  color: #777;
-}
-
-.rst-content dl:not(.docutils) dl dt span.classifier-delimiter {
-  padding-left: 6pt;
-  padding-right: 6pt;
-}
-
-.rst-content dl:not(.docutils) dl dt span.classifier {
-  padding-right: 6pt;
-}
-
-.rst-content dl:not(.docutils) dl dt a.reference.internal span.xref.std.std-term {
-  color: #2980B9;
-}
-
-.wy-nav-content {
-    max-width: none; !important;
-}
-
-div.container a.header-logo {
-  background-image: url("../figures/widedeep_logo.png");
-}
diff --git a/docs/_static/img/widedeep_logo_docs.ico b/docs/_static/img/widedeep_logo_docs.ico
deleted file mode 100644
index 5e699b7ea222f5e43e05c969cde6c92a4b110b4d..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 102115
zcmeHQ30zHS8$VZ>Qqon{LRnIZ2$ikP7G^9%#Au<0GE@jjQTXgF-+WoJkNJi%W7Ib?
zBcpCn2~iAVNh(YeSwhB@(*2&3t~uTJw%u;&dY|+AJ)Lu&cYXf<=UvWw-_Bte9`gyq
z9y~^$S=xeO=E8Mw@Vk2>RUUI2bhfq?_rn=RNsGty>-X;dumQuEI6zAMihDl8T=nNM
zd^ofDc`7qZz-At^Y0?CH4K*D#$Xa9Un30p&P*zmgqMa5WX3OOHES})N%lOuB7sIsZ
zJ9ea<!}820cjkvUc<|0X7g~AD^;ovUIVx?bI<MK7UOUzd*KV=3-IdWT4~2SpxtToI
z(YgHNc=O1fp$|8>jrn=n*q{E>p_hx6`_+L{b&b~j{`>kbN`u?*4rHyLqm*{$Y5K*T
zY5s46%^YU=HDB)+nJ+3C6L|{?C>!b_yclH$m61HW)vgN}hiJ`dK_*v<U%i^$^Xom^
zhL%P@-1OJ#J{GE@JGB2k<Y(taT`z3)z4dDS^N3~bhGulqITe_4a*>JPeqiBfQEMhJ
z*~T$?pmTsthdVJrg-LlvZ}yn9v3`1Qv9s%rnA!J^Ypgx7#^H_M_$fXC^Af+CSu|yz
zf4cCJot`QaVs>^={>9zCJty?>^RseEpQC4yzRoeg*+MmQLF95%v*qXBG%wUEc)9ZS
z!a2&!+>$*lE*&h&7wF{|9SQLbz42u8Q7bp?^M4!_W+%+Mkm<V6bJ9B1;Mk)PGv}UN
z=egEoo44gQ|6yCb_+{3g9h{c!pSxs3;q5Kq-I#B)XAH~uYk>WZt-qy<t`D+|y{Fet
zb(vv-?o-#;=(peGeeP)f+Ceq*<ihRePt190;t{ZFrL*e`VM^u*yCLz9qm(>ey%uOL
zw95@(yyFhXd!5$^dZFtc;;Ze{CO*z<#iQjpLY`@E+u%s3$)Rn&w=6?^lj1^E*CcGX
zbji;&Gh%RlN!g@S6Q-+A`tED|_TRRTvn~!b>lx;ea_jLfKkK3U``Yen&2&u};4$sw
z<cF<th5}A))m%fZhaFEIylTh;Z65#U`w=HXH><DHi!2!YSbdz8)%<Liele$o{5i_D
zwk6kXBG3GAQTSOW%aZKHW{$?`MtciO*A#ZXpc)*kdE+PD$D32~1b$8vPJSPdC-@`$
zU|`{-QH=7RVV#TPOm>(A9m|-y@xPwaQp1kMtYkyJ%K9N8&MmFDG#(Ot?K}JV#fNhQ
z9*4Uz=1<RM+-PNCD!3yI%T7Lc?|XHR+eh+tM@&|=4NdE8m8E5Wr9;lp725g7`s^H^
z@PjaZL~GvRMYD~qvV`lc2j5ukcJ-44L&Hswk?B`yAur+&^fe9Bjj?Z~&1ganzIng%
zb9CQyop&^kpQzMqQi+y8<)umv@8w$Nn!r0p>CJ)1drV$;Ik1tr7W?Pr*@i299z^`&
zY_~G0SI&Sot%G%Tcz)$w6o0_y*Hd{<Gme+Ilt+x6^x%J+zDyT}xy<nWGBNH{k-)<U
z%8&Z{K=koj`Zpg1*!!#*>6ZFBZ)&m2!Cm%X`)ePY&s#ovBr0Homb^V8umsDS{U^zO
zLI1N3dC}QF+b!&8w(xR^z>n1zq=wvS|J~K-;_O9(?Tq8iif&G^ObR!iua)n*@ur3Q
z<8R$kAND!4**f1sHT}@~@IBU|__c<CKA)$UFs<E#UR=|)*cq524BOKDhnGeFO(?w-
z0<8V`*AbqvPszoZJ=+Z#vh2>kZJJ+K(Os{ydv3;FE8)bMzd`cROmdv{yazk(IeD*r
zUeaZ?n{(R6iG6OaT)w-oWJKo;jN6%+Jr^9G?Y8#>@A~$L5$mn3@7vqtop)`n8XUPU
zx^VGBGxt@V5$9dApJ{q~b+TwR{l=hq7cxHS<6KY@6?~U(dp*ow?^F9HhWnSM+_StG
zy1di$nd1hUM~@HpUil*I(y8V7uG_%j4;L3cO$$F&xFtMu<cqQHUUM>%G}k!Y$vqI>
zYL{h@`NXn<GbT*r(J5a%-CR`sca+n<!1(B)hmYLe`&DP*FLsYC-+E;ppC|A!E>qzJ
zFG}CJcgtMOjr~&-bnUPF_~`lk<OTixdoNh9V(#l>DlV~qKl~-|kLbzDjA-eDV!hp0
zgI4*EwDw&v>g1_M2NqlJ@^s27Ox>uF<TlAs=;CHu6kRYn?9Kfw$DD%W+c!)%zaO(U
zEwdzf^P$J9-}=5VoS5alr)1={?HxVt4H|g%3->*X(o@c!yxB9tKFMLl#LJgsZ3cUu
z+kQpU`dCob4ae<)h4vyo^YZ*)kI#f-Z@<d?^~)P&LCPsn6RutxcI?`Kr7Qdkj?I}`
zY&zm)hJnXgFK;i^5VNUE(+9<UG3Th2r)k{Y*c<M}myT*IJsY{hI5saOB{in6%ae7M
zCoEc;on7!#u0W-|UdFzJ^((_WxOX|6D>%3%+~fWh(cN+1ug#uvx^?WymZdgjKAy%U
zH~ob=efLK_6Z~oJvTEEqq4Sy|hMBF*@JGw#wCtd(`l>`>KZnqBFi4~u@Lp%=wTgWj
zWYEs`@4I|?l_H<IVlHwby&-_y{>+LoH}DDR=yLi|hRglD+s1K)kIh}m=DA;tX`9vF
z@!4<ev`O-?9aXCP&F$txH3K8ZvteULO&A$9eAXHxxDI9&kcgd(y*qe}K|7{gDinA^
z1P}p401-e05CKF05kLeG0Ym^1Km-th3Ix<T#We%jTnF)}4m`+5r4D(q<&LW%zaDkW
z4;C=+s4W73tulzH4w6t^-jI(*9r9w!-Bd$<BkGvn>gw{tu*wJkw#a{FNT|9){;8sy
zFc$eIAfZZu{8L3YVJz}bKth!Q`KO9*!dT>=fP^Xq@=q1rgt5p!0SQ$K<ew_K31g9e
z0urhe$Ujwd6UHL{1SC``kbkP^CX7Y?2}r0?Apca+O&E*(6Od4)K>n$sn=lsnCm^9p
zf&5cNH(@ODPe4MI0{N$kZo*jPpMZoa1@cc7-Gs5oKLH6<3gn+Ex(Q>Ee*zM!6v#hS
zbQ8uR{{$pdDUg4v=q8Ls{s~B^QXv0S(M=eO{1cE+r9l3vqMI-l`6nQuN`d@SMK@tA
z@=riQl>+&vif+PK<ez|qDh2XS72Slf$UgxIRSM*vD!K_{k$(absuajSRdf@^BL4&=
zR4I^us^}(+Mg9p$s8S&RRMAZsi~JLiP^CcrsiK=O7WpS2p-O@LQ$;snEb>o4LX`sf
zr;2XESmd98genE{PZiySvB*CG2~`T@pDMZuW08LX5~>u)KUH)S#v=a&BvdJof2!yv
zj79zlNT^aE|5VXU7>oQ9kWi&S{;8syFc$eIAfZZu{8L3YVJz}bKth!Q`KO9*!dT>=
zfP^Xq@=q1rgt5p!0SQ$K<ew_K31g9e0urhe$Ujwd6UHL{1SC``Xy6}gv;^@&iBp$!
zZc6xH4R(r9;&hOIs{H`@r@J3O{*nJmLjaEdRE-DN{;~aI`={3ju>I4^J^TEN{3HL^
z|I_OO*#7C|p634F8^*p&l(=@4I)AvSc>b>oHs49+!M%t4)a!`zQf_+5YshPG9rN>+
z%meQk=F{ov+D3ztC-b`r7V*2qFKO8&exY^`!Bj(w<em|`?(z_$j}HNnNW^sLmC#+Q
zyWjxu{R-L<@f(<W=eRP>uJISz_Dpc}4Y=Oy<3oW?i#(TPF&cXDUcj$Z-lh41rhdFY
zzuy^Rjz}khifeVRAt}v(`!H!7*QA4VxgCuY4HVaUtT}M#UlY&Og+uyV-TEi-5lO{E
z0LFR?;I^bXerpp3W5KSFt=WnP0L3+)=#;^0r57)#4Zk(TQ3oGtZ<J_AF`No#I?UaA
z*TidG!eJcPq;McmTw$#qf_jbnlD+}Y4!`f;pF=UCie)-H$M2Qous-*||I6+J&ZsFC
z3iQ?%xQ?k0-%`?+fe&<|7g5Es9Oi7%QaB{m{OS;g<zH8^P@uQAt@Ps~>%eV|e!=$t
zBPU<P{$D{{s*ORye0lN@zZLyaQ20hctjB8Vo1N(lZMNQPKVl!!Hb_{5s45NuKPD6~
zu>NyM%wK)ZY~PQ+1t|{q;+n!3pby+Vs}J{VT3G+}V}C!!L-7y*?i_)`QdxbVWtX@Z
zo!$w`if1|2m_?wUO)3L(w=aSJI?n}Kc8)u)Z6F~2PaKg|JOsc8MnGH6u4O+Uggrt*
zZ445dE1soT<Ejwo*8hwe{4U@HYXccDFDR4r1>pQI2>Tj-H||zd9;jD*1bhN+@XSZ1
zcF^s8+P3?Eq{%S1x9?(l+OYSq6ddOjpI)>85kLeG0Ym^1Km-s0L;w+>9syXt{xC<d
z(rd;hSm}M?c#n9XKVW+hcL*P8e~?%);;2;Er|9=$x^nfzSMSb@N~QZU<{!^y14<cQ
zrSMXQsdWFEVS>1dXEop3NW_=-eUL={mF^|$D`}6Uy&?%bM8TTe>_wtno67eMb!P<r
zXbTby!ejK|FC4ZEV*qbKU<D6E01;?<2*7VtJdjaRWEi)$5Z+8m95$V~M7$PN#2d(s
z*Ho@6?2@)Wx6(i3K)g>tq{U0-+5&O9RuM0e8?UKcLtR;3rP+n{b4UV=kbmiZg8pCG
zPq6(<_me~pOiktz`#<deas0#aUy2Xl{D;#^<NOEbe^S~A``m!@zsl_d=YKf=llDBF
z$Z?Y<a|!JipZ}%%D~|shFU9d6$A29Er9LO&{HJn1f%6~f<4Gb1rY3WV<3G-Ss_55o
z{=;ddIRC-<PZjM1$N$Rh1jm0I|8f0;^npZ9GifrHIRC@>UlrpT&i^>ARG#xc$g3BK
z<p&le);y>~qp$x$nTa6M#)!t$X`ubcKl(qm|0?>T1{?pNykj6wKCrap^ncRolEH?H
z52!1<|5aMNU&V3QGsMxWBHl4>ybpB^-{@=6y1QULyoc$3@Q$XF@ZF7la6bmV$M>PC
z)M(|x`$~I&%{PG`Vw?pWqTpL(EgG#dja(4Cb4vs4&XQ)kSf|k?{+o~WJ50d+3bCEl
zoZ0VF_HJZq8l^aR_gYFdcE4{A&_0$lN`)G^5U^cpyO)AVXd_pxQHlfpYxT?l?{ls8
z8_A7Q*#;;C`gb2GT)+4Gim=kQh6x|#cgWyf&8pD0uT+NtyuWh`{C{i%px-pIgm;v7
z1PoH8VDJ|7Vel^6@-=~`QQL;81-zqE32>MQ^S^B{)*XT||6kDN`!q~xm@6WH2yhXo
z?)?MLI;oVu%>mDK#~~O%<^z6pJ+pjNyh5X_;?)_Y6|c;w>sg#$RQVMhl^JK>6VVT3
z7RdYeMD)P-L;xJr9xxwapUnqKh^iP6&>KmFb>W0*NrbWaH0nGC%CiHJoJ+H=UVcDW
zvP5_UCv1j97@JR{eh(bV!}?d|5Ahc}Xf4j@KSbL{|EY!Fh@+tYp#O+{L6$T65BiTR
zej`qU{)7G__61qa=s)N`viOZS4f+rIkJuMvIivre|H$Gu;xy<#=s#j#kmcME{sVa`
zfw24J8&b;OOx-~B=R>}dYr^ur-^B8T^(Sy<u}A!C_P>Vo9~IzWaRZ7k?Bno;{TrXc
z{*BL|oFx)vX^HzX(6is@=GJ96Cyf1Vi5tdVtBAuJ;tYE++rWOC-=G}vH)6|R?@<Em
zL)K}C^5mH(>=9Oix}S!+6MY%z&%-xQo5{00dFRFN8gIqHov;D?$wuB~$}>N}c(yd`
zIXVw{mM8DLpskMK)Rn*=_GC?zcbW3cuVNpe)V;L8O}36n)89i6dqGX0t~nAs<iH-;
zzVgJXA@hQLs%o%@ZwB~F0N68w-&LU2kmX_Cjf=p$T|?rp`IMh?XC=~e2DJ5AduwrK
z-r-UP+pbuAN?|)!ZO-EOBKX2mIj#WtKWBFHiav-f$oqX-@T>O6`aK`$3tNM9uMkZz
zn|-d*;^@TxFWg2_$HuMi0eP+l5s!xp+5F&sEl1~8q5C&SH;dKD<A6M|e8eC9@0|}|
z`-4se`ycFoIb#iuKkwQEj=%qmle6S8_h9pob^ZbA)IfCWDZF}=KMT@|$4yz!?7J0Y
zntwsM4_&?!)ZtY<g|$DBlgEMzpIF7wxm4)3aCA<rPSyj~VOzkuYXq!?WP+T8^#oH{
z^O7YUl(`dZ;rdKItY@gpl9w!LpzM5!@<mW5TUqjwB@L97CQ*JFtm!w}{~rVQ!=U`<
z;yOTmHk3aE%VLa-X<By^w1G9ziLi#Hkndf1w@!10FA~X57^WGlH2fo?GWe)*87xz9
zMWk0=G`mg#k;~UA-mg<A%eGDde!%`8QcDnJ5OzI){SCSmNcp>RYN8n8C4!s)u>)b-
zwHe4ckm#Dyvf&U9(|@G+0k`aTaamugO4#2^tK=WTqCvQ0h2%B6R#aX3sSqAeg{*|W
zZx_gR2%iq3R}rrRoM(Ydt<HZT9UF(OPcd9;R~J`W7^E2vX-h!vLjNz*IzEh>9ib1c
Ya_kfZ!`fyogj>HdJ%(XLWWzB32bp(QjQ{`u

diff --git a/docs/bayesian_models.rst b/docs/bayesian_models.rst
deleted file mode 100644
index 2eb95c60..00000000
--- a/docs/bayesian_models.rst
+++ /dev/null
@@ -1,15 +0,0 @@
-The ``bayesian models`` module
-==============================
-
-This module contains the two Bayesian Models available in this library, namely
-the bayesian version of the ``Wide`` and ``TabMlp`` models, referred as
-``BayesianWide`` and ``BayesianTabMlp``
-
-
-.. autoclass:: pytorch_widedeep.bayesian_models.tabular.bayesian_linear.bayesian_wide.BayesianWide
-    :exclude-members: forward
-    :members:
-
-.. autoclass:: pytorch_widedeep.bayesian_models.tabular.bayesian_mlp.bayesian_tab_mlp.BayesianTabMlp
-    :exclude-members: forward
-    :members:
diff --git a/docs/callbacks.rst b/docs/callbacks.rst
deleted file mode 100644
index e8e65c45..00000000
--- a/docs/callbacks.rst
+++ /dev/null
@@ -1,29 +0,0 @@
-Callbacks
-=========
-
-Here are the 5 callbacks available in ``pytorch-widedepp``: ``History``,
-``LRHistory``, ``ModelCheckpoint``, ``EarlyStopping`` and ``RayTuneReporter``.
-
-.. note:: ``History`` runs by default, so it should not be passed
-    to the ``Trainer``
-
-.. autoclass:: pytorch_widedeep.callbacks.History
-	:members:
-
-.. autoclass:: pytorch_widedeep.callbacks.LRShedulerCallback
-	:members:
-
-.. autoclass:: pytorch_widedeep.callbacks.MetricCallback
-	:members:
-
-.. autoclass:: pytorch_widedeep.callbacks.LRHistory
-	:members:
-
-.. autoclass:: pytorch_widedeep.callbacks.ModelCheckpoint
-	:members:
-
-.. autoclass:: pytorch_widedeep.callbacks.EarlyStopping
-	:members:
-
-.. autoclass:: pytorch_widedeep.callbacks.RayTuneReporter
-	:members:
diff --git a/docs/conf.py b/docs/conf.py
deleted file mode 100644
index bcfaa096..00000000
--- a/docs/conf.py
+++ /dev/null
@@ -1,297 +0,0 @@
-# -*- coding: utf-8 -*-
-#
-# Configuration file for the Sphinx documentation builder.
-#
-# This file only contains a selection of the most common options. For a full
-# list see the documentation:
-# https://www.sphinx-doc.org/en/master/usage/configuration.html
-
-# -- Path setup --------------------------------------------------------------
-# If extensions (or modules to document with autodoc) are in another directory,
-# add these directories to sys.path here. If the directory is relative to the
-# documentation root, use os.path.abspath to make it absolute, like shown here.
-#
-
-import os
-import re
-import sys
-
-from sphinx.ext.napoleon.docstring import GoogleDocstring
-
-# this adds the equivalent of "../../" to the python path
-PACKAGEDIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
-sys.path.insert(0, PACKAGEDIR)
-
-
-# -- Project information -----------------------------------------------------
-
-project = "pytorch-widedeep"
-copyright = "2021, Javier Rodriguez Zaurin"
-author = "Javier Rodriguez Zaurin"
-
-# # The full version, including alpha/beta/rc tags
-# def get_version():
-#     r"""
-
-#     Get the current version number for the library
-#     Returns
-#     -------
-#     String
-#         Of the form "<major>.<minor>.<micro>", in which "major", "minor" and "micro" are numbers
-
-#   """
-#     with open("../pytorch_widedeep/VERSION") as f:
-#         return f.read().strip()
-# release = get_version()
-
-with open(os.path.join(PACKAGEDIR, "pytorch_widedeep", "version.py")) as f:
-    version = re.search(r"__version__ \= \"(\d+\.\d+\.\d+)\"", f.read())
-    assert version is not None, "can't parse __version__ from __init__.py"
-    version = version.groups()[0]  # type: ignore[assignment]
-    assert len(version.split(".")) == 3, "bad version spec"  # type: ignore[attr-defined]
-    release = version
-
-# -- General configuration ---------------------------------------------------
-
-# If your documentation needs a minimal Sphinx version, state it here.
-#
-# needs_sphinx = '1.0'
-
-# Add any Sphinx extension module names here, as strings. They can be
-# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
-# ones.
-
-extensions = [
-    "sphinx.ext.autosummary",
-    "sphinx.ext.autodoc",
-    "sphinx_autodoc_typehints",
-    "sphinx.ext.coverage",
-    "sphinx.ext.napoleon",
-    "sphinx.ext.mathjax",
-    "recommonmark",
-    "sphinx.ext.doctest",
-    "sphinx.ext.todo",
-    "sphinx.ext.viewcode",
-    "sphinx_markdown_tables",
-    "sphinx_copybutton",
-    "sphinx.ext.githubpages",
-]
-
-autosummary_generate = True
-
-napoleon_use_ivar = True
-
-# Add any paths that contain templates here, relative to this directory.
-templates_path = ["_templates"]
-
-# The suffix(es) of source filenames.
-# You can specify multiple suffix as a list of string:
-#
-source_suffix = [".rst", ".md"]
-# source_suffix = '.rst'
-
-# The master toctree document.
-master_doc = "index"
-
-# The language for content autogenerated by Sphinx. Refer to documentation
-# for a list of supported languages.
-#
-# This is also used if you do content translation via gettext catalogs.
-# Usually you set "language" from the command line for these cases.
-language = None
-
-# List of patterns, relative to source directory, that match files and
-# directories to ignore when looking for source files.
-# This pattern also affects html_static_path and html_extra_path.
-exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"]
-
-# The name of the Pygments (syntax highlighting) style to use.
-pygments_style = "sphinx"
-
-# Remove the prompt when copying examples
-copybutton_prompt_text = ">>> "
-
-# autoclass_content = "init"  # 'both'
-autodoc_member_order = "bysource"
-# autodoc_default_flags = ["show-inheritance"]
-
-# -- Options for HTML output -------------------------------------------------
-
-# The theme to use for HTML and HTML Help pages.  See the documentation for
-# a list of builtin themes.
-#
-html_theme = "sphinx_rtd_theme"
-
-# Theme options are theme-specific and customize the look and feel of a theme
-# further.  For a list of options available for each theme, see the
-# documentation.
-#
-# html_theme_options = {"analytics_id": "UA-83738774-2"}
-
-# html_theme_options = {
-#     "canonical_url": "",
-#     # 'analytics_id': 'UA-XXXXXXX-1',  #  Provided by Google in your dashboard
-#     "logo_only": False,
-#     "display_version": True,
-#     "prev_next_buttons_location": "bottom",
-#     "style_external_links": False,
-#     # Toc options
-#     "collapse_navigation": True,
-#     "sticky_navigation": True,
-#     "navigation_depth": 4,
-#     "includehidden": True,
-#     "titles_only": False,
-# }
-
-# Add any paths that contain custom static files (such as style sheets) here,
-# relative to this directory. They are copied after the builtin static files,
-# so a file named "default.css" will overwrite the builtin "default.css".
-html_static_path = ["_static"]
-
-# Custom sidebar templates, must be a dictionary that maps document names
-# to template names.
-#
-# The default sidebars (for documents that don't match any pattern) are
-# defined by theme itself.  Builtin themes are using these templates by
-# default: ``['localtoc.html', 'relations.html', 'sourcelink.html',
-# 'searchbox.html']``.
-#
-# html_sidebars = {}
-
-# This must be the name of an image file (path relative to the configuration
-# directory) that is the favicon of the docs. Modern browsers use this as
-# the icon for tabs, windows and bookmarks. It should be a Windows-style
-# icon file (.ico).
-html_favicon = "_static/img/widedeep_logo_docs.ico"
-html_logo = "figures/widedeep_logo.png"
-html_theme_options = {
-    "canonical_url": "https://pytorch-widedeep.readthedocs.io/en/latest/",
-    "collapse_navigation": False,
-    "logo_only": False,
-    "display_version": True,
-}
-# html_favicon = "_static/img/widedeep_logo_docs.ico"
-
-
-# -- Options for HTMLHelp output ---------------------------------------------
-
-# Output file base name for HTML help builder.
-htmlhelp_basename = "pytorch_widedeepdoc"
-
-
-# -- Options for LaTeX output ------------------------------------------------
-
-latex_elements = {  # type: ignore[var-annotated]
-    # The paper size ('letterpaper' or 'a4paper').
-    #
-    # 'papersize': 'letterpaper',
-    # The font size ('10pt', '11pt' or '12pt').
-    #
-    # 'pointsize': '10pt',
-    # Additional stuff for the LaTeX preamble.
-    #
-    # 'preamble': '',
-    # Latex figure (float) alignment
-    #
-    # 'figure_align': 'htbp',
-}
-
-# Grouping the document tree into LaTeX files. List of tuples
-# (source start file, target name, title,
-#  author, documentclass [howto, manual, or own class]).
-latex_documents = [
-    (
-        master_doc,
-        "pytorch_widedeep.tex",
-        "pytorch_widedeep Documentation",
-        "Javier Rodriguez Zaurin",
-        "manual",
-    ),
-]
-
-
-# -- Options for manual page output ------------------------------------------
-
-# One entry per manual page. List of tuples
-# (source start file, name, description, authors, manual section).
-man_pages = [
-    (master_doc, "pytorch_widedeep", "pytorch_widedeep Documentation", [author], 1)
-]
-
-
-# -- Options for Texinfo output ----------------------------------------------
-
-# Grouping the document tree into Texinfo files. List of tuples
-# (source start file, target name, title, author,
-#  dir menu entry, description, category)
-texinfo_documents = [
-    (
-        master_doc,
-        "pytorch_widedeep",
-        "pytorch_widedeep Documentation",
-        author,
-        "pytorch_widedeep",
-        "One line description of project.",
-        "Miscellaneous",
-    ),
-]
-
-
-# -- Options for Epub output -------------------------------------------------
-
-# Bibliographic Dublin Core info.
-epub_title = project
-
-# The unique identifier of the text. This can be a ISBN number
-# or the project homepage.
-#
-# epub_identifier = ''
-
-# A unique identification for the text.
-#
-# epub_uid = ''
-
-# A list of files that should not be packed into the epub file.
-epub_exclude_files = ["search.html"]
-
-# If true, `todo` and `todoList` produce output, else they produce nothing.
-todo_include_todos = True
-
-
-def setup(app):
-    app.add_css_file("custom.css")
-
-
-# -- Extensions to the  Napoleon GoogleDocstring class ---------------------
-# first, we define new methods for any new sections and add them to the class
-def parse_keys_section(self, section):
-    return self._format_fields("Keys", self._consume_fields())
-
-
-GoogleDocstring._parse_keys_section = parse_keys_section  # type: ignore[attr-defined]
-
-
-def parse_attributes_section(self, section):
-    return self._format_fields("Attributes", self._consume_fields())
-
-
-GoogleDocstring._parse_attributes_section = parse_attributes_section  # type: ignore[assignment]
-
-
-def parse_class_attributes_section(self, section):
-    return self._format_fields("Class Attributes", self._consume_fields())
-
-
-GoogleDocstring._parse_class_attributes_section = parse_class_attributes_section  # type: ignore[attr-defined]
-
-
-# we now patch the parse method to guarantee that the the above methods are
-# assigned to the _section dict
-def patched_parse(self):
-    self._sections["keys"] = self._parse_keys_section
-    self._sections["class attributes"] = self._parse_class_attributes_section
-    self._unpatched_parse()
-
-
-GoogleDocstring._unpatched_parse = GoogleDocstring._parse  # type: ignore[attr-defined]
-GoogleDocstring._parse = patched_parse  # type: ignore[assignment]
diff --git a/docs/dataloaders.rst b/docs/dataloaders.rst
deleted file mode 100644
index 02f9515e..00000000
--- a/docs/dataloaders.rst
+++ /dev/null
@@ -1,10 +0,0 @@
-Dataloaders
-===========
-
-.. note:: This module should contain custom dataloaders that the user might want to
-	implement. At the moment ``pytorch-widedeep`` offers one custom dataloader,
-	``DataLoaderImbalanced``.
-
-
-.. autoclass:: pytorch_widedeep.dataloaders.DataLoaderImbalanced
-	:members:
diff --git a/docs/examples.rst b/docs/examples.rst
deleted file mode 100644
index 489560fa..00000000
--- a/docs/examples.rst
+++ /dev/null
@@ -1,20 +0,0 @@
-pytorch-widedeep Examples
-*****************************
-
-This section provides links to example notebooks that may be helpful to better
-understand the functionalities withing ``pytorch-widedeep`` and how to use
-them to address different problems
-
-* `Preprocessors and Utils <https://github.com/jrzaurin/pytorch-widedeep/blob/master/examples/notebooks/01_Preprocessors_and_utils.ipynb>`__
-* `Model Components <https://github.com/jrzaurin/pytorch-widedeep/blob/master/examples/notebooks/02_model_components.ipynb>`__
-* `Binary Classification with default parameters <https://github.com/jrzaurin/pytorch-widedeep/blob/master/examples/notebooks/03_Binary_Classification_with_Defaults.ipynb>`__
-* `Regression with Images and Text <https://github.com/jrzaurin/pytorch-widedeep/blob/master/examples/notebooks/04_regression_with_images_and_text.ipynb>`__
-* `Save and Load Model and Artifacts <https://github.com/jrzaurin/pytorch-widedeep/blob/master/examples/notebooks/05_save_and_load_model_and_artifacts.ipynb>`__
-* `FineTune routines <https://github.com/jrzaurin/pytorch-widedeep/blob/master/examples/notebooks/06_fineTune_and_warmup.ipynb>`__
-* `Custom Components <https://github.com/jrzaurin/pytorch-widedeep/blob/master/examples/notebooks/07_Custom_Components.ipynb>`__
-* `Using Custom DataLoaders and Torchmetrics <https://github.com/jrzaurin/pytorch-widedeep/blob/master/examples/notebooks/08_custom_dataLoader_imbalanced_dataset.ipynb>`__
-* `Extracting Embeddings <https://github.com/jrzaurin/pytorch-widedeep/blob/master/examples/notebooks/09_extracting_embeddings.ipynb>`__
-* `HyperParameter Tuning With RayTune <https://github.com/jrzaurin/pytorch-widedeep/blob/master/examples/notebooks/10_hyperParameter_tuning_w_raytune_n_wnb.ipynb>`__
-* `Model Uncertainty Prediction <https://github.com/jrzaurin/pytorch-widedeep/blob/master/examples/notebooks/13_Model_Uncertainty_prediction.ipynb>`__
-* `Bayesian Models <https://github.com/jrzaurin/pytorch-widedeep/blob/master/examples/notebooks/14_bayesian_models.ipynb>`__
-
diff --git a/docs/figures/arch_1.png b/docs/figures/arch_1.png
deleted file mode 100644
index 7d829a0c3cb72139afbfaae1470cf0997e9179fd..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 58023
zcmdSAcQ}{r8$bT`MxjE99?1+Dp%B?*%cks+y~*Aqg-T`bRW>0znU#{gw~)Q}p1<?<
zd_K?TIlkZD-@oJdISzTpecktUU)OoQ&e!=m&+Db4yyPXk>v$*>>XNjSm@*252_Mm%
zIOpI;4!OJ^3WZl>E-I=hEh<W-Xm4v`Ze@%@Nxh7WKCh^vam{ztgDvqn3iooz6?dXE
zB4L@i_q+5=)VPE)L4GL)k|b*SEI~oOmvqt@K9dOdy~1XGg?)SO*=@Y0RfDeqO{c!p
zR<Wn2vqH8jBddunD?GEK6{a)%C<e!T^#puoR0wrb)bst%9EJvyF?%Aum>a%9f;iF9
zbS8n%u~4cRv$aiutSGq_msWbu<IO`KV(}WP*XL1JpV`w0y^i+Vc<NX+^O9~5MRllD
zy%J`%j-@9O%PK-7R$=-qJk!<a8PP3L|Jah3IGe&6*eG;tb0jUMaG&UQ%MD%jg}|kk
zMK6NbuD%bh!Hf<LmKAI=)7F1J6r_~2)-H1to;+2%CmXRpWgl+LUm7h<?TS}iKR;r8
zI{Wm<woxitzTuAWc^>fzvVgXWTP+l6#s|&n5r19fZ+&o*c1)Wo<#jVNHhTIpcdKW9
zX04UAyJ8*ZE!=7zcA>$9AafEUw-*Y<v>dV|%+eEIlT7lB=<_*Qlrt1A+ZmlWsL*`K
z<dOV~n6p*G{Lcvo7X^E@*me45J*thT$uAdKhCL+ul6RP=e)XATitRP+v?Vs`Q5m%F
zwY&<Q{&~&Sy>&pWY3aw{N|E}+7GZ2P)o;m+_9oouyjBUesg@zaLDCHS;|Cd}u4pzw
zUqM+y+>gEjpRjHd;t7}O&)1V<Q^ZafUwL9L!xjGKajb9S<@J`?J9)Q1TA^aH2D6#h
zS*?mlsDH+i>d<S7ek3qUw2JMdi$Zs3qkNlENfSp|QCa&a&eHX?umr~6VmuhPsW6yQ
zFdwDhI{IC{Bf43PDPMP~b$|fNABXS)37SF*gH#wD<x8y3!hjOj=NLH0RA=*vpvU+7
zID^@_%sMwaQtk8QU%l4;+Iohuh8OHhpnvfh6~CCDSqM`a_Pl6D2yy*0Z7Mbf6rR7q
zdn^XhD<aBf*d_Q(&ty{Y%-B>f9HSGTw~4B~m!G5VIe$sS_C4DjbodkAj~HZi4o^s0
z0*?RSxMCz+WD9Wk<LP=+8(-~v;5hvrzbAPO!S1($aV0D&dmPM8i55bV2|uixPo=KV
zh&-l#Aw{bEhLb8!iqHPqym&xp`(HN#0-hMO+>!rFPnI?yP$O951>;XueTzGCF|T7P
zV+31pX859>J!*(EATPXOb&-p-5=+aM<vT@PQl0L%(FS~dp>ezW=h_HO-);W*F`@jl
z`RR|(iJv7#c(iZjQ*ys*n0&OJ;cDu6C+T(j&uil@;{<=&CsxPLuO;KyW3fCV{u)mx
zNqBh!$M`w#H_~t9-!6RnOUW-w!IAbH(SE&gy<Giq{o@Ii3DnczG%0pko^Tqe?6lG}
zdu7b|E7^ft^6(|uCc>&J&rmLrue}m9LR=wO(QZUnggfYeLm?`RJB%Q#J(MzROo`zt
z<|{I~w_$RFQrYR+Dg5c4VVRBb%??xh8@bcW)A3WQziKnCWFM$<IZ@w>)|3#Cs1_%G
zOvUsUgDZm_@nc3+rUccw$A3LaW7K3i$85qt!raHG`{;)1H)c`>H^%kIjmRBF9mR!A
zYEGTF^KT@zB!?9$#mG`7=K>t@1tN9Dgk)D!-Z3yU4l_tHvOUUI70erH7XItpMf{tI
zg)g5*bpF_5e`rYJ5vAW3pmC{qrR5-tUF(OOc#3jXSC+lr^lK)S-iP<(oL*ZLFA&`e
zj(e$@HFTv^$0E8Xz_i<xuXU!CHNneq=cbeTZ!rN`uZjb~ZL)>b>v7iwuN%A}zkZD~
z;2vq5O<Z<ddYma|V2PO~tLB3|<-B4|w%8_FE{;TvwyIw@Iv@A%^hQ~%_O5hLbl>fH
z+0)o<-sAS+VVChd{e09y+5Fo%^LgxrypG&{k2HlI)A`If+kV^rQ!{-tLhDbXDTDeQ
zia$aOV=UEe2MopT+TYW(Qnhw$|I4(6U7lmq+;bR<tMH!m-E7VZW4!pGSUfqGIOn#n
zZBJuv#;FKo4hLKP<Z8Z~%e}~98go8oHkLcSHYt~z+h!%IV%ecqYd|wiYpLATD%)t@
zG}d&`V!yAc{*~z?bEoz=gJtXLp)WrU-n_|vQ)d$0im4T6owDpc%r$K5yyJYO@?0gg
zhrl7$A=9A^Nh0w#CKH{fb-bVXK04o@UK{LeyF0v|HAyLNB5yA5B=5g@aTa%GIk|uJ
z<7A7|=;FxYH%Do^!O5=0oaH8`JZG<&jK!ZWeB(RMtXt%bDGh3!%k82+C)=IQ{h2H5
znoPJgF+Zs>H~1%gig^CfpY0y42EM_{QIGnDKGTYJ9=(qRwW}m!HgC@bHe^o+?+qU2
zZB@=G>=5i0toe}WU!lS^yYw9M7+rpD2W$TP%V%-^B6n`NTCRl?eh>LUT^lg*K=c8#
zXmX%NKpu`ILFNlKyn+j-gnuXs87H2mDW!21vAnuYO!<gx`|c3O7uqk}CN?ciF^h#3
zh2I9E?t5yhj`)4<+}GVKZ>0<>f5R5Es;U23fBsqSg<g^jHva0MFF}=a>S!)%-uJA3
zn7L!LTFK?6XdlzPUE_Bhb}QJkIZ{3esnbxTaSfeHPz~dZSxTt7=f?2kUixig^*QQ~
zgf-IJa=#M~?y}rp)*o_pcXT&AM}3oeGEGW4g8oZPc5DS_v5FNFE8#Ayvv!sBdQiQd
zJ=KLK&5+(OIm!4`GUc1|1YI>Zq7#}D6drzvx}lZxS+!*ACwsUtSyc6}?Y3h#9upp0
zw_&_xVoSBHJfnh<N<7WT&pFnmD+@=xn3kS>Js;`v<g`3E+Gz@CSC7BUIpytN-?!X2
z^jX_~Un5*2tlUvOYJR*zc~=O}XK6sc`bk5;>Wiaqmf|Gh#^QRf-G20Y7ae6EWv;)G
zCgbB8v>qxYE!QZiuzbB36?f%F*htt(o}X5{Ol<m#^wn~fA?MPT469w|F6a53(GA~?
zC~|i`uGKxwQ01q!UDZW}GSwEMy$y*+i7w6+%N7H9g_UNz(E{HD2zCyZtaG@sSce0L
zql;JfPqL1eg^IoN1yY<-=IYA*KloSQdvfoZ<+7#aV$UL1@AvJlgZHBAfi{#o{3BiZ
zeV@uhJJlVrZiG^}?LYj+rXzX$u8iD}-12_DPt^j}ve*-%o|2E|X*}r_<#c7~n`vW>
zWsfuah-%|b-Pa;^vYWb_GPx6}6B~s%yt#Il=6KdC{%khR9uAX~3e*rf{2Bex@tQpP
zh)t(r$!q@sU*+)<@ho#@KCiBT+Z$NWFTb}l$jZ3oM@p*QK77wlosT-siyFo+*W#OX
zN~)N2pFPrPuZ&)Ox}sA{=~!FLT(l_S;_0%AdE);yaGqd{zsZer$5oD?EOYJUxA3CO
z1%+H6j+4B?{`_6XWYe)XrhnQgb~g|7j+3?z`~8k)BW8`+N_2bhH7_My-=O$%Y=6@1
zr#7Kx_#OLH`ADtTGPS?q)b4Qdr&P<YI<m(2_3|I9s$=AywwpE9oJL#ofx!fZloy1u
zy>s?k4n|Js4w_#V(&?@1HP@2*SRSvOR-5u0dTO^s+TGH^z#_L(zJjtPKnuUb_H`yh
zbsc3Vxhtn+pZZFpPhVyptVkDP>g#c%E;nP~e8#PNi&v5^eylQyf%E=@`z@4~R`XBC
zQ@+7mg`jWLsGW;i=Z3KPJx|)l-ewD)D7c+^og>n<t{a>6!TcGBGqE|@unHvhm9e_C
ziJTmY2|nYXurRKpu;CL1d<$Zb{qtD@gW-(%VxZ6{RG>Ku>p$1X!%yTZ1iq0mfB(b`
z3qYNR|6PG^*A(=Buf{~BVE*?rniKwp5>XMAmWH1yhW5tBHV$UCj)vZ1$h~pxq%<5*
zC=xp48$()|b`9>|XRfO5s4geVYiMiD`ozfAz?jw5+77u7ir<wNK3W?)KB028wz6^H
zbrqmFyMh-!Bfn;&p*p+7(Ncg$T~3io)YjgZii`CQ>m3?FJSr+GetRPmUS%<f{|tx!
z3DB51I@<BFvAMXou)1)t+S;44-R0roVY|c5#?H<HSFkv^**HFNWwCL%`S(r!dmk}l
z2Sa;vJ4bU{8!F_!PYi6G90h1-kO%$q_4hoDUCsaZBpZkSycWD58}b*nyR3KE{<${{
z<wt(Yt7z_OY^5P)ZVhV&&k(%F#mRqm{r~ls|2^?vBh~*mlAG)PzeoP-FaPJrM-ImJ
zqPEuXNJqi{ZP$MW|ND>s8OYCuy!pRY;_qcX`z@@qARa&4KYJ#KmtnHC2ithVTuea~
zeu9x9Uto+V6vN*?k<ZEx3;Za-UW8H7Vj`-p7)!r#t5he+>egp%%+o)nUroO%^O~kH
zOu$-IM(ouG9O9QG&uNL37{d}DD=9lOQL~hN2x1UFq*-uYJsO*xZt*GObBV7ojT+nc
zjEOC-S=~*XscGj~b(4S9Ap7k(3XMbM`=5WLsY4#1Lar{$(p?rtVPFxS{qqTph4Q@o
z_vf=;NYP-ReQ7PMa$fywECy=qzXOnq!lZmrR4=c`=g|E3-Z)gks5+N_?kbG(qrF0f
zlKSv7Bj<k;z#&Afk^k?}!c;G@a8O}ch1pF1nk7u<{Qu|ENx{LQ@Hu)K%Wa#~5_j(*
zKJQ#g){Uo|Gp+WGbkV=6cSo67G)sb-?hY`baq4SP7{W*8)c4NbB<QK1B&C<?%I8UZ
zX*s#b-7z1%HlEpptO(bM`j=g)k_3&veuXfqWc)IY)**?U{G@_K7-S18yk{dM^A1^I
z!bvoNz*fkJL%TrPrjXl)QN>K0<p<3PKg{z@{se*y^Yil?8_jIZ#A@1?dnXGWV7w4A
z-jYKmSUB{8=gI!;Ml1K&FG`;y>PI=Mp(+Xjk1YB=YD`U!`<~(*t$imbvL5C-e6O4?
zW8ZSOVXxDbpJIQhxP7TATtQHyDS~mbJb^&`)*bI7*ZINXOo8#ub|K2`wpx{949s)C
zz7q)DHa9ol*crBM#=fRKetUu#41o}hs=>mc3Q4V7=*^uDAhQWk5Zte~O^k@R>hV~K
zk9)c)a#M>Wo4~T~W1)y-V;J4^#A~*<JvOOjl*=JL>p!j)c^|u1oE$8sI^X#HH1~0y
z(azG~&R?cUZW*`LKh>?e9_}oMPyBd2>~`TAYXrSq+#buUXeW-<V3Bx^S^+Dc^U^1g
zx`9H&*E4l94A-%ed`|W^zMiKX(AU9qt){6%tUe2nd<+wd5QlMX*tT}?ZEB50pk6y|
zf0mMH&C$j~jdGjB-x*IQe{PI=tiE*ax+&l^*CCQ2pU7lAR1(^+=OfTn`EKvTG3!UK
znx1u_n~|}xxaVmN-_|9R<4O6LH}kDT$1s=3<HKDKTXs6sTG7>gGJ(wkk)m@L%#S|?
zHPwdOMILPT>!r&l@)kMF%9C=K1drG^v;28%N{Ysz4fo}Fcs5Ne+!CALLx$cL1ZGSg
zv>)wn9!JIBZFP=irQ~(Y8Y`0Wx-%KId{-7ln5`q>B*a#|6~<Cf(a4e0(3x0nHQ2RS
zU~RYA%6&-Qt&Oq9%8lVW#x8jEjW5sS%H8j_R(!I_g4JgD*C!YbR#j?m22!|ZnJ)I_
zQTOSSz#LLhYN!^rn?z`wy$f@s#sUsz-dBz5gD!U!=n0(?w`7)C4>z^Max4?3Tq5I=
zdU=JeYH?2SFM$hrY8T(~Z%D0rdV!^!4&>q-)Y$y~qWs0t+{^M7x&78`vd_@2Xu!q7
z@7LnEteM=2(mBjYh^seJ)aX2K$@yd%P7?4f-N}BG8+8N=^ue??`nCp5Af@-`IYrXy
zSC=gg<};IxCV#$hu2$UTx~)|vyR-6}Ig(ke?3CxQgPLN@=d}8GuZ?dtXx?g&83krv
zSp3RMV7;E`hhyE*4DpP=1Gez+++sm(lqf!rnD^<i%PQ$il`h6WiFtU{;Y#JstG4w9
zO54ETtVi<U-fWKdUynMZ>^{Sie@((tG!{!Fe_!P7{Ry1f9}lBhBWq7jhWS>5Ioduw
zUdZ_#yLGWbR|mW+c5ZK5G2A^~zbC3(z0lx;abH9<PkN7Pc&TN7laeI4ar?L2*Jab0
z$=(*VLh!pXCw!-m-`XnIf07x0dyZRXy)_z}&=k}CbZac4Y`H0tnU0T7tEXq5Ez|lz
z>*@Y%a=Jxtj&rBsI|2IgQFqI%-k3lXx{*F4Sc$+%0_C@cHu+d2U}?rXNYVWgucB!F
zXxOR6Hlx8Y=cq4R@%=L$E{h(9+WmvCjvLj(vG~bj8@c&knkr>?<GXz_M5oIWy<Ncx
zmk(x>Jn~IfMk<m39QLgXobJY|KP&4drMLc|)1PK?v0?=4s`LK%SAQ@{&ez@zzp|?o
zgpNzboL?M|Yh(PyX%^BDyj^Y_!)YGoQtnoP)<^TLmdRK2&V4Ly+lY~`anY->?Rnh4
zJX&43YPISy8(gk?2{%F3^mlvHMV8_*vf0Xs2oXI|IRKBe2CP``!wx4o-}$?{!q125
zIo!9*D!l;*1Sq|>LL0EqBEz}q^ZJ1!HFr3Am74ouQ9YM`V&zkGOYhD_(&`b-R^xgL
zP<rn@soBXMp4&MPFns{wNR?`fjfD%Kss~-|v0FwSaPHFesksz#Vh5G?I;-H@+gk1O
zJ=5H%jyh3))Ny-)b@^f6`wMOSUezmK7LVRed)lE<Ea4}a<de;8wIeo`HmYTZUGkBQ
zW>Z!ZUac&sJ8IZVJAN2-fL7JU9Hu49yhEKEECn^(rft;?uj<Kep+29U916_TV0w%M
zTq1LKNz_8!u^!@l=e$$;afT_F{6X5`(`UF2-KDd2ewd@jJht{1PzmyuS*W_Hy4{$q
zXbOkYZxm|&oDMI3)o`nkFXeVOI=!8(ip#w^`iJv+aBT77j>ouFCb@^h&k>E^hYMJ!
z>lYVX9=x-*=_vCXXUt5P5HK0v?XUEIP}Nx{wrIG5@vC3ArI*XLqJ(^l&t*2z#oduX
zd$7oOnf+LP+SL9;AlYkYaDOn^>9R1DA2tr^%IBxq*U{L-pXWZjwvznLFUGLMn!u4W
zlb!0lNX&^^`z`IgiUF&%>=^t03I#{ikIG6t@BJUc6?tj}NiH>;87UaP#nm-mo)XmQ
zxcr{JOBz>2F2T|AlY~(lmGU*n!AWeDd~JDdG4h7#;LUgZ6Iud?HFsbL$E?Mx?%3t-
z%y$hJFKo^cU$o{sUfGv-$8Nf~Sk%l~y`}t)RANfUD!|6A#g4X87V}eGBQDCZ_IgWP
zyg_yD-0t^m%L3i%s?D40CSXUST%F9u`>`z@Otw0h<=w}EYjv0u9I~u-N(1dPu_)){
z>E>vHN{S2c)@}$b$;``UM7xpL@lw-cn+;gY&X5ile%@$eKJIhiKy`2o*N_Wj>I44o
zIyX5l4^6j}6|7%JZ7mP?M~%&^@`*Zp!$$F1N%7SWJ>6NB<)d=X(2eG{9UY0C_IhK4
zN@=(gn^bk^jP|Wt4QotXwdQuHec|FY+xjc?W_5F`7gd`TvXeud1oy03(j-fVB@QGU
zHZPvDjcqwT=WUF)OU=R2ZOCphXkF>FNMh6L$>Lzu%b)p2XMc#A`KGV%$OS@Fy%&)(
z1{MulN~}*waf+$wce(Aw{_;f~*FP`3b1O(ubBga2!=>pKmLt5jd)2&i>n%}s!`9`4
z?)S-c97Ibc{p0O0e4Up{t(_QTv&=`U+?@A2B`L{E?uTLcmW8h#)4AIQNk5vu>$Z6O
zHdo|rRR2ibGWXcNzRkvVndNxd+lTU|HHq(4<JOtN!rj)Q%4KKhw>+1JM#h(aKgJ}o
zCUDraui3f$dE)|G)%fpB-Ajguzv7_w@Tq)-Lwp^+o7iyQSx_WfPQZTK{O$)z*!F&3
z(Sna{)p~$l;-mnoW9x0(-Z4b*!FbKYFXu8aoC2)p2ToaazF-|^eoZL09w*(FI)|#b
z#O84Q7ysA3s9SbsA;sE(1U%EdsBbP!EU3AG!qnefT^M!mgihvIRP^z5#x`EK+ZLU;
zCNNvdS37==IA=v)shTBcpJ-ZiI<PV4QqP}C)4#2Ofq`{J4;b+JaLjd@5L-VI7R>>C
z|0L>5+&qTG2n5yk8r((QnJ27HEHX{@>MN*Pr$Ct*C&c@0&pb}tx>)H<<%Nj~x9_$b
zcVzuuZ6Vp`^2!KB=qg&e`qXobk6V9HfSN~+{zpb$H2&sB;s_pUs|GJqmiIYhZFpJu
zC}Dv{b^=*$qH|XYT~9}zT|<Y5xZ`0${)gwoQN{puF+k~G4(9jZ`8~kB|DAQ~qK|xd
zW8+FIEr(^_?X#(^RijfFsl77Aa8R7Y8Qs!7OJZ%8@@p`B`}PM6-<2-TuU#Q$Ct48_
zs9fQzxQx0lYS6NY;rV*PtJY)J#<u#FLef%Vfh{txfi?}#)*jUuY1_wr`gPovp(fjn
z?P<G@=3G*BDxLpdP$cI5(+}k;kt%o($H*?I<!Mg4Fx*0Qka1#W+O=b>>6E8v_}YIt
zufNcfJ<5wcN(fHClP>Mp_8^eYa#@0eYL)`k&{{#dH)L6`J@=RQ@)q_v)C`7=v$uRZ
zzugY;LN6}L{tM3tj}|%8rL1D1%-}z_6P>mnCI3#@z7HJT)6`>Z@%V=P_@t4DJ;@hR
zlo_5Ti(1x475V**%^cI?d%d|UD&3hgPF+-O($#U@4tF-bu=R<e{OT`5Hj_}6Mvo&V
zE8NY@D=JmuaO3{G+)D`YU&=YI+|M6q{6o1Qp)xj3vnHZn)6gmZ(b>60#nH=jc5hS~
zGbgblk<aB^UZ!c4lh1*yZ=H1alLhu_4bHXqiZk=yM)6fY%P$6#ax4it3IS`z<rfnZ
z;^QtjcMc^Vs74tx9lT9&5tR@T@<raed47aaKPl&iDkXVxzEl4x4Jm3*(2Au8vo@h>
z|G?`~mBAm^DMuo^k`)Za8?#T`7v4`EZ1OG-7T;=kbZ@Qec)oSiZ>&3kYvWk=+$e2g
zH&=@!yx%o=zay@U0Gz8*!nR9Jyp9Ni8#H#5B;&T(x^Ft`RN-8Oc}pNXznSUr$2mE^
zEYmy5t`oSmcU%9UQC`Y<4o8E<rXGC!yi+$-EyQ6<t+zfdesNl?`r*EiomtgDe%~$y
zg27Mj$h2<hF8nCx`=y(65jP<O_&qC5!td-jpFW><*hwuJH*r^4MIOU<wY5%O1C5FD
z(tn&T6KSP!k9dxI%yW_@$~mR9`y;uh6MjMhGu=VG-mUoHw=VtKyYdGz0yeams7o^m
z@vYknezui0`!n&QnV6ObU3HDEC}Dr2*``pz&$rpSUlB9&pQrTvB<Zg=rdk%)D_xAi
z(=PvO(Z6o1GE>dHL~e@|PZsi4-<!gyaj7-PguT&2>s^7jc*v!fx6A`a_dgc!I9Vlm
zZb+}XyK|%orZtj=>sC5juNDDj3NHO}i_4*S*luPyq_k{C!u<I5+)}A6YCJ-T?`@kM
zZDQtdpiFj_={Kk3Hs|ZY_nlZ&GJ<!XsK7%0=(b{S6Xcw1%Faw$5Fje}s*skEfJXhU
zb>R4<^0p2o+^<pNSt0Ox>0>H}-ze3>$mg8Rz)NHqxd(Eq)w@aM-tOtsCEfCybK!AL
zUgY*u7w_Q`qvrlP0XFGY-p_pZK;!gqa3qD?ri){42;cpN^(|}3TPl%l-S4Gq2sj;f
zrcF>O#;%Kb<yNuVgI6u0ho`h(cJcCL#N{<MuBC1E{Ki#w7*5_?RqQ?pa1VH3GhBQF
zuuu)K@ab?=9JQjEpY?E=n@#oh<J!0HogW}PI=)!IZtk6f@N)2c_oLFuNk<dMfoDmB
zJ=FN}v%GgGYm2Kla}2+f5-l%%ck4ZW-wTLoo5c?JF{fhZZ6W!<&OsziqY!wQ+`ZMr
zT;SUNX@Tnt<!=eTWbWUN;Vj=f-ita6jkKw;$>q&!+5T1|-gNB8C8B@+n@i|I;`+3o
zsIMbs4L9auq0s5kW=Lh9e44HA?EQI-#b`b6L60AFc9Va;2OfG+a+-CRUYLt~zuCsu
z&O84pv96KyaP>yR?mg|w`-g73zg?=i`*?>fOx10zTIU--%b&krPA%M7#nb1OQ>JU(
z$;??H=l)2v;o#8uptOd!UV1lO4R?cm-ixR05$Q2^3zC!W;ib);-<&*CWu_SP16G5i
zaTzmG=`uK&>BEC#@TK1_@?DS(lx&HWygS+Y!|3ki?rdk0>HSAOA6hU|m)4zxW`YZ2
zwo*lg<BQw6DjZC;SUE-{TGr0rC4YFAT;(oz$k6FCF1(oc+nn5tuPY9yXKokAq|oWV
zsq*4f&?-H=JAVVoD-3vj44yr!vevG*Jchj9AYfCSJG=r)oDtd&^(nYatmRTn6J-fQ
z7P&&zMQSh1Sop;}5jNm0H^@gnKVlV|P!AnyIM`nNq5b~TnBK8`EHj2dzW3A@4<hYh
zYex?Ug*V41Gt*-UE9$3t3wNEgI1(PV_&4Gm>9m!md<4DdDWfooFuGOZJuodrsyiqg
z?KV@akt-djzKecEBH<MaG&2?wNzZ(rMF68<Ga0E}K3ZH%;BlzGSj+q?f8ZP#oH`hs
zA6aw`b%-p)y2hTNbv3o{cD)utMvY46yqI9=t}Lb0EY*q9e%)|Lzy}Ru|A_4@XsP1Y
z%fmzSfz(cX*=OKK<J^bgZ?OsrnNmdO9Mv_uGvrb=#T+&ZFQHnzu#^Pw`2ZUCh^bIP
zV`ZsRI8@^|$FWfKlP_arg#;hEzJ@JD#I+i@NzL^txb!n|2bqo2$I?+dEmiLHpDZNU
zl87WGWdVXsnvFNpnc#CKnT1#(6Yrbox%Q8)MwJco3j4C34fFbsW+z;PONgS?daa`O
zAGHp2wKNhy>V=uijsISRB!<GM5GxjWum9<KVVbZ++40FuH~&%SXyZY*Dk{CdE&h*8
z_y5<4g{e_FR2rR@%J6%r#h>;C8QENJ_xnww;c;?Lnwf=h-dKzsKRP0;x7Ft!ZK*2D
zW`*`foDwK!!c-VYLi0%}`wFbFK6cS*=n~5&t;gZ{u<B*e^`-r`mYpf0TjI(x$+X#l
zNzB>PancKauDwnpX`dBTCr7r5DhfENLH0d{3y5}y;kUF*DZTUa8r|Z%d7AUi;`;9d
zE2V_gyWU#d$|mNG_Z`2Hj9qoM;6c~G2wcq>n_hz1JSSQ+dbpF!F<sYA@X~a5!_HE|
zN<${rJ<NOLcbNOsG!0|+<zY5Hgb@=4<-@)?dJXa+4|MWd%G;c-BPf5}5WMPQD06og
zH!b;&rZPe0uQ%TvWDzs*^TZ=WO%Vn6f;IT5UtU`p*5^GbmoCHKe(A^9d-0Nt(4V`~
zJXr=8BLZm7>}-k_%qRTfCUX5>BpvcTGfg(3l#_Y`yzw4b?D7e;*Dv0oM$D)_17>1X
zb|D>^*MkeL?9+7%-eKE|F&)i@MmCz$qhY#}N)EQd62TEc24}bakAH!w3V*zCV}LQX
zSxaW;Eq0-<vQ|w`ksMv>s8#5e=){Mrt5*;|A%P__GB;m<C3?y-K3J>49<8HlMCwiR
zBJxpD{zpF7k3BhF5^2tj-e(Fi9w3RLlrR_KWFO%AawYcYB+H_sBf@1-amUm1MPgt0
zZ*Q!0sqR1N#pB6QUJhDGPCR>)tFpeR6nz`37w}eB-w5|n4AxFrIn4NIblzZlAo-N{
zv|EV8ZAv0kt00f<$MrY@^YojJSkC@%HI<?wO!8S%Avv-g)))#=M)_uRgIY5Zhf<e+
z`{>Wsmb=Nv$Y%#~M+bkgjU)5#K1uv~dghqgz#;8kB>E$wdBuer>c7>B`rb5o{rF4s
zW+v#Z^(9PRvd$@?ZIZfwUrq>@V<d{6U5;yH>(xKH#XQky-5aBkq&0gw_AEn6;&^{1
z`|GuO*$PG+n9jdJ#Xz?%yh5D)TL0zheQw>1DyNgvJh>(jV%pU8{+Y`dfejD08EzqC
z6ak2EdlOU<TooCwEAN--`+035gK2;Mh%Ztm*X~VP@=?`hsM?un(OjiMF+8xsN7ikM
zetfW0Y{$ntPJ=}-;N)uB=4pY*tE+s=4$?fIpV-jeKEuACPhd08MFx@OQvA!Wg-)&|
z+*^pHh4q!y@H3Z_Qx*e@K@Qr~KObnA+0LKY`<|LF%GVZ+EbmS9pw?(u%QTBY8t0$a
zN&{`R#k;}o4zlhY4-?;2ofRLh`oaA#Q#s%uzR8A;eHbsjQ%0~^l#{zCma|z9DJ8I_
z-DtMvm%IIE95KUxA!?8`M*GTi(P{C)T@9|W2jWGWt4rkr^UbvBvI4U=3(oLW3D7o(
z{Z1wFE*ms<`qP)fO8>-GeOwsg?VLNZH11;3Wt+bxxU5-y|IYup0}6HFjE6$3Y)LKF
zg`i%_T+M5%T@j?4RAhACq4y#x-p<rVW<-Nvyt1Y8HE{KZeoHL2@$o+K4OWAxgZnN_
ziAuKlknUXhXFjk2K{TwD5@+)vZ2s=^>ceJ%#6`@eX6_eGi1+`3DvQ|9&}zj0ut^4L
zvjSXq@)_55I)^@G@t$g)04%1fhxc~o;2oaf(i&Lh+ShRwWPf_lNyk0|>!;#gBO`Cs
zA`*QvZ@W_p%M_+2{Aabrq#>-xd|Dy_Z~gUJ7!ZtZGq=o5qx4tb(^OdP=KID0X%ud5
zp}palh@&+!0knK%QK*KnOOI95CrEYP`}$QP<IV@4d-wYdGu6nL%paX!(4s%HE(QQD
zvCGjqNX)9hy|I`7+UWSi(<Ya=(Vm;{RP$>|YC+^ycoK8i7kffvyJL}nS+SiB$oQEx
zq#`ZbXjOgAoo8g_h*=n{iZ1mJx*yugkbRj0D}Ooqt%m>u%Rp2A^qa-RzDHiJaqx?_
zT69`O7SoTmqsB)U@ZZA4wEkdG>?Q0@Vko~D;(^n3rHR4QOv~xlk;t`I`MUTo?-BIh
z{(HY~BVoL&tA_~gtE@kEK3%-%WP@32{lc*zNn@a-&DHs3XLM4Dvb|!lfVi?A{xjqO
z#mEE71)adC`muMaqv<CacbEgoFzSR>_lUnH5-^(EckG`Z;o*`*cIaQlHsK}QrNflq
z3Ur-@?J&D}dMmN-@4|km2pj}F?*J}gSt{lLO0Cm<I-SIuq41Ske>BGs$(63|=0H&Q
zlVLgW_nFiD50-r|pbxJ-y4F5QsdL|ciJU!sYF+tbW847qUv!8wUICn>`g%QL0qoQh
z*T~x~-^liqp$3mUy(=S%Pm*2!_>PQ+Y5AF}yhUO7-u%i8t3Cd#&M%T%sxQD)`|PH9
z6Wd|l6WxVscVrM6OdA95|Lh(^uLzK<E58nmKi^PyQ1c|_l}VQ?IQf)DJ}mVAdQ;(@
zsB&5dC<(p2;St!9#Vx5!P|LZ4C-D+7dn_U<l+>ql>AV0Nrs&vV<1xv=*asdHAIC#k
zZC>4QY<nCUY#%y$_fQmGix3B}6$ka_=HEtt2q@W}Wc%vi(_N>kh_2Ff^~nOm9NDV?
ziG)(Bu&iR%8Vdk<cUeA}wo@?bd?{g>^rLit*jJ!X@938L=(B*&>j+lru*dr((n3#U
zOYpyYpCPy>PzBFBXFO3%iwYF}bNDwHY-{#2ZtGU5L*>n|%h#JGRxgQsS+rr~r1n^2
zYJyLkqEch3sfO~QMMbeee(=?^trbJ&VD)GLR_`w3C)+W>ik%@#W4HrSU?k<XNl44c
zi2HLoSmg8d?!@4(hinv|=8K^k$+X{up$fIcr1$J2Mw}0qJ9)4EUp`OBf~0KH@m$tM
zf4;q7H>k&%A1*H#dyju)S4nzqD`u?9Xzvr7%2Yt%J)?z@-mY9VHmGH_)QTvvt0wUx
z*2D_<^%<-E$(eBKeX?%~T4BZ9;%l}VhUMXMf}AST)$4<&=5ikQsLY<wv`a}x2LH@W
z8%xPF+M4U^%6Y`>2J&ONQDfLkZOerKu#e9G8skht=Ydd*UE71&XS(=nlCaoVMnwBd
zaGQzcdCcP=PS!YjB=NSH=&8uU<vhC7mD^hP$glKPy4ad`r1-TY31~r)>X+C@b|=J+
zwL#vntHwpJfHtQNl8H3KhPP!m50SDa?yUsEvYbIa5}BE>j3@olB`-)XU6ZmHJUuTC
zc}4NYbo1-D-Ylh?ojI~z2<_+x)X}y@<H4)HWI)QM71L2#Y2L>NAN+)M+f7XK&CK)B
zEz)9PDZJ?vo!=6UlgT-Kpa!~=BmMYsRJBD{nsj)IQ;Z+Xg-X#5Arr%ebnuywc6{B{
zs3S#WW89xWra6jb8q$~MeplyqJ)vJGKT2DxQ8Oz8VoA#EE&jDNp%KdO3tr6~Uy0VX
zfB8|Xyls@bGhe4ttS`j|2`ZrgrB@lb&x3R&M$*BVtn<aOF!3Zwz+)(w3+lQ1`5S({
zUbL3K=R9q%DUhW-s?h1O493$kYE9_*;$*2&ViwB3Ghn!L;`@p06Ct>PFqHxzotuFH
z9rC%VZfAMeI<6i20?9z}DdxG0IJF|5(=|MxH+oyV;2iBln(6WD#IhFt6YgDEq3fzT
z4z2gtofcI2(qu3(4ws&bJ%2Bc%<?jr&ehADfdB;c#Cy}xdRB4mNCgcFi)`e)jtq{*
z0+mrsCv4YmjVe-+A9R})rrj8!YM0>sn4^#^xWnWt`0!7pnr`S7I+-aqkH=z&Yp=mp
zukDUQS(7S+%`g37^bpzOZQ0U{eXAi(dkxQ`J^`llkH<m_HtG+W=I=>JX-0ZuGtEFT
zdHGHX6ljJx-%H+jD(1KIT>_b55eVtDhuppgAgcRekGe0ic$De!eTXY6GLhS)D!@@V
z|5%qzgN$C>wP0n5HP3)sT4+7?0r?SB<Rh31s+5^jv-DeY)e3a8jtlQ1Qwl}`#=t@k
zDFzlE%bh3Ru;zPnqkD4I+B5d~7qU|Es@p^O)^3CV=b`mjJPLU|X>79a<eIdEK=>#p
zU3G`}2~^Up0_fz3axlJ<nkFdCc@;sG8)6|~O;~nmHsJ47P2q-A0t$Y03fFNAzAeda
z&nTuxIUR?WD*Z7l<p?{pw?v+--=FudVP9w^mS!U=oLD?2iKE|ws&{0xuDkS5nKfCV
zSMg2cww*}6RQ)2G5)Hzq{^`lF{corK!FWY*MG3c!##`xB=7JYgslrJaHW$mns*QF1
zxLhn#d%m+p1@m%jM6BPcH6Homlq*N{#dY1l!~c_@>l@rNb8-R0A9<sW3I7Qp=W`z5
zOiuRPczKbWS9PUgHtB=fU4rw?M?2RWN4GRPDJxNuvMggt?DF1UFCI2B)b+UyPbK@D
zgtjH}HE+##FJJnWD*29H&TMo}{Q?9`L>_oamYf57Hj1@nUeX%R9i8lbAf4o~!ZQy$
zJ<1i_rFjtY36#MGn^%h6e-sJ?M~n_D1Z2=BmAy|i%6~h@d#w#Y#|AxH?Gd~AgX5Qb
zXZj78>)z8fkpIET)XS`5wcY2_KbQ-ew#Lz!COSo(;<0))jaI1Rd;V79KRsGd#(hA^
zmF{XRU9cIKMq!ZjM6te`NP+(erk}){>w}!{q3G>;IokI-!j#sjK^37~7Gu0DT!)Pz
zx9s+VSS`qzcD5~%xzwW9{{7V___5p3H`G0n=ZG=x4nEUPly#;Qm`v7n3L;&loR0tS
zT~*b;1jG*MAFqh5<2oka2?@3g-A2GKOa<11XE;$4)`Jy|RB%TeyXBP2ZFJ&iiaqx?
zDy;sD8bJJO#r%Fy5yhmc;+fI!stP&q62-IjP>{_W;^|cb&zZ+{{CxBJ#NxTa)3cCP
zp8#RJ<sL3JJfj>t;36pvG{4l6A8<fmn#<e|R(v5)SrdUNB*ZZ2Q}bkJ?q!Fr!;svo
zG>SpZu6LD^%J_o!G9$I)3wABd;yR!-HSB@j8b$rh1bx8l_UI7`dx~oo*ABoogm3tc
zLcnNgpin$|hT~b@IPf<kTThK5t{U2;>fD>3O~X4fjV?*&+Lfloohr-RxYp?<>24ZF
zU-BsT@y4KO@*vss?XT8|s*6I;l#giByxB3QNq{)*7jq2&64by7&#g|$ouyN#!y4AK
zxG|=f?&39$?)U-a={Pc_pmo-l)a5!n<()UCD;<fQsdb>+?l1{c&R98({_&Mhj>sM2
zSTmyGU;1`a@-3~L*^Lr1aUhX%2(9gpu7>5mfiu@_bgy+b?w+O5##G}Deb;>kXTMi-
zgS(a0Oyh%<5=N&a>qg6$8;6cpYr2}GWHik~4Fcmij(e8}Pa^!^vUWT*Fi%}tvKrp+
zzb$f5ukU4Vw?6U1Fgqv$8?2H+8N7;SdIs_eLQ8aWG+VjhF;t&z-Hx~WrxP67UY61*
zC*k1opX^O`WhzkeESGd!#dUng=ezyD!-N~bJ5Mn2dNmIvO-M#k>Z;r})4UFMK9GyZ
zZMsdoyd`0`y2M0+&H1ZxAtyHtng+P!%)mMg3S%M=uMWxqi%@kN<N>K5r;ekY->}=W
zFI%M{WB;S#`v)IMBJIW1g>CU{28+_u($WT)G2F#{k#SP=Fl{3X>je<nACE&rP9(Hx
zv}e3b;C0H0V72~P%|RE56zR-{eZB`=f*Ndr6!4X>oD)^ZL&CJcx+{j0hi-<bqGMxY
zyYjTS-9SV>YfD<6Y-li!YfrN%VR-BeA$3Ql?wXz_wY$Gvdy+t^E2r}th{E;2T!oiP
zu<0NSG2ktYRCH(TA3+RW8a(~#e_nFkY>8JFYikDTUc#%FI>??=diC&k4qUusaK?}|
z;PIha(O;;2eTny$%f}#u_2#NI7q<&q@_BYpjpKlIk5+pOpRa|;)}~8MBbSx>9Z;I~
z13Z<cj0xc<1$AnAUdDW<KF8b90?i(PfbP-j=P>mA8Ynzu4|bNB-;i-fagVxwHwoO(
zY4bpUSqbp|N;+Nkc{C0dZcDPzSjPTswfqFwr!MJR*m>VCVtHb3z5IGGH4fb+!<_HI
z3$}Gy&W2JWLqCvaW5H0oImKiO0ry{xx00>+PSf@;UY&4|M*~=9mss>RjZz7E>{yxi
zWGTt#vQr{E{1A5PvlSN}tlJ-~mD*EpQg)*tZO`=|?9kKD<r(nZKtsy!o@L(v=X)p|
zm6$)!Dz_oG7cfDH;yMFB+`Bx9SYV22z8U*f&;U05;+pQKg18wD0Re%jFD+tJ+<t)8
zFyDt!w?pq5{*;0WzWwm?X~Z_QA+xO)=TMS@+*3}!KVOHdb@p(zmhB*gtr}hTT=)q#
zr<_@)e4-I_zjf96_>gj0{bl~(!t16TNJ+jxcCsiZj6o=z@(I1zUr-+FU2}Tk5evO5
zj?e+~snW;;s|pQKFiOiN6bcTF&~Nvo;10QChsfx3l?cG-gdFVjsAlj6jjwNV9Ry2j
zzEh8#rO;RoQ<-9u)}yFR3_mH{;OqBfKpej%7spv>{s21rI;584juFri2U@F`Hge-4
z6y+J$Rw`FzhRdwep{I?j`<=<rlLu6B=<ZB~a<!No)og?A=M*j@PdoW<93q+i2O3y8
z;>XV5ADtwwU>;M@b4S>^EabAH&2uo9n(8IB^Ujx89cY``<CKyTPMFhMPOAYzQp8m9
zVZSGGDj04<S6m&GON`gXzuNzKaqU%7C{&&<-od$=jr8$B(2$GgYP2k<HYvs?HpjUC
z7eY{RV8b7b7ee3T)jZg<CFtdu8$1O-ZhDSOB(<qgB3IYxiFI<csW7~*_r)<=j%cXZ
zR3SP2O!PDWDTm5=I3Nt1;~k8)>d#M{`|Kld=riWEyMKF{1-*i^^t)0mziLCq4f@!O
z>wGZ~_8UgulbA7{1$2o39`C7N2o0j2Z+?36{apOKX>ZP>jje8lW?;vQ@*?+{(9~G4
z@2jKL#t<C3-iw}hgO<n_obRbqVvw8-N_d1Eav~zI5D|4yK{Vt$b;s8K8BA&qmj!CR
zaRcq(j_0182l03dM>|+-b0qVud+eEPKqv$AkHFi6vGiDgO9JsTn=rkySZ|B|d~%}c
z4}>^Vw0D5mB0Za8-TpODPqAMrY?vuwLex}@epq?!)Y&?KV|n2~C5U92r51y2J`ke6
z=e4n88A#|u(gh$YArS!a0}7=-Wd|7$DK7<Rq9*}7cLr-qq6z5*h3-rJZqEf`h>#P~
z<3JrgnLoIv5|V*y-T6VBpE$XQ##i_m!d6`|RRoYrAdDgDq1(=&DX?Hx_NNnKoyA$Q
zIQ6N%&L^dlwN``d2!oscNoE_|V4-LNI4lgSjb){p^IVu84|aMuW8V|{gOHrW{`7da
z!SEgf2UJ2Krb70_4#-deI6dumo`Om)E18gzP`K_A1{%fP!$S|XF5IJ&!&R>Cc7UH{
z4cPMb=fWRwpp6oE4>b1`w_ofyKS&IvQKcM70eDY*i1-tqIvLo}FKO2RMu*x?f!!^;
z`jGF!NE*D0WL&af_!K;gAB(4;Z}3{zb9w`2m0MU7qOeDTLobzSLPF4#o@;*E-uu%?
zH)8xeU;!_#h62+I-<x-go30*GRpva(1%eN4vQJ6Yk^bhZu>iuMN)A-O=!lV?xOBiP
z<zf@L;r#;D?66gX(VHQk&=*K<!(wlne|4B?355nSS}UDwaE|(a<Z!C@*c9$_)O=ST
zBroJD?9#QGKH%n#yg<EnygLS17SC-<;^nFi?Zl!Vf80h~k{7nO^QkdtjD!z_yFD$K
z>ZQZ!2cHlEIpZ`J6up0!nFkAkG(kMQ<Nn6<2W}jf)lmT+CV3WMhfc7@{qeqNs*oFK
z1b-kEYlc9*H0UE2`0F`jEJ$1BnIAwMcj@~o=drbnj7(9nq7UGA5xj4N-=F{*C&+5#
zb0_}Oj$k1odf5<g=x?A+A#AH7-nK>~$$f!wz9&0eHi<u$^Svh@%sB_<tTb&ri$k^c
z+huI9?ey;)5ZZSLuY#gNowED$;*I-?ZGwkZ@$@BtYt(%P&}T}e=n7W#jjZTAz_!RZ
zG!#cPvsTK=L^2t394FW}G4gzDw8A1(^n(!iXiLPkfzqc+j!3h@-dG>eb%Ma8#S$XO
z0d(=Iwz?j2wVJ-Ue$Q(5G{*2P1_(48RBO=9Ib2uax<0Wp>g&F}F#Ux2F@o0{te`!*
zF!uoi<HzHNMrd~ZI+Qu|Znh-wsNnNVh49-nS|FYV{l`V-56W$RTXa9yg@*4s1g$TF
zo)q*K9a4gk+l1Bzh0d)c=-?=u7vx}P_XVba#<3bIkw)6k_79*9Hx4R)pWmZAo)97X
zi)0`J&w1eGk>LoJA9&jAGUipzzc=oWJ#Zl&dJ77CGjMx-)Es8pB1Qbqz0ANhM)(9C
zGeik>kuuN*<7ZUbRlNZgxgwvy>=falIIus8)kR$3%M}L;xyzBI20vbjo2@RvN`xuF
zLrJv~d2bSm_(8wy5N9&)W4ECab9=CBNnmbx7+5zjyj(}2p%Wp45S}W^e}?{nhVBh`
zXo%k3K6q$B9kjC>6-P!eD1F0M%F-&&cOkWfH`+iKRBgP{cKPk3T<z|3X#1A)eJHod
zyML*rRCs7du@AmYte1cJ`x3cAS8rDaF5Oxi=Ji~7+hmBpl;;5>K&1a3NKkGrNct+|
z8cUxD+?jVvUd7*LxU5z1$>E3Btgntl20)pk{Rp=Djf;g?6fSY+FaRcC1iiB(!GF2w
z?gR<fU~$=um~}t5-}I^(P=|Cn7=?k-gIxGmxUVhmx5||-4<Vh8%3M|liuvKzc)YjE
zuMlc`Xp@jw)n9_^uSmdpJ&6l@MMWrp;YFnc?Y~pG1vL|XVx5L2?a&9~dwz{L=*^pd
zm2ZVt&Wy$I4N&-=Y?bB1k#d7eK)Wsc-irnTumD&lwZz;Nl_JvYVX0du459?n-!o^H
zd4+W91K4(D$dNJXR(*!PYRQHE0=+5YabJjXE`CVfy(Qv#&!PvA<{i&WOqFd*9Q=h7
zoKCvCNA;@T5pEUiIM<jf5N<2U58yxh!x)w~eSMkH&~o<POHeXfIen@2w~9Q^k4Sqv
zMR0Z<c=~`fnM{DMCHkgBaLCE=VKZbx*2(uZ%d9l|o4)x06FWsiEKVfPFE?z!kEh@I
zc7d(sRDJ*r>+={{pZKU()FD2I5rZK%!{GwDL|&(`#_(GY3k`m3)ZQ#RQ^b%iYZwfj
z_NCb#mCnlxe+SU|0N6sS63lc2wZM%E4JG;d^x97><J#e<K!3A1LdV9@l`gAM2#P`b
zyY(wYIpQWH+5XR2k$p&Hfl@<rW;0~&>5}g#vVaOcLs&JLZIu$74WRAU^~mEt-s%nu
z4GpC#O}!3YM+k5JEUWMm8khDAgA2a_21mg$i~(v)TvdUbGpHK@humSRgKleEpj)16
zUkBjMVUVoRntxZZ#!}V)`V6}Z)G(g86#u1;WB`5?XzRGQ*I<o)0miZc&n+Q+2HHwQ
zU1>T%Nk}-nt~`5-B-K~!Evs7m^I4juIm<XVJeCS^bq#Z%g_~&&g5pDj_c_jY?TmTD
zS|V6N3t%HvbT0D?5VX{Hf=n)>Zke3cL#ieqEWlX`OvJmTggG$p*^X*EuZ)-jPT76M
zmR9vZ4(Xg7n1NF}u)d&vB5H8!-X<I;8T=z8EhhZvvhcW&A9U~!cz8d~*JdQTB_9Dj
zlKSu_(!eT1XfMg#yF5_oQn0)QDUlX(j)hdv^S-&@z9v=36cX#9k8cCo?SDZmWAi_E
zNyh$i&O2dy*dJO+K9y$LYkQ#9ffi`o9MuRXO3YQz^gOt|pxvG#v}||f_p<+Ke&q@;
z$iD&t<dncN^kH+bY_xL%e9w-jaJ6Qh!zifUiMxaqg5Wk9d!>3;w^Q}KyHFiu(`mA?
z?0&gxh{KA3L5f@tv!EiBIpa;OnSxuUZ3#T~K%4>)CI>h}><YB*EcZjG)<qGZ7-Fz|
zR}^6^23aH%bqZP~Q$Er%ze3_I-^zWr!{zc9oXMVSXTXH?P=T^WfEUl_lHZfB!^ak1
zh!2526!!JTJrZ$ossePX#G5b>dmCd-lMq4roVnm8UV}T*O#tm?WDu|+F=2O@k>y#f
z+JyJ<!@2@{oANV_3CGCPv^lSQ)WUXax#PY6+_a}x6Wf_gsfPg%ZO2nDwTR^3Y~k2>
zW#GO0^#Hg*JZx^^AZrAkpXUW2k4P&E$BFySX{aOxb*pI&@gOW0q<MG{NxYy#$#|~g
zJ-gkw@YoNe8Rb6Gx6&PuIskn~R-;veCc7HI&tXL++;^7dD`R$%oXQWIorA+VPSMK_
zNvM#s2@m2r&<;h^=(~s@Mbs)xO~~;1@ARM@(t|<HWEQQm+k0&TNWuP>h(6>J;6viq
zbZDV@=BAt(zI=-W6$O%UypLqKiRQt%ty%IbV_){+!(wwmGA^r_?{O_iN`uFC2hKWq
z+#8<?SBQc$4iDc79?X4PjSzrIg+k1~%ENPBjEWseiK2lBK8$2~WEH0%x7&zdgP4}^
z>Up!_vb+cutv-mN`iP7$@esDSX6SVnjXt)-I|>#ME+PRQ2I9RT3sW>g$OH}1-a>A{
zRM$hcNCA%>$WdG4?lBh`BJ_`;3ikNVr9n)zIDLw_pl#(c7ovC|^2Y{9u3<C9zqWV?
zkshv(dbOh&xX0kg#vtYIyh>ZO9|dV=34o*CW>}FO>j>)vG?(}p2P6*#nZ*rcCF#xi
zTlK(z7%X6~-S?-Xn~}_wKyWW*1a?kb2kR}M12$Bp7*N;akwR?Tj7st0ZH*!^<Xl<>
zG%~*af^!3^mvEpd#r#1*?TNb^q;N}%d+U?ZMAwyH#r{jiph~L)feBC(WCIXOstk%x
zo@<B3BJS}T4K5sZp2Ag_wPw!%cpBgkYgB+Jyuf{MOFQ1%qIJf8KVDyZ2&`yHF`36<
zMhYxmoAq)aQUbRpaYfPzKw)L*c7)*%Uf}T*v|Pa=w1e!p?w&h-u&}Ei6o`Jo(Fzvi
z(2ltv+~~SDW)z|T+@-qXbK)7FmZDu@&m}h}eJk-QX8H#lNY8<QjOTv!z%mdMZrERZ
zQv)-Hp>R;>(_f6FKai7l_SfK`%?l8cT~>7=)BwP>dfn_h+f-OJb_@LH5S?Mw_)e~)
zN<e7=Jw&(4b%m_x9B@!VLv-!&PCSfu-?r#{FQTU7^e8woWP&QB3S6vlc&CT|Cl+UG
zCjNdbk3+_Y_u(?T{ghOf=TCJqr5j-9$#r0Z!0`x#jyf`M$?xb3pE%bI!F`Eg(Xu`j
zsbFo~`efWp;!(T`$DddaBE~@SnYBIt@l0RJ0{>j@%Zso6Fj(#3a#pWPPyb<N&Pal8
z1TPD@u`J7Cs6GzNx5QkK+iC#vyfX7{`hbTII}xjWhKn2y+~^P`7#yq2R>{NzQDf%U
zV;r#uVDML7En)hh@jPK$b};l985uWb;%%GY2$gP~Kp`A8qar*+xASiXpGm%%^5n2*
zAo&*RnaKnRqmfvrpQ@a`N9YO);j-LQ@UkM<u}8Til-QXO2tZI{&9ugwc0adpF0OEx
zZF+etNj<3gOb!4vPNA}}@)xGYhHTo<c1r-{#HH9%VE9!b9!6*FR^C`qY<e~@dc$9%
z*rXB22f5IT%rqQYU+^|GC(R>ZNI)$b$X$Lx$DWl*FvzePQk$eg%aYQ-P3+ONCkIIJ
zz@+thEWcZk<3bOkJ)fxpycI%DZWCO%9F$N83N;9g3q;<w(059P1+`F#71Yw1Aj#ne
zwFTtpB|UOZ49Ou6c2_A)@1=aksn3Kdc$Sf+zvb}D(<%qH?A~3Fbli_RlO%<OjiA`k
zRcfi$$^1dUeJiBUupx{-g6$mopN^1mX#FlSU+pW<V?)m6kstWu!E*c7onrG-yhCU@
zp51(UG!$u7(p0mRPxbs<VA@JxFG2Q299T!#TVHhnO0OPIW&kj9+bq?hlVP_pK!rkL
zG?eOOet&T?hVb?vPLb`BEnyHGX%rUY86-uYUh8NfaaYkgQ~KM1Af23Ea0C2+`zs5p
z_&dW%MozxCngY=Ler?A<GZi+}#27x4Ytg`oO(;N6BLxmliykSH_9W}mkm=9t7tWr1
zgcZf2WfF*nvTb_?t^j1(h^PfveG77u`$A<Bbqd0P2M~xtZ%PiG%>YgX9`*gfMBW>`
z56%`A?<(Z!R6hQ1rDcVz5ZrGH=^9ax{rI<sZpK;Whnjb1NF$OBlz<<^wU7KhD+K+L
z7VdrymM<Z=KSkuW_a|@~ct?x*qZvRaK?o@uem`<RZ~=IRES=@`HMXeBHzlN!1-<Mw
zBllqnDD*yI2j_dJb%r9zq|KLk`cCR7q}(}xnG5nr3M^m>PV(hnnZCM7HB(^^#gZ+u
z%>{vu#kUK5;yrtd`xxZ9Jdh{?3Hn@O*7-o!WBFd!^8idkQgj7yo+)&_0oFHM5InCz
zCJOaju->1frq8t!;p7EO;00mKohQ$>7J46oq6Th5;nbt_Vt_23<{zM`3PH#9p8_mL
z;1M%H*$c#Cwke81=%WS-!tDp4uo>2V<JV1q8$j`yGTdhH1YuY~)bn>ZQ{uvuj|#XC
zxU2>`pkg=?baSJKbP;(nj+n6id{;V%NhT&4pu&}ycjGKY>QII7BM8^s*TacM^WvBK
zoSy7R=vB|F*~;8(HO0~&#D>=nQSd{KY>oPC1D9)tjGjtO`>D9#ZruV7Z2>IWMw!69
z_Do;F5Q%P2AR&)zYW7Xsj@BEbH)9n<A0vT*hVhfvp);@_;LI@hcT%<9$21ilJ1>O4
zAs}Pbtz0ODX9X!<#=bVFlmdri<+fu|D@k5R?F;9sqOd|6xJ>KCG;W;5M$)e5B!uCu
zVMu~z9vAnDL{*rL%`#5Giz!{##Ad9<OT^`e^VxDM5qA8Eg`70H8m|L73HbPnp^rf6
z{*_8Gs=Ht?S5nEd#UrqyZfh6-K)f-D+s*`?MTz%H<yq?!!^b)ZpHRKG2?fa(gqk?U
zAb^c$AVocsi}Cj*juG%Eb6grwYKmsFg;GYA5+)+3LYL&V7!X+Hh?;C&g8v`(zOpNi
zt=qD2_XO8Kkl^mF!QI{6-5ru3A-KD{yC=94+=IKj^+QhI9{t{N`WN*5N<E{fT6@V{
zbFRIM{oOAo6;SO{@kJ&q1s~JM1N8}@Wla<PI+Kv2{T3KiU7)OHk%>Jrv!su=^Iz27
z8Ja$RF$(neVhIb-850O^X93u~sZku5>o4UTwme@6K}3&#5r&h$I~sAUNL$^z?Kge<
zf;<an8ML^hpq<fVL0YQ=>eQbh4aouQ+ukQKlERc~$||=x!sLlVzWe=QyL_2WTSwnU
z*J^d?p<sdU0^=BT@F}}Hh`z%szjx=eA(4;9<8~jTznj?keK6Bokz8E!iOtI^2-b`T
z(V`#L`%XE){HbL|1&w6ZZ)Z9OHgz7P!UWs%UOt-r(mba`0q2BoIu9f5?@5A!1U8)2
zcM|-OPt#Zpe|X;xqVKbBGg9f61Z#<IG=VgVrgLst0QEQGh=gjuv)?P}FUfauvj<A2
z9PjNnQN%$s?i=`U((+&PXw>rG3`4iybV~z>m8BToKu<UZ82gKTa4rb|TiDC=7Afjo
zk!ud%{_C##Ab?Jc$_{S(v;i9?44}$}y+xuxl|NqRCM3{(+y$!N0>rkE%XhtK$YB6+
z8uIYl66%Ao`}V^DflC1(mIkz84)*Ou@R7j;I0zEne6kyK!zc^@vJt}R?dMHsf1eWn
zABY56hd`+1I|2#-))r>xE&3ct*ZYW~>^c6L;cbWo65*d0UPd1ID;|h-Me^fs*uw%n
z`&A$X02LPVpBHQ*NCLHDus<XH@%%V?Hc)}!X@kFIo93_e1?;&QVBQRX_tyZu|6_pv
zPdxx#TmN<Kb%0*)mAHSV{38y``)`MfEpI-H{qH^t+Phm*TVp)oF*g(?qm|cl9&fUq
z_QgR*>n)HVNdKD|fobFgjfIQ;@y@}@wThT@KihcHsB+5fQq=^~XkN{!sp%lCubiSE
zs>~kg7TRouEBZg_w&d*(s0_OBstO1|xjt2^Kxa7BgASMYEF%?*mMMqvjQs0x>n$i$
z|1&5uuE4q)iG1kB*Eof4P8KnKf5=DS6`^b0^yDlbz5ns!2YOu^&g6S_u3ICIhVd$G
z6vy8V@y2IM<6AAd3guY@BWtP^F(W?7im!hPL|xRQVo?GbGu#ytMzx3|={x1ePG{+e
zwE9*{-V#C~sE7T(6T)W#!24<nkLkI)@F=<KPrl?pwJFb8cUU~D8pQ1IsnzV~9sWma
zYF2aK24@0R;-+6o7%E_EK$c>;<&TsU+zWS=!*t4HnALZV$#~3W{5NeQUI!{(Du(N6
zaJKIPjL2jDM)5iNe?+nS6g1#@|9Vo8ZakN8teEeLwEyx`pGxb+-mY}-ES6o2k9v5Q
z{<~Ht5nX?pdmD&v{VLYQ4q(?Gxo{R4f63vrCy5sNJLo>1`1k(rzyD_%i)91uwo6Y<
zM8cM=sm*s^;}FJRZPA#;YeU{qEAW^no1$`7@hg?2X$v!b#fp{TfBwYpdtYu$z(E{u
zsTw;(q0)F6#P(ztii~mZ@k`Qayqk^tspv={EcFrj)90B7itV~DPqk*1Wv{;FZXP&J
z^+I8tdsl@x)SVBPl}C}0ABW<8EbeC5o){GE>CAcLZcmrurTS`9XwAKnzz{hUwa;b$
zfm0D}9JX0J>m9$iR4d)0#~>!~l9VYn9ZJ=uA#EImuj4@n>pCgt-g=Zv9Gpe+8$gSX
zPQbvOi=Em=M%Kz0@p&{ku8Hq6i$y|Z{z_yV8D(c*6UWiyJYdsk^Q6>Zb2m-*=={KQ
z?>+0~sGfh}INJOCH0#zu-@$x){JeyS^DwBrAyHkxMYrNMbvl%=#wIx^EIB+m6Md~i
z=65{*5zp^whPx8<&Vw8M%2k<udY9jxj5}hvM|^1WhY77aHf6RZa-***4j;vCJ52}K
zYo_r_4M#WgJad&qD!*>CJ-0v7FP!yiYK69QyDb?V-d6m9>t8s(Znnvs+xBWWj+3$S
zt1VUIYi@bRY=dt3;B!s?`t0@48C1KvXtMU%wW0ssT+hXPt-3Jn`(ZoZ1NYhV)T!~|
zso%f}E%(T6R$RGNq?n7)3*Q|9*$~lNEvu0HZ(&sj)-_HJxL<Vw>n$;EgT;v%7|S0F
zcJ^k1(o-8hpY9SSxRJ{;rjKm<S{=$1v&(r()Cps9MD9B#KAkOm?OM30E%?}W6T@w7
z#%d3B>>trmQt9S8N`AeK>4ey>c)nUYeLOlc)-&r<5sT33X3w{%R(>hv7G2Z*bq)Md
zaJ(XOx;Y*PWkpvqbZ?7rfQLBN)=R#_*ROi5;FlymnVE~F#cBMR7HR-W2ok{OZd=f1
z5xU7C+*~KIxKZkQFzC41<RuKvlPQ}@p+tm#+8C)6uNW2=GkkhLSE%P2z&!lO<#sfj
zvpM9H`q0LsSi*XSa&fhDxp8b;ox1usRx4=p`m%+?<a9UfI1m#yD4f}L<Q87J`lw94
z>fPy<;Tzo=u}5FIavMyO^3XuAx<k`uxo6`j_V_J5dUejCpd+)5Y3cMam%Y<U-g_nY
zS=e>!8h){d%E@Fl+-xhe^X2Ab*uzO=8{YBrcxj$O=_T1ONVbslI-^(pQoLhZmZGUg
zM{tVK6DPG;Y8!dyRhPu!?dR6V@~OuyOg4erR13c4mnm4zm*0y4we6W}KBg%nIG2G)
zckRrtUQXd~S(Vt!QU@vPL*;{SIN?C@zvV4Fz<ngS>&48uDjn)L&&}hCu6UJUS5`pZ
zi4)8}gJ2ff^jC^XegS7|+hm%HPP2Wn4sYSRAbJ#UX+E!Kf>evWq}Zs&^r@LlpO2oH
z8WaS!&i%jp1IHv^?uVh1Uw>QGcs(NDpKN}-LG+8gNgOyVz$l0%lpDu9)L5WCJ)|{r
zHMiY8e65RhRf~;kC|m5VG#xV~(%Dq^B7dEW=TAYDD3uYXS9y^06LN0zS<egT{gZq;
zUz6cIX0da0qfk(B*cLGIKFD)uL9HHsxlBG20Rt#=cZ=bfegs<r<J<XZnR5$wv~rzT
z=5ChV%<0ydHh-nuykGoilkS7i?0PSW<6%O?d3WZRW0rDF^HRB1mf+GSUMpIxkf|m%
z=&*ATc6v3_nmP2<4Qo-uDzUQ0?C7-}-6`eNa_S@_RRjWVt(XC<%dX&<&lAM)by(Ee
zAe5VdGTV?>Y(Z+2TS+^L!LVItDz(Pr)&nC%{%VNX1EFS}6ZhfdXV(nh*K5g%8-BJ_
zfm7ylV>8g`5tSO^!FFS$@5F;_+xA+QF!y7B@+P6?hJ@G5R)7B5&`o6O@vYKnK)5n_
zd6?#!aRu8G4MqBlyr(Qau5y9O&e}@s+?vtu+Vz7vC;7S;&;CuAG@j|g_5;C$pD@=p
zCgA<tZN`+n+wK~krMp0F77G8;_Q&hqW=3vS!Vlsk<vcj9Qgz{^vgmd9t-!jF{YN!F
z&qD&@^u|*<Uj{nU&2~rZa}!3bcM{S<Jvl~qAc2SG&F`uC`|-4CqB&;D{GR7XkM|8&
z;*e07i-L0F1F*OsSbXLo@6Yb%j$NO1@ob0`w^oYB@zzSQJfU^Ngv|k^VQsXDu~JvV
z*2oU613kwCw^I7G2MO|9pLt*8nGKE0UvoA0bCHf>t>4;~IP9)W_bpH&8VQC{`_?EZ
z(pM%ljJZ>A{r9TkTi39$CY{ruKxZmRjF29CEp2G+YOiQ6uK48M($u*oiR;Xa(@i7F
z1>opkjof6|aOaH}Y`3}mW?wx2ja($_iK#PQu*<T2OX%MN3CQBGq|V&oh0RAVVus;+
z6gji1H8@Kw_+YtVAQ4_$&cGGUxv-%fGejTr4Y(T4>@8a@x+ij`>!wc{9ibLG=^C9f
z`COLW)6uf7QSn0fp0>t5hi>i7d_c0NPSzI+`yy&Nl_l0`g)5J!b!90}oF7n++&N{S
zcst_z>BxX}<#feN$7Bxf`n2242nm<P9TA_aK<)g>GDFt0UZ%38nXws!)wFrx{saH>
z6hi<u;K$gVJoa_G!Urs0_;BhfIF%N+zDfmhosMzI0wB24g~-Bn@^n%Q;{N+M(*<2_
zHXlpUS&qy0<EHJAbawCn^Yt7v;p>P7Sy{k-xVATM2j}{T=XotqZ8yn)L~e2XHW@jU
zS$4TqD2ct7?(pN`=)xHDVsaGdC`kgn_8&}jqHg!S5#uNBr?gXboB0Meah;+aU|pQ)
zT1#0}_9)w<Dkw7Bd))i-_yj+cDtgeEj92rrDB3M23cK}W&3^dYZa?aV>FqPn5ESHt
zY04_mKmtB`IQz%m++baRlj`#ALDck@Iby*%l(eg|Mrd$cY;*s0?#L7GuqX2qoQdie
zw^4jN=a0YKRwUZzO0^)ykF6mRy8MVd?S=t@7o!tl^OeR@ZTH^y1}LeS0P%9viJfoI
z^T{DEJ9|_o`*Mkb*3MfH#Q$e&be@L-)Ztu3v_XSs%4XVy_1v*VW-OfQhYCPO8M%T3
z1ZHKMkr0Jp^qmXK^Y^*!IP8s40jmT1JYo9oOIp?*i>cWk7Wa`@@1X%M{M~}Wlgo;R
zWkiRCdXLuTuMP!k8b^^k1#2*xS5=4m$s-8TFrUMM_D~b;V^{Z<3tvoJkLqTHxw!(S
z;~x7L8MnB$B*F+qEiaG&$3?Sr(?Tf}=8QErT)$2I`<q;PM;@h9#P^f6b11a#zucy?
zZj)b+5_Pm1m$FGurPD0O84m3$Tg@U7H0P2xT!E}xN@v2^IjSg{<mUk1pp_yO=FUgL
zlhy%k90vC2uom<jJZ?2#XY=p6gBR&LsgJs=_^b<KCDyjVbAAvMnJbhBD8F%?(wqHE
z&*2?T{2Ye1>WY!QJlBJ;o|rs25AQlK$w3El1hNi{vkB)p-~ImThMAl?d?<N3_^NAg
z$t$bS<_2omn>^9}1LSgm2?%iS$)ga6o&CZ)_2S;fh1_ExTBB+N^tGb}T_dVpqaNgl
zHd~7cqF<A7<SP3o&5+G0xb26mfUI;sB=8F+i$a^R&G98_jIjq#vAeq2-Os|zSD$O)
zp=1w7?%W7ubEm(Uu`w8Jq9AFD`aZr;Ea=x@>^I%{DiI=R>Q3NvfC&sQRUjqXNH9O9
zId=F#7lN7TTGZa+Yvkt;JQBG-@sh7kkuY;1y*hv0gIM#XXwQD!OkAroYfA4Laq}pH
z0T!VHZYl2)^wz8<yu@=|ZwTR@qX-H9d>+zbv#x>W!atd$FL*ruV|TgUwtM3J6IydD
z@G8Q2iE$3dbFJ;$`6|;=OKDh{+5S?|#-X3r{)fT3yL7d|Yd>SM#=Y-}$L;4O2C!Gf
z@Z8~jabfLsEprS`0q+0$4HbA=3j}e){sOv?%do%;3;?{oe%v`P`L81)kJ6#q-v7et
zkA6@?L{w%_KtyC<lxOII#6bP7nAggk&z&zU&;ltc00;3|uo{4XkcWf%N%#Hr33`{w
z&F!?~adoMG)$1OKXmzEPk^3>do!MzD&1oko=+kHLyp2p+Nq<Frx}Iom-|yKoOM_$d
zjf0;HGd}ptj{Go+=4NvgTH>Z2kmEm!2lYTh?JAO@H>r`kcy5W;m9&XYbj|Wy9bB_V
z34SO~)IrpZarCPzS+_d&+aDdfRT77a%`&nt^_W}}R`m*W2&gW8Z^j%RMON}ZDL?Qo
zsGlBOJRmsELha6n`7}1#EYp;lwv(UI>ymVyJX1N^HpI&PI0#z$+`(-Zb<_);0XB{<
z$Hi|)XmL9<;4T|^n0AXv7waJ7b>IY3mPt)%q`415X$gIE{0XxbF`Ma$5s9@<nOOJ8
zY)V{+Q)_a3w5{46OGu8I70lbg7`20;jk7~h^bH3OmXA|s1FU0_YS8V(m62)WyE?wr
zb(#D_9`n~eOQEAvt<Dhbozs6Deq82U=8k#zJbhy-yv63^ozs}wJf>3UA4x>AYbPbO
z<S*^)>$QB`)yjJ`svq6%$m}O{OG8{}>T*F&iMpM6tae?+yBHk0H3s$ZB4gcNI&RMF
zoSSL2A07ke?=Bnp^ISF@FUB+dJ~zuEZH+mi3<o?v=F4ncTrA0T4otVFNOhy%=6jT8
zcC3-woR$=gTQN0PYWrT+GgfNwc!sv|=(q9t9%z_JtZG%nPdjE%^_y`EuIkkZx%$-#
zSF}AtTX1x!jquc52&_;OjSL-4r@Q6UZ}0m8x&q*20^s6F0@%lQjPC%$xf));hVziK
z`EHT=14ZFS0I_0OlFx5fc+l^bR?$s)Zoma0RQg9qZ@~a#KR&6-Lt{cJVGFk-A|5S5
z!X!j;89FdB%HaO^@GNt&+S6$AYb3<;EHprOH-B?`<kB&D+W66wRo?q*LZDi?6xUJV
zr7gy2EYi9#ZOd!f!&h_7=H(pGQpT6y)JyEfyH%;8v-A5qpOWB(<9m<6$onn7{#QAo
zpsKmUFVLR$9=#AD`Hq<hp6hmWX-j5hGz9G2A0Yu)Ng0GVW$=A4D#d+=?#n*OetfSo
zTw*<p!>ZNSM%?dq?w14_%x+(-y}i;`!`$*`z6pC*4i9uYjCu}wA*}vrYt*HPq_~NZ
z8D5JwU|j@l5i<d8J1rRga=<H1{fio~R_Jkkg&$=p6w3cL7)x+C_$tf6_v1N_?fIHn
z2?!@fg9pMi?7u-u_(T375db45loUV1*D77K7osmmVr!I>0`}bPOcpLz=y<vPHNazA
z-c{1IcD$%od~(iZ!FH2qwTarelz;<nc-iT+D0;12CR#ta*nMc^cTmp_$N5&k9{)S&
z&y<bj=gFau$5OS<+_~Ug^PwwOi2d)ODig)%ucBHbl6*Xh*DfEPZ0%%!;O}lnw6MiP
z0zB%s{Raa#<DCiI8Qu~@ZnoIhWy$wU?bE0ld_Dn~j$Rc><P#|{$s>>O-H^;5F5_H%
zuhV`x4wH&*C;OKFX;End?^^DV-1G8nSl4}C^O<Bt5C!XsOa=E(g}eO@z<u{80)q#9
zox%BT`sq6dmz4A-P1&lYrN|ozvS2VL?yhyXxh3(eA^rcjP!EtIWk!8kQIo7)?etb^
zuM|oYQ*5$6A2=UFQH?!q^rc8#iPKAw-6vXzMv8+oh4XtoFNEZU@DJdT{;1!uF75a9
zRLas>U$^>fA5^%N?HMLbI^y9I-;BnRov7^>&yulBpTo%BUwgm|vQCb|w8?50oFk~o
zvv|C)A9Oi~a(F+<c74Y5D|x;pv+Z(fip31Tj83Maza=VUoorVfli#*yZbZ0RDb)KM
zjqY+?!9~hF%qj{mhMc?F$8z+EsJ(+r9)oyUU(qVR-Phs^rt{(m0eNAXgXUI;h0Qtv
z??xU5uN(wvm5@*emab3ot_RM_7maUhLy#!s)~FSFc#JB(YNz(?j4RsY*3~Cyl>jhJ
zoun@a?A1HA7>}ck!Ui%!qK^Ui<T7CE3<$O{AnOY7KvDufg{l0V7>aNe>m%7F1Dd5i
zn=FF2>gE>z?E14EuvuTuW9#O!k(&2%Iq7;P2KX-^))h9!x9KgNoi5djhpJkY+WeB^
z<_@d*LiC>1oB>xj3Fu}gR?83$n%28{D)L5N9B$0?F3zmloBo*RtYwFH9B~IoA9JW9
zDbGrN*_nx3P>2cVhkzDB2$fBUXV#m!wPCl*OY;*YY`1ywZgP&t4R(zUdVMJ!wahqJ
zRhCt}IhSx>Rg3?O5uC?#e3&E!em_zK_%mJ8F3Be%X?i0f8bn7!a$j#x?+n98seZz{
zu28_$xidj5`>maMt6%PTp&n~?#Ws?^!_TgO#8S9_dIlFD5YBZa*#N)_o>ag`Q?B+8
z(tF)dMLuHks@GLMUe2xMFN}sgN$7X#em)QBgd5A8^eD<0?Yb9y0Ej#7ez&p-Bsc0a
zpg5&#_Hv6_-;hKIT_Ryrr3iDCfSqla{nBsMYbJlVy%0G}L93;=@41vQpA;XbNpsm~
z27SZ<a{1F4OX(7<ht<V!3$nf5(`*ZUD!NDIbc+EfI>Mw>2lt?LoLw6_Q5L^ztcaU`
z1V=XAt}VExF>uEBHOT?i#l!j~)(24M8=V>>${x#jCR-BN5_y>1c$wVpulqCW7j}D*
zJ`J+Et$7;Tru=e)%O&9d<O3kTc*!U5@Js0u@QEV%zI;`thV(azFF+Y<C*K}(Oa3hn
zL0p(yjs}c)V87du@A+9#*)f^odUQJ_;oGnSfm3l5fBRaiXZ3zV4Zypq<iAta3k!fe
znll*JwvlfwO2iw>8E4-o>7+IjN9_5|(8YV=_d5Lorvg(q`IzSSov7{(TmFMVmG4-7
zGZcgzKJ&h!3Cf!FB5=uiaCv?16i0es-j1>;-wR<Xi;BpAZOd&J+=a$b6sG_N+=dae
zW*Iu&AKvmiJIJa){5&#CE5T=TgFjk}A+*yJB`mtnTkRu)g=vpR0ifF@P?5)ny{fHO
z&YEZ0a&2$^c+B`;&i0>7@_|2y^b?KLUC}N{bSyYaBn4H)9dBVkAu*w+iDol4Orw5e
z@ojNzUk$ls=x^q{1_-RjTc19Bl()Tn2k<G(b#3j;5&2DBe?gRIFUl=Nz?z7vx$#L`
zBI;}w6Pr2AIYjMe&~iL_jfy2|u#Il(XAO1!j(5U;L|S#TJ!kSGg&>HW5N>RWlb%mc
z&uCTZ$gfR9#Uo2h!x}oH!S|{8ifWqS$0_f4oFc)Bfl}*0z$=^Y>a$VY^ri1La#uDs
z+_h7hr)l2w7@yK&@$Ed>NYpir3jYN+XsTRUc2kS*LHZSXLjajVpDd}rTlz$zkbw)i
z(=6{Fe<3~u(IYSI@N83(jWV+QKg&A3X4k;T3-)dqTzfe7TfFSXA5u-v33|-?qb0$*
zev?91hM%0zJVXW-@sk^zL5@Dngut6eHYdV`p#cnMcVeQm3vDMxPR&uckhNLr3f}rm
z4E3AD<=Jdl=?*6go@e&k$Lt40bkJ3@oe)VI`oyR4sO8YR{gM-V9bDceOa>Y`(BJsQ
z(?h{_+f_0qAh0)4@mzZR^z`Au${C=Q$OZ4Z^lunhlg0?Dib4NTo(N*DGp>Ltk}CE-
z7=!KT2JL=8c30Om=3S|huB~rVxYW1TrH)LM-?@Y(eEd^rsrCfH)v5759t*B6+w}_5
z#)C;3idlC$7t(79t_=!Hs1;doZeslb{#WjoB?^HAbRt2#9UgYibSV*Mvig9JMZnWk
zsWGn{n0`>o430u+ZZp{dhDjM)mCbLyO8RvkzcUe=6w|06&fsZrJl^sr75Jo;7&%xn
z`fEJu%<_Br$FL-brW5avECvOGI0{2)7{ASS4TsWfhuyeZXNP_doIdxK>6QwGg|w@i
z_)pATkA6`IW6WhyO*XJm5Q2eZBICft`lO`IYo?OCViZ@FR4s5)ai>0_(OQHE@q#Ya
zzi|s&Oa>5tn1@@a87z-p!+Q_eb9(P_!~jOwf{C1-2f3%?=y%N&kP1QAF}+bvzqUlF
z=XW=_LGWrrGNsFd)}xI@zJN?g#&9Q|34S@;H-s`aGEcFPdxz;}+PG0;eZf->luvcW
zFW;@FckEI;bTcfT=V}5~da#J|s8G=R-tT)*4Tzq9paBsYfDmfW)5gyuio=Oa3!Uh;
zY_w2*06LGspb7)hAEBa+W$3DhFC*Co7FhFpj>OL$Q6%0=b9^XvFoQW}tJ`~YJ)Sk`
zyG8{^=M-!bHi8|!@%YMmKv@o1i*0@;$H&`RGQO-(@DW?u5VL!q>?odsOJM$v=x&#Y
z^D*aA=h|8?C>bgaqn2rcCe!0(!(95tL!&y<YqLWiif>ofQ$ZU%k&2m%0QLv0!NIoK
zkM|~gQyw{ycQ@1YGfq1bop?)-h~%QEzG<DCqzZj+5wk1#5ey(PV|0Rdl1?&V%8v$o
znfuz%CeN}W#}2NB^xucx^E=5r>L!ia@tP&IQ*Sp+vV#IqE}9lK5Z^a+VVq~>lG9+{
zAIB!|m1<3(5FE+}D=w9sq^|b(7E0<XEs;+NO0P9X0)2k->Wc`6O&gwTPZAt+*$wZ}
z=9tYD@7c7-7H87PEa9Za#ajIJX<{wDpbtteK_T$Hd(jCVz!?cm{9DTlV1p+e?%pSX
z`xB*~MH}dwnYLf9c*J6w*On+7fQ)VZAjGHjEVvZ|0`c0ZM+8XVT_Y8}#tU(3Th8^e
za6*Yjn4GFLLAqTuO|Pv^IYRioB?c2pv>y`L6+qh=?I&KJif5ObHyC}|6r&3qy^3Kz
z>=x8B%5Oi0n6MgTMl)e4#=sgV!d)tMLh6-GUbn(ZYmo3bt21zIy#G<1K@s7a1};#x
zC}G48<<A0~Gr)cCLL)(ab6x#+z|&r^A7ZV`_^O^LR^G3X?jO4?)U{>upvN8y6Ersq
zc9(Sl19raa5LFcW)DF286-uKMx#WaD@X0}OX|Py|855Zzpr%jciCTh*1go<Zj}_4=
z6qE|W0!iszWjyFl^RCoNj5HVF!3)1t>5(HL3nsXEU)Ft2w98t2Z9DhA5GIoUu3%Vn
z0GZPpiV`qJ#iu6)Lx+tMz$OS=gOCT#axq>Q;em}gLDe`Q#a`qn2)h09&$+^)cje>g
z9up1vFU0<h{@bkg?;xm$jcCbYtNP$)H5sX3!MgSfp{JMF$w%HNIgsc$_RdUsKL|Jt
zCuPPiA;rzm(@kZUans{eJ~J3)q-DW?(k&HWd}boh<~#pPxc{q~NnKl444mwH4rI?B
z$CEsHE}*Yk^J1{%8c=I~Vs&Aey65f#pvGIKe#H#cM@U?8j@)W~8ETS7)9*Q3IJkA0
zH@}M`h7D#$tR2ODfmN44w{j)s!p=$asrQZbzFnB}Tu<Cxq>itg^tk!xiD$|W`yn9M
z*GhXj!?V>pF?4ZF+)u%8ea`qpTe8m%Mi4W{{K-Oxl<z{j9A`pCj!%4rJhfwg&64XF
zpzYHyGt_v1AUI$mBGxM5&~a8G6_0eAO{u-3dnY2ocsJACfd?VDv<uyj_7j3ns_+vn
zQ_kWqNt|Rb|7fhH9(s5|`l<ry3ZC8J?GoE(d`q-ql+z!d6N>?4*zAaIrV-`!%P_8f
z>!0<acFct)57U?j8As}RVHvX2(^#4}G4+?L6MX(>2F_v#mL6Ifg7S#wf4Z}@3!xp&
zn!`vJOhW*Drm)NB5{=6J%9XX0Bdej{*dq;p<LfyP9x4R%SDT!D|AcTeK2j+#zDy@W
z$x(oz69-6Rz~*kAQ9IYb0od5lWpf%xmMG%vy_YK&d%T*tUjyvQ1fihUa}QiqC##T`
z(l#|kzIY0he>AjGu3nRc<zlR{r6Ay&TSa+1hF`%IPu5D>#I7<}Pf$Z@K~pia&OPxY
zKykv)jApB#`DK~d=w?wk;VSw`^!V~2@Y*`#U}3}smYhW$0V?}PI@~txQnfDV^arOX
zT0r?|R>UH0SG0#T)d{t`VrL?%VjVFxp|jMeeY4Hi%B&T7HGCb0+m)lR!ROw7PdB#m
zUb+(S57?Ei?+arTWxQX<0ArRcw(niviXRcm+`)#f^os8ac5y!1^MGx9I?CA{wMc#m
zwr8x7rAU{H-d3td5ztKj<E`xm>W{aBeo?coby%5N1VHVGgx5fK!;iJ2+NK+%*D3;C
zJ&C!$?a&9CIZVrTg<$POo?3IuG#7N?8U_>oIyc#pR?o$cx6V$KI3lXIrQM1-84@r*
z@+v`XowyPj_R7UYG@ykIMs4VY@qz=wg}`eO5LL7CJ6=|WxRg)X@;5?bY0+m*oA<L@
zbt1A>`nWLF?I5C}!i}Q#G*CeWED)IV?#mhg4<!c;C16K#rm<Hl#JI@spd^%q2PgCq
zuM*}7^>S+=S;nA3AqQEJ1s~L_HFj*ga9<flKV)6+=Cc3xB#~4P@PNZa8p-CXc16O(
z5HZ)TvU>i5N-#OrA}#L8sNi2kr1>QE+mm_XA*eDr-XC=x(TE@c(8qKP-oayjr~Yb^
znzGUD$uO~+TahYDK>XG~J{Ni7^+217^LhKkXCRd<w}y<!M>V4c>!12w6fa!U08e!5
zerU=<;kIR(QJ&a$FH#i?=#`IM-B?i_>ikeW?}E8u6;X>&HDJCaVA5%6yIQY)(l?}!
z`tnqx2;3R}^0GAe?b<S5?JGQH7RZ1YW=C_d=&q=@yAk|KD8?-dM!cbifcgE6Loj^v
z>Tn(07-MvRyr`aicGOP^B@LHnjdTd^V*6#&_&h23L7^SeA4-^fP^ccB3;G19|MvHx
z3pU-3CjMcadD?I=81<xI%YhF;bKYd|YRvZ@1|pnt%$?o`64>geBdbYGLsDR!Z{h=c
z@|0Yv`8u}{?5zR<sv&u=oxerux1WcQ5u&fYulJ3V24J_rT^GyD<<-SlDnI{xb*(W2
zs`?OB@z@W8^ci>?9N4=pmn6ni_IlAlTi~8s+%l=T&_~KVd341S>*bRT6{|~cQ+PPQ
z#+s5g0;>FXe{ylFYY<+u=RXS!7D{IyL>^aM!v7in9TjA<J)&0M*Rk2d=wJfZjkVE7
z?QB7IsOuq&0MX|$zXbzk8edbs)^<aBtQ@gPZjul9tE^Tpa{rj?NiIOV*<J21y5<r^
zUG}U<G9{Aydi=Lqd_K3V`VL~d$kt`vyt$Pu?1uasD9|)w@rdS9#+Ov)yCqjrn+OrU
z^^-KTBmzJzZrOI=uLpfX0}68`?o};~%1?Se_N_&tU-cVTJNA?woDst-TPYSX@ge$c
zqkm6PN8P%6ea%o|@W2BDQm!iOq$6OFcbXX8y4`5Z!$OYJs7S&6nNc3S62mM9-dZ%a
z<9c!D8Whn{z<r^Bl#nOmJsi1=y1+zU6(r!~S2+&$KGlLhye>^C67c7-7N1k8#j*ec
zzR9cZ`$^bew8(khCRe4%a{G3Aqr$<T_JDXoD?fPHI`(2m6_stF^eo$!wwFL@zZ=el
zJeCV}L=^CV&l(?ruJL@cFD(y^5V56%mJFwj0>xD07Z#T$p1au()IuqJwX4z1Y-oJF
zZ{>|lXtWmb&1T;P!QWa~Q5j4yAV?KaH|Buah6Z0bohku?0h)><G??-uHY(TjrHtpm
zgWB5c&0W<nJ6w#WK3A~dC}2{tQU5&$jV|q*84k!Y+a!2cDKw;+;eM~$@xl;NX6oWj
zim2eg(}%8gJ(CxRYzVe3bu_|7P3{F|V9bnhS5Q7s*e<}(O|o7vY=qCAMgbB)boN`l
z%tdBycE6I=V0~n57D#5O_pxa5fSfBt1N2K~j>|uzrFz4<sq+#bFRQ(4TXoS9bJI5k
zXMR_;TY0V>)>OkN+o6xlj||JsaBj(OmsdxrE@tQE<@Cc>?R-o~KBUX??U_af2`%pG
z@ReWB_>Sk>M9+2dMhopTL0pDJjAV4G)`Kr@?Z>otQg_#*#9KiIb(`-1p$g~KjJL>T
z5`J^D%C^A|K$)LSGtb)HcO0?D``qiSk$dfGNdHK?%GeE76r?&^?{(yH75UDWGYo?2
z-INbd^`yCjOS;834d|<5dG=$$D;>-%Mw;OM<qG{>$U&B7L*W2Wiya*7twqapUGJC@
zdK%1Zq73Y*ReZUixD!d0XH8w?|6!8^di>f1ndCyO(!kD8W)OF&?dL~Q%$b*adC}Dw
zZ85KcU~neoe=5OKyulhh0HU1h&r;RkNNH_-ggE0~^K+l2Pgz95o<#QK?!fi!k@_$D
zFdju}kcEJ7<DGeb9Jy+)_fy)#T^-hwMf*&4>yQL5&J!?~b<DLsJd=(8_|Jn@_r)9H
z^+yQ+0rbp2WRWOchnu^U{ezPa3TXvwJNJfgKtpo?C;M!n)4FjFeIM7_>Zx;^v=rb^
zPJrt|mc-IsH<--y0hllC%kZ4V0exYEC7(4ky7%#$s=3mS2zWraq&{9wtwL=>j$XkR
z2+O?>{s<7mPrQ2{iVlaoXfMx4JD2Pj=vcLKHmhlai%|*4x7k9KZ@A71POOdQ=oXd)
zI8|~StsQrJ)mlJoDQ+k6)@gDddx004ar{cH%B=N@h-0Vd*wU039V5-91~baa<bK`8
z6~DnR98TWdo$^IwU)EFa)Y{)*$!~`zC^0n~8&4HqY-eKPO(g%>{MRSYa}G3M(TTy(
z2e9AF(D{RW7!klHqNwA@n{?*NO4<De3QH=-iD}wvi|{54YF9p44vhsy(weWI(PZ+O
zz)C9dqo1devC=T&-88|0KNGifD*|R;Bq+nf|48q@2Yn%^&@aSv5wbvj-|!+2{{G9J
zFgXsAgBTRB8F9d(2PX{pf03nsFN;nhppY@-qj?a}=`q{v6PLT+wBV2m5Kw`2vH9WL
zbq_0;u$eUS`+>GrmLm|szqeZT(HiJ&KHiO<QYeb3^^&dc5TU5=O8N{!>kWRvgY3DP
z_F`d5T&7zk{BwM{3&E~B*vA}D{u3ps?Bs8lY~&wI=28O-zy^fNhfCCl@|FraaAi^_
z002f3W@Ex^R0}rQ>cc$mn3R!kBTR#O{ngR(Nl2@qP8#SZnGkn`#MhdfqwfcyAE&8-
zq`gmXbpDm!t!QGik|`u0f@SIQ80$GnzpKBn<D54O*5wIvW+4#x%#A5ltCRx|Fk}Z(
zy$2m-*d`RWIBxLg$%qKwhosB)evf&-G)IEOAiH_B)NIc_$Fu-(!_cde@7;TZU9um5
z|83FAO8k{30?MqXqNw)QD6x<Wa!>fbt=yg+)Uf!QZo>Bl^u52vvXBedbN$Q$bDbc6
z`0}j1=JODbT>2%PPglX8i&tLMcfLV5tUfr1TGcy}VgdOl%1k90O6lu4IVqse5G7d6
zJOzg-*V6*2UXIz{j-G={(t?Jb(?Z<xO;5<%<7aX|uUC262|N^6n<~sgleJEYY*!UU
z=XOkvLgRz}pd`+8R_W?)#H}qi3pY%?da+y5R}2&qy7G{&2oLG8Z{u?$8oBW6cZOEo
z+_Z!gob+7$2N7S&9|yVVX=R#lK>G@g-Pf48Agy_PVQ!=qGFh3g{fQK+7gu!@ULKC!
z2L0yB1NT%UN>#8Y29KdF3$MG0#-q<9$iVbL;s8_gF`(0+&a4*`NU<mOcj+O0aMWqu
zTs=JOts&EE{$T<Qj5Bm-(0jObYkksvbl{@Nbx8U3L8aHSZjMC2AtMpchZf`T1vRO8
z&aGUYVD#Q$2?PZg6DR!3>r_WyCHr)7xQAVfsNHz(<;id-5>IV@&taBp3x&<r(X)U9
zH-8yWWo9h$-@6z42*zTG2A&2Lb#+-fU)AQ2hZxXG81sS%fs<FWPHPoS{W5-l+b<Ti
z(9b!D?!l^9l~qOhQPC3*qEExpp}MiKBd4?1wQYC`&U5LhIu}THKA__I8o?+f^p1Q!
z{dDELT^A*LblE_m7U4!(6r{jx!@1bOYJhkUg{b1^D=zpZIZ&SFr4Ibnr#~HiHI3qR
zJ=D>_IgNE{`#m5&<#4m%6)G*O8vVkG6!}+Tf9;hDIA%|^=SK{v0nXA57hCy9)e&{d
zY2^XgxZ>-yY{!k6#9t$em=z2~w+OZOFMr-Mfxed(!LrxufwttW1qe`T2eVyE_7{b#
zR1G)A!fD|K!=cu_rhikJ>*Or#ZTA@$SwsFmKATtr>FQlA!;dG_!x$qV2KPom#Iu4B
zeX#6Le-v_FY>sp<4O`B$aj<sH(%FjAfVIrd=M5?lM4F^C!pr6y?>OUGcpc{tRDZ>0
z`S6nh`WI}aImHMU@mJKGQQ~xaMYe6gBkHN))QkWb?J<pQZr%(5xQlWg`-pYm{)j#C
zfuzpx5(k15JY;qM?y@hK^TM!F7<ppisQ3QDtll5ndM!IYtYI~eU-?~HM*e|Szx+`U
zId37lwLViIEGMcXgSvQ+K)Z9QrYD}CHCWft&-;bnlKtE&k1)vC-fT+Q$_lpJ0eYEJ
zNTk$EnGRzbj$a+${Te|;UzDlKwtxeY&iAhLXC$^h&}7}*X8KtQf-{~WBs9@GYK`1n
zsqz6!Djbqzuxh*8?dp}_5#GdGM=DXXlz1Z@`Y$FF^Vz*<OJ`luh=vQKte)A8JJy**
zTWpXugwb=e(p5DqBa4O~Nq+~lm(SdC5uH+ERtWbXen&&L0qZ*C^3r>~-4P2mIzmwd
z7_6v4A7r}Oj>(v8JE|c&aQz!JB0hLgfsWbJA;S2oRo-Ri%#IC?Op?LDMKCZoL=y;2
z*fuIW+dV8tWUQiyt&^?Pd<ER!)aMLvai#|)>_4=X=ecoosn0p_!(qRnw4GJm%t10B
z>k0dacdaQ*w|RzVoHKD}7U9Dv9*baVqx&eZu8jzZg`7?^Og=tsz&(0B@kvi<OLe7!
zguXKP1_k#DB(G0@i%E7Ob|?0ft6tpJ*<lj{y7UVJHNRlmyoX*O*U=>I)SNF)9(MkP
z-^gFRcu?9~C}zrOkC<XpC+e=tEqKVM8j`YnalY#r3f3>H9CRmS^0o9=Ye|V?#E<vP
zn0v0{9hiquEVS~9PL>Q6g(F97+Vr@l7knUT{Y`suoECOV)ltZ_A0nt!wd(s?Pi}zv
z5}u;MNo$78%3zaP0Qesg??o!qVlyl-!MP}FhosP`7Pc+q^OFQ`_{_JRkhXuNcNG$7
z)qoQ-S(=HT6O)PkXk@zovXR^|2`<nmZg~Lat{nR6)7@i(T35qFqI?*>7zX{K9)BN@
z5+>cAcKh>SP8|xb;e`S0LP|>|{g!H4KGTpNA~?T~orlna$Z#7BtZ1COHV4_L9DYQ|
z*cBQeu?SPPAoY8G3Cj?rUb~m)%(YD*)BC5B9vqID`Nea0Dwscc&oa71oCC!oD@ev|
zoIc&h9R9?d5(XN;&-Q%JTX<@;jbMX~4-sQXOJiKYnmu#&d?FeBdEv%ol*IRkK+8YG
zA;SVBa7okoa#yPx4ol8}om4<!zyMi(Dz|p}-n_)=6D>3J{yEtQ!_jJk4g%^#1zs#+
zrIigBfDn+qvU+{ZuIUvWP_4)Eq4KE;g@nkh^i^Ja-Lo=eaHJlUt>gJh>R205wgGMg
zbh}AkaM<I$M22bqYj}G?j{D?HVj>>Wp(fB?08TYHrTuceRvJ`eeNV_RRm!xQbnhQN
z9F}-J!|bUY$_i;UrF`^m%07l{fG?~p#YPdL&;z*AM_5-0-BS86<+3h2gluHlr#(0N
z&U@VA+$VeDIC86hYHXRERM&PWB$AAbCe^r$n&h|SplL4JicRrapz83@4(eMuF-h5K
zbK)c-Wkq5rMf00kn&1tNB~(EOAUJr%86JSslE<V`21f;mVFd(&l9e>0R<kkLAlSv<
z<$?`4pJIKKHWb|k3A?2lDHzP(mHud>|LwdT4)}lnm<I>y1d?#CB-P&25hrOPejpeH
z7eIYVf`RjYt7=3ZZ$dYdTLiek&K-e_8a*MilI{ur*t8nS0%w;sGV+reW{ZL)${|^Q
zrn^uLzar%uD$4V-3G@K80DCSmC?^;<x_7{`bkBSc)o+)|wqk3G%c5fC+a5uKxG%R6
znE;gkoyNz<<afZ2cIas9`wa~HYXbKB6c>U8bSO9W;EeV^4Tq!(Ujm0Y)6)LBSg_I@
zoZDOkEKn5?95nHH$O37&$?v(FTo9X;CN@gm%D(2obQ&wL%Jpr=B?`lazex#blAwi)
zdg{>Z{9Kvgan9=D(qZvyPHcg_<GG?CS2Q@V6#jkqFdK$zD_%nR%*NSzQ5^T$w&Qud
zngkCYPzCl&h}B-d&oZ4Ma>#dQzXSBAnELbyM1=XCo(wkC9>51Q5K|XK_(8Yp+`fOO
z{9^kd@{{Q$k>&msg39uamNR9qx<^-(Yk#7a$#7VvMXG@|jMwV%5GIxfNgS>au_~T3
z<xOomN?K|owU^ZVaZewPYh`mB(LjirZVGw|)}uzO12?F}ipDSlk<!K|NgD0gt`Flj
z9Qd)kPR$pVApBVcDsoXu0Bq|l#?}iC8nB)e@=F|b8=Pc$f-_mU$=c!Ayx#mbVyzTi
z^M<7bn~lIvhY?Jm<jSr**P@GURTkDvwcal^2||X3j9HXcF;K83o!GvM4mHX_7j9!S
zbjG}}_j;k<fnNmE-^PH3V*M72R?VRZW^VvuM{{2DLjt>@1E~imSm%9@cJhi-Tr=54
z^6o=vUWjHpqbd`%q0m4HTUBI)0%I~7L9guknz<p790etZRAkZ0<aODOrw9S3Mo4Mv
zR-ZT5#g~QX@#s}`(rFcI3k<^vK4^o(mjjb6kOp25f!XKVVZ4A0qCbs<L7y~}-H8&!
zUTd^!bbZ&bIcG+vrYi$ZhRPQ3mj*GI^9)?(X9fod!65X59Us@u`-f8!6GSrLnZdBA
zWL~Cr&8$Z2+a)t;fpu_^QGYmie8WypKP9p&Et*jlQ>+Iy^8zC^Qq!CG#uZAKIC4B{
zgAZ|d!5}2=C<ZO?WEjjUe8+D-W$T_*g*+a*fIw4h`y&xJ3;H@c2~y<<@bG=nn{L$K
zi$B(~62aofN?)J@3<TfaBgpi0J%yDoebfa5&RgU-7*oIUohNR!Z*w)*3Gnf5)_5F?
zb4is^evKDBhBPoWuW!&?icU`E9_B6c5VY@+Mlhu<L9wmOrJSELz!O66StLo%xRh=_
zDtOYo?6ljpr6nUtf&Qb-|9CJjVQuci^op<Wm(f6x02vH9`frb#3L1mrfCAij@#DRi
zsDU_Z4V{xxP0jvVTj!PBlnvTw!|XUrO%8_h7@JpcEx;efFKD;5RM5uPhjzrN>L-sO
zlrY0yo3oIzI}hTSS6mS_c7)+|r)ALMK{q2^-_Q>TYPPu@&hSE=VlRyJ!uufg-{DJ<
z-iHxKAMQ`u7VyzGIYJ0z!2nYhpCPs%Hh6q<*?YKh2c+uC&~ONg5q7g$4BY-(AAvoH
zSt1O07LcJ5dz6d2?=lv(;np1V%DCVJ|9sj;4N+fm^~^RQYIjFvdE!-*o;KCk;ZTkI
zc@Ts8-Lo=z8@@m6ZU^DFU+}({>ts_A_AG;NAgV)}rr`Tpa_J4UHxk`T^DV!*N4(J(
z%aDK5OVi;-(XtT&5N_nN17$m%Tsi|x&oba)5Cey=C-{@pW+fnz&@<-yXP@|jhwn-j
zofLRh<qpZNE_nk=(N=v8fV_(U-kc%BP0dj>d+8^2+vPkrthW4I06+5>a=*^KEvg+f
z_%vOH42wa$dvR3`&s&Vu+5FNWr08TW{>aq6lJD+vjJ5s_$a+4#5>WAx9mJGeFLtHe
z=^R{d%=C>(fEwPb1hHH0<%tX$$pYoR4l%l5)z44DrCvFx*m;d~+Oqss*-XFkax^&x
zWN+Z3k@`!+;TzZVi4rUb;{6$v;9!SDigquGKNFeK9&9M?NxkKHzW=8kssWC~@aT_#
zzKZ}2m0jv%m+~(4b{(!1aE3c?2z(2vuQDV*Iy_c+*|{Plt&H|lNX=(SpITEf=Gb*@
z+O=r!Tx-FqhoJ3v0(~Wr@SPE!FAO`sHe{*-h)_EVlTsVpd(gRRzWExyV|YjMvlSJ)
zVsHRaptHWVej&1%f)5Gd&d^z#gWM6e7Gl))Q$AK%z!@aTkn9qhb;4>s-k#1(rWHLq
zpM98*`iSHCJe=S`K#Iw>XZ|t&bs4_G?zvRcUyUl`O@JgI1j%Q_VC?^-hZ9eZPq(1*
z$FihW>i#<B?0Rwu`6@GH2^!^KT)eaTwn`4-ESVY;1mzmv&3E*ovA(7^ih#yK{a$&q
zWc1G32e*9D@Z6H#8Q?gQ!6H{#xfQ5``?oj2f3U|d9nm{_mjG1h50GhaSbufxYk&gc
zM(ukRkr{Gp)K1da?E`6WS#?eiK|@eCFjbi_;6KYwNKNkRdMCx!!|anUcq!HY__jy1
zPIvZnWGsn?{#E>96+nt#uhCy(mmKx2gm>wnh;I{W0aeFW&-(aqU|F{qNXAQjql<B{
z%<$wcZH+Zu!IT-wZq8~1<QTgZ$*m%yXPhiX6DRR8j=rlr6*M49N{Ji!?|>8A<?~wV
zVGA7^6hM>2lr3uae&<r}81R;hR~%xKh_k<Hxd{?TZuf>c3JWSmZSr~QvpjyXj!Mrs
zq$%bu`KIlDZDNYWFHHPbJWYBNPtUO7K*MrP{}d+sxn6~43I!;CWYNr7*>+jHn+;a(
zqWRZD=etSJ%<bLmLdd5Fq?{Sz4P5PS#;KJ11h3CuJ_2n_cTgTSKUsWc*Q0k^(@ZfG
z009G;qqA2+`h(4b9(F4JaAb#tTO%=mneuR_lj?A%J=+JdtNylRpQ?F6`dty^-$8`1
zpnqR}(DAfqw?uxHsSnSfVf6n!Ye?#HIYi8gO~fe^R(O<+ue>RvE?)|ocD>$@_hgA*
zlHy%&VMze|!#Ts#f<WgUbnuANhME+LyaUa!Fbar@1inJ`)b&{4KGMm5bJuZvj(4oq
z>_W_zpH4kuMKVlQ%x9O&$YlJRl0WAEGxy>TDbWR@>Ooqxe=A|s5U&9OaNXzp4Zi=2
zq&n*%Yx2~_1h@uL-}MCbQ_7Ga><}YT%Rwcq4;*uXJ9_Zf*^uEw(YCDTib|pAMYtGW
zeOJhSBeQqJWoQ$X%h}&^bQy1}U%HAs(_LIO-a`!x!_AMwbh%z#x6Jyw?y`O0BvCGz
zo&(#pQ8+}nQi)0b-u8iGAL7YL<Szi{#fAd-xp+WouK>6$_+O51`PyFj1WpU+3RFe~
zugalC$Gpc?&25ibgurtui?gcq{u02$vc!QBRa779_BLf&NT7*U`MLsSH>4G-n@|F=
z1dp$O%u;*Epux8N-p4kCE)sG_R6<?H&HTEz6oMNHNSiKI=5-mTcQGG%7{8U*yeh5I
z;`4Qi<N-B#QB6UoUp@fdkAsO*O0?Rc!699AAW%K%dhz*z8}qh>@^#x%CZMe%KEdDw
zLaF#|kXWv+xEzXRQ=rXj2{}0%+c3oNHynPr9M|=`bc$DekH1SH_xazeqcxZS%GK|o
zj%vlz8Rn}%WkD{mR4)UA@>eS6_;fwnnW$2A;xZ6{LFa^*_6lM_@+&oy4*LoC_l9eC
zIb>u99a^61tlWNTT(i9|-znA|mA-j!x@G6CyLp&v=ifhVzR=DZ(T|#@UL&kjUE~#9
zA!2+6NnA+7vqO8HI#=M@6H&c!JBhsfLmfI^S3)N0!H9z(!gvAw&q1p32@2BVU?I_k
zLRYmoo>~I;5TCK^G=vSTu!OFrcP9>z^cVr6zQe8}jQ8dRj46V_Mr}AR4UwRiF#<8s
zgQ1gn1c?E+SX=7iyPJp|tEltK0o1=Wk?G%S!6j&HTPha(<JNlT>jNKua8O6MXeCRB
zdPt*AjHG5PpZ@c5IH^T#s?;E8M4fQk_^=)sfF3<QsVJ)KFKNNhy8Z?_#b?xKA?kW7
zrPVG{qkurqdglBJt+-FtEX}JyH~DVtBDyKf#bMPp{}<s>Q|jza>LKc3hvO-xrOPrD
zS8!1x;P1q*Q?DzIE_5H$we<t9BEizK!C+m977{!(5nyN>SJ&XOZRqGbm0Li*-a7dj
zEhbwnoahM$*h_~7)i@CVv0%ae<EovNnKQ5iov#<AOq=)8aJ*VB>Amb(OR2FrLIH+V
zuNVI3f!8SVnXcf}XH@Ec_@19`7;T9({_&H4Ls9l};gH12M~fHbt|uX;tHUqLrS9tM
zR=17Z+y0K&e77~<AZx+S$v94YLBkD}*`-c((EJ%zBb=1f-q8A*P8WU(p(EAC_rY)~
znM{$tt%t$dCvY*fFC`}){5rR}ZEWL}MyD8FI<fNYZ$dy+7BAJm#AFvQ*>`lv^*Eji
z4f?@sQ{fDs<NKuX-xm!v7oeH5?QYHOt!nb|pFGhp=&^r0JGg&Y<p3`U;*YRh!G*^<
z@2_il7OFdzHC{F-oe1tYyS(=z;he5GUBru_rcYceOVZF86z@ph20-i;|D`Z33xiV$
z>)UUuUfgpN9qa}yjN$C={6>rLRSWXF`=lSDpZ8djknn@H^w8$zGu>a?22%oelN*e0
zoT@#DP6rJ@tXIICkOah$081}w&~kCx(^vyULu=i#iyrrV-iqoWg0y!{jyt?;u3Hq3
zT(%ULZ<+%?_`eh%h}-W1LudkISwwCx#C}g#rTFqf-4<rs-=`gS!j}fQw*B)aAl_`d
zk^3(xxhd=oAO84ThKcb1-EN>9PHgcvWp-!(&n^DP7X27vfc@q=Av$by7ib4buAMLF
z5WzF}Zthr+NC?0|3i;c35@RQShDXZ+z6-?<io-<#+~3yKnJC`;A2t<Ox~j7Bzm;PF
zxifmXn0iJqao5-fl9GTB&}HHP0oW=T6i8C$&m^GhibYKKl-+3hW@z|QJ0fqT+5kg8
zmCzOG?B$D?Ke#~74D4_$D&)KdnFwnY>KlMWg#~Na570LzrxF2OhgTsd`p5x(L=$K<
zm<IB631Vw->*?MKuvPv^Sb0FA4%ff~JrMMG!9~!Rfp4(@PSVB3iyARP7V>sIduS^c
zQrtgX)Sf08<L&K&Wk7E?&)W4!feYFx@kN!CMotJ--z)}orc3B;1BC!F<a_~4U`%oz
zD(E_ua<X^aI&4)fY}7}m4{vW*1qXV&R%rKE5mL~`l1a<p?a#2F#{)@5>VMTm*kF)(
z{#Sc%71c-6><bU@$K8SicY+5exCfU63-0dj8WMsAceg;0;KAM90t5-}?s|qid%t_{
z@2v0cT%5Je%bE*dX1c4ZyQ-_I>sKSp`Z^Ye0=yscDmDs>2+H+Ed!@0$-_db_qcfNV
zjxx<bC;^pLL<K*h2uPmjI~Zh<XZ8eOfb2n#H2o?H-p^AqIaLm%1i8zTF!cF(bR9I{
z==KR$9<U?8@RWE7W&$laNS^d3OMUXc?7;xp1Jm9or}7EBkMm7VP{tnQE~NJ>@86`y
zU*PCMbzqwOa-JvGoU`MU3IdXcsEo=8`S;}id-DG;k$f@(i2vJE)laWTsdo_heY)9h
z321zMS$8r+(4*%e83U;rqrJ8KmAgUtyyE^ge9OV&|G1rGG0O-E5zm4$@z>o`r~3Dn
z`2T`=pm6~fM&i10Ste=VS&o>|wSCT*6kf%8b{cX;{=fc{|I_ZDFX%JUvyLGfpvN#k
z$@$^!+-CD{PMoa_#aZUqI@8D)Z9E;Cd_vdAvv0Xa)zjs2@u&=mefhIX6jy4cUguEK
z<FpF#SEM%WU;D*Oo`n1wEq7;DP1g6%SX>X#jikW1)w0w7W_zx*db%6UOD7*&6F)Hd
zW#8pZ2U8yshB=LfHL&z;tBv+OMGA@Z*xWOWlti{nnqJybo=IJN{jIJ`yQp__ybTkb
zqnP&U=m|U#rr@;WAFfIm6U<7N4#s&-3jh7rwA6@(=n_qd_})0w;nZKTH5`_$ikJ>J
zVMPNt9~T5&qHwdq15{{|RsWuNJ}o6J`_3~MExe0ue$yK7`2BWZ8up9vwT?_Yl+s72
zE)Q~8e;yI)Ig8@cN3pZk>yUQLThKF!9eO#g<5vh^w6D%fL!eed?OGGc1+^H24sR9E
z_Lo_<^jvGSy)RH5H$573HZ~&HsmH$Gr=Q+<F`T0~ZkUCd%muZH>fhm|u5UF9^?fch
zPDDP_&6N*P960<=*7qZjXoEoPEwagXb{*AgUkhH}N+&lJA256W-861vMe%DO>>;?F
zC>y{)=@lQOe!uK@4L{v{VGUv+h4<^2<)HMh+&&mP)=8c|rk>UC7U~QbMe<4_CAw=3
zaO{!uI;Pu^4xC?(Fr{uZ4!>Lv##T(QI`U?nykQHvMlh}cjsrZS#uR<-Zs61d6a+7v
z>6Ke^e?Ps0;XU;0ZHx@9Tsso-O&lgIeR0jcbT3RVLZC`-quZ*&Rj!BYk+(50zY4Y*
z0xb-A`;&_YCYo9MBt7jLOxAUV;A0ht92|X>JR&!Jl?g5EZD*hX2KXLfECpT(9o^{S
zod?G%BrqW1s1YuxZhYY6S-!d0@_a&-AW105-Riw!b;#hXR@T(K+huIndHgzI{ws~A
zeAObEIyxS@&1wz4uX>4zB5-|&cQl^C_NUmR5_`Z0?&fm&D5Xcs#omEMlW!)M%Fqpu
zd#FbPRr$8#9v3=4G)Ee)PlZqAr{sESzth-_l1M?31>we3Lj!p&dhv4#my@G$$fui%
zXD4;m?uw7MXKuXQin_cH`%V`(tq<q9hxloDMp{QXC1q}hm-mmuTcm?qM`h8=eshI|
zuxBsmsKyAp)?kjej#>w#26f|)>U`)6)8l1ZyEJ(v9%e~fcYYi$wY>LTYL-<HJV5DB
zzEiO|j629v<<C4?hJiOeyDNdrvvc@L*OQE**t{f4FK?_$8X05{dU6$Ye7aq8s}b9f
zuzdA+`qFN86G!s^#(t&Ecdp*Ccz^z<=HvB++50DTy38P}jV9UhQ$}XOvDT-1N=FnJ
zF!NbJ?a1qCSlu{r;yeJ6LSoT<G&kmK9~A{hNVifWm6WKYY}jRdBWyhAv=#fuoG;S7
z*MHUmC~T+&zwAf1(0VU!#o=muqphHPblH%ooxNN#IV#xe($Z}7wCsnt)O|F?aglLU
zD--F&5_dLL(=hGjz1lx_pej(;dh4EEM(S*<{@XiL+tsJNzFI5U<4FJ+YfGWx_I`0=
z_o`$b$^<6->N;@n3~`C6siYE%$cy{ChnSP7KxQ`IMG5kY_^Odsx*6)2hh$8k<!O`Q
zyM*{dM)ZNWPWe{JDcm1vqMj)SNLa5VJ~ZIFU_;U(d3ks(Str5i8Iibm<STt=kW5kL
z_h4co$SPxbkxo~*$-@zA6lrqFJ`RGnMM^g16H08EC_GN_`u5kT_b<ShzW0w!`JMMS
zx*~6!c3kRYE=WcPL)5jg|M=IgO>f=xD|)&1PZEbpUWPV{K#}be_FX@K$%nCO0RH69
zFvd)F5vk~%^VNBdVxwIWV)tp=+<fUd!rS|(C13g5_ik^{4CbxZz3wK4&O#JiY6J)g
z5o1}+2i@)jejL`!wJ5Y5F=tSY)|D`0i0(rS3U-RrskTBAe>Ji<H|xuhCbQS)-si%F
zjW|n=oUQMeh3?XeZLv9*)*un3ND@_Br@F>Hu6&18F8p)*Vh!)@AwyZjd=v}(<r5{0
zl@N`-4J|)uHMae3|0`O3lyk|t($WXMeJ`VzK|!bA?<^Md?=%MC%P{1&QaQE{?53P|
z)jY~XEN*r}(@pDMP~9;K<|&8ozk6w?KuWFZ$%QM|J&y=;E!nwg+1|TkSL<f;AeuFN
zZN0c)s=kgrIp~(~jWKHWV1-wy+evV__GjDSTr$}@Zu!JJlG2GTRSbU&zM_$}9~Kuc
zl@mp+Zp{z4PR+k_Dc>EifNSKyd!jKy<lN~ZbFAa*nU?bm?gCnLlZ?F+4NAv}7k7%j
z)BUft$k@<toQmwH>XHzDy}04yF6N7k^bq#pgB(G-pXI@EmcP5t>abmMMic)ggG%E=
zx5bgj8;=KzXkA+g*)*6v+{F_d{5F%x3I+n&&*;t4=(p(F1(t0jjF1lBv^+AtUSQp7
zFoO;1VX$kEmEqob(SVBsBra?8snZP?Q6LPUCKY&J{COkOeiCw@oi;<>CPvq52kAI$
z#@&_PQ0S!G=-=;AlyNu@Q9(F(--6qEu)tup&Zk(j<`JLe;5KB&5yo|1V!k)7XDDfU
z=EVRTd8yQ+?dImddszR}%}A1gJMfy0M?c8#WSZNOS|UnD0{RLHsC)5^)ZY;8;awCg
zsVpO`<`8rV%+7nr!}Wu!k0Kpic2O6*-D16<;px}AZ+laB?ooRQO8Yj-TjYPziDnPS
zt_u#^EWWO#AyaIvX^y-L%Py6<^Zk|W2cv{xs#;*LWG{R@<lCd!QWxAXL3`9hfA{Xm
zv*R<Zg#=m#c354tiliBEoO-u888vlRt$eHygc@g~bWh<fi%%c!%5MSdKih*IS8SQ(
zGO)vI-1W0L<~C)%B>6r!&k038!+QL~(bL10kILO9k!R!D(6r+XYTb^F+*&jLy}i*x
z5FwfM!#a!&cVYr?JX2a3LrTa2GrM$><r(8T5IXom67Hfh2l&%F|NWj=s{{qr^Ks#5
zTIljZR&OQ(F4*7fJ?{yMNd86n=p>8)2jEfG@Rl25Zw(E@;eP$cCHXGHO4u41z?Tc8
z_NBrXHiFY14&3=E(s~&QpVwV4QMRhEH?6HpJgaxqo3$^Kud{SAZ6+huLYHGPiUb7^
zQ{s0zV8fW3*JRr)R>4FJf4EKlQY_*S!UdBvz@KZ!ut-*_cl)*F#eB!-^u4{gAl1<!
z`a(K53=Gz>eU7{IA}j7PU!7YuAEi-@wjGRg3XU`Vs1ZQ5Pk-OU@Av9s(@$(}Q)F|V
zr`reaMP-Uo8opt;(c!Nj!r~);(h@HAxNXc_iQllw!pQS1NetDK8Frk=to?TSBiWx{
zz3wWS5wsVN)8vK^F=F-n_#>j{?lZwlDrbs10qv0%IXkn>2RD<1NMiHSH;E1MtzO$?
z%nh5SoMK<~(7W0|BkK`|1smP@tIoiSuXciW(#WE)hhEy#CTIeUCVZpS1*hy3KpPb-
zdtmRwk)LIoW2&ben*8V*cZI{WiU8%M_*$7vy;UX}(s<#=aR>24vfG827XYi0rN%bD
z%~yvz%8mRH|Kf32nYEX4nAyW2RTb>%KJDEy795hJBYaU2*1B&k<_dY-8e7mcYgC8%
zyhc|{-eE%CY>T+aB0%KIgWI{{l#iv{3(gSARVSM5KB)v|HPv<u8V3@Lr}*d|;}}Qe
zpt6@=zzrYcERECFq@Nd<WrjjgRG(lt?mO3DjrhJ3c{4^r8Sbp4&}ME&u0b~al3xtz
z1xrcY<QXkx7#TtZhB4;w<`R;NiF>T$PnI7?UtE#HsJs%eZL}JtCTlh5bZe5;Jm+p;
zYa$$GC*B@L_hY1yl#Qz}XpGhIV&8s9(&F{=!tmEp(D=HEVj&3us(X#*-(B1v=zE`=
zAJnR>_Y<~27oYJ4UW0I(4AAPR&S=hPQFkJ(w2iXeXT`VFxUO)*gTNplm};R2r6}+D
zomv}5#~{L{%%A{6?DImtxihDRWGJbMMoUmk)<*+Zh}99g&;T6f0coR%RlXhF?WZYq
z)SMfGtn5AA6K2l(xmRxSW#0;kgWM~l;rZU8ETyY1bM;DD)+GEG_LN+l0Ox1r+w>hQ
z{keO7A8%)B+^i_zyKN+R&-sJ&<=CN-a5&o9qs&FVO6V2^>TIHn6|4<owU`JpFt#Fa
z*TpoCi8Q+4aoDJgidbzlXev<7>4Hz>l|R#cX2C=V*TZ@vvZh&J;ih?WYAoOs6j{@K
z77A9vZ??GVbMefIf-(;xuexY^=^++7)x5emPlX_=6uQHZV~a>$D)w1y^8EeJ@VJ}}
zO9^5rm?#At)M9p~^w)C8bnbPH!*{|})uT7t2kM+_3#8xmk7gFev|2c-zc8Ea;U!2~
z$eUeSF`Y?W&GhQaT3SWEK6`9Liisq~C}7<{ccQ+pombF&fw>ujO3{*q%~V!>GXj-J
zP?{}ZE^vZMj0rcJW!qa_Ou?X+Le<NF{awLn#skgJ-qsTqK*+qX8_b8lK^L6V7?wj3
z=ZXnlzFT04jP2d6{xbOcm1DLMC72te99&c`1%jU!SgX4$Iu#WIQQ6?I%=DtA5aYNl
zoaDq40<fTL@Nh^4HvNb$UJ5^!D-vhJN?ZGt`eq_XAz~{aJktCT2)X&BK2Suly|(e?
ze1)`wS&sh5GbijZ!f1w|6*Z}fljKol1zAhsXi|m2Nna8X7B)^&sUXWoED)|{LH0=M
zmi-L;?1}Ib)bKHgqRuABW(p=CJr#a6mlb>aRJz3FcYG(lfEWUq=b2)H3*8nPa^c~Y
zhSFD={rJOp0%P$qLZh+wg1^AJQjq-{!fR;YcEa!RQMC?n+YG~DTa0rLZG%dsS!6PQ
zMNCoB7M&ae#=fZp;}Z%V0&*?Z3EBpLZZ?-S`I|i5<S<%+??ck-TwMdnhY&}}9&3g;
z(sa{um*yypuRU(1-^+vg>EkTjUnK3ecOfY^@(pR^FIQ=pD1dOpWzJg+s(S_fRb*;L
zQzll*5QmpI&dqp!o4iIOB}5|oAjnx!3kry^X%6J&yfQQUK0X;BIcW1L_o8d|B95SX
z3cn~Pfb_UvL9Qc>6fAf0*~N0y^!CS6xzFkk+E5EDxws~X|MXd5|EB_6=d}sD9Xw@K
z=ROP=)g(9rnQu=o^^$W;cFal))fv9u7tR+n_xE`yU;TjfN3yw<WiwGQq*c}3?<_?O
zrj6~XDKEwo7o$A?J#bJ)8b@FZ#{?vy5pVRFSHGYH$f5$#<EK*?tW&pKED6<H_}@y_
zxDytBntfBLWPxAE5c)drkCmI0{FjbAOW|Rub2c7YQU6ce<cu5p0S7LWcfZ390yX{J
zu}#r1E@!O%yg{i=ebFo~KX_<A%x}E;5LIWbyEPZVUF~ziOalL>ZWFa|cMb|c+0=CG
zH%t|hK9p09Bhy`<*fjcdEU&E~h-lE)pSo`uyx3sn4#G&rr3aVn5pQHQm~+iDG#>r&
z`;DMZE0lJSeR+~ikiB`Kn{GONCm}JRdH#F9oeQYGa0-MlY?N<oZ^oZ4%5&tXv-0)o
zwj~zJQFI@_Kky_keVl`Qoh(F<fzV+`H_fwvI6od!>M@#^t&sc8u^G|EqngJ^f%>i?
zw0WJ_ieEHv(eqREZ3F<__mxgqzZSGf^x@zXuY!wY0*Po$z#cuRw^o^*Lh=MU|HV93
zlAK8L<-1<;3<<UY1tNYdg*J^LC-JJVpNvWLlEle#h*N_pA?(!>a5yQhqP&fQni@}+
zX#*ldwijiNt!fn=4TS7fl`5XF{%>^%x4VhSv-G`9QfWngX$$y$%YY4i-xH$%9<0A*
z0J0nj7aamt;ME(Kjp@%dB-(SxSa$-7VfSk=M@eg&S<v@hvAnu-<FVm`uXm;HWdOq8
zPfFRf@(31e&a)tev3(9GhO>U3j`-DllEUNMcFTX>!`8H=lL|TlN!L@Z)4&ps#fKpu
zu{M?o;CNXjocj2y*MWGyGG-|J3-8iKzvzIjv&$4PPt<9qCIR)jGB7&=|EFi7&tQFB
z1OOa#3vb*3Lhw{WB*JU%?It%6)iIi6&yX9nQP`J5hkdjh$iqj?a(+^sw&m;rKa0Eu
zHRpw3v}z1TIbs5%;=UArjx_$prC;~ExkJYjj{`5MJ0M}mndq|Nl54l+@nCCPhd4d~
zHbvZwaOV20O(Pm=jm>OY^3}3O)VVjesb9V6dZSRmj=SyY$3^?p(9^wg)`h@={$$pT
zsN$y%>Wr^iJ~Vvjdq*Hh$UcVcAn+2qMQW_09fG+WB=ff>jnD+Ne{w(cBx33#VmK2-
z3gP3nY?ev+l9TpOFDUrY$`TkHxh{4BvR2Z~rq&TZkYT&S^_$um4Fxrb;Pq|YrTv%6
zL#(#s@|l2NH1#xA?4N>mDCBStjOYJoJebTnx^2$|keLDc3PGCcN#b0Dg0svDDGSNU
z5D*E-4*0~u&DSpXMl>ccls-UVELHs(!xI>E%!36wNqvcQrsUoXkSL}IWTg+ioD&;W
za;JyShJ^e;WxwuRx*<1B*j)U~jk4YMLb3l9=a;BtK05vEJoz{ra7&+p7H-`|r<d<5
zi76J#B@lsu6sqUuhfr>ZCDK@iN_P22#;=07DN|%24wer_>j$*Jx%aoCjGWGuDXWc+
z=VT^5rS(bM-Gu?u$1!Kz7Bd<HPmEeSR)QOqxyMfa_Gd(zKHpEf8bdw?Fz4f8g&!qC
zf^ne}EP%$B@KY1x0&(QmcRlyTn<NJxZeqnIg9lF`r`Fn?$PHwu2+wINwB(oXErm2l
z+<zLp<7FlDKYI}j70b%0sge9)QTd0XT%o<WrYWj5Epl7~S`wuQj8lJ2U0?d{&xN?2
zq2jqN-s{Dct03pWAVq1eC}|OQ#g%t(s+>Fpsd3=aSdWQO97+qo4t~jF)|)z{kGv(u
zu)D8ONvg$YUd5{4(bmskA`bYga>IotnXn=vA(;;zGG54$!Bm4@*%_m5Cb(fHO*B_B
zsvy^junXJ<={1GLmHC)gRjgq9eput1nu?5IZSdcbD2>yFYNJCjM71`e53JCqhSplC
zv&GSNP2O*JUFf%yNT!_If~`b7laGTp*?Ac9nRmve;b+~bjMbrGX<w{pBg3jHq{$vG
zbr5s4A$^Q{O)AmkV>S>*CoKPEb9@w9b#~r|me+3QZjoPB5=TT5@V}wBmrO&f>Plj|
z#FmT91Aq;KVdz2&`Q}8{UB=djMSkR4lB7XCY;XFzSso5-LF4I2>2=914{-IfT0sUO
zR0wJK45Dn|$;B057}8w@888Jbe2dKS{qAn0b}mov=IJ+^WXs}~z6eH0kBrcZn}vYt
zgo7^66#rdlq$d23AWv*2h`mJO8SRNpiPG7or|NS0enu3$0AZm*AgxdhQG3aCTtOW2
zkq=CWaBYg*SU2BgF~`oD=cYF^p<c=3^w~Bx_Tz?bTf5-v!;bj*E0n+PgMfQH31eqK
z$)o<!CzkhTW|OKJx-{H<IFl=baZWcWu<SWYnAhZO>)ksq_3<EFKL+EN&b(wdExq)e
zP*6X_#;{0<)~7tw&IQlm?O|8pCe^UgvmdqgQ&MK8XVkx6;1Q$yXisDp_!_9vmc}RJ
zlgF=E9wKZHMlz++1rf=>smO<u%l)mI<5g`X1Y#vH7)>#`se2`Fr3Xfm^^%`ABTc=b
zN6};Ax}78EH%A6!LCHh2g=p$w=~vYg9XH;f5emw4jnBH)K_<{U$LzEe@oM)Pt!90g
zAlIcavfY)Dw6N&Q>V4oGym@_$kXm=Q+cSh!-_$-08i2-c(?df{aYyk^r)Piwxx$XG
z1k<iAZ!-ualt3s|a*R9<$VoviHGEyUrb~h|Rl1HW3@4h#!w=t9C}xYuD0S(Zp@Ez;
z16L>+@>tfvDEkJbK7Ci%s$+vZu<%XZ3n1JSU3C}#H+SQficQq=-n9JuCf<mo{cXq&
ztQ^TopGohji<)B-*cC<7VVU8C_kPKUwYM2CR0>OYbo%|ap{&NO4G|8_JXzj8NmQK3
zecnhZygkoBOLB=O&oUZx9Ym+g1>UCCxbq6Hwt&$pXL_$jM`5klf_}UK42FDlK)7h~
zYT2E7vlON}42+%2FQXMeSm}>t&9yQ4uwY!w=c2<Ezbw1p4+-y~sQQ~&j(t9|w+)f(
z_f1@(`zV5dWmwQDM58uPCQ2bH&HkTLYf3YUaCx$Mk`cDeQE&U!=hnZjR-89DmH#6L
zV=n9;4MNSdL}jS9-!Pd7SpJ?qM)L^YO}bjnui00;&5&}}>=W7_eL*$*hGM#{uGaJ8
zK+WLRZ!WD5`@Q>EY50|bka=mI&`xw3b0{+QFtQKsp|7FBpirzUUhVFwe%#X(A7UjB
zRFCc#uNKX~FSQD_$p0>Rt$?EYM;0>=@F%ePjooj-RqXanNRu4EKj5Hv>EdJ2-0)sI
z6I>|Nsepd!Q-?aVAT|h5DHVu4au?`)A>01Z6>E)`>Rb^&%`5*L37|ngpRPb^mkT2P
z@v`a=f3ki29xRLX&y3oz?001xa>%{0CaB6($VWy}`xv~)SMU`B$#Gw*b!Suq_a$hv
z-XE`$?TTbdWH$UbONH|9q#@$Y_6a>$h>`xZRAFzeXY;ZomyCBv;D;#C<|K05F(ew`
zMH<^fsp~3WsfGyjjKdqS+`W1uv=+Di%6~Kxq^UsHh&N?{5H%Q@Q^8(PVk{J6wan$D
z<U*Kaq}X01g1dbN6I90vsD!4{yT8-K6qk)s79#iK)E>bA+T_5(5f6nEnJ9d9%GksL
zCi8t+)dPzmqB61UQeJ1~l6tZ3P>4?;1|Y|a*_GA&gYzT0s#eR}%@mDHaHFmq2z&t5
zvVQ|<b*=OX!t8C=(_<TdzF$P>`4RAFXp(Xjo+82;w3G$wKlCcl+7JKEqas36+T1v(
z+D^Aru9SvBm4T{U+N5WeS{e$#j9}dhg|K5Lec@46LjZabF$Ag2&8(%Aw;s}$*>e$O
zxS)VX`)%5ow1+RTU(Ome?PwE@KI&XmdQU~0fv93tlB8sx>U&P&Mf3y=Y5OwL<INY%
zK2%R2*n+kO(3iUM4<g=gl*2(^6NkRDK<UfA=|%C^3sVPJVbs@QQK2|sU0<{0tK*P;
zupBWDCcM)<v3`k)N65j`%_kO+0;Lnp4$$FMmyNY}T4ZH2iB`j!=LB&<y;e?A`WW8h
z0GcKGQYa^<i>xmGJ3VZq_k^4<L5>}lLJ2UqX)bI*Ccz)l?67jo^=rFJs_Jsb$Sff_
zOurD7sqN8Bf22g{V$_`(n=bj?J?<<@IwSCOkaxdkI2*wBwk)2*6K4ycK_sMpWANUg
zl;Vpq3yyhakj+9V(~^Cx??u@0$E5F|hceFm0aT4Xep{DdSjFBiSGiYI`{14XFZWmn
z^H5FvI7)@Beb5jH?;rzX87Nv9o7btMf-^_#I7IHf+3$pTBuXRxct^T>bn%TsoZAGl
z+UyHTF@XJwr2;*-C2mj)e$WhZF{}U;UjLL8xbyYX0K&255BKjHk=QlO&(`E=wBp&*
zY8xJjaRtmZ)*Q7wxOX2i1%|mC$_T8w_aN4%SGOE4p+>qf_f2Jpj(Y!Wb^dl7Zsu*{
z?WrO);~H%l$hBd*L9}KYU|E1R5#)KBBz5g?X-pp?^-6~s##l#HmJUq+Ft9)YFwo*Q
zA}HqOtf_Rv*Y5{<w76Gy;bE%^y0J4=UDw&YhhL|eT-~8H9=Bh}?5OT5Wzaq4*`i)l
zv!Qv0kjOJ2DKXOOuZM<2>SKR~Eb#G+8_>Xc7|uX-V5p2igDI_J2ISTB9bYNJe+|oc
zL3)ZP#qC9PxZkmLrhhTofi4yTx9%+Eqj$Gz#8>!V{0$<2E&@qWcP+-<z^e+4=>nB(
z7r*A^dl3!!;0Xrsyi6D90f9p2@EcukLLke~=2vo}h3H#ir7u3ShQJ>B@K2mj_VAnf
zjzzCFVD|SB5cHuXBE?euG9>)YSG`#=44bKf`X=?m?&wg{lQKd68FDZ&PnyxR;eG82
z#6z#!(-V`uY0y1$GwLuu6Q+RQHD$W>d(&$f3%BVzb+KgqWCC;qwTC!3d@W;sD>D<-
zgAERwQ%kM$=peHFpzMpl8<?eHz4toF(#o6Ka_pV9cq3mll;@K4vr&bMBukgV&1CGh
zEKa$f1^yYt36|7K`qOn6xn7s`(ihLZgA4i$JgGH@#SFF!-r{T(SPz0igD50n7jVMp
z!%^tQSVt$an@D0Br&(re1Slde?UX8Lc%v8O>5jr<hnZgK6y#FF^1=X*s7E|{cVq5|
zBuN5dPf>L8l^p#98-_UxSx^3LFq0VNkXEMPeGYo|uq-kN0I~5W!1}5#t)@DXCCHW_
zLdAEDOHpddNS$!zOXFP1K9IpXnq&~b)REjHLo>xQ)P`yB>U*#teyhXz=Mm%3HLf-3
zEndwV0WcW70}JG~v*8*UmM(nhT*1$I%EGS2HiavAz)K~*W9yERglCa*HBmFh$|=^b
zFev=2vpYD^-+9iAl2SoP&+&TuH>L^X3zi2WP!FV_Ceazpc~F_JI|ymRd~l!|(w^EE
z5V146MCVA<Fh_EAk-k>W776-{12>3&t=|^9UOhV+4(<uzsyMYLY4t3@q$u7fg;}t`
zZ{(n<GRqBhXDsO=N|gP};v>^(kznJl-FVlIu-L}dMXWyoA|EcFC?M4t=N}6n>KjN@
zbBffb$oWi$_yS0)_oY9{Oga8G6wv3K%@VPEx<=KqZxiJBw59Fc_lN<F0(T>=AM$a`
zRVgqNsO~G$qaq5jVIGnlY7BYs9kDjm4?kP64tIm9V#lZZv*eD(v9Iuloc&1XFn%18
zvF)6dFD7c_dpn14*C=+Es-LND=D+OXhbsXy9u>_x>}465HjsR3<U`C?7FnfMNZ^Uf
z;F6Oo+3^N8o8TgNIgULz`{6KGzP;g-(8IwdYK$93*q6gIp)i7tx65=9LeCzc1d<rg
zW^HeFu^_epJ%j%0J5w6<2{TbiaJ9fU0sq*0(G85qr1khWsIyV}Qm`T){JEeITLai+
zcvwdQmCl_#9li^0Ui4<TbhRNPi{}ts6B~R-E%b3V#Sy7BgxwHb>V4X(cb{o!xR5zY
zdjDE9Qr%;+5gl2+nakF{We6?*OD%jAl}0X1KSOvKriT6Re-uVq6l_7o3jix^<IkCY
zurQ%d3GQsGfH%4p&}K)7*7rpBha~8Od0jcZbEd7{iLH0Z?QMhDo4q3bB!O<mS1MrV
zV1)|vD$G2nFw~?M6nSj7tc-;4P-w~{;S{2HJh6UR>ur{&N^Na!1tb3SF@jhw9CqU9
zi==!H%B)N7YRLx%_W}0v8jHV5$sw?G6kdNc?@f)Bov)jk#u_pnzq`C|*OUA8;eInK
zS|rmH3-UhLOR@!b0W};cJmB^tMwF&0WCV;6Y++)S%Fi7Wfd1$?wExXGKxWv8rjnvS
zRZ6F>_`SmdYxQXj-hj5MD7JajYQgFcYLmFr)I4-Wx42PcFcd!Aq5Z&amKstdHHg<v
zB5o_difBk@9SdaI6~Ntv%=?Sgw^c7BT8>4vV_?CtN(`vYR;DVyC>;tl`~K^241nKS
znXCAOl`(5Y415^-(doacMa7X2JuzF9jA~8!C-kwxvTyV~j1G?;Sa8k%j%*bP&N@}C
zwQMlTM>?hQM`5cvo2%t01`A0GwJv=EOtBBMT7dYPiIi8nGylFi+qQj*-XAUd_HbNL
zf}D)~V5qglZ;>)nd-xV)2W$BIYXZTOEQG3WM9hWVYrO_%KU*RkI4PBH8z&Z-{Enz%
zixg{gZ`O)NDIparvM(5Fc^bK&I3pHaT(^xdZhp<e-a%9r`ki?f=O{rvj}%3%!hY<h
zM3qaZ$su@-QabTvXS*7Uj}Jk}{0&)G3Pck8qC+?mExhEX6zI%Moq<G{WqHvL0h(-#
zG9Nt$<Uo7(n86M_j~)oRg68^_G|Yww&FQ;_Q7zxGQr}sP>lvbczqBF*Fulg&_-$5n
z!NIB#NM;vDi9eJWm@Kw<Nt0TnDDD}k8Hw4%*4OdoQvkv(6hdfi0Y0ADJIvrqX!3G4
z>=z8=DAb1Cod$xRAgR_U8LXe6zwqDKsbmK!j+5uJNpqzgmvY9fBC#LRm`DPE`cSE8
zv}lJ9#t-}fbTI5|g})ERDOYq~P8sSa8FXhYQPB}SzV9elJS6*?*j%p<S^<5_;U<P5
zP!Y697^iH4D8Jxx+2fTEy76}Vtr&yOUdQg^`oy4{{(5JGhkN^L@bgP=w!1K{3ox}~
zhJnP0+9<cWlLzUM#ussbQ@s0it6lZRcJJ{LkD2hwyGS++9<pq^?jx8m<?~z`gZqtl
z*C|GtH~%bLVKp^revh?{hr>a6PwKaFss-wMfj!6*tZ>-w9?fu{PU0w+oU%j~3leaI
z<R$Fdz8;zn$#<Y1X>e+UNOEomD<DT)R`!w<=AUQTyn7fxHoy>a<iOfqnXJ+((`)hV
zwQpTqwwV?s=X-P~?pG2@>IvzAi$R!B#dA-zXX2xu`aWKC9Y0AFM1Z^`)&yhr4T|2B
z-1^`%@90hP+|_cjo@Yz0(W6K_LAKv`bnlz!k6G1L3wWoqcC7=NmNY<$$DJF7A_3TL
zNZZk6=^1lal2`Z77e`AYOqqjrcbB`GtaJ<$UZWZwq2mr+uaGZ8L1|mU&tmp8h}JqX
zc8_C4o#CBcImtM!xnQJ~tCI`NeNf8(Bz$Ka%z?5(TKZcW_4-W_ZcF9_bZsW!-^6-N
z`c1BdwJK~m<n36v=o*4eGgzAmGnfD!CF>Y%HC?m=<<b=NwH2n}R(Ow5G>Cs^a5^su
z2`n4z2277+&v7CKO~01++IC^`RQC{O8JJ#a&~%iooY=$(P|0)KxI1?%CjhW}a7C+|
z(fs=Xn~AZHCet}zODHaamOX0@jn2cG@p;Ui1-i2f-ZTVErx>^SL61GJL02~Ls_?|S
zZqdk9N{~GVp@UEW`^~JnP0JM@K$)sOv2lzHV+p!AN(%Eb#JOYvPw&PM3%;!c3yy-3
z4fiWHr=Vn>@?c?HjouyvcOH0aHd*+5IsP?z^8@mtxdC<roQ)5c(?=mbdgl%Zh!2&H
zkK7C&)_CUO&yzKdCw$!wtg1ZU{H-ghHv8)AZ^1@Js=uf>9s{sf?WB$1OX8sSC~y^y
z+=35S5HXaZQI!sYVAv1BL2v)OoeyU{fDK}`LZU`CCDA{{k2)sJqv*?skH8h^jS|!9
zWp%1t8&q_`;V&FR4S+*Je>3E)b<^#o%~1aCLBIP<gX`vGGk)oe>{zTA#sG38RrWiv
zJD;q-h=Q(p<BN+RBuU^+twj|P-5cw6?io&k#i(VX47Vt(;B4=}A50lnKnEiZZ~bB1
zq@#H^UMcoO>ha6&vkv1uqW40+t!@Z?Nj7aag+;8h6_UqqgqltI`)>YX!d|PWHhp}~
z2*nEii&e~}4Fp!yTZ{$adVP|zbn<3CiMWfG2hYtA&Kkik`cXA2TQ{Aq34*EKrL=#c
zjAVOz@os-J+ZieUGK=*&)z0`3e<-d3gLLT3^3qqNT+(g}VPQ)XX8v1zVebwBinyeR
zxe^W7NY{TPrdBQ^CPY4y;r+Mf3+|`qva>_PVDVj1J=^y*lF}7tp#*3-8tqFjxkhOE
zf`~b3o!mC2!HjUtfADw0|J$7XCF2o)f4uO*mYHF*31g%0QyQi=xn6sYN}jeny+)Ju
zrd<ya*k>fdKfqF{JWCJk=Oq3<!haw`qR&Wm`(6wS^P>^FcQ}kSXrCM{#f|GGNBL-(
zx29HP+Mko|1HoL{=YoizQjAFcCb47vt)L~9|8F$O{~wx!-v9z|qaBiikO3$xX)X{6
z@%4L;hVEYgff85?0nXj;`Od#CX$R;uDCrZV&9&z={eNBl^8V2N@<9vFT>sY<z4{Cd
zaECoR0q5qwt~MHBsJ4Q3oBwL`xod}K5Jpqrjr)JS4>qKaodxJmu5kUgt^mY5o@c~^
zZo4=C|7fBDQ3<H9uuuJOUHL;)(w^Z3<@$mo|D%atAZ5Qe>cLV6|DskHnqI*lS~yId
zl{D&qG+_-=W=;6}xklN)t^gcb5H-=wkQDW|xYxg0;Q}e+Lbd$|pz<H=5e@?f5xt5#
z)cY4Z{jXM10azdvuI}xB=M7mmIEa6<_uuUOM_m5BdjHW&|5gsq>*4>LayZBhYr9(Z
zV_|8z&PnswOjgMeN9<2vT2Q{Y_tyAN1r?AR3+m3?s->H$FyZz<Sd)YKniPppG~ztj
zMCJhd;`b#9)(f@r%&qs11h0*y$vj?zTXsNhR9k_3YMxe&rQ=CAp6tdze7^h1suXE(
zbVj{=D%a^rFRAS1{+!9}`A&A6;o<Lw(~&nv?n`$$UzorCblLMHSyKSZ0fHU7)19Qn
zQ67V24q6nZ)A&7HB>HkWmx7<OM_K1Rs$WCXY;eraP`*AJ6@uxHO$L=Mkq<=eyB2i_
zGTsb2N#57hZOY_F+Lo0-WaGQb1IOw2>ix8t-d`V4pzEOr^2T$eyf&Th4(gDX&}p#x
z=s?9yK;66dHA6U8GNSt9V_vMn*SE1WZ;Ms(p*?7jfM=v1mHiYp&pfO@3_Mz2B=H+a
z&<U8czxk;J{WBSmrI*5Kr8eaxrVI-fYs#rOmW2axeSH9j?pHpD{x_!{0<-No|E!EK
zXkjI)WOPKLMM$J6ls<@PoM`fJnrroIW&BVpcldL!jPmw16=?chw1L$AxCi-#12nDJ
z>-~I&3F0wl$U$2??<ZW886XZXnkyhz6WAWl4Xw7CRS@%Bi{Q>PAJ4IXK|V%70-!pK
z)ezr;o0_N^GIIRTGmy{epmLuong{$<39xU8NQ1J+`g0YI1vaSc*u*@D$f(z%z2tQy
z;FDRH>8lmG7rs>xORJc$*yy&mjXGVT9;JQSji=Xle{)8;e1IAZd{-@uYz7mcdrAG`
zFexSb;7Ps7ye{`+zy}7I>V8vf3??R0NQ8AAK*FthBZ(PEea>UNuNK|DGN?1B<5Bo&
zyZ;GkF<B7-SI!jK@yrK1iw{S`soaCbg&D74k#M4=NIb-S!C7+Vg`&*lb=vGp)bpiB
zaE_)`B=I<IhtD_e|LXYLp|=}$4_pU{Cu_aPf>(!&B@5)@a$cLsX3nXE46!bIGdr#<
zW+Q1Tm4*^N2dmv*U`^(Kn0!fd8sSwbQX&VZI4sS+12JO`g_!SM>ir6tC`Y+QrD$fT
zIk(5zX2egpk#TdU=85(oc$ejXyWGW4()Od)r?OmnaCC}wlb=f06Vup%m6P3)a!V(w
zb(fv|M46q5{EZB+!_UdX6Zx_xogYzed2aE_l_srbN-^)HL>r@SuVe{uffa85A~)Ft
z#+dS!TN!o{jBv2Jl=u~I{Hyn0okj&_k7iCMWLb88a?l2lFEQn%zi}Hl{?lo)))R3{
z+$BiVdDj=01(`w*`3%AODqLkcjP@-v;<21?qJGf*a?Whe%C`AprpU^4G=sQ*blpnO
zX}u5ArED2*dYt&3FHD9<vBFN4CC&xg^shS;Cne4Tctzd~oKf#cp+|S0-`p@HRG?wz
z3C(Fa^=S?+j$W0<QBDFX?55PR!tXdclq4+8YM%3~7;XSQo)z=V(<#fSHxfQd;8J7M
zd#vo6NCofog=V9xjZZojAKJI$e0Zm=k%QzYl?0-9T>E8vm13y20_AL@27Zw*xkuR1
z_nU4uQ+>{OPr9(tFOQamHZOSWA8)qy_}}4V1*sM)2owds80DXC6Eqc+5&oo|8%#fX
z7AK6uYqmG@J!9*>wV54x&i3~1+INr*J~B-@E;~FXjo&!jnCCUF%s>En9Ge7r6#ZH|
zHa$IoifBhbTwI(qk-d1#Yoer)uE!@qaE+C@9C;mC)n`84{hqB_9{pBgrRP=N{BiW>
zQO%4SYU}%Dlw92Or(Y25WnFjqhN=78S#th){P2{QXeR2dZu^zJu8+B~LRzj)Fz0`2
zvCL~ZT`3dWh@e@Tjzx>DT|w>HzBQ8mW<i5=-p4Q@i~mc^&8b3oLEN2o)6?S}6RN>O
zKb)B+)bUoPUo9Kf`_)g%IhQ{MnXT5yD&=$ZGk{N+Ph9)5iT2Wy)QQ>?hrcRwIokMp
zGZmn*l4t@dPE~dRzNJcU@FZ{WvVJYns&iM~L$c5+1Lzy!ek>Xs5qi4klGNjN?qK=3
zopZVD_e2a!z+Ol#2uG$M^l*N;`h~?dif^~bF>~bDuJwTnUo!n8k9MV3DBph5*@#t?
z(Bm~0oJHOti6cUWsseBRU00v8$5xV2B>mNy$50Cz?D%3MIDw6B1Vy}e`&DlWoRhBR
z%rdcPythyp^8vDE%@+n<ttCgrR}Gn&Zp%JytW0}UfZ>Z~Sr*a5-%g`87NnRmH7h}*
zpx)t2!m~X^^TP`za+Jn<;SF}ZmQc&A*@dFi5J`|ZUg2Hw0Kowk^xz8gg8^J`wO0nH
zIoCO{nYgb<?HUhiPs7J=ythEb$#T`pY;JziS_*u!IK~o09n|xCcpn&P@u#e^|GF2_
z#?pE}VY>DaKxDV0MqxHgX)xjVXpdiWvlb=9#eWqFp{lsT{*+I@@s##6M-QD<UwITG
z6475HOk+1sBXXigB^)}G?{h#%X)KBNar<d$%cp#Drrg!*T-)3Is?puvgHCiI?T#!l
zp><0rVIH?d7j@bDooy;JJ`DUy4jNKaUvLs0y8W!7NzU0_&q7Ik&5bnWU%m+Da?fUN
zJ{x%>TYpUGR0CuYb=;d`Dk0F%=MAA(fi*WC?+UVJ-U3}eDiYd%v>@aOn7p7o3sllx
zHw<z;A(hSkX%$Ryz{0#x&Ld}|tR%QB3611$6KpqihS+u6mGChzY$p(znJ&W!)u`Cp
z-E~IOEZ&Tz`CP?A66R;$cM1@v^LU5Fh1#JMqL@NJO{rr9ouxbPyzFy@O2ei=&FnDz
zGM~_UE)^@fZ63S+@pf<UiN~cQS`5dy*IF821!81HFBj}uFTrKJgo7gMXZYZQr?AiT
z>pbmcc%&J>(FF|*$+baOJ;`++U<K8Na35Xrri+m~Wy@6x7(z}&vp?16@E*zs^mi2Z
zsei4bmR<Zv`BhcXub94kpbK41?=X#;5WPVscrZ($jto4~tWMn}%$+#X10<f?*-aZ)
zgjh;IsX=y99sy{J2>!Hk*6>O?41g!0i@=Rn)cL&Zb=15urzrFgS03(LP66OU4M=`<
zX*7!T0j^(S1H8A2ohnh$(|@`*SFmO&qxe;XPPIS@0|HoIC_VNXogL{%u8N<mnn?pw
z&<1ki7_3!!nzRoOSIaL~k=9^LBssb6Y~9tK-1olM$!K-(DL)i7mch3D(uLxi&<xe}
zz1c|ZD6+fPLj+d!36u*c6|Bm0nP%yMO_xu50V&xdhf5SdpL5xf+1w1pEFLgaI~B=A
z(DLOHXX?nkc?*#axa4;-(hdM3Ah?LR+Pp3Vw3n_|8efv)-HEoRLp(R))m^VTQbtrN
zODTY&74{ahOdnT{dk<)T<t&sSm+xoVYB>h!1{vI$Z-t?~$OG=5GV1Q_V}SaH%LQ4V
zQ&0&+s_1uG&%Al?kuV_m&YxPB9k1Z?5pzbDtGgx~xW3&hm&M<AOWJyT2`0gW@rak2
zm?j0_m=a%4?M=B!0aJHVWeuAt@^hL}P;DfU=z`Z-CZq5+TCe*zOiPgbB{u32!?<TY
z`a|p6gmna?U>410{D8nG&QNE7R;Xbz)E7P;F5+=%a<l}85(Ix7%3#G@-px%!f7Tc^
zLg5Yr_PJlQ+-6KzNJcxyEueU2Ksp#ris3sLH_R#U5dDdcvzQHf97>r*j8{Sx_2^iW
zVn(k?kSP;~SIng87%2e_u2_eV<h60{GkrG*P({eS<i1AJi9+D^(EJ8^1r%4t8v;uw
zP_}}1MiX8SMEVYAwdlSF_@D02oCe23_hf~kDq2j81_FfzOok*`x#~?!x@rDsQ#gYb
zjaw7`%@%L`(`#E?D2&T$+22Nc#-E)s0R1YR0>8N<%19o=<kD>^Xslnd2;QJ+oKMmw
zI+FB(3fM*4{BiO2*Yz?6aSv945BI|bQz_;On7i#~@!1R%yN#`)g>OJxoN0<7O*6ih
z|D-(k;?xiCxQMYD>bgbnLw*HR*o+MqIy!<24e&r!=^WQr-CtF_X(mSfkUzau?_t}T
zYAZ%Nm-i$<>UB2^EIlE+XhM{a4QTNfXV%(=9w_lI66h<AqvKD0SfQ(90yX);zLZ+D
zUKzkOD9N&f#rFN9Iq11^e2CKee30FrVH<Qt;AbX__VQlZ)*Be;i0X81yB{QB_IaL#
zvyW)J3J3M-544;{T}hzgo@y<H)*HdT1Uf6D*d!BHZT1$e6Eisyq#NJ7&;DbAW!V>i
zXDiJ~hL5d?&3GUlb#+|=YV64>`Oprw&3VxEHSd>6#Ky24&@{9m*eU#)vleN@5xVSL
zz^0QJfAVZ1zk38}el|N0h0KUPQY6y?JOHi2`hG|IK8ae8GpMJYgrMPx?1hUJKbGza
z8^-``VHz_YHN5NZ&3oQ9Q={ybegA0!0Vv0lmw1@mZQp&s1un=VjNq*v>L9?N-fN-o
zysJe%1YkZ%TVY^-P-!dEnE|-X=YCj5^5023(ar2EEuFhhE!5F<nSQU+VmZjKC~)yk
zoxn-5O(SSNv@#5E2dxWPlc={%Y>9(kdrE<H@6&#2BB`rhI!`_5NgAaOIRsTsdJ7|@
zKcIsz;`jYCoG0FXfbK!y$!!T1ORpLF+%AU6L<v|T+iHNx=$5kVX&jB=lq~&0nyZhz
z0D)vJ{q<rx0brWJ-om}?%^t+EeDQYVAvy8<-Irjt_OHG-3RUHU8IYl5gk;G}hj3ol
zHvHtBC3Qfq&YCv1>Vs`Ven}gTIn{UWvAN#q+M6n)Vfv84pYI`n!^Z9VS5w&$Z9^)?
zCU0l}jf^<oicJ0)R{M_CvGa=`9+R1%7`8x8tF?Ba;ZSckc`L0G;LwlRsep=b*Ix#q
z@we-s6-+S13PW)=W67TN)deevQH(tnlqf|vY4T;UvKPRtj+;*9e1b25hXQtASIxF}
z?5$j)1dSLwTFYK?Jo(+jQS+5<9Shmnm;Md%L{ni*EHM{>J03^l^ts>Jp7--s<xJY4
z*Ru&5(xb1T$tJr^mnwZ=U3@V_Jo;WboTGbm9+i~0XpQ_PZ+&R(=HoB3a^Q+9Z}Iah
zr-g!T^Gx_s{0(7BYRWACVO)w7m$W7XIb$~TBd(%@2uzpP#nzY@`oSx6H0I(XCjpKx
z!Z4{8ZDFu4I4ZV9ay#N&1=qaffaro-aRWko@ph2;la74bFYTI?P5F3!=+HtT%0&UR
z6`MV>yI*8knyXDDqa&ZN75T?sM=Q_Lia`Ufl4eTxgol(>NR?SCaZW)B4$xY1s`$7j
zg>@1s+qyutXy%$OI?>_jH>9&UwE62#NDV-SVrretX#z=!bZ;50k{I;s!=s(ubbQO#
zxc`2|8t+uXT|;;`m1CnBnWd@1excGxD$$IuJ#P*4e;E_jz8(y+v<_LNYP(jS^ajMe
zaXZ}Cpv2#1ogK>bz1@*NatgX5K=+;tY7fvH+Mlu!zSWcj)&s1YU|>*W@4*c~=T1Ts
zombAqlxQCw1g`M=zVw>d9x9A}D`d2a64SZdIjydz(~*0yN|IT*At7dQx^#)H1X%kK
zBn84kiZp&0uBK88UK9ttJH2p<%p!Gsw^Gw8F-w%X<Q5+O?1tt3VZC);kCAd6-9iMo
zSTj9$i*x@N&zlD)Sm505oY#R1?h&#}EqEiO=Qo?m-3tL4y2`vVQgmCFTYIoZl5(AT
zvm~F=ACEvjJFtFg0k&HcTR+2`RD}=144p1MUV-6+T6%o6_k$*{cZ5P9trkKUWFAp=
z^GPkTP{INuZwS<$QcZ-7A}3d^&Wa!0zzD>X0^P=g$*w2gQSzmC`6G#CUA_9K?piBa
zkl6+B!b72I!DY!3Q@uBN*2wT1TIb#cc^06N`Gb|360*(6&XgRFLi5{FU9oC)D_&i<
zm^5+E^TFaH=g}RwE?*tZ;DyUg(}*AjVd5x^cg;AGjIb*L7$lQ*c>=S`GZrG!6^OPa
zdk%Z&o=i}Pr7QtDpijj*E-la|sfL2e>X=Sk2X61b$!UJUE6q}BQ=(LS6zg?}F^XlT
zymHx!X59vybfKy$J;;oNO0@vi(<sd$UoGinCHNXrwMWyo5M>)v^E-pev`-@G<h@1i
z%tximM*SwtTLw3%vPSs*q*|wO(Eh5+dR_-LKkhj?gT`^ywZ|h`(DTWUViNdGxR<GQ
zFUeV<pTL!o*aRvmghT;McoKJg50*}o{$Fp7nqrmht|h5v2D{y92je>@?A?AlNL^hy
zj)}nAxO4lRpT(SP)<KiKJs@6w5__F{{H(CUAcmezpT`Sm2zO}sp5!^^89pAjF6G89
zcTHq|JkH6sGK9<itSVZElfd$9$B+j6`ZV9Al0n1gFYnxv^6O}-E2|^nH5O%^yfb4e
zzP@=*=Yn`)IU;6-ibmy7{^x+mO)y?zokRIse5>_yKv;u8@Y@NRF1amQ1yozZtOD}1
zQK6@%&pt3ZT85gFzgc@oB;lJ&DBohcthd9H{uYO?|BSZ(ivh*f)#a78m^f!8r10~|
zBis|4=(0oMqi&B6Xb#8^y>NDU_!COH)Rj?fqPushIB&sXIMpgEi(<)iahOzyAtlfF
z@?Zdw=Ggy;pN4#i@U+d#K~NN%qWCjA6qmS2@*movYTNpws2xltK(1VFI0iyR1_f@x
zQCRp`fENdpBn`M6#KAosi&;Qc7#b_uA3`-on+0rRpVUF#Qk1U)0Cd}1agld3si3{p
zJlH!0^6L4inGEpngD-s$8GRT~1HutKKEL|6?f0Oo@c#3T9`wSqsndKPq4Q6>6|>mF
z0RPo*CwYw=i;h*fI(nD#(Kv$WwqVdCrO~c8mH&_DXc6>>8TXIu<~+wUq;nmlg`wHO
zCJB(s|F-`hRI+-2|Le!AB#8TuexLu}77kL?)4_lUzVml1AVG4^l%7gh!%;s|_n&W5
zW&uf+$1?wV3cL{ujyuU4GgahYPk}ECA<zM)j7?wvdnfD9^y+n)D*gAj|4rgQ?ESxv
bi^rtgcrs#eZS%JP`14jmLA*><Kk$D6r07h~

diff --git a/docs/figures/arch_2.png b/docs/figures/arch_2.png
deleted file mode 100644
index 197e163a54eb0cc9fca5dbbbeac0cdace29acde3..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 51944
zcmeFZWn5HW7dAXI3<wA^q=ewmA<`03BaL)Px0E!}IfyhOBHbMd(t_lO7=$3G2!fQf
zC@Ips``{h_`+YyWU!IQ-KagSOoU`{{d&RY`wYIU^n#!ca^u#b2j8s)cK^F$Y11}ML
zgaqIhp|*Q43`Sh<BrmV6Dld=L_Hws(a<PNKRAN)p&uQx!U&O8l@#o%$5s@cR1X7kz
zVs0)M{^8(ZCPLke2`{iwrZKd-8WV#hH816Ap}~yC;a`cv=Uu+XOWe0^^D?UM6wB<A
zd3w4e>;7wUJ=gD-#M0CYheatEm-jQHoC{ZANz8p|_xD?bZEe11?A^fPZDC_%2-DNq
z?W6A_VftlDU47AfFpUAf_Z-2;+lOJ)ik;~Ab1<rVUM#Zl>ET<R-tCLA?Bg)>p-#uI
zWS32(<&8|f8<Yw!9PT|X_rHCQl9MhXvoV%%8)J+QLuB@+vf*Jy<$n)cvJhI0UW=`d
zjNzv$Oz6Z*Pe@Rg>2ox*y8kssCvRiu=F#K)`K~?nl>K?H$97Up>8i~B#0@<wlXj;|
zo=5JzD(RYU1u*9%6z3SC21#}Xm`dyp`i)X(sifY^os=B2m8*oXmS?6fkc+(!epcS)
z;u@%p6X7K4kPwm`&xvuOxg8Lx)xajKPIE<d?q#0+v)dfcM6T+VX_0%}KDn#MQdBOX
zOiL~D-q>mOL|Bwbs6&CCqu&y}<(VJ*<LX3^(rEtfmHBt0_T>tDeY=CXy_RU3p}m2)
zM+<K*`Uk!rQ|McJJ^riSXl@6U*@6D1TsG84lwSQ_iGP0JD{7ps%<EXLjLsjykHX5R
zqll`o(zQrl6fx$x)k+T|K2zqL9fh^mP0_~>Ze?P7$u|d<1gd$fTwoa$<CRx7`CRI0
znBQd5nRA%PSCKj9x?~Qsry)L=!LWVBbh%S}u!?<{NYiFXat`-51qnDW8qQOIXHY=o
z9ZoJFzukbRi6ebKMuv<aM4hKWFsZ=lFo-lPwbfNFn4*>N7y%E?t(NQ)_SFoRBSASX
zz=O{0T=UD24ceW1@D1VwESVL_J+zcUxMLDe3I2+FSrT>6Ju@^v7mPT<rVz<RM{z^f
z5x?;Q&%K)k#E$&>=Z_J&_Xp(-3pJOSKb<4J;a<oufOu?uy$a5NyK7A|5Oq9D=nu~!
z;g7mI8|=?&cERvf^bE(IR4`*F*`My38670rix6*EX#hnt7mj50RH0zGaf>-pg--W@
z2)bHD((B@iV$`D{+DlPU);0qInzWYcY*Eq13C5A!Z}hG12xw%)XS8L=3=l3#rrk4m
zn`OgTcgclBl&%eFioN=Z373bn=$?9e!Af?<L!4j`<?wL(_3JrZ&wkI>Ex9d9lM-f}
z&n}C_z5Q;mS?2E$ERYvJ^ycD>-wfI8(A@gWxs7~6FXYvG)GxD9$|&+JLc9CdyXm?a
zyU%yi>V#Kld9x8v?)TXBX!IQS+?u;O2lGrQQ4wO3c+8?wS<+PErHi*hQ5n6X3BHsE
zZ}91Bay2VGGy4@YN&Q0R#n5f`dZO`&OIm5kV##F5Lys;ePwQ||;l(kqCnjr*t5lYn
z6-bo^CztnT_url0-+H=mWg&Zh{asfXMdg9Ms1NhCbQ2|Mr4B{LTWB6yE`Kf$>Ra6U
zJURNyw`dJYxJ`HnuGn+YTp8uIFu0`OeT9xIfO|7_D|MIKTzj>gS;Rc++yiA(<q54e
z1%`s}%TeAJq*E;vWYyOT9&%mbp5RjE<~MkzFH=3)kD(<XxxkuR7wefa{CZDF&6Xx8
z&1y8tn6%;7z(IwO>1z$e0^N#{3NOorcpkmaYS%P;;_o!9QeI2QiZ!YDO3`F~C;d~D
z!$$|n_lxiOazea!S$&MYDM+h_yf~2g&9GWbpG7Z2Z}WhW{-Q|KHM%Uftjes?EC-S3
zMn@Aq6S->L>IM`3%szEd;auav_IH<tZ++SQoOWmZ^REv~9|b?fe(L?`^eLc7ZNzSc
zV<m0%`AXul(+d7-^@pclf=aYLIjoc~yMJ;2a_VU1h;pr+Dj2u=p#3_@HpAJ-eau!t
z(CeCsi@vM(5Us<2kf!jIQ}6^*wC<X!V5P_lJL2rGnZz1?S-yiW2R$=bv-G6PClXxV
zi1rIU75gFVkZ~?!DN`)FEAOe8nA@+k7eDWInU0y1n65SZyHws@amaKSzq3Eu*AwSp
z;N&xuW%JXu<7?~dg9i^PAK>iM-{YBPxfc8koDiLG_uciSXd`H24w61Z9`YQz(d1Hh
z^Vpkv;;y$yR{4rAY>W>N3QlZRe7~$|uj!=eqZzSHvP87_Gylta)%O9PsUMR+y1i9B
z#=npJc=EH)r`k7UvFyhiKgpThd#(eTc9(6se49PeTk<_lmuHvjM!x58&aHemULKz<
zou^(gnEn09^sVH0+f-1`+fj!XLlTx%HC^j8({70b(Qhjk686Rqt9RO#wRXw=)NF*&
zSW%#f97*rv9V40vc9ARRV((=|+z{aOciwo6dX@B=xhrZ;PG0Vce15cXR5hV9S$QNs
zan1Qt)GSjS_nc>mPKikU)i`?U%LWX;1-}ZnvbBoYyAAYZ{HVKA*FBad9&DyR8QwCy
zZ?WC{{&Gz71OAwG6RQ@hm3vRmf2Jwpm+FXVjcHppLWnY7FXWrOB9>wLo>60-?G}6D
zhLrzAK+T@pk?u(n&RCnp|IvJoezHi$T2A}50It{9N_p*!mYJ(iovObzzU3YWUKRgo
z_0>PnJJ6PZnU(o_iHd3pM{7o9<_nPqJr^E6)E_=yvv$|bm>x?n^!YxMq|eD3%Gt#X
zx~wZ?Bb}GhbNX_$)QZwBnLcUJZ`^q!^w^Fet>fMA!Q%i4dkOb|3F4pB&W1bHw`(Tb
zh)q&kp13ysT0Q!V=NvrxsfxW?!!$^Eh^2;Y{kV16r+S}$-+AjWY-7K$6Vr*&{m?Px
zbo}eGpe%9N+L%>`_1mcR$fIs&MH)prMa%er*Iz>Gk1CJKHy@}nNQ#cTer=*_Hg0_3
z9RDLNi{f?iWb#RMxM}vy%+kox_2#QzeVYc#T>kiu_^#|uZDF_47y~6m*Y`{w>3VjJ
zbkx_~?6@QU`EBk|uAlFVpLfQp>)IUuq)T^8lkFa?xjqrC;G2k^NN-r*KdCtWDccbG
zOuE3gU>VmOQ54Z}&HCC!=bz5bKR*2s{ru|p$U&j}X0+SoU8%_ttI^u#N5e+m$V-oy
z0`}Fq`OTG&A3kTaWpoyQ7S_Is{Hb6~`KhtWsYIgmMKk;J((RJz-siW<M=86qP6Ibm
zb}Rcn_LYm}GUxWn3WtjRSzDIad@;M-yL31~(<I%Ax;s1d`a?Wp`VqhRi?xt_Imx!;
zHR`1+<<G8LNC!Ls0&V^FyNuzvnC4_-M?lf5XT>XN$JJ>Q7n)5amwfVGd=FeYG9PM7
zU-$fF-gVi#tKmxhj~jl$e(QKA5ig@x$fl+G0z`KGHOQWqZ^U*#t}kEJdKxBtQeF4u
z*&pwGhv^3nvqMaOwht_i^L`(G2|rp&S+e7AwD@$vgfx$Si>dY4>!d&2aL&;775=I2
zk>O|O;xBJcJr2LWQ5krLW9ZG^Y<|72Kg}5IzTN36a(ibbI)Tjg@_E_H&?ox?2a_l4
z2mSGN?3SCB{atin&d0w_I~=5JgUtp~JvdF_NJbA`3Ya??0uzgm^<{vK998B8>K0U<
zVpS2RvE>K9RO|4pEX82t{cyq-B3vSIW2xe?-gh`*VNoC_%+$31jrXbK_*1QzZf4jn
zi7CNXe5v4*q3Og*nG>ym(+~nm3s>CqQW)NwXu`Se?{C)uh>f!|QnlC6fbl?p7KVh=
z!|=f?IQWr)GyLbZ5}fM{`NClc7%bWehWytxn&3C|l>~mEd;b2$OOAq_1AkF~AO8Zx
zzpuuF72y5*HG&U(2D_mruc``u>)CqQ*}2_yboYM9sv8a7Ao5T#z6*oVutPs^Rb93X
z@cdCHeIsup4fX4`?yh{+x7}^*`21Zxpy$A({I7$Tu6EwmXn$81x4YN<rCH9dxDH-J
zZ}YRD&o1$HmS!>1&_>I<d)c8y`2_d`SY(LNXtb2qZTsuG3QGUF9sDKD;^^(|ah;#v
z&(Dv~Pngf$%Yk1|LPCOHK!{&R=qkA4>fHc0Z)^XnZg*M#KIGreQLwvf>*eI(?d0x;
zhMsF}<L=`v&B6i=^q;T4^R)AK`tL|?cmK64ut0w36MjKH0sj9y8{8@dy?b5T$=}Y!
zSi#8^$PA1jbM2bA)Y<j_pHKcf;y>;*`tO}$qW^Q}KR)??-)V5y&P(3i6%6Su^WW|I
z*RcQm@LxAd@k1;Bha~=r`RrXFXBlEC{{QTm3~?Xf>p-xLmz)%|^uceSWY8D<BKU{v
z?{Da}Aqg{oS3eAffvGCo(D#S0<(|u-d_D0t-+9MEKd=VDg1Cn*K-}clE&0P@9(Yay
z^JIq)-QibqMMVb%Cr6g}B~AVKwvg+v=RU=j-loPkuHpMfEL5QPSL2doQ14Nh6r~k;
zI0^$p{_|V?1{#LIQT-%gz46ZvFcerAE&_%CAO7c?g9Qe+DtZ%D`JZ>rUcj*q|GX21
zR965`RCqwD_dibrs~&<8V*l}-5d%YFW9b>n?)~FUEHI%wI12WUZvk>RA&l~xs5<^X
z-h_eor11V{CRQv66pZ7JO9t_O=L{}D!TbsSxnc}X0f~mGe;N~@{vS!8VH+g>oE{b%
zONarfO`SE*|4bVLo1y#X^k8sGax6?h$M%)}|4bVQQ1~Cx12g&m7od%*Rrh_OM!S9*
zlQ!S0uOBdorM*zdU)%jTp<gJ6PcP*!VDQXL;^1A&d1#4zC@jnaPEqu)Sz}ql1#RDG
zKV*{T5)eoVJ=u5Q(JM=^rK$<eleo*#<n%F0PEO8oyxvZ~Sc$Y|K3*M2Wk3z=Rl6%$
z|KGg=<L_<EeyXvQ9pBHF^y9ncHYuJa@gl7%?DRy$WlVth!i6%$cp9DD>po^`F-Bkx
z=$ZRq7b&?~kO+IU#|#dB-FJ=l_|J4$3b!5yk6DYr!eH(MN!elPgVU2k{%>u5TBBD(
z4t7$Q<q6sXO?YX(c7=7dr}10L+{_WL`xX-XAraT;^s&Oc&9@#Gu)SD0wQtD#<6}A5
z16sj*;LCA)@x49}!2*p1G-KRVu)S`xa<@?*WWzs}Yw10`Rqgoa*EiocNF?%|j^eLL
zHm$tLUwO?kITCl3;u4WW6dx{H2G3Tz81zR=&W9aso-^+VOf`A#QfVTQ<N5iCfzzuf
z()xsCcD0c_>7cc~p#3fO*YOv%EGF9g_*1!bx_`;YILglq=1QipDqjfPm<gwX>y@bw
zms#l$$K&el`bYS?j#yq_KkuJMjycm3KOzjQMU7<h2IejDnb`{#$;6{R#!x2J!AxP!
z<Nc+4@0AgDC*#ez*G_K}8GCZQ-J@H5evE7_^j8M{UTw60Pw(!q{`<(kQV~pSLFnPe
zhDXhkdv;+pZ05(?hnLI!rbE8aFMNJ#II2;Gg5BcH>Y9BNM$^lSI63&iZ*ZbnB)kfq
zrC)%72kvb!YvhPOf;Pz9dlC;arX^%g3<5!j#i|nOKfU#yrIU!)M!3-K$7r?ht@vy2
zx0t5MNJ*R8ma55YzSZ-}6T;l>D(}dgzsQ&Jn>t&{;A*sRjc#dNs%-O(=nIT%eN@7*
z(L*`d*tb{bTnS^y{o;*6StqO)`cl7At(H)o9L_tiDaKKywfU`Ij}2FYv<D>(AArTU
z>mX4AcyK12#Ygg+shDErhh@LMwJ8f7_Vag@fVK6rW4=5yFW%eyZu)&gmzF7VkuIWn
zDzHB=ce>qwQ*MxD<M1B-dEFv~*u83!_W9+}YSXRbf*TR@Tamh@H_6?L^btWivzh1e
zQfo=qPLjU7DOb---kR@~*UFb+XF8Vg-$0k%OpWueBpI?1Kfg4XTg)Kl_RfEBSM-uh
za0XBWB|2gIl+zR`|Jb*cdgb$PT_>JDT6XEB!&-(N8^p3$0<Bal!n5~#|Cp?g1lBw}
z9d@dr`k15i$CnyY<qY#&cuT-GhwF5w`QY;Hp*(XwI_q0H_@R9M66e<?W;)P`*3+FK
zTbaG7S1y)n=6?OPTPPpByuZCP>U=q5=h5xgW&skE50kdnM#}R;Z?=biGq#$NOXxdz
z6H6K0GPt%S#FL&iFn_eay|X*v-ml}o*zLMM;ojA|#2I#SG-B^y7Z6wtW-Ma+hGbl{
z-H+qRbtQufry8CUdE%b;SXfwMGI>smZhl`oh&+Fh%XzeFYZOSj^6t{$V9_i+<ogsf
z(19V?Hn3qz%;-^`ejWMY-sdrzDS8bGOO|iQr)g=1E9>JQb7;^9tRJp-HMb33R^Xh@
z()8Y2`z7Z10T=ji`@w4dvi5lCce3*|yh)p~`}36jWg6MMM3fwjYD3QnwRj(mY~s&~
z>b#+T2DqY&_vEH7i+dD&3z@YaewI+ND(O7R+vc4Ys%;M=AfdG)cRsdbv>(cIeLtE?
z2V-8Va!L&pE`BI`GRAU!f3)3YTEH}MqSeQFqJQ*HUyRsLlsLh#akDLZT3YXtQoya0
z>l;rS1Om2h$D=ZR13b)_1+LnV-EUw^^Zfp%cdyC$%k#CbP5mtQF(o?#y56<9Wq;;b
z?#>@vbDx^5cDe7-6r<T1>5%M_$+aH2*R(eI;$yXx1Znxw(j_<)Q6|}Ej!ehDnV~KN
z2enS*z4}SjmsVzPCW7lnA0u^LrS==s`rEPQQl)xJ&tO)wFIljsvBSQZV*KXX&o-SK
z?BB2T>B?{+us?cETWrl5>@_lPx#*__c$VMv7tu{Nvb3du+1$dE`BA~TPEXpJ+-wv~
z-^?baxrXd_&`LC?Ej$p5gV74zj!|5bOj@Af*6qpiQAb$4j-#gewRYG1HE)8MT+cq2
zjhsvC^ZSvJ#@l*RJ#4TP&8Na}bnuMn{X@0*jOwI?dKF4xg{_VZpQM_boQB+50n`mt
zqYEOuZ+ezngD<3bPu5*zUA*c2<0%Wwh-6FnrCmJfS6#x+KT|=YLX7#A&IC;D*4Ev-
zG14%%1s3~>s^4?IGY-!jhVp1Eh|_tEg#sStbNk)f`eWAW?c|rt%m?Y23KXnS7#tf?
z=Je?9_%JG(P|>+gsE9|Cu_^wdDcmYMwk-%o1;^#`)#-P$q%|8gwE1rqHQhZtIb1ib
zx#Dhf1(vdNv>GoB>j?VuoRCao^g^+bJ?m4dT$q($(LmzH6nr94W@w)4kJ1BrJLkOM
z^;np;ke<C>%AgDIFjsAiGa?-XT7K@0H;_u<v8Va;2Bq>C)W_H<j853Bq$%<$C;u_R
zN)RZ*JUV&pUzZs<Q-#V<nNBpjyLbG0Aogc{y7@WLP|6F+o)Mm>)rl+O)4-BVdW0n2
znhuB3I3V6hO$RLHI)0z!fR-!H3WLui5I+mY!il(YQL_jUlOOk&=*H_R3~Tbtr7VeI
zOykdgE^|}=`1~}zNpE*~n028(`14a3aB$gGT_csZ(iRmsVgBPNznB-_T_!SbVK-Ey
zr)!MrioALcRyE)_J3=k3(ROP1%pNb%^!}(X<>A$;%~Ublw~0+t7Y{Kh{aXg)u*qS+
z=Pv3k%BfBr9m8fhFRiK{kGWeJmM_b+haP`rk-F2{c-5pahVO~#AFAgAQco7l-RxrC
z0cTJK0m3JXQFZnlxV*5I23&E&aHL!$aj*YEe|mHLq8c2hCY^&nd-Ky*Zppx%3UjIs
zVl1C|tBPbw;<GVrS8GqoT-Z#abn{jINKcyQF5@ig<r>+dBd+emFgMPLWA=72g_b9c
zgk8UV7wX^59>kGO`hH0zvw2+8WX|}J$E=8c|5loA-P<ef1fCmdO`0~8t)|BDq*J4f
zyy`8|J&ShDTOJ8gVJE@Pc~T5fXIShP1eu~~=I&t=iMSX<ZNK8)skye?U2Nf-zKjoZ
z7hi1M>hk$9I^L8xg%4}<(;r>e%DryjcE`ZQ#~ben@5EfwDVX7dorkY;l1DuizGWlO
zWeavU@nR`m>_RT36L9|t*iX{6q(MDr=74W}Hni+*oT7mNt#2Cb9gckFdYsd7(ISku
zZy?L&B7eHgOOAFm2{nrE*Jh8EN2Bw}J(Uy_Hp$Pthzr0CV<WgAlz>2ZU+R_q3wRHc
zlrE+;jk78x)W$D9L||Iv8-4p02QqC9Ej!M`M3u%#2C_{hHV7sC(60Dcj~7mM1>Stc
zK?{jht{Y`Ak4bG;)5M=ac6g%eM6o=C7qdtBEEeq;g4tpImXW%g7Txo1Grn`An`@~z
z$Bn8RDdTAz)S8Q~MEr^KIH<7KDN>6Mf+9BT4DYMUVT^rWvL5N+&p!UPs%ro9Ym@Kr
zkq&HZ5LT<EGGT>;<DLnmYO(}z>Wa2}C5AnhxVtzY#Kg}(%Pu^8{C(JGV&*)&fH#(W
z32*Gy>w}Zc?{9ClfxzqBG3X9v)Z$`Cl`nlQfWp5)C?0NFqqqim9^4_%-gt$<(=^Fm
z)J4eAvOzfjA=-)pA?z^N==-F|%i5`}ZezapvF~l*Bp5^(<-M0+V{aZ*Uo5EDh4ET;
zPTqO^JxW^<rm(=FLl>Fo_UY?I3v7xsj9AQ0*aO5b-u9?`$&f?@=F$vDjk3>TcyfA!
zA-KFr!i=ukcacE9SF6(2n>V%PsLw&HmeIKWc3p={$k3q#)oii<4PCbbU0CgTlV|C|
zHF`IXCj%CA!hC=C1=Jtv8{)-_zy5tkf61yP>*8EEj4<OdfXxyUjJL9rU~Pu_X-OVS
zV}cuHJvOi7j3rqUf5cH=J{c6v`WkosZ3NDxe^!&%&$oxW^?7=6a7_z*L38<4G#Tsb
z@8s@-4~LK*Bx@HQuW#Dxyp=EPS?M0VpzobKYVh<{uatN`5REv^?B}7rMFVAcfw%uY
zj@Pcbs~DzjCg*j@??>et8LFH|xjy<!dBe}6p=<<hxAvODxlr!V?fn3soH}iPg(f1S
zYH1jWvV{7>fQ{Br!&i?vwTcq@y5qOMzn@ARa?+1=$ya-;F``rY(g(_bYT1EXP{Uil
zftx}4+t!Q}GKe@P?eM+gm~QcWRg^He9rDXxl7LBdDQIP+w4s%8o_qjcrSWQ~L`~R+
zYSYbq22qz<Uw{AJXyGq$aj+Q?sntLLlAW8%wDR4qHiJ}T^hY0qsbTU>&R9vK$=UhA
zV%>+!NZ5INb|(JenTQc_51ID)7;~w>D+=r|7csmmKAu#tV5?gPJ1chP<J8M_wr{IG
z&s`Al%X$$pl*1Syq}R=P!ZUT-zeDn{)o{|#>%|lmjCV2R{i0Nm)x<LZ;+pd=4E$1K
zjNr$I#oxADj3MK@%YO9xi)9*{YO;~h(bv{`yBf_kQWXiwm!BhWd*k0TncOI4<l$$Q
zY$FfMM)Xq>0n+;>R?;HaKY8TQyyj)U<<y}gdV{z20EmwIx6>t)Iq8m;EDhyP2l8Ks
z<Dwb4%PYN%rHbCWDZePD<J8PiDgN>tfvIHmyOg@168zw!<lV}5tg|`g$%**)ZJg(N
zVymMy6ggonnd8|B$>)O1xXoH>ec19|KWvuMM_{)NMe(lu_Mn1YT6*`0)r-nF_SHSC
zSPTz?;r;<gBHrr!>`r8qtlHgF?~lG5>Nxv2OG1WULN_f-oD$X^C2RWJWpqc1@>Tx+
zG=AL;$yOZe^5g}x3(b_x#dDQ$HJc-B6L{}a+Jm`aEf-{ucOF*>=WD!Nvmx$Nr<y1c
zuyfali{|WnGfDl3UN5w7nz`+_k8hNm`idvONHAmul=S1F1n9zp{`@k&?)S^(i{QMb
z%Ic@e(jnisrLx3KzY;cAo=k4)To-P430D>kh>aL|;k)y;ro_-9Kts&0C{-%OPzJXG
z5TJHZFR<8JZEJqno6$O|dhPz6=O2zJQ`7KUxG-8Th~$u!&kinqu?)NV14U0yZ{QV>
z8&K!0VcI59qdw>T1ps}8?9trpz5}nf?3kSBpY~RKhq!P&&4xwIl4_iu_-4iLBE|SB
zVZOvz<-|))f23+1XBzPR?Ux&dWFF`Ek0;t{kDgpv80%E}!go^!AcJPNDuWR(hYYdj
zN<Ko*M&OA~L`hlMwyAne0nZ~H9BqiDp1yQGc9lGK;pg|9YxhsXrQ!~~QjF<N7?hw4
zGJK4JaOVD8JRwjf+-+pf79#~8v#NKainqP0e=_qhbL^vg%Nt3w0c>?sIrlG8Qsl1E
zeiT=lsbTcAO5x4lCTD%aHu><n606{<`UCoF$-gQ3T~&C)(8G0X!-DK};rb`>Deq<x
z6WpTQk28+Zf_>_8jp98XKa0g(4Hj7^C3Hutj8(l)RALZcD4Z|-0I`TIZCk~y`z~oL
zJD1=n=8`DKIB_AT1Q9vPmfl@q#^J^QO_C9TLeX6{8yeh7!_0&F_SCl^i&GuYez%mK
z&n-xavDsyPvliKs<|N)RrpKCZX?U_IZhK<7AS6+3G^jXdRc|EQFL;Xzq|itbEx=H0
zXprFDz;q!m;bTLu-gy5&35#E4gkV>EoZ-VPm_3_J7(;rlE<C@dHEeu;=?kDul8r(y
zW>*1|`)`<lAwnUt!#$gTni6?mHjF~8WEtd4Y*HLL!$pb)v-RF9x=$>-I(gN2Ig=aQ
zpHSUb2NZ)kFkYO92C4iFOc<bzcD~r#KNaLJ{BGw2GDh9c-gWy%#IU-Dn9p8G5+IM3
zfGDHlTx)j1<zy@i?EYIyio8&7Cp;d~e{oVLO5l0ZCdl7#;3z3(NSoV7v&1IvG;x16
z5uf@uk#hsYe%eNBYnuNdp8f+D|4El&|L?VDLi#@}2do95f)r<egYfNuz${+=8j471
z4z)eoqL_4r8${e_E^-|YUE-MhNCd@vO3hr6MU}ktUj|Jd0Hgwh{%s;?zweOOws*>>
z1$AlnbmUTbhNUm>onDEnZ?Q1L5aM&}V-dQ+a^b!EmfS@+|7YdO>VNseZ<JWrhY-qc
z82IxQC93#{ym?e_?*TrQr{B}%_2_`1+k%p)9p9_O>K|2N!XGMj!l(KfBmkxW>a)iV
zFj1EajXp7?`N;6vb5x-L$$Mhqyfy1j64}t!OvHU7(f6h*IlVEps|$tD7~p#vz~T*3
zzmzlPRELr<Ov(CU*JH*oi;p!vGslRR<54lj;ZY_RA8diAf^QLMm2d#EpjAGjh-v8v
z?>_mdkhXNuT%*XO9XDUfEk2}Atde2aB$OohcY}6;`7}BtQb2M-3&90PP7iKhiEDYm
zyCrRDL7p8I(Gabxy_i=-Gj@5VzYH@Dfz26+-<x8I4TgqAry+T@PRhP0JFEKT)eXq~
zz8zl4qodNfufWW$&wn2Ouk};_BgdF`zGH;uS;UU=bf)f*nTp{$3TWIsnzhFq4Nz^J
z(A2SUrOWD{^Eq2RDh5amYjjUu2$IG|3TCfKg37nP$>RszFQ;2srDQRWvzY8GG%JNt
zhpW`7kY}bmV-8JiA$Ag)8jc+?oBlTbeW$sinb?I*FK!1PRz~p~`c0NFjcoCl)oYha
z+^qjj34@^qH$JgaEQ1z^x)vV8aDJNiF!p+!<<#t8SZb$&d%P=b;thMDI`wY+0w&<7
z{#gapnFukTfC!81BP7p6<AWdvrO_X(mMF#ct-z@}_KsinFKeW5lx`4F@l!@pQL~_b
zN#3D((Ee93+tq>N>Bl@`gdU}!h}$~4)bZ8QC~t$o;8Ixoud3~uPr_(b@;E*3d~H1D
zKKAy%to)t?P$*e<v_2VFB^t?Vy>not8t3niF(gp@vTsN-D~z~zH$h)!?k|~+8v?Ft
zF95GF5X|un8s+PTj?5?foGSRt`=s;+Je*3`@f?2Ho`T_SVLh}8t6M-A_O1bAz&{qy
z!TZ0n?yO?S&0idPzt}#Nje~Cr>E=+B#~{8tzoEwetKvP0K;UaF%+%0ATGVHL=i713
zoz(EX{o|wHCwY1^B%&ci?S}C=p39qm3js6bKx{{i*4kIW_S&M7IFFgCOBrl!Cic?_
z2-r~#u<}U50`Jzki_~WY24PV85U_pv${t+pN@&Eg{Mr3Q@77v^=t6)|oO15zX)0q%
z=QaGL%vo!Vq)U*NVK5qC6?aAyr@)fRIDpDs?T|5%zev(Pyq_LL@TOS2lur+n^QpC#
zAr%Rk+dqfUAZ(Y}6I7HMD^@kmqnc|PJU!(hPw_$FJ20%NrY6z+9Y7Hbz|*Zb3I<qY
zw+aZ>zjR>mVIA(@wmn2}t|uUVE;!h{ooFlQQ0E4u!vd^e*My&6cOihIv~w|InNw7u
z!(C)=b&I!au%z6L4-k53djHw8hahO}>DU(VKzqi#amQV>UP^YonV@of-=R2Sq^V@4
zz2RsyRG$Qz%K)TD1Jz^`K&|wdtpdD_E*<?2=Lra6)oYSGHG7LqqM|C_aoyzjz+j-n
z1nn^lVZjE2&nRZRJ9A1In7z_ED&N*N%kB?F+cmUI<zwW;Dr={Q=#+!MS~MUBhTy$Q
zDhY-VVZ}X@9mlWe!oMnc>JsGfOO-06Frv#E6LMyEJ78<oPYGeb`C*%h+P|h|0Y~RA
zY5H*MW9#-%Drs=rg-9x+MV*dEv}c0E9fBL$ho~fgc4uT@7Zx-En<O7tl5Z$3<-?I|
z1M;3S*JllRwdo)!!r+_7F63}DG!hKiE{~pjX%QGwo8l+5Eyvfnder|UUN8K6=&pJ@
z>Bc?wcZk0um_j2cQtU(S2#ZFA9xD?cmS<YlZa6KSh9T!tTGfG8@8Of_|Gj>O6v|?W
zAVJM2B4(GOb;n-7Ed~UAjrZl3@|~G<cf^R*O{9F4&}Ryd#qxmX>%WYHz{$W(++|JJ
zZu+3CEFN>@^Ek@z{_NkL*l%YKO9$5-)f;PbgLQOJ5K4Xy&&$b`!&{>G?W&R@MXn=t
zc_-{{3FlYQEG~_k)5$qu?TmJVhPeRWEltD~c>fi^6)?Fg#O(mAz~Dnflar?TPoA5}
zMGvB`t}5%xubtbCgtgT6{K-A7^(X=@86Soz4IRHWI(@3QQyYS_p8NZdhrlQJC!pvc
ziQpIzfDI`n&ZjPOK9w%n4~cmaj1=Opzgp1xlV$`E_uL{PDF=Vn$+-TijtmD-o$myg
zP4MR%^afy;^+1(|8$Cx?&2*FyH!m7xo@YX?>v&#~?KlC9EDv~wLclDK3}gy(v$6Gj
zta`+Bwl5%hpkX#d7kZ)H#4y(e{=RePLjm0GT~fH{V3kUyg_gEk=E=2^cTB|Uw{*Q|
z0sEYT&t>|Yx7wsBE}2c$CQt@;rp%on_RXAM<)Q=*Q2<${o+$^aVMQQsA3pf~gL^#w
zGnH>qll9v6rzCOqAsx(7)u_wjP(J@31mGp2MP0utKw0n9D~7+aaRs*L_=>I+$c8on
zqo9BU?9+oc4=-z3hVHXpHET(ldmXP#{wK8XOSFt|MpK;#C)y|nd1Q78dC4XH<&f;D
z`Zed#6k!<1xhSc*v~i$51&J9(_>VY<!y+x({h8B`Hb(M6j(3wBMN7+dwW+;Dy=M2e
zP{HS4*;Gkr$=BS>lgtuCpbAwSM}1`k6k+DQ&=xfJ&lZg?fYj7gyqF41I|9{mcPVjl
zEeMqR=*8S}%uZ|_ve@rg{JN<`wLA8!o=z57_UtAz{uf2313;#i5Q+zEvAxf_7M~^J
zL}6z(d1hLu4JPo!^*RMju(Jf59=|3fo@f^$pmH3|RTG74cRqRC+5GV;U$5tztLA#<
zij7C&BdU}a4vieLp+`o1g<m^EBuN=W6ApHNGPnj$p7~YOMh;+AmPj7q0n;NQT$}E)
zOgu6LMaEj|h{+v>)bW3rm(DlQ_rH^hx^%2Y<^;bo(9kGVn>LrdxVxka*!Pm`4>MMW
zA$WrJ_Xz*$ARho^9(2?!kk3fPeQk2C3_U&=u?1a$+1x?9Yr{n`EnV3br^eU6ibr+P
zr6kRzy-=p2eQT$8j)WE?AAO+|RNy`WyW;ko>)|U`RQS8|Uw~D#3=#_hJ62b*0=ceN
z@jA%7xxNLL-Av`NCr<MKu8Gmu;`riSvxql@|MFzdePmDeigQTcWsC9GMWb7Yd>EKP
zz#_zA5+MCnxDU<Qo(-oKGWMR+ea9|6b!|~N^k|Q-7<hoZ?izI;iOyk`sP^VK#Pk!o
zD-Hg#7@*)K+1h!Yz3KdxRa-Y${!VWRJTB+%O9Sv1C8d>EK9oB4YkzHa*P`UqWPAO-
z{Hc+82ML$B;Ona)YP_ehOFYk1Hcw6t-B*~GdmnR-Xe>MA&$>jaBzgQLK~#YrrOhOP
z7=p_{{+jBzB6he{HVvp4-E=;48`tHh)7y_1dN+;Azc_anB;<05e?dNcIy9BGSB2Fp
z*BmC3J;H#zZdY9E_FsiW4?0kNoTS12RVMWHu#<N0?STC)yOqVq=_mda-8Dm{wP9I7
z$&n``1jyc3U5Gh{&70r$3j}1G0)Uc>1n+sQek^}zxTVMeZL^g<ghbpz#x8)@?6`A5
z29>&z{_3}n4}R;n`w#rOQXN|yNV%z-L!4vmLeNQPK=R6*EVwOg?5M5#sc%Q<ah6Hb
z9Ys(GEC%ZPVQ1wdk(*pc9Bn4V^(6qU?4`f72oy+xqt@-)^_4|ojVzI#Y|Xp#@5m)|
zd28-l<|<ipPP`Us8cg0?4y!AsN?JBI;F$uIc!~2HpiKM`*yId{K6T$aDj-1;usH)C
z`fAAmD!z%3${Jt@iKn^BqkIBH#b`Ha^vx>0;P8Bw4L?RZYL_Y}$g4`=MOF;i`S0?t
z#)MIAM1aGteyU0iFdcA)Gf-l?!3C5kBN@j3du6yddUDycBhbgB-M`M%P3BuTji;Yv
zhm6@&F55voL-Nu4uct{VPrigqJa-+@aIC-GJD>2rZT*Sx_kvbtF7QAYGFSmbaOD{t
zUSNmogtggPyIimORe<Fv1CrHi;(d8r)6#ofJNH-xGuyNrNMta=+lfrlJkm(%kb@LJ
zILPM*Z^hc%6{8_#{2&QDMMpDwl?=p48i*s(X}a^-5!6gY><4I9zcxKJU9n}9IEpnQ
z-jOlR%YEQObCM;nk$XUu9QJ7ObDP&f92LA=E5B^K!9fp{(XZ9u5uf=BX<#`YY@Zem
z0tKjJ!MVwd*Bq6qr8~a7hwnwFS|t=`$I|}_Z)eqto2CDU(c(;4FHj;HhOko>V$IdZ
zebT{u+O0%T5#OmNhWyi0L;n0vc##lp#^wRHzFgU)Z3y@qBT8{lU<QpFncc5mE!a;#
zlsVuU4G2gUIC>LIEIHNlnD>a&)k7#w?{>`{i!ifmDc|nb2WivzTs3`O;`FiXrAEcm
z4HN{wQ@}o{kFN>PBL@_*#|K}Y8a|u~SYmwPB;qibBZH3$4JC}sWtjhX-QUdL>Jo84
z*m?Z2jr^uRA&0xA>z#+1OpDKh_q}eWa6Ni!&w>X@R)7Uami0N03j$&zGOqjZ_^f`4
z1+_}NuPvUe0pH%2H?jSh+8BogD`ENMF#?hErkL>c;03?WOC2y!QZ5Aqiketq@N+!S
zL;$T@8aywEX%D1GRw%^U;$yPJIdbc%oqjhNN;Z`l!_mmx6^+@8$_ECyHdX3YW}YMK
zUc~C=*WOfg#?|`#xb?)mO_W_L&unmS{(bfrLxUSHAk&=rCzR15Y!BW~29ZRrKkaJq
z1;*<i?h{e!`)$)r97I`Ox?$~b5I3JTW?yLDI@3e7HFdqqWS25^Gwk^HMS~~$v7naE
z@$G&Z=uLT2(E=<T;B`=hvufpteu=hnIh*nBSJ3M5fJr(F^kFRiAyA&Q(5g9gy18@E
zIX!v}ts^72)s>8(SlpVpe*IR$xBIs$w8IB;B+hUL0coRTIj)Xxj|6)131kHMCU`yo
zJ#EC`Kyke!TPxJ=p+Gx{0{_dcBbJVnHc?}y;}t)H3pXcH?rp2H6DO?u`NUVst#LGh
z0%fD~7rqsaMu3+D*<Xg7?Uxjg?<+Au5+wXCYHxc;wlrJIm*@rXC_KjXB_|}!_jUG4
zJ*j+8Op7+kWol#(_s9m>HB~ju$CrFIOAs{{9m#HT7x~OoK)mCaqGh6@npaIzIcp|r
z1wgCd$D;fKh({T)pg;SoPivN|o2QbqW$rGF3<*&?Ty+fajO^C*D~>t66*nO>#(+w}
z-V+Lye`$QU;`M2oPOo#c&Ci`)!mA{Q-A1esl=c&Nc%DT-YEcMc9c^oX^(8{ueAwlK
zPj$A+Jo*($R0v)~0tK&DqTX2K`hAoUhc~9>z&`e4aD-6f#oSXqDimka!NPvQed7ju
zoi^W}6_{|uhz3_aY4}+Tu!pwpIZbrOUuA26nv{T7f1+DTeAT@5)B0p^;Q6M7ef&5S
zCt>uokxH6_Lr3b*(=DO`+--I)0S$R#o30Z%B(1c|RC^`}!1COBWhuf+0bBEet-ZBp
z&T$Rc@z^X50<zL55CwtbOwA2tdY>!qS>!b8ZOX({5YuMLJvXHp=iidgH5WOdL4!)Z
zhe%FO`z3{^XHbUSGoBojUnfeuB*F6SekbTwX(a-aYn;-MI|h=*;Os!y;VYJdKtUMV
z<MbqCW%2G%o;1{L1%)SB{eCf@7=P8Y^75W4T-V%h5n~GrcC|6vy70YT25VGG65<|3
zyv{LO{o~6?=yLDxbV=y83U{pg>Y8$?MQ5;aE0J{2F7NT)_lNp57Sgg$7@r^@<y-|l
zYuXR5dVvISLnu7{B7Yxg4yZkpgPtp$Z>>HZW{7%WhVXM_Oe$i9T_*=uK=_Yq^;>rn
zMb?7a8}2^jipG`&=#V=qGam}&N8WStpIVFM15$LPn69GXGKG~`A%J|vpm|CU#0e)*
z<R~-NU5tJo&I2N%BL<K(c<;DC3WO6zK=p!4KS80jX|gd>($8&3NZ}631C7h>8`t0(
zLg)j#O}lDJAXa!GVFN@woH!Uf;bU4Y5=o+J^(KMtQ)fs};OX(7((Iw;>ClNJ-)^o`
zw7}g(^%V}Pq3p26b9VqwRd4~Aj!z2<@W^m7qK%j}K<Ya!e<<`?A#2<3d8xr9r|6YV
z=<>BuwUy{~!eg}-6cNZ1qQOw!V~!O<gtWP^(c26ik;l`Wp!C*XifLcE4XU0yn{O{4
zhL!5bL+0u62n7200u(8@lL4`oS_-vr1^sc8IGApBw{#^|;zrM4W=2ZYfD*TBN?>0q
zgS-Aq_X%8p|MD=XGI|8s%Q%ng!+xiVXVSot!7A9H+zOrID5?^uGs^8->wc{$7d;eb
z+aY#Z^3tR8z{aAHwe#KbSe0k=aEs(OoLc<G^$Kn%(T}|!pWIM<1GZcS)a7RMj<^74
zfh7VeZ|v5pp2|bLNiZN4umvL_+@pq1rA#$AXweJXlboR+y})}&qN68?6(na|%S|wt
zfo{w>AWY(LV3Nb2ilTvko+iaj2DCLW8uZ;LL(h;RKp$Q?@XOkTa*^}f2cxF$0j#F;
zXG~!h6mO}-h(YgJEjtLe^78VY8_8Fx+W!P0K*tr^RDO4ESY8K!%IVWz=%~+Pv?;KU
z=OP;*iNjDL+RIvjf;Ku1v)uVuMyio16bX6)MYtIvu~;#}BM^wUg5$o_SbhYqyW)0l
z^3*$;fU~f~0k%=(Wr7?K85MBy0Oe;v&Xr#1HsSB}Y0DRphNutJkQLk%D+KPb0Q6q<
zhw!!v!BPPwBybgI&PP#cl5_B%L}rjl?MvNVBzd1JnZEj^rY3}4n}yK-5v+}<`c^gl
z&u?w%ZxR{Rrh~U1T)eA%3&{O{RPP)DRE-`vALq-4X@XX(LLs|8@gAHq>H{ZS=4uP5
zgEqR13*DLSl&;JEfWu)z=p#&k2hF$#6pK5HPU#-joycXpKJw0&XBv>Va+3jw^Tz~n
zp*JvNxG&|`Vr4Rk^V&^!3<B(7llKGZgzAB?H|W6%Y6?OBHw-aIc(n0~Y~;N|I0C<9
zqDRDOL_9vq8Er*~0N4?9pIsk5cR3gXO0Y?J_eiX16Cjj81#XwW#+d+zpRx$}Z&)3G
zKC=a-UeM*C2YT*~p5Ch<C9J?;B~R8kQid9V<%4R!ej(Lg61<KRJR-<f1u)_$e7Ns}
zu_DFuCbd>@0O8_5qD~-|uT$G?XG;M<Qh_%mJrffj2+*L@G9#*+2bhOBq%(F6g(K*h
zE+msoFvyQiw)*);-Y;R{ct_3k6?B9ZD%fukP#gxmC2X%fKt7|-OKN-;cz%G5H*<*0
z01FpHwE!Rpa)g5&rz~NI3)dV!prfc-ND*&1Jl3bAS2(-@pxgnit9l?INpKaY0)qR1
z4_0n2LdipfIMJF}9xkrmKHXW-TUp$GtR=H~db2nRT>yu~-r{LH7b3tY>01Ii8D42r
z5KQVIK~DA$!j_L>tkvhK_jaaKfET#`o8PpLAbU|bs2J5|e*iW;+~WDY0`eFBn>KNm
zWjiGjqEo@FLzRKit!E*r5%3W5nNk_ouZ@Ml`&+$(vXjB9Pmi@tU5V5)g%m+h(JhJb
zrp&tTbI0`xb_Bxp7(nXdY=QovoDpktkIh3uH(y_Dw0TY3vY&8Gl}&^){P&M9dL4nk
zV_plj5xsxJ8l5>1T@8(i0b`QU!wmp&L=J);1VkZ=F!bG~yL72aOjz3zwLupS648h!
z`Vd%2!GEcIIb^<}wZIDO*f!J-ZJGVb20}f+W0hnN<%gXZC!Ubei@Ym7N}>h4)eHcR
zAVmZk<TTqIxwDXNnMIAi27=KCPz%Ps!RcHgT+YOSqG4%yK9v5gjW<;46e|Uap7S{e
z&!=n&`t!`2@&K?w{iNN1e%T7h)rKgl?;ukb2xWLBhuR<x2i<X|z9;!o0dh|bYs$1_
zk3U`6XQ4z1NQPhJw@?x6@LIUJI{BhOiXSZ?@L|o!N)|ZLIG`{3_v5Q>;JA>_-KY7|
zIoZ_G)B_K!AId_B^%N`4O%4DNo*_cukO@Fr6d}m+qAx<#7jdYy2zKuwxNH|0dbFv0
z=UeNwh4<Npz9%2cG@LKP^C5?=WHnNvy45%XYRJIXRO*&)nM}PSgnT&_aCF2(5bdtO
z0xHut@y=Z+0mK0%zm`glxMzuY3)o3eba4LitP~JUdad3ofT;j-32KJT69Bv&2<|Tn
z14mboEKt5KwK`OM`aWB%98|P%?;UQdw;&Kt$i+Qou-=q?Y7wF?V;Im9s8`K<8-P_S
z8F2HW^Iann#4w25`M@?=EdC7f$?O;KBY|9h66HwxX|@vCt(&%UV*e1J&%CM_07w22
zEL&%cknhi|ot?W|^Qu|boN2B(jr5??UXnm=&6g4+#s*NI_au4}0KX9Wdyq>5(FRni
z?p)9(Kn5$%jMdrJZx0E>K>v0T=%PM&Cd5StBK=M5xBFnG+9&3|D<hGRU-h88^EyHd
z$hZ#BaH#W+DhPxUYpw0u3m~OdO=cVUsC9C*|G=JQ4GuS_jERXUaPQoY;?T_DI=!_&
zKjXhSxAK)k9|RMeAK2;Ow@mC2SW8rfm^%~XjsQqul{{SwKtiQ<St9iefg5=jaoP7k
zNYMh`c4sKA2y*D58o)kkvd^XB!F`BG7_X;*mQ)Pra(?V46F>li?(HI~GH!^X7jeX_
zd@N6QlWVZ+U!0~^`U$rA2$V0d)7OZ2kcd@E!t>{g0VVhWbf;H<R8It87GfeIoi3{s
zXtSvi?c6PZ3!*}y(BJNd?#)J9^j^$_^7IJs42iFFx9(w6ao^uQ1i%L*W4*(t%<@r2
z>$9XlkwXb{q=RljJ7x~TTG{I7r$5TKD!9QXS^$ghQOOdYgS!%$b%mM-9!Pl3-M{pE
zX*dkp>SGFs+{+;XfKe6@4qfkKvLdjHQ2bO*MBKoMArmjl@=2bOU6x7taUAeX7U~{9
zAp%`?X1|DLI^dwgzZ@89Z|RMLW)i^85LPI{5SXj=?=B*bVsM~2UJme;B?_?=s#>!6
z#8ex_UkiDFNb?}P7_5XVQGGm5(FHICPNSk+8&J0-P`Ls3;QU8`Sv-^rz~O@wfDl1p
zVu%?s$Tw^P;D`c<c%aS0q}h#e@;-e-rZqL8KXjYz{dYj#>9~#ASCy7sp<DWvGT$z`
z$s9LVYYU?`Xy7NNOOGRj?FYEN-3Nf+v#BxbWgvhEV8;R~^4%nVF$w^$Y7GuUWgHq=
zAHb&0EPkXxO+{c|5*>jcs1G!on3xzORe*lE;}LJo1f(Gegf&?h*@7;>Lulnm6$0Ah
zBfzf}AU&a?r+*S*QXB_nKE>g&I6$|;vGe8uL*wmU3U0!7bAl}>=5PYT2zkzMhZLbn
zv_TdEtTa~oWFSK@B1_Eumk1LdL_<d4eoi!(W`Bq{M>f(EOIZn0{6&FFw;<;M>Yr#6
zsTLx{0rLjnhu;mJ9v?!`82FNZtH2%Pn1&-DwwwTf{R9C%K^x@c{U=HcbP-q?h`A`v
z)mB5rq~QP%tptf^ih#A;%F0s5pQ(6J88>MBcx-j{hwDD8AAld{cAIPg)Rr?yXR5%`
ztGG4@kUOeYz^?<80a9bTC0Y3>k^>QB(^)kNmmz37&x?trU<dw-id(nBf`b^C@`fyM
zK__20D#)RtUfX(Sac9WkS;ITzy@6h|=FpIXAqG^ffT?||&|&RM;eHGv9=jUN-Fw*2
zXe%4=01mg{jAxb*!~-WtMmji8lN#EFl9Euz^!KxX({a3M1%kYOwaGOQ?!UizpkJaw
zuF<b-4fdlGLdT{jT#U#`4ggI9NY??qyRdv(YyE0+5D(Z4Aj$x%2Z9!Gs!o8zA2CH$
z{L>cc@59>mtMn@=018L|t!jK<P8fPbG63+M!9z5=eM@FM1zfPISfNxy4A>&8U@+Y`
z?zTpbC<JKkN^+A~pXyM%a;vI<zjLqq%z47#mK6TGYjhwCJ{$!u2!v{uDO2DLX8L&w
zU4W{lf@xKd(2hp{^b0!H)mrxx0CL#b(W|w(;eI<_3Ys(qvMi`V0UVe%$n2b^+5<c&
z;~en*Yg!36tibt`PgTakeE<Zk9XSkS4AVxvBnBgNu?ht4I0E~szpc8$Ap%n4P(x*X
z5U+4|!SwTl6)vC)=))heD(mLUgj4_|0-acC+1Gy{A_Q$VA?Ufx0VM+UHKvqY(%XYM
z#t_v0hmF~i6OxgYgY!)v6X=91tY1X|l>yN%!H%s8VogjDfb2_klN;|4Xd?pEklN`{
zYHDC39{Iol9u6cSkoxQ!XMpT3h_PNfVc_)AleFl8v&@<<JnZz?5V%0C8-P#oTIl<G
zoB?|QNNu@xSljM9g;m((5o^%SvJSvaJRkz}wY=$ungF^uak*h9n%9H(1T^xbGQU1|
z{qe^~2_HPx2GArZVb&fPT!S!2DnK{!d0n=aMT99ho~6;J$P5$-jba2b?*R1v4o8tP
z%vZ5pmLU`?943DyjoJu<bbMN=q7W1(raH2p7}jV4$v^(#dY+o+HsYx@3veG6gd3eD
zpiDCY&Oyy*!U4)+wYE&QFd_Sm41Wjgi$(Y)HR?kO4xCW}$O<F_s$%(>LUtpK!BhYS
zK<yaXQjbH9Hs6j2(@oxQy?J31#3NwTdxk*;q8zLOcRq2gerpp;<uf-q$>2~+8@P?M
z0^Rr6DMDV;=Lrz822wj9b3w<#W*UW%8(2aFObbr9hlOdC`z?P62db?JfMYo*m)KnY
zcJ3#Z1rm!g#`rm{n48(bN_jqEKaK5M@=6TO5KK2%k!c|-4VcC}sn42en_-Q`4#2+F
z^0`j=1hf|r2kaiJpSV}5W`GWic-{cc?zc0^oeKyKh})?rh<?$@z5!0tsHO*kyr~e<
z;cQXYU{mIlGxClJReP*y`{gyDvv$#9Y#>?t5C=5lR*6pH3ei|%+9v&15I-~Xh+x1{
zm-AsmcR+>FkPm-#qM4tR@j6vQ)`9yiBP$QEEd=gX8B3RuyKCd}puos#!QZbI1`(mB
z6|@k20E*aXY>0*hHij32kq6h)cPr^EvRVBQv;~@%B5~LL%n88|QJexcJ%n?{Fn>|u
zV7*tETmVTw1mG1NoM>19Cl0w=9)l-T)#hz!Gr|*H6X%<t1T6=a?+P|tuP;x!4fr%c
zTcsgk#S74=kXtgyc}IerLd~pffc%WyJ<j%_Z15cp{?i*k7eO}xY7excLVkM7`!3Kt
za26u_rOOy0qLmp2FUT&FQ~WJA1E}x%k6qTU2qX@M^yc7jcwnW>>RFSL1+xc0Z6FYV
zBUgaVk0hpUPv#ebtnWX?G$W$PRv+M9r2cCU&K=PZ3PW3oBE;OMQkaII(vJk|*Cz&7
z8kn&i|KMz;DfYl|zy-&P@mRpE-()UOmeh6t(?)G@K-drzENnOlL5M2<@Il+%{$V?C
zS~fSplW9IAA(5Ae*1okFw(gS<5j}E)4xX*i;^&`PcO%l@QXfEZN6-QPsn%LPOFBsO
zbu8uEMky<C!o4h@1vRFyg&%o1P&fftT`2{`iIq)+n~cwNi2&pm*%5%pNpFDJ^`#U7
zOri~Nxd9K+I21xb&V{~yz2zLp8EGYbd4DetN8C)~RRzc5BC>~Q{bxlXI)oRTi2xB<
zQ^IRLDT0Ge=r#<hjXnU*#gXLFYr2TDV(za=W49fkoqQl@kh#e`7;dZuoEHWU0cH+R
z10cmQz&BaFyhooB#{yRV?<4>lj~sjN07CRaOc*%N=mg@O$My<~qykVD7(5aeOUYUC
z{r=&fbxQuyqa)DiVCoN)Upo^lyAS)QqmFR0GsHCTkA(L!3Y_`Val0d&7FkOGr5TN^
z9^c=Pg6#&7TO{Pb`_6Dtq;F&~8%jVB;0{VL8FjyhIDlFg9H|eR?8Wi{YwpR9*8{vc
z*s>UfnZLC?5F4FvSh&dTcS$OqP=ycI25h~(pYjD=3JwHr;F$^v&?}1#DvwwQnyBod
z)*b9Fvb(!``A1Dx+2zo^a1a)FEyt^cAOLwC-27=a&Wuz63hp2VfU<hYTVmfov0x{_
z_e=-^)NLNS$;lqB34?Tk8c-XT1-+1PP*sGHAqW-ZttMMMKUJHG@#qxYpWMl2QH*o6
z`Tn<m5?hX>;WOg`1^9tXy*wrovBEqG#XwXHfQV=zN;5dzMl9dF)w>XoFFF7aIf3er
z$9C+}S$l>EOYPkyO%VC4yYK5F+rjB3`qTs)4DJ$`@WAk3?FcDiC|@?+O$MAEx}tol
z{E&%MEZ=QIUjIzN3Btop_LZ+&_;1X>H+2U90t@;}g`t5zo)Lt>0ZF*{xa07Wlz$S3
zdIpB^x_9l1_-1IuFoX$3OXzW{H)Y__o@W*~zzcL#y?;)JhLAQ5?6P$>P1Fmu8PMh6
z)fjuM%Wu(c6nN|@M%HugY@;^#!olg#{?ZK7X1CHC5d_0CFC#ikVP1G}O2DTGSQo#V
ze}@b`J@$KI()7e^wZZed5|2rv<Gc#U63+xqi2!;D9U2VUb-6QALIjoXXE;7_YJ|&?
zLhU(dQX*j+Evw(FpFb7c!0XEpEC$pDnBtg2RLbA^KOrY1B`podJEsed90RBdTIDNl
z(EwP$3MJ`YKf0gO5&^hk;kI+tqP+#&4X7~Mk7%F{kglu~4&=+`gM+!MAkL&Q-5V*_
zT5szs00(`!ftu!uUV8GA4(_`>-sEifVz~KMwFyufa28ampQ;ja51?^~c>~l(KBilS
zssgB)%u)bq2U#icau|@=1NwL*x%QPEgcAYg=ur#n++Sp%2c|w5+^S9n!Ay|<5!N~s
z5L5GfIHNE=0Q|78{95}w>H{21fKi0v$+9%U@7Ls`>=3d`;+Wt*_j1kL_H+p^P)Y-q
z0s=JD`jGtrRLcQ{4(bM<&jNHH@aF|z(&Rh7ad_;m8B3V-$kKvY0OH4ODI81;lKzPz
zNP8EBY7fDG4>17!UjSRWgJ3IEFBY!xPyt&*L#h7o_Dk~!l+{HqfXND@$u5roQlQeP
z&W50l#2$-1CC@ncf7pBLpsd=jZFp1CAR!?wQi61MC`e0pmxOfZg(xB2-Q7rc2uOo;
zhtl00*So=IzUTd(_nmL%`~Np{d(SxAYwz=1d)0BQb*_D=3N6O9c<NU4mdVg58{sG=
zfrsSPD<m)>gS<h&Se~pj_-al~I6iVB5u|}i^(UF&8R-99AKgg2?<>B?jet#Xk1lP1
zC>ONht#n8xJm3v~1{MAZt^vIC>7Fvf-|6Z=&F{?#$k#Hn!j!h%s_4LeC4-?9Dyn#7
z!qH(hp-2S=aILqK8Q=Lq*@nmn?;d)Cg-0RTr+?rAU;Tjf5$-uW<fU>X4fygS!P_ok
z!sxH%BiBAYH4HK(1Zx3&0Y3ikPoV%p?$|px!DTFB{l1Y9Mi&N~@IJmz10^U<opJ6W
z1P_T!r=9_mq71&LCBPuT3ZobGJ~r<3f{#210GLn4;pahRag-3=C4=DrM1TU37jB>k
z2ofPv%lFR#aqlF3B5*|qNRJzRMveq<6ugoLA4!Sx`=*Huw!K##$LbNZk-(Jtz)ye^
zMlznMtbuV=-xyZY=TZzHxHkXt8C+2#WyX0p<Rt|3bN>;-e}wSgn$Ru)3)QKY%J=3I
zReRtOV@eau|JsMG)Tpgd*lyNMH#<+aIDqy;h320^@5p|K1@%?9-RV)I>vh?tjc!DN
zg+|BFv?U*7wZ4oMeW@5$y4(+^af?p#nDS(Td0JW7UB0VkgYlggwVvf`sod{brD9n^
z&mUTL>$d)sHk~+p$qCxtXV@;ijNwv~OUbQJE2mTOXOH1FVO?j{)gib2Qy$IObpxdp
zp~$*l3SIDql6CSQVw5b31S85tjHG2xk|{ID4z(Swx)P>){g-W%adCft3PoCv3!RDE
zFTHGi4hUTkyhr4^`#bUVYnDUW<?1WtKfCT{&Qoa4c45}gP*Z1O!xqj@16J<z4!pZ&
z-q~w9&8BV^+|9-3OGyeszr%Rw*Ee$tm45%$L8WD;?R8IIMq`+dInZs9L>F!PEm6}}
z(G-BGfK#E)FhL>zt$>od0fzwh1J1+J^Fs8LcN}L_cydOqH{x>%*DVV%&}htDq}Gc=
zghu-l29<+}gv!?R>ECE)pHiMo-`!K{DNpUhesV4mt+>E3HdO|iTcu=`F29%fW!Z9G
zKIYOO+Sjg7v1JBgvBZ<j>ZE;IK+sY(+Tz0EbIXX0HWol!zTWk4=6xM?H}&$799eMh
zgw=1)52eg)@L8>G{`EPL_Pt`$t)wDVei}XXoDSP+fYb6w?q{}mEa+Rb<(9y|)PXkQ
zXpL1GPd;-|;8g+hf4NA|AxZ0@Ko|_Ws%|TfRjcYL-S*<|D{-JFvknaxJZJc%g_tpR
z#0bJajU_+Ll%odmB}mz-0Ba9hUOU{Id{l{^-++qFKgR)urHH?UnlKf8<%1QH3;hI!
zM*<0DES4n7?wJ&?aV&WA;<04eb7cQHv=f#u3ZO%Cr+RW|DR+_p?Zs!P0YWJWo0j)p
zo87bAzBI{JHTZVES=O=CfoZxuCjt(_S-<b9w=$A1SJ8ysSypY&*LYBnkTOKH%#?OQ
z%uCI6uz)&7-E1N=#?c{EHCo-DSBA)n?o!E=I8<KJEul$qNT~NsP^Lz=1J)Mz<0w~T
zr7%p?cTIYKe~v2QGG5v0!{jnhx*e|(lFT=xA80qw5gt|I;E7qlyTAj7i>{8htu-&V
z?M?HpB(brAMVE*@?#~e1AFk+fWuO>#W<1<-fgFr&f9@$ZG491Mt(P)sE=hX@hzr?r
zvs1oAuPQQ8Q3Mat(+n%5T-lZ0Vm<Kryx%xodk}M%-oxorYy6Z`2_8G|KW}n7Kw0GW
z2(s2`mTLYgD~Q9x7R39yE_CPYzRR>_dwCg}6>i}rg6{8Z_p)cUJMhSsx8bm20jJXk
z@$Sx!?_^L)`OjW@PidVxJAIYq;K9`}oa5!3&xuB#F1L^gQ;PhrmbVYXq)Ux756{?8
zw1OV4&oO5)+=LiqY>->;_)~B00~hp^wt^K$=LSD_j<AfNwNRc6od5Bt3@wedcXe3Z
zGuaIxU+%xMb{<tdrhUc#MQ)FWHX*nzV3TQW**5Y`x}vZ?L=0ocYvM7u3Afa~du)v&
zU*UX=ZVP1@sL0?M-@DUYk)%0ONQqamBwT@h1NKSFE1(`{<Q5L&zB;%P&pp2JEvaP<
zdmbS~QBMUsH9*M3EZ-_vQ#o5g+Hwv&sOh4MhnYmqD~wnQo<)c8^;K54nkt$3k-3LU
zH{EjVzS)nSG*;BMrL9{v5XN_Il02Pdv6wWkqb7Jps~Mao;GX4~b1X4Ez6~c~o;Kst
zK_W<HOqUSQ$V8IwdA3u`2di(ee7Jo%IbX7V`fGac{mD4{F|jSpB4@}<-Gp`YMU*d&
z`EvhT<DkHbQ0nDV(?juPC(7D!7HvUqfAOguS_BoYj+Fp8vF*yEpGHLLR?dk+W9B_^
zd_AbQq1jc4;j`3{hi=g~0UZ5`-<;1WkvZ@(7}mH;2sl$2$%JID`#nFT*A>#2A45g|
zcGob5SLxm@6D^T3-|vbT_AR_^(&{iYa#|mjEzlmAdpyp*?*N6i;^7IbhLvQCC41*~
z!~K#T5tC~6H2FR@J-)nGqrvNny{~bDb8O$VtoFG|P)QqaWQXD2xpIN4)BIY~h@#hR
zwbE#Y+pbl8r#j_l=Te|z>&S?0S`^vG;vl#^)8m-x@itP1?xK@Kvpq#*nBHadJq}4^
zh7B50sN_bzw$iQ2lWVEb!o7Yfni4(cW475Aj1uP(;4$j24U{9919u8B*`Hvpb+LfG
z+yDt_8g{oT<E&->Z9`nast}xLfL8NdiH?*rBwZpX>J&Tc{S?;H7JPd`qfVg2q}y5|
zUv%QhdQ2)I>HWP66pQE0_2f!e7i_1NuLPHyAFa!k>u2t!uQCzrSgAsuqK@B3`VC*A
z-HV5i`j8g(nuz87E;eJ@`n{pIeLkQz7Crby&}iL*lxhQm+Q!wp&JoLki;`W@-A3D1
z0=TQSr%%P9oY)LcAx}eJ87dE`+l!=x^6w<`Drq)4?=+W8b*&QR;?w4a@f%fJ?-6gG
z2MM5d%cn#P2g>n;Z#b-s+Ifq+Ixca7BRI;B1@L4vy8N}$I95CuaCu^^)nj+gY-pC}
zA8d{`hqf<rckQ3~e*5&Rd3HPZQ`K%hlaxYx72I2zv+PSDCX?h4e-(4-=@ISfrj0A#
z&jb#(v8yI@OXA+M3biEDJW!f$TDdQ-jvL`^>kRNN{u=CUY8Tlb%<<2>DPu58<|)h=
z>#ZFc!=8^=b5nWHyg+ycKh{E27YG$k3fZ9>r-y|I<$Sosdx2NQ6u@<e$M%71&8uuP
zU~-^;Gtx|-6~FhrK%9KUWeoJb))4hA_A5U)rrQD9i*x?gTgby)t$ZBG<Vk+&=~Uos
z<Edi3eT&7pxftJOdF<cROPPuu=8fLNo`(iaI=N^yFsG*g*v(aOpU`31W4hg5quGEz
zunbzW_^+u$as2mY$$TxHU+YM9QGSYG0nPBH2{X(*Iik`$th$XNFN)YQS{~J8CA6&u
zu{!&~<YnJ5VtLaB22wx)cn`s6o+qZ!ma&YoTfw#G#-8gw{JbNnStIk_RPmFv(*j;9
zR+8j<hx|6Lj|UMEbzy&G9ID?KEsiKgoN4j|jeVzL8x}1O-}z3(g1Np$n_v7&JaC|{
zj9%cC9XM-Qj-BK_uWFN_g=TsI3y|X3!*EzW8JfD;ctP0}D&dTudYWI^qJb8|SE(Ar
zsHfNFN)_kV5=Y`QPt=5OlVc)AJs=$_;TTV%GJjhO9;E)v#TH{9sDX&F;6|p>R7IR>
zcHlK7^F#Ix|5-{l7RM65!+GoeJE~pP>ZXybP=v2lqgN`FrepWe(53Vlk3d&C#ePQ{
zt#}GM@xJ$Fw*yNTL<`q9F>&{S;}r$lmHIJGzk(L0AAY+f5VLINntM&l$Gh3T?D>7X
z`2B?NFP5wBROfn(xw7ncT0A9_m#W!w_ib@~xG)j?vKBk7#woZc$@z@bq@!k7v)4UX
ziqFaLrTy}12x-ksUH!$8_trmYcakbHsO|lZcoXC!r60-p!`$ZDrI)m6#v#6{J+1A^
zCVh8cgc-HyS!QbP%AZ3NJnGE0L6<n?8q25nUoaKZ`*fCO0Zd85A-5bMJm0`$@#Xt!
zQv#=3^h|{_O^(rb2AfIKSS06T=tY{LAK?L;2a+0}hP|#z4|&pW#+Yvwf|PiN6dGXG
zvBqo1pGmu9rI3|7nc9dEXJG^US$riHT7*3s5{gSlCs1`Hr?<*tVKWXimF|~#&Q$!Z
z9_cr+*01?gn?r))*?3UF2>~8t)8(R0h^^bz*2y*6&R14^9>ZfE_Cq)@?dz4nY6q?k
zO(#*9c%BPbB#jQf(vyF~oYJGtO%Uhp8t~2Z>4Da?hf-~Sqhyvtu|F6r>Bt82h3ni>
z0rAT|0!D!g^zDa>=m&c+u8a@jjlsD(#3?lmKLZaXrZZLHfu38@`gs$DUXdTuwb^Az
z03J7kBln(Wg&c!Uk}22R$SvwGjy*;Mx+WJD*pYYpy3HcPzt45J-j4BlrNz#*nKmu-
zbl$q9AiM{TT0#srN^i#w*U^4ocS3tRzM#2PDI88-zz%5$E;Z*p0a`9COt&X4E#T2d
zA5}EUMc*m{+@%rq3F-_Ag%hnXu|;aLjNe}YG3!|JT?U5tf@MtL1O#b=jdSAnJ|)oi
zVbr{Qh$}jFzku+wG~Dch=<ZK%0v?FW`enoX`Z=PdiHJvFRup=cs9K|lu-!g$F(Loa
zfHOmkoK>0e!))%b`?StQ+1{M<C0>==|2WjWH<z&)nYU;-`V~lcn_Q)J+pn^2c1i~h
zlkw1fUNIrGfR{%Q-j#&gZn3wCtF+y`3`gf%$EG#;1x}$lH8l|YH2mseRl!cz!G=#d
zC#UKK;8!>FWB&QD-HZTD)Am}fA*GLA+3V=?X8z)yfSUC`I{erV=Fu;e>lG7e%?$Uj
z=RHB6pK^biKrFktwrrhFk6BXHC)G-#*$|5RJDsz1+wqw%B^@l}8@k8nU$Z2*@F~5A
z&5{|l=;=midm?b<H0rKN$~y^8o)|0rao#A74P<wZ@uUr{YUEq!B=^}{LGu|drIXoE
z&$WbxF5=ExN&)>JWmX@(9XAC@W*gN8{c|b~a4M-4uS1_u!y7Kz$|v08^2ivh$SrDz
z?t*i0NUjGF?1L2!;&WVS|2Q4I{dE~lghKN9BRph3X_;qHymzRbpLND^{hcCbv^WP`
zM4hsMg^*>z46{Y1$a+_Y-p^BHCZB@ELrb2YgV_aP;Bcr8%2xiSb`k}uqfhKE;=z-&
z_}jl(spPEmpm*l@LoaIWR=iO9mgETOipiS7bNq;06dM;b3J>QBBM(kNZHp;T^4-DQ
z-eV8b#23H<6iB8vO3L;S0=T}OnyTo!ccwb@JST#wu*@XbCgmL}1Xk`ELh;VYY}9lu
z>~0(*i_#TXelVs9;C16Zx>PT2c~TRXv#(v#G<91v&2S`H+TEy%`3*K{WUbajxG*1E
zRZ?lWBI%F8N7YuTFLZ1QV2Azb*l%)_a%D5bI%!d#>m~{!@dK4e>;jT*%DZ&YXZ93y
zJjVuU!4nSvd<saoNVL1X<!{=NhSB%H06ZH1aE$MzevL<&Y1~w1aHIxiszNIX>!*t(
z?eDLaOp>!wU_jTi*<f;m{TDoJ(6iFC7;LVbI&42)$OhAXJnQwx=I1<`%biur$<eQY
zqGk5j>=fb=qK>H-JC^~|+ydlM+bPEsT9iIo8`bLINQriX%xsgRcfcAh<14piKYCxT
zx&yr?9r1Pazx-NJ@AhzzQ82Pv4258VIXU0~{N8~n*Jyf=aRZh@c-?Yc7yW<_9&j<Y
z{f`Oi&?md3^S#(e@!8nst5`GsFsnteQKz^UV7BGu)@kD?;`Ylb9%$f&J6dULGECI;
z+p4T<;=1_#au+RjA01pvvH~gbQNTY=K&YB`=ZQCj{^OWd?!oa?n?dnRM_X^@k6jXl
zc*oog%wQp0mT!`Mg^KRK?a@|UMkJU53G1EnmnF%*ZVkV6S-yB4ANr&Y1KgFHiLgso
z1G~v?bg%(<t2c_7C}e!%p_7~0eBRlKBun*%Yv^IkqoXXesV!Dy&C)vfEyqWH33|3*
zg*8NfI;*{qmg%o6F}hr!T}qz725Ot#E_~C21f)4NbxNJFlZ_o-(0?KtI%wg8ZJ$cU
zu=%>tf)!@qxtcac`P>7>LclF!_4Bm`Jxse#@?x+j-3SV;Y76m;VvS*QubAMyi`edJ
zBM3nL1+evea|E^sfoDxVfSY$diVbb7Wdl;inS{!yFCu<ssxH@_Nv`gkw44Zmdq3Py
zsETrBV|EF2&51g-0|lg+>(GH{cs%w>H|+T&8qni{<|!!rDOYtYywNBz%x>{|&>GK8
z9yXwWH<HrAIl}Tf)yX=xGK3Mp)jQt9W_2_(mS@&P0lx7xv7ndENV%8T)n2Oy^^_lg
zF(5y?o8bjw8kKZBeXys^t<Z&Pw{Ec&*U8+{=A|Ywz$iVN|F>_e<%DZTl2jYt=Gl{`
zH_y8FqCRej|Mg>F4VZLFTF^5E>g}6n<G9dIIz9}?!M=Z*XLI^;Z$93pp5bY>jd?0E
zkd|o6`M3XkW2mv&oXaIhbi(e!wsp;JWpvz{Y$Ar#$~7E@%+!SFYE2`J<xjiity^eY
zmA)ekbLqY?Z|g3m8|;DYkC1^2c6C9=?_LL|R-tT>xZ~9NCPOV%vh9Psjb>lCLaWL>
zmIxNCtZd9=Da#uufvY3t3$$*;v||ce4)@yXfobRHiv4F6O&|XH#lb=l1W@fS7P_;#
zXLa#<Zg{-?0rDFm005xSuEL+oz<<rS-A_ecQ0avBw7tvvit;V+tKR#5?7k>tbQ3!1
z>cSYQH`Sly=PskhL=~~$PD!D>Frh0A<g6~l#r}LLl<xN6km7OJFye6Nb-**?Fnm9_
zt8%#np8ia9OFtbPG~3`?=6RFzr5;z@TWCHvCXoR-0zih37Oq0IIri$X&pJpya@4y%
zNey3Lu*a92=Xw^va%ZwHKepK#XTNr2K<B1G=w@N)!?UxM?U-V-$rv~Kwo$qjmZld{
zzJ$?at)0Ada_)gzVx+e0P&KhVId>z_ayW1@T5qm+GdHjv#3E3-nn_8qZSUYUdp^Lq
zJJTtkDVcj~Q{&B)cpNFS&8b}^;M}HM>Jv`v^7Hs+LCEL+t+eTIG?zyhF8c_kqdDWW
zp!#AXW0RgK`b(xNJ$JDaVuujMfmoNtg6KPkX;Q-3`e-ZN7P;@ZOw=Q-95QmW>g~tQ
zBdSGjefF2=FI}wOaPiyM84=fEyb3QCx>sauIDTW!T4||W=hkxiT!Z}_t|7ya$L%Cm
zP(zo^<BW*aa>bRTd{$Z@XYgEY>k#qbT20rxGQZ?uP5QoR&`Iq>)u_PFvH50;RQGAk
zEhC-HkZ7e5LCU?mFVm()X17_m+_@QN*Pa$L4Q{K;ahT$AQS(Ux4ATP2I$;8g<$`TC
zhv}8J{T{5Azv1tj{;Y&4{+Y7uEUSSMXtt$#La*%$9hXZ_Y)4ctl){$L(~V9^XwVUB
z;)9E|b65A5D#iSTrW9(+q2PxUw)oBYOd{4)>{3L7S?h}1$|)7=4%rNs=<;2;IdgZJ
zuVrHM?f2)`JFMS2Rm)QkMHLE&O9e?Y#^mQ?jP^`1X;H$FTD|RA6HxUpag`wJ^x@-t
zcyn)Ahi%Jq-~4xMFZbXkwYvfC78-e7J{&<7R~C_%-8jI~L+aP50ONCdHIFE9G>G1@
z+X$safh-)&BqCyy!(jK4uQ*YzJiA<=20s2Gyzl8>h+m%QZYlS|DO9Y>dq_(J)bn|<
z{K&k-dpdoNQGl-0`!U?&PXA?R7{C`I_jkn7yNixWJWQ3O+ps7*TCMh>UQ$#dY-{ye
zo{=_H$W_Ay@7x#nF2JMh%GGQ!NZkJ0Ft;oUF0V%IU4;k+2{)}<_y~<WJ&e;Y1gflq
zsoOLfV`#e1z;JrqjzjPAG0n#G^1M7*=6mId_jH00f#7UnJx-6zy`@SW7rD_)!R_jw
zyb>H~*0k$3WqS!t`$XQ6-bH>5(D1gi>#7Ge#o)|P`c@^9K_4eG9JH`uhiYt*AxE>t
z*D)mk_tgl@wn}#4!NRN3?9;W#ajBHT==X>VA(LsI?grbu9?itbOk%_dw{t=`FMt3H
z(x{Gb{<lPN<&%9YI$z0`PCBW1Jhvj&gtp(kMM6zineZmMZ;`IKxfyT9K|o1;eSMNn
zE<xwD=p^SK_`R972^QEcy|vFpv-zzX;R3&g{&%fTwoS8V&&Po`>ITyg@o(=M{-H<5
ztNg8T%yU;o#^xPGoXJ!9@?CEfV1$gt<^$|&Vb~wYu8T-r!;&Xv_St|iV)16GtYc?*
z<gm3v&wyk;UX%)ITmgn1eh(q=e9PHVtXN64A@1dmy)+iXAO}-+0sCjb=EKZ&lVfvh
zSejG&p@c_W>~{|9pK2t`nvI`!ZZ%`{b0!^+Y_Y3g)RfC#xd^H-<)%VI_<|2+<W#dO
z226eO_<n7;jz*cGc8x;c=$T@C^Qvvyfa#DCG3J&!zql!$jrP1|Gi7--l7Dh3J=EC$
zJ-jW@@|nK<W@hC_7H5?68v#nSg+!LyKaLM(9G2{@=cuqO`mpdLrQt#uW&+|I`$g|3
z;B>TlYcLlKvU<yxOVvoOO>gSP(E#z*Q5feW7-2JK-<K_E&>hFwDzErh>edV~@6aTX
zpL-B29Rn;YA#rwhxhc<!*kSw=eEFH%dF#Y6lY)|zw%2<)47!au6Ygb+DQcM^Myzl9
z7YtffFOUe_$SO~CZYg$~uz>052I>4z9Mol3J?i4y9yU0AY)u|0Y9fz*Gv3e!!P5Id
z)rHaKFm1(i3x}B-;_=D=hV-i5o6pd(yY)?@*4qnLYp=`3gM7etSCUod(Y`)>flP-A
zV`~haM5R0jG)&*%OF(8}d6S@f^X0PQ%>BFXue3sKp|$40cej_`0yq!_WycJH2TDLG
z|Lrd4j)Ko%+)O71$DDS>2MV>Xm@*wla>RF+kThiG?1_uprsMnaWoX1Qr%j~TPOB@~
ztAV{=0#(TyWqNf3hgvSQ(S@ryGuLLj8YO#qi8#|9I+seoT9?v>fFuO|1Nk$Uaw1=X
z+1@Dw6o}QI)oxI~%s;g6Hm1Xpw=>P=$oWx~D|V?Z>8K1hTzvuJCsMNvcrWh%;7p3P
z?z?A_?g=kq2zj+=a(X5k1emu?2lb0jTMOz7^&ED*dU7fPC;(Rht1LFTFFDT9qg0TN
z=8g9_4tJ00mEEevA<{c?qs$2N8z&x)f1Tx@A1u73oOJPTenr;G&9j47(P?#g5m>Ow
z|C}FImkZYVc@ALf_h$Sx3^o!g%%fF6fHqkXT!0`}4?1LFa(J0+-{O*(qRwq}-bxhC
zLIV#pe`y!<MOM$Lv+KSTh0za$hmP-xx1(p_0@fc*lEB@HP_H;G%ELTgsiO;o1<IO2
z=2@;#{Cj{pQ8^??ZzaL>H(Vgn_{;75<Ikf0FFy-{CHKIL2mg2H>SPb!Yz-eZV6vg6
z02mJkm_3O&3Nq~^gdn_zbM(h%Pp)3-`rz9xBLm)B-P#yjHB{Br7eKP=19`IAeFmrN
zY`$|J13gv0PJv6%APT~rx)1c4b>9!aY?ZOOv3Yfag_`6l++U0BO3}ZVu|7=;I`s1T
z{1)Bbe)N^%-@mB2fpd`8tnGVdt98rUogOZGQFi@YY_NBB_yUJk_ax9%#XV&@&yDRV
zn_K+)*-qGb9SgApA(w;5Rn(1X>qW-;lc)s7&Q}P4r%~s-&7N{qI-$kuHLLDxhIJbP
zqS|-Q>Zj*VI>lQQ1{>lZnEGpU{pTx}Cr)`K4~+@jTCP(E(SU_tzVyx%-<a&L_=QTl
z2<=<lYkPWTW^pb%pK-dz%g`3y2^!TYU&#lwnbbFTvh`SfouwP=S~uH40=-%e<0drs
z9n29Ksy8JJs57oeu8->4b*gM{N9-6PAuuZIWZ{E3K*h;@eUaVmuC=Eg3>k&-Db(P)
zeh7wI4M!qzIy5x&!P!f%&Ka1``B>eWSA{4NRkjCI3+NlA224z?nv}PVd8_;A@$MSA
zttYlBN;x$agXzqi;=Nm{kVhAv)}#tiDgP&-*B-U|F=1R2$b1`Tt3^qZYmxYK?^1sr
zq#t5<sT|r9x?x|JPF^E7RL^ll=65jyqzBhAo;NoPRDd^1aDWaR5#}Lr4IDM6L5C>=
z(tW&K@Q&+<^8Db4!u(LW+n<IpgN@+F6~2B0;I&<6%6L7zpEQS-Q`(GmT8CX$TQaPy
z;WjR2aq=ssdg7$%9VIP^ILHj7X?{pp%};g6b4(PNx?3~*{um2#xNmuR{->QC9nXKo
zy2ZB;%#?6v>DPMLusq9qEyQrG%2}>!7CR^j=uh{J4mAyYcT3+@*s)ik-+nvBRg7iL
zQ2G87b481`^X<7R?*^)P-R;t{qRWH)N6P-0&zM>zVjuQqu8Q%Vf1Xr5=^hs9(asVI
z1(+!}9+MK}Z}g67@gq8)_B{jTnp>h9!NiFbd}CM@lE}Q%NHbnIO>{}N<~99_Wtvo{
z<2PMpkuw6=RcL1MbX55TBj^KdID2OVy!P+HBkDXX6ZG+AJStED;vR;chIQ#(1)n31
zD(vSlAVXcNS9F?y1FL^u$-%nPG39RQF!)iQGJeRy-z+-mPsf%(ZJEvejWAE$!h3`3
zZp=XR!7xugn;&WDMf02HZrMFZZ@*Tw+#=K9{f6ymRWee3#UJtc(ZS3PSBSyl0vI1%
z1K~eF;8`i2d_zQEXYYPKb@e`_UMruTi@sLKe(s>l!+ep)2E>llY*1MTo(AS<sUqm)
zAyBe$b;<q?)nMK;*>be-kk=#SfO}p8&GNTyt5&m&oWLfUo4qL6XLr9-W-`M2cP<)D
zTb3;j2Jx7k5<#JWuy)8RA&$bfgMz=82XFauY)Wa)WyDQM295Dx$ap-}A0CYK64cuT
zDAfI@ECb5ztCIHXo1DDv7w^pQftji1zTPrT#}$6wai+Q*y5(V=l5Ce0jyHpMbor(L
z`QF1^3r#;C_F#$o;-d0qP1Ky+!ynSpGnag*`@*O?b0^#nh`nm5Px->RvOvda^4S78
zU8ziF+DENCwtBI4(Eo)hegx`8V$GvKG?Mt_)gbcA)Q*E`Z+z-FEgNo@v~q?|<=@Ay
z%}Uj%nAw(dBLfK&?bRC_;x%groajHx1RPf~9LqG-E*@&NNQuT2`j_oXok$P3(u+?5
zA3`yXskn4aQakY-i9J+pUTp<l+&D(1sW4-O0PP>#!^8DzzWVXl4C)*;_1vlDe*AIt
zdD@Pj5C*bq`0b#MD{6n&XNq$_ylsYqAxQ_AI9Sjw^C)nixh@s59y93{NnoG+H2<47
zT?w|mAJQ4`i&;0iOI|^pxgR5f!NQ6G-be3+b(7pnjGYX5hBQtsRLH<#y~qw`?{hgq
zwIir9_E96&2a_dPO<ceqUg*iQ3&QG7--D%ffp~3v)KpbRU96<yQFVrop=dwwV2oH8
zhbb~B12GJY32v2{t~dPTYXk)MPGs8~VG}S6-g&N65{3nT3#|`a;2%H&LKrZ$?pr0t
zZoWcqGtb$0<4S4Jk&;K5BNZut3UUIpFxpeHz_4`(yh4P*{tz?IdctrW;9Xqz8losN
zsyqFHdnr5d!YJ`D3l2Fw92AOx>I$7XZ+h7$R5C>@6Wcv57{fOCa~*?CTwJ`yn+OI_
zDosPBMwyZ%7hdp!cGtPXTq3*}&^W%Ps(FlIipjSj6STlWMn5~a(=Rel89{r#%{UjC
zH?@(LNrD0Y<-w;#x?lz=2AGPQaV8VvRXJYw>tuTRaLyhH-8V$7I>iDPF)f){>?D&z
zyf({)7ey~8OI9zKjtJnI?wevHL%k^MRAJi%v%-eic_$+&z!xwZ2<EQ6`O!-X>o;=q
zbrN~WCBPQ6UCBbAL-Vuf<i<ISY9$X39B^9lh>ghFv<qd@A1Y6bTf69iinOfbis21b
zZ})dks%oh4f2wNUBfP7-4Vh{PXRl#-|D6gDOuj1v0qw%OfioOZ56zaj>F;n%f?K^k
zIMxbjU-kK{MO9$|%54G#35X<)O!6x+R#;%kdBq9wPuEWx=|XzOxGxsUM3%&HL%Ib+
z-=kgtoiCju4)MIQ_n;%OVO0;O8_`OzfG&jE>1D~lcZ50h4DF+8(I~aI*m^bkh0e1t
z%YpjRSuw?y^=#g^51S-lHU`5bB@VOlOeIee_Vsh)e;Ov}mE7vNX_R>6DBi<w4^u>^
z2uj_08-?(0i;eds|3MeX1t8JTPDk3RnE%|1&+9D_!Kp_6=umUSUG?$9^;%tiYAgUD
zyAiys`7K$SG$ZdaK{*b7&!=5!1QIa!2`547a`8<~?0~#08LNTeEzC*xS%YLU(1AM6
z)Svu2w;xU^JCm^L%?Ca(qmprSU!E6j0^WP?TzHAQ+{tR>UwUZ>zT|pqjYEzChz-xn
zKW#$d|FVfGy3;TaL6tfz8^6Qf|ISuP`vV?W2oUsI_j1o<<PB0=axz-WW$~Hkt6kg(
zn@%?G4k5OCMGphvKvegD{Pq}Kqm}36PAdQsmgJq>d*qaeG)dmVwYt;yFVOM90?y|n
zx;k3Pv+tba_EVbY5P@3~eOWlsk!zf6JPjq_6MMsh(Bs-5|2H?rV7Tw<Me+#ewDX!P
z*#nRIPj+2)YqVrt&+JZMC(bu+EH+n4=Sr#Y>J59QUB7t29Pbb9_LA)AHFqEZj=x6f
zhSgTSR!Hk95+xac$9PnOPvu~ALsu4&V7zzAo12Q1*mq~<y?*sJAKRG2fP0CdRm-4E
zc8r=#_e@+z6K;7J>3=a<tiJts;G`kgSSFbCMtv;}`2t~R2B+h|g^ce!%(Wz2xXYvl
zYo<hG)#O6j=2LvJZC{c`MKZ5bDw~{~z->syR(b6%leDR47a@fCO~YZpbSQ)f{+}V^
zP(gT|M<+0qb<|}JJ{^C^is$<UwLa!L3AWIq6w73jbJ4EDBMfhoB**)gTdT|eMC|FQ
z+kUIpQYU<EIg-7|n)JSECI=<?V`f#z%4<N-$FjD0ily_#+T`T|!=;l|5!aEZXGo<{
zI^<P%D*ce;2>R5L@$)*>d{f1Si~TsRjPzFB((+QpI)CTVAZ}^`7|3$BbbQP&i;bW4
zH+NS39w*|LSnft;T1gYCmWVsa16hJi%NGf{8E9G$;+?#b57DUonSKGr<KGX@AK)rZ
zxnR6+B<i-xe3nkFjMm2=NbgCkm6w&1gS~EU_-?p32X`Ju;UA;pC9LCJ(Dk_>P)`Hf
z%#3a`qu^|_n8~-fpC68ttWz<ftX!z2bf9wS-#Z5)N*SwVy~AvXV<k3lp@C(^`7GHR
zzW!79aI3RJ9$A|vmETVF<mUb^Qw7pXJ_?_!zs6n|Cfi`A_QDTdgjEDud(}=l#_-&<
zN7T0?O)mOm2#K^XXs-GzaJswxC7xWP10<{R+q^Qrb>I63mlcxFga`P-M^XJtH}<0Y
z5vC0sXdJKf$NYx1N29vek2*Tt>r@3A%Xqy()Obh>M0l|^h8)@sX{?nP7EyYAT-hZ{
z{l(uyoXyugBQj0OybCai9&!3bTwblhXOrDt++{9W{v{CCZXE7YcMV=fa)ybpd$)Gx
zFfztBlqD7nNp+uZW$DGnrB1s)T}8GT&}w1B6E@<^{7snON5<zfU3*B7+w-Xf`8F0m
z1^>Xj%tiVai<Pc(5jH1LtU4rsIFQ(J3hTE*8%P>~IAaz8!V`Gavo54tdta|H(wbpl
zo~>Q@YWV{2_BymQ-|zK2+5ScarK3SMHGTd))u&c7qKQWu2Edv=k9!uk`N}H8c_rPa
zI>NQuMXOMKm3p@}VShV&3b4|sQy}xJf1|oX+=9wH=G!}4il<3w!+iVYI$l6uiJhT~
zlb;y}F4FgCjoyph%)`^$B>;R25Te|SC*UX(10e#lMy_I$`^p+s)e3(!(mh&$;q_hA
z>whZ$noQZc0WG!VN2$9x**>*=e!7}=GW^g>ev#xl?jeAjk@w3XrjM+k+vhmvdGoTK
z0{dB_vX~~#Nt~Q@TykFKFAwLp>03j+GQ>!ju7?fh$B5a3pF;r1jO&N}cYLMOw2OV5
z(`g|0z(3a{8?*hL#|El>G$(VdzqUKYfH-Vmd<!Ct>Xjco-37M;Z#{Z{#R_c^K1&(M
z=RJ-+Xp{ZCG%38kN~Ll2%ftS@)8(5^@I9=iT0tRC4Z-CXnSCC`@Vn${dEU~Bh551{
zk171d#~B*D^)R(24x`8l<gKr2YD%LB@&fPuIkqBlfp(|o?nV;~&tz(4+SwXQx5ex&
z#}S7p?vb(lFAWys>CYHjxsiOG8?eP|csZjn^^zDIdYTKlu8O9aw1KmdCH3H_3V0|K
zk<;4mx1nwNlccG#{#(-Ite6RsJt4dZ!mkM*<YNBe;4FFC_+QZ7OCjT8Y`FYbZZ@mS
zMc2yUNTZOOgT)_O%!*z`FHOX01kxac{;A1zpJ04Pos7m-9ZPipeaed8krAxx*BaqV
z*+y2E1Z$2h<3DuTtfEtQmg&0XCZ~<2Ua7VRJ}@oc=iuY}MK8je9){_-rwW6VT34es
zS&s~yxW{wM6lS;x4PnB_9v!V(X$7i0@|RKQ&smuT-q=cs&}Esbr(D=Dn~OEyHE+<Y
zcNapEOz222_*sY`PDuK{B5ognYP|>qwjI4lhD`y43qjaMcPX#`*>1Ev-_QXG9WYjM
z#lVQ%^&ZSc$jfA?2tbxVL|-r*BU%dWb!>A<2x$%kzZz$fTrzRD?D?Jq?6~M&{1?K%
z_LF5nJhreS7M3c_Bt+h$yN%5H_6oTt?(J0c@3g#UXhE_?>Or+?+>Ca-=d?e5$|Jq2
z>e<S2+vUKHeA}3xhPs%uLN1COX2fegIEaU#v&(Ly?3`omkw$(oOmkIrw13I7xJtv0
z(}#cST^YITif_mRj@FHg@&rICgmOjv_%U20d5rJbE_PZxbBkZNHrDB|KBm?m7fJAw
z`<NG@&gTyp{5~lC<6b-4)k(wKnVsYuxv1qlgK7EWqfV~Y{*T+vuz4SSqk(T028@H6
zvh6pys4?STO{#<j9I=?28)Owt`6HY1@cE#`=DbD8x_l;MjyE#mfWFyH5Xm$D#EPWq
z9~%d%H9;F3HP+)8f`fM6&p_>ZWKoWm3$>L*Q6So<v<Z>gU|tHcGPsYd%utK@OBe`o
znva`?01mT6DF+G(PZpa__1diBL7r6Q))1y+{^hW$Qw=jS)m+`B{C+#I5w%sXr@j8B
z&qGwOBQKV@;cEjme=9|3um1A=U5Fypnolg#xM^;&kcNfRj*%P8joVPTdv>oM^sbmh
zP1{w)ji<zvVkPY5ZXr4B$}d98Uxwe_AprdA=$`eVav0QAn<})Wvb+?YQ$AJV$*YDN
z*#iMgW)>TS^g{dxdjAr;2QM|xc!w$vmc6rpGO18$&SepOe@SxGM=+{kOj9_H3rHHA
z?!O0RgLu;OOekN6^wJl<T1DuS1fqP)lpddX!h%}?L&{C7eY<Tje+jWj0>n#Uis4SP
zJko;WM|t}^A?)CHUGEb&4UofJL_TD;RG!iNK|<sci*Lah4OR+bS*P_VLIa#?wJF79
z&`OteT?$g+0p6<?3-)M%dFMCz_%Cw*tYAzHZ`&U7Z~dut=Y*AeyTcES>9d$H%Yp@l
zU$3~rZG5y9K<ApGZWlOLI~<-SW^Cr|T(9Ek8Cm%+((i#U)>6t<IJ9zawpED<xAwMt
zgoDsn`~x}ua}ZY9X7~R0E0m=g#gcfetmuoqyZzKQ8%pdk1wtEz9>3y^S_?DxuSxPk
z)La~cS7n$5(Awg2VCEJfQ?7jKi<hNL0@uT5w<dXi>8;_z@Sqb-MEum5{Y8-Lz$Dqr
zb=b3Ws~NE1Zm$UF9ycAx*?eY>lhG?+xOa^G0x0JnS8M9>$9Y!YpV!isYU=Lu#NmE8
zybR5y0ZSw%kEm#YRuBO~r0igmBnPXn-ihJ<r(6dZBxaV|2uUn*8RURn0V%Vhd#kjm
zZ@Zi>%}dEtl{o^%T%&oyTvJUFO}d<$dP>e`>1_fud3@`G%|N2g-ygOO=Q%LclDjz6
zJgL^6%FqMox|aL1PG0ldr~;sUxU1ZSG)t-7n|YG7n98JBx%xu;2h~uyx%fYhc1V{Q
zFCL_ET|hZw+4?>EujS2R0W;(4LupJqKhZUg!%Th}F}-!x(F3^#Uk#ao<CapTUxP@u
zQ|u(MKUSNZ6utUn05tD2!j@lvu1~*qGK-}_+eL3=r+W@*5BO2X8gw}d1sF)8Ed%c_
zw6Dk#n|+k{%FXW%jT}HK$KI;7c>`ByJn)}=zHHHEc!L}vd_eQ)OP*mp(o!2s>{XON
z-lYeK68L0Y?kIp1*eRIyWK2997k8%4HN@jN74MG^${tnPcAKHt4L>M`&8PuhwbJ6n
z=vTU`nX{iv=5n-?ctPueNW$&Dj2leIUkGQMv?LfjR+=Pb8v1osJtdN`dWR<XFN&|U
zDKStWM?k`#T(z(NqpNP<8rnkwn()&_222Z52TtF=m<XBbME$t>C$;pn<-odVF(9t>
z!ro2PDTz|d6jmR^vqh)(xhXn%F$<?w*}c-8-s!muiKOKhY_=$UC_pqG9<R|dylbCd
zuSk2@?u=I|Maw_7GH3d>dkmyV*6g1+>y^jU!{R`e4Fo93f#=2NIgHu?iA)wA54};&
zMJEOAY$N}0L^?8Ma)I0(jb0IY^{}!vTTfEwl$j~)N@_=WZ$LKnQ*B2#y~SE5dMkbp
z1-QMJNwXa^SEv$TY(otHFYmZS5eG6??LAN0D*PVIJSP{DO`YKZtG`-co^Xj)U)bTL
zc$$b7E|gb&@&2dnsYcnHj|a>kF}LnUh_{%;XD+X=xKKSwhr|c=R0kR+;TgaRgd)7U
z!FEhrBmtd_VW)_AXNCrc4eN|!9?y)OW;=W^dRcl%iZ3|>Lz<-ab&d?9LkPq6*63Ju
z9xf1{UI^gRj6X;PYDLnMwZqZ>x0=$%ZoeZ-KIAs%E@)XoLhJbxWL7^%segNuj?jcs
zm&)3EGHEmHQm87wA_PqdW{SWw@-SB-Nthgb$u6=H>VrB;ya*wkfrigT6qo9=F<YHY
z8XK$8IjvJQ8j<p1D+W7m4txn>%k9X(Fz~?Kg8vK!kwFuPGXC5&UPyf<Pe+>3%g9L5
z;pQC~RkaFCeFUe@40sJ-8rx!!#1q6ST2Z1`EA9dBZZK-xk;a3z4H99G#SyQiw{bC`
zq!Ir~`Z33Uk~Bg)h)*1QTsu!{8Y9$^f<r68BBvw<8E+0>*JG*sb}c-Ixuyr{+F0?|
zUjxUrA0CJ0LRQMCmQbv#i)9ddfF(U%51;wcp>Xfzn`F7#b~=Q|M7dA5MsG;y_F~sa
zZL)wm{LDJLjqkqB;Z$|oZF=>~;#aJSkA!?gMt1h&@r}~6yXDo4S~dDXA?CA6Hn#qL
zw5!{jX;d@Sdtjxg;xEt8S9O|estWk#vRgU;_V&#WeXtPX4Q2$O#Zvy4#fAc(SjysA
za|<)rEcQ;T<`COsP*RWSbo`5SeBukR>^a^&Kr>ypqZD@=Ozm~Qu|8&+e_Q#^RIXir
zIxcSaNDjW%R#~Mk)V_7DQ98n5K>+?*-`hpY>~ctfOemUAsV!Ku_!4$igQ+@VU&R<Q
z7ykLY8IOqCx4DNbpOe$_HCTRMBpyrW^I3Ysb`w613Jyfz6Ej}vQjj3Yoo0ooU2B!0
zH~9;V`TbLU_hvN)P{knf|7`x-qs?zeQj7oenQwlNaql4<e$`1S#}8vFJnucn04<7y
z{>>d3sT?Ow+(>V_=#}HZ_E8@a*T*HRf(d$-z;F*%<MeHOUs>Q$mSeyA0s9qo@axB^
zJDOD5{GxAziQVC`7~zN@ws_1*Y?nylzJg8|9FYXZ2;u*Oy%F(FR_6o@fy-ErP8#m{
zqxL6KgXR!{{8Zk6zy54E3)0P=5>M%j@c&U{TB{2(0qY@0|L7s$e==Lb5zEwZp`bFl
zJS_lt_dl7J|197?68Qfo0T)Pu0rrQe_D5qYNBbr!nb$t>ft(v;lD?cWXkde-IpBE)
zfYdH#jAc&+1E%BG>IsF!uY_SD#8%w29*f?#;9pkWee3x^ehdDpawLKdJ^)e-OU|93
z9z~9&Px(A2zpoN(Yl;x|gjc`p=?VHV<_91MyYrjHjfq1-A?VNAz!>@%GV~YIsWFqf
zLxRD}+6e@7AK+(@;J3{y89=A?`Z*cQ)cvuYVaXXe_;AvZ#N2A|=K6je<&Re*sKI;Z
z_xYK!tb)sVj!!L)f(^q7D*}EO2lcV1s!Iam<l~t_Krevw5y<IrFs1tp-h4*?i2m_v
z4l;Oe^OFpX!Fup=*EjHERqr67!VziI1EP<O7{^%pi)3Upzoalh5V(Duq9H{D-kg)*
zPU`WhxhQz=B>qo$Y>eP#I|IX<|7`m|Z~H$={{Lm$L623}|L4CiAjv{bFcO&yx)X9=
zZ8817joQ&KOu<kK#!8WuNW~@7XxMhGrDme1{>U`YKdg|kpn^?O5r(k7*nkzh;+-=y
z5BS^mCwK%$yX*bGH>0>XJ$p1~J(Z{6iT;$W$M`z6{U<vAM*>93eyJP2_dWxS>*)h!
zZ<E(gk`E(s<oM~P@m=rxnM`^%(sO3q@cVe6P3+T~@q~km|FUWO%iH#aZpoF>j?}f0
zWf!S$MJ|0*(~EeUDj^$cu*G;ou|vyVM6vg;%u*=TmYjVi?1V_3pga0wRtP@RUqw$c
z-lOdIw>>VG)D-XG>xS0$x8*!y7k9NPq!vQL)}_n5|NTHXCcyjF#IHo}hqCwbLou<S
z<A-6|YoXE!)ED$=P0R2bj1;f}WX-DHSMHh0UiFC+`>d}rEpdYGR!^9W2f{7a!WY{l
zTkjCoN)%1jQ^wZUuV4{yB>nBGB=tq(^G=S}6ch?$TJn<k*}Ucc=TfN=ge%gSq~ke2
zjQ7xv0!eMNj+u5Bq#M|1qtB$2_}NS5G@)QG8!EB>z}d1EGd_&O7r#h0&si3&H(eUl
zd@UCdT;4*})OUQtja>g0h%3JQv;(;~8Ne*gvj1AR^Kyt!x<*3tLz+=oC~rCr;vIKt
z8dAQ~GD8&jr<U>B>V|&j(7NU$p#H)B3>85*?C<5!5OlFkp0n&heA8vu|IkFcNBqX4
zN+fzcW_+Z&-)=`^Qz|pdOrtHewVl|kv%ef{Gxb>iGC(igXaB-zs;7RMtjT%Hc=)YH
zgGG5bru0d=mrs9Q&A>Se+RHRoDXd-o0|#)R*V!S(kVU_qhjX8-Q(DCLy6M`#8M3tA
zf2gKxyeYtzB`{SGqYLIHI3}A}liS7QKRl-CMR-xBKv_%E&s5~l`neO6_nWj$y_jaW
zH&^fkHlM&<pJwHmAJ=-N1;=_HmJN+FixG`%yDZGnpDSP1hLQ2O2>$JfEc*_VSSPV6
z^C8yH?q5T=JReLN9v=K9<0)eCZ|vCL@~!(ch>2Z0&T*qa_Jn2r)ZhYiUkO)oydS*3
zIW|{m$5AzxiV(R~==k)wLhtV5(X8F;O0Mr#D^5Qh9(p#nEX?M3&Dh;t*<Wa}mUMgF
zT<xRJ_rr}`pavW5;Gk_erq;e2IY~k)Eslu*-!!WbiBNDHEtS`%@fjO2<q<hDH<%FG
zLZW&z?|tqm_77?4o9Z4IXmI&8UY?UOt(Udwgz<2e#}0Cxn!Y*pjn9#6Q|pXrc|k#>
zXP*qT)Asz;DDQi(wnR?VuiPu%XMdS%jSQ*L8;oCaDwd+fjXGs)P$b>uP#tc}C_z|l
z7rI^d`*Ck_Shj!iY4gw2ams@zOvJc6`Sq>GBGT*h@fkHmpv$zA4Mf#SiHfaDO`&Ca
zKFb%uB|4SD=_R(~i|!z`R#IZR$I^sP_>l?IEi2FSbX10iSDv(CFYl#&Wzu9%39KnH
z7IU)gBnjp_-8b2DQwpC~Uvo&?hB&4_)NmMv^EV%TctHYw(RYL%Ym{PdPO>Gf@|hRj
zh^K72*u49%>)cXTX7tjfD~(;Qwg_SVNB;n42K!ZLcoydw>Y!&@@?Bz#A|*5k9x3p!
z?M4O_$uaFTUHgNa(5)w(m&yws>O;FQ$=Q6IWzxC#zs;MEzVEKJ=DTYigWurz#M9<U
zmCCfM--RDyTynJrx37|+|7YCs8~%vqH*hqV`FPCvGGpX^vUkkA#SQj&h7hzuvG%T7
zHC$WGtDtaVpO^J-SRZtu$)V}4-s;DmswMr!{%q-+MMH~GRXQs6vT%)GULHQvY|N(a
zxUpQO##3G`ADGus<Wb_aeUjJu9ApMVGE4N&iYq$<&>wi9^yT^P9+M3e>z05%@5R{=
z{??{1-InfQY0PTa2pOBr4>|qsuIoQ}+(}TD{KhA49fMdggU<!;TaAKSHznVHM-h+u
zxFOppGQjLc=RHcb<JHWv<6M{`7cRx`3#X17(B_+}#o1@qkBfm$`LY+527j3Sg)3CO
z$+%ThxlUHTFTJo#j3-=iuoA_zi%|iG{BG#}ZV}~NuoSO(+;#Z<NQ>}rYk~GmhKTB=
z{JI<1@v6MBnDZV+|HBcRE^Hr~W=2aYOLG&khqj>kWxs^M{{6(m{@UWG*BKAPn=p2Y
zo7-@KvCNz9N-gB@Q>GRvURjNtDP+<KHtbL1(kcb7Noc%^zP&GDwZ*b!VC7Fkdvu7x
z$be&vrs|u#$?M5UvY0$LJ^3xd_FqKV4eAWUgx^!#!q~TXe%ayv40-)d7w6hz=I>{F
z#mI3=%-v<ZAvdjG&Ym)0t408ubZq1lQ%g1<^~LH2-(ChHS1|>Fq#eqZ)uvEkgl_o9
z^7&vn1#Ch8LZvl)lB-?_!jI6~)>qw3_1q^hC0A!++t8WM7CGyNZiYuhtku^oni(^c
zakL(ZjtSgCR%bma5{OvY$Ip2#j)bU&nccsh;}>ef-z|>G!~cZ+6GUP2p?lt9`!MK^
z%cozOb^F^CY_g|EyT{R5;I<U5?q{KrHnDcm-k-^`lL1M6z4@fSQ<Xg4g+%;uN7BJj
zWJ$File)mkb|aG|W(1q(k)&aNZik7FAmz;)MBx)IZC#eMPrNy?!GzsPM0P+}@?u5p
zLjp>63C3G}SC=t(Q3<Y_GM%$!5X?>8q%KmxYI^1M;2tjuWN_;!1;^JIUbuMo`z{8*
z_`GG!K1tkOCLC01_symSlfYa5RywtXB!d&9@M<aa!81$OV(S5B;aZnX&bPEDsUL;i
zhc(NdSxSK;=e_cQK+&IYv)~zsMKW5LijCm#+{<hg4)2-eP7I+)fXLAC_4gDF(P0%5
zg^-_}RcLtLSyiafS-OX1AulBpE^mS><wE@_OO#W!>$AIh1;jgLA;{Q+5-G5G^d5rh
z8v1fyQ1S;SRn7~T@;F%B(Xf7->})Ee{*6r^no9Ly_#i5A(&({k33I$2Fb<Czi;`|v
zTm6!Ua?pEp`yq%ldF+z#T7LoH;Pvt)o$ZEwBa{4-j1^Y=qsOp1-sC77*qqczKqz>0
z=sko~#V%B!fzNiYRrOx2a5hU?y*7+kkYPl~vn6fneLnsOUTSChL`gsc27ZnrAhS2B
zxFJWJClR5m-AG@=Z$RjH_2cH$ZLGZG?U!aF3c@8scB$pT<lJ=nDQx{69(#&LEMSYB
zjQ=QHx7QzGJwf%sxx-`XZ#B&`d1NiuxW#S9fLE^_KiyPVvwEqkd7(czA8G#~*_K$J
zC)%}Uo?LR88!NcPdS&jp*>mosUXQ3mAA^wRew7UI*_X3n2-7Su8_@>KNj}reL}KmB
z$WusvY%r_v5+di0Pu=geZw9Y*e-j?DK9*h;I3Fp>XM96ovCVWVSud3(V@hKZQ_hc$
zQgbqn>QFwmDajcvIkGO=Gj|?`)n@F2&Ip-;Um(HPPrn_i7e3M(iBt+$ueqQ=KD@e1
zC3!5wfnZ@Pux+d+aYN{DFp0fkrV@@j9zo6B<)N2F*Ii7-Zk#8xia5W97Fnn0YlaWs
zUtsINK`<U1Ty%;gSQtwcb&`rB?bt-WV;ydiwaPJZHy*S;Q-Fod-riC+ayD#lJ~zn2
z2Ky&OX<QWvPG*^YR3=Z@CCHM$a}HsC5lp86&rtU3m8AzWEYQPDe_Zl%mSY#2`F!r6
zcFf(eO(r+_jOVuIVwlR*nYq*G=a)hwM?`FDzCv7JtGUw=QK=(=RPJ?qNWRgzw4($W
zu_jt_9uYwfOeQ=F+;sKUYnP--W(IRg!1j4-#e6X#7MoY7c=^8Ure$=Yg;UZ1Wop1T
z9ZT=ma|ubiQTKOusnhG-oS3@<1~$QF#-=n!+O4iobe;m<Q`hi)nCm@-+m`}HkdujR
z_K8M}DlM-awecz%b?wmiQ$IwrVD$|z@mMcw{%B^PUAcNCxM9Q3ZyvUmeGa1d)q?TZ
zz#kL-$jf%fv5oRtcIT575U$U2#L!zesD(K^-w~p#8)DuRng5>Xm=Bqq#c<IzMAz4`
zUo0KFSjSAs5+}+2f0?TO9t_J*euJTo3Jg2Wv>?*Z9_3YCaBhXdMUO54K3l^(g!aut
zxj1|+p&Y(|d!ZYa!T;6XSw>aWb$$OFBt#k{rMpx@KpK%!5kwFHi9;Paq`SKj5ROU-
z5)uN3&O?W!ga@U&K~O;QS#Vv?b$j2>_xH<tkFf`XF<`H~_S~!If6m`8YTMoJx5gC~
z#MTKzYWz@^tiEF2LPezl5etrrZ}l*h)q&enZ{I5`3W#;5^aMRdeuoa{&IFMpep5Xd
zVrSU*{DviCCX@xOd*sQNdKc}}v~y;8%ypBnBn>*{T2jd(@Y5}$ER9a-1H{kuTFNNQ
za6Ew}S8+;UcsdfUl9-(4qtUs#H>=ao*zD7M+%sYwvCA98=djveLYPMaNrA2uC}eDI
z?GQJ0UioTHnS#EFT->R$=i~<8Mu9DE2HMK7ez#WjoO`Q+bQs$uQ9H|Hb-XAuvQjSm
zVM_a~9mMuff%m1b<9*G(I<jpVKQ&cpW-U}GLhNL6JQb5w>t~|k0g$k|Lp1^}PM8T?
z-h_{7??amkyG}mM5qL1RbJ2hMvb@Ee`&j8EJu;4Jr}_O{AqJL=W{{f{$=|&sA<d2r
zX+Zs6oFpsx9C6jPvsx=9f*_qQK(tuhxS@ok(Z|q(ga>Ha*$-?B`Yeo`tqord9)*w|
zxnwm?8Qa7*+HKpiK_bkDS*!U1Z7L+u;@|1?Ul7A%Ok$HGwzaDKPEg7pRU6*^jKWfO
zgY_5_?Mp--t-0DfHI=s0${-6@LE`egwQ#b4-sGiMV`<x|VxXSb!7!O<cvf2%BQa%X
zv~W}K)TRexuZVK$1vxn-JG)K6>yb|WmRRf_OcqFq3S?b@xfRthYFZ<J)uWtpVcIE{
z_-QJBJKf9-`g~W?EW`)Xo&XYw-dA!VcGB(!*mrtT9?(>}73kbozatQFmI}PGg+tZL
zH0xGOcT1%*PoiDd5L}%%+}s8)6w>c64(El?7VJ7^;|;%1-x-u$je{|I5Q@A_C!bvG
zZZR|n%MR2Vzb4(JqIE-Aa~6P8OLC?()ga(X@Xf$P^~Ossg1uuoZgaH|4yk^9Xbe6@
zRllcqwzbx;3SnfJms>ss#9kyM0$+GIS=aT&!sf!Iqn4CItnn;g*Udd-QoyG*5IV*I
zo3%cuYHDHPiP|2KTWaFh8Q{%865PhkzedI1d{yL2XL0o*9rY^`2ppT^Da870>xB(_
zuqTX_o^HBpCIqtLHLzHw)_{X}%%Hf`da|Yd=}n>jNxZJ^Z@9b1gf$3#3d*y1B0P^6
zakmYdI5B^;t_#$>=!zu@D9pg?#O?fR_7^?s92rf2f!{_HKQd2dh^aRQ&Zyf<3+Gm^
z1CM*ke&+@Td`1~XgH`>xVL#0yU_7a-&tF-hO1j~~)lT%+h8X@HW5#ZAwRm+gcZE<K
z+nn!7f_Vwi!%Q3hP|<QLm9z8IFF6qV_btZVK%l)vlnBw<S^7Mt5)epX{U!34)SVbU
zLh%)~M6*B6Q4svb{%f^)hur34#!;Zky;frNQX+Hn^LA)B=J9pTdPWWRjBxtk_ToIP
z5zMN%lFkr@1r2QLGHfF5t1E@;Si_d(G+W|B?(I4e8rL8bcn5M}v^T%vjQZLFV6hm%
zfPy64WJA+pD|vJoS*4q8qdME2`#UZ_X6Qzus0uRTCu~*(3K_zWRk@lX*7+vf4Ff*5
zmL#P)P%?Qc$5C_8y``3Xy8jv5!g1dH=&r)=K~dLh;lR<{Q5HKLA1tU(-g^fvDP(w$
zZ^N&2aAN&ZtMmIpt3$JK$FLyQgRI$t0HMBGN`KAjBdlr~{?ii|aoVDdGiK3j!!Lo^
ztWK;OXmpb%#&Ewf)Zq*9XFb8)z22LYmN+=iEJa=)N$NZFFvrHrOk8eM*dTm5k!mk6
z(W|Zc(o=r%e$Z&!uUWW!xiw9jc!q*kbB5nqLjB~GwR(fpPH@sU(=A+P39!<_HfCFY
z1THu)!oP53?1htzYkU(KP-EBp#HG#?bTp}GF=W_uFg*HjHMkzkUpwX_zovM8*}&K_
z9OEw<9m<*%o>Y>98QFu}HMScV&gk4Gb-MM)z&mSKRdT@Rp>L?W_WUl^uY)evQ9BHX
zmEfQ`bK9HDAr87TPlZin9O78>7#X{`dZnA42crC%j|Dl0BpU4<v%<Kob5%=M+Wt)8
zbn^50rIh2wU+s%s*-l4;o7>(A;#kk_Z7{k?S3lAG{l-|Se5G%PBSTk4Lnkk@3q-*W
z!3IxK<?%_I7%qD&1wSVWq@b$z8;+e~D6G(R`tFL`Z;fM=Z92x;oG)Kh)iiP-D_YXj
zH>9{sl0qpECsY5oRUzY1dB!ZoF$q%P$#wOGC;q-iV{Q#TGau~s6{1<I*&sOt9xCvL
z!}>QCoa|bd2ql@^51$*J=LLV{`PIc)U!1Y)Z$`&!B38ArsJ5uY>YZCXY4!Z+4L|SV
zZ7AjI_+|pA-w>5@1+&DCvO&#hX;eFM-?NShva!TLRKdVmHRjvn^I5s+?eev{5&u0A
zx%P@&f|!mQq|8T;Au!ga2f+!LQ<;1bu|p?rfj=~{L~>PQPx;o$*m^t%!!(J^k&Vf(
zPf042(7JJYP}rB|Tz8klt#ZE`os;;Nv-_pjwOMr-zcwSAubW4sym~{8ew6TI<*nCW
zks>%wajyD3_5uFEbt^l@6g2uj2kBeh;Sk=5;t<M?uB5k-3K;HX-XHV7VT*rO?)z&#
z#cHL*!hHHj1W)c7hOQXPr=2YvB3pZSd=K=lH`y7b*5#~32H9?!P@T!8huU75l^W#A
zaPVVr+3OTN8uuuBg6K!>U8^rc+bj(=`!6RkBt0rEghP}S+qx5?)ulF;$m}C=DN}x6
z8#lQX3aGWbH`8#k4YfTm_@HR~t_O!KLRcheWa`yZ5Oi>MlP5h>q*Yc?^r;(a5&n@$
zxDEngSH8$`yZk13^*HAX9<dh9wlMk$!9jEdKfW->{b0L*NoVVYZv<`pOK|gr4&HMf
z>o=n07>tNjr-TC&sdV2Jv4%k5v9gTP_GvYV=IR)%(0l3DW3p(URQ^_s7aKq;2D^8g
zV!s7>T|(KEyu=pEcfR9UzN8d|)_mjD@UzF?h9B(L{oBS!7mmrd9)YmA`<f4m{Dkg1
zo6?l*Wm+do_1cgYcy1$O-h{nrB)S;z8=PB&x*N9Av)CkF!R~G8*rmay>`NJ)IRDDv
zL8jt%Es)Co8k&!EvvC{;=j9;9^E{7a*j}I9WW4xfVCSXR-FF5P;&Nrn#X!D!NymJ`
zPA#iai#I%`d8kLhJ9+oxUii12^;3TD3RQ=VvryQ<wSZJ>p08c@vtjW;{10hU+5?vL
zyn>y)tt@)0ejuwjx>8)Ia}-OAd0kI8#TbZb(GC({q-b1@RwS63Qfj@{+7S-%yH~Rm
z_nvi99;EKAJuy{0Q%Fp{!Yj~jq>j0OXTeY)yeP|AsGtJV#5{IparhZ-eHNOkFHI}*
zB=?K+H_`?RG9Q!+DUuZp!<~IY6T8Btdpdu9E*O9%omvlR*TIZB%$q5tgkZRM1x-p|
zjH!Zs<6CRJ^VG9cFxEMhs$NlW9;|6!anWY`QnC--?znBaZ+f(2`;CXl`iWs=gddir
z-ge#7@1aIyd{MT>tz4Mam?4wiL%0nU_pIFWCL!JvC&|~KehU0Ut4@yRCQ^o5cNW6m
zcnk{`rdo*EzpC)Ooi<cR>A8w6;vjh~XSI$KDe3mAC8i%eNo9lw-IkQ_%;eR!=?xUR
zjUL4D6_JGBLQYK0*%2bGT#|i=KcZ);SEVLi#<|?I>WNy8?x6J~hUxeoE`OIG8CyC#
z{8>v<GA;gWnhWw&Uch^}Iishz=&iHlQ!T{2VYD_JDma&?%EzdowCU@3B>JJ{Y)i0C
zn0&b;2eNt3U;Wy;|9dx_drx25<&fb2-R;~6o<LF9H>@y}QCC?sY9@^d#5z;iZ;xy>
zWgM3JKpM|T_*%Tn-t-4<GAUf+D+=quqKoo`Ee&pnCODy&r&7OHX!mawNw3eBfHZa5
zD`6QdF<keKmH3&P+ddSK-dv(Nl(?O7#-v&;BD?s;`NPX}+@reRM08KAu`8GMV*4`l
zRC!GTJ{=k@>r*-3LR=~7R#?VO<t*V;TN0>%KJsY};aOq5p~7j!l4Nikq6ai_GZBPB
zLDw<10;3IP{IS7?SO^`HqXU`IN8h5~(i_Sbfo65WzbZN67qgb0`{R~GcRIEUw$-_g
zVVV94dqvIFa^z>$(HCx+4Pvt5LQQgpA5SIxF5g~>T<34i@fi15>d0O$54+)YT<}X|
zrzQCDPRaa_?X_(z-_EOA6EVNo5m<b|&w2L_2!?`(Wa>rY@vfETfi1Y30ZGXV{xhkP
z8LH@NTrE6I$~t|Vi#U~@|A?XSzYcM`H+V1gv<LQEDH{Jhiw%ZYQSQh7`Dh}kTU8@g
z<ia2Q+QV93OnnkhqN0<N!8v<ftNM}9m!qO|m?eOqQh*FnjZOD^))H^`7tiZ86@nNh
zn8|gP7^<RJ{HEd;MU-ESG^XicRbeq*(UwQU7OV_{^eZyWIbPFgQC#Y1hGKP>qwk5&
zu@bMCQq|f&>Y1Qv*&=zXYiRNn^RTd~$@%uCb6#v&xb8t8^X4<?6uGk2>2HZub&=M#
zZVPNw^~-3dE0)EIq4sW{bu0;5jP!2NcCu>|q%&l>Zpe3zgQ@Enc4uvJsKN9{;e*!=
z95MH=qMDp;R+!iSd?qsYL`w$2A3@++#17-8#sldCFiR<^rQY0;)!!V`z5USf8xBkZ
zaBUe7HBcc7cCl33;z#>?^W9l@a=~$R*(*Pp*1A~4+pfN8WmapDZzdz=HLCSxNqc`Y
zkt@~0;nrKLJ`ROVrjZTbZQ|=hQp2iEl<f3UOiR<?l&Vl8^1QEq>~73arg=DCIfMOi
zU8JKv*5xO<(AfPEtL0A)Sz_})y{j-iq%ChY)BEbfuUygXpSX^gPI-6*da_IugIs*j
z8;*2<X3jVF?wOzaz!LB<tlYNlod3ZbsAlb?0>omRRP9b{jBsZIN1r}QhDY0q1#9W0
z^~)*M-CjmZM(e864~86XhkYxe2*4JkL)1KZw3Jqj?SgQV(y}N2%q~4%j;tk&Y~Xu}
z&mV9wUp~i{rLdFw1tqj{y|qfBx^3^)-46*HrE<B$2@OQWt<$tUb&qfjQ}$rOH`5Lv
zM*AYaw&rV3uvCX`-Hld^mD>$$QcI*{f00|T9N2WM1;wC75`?82*idL_AP7t-q&3Cz
zCAOSd;(F@emUBYzaeXdz8(x<Sp4*VQ)~zNjd*2W``~l74d$aUITqwCbHPMX>&nKk6
zjOG*RW8)#X)v6cDr)bjuR5o3Nmm-h3DMW;~_8yLG6_j}UIKpuJpP=xuC9+ULoL7LL
zVCzz~j!p1?0-#sFV+!I`H%??LM-ju8i9Frh=uvZuDI`pU0h3lSQmAv_Ms~P)8F44`
ze*(f{r`ge2s$0XF6pg13(XtBHaq%8MdS1vZA~C4NJ*D}%^MeV9odk-{i&%i<%UFN}
zAOY*Tcx#}k+YxT|p4+mK{w%c^$zV%~#~mxdK`td(+@25Rl!}zW@c%12g2aDT`s(iY
zwlZLl!<frp3cvph!DQ^RBm1m?!8#*;>8*zR6>RZGJ_N?5$GpHD4j2R%HG_uzXUzgm
zgiEb~I;D~2Qtbx<_WtyUgY*plZz~`!)5~|^ypNNq#C-xWxxqws>FXaM$$%))J<5^0
z5Yn*DD1%@N1YYcAyNUsBrY@3=dag@_4mtEWmW)okiQz>&?mrh$WpYBicH&axFO*3C
zTuS6XsiYBaB6d;eKNlceU57}yDfKd6Rs_`L^2&|?a)OUa0(Aen9~86$g<gX&NU*}=
z|9G4q=2Ll~RCvaUApEx?0Cot=kBFWWd6~B5pC{&D$fUSmDTn@B5neLjC2@$twZX^y
zr^bx_8W)xze~@$kj~dHhNCC0aOkijm?}bVJxnw?GsF)&HhnfB?0?Gg^w$psyh~nRD
z#P|%@=&Agx)qmX&3i2ZYMI6pB@kjns-HTu1xPguGnY*3-Wn*CM|C;%aFZEwD|JgSF
z+nWFAg8v;f{ut^0&!AzSfCAXnY{0T{=oej90+_$W%I~(0doIdv!GszLriF!_D>Iva
z4_=J9xV(TDR9v$}lNt?(Vh;de%2Di5y)+oRBEg8J8}vE*?|(%RVSdqr&`Nn}RoYH+
zo29wu&;mh<kDCc%>E-#^X_@P`J{RBy<@|6H@CHxG-bR{r|9z_rnUWcl&!mgk;(KKg
zt(cRN)b9ndnyuP(uTxEp0?o%#M=LC%i;3zc;OULG@C3K%!L)FI&bedy%`E4-nQVB4
z-L$@Du`U<DgvJ5z8>^GeM`qP@DZSUc+H)xr<7VkM0hGv5qBHPfbR{P8ccOp*LgFic
z&a+fz@I5Kj$bZU4Z&D-NpY&Xx%fRRNhwHprPX;g+zC{Tbi#`OCjpC0^-#w?|ck{Um
zD$2^ZcH?F8w`y0Iwd-7p#6`N9M*;G)3oxJQjP<;KjRqV9`d@s4S8Q?pHuzXFWp)Ig
z_5?wd(<K@}89%}tzDNm*T=fHFuAaAcjnw;<jpt{_Z-A(VisB4RMqNRu04zKwy)xTL
zG0`7wWFKUMuRU$>a2^L_c-ct5`2O#9O*f@ae<sl0wU>FA$jMYS;SdR!jOqYNoA7|}
zO#D|Xk#yJLk9n+gzGtHAg~)89_K2sS^d%+dj|_Tu?5HO7%gy88d93B{0>1aU)p9R_
zrE3_#MHhb#t0n^q-UC1rW3!v8pQSuM|1{egYL&on;<a8>WdcB*NBD5{M@ihj&PxTP
zO70#-@DyJK8GEic?|N=GaIAm*l+2<SW6th6>6|>_bG*s_l^piO0+8~=YFdKvKC{G1
z%q5n?DBJ)D?})Z1M9qgx*lwyYJ^1$|0CEBF97u6E@$yTZs~tG-_WC)b_yM5dfzIcF
zItCNA0r8V0J*9^CN9ntRGm#Rx+z9|_N=WC}%U`?EBPIB(yCb!q+U#BCb`cyh*Vqvm
z*2&;o-=9nh(VvJpxL8bx`UCLYs#t3|+tMD|IF?`%#5j!A^nRln9Vf~sEF%A!Z7B6j
zQ8c?v?sc}9>lUrp(MlHGFxS&;C8h9y6bp&7!}%jIHZj>A31JNYk?nK>SW&X4k%n}`
zqO(C{w4eAFOMGK)i1hOlI1(>DV`hSTh)g=EuK=DFy9at15FTY8hG~E0<|lTcmj)E6
zw{fJ;#9{>2ak=8jC=|mP);&BMjlG{6@oadUDXC(-N^9RY$RE~*NSvSSX)fk$2uk|8
zU5BKxEE9lk^Hf=;k6Gszr`S{y&vQb_eY9~VV1|Vze!a0yD;ABwg$(~zkf`XoldVvF
z-*QB{kW3%&lH+KgvDyJ&^*#G)8QK&9yYM~$PZ8>Q{b(P1iC>+kX5D#b!~F@%_z_Ko
z0*}*js%yxtx(xvyf(5nleeoP<v*YQnuj{_cX`aQ-k82zj3+I<kz(eVVQTGyuZ>5bC
z%H;tAX96Irc(%lj)+)di-+|KrkZ&3JCzI}1hVSKXDiiVy^L(l&P3m`@_NIJo#=^-+
zqfFhK`(`t|HVx-X0$9PRrio+8bZp>Yp?q)innBKto9#faEyX<nnBg%_qyQc+mt#vw
z7l7I0Ee@nluLX6GLI4N#J^6CS4Lf`Qzv_Fqyxnw8D7k~WGWJ`_`$%hf!{GHTJ(`~6
z+vzc(F(L|8o0)*|t4XsmD2`PJk>NP6Sx&>7J!AI**knx_Vk%%Mfm~W?q+r)bmLk!*
z{>mZfp4(%RjIa$Pn{j7MMZdhyy-r%w@as1LdAmGj8Z`a*WYMz@Ui>f3*3c`sZOkYk
z$l51X5iB%c%h=m}M1JXj#_dKgU_OlFC!<#_r3XG{4%X>`TItYx?c6(+M7lCM5I{Db
zNdt`J<}E^tRM#nYhiUJFa7$ui_AANOA4$F}k1EAg_C-aVv{M>fJ!66?s-TzQ6eHIw
zQC}V3TY-cuvO+Bau3Ne`1=ps(f(n1i>(ClWtQpBy=Xf+d1lYxcm6hziXvyo9O;<mv
zGlFiK2xze{T=MVYcp74ZrPVfWx%K!{&`fjeCUPF%Se<V2)nLzg{FdF4^##Lo3`R>9
zzCseeKFPIr2`XX_s?-i`h;pO=Jg`pxg<q|R0LteSP}IJLChgc4ij?A&uC+zJ$&Uj}
z())>baUWqjuK={-&r$%@{Okw4($YVV(-R=T#OcW5ujf@5x4{e>aW-dwZGVDCECN=0
ztWMnpO;4}t4Dc!)_WDF#?4QNE-_tSu{5lEXdc#R?zK7E}>6quD=+91uLSU)D@ulxq
z#@x2|MuyiEH9tj`;l1SBt8SX^{(CX`MHa_eaO_b`Jm;v}iK}s_oeNpUR|8+kY>FiL
z4}s-^Z|9z2G}p4fKK-Cm0&iD~Y@l@@_2C?+hM@bzO8GkZyrZ~Qdoir79(#G|d#rvC
zt6f_NHIl6B{W1^K&d`Ia6ZWkHW77pRx(@Yfn#kT1AwJ)|hjMpw`~wH1jtgT#5g|%-
z2L`^UqXHsNBfX7te)NlPYaKi#<a<9iZU!+bA}{&gUJzWNW`QOGapbUc68G6)%8dsh
z-p{Xy)k@^1y6B*PbTI(9eMOJ=X2sW#KKeYYj`$(tvn6~P<vS9FEHY$_H=^F1JUs5T
zKgur~(8#)9D=YbXA>n@2lt)3Od;5EBWF8d(&4$6~Rc|+J>!v6O-^(BQUfZGc9xIt~
zG^eH9*0h}?NcFDA7&7jDgYF%&b>dsqEI}Q(vfYi0L|%<YYdpE{SB$}_M_tKt`I_eE
z(QCnjEAt{}%@ku^&$E2^hv#sCDO_=9?C5+WOU7f#K{5|ONJO5caXfx=od-co(fg+9
zn4Cv5sNM5m(zVDDL6DN6ZujL0GThK&Mo|J}llj3;Q0Usl%L<YUg|tkxTKp)kF#`zZ
zBDbCo<)Ryh;V$jj7E*%7i-}jU{K&}Z!IlY1{D2k5XIN@j^0*hw+HBdqma<w(rbL!H
zq9d92aSGlCOIj7eC?O>Y?MhMFGa;%Kf!Rrf^=f&WU$b8IfOoCg#LLfjFTiI<mgvdf
zH>RQEUHHtMm+#2aC=mQ*x-?F;W_39EO}n|JU`64pId|}elvg>)q}S~<5D9Jd_cbvx
zY$t0q#fXonPVA3x;tkj0lrs6?y;SS8b9eP(jeDYV$p`ZHZp1?T%;*z{7WMDIYhE&I
zL;axF9f?MqOM9}Ea|+!MxL6BVf^Bd^!7m+!(4JRHU<(?gdDQU%(uFn92r%ZFi^Nuz
zqf)19<uZ@PDMO+jT;+ZX$kB31_(@^;T1nf>d3$&LhF+>?8XBgkqm;zvU~N<f(fIp4
z)GRwH%~MPv;?=#fWl4^3i#b}c?GPM=E+Z@!2<#}(VzCQS?1H3OR2LI?*rh-9P8b@|
z^KKKOdxB6qahpi{h6R}l2c25_=W@EQ4i^acC*k{Rr0_c4tkO&+BoZwJ8Jkf0p$K8-
z0@cNN=?xeA)pw&p@yq<*si-$>lzGV*b&cU3NT8CjzF{m<6w&KiIL&9*!wXZpLvicv
z`cS_QzyKrkNS$La7_Z?H{iSN1`^DtRCVSj#e$=hmKg1=^;BR$Dedr9-WQb&hgasn7
zPVBlpHTkeJA8BPL;fMs;qK&7CsywuVl|WW23QxWq9PJ{m5`)%lFmvRy_>HTmcQ*6h
z*6ex+L7U->zMmr&E7Lv=>}!&QDWDdx8h;>b>6f1u>sqMF`o=0kP*-AWp2J&kaM{0i
zpeKEeV7JnCFA(l(i~KkxW>C0=a$_GyJ=?5k#|=)%99z;9Na5hnyeE>%`pryCI?9B{
z%+oR@AH{Bzz@bXWMMTBaD>jopsUj(kF#}M~72KO(wTQMWYFbu+`e8eqo&07xTnI7e
zSB?@#$pQU`OQqTqw~PHg83te+BiA@`wO^QktW1M|2h2mj`|~k7=WSO_8uVSBapG7A
z3fnN}N|6g*|4+^CO8z39in1-}!H01IRjJ?EN%Y^ij%lvsBI2|TfS5-<q;nIj**ESu
z%!qFavMs#!W~b*+$awO=IR2vrzL7Uv5dw=C%Z_X>jb{?Ug;YFl(3dk4J7S-XPY=K5
zhfBsTNAb1^tUVl{&|_t@!Um0v#=X>DW&S#|wXlNXTJHR(nKAMW0ZuT>lsucd8Vk^=
z_1%?yW(%Z|u=MUA4y0S4x0<7i2s*$$e>{$W&aLo8Yc20F+z1Kpi{P>9A;`NW-m7W6
z8ASVBHtM=LI0F$qakEHA{DQM}$MmC;a&}eigfQ$?!($HBRVB$~=l!BB!D`hsbsz@f
zSA{o>&e)B?uaSqqH1H#Pl=Tg7MD}@>?6>hX%L>0!)qu3=2$FUOT)aR%8>~KacKhhn
zqg&w)12yh@OC_mZAPvKqvYy*)u;%0tK!tY7!a|skwHbhj3T(~#X!9OvQp~pk*6e*9
z+}U9+G?ollfn?Frz+8qK>+InAQ*l!8kjJShNuQs1`V7L+X3s;27;L3&ycHFt1jT(8
zj}2}!D$v8u?GDmRTnQqCtPlud#0{h8rt0*$-~gRodiX6k0V9<5{bm+1wIqS;j4Oo5
z$lG$?D@59VygS;#tN9KD)ZgJ-(zj$O%9~rDfMy7zOLxPI)`p%aVDl0yYI8!EyLUuz
z4-+6_84BomyB9WodE(hCheXr@Z4S#;YkV9~zgMjlIg_7>dg*3Iu?5E;z2+VA{Fspt
z5nNCTr(q;krVBiNR-kz^NafZphNo|ex@`6gZEwuG06hrfl;pt#na6g+f$YvvNW7Dy
zWWTBDS>VrE>YJ)*5=Ut%)a^Q06$r#F$}fR9te&|OX&i16f}ZIxOa>!7Qyj7lLq7pO
zYK3_gS4rVj)I7~@gs;j*DNC8?C*fyzlG{(pZ&!BE-|WJff+ms)R@h_F)hOQYZF6HT
z0S(rnIjbU;aZpO|$RA7&8*u4T;jxEF5Sb5_+~&Cih7N%sU*X#yzlgS}R8oBhMWO|<
zWPa9lv9QiO$8aHIP=7uyy%~5#CH0BAZQ+eF&-~I{!}A!#m=-h#!FO8T>3T${PLWH@
z1$kH8&OdR5ZrUwi9L1}tgQ$r}lk%>8R@G5UKZklSh(3TEO`o4F-(}ea15s#{qEPmE
z56>qP9oCUwE+h+yxtaGv;cu{2vD3kKyyezWKA_y-&91Qb_(N!Xa+f!x*H|?PQW86O
zciiGzgS?za2cLUsWRO-sds}<%%j0Wbi+*56#Za?TeTXFIgc!xoz9iid@dq^OU<Bm7
z6iVhn@-00lI*SQ7oE@aZH^NwlD0>@8G2!!lueG0T;l0;POTuyhUzp7Hj}efVMTz>4
z+p!7o*FjKSP1-*#zo0mFLr+!a2&hOpi+A>8DjyIn$u)Q$^xw(17|sdqN#dbw$F!oG
z5&LfOlfkniMG}MW?tJ-rY0YYhVQXQzVeM-9bX0ic09alJ^WoW!k*Ac+ka@=x?z5ES
zwr@9ssW=WVjb~F(y5}bst5v#1y7tT6>DX(7%9HP(2GQlH^V!NUv6igx2*Bt<skH~d
zN?hNKmU=myBdvtnj^kMCSS7yekpu>60Z)CbR5$Y59+V{L1QCMDzt^HsPd$0x9r=d6
z@^+ddck=1ki3&I&-NP*5bOK1)Olm=R(Cga<^uzs9Y42LBai&K3FbMbPHdtPnqiY;<
zg&AaN_*lxeSx9+q)<O7=A&jYj5^VW0pid})fJy=YqMu!0!M`P%&38oe2Zs$y3R|Oz
zTX=B~i}mV+KZ}iUMQhOC|L%u1p*4ZuIMXJBaS`~6H+AKk8T^d+RmDJ|T2{#Dc*FiT
zD`(Rf6+VR){?l;6;#K^4N3c7Lq2{NvX}h;f#d5DCI2a}qI&MpL!-|9|=B=k;5Z&4P
zp+Q5|BK-!BF;OxOdMHfTq*bUy4FIO|izoO&FXPRHJWpMsPuW$}jVj9=k~N#UN=?Qw
zHYeI^l+j#Ommm1am$tP@Ye`<keU2Y2PDqu4{1PBa3mzS8P|^%Ym(Vt3Q&7<NdN<n4
zNL|R7-B_%Ae-*AAZRDRK8lvi?&k_i9kUh?CRD>*A@i{e7PDSI+=(Z|fz`)j04Sj3;
z10>p;dOX$Vysw(OYDX(Le8OHVN$~CpnIoZT$f<=%S5M=e*C`Rop%;Uy664>=nY>8D
z*iqDe=Io*_LHmf(Ar7nbaJ3&`qxgbOXSh@n>}GE|(x*(L?qIY0R=Kqr1u+N6@~bXs
z`+co(^kRcMALB-WOx?(3T<AB-yOa>hvXg=#(m788$q2TXL=H%OukF`00Q>Jvg@JI+
zoZQW=6Z`1Wx^^ci+2j4qU)G9%;r#AZ7<XvRr{5rdbz8^9p%2%^SHlQXX5Pf5#lrYF
z1n*+QOgzox+#^+^lTk~(U2`}WDUUE^O%fM!;Q46G)r0*+BlNjOu4*FgBUuYBK}t3D
zyJe=IDb(l;)gqB{?xEdx2r5kLIr+2ZIggdq8uzhQIqTHbd{UkD0sb9fK~t*ok+7EY
zSS6LCLhBJ&?n|iA!PMx`TFEjC5oOOtbw{sOagx!hrJj}?S_bK-e*!pW^balRM7_Mc
zaSS!Qm4)|_w@BHp8Oalf$Lt2?zDXq(TKly3UOB#Hj`iq%ILmLaBfZH!18qd`tHGrM
z1ZKo42xSwMNGnQsH^$uHVUFXb$-#8XL>H~1>j8#}jk3y8=m9%a8nla@dG*hW?(oHT
zpBadeiWi;^DRw_$_y)T4JzPw_&wTA3BLiiqHithg6R;W!I?!AFnjcKs;C0N!sg}d5
z3u?Pu(e^_R7YUBNe$NCM+l&8uktXN2dk;#S5!(I!A14Q9%te$ay+uD|7zX2VI@IwW
zKM>f`1K&p*bL;#AuvdeBS#bj?Wd7H;Qmq2KGtO<RuArOcObeQYU$n2w>*PN8i}XN*
zC}`o})&9Qxt;s-h7nWuJqj>OIXP{m)>1OTk{@ri$r|;{ZBB19NewzWLk;*^Mx_Bbv
zcQ65!MaTWCZWopG^S$ufENk=Uf~WlJ0$Gi(L&g=l{y2XHMWCcX5hr_vDeC`Q-OGn|
z2?O_-HH*m~_k*%#7(fwc?wY-q4!3^{Vsf!Ks3Z~iS9LGS+93jmisuqMOCUAMrFH)L
zS@sy5Sbo2X_+M-ORmx!~I9%~fBX_(Em;a~I<Luzu*#s&7aX%>K{{u6h<5cfa?okn~
S{RZ0y$OCy5xnfyZ!2bZ3Yzip=

diff --git a/docs/figures/arch_3.png b/docs/figures/arch_3.png
deleted file mode 100644
index ba211fbfe5c8fe3f3215d8c2b763ee744c2961d5..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 61499
zcmeFZg;!NwyEjaWfC5TNmvl&XC@tOH2uODb5>kR7-7SrPl$3->r+{>KclSFNKF@RR
zbIv=y@%;hs9%H*VbFO*i?}|Cs+J01!lX!&k3<U-T=8=@7s1gheJQ55H+$u8o7v}ur
z!y*g}O1Z_WR|-<EUXdy|*uJr_GKGPW{1_YmKtWj(*Jtk?bE-egL(GUL9#6BM3QMnN
zo;{}{dx#+&?3-aEfunA~7#!^LNH>S321j@_1d%=jkzw7B0i}D-=x1Q}oe!B+;@#bv
zknQgDUaIRZ&)RIQ*(yH_jZ=w6%42$%2(s=t|BD(nW22>na}gi-L!V$l<oI~1H$nag
zFe<rgP2E9EFtWX_ea}5_j;_41#TrRNAHZPwIgks5#`_-HJN;PwNHqaNdZpN~8)bEX
zpf8fhB=S_W*32(D&&|Z|DK-9w#LADzN5YzjFmQ=Iv6S$_qpwbS3H4YvgSJ1G2Lv-?
zWqxXekN@;ZMzGsl$H0FwSTSvXK>9j5eYxpeCgx(<A=;F`Dqf1r4W*)WW7_m?&Hmc9
zOEO-rgGKlOkJutXU_aV%FLAc%WsgP-9u|M!%iHW5$~;LgioC@56-=%^&yu_*D{Bvh
zW_Id_4Lqzu6Dh$KI414^@)eY9GC1^7i$Bxel$bm(VP{mzmB+L*xph<~&&uPGz{6(m
z)3jK)W#c4fZ4iC-yhoq(&_4a+7UR@A@zL~C`sL2iH+iDx-KYJjUHYU(1LwUVkt^-E
zZXSK(qTSnV6T9Uai^mv=4Wu&?xdYt~<BR*mnU{MfF(&YH9d2Ic;=92yWB3TlU_AWl
zBk&D@0Ru(&yTL{)5h8KoqUjSu2WigeFttRVF3f}8HI`zAuU0S#`4a{72TWGwIAraK
z_`1)vUVTM3Pqj)Mq>6+4tpnrJ{RKaDmI)^R0*1ZnAUi6Bc1Dy3mVp$OE(2aQ<Dru;
zCd;d%3V6BZM}6by2p^C!P;uahC1LS};o^L-4H#))#0=QRk?5LjzM=o|`87{tj+EEz
zZilb)p!%oRKHjk(>^{mTA9MpWKT>{CU-JmMY{ZRMxe?f{embPgG%zS1j4}~u@Slh%
znIl#{rt_1|Krv@lLA`-X_3wYBo+-CZ_UFMP5!*~=7Px3b-mkC(&5njRy@59i$ZoJH
zXv~3*3!ZKiI*-+V1<gM{=l3LPL_cdOomWI4bwGw66z|2rS@cDqu$O#7E}}*jAc?OO
z#!gx+$>)H(Ar=@pfJYb@XlT^SB8R6hLm3#P`AIW?wq3>GEsJbIXhMC0U@!72U!0$6
zN0Jdy8KD&#Cw@JGwh!Yk;^wqwy_VUI#|A?4cHBt)7-pZ3+S(SC?0f9nYEo;&r+IX!
zOGvmvI+j!qa^1{4S<*rW+HvPy=g}7i7Wd{K?586;ATau2|4hb^z`#62HudLi!EYgI
zL2bcP^v#!dqC|Rn(Q4W%+j`Tgw#c{$WB(~zl9iGtnq0CVyDHm334Y^ALC~=rcuDk&
zFsaDVREwAB>;_L`*9z7Sm{65JocKT}9~Z?Hg&s8!NfI@uNP`6*LO>NBB|9NmkfW2q
zpW_*o*OlDkxO{O~v_ij<yu8=hl>4OMQiao*j3Zu4TtK`*j7W`?4v)r-#tvJJR)sD_
zWnB$VHJet84vGE^4G#S%t)42ON(((cjXUi@>|yLFt**jm9vQoC(t|JwZHX!QdQpOm
zrS(9k#{#i>qCzrz8J}tBX{Tr;Xqi<@R0NBsdxY_j&>mBymVLC38EiXeeQk{MF3w;y
zQ1em6ZtrD2t9F~LScX#mP`-oyN+_N3$ZHN+=g_wmn@>4DC4JP&pL|lK`!@bhpxLk)
zU*BpUQ;L_<DTTAfjHrN&SM8<X3Bl%<XGzZlpBaS_J;P-W<iJm|Nh(OnNit&(sx;SP
z(t25}R9vCOoY*bH$(E|w|D%&|P;KmVB<}6r$nJ0A;TL~C{^=UF_~V}SddPI+`9|F4
z_l@v%iw(rh;@?GM@3Q6pm~G^(+m6|e-I*JhV_1Ki&6qIwt<V->oM5S8J8mrc!hu7}
zO2yh~0MD$KRgP`e!gC6Nvy4OPMFD%QDN6EWB8seQl1u;3e)|N9BxQlTsZUnzoINj!
zxVG5L5*{S1C2}P<r4?~;+3d#E?l?ASk85RXZ&$ln6_{+8C7MmVy%_Co4KY)-a2`l9
z+Ock!tZTar3o8g~eiPpZubpI_vEwntIc4i|>hh!>sh;eez!kz3-IWbaDs~Iq8(sTm
z-WtBIF5D~o6NCLPrVjF#NaWtgS;#rdeK<l}d$_ujKDPIDsn>aSYkI52Ny=_wX=tl(
zr`x&M#cMTptKF4v{?yO9SI(5gsL7?;F1{w+?rwcyy=-VHg?e#gNppQ-A!iwTLv`Wg
zk9G&&ME&f$){arL+5sN@ucb|UICD1PNI@M1E1%9Mu8NQA*X2*q&r0{baSWc2J~V&i
z4}Sw!jdY5z@!+Fh(gzV1YB$UMXpCPGZDdV>i!Wcjq<@tjq#0O@Y>A#1z>HFgdWW$<
zTt>TSpRJh9Ud|Zu44Xui;N-<5TODN`*BhJO?u4zfw`DEkaonCdD$~9-gBN;7)qNzv
z)nUxRds+rH1{;1ws3SPJ%=`_(b;0%P8gQIsyqQc3^jrzreMGX$lxkGr`}}TG?xp88
z*Gjh$&6*12Zjs9=DpBkS+bKUd+-cf4au`fC*2%tNG)kSw&ZJ(xVC3F0m~``S@-Rjs
zqaa(#mXwNlUYAgiSj%3aY(>X}amM7L^TYZexK-bQ6t!C`VkAmdBKZq}62%7kP$OY{
zN_UF<>#R6J?ZO(B%Hwv{Xj6i?hR&1z8+V>JJhtvrC_C7e>c_<<rPK8&TCp{S)>XTk
z*CX(jo}+)hQWeW;zhfI9FQweOsatn0zIb+Fd3fc$f05ZJ+$gN{yJ6PiW|!oJ5Q_Kq
zxIu$qN8nz-b&I7Kj+m*KeyDrf*!%M9g6q74Few5)&I#+uD*S5A%390Nt+=EoZBf%v
zx5d8N$<m2A0XcisjFT=^y}4FrE<-LGr?ZDXhjBz6e4KmdT9HcjO+yXkWzr3AUyXF6
zUZ=Xc)b6|;FD|P$KZ_S=5kNn^+_o;{%x9Vknu@R3ySUB2*%7LEUm}p<lCj=g{UPf^
z1BW38uH}xU<<_4q&XHdyLzkJa4uWh*PWh*Y3`W0IM-FN@ArM9qyI;I+Vb+zn`TU*8
zn8=d5#QVo4!j7on(?6A8EwXuXYOATf=Nx6vb$wUM8-3c8bmy@jb6U_n+?~gjN|xFs
z#P*)^Y<r#OpmyP?Ywc<Zr%Ip^!*OA@?RO|q{57*~?Y7s&OTPM>ZR|Dryb@kL0rxOq
z(7KtET!Qaha?_O!?peP|zHG$Z6vs_HuGZ#Tb55&W@>sjp9jK4rv)|QiB5`V}pfBGN
zarJcFgTMXoGiU>Sj=$TT{nSks{d?a2$Cl{wyiNHcZ?@awvaymgr*yNqFtddL;<KYm
z{hPFtt1;i}wU{+i=1RRkkF_49Jv$_>yK%Vf@l{_`H~xiqr*y48V)<pP<Ie7Csa>+S
zvzeeP`JlROPi2nC)Ap#*n%(4hBj^*lF$t<r!TZ9C-plD*s>`0xGAjK8{hlU#Z_AtA
zy9P6UV^5vlSUYNMSOg+Fr6(}9=y1Xx5q(?;V1}*>(ma$h3hsQQ;O;)=UG7Si!5ipv
z!C>~lBG)`@4o9iX5xY@df<?~E@}P#%*6wL{y5pNDk`HbngE>XhMw&$A_q-jL3oj77
zm3P1MLVBua-8{GE4c{Jwym++Ku?L#i5K|4QH?p!Ybl^KO3<B&k7)0<57Q6*v3I6k4
z9F_(K{(c+|1}4Y?2H{`V$bnDj6#?GRn!lg$QGqZIz+X?m+bsj`-&eyUWx)UYJKQRm
zgAq}FB_#zum5m)tO>G>_ZJlJF3j2Y9hjx;hjxaDdRL~n%N{MnG&>yu>(Qwj`mEkqE
zwPrFju{APfa<jIB=)my1@q(|`rcQ>WZq`;dj=XLH<o8$bg745UGdbz~B~F$C<QlRH
zq_1opOi4MJSeRJI1yM*zN%<X2-ta1kivMdl_)CD?+{wv~mzmks)s@MWjmg%*jQIr*
z4-Yd7D>ExABe;Um(cQ+$(2dc?k>W3rf71~)bu@OcuyeAowIPM*8XDO;I|-1JLyZ3O
z`YWfYo5g=|vT^*ET0lW&XomR(6ASZyXoIEv&?v8hg`26Brl^HAuo>VX$i>dee}Db|
zn)xq||5&N<kCj~P|6KWxng6*`)zQ@9m8~^k=_L4H?)ul_e@^~uAwM&u`9Cc2*Ua~$
zz|Mjw{LKIHnIOuRME)6YBcX+;ybAaPAcJ1800J;He?Ot`I+RI`O~f!T!Z1>zA}VgM
z+o=y?u*Aj!_m~v@qC_!7VMTgFV9{ud#6{Z(Sy1T|g<;`I^E3Q>GM=C{k>@9fy)b)W
z0MqL;A4DS=kwH_=z~?<TL~It6IFmY)mbkf_*Z(tdW4EyGthOP|gML*sg_06k7zPf*
z2L^!@1{PVE^bdjq22y6YQ2(ocj5cEk<L#6G^EwzfKbCo_f3Ny`DF)IH`2P@shI~lT
z+#cSO{SPgF*u8&o`q$7rhCvu?DePbF{~GgQnJ4=1r14(-&k;g68LCB&|9;i~5&(wc
ze~I+Ur$F=v(ti^Zrp3qgKLXxc!=Uwlj@Tiy#0HQ3_p5+S9{+cneEP2uelix9Z4LL8
zSoP)FwX|sISUPv5H=blgYQ<4c<@pr@(eTNMc%6cq`fMSNrsT~kS*!~bY=7+_Y_i-L
z>9A8ZCT(L=^5bGPdFZ?KRHZ>8yG4}K&ZH`xN<OJP|ABvs%cO-xLC{Mw7+<?5!g&5{
zyKaww4`JYlF>7|dPb~)$*sx^HX9f{Rp)&vY{Uz!tuH#Y|S9fvsEZs^^e0+g&Ubm)M
zYV5g4$xEP_WF$-rsh9!(V_(uI=gZOZ%SEM)QP}iqo&9`!3=fFiGAhjn$%9KSTk<ZB
z)_RgSEI$=LBH)Zkb(t)Pn2D4W6}&w+cC;>SIE~sIDZ;HA_}q(Gv)6dFyjfJri7j}&
z|H}dwvBV7i!Q&WA3USF=hn3}(c>Orz&jO<jZPY+-U!|wQcrnk2@X6-A;ZSiHBIrwM
zUe3PTYk1keyu6&CCux2B=c{JLZ-LVH3_3NxgCprw3p*cDr_8ICS<p)mY*v^J#HD-R
zYBDulA6AZhL3G_7SA5f(V7SQ*c(2@?%`vOY<;o^!JFIlCPCg*<Bn={R(a5~I?nLt)
zDN=u3@utsYrp{TMaN2P|AhBS)K&)-yMF$~;ZNo_%<^kQdGlDMyFm46|0tS+4ri32y
zc|!A0&TH8ZN<ym0<^zsLt9>gK1A@_1ujB9r9`h_?xC2{@671>afl=*T`E;hdG>=05
ztL+Ls+&y=9UehH%R3?^TvF}zBi|!X?i0vrw4Ywy_`aNmycJaL^nVFgEYOoD4+M*7a
zd6hC2HQRnp*E$X#x+=ve3*8(?Zv(33?{7S6uXgKmBt8?13G@p<u5Wv4=RqWi;UkRv
zFvX#p7IW<m1JexCczHx!pHZ|qk<EH{<Hqn;Aa>o_2)dbGWx+2@Bt7iJQ3qnLV@2cA
ziW>If6Y~Y?bWewvruK|?XS)HLPU2`;XGDey>%>B=ubMY9KXVVe$q=S_Y!>vYBxT^-
zT>Wv)Rx8$uoO^%8`17fbABb-S2e?mo+;<XGBBWLbl@GMuJxje)?M!u@rM<a4nL_!w
zk(bUiz<=~9QTn)txv7(L%Ic|?c-|lL(yDNlKgHTy9=c7}z+rFxh&1h0&%IYF*5bH%
z8zrZ!RqK#*A?UVm!1gFhHkmUklxrrtzHVJIAl#_+`An|j+U!_~uC~jZm+QsRfRNa`
z9$+bD0Q2zm7y_ab$rv0)Ey|vFX08D$Exv;`T>Vac?;EPHXB@0|)2vQ~oo932w(lEH
zCoLD*TY7_k*6%fL2^tBWjT@ueKYJ5U61x(_XO!{c^00^5w0`G%v}sSA*t_i4yhi(5
zrA^)7Z6ogEGUKeyb|R`t_4-n8y%JA%IKxtQaTvh1m#<>VroQWR%2t2<bS>SRiGa)I
zlOEsk?^loLR|<!-DkXF!zD!nGQu7K4Dk{x}OLCm5ebrF>^JXTAZDKUN+vH%et^C(x
zz68B@D_La#(dd0w6=$>W2-2U9sE%5y>$&I1q;L<mijS>QR=m&@xa7(yy8+yCGBSi?
z3a*$n(?rL-uMZ?FhO$4_a=huq+W?@?>Zn12Fl~Y$T|(ye1P%ik1=ftIa>LZ=*&Jq`
zee7>W39JOOl)Grrly<dDv(2$m+ua%GG2MWYE%dwtlD;)|1k@R!yX&JbXO+_?cPW8t
znVy%%-7&PQ7hl48=~x|CdnHE-RZ&;e^;1#}zjx>1rtA>)RrKRTJ>27a`D>bl-CQB_
zs^29dU{mtV-f~wo>I(f9x@+rvZn8}Xx8=i6kI?z|g)7IS0Gj-mlg6sF@rmqPF);ye
zM*7px@)5Ior;R7?crIsl$i@bv&zIR3QOg{1^Ape7z0`FGj@cBS3BC7h#Y_+A{%uCI
zfNwvlLD~HTU%;c%{kE5F;M0$zem+bIQma^fP2NX){uezCiadp0@p>K>(rw&+oQv$U
z6ugI6J*_vp!Hb<!woR!@iAfSiTkquVL5>6W@eW-#?nxIaEV71thS@+`E}4BZ>;uXw
zRRWfNBk`kYklfVPd8*6e3t1FIvKEE!cp8p!ty19@^o9psGu37*=<cGc<K@d<$`kaa
z@;bA+ry>sJGFV&j@G;bCW~H}1gg3+bx%?Ryi3CLFAI`i7)BIl^ebab%Qa?OQT-+1N
zT*89K>tx=lIcBgau%7WDOAMD9EzR?=D>wwEY6QRPg>qyip}GloXamK?#l_&^G&3Ms
z{UTWskNe%ZM2a;pMgMwWfqGjqTb4;9-xsXaVe9L<liKwDd~qkO@DTPquGt?CYy_?M
z`2@Cimj>m5tdf6RUv9bJ1QPf{$z6QwlehO5#F&Lp6r7U|P2&h9+WvK&KTKgC8Xd=U
zU3EqC{*2@+{+b?krHfs@6(Dh6jw~=)(u5L~Hl{G1Re(P#2C|^<(r2MrcHxv?ks(f_
zJHF0xnvBOf76BqwWCbNJ>;il_+f!(9G0l=>Dr@d;_Phth_inHDNE&wE(-yPX9c`I$
zj^d%=v5+oq<6&D%@(w$Hk`j9ek4?L49#SG3j19wkxHMb86wYTf!#3ALjyBygJ&0Q!
zIh7}$9?aIjkVQfKjL+-LC^}q&XZTldL$n+(?V^lC$gE4Qk64O@>}ncH#rZM{Jfpf;
zNd+F)WpE=&Nhv54p?YVFJs@DeycmqEQi);{qu;dj6l6S$)3*7@J-oAMi+wT0Gu#}b
zS*q7Sp1$-Ps&I?Qq-Y@&tzO6o5k6Dg_TT9khxp8>35Vyo&!!zw1>$85GXxgO9tn?q
zyS=%RAh2w@yRCmW;d!Zwcf2z-HonLCeTG&L=$86E`;n=&LI-ELF6m@N>Dg4Z?Veu9
zdk<Q)%ExAF1?CeaEWzIKDGhgD>>WoPMbllIK0V{0^O(~#Th!xBFJV!DjWEAVtOzUW
z+B$)AT|@hrFWc`C_>vR*#2Az@OtT>7=(F-3Ci>06nB7{dSod-ZBU<dvU!ba(KO{wa
zN#H?<&IUX4QIY_2GFe;Y?L>Kw`)<urIQ_8(XMnF9zEn&Kqx_e!D)*yRY>(E`mJHXu
z*>?15*lmX8@ceaJX*}}}`o=}dZ=d9VwAD5}XWv0oUAL3N@|(M@M1Qb~x<4)n(`@eM
z<w@bW;=9y<rGS!pp_I0^!J76p=UU1wTGu?2jmJ8A&)0;@fZ&T~F(EEEaS}&`27_+g
zp}ER*6t@c!#O{<@x#4M%Q2QAZ2h);Kx!m~xmgK1&ZU|1bol@Sm`4`6bvrS<w!nnLe
z#g}^*Yw65bL`|shGl4_8BzAiZXGv|9C8Y;bv30{uPS!s`wfHe2gzauKdrs`e93Wyp
z{AU`5hx+u{yTv@b+B4^8QI@4Bq2nqkEZl1;jxu_s0YA}&t}&`;-xtkkTs`q}bs-C>
zp|CH}=iQMGNU4}o&0=WNb9A9#JzeKpOes^`KH|Uczwu%HMOayNf`ok@I<5cy$xc}*
zQI)^ke?$DjNHnv<3Ja<Kd)Q$6<Og&?jZQrvu5-Jt|JTsB92TE{a55MN_*QV3$M{aI
zGi)jo)Rc9-HpFO(;QO97zPH<AtLqrWz@zQ^tjVcP!>T)u<+n3Ii7YQR-AK0mP{4WJ
z>$v)isj%pfL{a7Dkiu7ES?0GUUzo&JR)`_!YQQ0X(0Jw<BrJ(sGtA{AN~-B_8uNMA
z>Ik$;*dD#&h8UnWYOA;Bk|M*;ln-0;Z`w%hwoGamW(e*9D6R6O=4L6|hFs<~9F)-#
zv^1w`Ev49ESV|vXk$7KYo?L-0HAHWWdEyC@R`t|YDW60v)dsl*_QSGznhmB`1(dqg
znS1T{W+{=<oM{1|#rTE|>;F?tvIfR?{%Hccg;Hf%tW{CpIAYkG*O@LEO$p2IfXst}
zd`C2+bjF<$(XLTEnRG#cP37yCD(ZNYQ~iTGRZNBPd<6ARYAw-pY<9_)Lz(JmvMdoQ
zU*IvLrF_`vJuxxktTCm!bP5nD7u&GebIIC~o;lPZ2sPReEK2t{^}igfv0obR8QiPW
zYsD7bEs@#nvqFKNp;O<Hu0)Nq)Tc}5mCn!}dYf3+XDJmO!si+~ZnEYUQ^}C_0Vr_4
z$pKy3I%mYL3YIl{2emHRM4j`NIR(}G+46TtB3XzfWmy!rJ{990mG(tpq-%fX$9trj
z>ZfAgE;)fdN?RaLU|B0I%>pCcX-cjkFH~g-z1~4iIh^xr$lG4Qo8#eJG)A+9mVg6Q
zx63JAb^>O!>A-Kq2&*Mj!MI4fqp_DVg)Vhl=qbwC)rS&mO+|Z`0{}1Yf_1V!^iF%-
zop^<jEZDhIYI~-Xa3G;{2v91xuh;oT<$|uC(n|#~F)lB(cWG|eB++QD=e%vA>(Gt}
zZ2E**0a8p%Y99OgJpGN>X7kE+fBxs2Y655Xi^CPnYA*df%Mbj576bLOmcKJZoZgA<
z91wEZ=w$K?gTy9~I9BZ4{t16DFhY(<suD4q9*zC;=@64<Ipy0?|A$YrPG;Sf!Unf?
z<Hs49XWFAVe^inchDncwXZic0AfNH1Ze~7cwit47+KjRh+fbE$j%I{QR{y;f6<z@O
zD`<P$)n>|%!M-LWDd&-;wT$I2eVz_#R}(RLZWq_;!HeTM^``p1+|jKZ7KO+)CWo=X
z!10>K{T9^0;RxsCk@Fakpcf~IEQa7Gq@z~*Qwb(qrmQLGLX4}YM|5+YT*=S2gQAG-
za`)20(JW++(U6}&p;Hd_W-+m(7i9CwaFe9NuQI2RaygNzGWTvr2TUHN9V~T7qz{WF
zc$~-Zt$BgO6Fy5XjkiuFx4TiM6@Tipb(%j3Q>;<8sMau3*3w+1ND#AI`Me?b;;Hpb
zzQCGRZnAtpz?S97=E#>Ib8IR!_TIYsFzvuy7{__v2jcH`Rk7S-qaMOD&guGZPhv8o
z9VhZQt|sW2o#A7n)Uk}{vJ{O4p_7z+#*w!DaUlLjh$ztl^lYN?xua{&n^AnGRFN#d
zIp9FM9kx0aNpMqq_udI75I456Lw70T#5{#Pe79fILpG1Qjrh-MTr98<=a1u|*X?fv
zv|mkN$4^JnpLCfP)=Sj#$!&0~?1qdjc01lZcD`B=H50~)Prrh;;)|fFEPmW4fFmRS
z_NFgM5|m674!kP+a?rsh3#={WVbq#eM&{;qz5UQ>y}W+>VCUgT!4sP=e7F3tTuoO6
zj_p<UJ8=u|H-D;tU7!mI+kK7uQvbO2krdYeSrFlqLA@Iji^)ohL1N9mFDvH6)-ig{
zBdYCz025R_HYN1TGjQyqwv3i$YwS}WpHkx_a>XX^UZ0#7h-!&%lbaM@Mav6HX;gAD
z83&~INWH7glS_R@l=*SM^J=FW*MmewW_<#gH_GXFP=Y{?_l8=q))IBv!$>GJBviW}
z><ox?e{Zdhh?g_v%ep-IHpRkZL#N1#yHb}IytV!*d>t#^2}^YBNoxq~iF%&2g(h#I
zLR9wF`)NhO8-nK|U~j(fCmfEmcrE`CY(f1KeApgIrWDh+S@?V$?!-1TBd~u)*Y?Ez
z*~`Ual@PXc-4tEbRtqcKWa?Yk8~q4JjJN{*9(#|3wu$E)w=e8H3UVCR`rDbzCd0Xh
zX<9ApLWT{n%@ztTs#SAqP&e}`xV2ISpV)ikv|+N@%g1czIo5rp?U-;;N@BMlm;0hX
zSf)_ki~s)Z#!$|p3@Yn%N~%+9Vz<&)#D223fyERskEkUGv64GN9~|r2iQ~2=%2CRF
zQBP6);J@dG>+G)DJ|J|*&>tM=?tPN#ApW=&pON;vn}ky`f6|<0h4JiM)ODZ5&&%pU
z=eK%hS_#|_N@6c{nMM_e0Zd+s!5Cr`+AT34T)!sO!#f{)%7Fe17W;d@3LQmmisCbS
zaUp5S$I`!kPH+pSI3tQ<t*?bLH`4U8Hx6Nm5&yTQ)5j+a?BhAb5tDV``rQ#0WvZgu
zLUBVYK136G?*H6nlotYJ$)0EL$B~R~e<dZG)|v$(7T7EKCxJtfKMNi1$mc;YEzLq@
zC`CEE*z$SRRTjg~i;+{r2E_5Wx3Gq^&;!5$+5&|S3$lLi0Ulf<<va?!*#jJV7vjSl
zbiBXEG#K9!0gcsl0|Eq645|!NFK4D|rsKXpM&sryp1z<)fR4c52QWksX%~aU|M#&K
zmIREU0wLM%KhNk$5r9Hu$gE7zS<b)D#WJ`-GG0WsC;6`-7&tOedyqvU>@oj2*aq5h
zEQ#RzaQyd?9WpFvMv(nJh5m;gAWahvnknv=(?NgbyO&BB88o&pF-&y)|2b$61Zu-v
z6uj5tU(&x61}^j?T_XSIU<fc1O*qCG8Poy&OL`1AAUOsCKJGsU6$pVq_Wm{K|8*UV
zF9tZA^e64CLH_69e>&}d)BFEp%6&{<$yw=-*Hz6I)e4lhRifXo$;wTC2@%H>J?B{|
z3|09`!K0eQbC>j5uB}e+@Uil}Y^13`&yDQY6OfgLWJt>wENL~4;;_DYX_H&d#dogG
zDHXoRduiQOFd^<>NlaB+H7O-m)u7Zp02*pwFAPGU>3r)Gc~B}MFoLgdqOlm#>9phI
zBRkXQ+&7)&MrLw!Zu+CN?>xH-raOnrY0>T-xBc9Q<r|-%#si=?sWj<f2VvLU0Vl1-
z#V5&ZQV#(}eFq1sY8z_WxpR-Mf<|wx!VHCb>wXIbpeu1Rcm&m^V&wIF;ak7=O5Z$I
z)y`P$$*Sev(aw2hubaa?yyxQg8aQlvd=Vdkblw$C`!?#j?UzYUlwv>f&(&>@Xl0S^
zooV%Mu3D*Vo!ITkmpk_=Mc@FHDh+b(8JbdHXbNd_dOL@Ua^7lu_v7zRt&lY?_;$7w
zRWb9EI9|s7?;2~c#!ABH5;AHl=6JbNN+K^MqPx8-!$qR@pbR5jwKDo^v=+2`CHDP*
z5~v|Dut51{RJeYp%xhaFXBPsGKcPcIdg8U0xRMtFBv|&j`1d;i;pRKwq}$JA!~hQ>
zqzc~@IS2TLH4Uz#N`G_}J4Rj4+GYKjZS*bOZr8Y{KMvyD!1B2c6onC*aQ$7a$!jUg
zgi5D+Z?;Y{z7y=?Cf@2%U2KQ$W!v1e6URtm{#AkzQX(=>4<aH>S}w*pATWVX>o(!a
zuY&a{iOIoKBC<Xxg$DaHc|nGrZjDcf1@GR>w`8Cnl!f3BAqICb=N!u@B;TEdm{Y{7
zhV^OVRvv%lnM>qhy*z`i?=p1w0OR`=oR*phAmBm3j%|*ka^<N<KRHRg`Y7tVC3>;y
z%CknKvvj-s^q#z@D2(C!7Xw4Od$>gktxC>rj~ea7WlJgGQlZG;n9&c1Jn&}V899?z
zf{VY0b+Z~AGQ2j$2VKz8**)au!&vc}RwDO)zei6Jij8;&TwfKSVs#TBxy#vjh;9t6
z@lOT0YTFj(7UU&->WHR{bSq6(x}}r2oR+8%xax2RzB7)XjM}D)ziHwMRQ>JGm^dI|
z!IEQKSY|GJ)#OzW48ECji!G#K3Fiq)Q{_J<=qecM<$7;K11bFtDD8jqsM#Ap{tE)<
zzzz*!QHr&ISCljr%IyiG>s5F^PVT@}t8(vO7XP~4;ERSyqrvr>)7!%>@Gbp24pU4-
zn;q=ONmR8QkDa{_AKqsU96(!xkWdCR=YC=aM?d70z8z}nbFT!@C`dSNr9H<!^=iiz
zG^YHYa!??U6W~hCeqHUK^z-}lM?FsZ#}mZ%z!aSVw{f_xiCha!8FFT@2o~7{%H{A8
z1#rM8Am{}9<wU;}KUgTBIUmy&*q1>Uq=0I@kIxLis!PX%EugS4MzG}UJ3c9sVP>^=
z(T^1~9<QmqCB^=%Q8@SYlw|3?Lj1$(7SsygcD>974=^oC^BAzRUt`s9s&cOjm|;2)
zoKdIWW7L2Q*55Zk<R`fCL#lyrxONuBC^TbTth|APxan3aE|7C=V_a<NV|ah@{HHDP
z6a?h^n2;p#{qgxUwlk1<SX~5Py5FBh;mDzIhXNZtQ61@Z7s|aAH$YaB<TC6A<)KBE
z)H~J6Bky}`J*KHNRswGGL~6d$)SPJd81mVb&l$vk0ZL-r+zFtxo~H}{+)ONY@0S|o
ziBeSDFFnIP^_Ftjs;mx2M7lRjvk)-MkB88@AxVRtJDd3H0Qez|SDRvHudwoCSCYdb
z9O{qdXAB(fv%au6s7kDUiP8{oo^;-wqD&n2+>F+wGb(1lbaT|`bnm^Gc!q-qcQ1GV
zKqJc0JTBxZ`G;1wKT@&2x=#(WzSh>@6!V=-(cYpduxr-N;Yjzqw@kzv5bEskxo*IW
z(+G!eIHEiCg-nQEei!qF>ntcQQsK8U9Mv>{Vigvd60%BfKoPW(pDg0c#_Qwq&8C!Q
z4r42lV#fD1m9Ny;pLO)KxWE|&WNQoPdNzZQdn+0YYmb221T+YlhxP&Q-z;~;5{b1t
z4M?hX(bKBS(p6niTzKvN)mRNEQ67@s4ekhKJRLt4uPyFp=?r8!zv4iZT#Jd8?pW3~
zKz{!BT5=%2sLK5Ta5Dv5@@(PeC%o$wMGHaZTpet))_M7KKHm#S;D3|=16bc5uC8{Y
zcwZ}<qqdVHl*>vg+6}}`AN_>;3lT{e!w3x2`=agkigfQo?vsg{@#GdyJ?VZ<+3dxI
zP|sb&Rj6%2W&)+$&N$Qz;2jYGL}m@v*JM*)stGQ#>UKk=g$(p^TV$NSlD-6z`fJ=5
z_+|o-9d#JA;{>*44F#EVro(ZiE(God*7OXZKErn&?6nZMzPkZDX$TwiEE+~NnMWYT
znaueg$_7c*KIeKfhrs}J0+;#&Fg+}@gSsFeE(8=cLB`~@)P;MhEsgS0HZ#Ls|5sVb
zli$bwINQ`VkUEvkcWZU%Do|m*3N-k83zL!(I4(HgzGM195y4fID0+MOvmVY%LrE@H
zH)qih)ytgc`Mm+qfdSl%!ty{ah(liXephRCvEs`xVk1$t@=WLX%300aW8-g7s(?Xw
z0<@xRZMp9OXF)0xQ05%q>AI&HaHAhJ&Aan|qxv0Ll(Xmg8iL8h4^C|ylj>O%j-048
zw-jZvdKU%0dt72kfku*&*aqqu_XC9!@}9m3>r8eHR)RpAoea{|j6W^ZJo>yMJ-k|r
zI_vVyG*K)YF^j6$fz3Q7%7fDW`A1NT<p*2TKq(#rt{hcE8ExFMBvIlJtX_>&T4;7v
z>T5rO^dOBu1g>Nd&XWc~lLBns7t!a)7ubD)a@9auJC#mImu`;(anlDlzJA8D9SDQq
zDM?D6Y(_$dxFvL&xl@pwQm<=SV7DJ6*Pil+EgJsbKCE&8cJ|9Epat|#6gIh`j^et)
zcpySKkLQrR_>)Ac-=!FCmwFdUBFK*++!=kO23+KU*=IDrGkE`@9*yhobvR*;ID98d
ztNX^`KHUhbf&5YKmnsdSiREeVa&c52$qq&24}JKLWO+sc1$LeBQOfpL!k0nH_hBLp
zI;B6Iiy(s-aQOPo=|%SGuS+$HBrC;GU~BcqOixm4^!-iQ!UmSW4#kOfh(LUhrj`fx
z7`5L&OOC8#Rh6n0ZvZLy3!5s@$vymif&fs7Te4}O2&2UeUhY}tA<CYwR#)zRv-PXC
z>T^8GRnxlFOO;u*r)8-3_6PwTJ`LRC>U%Gy3g41MolRgqxiGewERr@u1XVsd<Uu2~
zQlkfNQKr@3z@-3YdA6X$4n;;3MzEFEa}PR)<<bBu_vtz_heCF?Jc2cq$wA&J7UX*t
z(!k^j4r=^R?Ig8EAa|!vY3uCZqj+#Mq$a_O(qGV6S}}gEYW5(DXh@OY^nOz!j0D`u
zikw^s$%Q2aszb%<JSDA0ULN99x`QR*=2u#`u0%nVU+;~}F#ak+1r))-M28H~N$Tsz
z-=%LCW~J$2i$pKmQ0Ty?%ea!vSC77;CL_Sc15p)jmIiGL{VzX7VA%18Sl5>|R<|Xg
z!4=#Sh06>~CEu=0x{6)DtG+QryAOaEP&_lX(P&{IM`$67-s;g^9KDnJ*=;tg^u91)
zbyT2_=ZnGDWtCalk+1xJ_Z@=3y89v#EFg0rbNTv>F!R7=ey*?_^!w`5Suiw>iQp<_
zlNw#FBozBs>o-st`jw~!$rlqQoGpx^CKG>667<^SSG*~Gk$<ftD55I_=rPko{SJ>A
zdl(Iwh@lfg7}zEd_vaiA<6yh!A0sS^GWvrSu_w1PE_NW;YGR%-$YCT~XGSin-q@_G
z*tt^KCA@k?`Psc*8lY`Aur_CbwhF>85WZhiAr_hOzFc6W)wudDdVAfNfcDXLO~sn5
zmgBR7zc8O!ox;0F2h}G+ovQbEKxPMc$oLiN2$m-xiyo@+EY9;);QUO<`r%$BdYFnC
zz<9T7rBYJ298>ySDHA-j+E(X6nm0)twRC-dVgaK4e?5-zpXWgkQiSFDg|-mLBPE*!
z+V1aLW6+=ij+*O{xIkhbbYL=13`!ZOi!%VbP+vrmW&}vE4Jf}->vn_0@Xw7CsIdP?
zyrdXle~j_K|GwYx69?ZbgBQ_(U3`zhUNJj}J_mBlw?L4#k`xS(A%hhGWPW{&-VOym
z4IhwX0$&LZTEPPCF3Gyjq15u9y+06y3RGyC7j(psg!A$MbKdQbilA(00$nxST7V8&
zJ`e&tTFNHm`tumTFVFYaeZWycz^0|WKMBUL0m;CwgJcinnSX9(o8bXjbW$Y*NWf%3
z$twKucSyi*(9uJhhQUKfz)DCZMEU?oC1hxSJpVG}6_k%qKtGGM`jh!@e9M3gnSiPe
zNr(UxCuPZO#e`-ofFP3edMKdNAdT<>f^2udpx%q?3z(oQwunPlnL-H#p1=H_2_|3?
z>=y=_8DVZ9RSUlB56B%6bYQ%&)r}Z(-~i-69F|__s6K)UO8UNeS`ei~avzpXEQ?KO
zV>NFJ*{uV83z7;8AXb5a96IL#?16?p_mA5MHv<-M#0Jw)M9c%$z7nLgNDzx+0I=`k
zNcE7g<KSe-K%M{F{lprWNTbLRhS>Q6SRk8d*rR|I&E8<*u52Ehwtyv+)Ijn6Kt9M+
za3TOB!7PiArF|=b-Li>ZV1v~doluwJa-n=*Et`O)&#9Egp_G>q18C(SNN_?-$e?2$
z&1QQ5U1Sol;M^#@10pN^3aIq#?ee{@6hK!JMAZ8;0~6>`o6!A&XAnl7L+L>zk@w!{
z>A>iBFIwRtvSJ|mu|5VRL!?OenPL55OYb4+0|Z3iZ?1t_MbNn+!+#c5y3c0X5Jtk#
zT<$UQk6k|O0GQyHkoLXT6_k98&TorYkgj)t_pR|yE{Gk5BveIH9=Sq-Ab|ZfDTZO|
zLukeVG8LZ@9VADDE`*W8JQrx<J=nGGzs~*!Z8C*v0Tbzm;Z_inmr&4^`z$emC=~<?
z_S*<QLr#tX(92-|(+LlWE&)s^h=P*~fvS@n*bQevnFEqa1{m7JkJT9xYF`Mr{LVP|
z0Yo+gc$4M{Iu}GrcwQf<u7?Q80Sg2~cJr}^V$KI5OYEDC2TgnjCLDi;zYT4f4R8S^
z-Gk<5&_o)rp<8$i+<iPm0bL^u+i0PQJ8%wm-Qj0@Z}f;1FvCM|-&`Xj00~K3f}cV&
zLy)OlT2&x9Nb3Q^i52<#qmS#CV51!`Y54bnCcr)B3E>1}Q2$SWlnHEyE?D3R7A%DF
z5JPPs%_Bh8B0us0B>HD4YOok&?xRKwikg<Zw~$ncApSZLOgAA9fTyF7;qN*y&>^z_
zY~XyFn*iY-D3E|~KtPOrm?h}m9V|qaA5Q88G+_ne{Tm1CI3nZ$CP1lv(U$|7;0HF`
zC!|HZ4{=IJ*Iyy`ea$UE_xWeUIf$JwDG!)&;GBjaje!g#6!yoYhGw#%UFc450Z_?D
z*abkLyeEAUvhH~RfHZStiyWj!4&*>8?s5pB$k46MPPp)Tu;3g>*n;cG59JYsXMpUG
z0tJ+?VM%~H&S^9fpv@&|C{Rz+Uw#A<mlcF;#KZ1C_Xj}#INZUY0p84z<e$Dj0Pyt&
zlDJ_*0wO@ZoQG|{uQtShtQ$g<5i~Il`ZfbglW$P#<f98%xU|7rLKAMlhVwj1C=ewK
zNl4c~?7Xj3-`ta`at$@?9xki!&vD~+B|p=lK42I}pi9BEy4!yY7YER*@dUXS`619x
z5-tqHuTLHzo3@zEf=mgFDJdSfg(3L>%<|?df#(7+oB-_%3A{T75KiAHF%mGJfPn~v
zXAsWiq0nuUGQ^MsAr<inZHz49F_?8y$dLgSSqF4QKC|3v0@xWyk+M)wf&sFJ>`+6W
z2GHjg5)GgYK@T6l<P|9qnBB;bP6QVJ4e0t28(vrV1NTG;8;DB+7rcC84%v+#U}PVQ
zIzWW<KX;>f!R(MoEbINo5~Lr2mUA<ppD&ef^QQ=)us`MR8ju&=fGy`~<xoES*Ihs-
zFq{2>4(b3!pfL6DKNtY|!M}#{#U}#_BTHr2gZ!}%RY&C%EF@nOAY@l3xR<yAKri_~
zwHrv}2f_Imc`2O=OvWL=vAqNeV?+r<k5$am0y*-y!Kr8$5Hc!qffr#20>|UR7$6Y>
zgv)izcA?i`lHno38-JiMDWz|_EWp1qkVD>H2@fF(;DM6jmoswq2QW*dLJK59<^>-5
zP4UV5|2g#kIrRTU=>J!QGLZQqqQ>{&n|+SGdp9p_QG3Tk)!Sw)LcTw-7P?Z-6uq*s
z(O^23%Xi+v;qU8h6p+&h+UI@ebCy|G5YUaNnhI#QyI1<zGxpV8rA;N#sLbXR=t6F$
z3Oc_&V<GF<4K}czlxdbPg>$SFkJ&`bB#p_Fx#EzB;hA6xss@4c2U3~A_OWeH)MIFo
zCTmhzPv9?hsvof)%=UDv#63!EN;ne2cOQEVo<wn8UKKHu3_w{KG8$FaXO1<NtC3YJ
zxz?Q|Y)t-fIRrEDCOIUha4Y4gL#n}pV>zTJJWAlqT%(ZHs_Xdr)4<BiStFakn!3LG
zSH?H7li?4eXR(Md%^lrPYx5nG{a3C})YAyZ-q-YA+WBsO`{i=DS~PY=5e^Qp4wlIg
zen4s`(X5DM(7@rKe6Vf05*sbOE21gt6G(OW2JUu0_mU&1orKXXGip#N(fP^F=~O4(
zm!ulKDs|HSWmqtxA9s#=rF%vCww<=%P4(^tKC>4YkvxIv_T;xykpiu%s5Q=8+b%ku
z^@wDXlTD%JX85Fp!1?b}Hp-hG-KvekC@AEI>%Y`{F4l<EHXW=w-9~F&d!psN$@5Yj
zGWd^{IXDwko)eVR9KNr;`&pecqxdd~o0?wcZgz^uwP=0r>#eN!=>g-AdER6farF{!
zOLI84f!I8#C2%B0gq0ya0WI@2rxeR4T#a-%!#mZf^kXMm-$oqweq5#M@EIg+6ntvZ
z%#o(r@V0(X`&~Xr=}e&`?juWZj$--V<gdHdXp&^TgU>>EbXAt>^qTFI@?>aHK4tR`
zQUSh6g)hrS3_qA0e5POd;rKYV;ImUj^9OPn&?CbJX=Chz+mg+EaWM$Kiw`vz&??QE
zq?YQWyc7)WoazNoXS~-JTywTY&^uTAw0lo@oeN(l|8j|L;<Aq8ag0y6*!D0!XEf{a
z+2Y0N)=nZp{_rQi@75`@Qqc38C^@FP=Ua51tKQK9?1*^_e-SvEI2~y*y!@_V;yZ=@
zrhVeBuC7RVp=Zw@R9abmq<+{7YKcz^6|$Q=D24*c(*D8@a5$(R*yL@ghDo*(r%!A6
z99M7U9}7A~EU($V7!JJM^DW6W%nU=5rNe2TIV$wL7?D4mUoMGCz8W~GF;cqx$RhbL
zo^a46|Hs`=&!_3kO?V4ZXc$NZZKToid30z_(&K4Yden0R`p>ODPwURUziF3AO4iZ^
zYPcP+BS=GseA0F%AA}2#5hU@}e>m%|=R3wUT^;tb$}7@79B?!IX8PK<_v(<^jBLfT
zt4G;)NH|=`hGZCCOT}I2u3UW9<BH&7BX7<Gom#&B`#b5dXR0CH<leWnQi2H_`f=Bl
zT54Z)m^r=|k;^9iiTy6It(a8>dxXotg{rGwe5rdnU6DyRRG8R%Yo$0jinng=D~COJ
z+*(QOFp6IghO^Fp9Iux~ABtNV6ZpuoOXLCP3%D=YEy9|8snw!RWm}OFl`LBXPS%q4
zQiv->rQWnhYs_4hs^L5yrR1*tn%_N(8MG>gG%*@d*{7<~G?|Zlj|9PU8O%fD_^^EE
z8M#tx10AI=#=kAtUYtjhEDsOhye`&KU5Oa&Gf8hCz&0y)*So`snONu#)-3wq%IHFN
z!Q3<2EkVK@W7@w~cj7Jmfyh!SbjBs-*JEovE3ZqR#-?ud;oGYnBdeXktA!`UFIW<9
zZw8FzJ!#gQ8#}Its*py*pOK;&{c3kb&JF{~$=aM25AGo{f+(Kn(au2m*!!EX4`<U}
zWhQSW;=vPg0`qltu7iu%0m3Bit-FVly|7AUMi(!~W=KBRyq6ZAK6vSL8Lc;lRD=?e
zm%eWO?8|P&?ivw=K2~3KS0XufI(ROdY^g(!AyrO*=2q?a?1W2zta^^;7sI48vs#gw
zCa;URMXuvIM=37J7JKRI(gYLHq4qhFDDb0{8dcn@0!z7mm(k~6b+61a%{cnc<lJU-
zUVlH=SSv|)>Q2pSq2%09snQNgqf&Pkcz)`TxtAm{ps8_rWE)Vj_vhPpndG59_BzxV
zyZ4`Judr)#=dgIb4F4S3sQj?J*z=s1{DshELb!^B=aUfvdNeu5#j%ybtChW`yU?;#
zBF<%*+V{=0gNH^<`4QYJ$MrH40;e7)o3W<z&Uu~9w;%Hch`g??wO8zqityZ*GmE0q
zUSz<jjv;!T55jSUUi=}-QkmYgdDX@!7~WT7Z=9x5Y+sOQlY8PKc(*@3b?PaBH~tir
z;o-kevqmwIgIVxAw%AzDSMy}QI9|w~C0L`g{hY)`(u(J7<h1U_-BFzF+3@wgh;@IX
zPT7$6=%<akb;C{e?s(QPBYTxiRnbP<C?kKm`>rw@72O1RFRtG8OSuhp>1UkzR1XM$
zk9)oyD3{9{+Q6$jSzAk=Ca<qHH|P$(`5fnUscGDsv)Z@$`s(SK<Fjc3lU(D{aN?RS
zOGB3ZcIr<sZ|Aqw?~d-;>qoGz3f2g>-^~kGR2A&%W&25pePUw0yG+g*y#DC#dwRO{
zJ%6v^t^t$0Zt8Oh-(cFoK-2jk^0)Z3rqSSU<|23R2h;B0wdZJh;!C6JoU3D_<=<N5
z3O1Zxs=tX5m&kP@zU~j(#OmHefyWl4L;kb+c;_sBCk2ye#=g^(f2sEUs)yAZb-eXs
zF5RF^)C&0>w~Vr*T3kUkwC;Fj>Z2z7+L22FuiHvCiJl{7Pn)zcPY=#!T*vq9VUERX
zWfJv0JZ3~sv~w=ryN`7pdVl_5Hj6t``x-GA|EJdFXVM|dItBj0(<)qjP7&g?!$w+(
zv1>vn1LdRHq69}oMY<MY)WW&^WKDFZU(L!ZamfShrrKvD{=Uv+KQj!#o(6%f#Lbl6
z*9RG1_gg+HZdGPcSguSC2&1rXuCq0RG9-hS&c(kTKPrvmu;ff=PvGn}isqF2%<qwa
zPmAZNOpWg_yfHXc7k~4D8jZ`8pSw0x$6+vX@ZF_Yk6!gI^KfTaLuMGy&do@6XpwWI
zv(-?0N%&Xuf%-HOmZN|z84B(6L8O||f-lTTKX8yzQ^dC_2E5;ROw4&-rPGS#W*OZ&
zUQ1Dk5-d&rDt%zEQ@z*!JP0L&zFvSdn#67DT8<K}?&MmCEh|G{Z10DC^mhG#(6F7+
z(c2GRUFgQLzh*eZE{Hu^G1#aiwuT*XR0z$7Hh8F=YGz~(c#b1hS+{0>+Hf9ipupc`
zD!={n##JjJ?OkSQsrQ}N7fcg6X`^U*wBsa{?`A2(J~NTnUeuGsj-l>467>B7SHH)<
zvbwDgvmVEp^&$@wl<Mf4qx|+EAm*YFt+U>1+1q7($;w%AnN6W}U?!zAyr*b}uxxs?
z9<=x9#qbBCqm5mqx}KO^5iS>p{>Jk?A~i$1P5dvZ1=`6<&tCj4k7>B|TF>u^{(Ls{
z`jpDw*Xza4EMdr9tY5iJ8DT@gWBiAFY16D^b)n4sExpXHwsp#6PJ+&KVRBsF7DBop
z4gR2$bdqzoazO0CwASjZx1Gg$_j+1<OYhw-<&hvs<H*bD*G%c=9`??30dfU27L#6k
zmVJwT(Ww?9%>2XE3Y<IGqYfNYZ`W%nFg@lh4KqTePd<gH$nmHhC|B%azYU*AQgh7;
zkq6HLn=g4!O~xly;kkx$%U_=G6ey@MzFX6Mmn2@t<S^1!VlxyOANOF+;QiZQ`iR0-
zLmMxGsv9Pbd+*9teP2F$3t}U`{WEI271b=$7ONETb(5>?Y!y9qa^&-BEAS(UjPvyy
zGDXFVGv^!W9S_|w7JT=si=nfmQ{cK<<t5@QOVvyLcjcdIPEuZ;Idy9;tv2+n2XR(b
z{nU-BG+1z$upW57I@tA;DZYnUpiqHReS71$DjjErkv9o5*@<$|H)vK$6R*ab>)l>e
z^g5OR{nM7_?r#Pv`(TgjhpWtAp6Juol0joNATT2#$8+JcPQknKDlag_?z-{t@Y7b-
z*zipoHd5DPZbyQ<j|icsgx;)ILH+_)Z_oa8yy-&3ouQ|)Al%^Ji|uD8wdOx1IZsFo
z$sXpdT;t|E*%DF}4@<M~_!NJ8E-!Dped9!IH);^S`^L~RzGo`-JjPs?WqLI~v%S>>
zRWTa;4ugzNoSpI9<&RpHY-zop!*OT-Q2p9Nk1k5rka#&httu-&@B^NLVHYRjAN@gh
zCZh@!8RHEA-~w%DV>xGa@n6!*8n)-Wobxp2w`QS-vElot+wX{e4a8br-MR(_$q7wm
zE(k5Yh@2lj91yfkk@4O%rtLZ=6zF=lQ|CSExIPybPV=c@>*dD8joX;Y(S+yc#`66j
z{k|k(-KnSPN;9#YeS0V<x;+QOi}f=-E?(ql+aV7|5x|cINWW#ay`YEsNvE?|6I670
z=hcv{t`3`_0;1B!)11V-yTL22`uO2nO)bL=W!1v5x?To`k-(<@g>d<vl-eC$qE~Cq
zmU~{?i>B5Ijogmm(~fgnZ=^#xrx%sU4d>;01`%^6eLl%=U?K5ZORrtJa$9&O9&8IW
z{60A8xNEdsBye#49MrplEi{Ll+W+)Qht-h9bn}730@Ycf#EF1(rNB=8o)@#nlZtR*
z0VnS<^D=d3Z-wVF9>d4u>5giPGtp_D9^Vt6vu!l?tc3L3Zb>oA;kC*=9I_CXsG6bB
zvb6~CEitp0J9w<zzmZ3@ErULF+4@%ZLl&PR)#6<TiFYW|<FQ#UujhkVr*4KBS@~Lh
zulthXOxFBj)bR3|H>1lH#nqns`^Fmn%%2BGC<x5CZYFFX)<x*n@M=&=*=Bxe0Ly5<
z;&k%({W|IW-Hz`|WjQBY3f<l!skKIui8ro;8-%<;*R***3-dajKS6r8?U=gGfA&;<
zeC#W8aH&vgi?HkX;0a8fcHfwz?dMY71=|?`i}M}kgm=lD{5q0^wfIpFKNwaE(0>Tz
zUlCMWHoMk2eJbDhO3&R{FVcNF*>%R<S7&%j^vujjJ<EgwsV*$vrnqRs-K;Qv7hxrL
z2<3wY>`GkhHruTy43R1C=D2q6#giln#M+X!-)GViElv*@WDi>AgT&~wCw9Gh8?M^*
z>C|0#COtpj$b+9Kg*IH``(~NGo%r8o02S&(*^*x5tXCf1P*-kPHU(M^HqzvhzVQ`(
z@AYXYZ?(P0*Ut<?{4=g|0{OCeBt)|=`I<hS6I9k>d3H6ZDpeS+)-TBoGad=g^hulz
zjnT3k(5AK+EbB6NRl(;<#e$zy4LGB>tEcX}TIs~|)XR>Psv8%r0ej@7B{G_&v5oSV
zDi3_0;=HKE;M@LYok^5(sdJV8iTJ9V=c(~!WfuB`<>=S6iZ$=PeBHJ%y!?5Eg-j{L
zC=@ELST#Iz8IJEsPRqvesnxEb)0@fzuYv_of9h6#-HT_6_$(BrHO8?HcL`AO1+6ad
zRrD@lwsOp(vW36zt+mR;?i02o9HBP)#_O-Q;&>+5La{;Mjt(h*S-Ua;)CR8$9`SMQ
zn0|NrBQWdieJPVYgaThB>zXiVK~?bSlX>G-3I~E~xW}MOm}b-E?Rv>e4|!$9mW-pe
zvyYQs8-DWM&0rJUEn`XLu>><}yS9H!MOhssxw|;7sOaN?^SJroiiA69(c#6J$nsk;
zI=5cen0VNky6!eRyDO|nPUv!HOlnX!Ksu_4+c0B|X?^nPPS~C0&8q-$5ovlfCpyi(
zI^EtYp_RLmDAa!Xi`hGyy&pCeSC>8T)ebKJ$p_U%j<uN8@p_)w{451MYVNNoz^drU
z?2R1UlGsz~?P}FpH63sYjEVFKSrD8e?i+s!;)GgHg=~*+!>DY6Tn%fJ1iLn}_@A}&
zgk!Q5pQSSo+E&AACbf0glveJl3hf<X26&`C)fWG83<E>34E<jKBP)82xEPjSRM%#<
z;3meigWUU^<dEgqjWYaR1_V6UP53&{`g~__EX_++Z%;b=K^VtGakaU8MSs^=?L`}v
z?sy@p4u^BoB{x3TYSv3lBi#SP-dn#_9kuI&0$aKf=`QIKlrHHOkdiJ1X^`$zQd&|P
zlt#Kk=|(yQq`Q08hxdGE&NXxXgE_x>A*{XP>F0j#&A>2IqVDp1bUq{|4N0}2EQz8V
zEwM_exFM$pAF*C?vx4sY$xOLS-|&36lO~N9k_P23pf3olCX-LS`j_3Ae4R+fUs*93
zZ#%R!poQYGMr*cZhW&Q7_qYl5`!^Z1OdOz7^_{OdlKr_uuG>lT-a%Y1AL3q#VtvHU
z-^*;p>5}3%z}6O_BK?LX0^z7F|HVy$0wt(#BI?m1_HargKBtt3C`Jl=v~`?H8q5Y2
zA~W;mD=5!Wqu4Yj4l~1$)eiKtdF&>WLS>#T1yZ!<P1%aXlRWfagOj+0w#Y1AElaLb
z7EJ|pxj*%_!7F?}PPQ(ev9LSfa$3I56ZJ|N-lnndtYT`KjhlDVhFgGV)9B;aw8=T5
zxl#UL&}2aC3GHf1upp7w9Ec_!FY@2;)!84!nu}E_T{2nrhFOKCfBz9y=6u_EtbESH
z`NxBQW^{yaRdJ=55zI_n9^RJTOCvKf2=Ua)#ee?iW_`1`I+dl{bGCK382QRwEx&Su
z%C1*OGDB%d)jo5PXR{8~y&&BN?K{TS6c{*c$}<zW)GFP`dauWxxlRB4*YCl%c6Vw|
zfj#|ZnXMBG;yw&FAt&Y7$n*_3UyyjSI!*X4I7%luoorl}?mxHi+c?E%o=7<+{x_(b
zC4!O7na!WFzvn}qarY(Fqs{eLO&Wt<xwah{H_hGYU!9AzvKm_<hl+r+hq_*B=?CX+
z4J!~<%nTc9l|f+zzH(!Eny7D}U)F@g7bT$;Hb?QIF#YvKD~TbYAdOOnl&htGjGGqV
zK_)UIbgeJytKXaGuP<3C*Lkpd-@ER@*}5H^9<AKJA>yuDuPZ#ZJp<!IZDvetKmBqG
zU54<?PP)<hUCu?Ils9<(s_3%NdAIUh>7OT@wc++#H$B2)Ml#?GDn`gH@$Kt{|K5GN
zPgON!QaGa6{qjaCS<|EKbPlOX!v$}2ETl?PSDOw@lI1jPXt!)*D)&kUCC+@4B^i<@
zh+=kX{i=VzT`Fi%Kgr?P{q_272Vzy4wb5i4DvM$<=E5I7oZ166t@3tJ5a*xov3`ux
z7!3X7u~i;9zVS2dqVTO54x*0cj<Gd9=hlyk%O1Wqt$W0AAN(+P<LZ<X6+yk@Z$pd*
zLUpyqzZ}eKnDN7}l(ri3_Ippd+`DBN_V&$r{9j|yj6MF7yv1iF`q5IWkI(wj$9((k
zxffF*husUDcR$sv*WX5~N})?JSMH*<w^oN5DvLrTHe#*0iZ~G2QU2MaU^CLQ{noUI
zrR-+F`{81?>4W=xRpQORHj5`SDD+pNjLo$kqYNo>IcQnVaMoGxiEW(u&%Kv0lwT>6
zPHfK@ju)y(q{f<>3|ZM7zgxKtuk0e^{D$QAeiX5UTr@Yt_k9x{L(BWT*3R>ztxA49
z8JAsqk6Gtee~Y+v`i7}2WboE0Srj+15p`^KsH_fky|!*&$=1vWmL6Xe5RO`$y!yKt
zb^6>wsNfkG!TNb#cYhicmae#T_2cq1Zc!<poVFu1sB-2)G4AJNqx62rK(}AQ3Vj-X
zw!pu)>#20#ven9g3F61)UYdz?==ip2uB&$b)g)?K7pjz1nB3k!WobB{3uuZ-Gx=6N
zBISO=hiKz%eS<}G79QSm&cltnrmt!58RHmpRNOJh<YeRT;q#B#F}E-NTv{?yU-OV-
zhKTCrRe?}0s5gI(ByGIhIyKjcPEi&)wcW{*X9y*aT<3Z%Tu-ST^gDi@oMl9vdY=4E
z&8I(cTl^@ojAQp=gr><aPlG_qiZaKqYnKJ5Ooh-@QvBT5;AqV9@^YOgkD;=z(nH~<
zr%3<Lnk3&0!rhp1%U4jqN&2o)vk)fSOm4kl8K6j#xn=MH>!f1Mif$5za3(C}S>d;&
z7Q?S&zBg)^@1|dxG*!;H?k;Q_aj(geMDiDJr&eOhn*MzQ3P{drj^B?REEqBn&3_vH
zK>A$!jRrI9Y3Xw{Q(~U)tcoc+hQuY3rk@r@Rg)#dGzpDH8r^QGCj&ZB5%iXdBh1o%
z=VhZe?A1$TE?m8POztz{Vjvxnz^coDK|KCEq@ix?+vo3c(yC9{ZvKSZb+){?O5e1R
zBCdS3x!>|4#YI)wbK;`2IqXw<R8wBa<>ptsE2XoIkB!_I-yP*HP5OGLTmr>@tEj2g
z0<mo;^JyG<?)L~SKp{J)c^tVmn~`ZE-D`_@bNH@>k*P4=baL#ly_Icyr}CehMTal<
zAh{JRt^{GEilA8r8NnbBh2|Ejq}~($^~(U3=;ldr>-3gQQ}R33K!Y?%!knjkdK@Bq
z>6&J(&Sa9xDrbt~gctPcgj@dyU2+RLr%N@Vj|cU4IfXBC5@hoWE(VMO#Vm2gHYMr@
z=|^;u$K`qt#*hQh5%TNHK+YP-b99kL26{d_cKH*nxOU1`B?c|Qd|i}fk-_tvfnrmO
zk8XyrfnyIARl|Ux17rp}Jm{K!Lu=Q(KMSd}wy>@gD##B~OA-Rzjk8X8d3Z(8uG~2I
zo9YaHFcAa`0R3eO&=|d-lu06lfe)g5g+p-z;37*IT1Jcjs{p{EenRD-F$-Zm_#0nZ
zkbxh73J4|WS|WipgpqOS5dlJn5?Qqq=^k2%;f)q(l23s(43$~RXotZs`C+mTY6B>g
zp`Hwc+B<-NR9OCjpTt086<PerVH<>}zWaoq0D7}$V2%G*sSE@#1i0t`I^hY1c9sGt
zjf}A&M}RHqc>rw>YU6;`_WvI7e_!!`1@-^dMF{00Ovhn8^L%rvJX`(8n}K|CLtY5g
zv;;H^#*p9&jT^LT!k)5dQ38gUhUf2wMfaMoZTL>d)x69^em~%Eh#}|!^=A>FUsbT9
z<KwjvX^MD2;-uo>=vZ9E9UJU*bJ*no$fbH4-HulQSyO`j%I_XP)RZcVSMVPU@Erhg
zENK9w3}rMNJcai~=36lEdITt!dc%phb0k7Ai3<KuXcQO{12z<NGJbXHFQIdu#{;83
zG}##qnj1w(TLDNLK&v2#h76@W09u#DwgSG3HNYwrje<un(-DlJ7<IvzpK6?6u3Zg{
zgN_2~9-7p%1rOVfonZ;AfeH<E2r)}32+Yj#C!Tb0G$Q(gDU&GK{e>p8fn=W1CJ&do
zwFKQ9%Rk?Vs@r}3Y=(Y5dYw6(D^E}d2&^Dlp*u%d*aM6RfLK~B;5y!d%kze>FL;xW
z(R4bYJ6i8-_4TEF7DR`1Mq=|N$mi}dAD#GlH;Y_}c4gjIOQ*ZbbzM`cJa80jWO67M
zGyW_M@Gl9-A7*~a+O*t$9P7}_mBX(Ciw(5Tq^72_Xs_N|wp?v7cc@Ih`)NHpp};<4
zT{C~PPY4Z<VV6aNjkJSGG(et+z=M(LU?RGU<q(QhK!gy_Y4TeJaD&{ghXy^}_BSPS
zuDiS1mXyXt#3X%&fQTO58_%r!j`aZ`3jzB4d`v>tz#Bh?0j?h^Fw@UH8_jFG@OH0Z
zU-<oGN!sIt$3uY{Ki>54OunU2%U6mA{8hb8=L^hTWG-(HNPp2lFQg42Dsf%E1JKxw
z;x|C|L$93v!M)^#&D^uw>+=^vGx1CsM>QtGWp}$Zi@e#~$Lm9hfcHjODuS2}42>WA
zttCODldS-SiqNnU;0*Q?1OvnE%z6zU<EJYv`Tz@-iF-*3kCl05ZW|9HJG*i)76luP
zR5&)i$(5c-dz*b*1YbIUbZr8BkBo?kB@jAzaYG;<T&CUFmEPZ!()hU?R%Pmz{L$e5
zrisjJ{e1W3t=H+FqZzAG(C#BMeacYtVR|%U(C`8N#{ztwDc>VVb}j;10g;pK*@Qv$
zhp%tFFW1b^Gl_XDFFG!;wfG<%o2@C`;15g}@<<+w{h?V(4nOoT*6Itv3Kq(9pcyA$
zSOSZ-F{}+51qRdF8Jk~hz<m27q0<)m?a9&?R>Jg($+V~2lP~fWG6134oeTN{vd%OV
zcvp#Ug#c1|K;U<C<H>FI4q`G`sQ2EAe7V*ef1F4|_48~@S%haB<6r+*n%&u&L@>)Q
zzw$dmLgqh1bU6@ai<mtfz}mz`4JV4U|EUz0w%q9`@vS`>EBi22W`rz=6&a>m{ozZw
zNmnut71NQEbtq~BDn1jeOIGj!H_8dbq)U_$6KsRskH=~vsiV?~WE4Cek=Ssi1kyc!
z;2I#LYLST|7h0XR$cPs@pBqc%9?4TA>4wDBR}>ngg{jm+ljBjXag1s?${E5HHuL<O
z^9?oHjXJq<@dtj$1zp@0gK;+V^`A<J>UJvn`g!xKtR^uFYHA^%;5R35ow+MgY~VU6
zVa@k<9^BR&pllTn2&wv6*cPlxo8bSF*>vI#2W&XWjckl$Lj+meNZHPNbJ`9L4#pHK
z5Q5fk7>ccUPhMjB+2LJW@3*W0U;_7SB<N8NkVZTI!q*&@L2t7<9C)#q6dSt;dK!Fx
zLuvX?vV_j2Xkh6NL&JVqAf){KuLHtS`oi?RPBk2#P5k{8HUMYlGVh%Dq2=}r?&L{g
z3{L1P>Z4W!tfXB<pZi;vy1n{shz~E?x_X*k`)SF;1dbpu2TKy+0YKvA4FNlaIawQE
zB4T6J`N<*l%X+rj+4h)793F@jFH*M9Q8$gb{FujHowSnRVJCJ>s~Mzx)W`@6VDyRd
z0EUW++CSsxhkzufub)2c_O8_z>f-)ty^ai^qd5RPc3(Fo0YzGIs`G@lIazz8z@An`
zzaVtG;7YL(;ZSI8z<QrhJb}S7w?DMV@S+C=*v5BJc4K*pcq09ORPd%6-72*mv#gXs
z8_ss(8)VQ)EeW=2;Mb|J0hBYq?iYzI6*e(E;Q0yxHc;kw1PO!l_5_c>a4Oh$<{^c&
zyz&xQ^to<I^UYD;XoZ<PWIIV+m;7?sqf;SRr6+`6jTJQlxH>@M2643Ujc-jF`0!*u
z?g+t31a`1`_3;5Mj1(RsAk>b{+E+c_n>PcJ+$h~h1f=a7l&F&Frv`}DA`g#|l(<^7
z>yo+E12Z@v?hO6`=<3$Oxd*#n{ds_HQm%lr6G<xtfj5|hIoVtksAijP3}>3#)Ow$7
zj0Dsz>jz-kyA^4m*dN%F!BYWShpbv=j}I?Ki`|f-0B9Kcli@$cq8R5??#J%g@Oa<2
z)@|mc^zXf?56-n+Y~Hbnza55Vv;G@&yqg7X-ly5;{_bcg0ITBO(|jbPnO6ua8xVLM
zjjJhtJ?5ZSNTLJ`LZ%yakAMpOKM-VOd^n-<Ghpl!yl<~QrE38WDGDM%R}O}W;4Rj&
zj_E19pX|*S#&v*f>ZRCh<gnpmaE)LfCE&F`4M3eoyTLDd9!Ky6ZUJ(;M4HH#?JUUi
zfFq2{7MVvJz?p%d)@aXWS-t(P@q^xF!7cx-Ll0d{Y%Icqoo@!v(F471FfhDKQ)a&+
z1vyTIY%7R~4iGAK=O*gQtuT9*`d5*Ib<WML=3ExNG%*EsI2=dM^gVXXikk?*RGcU!
z1mCA-dJFLMMN}M){}ddj#$)K@SF^&{V!qHw%k9oo6_;a4P-4qJFh_2eG$>6J>ssVB
zY>KY|#>GzIb{9zSCx9wzkr1PUcSZrS#O^S_GuaJDG8jRzfWWmHup!K;94rvPfYGCw
z)b}VZka;&=@MXHE07M?mWn?q}NNNY{e4xOcD;C@~@`vdP$-Sm?{kr8~5?T=v0~<g^
zGS>ZFCDY%z+87)5nVQ3Df`z+j18`|10F$c+IAUl34-E?w+BpQ(HH`<x`o~2fDLi&4
z-2tE%lmkS7W`n7Gqs?9(V`DQAQIGTmCY%9|9o8ZV?}wz-fB%xJxaJ#N)(?aXBymir
zu*tzQ9dr^aQyH@Q0dJ7<1&)ldz@Vi$oLop}Q1Dp6YWEiha*mwnWOo6jD<BUv1D1wc
zZAGWbGTxH&VKRsknn-y6OMV%sfjtajd~9+*je*=ZM7~(F;Qm7$5S!F~m<#b`M-c@4
zJfjURw%Wk4O}lHl%zjtR4rcdidk0>~6|B=1#z-#KN&o|tg{Fb6z`)WYkvT<gidh4)
zE%pyxGRSd3t-%;1{m9c7hby_j!<z55@vWz-B~U|jbfT?z5VV!!H8K#u*YIv`ZvO<~
zcNvS-3u89swt<=ddO1nQ8n8}@4TA?{AUsx<=0oYMU8IiT+D?mrt&p-d7z=151U~j8
ze%T-FOM%h|&}rsZoi~@qYekiVTtd0^dA=|(SNrVQGr*p{&Gy#)bA|U7A2L(2>;pa<
zWW3Su|BfU1VdD;;BGi{&cVLOI02Ub@p=>C6L6%+Og5lz6l^X|XdmLP>M9Bl@u?Veq
z;NAX))IGqx6`|{dmoEn!k)g%?0w_C&L;V5U;F#xD9W0(%6A>W9qkUURY#WWF{&-(3
zBW=#rRq|~%fa9U0uR_#2aQol@jTUmBz!DNmfcJ+hM8FFKYk*EwcMM@BOA>7gi1IJ&
zsh{s-r_;89O^$|0MM0dzmT2Hz2WUs1#Kz7-$LR&0T!q(r6?pQ$$Q#|hY|qGfEaLzl
z=@`rN=8dedzv+Jg<X?pp;2@tSjRb(lme;AVVZ~!mit{>~&W9cyMB*xzkS)ljp#h+8
zWl=2KQL^9!Ha#XlI9T{D%A#1O>MzC03veiSKSs4YnHKL`QxGISXx36&KrRT98RJ^B
zq3y^)+u>4z+{Y6!(IC;GbfGnfW){STv#GLn8<`<bzXa$4Ib?!Ol;|;oI4}{oTLWU=
z56|2Y#k(TO)ENCOps@PCP2I%cbPo+BLV3hDv4DQ@l@yzL*{ic%W<V9IoDj4P79&FT
z1OzwcyMy3zphnLvkL7p)I$aDt2s8fMa1Gdgl*rxGbCrJsDGY#0v;qfNV8783mG0mf
zxB_B$rf$Gj%KBo_v$Cfo1E`32bRr_X8wd}?!Gj0c9e`BhS!64q!dkuBE`2c;<RsUT
z#A%WP;@@9pD`3iZi4!iZeG;S&Zp%Rg{aLFZ8R-TbKY#66pMa5=!2d#nY9hydX$8jB
z{HaKZe$z}5BVbA_0Qp#Pcy0{fsRH9>_=rO8CmdmSfJ2fRIJXx9{_(jAM<7XAhK7bL
z<_?L$xTRLDUFI||6q3Y&Y~-so-Z_YS;(C{ifS!86oKg(ui{I@`m76n^fohukAP~<J
z4VfNrS(K0?BzlwT4~C0RzR?N2-2}l-EC@e-dwv8Qz95lnD)fcsY?V<qvR0W(8(%wp
zIZ~A-GsAp32i&Y+aX>qiG(rdpXr9@R3uk!5??*|!oYT)GkZSORk-1D6g>c2onuDF(
zf>F9qh2-b5P@=&f^DX1Ql8yWp2Phi%1j@LyePARAYh)-h{)Ot8G@2bH%mjpsTPRYR
z!t3WbzA9j3iGBmmtXUE#>~+;WUZ`dU2pr=9{jO#t2)}+#$O5zG+X4=CLncxb;%fl~
zJf=shJ$V*DWFr1?(9L@lyv~(N|05;_PX7<35hPR&r++m2Ad3NrHe*$@r2wM#^ugHy
z{{f7@76mnWpy(CKhBmDJ;}UQlIAmlHt7QK>u`8=QL~t%V*wN8Zzl|hFHaLt8T-c18
zROq|akYG1p-fOAU)rb{T-O7wQ5Q(@<L;1q~d!q55`-Ag9z5!IJ<|j*lSvL`o=1rwa
zSKQxS&zevs0L|A1IdV6XH&ck{8L}XtF*V;mh8TpM_8oU+i@<)*(6gZ1HGzS=i{9({
zEFKv1uiUaHC#!(s*rLdr2^7b6fUuNtvuQo{`-!7fVfqON#%<n@EfglIEN=}`!I`aW
zLtu`T;GtYY;HhHjOH}~ZV+U6jJEkNspkz`GM$@{H{q6yo{QdnvW-8UA|Kk4MNkeH2
z|3lJ6@d^z`^;u4kI8(fw98>1c0v7yf@iVW*$(`$Dp#~XQ@i0I+tTIs%h^jqizV73_
zez&-pR>yt5Q0Fozz`cK?l+oYckh*r=e4ctNur4?x>n!=$h!PIhPRK8efW0@GN+L%-
zku8Sb*>ZEH>c?~I8THW|*_Eq`N?%%0ct4aPgBJY-z^OM4g3m@kg~nyvi3}y@@Qbb7
z;1^2hqCW#tvY~{>G;ml@ezFnlvprFK1VVzogXIo7&62n2$2a2ewAiySHobaUkQK%b
zrU~=`t8)5C9FQ*4TF?9}#|X}4@FS!VHNrN*O8phV3eTXN9tYH<2RLq8m%TY1?8b|~
zVdanga_Qi3Ww`tRZ?6XN3mX<Fm-T;))(hDO*TcWE%^3tSk5FE{fI|(~@b>N(LV?k8
zIB%)HAIS<z<#(22Cu$$fJlPyWYD{O4@rB8>0)V<jhAi0ATB{$z>mP4ep@h=3C*~Lb
zfHE#6Tq5o_4!g|&vvT9a))Kvj9Q`Ku3eQUi#y3St<cAG_eb<BlSzZaAHkt(<2vw=m
z_JkrL7CFmTF0&Xo1VouC%Re#r1h&+0xS^QvPY_2AQ<LOzW#U>vQk@GZG2KnExDmXr
zPQHMuiu`x=LfocAg6Kym(V~>_fY5W2R}LZz$EB0&2Km$)V1-6;;){v}r{e^td*<!h
z&yS)7gWLy*RuY>$&gDQ>Yqr$t7sIN{yZu~57Mzb#61LQ8*B*#63ev$#-~n=h0FQ#K
zu?~<Nt36@2r3YJM0bBcWDwsrqi~PBDxg#VO&~|cxXsahfq=mePS-Zki#s0t;+y<@;
zlix~Llzd+Ts~O<<bpXk9U-uW9T!pV(vQ24|U%=L0f~_I@i1w4CT+lpn@jrM9#b10b
zd#QJwGWzvSn>TucCO~Ltv0uY8zbP87RuFL6i~EevY$*_!%I6sT979S9o_0wP4i#bj
zE4L)B42~a&7&cFvLC6-pp6;Op)ht}*?4>{dT^)E>hs6(wfc*X4ZLVgizIzLTr?D&z
zuZk(yY^V}+<(G3w+k~_SAcNT)2r1(u+Cf@kgc!g%W&9<f!_&fFpY6sXVi0$O9CED<
zi6Rap;#)&9y)?%_W}}~jHDp&hf5=c4!60v@nEV`P>PaI=VrH(f=a9G_o+}4CiQ6J_
zfqM<`M^AcCo@|UrZ~=m4#%U8_AY?J3zU`DK(XJ>cfnlwH`>cp$N;2%J>=$aupL%T|
zCL%4_;m~`y-RM?ed<vWJjXs&2?M$0>M^nAd6c2*!FU#Um`gKzQ<=`9g=c58@hDN~l
zE-@QKuZfUDIG8R_rIC%Lk60)6rh)^%D?@hxoRloEK94gFbYT0E2gaSyrBB12p;KR9
za=38HETz~skMsq7+?k>RjP1!#0ni2{6oj;G)R2gA!?`Mxa2f|XZH>!vOpTPANbhdN
zEsSMd7IN*|Fo+s0oiX$uNn-ec_(le3yBV+;F$~J&LhWZa$3xPJcVAVZ6CFq|ffB`x
zl;z%BoqLz*PHH$=91*w@>ZnISVyi-U|DOFuHk4NV@H9{{piH<1Tno5WmnRYB1b)Bh
z;1cnCC#hZ8{Ry&xv9;37*RC{ob}Q5t#0_PGUiP#xNjDLv)gcqWg?m6<ZS507`mg{2
z0gKkKS+1!)q~4T~fbiTR2jq1+Pg+e>K|GZM1Sp0{gNYi_$wy!roCEBnpedTx-<)UZ
zeDVtLv<DP$J4xu1eS}XK^&4x?YJZD;xxY9d0gngvT-(h(pG0>6xMH_APe4#54>bH`
z8hNEHJQ}b99BxOZfI2ZgRoF|1Jv%V-2ry+!a{w_lq6TkAf1tp7hTQt7^!-@g5lA8B
z{NNv-RAhqqi0e$WtP_%ELz<;+SGe?Web?p3v-mf1Koy<<Rmhj&PD`S*+Z?6aH9{`A
zV$*Ld&^t2<WC4bgvoG*519);c19)I8M*G65Z5Nd^2b%3JQedRuQ<t#c(c`7Scx~ni
zUR+LtC^wE(_YGPgC>sB!M^8wGfm_0*GPK^C%9h4z%A|SWq5vaQjuj0>Q6UT@e3nfE
zhwVE%f=R~T4bpO2kFy=9pjW7p3BL>p{v#AxzegX+prA6t>+GxGK`X+{%er?XS+cdU
ziQs>80>PEZZ5-<39!MC1Tx=#`30Sh3Zo-nwl9oGn5zQlPXpv}9aY(oFUq}mhUX<yZ
zkAtkJ2h^#=YYs@kf1)C*5av80x21y<@B>!PlB&$K2iFZmxTYZNdYvPKb4_;xE+3L_
zTECXNEBU2jWT~F%3bWX@03>-(rwa%VN(3@Z8w7Uq`q?2du8eOh_y&WjKGlA7;Q5q*
z{Ln!r6ofnsb_B}6jDbzTBn&}5GGlH9e8cPS>Woe5pk%s_PLHZD56l8GFmtaS9X>~)
zeFfEx+}HntTB+=>&-hiWVv(eLxb;Enko*DVmt8tq>zi5l27(8Ny*a)%ixgh#lWz)K
z*{jxI%e+7Wc?rQo%hK8HOmQ|b4uNy80@n6P8qHj=wXNO8NN5=u%EvIUE;R_)Vu2dS
z-rZcDHgf>CuOJrQ7!Q+q1Qh#%JX#bRR`Rxv8joIKm_{8cCOk!&%2-OaDnX%sst=?I
zx+7G0;OzfY>w(Ao0?G`%zHkVufNdRBay&c$W*h_VNJ04HQ+_@e_{NF6CgURgCLOuB
zXPNnxATHZmef7)ZKL-<kwX`!`32!S2MMc1)%4Z0B<p75bl<PZoZtFW4%8%gPItRgK
zV`Y~>DK8#mWl&;43pn9thHF9E+!G#A^PkI3qC5a~H_Mb#z^LB?LiRi)TT}wJ57dmm
zWkGxaq<BWU&p*h&wLJ*!lwJS{i#!f|`-B`&XIl9Lmdwy(dq|2Np%7C3cVEBHI0lA@
zgklXT8KuB7AhuYlm+plJ%YX_jQv}J21Rw<Rz`&k#kAY7o|6D+X^dOGDM)25=58gZn
zrKFyC=0tP~;kWwkn{*w8T<*A`-jM0JN7wep#;tYQj6o;iRp58oW3daH1U-xQ3ypP9
zQI1L>NcBG_dnslJ>M8E~0mZsCz4;m5Oh6tTskQUO;8o>;Td<@c<0iL+2Nrbf_b*Td
z9*IEuINcx1pj;4Z<Ow~OC_E-^t?OZyLJF_qg7>wxjo>Dw3q5`sc<-#^I?hQd`ZAAm
z8^4fD;61m90)VWAynHMr4c<AlwSXcK#{X;OKg)O{j}O_J@6)0ph?KcK<3#~v$a?zT
zmu5X6{QCfJSSU-W0M=}WrT0Mb6DyGK0xzfjm;XmUpP}PsbvQl<P==@iG6FB=A%$Sn
ze<`m54<<@3(UxP;wS;2aAvkz#nr~sfNLB)FxxL-BRsRwAQDCxf;cu$(OND2`6SpWM
z%Y2Z32vu%XpI0`*&fxjaQsdYhe_p;s@ojIFolyt_;TZ-@;m_?8L>pHDEkxkglIRS-
z+uPwj3-XKo`SmGEv@Z#A()Vjd^tpqNW$0jYobs*eD@(L0s0a-2xmm#{yujwrI9DUA
zUEVc>{El$~*hHJ`w<m~7qvkr`4{Rt6zwhO9*mcadL9V=$1x_s@AXbzSJvC?GC-{Ro
z(T5E#1=j?aLHpH)&m%@+@Ow@8Qa0s1bWlHV(2Vw03|pi=NVzIu;4YNlb2;o{c*lT?
zu|t4Ua~28mBU!*q@B><Gj*?_anhKLMTB5bilLY1+5k7Cr8x`S2BN+<<uyy^o4;K*k
zzZdZT6A8FOxw$(ot*{y}zcd3a;OikGS)Yzm2l>0%+ym~Y9*h@@8~60&7$gs3BsN9P
zhzEjJO20TfASXG37P3)Pr0q)^ev7LaYCi5}Tmu@KvcV+WvI(Rw+{kL~FV;^C9Mqk^
zVcU7_iMvcIZ|}dEo3yACtN1s#%;$47tjgzg5_spOm$$w4ud&8`P}{(Jx?Gwqy@gWS
zaV>h=wy}-Be~1?Q!mV$R_U-8+-TTq+>zW*Q$JhIxr!K4aR@lv?)-$i>k6BLxh7zNb
z17<GC|Lt?&J?LEEWgC|pq~tOvK&d3;j-Ko^En_3b>Ym18WO(ns__3*x-_#n5B|q(M
z+dWXTepLVOnth`CI-=eq(pg(j%%*<#>5#2w`CspyKkGSi`~T7%kFO)Z_|f2>7H+dX
zYN15s`0@9nu7fM`>lQ!yJ{d%H>k1{ym7|(94jxJy!h^i?OYnIdhNoW!ci~u0LZ*l(
z@jlT--T0pn$^t}no4h{PSudBqCOV3dzZW$WJ_!aN_r5qpEGEoSJX^9v9MO6SX|CAw
zW{iGr$O_(MJlKxb<tK=frnO2A^Nt~Gd(%F5@v$M1Jt97x1GG6!{AtfqctwxqQicLL
zGzkUORy5qI6v>9%JY_#~v^wvjZ$HBl)Fd?!@@of!s|D=Xh-vBG0t-U<m=t4O2Q1vC
zJ*&>bCKLUmR~<(BUB-u93<2~#qvsZAcTe3*Q`s~PT&!7)L+iOxAe)!9r(qZj<#?b8
zoktvU|C-~THMO0xl$gqz&m%NHdc)A>_DXlYPUL)_sHwWXYm#tUxw!NYA92&R`&rW)
z1w1Fg;%IdvJC|~#oRQA+qS+IMAU}f0C-;OJ`LQ1i>cmh*ZVSfA{_y-dtmE`w=9zsz
z>mSk=AGd`tQ(GQd!V>#n_~J{^(10DoW_~byM`5%^hVj>7hZMbC2=p5nWCW4%q%QwP
z7Fms=ZzJa%ZkDL(T$i)dw`b)(H{XOiTV-)0)_f=WY~ZpNuU`0CHjxgGR(CHo-6NJ7
zJ#z0}5p!RdQri1YvP9OWSf|(N$7&;UlxclWs^3O@pXO?pZB6%=CZx5SX-EHD6&In`
zFU35<Gg=nu5}#1OkeRLcdZ%0N<BR*6<5yZpN6^bWA5-_dLY5+I<y{>z$rqfirCYRi
zu>Oq>UyhtTCn0jnVsUpRZPUop@D0yPqyA^Q)#2ka29K6mHXl+xHlK_eiP=!oKF6?C
zwN%f<aP1f3o_7@{r8x~}mQhU|``6whhc{kZKj(6GR8Zre5wo*ydwWv*NT1zNhn$l}
zXM?Y%W(gYGf7L9=g!Oed@#<E%NJ{t8e)lQknLCl&4-2)H$qND3%Tc=NI``P>y0%5Q
z1rb5%{!SRG2d~X>s((_o`AVzJY=yJK-<rkK28VTt^WHQp{Lx8rH6M%osC0FofI1Cr
z$?8(A(hLbd28x)JCkBnm4HGsKX9AMFf?o{39)36=H=$>2_{@G!bG~V@(-Sw8*x`Zc
zZjNow<kBYar?Os(&7ht|pn3XR(vOx9Zi~~!gAR6K*235MN1xi}lph>foyG4YNa$K$
zLZ0$4d=@wKl~I}3FGM1Zn%F_J-Q~2^yVp_M`w1MP>*Qxhi(b`lZP;I{==2kAFsd4J
zBPy$Vi#ap#ZYd2{R_P}m6)zmlr9^Xx=9^0;W_~yM+2+`EN~f_keiY4x>(#7^Kd)U8
z8S?k|f<45{bT{(**ky{Q)5ea7?y<95oup#v%0$!~NrOb6`<vM-Mu}zvN|EAg{k&(_
zm`<?b=<_>|oVB91iJ@q3L>lzGDhgUF5uGhr3d9;0Vb9X=H~Vn>`)9)zr!4F%2Vc`d
zdbaAqz+ipfS>K5Z;a5$s(x7jt&C5&G^P}gdQabOdRM3CF*keSB*QE6DMYg^YGiV{x
zcS_E8CS>v8{ZO@-)zb0u?#t1T-1L?0Z#+BfJZx8FiZfywq?%jWpmg_Fa+Ae5QOg-W
zC*SUBo3Ha-+_RU75UQ1i>A1U$Z;xyAertFeYx#OB4OPFZm;z((Zpa4F_}#%j<<$mB
zW>cwrsu{c(XT8k)&vH$i_SJ%w^_Ypa=3ew#)ioa%UJnTb{p3J<{$m!|Ix)b*e#e?_
z+=yeB<SFL0DC^f`nnbqWpEOdt1U!`wk#f(<l=LgTCz?#Qoc~ZyTtyHy7s(N3+4vGL
zFpI8Y&7OUY9O5^>8iT?z!>b-nZ^(jk$bhiQkfZM~EAo}mqYhK21Ds^<VdXt09W9@l
zK)6OkBPZu~RkgVUWWr?GQoFWeleF~cpPNs9KE=<#{7Oy4Wx_b>G!$wl-fk$Go9D0U
z8m^ul`!!!hc)q9Y-C@+C%|v!md;>2oa+J#B@Zfe1d@`{jp^)v~Z7#EDuau7kgGc0P
z{klRag(Lq)4`Wj2y>e3T&x<_IFH!64_8QMVa97QX-jZeaZn(A&*BgEc$-SE*KitFm
z<!2b!mRu)qT1q^!P~#S2c=kxDCgrJ5wgoDO9QcA7rAzBc;k06-*{7DJ>Tk$7J5G99
zKh2=w3zF%1c)3p(^KTR0O1Hk_U%(0fHyc2QZil{3nRCP>VbIUDL<*x7|HQ;D`z!MF
z)a6=4Pku7a1PODK2nl(7?(trn<JMQQtM@HW(UKHhE#4uyeLKNBo6VUF+waYszix`W
zyb~ot{E)Qnp%qr&T*@;#gz45?vzi^VS8ru{enoLo^q9W~d+c0Dmt~<zZT2_HhI58z
zp#^_*&8LCu0+;icxoDvuIZE+5O60;!7bJxfrB%&~xQG{D=++gFqLYiKPcG$nR-Cjl
zq=hNm*0gVhjeq~EZoka^lbU&zdODPhMbPI&CyZU@$x%~5a<$z_QDL@enfCFUnR;Jd
zzh<S!_0MmT&HA>Qnh9P9<xfw`dVG<%?K;o>NJJ`P99JJ?W2t+^gc?c*41*!Js9<y3
zMf|?~QCggzZFThRcf80!JN7jpWO+Gk79-!>=S^{$=I0gr8(&UvZJVCq3-Am>B4TLw
zHdD=O^Fm!?p0Tao#z_Gi9tFWkjz2z}yTq&fJrCpY9?6yaHbwZq>`L{k`CfOoNN2&^
zj5pdPYHPM!xuu4AFgSU}ylRSP+ai~8Xz{#`A@WL9C%+BesTIZO=s4;`<^63Y_!D~M
z=*;vUOL0TN1V!rf&OS}l=RAK^i=(|&4#`G9bLes|b`jq4)#Yw8t497U#cd-ME~4$-
zRQ+2Y{g)p4tjrDH9{b&P_7b6)eLTmx8lLsi*}^QkZPX*AT)5glk8T)ec?{2gddu{D
zB>Qq{!RAipy13V5SbtMZcrVQ(Ti1k9f77k>7~2v1>7CU6`l*|N(yh-=pRVuwF!^Mg
z#x?UH2l`iW+Rp6;R@>HEQ7bfyo0PRk#`&A$-swb4wbNvaIf>*(UAQ%8XumgJ+DF;G
zQ$KLK#Wd<9?+b_?RMRk+_=`%tUC6{P9~60dVx7Rzol2APGnG#ZEp$kF0K5N@=v&^e
zbJA#Q)|Q_}HtN$(6)?@6iXN592mJUsN_}-1tg-s|v<07XAy~uq$35n<7<zi=&0|;U
zjz!_4Fc(7c?P7AppV;DhaBZV2+gLaAmODugu78Tg<Ud8LbVvd`2k+&nQrjO@BTUp=
zhEEv3|NUJ%%qbv>^o=^%1$gw;o+3JQL#Mg@S-yHlv==#qRe{n3PJ2azPLwXWghpO2
zPMW)CPY#V(c^WCGV7Q2caC@ht4OhfIWC~u?o%f=YJnXOaqYl8tzFeP<KI4t^-#Djd
zsv0h}Wu&NmSvQa6376(#ZJVFP^LII$Cq|}Z3;3TYeQUW^b6}xsf3(?-&&t2*TB5}d
zPZf%3Ib@~=3SmyQv-m!0E{$G^5mfLE90f1(QuGH+q6j~-Z9FqkM)R;BnA9RO@+^H(
z-B2yrdTuDe!HZ}yz9=JK^$NXhk-LU)*;Aa%8hmg)S1&6^C$+zZz6a6*E+V!twkUF6
zzIRrFXvA4EneT1;5)*5cN)U+-v*AC>wQ%LraQ$m#sjf<dR+fpH<Cfu%;$;L!1VB+7
z>;5R{^=Sv3gpV{U&Yn_=m%zKo%tkc^rL?$;yY)Zh%DwrWwNMGB)#xD?4Z~3~!Y)Xi
z{E^GX{@hhQ-8DOPk}iRxEJ6FGm{)fDU^Qh1d0OW+cdoODcFU5L(A9=5MS#C$6f7Kl
zUp>2oW_*KLyenwmYNeMg*K&gnxA~gTFs@>ECBBM^b|TW`J&HNejdj_-_~j=zzD8z4
z4)!#^wUYhlh?|XM{=9MTK@TrQaQ+UqKApI**guRo7K$YQ<;;4au+Mjmj9y$WNGxqR
zZLRsWy}a=-qyGISS5sVX@-j-C1tFseLKE|rNO+csVvE<Cp86p+^m?|m4YSZlt0k!?
zd#}n@i}q*uCakaU`WkhFUW?hv1%Elt{}G=UqVLMI)E_Z9P)*T>93Vtevo+!}x3C)~
z>rNxAtA1rQ<ltOY=3RgF!`p(??h73{rR&lkwEgGsQs-DUi`hFLQC)XPZcqa7#|XMp
z$+H^+;Szi?JhR;djfM?a+d6hA%nS8x^d<65?z?n&lei7^H_Hrv-o~vr=n;x`>IRSa
zkc-fzBo@7{WY8A;D3hLo?!yU{7zn-X!qcHrodT4?C-qm_&Vfu}V`=ftg$)mrw-QSf
zqn5ORl?;+~_eP4QJmJ*j6Xg%!%VMZZXv+uYjEsrO5tBKe9dt^J8opIFD{z;rk9$p$
z%&5$wC8e4vAtXc{p{i+M<m+h|NLY!p@l6LVdvYU3Sf^RNabT**D(1TGuyTz{HXZ}+
zBle3qPm{(1`kEbPlBaC3x_ixfGd_Y$3vQ%kfo&`JeZ493DHlz@$J^onp<T-GWgm0n
zQ7bl?RE1W1fKr34OeBYvuwq2`Iq9w@A*Huq_9j_~Yi^?AZuRYf{7`qc<-CyKGrFh#
zIEZh&u^FV+yo9)yFG6+5;%c(?Y41(W<yU0w_pu@&hnfCEkMrAMy|b4bU;~&yBlDRu
zya*PFOpDg*ZNTm9p#*w_*x%fSi^u`nKbgL6&hI#yeUR~luAWy9?W^ZdT9MhKH)@QM
z&r!B-@@PmvD;C3+QOm*5g~l?zGlRo7pR5nLZlH15-RtW6bQt^~p;r4Ndp2@zHN!x3
zlHl_<gVEmjDep`)ecB&Qb`1O_Ub{sk6K_bQB=l?^FRCZU|K*&C{4udOfOp`$;V}Ns
z`;LD`UX%T2DZpG&dQ$qPqBG(kUdA!YPifH7wHBPsMzstlwsVkGG84s<$tC|7?PfA7
zeZ*-YPDm-n;uB!c+ITH$ssp#Z%<ODKICR#^Z|eo3&wuBd4KB(_B(~}cpV2assHPiN
zNn{J~tWgjM2|I>gW*Rn`oaMffP2;>+ImffCo7F{;+;_HFayb^_);YsV;P4etM<;$I
z{*YL|Y9BKrhb#c)SLk+n`jskd!k!rUD&Z%H8U7@Ye+*(X5zt!%2bRi8z;OT>u@e&U
zf&y*|=cA&rE+zNy=<=O+@x=BE_Z?B*7U-VK#^!K{U0Na$bQs80h&74SW*H|<&$U}x
z?j0f;7dZyaXwjLC3^z%Nl$OY_p2qzfTM-<*&|mpzWXLKhR%h^9UL)NjKWoy%#4VPK
z4t=MQQ`gVsZ^~CRAvPlt=9Rgds0G`W{%sH7Z};Yr5U0wT4lpDZY>I`Rv@xru;iP#f
zZ!apLAlz72dtUG~O}1s1#>Ge*u&%ujnGmbZ59&U{c!Jpciu8&s@^@Qi7AMg|L@Ex7
zr*2xT2u9c8aK=~eN7C?7tjMJ`KALjPx1aB9Zda6nKS;z|M-7i2QEl(0{|jZ{O>goM
zNW6?b;U@6Fk~v#Xo8hiWPFypq&k*)g^7gCRZh{45rUWTrTW`<Ysxqtb4<<LOMjaYB
z$~@;goXw)l4u!@5%OJ}?yPBDm%bA&gNRe!dZGKPizLYxGKeSn4{om(fL`v^$?vWNM
zmn8daL_df`?Sr%Bm`%6knEqV6B2knj?UsKQ030YcVy35DI4E-Z!bs_%vStP0GT{?c
zT6DuzG!w}W7pu0is+2~CFT{Tn@0%cDF=!J`MMM{S+@c`#n_iG){;Kr<!Vw;{@7);s
zJb74vHVP}dn3dd9!(-eJ1A%w50%_-G7|~PM6lQC1tASku2jQ)a#dNi2)52j*jSi;y
z;_cIv_eMjH`u8sIls^$s=vrOf5>W&=i|@n2|1AHYK<wc`tUf(|cT#Q`2*)cKkk9mC
zHF#jRx5m4RS--))V%~0coBFgv*62|tRwkyhF*f(z2Q$y(_3M=O*2GYWmX48^FV&yZ
zOF$KDDbNU@#j>I_@|{`|%iG_MsIqlr`SPCE**CwEK!$KhxDT0V5yw+{pEKL{>*Iz9
zxRNoc?x8;iI}4vpn%fQMy9~C-^p|;`2(-3$V6{Uzy|X2|AC^btDEKZCm-mE2u`$gY
zU+-B61!WV$(~2>;%g1|kup%)TCiYDd^@|EatRJ889}vv3p(i{W5<`nyr-XeN55}0q
z9{xnWn3HdjZ`nr@FMN6L^O_pHIr-#^r?6%H%=8888e(;-#k&3uQv}_6G(_czLPJGC
zsd|JUa3g=q&fI(%a&!V_H!%k7NE;|S`kwMS2XOL#tW?S`IV1oymSw-%iZE&oYmTmZ
ztC$zGK|$h!)>-UKDuw{-HMihz=O>8Yu!BKc5OG5x2I%s4&~c;PKcu5Y$If0;CR;3@
zeJ{I)ic3Xh<eO$t-aLaUi;r0A-7}?~9!csw8w3|9u{cTM_K{dAp1+DKd@vJTbnJai
zG+~ouYv?;bcV&gednJl|CpJTfv)Jqq5Q*^-k*H$s8%9ofCcZg`&H(w}NlEhzO_J`f
zEqA9oDq%NmXiwnJ5L+u4$EXNbOavWk(*@l&T$*4EyMH!M==Hl-n2Rk@c=L`4I7oc8
z>HBn_@X`dx%-nad{W%oO7s%8~cx<wSO18DF)QGR_5Cq4ozI`hv6b0wL_5A%wI$9_J
znPdMMrJ+;fc8shgO_|eGO^c5|KH@=Z;2NJD>t|mVoS~y(Wu%kzr&Ab2;v*T-v{vtv
ztE@P~CrV{~(sU--N1flN)*cO*$c0B0*Oq;m4o%+uscS?ku|TO4SxBe(49ox4R*6F8
zUKDm)`!Q^t?Dut~UIWg8I!P*>X+tJE*F_Z2Ir&fM!q24?A)(BJD)9>Iv*bQ65rc5?
z5I0;;J<Ch>unkS02+uAzJIo$Af6tzibO}VXx#`wNzira}s*^sbbNJa(B5%*0gljp6
z=W0*D#MhC7d`~6ob9!#8?V@6TS?BxHR7v+r=J0+UGLg4o(+IUJUaD?%?J=<`NKUR6
zAIVCD1}G`s_)Zdq=6Q?{w%++y#r--b4HrOq_Hl}A(uq*F<;TszY0c}4P3hddf)W(^
z7nR2xYhv%dELxRubANn?`J@c(Fz4U?CY{6t54O13xYGDRCrkhq`lDBmqz5nFpp5Oy
z;>TSzvYRV#$6n}5cTdPneB?h3XX=$mH6(g*7%X{pkq@i-yd`Qk;cde;IXgbvPk6z$
zigqb4=R0y!#MUv+*0tTZAo<}$quj)+3QJ6^W#l>2h5tHPje+l%t4v*j87#ME-S$Tw
zE|fMf*W2X6(U5b7opL|ycN7R-uWu`zr)w-Fnq<=l(CkU~sm))Mk#NmDxem6E@aRs+
z{o#eRy@=6uj(V|o-9*)iqzLtQsJvCeANriBv=5dL9TbX{64=Mz8(ktu6%O4wN;Rk}
z4kmA|uFCGCdo4v*^R@QFN$K6X*~u!++0?D?$L6mM@7=m5v`&WMq5E^24;a6~Hj*V;
z_=L7TxcwEuN`3aZ=F~*1-j-xM+XB)eSBjDd-(kn~6r&kw+S%5R(>#B_osThiHL>Rp
z{iIaS&rtix7bdO%Dj(Q?KhP7B1&R5)x&lr0h5f~O@WY)FWDQ7pF2vNEasIz2&fN9}
z2pTA3v>g<-*@!tch_LV2IIuhU)y9nWU1rzT_g5y|X2$CEi{ZRq?Yj3Yc|Sz&J*xlT
zPLgzJuVFFL>3V~}E7@9pVuiJ8*9{FlsRj$w*q}S7jVIcI?&giOWrLNw(lje%>u~2d
zP|rvffUy2QO*3(I#22Ut;eYpMf9e%82<E+o2|t@$BwSadeAV5Ug%ZMXVb(>y%h}e8
zT%Fm4`1}dtQ=D_3WYD3amHr<(R8&j>Btd>&>q@R{zDNjap!3jL!;%mtiHcyrwRdX`
zTv!a@|InEWEI@yP$cP|o6wT$`DB$)Gv-Bw*?PYPukiS@fipLHA>GE-2C>g!@ZQXm$
z@ix3ohDS5Afi{sVPVqa(%d%4jzS>=eD*FKuhpvF2)#~e`Z9Psr36%Ruk&^i$FXWD2
zdUsvq2Mvso_*fq&0<U>4{0HqVY<8(_7xT!DW}`&<Fs3Vet<V1@7Fn9$FZ!-YT(>Od
zVr2-si_<eLjv^m0q~D&+T?yRvfZTb`wQnGO@r>OL3Atkd&oVouCGwtX(p;f*erSzZ
zN6U7@q-El2j~RVwmPFp`?898%O}GI~9sas*e2eLaS1Tk-hzG;U*Qbat-S%4;($QL9
zo?CW}ANS_C*qk&iUZtfHA=K6&OyaeWk2H@jSKj8$I`pP(P3w~wGN|Q*M7SNWX)j~0
z)$nU<sl(sz9}r+ndb<8gYN`bVq6xDbbWW(75?cAy(CG;x%3lZ?1p1QF|LLP*RW)FJ
z>ktGKU4N#9)-+T;%pKh(TAiFN`GrRf-Fis<p4g`s@oxCFt^eWm@!R>{*Y!$tr7LF?
z(r4sxoR-;NO?r{4g$z5sz3||T7pMC?gSEZCP2kC@rgI+Kpl(5`@BRD5X6~^?%fYX_
zKjVm8wo_O=6w;CMA^G23$&SxgMviiGwmew>rd=7|gjCo~A^GbGc_AF>mOCQUwT!Pc
z=XbK9FKkUKycyK`7TBg=x?Lu33<AKF-HW=Fln8%Cb8N(-43bibyPj{d6(_gTpbgeI
zdKWvQZ$IM$V@99wtkwE=LR`#$eD8cvd{O4H_QQG``Q;_ryxfn7RQ2^m)TRXGF@fMn
zt2s3^^z92pH=8eP^7L9Fmqf-;iv-((yKMtBvO7Csp~@9@)I;x<lHBr~ycPN0V~u3v
zi%~eL$idu-du4pagcmzf&HmYwxeSh0wv_!BJ8UAJRd1p!7ta+qn>233c&IT~^aDZ9
zs|WPH4mwQ$a^JNDHKOWSEAQcy9=7g)gdyqteaga4?%EEO<r~T<=din<fqp#g-k`T2
zCn{KU+_)B|k)B3AQdi@mf6sTD*sqbM2m8?Muk-@`BT3ahs=H6&Uk8Ii<EzwLik+BG
z^IV+FUmBM5Vzj;P8qE?7E&7-f*?$zTl_peMHn{9uy~p}!<hzoD-*YVrCnW!<d4dJ8
zQ*5QjYo}&QFj#_6OF+_?o3#I(VL)<#rPPLRGEVzDpRMPGxtHN$G3U(<O-ErA*i=c1
zmv=vS+|LSh7T<6+bjh$rmkr!HPg)GOw>5K{HOjbB{(=;kaZn2zM}-gu7nTU%0@1$I
z4{h|&%t<SUzUB&}Wv=;h>*tv@I*2EW32PbXsn6uQ21cHPta~U}IbM<`e;d<lMRwoK
z+Qnu*Y|wj>4$nDcCeqn;;3L!t?MYzBfBtn&lYP0(jYYz(Ma*To?n<CZ%_Q;HLN@r?
z>eebkXB%EGU);y``aw$M{7kn@yd&HcD9(bhCZ3+4g>rd5<kR>HcJzpfyU1PFGk$%1
zr@L>!YGZ#w1YFVX{C4El_g~0^*|9>+@0Bm_!g#KZFwuUyvSy=BhQq-Y7a8tD*Naj5
zs@z-NDz8^*1#um{iORcbXz_7xRFUSOf#LW$2&=8|04$nw+i}W*ec2Ng`R~L9#VbVp
z5P%`6r}$or5&gz-`2hrn{OUi-8Cu&HncXq5<Cl1Pl+>lq%&Ote6Q<eD`-`bi>F@hP
z2K2J$Nb*h~XI9m>Q61aXXQ|t~QQ+b8+R16;{uv;sa)mPFR*SN$Rq_2UBGO?wziBc#
zTVlHMt)3WC)p_rx`_uM%@a5TrZu^I)NaSKhX*CH0OD<7n6f&S0ter2I(Sw7So^tEC
zdFN?;>9xr2K5)~J-QccgM1UC2|00fIk!9Wa?jRkZzUU=j<1PJJw8jTGMwWVfGuQ(b
zEHigm43CE$CJaS1JIN{n;k@w~uXDzU=kznQxLrOyMZ{`4h#My)WGMK%x73med2wVj
za=fsxc1lihL1if5TKJEVUr5!vMoZInvcw_&KB7zb`zl!UcNYizZ$6k4_K4~U243=M
zf?`JL`1Wo2-Xdz5ZYwTr_&2PXn{ophGkK>A8&q=?gz#5bi_P7iM~^R8ba?y?Xi7S!
zJ+DMdiLke`RFDw}sRP=~s=|?tP5Vaw^xEmLA|<ajk#@78d-fTxA;F^tcY=NxRaLN4
z8D@wz;}-^m`+N9~WfA4n*EIKuaf%(J+T1SXb6Mzlvp#~o=dXus<p~XYG>6vI9)Bc6
z#`Y5NzHZ_r9Xjv!O5eVu`}t6y7%*!uS0$wF53zXYjV1mE6q5K;-p5wwCPLvIaZPIW
zO8dF9#XwDgUtrr=Jc+4#n$pz~QaW>q;Lh7^j~_QTw@SNW1;c7lGEJQGg6q2i$HG@~
zj6?R<1g>R%dad%7?<41Z`8bUXDg2}vNYbnqqe(&0EzG)avZr@5<GaJ|==QqTied=|
zp-SD?3eBm)MB}O?&S5<3mlo({{Z_wD9#>d&yr~M@vQ<tmVh71Y*<=BCW2G~R2pLdb
zd%I`X{6zl)Y(_?$=piK<Klk#=-H##LJ`-(U0=d1}n;M)6GZgRk0F^PUE4LCqO)95@
zSX?>gAt|jCo1o4>IH?6r_$c38RW0f@k{BBFrCTI@U+j{WL&Uo6We>V+;>RHvzy8tv
zfZ5Y)5~7#>(f?Sykh6n!Bp34d-i-h?k;7LrYcnl)qoSRIG<bx_wikq1ey7!I+f`Q6
zIj8{@r;Bl*SwCTaNU@F#VQQHDkpG9+gEkYLMEF*RzUK3G*)?&qP7M++V#DM;B~*W4
z4?y}R+I(-{=G^;>?yaSc5lF5o3=d43;tDdkT$P@gs(PSKttcrFwnlk_Tx;`LiqdYW
zz(MmN5^()|@0L<nPhKY*mnMFZ>*Zl=q|!FBZ16kUy<v}GMPI_-(p;P0@Lt3DepXhi
zP1s3>3A!M{Uiwx!dhygEf3B(~{9Bo}Z@gF5`aU_vHpYdvw^Fa~<byhT&uf+cAtc1@
zEz@PiBYIsG-(0cR%{F6Juy5@b2sf8n&2&}Ii!HTYz9FxCy+C|W)6_8wl|p07d<m{o
zWx(e1;Hoi@z1|FkuWKW91>hqdU;h|92&}WrE*a;tTWNId%L8d|nN7=pN&JZe*`m_V
zv{*Gz18zZ_QlLC}%gd8)|02jw3h(e9x@DG^|MMktr0;?x=xx2LD!ID8vE`NPC#bi!
zdxu9JeB(y9b@djbO6|yK*BdXmYHFF6vCXFT#<PRz$p1>D5S0Hma<VHu6@#}MZODG`
zT6Q#}PNfm7-#&KL_OGFXT}-oI9dX~W9g|)c?7zIU92I0mzbZ<{A2Qo4s82J!{*3Wy
zkWigsSc;rYa}<xg`NO4gRk+o-1jX_Eahr`ygb>-zHKL)#-!J=4*r5%byf1Y*Tw_>>
z1aupMgSMvv{k?kBCv)-W0PH|N(P#LhZ_)LI{2-yL>Ze8u2<s*PAFLLB=%cNvKaRqh
z&PJGSnW}$86CXQ!P9`fJ{jfCfl1NB>aa|R5+!Wa&ONsp-s1D^{HbLP+Ns8%<X@@mi
z;fRkE?}@gJk6AyI1SviD7cnaPxPbe0f@;ug&7@SAq12@y{{?-=DPeq<Fp{?8WvN2G
z3u#u<?9$BVv@Q8Vxgn26yxm)yhUOa6D$Sp0DsN4ohKCE(9w6fxGpeJ#HJ`BHYPgnZ
zTGraEkB_q!$JnlR&6w|N&c?_#Vpj2RWsvoekb=M$q!-K5O+N5D$VI+hhzqvkdiS0H
zqp}uzuW?JZr!S(Ol2@F1Sa=r^(OU1rr|na7SMEVqe|B}^PJxi~=~1Z)jiqyRY)I6W
zl1IZOO_Mpz4mSxMsLD_NU+ukRP#r<Hzl*yC5?n&i1PGhp4k1B<C%8i(xVyUqhv1f=
z!Civ826uONcm7RsUOA`EmwW5ht-7~fYJY&)!%TNi_w?%3zqOu^(^#7Bw{CB<Dv8oA
zJOy|HkzsG>QP^a?e?!5jb?eaR-jwD5XMH!=3psPFbwvy=Dmgh>9%4S@jLvMt*03Pj
zVN}uxbTJUTRqxgPPV+c}xJtoxifUuUp$~9~K*5sZV|n7$gXO0m<&v_xKk78yg2lzk
zd$=29rf>l(<Ub3y$eVXR4*5(}d_$kdAbgLCy$e;d+G?Sy=Pn=(Q6y<8dq_H>Lz7W6
zW+7Zyddsp?!1)pJ#!IUR#)f)C&Gy?qy>6$La}zy$9bxkg`rBW|MYzE+w<n#c)9AFJ
z+CEu3!yMk4x5CA!-qjQ@*f;7w9prxveGOwrZN}+bEb>KZVv4^p<u;;k2gkr6WXhkO
z(>#(tEOpp;hbcE~$dxWhR>N`Lx#Igg31MgPOI0>1hjY3o&|$n-{F`kQX|NAYwZFg$
zqy!aq=Xqfx*Ik8`-hd#lsy9NmuhmtM%M9%0nR9)K0pd21S%d7bK|f_&w_q1o<a|bQ
zfyin@Sx!{gNQTi+nV{L<?<zAu=J!YBNWt!}bZH#PcAg0u#@>8Un4ryCHC@>J8UB&L
zZPG`C`7WZEFl~Z(Uac~Ad=FI~h&Aky=b)ar=I<#t-o`|7@DLzTst+Y{_?H_k>HXLc
za<p^P>DM=G2d7Wr*o+Qyz%o&=g|QpV_drD{x(*vSJ~N1%G+17B&Sd^=X?@0eEAu_7
z--q#Dz1!dTiz+r)cp-7rbXg38`|k^Gm{8O=fBra@8`<`%zJ(8X5?t+G^~HBy@AoeR
zFSvQ1p1K?1y3gFe`Hkc#zgNdyb%8$zL}&H&qJoaK(qA1UwDM2R2i8XTBEUB1VPDW9
zv2Y6+gQhqt`Ar3WCfP!@|L($dR7ylsjq%5g1|{E8Bl*&VS7f@M$94xY+Ef`G=f2)y
zB;dU);$%&@OzGR=6d!o1WNGkgOW3i%l*IAy3VWQqNE#njTCP^Um17EJGbvVHewv`{
za|{qe(Y!*lAcI@vVBPV|)Kq9<ex-UK`BMG>eJA|hhokkT*tr$cX+nqC$HG|@IV*@{
zyLY*SdF`tu*>*^8?!^Tz0&nlwkWM>Y%fmyHpAD84+WMnAs&C`A@hR;ikIQ_CLJK6Z
zia}sLi1W;b98$F|G-LnqI>6mfs#)DFWZy@(ic`+o%rI!wqSItL;2U<(9&M%5z*Edj
zjpnd$9f%>vMR2$shU4Pmhg8KNT-8pzMk{Wf0=lWu$TZE<FZ^76c9_EV`xo}RZABuh
z)z|#ip(fXEeO7QNanP5x6B?$YXj12yEI?W`Xoo~lrjKD$%NH7t9E`EVWziBW_!j!}
zJ5p)lYxPpP2zaK5Stw6$st*Kj45IUmggX}QaTI2;SQ~dI^XkMu%apyT0r6J)m|7c9
z`e+egt-7+PC-zrk3-crmg7nU>lOp2pVhlT#?IZ*F5(QQV1hPEJgIBQlsjNk4(fUe4
z+Tu!Ba9}^!^C-z+YBbT!oEaTvEJk6Xp3665zu&#^B3Qc-=OwiefKQpX%iRgRzs;*8
zY8B|2YbPCqWeWUu0~Y0<fiGrNlQl6}p5c4lohOXbK}^OZe$vfFFi^b7Vp=r|@9pcn
zkL0n4oFxFSyQ#(2TFgzVYW8`jGQ!E)Pu%{ZCQUDyaCyj*|5~#{SA1>n3SXk>EHjmb
zzkD=%MYSuvekHd&r&W`6x4fZQkYnzrS#75tTOIi>&$HI7AEczfrOfct$CMBTPH4uF
z;&ius<Lw!I@+#BWudZ$(kv;#40NU(jhBGPn$CdeHvGun%NFB#YbWMD(Qmw{|9lP0)
z3CUV(XKgBoyX<2xfxIrlN%xoG3_YEc`=gCN6H^rRa}Uvx!M9@BPg0Yhf^!pW`Wbb;
z7`Jr$1vHG0a^<@NsnMA5zw-0_=Z{J*@Ar%Sol-5K9_3&!H-Gj@hMd>B_~X<Y^%@uD
zjn_9MHERt9U%R(4`6rT>@Dde9yqbHi`LiE}Lvf0Cu}Q9-CHP)#71jzD6VSw%=}gh^
z43c9kXf5pVZm3sv^R|<X_R3ckDIIL>A|8vM_J;C0!S<mV3>f+oT#>m7y#A?-TK(Iv
z9=8p8R<f0vK?X?yzsY(btx9Ecv-tG<{002!unf`<pak0dTp#yxtJJ~8tM@(Zntx~d
zUrYPB(*p9u#_V(ffgMz9rEiCA($4o152imIjZ(B9*i+oMhRQ8a+=Y$I8R>SvXYn+q
zD(^B#=pWUZ-2QP>j&b=3WulyJv=rtq5uYkFHN56laLqO?PLSH8Ic)IrSQA@GJdFfO
z!m;ENpu%vN`auGg_H~L44N3ggQYbMFO511$yOv965ls2Q+;c<cG-!iI2Y2GGS1VnX
zYbGmCezIoN8)Q3Fe<!_Fkl=NOHXsCxzp_%o17;;{C>mUgOV7`rEF#eDXB0e$_Cio&
z4Kc3BzFJF~Ca+}Q{GHimANJpwA`j%r#`C-J7otb+-Cv|v3IKsVK9I0mLc+&9WC6zN
z-AWJGEo{<{-%O)G%NtTC9!ht$YnpZ?ao!&#92V3>j2&k9+ECaSYkn5Wqe{15%a?Y4
zr9MTZbwKFzsy-3*e6FNLF^VrT?7E|-aKG-08V{qB1y#`89@uu;h4!iU7}iLZsvY-R
zL3e&!_kB3vP-3Jyo?Gc~Q!j4O65nx~wq;M^Z^~rfL*8Z4y{UcdZO<zg=wiUu`177W
z;`g?<z_qw{n7t<b$LuE}p7Cv_fu-pgwzNLQXpVtQmRaUahh=|z{jQ3U8bE_wBu2p+
zw?1QYYrDt@Z)p6=&?xzSIxGgk<|Zjz@kiKoXjBpP?~&e=1ZlD_@3*enhEt@Z8w=bq
zCn3@U4h~Y&4Zw9&jB^s)FJy~0IgmBV{VY?aTZG5CWAexP1U|)I&qWDdAc#1_w!nKF
z3{lavAOL{mzWmWLrups9E3)$z?QThaK<>Ho&*I+O{;>t*m<g7Y(N60gl|mm^`|Xgx
znWcGaIae`Jd^E?}ib`z{ydPfDZNIH159(d6B1xw@ndp7J_Iw(#@*z!qoWaRVy4t=#
zy)4=wg0kc4u+R8OZCE3!pR0WU{~xZD*nszs1tR-Xpp1VNX#IkWQBx`|4<4iiYJ}p3
zb-?p_Vx2v(9D}j+tFnC|ym55?4QAtI`K8%pzzH(4cmKf05wnk$>N?-=Z;S$?t?IWK
zL4)LTXt9^<uM|Zmj-E{mw{WFM?#ar#t4W6?``RX@?gR7dF)ZStwiO%rcdFz5NT2l{
z!zM7?RltV0UC~=_cK3Wcqef~C&-@`*2dZ4=A7vXZ2eCVb?Ah=s4x)rP{pIGOqWHnN
zBVx8a?4tW#_T71=MmO`@XEzFKc4sOTWZsS8YT8=u@UX_aH&8{cSo}znpGj+fGq>UX
zHK9%FMd%rTbE)^w0<Z1e-+uEATifIJ(6yq;57Z&x8=Tf?s(P0!-oSA9^z`qNVgV(6
zJGcKX2;E4xST)*t0C`+{z~F7m;t09C-cek9<V#iDt-If71H8>{Vm~3p*#>LKowo$?
zGev9mz?J?dT;xDqI}kYfbprm>+vi{PR<d5G6LqZLXf9*b1nSil9xUa|QX-mGk}cQF
z`<dRwD8VSbt(&s5Ow*09Xtj%3?YS)e0;yX5qIZ*%CTi)V*S~~u7Dl+~o#&F;$RLz{
zf|m0)`{T}}ly)<|WgBa`c_Z8P1TIB$S?3z~>g;C1lvIw#A+e3=-hXFDt`vT2u}5c$
z+v8BWrPkkk?(0cj{Dja{+y^>H&$e(x!@2@^Wu(k4{8kRpSl;eg9^N!)!qw7NC9zJH
zO?cQgNc`M}d3XSu6*M8#r{fZC#XJVhi`~;ZScNAvX#J_2d2EEnFDF;gh~mab#y&kt
zJnbio)Ea&Sd&hPk|4bGA-Mm+SE+yb{UToFzHh#|4r=chro2H8f=3JyJN1sA2TWqg}
z_{A3>4l?rBkC>Pl8zYU#kD{gU>1+7SUGD$mLSf+P?o}%m_umP$<OBmG=@Ni>{b%C`
z``P32vNcsf+_$+M70=fHYJ-8>R$I&CD|!>gosGb2^|~DH9@BO731^hc7(1!3h*BM=
zq@g!lmy-pTRjKE6-t%Q%^~m>^cRoNX;~+)G{WF#+nakyaNFuqluvjW*RFcBYUeeu0
zFyHwmU1o++D(<6OFO-ajo>$XJvDo<y3|dA>6m#&8oshlPzT!{MW-CpvxCkbyv*)YP
z5oJ3PPv&1u2Jan!V2i>NcI#!%ZpK0olk_-N_5gDlL<F_L{28#j|21IgwFkfGm{(9{
z2lR(m-1C>XqUFz~NAczLLMnQXxLAJ_`i(x2Sv{*3dS#c-ScUiU3(4_yTLr4W>vK)%
z?pH-m{*enHrKlZBN@sm(NHOKx^0yR4ZVGlJS~QEjC0;~5vje6_x9RPv4^gSXGn>Iw
zhjg_ciQ(GO?r%plRD6`4JZjWmF6377T<LQH1C{5#SNqDL>7A*C%l-n#ktfbixy<28
z?8P&IEt<OhrLf7rK~VuA7yg-);1%@pPmQV{YqVEb4zY6EeU$O<*QJnq_@`vR(bI61
zUZf?-GsD-Q%*KB)m}l>Al)5GF9P7HHVs-YEb?+mEBPu@Kn)DR%s((_Ns~lz{o(6}4
z#iC^9Q<^nkQ?DnFFp8{BMoUU+kNIchdpgfQiSYq52Ev~i<DJ<5vKjTCaH;>qi~WBk
zV9q}dzQ-%7=>a)I{F}IYt<@SzJP7r}N`nN%i~RO_oM0oMK#XAOtKO%`APDot(n|=Y
z6b{Hdk-#xR@fhxf1=3I;k$(_=|BrxSLT0!Z;`nW7wzw?0NX3=`xt;BOj~g)pqDaO2
zn8?Dy*q<LGM0Y)zYWs1tRu*}zvqT@~jV0=)U;MMu9$Wt2Akdd1hxvETi>40pTv9w{
z5Y{XJLrxC9r^F`^iw5hmfKtScT`p;l5bu+|_!o~cP(F!JJNFj+&{<@U=`5Dg`dQM0
zposi)oYea0|9tNQz7N;Zqm&f}`m7#O{IVS$Xf}*~W`iJbaZ12B&%bHCgNt+H0Fodd
zZ?2KxS1=~TKejEI%Km(h4Zauhiimhe@R;GkS*x$`1a6XpV<73}pUPDugUVgtXk9No
z2f`VQ?GMcVsr-MQ{C~Rr|HUyc`6wCq=O+O2m<8ipAdcjZl7JOP-o8^MP+8;gFHr)|
zT>H%3djWfSe7F~_{}oCe3f^26aLhQsLu2v8GY$*Egjnsgqm&j5;m|(;+>uugG%ddt
z{#}mpgjKhxqZz&E)nl0SBIO^8;<MJ=!GFS_{}TsppJ997*I4S@tHNoynZ|QEn`bq5
z7XP9_>!#8yJRR{uJapzDhUa3I?C9z<0!8BArBG=qx3xh)QyqEoYLR?rW*0M~*b+@I
zp}6mmFp8&XUSqg-Z9`aWaz5vY=uqjiWs<Ng*vIW0g2&yp84|<0I@5F;$1`p9xO>gr
zn8cG=f+Kk2TL-DDl}T;KvN?g5^68Fk8?gkzCVvC;Q#agpCsIEzVD7Aq9yE4me#N8O
zAi4PRIh9J}bt>Hza{}_I%L?+<R_&+bKGm~KmDq@mpi+b0DyuI87*{8&Of}T52sLMg
zZzHptV69C3f;)!w+K!Mv?8`|vq;OKt2fhu9jU11P>EI@nM0`vk@CFRA$V%?v9}tU9
zYrU6|e-kAfJ{VZtR^7cRFu<9rGx@Go<1)1)2J-icle({&{v7;KrPuH>d!i=#o}<*g
z>bBB3F{5^!tL7qb_+?1w1x<v9p?2~6dJ$`c$|z0uX<SB;xMR0WZG)GeDk3s!UYXXN
zEOB6kj2t&xOxGJ$?9P{WSogA@Wal*2?g5{TAoe)pQv~gHGS<LU?vt?Hg$A*Wi^Bo_
zo;Hk%vx{qQ1M&4l^P^~rnF2>DiE~xn#yi*M(v;LHvuI}b<QNysXR+M-&XiiN!R7Zn
zmLEl;ss@E}>l?=DddePdsSxeVhB_f}4zBC@DJP;EEj4A=7lYSpYX|&NU1gsLyC?<}
zkJY@{$QAEfQWiimF)@$~Dg3E;cO3#VocDIa=2fe-{hn=PQP+x*&xO;~C)??;+uJA*
zXP2!JzJ9>?LI2ggInhwYkNk&QA;;p6j)rfJDIV~SzH}kSJw^CRgB}_0s_F=*pJg&d
zwcy=u3u~j*sbBAd44*q@To;`qL$~@_Lds`|<2okEO64NhTvI}!KUq{;3mW-^##b<o
zOdy)_p*M_Q)F1Caf~D%Sab-CBG{&Lx+?4x64EdQx42f~$Iak>EeY^ck_gC8)N0QDk
zbcyxuL*b&Gm4OMPm)~YiFMf68N<#)mZ1=f%=}goI>sC#{qlO)qZcLprua&rF2-z{X
ztoJ|^Yr+H>f%s+KT5k3&xQo}~p>%h*Hg?$(n1@fhQg^#jrNrh9XPvxj+{%+4edALi
zSM`9XK>?kDDtAaV40h$;kJHCd+vB;4o6#vz!c%?SRh2~cPF{E&#W$7`0h0f9?fHJ~
z;jX9YD!4*%)Jo!a7UoNZ2uP(~s?YE9v{>`ZYJw~wcW{xZ4IrJz6(`C)K*KM_w=a{p
z|D=j%zAhg;+xbb+*wb&@%j)-Hpdlaen4Y?Q42@qO))jVp?K^r;Zx$ne1aN(Ic@H$a
zV9)sk73bTO{{lvzjMVje=|RDptk2KJ*rXK8P!z>3t`;1HFC+Xfa$`m(6u-<%1b#i&
zxR5%S>tn_*Uym}pCY~Novd2AC^QX#M+b%!4UB|-~MFC4l_dZ6d7P7s_ZXvi{U#|XJ
z;e9z~_+~hyuVn0Q>Avt>V;}||Y~VG>qwuK66^h$eEadEtM3SNJyx9D(?tUreSg$NS
zaQ0I^JBFK#HtS@KbMpkAf{Z`^Qq7sO%V9<G*NYE93~eCc%Ncg8>q1e~0_JW6vk~Fb
zk)MsX<)1M2=6;I&N_8wt`MAT?L({(cz~g#rKUL-kL~4~p54V<bpE@|BOD5IDJ))76
zI^3mFhnd;NS?FHb_ATz-n^kCS<ED_H#M06k4y!$b=+e^R-(`(o>4vzcIFk;_chu+c
zJhLK#r57|lQ{%hifqa|0DJ!r#tJs2FXBb5Oz-q2u@o}bRk+)i`UfOr;O_{*K$j@|t
zn}qmMeY{|v?|q0iLU>g+)Dv^QV>g#Jb*pbV8ORuqpJO!PW4gn3Vv>aCjYV>?#C_pl
zp0&{RDrq9>1`jiuY2gWPh5OfEm}?(0<EU`+pEPL2VlSER$Ex$^To~3aCU;o(r?984
zHr((PHT$#;W3D*@og(c?oRZp8guo}uj+!vt1bqU|H%9^ck>}gS8GF%;_0u<;u+u#)
z7dPA15dCxYpwoj?chMWGiykp5(fzsy*)101ic+DvGy}+B*21TZIGeZzIaG{=I+aF4
z7n6uZ1KP**FS;uDX|Q9}N*D#}WONsPqeLteIR?J0tN3@m5vAr`P7<U)r>v6}WKoS*
zn=eQg7Fs8WBR(jbpY!Iso2y+%eAqk;O*=8cteX8KlMr_I(Ozk{v*19<fH&iw&MN}X
z9Xk2&G`{qUv;X#-@J%h{rf!_qd$*yI3v@_Z!H9X#YL9)<!OJV{H64OODP`6-)<rD%
zZ&dwQT_jQQZhWhrImt{*tFN=F=p~DEK1sj8;y<?t!#2e?^>P)V4GFU^&?LtqMnd=N
zj}hrn7+vtbGMLIQd2?64?#23W?s1XU-E}2S&!^9y-zYTscCn?lpV@poR`&O+9jUy#
zx4}6}Zb({mJ~}G%L3A~#_;)u}+el!KzfSQf)<?!sGlsaz&SAa<NUb|jb0ot;BQH|=
zcvK^n1Ccl_r3ikLnVwoLKr-451~!Q<$$vAVq5X){BX2bjZ$uoKn7TSEq8(0bRN^Xa
z<s&#kW4#s1E4)Ehxw`^?xO`<jT&Jh8$C@)#jJdY-#!|w67td54mhH~?+mI;%LcbIl
z{}0~|)``#IDX5h>f=S%Fv7I*{P7xCsGs6SS-~3s=&7tOvF?{Cy>13$m7rM#r5NPA*
zq2?~9UeicLTNx%JOlfuq$9o!li6|$=Swi#Gr96W4&sBuHB9pmXjTVW45pKaLs?ZYg
z#c1uCP2~my>_Mn8nQ1RxT~0DPMdOd8<D2?%Y`!rtt?NG?H;?%u3nfHr14(r$@WaI=
z!X0>qFFF^;T*X3iRx)ud>b`~OY@*OOetgyDX~e>vN{Z~agqx+gG(Z4NQOL_bPh151
z1*p!TbSbZHYnT(B=-ax<m9dPoSX~LLXS$@n)|mR`vK4vil;@K|Dr|s1fScS=Zxqw^
z3zOYP4*$d6kWxb&ms%J{h=%(M4ad4Sn;V_y6`EoX37PPY{?J78h%5?;4Jv7=bSBRr
zzI(<{<{$_<i!q5m4}DXsD|Bn?Al7}IDaaV27Zcm8DxSS|_mn(G=)~VX>Z-MN+xf7{
zdjrK>(mC4PYVu60r(tLI>}XyjAX_YXVeFz=3rnI>2ABS-bph_y+diS5I>j@M*SZ-6
z_Euf@iU8(bO3T})k+;uE3t?xI@#fTetM*hxkfymC>?v840^PRMu$_nsIK6L^-G$yP
zt)MKI>=wN=*hEEaoFRdNacpOi)!UqgEmIZt%IKO_eLz#e`zwA@&eVykW3EmTdHJ2n
z!og$s)+IO9qr;bp0c}t)b-1Fz?KMRs@2d%eZroAk7TS#(_Xk;u-T3-_)r%`)f~ky1
zJ_J}={I~EZ&C~d4D+@=NH)lDWcT~M%LadQyAvroebrY)6Mnc&%nneWCN?NBR$)<h!
zXtzeG6OpLF+pSF~DI|VwC&p>J)H@t8n$Iuo;vkt{f2zab+qPEEkb*5v-MIEI>B3$N
zwTK^<Dj8G-1`wXopXc)#G)~NKwSB<S_sK7I>F1a$X&X)j*`#c?OW=^?d*bvlsI@p{
z{H=T|prHJPTQ|1-5G@4+1Xr7UUvmv4>lYjcO=BZ)AZgC;jiLpyqK9@Aua((vXDVjq
zvP9=5`#aAW-yu~-RP0OaZ%j)YbZz3AZ_SjX;0<&o{BE+ovKum~N#nY52-}0c+Lhh1
z<Zbmh{<y``XHBNxdOc2?X`re#ioUWhK(3?>sc@#a(wRnLP(yI5A25!`@tK|WWj<B;
z9AJ~}{iq_d`Y$^dUe(~P_}H2i_O9;rZEL1xRNfuZWqnUQOAP#X)AqjFMvtl4b4J{A
zG?Qm;rNQOI4>gSvD<tWV0Dm7uxyR&JWFsw2D&_%Cwz|1rJJ?3cALgHK@@COX=T!{g
ze{<ic4}?xfRlv_z=a3g>2f@^Yh%Pmqz3+*zm5-L->f4ZvQ-?fDV1(s%@akp5a$@pk
z6kb(23~*)gH{O}VsCZ#dICezSj(<qRi1UM|39kNVG^WT%Qj%?mhmbsmhBR(F?LAi0
zI(>YBYg&~$`~&(N`w#(c=E=ffx|JnwB9^!|lsSP?3aob}b^7`Zb^PMNSn7vwB?a^2
z6Q}BR@-xzt=1Lhc`M>2-La~Gk`1i+{Dtw^z(XM|!F|M8AQ(H(sUYY(xV~1$UqeF_(
zj@rytdh`P6*rT1y?#0}O`G-=5j#)txz2>-iCpMGGZK0z!>X?ZobEth7k;0QkH4Np7
z`Q(gg{9waRpR_mv;(2f9rjl{sluyGo&47BzQ5Yux?6E_0@T|oO6x+tQJ|8sqsr0_!
zIgTdPDj_*`d9OMRI`*mDN8+}8Dg-aBz9gkIh-fg@0HV;}hSzc`$|}|Ck(sM;aPQS@
z=D}>7XnBrHq^#X$t&`0)orc{IRN_PNxLw}2J0i!}i#)p2KJ2$HJ(m?lbTUVlV!Hgm
z=_-fpo$nNgqgJ53SIIlcy|kCjaG^4;*ZBJBf;PaKXw`gL*dEv>z@Ue3iG1!{ykXea
zA@5`ub63|6lPqDARp7D4UU4vx1~2l7^+)g>jEgsN(YqMcUpkD65%xFIXOeH47Zj9>
z(RS+xD%)#uB|fR}S^M*a5<s)4DHQua?_z1iE)yY)y`)hcjr`#QCs7=dZ%){J+kxbq
z{DR=DtARfvK}JnM8M5rJ{A}UyF6jN6V!PSCkwaGhZy{243lgUXTE5Ww@zi$t=?A`8
zzIUy#(v3L=rMJ|Ji-Lc23s{e;)mn-ItsQ^_eC|NjhP^02?rrq~`-`S0<}dbW#_W($
z=Cio5ciae_<VtSQ|D%l;qhxtM&xcDny=-YC+3;o~e20iAT%q|JpNil7Mv8Ib&^6~f
zE`r5ZKNJk(QdEBBadys{KRdc@ff&pR;`jc`<L_74d0sW$LnGAA@lOW`r9ghlt1GkJ
zxbot7N^#V&`MlEdfu~pwAghI1sd@~rF4lK(@ojEaU)Whw5QU|<*u=U)zyWOAJj`yl
zGg3+x^u0vP9s7mtOud@~#<lt70kOP5iTu_S>44+}-{lmsZ=phKHC05;{Z!Ow`L<;d
z09P$MaDWj<-L4U&D!2Ml21b*9MKGF-*v0#9s!D+v^<~Qa+u|0eUPK6yCjmEI69SBx
zYpO%VFD<qX`w@yAu8~x*G#<IC>;ySqvYQXY$iXi4rK00}-i3_*TtsIRI)a54`g!xK
za)v_(f4koc_we`2S&%~!-_A&oaPRRIp>^fI!plvAq2VS#1SF;&`LY_b-mG&<53b&C
zuzXd!$H=6|=F#5*Wh^UnEej1*IdNl`1kYr)+|sx2dTYO&MsaJKTH;Xl=}P8_+>(R{
ziXDl0emzgaDMGIp+hm&!{vnU58UkL#8NMtaP;}x<@GJ~jTPsArbwn;KY{hP)d{|<)
zOmeICY+Atq*NDR=bfMQ6w@#((_$=p%X$zDNA!!yC1RgUnI0LS)?Gt0T!a!I3Y+6*^
zc!qP`Oz7QpS-Hctohxy=*Jk+*Dwyu*LVs(zzslpg7l1HpUl79!>E$!~dDf0JKT4N4
z|LlZ?mTRGUIN>y>h9oq~walZisTQMtIl4CxWFYg4!kYAKu7AUW+id+rEv{u8y%RDs
zBgE;Cw5mXOWvI^8XqPWjDo65jNH@Qc{um#bGXAeo1TAdO)bkaJ8)_1kl2vO}OKWfA
zY!fzS?c<lLl!;*PF#?PVh{w^>;2g;IgK#1=vOeCt@*+VuDXugjJ##eo2Vktq1uyzp
z-JQx+d^Ao_DfN{}zrkk^-ZiXxWgE-(93vCr;h$Mp^lYqKs|GnH`4aRys50p|U$5UN
z!BaRVt>yzGi}sq>He9GY=jF#mci--!??XQu*^+kUG~P7i4S}33H=Z3&$p?c&`L;Gv
z$Bj}0d}jNmQ(d1APz12m00=#{zJ=SRen|pyBK6_6FI4)yVDI_Pfz-Y3zZ(=vEh9Nf
zcAnnw^gDSo2e5PV%{!!Z1I<$XRd;1P$X-y>&V5`uow{)J6K(XTk~L<ribVD!ox2ZO
z5Lj<Q3%22nlVnoG<#Y82oM}q}OJz10i>#r&baXC;XxN*R45?({;tvs?aFODFJ15j(
z!4n4)$hh`(PzE0+LD6k|Q4+h@3{CKy4cB~^_e*Vvy@iM}<X3D$d>1*S0gXD1d1S4^
z|C{Ho0&gS=BP1t!aSIXQqa!Ns(<0*Lwu6J|{ck}(Lw=8Mvn(2oumcAuetmN^!`rp-
zyVWcE<zTx#!$d%R{7i3NNksYCh^34se$jrXMrUp(?K6L8?Kmkrhsu1sg?2fxDQ_nj
zK#JG`l=OT2J87fhI)i2s<2>ZAD|3}nj?9Ep<)s>=9CjzEN5N4AL~^gt@*NHhdAVIK
z3^+nps2?YKy%g6KFchg;7R1Q4&^s1Y==XXfd1G(C^VF%!n5sW#G$F$<VN4ATv2K9&
z)<%NvQ)$g3n--!+^;*H`EhAbMRt;k#YhL(3d5+*!Yk`JRakIGN!R9oHR-qZP0k;ef
zU6>uA=s@Hn!pSj&wAPd)<?~Ptl4$<oU-xbPadZdocK&2>9?J~l4*nzIA%*$fP{Tsj
zuY5TPi$!Dv4V_)%<Dnx@nmFE`Gi8wcO98Px)quqPs+h6W<3_4_`-4&OxJcF~iDWI4
z5R^fiO@$>{UZ<5J7p0(~<tgs+i!7Obd=K}PWzI8d%}s$D6zpn*J+*tmWHZ`&Y|qR!
zQ$(V6@;L#bd94vv7C43C+Um0k^k&xaQdSnXmCD;aR&^2-?eHW+N4kUVtKGBpD5p+~
z$)2<)9SgE@UTe#^#0W(kIq;=-e8!x3lR6i}0?>YK5L&-%I)^%O`li<i?kh18pKi_|
zyTZSan(rm2SH$-1!J5|UBr55K7A;aJ<iZ0=-;fCh!yi|n$z%_#%%Ua|&WkY|`G&4K
z?_X->ggo23NK>zqt_dm}tc#6R<k*cEe#`0BlyCZ+QQv|pM0t?e{oyA&p6|w+062GO
zBJCe@Rp&>|tY5x<<vEGek{{BV%es;9?$mjde}EQ;w&UMFzozh#Y#rQ;RY<Rc48QXy
zuTX(?f9rUq{GIKMDobJN`OnZfj3>Jq;>FhHM~cg8k~*u}N!aKy5zBiQeDvxY1(=sH
zZYM)}1QTp#$T;I-^NP!k$(Tq_OzQV*?+NXoHb0U##=)4fUE|cKI_3<eKjaDp+r+v-
zJ`~8Ok__y?1gmPV26?!U@0S~uG5HXUiK52PXuH-?xt#_+z4uNw#git^(B0y)Po4*%
zbj55qWv?bVP?nAqbGLtL-K{)4D7{7CzjLHQ(Q^sjkv-C7goWF+2(-OIDjwL>%*!X9
zuabBGFFAp_NduU4fz4ALmb!3k^WuW0+M~RLf_VNPiRaiptc6NDK~SyLg++K{JJKaJ
z1j4N)5x6jH9Ia*@n@KT0r_|<6xMj$y?vZ^V0=fjd<#pfOtoTYyjJ|juJ2A6Qt^*&z
zT$w6U%8=Jky{*u$@qyRY{cEEuG9FcJMC09cv?qQZ7o;g2GBIsz#9or<Y_D1tB*UcH
zoM%s#%`wU>?IygjiYJE%j~d2+j^ESx8^Hik2+z4p)3L`B<1mgF)di<VhPjr-+1l6K
zzMh6H`(b3>3^W*Iqi0<qM#^L+lh`P6f+DV7o<Hb~9#VkwdF{+Tb+G<B2Vx?8IU>QE
zg&f52NG=ZSpmG9#?msRL4Sb-^X!l^9FXk+U5vKgUyhmy?Xt>W}{mo9^?+MH#4bZT|
z1qUpoWx<zJKA}L3OL>>+9om_v^jfwt{aMAt#@0&X&KfTb=EILLT}etd7$HP>5}OYU
z(;_jJPhxs(KIwG)Lgbra&UuKS$q13Db7B2CVqXgKXn2RdWaFGEIXB*ssguT+X`XNm
zt-D%DmgtAhzY@+yNv12VE?D$7(v3g$3eg*MtmOEfU-8Lpk0>JZ9li%0*EE(Ak|-k4
zbQ1+e3Ims=QX-*|wnus8Ps#p8Kip(jI<4}lX{r$}gxJ{0#%2^H=dg_+g8=)3aTniH
zq-fSsxEt?roY)TIy0_$L;r^>1dQ}~z`?FmUb!Wo}ILB_rnRyjl?t?p<?LA5lJQX1(
z`~D;;(mxyVZb>Mba!AkVk>Me}v9EUyh;-1>{T<9*&<V8rLa9DDXg(?CX1?ds+6?ZP
zLn&tUqSN}fRnxyG_Fa51N8~D$$U=)?920%PHK`zAz?QiEk?*dfR_>RRe32{?zF_DV
zH9>neri1{GP}eOZ$!M(247U7S(Cz6#?gr<<X0q><Itv%APIFVuOmM*&IzHIxHw{Zw
zb2AmPjQY^K&^s@N1RYf*)31rHI?<516qL&D2goeti<B0sRi~hdWi?1;Fru|5bA`yQ
z8v|^=P1d1OG%I)v+@bt-Hf9uc9ZTq)OH?ps#5_FL0^Wnh$cC}y81bt_rsCP(*T3j0
z_C-W2->n4obZCdX#tYRwDN?EL*dhrU`pyR06hpWOw_!m9_=NYBrGr+mkPgGtWeM`|
z98L|1k5Oouh3l6T#!=!T{h(R(^Pa=Fw#JT#^dQa=8tHPl-++aW^7{=6a{Ek$q-LpJ
z)eEYNlbv{7mh6>AlEZh<OMdob%)AYW)5>_-WhI!1_J>IP#E7c9(YFdW&29^>I~qGU
z>9<%G=Z*GAA?zv_69>MwBz)w@1{dZOs^3PSjcLq@Mef4yDqW_-YVPLRIkUaMg!l87
zg5-b3sa@8QD5ZuDIoaq8iB+$K!d#DRyeUdm8uti24>3~w7=UDho9}_%S@=o94e~Zw
zsp-vkZ|<tiA)S0l=^#a-V5jsEtF2}D=A7)tP+grZl?{8Bn~GD%8-hFLaCWkt=~Wk<
ztd+xg^+;ax28Y^{x0_n}l;n!$-C{TwT5~ihm}vDd1y|B$vw?Gn6wJZ&LK%y9)h{PV
zj)ZUT+C7~ImHo_JEW~<h7a3DlNm1=o_-eKm`EGxIucL_;=qP>O*Fd$#>(RxVHSanb
z%crZq_iFSQg8jkmkUi*FXKUj3_SND6pNxk^$1Yr<bP=0iPhkuhrHyMENC-9y)bTw$
z)VO%Qf;v79f3i>2{Z$Lsan_#W&6aC-cFgwpzMSQ_MpZKUc*D1g?;_}Rmb=2pM+$T$
zkVkpF7T?B0rhxV#8LHbKsY%A)jM<dF4XA}Bc8R{SpOYKpm-pDdR*aEh7%^gD#T|KM
z<5oldq{9ZAV~-(HlK+c7ON<sa#<=PI26V+oWN14$C38$^B*?EeD@rwQt&8P!%tJDu
zYYF(1&%3ngf62!k(e_5G{-&>i;q#b?d>tCCB{!NHDvz98O0Jp+=}T3nK&In2;0hLx
ziI6#zU;oc80Wy9gm7-akI2?B|9*0d!K<$$=zrqxZH>=|m6D|>?Ztn$=nrM&zjNf0W
zc={AUgX7ZsF}VZwUpb%XQ(nf$EppGUzeQ~OUyAtuqm<!TWdh0;fA;^iff|76_E6ad
z)O{0xKCtcv!m0kWgy<uH<fXLN)t|rgfCxYofn^>*QXY5oV+GzJ!!~lR3jRx2@OT>Z
z3XmZMQ|SLW37q%$gLU<&a(W<O^8c>`e-CRV0Tq;G+Xs-^>fb&uQh`!wm7iig{J$Lz
z2Kp#SXxyLt^na5719?)oz$!QzaBI*;{HPcD=f}emRI1|%^S`McK;Z&hz|n|@;5ecG
zR%VD4NI%b^L;cUv{`n<B8VH(>Q=eG>kx~7#*c~8i557p>e=C{j^J8%I-957o#$z<_
zKR+O4E(+{QK>R<8_2-v&d_d~d<u2Su{%-{aC<D;+TEt8^e;V;0<@4-%d>~f?0h~aV
z`ETF+r;-2D$p7ide@EK?Gp~yW;{zU#%$z$8(sr+UEk&%C6u9r+wfm9Odex)iB9v4S
zw7+Jf=XeqP=G7|&VcBA>Rz9UBKd%?K0w3SCzqSK#*&Tj~eY1=G`J?%ndi&JAx~Uk+
zt5prQh1FO%?ZL%>z(+7M;6VZaBcRYt(Re_*+z2AxKK}8K<18Ei)9Ot@4dBB6I;fWl
z)$;u2^E;GBbmKpc4=w_1B6?^X(tlm<&lSR50cmnK_Q^}1f8GXA`Md#ikA*|$#lP+i
z4xR(1U#X^W2G&3C4X%tX1@(WgpaLT=4)1Rc5&_0#Jiq>YIty^Q^Z_%`T!ny=9V<r`
z6pN6-p%ndZ`E#P10Qpd-a^7j@@C@~JbrTvsByBj&0TVFsus{R&p?GKfS3Pq*1qx8T
z0p7jS<!UWfAK+$LP#MZ)Sf45nvD^b>&iW(RIIo~mMb@3i<Qg=IWMgnQ`QJ89(?A|A
zh2Zlz6m#z6mjU6#gQ<qg(fcxhu^XqG>A=W$zSv)nH%MWk%P;o20`zr4Zl~p5d{BT6
zl#GT*?>ht6h<s;`N1Qr<v#?1p)|V4-TItQf=J>M<hmo2BUN9CBz{eHzA=4yb-2^8-
zK7$@|nckMgAi6@xMzXy|z!u<hGIQ@13vqV6*)cfz&CU)uG{2PXrargIdiPOB`>>*b
z`=x7{-HIC344b)3*Dx7<aReZ+EI`cF-yZ#0gGscNYrfF14um$LA*Q_yRV>%8i;oZe
zqj*|OBNLLhxI1f|B08x0n2h+djFK2S(<SkdduG&@#lqVCH<&z!0{x5|N1bGtHP6wk
ziY=f4{*xRK{TO|0y8#4JFGg9PpsbrZF%CSo-o8ZkBB`jbXZ=E$yn1|Bt2iecsqP7Y
z1e~aDKR32pd(e;tNJQ@ec3{Y#%<YR=<DFf4`Uj116=Q}+?#@W9{egv#UBkdE^O;{G
z00ir=eN$uuXF6Q&1Aeb*bQU0UC45p;A2X>KQ&`_j4`chvP^q-v`_M?pW%K0=S4Qt6
z?=_f=pU>sT(8T%P%)`QjJECOIyh+(_tNnKM94S(6yZm@Q*I!ovyCy_^vs)H61v(w}
z218ES*3aw|6mMfd{o@4K?iDXKLe~4!5$RTgo<w2zBtWuPP{Q56j+A)K#D9C}t?Y8t
z{u}WB=JqxnMrB3fpDC&_{aVim#6hx<jPxjggS{C+f7lIrBKamvEdG&X?fo8CnX<38
z;^D*bxYosll5PmZj9X_=%km@kA+`klldN78Z*h$c5&%x`Kmu?intbNjsx}flb(^6@
zfFuu2d3MvG^aN|t&un$VcPX?InnWo=HgeY&PXnJGAQnrO^2>+CJ)fZlY7+yL_mZ<h
zFx+?_+ZVvJGT|UHDHhbCS?cgBTaA?HXjnO3A~OdF=t;<|<l01#f7_wmyyM0q=vr3u
z)}eqziIgw!b-v;S`9{raFvtp;f(i!1STRxZFT0O)-^-phr0!=})3sR5Z~QRCfk<*Y
zS0vV|;35FPtK-Kl<(mQFhbPa`2uV1f04(a+ho(?2;j<jV)ftmANsC#_PWWVQdyL&z
zJVVT^)L)TdCun2&xocR9d6zv&sJaZ3$ZQ4$J@=TT`<elcj-;JNwiF;Q;lXpY7zC8P
z%?2(VlOlT)d^bXq0sG6)WLVr?z^@gIeXafj5-rSa|C|jC`8yb*uWwZil5Kz3bJ=b`
zowFYR5Wc5GerB&y8*)n1@dW|;)|-^q3C`|)8ttsx-8Ge;<xWAB@cne9X(EOGuFqx?
zPY6EKkLO94e3#E@eoK83{ZP$QolZjX$|03VG7+jJEii_$WwCJx+o$=KjDrH<MA-sx
zvMF&c0hBmh!_Zg*TdAy#cwm6<#-w;uPU3yY2&J~gh$Q=aH#^V-k6<%d4!|8Vwg3>?
zcLPPgwsSY*-QQR)x1|A);&*=5t}&whTI0eRY7`;#JB|%hSk`-h0lXDB;=Wjok`~q$
zbpQaxcHSV~*P<IEfZ|QqP9=B<0O2B*qLY{Dbbw47ma>+b(_C1$`ZXTlxL^Nj1hC1C
zMqK9tSt8%`A$VfAI9;s=qTH!&VY$lHN%!%vyX*q&K0x?0)N?oD4_)FKJBP*I->(5(
zHxUeH79Pg)#O_qx=gyNfDGo2x#-+3?3`aVMEt;S^+-PyUhlVZBSDx}!@rr!kGicZV
z^zPrA(o)>vJ@-{{WkO5B#-Ao^Lp|4xC&S7J*v-_wpypN7K3uL~>hceFVALm4gmU~?
zou1I`2u3z_RKG4)%{BSDW}@3X>#!2C4j%B=U5&F3=Up6NbQ~ckS@pcs2c$#oHiHz$
ztVb**DmIhRD6Ot@L+gzK{!<f+EKa1-Eyd)rS*u>(1zPo+{xN?|*RxL?_JX$N4N<PQ
zfa***U+G0zIX_z1TWM*&+vRr!0N}yT7&x1AFi6-;lpGF%nfoE$2<SbIg{W)q9V5&1
zC4}uR#0rPIJ9BqVT?buYxJyj{;#C2{t^-ho+H=>vjE!;Xc2iUd(&4GNz7+sUo?za1
zH?G8_yG#2XKc+!mr|1sp6n%)tM=tbJp!k#Al8rCv+f*KR8@Hu-j5h<4>~k#JrX`;Q
zn`@j3cSSzCwJ{kqyc`0xCz|XwfnTEC)e&k};COdHIb}Z`Xca-m-AzwTRa5nfgehFz
zj2P8kx^<9n`t`bA?au8^^-%|r|Geo6zv!!4ouwl*7_2WGFf-<juBQ)1+{ro=uUKCd
z$Rb?uo43C=!T;rWK4sv4&+;b1T=AhDPxGC=gk7Yi4t^Nl^_Qoy>JI5msr)+OEO^aT
zwN*Gys4a))AGvfE!_w3o*S@)hb8b|o)AS9z%FB<m=6_fQ6M5M29AK9v4Iyxsa4tA6
zvr-@gela?R&U*JrWp~nj9aqhD%6)o1qxgrq+tr%)oZSpV`Oh&ub@q5=GOnqKY0|q{
zI0r>3e2zx0>3UeNRt#?X;`;Zp?+b4TTSBdsu@UUv2SI)(ELH)!C&Kj?F@Y-_==O~e
z_W6KbaR=#xuGdQE#XVJ*Ak0%B1X_JZy0UVA_8gnquN@Y2A8q7KclC#_2{_@a;>h0C
z{hTz{?hW8VkzU9wvJ-a;jW_$XocE;8yCs{-C81AQ&(-nmPMK)X)T1fT^z08>23>-5
z3gxuZI0;{eR*SX1?gxE|>)0T+!LCX-?7FZTGFO71S1o{rVEGXwiEBwi=_S1x@B@NX
z0J(%UPQm#C6ML|GXk)QovmyMfc=vNu5+j<Pe_jd^dx?}tt*-vlP=(IonL#|bZ&cb^
zv3O{^Snkn?u}V`N+pw#A!U+OOJyYbDU5%_iQ3hB&2Vc|RH*8ZkPf0F|!?#<5>OmtZ
z$v1Dhk0xYNcF2h1pKg!Qp@@(`Z9fgXiMq6~6>adSUW-xvVQ`Fd(Xc<YCQRXO8M18U
z<EemRwN>1kM!l|z`RW<Mwvwe~PN76k7;Amd?K2!K@vp+sb3R7Cpv$afzQ)j&!Bdf3
zV0^6HC3qc$z6!}8S8>N@H`9r$&_qiz41D$d(N6L3d4;FA1o*(HDLZ~nBkJ$7kqs@S
zcn^-#N%h+k4{pU(dCx8T&1sFNe(4kri`Oc}Ib{jguY4!1pxp5B{1CvxlYG!N#y(JD
zth3>ae0!T^D*iE?<G?TGg-!ka8vVdFD(oFK;NaptwykV{6n;)88i0AT@Oj$LJJRHi
zROm=Y(ovz^a`m}Vh?DG*!gCH8Zf(gOtnyE6>NT7+GKpoqbLtF1i%sxR-D2-xAL@~|
zAFk-j2;!9N822%}XlN*7tUr}2QA9~0M?(o8Jw^8se1_YoS)=u!tmN@cVcM7c+58Z;
zSs~0lc=e(JrGymzQ)608e8_VYn!+c$?jOtO(cqeXlMNobXR0+o1~-Ebl8}Xda-!aI
zCkcVMQA8j;y)ZP&QS(nRhu(3=bIQP0ji#<Yy_BLrdd;9R(>QwL#0L<e`~in`#M)`k
zJk`|N;UZx2Pl;9-<#RD=t&V|FxbSqfb8avNm2k{*%T5I3#ImBFAV=$=`b2b(h44wK
zzwpTC&cJ2qlZxTZcSAj)#B?yTx_YWdPz=)g-n(#L!aqR(4QB8${$g)tB8-b?;0I*T
z{{#R_G0U^6+k$nMV;J)DAhe%5RiS)|fbpM>rQ~`xmVW)xELt&;mJ024?QSV8nk1yK
zZR*p-4@fz=Qf472Y{-CG>-2u88cVKtBU5&=oyt@H7-e-C`|Y3E!b@A+YXjN=M4Vzl
znA+j*ve*tN&~o!<OCDGRg{IuMv(rfFKFuc(@9<%Zb%Np%Zrd&y$(c3uMndBFT%9-^
z2aus+_iuI8eVa+_XCqQg3(288(Xb9hgrm;AjwbHIm=(vqq~%dmPU?IOvdZLAI16ft
z=+;p;-B8b1LxL^9ULDS>M+o9{UzMlJ`?ho2J&O9vG7i8JgsqJxjWv?iU9S=|c!3%C
znd<jzMZ$IU^&&BD{@7<&DG6}E;QscE;yg8zUHfM_yNHC_^x0W@n0L(3N7y@HFJhh!
z4I=RD%~XY@Fa|&dw~6*CGQ@coKEKD$!IO-dQd9ZtoF^H`Qq|5h1jof&lm`(_h>9HW
zG6rMU61jS8>eRh{NSNpuJq4HXcz#my*A*Ynb-e<>iI<(TJZi`5)PGBX-M;Om8_Ef6
zgGBc!-j66PCC&wKc13XloxoD?j*{Z~sYMPS;&)w3fTwCJwPZ{=H5Ql|!*({3ZtllV
z$ZWGI^I&OHh~QS@2xE8PWwBg3oOT6&ZzOe(ot1#2*!Q&ieU5sDy4&)49j807u{W@G
zdU^<Mq;bqHH1a3G+aY&ReJ#h4*z#HKoCWo9+4B|wGE%(Qx##)Mxs^ej|4JHdGOyDq
z(kb+B`-Hf{y60%e-4^Jc+mva3KXGoa7b=Ok{m?%ldjZs{NH0b$7rhid?S=}>sHUx`
zew4~*5sC!>BSpIKkzM(OqaP(qO-Zaql79Ki{(v;+D6jkrpJ(42Dvi?*>k^jzcnHT;
z3vM^?x}Blu<tWKc^K&;&3O&K1OU9X%`o9x=EsZEYS(EQln`$>aO;ON0CfadJD2GX$
zseR9TkFDKP8}|hhVNI}a)H$`Ww+vN;uIXjL^;g{@lC(MNTICF9hNDgC<8e}Eaep`1
zo@`;e3O(C<Qk+1~qqJ?X{>?2$x+6v0<wr#=meXEL#H2FCSI}$`_*Rfty~6gZ+Y2Hy
zkTjDgkZ0Q#2s30)X6hrjWFEndm7@UD@H0Fw?>2qG5Nm?N_G-M2(tV7WAMHVnqD)MR
zI^_-d*<ELHFC}&V8ps-DZ!2w?Bd8W?O;kqZ`^iLXXDD4n#MXHD1{>Y-ZRbMkm-@5u
zuOxz3yJZ6~RV|4vWK25mhUkA5UbM@qC74w&`Tfv7EZ)Kr!2hVNtn;A!aGtt-XaDWH
zlhN@7lLdd&H8z4<m{bx;PNUQNdvTK23=IyV*l+raLd_gi7?@N;<;@dxe`wTG`a?ut
zw7MdFWcSo=^;?U>&gGLq?0Ju;{&YOT%yB@cMF5WS4a-2&7|lLybK20<(Mo45995a{
z*=cQx6P_R8ph~>R*B8}yI>)dlAT=DytRBe86)K*3Zv1lMC!*_Mnjj~-1zt*DBmxGp
z5(@wa3&SUK*}l^HQS($~ncJ*fz<Iu!7h}{Ar>eK?s|~RIWGCN1hs-c`UX57-W7<>r
z_go!(EFQ_&I&DR&7a8uxm?V_8jPIKN-3Ie}i2}JBX?IF$vj^x;!DdKG-!gZ3R+3Z;
zKf(w_{XR}!53EkP1TbNLoRpKhB!}fd*-TOc_nm{_pXFfASNp|snp_T-n~D)gkvZV{
ze3|bf;jn$ZXOH>MC4Ki{4Yh{bOHO|%xM7;K?i^t6*+KOC;<NzUI?(QRE#u=lzsjB@
z@A@0qVLn=0TkDN}^xxr6{rVzqU1cI$j(t66It7-3iP&F1Wte6um&=yQd?}v=hsl3y
z@dcX&P;cLPV&ioN6EMSs<OUo7*s#R-cT4Tj4PI9w7E86L+;+tzi;DMuY8rB9aP&5B
z^u7#`t7R#wRjbU78~UlBpb(tJ-U)uxrYh<vaq4Fc&h;xYrrhDMTJ4(jsYa;2%^Av1
zp_rN$7tJBxo0+AS<Qs9ti*;J9n6KdT*mm3jxbV}ytCu49)gkMrp(a{1Qc3s<*V_g1
z1Byu_kw0Ez#A9?7Ijx;C$x|#K6K~KM4yOq7j$$3gj*Mu)|H^KpWX`N+u>_tO4*_+g
zHe~N#bzz!Fqu7t{&=|J!)OOTD>>FuQ+>Jg{HK7Tz%5khb>Z}@Pu3Xu~gCxjcxW#O7
zKC0^aJ29!&2D8}D8w>oEUxr=~E7)K3JV7uTtu3#;xYeD7{MzTVhTfRK6dGu2<CRbO
zypV7pK2<~=n15?g?FW2WGK#>1^b5PyodNi=K3Z<+KE$gze4BR9#j6KTL{?~m)nNvh
z5%KWd!6MXb+gD+#Tm#77e6H7Ur4(oE@8ungl#IXm%L$c>f1UX>nJIK+l;Z8Q%No7E
zA4`GuPNKE7K&2vR-uj@*Wa^FDK4WN(U(MZT#Iw+{oo~t>Y5+9fIurrCAf52C;h1<u
z+%M2{N-S|;uyPPVv79_au~>z)$k#^Qk}N-oH%+q>8fWR4v<^-R$8SxRq?fFdJX`BH
zKP8-5WEt-mob)R$2@%wl)ZTs?!y3yp9^KiO!aFmGP+Ahz60Ba3^#%X#`ixm&)yFOW
zF1y4T9^Y4YPbEY>mg|GWOr_~pzlhU8w%L#hGV~EWMMKCV4ZKp@#H5vN>sjDuT~lqL
z!b*`@C3#Za_X45rGwEb}2f)7Szso>1wl~_{^eVx*_~4I068sEROv8Gg?qS(bggKam
z3TZ?~GQsM0^sFU_ni*U(AkZPeoQZnerHdBRX`sIhqQFDI{hfiv=v(})4M37_@Zmi=
zyM`&TE<(hZdE3xE>A{21tFbYnH2pZuDmSPHTE{~G1@1J2fX}76gd)8o)Wa`X=q05$
zExF7y;H>&2`$3XB0?9i|y<rvO@!W(*9Js;mZAp;G$jAq7OQEz9@QNl(<T!c;-hEG>
zFCqNHbBguF1~j{{h?LJ2xX{(RKRx8HYcxa}I502!0-3^~&Ql!)s{iE&eF-j-0hLT^
zjjZ($-|J2HPe*Dv0_e+xqk)eF{-@WJfC4yYl{~|V|9Kj?euf~Fb~u9V^1m($irs<q
z9PN$5b4tX2|98X3n+Ya!{JX<8oCy><_!&~S=RfZP{+$riz)*%p>fi697YzKpy6|tU
q{+ny}KgIk{YyNSc|G(Hb?-4Jtpr;&WTMMDUkJx*ucZF}Yy#5cHc+5=z

diff --git a/docs/figures/arch_4.png b/docs/figures/arch_4.png
deleted file mode 100644
index 0c59e6ce34413d78dbbd0c918dbcb73133311edd..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 74000
zcmeEug<DkJ`ZlS6l%OCY%@9%o(k(5efRvPUcPcFs64ISYh_rw(v`R<~Dcy~9H+*Xd
z-}n5^`4c{_3mJyld+oK>6ZdmJ_cHLAyd)OJeGCKy1T1MOF+~Iflr;ndBm=Y?;GL|u
zR$K@O7++0AMW0EFijqIGvo<odFhoF*3XG1u`AkWj&};n-OR_Hl`t2~h*Z3Lu!cP~|
z_h}wdpyNIb@J`c{Bv9353JCDR(#oW(AQ0|<kIL{Km2uIB5u<fo@4H{?xfg{+{Q3EU
zkoDT=db0Bx&%!v=c%C1D&Y?gp37Y{SjG{Hh_ppLpUvDPvK*S4q%PT+-EjAWn<nN1u
zpq#Z(*XqxVAlL5PLF0D1ee8Kxyp}xZCc+&bJ4&ITSnn-chwAx2$Pfbg@$;Is2#ZY=
z9g%ou5qvSIu}@^Si-8Y5Ez#Tf(m=FrVRcjlr1-XIYGmPl(cSh3+H8OPR|3Dj3t+jE
z9$bqY8yqYv*lMDw>pL9qJY}Qv=}BbjY~6uu)ZwgMq#=J<tTcrSMoGicsNwm7?TK}Z
zRIGgSBjKAo;?u-_FwC8H(hS3+HnpgGclbM=oMoI+XG^(LWyi<P-RA0WE6A?1uzdZj
z{xL0j4G){pP*Q*?fr0Bgg%WCZSpo*>>F+5<1qL()kC_y+6mHuXoY^Z;e$3{Pym$9;
zhq~$189OH_TaDO#nl>HsE!)(<Wu}og68))r470!bjk3iKTK8bdEjr|Sod@mj!{?d^
zU0!z#inXpZ4y}Dvo8G~VuOT0k%<61KkInCpV3}<n#vLNcvO9f}MdX6Sg6kzHi;JG;
zB~XOIh>IcoO?Rn*1eG*?+7R!R-BZrU5S4hZmfM@{3y<;{^DGeJJ`d$EY%*JXC7@`E
zC(@$P5Y59eNw$dZhQuIsX(D*Fej-X9XGZvZi14^<Gb192eoTx9k&ztnVH)y_G;{~=
z+mA%IOOWO3u{s8EP~M{9-XcIEl|m#EMvC#etII@(Ag;?kc;jKcRS`~~*Uw2hlN;If
zt~NxPH_N}fZ`|AQLEOLy_QKJ{^daXL^EL^4n1Q+^niY1p!AFyvg$@DZtzJ3`9TA?0
zq6um#_CueiX&5Ff%C}CDl6_&Ks_F8J6n!_bM6A<U9w9}(;>|-OuD5?h(C&9Sh30~o
zgvsJ(Kjr2^rHQTj(|?lYfZvUz7H9uQ;pB4^ayvBSZi#kWf@yCQDqAT$N)Z)`cTz-(
zA&<%PrTFX!m&E<TJMTU4^LwS&{z(3wjx4pGzk0CxJNhPNU9(4WaY1oaaf0n=^L#Nr
zFPandNQxg=U~&>wp}h2B`bk=!Qm_4Eyct_pXwruJ1`OBu!**liw4!aBZDU1pg~TY2
zCT#&3*Zby~7n@lw#%_;Nf;yWBC!Hs8raGtBCvR?~qS>J^`P}`Uh%1SEdkf9bm-h$J
z50W3Ze%yQR{aL|*`Ud`CgJFYQ!)b%cG}APKZE%JZ8#PZPrBqHvS%#e=@)BN-|BgKP
zl7xvcE6dZBOB86X1&rQ>3PL*#AYajk-ab%>iQtOBiRcU`i<o#$cL(`BF(foXZb&L8
zQ!|Y}(=8&qC9%zZ_HZkAj$tlwcKugf7GBPgGN&U2N34c~fJBWri3<6{dvq>zHg{F%
zl^-T4FRI*okwLHV@CJhs9RWi>z4nU-%0C#0=v?VHqqm~>=(V2x$)<R$m2fje@}=a6
zLX{YC+RUP#1GYf4wwRFYdfEp%2Ko^?NqUwS1<Hc?qiw?XZeU_lB^L+UMs+tHu*v8X
zyou56_fy9zS!+M~%=WTTPCQNVbI)fxow=ZgO21_|<Q#*{O8($;1SbS)d>+Ot(=v<g
z^E2)>=Ifa6U`}#(*rRe(8xs?db%!1a?h^m`bU)#~;C;OilKX^@{WypctP*k(G82p+
z`<I$%Fl#)?SIjTbV2N*)<z!D*hgJW2(5*7C_dCXH{r6fIX)kMEU|&nGX`kyynI6L>
znx&XO-<CobO_xyr<agx`yvb1LGhWJGv>vb?I5*KX!L=+JPaD$hde#`GA7`#+J*Y3n
zYR92rp={~UdC$0=O`d(+)NKTXvzSAgHRmzZ5F>Fo9z)JK!3p*qW*bM9pd^qz5^T}L
z*~Xg7wajiDcQbAwo-45~C6|lKYApu3YG3zqP$T2zO1X<gj=_>~yz!9PVSj7Gd*c_T
zj-3g5tClsxm5oOsAvq!SMzI~pFB2@&R$q^Bj#xYGIpI~^sG@iyaEx;N@YsqV`R<R0
zMq0M@ycK+TPTX@FL)|dek<HIDWb#Jxrt*&RZ?`cQ(C1fE2iEgu+8xK2N0)y%NZSm}
z^elf_ZFS6da-YvyZgS?E-1D()mp3HSt8*&1iLFSrIbWPwEbf^}qMcrvQC}RI%ACEs
z^kQna?`1RJP}TUGhUR`_XeW<OUSZuj!Gu-l4gcnxx!{AL<NTefMTI?_{lX1T0$n_E
zbQ3IJ<Wr>b8+#~AHv@eV-ikb;bur(F#Qho8NKxlE{Y3N$gJ`P1x?et;IZpOF7L3AM
z=eSd(#q`s*8P78we`R`q|1Q}J;$7BZ_DbqXE+eb<*0|+jv*I6vG2CvN%A?*D-G|!S
z<sD=J<smEq>l(Thx=TK}w|*03vGCUfR0dQnsv&Vw@TN0QF>uAb>>!bwrB;E2Zt%N|
zxE3B*ohY7#)vG_FbP1nLQjT~Ww~|!N;Y!!Yk;!PNwn&kOTPwXQH<o<F%EY~@JM8k>
z;kEt^3Mz`33@Pa-n##DGc<AF2C5wm5xckgbn$?z@0S!8K<hNQi!hT1{NhW?GR-{_O
z>8X7Xo79@5AoDTi!OJfd%B4F^Y>|e<F*Uz-VW+M<Mm*N8BN(f9%~f~u4GKrAFf^ho
zzF3y6{W<xKZ0^?Imj}t0d-;aFld_O{{j_q?G5_%Xq50Oa=f+`rt#GZdVpq+$>FF97
zs}P3g%Ajt|t7gCTcPBs0#R<d>#dU&Q8wWhTp5&ZlZ-z({^KlMY4wn&?tCvE}gO+0w
z@ERjVBhK=@UnV|{&wQ7;Ud}Y^RMwtlvG3I5w6r(A<+T+<@|urx{XipJ(YCIq=4<iO
z8Z*(~&B-Up&Q8!(v%&o0DwF+Kfgb`mdq*plUpPNAkNA(omaHG1eLh_kD)A@~NOMYC
ztS^82@of#qD-J^QRde&@zGcqeKX-eM(nUA@t;qKHM|*Vpi^{{h)f`YBgp;}+%KTu_
zl05zJjYOZsoV&oY`VY#g*em?L(mc})o=j*t<Xh%;#zf0EmF#}}x`gxB8&P{Xt-Y<;
zT*(y4Ekf)bock+_Je$y|?Use(5rQ&-T3q|7@y4zolGqa#E$E8-;S;{9)0Mjm4A}*|
z+5)a2U_mR#cC(1TamkOC*0_HBS@3Bo<}^QM1iSnt--2TbbmsNKiB@M-?7HomRvno`
zT?xb2Wf5mL=XK<>x8MDja3=U$T_5kc$l-j;-U$2=`8E5GLarzKS$^?A!M;PP@kEI6
zR43{F_L0tM%I@)i_sK%kf+0((b|1C|R?7V?(#li2vo>$lX;uB7sOO3&s=v)Y4K$zI
z9M3dKwg0LoZb{rMZ(LWNAaS$auC;t@u(RYJjH6F>ODM<V%VGP`=o#dwEvOixv#HZo
zN91XKx^`Y;%&+gJ*&b~}`w|g_#6}Sh!5RliI1ts#i5Q{hBq!yyVp`6*mo(COVD{0P
zbTP874j01hHbk@v^!iYY(oFGFr5QxD^pCG;5njG*YjQZ}8_HD(_(6fNhxzixFe<;>
zS?5G(j^LSs>$&?4d~M76i3LyOCV#Z)?V09v0Ak-8s!1Ek$ss%huh9@t5bq<Pf>(&(
zTM&`>?`sJ}It1j4_mL0~{7n&1u0A6Ve!{=Pz&HGw%b&;*eh4?gZ+PI_B@OBN(<p0c
z$k(rt48Y$IM3h9OrNK`neLF)#D|-`b2j!M1_?753QtI{y2m}!LH=?v6^#-{AfT^;Y
zgPNQyufDY<^D6^uJws*}OB?uo5cpkq!Ana+hgak-mKIj_ye<Ni7oXq-ui<aAP?BGK
z#KByEQcdm|xu~_BAvq`WBj!hxf*9oF<otF9M!bq*5?7al-vlU492{(TSy-H%otd54
znXT=NSy*{^cvv2>v9Pf*fln~myIMKCa$&Nvr@Fk!^?k$)?e*<UZ5&Lkt;pf`eWhpZ
z=paBz2_NY1*X45>x|seml9m0{w7>*e;D2FZWq!o+_uk-Ae)zk*&rDqmE!4$KEy0?B
zF$6hT+4wI$|6hOkXT*P9srJv6tZbbBap}MQ^nWjXVQ*+BYHbO|bP)V!zplpp_aCn=
z<Y$3T{a;IQxy~2wf`t~u;Ai=}X@VF{4Wdb4A0L>CDJX-Vz{=oXh-}~=y33#N*FGcT
z$u?*R2*L=`Vj{{eh$~|^qm;$_>o@trlS*b8=>pIgeF7?qd?<bJP^pc>aBx}IDjz=d
zNt+FxrSm})d5#%C`Bd2s#?vf)x|=_iV(8b>JLOn+&Swm3cc@c!T;6}JJMZ*6pWJ^N
z7aIjv7y<FW{wazQ#tr0?8N=}r`5%83u15>2n0e=gfP_Z=fB*49MU@&=-^jwn`yYQq
zK%!oA!TPUj;P=rLLBw+=sHXSz{a<%Q2;|d7{_nxSN7bHrH6){2V^C21eG(Ui$xj~y
zBcd5pTfZ8%QH%Ovk}e&z$Fui*eoo~&6T+2YJJUqE^j6~b<;=-9P^3mt#{~?<{uzM&
z&OKH=UrbVdHJ`U{=SHmSq9sXPso$w)t(CS!(#m`w6`*EhlqF!+r0YoFkkLX;3b}gf
zZ+FOTjoU_GX;(J~t2getK;pXlTUD;dswtRgYt+8e_0C+`fOcfMsQ*(Flh5tRe5xAX
zES9>@OdeeA#EA}=)cJ&{#KoL+^}TnO2im?_j?DJuC@EIE*ygC@Cycu9k8u)gu68-E
z4nr0?Q@KR$&?}0)Ct$iyOYZ-ZqVDV{*L7w5&3d%O(AP%S6zgi*N3VXMER9u_FWJnu
z$CD}yLnZdM+F07&tk=*|P*80BR@ISydjm^TVUy_EASy(?8pcSd`7f8&YZujNeVl2m
z(lL_E^MqBtNJK=$h~NVa#BQ$j?gEdqdx>!;X(VHN5>Lrvi@`3fj@Nl4av7(4Bi431
zi_e#a%W|8Ky8RL#n^LBFo<Wyp()~#zQaz67<vq)-#v+1=*b|!j3$z%@%m+fZ`6pZ?
zCs&3_6u()hLtQpMN{@#SbHVsX_}+BSTAm*5&Q-0~vP(P3RWD=m|FvOq3SdL%IUDyb
zO^<+p+io^M`e4ymu5YP5A3SBtdfhp*KQ;+lxX{7m+pn*8&AMk3sy?XXYqWe656Up@
z&DvVZPOYqREiccO!5b+vH;xgNkd&;dIUOOc7O>6!aDKRuYS$^S-;jcL`}XasL&U%;
zHG)FVboSN9(EOIFjzq3EvkTNYA|!%tY7bQl7-N~WBDz0G%G7zBM(`#tT=`mk6Yv8I
zw{0R`_=9?RVwn7}INX}c`gKaE<3@3Q&@7~Pb8D??y_KP`TCdh~Ilo-Y+zfh5ZW~)+
za?+qRQR{Bl9>-ct7*-LKxiLhTAe*n4`Q8N2xg`R!)z02|Kpj&);qg-2^VG2=nXi12
zQRTGV``W7-2+3^;b$r(^?X#%Ji-yD@E0npMSkSGadgI3}XR+MCfJ~X#gv_tap(K&A
z@obGLpJXIBTr=2rwiY_cjw?KDx98>9%@ykU8zlPk)OTg<7kfr~vlS8x&vy$w;|)C=
z|McBvT3hFJP?b+!$}T~$7J?=5P-z@9DWvc#rawW2I;|!wT9jZBb4ra@yNbs~{mJ|8
z8|r?%*Aj?N+wO|~du^n`_LF9vht~VM3`T;QmWztNlKGqytbt9NcT;CI?f=9Ps*q<*
zWl+gm?Dq|1u+82#S?<q|ppwMq@i=i=2`-4DS(8uUr=M`!4v800&ypo7DOJqJ<J)me
zZxbxbO6lhf_B=V+?wsdPs&#X1iDl7A%054s!GYS&NUZ3Wnf+!{`SYdC!%-puHt)96
zEh)ZS0&)1f|K&xlZ1vRC)JIk>vpzZ}r&lNDeFo=er;2I9i0sR9T07bWFUw)o9uHr9
zF=%>l&g<+pQDUqR|JXDF%B4Zf1_DuznTluD&bZY?t(Kmio|v2#S&r#qy*IMr7iD_;
z7Jo7iy3^x*g^szTpsW3Ix@~hd-Fdxx9+Z=LjdM0<q0=@r%@1I+l0MJ*BAw<R2z16{
z2HMM~KY7<~PbC=|yj|fjBMWQPUXbl;aILi*`BvO%S{3NdjZa})&}(hfnPLemR4q)=
zZ+w3@lu`X_15~?tdUR+zba}^{RN!@-J(=rlI7+HB4N}MU`N@LN41b-?)DLcVYJ&AB
zdZlU2*m3I=w=L=3LVKF%pXzNeHv!Ax(m|tSjy1g*I~o>!vbxh-#!oo+G*m~;b~b+E
z@FlfCM-)O%H7s{W*T$-v1)YuwVIjEf+xGMA(9>7jW$Y~k#F5{oG<__!xe5l}Gu${$
z>SA?TQS)c#d611IFc5#wiQD`H;t;6c_xbt!2?3J^ckTJv8-I3h=jL6rCD~wzi~DO*
zWyz32`vq8zSyQ}?6L+LJwlUH{s9KqTanEqEQAdJuS<b*>*%9x;--L&UCw0J8<O`c+
zy-I)^(E9C8A?vd)&q?o_cp>8Wm@vKl`Rc>%g=U`4=wNBfX?c9Vgz8?Lg%!@;Z}z4-
zEBZ$++49NLJ99@((G|UMJEHg+3PPu)(0-`5pQ%hF1&qW_E4INyp4oOG>78-Ght!JW
zxmX=NeR~)=`+|v21ILX4;|1BW!P?QycMP`s^NH5HNsha+b-!~}KUnfY8^)#^gX9!d
z^c`WZ9k^qM6Pz@qg%^y-LqC81Jh-Q+LmgXT4@ty(I*LWgKe!1}3r|uBvRrk7rX%dv
zV;w%@!^UA#*-e~<JYjzZ`p?qJ3l^N)yf#|d=sXLUWe2g1G;5Coa|_W|N1(O=(v#!P
z6Yj<CO`fOY-OuimcKLn`ZB(COexb@|T;9}_xi6w)&r*Bv@h8+_ES%j~emkkQK+(R!
zBL%o#VnQ1?;Z@eKptf1C*5$m<FDib49qK1gNZ+imMv|L=Xc3WKW%Q_Kvx(RiCSyLm
zwkH8HpQ&D#ulQ4J)oJ-i2n_NLdx5QMunn7({~lk}%GvSy2lvrkaSeRug5}*3n%q)r
zIc?V?8_1gW>m`W4eX~w);ohods9B}_L)&a=u>lJ)xxulNwBGp0f`xI<^HV;*@RWw+
zT-JdVaaBCC$$D?>g)LKm*u6g1Mf3GNQDZmWSfRv)6qjjqe|xCqN-qoo!(RBRwVgyG
zKSNeP<UCwJlb)Q&(U9`75NAN`cxMU!xPq^?!e;8pia>XKP7Y6!OqhQ2{c~Q+0re8t
zl%_oCc@A|Q8Lz`<(-Ph9KGG@8kyo%nR|jOLy_)^ti!5J{gR0r`)@rO0LR4c<s}zZA
z(wilhwCxsO(L0w?PHUp+jYY&p%^7cZxMeu-bH1dGX6**neS>Ld$WKifwaBNVOIsRp
z2Btrj_o(A;i5WL7ZGR9tt5{AS>|B!_H6?>#N$HI^LDr`0VvJx(L~kaNc2V>@o!vLT
z-y~}gT+g##d_Cv}{{C1bwrlIpaG80cxl+P|&|Vk3iIbz{!eUmQ<t2!%(P%a$6%`4L
zB%z~BtI{Fx*nqdze&~y<URzpkH-9YUB`y#~0C4b@SK=?+i~P1ig-bwS6mgT6_+ic`
zv)^Cl`4>V<Y~{BNr{!bCXdK**ma>amqL^NOYv5_r!FN92_dGXlntJ+-hwI43M3F70
zA(>|Ou{X#$_`E6wnx-jr%LP?yF2l)zd^HN?0}6e^{+}d2IJxbQIolrnp3P`l9#x};
z%@kNCB`P$G4;&0sYT_lD&L=<*`jix)3HEIZg|$ZyIcXN=>2l0^p_@`GWeLyvj;BVI
zsa{w%2)P2IEVlf!qVx}ns6_~@I-qp>a1nyN-l1v|qf@J@>BMH3DMCEPco=Bb9H=lu
zUI@jo=#WZIOyD~&z><TsF!(|0DH-xH&)ek6wqdgFA9f<)bE=*ZjVvlKpXt=KH=bsO
z8V(i^_oay-<13s%N(&Yo<MSD`<xa*tfFGNWRhB{f#{#Qr-t0Fu&5ok(?}bI24nu?T
zesV<q=3v%R$qQ6juxf14PVGX6jTy-CYy5n~%gdXt<brwadCbDouz3~)zl%qo)IkE6
ziPLj&IXEK&TXnT~fBX##%v42ol!V)coHL%y{b+}K^XSdA{LPu+j3-=LU82UNrqc_1
z4PyXq>SJk2r)LP7BmLR?nGDv0*nM=ob5*B`^G0`qe;^4Fo57=*p1H1avf8JkrengY
zvXy;YW&Y!|W1q#Hd7LZ{rYaQ@EM!dr?G@ckn8a!Xi`9dqwc6~jeE|pV3l*Kgc<@Ij
z7e>)+!*`U>(rX?nG1hmc(E_M1sW&gKqRPE#Y=6R&jCNdDYV>rcSD{#%XKb~<Z1cxz
zARmcCAReUMwGvY)Hil+5y=2Hr$Su6Wo{7XAq-<*($lfx8YC%Kj8-TitKP(f%A9SKX
z-M(_$`Ze~Lm}sO+`MOVTRYTT%SEYACg*m_?X3R61SE-;V9Ml}dV-|kz;bt&-juc6&
z5Gv=l$ee*rQchSsQLHbSq@)?hH^xnVmR{DbS>pdVK4vhyU&|0b*S6fG`{SU;jJ5LQ
z)6w7gFS)+kXqMDQBtYhav<$U4)-*+`ICB=e(%*`j_vP51G>ox#^6Qfh3~A;FXa-g_
za5C9;U@HYRV5<g8tB-rE<`&kPO*5%~on6Ig_Q>DG&?=-ID5xx;b&s!j1Dr`t6Ma>0
z<vj~Ws$-1S?w{XsiF2*dGMkwRZXMxN{!n=)p6e3N+ZR+ikc=Ptm%E^*tP7xIu?!PO
z&fW7ay+aQ3{$M%EhQh=bd1|cNnoo$yM^o;|g0R{`EB8woyyvlx?jiJM&%(Zz<^wZ`
z2hBXZKV?VSq8SEAI(Jt`2KciR?=^rgSC4@Lt>VBB84+@+Zd-w-qQ`>(2PJWio`#q!
zy*mixQ~mf*DN7bxm8t3=unLqx3EaC$eN9VWJomdqamHAw^XulHra{?l%K_{dbaGUA
zbu#mgQ2SA*`47%BOp#bgy+kFKFHOl!pG#&nNj1uuGn+xa#20vhpEaCi^#t$SF=0@=
zypDOsdOUoPgFo~NOYPx<+N|ykEZ_qBo@felCD}ko&)}eS?5U#^t$dPAh0)KY?Sw}`
zo{Q}q?ge1<&>%vVglxyAt~gfx>Asbx4{f>J4mJkQgw9X42gTkohiH~tX`L|3Jt%N-
zjOhha(d&%ow$-z!E3DbmSeYqcaWG0-)=;$*#ivT3>&f`A49&oEPMEY$&Xyr6)T%N{
z7;thhwK6cs$gOhllxMVcKBl=_pf*jr7D~c5Pp6m>5LgCmDHK4s#->k!EOJh!dh*F2
zF~tXhk_~%cEkmQza;YaX6qL8kp%WlR%KfP16zLv%T|P0;DR|tTjxI&AJo)|YjQ?_P
z=GTN#r;a{QRV|PdLw4%tI}%BNzd4)q9BoYfEY4nJ{$5#?XQh~~4~DZ`v6psFFBC{t
z&l%v~Y`~XJZvE@L_P}{LJrN(dUHHm5ewS|xG7dj(VsN)X2+g!6I>l*{+7rE%!Q$ct
z)9^EmGPA~h&0OUiMQ}^VntZC@>aSEG&lM=|+dPmSXhiOil6o{-unq)Qjm-4=elQ;@
z5G)ule*!s6XYVYP8z@USun*yQ`ON}b&SnXIhf=_O_x<);n^e!&C>xBoPAGL42v?hy
zA91!+oO(~?F)iOYRB9R-UpB`piSJrz>uoNI>V7ijJn>s;=0UD%VJJx|%;IEZ;%pw2
z73Hq7vGbKN1~gF>z4skK;S*Y;Z7ld_|H!r;YAdDa4^!(sPP1Kd?>Sx^K2#&?g;Xa9
z2YVja0qffWNG>7nt&h(Gv)Iw8gu|su7PBu^=YPywcH=6mBHCKY7LIr@Z{4zA8HKV#
zdtFJd!!#oZ(@k8d=~VFic~Dpn@FfYr{U0K`x>26gx*a}%C%6V3%H{hxG|!I!<fw9e
zr~Rqz*nuXT8Px&OSSqNl=-<!SI~&VjI(4L3<z(4xU3W^bpa>uoesHCE9z*v`<vhsr
z4vTpe4WX{v^Nlc*9`jO?8i`1Wn%BE*tkt{yF9Wr#Na2gY*F;qQ?FC>Jnlq#$BdR$$
z6e|B@H|jjhwH&RLez#ZSae9Q3*gxC38mJx{3Bya|bGBN+Y(5~wA?9R+EsI3D!+}S9
z;NgDV`55=oR{w#2?ziOJ)Da<CM;*;ox9twDVq(FHg5T6JRuyT|<GujsJ%na{O&Kkn
zDsCoar%5>ZO%&cj!W;C=d}u5suXpp0qOPTk@m%I?S7l^@de{;BKh-NW1UEy0*K=v|
z8p8|Sv0=841}XDBy9gzWI*Qww=B_a%#ZWjm{@v-MB`Vll@bu_irKFZ3gTg>hpEC^G
zlLOn{x^FW{=@8eg*+9<(Zev&J<GocmO_*WMv?dK5%r}fXcJIBtKJvD0v-l=ymn|<>
zC+>oUs4eUdp7B@KaU%NTeklK7sr~7ZBmIjnQQVQ7(@BoeUxRAg`opOt8=xz{D|nor
z++9%YFrcb%JN#PgNMjNb$?h7v!m-9JqH_y2<T|a%%mB-X;x-<G9B9}`>a3Z~EOvjI
zPP%QPU3ZjV2dl<ku=r_^=uFet!STAL#3;k{G;7vap142t#i~-FBlDXWIH>6;oHIb4
zL%GUWXlTGK5p9WyP%i0@{<{IG6(v>~Vn$qua{PR;7>Nei_A#Hhjd8N*Tw@7n7n<0n
z^URGSoQ@8ISZc#S9hi|xUvSO$5u=Aylyr9(eZI^U2TuSa#nQLLo{2=OkaA~1QE#Zu
zvyR(&GWtG;S;kI}a&o!ucO=)nA!9imm|?lmH}fy|Ck?K1X(q^}FUOs3T(V?d`ydUu
zb|$3s(4@(9I37AB^E&qQ3u6@8S-pRS=M3X@rLXqsz2-{R09UedhM{|M`5-Q8_0bjf
zYiabbWPO2A;*JX+x?~h_xxq(~7PQ&$F2`l0Pa_sFVPYLa`|&anqK4;_cAW@;BT9j1
zs#5l`y`W&O62uia0bV(dOK!V>Wq5RLW>*ur{FOW!jH8I&wDiyC|MkiRaAwI|QA@~I
z-u|yKg%P}!F~TZjAosHW_1Kpn`T}KOfgK!wmWA^l!(9`wiuA(M%+b0@*XitEV*(P1
z3%;hJ@q+&`-1Q^;|H=O4od0WQ6)7hJ2%0-YDQ_D$-gXzKKKkQXfYI_$gknhS$M(}_
zP1DmSG?08->`SKti|&<rE39IBn^@}LY6m6@k+5wwhaR#L_Hzq);5C=6PFK__G~ABg
z@$&fzp;MwqZ&iHmWB=RMqc1YeB06~h1>#Q9c~v~YGe9zMkGd*Dl;A5nJY&SQw@q$H
zieqBUj^Va<Fdk!zN-L_y(@cY+5Sp=k=P`$7%H6;AR21Be8gPS&31rGEbM+P`41m&i
zH)lQFNylll{mGLHdn@@Ys|Qc<<eWOgMm2mgm~<sD>mQ1mp9y57-XEoxB@Y79-rX+W
zv8isD@0gUWSkl%P(Pu7*#z@v_zIWcFprokIEnQa}xHhR^=zaB`zidA)1FpSQrr7G@
z@+Of}F$vAqJ6Qd#{6{K2+mB^uF-aw^tr@rI3%D|OoM`&V)m!Ix0;<Ww`CfCI4>EmI
zKKYKSA*J?Cn(`Ht@h;)_zx#(q4T>CQTjG?z(^rH{R#b~nmVa}AedUY$vRv(G_bdw!
zPsKGH#jtQKpLyx#wHpYNlK{`{@ovt%_8l)7@}t2!fx|<p)HFYK!#IR)`pYbi=_e<u
zI+Pf>iM{(97;qWDs4+e)*Y4$AiY`XxBKdKzHsM>4yTc2U_;ftQt-EBq^5xg}MnIy!
z4JcbY2<-2p;@u7y78rz@uLOvtl&$?t&Bu>F(~?%N++t~oqK|Y$hOk|G{(n-$;4v~3
z$jv5}@oLUjcQ~fs=5m-0^b_qN&xxtDdp@^&K|24IZt<d)x>!td9uP=c&^P(7jW*0n
zc&FGDudK!#sZ;GZeI(K;wT>a8gJ+1jbXwWCql&p7J8AKHZivJnNR>><8UMPPN|>N`
z+;%R+PM_+wr1H`It*zhakX2bUk#szjT?TZR)<oOwZj$Svyo43OB1iO0@m$Y^JjIJ@
zI#}#XQToB{>}di0Rcm~?;Zv@av<)P@y-7}Vj85ZKZ`{8((4_(o&sjNydG~7HdRTCw
zhth#O+u}#WTY9%zAgh|`C=2F{uU-f`ySEu%4}*Y626xO&th6^*Z*7oyFX}5LIj&Y7
zE7ZfJw4|%Pz(VM153J>?GH&2%*nTY;*DBAD|K~{&5LrOl+s1o5c4cqC*TuA=0?G^<
zZ@cze--sB_NXBNQV=VL)Ds6`9i_6y$(R%-NWbcQ-MxL!R&Rkn`IfmFODP6a}1hh;}
zfios1Bbgy3Y#N!mbX01f-0dG%*nroXWZ!$4uGaA2gV#(GV&DO^+MaMost%1;K$?%e
zKJ5{DVU3IrTl7xQCLZ<ma^rr6C%s)58}rqqZ4!n13Kk$o8~4h~7Fzszl1q=RZ73R#
zq)iUHQ!t~y{p0l!aP!2(Rk+GUULB~EEWh$o-@g~rDdG36if}SYqfn<7O{RkTppX@k
z*tPE>z3^(#N*h;iy7mvN+W~^ynjiP77h}Hh4+Xr8bUMd+T)J|Mr>Iv%`!mb(?-qGe
z!z)nlpGwzGBHX~TA2{<zMMl(x(li?xyH!K|2id6JQZfF6zcc&=pW)#e8|T$+q+-2h
zhM5D^oSiF&H+C9Ro(@B&u?x$Te7<sX&HqJ)AS_mbWi5>CrMO<!AmN=eE{NGg9V2l;
zcBB*UMpuvZQ>mN*CUarkzYjzXjOf;ERomc|JCJK3g%kggYHuuCGW)4S_=d_fyZGjU
zo79gl=d(Glx|pvY(@Pi+Zuj+5^jD8rR4m-i1TCZS57;jGjcpBCZLfICF6H++Hi@Vj
zQ$?Sg^m<juCqc^;^@PdD>)PK4yyJp|WM=*Rd+c%~3pdSr*6OIxb8Gu~whO*VHjB?(
zKP&>0GjP?aUSG+-z8fgWukq1ow>-+1CBg{<SL2Kdoqn%6lM(I4brdXm3zmg%)UYYw
zD!Szm1Pt$1v%7N9{2p-qgQNplb>IgE!xQQJNr2<=@Fj~2-+L)Bu<w+Q_^<QS9znq9
zPDMk~X)7LDt?V$pu+6vxp3^yoj2iA01_gaxTHwu2l_SB|-mJ)oTU_vwyI@sn^Du?C
zdqA3xMD<w=2A;NgRkoj0)jv}U1gxSAil6_L!Pbu>-4-Ia-JGv4C|>lWM;_6K?M}>w
z|A(VKAJ|`;=7UGB87<@X>y^x_&*$I~|Huq^laei8^t1;S%<|aZ;InhLl4z#Z^#&k#
zi^0tj>F3X@%XF>1U*`T4d+^*Am+g!SR<3<G-)<Lf5ntW&VmcY+-BZ7#@IawX_J5w0
z6u8pDZI%1iYvoDEUCsS=MTmd`XMDRZ{D9Kdd3*VMcfF+0hyAHv-Nc6azQgq@q74Ff
zO4eHEpSg+%-YBS?mc!Ia+49l!k<BvUZj^jRG%gAGq@(t1*|Mjf-a0Jz%dJ5dzR70g
z-?_(lZGe;RaP-KPEqQH!ab7>C!T_oBE77`^Ug%`wtwBe^i-du@t~tM7&_D0e`+h{8
z^v-<y>4*KoR?xYe>y{*o2JM-)UIihhSFd1m4G;b`LBO3Xuxi}D4qCalK-norz@p6v
zD$j2#BjpP330ar{kdHt_z9H2_^5Ol=7MTi}kLvWT)-x|nrsFc%6#k)h3FTZQR1B#?
z?V1eGsZlI7d0yBcLVG=o*BID_q18}!?W-w-fo9e(&=@wD{S|Jv)C0*;&WVujcU<fe
z0d4t^XS4k(c=l$uHRQb~k(eNPTTk@UOYl|m)F@_}Kg5Fe|I=sB!X8@=E5F`fV*<n&
z`@~znzX^Sk6s*>oZN}ZJp*E1cS4Ya*K0p5?+av$ea<C}<WIOdd3RE|vx83{3RUt?9
ze9rt<%{QMst))yi^c!d0`RR|-R$)6cJM%#(2J}B=;@J!{bm}~MYTe(oz&Wvn?Z2aW
z3I738%Y+4$Tw8M|3TOb%E*I9tc%C0mY=IX~Lm%+W4jHFLt7^N7zrRBtv1sz~Lub_I
zI$oNi>f{u(ngxAyXs+l-(a?Z^uy?lzTYb^-XJd6d!a<K#8W1)ejMc~Mbwb_CE=bp|
zE=+C;W+{EVacv|AIrVdGv4-D$P}v+7pLNH-;w8?LiDfpDP>W$yk06PjIkEz2$L;IF
z(3J1Sa1}M3J@gLNZl}<B6zF2Nog8k@{rR1%n6JU%dUm*w35W%n^w^IQ+~|J?{7+>F
zBE)909x<NNG6CL2rLo<qN5=UD+MStliJUnFbp)HDhyY(=)1~%d)6ntP`3?v~54X*>
zMl%>mZ-GvuSfp#E!#DGRf*!!3(gOzQ%g7AUwbRyzf^fUyGC;(B=>UjyMW($pN_AFi
zqfk6NJW4qCy}SGcSj>F4gq5p=o;-`(_O7?hE1R=y$V#n34@VU6jkpd*K;tt%fBrn{
zTbh+oH;{<f7xw(-Cj48VAp*9B*~^+)RP+*Bb)h~+-vP`|E8MGyxvZl2UALsFoK|Nu
zW3?lEP;X_uIp~ApOBd8?nT07N^Fr`x<sY+CIvVTbvwlXemQUt|&Us|g5WucO1<HvC
zaDSmuh~8Cx_2xz*WY!Ghv7i41mu577AdAJg`!LQhE&-(#u#idiy}NRK5a_O`M|fO-
z1IJ7`Apos@m9Weo={%#nzKe?{-9yfB4_L1~mVY8DnkgPcKoh76SfcVh@hj>VR53zu
zu(VJmxpL`!gR;8&Yh!GVf0T*%obpC1Y()V#2SL7p`G}az<vpdHJw5<n(CmbAhpO*~
zDN#iaQ(Yf70p2Cym<KSzJf3IOfFbU-ApgjB4*JVupWJ)7uLF~=BQQdhI?)FID}VUv
zarzo;)YOl65fq{~O#AcVtIv*>X<BAXQ{vd$Aa?dj3eNg1dq2N0MozjZxI(@qa#_Fc
zr|naBsa)YP@6YQ=7rnVOUQ?x9pgF|A+J*jiAP)FIRa6acuLhDJr=<MBnC%7>6b8UU
z;C=eYUqt*aMYA1=18<&&-UA?_oV#FsQ#+Mlk)F<89UZ_3YP!MY?vu@CGPc({ChFBL
z#qsLufaEO+{s#0Rc%-CjyDp2k*9N~44U(x&OE1gSGf>k3aSc&Chj~o(`Ps4E(T*vc
zBEo#Y{jAb)dA=k=Zl@a^uk4ev&*qIkZ`1EK$w-B33zDStvn2s10gf#05bppOzzoR`
zq**DiHm4ii?-M8gwZ;Eaym}j;o}FwfM5e(d28$Ma$%ea~o@YwHMPpaWNQEfo{lrc*
zwI6VOaH(csaKMc!{_Hz24#7)m!I%mFso_PgO1xny5K%mu;4<xLX#bI`Q&)?EimFa2
z@AH@V7=@RHV;@Ve_bvzT04$B%X{BLXj5N>@bx7a?syhMn^Eh9ux*W~MMQ&5uy?hBa
zFmUhZ!;a`lpH2<Rm;;dq0<V>6WaG#|po|5rlmw!9(0*cV`@B8df`9H%D|wAAPc(ox
z=u8OuT)tB;gz6wUs33SKHtun3IaFpIdxu$5jo0xHHIOPrB=Oj5I6T7+Jjst~&^-G6
zaT7|LH~9ePsalBRnrZvnd@%`$)(D79J7YBv6(z{YIM5D{*SwJd0uTN{cDa9+6`tG#
z%wJ#P_WC^}&<Y3xtfrbo@I4A3t%CdCFE2!7*&gP{kNdU+Lm3hw)8%!64Pt77{f|-a
zJz$>V@BpR#e&&*$j#jDru@e!yQE+=Amjbg+EhFe()$J?~C;%SZ(8%`aA@k8+*YAN>
zl{G$^*L8Ob#$prFCy5X?Vk|L9Nf|K3+1&x1SkRuOeUtxNK0t8CTWr<iyV)x!B&Exa
ziaO(fyu5TSNTNWiiiX`776r6vZ81zQsU<@n^km8qmK3)D)_XN5Y!Uz3UZr9I?d7m(
z%X)3EG-<>bZpW+TTm3Js1N)1?<%y+MV;b77b9d>Ve-zV;J^VfcgOqqa4vX8KI1<UD
z$6@8(pMb1*TPG2sG)dqbB@hdFRKuIPMkSG+Co^O{xvDItqZP#mKUZK;6Fji%s3uGR
zgh#)eV~a0^dXvcBfQoMgvMU473H$1K?h%>MtF8cOaIO|UGT+%fW|G_4$-8T$cbY>^
zt*l1~v2GYXoXa!c4?=mjurIz(y1ET|9!59Dfh2AYG&t-Aw4CEnv_UJjV}Nuusp%RL
zdON@wH(K)8tKF{mM8PH#v~W5)0m76yuu66l?)&jajv)ERPEJmG>Dy^eoTkod&AQf+
zzI>*(c22#@7Q4M{OHJq|D;w+v(w-5)IPkCz^R0N}6jteK;MX+HJe_}yNe~VN@tt@t
zH<{c91?cvS{tUo%f<8Vz8P7jSWO=~bbjZeGfx*=C(;9)J2XRt6yJE1SgP#=fki0(+
z(P4*SjmVDHE;q3sgaNT){A(BE&Xl|Po&)99I;H~!$vHs_<Kre+e>b%XY-;I<E#z`j
zzuoP^q5-XQdN?$Lck?5FTEleYTYmLUkBnl5)a{qh2F(+7N-hBkJja&0CXxivA8Y45
z@uk~!SWiY#-Ix2B;)OhG<KLCP`gx-SdDXsCz}RT&1qilUV_koFurMPKY)!Lfo?Is6
z71}fzU5@TeAX|IS9KxiO6+Bd|WXxa)OB(VZKPM@kX*FWsmN%a`{Fz?1;*h2F?AyaL
z+yYg-(Q4Ncfbb;XTFOd3OR!9t07HFjGi5jUH5q+x5#@T40W82*YSAXH_mKptvc;ER
zf1tt7N)Rk6mQq~-2xvyGD<p)28;H#z$av7Qcihb9ua>%R8hQ%<c*qiyCJSLA$u9Ki
zo!!}4>h16&;B#KPNZ%myWdfK1S7GVA6R*3%qq<69hMYB<n8X*);2nhq&k9){W7V$P
zaQSBfP%^?}-;fI>WME`1Defg+nYou3S+Z_Ry8oReNcx;)APW0iMhMp;zzX5$_%v~H
zOb)IG_zL_KAalkvlfGdj=hBo8L|?e38=OHU=sJ)%ep${HRSSd8pxb66AsDnNgcxEy
zUj5OrPTOr$oc>vw*W66`6E9Zd7@0np$8?2j1ICopv!;)DulMhv!}g;f)0^ew@=4sU
z@LYKD*9VLB;lk0S-s}$!D}&Fs=Gqple^OlI><u{O;^9z&c2%S)U?d57YJ)t^^wSRu
zUrrgG+WQKZhf1;y+T*$-wQd@XbPrIQo!3E*y8J0ar6z(|>13b0KTq|IIf|8%BA95v
z6|RvG7jFUnC5N{TU}@~ehE4a}H1911^J&|^WZoKx2@6)whcBU3eJ6_3>agJ-*QJPh
z0IYtA;s_Z084v(%e-BPPbHC*Z+h-qfup<{|*u|1$Ol$18V}JOK{+5376w|i<E5lSY
z^J;(K*=R5cp9;uJRD>+rcgEhSaFv(<VFjpsep@{VzLqHQQ-h=?TJ-o*W#{!9wLkx*
z@t*<EV{d5x+$)ycUzqzW-RHOHe%rx=a(Qm^iICVhe0u;#Q@9^X6KZ}=_Pw*jBtnGY
z`tjCqFs%(}1hFeQo*Zm`bzD|yNfD@0e=_n(5{KQKV|>Z&TFHl&@f1|gUn;n-)V1dX
z6^_d>j4!{n_Z1!k)mbJ`3(>SZGU*~$YJ=5DF0gd-T|@^SsFWVela}^_RV)&;hxE;}
znD=-|$IsE32Niwn1!pbj=vG5)4J%Rl3w5lWw9jTE6=p$WcWd#}hXljY%A6ppwIA2=
zL9`4N@Cas&++>$dC3J_|{iqb^b(Q>Z$e4{`N#Q^*oiJAfTKlYyl_xe*i1>Cn5sjJJ
zt6$j;J1POIJ#ttq`QWy(_{YNcR7dZLjGswy8i!Qo;oV75^<j&3^oWW~%exDqGbUNk
zKSXY#e0IZbZIlPpMeh#>LAyvg%sgM})Rh6mpft?LYcud71f`KHVm$Mu(Qic8gNP}6
z$}2}HCMFg~Bn&!TU;1)YJ5?#V*DaBVIW59KQ5Ki+Wo@+Wwi*()zo-hR_$q^)fa(G7
z&K+t{T5yzX>=-|*`$y;(fd`nbDD2Lw)Z>qoz<4m_PbPa>33}wRUkzF)fgC9cUU#)d
z(a-o(VT^|PV-it<>N5&d;nJX@Kz2?Gmc=IFHNbNL+UMd$Ots|QVR$s2sI377OK4fo
z;I$e`7)~H9j<ii)%x8r<jqeAucC{K@2Hs7=;wXKQqap?Yb{P)*Vcnzw6sYuCm8J2~
zb6g_$CY2}-^UEMB%Hpm6cWtL-X@8x#QBpw6udm7uMuYEVy%7?;P5|zmNaO@ZEN|AJ
z&D6`h4vXaJ>FJ!k8>v%P!==(+RPrAJnHf8}%6VXt)EFd9W~e!sh-!PGGZrM{rywiA
zgKtl@t9@jby{b^@HI6<5I0fdpP7;0*d<Qjkfi###olCz7GcYhP9K<nM?Ki6&IWKZ!
zIt)+r4>Zhe28&+VO*fzeX>F<}id+=w?FzfO_lupIvn`Qxg>`3aGRB}}dN@|)Z0uSk
z*<gC-(jd>_xl?HWz0_s)euQ2LbRBI#e$OOyx)W*879*G9zNhZGlo^{L7Kn#-7%mJH
ztQ~gq;Eux|KGG;F+{#lgX=41ORqF;76Bm~T(W`@PSR&-Uw3Jjpc2&0fRq<F92T*&^
z819ctW*+FqwInY3mq1iz2n4+}(gUF4EXk9PbbfFKR*4!2i5Mek<k(8SoG`rpxa;R-
zrN(AEEmjhB^edbSPGc+q%<b@}Pa1=OaWMpU@w)lPNDdYbiOc&^8HXz+I|-B)@FMFF
z_~$H$!dW!K09dv&RsrnMz&v9cbn9eU=bWvC<^u_z1YXnhe(S)LxTItdto8zCf#=Jz
zG7KVM0+1G<!ed_kqif`h1Fz>EJ-TX$h7ouJla4!b1QOfuv@~)!A~TUI1vOn}CQm_e
zoR<~(MKu&qtY+e?|H>QWqgIe34EI)sriTNvtS4%XR5gsoH4d6?-@il^OQ0nhi_On`
ziEHYUk$6jt!0N+!1ob+P8v17w1zLPS78U>i`e3qFcMQ(OX#qet#KIwrx{@H~yMj~Y
zNs!0btjGD;Z&}YYg+y?et8{B+yxCc#h<^BDtM~bB&TC)_LNqhQUDB)65QGGz^mxhY
z*?hrfOu6tdRcQ>*p#THEVa2Q5@~elJ<JQ%ie$WAGgBl9bLH84&9c43WzXvbjOr8`1
zsbv;{c8O63H9tRp(ap}Qc1MvnFa_dK1HTJ4R9DU$u0sb_+R>g15P>wnBjU>grl$|J
zNZW45Ayv|g11<YpGnpr5J0jb|R5Z;Bo9`9|>E~HMRU--X@z&!Ai{PNc9Tx3TTGnp*
ztI1@5^Bh`7Sf+@VKtdVaVz5X=BJ=_MEkfp~qM}Ep!=*VnI@#zXq4fk^B;J6LvE(F%
zQ!L1Dih^OZ7eOvx!!V}%PtTjvTIom{BXuWm2*OpVp4R!D{H3!<{U->dCkO}*j)F^o
z`U*#Y8U|Fns=Q7sbS%{yK0_JgjaarHb#7H}!}w^V-z;`f(5vLd>3n!Mna^Sa&T7Oz
zdgUXVDyRk5&}Tl4ju3P|ihJz>^b*Bq2m#l%uWthh%M}~LPiF&<^PuX*BoovGpo5|^
z@iRf#IuHiK0mEB?-qO3M@ZbYdK?FGbG4y`wwvrddBJoX`7)I8T4RGwk{A?AJ*CFxm
zn&p2ZzV0O=fG^#}L8Du7FI$8!Vrd?Ak^O=^yx2Xoxf?ui<ftI98x{S7nt%mkk+BL7
z@-)t{=SX8xxSkMcq})2TSiiY-6ly;Q-0Wk^Y_0q8(_EGOLCj`CE?>c`HIo4Q!Y0gr
zsY?(xLvi4QGk^v_QDvkevpwH22gtZ^Q2k|O5y7y8y+J9b;|CIh#zKKXO9UeK6Tevn
zB-KJ~juP)(EWiAh<q23yyv}PDoTYR+wbepc@6;~GK{EmP3eo)y(M3}O^y=ZdU$`{R
z<n${z)F1<*2Hnxk4Zw`O<5e85KtAi147tzI-h{9`P`LlWE3k(Xsq*LM!RE{y;AW!(
z0?<sm(!A9rz%dDZbS$j05M<;luY3<Uu{;;mfD1ydo&^QaO%QgY_D3>go!|%tuZ(!C
zuvw41bI_b{JtQp=Wzpyk2$7NB-`=buoo7di!RzvDrcp>?1me8gTEkiuHm{Zd<~!tC
z1q$4+%6TZ4`M^sHPSeiGv&&zE({l91uV2B*jn-1rUa_FsfkK@LMt}F&aLLjBn^>qZ
z^&mnTcBP4Y1&4#^!HspXLc39tg#V+Mf^c)IrMS)$!J}v&h`f_cZC+Op_$zQvh6xhA
zi#Sjed_`%4z^3E2`3`J?xJJ1Z6HCnwUf{j9QONQQ#atDpZpd%XlTClex72JXc(`2&
z!2lfqNNBgWq7FS?tLiW!SRF2le86KzZG(z)i3z;C_yF{VDn)*~7#B%?@@RMYt4TMd
zD?BBD`rsnS1JC9M!oRGzH}zO?h)}J`hCqCymZ`3E(K)~fI=wl58~`@=@g~3LnOkJT
zNYSgeuG=3$K<NQ!FbAB`;d+vGrR}f)Vh&hLd9uhwMx~~;Ss7$4xw8ijwnTxLAO*0D
z+$ry=C<iq)HMGr@)LYFPevAmDCPR>VjlL<a;JbNyrD9lsq}~)t!f=a#5!c=R6M)Y~
z?6<V!69KIzL~{4lWv!$u2duBZn&q;cJxPT)cGn4<qcBpBWBvdqx~q@=<ZeyA#W|v*
ztlV|>2B%QefVz_!@U5Hs>9|5jwkSJ}0`5l+CFzZ@WIlKi0_S75p99>)y|9IK9pmJH
zFAa}YV8CM>xfe<)?lCwoGn?!@9uCNtry#Lp0$CxO!7p5T?P?c<kf?N*my@6jQ~dml
z0)8qF&UEZM?eF+|4WQH>&J)99&`{r#b2)cVbpdW0pp144tEn>K#S6rP;Ee!S8$w;K
zsHux`B13Z$@W$}WLd0ni_ki0**uhT<Ue_t43P!-u9alp<8Z~8_`04I$spSYK6sV!`
zd4l|Jqdo8jWNGtSWxh@<-SZ#R8}KLzPGj9l^1c=lRl~~#nJ$M5^VcQu7WK!D&(zZR
zv;-mxoD5g0lXnip!p8n4pC&fFj|5;8ezIe>Oc4)mQKYc4aulOhWncS`V~~#^{QR15
z9w@>EaL=XqsQOndffn%Fp2p_Ji^}tiIt}%AnMX40>n#k72pYMBU*H@VB@X|Fmk`!l
zwNtT45Y*twTZw9O;{2>URZK1ptwKi^Xl{+zgMgqj@%4wwAq8;cECXO+BWI2MeW{?!
zk`^Tt4jQ4c!539EIU-{6?j~Hsc;r}jx|axWtPC76!czoD;@sX0E@+7RXrRuOe!aIs
z|FYaF`f2n-X!dgrk@^p)4mUOA#+N$%YQsP7iaDZNf-{jBdX4Wr_}QLZQFr!WQvAj;
zZ5Lw;-l+h`_zsb64>qRkW`EtLS1m}A{<ugM5YKMB_sYP*lF%ERZ}<g5kzr?wfD;kh
zj2L+%5;$6dpN|MNXu*SmVu}A~@sklzKFk;U)-4%ZzNE_rrf?b@aPRQ7AHZEV5q)K>
zYM_0-YYYqMYiZzTxptS~v89B+-{=A6fFzt;f7E6G!iTsk;Jw`MppH@Mx+7{S3L3WF
z*)9e|6unweTD<o_nYk)G5a90K6!yJbJr^}VP5OJAUg$XM`H<j8EOT_qT{iXDU(SGY
zgE0WGtCz;$-9#EEhy!%4$+So;vPzC(<H%Sq8IlJr_#a#h0Ad3C5wO$_Vy4SV#UYX~
zU#9i4MDrp6L1<{RfZYi0L&7MOl#~WTUkxwL0)nCrqEqYElcPjS^w=b9wlh^oSfb_8
z$@4V8etZRak;i7zOUO^59qm9_*!isAuoPaX3`Il1`OluKN+n0>Pw(X7)hSaJIACPs
z9=iyD89KtTDjjjGqJjsL$ULrF8Ka>0kOgeerZtR0^oTDF6{Uy52DIQJxk(@CdQ}7O
zSuzQfxm{H5Rm)QX;$&n4{9qF`2=G1MEMKzUS_jr%&V~Ky!WN1`KolPtHM;>9(;w8;
z1JS9$bVDGU>y`o3bvtWui(W32|8O?)Oo&?o_Yo4HG9U99S^<m%A_&2w89j2e14UtH
zV^YwWXipW=DbTKY2nvH(piGqpSeRPBDYzChr1mn`ZhQt~ec0-`Xg_%Ip?I8ZN&*Ls
zZTu4)1h~;w8I~GqaL%cmFCiNXW%OZM;4Fv^0nGcKm}6g>nqneCfnw9d&aEdydTT((
z6JGlPQ805)%^PQsy)d6#IZ_gEB(JJWOn59p!$eGeUJh)ur0>}|H-qx$F#5fTI!{l)
z#wzGyMm$GJst3);cD5k^qkaY4rb4oE1J^O2|BqTlBcGKB4^J=iSs>~qRD29JNrs(-
z@S?bMc`^yu&X0U{q4znS!cqhn?mkq0)b@(cX~o1TXjd$?G|;OGWgeEy0MZ+Ne)EeG
zBROXd89p@fFki2J-rqznN1n|<p_%~{<#3jEX%t#=hwd49AcN~g&_+O92BVdn2~fg0
zy?O$q%2JyjzUW51=*c|Y;LwWe;aqGcKrY|z3BAR{^ae-jS^&dH16mb^@fNzHiV{<j
zNZsAtA4}F9B!{n>PjXn*aK;@x(5VTv@5NaPuljR<A5J@c3NKP^H6z?FK+H698dKGA
zhp3krMT7SDFJN{V4of}Sp?+()QmOSIhfo9a&+6|7`6;fpsp38&N)Mz6fC?fmtE6_o
zt}<&@=sJ-NKvpUlEWcR{_59|%YT`}8^WS)17Rv=(p@P#jCG3+$uYQW1SM(UlX{c#B
zqqT=q-F%96kXirZ-K|W=<^C-Iv|0cGH5;o>Zi0d&_n&3UJl03R1xUd|o?_-xZ%)mo
zi`0PY1tK2OMiv`2eEq&4fm9s{jOG`R_I6+ffK#CWw(uhRJegLhF>i=g^ake`<1)rJ
zXPN>2;m(Y;B}GDWk)8w~GnQ4q=@&TpCyxJG<9&?>zV-G01t4NRwS>11xHJ}V5w1&9
z%#a6p_fIwV1xZPcg#`Ls>}{_AU(8g=zZXcG@@RLNZanpdEOT*&7Y(9Ae4f=uHgL33
zrb+4<8u<npq84v1vHC@G{s@*TR2RQH29i4{2kyyU-d_#SR>_%GH!mc3-c-2w_*5F@
zYoM<X4F~d!-j;v?0SD#dgo2S4IN2l4G2jPE?|j!OF$&?|`bAX%T8(6Ai^S3Zd$YBf
zfbRUOaW%7^jKKJJ*~HYSsHlhE>CkRpV1_Ob{7{cbjNHJ$Ua_bJnovr3^*Z&_zs(!@
zW(GJwkQEcQ_9r`0!G;_+07+wH4Sp~mIN8?~K3wZZ`WP<WaJ>M)WQIUAKfj>HCDn6Y
zUP`K+!taXYae{eKxYSpIJ<5Ay9RQawZZLSiy@}Hbbnor$@7iKnWB}?0q!0)bitBBK
zYt-MfR3-s6XcV8<Z<h4v)e|$81#sq}C=5=^E@BRYoofSCI0VEWqjGz>X}~{sJPF6c
zfFq*+0Lk^DNE$0|0OtD0XJ#gHmYwptwXZN&qwI_S>kM_2JV=qn5S!Ad2XKNdxQbrm
z`)z=#GSnvluMu%kJkRnbYM$#XkCYdJ68<S0pz7wGpqGW1cOP6Q6GE_X5hCt+fBsB&
zxKh$NTeCkN2|r>D0+`(MGm*D9TC|+U-r*sA!!v<*5Efki^wGnMhu?!lj7Uzn6|Z-{
z05585e>hR;m^<4Z|F1%Z1nnNx#hh@!4jtmA7QiE|Cq6u(!^uN<mkb1fyQgn6K`k`@
zAr|y84@YbQ2)!L{0SZsb8woW7NCnsoTcaGxC6R$9$kH3+rf$YSol1wC#jnEvYA0&A
zfQCJdt@Y|lNeB4yO@Ow|_Q*@gN%$}OC;&d{zjrdns)85gKxP7VDN8Q#>vGDz6|#W(
zvq`vuoi^1?5#ILrAg~vQN<|HvaKpDAP-1dlwek1Bss9Mz+cE%Lz~ea_bt*VWCxW{p
zAaULW2ZMTvJBz@)0Lz~T4sioc7BH!RDj02lcps1&%Ahp>iM7W@LS*RzN58u7J0D3_
zp9R~1hpV|zasURGD^VSO+QCTm|FQR;(R8(4+b|+Qh#t~(B1MUkkRT+8P7uBKkmzOW
zEkT43ov4Wx(W3^@Mejt3E;?K9?VZbY-S_*9Z+zdM@Aor?kufrCd#!bz^PKaT$2{g-
zfCb!#p`5BwlJoQ4l`~bJrx-$VzKu8=I&pfrE7hY6TYPE2eCWF<!Wx+E6I{7h?YdIM
zz+i(tayO}n+CJtr5Ke~ByM{K8`wi}76?<oO-0~m#F^&%<d*{VhS}38CWWR}lzW1u@
zM3`r2Uloy`-$$R5M;`1kC9lg)7G(0HZFzkE26f|Ko)i-uAL-t7PBUO&XWdL{HMoA?
zjIm4;aTa;%j!5zRF=;7;Wfs-EYQ}ZDbLnuR#i2za6*d{%-DMh4(SJm)Cm?w2&{@4<
zja=D+bg<H`e1BShxX{!FN=Qzi_<$USl*9rnEynXQ!iun{7m5RYB5cE_yUKY^nAQy|
z_JMgQr0JhT#-Qae71EbknS7H3CtnYq9<kPbWb4h5Tys{L>p6mma;iVwf7Q19D7!tG
zueKP<l!fl`X!2G7PZIc0Fpb69Dx+%vCMN80j0uSl6mrY%ma}CH$RxlwGl2<4K=g_U
zhKlg;7!C#!4ZP~XqM0n}5Gj{XHQxLNRXiQcuDI77<A55qkJ?Y3gn<uHfVXi*9Ip~%
zC6KEAbS~n4*;V~odL7zJoxWWk7KKcNyvsxIS1^s`gQBi>R%(wUYc71baa>$NLc(ei
zE+iOWcx@r4G?;n-Ii>FON%xo1WaID9xNU{JWaZ1&NspCz*&Yu}kl@wxdi0QDBnY^)
z07KlysTQAq1`bn{Cm{bvNXKr4%Ln$*`6!Jghsx}L*#%~?(`TJnUKn7<lu101dI+Su
zljpQ)yqR+X44EjZ8VIM)YJ`VS^E?LwR!qseu5v!oDANAv%;|A*#9dr#1PcJA(XCUH
z(RMco`Wwf)<FUgifJWRB6|m^FXJc7(|3~K+E<YxW8PV}LE&>c})_<q@)BQ|Htd!5H
zQ%c`nOk9d4p_?$m%qGf+Ed>h_DXiwl0WUJa<^{tXT0l4@z0w9Ju^Hx=Fhh=v1pVky
z)6ssSm9vCyFBov#=?@hV1T-+-wptm!AH!prfDlaJ3>W}Nhrz4@@x;e&NV!`=yEP1O
zjt%B<Ob`8LS?p`jGby;!nIOx3o4kwarTz5v(YB|kw00=Z%#*5XrRl)51oGlq(=!j?
z1w1VqF`C{09GeC{XchX7^aUYGc55vGZDqDNuu#59M@Ls&tha_+sv%@jNCC4}5P=M|
z2RfiO&(skkba@t|j}&*Fn(U92*|)ast-!1mZquH(kBU$)*AYbuu_J;HRueesA&B^U
z{?3Ji7an7j-uxC^S5IGVGIfxQHU>d2GZsR3zx>byDzEpylcwONuq%MCky?1B2;naB
z91fv4+<l76F1iP@lucjnlBpyb#LO^P5x1Gc^AoXv1=4pxL6Mj5Dxc|y4hSpQyjGy;
zl;9e76Uzq~m1#{-AU0pzS{)xh>UIUCN;Aa0?{l_T$R=4}j?&IS<0)ss8sbu6(o0XF
zsbWElkHx3taoe*E`k%=pmg)pTX8&#nc>5Qs@d{;jrU;pnp?4OoJw4~n+Zn$CIZG;J
zzs5M?ox9>h$dT*8I^gp{KU?}%u9l<uFs!<lXWIstnym)@v$EL)B`ny{)B_TbI}a)e
zMb%2j@pd06xWl{|r?na!a4)0HoV>h)IV^$KF%xl*V4O`_@hHOifsSrlg23*~b{L6C
zNa!F>fM?kn{|3P+hH9pWVhk)iv5SfSOcFT3j9lpNGm@KdA*l%%pTf1hv=wDFs`~}-
zDC9vF{59ZZDoMJS7RvAf*}{y{jmJmMIm6-qdmbE-*-7>%D4A>ibXGF!`$~tHGk`mB
z@ZMpTo?K@*mxT$Or9#OYUGU`{SRgq24nZH`DiEGJlv$2?p)Vs0ghE~9NyP=148*E0
zVj<Kkq+u$L50|;_z7-}wRxA`t-Piz@$awZ-XXOtQ3O#}-Jrs?Fr7R#AA?cMLa&AiH
zMEo(ZFj=65^}J%NF`}OYh6!Q6WZbTeGRa1jA)#ya{Daqny2jb_G&L~PCT{f{JwoWx
zUd95E6-Vn!@GWcy&-9Qg%IY<MMx0!SirwVt|KuWS5ND!l76<`GS>Y{@{rq!8DgYVu
z5BK9TpzsFza$ohN;PKV|__!D7DT)oOseWW`;v;)W7{g-}@)Z}Ukch`uLypwmF?dkk
z2=rEa(3XJ9PB?H(QUDng2CkT#MKniIxROfr0>WABB~}CAlEIolcu;`BrbFLTEB2-W
zjPb@!JErc!u?iJ}gjrM&YJz&HdGC-cN<2)+&aQBNxg3FOkohs|>FFUQE-w6B2n(iW
zVF73R!_NTw!<AErm`puM*XS5{5PFywR0;Vr5~JCXxko3*hf*NVnu==$^sUS41u4A0
z1p$}&FgMrT2K8<Pw$Vbj1UyC)I5<O?E%0TAuQ1XCyi{9GA>5gi&a~zk+x*c2l9Gj+
zDR6^AwANmb@<t2pfJj%ko5=x6A)I{Wh|Cq}vX6T{{1B2ONgQE@RC^jnSv?CdaFPB2
zXPh|dA|llTZU7V-<|C!@+@|UG1?+EU-`wOqdn#Atsd&&^@Kl&=Ymm^!pmI&xqUn%{
zO9<>5)hoKf4Q_ca%G2VCBJ_CggI?tLuGJTWnCiKebbSM<r-Lms@>q^>+dH~!FET+*
zgv;o0&mc?GKhy!}7qzZ+l_L8iE?zxp75#_<G)$rFRYwYdR9Rp)ObF0AQoyN^mA!2Q
z^PfVJlrSJcS^;?@UW_6*at*oN-80=mnp~pA$)%BURPIpqQz^H3zsk?Z;RCPa1Z$SS
zlZFF1w-u(A`gdqKyb5!Rpwn0wUADxw(4Fk3hBT<6F=zzpglY56pX_&fv>@~>Bn4>?
zjC6w|yODdw!=H~0`zImMirqq(hc5+d*SKHOf}g|FX-Yne2@4=jh_ZNk4eqP_;yld;
z;(3w0?1$}KTUH#%oTK$S%J{A$i&y3O+rFpW&~8BVpD7ZMWS`|BGM99+$2Gtg7F;)n
zWYC9)#mNUbq-YqzH0Q3UE&|wrau5_S8Hb#pqV#|NJ`_qznn(!EfWot9aNH!FF8i4<
zSzvd>U{PrYJ^DxcjY%Qm2{^GrLPCIpC52KN<kz7b2G11fMNB4@a<v6wGMykl2FpqX
zzou}f!3nJOl~;81l>!lwr2#htrKT5w|7cBm@&-w)&H@|uxBnK=ZtyusL)nA;Yn`Dc
za;>uSnz1(RF<fFuQid>*K-`KU%sUixBvFUlh~pVVJVN~HV*@b^5GEp$v;(PmhzBA8
zoS?-R>9g%dg~5cdgyrox`1r})q*w)nadh9C#5twHJ}WJ2lm90TVM*<8cY{!H8?eHI
zA`h4`FxMbHkI)pjH^2zIY}v1Z?6Xw9TB*tigcmmJQ=-UFG(>0#6J>7G(a~8zha1@1
zd#q1i*)@u@0;vVm!Qf3Ai2ae`NM2zz2>fo5-IV?a>A-uo%xus;X85GXs;`xYr&?0g
z(0O_`0Q6K__9X_%)1I;}R4c!<>e)xB3V`lMe{tY=#nNNbTz3oFF|{u{&c&ntA~W4U
z6D}1(FIq7SgFoRZy%?<^up$D8p)RRY8CK2GyshOS#WK5vr&qm&%DMx=LGj;&BWh5{
z4bk|7o|uY?6`78Pz#K%B5@|o*@$yk!4p25HoCH%KrsKKzB3r$`&*%V<>FD4c8RRn1
zujgEIa2VN9dkDvuiJSWcKsICJNtit`P<iic;EeeHEp>U_wSu8C`+L*DMo3&hs(&EY
z+MmgXlL**2vYLvQK^Vh}@jT%TUfl=r-5%F35M25A_Xj^j6mW0ONQpECJK==d0g8SZ
zAe0$ud&+b2#n)<4;0H+0u{6LR=#J|=1IZ{J?(%rm_|bP>(;mixO*JsncAB|^97`)J
zD=d{YVlWzs35df{bi%#}_1o^t?Qo6nj1;f9Z3Sb-6ca>VU9=p`tEsRahVpU&$snM`
zx%TJt_{bh+L8NOPyM_g|Arw9$bOV_bBKO<~gdF181tRJ*?PYoS7Xe5G4ZJ`$A`Tq^
zSvpl?!D3J5!LB_$@LtAGA{q8s0wOxDieR?+)mKo*Udyc^MpCzlXJBs@^(nXEf(f2z
z{T;~e-ghMJgX*6V2AUxNY+<}g@O(%%s{9=;W+MK}$t|F6Vm-LH^Zyy#HKFANFtG(x
zZXznrmWPX^04kZdqk0u?GD(K7S=`$~Ac(3EoSyR*Safh#hv$KbVMmm*M1n`pfh;vP
zOW#`gqXM#5!;2l|Rbq(a5F+j>b3jC}PzINTg5^0q@j#@$ye5U<@^SaW?0HW0(Y~iV
zC650spagE*bazt0L#Q))0KBYbnn+~1JowY41ab3_QQs_U`vO@3NEE@|+k1FgFUabG
z@4)i{+|~>Q&Nc!qDQpqqGY2^<qZ;qU17e1U)>Ci8S=4&^YI=DLp=ufc8%s;vg@E0I
z9Zp{CWd_JQ1cCfYblZu^c+>Mu>8@fGI>7~<PlUw)5Xug3et85mFK3<_SY%lh4FsFQ
z+Pfokh9Z`z$)wxq@$M|D5+Ld)HKcfA=iyg~@c;i~-Gqj*?UdUPMQQyz5=4rHh~!=#
zhXG9-<9y5XRYWHEgB&avuJB-+y(mrLht14#OLZwiW>gV__`y%@86SAzA9f3&;1WfF
zLt=o~#ZlgS&_IieG*^m)R-haPJOaU^Dm4L#6;K-F-Gbi{nLjYR51Py75?o|!>*}&E
zvvk8gj^hGdD~4!*ky=`8G#(;z$ty>=Ohd?nkz{TNoIe~wVWjQ?sDb-_&<eV44|3O2
zglahzFvP|kd^Q0M7~I=SFv!W&?76Al6vUn;!IO2OKUeNSTD%sBpr}+pdK<s4TQ)%_
z2|7mh<(ZPNIvJ^;a22cpmeF@tL7Ff?4+Hd*B=iLc4D)<I09FEsqvmd_K_ZJ;f;%6i
z4@w|toTxllyxR<u91C9!bSxh<DLg14?#0Ez<vkDj4d$}VElB*BAhoDJt0X}=2<Yfn
z557n9JAFrLZxA~STQZ<esl7sCx>1<H4xalnoYw1OcD{k0g1G+yEsFyyBR_hNLV?X^
z-^?+E=yIfoTMzr{4zj7|OPsFpLr8WH)^r7ngM;IX<<X)s1&DFNCnj{E%K=QE;us}(
z>(;G%7g}yXxJyDP#H|%0D_lT|P?MJaKpKCHXwZ=00168q-@4k`BTFF>Ft13;uD9_W
zzZ_SJ2HF(VT5+U3&#XFwg4b~^Njm!86XUjMy|J&ip^`rzTdPjQ4y;>-MKC@BR4kD;
z(May&Ge8a`24}1cGRE^ua&jto6L3Qe!Th0r?aw@Bii$226Vl%8gRJc@4CIutwEai$
zt*hq+HMM4N7Es>Qao=wU*E<`aK$;7gB}GtN*O|FOlZ*4*RURw~;;tEdM63^cbBN}H
zWG&Wdz!nCYDSS5s_Cp*Ojw#Mr4GjoWBPa$sTwyaf6JPJ;dZOubGWiSn1JJS>P!tHO
zeg(K0lo=$<fDx*ZzC{-6>$q5$E_g@=gXlhKT-Usit}e;&TY6?6w&uHqp?+_{PVfx!
zvd5Qsq1^*22SSJz0udR@8GN>dZ^FPF$JMNKRYMH5rRX&bIo$Z$%m5Lb;&`+Wxco}I
z_u2@>b_z-AbSjvhHd)tIR&@=e7KB9beVEB&99j3ka!m8-)29^_I8;b*WAg%Ghy*;4
zXw;GlRMWQaB_c-_`}LhX<MwL>fervHNiSb!0<x3a@}A(y-n6kTe(d*qui<DQHb*nn
zdmqbzWDnv@yUidp!^F_>y!tJz?KpWpLq6G83-<~$oaYy3Wi*#v3$DX~FLb15@)J)Y
zY`eNb2#(Rg;4~!j^&6xG`k*u}esZ+$Mogkb4=gVj+|YIhmjyOIOVV}edA<mSXMx15
zN15gJ3+#_T%^n5*$hj2t38{mHQg+2u9W+M$?sr}Nxf@#bZ)W(KdEy}60Z`1B`!Qgi
zA}Rq}=ur)`fgE!_q6=WCV+dMwvvn2$^F|8uAn5pr1O_NUFaoC<0?L+8sV~*O8{*C9
zbMeAcvtwqP^Z`pb1OkK?z)4qmGpnC%PcHsEfjR5pms1bWEj>ZTRFXPEOb&xel(PW!
zBd``1ps5NP-HgL=9W`}9Zh({u3gU!#62B^@2W9B&0_i1BR6l)S2ODEb;<(YwYy?=|
zW^JOTP_yIy*#UZR8B!nLA%EpCM?{5jrrL8sKP(8}0pl;qxL5~0gmTFF31?ZM*Cq2p
z$_PNW!f-DHxTqJt)LR^o|2ujU=9u||#9^M$yG&Zy1bZpe{ofI7J@k3dKm=UGZ<|V>
zDfvOk4>eWD2+bq=(^6&5MwnQ@nxV=s=rDK+y~l<bb~9mek2yS+7s~_xr2(6Puw_4w
zPa!RTJr3cv_7rr1DgPoBTX?|)5*)Xy7v0XXPi#+QJP%S(-~<da1W5@Pe)W<22owYt
z`EaU_jd0;_A$GHGFI>kIB<s=<{|~cwL(pCGj&jy5P-=1A{{m99d6Q&1m@k%B;EA@#
z=F_HuA+gXBuq<6OcxKQew~$I$|2HLg8kl@4-KzfSGZ5-MP<lJ*v`)S+L#Im#Sj6E+
zZy=eM9WH_-fM76q5MPSq+#tJwoV)KUNtJ6#U4#^b5FRXbKgFo#MHbvqald~RiE_B}
zvBHneSt2AY6aWfLV{fk68KeMZ;_^jm7|BN<zEOJeM9|RzQHM0h{j0QT`uGIVUS8$3
zN`f(y0_a6=2jsx#;sZ-81mDJhP*(g~UBcR{iG*j5v&@uaWQ+tJ<H>BTa%H>J_a}lW
z5*5%p@dy7+H@&COH#j?~p91lTabF{j(EmjK5M~xdk5UvFYe8vE8WWXm)QEWhy>cNy
z6-JPNBZGt;e(vnXN+9E4JQMMb00rOl;xXyC0nz+D|629Ao^&}HQ$+IJ65ibTOt=(K
z#DB-C90?(c6XQFl5bZ^)DxpT%W=y7DApkT#(KDfU@RdRvSY;!ENS*zMt%)KQ?fzZf
ze^+i7|7DK_Fpr!tMY-Eh8$wkIq-e93Hi6LiQ6&tJ46_RO*SV6CplTAUU_``h2!3;x
z(hu}n-DUPjE1muFNbk99Bm?|S(v>R7s$TdJ7zXXd-`TI?tKa<-N{g|;g(XW&uxgdP
zuv5$a9R@AACG%+%bU;7a>p?jLYIdZk3Q`EDo@j?{P|dW0!m54lUy>d=vp^j={QNB)
z15$&syB+#_wJ$z@-=K*N6>p7DHz#yS*HxwCA1Q&->$QUbJ7^sKQ9E>R+=q@|T{-yr
zor=}K;WR8BixJ}N^v`(Z;KSdaa3-zJZZ8k@N_Upr#z#(ZEG~3!^s<EYlVX%{`~SO6
zcP|1HA7DY4eyXw%yYl(-=bkJzu1y!df`)9zMi8cTA`E2~dp3hE9%j7Sm(%rN8bNKq
zyOs{Qi{n*QP~U1fqSy7O#Y4JS*Yc|2Y1(KW11O2CIz3*cMt_(+vo2!ss%mh2ZL8~u
zzOAYe3X9#0Yj#D_utmM&n5<wOGhkn8Cln;KSf-SNFi{wRKR|c8BAR7T)qrKgq+$t|
zWKATm*~{?kDAf32lN!hG6w1STv_(ydNKtiFFyrr60?z>YhLw_W5vkGn(^tp{t=N{K
zejjTFT^!^@mbM0VD`S01>dn|jy0;nzw&P0ywtsr71w8$0qZ(c0mVsfZOfGMR(zHUF
zbj*}<bmy77gz+!cG^d?oPSq0fEP549YBn9>q>N1rc<<l+rS&%a7{??*%;fL=_(`+l
z4SCMcFmFRVQV}6+CUMfYVR0mwlQNere~<ngeG^+Azp`*RoH(*vkM`)??swPHDl!Sq
zO3|YWE=Y}H)ljZjgQF(~Bp6E5<+&a)hSxhhq0)e8q9AWi1Kg1^`vBlwWFgMobjUCX
z$pL1*4>GROZoV{1d3pG>5Xl7IrH+n9Y7~(Bczp&|I>Z>5$WPt}1kk>V>hksV?ExCk
z$om`6Vxe7FWIM(7=cuu<FqttX*(TntV-VxActKXv3xy1(!#=3^EHqH+kgt|O3l+VA
zzHaw4nid0#kiI&h5pWJ|#~{!K{5btLl+K@B2f1N>=dE`@S<-4d9+d<Mf^*cWSabY-
z$6<MpWE?~62?lofO^iv|1@sdTI0|A0nb!>IQL?IwK;-_yGwvGZ4D6LZ^O24Z16+uf
zcoTP|$|DgRh8#QqMd#aSF$~rVXV-dn8yf(v6!h+<K=^;Is(NJxDxy;pASZecV2(`j
zUQ#@5(q&9sl)63hwPZd_*-KS+3kugb^sxcZhQS6)fY=QYOUQ!<w6lCXjDrQ2jKp2r
z7zA)9anuRPsA?Gw8`eLMhRICgxHjSCYKe=77g9QaN6Pq-Hx~|F8j!nu8MnXQ#jSV5
zB1Y>U-NnF=rLC6Ig`vZDuIf9PzJ}ViNY-==SH~?MEaVcj6NwmdRxFyOAED{sUDD_Y
z$0RPUCkC#kXBsvF7+PNfjFo_codm$72MXviP<Q=>!jn0?(cTk)@s{uB0a<o;AO>^H
z70^|@bBJ^O;<~><S`~CIYjZ=D{A|x*Bk~%b0Qjq)wg8HgBrz}<xib7Dad?69wihZ_
zBK3c&JOr8K65=4D(@B{^k3p_=Yd1Hs)dN_y4zysOwjvgI{ZvpdbwfgY+hU}&FHcX{
zUq=Nx)MPXQjlJNeVJ-($V}oFnx-@hqhFw8<IbB3wJi8PIRtJ3^v|5s^@oMjkdz{LE
z80MoP#SNm!Dvu<1;IG0MwegSKygt=iVcp<&gO>MDEB)26pQQZ7px^5Q(tMr5f6}qb
z^~{DvJR7@#*lXb&g06b?|FJPzc$#H)5~ONYPXoZ+%fre2s#Wf=;Q<u9z4?RT^H@Rj
z7?U0HqdqN*;z$hy3O<b21l{>z^MWpt-x7irhjgH4$tn#O2d7_>!9HkV^L{4sz|>>+
z1}HMLt;&ysErfym+z^`p=w1j@>DOzgQ)HnoN+D-2Yh2?YCJ(>$7Q?93S{s~h=7_6?
zk1-9^1EDYv!2xnrCRjBv%`37O+Ht_3yyofhpcft{&w)1W8!YnH2ssM#UrTGpLp&Vh
ze;_(Ht%o<;Y|Mo5I^G~+hnWMPN~qh<VQ%9Jl`7`SJ!fb?)2UAlz?jANysR#uofpY7
zkmLK;2<928uWo|RU~4smb8^Ft=U4nc&mly6fp}sN6&E4E$oLkH=98n6LeQ+!PeH!O
za)DTg?1|7NFy_8_oW7^FSVv;P1}q<LJPd`2&$l?123IOzok?3|H)oTb5R=m7!_?jg
zW0In9hPDqz_pd9LtIxqfM=mAAa8@Y8bv%mqE$EG8(^9W&h2kv6q#O653Ad;Wa*<`q
zK1nhxXo;cZu}w%Vvzo-Z%&H#lyfyy>>^==V(W@Rw?sLf3^)XZr{Q#m-3`~{zM8WHy
z;{<m=Y>x$^=5!D$_JDCfx2kaXBYQKm(X#vY%Yz{h{x5IiOc27oPQt?B`(g75(Jo>G
zcP0eRG!5GLiA@5swSShe_=)Ncd@X38LT-J+6xS*=5$V@WkpPF1I7=FL?q9Q}CC0<L
zAr(nQ4qV5xIO6vP_M7j&Al#KrfcVJ8bZ-n?5g{-IY%42VY<l4d@VKG=rl2h;1W+s)
z%7|x;p|X?L3T(BZg^%q&EA>UlvvC^g81fK~Ucy@f<V(re{&lLD#Ezt1P<W1EffqB7
z#tXPaLZT;2^~f#>>aov(s7$bX81x1Mm){HHZ8<YWtse4uVA$W4^STUDJ+yiI-MC%y
z266nUD&0~~*!(0f;%r0&F$t5jVC??7^DOP~Eo2+de!Q3=Q8=fbukYiCR4YOt^%Vor
z3q#NfO#B_HR>6?Nz?6Zg)LUrKomEXXF_Z!W$GvaL#)+%sXnqE2?>$Nj&@|#JymB!T
z{`uO8Fd>*HV(2<M_@$G7GqBS(kL=7kV3nc!Hi%l_j`bRE`60OYug^a}rJn#$8%SJB
zZ#80VB7j`0K&Sf~1}RLa4X)3f_MD3=TKm8=_HoZXTJ$2V-N-fMyXc?LLXG(a=;33w
zgX@iWFaP%fpYYZe5pjz@<rwJPZbK5*2c;!{AC)}anwRd^EO4-#Nj<I7O{%^qbxV7m
zf8=mDyJrt<5D5`s&`=jBt)|%D{kuM9#0U@PDAWlm%*rcY<?r@k{pT`>ws*>>k&qa*
zIu^0KS`i2($FlEP6mQ|)T**AG=3?=8ygi9Qv~kl7M0(uN_43Gi3LA-8^^Yi49E)+!
zpM8}OD;83kl%UYQ8Ni!twtdO$KSn2w_^u7AyWHI2K{SH0pqfdC;+Q<BKA5z5Z-Faz
zgvXd`K9?vYgo_j`xh+O`KXDpe)PlD(cxO~XAIXb!S*nx=jeKXv<|3IRjDiH}#1-(|
z9iLbJ-w<zp_G3k{2RqQjslK`D0Ere8g*KEqZ$Z7z6B`aYv8R>a00#C5uGP(_+Jxm$
z-5{Cg#a95;PKM+9l!b}jKYJ@AiUqfp40_{kpx81%uy;q9#Qs0Qlz8n4g7ZVSf~lAX
zUp<BzDJf1a%qs}LqjZNxMVAt{{st0EWp$EM-EpmkHyQ#c19Tg$DIsj%wipKYo(kOF
z;g5^u|D2u|3|+Z8f(Ia~nuiG}_coN4ZuI@fXD4`90}1uFTuodY7b5-#XhIQzN+~Hx
zE;OjgS>wcEVI7cw0q7)t5tWnh><tCVSS_+X;i0YX#%#M3L<i=Ceu+5Vb;Yss>Hn<J
z*QcIQO~Ap#Lsv|uWW;l#e{mKDW&q+<Xq#E!mohh8;#8hkbK+BMsCEfsdPP44mms}h
zTG%T(0m%(Fr0sa`oI!^3?fGXc>!UL8AGAT<0i+KQ=TohQ?S8UvatDw<OTpyC@Be#1
zor!h3k&sS5MaA#<Tv;5!JD`Ubht{9&<dB!=|Lc0h*|A_KN90T%LdMxmgD<71LFOep
z-zU#TB8}G-ab0No*lK1@M9>p5@Z|!jP}qMQw+9mXBOqEYz7wW~ante@YXFNxt*7d{
zX)x5UFd)b@WR%+WS~~~nY-b|7Hx&-+9J?(tU#KR&Hz@fZQ<{I0B;<t^4u!8_U^=KT
zJ-vJObePyrgR-oJSmi~j@J1?JBPw<FSlt9Y?rY(d3*F{S`5Z*|=JqSEkw*_G&3oPn
z&ABSkk6yz?g*@}cj}!6K`kD?f7>EJDrND<PfEFgc54-Q56F-56jSw)chByRQXv4VV
zc&YsUb+G>69baPk|53fUwwCB;w^puIXuI&pjH7~xNjz{66{$D%R0E<z!Fev~DMqZr
z-PNdNZ=j4F6pG|kh^+`PuPg^G+<8F`F^vREnZQBxTacNj5HD8Vxmvm!Lc{>xee}G{
zhe3@Zp>KHdF9^e})(w6~7<7h{9O18a)=)Ln#LwSgEMwe1z$}74vLeL1>j0Q;Zj0UQ
z3^lcoc-G`-MkPH3fBc>-Pe1SO7~>1}3uK;D#Bca$WB@$miB!L*A~vt3x?nEl<{JPD
zs(%mj?wew>(tA6ZzZt#~0fJe?w!3nuWD<BVB!=PO>6W$52zMlatD1j~c4VP@7tF-d
z^BkV))k39LHwhg2ow2vUB%a565*k!32FeQ_;#L#p1>aAA^PYtB&kYJ3P-i3eOM7;y
z_O~@T>cL5m;U>S4b?Ggu#B(f~&|nT??&SXpgFD)y)ombzLnStzd>KAN$f#@7rJNme
zZ%}E81UZ>P_E?VskH2v}J6>kM;H!T8+mRIRh#EU<5R(|aHSbagj<wngJBwiW+kLVN
z<z@oj2MA!474pm1vH$fo(G6P{XRz^_9Qc#2H^{Qr##6fmE5m1aKYB(y(I_&m)m6zB
zdrkw7-H$g93)69ByC4|W{ZMsU0`AC}I8Pr3LQC>$_eJ;&4rw0e%go=>SC+qR<1T{B
zO?re$^~=5gYO?zdi6@mV%`1L9xN9OT<z)<9l=GH=Iws(FAuQz!=KY`W-e?t3KXHS{
ze!-knH<<g#ps({W7{;=}2WHs%`6sYnkn;priXqm2L%g~rDO;UOFYkXh>wh;3j{g6i
ztp9%}%To|*PVVH1U&(4RSJlxol;x=?yq__-ZQn+IazHzUd_nQBoZ#>c;w^#Q5Rdn1
z1Yquoc!&FD|F4dSn1FAhcGYBIr~f+jhd?qOHYS<)I0Yl=x%~0tx|JoK1#dbltI5p*
zfyBn+yw|b2vXOVrr#44++aI<3z_&ZDHQJBkb>fsv<O*1d{$V{kV6ZhY<|Y|-pMgm#
zzINn8qTG{-zjA&FQ&}Ez{$!o=Ro;__tK(ey9lvvomQ2w}92;n-k^{khozK4+#&^d)
zbUwLx=0qv~%GOOo5YuNnbG?P$;5mL_Y-4)ql2uE)<hjt=bPnh(2Wtjc7k6NL>)}YR
za>Y61j?9z2A{Bmd_jwMs(o$b^&s@u*b;pVQA5(>aV9mqFdk%L}TSJDnV|dzJs#Hrj
z?;ob5FAergD!x1}6y12}H2i{lt6YtoPxi)u_1Kjc`Ak|T({g5yQg9+Nbo!^SJl<no
zo9|{=+|ay#%Dl%*$|w8!+U919mD|Mhi)>W>#J*E9D&@n&_kYu$=0=jPA1sH2+0Axw
zd>hJsIOF`;^EW}<An#e+W8UFjBmj4N{_NQ<5&70wboiDRR7;ohU79mDNE>xd)Jq$E
zRIaO)Q8!4#;^3I$j67rXlzMH7|LrB)&WqvYC97Q#wv-DyMU4rL>!kJRC}R)VA^aKp
zCz}*#?N>@si#z74k2<c;{$wCU?^P5V;B?_f=uF9|cv;OoCCRu%fxe*pk{bOfQ;*V!
zX4k0aMYBMVVAxHjpXF9rjg`@Rdizc6rYkA?^;P(ZuVaMqNSWXJU%4@0e0Kkh_FQ+6
z`^P`Ke{-U=8g#&U<6$>>c^4Yv;p|-^_@@AWHUgqu&0jy0Oi|is2O-(Iy05!O-E6c+
zyML>H#nV?FyUnrtuiQ6eaQW2`X<Ok_&UicFb1+ZmO$y((jlx>36HfN$1E#0<*~1cv
z(fn2-`@-UfUR$k^qc2LInl2J^^0w^uu3wl|uVT+%k?eDgP<J0wX}oDRbo1;US#{2H
zUxavycNR~J?nYnWGagoQxt`UJ!?p6ebX_Uhw4{SAT4nLx4o(<NLN<k$?l9(9ZM=l>
zXt+<|@@B?{b@c$*Z3Fc3ck_#=s|oGTqGS`3adv5*2k5zdHC!2JlS-Voj%rw2JDl#a
zhPnuaTb}Wq)b_|``;}f?6s2r@YTmeex}aRb=eH81SEcL=o~Q8--F7OgW+HanS|gzv
z+9x}9I^rHJKjc|g80SyDD0YQt?e9t5R&b3UK7Qh?O>21xy30$3Z=okf&<B&f(~yLV
zn2EEND)(2Z_4>EWYcZR%Ng|z$OvN0$rgcA(nHO?nOK#go64cYHjO?vz&TU+16y&d*
zqb%E*V=A+s`BZ;PI#z&d=02|zyZ-}rL+ib^+{WAPU$Ao=UFPKCLJmjDoa1}0S`#OF
zq#Bl*#0+f8<%}mbc0M1>A77}=5PZj+G!x$xJNi{va#zvKBG`PJnrdjBEFwcC;uGP@
z<mL?>KAMwFBB}%T+<Mo)<LBH@OKFC}x^*7zOg|Z*qbQfLR5nIg^|F<@?o?FV{SiR<
zWuzf7%9j0FmB|3RCWkvt!Bumn1EWo%9-{Ltm(%So=(r~mt`0J<GXyBPJ*<6TJHF=9
zmJoe4@Nnj?lE}fJ((T+2c?RiP)TOSsG-^hk=`f}Eo${yXl%O9(U#}wa7>yX8BrUr0
zJZmt&0!>^Rd<J5?mWW2KA~`2L0?vZge4p@s4Xp`^*`ck#z;rb<B-p`F7GBUWNYh$C
zuWLL6G*tDW&g+Vp5EEbZ+tb$BQSu(WB-htG7PObMUnm#LOa?XZ1bLLq#jf5`$@lJi
zg~r|NVZMlB=lQ1Rx^OBl^8UT~qI~>gbFG1Aj3fE;_jccK_*&)vv9J6=a@XPY9ScU1
zuVY0q@2<|C%<nlZNMCX5w(z987{6Ug6<~c?V<36>r@G5BkKopAwP5r}sgZk1_n}Hx
zx7MMlFNw}apKI{e)q@+`CuZUM9O@p=ei25GS~9Mve*eAgD%<q2T!ikEZUqiY&W|X$
zs?!z*r>2T01`CIx<2}iKM5A(H=CV&WetWo(l=POzJ>I*1DRV&SZI||iH@8Q9drTQJ
zGhOsTN*mks9-qGrm9JeI(MAUvt16M=huG2?k1l7Z&)uBFR;CD2^PKs<HvRmO+uAJw
zol0`0g|6p2>w@c~H?3G1P%B5L7aFVZSGK(J3y-`TKbJnM4gNH$@jSJ7TVN-aW~}4X
za^xAsU9lAnSr=vVv>)fhq;6_#R?M|VrO18`Hjr???|8%fc=7s_(E=YuuKiENBbJ>n
zM~ijqq?Dn#Gj8pd=;F!D9HVj@JGdjU*^@k|`6<Hod8_I=e{Rv3@i-*;4P-f)zf7;8
z=ZsAp5`OT3Ze(MyI^V0kmi}n)&3Myso152(DT3K4=e28&k({ZT2=Qr&8t2VR-Xk7E
zKnJI-5Ipkc2U>&Tt6VFxv8KCdVb=?oO1NKN;#f^x)ZS=GZETwiPdC&Xdf>y?anLd*
zVW7^htv={s^mTGOQn<sZM`0*5SB1FSa%TQ&I5!h}QT+7A*Z1P3hZTB6`s@#Ow)9DE
z9d0^_zSXtNWZ>KNr+l7Vqvh{YGk<4UrrsZ4_3PxIiI0v;2r09iB=gRR**mqyzZ^1t
zl?INll<;<&_G#Rn?4n0f1u}VhkE>QCjC<5pgFl+@wCL--BeN!VO~tlP;AI@y?#^f6
ztR3AEeKlV?N#dYCo?AbLiVN3UEzxM`4f>hYW85iXLX`N??1R8wk>TYBOHB^b8zjAq
z0ryP4$%M1u9UH$hT^4Bc`PsckrDw^ztA7zy5geS~71Jw@U8C*n8hBotEtlIR{u;qN
z%OCV9K~B#nwi9<*{vB1J4rN-NWAm+{AIa$=ZPR!Q#Rdnx_)6LiRe_Il`a?R79a;<$
zcX2H0=E^BuRw=se2V=ETS4%s5v{_}=$(U2d-S^ZgPE_ml^UK8>I45)Bx5o^;Tn09_
zr*1dd$Nt__%)hN*C@!3D(2#alkMf;|m^y)nN)C??-^s^2ijVGCIP#&jem1NP5`;}_
zb}CrD2;U76ptdFN5Obpl3^AA${6pz3WlLcF5#MFVm}^RTtitt!yZYfz{y*cKlQLY9
z*T(XhMXY9>`(3KenSc5rErQZ6s=N9%hQTs;&)R>VSHoR>pAdJlBQmE(7(9y*EkBr$
z!(KTyD)`KVPTiRA^@5A>824?|h&DydN|&Pqr?_yxJ{7vp=gEBY8TW!{IzdQa!?)lM
z%efVXp#ir?))l&gKM0-BD80Iy@G_S>TlCU}O4d1kiPGt}<l8t7X_zzPq}`s=@AIvy
z8xpM=ztmK#&@k(K?N2_hv@)pL!E*{bM8WmV?lohpT>G*7v1u_=e)kEQ%a_FEn$2XY
zcfrAjB$`M|V!Hg*FV^ABs_KiGTdciZs*jwtDnj<k)P2R2XLc?z)v*O=Rn+Irm;{Sn
z(-{ym(lLC?xB1W(=fme!$}ETe63@wyepgjX;TPqs$L;iEU)@^UdS~X2%Xb5fdUr+h
zYz6a-&;#73$?M-U_tSrh>e>jR*+$mg_~foVbs}IFWMe72lsm3fF&TY$C{<-u)z`G=
z9QQbTb1(kYksgn8hMs-!^ijztA3>cl%K*W65tY+b*4?FIfqE$iMHAUo^_*57Nz-?R
zmX&icjy}5{-Nl0I_gS9pMBg8oncN2$`l;vGVEy?t)5x*%+ZKWSYpXp)y7o?fgVP_P
zsy1v}s+2V9<tU!C@?^?v<Jg6^N>8#G{VMEp8M~$LWoh4}sZaEX!lPX7^u^odH@`@S
z@u)8NGc&q~eDX2a4>J2LCW2nn`Xqm8+M^(^^ilDQ%Z5wS)ue}?19NL*Qua}aORcN-
z@ITA=*-?HDZM;Z9?)N%wVX4P+rC@`KxK^vW>>LkL(}S875E2ZP$;V<!T!S$f#=|sK
zIMJq!(&fAFs#|04^}1^t55zy{MUO`95nz2I#!aKD&R=RP)m~nymm(^0E0V1I$d!1(
zw|YmicR=f_^kh(&0>StXq5AaHy7YItbypv%xpFkHd8io;31>{dtucI$d80L;V8CXb
zc_bgFzcCjTZtIFCWJCMuR1dSjB>&MAUE@^e^`Czn`6p**GW$*=N<@e!XDTa}2UIwG
z_#P|gNU)mm&gTynI&5H(n2r2v7cbuOxm%Roh!MRv?LLzl!qWHMliRL0EBzY<zW!=e
zpU7K|Rikd#bEwH2#d*K)n*GPWZkUAlWEo~Cek{MzFc<ywvL))%UPyiI^h#4?Q+?@S
z>mJpDIg8iqRA#5lGgp3kx;AovvESma1w^<0F?Ja}|3I<SKri&XB-?nqx&_O6{KTYI
z<ND^a`%A1(H=+|&TAYrET-+^U$`$za4_On5*$AmzywI9uvQsZ?DkOCpj<jYBh6v;a
z#&kZA2e*>Bc#wn$zVb3z_{o$|`z`lSWbX0ZaZI`qQ4^vGZD3B2;)n8u74ea=NX0j%
zd50GzKfr-5A3fUeig9QwjI{*#8_(Bku|wzX?P-SXZHlvJ5YndFqBgyfwq-N@i8?*H
zHvUK8+VB4BeN~!d5(;{3SL4^wRtI_w&(_EE)@xZ<*0K|e-(8I4iuApfpyWr8o4W41
zOB#i#BNsYxNb|ysNeUyDaN&v3;^}pwz@M&TcmIfE&Dbt;YCme$b{<OTG#Gn4>|w*}
z`T0JLi{^Tdu(tQS*0ZX9vB<YJHiRBu{BDor?@lpm%t=1IR7u2MB#^)QvCwNJUnu<R
zMbt{;r7!dm)a?6)-HUz^v4$CW3v>&)9dY$j#p@RQoF@kC+(EbJ&=IEEb8I7*wE5Z@
z<na;<9C}<O=o_TI`K_qxYTp!evMG|wIH*ssSawqWEJ}4pJGnxqQ!F@O^s&gZQofyO
z%ysSQRzb~Oru9(kRn9_V9Foerg;`6z2Em=+swo&B-sP(5)g0tQZ?QK9mWG_yT;F!Z
zL&eTzC7l<3%Kw&6dy4;yb*EMBsL3x<+nod>_Re7ejb_`CQx4`Ou9<d;V-;yD!N0=}
zlxS5$9x{`WQoD$q>>*=Enlzh+8aplWiA!fYdcIWA$^~I6*TjI5T3OYu8K#kbL&ddn
z;sXCfo9?>v-Rng!$2hE>)SD1L_Yu0cf<ZB1L*8NO7;!9P{7Wim`nR#22cF$(dlNZ&
z#6VqUX*+{4Lsu@YGT>Ke%-`z7MlG+XfZji0e%Jg7sJn3@)Fmrev~Q|bc`&cf>*q3A
zP6jR2P%HC){$-G_Hh35yn$lx`M`DEE*2C`TXgzjyUMksKlb2BE{Z>ZZsDtyWUg6=A
zO!z|&w#5<_#u4i=y^AQL>o1aT(%A1h$tF2G6D^&d=@R~6)N{gFTQ(3=@Xh86sqB|(
zsTXQ5HrbTlmaiILPcRZ@7H1KGE3bCCahp`X44#8TB}B5~<XXgosY>e$Bg_#^xF}<{
zy@cid*?6_6-C)b!@ummmej7hk!^_<E_l_rz(GBG2J0-6scz*ahxx;x!X+-)w48ok(
z%F(JAhfW0y46I-YktZ*t{XQ(0vQP(cM+(_`<F5ith`VJt!4p<NgfmkRvFb^T_U=(~
zyoHoqh}D3Lou-g}GfIQsRv5;i@g6x$E+@XaY4L^+V--z?1zgCWv_pkeQ+RGfVD+Kx
zokr~G*HINIZ05z%leGJ!XkNu{F)3eGr>se7%>NuIxeK7d1zMh%&+|_)UcMo`TBR}D
zvvy&W>8dyTWKd_-HHVPm>qAm%dDfj9EqIvP9_gu$m)D)T<wwZuKa7;Wm^yYEA0Sr>
zO_P+ZEAHHV%#sKY-$#Pt)(VR}33?;)4!!kpLRw{<o!;Dcdz(8hpYu^uT)u0R%oaC>
zjq6(t^6wMZQVD*HLfLMd@91#eUv^Tle@LxS_e86<q0%;f?eqhzvy>oL#j|v_i*x<6
z?s7Z#MoNtDiZa^7)KW1gO^A?_T;8Oz)wy$6NR1lk(0LGP%kyD=hr03EvfAPXuDzh{
zMO4GLN|rB`hxxkZ8VK-UQ~l{P2P_XrF%V`p>3pv)gpbn_%>OTQMok8Z5^2vijw!Ho
z?MCmbl{%_czU7mp*S(oBF_T@v7+xU%xOqBgU#H5kAtJ>)s^5Z(zgYIzwLZNu9_5j5
zLd~2qv8PpG7%UYlfBaRo$Yv~E{h-3**x^&(48xDebE~!(?yLImxjRqNX}9Axc@4hq
zJ}kpW%_>M)>@NLMXlpUQ?pVQN{9r-|4~4E6`V&0vpS@Wbxt!~o6&q4Pr?c<{o$mBn
z>{+R8XGaU(Bi@}v)YZ`=t>$~JYDRy8Y5uyYKK{crvc77(N^03Ns(~UPs+W9GX2F!W
z&4HSIK>8*R7p-MSf<D2)z&JgPznbs#pD`87@#ibZm9GENDZ9={ei{EV_5$&_2n7*I
zVo~DfU3axedq&0ayi>T}dDGH;##O*lpic@9FBGlg>HIOiJQ`h&LOnn)FWb5c4*GN1
zcI+n(*^ax-3am=lh)I#Y<wg%&eo~5W`2>qm2&-F|_NGVbJImMlfyat`1d-WNzr`zS
zgC!4tFkG(7Uv+0Z{B0Q)ZmJ*=mHRm4#_sI8?&0RQ4;!WdW1)}S)UOt=;M<>6&nC`H
z`_g<{A)*Y5ytG*vLNm)2#(Dp)<nf6))vhYHW2h^ilNbMqo%O!1zK*@?{@&1c!@t}T
zVH`gR_S_>0a(X3Ysd-r}cT8E0bTXGqmB&jR7qeb#=4lqQYB76IKm1G+$tV#jUi83H
zsv*susu62c^}cDP>%%QB``F1ploH<xP5CVzrq8`p6Ew+4>HH*d!8~rsni}UZi-yX{
z4=Xk8rGrZT-QStMPEYNvG<z++?;d6=%<uQ~yw;RUd>_+)Y|!r)9Dau-JMGfO!l$e8
zzRhXuDj5|+o3oLJ_%l&~55G%(|J(c*XT_;>V?c0YIW|{3-h!rdWOXp%4d+(}eEY1q
zmS*bCk+ROJ(0w<-kS#UNmL!_FADWq~aRLqs4?1+%1X&MdI}&-5X(CgaLnTMi?Wvlh
zQ8yX`A3VFi$XIn!s@U>72Uft`JiNjBbvN<K)sxp=jB+tgACDE)TaP#yC}g^SH)wEf
z-k$i`W4@5yk$&S3>%|Jj63fv@-#2`c&1Ri^LD$XaJFg7O=c-6W%#_Cj7-=V3zqj>s
zqRBL<0(PQqvB*B-Pb$}3>QaH_wb9>egL#P*5{q^ToewKF_t2EtM}^#1hx*rst!N5?
z%d;?M=`6#}(#P@a0^NU8tyjK*$_m@6#QueTt?H{om`$Elt9yKjoHAzj(Nm8r2Qf>P
zX$;T<uSo|f4|!e(4-g8u@7gVFJzUt_D7V=jze{aYHn(kZI<l_Vll+vyUZOSQ9FxZa
zvj7{7YQc{i<EW-J%eaIgv_tjYu1gELD3<XRx;&+Gc)_`VDp7c;II>P~mB)~rfUjEq
zWACp-F*)(Xs4*pvS30j1G|cQA8yB*c(gYo<L$s<|h})G#y|Z0WC$2d>y*p(Qmb8?z
z$G6$<=d!cEIt^4PUB(C(*vKZdv06fVp7<_}cZ-*t3w>V7<$S{_5L?_6`>9+tq2O4S
zexC5wyhc%sW)-|0?2h(xbvYV(iO`$Y3nC0}s8(~FRh-1|Oo{){T4l!<H`O^lAVsq!
z{-mW5*o`z+9aW$oC;EhYA^Fzh$&MeKCWNPSf2#JiD-(-EP9CA;<IOm~1dwNM1lm66
z{Uhj+=^x(ilDTx46~m^sIWl9-u9f1Hr;2IHXB*w<bR)04h7j}0(|Za<#UE0p`0O<k
zVy&(%GrJ39CHQX_z}rvQPqP=v$Aekz6%IRT-cVspwprvq-F-);^Nr@~f!kJR$fZXu
zHLT?+YXL^73(@EUy{JyBtIK;9#q9D@IVYQm;W^q*BnPr46o_`?{UrH6i&*VWJ8%q+
z{%LR{&AW)1Ifyyo6?IXV`}53T!Z}~|jlg9L&E3w1s+1)z%cl10JVjCWe#|WAD?T2=
zEZvEsaS!EKt*<2S%Dw;noNl7f6>dql%m)#QenHL2a+f1061RC;rgk|3Kd-#+4}W#4
z71DG6PLsyD;U6Q@eV>zCENW|boJ&H}7hhOE%v~KECEWY_6aC#}l%hFWAYf}iyY|ta
zpc0F*aQEB_=DeY&>sXzCxVjfI24+?_qf+)-J0+6!;<wOids~f!rS99G3R%^K(Ib_t
z><@lp*97HwPkc5nHxfA}{1%v&TrM?v(MX`B&dX{D^A<h;w=KIEt~>c0wjHl|msP$8
zPzg<T<XV^*lhR_6;!<N?M9r#Ra~heHb?v2Y46__|+bo@q-6=J>pyAq!Zee)g;&A0q
zxWRY#13#8BDOzYXSjWA$;<LuLC^JWne&b)y{jQKDTKO&AEV+J5kX6f1Cc<^QwA}VE
zXzS~sc*m)=dR5=_`LS8{s)&a_e~J5`w8JQ`_1ihtOgGfe<|@A$8=A&``e#YtQ)}<`
zw?}BQ`Cn5ciiQnue|*Amvf1{Q-wpMp#m?O^a3Q+L-_9{joBhD0`qq$EsmA((rW(KN
zdo$;@w~rMjhDgn>(+{L5yBV#bH2U4WOeDqI(%m$P&reyGIJZ!4w3o&xFwM$1&3^b6
zO0OhoZ&^`V7ckMlkoL{Z@qL{lOXc5SD?gWcE;5A_Stglhc9G02QOYHiRvU6mLW%pw
zr6b4l=X%^ba3wNkI3y>@(f5SUv&+{o&7E^%GJa~g4Xts+j|XH=6$oFl-x#ME*9c7)
zw|JrDr@|tDJ_%;#xL{tFF!=;o@OQ*m4twj7R}4H{YpKQy8wK`Ym1C_Mc~-1D3E=<v
zxZ9CbzfPWWO&v4%c}6T_#9w<VN^6L%u+q_<;#zmj>`$x}j-`Pk)l##{B;me_bo4Z_
zdfw`?HI5#U-sO8tmv4h=tR<7(sU`eV)TsQ-@!RROTZ<>{jF+`B4{OA4UH6zHcEG|L
z+g&IyR@M`+(60K-kHPnnPHcnUrs~n^(RlSInx=@VQ?d$Nk*CK!hT`ePeC*dU)6Qqc
z6Q9h<H4$v>2VE9y@(D8upJRHPCyi=wHc?vb5_$iE#Y9!nvA0HwOpoIu+G+U-lRouO
zkq((}&g~Ff0^-`Kq8tuM!(K~6<<fHN?Xs=hlS+oyl0Amae{&93l}miAE?g(X8#5A6
zkq#lxFc1~6?^+S`@Zoce&_7t1jIa92S0hEPSRqHvduQ#QbkyxPq&5YkVXfJW*3v(J
zC*r+TmxyP}o?T(<u{FxtTX2`BrQ_Oi(fhU>{VP5~j3wx^1@YxhGsXAa)QoehagUOY
zaY9HZMYaxVr!OZ&h+A%%8u7Ym#uL<e7%yWukNA8xP{qdkh=F&7<Q~_{>!ZvZ!D{Wg
zb8srsy)1`u8fxB|1XvF5rwJs8F{Jpd+}rUZC$^-P$Iddw!JvK{Qkl!GC^$H&*7vHP
zyg8+TZM49m;HKfWmB0NGLAlDzO+t=TO)Az0CaZ&kaZSmmGWJ3CnK}U<DYX)_b1ROl
zvTCY#6uun<9GEgOnn#yN`}?oG?(w;5DD*CfRH?P~u6mY6y~rg8Z`%7B0j2?asg5-n
z%=Mf;U-wmi=pWEU$+>+?YkgvlW9{T;X6^k!|9#oqB|DF&(sDyZ#$j^#7A7c~)P__o
zW7WTZgda)zt4swqJ8N$C#Pv9Mej?2r$9QTqv#YJL?lu`uw7$8W!1{M#K<9TslmBtm
zD=f{xVV?K)0lc-uIf444Pp#C`odUAsE5(U>`yW0Lcag*&$Fk(tVzGa`T9Mu8`0VF%
zK^@YkKid{KBm-7ThGx{-4)@+D%q3rON_vS|_owjI+V|<Js16Ldl3<j_ifdDTHa@EJ
zcWxT`1KB$!0gbyf96moA%Irr({s`Tu&GnU$*DrCu(Mex%jwB$r3*Ts~E~wL=^{}ZU
z;`)ulj_d8r+X1^ns1C|+=3EXKKIPB1FRV%-SP+jGc9GetH7~LFOHBvQN`5frc;Ti@
z`Jrbu&q~N)+UQUAA97gAbRVxccq$EBq)|tY;jpu_JoEA?Y^z`9R787x@<i>-OI+S0
zOhB!vd#}}yi(v#<$MZXiI1T-1yztAL<Z6M*C4XK2+r5rAEi}t>_*d<6^GZMY7~|(t
z<%|E8$=2j2LYouHWO$8U=E!C@3ouC$kC3zxw6m<RW;l13X-g$`|C0D5+tzU{DMV`V
z>W`i=PyVbu^LM<*L9nxz*0OkIFHMV?&TJ6}xpS8hGP+ux75b39w9y|#Z+`p)g*h@X
z<n^Co87FP>yUNl?-%W1iwPB?c5csFHK!Z%?of2faUzz=@40#3zNrQWT=3R4P26ro{
zRzU&BYNLVQ6|{r&Ab{R#CXRLf4@m8w&sdCAs6_<qd>Mi18aO{$=C0?c7fB-BeYbWa
zi$5wrM|%o1&MAPzZ|*gwIkPVnB<xNc$k}1!#N?{NygkBXlb-KLO6xVxq-)$&e2w+m
z^2s*tc;R8=-MAn7X`{9?lU5tMsaY@Gd(rqb9-GK3><)HTUOB3PjxS=uz>%~1jezsk
z$FZLVDnA{nl%T0BB5BT@|H|PlGNbFM%Gv78O(FT2Y}ez7{$9EArbyoV-2=0}Tgi%;
zG}mDsB{UZ{tUZPfQKVn3&h~^lg=&JGV;0^$_|5^#*!`SBX~0gp?+W~1oEyvd!iq@d
zYko-If^i6l!Z7eY3h`rS-P@^~UYUuS*1vyM$sS%!r2vhR$%<*gHPhKJ^;A0-URm$n
zCD1+Lq@1rWnmf}0!w{fvXX$=$6*Kf9u(DTPp<HU4KY(#D63`zKC>!&**kMH-hIsE8
zI8XjUdiIcc8*IW=A00epy%WJ;87Hn|A|}i?KB)e9pv{7TDfPiX`QJ?TCO0{L{rv;=
zZ0~o61+Uu(%x~{<nPoU+-6CJy`XlEuv(}k^_kcGFv<55950kPtA6ongKrG1&`(>U+
zwwNjngrH2ayxKt`4!kd+CGeW8hscQb5*GKwOn4t!mgbM1C3~cE5M+DK?bE9g?|g_a
z{Y(XY?YG;hV;g%eIba;#3r%ARFtO*nqVJm+n8SG!Ms=?{OG(F$!kDiC{>nI5Vtl4q
zI!xGfUMz5^7g5$n^v+T``{@o*yD6@37D+}U>%a1Qdl%oOK2&~a{FLVQ+H3!hDhADs
zy@vbD1p+NK?;pok;&yDE502gROvR19TFBauk<H0HnEjg}iwN(dXFr~*-ZD(CE`YAm
zp}XHe>IwR6m5SN*U1YS%49JJI+2OT&snE50|Et%g?cXLkcmdF+x&`!|&VVV0a+@mT
z?jJpjxvRnkLhQf+DR{=TmIjkN1?Zdj3*V=0>F4m}=Svi=zk?9%mLn_J0ypVB1Dxlj
z1<hglVv~g0R#lecnb|jNm8m_-q79oU%CEtTYFIelmQNoNHs=bW?YMDT3u%V7GoP2*
z(&iugBKz?C|FI8plO3D4KZAa@K8m66oa^U$cmW9(Be7U@AfPn1ARQ6QA^B21)KU1>
z*DmpGDM)%+k9{`$vVuv4OKmONqJ#y*l)`2eOKousrro&S6!%+o>6WMm=RJn6QD)Zl
z$xl=?5=ab%;cU&IS>%_)wtA~*bk4I;N&@jRY1nOpnB?f!8~2`<;%7139y$LjSS6ML
z&glach54K>R+Nv5E4W~}1(4;Qn<7u>q@YY+Gn{HA8OE%9u1|f2Xc%;V5hhttHnP(f
z=ji;r>GInmvY=7+A_Yg2#+qJpCTq$>rWML=7xQ89oTujgp$_B#PkvNcxl7CnzYh3Q
zV)3f?5vnOeSWKxc<jCl~c!C%@IBIBoj=edi&O^Mc?JCo*@NP@dY0k@D?cHX79U`e&
z&7qkS<_)PG7h4z2;DvlK_p#g{os-Q9X$w9f^MR#N1segYTn3NGk%x?bMjui^Y#7MZ
zyv%ZBb@KP>{^1A{Wa#mRov|9!x7EZI(j->m`tw)VMg){|qIK^qeKvPtR2uHH$^EsT
zAV%{s<-)qy$>9>sL#K&%4s02JT{bv^qgkL_FW>9NW!TR6kfv$rR<B!KVB_eAR|A&^
zHa+4Zt2(s8$ukt2wRY!1{0b{&fDmvY2|6}#A-tHtBp{$#=*cMYNyI|)CtLY|MW|qK
z0`b<vw3Wy5m4yi(5<z}r+(l?}8LN`&(vBy|#s%0)`DfL43_M^oPYS$$!FLNfY1oB*
z2{`yE=WCErXVAQI$+8!w>mos5@`n)77wK^Ga2U+M1mkA7A6;JKxOd@Ia^DwG{Hrim
ztUJRq_|lboLLlMScT9xQFIV{<@UDlO;#{Gh{Rg8)d0yr7tktUE%|>B)|K1~VHq{S*
z^?aN(m!DnGqg<X=wV{Of{M!ONj-P)AWUM**yK4Iudc+%ybEQ(#?5EAw7RG~`_o5SQ
zAZAAK$JiJ~@EjCJdUr-b!$3T1z(7-B(3=@O8J5hoj#r!+N`gv)bAh>q7PlhCGvW2Q
zZ1InowGyQqO%MK)L1_k1N#8$q?z8*hbnM&su&+dRm*{WvMLDN9TMG2p@}Fmrf2xw9
z-EEtrs<y&<?4(f=@sG1x*WLaPdv6_8W%RX+3erd`AdPfOcPTAl&`3*nZn_bqLFo<w
z0qK-(0qO1rrMowtwZY%H-*@g9_ng1)IAiR;_IP8xYtA*-oX>pbn%gx!QxMU&-5;$r
zmM9ZExjjOPZXAo}Qbp4pFSqdGEm9#jPxfFa#_^SB)+Z}F_!O^Z$3za9enkbul(4rY
zatdGg^3u0!_8jUmP*d@=&Q#ztyl#rG_u|NyL`ET(Pd~u63<@>MrP*OUEJ8{f*xEHh
zDB?OZ>>LH9IE5+-`*U@7C(?l<TV9=RWorL^MCOhCyEKWh7u0y>GuD6>B$S>+rR6*S
z*@myrti~5WfrlKZt$Crj2WTzIb%0Q<x}McYUjvx;lq?WX2rTC#SqIbsw}=|OtW+^o
zK%6PwS#0R_krYf>Uff%c7i5?&KQZzTeOoHfbW|i6MQDJR68kXOI}Sus0-E~NbFCCt
zm993^RaesMPgE}g%VYi&TiVMX)3>i+$%F?%;tbe?w`(Jt;B3lUORAK!v^D$v3xVH7
zEm`nI?!nFRPgI(ls&v7(#|8CD%@UqlZK->iaY9CD3QJ6xTMZ85SoVK@xyE9xBv;g)
z?5{T)`O->JLLCm-eCa=)T#R<oc*cd96aN*~$V_=${Cc9S#AZjcLKctSow20J9bJf7
zgKNL3UGTi`P}l3TV3IGAb1HsB+<UAp*ZpNplV!|n*%<0*4@ojL9p|M&ceob~7W_^W
z<$P}qP$B<dq45OcH)Oukdp$JWT#>!mNzyb8>q~G`&J(shY8P*$wNEjE0}iBaTu-Ie
zJ*RD7BPY(96s<(*96ypzrfkGL!&bvSL`;0~vgKC>X;M-`GZTjM@s0}(*a#ePO97IU
zD%`C~jFUmCClS~UbJ{4p?l-%`<I1tbZ@b7n6D5<3%ilUppjBqYh$N@i=2IUu`0dhW
zcyagQBGM@Px(1Z}wE7qxuEAMp=A@liP<EZZXV*a0x#IVTu%6R6vzeph-|M6Lo{Nbq
z|3<Qnt4~SvH%qhSg=})tH&-JT$Z79Lpx|2Iq-t{F%h;9xC_3HWFRf0R41?k}ZBdDx
z4AvGlcz>_AbqS)ejM_$R$7WMY8J=+Ox2er`vAShqn*QZShxELojyGO)V=Tkor1VoT
zpI7tJsd{6y|HGxBk88ODcaP%dt;2Fmw|RX+#N6NUa~18rCRL4vol#9IV(8@NdM<pV
zT(F1fCVIHJEPyeYB6||a!=t*Z7~09@w@e%vA`yLo{~?IdiFAt3HRT`jnHMw^3x<aW
zm_l|&@{|NQ>Ja7u&f|(`;A^$~MNKGsNIxgpvieZ`t3tHe_{ktg6{9TF%Sy-`sz3cB
zDcE7!wtzV|%>d236SDoYaF?k5eJI1%d0kWb_(7d0h{ztJB%|;ypZDRI#%2DEmO1$g
z)yz$8!&LEuKLGJcS{eF#o!A?y)T~ahw^TP_((f3wl$So47iiE75V;ucbL}gJ?!2eV
z19oMtQv2}>v7F81YV5NMfTZ{e(`32W`h4T)7qN==097^lOh~}D=J8oxyA&m6Q052c
zAHHl;{m$+d+o1%Sf`i0wfc2-1^ejZxRVpbgI6Sc2t`*JKR_$heGiSI|y_Itz`5?dy
zSP!ewz`)-xh;}I9`(|Svpiag40LIg-GbZr{FWd~A7u6~vY(*>HuV{EOqF@E=)Tktj
z1toR9R$-VK#{bm9PmRKdZ9g|^Vs+JbX49m0xPc8(R0>WGVyHe4BW_xK)h@~G_~tpx
z&&p5;9RJKZ*ZSGwWWnf4zk9Pp)SS#V+yF7Nnlq@*1$!&ek8+N3kDQPq0=1(pm5Ep8
zmSe_olA(B1J2g=7y!!X4Z_B4Z+7MMQ&7`oM0&>++S3Ih&)NJ^_i!E)q-XKnQ9he_6
z*y{frjvm)HKjTNF-9)mPHeObO4oMi9gUmk?ClR9#E^oHJ(qLgelKIvaaWjX)3dia1
zY-@piF>hLmJP-we;UuhgI#Sp@rD=m;itLiz?f@o&dP(+fCI&uduSZzwHtC31kP2!a
zW|xSn7=*aftm5sRc{Qtaaqs4q6MR!w)S~1Lzy4}}v8g+TR*IO;M6(~czs}dZwrW;>
z^Lg65+0kiL60;PZVP!;RG&fCv%{Ds9c5|l{3FnbPf;$J*oc`7CG^gbC>%>=;&bw-d
zvvd0vs<p&dZp17)3vA{EcJ&%|OldT^-K4qf-n<TFBr`v>lAql~y1(<p-@|4r{quW|
zZ1Pyl6czZdFSV%t?koDwdmA_S0aNh<bqa82_sBkMr-Bt=o7-vybERYw?l<1H_jtnH
zTMW8C3H(J=PHod;3e;FdmaU;XB}R%R>~1_I>FRLRrzfouLh4WO;ZxYY#Z+^!7-aup
zpj^gGvpCRu**2yt59h3893WV<5Tf!vn&U$sGgiFI3l4Lg55dbdhMib`?nzJfq<H^4
z1EBWg%Ix~*;?PyqAO+@S%);J*9iF#UPtb$(mczmzMzLRJ;7QyktZ;&}o80D9KDoXu
zsJ6`5>-^g$4YQ0*GncBk7|x4)>;$l=?>EBE^RsQMWjgR#OPoMrV(_;(cW(1>q+Qn9
z!jBK(^1Ikf`E_hz8EODU-dl>(<|r4Jj>TZAHLe%T^ZL|nG9&+VZSUriq#K}HRH4%H
zD({SvEzk{IUTIWa3oHs!F7N#(KhZbA`H|kYA9{O>22sCsnj+pEQywm_VtO^dM~<Re
zF+LhL(EedGOC>i@uxRT#s9Iq6^Grz7eniNrK}RFr%i4D33EwKspBPrr*kgIiOA&IV
zqq9Fu52!BzwE_(yfF0_U^tAH(#ABiz)oq-KAmB1@!M;x`m3^B)1itcNTXzyor_xde
z&$VfLgAH|eW8L~Xei{4B>fr8dB9yqMkNn#E{AUWcl4AR`0ohYmAA}Xkdv6&_o{6+e
zj0@oP1#<meXHhTMz8vQ3c(ly}>@60P#-b3Bnqu!u(OE-x)gh@ZevaGsq-P*+6Tw#4
zeHaiR*fI9fQaHz#*x0=&EMygYedtG1gK-mdYJmQ6SkAtPG;8%mfdmtu>nHv&7bvKQ
zByr|tYT?Te^<Ve%w?4#9^TC#^_jt1TZ8HUV_R5dcmE0Q;2-zo~m3_Abvd1c3Bs=xa
zAG=q`AMd@l)o||qm0?xY_1{8^pt8lSFpcH|umNI=OfWNe!6!H0sOCb53P==17NAZy
zPoaFA|58`oPa;KBlr^cB##Zcbu%dkM?xR~eAJzGe6*K^QvB98kgBiy3@Ojlsq0B~9
z!uNTxnoiaK;zd@4KN@AxAmt<nLakc8<v?oL&$TI$Hr=lJq47zR@$psOO|;sz{=rD>
zOiovotnrE$8Sy)s&}`f~+XZ_4*!M+op<`f!XSm!#mzpN&ZX)D*qpGBlA5ia0ydiVN
z{FHRFl)YGkclvsM0zr6hf$S>tOTOm9`b=se&(98g&#|(ucEML=@o7#e5FOKO=v;J6
z$w79618=c*eA&L^li(6mR0vSi=qGp^auL@@GJ{V(lb%1Hu}#CBb6EedyZe?b-twY7
zP*g94aoexcUJ%&*LT*2#`f)dEf$etQ-Qnr|=oxdtFlcmd+9{?8u>7qiy@FRt6p!2H
z_h-+0_XWM|l#w`VutS2|QXiu7%HKz#oT!3x@d{?VuVMK_b!bI2?cASGev(=D*QHqu
z60Foz%X{5{Ol!x*RAZO0yyb53{;Ms=vQXugLaRT&pY|pX{QT~+Ax$;Dl~Y@*SVPF5
zc23j1Ro)D(RMXaK(sy#r@-TDqhKg#}wu=b;OfpaN({Nb)s<_(UIk#ZA9#K7F@zvO=
zYRV=2q)$l19VCueQhe2&_9OOa|8SUmN#?cg9R_(T?eoR&D^q_6pbc_bh&7TnRa(dG
zxkoRulgzKrzRz+rFVY=Bi)hS6XNBu>&THO{SKp=|7nGZE<a`E>i9qhYfyyGg!jIm-
zcixO|z8`0mu`YNFr1`EBPef^IPGd{6rSh|J3Sd29rttj&Gv9$z_FH04DljLtMcl<l
z%tjMZ^Ppri)zVU9Hj$%5?X?$v>M5(+Z$?Q&Lo!%ia_Mu);u@Bz>Z(l4hrXUq_#2o4
zb6D15k18Gz?Rdc(o<(7!1j0~5%0G+?%2awmW$C1{@B%<Y0;pcJw7##W;P!S~_swpK
zb8R;jKVIM7i{?1q{@J?d;kG`_nBEqXM8bQljaV&S>mHEd>u{?u^Xnh?E+IsN<d^UD
z2kydBr)Tc=7VuLTbs$5WB$dqUYENTqG!RRwL4vB|r5>wFkPZS>nuN)}`(K+8$XsQD
zX@(q(cKoof0tebsy_ij!+!5fP2GHlg9OvE52wN0iH<8Uj(10i^@S!2WhR?aqgb<`h
zcaybX%K9M0ACi(*Y+6`=nLqs=mZKr{+tkFga@)sISZfa{OxQw?#z2MF0x5?vo?8%O
zo%sJum&4;zA{WSQApM*t!@9c>_WE?<cnQy7dTE9Q@fqXWjg)C>?Kp7qxJ=F*)_h@O
zDt}ytE%ITN$=9xkSlj&{`_^?{rv<nVh#y(9eJyCt-wXN+au1aLm4pyFEV4J<6d@a4
z7b5&WHhpmIC>3>v@E?fJKNlt%%}E{2uMtGWM;Uh2b7Sie&ruykVi*{>tRp3Y#a=^W
z9XHx$rO77UO#adokC6lyE0IgQUFg(c-h;MQN24jNm>@-x`=WpqRv_upf&dz1PomZ)
z?rxt=)tLEYODd5xsCiDnET55=g0-gd&V(Q`r6AWAWW*@h5K@c`MdJ5GbeaF^+-fzy
zceQ)sQJkY}0j*xrkwP_~6q@kTJWIt5I}u8_4H?&CYE~OG)GT0~6Q}mbR<h6-@@~NG
zKCvLD*AydqWVy-S`A&5Irf%=JWG&gQf<z~f4_T=k#JKdCBm8U)bF@`ERMO7mugN_5
zkoVilOH`t_c8M)c6DUM~X)_KsY4oqpY^xQ^&1P{w&cpwV{Ij8H6(P;pt=CbF=iB(V
zZ+(w7pd=#0bnj-c-iE6E?xsvr8TZRJ{uM1L#k&`i0Wx$HLyBV!NbFS_N(YsNN4Lap
zFQIN%l&7eNj|baZzt>kH@>>@gVylK)_!*}bvzL3cS!!_GHNn!k4Z<BNwcA=%-*EA?
ztuOwjidf{ntL22p^D_h@z*eTp&>GRhLLq0v#Dyc;5r^-Y4G08o>}{!!0VR&OkF%JB
z<GUe7)F9xc3aJ<QTzwBvUwm3_5)D82C8{!s>H*2aGn=Vml2>SIE^?hIoQx!g?cT#j
zP}d7A?qsy)Dl!%=2@dmfIr>ykg5q^;7I~8gmFmytXBB$AW;b179je5>R}6(n0@qJr
zfK>3Cv9+*fb-r}b!9_1G(_E(fO_!k4LB735kF%1Z)y9-CV-60<%Y>}RdtVDdgzoSA
zn^)H><Zs?dvj4pN{CudbZ|f+X;ERbmW`802N)j6?nJ3{I#Z7CnLi&}t<M#?%UblS?
z1&R7>1y|wXhII;4{t__@$HrlHmyy)bTce)Tm)clttrxgLzi+4hCTpI+zcWwG3()}p
zRfvE55IK^|8D#|Ak?*?~51D3y1=2)arR>d~<u`k-&f%j(VPoRtvDI%3Coo#=j2UkV
zUGYeCuU>n7u1p+xHQ?c@eTApv5nO?6EI$Fm>8drE?hwVS3X#WqSrv;^RpUVMsbYn<
zNj-pn;x{<f3)wmPl{~Zm^Y}=v@MAgfeIp8<74xL<=bA@6N+=6N5HA^RW8dP<qsp#^
z!?C!z7G7)X5lzJ^T+~#lx>j}B<c?n*5Oi=0k0!4yePt^*jAx7?{Ps$Ay3C+)vXcS3
z!XuYw{t0JU%DcUwz0c>}=eRdgr*XefAi!med#e4ho-gCKkzTHr5zE@}JEwwtTlFhD
zubK4w1qe+Kl`zY!w~eQd)yt&Tvx5ahh1{R+Hpa@+<z(l-hO-9>?xyKIG7a@Wqh!_w
z1cE+3*!3B*t=iIdTE;we&vy}DcddDwyzD=<E>lO=poZ40uFCW4uhb_MfGOo82m|^6
z7A6JC`9l+A6zky6X&hGO{Iy2$_6a@}h$~5}WP4_-H|rb~vocP6gYD;ihU*~=c~Omv
zR(8R5ww2E{Rn9J&UUKY&J2TGVwh67XX-im}>3FL%<5VP2Ea*pz3u898Pfwkz)xH$E
zfJJ~ce(ik^CCSw=pJFq6FE%OhkG2eFcOS{>q;iFAwHZb$+=p`I*K_4ndA26g#rmDh
z62OtR%G@d}xbv_AszrXlHXNuYd_V85h<?Zqd04nyJUX@H0=(*Af^Zh4-_9+cpQOh?
zq`Y$Q*NDM-(oC^p?wH&-&ScCP-r37u3v%tNw3UoqS75P>N$YAFRXKwmd`?OnbL-L*
zm$#w*g_b;4kiMf+FZ;<v8$S|~n0dMxjmME86$WS_3TQ%FFOnkm_S9>!o-O4|rN&|#
zQ_qPfZ0ib3!O%HaB&8tb9Q_3xuwY+yu@uZZ%d{^V;8U<z5-wnUlHS}#`hty8{z=;_
zeSHb$@_f1g{+Y8=%}z@1M7q=Vn=ssaoaD`sZ9skE<GnV8ly%8xAOx%)pON*SHy6g8
zx%!PHC<ny9Uj#Blg6XCr7C~jAN!O895E?PTme`>Ab4Ni65YN(NWHx3KImPID&oK-v
zIE5l9#n{lXMP<dNPXv?~qoL#DTI!Zp&lC?Yle(74?o@-sFGMO7GFjg%XkO#>xB6|a
zu_W|MQQA?U2RprUYujEQty_ojVx?*9uirwyYzua(zixk!tM`R*2C6ci;o~|g5)%|f
z0gU7IJ0r@{&Uk<y<~oruUnqn{Nvi*3&n$g)7~8jcb`Oc0G;g-xvdr9tJkQm?6AGFS
zPR8ZqF^R*g-mhKayS4j5i+moITn-T4yT}!;95kR_oX0furam^A_kRVfr>(0d)cObM
z67yWR^@t>k%rf(wC%0vJQ3~InWUXMORZUp_qBwWiWC25ru#I)(#@Y87d4)vJ+ubB{
zeKFzoP0f^A@@Dm?`v%?%2J>F*rdhN%EXt#9vjZf#izA1hz7@`=*<_prS!?GDA(7Ih
z^1v|BU)NH`PJh+sDFOCwUUt1D(|YuE?6H2{tUV{%Ka^(KA2)(lx7kgBHI+XPzqF+w
zUmgT|X$Zd_&7?tO=cu@*@Z!B|U-5EG0U6d3fK+7ydSnO)drd@Rd=z~^uIrhANcheJ
z-N1UMbZ?sDYp*OZ&jyQ6h6RP3b->{>FYLdiP7s-~?-lE2@D#9a-IOy&mpySNndEIi
zg-{VYDzl%!;#$Ns)=b@%iw4i@xTCOkIQi4gF!=1X65F7~cJ)UO$-Ld<l9&^P-WEiC
zDSjiKR`gmCP&~e+ub4c(4@*O{kOLHG;BZUvVH#MaN%7*J3L+Sk9xDKUBIxW-SW5jY
z2-itwip=uC3Pj=7x0n~j7StQYwz}FeLI0wMygWcpIY6o!)cK5HKCVkKT(U`VeXD12
zlNBV`)8cD_98YS;l5HoUfgkN}JDx^bgkg4g6ez%Xb-kN=sRL*Z(A(hIn#;a%z~EzH
zluGv^0Y??f?`#Jj=aFybLGNdL`fr^cI>_5F%fZY=RqlU`g)iZb$0#ev2WNVUP_6I5
z?0+El_ER@t|6dGF$<ZJfAoG==2^w_A#{gDaWbU5@J9U+6z?h0ShU$9iD&DxM!@??$
zj8EEoJ;}Db#b&$;T(gYl_%7<^UKhpJUs<oug|2*ZdJ-1a6PgTFYe!4E;Yk30^4)?b
zHeGc2M%}BcqQqj8Ws@)1caeT*Opt#+HJDPeyG5-j{(n59OFxTsjbu_)3&gdXt|U4O
z9c;8~jxZ3I8k@!%ni`6rqocLPNf&4Hs&(79^<TS>1_1>?$);^SCK@CTFQ#J{q*ebk
zd&R91dGo?Q*-Pmg-3~C63bo?fT(5?r+<}YJ`}}&L0Ihu%aF-WV$ZoZ+B15(Egl7k8
znrbui^4d{qU>;C9q%o}~xW)kV6p&O}wMQOI>XTq&6*>_mSYy){8gB6_Jfe*0=N5}2
z=C@%-M+n3F%pxzjCL$3@RWAXxU5F-^>XFD<#wrPq;~Na78n1>~C;g6p(WLS%%zeu?
z3ax#=y^V>V)Z&MCi6+VjymtA6a$>cTS4YYWfEG$k=6dc?BPn8rzx{?#CRND#*Fzlf
z`_1FokGNRa{8HV`FW-Mt1hPq97?JB4Q4Ihpqz#+P`n$kK5IU4?4~y+Qcs|@&0y0yw
zpK(&xKOKF2!dSE;u#sA9zj3~5U&Eta1)$5Aw|&`kDf@4ht?E3ttLM%;s76*QZq4PG
za?@GK*j2t0g*HgF&7U%<8C@jru7k}nMnId$jHftH=zVz<?igUDWwVf#sZR7?$Pq>U
zem1=~@QvWl;U%=F#K?M|ee1_Sl1s4LGi?D+MGo@?582N&PVXN^+plSnt__ez&!i@Q
zdt!C^*_0}vYZ0>Zj_DWX)N`#E8l~d&4U@<J)XNXqUjg>{wD;=u-xjR<`$ga|kc67f
zl>>-A9NNW0j0Yh4U!BF$56d35tZjrR%MA%M91{ehpmXIYrmF&mE+ar6lwV8)p-aE2
zcDHw>NSQP#^5ErkeuIu;TSxvwNs)vvi{-*rg=6y<ZTvtA@8dn&>^>8=e6t?}N~tR6
z3h6Z2Iejs*tm*u8YJVHEB4PpYaDiw_5kTaSy(xx2TD!t^nV;-s)Wud8`@9P!ITzB0
zw9rJfM(m~myl7<5*95PRgKG6Fai5I;Z@1ay=8A5Fhg4cn#OJwEmV2mVRs_?M@GkES
zu*B~Zv@k#_dT4^~wLj2VNmsl(C+S@wY~yGie_G+SOa;OokD~9dW2})Y6XtZ2?KPEE
zWFtk-<+eY*y=-*bno?`0?d$z?oMXD4_qFF|F{?^YFi&1L{dA*q#YS{)T@F-q-SbBL
z+wq$)q1Tl@UO(6IT${<bwI}4Yb2<c40ycI2Y_ZAJ{b1|yAtLW!nxotY8C)arzlY`C
ztC%|iZ$XZBd~7~%TSaA3P?6BG<<(xr1oVqce!5~JZS2_ZgJPj$DHUkcl@Q2nd+<A!
zYeucic$ckEDC%u78=U&8MawDGzi&_dO>qyGkq#nh$--0XiQjvO{{KA0VT`i`RL`82
z3L0ho>}1DiC~-B@LoQ0#qyTnSrIP7~IoMYEA<CGg9Ttf2(a%;qa!E@C<Yj9eK-1YN
zZ7X0y9W_$=>nvYvmgWp#FnaN_erTF^dNcJV*I<oS(m8p<#**izP21Rfg|w2~N6SQ-
z`VH>iLf<E$I@l!asrXh3fQ_~`Bq3;XIbX8b^uMt)j-WPTW>8QJ(2pJ>@_JAMJYlgr
zmqY6@Ho(ykQ-9HdBmhF}eQ5H!-UpU0p@3V2A;_N1?k@Hq&cZSgj048rc}JzXhX5Al
zdYaK4OCSS4Vt9Da5|*AixHol(k)rz&FaXvI#$vlx5hLUn)x&!D0Ur?L^_XCHT8&d$
z%8LQ>fDjN#0O&3fly}-zPw(-1>H~PM1&lbQmAzqCmC;Jp?Wc8zz@x(foT{nvvul<=
zaKwNAHmQz5kfCt7rxN&er$e5>hTIGzZ9o|*9wnNU2ElQJ0(86d3GB}5Eo>fI3a~f9
zw{?et-7G8W{$n4D&|7i<4d;g4`2Edst>%zkq+u%y3>QCug_B;oGd)7XFey5oaKe-U
zTj<`8Y+W4`gyoNXbYZ@S*AwiIzqqE_A0$cwIC?4scBhsT3Naowm?603)lTpcZ)EQ6
zC>r@=o<soZ62Weq$aj6uVp`f=wuBD@vM*k=fVIT7`yuT&1z1Z0SBc+&%>jJbJ8s$1
z;b!U`QHu?xkj-c)FO7+I^nHFk1Akgd!tVSo5D<<U3}(pZ;LrdtVd$yvH8~IqEm<B2
z00VZT#h6er$cwG2+lImz+8&fJb%=N7P;!K+11a>EDoh>zMXCoE$BK6nAb{CDUBm!N
zPK>kXV#YY&&+2EeJ5@>Bx)3D542fO101lSe<9jSBv<*2>fffh=33em5Ihh(d(T*K+
ztt3o2ny^7GbN`iKBAoyV2reSLjG2H`{QfaeTy9+H?j&$WBPxLhc||05LW&9$@TV3I
zxYMQ(Q{NE&8JHmokA?)G2>*|J*hULfO5Go}&k}J*79wJt@!`+iDTgi!a0Ac3rvAbs
z1<EHOEm{!^M$-C^NFsFGfOjVWN#N(~s=?|1l=l~Y6r}+enLgiPch;x`m9oRZfM;wG
z9Wa;Jf%nRnbs(<)84SKi9OfW}Cv}heA-R*FTi+=M8w|L=9#P!*gb9={D?5E!4oGF`
z-U&n}^IrG_lK?ni*m&ry(C7Jw+*UHApUfav)1nHybGG2+RDutfArydffVm8Q@h^zE
z$SCargP)*)MdMQatLEbV!V!oycgpd9mnYzU{huw*mtaW<!UM18jnxl_&4}HZE+k1h
zz)c6pxFt6bJR-`}ayN0HAi9uD5j;?B37GPcziH9X08-4S0%WPJA9H6Ixwa<Pti}&v
zN+9P0iXCP+i_~)Pfc5$#{PPM%=!4k_SD*mDDpq4Br~wl`3EHL!6hs0JyzI?TkEKBt
zc1CL&{gej%-NitI{T}Q-c;)zz2}c6{@tz;b=bAem_C|SKdmMy+9OJVO$aU8PA^hN_
zcT%?oIz9DWdO9rw6(S)EuS(q_J{!{5CIC}IkPNil!D7)}NWHrj3{xINLb2{aPMj}&
z+ESmIH_#fO*bYS`6dOpwo(EbC^PM4M(AeA=a-e58(qMUaI6Y-bc>o5C`e5bHPudo7
z!Sd_KD!^f+fa3($Cueq9z(xXoEo894WRp*a@PWavsN42^MBp=>Eqyml4O`YSJ`cKY
z5gr_J{rlFb1n{lI5bG~I;cePqoINVJlKq`4Pq=1*xfO%A1JC}Cc?1(FK@8h-CRWdE
z*a8Kw&+U|JqWqneA)>@eU`c)j*GijpS4@NZV80BoTwA(ZtP$wvror5h5x|pFYTMd^
zlp5Fq{J<#ke;f0FjpT0Fk3f+1yraz@M;@?{5FliQY0u#>XHYo}Fkyo(C5DTDI&Z(J
zzPlOR;XwyL0f_3ar%%DtC8b2m;%HHAhj>sU;_s#$0dHxFyfwAE;Gg`(d|*2sN>sd7
z1fW><IiWH=mItHpm;HPWldSJgik31_j;l{qz}|z`QTV=w8Dq491``9^4BPmG6AkB|
z;Q5&R2Kaa17Dv@mwrH>2h6Z~ZOvY0%GeK;Q|Ckocc^g7fCp9J*{-^fTufUo|i5cK7
zO{IgU@cf%}YF03hpZe0jBBuk^@)GAdDT{=Ue|4aMzsenl@;MF|KkX3n$A7cKECW;q
zQ7}*iEf`!6+g=LsmNLL$Fa}WA^if>ER3@EaU*>M&#o(h~0F}u(dA$9H38+j{j6qW{
zP)WgO_2IF=kWu*hpa7M;w`zBv8w`WYv+5&bc`3m9cz}jA9rOU%>iUF6>&4xM0fxWN
z#ISManScqphEZUeMZ(YU-vBNxUCYMqW`@-<rtvXsFfBmAydKgE{YQP@z{c{$(eW|&
z=Qi!fPR?9BvHs3nX9BY@GrkMcnQOe3vwJ~FaKiL!?)uD9Fc7rgVbVJF24cFTznkL|
z*ra%R4&eW1QQsj7e(BDXV9cz-J<f!<U`~PqkzuQ-Wdb~T_J{j4-<?2#Lm>yIP)$ha
zsV_`p^zzDjo4^$A3Nmfn$-Ew%Qfw2S_CfGo9c5a)fyKBn>0&BU0E@KhI;aXg>H+fZ
z1Gpo5*aL&bKsZcQI;w8;@A04~&<q?52Uq_H(=bOwCeZqsr1GVB?D^gEEr3Sa?B43{
z{(C;f7od>=F|7NkgkU^0NKSE2dBFAUACnMaI^|;oBvd1>R{HGUVgeN$Aom=Rg7`53
zXaM%Zja|HWFrJR}!I3+o!49rhdbGv-`JcuV0k6r2`^vZKuH2{9ut!h>jNb~~8LjY6
zFqE`F<2!s@-T#fh^f?$m{vL^H|6`yrxuu6$GL%4L-nxaH!p0AFO)&XFarfIa%n}fR
z>!T+Oi)>!N#xD`PNtFb~6TBy)ng$z!Jh<KljhOAuaDsPefd!=(YW;yK1sG8WzgC57
z93ZS_5hGY+E<-^HrW5a3$ML_f)zbxE3tG+$&v1bKVCMMEh&ATSn~`BlTG#^O1JB+^
z@K65F)EI%Ol+Xy07I$EnlN6_ucEIB|m3G*#*aa61%lR03SDp#YD*Rg-&nh2YJe~>%
zLaINdpIv<cmK{3g-kpLe105iLGXF~owq{{MpakFeQQhUaAL{Gc(4_&xaXgse)tCBp
zS8wVJGXmDay=Sm%?|)(^z+4bCdXhh3*^I3DpZ4uN5DuiZl@vDDC{KN#BENLDSOA_A
za2#cqz6U{V@B;<^CLW^wlLc%^-u;5zfHldw#i4ir{tyL&IL0HaxYvFJDIX+vy&D&>
zQHq@akET?boTi%ZEzF<$l4c+bbfV~v6BsF2ctW;)H(neWa0`MnQ6X4(QTYhJ2zNed
z>f<ZI>wudM;DhpDf@tvpt~rHi8!57tKiM)RY#awKfNQn8?)=F<#LzoS&m<GfDvmtx
zcfAo-O~(DfW+oQpd*lrMt){#4cTKLl{V;&ilrr80>oa%3dfOG4BK1_5YhK7{3~ht$
zX&34hur0~`KE|(*UaWWmd$tUCpH-Fd$bV%u|1Z=7;JATp?vBa4Gkk7|2qpmm7Z$jD
zPyDZhahT&MU^v0%0b6U4+^>K#mLlIr*X}36T3}YNePMBcss$~EbKxn<Hq0e)f`v)n
z2NF)v$$|Kn4f|uj#O&O|Fau<sgR0;|B*3l<u<N8d?E3$H+W(KFmh+RscCQ0xy4=H8
z--xMkUK11vSi3|ZX9wNKtS3U`mJGl&blz1cOdw>!z(jpdpdv+zuo&>aATb(mKY&xb
zy#gPZy`3&C|Lk?E2J-n9{~ymngQ#v}PtXmc0%#`-;=9fS3-FGjG+Ik>fK}lcJH&T{
z_IlgfZbsVQuCX)dl;!61M=>lksPZ3P?|nboEL)^Le;C>887XWQYmWGp%sch(J1=#v
ze!G%(_v)SvqvJ7=KtbfV{DphSc1!wWIIIXp9)zX49YrN?rn#oV^AFUy@JdIL&j3pw
z)$93Oou}5GXXW$7BdxVoP)lut^QGt`A}|;-H;OHMi`mAgD$$gMm~sQxWO6rhG_U>R
zl-8qPnJD!Trv}NcOIhLlBqgaNbNtU)0MmI8aGo^PLSZhE_%dtxHHUf?FRZ=-9p7`|
z&b7Oth6Q64!BHl8tz`BcsQ92<3-uYA1yW&ySvOW4!~EiBDtE89yZb_(;X<lEJk&Kx
zpBuuf5?#l$YBPe~#GkEZxqQzDNEuX%5-QC`rF1+`RaSeE|NI7>W0dQ_InyfZpnt70
ztXnC!`54Mu)UFzhCXc#DqyoyI4(eGAw>a^x73haHR-n>%;YUHL4UVU0gn))a1@9{f
z*1;j%f-*FOi^hPC4Z87Si`Ug~Q8vp&KN|r&d(fmc;t-KGVg&O!9Lvvm@KGE?6pImU
z+EKnUOU`#SF+9I7OfA~lqUYE4LpO7NqmSndajIl}zkD;fAWW><Qw9}vuM~7oyUzI#
zJ1G<)J#ue9rTv8u-2;%^SI&KFhyrHynvVu`%k?^>k9L2R^^Pi*fDkEFV#Qn2XlH-o
zdot(^If*UvzZO!t6EA!PN0Q48aSA<M694k4Jp3(adHVr$8<fg}7eVUMv-(1j1<oq(
z{jJ>64LW>MX#a3zZ%~q#4+Fg*Ck|4Yk-_0QubBBROSCW5B6jABt(~_HN;vAa>&*p?
zJ1l!*j;*BubGe63!(o1Pig%`^a=g91rYE+Ut&gg+ug5>|m2CMMLs&UlZyP#ejt64s
ztKKoQ_ea{$a~uAvK1C<yLH|hEkId7gn{NifE!4(uyV<$d(qC<^cV3ape2d=_IMPe9
zS-1%#xo%ykb%iGF-0FW#%5{g%Jn2bgYd(@`-d;d%#1D+msh?nruka>OWg{+MuTw)4
z*47xw8T@@Jkn$*x)b^&u`f|VImHl8lBDHpI(NX$DUXb$*z4W<N{G;i;n48XYRV&5P
zkzd$vVUfNTD&*oBw8<j$y^zBT>-h!-LgwctA{KiMl!$}LLGfaXZc2T=>0VI;3t`b8
zh5Za239FWd1{Ca%hD0v6+cZFRY9r4B{4dvJY%8M=tQ<wjIHmQ>BhQWccMi!22s{e>
z98=0iR$Zt`?=~(^kUVRa2CkGBxl5rqAI8~tCj8z_PtD#=QKbULW$7nrSk!0`G=G}g
z3Q)@xSrf18HO&+Wn!?M0gYz3f6%@75LV@z(?T8a-NGE<S;lHWT8S<o26#|+|zOAuN
z@mb=oi2PTbc9AtX2O4<9fEGBepyaCuv_t3xEztH)%b#)@%Ydc|;CbuRyAO2f5k&_N
zbs@5oYG`yWtBm2ZvW<!_F^;vN`{YkAZwt%!ySZ66-{tjDlr&6g=$K7Ruk$${K#Uzj
z)s=?hlPcpUMM+j-Oy0)R9zp$j<Sw@B^eV1)b4*ONZoT*+EBQ?p7m3@qK}H9?4?k|g
z3tx=Z==OfG?k8!>K8DIQDobxwlG9@g%#RK(vV2A*C8Ez9z9f;T?}F0GePdh@R&O^$
zeq?&ka8qvfor{=5n>FS3<<fy)=~FkkIH9_+zouUr4!W+dD7|Rm!JhBK5+I1+ciUjY
zG+lOP5nt2y+aF|He!d$g{%Ns$^(wCZ2%$<#k({5N4s&mKfh#ALF0J_SnESEuAs$MV
zZOuFPhz*QwW^a<I)i<aOhy@D`^~x@Vk_|RlLab<qzA`Dan^-aav%EiBcZo^kizfXV
z)A(xD=;!#OsJD<X&N^BLd3QHzXQ2v$Uh+VgLU3CUQU<Ii3`{J&8*fYL=MvqGU)aSy
z^%tHx^+`Z@)1w92+@!#|(`w;24}vC3vZ}==w<F&{!}@YN!?{JzW_0%22L_Z9FKxbk
z=!r3x`&J`#n^ve<RiGzOUZdRLV%Pd*SuYoq?tu;;uNoF`M~!Q2B2TA|lGuGeS3bmj
z@HlJK+adz70;&4lf-V8K??!z8wz90Gu7`>P-_Irbs+2Q9LY@luwy}#y(R^^?15_@C
z_7!cXP7!Nj)nf0#ivMmn!hYijTl1dRw=f%BT^`o7Z%}+W)<y#8+CkJyQly3_#4*cN
z)lja(lF+^?&g+^t6}h&yt7_?m4(9~)jILyEbJ19!x;feF_Y~FyQo{g+dFv{hWWm^x
z=_eEFcM)^;XBYduln6p$TaU2X)2igq8m5;YA+QPN%!?Mi4)twWEuBrlV1@tco(fpK
z{`tWICcSCzLd-prg(T&sD;*=2>SSYSQy=L<Iy#9*8VBaEV0jg>4s^&tDe=P2lOX$u
z^;mL-t1m>Qw?h(Dqd|sGuIbX<HD<i7oL;-@$rlQguP0x=EqZ}r#(fr=v5?Fgnl45k
zSik=6W8aQADG9u7E=k>5w88q9?TcYgK@xAvq?2Uo!KMl7Te**i^3-m#08#b+EWON%
z9nxAKhva^se&s;dWOg2;YPn$QG8*v&v03XNvEjyA+9*L+1oE*h^>3uy&l2FuBiFwL
z{bl0%;@R9At+vL#t%L69I>#6892OHru-2F*@n7W=-xljM7U(p&_y6whA>p@&bs}%j
ziGGhq32U|uI_#V#4W@qveehyHH+h#v>O|V4jQ1W$OzS<Lr9tt~`l+)iU9V*QLs@{z
zXA-1X^xxoL7${?1+RNL}7&s-JeHSW%dQE^+A5z!0m5{$VsPD9S7O><RIDfkeIQs4O
zyJtkM_J`B)$D;=UL#MXG<s(iT=W^I%`+shq4s<dst@(EfVw<C~Rl9kYDR6w~U(Uzv
z=w(^_^?AnXs&4G*vqkCgH04o$_U*0qZ-vyJ8<txF^2ekn&WKiMTbMmw=bjw4(@ib}
zK5~ziF=LtERi$mX^0p7>_V;b{PEC}(U^ZrXiuwbSeh!5JCR7P@$m?jq=J&5zQv_Je
zIE~{V$coWzgH1V)jZ#m}#&yj`cWCJ$Z#MbQt}ewyzSGW8F0^!YruSvjq~kR-mRg|2
z+9KDi_+se3wkal~SAM(j=WpWNv5Yu!*S1>8Pu_Hl*uKp{8UH;)l5RZr_Fv@N9mNh5
z>+fpO19T2bzP6Hi<TlMkgJU!@AIH;aE>w0`@i~XKr3&I$>?E^K`D;6%LDD1`=3F@r
z*HK_bHIOGVaS{G|dQV{huS<=7NWOy~#BKKIa;%_)MXEm@H`YQhU3J#IlF#mClCo-Z
zGSL|cUHo9G>I=QC%0FNCS5jkM|Eje`gS?`{jH`Poq71JZd6}$iDH4hFl;mP}vwvi}
zYGg}3`7reB&*PHXwV5WjvLy2`|M^Ypv4*+6+LTEMC?$0-8gob*y7>OG@})KfPuG`U
z#RcPzL&^+l*_k?$ULO--abQ-UVC`&ig23Cx=xft6w|HK}rqZATa2VvN+=c(|SXKws
z9@qVsN43>knnz#0J+fIQiNdpCq&Qofd~an@^9g+~lvh0=&Cl$|Ax+2QsndC0#^=UT
zEU&2_+vpk`f&%q-3d=Xz#4-aU!qjS&4>C}wL&Uz&d0$JqhUgCnAPwo!5Z(qkTYf>;
z+-dP<(;k}3cd+OS6D>jw80?zu@@CV0>>{)UC$;Pq2K5)U!bV`}hqh$D{XNY%GuwHw
zc$E2F&+$blXAq&oxv;7P;;UWVzjYbiwe{KGnI|$=s0u`}XEXke%=c=w>V71|#yNOw
zhkr-{PIOfYtTY=t6jlm;c>-K&F5ZpzehHYS$8XlN$OI*+_p<{lH-|Z;WS{5tT+T{u
z-l%txF8VsqManT+_42)+rzG08$xb;IuxV#ni|@DK_Ft+*Gk33XCY?#_2&$Pg4ixm*
zjM!fUuW{#IT?q@(k7in=Vx(Z1>&Bx>d|HW@owm@M)4CU~=rWK9BPal-&dhu|AB@CW
zRA%$9x;OCuUES+RyHrBzv)I_*x2Aj^sbotu`=i)WhexTPadCVdQta?G_w-6kb&w=a
z=-Aq#Ss&Z5$!<|;y-uJE-H30~X%REemiNcHMY+=@eCvKVT;JIbHJSxzm4m1X`{DB{
z=am8P_Y%+F2Ta-R<15nfa%EP$Nk@I#Wc{w)o#vx$kcgNrAKh5Z@G8TXsjC5>o+J71
zQUiH)q*B0-@(F&jM`2sZoUB#X(1vjWU9Px!5l(1)kk<BKHEn5crgmTA<@p;=#pY}7
zz3Xh=G*yS*`A9?W*<QA_isF5g4wg;Fn(C`pv#75g`5QbVU-1YJ`&PV>E2x`i^OCxK
zi#@pJ+_^;Qq#e%lnGWU_`f|T;t5NNrC9|0GXu3J2p5lFQ-T|b*lpQNA3QKAr_D#P9
zgZ?o8DikqPHvXxF_R90$o0K~(*BC1~4sP;zDmdM6hlu`le4V!}i>L_VIanfT$$C+5
zVuyE|s~l*5{(`eYWZz7|GeEB-s{eVw@+iOrWk{X$G?3Te!~5jP*!(pQVFQ&VHxo)e
z!qK&qzULL;wM2^pgjBpX?;&+0c6iSf(BUlDo#us?!wGV!>%Gq@-aC)IA1c!<s`(&Q
z`*N=BQz|jB7i8%+l8|I@+;g%pYUjJ6@gPu*{<tRv+w^mRAw~P=r;~c!sUY92-vm1l
zqEQ7Liug(0a1qT=6clTF;Mej6(&6bw09+^Vt?BWF1@g|-E%7d7=2gL48piP$<)ru&
z_bUah94t&GrMQv;uk;`3-6CGcj;>#$>|eBaWtuR)x|Utk<7|DbxU8uA_#yi1CCIlD
zSpwP$;CVdKb)PyY=NX*OGnm(Rn!7OmpdnfN@?nxtv{dJ1wynujs=lCU7iO^fvh`vv
zH%DnJIvVRrALT%bC~D!n*n<`V_H<f`)@=-?SOanRVC4k2*}vPMz0Z}};FScX`GSc|
zJ+~biI3t1qd7o3b)g5`1IUILm=={5y;`3cx)(*tphKz<HY{JJ2n%MhpzpjU0{+a<%
zHtz28Niyr&*FPz5mUrReCb~SJ6RS-pM1-UHBB-RvoVD2=oUREQ7-zvpIWHb4B9H~f
z6XFWRU3t~Dzswj0r_Q`?#Y6A2^2b@BBN=KSIek#Tt$Ff=h{df|`Jv6jtH#SBv@o<X
zG*9$5$f8*%+qUA*1M8p8FnSakhpF0xiA>usWDB!C;hLJhUNwI@QXAuT>^BY?fz~Fr
z2NFR;)Ll^)wwlC308xDBhaI#)^8mh$3BKaC!TOsa|0&Q*I}JwRAv1d<g@f&(tl`EF
z!@twEh&&)eB`AJwrkjzB#ngV|XYWfpS6u^Y`M+MfZwWASM>6!7qUTnw3i_&K)SDG=
z?7iv}R2$G;#$2?JFOh0HM__+`IPV`%Atd&9%x@%17wP+r%F+V)zz0xlZUXgS2t}%A
z`|={WTNg&2@Z64+0p}7e0KoOm(!`cEgK&AI#%3-I$3$RvwkDqP<$JYIM-lR$&Y&T0
z4(LAivBq?^`uihdo?i#=GmziV_VU0gm}8=;q={0|F78g|Fdxne4Ex`Pl^2)Ls)q;e
z&YFXS>5P61@u}R>=-CLf&}{R-*q4X{?bY2cFw}RRWj<zI#w<H0B9Qs&wMeXhex^Ez
zv#){|Y<(`Gsjl#syEjzbdkCwcvjLE7C0Kt%CYC*XrdsPjX4_?bHA>Tx!xjnG5Ks&~
zFWJb9>!i6hW_i9orRp@!&YT)fQ@Q%;wyW0P$NxdRdeT!Hwj_&?=DlbHtO_q1k?gpi
zUb)VOSj$)o7TE1e7<`te`>gNKPF2>**BZ1$bpA?vjB`7YjG{-+Az~nKb}BPbGhg|3
zil%_F%Volb2ROgpMb-zwRlbYuj^)+%EoUom7?>0YeQ}TzApJFMqCYu5fLz`RBr=hQ
zG2Xq$k$>WCdn22$H&bO!Ni*d8GISXI%Y$Hdv84c%Pv0uAYku@!wysb7unqF(-*=aR
z>W@8aS?PAVCLn+NE|h7^hSkQ2eAoVa)9OCC1yrCSU6eJECcV=AopwZQ?vV5kUWzND
z&-fQ7AJ2kF2Fn~?2w#tR%DMi|6h5sFw||dpGa??&n9mDQKY1AH-2o%J8lPBG7K)H(
z2XyLNctH}o(4#N9OH970R8nLG2?Dd!uH7Bbo86j*ojH_#4tcbc?7VnIjk{P`UlVrk
z18JG<bcf`%2VtodCarWjf&QyGphw$Va4<?#n_o9Mf<RGjh<zwNOF{4VH|@;?hqtP@
zOnsm|=|4m!8S#H2GK*T5vECr6Etu<1<}I`z*7>4UUv0AUyxBTMY(u3`l>rWaePP6s
z>^~*)sPF$=n}K61D$rCEy>}mMj_cx3kwq{cem&8o|JHDc{xmIZp}x-J`cnPW9lu(%
zsSkxb@J|)$S6fs_zH|^8AjGHIdidrgmYAKjob~!5Hup<;dCwh`<14Ml^=bZ2FX$JL
zO?7_w=olQi*MU2Z+&=|}!!-Bj&~<5;+i4MlfRZUH@4-7;_COTfmVl-?P_rlMC9EZ4
zcKs38>7$ps3`He>-u{MnZtl>tE+PaZ;bki+`5wz&svgtVEh7V~GaVsXC`chS@R^Dz
zVrYs9d+Igq6x`e+b@HGvxqd@D_(>2@u_nS!BA@&q7f<#6CuQjK@6spx7;GBAiKW`Q
z49p2$Azz2I!nM|Pawee|y8TfgjLsMGkNcaoa?Y<?DxVx9EUono-)t_<fwe6fR0*$I
zmx9>;R{q2l%D^r8!=*E6#QxWd%~^qJDAaJbc4+P60J$BSJKM~L19J;vdwuW9AvBiv
zPd)~naEG2ec{k)i=+Cfu51xgYKZ6k^g}o8`2XUJAWNBp=3pA=ig4swwW5V}7Cv=;Z
z($PYv%AuqWQ*=FOl---mB8zx;_n@-N1F6B>XMm2W<xg#)j|dV$xXlDNRS$1RSF)RR
zg+g&>7DiP`S+31Jn~9fiw6K~}vExb7wt8anhK%Pe&k~$gb;S4r1#c9)_Sa!RQwdx3
zoi9V)l){?sY?(j9b(<GT*6fO10~HRZB7rlI`i0NOMM#4Nky6gOP5^GNL129%RI@L{
zjJ%NiZ~ESi;8I#Q!9JT+FPYFsT8Kpf{9e4<w_+YFFU5xQGqu`LmFo)x2Kjl5>p5sa
z!=b45TXaFa&BZ=~F|_9{p@&yNFO_$m$Nu^Imme!dTSRx0!)lUW_jxtDFu|#~Hc}%r
z>j=9;Sxw&6NpZl5sChRl$!*6&#{}*W*&sgcdE?1E-6)gz0ZTtsVVf4q$-9Wb*F}tl
z-6AdItT%ASbbqW`xi;Zmox8jb`IfM&>++g3sF2?aWm}eD4;vZF4rhM!AXmjGkH=^t
zIpa652<IF()7|8`xssfWgM3st)3c2y)f90^4Y)9r#mr`LG5mCnr19{JW@)=H)*1B4
z-qKh8%j_w=GVBl>Z>+)>t$18zpFy=;+4hc(2!=D{+4ACl7QGDlEY%-{diOs1xjMYE
zt|pM`v55M>QX~daJuBz%2E(4}f8MCzWGB9s;_fIS`}OF>`ybBftfzgbNSRsBXHi1O
z^gA_*gsvBL<Hy1nN!o3)L^OjlF7P*BdJ3ramb7i_i6b-bVE3-=NDP}@nYRc24eJ#J
z;NhP$a~vnM3g72UKl%Sqa@$%i@m?b=$Araf+oW8qE)i>Q@0Y=0q^Svf9A0@`OJF#6
z`HG9{@_wy)au=5{lT0fpR3$g;|5y}Xrxep04jixO>RA>R{q`q#Zywij*}6q*T_udC
z1A*y8{>k|Y*W{&EFDDA!kAW+lBZA2AzCcgUTOoYM-_TAT68jv6`J`{WB<hDq&z_3-
z$1d$6M{+!pr>h#*F56E7H8MZCbLwp&mVXD$1=xFAW{FW$_ZS|xX`d%`t_PoyxS+Re
z>0o(!56wRLu-|2~{>lEy*Y>q1plqM6<7XG5;dRcfH2JGeBo@+5T-?2rC#glx=J?q{
z4i|<7zgkvJF;qk#xlRxHFK0uSmeNwLo3gvgGP*|TF9BQAmbKw>>9ByVW68a8&Ed$g
zzXow3Y{6jb&t$;LD5&ZVaYL|IBHHmCLLaCCmJ0X+#OMFbBLK+=B5DnECXmUEx@de5
z6oPN$fZDdRwC-5ZIvP8QV318E#b<0t=~eGAvf8*xXq?!}qm;_(-wU7j@J6+sY5(Ed
z3M017zA+wr(vE%8fE1#Oezpk2r!~du^eD>#0k-c@1fS@Rn9M9m3N$Re#-BOtZWRWn
zHgQzJTY5}hxs&LJyEUq6f*8MeWyT-NAcRW0{-}fk>Ab6x^Xq(vQAG6;$q77P!BQYy
z3bU;a78GF?9o>UM6*5dvVAibF>=^K%t!e;uy@ruMa1>LhkaxdWv7O9+G|K%q>!t;C
z9u#x3m38#k+`F#xH?kac7CUc8sU}EO#5bX>#m%za+%Cxx7lw&dhzJM9dhPL?W5fY^
z=n$*E%~7GThzsTZv(IsL!R`a^lCjk*ZnpcsOpabjj*fB65jrlVq4Un`<u3Q@;1vD*
zgb=O3vX+VQ9)NQM!HU682<4YCjWXq_A^TiTudmr2gJV@NKE~Q#d(>&O_2wxxEVm`I
z7CoB1S#|w5NkPx3UWY6h&b5r`opabtgj)6h2X81oKvd4g@X~JgcpU48?03~P@o$5-
zcIq?2e|>(Cu2_jFbIJjsUI{PdMv$+#Y_B&QvEdiApsA}rpqc9d(%%!HRVKqY3qT3H
z6<VxxHz82~VW%ue)zcY-djbyU1VxXgx-Xy#F`f9)2jTNoQ!FDoO}uwqkD(=;A0k0n
z35^x@RwvNlpyXjC#{c5u@^O(VxDgWvV^jyHNj9=M+}*p`Hbj1FjxgEaLrG-VH&%5=
zoaG_`C0)FI`BL4gPBdiRf_zoJ&c@Q$BwHZ~2f^3EdeaLTA8YtC`7`u<Wqy&p^ov5x
z#I^f*c#ZJQezp2YDo@{B>o3+qpfDG5qyt?<0S`loKzM)8;(6h={bnDVFoE@(vdUA7
zFcJ3&R+{GhnJh)4foQFDQO`A>qZNxeNY*e-Zp52f&OnwOf_q=lAM&^%C+bFffN+#A
zsPQ_6C`!kCL`TO$w58KfiJ;JrWllU+`n9b~|M>OuTV8BaJk?1@Z_aX$UX(=c3>HOq
z>R_n@A2j2e;@BDGwZSTe9+_~rNHv2@hgP?YL9ivOQ`VS#k0~IO&+Dj^JAaI}jJc3}
zVHhOJp}qqH!S*`4>Pt%aPFxZ<zm8UplU+W?s5gra+p4{;ATnzsHNbj?METZ(EP5-a
z)ti(_Buh0lgpDWDVG%N^8rQkiBh^!6o41Hg76p<ZlR~nhYK0uDeE6PEo6dsMXe`q~
zmakN0pxX~n*H9P(ra{~&IA`iMx~J%O@27!!ml`D5i8eHlEmzMZFxd84<hW=IE|qd?
zvq>UYaM3ld+RG-l_h<t-#UFV^N$g$gNsp7LUg=x4Hcrl*IeePO*e`e1ydb}+{=_!H
z74ZB<wHBnR{AzcD*}@mqq|E-t?JJ{@$WM>sb-ZvXZJgi6YbHn^+s9d?YrtA=a9E<7
z?1ZOOZ#vQ;Sa&s5o?JzC?yDqYC!5rd?))g+eR*<3?#Ur{EHzi+y6-E9i+|p3S;DhQ
z(5+_F9J!6o+QqAa-z+=qz1J)YWep}kkN|{V?6NjenLlKfm#>7Vv`xG=IfWW%T4v#k
zz6j`F8%4HKE4R)2;gI*NC|#MQdDbDeUP5KDiQUE+Gg&f9wOpSqheogo&pDNy@c3B~
zVQ}4kSx&2g%$M7nc1IS3Eoi2|--*x;2fOI;pHpza)mMlCfs-w(AXvZ0T>ZQPSn<8#
zGkgSpoV51Uihgo=?CM8044flL&vz0%rBbMEC;nzE$%(2;{Be?`Vbeq5YOC8;VgF#-
zKC+L;qHs9E3i{)Uibua&mkd<*5+E?n{W)*VTYyaQXsRQ0zf1(gh0LPJME6lW-j^&F
zlfT)>ZA}2ayhhAoGvhs7srr#STx*@w89rC;k1Q6tj|P(ZZYqNvj7_UVV$VDt0uytI
z8Vr2fR3iPn<GT0Ra;DGh*O#NX13|n|!^145!fF=-klDp|SqfxoE!@T0<+Z+Io<;x-
zMNM_-@$fZ-uDfwqa<4GzHkbWGlvD!gKRtbML7Mgw@A@ElVKG46=fWzuxf`1Y9GbSg
z&yui+1fy)=u*|U}b4hGG+ROhjK(CjxkY{_|A7EC>@V4M(y4OYZO@;z{XNHf8NUC9*
zctVR@T9C7KOn}gyBV4Pgk#eKurqZU_$L9%|BiBhMon!|{2%&aMRucJm25?LL0dKCK
zkC)DeVik--{QuY?-?H?r<G7%TE!h)1Iz;~J9&S)DcOg{s%YInxl|nM*XGQ{VzA)Td
z!nQyyrR3&BIzz2-mHNFhRJ+&dapB~nc#zvw+<b1rw$wy3d$;+1k2Ckxr@roj2Hb6)
z3zMS7x)FcAuNe8X2k%F?-jt2zG5POCsUWX@C%UWfQU6BBgirScJ054z?k3tvN&VvC
z`m@XHRLAD{MA*T7iS{pCF)+?vkDTr=fZaA#jjO@zuhaJV$bP>`qyNwVQD>jR!82nN
zA{d8W+`pArQV0-Ia25kC0oh0_0ahXQOtZG$X*>61WFwS(x?USfUy|c0OBpQcHAFYQ
zz?>rLs@$aYTq90_gn4kxy6C;eFx^0*JP)amLa&NV#t$@yD?0P6G_sCcUpl#b{_Vrl
z)ho8#-VVj^vH#KDcLznaE&U225(OkFC?Fz81_4nrN-zK;8Oef@qvY&}NDfMrEXW`k
z$;^->Dp45886?9Hhdl7s;5qkr&pqe9SM}<v`u;f7E^5#0wR@%RUfsQZy~aFuLUngA
zR!y9VO8*wdU;dD{@xAXnbl*tHD*;mTR=$*4E_x@CRqOpoG*aH&zGv1#bqD3bP7};%
zNP)-lo%mCeirE8km7CnJaYF1GO_5FaotW#6O-QZS+Stf6T|O6$kx>OD)dvZmS$pt(
zG$bYaD2C|eqHZFO;8a|9cF{Vw67<k+P+w5wCb>n@_Gxw9TbJZL>#itfe_`i_ol+_E
z6CK$}w}52t5|L+2C~7>S)T9_D>%t$y<EBGC%-aa+jV&aiJdYc^ovWvk7xJp&>>6uZ
z%$&ZH&B0>Rjr@)!3uUtO3c}Cis0rmsTPbbz+mXjtG)7t&zdyp!nJisDT6?z`%3$L!
zp?!GYXXfP!gDr*a2KiOoPuCbY_4Ktv^?qpGzSt$M5hSNWw}Vle7`tqsE$)!0u!M&0
zk8SN)_!CtGSM7FAcEoPd-yvwnEJ<df4ovzfUQ#EaUmd;P^|m{ZE3IFSV}$t2lJ~c@
zWCFnao;#-%(y;svZ$%$Caahl-Vs%{+AxeSrPrxSD+X)!E^<gt%;a&~#03oOq`CEYA
z@w8YWxiJa<>K0x&gX@Re1uuw{bB320S)Rq0a1dOZng(ZLXzSsI7qx3kDO82FB-*w+
z_<uPioyQ70-R${cd}xMJ|4Cd;5Vb{wTEs)^eQrEF0yBc_h2?jCVsS)46H)s@7mJ?u
zkGw3-5qWR*;kgjq1Ee(e#16(+>F-z7-grsocayn!8ZE5E7}F5Fe2Rbd%IpOhvH(Uy
zdmpILTMlcHJ;hBqnYBkrltcUZ8sj(Amu?4<%2b@zBzQtD_2!WhM<}O;pzOF2`bus*
zl@+jqR`#A`gqW+#2igzFzbx;wR%EGh+B8|lSvig!U%BoiO7eX9>Nfp8wO4@YYO*|0
z_P{y!cKpWLrCV+p!5?1*`(7EDlrVvPofIi1%;8mBI7*&=?pk887`GAios5o~2e;;K
z>F!&t48n2QicEv=HN3kFSD$oKH>z&M)KTSdEsfg-WCYMd=kRg&BefOIi9h*7snFfD
z85G6pC5OhmNDv*_K)=$8E%3P|Gfp}TW=_;9dh-(#-Lk|>qnlX3+^!^6V3!rne*>82
zZ}-<uAeo-nOoss1$#Z5*TzPRfRcd<0?D=+dA>;{s$A^3FtM@a?br9{Vz&1$@G2e?n
z?>jN$i#M64(x4Ils9%xQ(e>d??xbn;iUuL_iRP009F0Keh4*Z(%%2<1j@hFY8b4QC
zbf0zYzRi|9a2jXQjr+yNH*=T7NUnGq>L1p|8u^A9%_RA1UxFitvSDUthxiu#BQ3_N
zHmf^qj34}Xb&gb5qoL{fKtbtBfySrHAM)@W?GEtG`7LUqKI#rDw~sMuwxz!QqzVlc
zL-A!08&OsblIygKOMkhDEDEDBwoSL7?KJH+;t^aBR(7@|VRH4mQ*rn9n#SHf5^C|-
zZf-z|d~IWvm&K{1Nr8bEBUS`wDY>X`jNMJY6;|P^Wq2C;@JbIn)7CUBdZ9W#=UTJ8
zZ3y+-q)qJBa_{Nn4_YMkIvXzWiRB)9&usvJB8XDsaS!Dyb+LoBs8@42%0nNyiTHhF
zH2wL4#SDoH280nIe4g8*tEYKy6XbS|yAVLpBIlI5{2aUk)*SpjuOmuj&B+rWkPD2@
zM~?%(+aUXA279~wr?HMunERe8KB=>h89y%;*E!HI9FMv8`|&SWXy^-Dy!J!<5<k1l
zX=zVESHu%IgLmUNW@36#Rnm4V(A<N(zG$X1M@Hf>t{+($DSs{WW(GBbKH)Wx)Uw^C
zG-K^IU{;g+fT(~!fQTT3u;9woCv5MZqc;z3=DmTE>OVrP=QU@~32hQWKC9NQ#Om%b
zDC5R&iZjX77xPup`0x%8a-M(hQz+x=$v^i}59Tcy;JnuUG9X}XI=_m{-L-weElJXz
zw2=$%@)=r#<TG<ir$@J4;&qv3CPn?Xll$?k8u4H4Wb=06$jlcH6^AeNJZn$WFs|RS
z_o1vwH{*rfocq3icKl%X{3~nMohv@@_Q%&^X<Yktj&@O$?bE=FG>l;k@c_qP;>(Wt
z0`P7;)}{!D9U19VgT%{;B(koPQ4+<XF7m){7BM*w9_t4qg{xg0DtA7yWgLuzzV4`3
z8*(Ae)MQSr%}Jvhi@<`m?<fJ}=jDwDdyh|%I!i+NMZq%nof4b$flj)23Y)`o>3YQp
zlvUzq4$mf`^_0mStMdE;DvdFH#j}c$MZIH=S>#3@9gJhLOlzNw$vLehv?`(=GGGzD
zTHz-Nz<H_v1+#Y^Ot1LQWI9=?abppq9lE9;A8%6A65215uX3x$J91o?gbg#3z1`%<
zT|9LRAn}D<Cpay-yV#uP|2r5oIHE?rs&vqIsEy1(!GW+ae9;)^Rm;QZ*g1V$j*RT@
zfe|#H;sK%(doG6ZNv`tWzzctEs@Ayi%n)&Acw4QH-g)av^{*fE*5cW9lu~FE0}5e4
z+zdTV2ulAqyy7ne2h&#v0UQ}hfW_mLX$zSDdX_YR0XjA38th_o9vCfS0G`~C>}P7!
z_Pj?K_N+TxZ1!SapTd7TL=7xr1HLDq$|Ujq<UF7_0jU1uFpx-Ws^=>-$b={q!KzXX
z)AoJ8aFAEG_Q~Wv7@6&#8KS1)HKv>Zc*%Tna=JywhyS-}{h#Za{BIC*XJ(L)pj7>}
z42u+|!Vx`mO}C>}4!^_+j;1%n!NQBVU%XKV9mZbJk05=4#{wQtian8l4(G_V__xEH
zK~Mj<47mXmJhzuImlOyE6&4RFo74gO4XFMTe2&#jgrJmy)bus%i{vEW!%OKmJHOTR
zvp($qgYyLOcBTe0Ru_K<uK$a|?>Dcoi79IWrh$Jn;n(JX`v@kGC_u<;gn!bFKZyVF
zkw6lZm^AG?kIhi{%Zo`GH1WJlBgx;^LhcPp)ORIb`*T-+Z)k+nOKbu{K)}Vnt^%7~
za0p8HrQCjal8Nz0`~DagC>{XUKP0CR05kU2m((sOK`iMI^LvWTf3}zqn<~V_Z1M-$
zKMe|WjS|e8_bk_&zZs4fc;`WW8jrM0`=7@8myeTdpu}POW9GkUC^&`x9!Q~j84w5k
z+gj=&pv3OGVwt}hi<3s*ivZJ)N5OykZ)^GgXAtOdsSEG7z9@&&b^f{>>@06>;McVm
z<nGhTWJU4t$ZBTG-uYPSI!et>lqDPdmZ+3j`MgT&11k0hJ3W3x)CDZj?ZU@<eF7${
z0v@XlogSTu4zn0>%jy+@a?{y8aYXgb;anxTnSs<qzL$e3i`0u#IJkHO<V^p%;Zu>C
zuwi*~`Pc7$dsxEDWY=F)DRBOe(!alUft|@t`0bZ-|3>QP!y#Jo3d`}4i-f;a^4n`6
zROA)(_n)5q_ZFNyTp=bHDYZp~`~^WdB7%`nmZH;tYB+cex_t%@H6-jJ_ZI}Of=6=a
z2cduNuha{v16*8}l?2hNe?jmKcyu*z`M37|+JjUt94KPDBk9e560Exi9=(xq{?B1z
z`d0IzPVZp06XYBIxi-TOFX~3`F^m3M$#13pnXX?Z;GgOGm2>`)uHWQEfYYu+sQ>A~
zaxtW==K<$4kCjr*vW}a>?aOCwi{jws5deKszs&OQ`owSpH&*xg{j#sD2~ih4cN-Wx
z&)xnJ@zr$p9xmQ&2$09bEDqWSCsIhGTZh~~BOLWp?Zgp6{lx8d=F`JXTt=NVx4~IX
zq8{zW3<8wud>M0|YoE^+?`_VRw(%gAqh>jaQG^5|P6bNQ0Vh&VLZvm}vqkpm(@$m8
z{v7JJCwd5Ib)J*homiGU-h()I%hEneu&vqSD#)9REuV?eF_GN+3MtBzKxYkl?TiY0
zn(ENH12yA!#C_^#`&a=Ld7tmanP_4pLhmt9x8Jx=2p2Y-Ym?ahOcq1!{`m~;Q(m9F
zwmHWcn~KTTg9YB`9ILO_V!3oiI~M9IcFlpVPH>(IOpy|X1tuu=Qh*$fwtV<vwpHM@
zm@i7AD<xU2&Uf|_-DcknX2j|bxwZ#!=EqfIR%~rYOs)aGY`$8^cRasS;w`pHet?~s
zIxA|exY~v@D3c{bk5lOtu!6ka!0Y<N=febvuxND&Uq6qk+U&FfbJw0<y*<R<<s){n
z2AyZyQFA!MD~y7#H`AN0A4P!Wi0?7BJI&!oUu!Hc$zwZhO{D8O={Hyo$}cz5+qNek
z{$Mj*Dya1?8+9IjMkcma_lh+$z_VsEY4xe_xTnMRS_7FWXF=8T?QG3zj$!l6q#{|G
z$4_--;ghfFizp#t{K5V6J(Y(WZKm|rCCX(xV;-@gm+_yS&->~Pen-!=jjLFpY}f`7
z-Ey?sBH1|=Z&Qu#TYlH&X|nw`wu;4LGhsYYe3B4i);>F69L|ksZf+Lnymb|^I~Cl9
zNxRapm{Tz3prA0co<>{!(NqsvPv*VmyPW9I`PHU!CT6+z7-MU`5#L|MI_x}beKstC
zcE?Y0zdO8jIY!@o!75yQ+pJY282>`~sIyKbcPmZaq6!?dJ=_Mr*T#6f+foePUmc`O
z*=<|~{e<M?^o#F|2zU%3lQp;@GmPsJB1gN^#bRB>!qp3kg96FxA<OWk{5J1>WJ-(^
z*f;3%LODE)EPP`El~R7(!$o;rVq#+|Px*+`ggE}0rZ6T}HAE+3+`i?q+q$#}hHXi-
z;dvXkCY%<EsNr7WPWIgRy0<-SJHA=4Ys6iU1H{ecN~g>VDYi2-v#W6s?~Tab9n0<x
zlg@rydta59%y|=mxwf=ZSb<zS#ywamH$dw}qX?(b>)}40NBjM?HA6R-B|Z`K%*Zlj
z50xiF4lj>0%C4EP?8!@PygB?7?qy{x0rA;KZZgLS^M#jJuUOC6%NFwSx<}1$<`&*D
zmaxn#)zDYy`4-nT70f&4s^H^hw3lOmu`p&Ze->3vfFf;QjV|zfcJ$?H_=iys{o^%%
z$se+(N=3>DrqEX5e&dI;!ftTW;ME!CTZ6?Aijv$gTlmqDlWauE+-{a`#d1+wwk3P>
zU>lkE);B2Ha{_u+X(nq%=spfND(LvGiH>%h&fRuLVc6iesr&_umZ$kM{-p>S7e$di
z3Z`!?<d72aysjINrOY{VFQiAO2-axP;)53WV3yP^^O+4b%I*axOM#3QH|LpX%^2<v
zJr{%O^qjw3@&C}2+>hBAmlQ=+Ot(7m?|n8&u+}ImJU|!tBvF&Hw4!ZY8|Yi#L-9L0
z_d{(xuMPu^(dHGL`sDUS8H(ICBdyE}+SgTdhq0=_Dk~#~BX;RH%iwOwrQkky{!7Xs
zinQGDGzIpi0VlvrY&}M3y8hkR+QSUpFd9qQQ+bCTNltxVlQ*I=)ef2%?HMmFC8$tv
zesvyv7Q5l1Y!$BYqWcbSUkkac%ea?|o0je-ZS8IojH`%J>DHBD@LijAPCnei8QrUC
z8q5PXtGqNlC&$^S28&b@VGpKpKs#d4{ifZ;92v~tVn=<BG0&>4;0rKwr#-qnt}cm#
z<zy$-O7grbs+M&hksheZOe^#SCra)WBaiQvA6$KO?VG4iTW4;-e+bDKzQvLBS4@A=
zM}9m$WXGy~xBWF3f7`q!8l7BEj<hDd$S@~7f-PcK?PEy_$kMUtBrkRr;;M0lM;%pr
zvvcMp-qDhcyYp-xZgzF0Y<*x;=gz<{tvKk3tx;Pk24sH7KFV&LGdSMYtXaxOXbgwn
znhLPc!GxQ=QOC<@#UyVg&PC;YT^yF&eAlJsCIfj7i_#vUejo6BoXl(fu3MsH2IJA1
zzdA#Ad?CTB48_m_;vNeL?Wl~U0U&6bwLr7>-VQ=#i|0`WpVBMMI}T4nJdcJ@SmoaI
z*)C4W>#N-^Ey(L5J1}O|>FC!P@q1X}ez;=mGwe_~Y*nnN4!asIR%Dvsw|$1Wm8;_h
z4pbD1I4b{8r&k;B3>OcdF!6aSdoNku{FsVZQ%5jc<DUCcUMwv{F~@rSuJyO@qc4af
zUxcf63axSCkEXa=;(B^PY?F8Wg|2I#Q=1EBoGEZ^;@p@Dbg9`MveaBMHha0-JrcFp
z|41y4H=I4WFZM2>RpDoRbq3{_eX@CX{qDT^3|=quJzg7$9<rUOEYDI2YhG=E?_?``
zqOWZ~x+rrb_sy+O`}kjW{t~O_JjgLVW$x8wkM=VXduLG#?jilD7g*EuVc-_?ZDZBw
z@LjT{(x`Kqr4q+o_s|65K<olL8&)SezSp@ZI+f~s&Z26KU^Zj*6Kr+O2k3A&xb1cA
z&oi=?rTlv`vE4?Vl{Zq1maF)Ojb#OC?#%_hc%nK4#-Pn>s$4FyT(Y;eVf<ihXWN!1
z)?f$TuRa%OQ`yk-^t20h)$b)0q%)_=8);6Q1ww1dP|KUvu47%=8@FOE`$Ij%UK2%I
zYw3M_Fh185=d3Oz<V{AE*WZJ>Yhe0t!X{c>!`w3^(W&pT`sgLQI-I+0M$=os=~1j*
zuM?p1m7Hy=i{FdUv3a)&mPfa4v0LE@uLJQ>%qhF6;r1kHw%#MDP5$m9cp<{niY}Ww
zK=;o$RV_c`Z3{N(-s5k%gxvkC(It?EPB|Ev+o-r5?6_LJGn<+0ZlN!Dx-%FG*-Q(x
z(-%Z`GP=*d-+v<R)8fV)zDp|{xA2hJs%L+TZjL@`3YKlwm(b$x&i}=n!GIN*BX0U6
z4DmjTsws2=ol>e27;aSk%x225r&}@IOdx%cw1GZ*?PjNF0UYH6x^3>%wZ!Ya9&C}~
zb2!Z?J{(WeL%SxUGt%UdI`5uD>zgu*HPZHEsf3-H42#^v*n@dfl(Q~8Ers`e^f!7j
zyclsy!dtNMK)6?DWZ)qlL3_xHOmuQzSG#Bz*1VB2ls+@Xz=cWXVa+gWxB2d6Bwi?d
zeV6uP?W)%*XBRd1!9D<hB2$jk#OFETJ~#E{bGhKpHTvmy=rLIg74(8M`d%aD6{hqt
ztn?K*<x(LY_0r}kUfb+#stVuv%wnHo7BW-xqY?8jN08bDal1zIpU|Pj*X(HdU%=EH
zqw`uX-t1`yzGz28t&PI5Z%Vd}0<Y&JaSmDRR)(fe=<JyMWvB1QN+WBUz*6k6^%meY
zxn<GrpcNdnADJ%g8zg7QucsrHaWNWHAN<)6=@ze_UMg2|>S9dVf+J?Ysb8;rL>$-E
zchN@6yNrT~&kHxX&qG_%YnA6VCDyWXBx-z0ww@rWT}k#J9oL28)&1JxVE#q?XR1m+
zBDN1kSH1g-$FYpT5&mRHj3VTr8`X3@`UeH^G~tNs!;L6;!HEmlt_RC;FI-ZLeDJ{m
zYYOCVb2d*r%%L|3ey5e=r8?G-8-L^73KZdvAsP|tHAE}{4;#(X1*2Ex-B)um4q_cx
z|NNLlO}nftm~6r9{dPr*c>ca&sErZ$ncdVVT$eYHMU}FtAkNwKNnHh9<r&IhW8K#<
z!312F?kWC!A#Z}J{oaDk>AS+XiRi)i^_dqwmStU^f!|_<m&=ytgKc~KX+k}&mfHCn
zF-JTw3C6fsQal2Mf)h@Ftcoh2um%L*W!&!IYnT|Ul_t4H1xBO0fpvzmws-m{W5EFL
zEYcf;Q&ZqM!`WLlkh!p{6$>A3mQa+=S;s@A4coxOX22?ISZUk<1lyi0s*@s=20xk&
zPLqjweP(w`JI!8>uKsFRpoaVOyA0tlMf0+=C?nR<A6M*REv#qt&!Du+`a{k4;rtk2
zy0oX?xAmc&oDbTEl6j`?PXtP9pUrTe;K2tR(b;`QSfz>qxCiPI{26<Pdz&p!g}_K)
zZBzYr8@1>wdR8P~=nM)M8g4ab(M6vVT`sgZzCe5>EYv<b&Exq8!M>g<w@tZc&wxjR
zWSKCp&O{@(`rdM5Jwk<aDFu=U@7NaCOy8c5b$S>HmKotibR#8X`L-SjrQl-R(TXB8
z33Pd?G3!m!+lM@Z4wo^+k)$KNj27DlJE=4C_+nht0`;@}&3Ymdbh3xQsiAlN5@(I;
zVtg<60yBv4;F*ftTxb>)AB~giRmU6cQGH+W>2j3H1P}a6a~v^-J^90TL`t%K^F<8e
z2b#D29g0JAcFKa2!FzWyyH}UwoFqr@X_xhDr9&1u=n!Zen5K4JW+@O#fh-k|AosV&
zU4BMJh<V6KNma<NyBCpQSsQ(JbJZQ3Rct@cOdzyu5Mr(g>Bi;Ndl-@;u{=<+lAV*E
zZK`}Eo?NG=pcLIH#ASH#qS8$}%@i$;=2@kFv1K9FoC{IdVHeYug)M<!j=|}3ZRUaO
z5sh3UESRPML8$N^DwJyYF1@28*YZ=zh?ZGFXGMk92nnsTiga&qT^ev7`(Ju~FB+m6
zVUdrAx^q3$fVTL0=mQKCVK%A-$;#;Z$iKPRvYT)Unnk%*BGD$Emjd63OvDD!QuUyw
z;!BTv#pRG(dG<n}JN$!B;NyXW*I6TZs@Cc`!V6x!<~?<aJnp`Xw3_10!KXJKN412o
z*|`9pNNsM&x_tC?vSrau;HviyBFCZehbM9TIpErG)MD56#0xjx373jpa+H7~q;K>I
z14mOROJN#LHNrP<Ca?kevQ*hbJ1R@7r`Bmh&j{l$bv?`pZzyAeA|fFYmHUNZgANBs
zE0U2WWNw_-EtBI=L*pBPsJeAof$qA|kDrYX6uBsw_%J~Cm{tOAJC<<3KQWOQpzc?F
ziFqng*ID?Ror%uNE`2&uOCj89;`O#ap0($0@lqL~Z*((CtO^VpjZwM*CZ~NPzSUv=
zy+7<Oy}?o_fzjKlb=dw>`eD#RS2D{HUiYUN_TR3q(cBemB9{TKvz2;Sai>Vm`oK{%
zy-mdvH-)XOAPrAz7<iNmACoR_rJK(0pu6bB14VSpsh0WMC?NYo**Vzb-C8K4;}V&@
za7f(sBI@))3?eo{@6+!it%ycOfzL7%qQov0n~yklN`oQ_yjCnRO=Su%>bS_)O2&};
zkJoNG5}qg(La1Q3{-Ki3XJY*b$&D{p#Vbd=4_4elJ|%g#2%kY2xuy#BX{8U%88aSe
zK^V-h(1|4a9POFgw=e08ZNzYdgMQug0sU=W-UQD5>|C;k4MTw<Sb98KM926|+E>i9
z$VVTly1iCpy0DkxzL?cp<FWin9?4GM-{fV0+3Eh&!Z2K6fH(0vRMiD6-!YUrjp*iG
zXY`;eL=v;LIAr7sd!lEJ{OSTM@5MOSWNeQ&a-G9Y_;NNfA+v6DYk&JkK0R{F$r!#;
zHoOt7p`EqArsKKZ)b?q1Y%_W1$2!Ygz}2uXng(8Ii-VUyG29jL`rH^9b0nw(L~~2b
z<ko|JI=wH>qfY&Yz?8~XFw)2vkN|v9$h+@N)`mw=!givAR;q^+Pb@E(3KOpULb=<a
zgwHhnxEB6fW<!^)ZuvQ^y^lsnlz~y3a?eMg;8hZqR$Jsjk5$qszJfqPJfqWh<(YwT
z{`d8%F<uk{IzAY5&ZaeRirw%sr?id45c5!~&rKC!^U1?ei%`UX?Oc*eXV>kyL@`#X
zkQmE_-GQ6g306NCBT?K%UzlfZrz_tHY1)~SJg(%@{6Q<Q*yn4Gwha2ZHFd<-NzFaW
ze2?xKS>!{W$`4$3<7Ai0<6P6&o1S{xdmdUAje|MPSplIARaF}CrNzh>xuCr>PK_Q+
ztrtUCwnrSg_&*OPlh7DSPuSqBnan?eB4{R^jN@x;6xP3A^NMJ`^->XXy9}r}i<C2j
zK(109^GAW#(=KXOhKDK}G)Y(#aT$2xFP3b9&fN0jvg_wHepfd_wN`S(c9*jgx&K}a
z04BitPD1=0Q4c!^-M7T-c&V;k*YhTMqnd&)NtVZ~n*HFrE^7;m+sdl(L5_FN6tknx
zqT<6so$SL1v_jdMD!hC*0?|5FpPK5dI>k2I#a>1?goUT-t1@jwsVZfKd4T!vAHZD?
z6GeSNKJ~H>1HPB}L}}w|vnDMv{hNCIv^8sf@7)xf)9Xf`&X}NXK@l-4OC64+A|f<C
zMaCn&h!(je1WZsJig3k!E1{JxWZ)A|3PtQP-!o9)Z*o`t#*?ozkzc#bMHx2?bO%|1
zBxK3|VBjQ{CctYqV&8(^iVr*ptLWdGmzUHI%c7}WF&-_f(~Dw$`GUVw4{m$EfZ|*5
z>vOaQp6g%g<;arF*kiTf8Dh+J)B-upyMx{0<?LWF>F(iCY>wCn=&z`0<E^b)SZj9b
z_PV7_(s%kPd+ofl$L^%RFhi_{_rdUZXa8f<Bz{)@nDIUx+tGm`8*n7Vp2|aRbq2~E
z&F(w4k3?qUjI*oK%T+Ila*YC)*-cRD{`71{QKRKnT0qQO<Qu?wt_DKajDd=69}&S0
z$+S=1mgJ9A*jF?gw<6;g{L3qOC1&EW3h8bRB<F{sU>tUuw)mrhyJlXk>qiT{NT*(3
z4`19+^Kn-_FgL9?Y^v6q(Y>dS!&v9PvNS)w5QT@r|EdKsL=k<jz1uS<h3Q4QLzd=H
zh~a2p^BGQ(2Ye;Uub2utM2ur>yg^vt+$DJ%F}m(*Ay!l{jcIoWdeWbfq3enG0pL^d
zXTrl@uEK}MfDbJ`oInGb5A^8!k!u^bfm6|%I;AK)`rUD$qOS=v-m7Vl<g@BA&WWfR
z1OCtH(*cSX<sNZpUwge;+OIEat*4UEIa-3MIY6~dGf1+I5{%ouCHM~ZR2J#D5A%SM
zaWu{@e;UGnZeLX_@7^?BBdHY$&rlR_U@C7+P~+6aK|e}1-SW%2Hk=mBqpcLsUE76o
z_R^y>9r4XCgCFZH33mglG-D}f{nC`Lms-%oGhG7_2vUobUrE;D4&%%U<EirFOXzor
z@vXhJwZL^XQ3B!E&)a~r%0Bs>L{$}{rBo+uvNvMLpni&AduxUc7-hG1pCb5IDyGBK
zBP@zLZgQqn$*gf`CpQh8qKJDSo$D&TQi7!8WjXW3D22d6)?w+wgb=P5#e6o|mWY{L
znpO%>7DJsN<gk&YR`PPtW3@WR+B^koH7+<0N#g5=&UUP`u*O%&JFLS>7GMb)Ei%Lj
z1H$n|EtD{V`ixUgTV1{>D0N%#C3mhMR%_I~TNr&jD|!0J7GDoLDq_Px$-t0p$P>?F
zL>Y6Q#PzkvK~(olw*wbcAKd7NYs5Gj({+lyla{vyypk<yP9)RYViaJ1+Ndn8gh&7C
z{Pz@Ke>LAWqqAh1xjg#pY{rE)CU_>y>%bruL=B(k27q8J6hQ#bu!OYq8YDEo4sbFj
z2`RA^4<DdmHP?x{jjz4}Nyk^bOog}7)p7$gf|X-J{#d4I8FCU>Ht~M}(6`ED`DyfE
zGfY4&9L`VrI>kyDX69E0SVbNBkt#wPc)i&+1>K55@CdoK%DB4sk{NZQK#|6_Ok`lA
zrVww6-Jq;W`A|{v{-Qo@OY|lBxlsG<v?iSpvN!p*pn(;g7d8!>s5NuL9<~$|))Vbj
z&Ls~UL_NRVY@=lyD&!t^-}h^?7VVqajs(pz9sseAZkste-ip1NBwS52f)0MnDrK0B
zcQ~KIaewb*O91M8#?!xiX~k-k##Mo;paLh>L@IS27D+wISIle|Ni7{|wNl#O8H5Us
zguhM8P}9JBRb+HnoJ*kgOtz4f6Yj2by&z50Ov6#V0~?{vB`|!}q#!+b{vj8?U^q7V
z#`!X0MxrS-eGr6olVv5E46_H*!zHx!+kibPA{Z`BA6%!cL87g#ugdT6sH~ME=c()j
z2PZDX_gtWqqXtQxHk^jLwJ0seLIZEXeVbRiF?v{d$Sge>?MoB%S*bY9NnfLVHY%@j
zHvTb1JAH{|*Zy@MSDHMW;<RWzH*VS3(PS@H_#-@_Vi57Jxj`7dcG*P5F}x#<#9o)b
z(?A1HxY#HrJy}&gsY!y~s!+PiofsS1x(|h|#=Wazm}C99r`F6yK0e8oXm>|L@Wz**
zX1p1%k=?KoS{z&qm~9Mm3>(V?%;Ju&GqyOCkYG3L-KiM8lfB$JuJf)S=2--yU{*b(
zswT?Hx<t^6*d2s*j<{i3@S073N)8)#4}fN^#QSD{nB~>q_wmf`0tqFZIhS&GBEKOz
z#|+!DC%_9Zt-wu8J`rcEf8PxBi%AvH?DiSpWv}czps{Az?<BBG<T5p=t=dU|Enh(x
zc3WAI=k4$Bp0F+;>I-_|P}pOFytdRVm|)RR`fmh4X}$C06-z4C9K<Km_bYWNQ#?GB
zh-x&^o!^N4@$lO<Ae%hBu%bUx!y|B%13eQ^&tm#hTmPx**SY&=0I^QgPlo!31OFw`
h|FFgXKDIa_d+;N1L7DoJ@hR|kUsg#b|Bm6y{{!c{5E1|Y

diff --git a/docs/figures/arch_5.png b/docs/figures/arch_5.png
deleted file mode 100644
index 035af8a176b9f84993aafc9af7f8dc06ed4ad816..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 76885
zcmeGEcRZHw`v;D5yA=|5gd`!O?7e4^k&KM&QL^{m+%!;>>`mD_B%3>xvLkyJviIKK
z^TO@*dVk*U-#@><zJGi@9&&fNuJbz2<2=Umcpk?oNLf+lBHmRz3=E8ma<Wn?7#P?v
z3=CKn&N*;}Wyk+J1_oZSxum4BoTMbRvV*OOxs@>nhHOw|^m%1fEi&YiCwt;E4BShh
zL>|Ox#1i+W-)%84(cmKP1^T5tmLb(JWDN{NUers!RZc3=^Ael+B{s{nKMP*d(&NuB
znvRe(R<TD%Q{uLZLraOSi-J=l6{dehFm5>&XeJOaV}#N)MLpXo=Qesg9<zNLiM5Ih
z6vK&*rZ)+A2FFm#n5t_EV8c*oacyPrI#}EDCAnKm{pvi%Wq*eo;;*9pR_&dt{|3?b
zV^Hrss96lNT7etfj%B+|ELCCZAD-#<#Gm*k`P0~sK{#s?TG$w{*ycz&EQub;jh1Wr
zT(bf5LB-Dl*)P8fuEmND4!$qeWTtEQY#{JK(sKK~{qW?8y6yWBI};A!#v-NBax`vu
zC0}QTjE|=5_iY<xqZPk%NSqhE`-kF18~%FB^)%z%X3dBzmql9d9Ht%6Wy*TfX2wQO
zUgB@{D#)y}vi49$@!Z6%5#$o@PY5(8ed7LHsf3REJ}I-@pU+7q1y2|Xcvw|3lrGsl
zIdoLL@jg>f<_ZZ<tCsoiLvFt7Ts2Zx8JZ2KSM8I7=2!<krF)V$nJ0eqm}E+AH*K~h
zHX2YrZr^Ts`F65_%*~^<PpWDDTmNFQ=AU&$Yz_6WOh$VXZghUDH2Xx$0HU8f!{Ojg
z2DuxI9f1_PkHF1Cihh8zAn+tU8P0sA#J(Q;$C&7m!#%$6kcY9z#!D+LQylp$c~%%P
zS^e3}D{NN9q%;k&<a!L+l6iz?iB_>4^ii;%x){i&RPw|THjJzt44%@Jw6FxmVJSgO
z7HUkU6fE@=TqnOv9Fl7#Sc<5Nt$l>>r#OfUq_FF<nB)?$C?ts?>n)7ChTMJUm{2w!
z2)mG9#%`IN%S5@`k?Wo>`|Q1ZW!)ch87~+~Xo&AmEh6P-7Rr={JtLVBO7hiTmzw<+
z2Hw-h@8Gw{iEgWyVSglG^1qjYXU48};Q*HStW8qmo#Hf2*ZGULZQrqTz``F1<zZ5w
z93PRkyg2xc<A#}l&;G*kx0f5OE`i3EfH8(`5iiPG!mawku?KK!2OO*p=@ta(A3r#)
zy)4m<+Yf1;%aW^v@KEQ=3OkU^+<o!3{mQi$FCIN^;ZVF{aG&l)fL5^9bH)ZWLkkXt
zm{&1XF=8z^e}$v`)xXC*rYyQ<h0jM`1=m5cez}fHLh09!d?zpzAF~rU*M=~Cv-a)V
z9~Jv%`)}om<<difx;G1`_+Ne>S6|6+Gxg#~dez=QHs(4;_`Chj(%AXsWE=-Lt3S!-
zc!Uh%(khPeGogC&ddm6>^;aJFWhpt)og?1)YW!8<>%rHDe^~!u*axS{a?uHf-;m8t
zD@}7y!I~k;4p>(NUovgC+0+zol}Q)qE(Q*fRESlyKcO$i?SFbrDJqOVj4-VIEmhd)
zgIkxeUQ*D%4pZot%}&=%5lQz7%WRBqcAVH*&7EYPjGtKgQI|oKy{pFOOv4+kEiEct
zbC>cVHPe+_Znx}69x|#iC8$k5yrQ1QsLgba+2j@}a}T4w`ZcwBX7XF^j4P3=k(-Qq
z%Cng?JbH2GLu7Ph29>I$C{o6!UpNtnM(Rt6-(O03bBmdA@RkfCyLy3|SpHD6#FcaS
z1hk1oLG}?H-?q8rjYvJC40~Q^T`XB_+0Ek8`KE9;MJ1~<%fVps6_aYWJg<WDD~pm@
zV&34mAnmLHqEbDJ=&l#0zf6T&|F*Ivcsp&<I%^I~iQf0F*cIELm`%MJcUA1_;}FWL
zWIQi;$>VI|vg6X@OnCx6nrX9X-^o|WFVSX?ZMx6Lov77T{o`84!`{vAD2t`;#h=%I
zadriDHU2X1a(^%1X*|O)6E*v3=JmAs4EAjP&)i<mG^H-nnapY1UfbRyGea|k^@ov^
ze#4*2-$IRIEH!QWjHEanc(tw6tex7gn6_{!a*vpM4Z`_~c;z^=c`A(Y;s;{!6kOw6
z+CI10$I!;9ie?T5TQ%@CbLR5Taht}RkC}?)kFQJ0<>$9qjH+00tkdb!PScq$bF<2R
zGGiKR+HbMb)AaSFsk*syd)(s%>zaYeZ@VEO*&!&C=vFM9IO~)Jk3qgcTbE52qN;OM
zG@hb+@I9tI8`4CQdL|P+dz4VQaGs06<Z^#U8|UCk);N`-iK4lpv*Ob={3+bO3(37p
zdE+h4BXdJ@^-gkj{o|c;ISWnB`7YjnGv*pxg~vAity>h0sUFw4l-Wg>C)*uO|DG=D
z98b9UXJ%Y$y8n0j1j&s0?~N{<@5236Bc5Ns_n20+3mW7V)-92a+Ppp&@I8Anc)Ncu
zf4yp2X_IiPaM_pCkcb-B?BX-516bL)P58|DApf|hw>fUQSuTeozJz|GseAF~j^rI?
z$>ad77x_4rgqhFT@d_^-A%0&kV*F#D_8^U?nDymV5-N3y4bB1XO1esZ6PuQ%n7Ja0
zqWZol0WV#(A;0pD9sRYkR;s|V5ca?&ZNqZI8UNf1-J}`pA~k`PfmPF*Fg_ZgcWl3z
z`D1iiDHSH@9@4*F7I7PNFWk1-S2+wtX(``udpnV!7RD1ZpHR)~e(M`=I*YO9G)*3&
zR&GOKIB}PgRbatzz|F(S!{{6hEzNkEtXu>`WlVN#1y6~p6%!j`i_Jy1+Il7MtAPXc
zg(mIL?l1+J_*4oN+8M&m+H27XO$kc!@1w5i<dmy@TyNkCH>QZH`LWS<;4Ww)XzM<R
zw?Ja4v7Y~=aHtATJF+~-x^!`NzZ=WatEVfEK3_q{le_&!A>GnJ<+OAD&ea{u)ji+k
zop-eowGt{nYeviu7O6PJ@qFj|3~L^Jf3fs@zuxjL>0RTy2Cv+|_4*X=XYXgOgve0{
z^YvQ~l#-WeeXOv2H5U~}^et>C>@eR?C;nb+`t$UqGS&f?(v}RXEtgK0naz<^<Z2Y9
zhcMsLw)R^U`?}7W;-Y&s7LwiH6ZaEcT`CqV`tpma%(kLM>qQAScjv8h__Ejr0|ui@
zmUa%a4i>~qd<sNUTvDb{Wl!Hft>JydOJ=!XX*t(5$JhO3qjUG2<Vt`I)uzZ$r(w^B
zvbP<YPVj4Que<Na*R$)%9K88NX+&u$P~clV3ty0WMBMc;&pb^qy`qf%Q~FxkXyd1c
znLWgHaYr7@5u4dfznU`n6KN6~#kqa>w&te=S1Nw5HBRjfl9r0rA{>8@eEa!|GJ2m~
zuVUVN=Z<jI!92+nb7p~%zNmW$5NPG_Mh3+ve#N1WHSX`f6r|2X9ppz15|rr(PdO)5
zjC)M&>$O)!FWE2Z)loUsl`t32-FEeIUBWti`Z-{Ra8#tpooCZcf$&r2a!`GEaptU2
zt}pjteo=41mQ%9nXo%_W_Ul_~y9NhI8+*Ne`%@89#_S*Uy9l%|CS6^<UU}ef*zBkA
zN5kj~_L0iIMz>{Z@Ao6Sz3~RwmLDjJ#`u-8Z%b;UlwP)LwbndO)@K5O35}>Oh-drc
z?6m9-9n$YMzbc|PSTSg>BlopDSUjpR6*2PCZHctIse=ipv{NC%uqA{^1YsjxC@?zr
zvy(hjQnHVba<HSI%-uz~A}m7#evC`am^kIQsMmNO)9)Urj$`7yd+%`*Lr15%!Rbi2
zKUXQRo(5wRU+3HawuskZ`{?Uzu|p;IBkyy>`qrq?DPOFH0GvN-<KLG6h<$0SDQBXf
zfWZVl<6yusuVP?>Pnh6a43pyQvoz){46Nhp0R9G;W57@EQ3OAsuTby}J#+FCE9?cv
zdGJ3X@a>iYL*I=JOTj{ahGl`@Vcb@gl#>HLRgD~sjcpvwY@H^STRwmbxOTEyju;rE
z^w2k^oC@7C7{AwCP18wJ;l7ZOtu@=DC$^7`+1#w{pm8un+=ReKYh$NJ)Na;RHjYAW
zqBoB35CWf}%j`F(k8g3Z6uqITpiC`k>tIaH$Hu|NaYGD`nwna~;faZmij?%}<KTay
zH_V)z?1b3aU0q$-T)Ek79ZcCd1qB7!Ik?!lxLCm*td8zBPLJGJZ5(M&20@P_W$b9=
zU~cDRZfipgjr-`at+SKpjT_L5&c06OY3yeH&zo!<PnQK2$PWF6os*4&{cLRTs0egb
zNZH)Y*h)*v+#1LXyhDsnfJ5Z?{{Q;TKX3fklbZiL$;l<a_3uaj^{fB;sJf%EgQTrB
zc&C%tKl^q1@V|d~`k)9qwDf<8;zZ8JSAn3#@I=_pHcbpKc+J!b?Bg|aDJ3=V6KEOq
zg$aGWb@CJX?EKQHXTKW*Ljpri>b9C2=KL^rf|~l#_mxpnMoOYiYMfgshKMk{mv;k+
zRuM06l}~hb4zt|AxpxmQ5cbla`bJ>Nvn;2l)bx}(YjsoGb>oZ0RrB+TL$NzUy{}CF
zjClU>mZm!JXcNWc8%Gep5fT`f|MkzZxgUnVMDle5p#TgR4mI-s{DUN=#+eBCzgLfM
zLQ>%%;Jcrxf8G7BL7^Anz;-VF*Kp8-W9MOTKOc@CPM7|BHsC>j2_LNgo(Fhv+YjS*
z3Kc$=p49(Xh6Iv2boBqs{q(;+)Dnp4^H1-SUHV_^2g~FC|CaY(JMsVK-P>0(>`ap@
zHfg=8RrL6InnHXJJ;qdfa`aS^r{Xz+Yq!{|m;L2uQ<P8D(k=<;%^C|M?!Oi(j}uHO
z{II^ho{ob<xpr?+IQv2B3;6+F?_DSNy@fJ*Iyxy*c72wMS9$E@L@&QQ)ugsd)HqJB
zY199u)`qx#zrO^?kJfrS;|N&xu$cE3r^^p^7aE9@^I65f&!2DJ-&sqS3uj=<e^Y;|
zfYg(~02s^0710xXi)t4?V)5LZe_MBSsDnqsL`TB>FhV8$UPTt)%bIA^;lbWyXS)1u
z@%ig+5_p$yA*M*<TN!l9Y`RAyS#=(A88=;?5}_Kp$>BV!=Ds!T>ipdCZ;QP<j@f3O
z>(U69!-Om@9$r{s-Jy1p`(N_4%Y~=*Wk>_8?ZU-#3b&6f3N?yNa7}>Pwq_%0fB6fc
z{Vf;_w$4IJ;_#$7inUG9)LX!9Ifa0V`^gXLjoEI;hed46GT4rTpRB`GGZh=-ZEI(W
zJ=8w4*Q|%-=~k%pJ}9+N@Be77ny|Y#R6#q_nf`0^V0-)xSXQLK)Xvy-zYAn+vOz@j
z6RVTWLih<>#xFgY_$(egvP%?lesZw8A@ot>c5NKZ55H6C_YhIzeC%=idG$nW`wzz^
z8pDMoR0^54Rk@q5doADInT$4=*j^bQn9R;h3Ui+83BsF8@UGq*uq=MkNaQ{lsWGwE
zF7E2GqR6hq`q`@E{d4_8a!SgD^;ILXE5bNxqeq8(ODY-n$@l0lUAnZ{dJ6}(z+&T@
zY68pop!?jf%4NZ!(cq?>^X%6k;?*CtZ+=8FYtV^^=#F?U)ugdaT|4!kZonD+#F|zM
zI-R)dg)EH4SM0Sv;|cP#OYbFlZ))w`rIij274zPmZkK$4e`(`Bmnjw8FX&YRAxzk1
z{#N)+rB9<02(!%PJ_?eyD0TyVwOiFrGb-j?S#M<qxSXby-H#5o=z+$(H|fvOK)d!2
zuYsmJH~2@LDCk%jjOnW~cTm@AJ*QEF?AGG&QnB*q1|aT@&aq#aO7gqwGYh?0^u<iU
zD%?1ZQ*G8q`ZX2ad!Lfjvqo}LCV*Y2+HSmyx{)mGs#r(GVTjc4(ahK8eGtzY6ENVc
z-YTV5a@mLqm>${2eDyU!haZxJqICQ=LkM09o4w72Fm6-D2l8g!c@58Ae+eLLj^#F8
zcg7Xnv-|#<(qZkd0;A@KltZJacTqlzgJnv-`<sKdcb~t{@uuLn;bHG6OqY*#vA{4o
z=VuY@v$w$C7)JkUf1}r6lialNBCI`0l-aVkpqpFd6&ZU$bQn|aSAvZy<!0S(ZrtvY
z8{<^*{I)%Hv0?P`_X>0?{v;fNCntOIbvBonGz#+aBXYI>=9#o)_h*z^bQ5iORXrn^
zJZR+~o!-ol3zt@lPxf3C+?)Tty>%;`R_67OgRRj*(@91$-YA_<RvmT80aQMkTY`t)
zZQDb&-kv+tsc+W%vP>;AfGc|UxZ6P0!s=*$O(`fS=#sI;p?z9Ps?3{v9q;dMuoB-?
zj8IGz5_Aymy>SwCkh$2wzGHvSLX!xSp-6L!$v*d+R-8dA>6kD>onVBrL23baeWXC|
zL~Fc8GKIL0SHe>FdD{7sQB%0MeH*{+$dHd>WPBeLt!9v8@-Bz%h={j$YaFk&f%irv
zt5HL+{i239r;AO^oY=;<LEF0g!u$uR(*9E+h7XdLM(cbN)_@dWtI9lHNaQq<PS{h~
z`~8i~pljnQca2p;3C6~rlGx5y{k)Z63D2rvj;=%Rd46e>WnHgT8JEZIOs2JT9~pSY
z(>S-ly0;+lpsLR{ID~P1M@jsEXR~*==F&j_U|JJLRKnb7$c_%@V2VDmkl6MZR{SNt
z?~Iaqmhe`-aq=MC;m7eDkNj&7yEP0-y@7u7(ITqne2mTE*0>|daqN+)76$Qr*4LOC
za6VW|YnpPyCaVT=nz--S4ZgTYag9lJICW9I&hg9`q9p2%CXDj=I!_%2V%gsB=TBwI
zYa*L{MnRiu*rW}Av34t;Um$C!Xv25Cb`hqIkzFsb@jb>nvzaf7YoJbbw9IB$NKx<a
zd2SpvYZYHLiJlLSB)toZ-UYGAKWA<r`0<x@v)2>buYu*Zx9>jna%Qqmaan83?~K9+
zey{q^uXV@BnHDya+qO{KEsuTX8?Sg)IdU(OssB~224cJ4G<haeqqI9F(wsYrbk3ng
zxuY-P8z~8iF?+k0V<GiV{6Zo;*q8cxD^{IPAIVF|oITYwEnBX1FNJ6Yx(Ih?YzJ^!
zc9^^fW{Ti78JrpQb5F+G4kmY6u`N08$8`B&9j<y=(c4!9&zd8dO$)gXm$nN0RhFOQ
z-BLFQHiZj0m2vL%jIi+%%<}i9$`HmEsf|YQT0ZnvJSey(*aDa$jl#tr*1E4wotO=x
zd@tE+NSRUnEcz8;&-Uc!Vg;PLc=hYyZo$@H{9He*^9_R0ozxsIGzxJ%9oEX2vJM15
zc)wp2J?k`K@hly6neXG=n=b=FuCI=NG>r#`XGx=wX<x-Tm!qXSfE>O#+udw3Kb<$H
z?o82a>)b7e?Z*_sV=AA6!QnENY2p$$CH>~Qw+6ooF+A>hYzcF#axU-#d`1=J3aw^p
z-pX|~eF1TAEm*dCre=E!ZMmb!29+q}E3F&2jJ*}Hi&h97rBj$Ro)0wqCEt|qTKY0?
z89+ceKPo)lDt@$YL`BT&*z@*_ri1m8<^G;pMFaPdRm1Aec4pYVh*cklo=K93dsy}B
z@Y4V~<^uxgR=g_hsptPn%<(0#Y{Y|kB(|I?B+<ed_4A|oFM*pP<nWxy{30x>l1HMR
zn;!$XH4624>v{@DuEJgBdKAh=@q1h7tH4q%1dH$*)R3gYYTUQGT11@>cczm0>x%1y
zfDE0lP}KJ|^F2jv*QE<O{_XqT>UKT#z+<ho+<tt1Dr8D3&R|Xh+~;R-r@1j#E&KhF
zf=%DG%w;+;tKrJ-5r%R1XL7paTWU|9q_y_>>5PKFJy?EIb*1s6`rjbdJ;l2#LujCr
zjiAQ4kA1xi?V}vwckbMAy7JD%lHOTlcRF=f$Vle5K(sp-j+(p#u;rtDM^Awg&#gs)
zXg5nlUxlOTFd_Fc7oUDnb|`l?wKBFB6GBD)39(R5o>pr6RCDxF!LK%aShv78pTjf{
zmnUv+<<=nL+s~>$%Csmf>X1)ZY9(wiq29C)@hzMQ>$@-DXfoN%US~x*710ZD%)khP
zQOVVSCLTWjqS0K~J>KgL3^xfPFCG1<aGV-wRx2?QJWX~`LI7*^kYCXDKh|U9C|(SO
z@Q`IHRw-QLInPS+py@YiW$XLz2XYR*R=$zhH}niXMFD3x5H04r{X4*8?LtZ63jYPp
zXvbVrtWOr*vjyTzUA#*n;$PZgZ;#bPCB|c|w{W*lZ+<o4vP#lue%ZR2L(REqUr4<S
zB2Mp?kT8yj({yN!LId0PqIjqFtp>Kwe2y9U{p_W|+Sp|cHeFlQ;_og55BocNc5zdu
zrlt~CK3hUhUJ^|Hx__qjDNx*28hcKr|MRn79A{+LBe5#$i63w6*-9QG+}5WbcnWX>
z)F-g8u}Q7$+%Uu=W$pitfOlLGTdwFS3Pi(!i!Gsy*%~Bf-^3Eg;SHm1^e&=Bqwy4_
zS*FqqE&xS8+>&QKdhPATgrnB3$1|7~#V72MWv@apm8F;0dQr2*Ugfv(9r?=e97$1m
zmEq*Ryr#6pNi^mue+V#7X>G*oQ+Lsb;D|QLA@>cUBxBS4uq0CT_}fc?Xe~~hU{TuT
zy`{Q0zw`}oVEontPY8FqY#RPbUnzPj(LGcD{KDK^SFvho1Yfqe#V5V_FT}S~D%3tb
z%-1f8f9&hKFu?0sXWANP43Nr-8X{|!&vYX@F;UeqOF=>H))miaqtXi>XahVJ-{Zjm
zjF|rq2L`}978lxmNp!*h1S$()wutV3_PUP(F+W_z-$rkO$l`Md^vyE;99t5vCJ^2C
zwo~sZv8VJHKoavC4Wk0}_%PrZlMEeOw9MsNAFq8a`23d})+!wz|6+g1_oyV>bf0l*
z`?g`KbjbRa#i!U*;L{jB1qBOwDrnKU#8}^N;@Yh7-rMY6U{y(zoiAKi>R!9vIvQSK
zS?t~!e%tWNi;K=xL<8vqR>`!57Nmv%Ejz^$iftPf+p0gZiDJ=O_ihTqI!$Hr=mAK~
zUrFIT@kiU~D3FXS$jhtCk{&6?lZot1gl{U}ICT7uN6M1bPXCeS?tCaGu*c_GOk<3C
zl}`J!#QP!;Sg65E1u)5>m)GK40uws`Ij)IweMPBanS7Cf0LD1_9338PQqyNITl5zj
zSK_OG|NQj)c*O5{!oLh;D^Df3x9=s3)wFt%8tzlu^?i6$nK*k@X3$(r{DVbzj;phr
zXk8or1kgG*{pvpbg1C(<N7ExURbqv(v2E@v3U9vhU9j)whN)9@vxxnY6y0L4_`WgM
zOI)Ld{UPUmDdYY9QS-<A-wyQ}M(@)Xd2U}nUH#Bakl2e^THHC|1N`XVUFWDxLOBgl
z7&R36mHRER+<cCol7tFj<=X*qTA6K=?a8AqJFCXt+O1}BBJK_erb+Z9@Ym$r4f;n-
zk<1I91GMofjhmyk)n{E|7Su~VyrvSjsWTb)Xuj#Ow=rjWG~$(N!WOp`_b3QYTWvY*
z4&9or%Iy?9{MNay&Q_pr(|^<jEf^huht<-{F(AGGGqSz<_!GSk!fkimd=QIJBxO)L
zMFMj$Nz`+G;lb@VymqypnF-@me_I+;QmErA_VPTgj{tmbP0sb?!`5gl7~8p3!Mw|=
zzHfXNBsT+UDcQSMNl8iTDz58Y&8v5CWH1Lnd&xrQj=}suX>*jIqe(p8XQc#zM|DyQ
zuzVN7hw(46YWX01$d=7NTb60!lj8~EVsck{g5)WULWqlid*CElptI8gB%a}Q&v{PI
z^3-}NlKEK7a1RBRvF)*^g!8@G3d8_ETlty{e<2jN5|36xxGfF|>VO1qp*M%z=8f2H
z7C{0?&K-NL_#CAT=Qp!48g(=BYBjcUkqn3LOd7JeT0PZ6%{F;eGw$<|#AG{ypa=GN
zFHh^mz`Ii5BsOE+nc&VnZ#5BFP5k^9IR`I^<EZt{4^^nPGjSI__?$8zTR${*$6NgI
zR~%>a1f$_f=gq_H*k~n@m1nFVA=(u#z1*+sAE%7SN?Os#)8yPcurSHk%{#ot%jY0B
ztIR#yC6MKkZLczJwrG`Kt12I*PECHaJP|C>9o3zyF~@fv4)2Q?t|4$V$q{60Qd?Ne
z^O67S+BA%xyQH49)B8wPHqaWR`&*R>YiJaJ2~YrGiYomn3ZP@Lo9U$2$y7}I1D9e7
z1zBP=tBA-Dsju7zt(SPlLQd0$qxYqTQ=_<ZVihfBQpt<6r>f|kvrKEVIa={=xAZcp
z=k$T#Ogi-t3g_tqBz^d=TRf9IW*Ot-N~8OV3_rv6f|;G)Fv$$3-pO(uahEB3^dpoe
zXt<zGLqA`;^yM#G^@s27Jh!6qUVjj_CFIcQb*Q_a{y8VO52!<>FF+OWR}UYl%%}7^
zsg(5nZBMpM_9H*~3vz;QJEB?3%K&xsa^D5GDnGHyVPrdC$4a6ksjpBj<ds(2K&j;d
zg<Z^6j`DqL<hwiL&j7xw>g_btuzCNW%tp`pef<+&qc#12xXG*}>?^|ist4-w$?94b
zJ%$bh8$*hYQ5n0IF=q~p5s?ySyq_$3{#16<hUAz=T7}nyoMeMhcY~GXgA+}^;V>-r
zi_vZ)j2ObMH1mJ!+fWO(TS(gaS|@Rk+F{^(kUpm|d4>B2oy_ZNE9}HZq;b_N4HT7n
z4P3gMQFmWn;s`2=&<YM=SWm99dAap`HnS);=t>FAw-{#zX*cqvmlNJRQIc~GHY3%H
zcVCf`(87s~9%`=IxU8w)R7$#R6kF)R?NUi%WIijZW4AG@Zfyy#9caYCT5^BG&8A$3
zZM2zkrTOVMoQkh8Br#*QpUIsCbAtgR62xir;K--jwkd}ER)f_KPg2`w^Sjrb>sDG6
z4)zq<3><R|H+pk&REI5D)}jEY5&Bj)>B$#y;WYSDtApU5{b$_dH25!T_-)@#sT2BX
zJBOP^J*as#|BI>-X~vZjig1#18EH2gq{SbLkSPZsXPvuq)Wbl=A94AcC|X6&0Ttz^
zR>eLQ^ESkZErn#WZ%ohkmpo=ki~?zHV9}Rik;CnAwwaOlcVDGxm6%kr8O0VQh0KM$
zymk51=_cF&PWQ4j-U(%df$2+3jk8{MX{+`yCw;@B!UN>fx&v-T;Y5`1tL@}Y=AxI;
zkRY23fXW&g`qTUYNre}4JEcf6=ZY}a-a8CE=`&V$nGa0XmV$EtoqI$flqnpuOpCHZ
z1cyr9pVJE|(*DC`S3}6dGT`+K+W$r4hI(Q8qb79h(YAS(yc`QgpBXHsR8r>4|D=Xx
zYcXycuI85${fjs}-3*Yf7m|R554=nLeJbMrd^?>1lA2@Zp)Q&O`j4BCy?DUoF8u7U
z`)9a+eF|fjK*V@47yM728)b=&1FLbFHWNTw%zwPN5fXAOiN+~%KSg6`Ehq&lr7!Hn
zbM1ev$Zz5T0?sh{lKFoMV_6$)N+G4tRrGfL$BHCY!K}h<zeWDXRrK54@`1MX5>Q^0
zIosniz5^jay#r>|SoY`MzeMo=|MntjVGnx0x~w5rOzT#rQu=PN1xgu^#jn%|!Th;{
z;KMgD%^mJZydY{w4r@q0f46Jv+CNiP1j2I?V#GcZo(~g(n1(aDhupVEzD0cW)xv<{
zC7ocSE0}&nq1pa$lUy3VeoJ!S*+6)PeI9<nKPoJ%47UF^Y3)B-C|e-WGRaEhXp;1G
zzedXHZFHEYPI18noJKakE&jKtYFk;uo~FjifLH%4oe0!k?)N>n{bxVIE`N|~*-<C*
zc8d9`yCc}hoaoHO(M(LFmR1P=>+$&(7fS1!3ISR%#`u9?eYm@v(6)=Pdo9tDevzoO
zuB5Nv=x+jyl7)V(R)(&8LdvM$PT~C0gadjHP<3EK4?Op0MA4&#T4CyZ9VK#3i7&!;
z^C@1kpM37;9xuQAbUe6KdePOGHzEYR-`l}pzuWQYDbY>>u6q;VcqkvVpBgDUAzqM6
zM^7K{Bs`R>BxJIlJ$=Gv@$AO^d%&6tU&Oba?EwK^-+Q`QL)#!Rv7XNq!=CQS?-q%4
zA3s}}Mj38Byl;)(J|r20V>%2w)XvtB4^)B9VT+l_(Y!_{SHavE?3SEt1<8%iu`1f|
zEgH1th_M1U@av2G6k7Iv5x9Lf*zZlep8D$I?Ae63{8Nu}OFcrhKjU7I3W;bJEaxs-
zlk<QJF&i#=XM8pf8MxJ@DZ15O?S=xBuUpz-#KWtUw5xy3-trDUeX=w%I&rw4uBv_c
zU$18buMhunFZ1m6m+_=VBLWR9UoJF#x=cD2JJ^D-dsp<*-Aw202bVYUao_m@;l|7N
zpJtJu4f#U~5IOew=L~0yx_;@yw=3;%D{6}XLtCLg0mxy4y|N9z3VpsL^75fR3A&&0
zm6AGVx)lwiR~a1Te|BMsC{U<R{{5Sf)g0-Yq_DU|9HyaPeF|=AJ8w14BwPq-iO2Sg
z2wF>)3y}ll0;|134d>%~Lx-5*U6h<AST95>6=5a4>v29E=&Fna3W{$9Vd=VOgXzF;
zu*-c_n*8cA6;bw4GmHdw=xYn>6r(mz8ek;~8@Ak)kvyB81}HvO-ihQoyTDEqxSJXx
zb;R3jsXZ@?W9Gti%fJ$sjr-UB4`%f2#-bsM5S(|qik9292Fk@bK*}ojNgmGY6}+&B
zXUIyU2BP0QZ)~HHEpI&Tiks7&J&gxkphlJo!`TIT*xSM6sF>#X>i4CL1eiy4u6|m%
za<b`E)$a>=&ccS>bFeqeYK+9_moCHnVqRI&Ied52|EXd>9gCE6zBSQNNY5by$_T&F
zDK+(237D;z@JA<n^n<e6)T_OwbbmW5)7Xnq{atPF;Nj<}d1tzbqSB(eKWDc3-<H3v
zgmSSild@tx8YLt3o@zn-oPQj5(^XC)!{M)t{+ST#K!s~P{#`~YTANY6um_%_QPZlw
zj`r&Pmvk?|a%|3t{#5j*+Wp}#njwe24+(`Oy*c@rvkMz=s|Q3yoNs_3X%|`_jo#3B
zj?CFchF%NH*wq?W_+Y%J^aZW(nE$n#7cP9&-EZil`&vn1QqBeE380_o;BCmDac+H6
zedZc`Xn^66eYt-VJ%7I=cwRM2A0<snn4$Z!Z{7KkQ*%HbLAC03&0nf$ZIielWyMJU
ziDHj6^LGJ4*;$0zNKQ`GJ$Vw}3~{!<_n-}*RpCQlFysV2P;h()xuuh*eq#|Iv$ZWV
z+eZ=BIl(2qiYCu+wyPocN61Ne=KT+e0|mD#q!w+qf%M&`g24NP(}o;yTZU0E7!M|T
zd+pwU2R#$AvC$VkJd;@Octqr!l}tbI{_5{+gmO2fu2&YEWF~jS;S9?Bw-|h}-yfQ>
zcKEh_f(P7`l)y$r6h^|MV~ueyTt!#YaLOftr=@J_I=i4LMeX3p8&t8M`dw#y2BC}z
zH_2zBiR5!1@;`U>u0>FUbBgBKIJ;m&D&V#K;o0-&awR6MY4+m{m79T7uYwkOF7cPq
z**HJM!Bl?*>m&>3Q78OneJ%NH$EiSYTmStz-SSjn#*$&Zg$B`&zPxCBhD%Hb*kdmH
zKT_tuGC~>x`(k9QF8O-mH!#tpz;I+8>(a7)dW=Zbg3$q|TnDUW_)pha3q!dq%A}+b
zLa~69bokpsZVrll!PiB#S0@^c3Rl{8_kt;fZ2H7=`Dc<s);d_JUVI{%IgHcExmO*Q
z5*+raXX9(T<2m#;EOP=qprTN&e0uuAL&jymM#`F*W$my3Qd<2=tRxF4Ob9!>J{r;@
zqkH@+Xsza^KWcEa!Q9_e3=WqY5PFc59j}d6R7^skj~*C_OlKENaT7q5ZFghtCYA4g
z(U9-afrp&kTu(kd=#u%&w3yfaqy^^*X+i^TrBXqJC=008McNEk(pyAP(;1u@g#ImH
z5YL}Vhogt}`|(3|2h?)YeUA<u_BO3&0I7q2O+cB;n9IMG?sZ^@dx$A~RafNh<xqw<
z_Yc#n^zR+am$WzeV_$%>DY<ETyfY!#03j@07uSAsDg>WfA6yo+Yw+DbHmp*pFIt0N
zE)_)77#7&4d&i2upHbrx_LEy|<~i3Q$hl2^fD&n@sT!MJr5ea?BUt3w?w})RwF20@
zLFwSl(`5jT7E1prT^5*_H3}lmQ+X*bo!cLBY!{rkv?G5pXvZ{Fa%+3`n_qv9yzI9n
z=PgiDpb4Sij{<yNGbjnC>s7hb4p{F#N6(W52<7`zrQTDapoHNC26b#u0k+8K`@Kr1
znKY$jF(Cv#F0Oq!)&t*K#*R#J`)x6UYMNWVI=E~HMKzY7oc9JwDvLcfEX+r$E4<b_
zWI&1~z2`o1W(gx~;5`epvU}+Fm||P;#k1&^D~o?w1X-3)+Er1HkNWj-or*G46pRnf
zduFb?Bv-PRf5pLN8e&#Vj@SvMk$eIQMJhRJj9Q<pVk20zxY~HV&aiT|I*^o(%xjm^
zR0pYa!D_TtPy9TjD3BITPv*3_x#&R<v10CS@+WzlsXI-D5hz`DI=3C5Tj{`_9Oimh
z0WYXD0m^I&%R>J^)M_~}dc5?jPw0JUzpxJqpPitWfl_R;F<b^z*%$!35QaozOs$Kq
zh7IkAWj=^%6`XXpX2cBp)PVgqbUtWPLX`aj=yxbKZYG`S$&c4hN(`Iq3wnscMSEUU
zD+p^g=Q4%R7pQ3<HH`%g0QgrpBy2`&^IeP7m$;3aB7RP0@3Q^LEPh;dD3jB)ycv(R
z$V_XVw0n>=tUUhyjzp=S>||4<F(?g(iTfUUj(o>H3qK?v9_0%#eMEPC?B5#$h0pG0
z<kr$Cy9L=8n4%nLKF~EmloqB+B`5qTUafgImI9-sEAKkrXftb??9E_3_^CU-G;4~l
zr$A4*f6Ia9EDn$WTgVNDFQB)kj#vP&{7y5ScL7tK&Yu<o3Yi2jy^ELD+P8>y<P|hD
zjAm9PnQTH)R=n3NQt>P(crECFYl?)LN8Vf)O(=UZ_%6zh6a9ivE|9iHRM?Z9U3lyV
z=$M(QR-ovL?HqH1WrYOr;mv(PonWc`Py?$=rm92dZEHEH6|T#zt((6^N(5BdgBrUz
zsK>S>iyHvWkA3ZLYq(W^v7jyi|C!PzflAN%Ga+a6o*UrxS@CIoF!Tc+y|KT<G#${P
zTozroDnM(*@ZzOXakckK_gy?|9%gZ58D_zw6C!dlyp}(o-|hF?S)B|AHTx!`H=<EF
ztL5Zq$6*%$qMC3-Hk$L<*Ea;5r3Y4MvhSf*lGmynE+JtwsEWF`d~-3Xld^LEku^H%
zLP!wG6S;Lyq9H8%lZMeixxLW?VbR<W{Vr}ey0(LX$qX`9ZFeU#ic>MU)gbYLx;jTe
zr@=i?sar3ZJlv`~()!cy@U}`nB4#bOdZh0y0Z5e|+-dTi`N1lu@452lE)*N`v`VDA
z%0X#fy)#oOc|tVdEOv_kj(wWZeHM-QlSu=EgD;nQIgO$GAeUOhyo|)ef|?~t?$;R|
z_M2H<Rr$iTU@s$ihsnp*ib!QRTRoe8)FJ`Y8JU1IyXBwjRm_#BQYbw_hdvDmmGTZb
zk)Dcdh-Mk;MHpK``C+Z@ueCR)$%Uu6t^8Tc8u%@A!&U4iCs~hxS2F(yBBmtincDV%
zvBD8(o=A6J`%84`5)BB%VZc@^^VsmtGEx;l+r$v@{*!1CA)s!akpoxI%A->JL@rjq
z{(V=j2D=|~{#xWhgMQHV%%$-Meh+%gBbc>`5G=szTHW|kJlNjPy$0<mO}T}&OtV^{
zq8{s^AZE+i*?pK}N_&DfX%s|>p8rsOmU+s`jsfCux8HaC`<rNBCvtT@2WcRP5fKrE
z{hW;2Yol^O5HM{g6{*^l>&$ltOYk1fy88hlEBxClGWn5e_hA+O$TO{#1IY?Q`WHd;
z>!@)tH>}Ec)~4h&HDg8uA6k=g0-bG46m}(Y)mm0tcMj^)zHP-%sP(}<+g3&5#h<Xh
zjfppIFM5lO1B@^Mgx%HtTy4n`wAbvmDZ|c;4gh|X+L%2$f7mWW0a|w55u=;vXJ(Se
zz4|X-(g9_yf8lXevBHf{8Y?yQ^%in=>8(VellE_dTSsa<(*dXYK8Z{a^k6~WklrPD
z6X#qc+S-7>f{BX3dCo$i+!zQHon^KLHRR8<7X`hx?VyG-rDwt&GBUDV@5DxUu5DRa
z+O7NsDgTyeq2nYr08PEzSlvXwqYZqt{<g%@Y85}RH;()#8K(o7Acp$FKOn&#yUTGD
zYQG>CuuB;s;p5>^+ud3!jP`D*>8ujJdvGWoB*xuaCA@N!PuMi5;@{j>bl;%sFQ*08
zTL251KqFeZ*UoB`ImJbE)_w#zkvH}k(WeR>is#2`*&_=G@b%R;5O$9BSF`AG_N|Wk
zvVj})4l<%a6(2|HL#kbWfwV9zRHio&u4(xtDqp8;R_{5xZaMX@3<cs1T{(0!hXhIw
z*vMj7#`V)JUlzjL&@BUS;)jNTH$5nl5%H$x0zjnSOBg_L&G{P+*4(|8$YjdeZajXt
ziF-W`X5A&+Pk&c+NML-HJ@<h_XSN>r_?A&MV|q9v-%y6)6qusQz?OJ>dd*OtDnXhA
zq_|w5_)jkGQ~l&e=xwl|OjG!aXPTBST6mRF`#Qmi&uCq19=G7{v+ZnYjy0j?OX<`0
z@yeDE^Jcw9j~}mhI&1_tI!rb-f}ogY*75!|>6eYOKn2x!3GD6Bu7A52*=iCFI%m|x
zzb_0`IC}JF49xJlE)LZWEqvW0b(FcW#S<Rm*i2m4aK{wp$R80JKnCciQsvw*o2Fg^
zUshJ_j|dX{90e(TkD2r|+cB2^pW6KTxb4saD-^^zIS@zbK`R?n_!*rseChc(eA*~A
zWcb<M%f9!jx-aEz@o~b&{(Pa!vLj55!-#%3`Lg~El{g!Oy8r{P?y<GV`t&?u6ewe9
zsNIc4d$uXavmIik?44ZLu0laE1;8yX&rPewc>coRE1Y+<KUvKzvAAxC|4pW%Yl@*X
zt8&~ea7^02L2d1W%V&al&ct&dyj9kD7)YVx_WJtFuXO2<tHov=H=wAbs>g<X<`{~=
z?ipDwqbr2_($qI@++k2kVtIP*BGHEa0^U%auWw@vXX@w`fuWA9Tq5Uf?;tV3Z{RRu
zh!*lTwven{5luq|bu8uLOLbdi`Cp?xd)83r8H;|kW{!HUNq0sk8(IbL03XMGjVBJB
zhGY{fw;QW|9tE}0NvKX=g-OQzkj44=ZpZK^OJRBoUyY!hDbF|SLL535aoGg*VO;9V
zvIhkP>szC~(O^Z5Z)s(otWS4%Oj&63%I%<KNCjkg$l11z<`eKilMR9FR9oU*Y(GNL
z^-uzgWYAG)rE&}5=>AHIijskotoGg`+}FdTF>CI)Or09Al7}wy((KjR3%bJ@3KUwE
zPT4$Geaa@SarCj=du=T;?AM4+7lBlQ^5m}^ooKWsdv9#EI~P*_SO94t^&jy)@)o7#
z5D*Z^9u$bak2x4}K;#Wr&4fhuPdeYj+%;C&F31{a!q#}AQvj<80(fIpV?e=Gew&2#
z*>3rlml<z>cG7UrLf04IA)|q|yJ5(_Ni)Sxg=-fNnE?oBasVAofF9b;9Cc=LL5H-7
zrpRV%>*d0iR>0{FYjRcVKW4KI1r*HP_qGDkh=`7u6^*;VsqpY&WYc46F_CY1jw+&F
zI}6R^f;Kr8kXOEKj%IJ}%2w@fVK8Sy^NbQi;QexAcIb4cTEh@vY7erWg~2lUNG5ft
z&GR~_rY$Xm@_!m4rTsK`^O$6wN|BwwF1oA7UH^IRXp>=D3>T-3jwnnO#udSFC~^yP
z2JdQxK??t7Pr9|3>tX_r^?+JWp#eJ>wYPZyIJvn_Ax1PC@1+C)_*O&2b(&*ha;z6-
z+LrJZG>QI55Okz7sP*CkHBa~P5dQsvXKi~gB%GW+t}hH~*iyg0W4*DD0=*M^(o@su
z5>NSEFdA!%+DYRJ2CRTIFfj0E$sy?0qI>)Ht%oXNg9Q~#&Rtn)(IkV)hvh!Z@BHl|
zA%WE2)vxpMF6&W3Q<`HnKsF=7E+5eoJv0~t;lDH97s}K0y;knlSOIH%zYQ@~GH<R&
zHiS?#w<QXv8`Sx5*^Noy;y;lqZ6^?*JgGRN<iK!>{%KdxTU`e=IB&GnK`{fg2|HtK
zgN~Z=I-0IP<aszq2UsbEoI)W;w=hs@{JZ|S)w6Lw`;^mi!GC2fq~#tPIrx-9YUh0|
zW(u`o*REb=`uzDb4$YV=ti-fE%5~IRqu8{aO+Jd{Ehzf$C1O&Zq9nfw2v)QQ<)U>G
z2IGrY_}iAq44Sqg)~4F!fl6$cP**ri7E8>T!C*<meKnqC`b=K?+xFx@zpJ-KJd-T(
zdC}>aG4MvL!aO`^{_gPw(21oUKvB29`bsTZ<*qIMdl$t?2@F49Y|wpi1In#H<F{-$
zgTlo48}X?7v0R_gi~~*|7;E8jH9F)n;{lzJ23|Mc6!y0{T1oo#HAdk1KRi}7Cx%5K
zuJBq!j7X$`m|JYs&jq4fF#W(?bkPj81~wpx3QHT!f8*jvNlE<xAQMVHiO9*J){u_h
zFFY8iL0OiL74Qe*KP(Gt7<voz!e@SE%0mFN)~^KZpvb^*KAbI1rw(d~NVCSHbmZdp
z&v%lmo$~B63b%pL|ItfKDaCQqgaFR!?p$FrP?B_`@2^4PVBVRwVwanRcG@Bk9WLiR
zi{2Fh7)Td>VUF>SdeMUXqK6wc*Og5?bybWS3Cd3IQ&ZDHHyxMDyr%p9%J<2VcCl~(
zQyA(n&gy5$fU{c9{+-5sGGVt&<iLdC^v`!f+p6-udjR_IpTi{}*lZ){uz<5Lr0^jg
zPy}VwWq`6cJLn-V)x2PiE~)%q&I}jt>({T-3KAZh^NgU+DU{E8;78ykS~_62xccn?
z4uv}SfzD@n&<ykhbS!fjqA(6xT2Uxe1lf;7wD_EX`1JXo@LV}T|D%ZQnapH5fXaM!
zCdsn3OLLwD1js+fBk@XgT!&%4!W*h|mLGASq68N4T4Zm&q!_dVfY1?eS4X}Ct-1YR
z%r??=ky8_sRT%>X<}iRqo2%W|=zI_Np+f<H$Z!wavIE<ez`VTjooa4!3#fKy`$exm
zK{pe0Fw;-`5E4lb>9HUS&#7}9B`$ZIdhK&=4A2#HOiU1i@;;iu<VWb|Eg|_g2*7Rw
z;C*krA!HJGS*}O*aOg}zO-eTp`oND0Er6s~MHf#~D!(VVP+$DgxxAzYo2$+2b3fn7
zzP-^{84YS-C}&tSyFpYGt4<RzR*1jgGHSqoK(T3e8uO91pq9G(x4e`2WUVYTA8bd(
zX1cP&y}&^ZsF`1s7VuO9QDzh&$Cd^-nUOS^g&sYS^c<5cOzL=NsPZ-l=~UO$DxV#5
zN65@$yR^XlFn|Fic*G<ePV;>a{GUD*9p;&p!SIuU^I7##1JPCvRlBQ#)YQOpp;W41
z!tzw^%MZbLwv>VcC#5kYHuhrx2W`@7JT}6h(+{8p(|CLNH`FM9;Zqc--AH(0md3up
z!|>f-kr9n|ZHeIo2@=r}ml)cmL_?M18B~YW$s6(*J^-Et_u|D!2!sKEl3|Jkjth)f
zw5Ea5za1eMO$nq#_Y(vjK<y(SHwY40MqzE0zxst%zl%UsMmX+$Jl&iN7qZBuEY6Jq
zJ#o_H09llIPhRYEo%_E)p;iz!tK|nUdK5_D8iA3efqc2KBE%A{b@gB<((BjaUYtlT
z5*zddrJCYrn_^1K$jAflk#0L$8%85}j+sCH6AcoH1v^i0EfhjNfC^xQ;3EijvLFOu
z(c3Ye!d6X)mAaRDTj%5@%S3+Q(8WD~ekOmsrCn`eF8l%L;xxP8_4=J2aiH4%8Yin&
za-ht%(DVSbrLOJrf}Wt)mp_-DF4yo9*pXNG@YWON21NniTpZ5jM`%{L<VS$BGz<!H
zbfAS{^+m=nX*fbQ9R*YvI32dIn@ZOq;G_mRiX!P9gimMppTskg3=cHX-DSUk4;T&L
zy&C~BkOq7l(dEk%&uiW_0)L?5XW+AQ7l0ua)Apop3RtOSFI)Oud#cm%{`2SvvJ40K
zmmk1mf~Z5rt{(whpy~@kDmIW#E=RHbRD>h!icw&C6CwN~p@p@(_v#em`9e&e3iZeQ
zIgyuLBp_dMpD>c0I2U%ZzH0Z9XD?nfy`~hN0Dh*Y#247r)MU+jfH%kRF~PYa$pC^#
z05R`@P<(fRV;}T=38k7fp_wi+pn#Q#==Uci=d!dPU|r|Pk;y*0*_UrAOMuO!SBU4^
zXejsz8dvO!P?gSeP$$WImrKclju|75`o5{FpsR&xcR-b##O<{07|D*C<@;8n>L^D`
ztfQl&Ghc@v1RFy6Lof5tnF}yBgn;vG1dn;Al+W(W&4aBFfR^t8+Tnu5ds(!b1APXD
z7T6V3CyS**Yy)qyVUq%w?6&~+`~XLT(t)Ok{p79$p>GiZh@37O=oC>pXj3e9TY20V
z&7LfrlY!1U?Vy7^m@I53ehSqKKLn>bW`-(C^7N`6Hbt@KCSz%qm?#*`F&u!n%#WLe
zp=`|*0cc^Za=UlSTg3dIRWF`wY~y86%4fJ7ed1Ba<d2;i$P-@0)r2#uG8~|e2q=YQ
zm6b!;ox^LXkpTB>E(~TSla%`&`5a{Vfqubzz%0&d-$6H6AfWz|TAbw5ZFuAt6LTe&
z*D`8-aaadr#c8IC_!NBi9VUMK{!)_xYAj<mexN_`hmLJcx+!+;_McZ=u8X>jalE-z
z@wdHCZ36-DC&zUhs?#ym{Xiw+!5v9SV_<D6H6G5fJmz5$%o=SPRq}PfGs9q1Udw(y
zE@Pmtks6ptB<ShaOi@RdvyPztn9-ulspHXoZlt)Z&v)lWLu4fAM!f|>Vi-tQhm|~(
z_%BJIe!)QdeM$B1Y_7xMzWamEhQKw74$}cA3+*M!7DS)}m+6_~rWa)41#s*}hv9w<
zHH7ephE6o)4WRKydaD$V!2OS5P&oeE8sA(t>cb4VEP#QQw{E-&xzu_Lp_U1OA2_;V
z>T%qlGiC}i@LG8e@wcz9i!xoL;EPxp|1J+&==SgXE#5#PBjH`Xp;vg!LPsPfeg{xR
z0Ul4T=<&C_E~$aj#`1PRMegn+e%(ISMPH(AaMCOtbb%Fv6C@9g-sdMBtBUW96ma;?
z(EgYoc}2<!g-?_AB-X;})tBFN?ysFXIZ80OQF3{Y<FUuSL)y~AZQZq2#%0>Z0<ymd
z2*mpP`|oQ$FQ7#t)v*;51l}c6lv@w-&2*#&;8Teu)EQ_we=s}dkC93M*Y0-Nb&{XV
zjaXTJ!K5vLvBq;N3iJZXjCjnk9^6L(H?v29eE^i5ofxDFp#Z@{Y0`$`Uog=y5%*O$
zEp{`u6MY_mNKw7<yOt-q^5KHjz(+ZDtdR=G%&x2llI&PM6K`ear`#d<>h=@`#R^cB
z2?MIGRqa+Ja<Kku<fqj@;7Od&fMVL@hYquo>5)QqzCPct+Eph)#-gbV&U>&H)@;U!
zhyQeiBbFWf#1FS6uLAx&fRHNc60J0Ju&t(AhUe7kEx~Boj=E+fCo^@$g>+@<P#C1J
zMg3O01`*w1V}L>!b!IhyAjAlR9cU^yAN-W>wb^glnQ|KoGNUJrVc!NTZ+o6D?;~)c
z=>Y)^C+53ch<ZuFoCZ|h0UTx#wKUn^Uh&|g76XcoEX0Lktu3`mmNE^5E|z~_8Bjdh
zJ%;gPa$sAPW>xu5X10Y;Zk`l!o~0Z0-by@-{tYS;GgFu&<9v!xu-9jWB0GS_LAsSr
za$@_-lIGnxks4b84JS~*@IOls)KSO*Ot+sYx1qzJjR}G!K1XfBOTeQj=jFOVeRSIe
zD8P?G0BTqKSr%o-moH!F0nD!*eB^N&t|`Eg?P277k`r%0r$NoYAT72tK@6Pmn=B6H
zg)~8M?ywQKB8OLFQ0)d~3hx>v2KbpBK-_3x{68RS5Fj_PU*bZiR#64)c9>Q1)Wgu-
z_+b8v0YWQ48D=KyEH;}#YLW1x+Y7=++iy^SYM2C}>?dWCuxml}a{YzFU+x|!b#){$
z7%DMP?!n3Q(xf22-C1I)1o_&mfN?+*Xp2)Y(;M3p0`ZRJ!YoSCpga`BIjMyhF}G#I
z>(OJmPZfaY0xm_D0q8H6_9;J3ZSFT%X4?$T(X5B@g9!NO(W6d4h07(2d3W+VK&<?-
zs-Ljy;&kjU5bdJ1OD$%OkYU2EHsakWgD3M5fQo<R<gO>EG}OWmoaohsN|=A~$w+WW
zSu{DiCsiPIA0vcWxZ8n@h8}d;L&s9HRnlLIJH?CLKu+YjMPgv^-h=)Y0Ca1p1w=m+
zA3p{(qLFMHl;e(liqC<EJ20?ywiwW|tmK1v(q@=~y*b}c8*-I544gM?0?uy|fO;8p
z6?+jhOSlnIA1l{!s4vA@2K5C{Wn&Bj=f22+b4zN4kV^0$MyOP?Ebj*nIy8avOY5m5
z;3V*VCldW^ATS47?bD-S$fqz6tyddwCd&eOL5F!)K`o*gB!3$YeCJ>NIcq69_T>Mk
ztCffh9I~NWms|knlvzRk-_?u+<WT(f#e~}^GWZR$?K|w#Sdi4j{6b9@b6>>)wOfK>
z!Qo8|NJL0v3ASRA2qRR60KL0%z+*oFpm}vRw@|3+YLg@?4nWeLYh>D>7~BH_(#OTV
zpImS+(4F`5{P{?>V|#X!9s|J^N+8KOA3w8^0C*W9s(><2#h!b>4Ev`<hU5oM-eYaL
z!v$Wg2!@z@5q$308Z{png6daisilUX&whE*0XPFIz^I&pB#FNlkFbNfA1%EZDKX*y
zFg%8jfG@Z}NT6`ciF~L<0lP|r!o{O6=NL5#5_|~8ekqAtAH4W=2d)b_TPCe0;F%4e
zpxXU5-2g{Z1xz69%bnC?ea>TpTFxOt-wO!zct1tlK76zt)rVnW<&QWG1pKQ(XUr1_
zrItTq9S8FQR6ujHGrM{$%MCclrY9llq1Jn^X|g%md7J;q4@4u_k&UbJuMo@6z)Eg{
z<Qo%Stp#2mFZ1RT?bv_huP9aM+)V^HXSuUbHkt;ip<L#jbc{Nm@-pKjYj8AG!DDgb
zcL2qLs)V+Nl0y3hk`TD;-LaHT`r`mFg_j^;f5&5eng*Q1iUO=wa}1{u)LGJ&|Ev?S
z4Ea3+a=Bg5xAIRj3KA&ykM@%?3dqW!&Io|JBcbRZA%P^o0!aWPz_aob1hs%xpuw4t
z0S1L50C|BJG7Va!L=JF%s^7l19$RxQ(+9wsK=qny;y$`@Jm&AQKEtr_wk=#|$Y5=t
z;n^vPo*v{cz7SdZUEr5IoCTEE%-;H?kRLzagXk+UeFL@-5WDO^jM2->cG(Z}<9M&i
zFe8?sR&3iV^icGB2pwtfH2X;k_tFP~nF@#@I4%v*$&7)?LUci_kaNxmrW@d8Aghu$
zG&FRunF0q|Ie-x;;6jaC|6;9Bj6kG{kJ1pwa<nB0eEm8K^j^q<28~JJT06m6<p(r|
zpRD?;OXWXOV75bXw<mc3z^^^9{2JR9GftR3)GhJtsm!nA(5G@UCF;m!wo48=ft{^a
z_33^bj~wtA;gxqPC@_6NzwG*!0mcD9X)<6!lqBOGAZY^aZiY<D<;T5=C}9Yj&;XAL
z`8r0Gv_L>PN`vS<k(l_d8%GoB{ji|D;|cU(7EJyj6%A0;|G+DLY0!42OV&YnA8aOs
z3?SuzwsV%R4QvA~c<{BEJuA>*>N|)Zg+L94phg&JtDHNKN-QU!z`g0<?<Kr9!Kwv+
z4d9q88uL3Z(o1b2jnV)!_jY~)wic%x>ONI3jsO7@7wjzqvR>mvA!t*Qj>rI!2<bM6
zbWpdCTE2F-tB|Y`;zJy;if-v=o4^}&gR%0(?tCP~{7=#;0|m;M#wa8vm-PTo4c0PX
zP9Q);bmfYfemy8JmV->_CrM5OK%=$L1~jM2k0L%KfPp7pJUe&nrqZOTg@m*-l#<!N
zNy%`?H486Q(?Urzs9k4dXaY9HZvyNngT~7_;1d9+2>yl#J(v>(_%Ix(pnklP0updM
z>!}miQ7Be8&vw0q8X<rz^>i51fgR|AkW+r$3zFL+a64}7G;IK2{zvRhx+DOc9Mm5G
z^+?=(dAWna57ZNWa)-Qo2;bKO3mNsrTnA|E8gN!lYQb}tB&bQi7G<NxA08`a`!R(D
z!8(uxbhaPP^_OIVvWH|Rpu_uLJP?K*LLz8SRT>3P(t{@%AB+QrM)n=hE4<(37r~ZF
zph{pCn+5>+O+tT*0q}`)T1bFzwyaXKfS;8DODBOs@oRv5h5ml)2EmjG3Z^Ue+{d1-
z3fpCABtIDogna32<#+xN#RiJa!^8Nqk{Etpfi)0^|Mmp$p$2{$^-OOYfB;?S<O9(|
zyyI-p?h~}5P+Qe%hYS@YOk;2YXSGMic830QTfwVU@aR8_UX}+oUw6$#C<K`a#ulYl
zsyW_`Mj~LMJ--1@elP*5jH`=POHC=apVC?O<TX|%L-f}_y)w%lV2nblJ3>e#KUkH=
zFS8CvoTzp<C@2(0V?uwsVigeQKY&2eK_=sqA>n#y8T64zuv2iv0gb*6S?f09CkA{U
z;#88Q<7**jqkqHh|6%Vf!>Vk%bzzuJaRLfc5EMaDq(kWrMG&N0T0}ymTT(?71PmG^
zq`N~J6%Zt(BqXJzyK9fz=UMAp>-hFQ_OJbKy|2$7VBYh(M#nkE4N)vUAt3_Z_`9Xx
zHZANuS$+4j=PvqCAO5n_2geCvCm{BmqFzI%^uI4KVIo9|U;nc69W(&vAk8B}Bh8Je
z7-pjX7HrWV{;SI`Rqwxg0vy~rQXQyFIjxKuAkG}>2jR%VisQq*qYghJhVd<s0Yi!B
zPKaV7qVdo!>*jw0sE^zGAra*)&N0-g*I^xlJffhPYK@aiA8~>A(5b7x@Yx<<BD;t)
z9wQRFB3n-iPmNbehaw6@grD5jK2`$>J!Hp`MHUdfOam$)tiE2-31}P(DpcdXX#r&)
zH$+oO=X+lPObx96(vKc{gbO?<RaI5tLv^}wsOY3X3FU^x4_c&gXny%tF2paOSCX?H
zYT##gVX7o?CG@)3t$JxV0r};#F-WkyTUifDT{`d)g0^)8g9m(^P70u2-?HhEKp!f=
z*VZ+~j?()|&ckWcZRPOcV%Bi62kQ2Jfk~3`ry<x)6F?z(#|1X6yG<ci*dVc#mk*kR
zwpu|Ed~djH08)7S|G-fR2tx*wQcXb~eP>T2Gf+ro-jyt?0M!VjnsDYR<P2M$lNPvO
zrf$P2`0~OaB(FtAXR1sUI<fx^dR|Lg0Ckir`>?}BK8EknHP}7z7SIBF()~yrcNI)Z
zR2UJ0v-d+%gBAV>N~jV=!^yr|upkBSRb_;ecK}(5WRyrZ8bJpkeHtdbn%ng>@S!97
z-Z{|X6gUevpT{srHW6?_7}GTUSEuj^^(*Y8dX?&tP#u~it-#Av=x~FrjaohD34dOT
zG#Fwh<@14WNbr(ZoC?5|3ZU>RCrRz!LYLhwTFA^{pGVrwda(I&fZcfjcpn%7CYKGI
zxCiI_zl9;RjVL6qLo6-8LWwi=%JY!S5O7Z6zt=0E*y@QCm(Jk5<pjd(!L>$O70oca
zRX~cm6Vx9GFjH?(eh|pd2TMXlm$NaD>^;OlEr2_Xd-|>dAi|zV2s-;Bj0hMS1P?Gc
zdv^^(iv=hwcNyNwU-$j)f{KEVZ1Ck&R~#ZDq6nz}E5JsJtAKpCGunIyZovRzDRJ4l
z5*Vt4EH@%x@JG5TV58p8enmWn?-(Kby%IUtlr#lNb9as|4+Vf2r@lvat{A}DGCwz<
z-<squlF1JPpm>#9G7bVPbA%!4?~4%4qPZ{tRRN^r7%Fkhj4U>SXvlp&Gu^Ww-3icn
z{|=NHu{@Y0eqz?dMESZ|ZkX%YzJP;#`Omyx3{^fN_S(As1k+*Klw$V%JDz^N({Ni3
ziCer_TqC5lliHQSg&dXiG$$ZO5buZtJsojEX5j0(%O30DV=#92IW9mBAdZX-zoip%
z+%BRLaWU-wz-jPOgV{(>NrayM<5d3@*`+bDbvvn+sIIM0=O`ZA_a#;$irp#r5hDGw
z0}IOSv97`$VU|60RoO4i=SA7ay!#gNR!H}yb(!Bkxk4PtLAx^&xoj9|Pwm$WHae{%
zv<iYNIv{dDMa5t7_VUZfU`B$v&6P2uULIUz+YeIEfm!7P@7Xy}<P?Ymxwl@hFq=&M
z;YxL_YuxWBvAj9e>QB+Mp?7nUF*DkoZ~knl&#{S@Po`FB%29FlXazlHxO&>xdT-9R
zrX?#x;#ib11nqY^=tXOb`o$%)tJ^a{(<4)<7pWB&dB+&WsSsl%s73$^2|~}UdsiRn
zckBlIE!IL3vOb&OBM*bMX1~9_La?YOe?)5l4>t=Tk#KWYgW7&|ozCso``zCCY3Gho
zt-?>9iaRSAm{Mog{^8t8$Ts$N8q8*UTE83o_?Jy(TU^a`87bq@YF$R7QR}s!5+B3r
zaWNLHP45HqQQC15I7?)K8Z(>)OnX!T;TYnSyczALzMY?gA#qO_R#mHZ`=`IU*{$Gh
zCTR{d{ilcQloLgiR6PB~^3jsOIhysD&E>tZv1TJ{ZY{0cQA-I4ADc5Ux@~q~u3b~;
zf!*~U!@J=oX{zm<;Ba0JCLoR1<0_!1^<PICtJNhrj;;XC00~p2rKJ&QFj9%0)K#*q
zF1}d;gCn9ap7-bNb+_a`jXkG^(05`7UJi5J&hE#b){FLoxS!}b%y{X+q&uhk`h)P|
zrVVtu_mA5{<=KxVY-?Wy21b-Fct;+c>G)&*2T}vg`CiuS6`uS%i_V=ZOw-Nn2h;1*
zv1(<Dr_}aZ&Y1OSi@e`@UBA$8`@Mf`G2%<WNd%OiFBKoO%KM#hxT>WYx1!F&@Z&8l
zx3F-N&O1+)euE^>q^kHU8N(;L8zqHTMRch1HQ1FXO??C%HscPDn^^{{c1kj4g~WB}
z^TRkl7!i=QQ5+PZ$6&{k=1=c(SqWk&go~<GJ*1cvs*NV8*79;+{h@1Goj=f-ZB7lC
z5h-Y(QJeEU)Xp0XqxW~5GZx%-a5zw`wyy!xwGf;M(+rDX6kG^KkpS$NvA}LZ8i_9I
zj`tN}1Z+9Mq(wk;Y5yXs1|q3hNI-?hcqtfbd+p+K^h<AEu2s7ekw1|4B|gYJ9r7Jt
zDnYl~@)2{Ksq!;D6MbBD)NkK@F^r5i;c8;pD;Hy$O2wO;+I>Z6U#m^&b(Uf`YxB>s
zbli%mkK0a-uE!~2GVbUA0>(#u%p3}lwh9i#*uGJLv3NSzSn{P>vo3<=$Tvs7%OF6*
zUg_(>cpmNQe4pMs#@X1)?LMOyEgM^{&6X{#eIg|Z`|JA>JS#)5rw{kCy9;}$bc>uG
z#T~Q}=T~c};t^Z3v*sxLiQ=AnH22Hr)o>-5i9{~{s#>SVVYgd^Nd!B^;#jQhP&J>y
z!}dW7?=b%am#}$diK|;iml;G{uN3>_j~;qiWu5=LE;=sAyu1$r#%VbZg=#RVZxX=q
z(n5tAOvM|>s|TThqmiX?2f&BZP;}YkIb8`pjT6aADITkx&JLFBeL*|fT=FCK#qcLm
zmMWhsbXKn8gN{CLe%g9pwJpufG|$Nl=HK7+(0y^v=02BN`@`(r2NFV7i=sKG>Wd57
z+)s0yD({w_4<Lw(@cF5$p+k1E>o$Yuo_*B3zffdGAU9%8^RxIz?Y-jf)XR)zk5)rI
zG^bbvO9>CFb^gva$bOxn`^qTiP_xrm!(?1J7Vl2l-B&Ba3mmY>LsUdiP%hr{V}(YR
zjz(<eVB|vWfk9`zhTo$)@%*n$KPPM^`>)MKds@l<X;lvY!NV2lRjX(-782&+{Jn*h
zf%hB%!E47ArsCR2MTzAe9*Vw2NitsVvQ2YU9^AgXuEwAdE^rq~E3Yc9_v>82UYPY3
zuRvm5#UC0_fraGm&DdXJGNF)tf!;^^EzLAO?9lXnhhSJ?xzuLe@E$(y2pBt8Fm{zU
z_ws~UHRc(rTxcTa)W^d12n7~!R-WC-P5C0yDOcqx9zl^{{pM5g=ym?SUBlY33T>bK
zj#$~6i?>?@#*VMSs)_e2{af!znu^q&GbGpg(k490k_aZfe_dB2xY%konr2R^mC>sW
zQCD&#1D8?vV69#u0og+D6;3M_YiX&(ds+Im_icro_uo|;qeFvc1LR+vJrR%j{k0`%
z-Ko97!{vUgR^Rmdxur$tCZEZ3Qsd!l0*tQ5u1*8}LYqR-++{LxHW|lRg>%0a!mB=4
zNCY!}4b}EGvKg*mSaIg&U%$NbB7QEDKt*c4rxb6&F<F*3yub#(bBNdH6f3iaLfnG*
z!OOP2lp1~hA<Zmn*DC~By059I5ub|{?W2d09YCfDEpLKRJd{HMCZtSF0ej-sxCjKM
zYIa6@R(Q-G?uF7Eo&Cd#T1G@P!8s=;Fnsajsq(?w6^@5r-6^;&qN_)zSZXemSntI*
z-K|%5*c`uQIcAbZbU12ZsCnjD7G~>io%N@&?o`GL*D@&{v9hZ*w^1ymY@K8+?9BT#
zDIdyLZbjuwtc>v7=8`B&;2n)BK2J&j9iUPA_R*-13GZDk#?URc&)*ah^TjH1P-u1L
zi#QnHUsSgrO}~cmoNpmfOI?3v-^wGg$t_)~|LOSPq_vm*oVe$dEsJ4RHmAp8#!Pqa
z_l<{qJDyj#-J;J@SL#dftkow>mS``=e)iyaQ#h&p-Eph6McZimrC00mD6BTRI3N97
zr0iVJM1oM+y!&gWkmtYhXM>SMV&trnGiJ9Otffz=L=BWqXf7h%XNVMA+-Ua<Ghw^T
z-$*fneQvTQ?Fs0<D)`9I)!0}+cHK*??!o7zvM7q@sJ|r`Y%fwV4zAdyk7XyOOHJYz
z{r+wQUY_PXSHuzJ-J=4_1dV*LTjd9<$M|X`a?4fw#xGtO2#cjh(!^ej>P%Euc+;mg
zwJWPuh3C2=WYna%E1#vs!V=5B8S0ZAe@>^ujp^uUdV`0=6E9vkI)-*NPu|q4aAl`Z
zZN1?n{zGgGtqPgm@1#73Zv$s##LCxl%Suzsg0f(Gl4N_(t1v5MW7}W5snyD~CtToH
zZx}_-vDa;rMZ(^@N_@#Ywxc=g{c9|R%7bIgM5sugpE8%L4_8QR$xzu8DX?V5={sll
zHs0E`JaE(c#NrU+Dj+d#s{%4_?`w_@p%VJ&dSU<zKre6^>NkbbaB<aM+DTfCx1=bA
zh8@z5^LX&G=YciL+qYds4>grDEv<jN<!}|L(brxgI#g<o7M_ZhD1YXyl^Bz%oxFdu
z$mFra$j=t_+P6PA9k&jC^*{U-x!trZH?q$#{j2oJCePu|+&NPT&AH4SnaAR4Cwm0@
z6wceECnwi3X$xj}Zf@N$E0t<=Y~oq|P1DuIUqf27e;lxNw3T*gEnhW3ofQ2u(?Dxp
zd_zvUG4S5IocKr!>P%QUH$If2TLE)6K2^I}2626D?OGfO5e}gjc(VgHV#<xr)bteJ
z%gc>Fcj=_ULUhc&Z~nvnf@6AmQ*@(+;UhdhR(4T)LH%Aw0wMORP=ZWqNM!jh872;{
z1PPJ>yuL&S<xsx(=CIqa^N|5EtiF|_mwtIbC?-TA^YUHZw!cX&&2!yo!o97tnHbgZ
z;ht7@d|x`MVSC9+Vb*5jLH88jNB%gK?>GCpl6Ryjcx=Lna5S+|Ki1|K-fu3~5r?;O
z`5f~PdEbdgt;`!|yZ5lJ&&9bef!}*afC>ux!`>W{Ai$-#uF2u9sbEAZ3=7<9$IAWA
zQvVQ4y1%l|St^yts^af!)w1!qQKDtVC+@X+z)-mE(o7Ej!|TMQ=jHa>a@#_8JQb~P
zr^7n6H*QDeUMspp7b}oTbo;nQfUghme1B)R2kJJfbbhA^$>k92C<1&U=eYMRL)9w!
zhmS&xXGXRYWKTIrAz`~qb>bbtRBsahhFgU-E``v(Ld}zc!Kg9UV3N{P`Fqtur8ifS
zye7Bi2C9y;&N1}b6u|<y?f4<S-HlX(#q{rDKlQwSIZChGJJ@QiN>GM<IJ<`m$<qsb
zRnvvCK`%XQ4Ao@!9Q>i6a~9w5x1yBsB2k8~ON+Ul*}Uw+<_!yH8+kW2_wpm2Laz58
ziCuo0<lXC_94a)7b6k_ipm+v`V~rRBX?pU5yI6}t+$15)ZW$O3+fc3=XdFmjcVWpG
zBS6KVrb`bf)psrLnZA=Mi%y&oOAmrRGtzJ@C;smD%sk~mw2#1s3gy!b1F!PO670$v
zdpn<=2ssQh&7Q=hna4!OKACEZzT=aAneyP?R-D9Eb=|IP(*2~5E@f9%y>C+NdNUl%
z?Q?~`&95Cz6-XR7Y1TQ|nTbt#TJL>I{JjZ-*3~_0k>a=%!x}ye?}5|AXFeA5#rVU1
zqmyqzT%xQ-oGTCGyYqvMVy}J|A4`k$On2g^kus)ITkwJk_-ND*dbFnw@G;;_WFDxx
zEn%F5kKz9I5?g}j`l~h3MEx7cC<LS=af~vx9Fnjgt^OT29~&C-t=HmPIpWUxf*Wr$
z#<_%rZ?9imX70}$iI@#$EKT6pckKy|E*6+KpKly(+?i>|w1m|JfBia;@}z(N#={52
zf?h9cqU9EYul35*9B}+BChmJu+N;AM6yg;{XRx^6w((utI`X#S@yI#1rT!Wjv3xGZ
z015}Oy<7HsmOY6N$W#j~GU7y?RjG6@Q$4}ouIOq{eKb2e+1tmrs2&aB;E{ne51Z|9
z#rtt<1?Oi&IWj`chDDFb!M9xOE<YiG7Gv<}8yPXSA6;K~?^_lmJJ?jl|Nr9^Go-qN
zbawwbSK8fJ4}N3V$i)Z?-wc0}BE>y%q^7ZLa%6Kvqn18uXRz{N!I4D&vK7C4JG(kh
zTd)1Xy!D+(&RcZ?&*;hcISg0ht94FPkB-(zFQynCsTSFu7<upnaY&qfcb202kK?>g
z@7{gJsz2j}YK<S)m$YlPNJd3m>7CtDT~2GsLS#bF6rabZ<FY7vE&4_(kIVJtG=b}?
zo!``xram|7T|Ine>-40Tz4>bhae8hzq-a|{m|vf~D!h7v)zi7=u7o{%idw1UV+57|
z>Ur&cX%?fiQFwxf%XmHA_eHwIN=|QVg~Jw+d+y-WiRp?`a&lIWXvm3!9M##c;v<Vc
zUO)AH$rk8KC|p7NvWh2}nU>WeNNq68n*GkVzyS8sRLVQdw7=Sd;wc6nbj3$sYz^s%
zo5PjHh{c~LLD4I8oTYnncX^5QtF=yemY(b0&mbjL=am!j&e_a^&qiZe*}5yQ)rS_%
zIepp(xw#+SILfW#vE6?6VXr?X_BgKa>vVHh6WeLo5drO{vE#bsw027KSfdSoe(pGl
z!zG$c<%H#kHIYEDv9;A>;mR>H3Rt()z~FdN^pM4+u&2HUh{2UG6^0n|BEI#nGDUoR
zd||tBUvOdh-Q8;EC~ka}xzY`}xggRKT0eihQCZ`0Y#XR|?pMskjYgW|jOJ@ov0OAK
z%Qq=p7x}0G=+bK>7L$BM<a@ghWgVT{h;x68KneHqZR=q%*Od+`3IQ6+V>bbB{|N8h
zPkTk@|4`J)>?Zge(ooQ8*VWeXp33?8RMIq{HLE9>u3vOxUUYJ-Njsv-HR#dsY?wP=
z^4d_gdXq(CbKUj$vi%G}2mSJM%Hj5n@oU^R77p_Pmp7WIDEn*DUy;5jAK)6GEhshI
z@j7DS`C7FVaU5xj(=CeOZ9Dg5(ggkC#bkn4T~A|BWLtkQwsSYQ`RD7CBZ=C+sKUXl
zOd(sFU)lX!y}9<+WREpCCSTAuF-XkHq;_}@d5&&Mu*~`7kDW}~@D%Xnk?=kJUhCy;
zSzA+J{LawL=0Q)|VQ>lr3Rky;_yLs-@#*k4OQ^TRF{`MmCQVH}00fW{v`Vz>?F+9d
z#r^rJl>y_4!_c$0*n8f8{Kci_hI8g#xbb9m!;;wcqS(i`!P=1&p(o<{Sqmpc2h;7r
zD$4>Z>^(1njk1%HCe|04`ZX^`R%acoJUTjYyHWNba_h-PpdiB`4|P+x|83u$pRZ+k
z592poZnmBboqTW?m`qE|nX9i`-eKFwZndr3vZ&G)y8O#_%wEn%q{a3{h~hzW)3MBC
ze;iY3#omiGZ*}az5Sg3eLHYL1=G;l&0*^NvgXd$9OB2(Z{muM_b2}y8U0#)Y4jZ&T
zxbKzDd~X`NrJuh3MN?vpFDKq-o0@D!`Np%JH0Ripqkf-X`8(?Utr+Tj&r1!nahv9R
zbB#~*4n@D|@ksn!5#Ntw6R0~JuJ~STUv5s#52IWq4OnJe+!aWGJj{POL1~caLPln$
zs<(G7_|e{78ma!MI{c`pC<}2{W05eJeldg(IUR-t$7)mM3ECK1@*7KrCOr3F;xpH`
zFFUDM-H@-ZdNF3#x0<(iudmO%oo)Cv8`hS3sJ~jcV2@w>pnOep`b$7)thVrFZ9c<2
zzM1SL%5MkFtUfC4?%}W5HeQVO3cN9BXS+zuxBO&YBiN?8P<yi9%cqaN{*kTJWcj(e
zqwSQB$s(2m?MH2Fgui~p?@%F2|9$CH=~2r_U9*#Y4h@ZntH%05$k2LwyqI2icz6tq
z91{={_Eou#OJJAj$?rkN&4d)WOyql^qUnNkXM1=vPUH4NWKrq;l^1@fL^@Dzo8W%x
z4UBL+g8iH82S?D&4C@(6G=#7t3!JkylkynUVRJwp3mg_<C4m$?63?%R69QLBL3CDj
zc6<IYCv?Qu*c8RWBO}wHzJ7YsImxcKF$Vb)NpNhq_Jznm80}L@aWwdG=zJym4^Sca
zYYWB1ygVhq9%(*&_;7Ls?<m2Y4*~05xOlODXr>y$@(du`5JoqW0Hjlp0G~X1C@q7j
zK-_ENmsCFZVO>3J!VZ7)CJ|J^>hp7aRu-17H*~pU;a$fgDo}!?{Q>RMZ_P;W6mu5}
zgxp+Zw{iT4prYyWGV}w|a~?;tva*JSg{8or#@GfUP^);i->=5WO}UU}h!Rr&iNPyH
zYO+vD{=m&>b*r8aewatW82tB#R#sMRW5;Z)tlfP5`pK}(HcA`Tg}EaYqdY{I8ZTL*
zaMA35nk>P_-lX%zk)ln2(*Wqu@5KweTo~46VPVP2$>{}vnx=|MB2>2Z?3{>@T^2fU
zb_+WG0BM|mwu;<;&Fkl@jvV6#%RCs}W&+ss*<X$O4VopqfN$PA@(!jF>MlG|LBNGS
zkbI*`v>Jg0mcU7@r9Kk^GsSVA)~^m0-GKg&eqv?iy}7wLm|BbW*qTp0JoKyq*kWK{
z;CjW`C1{prBYBFLVGJ}9xq<U9aKjnTOD1x_6UrXAL30v|QTxln2zJ$n57RmlB=UE6
z-P%8hXxrP{OXYpDLGqhtaNaAt?D9|{(}QOW-9LN5&sQ6MSaZX2A1<MQhKTR}=wWhB
zPFpph-JLtH@QA6cTMC6Ra0g>T2>bM2`_Tivq=4!~m&sG<$6*ng<ZR~~BKRR53ZQdZ
zYHIHQy`D&S^Tjs|4%6Kz<7cg)_8=aJq};b70?7?Q7GfP@EI}=R`NEd>dchi$ks{W>
z#wHK0+zYFI+oO4!tLeF~U2AeFRYP{C7{3CcV{D`Pt8t))A5FlWZNsBVPF`nVanaVy
zKYsj31mqGMBcqI_rsj;IBmn_ITkO+;0)2U48#)Ipcu7YL14M!%aOaWhvXxLpM{YuK
z!3=U1FErA;PUuXBIRh)`9D@#4!pk2P)oH^K33>}+^EySm85$8xNF&1Ds`s_8DRAYA
zh%f=<+u5gKuV>!orW1!ztv-MtwZO8aDSgO-%6zuMgd0MCFZk;v5lU7FvY)^sh8#IH
z{QpTng>{Ccar}PZ_xAN!uKjLoZEfv_zH|E+4y^CU1Hjki%4Y>6K!kB}B=)}hP8{eB
zt!K&X6d{-*;f05-Uq@O@L+24BHVG-Irj}OPD^jKy*HhHI7Ahi7R>_f1+rvjfeQ+$G
z?}~!H>)EK+SD^3e20}ExGywP;eRdT?!@QJ~tB;2(R0T$y-c#VfrA8aC^dL|kEIs@o
zR;uVM$Scya<4I0aMK6L$v<P5Pfj->8?Qs|_ZEbJY1kDw8U1pzTt6y7N3w-seP%-!d
z(vBRs2Os-6nIs7EfLNb%0n&Fc725E_n&MO*kYUNhq$DYr8B?>e%7XvFU>onoLs}SY
zATTE5|Jx$RO3}jTBAl8V4pUz}097Jk$WaIA#9e?h0+_|o8a?p@Wq7MX_n)rhcFJck
z3;Ow7ZWn_0GDeEYTQgM9CyPUtb1m+vilK;*0@6S|Gho@Jne1TZDm69L68d8EBUR-f
zK)eti%gD(1FVp5A?F=^rTJnfA-3X2SlL1OCdwLKdCg}88oFGMLAzVQ@J3E8uw03m-
zL|9mb>Wt=*g}BXG_L&o6Np1neiu>I@6(tf#z#+b7s`8*h5KzP7A7+?^d-@LxNl2Xj
z{Gbzqof<QWQUEi*(5ANd7w`E4fFZ_@SpVAO61c@c*|`<4@83YSZYs4tg<Hz{t*<+5
zwAMk#+Y$zn5IYR$E1PDuOk{0^9g^x5&&qE7S9%a5g$=jek`UjEXh7(u$Hh_c>{bD&
z1!jcxH=1a#U%zf>kwA^K{}$kjas_5zf;d5+Skk;RbpE_u6g=|_-tA8q+SLv8<;#~#
z9H!+{)6%lQs6oeG9aI5^nf5lwjsMQmfT72ro;f0BeHAQYqN};`Eanz;F1b;Es9<pD
zGh4xv9UFMlfI`z4Y<TIfzK+a4eSV^%1$u_Y?iM448~uM20ug)-O-&XK4n-u*gUUqm
z)2ArKcu`%*c3fEB-$UAmu88Y>QtJfPC<#o+*%}r$Xo1IqF?ig)iU(1$%s4>AW;A^U
z2M2G&GhMsJ%D`|FVWK=?FJi<iWI+tXic?5IkP%h5Y#svgod)NkfpgiF)B}hm?fG*w
zh3jc=Xi946>*r9~R(ExYsUlMf;}$?T+0fA6fJ8yTcb=D=eS>jB)S#+3-4RX2VF#?=
zXq8~Vz$j#fyn+HBApc}{Zb2i~*Ov`wXK?*?4BUbcSi$jiMg;>p_{|(hL&a#fy{J1q
zHI<Q)a;vbg5IQ6A?xt@5#V@-hRfv505q!iz<lbe(jezlefgfQ8ZiEkX>8FMpU|_77
zz=|4=qObDtX)-Cr{eS_zwCZZnt;NBff9>$NB^+KkG+_n~kAct8{uqG5$5RmCvsxiX
zV3%$nz4n&b4NJh$RIbz>v{61A_SzYVxRP*+4(uof4X3??LmrEC<bvV$o@e1h$ka1o
zq0j~R^)(e;-OQ2_-duRyi`V|9s)<Pw%vw_o&|CkVcQQt>7Nw(9aM(DcXLd-@jf*0J
zpvvh-0W%qHJ_Z`~4b^+ifDV5ksw^+BcKh}Vm?is7hC*B%45h(mh6h!JKogN(|BYoT
zGTcxwpdL=ECBeQ1r(xRe!RWb)>S}!$moEtn3gUxd)2avs10Wv`svYPT8?<T*!k6d;
zx8L@1LktHiTuDud9H?r18jN=vjvcH;soa~11^o}D6yO|~9v}wEg&2qb^v91M+jd@4
zs;&VCckt~(s%+Doz_cWR9Xk7gkqySRuqYVg311@{hs>?9smTf;D}Z{402t*nonQk*
zegRlei9^@W7d(K%Wg@f-5pnnEg%z3;h*2-+UI55dHmXVU5}Q^Iw=JH-@2~hZ$NLMd
z-@mIPfaa0-e_vXK3CE$6%DAd~2HcTd$S=el;h?s`mX$P9WuS1C@sS0|U`zq$*aaYA
zuDY@Os^!Y>>Ug|R8vuVGge(2@+o0nEbPjf$s$4qS6a-8i2n4SpnZI4{hk{^OUQrPc
zP}#k(e1eRed_G*;(B7B{iS9uQ9^gu&n3O|@&cj`1cmLi6U5&73WUxmF6@mW`BLmj-
ze8Hs3D=BGtcvJ!>@9UnU!u&(bYjA*T_{a3)O1K^Nt{vfd;1_s7q>}6(kU>y;(-_7D
zXGVO9X~c8P<K1tA9ltDSXH2QL0U;N{&54V&S21<hWhH6fq2ear_RbD~QFH}StWtit
zkG0E?)YXlKNx2fh%)|6XTQ&V^VKcR(g}^_1Fvj8QQ38tZ4Y3w1^+KcF&ylfnziS{B
z$eDa4<<K4RV%dv})*E=VP1-)t9Lh%~ECAO`r<)-24;#UW&`JGuB2R#gzn+L|fJu4_
z%Je!%LRJ>S+jHm6-K+Kd)ZN|fv_5sG2DU{Z4i?y`-@^b=FJg;s`o5{Z;+R5+9wHg1
zvQ>ETcO}&;+BeG3m2%QChWFyE?|BYYx#m;5jpJ@uA=}(t-3-Y8ijgEO73w*9Ld8rG
zy~t26WRsw)f)9QgnHl%yNuoznF(5cDFy&UT(FIxY4Vldr%s+{wdH(}hTTR2E^Ang=
z>K{EnPPNmKGY89AP>4w#el3Pwt(JWJIM;n66#R)nwVP^7OUsP*)8H}m6yga%T1HM|
z@E67D=5HpN0t2pWn1W*2Ax(^`Sp`?jbA4oSz5Ly~8sLDD7aWn1ky+OJ;?hYF5hB(D
zM356Cz^<i;6T}uo@UPgS7olop(fLV*41!8z9Sy=k=LId`;9c<T9<XHVu{P7`zqW|h
zjt?G?jlU+2>L4Gda@{^e28KU_5MDAg(vq!TLx$y5U0lkL_jXNY*F{H1d(3`ty#TLw
z+8wn(^blZNK)}&B5rZePzK+8p#c8k%5CZ@=ZabcBNq~cGZEgK!U{8-TFi<f;+~Cuv
zPxF95VA@Xyunb}Z2(Tm-u4PUyeCg_iI%bilT0pW@@R)K`3J{4fL=f3I+XLWf-iK|M
zHXhZ(V8l+&nc%+|aljY7sW;e2p+v`k{0Ll}fF#kkJ`{97;@;|6+)hFOq}+w9=3eo`
zU}VJzyqCrbHmS$E;{J1Z133J04AThh4}D+9>Q9&x2%^y55b7RwJvGO;X(g7GX*D)B
z&S?8^8CGY&B)YPtrDb4n@Kz4~zX#BU1N@EN&Vmk#pMkHJ`~_eC_}3~CfZTQJyldXQ
zd#3>f6XZo|F!R$5<BOtQB8q<*=%x;zj&)3bw1Tk<dGb$Pzz=*go=p=qz*hkrbxK}N
z?g#A0ar(|t9#wa07haH6@lgQreuxPc1MY1*xe-IhAB>>S3nxUW5ks25!Ny>$;KyU$
z!OX^{0FZAjef_u_JUo1kGk0ZVWq%bJ21@^Z1Z^)+Ac@>e486z*@}KXn0>gC!2rkZW
zBk?l&^EDMCqkI_5Vi6NF81p%)dDs?}1k&uhGAd@>{J`Wd1JR4jl5)n)nJT1U#mA|L
z|5|Zh1pPcBpe`((=wA<`0Mi?LfSeZ(1_X$Aoe`{8HxF8J`*%qr7Y0k>X41O(#|&u5
z1p{a2CC9CRh}xvTz0g$bcJLyO)RYtph=}IL>gvcj^)n(3Jz5&Bs0#mM$F6`K`%^*3
zMr&*i4-iof7y?7s4G(x5kJBtmdlpq%S_-MP8l+dq7#{%0jWRPcfsN<Bbm{y%U~i6(
zB@ujNs1YQY2&(88SO|MR-H<l7LQk5uGA0LVU3c!>5q6k*8SiuK4qE}_1cvRNF?swe
z#Axpja7dl1_bI`})(b8F<6@s91BHi8RJf2^!du`HAt&_$=uPF;Pd&|h_p*?69q&vX
z|3Q|c<E~1|ji*?`aD*Rl^9&*awByLo+DcoDk^!q7yzRiu%*+z-kKmPg5<26A)BzCa
zVA)&(ZVQpSW;6z0c&2$2GhhI!xJ>#H0E#dFvx0X8n~|U253Tu}*RRhaNJ5^6Z7NDi
zAuvn&(@%5o-=()ZgrJN*O-_Fv10`)c6P6)7`cFr%N8q{Drw!J0AF#n&FvHrqva=8(
zvRq)6d*Pom_>RCC7WvQ5OVO498I!giKxxD-60vAAcLX>u3VLj2z-V4B;5oAZ=DQEi
z*!&&DRayOL=<m#iBzz3y5&78SHIT>hS1K@hG=T6ysV9*fJ%r83&E;D;j)e*jlxw0*
zjOFD6@0*(b8h93oMCXW7aDY<$Eeg1TQdOCTAZf=0)T_>hvSrg_Z}=f=fWQ(XtTc3+
zIL(8thT!04nK<D;CUG20A}U`o&btN<uw448hYNx-Kq+5(N&bu8I0;#+K}JK9&sDoC
zT|VLxi-e~z<wD3d4sqyTe}w4t2gg)4bPQUsIri!<5-gN}NwK#4%!aw8fy~LZwzc&F
zdfpkT$}FItwsl8kFtc`{t5Wda1>`p8X|LLSzdCoy@8@a3W<A#lO}JS-geFstzP^g+
z`GtioKuEJdR4#G2`fSv5Ly<$LKoVLx#drL}{&g;p<>L<{Sw2vtW9{EE2|R(v*Ovob
zd$_K!=PB;B(JtWeCn;cR=fD{z`+t4p5B!IU^^buMHnGe{w5#gy=>(5e$W|oB<^4|1
z;NYB`?9ELD{l^<t7eAb?zMx6jWF%dga<f(PYO0i~WXjbGS8ri2ys3~kdV@DN<1mNS
zxXK9sM@z{|w&Kj$XrxTU?hM;T1P|6!xo)?0bchDfxH5Ib?IsLHJH~r;x=)H}p@1Kl
z=g;e5B|`?R>Ene|to1F!ncz#;pMoc*c~?ZMj>W-6{uL1V!6LU;Ssy8_BgtjOmTu#W
z*;kbc&J&_&u^2x$Eb3H)e+4a;3I)54l3;fj)a7nMb}TD>_7E_^%rFnw(1Mc*-}cWx
zH%6Sd^=y~ppdy5=cO52ae<=^YkfX<;Qz6m0N{AZ8u<I5*hYbI2b0|$Z=ubCH&9i`~
z9Q`;c`0q1(CEwsnUd7x{xq1e5jlp-NqUAEW5f>NBO@@o{OS2-n(A0{C>Ev5b@Jxqt
zpL)a7(=5WmF|CnYw$rT1Hncb><e#xT3X6Ytke&Gg1|5u3J1KSZ)pJQ4G`k#*bmD1B
zGR+$okpW~C7)7xJRACsaCMy_7iBg-97?P`U#(7Ua^8UwzNZ~=&-9IwY9^;RT*!)V9
zZIlAYIxP;}4`R8OD3m>2{i72|n1iACCIj2g(qYD^3p^Dwpjz$k5cU5jZP2f9Zat;-
zyLE%sSPius5?f-T!VWA@$NFL|0v}4!<=i%gS+ibP?Fj1v5~7X)U!}vUk@2Q4>(&0h
z50TU%@s-0yw7-yk<~xR7y|P0|M*kECGw=rIxg;gOa29*$UzUS<^TJznfaL*pXk-<t
zI|aSjzYNFm^F^q8iGmZyuYlfn#Z8CrA^~vQm>Z~zcS_df=T*Dejw^Kg(D->yE#Z9^
zmN`*4FAasiyQXN>PLQniFIlinXj+oz_LE6};03z4p3O(T&za#@WsE?VGR4rqGePyg
z8-p@orJfTc5%(96aT$U~^~R~8PQZqL{ykN|p_j6Ar{}|RQ^F(Kf6_A(V|;z1F_IWw
z^+$>9CNkpymhOcyp?eSv=$t@1H3p#97qy%vO@If;L+V4|7z2fUs^%i+LEw~fgDZg`
zXn7oI<HWDnv2FCgDLaJ65V?d2J6j=6a1|3rD~0-1jEnOxNib)C!=IDFRc>QnioiK0
zJBYO5Avy^_)rIowR+p}#AZq@Ysh$qBtoZ{|oV}IKkLXYU$L5z4qNVr`%HlF$B(HKm
zy|E7UI5>=`)8bnmxXvCvT*rqp>G?I<0{4xF*UupfYBiFiPhS>sVJBnP`GICc)`<U0
z6BK9=t;+qE#Ha}woUyf4E3PndxTX#Yiwd3&9{&8g9_C*&5f1wtd$vaNqyvWmL(QFi
zIuN+{kL-`%#i94(IcNRM*Aqp0qQf9TI`#J#<n-9J`(&1-epu#n93({Mf0Y_$(tZGy
z=U~9R7l5U77+5*M8<*{0^z1*Bni)-t{>_hrYl>VXLcrk^E*9eh=lN21PW2f#eE}>!
z3w-@L1>Pm80$^u<+9-H;2X_{3AOUKI!57zeZwSqZ0`(QZjXs6r*iV&OOpm*HDrX=o
zgA}N97j@$Rw*V71=pZMOgP{_2Q^`410YitZ`V|E_X9{3{g^A{w3*_iFd7M)?#bsw(
zPlK##p>CYPd6>)7YG#1?b4I89u__NRK?wa|`xGs@)siRW@<TKcjrd*IQu7fuH2eT{
z%z#KNj8^LDKZKfLgoC;N!efNHa5Mcq*w0I;MK_Rx4OeDR!okiW2YUhL5&v6-%$tYY
z80+Y&r-7PW$}MWCnA;9=r_fJtOU@|JTKK<aq!T?b#eLp`kbnGkJwc~;Ijc#36Z!{n
zf&FCiJp9<6p0=0<hmKoG@oCUsu@yw!Fhbq#L!H7&mSdoS!)Id)J{OO%KKAqY!{qDj
z>-|Cw7uat2&y|K245c4~#D9L{+ZO4oD#(q)9mu;5HnD&Jmre8_x0^g8z#Ay>$Nc35
zc>JP_<U<Ltb2ZD1y&ycaXYs#z;$QMV^*>+sKV<wL0r?+2`+sCXHc2Yn4$mIdKIZx$
z?(}?lxKgpDCnd$2OI*R9H6n>Ox_eHMy8leHZ<{yH^LknVa(HbZ7RsKaZ}vzqF1j_T
z0l2;bHw1|gFE)hESzYYUqX-##E6Kc%8>tkZs@H$w((r-SL|XnWCtqs&yRPYqjgH*d
zlOK#S+!OU{ju+e-_DTk{dX%F1&!u+0ZL~ig`lR5r(XCmS!0s&H8g@X@EaW`7Igz|M
zKI$c(AV!#aH}baFYh?)x)!0rfO=0}4JF8*c-g{!dR_>fVuwL3-J`>Ypo>Q?i{K#N2
z*P++i$#v(;0P%I)=p!7AADxZ<17hT)h3Db;FF96>KM*(K(CV1U3ey^sp$gEL{D_jz
z#S+yolS^*kKmSOxY8E+B@@NmQ-HK8BR;6ctWTMsM%W7%n#Z_PToJaU$hRW(Db;kOP
zRGRb$sSc`>(hSm+%&bH!-cJs{o9LMfpdf7Yo<D18Ii99_K|%`E>^}NuV<y09tL?NE
zZ{n#MwfwB_LW^!ptGjEa`<tm&qYM^tuM%j*gGcIylh5&fytj`f%wQN>V4jc4wliea
zNeaV$={&ZNQ#x@8vnqKc-AUeJZizRcBlP!n6K=AJZE(3VdfE*};|t$Z&f@eMEqqwO
zDBzf1o+v-3;9F5n!H)a8gHuEM-ozjctIM~3PqcIwu}Wj*RhD<E+mZ34D$B$Bp?M#w
z?v~qp)gcfM7d-EL`0(m9dsH)Sy8$L=SM^HPhOPqKL#1b%5$#Y--c^$!H(gXchryz*
zFr$Wy=?Zoz6-Dztc^?-EHa(7}e{2=Gk=ZSwXiyjIBJ{BNLu#^l_fY1>t2)AG0-b4J
zTiy+wzd6vSi+Dccfe|8RZgAow7{v26-UvH|lPmn0L^3GwQ)mMT?efwoxI!`Ebhwt=
zcECHDb^eQfD%+<V*vM7hZX{Dl?s_>Ge{rFk897=h)1iD^k5@Jv#Pz#^&<_Oqqm+dA
zO?KYQ1|vG|6*ifSrPp-xX1__L6A6h%L`e0v({9GPswN4@t7pGDa!NOEL^ts*aQ?(&
zS4~EF39j?7X?4AGem6)AL4S)LUN{6s1thr9T)4@C{th@e3OLF4bZx{7&@V+`W$D)s
z2F&@06onnWDgT6dUPLp_iOzcPQ}HCCmU7R3V5#&zftkB_`hb>yeOu>rB3pV#*RuoS
zjJX0%jrFC#`P9ODXz^qn+W5UCuE2|2)LRA{r_H{-?X%C7kRDGZHJ>|t``&3{*HZqS
zXyy&RJ&M+SX(jm&s_{1innS63XnyT<`PV6R)rh#hO6RXr$$Hbdbs_X*Uqnn}*Up$u
z^Mj$s?AXbl`lcnP>CMv(zUj$D%&ZJ(di1*W+d3{I*9#1K@%mxBngqYa@kF`=cV_J?
zk2*QzckVvC-^Fh7F`dfZ{aJtL;X>q1r^(M``#VGD3Y$z{#0@fses1w1sPLc*YELIV
zk$Ls*noEwh+uDHT+m5<^TeaI}S6Wch@*Y#EhMfuewS+z&(l%t3yWu)7eSVPd+=`Er
zr-*k6$@+BVNL65ea@K28ek7~c!uU+P;Jq_6VxDq!k4G4OwoR?zj`<9SSX<6<P>p!~
zun%l>>>G+0yfR$zEHA!>c{zMG=wb@|hqdQo8!L{D%wrjApprLuMRZi5UUIXxKS=$y
zKi8OY-AT1}PMvPa{3G#&w#Q#~uImLp)argxsA1C86e9iUah!xls|j6Oy2F%@p=I{f
z%|8<i!qsMxpY`{|hie&Xzc5<WO0E5Fc2tlW%|q;>r2Eo~FeJC3Wkx@!ML$)Olpwj@
zgM$-+<ISscNe2G`4)%5*ft9KWK2{^{1>-}1^E#7)H*e?gqcUj{KMQY8a>fi~l@A=b
z>DzjI>dw9o`=xc;Cbv%c=ze+?&k>U#+)0sYcsU!pJM^TF?7T@iR&3PTFw)A^a-!mA
zB=#G9Q(Ubn8Qp_OhT6*WT;86nd4kSv*^f=>LNc14t+`Kcy2q^AbICjqUU)|p^u(4w
zTfx@VX2wB(WKOM!^Jr(MZk|)-sPgsk@EV&}YEI|3&!Lwc>JCX{Iw+Hu@wRy0c-ISv
z-R|iO>K9%&KSR+O$t?Fi?7+8Cc$V8~LG+?Ir{lq`o}d2Nb8E@69bI4fMf~@S<;;J3
zR0gP@3H?z}`Pjc;NY{kBW<*jr)osa{h+Wuq*CPs+8PE?3h&BJ{GB^{*b^cfNlw#(P
zty<~J9Wk^X`A6L!)2mB~El%^}2^xu>ELq)Ieu7Ot@$&Bmu0I~NUp=9$_8y<&_X^M4
zjMP_R)SxQpwHOOyh1V>=wf3&Aym#i(J)|mjWjQLEmyPKWk#}R&nj~X3dN*`8L^sIM
zu+F~3Y*hZY<*o$Ry@iT&L9Vj(xSc(>!n|un-<xBYKf{cXJu%zII;}KGuEYT1`(G`^
zJ9kzU%!0a_yJ`$_Jo+9nmCrHVdgPqI;nu?@%`HLc_SlTCAcMxsE>%uh?An3)-&lU}
z=gobv$h3*?krbfg<DS<iiZeI`XuHSaI;eV*`mGS!>3%yMlujQ>eNAp5F{%hZs*t8W
z<d$0?R{)U_ar#Aha*|(J*l&hjgC17aJyDrqVv)*PIqMFu1Pc|<aq2Fdqod~x-49d~
zZC4L67h$jU#P?&MbfPUQGF9Lu4|gE{#&GLg8Qy{EfuTw2reR&#S83A%l%7Z-13tO8
z+fs!z*>{zqUY0)t@m~T3T{Ir_SQyJT#^eObMr<4?Cq6kDNiwQEdAmGT!z9|-F)uD}
zUEFtxP;v1Rqt7Rj3)jtjo3|PSu`jU|k0&ngD-`O8l<jaAI@5}@3>13=6`&um6`U0+
z?j9TqeRW@kMxU<wcDC7+_hh#!&+|x&?OSoYW)8w1UpiWkH2dXF?^2sQ5}QpV@bi4#
zS!frdXW!PK_1rV$v?<Z_<^GTRHS}N0HWIuwW4fl_&pwx-BkUGh7gKf(=NH+T#it`$
zCm!Z#+D@U@G<2+ejn`+bI2#gYCp1(0#re{wM{f^I=zbGDDbKYK;dE>uNM%tiy6W^`
z*4rff+s8{mv;JeAjgv#VBlpo8R}_4(5wG>jC_cOh6qku<n`7~=-gR0!Wc<A%*E3<z
zxCOiru`GU=@;i`kCVoYLjXZU>(Vf#g({=R14ujr<8P-jDXI*ZsIQ0|j0~fw_t*^I<
ztWU??)ZN)g$QG~|8_K1(we9&R;9YvFa8&FX+L}{)WI5MjvVRQ!e1bVyVSxS2w_nLF
zTw29EQ|dF9u+oWiiZAaX$tUeHVp@J1X4U8MP9mm7c?cT;qU7cv@YTA3q<$y@XL%{u
z3;RZ~-uejVILWm4;KST=`pUvxKDo8!@9MXNGNfshgRi6rYW*?F^)Cox>(96b101hj
zlEz}c5lPQ)g(MnIOJM2IuvJ$Yea2RrHV9k8_Eqpp#$XbQ=Hn5Sui7UCrPM(Tn)m9M
z(tc$%Z+T}q-%0DOJJSC0l7miYwfCE@*zK;4)CGU;y}{4+VVcqM*<|8RQvGgdZl@JQ
z{9dqPB{vP%<Hu5m>55LWgwuq)JnvPK*YC)VrBi7)S4t=>ac<ZNS)Wm7Z;Ck^q;acK
z?_-Vb1*+m55x&aZYM<Ho8;XauSey^a*73zpXLe|~i1;Wjv!K~#H#($g(<!ez<J2VT
z&^Z{Nw{@%8{(LX=ldI8<hCQ{$<P{^{5|=iy{?!c^jj{$k&e=VKcIl51&Z8fb)aUk3
za`%W%HVM}*OgjzU5K3cmlvZZ@_^WC}|H{Pb8a{^yiP4we4+C_+Tp`aOULHxGTRMYJ
zC>&ojYC8Df?KPuG>Y*7Qh2fjlZ<~9(gc8<isyV)DkZXPy@Q$6|vSQ#=_peVcevtU>
zlj+OTUsjkp=*m`2uYWp<yy?&FtrC4hG*gqtB(sG))@r=;O*8K@{jH61@yB=j!cVM>
z3;FlI*!+I-jk<QQAyms)dE#(nc70u}?Xy)f?vnp%OD6$2^BL>idkboo%@0Cw>Cp0z
z_^+Nt;@bu~2m!HiBu(*wgLpXl_aQ1Np=8H3#veaEH#4VSfUaQnJ}S>*^nDU*N`rnt
ziZPS$?LSuUmoa_B9&feRQ>}U5w{t5mALBB8xz7P!^)2Z_a>MfgMfZ?{Lfil&xo98W
zNTn&dF~b+dk#SFXUN)<u{!vn)>wTip=aOZsmo>6WUnVm$zao`7{_@l!{bQ;BX_}xr
zE4TPvTP6~+`!mfy5kIg0+!N~_Q>+CK7vI@G;LDUsxU^=qRB~E+*Pi^ZT-*)Ce%pK4
z7Ose~8_p9Y3-dQ3Pr<W^o7OJWF<vm-_M$8Kq<m@?9p_)q|6bx^kK+f@8E01AOA67P
z=Y)dvsAS1%-KyHdj_-3wjS}&qi*AoUo)#g=yt_A(*f+z<AikoYqwX&mqFt(Zla6Sk
z?4FT%gAzF2KMxw-e@W9-2xn=3ub*?`cGPe^n8BZ0%;m)4eCSDm+5C*qcmGD4(A4*$
zKi5pz`j!-azH%0KuTjoT^w!?fCu$@Sa2T~PIqzZJNw=0Pwe8udTHcr^ovS6>AbQR5
zEQXFdio?GIQE5pO5<}uQ_TBv(>dJgJ!Cy^@?rk`Wa0*_kQGhO}W%M2Txr@e4{u@W;
zhOmTZX7@>0-pc38nIMrNL-kp{KQTOs^{73rwY&UL>FrV0g5A(3>t_X=_pjeN$A0OU
z+mvqF#~_*@TgUq-O-D0WJ11Lk^WJq$!JB{d<`}L|7k2p)ZOYd$Eo2swsg-Mwk%kbu
zEeXW4wrV{WBB)Nh2BQdu$HvCNmx8$Cty<JMM{f3>G5C5x$0g#o;w@Ew$+A4EYO@eK
zyU%m0y}wPL_>ek}A4k1T)W%o~<q?zRUEpa9mnr!0T_7;CE9-Q6`Hf=rw|cELYdU+)
zS`O9M5{mmWJU$d-2W*$lUTnTjWxmwA_f@IJjbN`J;bKPQE9K+DGKEZ!uMuw66g#(j
zhBT@@-18%%IWLMVLC8uy;w7omzc!taAA3hsW>oBKPM17p&el^$w&&xU$!uv_!nj4}
z*cD46>l)YjAF5^}?Vcf9_@&S5Y0SBjPoKJ0iSZLneWLM2iCTIbV<s=Aw^Kwm|LcP|
z$)4bes?T9(43of?K7Y$znCUNF%!4jN=LhRM;R_|M?aG195AHP!e+&t_YWt1qlr$!6
z*iDWasReJ6Kw|krkKZSscp5R%qk8y=k+!Q7F~&D0R>0YDX%@$PyM}G`#v%0hL@=$H
zGrIh4zALQ1;SqJ8x*)Y(eibWPI3J%sc|&D__c@2ZxZWkjtR8$@a=$%8c)QnK$_&8>
zx4T|!Ud%r!CoSJ@d6b<eH@DWD=0rOP4WM58Bu<(aULFt*mdWfcWqaj(rf;%gceuuZ
zv|VGq$*=$V-S;dXWQ&Y$6D1q4@vD_pcdCW1T=R&VJ(Y#swK}re^gh{+v43!-mjS)L
zx#HJaVrNV3^Wl4&q2r+aS&`ClmQU>6p8_f{ezCQx>|=OJ-4{?oUN*^tH6QiYzrSH1
z4A0PfE6=ttQy4HgAWbWsc#m4?eeUwG7}LhuTKf~$(dRx%7r$5EU3%CwEa5o}9?5t*
zbm56Je{xel16p{;O!v<u--VGc!QZv#TpJIr2fa-WXBk_x`zi*$OT=cb#v$uq?#<$N
zgyd)Myr=)$4_i$@VylzX9~XHbVYFooV8mgmoB2!v)Xy8k>Y(04)tWmi;Ph)XQ)9j{
z0p)!++HkgCe}9x!D4-rM9YrUr725g;-)|9L+xhbP;79xWUY`<X^4G1-XAt<s?asG2
zzWVTD_44zZ{GCY?TfYcrFWi%+ZLae^zVTSSx~B+dYOyi$3UBZHZc>5AZ?^_aM@Y}@
z6<zwD3Qo$}WR_|vii%&#+Fd0$wl(SA_$!f%2A}nKUN4$T5afLRh=og}Vc>W~_<VTo
z)PTp0aQds)<f44N)qYYwSkroe{YJ6w<l^$8VbnE0zUB`zXTEuVVrM$_qcg(vhdvBg
zI=dE3W7bxXKed)?hh@>}sZl<?Z;=*h1b&Oy=U1NX63mOui1G~>1=Ukt%xCTCr`(et
zogU6GiV$Si`k)NN@5;)5MS~yL>2IZn#2UsK_sI63hvMs-fW#Vy>`#|1lBjSXPJOb`
zfY5W<^$a`iq6z-^=<>iB+(ESTAh8`eO83@;Ww3R7<5Qo2d|9^~rZh%$k@}fFON{dD
zG)uwaV_yCs$r+E0b-sWeZ2axR_xDv@?w`=akozq!)ki#Aq<!8mG07506%#rqC6isV
zOwFfsP<LDqoaQ}br2jnxpuSI^sK_MD22Q6d6`P8Fu52!>#AYaQOndaJrNwL6sE$q7
zs@8~oCqaF1FXp2kBh{P9!;X6G>5DW*HwC2{xrEFVU3m%UU$4k3X{$r8mg%zYXirE*
z)!>y=jWr*Y3<$_$rdA$4c;!0RQFp>Z{w_Sj>4SXCv?{T9zMz$f>WG*m#)*k!>PV#;
z{Z#)n^f$r5z98|%UJ@9dA9C$Dg`<P2D0<F_!;Y)Jf}z(z4Sa!Me&la3|NQ>hj6ULV
z51U<w>xHtKM9xo2(g+ks&;OdI;h=ldxi;iejYmt!VEyUq+%BkGY1X$l3c5_8=~NrM
zGNAHQV`9S<cK&!8Cs9zCTY8)RWLR=LKTbUVu0e>ddV6dXZ%Ie`{agu-w+ijw=cOSb
zI(nY2??TIGoSZ(Z!{9F&zL)ym+_GMehLE`Yo|1g8+abkE$QQRTjaK2O^x7Xjxn4ft
za{Og9ckp{@@fYIRjq1l?*l+Hc3p(?KGwOB89^K<bXdW<epXvWMUaOn^@7?){*>Mj^
zQ6Lr`V<0Qx0XbsO-Y_eas+^FmXFtNZ)~qBn@TorYnU9sy`R!|HVUcqad_$ZG!Yg#G
znH|z`>3ELu9ZENQAC*GvpwU}Y|C@Ip^9<MA+nv6Tz1u+zo^r0nkFq)%=P{w>)p!$U
zZVFeWxF7$yV<OdZ#(hb`@RSOhjsYgFx03kEaLIc$7NUoEY&}(zjXw8Pw^tiZUDhZr
zT@}n>jA)u+zg7CR-+}ydj@Jn3qW%m-<j+sN4ZZE$V|Cdcg3#5`4^^MvZ%)dF{%L>c
zoaxd}&N(N{Mfbb9I#A)NS+gAIuNV2uV2?mh<h@4*m64rZG}4J8Z)sNoeludHSZRfj
zo+5XcFB#smB)9D^B(&SVE=1rQ<NJXFsELH&tM%s3c(VATq5;~GRR-fUZ=4=e;`}f3
zW$L#Y0=}vniga9_!@5@E`+0tt=Y3u-wEO5hbKDD!<Cl5ypRkTe>-`3p;k6NB-j$6t
zjZ7J_EU3mcdw7XX+c$S_1>Su1>g(r`Ctea;t4@n@yhN1Wo7z(N)Mp4~2}GTB4LXBQ
zUo-Ss=}ltz?8u%U9CcTB_AlA=lijur(%iPEvDeLTrn$7Gt&1)AYmOw5qC{c7M-aKl
z$w{&-Mqk45x|f7Kzh;odt^7KpXYulJaw#4QGZOa$_lm9?1$w!7eu=Oi1)EQsem%WY
zr;zB{6IL7XG!e2Km3vpcmWCN8P9OX(?D_0|cdGh&nB1#mvey?-?i`9`tq(Y(<J<E}
zRDA-<P07^+ypOINeR7m2jUG1YQN=_8g|*o5)^wN+QBv&U@1XLAgZl+v$AtS*pn07D
z9i%|YHYPf#T_*HrlJOQXs1P!-Kd{xE803mQ-%U(|dN_Y3qZjIKr`Jd*sODX4WyoTF
zlK-lGUe6F1Rj<`$wW^Jo+I^JP#-Ld-;bjgS{%qp>E0Cqalhg2UH;~~yd*GKBw*tw=
z?=&8)GKjOY$M1Ve7`R1|oq4(%uraeUoYa2(K6yTA6#e7N6F;UJY7Rb27k)ovZjE7z
zl(o6J;3-{xvKP7=AVXyRw(${<<Dc3GPowZST&^DDf7Qj5?~U?Fbd@?@^fzNBe3Z$P
zMw<F0UBQu6?NiRfJMOu+%jh+0gd{e)9i=ab6`r^@-e1YXeu%;qS;^?<k$&l7C2dcJ
zS{eH!`)tf|((cli_+Xxy)@$#~=t2(cs0Q|P$6}&4HQrr%t3u^;K$?O5M*Ysca%s)0
z?p4^X;S|f#?6K$_Yusp2XZiakI^RtTh5Phtm<C$zzME<ZbKZNopBsE$eI!uUBhEj=
z^2(sIZ<D}+lD-R(K0i4N7n^!UpJ~UK(3UW{*vYPY!X)*>ApNq#snuiOZSj6Xjh;7K
zEX6dwnGHqv&dnh%Me+?&)J12$EWBn~h6}bbo)>D-FU0&N%m|SPY1QG8oR~Ef#xx}5
zL~}4vxis~wIu;Z<#o<_-f%Ul!XC9W)->0|7)2;+^|J26NK#sgUe(2seLu8+3eN>)d
za^eyqI4{!YWyLLPY)&KgosV4WW%bwti>u)tvx=h^{CqNoD~-dkOoQw4WFEU$MVMHi
zw%eysjkj^3+xQH5#mqJ(lW2p3P;b4ViSAXeOzZ1gZ@#As^Yu+97JccsbenbY-Bsmz
zZ`MBt>h$|v$LD@!{-8|F?EC61-jnc)>E7CEl{`;J07IqPtC-Q6U-4Qwa^^0V>C<qs
z@e5XUOShhUR=?3nZG3&J#{2PH=}uO*>Yv4x<_l6>(u+HKB456!Z;>~;pCvT8r~XX0
zekeuff3f%0QCUV`+aLmx(xr4L9nu|wfOL1afJk=>2+}AmEscnDr!>;t2+~M*e&<Gi
zGw;mzeY0l%o%OEuEdP-Ext|mJ?6c$A*DklY_CPLm)m?9Cn=1-kA*{kz?a%;@?(7!f
zv@SDqy0Cxh+|O;I;;#`kKUVdBrtR?-E;76W!9oM}q^gS|{$2ZZB;<7IS0)%=?Sy_q
zv1W$|>kkzd@Sdw_MZX6ZlIj^BGc1$NdL~M6?))DAa^N2^Yq^=9Br2eRv+l{NvtUth
znHN!@*ul1C)U{sW0tdw|BE8om6!I_*J@)v!K2<V7$#F&XA__f7?;$|}aes0=)(O*}
zJTQLGL}bhV+6)GUAilK-uEhSoS7H_!dA&T_==x^(o1o5t7i_<~3G+AymzA6Da9`{n
z_Ba~%i&Q-_Euv?_D6*HDrJua^{6n3}?;xuApw3gv9FeqOBrlHA?-19%i3W;GF~X@8
zaZc;H+x+YUfpoZM22Esh+l>5oHQ9sK?+i3jk_6&-SV6o!c+#1B<zrRM<h{YpQT8l`
z0D>;H`vdDygg6u@Pezm#W}k~wvls}cD)!Udx)#XfQP_p9bWE*j?3UW%n`!-8hi8Br
zI}{LqGBJt)0(S`#ewS>}9$+@TZ-v+bVBaYhK{><+FMxcN1|6p|W-3i`?K*jXX)^0J
zdT@do!H+Jxb5c>UbV<64prOog{azDMU)-g938+T=U2oEb9>X%3a0DuizcmG#)OrAf
z8wcpuwd_SkZ@q>`ryMv_Wg`RdQ&FqZcM8E-WJ|uAm^hH*@l#UdK3sG$Lj>VAKv_71
zRbYN1HP;G5{=5dHw!#x)N{x`ChhNRv_kts5-?ny=1u7m#0p^x-P*Oqy<$=^u_CSF5
z)oBiZ{8{IY6sXWpU|OY(xkn%UZm1LmY~yr*oo73C*W_oSmi0sYRZyT%DZeSi1=yXp
zK*IoYJ}fLO4$!Hm-_f7+rSM-o48YgJvUT}k!v67b&AbJcQ!WjZol;$8{V!+;c7dyD
z$`X$g8f!hvQoV}TSmOmMlG~~&QKjPgppS}|{vDu!MwnNQWLlxu__o8o2wJ)+fo6PO
zoyf<(0nB-`pF904UY_#}U~0?<Buv_Q%U*9lEx>Un(Zb}*l|YOBWO2}AFw&89D`*2T
z|LHz5WW(KJMMD{#4Fe)g>WQ5NrPx<V3<e+r6I*Ub+BephAG%*BRn$3{Rb;`eoP&Q7
zA}G;=?t9}95_w0sQrg2?5*y0Rvi|UW>Y&lAHe@e{QwBZoQ_7Bk&`@efiNB@?mvO^X
zn?wN*jU~W{HD$qjeFMym6`*uDeKL+t<voBoud&N#&}ey?VvT6fYL#Utif%K=XRKR|
z<-ZWPnbDxtZ{Rt=-86g;$l`|qC%QLv&uHM;&h!d<q%h4e#(psNGK4V$sB?gN!Guk>
ztLli~nl5-7HKCf>!(Wx}_?)39tipwv^r*<SJ@!Ftv+f5&>XM|1!%~CcEh$IAfzqzr
zw_yS7dOxV_i2+g%%4pGt{$D4NPAx}R=YEu{kS+D`X(j-tUxnjr^Z-Kl5g>-%`th_%
zly#2l@`@x+Kwm$9(1C23eGxR%8JVwlrSrKy7g~Afzy=s5oIs1*^n|CcqD6l9rMIGh
zW-d9P7uuC<<aW==s>e1mfWc%2TZ0_R&a1yLnEqW_1{D7{Ij$g#(!oAt)_bPSDxJjK
zIh!7yix!0GcDb(v<%uG<cJ#`F<ah1@$l>{-HGRa^WLjw59>Y8qC%gb_qZ#D)B<%(h
zR&b9R$v;i@n)jWLpKvTDB6MWZFz@&OJg}Up>MJe0KCgG$rTXQo3WJ1)CUhE^1UZ#v
zw1FaLy<XghsM_2&I!OMzhKvm?P;Qm$5llE`1n8*^|5dc*(V&kk+G)dQu(#r1ea-|j
zSCDcMw*7)-pp;Nc)?YM`_HpikIunxrLif2Vv1o>G3MNAhBsK4Jl=li=ARP}NSb*1w
zh=f4{w!>eZW&{89w5L;TAfAgM+e3d_LjIe^R^ZXG+6K0&lidn>o!1s|smh0)hx!>i
zp(d!lK`oyUZ?jbp@3Lp_=Cam9@iKIS46@;UzM>t1ES4GJwmzTlF}Pk<JJ`f&t<!Y%
zr*9ok5cC9m-ha%K64y!_eBpXmUo3khC~I-YBX7x;m#6#CAp4lN%uRQ2)L-HHvYy8W
zr00`|EkmZ>PBoL5`EUhO-HvLpDb~G7BEA|RI(xfF;nHq@GADky-is!kOwXG$m=<&x
z%lH!H1nncYrd$YM4d`{k-L`gz=Mm}b%bv!H{koQ2u%K(|ykxq&xnW7YItp}P7FzYW
zcoxNJ1LgH{x$|T{C$kBDo{!19lF(%YMo%M6jlQ6x|80{)Do6??vaR(>4>H(%28qiv
zM&heHbW)0@gt2|y;cHwLAAvgGbyAY4npEag0hzfq0l8Ykpb;w$F9d*UNDzq*+v4PX
z1k;e!iVWRjLr)FkEqwPD8ZALJLb6#nz{mlx-RrbbPtX#qX$*A!>^BA2<(Clav_Re&
z`FfALbr8V&tEj5RPSDMFC9;-Jpa7=WoQ;jw#VU1C5F!&Wjb|pwzkCTVmx!5il0S?w
z^fEjEXdv>XxO-fMNXgWXGk8?fPr?_vSk<}&TFgpEXqP9!Htn#9+!hp!?Q$Q^Q*Nhj
zP38EgW1hMz7tU=77LJtpYdj~o<rDuzbqVD}beFRSqT?6Hru<Q288#hKTwR!FMGTxt
z<XLE!i9CO$F8D;#raQJLlO!vTPoSk@3`d}9<Xc>HGIv$DNKbxyN%m-S$38S~;#<gV
zGU;6Xo_%DWPV*qmu|tlA-~cbGe*}U{cE4jho#>JQ^MO9&4O=DgV(`c8S8EQ}`|<M}
zzgF~@r@B*71>tM{<@o7l!)gn<Mx>Z0`S#%H;opLM2-9N@DXhITBo8Z(9+kXozIknI
z#}Slt4NK;Bp_91`xy2z)ah}wPw;7nY<10@b&A&8RHz~RvJTZkXvlq;TlfR$C?1v)I
z+@F+Beo1LJJHsxZcDv2`4`>Np06G<pWr`tstdrD6u1`_MJQO~S@4Ene6RN*;>RoK)
z%v<@o(*i&vCn?Z}BO5fC{BT)iJsG(G$`uXGGLr^C)!oY`fZ)ZvaPa0`1VCr2PK=3G
zP%nV|BH_sUe4nB;&k&z%8-TIDG9umhWn{h-1-r;aaSsz5GUmq8*+(n5K`vb-(DBAA
zxTepMPLGGOs;PWV?Mgbp<LB2@5&olaHF?MQbpl1hD@fTwRx%GajN7HNHFJ7nR~0~3
zBAv$2s$7WlOHwf(e4@iQhfp|q*fiMhmO-*SDEcw4uypBVUx7ZQkQ#+zWrj@R2A%kG
zMvC4B+j#}2m1Q-os15;((yfVp-`AOxZ|%>|E*IM=iGsaK@bYjK@~~&G*H@*@3hZ(;
zh_ALfff<X}TAu!$?`O>2Ll5mzA%UhnSH7EBgJaFjeb)N7VVz2A>uH(<hm|11vey+x
z5amG7L-Q3Tsl+O$+t{$4CD;L15-AST?aiJM;rRDYZO4{IrHQ*qKb@pY&jkuJy~;Sr
zbZA|Y*^?az2DWL7>RPj?bYyPpd_I3ix8eJW+E4cUh25)6O=fDDBvGWG24<>~!4UFb
zD4>^u?U@U%A{rb}OajiO0fa3&Qxi@pCjfBIo$-ZyCUDLA{Jr>2s-p^&&)FOv|Isd0
zLejEne*9H-yMK>%5o8))gN%Z)feex^?_2!h<mWm_Zhy?tGcDpQGprO#%<M}Rlb?u+
zs0<#g*O<?5p4oWPi4A-myX`Y-S8?FouNWyvKh9gMy3LXmSNqzjUauWd0D^yJMP-rY
zk|?dAXC<liFMn)T6U0X}1?JZ9pt@&Y&j9w=YTcJ&1D}{Ez6kGhepb~g_3^4#3CF><
z7r9trgs1kRr>IPxQV5CiN)xZ#BtaM{*{=GqA($ZNl7vq}PZ9la!M9|7e?Rx<XbEET
zyhQ9toQGNgh|?SEY!CWvF@0bX(&G5-p?<xgS2%tkW;D!OKl!*fPd4V(^pd<-#-0)B
z{$5cOX#Us&)50Pf0uNa`=&%4a!%rW8Hj5)7+;UnC0|Tx{_`jE}p<jg2+6Lp06XRi3
z;;#1EJpGuPqGZ&v&#5$=IBs15FK1%j%wg<oFWF>XH9xmwZG7I)X~9(NTm2@9|72ZL
zpor*6;lBTrh0)g`qHoq6rG6aWB^X$9O!n8N+j<onHSu7l%igxUFCl@=^3YF3=(~dB
z@{zMFH&s>O6<VQ$=VP9j+PItOassH4(lxzpsT*Bm9#ns{OD|~<a`>?-P!iGYyxO9Y
zy7=MEE)N+!#gGzAM8t3eUzjttdw*NEy9qFriIsyy78tbuI9mx(n=>WZpiW|<`gas1
z<V8tzAm!`hw0_`$plczHnXnC*aDFiU6nsFA{vd1qnzp@?fQLZ@(nU=B{Q<OOKK~3v
z4V*y9r}!nB?aTWG#;F~VjB^1%EALbU4B8vM2nnJ%+ZXR@tysV6H*OTtuU0^7*d<q>
zGe@l*_G}(?qR$Eq$5KI1^fB8iAylP^X6o-G(as@t@wz=IH_kj$3U3*+Jf$T-mE=@@
z9w?&E9rI+L?>VE;u1<ZNKI0eJz0^&<#z`$Cr;;#jQheFz14$6HRXWN~+cIMH=Eb#!
z=R?`QLvuMkI2+#%HCq%jlWTieVe`wXH<yQJ^I-nvS$?Fq8q#mNIXLcqN`1)~xUdG_
zH+O10K>rqgrxN0NCj-HuYo48?5hB1WOfNlxoTArQ0B&heq|aln0Px9E<?~9`0P|Oa
zb8jy_K67E6kMmTw&1B(gmmS=xG%m!nm_yx^LlUTPCy;wHt@Dxv1TdB#a2H(VI9*RJ
z1_m#}uF5EV?n&n#U?IE0D^BCk*h%n_oZW=?Ok8!ov7%G4NhOJF2a!RYsnYz;<z&y0
zFwE{zGmN)0FFsqaXl$L-Yu*hA&ex#$12A0vSMKxAiQ4n-12|DGnH5OCEoX%X^d%%+
zNfdKm-fiovH3y04+m^1@2$4>%_~}A7hl9-RQ|Cu8Y2RD$q3NBAL1=H^tq(k><N&oT
zIP;_u9yqY1f32%l>%8c^3ZFG&ZGkt1w82-nx|Mbdfh|mq8bb|(T}8uQK4rm+0Gr|3
zZdB6DxN(6~NShYyZo|t7CQxQZqBCW`R4^d0IxW~Pj%2#mu#=M@*6U=s5rqAaF3xzh
z)0%u&?+C{}8H8@SIQL#O+s=Raijnx)gV}TWhFevX+!HNro?m(_j->WG)hE>_qso*W
z6uqr-{2w~heVJg3#2XYiXE?Dee?{5f`y7Q25nv=@B!qyts#6$(_=(yK?8%S<%ni)V
zqZn8q+KouvJBT7k6+?{>knyriCR}8FH(#_kmz!e0d*B9lJi92Qlhk7rG?mw1K1hO4
zm#4;WyAD9;a2Sg06||<)qf2`QRY=zjSawmvr}4Q3z8|+?__*`jIuLVbZWKcm$#k&k
z%fcRGZfBEnH9hCX5prp@G3U=Mq0lN8Edmf!Yvf8-YQSjz>h|WAhp)5u<0yI%X19uK
zaMNu3E`z*IjeuORD;{iy_T;l4d7~KY_@YRv{)xvbz?9FYH-Rz4wfqeS>Id*34W^UB
zq=<0?K>9<2m~<O7b}y4iDgkOBmO>K0aDmG_!n3j9w#fz~_2MQ`k_`YgMpF9Mu&@wC
zT2TM)u6{=FijmKEaql5<+{W?1Ce?i7Cs9^zIIy;Nj$w7NUm1u!zH6$MO%RbzdhjnX
z+Ics<r3MCUhmLC0N-45`12sj7T({B2L@jBW?S)&0=+Z~5(DRcH(NF?mp@^BrZWF=^
zQi|xw`Kzz8aI7~I^Bx09qrfVn1BzpvcuF*e2~W%PPwDZ8XPJ(~3+g2Krh{k1&N>3E
zHx+%ib(@Q!=U#S8om!*=n&j|xOKg84p~``{PM8+jk#69$w9rG>V70VP2I(bo23bZr
zh4PSsVVA4$(B!-X3S4|CqWXq{qLk4;d{b1#VUu?3OsDbF6?-gnz@zkc4dgR^cz;UO
zJ7%0rBo%TK>A+?A)<4%!lcwfV@L0dOai3ndTGG03=r>6M{aMj9mY$Gv+F_%5$`qDF
z!})>kfak2e3&-IEhz`{qE-nJV4r6aAPyADHwgj0xuS4EH26yF{{pStphAiDXXh~wD
zMtb?bXYnetcq5rexgsV8b)N>dWa_Ldh32X74jgp0t2b^AMb#TuB3;}4wouKIc$i>)
zRy->`I)jecl^n4v-?2cy&a=03{?x$-ca=G1(th`L-jgF!T)~)?n~TD3Ix(rjyw|HY
znw{Z9hy6#)`-%5QxEns!7PQSiX3ifvSwwv(jIF+-;*b7;V5MqWe|<MEEWc-dEPOl`
zT##6Oj-czs;RDJpFjCOLM^xlJFlh!U_^QshC8-6<!xQx1VQz@Li{Qcjs{*?uShvmC
z`7v_l8Mv`LU2k4t!K$qj=An~NoND%EJwarm*k2d7-uRqm(tg3_Os_JZW<x7=_&AT^
zjdQib9HY^nJTnQwdX3n9p+17iJ(3q@L=CI^q}P)y!R?iJ$xUJp8DjA_i+sU#aW>sn
zoMYUR$R#6TpX{rZwCD>fs4j4!ZaFTZrL#|U!WK1nJFbl+N!>89*tO!Ja>!Az(RzdE
z<Zwq8%c{Xuik{&yT(XVJ(q(fMHqVz{nH@?tjx`%aDwJi5D3=B7`LG+OLDSq@VK2I>
zaiB*7KIO{F7gL`)%4(RG^sjtA%eG!;L5}%$^%0qQ;{8B;7>J+ey2dqVy=k;Rpzq%j
z`Z$$|7(5jyV!_bkVt|fkeS6@{&0_H7cW;Hwy5)eRD_J&xPh3>A?e|=U?=YH)A5Wsw
zu;?aWg$@z2q*a|4d7!yeJK!R(L^(m4dIa_E?$O^unFDc=PAG-m6ta>+3hmK$f$+CV
ze5lxGJnfe@(UTh(SA2oAHG9rL5m&Y;jM%>rbrGE(zYhRNS{F%JU_3!Q_2-~jUho;&
z2uOsPeEoy_U}K<oHd&NhL#~o*<<B|tvdX1J8ST3n?>c~3PUK1fP<4vPoSu}lFFde1
z6lIe}YCol8Iam+skgDHa%e`Uk_MbG~sOIW?5$us!j+sx|2O#Rktr0;?=)8x`A~8o@
zJb~hzX-DhtSqMh6GFc;)r|O>DOoSf;;DE7>K-G-Nwjk%6dczue!4aLFpyLZ-<G#8_
zy-sU5p|871o!zb2!(50pzZcCm#!amW4~S-XM6b}O!;hj$Dj-|`3{-fQJ#f?+;!tFN
z{;8w-m-8%ESQ+t%skCHIy1d+*f*y*#Bi>kpDESV0__D;=5^46}yp?d?atI9^m@)M)
z7#>4<=We$rgr+iqB;fixkf)Q<I-bB`q$j`^MJcgkz&cb6pP5JjaozIQ00jUbwWI5(
zk_x0Ibx5H9X(puR;qU6IBEG*>rFW9(5n$3}?b_X7*$_Yw|9KP1pO0TahRem0VT|1c
z9H;gHB-Y<J(Zd*aEYLPlPDL^N)j{?)csK@KS%U;52sr$qiedm$A6C%Tm(_$js$a<n
z%gc-^TGZ()ChSul{&xOavQbwv34(80IiE-KFcr5fQ|XLB&`t`G0(-YQ7SF5VnYzp_
z4qY{X(>S<!K=S(6Qjh2xl@jl&*FIKk73E#A>L(3H>$$e^BWF14JYPvKh;{1XzUfIJ
zr4QIoqu6@XKX6)ZNpD+@9xOdsVK^xq=8x%*b&$;jNPujI2&-CM&%INUE!B#ur7@RI
zOzr1(l%>xy2qFJ%&&7I^?U|-vSNa?^J5?~U1?84K{)!cC*m!R5K1YQ01eH2Pbcso;
zYqIfpw@U4JOT6FsOoM{jEQcz_2L*hS3m={`>*i(h)fk>mQ5$=4^5RAIcj^jBGTXV{
zG9V!?PZnv12a7oInyy_c0sgL;-SEdGCTwNPD<0~jXXKeR$x_BJ86O@ZqEUnDxYm}K
zROb7`0_zvXBo52~8BU9@aYk^cT(8`|Y5K6LFISGJ<oYHbHtWCpNcaBXo7ekS6M}7u
zIlv?ANcPtL0NP+R(;YCA2veDR9%wQEusm(tNA)+q=F(|4N(|6Pe0v{r78xTQyovF0
zvKy6eo<M52TD0M-aOc+Jb(a6h*motZ6jeFVxJsr$0RqRY3l~&0K@@Uz7Ho9vvuRi|
zJf)e)?{5AXCHptWdne6th&{-xobhDI{_s9_<#<#5GLwO;+6Wz6;T?RwusXztVxNLu
zR#mCwU0UgFGr~vmS4d;6k}{}xDWF;l;diG9KC;#;aSms?`9^z}KJf8{?x1@30<6wC
zFalV91qS9}@+(9%x^EV0N<Ob5K+_>T<ijM6;_W!iSmSeeQj%8xDhW@CbEwMx8!Bot
z<I6+7;7z=+zV;sDTnGLxrE0Bq0q=7b>3F_}X1%HNIfB!@(UWwh=16N)_knDizk%!n
zaZtMVzq(GmX&rVCJgCfw-Q%pPY^D3Ud{${;OJoK*DGcb}V+ni-heHjV$B!T|k(YdF
z79d{QV{<uD>6nm$5mu^>_H167X{(mlbs-6WlP{&T%tIq|$6Csh4`Y3jPQOhrk7^;6
zTnOnTE%dQ>q=?0@J1PS!8z)N1I|yKO5V`+k07Pc}<BELuec6V>UTlSui4yAgJLRaA
z(aV2P$-7%tXSDes5fuXbAUJ<as4nJtp)?dfoguzx!6x6Y^aseLNY6>x9H<0KRzN9!
z2wOy$yy7}yKo1X=cM(F{$NO8yJ$cn|${!3BG_=_^iRv`p=!Y0^@^j$9X7UtV8wFtN
zFN8$UXtu04ui-qC5tVe=B|v!mrlEDa&5>}|Xtuh@(q)g<o@X1ib|a^(BQT~XlgyQ`
zU65;T=#bD0#Ci!&d@M>Ea9ONRj5jHd6-%W}*W(2GLOVoUF7v;A*-PWzL!^g4^tk*&
z<cu#>`k=pD{0JEi179TS4=-c5<CI<Y-qxm2giuBAJCk2>_Yr18w?!f&0}mvBqs72{
zzXy;t{fM@y`@XpIGZmRLsO#`+_2EPU;D$ES`d8orn*BDcXjB64Z{*bVA1;6)jQ!Bg
zk2GUGEq!qYIrvJAfdTDC4eE7e^1S>KJLmncN&`pdgockt->LN;%BmH_l*z^tQdew9
z1^Zr#|4i9>Zq0L$yg}?XF>`zx099p(hHsa?DeHSwCPOnY5<P?Ew0El4Yw!)el6bP~
z*Q*%uj*X}~R3kYVB-2dWeSR#c2!md+370c!>&1tUPw_9x*t=OHD1)3SS=`5g4{xu#
zt&kx~&cfNz=WyVly_5ejRlm?b!Fi_9_{-A1aZ<wgkMoLuVP&q`4?nf#ko?VUp2YUa
z{`D?sAn$@pQfLdtPX<l~X3+)~{NuIbH31wb{%W$VU1^|VG)UDKRjwWYFBj=W<Tla2
z3Xb0(Eo*x2bZOKnO0RvwDeksS_x-y`SNBKf4wpa7OdI`cth390hwE8o`i6H_Ozm%1
zT2u37Q+v1Qv;o+Fi4=cDq-?OzlGJ+E)}SoX+?gzgm(%U6SB|6nNYR7dp7PhyM?l3G
z3_Sdy9etDbF+=>Fh9HrkmXbPj8d97iY`Gmx_A0a7hSji@397#lI*&6M&GFZhhiJ#3
zuB(m8?zk1tNX-S*DTm*eOsSGUmPZBep%R4A51FR~=z_Hxi+)WMfM@CN#xp}f1J;M=
z4?)%1!W$S13On5+(zX{N<(M^2V~@1?ZT!P9-JH%I5RTS_(O#}^n=4=LXbBXuLO~Gm
z^9YP>B*_S?K-*a7Qy=|?iJ~KM1M7z@q+D`X*aROef2Nv8DSvYEI#iusu;swIqH<-z
z#*c{nBv0gz;;%#CrQmoOjz?z-@|b0(M&9&_nX!r3e_|QAZ2qN7&}?bh)h?i<z{)b-
zFU7z{(BlmT$qj-d?*Eh<z<LDp+m6$^P#Z$*yspM)d{9xlL>mLH>erH(ijXVu@q1UE
zx^RmDh21E(%;v2BC;0r*s|5bsQ@&3`iw$|pa_0ogn`Iz&=A5FOM@B$vplrW0ylAgG
zzyAzgf^(C8f?O8hZdZMa13f;h9D#ITn?n@F{+M~%K?WWy9;~~-&jG0^N+q-doTnK8
z{id`z6Sn~(k(&iiKa8oIS77}$3%KfH6xBeiVgKoU4I3DCIHgUb*YP75Lu62A<v7=o
z$r1!cotYPcofrgj`M}%~*`zl!kL<f?Ae-KKSdb`;DTN}#KA->2O+sqqvHlZwyu=k}
zhS})bT8C8WF|Pn2;^FZXWdl*e^ia9Yt?4k*$=4(yiG{N6KJzM!ziYs1TU|En{)*h2
z7|rbG?;s4`vUkzOX0^#MNqu?3BUG&LH$8Q-M>d8DQ29UqR%s(ZRoXyn2tw<N3E=p|
zXZHCrMnM+7`A;Z)K}UAs76BWhmr}>aa4C(qk<(Ws&!m&vzjPO#kb0uM<!y>uxn8QC
z9hw%7RB4(F9i<6%TO$Rf9H25y{bh|_EY%&V>Uw$9Jd5+RFZEtIRG*b#vL&w@By!^6
z65`<2_twa}+n6S}LW_OG56z%RY2Wf+PSC~yCo>rY2Yx|uo)%jAf0}*c34rwh<9e+o
zOw0fhZ!ra|XU|cAc>f&1?u->v5MFA(6@LBl<}@oqFkT*e`=&Ob&NYa-J<V(JvVv7j
z<F?4im_hvKl|6yk&$x)oZ?~d?JLkvG;Gtp^b%ivU0`}~jR-HATZ^%w!C72s2>Dtsn
zxvSUhyv$X_lK+-)mtYJD__W6Nsz&radD<Tnl?I_hpgX6Y=NP(kgpYBRV9TKuWMndu
z<pnF~*54p6Zaw;c;<O%Pv|9C~CSY3LV~&qQJwoFn)1<8P1_-X^<7G2;SOnL3x`+y2
zb)~s?k=i+{*4oaiCr`Neg)9PFuJ=kBE5Z6^-l&Ug2j;2kuLfLRVfu-_U4W}W#t{|=
z^sGlAp8H*?90h9O9beMe+@FTS8NK*>g=o&f(mn-2mehMG=)n80QXpU{rU@%xWYon3
zmiN!p+x&7au)GzunkV^bJ}~S`9Tj;WpXRQa)Tw}ql!x@EbNo%>3UzKf9A+xH>k<P*
zVjp68td@=d)E9K15Xwq<c%(SXf)Ga6ATaMToyZcTVHjET41S^BJ6M7{G<!5SO8Mh&
zDF~2qEZQR~n&El9QM&xogvutlGB!D4^TlSFnR5B@`J8~o!~Q*CeceetEUMZR;<vgr
zq6N6tKY%C1|IKKaeJg<Z_?%S<6r}`q8J|3zj0x&9@x(yIXcMa%$bm=3Da<>tg!>zH
zx3PoE4248X?q0Om{;Oy$Nf7}&7BVXTTL8tf?Q#)It%Q_Je@CI@$uV4F4=8vH6mjud
zw1vgu|M}`8<r9$fdn)L)FM?!}&O|7=nbx!8Z`bOFe7v&rn1+gR0{}Q^Qg4m|m;F}I
zb9!PU-|99L(!o230#_4f^kYeSGvkQdffK#-t&>%q`vJ;)t+Ng>!;Sp2OT*U|rNt-f
zy#+@c-_@<_N<zGdr!sp$1u_y1wN37E9w<uWW9F~1M3l4Uj@Hco-QxTccyx6hYp^$-
zH@1=jOntAqq~2)8|IO3&EpX*O2K%k{w?Y-^-+nhjr#||-3y3K2?yf7wwV@2&Dwc{h
z0VWL}z@+ig{KJ@!$n2AVIJzovarLcBZ(tbZ6c!VOn#f4@XH<^jEB*`WGK(84z13fQ
z&G+3t*qAjh@#l>&6d)Sk!+SA^-(N!NaPhkaRHfyj88Blllck&XwT<jGNP*~qxR{mi
zGKllmtVg9~2L;fK?#-Saexu6nNzs>8^C>n$vu|7p>_pHx@E$gb#v~aZ1<Bal89kxg
z$;FaPQtWgM`SiR!i2+P>(cN1DDy~E1H2R;`By1&6Jyk6N|0((33zYN>(+XfG0F4ZU
zcW8SDK?T5(<%0=HWb->rV4y(Z{N|2v>Ag)loEc#QRmN(<Ca%b%+t7iq6$aCb-*2gb
zQ<zx5*;d5(=85aY7_W!jS=x6PQ;fP>_%u?lUnMF|8l~_6&UB&K!GFeaSL+e*jW*sL
z18@6f)t1S2iwK;8T9elyF(?CnKf<6MF)*4qKWhBoQyzw+-lsfysVBHtLmFOZ+oiUG
ze}%NzwK+oq2r2%(wT?^w??O?9kb@c|XuzC7M0s;A1qFnKi$Y&>K>=Yj!Y67_K$!mT
zo%F(n{wV^K4`_v;E11_KR(T;0{J02+BT|8;GPVeOJn<BYymIOwRE9z-bXgP)A`qJT
zTQo1izo?bZ4-jygn2>Re0`vbeU_03uI0KM#ffV<!3_HXBhd3zoV_|<K)N@`2%8ZFT
zhHn1nD2V_4WB<L3m~{UwApiTs$p1gi8N>=lea{LPD(G`Pm5leY7|MAd`RlhV0Zs)N
zq{JWymyQYB@4%f&4BgtIK<C}s8AUh%B{TI26tR0&@mtBkQ((G2xPLwc1|fjjmK>Mm
z4YDB&W|ds<yMNqLes6&e-;fA_Z;J(#AkS@haEX8wCj|JWX7Y-aR7}9*hK#nq?^P!F
zA#+NgQj@gd%ve8yAEC0OVu4^bA#h-9_I~eWL|QOlAH$qKKvsf49pD21Q-Ff$^^Yv=
zQtaotNa2+)LBk9Ra=!!-qzoz<=(|X)4}ifs2h)9Das|F4`U>X853@~jf&vhkJ<mRm
zsntSSdS)0OFT#8OMjlwVQ=%?HUVi|VgUVZ`kut~(tMFgAhXangA&^?>8iU`*e(rpf
zVFw0GP96l4cBA~AQVIGo==V^e#DF+^W;8Y+Ig5qClX_3~paH(Eq+ih(`umU!%%D#r
zyuIN4+kwhv0hd5q^&dqCZ?Gs^E5SufqA)BE!Z@B5-shDcV78PC!)S(}K$i;m;6r9Y
z2O;kG!Dp~S%}NwmamZl!s}th$(C|e*gW=5R+Tq?W4?xI?0H2$`f^91wiaFuq5toyI
z(Y%@boqxa4m}xgP$k6wowQPdc(sp*XaScLXB4*sagVBer!MgvO4?IXE8GQcI`F9lL
z86d7B^2}eM9t+T1RO3r}bAnMok;Qu$+X(cu(@~bgzo(Zz0q;xbU@1EY5R6sWtz=o<
z;<U)L_^OB8gDcf3{J@dZ#y<JGgfRXs!2ZMJVb?W610-a{!;$$x^002m8}K20_rwX0
zke8;(-bcKr{3m1pFAko68vTlT2dw#{dTKF$Gt3ql5d^jD`}eO=gZJ0YBIZDJG|(pr
z%rshx!18kvSVQ(|^j`DFU{xN8MTFl^mIDl*TTUf|{NL~mdB7e$l8k&LCkhr(iV(H~
zi3vO}+DBFkjX;DJ35*A2tNR{Y3*PYa1z2>)3OH#a0U)@Y?2?hxPyG`7kcl3O-@|Qt
z!21&(J?{C>8TkOFYq;F`VmKRA3en=RU}F$~=?2QkWTrqPz!U%?i5T@v`Tonm;k5%Y
zAW~ELxR;0#tWSVA_GTa_Bx7|Kif+&ZLvVo1uLPnI{6~hLK{EWo{N=M{5kQ?nobd~L
z8w;$zhfOFU^oYnCFrcUi%z^(5$Qc^Ye3f{=7Y@L@e#RJ>2?V0RW%*k5eu^nzzI*U+
zpZ~Kw>_FtFzP2jqA%f|m<4%2N2cy}UyM!zmKRvKnwv+LajnLMCj~~(v%-lm4h(+)l
zQ<O(N*r=9=h%WaFj}OU>IQy56|K6M-1r}0{k>@oWn5WH9cfRosIC~+fSjP89a~zDr
zwh~|58~QRN!M<Q1=YuG$#gMR9&frvjpaLUVMnX@xpXOs6aF)V}FvJo6$w4ZV8%D-V
z2xbBUH=1>#b$brP7*jU+6)N<?wg7OLehtghKKplAyCIA3QQ<3gVo1({)kUA=067-n
zGd8_Ho4Hg#&Nh?bAO72@2S1>7e{5xH$_B<SH$hG-{~G8TOv2r6Jm~$RQZT^wACX-6
zH(g9^NO!~wJ7AE2@no7rrO3Smx&Vh0Aruxm`v_Pq<%{G)-3a%yg@Km^stg8)3nkPP
z2|WMaP=eZ=2FTg84$|s9@zN4B-*ed*%KzvZ$lz?t$x$3I1pE5<P1Y+GMrgkGbd!*-
zV+Mk@-4-17Z>z|^Lg#RzKe%lhkYOq!3g}vq0KwGEV^YF_<ZM3ytk3O7QGvgw3iCTD
zc>mYQ-250{pvx}1kV_fmAzjA(yzssdju33`q~ANS|HusvBsbZaN^%%rc?6@E{74{G
z)}=0)3I?zv4-{qn;)8tQzj`<olF9XnYS(!R_?h|}rK>xz^f1)`wg9fAC@_#yYsaz2
z|5gSpF8ID^#{3?jHNtE$#N_^;o+IagHzz@sYx=ttZE-D#EZq#efS%<|4BNl;f`sh;
zGnEGP#Oxw040IaF!CF6n6?6JKX-BLUm=@hX{v+VYL{MD{*buFwu(czA!c(_a8UuNF
z5iZCz(1h=OE0}#xC`B&v7-oqYQbhhONZrIR@aov@2oL^)Q-G!e+2?-0M9KAGPwU*3
zbigm&Ku*+g!ay@FK(<BU7(?>xCk=TbmCT^z`xwp+0~)G`;ond_!UTQ@8YD9BApq0!
z!h3)FKOkoi5>vNC9TO7<Upr=hi2@!w{mWiBq<@BO1rfu(|Le2){}*^S1bT|A5)nKG
zkT4Npv23suhAe;mra#UsnN{E|Mz8Ljd*c6!sQ&LC3vc`HW&HmjAQO~!t2@XuwH}$~
zIfY(R{(!7E-yI$CkJ#Tpp!Y7-7%*<gW&g%f;i0I?|At<(RU01tjg}Zyvc~w=v10>U
zrL628j=^6U-v1JcgWiG3kkd?e?=(jn-aE}W%<zWDn9NLgbq)~k8J|b@^c!$XAbE$)
z)!&ejOrNpq3A%=cYJ3yxQO-s8&NQZLErqM!7%6C`?JIYuTbE(!!sc&~zw<u9zNz@l
zAek9OHYdlg&V+ddNY6CiG*Uo?P(u^d`1C**V4~ljvqIs&o;NVFW#eH$0PiWOpMcMC
zh{JEDM-1?v=3YXe6ZmJLMzGNR(1dl)yI3uctQ3v49jTa`G8o*XSQ?u#U!=9!xq5W$
zbEc$v@<qP19er=qlbbP*wb=TDG`OwysvfCtnnCob2t{$x=#XZ61!XeG3%W^rjA54i
z+xOu>R-U+a@KL`patqv#Bya7}4*|^~hc0&>aE+>U?*_|OvRUDO9`6|ZAB9co@c)5Q
zmzqd=0FvySHqA<_xNg_UZ+euAS>j>J!RRFUpms_5NbkRvG~xQEY|jE7WRJ6X*POL_
z^M1mZJ<uKPWs|PK0N}c<1CLj~C4tzic@~3vG2p~QaWg>$j51)K4TxvCeDFf-$$B6;
zeCLx?3Ib6i4tPgRGs=7V)vs?K;&NfahGlt-3Qv|O4u@9e4{eXb*5ca#h;C^u-rgA7
zYxam0@SaFfEz=f^TKVO4ud>kZ<B^_1k73*_Oy09gV8Fb+yW(ad9u4zJ6nvoxTlk0^
zzVNB&o#lUl`-@OgQ3d&ByDP7^g8mE2pgSt3%H~+XCrl(xg)M<@cZUV-Oi&<&i-(uv
zEPcjm{_|@kY?uS+@S&aOL`KABigdCuEXJN15`vWc-00m}tq4XI876J%QhT%K6@DsD
zh2&G0+jBCGFY8^#{Kh|ZJ@{0(B&naI^~hPW3$IK~&MaBhbz$t}g3^s_fsVcr#~>M{
zcYz9);?D-ceD{~5>qYFPW1kktH1Jp7W`1h^gVR+ebcUVd#n-qZu<m%bOHwM#mBn@A
zxkBGKeU3waq7ElXoHmhi(;*spV`wW-<C=heU7;>3)8u-_W!7_(^*wPgafLTs>mcge
zRrS~0K8wgOGL{ju{ue>|Mbc6#*@x~V=^SNbc9(;2$BT>H8%YF*Du-j1o5DGAQM)%u
z@}vu5bfae31Yyn8Tg29xSTF7)W;!VT*Bt^?zBGExZt)Pn?0WwfC*p{C4|63&Oid{~
z0+(0B?e!{6;S}w=<Wn?x8i^tV$upZgM@fv)#YtwBkH6_+5v?BdCY6L3V0aZD+cCa{
zZ`zrC&-F8P$KKvwgF$IB$yzWnc=Y~^^W2~Dy$cagLeu%kBAggzN4q?v>B&&!Hv>MJ
z!hryeCSe{CB;3cc2VJ02r>eV1zp+k%6x8QsfijMBP|PF+{!hLCEUISF=gMY?X?2S2
zo5O0aP#8a8<F*=F`au4Ry)2yukc2DPHeKWnPJwEs%=c(SmrxxPs45!)ot}Ojm4S*T
zB~Wta-FdM7^zy;2B8pGSsP9f*8YgO3vR$QlYJpL8O!C3*^%;wJrTb^}yN1?PFEG;r
z^X*;Lay8e4m4+XCN%l4-j9QCUq*8m{Ty4(=j!FA1kJ_GyzS^m_3x3RBfW@cQEv_+;
zrs~Qm%)aj7|Gwl#<tSw#p!s^Aq<?3=m%@}UPh+e2QdQW2w$yHm)aS+?Pi}MMB~rq`
zx+eS^uGfTdg=@2I6Wu;7(bTp>wzJ&IMOX%-m8cdWXs_LpFgc&TN{l8v3(XDPt5(AJ
z#SRZy9&et(AGXacK6<0z$^Fgl;^gxzhkk?G@fN>azW#dau~X;Q7dC6CG?7EoRETi~
zq92u7Ckx>@ZJw^B`n{pBOLrh+dOV&waP2<y1@oF=oIm|EQ<|*e<WeT7OZ+27;7Ub{
zNHp<ud0gbq^jeK+SlRjRG*vK?Xa=0rEAHnoNN7HQyC|Z5oaup^U1tDCmFnv`_Rutz
zLzGJIXXSnmDW1PQ3lA?cMTfa&VhK}WRK7p~l+ig%I#uVP*!#8gYK22Hl}*0w3@W*g
zY-Ue?H<H;=6B2ws@21QyQ7ki+Vhx(mND&zv&sPd?yR0lxY;dz*rW<K}1gZ|RR{IjJ
zWIyI1?@i;FD6O#@GlG)4kcZLPqj$9LQ5@g9;7?di;<c_<lYD+DFTUud!)o|yv%B&u
z8~U11x#v<2Os9|X=hN%&vCA~CL^>9{f=6gg%&ORaH*q6YF*EAj2%gDUI+&WwRK2NU
z!<T+_lW@KKeQAa5(FM6Va;Qft0-dojMuN2k#xAY7^FgbPOf%NH!zzokR{G$2NY{G;
zPHmsHDU1mOosQC1o%kOVp2>^4%QVOru$yM~<wYUG;jCq<nOFJkoV}GYw)W&=Hf9h(
zNA%IAN-6&?Zo22LqX(~W#uhA!WE%6aFNM#Wn!d2YrY#S9_%!Y)Wbbpk)Thx6IU3!s
zob!F*-nI8~Km}j+aA`xGlcZOueq})ASnU8t;7f=TL1fo;*1p5V)|lm1Xhzk>`RR$}
z`q`p#sdQz*O<>0hIie5lG~_Bsx`K`z{Uj3|6tsB*(MgB63=FcX3Zx6$Ztk399I3tI
z*1L_-2x-WY*5rb2V+Mp2@LQRW8?0GjkVyMPl;1rAoidD<XIioS1SM{3_E8IO?LlXy
zX7^Rfz?K$UyiE;Ybf4>-L%iYrpSisy6h1yho^SG=wMY?B0F>|X-wc{}kIG(tz#aze
z9^0cXgs2eg*u>sHa$4=hOV@a+`ZA*4bx-;F{J^^t^|$HT&!kO2gfGM*l@anhg~b<v
z!L&a)4s_YvW9R)wv)*?yfdjluF&3W5PBX^uJasJI+)d4x9d+I(aE%$^>EquGO}VvM
zofCPw1|;Y5@Q=nuNjXta!pheTn3)2<2V?A_qKe}W=+j=JoTM~rA%$Bd@1{%ep&EUm
zaLH|xkvYIHyCK-ijkRRw32k3qAJ`*MoR199?fsc$RHDriwmBrguv5-5sT9hw+3qY*
z>^>D77fV;|N#|Xa!r?S-RDxbBN&)Xc(Kq%wUlU=#Va&3t@PZ>#Ec|<4d0_>-F(U4p
zSI<B~#}JhwT5WfC9CfJjD34C7kmLAZC`q+uPDk`WUFYrKWlnEJ8(o{ggnI<+Nkf`K
zIe<Uw|EMxsR9yJD`oUMP)G8V<y*xP?%Ejwou^z;ymtsQjeKGi(ZmGp>Z@}jAHY~>J
zXiMB`#G+xvEuVY&@DRf}>Et&Cg(61dl0n!5k^Q90Pm};&lX?6!QV%c!jPaPG0zwTe
zN5Jp{Hd72#Y-W?gd$1j>BS9MVBM}X=i9dRY1%W83&f-C==6cJ-Q7Zj!e0}d(3169K
zg%i#~OjQ{4+dC1j_0_^ZC!c<P#a*I{*=GFFl<xIXnK$livZn-!KSLf`-UXEq&O6OT
z+v>1NG(q}o`=t^C*P-<j=`eN@8kR4Vwqs5oDlKl<SSyn;PLrDDO50dg9Q5argD}sP
z3l2l=E&{~hf0uoCd!oeFNUPpwWS~4fBR5XDJZcDePgR(((+zI@2};WZKlB}|Ypy^3
zQqU=jKJVIP5`ATg#pIq`5lh!~6svGhnDAW5xc_Mi!K+mRWlaGOC03?yvj;uKeb=Z&
z3SGY}dlq0zdcNOv?RALe6vPJcYsH*dFoAwhWFd4#U9pIx9iJpOtIEh3Say0*1JbUx
zxs7@{9d9rEiNChMefZ*8J<MtLr$dMP8=d)z7H;)13++1--V)~q=OeyS@!Yy4hR-@j
zw=o7)*esNvSEn0P^L05_Cl&UZMIH2#hPBL6h4jC$tjmbEFINT-C16|<`bXgn+UwD;
zWBqBh7JH=cE~2Rd-y2Q#y#a=)M5X8Kg~X~4eYPyonTzlNpL~M+O-`bX1+_j-z_yoi
z;_oim4?Sts(GJ2M-wv>UTKfA>4D;Sb^sRTeF7?)aiq%Y_9L&986fJ6w=k0P{FfWQ*
zba|u7_n4ra^2c=Sc1_yB^*Nr<H&Jrmb@{s{gW7a-y41KZW3=d>wMQWj-%aZZ;l6+h
z1}^{E1MOx|`q5e&wr24eChYu&M3cpTg}Ma)bD{1HHRfw!zplpg_@1XDXVivr8$*6B
zU$sRlXO5kotn(3U6%7h!o|My|^tDR%K7K#BVasJlA6G)QQq#`+?W!t9bg9PW<n`oM
zQ2R;*MG<-}LC5xO=Y>i~@;iYzx0xniy_cL#A(IPNHFgelB}jQ>Xo;eDgcW1547j4?
z)eowJ9G4D#V)y7M7OEbz1?c;pQLT?oae=l0K7Ok~BKWd1jUUAm&&#H*KMy;?TJh<_
zMHbhe`>&EtHB?WxSSGKPZ6zb^PS)&2&APn!+2blaa8*CaoXg@-MULN>BF5EGuaHOA
z$4Nl9>ee8yt=i6S{I>bXd5bIT?G8!EB*XRok<^xvUV6g<W<o-cz*!6R2ve}H`1BF3
zld$*|cS&vxQz4zT1a7uM)|$$3-X2Qk2(R%s*pB?JuMy1Kn)&Nmw8q;4`249qWI|u{
zZ5!@ihPOK5A6^ZReRwE=o*te`ww+Z%nKhdpag;l>Ly;OXx#Ni5z$JEtEW_dZ%$VZy
zLz`0{or4>s<f#Ke3T0k{L2K{*lYuse=eZC0*0B<7^b>vuYpray-gq)69M$0nFhvkx
zt`%JF`uoPL&`s?w3gU~_(^H@ApiOB&g+tpy^4kltNA*8ON!-vr9ij=H(#c-YtLP5)
zcp<k}%KGagQao2QZjB<_0H8Ji@|&8aAZB_nXZ?_3C^dxlw10mmxEyoP^TB|qK+?G*
z{0)pRZBGQwYhlB?wY@m|8J#S}XN3(iAMhB2&h`_QIbzTZ^}i{Kg)7Vt9tlN(#*B<l
z!wKD*hql$|94^lqo>)IuMp8|U{Vs+{ns7;^n#t~6rOV;5V&C7#7e7chfnvJXwO}5e
zB#!hOg@Ywg_3I}j3dM3>G=5s3pxr6<_fOF{Dx}1RH5-J;N=6A@8QMxP((V)RS5v+!
zRU`GxGDE~-^f*vmX=Oln`}*O#&%jf@EX^)^o{SK^h6xy*!&2uI!l}BDyf~|e`T>T9
zKg!xiWv5N~e3L@kP{IkU%&gh);jWm^r^Sf*znev1K5#FF<>EJgm>%@<);U10^vDB|
zgxY7_F;lttu-)a&!v448n{cJjN^Mf3%?{M8L(%VVIy(KC3K4v&C&l(5Lco1v8CI9U
zfpE8*55jm1vd%ANXKb#N{#1GEg|;@UKeAGlXH@N5Ca7i#dnrJxQnG^jjt+i6xq<Q9
zXHNO+EKbXHwW!M)Tt$=hp}}p-m*w=!2^6R5{my3P9$Zh>T~;d*+*$QpgE;{a>+<xC
zr)wY_Hok*<=9~d{aW>=<)p}!dMoBl0x5Sh$So%i>F4vTr&aGFN>I&O)`nuNWxhu_l
zx$?o^b~s#ay}{o-t|8N6Or6fFtl6z?%L&OdbkuW05%#5->T|X2s7Tvmx1~^|h;E(~
zNKJwBRd7R7y4@|sqteOOSH?$Pb9qvAim|-;NCE+3PT0DS2{o?cY}_;dY4N2aOUl5k
zA0BZ*$oI7=?!NM7*S!S}yQNQ0m<F7Ht7+Meq+y#euzvF9?9Xh%<=I|I(#t&g)YoJA
zO3CZTq`w7tZHtFMBOTBqdxS6N>M^_dgmxyVEeHJwi-G@H3c7A9f+jZ{qaQ*-GLUho
zU#FoSlV3jY$42$J7P)#LkakmO5NPo57hUc=-Ve0m7Vk6tN)%K$G@(InPnVR|0ZC(2
z32>>q7?odt=s4H6CafrNh_uuT(`@gHss5h0`xP%?Qrh&UC1DVd=WGAo@s>3gZaEaq
zID&iAb}GY+>T~I+Q;Ih_fA^LS-$|kV<zglyE?=g1_)^=qiRsn4YRnBne@3K(cVBjT
zKl|!ks9uPRRwf#w*1U+&Ctw~oD4G2!_R4gd4}Wq>+kWiQgjt1Fjm2NVqF2JCI^{rp
zj)!D@GnRPnA-2A5d|o_gB+9rG-T~3rl9@1w>+7oNRqy-$nMa(%?YQYWkQ$p}ARG=s
z)7c4a?06qqYS{J8crW8!W57e)(oAE5ucbz|QvtM(WsC_pr=%FMB3P}uZ`b*+Ey`U7
zcLfLM|B!lCr@J>Er_Ii$&U+8qdtTKq{@j4O2qf*Rf76UNF+10YKhH&Is+^n0CgkmY
zd0M1LE~V!_`UsvrB#*g~a88}zyH#l2Sz1GXFY^*jFcnMV=tEq!VAxy~tHPr1@67TR
zxT><P$dJa`9unyE>b<`4YpByA`+lMPjlub>G4gq%LVbm)&RzCQSo7kyiLUj%<__Iq
zUxn&gb8Ho&i#OtxeW)(vGwFW_;2NX8h<EY6u0GL2$Q-aJQDH%@R;k*K_O7otz~@8`
zqc)MEA4h<-C3eex;y7wYkl(P>gCSkV#ISLF4Od;GHG&&!>(p~dIh61C3n4>kr01&O
zl-*C(&$MrYp+6BBQ_$x|j{LjsQ?kLM<LQzoHo<5_Yv#q{7{34)4Z9?XbL{4HxkavK
z?<>_7h4NLa=Ax3Oy4gK+e$}%dK9z_CNdX`{WJL`+!QWapmv49h+8Aq>8OuDYqC%@1
zNiy%TKd!*ae%s7v@W*vMbvCk5U9IEKR{_xKl3-}uCvRp?Aem0HwM)-+zA}}G|CVhL
zj_+;Uc=EUpM##W!5$kE(<4>w<-^DDPwOm%G5+u<C+5~(?&31j%7jI5%wSJft@Y{Rl
zDEW(!bd}C0xSXX*XDd<bO7;~8Al3vvp8u?Oywg?snWXks)I5}A>-FaVh3ECB!rU$-
zu>PftLHdcSXMXV*@RtV>vGy)MJ_{$ncW$|B6!ewW$!?N4d6MCSl}-0uQg`fN^{gQu
z9_m_TN0}4vV|Ufvu6tx1&i9bfYq*3dxOlwBOt!<j`!)T3UsM`C<LJ`Y%W=x>tOrJ`
z*}JLrB>e5p4<mNj36TW)&3UXO3SJ+yrqq-Y;yxX73r;T9FcFSVc+tpZH~r2M0ajtb
z1@Fsl)%?!!)(tb6UQ=J_V0!(TgyU+dXT9@#fx~#ByXSSSVpXV@HZIi0aeXviTG%0n
z{vu}ZX%_4wy^A;j9>sfgfhDCKk-2;O^;zTZeKkA}Fu1=SVC<FHlOjgmsswi_H#ko9
ztTkb+*SwjONxS+gXQws4dSWZ`=~9>LHmv>2(Ozog0HMJ0)Pnh3BX{jjy_@>a`YR7Q
zQx)S0{A+H^O2di#7*&gk%4Y`sHZUyz{P`g3x7G_!YpVe3ze;)Gwt$j;a}oWU_K>{g
z%x`lM1#IVe0*S%x5a*h+f4$hw*RpM!W{q(gtj5XF#!sEj8q_zB*au8KA4b@YGUR(9
zMm&Pw64E;BllfZnp7&^%pbz8O!NruPL{d@YY**@NNW$&#TAJj_0+}&MoTlAWP&MJR
zSM7MP{`T}8`tg7FbiwOXX>B-$px&@et;9#;j9=<5y17yhH*VNzz5uZn8FLIzMUU;1
zbfv$@U6H7=pNt+})U=y@bv+h~_X_pQ(5JmTuRrGJ!zGN3k;01+jce+ccX#5#AqsVy
zn;+ly4xfyi>&(YE3-HBLcyqZv3E!G2Vs0A77R<%+y$s&ABJrNrp-s$QN@CfP_gpG;
ztbsAH{meSicouV;v(Fl)6>u|!RfoCtPE+n+J?riKy3o1DXSwkcrxtU)d54^d?amnj
zpgefq5o(k*JmP;%bX53<QkPCd>rY3s7umSLWYBKL__{-qr~QjfY3B>aVa=z#$&4(%
zKAq->#5_#p$0PRX^~kEdlfy?=aQ?lNjk~dy0cZ64Uk^|@w-eaZ@?8I_gq8ohO1Ld=
zOk#VG)m74r3?Co%d&Ql>>vUc0<CpFLm_@i;UxwF=%`4Tq10=6>6^d)9R7<E%bVq7@
zZc0497iJTpMdU0fCUYpVqgb2&^7X!o*gnWHtD_+_4Sp$|%&64mI2Yen_bZU2^*!}W
zciGENZ1Rs!UhcdMx4)Xil*@bDoa`yp7&m53r+tk{sm*o%i#y_da41908Bfm<QoD`L
zpEAd@g|QCSWolF<y>Kke;@4c;78imUt8o5j`abcs+gU1llWp4z*9BWIZt;eOo<3HX
zbEv)8-e#%YO<3(uaUtKITlieVy33I}(@S(hyO{ci{`t5W^JdT|TlWdZ`pydb{^PkQ
z;S6LhgoPU#CX~LDJklK>|KV^6l)6pU;;c6>%^EfXcG>woOVSb&v|;^)zLX1Ko)6FZ
zy3FTQV%*ea!)UtqP-42xL|P|Goq8);ip(9Z><<PXZ@yi4d{(1n8Nl?jbW(7OAUF!;
z?QZpQ1=sX;Y*;_O=NX!TH4Em3#ZIpb`Ex$09ttsAwrv~k;xIcmQM}vzPdFRbtBU$r
zHohs&y2BowabQykyEUiZ_jaF%n>=Q&p6E3HL2~ujOh=vY(~#|_4ij)r86&yB$-jkJ
zRwb<eIwL}!1Vdt-ZvGRVR+#;NeMqH;cpMrBBz~Z5cI^fX60LrG&QaCSsi|Sh8sSUE
z<kR5Aq{71E&7E#`$~#XlZ9LRCQ&P`p>9XT#?!<mpP3=GCUGaX3@Ooe87km<nkVnFB
zv>H93zRZs|&+ry;UJ3tXkWsiAara_7a6alY7+5ktmz$}YStab;{HZs=YT$cY`k6pP
zi13r%Y8P1(H{lPjIdAUIw;IB@X<_;*4I&>v&Ol9y$YUPEIcHBl*-qRfwWMj<<+SM}
z&90luhKOm#w;3(>1U}YckV@W>W&0}eV{~sYeb`%_Ka23NMR=cu{v#_2i}n$<yT2_}
zJ{g=J2fxMkvq~Pm8!53lwr=jU{oEzLkzv4`R=+NBhAKJZ-(te0)n+~<w>4yhUbB|@
z(a@-ZApwDJK6|p2nPEHXi)5YU><LA#lWESgJgxU64aLLZMf4~$wI9@W5Pv;l|M?oX
zeMB=1==DmLKPou_o&!u3cYTzY8|C91IENXY`lt0<LtlLSM_KHXFU~^3Y@_BS0>*j^
zy`pE#QQ^0;O2V2$JC(;0LrLiBx=)6kPEPHwekgGApIGz1IwwfK9zR}mlF~@+WKfD`
z{-#QVbnHyaWWarTsm9D#UQUfZ{Kz6XI(PV4Uzo_3fgmD32`QSrU~u;n{g3v(GaAnB
z-J3Cjh>~d0rRZTq841Bi9z;Y45xpdO86vu27#<Nlksu*@P4wQvL?k-V+o%zB^j^oi
z$@6dLoVDJw-Y@T$$A_C`=HB<du6>o?RqnlOG}7+-KBxE}1mRYRpCg6ny|)j$XvQ{N
zmY>!Z5@#hMZ{P0LnQ}9D7Fl`RVm&o%?MFj8<%~@!#526rlr*N&o^FI`kNy~Qsi~a#
zcC?upbWn@#dLk3XQ`fs{$Q$k{T9`R3?YiEyu|4{=-6Xq8+%Yj+%XGnZWYdY$^Ge^O
zmuxA8P;7x_4ek;BMXd?VsR}&d5A*D!@aw1#&oBqLW3{1(Ox&NZVRA>E@o!r9i{Xjh
zt7SO*WbSAQ%MppXFoosvAk;0>+0u40pZ60+v6h&Ck^MKkHTa@)JJR@t3RN3DMXmy#
zvKJ0-f4c_(l!~r_K;HwOT>ES$wu;gNU!3PLu|*8=Y_u8u{)F~0261GQBtIP0SAMY8
zjD0otK^$eX2a}92@JX(m9x>f~9+$%PR<xv>?RfDzZfl<hN2+f=j=5s6dcDe2Zq+6h
zgJ}2lYItB<Ihu8lRrNYL<{AXkBHZ^~lSxQbR;=+>e2ZJ$?#K_m(!;8_SUN@7E(;kK
zX*sLX%8~W;9h+iOh=glu!?Q=qErZUmhQpN#p=ihAVD7M`i^zkX#kRUwcImuc7O>v@
zcQ#-?Uf(RL#(r6h_CHtsDz^Ah86EXP#=_&gRj=|hl1{$)Uq`PfUfHE;2fw9I7nc8x
z_Vg!<<v1hR6gZ%}9IpZ&zjHlT71MQ;^1^XlVMjb}$uVnaK1&|`a5rm(CKV=AA6n_?
z-V7HT`u0w15&E*4&B?WSaJB`FcL<d3u2JpQ89kdbZ!{U`gSHi4u4$O<KhiSrKz$Xn
zZ{l=$xUEp9WInicb0hc@FUgk6RT%+kjb_A3?IS6VR7etz<9TFjXsC35#JG2R=&hpy
zIL^Ae)w`qO^POg>%^MZFvR^oOwfeF^tlh(!u5b=V<L*n^H6v=i?|UV5cG7IH>!|Ke
zZr{$hP?-Jp1(7?<uB~~MuW#<nSW1mt+-JChwXotD5j0Ndo~h^0Moh#rB|KfO{%IHI
zQSqwFjG~X86!u`;T2(svyRES1Jx;cL+J~M+88!nGnU0(WeT%7HqnPSk%Q4inM|lLs
z^+Vu#qym@v)IF<JohYNmc*`<bli{nZUEY@G$wOULMyI|#vB>t02wyJxIbLfhG1Xs?
zNf@>mwHkfC$v>@EI0V7$e_x#Fn}x@!>gjs7EaR=#iuDB<jt6Zyt&4};rE#m~N?XsF
zz6%=fm9?8)M*Dk`U}`yJ_RD(49qDSUJfcsuIl7TOH;&aet6>u;$GhY!Ty#lDX?kwD
ziCm@xQ6@HVWR4d99Llfi#oo_Ad3D&!DTj+wOdEqHn8&+|4kFCet%bmyuMgbtIarn1
z=$qwyzsF?M=3FG_n$4@7o^o*Cg{+^(a$~6Z8k2#Yb?Uh~MF#Ux@24w{v+v5^JXB*>
zO?hYDl@?>{CE6UWd|a$+HqRhEBB3q+V9TW2dlv$C&qD3ngaE@;lbqLh_HD<_aY*0e
zx_u$8S3vu2Lm`}1GhR+OkH*UK+ap8?1eui0naaK;w+!P-1#R~6#?Vqjo2^T!x(2DH
z3;dnQ@Abw}TckZkib*tgzqNbeci6o{U9)?co*e~ucKzfX?}UlRvMFGr%?!n({m?gq
zq~%<47qK4NlGV>+nKA1t)pcL(eVs_#T1Z}1A>zyKNS2SMwH4_Z9@<qDx8E$s<qQAe
zsKBJ1o^v2dn8dZ4AGtcIwKlr;-X5nil$_PZ?*E9yIQ5W2`@4lYIZ;9pmMev?A4*{x
zsvfyv!i{oPm}NV|BrgY2ChldmfA4Yo5xul$%EQAE%R}gda>WMeiGh<A#PiY#VfYk7
zBk}Vg+u0gR*yi;r8;^u(g{0`IkLIlQL^cmwEm%c~1*#b@(E$lfl(1act8Ss)?yFdP
zfg63vph0I9@Pp7Rl9e@2y35hIsoF#V&P4YoEjr{-e~Fmr{dj!ca+t9FX3TwP^p}Hm
zafSFtKL`|Cg7xy+8Va0wFP$$&W9XeQWHozau#h)=Ln>rWel|>;b#nwKQPc28#qb%5
zqcn(0UP%UiBXP5KcWPAfUAEfbKur#F?Bl)B{^c9=?uVZ1yvPQ%o*U@1DRb@;Jat{#
zBev1SVog<xvaCcsXiip_Aa=`TnzHPlKe?;>%;;m^-d1Q#FC4Nc!d}gCE;2>Tsnj%-
zZ`3{5xgCF8ZT`$>qsgV-Kq&SMx;ZM-WvCCv+!DKK-G|><Tso5;w%kD`TzbvEe<H2<
zc1f*9uadD8(q!0>?40Y=f^}ts_AQ@dju|Iu9{+p5TKAz^T(hgz`00OXl;MKmI@yKi
zx5heQTB(7F^A>A;i+Zu<Dhebu356Sa_A`QEtjTue4=7;^MyS1@gSKg<o#$_IzJ*D@
zR^(D==Io=UU8K%yIJVBI385LUqe{xoiM$gyx`=js<YJS0kawdo{X<NI_D5qDiG~*q
zGrLPtaqrMEVbqol{Rc*37`(x@G+rRpc{^?WTYF60YfBEWg~_?99R^9FqW(VY%_C8!
zBT`LbywmpZU|;oYGZRbu;8Mu2e%)aZEW$~QoW=+J{0?LheeBt`ewlVuE!xM4<W;&o
z(&Y_>`cT+?-xk@=u{8()%q{c3al|V4t`N6(Yq0S`$bTttwXP5ckV8LRin^3>eOTNg
z3D$B$qrt%Wg+6KXgBq8dy}q2{N_lyS+KIO?o@&*%Yw!s6q(?)L1+jcb2z`mVuf39s
z&#gszW0&UChOvikhV6*7pJ&)hN5dFisUsfve^3-_ED5iwwJA`?wtv{@aOg>zb!$xX
z&bBM4mz_zvBw2wyI;MZg;)$TynAuR<S3q(|Xraa@-I%HLVimjIA<w4py}KOpU3<>L
zA_3{TQQIEfYmzsnJnNYxGv-o0Br}4P!5}IH3Nm~@1#gQLyYz*Y+UU0nr|TAWQ$uZx
z^CP_5&aISZQ!dX}1{XCGZ2dblk2eynDHo4@d?p+%%1s0R{2}3*Fsy~l+8MjtnlsnX
zuxe^#b@*i}50!KEZh9Ed<4*~CRirl6dLb4ARfLi|U)!S`ZBvoLZ+c*%af=GIcb0ZP
z#0IDwGe(6U(;TcQ9EWR(Q-n99<~%a{q+4=?bs#mCSbi}kdA2Fvrcr!N;mwaMm7=kH
zf#QzqF$X)zC1{sBs|uz8+foFt<B}&4x<*hImkT3A;`d^YfW~H%mwdJK$u1$CeTW4%
zctJAl43w5H8WBBCa+Z#~S6>nHNlD4bjHCSh?S<-<5I^k&EOvf5@SV^Bo0h2T*I&wX
z?W?ub6%&y~uO87i;hra=;j3<+D%S_cW&+3@vM-YN!q3i8$=OIsQoizIt>x3a$9%VC
zOc0j%z^h*MYw|U$RafZ!vv=)m$ehDk7d6kMLHJp;Ck`Aks36(zqj-iL4|#C9wk{k}
zfhrB?nFjWk{f7AQ1&AleKg6`T=FIoF-JD^QR9T~y#`S!QNfc8udLF3jlpS9``g!e;
z&Z0@Sr%JZb9Quhe&QH6gtY^?}S^<+h>Fri(N+0etn?5jGYWjH5XS;7k!ZwLWj}e1d
zQW_GcF_0Kky~4?^uS{Gg+?r?STKcB-8nz=tuQYA$LsZ{(b9H`TtAu?j&#f**(b@gz
zm_~=%wos`Ja!Kx^^r;Nej8uH+#ex`VYV#_a=i&F#hwU*@3kyxQYGYDgZ(1`78PjL=
zp^V>uXlV-*oLaof?q=Zr{Nh}nRFPKyassd6#e*?ezLKuj*+IqQ8o`2!1IO9w9R+7^
z(yilXgtZw7Y2{Z9@J)q88kVmeF+@cC!peWhKZ<8EZdml%)a70=Jodc8QT&-){O4{d
zjXe<ycRQLEk)#FHn=A(VbNOv#83PsFV$~plkkhN2YUo+$etB3!j3zvxidTSdH7>8U
zTItwxB(;iPprGG6Sy<RAYr+?1jNVRu-{(51bw&@RFp8G*#B0S+WfV?JiQ;pLhUwEf
z5$P>9!Ibo+<;KVRaj;v~XI5(7qhiHzdB)|@O6H<y0s)t?aiO)s*Pw)&6dtvABb6O1
zvXsJ{*()2XA#2@MTe35c*Q8Z{Z+iCL!qFOD*bry7x#yA^F|1g3oeClt6aJYg1vhZu
zRyqp74<!=X=e@8+#ExOPS&PHrpK0iO314e>N=Fx++VK`R4Yxg;5JK33S6>uL9w)o+
zmiUOCPkTGLKhZm3@4lrp?ep}lH((oL4-lG)M-*!qju1WSo|J$BR^^@;vvL{1XnHNc
zDFviQ2P-kuhM{R4g}6k5DTV(rR*vcqcLMLnp10^4&$D@OP+u`vuQf19qW!*}lp#BQ
z3to1c@v<*s1_HEARBrcpoLG`5EYd=boU2m+lDj$t4(+b<=!Jo7)IQZmVq$18H$Wmp
z5RA~KQTL!buS0cktD>}3)7|1RS5)+e_y)yl(?MlL;ydQzHavZK**S(q)M~T{E}xU}
zSzOBgH`+`NE$c)nkHXY&$#Hv~t}>6iZuqjm9A<Pa-QHZyLKXHu4C*MtGp@=(X`6Bp
zfGLXc;{VG@ZyR;HyOdKin>38c6q$V`iO=9k(?Kt(wl&mHi`woCmVWQ}&?BGZLRtib
z@?O|U+*iyArL|%>g?~|H6So$aCQyi@7rnkjkna}&pFtlpqc_aj;%7duk6rEYHmhl7
zkA%|@v)Al$iAf+&q1D3+4=6DiU`Z}_^3nfqWE=_!*)sZRcOH>MaXO7)|Nq@Ip4P<N
z&<At9jkyMcg#Da2Nl-145kvJ9k{DzUz<zKyBK83QQpaAB0M@h+3h)ntt3YRkyMvJ!
z8mrEA66)TlAhvodejrbTd3^o^HGy8~j#M;pEm^O_T`F=e^=p`GSTb7jYyGI8Vg(`$
z5B(`w)c=3Uq5+yf1n5)-E#QKsJhOg85F!4~vy%Q$i2izRv+^Zh#ETPAPP{~;bp@;&
zIUqj_0)^QAX)1Q2L=Pb&=isM4=Jd0_F1bR0JEgdo;yXkbMp`b~K}c8=KiBWu|N6>J
z@b7p2iW9Wp`IviPM8Wx0mXqQCH4Z|Fl!b$QaSl_+@t+_3<11;r2rdyB6`R|Df>Sd7
z{lZla26e)Tvt9fjZ=R>1CBOe8GWOqZ`R6cAxj-~W;8bKU{_&{)n3pFQxu-ad=f6J`
zjPP6#OsZV99&_p6Ncrb5uM)t>o7X;4{0}oG!i=1yC2vFaJTv(ZKK$e6zsddQ0wMlh
z-G4~MiDdpQ%qK?dw<`awyZ>f)5PzGhe;N3{E%U#5vHwR$^oVzLCe9)DXlH!<TG`-J
zi$w43>oT63S$U9tMIuZzBk=Xz?XAuK#rM%Gn~YRAS{!W+xMg}Re9Vl+kdQIXZ{E}f
zkV^?54{5~b&{KPcI8WNd!}m1XspHfG=5ncTo@ALc+h-Gf?9m%><L`_Qe=@0#r^y|z
z`o9-751t>Xbm~c@<b6d$48iPKk%dD}?EuDrg8qo-vUXTq-EW$~0kE(0Cz*`B3YkSr
zeD{9%5KI$1W|)h+6j`+{#bC8SW3D}i&!MB(%IA&Pd@MoAhpg$mL^^QR8Fp}LmfwTg
zkp_FwUD7S0DDw7sD1C~BTp<dzFA|+PZqEFAqvl$nCcBq5WcEh2$lo5(Zkt;g?XmWq
zsolV-eYyz3TA<?kkO)jub)?A|7mE&a`x(R=J79!rZ@b>eHUkO^al&Mxo|{%;;7F4}
zZ&=k_%1l8$b%=LZzzOP7f`W(3!{wF6F6H>2Fzb~z=CW};1$1W1u-S3?1i&TerM8&z
zuakL2J^GlRx5guTuqakM^1StZ`Ix8R*1ojHWE~l&U0&6~r$Ma&LpQ_J9WqitS(a8(
z)4)ah%QoHv1?;C(T_H_C?^`=Hvb((tqT`LdIs2g}r*?nAA`V%0ecb2RTkB0gRsuLx
z2>1Dza4?gZh!zcj)m98WZxw2%mR>Zmc=zN5b$mDYf^8m_;F+rA^2E^WwL;slZ9Z~;
zbABq4H&WnX8P4g4j3uLv332gk0;&qhiCVm#Y8oqU9jW`Qj<nqqpKG~2>Snx=ZG0q9
z7CTuVSe^I9b1koWS=6*K+y&2n{poyqjPQ`+o2;D5nON@Dp9!}%JMYW@3dK3SRxx`k
zKF9d`fT_~OTmAaH;nG<LUr#ziv<ASDLp@-*D?Wu?kdi%Iy|WYBp{nas>$TaXlqYn!
zA@<`|kJm=Kq{Ym8OSnH7&2~tAW6jPOu1#{>YjQ*Uxbt8x);90DJncJ~?ZNuP{Q)f8
z0ee*D0mRxMZXWIa-aOiorzlzcwJ-9~y*FnEY1+Qj_Ko<s!n1f}LWXH@;Rm!{hii?P
zBa^(ENS1N~_sM|xYAR^I&Yhz;aO#u5peqikLMUdkaWjC^>Ks#Nfxz*skoqs%wjs+z
zb|YhCc}j*4=O8PGn{1i>!xPy*wf7>b6JJPZ&gE+n64WEa+Ql^sT7|3s%-5<?E0M>3
zIaBu4C3S1g!BEalB+GEU=Yh1)?ljDYS^vl3#t+WH36rPnfo#uT;yTW7lHjI<=DO2W
z>|_LU3T~V5w!h4^EE}>28O=EK+B~en=Y{0f!+Q6IO`bgUj&rE>Sl##`chIN8aSS+d
z2F}H5Z|m>JosVyBg<;utQE<Z;WI?eRJ1dl`JVKhVZVsQ*EAoz21HC)693kwPA{WK2
z)SpW4;0IvshML+95Hskok8-<S@Q2*%HPy)JOIJjey=`HBipBR{pTC${FA4>3sdvJb
z&FC)Pfy={(ppGjyqff0(FYmJ+Rmp%M3pmlN=uEFBkPai!{M!9btrP300t`az*#zRa
z<kwo~9U6%%ilNV1#hJgqV6~qUB|6^Blw*#2HW|GFsI%uwOB-?nFX31e1J$f_LDF`s
zc0BR-QLNXo=Hq5y$5FyQo6IDj?00VMW9P2(6{`;Y$%0lkJ>Z|ZKl>}Bfgp{~DL(93
zeb~6WgFPOV(N>J7YO2I)>zXqF$=di6KTk8NWaD8_Sjbe2v#+jPQSda>4Q}D;eNNRZ
z?y+uG{yuzfy%iRb{nY@qR665JLj5e?WofwYGhrWZ5v^<C_(OtFP;~X^COGD+n=cin
zH|8<Ro|<4>JNPAeZ@GHKpxUmW?hLvR^@wo}SZU*;Ywf%({hLF<Ha#%xVYj*YC*)g-
ztbx8f0Z*UL>n(yoW!4-_F6msa9Ftxzh|zQDo9e-TZ8r5c-BC0d(YNi_wRFJ_zxs>y
zd`w(%6F!&C=njAQVNV_Fy5Q;<Y9&C|vaH(5#HUU0<{)o^`{j4u+e7vD^U^}U2<BB>
zHn0Q>LuL0$FlQE|jI=wmwewfEKwIIT`3lGC4ezL)S8he?Jf(<PV)Q-&;z)y6SIemQ
zDF#7W)#svG_%orySXjiZ^}9NOMeInEb}apYj@o0a=-8p-=HigpZlfIG{BphP525CS
z5@PIX>AIZV&SC)v>*sUE9B@0G3a`v~*RwI&`=*{2TS{V(>6Dx{x#PVb+<~}MrHIuJ
zFR0n|)>+in1(W9B>B^+!&<o_(eDL7HG<g$3ewAXxeJX5{Nz!Gu5GzWy&9rDz+V_}p
zF~4fkcK>ILGhK33s__I8E^pHKl49}QZEi!<B5aEx(6Qh>)g)n)+o!l4PPqtMue&Fh
zOD{a2es2E)hjvP%NioF;93J<DD!vmO!&X2Yud+$$Q6lV;x<9&to+b3Ohhonc=#or6
zKdx#odugw?Uj8EcrE7E15ENt2QwH84OLCg+cv1t8OERmTqm$-Pt4L_);hR63{^WYf
ztXVNdN?a9oX;+53aj}~JqhO>Q*>P4J!`V~YzCq6B@$2=ZM2gQbp^z+ZA_*J*-LM`o
z#ODm5w$5Un6dsqSDzk9ewsF>JYp{?E_c@RBp6u$uLn6^UJ3zZVyZ1mRx>aS91r7Yx
zTkK*2aG4#G=w7NyjgJzvyL+>VfulxIn@@^YhZkpnUm)K+x;^{(JEMMy$0$tQ&@~iM
zo=`jRd98sJ!%io1u5;nQw>@toTidHmh3i=}cTA{v;)vV0_Y)i&dUulHM*!2t6%bL&
zAz!;+brtFOLha1ryfP*;=-9|k5=YHG$CEaUNLuI$xHE;YvLhK?oxhwM7aYzQ_lfPp
zv#%7(H{fILQ(rPLWHCmb){A83B@d2bpG^iZov9kq+*`=b_tST%ktneKfnk3MFW~RO
zsH)JQfq9;@;P$bV`*i1c{zad$v)NS4fzlG};~Y<W#&xo)yMywUB!(a0!&a$ZsApI{
zpq49<4c0HSGjCl{7iKbaeqFuSw>NhNHve=E&njkd-+knmu!BAN8QBP6jI6;d&H*dr
zA+#KXP124#pS9z=WWzC7XXOu+rqOWF`HdCX4%Mbgg0RIBgY+EIJ;#Pjdg56{P}x8X
zZH<q2HNQFn<e28Kfc*i6pgptkjYeRr6=LWDL#NEUV_76GMNn>D&A~UTBoY31S-}x<
z1ov{7IY)Fv$j*Fb;tM&p;pM!-#^iu9yp8+ks&PJ87s8}l$FIwA!}-VKkD$6AYH@cV
zLFD5i_L#8jwfK!$IlE#S96pG0A1iBhQHCeSzfAjC*2T1Of&9@quDrp#x=D|g7u{?p
z*_N>f&bWPJlJq{LZJ0zqxm?bng7|*c<~F5p#e|P^_?{pO`RD0qeTcNbmR)?5sB8+>
zj+<)z6Czy>eiYnm1jHw+&f^IElOsws<y_P~g^xeGklKbWTIE#`KV_u$M_D6~P+EjG
z@o>d)Rl6}soMxoV_>-C?H{<nnca_s?5J`Sg1XMNq6zn65WZ?_r6`0SSO0y@;tyP4M
zQaXx*bjOR`)u1L)YBTK$XkV;4ka|n2aYFT@$k1`6k*l2O#MyW^qlyIf`c7E5cZt~2
z3;ceSpze3A$Q^`9i8lqbq0~e+<~(_uWpzbr3huG>R9YWWg@1ORtBj<v$`0W@C+KhA
zz-)O=E~`)If!Gyp22S!vGPRVWDIvlvbM<2nu3qR*9gxUVFlg2URloOAkG%_bk5+s*
z=k6E~_slICuNRV#HO2?)XCQmy)kX(D=ZmszQKlr|<y>BG_~Zj57v!=AfsmUCp058;
zZG9H|2|DGXB;G1o8&8ihM(=$yEMX#lM8jnqQJak?a4*Y|vL1O(6W4<-S}8A8{5NF2
zc`Zl<2T}UDT&2yjZz0@Z)HQqkZKI#E^bEv>=GRX0)t6}-54CNuM(MuE;R?+h=%x$E
z)N5bE+rH)?y_->7>SEj|8VIJgZ~gc#mq;e^kMi8bDx~*%GcQKtU2F9p+4v=8-gtWS
zE9=gYv<p;F8xf;)x9rx!U=Ff#7is%@(p6$iTHf*_KecFbtpYbiv$&`=d2`plkEpWB
z<bcL4Y@^QzpiZZ+TN2TKmi65*UI@?q+4xCn?He5~J9ywcg6j%#5k(VE(_F4X>1n9I
zeAX)E%yEsu)OVh!Z3w38fY#U;6)|A!<K4V0eoX^~dyC~@;#;`Or7pd#%<9xGq228G
zRt`wq#iQ3HhZS!UvT;u7Pc09r#1)gqjg%RG$R60Gs}6q#4Lnp7lo}`5O2|`2txlw1
z9BA$;jq-CRXbX<=@t!Pij7ga8L!(U<wUX*3&{M2cdylq;NNpvr4oNLsGy&l8+2Vb1
z9Ul}o3wf`C!^iY!0p>g1WRR%GYCw@2C|{|XK#C|>j%!ULPD$CK<a4Yei|Xy+`3TlA
zl+Za|?qDn#p6I6+Md*3w+m&)o-sc*NPi=P@p%h>gA-#ZuY9P$2j59l0{6wG5+EPIk
zdH1^qX9({EWo*`8rLEV~$U$3%Hyf8YB`)bIHK2F4E|70txRt>V46AB8JwBLlxsIy<
z`Na0f*Fj$s#~pjM-bbBR_2(oBCJJIoLPmWKA4KUXp~9tp8+|C(9@%HaPXiS=5Y_YI
z(Lk9}E0#|>Kxyx)`PxC~Ua7tsI|%xnR%H<^=)6rCuyfI|cy+;Qm=bEUuIz(NrI5Gb
zSCcz()@j_w=Nqdq*vExKA>#mZKQjiF)bhJ`pJ$?b??a`4gHE%ct!VVoe!vzLyHu}g
z?|l1|=#@9yN?an!FQ&K1X~}bxVD(7vf<NU=re1@|NAzb<&H|V<WdZ9CEY3(7@ab|~
zT%dr416+N$I<VL@%^d++tcEj&(j}^k{bI)c7)^AU`~;^R95bn%=DDQ=tP2Q0MSx))
z!u}Wpr85wC!698~0lbM`qOQCwsUu|ZZ$}>+rHIw!6?aHY!`$X7m)|%$Bpq3OsIzIT
z-H@IK5>~vszrieS^=e#v`+Dr#BxOozy_7eJb^r>nk6v??*}Awf)g<^T^U_E0UmCL=
z@%0-OE42qt%Y}?9`t@upk-c=Px-Y`6(ed0TkC3$PH)$ZZGYd&8he0s&x*3n@M$3xu
zW$B6<de!K)F0)GnzsOA;HP9L)x9cepr04<6%Bnw&>@zXfJ>NvR<@H9;wJACB?9QI`
z1{=subDf%vo9?X7$xnq%f_yiKW=qC<I!c=wP1u`_)RWzxOGTKzJy=<D#2}~wOBRb-
zU*JDDHZXI$?DF3G;0~hJ!yb49*tEe|r?Fq(f&FlIDgxR6UdqL0b+w*`%rjK53o&tC
ziLJ|`?K!tVmUOOM;ypU4PlOXsYfui}PI0UbHOx4uo!dzTVO?Whrms=DBLT#hf|6Y?
z!?mYK6fN~@kG663rajLdMO85##IJxjZW8WTue!HY<bIX*U4V`30{9i-%6Ic@71DeF
zi3bvXj#1yPaJAkU1S?LaTkPJOT^O`Ai>TS%g3WG~!2AJF=MJpJn#{)}tw+Wnk~Eei
z{G;R^yS&HuWovh0eU3Z-D7Kj20^EwKz3~>c3Yf*hd&;aZRCyuqNp=w<PD=lBeY}0^
z$P4U0;fCQSC4$xN{!FqR$eY~ptl?lH?_K*0CNA&%*aY+>U8}fvk68}w1;SaTioza=
zIt7iHWJmz8!k!h_<<V-gv##?x;v7kyclmWbPN}>7syMNfHSBI9*WFj^$VgOIv)ZJv
zHAc4i`UMLES6_;?iTL9(BiY2GC&vlfl5R$y)Wmp{{GGZ?yMMi?lIyK~+m)G=-C@WB
zwg_;V<q|WWwZ8nSBBhz3w1yB;JPzW@`T@7WCxI0L_o5Ye&2?S+v_>80WRf->b$<sS
zew62;frdQ08fZDf#B8X7#6kgPt}M#1T<A#Pm)tuEYSWQ?5x`=FiU*)u;z)0L=fwJy
zdSp&>(+-|M=^Fqg93Ze&QA|D<URoXDdM9YCrmvk}qad7Z2+H8$e*VN!nMhIoQBBxC
z2%9tAJn%LUY2mDW0S)f3Va>g=UmnR58SJTVeIajd-_a*K#MS`VnFeIG`EVe_gq@#d
zN8}@i(71DtDqS|SjHh$bgk|~?!_`nqJAm=RCS9>dOAe-DSLF!1Q(x2)vb;mtZi=qy
zZ2A>lORejyn5y~}sUpz-5FY}{S5d9Ut^lwMD|VHboxvb+cT2<1UtFTjS^QEF`It)`
z+qp|)W6l~Ka|;S_d9ub-#ccWEI-ERfiicKb%=4B~QYA1GStDpOx<}iSTBoux4-<L%
z?Pa8qOw~$fO$UHl%MEEgZYZH_dh%XfzN{=q$dPx5&m&*;AUcr1K<=3J)!%*ZB}?~X
zAldi5MMvN6)IvC$b_xI`kp1TCUhJN|Yc)V{9pf5>HQlVu#T7~HYh_);IBvN=TXaQF
z#=DI%=M=RFAn%Ja{(P*DWQe1>VczoAWVv$I6*ot?$NZ=AbhiqO3jf&Q8dCq`{e)XL
z)o+Jzx^*UoaP+m_dH+H8treI0N_NH9i{J%@J!8caH=gFxN#JzX5OCd$Nvfc6KZ6lM
z8kp~RXr?>acJ`kzMkIEdUGk!xbFk-8B%^jp-Pg1)=FZi+mhpY@Qy_R|HT<w!uCZJb
z)bR1H02K1ZOeY+%arTe=O9Usb{ktc4wf$gurU~Fx%u{l!^_SX49^4fm`{So=>@DHQ
zRvT`WC7sWs@ksGQ)fQp1M~!_b$1MINY;_TQ%)VfBIO;YY^_gSmDhv6E0*(rNvTS>I
zUHI-Eklyde{$%s2V-(0S`lD8A23*$S#jQ1-B1H3UlUBZ!;2%H9ffWl^I39%SZ*L74
zlT|eiT;*Gjq9euVL4nAp^!p-#w5yiqT4kg+yyS!JOMH)@OlEybwauc<XypEc389I2
zD0V-JDQsg-Y{}xiIisY!@}5*6C))kupCPO&YQ@`%4lUBau{Nf!i8v=)<?~(qG_`RM
z#EQQ?<+UqJA~n8F9MrmVKK`TN^5>R;SrA>`mjfs!a;cf;ZrLQmi4+k-yDz+?gl1?-
zZUN8p8KfWGW9Dn$+UrNfCH33u%b>Q;`5Up?S0o>p->nldZ?Lu_C9U#OzaLRP>Z-qp
z+4YU<fB5SD{4~omZOk3gjues=p7tB*(`e(CifWLm8_l<9>7MV&RNIX;&IFJl{nNpg
z1Z`e-<(A58jr}MNPHp(0iT^WJb0TZQ^kqDg2<es=qGupZTr;MKWeYz^OvXmr@769c
zP+_jeA-QcbvQ}^om`MOd<;@+6C<G2x1x{Qh5#+aTjzFHzQU!TQq&?M8FreYREi=^h
z=eHotiY%T)zpL<?__Cix>U!&D7T05sH4Gw*?!4q<|9t&X|2M%6e(G)pSo=Lw@UG&B
z4lCM234%HMbI)`Kr@%SY(BCO4_u{3oq+X!Wn;;+6*&KQMRuz>0AP=clEB+*I=Tgrl
zl04rRTqyf{LVL6mEC~^=6MVs7lLwL6C9uxF&U`AxMHl@7%2O&*Fim2hB2@Zluw8rf
zd7t^?_ujGUsh@Wd6Bsg%i&i2-q>MUAX|kl<;!!Bn6nswSvCfl#NMxyTm}XY!t+4#b
zh<w0K_RNM*9p^5E`lZ8d!zyPA4iLbuYl4i;_V=PzVf~^o&Jk_XfUFZ2DtzDwGQ(~^
zT938S9B3@zHNvFw;tzmpg^2AhcSoji{FVk5!7mb@3&$3zh$#+l9~Vn5yk(;tv{!C$
zm6m|hn%U7d<dk&ZYj7RNnSY7oi76!JF>Q<xgRh#~tu_wk<!berS7TXl5(8e_LzWF^
zzJ<n9w4CIc8Spnisx*wsrpjfJtH1(e%HW?GsqJku9>gz%rvq=sfWPb6b!H)y{f~b3
zH}T9Z*D8NSHk|nyN@d24Od+Z$B>5?&9k*s-H(+4ukLJeXLS=ULL2}oy=+=A?yI23<
zqP(dkUbI!H{6k?~U7D6kH2!efx|kuX9pc>hw3;s_M{|je^{NC&96YePOtbe`v~Uy?
z)*+7@a=Z{m70n_YqSc(&XQ7v=UvAIc>H&HZ$nUFRDs0;}Z*gpg{Z;s6r|XnB1P_&f
zTh|T#g1T`I6bk85o^<eV(duNg;k3BFsOo7M<eU~3)B}hH-U$@wF#JyBvr%{*AMG?y
zk3MorQ3-z#a7RAL8W;h%s;0ya=J}ick)_72li1RO5d@k)a_bBKRnPeKSE4;2A^>4R
zDJh(J(jsu*66kS;6G;K;_p+T}g^cr{79y_NH$rxgtDq7GUfTF)iSn;vC1!>ReB}^1
zNN<TK&&{ynp}ufBy??*x=>Ywql(bKH5N5CMVM7&ulCb|*vGe4wU_Q`n@iMpL=3o68
zfAQy356S5tfcqz7KhP=u*QbMV_)>s@xm+eS`@8t^A2)KT-vFhG0NdveF8;SCg1(QJ
zAlwfbT=*dNTfKfi?62|?ViNF@F|YaNlm9x^6TVyqJr39YS>gGQ8;KwRbYSr+cxL}R
vO=<j1=)aY6|FJrMujq*+|6fEGb4=nez_mhlBb<>4{3t6t_#^MOiQoSLzo2J-

diff --git a/docs/figures/arch_6.png b/docs/figures/arch_6.png
deleted file mode 100644
index 3969511041e5fb8074316475a6bf74531368431c..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 123034
zcmeGEWmuI<`v(kDDgp{Bk`jt^3xYHV(k0ypNJ&X|OGt=HHwuf85*CeslF|**N-t{B
zz2F`8R#^AG@BO^T@qBr{{5cL>7qjM?Ict7r%#5!U<t46Rkzk>qpj?xZ6jMS$xon7n
zg5HIB8MNG?x$*=B1*_OxR8&z)RFqQD&c?*t(ijCr@^y6V6-8xDLLaz0Yw|0Ut2f@>
zbi0*tOIUjG!yXO8-K%)g!M<sR5(Mhcn1h3TuIWPPDhPy!LM}0eTw+@EW5Q~K8-5FH
zJN3D18Gm}ZAY`*P4o`Mo<5`%jG@a*1p>rtENV?95^7d|9%&UV64kN>vxP1{H^i7{&
zLCn}#YLlQ>7$_>43w3QlEGTjv&Yd(Kh^-^<+fQmKL$9FV``O(S3XS#Mv~~D0|C)LP
zh4ScW&Dwj*4GevecovacVwI+Tky$R!{ch0``^T5O#@rItyo7=l-yVG*U3f@zr{j*^
zgQcLA*Tn(BtoR?oYSClE!ej;8%ygc;8V!D$vfeFy9GN;>w=WxYFl!fS%wHNSb=L(8
z+6WsrK3%XqwrQ1&m2Y7azQXflnl!KrXS;(u!}zdWBZ>&0zw_})2I78}q$gEYeC*r}
z?oN+_tU608H^r}<v{!3*9te#j1)CE*cMVX0-sg}dV3eBvmSR%yoTh-2St(QD#>?j?
z_R9A@X7NZ6-RA7nH2-zN!A1U{MvR1}U7vE(Hud!~^O(E%Q0gw@?Drv)EV2Ez-LB+T
zeM-ab{f>}#bIpV<Zk<2G+E$uI)`~TzxAEd@C?_N`yW6hD=68y-&UTFAjSy$rAs%NE
zyP&b+`3TD5UCr|m_>94Xhb3J04Aw|?i9CMV_@;rKG*@J}TD(u|jg5{4wtS{MOO&|m
zksQVi7RzFSyUp>$x-?p%dAMfDmhrvRF=#zHC_ZiJ#L1H^DA@-noTVEX?~~{!#CTAd
zC{Y>G&{fl}I{4mT6WxNM%YVJr`2!cj9}^Fo0F7J{l~@=p#^?4kW;&E7&p3WuX83CT
z8F#>^;U}Hh<*cu+FNt-olz;PFC))NyUB?RZ!F`6~N69bdYxb5Q;}T3X^X=_MKOIU|
zIutB_!w(pA#5YBh%r2E&XYi9w!!l!4!A77Zzv>cI{~*73ci_r35t|RJY-o`Nym_dk
zU+oPDIsy^DFkMiSa99KFe|fl2>0DQD2>MC0&+kE2i@R4}`12_Sr5z@EuXqO@!L%<1
zm96B>dm?Ig10;!+!Z|7PCHd?KVNU|zbranQ3^Xw8V3Q}(m%Setq#33eK;NwL>;;=#
zTxeW%oL~p$JYS5TYD<D4S<xL!94_K&3~e9g2J){dU-jxITdqG7`uXzF<t{wah^?lk
zX(iis+op=-3h{9s9ohm4?vR!l)s0LSQxCS3(C%i!pUyvVe|1m8e_mNn#k9j<_PhNp
z5l;f|#wMomE8cqIda`=#dZMSk*$NK#FW)+7G;Wk@L^P^RGf$(~hGj@TxX%-LPckQ?
zG{a5_9d<J(Xj>lqNOXy?sL0cmix=pu1&`mZ6s+uiPF;L;#Q%;$%zN(lxbM5)QM{je
zN{5dgLQ4JSz1)ao4n!x7AL8*ot2MFRe)eE9caCu`aTfl)F7sy2p$eDd-G{MS;sW9|
zPsr3L8Hngy=w9AdqgP=_Qdv|ZQq7>(Vz|s`LPx+jM6aiMN2Q*Tn9h}cBYHD>mtI$K
zDeEq$Zo-vt32ling=#U<w3)>~hwB2-dSXJd@U#dzM*1;233^u50u{mh@pfUN%Q)Am
zl8av3M)fxBKaepZaF2O56sUO(y4G=+{Xn}(?n#<bc3-xg{#+=7@}SH^Imgf!(4|`s
z!xCO=Wslx0)qN2=5NO(O%GWvH$&%#huuJ8rF(D=(>sfgyxI?;>PLe<(NMaaHMncFL
z_>efkIw2<kl3>aiRAQ#ZqV+gmDIcoE8s8?%#gVMp_2v7WUbW%f!I&5D!L=Upe)fUa
z1Fik$1Fj!s`ix;Tu$ZMX*qcRj*rlcXp4?&g421zxSk|J=u+8wP*)uadtIv~ZBhPvi
zo8B75S!me&FcM?8d#GipV&%|HWZLmSo@3J7V+@0<=%EyQ4riq?R^n(pmYj2fQ`fgH
z+c>HOWr3`*Fw17HcJ^HEWe(H0D{%|)+=+E5x!l~=YcZ9p_I27nv@*0;%3UmTp2JMz
zO-EiF47D|en5vpPb|)CFTGfnJH64bB=Y)SXiS0z!POwT_bsOUvvvJyWx><d>`mVdc
z5ylb2ku^c`?Ro|iUE8m`6?}P4kLK1#db`-iHnL|Z<W1zw<sIeyw{RA&&ab8p!}Df3
z94D8@m+Kv*UXIN4Eq_{VbIf=0oX=ctcINxJ>u1#=Z%koW=T!bOwj%ZA>Ef@&qQ03V
z+G*H~=HkdN$n0&H>aU#v?H0a~>Ph#;mLb#1ZXW%-!a6v?l=Yj-K`l9RVf!OT`P<ct
z3cI*_h3nn~&u&s)HM{l-9f4MUc^3nA<+WdezX%(xi^X~*Uc=j_yLExnk3}Cdilzo>
z2Iga0;ARD|VijVa;{75oqMx?Sc$&dk%p5{;n?jXzhkcZz>V6fsiFHR?+;Y*2qWT{(
zk34i##(gV#5A?RmJ1K(8!&!sjTF)w;!TfTu2MID+`D=o!f~yxb(75jMeqi~<$Q`HM
zNhUXYUyb_BI={=9YvI22vC_%gubPVYT;9zlsl4ZmTS@xz(3P&~A%w|TWASbtUai!Q
z+(hyrJM*K}XQM7|4sJ%5?^4~J$&i$aqN$3@iLc~@DqAwJ;O((E>3p%;2yWE3qr`60
zdOP@DP9ia#REY|P+gE!hHmNO1LFQx39qmsQDka;^4<e08V`{$dbRk@MOn7Wu$FNp!
zTc~g6KQA1w#?p$e_+(YOwsbs*Zs9RBkVl;_r|r(seXsC798tCCn14WWV6l1Ry?*ea
zR=8GJsi$Vr9I-~hE`;U1^5a>JK}#S!;JDu63BeQNC;FkTO~YQr$2rGY8{tx<d|V?|
zqou^<nkAJMq02D|H=Ev%zdy<M)lQU-hXg?2<;<f_r5%};dro~$u-(Z`pUoIDH$E=-
zzScV>+q%A*;v(sq7ovkL$;ZjgPL-=Ke&iQbo9)F4)C=J59<Eq@;>u<j3mS`s!VgZe
z5vxK_uL6NIr?kbd<^CW2YaSXrB(zwyuvi{g<{E6+={x)&x)EeevCBW+_iX5M`MX{X
z2aG%K$XyR)>RELq5D{f$Mr0O`3cSB8VXTT7+!`p!Gtc0GRF+ehLAElcTFcb3hHlj*
zoVu+??dG)gw`FlB-%V~6;_%|yTUq4UsQk6nx^Of`P%2Q1Xa8%msV9^y_Lx<-a>euD
zF<&)e<@N$&RspY`fNMAqXw}3{CTSVB{CG)?>&J$IbXW``KW6NDxi;T|V@l<W+rqJK
zcXce>c1^dA!l4ezSiCIa?BNVYKk@$-1jC);Z*%3`b&<m@%UXY3A6cBWq>$^)agtv&
zT(IYmYC08e`m39KZ|hJWk+O3%?0dWrwP4IzqBn3|>skuQCV3UY?xfvUeOle9;nJzn
zvHGAz`f$tX%cGfQ$&T+|Nm~;)%A4RSQ)C`CTeVi4&$nSgVYo&V*g`p8pAI?>$4{sa
z+e3?}^*8j}>xjKA5NoG3ru;@8Ivvq3X|+)?$X+VlM6tm|6MlWk$B7iB?>HyLO(`ws
z)JF>K^mW$Znp6?`GktE98||o=6<5E$!770~K`76lVt)AOMvJ1Y-QMhQ$~Tg$5L|y3
zWfw>L^5`XgkCX1HH#veQ3a+P~mv8A=eVtnHMsE(noZgygfrB76#8^YhL{1Kc0X$=(
zV4#wqTmnz1;8zfp^!!;Il@0~{uXZ#Plpu2yjNg6a!ROh}Tkw1K&flNt?*mb;fWK~n
zUzaqrKfNy-rlJ3NM(+aOp@=ApN=boFWg|OdV{3ae8;7@F7=%H?)t8c*_9!R>)Mvk_
zQcCyN!T7`GDjE(Na<aTeHdZVK&ut8iSzN4Mg5D@7{4TuU(aPAtfYQav(%PQaMd03F
zJ$S+MSu^WB%D=ieSP0zHkW-`-wXrj%<YHlCVY?@YMM+7?Z};4US4m9#_v_#<fqP~S
z4lj9GS)HAoS)4goZ0t-~*?D+)SlJ%1K6t<kdNA9&T00oHFk9PG{XNK^am0-6jqJ={
zI+)v7Q=W}$U})p$AaL*A*+l0*f75B~Vtz4`wf%3l0E4V&->|Z?u(6(x4PNCxYvolm
zcQLlq6f?I1G6Qo6@^EwV|JDD$zPXqY=}nD`H`%#3{^M1oum1U}s=cwDsErkv(?Re;
zU%%%?{_^*W{H$lJBZ=Z~IserP1TBcg&w8#jK`giV!z`f3JLX~vD&P}X+1U^30{DmS
z@6WU6a)-N)RE{Vp!YEQ=A}TJZD-)M0o*N^stWVWwe`cD%Wa^{O=M0w4iFzd&>u@y~
z3+pBUrqWF;$sklrNp5dpd@<n+(wMdJimlYWpE&mL$<b>2wSfixZeCOQFB`6=sS;hz
zQ+p-1L$rTj;!z6wprE2*{P)8rmL8hmoT?nOtN+?3-iNXo*Y|(F4hDQ1=|f3FtI>4x
zU+2JJ_d@?4Q~ynjZxRLuJI^<Un17vsGI)dZU&qFry-yk3dR49HUnf8n37h)Yv4tsV
zQAK3sKZj8Nn}9Ge4UaGXx7_};@La_-wDVJC`k%r*>m-af!~L&g<9(MB#*4g_XZY{Z
z_amzN*Rg$s<<b1&lAhg``4{bC;^8&!U;9_hV%XipYn;W%)%Xv~$Y%AOi4YPjsGLuz
zSSfBdYz)3p$+=cKtQYDytjm3!Z{_oTGk5Jf88WPMp_wrODH?^x4*iD_d_}={!!sZE
zX#FdCdQFXh^_Pui4WsafmldU_%9FefVMXmM)s3GG8m6a1S>J?njb_6WbwvJf(lY>9
z%OuxJrb3pUZz^KN)OP*b71O&Nnx@IO$1I_81$04o>?H3v4m@RdSnPRpdc0)>?c{1N
z6%4rb$7}}5eJC?)TO^kMs0{=2IhyzB(bUYhD->bsdal&s5#(g;%;h>vU)Jj#Uff?_
zOpj<zw5g3B6kJHMZ`QHz9L%8(czk!Q_GoS82cVZ+SwX-tYpWI~<tu$w(#>jkt-IQ<
z5!2LUcTDA_5nS*6hW#d1uN$I)IInXHYk6yTOADjr5AOfr#f%*YEc9B%+PT0uW4;<i
z2oUhWzu1pEjaem=Ob!jjAQ+Ff-?|>cvfvH{TKeSP#~Vz?#5{J-^PpY4gA82dc1mn^
z-2!`!*sNdh_*P4lt#>l~hV{J>71JS%#k^{o#?cjs-7&twy;j=PMto)2cVyKwZ}<%7
zqSe<NCS$+da6#-08~I)$2ri%U;=?wQ7d)8#mQ$f2gEdooI9S*$$2}1hkV!8o$MM|q
ze0}B9QU;sXun!ykF&f{?H(X8*7NA%5VH}rO%}tv8uvv&sj~9eyfIK?giU2Lm*SN>=
z=$UeM%MRufTc~4oZjxG880fgcAcHKmOLGwFw6f6`xt0exS8vg%)Eu{=G5W_j9DfKR
z4{0LM&hu#fQDoH4>QC4hrFq$tw)2&gfiafRTHWc1&PmGlP|iY%J4@|)0Ba)<-Ywds
zmycI~vDWCW-TAX{gd?#q4HF!mIfniLg0ziGN+@;*bFrqZh}z9IMx*@FAqMBA1))>d
z<3$Ofwpt3?#vAV@rJk<J;q^n!hAB;en8R)<@E3XbQ)bFOg(l;_VJ;m|diS_e2mw>r
z$hYy#rLp4K{7yB~B6MQg!PbED&Lke9KzdGs1po>yi!VFpIF}}Z(I+~)%?j(^6qlgz
zbhz{KE9$LMegTb>cKwt0%yPTH%9^8;<uz+*n0uq^eK`G1Vl++YDHQl9ypL_Ua%onx
zt_-G&hf7zF#Kb&y-x*YgB1CZoIJMQx2UXR>w~M#>h9)P;&1vKlAj*|ByQ3EGCX*k`
zgmE<A3uU@V;ju1~F+zE|-y&p~8E+C&@=85nuzzBAHd3KEj@_tuxypU7({pcvkiu)<
zphIA9d?Zzg*c_H4LGF70$Zf)L_z4z7>BzcfXQ+>QWZ`2di`L+kpAbr#@LqIWe!PBp
zKC3U=eKK>zgotZ~dR_-}6(@(wP-6M5ZXxfoDAnR6bD6r(k`6UDS~)j$#o^V{6ZmQI
zRL5g%W_*rre*3Kcy-9Z)kL_NGIU~mW5+##po&E<F=FO4`R;BX^7UFJoQqJ0&wSXt+
z?sW=5ex<}=UFV6-mDGa0`;l2GZVV>LjzgAe(soitt_#V|CUvRQF&g)l?T3#(ZFqvN
z;ebG9(!_0R$PFrSS*0QvT!myTh{Ed;FE8)C_?Qz}X%Ph>ghK}~s1fG6gH9X6RL|Y`
zs4?AhpzXmyYdZZ5z3~TO@F*2c(VCn!@k35)p3()91xz=rSZk}}7!+IdIW`52B^Shd
z?v6n1Tgcrlvu0p2?3VOX>K^Npu;UnQYfZt}wpCz7-<-0J`eN<71-PrvjKJ=fjrWnZ
z!Ka+eB?s@Nb^ulUy`HeIEY&OTb*r06EF9cxiB_I3cyC6gI`DdLG!Z&~O3&IoS#z#i
zc9@#=I+*9KiyzcZf+^C~V$+NX0oAuP5lr1swpM2kW1uJ;r@w4D=Pt7<FP@uUruBiS
zOhr#?QlShs3isNpo->N(Ya(ngUFZffXloepV&|kUr;mQBv&nKa5#_}Q;mYV<D~pup
z%~8ZT;*EY+z<k*=V4Baasv7cg+_uGd*rp1W8K7-l2~os(PE#eMN#Os+8lIZu-^yNU
z6GY||mth!!YWRUD20z4iQp`$j?zZUn`cCdCektQj_r4mKv-LUdn+c$D7>1nM{1E@+
z8-&v)G5YEf6yQ-BP9w(cQ1z-Z_Ri;$9PG^z6cmJdE(RV)V^b{p4_fZnKj)AQ5s_U1
z_%=6VPRkq-Wzv?x-036f7*#fPvRgi48L%x-<0DuzbB`GkWg-w&#(8VHgQJ_tli8bq
z;-qCr+eX*xuB#jSf!lI^Z07*|WsVKdX3W91R<&HPKTTMtQOmj-Wx|T@A4exRyXL^+
zDaB#g*AZ)ock2uo+85x!W6)j37}?|8qU<b@)Mi&~6O=iPG;vX>ZGbp<q@_ZL*@5M`
zWN3=zz}s2tJTl@X&6h32WwjVA&fJ(uVTPuZd6{AY?ERZ8VwvSUsC>rUV#qh?ESk&_
zhSn^i#^+E0-ee{(so|^<(i{QZYqh5cxI_Z+`<QAOIFpo7F`&R?RRQ|4&oZ`8?O0)^
z&w4k0QZ#BbD&|1isOWGhSAyNL&7?2ms4py*v!5EV#!$|QejjlX@<GSR^N=a<!N^XI
z_vy)A9kjjr#EY?=XwmD1a>7LP<c&x#eZ<6d{`Ieq=HrZt+m%!MZ61b8kt`U-MTILD
z%QYh=q`gxkwHX@DHP>T^mLDqsWB&t-D87e;8t$v5P;ZgCUUAZ$(I0&{{tegp^7<Vi
zn%5(FsWOY-8Y;dm{jgt<(AFTZ7HiUoe;WBoQKvpfQO7CQV-=u<atfjdU`=d9tkYq_
z4P~7ICJUKiVs^+;zv;#|7bm%d*0+0V2{*R<DUQ=#j#-r(?__!r#-6r)TC)<DwRJ7p
zwEQq#ZH(1KnLj?x3C-b*TvV?+R6TGp&ng6?Xx?f4n3d02Z3gVQeH2jx_bgPzA;B^i
zS6C7~TvfF;7gw{btXvr%t~=RzZ0ic$giXJ<3z&XjK2kZYWI8tkN$v}aQ?X`WXT<lf
z>C0K!d%Cnt?!NL=P%k{$a51)&elXGGja-|W&Sw5NNZKKGp>Pj+eDsupEH{MLz*7lh
z<1Vf-%No6f`+sC6Y>(w2wD5)J=#%=nQ^p_>Pupg50a&+I+S#N@K7u~UT=wH*zhfzm
zVc(`!H14>}HNM3(|2Eu~DS+yIzgD16{o@dOQ$iNr^y75Q#6#M6kue-Hs3KTw45HyT
zC2hsUooXK?W;<jbYj#r7lp1Vxa<j(jVi{JQ#B>QBt!!4@Hl>a(WL-Tz9Ij(YWv!*n
zc6fBebcvn*CCr+qI$B+ir@Fk(KtHSA{Xi*+Ue#LJHb#Y2-AVAF^>{`7_jgoKcV~bT
zU9_|_+O1sidnGI5&+*}E=P?B@AztIa0g5^KIoB3gIHX)m?nFr@jN4_*(2fTqju!c)
zYFX_sh#T=^?=KI;^u@}s)|4E-{8DjW74~v>BwIw**1Rwe0r_e$ko`0O|G^GHV}=j4
zz}qSFtYlu|s9F%#JsdfhexMH#o$G#ancX~zVM-`!l1xSCdg04{Iqv9X89sVT=~Z2i
zjV3-aR!`TZoUFE)(VfZnG720b(j1{&Kc4w3*{U9V0jsFp>JYeob}3_m*ks13M6{2b
z<V}!ynw+o#!c_Y%p1BA8D{TMI;ea%&8hrW5jQQ2i+iyLNR?Fc7P&XXI5O_MhP#8;X
zR)(@WsX(ynOgML&OyTV*Xk?9tV7SboeG8eh!;*zLPvvyA0T!_D2{CV_Y_+QLWt>RB
zlP9yg`it#h5P;?h=*BWJCM&cjM2{x=CAH-jQfBV!rLRoBxWcZ$zadsBH>~GwWgK}p
z?!Esj2pbY_0`Y<vk1M@&TFx)ub@g^8#Gi_eEHw?G&!oejg7!~Pzd3h*oJAnOetkuK
z_&mTSQU6IgwJ*N=ldF?x(TcIAdWNuW|4CN+5_|AjK--Sn(#^lsA-`(1O-bWu0z6RL
zVfpNP852(#4*9Cf>}O%eNZR(>y7F#(Iu+|d978wvDS6{$$lg|l^W~kAa3c15%j#NQ
z1$Mcrc#)IU$}g0)w5*iMIpt|nJ?sp^9}dw)fmV_%d~(O~3>_QC4CkwmBliT=fw7vd
z3$kkg*&8j2xaVXiD5jO%QAyb7>)$<GE+p(b(&$5<1bIqczwr)Hqe@qB<y?$5Z)rKP
zU)-2YUCo{1agY=R^o1@~cLm3rrnnX__Z6#~(-%&jWQ@>6Ycu=B^(pCE6jYiRE4LlN
zYxeXir%aO_)ec-na{%2-ES)cSj@KLdhf!@<>yGUR+A@6d@tap__9i1$^xWX`RnC!e
zT=5x4I<kL!U(X$o7tIq%)SO2iW)FP(6@+Szd-g!Uyj12K&5Cjg_dXqiB`D!V>Sati
z%D~a+1r+6fNh)Y3puQ_uhKCM-Xm)(%FlF+FeJgcr)bi@#^sW*f6j-tISW;hJ*rbVV
zlJlgi=7ITT(MfIC1P#QwF=|Qs`MW2@T$a_VWdYl7VrPdR1|JNS*G7{mK$&SA885M)
z<u2vrF(x+mLsQC>NZp}%U-o|b<93Z69dvuwZoYICs)1*1yH|nOuDzvSp4mSxoO|+-
z#c-bt>qL$mI^w;_H>SGdP0e_0mWpR}p;88;CH?Y=6R}KI=Q#I2k`FwW%~o1$2qa%g
zve!|D6<6ON;C35sKy|IN;acTFSC`}7<Nda`S>=>80aszDM<7%kEj_fZi}j1czx-vZ
zi|>|hBwDh{<Tj9DZzT{52Vbn-(qwOMFYGuyLQu==?S!_=I!Fw77^wy~_GcoT8PZ4f
z-LMR$zcrji=X5pLnHG5VXGpy&J1qNxPX-<V@-9`HB_%g+DSEsd6Fhqmn#H!uLA|Q1
zuiSYl>?DZLtM2+q@0ww|YTdp}#-3A9QlimHRkH`F8#{v6P0c{)QDltYkUV_xamrkQ
zTlIvce}(n}I+o8e37d|cxbtL21fy)q>uA$|7R3K0>uc3;6J`ny6WI@(+SCI0@~D!-
z%$-b93xa!<-M1&bj#gJ34ha#oDK3Ut&U)Jfjdwh^d*==dLC(18zfgidnCv_`QrebB
z2Dvh2PEZf+7SyhtA5d0c9(MnJHz*2t*}jhCz8Xl-ApKh@+>?IM<G#vCAj}_81u`-c
zo**~Z+{j5P!5^)#osk}t+ci}X=^(ZF@>G)bIcq7ErxVBr0G!q**>}b3at!O(-zN81
z|7Pc|^!8nSD1OQPg#vB!`JT6TMX>-r$U5=X;LfpnAIKQYet$RTmP_22p~%eqF81)f
zNx#aQW&P8`;m24}eU%{T_-tWs!AnIG5Dke}sRq%RnAP&ySK#6F<4h9FvFKlU0BdNA
z^#H~a0M=tB+k`{EvFC(cJ9GPj+n7{c4d)j9fd^t)AHJ%zZ~+Gz!g&3S{<#+C78uU_
zmfVN6dZl<obbOnVTfjr%oAf9v0tQ3flA=vksimnY@qk3DG1Nhyzow3%OI>CTb_KZ^
zHgH%n2cw(~M)Afmt>pW-6oMWfNA7^&4^jr{-bSe_k<~g(SyRCMUTtk|BFO-Cu)(sE
z_DptL%E6}pEV<x@ZKxDaYWZ2DG7LYMseIgSE}owL)u*8$WxU9?sTaBVZB~+2Yb1>s
z7AHG}-o{V{X@uN2S+{rYuuEI8{ci7*EeG?g)j(2fy4YkBa!;6yKEqj(I#N`EuVsEU
zZ_O_bENPIO7kYRn#7EU;37zFXS#DKcQXJBKk20Jfw$d|arIpxrX_-ujaQ3oVJ286>
zv%0RM>QM7G3*`Cmuv$XADoO0~ssSY#N37xZ?(|G6xnKIG8Vni&Z<D*uzN=+%PB}MO
zsY@8_a=f3bo*?0xP3?h+0Z6o9+dhH<_fuTw<FU`O`47wQpF{Mrf~MV9iX`p|_^=45
z$hXr&ykV&^dJF+~U<mRWD_AlGeFt6i(?cECxoGUYaeIX<Z5a_+tgtTNWH$Ye`UDsY
zDh%&-YQfB_fBIp!aa`{c$VqZgz(NJJmh!x75_fFH6v|CoB8S1>!#PtqdA;cpgz})K
zq6bfoiiGS%ICx6PGEj9+)9>(jo*>}f@Uq2iwz>yqj1!RnEWJ+-=7=SAZBBr(j==Wb
z)+g=^X&1gVk@lQp)NfsqZ-4DLAq@|tA5=aV8LNnDs|4vVb|`b1Mpi9E@g7J(0-PqS
zAe0{qA+4JpuMwjbg~_En0j}oFO8VLfbCHvm`W;P_wcMDaqfI)uO=aL#44q2S-a$yf
z$k-E8rFSg!lj~0i1p=ofjkJn94gQVBTTeZHk~Jybod+(8PLOB~+SGKV`{{A){!<-6
zh8P{YR=Pk^&&mEBzwJ~U)|zc4HE)KjNB;Lrl5Cp`VxQO&h+Be0U$Z)--RFs|W+t8B
z&j>?dye7gLQ*WB8oOa)VZjjb8g+P5jK3cvP^6P0V<KbRu#T{+6E|A#}0gSc5r>K`;
zk6N1Q^!FG9H2{hx<K_qDhCMf$AVr6kBDV2W9c4rn#_~;UnceQ6Y();4@lCtMY+Z}#
z0&$)u-R@tt`9R{Rni~dkySpJAPQ_C;hjO<~o5gIo6jPfBMY~o~9K1J@Wk%rTLf#8X
z-(c_;pExG8bxc?aY7duCPh*ZOv$4<D^pc)1p7Ne(hVS{B#vl<*v%u>o${%V|!;=&-
zv^6hd`ohrlIAnSy#Vddvy~q`yM*6;99MdbQR8TE6mVG#=f==Hia6W?EJx~;T?>lhq
zLi(iyAcA$n7??`@3evheO^~taoUD`(ZJU~hwV*b*>vZ7Lt?SVHWFr~wL7SUf(V@vj
zYV|ds^4<9wp$C<5IrcE#^O~q|+AeVZqY=IY+bnNmc4gY<@~f6U)#yKpH!TDYOu_WE
zrX~+$?D)=uW)4ukPB6gVxhUTYlVL{r#ep^11d3|QX-^XVlqApM3Exp{Ov4vKm$fb$
z)95h-L4+a+pN<2tY4}->d<y?15=lP7%HW*@Hy6h@e+d7xr2h<(4FOSB%;6HvpN8|$
z<@ZB^7C_m7hFtq|dH&-;m>IxD>3}--AIAPU)&H&*BlmtmPZ^v8sjd3AO1p0+&==><
z`qFvT@Sk%ig6v3^d1p}WpZxt{hqDy$nG>Xb;Xcl7?(YX*El`JikyB`WVI}{`Xj;x0
zEDC(#mH4+g29$vD%6K}Sm$v^|+?Qhpmbmn5kmq0LhygaFwUnFqPyYUB$Ls~*QyI7L
zB25109Hsccw3YNH{*(9%lK)?sL|W-38V{m?tMDhXE~m;tchf&weIK|XDqdC=I(fmZ
z**!qqt!tSDNHi&Cp*4Ml%p~)r&*YD#uYYT35{h{8E4?KYwL@Zp+vR&yFV`GeFV~=7
zlTfJm(RQx*Q3>N!v#MdcsB;kr>jEQQa&zHCGGbrx+jL>>0=#;)k9hSeGl=TP$$HtH
zHmmvszYsO&;jKcg309$2<W>zP312qHk*wlt8d!Cg12n0T<WmwN+@f9H_{uaQ^v!VL
z8@jOetTC&GkDi&1M&B0^AyDI??^)L&=eq%*oOl~Bs!DYGF_Ktwu^qmgl*g@aKkePv
zP4<`W6`YV}pWV7b=pVmxxl}LuJg>o{>=6R5pC;*?`&Xb{U+YU)6oNNqC`LGVCo9Uv
zw)`DGdp4IEE!NXM8T)>4C0BX}Nvb`#U<9GJI!FYd_`b%#x>ZcfHj{dyxq)C!m3641
zxiTuz*wcMVmw5M<mFSB;jA$e#dNjep9%<l4w#99F#r@~9VczRwy&FSTo_gA$*rQqc
z%z-SnPfNn%Y6u2%BhS@>;)?}{xD=@S9|KVLYz#58pZMBH7_2aQ>T30GSv8o3f6%X>
z0|TXJt}61JFMMA$P)8^^WEDO|TK9|P!Y#SpuWfOypX!8ITGqjHU;(QQMn7&6iXSnN
z4wn!!(EeEqls#==wWyJMdR{aMr%j>V(mNCl^LEJ!201s0dG%@Q?tVYIMN6T4EyL*O
zC`k@%RQg<R073tW<37SEXt$b^NM|TU%QT+c?SAn|Zx^8;mglV4I6y_UR{Lu<&T3;K
zS1$;8TXs<w2(vl@ifmOC!Yzs1O%+E&^d@XGB+?%rysA8j3$8RJC1{ALq!~5UfPOgq
zfV4MCayG^yRW<@?kXaf<ms273T?$5tFmKv&-c34}k9a0Sr-Kh1Mu}>VPoIosH*IFC
z#Us7qKt&l0yWDgaSuJ8M670Kcb05L_yB5%Egt=9xS0YzxhNJ&0^~S0)mr5Py`PPSz
za5?}&635vRBoOk|xzsFFY%1$=GOM$k?exntfDLjO*ikBhqr0Ecr=E&5{VW~W^Sz<o
zi`2xKDcrKG-csS6KdcxIw*i}Q1D_$M1gjkR+J{TX2KC?D&Ayj!FA|X&!F4#XC<I2>
z$l?a@XdLsb+71l&rJIb0M}xR!g&rb71=ShtwgV;cNM<^YLBsD3jC$qkcw3><H}tCp
z(pE%OKYQ7qZ=*sA1(A&Jy9n6vMgMdUsmV;%x4vT7XoYO+Ntk!}U0ybxyRt*yKJ+)P
ztEPmN8E`xl^M3gLd@|wxHNOs+-_a$^d$UKUt-$F+Bw77&p@`_hkvUHM9_^rY$|jO0
z+D`==`U)2j0T@{-Z6=b){b>9t(b0<ucNNB0Qe;-7f-*SzJD{j!C_dm>_weoiGXbBl
zWnDJHh)&n#RawhNFR5|7qA#^t{8$%pH+8INdyw{gMWUjq0`H-6L-{GP_6ojMc;^w2
z*-YsTlYOTG(_5!n4-%CfTRBcRhoA2RQ1)bl)hJHJvx*c-q`g23P|yo0D(}=}DpV|e
zkgKam|BCqGgXut{d-Ex!2@gw})(7>1%Wl}KKoyOVAW{*~u}_SQc$;XqQuqX$g{o{O
zcxOrLJw{hQU?XS(J8BdrNX&@NSpbd!gbGXF&5^?q)muc)Q@p(!&84c0g$b8b1KEZN
zP3yx7mSp3)wMGSJM%gXnEd=?VMV!kK#rG5N6QAm{@sO4giCP-lUBOD@!{810pf2HM
z##gPpf(}bWMa|kxj<*5_lKn4VuweEapd(d8O4)^fq+}aG3)nlkZ3NddDQ?ekLiHyB
zrm)eVUcow=saG|s`}Dk`isBmsLY|WC^=1BZ)=&YmJz{P?bsnzr%YK3xi4w?S_W~#0
zKDMg3hv<ULa260`Y~L90x@f|nk$HA(bOm@gZvjnXm92TL0Ja=lQE@nhC?QjAaP#>V
zfUuA7AOJ+wbQL_LX~XH*)=sXa&y=<=e}t9d7~B*FP~7~Pnl&FAJN!?81>71FFaQ%`
zDa1vS@H1gFBlzKC{7LVZ!h334nOZ??IjmOcF~PqW1r<B3>CZQPeT3VAm)AO>MLEZ2
zA7KmZuo}-bI*(%yxf@G&UrC`@DR<m%Q1>jzBtUipmS^*lK1JL;_YA_%-eUko1qR#N
z$n^;feyPxV9G9oikQs5q;)`bI+W+5aJ?t9RgX=5gF_rTMVasVAuG{0mz0m8@ssieL
zTukB~j+WgE$^S7Ohz%oU=Px74JV7c=&o3^bFc)1_M3xN4kLKK(ITJxFIwG#(jkNIN
z?)z%7%5h1BcODtaM9wkidHfm5mjJAVlAmeI50F%ndiA$tj!my<xfEK3R2uGo>Ms54
zQ*dE!*=GgR!%XS(gAJ6zX}%cElcmprI?{Z#lJu3w#69l@7Vg8385A!@|F;Q`_QZQP
zmW)%$%zoAEYI21c$mosMqgBbDD=h60C}Js*;XF$MVa?^6M5jM|*?)}~kz_X=JaqiN
zh;F#(C$XUJ`~CuDOVR+Pm#*l>UNqtD%RZ~e&T85}T{|4{C72|w`kur7yh~S0LwNY2
zmx?}!xILArk;58cu*TLRikw*|gqgbva~jn5)l1}RRJ?A}T{s0{Y~UxfUiLU%G=Xr&
z_pyYBUHkTpnW_9oKm4jivt9#px!pT7>6big{6`(AXa}Iiw+U6f5Ev@jwR>!9wT>TW
z!YxQMN)I@pXOY{F2g9?<6-pPxmV5?W23*lcq91Qx<ypNwZR5Vgp1Jv(57=ZBqVtp0
zkZTU-#TXx9av)TGm1wDRG5ZLMUd0!5rEi`TR2Q?2FOC@xX@7T3f9JrAlwCr`Nk8ji
z%2|+DwJNKRAj5eWfb#-!GMkAZw$*$S(Xq4mj@jAbVW<iLl7XGx0su^ZZnEGZjgIYC
z6Nk}p17BH2Q_qv`VBkGIP#+caUkttpFl!dcbisHad!LD%-h7^(&bAxfY-SU!HNxdF
zG+q3>Tf8J9Tj_NO(ke<j%b#)#Qjt(Nh9GWz;0F6iYidc<x1NLR5qGyvDj<h>NP*U6
zWnnyyxY4D17pp(5;1br%WN?vl2O;}Oa+S5|APM(bywn!i%VpX8>2?m~g*GvG0q*d+
z?;~SiC~%m&!|W$T=vt2r7X4Zp-}WAri|IQeIr@A6lcn#~S&>AxL*K*B1(ufvJ}4gw
zj;TY`O2d<c!1783foPz5j#v|tID3+TG4S1PL4p7?bKf`I1+Q)Q^2Y`){~aRgROcLJ
zBW2!8AV5*ZO;5XaA#==u<gfb@yCGM*Q&pT-KZ*N;u*?*MWsNWJAvj2E9)ja$+o-dg
zNCY!8^laFMA&%^xy3|#8sNb@(N+$^YrGIE@D^*ONkgZ;IkzS;E0f5`#<I+MFQ#eRj
zj=7sBp-PSW<rd@b8B~QFs)$aS_|mODJ^7@PW{AYxq6`RbW8ZEe5nfVqmUiJ!5h{Dc
zO1J!Ib~2+N2W)elB??7HA9gOflE{LWpP^?~UNm9i)C#x6Z5fS=Y7;WD;CRic<`e-t
z_OdSNt~FMbR>nl)69Mv!?WwkCq_;~xe|^Qi(71oPqS#?b!x{5<>@88%k9hXyyH+H*
z6I$m-Nrcm+&Ju(NZ>uf}JwC!GXk>+Y1T*BvPGI>dY!>E`JS>R_HjvT5Uxiv~g!e@u
zbp{9RFxb(B?Xi&v%4Y;wk`>63I>D-#s}eUO71YK-)HgLb*XOAXuMRxA&|Bx3H{ZQ6
zBy`b)@rgxoe|C=O^d|wx(wJ3Qt4S`EDb$JM=uX!I2+)2QqI?-?vdFuX!SOi!<;WOW
z501k$$-2yg<&i@bDiiIgLxH!d+}Pua`}*$IG3Cd7YohZ#4X_$Z&o+F$a7r#92mm_)
zoyWFHnD_JUMHj?U!tP(`@?Y6%)Yec58Ipe%PJ;qlhX&ao$sdpCt#AwJaPzC_mAE(M
zaH1XrdQFaJm8zknErIM&m<4VL=H5jtMfo-zoMV;Hd2-?0D0|${0EQ8Ops?VP=)6r6
zb$jzXdXFIgsHqk!hzj*f6za4D9qFQxz~p~~beIk|i1timUFN12T@Kzax-1kJ(rrn_
zV$*C`90i-DIyIT4T0w~(EYII)Fo|nK)46bVKEljr5TK}z96slR^w-w5wU{q`eRs8Y
z<D}u~{(8}LY6}&OM^8<|+nkS9vuI61*y2aoPtvWbRMS1HqVJ7nEuOeZTwpd{wl;_y
zC$5JFA_<^p8?DJFlkFD*gr1(GgCCS7#@;?fi_N5PAv<^PX4K$4#_+OQBu!sn;+gLJ
z87PB^RjiR~5&zjC9~{^<hx87qBU#&$Qd)jKvxWKx#q9Q7b9^MrLI-AnGv1Yqtk>)i
zplgOOtP5CIj0Fy?>JC9TGC)TI+2wPN9-e37Jug5nHBwu|`MFi#an?XDNACM1V&{ej
z5(>Y#Pfr<wFVu>Ph7NocPDiKi1sx;+C;KteGr}*LF#o#(sHFpXO$(t+AlDE6RB<n`
z`a6KG$k;qPBo$HiC;(XtebBjp!!%TYkk=lWC7kaDgJ9YogbE23{5yf?<O-kFTa_9F
zk;I7cf)bQJQL7j(Cct^`Luv9hO;z-qaS%$S0BH!cKDsbLa8MG1JzOZz4M~Z@{=l7)
zc%VGI@S?)3KxTR`laS=^^Yz(T8BA1@>_R{<r~nn=)mj%+fI=5==uz}57qT$p1$_5k
z%ejthn`U5qaZ>ix3)S^_gKnq!I!Hkaimwi!zw$E!Qns!Oh*&Zt;`BQg4~nnaS>Cg`
zxO`D3hXRfgJYVe~XJ=Jsh=jm1r#zRVFDmGES$07j$#gMQB!IsplYm?>kSzfy6L=HC
zh0K5wU?7IXI{BP||2_f?WZ?~Z9g^N+zL29LGZ?J&e)NB+2^EBvmoXQR_Ij=X{hIrT
z^ob#nmY@x6Z+s@#`CI|{XT{;wjbV4BUQ6JJy7CqOw5#X6{^zmqZFMl1KK<|oPiFjp
zvYGIB0%XK|4j7;zRh&Rto4_~ZeEodk?tqzT0j0e7P#}LnhCda6jXS2L7g7Xk^GuEr
zobnfKW-g$m0r$i8izdpR#WT~O5LvxgVLfNgz?9S!DK?;FqXXSU4Aib(XfxrAiuwYj
z3o0-$J!mL{4?60Rl4KLG@;S$XmvGNt1hM$p8R&|;{m(95MF5-HPHYRfkRm4DcVHTk
z5xGe5`G7y5j9grV6p1u5bHJ1gZe7>8962jNuhuIXdq};q&&(YW@s8jEd5N;Y3Qn_I
zyQq%<I5<N+RX-J+b83BgL2ftjE6wcYc@rvHJ~dz<9~@=;!+`G)xC&4*<g(0+)XNdn
z*>W00dhh?~1&TMIo61!sc4RjHn-0a(GdrfBVm@~qK%1=qN#q2-R-Nx{^z?yVG2~v!
z7n&1JBM0>Fuwji};0+}O;9Rgk`h4Yp++Vm342H<GyL(QpZ{V4p#^0K=oj0MPbpmag
zm)#hoxL_a^z@;Gc^EnY<9^d~q1b?ufM~rV0jbvxSX{LaVZmzD2HZup%k{*$^d#*W9
zp;!k@1yVdFyQom~fXE_Kf`k@Ho0M#iK(}wUY8N5f|5y|lm@2?T8Q4}XbZ+(lOc{AC
zd+mY>${rUmRx>FF^F>vxM;~<ahUokUoBvh%1bcx19G~HFA{7IDht9xpcYj&nj~qc<
z3Bb|aV+VO}<T42LIloex-}}=G#rMBK3E4=_l%@~ptbLV)q;4eNb2bo;BFM9ZAC+H#
z%!L&HHi$!Df7{fve=1g#xW2Z8vrj)g_{*aRBd~Ce<Pl|Z?SDkg1Ax-5QT0zof{ZBt
zycF<{?7^2G7@})7!(X-IC9sd7+?}5Od%s=y4PfMs?d?B%K8sHQHd2M(I^(w|1AIqA
zrZMpw!vBk4pcwK`ZD$L^*A0kzU~AFrcO3lB2-FQM+-=Djj5En_<^UU#x}sM8H)p=J
zfI82sJPN-B_+1nrP0-(BNJ{~0V~3x2^mpL!k7~IuIrv(j@6liRgB*y~%=>%?ng3oO
zX{>-c{m17>v<b94`{9%9*|*te<}VQOrtG(J{!g?RnExkQaR2WgdDH({EhgUo0m6Sq
zZ~uq2Ap7|L7;Ak+@GGW29ZmO2w8*N=JjU)MpE^E#aEq*v|K)CFd5cA1jkYV@*lAlr
zXj7j|Cr7XCl>{;1*6yR{kpAt*zh(euN~7BGr)cK;0z?m=K4`=m^CLPeY`cY0!U**}
zIGQ8i!A}U{sbaM396BB@ggK5(n%E317uFF*2yNJ<P9&Km&>Iy!Sy<VsQmdduc+tu(
zey>V+{Ej}8XvnIZ`?Tw|ZENaU%HsHZVgVjua^0|43nQ+Fb&9|YBzE!>f7EK~exj98
z_r!3*Vg>p?&RsoI0=_MhVczX;cjhDf5V*-f#k|}HDXzPvKNj2<nxmUE(a6XG<gF|$
zEJCfyw5YwDX6M;kR{EWJ)!SK5_ZO5mW|a#Y>#;pa2nLIpT?ajTshz<s#N+|f4Q%E|
z?);lQ?4EkVD}?+z+qvvhkB_3OT~B!28=Y%wln&hIy27~;v};Gh{d1yONlu~}l*K|K
zvQ@i?Jt3B<BLszB+$y$V+x^Z;s$laj+d;<PZuczHPZeYN6YR`_WUdvp)agkkF~-#R
zVw}s)phBz`36Gsv-j_Z<_ML3{=?|x0@FE0GV4HIK`ZlnN&0_Qy!#ykyyV>0$h&bj~
z_jd}*!r-gbda=;XjnW6fQXG+iIOgy&aGS2;S<=gKu89;2h`+??VH;h*y__jhn+lKP
zgZG|CPd@k3v@{ueBTVc3;tDQx@wXJbGhw}ISk=Zk;`|?hCj#t&ZmYLB{*Li{UjXCy
zl%v^SJn6ZYuHlj6KYc(r84SxLB{oHP#cJ3jLk#!VKg>DldNtM9j>4_O?7PO9!O7gE
z%?&Mhi5A}0WTI(fw&5d_WzA;+s@KEornba7INatXp<gnS>|DMG&X*AJ*tv9bBj90_
znF{v^FYaag5&&9C7QZfY{T+-d8Ub;JUrBQtggbwA-AKGu@TTPW$13;f{lu&NQ+@ri
z2?~P56y}<FnkMm?{Z<s-{49Na22F-WO}7aaZE0F$j3z^_P<RNQY^`IDyf?#UV$)XB
z)3VO1W{!?;lOGdLBnRi2H@sGg#)d<nL~2a>g5c_HGhs|z^>Iyg^3lnT?D5uQR`HQ(
zlC26;-T0S@+UEi_7PDQ|<g9`{6a?)oTL;qu3N4-G{CBWf`6WCYJDZ?aM$OjI{`_A6
z$L2#l`Q-?1a9lG}!+90*v8Lt+jKM0J!;pqL*mbvZc&2he@Y-;UhDX6`t*x*A-A#OO
z4}5U0(xO9f%;8qD!WF^emzMndJ@!pSuI%m<ZR7UcMA$5B?o0;(l~AAbu|$UJlN;U<
z&4^PkE=JRxL4r4PG0EjEQzA`P=#!owz^#zVg$X{afX6!CJGe%xg{Sw9<PaMlJq;tn
z>wcJ4`o#&p)<L?KucZQj;v;(D*x$w=91d3DLvo)b6(O(VS8<+e0>#wPWj{P16+g!X
zXmWl|nq*`UcCc0~o@lzBemVNi)R@i0FEA@K+aoi*f2%NlF_{xqrK+Y!_vVphzlPDy
z69$`5*ZC=0rSCmsWnYh>O-8t<Rbw(r!O&kcEzPCF0&Kkpj>DP{SP(N_?1qh|tj^x>
znh2u=V&@LOO#Iy>m(9|-Bd*bBm3hfW-lxZ#MNbG1bjF>#pC-4_rQZ44CF~z3dL3bO
zXJ@P35CBtH(Aw#o*u%oZxdy6B)Yb6A(nklcR%v)ohZ*6ax_ghPv=8%_Tk)$ry-#gS
zP=DYndzh$Ju`u`U>|A^E27f@+8r&pMz2mrVQZ&Y*pU<yTint|TRkoj!ZeKjYrXX~Z
zNVl^wmY7dOm{M+<x`SzWc;C*Kvf1W$ByaW<7;otHFV??v%7<q;f(yzgWrQ5}DQ#`>
zZg+A0i4qQ_(q}gMu~`iED<?nw9?k#AEne!Aea%T_46Y~zgR}kA1$7(WBvHS|Xy@Ho
zNUm|uqJa?e!$iDCyQ+V_S+S3YEWCa~7|$`La0=Zwv~L!<;srgfu`^9vRd{XRT}PFO
zx*cA~O19O>XTiU@C}Z1MdfLS@W>7si6V_j+T?@f)jJj@Fabm}Qw9@~?P1D_6hQn*<
zc++qr*Qg-ac7~=gm1UB)w)E)~2b=WX-mYdS<@~SHTB1<q14N!?PGN3di6T=;&&ozU
zwoUVdxA(m*>GmH2hdqU~30I>JPX`^n2+Ai;BniYJQz}|rYX#L55$<qz-ozXBgL7A!
zM=d-*lw0F2j=!)VCoiAcFk68R^KK;S+ax1u``}E70zG%wfM5f=NQci4?PVGfS&Q)U
z$=@loF_3P2{q^5j>oYuHss6#q`l|G@KRgSe#aMct8&Pqt^Ih$sJE~hiypVY2<-D5b
zI-X5gxCoh@nuGbHjWK1LU|lCY=rGrgns-iKTlZ9oo99Z=9!0UJ`L063C6!Tm!MVd9
z_NfZpkiA*U^3E3{Ic?zj318i#rLmDd_5Q9BWdefy@--#fgVZ`g(V*+E2kY#_2!!u!
zwKE%?B8C`m3uva6Vj6n6&FW8oZa7V<p{46&A`#e=ofFJ3t~;?C5h^~hUhgz+Z;|0^
z<>tvsn~x#&N@6yO>2h?yFDB-y<C~d}a~gfOv2v@$G!=40v$qwkKKXjvU2<pKVdC4y
z+(Ux!7IKPP*vwACYTI+(BSJ%`%@1nSsDfe9Ev=wHzrpw07yawf_BPuvIQrETv<6G2
znGxi7em`Z^r;XP)*7qD{aDoH%;s!TO8h)4uv9`eFC(MP(mhw;5u$V4Un~y-Z1(OQK
z?#bMV>)e%u>aigDy+&Ydl=H0;5uvU^LNZPsU9P(zlyJ-KUx~G`c97L^9ju#fxnhGK
zL{TL*Jbh)m=0Wg?dv_`HWrl@_IZw%Ixo%DWuU5?j%h-wT;C^p!Wm%5SsEs}-ZHtO>
z_@auFg1!>w`xx7!pZ*Hdi~1h4*Y_uf`E(PS-M5e3TdLt+_}u#5-^JRy9IISz)+M<e
zh8K;cm^R~ZgoWTUTqo{JiB=@(2cg2lE<yWYJ(=5$+if$?K^<bB>~|4{`7Dd>^&PhQ
z?arG)34jDw*zwbp^Qk@5xg?C_;gbx)ZwgD`mL(8U<Pv=Vfw}k0wb%&wT8T;7m*LL_
zqJ1B{tZD|QbW!`ei|q+c?l^Z06~QN$WgS{0ygY1=k7K<%V;J41-x)!-1TL||AnV<A
z>*86IG&Sz?n{|_k+ISpK{ZE~rA*MT`I2p6-5|b$B``S;5Np0M8)W#PU;(5H*5j#|!
zUh$4Y=8U;<W_5N_P|ir2n%u1An3#GQndu$Q%esYd_+DZD(N@K;=Ae%jKka%VSeRId
zosBd0`oC<})Ax?SWk3;lm7rE0ufN{EaQQF12M2e-MvK6S+wWq;fHWwUFdlpwa-Z#H
z38kReZCUZ8Hxhcpr)~X0iD8i$6UG?qx?RNnj@;vP?;T;gY9@7VAK#O{l{c#DnsgTG
z2hX|rfFyP{O;g<k`s~cit7q$LjQVXnbC#)}YNh0Xog(baKH0b64%VAvSms3S!t;{@
z({RqvzqqEUS)i`juH6)PWIi;|#>h-GW?iFYS=Nd*;;zoHEOP{#;DeFzn~%A*t5zi&
zI4)a7G;(y;#xNsdVtjI&cbiG!EtdAH<TEV_!9x1QHUultZ8cj3ErxT0-U*&H^KJfa
zp4{P-nHlrY|Ex}To|V?uHU{~A+khSr$`0>?$bo{XR9E|zFUxSKdy41gt2p-pVp`bA
zq~F_Gzc`iNBe?e+??($|6+b`n&A|8g-_=Z|5BSJ&ofsw^1x*-*&<93em)joEM(DEE
z&YY5mtJZnAIPH(-df{1kTD>lab-=nwehM#QKlOkPfeVEu%#aZo#0U*-Fnx5YNnFOp
z!E*GT$#uw#eFwXzj^lxE(ZKt3nV{lsDlaGV<Jr$bj=V$tv7EKuUl#kH<z-nMx*vB4
zjg#<s?QO4YMk@R$i0plJm8__tD>Y@=f|wuF6q~g7c|~Ngacc|ya!~&)7b-FWfX$!H
z)dF`N1<2a-UrMUY$JZULiNT3sxy69%6Abp~xw^xp>Wu{%`0IWzXQSgI--Tjj_the&
zv%oYu)^QY`e<DvvP?xc3nrvUbvVLh{?S{5xt<tc}bj3{J8#tMZ9@p|LqP*+73Fcsb
zSlqYkhSI|3fgR!80#TPt7IfFW-%W0COgK=5!VP&!`<3*)58DNbVC50r*423xD&SUO
z7}IU`-UV=#{#Gc{apmhbxU*k}xgYVKIPD-jCwaFz`WxdG<XP&3GMBG`F!)}r0)WK$
z;KtuxBke4!$Y<|z`@QuvaCX*eQvCVX)q`WSBNLe?`DIJ>w#(r{DH(Qd*pF!*2Ggq^
zZ?w!9?iAj+v3ESvx<W<;)1?wPTkjJB<_Dj^l@i*sSm)8K1gX43tKLxx87*SBO2aM3
zb-LF5O<NbXAuk7F{*{Gc9eP*i^+~TS1;glQLI(Zyf&Ri~5A4-g&$`19cnhxuEBLP@
zhPyyCNCfq;E+^u1@+fl{Wor5iS>ElcrZ9#>q*`pM)_(49l7u?d+&^LohQ%~}lLjh-
z1bY59+yKBFGv|-0-`l1G?*MaeE*oto*liaUhQUwQ>+{}DrIy*RR3Jzu2=>RfY#J?_
zVoSk=wh|AoZ4Y+|-rB7<`fvIbMshs(SCdYT5)A<?e}9A6TB9-=F&3PH9fRQdCX9?}
z!U|UF;LUrwMr!JCRPfGW>TxI;NNNOWYODJbY)5UJR|>M_TZXjYgtmxlxXq8TyT~K>
z9b~iYVIn6<sZ)%4?yD2>a8~mH%K0pb6YA@H$KC99v%xM~(fzzN+Vf}m2W$Ru<XJxS
z!*jUsiB?yx-O+5M?MU9c528UT<XyY{H|-en?99r%f4RChmvd_u^*=m=-!F*Wpp9gk
zLi~kPs6>{%s|OF<kI!&<Rle1WidE=z40x?7Yx?m-*EhOQ$olS&RSVm`jWI_c(Aw7d
z7E@^PkhXJ<C9Kq%b?MhL`q`tA8;LErxLS2Vxp%Dpv4i!GeP!QPu!|v^qE9xFtoJg7
zylrZ)(DYtShF6{EP^aE;ZGG%KD@*4fIgEM_Ms}x;cfRe%Sxi<=YprjMAki0iU|q9X
z$G)&&5)5KkV_Ck{?~=GR%E9kV<Qza;U{MXOxWkGDhxqx=qTxrI>z&0%{Q>kil?O3F
zMxpr%A?1^YY285we5;$)k7n|;4FieAn}pVQ@698QnMR5=aS;mvOp+Q7@49645j$;1
zh7C4kiMmkWrkaZ+Y~pu#m}{aZsos+>De%m#SIoqS8i6>ob#yXYq=RQ+f_!Z4<%^*p
z8J2NAF2qo5F@m#eYIKErgeltr+`fur8qz*h-{_OEX|ipxB-1}ynvQ>3wcDDd_gY|&
zhp(V=cS`f~RvUI*XET2?qZP~<I48ktyS3$Tw~>|fcT_>9SrjMm(2W_<QzG--K|%L?
z<tpQ1K^drqny&GOiq*{2Z4EW82h7aLTp(s`3hho(SyP_7CULS-yuV@K-4O%s3+f&U
zK$sC^L?F<Lj+HN|>*Dqz9@g6vX8O`|XT<W;HlH*+i+F2uY;I~#jGO15+b*BLo>2}~
z`r|))50PN^EMDyMVeK)5-KOb~6v5laXC$qiaE;ya$J_nI^dgAp$a`J-3_1Y~g?1On
z+?PHs!^Y1{UtE5ETe#Qhq-nTOIsGjtrMt`VnxV9ek1=iN=~m@*`;=*_=b_7@P`9aw
zJa5WU-I|TiqAhWb!~c)Hw|=WKdiFpCK~O|c6r@8!q@*OI5u{PNyE`Q|A)<sx2+}1j
z(%ql}BHe5nl<wHTrn&1y@O;lX&wcLw1I{lzz~1kAXU)u-HEZTGvpUyY_=qg?kce&m
zC7NyXCoSNRN{;B}vOHdxYFOs*rJGQUqYvMM4yEPCBi`h0^nG_!&7UshP8h5(skZL*
zpK(~=A{<z+sM!^rl<Z}BF_teTj8yU=RxMeZ?=~8Qq`GKG<pM#;z)$8`N-X<5)5K*z
zc2Y<+eB|)WEVTx1uKm!43%%#s&cVcl7dhmEuBshkX9ztwq1AES>RcTLRm8Wu1)cvq
zqm;`MIozU&j5VoFpIR*NE;CM_&dORl#yVpu9|c`mmUf7B+%nMVs&{mef)Ve+=|4!N
z7~R;`aobG<Evt}!y2fq=-BVT5ONB{c^wk8vnwaTWwLGjT+!QHknUSvz?;LQOd#Bju
zZ7KynvU&l`oA0dzlZyjY1OpI@mN4=%XLk)s^#y2XPa%H$8_saiX^}SY?;g4qaTBMO
zW?=L!2_NQ9ok*?LAbHh=6nLXgo6jbX67jN<yhPphq}_j=<^|WsFrAfd{=qQ9nF1?p
z>68ODJ)Nv|$NZH{T%_V?pYIW8L#^7L9{)QGPiu4Wu8^q{?I)UPPS%eMPiz^9vl2>A
zo=Gcb5Okg`wnR6zW@)M3qVMwr^}@0w+l|<U1^h$I_aZ!LqhI!KgmtF6t!LvI<nx$z
zgk-s-q*Khu6Kq86a*>YPMHtQiN03-hz%`BgxJR~lg%;_}Ts@mVY>*RCjB&Wp;B4~5
zFss2>=b*w{EROTjWKcx;-d?$yMQiT}n~kd0ZUJ{~rH4A4f>t|PB$}Me!(M9-?)|f-
z8JVMGlbmcizu!tp+)}PtFuIl-?qp5W?d#?A^7ynP{^>-rNuX8j?```gd9Glku30dc
zvo1bGrbCj)PUZH^sP#74_KeQ$D(wgF7|oXoOW&%wXdY_Y)Sb|e+;Pr8^vkdtKf$wV
zXHsD%UEV`PKi9qOPws5)vHzQH4nc2pthwdMqpN1HzRmg+n-Mr!)8G4@{_a(J4q&f}
zNL$eP`{(O#Lcr;|8kuu+)aPJeKL3NgyB4%iI|T>bl5X$LWRuT)hr`n95@BTAID~zZ
z8(x_hGxB@~4)*l96^1*DvBelW5f0wjsT`J+o!=x&wC3YohRCJ_6{`i%8M7x<bv?QI
zWa5CXZ=Y~wtPq{Bcj8L1vroFt)Pb_l_7>T;sHh?L)go5b-2p|D5`TrBp(&<LCAaNt
z)e;J`^(xbjpEU1;g9|f^he2ucZj5!gdC8P`++1HW)4Lbu=CyZ^fswq~EYRI&s5h9r
zt)*xRP7%DsxSbbJifaHO#IDx2e2J`30s*&2oU4!;3o(F5tcL!l-3wIoqBw}hV80i=
zjLnS$wp8KeoJSaz0nSm;^lB>4-w(g(fqeKiX$}6%@BW7~V1^D>4#S6Hfs3>A1b9oe
zu>B=2Dy#+o=3ET!L5CorL@>;<l@N#jUzp8(&=9agPcrA?Pl^{%_2od}`3WHpf~vfG
zIrRL(;UETY;d=NF?c)GoH?wlNsL-ch*uiIs*a7nw2Q6O!TOedK|5v8=2uv)JXb(#M
ze!d4eWslnqCH@9d{zr+D&^qAec%brm7TFoV0<ud`dPy$Ss1Q8rOOr=-p+^6SpDKL=
z+@E{m>g0bROFlN>s}rBOFVSMZKs{B&)1n)Y&Hvm5K68FT5qAOAxfuoE8HMSCm&lzj
zAf2mZF$`tDeN;eVwe-isl?Q)a%AR2G7UH}9ms>{_$hbSush~ig4uJlfnO)bOIbLLb
zzk`$K&8SlM%kM^liAiA}LGtUl0e)AxZ|r#iyQsd6+vMc?=d#X%!Ub@1z3CBjwD;i!
zj>f>qYqN_`$YV9|==X4yOO(rh;-oboq}z?O+|a)|BE$v0YV(5R5~DN=;G?aHYUt3X
zpb6UlDeeC$?f)q)Sh@edEUkFsn)gocaEwFSZ)~^?HbC$Q$pWLsIj4G2#QNeXShQ=a
zgBOs`(qdo@^q-$)#-HImUG;E3hNAKZbMw~2US{0jrPqK#VqctnbYTIO0WL(5d<GcD
zb8{h-1I#p$O;bw<DY+o`ek%9+evaYqck=;3EW-@-g-zqwaGY}*I)N$UOjEt+)Aj;F
z3tOKIUcf?2zXBRk#7}{98p>48OAqBqiViXvnnq05i5`gGwEQ9f8p5KYSpdV3eSm9r
zfxLVH8G7sS!T-YJQlNt7s&cEK%MUBmmlCuKC52jg`Ic(2U&l4O=2Ko?o=f~rB!#d#
z4d?F!K+PJkqNTHiE0>TS2vx*Vzi(hwHmG6`I=D7{&B-T<)_$c9wKwscn1_%;UqO5>
z`%%`9OOwe7?Bc*%?Tm{tQBZdo?HaMUnrP##?%>hN1b{xDtmR`iHSRX~0PWJ#1Z)~D
znU?ki&Q1f`UD-tQ(*8Z}dA;^%y{2d-zvkyV0-gI%s}U``@93<cgUya}JXDYHOBU=C
z#CdNnfv>(`2O{LKxc3j}G*I#_uVONeN$iP9N6av20XZ{l7#h5{x94tC4AFoJXu!X3
zq<taRIMBnQe@8Ue_a6;)*FO+*F^4=|ujB@O-VDB@6Lf%fN}x|o-NYZLrNK2o0lV42
zqkmQ#P|zBz^8MccyUd%LjW{j%npU@Q_%?<1<YGYAGCGiZYjd|P-vcCwKqL@ZX<y**
z;#2?$rJ^~8|3;|04)sAP0`2>nocd)6Ko4AU1JEgSZOvf;+8XeH_+^KcFU9`_(xR^k
zH~yg)zqt5B>$uaGkC?<)Zv7-S1YLCAgU%UzAW4{cK|(4-0;%i71<)-HL>l{*8nfB|
zgM@JpL}w9ryalwkn0bpQ7eMMVUR2QzwGkHTgjN<gDmJtM0+wLPl!G0*`1-wzKLL#!
zs8EltRnW81e$s8B8MG{Gt_1yJ4C)W&bC4#M<lre0+?nT>514s?In9KokX`sEXqY=x
zcAy_y6KLv52x;XiP$PQ~`*{w7!B||lK{dY5A*6iWYG#)}RK3q&Qa)`}|Dea9vIz9C
zOn%3VLFTFlwIH?s{f^-0b~}qM+<FKq+i(KuPN#cYi5KhYf1X1D{6uv6<8!h=&;gJN
z>XJKbmKI!0(<&Z9GSZ?g2OP%Fr(k*Sa?xsBx-l_Iz+Fp!U-hpf4A8(_y_h--x<wg+
z-g)*z>ZZ+H^;^N!HU&R{&VJ?s^S0&v*CB!kn3?|jGwJ_T{^E&Qd%WFVv{pX$6EsL8
z&^_#n27Q3@p&n^fp*0DRo$~}v&N9rS^b%Is1K~WqKWSb1OB65Qs1*~hG|SOOK!4o3
zfUR1f9W}G_gbq{@EaVM@c@pVB3JDVOzW!JBP>=sN&`paRbeB{;{*B`u02;S3GXrU=
z>G_}^R<@gi=Qu)u4%+{elD=G*{J=iChubGyuvQ50;k?mjmrFe!=$WSV<m!AAp8N+&
zQNID_cut7&2w+=cm001I$H`@RU~#<-Q^o#ECoyGW9#bkT7N4CSwJi3gm8d^v$SoAP
zP=}vTE%ExW`DM?7&mh!DFS&oQ&VdUI^MRV{#G^X`&HM>^(j~uptSHN=JW-)}6;S;c
zoNhaW=Eg69o?jp)o_u&0^&c9!3H2o`pLjI}66i*tr4M1z_iMyE4rPjfMBx7k%Q${W
z)4BNme<H+>1z|3zwu-w*`{S&e^dBF7WWynRBr0|FBNf#hhXTtsK4Y1W5)auU-Cj-P
zVA9aMRN(kb9quM!LPgESzff`9?b%g@u=A@!9<0Jw?7K#0G&Y>qy-wFQ)GaArUqQuG
z)0qa`=&N;eRIhf*{=}lI6UC@bRpl^O>`_Hefr?2njfL{&zV1VZTWGjKS5UwYib_;8
z*6+Iu%a|0<0ML&>%v|*n@#UdB2Gj1ONYKq!0_<&gBk{7})7QZA8YI9o2QZ<B!3ZMq
z;KZ^snn3j6^6@AFZkP20r=`B;o5CuYGK8^DUGncTsB~=4Wxj?+C_u+F@L*~?xFFz)
z&o?aGCbY=}2CB;uFsZ3FLA!J*SJ%qb$vRDq3LB=GpTQA3OZ{{XXz|c<3pa3^xGSu)
z3I9@z`8MQ=u*_m9C#Tr?q`(hGCcA@eV?sf%TBo&9WuS1(f$!gkgI3E;ea4w~5FHJG
zj$@{dgWG=@aRucJ(6K)i!|;z0nADVojtg?vv9KbXSB50>G|FZ{^RsA1^<rxKyYbL-
z3*Zs%hTv?<3p%)UK_7qvYZ>f+gGysDYE{0VqN1W!%#>j>Zo8YU+fbj;F8v!!!$$~A
z15F|Ruk4`Lp56v|!cYd&f78H~5SiZJn(0dw!%{Cc3RKEbku~W~VyXgU{dxEV;&}><
z5H6m^nymH3E5F48fp%G`TD-duC#IUlH0T)>#_PQNYpFkLZZ29k%I7MEMs@AoTR@1I
zQGj)YvsKI&Q=piB3RGlD-GTN}MXvZzqn(~0b(e<n;=y5E5;VgTfP@MV5d4xP%_fi*
zIRR#hRIFw5Tr%Q0U<8k~r0k`XQKYeQbsKbk{P<zC(3RL%Y%Cr7SpSXDudweI@(?Nn
zL%f+l)gr&pEL@2iK-nlZ&(%xHUJFs7s^)8b48)_8ie}cPmQUhiGwmYDR?1eOdR}-P
zdhQz&xEvsA`rhRjg#}lHKX{IXg#U+n4wsmQE}^G6{s2~3RA7JlYHJ*4&Rn}J3dF%2
zOxz~544Q`+moE)-M|l%7FlJAE`45N+&@Q~_t-pW#?-nPy7(Z-c+G4JAy+r$iM-dPx
z|IYb*AwgPJ2iD8#pw1`!e-zbz1N5gW=csJ=J|~xqe56dx#-^}di;!!Njz@*ofXZEp
zfFgsO7ykr60jii;=@ACD|0R)UXhV6LbY+(Cs0ccFsqQ3x<IU;sfvcV8&-kQ+mxkXl
zczQ{)o+l9LO2uuOXaAN5i{0t58(uANaO9*SAJI52ba56$v2de9-(X}G!ZqBwmg;&r
z3n4aWK%XY&%YR%!y^duyTqsP$Wy3gDZXGe-86VVz5y5Ks^DT}Z7W7)7Vj#|N+2)YT
zM_wC)e=v@5+`aja6Xo9qb5xDDXIuNS73o?d8KPsc#24P`3qwS1CIZ22F!JDC&H<MZ
z$U(2wQTvh!-+rPV?X5*@ZrZ7rm<013%to+zpL(=LF~tX4KI;Xt^g%;o7N;xp8T?BE
zw;F|Lj4#9UL0rsQpjgwLRaBQ_>sk8HNJmMisl_Z7l+ww>=uM7Qi`*pxWJt&W^TtYA
zer{d4m4Q-cGfCIIp$2P?+Lbxza)}MgPS3VR^i?}ssu$|{DCKL7+!T8Snxc>75q%<n
zhKU7lY^xn6hUWf_5L%&T&n@%`fmqr((9zFl44I)^hC+kp8|I^>*(8EqTGcKa%(ivA
zQA@WCC+;Y+L63p}539hNS&s85mEQr1qU`umZ3UPiwT9q_mu83plpkObF#qaF5zPQ?
zrWI13T|Zy2u#Gxl1{?x$@Fr9sV>~+t<^X64dMJPpEDlsLyKunu2#tv9e>PCy?r^1j
z8i&oen%CKhOJG1i7`<YqM5auvhby`qWMYPe08_37Kf8Tye>}jjGmZ*!_eTMtDHEpy
z5EIW0Y`_D%-DTz2C)RP`MhRhu`40LL)9xsX!LQ9Dq4yw#kpL_2?K5^oN$3F|HSm0q
z(0+i>C{Vq=`Ig>)dX$E?w=u;c8^`exG&dCgs*?8zG`3AK>Q0(?>7336jr1NoqR`ab
zb^Ztlf(6)1h$>+Jua%8!<dY#C-3Bgtpcs8%GG1wK!d*;35ZDfxithvz0oT2VU!a~7
zbGbwDYJj=+c1q7(S~Z_^bh$)c8tbw0NH@fMD`;-|$Z@{oqsPHEjVA>RVqyvf;PZ_<
z8Paq17yv;v;s0hwH3y=T#3RS}XO{Nx_m^`D(#`ub9cF*sUK%aS0j-&3`ZA<VH>6&=
zL!yrc9H}3e)jFTmD=@1fY3Y}8@qlZ?=_wCsFLilZ+6OpA@_snv%msCOad%mCv{KX3
zGGt=e>TA&!M1bfMD1nSVyh_JEABi4VEXh~`*n_aQKu7NgSwY%yKEDLoLi2uFX!q-k
z=Zc2|(ptft6Vs&@3gi$ypaT_az5OiX`6zpkDyqU>YP3VDSReS1;vYRveF)i2Bx+u*
z&4MoPk+JNiQoGB8o-PhuZ-Mr3zXTi&%#%tye`7er6{&zV%ybE`?8LNIKS0_ExQIEd
zIQ`A%>YdxSDH&ArKea_Nw7)^eqJADivX^@^<6NKV0iPwT^UrrZy?a2Pb4t<X>;iy3
zixP!f{L|;_*FmRLf<Ax_sq#1|hMLuay*23VD`S4>O+r?{$P80?obvz^9sKda%y{{}
z5cuVafg}B2zJTp63^hwWIcjfhOxnq*R4G^OWol|_x{wd54<jlZV(}^qu%Ks8(y#qx
zF;FJSD#GU!VSpSNDsC(2E9VnleN<>M_=x@P$ah~1po`3G22Ge%z*$+<AZ-f<(?)~5
zE7v_OLg@Pxu7Km;OBUKfzy@J&RHAT652mtD5CKaY=m4G}pDdt|E^)gbD--BoN}F&U
zWCu%uOwuR9*#B}84EfW`T0D>mY?X#f^C^%sbk1g>ZS-=h5vpKft|(pa6J^jVmzv$A
zBdjBqo$d^6H4yp^t}37>|GMR0{Q#4kIewDqhjI@rmWNyfqL<S;e()H!X9W}{*r+|K
z6YVjl<<L&|J8lMO1Pd^Ul%YQZkXaNG0(-(f8O5|BkXN#U17zaP$Dk-g<;9RGG3hjF
zet&NpTpZH|ghXxHmEgmKrV70#ff2lWb|}N<ocQb4ahs^A8{N?qfHM^F=*g{r41UA1
z3H%)?-)q>jKi*-ny;@Z>AN;Dki6wR&dJ|~5(Zro6mW^{Rdk1KfOKTFqq}Bwp98p?5
zw~Y{Q)I_HN7y60}p9g;U@ae|AM-o8ncONPJB6^p07f|rJA3T~clsbTME+^0@khS%t
z0XbSYU~z=iqgVg7-8U?S?z<M|ed&S62Mfux9|$=uK52QM?0#ptPXo<7;SreDKvaef
zC57D%xXW^<Po_1RC2MC<LvbA>e{sKXD(^&pqf0(zBxhD*$dH<k)ZtIsYBk<T7%L}z
z@0f4U>>$0SVLX$Ivv}i8%xecn*8k<93QYr}!sfnXwlq;wahFl8XXj1t<m11q*bCf-
zg<E2}!Kz+k6HUUs8Ld_AB(`2f7dq~(D^SH?drMF?rXP00>U2$>$8lb_TUoN6MU%F{
zsO=MF>&fu<Gt{TA$f8nd++(ov&h*1RF=&3HBj9)2CLgI@(+gi{i$|)QnnWnQuW39L
zl&|!1+KC<sqiM%hs2<SPd7AhXnQV3JtlLp7bkZ42auU0<N=oClZ4`YKHWgImr7_8I
z1ExCfxi6Za$YHS=R!cs}({=bd|F7mb)`NA97>gmgI77VdL@&3Qc%-kdU##DF%?(_o
zV?30r?g09{kJhT(g*?o{K%v*ZF?B~Ft<n<@j00Cx`+#S;D+2t=L=NliFE<GXB(`U}
z*2|b;$cOT#m}*{;fBe&&|4!^XV^BfCiNqY&Y|TbR)?(?D$8I&>o>|dd(wTxp91=cy
zLEGU63*EY(uI-ZVz>OJUI^O84(gI@?Wh)^P#gpBGo<jH?D{-@4tJ6%M_5J<Gs%Jag
zN=Tk^_gea+2Mx6$f`9(Z9ot@cT$Qz-$eMhz`;%O_BU<;w<WzjZIO(zZJ2PB)+!_4e
z_fS9psEIp^Df>F4yg*60Ur;=HB&AViK?6<-AAzSYy*64#swVvvva@Px6t9Ip?7hy`
zwtc&3CsjS}YDnug`#aU4WQE}OD7)cNh-D*u;H$cLl*d3no0;3T^CwY%xz=C$zxyUQ
zk0T<Y+`B`glD#HplrtKi%e6KKMbA~Q^tC3s+3b#HlGh)-dOcgmzJAF4^qO9?<;g<(
zbcH~8IL&t3ysqE1>Gwm`ZyVm+$6jzpx(Q|$!?ba_&YX62`saF|p9;Rs(OM!S+z|&?
zC%8uN`8-+6YWrl)VEF3Pv)>MHr&@K#l}AcExC(z@v6@_^ff7gM;p*gSg{n7j53q8O
z38DA}bOC|bjizWMV5gmVEe<Lw{@Sx?KA%Q^9BIUSr}0R!u|ko7*a0vkvwLH<E;?e_
zw*lAmoPDmKvu4SkF=VW7F!54%GgXW~WX~**jN@F7{y^B@g6VnOD$i!#7R?58haniY
zVz7n1W@ag91x6`ty%{}qlj%TwQA~~`>FyMBl=^erndFpK1FlU5&#CB?MnU@%+p=7T
z+3xE_Tx0WyikynpueM&%@WTOYi&#S(IH#KkgZYU;WDJ|KM{dLM=1f;ice{Y6Sdx3O
zQ-`T{9c92)G~<F81qZM#B58@)G#7FnmY4>WA5TQX5}&%ri3bzW05O-mT%vyjKA(OJ
z&1k~C>m%z1<2_QZ)v?He_@u|JkpoE_1k76D24ar!7~hjxmco>Honlr>6}D2S9KeND
zZZ;uq-kUv=Bc<jI$K`YK?9>_`sa#?ZOLm4^Ozv|070kOtAHwnIx};k6<g2}!wOLJ&
zRwHcfbr~BW2P9MR$y}?EyMk*^S{8nN6eD}6`=V_;S+FZVnnl+h`*yh~IHl>$AE=%6
zlc^u<;?wK!g(~G}v<<=@rY&)%<9tLcbkobLHcaiP1v*lZ=So=Prtv36c~?o6BA@;K
z;qnVlS&H`lmy;7D&%xjzS@j~{1N7}F{H)ng#percmAganGNA;>Ti&aJlt+7{T;y4D
zy7VOr9+n}=b2m2mqPOR6RPwRA=Sv}ua#Rc2Scs9a<vltI1S~oZoz^ZjIoeo~zeTF<
z@HKQ*IIIp6(k~xQ&b39#b|vxKl9jF?WmdALCerJr4@plA4rtjcB!2&>noYsLicgY0
zjd?)(N-UhXX%Tsn?9Cigd{T5)=~2Ez&>FWZuDa-+JpM{kr9Ve}C5*Chcl+4!-XCjn
zey`m)>*9^j8Hd@$DF^9t;^3{BNU2}4T*mLUOSzMGZFgIopUTP?%(Qe@&I|FfSS*}A
z9k{x;djG)zqWPq2Y3jJonAo@_q%qZdN?wJ52_0+y)L%p%PDYztlb@!)pzEFJjNQ5)
z(-G&j=8dCUY4@^t-pqJC^3bIKvD@K&Ml*A-rnq{8EjlYowIQCjLVA~O<6a${y6lU;
zAOCXRRGmKq{>SY82Du>;R*g*KcJIiEn$Wfhmxag3FCY+3lPw*$wi6(UkN1>PPa5pn
znoo$|U3!dkHxl?^D%DO)RFL1WJ=YF-M|bHJngW$G<e+WR6Acx&3G6BA4pS2^cT&t|
z#BrpZj9Ts<P!Bzz$xeyg5k3XRqjlItFqEAx7+=I+-Q7{>s$1<gzm9R6?mi8IV%X9+
z7e`D>zj2T0fni?V-b?buAy2PQj7NXq++)eF_&@1zG{nQ-A`c6>Ekg)od(!p0Yi6RF
zNmMsRwlfoWwLZe=?Yk4V5u@LA7VK=LS1c{w#Pk|Q$LVkYr&+gSj?<~X$_Srj^|$+%
zHGX9LHP(#wS_5~92~X8qBik!>1^%m}jw#!CwsHRLyd(*?uoiT#-O_n@GNm?s8Z(`9
z%`2(VjHc3%jcd^*PhpU2Zu7UIeMpb~xQ^9UCFn!>E?wT+Ghq#zxZweU;P_$M^JkT#
zjCE$9sbB9z5Lv-gqd)=Qiq72bS0BZN{1Ss7!V_QM=J;DNGVxnwBQ7U6zA^!(0{i>P
zv)5QhGvmF-SZsF+SZr&n9?#p#HGU{ci+|5s+3<i!Bv*ZK%2Fa<|3TER6<d!*<C%P^
z_3?^dFhg)km&Th))i-HjW|u9f-Y;Ky5kO=$Tk|yQ1JUHONBk<|mRyPL)U_krJPxzA
z1-tKV%!j{2)X)-sawGlJ<nPab9k$^;tiN2mndVF@F<$5Sfk(>!U4K>lPR-FmvUgxt
z4yUt%CE{e8FV9aIyK$1Wk=8vQzG>g_3)8&+>(_<l7$ni~Td@xj$OTXG61k}_wg(RA
za}hLx@j?sVCXuz}HG&;6Wo?|@a4S5!D)>?kS&pcbMNiDI2yR5otufrf3b4=DjfM7I
zoSR;YqSc?B!mi`sxSYIkPA&z_E`NR?oN<4>dDhs=ogSxj724{Pg|57bDNs43l~*D+
zu)?oK#l*%}C{CvdSlX2;{cwLbyDCB8;Q$#u*OZ6v>bP6e&ScLO>e{UbY^Ew01yoDE
z4%i`1v3PZ(wNXqFx>Dm2@LIUHRAdK39jsZfLcJd!kDAS*>_GX5y{s)jpkq+y_926-
zw#JhJ#(vrg){R|q7u*@0mlbku2t6gb@>eXQmZ@KJqFW1ZC&wh=l6!524-}|=uNyHf
zpvN}Y5iXWv%*Syps|Y<UJ{`z*OvMYaq)<*f^qfbrUY(RWoIYyJnqq6&T~tvNzgdT0
zevF)=scP^bKk!^Z>BpPTBFAt;mkScmVZkEEW8WS0AF4r;a0<%<r?Bqk&ye?z`ES*#
zWd8OQScGq+?om{p)q6Nh)g3&8d781%SPN<&?&i^5JrX2ec(hJIYdO@B+PGsL`p(oc
z+7Y|oCBkFcox*E+S`shU>d5+80@;0HuMviO<ivVEu6wHGx0tI(6%1RE_0_PLWKf8=
zr+Zt<SGk*hAUQFYd7bVfljesbskB2BJ#n3VA>8y=`S3kdzN2voc8r%is-d)rK@u?Q
zFW(~>RAq69P6V2`?CZJ&%Vztr=WF_^W)$aZ()3&WvEXq!bZeu0lp7yxOx$NGJI4!%
zij`d}lMf~*Bf0!np27-NVD#Hdeb}^%J)0&INZU9a{&mB??XQg-78`unQM`2={Rzw8
z>XFp)>#Or#8OS_U82lqT7Cg49goAr|udkmqdf>plO>igfdN9Yn>rW6ByXL6tRi^%G
zo!4NpVE<UTm&e0^#TG}Cv7mm>M_oJL3dK>rSH~*ZyNGojmdkJX2g{%|dQy>d3LhKJ
zd2{Hov@hpL{J=)pWSR=|#>y^|=;#pmE@Jy1UOPgkhW@AlH0aa-ynm~Qc1wLU(CvEZ
zss%dXEV=?9X3ggKW4Q3qUFZa8mkAX4J~ke&Q<DC12w6<nAOBeM>jvZenjAOP4qMxG
ztiOn&1}9gFjF#uenZ@3zsfS-bL+T$l&CN$29KM8|nXynH*=&An$6yrj%F7`#BRgoj
zDa0ESD=-@LP~TheXvX@daVFR63h4Aq@aYXEQB8T&45ASb>Fv8qO5&ExA+OG(l>6Z&
zK2_mrXo63YRxdtQj{JIy<9l;q1(9?CS$7k?MYK(c>;^fux!0k6A;Wm3<~YBO)Be}8
zsXB9!P;QJ<Td~gz0lmNB4)e7mmTKId4?X3Vbmmc9MI66W`rc=HwMzDqNJ2Tsp6Sbn
z6VG0Xm3oJQIR7o)oe{Y1Uc^&voL1-FNuALZi-Y&F^CgjQVl3o8h06=B8_&P~s<UFv
zo*w7-arCOV?^SG64&0teCaig9&N;aEfPyF==bjc{<G89${sUWVi`gpmC)SfJ$SBX|
zT$qat`_jPg?3gv8fqRdJIz5Jxs92oyZ<okUExJ^U`+6FW?TON=ZLWULSkvCk{!Yu0
zJ#*vRj%^P?yEZy+y#6Rh%sSk=p6Bf7Wnw4VUN`4`>o>vO_zjFK4ZQH3$IDY2JcK<3
z^;M<Ul5hmJs|mwj!sha{wlq3iMEq-Wc315jy?auo%dJS}jk6m+nt#da(r4jgEHNz&
zCG2|Xn_|H^$;Mv~w2Fh9K{F+k(4<ImI4z%|%DIC?*8;gXV;i%1IG5D@tA{ce&xk5O
zGRzMfb65u)^7mvV&@ax@-||t%Dr~vx^o9y_SwQ$DVx+`WI*L&;RNWeWs9*!s?^_fq
z3aFdI(W1DD!Nh@6z>*ZS;~tPmQquG$tjUkIA0w(_-olV<wcJ8oEK|lRGNHYBgsy0%
zWn`r62kEE99uZzXl~pnFQOk|{YrDmZjg~k8{Jt^TW8y74?|;QNL@Y7=9yw}0X+L}G
zuG>_vBPEzc9bjLPw@1~ZKWbA+=?kA5=MU%b=)q_&A~@l!+w&T!p9$krODXJ#85M8i
z>w)i__V}+I_V5?c^a#s{7!wLO1iG2!kA;#ZEbeY<!mE7muSp^rm$_=)Z<@)6-sBRG
zAtN7U_{FZ(KS{xBU&Gv=Rr(RX^HDr<-FqNUtu<*r4QS7=*flZop_M=RH4C_ow%FQ?
zo`;?k@=8qf3<4X5d(z9~$SK)1)3&(Z>wHL;Bg=v?-~1#RA`w?+V}>2hIt!;Cu*CgI
z$6g9)n48(_%7f?>d-pL9aAr`bIDH7$;)<OcD5B4dj$hcyb}K;1*`rY3wwH%*zKfT+
zpZ)uzp9JrRnV0Z!Zn{u8R?4uh$n-k>Qq>R4&{^|Ga{;=@mM7h~X#7%r5Wy~Pcanic
z{vMITY0(O4Ab&y79@#*{$o`xAq|7gU`&UP7!zadmR=PX7;c#&~?bD@-4x7tA#D0ON
zfoX6jT%R9x4}_qKyonXAUPM?5Q=mjH3zKHKQ*g6qnoJ_EGD!GYHfp>u0^guOgB&%d
zwcF<NX073K*LV={(z|K2_;0E4n-<@Vr@X6&>rSoIUgxci76_184<Fv0dHQ^BOf!D-
z`NGwu(dHvKvby0&|H~fkaFz2X9x)N&jtTUd7&j$8ASLvztv|_kNUkx|vt<}*OTCFP
zT@DG87aXIwj;_u7TrSwuFHV2x);ckPEvuJSO@BvVN~85sp}ob{uST)C_yQeywo)t;
z+_X9V)j0WQjp@!SF0pQ0rEiY3a4MZAiMyLg#TN@bMzN^&Qhie7UX#@<gmq!}S5GLF
z)SRv@PK7Om4<M%;r%G$(mq=KXggdZ)Z?s_`yf=ybWES2mO_<t*gLAS9kNavV;rqOb
z9G8Gwz683iGwb?kh;Jl<9KAR7=hb!B22&d?lwCiE9aoxsUZCw~e;&%B!x(y8Z(a$1
za`M^uS9o29`UYY6&k@#G1`b#0WS$>8u1Cv3_1nlL$JF0%_pbGC`0Y*hezR$;rrcJG
z^htI6?t07k%e7r%op|~AjTuE*p}R+YF5`pWZ)!eq)e|jn^Ixv?^X%u*+acSZE3QNt
z{?(f-***Dk>l$mPZo<P7s)pSgVf+O-J|AyIXF3inx^V8jc)RWic85E)RMULGQ(q{f
z=e&%IVPTQ*yDMpwno&ZRuqqYkXq$8;EFG3jkm575ia`N35iN=<!X2L}$t4?55S=Wo
zXGf+{ggOi~%Yo>WlKcTfl#V;!le@V7=s5jx^`ywJ!*XJCOH>jlq5(ERz-EfPMa*Bc
z-*|GIx$pC-Z;{Jw<ff6kwx^z3zih0`ei&y<kcn=$f1L`B!4CY0*RAf>$8^WWCLOs-
zKkt6Gr%sYqYSr}x-FVoqoqBaR8>;-2`VO9jYoL-gEgln(dLaalg>O|Y=iv+*vSfoP
ziYhnXGH_DZahcRKai=_XHgZ^%mQ<KJ<HksL5c~Y*{#tW+)4kQ~qP=T}M;ngzQY5;#
zB|C^<g_5qvB^wyMi5b5UIL#zkj{HkVCI6Ye-&+I2j^>*`)YwBmOcE>)(Jxfd&!3RM
z6P^&&W|)NJ#6%o#1n<4=C1b5NW1&4sTIiS<@)+W=a_Wj<PW;+F{8HBN&Fn&Q4h3SW
z;gzmEQbd|@cqnDa$?TgrHrXB0?T)&;5Bml?7zP;su*UM_CBmFLEDeNZme>kn^!qrB
za<^RrV+_Y_yOQek=HK;oYkHC5XPJJ43BI{sfmVRY4Fz!Btfhi4ZF?#zxT!3U58Tk?
zu(ztdG*(ei3vThnXQFRD7k9vIOb{43$T{67Q)u}_<HH3c*y@IxhoJ39I0=5%aF%Gc
zfz|E5aQK?t!Ni|i)WQuP2obiPwXq9dIb?XH?Sx59dV_xN4q=%_TzQP1%ErFur^Rvd
zSF5+v5Ya|mqkOM+?;NiZK5m{|+T{}X#D&4amgDS7zgG65T;_DRrul&67lpX!{lo{N
zV^ZF01$8&J7uuJIQkQ5)ie?HuN?q&dbv>rG(%qY=?>1E~V9|%2JsGJx(O|)pe>>}O
z#*dU=I}$=?O~Pwk@pu|ZUemIZ*=M)jh;Avi01IRZ=+$B1eH-6(FO9#`DS+kVH1}}N
zg&k)Agj%=|u^tFzL3XsHtj4P_3;mueSGXQZJ`+~DcNEx_7@i=upcIJN@MKoQVK3A+
z`NTU37hBDFWN!?UUW2+KHci|Y&w7C$78=_lwjnDU7aaYi%5t{VB4aM7J0ClHsCd^X
zmen#cmSDcSZll@ddiM4##Q|uKKx;OXWV|j0`(ySw9WlG;!BZ67^d=_W*0XoFVXM1E
zfO@i<mm~JpOZ+JjT)gBPE4X%k;7r3l`cdN|4FCEadRO98I&f!HIN_UIkhi%*O3Kt6
zL{MQDNKXXCfP_9kZXhy@)4g+lo+E%>8rAj=mWaY3g$CV}PN7+zOx$*c)$uNlqgbNR
zX=1M|US+j}0X)<*EJ;vxa?iwhH4r{=6hc;`RE>-?oWITkbBJ4_G-{y`Ur{VHQ$<Xx
z!R|XRDK%<0yXuy;$o%}UG-K6ox7yo+5l4nK`c#7aYjov;+`&@I_rPmL?x{dDqw#-I
zo*xByMI$ENa&jH`dyuO|h_z*Fh3w_BD_zmTGDmX##(!p6IyV)0Zz`|SWK5k6x6mOS
zDc;=g-uNEGL>6to!GVmE&JxR!q<s+#Wa@6VL5Qt9K)shbmGcf!-Aw$!;^EidU#_Jp
z(w2vKet|E9)2x4BKl=c`)w;hq7A2T5a8lQqgmHGCu4<#Q`6fmtWdMw4R-*t>;k3%u
ze~+4p>-Wblru3kvxqWuaQ~m6n)x&3%u!1HswoN&Bf+7Y^5(fggxM_bbs~b2-N9^wv
zV`yO~hXX<2ZKl}z1AO)GK-}-%5askuU`tI)AC&#gR9ry`x{<40qXGdNKpMU;Q<m5b
zIpLy>^XZR(QIXjeJ^s1KfWX9F;RGr!75yn*Zug>hkY~h{T=O+s>wI|-$Ys1%t!<*l
zlxQZ90@+WO4=^u3E)0$^5GxEWkp`iF*}5%ViP$4Vd$%P<{2I9LFUgi;2u+h$>0_76
z$38L9Xf~W3R-A6<JA10S`7<|+Cm+{E@zFh$U##`HVNh=K^dYZ$E%QdOH8bg$;F2;m
z2ub@+9sYVN>s~bDPb(QhqF$RHKf2KXmki&c#1-yn&*#ze3)M21NAK&Q-}B&>L}%r7
zd4G#NQ+UL2_t7s__&8gY<CKBO%|u6utD*Z*tQ#K86Yk>G^PC;j-sH*Qe^T&fo|+o0
z)E{}(#NtUJ?YoZ<BMS`@dA#uXtX}=oapVJ}he8qp?z#@sn}UVYxx6Z9*RFL{w`uS1
zLQ@=N|Dx!hWl%1-|7&GF32E1yT5O_x7PGq!Oc(1KuN)W3t0e`O)&jwXH39180)UO6
zow<T$^#J6v;xkh}U&Md)=zQ9vS=vF)n~vXoC#=>D(U!pDINSTY|A~@s;<CItl#y-0
zM0*&}i;;WySvfj8LL=z(4}bk!=n36yh}KB0yWA|pfS8l#3PH}V+3WCm6+7~$Zp*5D
z<&7MOmkXK?Rm)$)BHRWsPOhJ@X@>UCI3F$H_1{UjUk}{NV$T*zDC8kS<ZVqxP!QB~
zxT(}xVGNEf3b|%BBh_wi;X`MqHPmicPET+G<GJ#v*)w(GiGSn^f9P6CMbs>!O%Qv#
zx4+fUxhMC>_vp9HP;RU?Lh2@0BhwNIA@B2mu9URjT{DpzlADa8{@zqXcK}Kg8;UmD
z)0)d~e!FWkynTd1?>5yw|K+x5W5)A}I33d-9Tg&}r9bUttjQjSB?H+mOw=D(uEaf=
zu{0li^}fo+W6HbaJH{CrRY0%gA5P55^=hX~CP#2m%b6`pB4tiyI-!|NX?1G(Dvuo%
zscZXDRZD+T)0fDNG=A9Mwtm7wM18_^Jwa|;r15>=TGG6AGRxRQ5SlcppNj5&9;bS8
z)WGij98~73cJ`R!wUlwd37;4Q?yR$`0t^T<#wpj$kz|b1JEU!L(cnZ%Q3o4*vN3UH
zMxosByG(!8NsTQLqrI-whD+J$i^R#;XpLj;q6f>mn7`G)j8g!A_exbo%|_X%z)C&&
zr`(jQ6D{ub)s16u@>49yULH*2JL6<MNWvee?mWzXulqsbF=6t@0M5nH8<n}x63*O@
zB@SyG$X0B5`<QEL^@O|~h2<{6dGbyoQRp>rwnrtsgYys+i=sknYcD`V$8GWZk9^(F
zx^%p)Cp)n@&&c`8t#s6XHjGha3WTZ>Q}ed?p7wbnSVYC>28YO)4)tDmx2`84X!d-~
zNZn=pgZlJKu3=l5g}@^$?x*5d5kIe`qmw^<0G<C{mOTBb#j;22*^`G;3of_|T0xnO
z2e()zDtRDmz2mAxobEGq8*hatXV{#)Og(a1+Xg)m$ZVE#8<gM&r1Q@Tp7?xUE;wr8
z>nH_xsea?^==IH~4BFYvujMgX3Xje(3as^;`SC7@O}{T1g!9U_NB%_5?AU=EfF-;R
zUunsAp*%17aZ9#XAPg2LVAtj1B8OOkVO#vs<bx3`&d!sLDzozwIR$VhvOM!I+HOv@
zj%9awdazPZqEN3TeDG>+g|#u$^I*_c0eSF}6=9mIb*hoD`LN8fY0-o1U62bGUhYZ)
zJ&{ayLDcgFK8qoT)L4Y*%Kf9GjUe}$?%M$rCcP~yVhc^Z#<PWXOOv}A!x`7Kt2>{F
zIxUsIoo~0t_<X3Hy*1#e@cEnd{06gP=8l8yQjKhl+cg|_X6>DgV)wj?PJT+o%$sbs
zHUw^&Cb7VE@eQo-_xwXto*QXq8&=9p-f<?ZgXs-_>rZrru6z;qJ&}rqn#bPkPK@@1
zBgRtQ<_!8ohC_y>{@jKpjzqAAsH|Nc$_Fo9r!E@gcKc7yOpG|Mp@!gQELgQWs_`es
zh0};GZPPedp-m9j#5U@0b!XKTa!~i<X8gGN&dG&6clv|+RAtE7sB;%t;H{}AX1|{I
z*?#kA4g2kmr&Qy=-+xB_*?R*;8`A)_t+P4dGQWOd&~Sx(4iF0<p~Y+BLL1BHR?G)3
z4@IfvjXrntHme{FWkP-zrRyv>a!$fVrs7=R^%)0X<pmvMztz4l5pvw+)JE4OJDuu9
zUxmcCM{rs1n5t(x-@gdh!6B7I;_8{Ctt3CyZg|waM)cDZVS6MSmbS8c$a}r~R$gRJ
zYNH&#G9oX=aECI(?<SpG0w+T1xZFNQ6^D$!o7fk>?X?e92fZ^lJ*mwiG7qEEPPm$F
zFMSGZSEbB{cF3YfR}J3;Xyud1@@S<*%H`(!#4?YH3d0^%5@1V7MXxa(er+7|toy@>
z=ozomZp<}JCN>vv-y?Cbb!k`@b;90%a$F40t)7b_rfHYuiot!Zn!8^c+Wq6cODHyf
z!L9g=M(wpR7cHk@HgF*ENj-YHHNkbYmP|bBNkFfqxm}h|(6x}m3Wqb#gAU)l2hUHV
zjt^|uDTCJuj*bde6ly%GokV5iN%;L4kgUH$>)~X#*XOFLabg1JCXy!{BNmSz$o|OR
z{WR1g+=zj#!lBpOJkj6iSRbvJ)g?J{!?Pnnv^unFMB&%2H?PO%Yng}NbQ%<^&A#!t
zKf^;zo`mj+z`6$09;95K<GiwtXe-(Nxw0$!dRahDclQ`al}K+?L2w265J@^R*Yi6f
zb*?WXiXlOL`=vZWfay#owp?MfG?~bD`1VolM)#2p5tmzcZvBp}kxW%t2+{jR4|a`$
zRI&BPNb!^0?~#5(hn_tR&BH-8{7!X4ivIll4f-dVgRpeJuUqJ>)n{GW&(r@Hp3LoU
zu^;v(2p%zUwSt6v%u!K(|D?gFC1Z-%naZxM%Gs;3RU#vDp2G^BtkVf4k1rA;{)b-t
z0G21qV<-VIC2yS&if$JHh0VxsTQe;g#_dtyUe9ESGK)bYka5pHlW3pG>31Z9zU8BL
z9qn;I@1tdwf|lBnf#Ygz^8G3TPl_xYjj=MfAG={?e(-|IiH3k=ZZ|QR4Fb~lU+XQ&
zFiG>XU9*$szr#FmQlmY$atL{)VJ&6_BtXAQb5!Wf$!^xb>?$jokze^8bgbCLUabk-
zDN*l!xe)8X_PY*W2WP?=W38wsYl;t!tEn5{#|WHr&e}Z-?WOUeYFMP3ax~+JX57r_
z@JYy$b}I3Wdn+bn0^a-Ir8DcR<zTuVwfl5S@QS3Hs~bn;&no>t_LB+G-IDK<rmZr=
zNz*BK7T>zc?@|x4^(ILPKZ75+qciE>o-ta-n2Ob*;hlPWO}U65abu?Oon`0poV0Ev
z2w1+a3p(&P+-c}bKX5$&NkzP{Dwjvf+stk-i3i!cPK3y>9Ca4)d+Uh$oapZNG5#k+
zQs<GdxXG@!Y@_W`$fr9ku6S{kEcYj!5|+NF)|KFQZOrSCA~b;ijwf^zmQ%E|yd~4&
zjwE(o>~ZrW?306Qd!IPd@+A<DNZz=&L2H3Kg`D#AkkZ}V$o$DJ8o?*y8-tFh{<*qv
z%V08enK77%&uHj5N3!0J^z=9PC)78tcXJUL{vdFX=Q`1ZOD_1|GH29=J&SXba+-`_
zeel2_f5ofBWr`I#02NywQ6iA}3q5v!{K;K@zWvN=8__j&7dBFYjHw;)EcaMm&Yr6;
zUr*B)CxB1o_zAAtaqkt;9~^hV+B=;85LZ9mbBpq-e`zs9XrfZH5MjTOhD_Z{G>Os2
zomkmXUhpPF$~oc<yXCtyz>mx;k{mm5jn}Ve!N$LsgIp%3Xf4g%rbQkIv@rqe{5WV?
zUL|j?vwM1CvBBGysy-L<?UyA+D9+)@doxzb`J>$UQo4w}w>NE%16q>2yrgkwO!ef*
z{~JgNL_<87Hz!=CB<tLFwa!qEDr&*4?tKm3<E{LjUN7a&VM4g~eXd}zu1hWX%8|NT
z6s-Kt5*TN+m!G62cd)SawI0KX)30;-Lg{%waa_#Xv(6E&&*QrJX|_GO|EBGy@R<pC
zukIU59Fny;g)e&fXbZ{q8wDsqI;t?Q#YWkaNN;con}{bxer@;w*P$Bim+)Rn>IpMA
z{Ze_0C(9?k9{-VR&G^S}v#aKH<pRHk{Ev9g!n|<!7byBHZKv$jq#|pd@M&rk+&gl`
zCK{;dI61g#L0m&}dYbz^e6Ozu_ZbXk_F_e+)O5l*znk7%7h8WtKydUeBhmfU(O1@(
z97rC1-Ccn+e%r3Z8a?-*CU{Ph<C$KApvV3v`i%@})%4FRu!1`M@RBcqlkGc|Y{&(~
zjfM!vV@kL)vNA!dD(yhCJLx2ImSnyx)K3zIBy%Y<e&T35Ko<J{w%1M5`FNk5s6q%>
zJUY2gpcG6VLNPuIeCuzsrO3+71f8m03!=Zd+q7RIl>l(HBGN*zKZ~NCYq0|$;<TmW
zwYVw&Y@xcBVh5b}>ipD#kXKZAFX8!D0nQ5Vg$=VG1Zo4|8M)hP4PJEs(cK<G`V_)u
zv)PPu!*PjGP9x(KA7?{ga6Myi?qjI9Wh{Ed#|V?d{E1HLd_HTVgD399NRDd2Uj32#
zAU?atM2cuuN7EiJ-EAyX5!}Y;D=5hT<iGzZBH|oE3}MC4^R=p|W?Dj{Z6>O#XgRu=
zK7RbDP~~8pr5b+-eJO<j5Laz2-+9|E;2i6<OB}+A_iZK2w@SOQZ?4NTh%i&mUufX>
zPp97BC~z-iX6&a)O64pfa{vmJ3?gfx!UgDi+FREdE+Rx%eD=}zR!0oozk8gWAQf^{
znAq>C1M5696<c;beb9eBz+U{R0u~7SO}CUi&9L@2-(JZ)$#;|1!i#ogteA*Wj(f{b
z1toqEqHb1ytRF(=-(f}3MG&Bn_E91gFM(GGG<;^Qhn;bpj<N%>lO5Olzr1{~pd3zg
zKKmvB6b!WdK?HKLdEc@X7iKp<Dgq?18vx*!ERm)aFk&`=DFllCxPjJ31)y{I{2`Z>
z5G_~<-1a{@P!JcOc_T2Pa4uV&7I^%P^LHPEu$W%~>iKE=Az=AigB(RuL5TOaqZKdH
z#dYXWOq#R+&KlitdMNFVTxSAZC;if;qq`;}Q+LjJ@edDsj;Rs_u{Odr2_{D9a2cEV
znd1CAKpXD)0MttRXJKC>0K8ZDszlFDAhXn$k*C1d<^w4et^~l03RyTZo$rsA*kgat
zKfr*bFYG)o4FwQ99uxwA8icV07-GpFd?uqGZ?A{58j4RPB$S*7goZEin&$xGK7g-|
zcqQ8P`n5lZ8ev?sJ^wF=*gKtTRRpbUIJ0)O1kkV!t;ax*v9)n6LLhqVWC43@|3BEn
zj|L#EbP%)}0Q04Ti8zhc$EzGb1?X_v=BGQ*QR!O)z&4)l&aIyFVF;>JFsXU*XbkXq
zxP0j$H^hGMx_yypP{0!v5P&HO0CgY81$BNjHhS7?be;2N`Yu>XS*6sw=OGk{KmdxB
zDjg>9p@vEZs0}bKTnf9?Z><q@8M5(DwtxGROJzt!o>xlLAKB2UMDm_1RJs^YbS8I_
zb0EH+Dj2q=oMf*23*e0yv7f=^k~>3F$yK8SN*Q6;5+YfkQ)krkSvd3ghqL@&vF8l5
z0!z+tEHiEToP2Yz93-!j`BVKjxGR`^3Mv3mg@H=7UjU2@jL{t^GzX$R$)c9hW#@~{
z1E}o`p19l_i069jfS`EJ`^{z5fS^ImfbC0y<k6nkj6a0niN@{t{lq4H>5}2}inPfI
z@e$|z<5mND_dQs8aP=W5W4fn~n0Q66PE|1b>tl0>^5+iBVMs5@)}GN@|Bs+Y7?BSI
za{5HCn68*57tU@f7t3iG1r<kR^W1AWANy@8;P=CFUgz?{ZvbuB_D0N7TkuOOMNLI$
z6kk<|W@YJJY!qU!zK1Kf9#-E2GRYl&^y$3122xn6Lhmy#s15*VSM27TFOERi>c~N;
zbMmyq9UwRgB?zvTks<muNz+S^!u!mVxs{iI`IEFw4U=W~5xRr_8?l<Uq`H<4nuh(=
zdv3)VOF}wHmXx@>5H7fXXn+G;^2UmyKTmbB(s0erYcqgm&jQ%@_9SdWvOiPysy14_
zKg8o{G{9rJQYz;Fn#%x7(a_HmZYK#tx2gqpdoDEl&RQJrFkw-F@M7cb_R?1CSy1tZ
zWnf;(fqDxc*gd-$g)%*0#n40(EibLu6dDBU2f#YKwUg~%4?=%|$Pu;o>9JQ08QnSB
z$*vMew2_#bBL{d)#ik9CH}EU@h0#D@?u!Q7_cbe^*%*L@+?Dp(8jk@--lTMKUM$sb
zET{m(&cUyHi(G&aicCTJ<P6_pkZjO%RQT^KT&KtsKEZ&}D3r=BkARl9f)yu8d$)!%
z45V~oW_gMTE>@hH+B7)nMN~|AX<oZ|p8~+9BffljNUvF*`%YS>_gpvs1;|<n0K~53
z)dX{u-S6<b3~`z^4HK~S7{mfgTlH=YjW+Mc_c-???S2*d`pdOt${x*CFnL$|h1ES0
zdq1w<rCO_&gvL;4j_~4fezczOgl4@;5!5V%PunLEdrcOq4xN>1$|V5!mD^_zVR}bz
zuWs$y$(7TyV@Kij29{(fOtlTyH8t);j?KY!&%(e#wtXg2TEgeJ!i}^M1C0b=xM#l1
zr?dUH8Vq23Bz(VtIwKlJ#ts0jjqgZ!3@YkA5`*M1w(0DPzczypu$u3&>&ji9forV)
z31Y3Ylw|>1JC1bk9sZ8$UBR2-4#rrn)`BkUf^K<de1u2q3$(AdMi&n^7augwcA{Ww
zp|tQ9aY+tlU2AVEsAy6b1WC9haBQ<5g5~$T{<4lpGa4aiaL<3Tk(_Uz%osN%3yS-2
zO~>ktb#G(8j`A2+tk<rS^$S*`0#J_Nkqtj$wqGBzt>@nP-7_58x`qO74r}86%9Aa#
z2^QF!scYbRrN2{9p!|GB$>(|KwA7R0JMO-c$7cBRX7<Z8@kd=vlz-O*HMBQk-ilv+
zgbVsM!x?(n3<k3Nay_cvr1C42X0>)DJ~ujuj-Ty_MckPj=4_bCy=AX<1XBC)+9?6B
zgYkP`6P~&VhvPMIShEUKrK!}oFQ-SCT10*nL3eB&vc{N07!!9cT>zNulX>R{Nt=w?
zU!@%b1SW(6&pxtpt8!W@hH^*h(d-MI+b#Jn)Q<BXSX;o%S|D6<CVh4QClE$%@2yX$
zmouZVoBoEvPS63`{?GvH0hP>D`NjPD_=dd@Dh&V|$EA9YNK)JvmO1^w(poPVA|1Sp
zZ0(cR+56L{-|$mji_%(CHFC@bLAKpt*>dLjP^~m)bp*!PN?`(!E`3GqGLA}4M$gU1
z7k|g}2ges`@zyz6$AfZ&$x?-mC!RC640b#LinXoAFfR7-b3qu{5kLb5-m=d;DxvrX
zX)MnN>)UzEFZ{w{3-O_X@(fPVkJ{Bvl0QR889?UzgDH#3-;bvs03$>$>_EhDz0x-(
zRm?#;adfLYC+qvO);n%>?-NH;T1)8=c~zq^ZaNxexl|yFvCV5MB}mN`kbT?~zuj4(
z*;!~hmG8mdWxP&5(Y_Ost(jHJcvu(@f8~OU(3-$7`d*u_11jwVshg#DIaZt&i#6n<
zw?@0ba)GLO0!-5H3<1}vNKW$wWQ8GRl6W=^DrQaY7Xv$xtyYu@Dj1&Mz-Ns4b^D_J
z<QoRCv;so2cUAE0Fjxa4SJKcY=0-;6YF|kL1TGx}6=ytf^5sePBevE6I3HyxDZiU=
znDzr6a)5a?YN#u_?kye~5SFj~=a&9h))dC}P*1Z-WpSeBb2sf#(x%$cnvk1rB_HAC
zY0u@Q^J~{StDA$|CDt^y{MB_Zwop)Nz#16NYkpYJ4AubSq|n{VXzVl48-N(vV|+1i
zq~Q75#)~~E4xsIUurFltU%x&B-#@nPxQEI-eam2-$+P0wTS20YU1xz$Ki)aGSxZaf
zrqxL-^9yQtO}SIdC-9z0@xz@pIC2<6`860#77JoX@3=-JP>A~r_y6v3ish;Z;`nl=
zmrzpWf4~30u<Mvp)~1?Xu|oi)8z8e)Y}7Es5Klqt_kjzP`Ulu+1Sup_0PBucg;L3P
zVSlL^?Y91i5k9es*;#?h-ymRS?CtGUsP{OyEQq{<S`T)ct+Iy;C9hvU4d}HK5q&?)
z4B%TkM4-$%I<};NHDcnd@>P~rl>_H|U$>yDFqP@6Fk+iuxFyUfT&>p)R0>9DLzM~5
z<ce1NRcEv5q+58jjRq{tM-ASnH_RYk>U_r&mLXwQc^d~d_%CJLySpVoiIaVJ&P$sB
z=hEf$+M}3Em}5YRj})k=HeMdchH4@**NsU2RtpPR0M05fwAkb0W2)fJNTxQznAA#D
zXk`x+o}3=~j|@~9I=h)VDRxi%@k~yj`02P`xcDoiud=t%zKhd+Rbgq;G6YnE2zMnV
z&@T0#ojo>3W3{hTyM1|{hIa#%^SU<w*VV?P2y^g(3c^5f7`0}(6&tX6*}yQg$^W?h
z*V65a06&M|eL|xFstTsTI5kvQ(I#I@ZwX%ur+T*$wx%gw%|){x2V{v;Ka+n8>_b!A
z^RKUMYb00OcfqSx7~O1-2^8i}kIx~$W7iF8x%YA3L$1c~^1w(aL`~D&z~7fqgt4I{
zUd;qnso3MdAzQ1exFcCm*QREz{8QpvsE`UQs?7|*QWj30XfZSe;wC6PbUN%UqS2rx
zMk{YNrySp^*brV$ni=c7;(ecbt_^>4G#~NRwd?zzs+X4uA}vRhOvE_Jf~SM;eHZ?q
zd!0<0T%nu1-QH)~;kD?z?L~%J8E*YjY=#z^1#4)mIwT&LVo)KPmGkoIb6{r#|2SA&
zF7f(r;CcvsKZ>W`r?npl*~LMreB!b12;N_d>kZb|zH}7g+RJaGFVdYtVp?*r!<DCw
zncX&Om7kok4Gh8##Cyq@h*F(a2npF2q*uX`Aj?s5{yYFHuf0h$+(S|@2!Cue2iHuu
zZ-O~&&kEkz`_XuZv&Jx-trgn<pXES(d5D|_>~N9!+DTC_8yjhWeRSCRV{AU+UfdC=
z!vv)JX;2p#4O}mnb~MG$AYYu6hv0boK2I+31pG4dTuTlrbc2k)Bo^?p#6g)lHB=c2
zRaq%1efobW`|o(H-#307=IT-;MJg#|XM{rbxRf2Tvq?r|Wo2_UB!r9*sce-Ik?pE9
zP|3(jTqUBERVeE|&ex?r@9+1%@5kf!d;e9vIbY{Ej^jMW^LU;+M6F~dHuNJLFJur2
z$WJ?xeg<#GS*a4cbtLa;;6(P^-O~Xx-}04JgX@w-{Zicq>WAH8*qf^_Q%UT#UK>o`
zU-$XVO>3#Os$slrt<4wb-=8W@<hG9R4c0xYSJHoQYjSBM;Fa_DXW23Hw`*pe>m2&l
z8@iScplIh;&Y^S9Z(kC(BjYMWHR299EmjOa;z({<Q*!Qkoz-E({bb+aDaAGD)aWLM
zYNDXip~A>z4Q+e-b&2($ldv5JE;)*Y`kTU=fEPgx^=5+SvzC<0e^v(L5dgtz0>N6l
zT62U%GpM5D9elf*J^(!biB9(_oy>af{DrpH(!-p;zL_K}JeNxSlo!+rrLx36l~|6Z
z$r#(bsc&<t!7l(>-yf=t$_YgldAYG{yF~epeZFcw^*evPBH3;-)3|nOLEihs5Sz{W
zhtfK?!sZ+XFNz^o;m~`JCdGT}iP4}}a_!Qi9FxM(&LmtqU&3>A$nA4Y%S8?851mhT
zAF$~@>e?&xt@7qJS{bCGOs_Th?xRrTX4P$8iZ)%0B|%4{Lvs{U=Oa&BLH*ki6Et(=
z3)>=fR*D2LD-N%G+Pig|?KrHDgIYogB}ay=`M1lGy?jfZatHd}SYI9`KQ;m=tRBEZ
zs&=s7nxua=Ff&-Q_R24wLs2BqO3`~WODi9%M1FiF%RVLor#b|Lk>{l$0;O0Eir`Q~
zD_)u3H&XzLr``%*HQyuUCeT%I{2YVuS)=lJXAO!z;r^QyvSR{Wi;=`X^NNO(F|tSu
z!lEZmyA8ybFEw4>R~6IQ<m+be`+e<xpi(4=^^z0*n-%AD09$&bYNDt*92UAf(ux(F
z;Y*~W<mr!35v9ZL<+=7er`I>~S#%&`SC}VbX)P+;F36KSIQ>-r05-2Cl*C9$O}d-~
zc7423_AsU2g7Y@cHtctvq-(b(!i%gdEu;=L1aJEJ$BNQnPm}~X%)JtSffC`u+D>&v
z=Jc7Ov0dBrMJa1`6ZHbnP~b#fs?JoZ_h_pYYS|F6=pL)gUc?TPYd@%~x31nLY#1xB
zm>~>|(?zf+$7N;z@^2K97KX#Ip6rI;1?>X8n+_1FCMtync~N|kvRm-;D+96HF`dY@
zhC})unMqB(4fc53mZ9~q3wpgFdBK~b5q}-%o7JF!WIyVr2J#9S_x+=%DHA@A8hv#_
zOE9bPj}+iC?2*jFlSLTXf(l361yn_sFJ8LT^5@sLys(u&;yz<KEs1=`ZpS?SMVZoc
zF-*x*QFbpTi@?=rKiQ#|+zIc09;$K?+8j;sH2i3MS5tv%;n>)#bVmrQ4NGklzwNrd
z?iMC&2g96v`hgfWh7pXB*_qh=lpDs_<w6^cB|9P_*lFT86otTHOg^r>Vtn^RsYlS`
z!Lu*e{qJvfTtCV%7-!Mi3<5leXrTvrxAP;hN_vPfbx18WoM?0^+#AM42wd|{K8=Kw
z8S6xE=`C>YUa;=Wq4aUn2EY~sxK3jY_kW}CBIgL06b}E|k+Nv7Xp-{UGS7!vKZ&CW
zDS}4ozGmg0p2f)L1$D10iYI*BbDI+8*Ioo<lFsRe5|vxh&`b(z7%4NEXECeuFGktY
zA0O)SLYL-IJ84S-*M${fbq30ZG*JUL7i$l8-ZJK(i!H^+KAcC*@OWQQtd}?<;sajP
zw($U@WKvRX!^~&LUJFAebQVau{?aCZ#ZLx5D|axK2OIWZTlIBl6n|jsqraOX6ZiL1
zs`q0r_Jt?c`h9B!0vp~lJ@<Qr1^6hCM-n_&d1zICG!xrOQ-p>MqBl^=KpyD*#{%o*
zC&V3#V4*%gW3f$!+q5-TD=Amy!bW1f0@l<G$&E!A<+0}m;M;<oU<`m$BW3ugX`o2I
zHn50|vOPu2GGWWrbauM{5(>@65)F0$Z%C?5I=h4N|NX%aeBkrVz#mPX*bT>)YZ^Dg
z7=?PPZFZSbl!37phoG4c-TDntDgrh+1;qH0=*GhuSfmKqyeUjX3IBhRodAD98FZ8k
zcpAx&1TdHrD1+|MlIVXI=SWGbk``b+jnbpB*sk=TT?yN3aKsYr%H0})1C;r${vXM4
z$Wx*=Y6-E}@RH~MOqw7LlAjVLH9^1b8r@Y@qzD0cOt)Zb7BiEXDT}h(M}*YJ%etvX
zVa4G2<iQ%BV-60vh%Q5E=j~i<60ohtu&uXeea=$O7CLo7QFZr3+f&9MpCQ8UYNC7)
zjFU7_c_xFAADqX=#w}m|HlY7az3jnl-AQg7eH)o)L2G6OLimY6b%rvx>B}UrD*_@G
z4--I*+Ah*De+Hhp=@*VN`zfoYm9dK0bYrWh-QCL(8)&LMWy=hTw&SEAHTXb<$3T@b
zxez|DPt9~LA8IQPSUl9cYh1*QTo-^E{rUPc6Ke*C!#bh^19A0`jajHS(RvMg6L{cB
zEjMs}PCZr6c}ZDyuDTsGsUqSgKu|P&es{t)Wp!mKW=Z*op^sABKYozGHOo(5-`%rA
z_MnQ9+~^qNl$}5eZ%cDNKdY;KX?<zCWWXKV!X~3X6!c9*(By8is)oifIG=`dpZ?LQ
zBy6VUIGvBXdK9F(5I6Yfd~5qNCR%gY`ULkNAuMIgxEZ%s$}Is1R{CrG?mcvUrw#8=
zS+_uLQNZ}7++Ukbg%7yRO64F5#!d-swvnD5R8{^4@|5@(WD%V^5E<nKFZ}Rc`u-@R
z#txV}TnmKsP5&M<k!qWy`&-Q&Ow{;#e6?rb*=cH^DM3stoTo)8fgj;EmY`;0Vw(N_
zXjmH(x_r>`>`^m%onW2yjdDLn5NPS92@)6gWZdN@GgAMs-0Grvml*LH?4VnZ-42X)
zfQs0R-z8$f3xxpby3_Hy*{{daP+Vqk#8P1%D|~H%OaziGY|JwDN6tgAGtItzl!8%&
zE?zH7K@e0S1tBv2lbOy)&RUn#hzM(%fZHy;*K-DGFP2a(6u)&3c8wz(1gGG9QG4Zj
zI6)*u?-b4j0vl*OJ%l>)(XkD#o&9*B%V+cY&JZ|Ns2A48s)0G+{8AZg3S(I!)q;p{
z<wi#pPG4Bpk%HnPK_s;iPw}Ew)tta~Aij12NM9h=0)$mlWfq3)e|NbYR#!YIm&`y7
zTt(!@od|(SAPpnyxNB^gB8-bFb`Tvpt)>F5Sl6P;O%SnfL89&PwiBO{*R?c)@C2vu
zR;Mpg1oGUvu+2SgEZUd;Ph1tr04F#3qWKDQ-kIK#L>`T;e{<1scX?9&%JdTLG@(n$
zT}_LLK)wOXYNP`b02s?OyFwpb%B&5!{*!aog7y_YIXMfDhF?9#nO8u5F?&cwJ&XpD
zj?09T#O)`A3VZcfLDG5^N`7N&A^}*kI08tBkX43~0`!?t3=s%Xg`2tH?scsPCb4aj
zN3jjnV*+tbzMjiL+jk5U%;vkA3wCNpbjW}a>ud}S4uo>rD9{3412zQB)#cwub%<Zr
z^&-Lqo&LM)cYv=YApws6qJvpH5PU)YcBVk|%lf^~fTe;P>`*Egipz;`+q3|;UuvX;
zH|8T_{(vQD>YtvZ3NDoRsP_DgQIrv^V2i5x?^CuBR2ZgmYYVHm>c*{X0%|e44*NV#
z7O|8nbo}(`--(65iN#{%R8Y+PVQK2Y-WnO<0|E)`5#5;6NQ%T|NPAu{D?13C;7RG5
zoKQ7N*2igY&qeOP;fw(6v&0UIbUAMF0T|_t>v7i$5!N7vlbwdiSma6Sz@d;35d|Td
zuHgB}^&Dno&NrRE+&QFGn9W11CFO{2#F8mWSg4WqfFSmq-lR>bp*_KE+RAp&=W)od
zcP9*sEe<0RBM4O_E~*vEt)Ew)8W>l97r|EWDLi009&|Pk1th>Q#O^T*%F>&+FaGR0
z&d1GtMAW8U;MDWjYY;Ub>fs3a$IEeFn6yFnTNMKz6n7V4kDpCFu>ezm$iu=zW!HJM
zW;SOkg<8n@j&m9q7|gysx-GpINNAqdi3wAb83gg(W{|<VB1db$s&c_>=vvCaA6o{O
zb>z798%m%Hsl?%sxvLX2KGS5CiJ^u%bZR9~j|`Z1*txzcPCIuvbc?!B0d^d1Gnmz#
zy%!IIYZwmR+1g$o8j4=TMr>7{2R?(g2<i>F`ge}xv%K1{O(A0Izd?}X3}9tBE@99J
zhm5Rfp8w-|CF(Qvl{>~DS5B;HnHyF9w37H+u1XK>7J65ZwJteieHcd$^RBn<0-Yc~
z&vOMv3DAirSC3L^3z=L8-3KHmx{Hz=Uf(eJxgyc7FcKSWK)-HwXaiw{$mJ7M0MiBC
zZ1dMuCo*{f;ZQ8kjceMN;WjfM16@_;eRB!(#*mus66^OD4+2$X#?Bb+^uIbzG)AV~
z#ElIoEmU^EuBe8HUM0|ik8#m!#K;f(*cx33(0w=wl&{xkb1ffvtk&&zrf3}4iu1gr
z4X-&L`<O$Dv=S84xa%x<*tsg;eK7>EQT%>_8&SgfjV3VFA>I(fJ4a|Bkr4;2?7Z_H
z6mlSBb=bH-YBVHyQV5*6aARFE_AxTY|DTm&e-<+!CzEvPR7Ip!)^+|V@JveUuU=cv
z1y_Tx_1!@Rp5rvX=F^Q}+6%!3#z<{>c7*g1gc$QbFe?y_ar{8wm}lD*W_X)G-9<lk
z=Q79a+NBz-s*8Sz(E-NUNXbg(nt@V9x5&UeLl85@230+ZQq|%RV;iXWZ=j&bJ3I0m
zu@cry<YydVxp?ERl53)k&b^V*%1x#PIgUH6E<}v@j_%N2haJ1oT}cwc!4@TNWtU4M
zqeSBDWJSA0-s$AO0XHLU6j;&?PhP3P&t8U$G$g8}2-Kn<R@IVkzaDaz(6K3nn4voc
z((5S&x56xdtCtH1E6@M+o)~Fmk?dF{8J)ztiGl23ZH;9p4+l(dN3aRwkJ<g69fdOo
zu3Qad0Kr4tVro$RkJFIR7i~-fodE1ou0Cwc=Er^PsF(>G^S4?{_9cSzhNYVa9=aHo
zSji&BNc(5evJC_OaKen5#$a!*%$9+dazFrPbN=d)YmBtOpO(gW+Tkt+1xk=2PBpFB
z+>mtP25%<?LfV&bY<!`Ae!9pDYOL=t6>cjdnV7#}&ov=VMtj(vBljN{iQ;ylB);ZR
z{VkYie4~Smjuw#23L^g>++9luLwU9-FPHt=tbOI5R&b)j(ww&1MU6KU0X=``%LWtq
zSH(ubip&W+Ao1MJ!6Evsr#J~13tQm|8y(B=^!K>*k42O7|AAAOkU#*J{o$EjJs|Zs
z0OW($zhniA-mx}^%Z*|#E<QvzXzlMJ1zZ*AsAHQ`&fEX1Ar;)_pW_CBK*Z$;LHR7a
zVbTIto$uU_!A3d7FMmA)JllTIuNQ{}cOD_lzz(quHybljAA@+wtlV4x3#JISX_S6{
zW@dhZGKVPCf%L;xU#4n~e<n~K4t}7dMc^Qg3xr4ba}$n=lM(O-X}8OK46seUM+?<3
zy+8Bch0)Cb%7D_2h}+UskYn_(a{+v!FqdP}+b+Qk5dg~m?7|ZvxFLX|GIsb%4Yqc;
z@_S0Gg^>{jMc0D;c0^eauA>|8?Mn4tFC<ant~0~AkKVY5<BEXP;NYV~Dtie)OaPMM
zx!C>89hh#l;($8kdPj~%+r838p>sol1*+=mt*7D@okqEhr2d)>QX;xS9V+w%P_+l7
znnEqsgz}G9uQOwkW@bPLZaOWN1{`@RUwcc%AnTP1(XqvY!ha=w#<F@iF_B@_9gNg?
zdpfV-XqqRWo}j-fI)t<uTC|rDJMk!b1jh-1Q_>|}5+zT+C2f9|y=DE5h=_1U4iAQ*
zxOgATDi%f%^pU%ZLJPE!I4-Z{#j`-bl@bE{&9lJz`x2(|D#3wRE!F#Pr!x=0PVavt
z>UR{VOa~uj_Lp>TroIO@-mKx0z&<n?!VSFA(xik-e|cV}oS(K0KpP<(T?!8<m}qAE
zM;XV-`D4x64>$)Uk)l{a<W;UFVW0_81xjDdpLYbCxzIG|s&-S@3d&e{10S$9oPM&9
z$!4~{yG%vB58djQ(=s>161YK#IA2*o%f3Bqb2IKE54JrkG+}=x_M`OT_xF#E0WYQF
z&+!f?o83#jv3u6z5-MEJb~yRUfeW3mgidtwQuf<Gh8jSI08)z$L`c9RblBgVi1(K_
zA}FGSyoQ2i-pBHPn>`1cJ%PB_Xl1~8y541x{D;#HWz&KmGUrh`7LM3KKN4chgG9(R
zloUZK27&TAWh<Nq><jTB^5!TAn#T}O?%d*60-n^`CMv46wZ@apZ~nfYJNaQ#z22{V
zq!~}%BL>?=gsvIyQO~>6*?BwfqfM}*XA4KaaIiyNYU0I*lE#L6_81zrYIElqzg)gG
zHMQD4Qq$X@&>q^udTTD|U1-*q%(=Z&!)L?VWrx2k$nR&X+C|~~B#@rr+LhPpQA!gL
zM53?*RzCQw^3M3bl?UVXWIp1J%sfP@3tkHd;=)+S^`i?s5r23Rb`zOtn|U{8Sz*$~
z&9Vr=4|c%;FDS%_mJYxT;td?+YFP&b{~ZeLMP3ra>x8sEU^HgqB?Y#rTvah(oW4dy
z+`aFxlnF|d)bo}(v}`Z3d{lD!U_Q7LW6DePm8oB~tp8eI>izV6!%e%HMn0QnyTLVl
zfD|ZfcA*#kBu0%pG?U1W2otG@g|w3&>P`gBPYOa^^Lm?TosOZAQJ!0Wxq()O>c6q@
zPngM(P60M<;4WT)#icAt-Nw@{8bIIQKFgbONUKG6E`Ev;;}sOt0#<c-3BG=y{zKMN
zw3y-9KR<^uVALZjVp>2<5-3g~EGdrC0RDoWP0m*nwu6UV&Nl&^wJrtveBgnN6O+am
zVE;E6Mz8DeYtEIM@>Br0g12Ap?CeHPzE=_H`PIF7ro|TJKm=7Y)wra?I0ml{__(+W
zNZ}6c=u?)Dypkk|lmo81oi9zg?$G3zfeUL4JEG^#+IlJC5e;o~KwW4NxVW5TS0dEi
zv><ZTVMpclmX?;1l5LkXIP>9~_N~W0t&hXWIPx~~jv3r%upK_AK2LUp0c||2zoO8*
z+Sm@U3+HMv2Li$S=ll$hL{wj#`Si@%VsG?SLMEQJxtZ>U8@4_Gns7chc&&1h(dz=N
zQTpL_M^#cVe6N0;-`cB^^xm^c1Ty^BkXA!1ldBQWfdbhRy;q*_FtWgx&z|IG#w^8U
zI>xB8W*79?@X)t`NT5~en5MH~T(K%4Ka_;D5||ziU(=czYH$QDOT$`UmvqJ8DyzWJ
z6w&*1YU-RS@a66J<5k#7>2`85Ch3QXYmlzOKM(e~CbLp6(8Cwjd-zGeaQ94$6LWKU
z%^QL$CEfb;)#Eut0%t#;4Q(i-!kvX*XFlN}jwQr^1e3V?GE_1MBt<k>mud12YB8AZ
zjNutW7MMKA5qEjpzU$x$YC$CyMG~kITQd?TZ)Hj!9Rnmgw9PWPTml=>X%ej9*qT!^
zUNKIeN2>$q6@df~9oi4}Iwd2|aQY&G>E!u;AD9KEXrPW776C_5^6~ED#&O!53AC_j
zSM>KE$239}mZarNpP-B@VTPlv<<B9}1{{H--M=0d{Ma4<rHNddh9K|pw_J!WvAfu^
z*Lvsehlo0q5X)?NNgTUbn=K*;I`=PW?W1l-E3-;6<icBV!9nbNt;&V$fhR4ZTtx8&
za^(@}qWGDf12ymxLA`;jE0K^5%i^u2d5O_ab26dBxTBXvTMgsJ-ph!8!b7%znU}cB
zZAQtl>SK10P%|vhV@J6$-yehK2uup3NnR)T1VnIpyI?h+d+vZS0P%mY9Zo~oyi<)-
z0mCZ1e`vgg8gj-_XMN}NWzhbwU`)o1p-bo@$^|;>#2Yt+fP~B^{8=G2P7iI(!(De2
zu)$^+VX$qEvjmb68th%B@kZ)iS{O{%$N-_B;4DzN5lKd+Yf2#hE2xBweoi{>Z>uJ7
z2<HL6JrZe}P09Fnz%-fN4LXmK=FnJ}f2FY*#-jKX1kwjgFc9B%hLn4*-Q)A(5aktF
zRNk!<R-0h*ss#Zkc*Ho0iCNSoury|ciTt=MwEBs$Ja1YVQ`o&z8IkAF8U5ISV$Hx^
zuaHs!z%&)moo%ennIGS@>bv3xdEu~hi`6J8<!sPRev8E13lW|me2e)-P&15!Z3JN0
z?Srrn5p^^ufd{ngDNy*r0$@mb{3v0}(;(@>S_Zo<k}(o+P7C;3HlLOp?V>u&#q-9E
zXl}9qoMfEq*3BrZSZ!nn2oscf)zZ?scBZtUXIaD``=Y$?(IO_Vd$7PqfGEX`YLqI7
z7;n&IOoSN7%3^tcktB|e1@v^9|27q6PD6lqEV8bv8c%j$m-kObC;BE+_Jg&cc6`KM
z_@(Lx?*zcv11-vhX>sp#2*-!)hjpJr^)-lxiMur<H(Wxl(BS^tUGcxZ^|s?}e1FEo
zMIr>xlF*SWOw-ZuQ>A~wv$Ps8Cm9Pjs2m|Xy%o&6CJd9NT9}0L*ko^_XVZ#lVq)TF
z2=FCyPQoh~Vqj?f0*eKdaH25zk=c#8ltN!WY;Tf?l>ZSDfHdrv2mf@l6PrPov@#WC
z(WO(eBVfMYIS=p|-tZtgf|z~=Fs7fQqJShW3yEmWU!RH##F38riY`QZW-z_7jir#1
zfd}R6Kk?}*p^a+8*g4VR0qu4gIcDR@Hwtq)slCykc*tCEHXN=xcQFJlV>zrPp{NfW
z(@u~BFJ9O=ZukT5k(&xgsZ;6dXIq6aH3U-kpHsuOfPna3zDM5*1-+clH(k})i8%vE
zc6@fw?c<CfJ%Dpydw&^_?||>e9eQw<&jhIrGY-lnkJ2OE{`{Ok0l${6ZVD6(gR(u0
zasRO?aB~nDVFni?n)%U94`TZNb<=}Ah0fztpyF(S3UMyPZk?sJ%Ca-f%+{DajYlF~
z_@D?eHbxK#utgE)Szxdmu-Lbmec)zh3&XgzS-IGqW@ERux+3mHCQQ7$mx><2f#c-v
zwFxf%_Dzk3ak?JPy<PgyPR#3jiDZ;tAOsR5FVK<?u=hZiaUVX|tbN@VZ3W9_)UFOg
zIDml}n}Wgz0=8mT5QOP?`;fkIV~lQ?0Q=1d^Gkz&e1bpagJ+Ba(Mu~ln1~_*au~75
zs6ZPK>%6v&At?B7MXBahZG#Q?Wu(f-i*>EBt#^Y>uEK|d2&lrJxQ;x4qOqz8Ow(9a
z3nd-BZjq|o-y;)0J8-5{3o2%68^l4G{V;<ShGZ0T??410d&)S{?qASDs|u&fk5lkh
zqawP2C2f&nmq&@$A1r3oUU$%Sh#o8m{4-II2cg)H(jA8hq*5f+1GC+TgMAvjqTS(-
zlq)qj_rgDy6Q7I1{y%`_MbT*?&lo0!3Z+1?dHunu7ut}J;5~fUL=>K%SunW>RcsOy
ztVQEM00x^P!y7h-R$b1xMiY`@r&Z|^9YFNqxu!!W&ptRr*{q$yrkdW~<>(EBfhu?5
z4I8&40MM(Z$-bM4x(QY=v&2MLLHiXB_IJj)&gcxD8(Dg4jymoLLKVoYLq;4s`hjzF
zY%*SijoUad@G#^jS|Er*nC5fX)kEtb0$X!tZx80-1|UD_-23Y{xESGGyhDf1^Fi86
zCLb1hYPJj^`llmFkWa8ve;fjRd#Lu>0#(n?$wqoJ>Z64;kSzfV@KF;!3qP6zo2CrF
zadl2^7>jA<0hlreBREN^k9|;2VDyS1LD>{lGZOb7=U|Qq9KUmU(9jp9W<Mg6pb<lP
zq@tc1<U?3QOOY`u17=cfkOXY3c=!<A6Eb2sCG!!HzXi^q19)!U@Jh`~sL_qyuXr<S
zxU&eJ7r?`Y(e;oIBmIf6McR~?SF|5z09J_ku=e&NR7AjBH(w+^%SS(8iI-f=R!KuK
z`7Nc;T!^ha1m#9msfd%X)h#7#OPB*uZ4Nt?VHsb}lK_wV9iltA-v}RqJh_F-RAAj<
zP9N&dwHX~(<jYaNnkZaaZ-|_Cpy#l;%k8H^MMOI0!1weEe31%ggKof4L{Hs}6jH}b
z3*!1X9s=i>4B<YyOa{59QJW-4#PZ@ueCJ&gVOE8JGC;Vui99QSA~j<XY>XyNKjw}6
z&j%!4gehwYD5vv~En$+4lv14ylT5($?Jewb#t`Q5!~0)xuQFwgJZdNSOO4|Jwn8Wb
z89$O@FAsK(>OY>}CpZUw(N_*&YFW|jsXik737MaoeI+}`4z)}icI{3;w{%fK*4KQj
zRu=`=OxkI!L@dc1kpaI}T~sL`jEh5#MU>NRM%qO^*vP3H@CsE##0k8ip&=r*clz^y
zj+q~j^a`I+?ixVU`!Wv3nJj#OQEx}QM|Wcbji55_Ea*G)Id5^WX2&83TP{<h8Hp3&
zHr0fEcX)iB#xPHNXZoR2k?D!A9N*ophP`r!t?1arr-*;DD{W=)^6Z$U?XgR<EZP<O
zJ+=j}z4A9({>n1@;an42<=0l>=r(r~QTeYvN(9n5*p`sq&&Ionn(%#9p_i*rtAWRl
z4)q(HLJ=Fd<cLDQyVTa!|Hq&pD$wk|6=FCkc8yMGdp~zX1Ry?|W%BsSt>Yl!*ph1N
zzU9msNQk+ph>wTaQozZPTRons?(bK{O=Q163^DxksiMmS%Det;kIq!#mFn_8sU`Bf
zpd&6k2%0f|^u>`(Rx_NRwV!?0N^Q`R-+FUp^k&{LTZ-vi{-d_EYku*66h<~(-pYPG
zUfDc#W<+QB#C`oE5GP-okW*46J%GiqVb5z6Z0_rQ3l6=%E0Z9!2L$NDd2HI7A_VoL
z1w6D(@t`y`de#Rj&V0zK)y3LA4l2z9PGlxf(<@Z%taT+TVfT_7?;kb0R8y$R^!w!N
z>9YANqt9=J%_l7{c1bbs?qJ&cz~ZY^8mEMlh(wF6)z<^RJ$}9QHP;Q9cMr_|-A!_S
z|0p)~$)!_kTPn@)_G51>x%Nu`pzU@Xk2>M7G;h7#mD6eAM`&WHT?1+LMbLx!j5~&R
zKbyZ8xafx%jtH@H0~fb*k-gwFwt8|#Uyjg#BMP7(d;&D=;O<IHPUeMxs^Q`3jw6v<
z*@gDs^i4b`q=~!^=q0hw9!Vp?tIXv_B~3-C<Ft`v$GP^Vj7J5TiGYHB$x^Q^WPQRg
z>zI4`j|$aB*}4Fq{@P#1S*2vO1}bahozw!$p3?@pd~_!Ml-OwTwfh#Qgi$B%aBWBJ
zROGQM^fCCV_;)d|F7fNXo?HHTxv8z!Ntn^+@$XBcKfC6fbvo;NGYt1N4z~r(TfNcg
zo#VpAK}0svmw(5O7G4n{#2Q&fdUKJH;0}4%x<YYMbeD|pc*2%_N|5oL;6??!_VXxb
zVP(Y=4FSg?6wnzBsq%*zO!zw`YhCtVc$q*v%xs=KGR9s?(0M*lJJ$5pl%vo_Nxv~M
z^<k2qvIl*=?eDY7HGl5TZJ0Xs>&XN$tvmg7-q@)Wdy~*kHO@n@N!v&CN@%OWNsEoy
z8=uWXwucFcdMKbT4(&uOk==Jqj$IeC72T$^cpGe;8UV=0KWsLV0B67o4BO|(t-4%w
z#N9XfCo58vuPBL)Mt&396{@Fgn#1mEA$jwMHqbANCL|r#Q=XljYFgNsW$U9gQ85}T
zp+|SxMG4a5(o_2i4_U$6BKssPMmqNN(o_lRlO5&;lS!MH8VkLw&lyRxf0%i(Z!N7u
zT5hujl%(JDIcIfsC(pDipKF76j2tI2K71Z)O=U~Dr}zHXpTts!l|RF(U#^Im-cK0x
zd$BXSeMokweEI2=F6;bvc}_Wo_kO+ClfyQ!ME;E3T^Tm_z0~n(<nZV0m}u=3OHp0t
zp!@YatW3;$oMX4b=UZ89JtW(EYVz&rz<vA4<LHa;J+AD|5~|+rnCi5e$$i**?t>WM
z_mwSqf*;-Jkdgb9?qQp&NjzoJb-&A*x?y2;4`^9|)mgPeHNW%4vw5DDTQpi!YC&|&
zY*sv~G?B}!_WtzDJ=>Q9m6ndfs~fUkO!I8B-P;$(@;~pq*%r2X`E?7_98|$6+kvH2
znA0GPg~C#E?aZ~Kkc!!V_>A0AZ~NAf92we0bn836R~jV54iQRM-%jUnOqWQP-TG>5
z4ea6r2Bu<-Kd&e(k892aI`Me!k<8*XDrj=X$?g-I;yGa*7{+;ZPRvzguQYpLq?Vo9
z+n7YtvX4uimuW(`_qQ234zz9i7M8MUs+4n%B}8avtF6GTkiK5t+S4;%3|AhX^bW6_
zo$Zmor>3~8zEAJ&@Ci}t{L;qNyzUa|A$;8V6Q>!+`so-d>J6I0*Hns}^49+BwWXQj
zm;0W&(PSv(+ZkrxKdz$@zTdt}^QbrSTL-^N0pr@g@5&Hk?M!5hW5@D5RKJEuU(y?*
zPH?JU2-q{-sdnpVZadFro5-sHCUOetLhzjX4Y&S-`~fL}#XN~Ws{_OP&uJ8QMcCc6
zkk(o#*{1nmZZmtOjq>Ra9{I8CYx!&n0Rm+<3&FR8XtZwjPZp9AJd7vuZDu>7_u<}$
z&$xs$Yu=qFmHECX+!(*wCnZ?ME@mYPuE8jS<_uF*6WnBr8A*`81y^Y|a+TP4rXg>s
z3Hi%WgeA-HY%~XcJwYIL%{tYpp#sqc)b{Mzh6w#)HZsKB?8NDaU(3mtVoXuAsB`d$
zXL9)Jv!D(pw%S)(E?n`7-VP6=t;;{ljqv)t@%Fh8Jo@7JCiSxVrJ+4Fqw@u?omT!R
za9hircy#Vi_<MiNYwYr?Ql&;t;dZNECG8wVWyQ6RXb=?oXBvH%)MoR?17AD5uWE9b
zEUfLUf9Gp7rW_t@3Rp{#n{`V2FGSY6Wf|r082dKAdgY><XWGd?Jm81df2RH6_4t*!
zH<jv-2cdO1_*9Ef|HI$sB)Cui4p!`<$(+1-c=5Iht#wJMacitp=e1qHN-e}_B^Vrd
zsO11-jtUP$fQ!7#!DF{(a&r778S`_Zf7H%xHm|>Z!9`xaDK`AW_NWI1m)kO=AD!(y
z^$E;+(z{`|D)pKW=Pfn?C6x-xcF$r>k{=&sX-l#mOKTS%_PrvzE4(Tz<>B;b${4Hm
z>x?1MQ|dj6u1y6?VrBWcOYioKK5S*%m-JjZ*+cQmjSibVA%YgUw+|n2j<E`OoIX1;
zz@+iLeUm6M%bfs=rR4>;Md#CG3oErU{Xy>M?ZwAG00QU`4ro$Q_S~YWb;Ua9=lIDi
z`cvVuzB*ij5CzFtf}T~p-+#XI5_8l<{};81xt}56FQwN13S#luF!*L*tNyqg`$w-I
zvIXz&y|QJXwF-NeG&^@)`SZXT(ZtcIUx_O{l6ohSgS92S3?^y~XIJGv#Iy27^-9?5
zDOHxxS=BH2p9_?9dAR)N+pYXY6ORUCkJ`%k150+rtXw%fCwxDQJAc)TaXOb4<VqT8
zoptf+w0oIRs>pX~bDko}rog1G;Vab{?GgsV`aag#?{~QprKhgB4ER25yScDMTm`S|
z*&Q|DBpJiMJhT5>?$^)@j$ht{ORzJ|eZL{XznJql>iX-cWar3t;yH(XhufR_>{^Sz
zZQK@MC0EsSLivsF*`u85D<vg;?~=ox^!@3~HAwOGGynCAAKu>*0!H2OAlHMFKU^>9
zlGLtRJgOHiD^(oOQhw={oYZh9bj#_3h8o?Hh36__0g4`*P&>Rw#Pr8k`p*?RT^_3i
z?^~_?cP%CCre3nJ4B!12<pi3+E!3&cZ9(<nm#TJ<PVH<6K8G4_AS0m;K6$b1+tQZ~
z$COacb=~FuVa*Qqyq@0T1|d35iM4O)fp5y{RMU#LN|98#-?v+d?=&y+d4`Fc`;f8g
z^ls_etYz1$avdZ+cp~j{Tl*il@Vky5PZ8blO;-VTNbYc-)^FF0`zJ!)TB^r|9UW6x
zy90C^3BG~lwL4^T?$GV}KB%PCtyiE+&(V@Yopi=?+-2VOr(tl-RMx#N6P`t<waZfr
z;_YD%q)!j}mMh3R76t0Hi0yr`hb8#tYUKs%^s7-1E;z{HGzwh0BFv^cAI!P2$}R6K
z3hLcC`!!qd@x9K5I9^7d_75l0zvSc|n_lyb+;Ttt$B1S=fADCf*yo|IM|Fc@So{-$
z@7kn@em~U^s1O*dAicoDth!A8@X&wrkWIk{ONXnm9aL90x{6y)5@Jj0id!AaKfim!
zZ<o*GD#_@x<er+*F8)5X-AVxL+@-7P*O)Z#gPm*slO&G|LrT+oy<iC3WiKKCM+{Lq
z0f1DU4_%B{l|rha-$MT5Xf_I)43S6SUOmDh-%W^BqH26uqpN$Kbpx%xXnYTwh|ibe
zgp%VQ#EvhNoD9$WQL6Xru*Z`Oetj=8iEFMb_~xe~gA10O?y3D50eE}gJrAEaE1xX4
z{q>t5u|2PYIY~|9>CdU9LMxxpZ9a3>8nwxS?^|@#@_58H{t6$20^zKDXKw!vqo#0w
zJw?}x@o~3bo+xWPO?LfMy|J_UV=?jWKs#s0rUqr@I!Ch4(letES*gSeERJ&AV?#@q
zTZMZzjO^`cdZuvU`Ou5idps(VwEjKjG-?+V%tfw2(m>ltS-D>9@Q1IOAQa^V`W^hw
zry<drSXF%Q#>BK})8wL2$w2#Bi>t%i@duuo_o^h+3JwG>6njjxx;4>lZ4Rk(E!!C)
z_u{@q(?{vRUzK97yB+pS+s~kMeY_;^I|64I+!CLhb>Zy<F9;)FG11$gQ1vVmaus9Q
zWDVuc{Wx%mp5;0`_R{A5=)<nm`W7fmDnL@|dBxSeuAF0)y-ngqa)QlAw;4ZFKf}g8
zXgf0_C-J(kVZn+|tf`yJsNzH2f>!@M`L=jvcZqMmYWPYUGJnxtC98g7iu&!c;xBmi
ztYy;oNIxFk+nQH@{yx}HtIR7`o_oCXa$a!irq8u|gQI(XUwLlgu$=bCX`lRC->{h{
z<*S#S&&TWwu&!4Xp1Yq$cD^?LlTU774E#UsrK)pnZ&kF2-@miG*xa0Ha=#{>zxcD7
z+UtswlcAGMO_uHD=F6=tW8ddRMvA9s3kL3Wu3Y71G@5&$ovyT?E;0W0N0#qz)$LIo
z7k!fxH@=h<I3*^REhIY;RAlsxzlOa0Wc~%QDSV~UQcu3(LHeKKkQ(~YPH`9NWYWeS
zdWQV^rb&50vxB9RgR5@ScS&dD8GS|=SS-6He$j@!p^rzom(x!TMez2JmF4%Z%7tpO
zIm#~$L<0OOg3wKa`u5SYi_q_23s8Ph=tWQJ(vuhY`a6LWs`x0@a^9=hO?b6<#Z#-|
zde;%}>Y7=;@}I9?X)sUrSF|gzj9vC?y7922sIWa|BCz}AwO8WN8l3!#zj>P0<P7wb
z6`T5q%ndcA(qAh*EHv0pX7lKBgg;DtmGy3Lhxx6&bF-h?N2AN1-Jj^$=wQn5&0O~8
z>!YLl3T<r0ni6*X>~J2>^mpQ^2pzULoNJl76h9z+5A-fo{ni=gui4uYdpET@SHbpy
zn`0%po4Iwn3#a;=*}{lax`JPkg7UdbZhecEoR*>fz+&qcALRe4_nveHr=8u?Q}r@4
z&6(xl3$Jt@m2Ehrn=!~Q_lzUv1&Ye*iA5Z^4ivbD8W}=fWdwFiwvova`Kbm>X#m=j
zbWQf(;QT7>5Ri|I&E3%8lh_U@4``H%nsLjxyeBGfq2aK1+jp03oVTQEu8c>;-@o2<
zyxlp!z11x=rl-qsa#6bYa`%^A46Wmuj5nX_6iCPqZI36s3Z5#xmf?Bq-j~L3XKg_2
z<au166F=1>Zln;p)*IK7ZDikX^kAaGV3(NYUh3|TEyk=;pG;&IGP@k-{&b|9ws16b
z))*aTo&K5QwAWAOcXyCeZLw7G-PWi(EgIi5OgPo0>)!?`#c#YKS>1K_*|kk?_oYPr
zQpx+K-Kw?!W?H|{StCyVYTUiXAqU-<g$Jie)7#Gcj2HE6>QM7&80{oE?J1KR{khR8
z*{E&oT<e8}6SVi+7LQzzmHWh3afzVgB*d{AONhO^H}x*pe{#_>RbXgOEtLVBhcU_w
z6*Li4B5I?(NBxXCg3!X7ehQy_u{@<Fl2i7}sY4g{MA2pSxb*zYPqdngEMv6^9}zAJ
z8n&w#J~lTHt#>ZJ-MEAN{6*W6gKTliO#`^cV(xbwcvozI-jYph2tO`u-mYT~-Au<1
zmX_94<&H~u`?{=OPlH}7&&u6g26mXtS6S@8IiOu2NpIu(^$`2-?DpvNq|PS_@0)E(
zjj|`YPrv&iyr16I%{iQHG59{=)zkR6WO%4r)uW~+w!R|wjNL6S+v>xNr+SSvEUdi)
zHAdb)SZT;^KTl`lYoWl-+LddtGyR~HyIc+V*PVKPXcJ7lT*bvWa_4!j5N$Imb%bQL
z0KT^q1Waq{G0JCg-o-(MqYBVW8daXdqNuZFZ4}DkcLNg;q}L8IGy(Gj+QW=FRP$AT
ztWr<x@=s=y)uZi)H*}iv+-lyS`+s)&!p_l4Uo2wuUYLwU$Bk7SYs(ZCRt_7$6Uw!J
zemW-qazWYc@V4sSO&?y^Yo)IcyYI@|ypb8%@%ww~LvkBLim9Ecev<u_E`F&$zs*Gj
zewZkd6?f>fm-zGLUW!Xs0`MTJ*W5kAQV}-WtloKh==E6TiZX-3VHc@q^$cNe>%yi<
zPk9_3XSc^bF3ot6{w29@38?G{KkA)tkH5?<Pi0}2R}KzcBhu%ciLY_WjVtMLe&*oZ
z(Ls<CD)gVdDt_uK^9$W0w|*y;+70})`lzEj|L(4?x8v^uoj+ohIahvEs_*QN*;vF8
z^pn-R`gC4q=)6n0<I~;2Cgf73z5J&V+HA}}59arr%dwuwkFDtK&&hZr|0p2@A|F9x
zfu;tb&rOg(%%6Q83B~b_xNaO#1}x<j9w~DuE)d{Yu=+&=;U|7|X-avbzoK9Zhk_Zr
zr6y8TROJ26M8m^QRcyZ@^2k<EO10whd3LLg%SGE*&F0d6Wy-B6y1WfY?9aEk*(&ZG
z;4V9wtT5IP_j$a3Nv&r3iA3JH_{(wHU+!pRUHeV1RGDRc{dmv)J=|*1I}W}d7ud@F
z`pz-aZAH?;#cPM%W`Qwzz9i72tzh4;!|k#if(@qOa}|Nlb<&T;g)P=Lcov5H4M^Tp
z(!I55u4|#4&F4|rhFzMEvcxoJ?44gm`nmYM<$U6P?w2$*bMeT%7`c{rCvJLnc6`Ym
zw04hLo_{YY@zs%dSm<S=dCuPYU-Pat4Xzdhje^%fqLq29B7lqIDC+gfp6U87PT%bI
z8zO(BqPlTg2J!l;>ZD%>|M>X%NPOsryTNiP+`ctQIou*6J%x3(H-F^-tLz`Yu99y(
zu?UaxPG{dEm$|9@ELN%1$&L08^=RIlKAqTgyS>|aZ-!JiRkTu!W<0$$d}K@bn8`%f
z6yHS0FKU<h=I@Yt7p*AP(~>&b`JSIMZ{Mq%UD~b+zE{ni*N)$!&dmvvOfUjB+u`H#
zym#Er$qqG*PzKi3=)kb{<r?>jEs_w39S)sMCBz=zbHkOJQIWdNvv*410J{C^L7AG5
z-3WpBRr>#c9{{{HD{zeU+eXzMY2%O$N7bO%Qw6?@{kIHK_fxv&(}(JO!)-Qhv2f77
z-`?@|a^ttt3=IaQ(rmukq6~K95{8qG1F;LaVL1<};*@OjJ-%m5nv6}YY}+QCv(NY9
zRlR#Azw?iE7C#zBfzROC1*#6`(D7{PiM1uqrsdToWp9Hk+_SGwT&M_{zjE{W%&O~$
zhjHmN^5Q?_od&nN^gBh;iLN{lDGsZ>8n-GK?k?S5?Q*#$JzQ(TC-(KZGvU{cUf$aF
zEbmR?aivM&f~+9#=e)X{>Knvs#|tyJv+TOo-ee}Ww6IT-dIQ5l2NNmwt2ru>;j8^;
z=6ebHce>^>Cf>M7gz{fi%HY4}I8irzgOGUL=>3TcmU7&h8e+vOPKC)rfu^<A6?>yP
zRDV<l->(1N)61{Ou$X2bWO`e5t|z3)x?;2tuIUrwt+UpzUJ87GQlZaA^|iv%(5_<_
zgq82JdZ^Q;Z#loZFe`kkZDkvS^zZQ(azUD1<_jaG&&`#C&w6C!WO3zCA7%J_`mF(r
zdy~ZGoAx&C!HvJp7W7E$omF0dtM;sIhJihPJ1SW|W+2SdE*gWga(r)=D-UQc&Clmg
z;OY20sE+hV?z!Xu5Lj=>SEwuVz>Q9g7l+bHQhM}$rLHe7;|!Df>dmJwRVsgdcdqd?
z<C8Ff+OU##<C=o6DV?QLJSfloi>G3IJbruL`)H-Tqo6gZe5L+XCumk(z3%Ar$7Sd6
z`tr8cPN_S{?olat?fQ1n{P#?OJ8|XkqS5yuZ}Bi4MM0~@Zqk?jz>b^+g^4p)tEw9f
z1%H!2WWCU_H~qTFux=<og8Ni^$;3p}yDtgAL9p$5sqVsWNbM(`&G|-ie$#}hg{Nd;
zUVB#h+_J`4Wr>(RXQ@K}M<qe?^1|)_H;3!Q-5*Rh2(rt!Ur)JJ+WUy8c0uJs?VzV_
zub*|Tfeex1yv?hdkDln<%74rf+;L4uqTsNu^rf^C`mN0lQn+VB$`ws}3bpSlnqCiI
z47ID=IGr^aaBHTi*1k6fcc}Kxo{2*CmE-X@$!ZsP7`_eLtG5+enzs4)C?|3>j9pPO
zdTt}hXm%%rZsPs7lRozcZoLl|J~1IYCARb?EaS(Nvv8Iak6UNx%e|hEQ<}k;**-xa
z#vj+`02N1Sh^eSI+aUL<pQW!pumUg@@TlkL<4yaL0luGZr!#NlfB-g}I`ESx&j*J4
zuwTjM*%0=7^tkMo%q|@}6%GC7GaUx|_e{QZ4jC`Jc0Y?{d+Dr0O8mynJ(hYwM(PFb
z^7l?ZzAvS_P_uEbX=p=BtaeSd+suItMic&<gK~1rzz;sD9yC6l_)2{k?;h5DjG;Po
z?P)3SS?%+NAgP#1L+$nHf*h&X=?$+nHi&=gU2_WYz46Fv7sF67)D~FzOlGsW4JB1=
zUmu;bf%vEY;l#NSlpB^{S7Fu*?CZGz_Cuks(3p<chwhqUu_6%DG5AOEb03Y++8l_`
z#bJOtxi@X&NCsFvqO>?Vl*#}_Qeq*<`;|R>bd=JbCJ1`Wo|tTIAaBRP)7#Kr5-|+M
zY_a%R5%lh=>XHF=m<*_;HgM7NpumJG>?~GZD&3v?xzM-%Q_GEr&_F&#E?3b;+zcu>
zDxdL&f+G*iEY55iYWIw^=41dpr9fw58*~n%XMH<!G`5{_P0Zw)YLBFZ5m0WrnT*Hk
z<5mc$K!YC8{{IcN%jPcg99F`5!*(Ae07%bK{c$q94JG7~gbMMDfK*J2TI|DW_M|+R
zhSh-Aj=lebZjhZ}&0!v)Um>Y~lJ34ugGFLe(kS4H#oI5k%6jL1hAz1Kl6?re6_pNS
zrFA48gmEJxE`V<j1XcojaBooE{gwY6W7+^~*vekZG$1M@H2*`50p2wafrI?M8rJ|-
zBadDxY#JN_Iyj;IS15-75p3)+)W%6>Lx<o(uYEq^7CMBY$|9iuxhPTwTxbYnS^ulY
z(y$@OFA%{^ygoiTS@3i#+O=0OpXSW8EvSfzXG3`jA_Cugfx8U-(%bTmlJ3Fk>j%oa
z>8bJDyUgGvc2v6iS#$7c<d$oI#8A)Vr$b6}2pyY*#dZwU<5dQ~zDatk8iE_+?KAsD
zlQ&bBf`d5|L~o=Fr)i1dgP>a<x!zX6-z$-D@M$O0%Cq~UnS2rg#Vi?gKv)e<=s1Le
z73%$t2!^#)7Zd|T)dm5V@_KSHi~$K$Bo|<l*+z+qK+D!ych0i~5NXX5iRkwkec<;G
z?dPY!G&jKpp0&%-=f!72QU7`W_oEYx(|2LMrBw%hfzXv<Pob~(2B&w3$!*RDJiy7k
z$IsDz9fAGgO}_OCld|oA$>3L+c8G}xhJUVp^j$y}=Ygu!V?8;O@ZFt6c%)g=&@d4_
zkBz$ASrvmG0zIkNUon%0iJi^i=Rqe+y$M|>`ls}F;0F=Ljvbijn5F@~vb(fW;1?9b
z%kca<X`H~HmbCnPt_?wj0H{iTCR_?_IUCf$pC8gyfjl_czQ1-YLK^8+4to&|&I1l;
zq-s}AJvOmoOQ@{CGlw&ggV4pyPwO+5?ySz|jv`3>c2vUVnsmV0b`tDhj&vSW<r@D1
zB#I#mdPLX|SOsa1*+G+DXZ6`r@cdQ$mOYZ1@B$7~&#D{3jLR5;eDY*mI)X`SGbA5%
zVu@ND)Ke%(waGB#!4D!uWhho<1@A@(Lkpqz2D3uv&5QK!oX^e8)hsg1%TAQKf@JK!
zz%ck5RL7I3e9Kh<Q3v3(4`In1)Q`~No@q0H*SBC|(7%F+IZ@l4?_{3=YDd_g6<}}X
zQicIYPQ%LLJUBLEZH)hea?}G(=y{#{vDb`VfRx(3pF&R>MYEMnhI8f5F9~8QboXZ%
z9PG6|XT2_AWEnKnneCg=v%s8;o6xZm`04TW2n}ob9jKqYVqmC84|gDlL>bo70BGFV
ze&<_SaG;KpeGd|q!&l3?PQs27NIQ|i>CoAMu|smU!r!vDXnWHJK*=h00X@h}MyMY>
zx1Fu<Jtq2bA3N|YcmRTSxcy=cyt&DvmxM{wfANs;d1Sts4zzHSt>L$a+$~)rBa)Cg
zs__#5!UnJv?(e0FY?~Sp6R%;Vt4X??-(8@skeu}GLk+TCH(`w4!>lwE;NE#KYXY2!
zc-$(|#M1fvDq0yen4poxd2djZW*j{*6v9)wEeP|2pgCZ-HP($q2^0Lvjla)I7=R^Y
zR?|s5f(Qldmpk5cZ%40)BTguNAi1WsBy12v$Ugu(Xd=)MAD<e8LG1o=lOPlS0h6M?
z{SP{`2BI+I`QFF}zg-aI`P!e)fc<8ozi;;72-s>zI7!rPiYOldAtJmU`v1RMqZw+Y
zOn`c+Hqc*whIj9i0H)KzAU()vx(Ad00BzWif*VP#NP1~6BCS*vkOE{Us&R-swuRfF
z0Jbr}H3={Us%m8Do<Eplhq`aHw6zm~`@2oBAai@lLoM{0p;s~5e<^kvjI1Dv0Lp|9
zn2ZL(Ua<ESABqm+u7K@H8sEhkj!8fCk0A_9%T4)5I+-#C9AS$e%jTU=Ap!dzWETEL
z-VM7RM|=vcH!pQyP$!`fFuM<6Eao#OJ2eo*-fg(i8aV}!mM%vu8wjx~yEIvmv#yv~
z2&i%e_|m{c!i5MsPsbelnhB=3<pnI97tqH#+}$mJaeuPitUllY^tlkATFVEAkOVlj
z=#^)J_b+0%<up<`G1RC6hKTVc?4ORQcnI9G=Cpka%+S)C`D|ARGA*j8e*05b3+lJo
zyk)@)9;i<;mR;Voq<&T#;S8baNLA1;5g9ZEeo(_x4BCE}aXbR@%DkO<FV}woXaDg!
z4&s$oei~`%>0Jjfr50dx9Y_(>-kn<eFI~fBAV8cFvZNIo3u>{|0s?l+Cg>Kth@?#C
zpcJ-YplO?Vz-AF1;<s`r2%y#k4#l7}XA%Mt37sfmEB>s7iY?P6u-vICk;9bbBD{5v
z)O|4aI&i2Ctxqn4H8};^dT1j*kiyeilc5Bfj4+TI`$Bo+73b4*(hjO@-XXvkgmXg#
zAd3iab<6TrEdrOWqLRXJRl(SQv;xnKR?oCg9vr`ic+?k&*ct#e*8)TW@OJpVtEQc~
z2*SH~6h=}WNVmXxuX7<0nKX3a93;)HU^*}O?BOzvB7*GhzMaSmix(|B^<t|_cj2`I
zW@QA#*?@YYpx;aSC%%tj#&ltyG_bD9h<!+6AG|A)z%e146cv9HtbYbbVL9Ck4_I*x
z2_VD#0Mtzdtf2OAV`v@(llI|U(F%Ts=z0r5=jqI#=ZD?Gv9N_(@vu8aq{b<)#p#a+
z7aRasN#I;UBYM7oj@txXjNbyao^}@+7*@HNBV00|cNH><9#F_?z#=^*A}!@4YrqP_
zfEg^7pb8^!e>&w1l8E4aZcoq;I~>Q5GMSkNpaVYzyp9%NEartyk^W<`4VZ*6Y9>&z
zOw*m;ZwU<wq~A}lQV)UDh09D;{7`SamHGv`rrf&p55f)Mg)sPXzJ<QudUT_A2x8AX
zTY5IUvrNgA!UU|5iHsYP*40&GiZY9sQ)|A+i>3)^&pB@j+VcpqpWHh#z<r#jk#g(f
zgogqW0bD|G?Jnd(?8VSg?WC+SkAYbUoR4u<r2W;@FFB}L=)4-e3{DvsgCJg_`r>FS
zDUqT|Itz<t>*d`+OYKQFK4*L5@ELx<;3uF?1}H~0n5ej7#1Z6U)S~3j{z>CKL^l}c
z<*vQ*=$1dkOs;3c{6OL<@DgM<ZoP*cMfmc}Wz@0@0l|KF6SyKh8Z5DW|M3Jx><}5f
z(3Wj=-4cC1;wg=ctxP>p02&Sf_cm9K;ohKS7OvRS8#fap9^tPaFy}@uZXyI56f5(9
zme)x&_K_=Gv`9Xd50Gf;F)d)sqWTguUEj9{bS<iq>dUAZeUcI!AqWt^#7-sXJ9MK%
z{X+;J)UGfWyI$@bjTSC>X=*WvUFQ*SOIAmm^ZuqUh0o-J<23)S;)1^Orj2a_$QGIy
z)_kELFg==Ts}24hfJfDDNPCJRA6X%AHlQruc%AE)Nt7(R88*b2`VGq+?FXn$hjAZ_
zG;@|8x<)i1k;mV#X$pQohX^v?62-*xN;B=rCHgi5)~tH(K`*frEDWrMK(dB&th?6$
zoe@`dkVCF%hR&jkIc^uZKI9?YEwDY4pk~cOX>5$s_elsd4Hg%p`b8Ynz(L>*NZ#P8
zM(CS>(a@DdA7FXD2{c>*=JtZw9x)s;+M>Dr;+KZGfz^9@4Nka!JBE^RA20@U?3P7m
z1ec#%-cu;3A-SwJfN`X&_Bg5o7%Ec?14F@FOhNW55<h}oM&*GfnSD)^!g1{AtMMtI
zBcUf(5M?52rW*zJt_3??V1>bkGpF~WzE#O^1~#MtPVy&5Ju;NrMM#0N9U_EP+C@Oh
z#wx$RX~B~qLPR{h@ZhE}YQ@e{3~oiZQjpyL9ApMMNQ@=ulQ8KwcjM4!6kxC&kC>aC
zxrnGF9w2xyy-BAk=@dr(fRqv%d-);Yyq=uAqXs-9Mbl$<F$IVqjKc|B_IP%kcF{Uw
z?$U5P)mIoo4-{Y$>n!=72~BV(0>obnkf|Xnzj~dt#TlMw0yel`FP@zu{m}NqGHr8O
z?P~-f1TSOT{36{7&V;B>BO`WM=$Stpgf2)~M>nqsUV#^E+({H<q7E57<rUv_Dfb(L
z@WL_2T`m)J-C_D>MRFdFQ)HUXp!!1JglHhBs0--5Z2S>J6TTkqs^=NswTl^}G|BMB
z_;nOl|0mm_b?3+>ZtWy6%V@aiL^L^9-LK~fZlIdYJ*nOBKZssva>CE&VR8&;a{9as
zFu8;0eptD5=3W8#k3Qn$W7sS8Q~!J6X3BYXfGty!i_1M9u@9EHcqgR@5I)sfpyU}A
zG3DH31QZ=|Ip%sEOfPg3rAnUZ(Q#_e5=jv<NkqOC&<X7}xG_oTM~yfE$GPCzW+;Zw
z7K7vblb?%v#Z|dMI<_m$1C0G27@5pPJU1MA0$d`Vrsdff->D(%TL14tKr!-DpP}gQ
z<aWa3P=i23Dl9Xg#;*vhJ;?K1`CiMb;ymOqnA{J~qpk=#jyT=dd8Rd?-5UN=YfTlq
z_5EibI3!4&nrOYKr&6!yXVI#P!rZ)%VmDW|Ak6K?ja*1L-9cWGN%VKd=`<1;L>{=2
zW`Uwe3lc7lK=6ln_~YLte#MjubybLxPmO5br44}jTw<|AT<sAVZ~}ETzXarQKhRY_
zRF(6Y*5Mc;!V`CTwHrA^I|TFz`8w&`@Q&s2s!sd;*wpaLLSP?lEwh4n7#op&eBFl#
z20$^vJ|e}WbwHg$!mLUfqzgFcZpAuX;ZEX%K7u5G;vD<<<dUM9(Orsp1fi&oh)203
zG9OqDOi%!=%?N66$uxH>Z8bukE+L*e&`}nf0F|tOPjtVbL)f9O>?E`mCjHNNaB@DO
zQGn~0_M#B%nXzCM()Le0uxCndJ_Ar?0xVkY0+}6pWBMx3q(1ccV3B|PmT0e)&s7K&
zU#hWKf369FCN*>b_)mf)nh=G`7JBM)^l;CeTQUd&S%wS2aSFZKN@9R}Zt$&kG855l
z^8w2$9oOCx5r|bP5St``lDI%PUOBl=Vkro)Y*nBHSjz=D0PI@a{$?1Bz}X0PZRDFS
zcr6KVr$q7q{odxu;U?RlUtjq)Y}%SkFIN9H_IyXjL6qzV;pAjwhm*`1(G;RxRDqM+
zhd5v0B=K;P80LEjg+@24a`|a@W7z{_Gk|-e^R~(cEV0y<y6rFTP*gTd$)K565U&6t
z_GE-X=NDWHelR0zj8Nh@iqcgj4C$cH2Z0$;TPjK&0C8F#_>gZY69Tsk59dcMq{U4(
zfb-*UH@gesS+Wz(Zy+Do`~TOim%xws9*a(tX4TI;-|OA3xbPA#%v{ZD+|z%@ibZ2#
zL8b~C%gq=GNeSFZlV8VjrfI8D@7kG+xbr*-{0E_f*k!Q#0i*{tBDb(|gL83VBSm@e
z_HcuTV5x#_;9E@VpEBGDXLk|K?l__bfki@uj220hBmzadCA5E>QExIX)K@{+cxd+f
zHz77Op$+2ZoMa|6A;eonh09`s38m-fa?@5rgb{o1M$8lL1paCCB0{l6r30v7XN`|D
zY7NQ^_;lspSea-nX32g|GBq0OwCY_r(yZ+;RuNFE(^jKqxHC7R&vSEnlS$M7Ai=t)
zqFKa5E1X^@LY8rzYZ>4syTH`EAv;`-hyhbm4UYl<_fCKzn40|=DcW4*05COip4^+G
z2scPzoB3pA%~5wg%H+jzC|b;ZY2Su{{6V642@&**_}fW*(7{X&`H11}ILYR4Z)dq%
z2t`KZfX)Dso&X!p1&5w}0WUyns6NmUG;fx)VH<&Rmq9vZ2TX~$=x{Zn4NP-2g}^Vs
z2B!2SBW*J`SrbjkZP(^YFeTJm1hh)fCRX~=`rHRT1lY({i+S>IEVMkV9#X9dLXM@0
zBO=5=$d|Iyi+FKyD1xfyze9jl7)0`hVoPuWDCwgrH#!E&Z9yxe3eE(j%a#U|&Wgfh
z7Q#mg>_72Z*OAE!`Tj2?X=qe9b;$s5JlvHqvBLqV&?|HE^R=Ow^dsdyM9DQU$G4x~
z7C;nqbC;g(jrs%zbPCO=q^0QKe_Aba=@5}ZBDnX;D?gKbzjkXu^BT{8*;Cl~!F)d5
zbMX@=*$(7js|Q;&7>ULmAO~$u1)a3K2*v#*BLc@skpsjk`ThNgH}X>oDBU^e^^}34
zd-@{9qEu7_D)fFX<c5zT?*wBgWkXQ|mWU;gdO@E<KP($_5HCWoq_ta-6?CV}<Aa>U
zW_69H2uqGy&Y{94MCoOP0LBhX4EvxAJUMY0dH7fZHKIiL;yAnyN5%QSUYt%m|M9E?
z?gJ060c~I#kcy3h3Pk=@LMu*KC;gr)+=%lH*168UxojITCw>_sxxZn3*8;9-Wj+IS
zh@t*baKq|!ZYf#6QCe~vuA{=N9%>kZ#T`3lpoia%?&=wL0v+;jU<2D8@2OV@0uU4m
zrb_<ioDL@<rqVr<r#X=eumuwI(10j1pGuX0*>$jS>^T47Q1F7z?MO3(POc{)u)MKZ
zvB)dYfjiOp{|cPlP@B;)=<;w_UYzkBO*x^htsQL@G?6#&x{8Iju=z1)epoED{Q%4l
z6JGFJH^Tf_2?5XwdKL^bGmDe>OdSa*b%Y>&R}P2ZGThjpu=2AD<p+M>yz~df-EaaH
za01rveRr@CtKsg6l9y#g-+2Kd5~9Ol3R8}SylgY=bq6MWA|wZHOuT?rXZI{>37}L3
zl5BCA_1l8%jRqQu*s%}L0Wc7T4xqwu(R$fzr8I+n1xuK%r+pqYX&DH)P`>l`^wVvK
z0T;cu1RTLs|Kx%i22*?+WlT+!0lA1mmJDXawHXFuihd2Z*<CbPtCl?srjsG13zq~R
z2);OV+6#{0Eqf@}D&sVY)Ua++LM(Czcc*<u8-EbFgLKcw;ELM`ro!MPFB9B5{*!%N
zz>I!9)h0*_*yT)7lTl{ykD9+WHR1*A`-0g8Ak(u4q34dQ=0%KN4o4xnm3rF;xD;-b
zh&seY2N&!#G~^wFjv)ucM*!WQF1;l;`)<ms|J^JR>biJ%KdbDKkU9i{oL|9YU|R#{
zs0v!d^M6J1RV0WK(3396TdG-}ZG*<0cgvN%a}+*9;HcOS3a~JcyoZZX7!=?^BA+$7
z^MhfmiZ9P?V5O+G^K!ZG{TCcCAlE=<vIVSl9&i}KOJ?CH^VV)s2=OM`<{=gc<ColZ
z7?_~l>r2dfyBD|vZ`wM}^1}5-3x1LgqNs?#u!ty?j%Y0ak#VT~e24Rsvh_PQLK$tD
zhP=@+d^Q(s*`DZJL_(zsQH=M`16mMh9zn3N+MSl%cKH9M12}>w53x#nWM=vgxQzcD
zEFKMJaLUL8&ka!|6BlP45@7?lvDqB;6CprO07<>r+^h~Vfs$8<R%*Z{o?X62=UC)l
zJH2=u?DQI9QRb|JO)nRI5e91}xPf{7|FHL#K~=rsyRg!o5`vVJg0MllOG-+*_m=MN
zmPYB41_?pBn+?+KmTorP-OX9}JLk;2@67x8{qmnZ<G`L-vet9gbzk>-uBW#rQ4LTb
zoCZ8AYQIwlDdIPk-yO}9o)5rR%K}@DfQ_lZ!736@Y}gZO^M1mn{|4wW*ua|4h#CQX
zF@UUqH8%>A8wGII&o4%Z?5mTlu_oEt{-T1&t+z)`DKj3)cg?4u#(Ecu#?psQNAW1a
z6>X#wacVk$Qxf>gQCI7KU=^<s@ZDJQI0OV=e4tqV!{^jX#DEQKpXBZUY5^FR6blt9
zBp`G9H7lYBjjMe7{)WLN)H%)x^cH{pl?;ym)>ER&G-**?=|w#zW(3Ar_RU`)w}LmA
z^;C@sz~;gg7EwQZI|sM5sg^VS{2xYbUJDe+`GCkHo|C`(83~f1;bMgGlB<6-NejKE
zCREmOUATW=J^PN5Buu(s*`XsyJ(}DKJ(m%nYV8aE`=662D_#J5YzY`d{bJcX-#<0_
zdh~eF4*>vp!9m9W(i5;n6)#$mc6VNz@ar?4$=4%B0K0sY``_gvl>6hrEb}$~kG8N6
z(LPUY-6L98ZW~aAvsS;R0syoZd1b`HJ)O;9Vzo!7W2+yST<S3`O$kRobo@kp*A3+7
zAoRcgC&@n7ME~#NB8h;Gr^Kd>*_Aui$L}@?^j?v*GH2631l_&)@A7DA0T<wM8Kf=H
ze>WxiTLj^F3j|Pro9;RJK(ci3cXuYEWR%xBZ>aqK-_s>wT-CyI+w8ZW%T)s0paH@e
z$$twX`dj_KiyL-gepEa;*(C5gXVsq!u2?jCBbI-C-*b`wE<aiwQ3QHeYJ%rC;s4zf
z-chtDU?^Z`F<<5{K;F|`kD4+gq!*n~L&&y-{$B|J#=XA#r&}OB&)+K60PT_d>|XM}
z1;MjV{O{s2e!71;zA?3;g8Y#sPG$;p2=iLu|5o;Q^naJ%x!0%yydj`(@bLP-n?kyy
z{m){4QH++rZ0&!{{XgdZpXmOd=>GpRnM=HwZnBM^IVlPM7Th4cmbU+BmsRrU7?}>a
zFyz}_6O3h)!9a5trKY?`_9w-C^+xS!5d7&WU(57LeId8{C-l*QKVA1=c74zZGCt)I
z>09)>lxYnp=;I@SyfG=(V?+*rfx-B;E;8_+vxoD((lhq|?EsX6D8#jji8B%$D?C&p
zv*8n%$uKJO`Z$MT{2ZBJdQ6;8%MuGv*MIdlPky;@+()eM{rT$tz)Rfs+92>cYHoA?
zW!z~O;c6Txg?h>OzrvdCjRcVN09?R&x<!K*p$eUNGF_3i`{ea~Z31pTnSc5|he3XQ
zN%x&h*%rq2Jw1f@4Z7}z)Ur1DR1SVbDd#TVuTkQI{zzBeEBSpR;yc6YXY;tw0@Gc3
z@z$}u(ck4Fd>CF9k9d1A_;NFYzTgFux!({R59v(PX89eTllH#b_Q*boA{lr_0~BC(
zNoo-Vjc{Rdb2BLvxhssIIu&mJuR27%0)T+@yAB^oKm4-k*!^u#lvcw!(Y>TOORe9b
z@bz^wmBifDqwactmgb+PU+!ABycVl`Lel-Hur?-oW<6FLQ*2q8k&W{#YB@7jHV;k3
zGKOepE8<hfyLzu^=cfFSSJQsK*UryIwH;@*gN-slFM}BDx0YzMFK4Ezd^(9|Z1lZ0
znca)heU6bGZMQM$H4oy&<$_A9{zDb=b*Uj4Pk3w-sPtp@;8Lh^&WCxev_*#s@0$~W
z6oifH06G)fB5L(CVOB|2LC^v>1bUq<Z#lzwqz#)Y=u6qmB6KeY?bVcf9pT6xz*#E}
zezwQ2mlmE5-68}lB^AIk0B(K*t7M8q$hMfd?eVUd?%akp(Q-b~mcp)zm=}QYH4S-|
zljO!#4AOsIy?2}#l&%$HNb|g*h0^eyRo3N4Z9<fFSliag2qWgJT%p$BYQ+N^bzRn$
zqKE?Qc`d%S3lCt8qcm>9X^EFxa@x?OneXe)NVbLMPof@P*L)w11^X0nW7(`e+Nn2O
z&5Z6w78>lY=0#@Ug(+WddPZ`;i@q>tm6Yo;^Q@{b7*4|7gtBTb`a<WY@}uHYyQfU9
zkAkPxYaOcBqf6(a9slCrOw3IrVNBR25&~*^zs|p2iTdR=I!x$XJIx)cY%dECDhK`A
zc=-g{<kiw5x}MtKuBYXd(c5pyaC&W5vJIN~)0`LdeDU_s_Ifo5z6KW=lN#6p+0N!;
zC9WhTVQ`A}h{`8=;tm8VapM3>mkn&-Y7WV5>Q{H0{{ce8<e5ZxVF3M0Zna=n>N74Z
zQ2$^mJXSa4=jrdf>p~Wy2#WZ4nL-7B_exmdySyj(3@-k7sY>s*7=Pcl)Wm=L1i5$}
ze~m(3RibSI&d95?UH`L9=5YDD+ZjoAZNj1q#f7`>h|`B7L%*b<_(Nm9>+!qx;vro1
zc|pr*bpK@MA1%kFUmlv%=G@X-L~hr=`2WFEp7#uav06!?Qy`b9GFm;$KObs0eWCfe
z6RGvectqfR?%3zuP#-?yGoEsW3c8Vg{-<1%yj4R~_|&4HKVjn7qmQ07&%U-)y83e=
z{OCM|AsL^`ElnMku(iLSs4Hu0*}jd6((5;<>n9Bx$sWC5l#}I==N}i9-~Om6J6MQ_
z;IWHoTk>-?>}t57XJI|<QY5je9?Y!JbN-E22*kkF^q{&i!$x!fmE6Jc84XuIGtRi1
z-5i0u;!A%D%nz$X_ua=tz>juoT`T~MwMqoLjmN{eZ4&2pMBd>#W7u`c8}s%0+5yT(
zJPe_Kk%W_Pi-xviFAy7<ZHwt}g(f~=WT?_rR7VMVxHx}H5oc9~abUb)*a9AuIsy+u
z5QhG*|9%?LT%Et}+}<bnc%ANSG|YOK77vL*mMBODHlitE4Lw?;8ahwkMJco;HwN!w
z?Yto>)~s|BItWX4D#hm&DZd`M%PH6-SIx7ptVX;IaWqoc99+2Rl)84gHywd&W_Ua_
zI=c3+s4NXl3w^pj%I>Uk?M^z`jZQO$FFt-K&|vmZm@yJoGP~D=_qRX7pe6qAygZAc
zk9*(ClgeT!()_qk_$j`l{4}f#axzlL+OOWN&+s=)SKaYG9hAoYvn{dbckK(^yS|0I
zb+%O=`O*EAaUH|D7<7iP+?xpR%W6A_@q4%$6I?B@KX%GXSWj@k;P;ZozOZ??kOs=l
zy#MBe^{qv<>}FPmUw_J^QxI#h-tn)Nz5Da>?}t-VJcE-i<fV-Xe#99%uvdgXzv-Ir
zvbc}QL!yhOv?qPfesZNhu*!NiDdf)?Uo2+_qDoSo6v5<2`RdNE#$3l^rL<Xgg>@;k
ze&#0ZjhI^2dv5oh+<v!V*z!=m9$e2kNlLNttA5D1uf50Jg@0x*E_)4cx%_Go`kp5-
ziGu|ql-YW_g?W1Sv-q&-IN4>gZe&%<Z$W4L<Kc=`8Sl<isXu}asIF}81u0EI7D@Bj
z>*CUrus{0u`&CNAH$Gk$seYYA2i5tqefVt_1pgfDFXX=u(xqCjpMsx*cdn65YzD9u
zx0TjJ!}eadL9F0oRqN@N+k2?#h?$P*g>__vKk0o``PjQ{w=2QxlY#3=^Pr*3(8igj
zsB0Km#t#vhu|u;ImJ2sM{Q~&8QcdQtIp}6c9^2MLucqZ?f%l(=$a~-3jnb2FFs}9#
zgAUJvQr48y&5X9lu2Y~o=(dr~mH(#x%H$`VX6D~ap&S_NOqFY(`s0G*H}yq=byhC8
zbl@1K@BP>&ypLv=f2X6Em3<0P#|;YpyW1$fNnF_~0HGM$4A?o3+_=1q%7~k@9CKn7
zhTlZ_c-Z5QK_}l<x#{#qL0{D7Z`YX-T`k9}n;H)s=s)&ZNMg!0df9SM<y_Rw+!t3X
zZ*f)l7PQG^X>uCb_rVJRy)LDZ?ftW>(badWh7dSWO_^#y)&g0}LOUYrXcYY}{Li6?
zUA$`P?mP!fWUi{|lcF#^<S2jRTx8P8^t@rrWcVcU<mQS`!^262+$vA;$nm?2PuY}-
zHp|W7{+vlxibemqr(%|>?ncjv>Gyz$+pTKTa}l9GPTEsH;Jy{3dbfIx5`0ecyXRrx
z)vfbd#rmWLGs^C#E;_!_#p6k<xk2rzos_e91+iva)Vke(Dw*iTAM})sY#nT1Mz&TW
zvXjY=Z1$FTuj@A+?a2MS-!pi=xf&VPcwFf<+Ul&$lW|TWgzGGLvfE^YY<L!6=?aUT
zjNDQ}^q2vi4MIG2vv=I;<%Ku8zd(I>{m#YBtaEKQ0%+Vtu*<f}5sT}~sqP$>TzbFZ
zhWjelCz-YC(82xg5{D`>VYxhu`|APBpjaGOWCdL6+>X~jIf1ovJ@;A;;d)n%(372Q
zkspUsi=YKd??JCAoZ%jM%F!xwyE7CA+0GP9QW@~J7Nv%y)U3y3Y=_GDZzqew=VGV{
zTjRu7)pi=ys7*e}UmH`L0nXn)#J(kz(HbulKnmuh;cJ4&9~w9O=khG8Q>BrIIPrc#
z9$Y-K;Yc$zH(rf<8&+$77@|{QXMT+*YdG?ABv(g|ptEas!qf{+5~TLsv)5aTlP+r)
zAzT3>I4!o|n?9##&MVH9@ZhSGiA!VGOS2(o#EX!m1wG|dMHw&R_&?;WdySeO3ZCC>
zb<_JBeLNjiR)n2X8nJ$MhTNCb5s|W(-Rt%>yrbkAOAW9o+v@%f-C{2xVT+V<ft{gr
z2jxTM|NdPBOIKtVAg`Rw{~+<X$Qh%H(7FHkc`g0k<K5%UQ<3}ue0cS@Ko>}g<!2<%
zzr6$E*jScu^qV~*`|(}W3%%zVbrv7fd^gBwubg~Jc}$9HOBsm{n&}s+xdIT`>&HEQ
zry5#>nLnd<@3*FlQS8p6ea>k=`glK+vdH<k<Y5&g0aM;UaS_>VOl}t;)#YEMPvQA&
zXwSV-yC`!=%Bk9}Os++BciIa%@rGBPTw5KQ)#*5M1`ETFtank>Una5gOeVDX-qNo2
z5_p%ojSoEf9bFy@n8<h9ZuC00&uzK=-nCLo?7gkQc;G#1nlQOcJ<Po66owi+rnu~r
zwroY-F!o%&pNenKlD^Tg+bezyJzeD8xmkO)n|)bYa#-W~g`T%ODMSm#nGts(uX=wM
zdoWzGx}lLWU6O?K_~YZ0i>~0?LDBRwTo-NOORkg!qr92B!la&WV6)JsS9Tvh7P&-b
z!!jDLW|@CZocT-B7mG<lQECvG+sU__hk2%oO!dpBVp_z;2zDxYe%d@PDRB7cS1VaH
zKEw`X85B*Oa%8@WJ3xh`hA9%2!oES9b`-Wpy*<{jobKkYuAo&r`_b}vQapdXo<){=
z5rITFIx!e**lxGZ6(Y-q*YZ;@Q!khAK>v`2y-v_S1vS(s4$I#DTwkB_DR$4+-G=&z
zgC`*M2_;&Sq;r5CPql_LZbZpx-gV3FAYqB}+s(A8Ud%4F8cMcbVz2Hy5*|4-X5VqK
ztt@N#Cuc|3vA2C1izW3`h4?{#@+7)&cFht&k|En7lf^&He=zEVKQu{Et(B1^-k8)_
zC+^bO`QM!YSv3{%!9rM8sebruIS6+7lKIy4Ffp+l%6R)Y$4EAvd_K;0ssoxolWKd|
z*%=)m%A4tG9bs~8a_&Iy+6o?so(zl3YQ54pn^N_}zr5ZZ9FifWK+eWBUN~7$Vz^f>
zCP@<w%}9y*p~Gukbp2DkQcf-vqICZmS=!k-oYnKVuvexBA~^%><B^)OSlS6F9!OT^
zY*O^22X?}&S)Rz$n{K!qJ$f!^Lwx71!e1)qK5sWPVU2q%;l5qJ#b)5mB{9I3c1<ik
zXkmBf*>-~y`h#}2HISeqkT2NxXWc)PM!}YNHh(2p6y_{aKyB+fd~fY2M<D&%<!t8x
zY9RA-ao*MU#ZEfEPgaBNQep*l>I(N!bb1fO-fEp;^*e~;oZ2hi^Qf5FGgdcKgpKgr
zF!!EJn$tm_jXuj})%jy}&^G)k`X*fD{@k>;(SpRu6mi$j^LajrVbhj#P<P3u@HS|x
zRj+D(En}j_+B0`)lS?G!S&lI0nSoVm&`%DM<X8dE?7TL;v2aQsp}p#BhI{vWSh)}2
zDF6E-=`MtQFX;|`S&zerM|gNB<Enk`HGakDiPFlm6SjO>-k^oEHwU`$QENNhYD=Rm
z7fw6S{^Ck4H8yf+cjL3+XvGhYFKvTB&$B(AHTo`kJS02w^iFBD^-zsf6!!~X@%OVv
zKTxPokvZ7hi;GkjCTq6cIy6y<Y`cW`W?|Fg<(`k<yVP7GYksej(u7!7Etn3!sF);c
zFQWw?RQtYg)*R~%3^xK7xO(;6lx18Pu~F=oyNs{9`;$T!N3DeKGYfO=#%0P)xP9-T
zPo2K3gkQ3&bguN!?$RS*_xsi&+TC-qPX+ReCXt`6t&f{GbPfb}Z=1=aojiOxUOAN(
zTI}Eso~8V{c|`~PT7Lf4!YAMVYeeqFIjRc0^)bb|DziVQ`uP@ac1<Q(&`8rRZr7Oy
zOVb&Id+siIsO9UDpHZoithD=9twuRden_GUoj)ly*tPY6ea7ydR{@cQen8(8!TnK(
z7F_diBf*7jDadE%+_Vaj)_^88^(Cm5`<$VA-5h>E%xE9npFI8yr$T$s^Bdx7kYeUQ
zkO|D=$B;_m8q`SLrG`q`ikP`KQFT4}vtl&#+eA^a-=XV~q~DV3@O`~5hpRPLuJajK
z@{p%sd&7X`A1NI)LHFO*1<j{Y?qqd%5XZOQlL{WfDxLwK$$dwmI@8p<=;YfMjZhq(
zfO%GUWEEi#%ft@IUTU|ITH2DWxces;A2f(rYPGTMxH_>o5o%=(X7dEN^fTe*^7uEy
zRpzXnAmqC%D__q@Ncday?G)kHT}B(>KW5l49;<hAAd0#$(ic|*MXDUbJMZO%wXf>X
zUvHC)5BYlJOS)H$=59<`-AHO&j)mt6kclwaDP))hklZjAH%W}zHc4mqZO5T8;u#(O
zYU>+?n9i)T7)2)k68aVBvV2i)VTU~sab@TaZM<#<XolSmyGP~DDR>rCEAuek)j-|r
zGb$k)+9EqIUhXz`+BnbAkt}u&R@um`9zfT0o{<=~hbv;TANki99UL!~u7yY+;(%=e
z%*gust(^zL5gGdxt=yYgJ(qLae8)YKxx$P=1=?cKq5M2`aFmBx5;-zGOAagh&q&|F
zdrGzoun6jrx2H|-Qe$jwH$@R9!@nmj+C(99)fHKsG&X9uI0>46+TB>&QOVU+tqupm
z9WxQfNh+Bfy*x6Fg}*}*q^6@w5=)k>XQQCa-71UAp&~2&ZYCT|9W8TU&dPsi8~g53
z>v1N0G5s$&@{OHvqn|+ILA$`7`u2mO<xFoxNOd$iwl)2^jSYg5XW8Fp%5XQ+{5i%+
zyZ?*L!%|+yJ{T7Lg#=BG0sm|yOlT|v>LK?O{jAkdIACtGN{?cA)|poVbSt`OQO;*Z
z9=~MF92wLOR0{$P7_;YoZurBm0!S_6kQKCN#EQae<nUCVnK9*az{4|nKQ!VN^QD48
z1+HN?=uqn;id}K}=6Tj<-z~=$(Fm0H{JK*_WY#12f;WgMOOoYD&Sz)(ljku}SR#`-
z7B#JcHeJ9G9_+5yXI^H5Z|{eljZ^D<HT_1&W^zSFD+}_g3MOk$>k!#*TBS6LZAuQC
zFWz5nJDh+&&SKx=Ls%2yOU`mDw-N@(-i*cGyllG#5_9^wk2dhb(wpKGY~sjd20M9g
zVSAxGk*gvz{pyW6U+;<S*_to$e_Uf`65N(2v!xUHH>4G(gzcMCB&pNyX@MHFm1nU(
zCe>e~@X~QS!RCgRBSXA=wA0O0J8yM8%R&0yVdcwhT1xEsX5w3eOp;>NyX_p+n(gXo
z&WhYA;)T-Rg+B7rKpMg~^V;>4DR?vHcY0$73A9jPJmY>&0rg|=x;f7G<=)xto^*J*
zSkwP6W{m}z-2b2Od+)=d)>vcE{-)uUIa0>XcHkzMi${w{G1N#?v51(h23VC~i=%Ii
z49Q2MOZ^)t-hRw%xjbY0OTmld6KTyCIdc|AV$yh9c+R&QWxfV>Y469azpDd^B8G1g
zFv$d9G7mc>XRewepQS3chVO0#IAxO3aYhbx8oxKTmyQ*ioG*P4z{?eYtDOpa)JW?b
zeeSPz8jieNN;}woAJ&luh})z>efuY#h1C^5c*-K$k>R;}ZkNP-EO;JOVEfAgf#PLB
z_81ZX6>pGr3Q#e@XMB>Lx!Vci$n?AoKjx<WQGFJi4XIAwz4W0tAo#;!c%k(u#jmCM
zXFE<iZBDA`LCYT-`1^C)AYv}wf^dEKQn(cGyFUXq@Ak<-N$_NX<?2|Z?+9ckQs&?@
zLLjXs)Vgl$al5bN(yYzeH+Irw`6`!H#yMf~ThbyY%wu#@$fR!iQNu-_-HJ{pbTcFk
z(>vo*7m?kQX2QewMpWB$`@<>6Kf>({&`9D=eaz%!xd~ql9|eH%tH(DEe~3qRK;qb4
zbSnaJmb<tHvq@gRkr)TyPSxY&lYDDZfci;rcm}-^3n>X&$29s@^l#1v2ebIzlEUGg
zS*o<Qo7_x5ZG&e_7h8+FcfO*Y=kHsS^V>9EKgtI0Kg!#)b2EPXqP;*g2$ZC)2p#@1
zbM-pvmzp$>4BeyacpwLastU;|dBh5KDVtQ|Zfg~il2DqQp;2Xhd!GF`U0Y{u{6cz1
zIk~C~+=n*8J5Xbt?=Uv>MaG0Fev=baCkR)rdS;#Pr(}om@Qp%`-K|#?^ocWu02h<-
za;D9^#e*RR1(Op{WGgf)SyJZw&aJ67c;b2LzD)(fb1W(B-xCel$j+5^ej(d1czgm&
z&ZKxV0z3EpvPgmk)__)b$RieHZL5`Bj86hU!SC;ky&6yTU_p4&z!^@UeB(Z_2eDfR
z?ltNw!*h3<6m@F{;<fXkh~5Rpm@g;(Uw!AEx&E7ARb>IKxC!B#qi(NM#;<=m$y$Jr
z@|tDv(Sm610@5KSi<votnLK>OqC-D<v{fCn+juc4C+%RBbEg-crUUdD<aH{M%?R4N
zAei%3jqmjY#C=;$o&Os{HL5v^>$js|;a_*#USq9&+iFVu>#9BLDLNFx38(-W6<cC!
z2teig;0zptsS$#vt%{4RR;^Q!i$4h9q8#kiaw?g(H2Q{^#7u86a&qEw2wtO5V<Y5r
zNj8BN-ck_6<)F!g{Yc;>Abt~uFly+(#sky)<H&sgZFX~mWiFgXh|I)0nLaojv<d(`
za#R^&hh#7<vOvY8`3Dk@3VHD`7Uv2cpQewYrW}IF1k4<!d+nw7-)?ZhueIX4_qa^v
z&4>+~ShDg+e0csWJ6$)-ob)gC^9Sj-VLJ^KAW^h&M?-T2mbS2jicL{+5Hpj_;^v0)
zGel7)D@5P6$i)!sR&=%f)xu1HO&0Eq<)U1YiDMz<gCUA;!#Y8i1MNb$YUz7k`Uh97
zL8L*5$zx76X?TG#j)7pib^5N)8l?O}M;%4u15m_>DpCWq(?y>&XSED@NdLPoQBE#X
zf!8Hn7baHfs9eQX6mKDyp{vQNHkA9_waI%mB-!_aIZ&BQ#(hcgX`R{dH8_!g9zK)T
zaycI1SGdelM1?Y%Q_zd8u2&LaV#!6sV<=mKLyu==;~%`VZf~%I;m1D3t)-w|GEX!y
zQ##srDSYE44u7T!+QhLAxH?Wc5ZBQsG{xO#=O|lxt#TsEH<P2UnY(^Bqx!JX1@(N8
zF@XhfQPzVX%u-Y^??pUAZ-qRLHV*wH1!s?xv_qx-y5qSi!LR-4u%2j<B-~l-@s*h+
z8>-tgOLk25jatQ}5r6wX7NbA$(Nx<RoCK)faZc7@(a|#o(b0ENmyq7D$Q*Hsy<tbA
zm>KtsdB|VOJWRnwdXHvuM2FzzK9@##bH_qemUru><@U??JSYtHX?<hU#y^%qEHETO
z-n?8GWR%a$!!u&o?REVX5keKcLQf}(KnI+QoX-D-2s4rdJ<SY_9Z$`cMNek9;~G}`
zjAl}-P^2aod^HzePSce<6_lK(ms-yXE;Spd&*0GmU4=@*2?-7HQ0r!{+}BxsC7Zq%
z=IU22=i|7!xa<8c{;e7&#<%p`-%$dOji7~t8sm2&d3CCI%Fw~ARxg8GnKgLEh#1Fo
z_47zCjwM4zxiK2MRtcWSTB&?HRte2rdbzf7>p-+3(&F=>XCKn7!ss&J(u)u6Y!95$
zKKNvbW1UEo(=66rdn8X>{rX0J*RYd(GZu?x{CW5dZYFj)VTtBiiqz!cFC#*dcV)!g
z>FG-chaQnMU35eQ?G22iG(CG0!Tua2*{dlJyuK=}ka~r1u|V~{#_=hy&?>ed(ys1<
z7WZ^BV*DAmqTyU9Z&7FJUvSU^Eb8L*wLjx5=4|-i$Ms-P@243|zu;t<J1GPqCsMs2
zSxFgMKUFqytW1A&-Q<Sgext2s>XgnnXlaN0!s!QmAHhOD1bjnku)O(*?t@CaK|~N2
z$G4(>nQnDIG#>)LIPGULOq?^9yY+?k$`sDuw|1`vsKB#FlTr|L9|ot7AH(<>%*w%T
z_*0=D!^9q|5>H~p;neH~tD)EeNbpS=tuz#?ad!rZuy&*-VJngGMTN|%AWJGr{i8P>
zo}B7)4DtwPJcnyRF?m6xf<`H+XLaV&Y7|m1cLs5Bk$i@iuf4e$BIy(3yjnM*AqB$X
z;+I-{6Q?zcJc1)bKGGQM!pgxHpMOP2nag$0%X~qS`Q7^DKqheF<2K{i3Xc#xM5Ty^
z8T4@XXv=9?kY{N30ba)&Y5x{*;t_q<UBfXs^b0Mkt$<$;7-GD?N8gXZ-ms`8gKzcs
z?AkK_to)%2OgfT;7r3iL#aCuZ!!m(*lzePR_YN;C=6P}@aZyl=78+4-!gK_gQOsRW
zNy<<7&5SUW4H|o<L#z+@)k+}XI}7*Ifu8!W{JsTAZnZtQ2e-bcj$0B+kVWS!R4Gu&
zel9$=l5*W!6Km#3$u3nuSzRM2WEdfcr-&OqVx%$NFCiV-=@~LntiO1nBhcg0J`o9I
zH)5<$%jXP#PEYvRH_>r_><><Ebqf*_vS88VyFOly8^*&@?Jj4z7uiosg#CmJE8e^{
zOukJ+vC&^TnSKv9LSvU45Gj<@Ic=1&5t!H*0n1NVeIEhb2^4)10DNAb`Dw-uv8!El
zKxh~U+G2lCq2Z(8#^QSsqt<8`SBPVwRFW{k_Q~IyB?vPiy==LNv{ynWOwIQizPdbD
zM0(w|<QpU%qKQJ>%tOq~b0X=^R^k=Vs;trvp)u{eGkFhP#mgW<`NRI@ASFV~(VYhi
z{&V(s4QuFxI93i<4NGQ4h@7DhhzG<(l^v@rA`vD#fcQ6F;^Sp=QZcC4myb>|%TH$q
z+m`ARcRxB!KRUZRDH8kH&!vg8%R-K_P1;n6Km&^NS?01pO7eao_^Klysg(v+;wY4j
zfP%f{+NQz=zHDL1^az;ZC`d(<Dm956gbtV^X=5!a0w*maC`<e<3{+7b6P10zNx%&Q
z%r9o$z-WYmFOFM#_4&&ju7z4V@MSRBazBCr5=X~TuFb?avM)NjSyWb>;!n#)VoXS}
zAQR5sU3-Ca1f#h7AM4vaX6OU5I33Ld)u`AvHTQwj(HHxEr`Mh@FI{nc><l^fBE;O@
zJ<M44P-nruy3q8X*#r-?<%vTQZ9XmRLtp+9H`>?CuI!-iO3v;iM?FO1Z+}d`mdo;#
zW-)~v9fuAjHQJ!Q?)niQfP+1Xw=NVg4bQaLN;^a+uJ^8}*%6G8-x72O8ztfewhcE(
zL%uutJr)ZF4fL(554sa~+dL`~rFLreW{5|S=Z78q1p}F77r90}EE{TXCp3%cM!7ue
zWJ&e*|N6txpAuC<UsKnY`4B4k+4eOGKUKgfCZo4ctZks?LLIEMoKqFEHpubg6~360
z4#$khOkK&Rg4c70Rp&c)aMjT0ZPnzkvvKl+<)6cspXVSn8wI6a#`ckuH?Hyx%=U2o
zOzrJ+`z4}7t6`$zVA2<5kvcGZt1RL24r@(o@L+L^SL<xalw{}EE<uNq7lsUVd!@Ia
zBVqr`$a44!II+Q#nvbT7il(QJ{66HKY5aJS9<t5lTJb<IIko+8ZQP&QTVJ5<2;({7
z78cpz0c$8`g0vmYlEB3U<e>|~kz8MgXr90AYM{Y@DFVsG{bR45>{D{_MM|ITuYkxQ
zfpcW01tH>CAEP80*{iV%bwcacQh1nmZC0s(Z~yoRTV%f55)s3dH`)+Bk!9sfi7jC7
z3rA+cKfyAHBRCCXKVbKJIKD-%eFKr$AKkx}KXvi9&AL)&2`ii|5&LM(Mpc=OPdl5`
z_gN4$8BA(qO6dFJQt#Yf*=DLBQ9R=K(-70{wYS8+DWu7lt8g^m-(g^v1CIvf_42dz
z+F+#eN^fMl(*c-$f#1V^s*3XcA+t7p*V0T6zf`+>HK|{2``U821Y2*q<*7Nya|``>
z4N3dm`Bm!_22IFCM~2XTut2o^{s<UJdh)*n@TnRy!5`#P%sH0m?Pe+BFJ0gER*MD&
z#_nrIBk#E0;5quxPsVh2VnR_htw{=km|#kUv7ZDoqD|KDIql_vpHX0DGbY{<8CD$7
zGMrP>b1x604ek!+(h`VAj4+^;s+;}x0+Y6;9~bIaIV2p^<7g}{nc`Be8Zrkb+sIAz
zbrD0)auM}6-|*4U;nDQ`5)i)hF6=7m$vDqWh3YUgQ7Qt)-27@|<U{EmjXf_2vr-`!
zuc>`|QFYMSE`WCUf*_bQlBVVHLry&YA{o0I0rhhNG|=+`*Y_QA6RGpG2B!psh83ce
z#JqwHx6<dGeF7h}4aKp1@7%n1u!Wj2tejtoxOdsnY%~M<<i>$fWOWYZAR3fR{#HRe
z`&|6o#+YxN^LpJzIFcPDKPDxM2mBs0C#?RC@5Hl_5B1Hp{Lc2kmtJi-ez_|?bZh4l
zyHL!7zM^`mvAjN%ks?O|!k8aihOWt#U~qG4s~jQ`6HBHaNUII)H3br^h?3hFK45(a
z<MTTZu~5(ZcJMi=tN<zVavquTG$+^oXQ&*s-1sM+QP5rkjT{3D<+aXkpXSTq-Zx#z
z79U*Gp3=FV-47yhEC85J=0Y2KUeD_)?$ZVjfUlUlKCJK%EK=M}cG+<ZEw<GgD_d~W
z7>LegT{688k61<zVjCn%MD1x2HlqJ)ui=lzE_;7Lp;8>VH`gLp>fsowb8D;?hX&X+
zMt0K-{=$839<sx3kt6PrLKu{#2>Y#jUUd-x+QetzCjjz8(g$~=4;EWc(|pZo{WBNb
zH+DWwVYW^g6*fPC#FUXqM1~<);<y;6w6cLUv3wvn7}WY!@CP2stc;X9>z<pga6d~8
z?}y0j+LiIV_AF=TXY%hj8EalX_?9W9^>}O987WNiu4@M&mA`f;x&A1iX&X~HOtAR%
zV31MHl?x|E?M4B;z^4qOfNMqf<5$b|c~*#k*;nsrlLp}sJeXD%!B_E!9=t_AaTCrC
z-H#jpYQY7~(zs%Xj}G-Z{nwyjIr%U#mLtSMNUfW516$C%C7;a?D|;b!8k9=~%LPr)
zbd(L9(%}pDq2Ep$i;PsUrmL%jtSXO&%H3rqyE|0hd&HDzP^!&LU8vrMc|`<c9e0wU
z(<E{GR;jRT3G0L+yO-|HoviK3Hm{JwjyA`x@T0T^xV-8eOE>+W`(o+P<g@x>b*hU5
z?tHoaSjpN&dR(b|nQx@76;ePoLHf5a`N*~<Q#lwBZR$$}_Qs0{ylTf2>64kQmnzv7
z*c86)TOX+DUW(7Nym~JwQuOdQ3PCBVONAPMuA*0%LG2B~`Z`=370^tVu+^;};-3U>
zPRl)M#u~IG=pvhXG^<p?#QH?t>HRUqr8p_(Iw$-k%zP(MsRpB&5_IC(h3L-D3w?+*
z-*jDd5yS@8kkpVYdmfkI8JdGAr-r4y@XP9_-CF;z-+gJFXp4;?=J72zqB4jpuC?3S
zVWt*5h{GSt`%5V4Le<fLnw5=b$o6@mY;9KuO<d+J*7r_Aa>|(p1_0jb7q@=@%5p1H
zm<mEi!VMFVxe}`PnbAoZvG_%)rDPrJ>Er=`+M~4FT;Emaz7k6cxC*R1uR5R}3uPJD
zr<t1?GhzE6bDVxeh)%Qd_fll=_mnOZ>!a-Q<~KR0*#5-f1yZHKao={;hx_O{OSYDH
zLA4EZ4&tf0V?|`9qQ>`V_ML+4>{B+^Kjk}jR4T3{_~}S*)U>8htdb}nj=$6>iz_TP
z+fFF4dF4Q_Y;E4c*IayF1q|nD_&E33y%J3}1>C@|+^cmyPq1-^85XZnb{;`p=1+Jd
zAVWbCrM$9I>N$5xCEsJMxuSl*m{g=L%T!-huALhl)dmfS7Wvhg_VQBfP7FTiZkAfu
zx;%WjSg4c>;Vn{Tgeod5n^7f6<u)~S)F_9sqlFxEs7cXANJeazEVq7Vu+e#|vCY@G
z?%!<GRJQ!<Rm5yu?AZ8^6=fmw<i|{bj$_h#LHuT?M;>@1MW<p<iS}CMz)xt@spuZQ
zy(J?pvCew}ZqQpA6!~Kb-0%t7Y>rd;eI`!<;2V)&c!%>c4*-Dnotf4y^LN@y3noS5
zx-a6IqWNu)6YkLA#Nd+3iBSHc;gQy$$1e%Sfb@b~2cFy|;?8Y)^!huS%47@Qy3{$;
z^Q!m0sS_DDgS&bn+JH(#@m9lz!R+xy6K@3zeH{?bM}aQ*w!fhWQ8)}b&H~Wz-;J%V
z41RAlm}Ou^ET0*-e$#->4koemS>n=uKnify%Io$Gu>}5#Q?G^R0iTdjw0@r^Mky>9
zY80r2HQFDvoR!UgD3=Q|d+qD+NB7h~6_aKQ33#+=n;B!xO*yKwE5=VWDBP4ZxEF^q
z6^-sB_$r4-(QR9QK*ZL;Y)LGBw$Y4*yyKb@3oj45BDFb+wLM>|F(Ot^GkEDoBUU#>
z;&m-Z^|ehkx8;^!JNAeZsTkDE>ook>iJ<S2eAMoCrabdxK6i6~V$EF_>Vmx{_WEMa
zXRV^S(nV*e2`Dsa=08!<ox;<Fe>{kx?gNdL-XYtfo4YU#bmVk_ZfbFaB>0%BU{>&{
z3ADf09s%In(n1atZ3>cnejPFZ2JpDnJ$e6+Z*U)~8uB-3<7@Be?ly+vn47_jNQfOW
z+qN?0;#-PNX}m-jG$<bAR#JxK_X+Ojuav|PHY3iq{}y^;;~2rUs43daIE-7lD|fqz
zYkwM_Z$=Ip2i>q--rwwdQi&t@!szc%gnlsoT}EOkcF_aS#Nj7*VQ2T9R87Pi-p9M+
zf<KrZOP90+4C`k%jv<TOj~w)6S=Qj1k5uB1OANI=y~|M4Gdmx7EeS1N`K43&SUkf|
zDAfgMv%;4d^;@!j$?~CU52aJc-a=z14%H?ZI)o?D@aSE5-%>2S8;*~*Hhq+!GBjY5
zg=P-U=9fZ%D!ug(6#LfK3vzYXQrs5qwl={!?A-QjYRL{KmL}@aQ41c&0%4_?Sm|pG
z)am+7R|5Cr-=gml{U!D|yySn;TYKfz&PTiOa0DH_>1^*QgDIWfMt`+T8Q0+{AQA~<
zoe<y|Q^p_S5pUVPJel((Ab$Wm7H<|6rkvcu+O|E#!#wQ{?CV?K2a|HNv)KSGMo^(U
zdsZp_pU$b9XY7z`JT*KBxRk1}$mu-{2S;r3tEquEy9anFTDNFZXg*pIL6IWi+n-5C
zkPDqP*b(TufT5ao3F_zLz@po4{c(KK8bkq>xdaSNJ5*AEb>3@W{Msp~x6jlPFF#KC
zr3@)=PjJuBYo%z?^xM-ei|s3C1x?14;Z_IR*YAD%N>$4S@Iu+*$4Sk5p<DKXHys=&
zhJ*YxL<TWD1V|yc6Q8dDZtDzpASgFS|0j*gLT(53)yEK)bHUK4G0|?Ei~#$BN`>bB
zKcH6PAtp<~Cf&H`Kyu#sb|3mKO{w(tE`k*&C5`Gb>H$*p4%O>=FyT;CN7FOBYVhu)
zUZ!_xj>Ff_JvxU23g@i_0zx?AU%ue^2WwV7E9T2yt<>>wD!KXWf9G(Pcb(uXlJ$w>
z^)ic57{rz&*;R^bj}BfolQS56avlHYl?Z)x6HbzTF?SO*yrOYJfGcq>+}-U%tljA?
zQ`4fusng$n@;a;7>lmv!Ji|<lYN}YC(b<^+8;B4b0@Ur##OsWK3|7Ykw=zIrx4p8O
z`P}ovjO<ZSX{ORUW77coSch>RJKcPE8iGfK6+Em9h#@^~^s2L-$(gG(AITZ5GLb7(
zduxzy78brYU<*hK+%%@~*iFJ%RPw@}PGu@mlf`S%C%Kva0sR3zI@SqV!`nyR4?cc9
z0w$%)x#dv1$O+YP!d+}_6|cH^ZM<$rcX~?0=`Cm<_nD16lT1qd*E*`4NOp(m#IcUO
zcht$RN5>oS#@!LryzZLkJuGD~Ul7mK;^Bs5r#_<s`Nola@JGRS!;I(b8Rv_fIY{nV
z4k9ajVUZ8GUHLzagvSvNHy-eAtd0)8xCMgnw@!Y_o!BmuEZn?5P?DQQET@?hIpz;;
zUAC?*8`ubD){l3$$7j%^=Q1c>?@X-GTAiuygogS~mrvEUyH1%9x@fUlhcfwaImNzH
zUOe?H^v#d`Yv;WaNs^~h>{%9u+r#JGkxNO4<U`NgBfWfG<$Y(}E)e`s9{cHFj*^K^
zeUAG?CzpV5F281ba}u``sg~)GOKlI<Z%kC%ktB`vpeRf(q*^*xB~BfctSUvHFQ|EG
zBJ`4jJ=$Z{vqiVMx!6qt-34vihOUH4r>x(~)-cBbCsU}pDH>1y4G-hpeM`}6J?al>
zd@olsB-Q)2JvAj1?I5ih3@wD^aIpw=Y8rxPJ+v_>cDhR^5o4%#SBx+2!`CRCGHyDd
zar2U~yi`oN_+P7fJ2)cSX_;HiOICuPr)Aae$Bs!y6<6>|(i_zUz06p4_oWLWeMcOr
zT`bk{g=YT6+b+@%hL2+yx%js)XjLU<TYfjJqSB+>;a_qC+>@{Xx&QhWeBzNw45d2q
z2$3+8JvlhU-#Xt4i7gkmdiC#HL|P~Q{1(uA6Brrsa|~CKLnPCP>{SDIo8&C2C>v*;
zC@R5sBoj`@!tRrNuV7f*u=voZBDi?3n&4JV+~yyIuVu|I_MAF={3^K1oi_gvP*ELL
zCp$Pey#M^=@=F9P98nMuW6<0l$qj`y16xLaB9mn?mQ&_IWxhpS^37t}F}U)HdoA12
z?vJu=IQ|6#eFWS!Z(lOcmv*^+=_|x9Nn|1~CGBMGuMhI+P4LK6T&o<G!P>Mx+<8m8
zWY8i0WVRv)6435-chSvwWeUrZ_M*OKHCc-m{~__XKUAseX{d=ap(OLp*?vvVs}yO=
zZbj(f_Uxr*8&ly$usLG1mwOJi^~<0HQGy3>^!}rISu>f*Gs*{lhJ@#fJZPcmEPkwQ
zi?Z_hfv7l5<1=cH_=F;~DFOSCD^*tv_J>c7OI($2x|Zbn(a;0fD3c}0%3h$;^i;D-
zY0+S?XBr#2!2!zvZ+q%i%eW=H*7Xw3Kmxc!1V+4w;Xvt+-QT7JtG$JM0FlHeO+83~
zinleeF&#J!b(Dr><4*?jL=iouX6D7~?kf*k^q4Oa=txGOoR=^6B{}OU2Z62u4;?{c
z8WV#()_mrTJ40~t64_UAdHXwew*fRQ15Oo<_lt?Ip1UT*(RQ6aao$Uc)TC*Eo{@!i
z<9nK@7O%3Yi;v&my#>yzZ*-+b(gikL#;*@1F{Y>jr$9c5;C%&4c*VXU%LH0j>gQQV
z0hJ`e91(E<;#MC0^o{|zAsgV&fZlXX=Qq--#++_G#4j+l^66n#{<<B$gwyjBOS;|@
z8RE4aEgewR>aI{o9wzBhy?dkei@6agsg{b{ddgv78LU}x)hSHE&wbL4pn}F1;C_mh
z`<w@wRqs=5^wCr~JZRxYRLox>-%Omh3K8yIj+@2js=`osbuOb-O?|nKe8&CYFGo*Y
zdy<GZ`pf}Ssi-^rk^eX}59S=`irvzzwfv#e!ySV?EIaD2!EN2120O->?DKBEwr~AW
z!F7U!0EY~IO`P1kpY7E$FGr}?6<vHWP(m_x(gL4W`YMez@7_*S=C4)$+(F7|F;(+G
z%_a@Z(vq$vF8{GbPO3?wyC?J(G;e}$M7nJLcjM@f$5Kjf<lxmyLj(=dNIa618GZWS
zQ3Md=e>^~hC>u8jIN%64L@<SpDw$QEpU>+;Cw-R=ba84tS)#puwr%Aw;Ir_ZL4l0V
z?I3;XP-InS7SlT~((V6f0R$q6{3D1&5Zb3PA2UD0ip@cLGpz530CV`D|7VW+`%dtC
zv=>IHE^fLF7`WYrEpYEvFlZ-Xnv5M!yoY<ARm9uJMuA{HJUDca+gD+L%af>Cn9vX<
zN;w>rRxy{4hIonmm}TvuAUp+kcV7XsEvE4BJgGV1N;$us`Tg<&xXF71ICVf-mj&al
z={n{P;Fq?wit^{!Ap+5wZ)t09YA*PLC9_GeA4OR++m>8#)=L+ErhxH=PxG|H3i078
zt*MQl(Ah;A^+UR=$RhW0NM3Mf{SIi?;w80M%J%>x^f3l94g%2Y96_H#*aPleJm>mD
zeCVc8&P1{NGSS^p;S>A&58i!6`()_6*B=)K3%eT=rw#J}R;{?ZCVIfV)}9!Ng4Up&
z<y}Y53(6-`93`YTR^@iN0puDGfV4mW=Hd~OT7eawc-RqnYEm!!;d9f+*QvVECXCMs
z!jPUb?;!)@jhYhd%&)<<H)<C7LR3rVNA@#}X9!Iqws%rE2Q#}&*&vz?KqB(}!9Hb`
z(>lt2?eUTmJ)P4ZYsXI>-ghlF(XI81OBX6<*9H35q~u2zTaN<*IzjzE1m9tZXzH;L
zr%7S*_u{-76<3_!%O2QWCtfAKw?JbLL6zO%oMLX@P}bj|V4lPvb{7vwHYCQZkQo=G
zXQehcejD~m=2G}N4E#&(@RuJYld1~kJJpV97oUT>6)SpP7ee=i7Y&E3wbN>@Gi|G0
z*XPZEl;!&1tWn;=9e&(xl+_|v!_2#`&s1QMdLi%fi-6~lPB~P=ApzB!6#~s~33#Wc
z7j`CpcT3L@dXJ{tb2?EId&fWntLTgGj{aJ3IaSPGe5TRVj>HZVJ|ey`UH9ClP;Z8i
zrz|)nY}n_X%=gC}OR<c9)~76ib^d=Dz{tGnx7MlUQ)Gmh0`44cUdD4vGBWS^P5?bd
zl<vRW@*&5>=4j}x!@cooj=GMFxo~Cuyz-f_W})y)PTB9#S-F=!=<MOr^6~V!rsmJr
zGx~kjT!}&kWO|@@noTTedB|9+uY{$?tw8W{I4hrRKl7pDc7a5|GH-`Yq}8NEt8Tq|
zaEA(*P;LRrfJK8c)lr<Y9^tRU_WPl*vp3K0+^yD3Nd@tq(O2lT{@{2wiw@TB+Wwz|
zNv!>Ceh;e*vrClj2-xLgig2)o$D^8MVoZ5#mUft|kB_B=U{e~gpFL$xPZ06($?&(J
zD>%?{J75kH13fIl9K7HmCSu0+eNnau?tKNsQ`0i1Uamz2Y<XtA?7G-8R&7Y@-%I#`
zeEhVr2{b6jvWqrrCsjb>SdK9=X)O*qWtqpJdCnXmg%&<oB9An!{Jwn&NXgI8uf%{P
zUvJz`0GwcM7^jpX;@7HHs>9W9g&@@EysZLA=ZeSG4`&Ey7O5A1-{Ptv`I@tg{$ztA
zSs)0OIXd6)$#~(@V|N?IJwu-k`9gFNARb|pI@c8N`AZ9c%$yraS3pVe02%5j)ME2h
zbK=L{%VS_vJ5D0*n`on@&s1Bs=?W5eckw8zOXu_bNQSlh1~<uT)aBtUc6D?+MaPq>
z<HNGq!W2gPGv7PRNaO&~!*Ww|5c#U9M3}#sEDR)UkIviUL5oFl;Z=4cTD8${k?!b|
z{~^p_Vb3VH_Y`}53)Z&7rL^eTmNrLZO0zM+YB#8A*RtgL(ANra)_1_mo@DBsaLb5J
zz*}Yd5*^vjrL*heSYfI#UqT~XCc_KNTc?>-mfseN<OK>>`Yyg_=@L|_Oc<v~p#Drh
z-|i<N`x~g*11Imxt$=lH>Iu;OS3Gh3Kihx-bqRW5;l<6|m6eqz?N4DCvR4J5_4uSX
zo;6crIq?w%n{4eg-67_pgSuVm-?~PNR$O$n6-*y8&0^lxqRV|ip!;n1U*W#+MeR`G
z4@1Snk6F}MMAwp8<f!#p;kw>kqXLU#MJ>M~Kv`pFJ4^w_4lHxRw;#)=0$g+>f?Uk-
zMzJVg2iE(&115XiijBh9u?5@HnT}Z)LZ)=UG8S})3CTrtmcgjF-{2-Tp(EavnqoPg
z9<j4x1mih9?C}k+S7FGZ9Gmr?xTsV#V1T%1gj$ld!;VYBl_}9Q7;z~QU$18++(-zJ
zg}PkT$dur(GX0*2#N7Ndn8m2X4Eru;sgTN7<-4fTues&AeROONlheNq!3)H5JT5Lk
zpBxbj=(_;k9{d&KguHgpOX;i+i~LJa(P17`*EKXu{W~g7EFkOOs#5uXnr`V1CmCF7
z4>UR581{VqX(UJTCvYIEN~5F6BSvgT@lq^ed$+r7>}$z0r((6jIP0hylK$5#$p5aF
zqJDAF!P5C=CL0<S7h=c`6h~``-xywEG(FtIHu_?9Ec&#Bx^zLIJ2mF%l&>zmJq^W&
zGRwIKx1=)8*Kw0r-db?vUACxS_?>3FT6j$RM=QV2Z{EEt=*%A&y%ZV0jn-sc<+oj&
zuHCU}Z(;=%euAv4^?_*V8M~5&<f$!1=a1yy;u|hyRC}FEU0e1P6oKKp=SqU73Waza
zN44KAp(%&X=Jk^t<-@)adD2)NJwAL!*-yN7u<`9&b&)S7yOv$SNEj>l@>J9vDB%Dn
zjJUb+Co0ZqE|UwWMKUj+KXESM2~bc=I4~P(L^21Z&s~)8J%u}x_`io`h#~$OYo)39
zAS1lPqd3*Wft0M=-4&F~Ma3GYy&z4NM1Sp}S=lkV68KPNmESrADJNRtkLi+X>?z-_
zm5h*!BQ#vnM!+Sc3(}G3<ANm+(3t+8S73@GW~%!1qlKwc5|U6Sf6ug9mr}zLFxY8#
z++q%=Y*cqU=NTwSS`V3Q^MaDkostG3{z$k0r|~o?V+cRQ7!RbVkLJl0cl}a8s2qeu
zCu1nUV@MV~#?%>YwE#oq8Yn2FU%tR8;VU23HX@+*Uqa)3niC^kE*{we;j@YV18NUW
ztL}NSb_~IPVe{4^<xQK}X#u1!fg2s+>!$dz)C4eAxv-smD*^Xbf@l;gS@nMhB{_so
znS~6!q=&T6>GdW>X3%4X%8z9CFJ?~9xy(;VX(X2O0dE8(2R2`RY_lA8(O}8P{u}Vs
ztkoZb9W?=U=m(F^?q5CnqQoT4X3qF=i!wJ8fzo^GYtQy?FEH7Cia6upt+%q2Ot(Rt
zuQHcRf`({mx~QGC;7MtN!A_6cC*uc?nZ%zX23Uasop2Usw0OkxuKX+wQtUJhp`Gw<
z|Jq-6;u=rWK0^e>T8|nM;x=>dS3v0-C;R~oyJ0X!{`a22|9<B0oIjzIRhq#WWIDx>
ztZJ*u`ah4Hx}g}!$#_l|<$iezm6rTb+$g)@(6f>$=vN9hOsoY>`R{u?o3eW<<*UH#
zS}PvKYK;0^|1#Tn$TeP*5_n2XlsCr)D}s#&zyY9Qx*J}>2EaJ2y@}Z-ysfCy&%-l1
znGbuOQ)!g>@P$)AD2|{DwWG_`S}J4YHJP;$b{Y}k#sEMG)9eeAt$WPcP)#j+=3S|e
zt>p{y&HqsibLcdfqhO8*v0<iKM_&D{lfXkAt^_1MzJLCVISAnrjzN;49zg#K{@{UF
zb@gQ8&-nj<7il=zm}2x(9vnU1PIcT~k$kHIq-gSliaw}vj|BYj-F<NzGYJe5V@a_X
zXSV1Jsi@JNX>z<<!f(9nmqx-Ifn2ujM(D=hbve--Q42k~Dir>Dxa4^$j`aaQjBwj_
zXvuYV+H}EWa{ybX=C=oA`%u*^&*m$TS}iZFXuFw&Hh14oRYu=LCMCviTG8YmHfcMq
zDx~ObR<JiYn9ZE7Zx{<1?qFFnN4FkD%s~Y-O5%@)y>^VwQR<U&U?ThckDVbD?^|re
z)4GN!nd*hK`dkh={5DP00;3SvS@>v7mt*>?toN+@9pu2<b=Gmrz#+sNc-9NwmEv9@
zpET)Th&B6H9k0T+@mHm~eFXrgxt-wW{@CSTJ8dpK82~s9|9?(12{rO20G!4e4{(}D
zS7VkZr=gxlDyblOVc4<ojeFS7aksU~<k<2WQlzG$iO$}$a)Elem#B;_fc^oK4kA7j
zay@A5*-LZdeOzSnQ>T|9?jYpMlM2iTz^buDR-&}2(cYoQp)F3eNtkXCuKMA;+Q-(L
zmb{M{I){%nF43)*DYJ>|{$`Xs@cgOo?PMG>M@Qy)f*wHSF1(x?(s@_y6%ogr1d;=i
zO;N4M&dzG<!?tp^|5Lif(Vw*cVAcqX6C2%^bMC56QTsitgx6)`yItmZm?sAtHiXW6
zC+Z^C8k^M5FV^LfOl-?9*NaoKcJ8gOo#din1zR?&S-?vM%%xPa4Z=P><ax(iwq^Df
z(*KLSw~DGd>e@#UX_Qn-Iz>vPq`On3JESE9LAnG1De3N(?rsF>?(XjH|JvaDo$uzH
zn{#u{81EPihp_ivd#<(Sn(@qM&7a$yB49zi&I-aGCKbPiO`EAR?@8qBA1NSZ$Tbq&
zxmw;3j~R1Kc5Za7b(OLf2$+=INT7B(jb*4h>^b~s{T=C*5lU%=Ww!;cJd@12nO4#|
zi1SN~FW^f~Zc$I!M-1FCY6@!mDlf)Bq{bi3B+EeL!gP<w35m%jXch<Ht#o=*p7)!E
zK_R9`yF=J|!aucf=fpr7E(cwPjbd)E6!(yekn4EYC!=BNq_uUeLhvs5Ids!!-?ysw
zt5oSVtH=QhC49j=6>g#b)@CdV?J3-kO4y3~s4cfpZZ|>-V~t+`Alfc6KvO1``P7*}
z5LJayXC=b6wS$2<lrBr7db}12uGM~AVEi7Q#z(kW;asv!_!)<gaqWAxMK=Q)7cnjI
zOOPn|#c{V$>_iu`R5V5(h4k!m%Ivq)Qku=hAFcZhJDR_+&u?=2BW84eqkg5}qAqiH
zR?7p^ABz7be(DLu)J<+5Ng19HCOHD-=oH)7^hF148xH&z7c~l%+lKw*GZ6_$ILvM4
zOJDirIS+~JN#R)#-wVO6P@1?ItB-~rSs3z(oXCICBSZo8+*W3>x`jA}L%mLSxpGr&
zqc-4BY8lU^R9(gs<hF1-BAbtHW555z<r%4RVy#7pw%vWUFuSjG77D?rf#7=%a8oC1
z(RQj++;(ok!^#xR>DOGsl&lS_cK_}H8YBS8TIq>+YUT|8a7^Alo*|EVs=8RnW*9SE
zpa7=|38Vi75lNB0gJy3xc#4HBcbjz-7{_<nw&e_vePjl8E1ezM7Sw!UB}HVS7_u0`
z-TH0}W(w+Q_ZSqVv`jyU{)#Gu(S;(UP>*uXfrVPe^Lj;dXU$H1Kenx<+XQlr)c#l8
z>VNCz^vfo6XPlfuu2*^+^6Ldmao2m3a<kk>lL8YrkFJ{8?H#$aG6Jv?d&XOk4O%$4
zSDOlm$~&I-HD717(Y#Ez*5FnQDlV}*({re4m$W!f<aLr!dzq8U&XP1Dy4DtqW#58B
z4KmUr>sk>DlFk5|3oqr)l`7Yx>2JK%D1y%%_#JQ!y1s-3WRH`mlEASESiLH&b3~Kq
z6FczA8CZD^sVOf1y7R*Majv_+mH=HyM2=t({MzHUV@GDm*Bze{y&DZ`NI1>Zr0MJD
zmJ)*Hb0ST6e{!?*6p4{gs-@h_dk{f+G9i%xek}eDuILV!c@GRL2)7X7M0)WQ{3=7m
z=d^gj=6(vI9b(Xa>H)^M7{_9C8J*9%4=0ld=QpNDfbk%=mv2H|0<hkc4VlkTjl0T{
ztY=5B-*=z#Ur#MD$ws0Joq7d&K<FF(+FIBf=UUOJlrQ-&2m}hJL%HB&qcFGJXr+R$
z1JO(xEIfU>E$8pk1)^rt*<%E45&)`HNESwYE!|*YOYnWYH^ik8q2MW%eH(d%oM`yr
z%3@lmx}TayMIgH+f&`jpQJ=Kk>X&d<L4nuo$1LrvLh0I#x8m<UHGHt1wJTM3YUzgC
z9&yJFyM-WeJIdP=uhHSh^wJ2@>akSZKF`MYml^#oJV>R^nFowV>g$+=L6Ziexgrr2
zh2t+4VeXAE4~&@`7eRZ?_rO1Th0!0QusoUam%}u0h4@LdxJL`GcRcG?t>%El$f(S(
z<O?If2gzML_1+<aCdx%?PmxenWptK1lDgq{;kNvsTkYtOI**NPUy-PE_cFb7O!Z|>
znwUO*doJE&`FIpu8%gWl27rUmf)J(Z6A!g3{5RyCpbut~m#)`Nhu1vFd^a;tj<9Q5
z&06SF*YQ7^F_Y;o*?3&<ACda3qThW8Yh^O*O{IGN3Ok;|SxWkXC+8becG0;?Y!eeG
z-|-L_z;{DMk2{^!1znRbFg!YYH}&u1!r@Tf(Y{OTt!ZBFFGa{C<b*UC*^97jrLn?b
zQ97EY`lrwS_D?t399}s#BkdAMkhFVyy`m;xP>2(C3TeUqQRy8HgcbYpuJkPkpAeXI
z4EJ%MRj}<7)eLpUx1<XOZbeqnS=NdrGFCH`Y&8Zo8dQt5+H>SCMYd|brUt#$-&wYi
zh`TGf9A-$jA1#qD{YEpM1A9yLcRg%v^R|b*MB--4<c*r-n{kKFU#T@kMMH8@A{kyE
zN;p*7%#e751LLA4=PNGWeL&caDCb*@&rXEn4U_j4?}IPJo1Z-(*fY+IX880TZj38M
zdjGu;YmYYN3q7t*D_*&OMyP?hjt3vI^867FKm5UDs{X#yrb2gz?}?_=TrCW9+c4GV
z*3BCYU<pUPx91BYc%L4u-@8GnkI{xLOR@$$WQOUqh{ePWx}KHb_r-*j#S)6yy$rPF
zAKy7ih#$R-=MO!sMJgz7eC+`%Bm4$V=0*7yZBrf(m^=y{@4=!9kv<1Ja7>5C><Xyi
z$985oXyd3JqqTlEkLwU(J|}d<EoVasppxm~`1@oYFwiPY?00|Kt*`!m5l`9_$;seS
z-gBynp04SD&|=C<qyEO-PHmL?eaXr~vk{Dx8rtsOw04<drY{2({C@?cFS9<+kr*xB
zS7Aaw2KXe#uMN>%q1XFwvzBBNf8{w?O8`8~=ib8^8d2y0_IRP^_mk8{hwe=*^(XJP
zfD-?8hQ9z{4ol;`e*<cEq`0N?<mcR{*`MSRURB+?HCn`_*D`+RQdvW+lnJT^PZ+*Q
zs6i=?V$iC}ZiPw?3l(3D&}iu?Ppa69bi2P2J({!ip8@6A5_NCf1Eav``#WwLzyu_v
zLLnSGj7E-lz4Q{3qj)`Qgbc2q>!j6{z2qF58<Ro(PUB4(zbi<J&x9_?mEZ`2pFeJH
z@!{Q-I;fgAalbZUxN)hCw;|Z|wwa1$d5-Gg;L_gpI|mtNu|usFC!1UM#vzoXZKWa2
zeUB}<QC=O91rm+caI=$8-gAHZYdL?Lo-}JaDb)FaxNutwiIM)r48mj39F4|;RZ0R(
zsD~IPaCJg8LB&_jD0zQOyRQnAeX>k9{!nQ<S5`Q4eEz*n>Wj2G@2HvCb{Ym)VOz^+
z##6mS)T-77Hkv-qeK8PF-MwjhsR7ew6*j^$2Cgs~wbs0Ui${K}PmNZ&K;sKRBR`#|
z!}&65C#c<)6)}E}vfLjAP-W+{ms&gXy!Upq{()TI=2BbGzD9vat(M??M%)h#k|M{9
z=@#9%Yy}wEg;IR8qn^KDaY%K9Evy>U!anAcLVs`!zB%LXC%Xo;U?|%Lw8*a|*pHy6
zkVRQJX$VN6qc27qR<@7MH+fe>erBZaCEK~Ke+CTxkJhuUrJptH)coJ)8=+XQhQy;C
zub90&O=po(pwVoMRWKQDvSskb!A7$1@_m4{q<r!GCn4^hFCCJ0lWKOTxbOfgU2fnq
z-i@QuI>|{?t?krKOT;@~zCmFa$gNW2l^1s%gGHnTuy15$5eo=4_NsnHgFy!I2_yoJ
zV#XcnoRj8U1xVCMK=BVUj%}_0r-1%{Y4NEAAnQapNnl0r%Em`>Yg<_R(V*ElBQk;@
zR2Hs&P>udI!zc*<#hN>aA|-#@{E&lzuo<xQmkd!yi&F#F9bDnwwB3N18(h3pGkRJ$
zG3`G?GdWsH!gpJ@tImG7jlPRGV!H_=9!FBYzFbAr`shaVE;7zOuuLJ}d*e(YgNsL~
zCLjWJMi(Pqg>k)b^m$eoM%=)RHz@3|9d*FTB<S1eYC{NMdVwFK7P<So7n?Y%A#S;I
zGBUo@XG5iGZI@p+6bO#;#-o@tK3D@9*6fSA0olic>&gnpKn8K*wGa)QNZ>Dv41Q`h
zIAu*uQ^{C#CO1iy-aH^<J$qje<~lkT^1aqA91uvuB8TB6t^(Np`e^>>o?@3wZ=+ew
z5b4J%@Ch^>_|TwPtB8z7mw6f$U%4{8j<)x?rMCXl`0v7UoFUys97v+S%|n|ynI-X&
z-R+F`<@n_XexojgbBCkKc?0{|Gqnl8z~?jG)U3{dR4S?)NA%_0jN!$@==1xoM?&=9
zrHjO4iT-TVkTE?o-E_=c7Xh_~_=~}#)e~<HDRq!zAm~ZH+C0kWE-@H<)0?7m=o8R-
z^A+LIF8b6hDCdehEjFSFIxXfk2J)FoI-w)bX-Puw!HhXqi7iM6k67M*u4XhdJVy?l
zVDgHF_>#gxL$P3_=P<V+S!Q*wA@q)KlvHh88WrhvL)AH7-zZ8hBz!c|QFl1;Hae+)
zp%?1j|B;&X!G?#WW-dxUq{_UYu<^gsLN_zrXL&3H(;tyZt1Rtj?T$bSs(0q9I^gE%
z40ysXsPEG;csRCp`hWk@`SR!Yrg_y`tMGh$(C<^dflmJ4IaouqTIG4$CtaOHouaqx
zo453m?c0+E1!1Pk+psjB)UNq?1jEcfq`**C?&V^S?|@`NIlJuiII=T9M)0fgI|NF<
z3_kB@NUaFej0VgG7OK(wqQ#&dl`#JFZdmJr0UYS}+m>X%riQd&1pLQC-8nt}kY#p;
zeqb7DE57gao%KJ3d3R2OG0)_MBbB~&e~Ld_$bS!q+2=^3ODTQzH8dXWY5#rY+&<-U
z>^a(sk1=#~*$*KS$}eYJVekIILuRk$gdugE3NnaO)6CFM=fhX=5BYIYiq`-)CNi>5
zJYP7yM%m{4&NvjD?ha|+$x4H$XE~E1yg7jH0<G-U|KeUipCEN-l0$KI)h<vYD=XJl
ziPgDG!)f;G#fr=L7k&MIQPtH8Jpj2uh~0&)+2yfc=n6`|FbH_tx}Hhbb6Ooc&o;m7
zi{X9xQg5vHQVK$3Fa2v{^rZYu+P3;WrUQm#GCE$mDTg)qtVMMn(4p@Ed)B-3jukqG
zZdaJN%$^Rh3KN7&0eM#o`UKGS2Nke;RAG}~Z4ekXh;%@^2HAQNYz7U5lrA+0)3gn3
zkkR2;*x!IpQew6HIieYxRf$IlKjGfW3&Zk3AW$Sf4L)#RDYyO-ogE4tCR&VG<#ExP
zD=_vS!tXdpnlzFAn^E~0@Esaql&UE@aAQic_d;v!RKBlU?OeBz=nP#K{$3HR*4nDW
zF4oM&%#7Q1A-LJO<>(E4l0pTmLq<(v2yO*zvKYTJ>?S?uzRdpOzkPhPDmi5X3;$N~
zdFiM#L{*MWxnpq&d=R*-15O$hasX6})V-|O^;gh9qU&^X2q)`ellakrqBwRcDgxJu
z4QQV6r<Lep<|Yi#DSynwn-ZD;N*gZgBRy&+hcaI;cEC9^4<LR3@el*5Qo3baI@eC+
z@A*^My+-w^zE||^Ld5_WbxLv;iVX}_HZ#%s&>~5|?y!*^4%m-cn-fB!PxzRjKy|+Y
zp-?jD(NS3EH$7C)(I_4wI#1HApPeA0Gb>3(!tZIC@_h0}W3tUb&D{fSnMOAu8r+f*
z+v7wMTmOTY^4ChUHngx?n@XlTje&>87OrmH)=XYNDr((yJ1BTung>N^49~FeQUbtn
zB7`dEhRYo^2K~x$CwOe_hY(-zag?2DtdlY5nCT4h0UB+)t2Hkmfv8GAbKpan;XBl>
zHv~{ed++fdPZP+k>_V)1ymG>c2@sy(SB)XM<<nbSQ+-4v2ZkD)PJSp^JKbdMXnIzy
zLyQtH@HiBQjA}kyIkSAcQg%~DxvxqR2$-j9K@I~)2*49z8Z-x7ki1w35dv_8C;Qgu
zS?kbk#nNU&!!s2b?mc@xc~=~izoUxN>a^NCM3FxDm%d=Qo`xkvkaq{$#;4s@-W=TM
z3$Peb^&^hUXma@`K3eg-x6;9Uo#AWk9<f7$9K8ISHXH{{E;s%L%~D~fdA`pAU9oIg
zkVJAB!<ED90Yvc@)H=u=O|M7L@MWAsn?eTla1%(<&ew&AKU1m<amGdq)pwt)tFJKh
z3N)%yxm6*7a;slP?=LMSY0ZTwG@G>x>P>~;ks1wO#eEpnIn9dkCoZ3QN3Y;{uHRBV
zOn3R4%>j58+!Kr<HZa{E^@u{2o9~OT>dm7<L&0=^^tD5=Tq7fH*R9al$0?(8@E&Kz
zb686yy47YXDBQm?`KnKm>0GU9Jr^rmL!V%bZ|6b@Zh4;J8m1WZg(x}}#%$J54Fu|V
z;~XCcyjxU>qFAN?z(Sm?4QKnmJ-+bY{lE<{e`H3(@G@*eyuRtoM;ttbLdU*+{&M5#
zko6Q)Bhxv!ZL!u*)r*<;B#hF9gZgso_~=h~@C}BZ)jj~vU@qsbbV&IBGUXL_4tyeR
zeS!3N+Ea|F*Z3hOk`*KdgGKx-`Xva!M0OfVm&rqJA;P)lk}4|1uXM}TXtCrRX+5o6
z-<d9v{4w%5EtmyF3Xp2E?Oj-Nm;opgI5y6Dey~2~{#Mne+lnSGubrg!V^?j(&Z6U|
z?YjmCS(m)abRPNawpNSv+u4{&gJ=|ftGF$$KfN*tHor#dvHKFDMobFFzvOTiPT#Il
zIy8Rq?2vtHaMTlZJ;?uH8%a6l`sE@*&z6`{>Jdh#S7w|aOCS!$2*kASTFA&oeJ!a0
zP=6f5Na<<Nm8dun#U~-<dwG45G<2B5wzA~y@A<a11C{Vl-S^UBR|$7AIjwKLDhkl>
zo82HeGpH9)!aV;&`+&HkKn}TGwh?Wl$5XT7iQj(X)X^R+E2h4<<5`k4r#@v-@QDr@
zygMU{`ZfC2C+pf2Nl2(zY-YacAI5PI*NgwI0-$m=5`Hgo4by2GO%Gwd#Om)0Rd$Dl
zh7q8Cp!|lbK_=sRYnPN>OH$8FI4iyPMo9&kT!qV~dr7gxaf(XbQVbpv*Bpu!_7@t`
zhfZ>fY+X;$Fe0o9T`iwuhKf5(-!P2cYn=0ag_Hxb5#Z$WUG%HdHe3p9vdO&Ck}BP3
z*go~Wb{TVdog+{6^-2T#3t8qr)jBcTRRr9tDdfT^Pk3lbkkY+<H^&DJo{Nf<cTcmw
z<=SLBu1N*%F_U^H(U4SfD~R3N$x>|Se~xaHQj&_M`J+uwRcPy?KA#kg_B||2?K~pi
zUwM1`Ko-EqwpEPFP+W!!?8~_FxVJ&zGD279yek`hhG>9444WrNG9g&!XPQIThtmZy
zCO@Qq<JtUKXj1E8F$sN%WT*4Iw1Ze~XZ|q36~*v-Gs`_-K>y9~$1Z>C@rR*wwSQMJ
z2-58|iiu~mgCu*U`WTD-Wci85Kq!SW>m@*Sx&3>G$>7+C>6uzm6L_pO=G_2pzw#sQ
zU3W|x)yfR7(P?o9O;m`Qi~6>?_I<n@)7+A_qa^+75LoJS0@#%#Cbv#ttR=&Wxgi9Y
zYK!T5$Fu-9DlOs{Bq>1vG%e)RiWWB_*G}YbXLoI<f2_?-xMaDE27ntdE#pqj`W~QY
zg%E&UNclt?rVpQ!T0izHW*)9Sx$oPY@N@seTEY&?hI14MjUw?`WH?&sw_%y)nn;l&
zW#8u0?)JVR;J=+V(e-~V7!^IDJf7whbD(kpjw=b|xQZ+47vO;7T1_ckBGBHXb}U}6
z=vzp7rKW!yiLE{od1P5%5bW|8+=%`Rcaz^jm9}bTd-vn@yWDN7$#T=QBb*QPWXhNv
zypJ?ijblQ_zD3M&db~T2pMUE3{2AWE^~Zu`8e@#TRxpXoM*D6SymKwQc1lPFh!jDZ
z%3YzHwI~s1E87fJE$CG;j-Q+S4fOoA)J_;>GQ^XVewUfNmaRYZ{37O-cya0r;g~wt
z8v5UyTY64>$BYj2DYt0aUbL32nQbL7LVL*htlF>5M?xV>XpemnV+}GCYNHNL@g^y#
zxvktx`d@{Hu3KlEgpJ*$^YQ8u4N4yp^EHK?Iv04Pemmah7dXBfO{OD?DIJ=g?H-TP
zhez4m4INF28h5H&u7MO=B&=d~SJA&C-zi{Laqwt<bhy+W8gHCrlOg=*f=h{k5c%lo
zi}Ks99|%^I5F@~vW7_lb`%^Q<@$^VVghUcxK_dO{6UGVO0$}}QdlEtR>{=Fnm3I}<
z=D7YFB~7Be;njF2?}`@^qLsMAdQ&d!9HR%N7vuZ8Zr#pkI{#0~THI@IO%omX`2;TB
zjl4|xYyBFXGH086f87<_dw^h~zV+WGf8z~msR+mR7r%Nkg<M;X(2cnZgGRV>GPrEL
zHSpooi7R2Hp%lt3O4KQ><R<n(@MqerkQ3#L4k0_?@Eg>cg8GA<bPIF{U~$Ahy3Ss0
zDXd^nIkE#F)8}z;w;1BkCqf@SA3ztE=uHa24w@jjFf_x<{R1B;__3RmHZzD(T)=x&
zbhCY^$_FD!1UBeSZixkwiSRu-73Y4)E5srY$Wsp{V)u8m2g@`IA~J;VVDlKpvAQv;
zasff7!3inS3@hS;d|XHJ@$(g*9kG5s5+s3^Bh)F>wGu(&s~r1a6h5FDW{-(PLH*Mi
zh@?;45C*g^1pUS#DLm98_w9fil6`vMK#qwox<W-R9Q@e7sk-+*+P{gLKc5bPpr1wE
z&szSi&@Xqo$InlcY^uJ7EF?la1cf}1KSXgu04qr(QTGEPS;6BbRh^+&fl|CX%nb=Z
zu2Ctp1R;y?SjlNU4}iR)_W)yx$W0dD;j5JXn<^{S174|nj2-p|2wFq{_V?ckfo$dR
z^ZMPHMk!zbF=8voUN%Yo;k+Sf@?dd$eTWB<tV-m;B`!^IHqk+T2ivsk%^Pn29I^<%
z9+hF93G&K+mEph2;3@Fzzsdk1DgVE-44-M*CQ5`=UUy(t?bU2Ef2UC{c%KOS1e`dp
zAYdg0Izrijp?&||{UrefXJ~qTk$!HG6fg;PAiUCDAJd(L9Ox#VM{m9}JVz`30u2O0
zuz_u)fMy37#tX_^OYH4C%!h;P0E{c+d|Mvz!<hze;6p%=^F@3jWVnbAz4PgPR=(;-
z$Q!LVo=kCNH9pDXg}m_sV$pbI>K<(9!xv7%yl#76d#id4g;*4ZPW)Eu;i@S)S~-mI
zacae;iWOv#B2YN-&ZxS43;6}|<MEn;kAza<RnVWHXHV?jk+vqtQarp;ln7;>c-nI*
z1bLgU2oQ2`!jzWjKSH{Xg0EANj_>z31wsnh9v2@#tRf%u6SU-BWGF0#hf&}`dB)0W
zxpeFCHSn)1W-{6`+Z!!0Xwsce2Xv28SRp~CR>r(74jG{t5>W*a?NdHV2xboV1c|ZF
z*l6sX5X?j;0=k&tKNIW(UFJ|>Ns0xSCBVk>gB8K*Z4!W(#)qGYcqnh2I5sqYBL*Pj
zL@u|=gHa{+I6wz75~;6ALB6h7$o-5oQZ39D;&&c?5U|1bEs}OS>xV*d;xbO=<-K?q
z(H#|hA5ZeZgD?H%jtvYt13A~jtRLs61=hr#WD(G~;=u>~N*8e-*LZ@IFAiIMw{r?2
z7he@{WnTWY$B*j_SzQX~7kYX*EUHmOm^xHsJHD4tkCMcJwJa4EEW&{d58P@QHmfBu
z#H|AbgXkC*_`Zo$Toqqqk|r6Eva%Q54+}_+M*C!{P6zSmA@ByPH-&6e#$@*R8W;%;
zAIz`$aCsJmtV!laD_;EQe>SVv=0BT-1_cU`P&$Y5|JH=~Ne2elxrt1pKOY6$69U5J
zmbo7%Jt*eR%7CXHXz;LK+n`#c(jPBN{-4G4<pGmRE$mfULcjo%^E|jTejHK~abQjL
zXB_H--Oi=}sY$tXhbN#=JZ_$fn7crLchP0`1{IjwNhOKd<3cYG0bRL&fA|J63gk>6
z|Ie9#d=NYp3!v1HKmj}<2xyq%e>LpC8U}RH<G&jA|7{zF@IQtIqX_#6O5Wsu%ykhB
zIODYy&$=IA{s&7)1x(7$2p%o+KU)C~{)88B&7X(RXyG7#1D1c+yZ4nZB=CMT07bCC
z=2aNOVF5bbqxE!$*%5YsYIbId8u<|J???duQ9##f4V?R<37YV8H}{S<aoY9rGk9<g
z*O=M1QMcWHF@PTzLIpw!%c>WrLqtLo2gW%hz2E46?3WNS_`<;<d0HySXyk(K0#fhX
z?H`y&|0BTv{tqc1cn3`Kamd4PC}4&y{G+j4o3N(Ojd;94hG6$dmTl|f?1=><st5vc
z@%o&RAj1m*DjMVpi?N%e%7f9bk^tNg{)t79Oq3s385Dp)izOf7HM@C}F_5(ob3y0_
ztr69bn1nA6{ejZpS6=z!6in5M76R7jFaD@r!dF2C{0aa_pxOK@FzzWDdIQN@q=4V-
zagX7K2+)S`C;*Ows>*Z7l7qU!FKc^{PSJ5`#L$3m*m<M@`ObceWdXZ_fC~J}Kgr0^
z%>EHvcJp;zz|i=TK>9_MD&2zj@MA5{q9?(sw&6b}rO{3iswCh%|A+bU0F^NQV@8W;
zVJUrwY%s{@)A0#GLWzv3ye0vJul*<dd4gdtM0l`zq>qw;q46GtS8*~^Ap*j8K%{?<
zWf4u&=TZLlso@z|Jx0;T)mzgt0Mr{X;vUHAJxcbAFW`|7;ujL-J^G+-Gy!nbK9H39
z-vYP&U(k^%NY^|L0D%+XQ8KVGB9EBl*c{%N5Af>65b1-wU#<NFoQKB+=Ii+f7*s3A
zyaf3$0uxxhf0B`-k^Nh}6$4!sFgG#C)_47P>;HeY^;A00$3AwEjWS8UNc-!_y_0^U
z9o*KD)FA9=RSKZE-UlkDl&;qc|5UX9vu)2z08~^6GuZeI4g6SYPPvK_&|A(&aIL6y
zy5k+hC}095F*c=(jDVa7Qqe~(^E>0dn|8f)Sf^xnIr*hgWB=i3?o05Db-!RJOMkEU
z%%6{gBE)y6RI45NW~<eB(iwK+A9QK%8VIzz7hUGNlCpv%YhGD|-+Y^8sGPYjN^fNR
zIFZ;HP>(3JIVIF^VHGg$cb)W_Y+qVy&z36|1;9h8exl3~fQLCaqDGth5=I7}gE%g-
z@Axv?+Ez{Mw?d9f;xhmGKU}bnvoH4QF}0Ct_<7dSncrUnF{$qNk2>yL2JhM<9qguD
zQK&|)Os{QMyHvAX5JvT;k-D&KkB)7%XWNc`B4HkNCqmHan`G=)+!YV1ZrC`2{qr!s
zD(`55E$&#R?E0ItWs5=#gTK+LZyINXyGBgubL6%6ZFw-@`;u;z&-NyWsZfc+MZ1(<
zhBHPE&6#h1+OIe*c$2UCR`GUr7ab6B$Y*?^APxYEgUG}Tx^|2<?8#~OHSMyO<y{E#
zCesBMcF^}I2F=jh+RBZ2FIXgQ<IU364@oMW!#{&U2|zoIiDE^SOfih2uzFu|cplC#
zz<+>*A!dTqYG*<AXuVUvQ|XOcGd#KGk5^&!LD;|Chaahu_H~K}?zl4-!sAic99Q44
z9RwWSehQxt7dyJXBRVBOjrjdKzn$)=inD4qWM3&tI;l4z)!_Hv+B6;{dZxKuHV-Q#
zMoB8vNYv}{FNlsYHY<^}{*mFOZV4gU4YLJ;sVT{)PSU@!v~%{q#k4w*z{S-lxhU}J
zHT|p)-HzmJy{z(G=S~rHxip<4M`A>`L^Q$%PBg^w-uOj|+g!Qoz+#t;6UlDzD~F#c
zG*37}%p&A0$R{!bCD%4IX}oc`zF9|At~BlCUrp}vUI^?t+~B6E+yoFmq2Z7~&(lf`
z&a4-E5okbV%s?x+JN|t~kKyJGEyGU)pvk^E>w;u_I`UZ}C60X;pOR*>=U-5(X4sXC
zA3|eTJ(_TwVRhTHeclY0Ki8`xA=(sznY2Si`=1;RHrFXmPfawG&DAgckj$8s@K7)K
z-pz&2+H$>kWrwMEkY;z=vMHafLS22@K$?=BN_UJ9g?2nUo||sDyS@V0;MPuydPB+Q
zLY;2sxAbu9{Rt8Mlqlg$RKZ<wnj$qpqK@AMq|()iSNkhljMMAB)-h27=Og(uh-E$X
zuy#Lz>ZpsTBlz#DU3l@!n*6*cFH^EU-!FZ{JsOj|+EhYdypVoxDMoea-2Qs}6B1Xv
zfRvXO6jAW=|J@nl1EK8Jxf<lcltFi-e2zQ?k?gbC6bi}6jdhE;I#u3Eu@KUsRAJ;C
z`3#8?)RcNLtPc|mz|C+jB@^nD*CYHcs++`sPAS5Ue8JoXUPoXYc>UMn@Z!#MpO1`S
zqEzwio|oV1!4~U~q%_Cx(-0S@#8=}7VKDK)pFplyof!6R%*sbf<Gw-Q9ouYWv5q?O
zLL^TazH5zSQTyxR`$aG^FEqC)!MZS(J%&SCElHJN^8S1G^;+7ZChYffSRxmR-{k?{
zH18YlZ)NwKtGwu=wv#*BtKIG+6SQ43joJ;nB0u);og2}`Nk-9c{Zi$1EBz33S0dYY
zcN}##c5&LFX?>Bkf3nbOW62ycH7nx8#kg8)B!eH}GCkf+U4GWb>lPnM%YOX2(7!f(
z;rMH+?ag|R$!>_(TU9n}+C0wu1G%YM+l2U!2JN30T4a^Q-WhUjA&K~maL}owtwuT#
zMsgB5OF53yzC2>lLmMiPo~*TXTKVf1ccN7%Ih5H*qGRlK|3%&|U$bDGy2a_&$9h?B
z#fh4>xvb$1lx-u!zC?BFdB>>azQ&!&HKaebHYqJzF3w}Q`1c97qA1zhzYb(%wkG5(
z*$9qOX2NNb=Gu9)CC6qiHLp|Vn%qh6;AHUcgol2kxj5|CDf_##8Foglq@juI)&}N|
zxUcNLDbsfB%-R#=F8tUNDoK6n+dMy0IKQv3KTO!n`JA=dvR3`RiP68ZlbbhuJ^w9U
z3N6ENjfguzh8ZnsIk<o%7IMB{t%Q<iuEnaIdG0K#^D)Gt_8y%e_Sy6cJ-%i_YL&S0
zY8xZJ;KnnjX^q^fs`t2Wb=)uI{#Ml5?Y`l5JUNKb&Q;1U<6Ayn83m02#0QJl@OQmg
z;y+%!pxG6c=>B`1M}d}CUsO?+;)J&=yrCjuCbRoyCb#2MLeY}=bSdmSyjY)`gxp;9
zE+oC?&e0)Hz?Ks$#Na2%V@W_+8|Rry<44k9l$VM(-s2ga3o%dXB~+fm%sr2y*E*?T
zaB5ILi6R_JK9JI1GJR7RhMVY9e<qrDuldfXf7KQr{vd<7SpTVqimuG%UX83;;H?5G
zT3qm}X%#Ac<KeEOC^fD*WzUT-f@^t>^|2hY^XInWO&uc4g_+&{sy>N`$m48VyoweL
z*H8LCmK^71yer~|rD)+I?)}jhxoFv#_x$H&i~2sp8-mRIA=Ado_Z5tCsQWK`rPsU;
zNVDATrHgp_ZCGL@b_>#avOd<`-exkCH+_E=`#dgz^n0=Cm`coGRs`{l2m2zo{kIWl
zioTa@L}Sxv7$mQ;au0rTor`?Lojqcq!-!^}m9GhHAo5IOpN#YSPcMKF*IV7Jug-^4
z&k&cZ){VR!^H)#po^qr5d~!3KA3M4%3$Z?Iaj3i@)=~U?k;tXZ+h?S4H~Dl5NubH$
zZkHc44eCPbOgzV`yy5yLPuiw4n{>FleR(tOA$Pi@PLMv5`#kIg9CtVRPZvP>c-_^>
zRfkxVw$j`7lPu*$3OeQI`vN-0cdfI*;UfZWdS-4hRJuC9h@eZ@bv>II=645tAVEYN
zJbj|gVx!H~Oi`vHjHk<HZ0}0(y0)!>u9TV;0m5PXpX2*YacBg8ySv8H(De#WHLpI^
zg&^`I7^e!GUU_y9mvE<$s?R~2TJt-IyXj|=@3)GB_~t+zR_|Gta_O=hH*Thb-%L#a
zAjz4(>83|$Uu!1I(EIaKJks{DvBoHQA>zr(P=;iNfH#u(9P76|jU1(XwbXGyE&WNE
z2mFfs54N#gSW1Sw>vf_~?bF&p%R6Q7q`69$?@0Xh((itEhQJaz$WQ+bA6(W<Aw>(Q
zM^MBi4ME)%CeNPUj=4i%+)z$-^O#@8@t@8bL%!2J#-q0C|L$l_L-|C1XfIfO9H|{=
zpXM&UUm{TnGym|`$4}})tIcb!?%RtaCr{3+m)CaA`Q!Vy=le{KEi92g-e26Z)$D&S
zZqiEHJGtEXXezZoUD!CV?Ao5LA;d00Kl#Pl+}dF6@&F_yfOK9x_2VLK_u8-p8)?j_
znS;)QJde@i$?VRhg(4G`KT@6ifv`916bZ+IWb%2crq&z`jZp)GQ^YrfA$khd8`~?d
zrZCt!5|Nf2M=#Ju<G>r|-cfwl!uJW9Gqr%i)Y&DKqZOQXxaRuj0}QI7Y!V?H=S@An
z`<$q4Qxc=&A%`nTobRY~B5t&3q5?67r*5)5am->4W;6NR1aB`+X4^-n{GaxFi1Ocb
zOxMHBJSTx8L^%j=YC^?nHro_k_4eZW^_zJ=^KSolWuRNpUyDLKS(?IMwo}V+=KlOw
zN#WSPz7Q2W&p>06?oHQZH22a-Voj)P+)P(?2`HgBt2Sztz?CbEPu{`t64V?FnXIr*
zu~5!q8P;!Bs-`y5s-y)sZI=FcAFsoFcSn7V8Vd<h=kv67@FvVyzeY|c6STcYAhA~_
zk;2s=o@xpl6{DbO%&E^0xRrCE#~IYb3fRajZJkG%ddUlOKnB&$rnogSWIjh(c#f|3
zQtzCsxfYRRK@Qsst*ysTea}IRqhD*E<!_E<)s<|rz_Yso^ga|sH!2lDBsZsnB@dsm
zVr{pv-66|M)JO(*Bk#egpMgXqlubeMz8+UGny-b7Z`_Hd(m}0!5xFvJsrL9!<y5zS
z;B+lP2MXLJ<3un_$YpuO9@Bo{K83T-x46o|-P*!Yrphiq!Nv_I%xieog8L&cmR*cm
z%8ZUm*Slh=y_t#CUbDXS;`Lm~)2ap@DtG7ORSeweHEHwKp7Vv;2Cc%wP8pRVdEe%h
zK!$bckuWG7*~L>alJ*(Xvk8>;wFr7mHKhWLm`}q0dc}wz#q!BU(kRPBGH55Rn1j39
zQK#FOS8xsV>GcP~6gM5l4A4e9FiwN(y+-A1ye*OX#8!6$eKq%n0|G>uhFOcIo*J0T
zXuM>6kfs|ROFwo(%CJcUam#Tp$=Y;06}F8m$3GapjWkJot@Ybckeg&@xY$MG*<05M
z2c?#Y*e_^diQLx$?d%%!Rrm0I1>DaT4Xx6!Pw=|hxz}V*a)xs6ZRWM))PglvSSncE
zRIpW<D+*(yotkZ@@8vT>;Uyx&zKQgW>suFKW<IZ#DPVaS5{LeHJHn=1m&8|UcT8bF
z@aii1Q%@{eGGzzt9eBE+Xs=gw=Wat_Qg6tIeaaqrZ>lF;NEnhH1?b;ZixRXP#qKwp
zHe#?>%vY&HDP7<axZ%(pN9`q0$G8;p6Qh1T-N*2Kbufloc4SnECetn@_j;mr>?Ts>
z%g3+yPnE<r-_2~;-FJ{VAqbR`l;c&35x!aFksJivJi^kzDH+v8wMsbcs<&?s?5Yce
zOk7aLaXY#GqRLX)bKxQ)$LJChrL+l9xA6(_ixt=Y3?C5bvQOkqdMiYr{W=dmw<+b)
z*MWZ7Llqh=;_Wn)p76S=*=z~37Y=@pZFj<umIDHAN*fwIL%~UcqalAxtfgc=J=x42
zi(MW{Wzphr%#rIh-qm5Sva_^P8txp2aa3VcPb9L0LbKCnnb$`B%1Hx+%-OQNeXW`)
zy9>Lcg%-HG(kCZ0rx~t=h->Pz6@gD=?epr{*f*QeXWr+F+Nd(Rq+|Vrb2%)HZq4L>
zKV4*;iBuWIHW)){fr@$<1d8r$!iILrz|*{lQ6^DA$6GorGE_WSfxlT`P^&4qyY3)I
z^t~5DIP~3T?f+!oruPf3g`>rSlpkULBaJTOg|gb|5TV_g)8eVG_$#q(-k1X3;-FUo
z?0ASv*T{iWc(o+gZM1C9ICL@KDe+#&rzkM3|0U~)fO;N+$K@P>OUO5?MNf&a__LGp
zz>|Nf+%6`u`iD5~#sCG&Wh=OJgo)#gBTITVf!e-i&>1S+7tbE{Nf`JRDx>~HX~@Yi
z?2T2FN#aRZF*g}X4*@-~s$GZewWORD#oJ<Czy*-^oPV@%1fEg8Y-%U5=0`tOhr*<8
z)Z&tqMf;x2Yk&HLZGWuE_-IQ9((B37#g2SG3nYUsPF$Rz=`uz8`)5~lIV0aDreHs6
zMpRW^lyWHShax^b*}p{@R<4yM6!l7EA(9>p^R=w6^d;10=2G~Dc=bke-8G*IsrH@X
ztLUDe;sq*MufjCCM=esQ1_j9Zd7rE{rGM3Sp<XF<+=Ee8x#c8s)QCFl|Jd>M1)-1>
z_1)5|p-;cQ!H{yw7O)&IR-8U&-31jiLQuX>4yvM$c466cNux7;zDf9JSwu3K?jqSI
zFmEtX5AHy})MKT+lAE(U{XzH)_u`VLvYhMS7sI;D2uXL59V}cxAdIX>=AROiU=M7(
z%CN2c!kp<UI9@!_N+~p&CG0&b^^xwsWFtvNGB*Y6i`_?~thXn$3dc-60#ewki{BYk
zZQj8RFaDS?>Lfg-rW?|!j{}M8h}nHrf&wSEHcbwE4>%aaJ=lhQx^jE7gUDP&F)Md9
zvYDHztyS6cL;EK^Fvrg&6%guF!5t;RG`0jQOlaUQcrUJLf}b5<(O0Fco5^mR>Jd__
zEB2$jT{ruzPS=KWQ`{#T46A;lC22Ib5L99%Ww~3-<UFg(@CB6@?ttyrVdl;|)8#K|
zj#^3WwabNQOqtSj+!ub$LN~L*rrbD%KeL|#;=XqOYViU@&AD22Y{WXhB^^#bNfoNi
zNiy@8dEF83vHNMT%5_QvRi${G;^(XN#gh3&Z}^i>NE6Hm=+P=r*H0Fq1zbCDwGm+w
z+lDx0R}{_k2}0PWP(zPw)hX;$#e$OFf8~QD>i&EiuTL+G{ioAk`pSZyoMEPPPSp{(
zsn00`lV#A<-iEZG6)~oyYbU&Ogu|!&aiV#YZzkYpltSHRkFn_bmNghq2$FO_fs$Oh
zU;L~EO^**I>=q9-dacCc)##^^4|EumdO9Uhg>S*N?UhM6bcA*-uiwu?{`%bdi^)}v
zGXd`VLI$d(i;#ZKH#ZK=7z~Edfq8Q@lPO-eBrTij4~^9FB@*V!2I!IK%cO~mk&GVC
z^cBQ371A;6JEDhM&Ye2zND|7Rn2kqWD~pcWC$I`S>?zw>dJ#Q&DXG)$y|Ul?2I-U2
zz8ol&F(GIm)GxyEXQ6^9YgE|u7`TS=5Fr&XIhOx2_~^6QJoAO~4ymRbeg0PVB`G4)
z6_u?;x@IJLBLPVYu;@sPL}!7+x(yhZILY*tZ)|g2XQK1n5cJx~Zf(}WG4x8zD5ft`
zN2v<#^s%_PA}o$=QzqJLK$jK6!6ko;MJ29Ap+kI|nD)=?`GWN^NtQ)TIy`O2WX<l(
z&Kn}Puo{vMQsFtHGw)Zgyr5Bqxxxhk&^a=NGGdP+ulNbY-fejnS-W8EpVhofDO^gd
z6(sWM=;jWNu-4Saq3LOFsS!v-q)*~h@GVJ)Ne%x>&z-cq&%-5#_p&pcTiMyqQ+!o_
zWOdM*!K{#^-wYkjNUjy8|2O(}a5k<f;^8fKA+mz62li;XOyE~mw7VkL-q!wxZ0>Td
z;|fR6s7@E;!AD$n;iU6J)R%j$LdagE9_NSx)-jknvbjbU!vf;Ri#Dr+M{{O8J4%UR
zPq(7RS&Qr{2j9Sq?K{dN)QK>nlUB1RgD&}4!TSs<VaP3LhGn|uuiP+N-0j>bbdQ9(
z8Ie!VGcI|&W`vB{$h8TWm(`FM;SgQ%KyNvRXVQjpD@GCzde-}(X9GlLoU1lb2u}r~
zKfCj=g{~)O@ceb71<kwai}HTy+M``#>$>9^sDe9D0kd}P<#zg@BdCVlE$(r&PVm;B
zQx19-&gWqzmJMOFf%B+0ulFP9vI_-YFUc1&Th(^p_#d-mA~M8a$zl%xBA{)0c)?)L
z?p)4nTxS<<8pZwWxn!aDyuEZL(R@L{*^oo3H5~z7uXAYv)p%@or}_3HVNAn(TvnuG
zJGwy5fy%{+XF<4vF}cp*QnbW+&3LlWDHr8_<+965&6j@z!|QExES`7Lk=6y|B1AzE
z1%Z)k1~xKtmzQdlN+TTdjJ{GXJIUGg@5$v8S}k9~=*?HZVJz|Oor&*TuNy|S*b^PH
zgnStdDV&fPRVI*`zJx5qKtE|=!qvV@d7)RM;39;QeGWyk43gZD;}G#ZeY=Ci)1=(!
zNP~>tbOWvNSR*QXye81^<O7{n1OVSAK<E69X;-s_ug#y4-NG25eiA_~EX4Gf-zP5F
zH!f|`fM+S_6r0#jH<9&<A<!uRB+Q}2l)$4hWL*kvxyH&(4YaD!gwI;*50YO|=Na%|
z@p5vmwWrrSXoG=S<~+}!B#6iotLvA*OF)>I>hAK#N=!Lj>Oj<cM->H>M5ONcrw#$T
zc4C%z1m+oGBaK|+A?KaltMnF^i`$U{{yej@q>){Ar9=l7{*AMqFo6In^gVd}<49Un
z(w8)pQe$EPD>s+N8K$GZ{Sssb4>KgAB)pO48Yg>W9@|?TV_5{688bY?2)PCgCu8fw
zcDza2nhaa*;xKpAXPpYi!|0rQ|FUX#?;c=4Y*yu`#omN7(mLlfl5-ueXgKfDsUFjt
zKG4fxP>QP~g-eysw>{k~y}<PIGeV|@Jw~q2MXZct*{oDTsK!?+cC*TSyhyjyHZ4L7
zQWp}SMb}sj|Lcu7WFF#ro>`Ly#T~BdvHawjVn8eeEwy?^4Ou#{@Jp_+36kFa!eXn#
zu|geNX}Tg%9r4`C8BUL|C6XOx(i5a>)5)aM09VVuU`AH?v8=Itc^h^fGmd9g=*98|
zg<N6vzV9w=`7LRM%j@BVb_S#+Zc={cW<8yl@N*G>qnCG+j!hwgruRmRF=B~xWLL>o
zRQRCZe!j<BT5jGsL$0pE{)$zkO*HyHZ|X=bQR9}66CKYpM$OXf`&xonLF%-6P^mp;
zE2Ki`wRA|;Je-HZ!Mar{U)pzcRLfmw!Gx92>i5Hi$}FU9o79$A4m=3s>32^v-aEbp
zho@iK#iE-D*%q0a^(K|5K=E+Y*1GF>;=@m8NqCN;isRh=PdeY%ZJQTkqQ>-hnc^=@
zI>xWAxu`!7&Jz)6(bB3seU@|ZarN|@@0|D=5kQJthmS!XL$g0O*%`hwWg=+YmAJ2j
zn<s#BAwefOn>zlJTC+5!Zqgtf^w<^%kfjac!O830o~bTv>$t`eTYhZWrMo!&(2}(E
z`Wtw%*>6P}TxMgr(Ef~iF^pgeDiem#_8oGkkZ`o85~oCBh2x58Xkp+I&0c9U=y@H*
zpm8}2h&rE7R?NU7PJH>adz116Tki{6(DgkdjE*SU{!{VLC~TL$pPzGS;{LGWhM#or
zu;m8}i=3UDrKem(f|NDUr&XlKeoE^bpQ<M;T%16hojk$!$5XRetB81OvIzTt*7tdF
zYGVB=@BE-DTc<^6*3O#v8B-xP`Q*FW>p2&@&|7Ktd%s{$E$1?es_r|?LZ^}EEDOgY
zXlO1qJ^OowNM*DJ9U0V8+7rc5+L!&e78<dP46X^|8P1Xg8tV4=&N9lxfk&QvJ^Skl
z`Pd^r`EydV!$-`=CgCksO>47x|7ZqrFck_1qVh0*BNjn*KOq*&SF_`_yVw`SxGfox
z(EM3*T=Q+GdToj+?K)f;jgRg{tj&$TU&VIimkQ+f8E%P=#mIMF=Vu$e95KOj4yCG3
z8HcqKm*5b_?3g#|2_s)-l0!3A2H<=e>5|Y#(L7k`;CH+5o({TlwwT|Yc}n%%*KQ^P
z6^kxapr9WQxoXpU+%z9nkJ&D`0sr-%ausVPvOCVT{q8tbra9kTd3xu)Wz$@m>2^51
z8~PmYA<5u5v&Po}Hsya<H1!GMnM{6Ylb4S(BL$A2Q0l*0@{vE4wHGAH^QK`ApLViU
zP3?mEAkNY&GgDAnn`ClVk}z(&{3jgmRr`9F60wL2LoT#?4LV&O)WSK%-e{Bwf%48y
zjk>>DU*(XdS*g}q5pugIod7YVJjFC-Tv|`<c0p|q`+B^7rc}&Q!G8IeS~tGy#iF4P
zft}b3WjgB@R)^D?HoYvfc21{TigvZ(*am(W+U`fBiSNx;VLhTR%u*<NO@fY6DF|dU
zt_2)E<S5+IM~M)N<lm2|_q!!IjPTp3=-Li{$>=lmVD+zwJlHCbr+$NWoS9!{7rBeR
zc5T^cH;bLSDw%tU%`7|b6c%GY>QIn6mP!!GD7f#`3#*_Za@IIW<f4OeJso<dKRG_w
z@3z{GMXymrp;SOOBg&IS^Gn2mpP2Hij0<wu%dyEOlwX|e%vmSXlJI-$sf}C1nl5j)
z7M<Qxy!0JRR>GSxZ5Ez-^;~K0SGrFJ)qYtJf!v+JxICX`$>x*pq1?PpLg<|R^Oj{}
zvAYl5i8a$#B2P_CUS-MHR~Ux?HNb0ALH!->NMsW^x#d0&SB+}P<;N#grJlzPm#7c_
zfrqrNGj#vfAuyL4zfV2r=p<(hn$dL7dLTAmvpH9Z&U9AZgyVi7p*Q^6f-(2j+{Wg_
z;2bM|4gT*)xr-mtobS|@T278f^udqY>S5+U#dIZYlT4rs%)wNW(-2+gw*$J6V<vaI
zAt#^9Ia|6s<cwTV+`FV49Pcuk;}Z?vuDJl0bKePPpnxUH(ip79p3fIhvD<C@+%87~
zDuQxePbJ@5l<P=q#ni{n(z4;^lhWsLXf;19#*nWuiP59)f{#+KqPSOuH*J1pfI}y>
zX_k`FlTl|(ZRB|?rcNBl7eGYG*RMH}7W5|BXSjL0%ywBo6jrZH!=9`2z|pbrP&){h
zbCCP^S^w~UMuQohal`c$&2ZW{PjP+Hg{{&~P0HZipNvR_YE}!Pn8YAHJct?%RM%Y(
z>8@Rs27UQ>#=9(Ex=4TRvvzlQRvNP(JGN!_xvzSrZLnj{jIdPGv6s8QZmlJYxA)aY
zVboaJ?b+Gq#)<{P0dB+^CimNm770Q1ORl9|S8;d6VFq(KFZ&OkR(zF)5^X$atlO}T
zQGCTceZGie>uic=zmnNq1JBqy$r5vOeXcTt;(qht6O?g<d4aurVfl<*oJ{}@$>klB
zdhh%cEHDU~N;JuTBGu@vT(%Nl^}_nxD%rmb+D-Gm5u^*0OuaiBy-kqqMfbfNV5Lwz
zH<oxRGH+6ILObhIGjFG_-9MrKRM`n!)lIk^_<H_LL0qdch+{N+AgZ-GY@Y5-yvw;&
z34z`Ht726xS3RO}(G6?OnwT&PXHq+MUcvHlmN9+>{=1ge3PbZG^O@A=w{a(zY^yAR
z0%_FO9W{s0Tq^#R;$|`s?I>LSKCbl_R^|$&b}Fy&R_pnkqBfS9a5<8*$q~23vvT%3
zt}I;V<Bt_Fw|ErCQcP$zi|Dm+RQok^cGNW$>;^aCq1P`HbLjC!iAsa76Tcjmmiq4<
znyyZcbNpsaaILeQuK8wvy1?xb?e59ei2SL%C#^2~E(z-YNon@gwEM|UpQXQzKyJHO
zPtLWQ@1ZSO(e6c^cQcDrv7;yr*riDR7<H;)P~O6IdO7cO%3R3JJsG|=#A{k_J^Wj<
zx?<*zkyq7&xeGinG{q*{6;5{2L6d2DbbUr}tk;|d?)hE;e3@H`ezY^|F#*%%T6gbk
zg~==&=f2Syu5*eyC)dEoq|yRrZo7$T>2Z51{m;MJQMc-QOt+((E4c`!^zNobJG6D%
z@y})1KA<*nT2@6&qr$rKB%QhrUf|+&2%{F4H@d}qbG2y`u8etUeop1cmj~rlk-O&X
z6Hu3uoml|)eLVR<Vu~TKmvLQt?Mu>h#r8rC=kQZvqR-Y$u^02@f<esZTCPFe!l==a
zs`a@}h5S_`rW>b5L^esa`v<5t1FFia=5#kjTT4cU*tVr>X$Phr$&m<)%!@b8hAo#_
zRpPaTEN!;Asi=6`MiY1Hv8Y*%YilgP>0KsPX=0D_W_ms|-7Wr27he`!!#Y=)A3_<x
z@n>W4HY9D|dKubS^u)^Ypii?muTq=Me;QgWpL>daZg6)Q^x3zd=Sr=7#cb|>3m45D
z)V(iS#}3W-ThnUeYBvBbRZD-_e74%SyveiRUV2nX?)=@=jD|SsxX4{@loLW$aRds_
zI$r6)l4SLcZIDId#=HpX7E%Yz)kXcL#RA)N=t}v@z|Y@UF+F*NZD+OhYnz9Q!(W}K
z>64l~dwM$?Z#R9scTx7`wl}9-waSSY_L<@W%{;g4d$z(P`2<TBKl|nCj^4Wb>F5u#
zJrWd6xICyPFYp$34;wCF?2E>nBLjPL82LBSST*|scT>M;ekHOLlN`G5y&H)y>fN$<
zR&84`F^TUuCoQs?;`%m+WgR0yi7{(ZDR@!GX|b4~^b)BcZvg5XZ-b}v>$yQ|@2Q}h
znWE#2B5G_zt+~eF+FZ<;CvMrT<&%tT3HmS#9P@S*j=!rcD~UM;GyYvF%U=>)7A#*r
z?Q?UEG%@dplBCIb=hG#e;i1gE(iLA;(QCH^TcWo1L90z0)1)`0?loP)|BYeYf9HBx
z?uZk(iXr1zVXG2MF@rPmfllvadR==Ki?Q+Mrj&X#!TG)Hy;du&ah3Na)@QT6s%7EQ
z0X@kJRRe8x>(!|e&VFb=r1TG3Xuj0%y-i6{?s`}Z17v@HDV)l=(mxgF|H(hjy_Q<K
zl_a;^ovr?yqWnU|YVeOHCMP$aQqm7c<2?9!&fnJ=u2ff_kF-t&wP(Fu?iMXIzisGN
z)_fd#stYUBS9d*H7m2B+ifRToy%3^pbmxu5dOYbiMP;roE`56StiGlG{GJ*u8H@2_
zlKo%VH;$u%$#-cMjJx+W0)ut!QUlQhj=uRmlO-yS5#>7z?Z-9E0W8r`--e0_jnBW;
zo7<tX4u|svXuO<|<C7Gh9!%~6<tB>pSOL`DmtT$dfIB<w(#P!0Znd**e(9nlZ}wB$
zd_dMaP*$DDK45ZpBjqk_aDtknki)0uY=rz{Fa51fA0F>obT`!+EMl3p4UArRFwwHe
zzU^&gaWngKyUgRVNiCmfxA?COhAir_UvDW4cf?)3-;6Bvh;V91Qu2OSrZJB9whG<1
z;^YZ4mG5Vfr78WCrP%xV`aHE!KO@kQXr-MDsqo{4RwiCj)4dDL#c6L;Qmr|51%jjQ
zP=bhN>Sgo#T8!nB^Yg>bYs**M3a8@w!lyiT(f83)W_aErrto3K&kuF-uc+#PB8z@r
zeFm!&F=7zqs*|nf724&Zh#?nKs((NK=4lmf<Ndce5>DLm&k2>M0)~INhdrA-Ycxh&
zL`=GGgP3cH*lx~QernTmE<&-r&g^0=mmjS7iof4@`15i&Y0YY0^yj$g%!^s&(GtaZ
zHic@wmy{zc>7j3aZ1CJ&?2bp1b<7ap`>$=@aebYa4y>{Mcp!A_5*7PfvEXp;?);L<
zuvfaGG~d(0b{@4%OLyq(R8U4dkrUrK)Txlj{<kw}M>>W_bo!^bNK2(|t}g#fO}5@*
z)>UDXKeQ3pFnSW&$aDtutHP4(m^-3V&?Yc?3NacHU8e!E`qgFXj{m2<w~UIiYyXC6
z2`Q0~wvg@+=}Q3xl<pV>X@(AoAw^Ic1woKhdZ?kh1f*f;M!Jz2V2EdrKL7Q8c(42Y
z^sZ;EXRX^0Ea9AU&)!G=j^94cb9;IuoLx%#T=9f+nsY!K`d?KDZW5wqud*$TE^2t~
zQX?*wd^>Wqd+REcEx(^`?IwM<6FawwC^E4iMi$&Enwn$VthN-(+wxgTs5N3Xc+vP=
zSfSk4Z!PgfRiLMG5vV1S|EHEvX%in|S%K+B@{^B$*(bI-`Nq(F=lFI<do4;NY1g|d
zv9%)h_p_jPQ5%h6-c?0>t4hS*)+`XPM!A*P%&Uc(Lw0~BA9*ziZ0ybS9DYu5C+8D>
z{)`7-3l{>{B}QIzsIzIGn7pz9uDlUtO#W7Ec5%LrAqt|v1IiOavq+zVAFKBJV-O#U
z+2X~jp3)i3^@pz4SEWReJxAWQw;JuuAR_)__Z4@y4Khz84_{h*#s=>xin)HVItL94
zysG-roCh|W@>M0R-t8P1j@g1RVPCuFN-G&^LzyXg81z!>%6OYjB&v3CNLaoHC1V!u
zIYH0vAM=D&VM=|>%p;Qd7qi;MVJqs(QY?-jz0k|C%)amf{Q|)2M4}*mKZ<_Y7#1N;
z!JbompndRqfGsb-?pNnf@A2>St-<TRGLN0n>G38T1ld}#xi)m{><b@l80<`ovcHna
z^s3rajD-}|j*G+2nkD2?h;h~@b(Pu5Rl<~_=0y9ZU3ZA&(w=vB>cB^jB`>EtZ5#Pn
z%1zaqxX)3AO?^8Z*ImZs-*`$H)xR1qSuvlgPbG*uc!s(LTOSxFTd*DLS6u|4=WfJe
z=PywZ{RR8H%mN3*zQXFN7h{EHuygRg2HwuwN}F%&QpZP*Y-e=6PGzn(FBm*=R-%Wj
zI7sYl&Z4gr4yT$WQBBaDlp-3sjj@NExhhiK@ktGK`kgvB>S=oB)ivf?_>p_$Q8|+I
zg4;o2*<X+zyRW)~C+tOZU$qgWxbAB4KjFwL&%zmbFh$H&)0jUJjWbKr(No4lRGz}=
zlk=lEaO!(3C;J*YCf0!4#`A(uQ@0fTRXS@pkMxV!_RnW4Thfi?4TB#ZMa}A^UU=Mz
zQ&l(Sx4NaVkE73Xz|I%jSiw1g13c$@!KXi-bM61}oFSaN={#m{pZ&T1qk<L%rSsP)
zbSr8pQ|Y^+eO-7`wm{JZ>Nl>A!!l<+v&_!Id}Zjaugr9f`Nl``3IFc+9aL1q>Sp41
zU%$H3mXEwm89VJ|q*4&^*Gm!d#&dITj%8g|?>`QNartBtTf#yh+g{f>ld}td9vVJv
ztp8NT16{p~_8C>3>X$y)tnS8c&f}?fh8Abm<$m}J=5J){i1CywsH2{qnVFbQ#Ld{g
z*Pd=A!WW&a*qsj)-Fb&X6Q}n1bo7|9=&dPe@NxvA3nLgeu_N1Q266YCpTQp;M(Yi}
zmRbt+QiJf4%Rt1Cx4j*HNi%z0_9~EHB?;x;?+<a9usVf&=Ck=Hrzw`1HEtvFdkH!t
zw6-sbS?5Q@#L#$MH#|G!A*cV%n?pw>Rw3qL24zo{)7=jDl*oOTG=7#-#HpHPb3`9b
zgzVPrv`q4rXqjM@mUy<qvm3>kb}dC&a}CICjCFpxrKSCPa6CEYb--I3O&aP-Z1cSX
z%4Jh?so++Zw_TqgC399ArR>F@Jt}5kQZ72lGm^Ws?{q4>dI+kvm1)TS*<=4}wtjkA
zXyeC012wccBt7UN<~Z7_6lCPw$V6mGdAGli(dSQ>Rl{eCozLUlc$i0MO9|4rRfz3!
z2$H(EFVg5Po03XqQr5498Z4jCeC$%5V;XLGJ*UYlcu=YTjUUYh)1i%{&+qYgupy+0
zWi!>#Q5!F(Zd_K6I_qBP`iQ26XMb_n*w998J3j+31pyB>L6lD|6@J^*Hx`|2XLS0#
zy{uF8=F%@)(zI_}=+Oyv-|KblF~RIeKQ_ttC=TL30RQRlJ6s5kuCO6leRjTg+m+S%
z!(%RExW*ox!FOPzdQEYtYssF~Tr4E32jWcFU16@t6G}(;!^<@_CVZcV%_6<})Q5!n
zTgBGq8#Hatts-D>S5OqsIK(&y(DCH+z7;vPuQJW&OwTpl>e_^wrt;Wt8ML>#ITWNt
za2lSb_zJ)1nc|Z%n@v0e=iKJ*qUrlR^d;<7UTNr}h*#FcGgJ*sj5kmlGUJzQle4O1
zCKxpI@`0x&_1}s?TfBT^CZ|TDdSm#pN=IBxl`mc9vBKaYu0;Q;y_ffO$qu_$9o@HY
z`^$FIJk1=in!qWq^^nxF^;|7Xms?APD&K_V>K4~WK%J@+Nc-M(Bq*H`j%++Ji|VaZ
zGt?byx(ZdYsfGAH>L%#3tlA~opeY?HY(B(&j_q+FtHEVfZf9+!;Ww10l}=1dtR>=6
zoSTkr-5;Usv`JL-RR6jOAJ*aEQ(C8+n&AUJe|v?EN7gf@jQ{lRUXp*{Bb$iQ6KAdo
z69Zoc$lz2?SZQR1VnPnFp>JY~F205E^|zc~^L6jwyZ?&i9LQJ10kS2ok2|Nac^_xH
z7#X#`PWT59K3g@a$bnngeWI$^-eRUWy5;p<->+8c;8+zVKI_`|TLC*sBWATO?!+Gp
zN1GN1A?HkPbe(MPCH#ddcStZ6&U>Q2mar9j#|)1mM-yoIkyHtz&T|qUt6FyEveqy5
zM%@R@R|64IOm4E-M?3Z*d~c=;E>KJ}bffCs{G<UZH=btJu~Hz7Ec4&(tD{sO92s!Y
z2`j_uZ0l~yhmUNB*c_fU+!GP)5QRm;(=*L<DLV^g0YGwh`zYniTyTF=d54fy&F^oB
z+}Jf(38j!_)d%x<4g1Ly&`!9PxY%qwMPKeZM)$Fn+0$ZnJf6OAVS%AN!8U|(Jw<g#
zV9#z!v){kn?t>dej@}<yQRPQ=W&_ReoBLy=QeBdwmtA$(9d4g&IIH$+7!p(J(Ftyq
z)8!3h$(eF#k@!RG!W#)ic_HmV<$3!ECm;p6fP#3>PqZR$N=m!I+_b@D{#-G|FJzqV
zmwdA=Z{JYEI9er0uW7jWI*-dts#0rc{Bwvgf4bbvmul)$@&JOrfZcljS#ISIzl&vB
ziHBp}D+~<56?f;p;M17&l-bwn<y}mUC2@2F$(G}RN#wcN9W{U>OGePQR!%5n)2jcL
z7{aKyC%#}7yEva{k9^;WHyF^ong2r41Eo#2=%7O9tntSi`D!CwKI5wN8<PvW^dj@D
zq^T&Z?EgARFKo1v92+bu79;dHGWd>IcTjna%nwFZ9717)>>o^L(GcB9hK;vj;=iA<
zqbhJs?-~w5CdInp^q0#4GHNdAbc)-Q#;EG+>4A~USaXaMVfnp31YK5qA>lDK7@hrg
zv_iVN=|PW8AQt)P7yphtLe}VKf5n(sH!W_3M`0BR$ONiAJ<P{DqyMIr>&9w~oH&HT
zm6*A~NOX4B-W2Odc~^vE;goYk{~@kKi*%3avGMXA$a<>(y+Sw)mV#lJ3~KI->*4`P
zy~W)AzE>!p2v8=rIL!HX{AJnxO7Hc)^J*HUY#I@e-t#6iZSZl{$2iuQ{43}x=ad5k
zJBI^G&m$ha`pm^<gQKkpK`^1LzjB$ZGnudjvP%I||9gd1P5JNLHtQc=LMUai0&ZY#
zzc1Qfz(xq%is((j2P~Y#n}32p%+KzxK?+y_Y=mki!q`#Tn2%p4`s27i2k?KSgK5G@
zzNuRcSH3)ojJ#$$g0RHcThd=&usQHDF9K8d0#pBc#b1`^@7=lIxdLJGSlD<<APiW2
z8ka<TVQmU3XPYkp(tNGKns3g$EYHU0K?WrEg;myb5KmGE^KrqyYGZp$dou(~^N0td
zwYk6%iAT%@+Wjd!cl~9de^WzF?`B**nED2hnu*q*D?laxKPX*3jI{0bnxqR5JP1j*
zXy=;^+WD}^Sg`TvxdD0eonTjsm`ly%n|}`*P;4O{JKx5Jt200Nw5;i0!QYU8y$1I2
z=l*~Dp-;<7@$rbue^C)Em_qhne+0DQ5w_h^P;URX0si;jqTR4@1oRtm9{T>LQNZv|
zRs3aDOw0Z|p?{b2e<k!^8~U%8{c8yN6Px}UhyEL>|C`JHlYjg-x&Ge@KX@K2=4xht
zxj@a$ahDHkSfs*^1q|)0&ATV6AI*IXz`ge6GsJhIM{Le#<!rO4sB7>=0)MmQ(XeLB
z)XsQG=K5F>WJ!|k4nT<iFMu|n6+jE|8B|GrIC!`=*ji9I>-D@h@`y6$8A!Dvo|E(y
z$S9f1s`c6WT0`rBG`W>wO;xBK0r!km!2Ni=mY!sBWI@%h4;GVU))9T#99+z<19BE%
zq!|62J>-$xKNw@QO2CR^q+j_l1ioFChuE>G=m(-*`YvWVWx@gE(kD&YWLL|dA=8D=
z69s=1)E`gsK*c6l@NQtB=R>C>vFIb&aQj0R-;;E8wWX`e-75{L+dQ`^T#iO`lalOf
zICiUc8_@Q5X>kNRxqUFuiIb+&Yr;U_CJAQEBHT^&4ewN$Ya_uIe!I7Kx|*7l>U+lF
zU<;ir6L9Ghx125G_DcXq9Ia0vEM*-B<!KWeU}%C=&mdVH0Bm2+t5}|1YwL`;+pDg~
zWbS);YMIG!MZjPey>ZKdC?FvR*M1}~?sTVim$2>00zWBOagHo&w|V}XJpG^Pfk@=M
zycV19>GJ0+<63u8lh4!uiTmPLjp-KsV|YWC8g+J0>n9a9-eL0kpdw3e&<Qg2@->a2
zod7cZ@ii2q4LFxD3x+=S?0&u8N-lH$a#YWZbPV%=Ag^@_>{uVMmJw$0-rr2Vi3ps7
z&=efK%tCQ>iE8X6dXPNdxBr-2>b2;kRi;+|!+->>>Iphv0?}V&<GV*~be*ksKGscG
zBqd&Be$&KVHt>&v2>mmlBY)a?6G%NV^pX*rT8cVbiM_`HO46I5#BDcug;3KjNf*ac
z{iM{zdo3iieSK}KM!c3L-rFLG7Gf?s{Jf`xuhjvHC+PEv_miavec#YzO-Z%gqJ^m}
zHL$xzH?*B|4jHv;jK&I$BhGS7UIySB<tHeOnTZ3r`C;!IO>b^qajp`AQY{x}LAHm<
zWBS!fq9rrxt8;t!t(95tT@lorpI`BNUgMgCXCK)T@oA0t6WWu`sMH2Hw$mv$R7<3z
zLS<E$zK(^k_>M>5@3%8JBmkw)6;!W$u}Fee2+a8$L|`mhVn{BCpqkJpw^c`au_H>%
zq5dfT^mhkK?~pPdRCLbA1(~nQC)B~{^2tCX0R>zusl=><zCO=A_X~4j?M{uw91JCL
z&?^QhHd<hiB0TN)1AWB$uPG6$Dc&`14_+i&m8o*J4VSrG9F2)xov+ID8oBgpSJj{;
zZHC#$s1v29ox6EfN;;Tbkm#dL%SUb-OMrP_D(!o;c@pJWRO6Ia{jK6^CCQ=2$)e%n
z*ifu#yo;;I&V3H%fs%Ik9!MU{W4h$jFNLbIANn*V*v)O`KI>_T_}$JxTX5Q;I5GrV
zj?<lf8$>QWp2H^p%4`4UgNRFx?Z&IiopV}Vs>|td+3=bS&$IRH8QS;j$ZQEJ5rK<M
z$d!$bzBx5w%)~oxdvtl!(7|nWXt!TOnP<6>2bLmY8($4H-<x05uu&-Y9O;(58S@t7
z%J;CD`+hqX-ZzfgBEccNceklN#TzE5OH&Q%aw(#>xB^$`xvLK;p1B@0fD%4gjGB4H
z;@E_3(UTx>^rWdo5Bn=biVu0fb>V$^x;ldpbfh38p$H|i9o|Dr!_J<}c8WY4CzIHs
zxISvaNs-z7qpX0yIZBCGkDWCm<mCI%&j+b?FJvxU4>pS$X?U{O63*l_=_r1aa1J>B
z{y|$iA9ef~I}J7K-QQ1Uax`$2Xqh75ciIgJw7%@EJLt2$xs-Fy7<egf4pn(Ur0!Ix
z^5pR?%j8$XW0pq5*GykLTE9mds^OT=zg|V{AF5QlRT@@(x{@@*jbA?Oru0gKN*BN3
z)2!zz6B5`+i~PQ{m?%Z2e;AIb(=+TQmwu@+{Gea)`felI#%VwLbt5RK)u`B190QVb
zT(Z;Xh%hnZoqtz7h-N?4ex`eJiofko?>YrG*3DjAw0k}8q-4f@DiK~W$yq@#QD%+o
zfuUwrORc9W;HMojmy~^iZ;Axvm(yc89dky)R59Vu#Q*A(<_iIc&+p*;HU8I~6pea&
zHDLV=q*Y4)U<I5}&zoFLRYMDjp$lBW-t|t@clqiMH8SUfy|+q27-;2Y4svwrIB$bz
zVvyX8=lg}{n&MBveUmrXX+sAGfgy|Rc!td82O=7f?NYQzLC()M)q!O)?`Tcl$T@_?
za*ADD?E3M2*?F*bQeGG?+~%M19xMT>(EqS<#@$fY$uO?Zp=yCZS6#1PNCXB_+Hd`(
z`n-&4g9r78L&}9X#VPRIjLPH3jZ!skoAH%C)7LI6284REQ@pzHjit}`9v_V8LaJ%h
zf{hu;nH~)@y7UOd=&x6gQR^Bl><>769EBU!nrSb)ZByk&;%|M%)`%c6f*p-`DQ#a)
z_|mw5$DKPT2)+Br-o_!GF9doTfM!F}s0xR<hUU`aSAKoIL_HIugyIradi=04V~^g6
zkozT`;S6HDdZV+fE=hK!&+NfEx`9S)nh(Ku3RpUr#CX0%HA^2lZjRFki+exrg@a~6
zIp#++635D~nMECArW{*IXJi;25{|A<SG(}f^jC@<cS;Hm66j6WRQ$BE596tD%-V!k
zn(Tl`;u4+h0}YECm!ttItTQj46MY(sfHb*?KBekGx*D=SSur#0!Rwx2@Lm$E2v#c&
zhwa_0R(X>F<=l)P`z?HLB1GL@ib1U#T7NVuK6S=cc|2*m>N>Mdcc>~Y>Q9{nA02cn
z*(&X>SzboBQK>H3DW|fMPu<~s<G_;kW97$n47K+x0GC{nzl!$MJK03LA-CW+QYj8D
zdXj>G>;#!#Jh~hqxNtdEd@|8m+<ac=4TnIAY936{d@k`kDIFnr<gA(fqL*u|^>kc`
zOk5(`={5LX7Vql|l83W8&Y?Z6s>8o0X>v<Wp*8DS;%Kujz7*8i@sz`=-GGeV7=iiz
z%4REYoX-du#~Hzj+LC0rY;<Kfbe+BcQnjm|y4W>ar*ewP?_5pwmr<G>^_S#fD?XOC
zPp>UU`bwm@Pg<9lpS=oU8cj}O;{)81+`NMy+Hg<I-mtCBe(?D14Q@-8Aw^zYzD6cj
z$IDaoVU#Tc|9eSanG(ON%RZ%wS7NE3N$sjvy)O}dOj2HD{de9-o$s~cdNdj><in?_
z(dbsoZ*z3pQ<Z#)pHm>bHES8-7wrrJrf(r4yV|E~D8mKmHAIR48H==}gQxWHXMY>>
zXdAsI_e5&R=z$<RLtC+3D`ua)=49yY8|MKXQruvKut0Oyj|k%@_^0Dfp@km1v(4vK
zIFBLxP%g^KWE|F->bJ<^SFM4PH}&2tUA~HGy#4jfulM56xd-@M=aN!{2xIqC{g|SP
zA4<h_P%`>zmFYH+EM+Wf(Onnk2mPL5<5nI!AoP;&$LV6ink%ziL@{ghcmGiUZK#(r
z{l@^3)c>TdOx!kZfG0j>Z0LC1*AhGp_Z0#63N%x1Dl{rxMB#N$@lH;A<8RraT<x{5
z(xCHA0n4SP(g0fwY~g4*cRqj?Dv@!bTjaR$XQ8&!(?F}K;9O;6HQpSR_sSJIPQ}TA
zXAAQ&U0aSnBDjj`O4%6<NyLjx-|;!ztouzZa-P5atK1642Jqqqp<_|M%;)LN1F0{Y
znhz-!SsEtnD_km2A=Bw0%wG^9B|*;5$rqHgY3vGNcK}eak`7S96<83oMD&V=O$Y1=
zl^7JV!h;wj-xQNPWoJ+K%VXS5UDc)!{oQSAEM-Zoi!)jmr(?ijSqmfKRLTFP|EiQH
zY});(M{8AyI#GMVK#nT4(FDqM+M*rQPn>~XK1!0K4u?8OSI68{tohJx0HIFF^!3I|
z92aAO9jcprf4JQk*<C?KoRYmkw+m9so=siRT@0G`6+HUUJNGbBa=)`PQ{AzhkBU0<
z6rhS3(#PH7D`q^FC@d@nr)N*(v>e5!ie9Np=hfRJZCCqjjSI;ih{(aY&fRZfeNd{I
zu)Z~;S2ojflh<|l&V9~cT&ld=Mw}w|DKt4DQcw4p&jgu1`>-lN%CvDS`G0ET2YVpQ
zCU5e_lCU@!!Kcd%+HP(eSq0kIpMYlDBQ4%wT!BZI%khT1w1rZ?Si--$jq3l8G7Eq*
zpBBaj*<hmuRk7*t_hA~g6ONnd-MqCu{hVIf{%xyN_P4$J#8>f<9+TGD8AlmxYa!|T
z8k@In0OnlL-&+cGR#tfI$f~d$VUX5v@+-_UX%2v%p#8AP0Z(Bc*|Z-dTJ)hM#!T1z
zR^1DxGZyp4h=(p^w~KL`e~aCtKrBW|5Kw<Rx|qEpP~-28)ZZ$toH@FN1Q5o%@fj+w
z9|HwV?mP4j2ysjC9r3JFKjkP0-4-gH$;3a+4|ID)Tv=FdoI~B`o?2eipo@h+(=UzQ
zs#EH#=)PoqnmKnrkMMb(%pc{v@IKzUHGSPxGfJsqZsKKPj;zW<H7eR3^@a{+?}Bbc
z@+p$q>!<TU<bn1BqsAUHIWvyPovOvimk_(g5AxkWicT{HZ)<O=A2e%l<boEg+{w}g
zEcUuVG;BX0K_lS3rmB=k=L60)HLd_UCWNn6=Mxe9uJ0|f;RY<hAB%XhzVzS2#N6lD
zoZ|O1e>NYdGY*?-p3;#upbyXh+~Ae>Y6)@ka_~tggmbwJ|8oq|UvIAMi#cLFxUKZX
zA+l=ZAd4#W=i)prP9o*JNq@aGJWPr)R9zKIotnj=@#1)1Y@zY;sHn_6VjDP?5Frq}
zy1dcV^=H(ot+^>&Ptl8ZDy$0FVS5J^z87cs?-smsy)=GT*tiWvf0K=Z=<=+ReDp=E
zq29%-i=owjMjqT)de!0i-r$}Xz;G`ULEXmXoXFr%C!p6JIYm=_oyyh_6avuH#x#hW
zh<>>8GLAq8lZVk>NTDyyV(gg@O6-9|llH8sjp*uX+{TM1&D4#_Ov%!G<glE36qyW}
z{f`$HJ}H52VfBEAp8HGF;2?w5GaD@~#tV`tvm3m0EojYWtSfo>g~XAoX+dPfMLV;V
zZL;AuJX6j0UFfNvHs5vl#o{j_Org0WNt4cx4M5gYHGd_tpIXj@WBU>ZJwj&vX<yFr
zlXb7UzH6L>eDK8zF+@HY(F*GhlT}H+j-<?S_*0J{Yg$?Z=q`>UmGVBV_$}r!-UFN9
z+Zg!`Ql2)tM8FxC{i-nn6q-no4fjG4J%Gp=Mbe8lWjo44w8w3pYHWJ>nPjy4j|$5)
z4nH)J*2_$%&(Nq^&g6`dzi&)+M*3~rddorM0+evM3ody{VU4#6>h`-z6g7A$GT8#8
zS<2l9S@+eiZf^bVrY5j!i81D(DsRX--Ki^6(Q3+y1?}}T&7dN@<vsmn5HtoFoLZ^z
zk+SVZN$iw!1SBXjd*17m;jEdB#SuVet9mpGvN*m!u(D!Z`M_1k?#Z0^9#a`}%o{GL
z|GaHbUt*r`t8(JJ*nr*%>zjU|?LWsS=CaZ=zAS7OC3Q>~f3*1o*2o?QN?(mfJ&*s?
zvJW)VBfMe1=Tz<oBsd1k;|TnCry`ejALHADkKN1n20U~kiWte-hF$Qj{dV%0LP0qU
zaaNT&(t7b2C)s1xv-+c|^n^C&=pM<X#}h5qmF>zovX$M6Df>&ueG`#z|Ngs$+KuGJ
zWi}(6*|7J@c{v4jqoC-ll20&c#lbVRIa$hu=wF2t23Rj@RLiQI$TjvCO>d5um>r0U
znYM+#gml3-R~2E+1;DaM^N;=aImS}G%~)u1y{V2S)=}DGO5#jTZIt38znC7mS``B1
z(tMtUXF6Qd>7Ym+M67vouH_P*09Y7;7*B_}T94i28pHt`afP+XW8nB^#MB0Rw=ni)
z0PKtHv)26(c;)OP_T6Eo$41{tBXn20e`sJc7?W_)PL)|xaTl>X%qt#@e!`k{+kiVv
z76x+Ms!8ffoY6B@w(~T;4G-rp=<W(oFu%fyBQPCCgkD9~g6et+`VC`cxNsfTmD=3~
zm;Sw$MIrQo+6g^qwF9p^38GiD05$Subl}+D!??%qt`lXU#pq|{;NIuQzkIJ**R&^$
zIunn^Y^VaiBu&*6X+7|3+hF731s&%(D>?n(9XFTa5sMl-sRm(hKN@tNNQ%z5B1<>9
zt^g_!PY(8mfcY7{i6fTfXa#Ut_&8jK+08IbdXwa1#<XuDkXK7DW9Hd=J79B;LR~<I
z(%A=U2|mwpa0G${eZc;w0cBe3W9K-=mKm(meZ+Kr1K%elP`?@tSR(d^$QjFpp-=Bn
zASrm_<<q3`s>WQ3R1cKcC_ZM1PW{6Q!{l|Fh=oS9eZ%RBrv$z4&f;edlmR$3_aZ>&
zF2w<$N2;8R>%JS@gH}vfh)4Wz11YH*>-ouqi4p4xa1SCJ4&t>O?TC~xPIi~$3Cq=+
zX@jC#>097^RYfvO^CIWxqr1x0y4zg2mw40q<quM|GI89m-!QXvt*);$OQf4J*BXqG
zygXUV@<#30pQXtKg)=cqJKSO@;D3<vCL_rLZ73}2l_b^}OYVN1+Ki>RO1>P9$sVyI
zAL2{oZo6O#(e(<?iW2yX5Alhj&@Uy%CDrTRJJs1Gi=S@tEIt_by%Vv$I|%bxOAqN$
zV1O=J#`%=b4AL=JId+ea3S^meM9Ozh1tKwg-Sf%-I<s0gE`xRjM8vkv4-5~dfM$v-
ztlL*IyLRA=u<O2Imh^3frvjwwgd~TCiRTCrzldWvKTwQsJ57td^YrDru@WAia5(i#
zQ07ZzZe|*t0sbR`M=u=u$kxvGp4l~7(F%VKgI<F6a2UIV9c%kIcJGIkkOymefEIis
z_Ol{X07QJJH=?_k-S|AhwOMmR!y4$65?MPL&aG;e<t`nG=x-%v8C7o3``64G_Y_yC
zQmld|6qajL0l@%Q7tempB~CZ>pd^(VZR1Cmpsi~x;^2GVd=r?QwViIVz0^#1s^>~#
zon#9Oy<r7-UwdKuvEO#-5Gtn7xb__)!e<U{C}vhm{vkQLR2OYyXt&Z^c_GMvjvg7;
z5%WYYN?XZbp4{rIWqb;n0j_YSHzR8OoQqUf4RmiUc1vE2B6rhTUbC#+ii0Rs$cI;h
z;**4SK?o2UY5}C8<4uxhYnk@DS`BNSPqj+nY~kDLZM~(7xYzm{gws>}B;JGp2r}oo
zHqZ<!<M9vQ&X@(jXk?dp3YJh3R}jA4&IQgf&e=|HhgCF3%%gozPGeJIDs0F4c!azT
zt?t@>Ki-~Vkn%RL{vJ1Nd6i1qNhh&Wg0{jHXeYZhZ@FAQ`I{qR`wO>&!ATG_(ZSBv
z7YQDX-CLTkhc1}fz3hZl#PtTQJ2|W*n30)rJ?ZGR@PZQ$L+m&V70VXhF%VF?F<6C4
zo_O!pi|MsXpRgOSN%E(`=GBE8V~K1iGju49cUiNYEa)eBLgy6j^O$atiaunPFIN7(
z=Wh%sF}`vue$|eQ#Q_-+qI1BZ1$i!rE#y>>zE?U9_|n}$V47i3YdvLO3oWjna%il#
z5@!e{TA{6YvQH9XtD0k2?)+166Uv%3onha=!yg12cqropo}+RSR8a)6;6^cID9Aen
zzC=;IxonV>;(N9($4swPDzLr~!a}xR3`paT!G_tfrP?#9XQllFeoFTFCGh7WJ74D7
zK@p^6pBbA-&y8Fa^Eq)8OZ$E`9c^!HMXC8-v>Vd^HN9CqmhGmEyu-$0T@fVfhR115
znQ8fY!@bK-xBhG+|G>QhsP6<9t`L?0;sqN&ALHia?q_%bGdI=P9fj^QfdSzsf$$UX
z%DN|_{wc^RfsHRJ#wzhj-P{P3;;%V6Lf=QtO)dxrMD!z4yRmTz@ccRfC+79yg6tg<
zMCT*~<the6h7%BKUVUnirIUQ)_%Lj~o>?J|zPq)ei*SG3CLiu>gQJ%q%sL{4nx>NA
z*FJeQcsqnKR_smUzJ`Zh&EP4F$L-_rJ|G~BiwkNP@`Z!_5+-!`<nwd;y&@PEQHzLv
zX)Xf=DggJfze?=ZA*^n>h0UYiYh6XD2hfNl9!|aACnvWlSmq>ZJGm(3$uYV*lb6up
zmuqI<PMsrn!`Do(+xAcMrektgDA%j<dH9sZ%_zA0;7=}72Dz*k@>6d0#c~>-;$dbE
z$1`&T%a&Ha%{WbjWq;qqq(if42B+8)C~{Q8eKTiEgJlRn*v@xYD6gF(YvXl1)1g=k
z2|_K!v{>2jlpm+N&D}heM(V+HUWKgN{YoO>S;4*8+9_tz^TMTiog1M<9@aL7`%7@h
zdI(_r@X)4)8(KB=-6M%u0SWXn>0WCTk$HgH+9`B!#eb;~$(U&9K4Bgt<2vJ$fg^C|
z&35^ib47RlOZ_QVzU-Gl1f^Os@f!zfJhyKOhuV5(%0@U)rbBb*<fW3w8muG}@(XJ?
zvtP!na7?nqwSW%GN>FR8VB$-%!Rxj3FbL$JYh+ye5|F|=eWI%_Q(6OZlzUOqXPr!+
z_mTcq7`AM%aI3hLEFhRzn+d_ZZzW@Q(D`zYdpqRP*ze*<-qR|O{Eby2cy?4MTdwvs
zyeY!2WhGM~;zQ44<c0fE{^dT@Wh~j?(FAXApr>!55{`hOU5Pw6Me1;|$z3cSVcb>i
z!c=}2Fja)3gKq>3bmZzPp~Db4Y8JJ9G$rjl&EjWcK}FBgmZ3;*vFFIl_LTRNELpb!
zCGw0&?dL*3Y*_?+DQz@!;RBh|a!SM0C;C-+^PH=l%u*4f$b7q_<>-dWClSAO-Z<v{
zSQ%6I!_*)DraUdycK3r6%_80n@pivCMhSPj!N^ezkSiV`Y<ugSP%;yBYD{F3t(uxb
zQmIwFT`?61!i1N_a(^lgr<dsXz_&TEhCBL6qEKhPy1jp1(`@9<_lnJhCwH_>f-N3R
z^y+|AG)nAk&e$I7rc(f&fXWwP4y!?mxTsY3HcG{|y!G2?47i<6JAV{hln0}aBgPd7
z-aPG}e8)Vj6+3)C9KyFT68T)?Q!mIxl-Dbk$9`io!A5&E16XCA7ZKmN$;J|@p1$c&
zCEd7}U-KYLxuVgF7LSCF#dod(YDinWj|A?)9{LM4Q$*_v1Hd56T@kr;!tWWsAStZM
zCe6ZxAf9ZwNtShL-df}&2XFB?!E7_%yDxh`WT+G)WFx4|sa%*wT$mR{UKd&3NxFee
zUQW6avCY>TBWD_(<0Te>rEDAC&#SuoP3pa@<6^DbLMi2ZD^x8jS!xJelk<L80BZI#
zbXngSx915~01XG&vpTe1Fo8^F-*zEt6MhnAPMISW;K<q@iM>VOJQn0F!K~63sd&Su
zf7pdwsx4~svSF6<X@2Ra6sc1`KrIt;`z)Kts5Vlz{ERGFkJk>Kb0n6TLzuPUAzGvg
z*F1t%HyXxe95fu{^2)y3|I)(Nqb@!dl4uO~T6wSJu2Rm*(pd1$ve-<c#?PH`GudmQ
z#LT241+tvqHQ@`BAj`^MgZqbb7Uv7raVl*l=z<LcgAK*6p^p+2ShL7pWRS`EZMnb9
znI|{!8+RX=nFa;ymOaiTw%0Ij7=M6)zEv`vpV?kfp3@}4Qq=>Ol#k`eWYDnuHId9W
zU1_iRj3i{0)8dC3#)DI^abQEVh=!uj>AKqxt+b|9^^-i_(%}q6u}(*6uOj<C*Sq0L
zXDxN@^tLzQnHJv~k_6mHvUqC`2iWttMEx{n_L^{Az}_fg^RZwZx!@2kbGZUXTp3>t
zWysiwyU2?OgKuw1%7FJx#<%yZgOb4{r3(`Lu_h`^iFh=)wgkC<S@$AEasn>sI4?^(
zfWTEyG*@?JTreB*#u13IQvvGwgx|85Pd%G`n5aZXTC_B;n%-5=qVp-ZozpPGpg&Ir
z+F0N_t`JwBheyGCa1>OfrNIm$a*hZ$ApQXu_lB*z(EH9W@76Dj<q$)q6?cwI?yP#T
z?Ag1gt;<-von06E3dy>?m>gU>&x1p#^^BD<^tWF}aFJJ*&QH4cM_O_QW;bXbwZU|3
zQI0l+p#EE;C1@=pUs#PhGtQ1b^qm7$0o82We7$c{nf=Toy*@t3NzI@F!!mfk6?K8`
zi2~}R#f@0TmXSW<nbIFV#Le}^NYUv*zf5ZF(Ws#@l~_c_(D*m20_K8Ek}?CY9C7o<
z9|K7?MpZo5v}dg2l)*+F*mv2X(Uk^psL}Qm`Dw+o2D@>;i%VQYL~DEMmXJjo!P|4+
zJkOkXWR$R5`4K=9X#M8=JL?<$h@o$M6C0H#y*Fe4xK`42!4@H1`_4K*r7B5zx|jQ_
z#}}fus=cZ^7T#{HuFT5hlKX8_Nn~jUq=ICESe>x1E#F+S=y5UhNm@L9&Ju;+F}>l>
z=GB}oqOTGjyl0ddqn|dUh4MtgyBMw(6$6s1d{fFJz~W>&4P{<uba6KFd5uKfA2HR$
z%GnEvZjVtT6#@L!a5fg|?Bt-UUV1j`wo40-sg}ARCuQh+jaT$I<+mw7&6UuS*H-(J
zEoo_($)UQda^WugGx1PfkMT+Uqa8%eHInSiDCtRfK5^&5x1ArKLU1_de9=&T<+vi)
z>;jE|f$U6ZZfRBOE`-`gz0%2xAS<0@_#gR$Ln9Gh>m;#h$H(?-1L=htm7MNi?|Y#d
z@E>9hgt+(~zhYBbb7Rl%gmG(>spc)g#$SbK57#>zU8*=t;9O<TAZY1{0%&eK?loUy
zK$~@LgC^007hfNhhhCUI_|w8!%JDMjq}`qBrJe0#GsCmQ5go(caaP$(emiASGnO(s
zhGDVeki-(6>k)9Kv9!9a6ZUq3*4~~(p$Mp(wVkr2S<;WsJ$Li$w55*~-<34GRr0y3
zRL6R@^_Ib(g~RVH9RIBNN8niTQwf|Wg4C)F9xFDZ<IaC(iF?2AF>eL6!>pjSZ?Q4;
zE!b6b5j@dy$+Q;hxeM($4X|xF;0@NUJ4fGP31OHRds{84ktx<OPgqsL{o9|CDXuL7
zPGPYXea$m}vg=yAdz^(`SbW`jobSMCsTmi8Vad!zJA%4CoQZ_Z-HjpIsWJ0V0!a`b
zzK!2;aT#hHaLetsENd3;3$qKxC&R57XX1|{@j&Sg%9lfYc0;6?Wf-PUVG;mHB#osv
z=KnBawUnh)-Koj;ao)oSsJVlks^F;*iS&fi$KLWWlg+G26UT5Uk9IL03xZdcm2Fd3
zmPEqW{)A`iZLtIx0PIol$xav?WJlv-x)ZuU3k08X+uvpkYC<sf-LIeqv;aH(_TZ%s
zP2={R`?Mz)9}SvvP<T_MUz0NWE6|8e^TY&Wz*J61p?LN?s~0Leku5JU$)0ey7|Yz2
zG*!}_3A2r;1_T(a#{CXlU{%xb-@lh?+4YGd3()WOza$TO%(m!w3%#w}%nEjTq)+B?
z%{nqgGZmx2ZudciVId#@ES5TBPH+7Nhk44fzUxktCt7>wH(~xck&NjPX1l%Gcg1j+
ztW@aFvIvJ0uw=4qw%(T1V&rrel<Z<S5!!!j>;oRV>i#p|iz8hG@`XgAp+mihpF&SI
zKnH!{EHbkZ=o<*ak3!r)T&)ip?=#nAbbR0!b7x(Of1|QR^;b3{HvwB9ax@_devOU<
z&Iq2EA;v5VH19w7VR-^dM1jTj2q6HEzhZW=d$%SKLBYcAD(K_w=au#|J=oV6UwR#`
zQT8m7;sgVlh*<tl;>Ku%a5x5ZrEILdc4o&+JsgZqN5}T3DMk3H|Nb}0&p1L5*il>}
z(9Co2R^+atrW=IkK)dghF+}2jZqR)SxRrm3Ut^5-&-eV>4~ozQ`u{ir=E98|p!J1<
zd*WYvb}+NYjEZMkMev^|0_^!MrY##!x#B-h1Pl{ESNQ+69$D7g!03pYV?pBix2633
z8xV^vrq$;^F6!xj#qw{m`L9_1)nxt;isgzte%mBs#Gcv^3;cT~|KdsEKQ9CS7cd3;
AqW}N^

diff --git a/docs/figures/arch_7.png b/docs/figures/arch_7.png
deleted file mode 100644
index c77805365e34413818691fb212e700357a380932..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 57736
zcmeFZbyQW|-Y`lEk_rMMQX(KNBE2aoX`~wk>5}eNDG5oDl5RG=r9ry8QxVv7ckFL&
zpXZ$Oyx+a=|96ag7z6fNYtH%W`KvWIA<Bx<_&5)6P*70tWnN0CprD{rp`f4*Vc!Bz
zsGvEDC@45k3vqE}8F6tMWd}P`3u_Y;l$RlK30P{X>qLHQ3rA5yM0cWyJczT1MP=cr
zS*{ecx9`Y)^8fb8rHw=nm-&&FhLgHPOBG7F2CJ4921%W5l%$wG87>d&E?Usij^J*?
z;c7CX&vk7;gbN{rNaiB*8~82rcsh?+G=fj*0p;&rS$SF0&li4pI^S^IXb#S5h!o$x
zz5n`I4uXBx7WG+?-4tHu#px+ReGKD=o-hhgh<yP~LR#SdD{K5c`Vk1qoNb>igjeZV
zU&2#U*)lXvpQnB6nUJS_`E;K;BhT8TF5eVkistOQjX0rvy=`i?ayVMX_uTi>b{pE~
z`^H=3^$rfExKBF{0x(zUTHN@A1&{0a)_ubg3xd0svsv7ZSm|Dzkgdp;yB{J32lt!k
z8FhK$Tk&)&@hsjw&nc^o64b5`Ct+2@kGyaE`E{G`d#<-F_QrjK$8uE0sDmPEWxRqA
z>rbN+{(6$h5ZkT$xQ^&X9a;i{I-gk-Haf|TQ=K4CRu;uKgLfQA_HmyX5qIzz-lG=o
zsL|$3Ul#VksZ%2@&NF;Ma6maok;mt$xZ}D;Vo&0E_{=Df#B42mlH;CZhw&-{vn4V1
zHF}5luX{Jq5JO3J8JDadhF>VNO4n99pPa0|A#?Du%lbuOdhp^==lvfv*jjgj*0q9f
z5n!N^{h*~$;<G!82w1f4_R4=EL>JlKQY+u}EA+fNMvK6OKoO-?NYI0RVUB!VQX&Ro
zt2>OjbdHLiX;0yb*_<wo!n`=$hg;@SiN-fMm>_c~ZPU7qf$=Q>=cyRQQOkYS=35V&
zF(J)ZXfzm)N<Cs;Gwji#JBZ>qrr(Kw!5RBQU<xCy6y2pom=8NB{jOrO_IGT}boAOK
z;jVPD)K--GKw1p3m)MkVXl8;@jY75P2(T&kU!X-jef&V?g-Vo@#HX&P7jOAhLSt#;
zx)if+on!JxBulc>rF7|MQLIZF(4{Fqloe0Rqq6*{C^h{A3t!TBnA}`I0E<mDY>3+&
zQzx8Clwzm?u3m{7mpM2r_HZfo-B&*GWc-Nm!&b?qXnPAjCfmCu7@84<9pgX0+82MP
z;N#!H+@VViHe1qmqo~IczE2&rw!F4P_EU42W&d&y|LpLL;>`96<%Y#yafn`vc<#2|
zi$}C6PZN~opD=FGKBUv5ak}eqdpB4jgJC!sE{n&&aF;6tH@GllHu$)Wt4+~}&^}7y
zkwr|AOllT3QDm$%GlLB+B~1#0KI3t;a8z8^t+!YXH2jII3><U`F{d#~pP?~n(LqlN
z?x}_f(g!^H)|Xq9y_NG^wd9otleijp0snZ+1G)Yz_G}I1%{;O^!aTd!*w|08bFmq*
z60&l!%&`Pa`YIb57cW(FwTG|FM=hSsJeV<<A)nEop%6&1l{!sl96f|v!MTqdkED+X
zd+<zjO-jl|-OC)}90CuVR}z@-k*JbH^vK7d#W}>$F;A))<f*9jt6i(P7lf%5sznwq
zXi%yd=S>$%=B?xvYs^Ajw2PHevTSpxzOgHFs4QmtC^iK78{g-;y6@HD&|%wwDa8@f
z_1w%MF|ki&Rm@b-QB+*NRff*MI8`dHLk7QwL2*<*rZBO%M&qz_m91Zi_=w>h12$vP
zS7qLx7w=DXh^xFR-|Nup9P8-oq*ZFqpw4K|Y*tZJQEx0c7dd4(t!`LP)_glOdgPeV
zDRI*l9TLR2Fe>R>?3C$D2HzR68KoQ>ox~l9f!hoV=AOPVR9%!7QT5HdVY>D~|LC9K
z-}O4P#Ujv*6`wRHuD{@QiW&a>jR&_VdhlEyR8tU;CsMSNvvC#hdYH>|n^NSG+da4C
zq<YHvG|^JUT*b;^FxIGtr~m1^^^E1kLf;%~i*>ekrgmCQJpW{UbN$4XKj!?8S)&Na
zs&)mr4Eot;y$(qxK_r$d)^TstO`Q85RJoLqYTH%pBujkx%_9TJ_ZYFE_=?}ju$VKE
zGpjMm)8~?Cleyj7%@7mA2u+(4I4?}8(664it-CzgIH5i1-AdXv6x8D1qbzu6D=<Qd
zPDv)n!{6vo?m{GB<667i_bzp#ex#=TJS?m*Y%F^qdvUytyWP2P(}pL8vfH4)@we2w
zLucJBmmk}ibzz>Vmv2s+Al8s_$cg8X0%zKz<Z@5h$lTA2gO0hmb73oSzm8)P=4OYI
z*7Ucpr#h#kmnu5fm^!0ly7`;k*R9rAj)hh)7Y66;(ih2nlYOInLwr5`JVf<GO`0=B
zGn<c^XPW(K*1lninFZW#NpIN<v~xWE?q!SbO4G*u_D$EhvW~gDxe0vvY2i}~twS{@
z%dRneRqFRiAH!NH=x72c%4mPR$#Val++~;<{Oa>7^;b$6+nKnTy2kf8D~wJ$>%)%2
z^(j0>1|BJrP7*<hM1&d{ZoC70UuXwF>>YmnXe*Qvkjco(d8^{!b&&eI#R7GdM`1H%
z2h!knQB1zdwc7t3qH1P((ic;qX1Z@mDMOp}IBQGkyf17?{qUA*O}?%w)|t;$|J52A
zb&Hf((~znbgV)#9M2+}~gq_4j+6IwU!A!{Vhb|>GCXYDs<Qlp|Zw-lwg(AD@+tbJV
zzmsE=myMk5<Ik?Fucr@9U1f{t+lkH}p7JNH;=L5l_H_6>M_!}gE^p0e8Kd2$(N%lc
za8P(4(ifiQ`8IV1vR_|)u5qZ@@2hi1Ywr;43hgE!{zdDL><{y}{J5^{FG#c5Ew!!d
zznk0dMt`Hfj8{o8WxKXXoG&jou}ffNwPPP)(NP(zcw{bVQ|3@sd3GWj87Jt|jG(zl
z_l{dGU@L5l$CfRLor#N2@G<B&^j?v^q-OECs%EfJxYS9gnw`8m88SJqv)kZfn5d&I
zrakyEpW*>eIZwQ`SDnQ~PVb81iXe|FZ!b?T%Ma$S)nBZ&VJnU%QYM;lcIwv}^TmbL
zjf)IZF5Y#6C$*Fwg0ik&7cxhqDJf!t6b1bSy{>m%S9U$h1$|sM2e;C!PT*Ti^M@u-
zH>j84QR5+Z2{-qp^C`4g(#oXj8Y_ny&Ik7`%^%3IHES^SUZ>@IEO@k)5Nl{lNgy=m
z@ZLLXCvSbYxbuFe?1QL}=lJO1<lzT7SvdpCw4J(Ng}z2!TV8H15PxJWP_#_e^9=PU
zfRDjOF>|p7Uk=psCHYk6nrQT1JBF;;rbY@Z`p_Qz$PV)-8?18BQ8GNJcluR+xw5P|
zrTG9_t7l{GXe)S<bZFO!bh0Hi238yIHqCdEN8C4RPme2juk7^dyBplASn3N8HRg&1
zVY!R(i+AVN>yo@r#~ZFZ#ZMC;G>|9mc^9QupLa4h7TJ8E33XjcE|;z_p$K=crl7^c
ztH?PK;ZstfbP*<B%e~5+!Sr1T-`BpK6&8aYX>zHA4Nq=Vw+D+k*bJzyd)J`V{%!sX
znj@Nv22h{OYtvobTo_y6=fKE?m3j&f!vnF)*2c)K$l^32VIj(|mx)I;zrt1$R-x&J
z)Ly;kCkj7e4y2?|A65@lKKGmSM1{+sd?G;Es2oo8kcb*a_}vXav}HP;27gBxJ4cat
z!w2c0d`M-H@S3Ze-0x){)h2pgW~Nbie-mBBMJe1Efiqfyc6~=l(JrA`<gHT#qGZR{
z@_xhy24W0>hGlsx^rl{#5M;2SCR#G43JNI9;2IkR1N8w4Cb&Wczrv^#|Gt(&eS(7i
z=Q$b*N{|H##@};5^84o}>hJr%KGCB;qF{k1MBvx$8`{5Sqf>oD|JOCz5OSTEs<?~{
z_*6A^Ffp-pG`DkN*w;4#4{pDEsqKh@LQ0SPMU_!`ya#BSuu#)-(o&EYFt)Q{fAiMP
z$b{X^<{c6a3dBtST-um8y`gclv9@&-a1(m;XNCZ{Mn2|vMDu5gla<gTEd^y7aXSYS
z8Xoqi>`xyF<IvF1Kpfth3aCg({k<IA2|Y4*a(XAg!Qtxa%I?a=Zs%ae!O73h&++sb
z$FpZ_U<R9`yRFk3H#S>Gy1#(@3yy?|qp^d<J0}Y}TN)(XH%4~OPC}0!Arbxi=Px=<
z+${cs$=302wg7`1$Po@s_NN^Gh7FcNkWU4aE!<43wIwWUfXo06Vcus?A%EuoKSut8
z@gFO-{;`su@1HCGG4g+}RChFS5Vx}dD4m4=qprUf|8wx~g%A!T=l_tzUt<3A6v$Z^
z2g31hoeAUgk{o^mYJ6xRp`->rfsrA9P|d-gCx3k+uXTR+-LL_Li=xO#h^e`uZp~xG
z-fN%f*mv#b$9}?0^!R~j`+IW7?_V67gK5acFeM~kp^{*r-hH~49?b6<9;|(rjwbZm
z6B_1#-+e#4j^k$?&4Q+f1oK;)2KRT~?`#wpa-SY?&8w#nbB2}D_@SU-|ND={y)Ig}
ztXByccSTW9G4A~Jry1uCgzwJZH-8?}Y@=aBRuWgB|7!~JDUI7Zynh1%AVp^|v8gN=
zO|kw19-BsVZ|VL&K%@9q;@*LbDE1Nj6TIJ23+MkNi;TuCE9DF2Kf$B41RDQMBOwlC
zuNnPdLf5Xlm!pYwr8hxI4nKiSmmxRRqhQYGa(kpizpipC@CA}9X}_hPFZam);w};!
zkAg?uwtioVNxdMp{&c}}q}pb-a?JY2``aV=s*Fs!)kToQJ}!roX|r4tk$a&)Sa|^R
z^1m?O#-oC0-NL*7TD-F@gm~qkmqW>F+qPzfwB~ZBE?YkF*}?A^r5;Q&y{L)nSA1yL
z?=}6F?k+Uuu3%>=EQN72Mom|k^^RoAWs3#gau^cBk_jVa8>tq-Vb@#AP4kH+VbPe1
z+ZZoo^xP_wZp44|IaoeTc;2UaxA9VTyg>cXdoZc*HsokX9DpFJYn6#%$QE+qW7}M8
zYQm!wP?JqyXR~WO&q;Rs6*8cCB?+ijVG!MWNluSR>(^ZGe)xOF3K|zFeDQ=@N>MT5
znnYcxz;dD}akoybAyLG)5nnKlSv^`+k)K|@J*W5~J{p&4U(}*6LSMiM9#ws9-eQ()
z|FQmL#>$0Sn#w)P>uSIA`~>EHXqvB_9&o)Cqa^&9E`smT;%R;Nr{;r|-fWA}oN~)?
zW(r{{uf3MOt*I@^*0tM=$uy%m3Kb$ma6&8(%=ef;H7~PKTIk}w#(oeD#w6u1uvXGK
zhTg)zUa8lvFg+O8H0=KI0k^pCO%Nr6YOW#)K32W~d;Q)Ecc7(}qKbj&1OpGI@%G$$
zNJI8Yc`thfY}9(Tnr&sJlk`3X<mhm>31I-HroowYtF1!_3>e%bw4dF)fI6A;P44Z_
zyDnz0EXuqI*QqrBTxmHjs{u7&%ajU_-kGl>s}@+K1*W-!*#~{diZt>OG_zE9)3z%S
z<u*K(@cQeM+4CN*)1ukkqf5cvdagy+d=KLtIoRdX7CLpwoT}vR7{=L)uPv<>^O2-J
z7wfsw*HwhI&YSUeX{({qN1nux1MkkHKI|_ncy}F>Vzz_U(tL%v1Qklju21tjyA%bt
z@1283L`5yqNbKF1-+#Tip^gVKizXg296C-=IrNouSoA*a#^;?Y7586D^(?7B`xSPg
zQAjTr|FS)lBy$taCVf#@XFxmczLt`iwi%|7#4}yUaN&WtK22a&FM!L{!He~M5!Wzp
zp?6E7o=IJ=q$%kI_$IZTG-w(|GBn>9k)=qfbqbFasLKJ}b7(jDG{B+PhCb)nV<q~!
zNAZsvEMv^iR}u_=4~ifvH|ZbL9rW`qc74RB$nv?qI1$OkYlt)5nX5V31|p(ceLQHe
zf40|3n4-S5*l@a_+ZSTkI2i4|thYZT{FU1(UJYuXe{;4R3Fp{oZ}_!8Y3SQsx7%n1
zea&q#(zft5Qv;LDLl15vE|4S9dC|s^=5vuRWcroYj@@}<to!sB%x9zEwf*Gda`rPw
zm&JK}w&N(&&p<(N=lilmC*xzCvDx>pbr&60dO{|&b0gTyEp@oHOJDnM0n0fl2&^?X
z^Z~|la%M4Io^9ip;Z1)z?L4W^xZD{D^UgHR#3?WqyBzjaoq>s5?FCvdHtLU!okPFb
z5P9EhLYsP0J$Edc9`HM49c@i>4O@DB1Rf+-{@d@w4{X32`TJt_HNa0aRU93tIT`J4
zYs<q+H@_fqL%+n-cx`UgZkC+XUG6rCEUsO2ybpuBtV(b#o&xK(I1*%oeM9tx=rwL1
z^%E;W-b{*|Oc~SJ0Z*0%I61GLtp1Yt!DUKO<mqVSjC23J+}lmOVjVudvo^4m%LRGO
zuXfa!<EKA^s0U#B<#x#|RImjPtF})orOs|g!_t}Uq-&u;w_}HGQalQWhoz}0Ie$LV
z(b&SR8g~|S*}8vee^hmd+hlMZVkrNz%gn^~XGLeYV-cV8P*X@er*Vgb{P%v%k0n+C
zHf^8mWh!OfwgumHxjq~e5jkkVqLz?HnSSG0^_=nKHcV+%+<ELm-}`vHB$U&i-{^;!
z1T?w*D<4z=9#N0Dxh&}!vvKki8qfv-!e0&N@s016+Kzwu9Z{M+G$D_pQbuMhD!14E
zHsi#PxwOVhPRCT0R&)pX6P+j^&-cvFh7f(@n7Y*qkA8XO_(E$p3cM~I7Wi7kH?t7y
zx6k?4l8s^9=u0-*9?#57;SM|@H*CyqUkYhlF3*<s!A6tbH1%nLZNhIGSbHB=)u`N=
z*JL)}YcU@BCU%#rq;2PPDUdLNR9|h+ZKr1K(Sv7|C94H74eD~e_Afq>_fdEE_lL)?
z)jwf&<74a%2?%okDA^`YN5QFSw&lb9@GFbaNmWgc5ki1*a^NPpX|+K9sG1(%f7LQT
zh#-0Ux$+kljDd~eCdIv>I|ZHZcV_XfV5m+_*C$~+TsLRoq)~0S_!2WzTpg|IOsgA4
zl?F3}9)0spqY8H$R!XJ9ADjpZ<;<}TG_4s&@OSi^7KH)RyT>HD_ar))<c{dKC;q@c
zPp2+@35q1CEjYw2cb9c&JeGX`F~YSM>}b7B<xy^CD--f%HaMdlxHY@sm<@CSKAMY~
z!;Gw!)@cKTsqrW2CZFp^rX6MQkb;_&ZjJoXrF7b^ng?0N;zsBn4Qd=3>p(~~PBJrO
zB!(paNG_Bfka5~wXrNm?sS7qJ8*Usm=5QHiv0Xjkznu5IIZO+#(Pz1+s*z+jvb`){
zwAc$XOkPS~u?*%3EY2*1@Bgf<vYIRz9#q{oTIq=!79M5^v?X=K)vYYvpS7rQop;u3
zn~S)QP*30O=Mu6`bl#j`a}sKl0AX(q<S=ugWx{CKTQp0_S6}j-T+i4~(79rwh0<$)
ze>4_}c+2Gm=QuB8{G!-?ak5PzM6Oq18h%)Iy*arwoU25w>$~=Lp$4-6un03-5BZ*2
z+L_uWNaJDhqDf@Cax^(I!&j9Y;zVt_T5dkAxECxE-&etrac$Z$DlzG5fJZG1xo99?
zTlTU>2(*Z!OJ&R+_C0TPnN?GI{W>+Bs5pjPDH^Kyp<%k#*{ZGYjv1_B^{W+*?)bQC
zMDbTLCyJCpG;MMa8M}x<p7rEU;1P(4OU;5qhd;0t>%yuxQ=CR*&NX^X_l9&WiP%hK
z7vSAXeTiI$#lHXpg_<0*1Zx<xcw>>~dCaVC=HG@I$8{r!*z6jP<F9>S^DeVLH`Q(t
zZs$(ziwm4|%AY^r98Bqg8RBc|7;<rnHV&rnn(nz(8ZN_}^>8k-7i&xu%-+eZ7*S1;
z6~`pbHLDMs{_=xYKM1ZA>HFrUY0?J00xny+cdXF&Pn;~kz^2NN&U%RPf^`WGgWd9V
z(ZWa;<A?oa_WJt}Cz`qmb%f_ioaVuC$zouTV$8;;!TW+PMU<@DmE1l8`fDeAl(Y6=
zd}lB1<4d#IGZyT8JSxfgjy0z!_g~e;N>^-J1E(HI@v4US>}7JVT9z}y*vn^g1l+E!
z4hM$?+La_Jb(<uh{j3^y*JLwB*&Emt0y7fX*oGM-eN0U&%8!SQ+aSg{UB}v}K*&!@
zP+Bs-s}u#Hpkd-vT1^h5=D|6Bd|(@BI28{f=!bnJ3k*usc<%A+Y|1!tr4R~8=IJ1R
zTN0S#Jj{aQ@#_Hvq;IV175Z9Mb!n3a8f9J9Gq*OvvB-I)Ei1FvO%dlz=A5eIGcN1f
zPGxe{0YMQ-_8VES&>H>9G^Y&A6qA1P(%qw&kJ%EiE|O|OO5ChjGN<vpQk!!EBysY7
zORv5VfILLB3?q3~_b60KEX+OUg_CJkf}s!pqNM>IRhcHl$#Q&(9=k<6H`ZBE(Ycz<
zY{|)bZ-|H2dTOX77q$;Cmicg<t4VB>V2wezHbHmj6**NPVF=Z)<C)nBA!d{knIxOW
zp{li8EBG(~iSNq@T;hJ;Ht-|6Laj#IY3>sh#afDMYEr)zGLwFy0!Vs%mj#&RD>PyT
z`QX$(lIJAWTYlrzzE_-vpMeMSZ`zl8Gq@kD56i01A$i`h4Dzq=Y9^m=WyBDiueM|7
z;|-;T>lK`ZaloJ1JKjn(v~MT5OPQI@n6%&03OO70F&N^KOD-q6b`Fphiy`1^8esgY
zILS1aZ~P{Hp!PyDts!IEd@S$PR_E}ffI;VNy@4WwfrM#1DuK9)36^12OU3KCtMS5z
z0w(40`J;EpT!;7I@^AEp9IZojNjH&6-99L4-0m*W`T1F)rsRy(s7ovLER#`g&EzN~
z!DW=Myq5>K_=xadN8oes6W;kWADIf-l!<rWaNX*}Y;?&P9gkTYFVx&EO}+U6U0LT6
z`bCzaq0uxd*0oO<!=)r1nfjf?rTpp#F2~ff%H;FIwa(o~*}ExX!+<~j;R&p5eq;V$
zfk9kV1trVXsRL((hxg7XXEK)wE4MEUnFFP_#2*j~JsEP}`9KBT*ED|^8E`8h*XmQy
zFFwMbYJ`-qS92!wwa%C&1^!CCqcRx3B1Z#wLWS6uu@=cH@zdx4gY6_VZuc}pkAd@7
zMx}hZll+q4A>YtId=|wOh;?_@EZ_S;8L#%k4ETq3T|f>AKT5d>+lCKk9?n-7XzdL#
zS_xkIZdXlaFzR?%DP%CorYxn;!l8Y*WbB_btmeMk4__}h^s)^PrjV7XmJ2YNc^Ks~
z)_RO=jAa6^iQ2BMWcOhhi*+Mo1tgdP>XrrZ0RhrmG>*CK`z_+sm6}ikAzjTB71xuk
z>0!(Dkh;_&t5W5)v-X;2SxQe_O{GWn@h*yR%pPdX+c5_Q0Y5;5-k?J?3xzgbSRPrd
z3gExHitb-Z|2bEql0yh#9XJ4~ykvoTA?ffCr67IffJ#n<F8`n~(+)bu@zf}b?hNAv
z-aS%K!_tfc)vJZ6P@&P13=s=^!@oMB6*4A0c$|EC0sBT=fXrwnN5!fy-tn6CiS@>_
zWiFDCLXK!H21e7I4KHvT6&DK`zk`r8`*ZSQYU$_LD<!Rz*+x)g3lEQFUGz4^uUqG~
zv|Ct#nk|zspM}*8{wAY>j^6~1_X3O3N%^QJ@MDP@8zv?p+amefoiRFLH#q1EUKU@E
z&uL2s1`|MBZb@veiGRc!RzG9|B~N&@5|owF1D<}@yS%;x3UlFy9X-aO5|GNYU+x&*
zKJJNWoJ9C=qa$#Ed+<LA+>ySNxg1V(R}dN1Z8oBQ508xlf^t6F^+i~177fO~s>>i<
z>q)Ba<{Y2yOeXI=;&OKVVgG7{A3@;ku_ZAm%4F%;C=KFhC=oCr`nFSS8f9yLP@PJ6
zoNOiTzE@N7*%-?=POG`&8y2Tpnu@V8`tA6AFMr6$toTuaUPD|Pe}1HI>c@(PL(Sdk
z$r#sMyfvusnN7zh@&U^eC^6(Y&3cjgKuAGFKPc)*hYN`?*;I4nLvAvPX3hC)!&8r3
zcIKuhehvukvYI<y)_APnt18=`D1Y!J_f;gdNaK*ryx00C>fokv+1_Bvw<VW1zrr|(
zNQ)VvWi2E2BZli~UI*Q9lAN>f-I}#jug<`aB;Gv12Db%4T~QpfKt<BTXDpa}-FCxP
zWSH!*;C(s|_p>q66}ml%I*KXOJ<Gq(zM)a*L{K|n;aAn@;Vc7vNm2z@e@NMTmx9O1
zvms8tcck!9J8+C~jwG@O{0d~JV*O6n@QZ$gOd0ZqiTEi@CL~cuz3+DFml$(SF1ir@
zyUgWYjv4f;2^yTsx7+%9=vLo0cG>*W974!(5q_<>oOI~p_-&N4A8x472i+QJ^!C){
zaMX?YGArMfJQT`LOUm_h`Uci2WlwD`*uUh;sp;8vRlv@`+Ftys9=5tZRQ-^PtzZi<
zY@{gA{{HhXo8+HWpqge~t!COZH8mn~EL8f;{gjZYvYO$w4kNK}=o|wH3%u-gNknkX
zylkIL)t5r5+mRpCwpi3nu5K=OjY_(vmNg@p99P8q##0}K@un}>X3i#Ys-B#&4K(qy
zX(5U?gq?LAXOq8RJ%~TkVlU4q0<5Q#(zv~^y|jR!{Uiw{r1}~6ZN-3iXvx}^yW&m1
zo(YlTagNr{%*WeLiN%A-C*Oz%-FROgPqxpkYDH&E-TI&5<qqg6iEWK9Rdz!n#rulQ
z%U`#kadP=}(*`|^j_(k<J{ooCrcapiPModY8t<sw3jWjcp#yEFr=WgN23g>X>Fq|#
z%7ubF7=qGy?3G0^jC{;CFo>Kndg59cbZGu=QGvO9Ev<b^bf&8X$`6Lj|LSx-1L<>D
z7SH7i6x2a%xTtMIf&rF;nPq8(*#jlXzgk#b0ibn(3*RCFg#X>?@-Mx82l8WLx$^I>
z6LP?x75fgP+;4#WADuSrB(T(4NYV2j&;ASUAF_K#duO^_rP@a8@hmTU{I;&9++)|k
zFN8O2=}z?BOsX=namDP}%saH%EaM5u4&3^&0|kGYk8gn2dY=!oU;*k?e&1YujQiMi
zMYG&_Wiq<XGDkvw>RrX}FLzXOI($^BSN&CTR>f3m4^nM;%=G`L3I+5i(dKHN<0EY`
zqWMHY&ZKt)X3}eD*HeAmL}C_W1XZnd5NFanRbztncB>TpU~gZDAo-)k07lV0Y=g9s
zM*z|{EwtH*ffxl`#_o6%>R^d#GOHvaXsoJi&$tB($pbyw>_Q$Edrs?MWJ`9*5H%pk
z_cG!uGIG-s4HZ71@TFqYMVuE&%re%!n$`)DYiF46x6#6nVGG^<gUjo&=-%EdvsdZ>
z7Y(_8SG7X<IxM8zV)&lp2Rh!wjOaz4+)rwGz9DMnZgIb3OQO8oV*zl0%N^ias|&_?
z1a7-=f0vw#O$^qI_bDDrAAHCzeXl~|8q;$*$7E8lo$^D42?uG~&0%O5gd8V0&R{QK
zs5~h>KV5qpSX0&07nJlvMO8~Jzg(qz&O^O%;hc#zHsjA$s0Y2rJ)KBpY9xK?Xsa#-
z$07?hriC6X>hz#B=HRUKxyKX#&f&YL%AKGee^@F9TemAh&=c9Z9AUmQ-Kae=NcH+w
zl<(Jx>#W+n@~Vg^g}UR<GtMpw&cm@FU&p42GC>WHju-TsM+z>VG}wCf$)@QE4<X7G
z`?zFoL;WlxYc!dN9-W~#m$b`tmCwv+=U?6DXFye5$=seuhDJDEr*@XvPT;S~cCM{2
zYKESEOE(_l@_iY^wb>Ia$R5vJ{4sI1L^j)v<KACb*#Oq+-_smO3U2x1JMMn&_pML<
z7(J@{)*AjDCM{l9aZr~+AE$Gqup7Zdi4-A?F-SW;y{e!?UN@j6kqVhm;qfsy2+D}B
zhr9@?ND5W(l>H#Y?yzw1$8$7y0+Ca@QmG>&@k4(btF`X0P(e4>9=+TQdtY2!N*8sm
zaoZ|%>dzzUN$ob>x&ST;Y7^K%R};o#ND#rqL*zvh1K(eOJ#_yx9JWYGE^ts_SgV4b
z@6q*`Yc8`MMj^5NM_D+?WU;3;0c;?@Q%s((LFl$R&)Y9a!t>wQiw9N6e7Hz>DR~$r
zhRmN(P!YhhUd_6WJp@Riwkb$^H#peuauVVm_5R{<nsP}D!qE_M3XLtO3^M#GBlJhL
zzd%K3Pg9T7?A7B^+llaU=X1u!e20)1(gvG76T&_6Wu8#G8Cl6cvk$-?Jn-B@7C?L9
zqB|Sf<tLkpjrn%Z(pxRwj-xZaOx$2gf&FQ5-`V?&($XrOBB>5m`fK?+S6p^am{524
z{GOi7$EKY5UhW+6nVBufqdG+m52=T5uYvAbc)S9E`M7DkzU-hc_qf0`weEBooEXm6
zDtHmZHVEmi_ee6!R%@S6`E`!$q|hu8gH59(5Ne6sG-YTMYa%pBq#~4;pBW8vpF91o
zQl@^AM@afs4DfenF9A?wcr9sz<c03e=kc^urdeK&!%8WE7ePGV)y@S}u7frE3Lh%L
z(t41!2=;IA%H#Lx66K%W=|88w*gaz>;8~lxjzg)KSy=7!vS(~+(Y@F3kMyZQEb$tW
zQM(IJ1(QrY^?h$`aAmzEo$8UYGn5|A*3^lqlD~eOYRMb-hny#RpkDh<xQL`Yl~{`_
zv%y;Tv|?=|o1{kqK6D*hG9pnxdGNmRZ{kA$@r;y`+(_a{f`6C9s8gqHbx#*PBN5p8
z{Mr;&U_Uk+t(t>_bqi4I=cfb|E4==sAOx_H*F51X9OKM&=|(`8nN9~iRzIO&H?gAJ
z;m2iukCbARmfzCnsSip>*h)CoI%n^#uS|+^H{!#G(osk#TtDOB&v^C(^!$1CbB`5t
zuckiQAuVq3w)Gc52_D3Lf!Rh}F_^sKDk#oe1>YsdWez|>R{jo(_;=KeNCSh=m!8bp
zM%Nu!DUC5DrHiBXj6swOsGfEZld7TpF#&U60>e^8{$Qn>)a&E5wceBPCbI_NX#z?d
z<pLw2e4i;tZP+&?gP@QCO_+OwAyr5^l(36#vQn%U50o@Ee|-RSWcQ3mZ%8+$s^hvu
z{Gr1OfZi|GA_jQ6YMToPBV)R%!_PO$M)yY#8wT5w2+StpJUyTHwUyw^Nh_|0sw{u8
z!1Ng;`-5r$gf6#2kvx1PM|sUXr;$@X$*HEXsaO`QRLzCB4zk_jrTdLV`nyV|ZX>A^
zJpfMSH$D@xkJ0r7iH=(n!ee6XYq0Q2aUEwhwDZ!S<dL&D(kz4T(zvy~-b1D;dw?<}
z#h9-6I0LAu)JB48NzNGGLHMVwu#`Wdf6M{i8!bc;0hawg-7dPhf^Zd%M}OKL>=k5I
zw|50MhyM%M8K|HP*sp)UYUX&b<a;cg{cm6q`-;;Cj4(1Lw;Fw=m>$phubiGPf7wYg
z$c^|#s-*zk&zMWsVV!T61A@3}m7qICzfvi)U$hq1>j-UpKuY`$aOA=0KiR<Ll8HZC
z??S?Hh24n*36}<}hYd@vMxfKw+II^Za4ky150vBJV;v6Gy;T!U(<xEnFuQv>=HTtY
zkBbzip9rw#8#tpS(wZeOHGF*<6Q?&$;g?SvpQx9mCJt}ZJmpE@LB^U!;4#mtH}jCM
z`P5&x&b5@EM^65j@tWNMj-NeF=UuOM9wFL;NH7!F&sj-t-cvAB|MjLwT6b-t_ltgI
zlct|4)e}F{dA{u<Q7;12KAW5yNOvRQU^3phdV96TO5EbJYEHozUynRfvMNLDcJ458
zt|3T_eYW{G03+$7*oWI1bEYI4;%ObAn-K|ygm)kXz#4y&OjWy-6rDdy9xTms*^^+>
zuaa~VOp8a7FN`F=aygF&uw8nR#2#I@F{>;Vn{V%@W+TL-W|LEfv@T^pli0e<4A>~?
z#8b6b#M-=@9m-X8vW1sf*4uRA<E9Ug<YGvI_nfh*S(HE<FZ_dxt_w0jRv~=PhC|iH
zFyCVcV{{@}xiUct>4C(6G%$*#EeL@$WNBRc9})|faN9&+)X(!XI6+uJ(m}lEG6F$r
z>pQURr4u$baTNb3>>|WZg6V@vU4L7fn%dl#D-Bq;YQvu+BLsFWFc!Fw{3TejhxWO+
z@NGX?LiY(90;9Ybxjij|6yuZ2xzVPDA0Q3pJ(Ae?)SMJRths+NLY`NNYu9Gxex+`S
zLspN^X;N-1?6wnPm=8$;9w6cA(K8zfP)=Z5e-TmP;qYNlwgEo&gG#;yRJ1Cj1G!<|
z0BOFjTC?B<vQVNa@7Fv2E>>Ogg(|o@ZazP}W`?s*-bh{G&UX4;q*SMXir$EznTvqg
zvNYv3fkoG~71p}EgOl6L`q_zULVL6^{LJUIlxkE+t6lO1L?E>Hkck-f`{tAWUXyv4
z{@dA)?Tyw->$09N<Ip=~^AD=LwSAe9mi<4xtRb%TV)=^+@5x@h^Dn!FF2LKX6>N#9
z$i~ffYaZv61A0)<Btf_h<uSqs$eI(-KF>M3<P^@0S9rfdJR8@YAun?1QQMdF&VT%8
ziwJ^1D9AB^+?rSPrQU_sUl4xQ1_uGToBBSdj{RHeLXxs^$4)}|50TTsOG|7jdYG{s
z5;iZIZhRvP#VP>_rPofjBvz!R<*;GTwnIQdfuugty<&`jg6t(avn0||4DL-eTIAPU
zaK5N!zZ@bSXVG1|?-;l2!KD6%Old4g_K&B^0<YLV8yLzWlOzaZ>2ccN+svA4%h`{T
z+WETcvd7sOHN^$i#jO?PzAni3`7}|Kzz4X@O(Uy=D1HAfm+6C3&vJ{O%ciS^X<RA^
zBkEsYez=t&SCbOSONhxD_u0Fg7|@P_x(mW?{V3TO22ev+fM^?J{fY$`WYQ{W5n<o`
zsBorYailx1R@D7d8q|%7<!mL##w)$~BjOw2e9v;Bc}z&O8pv!t1NLU6<Z9*yX4CqE
zXdCL3eG8N#{wE?0Da^S}ABf^wrT#2z0U-gSP{aZm5~7~@cTK35+g#hZuy&n<htZdt
zRHcygCX&mUWtAi13jdTt0xp?OD<7j(tEGB2M8*Ds0gz+la#;Kc8Au@j>81jcb*r|@
zyAN)c6#SEueLF2A9^JLmh|>9!lDL5`@6T@AbQLh$&lM<fVVlAd`99N48w{*0q}w6~
zu)qM-*1ZCx*FF;k1A&Sep~!&*;5iI7@Hddb`3!jqBdswb2RXn922$a-n31vfOc4y!
zv1k4O1EMYf)bz30E^^X8{2TUL8K7Z$Hy5O%IRmN8&qIQ&3*>?vupp7t%oln66hLi(
zOY0z^z6CAw&zzQjA_;~p*z9>bS)9m87hu6UuPGt&`mff4`V5lrtXE)xlwK&(q|rdq
zf&Ff6<T=ne4JFukPI)Q}Kty+taeeRd3n|hvnSkuFVq<<F!{ZPzix;Eg&+h@Vco_~n
z5qCr^lIiyf0CP1}ZUYiqGywD%bsCv3W9Nf8S(Hi@$aFOtB+|0Xoi4~Z-vM<$cYJj~
z#K8Uznke5HjQ=`u^LIZ3tbIo>y7#WVy$4K3L30P?0RiS8)6@i)ntDZ_z)G4Z5Du4=
zk$dUC(@{|WE1Cb=%zusG|2v)lG&MfYk$LNR7VK#HtC$Q{sV_DjV>a>ldj#px<o5B?
zl6^bJehcEoXG+(TP*GXn)|uX9-2rYh5xC8_l!}f*z#Z<%YbxKxm;+hMTv8b(aC(0Z
z8qx>?FT3@<LK8#f5y&EL(|rJ5wF^L?-F)|TlN_WiX0-vKG)o2`hAezyL&o#~tWFfP
zX%HRXcBV8uiXs9&UiLl{i105!^9ox#XjE?yxOd1Xi5dp#DR8o<rq;*^15QZZy3-W_
zC=%S+BJqy}4^)1<qy<sp8Gz6wki6<41&}0`e0zlb9hCOpd5v*_qyBT!QPc%cG;MDN
z#m<rdY!XS~h=KGLbZuTt{Cqyi1&|CnVUmb7H-Iv{fm|2q=`jc+P%-}98>68IY{ouS
z!3Hq~M7q1>1i3Y-fCaZ4mON~5Rt=kKr+*4r=KeXN`JZ%$3CziQ|M>-oYi|LBC}+O@
z<4*tym%f<fop+$%duP`{ii|ACqn7?#fT!~10fW0QK|g?w{?k1W(`Z5J{=TTyqQnfa
z0D2S50e+r9s-9fc$Z&&A^$`8Ps{60%{{Ot|{N-`IuDT;fGS}!1HhQm3B84o?JWeg<
z7c~tLk0?vLl~a9p7%WD<+v=Rg_58R^l|PoniV+I--jhZWt4B+qJrT54`yPjcc5hI`
z%4Fe6<abnHlI;3OcBs}Y!mtU^a_Z2bE&lfULcwA3o80f!o3)FkSDb9-zU}A-zhfic
zf|EVbrwa|@UwLQQR&{q9AG^81?DDirfm3MNG1MV@2i(&z<<(DV$KRi&b7hx>TtClZ
z8K@oIYS&tx&*w5(En0NZst~&taJX>~+TG))zG5kLT4$cFurQncZXcHH-a&gXg6oa0
za5zxMwX)?HG;?C$(LMak!ibPbow?IF3320D<;1Hge{{CteNC<cv&Ek@$+8N>r{L-B
z&LxOe5=mmI+fotFX>_dHsyL-upV9B>p?Ayr6dmxN8099{EZ4F;D5BY8hl{8dC1H+@
zK|isJCA8{|*E_#RGR#P;vYb|C)?}ylPAegcNj(!Ssr9IgWzm%4DfVTcuUj8+4)~O`
zcS2y2;`YPm;3NsQz30v>GF9(h=ykXq@F<KGoc@03@pAOhWbbbrYA-(N2*IC*X&%Rx
z-5G|AZyip_=O@k55RW_4_>?P~1_Tc8xSy<3WXUIv&=<5`r6FQhJ)J9Nea_n(9A0mB
z&@(gYQxqFCv{b7{_6j%`bSI4N##|gXc*~Ku(CJH~uRQv9YqEJ1%>ak$bnY_xi2CWY
zM;wQthQ{^bd$^(5FbiQP6{0ETfN-%%E<(V8V{NnYT0wem$k6q;ZA6>O_sTo*u+^wr
zVo$5myym&}T=%qdx^DtU%45N8wd^71XXh&?+jrO9=Ea{GcX+K&o<6Mk{VVfX&E=<U
z7NU?YTAPcRZ^fNwf_C_Qr*<tilF#l!8m~6Yz^UD)EXlBW6U6la!dyA|g59uDYUbgv
z5-F>e<)!n%azki^_QV9O+uQI|${<jC-913}TfNi;`Y~>1Q|gKGH24(YbJU{m@cS+T
zmsGspl{d*bd8JCzHXf1e^6ikv#&`5C<QL0q`m&b0^)F9yLMnx%9p>=@jZkQWw_&y{
zraPiR>~_;~>k$t83l`<&w_Z)IotO!%t;8_`uDcKy=%UzB!Aj|6N$f&{BF8K5T*z^m
zp3C{m?(Xi-<fFU$b8(4NQg<On+-~8bs~`d;R+uVbXXyd@GIdXbOt!wmGHf1tO=h2H
z+N_~dzP`NhZE{!$y?OLC#IQc_A!Na0nTLP9?K&f;zN@i0M?u#F>Rs3S<3q0sPeeOM
zi#XJLd$u6A@7f}yQpnUWk>@yK%k?EHH)sc=w02@=fkODR>FZEwniQm76AaJ&<b4k4
zJgo*pNj861s_OP`LJ5L-niS_*B+Z_W_?P)!SIKq=PrWMHbaOcJT$8lj;c<7}U_G~3
zAR4x(70?~H-C4yv;WsnA$Pud~oWfG5S)AZ~lFt!DJ)vjdyS!->zOMzdjj2DM;62g!
zEFh?vKpbhOzLl?3;oRP;u%}hykm4m`R+b{OQOeW51s#c~xf$Q*4<kbu?+)7yd!0@}
zGoUb=S$2`D?^>&09FD3VI$O@X)wCEd_XnrDxoZu!a4DCcF88Jxi<CpbxfeDY0-NgA
zwGiQ<XA3@Sg2Tk`8p17L`mWJ2N;6H(-j;?wm-A4Se3cqFbjRb_Rd=pPk4W!ir~%VR
zp-mYKjbX-4MmWj3#6>UFwQ6~jBWU~SM1I~g#F_DjdSeC-t%FOEktum4a6o%}qIfox
zyTiEuYxu#$>|)Qa#RT14DSX$@s!IILe2=N*vSFGQ$|EbuhdX5m;W`&#b@uN|!RFkB
zn$WkGycc43IZ{0?h-)gZvFqZ=Iy3pk8Qu7Zt|VzHK%Ud*QNfF$!n~BdWWPEs{<~_j
zDcW(smX4$GxxVFC9;-lysM-GTx~lHhy#CR94Lj748blCYqx{8!n~QLyb_H2zd}b4A
zC%eE?uOUjZUIpG{l1ux>N&UKYzg^d@+X;ssBJB8frs5B-aMQ>HzUysz+T;dFC=D2s
zrKGH>JIZNgeiZEVsc*Fs!5|c(6)K+Buijya7qY!%tfu8n+UFPV=cB~GyXfucyQO(j
z^|7`P>i%>Q19d<axV<>V$<(*dck6vpWns6z3MDy3TtB(Eh-{A}C4a)rs^_-<&XYpf
zN&GOwsNLb&ubaYb;~-*Vk!tAinLt**LiuJ1_sVD_R_{3(-fa4UXH}k=@8?M9F2RSI
zt7dy^^N2*uLu!YrW4cI%s=>>5Km2Tq&_|%(>x&^5Q=ZjD0g9_>KC8s*bI$c}(cLLY
z1U`JcT~$$-=EV52_EpTL%NdNcDf(k&n$LY=guVZ+b4NMzI7b<RfbrUp(870}6SE|#
zp97@2V`htQkY_?_PA%$oG3)O^oR+>5&`=9+$=0|x$GGrN3;B-|Pb0Vj!O1SuPkL60
zL~DK8Z?Q}l=XUjKHV5P~yuWG65ZF3qUR`W1o)mPiZxclK#4%4jb6%=m;JpcOf0;ON
z=Ux;1wyresL#OFUz4al(R>U(^SmXXG@9@)CLq!?_#DlLo@k&)wH#qt_sV7w&9AUjI
z(1Fq>6sxndtr%LBD$Dk>d8tJ#Avd^s-uwxiM!-*t(rIhp=8`*BeYGe0qa>v@w6`)p
zu#b)BQbPaegCgc^`}-GW+SS%oaP(s*CT6!QTc2Z*DGcnCkDBN&>JSf~n8+r?pDv`)
ziy*ENgqVC!xBVUZhR;veF$m{_Qr)0>H46sU0@d4%JV%z;quII%swQLTlv|r>*Msh4
zb7q0C9l12CS96lkp@%+ieJ^*V&_@$<sUOW&+wp25OndB_t_5|I<Kv#V!L98*@T^`>
zkIGYc2h`}~!i26T^|nXtDK6-m$<h{$p^EJjxvps;<yUFkCv*)u`C$>#)1JE?Csj9*
z2K=88{xlaRs2nb{%MU!&9sS%*yHi%D=?(Ni`)-S5<K8WhG2Hh^<hiARbgIW<yj`wb
zzH$4@B{58@!qxt7j`9dHV75*h5Jf7V-}=LbqKt~k4Afp{4cB^2uJNh}$J~Y#DZAnk
zv0)F~NOq5;eoQj_3Qg6cEZ0@osa)$VL>*eXrmEB81q8LBL9R8#5w{<@`hL4D#tsHk
zqQ{}~*qlVgI=(B^Vsn}n_<NnHArqDj-+I;g&^7E@@@%6pB0-wL$-bbrb@3Y(q0srb
z88`gks)Bc>P^S#r@QZeVsTaRtt*a2*$?|)saqbu!IE=@pZbPuk=ph(&vmv}aFIIlJ
zBRKu+SfjwjY>tiE`zo-3+j^h~E6Ziua`?LaR%A~c%NN}n+lY_EY^Ou57c(ds6};Yu
z<E3zye79kTCIgp`6&VqUQ{EP@rx48x&+HbcN3vw}-F-JwhO0O1y-#XRr%*Cx+pq{5
zehhE;ZT8zZ^lr1-dw6CNM;})gFZabZ>g+6dZoffn-#Z%WP{U()opEY=#oN0!>KlDH
zAk?Qww*{KzP2l*&@h#`uF`ycgZ&1)hX9ON?ILn>PGeBg&Y^S_yj+uKeB6lSNH!_wf
z21>T}*1BboP-cyN*{0dtcbfCW-*SfOqW4_UnH_fRI5^b1(jjcNx>=ak7cy#a=>$s?
zx_GHsvk|(SvA8I5z+LmR15vT-d*&0{kiv42?fz5GM&=$Q`EvMJBmLa5Q%dP1d7GM>
zOI_c}w8(PI-0a-Mpas_dmc<PoqHm4uxKTVPP3|`PVL<niH&M4Z8TCSE=xFfcOm4&)
z%UpZXuUnCZq*wF`Nv>7&ou@TEd86di?nnJ6mzo9}93CI~w2cBg3pq|<=_BWgh?7mk
z`#Hfalj-SCug<Nyjmr$_k9DBW)Jj~+@Oy1jRfMm@+)pfnb|9~GQ*=p=k>BN5)m`lI
zRYaVU)<%ld>qhX^e<&f<PhK_@IkO*tqouK?_AI4y>8|5xvYpzzZdI&&M&%*4x2J5T
z9H)#d#K-5K^Oz$;`oPk@S?Y%IZm~%1Hd_W^NYe2(c7D|%$;t5#%;jyGi3Gz$TC-%I
z()ueSky-kSUgz0m+bXDUDzC<gb!9Js{>uA=P0v(YyEO-m@2fLdCch90EbpuX97mF$
zxXrnZJ|-9brS7>IrHG(tV%4jZf56>1;8X8#PMm>mL(B%wV~1>Hq^&~{z6gHK#mR0A
z!W2AFCHZ4Zu`0_r>!*-GNP2^N#r(l`CZAm>1;lACe4#q4x4@Sn>3YYlaJJ;Qqzv|E
z-o|!%)6RC@F@9=^({3klq;2j6SsmYCvdr&$b*ZzHo0E3&1>83cRq#enXq50O({j7E
zksXO%#es|<0kccp57R1#nG9IQ_4#FHS<>{Rz*4<baqpIucX0a0aS1u*z|N|aju}oS
z&#hOwM~n4jWC1~Bo*x80h=FABqd`u%KgpiT-V?)hG{MV(Bt(Py(Tz((Ksqc0zNLPV
zWtSqLiB)4T|K`)xV%F)Q0*wEng*VF#>Z|mXr^HjhqfpJdGsR&h<zS$0F_H6rvBd7C
zD=DQ!5#v_@xzEBBkA0X}-5$FB5cQl*xH%}UK(uiPgvB`SdcGlq^l9u3adLT3s9a^B
zLyt*eQ}l`8yCv|L8Bcx0mlTUWCQZA=nyrz{>}wMn*Y7&Ni--}h?=8zNj_uY%`h(Lk
z?}W&eDjzy^g`Y6(kbTwulIl_J>J*m|5ad{&E#PL3nY@pG!HVb2AZ1VLOv%04q*7r@
za=x}JBWSInwfFsZ`P~fkUT5o%!d#DsRf<=$Ghn&%0!v;s%O_z(#YTZ9<%9e0aBc?I
zU1!OE%|$rcR}Z!s21ql+PGg}7f_DNg&wmnQc+oAz0Xv<BvoX|q9Spw~jGDZS<B=3i
zaNjj&u%YqP_@Eh%n6Z#@u6?C{apI}BLva&||L_5Xh)rNX+$?uwI(W73=Jcm^ZR%m+
zX?DUxHYNFun%0~Tn2|zBE(5&pT!X@mh0k`aV2|+_Yh}R~6Oz`QV(_VE6e)ir;F+o`
z(`zEclp;&40y(RQJ~o#JD4{yQ%#vBO1xk$f?FboJCvHbvIK9LFH~`LJ|CBF(ccL9M
zvoEEXf^+G)bAH+*$Km^GiOu^1CLyI-N#G@}U^jGfNoUq@QqQ{{0ukQV%2qSjpR!7$
zauKjjVb?{>d)m{wxs_?=son>v{2RE*%zZ6XD6o3xm*{@%0%y9r+b*`K_IqZ7519nS
zV-Mx32WYq#=Wb=nZ51<EX)1v458P?uV@wn%91j~j4fkpA-CS@nrB7i$3ZJwK@!6?=
z!s^D?)JHW{TLv%Y{J;imWq=^o{I=X)uuUDoh0jjS;8xt7Yi@lr%FB!1%RXz(a3jsx
zpU(DyL5^4Ovm|^g;-F!TOGCR<h4PmIdX53|_fMOcJS)f{$szYntj3iauRYEPwv~?j
z4epMy$a+xB7Wk%4y@~q~5JWtL^YxZCZlTu7Bf*`=!)HupX89nt38vm)r=zd#Qv1Xm
zZ43`gf5^Qdf<VS8)fgl|=6c7OM*%<8PZOM)U(%!$EkRFpj;BkBdr9W~JvoAZC`Q%n
zsoRzfd)cOd>R!jGMr%ikgxS{zZkw3rw-h(Gqy%hixwsZymykm!n(mQ8Y#VGs=Y)P^
z3Nn~<XGs6zWjncidW#T?v3mC-frTNwLc<BTWx{RMZC!mI!?RNFs4VZ8Q;`ehh7{L^
z5n<Si+_v$2`xN@57V%EJ^(opE1*>)HNJ%pp$BlUS?rTT91@NC~^y9COeFY)+%Sx9x
z9mX~@Mt@m8nea8fn2Ve;>5e8Tmu|J%mj0kmE2q8}oPzzrY-%KNwf;MJtvbar#`LdO
z?RZ6hSPn^b{FSv#cSIUlWyL!aLr6JQ&}(c;<A%jW192&ZMPLqJ4moRlqwq*$I$b#2
z#nV*mY$p(&Tz5F;AvLq7n#$QF7Gj#Yjahc-SRGE;d^MnpzI{lTygPU0?6We#XuruN
zGE!rHHgYFN?hRw~zO$R%jJ04@sZWXXCvVH|Hv+|@<eQFKMAr@_O-7sBP9dp&#n*cI
z8<H+wk&3yO&0W4wulC(Vcf+}*bk&EGc1OE50wGj9<g>2(%XUVQof3h)qx>D>X;vdM
z^20@)<y4&>5V9(GAn$QYVlAJla;NQbUrA0{`=BNF_Vu*G4;k3(_%g!gLaf#s1?52&
z@_zxqg9hU0yo&<|aXnre;c{6eE7r~K&k&o;P5Cma4cW}G4>a+n1cU#7GrE@N_W!zd
zBBx?z6gJb!(lcm|$PAL%*9^~gX|x(XNZDl%xAVJC`VT89trMqEshh<MjN`GUU5*kB
z&uSjYjFmAqUwFru>Psm35enyp)L*aKoJ<;e-*mFU6w4|!EY88#8wXxE^_9V<P<t-6
z&{It0sQq#^O^@34^K(Vwhm9Aaa?>_a-p%N+X(p5{FGSnUQ&-QfJJMFB*223hgz-wg
znK4eq>D?&F1a<}(Cg?EEr!3rLpr;+@A6$f;sZV~2&)@AY=LR{)t+|7&21-ThTe+&_
z32c|U(2a|C%|B0@j5^og9T&|IN-O0l6`%wYK4a>m#~PO%?#fG%j=nH488#=cpdH?m
zgd6KygvF<EnvwCHX=lxflZA|nA-0rb$H|hB^Eqs_z-0*x`&Ucb$&EGVj*eKm*G3?}
zp(36caX1u&JpnbG{Wwh8?+V*2IG$c3sM7qvZnroQ(OPEs_>9_&ff*l%D`(NlEPnKQ
zv)DSjU!3A}Vbhn8M834<`ekGpsi!M1;nEyK$PXj2S(9bQ0BN4FX-2e1prwz7;l+w{
z@aV?Jk2#0Cy&=%lrBtV$CMBl{vnTM7cy`_D-=PG3<5-dF8OEXoiBG>V3VY~=r{3{C
zZ}%R_`-z~UGS_^L7;WgJJ6IC8n}I!`Im)s*ISoU$2VJV^g=*OIYz1PJlISj`u}Upa
zJ#e*b9Id8^xZHH06aR<3w~UIi>)VF~k(3q`X^?Jd7(zli1PMt&QRyCrPC-CGTDnC*
z8l*!)8U&=9p(UmJ-6!7ndfxc(|NgFLEkAf&&Yb7ozs_UU9NL$y%@(t+#pT+(2!NjV
zVo}<r>)de~MA>E2E-Ke-*ihCNL2Bp6MWN%11;escjNSe6od=&xRa$CACzN}YYYmg-
zd!5+t;pxB*%$G3|C$bJuoNM2$y_>ainW#4_=uNxMy;Q!IoBD=CfN3|*vwB>yQ)g1o
z`IMtdPB)y?@<wrQI{0R|SMe)3JYG*EtfukB@Jufnn%Sni7~$A@ViO-z*Iv`&TT#Tj
zd8<YjL3h}_)gxZ(KT9msU9mc0WxhIL&fj|JUKJs8A11z<b#-&~*!^W)X?uXR^AL@H
zX=AU+)v;)e<J07hNT5R6-a>%K45QazZdaPe1|{4o&?^`J@9*6r{E6Q|54|MzeWkqB
ze_8W2tGZp>CihwCC(~TJa=NtQM2oQ6DBs}*t)`ag^NN{cR3j{^vx0Bpe9CH%4z&wf
zNGW5!sfCX&j~V||@IQ0TEU?5}11YnoVofTya1YA4AW=hv#MQ#P{^Q#1=BczeLi~E0
z)<CZ!9o3?D-rt-y5t*K4u1|hQkBt!Pr&#x#ju=Y}%c@zOf7Uk>aS^HMjOG;k(YU*4
za)jN|q#xSwIIC*Psb>4$YF763PxOdI1CtLk8AdGC2W62*So{5}B7erWZgWM|t_TyH
zIsKIZywR>c@4bEeQw7nHiYYp^VT>Cf;%qyY)Zg#I(EUR0t-+A-QmCG)P-E=cmd##h
zpa`W$p_*ERA)dsynvZ0j)-69aAAZ3TDv2-@8!Xfo`J~_Tk@D?5eK@PpAFWK<<N9|I
z<hW@}sU^md?Hk{$4eQiCne~j0JM+t|kv#Imd`rB~-Og_*uxZ!c%JMvoYObN96>ed^
z%G=<On{oUEnLY5E*<w(C>p(jX01B|x<_FSvuJn)dq-<&~W*N6N*eM?(JNh;q;uBu1
z)Yk(?F)jL=qZ9Z>nqy9DoO;Mn<Yye}9HsSjMa(|UWcpdUkX_=rUOazR&@d~NGr<3=
zWZ})A@yA0rVM9xqPUy^5a9+}uvth&GpVhSM$2t!A3P`a(j4(16{Wq**WBQ$5m)3dg
zd}{Gy`VI&2)`C5Xjtp`b1O=5wzF#$S6O;Os>WjqN)X$&VA_p;)Etn7fX6oP^u%_ch
zjp^S$ByINdbMz4P(m$!%O1phD$GhN9j2w&bEdKGlYOFcEBh68k2cD8?^ruOp!St)d
zqjzMoGptA1HF}P+vm*~FS9u-UMvlI>W*$ZN?O^@C3nzXLVIFfld|;qBy(-2K)P7u-
zN63!yj)%YCl?@bQ5ru!>y1)sVkX&2I2%ZoC9nCt7ac^Bv_&3>doutGBX*^AB<Wn?V
z$>LPQNv{CUlV$%`B9l``e!r(v8>mQ97`~;|jM(hX#GigKN5h|UA|b*3406qvl!}fm
zVC)1_$9q`M8+&cb#H}YbJQfq=1ltWqXYmdzWI-qS;2Xxy_hWHCfp18rC}_4odl1HH
z!=_*+2DDov>uDKi)S>RG!f&wS69M&9hX|kFMnUrckKWaCc{XtiBwnb$6ZjZB0C#r}
zg*l$y@?8R_?;_VgMP&f3%NUt3Yl7VmpkYZ*`Gg+~lR={$|NGMaCGY<V(*Lhi2Fyt`
zN27vu&`92XA?f_*d5c|V&sf0!rRfEhf%{n88XEOPeZ*}$3!AVkZg_n>;}ZWY>uJ}E
zw<#p)ql5RL2V4W&aD7b60Y>aTVK1$$bT4&9LvT&;8aayZpVNAm=c=V~jv2b7^d<{y
zs!Cisk%_rg@V)y*bFEl^Gb`U20xTKZ=H|^*pXz-1fOvF&ifGTb7v-RXU=$7Cnq<&<
z>GOS5QoytglZ@{jKxQSZL57QFza2=`_@bO}t1{{fG`evG&3+dTeB}Yd8oDpR&hX=x
z&9}$|ZIXW%=nPzCdaoqB4b;t^|H5TduUiW$j{Yw|-=%N@Ahj6nqTj5jfN4=+H}zau
zpy5IJlx9seF@^@WLkKKNzRG51si<xnl*8F_Z7@(c6pAtP;?X0+DMh>IfB2i_roge^
zS(=lA0hxJ2&{}pm4@`t+JGCV-RQ_0N0T`xe0qe=mlRadPR)O|fHGwX`_L+uHS;K$1
z_9gOf73u=;nY>UUT@vtWNtzen9A$o1(0pRH!pr@YK8FFZ<8k9PHm&?eE_0q$ltL-d
z(a~ST(3{2q&XvA2CH?SsYkl`~E$~tQIXgXQ(G3C3i2ACgV?z^<_07NYG;@^VIUjpp
zc|f=vW=oxsW&8c29mPM-i7?lE92+d0_WJ}wjj+}M#?`@}F*VlbU)BAF#-SNp$#iHw
zd{MEK3YvOE@dXd+1~5o3bGxhos1*d0<uGmEVfJ!hVXD2%tY_)$PjNg`nuo?iK^yj0
z9pSOjsv;~N1Qid!qdFnKCH-sjX8lz8<@wRpUw}|FWjrfrGtKOIepK1i8P8?hSkiaG
zc?T8XNi|YR?+>d<fZbCAy4E24cmV12CH-~C&CulsFdN!tvHK8utY0&r3sZ`Sa}*ix
zn@#|I_0rqbcCx~p)PiW%&}O1cHt;SP9l)3%!F4~=I}gC%4Ltswz6@^T21vc3!UAph
zFvm56$1ed$mMWy6#aIzXYaniP4`<ys-KjDSH#D!)<@=)FS5jPNBmMEk$MM8%0W5$U
z{AnIU;QtOkfzjDn0t5im*equNJJ9&+zcTz&8u9^Z%<NZ+%1<nsU*yxoJ!Lw457q`-
z#V_kEM)Er$>BW3R3kIoBBUGS$ohnqY`Cs9S1Q1K{m&bE+SD9XG89}uifrq>_C7^jO
zy7C6#sDZ9i-DGZCn&;63**I?<L=r!+MvM8TX^0i<oqU$Gu_<cEv6yf6g-`ojjgZ9x
z)UvJ8M2!@LFIUsWQ26l)xao4muxd0|!<Pp@T#GtLF~DFSNkYJFI-e7Wu=3jTfc%yC
zh38;TdAXN&pn|FBaAQR2yYBaa^k#tRw&aUJRr1P#PiAAB0yo^OC$8@$$2bY32ev>D
zn$9C8!KOse9qn(rCe*6A!%HB`n?nx=TJ2i>^LZh&FNB_Tygjp=6K@m+ZqgzCcKK^o
zII9Kk&00JGH!v@F&~PpdE^0KBTHI`ddk@gT9Dr#6w{yr*j2$Xr3h51OCbzApHLQb^
zP?kIP@sEw0BLE~oY*>DY8f2pc!9$=h3v4V&VD_enj?HQx0=@4hl%p`gh}Z$3S6*L5
zoSj(G@!fvw7d`#>fEM6QRN+$r(5Z%{rKM{2;1Otq*bW3!ErSCUV6A6}y~KQ&kqm|?
zNg=ILjAcDSaT+sn=U~b(haXGrIj;1Us)=6%oR;OatcdSb##;b>%joMvXNX5IC=Pq>
zqD}^Ql+`?C+hk2GQU$iH&u=vYL@Hyj?vL#FqP|Lm4eXpNfQ;HYsIwgX7MWyIs{ruu
zXn-+dHwm_a#>3InK;^eyJVzFYF|99Xehr5s5diuuM>dr7D1I9-=gqsF&7YyDJ@H)N
zHV}&3R!kkf_(^)Q+eV(zH`bdK)5n}(J5xwl6rlMufenVnnedr4U;?Zr%G&!4Mv9Fk
zSagb#=aQaYYChTpfK?My#a<^17&Y^LIGaMB0K5%{Y>wt#h~H{}-xB9xdh|Y$acGUF
zE9B?gH2^_EQC0l(^`LPDfC@{_nJm%g!m*mE9SJ`jER`_pjwyZRBjU2%D^1@pZT=hV
z0S~bPHDKd<4<4e9Hd<uJ2Ig#%K2nM4!Dn6Hiuw<>-#4a+Ss4uH!v;OdV)(|4;GBo1
zUX=|8fDMcH#}i==bI*o$%A1n_&ux!zD#6kzKGqFVTZX;&<zajZt4rr6T8%<Ie*BS7
z`6o~26W*qlzVbT|+88ZNng+;i6mni`LL0z>vH*v2GcF@Z^A7bwgR+9E&&&OwdjdeN
zSuxT5VtWf=*8vmCbxx9n4yg?~sHKugz5)AX^IRwmg2uUR4Zu%rHk|sS<Cf{bAoVK_
zRM#KC_5f@6M;m0X-ovJ3p-svES;g@2zGM29USu`daq~0Y8~VSt-flH3BBGO@d0s66
z7{<!-tn~+(;+|*RqKWaMVYL9NMg!<-_Q^P~t$uXr1IBApRmCuXSO)W^Wlq_MhOh0v
zbURq11$c4wo>aZn_Q<8rufq2fw(_7!d-ea>phIO&S4&RiGmo0KYsJk}O9q&HRd*?b
zJ~63cloU72W>?L5>DM}~)8UY^e_|`H*Vt}2Hp<h?4Ie8|xrAo(O`*LOm3FX@K)~lK
z?6@)>RPPDo%>fAT6iSo(PJ_#@L<M(`N08bK^2kD_96;iA0ze2R9Yg0>pk*gtNuERA
zPadoV$Zxjff+xN74I>xqHguUWQ`^YbDdq(BJGP*>F)_OC)^W{TqJYb`f#}IaE51Yw
zKtq%|I4S^-8?)_8Z-lrZ4h*|E-YodruRF5{B<?wP)q9-#-6T2&w2)obtF*Y&Fi|m(
zi%QI-8UtyUcA=h`Eyi!(HGs|d*W41?*9tHM8g@kQb3{)-1X-$1V3E0iD^6b9dVYrU
z7vT4iM2!wBOW=IdEtP`ME(ZxTn!G$kiM8wCTNl%hHo$5aE$~tS6-aXdR1?DS@775l
zDMhs*MXsr-0hlgT<I&Z?Wd^?Y0Z1e_rnbu1695J}0Nk_g&Pe(kqXst=bILwIVs6O%
z{OYM71c8<Zm?wg8G*ACy!pocP30nMS&VI-ziivt=r$mYH0x&{~c^WyJx;Zv8wOII7
zccpfQ(9!(B1RSOp^{^D06cgl1&egwTP<yDu0mK&>k7+=%_tj~JWBQLDB)@nR9|{?H
zUph|!#KNsY{~#*2cXs5;J-`wZ0Fje)q7MVsOl}^)4aKI4xp#w0Z1NIrH=f(%=~d?M
z^LrCw#x&z_)zd%`l*H9Zt150`dF9C0XY}gU(2fWJ1>ld}SyUVifrT%qNVKDexy-uC
z;fbHTJjb#7Gg>HZEro2QEwPyq5rv!ge&*4rgIkLpLHJYHa!7Lqu%QALP8JHbGr?jN
z0auAx59v7!?=q*#<Mi**>@ff)ojjjIjxCF!J|R8XuG`uC{N=xq0h87{5E*<*Ql0{K
z;w`WfY0eKuWN-OiV?n@pM@LZr=}-ZTy`Y{e6^F0Cd_waLb*joX9#EjOfANdjld*~E
z17Vtn-8ayARzUE}W^Qj%qo5h!texO`7X8Rph|UIZF-P$;An@sO7PbPm>`ok~p+$!m
z3;<~ieS5)6{xlrGB&G|*!)gB?d%bj#@-kHJn>^$qz)#v<Z#W8rFzgms;Cn>y8BDx2
zu$};{KP3<~nT?m2K=B)sZgd2se+S?gL$<wCfIW9m%ij)s$5%VwEFZ|Wp89@tf0~p-
zFB^)QXvR^K1Z_!EX0~SCcVdCx>JmGirl%Bf`jniU{8BJ^6r5%iSh3VUM9>y-DXN>9
zG<Dn;e3gP1<zFb}z1m51+KO5%L8uf6p`31xL;&DXg(Ly%*B2*yLjV!T)T)aWQa%gt
zkUaZrkRizXH-prHai-2C9=N40fH7+AT{>f)n*up*85$U>8f-5BGCF0Z?btO}XMfB<
zz@ky(u)KTgmj(#z5<$FZd9VrW8#a+{J)#uEP0~hQr)Cgpq$gts6u$r3@Bn(oc}Kg*
zfCYq*u>dkv2EukdyZZ?wHmznX-=KZrYGBT*_f4(Ak+Pt`<u$;xasa4}6TV9TqOss$
zeORs=^z(l<Xz+n`&JzG{wl$syU{NRLdE5@(<rz15A)dSw1_8MisIq7qVawg}UBre^
zV^eSS{U@~<Bu+=ieR$8saTM%P65V7&9|-A}0IN$F*5-hg5S=Kkb-e+Gw*;(Eza(!2
z^XPXmzv)#f+BBZNaTkr9{mbL9BoELN9Uqsxg@gHbz=6<LQ0@WfR_Wj05k}4wL<!g3
zWUYrbsG`8a+N1W4pTPfrgt8Sgj4CizI32$J0HjIJ^E4O|=q5<P$L{@_&xjUzN3cF#
znst}#aa%ug{OeLHh$xsp{Qbd$9&>Voyg;;Y6&PwJAm?9GMjZhjVh=)WoEFF)=`Ztd
z&fg0FAf%)dV+|Cga5fwawXy9(D|YJ2K#_Sebb<4Zy8jC!VZF5n9iNIF#3>ctqRtzS
zK#Zo?Xr~BdGzMoaeIEp^s!jxFr5#k3)YB6{h+=kvPzfbzD1cG1JK!ypq4|UA=A7lY
z+MlT-%Gr2gD*D+u-Vom9*6<aa_Z}xfu*_=+M7YV@dytLxtP1c&BSY$ai{KK3%0L6w
z+z2sRz+dGHn*nz3R%<t-S}Hq0Vy$fY3}!p`@@BAdfY+S7b9HrYyAGTj>~tv_Opy?i
z0aQE)&ewd7OB9@S9vEA{;rI$P6g4aedz8xS#N@nZ?PnM{c%;)&J^<Zy&rf|0(wJ#1
z<s<<W$Tvz{9z+`en4HHRCw6)IRYi4Dy5wNL0gB1Zs)pa84Y8#UEdDAbH}=?jzcvyE
z*%PBJ`5hw&wBj)XWWhwIMt)^TZ$)5<?2{lXALk5s5$qN>Ujyj$>uWx^*}MN@S8t}=
zsV8Zl>Y&!B&92ile<Fp&z;KPpJi{<EGs7`c7=TLm@|GXAJTnL7(2VDtYoJTK=JBl9
zYHEr*e8={pbK-MC%4z@X+m$NKNWTDjA%a^d1iQC=gYc<`ntgAZFSZ582-%xH6os>c
zICub`haQ{YmapU^yN%y07Q<g~t!HXuu}N5E0BCVKu(o$hH6j_o2X4_2V71WSQbbX>
zg+fjM+1#%nB6nIJVz3y@yc0$t97Xg<{-u91Ee;LXaVFILZbjz2KJ?yZy86l1R8=kj
zYJLWw&ZKby&%wVFe!LYRhsI|2`aar)%KXLYzJ}d5<EGTVOI`F$2oQh<U3uMIyp2MD
zX@?53hb-VF2LUD{i)_d}KC|w7`QX2Oavyeq52!2LYLRBJhT))D$;<-;YX|ueAC%k7
z4ojU5AGA8cDOnU_m<75S%1}{h<~e-D+>g`?v<ue&84iGF8w6Q1i|5(F?9<!V;7|gO
zZY`L!O3B7aHd~=KU0>B%=~9R|eaP1?WNotgq7eOavxCkH5HpkjakxREk_t;WJKj+%
zHg00Pk{kryI)s!XQP#odwcjnw&rfc(wBF7CK#XRkIDfP?<-pv&IZ@7HJz1e)_q@np
z;!BMzSoJ__QS@2$*h>bD*)feGgPJAv-o^INAb!073>1PAW)w{PP*RF0T-1Clc2Lz&
zrD||o#oPPDYSh3A-yF}6_FeV8g{k|g8Axp&a0G3REr|lwiXd#wlnEw!!ObN97}G~m
z7gG`u@Sdv0PZEvB?&8GWVy2cKd^ijombFQ@5qLyGn$=D^Ot7(!1@pSlErRo#w4j9m
za4-X#I+NDGD-X0q)Eh<+P@xz}%2Qzm-1h}VpP}7_*5!DG?TPtid*F0bBk5$Ziytl2
zKy;+BTk0efma|&zPkEjxgLMW%c^0{F3V}|tvRkM$1P@U#BYt5nh-1n~q5>r8$RxiM
zanHn!v0^pk&#<gV3Q;nWM4jO5K4fT!oy8i*RgDNLu@8b&Tcd^KhwH;ZrawQL|3nfn
zf!p~I`0QeS@{@cF!WK+}8i)5F+fZiH{+15%n=Y<MV4Sc6vT1Q|m}|GcqOvhr$;M$&
z{ps<WUU^Wp$pU&l#hu$pgz0mO*k`xd9UPA)zAl;P75d@!OcF5ZjNYe8uYpr~f`JD3
zrOZ^~c@7tj-yAP}^}IlvW6_lcr2KrBHHcf_xO%7rR2HsCa~c9Ww4<Flt`7a3nY#Gy
zI1Z)ZT(w*PYJSH+B$5UQkUKHr>Owe-j5iQqJ2M`i&(A^CBV5I;6vrhzG&(#$pAVX&
zoh+C_goxdF-|1;f;JO^H(7jDA06tst#XK5xnCHYojw^Qo0<g{jGD1Sb7xm7&mEMHE
zI?f`yFz}LVaG_lZ%{EL_6t$Nir~jQHjj<?9S*TYTRiLx?H_i?ag@%9`4aCI^$9F-=
zEOGkwYmhtL0~It$1*#zwpfR_B2vVohVq)sXHiJr(*_|s8jjLZ>oGzzQP<!n9YL)vt
z`QO4EPj0?EKVG)bfuZra{$1j;UzEuQ5k;S4*ru<KCzd2)yviz)m*5vxJz`IaGguU{
z`!eJqKkJ??2&0*Sg4R+~_y7V*rW1`<*-}?pO+=l<>ohEm6eCF3A?syH=(Ag*?Mhoo
zGtch}@cmTqaz)$@m_fAZ&#00p1H$0HB2;L=%H8taeYAUaXqhPD%zF4|EaK#q;#{NW
zvPQkGCl^fV&Hy^tJ(olvI&d1FG(ScIfwL?KTyg=Xq$V)%OZQtsNZ9lkFXfqmIA9{^
z>`woh3CnGP96nYw6|@BFZ;McXQ_cgzVqQF!!hX`shFVY0ZTqub6lJc?B?591I>jyF
z%|<2xZy}4x)LdY+x1m#ytqqiNNuwU%gW~Nzr+1|xC<R23+59WFZT-|INVC#rCJ|`#
z6Kc<c$Z1!YPLY8?4@22)RKOuyOqlE6fIvJ-R#}W<1V0E>ET*e@fE~p807L-w6ELH<
zNU@{PP-lrGm;Z3^oIsX?7-e=ZIUapAg0nsJemvXJLRM&s;se)G9Kg=Hm`8sJO|Yyc
zD-`eqg<Q5N_^rkXT5;(%lYBk`R<i4sG@E~BQIm-vVIyq@WI)3N(s+vn^U-azdWL3V
zb^v27Kj*QJyWE?giC1t2iXaVP=vPl}p-DDtqZ!;EosTUMKi}llUnWdIK`9^tjI-#X
zogs{C1|#N6*jn{5OQ2k_92EtNgaM#<!Z@tQc+fqI&;U{)OOP?8umhS>a9zs}Vaxzx
z@Mb&Ihz>s=E;JB1a@7QnUR|E&NxjF`@BxzgBmOG^I;03zYLZhhXCS%`t<SAeEzarW
z&~GOL!uZ@Lh{8Zso(Et04UmWv#XUtB)KZffRg>lUEl1HPfMkI`x6u%I22<ay&4KZW
z;W8E%KCM^JkV2Pg!MU-Cl8BRNMqmMnZvwgg<!Z0X+GLr2B{^Vq#{!sW6``k;2>Jq7
z+S7*thLP!lOmL~ilNUQ8Y<HCDU1$h^;f0}o<^FV!!0wUV*?~C=pMdSG@*Nz~>;&Go
z8bFyS8|EKC_mFH>LdB<o(HL!oFr_dlL<KM^f0l%ziTKiHQE(V8@F0(jKyI3sbiS(t
zX*~?~HE-!b`p;)I&J8(k*>H-9wp=<3=tLiM(Evy(&1&x!WP)N?=!44M^d}3OK9RI}
zBFO{fpuKrdhyXMArpDI~sw>pFAZ6~Vp<vO-#$>_V2A%~l|3rQssF6dX8~~qQ2L_<6
z31EkRqW1LoNc007Sc>M>0wF5JG%AV~8u0V9f(?4rcC==_pPzZTvVpQveg{(c#>!-1
z88o=Xia2k`{MR;nMuPN&BqM=lpXMbxI1cYuI#mL0c^YQs4=epCY)vI%9w+J{__JyB
zpk9am%17`Yv%Cq%%gjQP^<6-$l5gB39!4n|7saTuQM^Wok12xy2AJ@Q<d)Aao!!Z<
zNlkt?$P>Q;XINGz;eNEa5K95Oe?w?V0>s$?%btY}<Sc<L9|oy-p6mX~Jr?j=GAO{?
z(F4)rMwPK6*o|uj#4!iJ<$PWj2*uCb8OidqB_`6gS-#%?jzo-=EMza$UvmzM2YK?5
zbbpcR0Ha^UgDw*t<s6ub5^pm4AQYLGpeo@&TmUM+>d}lU?|@5*VSbi*OOZATgeJfe
z&&$8uTWtRgevhKoaW#Be(;VbUWlU{@_~AgeJ^};C{S&KEl13Sg)38<s3aCKLFa*W(
zP03KrP}!c28{(Lc&qGvp5kzGHDAK4W@|nK_ynmjfNWzcbp()w?6iATO^Us}%^MwER
z5T@P=34*4tASl1bZ}GcXiCqKj=1c^1z96-i-oCV7`$enQ<Rzw+|C~kUx*7`{R~ZC2
z2^js7&9-Rpommd12x=HYU0bzY$p)Cg3sMO>5HtWeR9AzdvWnMTL>9l&Q3T*x5!7lG
zLV)3+_i3W2KclIp!W8PIw7@CNBIJOVy3Ly*PjYjS8v$gdyS0#k$wx)JhskbO8v{Zy
zMSBpxA`;{vvOqcmsm=Ra0YNwf$sq26DDdAuCuu%~?>mNG>0hYb<$?aqCk*R%-?P!b
zHU3*w9)xta0og;azLs4QP=(--Mu1=vM8S;d`G0w$Q38SzoCpQMS1A8|0Pf-bi3z&T
zZlq6scSpw(88-TY2_`WN(i0z5$=I!+-CvQ&f9H|ZT7Jc74rF?r&IWu(lt|s|oEElu
zazvwc!)baHt=Nc&F5Y{0TUrSi5jTWN@nN09U&x^EK>5c-@Dk-A0;)GdveV!@@JH$0
zN79J#y{#vyH#e3;&1a>ij(*GhARvfVdH050A}CtlsavGw!f*c*Oz@I~Y+6`vc4#%A
z)u#N-Axn@`{{Idgf%aTWT%2)0of{Kh`tv*%L`o;rd@G57pl}Kd=%BZtm+&e`BR(tK
zw};$(<jo;XI;t>SZw^h3^)=taLpe3$B>`0+_YY#|=xb=tC%iE6DTVR=aR<EQO;q46
zcQofBIxv*r?%<HE;kNjHZw@U^q^@;(@);)mg9dij#P5d2zy8r&#Nj3?xP=K`()$Q^
z!GzJU4HuQ>CAd3uNXj{?rJF;0JOVu1<8cl2dxCh-0#Jj%(bI`<XzsKm1b7CCq1NeV
zg@-Mz%C#TJfxv|SJEUl%Sm({5Pvim5kgxvUz2m>j9e{O1V@akPny+=*-;jdlMlksP
zS1|rhg5i6M-tK%u<>B?nt^BDhhGpg$(PoXHB_W8>LgK@79RyTYj3FnI&F-ir=d-*x
z0|XE|<q&a?Q*luVeVY4v#>^4OZKumclJ0YR+6|Fc@@Sov&gEYr#U%I-{R32+b?UB{
z7G8<C9NaVOON-8w343_vKi_KmkF!lA38#d8>k^Tf9UoNN3p^Cb=6WoyZd)G(#B&)9
zC7Oq>|MVXFsxh1x!|BdA?1(VECKNe5=BZk3Q;YEl(YgG+RXf93GX8S#M=ZhqcDWs2
zdkA@l%|wNk7bMS8<<Yx7Gx$?UsCCo=UX1}qBij=0e`6-&^OYjDCGNG*-#;{dJR9FZ
z{}Ct|qz%+K%rL)77Nex?xkjZ!_sTC0hXwEDh}bWcucC(P)twVCt~UCO=lbAtsz&mE
z-mp8mE1yv$n6GV^+l*~FEnIE6|32OqXq(w$d7S;lE3`wiUS@AAMpl1C7BV$3Q7ruN
z%aR6`l5-Q}s`XmQ_@xrlz_WpTj%E5+Z6ONf@ZyZlXLRmKXeY=2RCwVP%6mC(QEkZ!
z{te?nHoHIF&fS;Ko`hW6NDcoAl)KYr_*3o*mcD8}urjxHkU@NpFrcF)WBA>lvK^Ku
zQ8Ijp<$H=Tu+n}M3}2-iYX^Do^*}Kja17qNzaO+%9LIl5ei8)V$}Kz0A7ZMM;Em~X
zuiE}KLC>_lFoR(#)5#Rr(i|=N&l;q7HXD^@?ap&f^e;bDSn=0Z{i3|{mN(2Bfr&rX
z`|W{Xyg;v^nfrqM;WWd>vzj}g<{a|y=rW}Hciv(M>MDvNbcQ!akgX#>sx~=p?dK+V
zg~?TSjm)fU^LsC0pn7gCOOshDlX$}orFuMDBCG|%?=612nB)?+Qflu6KO9|N)NL+5
zI7Xs|kcuS={#X#CfUH|@FC8RutyF>ma^qy^rgs<2j%i^R`vNz9<BEl=>F3{zbzYGY
zYeB<0yBFcXYH4ht7g9FQ5y`?ATzl7QRIto%cV@GCYs$&=C>IY4#+*(fZXe98??bUh
z_o0cAVl=;Wi7WDyQK#Q1QwVGES-!IQ`Z|)ybo}?T+LIkJ#Ta%Tv)+`Hs2qg_($u;M
zwQENMqbA2oQkAX822_G%<4=BO-I0%hVKFGgDvLUwEi1fZI-(?ujw>GY9PE3@W<;Tw
zDDvJQ|1jk<Px6HYQ&9`OOR1mJ_Be~k-WaVLllI;2Pi$;CeW{V4Y>;_p(A%|)JVFB0
za~_-ZCrSSuv<<nPTH<xSfg6&<EuPc+`R88Ta4DA<6<4ZQt~WkK26=qZ&`grcTFQeR
z^)jbCX7?Y*QpsOEwd-#W$m;f)^~HzM$%e~Jo;x32c(<9Qa>_Aw8*fPW>3{a7Bqs?w
ztISbr`ew!4-}9Dp$Y1n$>DRZ{=9;zd7hD48QutmBV<hc#ET%NsEq+7<asA42$7fd9
zSc;+Zv`~!Z@b7SN_V`e>BhjRG!QR(SjDx;wKnm|YTH9_~KdMZ+$EIraql%m1{(6a@
zCb0iqEg>z^=XC)yK{KmOVING7C$>{zh~Iq_cI1zgW|PbM9qL#(ryK0`zek?TZug14
z>xtT7{{9ZbDOI4e%YD*{jO9(TuId<tT7BvHMOD|p@{y`K!Qh<e(XT8Fe3k(@YITQa
zOpYnd)?e!~4K=@B$bG@1bm{eP3fF>6fOqWYw8e#=BPYe^L=i8XoZ4AqzcjVoe{Exj
zAy|`4enl)ZPG5&Ff}ZoB^p!^Jim`eusj-lu3#`Ut1vSi)SMpkVbbIgQ;*G!Pvst2T
z;!>Bw>x9N_9zDLJ{Z6e+Mi&a!;oSGjbBUA>pN+&Bix@>khfT#7{c-9W?&=un(Fip$
zoVM9h&l%Aq9SmLNn%=f|BR{fMqK;u%fe$=G@R1J%d+B{Q!p4`2sKH*%3D-ogCd>DR
zU&&_aA4D3x-7?x5-8B?8p(;#kh#Cq(-`=g{rO0v;InqX0ox_Pa9$r=6!%-y+juzTO
zYNTv)d=PeZFl6J4#C^FnyJQ@vm?4mdB&cOV`?xaxGJJ+lO%!*3=gRmx#vN`aqgllj
zFV!NStto;VO(Oq8E%aGO*X4{~z@vJ98IIkaeCl_n;}Zu5gTz%O>^=pRMQl0&ulANV
zJY26`+skMVMgBZ%(BYFfOy>2|4o_1eU-f2_O@Wuga~Yr4xLR-VrF&~ZCc7jNm~8KJ
zf#z*u*++E)TXAphh;ha7{;f-h5m7p;;tFaVRO=ZV_M_FLEJBEHKHAtvO}lr97}l?o
z@cwo0%yk((|JPUaE#2}LMUCx*a$%%e*r})fxt;g{la+Dwwa+3$7*2l^sX~DuX8M?K
z)96v&;M2zH!FHW3{O)_XXu-rogJRx2!J2v+Vp#Y#2$iJ;j;0G*9p8D0iU5VV*gq!w
zOV)7Xu&w322%n~v<Khw35WFkXW|J)02+qcFs%$pXZ{j%dzCBFOHhR}99(5Y4Al`|g
zJ!`jnXvD(m6%|n7@`tmQzVWE+)!zKLkQZ~K3>RsbIsD>%68Yh#!+kDh_q$8MCqeDE
z1*HsU0PR05eq>^M+nFGHcU6Hd{+pP~Cf}U~FOE10ub@CLQLO`5wlnqyGwdYrA~T9B
zmjH*1!MN~#V7pxLX;yMeVEnF->pU}V{#5W{=rN7|+NmOI<9wR1hsK4slz^3MJHC+v
zw__@W>iRM<sWAIX*G}Rl{lcg4-1pMC-NldVYwvMc_870Y8sQEU@5P^+Tv_M|yoqBE
zsyT0T{6mSoEo?$a<$YNZ%J`j{J?l+mq-@NeI+m!A)UO4Za(}pIh#H<v@ALdUPq1n5
zZfv}UrJmNkU|aLmZTORZ^0$32{sN3ROE|t3(J)h#b19mO*-W}?rIuq!mc&d(L9I0g
z3B*c=V#{qfHGJ;SmX>b9b9*9T_R`w(d})Dx!m?Reiim2ue0Qq0kcd)^{VgHkR?6D*
zubXnJLZ`^Li!ll*MSH<qaARUBic}=~-xn;Cmq@%CITGVg5<vma=anC=TvEop$9J6B
zBwCeop~(I7ix8uqVh!i=XP&LaWj{uxJ%sxhM@vU3yk!j~-p0ZXAHlTM3}q~HxnjK$
z4J+X>mM`l9mnFLGlUC{qIr+-W)lP%@TlL+z9;obu1j0nO&sKSofP0M&%jc2k9m$Mo
zO5B|%zOtCL-4mLSTydPOGLhUmDqA4eXdSiJ4Ik`UhMAo1UCK#iw;|=$!`~0>{uEmh
z_BLE8?hWsMhInVpth-r)+#c;p2_zDHIT5BS*lPay1p*77GzOO4=UiEg!_=>_vr+h3
zPsz*Z$$_GCOh@X~x=~&YUpKpkP#OvO7=hTcu;3nB)g~;BUFS6ZUQPA1qCQIQNUzH`
z(G(5>ZmmgAt0O<O(80bqQ$F#&C{Z~o`Z!lFd4+2}u(a?(bkv6M{KM;PWz&<ig8jqG
zvBd>XJ#??rK#5Qzs*SNXPWWz3x+p9UYX}EzJg8X>u3mA-OJ3&#{~Yj=VF9Ji*6JDI
zxF?ac=+@g~!z~|VR4pwQeurIS@7`|vwq=X|l>F^Dl^E+hXJf7$+jh*m-!B_h!iZX1
z9e(}Ta(`#_$IJAq2Wn!YW6mp$+h@8rO8jZehde+9=Xqpnkm5x#1wZu}95X1cO`zQ}
z`01KhgmfWdjc6xcz2*f+{!C3Csm#^iPuZ$w4?9bPx0sHc&*++s{X}Dmm@b`D1=K&v
zaxLDAYogoTkeR-fT4tTFDLV*z*W&Mv$9Pq*d!K#M(^{7>O>Oaf5vQVHUlvjQa{)2i
zpus!X*~q-gULAhubdM@^g~~w0In_<uoHncAcLk~E;4(+YE)i9@tD_i|MlsHw4vB!1
z6`H>zkc9K;ljWz}b=C#mF)jX0lOFtEa_d(jyaNT+6g%4M$!uL%@I!ZhrFKcr09Bva
zo!l{Tz2?^rZ}Gn-t?i>=ZKs%9on<AZ5^L}9ISyGG_Cq@;h7wXtZ9=M&^F`Y>`~tF;
z?Ye9J7wy}1Uku*w?v4N=XI1E~RCW0rJzh_W{}xRzL?Ka<I_~m0@b?bgeXRs<JAX^k
z+eZffXshEx-eU!u0D@w`XMkOu!(Pp^eIc8?Q;I}q7L+lZiRm}Js9z&mIwhSx{3=Wp
z*xK-L`gU`xDG2lPXNJ7_pCN14QyF0=N@tnUNlw?3C_BbHnO#Zf))G7vouwFZ`$+tS
zANxkzbG1sv?c8jnq9FLZM~vJLF2<*??C3O>TUbcE-$#$bm&$BZo%}9c4KXaa>h4Pi
ztG>TMwW@V+wwp{uiVu5kbuuX^Jyzi%bY($80M?cddWbyx?}T>KdsE#bULJbZ-{MN)
zAF%$}-Snh~0-t1)J1=da8VGsged$zfm1M58*oB5~&r27s+v>AhY#f~ph>7EljYE5$
zA(q%)e6CgY=3NTi*_kQRWXH5i!j3KPmyrf(`_F-o#s!%8<DPTxF*X~;HXFeYlVUF{
z6wDkyf_>+_OeMujTH*VqNY2_(AS{Kbjw^Yv!rAybwC424NjuNuvv_fLGcFfAYjgmv
zH=|&5?;4E>c6DXV6=pu6<942IWxq%%VMr+$j<~}*78r@_pQP4eLq1ear6!1e?`rh4
zFgz@U@oNwxjKZCuns6Y@RcAGHcU~t>=W?;jESZanerv;ngvx)z=^mREMYQw?Vmn}^
zMH8{BJ6fpJ6y1Oc;~W_wBcb}CG6@gk`nDP^ayN1@Vm0Ni0hNc^$3KpMI#i*tvWR4E
z7kkU_c0RR}l~dX2=#JGQPWGK9()g@hkac;#(uNaIZ7AZy=c<9<cJbocGCfXllyx|i
zw?1|#57axg0AaX+Qcsm5CjMOYso3u?cM8nc*Iqm}8lzR2Z1p3ik|-RP#A&m@eo9d+
zIU?7K{6$f$*pAxrp<rlHiB|mKx{)+7)wE-?80IEo?EyZmqjMuB{?*VxfqW^FCq`Jj
zlcQ;IpCV37-z%E90o<29=`gdkX(xVP6p<RV(xQae-C%S)Ycj9jZ@W%54Mql-E*Guw
zn0|PAWMgXowgPogD@EM)_zSGUBVn&c41fCe!qyvA;MJ1xZ9ebz?_L=Ft$);^HGsg>
zt|F6W$<SSy7<|7B?4JESD&p^)*l?_OectKUQi!i?wBw#g;Hu_Qfv+l7k&Or{hJy-9
zWrc@m(C%(n(wy|x<Svfq<u#jVwW`%96>0LW(Dzm9hofJU4t}!9gRH;+XYq(!pZ!O7
z7<Zq!^z#!-yNC(;iXJL&;!rVuisKPX6~wMM(Pf*(i_J533i>;KmTj}x_(rZ4b~F7B
za+^{7-I~&+Y5~X<r}DFp(Zs=?BvhVPV}bqD5?cbw%&-6q{P!E@uKL1VX+a#7R3kIy
zLkE9(%9qOueHmd0I>Gqcb*|@4E{f%rr=qDrS_q+^Ia|Ta<_lC$;PtHIpKQN~xJ9D|
zzmMVq+$RhOdP-;KRX8+-TW70`Xx?thy0)L_+x|m#etnyyrF9IB9}r||ehSB1SRfZ{
z3KIFQ^C2kEwoN)W?{=P~`KrR~Me{s35WmH#3GQioJb7+AtGB6;XE?5f5iWCH5bm{K
zC_CT64@X6W4OX3bQ^P6h_QUw97XNtG8KE8W(meIoi4AYVrb^06F!7D}b5u(891_IF
zW>Mr+^N;rCFBK8}*rKUP8HEC0c<%nj{t%SE_lX}4``taP(WFO3;eE*hqd<<Z;-sGR
z?a&@anfI3(G<aTD3emv$h<x`gNg!8sbFIs%vZNXBq;f$y)&Rgz;{qiLrubLkL6%s7
zUU&<HBJ>R62tPz!l=J!sXlfVZ&N{XZ?%{RS!#MD%_bQTY;vK@)CfnH@_$jG07R{si
zI!-EHbp)?ppXXCu?i8F&7OH#mFZ`f;Ihs~5!}wSAov8XYTg$nu-K&{sq*g7Wk`rD(
zAhX9x8_GCK?ROkJ|5o+dI%g`X_Lcr&v^#6SD01+PlNzgDUW{fGVkc68a#s1SjcAde
zr*1}o*PMN3z2&RKG1}F>J2ixaljwev+>eZE5z`U2jM9Jb?S<_q^qRd4UG`H4E$?n!
zZ0GurTeLUergZbgD6zQn7R%^s#~d4zHJE%S4s(=KQ{W+<J|7@faV67N3m<b!n{fYD
z&u`V|RQk<^aYkQv)|-O}6F*F5{z6X{2t@04-PMQ;U)@fkw{bwwcPV<V^vBK}*i~@f
zKfE55=LA$mk39b=21~{lXL1E^tBcr^h}^#M)U(%CvlMTYd&C`B482O27D^2_GllDR
zlDdAu623p#5pY0C91S#J7vyr1Hjs7YJ=H(vxvG!9eZ5d-IljuqYFL-qSWB|gU;p&m
zGsB`M&NVWH2M5>Jz>BL5uXZY@e8Kw_-+Sr0{IsTk*><f%^vizR!Ms7$Kw@|B4ot^i
z=SfI34b|Q(PA+~5!Ff=&!<p1p^)my*QcJ-dn(pxU90NX&+GM-_RfV>v?!@2Om!xNe
zi!9p7FYN}1B%&#cIeGJ4ZEKyEYpv!TZ;#kVHNF~9cKwlQa~!8Jy~h|l{dE~@ep8B=
zD!f6mp`G!)aQmD0o7=P1DzLl#_T!(wKcb#<Nj7eEBtxtoIrGNtanzU(D$`z^gf=qe
z7FsM%h9bL)n~E;WEa>~lg<gDtN>8d`m-DH6ec5*oW_p+Z3KS2uOLqzt!&cuYt}d7o
z`U(>NF`V;GwHpv)pN@W3C8s2u9YViQa{pRP&iMENgV%tY2h(_3@s3208OFk1pft@T
z5dR;&FON-*>C0wZD#?Mpo$al(Y!~dA+wah{aopW~VRSDj$YVcMv8exk9sa_C;^05?
zo<I~5PeVDG9%_&iOdff$9PciPa9RIsZ`u|3dst2lhK;ZK>4-li!Zmm*J5Tci`UTG5
z=SJGrzZx~u4^}CgC@CfUb=rp>4%8E>F2;1NEzF!d?28=r)IEtHmM*18$=U@!?b!KA
zZ~J2(t~nf5l<JNN;}|Y@KT%<a%R$Mnnd+d1zJuwuvqC9c<?GH+?IPtkeX^;EWUHwY
zHAWiUx=QP~<#;<LD5O;WWd~}8(SY{f&)2|?Ps$z*pdy=w??18>(5`>PlRGtNPao&v
z@|LhOoQb7_1U1+_b9^tV;yPs|c$Tbf=({abPEQivwH?0N7<SlUY&8Pe{x&nn=Z<pE
zf?=1v8P8SiT=5(Inr;laALr@2+k0;#F4$z5Lkp5jQ6+N2g2wi^MdbvgtiB+SfB91|
zj$!Qz1Uhc(V%`-UGX|8cLCc@tzbv*Wd_+to&`HO|*U|70#`K|`wds1@@lD(gr*}#S
zz08Yy&jc=vs<9(d=B|~?+mnf?SUbX*7MPUcql!|R*ypgeYU-PCcN~!$9-h}VQb|K*
zLSjU*N#7?Q;0Ga{-rQ3<b0|(Ap=69N;NTzkEUjnUY`hwh<*OT@zO0_fs>zK=>#j{3
zn23`LekePu7)cv8ovpjSdM5hqnTyExRa@7DKTbt|D!xbj!9+h=3Y`hswaeL_5)KHm
zomR$ax9~30)sAN(wd6GW$=4=b+?Y@;iAV2XjWEBeMyY)>X<B-)ajda!Id*D^y<^^d
z?qX+<N-;pZK;=J^^j)+Nz0EI3Co6xlEL{IMhY_Z`#790IOd4^bC$L?4QXY~rR>QL;
zA|qe7b5bYEe!oZ5$D!50ahh%KS!G#f#;G=JSIj0`ZA0k!(FZ&FSi_IwJb5wWKavD5
z^<k#yd-snYJLikMUtSRE!`6<*b)NlY9jWV{6qdR|#OSHx#p6O@nlpnm=8QHsl<V$V
zopiRjaOBePYr_iGX~R}Q*a_LZQ?`(h#|s-?TzH&6nUAMMd@U+M>J2+y7&pG|G@@+m
zXeoJ@1Aa%mpfZ`FPju3oSpeON1KtC7s^6mzrB$<8R0~xhp}J5u`in3~y863U9}3%H
zq)tRS=nJ?Bh<RSfxS9P(ER^K55YH~ZHwsd)J{3fX9BFt@NB@_sy6aIkj>dl6M7|e8
z!N-{ijd|3@F3yeh*X(c%kHH`Otgfj_2d@u_#PqbCKwzeX3#HC`g`x8BwD{B2XZRVy
zRCGP`gno{&CD*tIhd<n%cI}M&_JE9;@^{dYC|SfG7~5_&GQzq?3#D+|)(GHwJB>(m
zzwr)fxy?P$OQet6Z919n_dLxJt$AS4hHGj|;muQHzm7dq<E0d!qI?n5vDo|pNfm2(
zzdhb`k0kz%f9}lBN(9Aw3|pE92O=MelIxfQac3f1^(b_jhgt%T+>t|CrM`1ogKW_<
zMd>6OYFNiFcd?(>Td_GhP*}9NMzB9j*?Uie*|m4y$tJl2DTKzfy@%OlPgNXxFheAU
zb#dqCRGIz9q&vTt%kRHW59M;Vpa_BspCMn7s;q{m#2Nc&gsh6hF@;v%A4scH5qAlr
zhxIot)0#-Bm+e>R*XQ!t8x?;j+Fw;m?J~kA)q6@Ax|0-N+;f~3WV9#R+rv)oJX?SL
z6j_ldff=#O#xw3Y?%7er1&R?J9za*!8(rUR@eVphJ@<H^Of#t#Zhc$LzID4PBNHkp
zAlz{5`v_2U7EG84V6sA}>@Z!Ql094DO#QDom@-0=C~+oG|8L^^5CuN3@Ml87%|^24
zTlbkZ8xOmmLpfN<Y#kvt%hNmexMc}<u@)9yvULm`3f*UzC#L>E1oc{o*#d%GolK7V
zURhRsHaRZT%Did2RnCU`C~~l-`>=-lsNEen?KG^H{D6at=D|oOi3$}~G6QzGBRDr3
zU9Gkr@NPCbD{9@mu3wbUj%5Tsv+Vi+=X&CS-MX|NXb{#jLA^V1v!BwEhzD<%3)RLw
zZG(SEVCwq+Yc+-))6!^mvB^!75@)-j)*?lgP!e<>VS^SP0n|jzXggWMYH5|TL-+f!
zP40unMf38>zwY22W4uRXP)oCy4r<)7T~cuUZ|m;=*t)Z0v^(vUO&oH40OI(Rp*KAb
z`tyDu8)zXWF{;d-flNQ{!G*u%I9Bs^P>KoS=S_<=;4KSsg65?rYN%XqdSB?6M)HY$
zUPm%v46P1n(#mhI2@R<Giua%HD~#A@=D4?=Rp+KGUQka+NRtA!K3csubIavahlV@U
zm{t7{F#?w8tm?Ix49amrs01Zyq{a;%3dECmnf{_j>ffqM^Ex&cVpv?@=dXYL6n+L?
z5KI8Q;3|w9KGy_m<rNse+|8ywdRqYX_f5Fi+U7A(n+m5C_t8}|?++MPw;!3jC;AOH
zu&w%2O)!p@D&hI`mEKX#EBEURP=Oz6UXEsVUUBkGd6yjywfr)n7Ix9~V&Kd%b&q5)
z7F<GIvE|(9rVBJ1k1KVO(A?$U@lHJ@{IT{GYiiT)paYwbY^S|8G7(T%^_(q2F#~wE
z18Rq2wY)*sIhOZ1HM>8WGaDW$H_F{ZkrMYj2tXhEBXeIhIp;W(<dYaRA<a}=e<oCQ
zk%z_r4%Mbh&aW08d?;<+#xTtbNM(fj==29=KKRjC-;%i_MhA=(c#v(GEK_mMKI-zQ
zc$to8+UhSV)w&}EMz&`CSo!h^zjx9}u1VJ&4Srp*b^C_sB*7u5Zf3Y?Y=a4k1gA@4
zU?#kU&@I?Y&VBVB|HxIksn#^!wUZ<CjZ|?DsiDO9`YuP~H^>cS)ebLt2&iVu$)8sd
zr@-u<f3u7A4J^D$1l^t`&qtUU0nngCZ?7bn9vG7VhU+}&6-@_dn0DqMQw45@08+$?
zm>Ib<71TujuA;EM$b#{c4|Nsf?w2rKU3dA*#Z)%C#|XO!gb?vte6XLcfk&%ed4AkB
zXp)WCksu--o6CtI8&V&=sPV(?7(nc8*3?U6%+R<Yj>Tj1G~Xkxf>%s`26K>)PSzat
zw3bx|J+f&zaX`eu4bPXm4q%)D@8zXI8@00M?=c9`mdYzaK^XMK#q?+TSh3OYT+>$0
zwQ5@KTeVb;vWmQ~x;aBs<OfflJaMubm~+}&5_UUWBw1!p4jNkSl!>H!KYP;TwTEpe
z;;af=rIYn$0+VKGAyD}_Sd(!i>h;?3?#{XS>vE$Z+v69lEH3s3ZVyoJ<E28P^#=Dw
zaD&MqmnvaPWV(0%Ut83eArMhdXLogC?sm15aJT$j+nAF!-{yWSyRufIcSZ*>$5SW8
zH2QL{Z3GxJR5`Jrp`7siwO5wh)@%RVd{2N%4xj$vu^mp<%s`YNM&-i^L%u~#;E3nK
z(?)ZFC;GFRiWiluB>PIR&#{i*_Sfq(v*oAVdrxdO%I!W*U2exF5*B+KHK-Wo4}SL@
zzT7j$O}}iq$}|2W?DB{~Atm|e_YI}`haXpTW!NuPF+wO9%HRrDv$YO}s=JL&8}-A^
zq(V@yn=UQZp|&Vxy2&$w6fZ@w&U-q;(BX_@SK*oPF=LpnbMIG6?UF=;!f!SaanCar
zu0NvtOSD77DR183K$L0hz^%KEqujjGCl$iJdt=_^UdFftD9M5j_j=R`Ds>dnOc%7M
zkSp(-=GE>s;GKJ&N()rf5GHf4*Ewykv7EVF6Ruwuls0^oZ;!GuSv^RS6dkoD6vH<U
zcE_ia3ukxJ$0HK_HL+LugpIN0Fp-^1*q>^;hU;=u@49?DzUj)6(>d+{{D3>h*7IxJ
z(A^XJLA}N9Gtg%=d-&(jis87UdT)+jm?@BHfmbe-n%8|m?=r!oS&Y{I4?J$cbygWP
zwXjV{L&L9~<pYC3d-<MJ!v2ySb_Wx3@*wy26kHR`f_>O|>mgBakj3G7`nO%_tlm>P
ze>6W_e|(<votaVX(c9mzCC}&U7u<x1p<$K;6{zEWMpgCZwp0iTJPi{rku8XJ47}Y<
zyFttEgP;tml3ZVQ()HZ9M1^D81ySlFt;dQxC4&a0K>v%8MK*`OWh>xk8X=5<R|iC5
zx5A~fp$6KY)o)S9Kdlqhf}=!tuJ?5Bc7;6TV3m#73)ie*XCEIgCoi$0v)_@k>z@*)
z0tGF-hoDKL44O4c*c?=Ew7yQu{kF1`qowH&)yfj-$xtKq*8Aof-&F4?bgWy#j7%ln
zjx_1WG;uqQta!U`dr!h%D)Hiv1`GF+${s$ig=T%+fllKSXVX&Tnr^nWad|@i_lqHw
zbm22cu~9c$hkM=9o}?e)4PpDYAg{#3TX$m}p~-`)O{tB&UmicUEg$U``2Cxw%H-So
zNPvY99j)?QuXO@rDHC<-b&i_3)vnZ^pklo+2`JX*NSR4E)KX7(jX1`C1^KAGy?^7<
zDv95|;sOgy=o#5;kK*lkl}}+_+Y2TOKct5Oi&DSjCdBhv8J&3FJ2>O7d@%Z6yvRtq
zsRN_8*s&&1!{FnE4Wqqn*h*2bE2UvKe^jn>ZDo2$*k!o`Xlxh{uQ2>!hQ=tI&mFex
zPjI<!=B&zZCY-L@aO<cq2=v2BKy{k}Zk*+3T#k9;W!}WnEb=QiCt<H$ZPhJWdRUXW
z*ohHO#(Q1S8JA0E5e)x`Zlj*@X1+Wqn9%mxDxRz3k1f#Q7`I#CfSqwr$xXD$xWSKY
zW-~xZlQ`_uW!?XA`vKp+U48~`ZTEhHchQ&ey~VZT3jf&={jN)Xy@(A}P&>;sjB7IN
z7L0R`l_8QIR2=yxNv(DI9Ipnu$Ktu3?Y(1xzNzbm;e9(Ip&N}|dGKFbHYW;dT{~&V
zc%^7+H<M?WH*N&QmX*e+E7+y=&GD0oMXKWqAqMSli#3?A7Y#iJKU8bxQ+>891&5DD
zBorl5;fnaLL9I^nsqN>j7$!BQB+?>Us?p}{EDJN3YX;~}GraG48zzy<Rmbo0x3$Gg
zG1tUtXO!|%1ixp|m3P`bN<jbVW$mCpnV8&g|Bg3f>L+p2nY&p(=GAzh=rLi!4|3|>
zz^Myz!d@O<?)THiefl?iMjL;7n-%<=*PVmm5ZsEFyZ5B?98#2;9q9x4H=cs|3ZT|%
zST>R!?t;FbUsJsY^C=9)7-hT62HUSJ1T)mWo{k0bzu$O<ILU@HSisU#l5|KZr*v$;
z%n4p?d?%31`BM26zRDOFw=_bxLTdu!`?#80k+%C3W^uer{{JB*R02O95Dn6mV*D3y
z8Q@hH`1lBt{3;c@mfvgcTCBlQ&|tt9cwdU_aGunWPKxPQ8u**96Yg$j_oy($Llu8V
zj7AhQBCY*z!+!hY#!Up4kx>e(PzUj@7zgd<+V#(zf`i=8U$6E<4$sP3{xg-axyI3k
zgfsl<Y|#98*B6I}ZkjLl#;Gg+k5e}WCAss}nU7yfZ-kWa5Em!i!(9_)FGPt$;@gw@
z?4sNnELyXIskbW2yZNA*6dh)-(*!4?0%fptl*&2w7ox96?g{e@@;5am??<)31Oo((
zd%=Ln2F&_X#yf=Zr(*GZj-JAi4>r&1H*y18enh+7sEZ?*zhpFo)Mdx}M$%k(Gk@Ns
zMl!zz*?s<QsoZJnS|N!?EayzO<7j{)yV&;V7K=NWO2{Z!z0R^aoy+o4`YG&c@}$*x
z$#p06mm91$c%-=KgI0;+SiZapI@~U#W4J5RWcXdr(9ZHuWqo2aGJFAc->mum)^avn
zjU3Zr&%7Y%!Is0S;izkCBfu<pHsPj~2TF;0&jY;@Ztr~B?%K<CPQATY8x_aYG+9Z4
zT<OlRs&~P(E$cjwXuiXEBZW0V|CPcZz~!U>-3!?ph1t~!rlm^5Lg=IHQ4y7&hsO^f
zgZrnyh7*c@fi-cWb#3H@vY00>+0^+AZ19G|zs=B*U%e9O&sbX5?BRyRn>g)@J7kG9
zsuy`Vmljm*&yoxwudiE^2yBR>hmjvL4E$wnrH1%emf}5TJGsgp48^c&>dEa?nE$ZQ
z6LE{HA9cTa*Zo2$0wzwoBC5R2EtA%~rMB<`UsoKCXPtZ;=0RG#t@TWqZZ%M%4vO2W
zqVIR7^n~iz8x4lID5g!C3B?xbYi-^#PBLxxET<z4jcm||4_QTC>u|NB`)xf`ul&E-
zd&{t>pFi#w`H6(mDcv1P$HJl@D5=s&2q+;)cMC`gNH<7#w=AG^NGsh-cQ?yDi}(M3
zpL0Js5AN$cIOqDwu9sfl-DP%WzB8YhdA~oeQHS^Lbl0toRIs+k0jK;C;#p;Nxv953
zEcwZQ!p3H;I&k&@-6b@{{HqpY*n0n&tmXNSDXc)6+7FBF7oMY7=Sx7}rZW7Stu2*^
zr)_6eC<`NyK!9CwykgMopP!ixxWj8C9!s;tFa-;la+-Tq%+4FT@Cv#VMhDx)`UkLS
z4rR~&HyK|$llPa7BKlnMPua`h*cZ+YoO+3It8!nop>H}0(V83=c#rd@btK|SvwKs6
zcIN7VwqPdOhponReD1reFGNk!4qYKytg!N%VP_*A&^U<AS}~d|zz)Vopb*NBAW_?E
z&OHwjs$|)nnR+*P%>#m)89)B@I3}fas);HVDg@jcHqgjN$qYu~<j=&X-^$=AgD&sc
zn@AJi2pW@IKk}+3`T(!x9Q>?cx?l};zZJ&*qWVwdE&*cqS;n?Sp1&*LX1NFZXA0i!
zC_LNb(;ITB^eZyQK=^1@NY>Iu*dovwx(Q>#>b(|wZLE-~JL-kIGcE;+9~tVoO|M+C
zGE;Bo5g+a^w88Br;W|J=Y?z4{pY<{gTXR^T(@!)ik7_;&_+97a797Xg<fw%}G%aXo
z3)>eonP4cN^v8=O;o+tr)q?A}%bVg@;V&{|-|dd3DA|~?%$Q)4w)SC}=2Cm5fi;(W
zTg7*==F6ZnO*_uF!D`%GsancdEZd?tyi>Y{6H~j!+;LJPXF7JAJrZYCWFAA=Y)lF(
zFb>#C=YE`%Qre`0-*ng4hlfgQ)}It_m56A!aS@9payR7$_9C!d|E~noD~4<llN5D&
z#2P+KEHWWtFx(am(Pe);&1Rctwvf``(u9s1GVF?1CAk`&mQ$t0N1iCw5>?EDaEo1J
z3<OEeU*8qu@qw@SL!nV_l)Otl1R45VT!@Qh1En!;opf}kou1kBw){wuJ^{=f5$!~p
zSr6L7I93+|sRdyP5*M&?jcj~@6n(`SzOIj`z~tttEAhYW>(j{5ROkv0W+C-^WtSbo
zKgTF;n-F~#X?|)ZoCnhQ+AW5F%za4mij*h?fvu-92K`YY^4*($F8E;g?7rq67OE#b
zO2aP8*9D?bq?R0C3GEB8&ZJ-#r8KE39Oz4De^CF*;U!YC^nTTSip~nD?bbrxW<cBR
zZ348osr8!61;}?A#0a6T-@OxpKYn_+YuGQ3-zrtf87n@;tbtUUN+K&52-`7IQMNqy
zEKsk*ZhzaF73wfLQZtVAS2juT?{>@o({>9|nf|YYp8EgTTk(+#m@5ovl}B0dfTlJf
z=-W5iO_O^ff0h4_q)}?-fXkvn1Zb3#NFP;@I^TVHepfGn_8I7`v~LCPjC)pMGI(+T
zCu!MJHoDLndhec2_4c4&w{99b)Us<}ZHVeax)@>J5sg0>yn9&odhW8_Iv^@Bf1D&K
zlI_3w1U68J<uJ{vi0`OM)UYePyNBF-e1Px0Ed=}o7E0hAT|xV@g+>i7f6`!nj*s+8
z#4CcbS+oSsVF6!G(M!4KnZ+324$9SR==pBT7Kmce4BQ>>?0~OHkmv0?9w03|z=ZKW
z2}XJ=fJwRR+L%!0N*&m+3_ibpO8)T~N=$fhFd=Z>fvzQW_w6M%uv8==O-L?x7~DR|
zqQofA$6)?-_ZSm`vL!uQtVIPC!x2p|Y|4R}=Wd~v-1PuiHZppYDJx)bnE+0@fWG@=
z6Ze6M%^kH=;VV7TQ1K`Ulx@NfznK^qDmBxizskJ9ms~a~T$DDxEgK%FR^Te>UO_sJ
z^nP^d&4bH?&hLGQZXGidplp52U(SjpLB(KY`|(c(qp;JwA!gzI2RVlLWBfRw09Fn`
z7VymuQDsh0!%^7NesB<GVchQNGJ~=S8njuVfr?RQWy%^-2W>iDARO`Vwru}jZ*q)k
z9a|I>5mIFC>VKR6|Mlia-fRDTytM&+iiAg!kp|h*{`)BUPe=W~nasS~F~_x?i^TSO
z7TbN7fJF*v0{>I-?A4_tVd}jpD^yi!;FWu8{x>2k)@OK!gZxMs19C&1TW$8-e{V00
z6l5m^Z$!?bpPaC7e<ko96&wuFn6>~T8s<xUCUCj<@4B?cGxVznvXk$HA*cTaRhLzY
zmR2#nJ>ZN2el6gnL7x>sUPrl;0$w2N^$impkD6KXSv0s@^LP0;ej)I5<@PQWLoTIu
zUoWRvVc_mM{{)6N;I=|;m^+XC{pP?B#!hqniJ20_C@93Qz?UCS{ChI9Ifjvdw9ElV
zHLyv*<YSzg=pZmS+8nYfZ&#`Fy0_PX?Hjz{2tCmn+{^z~2xyxB(+atgd^?eK$Nr{Y
zkzMsW@A6F@jK)x!=##?y?wnUfW#^5fyy0;4dY<X|z)Z=Tbsx}86qx^73B$m!OJz2G
zLskRbdjDVn-_f70!+LvIOZ0p|@4Lq~)b)6SDZ4+~m-pR(RFFy*=3`1r?GVY`?@Vjs
zO}eV-O6DJiyn@YwZccY+?Dv5^#p&{1`xoMXVPlVQ-lZ=2^2Ij3?S#PblpkL{=?|?C
z^UYW;v7NACCtqexIdJHLd<4p7KTp6xz_(7+DVjB=rMaV@CjlueNNbJjmxT+jRnwg^
zr7EW>87$Z9$!_JW=Mj1D{#>Y*9=?=|=Fc8!EKAi9@rvxfar2uEdKhv*%tJA{hBr%l
zr;;PBxy`N0S%>Dk^+DO2V8ixM<?5-U?17EiY}Qwq^)tBP_XkoXatP58Sri)FFC%#W
z1dW)oIJw{K+F$+Hb&d7ZZHVVi7JZ6GEgESpjxj)uJC3mCa$Uv^*mjb^ncDr;DrZt4
zAqv9fJ#}l`&azRp_1WT0UPqEU6NjYP@#XMgG&(HqG=SaV(X609PS|hMS9z2JCq23h
zpBoN%U+^lNZSF$AQkn#EhmRv|cUActnvv@os^F8wrRYjYkb!8AIz`;|CC>h0w1<E_
ziN}RY)I4GSUVnb;W!}{4=ViiND-k8OVPo|(`{VG$$y||Zn5%?efa?);8kt_pKQ(hk
zHscrc7R`41Ehl1!Wy25s0%+toAna+KxVtI=JuYi2F(E`^dZ-_S+_{NaUwz7u2`O1M
z9T=D!S&6C4Yt2I=;xRD1{tm@K1)6}fNyPPQ*HY^jJ@I^L&<e7zHt}lbl#{^c&(AIe
zjzFTBPSJlmj#J7dF~#-IpO3IqS<X6tciSYdUK7%4rk+L5b8l(5>+aPAdfq9KKQ%!O
z^a?5pS{jk+zw}QHlu`7P9WG4@jOYH5v-1FF8{VOeefsfz7JS@X@+SL+t#ypY-o1(>
z+?^-e(Jjlf#S}y=PoADv$5ZSKWi-PVFZ<TLOq2s>V<-;ZwMTzyyT>txPV*`SN6P*t
zf<ZRN;yk#^ew|boCso4p`K8m<a}A#4oU>i#Y{)aH06*#H)e>vv{^pyloWfP*ktkW*
zrLnX7EI1?We4R^i7YF(^8L|K5e1{C3s#u!R%-x{RG$CljVQv8g%r8I#nBboygvTKg
zFfZAhsDSINfs-bSU;4r>(ruOUIU;HuTegPIU@zZSk!&^Q;XdSks!_y~!2F{6`>*Ei
zWnW8l`Ub33v!zMJ%2xb-Xu5deOT=Kj^>&10IuzZ>YUro#oAuau6C#n<@8lx5`Id=W
z_k7&aEcKf?Q(a^pDKcLfN7zO6Q;K}U<&sO|5^B6LV5{vE_c#a~rxEjH$vrh^3vc+M
z&+bo@6WMvCT=9E-eP${D*v^Y)y58m)0(-j7J#{t+KV<k_Kie~_>ASoBXROpDj+$57
z;JTV$2-}L$a@I6`T14BNl4<`~Prwau;tNZYw!1lN1lk)%@p`DsrHR*$bOsNm?hXK@
zUDJ@?cEI-O*p<Y%=N#gm!KDX0=ksf%*O`CO69~}Fq89U)2jjveE`3D9-z=&Xg>;^#
z@2-SJVT+G>zi9g)qK}=z*6zL8h&e+(7<SyN!_$y5Fn3JD<Sdjh<<PXZI$F(NtbBj5
zqdrqUNrdz`wOolmkXKl5G(4Q<%vZ+!82X)uS<KCLx+|Ol8ey?kLoVDGMOs=KYN;=u
z6X$kEN4F!{CI7k$Em}$8yQ2fT=ahnPk{9z%qp~gwXXY#VRQcR+5trU*4k)(PQ6(kG
zs=ZXQb*sH(W`+Io6<h9`-TSYc4>!CPFJ3e&aYHMd(2tAtwIvES{gzdWo+{?9l8Y>)
zR-ZF83|F#*r@&aACx3BO(p8)|@LaP`QPvc;V+htTS?oT5i;kr?2$hPQa~!=Oewd0@
zvr>0wCjY})qqZ7_Q;v+djQP*23o$QMp$mg<e%JHGdKDPwc!AKSj9QXotYO=Fj<6Pg
z4e9<@K?H@PJtyp5-%Z5XE)OG8l<gv>;wp4_R@>ULbr>t2uTX4E6p4q!rr0zK3~y(N
z5)2Efr_Mi48se}ln#Bk+HQru^M!Ig|*4%L2*AKt-Lh8$RN!3YtA>^*x%g7ud%|5GI
z&T!{1!a{E&bL5n#QC9TO_1J<~i0l}W0xT6T5Z|W87HQ%J@boipzJ@P$2=Hw76dLc3
zdG+nTEUjrZHqTXRHVLEtrsp8ROkEQN`N+H<Q(I->z5n$QLc4#_XsIW~GIakRvD(4s
zbfT$HH+G7$w5!qxBJO4@J;hv{>t`=sXJx03jJ=d8--v|Hv^*AHZ#aju#hmf}YOT!(
zDIEzowTKh{=ZX?x2=ScJx5QQe+GSD;1Oeh|g0P!a7GY?AR66&Lk`fV8^G}}Rl&5=M
z131pb#D5ta7WAThdYqfHM=;wo`hZ>K%A)VwPOVTxN^H1;Nrmk~?6K+4&Zg!0Q0l|!
zU8vX9Q{O~2S|zd4oQ~br-qiuq0iMnG^#?i;Hs&XFT-M)!hpPzcg^11w#@w8iiWKa1
ztFpS35p~<@<E_x`;K^JhHeU}n+3)U1oBQ)dgJ)ewGi&dPxi6+beJ;~`bm#J-!FB^r
z`?15<2mM}RB1^CWXZywZG+RAlp?O)a%iO8Uu0@LuZ$Vz2QOIZ3-pkKxvndp9yU-iO
zcU=(yGTd`)bCvVME5&>B&F0jponbdh&NyT@+crGd%|>I~sS(S(+nt$2Qb$Gk_Q;K%
z!z^vk4_a@3D71#~YSJ)TTEnYp?t3TrPWF(IB0N7YGRZBfZC%C01{Agh#SOAemnhB+
zjw|W+m|tbbH<O{S!cyQrTQXFtQ7h6l%a%-jJ1p8Ze(KdfyZp_?L?M;-+jNUrr|#67
z;_ynqc}1Be#Xrlsw2kJ?=wL(l>!IZBRbTlM<uMm$EU~eB69r~cW<ciEQL_7Mw|zP>
z_(1bgmXO+=TK1^rL1MF3n$y9?^8F8|6PFc249Y{wrt`2q?MLUE<9W-eoI69q)1zv3
zYnz`wE`8<oQkU>eRR1cKRS`mG%Mg2$Bi=vAgB!P@$&^M!1`B+0`l66E5%1BKGb`0;
z*4`k8WiqaQO)Hxc=r_-(V+GzVD1RScwwWsKHI9-RVIRE=PCQLnB2pK7CEjV&>A%dM
zZtih@wwU%Or{5@UO=`z^yro~UFe`BWbbQE0)Xn2OVvBC!ihq;u6>g`Y=i4aTirPSy
z?Zc((`Q+Lbqmx0L3KRdhC#iG@N6LUt<u8}6_JW=2LtyLsSUdH`IzYgm%5HHpwGdc#
zJ=+W|p?iC&8+6wTXNm?wVYT~LToS8*frwLnC|#I?9f7^~Cm>B!piK&cWP@v`!%CgL
zk!;5{rcNV$dr#!2|Gv8ORgDc==<aJ}PqnnBWj`XUu!x(ZZAjv1?Ptml?bwWc`*HIZ
zG|Vxf^*7aPPmaZKM=ZPNQlLddQuZ4?aLmn<-p89t)C$4|62F361lB~K;uPDsf63e<
ztF|S5-={LZU$gabQI92+@R3U^ROwVRdataA(oldcwE@228bZy0cn!`7hVnmnK1STo
zuwR*}i4=7oC(xx>U0Px)ZVWwlO}J<^SUl$lvz&q1R9V~Ox|j{6jU|&e0n-w7c>ZFD
zSct4_C$@uSw3lX$=yCj2D4gqBvwCmvESKb*veLD7q5hFP%TcD4N!V9$-b2rZHD=>;
zVCTJ~ch_)%M&tOZp4yoM!t7|pN~$}b+FjZ{mjYiTEU_t+&_EuV^ZXH~F1xmpC;U4C
zg5faAcU!2Roj>h12Z1TmLi=@ntJsMDv#9s*_veTPJ-?%^7#6d`y_m5lrR$rX#Y=u#
z)RQezIzD!@Sm9`y-+K+@e+uNetjUH9CoI8ZcN(*IJNNcxKYrLV;1wy#F*tptTKs}E
zmr+FUAb=auZZ;klB@0d7gGC28S*d(Ivy#lTtq90*oI)6Ose0v(Q;k%;8&DLn?;G;%
z@S!MGkv7t`<7wN=;yG-O^U2Yg3p&zj1X*?_JrDv$Y8)&5SMsX)@>H@s_U<})o!WZN
z@zYt@R{k;~_ZHJ3#Xaw0;+&Ow-9FDgxLmjoYS@ML&<U|hra+BruJ#!*c_J0)^E?|=
zypJdvVNWWiGW|H$KRBdSFe0`~=!S$#Olyizz-5h9MmiTqZ`8RM4?eE);S!9PdRktv
z(Tn3D@%@>NzVE~6+3ek^!pi%FupH!pymG;3bkL=T?W-z5Ro6S)-JP=fLSoVUSV_hs
zAGx*!d;Z?dR!!7Oz0AjPQUXmPS8baw(ywIOqF_Q?mx)R=6FdQxbrPd<Et&eh%1fH=
z)P!j_nFl4|nPa8;-8_}<M&pqmqbuLq%qWA%cKM1hHJsP+nEG0j#(kz);G>qno0BYS
z27QVZ_)>Q#;m*g2lWT&>3R^*4qDDOToeg(<>PZcGjRl*XGpM1kw&yuU>eoJTypsM0
z3q#d1bm!E2cy$tdO?Dpd0#_l@YhX>418eUY^sY~uzN&~Xh`H>;((1cVz)Qld8M*cY
z4wrEn+?)IPEHc@9*2%wT&3@5e)Y(#r^Q2Spb0p~D@D(v0%&A&iJA}O-piIbRSp`G3
zVvX}-^c|bulmky(O06;3bsN*;Y1GBrM3}V6_&kv!$}HUwmFhk3?Xj-blV+xcOMFm&
zj%C*!!VizLkpd4gF8qHGHJ*GL`cLggOHG!tUj6gao@Y|yL(`AK$-Of-L`zR^>cQYc
zJ@$T3jms<vK~9qbix+*9v~fv)u0+Gk5&DNasx^*wuFTRuO3g`!!k<F+fy>Ei?)~&@
zu0Vayfa(JFraG%=oAT#ftbVv&DXabzLWwRVU!b4Sp&@DA{nbvAz*!bg2dOIEhrJ%F
z6Yw$%=RKOLdCFa%#Z{{C=i}m3N3qd-;%;YTQQg*t)5DUvip0IQ<q`!mNyN?0+z$qV
z9xpoY$cCO8cGZhixApJp*U+qyY)xiY{@xhjJaBEmx65X=K4jzuH*}To3F7;wn&LMK
zmyQ;;+Ad#M5>L|_pKzNLovR%eB6}Sa+3TPd&Cq~}mJ^Cfod3P9>f(HwRm<QmWDuIx
zJowH!9T$6dx;(@pUoXx;Nt{{RXlky-H?8~fwRP#UOlqU23YPv`F4O)j9?fqvbV~lH
zNLn5pkeYgh>93|+K#D%;7Csrt*V!gf3b$0@Zb+ux0Vksz_6~*T#uHl9VZc|alk%{|
z2&~eN`o>z9bm;Voe%PQ^Y#!<k1k{$u0JFxm;60x44c9j(Ie*x`zSMK_&?p97X$O1Y
zrva%>cr<K%Ax&`E*01hCh0#Q~PWcG@pp~+?v_(pu;8e04d%jQ5MNQZdX4+b$uVk(l
zeT{U&{`&f1;m*{qPGZ@0KLjQY1~!=oHFIPvnN7EmUM7>_VA`nf4~jEv-K)Z%FEbYC
zM-OjY`cOrn?<VwXXP9%6m>{uATgr3&WZZ%w@=qZ3qZFN?*_LTu&sT568Jq7O@>as4
zKXmB8^7RaQuJrEIx;qm#vowo0vvSADF2A?$ES}h0X@9WrQ(RF|LXIKrPubM-g3+qN
zDZ4&AP$}-@r<C$)<PP?3me!<Iw#(TGCWtNAE=B;*Sc{16uv-Dck~paY7Bjux_EztS
zZ=QOR#7_g8hLYkRQOca8(_a0!mZwgLGPxb2mfe#I52tD7H1j(x?2szwGYt+giIb{F
zDEdMp`N*p9{jl)U<<srz<epF1$-^3`v-}<8tg63rJF7NDD`6On_EE{h2JVXKc}B%8
z94$HLPx={>#f(xssy89LaG0U5C&w-2T@q&Txv6svj`4FI2m~5|s(W8^V-g_Qt>uN0
zabK6S--U{PB%s4ZgLfajYV*W3ZlJCX!NPHalt9|yD{nj!)k+Y{U^`)DZVAqBODL}F
z6A)LjebD^$pdPVhX%agbOa*&5Cj2<uKVaV3R)ae^Qf%7%lrrwEV@gu1XO)2m1oH9x
zlbcdQ{H2uhUjH&=IwU1GJ9HTviWte&dr?l>t5koA4S3)70Ztz=jaMGIO=+?Hpo+@I
zcA0<wIoC>J$GLYso+bFu&qj|eR72r@-#&~HLE$v*{px<D)g`XsQ#WFljqSNC5?M{;
z$Yb!r$kd=!3IvM*>g)y$xKp@OyX)9K+o#g)wdS`1aI0lP3s$|JYPUZ6wxk1_%#4aN
zIo-3U0$_bK2<9Xgg2@dH6p{{3iWmi+5}As;E%?%F6O)9)u&VN+yqjDswe&}Zbdq1<
z`Lz%KN)mKgT>0jz!jVpK{g3MG;yrytI7gR#*<d}B<Cr7WP5<Gei{D&qBfUwm<HRQQ
z5^F{`9%<jCv<8EHqVkbb+xqf!IG@V_ud<<W=Xy_SX`Td~V0nCIB%8*aeJ1fz?h?w@
z3)f#1zxQ!Qwq!~#d@Gg^lGM=eUUB}_CG0{vOZS-yM>BCUNxIx7p@}aS?UW0((Lgqi
z>(Y49c=h0$iG6pBRMMYetDO(b^n$okTt--rYxmBS+j=jHtak}`P=4qi9ul2PbBFit
zcAR_R_S#=Fkr!SYm*l$BG_r+v;_d`37(83*aDlnJy%C`}7Lj!W(g&k<%cbeF;^8_%
z0mp}itH7u%t{)nbOFw(+t6{!aF^~O7Kd_Ja=Gp`|dmwh|<9O=b4LB51Wbtl_+_q9W
zmEw=tmZ3+j4L33fif--IS*6(D(9<<1ilxv$z&V1bhCHKg2wiMkHSkD|6ZAFd(rxpV
zr{=+UY;G-fUMuLiE#Yx)I{Q4rjgUb$C<FcdCr+m<(tW{o0X^=aqM4<Gm2h;J&f{Hy
z%C}=)a(5d~+!1R3GjY=b9pvq$QIDbD_UXXX12C&I+|KIme`j^MR(a?ZT5{+3VI44j
zkp5vE$`C33(H3eu3k!7`TIrjvyVK_OstisYBK<<8tMn39WLd%gaM`kh6T?~fMc-&+
zbzX|+?(BEw0O$<N_Lbgh2enmICpPVlZUStdCYJ}k6%;_=P^uPP^$2Ei=Vric(WH_?
zt@utgjfgU}`HgjJoDlw!uA8qNEZO{&t&mmC-{Pms{)T9H{qEGas^q|4PG`#=F~ere
z9c-|p&=GXlv+gjx#>3g-Tkm)S!zlMQ>Kq7St2>aO(5bN@V2lj4@VbxdISMvqg5q8n
zKTa2!?YlyHF@yBCwU3QwmJm#nBH>=kw#&RrGo2&Geb2&aBnF+FSI=o+BLzm8@vRq9
z&eo7g;r9~8GB6Xgn~&BapR3MkFHhS=b|m{3<~Thl7~Kl0&LCVkKipkP2us;dm40a&
z2At@;&a%^<*oV-kDicM*6(5SJb{F1wRF1=$a(8E8E2Hq%5aK;No|PYALih>^bR4?P
z_F+Jbk`MJ?b=`mcryvDvyZv(i9I!Wk_2Z<OOKluc%jque1EvEUFdgI${%x=H7{Bn-
z^Fn5!c~2cdDZ%+_)4K9sp`*6?Ro{BB#e@}Un{5${p*wqMDWsuVh@PIX&r;d<w)53D
zCEAk0_E!`@eg&LXRG&DAxQl72g~Gmo;}uhUbD8a~Utxpw25TS1><jLn8_wYxV+EsG
zd@(l_F`cRR5j{@UtLj52H*dDGZDj8)&kQP4Ua6E6w4KjGeofiD+Qqi{G8bn={kk;?
zE>hCbUa0+NlV7zYLm=;kOMQZV?F){4(|Ym?2E;j>lzgzR2y*%xp3W7t{yn*b<c<(~
z2iBb*`kB=S3w_6v4-(kg@5I$U1he{pRQ~8Maz(LGp3K#ZF-bd4$>H~I`WExTI!cUV
zCFae^CWsGS3rhyuk;zW2giH0cp=ddUQ&sk&edNP>B}6M`G~r-}s>|N53j7)tzGXCw
zI{XrmRK|f_l@uJ`DtXV#8JHn+F-dZr$tycf*NPCQG@%&#8UELYtx?}y_?{?g9yl2H
zHbyGCh8~{nm>X2<d-9xJzx`PvLc)vP5E{Ar*)i0sKKK35xL%dM0_$o$TcXkBSFqhy
zFk(|S(oi^~v40!m@GLMvgq6##%dQdbwFZ}=VEav|gN;%DcJ1;!dp^Jo0-ax}md}WA
z*E_Mu>Q0tI_!9|O1$IXD*7YVn!xEC=@5E-OCTMSc^wbXWWn;R$dHH4Nk{KVlHgQlu
z#~wijk3Q_%?}Q3Bi+KMNoo$IzzauMEnbdo=<k2f*ar!&?_4X@%rg`sergW1&t71K-
z@mWnfn-ceQ){n(KeU2uy+A=rrdl8;4rz25Ys*f*@k<aFms9-5hlBnOQk7EZvKpTn=
zV~aKId$*=*{eWiFs03oA*Tm`%&CX`b9~QyfDeKrKPrX?YZWemRLxY<n$KXoy1%}R<
zp@laMvwd;p!9<kI+uJNKb|xjZ+C0Rus@$7ybLG@2p7>LQq->hP<=Ihu8?o02*r&xA
z^0MAPJNyprg)}qAP!7oFfLM}o&vDOf-Mvm83$HaIgG(tBqDb6Z8%5UZAB}G@uVAiC
zSM2@rTL-+Esg$_wi&V|}P3})-OMWe|N$7yXtVe0Lf}P~MKsU<gweV2L!pnKMv+~Ig
z$ln7#m-nWd@zqpPE}kgL{!TMoiL26_59c%#veJ0mEM!W?nf&Wv6Y#dtx8Mk`O!!v(
z7f%cMm3e>4iuz641Js2Vf0eQWuWR-y>*XuuLvh!N$a>ntR&8KO{KSof9I<2rUI8V+
zbR1g&DRB0e2dD^Swo=VoMdTfDMkh;JGY6>&e0aAd>h_opKQ-H8-{dfU<UF%s1jP9w
zieHy4o&{4TIJ*p2R41yGV+nAjs(`ajkaQZUh?Pd>P?NXSvKaTYd&uC3_fqk;L@#wK
z1l_55q+Aa-p1IoJP@@Ii5CmQ`6Tp&w4Q828EJ}iu>BF}I;HOAQ?aox4!R1`V4}1*G
zzj-YUj)l<HL1tnAXu8e!__*8h)f4ABx}Rvsqe_gwN$KznAI~!*Pavf^Z@*9e)4$@R
ze-lrF#E1V|;>j8kAe(voHzn!qVK1^C%jl6xj#%>H2mhDzY!p-y25|6Njw)sQzcQvG
z)g!@3&57oR=X}U2|L;Q+B<2rj$9m9>hHS0>J~)txoDK8o`u^8JIXLEh`xtpv&hd!x
zZ&uy^9G4e82cnMPY=3^_x%Ph_-t9=GNYRh4I{*FDe+&I@P5-Bd{M$YLy%BE*kbg&y
z|MPMBvs5BLX2=09PBs9vyaK7mky-N>z6J-LcOh9S=~4l#O3A`ihV8-DpUND!wI>_h
zYCBDG6_a8CIt#u}%?*@3nn2>`brwSd{wnW`>O;;V)NkIjUjaanO1WYCyVQ&0O<7-b
z+!{3mst6x|Fl2zt{wgP16M#cxv=I3_mg@~*VO2sT8bDU`wq10<PVl}4!oC3ggqD*4
z8aK!U1WX?*<<)Ax@EZ?Y08&L406gmG1BuuohbyrCdQ3n<^do@d6JEXypz)+#L0wqz
zLt~+R%K>zc$e)^vD4bJZF!gi8*JQYZUw@FmZ${v-JsG3d<o*I6y>bCBUehcc&1Y{f
zdO@??a0n14X(5oGKLzm0Fo8aC+;dFhVa;3&Mt>t6%EAJ6F~*h<y@mvU*X05RU)S)>
za-4o1;4I0_-NnGouG@bE<}38u-;Go13-Q~tX7aJKLh>Y}I8ZO2E&KL#9v{mt!{zR-
z(!Pb>zOKYdz@nK0KiN$R%!m})_&;`$NKH)yD9CF7!(sD6Tp>A5Acud1MpUmzD!>v9
zu8-ud10qGVgoi7;MhT@_sa^_5%xW08)O>@GKU^JPK{R??3T7*%Bw~?rD}<5@L~=mT
zNV@?!(`>9nH=6MUK>>ghJ!k;Dp+)`_F{iLdCV3ga=E?@Z0u_*UUR;LHVMA4<D%h$-
zuQ3rxUAW%gP}Hh;XM@!mwF5ny|0mac^2N)Scp%Aq_}O``QcC2ZG&R(dQv<y>fse%l
zpy>`Y4|mtcsylsMZZQ&<i*-nzN}3l!B!*jeQpMV3J2%ZU1Jq+Le6;w@h}bR+zH$l}
zg^Y+as?xU|%QTUPV3FoR2XgYK7r&_;1{?&&D8T!D1t_kpfPSG`swXa&3UPTJp5w4N
z+FkJ*vj4&S%Cfw{^_X2TO(G?XQiOi)NmIw^nTH<;%l8z#w;95h1Hg74kRx0P`4oE6
z(~`?r)6!z{1Ge#Vn7ZYSYzPTgo^fB|g+qCPT3MT47e;|*dB)C|fW(j_E&)CB_}*kF
z;6$J!845-aB&dTR)T*y=$_n6u-U-t=`n0|N`owHFYo9FKbm?A@2$to-xsR!;+XHwa
zjYOJ|Rcfd3nc+_#G^sBv09fz@uq5saLgt<)@MeFJD*8ZN1$Z>Hbg_^Qw!6>D_Av}H
z0`TekGof^C_E11_I9==9Hv|m4Y>@F{dfzs$Q-?cM_`{fVHYvYJe=;6?p<?9f&}tUu
zlI=yu?Zef8$IzyT7x)8+x#mbDz^?brRP7yEV?0_u`e(_b47BQ8!xXmD^MK>x{Ib%x
z&j&zmNEy{_hIefkco-Mb1uiD^0b<J+5TOYen*_xK5CEo1d<5?1w{%=u71CbRA*sp|
z=ZtxP;`#*1==DsYSa8~%Jthk<A}3&yV{8o+f8>Mgo2uU=?}+0lWyUuwJjNl>W|E8k
zdK1F4?-~!7C2M2Vm*}|EM`Y;wThsNG50Yq~_<RE;k<Xca8BQ)xK+SSUEKcyVW8Z5#
z<xQT_07Yk<4i>3gJ?%t8UHbIA%nMeXuN={e^rScu_XCJNC*Nn2lapi2rhIyxos~t|
zf}P#<IJb&Q?!_i_E<0hasM@%3chWpf@A6t<d$!uR4`RMjIV-;FKK2V@^f{o%Jni#@
z7=vD&wSk3<R5F!rGZ^y*l^8I?Yieq;ei3{HNWHMWH97i0_jEQ1K%BZi`YfU+bN89N
zd<fhPV2sAUED2K^xAjsb;g>vpqvI{fNBC`98R{{K*de5@x?bPau1uwaBp!FW9Sf>>
z8#caI?^L4_;+x*l4TS?TYTG~e9$2T;(J$B>t-XrtP`C{0izYScRKN@dt-o8cJ5kxG
zdx-msC|NTTg*`;{3Q$1ycsc<G&`yw%@##-e>WWH$-qk#<eUf6=zNy^Oq41<4&L>UG
zX~%xOZ}H%5Vn&yc)oh#b#cK<QVRoeZ<p~$)EtQoVde6X{U2!@>iwP|WtUKVbm6<~+
zCnqP#`7#95#uvI4IcfEx<1L=o7XqVsp&o(+dB;9lXS>5mY$o01cMB3M<K9)znOEX#
zh^y!XNRHrt+s@q{i)+f2X)wD?Dha|=CtJX88`g;ca2LmvTEGfH=iqs4DCS4Eun({q
z<4TQyp=*5w{+&#vSkAl2K_F(l{e0&;JwN%|U51~8SmYuv$s6g$JSssk$!##uni)#W
zbgQ45_{nlL5Maj!%pyq9I3WD(Jw?RbOoSR<7m->~k@E1d+S*+0Wvta_rUj`~)q{EJ
zjq~?$Jy^8gyn*k|H8)tOJ8#W2jJ4aK+`+hFlAKbWDg{ZtF~hZQoT{#?*N%$KF~XaU
zAsl~<0e7OL(u#!S-pBa%Np8jG&Sx!I&tYAAfXhT6wxNSST#GrB)>_Sd&zCmFciBkv
zvIV>|4HCFbdvi49WlN?hSD@&g`aGw06{|^W_!{>@x|`|uSguY01-qL1Zo~ok;Lz5m
zbkMJq7j<2VfvYxIGB<cs=Sl~@)AuEGOws+~TI05X*2i}fKcjqver#^q45fvr;!JwJ
zTD!cnFm#$`GebVvpD_B4=(B5!5g1iU;BFU3>-2oxPc5c~JOR&(eFpF~QbM)wXEQMu
zVZ$3)-%GK)_v-}2sTAyDlYHXuPn}KQby!G{rHR-v&jPGo(JPYc^G!Ta28dFM=<-G-
zhyipQ$UzR+(Eh$8A*15;2+azGt*WLltZ>(X$LOCiF!)98OnOdwUf(~e+WWR`wd0Sz
zFf=^?^}IneZkCasmVD;lzzheZAu%!XzQK&u!L+teVr;EXwxD!Qie^vGiA{1@*c6|C
zi^iu;OnujDFnvTa`i2w~fIqkrj4_%az1!Ep5@HMx0AyO;G9Pjf<d58Y#BEP#V~Qq&
z9}Wrv=1b$kK9Re$FD1QepuuLBH=ZJ{a?XP%q*r``TKp}U-PF!C-$BS@OQr7O&W?5L
z7cf`Owej2--bYQ9KSc#`9*kj1s#(*S4XsH3viFjDQw!ioo@f!~9~QJoahvra2uhS}
zM_82rm^8#`dy?4vQ{x0-o9^zHy}`AJ_l$Av6Z6otSsggtX$PRxI=m-#6QLSBg7iu+
zzIIs8+hf%xjDG(Uln<}7>iU(!)Xgb0oy8CRBeXb-2`ro$6OUA8V=MyHMm&R`iZ3j<
zF=$sBD;7DB9V7V3=g~vG$14D2u-MXb!s}ou$ckQs8`RBj^IbK0>+ZK|3+dGiC)A1Z
z*B~;`(K7iJzc6es<J?76+88(C&Eu3NgHO$8(r>?<%x9uB$W%dzo5zh2K5fK^imEFE
zYD;dqa12#$iYCX1s@wh@F3Tk1QLpirXU_a^m4V+bfHPXcM|MDvy>#pcaCu_M*j%)w
z1i$+0`~4@dVZlL+$`2RPYk6uiEggkQT}lg}5{RRM4D9poa?YlS5yjy$I2Gw2Y2!r7
z^hd1xp_~j4O+4y}+)k0Z-vj=3XssFX#V1p9;y2AV1!{hs%@<4G%ch%5vNJr`uNrxt
z>J=|3b~(DUC?=}r+q=R`Yo{wR9q8XX3ggb?freV0j&%aVARQlf|7mhoN_p?yKkbuo
z6w_N0)T1&Qn7G?!hPzJti_gqwJubusOk@)+==GC{Fqx9n)2KW)v&9uH9#$*mveH6=
zyYO&7mHf}Kfas&w7((BmSn!UY>_-Vjn?E}x#Yzi+o@{GrsSiL!lP(ZONEJ)_+EAu$
zr!8PZJ#i#|l`lZ*wLYBP-vMG;!$!7lf1v(q1D@Z|<A+#Hdz#fB)GRYa8_iCg7)q<n
z>$Z9cW40fS*)?v)b$TU{O?tt>{8~Qhp*aP%6!$9|j5d`tP-Rk^E1uX@Z<owXx<6**
z1c1}^9l}MP($_VV7n}?bi~6ePFw>bpef1!o29d0th#AV)W&<f=aTXhSn*~;*%N172
zolUF{{%|Cv+F?k}Ls6@)Z!U3V<^yY>DDBciCjs?%Z^BSoGAVN<Qsl&BrqCKU_P8|5
zXk+6TF~dMVZs+3S%2zUb?u_wO=l6wP;nJvDin)t&z1yj!JHQXMF~o>=KV#q^U_J@0
z?coKW#4Nso3RCfb=?*!{^~4;f&ptemXiG5D?8k6u7&h9(p_H9F6wmT=OpI9;0wSF9
zPQFsjXomaQ9$kLiR(Gbxs6g349{{x5BtcwkUz<2IsfB?kk%h`U<|iUW+)aJL_NU;1
zTyDxJ2scv7Vqd<JoiXuT>Wa|ue8xIEE9+h@Aw#ik#%SIh$+VNGs3PM$9!eP_`=Dy`
z3e2N$5c0DOh^pe%!>UgQll8M>Og%u?pr<eLt}qLhPH&58!tIUavOB5L2v|JQ;1A_c
zWnQEmEV^rYX%2M(w*?J&k{b5nOFBoxf3K>XtK?hc4%{4(`V(W~J{fL$ut9n0BGq}Y
zrAqWWKRT)Pr3BOx#8FQOwH6ju_ro1<ulHgx9pO^YW-o}t#_!~ZA3IEQ878XKf2_d(
z#x!WJ+~z_xSA?QyZ}5n3xxA(#EzweqXmV;w#WJwcCE&rsNDO`oeHrKH{Noi<?ub=T
zfp+b$l8>2<M=k{I6%7aVR8qj_q*TH~WRpg3I0<Y-q@*Rkbz=^mj3H15Pgc*tGXYN}
z6|^{{P$KOBEW_9|ayPonJ{^HDJW2qord3nk86(oOtE#_s7h^-yM|w8px9Okz+E>L9
zSj{Tz6dTUw&G;WhbA^`E!Gs!W-_>YQ!^y6nXGB?%E39}v6xavInRYe&ALTRg(!ag(
z1{3@0K~+u7cQcj5l`OXB<n<VeT$L?G=LgICNUB~EPJy1`@6plNFM@tOZ{eyW4SYq*
z>6af;f@CPUb?WAOw7@`CG79@b_PAxa(h(3!QxyU#PfwssO4nvCJw|DTf?BHMRc0y&
z9uGxqv(o-hnQR*DnnUraafs3J^5ng1n#u4#dc|Xd=l$|y%p(<oLnB3R$)`U!gTsDB
z=Xcpd7)I7ZI>M+Z64rTQ*5ZZ!M63_W7$dyiiB3kWTgb?`+jqq#C=n$r?5qJ4V8pt&
ztc<<1@f}x1ijHclcb?Cq_>JQZyS8HRyZb>;WvJw4WD7}<P#=*QddBb|1N1tU_q_Cs
z<N_a5--*sftP7LAkEjd(rCRP+h;J2<utD@NB0|nehB0yzV8|8PNmF3%=e)d`3=#PL
zLjd|t567x!HDlL1CROGmCro%1mc?nz^^9FnR{0mM-O<(rU#B#rt(Jt4ft=yblk;Dq
zab1CxU1z_QXS4?t%)T_hRUs^R-jZV=7A@_Iu7M5&fTv&l#a%q=<A)K8GyrQ(eWOXt
z%6*p>L_+poQ9gR1e~O?)ZInb=7`i`L|NfL<zBAYD+>i<td<+8tnB3cPpC-_LW%k~C
zemGx*iw7>_LZl18yCuK*f*VB)<I57>8X77_xSn@ES3?{MRAQ!=bQ8}T<t}*y_;>~h
z;-0&o_`$fEh;HKH?d|Cfwi)mr4#-99e~acs?ELd50}J_L+LylI)!4#1K9at8lFs~H
zzTe1nW#>VCMod66D}+i)I51<8g_iu5i!pd|N@TY4&!-LWz5Yx@5Q4&jPe5KoLB(Tz
sfwJ(5fT;UF|G&NT?@RyxTLTDO!^EaI=|4GvDB#bFXG+o~Pv83d50(S0k^lez

diff --git a/docs/figures/arch_8.png b/docs/figures/arch_8.png
deleted file mode 100644
index 16a0bfdd39e1e5df7cc1b10a5aaf72d714cddc19..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 33393
zcmeFYg;QM7vp+Ztg9jPhg1bAxZE$yY2^JtoaCZ;x?hq_MLU0YP!8H&_&_Hm{Uh;jv
z_g?K*ZS5bhQ#Ez(+}p>x`}Fy=OthMcEIKMNDhLEZmzRTSfIx5|AP^WA84-9wz{;!u
z0-=7ilaf-Cmy)7Tb9J_{bF>D5<f79ukhD^kv4c+jRz!w@P>Q7Xl;5j>7>gpQI%5>%
zz+u>u6qQR!gLr!Cuu{EX1SrBHc-WxPOahnW3PB$?^_N9&#OQsn)ROCl?=R&^k?XMs
zx7R;6+<u9j!;6={g%$JYVFbCGe|j}Hlm~r@GuDF+&r%4Y;(-53>XaUZM@kC9%F6r8
zME)Dj*1qcv-|k_-)Zd!n^Ba95P@!2Ej+g$Ldsrgn4_;lGNkou#<eo={b!_<!S04{&
zGDTlv#Vfmrq4HPe*)w}~Ij3rvVQFwZDX&Q1VuNTzU{k|ebd&*d`N)RI-eAhU;lS_7
z`T4z4gwbemSR~Ls&8IAsz_XMKf&vtc0#cRa)Yn&BtX`@T=?w8r+kb@<{<7}dB`*^q
z4pY#^`b#BcCFCwY&b<g^M8m;HocI{T|3++L1bZ_||EG=jz$91Vk9~IIoKGBP={N`*
zj#UMY?`~Mmn>zYz^mCz1@o5ZKW)obp!UVd>#FX>&=gS&@zl?0DH|0`C8NC|QuQGpf
z<0I4P)Wc?)C}6*MTM&E5y6)?<P;kX;rnqQBA9=5EJ()FDz-~2hJ06q%^Xm^^pNVCd
z+_CiX?^(U#X0s|0u?gAgsZo@ax(OM!owXISeWG%ghmhAqUSM{tpa((<upXSR8A>Jv
zB*^jUtV#$~m<z@{5K_A1h=^9jB^t`QlRagrr?0mR;?Qp5*4;MVb7z&j_~zF3qBJ6t
z{Jr3^V1*(Bd|@pJv>WLr*trf;JA<=5-ccA3;BKpO1rzQli^K60f_Z}3I3zbvb$T#f
z!ST~rieW+eBwpx&TPcW2;W_mXb%HMoLAb%JW<r%<UvbtYOpIXaLiAk7uO3<#^ficQ
zFZDELL=Q6uqG%8ba)=8SQL!Y`d;C(&EJ+<K9M|BELJ$LPOz?IQjU2Lkq4qRlB{EZZ
zb)ml+^#z(cSP;oL*ry0-8pH$c4Dae;$b@_|Ltg^rV{)70A7MuJ$8OQQ!kG*|-<ohi
zFoGu!Gu(1J<-K8B!|V<jQU}q77MC=u+c40L!sCU-NYfS%RPt6zY6(;$&c_{ydlWg9
z+^LsTPT8_#$Ji83Osm+Mx<j&KCt>vx1SK`o_cOJ7XnU)Ba2JGkLJ@K5Lzxz&oQ_I7
zbSO<PG=mt2<}4E|*(`orL~xKd=&`(Xh-w+Sv#@MnTuNKAIAU`|yJGo7AQbg-Xv>A^
zm1ZtC8}d-E^$%2+wARp8?pDKA$~F3q7ba-V;n!b>w!?3P{REysPdrcfPY3}a6tdIw
zudqXqh5Jw?QHm0#C0Zp2sg$sGqa$FiEnabkx900fIMSKWh|+bXBqhxxy?u|81Reg6
zQuSVd#ZBKv{3o}3u$AN)#b6Rm=|m~wto|(gtPoe0qxd`Oq2#8KoRRpQ^PP$v)sI^G
zZu*~CRNvJ)wm8-tu`Vy-f~5=kidl+wG)5U^8TlDXTp<GR0+<2`Ys6pdzu>s~yZ$)S
z@>aa1xOKm^xP?E{-HF6=K}AG0K@|@73pWTqM-A1&)y_DO*wx@yJFFzAlHy^Rc>lG$
zY&ym^(pIFIQyM{<WKeE!H)cMj0{4EOp$#riS+yi%I&u1BrklZzp)vJFAtyCE&56-p
zOGopa=1*n<rq2wU+E2xOYJW83wU?`B%g<nVwf4ot#cfI+pAV$x)p@hCQ;2nI)sCZ9
zq&)Q2VIo@oWj?h&pYK{D$|DjZ<VrMtbbPI(GybN`E#0Ja-uyEPUwceiu~56*L*|fH
zDyi3^l#_{&fY5T~V5MsX``CX)WCcesLJ%tGEyyaU>K@&uv<_S6T31=;6k+WY@6`5D
z_QAYFxl?+ydVG0jd`FJ{0=@VJA8|9$3lb)hkLYgbIWNdblH!fxx#I2PeiK^8XML^u
z+8*be-9z_@ZiH^0j$g@KQ%Ey-pY7X*ccwSSdDyxA&&#Z&aXlLW8)X~#pYG?NbK!Ho
z^TqSapF@+Ig53^Jo@j=#Jh6hYvgxSl0qNf9FVmfCl|HoU%<Fi57+Kg@aHw6a-CW9C
zIeqhPb?tN63gU`W^KtW|&1+}+PY$0*8W)=qKX2M?4wZ6-W(|&yVGtsDMz&zkzZ4Z(
z5wH;=d;8EoGR`=^AN1`M+>h{p;DLsi=aDNgd7pc4Ovv_!&DZ$OxW!nYj<;@V&3)a=
z>C%Z^@9PKL1&IX*Pc+ZQ`?uGLWC>(7B9lJF!v4Z&BEB8C9aJ6F9r-?oUi9Al-ifzM
zcONbU4rBjZUaudl4lS6uEyo{miurcW2M|2SKbb#iKTUy^zyokEgIx+uzkli(Lc1hD
zCHRC8O!02|Qz3nCL~mAacZ8W#h*Y&yexyM}9qKV^Ndzmp5Y`ptnn(xDgL;^H7)>#E
zGBy#xGT|-fD0ekawU|%9#lfBfXEkR?=9Qk2qm`ro(G69-^eO@R;PO|zfptU2R<GM_
zW<{-xQdWVM)Q_o67q-H5qD)WX54tK?6%pL|Jt|9C9A7WL|Nh>!{pHW77q#E(0LH)4
z#cmdM+b+p&v!6<ntQL39cGQC-3l&c32KlX3M~CJL=OX4tnk$>r-Ti$0?of(CIJCVe
zrMNA<W$K5uv?vay+{f4Q1?a8_7GF!~$2P4r*~z)hv2rFqP!J8j9jThUZDw&@p4mMc
z3m<FO&HS*`ye(bl`Fo0ahWQJ#kgi?5LASjbuft^*?{sz;Z=Qq4;?Me*2EUq|{`zd+
z&0Q{@vz#fMZ#YM-OefVRb+ZMYA{CHlqgtU2Ao!Ey@%_GqK9+nR%2!^<VdD2^-@!9s
z{m8(?erGVw+I5$*tBNP4?4+``rfH%3(HHGqtjkzuZJptD#%RXRbPOl7wFQH(49qOP
z-TwPm%)5NMzVEO+<=$QBLbRH7544@C9t_fqJU*{<wpmx)O)3kC{oX!qYgv8cc}^BD
zBJJCCgm={0J@wkuAznh+oAnc4#g(_QmGR=#cmc_w_F?QnH!m{D-F4*e?rGc2B`@(u
zae16I8arYELeJBmlv7g26E+j~279d1k-g?4TN&L0e_c<a{w7V9jF(IdT_he0tB9B^
zp|;f7zI0G@I&@C7I}zFPEN<$r`f*eLvzsq>PE_At;g8mc-S+;&x0KheQ*i}&c`rpW
z+?^WS?0%J;&-^+uLjE%CHuHCTS8*d_(zCH`(&$~6-&xF$G#uS5-PO8fV~68U$D#r2
zS9gEN+VdAkuJ~7+1=o)DjHi9?(hqChK8rY;w)J<p9sllY@4Tk^y?zpVc-TGFF7`li
z+?x|AfJskQ>l^27bOZY>f2(;}!P`yrcVX6fG0?upt?*udsYy0;tyj1E_2ZKV!(V}0
z?_0E5g7|)&etOvsS*I?e3)j80z4-y#-tL{hE$1yWhn9h@k250pB77Gz4>o6S{s^Uc
zqMy9k@Jv8T6nlLCWWH^=MW3d6FUBqA=^uS}W?`|cyS5nP{B={X>pn10I7?K3aXU)j
zTP+Bc$i;#R<V*sh7y^lRqJZAcD5;9DPLF%`@C5%I;P$_6UV>9wMH0VJ0D<e^)%3%g
zo1+31NI|{0uoH-xgTsF2abSh-C1a8ha?ghPn0R{-($wGEWzKIye;H4@r_4rc&~NS^
zob!uN*hDACCIJu|W34A|qpS>K0`8GP5IABG0&oWhd`00%{`+1AjsXP!?>QI*inIek
z{)<KhxITa01K;O1|GB~^M}UxkU)aFcs}THusNq5i;s4J)7#Da4lF*crmj|wzmaf*;
zPHwi&?q899b^;GjT;%lKKp=eD=P#VR2F)qZ|AL*Cp1Yp1lAxvY8#Z$*XA5gKuQx8w
z{eXnM1cBQ(*6!vMUT++o+yuQusQyt10{734*{LZ0k+?gEQ0XbFQAjzvT2t_`aj<bv
ziK0?aPzbwP*$8UDWd5r;@Jocs*4^Djke%Jr)054Uo6XtvH9Mz(fB-uO7dsahD?q{O
z=I!Ke?#1flM*W{o{!c$JYd1?*I~R95XD5p1e$6eMJ={g8sGbM<@6UhcY3*hAKO;H0
z{nxU91+qWCVdrGyVE^yFfu_RGPX*QNysRDdVRml-o&jTs@^W(t|D*qZy!oFI|5r=B
z|7pqnitm57{9kYW?=7|6tX-v?-vC3pi~f(e{;To-e)(Swh1s81{=Yc!A2$Dc3h-GJ
zRha$1WhRQ6|JY~=h>_F|rm6*80VR9>z&-n24F9=4-%EAp-O9{@K;j^In1q%W+>tRt
zht?p0Xgm8JBFaAgdn}kZE=QD@90MgK14U73&@5|IX;?^96e=<ULs2P39(M9<A=Ub&
z#Yte7_nODyU{`0m@5ws%1DUUb&7_~(i68f`+^j|hDl8<KAgCFHF!;T6Pah2HqsJ7&
zYX~8nj5t*ftcM*;-Gj^lrc40gNJMpmjQsx5W+LCb1X{q!Pz*rLsF42<1jSAPRB<32
zxNi735YQy-6<7&E`2=?PLj?ESgzCHJe+I#l4T`21Fa@!R{_9u?@__N&kQ!?nnFAVj
zgVr<nZ}6b#pv`{`A(EtUbRk%QaIH80I_7|$3O&y#PCSab2ZpuH4k9Rco;@5LH~@(d
zPK?6kVo%*?JDR>j;fu>wpZmkj+xqRy<1L59x|zv5q5b;qr{b&Q)tp+dHdaiWV5gc0
z1a2-cy=s0bfr(ddE%L9)yp%<aawYjhk*myak4)Ot(+{`jd3E6{BWqvWs<dl#tXd2h
z`S|$eV#u4%xS$YppUGFaPzH@s<h)mJLwqg|q<?&k=ca*NpKMkP#u8{|^Vn6I_eW%>
z2S<x*zDmg@#F0oY+D0OR+k~E`BRg$&DU<L!l{USenk`ezV33M<p|N}Ma6YAZFjw^h
z&-VQ1#Fa0jMrmP1L4ZNNX{S$`&Idipd3M4U9*2dv4Zm~c`=$h>)^5uo3_-6`iuKmF
zp-$^<wA#A&SH~rll?*;-BlP<{!SD&{M?-|S4|j+4?oSWiO!^;7NncuzXU+s{{`o$d
z#thLW?mL*Tu^Whjy24IZox24@WoQ*r3&*p#g^bU4ckV6^-S?&`6$(T}(%6i%9r;Ke
z{|u2gTaV$ld7qb$p9KEBCzpo7Zj$3Uw=HTEFrEcU2nl-$@(p9QfW5bk@V~#(hEdZL
z$dV3q1AJs=)~&n9jIf=~%~wuiO7nAC{%9Rzdy)H_QhAYr*ttw8i_2-M>b62TM`t9B
zIYXmF27UMUdVAeW+SB8G`^Zat{PHLnZD<L~=U)UYdRWdT_W3t?0&eAoMy6~JXJhPK
zrV_TJNz}5q=HK3?lo)++jiEp={m2P3y8hgQWzqCLE{GF)8i8yxkzE!R8OhxDuF!pd
zR(gM-Hpj7$6+hOoVPH=jD&m|d-t4q)&|7eKRA=0t3QTImivrpb?Emm*C~vz!&_kh@
zcWo+%zn<&uHf*<PN>mfzAw7v_5r#(C%YMd$A0rq0-;V9X&d0emY&Z|z?}}%$8Z^83
z8Id`!ep0qu{#f=PrLVm_=WtL@nAJLXxf`MY=_4x&TBD+-F7pFaLA_eL#wG*)mkp*8
z#HQYyt)LDIm^GQG-(Dj>(;V*=+4WMRB@PR@&$g{zxPgpBD8Jn-4FYuprs$2h$;nDf
zjiz5wdeWi<MIFug2z|j4$<Vh7RiVNDXtk=PU1DLchH+Vcxk~RNtJB>K)9Q@%fbPdD
z!z?7LOHZ|Eaaxh0vyZPCwM-R)!3d=+>|Hm-k8S#xU=;{qNL5_HoT;C15m}g%Zf|VO
z<>@O<qm%Xaii^X=si)w@YHj9520t-4NC#yIBw0Od(lkI;vb$i~GDZP%vw(<BY!#?O
z3trr)TcIZ4L%wf1fYXvWK(+)_{=oX)8Q6klN@WUZ8Wt87M~m~JpErCIH-_YByRcZO
zmYQBC=Exndw4|Kjvzx3P=P$ECa*@M^!bn{$miBKdl0#f-+}aXen*=_3G;RI%@CM%o
zSN5Tl6nE;DE&%&IM&&HPpAx)R{pa!F_E$lSi*j1L*0*wH99$S}$kB2W^WJW?#LWS7
z3cadMz~4vD%j5?<ME$I~Q$*Lc1RQgYPdI2Y7<EM!gE3rejwJaZcNl2>A1YSQvhS1%
zWg!eX4tw7!5?NFmEe4I8vwjPIhhdS2boo*IDI?lD+!OgBZSv77`lmk989ZQ|LS|dv
zTHZo~Kp(Bg%6Ep7r1H6;H##jF9l!Rp``@Gb^SYli5eYrfr2*znPGN7ZDrFq$C6_Ms
zKL4hlWrl<omjEqNvIOZc;wngnsV)#OO=;KWpvaXmP2?Tl-<8B?Fo;wnMjBwGys8<+
zjL(0<2P5~TDeHInec^hu!h_Jiih@HYHy=&UjwqYN|7NAMGH?&CN}{!a(e3BB`iB3t
zaoM}$l~I5z^M+2{dub?vudBFvT?oIjTtt{T_C<o&!)`)c;8X1|$wiHBw<5`LxBBom
zOGbCcGLAo5=p{DKXlm^01{U$nayf2x8CQee>kcKn#N%AmBqmD|!wGhl^F7b+Z$i%e
z!LFJwj7t0n>~QA2s-NRonkK4D(86G4g`h3`2+bY+&rZe&K5iXN4bnSf8LB3czt>tQ
zBUj|ocp1C%iA{!wVDD26W}k?p5|Ec_CzsQv$=|_qF3Z2AQP}hN6AUo*3JNfB+EDxQ
z&q?QXKK9F($Ekmq>pQ3pr5vcl>GqHZ7Bi(hF0MBNg+@Yf2n)3r=lI^NFj!zngaleF
zG#C=M_Q1YF@w-iVXrK*-#<?|l-D?NJogPY%e6>GM1{%@gydR1^e~+9Vs8R(fAi>m{
zY?OD8KX0Z7Qp+Z2Equ@~0ZbnKhNM(@xR)9;9X|(@hCpC*n<)8tx}OzPRUGL;b_#!z
z(O1Dh$+9I(Hkix}A_9mP<$#rBIRW<cz@PNhMtc^OYJ22rg>hr?jk<TB&jZG67e~vQ
zU&oSZV~5m3$9|Bh7O_ch+~A;{di%-L#5z*nV9HBj_XcjalQ$aX3DFrCnNv-JrKRPW
zpmD^!qZ<^0D>Y`RgQYAjjVz{Jejy>=ijaoJB1+l21_Z#`ORV&LmwX`vrzKh^W2Fu9
z;*O@Jurg{jilJqiGjyd1Rv%Pk@7%F5Z4jfuNO&n^!%jjsUBxF>na}6A^u3(QE$;A`
z6Q?&YDk}t7=I?IQ%S9+oFyAmBiKoYFQ{R#p8KN<AvuH$Z%=&V8AuzGtP>@WMT9M?L
zD(#EL*{a9vA5HlQLvciwpH(aWybPRK>Tg-n>>1#kw*$XD`!!YHQ}?}9OU8n)7O*Ev
zBcOaaJSHu*?@uuo<w{xGjH_10mQ**Gd{WqO?emYi!NGzSRMUud9wIq6e#bA}f@gGC
zmTa7HXRv^P+8S{YS>JFE4wM32yYt6VP|Z++_kd?y8J+0e9`$|^gq_ywTQ95Do1G-~
zopaoDHsb?2{`{dkX|bo9k_rZu#qVW^LI~THTt5g*6uE82U3?ZYQd%4qSH4~E43@Sf
zT7={-hf+odtsj8@8UToa%xxzM@o%%<r1^TAhbtG6&wT<M`lp58{R#W+WR}Y)act8Z
z{b{^Kpf5#}y~%_qL|H!+;jd5!jJpFm-OW8%ef|F2v|S?;{?qoq4h>#D@tMfNMHY1q
zne0Jn3Y^~P=kD8R2cxC&#Jn*l&@Lwq$O~D>k<JjztigGlW4m|=eICc>o@1^}g*bEz
z{|Z^#m^<LXt3Nd!L75P)9||z+cC&W!8*@K3ShX_qgjk1Ko5Uv$V?ajGbwOA-Y5<Y{
z%9msBD?j77Uu9-DEP@*Rt|s=d_Vydl$YBch9R<(Lr2ix9!HNbGV`aA<2gaCs%}Id(
z6uDD_=y}p?P9NjN)(=jJ%mgfUik_;_;R5CzZtx7LA~tk!sA!*l6$IisA~yM6JM97h
z#HqR%E0~fdriz5h?6crw>-^S4y;fDSG5U8oaNW?U&DJv<{3_Gakh6%^k8d98xFssv
z*PV%q&mz}%qSvhXR-(0TB)+R<?Z`lR^&30iK0c1=4&sl#+JpfWwS2OaMc9dglMI`C
zgR7<URvGl9Dc=dX;5ByBD|r?XiZNE5a(AZoidTN||1k0m76%J-K$jxW2x%*d@th{z
znyyht3_xy^mVDM@>1B`i*C{SFB>?dA^^0KVdPcT|{E$rKWHoAwX<0)3LVC8f1Gx`h
zBrPJ!MCg2eTBJA=zj7(;paY(=ymaiILi&et_lN+{OS6E1EdIfbsvtsv-t+fbDx)SH
zcFxdyj_Rb-FX`7|hIypuW*0rd|IjZr1z@72S^e+!D`>9lX}7-Y;48MyCoLUiuY7h^
zwb(cNeDC)k%1!{*zmLlf&J$;elg1<yF8rY1l)S_@Ew7y#M<amx50y(u0U+IW+U@Iq
zCIb(EYyUZT#om9odm0OX{KOZzoc~Dg#Dk*6$uD~}{)6IqfVGAoio70wCJhn=NT+2*
z8gT#h13?ECRVXn<#Ql#gm;mX8IwlLk|ND@HIB>LdDo%>6oV%~9XaTR{f$ACkhafXt
zC!`$T(P{i9qubpm*gkosK8L4aae}o>HZa&yPNS%2P(1v>O5PaufJdvCjcqk?kt?jq
z0pCUB{yaw$1QtLv+z=T1<~wWJWi~dN)fQ8E@0-u6U;_<dm4$Kk_q@KEA8l%t2TG6%
zPA_Dw=)i%6qjp?iM40#&&k&0#Rl3#g%_MA(Wp>&Im<_c<!XhJ!DnF+N!%>FtEA<n=
z=Q$<gPUuC>M1|X(l!y@!QAhxVb<uNJH-(KrCB1^5tmF-uIJ{li{s0oeV(0M-g>ndh
zXBHWL!o*@89m*Lhu}k|57+9`XaZS8<)L}m<MtYXH{<%M46yQdJuJR=@-+N_*9oAAc
zqHw&IvgQJYz9eY{iy%N;=;Avxz+MJ?X#ZkNPuzDw-fg(c*(C_lshJWGOd1S0K*Bw`
zm2WWRzXWQuF1$=}@=&<ID<<61A@CJq+ih@%ZP|@r{~4}jQ|A0w$^^4g-g1~EEr6b)
z!nFp<oCGR&)W~+m5bQ;=+Qn$XvdByC?F!E^<{vVVJ8Henm70t<8J@xn*IUr7+P_Fu
z>XvjW4q{6zyqJ1V>*lfbnsx1fON8C%pszzQw7>?^A>Q-Ol3DtsmRXRIzat87K(?P-
z1NH^aYC?}TN-|7kK?x%N!D08xB&PEeeVJ{Xz(@Of;xHvMZfJ-lNnkj;>C1Ftmqfbz
zuCQo70a5S{m05e6K^%lQTK%ANhYp(R^{cM5So_ziM#gmoBJE$b(bQ;)q?J#`hKh|9
zqPWyS=+VMe6K@>K4{&u{f--~7llf3wnJD9fXoCIgqlkLcBl@=1q>D3ran|wIJZu-i
zqv{{3igx(38<Ze_<}#0e^2G=&gr?>Q{qXMV&R`{C((r1Omkifh?k0MBP%bV0#Vx4#
zoi^S<q3jh?XzE-NB!Emx>ywvYjGs?arnA}#1uaK@rV@nBqa0>hw)DfF@#;i>TcyFT
zM!P61@$$zp!-rwnzNK90&f)Gj@nU2K%%|eLyfi-(GOC$4o`9Zs2${0zS1vp#-scor
zYo~a(ot+>MPH?|(kh%MKuTI;aS{>=oL=Uej=aKBPa-ZlxqTm}S76$rEdjUORR6OoB
zHC)RJ4pLdIrQrlTCJN#wE+|N8P=>Q2Lj7nz=}b?`uSIdxqO1W7&)D%D>kp+~i(-Z*
z*Gq6?MTF+K{GWV61{xW#y?P%SIu2iR?*Zdq`!Hbm)6yu>X}u4hZenHL-|-VUEC}MN
z-}!QG(X|9}Z_lAe2T`qMmAn&S7UY35;Fg0=O;!MNF*mwaH4`9TJ7BC(Vy(ci+%)fW
zR-GJYFlIodRwJh@k2xNQyR3R#pf7LnBP7J*982D5QM3w@(0WOO6HyTe4u@aAJ0I5T
zSHRGx3Q~XNGo`p)vRpx#`=qJ>y*ix4iB5OMtO!NI)VLaoy%p1(?8QRv>81SX;wv?b
zN|_=Bw%_Vg9LG+;eKW85bO{ZfrU5tBV9WK`Qp2S|x(wubPSJ~djSI@6&Kw3^@fiL6
z&|kG}L9rZ=rQfg>-&)BAZf&EXh|?+c#eHCUr|^R}ny0E5iv}#c1j&wWStPInn-yZe
zrG>+S3*q&}G3VB!NMJe3{fnjyv6xcEJ~#0pE<Wu_4l)=%wZLIyamKt%4fGeM*ogvd
z;GR}<XhbDK=D2!d!5g@wlDKf@gb6DedMp86GS?tmX)r@2t#&|+-&NSF?;dS_wKwmn
zz^m~Secf$sMZ!$j)Nr@pC``)YKEy`xj7;fK${nnpjXuKN0})$!u)RJf@mG^!Qa>pQ
z_b72MU@^>6qoGY*l=wYM@>IdHXL=exONK}92a^|xzzmeH%a9<&*9%EG6NeJ%H&&-e
zC=~1t3o90B8kVgALy_@hb&)Ti1El*jpLTFaObBVb0JbLexq6nK(wEm%Jdl8qd+u2e
zK9gFcl9a5@-o-+!f;sC8SGqeOv576@*f0u$&6THAb1lgc%HY;SsyHg<$H`c?+Z%O4
zMxUciQ{;@9&mPPlP36C<SM@87<JOgaeVa3HX&nxM!cM%C92W!Tsh^y<XmcQBWmXdp
zAuf!@ztY)EIq1PN_r&jy&oqqj*|2DWLJXVcE22J%?PJ52AY1!T^#p)td`<XZ{x=F>
zdrIoj`mM(oS{zD`HEMGajuxREHEqn+g>oQ-q0x?{dG;T2p05a{DZb$ZLtof=={Hls
zl@22E6q-E+iA0rrI&jWP5(nfsmZXjGhir(6aO8%U7s}v?fudTZF%m@Epx#Uar+qd&
ziWFWe9vC0SME>_KK^#~c>b<zoCeAEJxY>Kqz%N>QzZ#e(j>LG9XIRDb>n_TjUrSFE
za8ab`zh0uUi%v|<x}EjHC=f{)@}P!Z`!tYPh}uKLgFj=(ux%cNxiIIrA|&(87tRbl
z-`!*<he1=CAh017dCBmvzcYRdyD@mowhuRdngNhiG^tVPcY8M1^jg(^x-i)E?GxY`
zTX(g;J5Ih8y`0mWE0M*tpObSO{XNv+x~o9U??igM_9exUFNekXv(x%s%c9A-WgV<W
zr#AWa_fn(O&)74n&9Ct!tLO2e(=e7as&tw~31m8y!kdMo_(2WkuLgWFNb@!Xz0I&f
zPwGZ7dGT7~+W4=bi0H*h)Pp}o$RtlTI_-{@8p(d&1lSF~r{wW#{ben&;d>$?kJa^S
zJ0E-o+sK!^{=IHDTPmOLbnAys2=Aa;1cdl_yw9w>nH5@Lhvp;uW+|Ya_llZmi*^_z
zQa#OpJt=KcL)&+A)uxaG)l2F9R<Q`l;;ZJo7(N67Ki>_n`Z2|(YwEkQzjJSFz!E!D
z@&r+244Z9}G*tz9BU8NGCbD_8nr!+ELj$hYJ?B1TeNq(qVE8$~#Am+@QkKi_OvdZH
zkz35Si^IJ;MSJt1+xZ|Eq9&s_+JSqU`&I}2)U(qor^_yy!rlP|KrMgpF}uac8L-7A
zSnTo(hnXUI8F7f|c{EVMfvC@HxpEG+&%mK?i~W4@e2p$WVZH0_XtPJnr~C5|KXFNv
za6o|ATKonPK*SL`=Zq4LipPYdWu;E?5pmu_i=77aVcSTqo(4j)sR)A8#16*xm%h`b
zSNmPKhDw7nfHzI`P{SUbNK-SlJCVsrz24!?-Ru4w_5{L~s;5NG&Nh>|S#_q}`DV?-
z!*V{S-{JIGR&JACkEO@g8OgVrqhMlIL}HRB&)X}ji8?#|98L;lGwH}QOKoW2$Z{T!
z;DLRLRGBJQ^1}LY@j-`vRg9L&P_Je#IV*t|q0H-#1Q)d%4=}6aGgaKq`l32}{!<z(
z-3MSt#$;u5k+^0dubEa9;`I77QYMWNOZ8HOAfaF`tp%SC;WhqLi^(W{OmdYgwNs#q
z5)p~9P<b(}jDQ#m0f~EUVKROY4=iXjn}079HYI%i;>~8)`eOv9ZihO$h*3c_!_NrJ
zFIf!sUNQ>m)LTg<(F!%*>Pqqu&d3#wBe5o4BxV5=#1sgHvtHe2hK`+#5$HN221#t*
z_xJP?{P@n}QltIdYY1q)%;~sa!rL!yu~@$FN%PSH#-~o^(;-QV#Biimh%y*!J$v7~
z(Qd+7G<CbX@<{ZK9KX$l9Ixpor*D2J2Y#&nMW*r;r8jbBGIBAW<!V>Th!m_u&xZmI
zcQCp7{nC!1<F5+!Alb1n6r9q<7{Q_!IPJuI7xl^!^r_yNYM)N<8qfAz<LwugM)aEI
zahTt;alfMsVdxZy7I&O0g3%s%Z>f%7N`e_mXt~<gTKfxjcbh{)#m5`TaD9KLe7CA=
zGU`8jS8;0bl=Z!$srXKsR1>?m-}D>y;D!g7g>VBGev_ilR82~cSm94!USUj6);&`2
z(zzz%Pg~u;G)@g#{jsXhLkADe%(*-0VlJ$KPpNf&Iadt>5xUo3JIS<mV3F7G9mkEr
zto=(~v8N87thpbHG;HvV)r8u}kd`EWn?Q6=e-wpPO{Og5XRQxyHd);ig-rK~$~HHt
znR!*%VP)o#Z8S&jo)2^>6&5tn5o$+0t+*?R@d>(D#n+rc>*I;OMfq9cJ`6pzm{-$d
zi|F%)IQF5cmK%03Mn}Yj#=L<q`hJ-ai!3RCQ)FXDQo2+LdARbIB0=|17dcUIISsgT
zoLVlR#g;&G%p0;IKgeho4q7YSVtY9X_j_jV^uLH4HzVIXyo$;iO#Ga-dc%qTQ<}nF
z7D2=5n9~^!harpB-#f<7#uKc1T3bqCra(2lY9J+Fg&ZhSGH)RDl$`kO@+I6jZBK59
z+KWt~AOQ$)%J!tf%-TD&K8PON$K-TL)FduJtpl-Z5*HRDIatC2v7){xWAg7p(WYrT
zlv)G1FfFRLh!_UlEc}a3-6e4QNcD4)kyH0-60fM$#?=)^*buI$TUq#z<=$bF&R`)A
zJS<0AOeJuKeK0^FO0VY+C%Bxtsx%m}O?3yf9AsKAl!|aRIPDtwQG9g5+dDDn>^pnR
ziU!P43x_JFNl#N_r)g*qKkZ8=QmkbwrI`U$l)zcSQqv9hXK>_fc*9!WVE&-RJp%@1
zFzD)gkUtA9_-ZNWF*cMBIWryEadAQYB0r2DZV^%Pu^d}HfK{KnyV4X+pX&`Ys*_p<
zL3)r3fuLDg_jQ%N#tv>tuvoZauAXLGDBW{wx#+UW_%8!5ekf*xteashNr}<%f=^J*
z*ugeMQb|O()$lL4{tanWV<hSah97@A+8uBqwz>SI1qWM=&v-Ik`Po*>^qqA`Z4Fi+
z-1%=mxCe<}1lw4(x2ho}+p+_gOEZR{Q~W+mN?2MG2kxD7%*s`1<+j>68TN*xHbye7
zax_+8r&RNe1^JK4H)ZnP6h_R-xGaq4QO3g<$euHCbWCZYF&6ezpn)0R!}rId3jT#J
zDHuQKJ7~hbn1WMDUYE#19Qi*XlVmkuE|VRo3!hkjEH8{YI~>mbqkr<lH4X;l5P^mj
z&t%3AM0Z_N{s}tXM8JKNbyL?VNsMB}{j=lu@uw|5_9cFtWOhC>h^Zq|9JvVVJ1vCj
zdk-=wc!`@_bNmnH3v?N;q{P94V*eM$h@EHvWGY;{4K4nZf%B3&<z$XlIE;$<m{!qX
zN%|EB_$~54KCwI}JrWUi!l@)R!-wzUQlm~ZMN%QiYLuL^g0Uzz-{Q6SO~#1R+fy>d
z@mEszIuKc5R0H`1SGIwcC6&=EeTV4xSluL4ytr!?5ig)><E)%a3{s<y+`ZCks$UE>
zb_q&?ANxwEItyICTg4{;i#lPrbMY-=Z>u>ejyl$+R`8>A1lJ!Ls^G{I2*N0slzlXg
z>TE>ulGIS7wQk}?n3OKe?`jxE*ZXlhyEz#>6K|N&3T6lI^q|&VuNPkzs9lJ<%0c12
zM+*p~TBxF*lQF&=LS}vy1~?q^jbSX=by!%`;3HG4`)d};bQ3LU{KjYvp}r?DjSPeo
z94kHyg)@2BgRjrA`DwkTG<GszXo+%^)BlBt3~n{PcJ^KQ8Z@OjgpkEfQE@e$*D{Sl
z0lK0eNdb*QB}j2BRosuCh^X^-_*(t8uGr!uI+6^9J~9KRs}+{7O<Y4ko{A3kNW7K2
zws>oJ0EH~r3Psh4i&PaJBAJ2=GP>Z6!9GVzUG?P+;yjhtn@uZ`8z+bwCcPOKkUgga
zN#1yNEv?xUZ+4?}w42}hXy{>ZF2|#Y6P#%5VZ(hVFuVs`V*0q@uf#;dN#b92`=l6v
zLzW7^&j{HpzD`kAv_c6p!;Viu{Dj<oydl>)%A1XL0wEJc0cQmkgxb7LV1lbW-~(fR
zp`N$!!6RIEkc!HINosgZk{B}>IJ-59B*QB`4i_%A3NftOyYf>HbHXZS%s;N<+=F1I
zs;z3!tl{jo%8Q3aE1!f9S@u|KIRbkGKx|<y-l8N^vw!fn3iVDSTg}+BZy3tZ@!gCw
z_tSv}41bgqSt*+*+UM#>{&2Cr0?3KA<DvS7XwKnrl5c&={17O+@!2dTlcO=(&q`;5
zdKI1X(bSB}rILDeo)?#z?{a$8hHuG=jx^`3mG`oQfv6RvS@t`S_jBb@D~6#Mw5KnA
zm?bm!FCT_|H(>9JB6aJiKP|nHTtoSk$m>_Ls;Rd%<9=>JXgxlB*wC&+P~Z^T)B5&D
zrD}m_v)wnkYJ-;QPtk7nZPGxDE5EvNeNClt$Ok(h6#9XPNnNioI&71e*Iw=JVt*gV
zxuQle<i3ugaJl|%%%s;qk;bf>pi}pvRzz5`H0B2@9galD^uT)0*?eP#819%}z3Ptq
z8z)2VMXXOja4I>IiG-XX5if`&@2^iP=nHnzq+2AiHQ!2`UF^-wbq5Bne0oEoU9B~8
z$ybmAB(Othbn0Av#Rg4ovK{*t4ZOW5qZ^a2GHIn%8hdoywhR`tS@uU;rQ6>AK-`_k
z$v9Z3RXqPS<=H+(KnQ0FZS}wB!(&hjV$!Ws8cSzA=M_kB<^UpH9km>d6>vXaF4a9M
z#1PUhIUQrMRjO~4S;W&g+`S&A^7#wIx}pZR!`O6KNHf&-7VF95UkarskP1jj$KcBE
zsdo(_d&Prf@&L!Ve7)UM^Y(1FGhqrE7Y}&(^_TQ=;$H%jakors*YE;eYH^%AaB+U=
z$ZoxL@<@+j>8m%m(dq9N)TZRbq?oPAWjAhv-Jbud_&JgD?SsA^OB_5<ybI=c-bgJ8
zwx6$_Nv2h7mQU5>#0BDfh}CsKLg?G;0)Lt3^t`b58KImjzV;Nd84ZW)j@(MSCwm&>
zkGkx+rX!XdKA?KQHq=sF<Myz=yEM@=662-qXeuMzND4P6KX|d;G@~Rqo7Z8sFC1;c
zSdY!P1E3#3ix+k7c)VKW@%(Kh=zqs!J(X`<Ao)=%LIPwWY56rSiOdd2LvqEfu~dF~
z?mxC-Q!~_{Qm@fsu11&r7Vcg|YO<#p*{IV;dnM1rdlxU8-#OoIYy?5^xn^O2b9Nnv
z)$mg)Tfpn7{O6SRzKSURFd?!T;vWFl?M)X+X?1wLwM3rKxI+N8Rd>(9ChX6ag#{5^
z)>a}s^><eAsbqggFyKjj%N20zh@YIwc>&Oc>e!aZ#M=T@24U(FE6sh<X#gz(SkD+2
z14%ZKtPEJ9FEoXmm-?B9wc1J#-at^eEbsf{^(Mb9qu!Aj1yC1(u<!rXO(P_@OZ41|
zYWj62jePY=i^I1r|NA{4<eti9JxVjU6-gM(X52OngyU6!#9hh(rA!zQ)#Nx83MvaJ
z2z>Gu7#JFw2!_hNfE$+%X4;$J-z?q#RzZLDY7-L*Xx<BxnJJNL(mWy01eM%Z5>O}<
zksUqu#2F&!aR37f9?E|VC60x^c+4ZOaDD;M$EmCSQTSp%&lD1ae2CfdW|eQ_*9{^y
zXegvuIu@Ib7VyW?;LIUMfGC!VIDSh2F3;XVt>G;Efc@@cT)bh*Q(A{{`#aZdLYc%w
zv8UER?jLjL;eeJFRgX0_=y9IS>zbzd;&nXT9p;m7*HFgyRB2Z1#JMje)2ZYV&g!{r
ztpm$*85(fW#g(S~fP;gNPq!G?sK*k)43BnX3R!8klZwS>rNWW~HpfE_QH9g1=T+)j
z=OcWTbgwmNNxmlrG;Q--%8!=@dUHM{Rt1a34wJW9T8jci6a&nJnF>)PF+M&`>Mx7_
z$LWNZ7K=O>aXr)!gJxR^#SFGI<^>zT!+L8DSpy>HE};Xh1b`nVzf@xexSkiYitI$6
zDOkSPAe%X8R-%jDe1``tRjSRttK?5fAe;-kB*Md!MnJ<)^(C%1?cRXV{wSs)2nIaK
zS)jyY*A`SFjS&|_56l@z7JYXp@H<1f+?xsg<hU#eSRqaqc|CVi=*^Erv7Ln!cqEL{
zt^UaSY|<fqPC$|9YSw(Oe~(PaG*GMqo~M$IAX@@IvRSx6+>1th?XPh}ozbIH5rof_
zA5LdVWN|)GJOBz;1(aUUu}~5)azYxQ{($AMTWT=pv7Ig)Ap3RyTvQZ{VhS4Pn#^D?
z;34ey%$3@s3_7L<f(lG;-~m;_6alpkm{0<#4LqZ--h`dvUeTv~VPC7bYiZV790;^O
z=&3IRb_YB*E2Mwp{5Q>7BS)nKb-48W8=%xh*+79UogUJFyp=EFyJH*VS$`geW$FeP
z{vJFiP96|I$oVcEmQ}V?kREyqUHbGU=K{a~g~gm0aLO|RT1JCF8w&id0Yw7Tff6mj
z%t2&fNww7*inN<8322Ur3@Rxt?(>H5uI)ab6uJOvC`KbFb?c1i*x%R^sIE6+)Wlk_
zp#Ys#J$Hu37%GZHenur7qqbOQY}Xftg2;*xn}-Xj)~#oo57IMFy>3uJwRnhoR+N~q
z=)fZ4yQlk;-GO2tLzCTFT4dM^oYwN5=10O;zuQkD*szSwtN`A;XIYBF+uxfh(fVj5
zAHuvfoJ1W@E3BA91F6=jRalC_Zci+eOQMp0dqXKB4y;{Vx^Jgj&(HDlVE$tx5M2o!
zQ6w=K)V!|E7xfqPI9lRtMx82~hX9sF*=#P53MeE>8B_MV*@|?%MR^3`qjY$|3TaF;
zK#Hr(pMr{ts%qv&K&IzemM>Ay$pOol`0n=_;~Rt{1KWf<lP*62bAf36CL8tDPj4pF
z8Q7$I0G1)Y+UXrr%X1v~9o}05U+Y$<Rs?>oT{1&L#l&0`^j2YqJ!hGcVsv6}`_R~z
zj9*YwQ_qh*IEX_69{$MoX?z1p6pDb&PtpKk`Dg;rPAe)y<S^zZTmW5kH5>z3k1gBA
z1r;;Q{&f5M<S*!V!x0`HZkG#BPoMw@BTVCs7!z|`YzDh&IyJnLc-$@6%=@iZ$V3i*
zB4EUw#w8o(+9Uv8@oPh0Nu;wH%3K~URycRvMi{i%<F&>K&?{p#EIVqox!d(sf>zZH
zYIT%ncVUtMJPN_8y$8&vn!9`RT78d2V;BSKSC%WFhOA0D7QfkLOVWO}G@28wSK|;B
zV6m~b?DsGU>e&za++QvR9O^sqpF1zWiUN#8(*UP0p_^3ZfPY8|wHUC&DPX1ul?3@-
z+bDfsX_p4VLOHQ|=%n7PG#Ak-rmOWwpvP14C|3Cb8{FMg`U&S0h-DQ6#WS%1Pq(kD
zH7gSSa&4#bN>I?z$t3e%QP`X<Kr~c#punD`d|>L438Jlb`!%VY!|#k?P7|l@e#AUe
zXDk%bNZvFczyUS@G_4RX8BzJyH&wB;F?{ExhT!Mwq-l*zQ^)u3W{Rcv8;8hBUJAX-
z?Cvq@3wq9JskL5aECaA%=Uuw0=xwP{o>n0CN0>MW2P6cX06=j=DZoumhPa?|KqR)P
z6`VLgDIDqL{x?}Rx$BE>sw7QVbm@XjOfjegY)al|J4L{0n%P=v)NBS!28(6KBlQG0
zTo{1bg^M+}Dd<A|Rv|kCFn2P>ZSF-t0)94@y{nRaOOIF+$c~!D*k5#F^+#e>6iG$S
z2E!v&YE@};(GN4)hJds@fwLr;I%G58UO9zF04mdi2gpVjCVBAyRKB@tft+zanD$GM
z|2M969AA|6&(ubx12-)=a*fV^b~X*^1Ualq(N9peBCoZ3CIaba23cD*w6|uEhvze>
z`#rB&TLznPF`x-!9*$i%+i39VJw{YDGJb?nHCj~`b$xH`+@C4ys;O`K+z%Jx8&K{>
zQs~Pv6i#VTl~I0gbTY5|jAgK^c0Zgc+J^L$VLY2#xX=|sqCBxc6Jq+UCQ%^#X^H$s
zk`cfG%0MbQBV~w5=Ywq@wrkSGb5bVXUW)!>v-742aU6h#m6w}dFGgz>QGY#N{ahA<
zaX`UB-x~})<p;Wbl35DcF8~S}V+hz)I(;sSIW4~uHbB1OdjlncJYJ_}UmDglRYk0i
z`Wh}GfD~{K);z8krQs(>M&FYELOTd1L0A)i0Q54;rv}vT830%(s<>wmg9qq7SZ0ml
z_NB0wnp5N}t6{7@^2i>FprjtabnTXNFC`?yQ)3|mY77qkC~<B(R|$Qy)|$k@)Pu-J
z`ViX#NOs2-6gz5;B!CeoUifS%aAy(6UjtSZl9&dZ{CaE(uY0Hn6|B5adg(&Z_Z3qa
z%o&#T^(=vkQM04F^Y)9qJc~z4FDw9*&7Rzb+ao^Pfu<<HTTunJ0J=6|>zc&#ev|7V
zp*5_GXfU6ctJ2&><daESwRN>1I*rDmp9Qq&841eZ6zuXo7rP?X_lQR&?p96Z3CX%J
z$45mW!_q0!h5$3UYg_pC$5)=$lezDiga~xV0cGDSA#-O%csJPh^J<BkAZOie8XxWh
zTMU!{|Gq!@8!2J1_;Ycq?Erc<`NPVVbXZH&E8AFcpfGO4rfKAup+z}M(-H#Io#RN%
zx@=bfg}6ab4py)Pb%<(WEf;I05y!tW-G|^Fp+WmwYJCf;Q?w7c4F3}CD?rbMBIUVM
z|9(Y00|&MF!M^{LDu@6JS+N{t{P)X@;Thg&8?&(d{|$ixhQj}!Ise}@CrJGJ*KS?g
zJ0YpPmadmoa)YzweLItvhCIi|!CIdkn3=R|i?_zU!pv@%cEB9qJE`w0Vi|iWeB{$<
z=LD5jEfxEDMAR4brI^@n?wk<(`Q1^&Wy3Vv<>4zYV$=NPSZ~y`kV8z7P!u>kr)gr8
zl0umHypqmD^m%$;-<Au3IcGR8&gY!>4onI5cdwKgu5rTG$Y*axM?@m`E}p2Js>;W5
zlk)9ul2<*NjKzOoVuOf4CuDA4)v7P-CUlexUY%*)9j}(*;*+U3$Nvo)o%fk+um+qY
z@A*srD~~q+OGMt^o6+{oPL*zUm-HK<Lf4hL;kspJduhd<2PX|jDFV%BO}9?VJB;d4
zcWn7T->oYV{5EQrFERDktq%W4Fsyts@lw@}`}L{bKu$*eOiMTWsP7gd6=NcreCWUw
zxu8?xy;ma`H;bw4(WsHp{b!H5cYcj+fI^aB+M|gD5`juCZ3jsLC6_hoXgYg_CRxI@
zaUz~{k@LN3;j!Yvd){${JW&!E+Jfu9_q$Ay_`j4O28DY=JmYeZn;zB2fT%jtRwdee
zX4U<VR%aw3KyKwcQgyoV&Ha(}O1piEZ9{@O16FSz^xNL&FJlZ+pT;kqibqm8Wp#uw
z>X2&0OvS~NGg;XqWo4#s)?-n+?O$~rVrE~+kYA?y1PCRd>M~f1B?e$gi;OO0{Hm4d
zlua|Owd>AU>V+l!MoH?b0A<y5=)~GL+rg@Eh(?wy-duQRY+Vf99Vf#OYj2Dau!PHM
zemN*XeCl3O<`6}LszWK+yJ?e7noe;>P1`az?I{U?xKAee&_pq>DGQef)JX80KfZD`
z*s-y3&V9OFt9F;UPVH45d)})Yoe4d&?uZ$wqBGj#Uo{YQ7JnFS>yr0Z7yAAW_^_ZE
zg-J<o>wM3;%~EE`aKgv$MZ&5Dyg2@+{aT}$BC*1WCt@8_{q~<TcAdBixwQjJB>WN&
zt^}X|H9)7rK?mMsii+;xf9Qyg&;Rq(Vz|p-wsf$p(&?+Z&q62~&o^mk)2;ot^K+$6
zq|Md&lpuOhD9O8}sZG*rI1%Ieh({wdDzECv^EoGrK;LZjYx3!OQ&!5#Yb?}M{}7)*
z{+R%=u25$rrP4QIABuU1zI|)pZDQ1J<M|^}J$9%B5v)pQm_ZY04L~v$y~gaV%&gFY
zR&gscL31B}s0w<Xs<s8**X?Pi6+c+V(Aq`C6dgCd@kJ2?!s(KOOc~UBhCk36bd7``
zGyJyP@Fx{9Pu2<S^ZPHZ8^3*o8~$s>IhxI1Vd!2NI(d}%?bk+!ds>&DbF*Lh?Wt30
ziJ4L23j(g)aT$HGso(hI1gA-TSiYA9vF`Vy=;lF6Qu^mRrNw=`GwPm4gv)8j`IQFw
zg>3?j_l47p6{Q{EUu%PCs4tx}EqT2CP8MF?{hH#|_#2A|ck?QuMzS0IQb%y6X554$
z@ds<gkDWN=VsaUIqxz>y_VqT`JgTs#yT*qvl?|1vVK3@`VUW>xDB3ezXvu%nV=c}g
zp`9nln0#XOspAt6^!_MAd&yZ4_)^w;0@1VRm*Jcu&xF89ppsC=lY$*rX-PZ3bGtV|
zr)d@Adgt5NX>KIMddy>msZNu{{5r8Hab;2!Vnc%3)CzK^8$Jm<0y0-pJi?j$Az0yp
zA6DRuxJsaBrtMmtko=zuX7*t+eBz!P6E>a!?hPWDp-+D+Qe>zYzm*nezclqcFc+X@
zQ+X2cR=M%UEE{|M_iY=(qf=~>NN&W|0sM{2AGm&YQj=Gv?tj(zrVsodx@Jqz=YP+w
z{YmB_!>HUC-OsK_QHVDw=a;#5ZAX|NJsUCARMKi*6&YT2KW*2#+I_wLTL4>PEJ;Kf
z)4hr!Bt>NkuSP3fQ0b)-<G1p6Z2?~2oc5U$Vt>2~I9UMxScERZAD8(UcsI^i_api<
zEC|tdE3#l&8#lqOXvVL_840YC-`8Mpj2z!dJRN0>Ga%5kDz}_jy+}TG{R&;5S5f5h
zmO$`qKQD6BUFJ<T{ToO$ZOkudz2n4dyUhbu?9Iki0{QKbY&*T<cgbev5E=RkiTY1k
z=f~w&R*&)!q=h#nMB6#AJz;;swCblgk<A;7uN@0%lPEvl&6oeBc6W)x1uFE64EK5y
zoi6HJoF8<r?na{%sr(nlFMcIFo|S0~p2Hxg20bpM?5rL5qaAwTT*B$$lhI`CagE4t
z=r2mn{??S!S^kxFzWzbCPJ_}zfhe|zy!8c6vR4+FYQNJAdzU45iVO*f30f&)OL(n-
zqDng4;XGl}_V!P;A6fhtOUrPw&FkW~Qax6rJRgy1^J~QIxXUtzZt9Q5w5!I)w2yur
z^ah)LM(ZPC^0o0+#4l3nEA;f)R2K}~p<(398K^tK)MOZ<_OB<NN<wx)@4N%=Dh#WG
zL=8Mr!CZS+^Ro6lw<Ii7uUf7-;+>!Jk5DWJl4(6L1nFh`UQ1g>Z3rLSb$a7eif3)B
zoNYr&LuY26JPddD8YVJDo5cxTg5fW=s=k$_!Qk3o9B~_jdo%?W4qDob`ee3V%YL3o
zsqiyHj*z(Fm2jlcelLuSH-M^|>wFhq@j7Rr2yJKzo1=<WknhsB%T9H)0On$KPL{nA
z5H!x*<vh$bTmtVN^PNy_XP>@mh~LuK`S30`nqqkVT>FHx+2Pjb7S1D&cXUMW>e~&w
z-Q{?=hNLR2!D5vE;=K9GN4OI{UmiSMdk}Bs+nH8O<IA$u>30gN0q*XHCj0x-k2Rbc
zw4tqP%zjV5T?9W`C#p$Dnfy@rWm2L5@xt#M&379bK}7p?O7geW&Qsu{d}YyFEoz#1
zy%+F1ora!ya`4_Et#sYX1d&>rdl@9iB2AVqG+?Et{hNI?c~LLK_yYnFnb{{^L+eyH
z#R{RS!4PQ(#f{LSQ>(Y8#3Ns|#A`-}G`g0m-6*oNrEh^8D;5Jtv*H4#euuovQN>R`
zF>>W-!(xus*+nmKdG!~6qCbUwVtJ6h`i}Hhm|Nbi5dI>A;tapP{ac-Nw}OUR{J6aj
z#m8F7>j1xBaCUxQ{dsRvkoo-Xaaf6o7T@mk-BTBFvDVm-oSY;#noh6iZ;qXqU;MoN
zBU&*;q6Crnfag~zGRFH}q-eCyxOP5LWWDsZP=R}fVeO)lIHVfk+YP7D*M?=zLQmeC
zK|YA-@m0YKm+T{O=T#4}^YG9NM|wc7zHS_%l~rd#7^2<W*DtQAMdh{Fa6Svv0go#x
z6bJs==ppCO!|w8N0dh^}<-imv$ki|zM#SHfH*H=A!V<sM_nV4TR9|5S)35V7QZq1v
zwT#hyKQ|3w;zO4ZPF*;Ptl918<;SC{u;G$jc%qyh1JY<hP(5m5<<;Io4k$_L*{gnM
zZj4h!{36^4Um(AvTQZz(WMtp5XWo&_D|Cve&ym_h&r*t%#(bH=B=N%jlmOa;!PmC+
zf3^3OZE*$9o-hm^+?@e}1PJc#?jGENC1?ol9yB<?CAfwV++7j~8r)q5w?X#s-@D&n
zU)<g2zM7utYO7OSr>CmxS3>8DL2n$BZKwZ6$+LX;)|u5o#;YTWq^GZ^V`*^~;*}dk
z9l2L{CP^DUA#?f1Z0Hll8A<fnj}1yt8|eglQ@DYM++S$thva+jz2h0Uam>=>Q&%<Y
zUeB?ZPZ)28n^b!=qu@k-Ge!7+j-`LxW#}ZB>uPF2D93V;0M};1A!Mqf2l?h%%GO}C
z^9`=agR8AIauLi6Al~KIZm!L6S1l%ya2P996>~D<qPqX+2q$NudXt+yUCYmdYC0Or
z{yH>)Xii(<PDg@12@iG0u&@y}HUhum7{j(LlP-BgTEacpYK}0Y!f@$;8OvWdAuM@J
z(FcV7p~`VCm94X(tJNV70w{|h%U)Bd1UgONw>p!?*`mk*&)m1<Vzj5muV?IsN`}SP
zF&Ez0o#d4ZqMvOXg4c5>5%46i9ky>$GHrF^o7V5TwsS}0r*yJzBiAMdj4$mXny{*k
zewS0)=vZVRc;0qRAcmxolG}9X+TZ^0p5G^u72dltHonKD1r1X2iaK*wS3?AVGZ3l~
zD&Lo3#gZ@=ieChC2}U%;H{NRIPW&`F4(&U!jSOVUG=U2zjm=8iMGXJkjHCe8UZ-Me
zA+^fRJ@~$F#7T@XZw&m%#FTGX=8aE=M=K;D17^A^yZz=Y?8(?3QX7Y5B(u(mW(ZGe
zz|VMjdvED8`3^fsgd+{X^Uci363yRc?6|+Wzi;DGRUZ^veyc7tYLm88Ko9TI0U>Cp
zcf7VRH-i}|u9XLC^nZy*A&<yEs6Tsb@fR;&H1Qo5GV$P<cr6RwGboIRh0`I@HhEdq
z8b3OZacayYb(+Z~jXA=6t9`vT2VELU@y|mwxyFaLM+k_vU6apk0f2%yCH=;{M1*5D
z6af)>w9G(Nq9$Le)hbs`R0O-P*g<^Ss@?Ohngs<|vTeo16odVj>TsYu0U!c=q=&L&
zB>6@Bm`&##^<|k(DxU1wfXBkcP~k!DM4@_O4F8<Ou-D{vT-JGG&6G)>F+KCvm(S_P
zC>V`Y`)yhEzCLi9&3|$#KzcNw52n_Ie`RHQ6?M4YMEXXjZi5y?JWk)>Zz_s}m3`|A
zJrlRj-wkC5Cp{7&$|pUbc{Pn1Hr7+J%5mHm3Fm)gN$QcJ(BlT8#INo)qzSYcza?xh
zvM*BHT$omF`XcyNChwX!NeNv3cT%Sc=M_!h!4B<uc;tLlLdowzs1it>!|1$@UEcrB
zFHuJ-hz2CrFx2NKCNNw%a&qfqSlV(gO15iJ*#V2Qinl?2n+s{5)h>zrVBtTa&Y_`#
z0X)sZM`;zl=1%8#ffpdzy<-L}Wk*hls6lCL{C}IA?nX-AH-ADX(|*LpqvVn9Yt_|p
z?-kUQ>OY6krS;VX^|Ht-k0{#9$Do)c1%3pr7qVU~3!6GG8`7w^bFk=E=*RCM^YNs!
zNyD|nkW(^I<BNp07HkDK@JqeG>1^<s$e_#3bhBOJOGU7Hl|+9y+)M()obI4&=>^ys
z1IhdoZ!P7N{<?d3AW4hmFV+Od(t{xq#{tV<{a%09NknkqR|?^0-VHaCHVE?KQNH`L
z;*g*>9`qu62%GpOlzR#~PN;G&Pomz6EtWk9LrJi`iEmC(8AC)xLUgb4(<Qzoui4r@
znWpJ#>pi^YDU}YCC5bJpumRJro0%ZI@7w_X@vt#^N<q5N!7J3Lh_@xZY7}1nKevVW
zh$hv9FbkwQj}@o&k6NVR&zBnCqyIh<lzb08V2cae)b4F`*bL9;oC?hq7!s~SwJu)(
zGaL4#@a3=lW2S7mR~kv>OPks>y`hj3@;9nz#erXnL+-pR{zRer#kj|1>-OwA?8kZp
zk@2%V>;C$(j|IW_m&rwpu)#?T{T}!~7`YF1OK}l7k!Fh<Gp?^SQHWOM|GjO(#Ckb<
zG1dF{Nw?~Kz<ydWpUhU{skZom_gP%BFlsFdWf}{7Q+IA*S+_<&9}LNxi-qub!`xdt
zAT<I{B?nwCRr`4!rS|pD^X(C}F$f#8wo~dYR)PasQE+p$-`*p0Bx)E-*BKzs#&?5j
z`n+X&4}YeGe<+Px$>Ege1?z`zRPGU2Jb{kj>;rS3ca0-Pr}V`3sC_p~QwDAM38NJ3
z54?FMSg}lDDWob-g5DVBk3^o?r3Y_3KT5Fq9rwHz$Sqh+v{Nm_82pN^Ti~m*h4G`m
zs2Mo@Bwes4C|Cf>L9wTZXYT=LZ2|p8T8@V(wIlssd1D*YHaDn_oU3AHyGkRqF4)#M
ztvvy!KcIXr;gC_T9U8y-8{NP7u4e|1^U7G5r@NhsT!ApwNAe`YN6)F4t#8aWQ~{Q@
z;C>S)4M$|()aQ*xxW10m-fpSBY}}xZVa^>WvwPanB>0|xr)<{}*zt3iMklB1QZpe~
zjB@$H_+%8!HUT3mzii<sm)fCUQK(xM7W+7i1nYPHC~*A!JHJQK1_y@_@PY;4ib2jz
z>5L(>xz|F@E(<vo4E8{iv?|B%QBO6lJ`Tn~2(C~ioFk8d+eOj$pu~7f$_sAVx-wz^
ztIBd-FbX)#8q)o58ULuq<%q_7(pN)jahWlcj&(IH{RmvalOe@+!4Nt85WJSB2m~`)
zMZ{ynV|?E9cewyLpU8vx<+?7Iz~)ar8(HMB`8Wr!jhrEkmT>-5d?)X7*JE`Ur%*y@
zWbIL$yW6NIfjuq0BjwQ0;3H^GV=^6roT$Dg1#L3GkPXl;U!z(dh=fc0Z9tcsr4#c>
z{Ih;9)ei{@l!iRZz#LK}9|EkHF>zj$<$ffNu6a}VKH+i+hA7nc<x<Ners~7X4QbGQ
zueQQ;xTrv!`~?NzAgFdmUK6`o-?yUB<IWl-o}C#4t$9|P67hZ(7%3REqz6$<qlw-d
zh{z6S<dyU5_r1b%NL9U8EEf@uMLA_54b3g_XZBDGZY?3*n#7u6)_kON8fc{^9YPj4
z5D|Oc`5BIdg0dp2&o<<8hg<}lvrO&@E#5N=3r_qihp*2x4458O`JTVx@z8(t+f+2$
zhU#jK(z`|f8))18)m~Tt#0Y;p@;f2>hIrRriDHtV1+Km!WboBOn^~CiO$8L6=mKNK
zr~GSQ*<o7YcC{ukYAL*ut63C++Nkpy39*w+^!#c%d<7Esz*Qe3v)?b}f}n$3MjUEN
zlz?L58S#`7q-hO`^#c7?H#Z<=xnI)3lPV4>NNt0-NfX%;>|F9eaE;=ffTd_vMzg#s
zzK>fa7fwv&oCI&5DypbpH&{FX7yd_3G@CAlOc#sO*!49&dnRT5fjN4hJ!5_#B0(KF
zTCgRF_&1Re-r7#Av+wXU51xZGC5!5J=Gq&Fk#N;xS}O+(M5cYEEqAs20@<8gU8s0L
zac@u^PR%6IwP|hh+E8y9C(zrx_P-g4iDd4%7u2aLc13P}(Z9l%4qo|HesG3A_Z5Xb
z|5dBBv9O=X_0C^$L2qKOFVKBbBgC3WG52i^HN2&1PwMQ>xSV8J+qJDy((dwNV&ksY
z?D>ITk#~gF1lE!h-huY>ViIsFa(1x6#~+vZM?jJYI6-E0iVl*k_CbBM;wA^L?D|WY
z<F))OKEVlvRdCF6)R<97g&_vZopNUi>ATuxvkg%kzBbqXg5f5JE|gvVZAOqSpk$Yg
z=2i?q=y-mHf&752$no}ND;{T~E@@9a4|X2U7Rbv;qvbP7CMMJUh*pOvYUblAMqg9d
zY1yq}8)!!HzQe~&Vf~?JD^p<gwCi@C4eR7`vpEsISjuR2%N_S>&#^gK+Gc^eLGQ9I
zL{{AxOyv=W#fk5>4N2sApmfdmS2CPsQ{uy>)Sbeu=8h{&{7T<wK^L;BX#&0UN*%s;
zMF!7tnvX3l0xc_0hUfd`an7F#a(9-Cs%4%M1kC}LNt(A4QV%)D>30r(?;g_!UEQdu
z!l?OfAADHDx;ouA8?#u6VFhixR&Cmyzv7kzYLixfq*Xlla(k31?@gGhR=8CEB>gLe
zMxn{CcV>NWfhJMxKs`EHf^B6GX3gJvm{ys{Wv`nALPUSh&#=PKX}*uFdDinrvsC_b
zG~AFnN?WHGh+-4ji6t>A{?#<rb?ucK$XJ&!4t$OVk0xPKW*0xw421?*&9N;OdG04_
z);dGtEv&ie#6cN!;*cRazhV^-@I{nb9C9l0njIY|(<LZ?@Pm~nZ5Cd>tRaB#K~&nH
zdZ7G2js9CRA_xye#coX~PyIz4klf1-bw+&m7$JG;Qt){hS&oOVx<lIb2nQe*abM4Z
z;jr_t!Fjj_U?>V8MV7<qKO*45sC^E;2&f<cB&5__sH-@6*tgY+>G{gIdDvEoqEmnH
zftunWE@M;Rhua_T(#!Di^014D*$c<t0wQx%&bBj428fFjKWA#-OaDbQ1~KG7FIy-?
zYM6pR;|YiKZ8sWmh;u=YpI0g{;JB_|&RhO4YBfDEbSi~FNo<AC)e8_nX;NyP%3%)l
z8me*D2M6>4F6A^xUIND3%7Iu8iN1_i!L`F524(~^g^rf`DFsTdEtZ=Pa7qM9CZHZD
zb4>v?8P)K`P-b9CraH)6N%TM??x6Ra^DiV6SA9m_FO68$22!^HS_6}yF@H^<pkTf!
zBk@`sf@1&shtV0po{hijb;5E$o89s(L|uTQ4MQGq?Ms@4bpIBX0)DqC|Awd%3M>al
zQJBFNAdslgNM+Ci3e7!((e=qdjS_R*yebB;F#9G>-2h-2x2^u^t=upCBjHcFK!isN
zVk-~II7J5vruM;i@(cj2pdA<M>I-ehm-KFYfTKllZwkjuUe;bEv?J^V6t?EiM&N+Q
z0j||Fpwk0ZswXWBVSC~E_l5Tcz<@!JZZ%x>|4jOv=7q79?x_OwKtY*X7~2f!0Z#92
zbgCOr$l}^0m{$Xq^XNW=>nRVg2BIvR!UHrl(9#*R6$CgjDUo4!8w(fK<;2tTVD+-D
z!rKhbaG-`0oHKl%1k~hS^%#9m6(0yS8yT1;jHyc!=LCP#87pDSe9&Rc81=$HElL)j
zLMw~PkxU@1j!6N_2ZDte0QE!8L`w&cOC59s$j4Ciy@jJfAXvT&m>6jME2ktbH9YRn
z|B<rP|A!ilF@sI~TPhyv+%mPRe5$moXQ*rNg~n6e9?u)MBiKZv-aK<yt0o>`kjfm7
z7gOK|<4?~ziTN>SiW;*D1>6~6KEP^{nF7MIJ`D(Hwtbq%FB)LvlF7UIm^P$G#hTRO
z@iSGZHD2C;x5EV1@!{#7>&|b`$#rO5ZU~j&Ph!%WY~~0`4Q^{Y&J9}-Uh67b^!u1N
zLFd{2tD>n;L=9NI9dcM#@4~|Mg$HNDP00D#h5+t8w6iw-Y;&8)bXA$ueUDi#jRo&B
zOQY#8|3(e}l_nE3D%bQZKZVC7*}+3Rx&(WOe#J>4QH1bYF~0`UH*yga2Fle=Qz{2M
zADI!+2)_vCL{Y%5pI(#ARVN>31|(cmH)$jcQisJlq>c{oFjvyF^Z6utpYWoDbxGnj
zeBJf0C#DU|zcHrmbb8k5W%S)M*Q`6}FGuz$dMF~@uUhs0ev_*Ov$qp_uZmy0bVU5L
z#`?p1MM<-21><}zQqL8SpyoGxkTzQP1FDquK%iVDrc9lsl0v^dk2RF4bsIJ=I!n)F
zb(JH!xiu+!vqe2uCUmuUMWuK{Ha>^?%G;-q++(t=Olb_6BmU4gzD>wMIE-4=c#8@0
z54H<Ezn!rej$;)YF_(!?Bz$C$Pf@B#3=TsxjGSfhRl#7+O@bk*q|(NV4lF?X!&gJ$
zESfg1%3Qs#7Bo-B%1!Y9rnwX;qKF0r)LnMgSBFYM1irhvH#b5rTqXPKS&(2)E6f);
zR4lEH&osYVFdDCIyh9`t_DSVM{l0}K+SnKiR6%>sXZuPJcXAotw=ob*%TqJJOP+5>
zK<ywBwBENG{o74=YAK3DCiIBtL@qJ*qm8ET3gh*{-HEs<OR`Os9Q~Ub-Tj{)?zX4h
z9QJ$-glKu6o(|RSKJ5$NT}3z**+wO{cq9!O4Fn-P+b8uGtLYB2tp>3A+&tv_REYUE
zsPYC}G@lI5wm<qe*AEUw;wL?~_$;!WUkx%_MSSy_uXl(b*ZJ}Dz+G28*v)Q?^JKnV
zD}KnVn9=8*^>UL%=&!TF?eoSFw|y;D0F0710fdM)Fh)FrCM(7)B=X^sA={-oxyCi<
z@}uVBg#aiWpQ^c_pgP`Cjo;wDqgnL}xwq$7TfgAxZuRjG$9a&5S{N*^s3rFebG>;b
z%x<?P)BRh29MkLXBRbg0cQ6bG7A^VtDdq}TovOC$^U*TxYS{(0vsxT<Qqrl>-#(n&
zZ|TgN>{&f^<GX5YhfjOmVI%83$-iHBK1>D5Mb@fSMM4lcavd5+O?FGa4*R;J`;Qfz
zRg`ZAD-PBF_<qSSl>aOpXmc{iCaQBYh14yA!TVjcnvo&L<aOo6UTR0$AQQN`C<?jQ
zICj#z)L)}_aqB66$_=0PL)YH2xuyUox!Z%nWZRAYGQS)p*>YXffxBAt62;ZyJb~f&
ze;k7{$AfWLL~S(V9?g0a`U|~h1b>pR$J_Ir)M5Vh7()9~RW)g1(_7!Q#0z~q>o&YQ
zHmVuq1hNx1{l<T_8|)IbZw&=TO{F3*D9nGck)uphTg_$*fqu-BUi;v+wohcGpPBhn
zeO`^lLzib?*t(8)bN<GX3<J@77hA}OUn=VIPn8(zj!jz@%ag^GCM&!AVUUeU0y^oE
zg&J@EaX(jXlHKD1`iJZ5K-gM0PQA|V8V$^`JvQ!@Aor<%nU9YH$(^(1vi!VSrT3lt
zz4s?i<=eASweO=mI%9L`xftI(5*8`R$S*t{F_odK5-S;;Mo-I~Y%HWGdo!$p!#X#g
z@YC&Sd(lHE3DVbSaz#+gOQM|6+k!U2tU}=~L@lYDPHvCH8Efoj4D44IXdDOWOmY%M
zWFhjYOV8D?qeq#^P6OjdjBBrd`);^qmVFrRV`tl2Rx{N3dYEeX%Gd>eL(0maS?(I`
zWnoit!?H6%vic>YLQNb64Dl;<sO&1;m23DD(T9G9DnS|fTT$EFW6=i(CTrh3Oz<x0
zAxO)nMkF)*)jm*GRuJs@&XA9>!Lh{<eHi(|X1Lg$N<NB{VML17bjE~Lb)UY~X{Sq-
z)9C?w#;~f?$9u0G5T~e<ej3%wQt!@tmuUp2P&g6cOZ=P1dOfb5(qm>GG%0cvzMseW
zPc4TZFQA<W$VKB6Y2_wot|A-a%LQL=Wn+WK-1+B}AnB78*=-`*229#p*dz_!KH6T&
zD;9o^`pxP{JT+`XLTxk7Vh^oQDuen{cjKCLhdlvUA!7t0E!bB01`0D&6Yo8Pqg2dg
z@zUye_N`Jq4=S-4LDXBee;xk^j?`kQ0$a8hu^{M<1JC}A9IA4;H^qA~SkajOuZTFp
zam)jP6>ZZB1&6Uw0!Dpx@3D6j?qZTG44)jn*YxgT((@-H(i;imot_`w_n?U=MyBj$
zp~2@kq~qb7W|?8)j8n>T5a&PLZ_*pHArSg9MUa1%uC&tnkt>;?RB4;QaAjL%Npkj7
z<7#zg32Ym0j^~F4xvRW*%vQ&lHX`f35M(O;%c6<GF9fcFa)IGbQ4Zr@DGs_tc7KFV
z%3dX_#G{I=iF!;YeiRE}YjAX=d`-7!T<FqoYdzGXVlHKSxgh3wX#Ek_bEiGm!kVw_
zkyw>hT9mp56UxLM3o*l`Ogg(Z3x$KRD|%^PT>^?HDBxz)664MO%2Ni<+quD`xdk$k
zm`nBzF7x4NOLr-dR$Q87pe#9kHkyuu{9MMUiYuyLPutXYbL-``5UE8Q8#{^ae)dO+
z>9xt2fggus*5)pxkS<l_a{v!Yejpja&3bTDZKn9Ax5|S2l;e1~u|r3ZPn_8{*_5?*
zgwUsRB9mqP_<`THh*#%NOGH+k_R}s@iwNiQhqKXFD4=b-N%wT?cYU@VUCUROwrEMG
z=`ryHu5Z9v-oH{BOEBjod941@*JV~wFEQ^b>+SsEY|$cQ{;hpng1wc<LJHxQ-E&cW
z_z<Aqo1R%}?&@8tYcOB+A#Klnb<xRpY!B#&d3P*9<Jf5~wr|a&1G9dc=IeJhks1}*
zTH}py9(a-OvolJ2lv=$o$|OU<ORk8^U6|7!ZThf~SZnCqoPDOvKUAjMA3lq^zfpT(
zhEybZD)84<N0JjNvCgU@yOJ<-5z^k$SHIT%-S|Q&LgdXl5LN`!VNtKzN#~{u5!u4v
z-V>%9mK*!G3-h~vwn}vAWDEHCMfgUZ%kNL%UGCu>77WoX1=STm7ndY%_BPa)0u%*p
za8S9NeIs=9!6cS#_5#rv{UhIl_0bo!eL_$#)rGK#O+fmo(MV;!@z&K}zpF^=f9Jn7
zveDC(&R%tuS@ylLxnMxc)q)T|%4->%Eoh6l4!YTR&_@Ml>;Q)~BT(+_F2u`WlI#9G
zY5jG)(ExqRTv(XY-rpGLf~H!aIb$5et1>|glK!`|khi|ievMuGqiJiFs^g_xD@(ws
zNUH}UgkLrjo>WXwe$nLE>jz&;cF9{*F0J|T3k$Lf(jhEmT!fPrqj)p3g`lWG$|En*
z$92_@_VrYJEEDN@XWZ~R$%MP)`NxAQI`KG9_g^6dCQ`?2%q63mn=SN{_7llNbesNs
z)<Fg}tC@#|f5P_jEtk)|#k?PEjMNte2T|a6aM)|`Ny2}9V01`{V$2c$u7xrY;%LxN
zdeSW9u1Is?^z1m|>i?K|Svu_hYB853n_21|Ik|yot3~uTOJ)MEFHbpIkS<2o2RzoR
zhCNTSr;R1SxxKIDmLR(&1<26rqdD1Y!Zv$2smZh4<IyuZEJs`Ag|}IvnjOZ^UkQVU
zSiD%lkKas+KQ`Bv3AU52cRpk8<)EMcwZ3n4UGPFa{&u_4ygu*!1j4HnF}=fXtz^$X
z^!sre#^GHQBun(!u?8edyLU$8U-_f|W+)*BI~~Wk2zyNj#GVVuvx!4u=PZ&$X<7En
z@hv$)NIzmX&rM=qtM>k3-Y?|(`8V^{*E1&>V>A~B|JiOqyYoISSDE_HvjUEi>>K%p
z)-y{)ezf3gg#+D?62({ttZ$K(p#J$Td9wE!$yDStA?+tZOmQ|$IjWEn$-rTs7+!W7
zK6p1`=lYJVf!0eJ;W0mJ8g&#`1;1gXO_sFgl2qRsh9Fnm@B97hr2nkUJ4F;#_PT#N
z-BoBM8!i%x2<&c0<~>?7RG92~pjy{&HZ_}DnCIfz{X5rat~~Q=(F@l}MK6iPZ^G*_
zLBEjzH0ELVCI~+ESB}13yIzxD0Zy+`G@yWfCeTY-K@FhV&zcZ$aU+^qsYwmxr%~?x
zW<Sw)&!3KaP{(t5$J_Qiy)U?;ZFNgq4F#_o{|xg&#CK6C%}~;ny0a8%;AxLr-w;gF
zo*Yu{z2Ou2$92HjMpG=6t##>Ni7H8Zj1Q;-`nY(acF3qncBLHU=kd47<R0XFYC?4B
zZc|q;kKXQbWg=QCkb>&^K^kF<Q0kr0_upjBE)LUmrvD;)xcyDyQIJ-BQofyLGwjW*
zUfcSniF&GaxeZh9sCg<DGv7#;1j?d_aI+ofq9^ZvEa&WoEG@hJB&70Lh@Y;=WAJUq
z2YHHLHI=ofA|+L`(@KYHqrl6i5nEgQLH&b#joLqSN!uwlbMQ`rR%^dN@cds<*w=_F
z+xqQhnX9KEAD+cfI5)oZUWrH^iz#G^CwI+5?v3Whg~N`_b8=VLMwy?(XhFlcvbMP$
ze5SFVO3>O)u-8I4KAm}Z3uQkuf+;6>MLw|X9dLfM>*0vgdg_CA2uqB!2{*p?n3i@c
z4>g<BQX4ufyf6JKXV-0IpbBzaIpNE$q3534&%sr;sGaJ?Lw59J$aOc-7I4MwoN5+v
zs>&x!teUp#y5e||T6SCq4;PYb?M+pL?Q$P1)gKE}s$tmwe&aTZiE3B3!OVRUT^d!|
zdkDPtHk=CoGz?<I4fvojywAkWvk5Sru>d)7ULO2yRpTax@-_{Mo<BU4j_NWw72;s=
z%<io0iav?hbIuzj7(&?3)~2Q@er40&*NLSBNv^>!H*X@LJ}lvEw8IdqGfp>o+qXI5
zS6+`MD<I~Oo^I4GPP-KLY9Tn~d?Le1Cz~9;=r8lUHs@JlS{KZ)WlyRJOWsFI+T8bC
zpHGT?JKX_O-R_-e>D}IL@JLCc1)VzS)yJ4<haquk2CkLQABWou8nd;Bt78q2A)#L+
zyj4b0g^yl3f?m|MlFR)$Uh}pL4>k5aS`h2iCGmlP^$lKK;0Gr$*NCpIcZxR7O){Tz
z%ppQ~h2^5ao35`g)mR@TJR*<gNG{0#ZDcFdf$04?;9se6dloe#jybH8@V%|RzTm)#
z|Agauue`9S@Hta5fkj}^-TBJ*d<Xqf(?{FYZzj5Ti^ehGVI;iW*lV3d=j|7_LCk5<
z-#ZQHpV>f_G5rN(;D4=yZ@#i8DVAjUbzFLSHL&)d{=PBEV6>zXvQ6wkh`b?(P+?Ol
zf00>GGanq33YD?ezq~jto!k1fEm7ceQw}fPf=-X6^?8TrQj*WdK0T(dhz@if)`b4(
z6kH))_G4iQ4Eu#61Zx;}wCtXIIFq{C7I|Ma869xCU9;+8CnQREKyBpfr`7DVX4U@j
ziDbRUgP|s0KdGOe3O%JZ6%PLT)4v%U_D;^d9HbxCQ$+WzbCQ9c92)JWu^kgb`HeoQ
zxDI2nYb&=3u><e10yKTjJ`rCOf9q}5+}&iml5C;m)0;ytt4)8^Zp>mN*%i!LNhoXi
z$@=2j6=o@Q>3=bf4(S@I{TQoxKf`buE@9kN8QCA99~z;@DE*B@<$l<==hgKX0^eT0
zTe@Luz#g{)5BtI1m4;t~0E=q}+KSe4iU3iLjXor9W_CxVtO>e~8u^18Dk5vYI!W_>
zmYsoje;E1FZR)Tgp4G!n*p2w(E}ME+3n2wzTO#;u<?lArapDKef{wo#y`)H$?j#+q
z4(|l@>HT~)QI5D>>tZQsKKxAQVuu&Oyi7l)d}~YB`bWQ*$l2TlYlo;-7Y<&;goUwN
zD&%G@nZ@Ue$;U{fd)Ze3hGOOgn6N&WqFHsf5)}Ki)_k47QJ#C~G`{hM+yzsaN+j7c
z!(olfFZ5n2CxKV;H`z&YDgyn%#|tbB9Ksa8U#?&039e9YWNyEuiQfGzwH!VQ{7s03
zLrGuD=<%M1$@Sm*c`PYwejRs~`Yhp_&g(GKW*m^SOfQ;*jJTi-zMf}gp5xhsMO_N;
ztbWc`$XGk0%WgV?ypx{GR3jc!P@E<D?osi~gyh-_FkPVQ*6yBZ1cb>Dv!}eNTSW6%
z{A*x?L=KL*n32VNsrb|V_Uc1k&IYS&@7*vMhGuRQkJD)XvDYMioqHauBV^`*yzRO+
z9hbSZj+1np1C&M0PDQ{L$ketsc7bsgro?rUIfR3gg=LlqLIzj*4_pLA*|iaLfyW4X
zc!ZV#MFDZ))^<|yez&3_g2-J;pLZsx%R}St=<xaq>dp8;k~m4Xcadi@7Tp^i#dhd$
z^)Ao;wzV8lQUwvcVFMMuS|UW$sQGr?{a;hFinto82=?U)EyBw8I0gO7^JDlGr6$N>
zY1)9m)e#4%Cj^|yW%3jrI5R?)LJ?5GiIqpr*jsCzx%u{QW6A!n*tB!=I(%<$`mWt}
z4Oqv5l3mxb-d?})$B1^iT~n`m0h!a0FnAA2Za;xolo*&(5v(){4x{(<bK_pxa>jkP
zMTO6O%nWnj-@sx?lmt;_F}H!sIcTzE%>oz&WRm9*?fTZ)8;5)(#-!@e;}!c|460L%
zZ1##j?9Wde&bwjW`*v;BLzJ=Bs*NgI_B#YEe*8QQZuq1XPt<i4tSin|VOU>$e;}T2
zZzdd0Pka1Vqb(XqH%vm0_DgLdrM)_;4|TYAaeveVR$x2tE}~3?!HZ9AQu>aE-14hL
zJHoyebip@d>nZH-zYAX%`gK)ko%YDX+M_M^(Ex*CLbxAEy9@WbCZ;$x(_!y0smd*O
zWtrZ0lI?6L1wom;V3t!s?F{D$1b1|H*hAh5ddLd(htVj{>^!~bO#B)n@tT3Md}pOO
z^2Sadrqsq*0lo~i7^`2t7h(R5>RShTC}gstbt|<Q%HBJqm*pB_w;K8SygqK#bGg{?
zm*fz|CNafSR;rmo4@JDWJeN=>!6rIaHvaykBT<z3YgRE^Hx<-v3_K`C&2X`KbuU-M
zx#M8P+RPG~>|E3m^4|1A@rj?2hv(nIhQs1V3I%%qMm6C=MmA-y!Loafnh!25m=tI+
zqRyf#+(wG+5_~InsxF@G@sevif8hor^f;7H=4Dm2-+l)$s?pdf8cKr+nqjX?OxRS<
zT&PeJ#1Bx_Q8dT-*_qO3veLm00%n@i_NxD6O-UCFp6&5uxlfA%m6#Av69PK{!Utoq
zjuDFRRp$)u8oB{txG^s0#SjtVo!40*W;Z|1bRg*@fQ6b7FqSjYE#bt&9nEkdeM+{j
zQAW!=?cqsJsQlDzt=Z}_TI8?(+N4ET7I3Bs0>smG7@x*Yl^5!+<mL-~@7AmJw7pL8
znO%$|lhT8bdV9W#6TY}u%)fJt{>SI`8Zg_Zagz|CCcvRuK>%Alp)P5?@{0q`1fcU&
zNOJiK2Q2OXdZVB&R0+aa`2Rgr5M6@W<84oO?joAinLgu-0iGJ34=~5`NeZjJ%mp5>
z+M{?P)_eYE&VWB;O*V4y#b{5x2KWr%Ys?7aU+nZtfOlXC<&6JjG2kq~Ww1stw(&Cm
zKr&#NWDrV*7ps3D;N}3g1se#zjN?oTxQJ-`HeWi%b_N(2sK*{H_cEp<(6eKxXVy!{
zAXUI$bc*)=3@FQYy?7;JvE!3o7}EtjN{~r63+2n0O@K3`31`FQrDGI+fSFy)>HW0-
zvzY&RGXMR=taNQ$6sCOAyh|ytUzD`<M+lk>0C9u!wC6QK2dM&#fZ%KfIdYfKu`B^D
z+6*87|5xR^h7pcqJT+mBca4t6o-slSU4WmE7}wyT*`gn<`F11P9ku22@f#7J>#>bV
zE4^7Dl+$y*&8t4k|9<}*z$CPxy8wXqa$`ui1~`tR-?qp*2e2oI3j>?Z^V5$leG{T!
z0JdNVh}H(jx&m(H$#)L^*uhL%#X|s4Ilk^mv)e9OlR8RI?Wxi$pbfFv4?7PN!eE;m
zFImCQM~u%;uo@QQ7Ht5BBxSbzA_SYC;I_6GGpXI@MiV7#&DoSY2wQ3vPrt+0OMa&E
zNvk-&l1)t&ZK6bts^SR1g%}zBo|9!*tW1*K!HMC8o2&Pg+<Pxy(9~Xf4S-f==mOJq
zS&wqb6k?O(OQ3Z>ub$%vKihP(dVshwWUvx_TWy9DUoQ~0xopY`oDb1{2c|JuYc2)E
z=&HrhN8^mY;MK4CUc??PHmU(|4h%p*dK#HylLQ(*V}kl!=lub52D^Finj3)B;o5`B
zFoZ~CMD4Nco_3uVMKFv^<x8#DwDr$2K1JxHNy@a<tQ#be(*ZjX5;$qvERu~Sbp<%9
zwB8d&QAi@{+r*2wzBbC%$pQe$Ozn0H6-$Ppna`QwJ_3nJuoi7PN(ZH0pwT7nO#>bq
za%^q^%$1)vIsjJ6)~`j!sl^@Nl5zPdY|pVwKDEp;>^Y{AEE*tZK|K&eJGk%K4$byh
z`3g+`@E8W|H2s3lgzJ6+l-O>i0wahI#7yL8xkJ6e0MNrxAQ1^`Chx~dWdM<qO5koQ
zX?u)oJegkQn|zt(_l?)AH68cm)k<!-ZS&4fkPpi(I<j%(Ok}(tCT^&&*NIQ$!e8M!
z&!Oe4BZGjv8@0;R2cPSHp%c8px1TQkdL@!$C(m=AUs2sMA~;8h^-Cv&M6+Py-4c*L
zB@DoqhZ2u-43gt%#+r}u17YglYqU1-G#zX5fn*JW1BIm72|rwffKP!z2b$3kC;(;R
z5rZH!E(C3BIwN--g5*P_!u#4Nk&lY=0U(QrWKV&CGSb`hGsgVdYK?*e&m3IY!z^g$
z^U4`H?2nZkxRi(??tLz9-UI+ubfh5jvG8Q<?pAm=%k5I2ybi0sytP!miM9%CORC6x
zvW2T<0+|g7LuvvNnB@CF+xb2L`!GB?Eyn#7ma#B|J(m5C(EVPa)SQXWNBOIP9@0%i
z$L80Us{zmbV?Z7Wt*T3ENdQX%W7+$#Dj&#Ovbccda|fh2A>XXs&G$UpxF_8Xp=#Or
z<{&v!u2a7uPTBfte}fn~6be9)%mkE|*c)qoS0r%g6zKST{BDrHwjN0Da=%a*C-~;m
zc8Z>Tacvopm1W`!p07^4$a`{Glmt^Pq7e_VPq5?0h-usIFUW(t<GE1c_}tJu02xAX
z9QzW0DiAD^rFzGH+=C@b<$s%RX;6zdx7Gm!_qX+XY>%WscqZ#SOL6iWs$>H>HaCJU
zU#=KLxJ{NZCdr_*vvbpVCM0K?=^LRFy?sUDf4e>_MWBQN$hCEL;TS%IN4jxCgFZ$b
z0L`h|L_m5~8#m9FJWL&G!g(Hy<OzVQT=(WngdiEX<e7V*eTIk&s|lY;qwHq}4&cJh
zkn=U@@!W>axcHL##U3xU<TDH<M>ZK*P5^$2OgFpPFjK7&F#ijKyUJVQWrc+0y@{u1
zri}6b3|*lhNXZQq&p?+NF?yfUZS@(5;D*H5D=+8ut`Xi}A3J2%C|h}xQ6O1UPN#i?
zy<;75i~cgGr3_qbu;^-z?fAI(YqobfUkZ6}j~a6a-W@ks?D>JqN*S^L3jz8gfN-L5
z)<suzbRkH|bA#<pE;M1tyDuJ?1@fNdM;4JUA|pAH45Vz<s^yDem+D7v#tHU3E5P9s
zo)C&7h`|P0?vNI!BIva?%lx*J&bDN8Dprxw$f!RKwEKH`fOnSo10DV(3W)OOxdart
zJ_jj;7=Ws8h$qW{0M{N~-GpjY8MpqLxh+?)7(85R3qO*?QBuY}D~uN{pX%xD2zY*~
z{rLhtF?yL#AJ!W;-`RFHK&I<IbB_drw0|TD)^|g-RgYcc26B<S!Gy(Wqry)HdJS)F
z0=wcDgTqHMNaP%QdCH2euxhNyFBEvRk4Mk!yUk(pkgmN^O6uCRL*88=x^uA%n2kJ!
z-na!17W#+#y5LM~;p=~vx>7*DKlT6aMd?hPzQE#}@BYr<8vH`L(#JsRffsR(i5iVy
zii;Br<hAMff^Y#j3nr2b{2Ki(hVUDQrL67gS~ny{^3<l$a#t3GUM=TCid;J;O*y|m
zK4|QW#iWftQ~(m0W^OT-AV2_W^o|Mcp|Ark;>a_fWW6pn$u#4k9?9m|>35XOGm*CG
z4_OhV=g5Ad<}JpF$B(JA9DFzE95;3K&t{B?e^7xP7YCHCr7W6kXg}t<Def2wSC|z+
z>CG5AY>{MAIlhF_-3sk+=#N*J9$I67sJ|_ISE&+06)mS4X1386^|gP;g9<n}-j;IG
z)C1|Ba`iD#gzYU5`5?7?+;x;kLyz{c;`**?Vvj#l(uUkD{UUDUt>mRqoMX?Hm;m4*
zD}3|SBmz*A*_-Fj70S*I;F1U`fBtCzK?s|EY~~yI(DP+N%)KaYTfTOih-GBa1`0U=
z@2aY{V38~oGgWS7>T5<N)iCPWEKNFe|M6GAx%GV$L-Lz8J!J$CmrUgvJ@~UzJdjEz
z%xusDCgn6w$u7z6)ePimGOm$(IJg?SX1x4NBMDa@s=%BBN7G{!2Iq!Fk1^K!aJ}fH
z;nlwJg}Ktrl@{zQ#$#88XrQrwD}yq9I9lav0=(bi-gY_Di5oD2HD=Ohgc!l1H61`A
zEkmr5BqGcoP<k-%_Wrfe73$0>QT5QUp<dyh6eVez6bYCe?MYP|?)e?x0hZHca(suN
zR6Sq{kxW?LvtPThB=dB!45Psuq|ptSp_14JKHayfkW(*_1ra@qcBR1+ALy1SniIwp
zwz21Wm_iD%h{Yuc2Wy7}*C6O9S)E2QaA0z)QLn~C^{+ncD2GYE`J24Bf%1bhEXGLA
zY~xfBrM9@I=myxqnR``gu#wZDujvBAZwc!{N3){UHs5CLDeVFjTR0cOb)Yb6!i=L8
zHDLJ1jF}owc>p+te0%l|Kr0Tddi5C>blaEvdBJWrWCg%!9@Mn#!x>LqZ1Zws#%wDG
zwGR#Dt?1xsLKjfir<neQ<*8)yYM`f0;5F0`#x<4`o%K!)a+^6;7hga8lY6K7^z77r
zrTgy?`<K(IW9QNW?@1vxzNfAX(%411PP2Z@?(~G%X%~VnN!D$z!(cK_ZWKz;FcH0<
z<!_;X7ib*Z(THDVw=5*GU>nhF2QvQrap$rv#EYA~zl@8rM+m$`J8j~Ce^qWrc#0&<
zg1^BOl;T%3Ke5}?so@TcA7(lI$$N8?u&E(URtxRHYXgx3m^lguHkemJ?h|;@g#->;
z@cHZ_l0`Z-CL+uH)NJWkfthrDXoF^Pu7J|+^5R?Zw0uD!dzCNu1!>+W-qP_sq%S_D
z_CUr<P=(t&ee@Q9X!_KQO(k4RV{GoSd^Kn4#8aJS<Qnysi&SdM62PX?fWFcclZqNG
z2)1+nYFZh3vfA1C9%Eg+*UqNeB$dcr<g=Mr^&r))g+C&C4f~`P+gW{Fh8wVd>ua&*
z0^xPTfXrU3MJ5wO7mo_gXWVvX9dNq^cxON|paa=uly3qIsq#`s&SGmkj5|h^1YM#J
z{hUGqJ?npdzP!#5-A-`pGX2Lwte=0Xo%iE{w&48NZ7YxF1V8q=(d%|eVWz2;93H^{
zz^D^qnIi0eHFZw37?EZ)-7KN?rcSG$y=B0JyCfG;0&^&69h>bE*fjN6xwd^YA%K1v
z9cv9&_Prk0uUQiJ^*jz;KA!cb_ys_t<ReZUsY#<uL}s>S9v<a-#!6dtiEdU|14jBF
zOEZhO{4pTUjinX0xrO7OhJXRwnbOE6+K<4WTRW-j)==4aN{XF@2xN4;8~_|}j@R)5
zZV(TcctNli2o@6c5e3|4GZEN?sJv6a;N(PyjZ7+8>Cr_)cS)BWYz(*!0)?!tOcKOO
z!bg}yk3ynAPN+OJEVYBkTzKC5s_^YrZB66;qQB1GfGu}|43OyYE|0R6&1F$eKmP+x
z4q>d4QbuN>M9ukplx);&?`Nuqu_pV45xut#kPoVS>1n|7=x+3$@HtWAz8l$c?5hHZ
zp9qiMOB-1p5JaC~pE^=6v3a_OYelj|D9i!I*sN0P&r|{N;>+3dxYKkcjCk>-;o{R$
zj0cD&1>uJZ)}~im(md?<H*ng$3<v8(SI)bPZ>^suI_(0fob)gX)+nCrB?TPGM^bc>
zOZVMQPy8SJ;Wq97NVO}cBO11YsuY*aNOx=~5OjsM=0TXuxVH)dB14u|IdtQ58K+Wr
z%KNmR2SV&GdlS_~KKqe(HYUh-LEn`A4Pnfm6|;|DHIb)VYlKVicvuM~Mciv+H1wrO
z8vuL>MZS02X@q#EaHca&gKe@=O-~~5MiHw#>K07)DiZ(hO1YA?ns367chinuu2TQ;
z9$hdJLY0^$3@2d-`mgJ+_1}r{gFlop?Qs3qmCS1i117L&DfoR5m<F(m&;*`3*uNO4
zM$`UtQIZintiH*kfES1@{QPiHAgm|tRf<B-MgmwkIDxyW2?kGIZ{4r#(KMc53O+*H
z0%k1is6|1*?#D>T0KXVJZZiNg$aq6<+kXs1q%T)C;W89}iiE)27yFix&=}bZ-TyTD
e|9}w>R25{o;{|n0-+&MbI0YG1>1s){;Qs|A2C2>f

diff --git a/docs/figures/architecture_1_math.png b/docs/figures/architecture_1_math.png
deleted file mode 100644
index bf81cffd799e1c9d2230e3f5966694bfc01e6478..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 47788
zcmeFYbyOVRw=dXOun-8EKnMf~ZVB%0?hYZiyL*Bp5G*(ZcWWGi6B3-p8;1_=G;WQ&
z;`_Vr-rt*fZ)X0TwQjG~R8dvu)Y<!N{p{UeRh4CMut>2$AP|n6tfV>!gbtj7A~Bu-
ze=9SL*+3wyRT~KjRXGU>N>x`UOB;I&5J>iGQVO~z!5nEYT+vq+BMuLVA(uytAE_X8
zOg!FL>Mc!3DE>!r)yT{qB10lc$?8w#R9Ikea0Ob;2dNrUlQ=0cJrZngP0wJN*(I-K
zzpXpB!$tV*HQdDV5OR$Ua%I<{8VJ?`^%GhtP~Wz{bUDU+YA6yM%$kD&vTqhl3nd~Z
z{>Gk~{=f={Q%!$atR=lXhj%{csI6g}3xaYaLldsw8`E|>^n3Rt$kri=l1rb3N7cPu
z^CvS+CzQim32{-*TxVPRA^6KW2{WSvgS%TA^jFRh%t-|Lj+1A#ex&|fq<}rt;g5g(
zixnaazoSq?;ci(&b|wxEyB`sF+!Fh5L2))190?Dr9Oq8=2}JkJi<)RE(_C-7DdH8s
z|AO3!P(U7%ho4?+J|!@Z3!MnYFqd8?b7vD`lKr*1-kht!ys9#*Qmy(LPpC)mrGRPV
zwP7}8&-}OMUm8ECPS)se&E4{fbSY1xe|)sj4yDqLOoXI=1iSNt1%>o;Nw9da9(=k>
zW*J7F_Y7x#)blkTtH&|OO&O#nyvF`>zdY%3N7!IBc&}ddC0{p#<r|jj)JW!t38keH
zBC&_Tbq(9!zt|y<=;FJ*lN7U@bZWP8+)CBhkSi6DU0pncubo@*`03=&5L1n({=sY*
z(yULuJn`&$Ig3n-flg^qP2<=0!ZVphhf$u$l0&!uD*-$2$E<Va@XY8;7SP(3m#acs
zbSiHO-rZC^s~v7_5{%la$X!3h@R_LFa}X(Rk%zPZD2EVvY-%Jy!Hw2q`<Uzr9unGE
zFzw012aMrhqVL%R&M3yRy&;r@k|EyJprc?sb7ThHr&wL`s;Kf^($z0DLo#!S_t7J|
zo{v3M4(9la{1e$RSm5uQZKS)uZ(lrt|Ds0~<Nfkf?TwNw?)z}&Hv(TUeMCD{*$I()
zB;-^XX~Nf4k?KhqLk4pgt$8%j?@-AjMnqF`Sht0zuyMqlaz@#)Xn%3)qH~8>{Z>px
zL;t0A<j+fFjJowJ?8x^gJ~#xoTj}WK6_!8xVTegLHGQzU__*2)Jrf~8k91fbzA6Jf
zgYC<*7g`}@Vq7`;YUCvbC9kLOb_qzMit=7sGu4yvgUF*db4hYja%XbO#}MO6+iu$i
z+s0??H)P@8#(Q-?8oqm__==XO?uABZY1e+wtKUER7Jr?YEt}Qo@=p`)eZ%SMHJhxy
zp7oeLn3c9CYs6lNApf=EoX)H8Lx7dAug797(%HZT-KE%t$K~#4toPft*UQh~5h`6#
z$K^*vS14B+R~c7ZS8AWIex?4E$9Q{$*&K=!Dio^S?IA8=nph&w`L=~1|BKC^q(AZ>
zQH}+VpC4C#|NcGXdzAXm@3J;w>V*@gg;DbD^4YSsG}!nViL&1lrV}pW=VdSDoCeHm
zG3TOwh>p-!(apq<e$S2*Vbn;fOyV2#-`E(04k``uB$=pT5Sn2{)dqi;Db0CRXjuHT
z6i-u8bC%IC$$}Be*r(N^C9PRn)>xKXmR7c>iCbc<POkd9G_B;i>==p#wS|t9ovW#7
zN-$;&=7Wn4bF+#sCaXC$NF$P^nR3bVJSM`S&?)jMv#FF4o$_o=cy9hSt@|^f&mAFw
zp@Av)KH(!zr=R}%*1$n(x%lH)%&aIpC0sOLH(zZnrq@a-yr{~u#j@10ZqtmVJ~>SL
z=k&))a{aA_aLWnH7RWk;Im^d<6U9MyL0mvwz@$|hQFieq1{~8FBWYDWqL=x`s0Ezj
znC{qo$gm_PRF)NyC0k_(uV2Ov=l|T(|3UNp$h+|KhI5p&jhohUfpZxgMVvew?U$US
zg4{Hu?{PGq4&f}|L`0u`6aR+wZ8!S$3xaQHz2a*uz2v<Vii%1fm8MfUGoGf}rn)Mg
zD()#!q{Tx7A;f9W)Ftk|bPmWQq$vYd9nA8IC6pLSJi}tDK&KGD;W-4>OVwko`%`CY
z>0uCGp{vcOeXiX;?Y{S6Piq=7U1doIQBF1DZ#BE%A(P2?njyrE>j1S!anPA-obRZq
zt6rPaT3BkNTR_aQH96G;9emiKbe>&t9*y69vHG*OX6d**p)O&n^q0-pTtDA{Plg*$
zLhn;Eu$i&@ub<wn%wBveCj-y{u7&e2dlZh^jysO%i%&LFN1hMQaTJd(b!F~#EVTE1
zUWs3?8Ora|fF<ndtzAzIjYYv5X7>1oekAqs_xo-*tg)O6Z{CF*3>ptQo{C*xAqWxf
zH)Yq-2<>a(V~!(*^YTlw3$L@$ODF6uR8+LYC+=?Uyn@Q)R=i_;&NhFWDb-cYRkg+@
zKSX`7akj&Fh5w?KVSU_it$DQ>l1rIL(86%$l4>)XS8dW=vtqgeT^Xgar^ce<rz(xd
zrpA@*h&_p!$2(`AXV!nc^jhMp+_%!GYrH)I1L6e&9sx(2#U$@aZ#91fWY#ydrcT}c
zf-s+qt#dklTH?3MnNIvc1Gfs;^~C1Pyv=d7aji$z__Pe%JE9XLj`U680^f(_mXigC
z%}yJSj8EFHHg3g2T5}YS1bc)EnJp-xLLM5sIb#~SZ(iZoNNMHG*dW;mObQo76u}iJ
zhQ2_ohyPg7TD&GH|2o;TRqI}tZ0;e){8s9FmJ~ROn~gO2iQMHN<($8s0ymCLB6tw8
z4jb@u6Lc+e6K0Lk^LPKci+}vRyqUP!&MkZEdgCrNc4TX#iDa9kf_d9$P`}0RWj(Wx
z9>VqaM%WNS=c;P9^ZU>D0UHfk+~tGX?FQtnEe)#0&Qhi@uX`Ajgn>jZs%7S9<=nN<
zjo<iXrrp7eqY<Jq)x^K~Pf<bM`Vjwe%aF+Iz3TPv*_*`kOtK%``?Ft`=$E~lrs~{-
z(ropN^<YJDN=h5wPF7F6GDsXdG*?t>+^}7u{1sFzoGpg5^DD`k0!#6=mr5_dj*CU@
z4hN063)c%yhez|{)f!e-10$oYlC6ga7qBHwGj$EMs7gp_KqH-{N?+_S#omSO0m~f1
zBiq|#^#04;*6xqJ-DlPmv#Pi=Lwo4Bw6EJ-`nOaSK5P9xl=d|nFm=`qyy{Vr6Il|%
zcQ3f3Kehfn+AHEDAT6Z5h|0CWjl+YT;itc9&=Q0G+R4eoPUs@)oDHG8on0ba8tD3)
zVYRed!>ssJIksMIK76DgYsb!H_WsCzac*I&d(<Nfmp|zE6D+Kbydq|jWQi^7=eqCC
zmBr@9CX=#GI+)*B$P;orU!z<+QnWWRW8nC6(fS~L(qp%13&PRl?`ZTB(j2fcvM6BW
zl@DLuQ{F#p?(5`n7FoOBL+8VlrT}|~`e@$jZn*Ri9MExfra#Qju1&es&J<tnPWBh$
zBD@3Gf)eg73OkQpUhDO^wQqT#c%slfxqIT&mU@}@P`JLatvU`b>vZz7xY*qiX}kZ)
z|B7evLI1{gAJWp4?sVDi<~<{f&wa8Uu!$JQ?V;cjIehqYJ9W%=ae6x5W8!SW*qI*W
za7TQ0Fs{2fn;Ocrp<ItlZ{Eeg0BW0yIO{tGsXBo~fAGwis9<njE3u3H6*)G?6@H@(
zn({=!OTraB;sP~lzhPln&Hm(15~_yRP8NJ{K(uB;rBKt9aVB*lsh`kiX66I|btNGm
zHMrZ`*L&LU+p&D#mjKPX(jjhK)c7#Ez;71tivJ=;H*r4)^_!1Aq>w)FeLjBrs!L=`
z1Suhed@)Zrl;|<^wa}HbR8j&l0_PYYR3uUm8gPaLe1(z7{_|WKi5`UV&vj%FD8dGW
z`XA3I1INct9PoV{^RFYy_waw>YJA{|cMkIZd>S1D1hoHiWF&A8B&H!DCkGrg%v>!j
z9Nny)+^sD1+<^<2&a!%LAQ17}$1jqcI?WL<|BQ{MuDh<1BEOlF1FNaI(?<(dZwKec
zc|d~R{J^P$g}W)Gw}ZW-8^5;@)jv=01Lu#I*{CT0dBojLh)P#Ul~TgV)q;|Xm6Mg7
zN*Ifhl2Xvs+>&2iQu;rJ1OEw8S-ZPC^Ruyed3mvVaj-hMTCu(3<Ktsvf6w;*Jqz#z
zi<^(5yQw#eqZ{?VCi$Q9NLsj=x!O3p+c-H=KF(|U(aFPIh>Gg*MgRHv*E%h{ZT{;`
zj&A=!7C<1|;~lnltn6(6IX5s=@bN0Ys*SgWy{@E<1Hd!j9m4N;_}&ZtGvNPr>%U(4
zuc2CQ7OoOb4!}rv;s4_Le+>TjoB!8=|GZQ0-`>f=`|q#(w>$qaQjqPD^nYW;zxe#m
zRe;gLSb}W-xoN^!nX^}Fz&^gRkyOzHj)0W?^N|An(gWYeBk+BnLbQDcNX~mRIY}{1
zZ>0T9^q=HX*AeK1s5`mg#s%nc#)Y!jN|7Jvk*S2-F}}P}+j&xeCP7J)D2u~&_SzRy
z_8lbz52M{yT*mp~G()6rbU#bT>hgyhTxjbsV9V!l$+f6MpjAKs#slL~GXMNW6oiZy
z3__&@Az}O<PYWs8age*z4P{7?{vV$Dk7;B;NR;GA|L>E&r3waBOOhlM{U2ZZU+Y4m
z4EX=B%>SP8zc}{4=l_56@&7F3|1CZLEj|A`L-_wkoe>J|A~EqdrB(i6ofUNNb2_D=
zl2)7@IREtJ`#%*my`(}p=bX>~F%gk?-Vb+|5T>fOk68g1dY9nT_ZE@O$Gv1xG;g2%
z;}z=PAqkC|1e|M~F1n32Zy`c435Fp&F#F9^vlwnLm)2(xYWiEil<R#ei1;@@BAP+K
zd%d0j`TOg$5~0-B^p5js1IrLJV%Z;S|1#yiv>|Vkb-UXx=Iy@rULGvWECwPDzPi2o
z#~p}XQjJxah;UdAr%D!XW%)T$7slEYrO~=%p<syS7^32VlJ|TkNdH}%HSw+vTj7fZ
zhjMh8=F+jR4+2emcM2g2Z0(xQQwaW%?J*8a|C^m+$E2%^1t&OdnAFPIqzVtplc(Eh
zqSjmhwISc}u5R|KAVX}O_Rj>5L{GQ#V_UT4=)Iv@I!Og#oG8e6CDI_I4_xnm{4cZ)
zU_2$|?6>C15r4S9gH)UTmY)_+NBPHBLh!ejTAO^}?U#n!?@8KE2AOy=-tN44bhkM;
z820Vw)0@)<P99t@-+c_7^MVB|<Y;RG-g8?K83QDc!0IFMZ;me~8l$59Vt(_ukHU+q
z<*;RM_?f`|m~=xv<G;wWMP_R|`m{4y99!P|!)?Sy`1VL*_;<rio~rO8anC4^4w_);
z#_hf@D=)O;oQ`@(OxjRfX|aK&WMWc+?6=A#mi|2u7!yM+QqHvHOe>Fluqn=*R@$}2
z5B`rp;RrSS8M!U+es-^)YcT@pcHrjThd;#s2vy5Fq_lfOuO$y7Y+@sKH4dAJ0>?$t
zvS&ybgpZO!&#)@??^KZy{M<-380KEEU(>JNEh|qi&hjlIYYH#>hjvjbyjjQAgW=g4
zGo$I>5x6|)ubf4#2qm)qMRO18I=27O-(TRV_RHmkZ?E`jqwGXo@Pb8|FaV0sJO)Gm
zT_{Nicvrtr56@X;AIJ;cZ0AEB$?D<C;r&OFtgVn?0hgHu48j(JggTHElfXIVhAH&}
zLyO%f|7<22rhWam^yWPR?p4FqdFxuY2W}-+l;Z>#C_o?zn$4_w?eV_|h8Qvv_-9@k
zyXmK>fIf1sR6udsKFH_&L%ygAC41nlcim>XT}u&b{g}Ar$@+j$?p*z&W{U1$`XBd@
zj6A?*Oo9&PZ<hVwX+B$-O=ky1A$Y;T?=V0+XuwmSf@x7Hf^LuhG)q5wRLEH`N$MTB
zN6E|K!03nnGV)r2x=(YA@aFTK^ixCJCzWR0jm<$JV9Fzah>YcSzYO}wVL%I<?(g71
z859p!eMV}W7Jv@2&`@f0zJFM-A9Is{x4*PZ&7G@|h4rE;3!Q6nIR6#Z`7Lz0IBDoQ
z@Ycc5pzKM=^mFf12oI!0EuX&t8FA2*ofdR|EkMKeK17rY8IbFyIr}=u;wYe5Z9ezc
zTjf$4f~b_L?*S=S@D#Io2G|hFh^}->CvN7Bt5wU@KVKo1626bSLN1?U(isRCf_F+=
ze5b=487jips7N0sX$Er2F$_JRT9zqBo-RF=o%c7JA?UBS%kmh|k@2kGAYETIZ|6m&
zgF{e*ec=$!NkyeCsV4^t9}<+f-(4>IRh9qU9?ypWHe<OXsn8M_5#lz&*Y_&$YK2@K
zg~7I;hXV;YQ||AqSuB@C_){0DSH9c1(gCS}{GUAqmhIQ;QIv#7FZp42s&r)QcM~jd
zuW`YtvwaR=GUWXbvzcXffTtTkDw$2#jq{Y6Oqy8?U$6i5MvC$*kNqsW+uiw0YxX59
zD2N2sXMz2=Wl+@n%g%>1pUt#B|J&m}Y2>zp#)T0Zv5TF_zBGwuz=A@=(8&pPG=!gz
z@b58Gb1*oy9fvnxtwycMO;t9|mq}=A{t2r=2~no4hIb&+<Edp@)MG^My-y=VZgUe9
zX#9YekYg~n9d%1HY1OZKEjToJz=hIK4x!~KK1aXMqqTjmb6_&xUYK@VZ<1Nv9M=x9
zx~=(CzFPk98E5zohQK*Fo=cAE(lH6JFz=Bv2m`3j(30C|`BPtng8697q+!n{l?5v2
zT!&be>a;f9w$klbPqQCx>E4VP0I~x4jVofmZg!gKwd?}JJNeC=@C{&k@%7)&xQ4>y
zx<a*|MA1T^Iwni9vlZ-~`?{x5CbxN1&*%W_<%%x~(jD28FQ**CL&8DkeABUCJCqJ^
ze2KZXpX#eWr|8e?&5SiST%of#J*W2bRkMS$N3Q~C4WMD*cLIk$%~pAyJ=(YW?_IEm
zd#}2A+wwFtV%;vGHEmtfFiDS_OSn@wF2(EYR=`X{Mg-umDD3*G3gSLg=hp`kpgDs=
zl&Nx(t`lgLT$}Ci<(cj=0Tpb<Y`Z-fQjS*{WJ@oZ7GQs>zAkWgHo2xjV(gs@7@7cJ
zA+J1VO)a;_ayS1FsBnIMf+K(1E7pv$2&jeSS~QPm^mpX#D7Fq=@DsqLx#NXA0Ais0
zRXlQ1J@IOTEhw#DZ>t+x7xBvVi(gE5yt%OE=t8FFe5pi5La0w2ZU`q}ZBv}}v1&xF
ze&H@g-s@Rqa*;=y0efBW=qS&+Czvrf#s304_GB_N_Sqof5qrIvD+aEg6uvvt@bYb(
zv*e%%IE%kt!3}yL%@K6BXR;*uD}o|$`$^AN3ckmnXiOYA=>9Yt;?hTMl@)knQ#T>c
zq%|cu-~4bh)4Awlb1F}!GuqC+=)D##U7o+eA=W{Ffw%n(mF6k|q$*Z!!i5<Z7~~b<
zsVW1kN*I|m#cow^^KQo2?;!FI5PQ`6T+CUmohHz!aAUL7tNK@D`T>}i9%_87bEV^(
z#FXG(+2S9rk#;%ER3#61fH24->kCzCjguHUhr~a0xQ`9mrj4D81?pM|(LB8wN%dOx
zSt_&Bv!MjcAm*a;Zyd`Aa!+TeZ6mN3|D$u|1Nd_B!%j)zO`L*gde(%o01&D$t=cwZ
z`#-tP`Ssgl#%K$7tx|imUvc^Bcx}yX|1m1DMn^ue%ZCP~82j2td59x@4Ky^1<{BnY
z5mbAwQl4Y6s=Mh2P|b?kmfXC{#G$2+DGBgyP(?IZs9-0X%W`J@)c+J>WxlLo+Psm>
z0r+Zhv_zM+-ex%ohI<A!aUV=s*?qdOjcG9~6;yyv--CxshF0SVk?XRePDe#LeJ+>0
zjJhfAwq+a}l7`tM#8@%wsT5$8QLHW2R|E*WZD(ZlCnxgcDCp0If1K_ei4WibizWkX
z)xxH_>T`_xtS%H>!4KMc7DOCF%ypWY(w_~}?vHcR8#A=0+aB%_mQgH?M{0B8kF;VV
zt1If-*YpenQJ;|-#kJt~52F#kKBH$4DD(cJ=v@Fz08|h#$D2BxH(LHN<QVk=NY{0V
zat2|5xB30bXFa}ID>(>8m+e21hBn$_<;p>c49Jly^80|wuiJp*(V5zaWCF9d9yA*G
z-+!TwVYkZivGm{fG7h@)IPIs7rCGg!%_c{5B{FFIfTCQ}ClH}x`fuQ}v-Nit+!yfn
z44w$;-x?Dm*S?@S%~sD*M#n^x;6Zs`UY<0yb9B9xEdU#SU#A!>YDEzDJ(g$BABe!x
z0YkLaEH(JCU*!H6*GkyGD9v&(`X62gJjOpTJB<>{_jOkQ$uF<H#@3bN#e8$WeVt-Q
z;&Ywu8wK1X%=z^}s&i$-w0uRl51_ik-Pml4F<OPn<cfd6t%s55QG@#BGUkjm)Sq7!
zjtV38$<r*PjE&T!rE-64bEB@;aT>nK9T@SL)+=6^%8nGzAw{Y@9AEEDbR#rBWE(;S
ztFXZ+BVPNKjPK&6;RNj0^bMQbpO8?mfB><j#|?qqZt{F3IzCxxpR>$JDh{bHU`4KV
zrVN5(oe(jg#p$m!%os=AWy$H%n}*Xy0_yQgf8YuSxsZ-hDl$KYpV6`Sw&K|fc1mGO
zbYvqX(PTSwG#S(ImzK_8P<<uw`i0EtP%TzY4^p~Q$4*hk3u;Z2-fQ*eaR{b!pYsPG
z055XvfqQ0pNrGmhTXr^5OdJbB7+_Bc=#)7D&Wf1TA=~z9s0dkT8+kw-r}9p;&KnmT
z-C}+`nv{%3zrcwzg`&90Q1HC08nE6i_d3Z>1_{%JKc03^D6lP2qYIt56>UZ1PN)ka
z_UrCUH5RHmZU%1wbLh26Ppg<3S1L89i3Fodt_=leYyXS^qP;g9OcRONe46DcsA~(3
ztqu*@e6VoTGy6#_IvYMl*%#5Jb=+;Lnx$`$b7@<aH{K(PW7BNK0<-I>w6&3d=yq58
zOo!vE0SoR35N#f)h<-{3eA<GRd+$ptciLTBC5*<Ps1j2S)Mp~M&VZ}h`pF!4*$R7k
ztpp}6P_4b$Ef*lDbtTA6Dp1dxXhkuxgqFB;6ywFcEdcxtwIjI01#z}RKv9Ph0;YAI
zvq(u>@`5c988}FlRV8#p@yuIOp8}NgmGcB47GR71(f&q*STB%}EvP70u7JJVGiUi;
z*FT@2?X7@OqAK}@aee&xX^d$gd5crSC(NbLu8O)5Ms%FCAZ;e<TJrjg`T~DwJ?Z=S
zyIGf+-ud9zeQfUWen1YH^{sQms03EDw?{ji*UqNtOmaDeodonqiq8e6gCrbZYcoM)
zs%GbLTO>Q{D83N6wCrKptq#|edOC$|UC!H89nKGj?>P`X!=Eu0tO$4heI15bY3X7f
zHozLxn#W$t20Pk%w$x-9O(g)7^3nlSrYdJw^Sb946?8+3st&peC>JvETDFj_UX5hh
z6t=o`lb=~YXHsSTp}bRxH%%AaG+iv__*wwo=JraY-7MhlEx6JMCpD(~9cl{;QhD|B
zODDUQMoWPT9|JEoJ&{h<EG{RaOxHn%crI#P4ssFk96J2!>36O$q5F0#F1VSb_Hn5*
zf8i);oi37+<7-bY9YagsK3lH7QM722n~Q^tm~Izb>JMmyF4@)7BDRgjYpmyF<+r=5
zdPp*y=KXQ9us^66oC>0+jQ4YT=5|RJhVj!CE=D`7KyVsaX8W41N^!5MiYimYG|7N^
zZ;!1brQ51A_BS!0R9)b!SVTeZc|&($it+BMzRnBQbRuiXX(Cl~mSCuSuM^&OGO8os
z0IrsgPJLFUTOcLnM&$3{x;h~qlF5@!;yiCY=+H`PQ<3Z8e3KO~s@Bj?@Lwt*+9)qV
zrJ25HZGAoc%gtQLQj1d^<yWpmgtXs&jST0K`{Y}&kvDnJ&1AMT5Me}$6eW%WDHC;T
zI=eoQiIo>6fV=?{7*nAEQxvP-bqC_hzVoL;Q}^LyjxY2MAd$WZf@wDGJp1zT&7j>2
z1G`$Taqi=|oW69yFZ=Q}{rs0P5Bp_b9bwD9WRy=#gKCN#s4?p^aPnnhV=77wQ<`qt
zp)`)%LyKqAoN5IF>_O3|v^*0pcZIPKzr#q3Uu==Q-z`P%0{ocBL><-bHcqp3P-$Uu
zWDMVC>h^AVRm`z|_KWy)xlLM%$>X~kqj)^MgA>_LPjSj7#AehfI6R5`^Ez+N_)Qd?
zD`QD1bPUe@kWmN4KV&#Genm7v7d;@g_vv}exjz$Q+;Oe<#Q@K1OYnlrR&%N|D2T`L
zDeTQk2Uk9s+`x#w>sO9ez}r#VQL)bZ?o}>Hy1jtG?b@IK;6MUTP!aw*d6S7aY-Ouu
zs?Xz^G34mz4A6I|OoPc{`9r&b;3`SB_>w7i3;&UytM@r@y#NXZ>xvBG@ATl+VIG)&
zyb;5MTkiOYGIh>7#JCIvx!&o9OXt&Ur}?I2fjqzkv-88-PV=fd@8_XC={eC`hEp<=
zHtx(T%bOWauQPpjOXH0;)W)CK(Fz3$TNO4X8!ukUN_E3kbfeyVb$v7x+EIf(8165T
zSfr2c^dhUUaD}fvxC72YduoMF<oat=R96);4xPNL=pY#1)gU1|-oxuTjP5?Llq*6L
zN2cyH272OfA&A^%iP=cs&ZUyPojMKnj<TJoi-O7%KD<x*FnTl6k46RId|=0H5;&;W
zaMa%`X`-s6#W<^G#6Nr2^ao#o<|MAs(s*n=tg4)L^-jjrxaIK3FBzmtfgk!nmjk7`
zpSL3eD?22{)3%=-7v5ILjr6CdI&hY^Ozxnut$+uog~*iqp_y}+n8bSK@u>M(jG(jI
zvv<!gUUojzW#2Jti2W*_)RxB=4*VH#wG#2$TI<py;4xy9e64Sq$|Kfrf3rt7A<HWs
zEcy~-I}3wFR=1AUF;DSwjkR_EyLDdZUOIb*=e$kpS@Vw)RCK)Ua8&bgN`I^C06VI8
zS*hUZ3Y2HOlpGFfZ{P0KamWWz=yDjvAKvMU4d!=b%>#NK)sS~;Fn}2fD<-qA4za7e
zY2d;kTMD>b`Z|qE#yxzV*+#VSRq~Z+4id6Dp3<n_jt>{Gt(%OoyMDs{z1IPIlRur6
zDn!;LU*9|%G1y=XiDXk)P~E9<{v{bIijMESk)*RK%(q})*L)mu$bwln=@zog7_<2R
zgpf9<;FjI8E93D~1MI5W#L+0{oo_yUR2$QgpF!2cncQ~=l}*d??(!ariX0wyJ>NZN
z0k~%e92WZ0rY;+5OQTWgHvCSehihtZq>Ykg*jcpnPM?;(_T<zOFKIgCYA|wZRE~p{
zS%cKg0!t5B^sxA$v)k9^d>6{(a#!fEta!CX@$h1+Y4&53)CASxE+vba9by?OaS1;6
zn$RTT94B6;7BT|Srv_@d_e-TW`m6wWno6~%ZzW+R-FEZlP!c=46TT+J$;zPp-f3;t
z>KE*gMq0eUGdp~xZf6Fuewri`Et*LO{Kahd3Aw{%H9_({72>BDKsF_e#K3v&6$^Xz
zT9B6~_q;(bUU_HF<={g=Y@bg>+sR<F4cD`DAP`yQR5zzOvq!ES`HQI3t{B{n3uVR+
zrY{-2y8NUFV+5bmZH(AJiGU1Vijvc48&PuJkY(_rmlu#G$oo=577jhr6Pt6T3N0$1
zH~u8nC$q5)Clkj4sQgh~sqf5I1DS_RYB$aA(xud&oXOY5iroM7%R(TOuoQhaMJ{H|
zGPPJv!bBII$2F@w49{q5bEjjH3&Cd<jS`8GKGdNa#4RtFUyy5b(>(lj6!Y*kk0-qi
zD-V3~?3FGFx=rl$-c}Qd)ohN6)h?H-KdFNSxYqjWYAu$!@~;NJ&9zE8Cj-yU(aLWE
z59n^?`!HstoZCFSJXw@q)m1ho_qn7vxt=vQCdq6~(x+gaOirI$Tir@ehL&#T@*jZt
z;`@b(^kJPgY$8C$lKZq9$dEHX?3~77?o+K~ICZ!XXv@>BfO%ZhnJS0*uUzXZXGZ)g
z+pcktD`|6xf=nN>03>fQvdbA+(NbjNw+53Cp-!N>M}-Kjr$GQ3LuV_VY{t+`JB>+@
zj~Kk^==C|mmx35m-8G)LQ-$s{&&PD}r;Ce!VFDFFE)aRE$iX7cD(R7Gdh62L)W=#y
z;ize@m&faU23wKzi5K=Q^&Kyf1A9v0Uh|F(Q0=hVv*h&ddUbAQ(ndh|es*R9d0_JZ
zr_6fgVu>Q=5GI~A7g94jCLD?7V_e!OkU!68DZxh7Ur(GaUYbBWc6Gt}s!#0Ci3)DP
zR5=LBHl6#U2oX*rcbO3ZZiAPo!4Q0VG2d%3EP)Nr1&5hW`e%z&=egwR^W#G`#tEtD
z<nF4gq`SWIr^0O3rGefDcY#?9SlLeWCG`9Q?ROUobc6PVW4%rJ)J+(m^2tP)EUd3S
ziMCo%i3`8}Blj7Oj=6e8yz8zU^#fmET)CKYeh1EPg!~WvH=rqbc_WGk9m!d3+;ZPF
z+Xj<auX;8PD<82trXedCwMp~bPl{@jnm~>(n+KMHfHxY#U~k4)QN>}YOcmAQoPoDj
z!(~2hFe$?C?>YX`pti&X3zysps4Vy<9rL%8)zZcV!Urlm;wcjhcj~1o)j|pUP8f7)
zte+Amr)+e#$qN5rzzvb12;kI4j{mp_Bqjs{5lZ~l`xPD3Vb7IF2?pgv%Rk?qYTZv7
zyqv(o-VMT+C&biWAr#)LH*x=#?}Yo1NnbclRbE>KD50d!W0(=CnZS#;t%<D8d6GN?
z_+0Qu0KX)%7-WfppF-#Zbj*h_G0gFP0_gqf<u`w<nG=J|__p6F0RYbA@^A@{wj{<3
zCE73X>0=+2zIYDe7OxDm;pAh4*u&m5kn_7+W7|!gMm|$|5%~ZF!UjC3b6L1mxffB2
zG3At<pV$=H2uys$#KrlBwD~JZr%+hgWM^N_FQtUL%poIp?drH^dQ8imX{6oQ&^%&p
z$F>6JB57H6nzPfg*9{Z>?^-)kXi}YM>OX_n+g+761@vD|Tuk=--2PMGC3sjB)kw;@
zfhJVdwxgD|$iT>XW}_KHTR-MHObvlAMF;oBoNN@-h;v0f2XGU$TW026ou92PIUO5R
zamIW)nmhS*^osxtcr3IIGf{uRj9*{@pS(I?X<GcAp?TRjqdi?<G>$2tbcg^N2wK%1
zXq$;>#G9yc{Nz7%>03VAz!D9R`9uWZ0inq;^}BURE9WyNvDrHJ&2R0VNQqm&o+}pG
zNYsd=QPSmLSS^!E3x_2!;+}pDQza<H#<1pU%7W{C5;18*eZSk3&NSu4^$(5-R2h^6
z7anuc5oT5%_?Uj$AqvA5k=kuP<O+eYg2)~B>C3awii!YLlcsSCJpkD*og==-a9+X>
zhxr0Wtx6!W;h2sgMC4Ii>`2R7mz=Gb#19;AWd{kz_ZFu-#I@!80>=g|y7X;Gd3D;2
z?M_;^0w5%@{7fd*8_;lvemZI#-rKf&DUrYlzMBE$>=a}00si`$diqFN2JI;4Q6&#0
zsDdWhLb&luBUt;rvdG=p*T7;sH(>X4h`;l5_?~U^E@k;0rrmHMR4K$dOl+AnL{Kr#
zB$1aNia0|tAMvJnB@jKq*A;{$LW*`|gM#s_Z|gJ;25-X)RN1eeRQ4HPg5zHxTw({s
z<1{eH;ydUl6UKLW-oKQmXAAMf2g&2TkwSa0KQmxRoyJm+i%Zj|fH_%QQ?vE2#s?|0
zHZ8hpB^6IxhtkSya8##HXz`QxJ)%O(pGJGhoED!!bDzDX8*@Y!g+EMmGV#w(e3Tg$
z;9hu(2jW(!j+#V7)WQ=j4CA3avU`lmOgVyn4rJ%i)Gz^4;d!?Ad>yAF0y+g6`jw5P
zMD;&RI;e<r43yFdF>MIqDvgtk3P#R*#XzY_iZlHC`S>vds1EI7OiEUOnDbUylJ86#
z1jb(lzLz+Fax@qxNMnGQ7!=}2E+`I2#;3{IKZr{47edQ+^u^}&mW$T_*(3o9Np=l+
zeVx&yT#Ra2ia(!IjuBv9T?Rb2D9GxM;BKM_@gx8#XYz9$3y-%d6pfyKx{WG;rSJ={
zgc8`&LjTm#77;aNX~6*?$RkfM+7ttiUcL*P*Rk!mH&5=%3=U>J0R3G7^`G%SZVID9
zN=4IHxt}DjzAXV5qTO4!P8+Fa6^*I+gv65oQU}EOUW&mpU<t~oZF|{bpPw{nG?XaF
z0SN|3ReP}8>I(3N0zdqL$Q7cE@#tY&dGCP4C3QQoc2s4X6nO&ml)L8q!<e~SzL<FY
z(MsN3+rY6Z0ZZlukRX@oB{{q1X5roe06H7Em=jJT%WNPDN)3>@1f1j5e0k@<A5H2v
zEMxKb60KoFtgX!B)4pg+01K;1>K*I+ub~suP`7t>M4>k@H5qa(Jy~bVKVP07a|s@G
zp?o&6g%>3;8UwY)dCV&!<hO4re`Oyw<Q3J(M}}B>P$M7Ef(Tp2>^msR?ZQkNa=6W}
zA2jCMy!3bhkG*SjKGu|9mC>j!Pa~Rx5dt_go<o6XdNg8&&%6g>IZLR)dsUr*i<h^j
zdqBOW3QGa@Mz1I-$#cnLrZQbwq~EDM`!pWcVnd6jpCuNUhXg3M#hd>L5naI3b^e8B
zM9~3M5ax@NH4EBM=jA|G09-|r1@jG8`N3QBdnLlmD@hIBW>!a(F8y-$Q!D2zrk7d6
zX*6{H9e}%(E!YMn>sh|j&KQRF*m99#|JHT0KlDNzn^*Lv)vyJbak+jYPh6kY+tAJZ
zT4PB70V>0bu<r`2HsLsA#KjHegx|QxEGx3nyhd)kI6Md3ZvB>HxCU%0M$u=o<PvCK
z3N+ggoA|p-HaVG?IkauQt-4wQe{_EO+GqW_868zLxnTwX<H2T=ta_D4Uf87%%t9Gl
zTHcxUzOXxaD{NWJ9Rh@9v$L@z)FVE>!2KMaM<g<J%0Hr^aj~%n==C+5KC5fKUKQHa
zdu*osdwzqna;Ha{ZB7oeWOh&3gr9j;yd*Y*Fozef1~mB8^bm7QqvzDzQyZByB<vr<
zp`I1hqq^B4z3EHNY8C8bf@fnA&^)h&(-tcamhKNGpk7<8h+@~vN~gDOTbW*@i1EK!
zdM(KacxMoFD>%z(#?UQVtpz~HmS>5Sd-)AUS&BKRN}LRX?#_E{SzD}X&l~<`iTH-!
ze1N<sWkThgMlY;U_KiNrwQgK=RWQ42xTYsEXgA)E<Ua)rcTcC@WLz&8`CDw%zHX)O
zC2O^^x6xf942!Aa!-&YkZKPfC?HO|;_a6v=R!865<VL)<tw2GPNtL#@$oP-dT>y#Y
zMMSfKfT?4J-V$$?N5#R7xPKnD=d#ZhxmiQ<>h}sv6gQyW*w<G%TT?W;c#btR-MY-E
z5?Z`^XBI0o4KxnW04xK7CLS|%#g$TT*bOJUZ<7H5r1$2??J|2g!$)lG>A8j7Gl*2f
zd=D7DUbyCk;f{uMt`g05z6|wNz5KcSoYZf>`u8*GHOXisD7+F^_43C8FgYnW{iIRc
zo5OLD*DMJ_Iua3zx6O!5TC87A?Ba4GTwlJhx~9Fqy|B`}m8C%FP#N~~ox<w+CAw!)
z_IQ((c6IudqkZyuM(^fyeWo95P7B_9bD;Rmtaf7iN3X?A__UjZ7!2)(_*tjdBJS-T
zH)=(v@MS<r^RsrQvpdr;;?1MqxL&aLRzF<AGqA_Gnvkt{vYx>+d`Zh{YI-M0`)mYd
z=-H5>zrwuNaAriKx6Z%WSd3luw&clbKF%xLkCm=5#lgw6ehzbJTAGgTOq|9N|K@=k
zm!#Cw3>4)CdoD$+7pGgjoPzp^kv#~;1d|%hEGqJ1GK$AJ4DT)<%Q9vj0qBc<@b<$@
zu-7$Eu(F!gx2tR!QbUHs?;`ZpkRTM?H(Ayt&iLoD$i5xB%{!kr;g6;FpxHtLJG8;*
z4FBAn%h2VB6hqgqWL7q}`Fv^+5r;YmcW*iti(rhS=koYW)k-C5uo2Nd3gbkre3hCR
z6U&O(tyP#Mn9-~+CVJo8-e34zet?ZUqq252!JqhK)&;qpSx2wbM{NE}!kk=Ezp^*Y
zU_2&~fN0E``9c5sAR37w-W=w(<b3t31Cn=ae@|01v@5^Ku?gUuDk8?e8@VE!FZ8OF
zp2syN`_JyAv$da1#J9`Y8CS(MTuk1N+??$s$g=GOaEN)xr`S-?rs>U#d4-8IvR9H5
zS`|$W0Oe9zRywD`52*vd&XW~u=-nYMaf4VPfgDVrV#PH4Oz+jNYAsGzcKLo&So!!B
z`w>3fr9Wd^f#NRRsz*;q0FspFCrZ=2$r-wKBb&k`Dgm3sZ3188LAG@uYRS@Ds;A&B
zUsp?^HcBI@CBR8z8qgG}v{yn;v%yq~Vax)s0H)Ezh&$1!g(4_;!j+X&7b#Uq{vKuG
zt|Yw??2kXN;?3V70^V@qn)(SbV0bZo2Dl{I#NrO=V9|0+>u)whP{!V0q*XUA!;ip@
z-kT4358P%kN^s;QJKqZ}zpSQo><tM@r5th{(EQinKLMqPvUJ+^-njQeaG2XpW#%6R
z9zNmMe6Hp{SOGTt=s#Qp&<7X(B%Qq_(jv3YI}BY5UY;kHa5N(oJ>%*~B}cH-Da(5k
zqLJ4A`ESebqMxLNY4Evz6{F_~Im)Q|!ZtZlWJ<(BFv@x^0g2Yat~Jc()eCA+<PbUj
zL*SC$mh{PVu7yHEB+Mr{v%7vDSlQpYQQ@d3xUXLgur<Mdr+bb2hrz9YETTT+^rF+w
zU<ysT1xVcm#C`gmXpxYO#BwsVLGxia$L&RkuGmV8Ws{Ro<{|C-<pii;4-;v6nisRa
zbc45%E%I5KRn}J^9ey#Fq?kq_sJ%*NL&f!~WcPQ!z~A%WhK6!AM%rnpY^yk<zD1JO
zqRF&W?fg_s814N!9sTemyckX7jbMYAaD~62&=)8z%gZDmNhQh~SbYQ2tKaKs2yG2c
z(>&3Gq%rAD^!>qCbT6Kso+0ciYNRa#&$AM9hHY{9gFH`cW{84rtl`5sB&kXLbl|Wp
zZFjQtKk*>e%n{o~9%!f%&r&_rDEez}N)8pwE6t9ZGoV{UJ6_qx0_}Qbf>Ot(p+H0h
zBTsN$VGpQem9Y2FQH1$)A6M|#BbqHygO`~r5|Q>JVe|@rpWir-w>JXKBHD%`Mik`^
zvxOyR6;IJveGf~r=ZL<>Ly48A)R4K(vC+=eFi*M>Vg$1I_C=qc6qyQ^i>1Y=pU#5c
zf7@r&ES`FkptV91aiW9}Z3nt;R>fR&!OINq+4Pt~FMcLn05N+Nk<c+2au;gM)rYF~
z3wgsJ7p=}AEZ^80yP}$4*yVyP1$9dh*Ic3ZZi+U6Fcabr-%<prbD7iiZ9@{WGS}O0
zl1~6gD*<Dhy!F=;vOjkdU~7*xv4`6UUJkqcoa2h#sOTrHq)SQ7K=GU_9c_nlYy)*(
z4dgv0hTd~o71XI~_a@C=nSZB%7koQ<a}`*Qc~wDKs3yW!<)Ri3f8KscbF09VZtMDd
zaz-z40r8_@{`n*BbjX1}CObYK3oQ&Li8&6)p5T$?5F4Z$9Mb=5O-nZ6iGqCpGJkLB
z8L}t?3=z2B^q45q%0Vu#d2fm^wutY17(SSz!TN}t0Qf;qjd8>#P<4$iy9vhFp4E<u
zF(uKZpXwGx;xF=@{d$;V^c&GxL48U?T+SR~3VV)g3&0R{@T^IYa(baQJe=R@wuRnU
z2nMlD&rfrV&J6<r(oCTOTCk}_-H($&HE@pnPb&zE^hAMLT(pl8YH^5Nce{4A0w;kD
zO~vH;*HDig%zbvaZ8`Sa7f-{&Pfl|%x7e+TOj2in=9unmri+G1#e;Jj8k`YQd+Se2
zea8zit|Mh)ZKHMNjZoSUCJ@x;F_Tu~b+)S)!uQra<#*XR`AV=^NK^)S(r)-$upzrg
zO5FV1=q|7e30?9L`<3Fa6dl6y#PoR8)ZdqWSjzm%M^}X}h%G@00SF@L)Lz=^=a|WI
zq{n*GgWEc~wtf#Y%xt<XvQ^j@Y>nK=^y$W9zBBtta;)%;wW0sbYk6$Ch6(<Cg85O`
z?{<(kbgOcz+A4SpN5AEmbSF1_7Z%kEjz&G~K20j+`E5MEZXCq6$t~s@N>8D=CrA45
zjjAU9()-3&k0ARiFaMalXV8IrWkGwYa`B~uc3jSMAIpjIl+bn(m4C~bHhjhNOe+l<
zIFb|*%(au2I@VBhO=Z{nGcN=vcbhD31;COvO*wus*~u=Xf<g|(bI8$x_2hl0|5%}^
z+hF-2#dI7ORByqpI-%ZzOx9u5v2N?}GL}+uKPGt;liB;Xax1l8xINHSHynsr&q>WF
zmn+w4J1VU3xAo3DT_7NrdUjal4%ezRF>f9-H%N0KqOKy)0*&okLicAvO|C5YVu-pw
z#+EMkoOwb;1;N(Drz`iD{L`0F9y1BDeMqQf&&T}TczG(WF(6Jw63Mmpb<-_vpG<WS
zGzv}8YgrOGMySExJ3~jEDr=zuTO@%tf{sQ|$>~MM0-v!n!!ng}qID^8j?);alPaQO
z3*@fn)Y3GR*y;6d^?AQ<_RCA^XuX7VJ0S$!)~kQ|pq}SY9j}D*iI}k()*pjdB9S!L
zNureGoC7axacz**Jme$l<%;Tq*}t)!vIKt8KOeX4vmI*Wa>iK53&2t5rz!|D*m31w
z<5d`g+8ekFO{7;g?1G^b(EAFCcu9{6hgH-dzC1^6YYW|$K)r;uK%(eS$%(6h^j9fQ
zocOKZhLCLPk@d!(lLJP{g*mrXJ9y)ct8OAEZ;UwGS{jt9b$+%UtL-dHeF;><T}SaS
zQ?mU}26#*BpX|Nkc}ZvQ+;>R_yQCSz5&86ox_v@9L20LY`27&gjgvk`CHM9F+vytv
zB`b}YnqkTCQN;}=DO~8>j0^#_ur3Tc{)!lLx=Kzez#=eT7D(Te<Or1R=MfIcjbK{x
zJ2)w?95$0b>;^W$l*$#txY7_fgWw?SwVI)KO{nl0<+)p)E1xQd#-0i#1P>=g6h%R^
zp}2LQhMTVax}7S6Rx3)vMjejs%6{iju!eEaQlaw4iwa<YDyy<05gbE*{7TpvCuh|?
zvFPq@+cQ1SbB<%q$JYkOF3U~G4YBK134P#&LuE;Hoekk#!AL0hcM8aHs-|2ON!s78
z0@s@$x~}o!Whe1a0`cpAWwl=ZA{PdbgF*@3TxJp^tlDRrlUEMO-R%_E49fAxaoQ1t
zK8?G6iJ%ZcWc?TTBW}!(=tHLrSLX_%7B<*zlrLt6hgr_`th6uDG=C%h88Q`~a1}K6
z?X05|28&whYxN2!GRDE4@M9YxQ(0k6d*|h?_8|N}Wo_#L_{GV?gV#>T8xTko^!Rg=
zgG|gtsU5?<Sb3UW?f41v;u?}XPSe_icn{{!HHiWAPi^Sma?lQ0%hQ_CCD}(Wv8#P~
zVJGqO$ya<Ejk1MPq3g`+rH6o#=B?$+WyJDjz{A7nRshg5EVyu;xg|JyYLabPtKc5%
zw`exO37LYuW%C&rpc6j+x=d_zGm(-d|8M}iKS+4Ese0H|dhqMK^wZd(I8E(SbOdT%
zO*4*OtqV%`mq8Cr5JYlQn}ut+!IG_O#=@fINjOiNX;Isj5Zp&&W9$%UG;p9@(&;!!
z8jxc19cMj2V|5N0=)7C#gcPiG107|n1J@&79jq){Ku@$U7Yhm8XVlNrru2b|q`7}R
z19ofabR*?>+v{}Ot2uYvUa@wmU~)4dmCnm^U~=1IqWFw&J#N>{aD!3D=J>uZ=(cYF
zDDo{8`mA7^@HQE~{W;<Rp4>o*6@Xojv^1Q>-%vN=`py0>8bmArYZ&nI@AInPc`SQg
zEC=0h1u37C59T~LFYSoo`l{;OM;V_{btY|vcXd7>I=y&z0{r*QDC!8KGVh@#cThzi
zr+_^JI*}t3kul?kHIe(Z9xOvs+g?#2>OO4aqfZ3$duP2o9WIPGJNKJT_q_@ATW2zC
zQGqwGhdY>23$FiP6pc$cL=jJWX=~^HfJc?zQmHG&?O$}2D@Xc=>86hJ+8*WQ;}|c+
zBn6+xP9vG5%e~I~Jq;hHz|)M_!KOh;ldEo$qIu_R#G+MF51ZEX!tzC9DR;<Dyv%Dp
zt!bbe2<o`p*2CgBzSKHZ$jX&)IhlR$?v<Tjf@m<=n_!Q9`xe>nc}T4PI`tAy7te_U
z?81SdcSX3Y5^4GF(ChMTwqGV+-1)8ro8O$Hksyx-tt@II2x$|y<K^Ab2CLWHx%RKv
z5+VMZMQN0WbCU;;6Cv2CkV6X@?5fu&gUzdX2(li%7nh*SU#f6A)OkB(v@AIHfFZae
zr?R^&6tHKj(>lLya=&5XOL6NJbm>(*@f#N&N~6Pff3@bNuDH2Gjq%v%Y4p&1l2p~U
zS82I8m0Gq2NZsXV{T|Tl7MSs@c?H<>p!&<83#%YcTKJIhMQv$YKZ+LXNq2$dhXjna
zO|Ggm$^~r-$s&KCukDh11?a5}N=X)6znh=wn9c}X#u=!Ji=capoFV>K<>5@F_~w`+
z`+8VOOGs-p_HH&-sbTFTrPCjgu?D+af+-b3W9vrv?HF-kZ(73N)e|(YrJh7ga7ysZ
z_`uPFj?k?@`6y4-lqjzrSJ|$22sEguK;MM*Td=yN4T#)Likxqrl0C{eE$liI#-}yX
ze(1rbGLe<9Xx=rI137MNYCllqXQkY49t&gTO!8d*NV+*bWcj9!QCt=g)p->~b|-MT
z=%xd`#S92W%UR2JOGE*<&C}ANnr?iWK4U*gW&SzsrG(n`Eq3P-wplGs5#QQ34$Jvh
zVQWA?>)iq;y^iXWNL$z^v&fuVpid|$<5DJYU557IROBJQ!sjVQg+3C-bim*?y5L`=
z+zZH^|I(f@&m@f9@ylaXTe+~cn+S}TP~Z7na7SSB*ex4jm~wGm6?8WokoZZUP)2a|
zImbpGXe;9iU|XB}vCA6>2P7jwQ;!i8_UG(aBl~7*xV2^Z9L$r{kERd)b2UzJH=fq^
zsC~;DQb&F@QAQO?rvb3bfJ)JZ)-<}WU9ZK0uOs;M&FK3~ZVF9E?eF-uesCmZU0bSs
zW}gnozFntV@HIxv&opk_jlge5KD7IAJfG!8ZB#>g5$$qSge$OL&9JS^yApaxCcVup
z{&L$4Y3oJYSJexD_+_9|#^x$X=T%H-03%1N9}3b9l7IOPdbja9ro`DFI^h}xm5vDE
zvs{S*tHV2hL#OMp_uH}jC>n`W!#pnp=?;QW8<mhSR(Y|v$^Q29tn5+neC*K`KZ}bH
zG-ssn3EFE4RA^dQi+#9?ZL;~x@O+>4A3+0Z>ieO5O9<`}Dv8kUvB}9Gk~bY>gK_)o
zIzi_;3n%O!<>gVi!@9WXK`!_w=Sdyq@cTn}7Uj?`^{e(o?LL8}9p!+5VlZ6tkXLIA
zUt~lkYvOOTdX1$u*cV!^4E34JSgvP4{#t5xEHmE4Ep;?*#7f|<!JM0sx4jMC3GHK)
z&nS9o+cHJ?R_(NhEwF~+H7|r5?%j9j3zMrMwO!C7WHS1``fieNL*78!tL6tGlCuWt
z>Qdpyo;kt|Qw`LFB`#%o8JkFS1r1cTrSp&RGlbZqMf5%`cE_QOBvLwF*G)l)ri{GB
z-9U<|ggkEtN7sZ5PM{TO^C@Z%bphYu&oxUhiONtBy<O{`3Q)nykP*273Ig8hg#1cs
z$UIYT8RfqhF~2%@E2X|Gq1P!oAztx5?YL|Q@~E^?{~`urFQv4c3_loRS?T!2{sb{o
zSP{KO3+zYRWjov<5^CUg06gq;`iZ11M~H3M;M7gz((S<KK0_)P-bTM(>_~{)RfJ(t
z3}!jW_YUTgD&a^9JN{Br#XhPr!*iUk8_FdGf)ZjUgInxO5c3dppquxwNm=&tI5x;o
zBqML%QJHTe5x8>b)rsKhA<!3Dpbki{!DSrm*$lz;|KJA>66i^CHNGCy;r43ZGtx5m
zVov^{^zLRmkj+;;Bc(T{?i-hicQ-)`qt8RnLJgvQ7}E_(WkHy`HudT(LF76?>3G7h
z#CEes>q2F73w{HJcaN3Wp}VEsm_BFa61n{?vN$HPM*8=e*G~zJ7YVEsbi!UFzqC&i
ziv<hYO)7fQ^Il}hXNcg3v<Hj6Q?!eGqdud#i8&qdc2L5LC5wGJiJo+6gDNreTDv79
zDFNRmKNF_QR;_G-({odckx;(3Bmdlo;khn0^1~7C!x*7y*LH0ba<$J?{%dHV6;3<7
z)1OPXuL_2FOlHM&M_5OZ6-!b*-#at6u8b1z5BM?f>oNAzsq8gsH%szkY>n?vQoK6(
z7{t4ZBtvYfgw9J<99{#zZfM#H5LEW#csO7}y<!D>RN6JXe=4M_w#fxFs@4W-$I85N
z`IBhB0j&!3(0KZ#+{TV%C3?jpEnio|_-<FnNq#*Hj5~d@-f$q9BROn){}=Z@N37^w
za@^Z5x*6>E%;INhOvcBNQRZnG#4q*NjiOEpu5&4Fa=GErj_&!MiY(#yIIe`pmz~!c
zoqW^;jJ?`Q?|Q9I-@yrg-(B23MmWQ~f=#Arl+ShXkvax|ZuvF_>1rcPp0*X71z3Lz
zV%sIw&+xRqSg5i!`c?X|4|C1*2S`Xo5&aO^`gn!h--ZPI5R2!c)Bl5}s|;(p4flY=
zC}HHN(cK-=DN4sn=V(Mgq`PaRG&m%rTe@>}h?I19gEV;Nxz72v&(C%5dG25SZ_@+s
z(+hs2@5fetP$x^=p;e-IKL9FYoiKoW(fjphm}GAlIhI#PJKtL+^?pc6vd$aM594*O
zyn!%(8Z`9AHjzVaw)dHEgM|vX9O5~bc*`%mpVT9v?=W!K*c)nl8`Vr_oKu$Y^wTFk
z(bH?vyRBw@AKN;nIJbD`Nw?ZZ_sq@Z)ns*s?OO^7{RPRk`{eX?D|CvQ-uER501ArA
z4-vh=a~0myj|+o0#8MAp7VY^B$S^3KcSUj$4ngZO*fEa-3o^g@_;2~(3Pai;xKL9)
zR5G6h2hCiwcU`jcKl4S9A2|O!Je%R5HoJ(?8`B9RwWoA^*;aqCVxl75Rz5mdS-Kfk
z54cn@C<iq;IB}pHJ5Z3e+}%$G1{8IKsZhxsog;Bz&1K$*%pHj&Y>j2u=B7t@FD@O<
zwSs&3tB@W1(;8msgjsySMyxzeE%Jot0m9W1-crB?64WT3jWzVWe=C|)leYqkaO#=C
zq|e;4I1DLPVU9eEhU4Ae;nb?n-!Zq$wCHM<0Ns!QXUmnYSRU`4gvu$IRgZ9oYkU~*
z2-0d7EYhtVjaJq-+EcmuHnpp+l1t<1+XMQuxhVs^EH%NdnRgl{-fd*yU8K`X?tNx%
z=2S5?!qwm`sP?-#;9nePri!~nz2Z@SiL1CPeCw2Gt2=W~;~F1t!yqM98*NjSI5+*o
zoo1V1@i6?%imH7|2Xd)IXAYQY%wn5BO3M!*r{9s6Y=s@Qm&+s5_UiK<XFT<*KlN|h
z2)7xW&FSv#jQo%{2L(6ls#ALEJ-30f@PzI9O%#@|8i~I@8Iy@6m5Sr-LF6({5f8&=
z68e|pcYrAoZ~E@+;=*o0F%-X0isf&&8w+?WMYdYhZB3-7Q<@uZSv2rm2Lt}o1gLpI
zH_^16^e*-h#YKlFY-#Ui>jQwniF+16n1KClU<L2*Yv#><JfN?nSW(#eA-!ycm*Iq7
z9#WxX?@yoK1n~vi8v6#azZJ_cm5Dmb^{d>i6*pu#m;=?Bjx>;b00bRoM4fFi{gpO>
z%nhOR783qlk`3&9Q5-m!$VAcSQqW>k8-byc@I&Kpl*Gn<{OJK%W+B&&HNgCnw+MHd
zYr%*?Bub6f^F%#!qw{Rne9yH5_iJbNtNZz`mp{rEt#-LTZ>0&=2Mj%T?_<-geV5`s
z!XB6R*xrK)gC)YHFD5b$w<<6qZW?wsKury~6`iGcfp>k_Qo;}sMVez7p6ZEk>`9#H
z?qtBz)8yZ$8R0z1&WSBCqM(0^o5U3*D57pv2M~Be5q*7ZE9J{XZSk8{n1g+8fY<S5
zwHRQITER(mO$FVM+4!3RgH(3K1>=B0^uHHP*CCg-Strsorx*wR5R*N`VdRI!pK!wB
zIEMzdYC15SkrOs_mYgxeG2}ChQlo=)5rfK#%42T**XfT`{9(GBlXTIs8Y;+_1A-IV
ze(?`Up$=wx*WKmaY<I{@1LRktZcp5Es%4S&#fOSJWfn!5h`9Jl?o_~YW*IZ_Z<5~3
zjTmh&!ij?b+zT@^J+c|X38n~`r?x{YReSOXsH~jtxu<({&g+P~k3JZ&Ic6dKjr04g
zAj)#GoR*C%PuqJ){dmWlNcg+)z>C|};WGAZY}+Xg%)BgoeTu*TnO5_&7moMjC^+<`
zO4Qc5M>lbE-1lAuo-!%8IW?a$C%7Tfi;Nn)P0Kweu6za5k3>n7Ss}G)Paw#54}i=i
z=G$nl77MgoeLWMOowq1*(8)f}udCRV8AtzB{;h3!hfDj!*y_Vsc~@i0lfEJkagcwB
z8q-*|WT$KVKnle{`~A2xjP={@N}L`8mJ^9@N+?~M^D>N;hnoS#D6IP*_{%&C;dKLV
z9GtBYn)(gbPs(wgoe`tnARaZF?mqCx2_rU#$Sw-H-n-jzQAPh_b-IF?Yah@4o4@z|
zr*65WlE__HxgrTXZ*MvUoSlDr@9aEx#B~Q~BBePQ6+H<0=X5-+zT7l<&i@|eAi(<Y
z7c4S(*A`~>;nr?PIAWghgWXtG7h!M%twyiN<n{ZT9hlN$wYt9#aZ=*$Z00RtFfZU(
z5RC?Y_7MNMU(*gp0V0*}HbcqbV0|Kgs?YA6<VdmhD)Ct{%JEn}NB>5EB7ZSJyC2Dx
zy8JELl!p|?T41oiG4)V@9HfLrJ&Uf8Qf0(cQ3>pA)z)RGDIzpLUnDGX@ilhV<LW!N
zf9ZQ|S+DxXs+uliY9L+~=unN3!L0A)EymX<BdXgGbMEXqKn}L*eNFb}cfr*70Bv!B
z;;I7--tFJ)YW)F}$D_5{bQ9vd_g|yrR5&0vlWe_LMPcR_ODEo%QJ%gb`%Jv;zqP|8
zz~2b4zOla2_J#d@0O}8dgZ%dV&N}B&9_?y8x0y;+(4F2asH^EzOj6vuXE6<KodMgk
z`!dxO$4BX3b~cq(U412XeE0kz;aa&<S+g=BIkNUsaes4pfSSSa{M)gMuNs<*LeOEg
z<A%PMs;j6Dum3tQXncJhU`<zx-M&{H2@BZ6w~ATmxsS(jV)eHR8GM#mmg~PxIoIc7
z^?bE^%Z2*Y5`1Y`O*9k7pk|C=p4w2SECQ^0)m>kDRx`wf@o#f7w2K*ZpVyg7fc}E^
z$mz(|3MwQ`F<Jzg-Q26bL%n&0au)daF`KT0>Y3%Z+*J1V^Fi#og-vepsm<gtr?dCy
zt|e)k{v-k_lZLiuc6hx@i7x`2ES}{&FRxo{y-1$-Esh!q{)pepx+!Br*XMs^XLkbM
zlA@e5k6~WB0e$c8AN1y*hgNNmdCj15t)s`TaB{W-b2<%iWzxlT%oOlVO{Cz!-8VF-
zze+5DEA9Ea!m5vEcj0`m8EiVVp}NId-A6=tMd+Qu9=%(iKC`WAp1`&P;yTP?=iQmr
zS&i^MKZO^MUdfxA0M6r5Tyjk=1}JJQp12gZR~itTI8LQ_+Xu~=6}F`+fr>ZD1i>qc
zNcUoV4YmD-`bS2WG>;KYRPWCxUDZI@>7R#EGLCYlx@vNd@o5VnU&AYO1#FyczOO_t
zYg%*oE;pmp=!*ysxK%rHzY{>O8A5f(98q@Na(-u5Sc-Oz*ivST!n7}}R=jI-GJbEj
zdO^on3|nzF!p0kc$&c99G`oyuTjcY0<~*^!9!tCt=yu#B--*0)@R4e3Z}2aGcQzFR
z2+lSO{Aq|*?l(!RUOP8A&k_T1EWG92THa0fOpOcs24-<`p1BR*S0(GoY*yO7f0Wn9
zB%@K^hH{fiR^KD{COD6|$!0KN*g7-JYf}g70gT1aiHWX&ik9Ti>)Of4>a%dy(M&V=
z;l3@T8}TZzX#Mj=l`d*57PPN1<jKqSZ1xwOP9nm2&sx5w#w)f=Z!VQAbS}Cb^uM*5
z2Rj2J%8y6qQjPdcy%3ms-l=Yt(fGVje`t~e=k7m^fWbH~Qr0ZjF8xy#>tJ4JY)C2U
z+C%MYH1kSWo?|;{$^;#JfgnIh%JT#e!AN<m^Gv?gyuh3ytM59%ZtBWLz4iIGQYkAf
zc6Q(EHx;bl$xxMPo!zG-zz0kekWotFu(soC=mY|Id*U|Y?;ssfm}U1i)6^q?=4(2X
zq+rvqc_U_1mfMYs9KY4R8oU=S0!Obq6K2%kp!zgBVoqu-ZWQi}`oBQwpOHP;gy=dT
z1w0_!oFzx$I=#+zeFV2%CjflQi~<_(TasbC)YTSe^mvxhOp_N)NT?hF@z$3BM?SL(
zxKjr-emMKZhOzM`@jC+o3`++E=*i-#>FVaU3nFrbBQp(mugE5N2Pyg8|EWVCMm)&1
z=MhU%4Jp2~Y0R>Y4xRPbP~2x{DkZCjDvXv9-6}rDtF!VeMDX45oqXr1&=JCABv$VJ
zZlCLQ&M{&Q(l~B5Qb?@PjL8%I{n-^vj64W++||fX3RmX<e7*;W3Z*$vh^=mcZ$a4|
zS*Je}0zS`^G?ah7b1*dF$lA&DZ*=fwWPPqv!W$@RtbBa?cK$2A#kwWMXugvUO9h?u
zX|Gio)Y9V3LXIV^xb9k+@rk3qkrUf1E$8e7LfHd{!F<5Ucw4%24K-DsK<uWU7W!(c
z_(+*t&`pQ7Y=cu)jKa8z)WOo14TR@7_nlD+r;}aE!meTFcAU=AvPbcnKQBq<RFz>a
z_sHx*Sa`q>DGQD!z=2|FZ9M+B0=7nZ^Q{wZEa|p3-(rOz!C5Y6dWwu&pf8!%qSpkB
z>gXFCLpG7ZSeQR!CdHRDxu7W$Yg%#d-=n5Ntfu{hH^4<sODJ=iRGtSY*E2iDwO?f|
z;7~hPY=NWKk0ftiMY4Aq!Hi)L2XEd>{!zsT`%y&I%(+fXpINp~kCqc{1!X?3rX<k&
z{;3G<2Ck-e^ygj|GFKMFaJXTj#w2={Q4>R&lr;(P(ysIH{OuGyx=qKFqr3C}kJW=5
zWjlRaY~&!P$NVz(0{8#?^~ku`7rgV+b<9KFf9JIc_iEc0%P|S>9&A${i!(JV4drMg
z0MA<4--DsmN;w@Uj46yv$0z<ntY&@$uJx#IboU$`{w+I`&P!30lNFJv+1gCTiAaov
zN)uJwCGlBTcvc63u#+X$C4eZm^UBi5`#B*G05{h0c{S_Sg7;+XqIe1u7D$JFk#dK0
z7S0~AkV;7Hu~2%*dbV#-*^WG27fVU#Th$)IhWU%MUy~TNw2%@94G;vNXV*Tu&u*@h
zI%Z+<?U}?pCgWgMO^yowg^H}zRv&m`hR+yvC%Qa&G_{#HcdoQ*a-<oBujq%r`(W#e
zmYagFs;ysZal~eqiq)ME__T$HlF6E#X(^pLCp*CL2i#g0!<wnBRTCo;0SZO1NC;;3
z?^AU`Y=uPPL0^)#Q36N+X5jps$?_%h;<1SnE3J7NXY05g;mF_LxzE2#__UE1@CY9z
z52*&W+V8vckN&%wFnV>}RP^_?ZxLL9=9$Bu7ZqQF)bTj;7M|0g9HMN~&O&mm{a9O<
zJNNO|Kfp!i_ipI4$A-8%AyXVAhf^7jfjyQT-7P6?ZbTeZ;l1ToyarOhNy1)9jV!n+
zCm(6smw*ouXxfxj9<L?WQvu`)4n|E*KZW8#B%>T54->5^6rpj$*}h<8>exmF<hRt^
zM*1po2IS1tx6OmNmhpb__Rm9$x&Hs@DKq+v;dJ>S2Eaut|8-?umI?O}B3_Y6R6UA$
zv{*QRUm_Zb!&<}IAwQFO(07#sB5~9$QbQGr?f%Gylyr4(+%5VzFj3Hkxc0Q~-!at7
zY<)5G{{MAYkvPcNtB<0nn{9)i8DJj~rCR<xK%)VFRg&P4{lw)kuD_J{2k}ZMo23?1
zrAxdnh7~kNzy3&YkWKb@g(vY?5}`wI5`p4Uyeejmm%}RF1E75p@0CR#NP-ylo~YDJ
zk>?jnoY-r7*K``7S(F|){q+7_Z1M2|u?^4VrL?H_TXuvk?8B}EtG&m2$Jz5<n6}+|
zE)n`ghl^+fPPrD+bd#0IP(M;?B6P*=>f`7tC%-DF&EQxC4kUf2a$NdgFNaww{)Mo*
zm_W{#jT2?>Ez9p2C3i}4iuWZerPtQ#9s1k*YacGvzOqo(wnu|wttZ<u>uQe4-b0mr
znBvC-y>jC7bdYWeP_<8*58K0pVCC&ow}s+j6IZY^XVt${L^`0q=E|i$Bc1WjWS45h
z$Q3`{M;&5n5{A*kDL|+nvR1E)gjGjiAd@$ClUK0Yc2!|WQD<G#p-#5le4-2pE^QsS
znJF%(iDJVxj*S=?`F{h`;Lf-vDtqk>p@p-Mzj<zcUG0hUc|?d%`<*l@RCy^F2}|ni
z)fH-owNZ5G^++4eS_2ipirs@S*hvQ|1S8`4a7nCAk<Ca3r&qU^ePWwl>Zo5=8#7%j
zSdZ0c=(Bqjo6k(yQtOw?D*qEl&q=*66vu_f&oJQ;yAdI_(1fQLy>?5c5VB`dv=0cB
zr3YM4MWb1Z-^GuI<h&mNM{0Xd*>cjSFGerB7N7YPV0GIa@Jz;H1urAGiWk;I%uzx4
z<y+EWQLePusZm58IFk>~_1Y8>y>I&Bf4v&>jyY_>KMaY}N8*wa@UxI3sHK18?4O!s
z##>NL|MYj8S*qn{mjdcl14d6QYv&$sNJmzigsIn6$$!)tCn@>ovXk=7b>2FJ@$WAL
zxX+l*3MziOxys+QaCo!K=^A9uq6+>aYDhlPku-`^gNL*zR}e}~uFs9{>J?0TNAk#E
z(gBUTRuWa7g<c~g^={h9uBa4BA1FVmOVlsuv%_vCI;#dN=r78BHzGnV@o@eDOlOgQ
z`&V1)kG9;f?-}r&21)KEx$n}_s?e+rl<l04^k4qpsab<49Kynd^3gTc2s%7hq>Q%C
zqa(O80CHZ>YT&wB6+1Z8>++ee-iIiU(yqg9n!27((%G%hT1Enrv`2o>DlB1@QRt$k
zZIBbXTdnBP!+CqNzi^R1n`bc4=}^Mn8~|D5FTL>bqa61Qw?kHXX#=YL`XF<aKkG95
z<`Ejn{DYG(YLpJi$$tr%mBLkDBcO88I`U>cC8-5}Q#))Hdo$I~W26oWzu2&kvPVn*
z@69CK_IteOLD-*)Y)d?*o&5fM8Glle`!C-u^<6iHhto>*6C|^B{wv0oH?{4wmlbb+
zLF8?s8}h2GSK=jU!R)&EE85BYF+^0`p*F(mP~M#M->iJA_9q+7&k^eFB{3c&lRT{z
zoaXx{1M&UwhQ+ZV)!un_N60qT4jNs<WV(41@CCf7Sd>W^Z`7~&M3}EfH2kcI+>YHs
zgT4ln1vhA4BNeJ}4oOhS(imXVPThD6EoQfh?5&E2m!x;(PGIIC&Bae5G`s5_rT#ZF
z5<^36LzavYx}tw(v@wn-eEHS3|8b-x6}8n{JqinXP7ENmj&k0S@kRcrGtHcGgr!S&
zij62p>sua;+fEmDkNqtI{~bi9tR3?!K9rmODLcs@&;9-(Vo9PH*7U*QR9W%sytt}^
zKf@`1uh(9f23b$&VsDc2k6qRM`~1zEaP|#HQd;pLuE6`zCU>c)4ZO3u%hh~Ytq~bG
zhU($*0Dr`$Y>uaJ8lATHRrZ?Shq=`6OLG!P-qPdthf>$oyfHC`fTe$rDSICWNf?g-
ztcE4$)~Ez_*M9VL0uGYbUEs9L(e276m2}c}VqgnjEQC1aI)rJ>H^T)N>p^bc*0p`&
z|8)~AS$W$)RC7`Qlnkhi2FOuK<$=|pNc(5u>-hW)B5MwaiMUSW8ibSaRXJ~xx41K`
znl#u>9H&czzI`Q$>^#yU@Ai#T=q2Jb>_4TJ8-K@*j;sBoMaP2|?4*lSw-bBoO6NJY
zMDfLzQNCkui;{lHJ(+?yzyG)M_SN!{>22%tjhCZ8tXYES(|DFK+Wq%NGg|WGUB?G*
zZ+#4lV;0fz0BwQlKa5rzZSmb7S&>Y33%o7t{>n6dzq-f~kKG5-O(7m3cpb@e2QL~8
z_GC$o_rZHJLnm%~e_3R6j-owG9I~f;KI=>we1%+&s##hQx}Z{4WZ+$S2Zt;mkhs+5
zCz)`p->A{*G-j*j`3mA1{%TmVKHSS+R^e3lIc~BodhI95hoQ$f&`wfP>)P|FaFhDI
zTMi<EseQJEDAKxFdY?;;C_>TRc~>G_-OWPTHR*rK&pleTvX`xBKqaw4VxP#qJ21tS
z+{JxDhb3+K<7(BSm+ab}kZYN=bt-((^A2h&4k;tQdAi~abAxvz1Y8$XdXkSlAiM^h
zrQ-koVK$c~I2q{*QHebx<q%a@Dv5Y%1iP)KOW4s=t7yAPDaV^LJJsnE_Eup%ZjV$X
zIG5;bhHTL6#_pG!tCbW*JJN6d<TXEf2;e)zh;r(HaX4^9{~hAm(el#9GK#;tk=C;3
z75}5Z1=umWej6p&;AUD<sW7@)bPW*Kn5vA`FK?+GXKUX{cRKol5D8Det|F>j%o;ON
zcgr#Jc=k2JFxVi+OvW_N7YLF5BJ3?LU~g7Rl#WFs!w(1SkzWJ<hR$vY$|>9dO54XK
z5#|wlW_uO$%$?&trXnWJ3t?<h^RI4b&^zg%2|7Q^5u6bl?jy6PXf-?dJsEHl=2=%6
z)=qcnS|QiY?Q)uD9!(O$+zj1V|6y53G1mu*Qg%UmJxbu{(wGSvE#@*(-(7l*+Ng^R
zY6^1Mnxfs_yb%NKOe~?;l(c)_F7psNuRe>A@?-Gt(zLxb#0k!-okT=8gmnS5t>2&I
zJ}Q3}&n3K5Oo9yaZX?eBR_8?_fG4-R+QLOZeYf*%OQsMIBx~M86y#_0!s!|>rxcf{
z<bbOnwjBlzFHW=FUsT(Pm$Oc2X+lgRN*92SP>~_S#>@pD<#UDmYZ!8E{6C98ZH#8u
z7R;HAv`1=g3E*Nr9ks_fu=@{1W4>!ApM~Y}0rd;s(g?r~n_vD^uVMLGMk%Ev&!~%?
zR`OfPoRBr1kjMOI0S9jA>l97d$Og1vkP{yKGfIS0M=EU6Mp91CM(a(py(M`g{%Q^h
zG913e6UZWZ%iludNu1{#K1+wy&608P;#vG=MOi_Os(|^&Ug?QCIYid?p>5^lqqcRz
zeemYksJWElt%f4&(XFVS2xf{sVEp+7@qSdd_%`{pA?~mMBl?tb0!uIdoUM-uOa6t;
zUFmy$)7~5q(!pefPPxSttnNFTqCj6j4ljZo^3+MGfZ!R=4qPDPc}dUDr}Kg@>l*)1
z(xY_};$+P!bT)CioqbHMt^}NQ;)cmU&jd?dW8JCF(1N?fH4Pwqulc*!zs3WqMQl0w
zoX@iFmX8@j25F8|@ocrO@DseRZoL6K&b0N#x%b9@5t{LKyF7&$M9LIu%9BEqLgfF7
zo*pl*Bd_Q~MV8ZlNS1HEI}>+$RGiP_Y7PY=N--*kClQ|gM#nt}Vkc;rYcL9D7vF6^
zS8JP<9Hz{tD}cLoA6cG{7#g>8eu~mo6aDTKfWsahOy~RlXL?%gf3|D&Mk|N^IWaIe
zjn<uN9Jl=e*R9&48iNNH{f+kjX*0q$8@PY|{m}WP@zG9YH>&AaZ1~=+Gp|s-8+Z<2
zT;+Ck38MZp<o%?`sucB6ebkQzjrK`aAm!%4VGlu~SeA=haVv%A?20&7ZyFjb`NjPq
zt*dPA7#C`~|5opW7|8wfC!<YDrLechPHake)A*}_V?si;lz%(4TME(}#_r)nLHeBX
zi5Ek%yb=GUCJ<PJ8Huc|qYGzn<ahTu^d2oJH#4JmHj9?0Q%z#t3vKhtdTZiSkFE=+
zt$z9&zsmV8A)IM$UHJ>9h1B;TG-^o=A{?HvFgj@fy`J~<PdrT4W{y0sY2Wo`EDJIe
zsnnJsm^K$B_0d87v757;<X>>jqCPQaK9b+^7I?mcy@NlVBj+JS72&5e{JPa+^rNQN
za_X^u+N~d8u6a2HWW!dsCn`JU*I!DAxyZPfLCXSUr)Qq$Z?>zuNWYntl%nfCe&jO#
zx$doxqrIvhLwg_5PUL7^gi;eR;yn+$Oey|+<}n6-J904Bn!TVj5l(gVpU@$hu(uVk
zymsdwOSV`RkvDT2b?Mm@RjZ_kCN3P)<doOhBH!#<ILR+16Bc>vE2hNKgd_LfZkt(r
z1CtO(R)+-QH*E<pkHR2~wv-Y1WNcGqT(SFmhNW2(N2I`~k7CtDXhX8KZX8Qm*1c4H
z2FLW#G*qcgziIxB6AeSt;_s%pL&qc2=J?uBE9`UCCB;r%X`Jz4N+!l!L|WWZrFWUj
z2SFMvcBrdkb8Bshrc|E|y$JopPi&#L|4YB_xap*Y)cL2<`Q$+a>#%nFNs@0!p{WpL
zlrU{kU+&DK2++zB2Y*0a#+3FSNI2&TyPsc(D~Mj37uDQK<Gt~@h&2=MA^bTXCx7zW
zIZO??A`#PBE-d~z;!F)@i<qE)0-P<>)xbH=NE-OI#GY>AfMUx?J|SphPtCsg)8N;+
zE?zWqoGuogEuh62)u~J5f8IYz{vgK?^{O4<A&;s79FfI>jg)+SYs2URuJrZnL9_pS
zR3=K7b@=sCd!mz>hEVxDz6BmktEi%brjbswD#}mBqFwt6+>2NWB*cg6^Ot|S04prP
zbk>lhAKC;z;QSo2ETZ<W#?}%qy-QR`h*)?HRPcbSXkJ>1bW$5ty8{hM2QfT>D$sz#
zyMH#Mw9!ryp6;hKB~+z=N%}>_Y@pq@slz4yDKeBlUpqCvhnXgZr%dP9Q)l7SsHC{3
zUE=3kuozQ}h~g=i`auatr);Gje-Hm;{9Ib$sk_=v|Ai{euDL&<!L+*Jj5oo1Y^Hg>
z*f@Ga_TB?iT~J<M=FyGEfSVu&)ln2LxD}eu8b8~Dv3^Yv+OtlzQ7T}A$FTD<Q!F{V
zTR`19Uv_%7J(SU((|spRZBD5pWjR6p<AuhGo@9VU`%X^z2jHuAdn<^PEcZ?^GOG5=
zjo4PNWUqcw$4x`^SB?Luc_qDv_G=+5I~IxEPu*gXwIEAl#!ZxvLvrA6X?>q&zFpwr
zdjr9QRecf&KS`bcBpfv9d+4hNUW}jWg+xCLVHc<d&iWQrck^5WA!29D@y;MNm#;T5
zB9RAGal`UHQ6c7+3iNz)Um$xrvlI!(oaLVGh*ltp)rmEFV7H692%qWf%^lcaCjQHP
z&d}=iV<zJHE;*O2SRw%m_%1Ye@_PLCjJaSra)%4S*`XLS&OoI23$S_nKz9tEe!;I*
zh5dhqFw~cD?{9EDF9VR%l<C|m-7f#GtT=51PRBaodgeX&4;13f2!BRoG}`<nce`lu
z96SBte((=u=Vg_Y!=pdo8VeON;$kRnGI3pzo;~M!li%nk6<E%OmU!mx`1^jCr~J_V
zBsvkGLv%)z`<xMoz{#7qcQ(6~U$=Z|4N2Fg{=sW6N2M;!&KgGiH8BX@R^io)|EF+$
z);0bktFTS+JvvmpUax56Cvgm{T{XDw-!yM@IN!FzxpAk>i6h3&lqvj=q@_f^2)Q+5
zeY)!pZf-Yhuep5RKeGewaOa%&-7-h6cbGiEYvfzJBgR8M)Gsl7Y3}XjDu358q!t>(
zJ38$n3H^vIvi!&9K<ZYQ4=IFKjpD7!k*?U6XSA8gduMNMp8=i(i#63xEA11)j4GmZ
zo~nyrRFC(_0F;VRjq3r<?e?y9Kbg-^U6E&vJ+|%mlplnPQ7AmE%dg6qNX3z&l@fN?
zUU3i(Hv*v)gtSiLh$gtDyGIqp(MIA3Y;J%o-g<CoO>2T^f6N`iiQ`A{i6~D;OA6(h
z|A9EI@yT^!(}p=F*3gM1T#$!@z$gDdl6igTx2G!weKW(ZiTrdwh}<?CcPwg#8b_Z$
z@aLXFw`>d#svvjYKZjyphzpeNfpayPMRqy%1B@U|fq>+gr7fHnOYJr-qnO2-Hyi~Q
z&pfpUm?wP3HH}z`FE6PGWYb*U0L}(E5`ojn7_rH<BdjDNfOueH4KsMM<`+I4$7w0`
z88t`5infye#&CC2Xx$qw9eNK)Ab;luW2qR=WeM;A|B#27M;NMmT{mR=AtEU;Isd<a
zlPn{%g5^ki{S+-^5h%z%{><YiKJ;4ZlD6N7xYl#r>7!dVt4-|GMCasA`?LGK&~IPP
zQ)1{e*m@72_hl349nNl?_Xxh@GT+Q$4ZdKm;8}}H_!Z_D$|`wSQcwg}9x?Bt(F5=b
z)z(yoXZMf9N!{R-0adscj<Ub|1JwM+N>1p9u7<sWdDGnq#z^hyL*I+=nLW}ROCY?N
z%=)VQdv@UNP<jHGo)6dJe?R=(`$C(m?K$u&Km`rV+)g`qc%pbxs$PP?$rxhlv*^S=
zMF0LU4<Ik%CURHIpNU{nWnU344mVl#&6WM~V4ku-?XHXE`p~+1QHExDVQ_opJm=h-
z<m@4gsgB!#rZ5`71SgeXAM}RYGS!;n4uD$s(_Z=gm4Byi4Kg5#u4n_Xhm&5#+~Dt!
z!vJ#q8Q4;X=-S(m6&>vCZEM<d>5?Il(g#sKcx!T2OUVA4FgcH490*&CK<|lS0j5{x
zP}e#!QQV&r^()g{S*TbBgjiP3={v{mp*2SYM$q8JO%2``!e%>3tFOV`6rltu*R54z
z?fzjLabe5Bm-;{GT&QDgLW8m!09SWx;s~{Q#&g}*`=#(Xp}Ua{$v~K|@nM1Y#;JSD
zQat<vgg6X{0<JSmxTNA|niBF~f@aFe8=d3NJZRsG@PWtatzwQ}o!pBk)fRSq7ri_#
zbQ@rRX|6{HEj6{!N_|Qx36iBPuIc_r)P2K9m*PJZmS}S4N82X_=Yl%TBDU#0(w;&3
zVgfvAF*9Fw6Y*Y*ZZlT#PB@3`if=vV_D6rzu94=dR>4j;s}~v8p$su+3a0IZ3MHa&
zl}VrP>?ZbbP*`Kl4Rbx`1PzD=E;_yx`yuQ;O3LE!HeK}Ay?|nwLwl}q$KrY82>Gmh
zX>Klb0k)!pgyn37R3Vjg)QkONV)M;*+YG3z{*6SH>}m5ap%3!NLv#kOsFl#StvOje
z9=bCTz8zK+9D=6y9nLn%mEy4ldx@iw1X-eBwpa$w`!j6J;tp2*w~C)gZBQ5KIC3vO
z(x}yS`e)pCq6wqmg;f~3&?0p|q`8J<?6QT@yWapuC^*0Ut{^@Gf^^##hGM%z!-Ib1
zV@V@mup7^>UL1vAY^-Xn<^pr?Tz`6)i*8}pWJQkVTb~L@^nz@|NHBlxxI2fU;77>!
za>n}wD(}&Di*&z;X~|kl1p(2p_}lyZp#LbB$x+b6xfjW=*hG>7v<%(2H*$G{WWVZ*
z`(~>=8ZbxRf}F>h0r-WVZI9)6QTq;@<dmZ{(I8|fOLqG~pR`f=RhqG0j75Ui<>w1-
zh33jGX=H1B{WOPue#|ftPw(JtaB=XGWy1|IAerPtxi~YkUp~SaU;^83|MHI)ti~nT
zf9Hm<5~?SNyMNxHP(<<X7eVS%_a<#y7+?Z5(n#-Q)1l@bZVctnp|FSX1K4J&--gm`
z<yf*1gb#%+E-_iIrV5ImfG>-F7kO|mlB{y~Qu`al?oxkQZK7Ei_n*NhFYUS<Zwv7s
zb31MjfWDVJ1Mc!1h@<MCK>)NeEa<cV9L>+XAIqv;th!nxd&^bw5{DxBV$PJU@x3ut
zOBA8u?=W(M>T&`w)nz&=9<4QEF{sdAySAa-y;|iL$G&2e)p35@h@@WC9nUhj%Eqp@
z53QQV;nFNK+2u>OA7#uT4>5&%ksa|2#&+XbQ~RPlwgV=##w@J%oVIiRyy`V-=nL<w
zVE*inK$;G>bxPhpE^$EQCwj@e8;z3Gid!t0LJIQMKkr6_lLX?BE>$ZXlXT7NM_ZTV
z0x!qUO>TjXs-6DR7A_G}nHQQ{SdOEM<QbQs_uDN4i^TNr;npe6DTBGy7Y9<%@13R!
znS)sruY@{J_MOHu_8pm1Y{yu^k}%$7To2wF8nNUQ+xpP^SF&nb?_?UZEtoGz{`t&*
zH<;ZTsN(_MQmh!|e@e5CJRtoJogd6Nn%uIs{J`~rN45uozai!5(;%k=<#~)cU-R*(
z37IVPA2H%bKW29%oZkqa6{6#=G~B3-w$0GPaFcwE&SG}6bv5q$;?$Xtm%mFEpPjQM
z6gOJ79skDZR*@(uD@UN2rv%|D_9a6`4XxwJH~Uet7`v?^6f*x7fW;sZ0gUS3eD@e>
zZb6J$ZsLFIpGRyl=U;n_0GNAe1StxDaB^DcoGhgx8r-D6SL+1dt;&}J)m?qiowY}2
zk51{PzR-k4+i@eDgq6KWfi~ZEcPJ#_>oy9IWcxaIg#m9^C2Gdno`NW7wDFl{JCrsZ
zh%{!RW8CtcYYq87b3s{NhN5`h(KyUPBzb8A!mwWHk%HhFuGQO}jrS|3JxUb$k)sIz
z@df5qW^Ywqf6_PUwNI7LAo=}HY}6-F`!6@j(uvCZX3RusY@rZUmJKi?i;2byZH4aF
z`lVYvMQI9LIllyKaYYA)V9XQgdjd>uRm={cn#d}QBs?0mz5Pvtw!H@WQD!7m-#B`#
zJ(?5*#gVn#p(X&8bgF`Zo8?Z3g3o=X@>P8kmT4f<#Ya6V++OP5)N#R`kYbZP(N!Ct
z;;>Csv1Wkzb%;#VgPed1oln%M>v3#AP@+)`+uX(J^Y{A@-o~O2@M&<5z)l~%7eE9m
zr6UzVr<U^FmooDfUyWUd(v2g;QC-`y%Xa|3s$MZ=K+Fid3|hI%VW6jgGCIfyjMLUf
z+D51&6XL{aF!au^J5kT06X<_&&hFq6*I|7(`TQ2dSl0<zsf!Ey6e&2a*3|SHZo3rg
zM8_8mr^Z2)(ZzxsKPNyqW6<n1i-rNI?pVDRNu%mK0?>Z1o<^*cNihmJ3E)=Tp^`WC
zutY7{(*4O69whbhwqY#}#=}+t#3D@%ReCcy*qY%%_-CBh@mv9<U6&?{iDu)q#VWOh
zXoA#bhEpW})$rI>)#xtEkyH@6Dwyu(R)ERCX$yPqlD&Bej#Mk}Hw1Y%iSC66K?WZQ
zH(aZ16HJenW+pzor>mI<>Zc+!6_E-P<fyc-;ce(7@nMSy_rZ-AikH}s7J!yXW!03P
zqwv~^fSfM5o$f9DQDp|Yh2EMcr!(p4ZFrU&UixDG5Lyj@z#;7o;Q9|GOPcZUpe+6d
z^{v5-LK!Pk0lo+V)}H<AHa&tI0Va1<knSiE<!g?C0=9sf?W&H+2H||2b&_O2^aBUV
zVEusHObGp&>a+(9Qs2Js^lXa4=`eK1F0L?Y?>YbY;zhPZxavwNHy)L~{4or7AV|Ky
zg(wv8ik_B6!!+Kjuw^spdP4q7#Q7UFo)xwN_2}?(6m;VQ2h+sUmAzGb)AWRUfDk@Z
z@jFwm>1^N6jzgBwKB#3O!7!Qq<vbDBX>Iqd2|ZMPTREhOK5af}83|3nDX#l<Fq%`|
zU!G!G=(;G7p}-uDrT=cpkgUx$js?jFFRW(KIc(ExweSsYu0$9v$&r@OkGT?7*@(;P
zOBlr~4svin>d8=@2Q_m3m;J|pg+8#*FPgQ9Aa}PofNBLFh<ycsD~G{4&NCDhCY*Yi
zFMgG119-RN4>cDgSVCjrR>HyepJ0^O;FQ3nP5FwyUgj@nO8pTj>^63?`XfUyuNyV(
z@_$Amk!F@-2ZZ|CuOq}p_&wG(Iv?i!sq#7&Tb)*;%e_2q1k!39RW0gb#5}YjQogi@
zF8bIqTidP<09{^y;ab}e|H`q&Zfk<FI1M*0xk~^}g(|V=E3z_n(&0&R{bypiS7vg*
zNXQs8C09q(z!T{&{R`CdMIgg{-#vYP{B%vm+Y7deBpms-p)$0l3!pe`yk1xt7$=#x
zUtuD26VDRQb>(bnkb+yn=}Wh<){@@byu#0+9G`9@_}7`i{}_^t)2O^TrbFH+aydBb
z%)H4p*s8Yl&wd&gLU$0ReT#5y%bWiJ+_l$gw`{2yHF==Xkbuc`J}*a1Iln>Ux?)D5
z>E2Wbp>8kEpZ(cb5T={iIl8|Qxs#k>J53`}1@zSlLY8%5MqM&oPL?vfOHQkm{zGpZ
zg(GPXur3i0M9&E?=YT2?_8RqvK&PZf%(U}0i%*Xhnlf}-jwGi)_C{e!F~#Zr5@qE0
z0@G7k+p?PdQ!3sYhEFc<ej3qG8TmbaHQxboHS-$z6zL#wg{=DgYDK|d*X^u)dm4;G
zNREk4hz2|pj(2Av+eRDjXzrA`XOdyQf0k6~h|H+qHD!sPsn%d?39nf^0J{~&ZE#_h
z7V$l#=+9INAS{E0F)*=K%lPL<x}QRE1}SJ@!M+TUUByY9HHyMUX9f%5+5c`dKP3<~
zU`}Fx_wFTxeH+zQYFa<#S8?Meym6{Z!j2|xEwQ089&I~t<m$e474h+X-4Gi<QCSc<
z{V0LD31%BPo1N(BXiY#Wzo9ykR6?}^kHjRZK%(~w;#u0yd~Nj3=ZWHC#krlVR1DG>
z1PagE6#lkfYUm7lTs{dtK@7l7GPlOj`HtSplqeNzf-gY#;@#(%X)}8V+C%*xqh9f!
zp>A|3%l~A<wkOvo>7G5`?N$t&=aE6LMzW}|ZUZMdZ{|rUb<!Pd*DC3#az4PM9{o*|
zY>=85YWGC(c?bJ3tG$D)S*r3%Su8!4gz&Ydm=p}5otVHDNw8razxJ+4$IHD)4t_g~
zMCCQ|&T~8B%XHTI`tS!Ryd>G~Q~c-{muE^sL58K*1_CYHtNhJsads;f^-DiD)TTM3
zF}B~tE<3#<Z1_V%jk#088`|KnQxhCVp;q%cT3*WKyN?c_#Zv$sORYlPYN6kAxU)j7
z!t+nZ+|2=8L-tm*bxeL|N>4&nAZOc1JC}hfFOI$zU4tC{-`q6IzPN0((yI+h<BMIw
z4TDmTnVXQkM;DHVl-rLP`bMM{PsKhCKY6^&S|@?8Qr~`Lz@lC^-S!H5-!nu)pcMq?
z6Z(@aS?JDmwrFklFIc4{9O_x|NzwM<KiR{|IPZo}UX!lJ+hqE-9I|7qFQyLOnQkTA
z1V>wo?%%fz;A5}eohgmPpKN-2!)uA=&mdO=pRZv2MU!h=`1%dYuEv$mrE^tC*9zjH
zOelG}?$a-WBX6R+ck#_HP=6@>g+Hv33THbGd=k9ibkGc4f7`zqmiR@#?qQRS52E7X
z<uxOIYrJo~M2Zc^8Ref{YjU|d_hO--_l}}(l2s&&<x$(Y#q=uR$-3r{)!d{x_eU^?
z%g-&TCLadb0^fu1encjFQ*FlpVcukL&)*)kQ$2JhXS+!VX=A<D;kS>&y;t#=+>B~T
ztU?$eAg5glUubLJvbJPzikCP@a9cPKH{sRVtCGo2RLG^PRF9ToKlMy{V(T<e3$|(D
z4r^>90;zF=6#(nZ1Sdf}-09XtVr%ob<yTp)_!=VKciCA4v9($vdEIXi{xor4&-nCa
z7}{CvTdvPcCb_f<*XvtNfQZ4&pZgMkzT!qL46GUQ7Ek`Pxg?%|q+sl=54klY-)mu_
z<SivCw}tdhW>K`?*S3L7*)7D5P8$)oOtLhXwIIAb-s!M;$m;<Or4XDjGu{gGliv$w
z+X!cw^X+hoG$53&N;GtIA?6#V4E`E}y?&&0ST5sna<8~6Ug$Tw&*9R4xjb9m&ntGk
z>BX=_1Bcr?a9<%ks-4G;)WI5X6?tby1<#W;jPFG@MXP=46Ua4D_p|H2_yRJ3<_ZI#
zv{P~66U&Z}`HHixgFiu}5r$6#t}&lqOfGOm8%^UUfcp4zpiv((k)TI1+nk@B>{SXD
zpfw~5#RKdxI>m2;xAZ-L82pE*)r7;5tzCR+n#!6_*Bz1xw8BVHOQ0#wCrOJ~`8}d7
zUg3)J@UR)lx*7Ft+&_3;*rX42%ocIW6jYFS@>=JX&NuN;pc;6yvoTO;Ek6?O!N{`p
zb@H(yJ(9&|Z0x<dfn;txnNsAT*?`2Zw3X0#E!3UG)S#dlp=N)K+7})ZX7Osk;JU(w
z@iyRB)^b2I0jaoBY!JHLPkS8;{BaE=PplC9ErF8(yJXiK!PmYVjscN>`zZ|lHQebE
z-3E~Sqik(p>j{lBQ30<4->>kU=5yg`miw_#XbgSBHy7W@RFk~7TKSmZmy3lqDy;M-
zvx(Pm!^Df))>!wkkx^@WrtkQQ`utUx=lq+}%KPAMi4aG0yN~*cL+>#EYDzvX3>z}t
z82tqs{IWk$Tw_M<>ZF*%Z1@wyK5J~VS^e5s9uYsb%h{(@GjN<Tqi-P<_Rn>vhc&8o
zroMLUvnyfTdbcPokn(mYY(}kfCyMwWN!#B4mu)vvb^a%ZBkqK{UPGP{Tz%5g*CRb{
zn*HQjTc}C0JS;1?vH^qbVWdp@{9Ju&H7_$i=B?aoE8F#UopQho?`h2hwxRcTxXzf>
z?{i82_K3>}@B#Tmcje|XhV1rV)suqJdlvvL3=qMS2&Hx3m`i0Bo16%#Q0HPwxV<y8
z8+gR5UG{!XzwJ_Gll_h>f&1tqETQ%8#y^(kf#NDE<zD>#T6^B!uOuIXtI;1sqI{05
zI4{JS@YIcwVtNSLNT3(V1zE*FlCrsrp!0L}Cw*?dqMB7;paQkoZRbJRUm8kJI!|L#
zNk=^-_y94rI>Pz0h95*bm!GJ7L702j4?}d;2&llbcmNjgknr&D*fkJ4v<x;T8(Cz9
z%PCj`&qoz5!_Up;%cB;T?(CK>9<p|Md?Izy^0jZ9HC4+1CHdAh>6+R|KA8cg*k2B<
zD-S8rNiF32eRX%V<Gb`=Nocc0m;>Qql{^xbA#*=plQ;kXUjLB<1B^*4)z@NfqZdU(
zf(uuax*?Uf1`80>9#DrV4;K=Wn9jd00Jv{haO@luF!W5-r-Ofq7ZE%lbZbuTutEPD
zd+2I2DQJp{Nd<=%r{;`sqlA`&I8;G&!plo618G)nEuQ39Kj$!@5(h-Ra@L2*l+_aV
zc}A0e`;)Z@j&kmw4t6XcxOukl>^^v%JK0G1SDK*+HC$anJ<h-ucBLT!P<23kA<s2G
zei%aAnR!ImI7_$>6sxz!>0W{1T;fjV0TK*cCnwo~_8ePG_xjW{`XS!MIJr`r?Z<eS
z5bmPnd%}U#$!C^qd6vJ%tZGa92|NQcaY)K*IS2y6Ot0)G#>L`KVO6n-7Py(}M)#wv
zxr!mkALOza{ten$)#^1VIPEDT7Pfq?yc4aI@uBhu3-<fK*~)rBYqTRGRu^SGLc04-
ziLgLQy7o+c3QZC$X;dVZ5fjnAjwsS2G1RvNor#L??(EKoJF@oXG}<m#Cx9*qH=PZD
zUcO4(42Ykzq~Usq?kYVPeEw=Il0b;|x-9%%EcgJhQG$o|_?^qWv~e=;TmMxVQ;qd&
z4-2*kJ;6^2*o!QAU#DBooBu=K!n(hM_*t~~^F#(zT!ivDpfi;9B|N}boS~%QS3;-W
z`_>;1b6gFdk@I-O39srYw$6<kSlArs;bLBf$??3v9u^>;6%9Lb)%0VMd(s1^JItjQ
z^F*9}AOpL|psz;CkW%=-Eb9!!{vD7f@kBdB3=~y#3psVOg3bTQ>P3dBz;^0!CpqW5
z538YjbnO8w2{q>+trBnzBG*63TbOr7Bd75O;nDS1oYzKXArkhy=8g0`bi{KGr@)h^
zm3;1fWJ173F(O{R7{!1dzf(ko*<i*y2;nm_j6{+oRQyD4!k@@cQ~CQ%i&o2-Z;_c#
zm)m#D9WSdQNm$RI-bhT0#@sto46M(&c%WGK+&_vs=il0saw0$bMG9J07JPcRX>w!K
zAOH4Mux`YKUSqb{H4;HCiDQg##n~I%j<*{f(Ly1G{hM!d3nu${>zT@b@lPu?a5Emc
zta87`=lf=zQ_UJOtS)8qWL??grSzAU8$!R}Le)ub`4Xvg?!T)1Dwlmn<j*L6lJAqk
z0Ov{Q<d^&;0qCz&<)~T@%D{U`nulD9pfWJo(WSq4hWkpw&zZ&EV`?m6D3<c1abz%|
z{x^op#!IOG7Pg{SmY#tp58g%mN=Z};3OI|mJmIcN^V>8@N)kx-UJNE^IDHA=d3~&N
z+U$4ICpuq-BtP^WPgV(ClX^WeJKYlz8VN;0A;<0fsN8y@j>MmL_63c2jH}B!=`hU!
zyo=0D$fS*mB8XUJf41Q%{zO~i`6zffW!UgsTkoaucpulDHXsPW_074nGAj>7!;W3&
z4P2sIsFWC_n>Pm;X8;2Fi`Q`TsfT^A;C3cqLE(G-mmdt^_w@vI_6QI8m4MOw59^j6
zfT?N$E*Gm{6Z$ob(%qIFFiMIXIS4NU9rPc3xKp7uQk0&1>kAPG@Dupy59_cUds@6t
zMCUX2+r4*$i`#_(>Y_ZTNj1j0RFseHfBnAP8NQ_-aYwe}Oes5zIMDj)dL|l3%@eHl
zlP#0aVZ+bQ*@%_7opLQ3%OV=2uHM&jb`qUjuh=eSyXepxY}n~;CKvp@#^#zznCT+%
zyZPO}Q2in$(YMQ*;<eVAwaO{S5YjohOL6Ne&jb)h{+|~|lIw}j^cCh`gQa8nfUlc-
zvp+5S=;MVj!w#d~aeUv1`u2W=1Bb-KdqVu6R|vHA_S8MnT(7yN7;B9n2BX)dl#{Z6
zFT|%%XCuHV;1l*|D7r(KQsH5=DH9CblaHFn^IR3=nMYx*=^ST_6xFK**XPo(%=s+t
zxU=0bOBZx0?H@#s@|9dPFg}qZ*|$;qwc;y3X1`IEUQE})TYY-*=62jMU-DRdl4-N;
z2CdOR#77K0?+vQ|Bt;I#x)F>>Hz3m*sq#c$1Wmj2mnYTcNbTaz`Mfw?H^|MKm`6){
zM`>7=ek8(dQy7AX8w;6|#;2lIIk00zk<(hGe<EFtb(Lpu_JhJ4P|X+>sGnG9l@y@r
zWpAod{f-%N&%QiedhK+kxGxvu&57sx#pzM_&XORKDf0K=Wx%^$eRvl+?R9E$V&gd!
z<Z4bC$^)8Yo=X!%ZcmZjE+=!DJsfo`R|<0POxbRam4uN=raeCKRuA%S{|l{q<_c6q
z4kT=`ju=M1XwgHfbcK}Q_L%1ff8{YR8caDtOISuGpO%GRw*d(!@l&I*+|><OzBp`~
zE<<r^aTLU~eDi;XjRrcjlc7UK-upfbE#(1lc{%I2E2Ebef<AY8Qo~^{D{fNpa(;V!
zl6hS;q?B0Hs$@qDUc$pm`jftonqru8@tXe?d+?eL=S5yb0odrw*5;tdNw*g-EV%)v
z;7{k^GPAA55o?o*i!^}>G>tpD$PMt>ag}F>V|uQ@DpzXgVEv(ae1d$}=WvN7qi$Os
z&#7BpE7t_ml%W~CK)kB+=!qsa2jz;$wQJX5#Z3Nu+FXh+bsx)sPvxu3ju?YE_v81P
z&_FVEI^My8L_N`*_rhC>%~~Gtr+Z&Yl;o~|fs(4C0qsf!<S1DhKD>uJ5zFtr`E?1k
z#!tL_h?I&10N;>3f^!KrH{RIar5jet;c{-1vP1SIveOKP+e?707#W^A?L9oPOvjQS
zOJjCNt8<PPoxb&-o-3W~-8zYMJ%(c1088w0VF0SQK;-JlN4Clu#135YVAS<QMZ|VR
z)H|9E6K%po)?4lNYt?vvvQ3U{=l{x;n~r3zbuh^#hX+G{?GNuh8%2>>4B90fYF`FQ
zk7F*rs<-=R85y&ES(1#-^+?fAvNEN%LBP1}E#L92_-palEuE_jHw8MVz)2YpCo$tS
zXlDqiW!`347z+<Y0$VAgXYpwZ7A1%$)0GfZtH^@JSYO0N*56<MbMy;-N}H;F&ZrwO
z&+*vV>-bH4!?TJ8;)8<;?}YR=;rBv(GfBvQ0!+%6Bp`Oh?BJd>=gFmnDY0bsfeYy}
zU<km!pu!jN@xIH(85|wIYU*%MjPDKKL^J8f{?N{%JsBEGw21oQM@g)-w|*D|vfBTd
zHA`^#Fz(A*08>TA5?Uu~BQV{4LA<Cc^%O@gQMbr2Kzg!JY)~Y<;vFcwsvcb?f!(11
z8EGy30m&b8gYC%qVFI+O*!$s<E=934c-s3d&k4)@YDHLNHPCeR!K0L!t$n4SU>EmD
z9P3^)M2l3D74%5LD9n~84sWLY)N}Na9D4cDjyv$Qqo}bhvBTNF`>srsYNMz+F2h$R
zV4Nbg<((qQSQA(RLsHv6`htM~?8BB#?=R+4S66f;pc%Z2jtB4s9?uP}k%H+alz8^a
z<FjrPN967*KA)HR>4Dm=$r6($Z_O&mTuP6{tEiF1i>_ZpTJskjJ-HvU1~aQ!^&z~@
zd;tn*8R?~N?XffHM7R~!nvyd_)FWF;UZgwKtg~+^<{z`>bziSRC-JI+G_veKgU#fa
zA7)5k^vTtPjz?Z_)Sbblaz&QZln6*}Fuk{bmwP9q*Q<3CV{62)o}J8JQ?i+7tp>e6
z(RzQWs=;%1UO)T5gm<<74;V{oL;K6@PcYP;j6xY>a#u0^nwYhIKS7J`Ks2LRV$4y1
zB;_gTo}R5>ODwmA=9Cpt+`X-B7s(#<*uK%NC}K_o_$%EeXx6;P{ft`8wVjY0&4<Az
z1@92-ebm<Hou9S(==H0Nvh(Qekk(gjcd&tdYA$3Av;!Z$SaPEeC5M$UCv4#_{MWK&
zNE8J426)ALR&8_t=JkY1Du4l{TvnRoMTNvtLtO0*G=_=D;DSELQf6v!>dg;%yRf%l
zlDz+E@4Lg{{<eO#FnVXSVGKglAqWOhM|44g=prH$oiG^D>ma%$h#q8=Nz@pfAtX_v
zB#0U<qE7Tk?l`}5?|aTY=Y5~|{qsK0{p<UDzu(!<?7jA0>$}(5YkfY$c5~J175tPl
zVm5Zj=c;xE!-C;;pa?-)o8NU;tEk7^{8qEH9lZn=CXcPCCkIr209^!Gw~AbvQ>mt{
zoF9%izrzlb`&7U%cnjTRYk}H3irwqllyoLSS)>I18`)1wHwe>yUd&^C*j!nx<84(a
zThi19nhG7u>hk1{RcjsBkR5IWrVcWZ_f10Lz6!ou&OJP|rf8K3_jjfshPOVxUcJHk
zsrF1U`_=DIs6XpLYaG2{fYJJfpT7m~SwTj_$xPM`Ml95`OU?`&c;5}9@^RDbsmQN9
z>Qx@^VngZ_WZ-skv43<PtDJF2g<?9bfN1SK?S(|$@7>&Qb=5#R5gp=bW*A+~r&3^s
zfN?>QW+rZvS#EGe29@j}LDU<ZTH9K8ybVEq*afX01<my`*`bhx(^Do#NjU#vtrzd%
z&I^At>S5aw*=tfPC*2)f`-eyY2Jk|6v)VBAkxkD{^|0sdmo#QO65p1{sK+w$J#fJF
zA8S^?QFma9gB9R?<ijR(5fn2LQr`BSFKzbKHEySL4$|7;FCXLfjM-Q+gP+hVnZ4*9
z^HEPh{AMUgf9AWyYbU3ih=6(VyEZRmV{y7bbG<KUY!+?NUh~KyKa9M4S<TmrbF5oy
zn3j6oDL*bV2^Fw<FacgT|IpJMOpi0E_6q99sb6N3v7K@-g05~Xw}_jc!kAn@f2u_-
z$N7(D%#|H><Z5)H0Dta1NJI|$^s#uP=o<aXzM0T?tXTG_IZ~Q?vL#OO1hE-9xEH{Q
zC>Y?cl;TU^&_DvOVjG?8dTestwjpbzjqcCI@mgGDkpan%1Nv?m-HIdow6<K4>FdIm
zS(e83oyG9!oBUbk!h7*t{hsr;%~qO8sRDWrohlYn$guoo44b!5Z_tGP9Mdw)ksWAZ
zGjvshExWzpiTEoshRK1cd#FVDQ=Z{22P5mPO%C*{yryE~pnEH8R%bP^XN>c?wKLF8
zX0Mnd*B7LYTPOR|bDEBlG@WgiG#V9{uYCqMo-eXeqAN4x60B;5mQVvOLr--@%mGO<
z*8Ir$T5j{GtfcODBMKtk(3doeWRSHjkc;Y+g3upfvV_idc&DmoopWmRR61so3-0LO
z$NK=i*Y;hnWD|>-_Tntf;4C{6{AO>1D)6*AO7Q7QR7;7Fp_)cKQ4qjH0rtT7F0Pby
z9SxgY-9v<+`eRYS(rex@m_-hq&cMz$dls412T@}|pKopTbgBZO1&prwaSy;(X?qC+
z`}2~kPL38-A;WtN$6IvZDahV7%$5^~Z!dyLb>8)y_0UXs6*$qPm?VxsaD11UzCw<%
z)k|=MM|!qHZ;$Q&h;FiYQA$_GwzyS<<SX(qs{Br7a+)7vQ^6a}Vh{hRci_dSW0}1B
zZ9-0$)~f8N?fuf*_^SYHuY@^A^ZJY_mvV5&#527koTNOhD+R*?@&5JE8%o<7EbNU{
z{mE^o602WbfQ!b-#k<K+CsBX<YBa79M8Ot}4!ddoSzu)J$bg$7>Si|r)?fAY?0Ja)
z2ZTJ?<>48(=CRu;Avpp#4AL}@SZ4F}`2Dd3YbLnxwo2)&8))>+TPAcsIR2>p5xpIJ
z;GJz$Cwmw6`@zT<kT>2G{qA~%7hF@}MNZZuJ9u$T(7^U|Sm5~9dF8$1WrEtZ_=PFk
z2>8z^y_()KP#gIiGx%LJWJULqwChPOcg#>#l1$B&xRMZHx~@}I%WXNzVtYDX3(8?3
zXUvgm;Zk#rzp`%M84wb*R)(Ooy`qx33GYlUmU)=iYqf%kQG=A#<0tO}Kf5h34XrB9
z#}OpZPD$zFOb8hD@Pdomhn|5fzMH4JZD+u>dG^icdYDTGM$GI|>2X(F`6{p)c+pY4
zG7g<VcEfSuC}2?dwOVhZg6BA`w%X4%(_Z}Kp>&wCujBH&#i=yQ?FR)sPbTrwg2NSb
z#QbHRAZ$#`cGqy<eR81xx*8GY+|QB3OA?f{JG?Z{%YS5incFHk_md@6KQMUa==IOA
zZ>}vx(td*a*KaFCmJvqtSjpz!ovcqQRx_Q{GtY7S5=8!_&w&RiGqw!}vAxSG-xaah
z5#__>iO(S!1&jAYZ521Gfa#VS`_1Z1dY2Y{2c@2{C3KDfns=!;b)iF7m?8>9tR%Xh
z0#7Nl27A7h!2q#VR~Nq5>cnIEZ?N_dI=zuW&@m=Xo=Bt%3w-3ml4XE5+Npb-BQwEj
zXB=o+_RdA{3E+n`Md%Jm_k@zuUU~A6e5~j%_GNY?m*N}|-OV@|3Rr*eGe$bpDe^K~
z2u}jA#6kgnsQQXImiZyW<M+^h!>^BqH=pg3y3n}o7Ax|_Ot3u&;%PZbJ!C9OhQ1dj
zlLC?*KKb+}-Wu)*FqS!&36rv@iQ%VUY-0sJu@U9t$}!D3{1fG_J*s^y6HQlhtf6$n
zF7q;}2r<9G(vOxv8EHM%d3Qste;o{O?d)9>MN$fQgYRYx%tN<qv`uSdZy~k!gh`qG
zl(t@_(cUZ5wp0fCyy%#aW`5s#aBUI;B_pAMGD^nhC`Eg(5?P$LrenkVsu7lo@b*h>
zFP@P_UB&s8Edo_TLs`#k02=Ks2N3TnYh823P4)?MR)!Zu8s@L4v4a>KdUD%S$xzkP
zX^X~uyXaxY5p}>%>sbP-qI&dGtgmgQL4_&jq>sZlN+;Bu1V|UHTtZboNDL=4d+)g2
zzBR&vcUNp*{_6g4`MG1LZqXm2=kRLNpj5-^4g{~#?xR?jFt}*rP%44<6-?0H2|bdk
zjH!sq@isWMx&M$qr;FS<556H1bqU6L(B_R!EK)E(iF=*@#2z)sjlZ;y*bzF6HE-Du
zyNlBVzErji`q0D(7Km<2IGb%yoZ3xo3!E@c{)FY8yumW&`z>E-&(o3;zQvrnz_dd_
zOnwlS@UU2L>fEsZuwUo&L9h?xa`l%xCP!f*tF$F0_;i((GdEo=yDzlHtwKzDwl4D5
zPf3dI-RCz8o!`sikK+z9)&1CVRT}dkLiei658s80mk-T9*C1I*nY%PYs-1q6J9fW%
zAUCQAIDd&yv+alB`f4UcBRXk+)231i4_I+CvV@${r?J-Gy3I}TYw!;3l%8@9)x<Hw
z*fDqNPWY+SW4+P-Be0hA_oNVzd2Ad<%~tgfe9kin5V0uN`DD*_xEmTa1w4d?9V-{e
z6xY+Q+_flZ?iciMW8gJV%VyZJHh6A*XX>~F8<1?YW}g?G7a%Py-P4w{Je>i}I>370
zawt2WewDI^X2R|c8Dq$2<!n*xeKkDtoh5ye6c)r1&F}qBN_5*zgf^@)RFIETc5F9_
z-5$}+Cy!YJy|o`LB+B2uB{$#aldtr(=l3D<mz-KfLA&aj9XJW@E7Zn0$oyy1Jror=
zmX74ICfg_9;}X5=ZZP&+SAaeI?CgB?lQ5YdbI`A4{&^#1d2#*uo`@Tlir$!nVab4U
zM$U$@hEfs>fFt46NzO5VS3{7aQyg3Xoh^4_bqvYXVZsFclz1X+TCSFvF-k*68^$pg
zeVW)k_<8$RK5<ieMn%e5)sy+|3|J(5Yf~m2R6w(++I=~CdAfua|COr{l5))%FUyvs
z0@RiG{B?=HsN)WOHyXZEcu_~`5zmG3EqSdwS%LNdjN}2&>Yv1sMt!NZW)rL*w?dam
zg(wQd`%_YA#EuC(d{UUQ$I*2eokaV~jHyts2V#=QS)k=Q0xI}Q-Sw>dB-~Pxj&>tI
zO_wy-nA#dpgI*LZzu)oQ$q+7fDPuj-v*~`{r{t^Orf?dnYF@{AaZJ4B@qsJ$zCWpe
zrf7#*TC!NHEpX<(Co4E~ynvekKg&gYH4why{FOZQb!Re)>x3bwN%XDo=ak{35uF1+
z?eN~&Vde070M7+F*2iZmt1ov>S`)~q%L1L~SMXl_?91}P*^iF^TMwc;dt<ERr6F#3
zGpV0q6R_UPRzb>>1`C6JyY@>>%lfy+#_67#CN`6S7{x9}+e@uK(z;}_R$vJjuD2&#
z&2y2^xbu^9Giifxw94dtiJy>2QSx3DFlYMh{`SK8DCcf(S?y3pHdOQMby}>K{mS|c
zreCs@a=)EPAjP#}^-{>EAt-wxK3xSs+Ee~zP@R<a<;*(P`Pdzv7~hxpq~<A30=s<z
zT6`PYGrL41gujMVtT4t=--j$`2eBP$^vaO9f`BqogS4@so>#-E6euMewN<ohEHq^j
zm(V*?Z48-@`1<d|ta%`nQ*9Akau6NJ6}*-h6_62m?+A4Ct9^?3JlZ=Jqj=2?U0OTY
zol{A7<iC5L^lfcPi0X3LQgk54!Ct@eWfs(zwxVMZrnj#P1Q=0Kd%5y>E8n6=FEpMG
z?jL*{k-LHdO%a{n;JY6urxSdluo-HL=gAF>F8r^lgBVU(qr>gaQ?M+q39LK$uuv_Z
z>L0$$leA{;MlV+HBqJ;b1H%eeD|Od38<KWv75>Q0C0-^YpIljp)HIuq;7LA;(O)^S
zxE-oHJs4PahPkn54Xkw)WlHXxCD6JU9P_fNdH#qcV_0F(AHV;zues*b|C>s`cuI?N
zyzLj=x%(2`HtD}L&JdeTMXz&O;>ZT4Mo~r1s8DcQkov^CX$LEBZL{(xRT#+nru>T3
ziCJT?J)4hcg7xi4a)dvu_-sJcwNiZl<6!@pnW`x7GH=aE{8eag{h)Vyf<N?oJRx!x
zZoHs<&7>j14gGEINBv9utBn5S<6>ydppb3rcV`K2-zeTmKQrv$GaaOob9{oVZK6$(
z|D*`BRxM=sh>@(4WLAAg_Jt6$6%8o`i6Ny#?yQbO3^sS9w9C06Ti0Dl*<430^O21I
zYX;%u5cemsF;d-{B9a_bGT*JXU)giyWOjjWgPwIMz7m!+<egKKuyxFBjwMkFmLH2P
z)fQm>+yQ8IB3~wP0l3g{Ojx}I;|S+_b*r)NtIwoIxJ<4x_-GeW%SCw`L<#`o2YRC(
z-sJ_(GjViB%SxGnmM!RF^v-Kw{u-hOdVuiPzVQj&xtbnuY#)%nny2Ida%}<G6*LrN
zqGvQupkIB_ae=96x292963j$1Fhr)Ch%I>CoSOb{fT<_uvP`K=7%*P+M{1lOF|ZNh
z?H@z4W96aYoz$7ETAWwu@<O3(QIC6-qkz^6p4lt&L0{|fV#`vP6d*aPd=)%NS|7nm
zQT>1xPEjhaOysO+^@jxt&9kQauI~W8njO9^R9{_gibDxXpYwHb8oIJIkMFm-(e2zK
zdy_laMuF@%-frkV`mvMm|I=s^*j+_T)os(xjP=4!a&)tZ8GZO<y0sq`nLhJQHErql
z<7k%s#+$c|ZY(U;KLACF+KwBipbCVf7PNs55WL8drru5mw|;Bjhohg1c5-kTk+C?g
z`CYYuB2j%!yfsC4jUfRA6N?LgEykSS*|%v-s-HztEz{!-4wS_7nZjh-eN%nPRvT7m
zIFDxC3?~B_J4W21N%o~+_}wQh6ka`gJ~&O8;lTGeu^6RD{qM2GOXxKhI`5>%{5{RX
zZ)88dmEr3PuUib!IawR#o@x0$ueL_;EFL92fX*ev6;}iN^KGxca&szhMRmbcg|vHt
z1_-j1K1bTwy=e5{z1rJr<0ZK5h^jMQYw;FAmEet*g?0`FrFMhw%cwW8J<a1CoV86m
zc|Ow}NouJ+D}lcsWt?jrtod+9H2*I3f;J&AolL`1ztAdZOJL@ST5wR)2`+`C;J0Ci
zCRCT-iMRBRPPgCW*;R!3sv};qSOSXgS+2WnCFJ1nW{w?u&so*vkBB;=S51?-WG4jz
zV?we;M1^`1<1NfrG7jPf*YDg(=m+z;chb710$j6J!C8IJbabhMO!rW4v9!%aSF4dn
z$mbRXDLBT}ew$qV>7F3=naJDn`sy-k=Qk-S#Tx=j3*0T?L3bMRAu!wIJjTSCOC!b(
zYi0~M4!nctof0fc;zdj642ha(7MI`JJ0>w&uEKo+145hM+cnrW>8%^cbz-vZr(0%v
z)vHza^?7=~Kp_W{K2^Ytp<lK8%do1~GLI~{QxZ7}&y>erPV{dKi{dwOMC%p<2More
zp6v<6_S#eJlbnjpPdvdUfB(4rTl3<ZzlmD?Bt%9F5pcDYjLxIciYLiP>>GhOFNT|H
zu+k?jh?d4G`5V}m=^RURm1%CSgJ{_}s?w~`%?GVxZ|+A!{bQWoKPSSvX*)g^B#9D4
z=p<9~I%%_&INWNrjnmK~v1k3>Io|UvSShbWK&ypW@-m;9W@g<;3X;nlD^i>6ZWS96
zOi?e>D^nzKYQ5fLm@n5aD{tdnT#xq9Ap+&1zxPEMc7K)oc&!A%c&*4bK(jev$wHI9
zfL@~(C{EGmS+t2fpUpC_kJ?rJh!Y$k9ECdy%LZJ|_rE!HYfwNak=CIi)wlEJ@L*aa
ztM3iN_B#|&kLw5S7HBn>$i}ig|Bc>M4vc;3R(T*kry3dB@THA^Lh41U-5%yl3r=Br
zrB!<NqiL9%X-To+NnLvx<4wl?icUgZ+vswX?CkTbh8tTUR4R;7iA!LXNb-Zglzqzh
zjag?-ylsKD{#uB_>F>v-T7dX5+uWNbA6U4%O2%Yjb$laiEpo3&rh(SiuEb%nx}NgJ
z(@1-X2HE}TQ>N>{gqLsUh)H1G-!@k8^s{dE6=Dy81e{5Q1Rl`cIKc)FmOW(7bZy!9
z`NrM-Kylg>G~ba`k3InWVS4=r@1_R6>;CiDGK_DpilEF)IW;{9{HXX6IY^K(mfdNE
zG7*Yi_C0&i(wiWw`hyo{^iVn|#10)`pPJK@;y3>iQ+{0hvDv}yXK?)B6DTDpID5T~
zu&UT$-K5KJCP3TstS%!mbjM<n2AkSSYUS}*b-pL@2-Lqlnk-3eoq9TJFjM(B{+G&X
zi2gK+BGf_lKz*`RmccIyv@Q1-&Un8dGJr;wCaVZS>4h1JJ>~YRK{IBhLI$tX@fEKz
z9zcKVs8RIje7gdKr}LZaG#3bZ3AGDPBxy><UOfsBNZi-kB_pN^Ry+aC>ABx;Dozfv
z#^~TAiD*l4y!VW1eNFFUK}p&cjC3DAZ<EB(AY{=~nI>@j!GJ-9#9R&!{}o2~ZpC6D
z?v>iuV~fWx)@1OZH-TxBZOiVAb9uzvY7gmwqDQTUAEz3$kiS5Fjo(vadGyx`0z(y{
zEHNRyML0}a9FyyZ9Xzabtx4S=kLm`5oMf7Z-TnP}aLP1lU#q5~e21Rdk2+CBI>&F7
zlkH@5-OLrLAj*gcFO5pReuaeb@ND>{35AWgEk}dAM(#pHk$v<Ht=wobM#`$;X)0YV
z0|^Bdk-KC`GA}~0EIPSKg<$-_tyceNsM}&}06hl`a143#vL0zj!4eS_LE^%V=ET#>
zFh_^`TmsU%=jz~-k&zE^@q5t_$z^Wz>m2!LQ8cxoEYhhSXn?{bAyu`3APc~9orJbP
zf<T3rVe-muC_n`0a29=cFv=CW73H32NY{M>@0RP0To!x5@5GH%K<fulVoo@}?2s8&
z#JLG$?$bjgAk2o4aLfm5TXM)?Zoj{a7_FbaRgq|CuXpsJ3zmCl<LRD=)@PQiCvQ96
zmF!fGaXxx?gpXqWc9PG;a-#*0nirV+bdZ1c+a>fiF<#PI|MHeZPmF}TYmWE<`{_W3
zImWJ!0+L&qmZM=a2~G<&QN?LK&~~thxAU$(fHTId%!_7E0#0UWDPBE@_x)N&-IZT(
zBP6*PAjnPrEqhKrLLGqxrb{0tHx}*>XVs%L*~>mGZwp+1+<!l+Jt~Kp5GUTr#N?A2
zXibpayCruBWsp&HWYcJA@R=g4^MWBRqR2fO?e%KYS^Z`B>Xic*m_@F^@!_#n9VpAA
zU}_ktwhGw)u*^5m&l^LgN8s07Qj6hDGt8aYt*hj+HU%9-vHQi*={HMhCJfsS;$}HD
zjohbvGCTP(?Ax=O8cInrwmW29qAt-HA;V6Y^?*HlOND;yj%7ge5qQqHA+RqGtr@)!
z&5J+JgVltO@^1UZ4CerD2Wq0%0O?;<-1J6<rs4MKqWsoF8guRW^_K}!sU$cu7g~fN
zUifHx<y>ZR(RI__r(xPkTFya^L(U#*oVpiQCUHuRqApR=9F+qdh`rR?iPaPB2KFx`
z0EbK?wIP@Ge=^UD2M8i6#dzGQrcPd=o@4C%=dpy-+{?Ktop8|SFo(4PWc>ARzg;P3
zn=ro@du{h%W!|GeG378ETyt7n0*e<H3%2aw=YlMj0B4ZW3)5@wR!(lA7=|kjAEY|e
zMm=&~sJzPS`lHf6+V$WJI*u8eNLl!SM9ft*guc~1n>{n#wEDU>z}?#>wWtJXDKc`1
zW|)^P&Ze|3F1U2bTZrzDKYx#C8s2^WRk{zZ@zK5{nW%WMOO+wtB0?sm8}S>p6XGD=
z8h>t2M9+Gr?;(7uG1hhA+)|yLuYI~qmT;XIWfCHWaXG*Xli{N0L_kL*pt;OXlCG3U
zT}bAn%iP}i(1j#=iQOqV<d=ZO2868+nun45rWKMefObtafW8}#kg*FWk$7eOK}9b4
zHC6<c-kt2$+X?8SqIvx((V1soCa2+C*>u~y<IWJIG|uO;HlIg0jDm0SyS$)99nceJ
zCJ64$-~=DsxEAi6b`+IHT9A1~t7yC8PBuZ5=k7bBR^i^ruKpo*Uz2njJt<f)^HOV|
z<Vqnx;8z;$<J}6&kZkOa6yUcVBId1V@8xsa6@#(w%#LWcyb(p-{-EuU0q3!I&w2WI
zz{SjOTQAGIKS(sTF|na|wcQna^VWgik0gnn*<KEIQFz|$Oun9M+CW^lQNTDwYmo1H
zJMJ!1xnNMXGv~5Jna0zKJ*CNhpkaNC+Kw0!Rumt;OJmt`6X$p}37p{l6P4v!u}|b`
zs6Mc$60>xEz8@&K0O0kCH6g%EROiFg;Mm^NqyRSu3PwDOV$BTg=c4!H$KA>*>2S0F
z-{vF0ikyo&z6LtmFxiHc2y?mnjR07F`F#!8?<E}-?|{k34}B;yXq;@?{)&O^4(_Gi
zwy=~dY`Gpr!v(h+wUWbwAHMqT^mtwmB&LfG07^fgC7)L#m+sR8vM%ThsZNxr3U!V9
z2-k|!BDZyT`93>e4;T;PZ%9^30BAi+@>fkgqm8g2*{(f$YLaoWB>q|%A#9`=7uS;y
z&TWk*V<1%h!$WT7_z-(RQ#LFW?;KS$hLv2@OBQ#wr#;f19PMezO+n5QTP1P&H~5KL
z@x1KUt$4(xov?M@ftrhPnmiE(7R2HjW1NW~*w#`Ml)Je;s&QIjs&x|Q%w}K>9hCZY
zy}5AnTbvL4X)Kk%Q4j}3hhum*eJTt&9}K9m3Az(4*iDu+_e@cIa!EhRkx!E}VV{wd
z>RZ7zKsfd~71b63eY<FIK?#f@Fkcz)o6!2hbU2i7q!E@fKBPo*H}i3{sg9^*tt4Ab
zjZo(9^w0&;DHjEh2k`DyyDjZRqhZTaHx5!puODaPU?v^l(6vrrVPHfPKY@G-XabHd
zdpn%wBhNA?9W)|v_W+ghl=JTKb$e=KRrR?N8f2L6o^rtcuD98BePuN7xWa1bwYW6h
zWzfcF2gXz(eCYfix)NYk50)dAiBzg!Bna@}qw9zAlY}$$l&Fx7=XgXqbC-%{2&4QK
zfeoC;h3LXI!m|cHxCoBKg~O?V`_7v~9sIRJXSJ(e*k9as5yAV;7`L~Py2SF(IRg5v
zR&gZY3hNiIKTs^oBb@PuNYf0ER-GPjqsZ~D5fGvqWMw#628syx1rpeP7aUY&@Ys*D
z<pOUG#K*jr-wx)&ILJTxr1stAVgHCRD5hOeVXHbVE?1S9oP00+_>c?p73vxBc#TV7
z4RJrU_Gv;rxpb5uW6bmEq(CX%@XVf;j5xw&RnYG4P1q33H%j|B54aKcOFP67L3?T%
z>P8qGJ`4HV601JSsx~5iv5CH~9`S-oB+(3mlA#u>4>wBIT((FeL^0Y{26`&6E~fSF
z&q*gQA1`Xe>4+M((^{$0@ozABc7HDB?cwI=<9y%LjR?-5ep=c-lxA9*8Fxg5ewCTj
zpjtaSrm$^7YA%@y%XJkKZ}a?U$EBOA5?%A{jEJQ_CfvvJc_zMJz-jjzN_)oCSHDx>
zhpDk>o@6AWbk9+z#tjitX-!WYxgE(T`O>`R0~s=r7|--Tes?=Sl+1HkYN*J*Y>jt}
zc>PX{!dbyBfEBJylm~n7)_^5jOS?4Z<N!d;cJT(v6ZQU_NH}8&Xx60e$`N0^Z3)jN
z`~e*Q-bDMMQ2_XgQsf>fh>Tm#ltnh%&jC-8`E718(~rns(ZmPY9qCyo(PHc_o;Wm0
z1jljB0FL_GjnPCrL9V!OEk%KGFXKWy3iH^YAD6b^tz6_>Qx6KjrEcg60(kb`k8esS
z41PJaO)e`%){H~&hc4HEoOiO%O-)xEAge%$WB37#j#)2hHY!aOS{u3|-lTgCg#WMy
zzlQepwZDHY>qS2i%%yOrWids3S@o-iiW_}zQzCYn2^!m(l-79fhkRzB;`T<%RR*V@
zeYsM@o|Ao^G6U@+=oKJYf!{|ZyNqSf{tp$nm3gC~KsqHjlWQI@!A(7<b>N4)U|;eq
z>H-b}KZJJMaOm6?I<jz)@{Wv4+XF1Bpw#yLfiEh;R>Am?Rnh&V**C1NcZIr(tLPYa
zHa0@l?RK5!e4{w%C{gsOMSzf3QT$VT+jwv0D8q?9`2uE)pbn9iY;gc4S2}xHnvk_W
z^3u$Lr8vubSMlv=@^r{9vF`iE8PA2G+c|n}lJgISj)D~Sy8gOb#~b35(UAef>J}{L
z_lR{}W`;<B(RK-Mk0w#?5j5f(E8Zz}ZyIy*2d@JKT<zW`r6;uoBchqQKe5WP_OWZ4
z<T~ZLX1b%CK8vynm*qrwN1m4po)iS>r#>MO7QR*->Md<c!58Fv?ZiIXQ8UCTkD0_&
zWX=y&v}M6yFIeRO*L?YQzekniMu|+}m8eHz((q_rvx-RhBO@YTN~M2kSvAdY^g{oT
zh5$(Nm1^jNhjG&x{TD{;vz)=X$z0t~W(IN<628(wG2NX9alzoAySUvDKpK(wsn(c1
z1sqL_P*q)KTHJq7lqVJqeRt%rkIR#@$9CKlVq}CX!xVSPge)rX`q3ERP5Lynq=hu4
zY)z@TJ&c@r;7UF}b9wR4pQhuZfLNuSA8&?l`D`&W>aB`o#l$(%sZ-4ni%49O*#?cf
za@BPO!v!@^)GNrbKalJ3L5q=KOMa&xy2^9{59WGNCyzFjX@t=<KOZW~iV!0j0})X3
zDW&2kNoVY@8TuBwKAk9yf=@HwVM?H#-Z2?#u9zC+$=Te?(zqN)kMMp4{A1gm``z^F
z53ZRrn91Ni0=vyA6H39~e1#_QgFE`FCTpQ2?p;$&3IdW~Rm3kgLdhyGV>RPFc^x?d
zfIx<i-=*H}t9(Q}bBTDI9IqT%JoRalH;g3jY=3fa2ob^=K^XnHzkgBp^_P|U+vO1{
zX|{6=*dtLm)SN=!gEo;p$eitJ>&P>Vue6{kaJW2ZQ(k9df^D@;Y>w6ld+M<cXPQbZ
zLzv^*o;mwWv(>hi2XR>NHvoj5zgCBvYffI(x&DPs17XgVu;pMjU`bearZ)M)##kTg
z;&g8YCrn{!p1V>m)2%<rm-M7r%utLnVXHy#y+`7cw_+jU@;Wl3nKJ4|*aeS7!I~j+
z6|u=qBkV)ZL`7Zm?wrP98#e2PQll?Dbz<Pc=|eWJ#&SRjkI5aF+4NPNmMONfP;<?L
z_nGEwK}OU@)0w8vji{}s)W{DwQj^PN*5o#pFBLTXfRC{`1!$@2h<;@g-nMvMynskF
zZ5N|?S01$ee)c7LDS3=-#WTs0dUUFNrc$&0AgX<@>ULX|7&<J;eUHH-G5oE84JP9G
zQrFm=_^oM=x7+uW7D2-_>`sQ}FxzRuuN;6x63KZ^SRbzc8VUOf(i|7|T(Hiz@$-8{
zs)*NXmMXuq>h2ls)g$^}EqRQhKA~0m7K-`KlHzR5aT?QD7{{IH7!RfQoI2Qvl#ISJ
z9A7Qhr&Tj3kVr!m?6=*rrqG9I<)ca(g_}penZTgUged|-Qk5{G_-BGZkN&;aYv$&e
z{ac}mUMa<Ged6W$e(PS0VY^Y!ohfcszX9f7CBjeZn`fpPec@@ymrEjP6)U)E^kvxV
z3Zr|Hq7Hdj-)5N;>>enk7iUKCeyiDD(^#?ZQ`nUo%;SyAY6Lg4SOwwtci8__8lFIy
z0f1eP5fcNe=|hH1iFVYaKUIFBAV#cUa7#JIt0Vg8;Z#Ilc%hS|HLUz?!$bp-{H8nz
za(gObCf>!2OhNdjwz;NO<1!^;rnvV(Vv*g<EQ?NwSYmPZ)LqGb?i^&HZl4y$&Vd7h
zeMJ8;2yc_Q7YKr;zFEEdRqw=ch&#upRiT4Z$JQLSG3}g`rcbDVt5$8bqF;^Qfo*fI
zQcd;{Dzq{869ucqXnGF6Sk~-!k+WU0J$n<K<(Zg+@oa#%^WlBsvAD+am#KAX(u4R%
zUx#gPLdLqL*e?0@%#wx`WS<lU#VYD7eqj@KzVjl7+$1ngWwxmAPqh~cafkBgX~u%(
zz@5wjJC3p5wF#p+RX@g10T^VbRJ7^2^UsXs$m_}^{T3vk@SMEV-2X7lGfABn^nIE1
zJ#O3pR*S=W_r!yx_iZ@h={OYy(aXN2*Y_s#-0B(2Yd>|^Ej2BoL-SrEyoG*6ztnkU
zj%}x^+^V$y&dC`G#1zKxd?{j1@cR69@BB>SKfD1Hl4w3<qF$N5z#zyAJKfluj5tds
z)!!3A49>Y#gK21(0pw(KdHPm;*?+pPmzCEkdxsP2Z=eg}i&`@_rXRDtOAs|aXGzip
zyVYhO$~^wlkvJQWSR*$%H(Cg)A1yN{W?ulc{%uU5OmUO+vqswG|Lt*IY_A`JB3R}s
z3BQsWdg_)NGBN=_;1s(sF>vlQT)gP&o!aK}<Gd3?VhfpM63uEB8{gBVrZA_K)Pt~_
zf+S==+3AX{#5IEC-<6cLJ^4elRE1h3jEYFhK+cYU$R=}YDsE~1^?6b9r%AhGS3=q?
zODuWshb!#v1|4UOoOl>%uZJsMTemnoK(xsl<c=uFYVw1EUb0qF2rDqd^w&q*OOdL~
z_ivKs=Kq5Q{JT9=NFgag4Tkx7v_;b@?+-u}=VCE3f7{JzJBBdt?t14m_X^vr;Blre
zX3nmDdHt6WB?X7?vzJ|UDg$@N23v)dsoaNmo3p;okjcN`ObFrUp9Q!kExp*{0c<?Y
z1pEX)q3Kk+yC3|Qi~4s@_2PoL6k!s&#QOC&Jld+;88v7B7~!tR@$9cPQ`Z2fyA3)!
z)c8^K^8$YsG4qVx;IG{#FqP!2i<mweh>)vOJXur!bF%mJ$Vb1pWPv)tH&r!++Taf4
z@OZM|O;A6puBsUGx)#w>Mh$#B_{VR+-y}d}!D^@atRv-{wkqy$q91E-Lpx@D|8E@)
zVI{?B5y{`{R*6iO&SxY+I!xS>Vt<#77bJXY`h(}AH>>$Op>Aga(PTsieZIAc|A9{R
zpRt_waoe5XDTLZl%dq@y+!Fh9H`Pt+4@CXhtno*S5D}-`6fWm^Yx)tHESr%;K&!7@
zmX8wlg@~r5luU*sU7Z+r{^z-JB<3D}{2T3A&+37(%<o_MM7q~7rQyH-kU@Z~9~APC
zFuSJKG;_uXkP7OeRcLxUQT>dFa9h>#DSz9a#Oxj1_(FV+JCIKM$NhYPwuoPACVoUL
zSDeyPJcJw8!KmzXykAJAT7g{Vng1M0eDm;vf|Cg0(PBs-(g3!}Cw9&3T%f%Z;^8ns
zNCofDkFxtG=A_qHDh#v#L(~7R8Y)^Oc=mgZx}43Om!_G#8bXkge@Sm)LE!r5Hy3bP
zr=8JStweMU>$V)uzYnF6hZIdRPejDB;6YXz*i|Y0+U<FD#IV#?ON4y>aB=FjN`z+i
zB_w-=mbCnEm7b}KQwE#+sn14zhGaipB^FJm+O1y15&tg<{<A#);jWQ^xQq1sD*ld=
zm<IO`i}U}u&j0q|5QMm>SFl5Q|Mu;_f7m&Bp$XL?!e0viyPdn3FC@|8G{YV}mHSJ2
z|IqdSe2||n%v5tgtq{+Jef;;``PZ>Zq!MGfIsH<8BLD3sL=v7jVxy+*th-VFd3P?{
z!+##~3Nta=YX7tA$A5nOU*0)xUZiMEwxvHb{TEa6pAWJQMm+ho1+~6^`ah7wL%xgS
z;<vZc>Gl7m>;HKlv*AQLGvtz^^e_7RUrbu{g>kxbr2ksuzdKFDv3BkgErmDohpgNG
zL=piPjW+FXD2V<KmXZ~Ss2~4dsfe@w|A0~*t5a8LU%uH9@Qz42O+tjy)vnQWjQn5c
C6i#sf

diff --git a/docs/figures/architecture_2_math.png b/docs/figures/architecture_2_math.png
deleted file mode 100644
index e0c4a9f394e885a90ade43c2d671c6732986239f..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 16848
zcmeIZV{~0n*C?E%r$M8Wq>XLcww=bdZQHgQHf-#qVPo62ZRegodY^ZEzrHc<&%4Li
zIBTuxx#nDR?X$z>WyRpXV15Au1A~_k7ghuVg9rmX%R@tgt{JUCQ($0l66Qie@)ANq
z`0@_6CgxVgU|{0m$*NFlN<*mGnu-!;{?Otu?J$Z3QZRWUz&Of9XbJqh-Z-G}DeVYl
zvkMZqvxusQmJ^L%`8Q8>fTu5c<nZuswd%ZU($DkGoe%2|;}2u)zT-b`auQk|#=&xx
zMiM1&<h8-BNrG#G2Ic2>>zz<bvGn-S<o&7aHm~*+UA(;D{20AHTH9K{fqQSiA`?z>
zZa$Q60fHQOV8poF(*8ek5U;?&E)_ezt3ZQIp&eJ4ev?dgqwIxbOvCS0BpFxP0bRcD
zP$tVLLO^Y=fyJ3~XH#Q=ouQVPe!Hq8h`px?TO)?30f6B!_6m3*$${B_?KQl<F{Nav
z|AdgPduj9$&Ey!AI52|y=UC&rAOgxrp?llXhbh;4)9wL9h~eTN0hFP9<Rq4!j<w+|
zjEY|7uovR=8hhZ0(5o!9{E|?VqDVxEQ3H>~aH^x;1Q?0|8#o;`rh!rz5wUv>1E#l0
z3VQ%7gBF2VO><mk!Id^)QYe`PyQMJ`%65o-N>tDMBkQ%%uV%>?rHECd#*mK`@j;w@
zUu{B3N#%nmg~(ot8la`nXwm_xXqh#ACNdJ4%c5wJxct8Dr<zWAWQ<??14X7ogviYI
zwO&tl?Tg6qX;@`{=7RGJ$l)h!c;%8ik$PzaVYuAW#K?27nY1GL(NQZaaZZ4e2`qO8
z+SA>WfanV;bDg$++~O}cgvlOMVKRFn;h4JIB|utn7Cu}mnkYHNgK*RROVZx5n?I2P
zaGQSgT@YW`8UIMfed;~)9`n9T^&o|~^n)|{L|`{%7Q&;b%!KSF10dN%3=87t-A4PB
z@hyO`<bye1Sy!7s#iCJsFCG?G=m%oH0Pbsryt{;Gexct+uIOv*p;wwEF7|MX@A0BM
z%FikC5O;OXo|IlY19XN6bOEPcnM-_buvq^EAUn66?y4>OO6Ms|EKF9ASkOrjZTA_*
zc3*APvA%zJF05$S<m`_p<FRHOqq}aQEB-;m7JE_~2(3$ddSoobkec?~l%GRd>Te6H
zzGYjHm6oUEHeZt%Baj8{AD5L@BvvegM3!dgxC+Ku91E&`#HUy7ICdP?gRTGUZ0t?Z
zd#R~gggNDQLAQjYK1H7o(C&^QzZ#@`mT8Cx`AJf8JrvBn?c2L!Za~RJ;c97j;`~yQ
zP4yL(Y9rQ<Nr_XFbR6|t?c(Jaosm6>*8&4IBAY88Skea4o#pX2r8{r_G_?2WZ`e>>
zRoIkuAJbRm;Ow{PVEvh$PV;nIJ2l9DL^xi-<>D1R$!HL-oJU<`!{}+BEH}YUuE5su
z!Giqk=)rsaiQ^!PdT1jzJyQJD^=Q=q!~CDsVCMZTsxhoSbN!Li!!v^n{R3kMsRh8<
z<ZOmjl|yuceEx$)ABGPNlSc&Ci(bQTg=E_M=}IIZ&W;#Tf`2*=ju<N*|BDgrHN2vK
zx#+KW8gZU$CI@H^{~uzU@hgKu1!(T!Ui=J6A(*))B{-a@cOpw>%o#zlLUmIR^#B@?
z)St>`IKQ#11?6(lrcTVzp5bu1z>0zy7^Fr&-}JGoVW-3x>oZ#Ms>CQ8II9uOfB85l
zQ$w2%Z663{|75yFYxh}fYlt0>uIpfD<kG^6Y&|HayJ{Qp%IB5IUzVPP9px29tJ_GB
zRUb7kN?u}7mR+13KBow`8;dZ0ka!uzdvhjDbd+G6fR@OH_?!Tj%mukM{1Y+qP};8a
z7vjEnw*o;$4D(OsY)9xvUrIx7W6DJu@jc^+29Jzj+!<OD+9JJ@yn<hYxW%Um7ZqPA
z?7xB=vl@Rk#wtv9NS_wMC^%L)EA^ixwm@)3YKT}C=aFd_^DZnbG%nmLW|5F8Z;&sO
zw5*!vs0h`#Fn?wy!wh9jF3P1TKFh@^UMp(PpE+{51$uC{$94SBk>iu_<@YTfn{=7m
zw}7ojGg~lYGuNH+sT7*3nbSX!UZ9vNon5MOR4<lG&9^Kbnc6!VU#OVkDezQs<&Vuf
zs6;DHt!k3CO<i|yFMcNW=6bX9_V+1$S9wc(UW1F$AKRpVjIu>vK`+MOffWAa`KeMM
zMj-f)9ON0~TgY?B4xAEPB)lbDEV>H=LlI8m$f#kZ-nrqro)5Y)YXpxZyTwsSzk;f0
zI}4`uR|8^ZO15P~r?taCh7tBv_64_O`?@P{l-1rLql6*qL>baznP*X~C<OE|JZ)Z&
zjYF=YlF~%;fFtagilS6$sL|0e!m$DqtWnvFyL6^8p;5=tg$z8Fiv~S4fO;6syasmN
zW0vutLEW{Qa=B8ukCf5~xm55T>WIsT0F`H%QCXjgXZg9NQiXR#UHO7WllsGo+M;xQ
zy5)?X<3NhRi6vF7wyo#<eMMHg_OCTpmoXQ)v*^>7dy;$j`v>epgg*$M5u6bgu)||I
zVp?LbaJYWRlhKjID@D&3o6at+sGqh?wK&4*M{S@FrTu1L(>9~EN+nE%ObxA+UO->y
zs~oGG7+8*y!E3cRU8>1hYg@fuZERU{UU1FX>{x7{dn*i0gxdJozmq|>#?<ZTZyI?J
zX+ORbw8I+Ry4QD^ad?-dnz=w`i)Wke`qQ;4Hd-!3u3|z)_tS>khH=NiE3{9Z59}+#
zOZ&saJ>Bib!H21z6u(VZrC;&R*#PW-E!GB(*|$O1RpiBQy<di*p9xB+78%pfw9sD3
zdpU)HwLH5pEAd%9>tjB+$EcLBJm}9UI}_;x9JnFDA|c744WTEYWTC2|d|}lQgE3}d
zhq1R2KTzDcBY@v<8YquR7otWE<{e+i9;MURw!Smda5DVnd{&53ilQ!ZCgim5?qxx_
z=v(fWG|1fH$7ZvUS$%VRc?@{eB}F3Zllzr^%l?N)-==#>Sadt8S3BG`K`>E;cumGd
zT!px)@UTEyrc};76Eo#EXM^SS;?NDLt~8SbN^wI$;<VCqQ!%16Oirzh<iz)?Qe!1s
z>AD;h?z2~?#3T2a*wM-KKbbXrfpPH}HB>u9bTO&}ScAJJ^CtH<9Itn;rUAe|8Uj`H
z^358L#8mW_&hs;;U8gKd?fH5Ng(Y&MNUD*#yTW@`<L;SQsjWn=<a8{ya=aXTL?3a&
zx(QSqEHhpvSEI6}b@kHA%A2~?mASg8G%K`XyED5q!H{-6J`!8Gk1AKy<Rv9Vhim0Y
z>MxOZ5hQ)7CR}4WoG%g*!b7o+mmIGg){Zm>v>7jUxSs1NPlK<cPu8&IXi2n(>gX+Y
z2ZYD@t0JAWE}Cr~S0}cLo#r$wRN89GZ`Qj_@W#u=wY8ZWpS_ygVLF7{3cO1Xrs1Xd
zQelVbzBem>SMk#7*LB(Qe7Gp<z|+WXX!8z!h>)QQry`-UE&rm^;MlN!oG`a$Nm^;H
zF`}W_aewz4<78Ylrfjt>`?pR_M;T?&FXO<mkYp@gmB?RJ3qkXzr7RWN8ZRzC^>&hF
z2r@C*GOa5YU$>yH5Sto>ZOnhwm6feZu4>fZT{R|IoBH0q)%dgy-AA0|xQM$rok=gc
z|5`8Z@JT(Y{046pn4jnRFuR7s7yLFmeK_50jOZxWr&Fz3^YD<v<kL|@=b^jev+))U
zWraA4YlKa0E0|%zcE=W((W)`6!HFotK0A*&>2Zm+pzDI&$gSnrdpQ{KbM|MUSZGW#
zHWoWA&$=t&i?HiTRAVa{Sx%da#dPWY%vAA2Rye!kHTBDBjidhoLe2x9?_t12@P6bv
zS|+W#Hs@p6LKW0X`${J5i<ZsN<%!uD_J;ed<Yh97YENZ$OMkn|mB;DbIMxuG&Dsx-
zx|5X6`I!~>Cf!wv-%M_<TRh$E*Dt>xsPNKwOI}iUI`5s_U2)ve?=Po+_KLZ-KCMmo
zB!1l26yGwu+3%r5lHJMiWaFMkAD%sW4`1+16ih(A48Kpl{Jd}Jx%uQ9^6Fmd>)<;9
zyA+w1I--NoxqRiZdeZ3s=nrLky#d!j<7@hAbKtr0V9HnE<HpE-LUd#}M>Zf=k@J&p
z18W!$GvV5Zdrk!`@YC2Z#O59X*rB=I#UwbGbHwwHVq{@&R%L?Aoa$%l5@cI2mmsjX
zHlQDRv)B?Kl%|*2P1R;DQ4s9-h=Y-taN_z_-f?O(JoM7W>@XPvtUv{fcPJzTk_`dg
zdl@pZ9lq-d;_gY2LfDP<QNJA7`IZ|>ccc@%mD)F8$|n*ER9nE<iK{z;fuWN8J;5au
zi7&yxz<-#3S94O6k>)hCwWifKvNbTKb+fhujRphbcH;y+S{pm*<GWc~**J2#@euxF
z1SjbEZ!;Ys{y&B|S@IC7$;jgi**X~Gv(Pfr(i8H2!N<qvb}%yGR1_Bd7dYq_522Zp
zlN~1=ovW)Wtt%6)t%E5Y0|y5O9X%r*BO?uH1dXG+jg!6`jg2GGKau<!kFc?$p@X@d
zlew)8{$ITM2DZ*lJcNXQ3H|5zC!fY{=Kn*<#_?Zbfdr)c+e61dOHcP7+#pcyzpb3|
z=5EGTYQpB$#x{;18oaE`?A-qt|NrUvABz74ss2BZ?DYQ)_+K6W0_3Lq%m07L^G{{{
zqZOnr-Y?vA|Is|}myOnjXpk|m&4uN@gYFQ2Edx5p|M&UzuvMK22nP1GUP4&lyBqjv
z*5@Q_5tIQgJWS~G^FM1O0C?c3fSrUL^tnI)B+U6aW&i*d)lDq`1^OJ|>UvS-1J6?B
zT4}oHklET^=5b*oW?|y-&a<YiJ)hMU$HWML0~hK7(*OJRE|i-H6LwF74iVx<pa=c0
zTQ78)ANo)ImH%!-hZqM4zd-+Qa7iFNcnttyh2h`${@D#a)$zZm!1*Hsr1`1+dtClE
zxHNyg?SC-?>dXazqX4Bgt8xCDNjOm38P<Pz{s#_nfi6HVGP~8l|H$&U9mFE!|EBSO
zlSMNq=-1mBxk{;2+OtiW({IZV5Ty8x!5Y<!|Iyz!pxfKs!rGea#@YRb??;s5<>sIB
zek@z3pQ6zYN-`|Tp3g_cb&gNbR8?)$V=N0=m6}a9PfCrp(9C}&B8^-8kLGf%3Fyrx
z@m+7m*ji62TgPSCw}ME5u^5fQPUgx|PZp|ju-NVNhFaVPJTLB-Oz{>PELEEhvOW8Y
ztyX|$0e&!HGWa#0{%u446@krht_vqIJnwoe27{oJ)fUIZ036p<`|~iO7`vVjct^|1
z7L&xAEAXJ+F}96W>*_y{HksCq;c`oj1oD5;k=QN9{M$(;U4U*U6-}#*Bh)(~2sp<o
zTK0WLe^j)cl^=Rh<eF_e-&iU5_vxN5dl1)<rKphr<L(DpY`)9)@cw=vRQCk`(U|pD
zf0lW*9vS8-;qw@-bMchR1zlffC259WO`oT|H6cWvq+JP|Ro4ygwZV9zwacv@rQ)+R
zf=!U~#Qw*5%mSf-NsJ$_H?21ld>eOXD{SiBFBhE~cZX9w@aXhicNZIcLy4Sj6Fd*i
z**u;hn<*k_+RS|h1Dxz26S`a?=Em#bePp!XFP9Yz?nJX?zku-c)P$2|FTPz3;4Sgi
z9h2pF)dp?+_25lifGqnKz;#Yp)!5GIcy=h7X`KDMs`K69<7ML`pcMIRmha;g#qH9X
z>tWTA$!Dd658~ylDTnK3lxYoQ_ia=)J^$TLh&;EGz_jbfomeOug9Hlu%hrnyM<rQ~
z!WHKQ?X}NnT5+4zLO8hK_G&DQ089vQB!u?Ikw;CNmOkIRIo0=z&X0T39Ph?d(NoiG
z_x($R&bRwa-%Wq0S8T`8Ew**Hoe(!Vsse&ZOz5!r!K&8t-|v5*@NW6uAAh0=Oxo5C
zk>>GOx9p|liEnt_M0dVFpR_~A=m3)E$xT9i_P<b-1CIlrDe?<bb(ZBAhlq7ZhefD^
z2a3~8GA30~85hG5qW#Q)T(4jA?7_N_GMN!~pA|KH-|hl$?m<>Oq;0=d<T)K@--*fv
zU$Smmt2J!Y2Zfgp!po<FoK{}%=VlN?RA%rTlqj@h5JJ#S{B?`1t`~5X(X7jZIKHn}
z!5ZaJQ>+?a!P6gq%kjN2n>kNi?!(aFzin}zl%qWVd3}<6*%f}>3`E4v*tQ?QX+mzz
zy3hj!uEjr;Z3?;%AnPN_F0a_MTHk+3OJRtw_5cevU|VyVImI?7pdXq9$HagL;irIz
z<-Qg-<ASAYvvUamjEvB>mJ0!&b`#}RK@&YEt7lX~K5%ML2u$e%fFcc45zYIMngWTl
z4_pT^<rK>CTc!KqvJXwy>7wJM5~WP4A{EP*OAIuVU3C7OJ>B7Y-zqG$dp_iXBPG^l
zdR-5#+?*`8Vn8^H@H-ySG%gyPgVa%VQg^yPN8qQ+lA&#%C86Ud$Lf<u*LE48X%sJ#
zaan!*W!a!UE)>T_E%8R{W?G7lV|qQef|(Iu0E((nQ*mdhx;`Aj&g~!XFEP-JnCswB
z*uYT+j(LOzst$1hsf85jM&b}-c@#dJhegBKhVbXEypMIy3z-z-X$dlCs)~BWYb{OA
zFjDiqWK{$aK@?4jE1Qnf&X0Fb^m;QkLqSztZ|A`mm1tBo$F9O`u_qW4M#@;QL>f}<
zP;7@VuB%>@5vG-;ngB4BxCI^0v8kedJl|K+iCykQ!=SD<nCE#-TW3VR*J>hk7y)2j
z4lg-reF$iTJ&+$pn$dy>kOmD@k70eta$UEt>UjR;dC~r`ie=k|6wn_bDY_9_25tjM
zBa5^AiRtQ^(9A78{>i*Ji<KP;c2kFv^3Rezf&_6jT19~GdCP&Mj>i0_J4QECcK>82
zMjhnp=bLdZx=YD}eb*&pG`<;WrsPWr9XdMFEz$?>qk_aX;y{|MWM;(o5xP!lcUG<h
zZq`d2$595E@rHTu0HZE{zN5*v5G<QRlHtjb)3qGOagGY*@{DZH)0%;h>qz&JkLwXS
z7Huk=OkJCny|!JevZLQ$Zw{w^zHLX+ofpnXXVG%FYz3o1M*UIel%#wAuFeBAo0>n0
zr0clcMmZ$8=A~lWh@&puS5-Go2D<feg-zscc$`#tr_yRmKK5dtHf{KL(uF6~sqSg6
z8piUxa^H+9byJZJMTxON2LK5;FC}Psc&s`rqWm7!LOGvK>n1Xigjd13In=+A{21{)
zsK=40UEBTr==Vq$!pqHTvQiP~XS8`)8<o%ZxEWY?Pnoz95uBG*)wJ%R;}s;7tKJ7|
zp=ZZ(=|Vn#*0@6Z9T~?~tf+3BlL~rdg7?J=8S*On%3yhn*X*qPdK4OglKskaN(60B
zz1BOQ!q^1+(YSpQSn(t4)i!V(sGsD?b2qE>G*i)t2OCx!rCgS5D2-bV;lYxn^ol*D
z_E=h+y?nExNX%r$2a1!PQvtJ1QZO3B9Zj!Y>NC;(jCBQ+L&IZD+RUZEGYgm;HQp~$
zl$iPS>RUb_H?vNVWA5AXfzF2eE^7S=90v8vU3V|O+$@PRC51&BAl<E|$cQ>$`MPe5
zRkJZ?8CYT1>;rNv*{nvxilD9uz7Oo<;NRb5e3T3IRtbo})47I4+EktE!iE67&{1wm
z4N4j31iT9hjdUH4+w|4nk-Lm!CQbu8PHRWLY^6KTt21?gqHa-KEQ(p4NsjmZ<W(WM
z?eC)^J35oDf>v{?udrd!X{f@|-_q~TW|ibBFjW)_<GvVA<_ak5D$)EDR)5AYNh2`3
zFKZ7GNJL2K6h#KQ^TD(Y57X`!?otJYgNn?4hoDP8mul52onWhixNV<$<DZRzV3=N{
zn+Er`isb5<9J5PZ{-4?bB>*be6x8}dDVm1w;xPR#Q@c7Z^7e%m4z^sA9?)T#V)Knu
z(TVmHeei!qesR%ctd9YD(YHPt8D{7gr}T9khDcyy?tP)_7)kqm-kz1c)ZLt_^Xn)H
zrHzMOL3s<U<_YA6TxL*JE&CaQxTEu{mXS1#<h$n4DABa|35JR%;`vx*0YLp=FN?I9
z6A@|dN8B451d$0tho2Wy(-4FinBj+a&2qDW-vgzd#G=sSy#N|6TIjbpf0GU|9@G@8
zfrBq-l>_U(pK04+N2gx%Gj2e(9swd0&&O?0ISdS#`1lpl6?^D(lkfTP=i_%fDts|3
zj_&0^n3WO_0C6pMz4x;3>A1oh4y7Zm`)+(5jX`v&@{<5oam5bl0<sZ=0ffj<L#lJz
zHhspX*+wWmPn?@aKZa?!q&AMFB|Qw<lu4#_%2vdL8q!()jO?FYtxEei=*&M?5mc4r
zzh%m>ZJodsRFt8c6(8n+7gG5;L2Rqg$0A;^ur&4Z=#M^+TcHuLrNw?>Lz%W!m>Yu;
zpJnJb4NC$x;MrE=HXk2=&IMi~q$YMf+Mg0Vk@}VgJ?8bNZqD4?0rXFXQ=A~0!%Xk4
zDKPPX@~;*M`i9&7Fr0fU+H*pj<S5R0*0K=lCjkNFr8wDBT6<7F0}aweBke&jVK7cE
z>+T2g0&oDFcw$A4Hs6M20HmZfGX<0zeF0HI5<JF+f*4M65Nff72|x6a2q2ahQS7Wy
z7-Thrc_>XUyE1~&!^>EBKGOPeZdgYR5(^`6r095`4G}^A^3Kewt`VPzOC`f<a<;U*
ztUX*n9eRTdK>qtE$H5OB#p&<Vo8Av=2QU!S8tTEyC26zY(zB^}b3ZZ7YV!B?iB+V6
zvXXmU9O))}6N-u?rjmsMczkLk3=?!PO)*Tw)6%#lRI^w^e9|SWsc6tLJ1rLz73U~u
z-OQug=F%xQ87?XwlR`diAwCZ?21&)4{S|4bED*QT?3bdj9+bl5tT7Tnvjbv&<_ac@
zDt|9S%U}!o46={W9dR~;*W+b*476<i*)(fJ7^lIJ&~ZMhHRc(|#`qnEQ<uoau=C^e
zR^n%!y^{FP!Vt~%6<=X`)_yyWxNXX`7~<p85A}6cW9OA_;)+<;t~R(-x&RIhpT-Tl
zMRK(?1JTaTmo2N#sGi`qA*Mi?{qcT@b%$t)=ob=mU8EI}Q0_X!b{v6FtxJ|G)@Ezz
z?1i~NgD<&*f<lL6=NKxGf?T#1?hx=5NZsWT2NpI2LG!~RiqrM{6y|>LWk|1QKv?{D
z5-yJiK+H*9mWHaXJZ8%@-8jV#&+DgR&}De+m_=Cy>MH{|>_XhHpV$n|@VMNIpNQYt
z3i`YLpmb85{UDL@Zqa1!<v0!y7iZ>XFGV>@`Y?P`Ch<aZ^JSKpODH!=&=;i_$S!g1
zi0<%yG4Y}>bpyx{CX%#RzN0SjW|334k{fD9Ay<m<6BkbVL*X8_x}0H33nu&=MQJGO
zZI!GX$OYflk}B2|EOVL^fC^?ttLN%V=Il}i%Mr|$GWtQhJZE6sS?(^wlY2#jY?GW`
zB00^5FK_T}fQ($cnTs=HlKymEA^xt=UcajvI57NK$X<X5HY`Dm$&w6i-<=RY%p+W8
z3bfjG$Q~P+P83K?`o(3O0vJ-~gEtYq69B8g*&!K*TE<90?H7OM?w6<+oqMcjaR2oc
zT6AMLh+9M`6oKZ)kHW_@H42D3RN$OP(`Eo{2-FQX=?Uhh#6}ddsKZ<eDs_{XJc>7y
zcW;_}-DhC{H8Y&Bp!shkUJpnfui(Se^f|(21^IW6`%`sA6)%#oMUK5BB}UwDr$#hu
zr|1j)f>G;1F|q#9-<jd~C861p%$@up;rv5h7=SgvcH|DH-|U=$I`Jt&PdA-r3YQob
z;ho>t%U*AW&kQ|8fJXExuaQ15xC#k{O%T30yxzC+%AJ_*Fz4<wq95;X?Zi3=&IQKe
z=9n08_LFi*8(13}rvqVlglcGWj>o)rpbbR7K>)Nh7*5ALN?;f=rj<jw4C|6no~#)a
zjfiwJJ&)oBsM`F}WSk)Uh95*>&Y65vlv!C2ZJ+%dpqpZ<bl<Ie9FrT<+=<_<2vcke
zDJzp6beT^0=w@>K68?mx&TmARdj{|=;GWX?LwnA4lOf974NW8>U;<49*i>+RE}v7?
z<=IsqmD|0UXnI=)@@o!&A<o<Dp=^-95kCH0G@~#`h)G{6*+VL6@+)x<HlL7&l>c#M
zJsGppIGiRw-X6hS@1=TsB`upF1)36f{%wHnufYOXe@;nsbJ-;vC=(E@3tSY_T0-El
zP=h31Y0<mTxo_NW(D~8p!)}KxwV9W!B-_8xeM0Km6B;x$wPucNKn|$s7)X0Zefp}b
zMCxFIr2Na6pXz7Kon=LXN|+Xrhs?+1w%}~x7fNX724FxGFWtSX0kU6X;Ddl>9yj1m
zudp!gM9S#L)s__&IMSKTF<n)}1g%AIv<9HJ*=_kzS#2OHh91DM*i--<6^0hT%^{tm
zRd{%$QQ$M9F`V|xZL7|Ik0>3|^z!!69K)0D$7j|)9v6VpqNR`8Bz+4>%SBZHIeCpJ
zuBEtiZHU7pyFR6nL7v(!&a%XJxmGIV`^m?mW#oZhQ$~HpwPywZ^pO&%{tz$iNn7)`
zy%nu1oRJ)TI3X^%9R1mA=vgKHD96u+B1BKlya;~5#J8CW#u~>o@pVa6TDh9X%a4ob
zuwihA8bIQ;#Z)=GUa)-b(r>81S@sFALczSEBK}CU%=~7ujIWUFfQUAZ1n=`;zoZB6
zf~%6v-rOXz>o;=_C^+;uK5hG)dT=S)7Nf{l<zXJR@91OEeV%%a{3Bo?uTv6t=WDHp
zigM#D-XBRrzb*wi{5ce>L?X+D=_9_Nrb*9%qv<jR@sb27OTR|Tj(r2piJ>^6n2;rW
zSG|_7Ku~!kQ;?Ic6elbA3dKV&5h=kET;q~fd(>+ODs;k@sBCUz<RMAT%|<4*_8P`T
z*z^m*-^RuJz6(cF<6p}*pI&C`VLuq^1#3cleTnMlHGQ-|-om#0+4K3{ww%x}68SDP
zn5ythz&~$;F~C;}NGrH6wTvuYbfF*zu0Ym#8%Yi4XU-vayYxJ^OBT#PhvK_e<{a<;
z$ZJTAZ(%B@AK`URuWF$`w3tJd9HCI0FjM-Yhih3>xo01x(;!8pJqDO?J=4`I!!em(
zC_ZL>$*jsRwdQJt^Rq_*9^WVD=~-UDtE{3?5@9fnq&|DNFi)aAmUMZ~!;6c66gCac
zkN|fqk5}kZ1o|O`E0IC=Kz<8Eex(sOt8iiw^VyT>+;2XI-NDnB!6w8YgzV@>BD!({
zFSD#80Fb5zS4qaSVK2OsX_*nizM&v~l$+}V;fg1N3O`@Mn4a#TI72CD&cVxFN>Zys
za-~KzJ6Rrone9wtO|Ck`3}<ttRCt#O!lZ(h(WCt>h6AD=1t4@>OpZsOp)<}zA$(~@
zn|GDWvd-g;Xd|L#5tR`NmT~7`&3XQSE;n+8{9O@t>tyVq3tinPIq%Fe)6{&7aGNIy
zQFmc~`U4yc?Bhc_$Ib=}3^(iV{^i-)XQ$KtZ;+cZ>FGzw>FMd7!%-AWj+u=V#KtDF
z>FH0PHYF1il?Nry2d&sbZSw`6tGa`1!wU_}Xp{H6kM1b4u?UAeSnZN@Y0@zYS^J3f
zALFhaHx1?mFRKT<O|)x;w9cjX?<|$}(fb|?t?R#Ep1QNnor=|Uaz0)UOq+OLw-DKi
zdGAUydvMO}#;A0khL`RhueZ@Uo4wEFLpi_S1k-sD&{jRQeY{<46miVg#LT^dHgR+u
zZOcBcnmXSqIM$A2*?RUt!?u1Cew^rjSj0FL5H*;egUD63ew>=}dEUQD={`?8KOAVg
zof_hCdEVQ{dTM3f<{^5UIi5S_c#at!@O?a)9(6qS(KSBMI;M{>tF{a)&ljv{eoP`f
zx8#uVUWEsiU6tv+S=_H`pF5=fnTVEfi;vVQ@E$0Yb^ax)JLs^_l+)7g+EAuVqifTd
zvk4$GyU2RoCCgcR*@WU_r@Hur_t>>`N1*HT(7M6WPY`9}+PVVoZj`d$e&Ky_+75;5
zwt4FDMDnn=@w(5pcedWC`|x{V>^jy(>;48gwORPW6LoD8;c8vZYa7M-dPL{GR^}Y*
z<8gp5!~1@zSjM&fCr*XT%>dtf4ad2C><ahuRIJWzgoiH*QuEuH?MCBBI0&-;$0SKp
z$6dt=b86@7UaL*)@B6q=?gIDM9lQ;%Jt|#{J1v<uRR2#1?0T%{2Cp<P?x9?(`qul7
z?~HhOJLm&u8{UUPc)vM7wJDd<4u=A+@mV6sj(l}3;GG&Oq$9kCEqox0j_D&frZskf
zBz^O!;{<WCUIgwZK{N8#=KGhSd|sX(#AF;2H(3{6WE(m+I5m+l45YfmSB*{W){aR(
zu?q!oeII-9D&70>IzcwcFvNRVJmaeLm@BJn#X}1?)j8l?2$%94JLti^uWt0=jP*Fg
z*G6^@5~FZe5GffLEYrFBwe)fUQZ-vA$Zs+Yac|Pbur?n1S(da;G!`-nJrB4xmb^AU
z;hC+}$JST5yCJ>_a6C@M`plAVnDV_WDcEYK;Cl?2%Xu!NW^%1x#r$07ApUFrV~q0&
zuYINk?)*xIBkmoH2{p%yOpwJ+*Bh$5&z9Pyfe$OTolAFlx&d(-!~=t&ytjGk?&MD1
z2`|^{;=l{91FnhV@OmGRA6gy^;eqUD`3rkj`z2ip@BK2Ajf{H>CKF^CI?HMJC!&WJ
zIp60QA>1V*^HJlCkM{>G_E(UyH;68<pA%?DfzR@O+4uFu&L+k<tDwE*mHt?-T_UcT
z*+@@Zf1wC`%nb%8-J6Vs(+c)Jx@qH2>I<Jjc~13L>Z2G(h9v=4GPWKw<<&V)qo$lA
zmld|&mMQzBY%N2-nQgsKG)*5sTjE$1tvPg{dHj%G$aA7Tv8)AheM=jcx=a?-Kz9_e
zMjK~!wDujPoab5Uivyku_ooKWz6Y!H6HC}{Er%j>%kwuSVm!y3RMV~Nca<2B-YA-Q
z=znE4ogiE+=Q8D5S(-;uP8m7eq;vZNyL?Sui}}#*IO=?XV=yITb>X@3(zvU_q5JW?
z5bG{uNfY)A@2+d~KH>Xb#{ts(qJiVMV>am3)*@N3WJ`=)5K(!Z8TZH2L{5W5X>JXK
zn0NN`Z1MQ;O(5Q@o=cVitEkz?bBGsGAH(&1Q>XX+x+zGe_E=geU^4tv#)IRqQg$s(
z57zo+NXyYPwOD(wEZg;uyInHAmX_{sm6O$y$=!l@XNIdv!wy`@M=$Rt?z0y{@kR1h
zpB0M#x_~tf22s>R1)+CZ`iDXJk(hS=g=lMVa5I(m2FqLh1oY&?Xm+y>rr&)pS7?VJ
zw8{RKyJ5})(efE6W<9Ug8&(hAQ17+M8AP)WaEwo-7PP-VbbffNjGyQ{{C1ofZ*4&q
zfTMX3CgW7N;Igrg)U5x!xBLo{q@7lK7yMR4<D_NWxssJ`IsLOm3BaB4&#n19TiX`A
zsh^8FL1^1agle0?$^QPa&fNhK-qT>5YNNT6Ff1ROI^w>Bnqw7c_E$-zREQWhEa$x6
z7BjuZxx$WF>fB^FUu^osfV&Jy>D21We)O|-)X=Vp>OAhz`7U)R;bCKP>IuwG^sD70
zxvRqxQeiL9S?P6ib}ASc80n|78K^|S^zwK&CNcfkf;;0WAV<Mv4+-&@$y~4ZvO3>b
z)mKo(^=Cjgm12Rl&7qb8zFdziYys$uoa=}+z36<dBAfD<v>n&ys(kJmsfP`@Yb4dN
zQ@-5<urio5MjX2KZ+`}PPK9Ct%idtx+g+7!Ry2#Fc&ZuRS;5%vuXmNET874q^W<@2
zyw~xW8}Ii`-VzUo&Rf!K&41uCn6i$fqv=yBh~dzp#G3=3d`Gorlo2f}TyA=PwJSJm
z!Rj`TIcD2;c^$Oxuepoj8xIpm!CC&DISh-`$crT9NLv9#NUjpSV@Fz%yC9><!p8Mk
zL<z5EiIhqK7t~H?&DCL=)SYsbc{kq9T@h&^4R9`gPbXK-zG%5jIICkwYi)Js9CVy`
zfLpT)eohkcbTMzHZdOpqfxO5D7P+I25Cpv9H{U>7>hg{Nh0h$aZ+;X<N5flz!jbE3
zyTPrqeP{dyNa#0~2{G+8I-2j@pSD|Qr7YsOYkGwGg5Xta6yw?Cdgb-kVh_wNy=$9L
zgjEVliYGMJj#aXh#~7BTE4JV0bz)uHc38f)RS<W}&su-HJ#0w!S&S$ekXCXoSJ0hC
zxRO#gY*!70*-<<ZacfG<nwK>y#1)FDG!eMjKnjfJc_4M_rUV<$WPCB4?-y~nWDa3?
z-kJgVkjRkd1)U7AuUi=Wd3&xosgUEseWG<dt5moJ*}xPveADtNyfvL{iSQ0I0X$U3
zY0pAN1TS6Vuu#Y1YV7SH#MQsCZi^M;Qs#itsHwA8*C*V`7d0!${EFN3&YtCroiPt4
zYMUTq5v=x8V>+ty_Y1j{sY5o_uil`@aU$(1t`lV!=(%>H^D~km1^?I=)hG{r7nCo3
zBIROE3^^zbC+hLg7Jx`%7-^BpV+rBnF=)vj=CY0%_bnYFQ7l3orW_;c4k_&$$*G@j
zEOlQrGDSyg4w%Nb30kR*<By1NufpyzQc|s^qF%0AnoxRk-cC)edA5Xx-p^e-v6MA$
zo<PCRB1IOC%JQlfIih#%+@;C5ZZYWq_cj-*!I`<!hJ}K`b7f_n)iRch=Sbot!m=&A
zr=#mL?PQUm3>mkk`<lkX<_BHx%XQk3R4Z9kX4UOn#iHoEy>aUR&v^|aai##Uj&S`z
zB$`35=OT_F``}J_*~mQLF(0knj1@TCOj4mImMr?mqxD=WafdzxE+*_}f~~3_=_c;$
zCRCNWofhl~KAWaXXLWXUGtU9XMEcwwD2r9(v#7b^4Vj}@<>{iXLq~M%(bz7jGu{Pc
zYCEIm@WQ~35hAbOaigWWR)6VfluFwxykas*RaMo=bf$qw9#FIYNo@#Cc=8EN|J<}D
zd_b6=8ap7RuOrB=uC}P|zP9ULALl3<Jm|~*-uYdaH-l)cre>J`ZNFFtuKP1wE#Iun
zXmVCX^Qz)HQU=fItZz0}7}o?-VFOd@$U}P;{xMcw{kY>6mo*r!kIwsbYASuyGuN^g
zllH;SFSj4Ptn*8SV-dA;1-;aCqa@)%*L_ineg-nztg{!kUOQM5NxkmQQ8HgNE27a+
z8!fOzxU1d}%7#sV!kNg1aIO^X^EQsupC;NGxE9V}<lF=^i-CE{AeExe3-kia)U&UX
zitB^xz+R8!e21Iny$&)UMkghAWf;n_v=B+*bJdyQJ7{SLSWVng`}JCG%TEToKx@@^
z^-nd`lgFpMYngy0MLyd61o}w}^^;jwB%}o$0n%RLnD}aPXPg?8Co?+l{V*|ZgPX*h
zS7O?#mwJAUU7SFJ>my26_t7EE_3IQ?NYWUO$LZ9LS!8b&eS0nM6pF{{w)LCr2dg9;
z+2I8iwRn5tfdWINZ<bR1D@+WFt*oVX-#C#awZimVxGw7e0;!<~NX%lbM*8uwu8#%f
zG|0qC8U$ar3j_^RTRB6aKLMfC=V{IlCZL&)98ietWBP04GhnS)2a(0(D9{!ne#Y##
z%x_ttgxu<awC|{L(s6MHPzYArZRlyUM{gZc&x=lH2rv@vY*Gjh2u*V7GtWeef`7wJ
zw_lY_B@x27{VXhsX-HhYv9E3Dvf=$;EvLa>q}DXnkOUBWNz(tc<r6tW)|NPz1i@nK
z_*IpT=<~#Eo6p``Iny&abrj1^*~D=zG!wfBp0n1n02Iz_|2k91575lBmC#eGzcnm|
zB+sEwByc#A(*14yvtx`-=z8&xxefo>ynagbpwq&KJ7~VsUCZc@i!~4*v-KI`lM1S#
z#D}s=NLAbTu9Nl#5i16orbr5D7;8~U65f>sY0X-zYZELum9o2*sJgf?0ju;P58HQX
zday{>P||)y^SldWlP5d50Y`Cpzto+)xK!l;&eru#vdVPvzSPp3a9n-V!*PqiI5n^k
zTKuM#G3#Zo(`;;_7dgYA-a=;@Yz{ncC=5+v$RKuHXT~X$Ja@}7E9FhhBJ;i{dHQhe
zYcj5-q3fM9*F<VBa!P%YI`Km#ZPP?`Cvv8m9xkmZP)zkSai=2;L+w5c3|N%flT6>{
zrglEU6cabz-Exd)k=aU$Y*6rsq(TnUN?X*P+3%0#HR|>N<#WcenKrt-QkUdEp7}pe
z8?D+?$?`F4)Q*?;5=Jgi(`fjNeB-U40tOBAZdSSVlLNwW=7c_j3op_f)=;5KFjiR<
zE5_O1SdK!qv8}5oxa%{p9v_)LH#~*p8{5NP@vg(zZ$TGd6={19*TI?Q-nP)1EJM)U
zig1uKSEgnYUl`4y+yiag(Ou3`ij(X{224JEDQg*er0Zep*Avx`T=X=)-RX^nE*>KD
ztp5GCNP=~n_wF9ilC`h5hxTXde1KsbC(8R+VP7P>4z6H8Bfy)^@EvuG`5M9Qyb$N(
zmh9$9s#Orb&a#f?J5lGB-bw-URsZM(I<8|k2>#^mj}l2wTkz{c_;2+W2qrA{&`}KU
zr@a$UOpOlQ6dX@tsl3F#Au}#y2JT)vc{jahs3OU#l@m%K^-7ogD7^<Qr}7(qV>F|V
zhyVoq!N<CH$XX{tzK4utd<!j{;0FdaiNnznhfB!>CNcO_rVsXKdo~@mdp`3rmS>A6
zM1S^HsdL|BtO*OQBU;|2|1M4FJpjGO4g7)p;l<~RXwB7XDLr4)Qd%!a=(&^(1V7&s
zrL>XGBvCW3?GwhM&r3xC3R={RWPX<Ro8AXThyWBcP%4Hv5?8cLVWDB8v?H|*2;UCH
z_?FwA#%yh90+0q~lN;Mi62lnEeAPw`!G#v%1>a1g@ZGn+&}flfP``)#^wDo~h2aKO
zN&~kVR@Y4R1{XCY)Cp(&KpJFF-J^P-^6f}Rn9vb&Un3h;(y+1Q5b-&{C$-NjW^`}N
z=sC82=~t{#IEBb)zAjWeb5qD6)U~0o9t@oQ+M{5(Lzn<|r12O-$W9Q9umT353){6m
zMe(ihYjcf3CApmSrZWHcz<PHb-FqvHGG0_BclCuRM@xeRcF`_iHZR0yL>+<%N(dPY
z5ym+TaN5uBXswHVn1D~BF$go`g=UV~xDmPcO!uc@F)WK-7-X^dC-W})c-$P~a9+dD
zy%NONLgKqPTYBZL+5v6pzISvghQl4M$@x_T5|;tV?h&;KJh)d;J_Ent$kM_O0`W#e
zPVK|I_Ps68o}OU(40N9J9?Aul+1bDNpk!C)xeIf?Nj{NZlnpBa;^&d@`~0*U@b8`~
zg2AGrl}+KE7NaonwTAT*z9<y2qtp@K3gXq04*x2xn(nSsyWF)D97R@HJngR6BN*9!
zxAGvQ-lz69=KmlJNZV*Qc1`@kmNI0A5gyF1)-8YHu;3|uiHfmkozjbuwH+KfXo4&p
z+2Ny{kVvq{Sui7S7Zo~9`N#UyqY+2|FP_7TI<U@u^=G5d6D^%SyPG943?EEfN`-Ol
z0e3!rIu1C$+g61C1N6GULL&Zi(qS35VA3MBfYwdg?waB{w_H$P8MhddY#_O7c+4N_
zRMVx4>X1xB?`thc*u`W??8rn;MRLI?5$n`G_C4^Sj|R&5>nM)teI9P?eg*M@F)WTX
zp~d?67CE@K7KIiH2;;{z=4VVcuehDX%M6%kR+F#5e1jd0Fm_F2Dz+5{cKpKbc!$o2
z8<o^7qMi1uQl0*e9Q-+ZTPm+(uIjat7B`9jbpP#gI^{+>h}G87S?vSW^C69y+p}jP
zRQlyyCRCZP9BP?;^-c}$F)o<BvbX}QzgchXL~3k|!WShgjWG)(lr|<%GzumFH5X>Z
z;;qNU_v%i~uWhcqwA;$pVXx}pz@W7m`ow6|b6!}P#&uec0Tbk2k&E?T4=s!h3JXPO
zSN8qtDqo`z4dECjFLFRTT8(en#?+A~(|!59+F{Jqe#Q=m>D7~|SQgZm*Mp|Vgur3T
z_z40;{q$8=%n|f}O=v8^q@c*^^J;d-0f+Pu{K{?_AfofIfnh%5u3J}4+TLr*+xXZ7
zjYhwrgIMY?HdQ;v@?+R@b#`sP$=-TfflORubp|1OOw^9VATpQ9NFsGTl~>#g5Epyl
z_FFIa>Kt^-$)FzUmQm%wB*uq~M5}QECYF?>C5gfQ(>3SkVAs6*<~k3>Q^(Hny(OEC
zXcz*)y=KkE?4K}%OqDU1Sq4jJ3kQw75X%~&OnLnU4&q4}`%!UI>%DLepMyyVUHUlj
zs5KFiW&ndE-6HXQfbx$N4dYY((R1cy@{_dNRV;|(7O`4Q&VwwN?KUQg8z#d}sziz8
zm;RTOcx=*XSLwM$FB);uT>{$8Ja^4goRJ`}B!;M)Yv`s_ml4_#_X(>9&n+9Ol>v4?
zC7sdehO~n_cfRr_ZCJS|GQ%14t;W+x-<gy_ITza=9t805Y=|1C;l7L%l0hK^Q<#&y
zmVmiQISImDq@&t~F=xXFDop?3H!C=kYjx#Onq#fs@jb&SJuq>&0JY~Z!-CiB+X6D{
zb5MCuMIi~SvuhyC8P^d(7-2PT5gujIPgEy|P_GePZO=(ae*o*XIpJ<Pblo?sw0#@#
zw8ktfF&SZTupOrfFJoLBEYLNf+7g&~bTSV6Eg`m@!xprC-C4Hs&~z8e*br}dX9mie
zKN((k7Zc84Ju5I&;ow;^pC5=RwV=YF7Ia=+QB`ET<*^foLR#yIA*ds!q5TjUYO0wh
zd!-oS#d8exn;o_4s^R0Mfikt!X{>hDMeIGcM8cQt>?n6SD!q`uQW?iz*pdxzn#8Lm
zlXg9=c&8&xk+re|H(ss<F!fGDEvapds9r>LaaW!=83~93u$q$W=uD}y&^g}%M{KeW
z1N#%BagaP0Q*oEj3>WY4MB}90jVyJ8_>(gA<5-_|KZph6mDkhoo`-=)l4zdTD<+2W
zgKDGOEyjFfILm`ts+2y|Hpi)Q)M36}dWN}+EGhbFdyDeM0=;^$OkWxpx;YPTV-)kF
zhuNi%egN~~I7W%us9efT^V+Ug<yVEQBCgxK)F|Gft5cmt0tzGTg5B1+gS$-ou+k3{
zMKCcCYI_+x^o0i>8dqGuF!YG#Y#A&)#}I~L5(UVy=wU20SP>jMYT|)k@j>Jw!h{oP
z0PsSq`40o@Nx|>6KQbl@NTRiESLw7=YK{u)DVG<!$NcHTcj9mb^b6y}Fm@}k2a8}>
zB1J^hfBV#?{K(-T?#S$#qZ-Fw9b~lggt5?7ZoZVVRMpK66G7Xrq#>mBcDE@$(PuzK
zOZ)s3vB<acN#)u5N9MA~eq8tAgZ@&2SZa5I#19$eXRV|Bn4ccQPsR((l`=o{#}}G~
zk-LAP+bpuX{U-J~&KR+Kp2_K;5mLdNB^V?}BV~Y!>MQrQ3)bEGMj^yIN-=D06Vc35
z6)rTL%wwBG6$!+;Q-d7y#h75M0DS;a0v9W=t@a#OQ!Zbt;&5gXSyQjIy7cXgC&>X-
z*!sIhoFhkXYAwDARo;t$yn2a{)x45U>vdfhb<8Fu8{@vy$`;oG-^h*0pk9DkzKO`z
zMhqT~?17d^@u|u8qWu;gBxssBgjH$-TeAuKQ|XR_R*@cIV}&gT#v@I1vzdCQg>bKy
z{$9TH@6x%b3^lg;5)3f=Sqr0&E^7PXHDgwEkH*_9?pOnRg4JkWJ5lXn0CBwDi)k=u
zl!dVYBtgCQbg0U2xKAijCR@hgV=DOPd~dv{+7tNj(<V}Q5PGA=0eD|BMR<Z(u&)+f
zEx@B3TF`I<PRE^awjOA=;8+v-77+tfiw^dNKP;%V`t`FE*$y0QyM|Kp2e*x6(IeBZ
zD+sqzXx2uy`uBK8ks};m9Yek7e8oy;X>tJZ7Y7$e8S!jKEqGjpbu2N3iXFr8!L2KI
zi>E(hvALBowd*{6vV~hyTa%DB@x3^l@Vnt99~^?npgBrvxgi2!wy&#7<WvAELi|4@
zO9_S*wDPq_ythA+HNL6~7zo6%A|QRMA4Ap;sAI2@p0qo=j{GeD%#N1uGGMUnuJO)I
zv*@Ty384w&&yLwIIO0r~9oZ1-DcfmEGNDhb6_4sX{PiK3s35B6I*Ok`FI$W>8s2E4
z$bb5maobQVNj#M_bXaXtYq&mYO~82F2=WJqXj?8j*KQy_{*^7Iy2KcUxM+-05-ozM
z=@|0G;F$R*EX#&k_mpomM${%(tBg4{9KEK{$au7>M%mX9qsT^T=_9@f0x;tP-$YJx
zg|6zVQOjIayc(b^?KUaf*PbsOs(n)yXO(Q&4PaDghy95TH-{olGJFzV(q73?L4O0a
zsXPC2q=1JtyQ#-pyh!xEOvB2vM^D%~H8!y>Mxl^8R$JW=yx!kQDX`if-Kza-f3)jV
zBitiBw|0)8a-eu9JlKDaJ`$!!0YPhK^o9TxZ`lI_Wv64)*8gl9<w3Mef40WwTnljP
zdCv2u<BL$!MI>tsm-^Bl>O&`^lCiz?xcl3vRPcTno!y+<@(-%zp)^--&`F)`>vA;Z
zdi(e8I3=!GDcDq3e7JM_c17ry>-}#)#HeNfk|M84En}B2#%Pi;M))O~jiy`OBLU|=
z_M(Cj&c!!WZRcGV(M|P9HJ&lor2SJBwhs|>@RO>8QO9w~DD8n^tSyKo#i4xDfu`3_
zWhGd_dE5lu?d(+wKOEO3XxcrXl^mCYq9wNL>XfH%4APJ!S`Y4|YPObD*x@W$6w&98
zFE6{I=HjOP#J~97(0+j(t<_xR`UNMHi`=WvKKK2be+0isE@3q>gnkRv4r1!4%?fWn
zWB>I~!qD8PMt{IS?Es|)?s$~&%Frn<TdNGUcCQ*YNdRd|yz*>+m7@K<O~IExLgV+K
zHxMaNHG|IKb@U@^)6)c{U%^vGq$%S)=VkL~s28_+MwoDT=m7rCD^pVCrFq0|saa!t
z`<TxAFaa3bh)lh#^i&d^uHP|2x_A?0xI~iw!IeBx3AEK0h#B_np(H=)qJtfoqLf@2
zcd;=i7C=$Tz);elT+#f}5Hq%3!|3tM3WUETut|hzUh+GD65^YGMR?b&Gf%orP`b1b
z8|h3mUSg@HgC(j{XsX3CA_VD7bMMe>aC=XD$s8hpQUH$l*c7CQe~=`Hm_ctEnq^^6
z??-CtwL4b%A<fn%UBqCB^_-otWC1O`gGZ*6dT#XF-js0-y<WrQ?#McjpCoLhh{96j
zFX?}9Pmkb#1Q|1QyE;AOr21rpnZurLbDJ(LXN8_BA=6HSR06QqWVNjL{x7QE0jNau
zJX87s#39nLR2Ja|uiAXBv=GVo=c|oXELG6|6zU(CC&NL+vZ`kz|3&F9)87Fu|8MYr
gyypMEHa<QBJ6=B+aiWX<et{z)A}d@es2A{m0K&Xt`v3p{

diff --git a/docs/figures/widedeep_arch.png b/docs/figures/widedeep_arch.png
deleted file mode 100644
index 882ac5f33fc7b3e8695e87967dfa8fdb48246323..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 157281
zcmeEuWmuHq)-E6*h=d@5<cKuV(p@5r)X*s)F*FR_N-9W5gCN~Vw-O>PEgb?wcjtL$
z@2!mcJA0q==lnZd7ck5_Pp!3{do>}-iqhD3NbVpZAz{lrmsCMQLZ?PTx;2A=4!jet
z=D>%9bf?ThLPA+aLIR}hU~6V!ZHk2SJR~;mwwkIIfiJA_AQBIR7aH>;36z4QB91HN
zP8b`Bib2z+thUj$aHkmU-o3(={9u18<d;R7r5Qg<S-)p|ykL1?t$B5utmVW0_4O$%
z#bfJuxi=-jqd&!Q8Tp6$sfxveeAGzh&S)ug*YaNk_~V(vUjq1*ebMeVfxunZxVXsj
z-(byM=k>nfJ=KNJJTLdpeF#m{>A^_8fhNgRJ%le!JW)wYP*t=jkZg#ud6J{)$mhd|
z3%W{C*}b2(SeOX%wlI7O?}(U?ls+>oL>A57J^lO?De+#mIT6Mno5)jxa0s<{8LG(!
zx!~t=$3S8e`>%J)-x~`qq($%1iCFmTCQu!B!g4y@tXV&a9}&W2%TM;N`hT6wMy14x
zP_&|wRyb)HQgfG=a@NNs5IixL>8dETe}E;uDa|J&!e*9?Rx2Fx>&fCddiR)_0@g#A
zTA7H_7F`69P+x{_bFTe|CCM(6tS--*Up;XOZN2^B@7IT<1Z%Hb4pwUHS@1d(Nnukx
zxpSDLl(qUFC>i<<s4;g^gV$MSJzTcZj+mA^$Ck6CZ+9K{rF}P`o>x5Sc^e0#y?=V9
z@U!jvW|rH`p5|boU2P*hlW1{2b8u8iZ!61U+l1ZeL$jTLmX3#>Uvb4tgRt2RZ}%Bu
zkp&}tC+H=w{WKT-IE9WNCks|qs~H>eQ!wO<JI7?-w?1UUOtX&Sw#n}U+LD@dbWVJl
z*!D9f&Yx|><5X~dsNMazYbb7BLFoGe^~X=f?{S3TtIxVHF+|bObNy*o{2#^p5q?M`
zsPQBEV2>9l_LTaIJf0~3%YeJfUr;pfv@74SGP<SqmHQ{A$2=05KP@kg`5cO_AF17K
z3UQ)?7CGX;W43S*24?zWc}ao+iST=JZ8-BFieO57e@QCL=NwUQo=1AVME*?1Ak`Jg
z`0}<&*o8ltL?0+F5@%6(@a@54+sJ-;oUHWxTpUY(MKL~lBJ5|M1|Gg5;U{I54Er(i
zibgwNOPX|mbA|8#XX15}WRRV&>X$ORGhQaL+pqwvR+<GBdyXk8UCd$>a6myv<=li_
zwVhrK@)Vumz5cfHT@6<rPmnM=>zDp-tSu!i;Q8YP(p}h2_0IH(*aZ(t%14D#>Kc64
zKv!`RBb@YjWwD<Myt2GXH4-Z{4RrV+Pcj$=5{RG6gUs+?1mJtsc%;EOA>?fo3wR-t
zgVb74Q!+->!VD2=@)MF1O8ioFGHWzm)J&0w-(6n%I8bSc>pv?7Z9h(<o~0I{!&Qx;
z|8^fNODy#~L0U^<T*@L{H|Id*O<tFpYQ6whVxpLyY*FT@YQHMwi&;i`dSdz-dP(|V
z!VmOnjAUvxxj%|_LbAonKc4=GdUaYcQXy0^RB>KW?ZiJFxFw~O2^#<pS*@C{Dy>iw
zCWMZL3QDCf7`I!tKWk44Eg`=DhCQG=zLWl?8XFF0F(-Nqy)3P)u&ig6l<bIXM+|@L
zo0x>IsxG6h@vefHV+I42E!9VP@wq2?80w*FF8Rsot40g)Z{#VneQT3z!`xEG3uM)}
z)R*$iazEr#C_}*HYN{_BmBZAN3N(sb#EZgSn<SLF7Cspz8GSUWH)`%o?p$2=+$+ny
z@l4?mmn8aW!&6BT8WL<0=u=^<&s>*JIl0CxxGjW7TdcHRNxkYFn8<h5w=#@cqpp_m
zm!XNqQE>nG)>7w{v}OO;OrDpz$8%{gov3tvxeTnZ&zD7ROpZsQTK729rkO&uUqY!6
zTnv@nM$>uQYieG`G!FY2a5iz~cXoOva4>loge{M4eD}k{r^JHXnIsN(Rk6FVb+B*0
zJqQ<n`+#`lp%7a(v19nqTON|?hdylVTp^rVMuRU0&GU`AIGLXoaeBRc#<Tx438G=*
zGMW6UkG=ottg&}1N~K-J^XlBH8k-PpeyvdEYE2h;T6vd*CkeWowL_t1?UjtB`Q>X>
zW?Dg-tmQVJy}8<7$u7h=t4?HlmfKf!ejYd7|Dn;&{oN%261GVyfF<CkS2+#wsB3J#
zG(I;wAH(^8Llsnnb4-@-F8v)7nGSg~Kc7Hn@><GS%G>1V6dnPngBss_vU+kD3o4Oq
zTsupLo^UOH?Q$)J!)0r47xNG#O=nhZC@&+gKKrPKv}S)AS_5?}6^at7@^Ern^Vk!z
z5=?V{=#JUw;vob}bmXi#uPteSG?>mvH;SCpoIN?<+YH;s-eKN_9INhz?H25LEZEMa
zEgQf@i06peMCP18kJxza^bF~gg6QT(_Rs^810JBKU}d98qk+*xu*)#szpe<(NEb_A
zm!MSOOr=d2y^n6xP<m(-(W=mD9&}A27LE~-`_A=&1!*nlGjr%eE+$KZ;qzD9D_jG(
zCOyV`CfX(s%Vf$b%P1E-e|0U?E#$PZhBrrjj0GvaQlzH&M!Tg*mbk^^)p{^P-I2MF
zS<fkN-jiZ)kv-Z|m_TR7ZnkSqu1HJWL9;D))q?wLY$r&?l_5zL9ZlH(t$&<2t5I#K
z!6FMIfsm5360_W<oTHLX+GyHVg8qA9(9u0`h=Dn(nRWL`<iiiObo(9}x*8JPd;)yu
z@sd(e4195)k~n#CBrF-2Ke+U|q%;~GeSD%>B|y}f@_sToCzMMXyx+U$^L?9Thd!6(
zgqO~xZ@#w6k;bv4uROlmVP;?ax+M+v3x>5^&GN&b)Jg1Ee5-{#EK-UgOq+<5g++U+
zp4zZ;CTln=L9T?kTuY;BV|uS<iCKzYN-osh8{;IsShGx1W2&|V(q2WmD-!BsZD7&t
zFw|vB(-PI4#hpc%WuSqnp?B4Jl+%^gRHCRi-So|(ceJvtGO7|f=2kqZldY?vIa)64
z<YmHF-;jAR{YHzFMV(j8dh+>nd~;a=N9Jd@=z;9Yq$ag2@O0i$vCYrIMz^J9T%AB2
z8=J!k?PdjYoz<%T$@a<ZO^lxdnkFi$WzjX|hOlzIvib?{<qO?<9V_VSR*46-N8Qo0
zEg|r_u1>v@$7~{h1xs_WkI&#0>1|Kkr(-%J+pi3{VkiOwPH!7rL|t@GOMDH|#&yIQ
zqhuHUd`iFnO6Kt(Z;5>=Lt|b6UwZ8?%^zi>52`<WYSzB4gVkK^t`S($jV5QNKH)#T
zRGZIB9~vCGWj@$<P|;`U<;s1vvFUgZHqn1Ot=aYKb;~%n&17EK7Ey$OL!;9M2h3rJ
zFo)?an7o9daoKhARlTamFK5_#FORNGg_HayuiMc6a)M4%HOtiUbd!VmM9RB*cIdV<
zt+UHC!OVJnn)CHxfdB1Q+&DohK}NU2gPMhqqv*uh9v`a}hc*6z%}K9Ai{YUlA^+64
zX4utLR`trmORaXtMp&hni&w|_<+*J`{8h$r*1^(_=x%e&b>(U7c)?>I*Gqe6Ht*8o
z%8kQgo`M<6rs@5@dA)VzlONH?<J&<ZF`mO0HOB-n)k#=Qe}53n^O0BHh2>S{lFm-G
z^lW*vPu!mA#efD<5cX3p!ns}wBzF{~eRL$}GfgES#_vCUzR-X5Stx7rP)kN~aY2#n
zzl~9gUYd8u%7XnmoZ8n8JsWy!ct&6N+xhH(zL#Xo(JgDrQf%itwbvVI&EzIt7ioJ}
z=XW3Llh1AMHzS*(VWLn>oT~vgY^bTGjG2N05+m>$0|^zG1PKjzg$#TPBa{Al{S28N
z>DE8qM?pdgvOq%p*Jl)g-|(+U;2S>XpTBQKy+OJS{DlvEyM9FZ{b_XSkGFomzBL0p
zLlRS!kdXm?tC~2Nn%X$NvURHU`wAb0Y4=>q5eex54g4EfMumC@n19SdP18wJL7v~l
z7Q$lm($?6N#T8-)p9e|Ml^=KsF?BKmxk9XM9Qj>^sQ&Q@e&99yZB{DKKR)7QB}ApE
zpbV0*bua~Sv#_zSQ3>Axfk1)|FU|N>B%l3jIPjMc)hj0_JAPJHC=|*9<z%sSFlS}w
z<Ktsx<6!0BU<N+H?C56WWaP?h<M{ZWll-1X($vw!!NShT!qx@^pV!FP*4asjiVD8a
zpRa%J)6~`CW+fZPe;o@rAS?U{D?1At>z}!Sp@Q&t`IRkPO|7*gEg%5RfHj2q_}B#h
z@%jJp<Yq;Lk(xIn*|~T)5Qic>`uCygj;0P0wh&-VC*d1@{cAAd!+#AFWQ89cffWCw
z^FQ7N2rYa^ko6DKgzr=iQ9J?oNM<3a1eZ!c%HUtf+rSU{fBuHQUN_5gU#3Dr5=D}c
z6jO6W-k81}e_y@s+io>oYviZr&m<ktY3M)t8;ia`l~n5~wjZ<8b9`4&%-4a|R<ppE
zk%5ux3;Oty8kTIdCqK(CU7zxh_9<t|xj{eY<g_~7Reo(mb(1j6u{K453<Knggz_JL
zh$5lsqEe&e$_@R855PwnqG3GWkrG8hM#cNz|M2t0!i&V<WhehHD<L6cHr@I!i{RNI
zqu?XcW#9TwivX*GKwqK%c^Am2C@i9)RC8@k|M|Q~D5%sYIR9mvqF;c+q9(`V{D-5$
zY2fRNI{P1K0X)E{K|&7kB^W~fPgDnpiWfO2@ZYwHOaubmvoq%Z-xK_wtE1q1-u^G!
z1kC}5MNyQ${a>gqD(b)eAB6*WfU4{3EBPg4^}kUa5DxPHvd#Z59Dm8m|2yI6qTAUh
z?_Vn@X&OA){S}*>tlRW0G_TR?+y$GASNV7vn%2krE8JMtp5$gjexHGxOJ7gUbq|q)
z0LuhV7o+PAM|B8^h;p`@&K14RkFcMol{6kD7T2vY($Lbb1>g$*nEM)VYY_J>)gMW2
zD+WaWXX9}X+07crs4cp4@1-dQQlK*y0rza1FRNJURx*6wofOw^4YuNn6b1&Oz9kpX
zkO;!Te&zG@-5-I((*Y#dc-@;*=Qk*TR!~H)FUlW0(fnAXQ=M=B>)YByWr22;^}_@E
zAM=zb-5;2TiuLL@g3z%^a*bNwpuL*1sW$6Nw&<^fOlrh3fm!TQaK93`o-V~X_i~N>
z#F_Ovp7y={Sdg>+@@Vq#r%42Zu&_P>oub<H)um9JtJIC)f7Asw)736*`g<=(ehaAH
z*OyQ|`*y-aZiu!Y6-!+A)!Kxhb&-CfyPwt7S*ho#11V~rW?6=!(4m?S7ip=ByszPC
zq1HsHQ2<Nx#h&k>2?@p1SCPWk7mkzC+cS-a06FfjuyS&$GB7Y4O+|=Y<pOgN*7OC<
zVQl;?laQwLsyNyhHwnOJZ8|Zw<9Bgq7t4!Q64r{QQ>ywZL7@DUiIx`u6X;6-e(c>*
zq5Q*+IXu6^^|8s@<I5wcpdQvuor+iK2HvMqR%6ALOCAR+=>UH;XkX23oBS*_I@@W*
zbv_z5UmsEuIfU+e+xA<(<#bqlUeBy#V^cgk**|M|<r~P?Y&S1nX)*Lohnk70JLzeH
zZ(tnFifvC920;a<ScrkoB^T&gQ8sb7&Ut;f$;)GNx*pswbevq=e03HT>WKYkZ%D{#
zs2I;r+9V&{(75#;o1bsL@Ocky$K^s8>r=xoNVbOn8PuRt_Cc2ya{;(r8(Q1j0lt#E
z8Lw~QkLf{EQhl!6Y|&8g^+O{Y*~K#6kqdO8<_KS%m;lko+D8w&C2)l@>$jV9xGaYw
zwVYq-^C1!}0#2}%7es$HHHU}1&o*-LdBXa9uf*q4u6oL$`;lqSt<meN<7Suc?6;&m
z`Qoqd%y5{kQ|&8H73Suu=Da&@aEpHb-UzG}f4uAm$&!uj(ea39&ZvS+n$6C7Y&W{p
zC;W;K(Whl7g$0L&MQbG(X+Rc5s%&RjqIJ^EdJ`>2@?&u*1nJB94FPuNs~3Fe8GU9J
zEB&IQE0(F!ZQo3%ICqNJ5!R=!t({21W%=@25E9Sq_JKvuQp9_|mxtr)L(Zl=)g1TG
zpJSs%I_HOCrF0;pt$~Zvef3I9b;v|T=Q)<A!7Y59D(itvndptsxaHoYNoMbGb^D-G
z{rJrw`Mx{Ra+y230O!ZId)Pb3K220w5!DEq_mO4g<aqQSZj29UKLnRw#%PHdYRo3P
zE`)HL;)e2vACDRMKsu6QVyHQn{chcM2M63QHj&pT(#`_dKcRl-Tl-Y^>QK()wo<*z
z=0-Bs*nS5Lz%1>OrnCG;9@=AB2Zuc)E~_hr1dn>xUE`&trOtTwE1aD4kpggmR=HVK
z7iYiVCsNynUAuhM2Y=Xv<R^iad@cR{*3CP}&6vC%)AcFWYfb{Vm(zU`(^Qkr{`acF
z8DN5o%*xro%{rO@&Zjrma5{%d-f)<F`v+RFf!$ZM8LRB+g{$P-4QOi#xSymWs+P_V
z*I`$C-K^7jd3k7SNwrgMu#})-PC7Um>HvIelG2BwkR6=~9Z#vwo!zGNRQ?#T@a_C;
zy@V(iJcDbV>XyAZcDhtQ*-2<L+kCp5G-H-RhCgG^C-!jr^ujh-G`hOjeCpduAc3Nt
zfLPFlRO9yBA_kA8Sg?EllFu({<*Koc@Q2aY6<*t@xD6&85MCX>@$I*;`h|x9Zp>@0
z^RJ1f6n_+ml4chY>^KPUJq^uY0Oa!;(wzpKaob{0&8|uL()Q@^ZPxym=I>`qd=_NA
zCoGKzgRPd|Ff(gYKKe~!T9b2V45bo;JNjH-9tVwN=b5Y}Zu8wnuws@74POWD8;l{?
zlvf07%^BPtebCjx_WHzhyY=O`&E>dLz`Y>hu}OY|OS#4Du=rkjg7CJYsQC3she!jg
zm$V!buEl4BvmfR<6a;f!&5GM78S6EV&K=+Iv#p9}Z}vYXTaMecZ9Mz{`*b|whEa^g
z+VCT(W{i4AXtQEyqYIl(G<~`rI%%-jAI~lUygDA`v=wYNgQDO&P#is|F%ym_Br>B3
zB#8WRc|5z_DD1c_&xhN!X}&M#wBPOF$m>UVzIIBvA9j7gM<EF7K3mgzwEN{2?)O4Q
zcE|0}q9lcU$ix%K>%ejGS1Ui#UAO|YvWzt%;$v}_8??R}M+GNA>XMNp{2nEaV14Vr
zeE!qmny6;QlWnQD<h0lI4(Y}0^<`R*EOI$^e5ZxsR_E;X6;!~xoUyggdxxy=I$+Vw
z%0M9%pOd!y@_j9l*V(~)nJI{dM!dTjTA<dcVH5Nex18#W=XLU24706%%3L6`?eg)Y
zZS(lT>*nUuN&0mUVCY20VkFxvt3ia0ZKJx9@M$Kq&9Ze<!E$Q+`39=`h8IWGx1VFJ
z``hC+3EY3_gg8;R$0WnjH>wTP{DmPW+1TfWkn7IeMGmgp_G$Ympcuh(i7@bXP`x;n
z&*Q#Zjsoa(aZ*=9?M=spk#<+D6PwnMUoKUT_de*Ao4ti{iRLJ#G{A&xWpy6STmO35
zq2l^FFsI4QnF4Q;0}u*7VL>M9tJ8j;x*UZhvvaO}mnx~xdT|xvE#oeo_3mw*Lh~*t
zvNQ9Eu`H>vG)jWIfyWIn1q&u>!R9rd2kqpkBeWy=>gB(hR<o0)`ffq1wdReM4Vydw
zQi*qrzv<Q&xI7q|?yFLo>F(3eu`xfV_vjr*>GGJ_|5h&EQn8NOG^Oa<?bzupBIww7
zG-1^UyWIVT={?S#;N2^m(aM)xRNCOktlM9)z8GjH;P<X2PWIvg(0`NM5xfh*_}%vR
z5Rk<HHB^6SLr$R!!Ie-B^o6Cb5Q24Iw2d??@-?4T<MQ_d)=IM?|F-zUNVZ9wC7+#*
zhd&|J)5~7Z8*FEV`z_8%@{mK4Y@5d4&F)9emdC2dAY+#IM!PB7P%OfHYPON>z0++=
z1PbH?a~evn?mmxa=MCv@gm|FfziyTT?6TtHCcXl-h9x1}RQEN8ZD+cOU|5CMAu6@}
zTzW2~$^CH6bp4FvfboYbCS`Ig=L|bvftf75){cscOI5vbduY{`X`G(MjxnFGoAp~a
z`$_xQPj2}L=3G3W4T<aL<^QNdj0#a>;|wo)OYcVg!GT&wa~+#6zQ|~yuQ@>T)+1_$
zPf~HyywCEEAC;A>M*v%SJM0@+zVx2F8TPe48Z-s*XQn};mjC!wFuv(xuN#ZlRoL@z
z1bkEnTm)Fm@kkmnreoS3jE=olDjCu;1z0slN9Hr}AHyD&L`OD!aqGWLXHj|F^q_d%
zp25b{*F39J%RFVtqK~x%u-OjYb;{&>YeM)Z?cLTBMMG*S`yS{9jXaxgxdz0p0g^Z(
zjsRR3G17o<<93i6_Q&3|3-+Z$wzt4^>m7Z*Y_v-gr>pYutsOvh)Cse~<^E0+Wy6N|
z6p=q`9dn>^jC8m`3WFO7R`Z#osgSDVse4;5K0PDfGxwKEt$Ph7JK+Sqm~b&nB^_?|
z@exRE(cyC|PTEWr{KbX*_CneM8Ix{1q-cHHw=I;Aslnt7&=mW)dBewHM650z8We8J
zrd8D($eDl+|DfHRi=xPaEkF^CG~L<yLhWAPqdx{WXs#Uatkd%7ga4djF6_%g^JF7R
z!$gC~I49SmBcF}3jyw0UI(!48ohHsBf69dF{o@>L(GU3?GE)qKaL6~NdhVd^iwgLN
zw#Z0saqS3hu&j;<YWDlUPoX0G?!wD`4;r6L)lS~o+3BJWm>(C?xhVLwtU>1TX5D$M
zLOovYyX{2Nic!whbgV%z;RKNE3mr)8a8M_g@fuNeS7v6NuEY)Ed@Kb7tA)z**`K%Q
z<9yAN$jVVNYD_xbk=sO<d=*?F6Y=)+GoB}o{N5~X+!e!s_Kg%9IfT;t<SVIlJiEyo
z9zJKl{7i6)rd6=!D<_W}U&zfADL(^U-wX8(G{^cxpd`qCt=zDEygj>}0>{ryA={Zi
zMv&3jQK5tDoZs!94h~9MzIsf>c+qG&diz#v9x@FN;81x2FTY2&=1Uv>u|ttjL-9eN
z7cqBWH^ySulbs2x5;sb37TucC?=E|EU3#Bi%J^}l`UcAI>YlLGq6p$@C`5j&X}zGD
ziP`6x-&q;>1UX%Bc<y;JA2j2QMaO%DN_}>)s=8<=*gVL(3cfV?7GAcQWKePYY-dq;
zXEAc;048JO1yHtL0SDt3m=|QfRSTQgUmiOS@^D4LZ>JGBzDS(aybpE#rWs{=RR`$$
z9CF*CY=!-_qb8nHQNh_9KF4L({v@9CyU&vy`}kaRQ}~@3*|I--vlUPQ2ASEU^+3k!
z{@(bHDge4dgiJd9{s@w<s2dOri2+BR-9!-3P&JBmSC6buO(;f*gyo~Af4vjvJGZOY
zX*zgI!u5G9R5m$T<mU?&(H2299^6EoY8$<pb1D%4u$I$LW*R+k8B9<}vm4h&3Md^m
z>(-0Q$K>~ZH8eD!jKwkQEPH*_8I(8xbXi-K(V1qGr<))5#btVpZy*fW^YtCVsZ`y2
zfQ^qUa#2d2!{Au+>tHc%WxD?Okt)Y2wxA!f4?4B*$=og5dNac3^S?gm*p!X+9#`&!
z;zUT>%M{&;m1qecD74&hjc>qfqPUGiMqibSiJHKzu(#JTJG8*KbglnD@k~#byzlYN
z{ZftsBsI#Z=FK0nDAOWG=T7X49?p1dJ8w@`_g9qz26?slp?|Fn3i4~*^1&~<1qRJc
z9{s68N1sn$AMf{z7#J#DM4{kcpN%y)T>4y{!e(xlnuVnTb}-S_;Ja2%{r!T2N~<yV
z?uNZCrWx<;x^<Rqw|>&{s8-dn{U4)(J6yP5dRwjgD}hO9?dBQGtA_bpcRoU9Dom<^
zyZZW$oWNUS3n=)f(K;UARCqyU_P?G%1JJQIG)cLwSM*B_{h)%&eobz{wkSB?YIo+f
zWbi2|xAg(N-Dtf>a{(BN&fr4L0RbO?gz|Qs(Sp;3#ov7lu7L-&3Wn`HKqSDHS#T<{
zIFe0FpuZ~-Fxbx`%Lttp0At@7oTt(b@wZ<eE!uz3+0*0X10cFh^wdPpp+6SOjy}`s
z+taOtShnoG)toeal^4bT9lZ*C-SDQe<cPK<%=YmnP>BU`>w2y1akULx=h1BWqNl3O
z`%`C<(Kjb5_w|_=KxL3gZYUtr4W_-UR|3#2Yf4W#8XEGwb?^Q=As4?maX$-9N+;}R
zH@PVTSCR=$J}I@1_Tzkl)lygN^zzrnS`9!}E8Ir4%-trR%gES6Hz!8wRlcH{TsYq+
zie%gLh|=*`1}JLO)sw{2f7Lak`VEy{zQsg|%FOX<K$?<E3X5#k`@9J1ZAQ~o)<Lme
z{NmFaA@i!>J!@<=HPMzuU*5IcPYs8?vV{BN7ey}k@8X=|3DK8@<Lwj9Uv8P~FZVV=
z5{feA4KsJ{brIs5Jt3f#FEx2us}Zhe5oDhA4%S3G1h}wuz0gj_z8VjIv#d(kDKtJl
zenr2|$-<(&_9kB9>#GFBC_Ifb@c!8BU*_*@OjLGz{d&vuz*QMS5k6Y1H+ebb*k6lf
zJF;joD@fJ^w}oU%6v)ea!%vs0$D_XOm(Ffn8w?sbpu`8Ar3RIyjsc!ht$A;v`p100
zi$tM=8|KHzRXK_g6hc#Ggc9~0fJ+OzM61g)nP7MBY29hQ+_ZJMiz|F$Xg{*)_6u;d
zb5ZuWAzU_7KQDm*)d$UEI~Mmng8O}sj3&OPO-&qJ0(XRHyuvLvMPzQ#o}+n<QoyAU
zDkVGL1%yCc{k8)zjgws_ENsdhmNTOjKP#FgWwiv+K?8HrE=p*oiIfBY6|hGedNj<1
z3scB2BWPe;+YyXpg>+|&(s~Fl#i$mBU|J>+KFK0C8wu(pd@q7wv}{$C$pJ}Fzu!bg
zTM}kI5_-zhgMfX$&8WKBk6<Dyh|vLzVBhlTD2+nRN3gFY$wBLugP9@8w==VjmIQrc
z_q(n6Cf9O-D3$}oUYvH09rAu!T@p9s^JS7tvcYfx;MTtm>pWF&VrkeB7ho~>pyrz#
z`R?|E875Yn9QgfcJ5u(wV!Ru_O!ZPB&ij2PPd>32aT?F_6>C+~Nf|jg)pE1mMZxe7
z`%G@LjJIot&kom@team>PgGbuHt+tZB0Xn4YYIvE67B5ZkDoQVD~-IbXQ!e1eY{Gh
zuhoUGfI9So<>V8qPi2`=D>@$&@5b?%XHIe~zJo?k%tQmhM!B=R7ad*L<<;4qZQoPt
ztD;0<1wHd~W`$(lvYy!`mD{p62C!c|0T7>dtWL(ffy(<2=*c4KJyZHI0%gCT4kt4g
zSmSkQ#ZsA91}TY!d=4LzdCp2_8;JcZehdg+i35(bsYNLNjcUQT&mbzA@%(`zg8c~x
zU(LG7_dH|FI#u7t2W5epkHO(Q)saSzV_*3Nz_}X>zW3PLpzlb@#aTs^KjXz8jqMjt
z2yC&uIyK~Gi>^R;$(7m_8;o4Olra8m<K$Na?h0PY%L#7K{9-!BD-p!tV-h_0xZlKS
zbu*<WN~*6UVE=yp{CQ7rp83x$`-g4-BnAjx$wj%`SO`=aU#KIwJr0vl2fxaGCM(mt
z`&V2PkNy*IK8}_bh#{~)tNn}Go<JsGzMhcguL$i8C*ZF>hQmXX7}8w-KK5oOcsqK&
zzJc2sF_M2QF2soMKj!n}xB~=(ACrwB4Ey&FeSrdLnkJnW_<!%<uSvV;0X~M?sUg^V
ze?5m!KMPnECpJ<t|7FMjpE&;yhy!Ak*s36OFyD@GIFhUFOXFK$lEaK$k}YHaw=b*c
zc2UAk4gOkoP6=SRCn2Ipe0+rreYbbi`@v)E{?cXw?Xe$?gL$E2S6vv(w$OSMQ`URk
zSZ#*l2~3BPm4X_^X=<>4HL5M*44@<6s<h1rnSVpP{;YF-sxw=iU6X>(YA3JGI~ao(
z<j%d@wS?{&z{uK1Nb_pe==i^e@XoHQ{3U)F;(#NkHua1a0lr1~&>V2-p_koVddG!R
z38dZQnw=whori?8E6>__E|SVeRI{S690?J{IO7>G&+wEit^Xfe&sX#zssk<~bTe8J
zw7^Hl+E&c*RQZUpxQ#^7Z~?uYr>PxNKJ!E4Bg-4i0{Mzc0_nZr;K?=+!kyh|7M-b5
ziY?!G2F&7q&ErkXT&F_9I3I6L2sYm_8qT0cL=(wG02rRvn}q*yyhV}x9PZSVS23CI
zW_3+oN%86I*&fzRnA4{p>NwDh9ND&3+?c3<`D*kC$b~+!xfOtLgF(C~ZXVuheHZh{
z{RA&Y$Kx;XF*j$yx|K7g5Px?BR)Fxb0Yqsz_{R1GK^QKW4fiA$FTBea`*xv2SCIbL
zkAV-A#xL-%An~Jru{Q=7EuN<|i!dds49I+Gx=Zg+InsS!iD$QWOU1MN_M$`W#So|n
zA&TNF`W%3Uz?if?1ma;-`0DV<m4LU-h;B{4pPjsZm4C2YeR$Xom5WW(?%duS0wGcl
z6$~)au5lFsR{cKS0d`*CFedzPSom@v0rOe7yV}~g#@U4kPrhZ4`NhB!$X~m_fbRk?
ztr&3^nZ8<t@5>MBL|yJhR_)(@{7s-<J}dh<CbNOEXRFn%K}v*77$63AfRLkhB<LfM
z1|wf|<{fM7<vNJY&)G)#oY~e~3hl9VQn4$I%SGCPWs1fOOK!xC_#vS@0^H7fRIYUh
znJADl%2rQrSL%TN)&{xX8Wm{s0crS2IS<&FndT^-73|RHd!Y0Qf%YIgP2dvy9cDQr
zaMS<EoW{|VYHam>66jc`X-gLCnX7h00iS3{O0$D-onQ`P@(ali;~ub&M?9fM2$DV*
zCQ7~hc8802K0)M^S`fbh;TfJ5AOlqE-t5{4YlaeipULV;OOzY04`U@T1LN=z8wI6l
z>UB11JP+Y&vF5&e8}UNQ!d>ntoG}Cf27~;Pv&NX9mss~W7dX%!UnT2#wywwg*gFkQ
zV7i4sBqU_>U+UG4NC=pKgz}zwPEjYlaJ7gUswgR+DfksbKn(03<<9cHAfucr5HWXv
zN(Dz1$tha&kQ;W0cB1gf6~EYe6DP-VpBCIr9)99Tv5S3`9jL+jy0efN9Z?EnXaP0&
z^ilDR`2oUPM%mg(={ie)ACvT8`El$8VR$LQ+mhVOUt^uInw%T3UL*v06ZK_I`m0z0
zZx^MufKt6LRG`J1!W;Y91>a=gas++T>ZXwh6#XUw1YG}rD{}xwYH?0<c^SUY?0<~+
z6@$|VoW(7>R>oEc>zhL$ujm{ua+UQ$(vAP?D~gI$gUjL)rB#i&{t%TB;I)%Vzm!8(
z)sg4oU+~$Y1`sTMhWO3v8-n?=qVO?ot8V3=dohqiarogpi<fEVsRv_9PZLF*4lp3n
z>jbFYXn``xA_|cU%6Pd4gKhsoDQVn8Wj6nN*=aSFLa{$L>GnVT#jO@iU~%GEvNs4I
zC>ny|_OR=sDL*Do-MCCy_pSz8OCK|fOy}hh|ND>_W%*cPL~4JL1}=%P_=~$YBIsvF
z@O@H9VZ8YOv)Tv^Vk&tRLg$i_59pyqk#-LR!N$Yj1O^+qND$*79I5HfVzI|FzV54e
ztKE+=SZiIuo}9A0%MaUm>ey!=&LAArSF{EbD50qqXmLg0#?A|n`RZYp-nD;3)-8i4
zY5?ELPu`3%q%ZCJrBMt0g;xpiP))+SP=sqAr~&er5nb9mVXO!^DL+<5f82uyKums{
zB0+15zt~^H2LOo)b(w#HuzMrcNUp}%soQaHJd@LLo)IH~E#+tqb@3gr&=JG~38fug
zLNN1&33260(b0-C2XmV{tHiC3h>ehRI}c->THZuz|KLMd0VC!Ka7A8H79&ssuL>n$
z798zx^v2qA9zVPk2~$0G%2w(7lNI6{z96H@0#@;hx_dMT)WVSS>#-%EhT8a*W!<IU
zVW+2bxl)o<z?22ILl;89d!2mk2CzGb%Ll@%TfW;O-$KxlAFYA))g$*f?x^8NiBkPa
zLXp7e4XU|MlNLjf*tGhW#CrnxG*3YA8gUaMeuWH`OMNr<yl-KB4b}~*j2VV{V7O<?
zAxg`RI$(^KsKg*x$e<q7&)(LI(6dA~QuSAFe&zp=lxYM9Wjnh61#15zK@L<`D1p{t
zHV)h8G#BU({O{G~JE;$rl}r5K)I%VvuWt#!Vb9nMG6Vt`(MC2J*6dYz;+cKA(`&cz
zjySu`W&8hfP4jlWF1<^;6YM9bk3((UbMO;I?)#^%{$-hY0xsBm!Tlvf0&)6H^?r&i
z-^uvLIir6ZiG-Fwjb8ERfd8V93_u|z`TOEGo&)Ftb;^P%=5T0Wo~INqoIY>Np_Y*w
zSFGO&0ilslngAzeM?KU5Q5@4nM~B@V%psVq@6l|Z7iT_>88&y^;m@Q)T#`N-?u?ba
zz`T(&za6~qsXnQDRo-M20hj~y3u~!4ZAX-Ea-oR&7zx<`?#(bt@gYptq71U#CpKjb
z-<-k>fk-nI&wOWV5V(slIWp=VaC5$wd_@EjDg}^v8b{9P`#p}wkj1Nqwz{9A&aI>I
z^G|Yww8d_Sf#l~42Ml6nY((`!=C?76&SX1zGUxQbtg&r`PbY+27`yxhI~l^x@aSa$
zv46psj3|BnsCzE*%0~^HRIl2uNG*+XSnq?^ovyi_%OTJagvSP;9P+0uBZTSvY34LW
z4@Mm*bx8_wS~DpBi4q&(?Sv108Q~$0R{jhSILfOg^hQ4brt+eVrygHeNqBJCnN=ia
zk%6k`CDX+S+2&vH<O>iOl{kJG0k1JaeWQo@t=#2mhsIE|>Xo4j`9IpU$ma_&?4z_D
z{~G-gZjXnbx6vaEr&kBru6J~$`sn8^Wu%K5pGVvWL!UpDf6a$56ps_QG{(ysoCsls
z=Xmjaj@dvZM|NL|K!I2QdXUiK#yB0lZ9i_19%~e1o4SC0$aJ>UVB|w4gu8~p3)jgc
zh?74<0ZL;k;iW+W89;GGfm|F5H}AX~WH3`E<dZX{W@cepy{y|qnh#+=!|WMk`HzTd
z87o76GO*bpklz@963N0Dx8=8gsEYRiWp#BmABaP8aD?K}$6J{l2GWTd%}_cIzi_D4
zojF}B_R;5)v)%E2m(7!8C6wtk!U`S<Wyn`yAHQnvxa2y1mB{3XAZ(xsz><}mBsUel
zVdp(oyDo;D6(D!HYe7+IITEF%ZiGT`8nNM)!(iLdc9BDM<)vHXcg8+X(<lN-hqBP(
zsIv2QVk%$UOWtWnG<;3>Y9P~wOZ~a*&geWgBB27|ZrH0SM5y&MMuX=!ryaVP+x3ri
z*(P}s3?;)JFbn2fx%bXw6^;un3D<0tINrPHi**l=s#Ay)NUx)~T#+uCZW_xOAI{Yk
zJ5HGecZTc=$L8DwM*)hVDgukmSzA23x%9|sXgv=URI}v=+Cm7pyf0jVO0(j&I&yxe
zQqy&y2*sKYdd6tlzxD_^T4mfV*@6aEA*XRkdjx)M=2Bx-wK-ryc}-nCskgRMBGi~P
z!IP(B$#(~V@4ljQaA}^2r$B&3dKJUNwUINR!USSyw71yFar)Sk4~X!Y2Ru;ES7WHG
z@z{Q1+AE~hHEv)zUiu_bSxY9RIkv#^7^Rhz$2mRwiLUCL?mTWvB6)hjZGOxq#Hghx
z=n25Tz;=is!gVYIMaITvSqx>n1>gaKcc?f?oyF}2LdPM?9a0offLHp>xWUM`o%@s0
zC4wvuRt98cW%rU94@Vkpt3JbUVgx+Q^GsXwe3{C!@dcD(3S_G#-o$=LDDs$1%J8yC
z2-4-Giy|O6_=4u8&dindeLWd5^5i!MuNZFP6S`4RZIQm??w!`HFEzwHfnr5|AiV)S
z0&-GAzR0(%krswkmDH!}d?3tEZKinN60^7Hj5gHHzac%?_PN%vx3@>@JKy{2WmonY
zb`U4v7|q%)p+Rd~$;m_a25)u|gT^I+8f>|4+_AU6O?o(*=_aMG6dW-$-cdO$vzf%N
zuefZ5gK(+vFlqs)ikwA3uypVgP-siOvNm6207?C+t4knZi<WKfAQDfAw_G9}K@l3k
z%0IpbMC#RY6v=H<13M@@<g2G$bo{DtjspZt0vsOPRh>y)4btIywY0C%!32vt3l&)t
zq&ebM$JBf0?2jKCwt2m}%UNM<@->z+k0FFV{yk*<X|B4|<MS%;P~Au<BmnE?{$YUd
z$N(_@YM)K?XXAeK51z}(4FUCB2kk@#KbwJ=sjIn8t%K>|_UtS=sK-TKn!+vfSlhxD
z$o$a~{BQ-b0!~13F-~KudVR}!>B)5t;2>}6oPy#)3!GCv3v+koU+bdbr+p0M-i-ax
zSiO+ME%NfIjtk|F07`jPMVr*d2QX`|!~T8uby4>Z-xTydZk`XmtZH2w$}a2M8c-pO
z!pE@)AoxG+n&4-R{U>bU_*gtMrtI+WyW4k3d29-SGOwYl!_CR#(mf$pQu`Ufz3-3L
zhV!C(nciPFh|It?FSG$?ELOlPOC*sAN?LqJnjw88EVRHabTG|J(rxnU&YCMpM1kHI
zru>15c+QrCdhQBe(Jp74<w7#y(Y_3MqUEb??jtW3#bVlv&$_ccI$2W<g6awiRbzGB
z$3parn}IxtJ6E;==)FXnd4m*wn)l*4&EG+pvTGe-?l}+#ey8o3Y0|(~W5p@Q+M3Os
z-O7Dy(y{f+_6#K}i$9PleRMxZvo3JcJZ+rTy1y%pvR23{{NgA)uN|5apBw0s60=g6
z7;N=Zl6J>#wkFRdw(U}pYW+T4y7QIEQ1K-Rr>UQY&5Ws1wS}6_<Fg9YB1RuHuDv*z
z5F&h{!r{|U*W$)f1>He)aCGE?Ogw!{DKc}FfI5=;n2T$KHj|s_k)p_@Q_}WVN9I9&
zg^cy7N+kwpYk|s^flT!+{)iK`cAgs9-M6+Mm}=)jUr*6s#jr0sr`0J@aLZzD3Rr!e
zfN+Bb^2YB;jdCW5URJoK2q!d2S|p5HYS!m(k+^%lH|6`wHuHlA5s1OsHzsYcvfsgh
zlh@_p7&nqpmB~cqdW6A9(&})Dfq{97^9Mc5ff54|8ag`Zl_y~oRRSJCd)y7#fB^~M
zeZ`9SlHj^5B=5wXh_U!<tOLy`<7Y0Y*_%O%vz&6vUl9#>QV$%W{OE9nXKiNH77vXT
zx0L#arA3%Ewl&@kXL$GiT?-BL55Qw)c=;7Kd&YZ#Lc3y!uaXptM@wuJrRMSO1GA3h
z&fHi2^Nn)TDETDR&N922<K7orqqs5ER-dQNVKQ6z{<moSY8Dprr)2E86xBSZjU3ch
zRa7jtD-Xb_8Z&2HGySKtHBqWDo?lG?2#CMPar-OeV+3zw=&{9Xy)n=*!oR%1CIw&+
zC|9hU<XBo<{P|A)%h$yKHe?KfVBY2U70;DM6NAK2>&eGy_=2zv^_kSPWOV|$<YrTc
z9OgJJh4iUgDt%WXHp=3r3{6pbhjaO^6}ovHygo(u!9{T~%7>-4#-r5)F_)KPLBo#f
zdD$^xB^p!r(5l_|GeL9+no1P30oNwe%toR&x5E!#v)-jh_;iuL>teSf5ipRZY@)S9
zWu2m^k2fdB0POhyV9&#8-1;0=trA>s%}fG;rz2V&Z_K~YKo1>E8+JYYO2l=i-DN4f
zLmZZ`3NxdXpeW>M(O@F_p4)!_4!C+40&qtgDo1>y#(hQ5;6(}8;ytNClao&Ep@dv^
zbE2eyY%+NT>IKV<4~lGYUcR*GIo{{89eE%iMG($gv_2q~w&jz|o;;YBu%=O<<~iFJ
z8XJH+O>1gC$+6q3{S%VNA>s<IZD!RFukVwka4Aa`stC0WK=g7!cp89B*RtlqMtrkF
zAB>Bu^KBQxKrKvm+tldmx&BWoS+X2ogZBFb`q3ZDD^qE_du>&{I^Ef9VW6E2?SdUz
z7DWqi?|b8nCUpp3y&!ej8mkr3$+NAeWn{^W-0hR=(rZ$~B(GyUox)0HPG5+vc8=!I
z_t4l1ccP*2$%s-E5}0e%Z#GFI8O_Jv@m@i65?;aENH(GIr+;$p>{O*dRjn<a@mGJe
z87&}nF9okneRKC=hN_Z~I#6{!V^y|8%R~pJyspqxB`OwrSqF!jw5qqbe3UB$N)c)4
zrlimb8*Q$_q8D_|%9X<s+j2)0wvf3;dDjooG{hH+&-5N{H*74Hepeq%kce@e!bB$B
z+gKt!T+Vb`8~>b+{V_9}8m*9(<fiyX6f^;tM3U#^X5PPLO7uQfp=O!*cJp=9B~YXO
z9-0l9USl7{0<7_jfr>L(+0fpg@3a<&I(@ANMIJcrkiF*GlQx{S@qD8zRWY~gmOw4-
zQnRG$lF-f&l7XC}I&!^PkG>}MkZ)m~1n>9Ptxdh3PM|ZYJk!lS8QsJhJsisE+y}2m
zugb+lC*{a0vKa@L=VwZHqkCJXm?4ONFf$;3WzNkBf3kl-Yz-)V7z(2FeieY{xZJbb
zmjk4F;Q#@_TYzV)L-HGmOg0o!qYFI%&2wgawB6Nx{@5P+0yPeIWdjx2KkdtTWeMz6
zA8@3*c?Fyw?$K4KTFNdw8VlurFSJBzs&Llwk_P;`yBMt7N-IUkbX*Hj6gA%%M$@h{
z3W5|!T_%!@SL=21v2{%|VIx?YcwT^fW^AW&JpL`8pf%Jg>+y^T;j{1PkIZ`FA64&v
znI<z6pd|<-7WS+;JKo;Dh!=jew(MO)FyY81QR2-dT~M{bQn@yZ?xNmdCLN>QH;XoN
zbp-C!txErH);p>|#K*)l@ibp=@vw!Cv!l5rd3<OoUsHwwd@W(m<y8Q#AvX}dZm|9&
z>`>J_k(Qblc*2`9QCo}dVqr6Tu9C*_o8f>l-c|=#U7FYEY(9VXE05-~HI)ai2tt39
zt(dZQF(}gi?kN|Sx_Pqw8*K5rKxYe3k!Hjn@Z)Ybs+Ac(Zr?SJRf5Z=601EPPqJHK
zKkk*B^$Htn;t2Xs=y!|v%?VAAq?IZCqL>)GwCjWP?UZe)Z}DTX_15ooxh(TqMtC}7
zwO4;Qc=JbpNPtw^omq79)3MQ-$4cgxPpb)3S}S)e3H`W>pl}D60JL`BX6F2Z)>Mjf
zKx51q(2VoctcO@3@u>_SZIG{H{Ko)HxQ#<A<_tj8V;h^V6n0h7`*R+2*M8%5or03A
zYc+0mxRt!0&0fTzxc3!uoE(j*XB`bR=wnYix!|^U)XK6X#bzDwq^iz*b9R}`z7wYh
zv;KAeXTu{V?(M;!3|JKwW8MN)zusAV+;^|8a-)@{cmhW7d4u)fvg@&(@Fxg|_W-4~
zwwC*Z&1FN=>ufcLLk!a~elxCzTfX%T_7HH2%_Dgu_BjmFK+I?H9|M6xIgr%VucJqe
zBbhdp7difWSLJDOtK9}4h$iE-2<;s@%=okRU&zWFKuB4R{VrvFmMj>6UKTL#DfQYg
zQ<ayG-goCcM^8@MELvX25NNH_HJX=cRq_Ht01nA71Hgj&n=M+ug#d`*W5J=lIM5X+
zOa~fyxF?gk-Et?(&6V_kVto>Rr(F3A%XfR24S=JmEpSQ<03j6t2HPZ`n}gky&;=@2
zu2z;Oc}#j0W@LC;?LlPX(N(o&uGq9r%<5O~8623$qGu3Fg7iY9m;jy2hFm!l4cu!R
zHbtzn^!<_sci<Ust`{m$7!^wBzTJ3i4KhC6YxaILo3G9Us2#sxT7^VZh~U7?z#z83
z!FIn{zPL&TMeT~Dy1e4>o0Ri|)$lCs8kZ!BhL34$ynJ~6UY^I3mR={nGEk=Z?NQCG
zvUvrvNn)Aq*L?d+_XXns{2Q>ROhEfBe?YWUz-6N}B(L{_Jg0fz!Mk#xREL|c?fd)3
zBS3NPw%g2!^q0=+LTTX(vGuC$5k4X9@y&?6>IUmyiuIN)s0{vI7p1RKzR4%p)<bmb
z^P?N;D!p|Z6wAG5>7&EuqoM`SJ)R!~{Ru4qfnPD84f7uA%ep@-Ggbfz2<QMkL|Jf$
zQ6Zj9Y`L6|5nNmdlpA&wzu{fYjJ^-Mph|trhEg79J4`y;qk1UI;4&hX5gqqP_gz{F
z<WtYl3v)_ncwJ>KYxHM#*wxc%PM1)J{XHj`TSq}ULGIk2#Q-GsTY;y<^p2y)|M21w
zVLXp*Dz<n$Bmw9;a*SUb&PjF%h1c0&lRSM+$XH6Q^wC!3U3A0Q%yIV5DRiQ=pJGBO
z4R`0AHsrc1CZ%cGi0(f3x)?H6ICG8#G+S0PE<a21n;uOB;1~UZ3kbl4@AE(IjOY_A
z>*+dm06oh|;ObOA)MayG6=+iWFdmg>qZf7QZsfo7E&9{^cam7=i0GOL>acf#kC{1j
zpzl(j?7O!nbmYH{4i_Mau)rEy%XQ~cYc^YYl}zD?5cB*9Pu#T++9CgOIzICH_4LSl
z0VRao7`umK226Bx9S@3t2BBx+#OXi*GEgwA*sc!qL_*{E?64tmdF!E8-O+)K<t+7Q
ze=)Pyfm#ek6r;03PaOOinq~m;JX$rPs}35^$@c;7?k62An%4s^+D&}{i1E`HDZr)B
z8qN*)EeCD6UrL&<@_A-lCCucqf&L4f*KKW*Dc)zR+!*!~7CD`s%NQwNP7>s}?ECJJ
zd%eV-=BP1WL09>f+Uuf98yDEgaPQruLae>^j3uq%%dLU?=COg3n;N}cgj^X)I+YOJ
z4xKlMzL#VHpcq~?miJ)&ro&G3msBAScr1W4I9>o+Ts;Cr<0hFCdO|VEIs6|Vpm^Yz
zt!%N+G69y%HQAt!!ny|5<;N~If30TQ;iTy0JZ;NR3X*Z9EBWu6Gu2=|M>EE<Sq4+$
zgzNmm@hWMP>?c3YHa#Gp^b*I0e#}5Nk}t=wnEx12fb*>nH~rSV*+_n)6cA+TVXX#w
zAo6);Jyj`vF3LDKI5ui5M++?+#dTfAO*#0ecw6KQ503W_e=Rt#d8!HNIa|i<kW3kc
zf7^`l8Xg~gRkUAvwTGNCvbpjY_mc~z)c#(~)yCtl$ub2QI&8ItkXv%}WG8ZNea+=?
zTg6A@y0~Y9EQ^}v)s8*$NDkyzaOe3wR~q-5+Y(F*M+_&P1gzhPp{(eqki@+@4V@hU
z`bM${nKXWm%8M_;dkASU?15fCCO*E}<VIy7Je1oh$@i{3j|G-^ai_OMnoHKy<gg`q
z#H<_-M}=InI@k}b&MNz-vch86kDNR0WrfWH@$1N7V@KVCKeukJlJG#@d+Ri|YuKvf
zBtAQ`DtS|xACtwm_UF<9M67}n@o1Ed?GF)oQD`3>24WIySOx>378O7_Rs8}ypS37C
zmVvA|{rqolNpG989jLJQrJN5{dJZX)L*)Y;DhW1Y9nCItd-{cA78YV021Y_f>P$7|
z$ALx&!xmKfg=EpWNc|#bm7r`Rc5ZZpoUpG5;LMyzyyyOX)BP~+YgXH&==)qclLr56
za4Sy{Q&EWm+DTu6CGHZleR0z#CGz7m`Hm685_#AQ>q_O9VKHx4>LT<3>o%Tcf$xZ8
zmODp}w^;1?kJK$~Q1IvLzXcu05+Rs5sPM)EyW~~0-@=TsbUokl`+osoA(x+hE%bK<
zifFm)7bMNc5(Nc{fgZj|P@5Eyo}~X;X^rY#syP(Jt2aFn&zNA<28;PB>V(C0GR2l7
zw`Ph;$Tx%|)7N5EJed^{6B7S3pB<z0W$_I>2DC8?L>eI`v_x%@k0hK%)WNGjVH~T$
z?crx2e6(aFy>9}n4A2pW_L01tn1U-7u<YyS$(SaH<~tQUK->l~j{JShFY{ROBe=7k
z`)~EIcTZty9*z<fuLi`y`fgdz-jZY`a$P2|o$N@<V55pN&GkJ8f3xbRw_P`D=MmCo
zh|_)C-Vx-Ye1K5dDjEZ%b%F;;yDR?1D3A*pyqj`gJpKZ&09;aq7ZMUeBIsJGhYgPy
z00ot`K(EJ6kgvzKO;+<>#nkx`W4ad+4Q}J}J`sGxB&+C`emJh*m392RchOu_pk+9Z
zNZ(bA((B+oQ0%wCmdIt5%~V_?9;fe~2Lz!A^3w-cQGg6-wc95SeV=E=!(FnqXOu@=
z8@K;T;RnMLCih%giCKQ%zeoen<Hr)ezP&B?3kIS_6LPzKE}U=|thnjS5}TOKcLeYB
zt80WPCQ7Xpn4wr>GZ^bxk7k{iBiLqihhQS#%+$d!#xATj@HZBN@Qi^q9yK95{Rtw(
zNPV$xKLBFMlSP0jJYkxYt5;veQoq3gl;!%UetPz{Yw(FPo+zjV73hxMdMF?@c|@V+
zVBFBRGAd37eU)9qTa_GIs?%xr+jjJehbw)8>I9j8#cM^MV#ME$pp~f{|FGK4+8iUJ
z4SBaUUB3?WLyVLfwZ<eTlVPix;$YlonsX_>`0V)!C)2#Z;GDMoLs%Bp`5?#Pd_>q&
zk5gKKP4vqAS1)IeEVZk(pRlvt<N*CJ;7cO&ggpAaq&3zC&^QclAJ=zZrC~N`nljGJ
z%>0QIh$-o`KFVC>XKZ~md7N#r{26;~HZMU1p8$Ac5J2(6A3-AY43G42-)-br<}t9F
zD^CRy6-^e~s}j9BZ+cp<K8Xn-v9OEK$dA15F&n)Q&ijXwAUth=xao&E(BBC4QPU5=
z%#~VLYWdgTTeoq3Isgs8RXQ9AMF~LDuDZ1zi)6`{b%Xv9RwnN85#-hgThc}XW+OAe
zlj>NOUm2-;&$2ew6c|+oWe}hP6L@PC+b$QW6qkqQ%eKG77HE1~$1TEf#1_x~TVQ-e
zf5J0;^*SxSzeRr(jaDhO5D=PYK&|+WO}Epa4BaXakFM^_|Fy7SoI-B~Yc`vt5IWk8
zkhQ~)6S?17a7(@^@O&mIUocw4=meAh^RuP+OU3&<y9)J3RvR&B+GVE^WSkI^k<nn`
zO|=5q_Jw>qir?@Am|XwkAADqCy?ps{rSW)XsJvge;B+50dB;pA8;hkz6fg?lt-@gU
z$1%a~emURze{7RN*M|FgQ;J-*pIv&&CC;6TJQ+UTAIpXO;;*|3-=ZOWJ8nfg`%Niq
zY!&W*!yT_`_mdWa6yFD8z44bSZ1%sY4{8JMs<`K^mEW5))-eQn{<(SDm`WNlHYclf
z!hxQ*)kLe}ff2AlZ<T!6<sm)WIXvKU%4){IMI*DE?obFr<BfcuB_OS&@l500mHnu_
z<+v|%&mB*rJ+Gf>saX-;+L_CTmMekWZ}<PM^8E@gDb}@h|KsWvewnIu+^mwP^xW}-
z<3ug+&j%((@BrsY)!3Ly1;Z6>s&7;h*PBg&6uLF$)~K_-h|}9JW_;S)#>e#3ds&4m
zUr!x+S4h7YWVi^KENrMOi2YGyj_3Xf>-Uj?a88CCpfH@KmCSw{U`9_2DH!2x7VdkU
zbf$Zf@sDRZ(b!yf3`v85ei$fmL~8nL6~nW^L-*tQS+#NpDIq&CgQ}_1eY1GfK4`?H
zem(BXmBFhNC+n~?<VJ<9gQr;{hrUw}S7j-LWicTFXcZzPi2b4ep8vyhK`}#X`&njk
z2&82!1~Q~vJb^6L0MPqrMDV}p`tEov+c)mqMppI;*|M?<QO2FUR|wgX>=D^3LPms;
zy~&onvS*Z$m25Jz_k7RGsOR^-pZBll!;|~IuH!uB_xK*ise0eCi-=j@p}+A3e5BH;
z$|cBYA%P;oZ&jbmKGApcNnyro{M<$2bv?P=knj)OHZ>8UNf9#ns;=RwuA~hA<x+sU
znyR7vFPw!M3Z;C{Y(@C@e!ct&BJ-bg(0zTy-TdK+4ikRFBs_`Bt6)C=hllYUB@yGN
zDG$9%Sd~l+(KDBy&-IE>9w*&+xVQK#A&)ER@gyRWMxsaV%E$iMY(*ueCnq8{rOUqf
zXoml7dYXeW)aOs0zTRbA!1)A&mLwcdk2}uXxW#K-FP{py{#A34PetARC2Mn%Z#rl`
zFO&Oau5%{1pk8XK{*Vh1ZA%C8H1mRI8Urk<J+$(TYZVpOjKi&8Q{7yv{q(D4Yc5+y
z<UDwH_7dv-Dq@!_%~rwOdiZ_)Dj`8YqLBl~pEkzXdNrx2&!UH?8heZbMJHQZ+ZEpH
zm?V0LxF<(@*rPW03fBpv*T34Htj!zo;ON8f$XhU=^3wj{p`_}kXf0Q251o#0YKO7k
zS*%V-TL!GM(4!_8>FE!O{X$DiUybTOZsS3|5#kLSwfM9Q(Y6|46zR6B*mw=51013b
z`uN{8&Iq3aEt}G1g%bcB9Xz4Y(OFYXexq>FquoDW2N_FotdVTvj&b{m@&<Y(2349g
zdv`F7UsN=#mUwqM&Y8^p`IWzKsL5h9`@uHSSoHWu2Ed2+GT1^pbd@{M#4#5KGzH=&
zk^XlG9Z-LC9xYuGAK<d+(QeB^M=SRP4i{2EoQ~w?;G>qc^am>flx{zP{#3Ni6HWYC
z&7oB#Hie>ad4jZ{i=&FH^Js!M=`s?1q~);we1D3&T=2@MynT*pK)?Yf+F$GQ>H*{d
z)ZmA;S7FEpJvuKMjD86SobJzP);Jc$i-GYYbDU`q`&2;$ptXCBNrr&b$sAhlo>%PF
z|1zvN_;FuHCJR(wSTL_Foj_pNp-IthrZF?MpnT>l%X}~Gn9)7``|2qFiR!yhZhT6_
z3BP<4)ay{Eq!z3W;F0iw;w?ypM=d@=VO=##k(%7Mg9L<hQCOu65u@P>na6B+#U`b)
ztE<2G^qHlc@{U3Vd#{4bV21{I#)AQ6`$vhdOh=L;&mN|V{d?CeBH-T!%A-0iD;+QZ
z2-Z)e66?AuffXFNV9$YNtl3yqsb%#a9?*joMnglJ`|}!%2jb9!?<HpXGSaIQ{;O);
zLZi|ts#R7a351tXnYLRqbhWQGWmNLveanq%{~h|&1e)E;7I)bX@**JyKE)4e^@Gg>
zGC#|$5(jv9hYjtI{*;^bBs~J;1x~i+ZCp4N5wh*pufyl#(sKJom2y??qm{KkN-Rs&
z)AYXm(^MmnK`5bcWnZc)d>b8<>jRcwq0DF*U|X+JfW?%1`MaMSimf$&ya^(%H3{tD
zZo0|F;hzV|ax+a?V*eq2s6$x#^2D{@SJ{D@9vHNAo_@=)4UWEJHjtz3qjAC~XtT&z
zm3?vI0p(x8UPV&IYEZ|19#z7>{6z{}5ByVS&@!=TTmof%H6et#Sx}{b8CWF_k&OT0
z*?|{>J3`7Ba#YWFl?t@OK=M178X}ktMn5G+Mn(bx1N*^lip=WODjAE>vKyN(7O+5P
zi{6@p_Ni{ScJk@1*iX_evv7Nn>VJ|#rJD2xW5kr9j91k6UfM!~7IXnH08e`Fc@S_i
z1eBBo&>$l;VG8Zr5x_UPjZ41{+Ucrw&y4`WMxWAp3(ebYCe|sY);0C8#3C2h?Wb5{
z=uO(cnankq5(Ji-TCw6^Mo_O~*mJRFuu3SaH&p^S4PoKe;_t{!5OKS~u$^wi9x9Tm
zo|R_CDye`5U!{|?^)sOXz*5ONk#I4Fy3i*x8PaTvU$&02EK8+(1?LLmCXTtEs)B$=
z=7{kX;V$Ff6YxrcB!_-4{z@6LkOm8M3;ZaYC83ORET#jA1}aP*u2zVE`uN5`;Q}#2
zL4EYv{IXxm@{fZo=c=V2Y~x!s8;SJ*Xyvl#)wDN%<#WnYm~%$TCL&X5rY5cNF~F*~
z3bT~9xbt*d7J{(uetx+ECJXJ%lGA+l+v6}(|J6*4Q9u=JH<Yf3CylD-+5}c*Y=caW
z1uW9B?0%*u3MTag0ED1XffZ}h5)E(AUE8yFuT3O1gf)Cz6)Z+K>e}nHqwt?_mRH~|
zctXp2i+G9NYWa40-FBLm{I{5huU3j$C*p6UMdu~({Yivge#*F`&Qd;V@kBakp<m5{
zZ0PMbw0kW7#AE<ruQE-@m%s5B^#bgdB{0|64<%TjHy@kpyuUtGwKwiEkRlc+8HZ>1
z&cG{*;WS0}i3N4tBb?O@;$n}X`C@m7vatP~y4JhLuvhz;`&~7@mmZNh%0{Xchqo)U
zhwfDA5hQ3=TNH>Zd@R@*-TOK6QT0*NJF8TXp)2t|&G_dD6(fM8QdD@aP)J;cLc=yJ
zfsJN-Xyp7uWaRCruj$kxz6)QDRv#}978P?#H;+}2P^@=$S8q3aWU}ya>R<=+E2%Qy
zZYva@%uM(gznmKueWzG+y7O^Wmduv3*IG#y#pY(jM&gr*YS&NGrUOa=XwoYF@io*=
z-_YM)Vl9?Wa?|MO7glC8vkSuQejQI({G*I`EkK@gY%gl$!gJ#6z+CoL2LP(_2};w!
zta0JiEl{1V)}6V7IbXJ=`R_=yPbS|gxc7CkRvYd13l%@iad>UR;Gte3Zy2Y$FqE&U
zv=_3FxnI%4f6c-8IrGep@rQCJ?`g98k2}+cq|x3sP@%>OzM^dJ0oim3i%(Zl%#TkQ
z@7+c}>Ho>rzbB4zWy!#{JwV7ly`O&N=uuyLj=WoZL#)#S`3>#rS0fbPd1o*|?g)}u
z9C0`?YJ_&ZdH=IAvurJD4Q$=v?;7;)6?a>+7K=R(>`xJAGEBEt5ntsb;g9;U71g>-
z=%m*jmi%&g#gc3xL62Q0`V9{uaUPEq+2d)g*h~g%A3%cwUjFuQE>E=xWz+oK#{Vz3
zL5D`p78<UIn5ZwkK<QWu#*Ll9czy|(Vt)7&xaSe1!M0ltTG1g1Q`A%+{igRGVz<Z%
zFK@*+e{mb-@uK&KZ4H?Z1mZu>QI-y)s9G0QwJyU%A9!DGQG8d&O&sc!?k;7f^`rQi
zGE)B;q9eQil=ov^9E3{g<dOQ?@!S1_8gO`lw7iPbul9Vcr+SV$`5#UBQ7)}kuMKX_
zeYN<iU1e9~<&S`fvR}jyBTQ8YOR(?;=-7bU{N9a`vPWL=MvmpDKQixaA7#Xo_T`iw
zI+zpnHT=%k$qy>mjkJ9kVRy(Uy)tq?Z3fL~{`p)d?f?tZ?%-P=<@hguOYTV^8Pxpy
zwO66|CeoL~C6+Opwp%d(>&|lnJDC<5u2Yss<w(8OL{ZH4k)>j|uiy}LD(`~r%UfjI
zj?|bsV?VlQdh*T<v|S&m(<y&D_(Ke}J*VjsJwD<6v_r*a6#p+4mI#c0|7OJR%j2o{
zfc0l84E-se$Ok;J5pgoE7fDB})d~PW0C`2t*@F~%q}Pmkcp9L+=PkZXz37ZH()1-~
z#bkncdHHNaPju7&YomJ&CeBs(1BK*g;D#9kNdfwBp4U?)K`Yw;6rvz>@?-aLjvDLH
z0Vv*-0M`tDlmV4}R4TCN>7t+qudJn&<qNO-PtZbw1`gm(CRa0+SQ2`pWfoOur$;+b
zK&C6mSNRlXS+#h3%yp|fTRFXRic9QYQ|aoV{%_`f5o2g%WJYG9yC$@QW_NAEs@$*&
z>?*OaPO%)TDw1rBJ3k_69Q8SKHy(g*-w;a9GgZr16qaK92N3%}WMb<ovY-o)&x-`V
zy}Nth;aW{9*g+-%O$8TIg2itX9>+GN<|B7oxuVYzBEZsCaDSlv_5L5_e|XIA6}!33
z(g*)xQ47<7Fe<ukcTo_)b1_`OLc7Ddvm+r^&60Lyz(Uf7Rt?xSqA@f-xUqF!$}x)R
z0ltCYvCaGE>}RymV@b~mwR;NUV+Qhcq~+NDv50+?fm9iH7C{aR4#|sz0HQcq*6yge
zf#%z*-R>Cfe%cWJ6tG;cB?;y6c|c<fsZX&V<+facSxiJh-WdNUSKlN>o(eTnGJm)7
zzZo_vI7n(?NHWEjr>Lk{Z{A@yAMdaP@aPXv2Ox&q9;|SK3V0=DW#xuO*NbLIy@c0F
zoEDVqefd}}UFqZmvH%SzTI6?!Y!@L1{1ZG9R4W&zEnW-~d65uGDkuyAYC}52$GeUV
za4y@=mNgsVkRfGjmYN1-b6A`QI}Z%=@lss~_z&o=Ht+w3g@}1Lh`IbzJ0k*+Rf_Xw
zU)kj%4B{w288L{I--ra~<6fF4pcc#FLp2M|J6KNH>R{C$mw|GNaBK1eTcXqKil6p`
zPs(-F-?9Jjwta}WZ?f^=#Gw`MR#*G0Fdc@4-avU3Q*@RMVK=LVzkB;QfZ+BiE|V9}
z31q-9NUu9PS&IQ{P%NrJ&&T=qM`KC#KMp3Om`}V`e(`r}S89xf2==X-QRdG)(?UiD
zJ@KLVREANPS7-iTvbDKu+%*!b?cU#Qt-1Z=gK4()j@&nvoG$e7hC0DN3g7zxO-h-b
z0f)21A-0Zf9i)&~7MlhkFr5nX7ZMSTf{00WP=XHj6Y?lD<4TYx11f|%<RrATwCv;}
zN2nBE*Wce#TLRe5k3S2CRj%+%JZW2|Ur6Ke$FZQ^v)aPevXP!&C)JUEcX!<R(K+y`
z<Cd;vld9FhG{xP@K^CmRIfY)#`H#m=6$yM*^X8cG>cw;HOV!W51v}^%R&>EeJfJ#u
z{c`t5CPl&Oz2hGk-BdU?VStym`(*d`PV1w#Z2P94?>Vi-O^|oVyLz+@zeGtz9D`hC
z0MrnLVK$kWOfv%>ELKNqE%&WHMFi%DP0uu$24SN7l>)T;5hO|sO-`3`S(lmw!M?Zq
z{zNW_?>{#IGEnOE7L;g5Dv83VqOqPDH}%ydOH9>eJSgq7O61sY|Go#{AGx?<VF^kN
zk#fw{o3CEJS{qN_ev%mJawbS`x0*luQo?tP<mNx3(>0hFfC3q%-&{e}irMvH^fGpU
zJ+}<Xj$$LQ{~~uj!9~8-d55`_4H?H@&1>v>pMCk(@?l&J|Cr7AhN()dz$xv9j`6cO
z-W<ByUuUT+ewXAg^=$hW#)^-(?sPiNXjB$|ju>83p^taf91ebu@XxlW6rm|AeO$q-
z()%q8RQdg23g5~bo9~<ZZ*H@}5+|DcifQys(Fn%27IhrQ`D$Hm!$lNZ3DJh`Q}+f^
zhz$NNOGj!CYi;IuqI~_UXlS*0k(@0n!?j5MeOG$BKe))7$+3+fgI3KCC9gv0xp!4R
z;2;9kGnl-6?=q1&rk;FN&F#GXV(itZ#m~=1fyG8nHTgB0(gNE(A`tyus4D3O5{;(b
z%(oNgb#IdZ2MVnTp_Bqs)lVe7TW@z7=6_e(PgF4dESpbaTj-Z(D7o?7V!`&g!js?o
zdvl_)5w=sRAfsm<w9@|gyY4>g<GtGZVz**%B~uGL1yJHA**}W^#yOInfwY1C$oh(Q
zN$v&E{UxyBra$QBz1;KNp2u6?b3kFYPX%!azI$-WU<_IIU(!%3j)h$I5On(w&Jd2n
z1?Z^*sr&^}O85Mg3DSS)q~|G#*&qL4WO|?!ahkpK#B}J+0CVXcbE&Z?9w@<aKi<E8
zkT}0T!Sz;ju)B8e*A`#FlUSGUjtOg9?(!<%_xQgXqQHz_uTJ|3o%sf)-g<rm-7`M8
zRH}^GXm71&WfTqL>BHeV0jx;DIk~&?kFl0k@?7>O^6&M$=%2hpPwQNo_4YUS!i!F~
z)?r3C0=l9nj6gBt8a`g<%6)OUi=R3C;dXXh0G19)mx+?T_c|?E_T$}X0T&)XD1Q-p
zsDXwYL1WFzPdahzjJaDjxNmO#@W3lW*}ypNJMq4AKQ7Xki+_G_cJk91)9*qf{u1H$
z^F^0N9v7PI;F&+Pzi&X-nfHL1hr$d?tNugbbz9o(5B6<IWvA<}UaDE9lJFs4Jr@Tj
z!fiU)Trh`s@vC|Qq~>i|p_A=!et~dF>EYqwNdQ_mm6-HTu5?|$m(e2;)L@6ym#+9_
z*qVe|DWoUW<c6K4VvF5tLpm`coDS6Y5rt9W60Y32wa3NnZ-^uiG+C9upWIkJ3=T}S
zQ~lsH?aa$V{^bs<X#%Dhi;P{iuGncUQDUVMkWtdL^JgHC^F(`_a<E}~F1RGm5j3p|
zYXsmQ8$r!GS7yJ^^C%(V6M(Is52^9^!wzwH-R{AJR{0U{Gq`j`Wb71`mi0*#-D&bc
zdUfoEe@a-R_^qkMC&hrVg();Nx8Zl{6YG6x#x!z)nqP6VrJ|-0Ta4dKHF*As#07|?
z>r3DW{Ne%Nr$`C>POt0h;dj&vuYoljo7=Kr3ysLc7q_h^*S@ECN+T!GrB8<R-&Y)O
zBJ4Jec>D<#D~GT}fLcZ7t$M^xMRQx;QZIbTsJB|oBEotyW$5?Fj5*=e_qDLr`-s|L
z6uVx&E)0psYCLFTrfmjv=nFr8ZUmdVIKV!}ii&j$J!Csdku*q5UIqj`8R>>YBUHz1
z^c38mG%?o{vE{Nqu(pzFa9w>g0f#R1D&<xqcQ0Qh0ak7~)(AkOjW$Y;_IV!5UnF3G
znmF)(n>CRPkfMS^_(8WmX#;zwg(k9hJFZbt?&MpU^5yX)K_9ri+|hTbKLI#k2T%i8
zY7`7e{^X#g$f)r-?bxZ=AGfe%HYGk_K@>&mLg6nD_uLkufi0jfbNI?!{4d7rdN49U
zg@^s5K<7gsXi9SY?)r$x&R&p0)V<7fX+Zz^`y_ZiOjdGMx(V@8Ww!AAgM(YZ^hwdP
z4!pF?%v2EV_W`T4H$#qW%ark{$p7;QNuE;Z9Mt+@V~t6dZ*oXDTw)Slqt*=i0s3ZU
zfl}eOp}CSx<CDSWi?^+hKvq_u^j4MgQ0L`~wT%u6{;RTz^|Hv(p&b41jEXNmPMC_H
zpB!u_YXF^SqfkJC?xdw_UO=@xf8r!Igzh3l$&oK;k#j7KQ{4^-WKZo8NGeLHm5RWL
z!}%PD5r?{#u52{Pa}j#jDz~Qg%U`{L<`%*;4e@BHr>CbUmzDxR^B<2^;+@Z%H)6?O
z=iXd=jh5FogN>xt@r^!8@vo_LE7!juFAAY|T7;rWBxK7q<?3S5+~uLTrL!$EBtD>L
zW?|HbdynfvQ-F@?k$|(toaxfM{8AMB3cE$TarYw&@`tZv^NYb=DfJ3#YFAfROd>><
z6z~^GC<qps53qDFAB{<mTeD9k`QOeI@D~n~l%~5~pJBo>{AxMRzv2VYGwE!i*bwc8
zQw-|OD!Em9^4>bRTl-Jel{Fueu>V!wo*bYi&U4T7FHd;}4YV&OLHn!;;HwD1;@+Q)
zz8ZjD3Rn{7xAEGbn}CDXtMI7E`em5}+VzL`ObKI^3A}Fju^aAHK7FB_cgNQu>q$fk
zrgI+8EK!*IA(rGoec4P-p~={I(oW8&yPOvN*(dY3%qIubWrrWWCRyeDw0`;d2|nVe
z^k4P4ixeH&o-}#=(o}7c<r4*i0enBx_y%J$LKbYkjl{=iV~Z=-tgP*9my@Dm6}Xj{
zFW~8?5Vr0fl99DA_gs34lZX~gw2j{2=aO5GbpNc}>KOW$gnaRUw8?=Co#X2r%aI+#
zKX-1TFIVkg=%5y~M|tjV8BQFG4m(eihH-x?EH4!g$>8oOlSh_Mjb?w8AbR<Do_uIP
zOXO=i%FgZBa|tOdJRFyV#DApNSR`j41+-iD!A3DEqJrPz2<Qt6lRa~axj!8!bn@j<
z$$N;@BI6u%J*qm{O!|9ShV6M!xjUWog!eK6OoM*63#J_xAs3eUo{+88XDbtmwG@=-
zi3z_POAg$h9~9uTdp}J6V~}!=Frr;KhHobQ!jUW1Zv%O8)C%g0UL2v9D;ii)s6|0&
zGuKI|uBmCz=z|7^18<X)M{tHXeIe}#XVini4A}NR<?Pb%2;QW9zytSYHj<@c_SIvC
zp>>cRocb&D>6s6;W~CN#!<5}8FuQwC3JV=PPukH6x0x-NOP-UpRZdFg3OFi}+I&Ej
zF;UvU+Bxni){rZ<U?&5wed*%vVsKEu*YHlf^e*T)lCrYRU{4aK$Ej<YRKL%UTn&6s
zXp4X~Sp+x&B59tRNT<ARA_=BqHQoH5?n_WI6<iM#njSm*)#ytbHT5_fDPzYhFW1B|
zZP6!eh)rZE-9v9m4G$|-&{GUee(%XIhv4WclhKalVHY-%Kf2o@1;1q_3Cj7tW?qSk
z3ldeZyB@(mQ9ZCjwfb^!+wxZhb$6kDFd!kL0(`+Q{uc2_H!+n$3LF6nJbIm-wUW|O
zE!PENk~KB<V3GCx15}~FytlcJHh>@ZEpb6$HI00tR-E3F7k*lf`m;xG*nSOTxs^47
zu<mv04VQJ2aZCBa(lTXh7Lo^l&9A)<nBPJ9^v6r{OGRsLYcmA1mg*%haQo8BH2{g5
zrGT(E=@DXTOLG8I5{j2DmUiR#R>LM2zrh0vOM-T}$sfVicgF0AEai8j`556g*wOv?
z)5*0?!GTZb2OEAV{5=oGBw{GAG`0(0I-3>->)Cc(%v%o#9zA+gSX9(7+Y!!b)Jbsg
za$}$~3>@1=S+(P`c5LK3g~Kv&jXgZkWDpe=``)wjkc&(HptZA7zE!}L8?#)=|33kw
z6nN6)!35*#^OLWiMFuv)2_}8IKHihiVVMSFV9g+s-3fWeaWHw43b_lex0rvA*Pa&E
zZ5Gki6Q#r9O;m)|!fGwRPx{hB0L6vOtc&y)7CpInV6}AQf8Ga<AO@^4IcwFR@LYaL
zL|(vQ8nlN}S2%9a<mTqCg9!Z9Ow1n_?-z*X)5NHilq|J>+))Q3BR97n(#PKls8b%)
zBZ5twxTDWoNlec<f2dR}(%5Dsd-NZvDL_*D)baUJYGMRErltFUQORN0iur<zRHZ=Y
z7K>Uge5N(7`ip-_O&kmvFS>k(H-Fs8^W6?hs&R1)a#bWl&eKUtS#o)4TC?U_T5)OX
zRL?I+6oOKaYNv@lVXcVL3scd}zvFim@0y@+q;55MX;1B{lYqdmndJ}G&o(wT!a&a6
z5~b^ocKy1bT{~g$bw)*57(>3F;hs=AN3zRf-k&zllsP$z)zKA`uEB)<2nOO0HO?XT
zS99n4?y~PU^!24&qhD9KGF3)BSMZWS6W4<-au5}^(1e6{pb0?)j8hn}2AEZ{JdM6a
zQd3dE=kA&dUMcVcuQs#}b&vh9nzG>GC*Q`95fP84=Czd_&^@lQ88|{Xv7T__Y_&ha
z`L8_rFhBs+2OF%1K`ZT77x3(c<26p!Ha48|&x<WC4FR1%dD3>+Etk(4Vb2hM7WYG*
z+0it&i7DUpr}DUJlFB+)ZU*f61vX8^lLN&@poPzf_UEE>IZS^r@;h&%j*pM;10CoE
zC`ec4`n`+CuD3%<!*%<V`L4d|(Y^Z4PP=JkeYzMGR(Byc#OiP3uX$K+u)TY=Fq+Ln
zs<xq&j1nt*<^02d|4>x6;#51A8wZA0MFnpSKp!^^22vh@oTjZWU1n-(N}uSDA?>-g
z)bk@Nms$Dz9)6@gWO1!pP=u;B0dRs<6?2+tmU4D=>$|HtZGV)y32-I_3-N1_6aGN=
zy^4gMG0?X?wkpm$mx-R87YKa@pyP;&h2@)?n#wQG57x{DQ4k|urfswIp|9?fO9_3G
zaJIc`u37G9(e`Ak%BSrHRVdql&9*Hg&#mg$`*b(VbMw}vCe%Qg!UO-4`}#`Z&<SKg
zP4L=j!~8-k;K@kJj~{b_tk@MWFoDj$=t)8VT_0}{;|R0)buL39hp*@EZ*ICgb<9Pf
z=Y?*i*yb@MT;e3T_pW<YG!vNf+ul?NQ|2f$n<4);$@c%#w=OLzMn+V?TmtzegPw`W
zTks!ftn5v2aPY_SxbD>R!+l90Dew}nBUbIvVF>BU^%}ICnA6|GvMps&v5{CTGV+<}
z!&uKIFE!{Mq;*{nqc8t<2K~;{GE6{mYXSx|xx5?*?%hbSUmk3|5m5U0Lg@n-lScL!
zlwX}83=cO><rNj)d6Mi;ziIpTq2UPJsKDR&>O3dA@NAQ?qM{-kGm%`~I-UG#kF!%K
zA)Z=VGF?|!r#@G<XkWOj7EQj-m_(rFmkeA*+*HZ&&qSXxLmC=8ZtkXz`x!BX&_uR>
zy^N92vv+BGqM^N9(=Q7-KN(Cm>&;oI?d2f_or<3=Ey&<<kocO=(gs9*kbuGyU=Asi
zsd3dZ3G*9B=me@%O!S2KQETG1*=m~B72KUwQH-YVTk8Z_Z%DCtc<vq&s#m#mN}*v$
zFwC^Ipk8`3=v!sgz_*6~Aw{_`ENOfIQMSeLz2pG#05~}=^I;qii$uVJ(hq|KtuhFz
zs9M+|t-o|bl5I*qI~I!wsFr*T=E|=>Y>FzJ*WxfeayHN^zukI6T%j~Gla6+=&FVc7
zE{e>Av^B|TK36dQkYpdeRS|J`uGE|OVEdw>CiK4#<0%gwB_8R@g9K3F5E8xty)?b~
zZ}EkNh4p}myAEvJc4~<xdQ%ylAQN7tvL8i;HS_8R?e+^z#CS3lrf*B?Jjl<y_R|Pt
zH7c>+GLgzIj1zEoerzL<ShNXwU$IY?CghHEM4hUqT8iSLg#h~m!jrngwGT2;jmxa2
zjUD4^2uk-vuZ})~t=`9{&n0x@Tva@KAqE1iLI=RK1sok6C+8DAT0tw)V7r0@C~9bR
z{bDTczRmp23ceA;-$~boHJ{G8-wjcOhrk|wN7COaVqsic3}AB8Y!D7HA}h`+IeIS{
z^43>%N3#{v+nyLc=`70iW$YD<l!<c+f;yz60MyY*oP?Bc<r}Ucqk|^`=#?Y#g-?SM
zNy^E!Y%liDy~i=><Nt|(fB@J&grII<1a=x-=*h!+%gbz+7QhwWu%(m_AMbFOH_l?R
zhI=~hA9ZBAIquWpHjNI6fC~EFtvLyF5;gcSU&ZfwWRs$u>wskI`irwLPe?k-j>nuT
z_is%$0GUHf{&SIY^@S(Gl9Q7cChCOVQ3#-eZZRJx_29e0l9C_B+;;+D1dJ$NZp4;`
zV-10fv;x+#gLOCVq?-&@DAk9;+6u-RK98Lxsi0l&A}6_K#&TNT6ndJwtl@<})KoOJ
zZq!VF`|F-z=Ar8BrxiylL9-TtOEX9RUnD~@xgH?jYz4%)`1szSEoKP7`xhgh)6waN
zpP@4<P)TXl+h=q6)j2Ngeq*_@A)MaVX@*3rRs(l-e(*d`$mbz8Me38S7{^K-qn7fw
zHx$3Xg+HUjhUhd*u_T9DDmhm2>Fbes6l*Mt9tboOT_(C+B0yoc79`3pd?_YX(ts(a
z{gMi_MZWJWD>D^roCb@d`J_KE8+rcJ07sUD?KL5<)3*C~NU2hm={VE)r5i!VT}DMl
zFBbhhPTiQgiQaE0#VDoWL<D6_Dtt8nfg$=2(!r)F%Q9Fk#u{N=*zcfv=fY6{`yqr9
zuSR*}@bmLXnuHlNEP+ntRJ|8sVOd$LaiW_ql;9G@Eu~LJ*Sv;_<2lBK20?{@Mrw<P
zb18hTk3N_HEhoMHu@|zk@bu@1=oe?NDV@t_4SH(WWhLObrCUQnkCb71C|sX3?aZ`p
zXp^bli@$ag?LrGZVF84C!oo$`ZS@xpQW3&p!-oR>gMuQ6b9KrsZ-MTHQPz?MHjn{>
z*Faxm-gS20qy$})prv1m3hAFo&<S+lvLkdEmY+YDgi1(7*p;!n>eqz9TBsnr1Ol5*
zG>5!y3_V~%(I>iuJrol`7QM9mvMl=CV{S50Wf&a-U~>*Rr^QZ=y>>gSM#m>A>WQGt
zqzOTi=O<KA!2pTLpC`7w{YRybOiUh`SrTv{BX9;s7i8v?B)lzW@@XRZ>yCg$E9L?$
zgn)$|<D!yCM;DclFa!hF0OPrShm{q|nUPRXIuk$qyxl5?z>VQw?m9&I1?h=|3#R#U
z!E^MocvvfbXxP!Gv$)sx-WjcdUfh>42l^HgRiHkpo5dG_hC|G2rY$5@AyE(&1H;?T
z?>e}p<_*~35Mz)P^cEOV5<G@$vm?x`jzBRu^D$fqCMCq~yL)IfT($;_y>{xLDwKWa
z&EJ^LC<8v*XoooUyj=7$AX2Y!<ZKP042g&!;IbMg2lqW;fZKbXbO9%R;p{vQFkTdB
z!TXldJtIg;T-<PSm$`Y)#EUQBj*C2B17s%cjScMQHkhR5m6k?V14fC}%gjH@t^*p{
z)N(<cF9_Y5Qp1*)pE{1r)rL(ySAN6u1L2C_yVt67OU?z}$fOEXx7l#fm`EdDI&kXX
z<bJ$*@^B&$zSFAx8qPsWPiiD#5s_XnTz3qBeQP_rW{_$1f;$XERJtEro>Vrz<VfX>
z81HI*ePiq++_i?!B_T_gU8N0y?m};6lJ|N~k%b_YEgMw({Zpk3IHdEu$x9d*jglnq
zWk&g%y3qJL1qip;*cAM55blbMz7p*py=-{8wV-`z;Ihi_n9od3C*LInJWz<(EUig)
z1n+2#$=+2#`QYJIQeu}YWQ%sh?ju=jo8DCGgLqiJ;pc%|a7^5I?<&r7{4f65E(e$G
z;Bwl0_M%i1+P1D<>&*S@*FK!?KKtrr^|~NO#Z4&n^$f#eUCznJ=VWw7Uw<vBd|D%P
z14%f=pvd&h$!9#|gLjs0CFWsS_6-2k&LFqM{Qbh4Ai<Z-U?D+0H&qhp$&PRa5*~9B
z08au%!*B(pT~tiW2q7|Nl)+0ZPq7;AtV|Q;9_?3><x;qrgK~6V{HvKv)>sSq&&1oo
zp~DWRJY@+LzR%>CS1j^+Q>AW{>=e}3S$!0_UVP(Hx-=j}C?WCdTtu!;{0jGz2Rbf)
zxV65x-ptR>SFg0C2YWv<YJ52iE;Z#QYMPL<e{;*r_Y`|1gX}!7wbvS+_b~H3Vpw07
z!W+C*<?mrR7Ve<BZQBIEb`VLkAHET~dQHWTz!fkrTk}ML<MoU|26RYlFd=gd<Z0jv
zEG!811s7LkfAq1q5Gy_rug62Yam#OgLh`rCje1370xP`JD2dybv3P?HP$Lt@#n>b0
zqa*4s01sU3M7l<0()geGmh{`EN{>&?mA&qgTxAIXAai?b!0;UT{N44?VW1jq1sHHp
zNeS2X>KGXSJG{fgorESiksVs$2qGBG4C(dXT1f}hB7O5?c9h@+u<EnQH#a2MX5L7T
zJI;R)cV-<Wz~~>Zxm`f5ukme&pvh3y8Ao#PMbV#G?iB%I%(DQU?23Y&`c|4tBY4gL
z5|t5qCbEm{{3CK$SlIgRsFfigfo{gN+<V?t4-lApiH+ZZcy6LWxIy`9%Vam!jv@#|
z_e^ibrtm;DU38UcBLCT`Ujes-D&_ch)9UM8rM3Fl6ip-JDD?4_{*{%xoTFkFb&+EN
zkoouiDb8~_D3-uQMUkAh7ra0NxM5^OLtYwc{r6fJtb=@Y6_&hbxN+bsQdm~f-1P{-
zb@;<W!VpH$$-Vyjg>sq%6)ELpw`<3hsg>@@UWpZYwOJ_LS>@puKL1K|B3_$mpBl#<
zP;*Vq&-;P$u5($Ou|f(n9!AhEqIX(0Q-Qa8jeRI5NT=KEw+#>W<N9r6W#!U7#<#Wp
z!3wRTx0X}&2`T%rV{re~spyK6e8A81U-$DnmnQrp4Ak}YpvVf1iwpAfyk<UJh_Jf4
z+G&oNnsE(zlir$dMUcq+A%et1GFP68_~U!J#G7>J(G~SmUz61<*ZBFh$#Hp;ScfDy
zH%<*4FAU=R`M-nOWcomm|H3HH@j!7E&1*yR`&&GVga8=N?n)G*92^`(H$@f!ejA-R
zludhmg#9}8Z+c={+GbQV5Uz;5O1AB~v>b47Ic*mDKlA7~Cx=yX1^+qsX&2|Bm<-v%
ziX9K+R6Y67n_F6v!KGkO&@wkSUkB`ljzEoA!>1#+(G{t3>ff48?VIdrL@-Zz-b6&y
zGc@|RgXEs-*@(>MBdXuK8*cr%{DZ_bzTvyQ`Ri;(`v(L)lKqh<whMR<?&qRAEPi*q
zz}>-NG3tV#Gq*?m6p9w`9T+t=^}^3m7B-z4R0RbE^YNOStWRw>6b2?pkQCp*vw3dx
ztF4LHY0>zWdf>|CL_HR(9+BKZ>pIyX$@h~;H9TQn7|HcLyf$p0itaBq)-t^cT=>FZ
z1?kVa$?GyQ7^<3uy_dp$i3_5JmUGy-7?P9`iX83tR>yE}-h2!44t<b|WU0LE*T~v_
zrB|%VSa_89bl7_KZf9k+p6P^IVd11VGvPQ&_6u7{l{;UC9U+(qAZpj>;|KQ29Dye+
zc{HT2eT1yK%}0jDFE7~&Y8{&;@ro|$(_BiDNo=9Du-W7b6haZ63^a+cWQOFT<#la&
zsGzMUS>%00gb&uO2kn7Rs;jEPd_l|dEtFtBe)31mH4MeDnnyeyU5D9(FZPp~S9<lg
zPVDEgLw7=w|ETOL3UU*^o%sGpb5?tJhw@vRPp3N?o8MSl3?i|$5YHoNe#AV}C-r7>
zX7#<_#EtOG*)HpX1L!&Gut#8!HDwb-bDI%m$i@2FnG}5ogSq-sjc>GSG(R}N$nu_h
zb+_dk%zuo{(iyz9w)Y|f<Sp9tuwEOq;aAfSbS#wRViYEn=<`m?KY}3p{_{&RJqtv?
z`x9@{*(F3za|}-tx%vp)>JqEoX&YNTij71emU!rtyeF>e)>XG*eo+o9)(1flZ0fgX
zk0BGwdPGG}j|^^ZAOgEmn7XOW--7=1H0Xss7U>@m2U;0hpI`iF^Vjs6gOxA34*9JQ
z!G5|-dXX@u(Ls@y(;Rom<Mo7WVRr<0?{?mKc*qxAMTYM;(D&jk87}9h#$Q<FM^u^%
z7C!c6`}6;*bVraW_#HimbX;!uVN?1R7vBSyQLwN-k3Zpe2O;7IZRmN0TIx*N6{!sW
z1M7?3kjjY?cG;o1BjZ%>{sT#!_k&_}z|zk|w$lBDIYsjauf!6{zKd&aAJ#sJO_Kho
zoL#KveHosl@j-TMBGBpx{hXyT2__sJ!EJgt1Oz^yR+%ZEaBsq3&oVzX|FA0nx|`<?
zu$?AJ>d|!H8_8;X*jL*x){bW6dWNxc1nx?Ha4iFL9*WAKQKb+7=nNK?eO8fZ8j-&6
zo1U)$Fs5id&uIt!Y>LZkJBwQ7u=Z|Y;RRI815%pE$Gb*9`drJxuQDNX*y09gW$|M#
za$@VLreZ<8EEBdibK{L1O(b4Tc~*h$SC<$bi0b-Swe37m1;k+pP4DUhWJ*$6x)I#M
z!mQ9gyU@<}y1U}*$YrYw{spNoWsTNk<G!SVzn}d~Wa08SgYM7<-NnP6Tg?pATH$vY
zynx%BP$4e}W|$EA(hgF<F0<e_7{ra8oy;LQju&}8(4bP(v^OZU8<}FB;)tVB)eRYP
z*GswL|D1Pk&&3x&03`l?tThRecyzg^BpkZt2n^;^+_`gS9U#$q)6H*xkG&{+cO?X2
zv^1RuezT=QQh`q&Hcizs6<SY9b_ze*r1|@(de=ZMh3VPFd>$sK&R_<n84cdo4FS4T
zSX$ZwjuND%XUD|#f73n1c=*ytcwz43^{>hl*=7&Pv^bzHUhW-M^jEmwK`^-pI99+|
z7lOl(&ovdUglu47Fl$fKOfzU{X*vAa2;*m&#nT@KP6Aie7UcDwc_EL=*LQbN)N20l
zDXwKzJ^v`^SPFsrM?o>RKlhEH#c*qDjt35_L112}VQ7fh<9Oq`DcA|N)$!}HMVtgD
z@&c*cw~2Scso8=WyIUB@jOzvUW6`{>Wv4h?Ke>3NI{b&y(?qvO=gtJY60g39@f8q9
zKzuYIqzp-L9T>oAM8M3sH9$vd08qXUQue8trt8d3l^eXsKWL~ZN`8*0=cHGfGa#ei
z3B4`%Xsct%&2gO<0bkb&0eDelHg94@I4ex4XhZ$wBrM-YAT(C1yX_X@=wYgzlyNCe
zVf>}PPlZ5o6?ZlH^TP&=W;o>T$;bkMlZB}3j7lj;`p|Q>!$R=GINiG4x$<<d+%yNn
zh^rWGfY-~dAH*M8Z(AhGQqCX{TpBqZ!LI-id^?Tx1&1Q2qWPKnQ;ihE+RlYvh0Nw@
z-$5tg!Y~;Hq1XN%Nq=6-!6{(i=GdGXy;Ak|DiCGWiwxe>gxXU?ZE%26mj+>jXxS@}
z+OfBQOWeUSD<!h{69Hp1Jd7*s#Zsr+W|QhhHI2J*1vvQc#FCPOdH^Hhb>$;iT=a~J
z#YZXIkO;QbB0o8>y-XPh7;C`czA2yia4rZ%Brm{lG5C?C1TqPdd(Sbyoa{bT*W3&)
zp$P>DS#&&nHq{X+e+*j^`)$UaRnsC@SH9l}l^e!Nog<kUww9>Led7a%*1kUB3DrV=
z^yqYWPpzMN=^8&x8u#ty!8B)$ZA&E#dh(U7&$Z?R7ur`J4V0DSaiTr+`ih_G0S5LC
z-00b!m&j>|0xjM8AYC{@kW{~g<i5XU6^B+8s8;I6nPo9HAKz&R@)0_!!7v;t{`XyE
zRJC(s8S=<gKz%?`Azdfraf0vjC({xho-I~PI=9{`!~T2AG~o*Ihr0vE)?PsH{Tvkb
zqQOX$F`R<#uMO(O0pvpuPTSdbny9ZZa86tJ6@&*kx)%Jfr)Dfr4=MkhtIWW$^G8-@
zEU)77RsD^3{hP64#VFV76EGT$IWmRbiD}vq<T9&v54T|xw6jqW`AyhuqhHihM=O2@
zAiun*8fl=aQ53Fop?L@RJukRO9@sI0;DPb*;lo^=T2jFG=$vTb2X&@me0?0NgbghX
zN`WLs&?_rtZTmmQ-b{)FUIq6l#d^j_?B2V3=*+E!$bJMR4~X&zc=-BfyS(a$5$x!k
z1y`M3AZNxj=-7Q`j4o{2!wcL}y}dGm4>)!{fq@v!27>_jJ7D)T25veV4=Qb-*4DS(
zvVB?7Tl!XJdD*+gMFfBg!fGOTT)-o6OP}kd;<bs*Q;(k6;G9tBUAjBc74`Z*jtZ?=
zR8>rMu9|JC#ud_6y8s|O1P)Xm9ksRpJgz0_K`{72UpmG+az2=ql@%aeB7()CEb7$!
zvEc240b)@)+GT%z?u}TB-OWEsYRYn>BZ}w?kKyGgGG8^nkdW7Nt3P|1M<gZWfo;)a
zukP&J<UR``EALWxywPRYhRkl38dL!kaQ=%3@rD@O42tmJ)rvpZGE)y+TD8NP_&|1C
zdF6TvK#YE~ThH1us;xZ4Rv?x@!g_%Ls8Cy0Tu&vv3~>Czh{~|ET&@|MWSE?2rpkW8
zYf%i`43Jfj7w!u?kgO&~$O-tMFJz9X*R7=9xhy6Aw*{mOMOD-9oaea4Z8jWST(8|f
zqcZ?QvBeXnqZL(tO$O}4>lYFL$Yixku|uZA*9X7u^k!u%SZ3M^0dyGK%Yq@lK`Q1j
zS1ul1cGYMTFYqY?TSDOWutec*0e1Bl5d@4tS>ttD<W2U_+}`Qa8Akw)bexItUk2(V
z10a2sUkBz;AK7af5A=f~!*K!U=yhG4(COhayHi!hbau!b(1G97ID}*&l&_9&u*hJ@
zF7>0k^1q-s=~Pp!CqV9_p-p>_hY^|(ff}p!M{o3#ACqrKGyzrNM!V4pAN$GJ-e0@~
z-9Lu<vQ7$u3e1fajq_ewF#!%O%>%my&-Ha{u=a!mTq4IH9eKwj(!wYeoYx74|Dw?}
z-s8#X7nx^`8Z^-3%EEp=9eQ7K>;l((+E_NaTxjz1LBEoF8X$o2`1>?h>Nz=c?crf9
zOtex0R@tTl=l9FTUX~&#DhOXaIPb<KRUmy!5C{Yf(3=K-{K3wJX^=a47Z<;@QEjum
zOtwHMdXJ=h#`k9S2Fn~%?0LUPQ|nrdWD;{N@2vYBHK}FFH_}2aYIU|Zw=Go3K>wVX
zP`~){uPY()LiLT1i<pLh1XTMqn|2iy50BbuB^Uu29wl4#XoMjmZUsfo^C$WXZg<Fe
z?ulF8RoLtI6RbpAA8(gyEpc~X_PL&FJ1ocLoX$et>VEudhi*8tbK$Xegzrj$Nq;89
zIl(b2r!!I3w=xW-kTXll^FTIv8<rW4Lwc8DFufYX_Eah{PTS69F*UsXfWgi7Rqvnl
z5P6#U#Dk5F1L>`iuRcBp+w@PN!npngG~go!G$AxJ)Kk#yP!Cl7ps&yQg8$$k<nTv>
zlOLCiv((*!;5mNXizlwrt?b|*Q=mMYy<IQ#<Yl;1l`f^=7hGx*{(Q_k<mQP_Z+a*s
zenmXICdKrrf`If*HFjdVx<Q2KSitzEiqH1cNqDTM;3+x2QE}WapTyNnY&Tncj@HEQ
zgC7uIVzjO7-J{mu)$T%pl!sBr3yk$7gG>(Mfvit8ZGhXx8~ge`JF;{qZ~sWDC*QRt
zzKAVQ9&rsBlO5-Pk%BnAS=8XOI0dy*$JZNxE+gpyh7mLy*KvzLfdKv?2R*O`*-!UL
zK(+#01f#efxT<quq?iF99~|IXcvmnXa8so3#uzHd*;v0FLjUOeP_H2;FeJ&zw3V)3
zM}6mtWbE>^B%d|I%6XiS|02|wf?`tmU(P#@)ckq?Y#4d~h}`R7>Sh|;-N~(aSC}gR
z4XxIm72{G*q0jC~t)KCEmL@Y-`QYbbkqXfTP$CT=P2rB(w&8OXtDl5{d(IyLf@5;;
z3l9B8Brp$hA0>p^<&CW5GvX<StB3s0^U5j#6L`Er*m?o(fQ#C}rD+hl1^ANt3#lRN
zVD(b0Xq>EMwCH^f)bkHe^oNRs<}S`}PIsnrLw|GpLsgO(f#3)Rv@s*kr5!|H`09@)
zCIvr<W*!mfkwV>q=eddrpQ6y-8cSAq!)~)sW#ml~{JcFXfGLX%b1+Pp?8>XQc6bHv
zC!JI2G2nTB85=>O9SOoz#sFmysBpnS$|~vsAQga((Q!f1f^H9h)yOeRgHy|XNP>x+
zz=A`5fP>qI*!$p%j<x-?>(B2k(~r;zygkKZeck50--l(3g_{~pEiS2GmKI9>lZ#gz
z((^pms-YhVIkN<2HY>%FQDxJN(MIQ<%JTpfTGc2oo|c}j54zUTcZ~u%xZi;u=Q19O
zs24f1a{BdTmjfy;39{8RPio}Xw3JiIF9@W0hm-B&(aB4nNxBY-dGRCVzmaGf>PN;Z
z<8g}XXgUwONuoe-APP*oeT{~Wl0;SybeT4}<Y;bgfWD*ws3=f!9@Y^qjx$F;;H`VI
z&JRpe9rk36A8Pjl%AWu8IObOMv3h4FPp-w<5=3H4T0sqOVrJ*mnoJfbf5*o`4RH?w
z2}(m}AKt*%s?;zCDRt`v5&VUlzL5IDJwWaST0M2uZ|R}%$Z^kWXL;x=hI$}3ZL>43
z!5He}U!crx(b1PBa8#p`3TKwXfBDoIC&o9D+bw-DW+hSJtqb@bZ?s>*Z<{6_Ie^q4
z_}ZCtAEKh*4i||aOA4F~)<Me3<~J_k*kyQtNo+#!7@`}_c-(u5J98fUIzcV_8o)eb
zr9n_R=81sBWAW1mbepKTxe38_%S-@fc<!l*@dO!UEH7tN2{=mX4<;pj1pVJAe8K}L
zbx?ASu^VvGSTH|!ziA~`Jky9BIANkPUtEJW_&iP|3V!+eH>2-28E<n6HxBhf#$Vss
z*+()}D1WX5J*hy;K*6-}%0LM^m@W}L(4q#rChl{Q-T-?U13um}j<IV3f>>PDw<ZWh
z2BFA$UD=;s!R_f;fOW}C&2SAjBY<f+ZQk-ab7KzniF|uy&(bjXvy-6F0P{RY%}}ex
z#r38#$rjld0Z1)?6y}-*z0PeCUU_i{zpA6&Lpd;LZ-QMPseoS#(RCr%7u+bALSou$
z^&ULrP<Xmh5-*bcm_NwH8M0>Qe18(w1f-%zf*uxuGJNkyL+S7&w`t<LFf+L3gCMaU
zpXrkaH632BGEB)lflYqBV4-E!edB%cGw9u0nCpFz@bOxly^E}&pQZ8k!5lQ8Ze!Ci
zFw8Vz3pKBd6cgeK;^PG@+(!9ifbhjDZ~B{YKOyxx9TW0(HsAK5vb@I#^1EdL{;`jg
z;TACcJwuzhZ-{AWrgLHiE!%#8t!8Sd=K|?BqYiL8z*TtQ7b8G&-lkp$WaKHZ%S23B
z84qeEfmJKZsp{Sl4L&}Pq5-WQoz;3)ngp<$MD0!99cIU+e(8Bv6AEi#qWYbU{hs1g
zB-3ILimGcsF>@q9IS3xX>1dl%hTa!#ym*?K0>}ZG8Z0mdb|~-FBRID;j?UmVY3Ny^
z!|Fk$-TUb=1KsW023^sdfGGE#=}naJ9Y9u=YQ|^O<MMLBPMqBCMs*rD;}oj>kX~|^
z;XW|HS)?#-&EuELY}rz8o5H^B-X^9*1TOnCx_>j$kZXr&qZRaS5p;Ume2P+2OWbN0
zOqB2QhwwB=F5-&xBmmAG7t^C~h>87hX~p$6rdtXN6$cSTfN1Q$a;kqogBTGvOgQB|
zblLZvE3<QVu249j9pJM;_G1jPN5?@Q$o7C0Bg2T<bwt739S&$#gTYP6OM-T}%CoPJ
zy<vnm?|Wx>k|H%5#2LstznOb5(8Fy````G;sD}iE6Xf}g`68fH!0ucAf&&w<0fjff
z+;4$}<vyjQr8eHo!hb;j5+Nk&GKo9?`pbvX;{YFmDc&7${!1{L0N7<cpN(XEe0@{g
zj&_H?xIJ81S&^kK5pDy?ck-g+C6agbPUq1S1;8wgp2N8T=Pw$>f&&m&(6@XP6#Ak!
z1=^O1#sKsZCEf4m$Iu=`n!s~uT*P6HhhCsL_dZ)rq<fP>3w3nGIqC*Dmk6NYKFTid
zY%Z#6FQjHoT(CNz1%N<`b?$Cp>k@H({_mppJNS6WRfLXr{|Uwu9SI=)!!CmA@{)jg
zk*wyV5V2*g^yurzIh`W|i~{(W>|v)~aEb&_0W99pe=nZj2@gyPH3RJe46bCqFu7Ea
zFvzA-l3X$@p!fAOL8GHHvz~fIuqyN}3h^cT6e{SzSeKP^=fQ6RI01CvAwpuxx95+c
zBgs;t;^Vs+|InfJi`&s8=u?0JtcJF>wp>QwlX6y{8UE)Sw(sEJ-tv>t-+K@6#`JXL
z><G3u%!289Ci=<qG3w;!uQy8%PC!SP|Dxs$AO!q$k+t#KB5>0)ADYeny1q)bid4~`
z4p!^ibhhDB?*Bq10y;lokEEHD)`u{>WNd@?30S-%Bb}-AfN~~vkxZcAMVD@xh<SN=
zp=9@#WEV+B@${p=?LE(wJOz=#0))5pGLiM&-EtH+mB3{JZbVjhjX0_G?`k)|&tD@c
z8mZ)`9k~F0fTn8g<mzul+0t39m6Ks4$Er;)qi+uY$~1?@?;7T_RZfc=6@BNrE}#pR
zx(ZY-lx|jq;eumE2TBAY3i5avCe*P26#)bzbj5IEU}0hHgO+j+816sT&MGXnvm%te
zm0MVU^AhC-6i2rS?T~H{9k2L2?)){%>y!m}$Q4!VnH1Hez6%wi5bD#iL1f<^FEPA5
zKSz#8em|EfH}B3%{?_;<BK6O|xlX^3ctze|(X;+e{3BSYeK2t7u@xBqS;p<{kxtDP
zNGqj}8DxCY-sgDp;wYY+`@~I`0w}rz-UA*)(}xxTnwgoIk9<Z3sjjNZF)~^tpwzwC
z&_X8&sV~ZLP^?P3$Lw6`^;n5Ggi~gqpnh=+x5AMX?|ttE<JwGs_^Du)W`}m&hlR=#
zQv?hg@?A+b`?k%77ZZI7-??YCw0g$;Ds1NkwQl|C$$s@FPJy{`T-)z;kI`^8xr~1;
z$IW+GJbV=QcE5ohbR0i%LIDTq04oMdPTc&WLhlQkby3vU!Ui}jwWX`7ooM#iE~%Ds
zR>MEl=W>o~<fT7kEaL9f3};L2Iej7*ANFtV89P!?Z0sN^&2gGjP3sK_i8&@aEkDJC
z#i%yse-H_x2nj>P3DnU=4;JR8bf|bR!cxf;1OX(TGyvoWxi0Q~W^V|JRmAZEOgsI0
zPlQc+&;xA(jSvjm%IR-L!ooJ%t)AVoli$4C_8Imm$p4u44A#*Qw{aDpa+m-9RddT*
z>9Kp07k+s$mmIj*zTE8Ao#PS=Etz2>zKjC)l-0nY)Y<DR?B&^7lLq&XZO{@u9B_8b
zq8h^3tL6_qtLu^HZ63vJlkR-MD1Xl=P?9_Ugw!bgllFF{n3PoVhuH5%1P6O>Q2omS
zx+C!M(B6_(9C{?+?rjNboi2}&<Lh8V-fD}Zm#|ilMtU}J+!@vUBwR;4B+BfWkjmuR
zo*~|3zhjx}*x}mYWa&HZM5dXh?+ykH3T;ZI225Jx0=pk)GRnL`#=eZ(deKoo0PL98
zXm5|lS@<E&4v^EUKR@Y1d;xd_jim`5uZxH^Ye!Ih3}p<UC`gC*A~lmW9<d?Oa?%s-
zuta$&Fg3|iKQ75~D%C`8Ton|L6+H9aXTrk)M=6D^J7yyuvnzIgGf7ND*T45V*6yIQ
z@siLhM4r`(J3@Nn>l0s`+3R>QFfUWfhCvhr`@s7f6>e+xH9Q7+zrfN?I_3`!B2q(f
zLpz0`zQMPCuHQD#J&Sji4pJvM?JZER{v#}2uDbOyDqL>*l;35MO|-E|v+DP30qvx2
z)5DpXMpr%I6b{pxi?p9nk^ekHQsQ?Yk~t6NNM4|@NCBqd9}*IUk4po%l)-3d+}n3Y
zUu45>KCYB}%wLi{XO^vhmkl7c1tF)%PtU5D6}*d$q&%Kcq_5BI`K@>Ss@-(DC-lNE
zM54T}IwlMs4>r@2j?;XbF7-K|f(qMX<u$=7*oWv)^$j_NZI1!nAs>E^=U*m#C6_6?
z_9y8hc24gnvxv*e9zAD|sZqzW+RaNec5;gUuWKdXh&xX$NmeZl<)F|Vpak`5UX0i9
zrKG0LF~u~i$V~_HbDPUbW+g|Bh(9LiNz2eL;Cdh&p3Pd;VkuWN79{l%MqD74>^{_j
zb9S^f&~|J6^l5jZiDV+@a*@$Ip_BVbDD7I)`09ibUS?V;EbMW~3#vSBYH<av8_u30
z3?j2fXK!NCmFm9rpNwGCy0efPNYfO#Rq0XgQq~2l1vBfl9{ss1PngObGr*oWbI3xo
zDiIQX_uGJ5mc{`7Dqk*Q(!+#<Ovh0jS#N?9Mx`$q$t4L5fu+*Mg7jaFZQNa7qp(ZP
z`|ltbq%FToKx=yY(qWyB8+qsWYm&#&gFdn|z5*+p11gHPv@Kd1%CAF_?};YzY2*-p
zRN1tUk4C54E(TUI&^2q`DAhfcuS!(ztrfXpe*9%$;_JdaYU;rhoV{@MwG=T#U9QNm
z+aD!7o~<ct2=L#<YWwaxmB0P?*LNeC>ZGc~>oNBk6Z;pxuZ%j}A(_wr6~0%eo`V@P
z$l|vrIM%>Tm+*cX>pYApzCJ$@#wkIgAm~Iu)d*OjU<Pdp41aCf$r<396-Ae7^4zBT
z(bsRD`3X%6t2*TA9z|n+3z}SZ6nE)52{!U))Z@ZKuJQQO>bTrGw`VZdG{4!Aa$N^)
zu{8(x_YP+#ql`{U`eS@wy62}Yf8aZ9{3;-#Y)zmoxAU7#EKNgdpzKpo_&8xpeBIZ*
z@TOVW=|fh9$iTa9l`oG)99?*ger|0W`aKf*DiySHee+%bEfXJMt$3cdI@g1v!B298
zi;o<IhDQWrRkDWWTud6)7|l;?u3;r&y&vS|WoEFoGxC2EX)JH<w>RO)^QnJN`~8o~
zm^s%mdm6OQ=DTK_DjHR8d?#FcK0Xy58Q%c^l<bw^nd826U^ekyMu<;yxDUZLc_kJN
zU#Jty)HA=gsZR~2vm!q_7`wVVtp^zA_ue9S`)2N?j>sH(x?T0><DjUFyd{3S^tj1R
zSNQRl%F5xHg$aGF9}n&YAqWXcNqZI7<{_9pQ-#FcC2=0_y5=pP%YbmNicUvImjv=L
zUi&2iQf^Zpu(YbFw^vD(2v|gR^@ut?E{>VFe<^LVF8$Mi>}$kr(;7&_qA_1Ny5N&1
z8Xg8yoA`Z_zIS=4yE_v5Bcme%!q{CA^0?~GmXHfl6{3FL%v{97!`kJMl<jNp2#AU~
znhGnCEM4{T=w?pfJ)3`cbI2c?2A4)0!{1`QA=Yu717YrKMSCjpY<ypxd3suX7`x7Z
zTleNxKl0Js1G|S}Hw`Sjj_K%p-71iC+y&Sf^sA6m-Xw1K?8R_>7KxxP)305gQ!ikn
zp3-cjD6)~VttpVU+6=VE9+d9*(54VJMZ<dRrtKWPnqV>J6H<4Gv32Sot5dIW=L@TN
zIC-e}8>Baa5zD!19k(*ef+L7`BOPYf*<UJ1axE5#YL?ID^WW;93by)9m2{>pCMPNB
z@ndGEce&7FkK2UPUdH5^guVZJb_1?pNpydUfn%M`#`Td@N<Jqt_i;NcZf2KuvuFOe
zk7z#OlH#%?YARGIyfb%ZX;UJyI-P%#yMKoKeU!TZiL+{cx>RyS&=rG;G|$xyUMy{!
z!r@YTtmArK+X>R4yN}Z=<>fxiO%t5E0JxCx3-ig<3;z#YZyi-t)OC$3f|N*u(s4-%
z>5`Bx>F(~9?i6Vdkd*Ee=|&`^1>_>#-O~Nr7oX?--rqOI_XiFfUhg?)pS{;!d#<_W
zL>=6;LEZAB$Zywj_?Hw@p#UBlaMyY=Q*R%dnMn%hhfkTAnW;GQC0W?W6(It8vH`-#
zH<XNaUJ_cXb?MEhdM5)u+eyN>!x6L!&){@Flz+lT^mrGyGVadV^<!phWyWax59iCc
zO)M;<-W>U6gVl}?FMFv6Z?GeL1<3sts@KuzrWD9mG}h6tG(>W=I^aAW@3&xKMW|<X
z{@L>TIZ>HDiF(^K(TB8lty=h+>2Ry+-e-bX#IrToQP%qx6{KrpG>E|P`-wVhfmXlw
zUid+t3pTHrF4x-@F=kHl-ZFc+5iNJX+Oul+mW^z27n6=%txNF|&M}!|CQI+CF*Le*
zEn<^`jb$V%oa8!@QGAANITYSK)~cwQs<-oZze+PYq7B=Jwt!`((Ma-h@n`Ch`KT#D
zqPQ5{DK@;dysczb`zpu2zfFu&_~G}9)n6`LryMU;sy6FiFvfXk-(OQbBWhQir}D+`
zKFG7fMh??c3KWHGB**?LezCuCdT24*KuFo1uxMc(1VFGuCok9Fq$LWYhrK5OhPeN|
ziiD-eFihe#5Eo$3Yk2{fNZg($v?Z!uXMRi*aESysMV|}4eZhI%)M0F?I3toIvRYs9
z<mT1)?`r(YT`!3<ALD3cS<N6cFb>V#n`K^U+<4UM3>+O?d}$E~*2vOeP$`>nMh;^U
zAuztH8E%^UD_<oZY=3OO8i%|5_^v*n(uAkt+URGnX!G5G<E9Fd7W+x}oy!kra+8}f
z%39^f&p+wj=E_L6K&z;sev>pS1V?Yn!W6I9#QwGw-&+QqKjjQ5tts-+OVh|}R4Vn5
zOoV1!E(`7*Eq%mm)XV7#4wn4-yJ$DXIE|HtxC=+r?k;m+!-uEpiGcg{lx3w8L*7R|
zHX6uZdiug((frTH4jc2v-t6BH1n!G`K2vl})2q%5Cl#`5*B5blx|AfYx!dX6!As<-
zg%+NVp^qJ`I&8VI9a9&`V~-al9y#@ti(nftMh*UK35&xQEEgOI7B$0p;){nmtKUEz
zCUGP`*+8fK4c<!qFt`=%Ue2Jy&H%*0|BF+BdjUlISS-gNe`aQkM>6>T212cBG4{-a
zg@;;4>Ik(KyJ{U~{fN-p$EXxby@$~kpAlp4<bndZ9EW+Ia!VxXJkEP+x53jB@ya`Y
zKc)qd(9ena9OO?;(+g!J2QzM1=LzQ-jNA7`R`z99x2&6_?KA8($Moy@2IK+g@Mg!4
zikak%o3unEA(UN)XgNaThftZR!c;w^VxKr^t$_*b!|=(cOEQ|#VXHDz)bCe)PzAgY
z>`zP3XLRuD@o4xX-+j81N?6F-=QcqQ?leB?Yv4@gtSHwJm%CRo5~e70-8{3jlbiGr
z6;&akm+X60WlCxp?>P2efI;)U)_(m)It8hxSQ9$FnRKM7yyVB?-f#A)D&^^~`!iyH
z&TjBTnP99vdr^-`c-Igtbx_m)&A{u7x67@n;fiuA14BEWM6iE6aOgS@d!Be&biVLf
zz5EQ-amDqSxRg}epzSCzDaR!jlU4bT*JJW5b3J=MB^vPVgfhn7L84$#p_n}vrC68M
z140Gcoz_lpDgTv9rC_#&FMzVyjq6_^1oNwUqut)21otB{a_!lh8C#106U*WKpd{f~
z>Y=MCIZzli{&E_nD$@JoiZ)jN@!XH0?6DJ3nS9G|ry1uBQ+<8%%HXb2`||J6w_LwE
z77%;uTfP_{eUO$6mPWsQRiDD#`_<LZzGeP$iVNzZ%&e)QvAA?8<=}L2{`mA53oB*e
z5I+n*&cClE%!OO+-t)O&#WkV2oj~m5%ujC}Z_eJ};1`ONczeT^*vf2MoK^b)lF?!B
zAYM+|NLp^bXPNIm6MfICbTePL9X#Kq)Mo`*R}ntT71j0G(gt7EV&Te&?9|CdK-%Wn
zy4B@L#uqCAHsU01hnq~kznd>vaDP`$VL3S-(Or0~Xx`K;m3hm3t9hz;#^@z1JS$=6
zA}FDy#Vc^VNp)jxGO(Kj2(0Fkk#&?(+v76XiVfF?VMIY`>{caqV)hlxrZpsn-Cz}9
zf#aQmFf+@)0!%CdD1e5_ec>esTrN~RhK>eD%>^r!SJ>ECR(l?GGcVw#YuG<prb+Yi
zy~y>1N`ajN95)!d($0DKUk~YIsRvcA)?Zc2ze-RS7EZD$THlD4X`Ua_G3ha%;{hVX
z0ciJoX;tlHq<^bFbj6KlTg=#}=#;%!BNz8FSx}RS&}}DY^kl3n9c#Hy%txqn(N@I5
z;)iT3P?=Y$CgHKzkug&2ukAmxrt_eKqGF)04zZAv7o1~Bh27XRdF^_Pd#$O-tp>SU
zTIDs&J~!WWOT;6tRB1EmInRMWx;l3c<nH-%%Qk#0#`7p#D@jrv!`0HF&w@U$92P5h
zWcs1mw~*bgZ0$psqZzRnj0Q4*pkvtEs3g4p-nO(+1j?s3&1MlT%2)W<SiH#NmoHC2
zMJw;2qUGyY@+pb1Bndl0?-+DuZc&MY8eVcZta37j9T@~-JuJ(0*czh(BK5EeJIrW?
z#ch9fBcrk)2;^DC^-~XxhzR(}7i;<#W?PDv<d@xFjGY*SWT})3Mr}MivxfvvMCy}D
z%+r(9g>!ynWmer^*gD&3<Ia43{<DGU7*T`@!YW@C@AKAL_YG>^YDef;PN(Y_!Rr_%
z*?v|*KSOLR)Fj{pRehIGdT}sV;oj``MAzl-Nb1;=^|Y}dvsxO69!U$_bu@zDx!xdL
zVJNehlq3<iBjFz69fJfp-rGtArt>X#=|)~52(fdT9g7VMmO039Hplbp_G53xo0ld(
z#`zw6gU#u1C1!8(OtzI<*S{(XnoJ38pa84e#olI}ugEx=PnT{drOY!Wsmxt_T6r;R
zIVVeLB@XM-?Kp(-Q0CJ)9<WWJ#|Sw&Ilmelje~=Ov)m5mVcnpsTrnsbky9DNfEw+}
z_|Ty)#0&?beo)+4`mqsT{R;8#lB?Y-nt?nzQ*qg8>P$*;o{yu?n;0vDgY8Cbs@j4z
z%qa?o6p)b>-5Lwtlg$_0I}1Oej<LLz<z5k-3%IX%LnqJXY&G+fp%*nR)?(t?kK6a`
z`St`;P`tK;37xXHq3|-D)H)`j1ZQhV4|9S~kMsgx_@~&9Q8FJi;vu@JB?lXgm4-s+
zO_$H*8T^^>g#C&`Nbhbu4%sHxKF4CvG__w)kqAnOpKXmhhX9B%{`F9%5hOpkzi(iu
zkNt^>3T(DSGJs4Wj?aJMAyeo$hLOK9FfjaK{Uksv4`ZA&th=o7_Ekr4S3BO#KiqEw
z1wNEtwETI*s+d>z39}JF@?cuBbdm1;8pzE<nDI_qX{jKf$AwjHlnPSNGD8q>fMBwV
zPa6FNS!72*u<shr4~hmmL$5XLM@YJU%#-*}hTb%XaFvotmnItUWkDlsF4VqHDsfj?
zGOSL`<_N7zWIG-Do7P4+N31fR+k@#%?->=uN}+YGzQOHYDg(%PXb~B`dtNoib4T97
zFMAwNU=d$>zu}aQkn&9`va6Vp@ehM<`0CJnt^H~PQ&95$=J4E~jcCc(S0lacU|cb}
zCu6UeXivy_>nbQ>9`)ZeAGK_|SbwXr*o?T{`C!VU3gM-fJU8?f?q>rR3p1K#VY$_y
z<3bE3xfLQ9GEQ`C)fmXLUC3NT*nIL2?qZETdh!os1~v$QQ=6Ui;sK3MMEO`h3aYUF
z0OHCv>JA6YboiE*7L}4Xuj-W8pK~819^lc%Vt*!bvbJ}`Gf$1GmaydN{)+rP%Df8z
zm}Q*&;1;8UuCQ=f2w4Fo!Pv2w<#TV;V2Pe3+^wbiHNtbn>G7B5_b<?5K1IvF8az9|
zO}qP^hF#z9Re#4Vlrb-r+V?aYvkIyG{Ke3A(uiw!wf=Eh?1k+J2SC0eZq%+D4BxSs
zJ4slYP-RKI`!S>Rth7EOG^iEr`3<YAFKtF+K_@jt;L00+{oYxYX}joT-7ojv@A!Un
z1@f_$s_Tbo7238U{=j+*wcqfTKIRw!GH$9LC26WaTluW|a)i4-BOVXamK6cBDGnz&
z`Sdrl8H8s(e$^MWF-j?iW^C~6$09G8&ELe4{35;Odq(PaicgVuy}SyIJ7nCU2NY~S
zTn_Ro9qGK*+xy(mdG}i}?_()IN0aEH{N$54ybvSZMn!)0>iF!lbLB0=+__PIflPBx
zmQ{XXZgkh?7p5O-OS6eyf47VmECTHdT>HK0e5ZUdgXU)-2iUm~(Qon);VvBc+a9~v
z!d5Iq@Jc#+{?oYhC_wX7=F^uiUKIIToRa2*oMgSyz3=|^uj}(yQU_?KWC1<?4Gj&8
zK>wsjmv3jb`Km->YYi;G6B3gf*qRI17m~fFSU<SG77o@)Nn~&Lu-w}7ic|SzONq$Y
zl_Osroq^HojEp?zQ9-q<INysf8JLqFdZXoWewO*P-#&I{xbIH+O|b3RG*<AO4+E*w
z`>Ldc$aYG8=epXePZIA;^mMe+f1lKnViMfZ=F%P+Xxhj{*E!VG5C8G=9{6ZZqBU}J
z_Q=e;Szn-6*K?pnTFeaqNM!tf<UgG=CVhf?lD{f+R{j+^Y<z?AaI9ku1AV*u$>!Bq
zAXM%-=U7p^14Kr-{z58(Vc`>rUPRs9!4AphWVzf~30oUilDKyICvokM@80AwFWH0=
zerI{XL?Y`q&$#l&*q2sXSh$P-z^`{TFG%v+Cg*5nj?UbD!OV>2fZEig-mfp23z^`o
z`_U6X6+r+-`CN+A&gZ)}w<^{zLP;`kG_9e%bam%Qt3FLd$xx*xKDku7b}Y)8o!yRZ
zlOOk-X86NEJ6B>JYv2!(?}|<GP~RECN_dr_1~te?7rQ~zf_mKHEA<`(bLP1qwX_f4
z=BZW}bg3b%KQ96vc1|1$5DWM1c~Q1re}J_K3cZ7gQuhF_#yqaq<zIcSU7fcRPKQ`H
zG~ZN~H`SvQ2=H+n9W$-`r->!I<HsM!pqBF^8ECq=l9Sa3|71MLvf9^H2DuE%99weW
z$k~H#A6|`ru)NrZF-QJ!RdmtI_GteX^~-CgYqC$6PkL-S=(d(x-HS?Q0LB&5sLY#f
zs7pG>bZKsDnOeN_eN^*o?PLv_jZDkTUpCYdran?IJQsAJo*}H`?bJigd|qfv%$Dca
z(>6#-;HFQ(f6^KemaAn14P`bJfTV?4hIfgaFv?uzS~pv1Mw1HQh8vRFe&6G}r&bK!
zD?`^4&a7}*Ezawjx!=J%6|a=5{ic(48R3rFcF?@zxF{;w^TQ)<b(4bsE8b~p=Vu=B
zs)rPo=lpCH?;u;5Jo)wq=8eaqnK)l}<8p@8)K7JiDJ8K~;j*H%hJ#iDZz}wF<@T^A
z@on=_JTlR%O@HbdGDQPr%J?a+B<jOLM#U%$*Ae#==0-&7`Bn;(1FcUI7QR95M8}MV
ze@9!;M4@1!c1FZu8BJ5g>3}oXD8&!KD_o;4`I3YzjI`uV%bsA@zXw}<nKm0pVS@%Y
z|LAw&7!L&^vDu<$p8&5g)Bgq9XoW5xD0v$Li6X#zjl9D6J7q)-`HS2={t$J^M95g^
zaNm-%jParz6)8a9-ATxPPnao?k^BgHLpvvy19rQ26dmH`XDg}SiBQfTGX11p?+|pm
zB(ce(%etXY2z>EfK_|`KfoYlmml&ONtK5kNMI=&*c6blqgF4NpLK!&{Eqxgt*yGw^
zFIX#)V=vv;M~s>ZkJTxc$f9tHXrsO!G9(^5Yot^vE}OFuPwerbuU$*aS|@lU1QgfO
z@QA&VAIBk&VxZS<f-g~VNGfC=cqPYzwYk3c0-T1)<?TOTddI~cR_=c>W>rY1`4qZL
zWNryBRbHa^_ry4*a>INLHCVj6b9@N%c^$%awaO)<IVyN`4nR)@)p&vfuR2gmd+ivo
zxM;v`IR<}zu+R%rCIkApsqS4pXfephD_7O7Fo@L(<7Y;!f=C+bOb?DFkot_NZnZUx
zSabmie>gk|zf@;jY4+EeucDyVbCJl@mlq#?mH95;p8%A!L9Hk1oPP$qrR1+dHZ}}E
z3^;lqi8=TE#_$EW(kfk=EVoH+XisqPXtV!gSWWSv|MBl+H|VEZ?mBhgc{%#+v;7^@
zvL?7%2?D6p>pX9bIZQ6x*=Ycb9B}$*;7RL&J1f~I;>-Y43%mzb0PSbOh!9xXM;qy{
zRw|anr2F;#h8!r<OXzhqiH4Pp6X6u-yl?;TPflUv1>AD>vaRJ+@HVIcS?qHA1hQQD
zyQIRnN|!t6<u!F(ZK&TRKjBqXfEG$`>7xUd%lH*=LmffA$PEsBSVw&Xpz*S@kPxWZ
z`A5}!<OQ?>K(T~8P%+k>1*;js&37WesyKsR`0z6_GGOQ^P)^A@#Cu8Y4XW%w=j%5W
zj2a=TBn*L+NRQJv<oZt|(6iq^#Z+`0>l`O$vC~klegCHZ7RJYr^DLtMlwy1PN_zb3
zY8z+od(`E(Ad<PjS&nUQaZwmyI5R)BR0<sbM@&hXRhj$J@l?5^<>8HVwj4OXx*8NB
zI{-PA#pEXj&>1uUD22sDNurSvmunVCo0}0e+8dE)y)NJj?ZtnhKc1c_M|vgGmSSP&
zGkAuhL&m*JE-i_w;g!HXB9@5C^ywORYb-kmUjzdba0MmHR&`m=<2`~wzy&NGAoH<0
z3Y;Lr{wm7j(_E;ZEEf$Cl{LWl&#(qTkTM-qn0A37nnF$o#dh228en3E<$JM&82N&w
zb9Y)J9nDBMhucN7ZW=xhAKP(p+4dviV?Y8`{63&+3vUBN%d!vyuB(pzYGHu}Fr*^w
zFy)HP?+IcGLQ!y_HE~7yj}M@)<l{paR%@yHo0^N_0&PKS#h`3QlT%hoE$#hKRqS8P
zWdyu`jcNvv06-XqX$F7%_^|_Q@J9>xcaxHFwDnrlkc1hx$k`Kk;d~d$;ZV_j)Umo^
zWyOpiQztnlKxYe_fTQ)BpRmw)uK|_D>XuL}VKY#b@JB}ZXwo$<R=M)4<TJAl(VpyU
z=?}0p8!)5+|6(x0{2zpu0TT^di^HUmOON&2pKm1uy?&nby1LzYPZ3H=zuPb(gufMH
z5N|&Behsw9EHt_i^JRGPMHm@iQ7-~UEhK6pqNpSvg|=f6*<892)t>OrNChloNXFtd
z<5`bCptuIl!8Mabn7=)A;^NRA_7K{`get^=Kj@lR8ULG$GXulsIOMR-{v%$Ln^z8$
z6a<r<xr$>`FeR(|^GAX(W*Kc>hhY8wlH7@LLVSUS$7rQ9xy;f(_PG%3@YMoq=c>Fj
zv4(|PT44jA<O4gVF0HNx1GO5MICc2P8I>Rt;Z7B=4~}$VLb9P`;u3jQ<G)(OJwU5r
zR>H5BG=S#Q2^U6J{Kt*_g;BwxS}i?5Yq0lUefTfs-+u{qnRE{8e;iAxML@$823;m`
zpFab9hcgRsuAJ#|z$6ocUtKG~Z@|F{SuBzO0ypBIk)`CE4Ah{r1c9^yI&8&|FhyHv
zwWi;pm`TABxQBh>|KF}hg)-abLkL_tg=A%)!Mf#u5V%U%(CfzJl-SpeOol=zSQzhF
z5853Fx0D8Lt<?RigUSPQ%!|MbC`Pr&i&HlWum)``v^#p8zkir||Ibyj{(zN9>uRJ1
z8#;eDVKO>8WT1`R0eEV^fH{>7aK^yH!|NIi0#D*8WP(|O`>ULIX=f|E3Aa^sr{lxJ
z>;ZRdKmrFu4Bc@uKc=qn8v;o|k|?8B;5n+X01KzADLx9;g=o+O2OOV%fMz7cd}&+{
zOlvMKu5mOq!#Ec5?61#_8>ndjHJo{5&(7Z1)~Skx?4imA8;@B10CvlvRQ`=XKt)N6
z$E?lE6Xiiqy3rdxhK<z=QGm4ty{^WB{RhCE0D1!AF(hab5)wdmj_B_0Zu{xz`yD{@
zVWW-sS54?fAp+^i5!+oyrOeYT@0S9>$~F>s<cH;cy4|~^(FQwXj~(3J(+W$qB*69K
zV2ms3H4ukf$E>V9ZAfzPZ)ZpOJ(V%8L|-lOk^^B-#sJ!1xQG1g8GGf2wQ3H$t+CG|
z?>z_}UL)YqLCB_SEQ5f!L78UtQ&5y*Hj!#!-~zkF|L|cH;ydnE>PL;cw7O{WU`5o|
zTVK?ytORDR?%{?!?>>S&85FY#`P82UnA1s2D+ndRVU9XB;JE6OFdAA~2K494WS7X(
zX*f;msLrF?up`KdJ(R{|*Yr$?zzj%15fuYv-`guBboAbfqZOFi9h+_OQEI+dslm3v
zN_WS9d-8kmE_NI+Jfp4<!lVeEZSKdIR_h13Kk6N7fiD=D2}MN5{<W9Cz4^O9Iy$PE
z@D%>>t*@}crpWKI<uKh>p;<MB@1r-IXn<ajQ0}`Qz$G2?iVSZ#QHO?4+p;DY|BX&^
z5!-p2`8DmK-|ifuH?Hb*?~x!K1YX#l+&S+9#aLW)1Fb8<<F~(RDX+ejR+i>agg5P%
z6n$*tv_ZmC(ZL}khPO0&giMCvSs4&3gmjSp-9&_}Z)m8sua&L+(wno<&1^X3*n7mC
zYw-ODPXN7;aA2@#)_htG@IxgcgXz@++h0o{l(RkCSPk<o{Q7#`{M=>oQ|w_a?dgMn
zGkjV{=bH9}^)Eze{<wltFOIQQM;}}~0;l=c&xv>|T++_WNl<G);o<+T$Z|5JU_0ko
z6C10-)@rqjA(o5iQB|e*JKO#Pi*t88o06_XCp&K&%7A~E?Y1NM7WK2;sBX~o=7eI>
z#-Dk{8T8Vfr~|8+3Q8-5`anGSR%I%g;0<>HL=1<fj_<j2;qtTPc`L=PMH8<Q)Z$Ya
zJoip~X$0Wm6#awYk;8;qm@Tp4khn#G{{zV{ROXhf<m4B?>`zCvRLw=wBmnm*a#+VU
zy4HggnnDBII$`@>rH}G}`kqQ5C&%gBHP@>gcZ=w#i|tVO)7Fjn_tEO5MCH_I^u_Ov
zHE#FnqUp7HFR*dU_@nlpQ2(jw@pFse*CO*;RKIh{X-hi}t=JqD(r<ftF>d9V8vEL@
zVx~c#xYAEMG+ycC!e{Q}kB@G3=!h4*4wZobGuJ+Ekqh71j*4efYk_FJS*@<A&{rff
zg%dakL{bvxR#y@gL#QJTF4N~M1#lV)LF<NBuqfcn6TdVkgBKH~!0-&<e9&~qN$lv=
zulya`?qjI$d-R*{H!BnMWP!;$$z+bFgOcH0wuml=4$@|g^tdwH$CKBkJ_Fk+*MUbN
zGXTLH5pQSJmTzU1pvnz@W!U3TV29E}n8JCj)4|}`YIRJ~t(6NgdL??@;!7_IOc$K6
z$;jUdQ`BDv83XBbe|0#}FRfmx1`N>?^3?TxRDrWgrTVGDPqxQcSQ$Kycrf*C|6{=O
z8~=SW<~VKhZsh$06&lA{)SbuE6o4k&CL!(?hs%<)b&vR&l}+T{XBaYv+d_#hx}_bG
z<FLOs#l^<*6RY0ZG0(KyA1p|(WVTUgG{?LC(-2=o0+)e6h#V=g;o*?rqlKiU!-4Np
z<$~|^5DEEFAI*kb7zTH_i@1!WBzG=p7#vK39khXx%12Gcw)<J)>-;_a()3koeq3Zd
zA@*m(RZYr{^B!^hZhbxaR}uq3mpjBipg$5CnaO1}MCxVwd3b-csv?!MEM4(kMSBc&
z=GQ{0>4r6b>BR6~iTng5;<8&r12%uWh0L<od7%X8RjN6)UJEQk+TS$O>WP(dJ#i<_
zI=4$y?q(6Jd=h~~Zp)xvO`l%OvWZqxQ*EWKW^a)OiS>v3ICmi#_5EKJtx{^AHu9{x
zKgYdHG=f(Dw3eyHTjo%fD%WgR&zHulzpk^5*xm-nI7vBqT&*vgaA#x!Qmy!pU9G=a
zlm#pO1f-S!96^w%9|kIL%QwFq_zJ+oxy?F;&U@}KPE_ja1E!a<v-6>E>1(8hJ?c-f
zKuja&%Y!g}8{X^Oml=M>ZC2{crKi;g=S5eb)?4stAk;UHbi@8DsgAw&Ne7R$QgTSv
zyRl{TSBkMN5yd*v_K}U!Np6*yF6Ww+?Ki$r``)WN)de3}Y1oVPrIZ&4EP9>P9`6kC
z$GjWYj{m$BXQ+$w60@LJKivaLrwBC~%8cA8oY!UMwr*#}!qRpn`mAZb?DmQ!)`bl@
z7N4|P_O{X{)usOS%3V5@Y5Vw!N8lxm-c)t={N2U`y?&i&PerwFn)RLOA>u{?S6J*G
zlwxA1TvS3YLd(3|!L-v(*SVYUtO7yVPI(5eN)!E<;6KXzDF1?paLZf+4hSN;u$E&2
z;IlzNNm*+#N&$X4x5yDiy&({|96}3G45YG|HhoiR@gnIdrMJJdQa?U|%Ro9IJS!f2
z;(MHR)f@A{T}d<p0-4Wfk#&nnH=8^k?;_uxu5dM*Gb;~^o@_N0WMEdO9!sS!iK8fk
zeBTX8tFasR71PdSI~Kez{bKNw;bbF1U;RL3l8|Q8y<cjgTAZ(@3R?8X>mvCx1Bj7Q
z#jYurnG0qUiZ^9BFOkEr;+9-B*}f!r6#234B?zR{P0!<#;FYW`%$mj||D@IV72H*|
zLNigj(0pY(#FY3AYS6GgCVuuY+9QH%CYOpLJk+wMSLF9FL3W(aj^|3VR-IGfa#=_x
z^Ec=zw-dW+HA8MI;-@`j`-ZYG>egv4FP4-z8vdtpzeWb?v?yPPPffODxq2|_Rq4L>
zl^T2_D2N=EXH28g)?Mv7g?!&69#pYr|65j4?;&iFDZs-So{HmsSADYbsk_WOnByV=
z4fZOsvfzL)y%n&E1AGRvnAnEBx)~HLsx^KbGV&#IoWe$}MM<ZU_1m`Y481&wW8Uy*
z=<jP0zf&&a1mGn_)wehL$t-;KJf13jgE9~w-1S$LZ(=`{Ywt?Hd+VxN{P79N-)u96
zKSiikiIeDRSP<iqe3PA}TQoO6qnI)ag~C9!d-ns)y(z!V1}*J{u>~`qAMNrnPp>Z4
z2u-k_y;q~Misv{Ti#ImYTDX%RyEN*PluSmc{Mzr!dnX<e8uI5ftxm@*YV-Ad&26SO
z-==<yPO8UT8E9yMSZdT8tye|jKiR*Q6&zZyn7m#~d&25E(~dZQ-Y9VGTH6}k6B!fQ
z4|zK1y1Ys^XTt~)FugiBqZmzFkNJv^SdV^p`J+pFIyIU;_lsHm<>a|){kxh%Bi4g+
z1}{nK;&_*VWWB!K0lWKQnt{Dhimk1}V-DfY->S8z$E>zjy>nkqe=m3?>)IfZf$qix
zY|HSPPqE-H{{GDHl9Cs~al9~}4t!Jq)RY3wkXEZ_Ex+(G=u&X55+4oDs54X<)k!X;
zz<|I-OxjM8>}daVDNRt0PfZNWn_vgFbfBdSJzs{)N4yv+@@>>_Cp+|DD}81wrT2}L
zWHLg<jtjAM(&c%s-{@_9O})PToQJ3jU%N^B-SYj_OfC-Vt2({p;`G#W5_=j%&ikAC
zYP_zDn@m-LH%8(7Yf7Di*Soc3vQu-2F;;z~eC%HP)U7vH6cfdYYCC3M_-|Fjg@Lvw
zik86M{`-R1ZKEh7jRx=1FGN$^_M&F--vH$3wtm<N;RU}976pbgX2v`E_NMA>t?w|u
zDb15AW8b|C|B|k8v^NKpL>hsP{A?2M-R-O2T!wOtl+;h@>s1pE`M!U1Z6>alJc@n!
zlH73H!f#I-`Z0>Qslv_J%d>R%^e=Fhb;9G0b=HZ1+9?z7927mR!o?x@)C<STquz}P
z=5_}j8Tt392WvhBvH)$nMzMp|^9%wmyE4GqE;8x+gisRGd5KR(9_Pm<-5_CbA7{c?
zbB9SxLQq07WUajbMRSoq6b<(eLKt)wHcjI<mbke2k!01S&*zR)P6O2KzP8F0vV|cY
z+j<_Cdse@@W%*8mp!nvEN~oHE6SV%GjmfS*<*$A_ze1AB^??@F{q-)o=hl~yvnLn3
z=+l0B)vDIm=UdTB`ZE)|*QccP*u-&)rt1V)y3^2a1(F1*je;v;Jw<^SJLHCcQ+vz0
z(?;Gf3}(c>reytQPn~W2>G>2kmf7eo$&>kNdBFwAr|&z;1`MF)?)M?q9EaQMXN?=z
zhT`J8Q+~1%G5Op69Mah<r}i_2KG*YPr`syb)sov`%JefgFY`4~&0~CSSkamrZDC7h
z?n?gWc@DI|=KL|X$s14SvfKl~2A*7AUrK*}L^wh>;@_#Z*SjXxVILhhN;NeT2maa%
zpl@;m10^R6FqluCH9zcxfANxE8wSD*iNwUjEJ0@oa6BAn#xTFo@ed9jkt+W67Vfyb
zIQ2QPjN-~Rt}65Tpflxjlo$mJhgzYi=gbz9^SK+cstJVQG{|9P%AEYy?Ma&fd`xwv
z-pl!#p6j)ZXFF+?){gT7$(WDUv#-a|PkxKIx%9u;_>?wZKPgnClfB9lA)omb8u-U!
zv1olzzGftoV++s4>UbgU$xBLzlQdW;l|wvcGk=d4Oz|c;wV}bHH*V)eM+h6y7;LC&
zeq0+d@3F9$<u&pUd%W!@<CV(<2WJef??G~UwIS#^wPAMir*$N?=lz({yBm7Fv~o-u
zv(|`R?KAK_sdH8?Ybo7@pmASZcdk_NjN&gR`Djt^tLd+6S$3qqKzh}*obEXVY`q#$
ziRud4by6B;?$Y|xUn*B5EL*9X5e&{wafd&G<3Wmz{rULA+(VYoh?-273QlvmpnN$)
z3Hk8s$cCkF)_Sdo3uLk_in)=5@UBFXV}H^3_IP5v*h?vo_Y67VF&v_WMgk5jD{4Ew
zY1&+}S?eOmbIN03y`&_U_0c4aM8CaGO6BI5j~_^{R(lH)z*nPMuR7!R1P^AN)y60Q
z)1w!Z6EtkuW)7g!xTobD41;R1!CfB<+r}Ht{i=5Qc7P6r_D=1n534+%4dbu5^%AZc
z%bgT=J{{mA+ny8HpA@C^i9@;*qA>WGpZq8)aj<bcTI#H7HzI6?ubgxq0vQyZ8g(?@
z&69-EHLz~4a={_FQyZYUDlPAeXf^v}j>+ZESogArmlsW|_Is*5HisRR{7_<J7u7%%
zg~#|=56eftL;`)!#KDAj2W{7OZ97^12e42({3$h^GS+SglhxcbCptevC4yH~HP^^;
z16~4xL}q|Gd%Nj(Qdj%S@wi8|W>?V7BX3zsczH{GW3w;GeCE_v_rlF<$?SN&adKm;
z3g4_>cFKgLV54p#nz2-C;{H08$r#cpHbv#~=+Ud^Qlf8F#NHX36eRbz<!8u_hetJ=
zkX+3DZf-15zdJBj(?bqZFB^U4S1-;6dWkl_awMvV9<eEp<3h-f)_Y|*c8~0L&3w=A
z3_@-jKf0c|rkzieepchRjys5CG-lcTq}XlRUuUn$zY=>!BB@qecmHj{TwanPY9aeM
zNO4H7O>_jkT0;i|{n`$x$>Ut!#qKIwnpS3>gH5PjDE!o;*7KNTuLjz|qCNE~&PuO2
zTw3DRt)#KVdg&RMCDo6Rk-5$Vs$dzv9?CO9PFXO`)B$1wb3TJc9gdBvcY^_y{^Xes
zF=+c<Z#6(7mh@2}si3y&4-(BOQx%zmct=^E0o3d@dYjwLU;iBG^k#8k*HVzlN<<g*
zN1DAJXPPKe6DM51lQ+X%7^yAbY;wuD^_?rbE<-39FeMq->0V(T{$0JYF*Qz?VwpHx
zUq1g$J<XL=WsQ|;qVBEDI-Lg<<jx%U>z;i#(lww$fDXRM(5xUs4x?(f9nsk_98Ib(
zRjsy4IbA>N^tNtNiD%(8y=^mh&~ClyGBy8_p)G8*k&#<6b%bzzvq7?*-@Y(^RQ7E>
ziVFl8R7)BS0nK=t4IO`C;Ko?hks;DsGj^rD({Q>L1IpONa%<nLH0BsQiI*LT(?JTl
z56hB<eQB&<yNz`~W!sfB5KStlpr8Qs6bfDUr4YSIv9}9g1s_IH)^ba9yunC&*)C<K
zcUE_5tm!_=qbmbJN~bcwQ?c`zxSxt(P2+ersP)vtwvh@lXx{5!t>XXE#jp-v;tYD2
zy~kS0j>^_scU^ybS9RUAeRSRQFkndwm5S|#nbU<Sick2NE4<_|k5f2a2`JLYe^Cnk
zFrk|5oZurTF1*}Hty=58{RJA-evijtTj6sQ!D`KUvIS1jtvqDM25+&j=!z}rCzmz7
z0xYW=YBWuz<A)ezbt^8b&rT%$_T1v;-Pil8!*APsE46eu=jW4rf`c`6W*`t*2F4Ym
zGK~Z7(%oo&DJ!?rw6i8$H#DnUbqQTx=u!H-34~>PlBrTvaCYFthv#h->0@SiInDdI
zis}M~X$v3BE*srtD^2~Ubo*lOaDcNn7jkT@gR(vLD{!P$5DEb4QL$JA9<Rfqe@`@V
z>GeT-yFZ80EItT%-r17Ga#$M8KJCzhKb0t>RKK$o(P%=%8Nh^pw+KaS9}CsLqpsFC
zX_`7OH6h7`3@63@X*&L>cGIS8Rzh8!RXfGBrJ}wd_re?c!>{^&SfUNuoOx%ZH@$Gz
z?jvL!a?kT+G%#O#w>V?JIlr}4vc_jDQ!tuXuz3HLW^;0a4_B8}SpP=f&&5`KrefjE
z$ILgjiLGWo-cvFwhADMV<cU)yM!sQNrlce{EFo}rVQV#G?APfhz%_!3H|&g7p3ZN)
z+AWp!vCd*T(7PXbg=d3?tlTGfcbTi_zGL3p1`<0%8gJHuAXYi~7ItND7%NDtu*~py
zcbqehcCeX*JEM|qgG@EpB5x`qG2J_PQ>C7ScBwGqMl*FV)P!wz8&)>6scyDYtiJxI
zIm6=+i{E`C1)-Zud&mBy3U5)b*)Y>*a6Gsx5`l1Hrm0|RkeAY|6%*_jhpio$)}uQj
zy-F(8N-r1G<)XlXpbaP-xmoxI?046S8s4B4{IRq?J~0`FSp?6*#PKWk=bEkeXB)T2
z3u6F8yzctRKP)p?w8I$z4f)aIV}paj1<vVQN?wj)%Moj$M?*Qt;7e%&Y@4Ht`lwc%
z->l_j+%CwRbP$`97;HE<mLZ2lhU^b*i1(O54I7MwUh(tfg!U)$9Fy^{BwzZlGU}S~
zBOPbW*qEi#vem|?EcZobz1D6`UE4)GI!j{^983_!*T4TP3)f|XJ0Mmpd`Y|aC((rY
z%;s#)TNkzU_=aQMbp102xuI+7DG`Cbg48;BYl>07&4!ndek>+df!uNB+T9Pq3H*g}
z7W?vAx=xj5aZO?=>A0|N8&&$xzk56=cU?GkiJ>uWWth}%nmb;}LuK`bc(t4nnw1wm
zN&JZsbjDZZpK!0wNoUK<qDua3q>`FBi%M7%hcWU!pSj-F8aGO0(9|(wj|^+}PQj`>
zzFZzEgA7D4-Of+IK)@<hw4uvYD1(H1*rR{uYGULMkfP$On+`Yx1gaPSMmo&d8V4IY
z_u|4`wM>IWlG0JFnypRswbR$?WeNg#6bgqM(bO{{-V!aSP7gSOEP4mT-DhO`wXrPt
zUf7BwCy%1>*e(p}6k6{mJ42VO5_2cM>WOiDyK}4Oz>W3zlR4wcO{Yp(=0YfNYgVgN
zK9Uw$?p%z7wl|}MzA!!|$7!-Vay28Xc~W24J|JshLd1qDl15j!-%@<4iGDtG6d4@s
z@pa*>Q^TwEBl{~BBhGl0Ek9G`R+e@8#E;La&rNS$=i&@nAXJxMS~f6!Enw(x({<Z0
zxi#<60+~~sfZuJ;ZfLovd}^)1nOo|GYRji`MUt0kn)SwuszdB!sZISCrVI1Q4WhYv
zsrk2RIn>5%J@))wi5yN8(n$+{vlE@UbKV_G(AM=&tquK_ee%K2j0&qxQ!0+FkK+9>
z;fPK?09ax*yJo-vHfZD~tLlKMhPmCK-_~koxqZjUq$X_PA^8Hqr~$+r*1IM;9k+@P
z;Elk4=?FwX{SE3A*Q6nWH-|x<XS*+*S-5MzLm=tX)m<U5w+2{Fn)O)+{FxS4yQgO#
zL$(gk>e?|6EIsT;WO!YL3G0czN=PJ!d>WEOmgiziH5!Ko@p`c8Mlls78^ithofH?l
zIac@C%=^u6>A>`{lG^6Ih0`6CuM!0Ap1SrUY|3)Bvj7EC+-`y*#cEZVj-(84d%M7J
zExvQ=V$C1)Dau?a$FdpZ0)|Oi>=mAEMrQ(SSCqa(cB<8%FxOKrrv_`}&8}#wTTe9X
z=lJrz+@_NL_i=0f;iG%RKPP~fJ1sm=5HqPHgsK0+498(kmVr5Qc9sxi<oXkKM+(?c
zYn9po9H+0fPxbF>)J$_IPXP|Y8X?4I2w0#F!m@XfOQzQF-A|0$>anQ1cS9<Nim*Yr
zc}U`ro;@IEa4T|oxBxl(1$TROIt~;!3P3}k<=xFC^rqEZ9uebLl-t`CNy&z+cw|f%
zgwa9yxVr7fBBnAsIKLB|n~-{4QO2M=y^w-H{!YI2vAdKM)U@_)xjVsas$CGfd^I=)
zUL-ACGdi)drS*0_R`(CoHBTLQ;Jq&K0Bm<O5?UgIcbnINNa~#hH1=tEd9~Va&(gMk
z7h}inIls>Z7jHZdBZgg^tXXKf*x75l;FTBWpf9b(LVWB`{mGQAd|$mfu~y!R0sXhb
z;*SD^F(s>@^C_+LqZv)*Z);IXv}@BPlsj;xR~nGukS=s#7e;PhFwp^_G*I0DxV%}-
zw=@EMog8FrY75{C@|`d;NLx<0^ZOSWi+^uN1MUaOy|}$^i^|;eBs7K;?K^k`3?!=n
zY|FwL&jCH#1;4K7T<0((j!N5dHf4I&qLrK@FdI`qk_Q0eIl6Q(J1bz82I@c`j>BzV
zN7rr6B^5}5>^uvkGQh(6KBim)Vhrrv9Ka94{V^lo(Xe*hh>bOxR~4p%J$ncI8L<?!
zzl)#uvu7s|qp3F1<S0>8T*@YY3{^@6z<(RakKPJ*CY@uuko)@w!Dsm0dB8j$!^6Tx
zK>=PhTNnXRY@<v8K*ptlVgJju^Km^=UZz0#Dc^I89~Doe<ps+>9=`6LC<96ixpTro
z=|~MqWM%WT>OHoBOlE78a?PBca4_>+CydwDVl}oiYM_k#3sBW2KsB3a*u;$tq+m$F
zdRb`y#_mGD<4ptG`$`G1uSjM0wT*CaNH_%@qY2nhi?F%rOy}7)oBT1Vi85DHfdiCk
z>;^oz5LzMO9E_?)09KM=Y{Jpe(ZMh?HQ-7OYUP%*4UPfxCe!A~dA2V?p8Si{NZcBW
z$){iUlU!zS39xdQ-dTr*r4h|e1%L35K&ad2%i+*I?=NG+w0fsF3DSg#5*=taMR2q~
z8v+=OYW*-2F!YDg2EfsdfX}rI^poYS4J4I{hGFSVh-7!-l94Mm%<C||h94AE!FD56
znk!ku>CyQSFiKYgp!XYUgPyYyy~e&&>)X863+o|i(;HVtdYB1txKhVW)OmC5<|!xG
z;MGozaDJS;6PqH|A$;5pOH@%`CBhc+cN&<L59k??F*G#PYH-K~VzId1ZyRd5ArPJE
zBPMjO)6e|VCRCtek)Srvq^eu-_5#eVrvYCOKh@a$kQ2j0S3Wm$*|kS+Cbu&J^$a%@
zA}Kl5ts*41r}Dp%xM3i`55D9>*z}Tz1(rO}w`d6zI6ben3QvJ>kEc@PdpJ@5U{QZ(
z_#s;M0FxcU&T4h)euk$|s*2A!GC0gN*6+!|G|Zw!z`JP^LyXBUGPFUMs8q|QP3x@v
zuRZtUN^k~r?-qxBvO^ONs0~bj1hNub5Ol73prE0-sQEvS`S9;%hA}nld_>$sPv0%2
z;tmvfe4*bUGKbj%FRBDQfd*Cz(%94%mayeQC^|3GhKhaO>HM2d+^XPdl)XMTz@7#j
z0pvaf!1`iLj~N^qaIJo21%y)ZFe-o>ftQy572S!!?aiPHjL_92t1ztlKTV(grV4S?
ztpHE}5`1{a*ePGl&M!r;>`Aj)dPM1J3)l_7LMz!~99X+C$=onAQ-=bW=m#!7ej(6O
z*ao7V37WrC882Yh(9aw-IzYgqvKep7GttFR){81pKWU?ohP~Fe^bYK6!RuwyHuY`R
z<688%eKjs`rxV%#nMglY2S;iL(|T-na5V>U10Y|T%I%oL>$084^M0Kc0)bB@14|k+
z_=|XaD;5kI_tmV|?fxM=(dt1Q68bk~4Hpsu0>dVPn%#Wvkk|8Kt0^Tb-FB^Cy&{jm
zLkPWvKO>cb*4;|{ZV;!sKIjVk4Kw-%f|}!`8O44;tMRT?!5^x(TT_c*F=GKMX)U@1
z6e|~Q!+fSyR1%g~R#YqX_}lg4Q7=HW4))V2<~W33*IGalaC)Qa2o>bqQDMY)N6Qm~
zU5y+$FXK(V<|{6`J(VV2&XVlEqf{tc{W`EfgY+gKvK@0MNi9{wk4CLhgZGd?D&ikz
z_*OXZ9tZrI4|jcjP+w>@-Rf1Z=dp&1Mk+9ykM*6@wWl6M{Q2`cjgOCjrd{10Uk+}X
zD;+(B&Rkh7rEoazz@Bq744;U`*Jw9xhuSSvI#E%&IL4<&$0R)h^Acz@eNvXUGF>Lw
z<98Yq$PY}+vsG(GQJ!9d@9~8XS9sWkC|YhCz+1q{2MPgCU@G{4O8pLK`l+yx?Bbfb
z;cQMrAShiDOJtam^-pVuwNi>!z|(6(qZDmIHpwh`hF~Sc8yW}t+nV+Ac^e<<&vla%
zOCvn@n##bZ7e2Hj`c_J-)umENzCd#v6JNQtWKFeQ%cJcl7SWu<D_sg$ni(u(=h7+q
zD4+~Zxj=RljcYbbDPMX@nPUP-&FmQJA%d+WjBMw1>Q1763eDVLk{&g^@Rt=U4^>1k
z2_8MBK}qKM_>y_OD}#y}ykxb-JdQ-&PN>iHw}aS0(LE~4aP3?w`ck7UTKm#GBCriU
zJUKGyyqpTyn!b~LwO<h6upAQw`fU?{G~jYR=Y}X!ZE4H|vJ$87OR>fv_EB>k@aJt}
zVX}%GPQkAEuw4N>=-uM4QJij?{klm~wvCig6Gsls6J9R1XmW)Y|8klhpB@e{Q~lv#
zzVkjeN4?uq)rGz{M-l~AC|PYsb#wqvOUwm#^>69H)*rWKhj+2*N<|Q;Z8g59=hwwt
zGTa_*i54#3O8*1mkOf$Lv9YwNd@-dI{z7p`-oR`db`%257d2KB^en2%an3;_%+kW0
z8+j@KJ@`Lg{Tz$icX=>p5)`;fKnz0^6Q@Oj&wSXZzq?J28Nqe}K#Y-cbab3<aw*mG
zJ`#%p{IF6_S{!${m&nND-{OLf?eu%@FE?DNPge_q-tY5XEXG$q$`s_8Nu~t9#y`4m
zPmdo=xqbLOvy`VVed<6wGt$NFV)p0ld<SA?I=uH$W1OM{)chrn?nu~mgytpEv1S)6
z&E?iJ`1959EEOg;GXWeYaj~$LFOui5450VeBUikI7d$jHDQwqA-)U$b??<JZT-b+t
z1$j}@Jg%Q5MBEXZfnVc<m?||;oela|w~(EIrgo385f6oLyPY(sJsx_??@1CEnh*N|
z9W#<H(h;c!DmqX4oHQp}A>>O<Of&}Yvvgk#zQeif&uYT<B4DVI%3-}Ohm-@`&i=jh
z!^VzFEG&U`B;1E26!y-Z11x&UUk(BfW`iyn2Zt)Yo_jVH7S_oh;}}bDYHGw^r2Sv7
z;1UL1qcg~f{pUN?Q9&ehImdnjw~z6H$0@HT@?|cNGO-2$onEh{fe(&G`ajnO3tAWx
zY{#^Nup|;ilZ@vyFNdNN^|C|<xUr`*2?=*@@ymK6HRzBD2<QP>ekcfNz-Q!*%kH1i
zHHzSv|IP$Zg78!N)}KTVYniy@u)xCdUCVcZ$DoA^xRvVV@cc{U5>}f3YMNa^)ZGHJ
zCQydL-ya*6EmWz8fi1!2aeT1FLR;%+CfyQXAh(mKuJ`u;q*NVLaN|k0+4l^3h2Q7W
zN=0(FeWOx?gY|)~3CIbA48Et9Us<TNjLB!JU2b9#{K_v*ovcuTXI%OOVlX>M%;k0z
zm)|}LkPWCZ?zIA0Nr`540WALkA;FYn1py2tMiJamSo5C!F)W-aq)9e|cN36>abddW
z+Il@+NpBt%>u#ms783g5qI?blxe=G?2AOy;oc0+>T|8D=`GT3XW_`((c39XqD$0nj
zsyd)p90*cl(XZ!ZdlHWVj!|qD+UE4wD8NbpI>ySG0zSiDBz&-lDugA;lTr{zI{bSy
zA>K33o@Sy<g*rA>Li^zsu()k&&r;+!`rars*NIS`wr#?-*JcI>FIW`601vORSi>P~
z1nn1nr(2`8Za?w!@U`w-TXqviX*q_Ve5iU?_IuzSe;5zequ_TXMi7+CJAkTSC6F=`
zUOxx)LA{!&rERd9{X^lD{WMGMlPV?=Sk_J#UCKOG$O#K!djSs$#Ac>2senl-Y<bDk
z)4?q<oFQYS5rGG!m1Ld?Vvhn0Fu0u=#Nsz9#wrQjriRK*eH_hVIjBhVvyeZ`=SCsH
ze+JR^KZF_qjEnU&d*^Okb=cX&8d|1kV^fB1m5%~Ta@yfS<CRbUNU$Y({!JihS~8j_
zR~l}|&vS+SBNbewoOYRN+IbNu<~CKmPuI_(CSSr=qX&2J2<P>9gxohQY{W9L$|0n%
z{UPg}5?8dnIPAxNbW(#K!94~^FvTOj5NeRjpTqU_^?{0xvXN2FAOY6-f?xZ2q9PVp
zkhG!!F=oBi^V^JPvn_7A&?bz^!F<><HAmg$j?Ej5-<U=p&LH~pF0o(q@(nm>2+-sU
zgALi>@i9WGr3`cnphWciw>A5XEm2y(4ZrU^+GkDuAS47pL8PMnEEX_l6aqp?JSBl&
zW($Z0#h{u;%g$btrK$u<f*6O*9Th<mr>m<xXD;Gsn+5cyGFP{$u!9wbLwpA!o8NdK
zhxWvF;ETDVBsBr_FAudS3^3D#3})IWwXS-f>yEc6V$^)520=Uvc5X<>$O3S%f<XXa
zHcf?nJUMX&1Vu@})unUzq5|6(a6WYGj3-%zaFE&qEZkXn(k#SL>LHzLknR}tx%(qg
z?M$(kaOWsOGf}JrEj)^kJ*1;ae4u-QSrY_X!wv6|(E#w7F5sgD5;`b=6@ViB>nK95
z;de_gAQZl;<AW~Z=}iJcI1>8Ac#vn2vx+E&gvdB-`gh<aUkzyaG&{F4eFO%vl{=f`
z9{{E7|8b3$+q0P_@PmtuZOm+<TOj+AD@48wtlutRc7IgOx%(yy1esrnwuJDG5O&xx
z7s)0dA>NYUa!HnoSkQ3DZGJlSprvGlKvuZZJV6D_jEAl4IVhzr9X_Xi13&w3@3#4)
z0OZGi0S8r=l_;(zavRt=01@e?&N8XTuUTc3isIyCaUVu05K;+-4J3R;$8KU+YQ1N~
zm78VB)kObwSvCN9#id~B9PBEK4j>XeKQQPN=|F=sGYNi4?g-T9wn0YkCi1fj?5WQ&
zxNW-+S9_~#$F+WW`kxw<X;)7_{}_$|3j`f_lJy0b3}zfv8Z5`-ZiL9l3z|wW4x2ie
z<2kvQ1g?6>Chf*kDHDJm7lPIHCXD+rfCV%Sw#M>dvhY9!4u_P~J|+Qa^=u4=S^$TT
zo;gGgJ^@7$PragF7z+YK1D?cs4~(e`by$AtQw+v0P=C>6nl37pT|(lM@?NpDm#&67
z`OPii4Y3S7k~={3I#JofLB48$lpIn9<RvP*Bk(~%pJeR)2tWP{v|NCOJUl)4qXG+z
zamhZl2AJr=u$_i_NH@QGr*0<S_?4)&Lj5`nS-`JV1UpfnCJ3Fc&q-`0I@aJ0T8^Xb
zn)f)ndwS@}$=~KnCptAEE&Zc*!M+<O1Uzq!!bU2X29a>qb^GJJ@71hEmW-h%9!eY3
zwa&Jo;^AQC^*C67^znb)^C&72ud^iJ1^=3F^#a)%<n&;neUnfO5bFNBJL}MaCrq4^
z`vu-#en+43)|Km)=k4Ln(|pZJ840l>{X%LhWkKiw0Uby-zC%EaN%oAW1FNr~11REh
z*v)dg>`kiz-=N{y5!C<Lbir1u4Ia=i;ypiD*?}FyK@uE}t}W05ZiCQqaJrcLU7JFm
zXgF$3bHdC*77L<xSh~p{u-QI-Wq?=NwwV)Aw)Ca@1W2frfIiU2Gin5|ROdg_%o|Lz
z;R9m|FkkPH((6ie8_Y@t?meQS^V1dFEk0IQO)4bSm2G6ADuJbm(+Iws;sA~S4CEy)
z(EN^AA4)3(b5b7|(hk`>4#SX%{|rY*AxJ;H`=w05SO^E?s<AE~BeYu^c5>=Y)df^o
zjCX{f1V{SwflRU?H!}?DaW9f=I=5obQ#7SSdes^rMw>{d+{03Z2~eZ|%n2a}aArdq
z_RXCDug)FL53Jtl!~yz20=n<ufreofszF~WdUzr_huuFcUgSH(e6AM2|C)f;xrlNF
zLi^u4dlZm^3EP28WyII7mjQP6PW-=j?NAK1z9iQ-hLm7##0cIsWO#1V)dG|y0*~?A
zWGqq-H*8Y;_kIkJ!9jn5a6D=ISCvXsTd{Uqdy0%O#n-EN=zl-52)pIkwrns8v0^ZR
zQma2-v!h7`D?qP&5g0Vh=)bU?1W6|V_)c3-efZK;q!$TJR;aTd##q4(cXY$8S6>y&
zuUlQ;K~7s6!k8Mr+O@GuO0Gsk#Q*2y4?Y8sirLHHXrb50&xp7cfNQVP{oU=}!uQ{<
z)1a`vkEc=Nnnrz7BmB-<6?*t;x`{Ao6BbP$;c3)*#OQsg)1adKae9(P0w(@>LoEdI
zreIn0Kf~n!-Z^s8!lU3Aq1Q;sTPZ;BI}3|Snii}rpqTjVgW0$^_@t0!;n?Y)E=^v*
zQmNN*%AV#bi$;Nsl+Yv87Iv37th&ANe8!!vtLP|xJ34T-FXo0pnujSJ^FR7h+iuWx
zC$r<E#qiEGTVYZ54xm0u`ubKN7lK1cUOJ|Jc{IS>Zn63^?u-Bnq3qK#?3=TZZ@|)e
zcj*C6{S4)Cx4hM7@3Q+Ikdh9*fO0Q%>p}Bl90Dw(a;i~X+#DHq70mwu6%LC#tQQy*
zy_fB*O|%{P!e*-+(OBM=$cGg_`(NOYe<eJ%Tzb*lpr>@y)Z#E!YTqX+A61T1*z5Jh
zm6YQzmJm#*JHBpvHA7cnM%Att)Bo^NyeSX&<=eNDYG)=Y-j><zO^oGAOG&zSbxDC6
z_m@W)L<)=9DEob&eLj5(tyI(Fa<Q&RcFOnmB&G*7rUfu*4$En3LGpnPbI;xjDXX`g
zDF8HBc@aQ_Fx-Qk8j&v4J0c@rQ$h$-D~mAI>(39miyna54s$rORDQO(`Qj62{;~RM
zD>eFfbC8Apg)D;;f>4J&oD9@{vhdh%GHz8RB;>=N@qOL-rnb)VNl-BX>k1GNE9EUP
z;gD|V4p-yltpHX5Jbx2EGiY|I(9FIBpJ9hXjL|}`7D&x!-!X*#fc_}cB6%~2lDcRC
zJ7+Ay1>8_4N=Y^>7Lb8Sgk<rUjs9{V1UdEm+Dwy8u+%B#6TzR_VuDG71ScUkqNw-z
zV1lW;^1Y?RxPSwDf;xx+CgM_qX<YJOcwJC##K%F-B(x|WaBhB6kcnMa^6pk+&b8Ql
z+{Zr@7RNBS;nu%Hf5w<?YxGN@wda(lLQB%E0dBt9|KlzApIN`)IRPsgmZ^a?be(Ej
zzIA@3H!GT-OMz03rh^2qR2Va=8C?QXDC;9UjK=dW`Sn|%emYT@>8mALzW5C`+wKqs
z1${HFOe$*0tXTMYzNXQCj_&uPN_YtVv`zt;zB5H5WpqL%i80WX+vZhXt)yVlI5&JW
z*auh<#>VA$dT|LkZ2NfsKgQlVEX(Lw7nTwb>6Vs8Qo6f41SF(HQd+t}y1S9?25Ata
zyFsM88v&`aUVgRrIeUNC_lMWTBfQVM)~vN=&D?X(Ou7_rUSW_8==xkTkPtS+7l;dN
zE(aj@kdcfx<lZ2e3UHrQfX2ak^{N=0u|mM4=T7Ul0&^iffrXsb5{_f#QJgPR6{<Qb
z>`^#R=9M!=lp~$4qC!dzlHtI952G!xHU_H*5rYZMxSsC4<R1?dH9MDd*r-U;>8;w>
z?|B=Tp%Ky;ghbXXQJ2WTNk!1{yxB|<@Mr?{%>`QT&IX~zUZ+4l>r7Q?b?e!BGA#nY
z&dH<>el@!41`{}L!1V`XB85}XS}sMdC;qru7ai3n8?+9Q*EnKYEyfVNjjqFk4!+L;
zm|k`02{2nj)bv3??!EDxK$%3wGLXMh2z)^Cd1A&RYM(O6J2jT+*Gjd}wA2%|riYGi
z(9z3hE=#aK8#VAiw8_8kEpP8;So1iPYiDUGSa;lUA=m|So4*%kb!L+ifO$Nb#A0;O
zM;_Tg?OZPoP>9d`3N8Qb(~$S9Sc9wY0Vm%RT&giYa;0P0Y*vJT)nE>SF?7n={;{3p
z38j!>o1grP%ln_39*w2}T!p>_DgXm-5M2R{tHC*m1~ssAot9iA>z7<hc%hvv;wUH<
z=Fhvr|1<1={`3GbZkXQh0Ux{oF^q^;om?VJNk=CQpned@p%yKex&*wx^i|7$JpX@N
z883jNoxKt;1*9>8%jJGuxi?)>0J2>y09^`#!kqg#y7_}|_Jl_h{O>{o-+rwHjI|2!
zB2?h8)0~|AMBQ)P&QJwS1K2EP=&1vw<bDAg9aNy_|9?J7s>BQg26vtVz&b8H{n>Ke
zKRZDi8y^`yv}ghs!Paw3j3zp2h4J6x{ExKU-vGO)K4S`UO=C6yJ$5;7QL)>!Up^T}
ztD1&Oik`0L?ca5yI~V%@ND9Fj@>y-}0iX;CU@<T;rz;Ia^YZe{K(%D^iQHg<PAUm<
z0O8&K$MX31PbXzSvn;!+5P$*8t|Vw%je|Zh1prLm_d^x1QMVC`Z3aSK@PLrd{_pLv
z0gPHP&u%Uf5b=Kv8Y+APP&XX~h1k}Y#ilQd7(BMGfG`%||L>0h?QRVOcs&z44^%BT
zG(?L4zq!2mxJ?^SjGPqo#nvEksnIm9r^EB_u|7%dL=7xXfl3<CYgQPbXK<rdy;WIX
zpVU3*VqdIYmXg{@t^WUKvZ5=XG}SDK3SItyLEX`N{F}(?`#R(f_?3N`xF-Lxh5vn^
zTp+)CI=CwwKA@g=3CP{0(`~8+?LiA)VAC~(dpn1NQ6Is8^Zwt5EcBxuoDuhZ>CZqD
z-$8@wp3@7B_Uz_UH2kMsm>nc&=+4w&0eLxP{^<n%S5b3AzE(Qm9vnsqACL!EY%-eh
zr!WZg6QkwfnGb*2WeLnf8<Kwy^FPvVgBS?fisquvd{%J&Wr&(&xJaN1-r3l|i|!*)
zfkUG28TUV&pK1Vj^)Yt3fpcH-EQcJNQ0^{yk=O8e|4ab+FIon|4g+^FMuZ_4A{Uf|
zBv-jC5)K+T`8Ue{gHq@NC-ZeP!ZEtgk8tR)2pgk{ZFE-PU3qhb{nu&#V;UQ(V225G
zaDhvkkT!~i2U5@6bt77m5T9Il^FP1)zXjeW3O+_GL<lC<i^R;+d8&J$>kS}W`y5Eb
z?!S#4*qr}y2CUHa9sIRjc`|KdqWY-u3jU{E05E_)n4q6vuBJX=<LDp^rvX3@Q1q0D
zh^b=}^2Gv`T-Xq$|KB&ki4fSg@KR~MAQXTNzQ(Be`-)F=6iove`+pe`nh<o(2Um{K
znXp%<HcQ(|isceq+XLk8C;if-c!FMbA(XTC0Po&hB?aeIQy)UFnXfCsBU)WiV3aZy
zP>>2!{Gb^%ql5znV3YoXUzrG+;}bE(#u7UO>J?%ZFkST=DdHSGxQbUi(l}jIj-0=B
zF5zS1GQX&98O_Sj371&)Q$&HjyNZVy%4|kjq~4uaAlMiN6{D64l>va5MmpB)WbFYm
zSV0gnF2i{Zk=kGEB_y=FDl5My0_J?ifOdh1NL;%yzs(mk5HI=k#aV(u14U$BM^_E}
z56<7>C4Y~&as3TEKl(%sSrBlYKV07djv@>PU<>~khX$$fkmM#%t(*4qQp+!5%GDij
zM8-#Oiq5Ph5P@ue1Va0yMWLbAy3b8QEWL8HlU^7G>y_)fKN8_@!Qy;^@^bf#jj}l>
z5<fs_&{zo|uo?r3^kT&ib?;_7(}@W-dYc~XL=%Dti1@#ff}?;I)d;egNR>39!5BB?
ze*<eYM*<%Rvc_9bpdDCiHxaKL0sah8bqx(%*i0XEKxujv&_ZSrQFy8ma%2_*bHIGO
zGa>p7KWPu*Ky+d~4Q*agq?3Ss0Q@o)?(XzPUZiu_39E{HMZpJ<3U+|99Ruw~%TCvO
zn=yR~&jDFIx*pvGGBi(!Y$bJyA*b|(_74)&+LmC@)r!+R4nmGYx?Vp|_4iLbrZJTK
z;Q<{S99*o|-bQo{64%}}JAS?*L#r|YV&k)4qlbu15&V1yD_aK<x-!D`x=#Y9EjYaY
zlgt#KVE)3ORO`0)pksX7?X#dcd!W!Wv~8(?lvd!u&z?eg5a9b;A?)3$(3{MvFxTDp
z&0U4P+M6{LPp5|w=cISr35|l{f#lh<$44%*Earc8$vQ}fbH@BN+k<*^q_VJx6WDzp
z$s5W`ucb*d!Ep4odvS5`vw^Ksi0U_`1>H98`sG|q3sz|=`PeG~)ZwA@lJ?}KvGWmr
z>P{wFbPNRVX9#mrkcQ210v@h^&o+^C8|}VUn~Yi+g%%!7z!JCA?O)`3@qL>8GlmEG
z;|3ldMwU?SO)L+p2#oc!Q^uXR^`GB_p1}(Jphavc!P@%S`Il&`{u#_OC^Q5USoeSo
zbn-vDo5su0Ze4ZOZm()_pnUzGtBq42G?4j#wsxT)8XyURa(UoiBN>$3du%8KjAENC
z=BJURK7&Pc!xVQTeFja6#sSO5YFd<r0*e^`Lx~W4;US{wcTdqMn_Cm4XF@+15L#T$
z1Niv(K${LH2Vk$!X`)%PH+6^C8^@qgSY`z_LEfUz3f|8_E(Qa6Si$<oD~bNsR|4}X
z0qh7#psfm8-L9It(^#Yl5QU;D(MgN_o*{U9e}*NMn(INW6NX6OJ%Z_{F%@oM1PnT#
zEH3cd-<V^T_W`7<e@1bhd3z6HUjI2v4I(<-87H4_w5NVouJdbiI}CB&2TYS!NDT97
zfc#*9Gc}^}Xh4ECRM08-wQ33&zWy@$G$>%ie_+|P+sULF?rY$lk*=J<;xGUwLaQzt
z+>4U{@6z>bOZt|M00xLD#RdZnUO0p9O=>{IIRy;`UT8x8Em>H0Q;bZi5H?^9{MJfQ
zsOFNu*Z;Sm_OoZujEdf0gMbRbM8X8vtj*1hp|9iXXTqHdq~k9k>!B(6%-8?Rf)Wxe
zSn#xP?@!Me;xWIHW?|rk0AV!nLk-VZBVbrPT0(&*HXi_yDAk!y)4VHIbGSOv4Z)xc
zksZa2tsw_9bo39-hwK_5$aMFTvz|lV&qWS3{8EdK8cA4$k{Jsg%-;<<rED3EV&UNz
z^fNRYGiYF#<-owBfGTY3g`kf;Bfo~+$OvQ!@M%(DAT#^|9tzjLPC!JtQVLn#_pGa2
z&1FSjd7l0&;h*x`JolIO^VUY=a{EJ%O|KcX))OuZ+Onf%TCv#5O$b0<8Az@L1$pHg
zM6ME2vV0i7gm@?HX)x%8XvoYh^bPO72fW&Y912z+r}zUwHw277CLuUrE2Q}e0i47X
zjMmTu_Bcui8tNTnT%BJ#A%oDjhDM_}kjFao0Xs+t4`ZzJoAIy%sg$ND@)yYKlf7~F
zIbji<JhPdd&}?*}0kJ4YF8Qr82p8>+r1Qq{d@-L*r~j0X3`93?BVO|KPBQR-<14}f
z*)|BMu(ix!;$SgmlthWZe>qWo>;8)@+?k$9GeiWdlPFa2q1ALES9&U297zlGIoVgh
zVx5ct+wgEM+ZghSN^kIhM4E*FSr>m?Xf3?j(qvv(M2o^K6tJ;*F~<xc8+$|%YYPRs
zTzUnZIYY@V;9>9H-JHj9T7Q^LqxqD_02$$#_&#KWXfMD+*0av1*CQzYV}xG++X!QS
z0KB^hbo5dKz&eDpY@vWrGm!xqA$|`0(<H&E2}WC4dD3D2oKGV(0Gsudc_G;ND<iPA
znvjEXj`_9G&wC&J`sSS4csL~<)Qu1);IRBQUuV(0M8QJdi{?$biZ`+Y2iem@;KAVe
z%WBAJCgpot7-abWZHSRS5*V}#0pA9qI>}^glq@LFDnq~sODePep;$w~gIly1@6*nM
z{l|<M{@2$DeSLuqB99&$`}?wlbFUC#OR2zg%pD35Am{iGDR>~p8vn;ZL4;xe2c=+(
z)i$nhbzN7<SRQg*z_s?zbq*(?kpt@-_5&y`T2R;#i`UW8_4<SisEE<-PeHp}abReH
zYZ4|;dq?WWdpJbA+Ly;zBaoT@fQAT@en<U1?0$XB^N1&DI6w=Cjs}Scw6=&amFC03
zBC^*i;efwQMy$17&H(+zWr6QPj$#=O3(WyhIhX=!&mkkyhG@(9Z~HTkH|MzNT=ubV
zO{9c_F}rOSgAG5CuDpKIz#A}-6CWOG#4wA;oB|KUi}s{94Fa3}sKByxru_)TW$OF+
z(@((bLC(|DGY;@*m`O07qm6*KBW!b`{RFc|I0k3T|BAP8@tWZKpFDPgNab2n<(-im
zv$(e|(1$$s`h%!2AI3uJB*R?8uxMXj9_XaMmI%{#K*lc@21JJvHj{CIfhb0N<WuvV
zSFtKXBCLK1*{HH8WKx<U{kF!H3A_>Pu68e>G;{{pAxfBQD@%+C2yh+)gtjS=3Av(*
zH9%aj-fH?gYwZy+7)?X3M-M#^Us?hXcke<9W`XrYUdaR2^#O9;rIg&)Gm~7)WL55b
z`!{a=Nso$6_Do7sORb*F@90x|2NKds=0`lXi_GuQso*b9Pv3^1^B4D9{26F*-W2e-
zzNVbosy2}pKV9#Bt9^96OW?3MFY5YdjX^y0Rr1-E40T3HgRNmwS<au%8`-O!$?Xb}
zFq!@6uEM{A8pr*|>fU{i@~-*hk`W)(dXnGblyOOwHOb`^VpK5syh$@Nb;J*asP`Ou
zV_|5E#(_ZLpKF?f41u9_3wG)T?^GbShM4A}$^{b8XH3w`?Op{AyAz;IRHP{=1k~kI
z7A^GS6Hs%O5<$<w3W6*`$U_#c>mortvFa@`Yw+@xYCZz`Yo(t&-KIVMC~>51E;%*T
z5%cS*^(08>6rYAn)$-g0>ZZ*u19I3b7kKEBm|%v}7y5U@hZ7V|G9FpAns<}RujBVo
zO;^wS))vrx#U3U+g2P3#@@`a{mG(awhlhOr?UC^JmOf{2_Ab1<3Xi2w(PMACD22I+
z09z?JI#0=IZM)EE_7$6fvuw`11t(F#^D?TSyY2dI)olNf!+9iM|8a&Xil5kUh^3n;
zd~_V_Az=FMKn^C@>Xc+s{@5;%`LA5S0@O(qz)iV?+;(H}Z#2?8Tk!O`$eszmPFtXP
zi3z6v4s6u1qY(k;3tt&EVyU}+XtDAG;y6lG86J-#jzzB%IR~ACdiq+YSkCS+^x-2I
zv#C$nXxJc)U5+5(Po3Ls`NYZ=Q9+UVB#R+W1Qvz!m1zFU{$ay3_9__czEt7kzy3wq
zI~DDTXt(~`T~@U+ID-n-%O1HkxJMJJX7DHljj_C{zqdBKsu%?IyQ;rttnZscY!z$`
zOp3~c68C4Fg<Las=POkSsV3|68o<b?6776%#nJ->)|bJj+IVmRU>jxt1)9o}#kEKk
z=0_W&ekLrE2|x0*yd7Y-o5Hs^)>PG+-O_@q9^<p_u6X>w=JXd1^R?%%m>n9qNH{91
zF4W!l7uns2fKmwisEJ?2Q&&WB7-eSti!@Kg4fR8=%$LB)3cdD1n+~~JeJzcFX!g2t
zfoAnrg2t@;ZVw5dFlx?V{pK?XdhH$VwsOc0RkdY*+M)2IVIf8goT$VWjV>G`cH&QV
zP0fN~7z_~qD`)^IVe*n2_D0BqKDJoY5%$)--#<kZVoE@i2D-BYSjT73!jM^<Q|Pq)
z@ji<?!RlGJ6-`ymv|<nEDV{atD_gb`X|Qy4`279Dfp4sd)%dLSvbdc0j*8N*1t;-%
z$7A`}?|Kqi`yMQ{96RAyMwhv=zR7K2>;6Z6bL>Mwhh!OP>#@iStK$7kKaTT}mrHd9
zNq1JygOXqaZtkeO&T>zYYveP|QKBZdtyNbhIkQoE2leX}w<a1xoy6t7wlq$tq|LGH
z+=MzgY;=8Tvbhn+lZlS<WQr#5N{6-d7IZ&&Wm~z2%e1UIL1FtftIgq*u*;`dRCcu^
z=yJVRaV?3srH0Dr?}o!^pR>H<G+uReBKrQHMWMe}eb}xS*Lgu-l)tMf*j6Cls%Esi
zJxbesag-lQcJk~BC5Ww-g_hf0`^AvLD=2tJ*y?)kW4A3ON*VGoudI25G{7IDTB5g{
zLQ0>ZSIg>XR?M-zKtiWSE23^;;T%@DlMsIcVj!~nfZKN*R8cDe5D=TkUx)fnKK;Lw
zfo|l55x0k2Mc*Jc$d?v+27Zqv#=Uuya^}B+O&p;r6teniUw+QSs@30cMSJ~Ou8I>>
zbk$9c`^7W1{zIeI>wKAQ!pYB>_k)u%k5nQREzu(XLgAtyk6ga71<ISy6h4#|7LB(H
zy{t{@Z8K<OhIk+F1PT$6FG?+v%A6ifCvp2P$2D`R?8uggs?A&k4Q;!bt89Kkwbz#(
zC`v}?6|g*7Vu(cOFs7&luG6xW>8XrNc-Y<feo0l_iBDwO8psF$l{Oghe@qU%rxN_>
zM_P5fEUT9FM~UU(OX#iA;v+T-H?Et>ylB*Q%`smJi&x^vFji!=6LQs?s!e;>q%iq4
z_K*1TN#DPMc3uW-c~i8Q!AVr{qCue(RkQRdtu_@ObR(u!9<CNt?xNNa%*!Mta$>of
z+0Eywh7B2Jx5MlBk^<R%VQ0?tJv9gY-k+euB6<*WJz2!Aj}QDHP%Yz1e$1fT;v8+a
zJ%UU}sxJt!L3=S6AU3EsHf-9vC^S{X-sfe+_qbyl?|*lXcu&Inu3_RW5p%5c%1*|#
zc|8ih*}BZ_`69a$(&sA{hNrwcSx~VS?zKc~6XHZ7^~l9us2irvv{P2{6W)@K*R@k|
z<Fp*=F>sXDz;{lqPQ+8Uu~3;D38tM<)OLl{wGIt_qg3~HBy<re0Rf{LualRlSc8t#
zkzJ?#Cdl!KpDD#+x;D&)m7<=(Yrq$_9qLhH9x8@YgQr*>G~aW-&rmNDl&M&D|JD_}
zHg+$|8nzU+?fgRZ)A6?$@fy{PG_#%iz)^>l6KwRL<lozi6x*ngti{w<!<nnXQ9Eng
zl5h8pan7A>oqVJXB)O^Gh<?*a(dfpWM#SH_Oav#<5!9z+L5t5izBAox_%87?z1>0Z
z%K>}p3*0`6?r|wa(55fWQBx-`RO1NgG&nO}R??%N{N3xECVQO2JQRMDV0WB6y2Mdg
zMQFXrh}P|$&62CvnO^X1u;t9f1WT+CzXXPguwr>&*AN_;a(Cjm-Wdc8&B3aKH<@HP
z>rU}u)s`dPgH);L*}#hOVSddFAaqFT1PEY<0NOQ{M2=8Bqb%^p2A8ztUIKH~9oVi}
z!yd&dUa(ax?F!+YwVuH@-V^;b-!aR>-T!jtw%@XkAxq#;a2d^XMNDPAgIlaq*UcZ#
z#)j{{IlA;KGW^BYWHA8a))l4CDeQCvr(){se}}_zWld9FNtPbYoW^wog@idkfrOF9
zIaWgqtfWHwx-LV}E6;QBQ&#w1JUEw6P_IE>T#4s@<M$(#V!y|V@h`b_FqRMHosdeY
z74wGuA#9*QWcsyp5@E2FZ+PYY9+H$(?fm>D?+PAmS+fG>uTksK0x?9t#ipaUjK@7>
z$rq~zYklfdHTGY0FBy0hRfnmBMHYLEy_Cz$+jq-PJ7z1Ech3{8u{af8wz@kSt5@%l
zR@ryPy{QFOgle0LM{u`5+qMO+3&wHfs>A3b{wWHnXNE4=?3mM?-zZ<H-~1GZ1($n=
z2=|QL{W6pV<U#mlRl>fYY!fR3#GU17;ra)bake%5#!&*ae)R>#4+uvjB3F4J&Z~k~
zD6<eSbZ3FJE|Pki%BYCDdfVnP_=3$%MY1M+gKj3ZT*pgDelI?E)81`opx;;QDq?yk
zbG+=!jr))Cv<453c{^rHl#tJK+$vEty2ILCwkyuqf#2a9aa1B`O~yYsOdn@4^VL9U
zC4UsoR$g4Iwfn%dNx7DU_ugxt@r<KlNAj^mMe5j9iu!b-_FMh2!1zvks&acq-7pUF
z8)083Uo|=tExEmg{I_|EscM{`l9f%N6U%+mQ9cTH#%#TW3-0+$9YVwilzGmh<Eu~_
zyKnk9beffwY}ky);sqm17@assIs<IBf7SQ+ZKC^{JkLl5Qn>TZ-ZJQTDk0FIQCw7-
z+ONF^>c{TB7skSc8TEbC4=sTN&9ojZ;yX)C4{!tg!;M)ngmk+W0aUYGP&gvGy}kW{
zLs<<sl`J1~3vv(NF#LE1%!vgE?(_M`YbrOqTYI6kdFC79PwkY$T>WHEhd-S}=tc&!
zO~Orf)ugMu*_4-iDx!%@Bz*yEpO9sOxe)Ez&L*RM#e5yg{4MgNbF@?=6{h2UqV>WA
zmPHvFloq_)jTt|*SG_DB?D+%Lx<3)rA%9VB>%C8rJRigtA*@_u-scP3yy0?smQJHr
zHNs5x!sT&V=%8uLLemyQ-Phj`K_ajP$rmuCXg?o~bl$6<OXQiI4q4fK7tg$NpS~fd
zQJ0p&c6Ng4eXC4$!@aIcqnsnA{sE)61yfidz*lU`cX4MXotn*2%6{+nB3EH^%Wa-q
zf0~1SXyigzzamZom-EeNreKlk@vqk-(!e@4n|AkrDjCk4H0!8_={&@3-g_(8QWNBa
zCmh90k<-x_3eg*vnaE%l^L-S4&DDF2V9_XmS-Sa`FKmQV3Xe<}_>A7E7^X9&_@J^_
z+H({_sh@$UPO2}GzUM;>o)Fw{2$m&e3Rwb)zNY<ZmhZzPo2E-~BcBNR)f?b%7GHjw
z86Mc{{(>MM73HUx>UNvP>kvoUVM{lsKVmj4qFAg~QplzL5T(mw>mKZcny6HoH13cn
zia{aZ@gT#Z%7$qykBDK3rdNH2Hk`2bT<6kFC{xgDvhEEDz*dxj;j31W&+>-YY2B5Q
z=3R*Y8I}}IxpN@GF_v;{M0%5O_Z#GwbuZw#xAnIn`e)0Z9R+RJr<<D=4l&|3n%Tnw
z{n*Dy@}m@P%WqMu$AQa0!vYxiR8JSDtTYAzue)nnxHp1~hS@elai}DIUr`&s(F%(!
z@prJueJ?+Y3EdMQliE=-^;-GI7M6tj*A|Yb<GU`ZJVo&J4}Ydi1k6ZE1U`?yaS)8L
zH<AQ`Y1tlNVE&{?gE;!zO(b~8K+wN|_nS4$@Tj*wUi)Phf7OtN0>*AF>~j~veSp<h
z5lo6XgrS}l%l9hdb(q3c7%xrMXoT1Lg7?*`2lU5V8<TKkHAu37sUP%~b5sr;s$%n0
z>y6ai-%HspGX0UXY7=?+8~cKskJ9V1wv?}Bgu9)_aA{F|YF=QZ&#PtQW6lraiQJVg
z*<gKMw&&;~-Q9YnKl|YThrYH8YO2OqAx~pSOW4;>ahu_nWCepgHMqyVkua(6>mIQ>
zv&bhkiEr4h>=}NoQ3?5Y+C37GSAv36Z=4T{cykp1myT`kwpZ6rjYa{QuYx8*PX}HS
z_FvK>?gBra)EHa><D+Ce>ot%C5;`8cITkV=vo~OQh=ZoF4q3?tbbuZ1>#w?q^71(l
z={*1|$F?#CpTGNKFxqw?PwB`IN1>{HiqmMIulK9ce8f1m1+;DIhOYfwcbdAgkeLIE
z=$IGFX0Od69^uz<%KUON7yozRtoPT~KTh1t0%4n<#l+V6b_uf9t14E&Whdje9TutX
z%h<)PsdD5u`;x8#ODjC1RvGrz)774sQVvbqx?C&0@8$GS)Ji&oEx*>-)v@5t=%06m
zM~@*%M25Li=)q3PeZhVP8WzkdcOPupjocr`+&*yOalm(TP)cFNI{FeOI{GJt8bb?<
z<R5q|;-UkWYG3dTu*6(6$$?Fb`f$_>G$H+N1S!~PD-d^B9`S_01VknC0c17_ibfU6
z#M4>HKu^x3fvq%*o&y7cGoY;&a?)$mL(C}TQ)A2>tkIVWHq_Pw!S*^#B5y?78?#R^
zy*p#k8%Q4hS(pR7eAVAptYQy!v9T9R3aIi?eNh!}7F!?V_#W>_!bv{;b?Q|1g;gYV
znQ_FrzCcDMneu<`I>Vk~0dDYMqvso=u}d@xK6OC*kr_IP-Q?<`Z=>M%B1MJomXk`Q
zH9IH1xFg1TndX;fUuiE%k(>oKQZLo=EJ_*OSZVMUOf$iVXao#8ueyV)hgUS3L7cJ3
zTf@2ozIdF9yF`@&IWg4oAvJZ(-v43yJYBmTca;h7HDg4=QDa#@pq^CLT)j2PyFw+O
ziF|okfMR6@p_4b!BFbso1LKuVG7LqCv<0)lWe-E%d`Ux&Tsj47qlR}UeCI@f$pl{z
za)orIyZOXj))TKGpGK6unUGsXaU}v_b3Yl4D;#Tklm0lB23<oZc)J+5;0}-d$uX$#
z4a?J_Fhu6<Oogd(ahKNpGer1fV;D=6{`m3ub<9o6cbWL-kJ5a&_9A`@nt4q>O-Fq~
zsdS~P21z4z(sy=iIH!uu@U@uBN_6Di2Ldcqh08=k$dn5$1$pcrB5bA|ru98_n2g#4
zzgm!}zKn-KE3c!#)lSQ}9*)-$=vMvk{kDo-f}Ys_?vGbX1IzAEM^zk)$J<z=b)kvm
zw-lT%@}Kh=fLV4|HJEoE*sI-AvL1;NYxXPUCl;#&o2#dZKz9)FJI({3T&0gXo+FYK
z3m+QRx(Zt_vJ0&I*d=sGj5T0>KVq?L1Q@R^sKRCV0RjC$1C2LDIte4Y*~(3VP0MY>
zfcmLJ5?m@Sco_x^oD_x~-2VcDy=5^$M==%<atH}{i7&FWv;@K68N(nr+&Y2xk?@Zj
z#D!*?^d9yXXK)#>RkdE#vhS4HZ<DxO<7ELQRjxl4vwX2dy?X44++`KORb@05TToFX
z`3P6*`cwHdiavQd{J~UH>)0n64$<**m*I1QvtM<?IlW9pGOg$iH!1BKs)A6bM@aTZ
zZas>><UCpvgB&4V8sSc&pG^DXNiw7EB{fPHzE-tblzxst`(a<RrQ4SxVa$MfG>@v2
ziSNH1w5s>eY{PtWnk9mWkOw@=uqig`R~yTW+$5C7MTE~+y$24i?%UUA#W{yx#bxK!
zG{0${;%#$WN8hBusO@=qYEIFeR>)NDm9VDhp5PyTeQLNHUw_%9MUE30yKkox1{{(o
z#m4}()hoU(s}ASV#5}iw=Qr}d6@$G;-o4qV|5Erdl+^ZNf}if)mrCcQ7h8iugmF43
zQ^QxLcX138_Rg6c^!Ku^H?*2z1njmj$aqaUVd>p(JpXW?)=U)-s%f?T5wsoooFTw0
zK{zOAwTv8RsNoz!I@j{X>~X6-*zXH)-h(A>tVpd=RRM8mS5`mDhHUmrvLXaCUTx_7
zTbA~A<_ux&3e}?Vtf=w>FxDdKbRMN&@SNkn!a^M^NHpV2%iq(C+eV+wHZ*uo<9m_W
zt7pLd>;f5pU4*b1f?f(mDj<ii^mwsZ5~SfcM)(+Dl}iJyZn2~>^aA!kYGwSAm#x|n
zcib=~ExBm9>D!M6$AiwYRA17p#Y3;#Bh2`cUGk;1?GKttQjBmqpOl3awtNi+eEoGI
zmO~U{d02GYZ=~=7xV&ytps-@blZ|*c=$5n(<W|n^w!Dh5f_Iue&OJP=!|QT$jPJ<*
zeU06+0}w$*MvUG@kmFW>h8shH@;4_Ty!YUB@OHMGK#s;_6pMuz4#5zsu@u=~!*$jI
zan^@ioNzDP-T{)C7Z?YBvu#|Ms*t|^ext*TS(TcZLwyetr&5y$j&bzYS+5owtPqXs
zn~X9@>g|CE8l4CrWf_LnX|K(B7;}|&ic>}`H!2G_MB<-E_qc@y7J9Dde{!s(f-!o~
zw+SnR6+D`za{-t%SfK7T|60nXfL?9doF~B91crL4v7;THp0ICSx4t3rLl~mteSD6o
zqU+6GYvIq1fyn}CIz+kIAg_W3<hBa>a%6(Jxjyl^u$$(h&}~BH#M^;%a^xzgsIP=L
zV%Jxr@8A$Y1kE>pQ&30#X;~MhCL`+zEgFlqJLmbH!gDiA7*hZkuwjGJ)6+Y<xX1+z
zuSwA96hJ0_?Ky#BT{w&|Y9rN8_z^g3y<6}RFDzDSx$ov;iTssHURhycmXN}kjzC9Y
zICcF!txFG^#7!SUItay#;r~)uo8Cu_iS}e4)~S@l6Wjbg*DzY{-}JpC`^vGwT^EPK
zxgM&SlE>kq*gbfyCzxz0V%s8%lB3!QIMxc8&1D+3V$qgtwjX8xmI+a{y1PjFf5;Dx
z{k@&965#Nc(eL`6gK2m8kD0w=Nd4NTIlaQmi&9B`UX8?>3jw5rUFt6HF&&O%41H%#
zp;-a@OR<cN{<TS(<4;o!9}tU`ENF<hZPRBM&vISaL-#IA_zddlfy+6&J~Zp1(LeL<
zkB7E0hvAofrFaO)NEjgAYZ*quZ{{~gd$K#Jp-bdw8^@`kb!w?bm#C3wJt#bq-MUk}
zEV?&a!x8?_oUE)m)DF<k*O}}5l{TM})XN1${3J`UUi-R-i<gPquufD$h4)WsDyFs3
zu9fICJ_wb+b}T+&?P}*7=2ghY-j7JaqPXVX5b9FR+m6y5vI!yP52>BZrvGC4Ubt&5
z2%Gg)t5>bs?DrBvNQ@b=ZyFXcLtw@ZgSQ6UnvZYw8K_Smae)gk>dzqg6f?DKWGIAy
z(^Amfb{pR;X3NJxsJ%$NOe^Bxl}95a27BZ@%VL7YEUh1V3_Z_ljcokpgSY3c9`}HD
z7M^%0zTXD{PZ__#?`U7Du}2jbx8QH`JzXEs)0gbL6{H`p=S;5LWn!rE`oBk;LE>`f
zmIp3hjb2rfC9l)uUP_7VPEe3J_I_mS_w_#xO6Y{cI9Ai=l%<#WIGh$T_s!hQ5a#ob
zSdDD2KfR+>@YiLoWfQ%MdZ0N4^~J#rfehIpM_-%(Hz-&nUUJlx`sc2Pzx`(9DD-2!
zkX>M6UOGy0Ui}^e{?B{((XQXOwuX`g7iO`DFlh@BH6CZI-OcZzb6w7llEW}e^7hG1
zUNId<HEch|-H`}TJ8CD8Yn!*uq2>1sd}-hd6~E(+CDT=@dLzO(D7^Uo8OwHcnA6rZ
zaCtH(&h>;|<XiRMC;mpM$b>}Vp>4&5MI^i-C+d!VZXeSEzy_WPPa*laeGTkG5G*y7
zUTJ@P$N>p`${|>^eap*we?_^g6Be^RqmlZO`V;``Z{-p3Wzm8<0@B;4!inE)E(6%H
zupxJ~4l82O=~NmxbUHO@6e0%&w~1DL;@u_;qa12Q9-^&$Y5oe&e8qOcI<x(=NYQbO
zgfanSx*GZvhR4%+AEgxR3!r?;fHTsSjDLo?w_(1%_Y-m^AFDQ&)dKa@A*Q%Q7#{jk
zK;&tFBslDy>$*Ragnt~tF15SL{w%I9xMErc4CvFp?+SmNeEU(W&bbzV4MJVp@`31s
z+{Z`~aL^-iLLe|UAT1h&@H_wjk8lydWgq$n)4_n4ryLT<hxg&M&i#BjF%kq^*-4PC
zNdw5`BP`5eopr!cusOU9{pI8R0=j)Jua{D=$^FFt;+TTJCI9$*sgia{IJ(Z%bKqS*
z^XA`H=5ED|xk`i)21xi5TFO8PR-%;IJAzS|G!1iX=v!V45^S)(J_PezK>%ho-#;h_
zTUqNbfB|WbmL%Z81JYjNL8}ZQw?7BA5yIf2QUn*({<6^!8f+=W3C{@}kzRZ1_q!UY
zim!OKMymU#_cXH6Em)LVYONMr%8);H1%U!Gf-0&AOJfeaTJjzr0WjlE)F-k6Ixu|x
zKg0ks&?bJ>n3w^s(^MsyyA5xzuC1Rk5P@H67_G1O7g*1SYiZSa=kah%+^|=sEKR7`
zMY<g{B=~zkPD-Og=<HlkOz;QOZ?9{~x1PS2P!b}3cGaMo-zi#*8t@j4gNrJ12(WBq
z-+qA*zew!uR|ybuJP=!%$diquRwR~5VkU!wg;fH@)DEWJra<{90hbE_4vo>z`xax4
z&lc)1rB3Eq5FvXA)<ToF&2|^Qa<*hnTM3nNap~?NZdCfGqf%f0B;uD^j!lE65TA_*
z!gHy9PCR4%Z#sapM*l-iAgO^@z9~BR3r|=+e*XU1S&atE6rL}|_pc;mNzq<G2mt7I
zA^U^R6|)Otl-@U}yU3*ShN_0q@*%I?)*sh;>|wZ`GDp#Uw-73-F`06|t{>IOYQ`<c
zzr>_AxU;FMS()%zXDb;*1Du@1B+>NtdFAA-Iy1|AtBbzl@;=!V4(v;9&ac_M5oT<P
zA2q>o@!u>}!g!*qWO(BM(gqbELHv&l0ej5rhkM>Uh4d6?7?@oUqfjeUBHJcrme|&U
zz#^fyzuLu&T(8#Hr%A?w2fh~)G4>qD`Q6<;kh|a%zX_N!o?#xixwTF~ja+9uNUQ2B
zB#9)%FM9lICUG+ocY4=h^h&paz{B<S!^<~D9cfb*{mvn1EWT*unsNHkfvQU(-9<mK
ztZl<c{u^JXfa2R%;lY$(k*LalF-@SJK*^&!F@q!06;L<9@|Osp@&7yU=vIlBz7G%x
zbjG8iH!m<O$J*@wyl8d33$u%(ld{F296R{n|CRJ~Wsl*E*YN{qfqgT|WNWA=7BOrv
z?PvJ0OLCL$xt#7WVHghif-YiO2m4?AZrGpCbieUhukVY)mE4!}_WJMu#W8g%Vg{jM
z*qHx=8^Zj*a6?Aqzkw8kTF-k|P=JO0^6>X;BYfz1BQe+ut7{oe*`yuTtB<->bt}tG
zmx6JOit58h;+hB;0kOZhJc$pJo2hgVFnpXyChnGUc2?_tPkl6XqWe{uvgE)uS7*(t
z*yQ%`D&FE<qPffWtUl$}`|%yYn_#^(M_y1qQ5kj-O%2AXKTV)flx77$C_qPYLZJ}G
z6R)6h5-$Q5lMgP_2c>~H+FgL?T&#4!$V9)sMpoM<Vwc!bd0P12h{B!Uf5Qrk@V6i9
zwT~wsKBj~_b@)!4!z<<r&E8CNM!a$bf4H;y0}%njm9c)32O0GLvd_*jlHMA>_;S3I
zUi_{^0#s_TU-B{hSPnt*RTqC(WVVlO!cgxddp)x~F^bel?iYRW!?K9f_HhFFtliyD
zp!HS=A89Mw!KFqew=b)`am2dimWWp}(t5D~5=@^lyrDHtaQO7gh7ZXzR6zZ%A?~&6
ztG{Qbu^XPd#X(GHf-Z(KiM($p6^!UBt5fscaLYp-m-#%%mO8gFJ&^=m0@#`!Fmk0t
zf>kId)8C-BdH$6nWd18IMB3{1kYd;sPPu?+Oq&*%-4j$!U(CyaMNyqdFLj`@JF`28
z#qh~NhTK*modHHX3`<%=QsA1VTN$CjR<Q{udSZK!@w#vJ`n)?wXw{KUq0>2z-Y@Vh
zW~qvum!i$%hOVK_`PzR;z)KvWd~$bHRH$~BdV&bFD?B!XA2(CGMq|?6;`Gt33A*3j
z#dB$1PToca9dkEF4(NU_)Ond{AO@~RJRiyWrzj#EgdQ^!hUx3iKBWW?^ievZX%&Pj
zxJ~`S9v*6&QmIm3m<Le5%09HFfP(2OEim1EI*GuFDQK*>EX@4%CFtWvuRPBD<p+%j
z#shvC%OFnNsT4S$_qqX5%cm<za0^lwnk<DXy*+A;qU5a{@zjb|x06GR0v89K-fvUa
zD5a(31Fv#*+Hq$Fw_{^duO_qGdsIG2!xqk!Lk%CYh=eJ=Kk?$%R*6@Tk9&Xdx8MF1
zXi(nDr%Elpua>)&!xzq2HJDQti;LObYMtcVg527y(iTGbk||mx%9Nu8mq@GaJqCr<
ze*B;o8_AcXJ^68?-l@TM*K0O;3qh%kaKT1U5_(1R1)(b0yO|4>(y#cgU*f6F;evB4
z94GeBS$9%JG^2ipVl$ykA1?bO#TK#U+G&&V^vy~T>c}urLYXLBt%}S=oyX?Oi+dfX
zP-m{OhCyx(@{=7v`gsOhOZudkZ2vdKlm^SF+a%_4wvh#kh%p9|6KDa;Y=m1WY~bns
z54j!WeI+ryD|1K1*AAE2u7Vhn%0;%wW$SZSu`m@r9!Gk*Kg*PIzpd2NaGc~EDrkMP
zhQ(y|`Yh4$na!9f@D)+ce?J-#IN%zJ$?b;U@@<AH$C)aBtvm&83YqD}dv@u0xM+3G
z=K@i1h%S=C8->agt81JrTjy>MIkP@YVlFt?aT4dK)=T4B{e2_5_a*gRJ)R2OGT7O3
zh-)E6578Bd(gWI64M#cx4GoVC6fM_i$DLBKtgMt#0>>Hc3aGi7@a>)tSO<SAQbqb%
zpC5l;X_6849d*H#`g_OSRgSRr=6GITIs3MATt_UiQzojx{)kTAbVA!OMd)BfDvpQy
zMwVCq2>b(qg5$ZJ)`i50?csca689f#@W;fBmB)p143l<pdRpw3DJy8&^Zn?o3YKJ=
zHVY^b%RX4&A)1N7>jE%BpFulPg0+y{d<J3Uvb(n+6_*CDK~BL+&p#<}r!ZfO{Jdi^
zPA)DefX}x#k%t7bAcmzSgAE$_AQ3FvP~1QGq6OcH(VkygolD3ZCD`rt)R}d<&-0tq
zadA>gS*BL|e^=C|0(}pWB$S``^GBv_G||7kK1gADd2`18kT%g1T28_*DLs*&ON`Wu
z)-1Eub>#WUiAZKPA}v}6R@!>}c*-mE0}e$4eY^3p#V=<AuI=+gC`B{NpTvd*l2XsD
zEqt{e<JJz{-EV~zTUG>IO!_v|?{A{$q??7!FHv43Fuctr0TT9Oa!NZQ7ei|s>A7#@
z51jnUFDZWNlWMZpSn>kIy1wRLG3lwMi6M41Y>-6W9ZmFUrFWa1J_pupWyTq0&ZxmO
zFWRxU!Kv@|`?fFq#lK48(c%)2Q>3^lX5Z;<M=%!$zU;-YQ5S}O&d@6WpLOZDv-586
z&#e1=(?BSn3gwG3uJUG5A7pLCi6k$MiJABhaa_uTxM^9Ta?(hC2{LS&?eCy#jBHD%
z9sPF$<S$98!=Q=VbKxeNR48^IkEJ5WS6DB?c7_TP6Fqppk{ak`m?8u9!~sp%1<FHB
zU2ODY^4PAyVl!GEV03>cRw96<rvQeph|)Z^k*ICOU7bHcw&5k+Bni1I{+F2wXYzio
z`<~oEDo3BL@+i7GYpFWE-#gRBvl2_2+7qcJOht==FtlEqDoy(p#DY1(#;Kw99An$G
z1thZG$iIGCI8bneMe^0k9W+I~8T5rcX2bjLRiyx6<H4igYxg;y?l@*~m7AEO$ydT6
z8o#<a#>@|CPex$SF|#ILlC)yT2xmuMWvVrqA-C=m^7?HL1nVkIA5E$|@t49PE<TeZ
zyyiH^vSs7S7tf5xl#fFgh=D^Cc%xLuR+r8tq{G$Zh+9RfT<D5#&QB)Qhgj^BX&?-=
zxTX@Sp2rcO%RJ0Fvw7TIe3fwJq^$VxrE?aRS*sb1M$7hDI+sVRow1_mEr$!>x||+N
zFunriZ~^i`7l;6167pjAYJH}TvYH>N*|w(Lw_e-@z{FIgf~^)uGp~4rCE@aj{HdwH
z=O;iUWtrCjLOt>Qk-_g)2>iA<N?Dw<^K(^YOmX2K|6msv{@t1j8tHLR@RG4EvHxNG
z^CM#+JAC4nMcN$m@N@H@Y!$OoJk9G-LEArUi8i`yY3HUfkMU0U>&HKySiC>jd24~U
z8K?8MUmAzvdia)B<NM&RCdGE6iHNT=!U0E1gH$_**HK8nM59(3UJb1jY?KF!N1ZoE
zN<lSJ*2^MYEHZ*f*U!ejb%WIa7&Y8Nm5_ObS2QL_^(qGFq9UB$kAf@SQ`UMdF^)WU
zQHJWJ`8gI&Ds;4HSKi^1$f0{9#=^e37RZ7PVqI{nq4XR#MD3)|F($Mz8XCm#2BJXb
zb2g|<tw-}t8GCFblAt0?-pFUV@!Hcv|KE)#Z#}~%-$E~uYmC?6S{Hqq7J+a%V&gN~
zoh3KT|BeAZV;sMQIhjnZRDOHt$Vc&yJYi2RDJzJBVI&cS7?pj85yQ34mTs1glf#Jl
zT~|noI7m!o;<j~1RIKDi)9QKdvo}Q}SF&}LPoqpKljLu_E%&rCj%})@DByTdK|?6+
z01Ft8A`|WV{rh+RcO`sSO9(Zc%KcBFyW_S&$}DT&=aFw^HD9CWDs}CByW&L)OkD#z
zvYj7fC~38Z@B-fL9(}dIa=s$lq*L|h870haZrjWhueI-aVf&%>`|e6#A_^gO#I2I5
zolow|FrpOFtOeU2Jsuoq1XLhAu~cSzvx3M|YZ=N`vHx1Y*Pt1IhU)}ZF>A^?n2Z&C
ziHc+F1PQzpbLlhfgB+7(s)-aD?~cD1#h}t-Q*aSW2GU-aM>lf+riMi{X(k_G%}_P5
zRxUDMy4W1#+?#89J)AVQl#6$I@r$6zgGp)nhiZWwmX)qxD?oA?_1s9eQkk51ApFnq
z1weVBSb8guC&*zuoHnu3RAU@h%#-xM>zUqETShNNZNF}$oGlmLzB_OUAIBg|$Kzy5
zC?1K^tz)~voAk6W=T8e0-p!y}vT4Pr)5hf3(!aeW5Q;S<2f~5OPQUWbVKQ2>Z&5Rp
zKkjZ8EWh~pmU|Y$cSGIH%dI7rrZKkL2AO`8<}|Hhm!SzVyd){?%n!e%GAKeZ=mpw6
zu7UxApOP@NN2S+U5bzHq!qj<oj19QtbkJaTWrN`VIH)PP11dbk(Z=pFX}$~3W=7D2
z7{PTL2kO?#zL&M0Lzd$iSr!zHm|!(mv9--Mql)aR&;(g=M@0E?Bd6ovU(9Nv%L*l;
zcM5Mtgo__QHA<O6bl3_JeA*lwc60VyIY6mtHmS&#Z$lDOwZ-Tz7;4|lZ;-|axyIji
zyq8?a2XVr<@1SRKbdAbg&;^@~z0%6(Bbrj!Co`$KBeqM2dxZ;rAtIHHOg!E<Lr=gt
zM)RUdEvj%nk9K-zNu>5G2`Ee9>wmt8X@ZB2RN4IoHG93lN~yyge`oF}`qOd@2%C}A
z9kZxTS8B5}kh`mu6z3|h0)pdYH0CVj`lNjoFX{V<C|&fb)ePk8G)9v1ytH3@VG`<P
zt5Wk-+i7#Qm4J?xvp<qxP(<3X4hceFU$-)OJlr8Owxk_MJaIS`A^9f1lyru*7@T5j
zOqrVYu9O7WZ(VK|)Bv41`PY8AyTL_n_y%RSY7hw%PHR*B7ziV}){^2B_M(_6cr?$l
zy$2Lv25buFAz5{zY@GD`pw-9b&j|6189$Tr*J6yupR!nDMv<uCpjTsl0K~XRIY&Yb
zghqKh@0=UbV4fd}KunUGVZJ(JL52eY3M{<mY6<Wr3Q-L%XRqbWPZMco+~cSzWsDyp
z6OvBXlmN7_No6)$8ei+OCbC++t4JH=C`Fq!RbG8K28;MO$Y~~b|E=D|l@XuXALi$u
z=@%GRxkTugDXuRP#k==4vLu#tY3-&~>O~Tr<-e^H{Ly5P3M-zBC;reZe?u4l?N6-V
zOI^OLN;HbOyonvx#hp|vowVLby8}IupeTGRJl@1VBYGgs<T=KOAmH(y6BLf0Bv#x<
zrqn$KW=lqjn4m=D5ypfP+nb$7Osf3YcX3S=l(t%$BVjLawlxaz8i3~Kc?UW^U9Rua
zRPdf%AKe$kk?CoSZ=Uj#7fEFekvSe8`NIB8W-}tVd{r4`*c+n2IjWGgA-x{h_Y#*#
z^a(**6^&9t912)7pD}Mt1rdL=3b(X2*>Wy!j+X^$v=}N)Y#p|<>P!bjj7xX8e)Z?5
zPPi0!Cie-Lri)waYNXcBdFs4se;*}Oo#z;T=YG<dpOV+1!71}YoGN}7%3-}E7fY>}
zZ?#x?KhkkMo({<o2@oUn^i;L$Wi4{i8@KK?^HR+?>kjMDSBSd4ID)xk(9wg<_>}Wy
zHDxnwFJ(~a)qLQ=d!b-{m83W3aU?0E9V~CS9N`MaCDoPE#Cm=gf`uXIk(Ri2v7i;{
z3SwKm;32<K`;LG?n3$zkCDF+cvuXuRS#U}{tMbyU-uLZk1Tu(s7>;_&cksT6GrrbD
zxj2j_C%@Da96qUDy2)s?5OWzJ^wbYl9<xEfaBb+kM}!Tw@j^NjfWpD7#@SOqC2swe
zkzO*{6<wxwg$|<Fl%w+sQ@<-~nWoD>XnrAxD1IjF+ZBN-tHaTM<-a^IqEpGc*mmnb
zf2SO=CBI9r*J%P0ZUAs-r+`MG23=F#z&~S-qMyd>@rK3spCO@oJ2nIDa9wytZTlp;
zxPH+_``juQW;BW*;^!^jY8!cxI2F$e9J2r;W_}18If?oSckASH&;jW}HiSNjOnNG3
z8~x(XX1u+h$)vDdBO~;NL-YH1`vU(GF07J!<J%C7*Q1}O`=bI}Nn#cUrmwH?V2+E!
zJxciJIlng0>f{pMA-kHvd@!-AKrwITV(5`&ljdCzm%cqLlE*L|jG@NuIchQEeGOZ`
zFKj4`Wo{%fZ~lkr_qu^&-5$OErx_KgGRJytS)Sbl!Lkg!7kQm^1F3ZRgW7_hP!6>#
z6%a6TB1?0Dr?UqbbQckRy94b7B8z0>=I(-j%K{#}c1Dl6MeLi9F%RobFSC88eIN1E
zp6?vhm}BIPrpnJA*E8vvX*K(FWdn*}*H$o6t}bl}jr4rnMvX;VIkGCe%&eL~^R{XX
z+uw&#+$$1AYL26B6hzgNP6Gfl**H(K5uZ&=2L>&F+jRK*18=oj`g0<<O$9ygau?wl
z$JpB?Ra#!Wm~L?RyYTrheRo3;26s$<I&vg&`&299(8Ji~O4>pGZY~zN-=tB$hV##v
zwl!x!X^bP+KCwcJR15a^vavmV^qR8>geADVx_6WA$^qCOeEggA(6eo6>$eJl)sNav
z8Dqh$oZSt}t&jBKOf0ta054&$y*T`hI^VWkYSY}+Oz>huUUBH_x=0~9MMGu>x%$M(
z>3)}X4bw3r+dLfRyr#|K!&CMOAsr-tu$0~^vTj}l@a<Y|7GIz>({O$0Y!gikGE&B3
zN{a_9Z9nJFH^<lPJXDQ0g9+U62tkBCxVC;>jLXi8o*36D0`7NmyOUJ$ZANcXkgH2e
zNA@Xtk5`F^>M2+*4x1xYPBtSQc8djd6X!SC>+Dly*mOgjKGLNFS_aqA&-29N!oph2
z@{i+|Si^EJ3EdDe=wDUcwn12s0QN6Z`PGuu)Un2eYQnUDe(rj9!G7ApY9dEAW4?v0
zwdOtb+c7hbSH+S#Vpgp|pK4W0DSVKsF|zvcJj?G<H;a#Za^mY_&)~1?DR1vjklZxn
zBb*diC^RK`NWBdJmQok1<h?kYs}lX<q4o@_851M`Ipxl<HT<w=OJ8z~dMyzmRz@i-
zf&zW$wMhjail8~?(f&A6Tgyd8EgqZfh7YAyl*NLTtm2W_qG{)h{-0G7f)-Jyo7r()
z21o{<%Vz_FHl#GbuUV;IVwy%}qi^l*Px6~=t6vFqv`)XW2h3mpf$jaa(q(<;6y@kh
zA%}iJa`E#`{djTK<W3^i$8x(~36+E383+g>TJjwes-7E1?3UJ#764=y_rIN*yv8GA
z@?|LxOA#D}Z4E}1Qx-oL7JjJLgQr_=D(8Bj#RrMyNy32%*67>8D%1s8Znz;vH@(|e
zB>go?N;i16hRVIW0z8~5K2F4AQ{Q2BH`fb9=uhUE<*l#2+EGjXO88QkF~u1wPywSi
zhEZ|a<x{}@l}d`nDcnJiORLL&#nn2aHCHo+mm%-ANV|Y(vP<yqiE*%j%^k{vsP+Cv
zca#1GU!rZyiqRgbLOE}#E4<;AVnDON^{be3Nd6k%Rp)z&w>EPxUhOt#RDYC__AGd)
zGS>v=4<SvX`efBwnVWF|Z4BkNUylyCui}yDYPxL!IqG6`=x{B`h<&P5OG+aE`6Ha?
zqwHJ#8`YGl;N1yjy%5mD?>2b(?xOFPw8qQ=R6F{BbpTx~KWwnu!)Rd>mPzMWq+3jv
zt=UB3%v7zXv~l8E60)|7>>u<1hMZ6piMkLt)6*eATIEizd`8Tx_ka&i-exSj8RPhq
zj~a(Z1j)S2G6}{BY)RFR>2tg?c&%}>wg=9Tl-JT^nY6{3DG?%U*n=u{Z91vGHObZ#
z@!bFlInZ&m0Ql1wdL};a{ux&HdpsWH#q)eKn+XDZiJ?0l9EG}aUDzSmtSKMb+&lNJ
z(gk}a&T)@7=vx&({B}7kQDbBpzBv5mbQnsZ#c8@JtL>OEGZ=|a{_2aF#(V2DJ1CIK
z*?gt<ZA*plc}DYFW4X+<8KzzGu)6_wfvsno(*)rYjeKe!BIyy2y}nn7fP#l9_w85Z
zAX&@yw{(osx$ygUrH7N-hZrtdaCPeHaYkZQHs>t*_zN7gt8+J+PpK*C$5@b*)yfbd
zP#13(lxVB{*pKmd9d&S7;(IZx8Z?wF0Ng75W44TmZx*qin>u!M^=xk7^X~+H>~nE_
zR31Bm=|{W2`AHq?3yZ4Yd7BYQE)gOXtXdHD@^*P=dfjC0GbIlGoD>Owd>ON4TizG}
zcZes)Rc#m5&!0AS97Oknos?`q2p8_=0ObV*gGQW_RNwYpGQEen;ZH$=Iqtmv<mA7l
zDTSC^V)-4B_eisoP2<8)tuFQAopI8;hvyov9ifj6R~BE|{k{0ZA6|S*-RgWT_VHA~
zk0K_u$SubCRJo~#u|Sh=rK?leqatE0$%_UKkvn!!GS)wpy8f>FC=NjOu>dwOT(HA`
z_=v7=d4JTkhE{OhkD*)&dm7|aZX**-8;+$vFQ_K$D^8g?@Hyz1r6AW#b$oU-^K0d}
zp#uq>O0#d@Hc1NsXp=7bdS#10y4O%KWXNpyVUs{4&2W4(4~r|Ux5uh!PB-zbyw!9j
zS7pPH-=bXAPDfF1Hj8gfHDAQf1aCl1@_q4A%izpr%7P_h#+q9gaDQHN=xD}Hy})47
zYc1&9Z1t~7^9!0OQclX2eIDfz$cVJA0m;Xi%RrZa%KarZnhn!VG}w$^o{>bc_Ua_*
zy_$g^kypjx@*QV9EodKseN^Gr{%z6lSkOHmLI$!RY>53GjY+WVOh53kg!FWE9hdHd
z8L4iIUFO_%kH^W~1Haw7AZp8N?uKbeS3l0sYHeE`KoW(e*){oBEdVp8pQ0?do<@Hj
z{7gWugR$XK+TJXkj0KKsh|wnMl8Z`H3MUc&F17PXT7y8l+0gx&>xgtXX~*Vzu9d<N
z6JN9Is!M~+#LzcmWYW^*870+sy~GxEfY=A$*Ic5-K`Sb$pi^+3cofWX&NH}-&-9@c
zlNZ`q-tAVus)6dU??W#2>HMID?eksv$J7{v0qf=VC`WpmJ&HrCBm+aIpCI#rg2j-q
zs`3Z#TTPlR#~EvWvKi+8!`@p)Ro#8<!mw$iyV-PuC@I|?(nyDdgoJdLfJh_KjdUZT
zq%<NR4bq`VOG)Fqe)xZ$_k1|d7-x+0^?bW;H+${1=A3KJwbq>1HRJp~qXsTn*s#Cf
zeh>?Qhpf}$Bux*CRM<VA48Xpm)Hy}un;FjNNp&6AAP|?W$UR>QBx)fl*Zsm%)gTK3
zcYc3JP0xr(+)yWd<AcL9nfk9z#n;|i@U30?m;Dz*XTLhEW_q6tJvD6H2VlSY456Qk
zL_W&in7m;q6t~W+2NP{*JlhzG&%mu7CAmiDn~L&2)y4vPlOFj2`&`G<|3-07;{QH7
zgJ6x1Uz72&-oFoy@o`nvh$*OuTu{+!Xt}Oa{h^QPXyRI|=f@Cha?$X6CBw-im|n8)
zJQZCG1*{!&Kc(E7d`cH@#kedCJ-(>AGJcAwJ)Bb&HUHqvtmKFY)`;U@eX_0}4|ZRc
z8MJ-TDb^-rI8&^Le_m6Vun@D~8%Xc^#WORk31@jvwr9WbOtHQCTD18E;a*z1@rQfi
zHHVphLB4<p_PU<)KsjwbfNZhlh0Jc}ibE@9k#|_RPKySmJypZ64&C`*XEyw&+krPH
z>As|;3_>2660h`dbJbfPN~<3xF&FsWi2VBaEUD&@<=}zAH`~|fzFeCPteP_4JqQGq
zU>V~!lBJ=@Gghniag9S^CXR#A2al&l)XJ1MncY&)tsh6_MrM|9D~GfOy%HsPjj1H9
z)*WJEw^>QtXgfl|4n6o$l<+8G)r?6+G5TWP{vE8niJ?<WgAu;;0VN1{?a7|m6$a1O
z7BQ(__;&qH)r-}t0(rsx2G^)%s;7s+UVJW=pF^-8Ur(2Oq^7)<w?F(67H)LU=j(M-
z&^|%i*74@^jf>5{cGtBeHk%eISkVkEFSY-0mc@RzA=Pw5UZbXLb(cv$|DF-sALSKD
zweUzbLm$&M-RlM4&hPl<R8d2JUl$+dL&~}@J@P^V_vQFp;s@5gq|8aR<eDuQN@5WH
z`YRFiYxe!gS*dP8*_&hE$AzDIJ%*1cYpMN)2?D#oB`6*91tI--@REmpkNM33z6qL;
z3+G)D>MM92;Gfi)s_TDJrI!#z!T-R?Of2}z-<(VX*3xM>t4*dOqki=bsEsA+^Y+o=
z6#f0ZXu)tONalLX;(@D+v-8zWck!bcU$-{)Q-!F5U<kf2{80O3TEa){>5S|`u0@8Z
zgUIQx4#~;6)&dFL8XZY}>+6)^$r-<B1KUwrni3Z8p-@?<m~=m4>(iqzVSFDSVI-RG
zCj6ZI;HS38L_&<2dJ$ug!m7(D;$%9eQ8{lMY+_bN$#@&be|iId%S*&=@S2TbN`(7k
z&zBs2eiJd8MiG;lvKMf7WjqNI>e&b30By{{-EgmTB&WgtPgmQTG4L?-3a=`qxNDng
z$R$C|qrPXfmES+k0E>f`@IN4hG)X`v=F8{f<MSAzO1L=s@vNdy6x|NIMurO*JK3Fr
zoXm*<0n6-De9A9VjDZa~%dditPl~;hC8AGA#A8rX8J9v17bGOnDO#^2JwIEQS8FAt
zxjk)R#JNr~<`Khdp`6vsWz~J4Jth5hJsWqlI0!?F5&!h*%;R_LhjEc0{Kc5heO0=M
zoXRZWk<E&3jwVbiFCSXZCIiV-7ViEm??_q2=*<Bue|$W}K_*v)y|%|q!(Ev7=&8^c
z>Kc)lo_vLqGJkn+#2dqgTfOuy?a)%CMN{--pXK1wiV(?HEVgR`kcSUE$R=Uf6x^~Z
zzLaT=er9L=cp`V4mH3IKgXik!kcYzLpG?+7J+V^?6?y2kQH^uCur9J{RH4__T2}|Q
zq?7FK=h^j%(I|1oa}$cL_vJHB2}hh}=@nV~&9`U2t(^Q>WO<d_RGfCNMc5wT<xKhG
zd=kM+#2Ojt+xUxZ@_`dfimY0*(}iBYm<5^nNKA&bdPPvW7VgI?5O?U&-6O(t?qvHO
z6kf(J_n5Q{Ow<}?MgB{leno;Tm$TZc=KYf@c%saMmu441RzfkKTN`30-gXj8Yp!^s
zJx<~y@nJcOEdZ~C1pRb3D<%YvMVmb>3jmtV6nB9ZGfwp)v-hcX*CqgSl|9acuU}tu
zuY1UQe+=<2zErN`-hT{G$#(l2c?aYd6qMWUN;^OKJ0iP=wfm=uf2aVk$EVkLI9MGV
zbGq6;e7hsi-7aQj+b&!lW_o3%E)d4yyLy=}n-NJ5UB0#9o>j~C8^vFC&!yp+A#|RV
z(*__Q*@!?fhY0|P+R{%Qsf?!kJZcDFMG@v=uGSpeb00rFZW*X$_*Ycn*(zm+37(tT
zVLrz{ivi=nI7ILO`n2i24^f7jdx5Hl-MC3=3O*vn1&fh126Q_J$e?H4ZQHUN7`gb{
zrM~w(Pd;I`_LOnTNql5?zD|@R?yS?5_G?@IV9UK%dQ@;&NPR_u*MUUzAN74oC+obm
zKPiJGQuMwjtVFz_b-y7b>R3cR{PTXIP%vM~TP?U<VDi*`*DQ3;O``8cm5!=@m)F>B
z$+9?0!k>>f8{PLL%Z<fzs|`s64XiC?^4Vq>jcawLK9`3*aTgO_Ve8;(^C}dyrHkPI
z=Zi4@Tch7eq@=e-6hKKP44+VdsK7xn6q)i9Qt>P%ZP7v2Jh*L^k8`7|m*Bom3^lgJ
z>%R{3(q$Z+)NAY)S@iDDd~mogyhPr))k#7tHshaLY=Uwf(C0vVpmlEcM*2(nTC+p$
z$DTT3tVKBpqQh^&KDg7G59j@d1Q&jq9R9{3snase8%@s9UhU_L&1*jV+4<Vn{v*py
zyKB*2io1!Pi-+FAP|0KSb@C9K2}X{v&`Zn>Y#Y|Mb@B--pE&RH${ci1e8a1G>E?bu
z8TyYhg=?SaJGhqCrUd8a(mowZja}=H*8n^*ZzfMzQ}yJ8@xDPs3{DrH>~`=NjZgY4
zpIAtP>_?ip*S5O0wSxwP)?8^nwf_|?uNSKu3#UV~d0MCeFj%U8KDgcBAHLjW-KrYB
z9iPvAN6^x_;!WTfyxGGYejfY%TM{VC|7e0WRTMlW@+a$|+aRySHp%uw?>g`9_dbQ=
z%#nRC>?>+pQn=$UEoRSLUiVvPP=}s?jx!`r-)+sRSlj%W9*xDlMYoce(fTlanNt9Z
z(LsniOC;*9Zs{csR8LaAJ!@6vSp@&wvO0U-A?-HW)AHFM<-Y36P-L@FlLhUbnbk!S
zPB&KVlBEXw=}<9CQmO-im020LH8SCP%^^$Kg7C}g-R6a{WI@@i0?Ur4`GqZE;n*W@
zL9Kn_+4hq3$OC17fI&Z;*Gx#T@~bs!|MKIQVru<$#1QUL<RE{#SF2ijn@Nv=@8xb4
zo|UXcFRsiTl1dKt{A+QM(i)pG8ub$fu$a-~jYexd!OOz@t%IsOn%@&PuqTy($;Yvs
z%MQZ7pu#{yREFozpMwTa{KL5t%P&(k!Cg-kY^dyzPYUj!40&DLDm4ypD0J~=b!+on
zqe^rPF1J5L$q*|m)RoZ(Kl1lC0CyWr;>O@}DyJ#$)`-*)`6n+k{+O&#!9JqK_r=0e
zz$Ex$fcx%e-q)RP<*?xlvah*vuda))FiEwIt3j=?mihJuNZ=Y<5Lp#cvB*eSUgb3C
zv{@wJ$9wVqc1G>C27ZrIMZ}lA?Z?-wbipJ-ygRO*Mu@9NO_$oQaw?91@tiBjGq-04
zrRC90n=B4n^NR=g+G~f&`kxLEv_pHal)`^xBT_wItD$*cDl_z|>kT1VxlKr_SB#4>
zRa|_bs<M~*7k}-3Wqlq0MG~n3&AGrAATg%BNcMv(Ye^Rw{Vfj!y?Lce!P?$0^bWOV
zhNg@F<4nDkA!1R=^#KthDz=#t19K(ejJTL#HOtp1)h&Po?-`ztXH&`FFNjX?Y#j(#
znUMdF_*>aZLD7IYVF*MR1;}CS`{Sr}ij|2oze_*GtK|R>#Gbud+_Bl;>vKu|P^(aF
zS{(hT+JE5ml@Jsd0)Q)F<?5%$7qJt{KaR$$J;Ew8Vko36d%{oo?9MS)aYY($$4k`Y
zx5ml(t1Lc{fOJUF#HRB-L6Tj9uvk+ncnbhJ(v5sh1iohallKVz`dnLJEs^Tf>|(O0
z0nT`auodQo&bg8;66FEoT!ZU<VxBBruf2<Vo(|*kS(KOrt?uDr%WhBj)-|4uoqM@f
zUR!{Z#_M%oKGOEyVL*@YC&UU3|4v2U0BOtruc9wA6w+6?CM3?=w~GQPfVrSBlcOKw
zi)nKSRNe3!W!c_n&^=+x`<};N{cydndu$6QTNH0_0F<OelquA5{x#)y<5A+2dMet6
ziVjfBy<P8w&zRwrB@JQqUT-B&WCexFJvCW>6$Y|5FIvcFQHWv$q>%>zyeLmOT~vs9
z5kPFqMrhY%oq&;n@H?5m$>|!46lD7)!FWvNTRza3>7bfU@1t&6r03V9+6zKI*GI@b
zqOeZ?)o_${;Fdy69JCORMIm4r(FEM%TY$FBWyYGe(yrGOWM5`uij3YLpB|I?QlA?m
zx5rWOdpk54Np)IFb*e)jV-(%vfA}8|@BMwz!%T+<BT{w~x(tfDDgMDjx8R_W%}(x`
z6fxa8cKV;^&Tul<{cl3$%Wk1j-3ikS7$%|*=wumQF`z_)eBk3mOjFD2XG;Q4XH<bl
zA?8+BMajWYN?4(0;MEo&><R%Om=muKRQc;ld`7*>tN4tjD}_c6kd``ZYIxi|;3E1j
zD#8ByguDl`_aDX6U&-KsOhX+!kYnDg+36WL)--~h^dy(xU720$eghr>z1#y5uQ?_i
zDkZ*~ldXzv6JwBaNVah-G({r8JWN|ul#3=9EVwFvja29FjQdQhrpUA{0U6_|KRrnc
zN9u5){#y+&r42I5-9Yx0OUFWiL<`%08)Vl%`5P6;p!6d9`}^AsB+yM(n}}pQWcd}#
z8N*QsN4HZ09%?##O#H6aPZ0!<DDQ=%HGsX7<wW(2|ID2^>yiAuNIt=wlxFubassbL
zUTfv+W%(x&-|={5{bQ28%l}}Kqp<(>vSZ`PdkdIzrA<#cW>USSlbimtcDa_mIAH9E
z_7SmoOwkDo<7t8}4NLq?QqQU7O-JdsCLiGpk&g|g{6Dy(sI@eA*k9#M8{d~{ukM;6
z?0Dl3s4>PzUYjG4k9J6Ib;z#hXj?{_D!fXjh?O5}TC|P8`}}+Z6o5;)5=UCdOM;S>
zFN5c*SZ??~|B*=P%XEVKKLc_!!XT4&`7U+O<fW4HnLK6lqBg~Z#vX0e2kj6JHo2YW
zqk7v_X8DI}WCT<yp;ofsPSs;a8u9TG=pi!Bw0ODVFIR>sf5%%Q%ChE<&L>AcXcz~K
zt3Atx*NerfyQ%SBy)M3IE*QOW#3o*b7h=;t<K6h4zq<884$??1rD|Mv-P+xstdwFi
zF>y!Z40q2WAW0GOMD9<J98dcf>SLc%0HP?o`zpa+3ctCd0sojbXeu<E+KqyTP6!pY
zBa1vjfe5J5p|_p;c7tB5j^aih>GHd|B0)JP^iieb!A^rwljsBsG%XqCZwa#i>_aB|
ziT-$-?iX0_*?_Lr>(K3<5T!TK0g>+JKSa)8Wu?|GkLul<NA538I3V6UbZ=pG=1yx^
z&;#h+U%f+KT{RaO5jiQ0uEws>NYX4S1i48}*f@C4V&I=+|A0!MKGpB6-vJ2@@9CbN
zHnQ~(EKgy6?+xW=PdZW(?<ajU7yRV08BLVNk5h+`01xCG>w8ZbUz)U`wQkXQ&sE&X
zU&PWvD&6}G{eXV8=Q%6xzL5smsnEVW$bVQ0Vwg4PB&_&;R5@YxeDmJfmDchto1(h(
zorcRSZuFo$V^j5193G>$FUI)$f|=*y=!nN%LF*oUCr~NE6e>gS<<JjR^53JT%`zxM
zC+kDfm8_2$D`o>aH|<3Uv_nM6S<UueF!#O$coh}rI8Zk6$@8=;DZk^MYu+>URvR4Z
z7QG&$l>_Rh^>PX3GJ()(J*M1FE5Rv;Fvx4LVDDuWOEDQSWq57n3zK3ZVkomVujjM%
zv!{iD;zM>(Okxk#@rG&z)dO7zp(M{;VU`7A=O?#se<ho73b~xWYIJZBJ#71FT?Vi7
z{?Rd6f2;!_GME?H8LIYLEYmx<EPt4w+A6zmCHmjEm0R9}1CulB7#`Tmm@q2A&~*hy
z^{g}+i9pVqReVD^Cn6L?H0bFZVc-|+j`|JCS>(iun5_yvX;4IUi6b;0u0{c1SW1^!
z*52E4Vb`p10NPIc+iknHr#fO4wD4g>eO1Qf*-QZHuu;7knbWK;AsX`=!Wd#IkyY><
zCvD#BKdHZ2VZ_Z46nA4wczSHeu0Cg%BBJmFpEV-_$20Tu{=*IM@b?+D0JX~B9^`Zj
zlmrY8n`y-Y*<kH%%=h58Z*8_;Kh6^LTQ%LD<rq*9YC%|?C37^lw4!d|uy{*xi<=ws
zaew~gLr%QsPosh_fcheQUO5fkG^ybK>F1dyI-&oFh%P5S<1992|IUgDtLRH4BmZV4
zMi%%T599*iBt*86adnt}W=dM4+;Qc$#ve`CznDl^!*<544ke*qamGb=*5Rjo&J`YT
z1AH1ir+^;p#g1-IIJO+Zvtpx9iS0Y~E49Q0oRrN&sbAy}<IOO5`Q%SZ>Fpk(wf~$I
zj%>m$2GE=@GWfagGh>Uv!{^oyHVq*Xq?p3pzQOy0)m1{dGiSSoS}_L#x6Ugck6Y*7
zn*nO9PvyXsETG5mP-R_yVkC37#1G$m0f2+PSNq8W6{ckfPH$w<o<1A1$K*2fP_*ui
zq12Ar@UJm=t)9V11L`AmCvADxz|-s_i<ca~8xmT)pWi|1M!E0#>}?*9kj(k0a?%?C
zD4x;BSEmHayhvg@W51<h<ucFnssDjUzAy4MVu6^O6a3+WARza84Z2fEHrSxtAX>Ha
zRRiKiU_SV$-^Q4rP=E}~2mdQf;Szb{!}i?yun2A-M}ks^B)V<ue6Sa_i)Gk7N>gbS
z@RV7!fAe$V%c;VzotcAAYgDog9u4(;fS61<i#3khjNEY|%dS3P)UbT%{y>20#3hsc
zT$Z_o)J(#QPWeBN2?g;ehi8|Yvu5Ev&JR<n-^|{iwIBj_5Nk7-BF$ZWRXOzn>oD`<
z=`)hz_c5WN<=foKXt|7q1XpC6f|_q`dS8*pW++$l2~CGRNMoCM+D(3s<yxKxP2!EV
zr_xvRGb$y*{=x_Rx2dnScn1F-sb<qX=I%%367ZxJ@sJrIpzmuf%eLQKI_3iy^3{E1
zVSN*+-p>G8IbeJ8%wkr=-KEm|p`j~zq=n*Lg@dpZRN>IK_H5=EPL}`bQJbcpZr@A8
z_qU}dFK=bPWO7hvONgpZ^0O$)s!YpPu5uD=CVO(bSavs${^I9z{=&R?IU<c<W(E#?
zu5HRU_(^p)3@A%!wh#7kWAco=vhzpHLZ^QL#;cgq*Ri*);`L*>>r}O!7HQ0?i}aT^
zb;jz~i+a^BoGE@R>g=MX9A6|4osqX1kR9bNWtFrRiKwUKzrTO0K{fZv*R#5M=q;|0
z_~ZE7dyF%lwmmr5!nqL<`_vWDCPAF-9AIC!p>Y>osNhBkAM!8d53C(@ZJ9)Sr8NHH
z(~Tm02@`t@p!T6IJ3RqI(Fb!G5p8AmS3y18qJWETqaKiAZt~R4TfQY)j*i%;v}NA$
zY-Al;oo)YD9W{F<pU)3s)eyQhxU^nhU)volwdu6``wX$?6QK(wflI5xxV#xu)oTt4
z-NW$-tgj^yab6nN^m^m71TP|kB*TZq7<4Z4I;oe-^glj`0+=iJf-LLu!c>9AM>P`6
zS(nik6i~oQ^1Sg0MS_mA%rMJ=<mzpXSP+A8`ftj{{NXqK1Ki05or9SiRU<d(XG1R;
z1^DVKxZI<$b<eJIUtRE;@!Nl2$Ud=MhOaM36dK(t%*eQ8t>F4sb_sw-O!p~cBB!1>
z4cGc?Qq-8W#vNV~vv^{eJmh==@Q#`rI}f;oX3q|oK+Rt%>mc_TNK`zlCpo}bF?qgQ
z9_<zNbT~z8lj55-ipbM?$)GFeJiH{IO*6N$gKq=}f$QeWiS+VCq68(s#?>DQ*TDn+
zes;GnQD7HVIY<^>N@>@J6=mb^61+meQy~nBtj%Oz!tdr~Z>suJFy8BK@Pzt@Bm7^T
z#><w_av(@O{AQO`7`Oq&Co!uj15UVcz5SIH4?X@UWq2M6xCcGal*_ymI4}4gl0vtk
z&{pjbE3DcItflPE>R_xJy%r`i!K{-LpO#Ya+97>E{ZVz*<19=;*|iti(@=%UySoY#
z8CQLi3LjhfifZlNk=N^f@)?Z1YmQwfRpD?Ez&p6T`2OXf+W=NNLJU42U?k=)^tN4#
zdyiD+y;@19-=*7v|BnjugJ1Yd@M_A*c;EZCIfb~t5NPC>{!Sy;*@BA*2@wsrpH=gf
zU!YnN0r>>t8r%$dXa&30vrqO+mi>yPjhGzk0PiQpa7maJL*pK=jHWIH6h+lSy)F_g
zP~GexK$~HFRqr)%t!jTUl?$=oaj%i6?kICn{H<1qv7+|Ic_Pa(Lyd2_HMZDYW|z<D
z`_#pf9C2wdvEzjqexW5`<k2b8v>9SANHIab<7KWLx{+c45@bi;_?n(vZQ7;`5P12R
z#J?57Fki54D}qBBl@evZ9RV3Wkg>dCf<-7oOYm&d73Wm>pUmir0XH<*uIV;N=ajl#
z`0w9ebP-m7CTo8;*|=U4r>dVH{@i<epkX_*jGrP@joUJ@-Z>l3#Je^)F+-1vXGV47
z^$;L*o2czSnQ8AndA0bgPK1lGpWq)Y^DJfbl4R#1253ATR&O(`j6uMdSE7-t9Fjoa
zB0}(W*$@cQWm+W670j_215DA`t&kTYW`4^8#ZM;TJECXrC}bh|mz#OLjo@5zvVX(U
zR-Z^OS5#l@_8856MlbAp?@9G4GIAH;zp%2p#oK8h1-~Zzujg;jjKyxOKyEciIxS=e
zJQ=TkIwN@qGQ;SkP)S|Pe7IC7#LiO+)tW?nZ-|DudS(vDlAs%C1beB7dY@#O0<3&T
z;7t-tyT7m~dR?jR0mGAJ)@Px?$^U3nR~fy?x&b#VohDSU=X1p9w9uRg2w8qGH}vs!
zGl6xj9yM8f1FEW)ftX;W$WXave7@Z$1S=iJfS`Uh8wMR!|An7gF=4tIr3nqcNiUEC
ze=&hfBkr$gKVFm%8l#MZ{$i})e(~-eJt=Zl2=H6B&G*QVz#)thvd}(3T5WJ>cM-e8
zNaX`wsQ)b%&k92|`_Y?E9y{`&b9Ui-hnYuB?kA|QXFwgADp)`6fL+zlm6n)=0^+y}
zsf8dALmA{!{{v3W;J=BZ0VcA{u)!v7kE$&s_z8q#Bara{tN5?CkLW(Ifh6IM|6pVC
z{}XI14kDywHLSNL;<c1rt!6WA^^V)20ySRzU|Rp-ahB|KhUSRPL;-Pcht#s%&C&3G
znxhcx9tme2=<=coXrt{nN3!E~$iRT_p#fEJPN8Xaazg{6Fi_m_Qg8ml;H3Z0146(+
zE~BPzm4*$apk?>}?SN!3T5*VOk8on9#*7UPmuz-+Qym4&$&L^>7uCBI6M~~OK?i9O
znMM$+5Mb&u&<%t?ISg)r%|8j%$NlG?y>JoTwIJ2WiwwGeC<5dI)U0(3(8p3wu=yhX
zhcp<zeMhmb1}!=q_syN!KcWLXpO&ypZgin6Htj(uyF&pWS_j*c_um`7^R=YKHhcvR
z15I{|LC~ETNMsls8XA&nE1<-yO#}u>iCFLm`o^sMKh|T%mYPzC*Ww*A;LI+QjXpK{
zZ|-rxc#MuJ-nQ#F6!u>n?(dKTZ!YE7w2JaNJ0(B8*dhSRQkl5@CWJm43kGk=mz4p_
zhMPm&-2dU;#&Z$>$Kp=>zh`mJV+#RQnp(hTu$p{&0kKM{T}iLQ&Gn&t!z;qrI|uo{
znyQQXzwt=_$MFDF>jbDMDFa-@P}A%G>+$?Aj>aPHk=g+!a-BL$x&O=Y$Pt5rf|N4_
zxO$@rBOx*oP)j^j$-{>Q^8}Q#UHgh-a7aO=(0d~`SIXTjev|3$#**?sFxO|`{}<AG
zd!|MMv~nG*qy^(;K%9sedVF_=nF@TX1shum<S`I8neWDH{2#|Fee#}tZ{cGyM6eIY
z@W1^Zjt7dUHx6G;w#IvXPT-Dluw${7T|pB6iN10+M3tU*UAwYAF~qSURn~?nCC$Z~
z0-RxOcqeF(&gQz-1Bwp?F|n|S`co%m$njh#V4ft5SvIC*=;*BIvWkQnSvN;Pg#*2G
zAQ&hjn}2&mNI>Jt#d)iVW3O2)o`a4-5w-Q#_h~BwIci5j4i^1yS5Dd%%0_9L_9^`z
z9^yr)oay(zV?^hjTnW1B44@QMtWZ6m|BXoNj=Xzw^>nbo#+j4(7-Z#pLREo)C@~yo
zYeTlDt5g7IcnP41o}HyA5Bu*+PjMj>8~I#MHr2t`Bh3|eIWHLS*D<2a&jnsbG^Vb%
zx9kVt+S`oyE^`@HNN13OMy_}J6dh~~ZZ`&AY&@3-FzQDy@&J|hw||>-#Hc09v6?I{
z1`2`&f+Q1K1n7fN#hj5=kb?#~V+wx1;B$a2j|#b#RJ_hd&KOK7u54hR7;k%8J^&q-
zkpZW@4B+w40|c{=+x;)HY>a{U44||^Z>i?}Yl2Q%iPf_uiw$U#!U;Q{a3^=NK=TUM
zfEo1vjc?ltZmGLmXR+5PlzzY=%XY(2m^7{vu1WXMFMhY}?8|FwIcY<=7<#^xdJZBv
zI<_&qo6)n@?45;8aj&+6w$Xrn_m0hDpXI9s+pI&eQL**2o7jzA!RbdxQ2QGMZ~qi@
zJVYbpCg(-=SHv95ZW{x1e-4&;Mjq{O&R=;ap*KUFpQ`t1^tc7kU5*JUsnYM^!(p=U
zM06KU4aQ5(Oz?I31XpCCI9n>2p9$7rRK<E6E01VN0{DIZbC<C?D}~1&oK1KGE>9Rs
zc`Fo=9YIHK3nWYpA8bVDz}D$PM_e#qSrkJ5k<PA1Q#u%rhpy9gL8Wc5Nj4mGtco&U
zQ<Duwmkp5wDfwQ|T8aaJG+00bGW_1;);H+Q=wO1}+gN%`rqBd=tkZd*33du23#E!g
z@~hCpz?XZF5aZSlCExgb`mg20p)*p$0K%FS#9rbO^=tw(TpFEzr5SX+G;{<oqCc({
z0+^pxUh<qQy(F?|JVL+Qs24A#B63u4KmLl){vnRdF%0wv$B)7og+vgD3>tK^v+Gr|
z0^&3#KwtCr_Vqa<z9Jv!(?YO_#JP-!+<f7RU`-;09EE@%^$<oiw2pPZf#q&`<Y0QP
zsA+R8xZtPvcL=AW&>0`XhBF0837J*#favhW{Qju^d{?_)76(M46NIEyaI<h-BrDjE
z2$ttfg&6mP=D=Ja9!PJL0Z8xI<SXhEFgpvm@a1R(`~h?r3MLUAIPT)p23^-R?={Uo
zNFvT<f<?;(hs#<X+5#fD&f$0l@RSV}8zpIszhXe6@0~6Ti4wk#?m`tW(!<&51hng7
z6kjVY3-`f4->#^U&6kdl{j)pQt08g2_~7Z6l-5{_;b_o1s~w3g7(q~i?c1OuJs2ZL
zA=v8UW=k42Bhbt`81Bp)t-&V*42RwmZVTk0NDH&-dyNk|tKow)0uea0shctTzd~<D
z0RnN}!kA`=gS4yQlFompT{=j+$b2z$cjCi<#0N{A`$P(fZ;x{NH98|DEMD{|%KLa&
zuos**pq3;_RhN}K_`el`!^0(Lu74^4eH>bUsDBIuSVRWYp<UTMXAJ2&5~1I9)Ps$Z
z2nP(einrzg{*hj41@wQQS4rhu1AI<vogql{3UO44qJ4f*!G1u{t4r+`!<H|>nt8*L
z$ng=uR|sALn+}&64Fw8-!+52F*S%HEt(gZQqqg4Wng+H@8iFf+E}6n*q({V|r|JkA
zB()Jg9gdPL0+NX$Vq1FK16G~fligoV2#t6v6Ko}Hh64U8O41FeS@h#o{Cf>FU}58k
z6tqAVaD)&58GHp!1Z89dh4QhO%_D@e@GY?85?`3~VJgIf`Hfliu>gs~Va{N*kMq9a
zr*}e>1*<;%^?PlR=B~ZfqeL}0j9W4f6q{Ls7J-&?^-l|=BQUv3@hc>eA4GvAV8I6Z
zlM%rlNy>j~pl~-QJD`=wzDWJM1v@JQuQ$*8MV-b7g9%#Vusvjiu|;-1@3>(G<n7Fm
zhjHB)z;`|DB`u*sb*x2$r1)9=X>ui4v(-Plg~JBHw2n4*9!{r0j>eoG*tTp~KU)Zx
zK*}#0fh6=E;t;^z8U;38w8%lLHqbS5^;2GV`V}(pt5jhbC`>qDjs%fnD$IAbf(f=t
zopjO#7!9Z>3@o~5Wkib*S`l*SXhC$a0-hac07@q4oCW&!eE2n0@mAqUuT7CC7*6Y!
zSP=!G`9dke)P@o)^7kWPHf>AYj<i6WrqFt4{L=RB0RhS)Fp7eWE}4S9;L?q4dG%+{
za_{^50;p+<`0#vmunM%;@i>e3meD{Zb+XAm|4atV=Lnk5odoNofyq=RSc@x#f!P&p
zvFoLvHxq==2A)f8jAR=byA3k22g?5Znr>pKq4x@17BsDkE~Q&X0x}bn7@V=K<OS1`
z?R*7X7YF^dI^=g0xq+u<(GRL?;)Q03C236}3y;806Lx=Bs$CN2eKMxd3vMLZWEcXA
z?!X9t8^5iW3W2uXKQ||doP-SoI5=$uwh}ZrE!cXu-oK47LE9~Qlt<}mj4)T(m2}%2
z{05t00%%W1F@2Ur76mK77=6J+Nk9d(5o*=V0d^f;h_zI(l~4T%(T)-{xmrkqV%(xT
z7N8NHceDE(AN>jfY{u9<_WbthpcmA>$bxGrHEW_A9%3Z0=~cXMoXj8)9VaWE7z#ks
zfrB8$8IK+JHR3KTu|kW*^hwRU4+dn!@+DD4PNE3`UHhUPYTERyM{`W5M7=e$Jf;nm
zmYD^D(a)hl!$Q%P<bpG45B_i&$paN`!#nFlJuWl2n80!~A>U=wQLjq}{uCcZWeUOz
zoDnwhPK^*e$P$50^2FfMgzm$`Ll9B~wU8V4Am^YGgDqUk#*^oQ83v;_LjrW(!Jm$V
z=7>iPqv|j0j`8w?q7Kb3S%DCAM)VNS3?>G!^u)NX_eFyPA7d#AXDWIR2KbXZKZgN?
zxG+gs`7ag+K(UzC8MsR@;0g&?bW1Ud0d7CE8nQ{wD0FX}5K*6_D8Mrr4Pr_RxY_tr
z2RP#0DR2kfPT%kE?N!Z8=9fQrMkpEs#X3(eN@(@3;K0quJ<1qXWk()K%Cz23N-iHV
znA4Fo2S@J$^UtBLX%jKChXn}0FB?K0)p4fY0`jPMkVh?TnA4#L9yJ<xR1HrK$~POp
zrWB8GtWVpbJ7aj-Spn4ryj-V@o`l=@*RXuOE@Tym@eCjq^^2qpGXUW-;jI|t5G;O`
zhkn5Vtj!M?Mq?2ax5hoGIrg6-=lJ|2=LKZsyq>!TV}MqQNAQgk_$CO8BtT4}^Y0zW
zz)IP|!PX+$b`*F6enA8BBgu<j<e4Dfl+=o381S4v0W=A!^Exm+{P7kb4W74-WyV@p
z+<8hG1z->HUiUK<fJBrbdtmyM9t*{?Zpa>7HtX$`VxTR$_egJ{0)2O`L4k->QxZI#
zszF`Ue+^Dl>^*I{8_HBpGJY9r=EoF`4fGF+D8OZwNb@wrLi2@=0jB#VO<ab+JtS6&
z_51pfgaA_e7{D$R^W_ak3_u+FG-cndN49eZSpNDcjm_2n**Jnr-ZHOgB}`Eo<;@+K
zfmWcesRNZMjRaa%-8md)pffc-Z`Hex#G%b&mRr}i1VzlVt28dKU`RdSs{~MR$3V!I
z14@xZwtNjlONR7i_ilwa(D}on(~t^33NmSeA`B6Ic?9C;cTolpF;=oc8h2YJ5PTC4
z8S(`P-4n8%sEF15PLiu^(6l7UkQbVmHw|dsb$bibq4V)vC$M{6#}Q+&<wg(yuBc`w
zDqP(e15*Y%y0MKS@*(|Kkn>EyL@(MdO2-Ts66k~Cc3pRpun#R5wr#oJkq#I@okFLR
zgBx)E2Ub7RT!BNd1_xF=3FHNVxc-{rTjAt{1aFt|lOVS9SZI-WI3YUOeBG;EJF=a!
zKmgIG@zH>gx|s$y7PsbZ*}Hfy^BGxIA5C};slfQMa3Pp8Wq;TVUz5I~qN1p<eczvA
zOJ_*0w9rs(%MFeeoItVoDtzg0XYf~i7k0$HLR$_G_dg4cH?_%u{+=B1L+rY`g_^W9
z{jkzR*-Y#YFrgJ<WG#>J6Xm~k)FZ&zc9CZpz7oU(qfp)q=s}-z!kmqg-8sFQ{u)w%
z`3%Mf&-b~O;4#tBCH^ehn%wDg1N!-m2`hVK?_2zL7RMIy4_G6dy_D!KV*9ay)*Rq-
z7ZSt|RfEE*BJ%)+20ZizklK2)c2pABx9;OFwmqzbp&6pwt!O;RG4dhU(JrXAscYCk
zHfQn@^~B(Fp_t(L=?$u(?7g5=$cr2?tTeeUu*oVZDi#UbKRW$XM+G$98Y)??o?*Hr
z(bP>2^dS`XP~{L6W*qNVNIQoKKF^Z-r?HVg5-N}@xWIRr7uqK(U<Bu#EWT{tMFSkL
zPE#qkE{j^t3GMtU7TC;LpVN!Iks!T~#XF*a&qwH2n)rqL-E6ONiy=F?h78CH6ZF9(
zuvgDeZ*%5^9B~aip*;DW{$>203o5Xlk(IAntsw|A>r1~BnnJ<nw&1rRWjEZ6jzb$K
z5V9MQ2v!V?Xg(IT5>vrc8E`Yu$5WhAn&7zU_Y{Hxo~?@w+mdAPj_dOi1|x7B2yOts
ztu|7(Fn4Qf55&YM0a;A8bV_)*Y)0}s?HbA<uuD2QAWPnQb897%B?T<+p-=<Sb9m=z
zv*2Z76|l2T!+++o@A$WY4~Qv=_Rd`Bl>eW(+->mxXD<KGT>hVT`5(MXS#0NW&^258
zja$h;kUPHXh|eZ7KN$!K%pN6mM{%Ivc9-jbp*aCV=czY=7fl7YHAr^-3v@nRrV*w0
z#D5Q)o~n*G9LwG~`qBzr%q&}uzT2Z;R*UHQNSc+q<akbn|C{K}ufg8_uYRp8rgKZX
zL^l3vJE~BEhQ#2+!%Tpw>@#qf56eVi-*{00w;Lxj0!t<X&U<~{$7t!Vfn(tFux36#
zlE?W^_s&Uv9xc3$OQ(zbdz9IHxRg}zYw^R!)+X|bC6Nh|sIcW5_^T8)I%l8X)vura
zS)QP6yNwg_+^~~0jZj{suV&M_zapy9L;5egMIIdN(FQU%)hw3wtsi0Mp#PPm+V4E@
zw%J@|-~1l-JtpbRLA(rGh#`)5W%TUd(+_J$74^~W2P1g4wj)&DgrhxJAyyK!#K%{$
zq=xP$O#2mPx^3>{!`J;LekjvK(=Yza*~Ykeup4O{`yoRCCqC~B@5hMn@NnM8$v=S{
zdNut1M7J5WlXTEr8yY7bJWDK;q;Y-PZcLSCb>v&Zlf3)`^+$qnbEL|qnSVYJ=O(rn
zWep)>MCVv;Aj#rE8Pn~}nOUNe`gFIrukH%9@^nghvQ7OT(w%HF=&pn6m70--&S)uy
zBgKV0VH4`IvF7+Tx>C4?PveT3_U~z$Q1hmr2hQ0hn`id2hyR$5S<<OnqL_yI@+rd=
zZn=kqy0P~h(^~oU$AFoW&+SgjQ9-xjZ1rZz+QvzHZUK*Ea^3x6i5<**WQ@J$V+)2W
zEy%xc%+b;qHnh#Ycdcl?+Ul2$%^6N^6Q<jdD3&uj>=oIOjgCAZ!It;H_IX}J+OlLa
zafXvQs&g==no?yrCfaL-2AP@Y_qywi+@P2PuiaOeW!B4)CAOmCt!KgXu)6rKe(R@n
zhvV)<)A>HdOrfUn8Ms_zLQ0hx7oQ&KHtESdZu5J1>}DMI_t9ab_kP^7wcO;trPo(A
z8BFyXt@PYac_;XgYe}|_&5=%vC%RneHO4wml^YEL>QBTDwfgfB<FnmweJoGj^&)Qi
z(2GJQ+&v>C;02mQ3(1Sv_rWf?DkKOV=1?-LU;puw^dz-yQ`Om@)m%9A_L(1KSV7E*
zVuJzjq*?2%v*xU5UC8dm&&k>|VGktarK_d@F7;avtrTxo0SZm^DKXtIhT_ht=Z?J;
zkITOx(Lervzpm?%F3YgNYQMi05@H)7GX9_}*n4)!1a&n1FDWP;T3#eF-g8d)evB&^
zplqzWLeXK=?(LVIHeO$IU|d5yQ6xP!)qWuK&Btn_hQBuZ;JbrN$ty2Y<Ggk=PdUA@
zpi|D`(v=n|niEbl{p;(S_yU^_fr1zBBfqL_iL1vWa4trCyp|#$Am7R3cYhn*p?<wJ
zwDI1;{)&!(KwObZ>_56;QQH3#&}^ZTW|AoQ_E+0cZ)uL}#PAk*{Gmu#FQJx@lz3&Q
zo@y3{34J!p%gS7ykvE~14zT@W-+1j3sjoj-f4!c6=T*W(utQRbC%9}TFFO4$?){T%
z8Y|>ADaGsSZjCi}wux_BI~wgoMtVV<ii&o!1%F@63(*AJ8$Uw^r!Aw#WSh%8@#D$u
zxtQKi#N?EBCl{n9@FS(5qqoX?EUK7S9k=RP*T*~_$5rtu#HR|L!pHD<DydG;Y+a>c
z9lXHIr5dfIQp&#HS97XiTH6>Ob6e%Lude_8tKHF!zP$5+Hac{;)h>;hn<Rr?HBjLX
zVp60!w@!tEr-~3k*N=*HiTGSRg+H_e{qgS%)!{$2=jXa___V6Rdjd|&qu8@4E>7Q!
zh!D|*pi#u_aXb60sCU>ZDV~4EKjr~1VfXAyw5CuvzbYyAkjyB$yWbf-m9hsz&a(<@
z2Aj+vVHd@Kh}Mlz6&?W|jNR^JL-iyY>9v+&v*`S(Cu~&ku;vmyCJEKAJm0wpQiv#D
z@7tKJxxS*q!<FvBT)mWg9zAEGkd#mhBkc78_lU<jVlp4V@%^`)h`kN6tDTZ(+m8L=
z{Hobx-@CEqoV6pWxd1~;-VfeAF>A+f9AtOMWqSApjdxo5kN_7qMs#9s8}mrZZ<sCS
zh&dZ0BWTG^8;=P|s1Oz{Sn{7|F*w3w_{xT_vA#+&>O~AG3AHJCl?KOWvNf)iN5-n1
zGaemh=7S0EiY%EJxj$WN@vhv=!8Rh)A04=a%dU=ph)U(TH2Rs8@Qt-^GdWz=(Upk~
z-$=A@c-YC|$RPGd8*M2Ij*_6<j~xj*b_vpK6uCySM*%^;=opiaN3nr>=X|0@>u9tm
z<98ObggdD}LlL({(lE>06meLz1HH6ea%&L-Bibi=2HhR%GWZy_8lQK}(!HFrD|AL7
ze|&vA-V}dx)1N!uvwt1zupIbN`yyr7XxDd8GKT|C^Jk{UhbZm8*9>l?+@=0qtNyjf
z*x_QCO-dcP3Mnn6Lp5Kx*Y!3?P?U+V|3sZ1pAnANtUE7zoYzm?I`ei0oQ{#QYaXDy
zHD+|?)PK-vO^sU+SbKYZn0ZiyajCJUWnU0yK(0^94*zZ3e9}E4W;sea|0Z|Y-)+J`
zb6W8D<7T)Yr^$?!lp@^wTbxHOMp}bOBFkIJgby(d#uWA9EPpSJc5fXsAB>?{pd<^L
zzu2n+&1s}x$SdGb5h(-;m9}@C;D0Ww|1m<BL%txjTp)gWWtr21>bKlvG7j6|L{wdk
zA6NC($2kyz-%y%yx|zHkZ7pg<#r1EP@xr0>I2V?ePAE4y$`)l`FEHj5{MB15Y%^{A
zConDX?@X>ENynG^YVIsEV=q5fM6NWKt$3^cUR)8cdy}i-C+n~4SJ_v$r7w`a<8#+&
z&ICyuRh+w?iF@3;mYGj)SzEiUX}>u*dx46UeXu{;^lGByX7RnGVtS2ter=uQeVoqz
z!_liCf_@y@HwT~P%ByCU1C#g8^M+omJp`9NmfEM}OVAa`>oSQeS^}nC-S6SRXK&8L
zcyjuoq@CTAR$MJ0;<PM|`W*&FbF&a<gv9Si6I(?ESHI$`lS1*G81r!CN|%o1c9oB|
zd+*&cp73l(zW>IS!<|_>eSNvk(`&Isfw@!SFGxb*LP^Q_^o3gJd*10|v<*{)L*5lP
zlq-x_`&u_w8ix%hTL1DjH}ho#0)-k888NTRNw&JvFpWuHd`$V71x?M3ma<y18`b`_
zzJzZM+cDf4F&Lq3<bV8qnBL7S1{MmbJTbDu{jhoL^lTV=Z{HjvRT$md@1&&rXmOy(
z<<shjQTM*z?Hh;TJz8g-9hV;d5G|wI%zH8RPN?#LWuVQ$t&yQE9NCi95<6CGt>bSF
ziT=;FCvAj!r1qa{l@EM+>@J*#^xY)fRr5x_X?&Od!mY6@*w{-{y~*ZV^(2ykW}}JU
z{(v-)cJ_Gj&um+L$7uZ!v-2i+hl!su$f1#jb@*Hd;;o~tA3fE#(O&N@o@+qvrR~tq
zmSo>K$BOE%eLAJ^Oak}8r6%H)1_EfArX(w+^ZoB4>)%&fMo8kkO{-pkC;i>*F86<;
z?b50hJdv<wV!TbIdC=ZXM%N#E%=1YzPCLTUiqJy5t2I2}D&e#y>K<1r!vL*w_R11Q
zJ>!DazS&_W(x8=EXaBb6H{bZ%xH{=Uk0L5er*3z${;<8XL#jR*YC-AEFbxs%2%kmK
z+n*D|Za5cu*GyGpJc*}V-VTrXZPL!t`h9UtS5oOQc^av&w!@Zh;Xj!42hR#*zgqI;
zFXR?|T}ex!-rzQtyk_bTt2zjxSI-t@dzIQ|Vi9&`AL%c+T2IXzutM^{pv&)f8&!$A
zVtPN5AvfX0Q%d@Djxl-upaCyDcTQ{3gl;(k8QA*V$Z2kcFvS(=K#iM8E{_56%}B|N
zG=1R%=?-PtLzIpWP7dO_Zi1|@8Ej^;y>58y{55($e$L8IW}^)gb?*N@+QTD3qt)@T
zV0_B9sCn!9_ZE0asWcLe2Tv(4toZHa#~+g!zx=D9I7vBXcwN&amMHqO1#OzRgXR(a
zlQuqsR{kHvY$vx1Uf5`4rB75TOZh%ta}V%X?Rb)i2s+m1=q(?se0c2f((_~<Ym1WC
zMJ8FXr0c12RBAnzfm<gUH!4py`}+zO8ao}CAl+QMhpNqzIR=CRXlIkvtT$Ww4cVH!
zu4)bsCH*h<eLECZaq&Zb;W;o&vlWK-`0;QNs1#i>?)lz3?anjR<TlSx{pF<6S?T8y
zktShYu>A0oNm^z33ui`Uy>_$Qqkwkaz&J_cP?xgBEc|EV0tYhpPw6xobxM3YPAw&u
zJ46EV4fr-t-X~C=f2KNq{Ju^qK{Wd}%?$Ol($}4b>i$B1)g+SV%>l0Vsh@<mPucQS
z7&ejOC*N+7If8p>_wf=AJw6Ux9-q^G@?PE~eqguJJ^bv|JjW7g@-Tvc5)!+><BPBj
zDPvNHuC5R6=_Ql$y(3R$DlFs6BZ5yqHZ??+MO2-(-p2L*tgm}G_0A-BBKNnjiOJ<<
z_`$GjF^GqR5rRW%>d@)iwxe!3TB>vKwAtB0NE!x<z=-uLozJD7=@vYACJifsg_lsi
zt<n1;wr56@-+-f{xNvG6(TH8AV=86L=B`HhTzaj~PX)85{{H&Bxl?RjfA9Xs?itj8
zghYOf>7(cBX$(t&tGe^2nlxqBr*=c%wd~w8xknD%H!syJef#OOc-Kigjt!2*INr2K
zzMEx2#{j3$lf}S}p_%2!#u!-nI>Z%u)8F4<MpfM_4yHVaRi7ozP6$1Cf4uyygt6~K
z%)8`~%ED8WN>@^x9M0QYhT(u3aokm-^-VwJ(EKRzA7>c%uIfBQ*t+^#5%QUTqa6~n
zmwNb4Z`cTr5fI>^ey*oP^!O_lU_)DW9G-VrH|@cg?w%}8oUDOoq@8hwKR@k^HGnH!
zBu`)^67fMv%`axSLf||nL{8hY4B=%RBKKR5t}F@9A-9i_gGCy`M_Wz&K9zk(b{H6V
zImsD+lDN;tQ%?EL(?<t6Y2F-6++bI1X$R&G-;>z5igR_D+>AllK;$_i+xxJL^{1sb
zhv*0sU8o8JF_WXIiY2<-EwD~XO}fB1`wM#$Yn0l<Im4G}S7+cxlkB(TH>@yu_?u;3
z8D(;>YvksvoV+F)B@AXFBIB|+R!e|2ppz3^u*7lG7c;ta-ML@SYTDlD-8l8Ismp+?
zyI$OGq+;6P@~SR32!B*?x4Mrv04Mjn?M>OATJQ4b8I&By6mh3CR9}#zN)F4iN`{fi
zRg$CIWY~XZoJES{eD@7Latq}RI<4$=qtb|JKzcg#(#kK2Jp1LvPhQp>OI05FL7RtC
zTW-?pIa*2{6+Ta>t*~kYTiw4M$_*x8YG@+22a@#=TPtcB%VCmCQn-Cm8umr|`ZhsW
zbg?6rm09VHCYxBeth{x2{x-or#ZTLl2&D(><jwox3z$Migahx|<3)Ewuz$Q9iRp{^
zV4B90Dr3s9X_&oVp|AVtx2SuKIPLy~)4F%_W#9HAa1ghyWFA(0gz}vuTX{xP0>SRh
z%$u+;9v>FT1`D@o#L}$r!aA>>4R(^G?*2e5ds9uSFI+Rs@QEjrAYn6Wzp(e^%u*5w
zzE2OoRs&Mlxjtn(o<S{xRPwv^>oLS9^X7^A>KQC?H*PvywVy++e$OC&)ud_M;tdv`
zDtXh~(8thT&nxA{eOy$|L3SOgHBb9`)%Byx5{Kb(TatZGLDfX4D=hPo11vEF5yzt!
z6~pllVwqfF^c(MU_F;6ox~)Cj-buZiDO6k}V!k}}YfocGUx%s;s?JtRBukUh{ZtBV
za2hXZWXn@R<cMq=*I-ODz!PuoSR7(N5qM<N_0oG{-eIb2G55JX3ATMI`)%eR%O`$p
z^VMM#wZ?Ezm!LOlT$yHmg7V!kuLoP10qKc3ybtGKzD7XDvprEQR;KlDZOX-u5}73L
z?|A~uPaVXWZ@vi#W|qgjM?f&E#UJJwLMe>xj9+$KpkHwyI$$B8feTtDqUt>>7x11f
z<;xuBfe-DuX&?J{^DSp(n^Dr{S@2iwNzO%<OuEmaZ>c>Kl3r8UVz;a}ChIIV$|;zM
z<~(vAeMfDXyP#uv{E|2CxXU#T1>PVa3XV}6I3PJJO&9#}-XPIs#HZYuC`on(=rq4h
z6_+5{>97zsY4cMvH&~}eA+Oj>+j+Wwa;s>6tLa73XgO$1@KqWX%9uco_qFSlXXll>
zf7dr}0)p~_Biir<K_$zz6N!?ZKEY*u2Z2h#`BCcPM)%Wz^RELL*2Ow`Pae2^`hv3|
zb-N~)@{J~y2~L^EPvIDgGH!17etHeJ>@ntNn&~kzjSpW|(lx(@%5(q0(;LF6%vPA%
zFAaZ07MZ^b+w()0G$$_5uEfCDxQB>$Nzzq1`Z_Nc&t@$kBoA}2vH7!?`JsPz^|M9X
z;b)cc7D}*PI-w9$_f5G`4mrAo=kKD`d<TOIXv?%&S!PZzdY7uqh60wDbJsNPOGD?X
zhHKz=9@dJWv)L8N*rDk^{08s4#ZN``h!nFWNnW)=&lfknu6}NoZnDn=w;K=L(9)$P
zdXL(O&59<yaPZ+k&lIutd0>HxmE5~d0vc(~T{&~QxOMzoZcK0SC6<2&EE;UO5Lau(
zsnlm0=u!_`1+x5BFSgHchEQH3;RVXrIjc21D#0J5<qA}Y<#BO8v26ML?C;9}17`cO
z@WB293dZ2{+WzhXjSd+yTXU57W$spl&IedNC;saw5`y0{5+qGaDVM16dPxUHd6o=I
ztT}|@ms@q^$9HahpN+=P%FNnkVx&G#AhpP9oDZ|g3U-uZNf3fNG0-ud2!-+DOzYQj
zN?+QKiq~ng4@$G%qd&}lr?IsgkeeWreZH4=iQ_}|J&GtUAsAKcr#|aI7X=cT1t&5F
z);LO;*45=&h;2oM)p0H}Bb+iM3-g%kr>ObKR{hi~c;xjxvVMAdX_$``jq6aDS2Q+(
z!&hC!w37V<y_DbFFdky~YNnbvgOOxz$h9mj;(F#j2iw_SeO6)Q9I1MSG8dX7L{_o&
zHl%2miGbpw$d?n(M0<sbld0(tW3YReKl>s0Q0JgbCmc%RMAeaVbIBOm$*L$?>&TbR
z6YjB~TQ?HZn4e_8d>qXh9^drQnb}KtSqF=v1vFdtyg2MXT!-yQyykUb5Nw%e)kV49
zKMa%+MhN*%`;6e65?x;(+^ko(DJyosg)#}cQ0@^K5S!Dz1bM=Y;}1h}q@g&5NCJk(
zli|oC5_1(1%{yfZ0Usn=<apfXzfzD$zA%T6_>TyGnMyvO^Fc_Zy2n+7b1aWC)y`Wb
zk|&06EP=-^SJaCfzJ*sV)9}Tb__M6Wa|weVPAtK_?+|RligUd+7p-n(ejn&xX{2QA
zr^`-62VC<3k42BVQys$+P6ZzpWe`^9p+j^x#d4Vqh7U{<nq;0gp3rsD8a9M|Ld6!c
z>q^HU{C1!ar`#YF<gUd@0LN2cH`$}>{hl$J-apotFn~^%Khzr@-HSnRxmM57@al5>
z2#ve*Qgl*M&9e3<o6eEVO;j*KkbfbKaw*R1FJi~d2N_Gh*JRfZ+a|+*wXouvrW_D7
z;_;6C3H1Avi@{DJ1z&JeaYKqJH(<J{4p|{PSzSP|M7zgnN8MRPjEtX8@Rb!j3~Dc0
zp84I+?S+wAr%p|glAwJ%mkC8htxO_F1Q}3O<Y~@_KKPwf@!`BV(I7?n4PB^>M!^Z(
z`!?$_j(<m$%yZ}y*8ZZd^-X=AU5$ZJser1;G2V_VsN+DRoiDvf{W>A|e%X_Fn}RE`
z_it7zsvB|I_Qu>Wi@XU5G>`00{Bd)SP?2p7jPiy*hdXbXBx98?7~3~}$7qG81i$Cx
zbgJ4Hug#$`E{+(N?9jv9RO7HhzknBo=HbKM^^L>kRG*7V7;(N0Qq+riVQ@6EC#GH{
zvo57LNi%6`UnFf<VKmpszby8iBl?+^FgnJoC#69Xg8>0gN(^XmN@sQOap6SZ`C{Mw
z*v()+S$=9i4>`raEK^Tt;NV7BrPsgg{cCIN7cpt!6Z5!_Lj^^o=@}f{bXS7@7BDgT
zE?m7Hw?z9qc^z?UqxUNB$??&H;uRaFjBDrDM?+{2M)fb>VbD|^gh%J^OSJ5kH11Y}
zKZfs1`bw=BG?+P@(pXk@iYy$orF+Kroo0ayZXLx?Asu6%%8&c3wq0`CnT9d})&8K$
zKaY%r8?VyS<bjfP%O(5#t)1Zs{#4B`GtT*PWn7f=y)F*qD3RTRL&}E}t>U9G;6brW
zdwQ9+?|!S1=0ez1e7dSM%sDUb<lVYn2Fa!7x9(O;=1H`Sxn^4Kbk(<-we`|VE;D>D
zZmtnN#;4=Pkz!>P4e+-d&b5-d6^&?>kz6?$ya-NtNQj<!dT%)?be1LH_!YSF$<tIl
zS53BvU|@W418031i4)5(J#~++J4sJ3fmq6-b6Z`7kbsqPo##@k<YGJtJVL2V&OFW}
zxf&!o5gQ8xNAn)EKKq{3@gw1Z^Y6pt)f*@-bz%+2E^Hk~`t&b}`^R}7rqDl2#<;Do
z5oBqt6lKX%*CWRuqM_q~`5*|YIoaSG`>2HQf48<LMlB#6j7uQX?}$126l!bK!6%Ec
z?aGsrJ>FX5hJqeiJoCz4$SD&j?EyVQW0fQ_OUOiCe`Xm!3Q?rR)=P0|*cL98B#{Nx
z0Ud_y#8mpo((6%}j~(;H{#25?__VtwV+3b~{OJG3-dhI66?NUZp=qRX8V~O78iF?N
z7ThH`1W173);Pf(f+R>1+=6Rx2?U4W1oz;6H+kQ4>Z^OJ&i`9={zKJ<wf9<cjycA&
z2*+>!#>D7|lm357Gh1OK@vaw(a?DP2_hbx7SsOz}MK?bXtb*I*@Sq-eG=%lWNB#yX
z1)E{=;-vd{Aw=*#`ObyCl}rVBiT>b7Jah#*75SBL%=U6&^Wa|T50i3e@w6wG-*Tr&
z)5q21C0i!Nc=6KD{o@2Q4cOH#oSDroC`zoL(DVk0H9ak?7eQtrb_@-w!RYjrWyF;u
zJqU*r^Aer49m6fdKz0k;dY&pKm|hSEn(oU_k_U<87`N}3+#f3lUMxa$nh1Fjfrdk9
z9sK>bpxXon=F9W}^G@O;IOgk1x4Nm`shiutX<4EMheU%HowdBfH!@Q4>7)8_<~*n$
zIBXv~Ci!rg0V!SDY=kH2$q_p6+q1Os^l2JWu7GI!uv5oSi&V8^^zsOk9+1~{qK#DK
z&3IqF`{BFMg}*0aYtc1A50?}^GK0f&e$>U|btMDMAM47xf;^S8)JzYa-!ySk@v1+v
zPPU6auf@89+dXMUp<M&N!szWT6Xm|gd`pvKvs$+Wscw4-RKPtAjk5AN7v4f6Yz#EA
zwwlZKO3~!TyB+OAa)};l#me`|Oe&Eur5HV6^D!Z5rdJVyT(TZ0zS5D&n#O6REUn<v
zPywMC+>R~D1vBu24Xk-m=8oI!sL>8dv`a#w7*Tz_*|2NlQDi#?W8?e#GZ#ut`QTu1
zB>B;oqEeGCX2XovR%d!g)WNNIH=<$4qDF#q^!J86{CXbm(gigkrY9V$AIF_V_H#aL
zy#Smami(!5it-)7-+mmZF-08UKjsl&juU^K?(jhHdG9XyFaZa*NexlXKgbI^|Da{J
zk3{>c#u!h2@TRO3enzD&`A#O24>*4xIV#kS6L4>?DK_Zr7(?%{1^hLee7}{bm8dZZ
z2bap&3xAEv%@S1^USQcnr0;Z5K)?He3k|2|o>xL9kqdtLyrEFxO;lL?w%eB0;HS=?
z-Viu;dB|+o%ksXL1p_tjTKT9?%+{X|I;D!msfh-xG>jT*%F^Fbe^$VQj>U$jia-}Q
zLzKD7sLyT?mNd;szD@-8WilzTzDy(^1gV<@7PS$eYsuPN`e#*(u~H)(LF3<21<lIM
zSqHNgKg0>>L}}~&4m_?=0{U5!1l;))%7GWWshocp^+GZRUd3P*6*xhm?+JB$D>tRa
zc}j0+o0&ZAcCGLq0aJ*a#yQ3!;Kk@7DpBB_&v`b1VPUZtoHFR1a4o_{X)iCk@Ejl5
zhO~wG6Y(SoPI2911|UhvOMk!%X4%j1B2<W=%48aptUitlhjQ`~HE4Kws<}#O((U9o
zG6r4?JH;E=BlH*+@?&RhrkNAwQlK>Hd?>?vwl(VGI#RCVeOA9d)-slch~Sbhdh|*0
zsHw$^(%e-FYIYWvuIft22lFWN?+|BN$k-g*DT3bmCp|#9uUq(Ew~cp%`;?cvbwb(b
zGvfmTV73H1U_q3zqL3nTfo);xFHq&<1HzaFIm6zLSLCT;);uE*Rk$5*o{3>rk$kXu
zZm8C4nox8m>>tIUbji%ed!zJ$Dg95FA_+BJ<u+&s^;C&1MzHlfA49;s#+F}n1jPKR
z@uVRY7+Q){TIB$SOF+cEO=XW31x4z`IgC^3Ijl^7ft_ah6Am=!qT^S#1#yvC-z*kr
z;(*!tZ|k*F+krtGi<rsZ9H_sfd|r~aq)Y|93ps~wKL1x^H$4LCN?AWYJr{NvV_E+l
z*$I}IaS&JYIYACD64-C~Bkrqc;!D{Yh9CweK3GCtKe&(?JXFha_=PkkAR?t+ugN>u
zg#za703)AYO0<6Mz#?jn9dxQEWyqiEPGK;|m;(pbD*ax{E=q6!1dkY)ggq1*F&8%L
zMAhq3!A~8*wgjP@a3^l_Ob*uD-=au3vO^+?{*j4uig~E9j~W?MIxyn)iJI;>+v7fX
zJ&Tfw_OSFJqM^;|q%w1=LA(_LazHK5pTop_<6yvJ-=z2AnqUL^y&hkE9$3+5USIgD
zS3#S=tR|)KJ<VEkqmv3nNtK8u4AdDbH?{kvX_yRsSPU)#a*OoY)6q!9-sJZyg+wY<
z8*?D_)c76I8X+Dm-h6OVb3_)Osj@t7u{$X^ML#ET7MbP<3DN)I;Ha!QT5Ex7<CV8Z
z$V?q{%byYD_l9Jr5uu7;HF%VzV%!VmHYPc~qm790wY-ay6_E{Uc{O!-wbs=n9Bxb|
zfBJC;(9%agG3b4yeep(4zC#n6(1(FKA8DC^+We=X$GOb)&BvhV<}DNQy%MN}+;GB)
zYP_ufg~78%q(-I6yy<jOE;$JV^FLb^w@Y58n}73ui`KnbC^Yj6^!ZFrH_l4Sw?}Zm
zy(Tnr@It3d6!=Z=KnvE=rTHaIg~nEx5p-B-8aRn`h76eAk)=9!n{N;gqYY)|UM~Vx
zHvRhd6oA!lyVrhvSYED^AA6xqLP|CXOhvzvI-tCY3MpFzn>!UF*&wt0LAW6H-&e;X
z>>|D@W!8js4fYzIa<@xGv?yohuF93;@olb3VaL!g_&M+}UXfZ=<|@_L)Gg6m9L&=D
z&sG`sXa|}$KYfJHnw^Li_M=1^LB8tt$dqD#%rfm%+Lb`^x1QwHhbhkj>QU7n9<u2N
zK@#nV))dZ+bYm#f#6;Hjt}&rd0oAeCu9ulg&Ml=o$Jjg+@Y>)&yY=HZqATNG*){sc
zPLz|+cns<|NL13!E%mRupZnch%ZmxNTzpTTdJWHPKrYMJ+k1J1Y1HoWy1QEfI2^dT
zZvO`Hori8BR@N{8|0n6IkEjjkz(IMqi#^@D`0t<uv^Ml>XBM_~zx%@4ks$m#CPCuG
z5f9^h2xP*hWm;gy!G#=!Jv|iep^I3uY;rlHtplE~iz75J1PbmKwWDk5t#-vo<)I{<
zs1WvaiMHc>$MV;_)eMq>b3_IgTHKiLKCaq>2Sw>i7C0BZmn>Mv#=FSRR8DR|_ieNl
zacONO7epVdTpYS1CAyZ(^4<*vf>k_U=AJmy($x;d(7bjgU@yW{cNcs#ADzA_;;U!h
zG#Pn^{@PFX?6ed3z{&^QAFg6j<2%Ig&x0?@RKMjGy2HMH!~T*U;{7l=!}~YV6UX^e
z{Bc%TmX*V}bCx;>+`H1Pnn6V#Tq9exbTsK%52?1`Pj$%m;cCBrZdx`Rc4XORWZ&!=
z&?YIk;mt6)1e`Pxa?ge?T?XDEBI_FK)W6No!4{+lr=^pa4z3L)L-Nsz2hC%~x^2+C
z#@xydaQ?#b#o-qY6lYD%iFb@BorHuc2eFh}allW=iXfpt9s+Tq4;RT#wo7aGh_phx
zbSH=5LL;gWLyhQwGAROqTdzq_I=Gs8N$cr>S6FQS$ToZvR$BbTEpZoaPB;f~MwOB8
zkij>Sp-%C=^FOz{NAa{dM%K`1iu6}+ivIp|mi5z$^1zy_X+?`uG9(ElcDgw1h6>D<
ztd(Ij$v)!EEk8cK1xD@VZB<RGfSHg4IKT)$f@+w-{XHN|^qPE9lXJ_XE?nruJcc^L
zq*?6Z=}4^*!55<bKyLS(Q>W^czsMms^!l8-*FQLE5*z!q8Dj#9Ug!ypFt2mFM@Ulc
zvlH(x$KCBV$3XCcKG*Is`XNZJjHSwJ(zV^v4#+KALZKdPHEpAJaJ43Z9@fla7=>Y-
zpt3b-62yYTh-IC-!;>6-WR|d~DlZ+JvV+G<C-IlYLNn%flgNsIW0ZK>5lSsqeL%yn
zIK3Y}i;ju`xy#*w;S*k67%c_H@QlOEU^^bY=;*NcvCN=;#Ap>%9FCM}{xK#vAz|$X
zjQv~L>%JQwz>-?kn{e$#iomYk+0gU9O&Bw9_}3kQhq*%*31)$N7bh@1zC$%F9nL#V
zl=?73a5qb!I55oy|25%OxQR{eEX$AK-VRCY8QU%Ed8oJmH)w{spW-&^qNe>!>NLuy
zvMh1|aWVBuaK(b#A2#PjK0825lW@5%CJIZsIe?ocUug!fX&{pDB0MaP@coI@K_L9I
z>3-VqGd_0B<Lfrus#mba0B_r7K~~-tgcj_EI(z{2EtiV7K~ow5=89Soc4jKThoDzF
z;I0oW{Cg~pvb^KdNMFV$=a<`uZ>Lf(cQFo#r=w(-DG%`wTz_rJvlmc$cZ=6DOgQNr
z)tl7bFFL?zUWx0Uy010=rxm27r!yfH(5Bj?Oh%QIpQTrE38P8kqYmnrsyxO{BgD#~
zS@N+<z{jxB^wDYEM86~}4=0TS%DSRwq#Q}BX!=7zpIj*k0IkcLGKqq0Wf4&_AGd!|
z*(D21!$^je0EIFjP%2*(Anm%Wi+G&6<;GK-D>A@fhAMeW9|>cUqOxXI$2h4?kZ;iK
zw&f~r)(m!CAr$gAZKsJ%&JXA^^0rE<rw?X*9)~~9T6cic)v=ZbW^CduhRa3vN0BKb
z(<2KB357{mFG)cjyvMl5o8a#WyzO?ayxx?6o^1x`*)bkr{!91~KgiIn^@V{pvJ$r|
zte+44M~!eB0t4!mV9R+zwXo9pb@tf?WRuqf4(GstWr*@zo%afg;c#UI_hN%b{gyH~
zsyACSmx~yLFi_=>wbF}iSMo~Vh^aZ7!Hj1vWp3=&C?n&Iw@U8sEuOLJ-n#nDU+}iq
z=8d&^Ns3{nV%#g(&12TQW(yvs5}vOWpInKAK@X~#p{147p{yL~zi6@0=1JMiU5i!n
zLlzO=Ua^;mX_97^QTl#z{?YtQQld0z+PCHKA}hJ^?mK@j8Y(+gC2_5XNr&uw`5my!
zTnWKB4OZa9Ed1UAmlGF3b;ts)baeabn=fcqD8Oe$F>tug4V>SV^QjX1jSCfjJWg^G
zL`##<3#XDOn3kU0p2LM+eUz%i2pDnsj8K@R&1(`J5g%x3OTO^@@Z<DP!*Y0SXYG=^
z=%;H8#%B}rb^Q}(U&;6QAs3__M?Z9zUm^U^pkOOnjh<7E`1*{R8{`BESBI3xzI$dh
zG1#;^dpM|FdahrO3jj|*p#?geNKI>S3k=^W+6=L-fYd|k%14RdTZ4wGx*{{UG~3J{
zV5x__yZDC~zSe%1g}$$Jk*tnRvlYV|soJ^ec|K;ah1kgh9>%ecODtEa6t81ozoG?l
zk-d*Tz)T7Fe`r-<``Oqm^4w%>b<a=+d}S*N?m>-AjeoXt=Qhgy?L^AznO#G1EMMrq
zcbQ49Sv?pD(jqLPTirx+qa0o3at{13*XfU|nv)Ui<YsX_YHAtim>^It&_T<pu-}Pp
zo>MSR_g|0ytTW{Sc67{0Apv>zA}kykI*m6fnZBJqVV5e5AA;<vzE-UA;!{_-{S~RQ
znp9HcK&lpaHORt_9^6Qy$O%Ys(SUS$q&8aY@d1_SH=@qy_Ys(n2xODx102roNFuE8
z-=0HC)Lg{T{=!THxWZW#C}_xqu-eXXZD5RLULk=Ks3J}(s<=?=#^V}fA}x79sn$mD
zK6AHKtxH0GgSr!PeKUF4d&AobhhLb#zbEgJ0sKHT<%FeLIC#&#K7P|@VfubunT86f
zb&%*OQrzR5t`a}9!a!Kbt9$EY9F)-@>r=X{i$jmPTPvM8Xz}++SsTQ5p}Uq*l($u(
zOy*yk2t{b4QUEq^fg@^AB#<qUUb_wCm#26#y4BOt1*Q|Ku3bfzt4KH6+!3hkqe2!+
zLa?Q%!rZa`Ms~wMqxR;E!O^hF%Jm<kAT~M|Q*SLHgme!S*Vx*h<n;Hf4p0UY2Pl)V
zL=|Ex2~H(ODu?V@dZa+`us)Y<wdNi1$qtPnLKQh7m1m_X42`g2*a;5TVL{te?0P3P
z<@M{7=l$QhZ+<xYbqVF!RKr#1yHQ!exX=c|5`HgP2xJloLN~BIPfkFBf==0>8L7R?
z7P3r{R7HXFg%}~a%+CK2vIeM1Fyx}?b`7AkI=gG{vuBG-jt>kS;j+^X1D6Qs4I3tT
z^jKgjJ1r5hQ=1zwn;l`3NE9$HyeETD=db6i)N9r`SaMgnsufa&SMeFeP#BNK;6R13
zyVYTSRihv1V(kjx;Dtf=hO%!gMV(*DS+HU`v;ypPjnu!(aGCSc<5A~ZnbK3}TAFcb
zZWUrlbu$j>ATiO|qbt?=ZHHa2eHyxttWpR5>dNjK9c&6ABD+B*YV*|3uQ`NlQgAab
zWqe}vbRz8BJvN7VE@pzCtwFm3`hJ(Np#L-r`U)&uwD;95;68}4;I|74e>oEVfWpa1
zaEyyS!rWZo_Y)?C=)a=u<fS4nvaoM=Vq3yznl2o-IR4kHTkBwQrug<d;p=bWrJP?S
z1c+nyeFrTnkGwrq2=X|l38BPBtMk3in$jZAz{FB8909N;??mNur{qoi;qBl39p>|&
zI$9Vh`gZ8)_<qEEFaop~kn%>z)D&MxJumKb=d&OB@-63GzEyyg%?>>Q`^W7EK0(3h
zbgJ!oB7<{0)3#<4WZR)4@Uh3a;6RN$8(n-~=}*mOz}-u5GQLrFf{;{X{NjeJmf)b)
z#Kl;RM@Y6ULJK+s|L~Tmu`_gtJKm3#p{;bSRd4Y*oP}2t2IIY=qC*Z4;*?mI?Tg9}
zm1e5-`hZGDfxvF`ITU!SgZW+ld6M{_14VFYNF~-{^9CzDPJNRznyh?!XIVvmk`~##
z50wS13&XiA_d|FnV7$XmPAwR{g*<Z0o_l=>5(38d^!s6BhQ%@E3b3-pTFQ5}pQR+s
z4nGX%eeQ%KHpOl-TDZMSK<K6R4O>}qL_hpt;OLS<2nvc=gihY@etj)c=9KC2jf73V
zUg;9SB^bGZWEsn#Zmb+B<Ck31-q<NtPAESbxNcjwId7UoKWVRNRSW%A>Y7;a2>V=>
zYMZHayK0HD@r0j}+gdC)XWbREi6wD#cbeEsQewW$QKha|PFeU0xnAk%2!V(Y{ihKn
zJgw|_ygairCwkLGrUcW+1_mz@y+LlPCkW+RnMxxU@>*=Dkw>o=f+Qb%$OBuz91EW`
zlBJ9Ru^D#nET=$&%T2}I{?3F?j1B{+3;MxTre=N``#CSce+aC*^p;wUoaiOKllmk^
z#!EEgOn^9Il!+3XS_2;ZVFT1-C^|is4b;IY<eKLcBS>qM=@5?Zo9`$KRrT3bl=rIC
ztv+VtE>@-fkeC&>Sf??Z%e^)NrsCFOzsR!M(=zXWD{|sv--wonn5<jHZD@fPn)UG@
z5LOzJN1!OSAO?zJBhQs^l;u*Fui#7RltNaTtI!$AkJQ^nBKA#Db%8DWpe$gjg|3;t
zY4Ho3>xF86(PCsvJ^d^g-Q7@lPA%dq|F<f68OKQ0)57bNP|A)iQV*i5AR0+Tj7&3+
zfitb=!*RLiFVeZ4{T?k(-iITPt4xZ1D{nx->;JoD!}{2Cwd6EFm#}x^D|AK=p7F`o
z4ybDgBrTXnoM&EcYCt&fdwWM#M^!Q1`t>M7kE|dqUgBez_M5`pRQfG~!AB0|{)9?(
zeO}Lu*|Upqo?c1up)iMNVn9r68a*2cA6W(LEq*~^I9gzEO5-ne?EA}6x&jHd#rw0*
zidh(Vn0MU<A++We4v+0DyE`u_I5|nSJ-!9FJhJ-b=$?qVa8Co;h=Q!5r*yjds>#<A
z_)=RsHdWsWBEA@NmD%LH*B2C!A9g@>*-h)KgN6RBB6OjAKOB~hWjC(J$bQ<-J5vvM
zNTA|;Y;_KQ2gIe^b)SkK8<fm|dO+Tf{Rf4RLY`O(rva^iK~v=SH#F1}PVIZLut0cP
zgv2wDcT4a4&i@qb*#>c0b*QrPD-@+4aQmL2LDaYHUOF_qG5)k=_mT)Q-}9M+Q4<|P
zuy7OP4RYps7llF{)S|)I-5qR3x&dPfJ;zMs=SX#CEthF#=-l*cpz|niZf#gVpCp>A
z7>IPlFVd@5O+wI=#Mpkeyt@|gNVOkw!Fr+ch0%g_fhkne+bO~il<XRq;Eo+6+DrNu
z8zMG>>xlq4Y%ohfKoCnb1U7_#a2<!j1wTpDabNp@Tvo#?!hotps@MF^1k(AKN2B^Q
zo0nOEQW|ZYc!Mn9z7BVbB$GBFG7q@4_spOEW##qPcR!pQ1X=2tjSIM-o$sow?^O&+
zIz+=V@^JVy4}F|DOl2E0e$7x9+!<W(&qwhG)COzUU^H8<2o-u!*7<Yl7FTzefZp;g
z@iHOO2#_ZauEd!d=INbM1JzOY!+)!zNe~Z8gF1{-fOtyAZg*V-J{K_>xh#@=D?CiP
zQM@1N^DYYLvfs`tBJ{aTp^Rc>>V@wA`QTb3ZB9~Za<t(>smX>4`7^QAi9AS;9h<Nc
z`>g>3i_$I%I1$Hu41P<Frx?f(%o3l(##iRbIsO0!7A4D2eCvE6QWikfZ+YpLpvPww
zbyFtxt1QxOBdmqnS~a!ZreU>7`mpb%QX+X!*+{Y@98S3V8I!<Z5=w;{uljIZPez1-
z7GoZd(plJ?y0I$UYjCH-)h=0%>OkI~S9y8$%HWkmBf%w=J_M32wOnRc2?|dfvn?up
zDal6F=|pJMo>l^kG6f4c^d9aj-^NuZcG^vJAr@TCX*x&NC1wB<8@|)XO=9AhvYENd
zpL}UNFGQ!07XQ`}FMnyKDSXpZUtr7EmR2_W0WQrKA?LvfbQfAcci|Fm+4*=;tJ{->
zx7pu$%ej0&EX1RrVw+C92llwuXO6GCv73Oy+pt442^E~p5_^Kt22?G5w!%~_f{g+#
zL4ni!YnmNvW`K`l_XVmz6+*DMFWXggN~<vuHBq%GYuQIUH;GOVE(GL>@v6Vio}!pi
z+JB5AB#;!61-ZHmp;}mQB#1x3pC2XvN_l!anS`VIY3VsOM#ETe2dsK8F%lG1W|=as
zB%kN&Xd*YgpLZv?_L8ALnzLORnd(6!V`bH-(Bm9@H@*X|2E{aBVIWnrY`J^&cxtj@
z(-GWkqRz825`br6F_32XLpb4No0HuiC?OiE{d>UrxqG2AQFvRSX|oWpS};x6#l7@{
z?L|4xZ1%I?*)y97;29G`m__qD+Gj)x?T>#;cGX1U+9WBd)yV`J^Pz<NNm-ZL8!KYz
znbw28-^W+Rk4|dVy(6-Y#PKXa{&1TCOhhoEJn92xd)L^__!0fjEK)V!xYs=QP}=1u
z5tKC3?6n!MP~I9z@DcF#VZD{0I~$#=9+6M89%WHzjNB!DG!x~Mw?EohhRl0~E_DB(
zBXQ~pqmV31sQlWsHnSrQ>$2<CH{qcH0s$mGAKX>(L8@haBNw4GbS{)e_FZ)hi94`Y
z=sO3f5LEe<3Zr76vskDob2_g%p`nWxpT5H}ppq9Tg~abqX8_0SHn9iG+iUD7vRNsK
zAjPB5N)KNh<hroG3H;_Gh_CTgsLT%D%@OEN<v`A1EFPl=ns2{XEPqySczz@8F!!CW
zeN#`+Tm>T^lHA0R=<s%5Z2=~q_cww$++Acka@KOANvy)a^2Z=Dgu{1roixhU8s=h)
zWAmJL#m6;usQWUa=Q#H-#zndZAXg}iW-hQ%_Mt9eA+RPc9g!!dUfX=6XZM>KImLVC
zFeATfeT@z4R;uV{z><GW*Wd_)QP%mon!Vz1nG7+12r2;Shl1-FbO+DR<J_<@R~ZsX
z$c#T;9Z|w<Ue3JvTh?mc8stf`v*TZOzrgx<8S$AC2q{06#9{G)F^Gq47moAgmEkEQ
zy<7LDe#KVgM8+PG&|V8uCzaO2ielBBR)V!(*#W;o|K6v#K3~doZe2Wjd9j6aWsw^N
zU*6i#eXV=T-!BsXGwp=qR-t})ws{wy6$W~E<bFeypY3v(xk7=Jxgsn%9SC1u2u=I=
zmC{0ULH{1tFH7@VsE;UJsZ)^U^S0%ii%lwv%G^4Om{!pi*KKgd?8)c$uND1>mdH>m
zE%mb+hJ$aYTlq{GGKtyoS!P(BdJqJWWaLujIUs^S85Le)Q^<@WA%Mt+?R)<*ksaZZ
z0ndlk0oTD$^2?ei90Mi&4~J%lV7-(q;v7jLId#xO#@Lr|x7-MyN`d~&J(ejsgV^w8
ztuM<Uf0X$_sFxW9EWqNSy4NxyGzRB&0b^&ByZjVk)R*pg+VEtR!AQ-Yh|8?|W|Amf
zC`W}ESVJBX&KFxJc#2}-4(ZK2xjyCn994EC3=d%$x|NoIQKfU2WbYI(s(kuqR3XDa
z<<eNXpVK*sUI1^yDUoJ>(iAE&MT2b9^pXIZ4(cMKN>f{Ek*jb}_A^#YPjwH-!>q~S
z3j@33y}dETPUI5xuZCcdg;bvN2g4vU4=Ug?1;Rd$&Y81Ty_HVJ1G#Y-Y~*LW=JBl+
zCffAsm>aP>i#zBk(<1O@Ve`CD7xGQ7neJKsI0m{g%Hw>88SEwskKD08umzO#g$&4Z
zwPXF8cC^bckqa$h3M2*=rU8Eo<2rk-j>SoG^StY=EqeYmt{o7WXlFqj#>9Ck*?cP-
zmmXGSU#H|(1XX^e<DXFe1ewX>=P|ya?QZ^pw*;|GBw*5bg^_BV=RpWHESN&Sdo^EZ
zMoO{Q+Tk^qp5bpm-a$q;V_BAq8@h4Sktq@1BIf&Tws94`kh!R0z0oMKJBS{uc7H8{
zYmUZ@UejjhIm_g2xQt2hxJ`et**OR@^=3-BeW<b_gJsd$ulbsl(lsR1ddD7{mHh`2
zBI6L|ikn@kKKB>xB-1eNheTlJ6=QDY0YD|-@F&VeG&4xGqEF)*5D)qAZpEkdAT;@2
z=#)b^5e{i7v$ZsBLHrpGNT}{PyT1oLqo=>r)u444)Emwjh#|yH8FZ#82xHY!5l@fd
zP?TmgFBA{M-SO|%#M#R*E2k@qVoDX-uHQwwF{2e?mAwnTqkhJ!zJ^>lix53$2I-2{
zMNx+8`M_pB;U)VI4v1po^D19{zY#9gEFP>or?86*x}cE^3>1J>A#<}>0Aun4vE5TX
zb0~i7{DP6fRO4;+K-8Ti`8%WbrXMevS)XN3Kwxid*&OKUv*nf-{z{N&eAZPG-}Ai3
z5X%o0Ze28M6nj{xkLwUa*EzzehM|(!`{m^DH|mSUnwME<Hz6*K6rEuo?_ghA;S>b(
z{fd(9mpV97BqY)<I1m2L(|4^$--5z)iD;+&i2xoiSlLB^8B}`!U6^(D-jixpl0+Pv
zUm`0``p*tCbJ)MXz%?fcS#!#v>lLWD>A%eC^HV=7U24sk|2;1{Q*DdXeyP5Uc-@^o
zGITbRmrFcjLizAXuid3EZlP<07&}TyDKF|^_l7O!9fP+>SbdB@P1w#vo7zEJAItFh
z;_-6OPVn{jzC(iiKj8P$eMPmKmiNx?_aVzz@<iRNfjoH>S|?<ivP`%Zl(EY#{HuaX
z2E9$c3~r|PZkk{C@4mh27P(VwKUqtnKVsQ1@WYfoT_-2)KRHqL8$7BxU{ArfGZ&1@
ze10FHjkwVtQ}neAMcyHS-+vlYxJ~L;b$z6D((P9ovmLHR{mgJwRs_USLaa=hX?CMk
z+Jh4u7r$4%?Lom(mGJ5<1x_Lx`6A`0$uhxD(cA6;#xgE|u-A{x^D>?~9i7W_2hOhw
z3`e5EGerpAJ^Z?~_$f0?!=%0ozb>H@am8>fR{J`t*N=M76SIac;_zg)*AGaKqn>vD
zV8ZUWjD!PiEg*<8h7CV&I3Kok`z@67>q7cQ1gLsmJ$ZyB1&>0Wkv!*?Z~QalSf3|w
z1SshO8NDEUaTW|hv$HPIxRyNbxc5+;c!vWlV`S9G5_E$2Xw$DyoOLj4Z6(KVjja&@
z;RK`-Dv!tDtw%z3zf<{*H>dB@)Cj~15BxB8H_BY~ou|yhJ$#OqjVv$UMf<@aIW@Em
zI|-dHyhA8lX$;y4+L#<jN~{|p7#rO@)l<*QnLReQ7|q;M1OlUR8UlkzIvtF1K_a-g
z5=}p6t<3RULmfXGXD2r3EaeY#%lfukra9UWo;Uvjsp7{)QsT9qd+tk50dBN+Aj8yE
za`4&F)Ix(-vAZJ#H-a4$X%yG@oqiyJ6pn@CD>WM;3a$<QQ3!mBRJNT46b@pguWL5i
zSiaSXBzqo<-1TfR#T%AIcTEw^OzgH88gcMD<PW4{KpZSfN2<Purj+u_+f@pfQ3kax
z$c)24e$Q1DkAO29(J#i28Lp!4($xkV4cYX&A-;WyDi|R8UfQblM$yg*xTo#t^HKCU
z;}Uf;`hh8v3BtPiplcO}`1Ca}G0xepA_FRWouJpke8V5-e_Q0na-)?Ga`l=AJjc#Z
zEM3mEJ^hjGdi?u{05^)8fPxTq|MT;R`0PEg?pVbYxNYw3);p^fRGr-SIpG_my<gVl
z;=Wel6QWU_&?>0=_ue5C4#@vev(BNi$5x7=w(M>|EgZ1L+qB0KV<K&Dcj0M|=|}xs
zmgOam&(1Z|1!c*~F3EFD?EmwG2?0`{@&$OJcCr(5Ixw0{8?S$;DJq>OECvbdUAme*
zRi2<NfaChyJ~tRc9k0B15>uC~_79O}5&j?Yh~-TE@RK=kQsN)K<O?wJm1mqNP=IxP
z%|(8qY*67pfl$J_;4=|{1i<X`-xL%qN0l+X-f7tR4^Q=ff_}W?QON{gBR6)HPb3o<
zH9!xsW>S9t0aXvk8~&s2Nw|O$?L;YB2wGrIJe~g&SOs{nx%0!!!8alx4cZ7&Jb+pv
zO@;f9XM<X*Z~gbF==8dc<@clrvNDK!8Hc4r_k`k^1*j@dWRqL}HGSYAA|$w;pd=_I
zHl?G$pBuq2Q4mSh2E@lTW|=Ol{58^_C}jZ5<_VXAhe&}J<N}~uP{VO5@Pc?z=_b7I
zl*_z;<{5yS{Ffx7wtw3(4uFB=#r^|3p#r|>{|h_$zigcUf3tBK5P}ew_qC_`y__fn
zUxmBNYd&$6l*bP(>hPY>j9`ROizFz(ph&9zEuyj0{&=P3d%K~U@VH5{mVNs3Tsmtu
zVp8%yWTK__m$76*OtF-r3;<waj1<#?H6k$jKbWFdPf_E4n4$%ZpOk&E(J4t=kpNsF
zD#~d$YTwJP>5~OIvMRLr;RU^-h;Mzv;dfHn@`>eu2YPFF3J!o+00X!P8dgIeX}4>h
zSQhEbM>5|;zqB&xUq9qs{srNphCbnw^0`*|Y0RGdQv|sFjG#_{D<vW%&;k_)tL6&R
zdoSq9k=>tE23D(HzSdgPC4S2Qas{@;ClHqFwpDn|!f6i{_u5yFwdIN89>h@{qLld=
z&j#fD=-*X`yeIduiwJ_RN8D4GPR5J@Dnc-71NlGL(5lS;f(@}(W(TM}vhpW-fK4j1
z8;`4%y0x$08vZOg+8Y(Wqw^m8fT9_$D;s^+gI>(}nOJ=4GQp-tt@$!iE^`K-z+gI7
z=n}8}o<n%)0p8T!Gl}r2|6@I}A7Me?w~UICz1I%f_*2b-=-!;1H~{+V=>R~(Goc0P
z05Ll~+*MB+95P^~LM-u)SaAje>aSU~hf~~I{Su5k4e_Q&+sTD#el~A|684&Wr-u}7
z@3+;&qYp19p-;TKet^+oF)LRQb;{Ys#c^=Gb(CbW{QXVW=7&bwT%`*}bJihJXzYJZ
z6TAmSs|Jw6COgp+$lw~E!!O$QdMyuQi)08xU2yVOCR!RZwSg+Bt?0qv5#!^Xq3`(#
z(b{nv;#$ReK*Uh_FoPSxAZK}-SbKIf&-D}up)XZ<yc>RltdYx3kN-@*%k6qOA186;
z*#vMU7*K0Gu>4)pW1@N!f6{*UzW?pp=89hmZvPNkvrlK^KZF)i>gi0RxO6i#1(8$l
zCy+0W-xdmp*>JpRE8?L(naDZvi;jLSniWlkwDX<W?+tFQ;*>&?s2cba5SW(cvprdQ
zB^S7N(%p*CPI;~Uj>T9?du7g5w0q<-4f18Ut5fS=e)B0sd}i>1eZ`w6SMPrSD?a&#
z@X_bKuSMZd_fGa<_*5FMXwe-@9JOt}l&tw5$d$;oY5RIR1>W1H4~fCMKRyk<#q4pf
zU;gt=_Ta&>WcL2r!}Oy7;8GGg9`U3hL5-X!R*fb|+Kig2Ayi7!Yrme)!*(;t(tUL>
zu7{uSCrXvP;O`ElCWK%oNS0q3(4o%g&l}(VVBr4lY~wEC--ou$ACG^}!uQwj4Np~l
zLzR)jvL{(-x{xj#Ij>YB+dkKu<Yj(;SFC@pZ(a8PBGVX}y<nhDIM^-emPs>x{ys@0
zWS~d*cz%hvbKn&N9Zg_(Jx)->6V;2C#^pU8?$r~Z*Owqd;t#+~qj;fH)y6voiPT#s
zVG{>zUR8L*(DfGdQ6eX6O!QRDQ9J|yFZYD9JDd@{TYCnoq4%oxUsasJgI4OL&lk1N
zR^M1svQ!EUyZebDkPxtKn<<3r$%^VK6X?G+`FhCvAJg7VMGiYpZqi(uTmfAAzjgn0
z&JP3#L|h1Z(es37lK>bc|G~4Nv-Q!@J;nS1D#>+6mW6K^Tsj6p#m2*h|9oIvdku)<
zch@~ei&)DFN3UCQ%L2u~2=R~peGz~w40%Ev5@`V(w>ai(jn|vjFwmhfwPVol$iL^G
zYfJ)<ZGhBU&s+v?39*w#5Hvgsh*5^VW3{mF=*<_h&Y=14y33e|5b^Wsqe7hQP`3}i
z8Y_rQ{!wsM_|7Iqhr9mnB!Z|1n3Lnyi@awb3{#L_*#z9P5r&k^dp;bRu(eX<4C!T0
z6R7^%LR|8GU{}NLg;V?kwqg8lU|Y~eL3sQVd1!F-0BOA4fn&$=&1jL++7qRr1f2*c
zRB?$7P(flX(mb6OSVAT?2mnOHzHVhoe81M2LCP`!8h~_v>o?pNPC3%g`1#l8=M9qk
z@=a&)+VRr88%?7TK8G<3DfRo3%{|t%&(JL;dV^Xp^92CKV7<RK=A?<F4#HdqtQFq^
z7xW8dAV&Je+2_^q+;sk;S2aSlmw|Q9*T(ntyHl>qv4kqr0=Z@%r|Q>&l{gX$B>guG
zFFR(IBaO$Uq5fDEEb)f;t+xUjZ*;{i@4FfPc9b7=*PHGqA1oH<+yWd4i*~-R(k93H
zF&$c%vYs_!EI6e;k3U5R@BYq&$l-rKcF1m4M{xaly@k9miuaS|31L^mR8wp9G*z#@
zq}n^818{mTBfkgelhD(xEI;2f?DqMQIjiiw7<+=#R#w(lb>7<N{~)uUuqR_1kt5`-
zeCKrs|Lf4|xhFWNjkNX>!$fd&`ok~t#-EF)Jsx$|ck8o|^Zm{@0XX9QdBseWx1r8T
z47;Bu8IAn2Q}<r+=Eq(~5ov<Myp}{{?|p7tw*D6j;6eDLxZk)SPsp@)BG3Wv&v}SZ
zO&7}5`FNee_k|b3a=v}Q6Jq&ahRil^I0U($0FbJ$a7De(pE(j3z$N$QTh6Zg#~SA%
z3-z1K-XBrxk0i63jom8Fey`HHRlIA}WLWbS(VSfO$^FM2T>rhC@F(_!`1XARcX0eI
zfH>*Vc3Z~w8t@`AG0O^Fx;5m4Z3b`cLpNh7H)i8yHUsP~N)tsMjP}!fd>h`wvVQ`!
zkI#SjmftM1J2&j<&}VU<zPLF3#g`FrmY8rlFXHBSFBMz-d-8ph@KzExcJ<bPotTAB
zHZBZgDYzN8-hNkPaf#2S@oFH+5q+>jVOTEYTdjH(m7=bX{N(-&!`cF1o2q)aJ!r;n
z141L`qLe^<jtCkv6F~&n+8JefzPtO4CdMC2CX68h`kiW!<o`Hs&v2mP0gs=z_1ntv
zBTPRNj^(oXji$L~>|#d)KoAxkRsSAOtgPINw{mFg#5|BlZ(8AK!JOPO2G}y<-<6_U
z#mF1lCNP2T8}D1G4P9Kk`V5IczK@fP+p>Q;1buq0Y?>oYvB-ERwKmK{T=tJSB|{7X
z+T1<>ZqkCxFXumOCqD91oFKVZquaOI_c~Z~<_+B7E;9yixF<`thS1|suIC<<3%CNH
zCtS|j-6QnWg4|BzqPlF1D(V^r<wLxzGYs33Hf-x%qQOAr>o<*(79TfH)lF~zwqjbC
zR8+0$l8F4uto+s5ZqDy|IT6SFR(8+b-xK+XIP+05t?)e(g6I*I9?Mgcl>b!h@QIzY
zWNN@Z?HQ|;@EfM6et3fLTxk=)&cv_<)VGA$$%>~BH!*RtM|OE&zD>S$Uh|&5he%t|
zZ#jk3tkPf)j6U_RqC{?JlASB37wVnTfP8!8`HmNn@t26iqNLlXx1-{to~_zdaw(O=
zo@?zPKpG^pc7a^I>MQ=)0dD`Bv{w<{KD#A~c5+*Yx}<d|b(a$+Q!~IArDNPuqL5*2
z%=6Y=$3Ita(NJ?T@pV|#i=gUazM~RU%J%RLA{+s2VNnN%Duj(s`o?k*aYXHVGpbC1
zoz?eW#manr*c{sK4l(`<_AFWsNdc{%0qJQZD~4&fS^wo!8O>pt;fuiZF`vgkn*z7B
zV>LnpwUVRwA1#xvUm7-oUiMIvWRrBcedH4GrGCu-B=0ASts1qZr+(U4Jojz+Bg(zr
zn6KCodw)MUjfg+D>mxz&L}aR^dp{^b;F1kshX96F1V9;y;9NF7N~7E)?I&UJblmn7
z`)o>xnL)Gku}yx&#-$P8zU6yrAtX}qV2>3Y&FJWQ^A&&nst_4<kk<|n0FB<zJVYv-
zd?>NGj_`y+CAdxo{!vB$$Fapo3kSP}RHAI!XN<m(b9{gAWQtIk+)v9+zLfD@f<0$7
z{JiP+_yQ;c4yFh!A~Zsll(N0t`t6Hn$2H<-EBoPnQ>zd&|5ZhNj9-2tK<m+*piCkG
z;RPjcQz6~!P>TxSNtM(k8XzuzgFGvG&y^P9-j}0d6U-wz-cs~stonNeyO0oF85gpy
zGY5a$gSxB0u_J?>TlfIVb2DMU@CLxB=%zWnm6kv8VO8cw0^>_NN&y!Dy`yCDK8XpC
zRJFG%#2+*_^z&udPa^Jk@%mG?NZ%1uh6T#@7^~^1zYppUnno-Y2BLi&uAJ6lMX{3e
zX90Xp`EJ0NN2b#-8ck0&@RwZM*jSgvx3p|{nZbL6j1%dFf2D_WAdjUQAOdtfA&Yw;
zPkcPEBY2J>Dn9UHf7aW$TQJu#6_$D0)9TK`>ZIq}mx;S4kE#J&Hb#`!iCwmw!2g@I
zhv;gKLwC#dtNgp1vyuCRT81Siek0y!1H}0`C}N<M!SA3!O`^WaPqQ}<j6(Q;>!OnC
zVSU#1b*JH7+qgaF@lLh&E?M&Dn~F^GyxC+E80frf$?_<*0(|$ul(wun2=fAA`(=UI
zEuoQ=sGP`@s-2RhN!Pqc{oPCIvX>stS1{(6NWY0%1RTH3t5N3y4sLD2@!2%nmP(Wk
zlR#$I{}vCfN1){Ere!UqSf}DCf709SY@}1gp(4|6Pq(|kR5AF(Vwwu!T*XPf$SmUk
zoB+@CLi^#kjWWduY-tRjHn2PYezMy5>y|yo+IF*zo`U@cwO-UaF1ZhN=i9X?JtE1&
z4vg{zkQLYKpbpw*V^}Bam<JtR#6FzW2xBMKTW#D<RV}syJ8VsJ>?<B<Oe8?wkYY(T
zDHIVhGm`nX^f`mcu;!Do>y9vhhP6qqLmLbRIq(5i@G=_=4XX)rg@^P%s=#x+KKJrz
z+CEiBJa^_Jn;6-GMgcXqC~mvY;{TgAoPC!HY+irgVy1}e+v?V}?Qm<K|E6fxog|sp
zNFcy*wAAEV*Gh+j1;p8lG2>wupLX^C@aDT>SG~Pwka>0HC3iD>%bAYj{A`u$_{3UK
zD0ZNrbGy6^LXZ`s!T<70WYMcBlnsV2WrJFeS<3;z?*lHNDQ_ke{0isR9?$4crqNIR
z9P3l!1Fgpar}7j7xM&3^UI|d);qVU&XBSf7QpR||gXi9PRd2RJS7pDshl>Y&kB{L$
zv=icMV8>K%8Y-)(RpPQ^C_O}kLPC#PxD$wk%tnP=_HobVLP*y@tkiD<Q)U)p?)aOj
z1cMbQnxOI$b#l^SBidgKEB|#2=I)%^FAytXQTIO$Z4Jj}VBiVHZ?(@q{Xzq%vU8aJ
zghK5-pBj|BC--x&n<w~{^}sgCT1Z;jH-6iSl!?1?fT^?Qpdb=czNoRG?fIUOxq)SC
zh^Ra=OyrlQpWH|1ig&q3&#`orGg(syh0U`MZVxfq&p+9bV+71~8Zqr)nA`mV1h`!X
zz**<a0fam>%6xcpLjjsWYSi_8Z;7vPyMT$2rE6g_aKiiG=n>IUgsJKgsW~oaHN~sY
zYK8f=NR+jnyZf%?xSltEX~l$xj1-^Y$sD#!d7`6!xU^=20*P%V2?<C^%FY2wiO(LE
zkE&T*Azt!Fsxia|CmU>CH0hh?;vfGv@wi<-lOp7}dH?SXc*A3^CAJDgK`~r4-QrA9
zkx_3nyeF#K_y$W(K2YBF0T$p2_%sPfxQkR7Q?vog)07ZIv-XFR_SB1EN`q7&v0>EF
z`2B4dE2h6M$_@qzs)&4tihJL+rOUj4jFNn5PODzae84BVCf>**yjxVT&ol(BjLJeX
z@tv^<b;n3@8)v4PmJSjt3eLeVAQSq`QZJv4;)U^m4YDtn9IkQ#4wP2RxS&*DpkhAH
zD6>8@N8o!>d_)oBnrFrD-jl1kNpTyXbjMcBeBcaCT2_YrLur=kqTCp<(;zJsA-8$V
z!M!cade+Dwa4@pZ#B+=H9|ROj+6?N8SX^*`E(qcvL1h%))AhxG+=ZzWFU%xeswVH!
zZ~|%9vB(`=M{(LadH%`_(Tc(Gta6(ZK(g4L`Kp~J3LoCONNorYD#}Zsf4xLJpAzLF
zXIt(N&!hC65%ZMecNaB*gW1GZP{x(L*TUQT4u^`aILy~48i$j<m3chidY0ZAImH}5
zV}Lot#5Ic2n{%5&39C&}%DnXLhOc?Hf*k(Qp83f;qUeyGF!G<W^c2znQ1kmW;jY?v
zR9#@@Eo9^+zWkj|)}1tX)9-z4*YFA^>b;~ZSN8i&nTuMWkD8twl?fSN;O`NVRb8hB
z*#i@fk$FAz?mOp7Uj=Mx1cwcqzKi1_$jNGDj$E)(vBm2*m(9*lD5_qjN^eL_(OXUt
zah7OA)HDYEgR*J_td1Ybm29%v!gP9Ombtig0RP1OWpd#2xFdpN;!+MX;|D#59nHcW
zErc1bE|UNDrpoQVQ$9r+piWf9h3N*L71^Rf-bVyZAwgRdFjA|a@x9LaMe?uuQ4H|C
z^giy@N;DlN<*>yA1?;(7`=vE<_qwm&72gIS_T_~k|Ad+b{?@y3w@82?CI5ON=;qzn
zsANhOYo}PmZ07yF<qHXt+T2(6J+lzOpyRXnUtl*RLMUKD{*G7pO$0|fH8b8;B)>j+
z03W(IIIR=))nG>e3bW|Hx#lU~^4Qs|OruHkDNp|<PX&c)xXUa!djJOh`Y;0?25M@A
z)dD*&R<b)Fiq{#16oOsG{1GtjIysjY{3{xUvv6%Wh^fsPZTsMJCl`##Uc76(9gq2{
z(P1sX-JR^Ui+3v&Z>xhC5R}Oha4I1%Z@TWXtK0=WeJvo7X4+Dts%-a5cQH2dxpU4i
zm>p%3h}i_*?ef(XNl?LhT#qVMZdB0Z*f{uH`qr?7>D06Gk;;A8%D#o4?6>iCHeW@+
z$#=&$L$w#M&j&%WQ!=iNxJ`71gr@=&jKoi+Egr_pXf>WEc$9v4RTD(!Nt)?=12k<n
z{h6u?0l$k2qsR52Py+}OhY(6x2I)#FL}$Q-8&qOp8<weHDES*Q^ce53zNM(GTh2^#
zA`URy);fFnlUbQ-R5!ib2fCNW#|sL)^wZ(+Wu$fV1yM;Cd=~SHzqbjW2UGT3NuK$p
zPtwTRQzN;`6iED3j=1OgJi+DuE$&rO6rDt=KLwhax-S}ogdv~-y|)DHg{gg+In3Pu
zJ+!%x5ES_dSXcECZnv|B(ljK?K9^~GLm;-tQY&!<5C~aFe8=o-AP5+G(NOkgMW7vA
zL14L8*ppdHYji#WWTMo~oOC}`L$`l_nCL%EX<JW`$<rn*7>d5YtKvripf?l_TuoVS
zRYWQjnek)Qc-`}X;V~c{LPm?iE}7KtiX`opp`3e^o(I|{>@FY^R*s);88JNua*e`l
zrjHPWH?*1Eu!SjruW_BT5%k1hC-eMv{=el}&=5E9s1Jrjzuto$FjXkSPZ;Rbi@xi0
z$b#@vR6pl>sZx1>1XN{?ge~WbD{{sB6uIxOuufl-(^4l6@DWp=Y{Jue0MjHv(gy|O
zTfNEGKwUPLg~JNuT<o&*u950>4Vc0mlw@1!LSjd?w*DE0-y*_$OAr|%0W!j>M~a!M
zzItzY!}`gs48X|6iWtp5S}%V4j{;BD#WsMBx?P1$%Jp<-1MG>*Z0oz`H3g_V<TFwW
zz5yA5YyXYjXedJ<<TwPOo%f6jfxBSJxhD%vn~uG%8rrn|w$d(Mxil45NC{3UV;vNS
zBNKL|4rBXhUWyv|q;c1_Zn&nDeasu8qZ@n`pl6hEWN(bLy!B%1W3f^8mFk_R%l?+B
zmCthgZH#A60wvZw@}KXtbStUgLj{~K?tgk`5alTyUfeV{E_eAe_#9^6Bp#jiE&BA-
zn;zop{u!b7RuU{x2B2vFq~}O4Y;~5}+q4*eg%<Wj$-O$p=}%s7T1*Z;KQgYr=qPlf
z8^DBj2fb-as-h<?*$spKk&}}51>{|V8N7@os>K;<lTiMH-&INptH(G9NYgP4Z)3fh
z-Z%uEh-U&|L66c~W$*0mZ`Ir1tr9g0LZ_%SxnLG~wds<+ss|LLe=Y+rx0rO7Dy6BZ
z^H-E+#h+Sig#XcElO@HEG1vroUgvTX^%o5l>FLYo)lzF4MztEY>jYJryXGeekk2ej
z`3WS68ALCAlr?aCf8fMjqLlf<tG+Plm4xtK4G_Lc^NGc!{28DHS(S729l1&Wi<p6m
z9;proa7asOVrbz^BYf#EmJBPRjHo-=PK^oV5b2_f_aQZ|T(~ICS<8kMd>v=}wmy7G
z*2j(P{$A(q{=y+olK;=VA<dRxK(DJfqx-!Iev){Cs{KeJY3TePHw3v`WF1^>j}SJB
zn+-0D+&*OMIzAr1Z8R||g)ib1h+pmsd5OlEZe03SNyQ`~R8^$~f^gZWIU3GrPOhW)
zEzXB~2A?CKsfaH7c5)sDn=(B5afiwz^u|}##7Bh1SJGnd?}oiSZpZpyxgQjtN8qFV
zumD5~I8z+{;IRmh{`|f1B3}cWz>-F;Q{+?MUtSI!*MKip_6uQ6*fSg>IWl(3Va;(f
z>mmOU*4HOSfBiwj#@!=qqi>6^BC0x?hPYC%Ui9tGtC$LkFG|R#mw(62#d8%t(By-6
z+qlK1Pg@5X3m|L(lu|VbkECDVy)K;|3T--EqXgaffM0Y#n!g&t(JHyHe1B${=*V(Z
zlZ-AW|1<;Hc3tLGR1KMxNPi^4$OIyfnOKoOS^>)Q`k3@@HXzZUCuBF@MjD&_j+Oe?
z=LKGSTFVbXc|_-4Q6`Mv-YK5;J;(U65Q=`aKt>&ipCiL#te7#1OiPyvAjwyLW7`dH
zzDAJDTePJPYRWWtbLCtFv^XiQeBRRdlpsR9O0H|GIhK=1jN}gx;=|8jrCCHCA$F+)
z>$6X)owcKem9xo{A?WD>sc3p&N&BE9+kX}Dse=qYIW|;ZosT61Dq6Op(Bq@79E{f|
zCKEHP_sr3wCK|K|soTNljRi*=KgOp|d*og1k6=$WHBOyLD!LTxy}9^EPF-c&kH6#W
zkwC*4=8SrPrnnV3x@<)?$KhGj6Dr{w(N*|UuOj_&Rj4MW%;p2XO>6k_`hUmh>oYUI
zHhpg<?RI+YYGxbm{1ntn)nT@;wHX#K^%l7ePdyN+&6~Wj7^Eay0rUPvz<n9<u^!%(
zQU6=28AvKHd<l-92@$pNq&Ou5$P+!H)y>NyHuQ8Zm?vpC;Sg6r?h9HE7EPdIx}-Y9
zS;NlfsMl>#8$RGqg@XeX%m6RRAwK^9*r{D<sSyqs^vLdL{WipY6;?3;CgzGYB{?NG
zvvH%k#-KUF0=q|Dx3!XVffJ<DAJ1dr^==1A?Gq*RMVGU`@qHS0Ta=d%<Rb<UNKwnK
zd$g~Sf&<qVrb5xaNGN-~VM*rTF`tT=!4HYMkh2JWv)288ULKy%dDSLI82b#T(O0;_
z*cB(uq74-TEHzs<!q=@#uTuKgvzZ+0A!X`U7bR~a+8^(<I-pblDPQKnFbWJF5Xj|^
zTT})Z58uyK;+Q2Z<_rjA&JRb1zp=uKwRZSCbv)qR#Y4K$PaT6z?GRQw2F!QIduAev
z$@|NI)*y(!yR0Qy;y+bfPDdKG^e^E+K4$=6q^8m0yMyD2z5&H9KUu-TuHrsO)REu|
zG3et714roLRLh5mXOr?36}%vVquXO97A=;gXzy#3l=hW>s1hz-bfGno1;&LSE8+aX
zDqxfERYR#qwksvjqot<%Uiu)9R4<!xqu=C+HidZ7zWLo%<bhq4H3$yu*iV<303(%1
z209!n?4ZBHwTDv#7OHW4ra|n5*4nG%Ho-rf60PGnYxj$t>u;L)Pb(=x{%jj<GQZZr
z+6yF{xElduA1qcZNO#iRCT#M{gr?T{>F>|g8e=>q2Cz=W(yudihCFAY2S#qcXsIIC
zVQo|oOj=i>M>tI{d&fv<rGF`_;&I9S3egR)B0{JR0cw<|Yk`Ps_3f*lY!v!FvFEB!
zTP7puyq1R-2&D~KT=j$nIP#W#<RozLj@2#(6hopsr*DY5x&YjrD#xgRncc-5llhBy
zA;=Y$p1AbK8p!K(*w-z{`lwLu0c-Rmhx<|3=jAb94zI68mfN9eCYbR9%#Eto=Y505
zIzX${4G7OT1C}j)=Kn3akI~KWQAR<UC|n(%LC!DOjMvY-sn|KxfcDL*G&8C<DJWdk
zK%UBk&I)<#LJ-f2EZ)5%6LBVl{$Ndb{QqI^EyJSxy0&o;1SJ$vq(f;?kd`iy?vhsN
z?ir~8L_oT`yFogJ5RvX~M!IL{0fu-lyzk%rJRkmF{-56W!~H>zISyv_wfEZRI@em~
z-uv2#kAIZ<>7yg-qZ~!KGGh2YOmPTCkVsrpS<=xEH{FD<+;d|3Q1anv*A;L_f#&b#
zRx@dtKgo~myrj}o`%YgBNlD1~#E!O?omSJ)`53N5-`x`wN%PM=LC-%ZWj_oGsW!}K
zz_2Z@x1Z<xd{upGju85jSJ)_$I=mG3^0W3{3+|cWK`jjnz$!R}GI1{-Jdk^$YRU2)
z2PE~b@&!n$)9}ZxH8GI~!supsPymcr^4PTNO<VLjLbbz(fLjn_8X1$Eqn(Ew{d_(@
zJE;*5&BpM@A}G7kZq3_!E^_g4;bZq}10S>dQSN8ufdd`t=81JVD6;!5;rS`U<mOb3
zaHRxoc5C+{mE>D^pqUK{vXL^}>e4ojPbl!V)YM#W16uUj4Wd_5H)r5Bb+HivgMg<Q
zI%%F5kXAOi8svTaazp>VJ$7o&#;bZGpFFWMLHF0>{B413tiIlM=fQW8g2uN`4YLYl
zCluqCts{g}IDE3&uroeAh~3R34PeWQ#X)CptC0-FM1P>6>~rzu)Dy>zSKIG*C*v8n
zz36eUPLZRe^H5p>l&Cg2g#|0wQ~mZB2`j%$!Xp&1OYuyxj#6ovBx`J*Ghjs);B*zA
zf7;##PM`9&B-c8%rxCGAu=&U9Sh^)BB6Xd|PX~2V>{VXOxLyihA?V2IG^86>R0i-m
zLrJ);o%$yG%n>u-?;x*%`V8;&WT%0jT%n)qrDD1=WDNomiOq$VpI=c-Tws#ZG{?;s
zm4b-}OAu7vY5WH@#`>?qw`%Q&y=Lz#O<w8a)PT;e-y@Xd+|A(CY?6?xOTBUTcNJTv
zo{tr1ES;T9-2Pzy@$THVH*iPaX4X9u=(K4ZRw|-7QwxBJ9LeRpjYv*;<YrMEGYuTc
z%N3V6{Qz`bA<%I%`LcwK0z>0Q(XYGlkXC&l-C~it1(H7~I8|FVeB${25}UqL{zLl?
zRm}`XvSG4tw}O!KcPLT|wQr+DG54BYn_d|~3Qim6V=7k_8JASI<Eu|^d&4006pG=X
z4BfTOG1C)lEcP!7Ud(}9t|DQI2uox2qdFG9ciauN73LIv-;OT-Ox+0J5vIRe^kh&B
z56t!v{}pXEf_K`W$;N0E=@3jVD5ez`AnFz~iu+!~PZw21R@TyV>@9G+_;?|Hi$314
z@1)h%rmM{5$WgFLGa(lJQ=>a;qUmm#MJvivoaNl9`Z+4;X`m~z9-M(AKxf^|a3B{p
zrL)+I_sU9}*UN(FSU#D4FJxL7cCBI=`0c3mvPa)pYO@`lCy!CAPi>i!j-`M<mu$jh
z$|=(`7^_uqvDQER=$omQ{tz`tuM%wQs<KmWsA7_;+E>$`)kH~{OrR)qEJQA2t+WH#
zHaa<)xfydFNRIk5fe=C!v#JMB)01-4_#3eYtHOQ_KC1U*FYMvFzPVz1<?BSrjVm7q
zO&~+ZBg~u1sVRuWOpG9owmU@*P0}3pxPlV_4@2OLN>Gtvtg-4HH%{{EPkUZYc`r69
zU<DDI3EP#!ITkE5E%I~S5LpKE4U10=^DjM*GG2?$Y0|gw$>_AsQVDAGWS>7+iYr#S
zvJ7KvyEzHz&xvbD&R{JY`O&%!?#4te-twJ{gqI5^Ju|VgpSIO)C@oMV`5@a~anS^d
zVKSW?{G()Yt{gyFhNd+px8!Q85C*gH5_P{esQzxC)9l$1m%5_Z^_8oF%nbH)s?33y
zG_b2~*V?PMEx9Q$R0uY&+2DP3nIc$sG*^=3i&QLkNP`G@p=`MFkpA$7^ZvB>%MS=1
zUDFZ`bsC3+pj({x|AKgXQ_~!|{6=_P*Qb3d^=L?zV+_wg1k5vIOmFLa!-JQHl3j1h
zK)lUs40dTt%W#=7<dq)UdkHm~+7a}OPG3DR++Q5_Y_vH-P8BR&n>w5!Zgk(yt>%wT
zHhCC+BxZY539oDsirE#})`a}1YRw6gtht~Pd>`TfBhr)R0L9iwcCa&O(%8b_ty(v%
zRI07fE_7V%Y0*v<xv1cZ%f4Mv7X^XHvxF0Qx1+*7nk*E)K&Gt~_*;3Hb>w%f`YBfu
zx_4P{wXwA?OoSwKuPm_!RU-7(oo>!g!Q-y;&x_*J<D6d!y%ZN7vm1|jEctL4;*tL6
z8&YamS^55Zc&nE@-xnzisxX0aXo#_cMq9Yq@YBc6d@+P=Fsg3o?r^feF9m*-M2>Ga
zBmVm`ESI6T4i`dmRIhi>$ej%j0v8Ij0`}+6WUA*YvCS7Q3)-E=bR-w-u)RO|@sOoV
zb$BFq+c)+1)Ennh685p;o==G6sFh^?euojL#cFw!MeGM4Y@3!LX*Q;0X=Q77wAkAm
z>UvCYJAOa9wMNk%DHtOz0l^`1@Q7ouPDTgc3c4Ngji6m@#IQHixzqLlJB}>srCYb_
z609?)clgRi4ob>g%0eHzw@>ulSA3r>Tl%}eGQ`W^R%-ZA_^&5)sO8suW~AXSqjQt_
ziiTOt_|vo5UBq9AJWM)XHmbbJzU@HH`8HRNa|o52*lN0kx#_%s2pCaXDMP|0DQq_<
z{F3Xz$4CXNhl{9?$08gEq2uC<G02Xg_dNd!qkZe<MyPIcuFE^bBQr#dUK3*Uw#NC7
zUjj^lc>z+7bN1NCtA9|mw#vw<p*t*RP+*EDP+1itR5X5DJp?l3VIk+Q-l3Zg*P=1!
zl#w}P_xna-vO>Zm<I-r>Fz>3eyfJw-g7cBN3UYad(qJ2GS0<U8+)s^c7l)Pw3VD|4
zSjP1U%1w0F<B{5@w;?NWx~N<9_FaCX=erfG4scG9#4z{LWV^K)q2&p2K4b3c&N5Ku
z)!mja7Q82$5<}a+3)hJ}6n9!_gi`TdrbICG`d<rD*n(9l^)0)7js_b~MVuG;b8-?)
zuBgIHSMdptUS*{{lvCw%N@x|}GL?Dj6|-R{;(1m!<(jeBlE1d9E0azEUeZx&BlNmW
zu;K4zJJNS6o7BJAx$!{wPwzd7%3?MMI%2CliQYI!FGyUB5PZ9u_gJ52aG>=_i<Y)l
zIfSh)an_=sY%ptaPUFMd?cvQ;bzD)NbSvq1+`Zq+tMt+rv%dB-bUqww$m>j<pK;_#
z_H?kO9RE7baSWGBi`P5wMb*2uadi?LwU@<xx~8mqV^9#TV9=lN12dYTTe6nQ?;HJK
zCh3l&yd6KMljqG%H`|9=5SvM`QDo8C5u*Oj>cs{-IhtPlm;LzfG2G%KECE>7rO|Wo
zB%1>3ss=m_7ObB&*&jI~G_@~k;#_{K8l=Kt#@K0=AmK|`8tJHjrRg_M*iIhE6dEFx
zk!wQ0G%e0WdQv1+N$`_K^_8%caEus`O%S@js27mYxw-Hb1(gVE*9+3-n)K02X`fZp
zUIc9H>9L1-2{La6Qo98>cQk45c7%ORh?j^y-0%<-eQvbBaUdIjP+s~nEW`1^U`#2p
zV(%pbGg~cd_RX!ni6%H~s8{7=lJPigR8o$MR5IJn*19SE*q=7ChM&Kx*}=;oMNnRO
zd9Bg_jp;fdG(uKh)$ezr8(7a6E?PH4&5z;x{N<$qZfy~AIs@`4Y(8TSZH6@|fVb`!
zAqXk%DjTqSpL(!90B?7x=u^wBTP2gH!ox{fffuh?Cd2FBgUr9{>&5iNhyesuX!w)n
zlPs4-fcpYd0mTERgNZeGhk`IvZ*9S&qO_{_2vtHnjiDsmRaRciNy|tmo2idI_zr{>
zC-({nT|xU~Eb(IZ2D+clFyAj%gDpmYagxHHVl`@JB?TV~a^>oio&64D>)%9pnXvs3
zBw8!q`r2Kg+{rB82AQZ_4AmO6Yq@3Hh?PebWNh$Z%6%0Z+gR&4ivCjimHXu)&68Xe
z9AB|}59xqxLE_s#*@CJPKf!k_EFt%w+f!bX-YgB;(W26}D9FLiJVm%tLWXv4y*<AJ
z%NcL{Zl=!O_514%!k;ux;`#E3_|P|2I~ToYCm^1IH)cyY!--+w*{M_;W8t2yKQUTs
zZD;4zW^xr#L&OdDB0wZuKhvgI=F$hlom$6`mg8ehQx<ydn#G=u{evSl%TLo#bd7B)
zB7~T7QUB_;nr9!*I0GJ`|8k!}NW`6WnZEu*|K#hjxW4kb8EHpBOnmaJJchHcY8N;+
z%&kMeW3{U{<5NE?(VvNx6YO^#c@=k)L;2~!$H)gANosO_Xvc?RTitUt5`9ef6|@HK
zU+l+2gNd}t%^~$18?8yVrYvuF`=<8a52vs@RuW!a+f#5)34uBkkm~5-yJce|ud2t6
z*luRDr_zcSgqdv{!Cd2?_8NIDwtppxz-ey9Grv8cvsDUz#hw*d@>8*$M)l**g;t9P
zcQ(Hl_&+vpBsfP^wAuXJWoBzwX17GI&CS#Urzj4xNmE)RQ<?dxAAkK28zI&BC|PP&
zN26&7=zk&C$us?LcZtoXg7vIEoE_%XuM2PT7)hCusai8;SI;H6u&f|!O@f2MPs{m4
z-`$J8^eC-s7cc{kEiU9~9+P=}M_HeL6XPJ1nM<Y6F$`5!U7VQ3?u-}LMgJj7z@&Xo
zHfA;J68#^($Vqjg%|!FC^PdNcCbyMFefGyXUFfTC5bx*@)8?qH2ax-DhjvF7iTOrm
z!i!qPZ{-CLX}(+*6Y#{zE0aElpf0q@OWkzp^*;BS#)oUhF8J?Uih83uUTJ$v5hup_
z^jD5V6=D9{T)u%UDo-J|r+mZ;O*h41c$|U+F2S;NzPY`JDv@twK8)cdXHoI>`J!RU
z`HKN&$s6;op~VB7=F?ksIZJ|%B8FtwGOWOD@c`}dGD^n>?t251jTxNr7dYAf103K`
zeV)Xf+o^4f&OHf0!wPzn%ACjj;0-l<7$ld7z(?%Ezd}XFnP{Fk+PzL?8cQdkXR3IF
z@!Wo9h23FJz(dqtsFeGe6`fG;KafQ8dq*-9t40RQOu~E7F`NftG_QPf2YaK|KcO<l
zfG@8wMso901v6nilus7tnpl0$4?Lvkjwh=%E+atSUigT=#50#;#s@rKK8M=!J|+!8
z{J(tQ$F;I0@a4m&GRYK=rJns~YA7CvWoq0NZaOU2(t^JLrhO8?l^2Tzcx2vaM9sZ>
z1P|o@mHf@z6mSDyKL6|WFN6Psz(<%gDlC|N|KI*0t&5KS;2Zs4`V_LcRVM?!eDyb+
zJn8%ozhU1aU?RG=E?^NacRL#X20+2LWd-Iuae(YCPiV`0&@g}f3u#n!x4_s4m>_=v
z_Wg6@k6co2NnyKK4D<)FICsgreBP~vG|4>@x?c|Yv12p<jJPkOw1AoV^zg4UJ)XY>
z#`fTu=wCRz<M=Ne{tJiy-*EWQSGLuB=v(f#y3|k~n<6_O9QrROd+Fg*$Oa&M^Dn~D
z+i9KzbFYihV4=rqbI?THzXv#A6oFViv8=SJ;sXGb91m#5jm2K#VgkO6Nhk(k{Gnc`
zHv1IFcFrQu(4yb@_;Sj&h#SzLp+8uAkCv29E?ORdhDj4icdzG!?i9-v;4m73474Lb
z5mqM&nr8rqWyz>N!u(9|F*6iB=oJCrIl<r1I9p@{f~_Rb?h$+!M@yo@Ak?{QkS-GV
zoo|{%TwglCTvgtfXh5@gv9de(JNM85=HUaLv2gClbJsVzCt~@GMSO*SJ9>^82|Lt@
zdU`MGE)+^7JJAsMp<$Tf$(Y=AM}-bEa$9OlRpaU1Xz_qXNFsawHs}ld?&03EVvRGv
z!DFv!(R}Ehe9Tl(vsWR$vqdBDjLoiyk{5S<3qs=*{%7RB8GWw8hl5iI9!LQM$ut&=
z;Y)%^!t{^bUcT^AV2=*ZPQDxMXTZAMKJT92HTaOu=St~gfl6K#qmOv@9yK}|&@6^&
zw7Xvu@K=JKFyI+{5Pkf+!O8-i{?Euo(AU78T5I3Lm;fG;h{~sU@SEnR80o+5_Us|L
ziF=gGv&nCF#`gmn!J@0WYfy#;_}x80f0j65U?NO!AD{uvk}$<gkWc)bc{|`4jfbmN
zcYTw6>7$_h&&UrS(zLBszhuOecmlBRzY&Ag_iwvp$^ak<$;~NbxSI(Qppk!|bPxC)
zkl$$(?LGV$$%K+dLz8{^Q!FWWDU<Ea7EZK23d3R4+LCvDBLq7A&&a<kqg_yD5b?ht
z2nJ?eC6^f+0OA{!u7BH2N&+p(kd9J{=WZr`)A%Sv{2NMyGH4fRKb5lNOqGF&m=(Q;
z1~iM+_YV*w$$@9sT{V#1O?))a>Hmz}^Ii|)ytL<`ugV>@zf=U$L~#Jx|F#<wApnw>
z+f2m|?`8rEXyo5eiWCEWhY7;bWbptdqVMJj8qn+;6^4I+xBxuEd9{1;Zm`ks0QdN3
z<SS3bAl>FtN)Nuh1OnnS%RXNKh<LI8*zLbA_+J<NuM7Uy1^@3y!5Qg~JM*Tu6lgRr
zFqP&GZuJ%On@^P`tlsg^L?-<VA~z8+)RvA~b(KnYDgoHMBK*<jPBNNPbGe7l(H%Rl
zDWU143rfOOE-@2h5+KyX&bTpvQd9XGNUd{va84Uk|CO9a^IE*@mS6A9rUa;&>0Zy_
zU(yFYFF&B^$L)?6yo(VJ#P7<JcPW}!SPya0rL_K2phVV)Kjk-MQGUJ;5Us=$vGBk<
z3I{&pNTcZ(zO7pzy<_|bcRBfsxl&G`#CpI~CGnrEJQqK?=TOb9$^sjRm%(U21q2ZC
zU8eB%UeD35%JreUrX&dN>XK2jIA8H!5@JzAJB~U3s!Zr?N8W`e^u<UC0?|(eGjjEB
zh68;5^+YV5IQ%vEE+HZS^og}E-+4>$@CcvZ3occOAYTWfY@P&-r5}S+vqt)>xw|st
zr4La00Qm}Fxo6>MfShB~$aNP~3I3*QH9DprBYzO6nD_&d%fCtzS#s{E)x0B<(MLdJ
zdWH_@20-=!M5#;%%*b_r<Nma}N%%%{SCnNBvS9dq^Y?kl`k%5)W-c>`f<2Bg1O)UY
z>j6!-<XtWSeD<gHdBskl2gkb$c$jyU2!X8@DNtBqKGXT1(sT@K(6jH*-8)W>ov4ly
z_DXjK<qQU%{R2}@C!9E*0vrg8Jdy%1;@Zi5o;!<1;@`DpGZ#6?12k6a$x>CdjamNJ
z);2&bYq6moZXEd?$^AL<dv}mNfeeeUTKuZ4sZz_GMb=W#s)S(g8maFr^1l?nG-Ut^
z;dirvcF$E7FsPZx`dh4^FUrq>|5AJI`{oidnewM=Y)s(Z!JzdaP5-fCEDET1|5lDp
zLK?ewRn~=uiT}>qe@%SC|LTm%r~h{5|1Hay&N~`;U1nVz&Q}b~rP#&nzixPRH?wc<
z0q$_j+|42L+avUi%<t8AY!B$-#F+aJb2$qP@0|EZ5uhXc?1jg7Uk9`gKm^i_0Ca+(
zf|sl*;U1fv1p{u~d!KJEQ1;Fafz^s7R2@}~VNl>uaYSOEeYyo%S#7$d+eT(7LVbr=
z#BK9*W0Dchr&u@BzUS;&pk{A3TT`Kt?%n$pRY7TebG3?;^Qt#K&Qa)?oGNz_I`IXV
z<|1~Lgl<JxG0!S85#;E7+`)%CP=}(hs#|Dr6Czn-A@Oayz3gv3KTtS^X=iX{v>*d?
zTt+(;TDq(g2f?$$He)Wz)mxuKh)eg>+8lwgsUZaCumL~)`BW_4gV6J?ieggwJ2a*{
z`Qf;zr!Rk%s(G2y`LvO8F)wAmxq=_HgQ~t8*WvbbBI}UI?5nzSLbIc0SXHZXk<pjq
zLJ_l#^^SpLa$C!Qr3<wpzIvNEZI<EN3eoG20yfnWU62^h$kf8!*cx%J45wxmzu%kG
zx6{q1efr0fn;D61H<#n3vnEf+O=8LvJG!i=T)`8v7r=l?oc8QC$~LLMT{}~o;jAQj
zuiu}aq)iVXjl3=6bwT{qU05klgJ^m7gbVw&k3_NuVcwT;Yp%NfQM0AgSyF5MFPF|{
zX@hlxU&8%(o&rv7=)Btt@TOQMHxMQOJt?G-$jI}&{NkpB-OA<{8M*DJcwMtbvg0W9
zd+WdoA;9e3fVSl#LZmb6l4{t%cxq|hntof!zp)o(*J;Fy9OcQ&jvTDJcC6^2d6J;v
zxXv24X(UK%X5cJ~b2BXJX%0_(GjZKT6ZK1NSzcuQOOyCYi1HGjb}FoMpthu@)`LFz
z261~cyyrC|yOCSInQ|GX6_Va}fhwSGwiWq;(yU%n>rb==3)5_()=^j_bXByME0SDc
zU6~Ou_u4W1cpZq;_{s^oS=-ZLnSs7rnD@nO^Yu#1`*QkvSX-VdrDUGG0Hs}eiWgMJ
zY1=dm6lmMLO0}_xXyZY&oi&dR1%>C7<7&o{^^fYq7ZWy;?MvI8B68*?6Gb1AMxPzU
zDD$Udwobgj^KFV%ovk})(N5=3`%^Y&Vq9HkFeCO=j134IjU%gbo&<@WO4yruIr$#0
zB#_$K(}O7<(^VnNzVN@&|D!wUKVb}4rHIJ{_Zl0hUPr#|ZrtQuUa__+6i|{26q3+!
zis}Ja2UZ{S*oS%i1SvnaqF}`t^a+MtHcf+UX4|F-;I6OAOWysO7J<Sy##s8iT+ZuN
zLITa!tjFgq^@(X`!V4#q&U*1+<^@v{I;-Kcr4lwyK0HJcr%hRdlX3Mj2%2nEI(q0g
zZRk**?J^CLWTbyzbPae7&t~01I}d2>+CD{d%CPMWe|RNW6oQ&`S*Y}ls^ZWF$Ll;Z
z+LhlJXqeKUI(}E(lO2*=Vsu+QH9g&VU)JNIw}7GoH$7H|4|7NE{rcD(7g%Qyc#DMj
zMuSMsfUj-Sx)W=;zmItQF~J0nFO(tt4ZF}6^I|f@u*cBV%`Q6#`{2l|EUaFRQnbcc
zbobHe&k6fhAUNUv6()Cx?;yzBf)-aS!3*;yAfY=_<R(S3PHs~By&D|X7FloLleq%8
zBmekCO3M9XQO^GIAJP1~S0w{yQ0g`Pl?yWA>z{be!-EMab{+YNL_4p!$u685Aq;~w
zFLDMX!G_j#GmgsDOZoF;c!agP%hMU|J$TkF5f_^o-U>0V!QA`9ZDXaC%Ns>0Tb=vq
ztn4*i6~lwF7+v|{)29hdYG=J-<87T1U<FC|m&#wbwgz0i8X{Oh3K~}6SAe&J=wqOX
z4=3V6`}|3<g7mi>Z3TRDnX4k#w}mwzZ618E3C~OTuGz5GzT*6gry+zs8|%YEO@ptR
z&R6m+kF}&`u~-a*{^m1L9RP}^K_gdd`)(W3P#*6_qTQ59zM0alR(BZI=d1nc48>Jf
z-Z!kgHhV|6CyRU=#=F6^!o6UZiQrU-$R#SL-jKsy7}aob5on)=*v7A0ob4fcmr6p|
zc5{{Eb`j;V09qy@<92F)v0I`0R&=ZF`Y5Jk_p7q#8yNff!YkuA57vs=-;D*)tN{wL
zaFQj+r|>BGXJs^+W2aWf_jV<ue4+IV`(oe<g=v(rz3A<?3k2(M6I#3VtnKKoeRYhp
zxB3<d;Sq-@B4~EQc^9-!Try+RJQTod*7@Adqpm{lM4erO?A&wCVcuZi_I1^F!LwhI
z;N)SL_k3gF2aj>v0bUR4m1@oN*p<t;Zgc*GCm}P|<Yl$3rY=4@HbL~PYx1`*?k6$e
zyVq^b6T$lD%%O)(<V~AaodG4e<`oAhpd~K%=c1LZBW;&!7{oR%njZVfAf9&=)KPPF
z<u{`^%lP%M$<lhw@ek#4PHs1>+?nyBzjw;asjI!)v>wg;talIBHu6@_F_#~<(v&TS
z7<%l8cgtQkM;nl~$d0!t_)Sg~??g`=>f;d_d7h*@<LBHMeD2NDZ!0(3O|kyC2L!a*
zxK-u`($+72zjJY$l1bIsK-U6SbKL_BXZBURvRGkm!@_3IJ<p;p;JfS#iIv4S47P3T
zhMllA##^s(PwQhA1JMor<D>D%NrUyU#Z-r>NR82uE7(i3Ra;Q`nOjV#_t~h&z*WQ&
z59^e@NnItn&+6}{$c5;X!g@pZInT>w!3|>a;dg8uOZDEX)Gf#+y;<*(N=n6U^wndt
z$n&A_hlme4x3#XW$K&-zDRE?FR@%$<e-@54PWjfmRT>|wy`s4`uu||sH8O^N{?UWq
z74N|fMvqkgGuB#IMutrfeb(`5U$M^dX!Gjn*?<S$!|QA0g102!MtO5r)5aCl(9!Pu
zSh2?H@Lb=yee3x|Tv21-22Hv5rm+1I1QO0_Js4Kcox(d0Rcj=xhSguhic}A;tw9PJ
z5NYpWA9u>Cno#Ap6Y-b4KV4o#_fb2XgWyxNW-Z%YBMs{&T-{GnkbBOzv)(XH*y}_@
zI%O|-uuMzWZ8B7f^}NxfWUK_Vm&O74PF{~PMEwxlxYg?zBH}66idzF;<nS4Rj1Q<m
z3X^QHn$wF+Qxw(39$ox~eNIA3s9yDE%p>L;E3U_JLMM>1=|xYE>fdVe+CGJcn|{+q
zF5Oys_D`!lO|riBjSfl8O~U?p7XUkbV@`|^4Eb#+e_LD{YJ52yVOfOXd8c6Y908Ib
zVXPRiWos>KE&IgC+OEOht-A@^>o}=<Pw-UB&W`2Uf8%TLE<@V9nc#j*CB9SK#SpGC
z*`V9{l=|_+<kbAk5NxHTzSk>#rAhYj(QyQ<5&8WB#CE<+&d#6o`g(9g&}isGoqqZ5
zwV|D2-s6!}UH9_j=#3)kmQR~rh=FS7q2WbO-6rB~bRyh%4ai@jIYk53qnhf1OM_bG
z6T9u8bKJ9^vAG=P7RXR?<yzd^HiBz7*7SzwYPUx@$Yk7kz6zHzhU^4kq;1~nHg-Ab
zOfM5jUF;i?ZBy`g+8OWgf`@gjc$w*TOO=rat-lht><AcorNl^()9rj+{@SzNfb-`T
zy`b<m{cWyuw?Ny(_Ga#HQP+9@)?#+Znk4Z#sxuH#9H}^sY~B&!`>y%+llRr>Cbij#
zWguB)I0LiFN!9JnQzY@L!dk3+R`*(vqj<LIE=tRT!X$j(D~pneugIY|K~rn6{ylaE
z8{~vmu)*sioA9d4@nXX9^``<2G7jskl*9>f)Or><JKway$Y5jrcd7*Otp0}WA%dsE
z@HX$8c0n^W7zlLlpingSQs1SrbI|U!UP_0+>2Tp&ZNZAY`6W+Fzw3NP{I0=$-V=;D
zmqClf5iw84?v4R#WswidYPS1AYc(Ppk?JUow^Y|r{m0ZT%C>Fthc4UO(z(x6Mt5sh
zF3wwtO;fLWm@?bIigwwu<-(7wraNkOI9|Coj3I6C2t{Ya8r88f?9y9N8*<2T-~BV&
zPrB(mZ&zVTUASY97;HyM%P5qsmHab+4K)02V|FFUh_YXf$Xg0)cf&7U{z5$04I9}a
zW|wQYi(X^Lyt=lw&RuR#2d4xP36Q?f9oMgJ=R?WYE6QG;tvW^GX2#2@D0?pBAxnG6
zRcm&rk@fZ9fX_p4vSumXK&)Z?d*7^r`X5~}>|ANQ{d0%10be?+w_M=gnn~!oK3wE;
zFbl+A{4(|2f3uPtI$TICc^W-(v68}hV^Cgw0m?^A1TFv>z4C1lG4{Sz<W)$%V)DLe
zYb85zbSH`e*1N`9cuMX}&B5uy<l4HHU#g{RO1p}HUVe67yf84h^7jhgHR%EiceR{1
z1YaTAY>~nPVNX<7BVt#tA6<Y$ytutjJdW!Y7r#6gj_(Jz$n=Tj(sjaeMQj(r7ZmYP
zeT-qtx(ZRurd-{lo*!)zwBS;_*Xe3+sho6;1RtfxK>CEdh6l}3@tX|PgNJUj-H6yN
zx*CoL23P~2{PHKMt@6vWRsA%4_A;Ug5wN^h-N)^L9&`y}A|@KV$0O$O6f<r+X3LxW
z8Sl~Di<6vsB@nJP*3xCo=-UnT`j)9m++vXfQ7%ZIL#kzX1_u}Riv!VnJx)~qxlg_u
zM@HUcqLvgFUS_-Qr+N)h-nQP#m^1jn(-Mp#t&*R|owXeX;ow$rCn!(TpX0{fPCQXh
zUFxO|f%N$}ss{R{^-pX$-WmuzY}~S4Frgj4K1wosxLe1UoUd~o|GYtNb1i0Rzy1e8
z);v7T?0G#dIv4$GnskfJdUtJQW!<$L{>^mh47L62;*;HTga5U-JFOMviqOm%Jk>gc
z?ggI?d^2xVI-NZ2Xi5>?>ob2?k>l|Q+<oc<-0*X{fJEmLwVZ-AC85}~F?D4(#}l+Z
z6%p|`;t@WEaWdEHs=}%0PHK`7dD3ttQc@G0G@*(rbTO@s7v1n3twrlV-&ccb&BCG{
z0bI#FOpfJc1#^)VCs!EYx1H6PrO)17#uDk`)mUk;qx=d)wPnSrd1Pc7U&mLG$!Aqi
z5-Kmc=BR(Rt=zrI-&{knXd(kX2@eRzNH?xlMcSKa!@SRyQb5@4UPWmSbaezzh7wo|
zQu@*Fvk)1VvBXOlmE6>lU6%}ta03P059?!JEm?2<j1ZP*&f`7RY1vzD<UiG?(*>M@
zlQ_3r>Y<-6E?ET=IMkwL56co#Es~LI7-lB(7z>YZNClpqs|ynUHte&xU?yI5C;bXP
zTd>C{h#jXXF(BQwsVlqK-^^IBGG-plWfEE)(;Ik%&E|=Iyxxc0*MU}S_i0?$gh;Vs
zoV>E?_R}J=plSUBcgv45FeOuL+N4i=3?5FyvSaSnu5ukeE~0!bryDfCC))jv-5KT#
z<SvV)&#`OVKaHpkE>st!v8p|@oGj7m3u@DHvuWA%R;Xt+(`qQuI>fiDt=79PrleK7
zc@32iu187lYZmt!dGcZfg(wOC-cQs8$ZoF~VRqK2vxQ;kjd8p&01^xcf4kcpDp!Yt
zU$`O-2wQV~hb+nKkNRm2t9a^8t*agA!&ms2!X<6TuE!Zt<cBTIwRF5FtC!5?byme`
zkCP3Sa~84PZ`SWw?S*RZmHP*m^dnk7eRW&y^e>st@YpjnCm0<Hc)bU!QI1BKKWZGi
zEj_p>$3L6jJ@@B?XtJ)>Th*y_iJBV7Uw<_94J9OAI?qk{D9!G<)F(ws3#lJI?9xM+
zoY~HHk#eUuAheF_3|eZAimj8JyB6(|XACe!);mnc!)V?4`O7FxqJ_yG(=pcS%F34b
z2Q1u3<RzflM~=X<zd~+r=Gm3tvQyUBG4L#xb5e}`Hs?j~Z3YO2FARCi2ppk@*|l|~
zWg!iq0!u~(%@^D<3c?5l=i3PmB?hHLlxE)9d6VX8QduYx>*4-8piD7ZOsw%=<f#X0
z_mqTEUs%Q3zUN^Wk=FJOy{t2ta0%zg7q)1l(Tn!-*Gf);uS$x+wh?VM;8XpXn@lox
z!Qy!p<gfU2j(Ps@HIIZvEpW-nR3*HpCPblRs%Eu|JzoWFKkX5JxkMDOue}XaDwrSV
znmoOgG`Bsn%DN_a9$U@YuzAFIn(ftJ*a|P$piQP_gQ|HUuZB~OebUCgY`tFd@d-05
z?z8vqngeo4^Vb!#Alk^1&^}Wtcb$l*sVfS{tb=8z^(A!tZtX!Hjff<Z*wBtS%{)3{
zdpR;BoxNwYjn)7KA*PiiASRRRuHlPk@?t%lly+upL9$!H9M`_iGQO^x6htn@yl>(!
z0%gh#FULHewaWT&LY>^U-SQLhL4~S!#@XJjI2w81ZSk)PVhs&3Si*T^;T7+J9aSmT
z^m0<NT$5}2r3<W3M8cVG^3Vg?VG#29=JHsX&uF5VL)fxDuuoI1ajZp#vEb5Ts7NYw
zY#Og#&~NoivP_$3TS&s+%|FqAthznNQbo5GkXECj%CnqQvB2xeY`*?@f{~QP<i`O-
zk=}&e`v!``ek4*MQaiuA<19F35g93%=KN^BtL$sS=hps)c{TVK6H}@scO%9gPExwu
z38BrAvc=--GNiyvs-1S6yL^RK$RlA0xZcSeNjooqPElsW1~WZ~<ap6!tAh-25;~t)
z%GX4Czy2ct5SfJozw0@V*=C2*2y+4u;igFGb1jn2$0BZipeC%OnYT`VRl*(?q&&Se
zO0LS6_syK#nR^j!@AoZse3PouZ@#YJ>{0TK&Hd@Ox}R=)UTY`QLLU{4S;4U#n0NBt
zj^k*w%m{rlGUVQpOPV<D^9FnMG{F#mUZ*XK;~a8zEhIP9$GWRETe9XnbG^3rY`#}e
z7y;+?W9Raqbhh$8*7arRgn27%Yc4lH23qY>xe~W#<Gt-~B{sO^D(SkmaVXN-_Q#wD
z_l>ck?q{uLAb*4Vd`2_#WzXX{B+)f;;9@{w<<(Mk5$h-&gKf(?3Dbh@Qpu7MyNVvh
ztyDA`yiS0x^tCm}QJth8WxC>4*Oq*}1bxon-3Xd55I_oJ*n2Rn$~l;pmx)VYm5rzH
zIK2<VP!j2fFOcl_o!o9Cu2oCkXn*7&9juZC^i6Bl?h@;2JVH)91(JncQ|MU`d+B^m
zTnRLHjK9qDW|Ax?zPhWx!hNB<kZQ?xIe4e8g^fF~ew#|zVA;dgSf=NX!^X1Sd?GP*
z3gwy5OseoDLqX}ZVp|Q?=n~_eUF(mhOV{BRdcBz(KO~)LF}{QB6`F2^`Ld94jhX#X
zW<#Zx`M!(yyiPl)Q5|lxFS3Y=<+fw&da-*S(vx1$&1d(L)43=>F^KjI*eT_7i|F`^
z7a!kgj)I#?KvJ`9jFklZYcir_Z(7=OIjAfee+&~E)#v%cZT@zsKwM%qYGJ$gIzkIP
z$*}%%wMATF-BMee8fyHzUn5Lg_{;!it2q7EZN-O!v}ApgLqKOMbBZkHPL-RK+@E^x
zNgw7hYHk9kXN3n6#X^lkjwT97J04e2dQ!}$4;|2Z*7h|W@cfLpYt1~H&oU?V$NZ%J
zYsD(;1EX})?9~<I_Q{l<3jS{IxRuLc@;c$0eF3{6fnvOzEc0%Kg<EO)h<3dOBr|h1
z-8c{2LOTfrk{v=q;i|-cT3LdBM-~R-byvUec{LnBHvKHxXWCqYSn?MlQbo9CUI99E
zGPDRf_7}tj)rANuQSwi2I{>#;CC!VMP*y{!mIO3iTaI@Yg%-Wi@qn{ptoRL<O7zN4
zlC75nE-AnW)yLQMzl2x?z*MJyJmQ)IEr3cYc&+uhu;pQlvhD!?)tXnDYWDU0m_fL7
z&0)d9=0v-Bga0ch^*j&udNscZw{5KSRJRtrT%9<NVhyv_Sc<><%Y5!uK58{hEM4V+
zyL{r*-F+uQa$)!X2$mybu~Q%Om#rkMG-nymdqfd|H1is;f;=jkdf-DJ3V)J85W8_J
zPkJO?C8z<%qLhc4+ZDDzsp|KP-lEcQAeox%%zb$W@$$3}dkD7oF!(B=&zBrHo-`SP
zr2Sv!U<IvShH_%-B1K2nc1w(%obwjxMHEE}{GvX~vhuTgt&HOY<E<acAw0?HY83V6
zYvXchVmM3ddLRF3D&H0_OBE#aLhi>MDadhWq$hGr>MtGBA-Tr69P9C;-B%CN;aBI4
zxGZ>{PQH0%!E#Bl5coTy+Y4WP*57PiSL!(igpM&Ft)9(nE#EvS>)5;8dBa^Y(u*;$
z+1k-ncbQ(k=jbG!)ACCx(^NWR{lKyWFqF|`;!v>Qs#Z6S^?C9L#klo1{=(~DkfY6O
zb6b2u&Udn-g>=Y|oL1|C^na_Cits=@!s*#_e7<n%y=TW<h8H<szqU46TMHDJXAh=Z
z1Jdk~I+UOZFHL`)zLrNu$bA~$%p_t}zci8+?$J$3kj>ZTJr@U>e($kaB?}aK&&Jng
z(#RUpjHT1Z=4;?uqi?Ua7WtB+7KvH*+0$;ab*AAO>V4gh9vTInC^H8V-`HMl-`I_X
zWvlwVb3tacMSxSX4D{ymh!hRGthA^3`GlSeueWA^2g>b`??mtk)iip4mMnfW=u)oG
zFmuIriaNZ>lMC?Z{PexcTP10evN|M$vGV%n_2Su)!fDU@*@SJ8TH*Y~N3}sOR15jT
zz2i^D3OQ<uW5dKHjMG4#aEG5y>^Q_Vk!%x(z#^Xi=Ps`pX&Q3UxP74sK(;e}Une;7
z+S5wdp?RM2m1GlMfo^UNQN<)Z9^vHZvc4^_NM|h{dgagXXLHUGpHR1fmA*wy&^mGi
zKCxx@y}b<NM8&K0c9XFBR@Qv#gV*cNd$qUk*^K71o(`r<*H$=$Jty0_78!^7fD}_}
zyLH*x^g|FHo|`z+hFpc!Za>*bq6(t3z|9)ll%r5EEUz<Y$W-HGrQbb$3)I65Dmif+
zn~kEO`MZqbejnJ2nilr5q`$lb`jfxF;7eK5DQK?qIt`bhu`8<0+kQRH{^Lc0I>CyI
zzei2XDjD6H^2Mm7wq|3Mk={Zvj@7;ai78=oaY$XT>%23icA3llHo?S#+qGZsS-DK_
z$EUoswT&S<w}GmOBxhp<!Ji_E*CchOdo$XiVNoj;)Sf3pcBZBpS%~57;|*1>)Zw-}
zujrTZVuno_XIu&Cn(QGUXbIWgHDPJXewJ2Xe^6X$>r(!E$dK33H49jec;U2IH1%>~
zs*MWu%9CSkXGSp@o+MDvx>X~I!M9~?+V0B;=26_YQn(*q%-mwRn2jfF2ToAb!>#R?
zDp$VRnJ+HahGn#lv%p>QiEsrt;H5bRdl+SZrtJtbfarHETy=3`H|Mf%It+Qr`XTP9
zkGM(8g{gNie!(`6@gi>8Dp(=rc}9I5)<c6$14m16)RjYDZLrqSHdsUrNVaX(zGzaI
zL?(V|*VzbXPI$Zi>-F%pissZzHxY$yCF#w@rac!(44b&ry?#SF#gKUM!@97Qb!4JY
zc3=`@lBv2#vxfBn*nJHC9fT@feFS@p)FXZ>Aqx3X@-qTfl%-x;v(G<9u~Zk>zj`}9
z^l(2@rRif|huJaZbGtaUZ{iZr;u;nM=zPsDnVY(CZR#Q_z-98=fO{8+7IXu2GfWPI
zOn288`tqLc5Am1PC@bI4(`R5kL<s5ws!w%Ei&c!LURBiL5=~Z`p7F*&g+?jf7^i}^
zIaBZZS?!|CSC1y9XqNfcjl**&m6H3nkGG-rP9<C1@0HvyHd*wd0f(u~r)u*GhHl0~
zskaQ>Ya_E*+B!6!r>_SAo}=?}x#}gIiXLfPzYuqDH*4Mr^0MB*hK7$)voj>fumC|q
zHx_gtNDQcNcVhgD2VcD21GUbXp00K&s8VBsW|<gw(j;<3usR~f-L8K>jeJjG`ka#3
zNy>fde4dqNgsnkIkzBwa?u1}W`dy6v+y2$}43DY%3O<k=MMg4{)IA@tS}u)#u4vgF
zck*p+Uy;TBUv2^HCb)A8d6Dq-=q^-2JT47l?!9>GH4EsrstBz`+Euiz4+5NPdt1N$
zXb4|}!RWNf%@yn61y7u}I&=Se%v)hVNi|~5$iFW4p6Js5umxiiHnR1WucY(#bUHjP
z52MT~bKPbp_xsHnvAC(lxNWHF_pML;jZLFqA<G}LEIdMty`RXNlHtZbD=ignG%ZP$
z5EQz8pI75rF*ClXn<(+q%;FDZqh%3ESSDpGX$y#Fw5#8gb!TfsG*d0*b3p?pa>2Kb
z{acU?Hu6yQfw;9+O9S#fFFp?gE@+ITUGSLtgmi<JKbxg?Odg{!<J=kB^k<wJxH{W#
zTbnd*#+(kF5@#LL%$J2V?LC%T`Uo$5OV_G!<t<3F+tJXjf#<Wb;{Mnw4jCqy#;-M2
z!~x2jiG9QycCVE|>ldrDo`AYF@IG97(vFnQF8R%GeGf}-q=K^dI#L(=c}%-zdmd0g
zHoKWGIlspWI*GOSdk35HNYdUAvC*pGgrF8V`_BX64O5!NGqn-PgLNqFt_;Z5bGFvb
zOWvi<Exn8~l}X3VbSrqv1y23(N<cxIV@6-(W4~{PZ)-6AWTlk9$<W0}hZGb5GTaiS
zbQ1tV+qEVYSwW)Q@5;*lZ)DCZUl`=)C8TYG**IGn{J!zs#D5ikIn5v51=ryn3S4o#
zwgCUSbR0y=zQ7~YeUR->#4^uV_G8H`LozLOYJth<y@~hj)?LYEA$VaphY2`rk0})y
zJ+ro**W+1EqGOr&om4UKAhV()L98H_QHPhf0EAy<I?9Sa1R$J+PdJ9Os4N$f9jI@@
zphmLJH7Hy@Zu{<XNFUE_y!Udfupn$}$YeivN`g2ed+D)>!v~;#kx_=}CgeC#v(@)F
zu~IGG&vKNclFg+(kPT7cTXIz9LyU*7yCBs?s59~u|E>qy-Yv%Q2EVEpIV+(Q+@Be_
zI=SXE>j@=JeLM<i<~?|Ah_G0}Z!V&f+msa3G>%<EAnX9mH&)sKAoJXU9l&-xVwXHg
z5LTbh3^`Ky%{-JTZWnLrX~i#0@KkADmR(K<XPd)YCOeyZfb3SwmRhE{IJ2eo_NYa2
zrDosez}DJfcC`@1rsAk*!<mM}IRBJ}N}+AS8;Z^QRayObW`-F7wt>X*Uq`2~rXGvV
z+12IhZ720gMIZG{U()h^xKsx(t}RHP?>_66U<)2ImOnqf)F9ZAE-fkEglDAdMrE#6
zoq$_x-usK|cT+b&zBZbD6iSeu8_Y7G6j?vILV1`R^B;VtG`M;|x@}S#l46oN!A6tE
zYIv0uz59j|`SfLDGZ}J%SU`i>)HrfPk&czB<y4MuTlL4O#aJ+2V}JA$=9IgA0Ob6J
zno!foMAuz9uKZhOw*ED2Q3vs~lD>@X23T39Gmd^sFE2FFeTpVG2qCG34y?$?#si>6
zc}??VcppLXcQgce500RNpeG$J+-6--`@_80R9?jwz$)M%M!k_SycN<M5?F~6?@mn5
zxk(e-?4Vipt0?)(NmUBH0GZu~CVP)c5j(dU)k+8h>`dCNnXh&k&H)~|(SG3f4Hop{
zEPp=n*v@;2m}r-anAAuKSiIbLt6R!ZdMd&O`fi9>D=&^^(dZRh@p1pbD<CR7{z=Un
z*$k5HdGi{H&S`d;J45PZK!Q&PA6{Rrj$QvEa_$hZLZhPR_m1s&gy{+C$S}VPE7$FH
z{o<#_15KfagK5@sz`FIz-xqTPV)4@f&)p;m4lY9_$mqC@#y&qA9F5JL*0GK$8^Adw
z=jSg)I^+SzXTf_b-A_QO1)pav1HXbKUR7VonZbe8a2<6v*F!P#vf1C;$JvL^iTJAe
z?W`M6T^ER8;@07}fga24f{P2vAs<b00kBJ$&e$~%Vcig9m;yV@fNc+;f5W`(NQX_u
z#Ad4%{Io!U6f|r-WDZni*DGH?^lOIKVP%LO6K$H87l#A1eE=ye(e+nP6aR(S`OwYc
z87_?aRb0Fh&UygnWG&UOO@GT+n`YZn32m|+P#cqPY)RDYxmfesTOX&%(hS?L|1hW1
z!*%~ebJm|z=F4pWPvh3n{_x2*_5n~uj22K@aTnS)jW02bm*bUDZ2cIN$OujP%9=MV
z4jIa5MVXnujof`kPyLi$*#4xR;l6HMMFVrh2l@sGR1<qjQRYM^dIhMq^V9<j)>lg2
zA)a2hF36p-V)hTMUqmd`*TJrp80{@WJ+7Ur?fDX11Qtx&ww95sm;*;h66X!op*Pxb
zDNok>fw<g1QgAP@$cVb8J$W5Vx`_Ai&BFbtTVOAm8ErT1V9|jlAahx5xv!nU8&-sv
zmLwPGh^76MgnP<<#G`0CO(MT87PM17<`@<d;zD2r`PL0=?hOg=JE{EK+fj=NtLN>{
z?>#4-4&{vr&<veiMO5zkN+-c-%gcWBF#<1~D79wen0_SOPc&+FU>b*)v9$jINX2m!
z*fDa(NR91WTTzP<J>CEtf2j}G-Dm4C=KgMwyA^8W9Cf&E64?U^r@ffFlRC(U7(jtW
z5v+)!5mAVT7Z4jqm8KI>d!^fKSn(Uma+<Hjf(#S(kw88%%emaNLi-PF)0q8~`Kl`@
z&Er3~lO6~1C~aNS#&|?*@!SYvJyiZW><K`3)9)p+hY`JEm?;+kL%`HZlTwcTk4nCX
z@{w=hQ?F|`G=T!Juh1=if!$Ot`0?m&u@3X}tVqOK^YhHU+6E!no?S6lSzN~wy(>oL
z=j`*yE=Du`-4paC?T*Rmx=M3c1-vemRs3&OL0}B*S*lhwaf2Q7Pg`3kk-Z%X-F0&5
zQyl+Eo05z)Ot*f9{5ZjU>o{M_&~g=nDlIk!@@*GU9X0X_Xcnm>EG5oPJ579LYIKw2
z{>v#i=6&}^QFaug{R_gDxz7BVAElh62owQ>ZryNVGJ}5AP3^bqJT-e4;sV13*h}CX
z&-u32IPBF9t!-xZ4pydI+$9WM)Tf(A<{VPS;^00Z)%$0AILVc_q`CHux<SoK3HVqs
z^^<seJdq2(-BD{zoao}~U{IvXLTkE*V+_0NK4)+WkRq?fM=`^$r^31?j)z>J?zsw~
zP+cCfwj7q6@UL}!+Lx`2F1(ERBq>Gbz+%%?nD>uznW0K_P+R4^tTRU{P*MV^f(OGk
zunxv_L5ZDW-v({`xI>Hkxa<5j8e%)RH(%~_1m7@kEm?EQQ^5)<N=P-fYqcpXvNyfr
ziI`IAvJP>s&jGzbw4DaeZXU#>?TzJnctMHWI$ozF<MYB=)dZb@A$|~zmW@{2E+W7R
zvfm$i#{4z57>Ca2f!{lgN7U1sFP%T8ly47~9o=OFsle+v)NME4s&-U<*0J<`$L-H8
zbMCZG_~#dBFir6NAXoRCBV)?Nw>Bfq3}7*-#EPq1K0*vAl^X{ng<C8rXa}8*AZt{c
z?kcR7L|h8c<zjgfy5gLU#yLWOEaOQoa20><a^YA;9teJ~EjisQK~zlSCgTytT-H^m
zV=|=N?S*SBoEkcqoSy&9T?t*}opEP0w<&i~_xRcET{<KvmA`8xu(4U$S8xuV<5yy#
z{)?}kf8$H)x|{i9so{qfCrQZiTDOvxI_U$GPm_;=cGUF!zG>&~8akmOMV3#$YPRk$
z45P{<qDE;(2<Hfw{FOv{a<%+fda4aeb9z}5j~gW=kkiF3%%q%UBMG>XQMQjA4!Mu^
ztF|lW%XvAg?$-Jk#%tVIkuAKp7wAIe;w#%8b|5T{xG3hcKm^7GW9oVKsWHiHR<`vs
zL11r1!sN7^Y<od%M|bzTAhUDe9+p(q7dynG8MiAPV8sZpX|Sj=#hb1@@UQKgSkwgn
z`lwO{Tzf%Nc2wtu^O`i<u=g5o?ak%cv2!Yl$KXR8`40uYI0>W%R8VSOy!NYvphWg|
zO#Nn($nER`WicDU=L!6zm0+e;8O5R|2&MZKl^vfvFWlI(5fB@zqgGpAZ)YsgIL^n>
z4r&sgITZ|3aQ5zj-$74G;mCvxFN+->o76dB2gi(<w33{>J=mnHGMh}NzKU6=;kMfX
z+s0r*da+~R@4U?#E~Ijb%~X}|UiV;XLgn6;cxP$Dp8?v1tYZe4eccjsYbesUgr3X3
zPQ018S}rbGtsLti^)n^$MYaa$XH<<>BdT?`@#n9Lg%>Yw%;8PR&gSvG1TQ7iLcify
zqz-3Jecr}6>QyeodovfG#@y?_#A3}Jn<0=95;`0xE^+92GFC9LADr}_qf{by4y$$u
z6#le@O&wJeDajthvmg28#+P(sB^=i{THk`0PGho76*SpvDY|62mwvFv7xx@^2}I2Q
z;?Kyi&^{0qi-7J-mVqXx1H$1?)KFRShGlZw?#vFkjoDB&r-)Z8CH?%aPgrT&as7ay
zH296}5LC!L7(r0TgKpwh7Ud}Jdzb0Hz$a`ugN-x}^skwU1#GfG%i$*!Ngx>+Gq7D<
z)$t0)pL~9^Yg<sT;@Tx{Q<6so**<9eQOj{&D96i^rX>CAUR$?(DF${C&xEKD?L3vH
zrsAF6!-Ky&AAs{(pJdHTdX89AYIlrqq}ks#ljF2B%WavcZRCR;0I1)iu{v#ASnjFy
zsf+HV46pN8bW#;{9$r(0PXVm%en{!Q0Jr0*)A|Cg6~yMP6S<@($jgl)XSo;<Tx1!m
zofDy!b-d5Ey)d@81MH65D<1KOi)q{zNe*#8*5_MFQ~$H3`nXj)OUBk;-NA}}pLf<O
z!jC1jf~ByQb1;Q#o{69^`YVQO>-qb7NhxG(yBN-cJV4!mT*Aq7G~4cWzv6~Z*lp2O
zx8N_l>e`{bZ=hzB0lXMvPAZD|uQy3xz<<($UD$a`bV?w@tOy=_ExrE5%L{Dyn^h>)
zYI+C8Jcz6N)bQN%+zHXL^U@hL)1KeCjWHn7?dpV)J7npRvr}X)CtJx}gY%P-d5-AK
z1qQV0c4moqE{!QLFBIo)tMTYiVgaJHx7Oo?mRi7Q;YQcXisx&nxg3;ww97W{1p26^
zL`?5ZXTf@spFY*C(Z35s&el*HaHiXSFLv74MPjvdvR4FR?@E0~XX=>yIQUaJ<>*gs
zmfd(&ufengy#!&cc8|XMG3tU6CR>dMv3~k*)N5TwY=_P8?!kBGZ7W<;OX%-+PFe{y
zqK$E#=IpbUJqTxLJW%^FcdaDULE&VT5!?T&*XM4bz8u)Sq4e{{fkW;j)imZVmtpPZ
zs$C?fGx-8vDlT$E)^09Vdw$q0&ribt+B~W?NO8+vCKTWPaW0Owv@*MF`|ZapAt)j~
z$ZXH&5q!!Z(7^aiWRDJM&^0nT_2F>K0C)9`%}3L7DZb%Bjr-s|-)-%b5EL7?!=nb9
z{8``{5Pe}2G@UoM#^ec~0m<#g3RRDM2!&p<*_vMQubL&k23=`)iGJ>HIi8aroF!f~
zA<OGMC(cN<1%wSIE=~4Dk0AO45YotPl4sC;3V*^q#U_TcY3p4%R?u#sD<4v+wI?o%
zC{M9&Vf6pB_m*){wOzb0AR&#Upo9ui5-K5$A|Tx$jUrvr(jkb7gmg$r$ABP13^5pV
z=SYjh2tyArz!2{>-p@JDdGGT)=hOS;yuX_d{Pu=D*Iaw;b**cy|61$6gAi)rzoffz
zT1Ny0G)0PI(|)yL&p(#y34QLFDlxscyffV}`qBh7vv(V0IwNZo-Sz2Bl^w!j;I~IA
zRZ~w4d+P5$z!=E8Y$)kSbDeIuxH1u!#QKZL2BR772CZ}CGtn|T+I%wTQS1Qm#x=jH
zLzx6)kQ;2Jach?y^1(pG=E`V8jCQV`mqYGfpOzT#C10675J<|K+MIVE^X6sWW+}gk
z*@}OQ61E+%vsH8o>`N*rkZv))VOKJLUU1j^9b5ke(XAJVW4^iXFoSn6*Te-CS2j^<
zDHw8l?1Q`qlGTZj-N~kb!rGwmcb|UL6dOM%dp2cU60#te>?(0a=6x)X-^;)VG(DBM
zyPl1Js6TH1GJ4|J$E*z7>Yw)UMO$NTd<zNQ$@m4Kgl#ZmFjYQ*CkN%W?-dzE?~EmK
z(4>N{3PIl8$m>7%nZX|ZLhH)NXC5CXP<ZLpuSd~Nz7c>Wse=CwW*>q@OSI0nGh|dY
zrax_43wpe@U{cal;mrW4{gB-2G(9_wnwni0wnCf{qvu0<Vg6Iyt7;-qso6a!ZxN<s
zLG4O>Xcrgkxn4A+w&_95*<o-#|M$r;sc^3wCWNK#Pu3IB<@=StP+#p<|Mc1@GU5Ox
z$aN>(oI1@f>rzf9GeiTr$5VRL8Be%Ro0iW@Hjz?=xr4=88y^~;ycit2b|jplBTwMn
z*d-V~yDmu1E0^Dmy)}ND*B$YR-(9xmEd^<te|d!~y)!o;B07S4C<_+0o~_?F^^#)L
z&#u&|z~|an|1Esth><~@7d&2@e`<<^TVO`Yv>+@=YW%!gT+M}{hN^Lx_-LRsV2D?w
zTpNAbjk?e6f9`zPb&1zD*3HO4INBj;mEq&c{tibImCdJM1NZJ)pZdi)`sBOlX4APA
zz9>`RN$}e!dRZ2lWPhz7Ve1q8kPTp8HjfR0c-rfImS5-iT;t(weOT4~Syz>j)$_|C
z)$|neojp^K2s93Og2zSqmgQFRcSlg*+%%rI8G5SaezUg6;<TKzFq-%o@3WK7g^Rk`
zxSF&eNK*4h3vPNm?ZNOQk304y&QJv<J|f1LBD3uC-$9O8b7`BQL=*!T#5MluniN)?
zizq+gX%9uikqV&rDu$zUe4yWYC+gt|Cl3TI-E7d+-ZA;8S}B`00j&&C@tLod2|H*Q
zx(UVsmjqT+NqRr=tZ2k2RXHc9n||5czmZ`kx-x{=<`JtT+QwJVI!yGc`c!T@<m%v<
zI;co!tlF60@XMm|zReYA!OOs89_Wq{Uy2j0?m51MkaJBjb7V#*Cl|k!+lc`$Idh0o
zI8`c77q<+aI(d4obY<Eus5LoYEpu`3vUBX_i&(fu<e(e;cxKLy#3}GGPZb0DiNmzp
z9i>MdarI3%4b4&W$$PNJUutX~?jS0xY;w*@HTg^(au^}~#!O}#z;5i-`?^N0tk_5e
z8zfJdgxnp)j5yftdy9J*1Lv40`{m7*jNIsU$%)_9OLuxDmmz0jCx^Lp*-tw;nK3~<
z?HqYb49xCaPD%O$2`S}E8B4hqgav_vz{1$H93S4Y>q~zBg}XZe4xybocUfx)A5c@-
z&=b33fmrzln7|G%Up^L8+P_k@+99%^=+oxFZPuy>{AE5lP#QrkSs7s8%CVS6|I%M+
zZ4z$YKRjmGeR$@>m6eT9@9rcVk%)=r7kS(s=yD!x(;&M#Q#}yBQH55zdmf;8Wv?}C
z`FKYPt&2|uZQ4`6g5n&gycd!)6_`?UwThBtmr*Xv8CT_{EpTfDo0H(vQiP?3H|UY0
zPu7`T-)rXZQnMUU4<^w^ct6ZM&3a;l<g(+m=5#t~7A))P@uH5s|F7PY35Kw8;twm*
zuMT(j&&rop)ZR-ZE*0%@sJ?Y8xmTg;u>tV~^&_T&i*wb-7TD*iua3<-hG(fp8oOHx
z=bqqD5%X>hqKcGO<t;d$f_rdn!=0e0s`)o-Wc}MYb0nHg&m9XDfo^onD1V?ZnpEXe
zTMl$Jx4>GvL}`iyu#L}UAwq;?MMjw+T&UzRi}qrD)<T7y)m$GLzSZ*+30bwjiX=vG
zMH0>o#%GF_SqAf!94$X835|{yP(mAwpOiM6&QSi7tgZ%6)<RFOr)rw8>go0DeH60U
za7m?jZrc7VLVF~-ih>jr50tHj{P=1BLhQ^yV-LD~`HvRHTXAYT{P+zRcUJz#o~K97
zayIKjZ}WU1m%dVK>X!686^x8t+Z~EYl`2#k^M}-V|M2%K{xDFhaNY4ReY+2#xyla7
zCw|$D^b*#Z%wi9=iciMM`G`7Om%m8;l}iJvP8JsHEkt!W;0H?=c-vcJr8>mezR9Lk
z({B!sA#?lRJxzS3YC!Zbb|uTM7dB9%KPl_*eD2HUkUhw=>D3%Ap6QuAYljb>>B%W3
z5_iiuvqOJNb1bgmQGW2JV)$;-QD~oVY4N9CKxR<v#^Wy+Y5Vh$kNK0gawI1p!rRED
zJGT5N+NBa#g}cX^efvk4<!5MKUqJ-b>QhTGx2jxe{ZNzr(rn1yJzhTLS64K~vKzN1
z##BvN5wR(Y*jO>sb)V2+!_0!Agxg9r3W>LJo?T>lK1!$gW-ac`Fx}lESlaE3%d7QT
z2@F%B1#p>ty*ggd%~$6kt%uDgCe2FK8fHeAId1}3TyGugB+{MYo_91{oUUM+WAJ1j
zjq9;_mF1n}H_z876T(bSB~u+OQDhx*cWs@n-`aQnF(|ND?)oI$j2d2n7_At-@DZEz
zVe(Yn8dhpz9vO`?%#+wT`TWjD=AoVf|01>9=(nrS&IRxCO4i=NMsnMxpPS@QQ~Mxe
zMyL%Xb~~cAGPEkD*`Nw8X}ev;u6^C(U=6GBdio6}&TJB%8jd)Q=EJhwGER|~@6PZ!
z&jp{6B54&WZwDqcUjc<RAX!cIhIujS9vAFPb}33!X4TDaJsr{~CDf;~A9uz!HJKV^
zR52{S7y!`qvM9q2C*|=EQk81F`CewPNMlqW{$^HUE_L-(29GAY4kr;YclC&~Hq&C$
zKM%<>wyfI7Xg=TnF|c_uJW#%tkmeW-^BFo?vdJ-<T$T!Y7E|XD{ZZuUL}qESn`m-k
z|5fFt{*z6W^N%I)OWi96K6aV(?-EW;pwiDx#+x$s#6@hZ?9fYDh^DO1t(8;I7xtaO
z12WNaMOrfFxJnqokrzebW~sw=K}~9_FRr*f%j&O>wn#v1;fhGf-exiog-!MF#5h3H
zqt;F<n`R3u+vRp8D$UncRXiOa6cg7Oc}tdBH9U9JRmcP&ulJ*)l9acI--h_^=91ci
zVnjRGJ8PPfhoJh3l$3;kuEACtnS_j9dpq&bXtg4#wT`@EU@Az%uD0P<JG-u6!SXeQ
zJGCHTv6=UyYy|Ug|ME!v?^A!hD|F63R05i^Xbsvul6D@D+nk20wxCZj$#(rDRV_bP
zYyda&({KCEW^eX5;|T9K+Rv>f?m8j27wo9K*f>wz&r2g4#ABqVn|FV=Fs!`U%o^F;
zzE^{yL6Qw2k)H$q&|Y2tAI*gNouyZKdF1>sqUh;CxkmZ+bb9siab<paF8acmq^BY>
ztqR@<TTjgky-_}>88XAk0s(K@W?2=9+Sgg1jF!p1AbVHzRDB_V<Bax{GzKk~pLuR&
zR1Wt>SIkYta40IhP9;yS|Cvbr;j4CeMS<|!#4)+uZ~Wp{V-S<MhCz9DDhSoQqSpS}
zmg+LeN0$na+Jo{z<uqf0lpAR!yvxTY2(R<C0Vi*CYTbZb(0PpMB#lwE(6@dhd#iOY
zc2#+_Z8K@8*Kuzv7k1}VVn5Eu;rCYEmolfq8P6m1YL5RwEp)Hg4erwv@k|=xlD>)=
z+xn}uag~^KKA2gbDe$cN=p+#{tN9uVN*;Y4>Zb<R4!)*8q1@=PV;1_j4nN$xa08**
zd+x@q?~QpcRdjB0#B0-D=2MT@y?s^*{~VHvmJ1#{amYy)&2;|dqpEMD<c+NL{AigO
zWR@bEI^*awcmDc=xtL<_XwSjm9CR9jnEjNHr%o5tU9)!LuDoTCx#enBbDBS1>K=H8
zSeX-D9-psSf<Cjnfc2fU@85sg-Rf);sImVhz=Alrs$%-nt8&Q3(&I5ootc1$u2irl
z&_w-ujMYuKwWGq4RS8a-8(w>Y_+?Hn*>gC3xsXl?O?RHzs~`1jT8w|rbsAw3&^NNE
zLxRp{yWR*C;No(z(XT@f1->64U~gPEt+coxPG!J=5U^Z!&l$nBSGUL(Dnf5?EWTn~
zfYhDcO!m6$F<YBrzO8qKHJ*b0f{E*vk9u*y&ZDYyvsOptjbA>)IeRkh&l)Ix&6f=L
zXbKqVnA|F7Wh~g`!gY6-apQ=?{!<ULkxhJw+Fax<Q+Hv(D;_Y1nw<&TB}&gbj9iKv
zI@5y4cULw7ET7S9-C&pEyu;XWA9Q?Q#I;@D$J4!t>$D~W2rzdpUCD!Cn)qJpMta24
zaWxm`PwI1&V`=Ev$@X@yc)PH>otpd^2Rg&QIREf}T?hRZk~yHwomKfyAO2c`{9n!D
z9(NygLB^N`>O9KPO%~5%CeiVW{L&_GZEo9mSu(YmN=Yk$!q-Xr-lEH#eA>TA2;aux
z`geb!C>{UwqDz4k;d+C+fgkNqEGAY;?73hvwexg;?OO2tNY-kPD`(ppbp4=!TgrE_
zbD!CU%6Swb{bYE}>JJzq3i?vG{!_R3i$-Qnk=t^pu8qC`AGOa3zrY)IYhmWMYb#=|
z?La-TON-~Vo3edZC(oQy<5+2_E&%R3d(xzut3p$ndB1Rt#f~b#I;mMb6AB<eP^>>=
z1Btf9+{3qTrSdo?JeK(oB_8Ev1!=P89U4{1P29fMzyK+@IXM}hR(uAK<nLY3rVHb4
zpusgJU%CVu^V78?EL@36Wqjb%Z!k-jWcol3Oz&D(#y~f6Eky%!hlFkX%lcGvGF2m>
zH%$CVN6@a43zAaq2N$6@!U637AgQUli)0u3Kuu%%?M-R)Z1^Dm7wxOnGI5S>2hQGP
z)vSEO3-Z@+hKuQAO#0@V`(JgtK_fKXvshejJdfzV+NpboITmuXh!)L-&u^d?spna&
zFH}OttQastdbxf81!l>?H5>z5AN^Um<PRIu6lg_8WHTFqp5()oXsv!NALT=K__<?B
z6}-l{^ojN7AGB)6qk#{Aq5(Ppj+nYg9IpWo0ol0wU-SinfDCP@ulTBv-OGnA*}R~S
zF;CIrBIh4vSQ;);!pWQrSC4zSa)tpXGoC-_?o`abnz{8mal{gMVKC0WxFJ%tF4zDH
ziS|c$5*F`Tl>X2n=yQhNs_J@ai5u%}%%AevVBi@#{J$v<G?reJN*mW+0AvA*c!1gt
z&}HF-a}^+X3qL&UaHIo&L>OiNxd(u4*K+@1wfMh{)#49>7VfD6MZghF{QXd#8Ao{5
zk)<_?W7GKifB)xa{wLem|LGzdb-+x;>?^B*<Hq=}7yeILsQ=SNNw@I<i`UGN2<v~H
zIr#X$DYX9E?SYH5J8+C=?#NNE|9*Ra7TLe(#{R1ez(xNE>yLE)&CT|Yu>R4kf0p52
zmh2y!^_Lv}G4Ovj(tmctzg&R-n_Pz<6Itqbcmh%?5AW;$&0+fgha1s$elc*Toym5p
z)}bi)^tebdn$fnKD>Z`he|6a2#<n?Cwb)KUo$Mwo0$fI6-cF;<e&~-*7Z`Bi7hc#1
z*Y=2-P`dGF+jMi^+41$-QO&|>Z4=_K>o)*hPM74nHGsMoX=G+JdLm)3_O|BdDcGd%
z3B^e6kLmSCu?O2vScU}Mwe3r5o=MCEF2u4j-m>oOz8`plWgmZ4^V1YlR|(I>81V@e
z#Ud@xK_7g)`p%{bI$=_zSFFiA=nf)D9xVdTmA>0e>ow6%fwD;GvPz;Mjz_OcD9j@V
z5whFOJz#Z)undv-G$@UJHyMoGK$E98f!NO&lnuOR5EBfS_aO{EKlT0mm5e`aJ{a3S
zD_`EV*^1p<cTMq0zICQ0y%RC*zJscQ7iBGT<L>$ep|)WAqW){M@~T$Y(cY+5f5#O!
zXN!>2{Ri|qi!wGdt-s##-&q+gyCcKSiSsqH<s0Lebv=R(Che4VvFCt2Njl}@>B(}c
zWuNbSkR6M-t68l}?k}3qHaKCAxsu#&*A3f28H_r9y6}^5iH+J(UHB0o?;T?WzA$(v
zd_r%Me5#=N4ArZ`REsI~EXna&=gB)loWhTmh;E37$xqVpV3<;2`hr|vIP%<Afe=f#
zeL220HUQ4~&`5Tg&`>Vz0qYK~RI{i4d%vHYouIXD6e2d8wk*a5zXJi;*$!sAtc{o3
zwVoZLKDv@812>+LmdB1+fWYJ;tY2)pAyU?<qWYEJW+PHPAH3II`VWEv#2KoYOSY^m
zUzu!Z;=omJdfgJz$?;w&VTOcYStIv@rP-QEV-g`~J17oGw@JBeA{u@FIvs25{oH2K
zBy^fPXo1KCy4Xna^9u>PNWccvNJoxBYO^8#y%gQX%7h>(EpXyb2z!i`uya_z^V4_5
zt@(-Al>Lp*UDu-zzu{QHym65St1n)=0*58=_LrhLF=6t4bUa8P4@1Ex@jP2_858QT
zr<`qRVfQ~?&M{r12=ln6J#YtRc6@J2li5=BOOREqMxc;6;JQ_<cXKdK2Wp~Y%c|*l
zO`C&Y`Q%sl{p)CjcB(ghX7>Qq`StDXZ`4x0u5R$tI^*D@<^H;9S0^gjLo4*CmfSd~
z|Jd}LcBxMXPZ_~eHzkRxu%^2A6VLN-Au3am$y|hSq<+r7*UWp|Kq<#VfK&XcMy8l3
zcTQl9v1{XUS#y-(Ise>L4}X<&tlZfP-;E06&+pdu&ahjV28PD%>&@r^u(rf(-*BaL
zy~wGy?v!NGkd$zl5<wp>#x(_INqXP8;{54;q!aymKoqM_$@5#I)Q#;;sLH0kfUOq0
zvoow)=hBb@wI~Uzmv-`7u6AbFRx5U9gGI#NN>h60yL!FYXt9<!d*F7MRB@dHQ-s!3
zHKcXhVJP?h%CsBYd*}Bn$}20>#j;wGtDJi9;M2btpt`xW2hhh$7|meE!*(XiG_dw1
z{RGp`zLE>2c_Yll<;0PKG&~$Ymx&^JA<>C#)^Rc`f#hWe(DWCU;Hlew2`AcAED6FM
zly<<;15&mted(q1!$N9;jRmVes;&35q4V&gl`Hx3>O!rDACK0`tNLs+T~;i-Vr32<
zGga`qYCD-rIgD_kcYB30wPcUJAL)#>03(L9SSy1Na$my>OW@!jdiRtl*Q_VbLxn$K
zDb=t?pgV|2;uO$F^-nofxrohr>D`ckY>gN0vC`_+&3X>X42U;wju)uVc`m(I&S=5R
zHT6t)onwFr02p)N@V&(BZ2vXh{1DXytgfJ8Y7NMxZbn$dZ^F-mi|y&?j&{e@5;{K6
zY2M50Xe+Is4{p+<KR$kwN1v*SMh=QEcE*XLSNhd>$Grs0tk8%Zz{31YD_0?k%J$dy
zH_5J_4u3^wG76X!{2KU|6>ZR!Wd)B}&dh>${w^2qrl_FGoPO$n494v4vo4zA2CYHF
zrzZ+bqt|DJf1*`xTKG)bXf9OwchT$)E3+SsmTGku+vNmJ+=7njmRg_n*7ORZo;97D
z&mxz)6SLg&%cK)f*ey8xZo;#k&p<0+x2>w%8Oyy)h?4~BXGTtd+HZb%M+Y+!QQ_sP
zOlbG`R^S)nbv40R6MoY_=-8Jzgx)e7ym^i@4i9}Cf_-?aK&U%Ogu6;}^T<gwwu^Ee
zQa|Si4F406IaZIZYXfNl3MU%kvr8-WEYn`i-l&Q6x1Jpspr(j!%YIWXWt$U2nP*<E
z6BWiCs1fR($F~+ozzPFaAww_Z{1h|LV&UJHv+yk{)1t+{aQ^G*M1$x4xKUF;7f%Z?
z#*)@BisLdmQiA)WKbY)+8KJp0)DnORS!Z83ulL=m_R+Ut??f8VhCO^JNb17F()dV+
zK!r5v;jLU^Li+d5b?8Lp?<?9n#7AF~|M*bemFTQ3LFs<M0}A~ZonO(fbw~wLX(%|l
zxC+F$<S#yokZbeHnfIT!v)Pz*vfT=r@*I#v2XpfVY&BM-A5qx#_xVq7oqP_@dZvOg
z3Lqt$e%zlQ#;|ddaTM)!I?A?2GO=;|1PXJ!;(I+aWxTCygXqSs&tHoxpE>f+ly2DY
zWya3hf7=}(eEb@Zz|+ENb)*pD*G9_yfhN^>K^N6%(myf9R<H=mt~x4a2p>CWn}f9o
zH30o_LJf*$#_k&MCt(btWZ^q8ZV?+5uUt;Q(`k*7Gv4YdvW`@P#)Jviw%9&iF!H$~
z7q>S`mRjRrzk*M4btpZS!Fk-sGRPZsU?LoHb`sw)7qDdx_R9_E_)ve5L!={Q^>N#E
zS8E}j$^D80H?vO10b7KMo$Kd*u_r-S6s~8<>^4=Ztszca!{VpH7qeD>PZdsYz|YT~
z1d&VDS~Lw18iO06BmE(d*>ThMgF?F?kY5@uFC!)<1Rk0d!AC0tULg@CT7&Qjg4{a1
zU-GR%zus_9OgzZlKdji*n#svFmfJ7Y8t2aP(rp16+i}z*2l;t3Q^}vga8%COT7b`y
zU%iY;z=VEu==RhPnn@9_IJ25(Du;Go#vA&<eJ~-h9{e=DMS1CxgL}bF5HUyc&9P3l
z0Oyuo!KUwCO}hchIoWbJ&)@<^;);F1&kw*zr{q~z%Z`)88vE6Zd`{uJfcBQ1V6n9i
zu;l<C5rH&<+JdB9p9QTnc;j@GfaE2=cyJbQ1D;GPP)QWjk;yBpo~+x+kssw3BXi0P
zA&v$5h~88o_Nh4pM$2i|&{W&ne%h<!Mx3<nV5WH0U4hwN=TDm^Tb2lOUvrq>Nu^hF
zUk_$02bN<A_7HdTRt{l0f^qfrh#(H(R6^I5{fV?mQ-QKDh5D7=1tLWuUPC%D!Otn=
z?pMlPSdge*V8*VvQotk*<%ICAh;uxF<~T-=bOt>Zm^$vYJo9+r?m)~7BcL_BTQ<?(
z<0dVcov0<-I6;XZCxn`zZeI(Pmm}gaV`;85Eqm_J{+8jIn?ZIUFpsYYq!@mh_D`fC
z*Pg)nhc@s@mxnEe>Ze_sB2O%iT9LEIwNr*Kxpzb0s$`E2$@46NV`Yzjn_!fQTu*}Y
z(eZKP4+pZNZ5^*LNS1xlK=oq6NZ3let30V0mX3T;2?e48j2xkD&j==_$`2kmp7tA;
z!@>yn-enqkIFLVjjG8+XFC$<p=VBSHp2}D-9y@;pv9&Wx39g<ngc3BFXocZ1R1}O-
zha7%upeDNPp~n)hEy-l=r2{#|76gvRDd1B(xvWIj<fNMW7@N(5D%#3XF?7H@)?&sm
z>|%@BVL_-W%eG~UKU9e2SU<NdPJ;5InP<PK``Jz>yIifg>#I5Pq!egTmw+uB!9{wW
zHqX;-#m2o6HS5OcT~VTUGC5bpITlb(rbNA?c18lG4bPvrKyns6Tlp45xDB$0VX%<0
zW<`eZz1aAaPWwjhRK$$P*+jt0sw%7M>}UfesK;{8c3{PZfjbYASL~Y~si_wJ4!H!O
zv6e_1h~lpn1Ejr7KtbjXYnTHA2ghwlGp8&Qk$$EMbJJN?z-HqP{oco)L=R1TfV<Uc
z<Jj_Zx>m7ZvFAV$+EaVhfKTf5fN2z=vEDB}|E_ygQ579{?t=d&)iPM3xy?a_=wXIX
z@`S*nz|}Fz@eI6<plu?2!sBc53nP0Bf-XNk<<+^yJC3Vra4#3mpKkhEvq8Q#)bUb2
zDv2M2(Z7L48e|$`I0sSVEKB<cOLo<=g3yTlOYGs8ewIO4PT53NPPD~qB@MAwWwUu$
zCE2_IzX`@t!@}NwA(AOp_3iRS4&kCUW4kPd^UbmFd*Z8pBj+ie_@pwQ{rUM(EFbGy
zE29#0oFq|SuNPQb>rn_GD2QGJ$&p1Y-3(zf%u_wCSVtA%lM=o6)XRt}BeFz@jE6>e
zLobonCvBDg((xSeY+`y)+E(JCYa;gQ;U^uMcG#!#@!kFlijFQ%%zNrwWQB7=zvGd}
z(ePM=B9Awb{_xNVQOo8Kb=vsDUrg}JIVgr7`vF6Cvgq_F7zbr{jz<r#`x)?+Hkv4M
zj$O<bDG6Jl@{m67E&b5@4r^?afg#%w71JeYt#jlVdQ*}euoOo@D!6yWnvMci*DuAy
zxp;78qQ#QJkcR_;^u0-F`ye#*yE@P3I0>#?3RCL6o=@Y>Q{1!aCpQZQ_m-{tGeq5u
zr_~aQ^9NN3Xjduqg_s%%cvwGYl%N`>2xuLQ^d~T`{`Yp8_OVw6^UE3CW<rI4Op6J5
z%rdgS7V4bS{mz>wpB^*Gb=eJ#fH~Gw^Sm<D%QK&%OrR{0tkpD06Pq>{qhvL^qab7r
zvs`U0#3D+-<F(I}Irw3nOEY+Z*2uzdxuoE6EFOV1*;W7Fc{`0m9anl>Z5C78y*=>X
zh*%9Xj%j}_`@J|=-M`82-2C}5Zx?aZ09%yp*)akh+xaxeg&LAd%sB<Spg&i>I3(v?
zI4r6m%JIJL^3fOS;2{|Kc*MAS$(7j3)d8ukcbyd`4GyTcX84AtCWM?*&(h-L(}CIb
zl|x_+)Oz(gfa?8}r4$ZCna;mGa%OpKq<9%_SHkd>Gm(I;6#^S_(j+qsXiPF8N8Y?b
za=j=|wW6`RBlIom$DvE3!P=gz2Z%mx5l?Osn3@2|r!aE0SX}(U;eKt@aDH5NHS&ww
z<zsd7<PlrM;>_!tq>p|pe#Mkud@;SGvsD)wf$=O4udMa+&&a0qVZ|u2hnG^!U-}(|
z7pXR?*B~-*&Z<^H?K;>y%Jjj@l=_hu!%%V9qS*r042;tp6nJr*fK62N0<B|Ag616e
zX31^d5tl<C*I9xy-|_K}zWVaN!pQ|+`{f+Zxn_ze#UbDKS1OE0MhKx;^zPp$wj3mG
zW}h0rhfm5t=EuS!l<$<`DF49v`Jt?{@Y;kqPaI}n%ys4mESs@}-^dan*Hq#0=2CoD
zsW5ez2hoelt>8)wzj$(JgvnH4B1wNTyhui+V6fP+TCBUk^*|avU_d-BZQ1HPSu;@}
zW1qL1J+n7C7%WzlZ|K#il3YxGGI8b=_w<3)O~Q+h`XcUN(>w{vE*17Y?ngFDWGa7R
z$U3<4^7=>yy#u0IM>Or~(}d86$9F#d_L&}Ih}RY{mb2Y|`9t)m*UogW*~v5oY&DYv
zXBdqI-7`rI=yP5TjoFA9;P^pJ26#V}3(D;Yu_CQJfy&=gw)^0&1Ve4Nd=HH{R)7OX
zz>)G%c|b<yi`46h#GFbDv{ZS*7w`#hGHQQ$D<`%2c~BNI9S63dyEP-ah0HNgiP&On
z!d4Y?!y_kO$E=xy45Ywtt!AZg@CP7^k4Nt7FjOT32_e#Jv9niaosTl{g=WRU8g}<#
z1Wb~QGAOM4?3B_Lr(~+iO#<3P`)Wag&q8gULB2yjs)n3NIOl>ev&SI5tr)L4I!0Ah
z2{kSyY~QL7B$J67B;w2tIXh}NDM-}Z9c4aL@AVsDlO9Qy9?ErX^1kJgm|f*9&9W6k
z8kSB23_D%Vy7d5Q!G*<D;v$t(Z~m7SP)7rZA@Omm&&Jdz^_TW*zNUJxPrAD~z!tDw
z)j2kQN#RYzNl%J6IIG%dS4Ic573qXUFL@ol1&*YA=-0xs?8>iP=E2^)HGxOF!oO>f
zP?wxcy7<G<oHKg#?u<-vLFji72^p2+4PBsiVFTg}-c8Ku3xnUIx)jzZu>He2uD#iI
z_@g3(`z{~H`xO-zKY7oyxR8^@uv3S?x)sU%;PR&Y6%kGO2h+~bal<De`NY22j`G(b
z<I-!=fsL^IwG5H2dPKswn143n^l8GI(C?fk713?$WZPzP$BlD9f<&Bi{wT=tTmH(+
zzg@@Cdn<N=9C?f9Ls7Eyo62Q8SP*3%^0p5;RW{z7V;6#-?B1J)mykacnje;J>IrLp
z=Y)dPAM_tRv@#*O_(+{!=*^%(Snc@_xswIzeGpAQRjRsQ)uP6&STE`~W*n)rsGdm3
z8GG;7>xQk$9h@(Ont((*h`0s2z?lTi?hTZFq9VB1cL|>?wagzzmMRij0vG1dGN9yS
z_?qQ8AOVG8M|-=n12&n_3;m>zO8Dsp>kVFveHi%)pS-F7{MYQ)7$w%Kbr7X4nFMt;
zy|H*(>pT+r5Yh5!^@|%JO<G1MtYEuYNuYVENC*Fo7akVqtJFJm?$`aw5VGx}L=*MH
z>cHdCoXweu(V&x^&QZ><7<U3%$Khq{8rgTAFQvqsn)enW8mMEM7CLA4OYc`s-(>JF
zWr$|44=KaO);hhf+X#}yCr$lH7q$zQJ^QKImXW&{0WHho-3?;}Yh8Qc^}B=8pACKI
z{l&GTi3h8u325bxhvS~g-(T!1O^)1rPo`=A+>wYc>6OXY>FHJoMlUvJwPgk)x~A9J
z6VaxAktRNk)aKwel5hks!sOv^qWjGCA3IoH>&-H+Vvw?!#_im#^y7vgBQ4taN0v(&
zq8++FDnlPm&1ezHn+r*|I**tbtX6Nc;uH4JJz{^vYu00#Q_3h<C@`sy7BzaF)avD|
zDSv%KOAG<E;KhGa3!egbLY?6?OV8b+HA2peJ10uH6k&eomG-Z+-pZ+&N<^Hk=cikp
zB4Z>N{UW)uow(bSGFzJ7EZiJ<R%LRCkY#?$)tPbqE#|VeC9HP&dVkhfeM50^CBf)z
zJ`VxKsRL#Cj~cIzE+%n<Wo-~g#)37-j0ue(Ak7v+S(F8+d8&3sI+24t5kOZrik2OU
za>}k^f9J=ECC0(W%|BLc*~pmZuzXAUUzV{Z**XZouPw@MHKQNMh4g%9cr;Y-x-Pi<
zxur&d%uA#B&_8-u*>W(I)>_}ywUN)yk(Z&j6|@zgAn6!h{91;N;Nr<SUG8%Sf3nm%
z5S68Al|vu8PU+&4dK)cau0loZHMs~mz2bIO7S@hAncfY#)5Y6c$OPZePH9WvQhOSl
z&-krz)fbL6H&ce5W+oEnVx*mRezo_vTb~-z7=@`7=7%S%bSiCUy`ziQZe8mvH>oh!
zlT2I1;l9wdwi>VaP3u?*TdT<exv?@+sIhk+84m!cr1k-VY={kbw{pH_1ZUx&yg)52
zT#!%p*x%Rc0bW?mzjm5(@hc!m*a#MKh>c$A6m^yV`m)c_<zy!=MEadtG2?R<qX*_<
zrMnJ@z4vi}oNQ{942p=lTMAZ^xJ6tn=rHJjFC;x;HKEMvt>5)sOJX_cx#n9IEal3p
zE3Oyk_wNiD22_yVR+0ZD4nNYELA+OHn^fmJJ8&)7`<Rj`d(^Gf;bngsPxq9K;FJjn
zv6o4(G`DYygzvk~6K&VsfVUH~9~fyJ(~;O-hC7AkVIoa3JBz9>8C~${Esb*KuAcmS
zNv&yDA3P5r7ilHRJJVv9;_VZk$P(CGtaw~~{Y(HQp%<i6JQ^+3db#tlUTYHhqZ6du
zc`0rob$AK3?2U-LIW!*}GM}wpWu8IH%A_Ls5cZ!2=Fbl5AuoN3@ZY$FV5gz60$KFZ
z2G`=Yipx97>P$b|YM2=bEO%siQ9mN5b!*<~ST7>G<t6hOe!Nzm=aCd^w5VL7CYQ4b
zp2G|0o+wWNOrZI>{z6Jl6O8D|-=$AUv2nC&d46mpmlQS#Oo#)*OpjGz1HjvHoge4q
z(r9=TLqi|(8ig<NX#SL%Kho@Kz|8v2(?m{CDt;l+Sk>q!KL$7_rh4#6LTJRn@=Mg}
zWseJ{<qT2t+ral+)9gV^f|dDT{$=BZ;Gbb69sUm8F1^x(G(4tY>p9<Sme&(7Qw4O!
zrcenS-uA?dbnvCY>!i$;!5sEo2O=>KzZMJC=H)Pxrey-Ewaf`jBL7U&ZV&&W2Si*`
z{%f4tOTGFSwfEU^;ddlu<*1;X&YFC;(D;wNTl>QYR*oGsmS+4!5lC@@i_Wc~WZ#1-
zm`-IvR@E%Yh?dA6R+u%`Onk07&jPdw@k2V&AVINDB6s+uVUyYJgATvtSZ;-*c}4Ii
z1N+Jo2!!hu0VCV@vDPuB`rS*c(H+qZ`xB;%m)+KG@`)GXlNRQwDwiGRbCm76R?Xzw
z<inhFj|XM3&dy;KCpu6oOaIlv=N774PL|jy`FIYQqJ0#i6C2Jax_VaOnfivT%8Tlb
z-dS3&Lh=OIATDco9ys7(8x~{Ld~;5-sn}Z2&#+O8L+2MHwHP9T_RB3<(~a!tY;O`=
zRFB3R+@BrXkIhb*<+0q%5Se(ti4S8hOoUw>3p`cpTv5r<N)Ev$8PWir`sg5^lR$vr
zuhlMp6$&}(ZMb!skg~w@DHl6lkVr5+m*9%Sj74ucy6V>Jo*#C~c|@m0wbkOxIin%%
z#>+OB`dLOd$zb#mI_LbVmOxPTh{yi>tW5oB)#<|X1AwB&A7JOt&%%yOxDh=Mpxq~z
zeP+E8$19m`R6@7>(?`h@g+}d6<sWn&>{o@zRF+RU!oHMFX1hwmNd48~^W$VCefjzn
z@WLbn<rj1@L?0X%MwnW+Bh%{H0AQG>trEHG@6Z4;6bfOw8CvIHJq@zGW;$^)XPB@A
zNPHfL7Cr?-b0H?Hjs=>rh1FfiI(~hMfS(zZn<nqE{Y2sH7)sjGQpf{r>E)Jw%gN8r
zb?-lQJ^s!ud#Qf-1uYC-Dz=NBs4x?9V|%>gHSKtv|M$nQN^X^v``%k4(x*-+Dn4Iz
z+zh*kMg-GOwnR_WJEOZ9-eTFdnzmXmN2u4k1<N5ncm^isX)TlZQ@Og1o<J(2JoN5z
z49^UNII+VlWA7G5>bC|R`+<x={Y_7>ps1H4SL==|9~78{E*Hz4I(R(>_&{CzYqEvC
z$_0lxk6r;t=SLz(?*w9-z@|MtjL1!}f)qeBdg86^R9(BS=m$YspQ*3)GF;Ux7GQ`U
zdLi37NJP#UfqdaDUV=1)%B$rZL9<OZmW!g;{CSENS4+timN$;;fr5@X=J}3lh(qf}
z3}ODo+Z(=94#nHSAKN0rkNNaQ&@I{hatGCM!#0+i4}{%JfMc&t&j04%zC00YwgXB}
zh>&k=M<r5obvIfN*2P2_Sd<<D3!cV+Jl_kefK%=M;o3<{Pt)-MgOKoyc`&ok8K7Z5
z+T6uQN8DyPI(Z9NB(-t4*$x4{ptt46QhGrdbl=@-(;I_&K{ITp6Q>GataFOBX00<4
z;U9R|F&XOF7RRL~RW73-ImhP6Qj98|_nmR?)aL(zj7dbG3j5<`>(%5uF#`Q+rWl7)
z5lR8I3u}^g4B4W^I+4~cU$Sf^Pn7V&O36!d6<*pFg<x2ul&v#)57$Ca=J=!=<{0*v
z8}Ezqx5(PqKqQ-o=P;&IMqF$nR&exYL_51QoY{cK(WiaM0bmNyciZ<rj4lrEEboeF
z<sX&Wc@f!;hKFcr&C0FH<T@R<3kB<J0oY&RW)B?$#Q-!`TlPGRI*`LTpP-;_Q@+-6
z%fzs=;H(qsf~a*EEz)jCZXziSxXfUNL)j#7K%ynN_wBW#?Gi00fjaKNY9j*47{xI;
zb4~PDbx`t_%E_9ZiQyyJNGz;TD;dp0?Ut!2vp?2Q1wxo&?03tv-PAhHw751#018zr
zN~lz55%4t|-E|nOz|43KhK;H<wGT<+#MkC={+=op>EIo7w&50n)*z8J@Wi~n5&OC`
zUu2KgW9UA~xZ|$x7b2zz8z3dFl1(wMb=iAX{<p}S%|MnxkL}j;pP@%#pb(WzyJ0NY
zp~$46!0^Hxt7S%2S+w+4u*xh~DgO<qSJ!^XCjOg3wbTi|f+;^_7Rhd4%lDFA3aIAq
z46j>d3rTEI2j%X!#vWXQ@QQ(u0Xq^3Y1*ipmd{YNgtm^^1-R<Q{I1=40ojUtg_#ct
z`LP6ie~QTW>}T6io+=bs%Kn<iz%%UaP$#Tw2ql&X-Wyg%L7fn<HrYfhzUgz1cj>d`
zHb44eEaRyq$;+<qtEH-t<4~AaS345=9V8c64fQ19z*AKC^vn}hroE7uy)gdkvAIyo
z_E&O9*!!6sUN^&R-g%I&^O$H-8o`DWa~jPF&Ff&YW+ZM?;}hB{fmM0mj;JAXFxmw^
z?xHK{S3m3Ts>$u$R3Nf<>uf9JTuM+^eri;+$$5m#kr#po6T)Z_UDH=8_}K`<=tUC=
zjLOuxjK~_yd9^bbMab8O&jN6m+UCAPdUE(xm&8qI*}b7a_)zP<YYj*M!!}*NS)bKj
zJW17sp5v#}$Xmr;{Gn!M19j&we{X0k{IQ0`QBTst>CWRx&_TafVQ6F{aEq^rS(8^l
z3^)Ug+*Xm?$@?gC<1jCq$S=t;sd~ooo~qnWMW^}s)k`bz@@ohVd&u(m%XcJoOeO?$
zY{C$Ziybk^41Gq1qtKc*2RoMelNE9P08(-kpw71mTU1^BaUsYi(q`dtMI3rSWQx(G
zjbrJ(<jv*x2VZrK!vbxH)?6}65|q32mK{Z3@uTjP){vZp?L&MVpBtUY3_Oc8NoM#F
z*&Mvpf~j2RditUN>k;aQH~f>~rG>}qkv0`B(}p!x8aA^hktj|&zpnb;?-q4@Sug9R
zeWqRb$*1%Uv5r<uFk2Hl5RzU%<5cpZvr5>EjI~;kmdQl`Av=?TxY8874*?5Og+`3_
zmA9T-7)Lksz=*E?cr=KZU}>yjq`H9aJk7%hqxeLdU;aMrHBr$QGkl0zVNAQYb<dxq
z7EjG4IgQVWcLXL&PLn2xPe41$r3M3|gKy&yh5%cp0iw(s;IxJIXZCP;Z+ADaaf5_i
z`6NOm(AJU)uID9Pp5PX#1f5T6T(!#qTqx5Ay<fqPvS_X*mb=8G1^g~QM8u5;c=(`C
z0Wa(};R|=>3vUd-F+O299){vf8F$pfz!AsREQSD;8qlZhZ-qx;L*y;i6qP%0_}KLa
z0<#dO7GVWA#I$6ltWP1A2TWAaoQjJ0&zSgx9SlF)ZOnKl6mVbiGyBW=1#k=UjRZ@>
zFA=t<$NS>|gTHKx%jP7<?9jk!4KS)}E2}x@Cw4a0Oa>r#Q8va}fAdOGlS!k8&EY@@
z4UajP_NvkA#Xlp0X;<J02^urnUdIXM8KKwU;}(B!sMVM79^!ba#h=d%tr>y9LfdX#
z1kdDkrTY(%^FDOkn1a3}Hwg|pL+4a&0^~(l@L?}spVEJrK5*Or$Mm%+b;s`n4F=3z
zAjS<T|KyMTj0fInJq#F-GEt`qecYyR0kX{vBS2Y<`ayQZr1^!rH`wta!yt<fKqiF*
zhg^k4f4}x;#$aZj0C4y1X5an49s^EpR+TsfI)2M5zXM7@^tV8gpbWT0K6}I!Ve(%H
z2sz1G?_FiV%>kzj|NF0?OB@`-sr=o3|KWfCx?cMe9{!tfA*!fX|9J@9{aJ#U?k%f&
zu>Gg+`0H=<x8&tN{*3;XckO?l#lJo^!~|3#s4?LK$$y;dpR0(-Y2tYfe?7PV@87hC
z3ll6WIgz;K_Mh+TAJP0Hn!jnrKU(uQf&BaQAJP0<7JobXM>KyE$iGkj5zW74@wcOY
lMDsU+{QLC(GtnHf!+t+~u<sk=jR$^I6g3}KKClY^e*oq~gl_-<

diff --git a/docs/figures/widedeep_arch_new.png b/docs/figures/widedeep_arch_new.png
deleted file mode 100644
index 62307687d0dc3f19aa6e382c3d9194ebe0d89db8..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 96559
zcmeFZbySqy_cuH<gp`6vNQeW1l!DUDARr+n-JsG)NH+r*l!73gA}QS+2BJuJhje$>
zd*;3+zR&afKJQxZpYM8D3m9gu>pJJ`v(Mh2z4vG1rywVB8HXGP0)bqXk`z;fKrqoE
z5SS_!CU_?zZgCa@!6`5m6;+TD6@@F<SQ(i<H-tbW{lX$HDJW@>B3E2l<6c6ruLcvk
zTuZ)&ke>axbDN18n?TyvEAgoWiJJahUti>9ofL*L5=4(b2D3lLomuZYIE^b$zr1cd
zK~g`DIyspUwpt!qiE~`$of#=No)&;G*yX6lUSWm=Q#Xdc+$&>$_H-h0R|JW+j`S76
ziin^ydh-$;qMSNY+xUhBBHQHHeB144<G_<xyapa{2}0y;Ln9mz;kEw4u4>wkt{(zF
zcwD_4@_Y?lPb7*(<eFHyv3F>ilY#d&dNQA=B0sDRga!r#7WF%f77fuOy46If%QpXJ
z(XZf@FDuc<z#6oOz(5(HMiXuQmjk|!<5ydy4?`0sYj<Tn>`mH)8VVFgNKrfC6n>u@
zGCY}iacK2RGD5E59^w+O_#cYbEqI$vRLO?>ztulnClY9WaGZQZn<nXgD=jKw>MD1$
zTTWW-a|;)RIu3g5YF;+s{#ajA5(DQ~@`beQG9=7Wf4;;U<rv(~;kc`qDu315;Mi7)
zCMk_q;yN)$vxe#TF*_F(TeTSZ?caLv^%n_#3wH-yANC|{Gf)2PF-jBLZQO2&`=tke
z+Pd51|9<KRsgp}{pIGB!egATS`kzgLsA~AIL~3gzc0_jbL)OWr0fK(ARGXs*sbo$t
zRsy7u3;}i)QZNtw4gn6LM1Su44GgNNKZb-)Y^1qD->OC-e_dT`nz@&KC+j&RGQB^8
zd5z_H0SWbwC^DVfnxa|wCUMWB+Ude!ZQ2lI<0rDX5f(`L9)zQKEjc8XaafEOdIt_=
zN<>ph#J2OgdQWtt5KXS`a&sR(x(^lsE(wfE5=w@Eg(Hde?=nEd_1XI{nd&U_@Vk)T
z#u!X6)9Re9$+Rz(esN#DzUd8J#R){>>*IOD1;o5ef|-&r=0sD2iNAYm!&w<1I6hB5
zqBD>YiYS_36kTERmQKVmVO7RGg2laT5moyrH%r}h>9UB`N7j3=&?kIZP>MR+CnQa;
zkH)c_ps{$YuWiTOoNj4fQTz60?DnpJ+l?Chov*oLkI~^aSZM7Jn+Qn$c%k2VAxTIh
zqDuWrl1%X}2RvJn--dKf{Pp|R>y)owKY7}8Pwu*&4DIVT8i5+G7=I}1o86O*42Z0Z
z6l%hn<`4H)X^4J$BcJj)9v4|9x)$>8H>$e$I^C}$4OjGq$E<lUTL_GUHtOsDD8Bgp
zqP{Gy?BNivHhs=bZvTb}m9<nSW4C+p0j)ns#~jD-$6Nobj9pqyz_LNV>rMP6hCqVg
z>N=L;OTMpUUvGTH{d)beSGv3%E#|eo?}p!HzaM>9{d4yZ<V9ezBpWSnD2-%Ba&fYa
zBHA2b#+yw!@FmeA!lEq4Q2H=Od)ar0xLl~b)qt)5yWfXWK0Jgw1V5zp{mqck#|%Vh
z{uFfYLS*|TGg7n@1ybBX(tgGKww>Ht&zxeOikV#bS({3jv9HW!Pt6&j`B3m-wfGHH
zIMa0oCkAU`RYql|SmjyO>nh2NnoO9?Mhqm(J&d|4l*(V3$rzj&*TUArwi$I4=F_M-
zbfPc4mC%wHl&=({NSv5`ZFfa5Ojk@;W+gF*fthiTL4uK0B}Z8(d+0afIwszgTXFe*
zFFv%_@3K95M&cT--}74Ia^Z5*ema|0y{vemVtPlqjowrMlT!C1PFec^v%>jnoPp7P
zn&|_C#X4pYU9XKhjrp6Wn^|Jr?Y3{(s}G9_%D9*B3vE%%e<F`27b1W9_69jA$7@cq
zXv^q~=#*$<jyFXnnk<?RvK6xnHCdw?Ww_YmG+L^DQnsu1Zg+>9t#mK9QFY$$^6UE5
zY1-wS^r*vd?)F^xe97FqS<^X;`RumLUe{##F5|hhS*u>F-V+mj69S98k;H!eHii1&
zXOZUWR(;RJ?%Qx`K3BG|YrSsV#3si+V(K=C&Xvz8bw7in+z=;bAPPs;G1{T!OUsMM
zThU5_X@h~!e{lW2pUJ(zZX9_jawdv9rZzs4o7-|ZynM;FR;y1lS!=P>>3N31oN<(K
zzu8_-<9B~!6;u1x=%-5-)dLmv`)}W7ysa~eXhzeDwn$uZ8RQzYa@cktti-IOb`?B8
zKVUkrB#9&b%4DSTqK>bOKg)q<YPG+;<^JGW`ov8+BRNw!dpVyCycz82rG(y<tcfQ3
zk%gg!uXa+_{SzGvpO+f#vmM;0Qx|?X@{etMTQtcT-h5i?P--1fmSBA{J3gD=F%e7u
zXKq4cwtqZjl6X#Ke5*^Vfxo|U#Pxeak8yb`uU=Me?Fz}L<vYwb4H;8`yZr~*o0YTj
z+xR=VtDYqKgm7$=%P-N6V5OMb=yR9+yrX?Y?$JA$uZ9wQ3$CZGef{Tw=mTcagf|+m
zv$4$a(_XRS<l>$Xj8o+^{&|u7IGLm1u0J{PO%;l*`vdG1v=!V&mQ9V33;AaGU;Dy&
z+_aU4yvo}5bT>+yZ~B(LW%XUr)GyPY^UlQWCP`%#sP?V!t(;YdaZ&SqWEp4Xj?`+t
zAv;N{O80J6z-iDqch~Yz@i@3nLxINW{ba0i2uI{%Y!#<7Lp^889Ygh5>MVj9sV&*z
zxc&Qgd6x7CoLuZ&o?%koqMk^Wl=^VHA~GYYoTE_bITH)P4vT|!mBpIxcRd?8Zlh*!
zcZjS+%qI%PTXXmwHIxytjj{5NlENvqK9?yMZT?^jHKYiy{<+n1<jiZtYvnwMvqWsJ
zwwY~^J5-6I8CLe$qIh}!up7<Xt*0xCE?ZX1mA#cFmv-f-V%9!;k9^O3{lIf|?_&+3
z2BFwiJz{#aeDl6Aj^|>Ze)W@v*DJ3MznY7ah#QLQ1vuCDdK4UH9Hy<km7?J1>bDpu
zCM(q_DmM>U2#+SL4;czM&i2xZk&a4vm9kQLcfg^zDfRh|Lx;oM_Q*PNJ^Y3XKiA5x
z=6l5#wH?(3`O?*9qTLN~hjES$<x6IL+4+?wI}w6k1@X7{7cD+>rLzpa8H^}g**i`@
zS`sew$Pr9*NSv)J^-1!n=6u3QYQAJ{zR<P6)%|U&WB;S*+8fK8+X6!!`aOB2@7vYw
z&?(<jIqyCC%BmxA6jXBK*$s1^9M7tG^d+$;*Sd<bOp|$2%1h}=QZ|xDf0d}F^<1ls
zK5<$7u$|G^*_g&1M;-S|nB9YGXK|Kyt$cjr*UZ5nNwHuJf$jK6eOthdh(lJL@<sQ(
z2mF;si^Ma`X*qnlg3fP&Kr4o~QYlKf<%Wu?os+)he3}bC$_^jAQmVy2V;^5W;WBfm
z(^?s^@?u%1_NHBJA#=fkh@+e13fi&Hmp60xqXLc29NSK^_$6tpeqTci(&pteJ=u@5
z^Lul4>=KMe-x`m%QtfQ)>m9{!9rSt~&U~0LWG&L|x}teGo_w9E;>hOsx0l)<wP)Wj
zP81K-y3IfJHk?==O#F~+`dLTuD`u^<enolohMUz!jRl9n=G>b={AV|Dg)=-p?=|fY
z9n<aq4#=m|ThsepOXg{Qw0u%+Ebz=tyD7|?UJHtT!&;FLVucSw_+cO&C?Fk&8SyTP
zi5VwIDcFf$+WxXsKAOHBH{|MXC{`JE-8-D36!9aa2`JXbBo}&!me%hdb|?J(nex70
zsUh2VT9^YE0&d5xqwg|=j^&+C+%d1|TGWlsc%uDygY{=)qG1JqSbsxxDI-}~2ordX
z1wn_BLomQADEJnFQk=bh2xWkv{qsHy0(oN!0U-FFYvjOh)K@V0M&0x8Z?urtkW1h%
zLh$XB2s^zR6Fm{_^fgQsd<GFw5|xqyzm=ZZ7#dpInpoLox$8`UH?XZGHEbae5<1j3
zR7#O{6+FM!R9W3lT~>zgnUw|069cQKhAd7N)~M$|1f2N5OAAB0CvYc==a#m7PJ%T5
zT)_ukquyqvf&X)fow*>5x~u|R)XK&X&c$+%<sOX?4jc{_urV;=Qxtpn_jd4?AdQKg
zoi!gTtD~bMiz7RWm5nj$eO_K()_ZKMY;1SI6?bi&E$yB--L<s6_3uMYpCe{y`^?7F
z+RoI<5{`QAlc!epc7ilCsDaMD{+*|xlj-?LmbQPF1s2GP`h@j9%RSb!XM<Y>Q19|7
zm^v9g*AO$c05St(2yycA2>f&X|MtoGh!^fuKfm)nC&#^uw_f<@f4x=3*3d@O$^wjO
zCv?7Fe{a6{;olntSW!#AAc}wG{Li~U&_Xx@tY@1hgyU%5Bm(x4(o{@d8T<xXhWdgs
zgC7k4{zknnk9y+wX$t~DK%~S(l%1f9!<S--ySM7rHcAQ~X}GLZJ1SRTF(lI5Cx5mV
zbDgt8?E5<mbULOanD_l-&JwXB`lqux6`L$Kl6^L>XpMTdbVn_+X!W}vaVNA^<Tf=`
z@9<l_aBQibDI4m!{TvI9guwpi9|T=678p##-JOi`zgz)seTYOtZla?TARy5H@y7!M
z5{balHhAs(--ba#pim*S|9J|ixxEAzcoCHS#Jl=GkANB)26MywuTvnvf`SXEt>2T;
z{P%UBqwnJVA4CCzp`$+-qhS~O9~b%0&{$ZjWdBVH1PC}fIu_Q;I&mZ4|GrWJf-(00
zx*I@>SOf%EUI9%6SO4oUaCn{Zf7u8yM=t^bIGg}R8bwX>Uxz^;kgflHr(o1@1Xv0S
zo2b}-9R?^w!hhc>^#7|6|L)%ZuN7hit7>P&mCbfqW^cK2g_e(R;MQ9zVGnJW%~|Si
zuP=vcSygx6vanV>J563=L_jrTd&3@6Ur<eC1jfT4s=M;SM@27Rz4}$<ype1>(?T`(
z<;_hG2G8S-1Ufc0#hT-N^SREnAc=wT<ny!jg&rvCZVuzE2NxBU6}Reme}T<zR^b}n
z?qsO^r04OBr@(*@sh-<oUDqW}KVRQq*4jhG<(dP_ppu4P=O@iu6xcCqZHb3or>A35
zOTXTqxWKgY_TJu1LW)`U=Z?=R%o%FAx;3r8v2$eg54M+V=DQeety84Kl5M6M=j0|G
zDh>D6r>2V^z9pA>Pb1z@-Fj^WTPBw0@&49=G5O(Ur@SB)UY^AE;YRCkz2kLKY<&C(
zGIk@$Qp@4R(%pBbL&K4XRA7+U49CS!B?z}ecUbJB4iZ@Ry`x#gFr28v5)oye#myf;
z${N31?d?)vJ0o}PrSIFfVOo~uDT#6m`5UvB_rsXg8o&5lni_v|b6cI?<<Fp1txl>;
z)YAcz(VFTzx>Y<nuGrYvgx9V;)72_4@GCO!Wf?vYSbJCJca1*TVQIi-x{2%>k<-c1
zPIYHqw?F2q3Wp`8jhR;NFZ`Eh=lr2ur}G25P(TQW2Z!P1-a6f3YPj`S9n|)gMy)vn
zVI19?p(LT2t9uv7jUC24WbM-w$FH`T>Mmkcdoa8*8DmxRo`1Pa^tNoYa$6ef>V~OV
z_`yo;X!0v5DJjO)jWoe014v{|jq`?)X<y#Q!beY^CMgw#Baz?!Y+b5gL--hx6RFpF
zR7-fs6;N~?A04Dfh2FN=UeqAtcgVt{<hfTl?2LbAxZvM0fW6+mgdG1nU4#kl?AzUw
z!x`aJp1H0Jr4-kl<tsnaAAg#BMXGzRRU+^@b#!4>A~y`C5nzOeg?IfP;>TNx8A=4A
z?&E4d+T{)o^Bx%vp3Pj(qeshC8*&MohLb<vuXj9RmGUE`6H^<Wb-XAnwrfCGQ8>A0
z$AKE2$jxqVl%IOTc_F)WG9xX%!mP{1X9MgQC!^1m8+^2{aENWT+q-yOlr+_dJ}@f%
zaMro0U2apK#n#r~@~y~v{Of+(^XR16zWis?Ys|S-Pdno5X4^!H7&5D0y?W)BBh#jW
zYg#(d5EL-o!>Un`+?A`R$D&)sz{l^={aHovvod3Q7hgpqb)m)1^03Y1&#Q%cM#Wcq
za&(v@Sad>|R6a+t9#BRT8V#0MOwTR$7mEAb8mN7ROFB>)xpJADgQ1Ng-vNQhZ3kjW
zQO|#Bv%C7l7}LcP4JHvp6~=2j9Z7t%*sPn4(Wr|lQ&yNX`uL$aiOF5Rz`>4`N6p5m
zmctc_6?SuWR=FC54_WkT7OQ$nEVQ1b<U}7!8n-RXhqCBY_)gchK37`NC^U*Xekd;P
zw_OgN-D2h&*pU7!UeGo9d4GXXcumD@v3AMxsJ%lD%b~tN-ia3&Ylry_RH3!UTe;H%
zCQEN;XJ-ZCR6SES2Q2#w#1)m4#ATOqt72!<5<Ei4In5$PN8ERG$T&<CbgvleooTq2
z0%6`U#6a}S;9!YBnS09z_p5s?hV2}+xK-GR^Sq%(H-}p*ennd*tWW)p+4&Jj7P@Tj
z{&aZL@m--MB+d1t=3v=&q;tb)%Wzm<)-hIS4;X{yW9|{BPp<o$bIWCAWp)h<>aVZd
zaOr-r|25&{(74cQrX`NueH7<j<c^-l-jv;n=h0U0+`!MzwVnrqqa8OQM(In39b!y-
zqxY6d%rGJwP7Z2M7GiroQ;%)_#_{LLO>@`BaEX1k<dONf;oXhl9@&$kO1_|O)gAZE
zc8R#&4xv1pzUZCW-9G_EtT7Ee&8}(kE?1a|!}f;y5{N7D_XP_r4tLiqnP)#NQ&!{>
zTia-H60z}rKWgD$i9A}VSYX-l#=sp?)gSvj(cMuo9zVCX=lu9au_wOZrb4{jGmL_v
z9$D8?V)cs8xh_J?*Qbw<4^}qp<OClEb+__TTtICa*m)1?_OqidpB_{!acG=)7H@-A
z$@96U>DJjq_DXvr7aN;`!;OTK53bAQ4gDz(Uf~%1s3s-|29}OLcL7aSD~b7%T_@T2
z!0T$+$a`O!MtWv68PseO&Hb;%VOU!6u^&zZ36~`Bjb_KRt=FO9^Sn|jMw+$83pO`m
zFeBh;@(En*9iF@6mrOfT-`d6s<r&**L81AUC8@6R3A6raI@tOc=mxv1<1GijBUnin
z9u_<2u;<%Pms_gpe{(B6`NTfbIlWt|SKG6)jpL=J=dNO>^6Fi+{X$P_dVV~68Zj*T
z>ojnjT!nk~VXt%I+oDePD{{u#Xj-CgH&%Q}$O+(c)pO56BBPGtmt`0HmU9dHi-pN9
z$2(7xwMVT-zm~QgSm1R2<3}%ZVz1)ea$L&2fKPPraWBW7sYpKBLn2o^O)gFhmwurU
zCMc%{nQihs8cG?qdsyXhoHZ8QuUD(}1zK49&I<Tyqn*Hn(W05Mo`wN>ouwpB^v#X1
z!KJyhgz)xH5;4Q}P5T<%=1IN%IA)Ox$t7m7gU!dzs@#H?F78?sjsY>_nEm-l86zp%
zn~rG?{3GPPZ@{nM`eQ5a$+wOr<JSF_1sty;`g}|E7$NmyypEdKsb{XQkXr%SgVNc5
zN;<}Q4wi>2d!sIoRyo_w*dK_<F{5X3JV<3FQ^V?OSa4jx%<h*kc5rnmRKpV38M0j#
z4<Lzhupd&@r0TSD0M`}?x6ET_ZX*YAW_gCY!>G0!{kccoI-Q3-f;(}buBw%q2F3lE
zcmGUG$kryftxJ{Ke;dqZrnJM`?|)P|heOPK`Ga#V@bQ++OTil@Ve`K%QdrqmG#Gtj
zdzUV4<3I&-iJfe;ptle1ZtUeX4Agph&TM%*>0>0wBoyiBNQFi;OV>{Za!$N((apRK
zX2D_L#ymKaTfH64Yg}$KRWKaHeGWPvXu&s%D?VC$ojrZeE0`e9N?K_B=dJ#Cz0q5i
zmK?{&JGJtq39M+a=<VA}Z}lplwF(_%*@kWpmD_Iro_0;dM1PG(_H^HB)Sy&p&w!Mm
zjYHJslcrU2T6MXlk2-r-^Fhx%&H>$ws|Y4~5x(lU$;^-g0zP3p*C{V-pA%Co{jGo0
zV|9Dbiho7Ro^~;!*^wW(qGR?)?A<v!{I<_6edOlk;+wCTPx*Gzn|Dicxu$wDGcx|V
zK|x{u?x-{2_``9tn?RvOw?L+y!XA&{oAclU-4BFz@bX|hwA17KGZSH8kX+7~QCM0y
zs99w4WP4$x+O=ZXjf@~q5hG<G;K-mOS*k@=lUXJ3s9=vO9SUWhdN)_=J~7DDw%DUD
z;Q)nJx94)dzUi?$Zil%Y8pUoLy_3Dkzx}9X#~Tl8)?QjG;eq)hb<@v70H!z&bY3Fg
zY&+J(_}E!(DV*<*$=}O4kzevim3?9<jHKFir$k;2K5E<)IllRe$t*6-R&y>f_8F(b
z$KG$!(6BUHgQ2~`Rw3J#j7-D63jyd)8Gsgzn5tf%BDR-Eqz;5D%{x6hTEeR5crD0N
zChboU6na-b%e|YGImo@@7}EtYHu)=5PMS9JiVtImc@XEb$4~g1Lsj_ej@}3#j(=vo
zyxiAd6#GIi80r<<@wlHfU8~eGRhUc(yMM*Jg1WMnTtTvOiijX@UN`a?E+0G(fVDLF
zLZfEs!4%4yR{aI_7P(dV4JmVB9LPd|1}KACpI^!h_nr$PNi3PaXHbHBjyAC1W4GaT
zZ_ZuWV=okDA3}7Ib@f{~SnkvwPO;88Y&0N|Dx1CKQs%^Ta=oJne5_gh<=PVu_L@}L
znCSdDJh$qPv0QL{Vs8h$#hB)P_g;!Y(^9Yx25Ldn7BLP-nAwc)yGd@`u=tT;`mw{Z
z1_#ST2eVQ>dYvIRMmKF|)N|B=xZGjTo@{X;StPJYQP8mWbx(BLM4_hP`{DemH>dK2
z6Y!xT^chX|vzP)Nhm1=VSA!T7$ML-Qn!UHb^>D&*)IGwI+Y=8fS%*4L|BJWX(dF03
zx<B;+aWO~2hpRqHlspy)#3h|<P0qrE)KBe1{Qd58iw8w9tT31ZP8>iO$xHqlyN7!l
zEoH)clh<bK-N!PK$P$azb<eGCwXHc$D@{r*A2jmWd>>el@Uc^TOPW2|X5Z-XcBwPp
zySUS13`KwN2xy3jj@83}(<Si0fX;rj-FUGv-Q2QZdNcX1as}`RMONiYNF<3{bD3$5
z)4`9_k>pl0;JPaGEOBc*j`wDaPO{9f+%}rI<toW=n+Bj>a&dfy12<g5T^tGrfXBwQ
z^@wX<nrm)xx9TOx8!QaK72X~5vCM`hdXS;P97mkfoOO&7X6y^mU>d|RCB}uB;;J#j
z4U2a=srW&h)Hyoeotfs|@0_&-Uu^A+<}{D(-SPs~rgpE{N~SiN+bTXszIR_t4Wr0%
zcp$bAj9xDE7SrCJnAzx^(xZ!z_J70wA}o2XJC>z!HV>`BjfaYU&TaMTtqjFBMMhLf
zhcQi8H{tZAUgkoXte63pPo*M0m}D`no8>pa-yO!S@zrKGFW;Hn^CxBX=ypZJ#Jjg*
zy4K&-)n&J0HCi)V{xN!3cG2_XV04;t;73awe`nmM!~_FkU_X(hOv*nkdU1YUCvaLS
z?v)X?PpI1N%wcV-aX-j>oxHcSM6rQEMzFiTkQ=u#&*U38s3b3FIMUl{{)6~(iG-^7
zD?t@e#6f!Dz6E|ZZt&pJ*H^e2%b&fj3#O3-we0dX&nFfo{0<b4DCB$Hdtu9!P+P9+
z`0(xNmS2b7C3-&E<~9|kHiUw3xi@)jzFV}8^zLjubo(J|Jh0$O%&cmcz-#-XDw{pF
z;Vnf33%!d2)T?yF<+rJ4;|Io;%L=kr3jwSRl#TZ2JL=Zx2ho%5_THEeeyoF@+nPA=
zkzP3lc|pZi(eszps-fB3p-VdjxL77*btCdh71~u!?XkM8pBVi^LP8d&*N+4<k%)mm
zUmf(=FwsF!F`1*B5i(~!)fhG$!)0N!|B1V6>vqM+s@hCoOmzs2fWT0-<BIM^$@K3N
z;30<d^YdGFaoOOBUv5-773Z)>y6Vu&$1nEOy?(a^AVW)^PC^brTVhmo!6X3Om7lGm
zs`EUKTpZhz#D~f825UWQ3m-jr;O#oy6lHmrCF@!BO3gv+PJwG;f}jp|Qp4taH*=-V
zg(wXI^~c#b&$>O(Kv;b*dEVbq{wtH)YYVg1{gXzmvHd4o08U#xG_{K@2214R4nP3H
zzhX#Az~itOSt%36)>6YaQ4N5?^bcEQ<t~-N!6jmt=|LKI$kL|xeg#!7w;8)7pOS8?
z>CH?ihHI9%KhcYYx3=!Al}KcKq2tO(3-8mXPlvKKi*1bN&F>{AmSn+TE*_O-d4l1A
zQ0TDnaD}~5czJaYCRWi{mJXJh-c3%k56jV(G5n4>>_R$zFNW=V%O8DUc%s`$B<^4q
z-2{Vaq9qzNN4Kn1D2gU5^vbJ#nOWqDdy?BLa*)z1mHP~g6>DqCbnCK=!H`?|1H=KF
z#Q;aAM0W8<2VRS9Y2t&5bW>pXhxRoJ^pqU=O~N|&78Abx{3*FpVlikl5VvzdE47it
zFzREd{_bh=q%K1#ZO8}nvb5tTbt{>07M%e>hedVAAIc&nkt4B7qqVh4cIrXx9Q1q}
z8cIoh+=HAat`0HNFMDI=x0@9XTq_-xhM3iIWE|7MNv*4Ixaqk#geVCrAAe0T#1|l(
zxpebn_YFImR#5xK?0pyZ3UFFe@ozm0{5fyEQQ>klj_)bSZ7HY*V$NaDqXoS}K`vIS
zfxS8Q9)dR8WS|umx$5~eok|O`Q$^65gHp!Do_kXfE&hRlmulC3P|OM8lZ(yv7p7^c
zy&JOYl>H&qd1x@SH~@9vH?1{x)LGnUiehhI>@vTZ$=y(_kb@b-Hn>Rfhc^Xp-5`G$
zBpLc6O%$ZmtS5`+K3~cv4(7s2oap`_>B6ET9vmaP;L?}LwP_rA7Jq=OYAm{Th0oS+
z`g6pK@WoZ1^DyYm(Rn)TM7%IvqMWHJn?><~_p>@TqCPt6j;bc9=|EA2I(NUV!0m}E
z`^<CW-`>a`xKNotEdOvvgKfg{X|r_1B`T-7&EL|kw^S>L@+~<6>_2H{c^6GDeD-0h
zN}MS2rwS<Mdyu6u`xN6o+v5AmG}UNY+xSNjEK+aDIlots-RZfrHgscgs`?5g4_tNs
z=0s246VZIdp2<EmX7>;I#W3L?zi@CSon%B76B97ZIla_PeZE)p-|b=Yk`czmXjc`9
zn;84z^E6xkUV`3{j0Q8tcg-8`C+4#CV|X71m73W^wpv+wOmBYQFZ^sVpRc-4A9erP
z4^Gc;+eHTc^ISUmTLAJB%Rk_upMpFr53fL|*VxVbxA2IdmrXPnLuCl+HN&WT%-(2Y
zVB{FOsft+V&tpdFBRvxM>_306uh;pV!Gn2vI|BY4l}qo#FfKb)Sp*yx<i;m6Ne|tH
zT{hFz8N&=fv{qm}{s0xBNnjpztLgo!-Wf3;DCoeC>gJ^~^$ooML*r00tW3tkIvoN_
z7gP+sxLWua8bsSX>QuL%ZTs>`w7{P%2H>4Hb)S~3Z7u{eAg@KB-5c6|wq}vL`j^mY
zZr!50fB$}ZL*v*5WC--dn*<J5XiJJcPd3#(gK5^O*%q5a^{j}?xBB#X?q7&`0Twd^
z)6vCD<2@VnbfRaK05@uoXd<02@;mh{XOlUV5lE(DVL6e-PXEVj{^LQbuRtwEFTL~j
ze~f;buKVXfx?u8ShSt&NS&p+Q{C#d6DL`UwALU|B2maq@<fMT_A|t+-UipvF|KmY=
zpxCkciqGPFm;ToU{a>N{?}GjRs42)a48>gEgASG#^l?N|LaZTA-RDSj#b}d!%V-|?
zGna%*syDIbZk(sOz`kL*0G$-^^dp0xDGQ<vw=9%yYl!JZf+jq}n8fektLh8Aeyhua
zao^f>{`hCm<b2(tyHoS2Oxi9yFDYAG`zkMG!{9|dA@Bvt16%vtZhS%2hcZH3Txyc3
z_8us{b3gdag!ER(SH*uBvG?|6Sgqbyf5&`pv?mxZR9nt+^C(^M0Hvxrd={9_1-*jz
z!Wc<ur|QI{`cLA^u3eeh&ol3DC)RoVH4`snwb4CWS@c!z6cMG*MbUcDf(He$VCsmR
zg+mDVBd-quAq7@N#8}tOyD@merYz%~z7cDD#laI`8COXtfT6vJH}ST;sJ=*2V78o~
zQMWG~z@I^g`FyF6D)(R&(lLaGT<FR(o%>;QNz1N;s;Tldt@YR!E=JTH0O?d&$pu^x
z*`#z`oRfA)sQ_(8L)x^Nr_~aX>b|O<$!dOClC*24)WC&VL%d+10;mA-jfI_`JYbI`
z1q;POxJM?|_@KQ5R@tls?UT&d0al*bX=6MW_JLrv0N4jspU#W*U|wvKIOOCFQL&+4
zyi4<gJ33bhAJ14Q>D86q_;{Z20mJ=7`GFyZeEc(gfxvVLB}a}zk{8M5QhIPwqgKgT
zrJdfY)lb<FH(~SQUbqm;Neq|_33Xn^3vW6TTR&Pa@d~jSJF23Z-8ee2`uOvtC7VHM
z#v-(=?5ofU{*4QJ5AkXO2GC&nYvu*%IjO*%8#+-NsWy1W_Xo%I)i;l6EGYbB_MQh5
zneSVQ#D?2mG(5;ls6B*)DP7puU{a_Kad4FdLifsAfO{t<T1{aAWqrM`SQiEBc-X;#
z_{l8#g;`@IlfvP_BR<CeIgPFFc2b(2l`06H`pl*2()=cG=yYt3wcm3tjjz&MO;*<^
zgDxs1#vKGA5h=oZ<HD26iLL1=-zVRU_T`@wv&3HuI4Z!yJRIOxGhp<qdh)3(>Gy%S
z=iGnH<&XvxeC1&m9wKzr`ccP#7VXyL^MoRg?6=cIczLZ9ss)u|C6T6r_Wi7DH_mkj
zitYyHQP)6<oW6i4mEZ&mn3G)B_4OXlWA_odNt~aXI$<Ec-3I$a((~1Sqz$%{CJgT(
zl;XQ_?^BEUy*j)*1!ZETgLH%%H2OcJB+egAfMtYo%x?B%=U@w|heK;c!N7Rw$ARG}
z+d>y72AYT9W2Y%7;bSXpqqHf`|FZ^zx=tDEQpk71E%BjWQ0V&veI!{T^Mj-TOJcRG
zVjAxYNDr)*39L5lkmI}(M_>l~rF9_H_{9%;f6?s{n#T=8r<5AP$DR)mfl+c7w1l7u
zoL%p|dA187aV?=WecXrd^(tbDEPE=oYzs0X7-&u9-=5KZ{^s#QJiIB3QTIO_Eh{$g
zii8}^7gREMy^}#|WXpS0l*r`XSa18oE*G7znZheO&(eI(WsHSh2u`1VO-11Odg^k~
z2|Z0{mvM#nRT9`7eq8$kvG}G^_EBs&kN26az=d6dKv{9Xz%*R4XXPlYmpRCTKO~uv
z)}pnTtIGH4sO*#nZ`Kp{)_F8K{=Z2ahjxYPHlyq7gQYRS8+dJAZ+e}?Y0R!{Xrb|)
zWz0a0GY<vowwarD&u!r01GuRQzm#BM6YG9R{<RS-Op}-=YZi0`TOEPT8<ZC{vkGOl
zAGL9u+j4@hunaZY@q?!{8Gasb#<IaqJE?yV8+GdO$NRn;hj!yatr$*(!kAxGi~`Sv
zC`2qNSS=Z%>bP!tlg)f6^9xdspP=9D<4fX|NDW7)bIAgI7eJkhZ*{R$=Tq|faI_-5
zzWv1mV?00~2@~sbR|9Tmof((!e2Pd!I*R2H#@x(vuJvxWfptktro6sVxXsSBn5mE4
zzE!mj@k{4sc^2Vxu8Y9w=>q#chG7tWZuhV(yprt&X}7-1)FM_5?;)VtbXM=R;-!Nr
ztg0S^dKyi0G$*i@JhgcrubnRy3%wGhFSTC-FQC*JnBVXDfP+6=2|M3<zd$f};uN)9
zF5jhz;kF9@{#^_k7dJFVr&9WN_}!^@{L8_hP%9~eC;rC$_{ql${2mcTP?=|7W%fCj
z_A0&CcX8tSIP1PNo%J_BHf2f7(#j1!x3AxG-?!TX#UR7u!@UCg1yxY$eedgwRX!E2
z)ie*)5pM$_jbC8kX2y-TXJut9IKds%S(}n(yK=6HwZxLULb|kjS*`mstY08#1dnBs
ztORVs>Zg837=ld7ZA7Ao_YHmr-@&Eu*;$jh?#zhLr|G)YE=DR}ROrVr&YuKa*<+B&
zc_+4b0U0BX(Fvy?`j#nm#L=d@R;1}VgBp}%Y;NIN*;l#>bDMV)f}Y1!GVucaSyXM1
zFM!fu@UGnj4ON!A22a;0OfA#Sp=@oqa$yVL`q_QOM#E)Z=bMR&9<K5@F2}O}b_cbY
z=@(zC9?)c{b#RUaAnU^V=PJfx=%3stbnl!JD0Sc8G)1v5Vj*8XioR*$1sS>9rK28|
zU{!#t;4H*_>4Fs@U;+|*FeS|{cfQUbEL8P)XfuJsG^8npTYh`7|Jgvh_O4Ea-F;JQ
zgD4eI_}oj00sNxS3Yte!$}<yt%iH<S8pN8H$k7{uPHb92=pIG?`EHiWiPadvny^yB
z1HIs9z5veySI?oHIA7hWKe9zxuhJnqDkFzGx%hgHB}mGIk=)gcjA;X_d5_B597$;_
z>I_Bf@U@EWd1Wea`k9?P+E4JdYa>P*^*G2b<eO-U(wisxz^o>>>0)Ay@IfcPr+>kQ
z!j(~V{rNECp>s)>6Siq@FZ5*F8YQu67X73orJ!I2r^42fW8v~lmQ6o>j>+9++f3VT
z-Mp1arUB`s9IdIfCuYsi%Y97=!lT-iAm)zZ$^@A*7KB2w<kg^^R~Ol6tE9kZ9ox{&
zUOHPoQV&bJ#!X{!=){FxqZ4A<pAA%wePL8iyfa&i8?ZiFRT4tEhHqs^)V6fje5p}V
zFCUdMn8>XXY62-NUQVo@ER9>BgvaKdu969|1ETlu*W^?z+LM!?^Rlq0!LZbJ`D9FI
z!`5}f{QUe107Js~_8zFv)A90Z6k#HOCN<+g;*Lf?N#KLSRJD&@?nLL@8xXSIX<pvP
zzUzpcE2K<g_I7g*5phT3kG8EWG^IY#-n=c*>l)mozaZs%L&HH;uHCtt2Wkp}Vg-ee
z*h^>j4-Q9HR8*81@#GI{+MMeY_klfm@<g%JQimQxAy>CLIngK}jdD*rg!@DuvFlV3
zdiAYP=;}JYQ$>_OoSqu8uikI#0MKOW<?eUoKa)^$Mfg0>yp;s%7|wj${1*=Ym^@JB
z!#T4y(lb@|f|0hxRP#G5T3hbQ#c(y3_iGgz$<jlt2Q5mS7mad3<Rhi6Sy|UW;!7WB
zR;r|(Uwl}xK+X}(Zv1W|s=vrIl$Mq@GVy(CU_s|YPKyC$9Ac))-e(_e&vghZnHUZa
zR`qE(e@lcq5yNnaY&4W!p5!iGmKI2&#>JF_n$orzU*z{!Q}uC991L!`_5?ZUHiX@C
zKZ~iB3n%O_eu4Z{Tx(bOU9H$6#o<avM{jA6&k0FSXGquzeSPi~2!2WdD<t=z>Kqr2
z?ZDn#xnZ;Z1+>$hyukSp&aBpw0c}f@zsacb`9tfEkfJpfsiQqTA@k9j<X4zoBgDA&
ze%!1G$rUmgH<jfyOp6<+aj#5HO^yEAscHV86L*fu8;E?SV%4sO@B5eYrD07fg0WdX
z_MQ8@I#2X>H-(pn0^@+WpyYRGvda<kBW!8zLvePPl+(w!<KzU_U(s4@=ML?S{P26e
zSq`GN(ZObyDnT7DHv4(y8TN8e4(K#`{yfX%E)IXaLL8s{_bD6aDq)U`hE^49Z$ZfT
zh1jC@Dv#}SU}(-%<w{Mc&n5im8dEPdSFVyANV;b6=h4l*&+ljxLSMcz6jQvG<HAg<
zixm()E2lbkJISOCe+TGdQpV@SB9%*5DDORE(JV?g$;RPFC{(*_C6?+V&li162V8n>
z&ZX+}h$;^ql!PXSWV#`BfL;<9Fl}6Fm1+h_bXclm>WV4)7?ezPz(Sk*LOqI~wqPzf
zn6fg*q`fu&27DJ##@>ql3Za_<X>&(!O_1itle&p{RvCbg34+r*QkY72uEclHL2-pH
zQ!ST^TW0IW9LVH0#RCRfDA1H>iqvSBWIze#dbrEeXxi*b&fquJ&F(?Zg;YTBP%%B>
zGl$4_{#eYiNRGb0*P9Ir21yCFuHBF3I#PU0`vNh}PH7!J3ZQ=xPsBZ4I3gW8mfukh
z@O&1UB6R@?jM_IKAauGphCdU+XFpG5z}q_<QkLaa#(GS+iWRo7?4L>h*rHVZ8=a?z
zQLo#|NVO4$*9~}?^|*I`HlV?ER61JC%uovn2^D^GKGz?Jmo|WVKRJo{&q1?xCNdSG
z2{>Eqw=|W~%7!h0^?DCr1rN6XNHQ}4`_VffgP@GCnY9Y3nrcVRnv2-8C4G5AsJr&w
zn@^`L!5CCS!05-ZS(?Sta`6JDKeRzw+ztx6tJ!oBY3hPSPl^Y)pPw!py5QhoII89W
zgV;_tz5nom2$)6=Q_sZ@4B$|5=T6Nhe<=u6QRaD}A^1XXgyPPvMhD8A2i;lS&>nxR
z%=CMM#n^YWN}BdGGz*R1#tORj#HM>3?E{!0ET64a8V-t%dRl0dug-!x0ydNjo>pX6
zINkC(TiCB(zt#a4g^v*l$^mp>`GizdEL-#4wl+ySv+duqa8|YDVG~Um!Y7YHxJHPt
z$2A+X-ikR)co}wNA=LVHyabt(rBP=6aig%ra+t5&X)VdQ(*0n&#f&1m`Y~`-2lr|&
zm`;5lhE<Vshy1e;VP^K(MDv=gX_9j*puMm}u<CK`9h-N5o*Md4wbi2~z5vV9xf@*c
z`BQSdhT9f}$fmTq+a<r})eOE5Cp|^m!?-~;)7FE8`*|O{;(>&cmHqNiKlTq(i+6yO
zvo{Kvy?W-HkZP#>WLV6PQ?&0@N8DHKS{hnnFyED7bA0H`9{E`%lMjkSkrV-%Au03O
zeA+rp>ha+_akiLZu2a4WD;F`zt;fCDBx>WYb)SDowKIH|I$MBCWHnNy0Sa8SG&B#2
z9=?god1US=CN2&=R{?^jDB&-9vumFUA6^Or2Ep2r38;;RT!N6rm28EN-nL4Q(I9}&
z*|@~m&+3C{^m-9ewX}wpi`mfW>++1QYQ1jkx(p=ff!cCYEN|zY#z=*I<}6ME@b2X(
z!b@ubCN??Oa}#e&sDN_CIdTui1*g3n@$}4Pf$-xUmEe5xjdn;Kz*J3nTd!*cGs{4+
zNwLOlPho6z;Cg7!)`zZZ>{UJS+~hdS-J&!x@;eWeir!0j;gYiaEHdqkP1w?@bnxq;
z1Ef#l?6vlkN2D7Yy1?OvIxqyK8DEIfgh57PY=+c#`7FGPC*-l8|6r@MDtxjRM9F9O
zlS8G*<nc^uA|`}`jv(d|Mu?o$pU)+KYRZOv-a4&x<&;$rcDqDL8ep$*N<ZqoV-0wr
zQEZ0(8&+RHQMIXjbJPn!Y2Gh&pn$AvOF?<w4KOB*a@RAOZP!jEU-t>d$KKb%oC8HB
zc<lcwBE5Dklm((t>0lAG?GvhZcdVvLB1ClrzWiNJ)lj=yLci*dKhv%E79Ip@AIZJi
zpSU=O)vZl5$Rg{GOaH8ne=|xTq?7(>Q93-0j>V*w<JO&Db?#5bPEf<2khhDT4nG!+
z4z`>zQ#G57-IEiN=YGJ--Zn8Ykphs`ZLCC22!|t2%nHVkv6}RsfgGA&wM6FI?m_E)
zKE5=B-A{AM{EL50cr7ls!pI5Qh@Lti$G-brORY5g9Zr0-BiiXVrv_*~htAhG&rRQ$
z8rXd|qBFgRU*&P#&dSm#{89d*^m$ZRN&Ns0Da)O^nnl*CP!wfTn?)@+kgx3ZxKOEd
zNY6HbW5+Kfc_ec|GRV<T=OksLeIB*rT7dzEDks!?rS@dxP|J3Ch}Sj*l+~jkUOGMq
zHKM$;y~f&20BOCG%G;Y~^^c!nb@c(HhM;gM2hkt@`YM&j<l}=^$$(cWXNDhFO+i8u
zo@rYU3Zy_<D%lZj`A2S-P9<Q)^@BUUb~`a`!4&0-dd{C75+j$Ty+BOG1N8<(WB+RJ
zL_mCQ!GT>SG|>+^Uw_>W*aJ7u%d=<kK`^t|%a`aH!17B$lM8KUnq%l085s$QiRs=_
za67;tK(E7o{`{#O7aDh{ROddGNNcqpmmI%m81}|<TaoroO|}@vv8ML@u*2ej&ELOa
zfQ9FXLj+JJpcLg>ZY}gWnh}5#@=#r5YzkNO493;vf{E-hoW(^ROu-*M(1P&R!7`^g
zid_Qmpy>nzw8=S4#*!fw;6RJxq7`C9A^D|39EIo^4}R=U)oqtkg-A#&hX7`U8em5(
zN4O^|)eu`(4i`ABJ)IfND>m=-C*khR1~|T@_K9X^vPpYV&C;FhGc!z$3XH^9IKG`i
z`4dt}S=lh)Gf-ZjC6@7D8@$k!F)<{t01BO;KDKv)oPFYjl^h>S2eMHC2kXl)UOf-_
zKTOI>H1oBrxLjh5?mNTu4M8^}fDiiRd-YZ_VBqcv3N~EIN9LGxbjxl>qzY8pO!-Th
z_5gHk92VLm;P>|J<)_%M&y3`*97w^gESi%21q%<f5yq{{1I53wAu<U<IxPvpR>~YF
zCDX^fLxTW3{ds8a>#m9-CX2qtdk>hH63b;1SKfCqha7^8RsX^3EuJsj<WJfOb(b{!
z(vPdcm}eu188v^p%(i`8|8nW3B)}$9OQoaXt}E5O+()0fk=NQ(6afF<{`m4x*^5uJ
z=2x#?m4hgs+~|sCXm4+aatrYVo_f>WPe7$TtC%*i|4!ChRUaUcmc>3$+-?Nz0?q2o
zDxW9OA(oX(T#On825sc`^0F{OLPHG~`|{TtnR6re37-83eBVIj+Qf^%y_~k_s-<}4
zScP^fX`@2o=F0{trxf8wPTkZ=7%$le?NS{_YupV18tW*xHIBOX#QPUnLVr!wYlJGS
zLQ@|^YjXY{?`oCs*ee1~&onxAk&!><<qM3lbr!JcD@K%O6^u#EF<`n{nzI>|QX@B+
zT&#gLjg8Sl1@IuYW$B$UXc!|>2WU*Hx)8J=6fzkAj)NlL1;|`lkmIAyxFr1H!>t66
ztSOP%`Lz;|Q|3abdkn?S+)M{0ohc_iowaqq`ESH!t<qPmn8kSmIw0O}s`57Gi>ksj
zt@Hscxuy0ah?oeDVomv|v*VJO)9o<4NbX$0#r-|cQFXy}fetVPIUdy+YTF~`dGY6E
z=<(C_$zMr;{s5?yaWQJ-kaWcqX^)scyYQ=Rl!qve;^EPXCwBs_4s~L}`ytC^TXWi}
znOwT9F1Hs6dh~zfnwLJ#?byEZ%apXmW^Nb%`ukW;TU;yEQ#hh^(|9b(IR5@bhIV-Y
zs?nr2S+H?xZ4o7H<CfU>pytb)nT1MMINViC_BF8}=5N+SiDoG*tL59m8L(wQ<rk+i
zTl@aPKKLjMuy7ucLfSqcDooq;4yHe(LOLjvpM&78jiv*|rJe-1Jp!~;G=|CxF|t-~
zDjxf1+)fDVVt<saf~J+7yY#1~NXd@VD_NUB>NrK{l^D@gY}V&SN0m>_1DJDiwhWu|
z-dWKTKDHM3inyy~q$=>dFYn7x)(vjUSQ868E_57egpYi*iv>t`-Ilao9TTyg2hJ-}
zg=^m`&%>~|BQ<gVi}RVIK%gqml4=1pe}#{1#n{B`Y6RKCW_lfHBI6x9X5FeX6fr9c
z)&@ZUPKVow8sAtg%)BS-KJSiNwk93-uw7`?$?*Xy5I<R3<=nnQM_1n^cAe{aB<=0n
z(RL3iT#&H*UaZC^{Wz4oHewu{gMgK1<&6$_;_L_J{v{|{phk+y#|P8J_X-4gt<Lm7
z1ZBr0S!lnV;>lJlfPz&wQ?mup>_HbyDz)t6cF-}yexPieSYqD$j)*wE99_g-Rn|Gl
zd5c~ymI3(8FhDZ?4GQ-Z%Fmjf-*frousZf-9ni#Ly4f^uirCf_QjbY_0YICAOEBA-
zV5#|C&;psLoDpko;8c(hOK}?3tfm5UU}e@x@)vFHRwGj_HL6{TLqEB!{i#p&HAJ~L
zkZqD9PXs~M<L@0Sxv8G^s{yPv1b2&E*C(as0PQ94`uzv~BzNca;x?_VKmcU?17I76
z@`apAgQm#))6-dRVx~5+u6xDtJMuxjK}g3VJ_5oumWqPkavLOO!Vzu)f42@^gvu`W
zY#9~(RmNbEF|cg~0Hf2_e+OK7vCdckP^CIkW#fphfrQ(8;O+1vHi*1pF+2{}Btjo~
z0?Jee=zl=jn!V>!ZU)xNA2WPLTo&#+1Ma`9f@~)?=(&)jp`r1%tqW|l!dTq2s({I;
zdB+L44{R2r@|qZ4m{>WL%Y4B6C6B)IdUpm0oKdL^wfozDD*#?d5=m+45a4<Fppdxy
zEN+G^BLk2#ii03PBgijXsy4NOow!;M-&O6;XRZUzzoeU6RUkR1^zmkz@O#bq-bbJ2
zv@x$xnixG_XpP5|=5tt_J;0T~f1p^m<nSAGcf@r+A&0jmJQ>cZJ$5nPLiYGGs%DM(
zTfL#>@_G0-tCWDMKrjlQrg=fh+OFgTl_VjD6dJdN0}me)^@|Tz*%&$Bd$)9m=EaK_
z@yl0c@UMDV)gJXUxd_|M24lmj07FPINz9Lc%zflFFGxCEyWUsnm@5D%k?YejR{kVj
zhN5*5u>F%OI^EBDgET-ga^vLP_R=6f)Ef-8esV=mmto-VoV)~35i2{{X(AOVj5~Wr
zzFGeGXlFDg7xZoM6bk$kDt({pFEsu+WwkT2Kyt@@u-H5T)gFY3=$d1=x$Av6r?9lP
zVdTjayf(><1(bj|t*RujKl`a`?q~d4$7+Ve%-<Y5`j@b{p7)Jqv2;eWMXSpB{NA^{
zdF|RneSAC7X$%$24S=Nc##5`mSawy^OVD+jhY^_yLN{O;?^kCjruf^2rUC-{q}TR0
zbJ9CH9hyWHIEYTzTr)zJIQf;&w{PEk41IOU!1?A690VkCabsP1(4od5j!OYMzMUw%
z0m9)I$D@THi<oxYB+ETKd03kUTB*7c)=C2;Jpa{jtQTM=7g%NJNp+z4A};10(CARm
zVifZPc(ImpY+KOF<S%3_TGt69f7zkEOCDJP>^aKqPZvhWZ|2tQO9S>g4XQ&3L?W&0
z1LnC>;J|J;t6q@+;V~(z?gtdZ`-FQA2Sx6a0^uqVld@#d?YB8pZ7S&C2*4F)%YWBJ
zotk(QImO27JImj5H>TzFtl=H6DLKdJE{cNnMmqeCsb#_J+weilN?8ysEDpZwaFN7b
z9Ie&E#HYHW$(jfNR4Rida1^4Ab)i0}tbWzu`foOmL#Jza*9BdQ1Xk<Nhzz%B`DR$|
zc>&e*Pt7V+!ZYQ&KkQG$_|&YY59$rlCaX~__w40f{>3ZhSHX#+pxZvpEal?Ff@2zG
z$ZOb0)L&|qV5(=8+{gXc4g&%5($xd3<e~@+j)+~5?_&P;2A><wTC<xJ)l5Z3h8=tf
zoy&5F;b?y=H04o*KZ$J|Fc7FdC$zgSq|pWFt^a(-ZtoUzKqAomE9M#_s03YjG6&7N
z(#Ol?tc^}%pgQ1K>Kab-xo#|pjGn4l>Q5{`nBc&hZ{=vAI@WLt%m@(X5R=asT<C6$
zfK3gvwRHhmbX0x^)lvttXk4v0Ci$!KLVIHIuFJgb$nk6P0@S%x>;6P6+I@wWZ5_}B
zXstmbS^LSTsS7oT3S90CgHp^b&CL-OMePhlI{YnU=U<0{NCyT0tDCvC?BA_i{fHzv
z(kZt|VOLVTxu6Q7Qzb-~E`~L*excG|-t_vVo-ikJFQ8gtP#sj_#;flrxLbyb4pyiH
z)?Xtr(Kr}Z+a+#F0Z-H{{T4;g9x;jLB3MmF=3H<4-~{YZE=b%)Z!@W8nZW*RpOA9u
z*PTJ(NeKXYJT`eq{yG>NI*^CP&?q(wW4~t8UtpNT?(yu|vlQUKfI|TgSW;bGJ*VZD
z9F~(PoQ_UhXm<>ajL$9|RbN5J&~+Ne0GXY&hZ{xjUm-ECf}9nzd2db_Xn6Z|uwqm0
zDGvGjzW{1kLG;k_)Jpdud=<TD8kqxc>qbL$7-gq?(gInbP7Ex7w$QAsH?RMl*nbwR
z5!N`M?9e}yUZ_M3x<n;6f0)_X6@kXYCaG76>{Pdav$*E<7^4cAn%@!A0`@W##fKy+
z%?6=wT@bT6W^i=0**tdDX2+wXlWBp2#X_Sk1n`{RhQ5JlPzil71vv$s6N`>+wu<g-
ztzN17$(i#Y=A5GZhIXo2aCTrBqB2&={~E1HDfs>Sk5QH_8PY-aR?|tvYMa}%<9imQ
zE#?Hod7L6%0&%f%S62&Ayi_qYsN3%No1{5DfKYEa4%<gWxQf`86QlB}vX5s#?`}s^
ze2<-+rtQniUi!ps#10i^bu0>5%tom2<n>^xEQN7*oRATQNS+7Ib<%(5&<nu#*uWAG
z;Xgw23MOvrKmI4^dx)ZnCM&hu32W~5w%89<yE=xFFc^Hi?u5Xel`W|~Ia;}WH@K(V
z>2R&V5VU&{<y;QV1c}ZED70|6I}y|YQt!?)agNEL;s)(-t?;WW#+^M0MO9G_g`L4!
zwwddo3-w-kO<`%RI#p~r9PQbf6Bl*sUlqq92ci<OWhBG;cb@Ur+PWw~ccpCyaZ|jY
zRuK#G_Y|9R<r1V0IJ~DgXs$W*v1HIztXUjXl;_t|+NMcz?Y2xX2)dn9@3+j~hw4Nn
zdm%x;S$7wsGa$`Q{i09!09e66#lQTnu5v?<c-8aty*5r3L4=IrVZVIy<`M7%aS*Sb
z)sIyEwK{_P|CXweGpJnMNF>Q$YcC1CYhuz1iR86SoeeY{25GwKAsG}2_XERY8hC?5
z+EoQF5V`#VU0wztzEUhOka7ks3sWG>L#MC9KoL{{I1K?_L<&TXGVS9at&YB%J}U(4
zy`6~B4&wirLt%H|J3?Y(?>v6|SWa5_XwL*NXq}e^G{+hK>J}Ceke8QD$}IiA@}eT?
z1B~)eOMTEcX#?zDXVSDnFEM04#km~Rs29fLtzFQoZmj}1`5W+D#ckV!fzM?#>ta~9
zaD6jko^5i^whjp`07Z#%z(Y6KoNZUk(dNx9mKVj^{e()8h$|iSb7BF)IZXwj0<MjD
z_D$FBgq)BL6^zsWt3jX*{Ehd53mf^rbF{f@y*c}M#;6WtA3*!>t?51hfB3o4I3Nlc
ziYeL&p*OB#p?fV2md0pr@I6ztH8XcfRL$1Jc+fjH!HG4yXmWXbb{;?Os-VrJWGIzC
zMO-iP7qtkZHE3gw;xrH6;Qs>Lb`uZC0N&BWl~ltw2}b#=+ba4BH240B>47A$U06EL
zU;Cq+55WKoF!b#@FlKR(07sJrKdtvq!G(0CKfYD;@w$5(j>-GxUOi786teUS2w(@D
zRj!Nom7Lg3E;ZLtLUrg8GcoMC5g@BB1DC|jDWFCoT%bqpPf!8m3LYv0#Sf~30X;Yc
zXy~B3SK1TadWwPJ00X0cH%tos*B4LfjDgPN0=GRI_6d132*2Yp2YtK192mSB8)kAD
z61Udy8c;$2Utb(xNrq_h1+Pm~_cET@apGysVjmuio^WL(U7-?Sz!yHu(-2HNVL?r1
z8di~{;2D&+fOA90t#2{?<n=bln@1@tfFy2{eBCJ*Y)lR8)8V{L8`;16l%q2S>O!|6
z`X^s%jGD=9*>s?7z;?{wZv(WyJU}rni4K!N7?%hg?@lB^y+CC;Mq?ao8-Cv7wqb9Q
zHp+QTnSBtqgeHjxGy<jtHRyCDm{Y(ZkKVvYL6R)>pt5jdyK56hvBXTu1aT(%z3@a`
z2?bpb%zr8Z1n4Ne8KYtQ=U=OMwGr_EoROtn-Vv3bmI)oQER}kdwAZ#oceFH6e2Y%n
z#6#?XOBZk=M#RYJfL`r2l#>-$`+kiO<VZSL@Pw+7BuY!Yxe0f)%i6MHqj_E=Nzs4s
zV2uY7a)3p@`+bGr@c&`&y&tLm-~aJ*j)P+#TPlvo$P8H-#|jxqWhE=JLZNKVks=DA
z$Vw%ptfY(xsjTePFiZB{nV;*S=j;1VeBZw{&N+|!<Nmnr*LAyY*X_C<^R-Ob#_wls
zklv@T>Yd>5`wHJh7J2?1r2%M1A5w9%{HNI<*kDq2t(>-h$}Bz*vp9rMbF8fVfMtC#
zC@9+f%69sYrKr$GIogu$7DeS|_Z11Ri=ag~2@DBEu=(=f4a}q9)rZPUUuBSkdB$l7
z1lq?$sg=J8JU>BjrtHZ(Kem<s1RZwfxzWSdJvnFUXz<S4j2-VNXLLy9y+mcb3{pC^
zDx%MxK!xB#>2u5bhuTX{F>?ZlL|FFi6ERGcyY^sYs$&b<x!Z2qkKneBWcuet0GzZ5
zUb)|uf8qR>SYAme=G0M>t!R+k6p6%0>r56Hne?TvTz`R~c=YPxcOa3ZIos|>m#4I6
zd@Z#9OsfbIjPDj^GLv&s8_(X}g%+fYzj}V`{{>yzvv9)5US87wvn2x5*#VV}LxfYy
zD6lGn&U@L*-`Z$UJ~Kbg*<ZX^P{|jgMX1LvJL!1tQ~{cN68i`MlmF^>Cz?dc@7Gp)
zt{`ppa6XU8A6+@|C!B^g3o^I?*Np3UoGZDj>#Rk&PtMZ&7I3^;X2fzU30AhkPCA#F
zr-!toux}}E{1ZS_BSs!{_|*Ho1IH(s8Or9i<N6*qpQh-oMR#G6_cQdHT{A&V&^w6-
zGyvE!gOLY^o6!X|#c;KT(yPC-0#+u?=ITXpBkIK6;2kg-M_}uotXO6QOp5p~eCDM2
zGFlUNfw}%JDL=@n*!kYl`#7o*78aJT4mfdCfx03C%eZS8lz#q<CQkngB`Ay{D4y!k
zktP3vp0F;u*~LRL^%1x;=P25K1eyX(SN4ZP<n$9Qb-AW8JWdJSypsP4r3xK*2nCeG
z3MKAGnfrw(7V}6UEg2yjE`MQ^M)TNZh&2ct9lRdZ!;G^>Gf;hFJ-obmi($V(`6no-
zw{z<O5Ox!qaQmOXj`M{O;xGRj_dj=C2<=TmC@Gj^a;DIPsb_!kD(_pBfMuz_FTI0J
zu4@s(DSbv)b$EjXTGe(d9+qvxP?40g>RIiy!%R0hZ?&9clO%LvF%NO$jSc=wql#7f
z?of(a-lZtFtTvL&sycPdr9%snbZJEj$^Y_i;t5#7KJEW>T}0KrHmt9gQ;%M}bwDMc
zD-9{$PGC^?kDP1N5j_w^bf%5_sUlnOzUq$U)R}QwvR&=1hx?jRjKKRFclF!k<F|2?
zXmMTTz}3rRzIouIqB$fatV#==?;bv`+m@+L0lF<ZN|rDGN;1R@i0mT|?@;{n((46C
zDJ5O)BosnIcSDvgLGlHE3tXMnGg{h>U%){Q?CdJuJeUtX8i3rN>cl?SoP96ZIyQ{L
z(rO?arnKL@El}m2$heT~%(p!Oc3_QjbZN@|CbE=I(=Rzgr$#pG!otE6wWL+ouaF!0
zRe2ZxTMcP1>+ht~EZJMm)|sKPN1+b@RzFlzh801BYiB#qiVY0%eYQLEU2H@^pi8aj
zHc{}-LqOh0QC>Kf^qzx>N6||_`yItJH<g<&WNxC{n`@kT4Bl(~zD>zTFJ?i^t5LH0
za+65JjnBEHJZaDG^?}JGou1n&hHkRo(P04JC_nj)3;c(}5Ro0txt6_N3%o1_ggEig
zyIJ$lA1SiRyn~->Kw24dG0spyX}j}~Oc-Lqz=VRMXcElBNO|-Yrl*{Qkh%5PZ5fT$
zK>+YT(2^m-+#vOZTTHwX&^)S8_Vpki4`Jv$ngN%^fOoDYmcYn}XrrMv6gh<X>(b1%
zC;vnca|kt(FQ50HUnGrYUwCbQQ|8TOP@EWfSXNGWj%9auqeik<mZJ66FeXGg-4FjU
zHOco;Pfy&9hj}{YP*ftuL>NWfB5}m=DqP7lNLM<?E$?x;<NRCCK~-6*R$*owfeh|4
z6-CbG>aZWap7$L5O$s}9b<Q7hO~%aA?dSd-n?*sO{yeRx_v8$C2MjVF1X2h^G|G@V
zF+Tnfw8bE^a-3Ys2EM__9b8en%mM6{zAdzNOFOp_6#eEim6zKz7r*TH)CG;i_N#<t
zD)MEl;Hiwus<n_FB~bGQwdL{*eX*Cbt++{wrG)e$Zq}EWr<a-9L;d;;J(4DM6)ape
zbNQ#gBx4|c8QUhN?Nzu4cr3Ui2Y}y#lyNPL=fO`296qcEjpwqS1AX9?XZtLkJb8k#
z(_LmJlQ`S<r6S#>Z{&`N$7NG@BwAg@0iA)!20ukIFM@tU-^waYPiT4kX>8F;F}$Q;
zFrx^%<fk}Lf7d&-U{Z-y@rz22jpI)$12Tc5Fsblch%2%y+;hS;peZ4g<_kUD%lzvJ
zae!~9dJBAaO3wolNAkmwfSzK;RbF3&s0C_6xVBwc8-+eaJKp*vAF~cqg6uI@QjX|S
zP^3hsGmwd#-$215qNMCOpxmD54d`D=v=tL<waiO4W`;77p_$IfPF+{a<A!#<2%hoP
zWw7NGF%dXx&b0@ciFibQ8WvrT-#c(SqM4x;;IY12?BvPYPpb_~_7==-EkJmd-|^d&
z+#iddFDG&Mcct!@D-Eqx?N;8|CvyUb%(0b_HUcJtV;yuzB2fNe7QeLd?P6?$TS28D
zob(Kj2TS}UBARYoAOT?X@mB`dj*(Rww9FWsuiB?OKS5W2`}Pgvk>2^wB^qCWxJBC%
zFZ0j9iNqw|DY37Am&Q`n+MrYPx7b|IR>3N-6WFGce+g4Qli{Jb(?iu{iapl@%Mu5-
zAoC`BWh@brLJkB<At@|E*_`3L2(TWR@qt~615vrUWTuNSr-aq`7GpU=#xgiU{?j~{
zxv^UU^xsJzAD^~BDI#L!<Z9q-@Cv<A8G8f;U<;Y3W4l!$U{(6lm@8oJ*Dxp;Eg?fW
zwEl@Oq=T`1^&0_%;GS<&Em}isuK#4tbAxS0wMoGX>QM8xtS+{x3VA{~4Q<!97<<rs
zEjp*Bfuy;D{|aP@LJFCz=j7af2>ge(EK?he&g8uHg5XinDPsALIU=w^?0A?t&w@BA
zQYg*%9sA0^1))|hsG;%pr`b2aXu-6~K|PE3yq)f=0T6^w5S@%C4cnkRr3)o8r2Yji
zmE8xA(?c^v219DQ*yr8a05sdkZJ~@fGi!fss74=%=b`6@3>y)n!knhwzXz`aUm!@x
zSDWHKP)LF0W}}C&VoLR856%unAn<zrNgE<(B6Ly8wicyhbeHb95s4+BrPBp9&(|X;
z2GrRCYn84XE;USuc&1ttGWRP8THHs+u0g@C<CU3Y)vQdqX7zc*<4UWOV^ufmvBiTh
zi>GX<<#=Vc@(kQ`66LZ}2-JRBj`RW#pTff?YHCpY?tc0xkn82yS6(N**ZEiP<pX^<
zAzVUE6OI#h>m1PuMzNN6ZIQFUOVMPVhw7ttbn$nY7vKY@3p6uraD=!M#OT1#`5NlQ
zk93mBVbR=dZFMko>Es(uCT1d$5*RW*fN1Q^YWWkh#VtgrI902o$OO)|%rYw6w4M}r
zE*X9aP~thyh%(^E&A?P{ScMJmS+(DJD6W=#cG*L-)gxQITMgH_3C%gV=YdP_yHd+w
zgd#>4z6QK*OM~PAeSKEIO_x1>EHD4IS{}L~9K<vIzX%y5+tM=GAVZ%Hv?Xe!1g1fi
zT|;vS(p$k}+1r~7_xN?bRv**OhV{@$LrtEM0e@mIj13(H%i($Y=hL2BdRGmEH8QHK
zh5kkNE3@N@XA2N!o~r9w?*$#kbNlbk;jqc1Yz;0TvG({FdC5IKW-!>7bT@4F5X7wX
zqsk8T_`i!ylFxvDc;fOdV0C)pr%}5S2Dh$d3o+k_WDCCCo*Gw76`L;z=^=#m6}??&
zHUvczt^f?d&c1y*h8Xf7Ez+l+MQqz=w6Rjlq!yWtZ};JYiuiS}ELWvWh#*x5sSj#G
zD=uX*aT2C9xw=w1le?hv7>T7u<gT=dZxm!PN)AX4AoR=MEZtn5;J36PY4llmEl^?j
z_zIzh>g07d`-d{rF87sWb9F+H<8*}~Xm|qdb}YpWq!_=5IBT1r1AA5GOiaoE5mF%i
zw3XPztSo`sQBm<ARaTHmhVq89wI*zQn5OOTFLyf%Y}C$BXnATlH?q10W9!~JvVGue
z+l`$(naa$6BDrOcJqC<~$k>C&5DmooS>la_l$6$;{_{eEFl4F(hG{5O1(vw=8l6lO
z9`5|mp|E$pcad(8DmRT-#8+2=#Ffo*t8Ith0R$WIadEbnHW41}FTa-4k<(gTbCN;_
zwFs@4vDQaP;DZ_<*M8==@C&JWG~QrR6_!X6Z5{iC0&%-F;yN$=X@+4LLO=w`lS^QF
zf>S7d?huKnOEZ4bO58ZVDs?I;d$Iy*K7V(;Kq`~gU1s^}6-XKQ9BXa&fP6kQie7n3
zQ?MtdaKN|ON4O-3v_4X^0cyu#aT^3-Z8_*GordzrAUx><&1aycP>}5T;qRBPCaTcG
zs|+)45H5z54VQgXnghFa8YxOk+g796D}F%<vI>!(A$&~e^pRjlOL^%8goK)k9GlYE
zG@<)kRx|)A8K3h9pB=*$hnRt;G&WFwY0tO^kQP?=t3QyW2m`@{4;hW}&{5WR)(Xk%
zCj0@krCXc;?(xqFTa?wQ@<^gO4!mORh-_~tlh3=y00L47kTHNp)3E)ydMV82(u4^B
z4nXL1lz7^-rv@SE1XB8s_XWIcL?U)F9EM^ln?s6eJ&=*RHnA}=E&r>c;{}jZe?&u?
z9<lo>3=khaCMZO1f=u!-f-2>;vDk2!dW48ImwvL<LF7cyAo6kR(Cupp9J)@-A69?3
zEkb;K$gY8wr7w>n8q-XJE+dyC8ImZ#G**H}Y0j56sJaRJ=rK4y1ESf6>w?$u?J>xN
zw9K$>nWx=CzN0bQYd6ae^Sq}|L)hc_hmc=CYVVW+Q-dlAI5x#lEJ8cN&=IH3$DqGc
z3dQ!p;1QF@8kB?A6MLA6)lL}5*YA@ls2gEu&mW*?;-Gx156Xq09$xMfDDpC9ccZt;
zfVMz0f>BYVLYR;;5D9H?ac@XHRLCTeH%jYpX4&%e@tmAvS+HDnI6oHm5@mz}5i<&r
z1Oe`xB=bS!Fw&kF0{*N;+AjUHA5tf`XFwHu4i0P;6`)A~*Vzqjq=BjxV@jmB1z5Ti
zIxj3>bels51@v<B+16>+F+(qz_<@(&9|i;GRBp${%1?yvtk<cTj2usvRJy2HU5kOF
zlEaYs#kE==_8KSw8b$+q!J4=4`tQ5)DKuJ^<9#4bE|ve+;**&P4vVc=cnq<sEiEDi
zO8<e_(W*}{*)SSEnf)fPbCkCGNi|{^BwA<HVGwfs968JcEi*|rA>Y5M#zaP<0Gw`t
zre=8+CO&{G5kO`j>hX6KyPQ#*JJt{Gjkq_m!ujOs<HUKG&h*d&h-8!E?BsO&7m!?$
zr4DQ$X;XLwwC=?1;!v<<HHVOb&mKDvy#a3IYa4JC9vGBTf=-nzr!dk*7k8R?hH`ms
zaPTXp0>bE@e&50Sz}Y+Y-38w}H1?0E*<SASf+~2zq8rfMCQxko--=k;WF1qyy7JXC
zTd%>fOY@oPX{N!S1rjOlmbMLe0&*OJlcbYU$lyK-_h=SCQlnTD!f1GuoJWdcm#BW|
z`=5X7s=d%^`+ofnH5tR!)~$uehL9njCa+Az_o;<cIB!Tn?;OsGEaTKt55g4!G9BU>
zWWxwmQ#O_ni18&Z176y)B%0Afl?{+(6H?O)g>{JmHuN>2zb|PsP*A5nhTu12WL-zh
zQm(?fs_C3#h5#HLm$_=1Z{-pd45XYcQX3!RbT?$h^YHM@|8_*dc36->2<*LN(}{6;
znF*I{3DVZl34CYQ0)iN8R&!ALF3<cl1IE^IF{J80S;Zb<Ou(nWP`9tvDw!e%vAk*?
z1lOM22`sKy?q0s|zy6f@4{+e-Qah0$#h`zKMMn-SmxMm-=|+JH7}35dO%1km=TMSY
zjv+mSAW+?yyFJQ1SZH@+dHVYq;2?#Lxpvjh3{S&YB(wlht1+deKd5vT8RB?KWhHU^
zH^6EXH{}IwFeRvseBH}5vfJ67mq+Q9Al)?yDa)SRd3yf}T|@o=2{Pofk=7@I83tIj
zCo+Gh?Jtz~$?@^27bia--7axM`Al}{`J6-TS&|t#@7NPjG?1bf+a0&&DwbllM&+v-
zrd+=9!cjCPI*q%!qw|ge_sWh`2>U`dXV3~P<q%KFVwviV2+{~$8W*6vKr`c2wkO9R
z?PW~4=C6r7M}iSO!j>uEq(N(Lq*1U&B6C8?=X(A@6jHXZD)(s^**tap{0Ry)WZ8_@
z8Na|j@UXRg_vZ~>zDGlqrWR_A)NoUd2unz`+{p8C1uv(AHXM8`8jb*K7^?Hif)6p>
ze<20NCAYD+rEqlbq&maN_xii&n1Fc;6j{qlXT#{mIdQ&uvMr2r9%6@mvbM`+hJ7L$
z)C>X-dc4W;7<*fp_t@9V$v~MYzOnNtpAZBm6G0EIRTxi(UwL@o*9P@<U?UtqhSQi-
zg=}LK<%Iw*R<pJ3zcN?3mO;N?&cG)uEF0g^O_4>Q|2m~tz>E__=ecE(^^d#1%ol9y
zg&7mjM|1s07pz~`z`#I?A^kPtDXm!{#2s_x)&MhSLo&jprfHa-I|6e{Z>iI{ey{op
zXx$J_IeV1y9nkYB3nLW-|A}Ww2B=J*iH^md^PUZ$VwkJpP&|ggVN<G+7P6?v$lS;T
zU}z#a4ubH{EtBEK4xAo5-MA|mwp=OnjEf3}!S#fWPBe#LPj}ab?>Q*67`722MYxc9
zpRp~H$lZx&MIi4Xwe97EH+|Py(iFX>za6c8QWh}7je>KU1T04_k~a7m3^hqs$==jF
z=H!RS>ma_446h{12|@Xja@qF*%y4Z2X!6ImYBL>4JfX`3zfN%x3snstOv;AwnY<q&
zw`8{;_8e~W(Xe!DX2i?7|LDp(NUMaR3WwikBY~sD5J%YH3*{K4H!VAkXLplTfqkfX
z0nJ<wUq5+s4odDYh(?q1_C0<QsTT_bkE_>>Dc;833%=<vxVN3{@R}vHGk9){DFIFe
zLgVa%={{OoTJ^wvIRca#y;{u}6$3%LjSea5Z>e0`HH6*`xrCQ*0WTRLEozYKN#|JJ
z;0Z?~xnvp8OL24p*O8R&G8yXoP0%aQESb)YYGX&2--p4jG^laOw2`rQpOKNk1Nc&q
z^@BF)<JU<<?pF4At)4%2KB7Q=-i3lpU~L<8m&HrLd?1&OtlN6W!R@{$>_1!!mIn3F
z?t6%!(u96ypz<gETWHvM^7hAcaUPf_gDGY&cN-A~(5rLkI^r=o+H8d1zPPXQdj!I?
z?12qBM@VVXErd3xWsvhfL`Q}SUPGq(jjy!N)L_URLf{xDed6Q{lViq7lc9)al;QV_
z<~vm7wqVSU)b2ZH6_!0;7KioGNHQ8Jie>}qsoJLsyn6PfD2QutddRY38%V8c1R0t3
zVO;^xNtodflxC6g%*Rr@S}Lao7buISqevWiUS(D_azS4ptwjqII<@l1MEgSZ?}@(`
zq(+2|IW}aS#>NOECH*RPAY{{B1gOwA8^U5@xtS?L%1C%Nqg<WscQeJ0hZle%>~=P(
z+wjZ>*Oz9{6ZG0%yI?hXuS3BTA>Fd&-1)#A(O8vUQAeWMS1v?UCUTM;ItY=%)tjud
z-d7_cA~rAVPT(OjXOKq!_MAhdj2%C6q_JVCx$5-eH4EEMI6r-&x0hSHT_w2(SQ%DQ
zR|E4Ngv_LK-aZ^Cot%9OMldDAOQpDveWk@1?sdS4*n9}eWV*kD2!~@JbVP(vgB{ai
zKpy-6(VmHr9!T0qAfe;nL2)Ek!k|(_<)x*i*>t)1;Y`O>gTmYaDR2UfiiA&tX4DRm
zArs1m%(yG$(5>IRi(O|<H2|?K2I%iTq>d+pf`Uk~QM1S;ciF#z2zROE2~4#DO*O-Q
zIUiFd5T}!u@9ZA_4Q(CROgLSr3TGeiFJ6*y$;rYHzw*(UE2!nSkg>HtbYG1(-hL(_
zC@8olbg6aMY3^jT`^z8my$%M_+8SBH-bYT~?yx$`eKvB`;GzDge@2}!gYIF5EkoVJ
zhX(g_AIdn;i_q)_^t!u)4xv?LY}$Laa<gwUq_tMXY3^;0^=AKO?r>lg8HYz7ii<)3
zyr$*5%OZeK*#&C$!x=TYlN5Df=vgF+nSkL&&Y3>Y3d;(K@}AcFq-;qr{`k>2e3yvr
zIJigg!ls%BHc--W0PfWB8UF^v#dxK_YQi^tAfaUX`_()nad`4_G>V42ym{H*NsFL)
z2<5g5S+=FeqewcM0@SB+?K|$8C}bK`-N)uQ!b0|K6tgZDRw|nT!gK`X$LwN2#y^xe
zABPERk>JOQwQRxrbd+AbXaOXp?FGswEF8sbu%t+0fHrEX2xwQ!KtsP&{$ASElwRw=
zyg%+SX-4($HTW>_{dIOU3nnC3dYTjNkeLrf`5n41a<#vMs7w6ST_AZZqNP8DWObj=
ziYvmyM^x2O;ev_oYCG#syI!8tM`-9I>wuh70d*dG`=AAFFybKIN&>J+qf8B2dOk#&
z2#utyJbP;@*Fj~Btp@pk(`OwC2$NS@Y~OcLHYi&@pgi=T-wiFoV?2Q`&NwzY{V+xg
z;M76_4LZC^F#k8ZnUx(g3ebzKmCgqVSh|1!M)y+hfBHr?am%wOV7@lc^)?y<bjJ<h
z_P-EqcXzf^*iedB8V)DbgRlUpFYNn0>Yuc?bGvwwlafYPO64R(lf0m7^{1V17(dpw
zp9uvFaH}r2;VBhsJ?a(2Xb}!!L$+7U+)f{Z5(&A89;5xx;w%|uYu_9><iH0v78He2
z&iE2@P!x(My|ZHkO8g;VP|&K*9DT2i1@pmap@wDRE^`fs&%&tVeIVjvleAVY#8)vR
z)+3F>Ao4Xth4`Z>A6gI!2Vj3vk<=+bGJ5+mK#M<V-YpG;7xQvp4)YWk0iep*zEQ(>
zG5Bxq+*8@%Fjd3%vs;0gG?z@n?~J|s>5+ic_PTy^?(O`OLnuyf;dWcSTMG=YOJSDW
zy%#05A*^8eWM&2b#VnrMI6tn(GI%50E|&y9MHO4cHTa$^(k|Mu37sLM&hfU8_9`v^
zpyag5^nPvoh%PJ>oUoE4+KwNwr7SjBmI<xJJAK&vPq^v6vd*G3WI`gaHsa+TpjTe`
zSnr26R^&LEC8i6X2_ul=oaS*{ploInHYOs#>l#4CxPdpRdk?4$H$6~!-m=6)9As8K
z))-vqb$GrJ`PDLe|4o0Z8gfYSIdu5GAK+(B5PWC*9;~5s!(Qc=AGyI-`x7jP@QfME
zsRQ>W815wuU;U9fKcVnWwb}@l1cOv{UYh2)-b5cclWkypfrqen6V%waNLED*o&%SK
zNEWhSfNDIyPR#O^zzSr$BS|+=JpqO(vh7nUBkTydRXvrUfjnXYhKz%hT7QVo(+kg~
zSr#o(u8Tj^%hh<4?L5I_>jd2JGA<4u$s^MPS0bFmW=<msEjyJnkbDo^MnER%a2SJB
zUH;_0X^ASj#vz0(NPq*wMX_JrBR8Riuci+u+Gjxd-@e8=Ee==IBG6)u(iHd*Wemvl
z#@|{))Iw0P*MR?+b~|k}%bt00VBjv^;xJrYGvMMVj9(b@MH5hZ)yC!frYeW}S;&3_
zq6UFSxQGGn5QA6n8i^D$FRU)Q3>_dhjWQBd0<VMgBuX1)1#&8b3^vk%Dmd+N8O%ic
zGQbLraI^%>@~sdW3+C$c-ELdnYV-Gm#bXI1<<kImlYpl^3s~|QO*>L2OB~)aNfh%H
z9a~TdKS)x+g@FonJPZuI6N~;xXJ;U8UYHq3p?nhA*>xsxQbFwSr^tQes+|pe$b21`
zQP5I<KXiB7Oepp}ys<&N_X|`NI$6Xn6@Nz_#_h;z<P{J=S+kXfsll9QL}+D5EAXqb
zuyxOT%$e6g^mpw=F|1D?Bt;GY#KQVc@3~I6(g8A5h%5u=5kvywqfAl$F_bd(JQG$O
zy6X;kCLV@^SdWJXT91!?D6fb86<w{iyTuPF{p(y-4hbt^1*(!cC}#8=Lg8tVe_p6m
zIKOvJ1n?ML#)xm<!MfssBWXm+i{NQ#e5j|7>^)OQ{5rg~;2CuCb--}wO6zGDG$HHS
zrQ4#qIh|NWI1hFWr|sguUSHF^qL0g<T(e>1Rn1j74M)w*e_{|khu82;&g_c<es1t#
ztuXLqy2keV4#m|lliu!v!=vEfoTpV7OY_cxT01(LE}ZfSJX7#RN*FHwVM>&?8MLH4
zgBhx4eH@pQ!q!fK>5F0r&>Qc51hQ*^tCLZ9rakj|qJs=2F1q|(+Kq{c{{x6+T{_rL
z9KlIqnJ^>E$9K1uo>rmU5QAxh?b02ut7~B;C#s`*D53oIu8)cN5&m_-z4;PQRP_9M
z3}`MMem`*n`|mylgfa9t?`1N@^do-Qkon3j(;I8VS#U1Jrz#Zo_N$VBirs-!{=D_h
zdeGMTRYiE(Z`Y30qvNpkf2l8W(88xC_Ja*gZ2n!GjU;N}0A*W3h02L}uW=Uw>O?sd
zEuMiFjSg4#U)1Unr2FTXxsj8D%$FE<cnFd7!Re<bCQ9@6F;EXkBA2cf#jK7@;at#?
zqJlPSQG!!@-_qbG6}r=Lh!y&v219o%(-f|1NISLU>Y=B%7nl^$GN9Nk=gWZjOW;Gi
zQ(WxbbV1B({`sX=wevk_cy@QE``??%2Yh!-vcRImu(WK#_|4DXFnc$9ke}4t3HUh+
zBF2w|mv9JDxQN#5$UPztIu^y^2s#pT=sggs*E<%l>?{H&bD7~a)qfudN8v2)lu0p$
zhuA{xH>@5FTGY<pg)~QfnveB57lcJ}JMw`U1OY}t52|uszk~b4NrqEoo^V`n9m*w>
zbZjCfTKEm>=}>^0Ss-d%sFeC=*kF=U8HZfvVIIIUh_uK%RAQb^totwZz|qxJ!4B0Z
z*?q(U4$nKF>}L!>LlELg0<?@|Squ3chKoiLskDcHii?A^*is3Lf6JA0l#2i8Yd&N;
zNUOvAzU70%?m}`9W>_}=zFyNJ2w<wVS5yC>r64S;3)A7mb{NnkJy6F;GJU5yg-7f;
zl5`yN-goW+JnkkiQ-si#24x)5lF>)EB>Znz{7#@8h>)l_LJ=w8;BlbZK}Z@iwrYM4
zfrdov6CGQ~E+2%Tt+7(K$Rprrh*ie$wu_mg`?uN6;XSnEE-g|x(e+8Q8#9?rYY50^
zCfj*t7F<~JE}*kFTtNnH|2dZVTh$E-Lf1S<K;6>Bzo~>qvJM!M4t0*`5-N-6f7d&t
zfm+A|04lH^2=gZ<UYA1J#oWF3U=S}TiGgZZQ3W*-fB8qTp3TscXo-$?Y6C`0Oy|Pq
z^gL#=A0PO0Yu%!QhvFJ>2NbRl;f)c;-B@nX@d|*(-{h_9xo8xLZqULh{eENR??3RH
za6*{K5I821dw{F&7(mU_l6+A&|1HOa8vW@F5705IT*yFNMxn09n$!*O04IG&W_vgi
zk2b?X#(L?nFqJyA!(P8I8?P_%+KVkrKp3VaxCQeFfVE#kti9ZC$yKXb2BQ;mpj|Hn
zU%<^O#%A)3&9OD(0z!a9-_(hJ<lUlJw{-1)n~z~eG@7dzE+F_2<^C8A@dN5Vd(UFP
z%-_-WK#y>dyj&385u9rA1bx9?+#q1oafmITu{AJy!Dru{@1SY)!n28)OTCa_f(IM7
zh&UwwJN}f1hG@{{DiCz4qdH5#lc~R;4SxphXi=Yw%wC?=gN7C5Q@yr(HVR<wE{f$j
z8B5g7L^6VC4)HAH6h<;!h<4iBNiD)#d@_Xg)xNOjm!ZVx@U74;g`k@^wWI_`6t0Wk
zhk;7y8(bJ5HdEUx6zaNsnvB&q2#~JbJ63N3ANm9@%O)oeU35HzS3`zoNI*l`I}Mb$
zMUhtB^IJu{CqrfQ2B72@54ddoLbCXt--#sRG4mfaa$WB<)z?bH(YIrKKuGr6yJXxb
zZVQ6S5ARpt(0=%3gO~1rJhA~^0+X_K_aFgr9(b}X*4F1uz>2H>8risB>Vx`=-%u@~
zy~C$?i~9E$el0=<0$_wvE=S|E$S66<<Z=8CwH*nGcJMlpB*wKfko`uZh^i#Oh1%`X
z|KLX?$qZY#PyOQ%I8VKS3V%mPD1pohE@*zu19k@&&SJqpA7LD%$BoEqa~vAt0}d3x
zr;l7nPx{Zi3UCCLj}W%rdSJ_Ng^bUUr&okLJ&U04O3a0LV#U8K%InHiq%?G?=H5Ez
z?A3K@{VqC*((0FWO#gO1P6>z5{0OFuG-Q|~foN?)YbMA0n)QFC{)=1<JW&?(2ZpV8
zCkPtGLo?==rxx$j|3SRHUGljB565Ro*q=W{6XkIQaDKa(d<6<GG}rc6vpf}d9Q{Y_
zBkEj%^#!l7mYI^-H)?xdqrXuNp79#J>z!^~yBrYG<mVj!U6q9u9scXUQTuQ~UuXx(
zNS3d$24GJ7@0x_0cwJo$+vQvPM6lGUhcU1D3Y|Ky|MhQh{a)zz_OeyEWeksU?*6a7
zF6tuw!IUDW%PJB}KctP+Cy2hIdo9Nt$wK~ec<xToiAIhwg^Z&0qLq5)Z!1lv6H7$a
zhIo}%U9KyMee3o8UOnHPv<Fc7EabBqq~Cr=(9KP-%|xsT9v$;X1SgRS5f~m81{1Qs
zu5v({23=4OPq2`0evDTqt2bD(o;Q$b{ArowhjFz(q5dKkWnot4gn~R-H5PK0PPJzO
z2MxI^l-loe8}4h!)WxzL^3RbiEBamz?!y^+Zdiqw=PRs3D*&qq5DcqdD_^Jo{x9T-
zGtd82m}RG9!C2qBsB&nCXnV4?-Q>lHV+yYR{JMReMaW@{OEhUvP@DyZa;wfy!YT!L
zFbC>b=N`VUL-rmmggMZZ9yxy;u{XkN{NC@W{P%mX%|r~}P>Ei2I1y8+ya&QQ5J|sF
zC3z^0?>+ZO8rXoNHFb^1Z;@r;UVfLF5p{8X+A%A8C!Qw!-_>cB7Ykp(ht@IZnBO1#
z=%=xJ7Zk)QJgCF+itatrjsxhbyT7f`e0)Rfiaq~BgdwTLx;TNhrrLbKR6}&I8=Jq5
z7P9VLFd08&&chm%Yk=5;t~Yi|E+&vNJe*`UVjhdQlQ<%7-M!-wUmQ`A={<Npk|cge
zVK%z1+6exWh_%-sUGz7Ye=WVYmgaw(iT}TwS3wTg|2_HtGyDHL^90QQj~rfy3TAg}
zrR4HoyBln;uG~*cQaqzESg;p8XttGJFM-d^LPPZEMEO|9<Fbxogn(p_%QF|cnSVsW
z!rS50?}7Qn1tK_AW#g~O?$aBNsUH@iB_-osQE3W0v6rn^64=DeOr#x~<6V3I+>9hW
z`sZ>*|38=8aSYhvj6|bj-1U$9H?h(AF^}%*k8iI0%+l5%Jhl<Ko`HlJ2tj__q!tLr
zvL|evx8D_A{myMjwVZZbugIY_<%`O%KNq!O5xKbRknKktxp9w1C6nkDN8^8H_$#a&
z&&)3!=`3*6cu(H=v9ekGppj`H<H+{-xh3uIWmluElGS2;9&N{@dC!*Rc=g{2xW~WA
z)R4&YRKT}p%jEY&#PV@YQI7Ya{eo^)7r48=DvqV`m&p1_jVmyPBjL|)o#%ZLfo~`#
zdaC`5(5Z%u=AVu~f81THo7+5W)2^RY?WjHS<O!vsRQaI?O4PAD%3^^fI~=R_7A>vW
zZ10n9iPs_Q?tvd~Y?Mq3gg`~{;7zxKqHRM&n6L2Z&wEfoD>&hsC%EViT8O2?_jWJj
zQSY9e(s)nwzZ#N}7Qgu`dOCtab@y)_^WWel#$ftC(TQb;hLk@fzw;jXV7$mt_Dw0&
zc5E)dhD7n*Ycy`_Gp)tg*NB;^kb9xKihTZbHe!m4R?dt|p5eIa^GnM)XX5o$G=-m~
zc{%K=e=i9xu`ak^`7V6P=VM46cd`4kh(87Vp_75mYYoC8xyqC#C*S2-7xY#R|B$qr
z^iq`X^dF8|2*P1mg2mJp*3^nFZWr*)wrpQ{GFLk|C2chtI)BsZ>IDmzqgP3rhS~G&
zIh8gEima;LW;vz>50rnrzDjFzdSv}|)ySVI-g}{^wq6x<d$(vEV;s|8n7TAVXPehP
z)8ZtHEeMj;h)QI%j@628E!qv{_bi#=Jv&n&_Rw+srHgu<KS3D*XSHkJDzP+ZU#qZ~
z{Pg!)1OEP`1IL48$NT=4c{kp0e{_F}x74sjX(~5!Nf}VI*zT|M{%_7bD&(x8%3DhE
z3>Qzbq|CfB@WJD0nnH-ik%4o^-byr0X_2eP_yrrI3P$zr|M1OvcubeA-|*dwy@S)B
zDv<#x9x#4<)mN2q@tG9g=Iwt^34Xs{^r5S{x1s+oPtsNrJEL$>K##`0-rV=sa+}P4
z->+#s+~XT0<$lr7xWHF*s!wQpFs@)Q?Pb;2*QE0SE$J`VJ5A9mkC)sT9Cu^-O-gj=
z%7*W~pMJish{66KisY}_u9XCwob~%)WPI0uE>vR9_u%8$Sj*jWZGW{?3+HPKMpAuM
zAF$jP%__wdYTw-a^J%cI_G9-(WsTSO?q?q#TtxdKPi*jMednyN+Ej=)U(D$H;!L0P
zg-eldcGRY+HBhCgSM!a=1WhviCc_z0wtMP|u1uPl46p8gb}5=_C)Au~pf-_!UEPJ3
z{v1iab?<IdNZecRBw8Bu4c+C?ZMhhoP-?wnx;QOTU!Pv5gKI%=@n>wGZ1hXNU!1~I
zJoJsn<vreV81?p@G0foEa>a@l=_)f9vSK0&kk+2}a<Mn3_z&(c%g~R$vAcgGCTQhH
z;19RQznw02KB23)M5XZ$R?{ypE|>yxj-&&>Ra@(vxLSDP*n_UnU4H|{=|0g=o|B6Q
zh(?(Y5~!m3<NNe*U4Ad$b%p%-OQK78m%xuT9{xUj@XzFN584;{8U$3#iz){P4T5W-
z{R>+h+8kBODwrsHa8SR<#!vl4r!o%}S&O&Hu1juw;eq;q0JZN-w=ZL`UJqRh6A`^R
zuho;VPR8R<i8dYL7!h@xLXG9}g2BVC5Gfk8kksj*wVw(++PxG}@pv4Gf1S?i*iW-y
z4aT|SFXS0pW5>lfIqKcmwMtW5ZVV*eml@St^4*}Z5*EpcCZ*%KDt@sxq=_H3cZ<jC
z_OA&%wQ1gS6?b+{Ebbt;&w;oKW|@<l_J0Zz<#_4e9<f!a8cqHDqmh>nkH4N>Cto;n
zO8U@M7L4)r)s5@%BxNc4AB;LUEC1T=j_GD%>p2z-m9}T-wgHZi#ypVcEbhb{b-lmB
zF(tSB@9oVBH0zk<ExccrCV!6g?v6r9%P2i&;+oi8K;4-l-=7j?GZPViYLcEV5v0DY
zuSn9Mt5hE@_e!#0ZZ8;sDFkZY4JUGq;PtN((>co7>Pd?wgF}|TA64yKZ!V9~K4gE(
z+A=%ben^^zC17iD+fwoGT$Raq<1Zyau9yfGsrgj1yfd%$n8vSYbB?yYTI67hZ#wdT
z@tpRKYJkD#m7fmjI=LpSt!xxuTlGz+v~nO+B^8yNW%RmzLVKBMfa!hc_m@^_YxR_k
zl-y-35+eQ&ja(g%@%*aw+%MNO+O0OBkgwgHom)ETdGd409cn((i1-v|KECwKA5jzI
zlcr^sEv@$}{xseGy<4BnK2=Pofa5Q<LO*RP5Mgy-x3g6$obpwi|2elIYBd*kK={;S
z%%a1fTz2G++W_aL`eoaag3WXO#I(cJY*%My!h9VES1%k1nfYrnAN6GQ%#R};y^=q3
zFMh-E$B1L~!a~l?jqu&){2lZbEm6>i_U817eG|QU@7pxfwLJZNF+sDlM|e+r$$V1X
zcCQsw=Z(oxs4SJUT96d65F1y`H40e2vRutu6xezF0_#GkOpenp!Q-I=i-U6$^o-ek
z+3aeK!0$DiZF4@Zj(Q-uPtFYYg=~|wE4=AD3xSX^XN~awLodty#<CbDG^gIM<X(1`
zHBaHR2M&%yb`}wub;4y%0Bg`};Ta_Si19!{)yLzVBM+p*Q5sUyKi{3H)AwjA$U9qJ
zbt{Kk&H-O>)^}_96(jEt$1i@B8M62J`LH2_i>cEef1Lj#dgJDdwl-bPO5d~ir6)tl
zf3y3|cQZ_<(ix)7Cbk>Yj+~{};JI)$xXfWm{IY4_(}dm62~>~mHR^14UtemmKiH(^
z((m((bl`TxCyn`@)B8~<MMFI;^|QF-W4t}@Jf_a){v>j-OPo{AI?eo%-TlJmo68rL
z-q^OzuRRJCx%ir*Xr|7lD_}13!)LSXm*%Y!WkQ9jgNwBR+a84F>4m4EIon@6-e&J=
zm)vmb_iRrj1+%mqR9p4M``pa05c?iv8ra3X9%)H8CVBmK<GFkKb-M!<u9dULH7>EA
zv7R*#jEFbKp(O)Noz$-91PiUUL|mWiye+#Bh|Ruk-(o>Mc$#PKs8|=Bm9^GMYa7)O
z>#unsuDtpM7t1&`hPoOO!on--hr<+BxaOoh2>zbJWQotX0fyf*H?5@H#>Cn;{qGni
z*_=1s7QGlnwS9Lq*qa5zC0X*)m;JZhgX!;C&SmlWRl75;Mhf{~=;FR|zVhuf{K~^S
zR#>ODR^gpzUw@t^{<BDR-M*d{i!VWSI~S&|CGybvJ?0rzEm>&mA~Sv+VUVhu?={r1
zQ}f*2YFcya+6j#=-@O>Gxl*!z?Af0u`V^WA-<wiZ)*a1ko~=-n`rc_x-oM1wSTGPB
zb+x(X;urlpuj;yacIrHna>{1H6Z1{=F5XyksSJ=m>S0!~d8vkr`{X>&wDIrNqtBM=
z?}<KVJF{ln!})j;l6dAhwJ%cP-5-}CcqHNvU<mi+#Vm7{MbEF4Q*68nI~*nGt2mWr
z#zZ#17H&%YWQ#QTv)q5st#bdU+erA4b1OgMmiQLaY+K~)dt=pkOG0*N6-xsPdRh}r
z9}Dn*ZZ3|f0<<s_96c&SLzZ$Lu2Y#;(-_K8pBecP_$nfY*^hgGGIUynAX(XOb}43)
zD3WeQ#a2yh_X+as#ah(vf7cnEN}}Bjt_jOm-+hNpWjMn~F*a15*KAg{_$8{t-~vSk
zedN`sn%c<oVn%&3R*^(&1<Iba?x|jrg)ZKEml@57iJw0m{h%%R*KYL*>E%Os^>XT?
z7dhVI*tE#nBy$?F=fIA6_N#f7)#QiY>e<ZbA~|(@uN@}hTfI*f97SLInrB<|qJP#u
zXKQF?G;U=r`t%&_ua1ZW(>cbDgzU-#@fK@=r4LoRu8pRuw}z~q=WVFlw(h0N85pc|
zD>(A1%JlWs)axZN!zvyML+t)lKXDdPf0|H9t;P%PYG>Cr`$u*uE7Wu=sN<(|$OWZG
zR*%~9qdT8d478>4yrS3*u74iC*xz58e-`9lEhbtzxnYJj1(uSNmxx7#f`~JP^Ot>p
z|4tFON=Gwq@^EumYdW{rs{>V8=))sF_}F3n@;cTKm2z=ZQtn&qOY_*s%4E}Ddn}Y?
zB~66{ur;@%P<0FQrz8&Wkr;NzIh7`EW$bTnNJ(PgLp@Vnq!Zf>%9s1#qT(NS?R4jL
zVJ7m+*#6XFyK`R-m#91yb2t}rl)r%)V~jPJl{OINH}GFt6Jw`ZvSg26&bYJ^e=aL|
zv3=`iymOcL!M8=9t(l@^&DRG8H(WODrexy#et5rRxS(=j|NhFSi}FOWJvQ*Z+#J(U
zn~NL!>G-5_YedUK<y8Ypl-n!)jnz^fl<NqI!6rJE<&Wl}$6m*ztB$xoT-it@y)6>K
zUL<fmB7ABImhA4mWjcBwF{u9PErn)}FqL!Jmn{Ser0Kn8Cjz=%1;9riI6Jd2E7@Qe
zJHYDGcc7YwzN~;JWQEdD%3wxD&h|poTF#{<p+7C0JTm7nI{}BW6|)x2pVbR9=Zx1C
zD8Kc%{!I*!D=8hFto;!<S}KBe^R1{g>-V{<EMIZPT)M`;vTHrVx^B^#2F->4(DNdC
zF<wvlfy_5ez7U%wDzD8qvB{d~&{QjbK_)sMdQDR!)^Q3!UvBQgL9sKH@v;FkUw3+C
zfBgx5!-|1y$aw6*C(k|!j|}FyiawhMM!P3i4^&pAPL~gUw%#}Md+odapUl?@)!qYt
zwdRjsr)8^icelRMvb4^qS;iD_C!XCVWbC?S<KXoh+$9-4=WcNyeDLN;@PJQ-%;3F|
z;E^{6kI}fs3`;JH%By;n*dFj^eM3!cvD=-zHWGM@=_jSq1I{`h!yc&~Yt_Cn`INPe
z)2eYvmu5e?U}{xyx<8z{zb&+gQiy%zytqxre|+ld=K+a3JUUfv7W;1RIIHEV-O%!L
zwLSifU-9_(0Y>Lwlh*blk4Dsb{JDZOBCxfmbzfwCZ0oG-DSb8FU*7pLnyF1ux&6(}
z#M4<PynfxpM{4UNTiJX}uPtY2{TLoB`-hpEB|vm&nk`PHf6l)}iT-0%f8A4u?35<8
zh!3M`PpQ~mEOcoe(HePJWco~s7bwh#36J_t0j!2bf)D0_OhKQqP8F%}rN@hqu6haH
zoa7p|8#uO>Bw?z>lb;7>OiaH|jY{@5lf-A&zn5kT4$df?<p`BN<Bw&#_wkuz?~U=+
zXfrx<aok7Nx9QTd^{)HGChznPB=J)-O*B)Q+tPn{B>YSa+k1nZnyRqwea=nCbQwB*
zgSZ)wAcHk_+TCA`KMvmc{^*Eohur4?k241Qsg9Y~*mjPv;c2*RrG}DnLMktL+s$BK
zG1Naz=hlB){x+@KeuIaOuh6vz<Ny0v#sf!uZd%O2oinD*X$90yQnBW`3e<~+qP;oa
zZs#qJFgCH9|Mfg$s%d*jy)-lERl^NR4%h27x4gS^|8BHp-&b>q-Vr!;cWUPIh~1x;
zQx1<W@?mXVWPLx%Hi!SR7*dsdWq$GlkJ>p&RieSn&ktt|!|N_w9Q+|ZIlohTW@IR%
zGHTr-SN?SOWY5{wcY4(-L($)Df6FB9e(t(fGk)mxfMipzt3FS~qvUmUiBFmVgM+#j
z{FgZjEAQ5=;9M7lR^IRbdzm`8qF?A~E>~^#(E5YQZW-mAD*Lc60$3Uy+*W4a=en1-
z_wjv>O8lVVl~YyN80J_%aprk3wch*K6IqrR;eMB&p?fgt8xNz}o-W`JLSXpC-RV$G
zj6kt4KBdq49KO@o`8sXqa*yb|ZSE_%PUAzGN?UWsnx;C64@Ti%oSiZ?6eI`;R2T#X
zZ{|C$q#xiz)$(619@e8ftRyC7ce7-(cgA(Q!lXcycM8UOD}2(=T<2Oh*RQk=voE*0
zd*nwgTSkGojK<{ZWf6JhAg8v&YU8RWcTV?AHC1hy7Tjm%4Epuz#<G)no_*_$Yt|F4
zt_S7h{0?$-*<z_XqVUPzJmSQ5d>&Z#KH@*eTV+--_fE8c^v5RnIlH!0Vb0H2j&V9i
z*luc0zU?+Y^dR8wt+#LEnR?RHg5K!bUDI&Q8A<b?iAUGz1lNfip5UR|ms|2$?rtKU
z^aRVZzwl&FtEqpkeO352@9eWCGf(r3k3HI5)CJ+BUKP>&yPMog_MFl5B+8=|Uc+`w
z1rB}=w(k9ZqkC^CD`Xl6nu(D4)|)v+&TAyu9P=i7td(zioDN*e8Z43+N!aa}GUtJy
zuvj>#*}QB{R*`(dK_dJtU8ZjOwXe}hI4?bH@UTNeals1D&Ud?mY=3*q)VwQ1ogQ6#
z*OO#x;L*1GhcszDD6fgjJUXs8zh5-a&9UL-&c}JiD)9rJYYX+LgSUeRT^@N)r%Q*Y
zoG&ed4A;3oN+ecABC+}&9<zSov1-WD=Ob1*$|5HxFRcno7Be*JvY%{EiSjFZ)&4u%
ze@Zf>vyM@SxuPv%n1^o|hyHpYS<Nn!))Q+R?vEZgmuj}1^%M0ysLnffvUPQro)6WJ
zTRuw|Ys%S({4~Gq<h@dX+vb#tiQv*exvTSDT}eOZ_`2g_O){rf$ZVQd<syrg4}+G=
zl@=Gm9UHf<?vJHbcN<-IADwD_^@8Fm|LrRWHqc+UxyCX-)hPs@oS+-JM~Y-W(T79h
z7q@x+^{S}bdLK&^-io0PAmX`k50ld(G>XHCxk?(|YVAAPEv9{<qcU-|ZO!T-DJ53B
zUtSLo;{y|98jE9{6oT8%FZMnPbq!|w!WDirwTrJo7rjJ3;29pOQ(E=jntq2N`JKl0
z3S_-69}UMO;BgUgN=&adPHhN@bRKQ1^{R+3BINj)br;S?-ojTq@=SG{6&Gy%6dC^c
z*f(9>=-U)wkyP~IV;Q~~jK{L*w%2&g?F&ES%?E6kh6JzAQ5B9A+M*v{y}!sslHPxE
z#_3iulWlkPrw`rb1s*Mi=qNpI4>ynfSOJ@z!ZEHqTMyanR32Hws!TsBJ~B<tc5!<H
zqj9!TR%j(O1l!y|Q>8wcT$xRZ_*wO0Bg5fYOmo{%x%;xBxnE63lSDEqhDRZ*|LRQL
zQ9VsQ284@|b#nn&e88QN3?Se8f*5U~0%^!%q(ZnvIZxRJANKO+h1PLB?10!|Q8TG?
zSh|O&TYWoDy4(6RB`}$f-C;REZOzrKeM+}SZ)MnUMWUvWk@Z5ag(v0Z<U$;J_?+%N
zI`ghi`d{Bug*|Q!TMLV17rRnj&Y#O7x6we;xBg?$%8c{srpmV|doqo5!#-Eq1R0J1
z{N=5o`w1NXVW|qr3PA)*-PyxZt->??V11o8GMvS6H&KTb#Eu&N($~b<R(9z7TU@W&
zw5Sv;+OM4aPLmm<E3g$Q9W-HH`N7mcotfsQfB+jGivN*OiE>U;sr$*-Wf9lL(j`^1
z?;Doe9@vpt!#HBRGWYTGa=kUUtfu?GC^e6UmJ4;$>sdq%o0zd3_C|oQm|gmfVAhIw
zJ9nm6n|_ge*jX17GmmpUWkzH#ijzL|FIMy(+|Rya#)RJFp3lf$8L?hz#U(zR@$fTH
za}w^nGI!k~hd$c&3YILSuxfq6mXfKJ9!GCxTg_r*X|t61E+{4uPXv<E-?80Q<DG|j
zR?pK@hx2qa{VM|xW9`4c`7r37^)+x#hM@*nr3saXH57r+61V+G&OxJ;`8}}=*Fj0E
zJGO+rM|vwCva}LV!m@Q~TL37I*jsq6XLsJ%9-whs=V0AD&tq&Hr*S%Z{-W{acaL^b
zM#==LrhdDbZt$qA+R~ex_x9KIALn1A{KCui#x6A0w^0Jz%W*!El-Bx;JrDVtN|zlW
z!oL>x<qKo?r0X*|-3xK=ucL>%cSePlgTog+N|GM^*gW=7X1KF392Jr<)5UdnTTYpK
z=$HfNSCwehlpMClu{V!3YVL1t5}YXf$c)JjHSRj3ar1A#hVLS?kNL~F`EZotFZwYD
z9D2F)VVy%%QP%GD4xeB;O;ba<Hj!_0qA9Jn@TdipClUYRgj&Q~^;f>suEA&x<x|e`
z#{0W}-1Qs&JYQuu2mbJLJGs0-Qs-39J&HEo=P_8Sh&E2U5S`>q4pu7jb`3nY)8zyY
zP+?t=*RcK6Supv6|HaR(nGnm<F-0$q=j-{hhdGxCDwGG^XX9ZWOJfn|!$-%o`OoN9
zQ9M^=lI4iQ6wd2-2rBdADkw=5k=)@QCbMuCON`qC&*AFxeU<a3h`a&s0@fy1T*S=_
zYT{VSbCuFlU+M}v`4pF5!S4QbKM<SQl&`IeEB(k4gjWmmJ<iz9(KRZTTuM9X7)IwR
zT!7ZZl^W`<(o}yKmYRv!h_0b%Jg@s=UP8f12upOZ{Ux27!iSx&Evj4l6h_Z_I4jj;
zC`DmugP6R#q5Mtj{jL7_LX&XOsUz=67PD6emoRJTe&-co=8+hN_(XwnK_ER^J$zOj
z$6I)pOKL%R2E&3&O)-8jx>~f_itELiDyO|F=Eb26LQd@S#F5SJcUOGp`t{+C32zp;
z|Jp#`n|%iAY$U0|rnn9FKb&ucbt+6sT@+QsryYN<?Bn1*=$TVMLq9f0k$TIFSwh!+
zM8iVp+os(h{Vjg%^h(;J;i6l<ygiavt{KoR>{U?22QbzNP(jf;0RUwExf5G|;T;*b
z)8?IaXB2j~C5!)vSY8dth0dqc=AG#0=3j_~XSKM#7+AasZ|1WdUBB8~?iMX8eB9EA
z`^9m9C558#hfY)7)FA}X2<x3c7i4^XnO~re!aaVKEnD!Tiq8vE8&S*tJN5>Onz*6k
zAH#Z1KW65;(h#B1;y(A3&TqXHkGXgDE+?NYn_#J+mxjVcn^?c>O4A>u7P%i?5B5w&
zruLurkNVbIWgZ=7P#tKIXD#kueDQZ-3A;ES>6-G#+g#Ep&T8vii-&<L8(hQ8c)=Qx
zTX>QkezwVQ_T1xpeQc5*j<quukGDQost@Bl>G16LXqC_58`-E-XP(y(#HE&nD@*|`
z>yM$;Rq<+i0SBA-4`bsJD5_i{e{xY!5_Bzey~1Pou*NM$U-?eKJC21;vo&pA!lC^t
zsx~H^qv@;P@)52-cF<ACw-nz$d2v)N?mq4`ee{y*(f}P7(`e;xS;gSB@QjNaM>OR$
zrIX8Bs!j~^v}oEsk{+6V60fYOu|1IP9<c!!Wq#5y)@|`=qo(gkl|L%*hV>u?CO7Yr
zvN}A*_k*!c1kIaXZ3S21M!x<_M-JW(V0;|~K=`0X{6~K5zQn)RJyCB|iN$!ooqZEf
z$x4rAwx*Z7GwaGYRh8^ycL1e-O^D<A!StV-?!5Bp;i$zdVk^4l*j4xUghb=kP3yJ{
z8T0qwYEOM{dia;&Q?s#lVR(YMPx$Z;#%lu^P3#-2xv!_~<qa4d1>A+O#=Sh;ZR3SQ
zfBf#!Hgn~z!zp{Wo2?AQz7{sLrZ|ktT;bX+#`KE?n5JE&DCX-DC+PW%H}8*ie|+_>
zGHR!<<ocNzBUd?Qb;`@s)!`45)33bN3{9KP$@4?;k>*wITI+Lbou+yq8nMQ6)|yeC
z(!7jcgocV4CzkGMHfv?uhG!8{tieV%Q*xAfsF`;iqmvI8)_C#6?{+%OIL-b^R2lu;
zqLl9>tRM3G7q#2#fhEBRS4&=<U-7lB6gO*luZr1`W9E%#y2JPqvySnR)_qpkHC?eD
z%Y|1W0uReopNQ+CYIxDlaas7HvL-RArt`Ce%-MlE6E_AsBBF#j{kEv^os1q2L_`Wn
zVTThN9@*tPW{8Nf2876{zbB-6dxS!UQaFyDpk7lGGZ_A0RdkZle<DwS=|IWUvyr1!
zhL+F^u;}u}9jV~)BUK5tbDFsDBQ%sVm5q&kv#+CzF4e}av{pOvy2vjwHoCG#;aQdH
zK7}#<;j^XJ{c1@u^DzbgSm}M=P9h`ggzQ<;im=8t`@2l7Cxj!1&ka3D<CyuZImGw#
z%)SZY52JnA33f_955A;VvUR+h)%Z9u2TllvZpy)YNn^+{x-hwv+G*Ti?AoDi4<T%3
z^_9k)&{LBtxR|<Vyi{44?b&YMVQO`rOhIe4JI`&k9Fux&CLi|(|BYskZ?O^dfB3W2
z_l5N-p?f}bF7}d6syg~OLRIe)zf$UCk4?fZUsuWi9%D+G#6<Xd=YDpROR#W<is;A4
z>ht?MoG-5xUM;v*v%9lD^P@Y_j;(RCC!AW;{7JG;vg#M>W6f&MdtWP?XrLM{4fmsI
z97(uf>@rKq%PZ##w|rSOaU=EKk4?Iaip2VdRl>ptaj|YphgtT{X34h1#wCrO^2%5)
z*2ha^g}X-$OEY!r!gOiSR<EYgPfZm{e_I=Pafp*pV*Ps2jB?_Yv7F-VM11uJ&ZT|W
z;a{0&eQA}SXGTabJ=?C}!}ir*-8}yLMB9&_x0m1S=bQgqueW$}1LDJ-U*C;z#I^3l
zN$TR+y1c6K8~#pl%Nu`~BF^_0MJVmB_?19l!3884>)kZ8)*boIRk3(xD!qi;jQ8sn
z%jCYy%f+BnI{3KKradG>P5V#JxU$v$8Aj^`7u3C5?+Qa){a3b?rcH$B6p!-_Eg#S}
z9}hlK!RK{1d{N_Mek;(E2~%!fW1x+SFp?g_b5otPDedAutRlyWqjN~Q5{_a?9d-;a
zTE3s-L`7SSUS+%b^~yo1KU{ukEFyd;X;c<b0CRZOsd1Y|O`5+@R-OJ>>8^dpwW#1Q
z@n`W07vkhg-b7xu*?qP3E+k>#03qL^;*$}M)c^K;ZL-^lsEL#?)nmItyM-Ic8%ByR
z<(zP@DJCNBlI&Mz!m#awAC6x*U!D5@*m}#Lx|*g76bN$ggF|rFU`deR?oP1a1PktN
z!QCaeOK>N+ySqbhch|eg^VYppx4wT!A+z^Pcdyl}*EqF%uAcfqqzg2JzZa)ACtBYL
zbgId$YOot@T~p?Ts5SL7TWQZzdMDcjwPg`6J&-PsjDd@JO+qoasv+WHpNcO}O{AlX
z20cWg_PqnWi<_%E{M*$;SP<6b1Q@CvbUNfTSc^BzS7C<$@AEMz3e4dP!}m|`NhZ*4
zm4eC_u?g&T{Wg<7^zm3vHuu|)wePjN7d(ti;N~0xf#_Mmy!ROC;Ic_-I=e?Dr$QqL
zq1_;(t?E(~N9mZjgnz)%ex})@K{#8cHpB(&9h2jFsoGn}S!vPr76g3=GfKaYK?U<#
zuX@KI%0Q)6%PF}84>oG`O>E#eOU@pee@r+H0Q1j_&@89wmzdjdYx{Y7SZyNFddtI>
zD;JUXW39UrkHh)(cyFL=Ts7C$AKvipNk+VGh=6cK6gUJS+ORMVx+buSZiGuSt?RUJ
zOV(swr18QiUGi_`1HOQs!NKz_nwe)z8yh3mCa0dq9gx|`A=Pp7R>G62^{j8DX&jij
z_j|k1JnF06N>ikJzAuVL%<}307%E7Nm=3;?(5w6uf021mi^&h-z}F83&x~ETMl94i
z09%NVJs5a)=+EKUEv4NJCaC96<qE4DU#FL$v_vR?4OHRz7Xhm`uf-BwG}`hwC5qc|
zyK!I0)~~&enEfCXgXR!u5OYF2{NBR_<39f7aR`CO_mltZn@Fy#7=shr0YUl*%VAG#
zLe41&K^g-74N|=KS-RHQ^#axVn}BMk-CYWX+=DMB(4h)9MZ|(!wR><egw)qOZn(-u
zaK}>RCsdC7AdvJb^FtDW^mFT~Ljm1PcB{kFa7-dVyKGl+zt?ux1#v@1K&o3Jm5NOv
z--~K?R#h7;(2s@mo6}aOa48U<fC30EeAy;ROh}EoOdacFyoG3Lkfb(|@I2om%I|)p
zuWOS<i|f~mO-9ffP(v@PsVI%U;2_>(Z<{TkQa&!2w&)r=vLtoY-auhR5;NAwd-{r8
zbCH7KRUB1szC$WfN5|@RM`plb`DVN4x$?cSpiD<c-Po&#y_>F$mZHAzLpHp5&copq
z3{i0~Lk66*$=(a<=95k4Kkp(#P9h=pBE~tv3o4YRvgmw+rK&G<(B@iQduB7eSBS>6
z8eaG(5e9_E)&!`L`J~WC3A#aZ5e>)lapX0iFyYh<Q4k4jrUWBW{WFYGYur7tJ6bLT
z8z~hssdrtQt4t5J%7DKf!^CdWz~t-14&CULk*XlXc4dY)BslIGgrf^RyAR^~n-9%y
znEf2e1I>*?nl<|qqz6gKg+!-|xk(iO7Pn;DP#GJltRaD}z!AcgSHmYig6fpUQpN5|
zQMDRpIX#!{xTveFGIu%&MLBxXqPFP$f<BuCDuS2$S!<VIGNyr>@ONdea3hBt{s)JI
zg&;|?@%y3l89znHh<TxPk4RxK5vI)odAz_7UnRF1S!ry{_8Xr=OX}VHv;Q8JoEHpb
zsb5l$h{;5j#w$07y1`%!f}}tR@b}<2e`w8GQG<uv{2aNJ-){q#J?N(=@abo|9W~5e
zl;DcgCqIZqeh`OnJ-^~=8h$<Wr_c%>62;Ev6EaFb>1`ZS7cj|#=g$^)6Hu<u=JKtu
zT$l_xYz_iKq>PZgoR^Db*=rzBMOz3%1&qeg==JQ_{;?x*3k11YgsVKT?fvbJPH7mC
z)Vp!w05o#l{3Oz86oo&qKP^3<FRyLmC`dAGcQMI6eGZ6suX*3pvZE?vWl;fKU~ErO
z0J2d{#IJmScaU*+LmCe%*b6hezPOGw*<tzvlfuKRUMR3|^>JE<#*V#st81w3PIkTS
zogpJ;+e)jal&y|?Z^%0gB+n>tcM-kfLx{YLq|vrw>hgUQmQZTNCh-r0O_iB$yB-Kz
zsb{$55VgJ#T2v?lb*PJia)bqYK}yRa>%Q=xB0)R}`3tyI?9N$#Yf$ho;Z;av!3YQm
z96>OlkI0m2!(6fNIETm<klBPP)?d$?3?DwN!kE`=e(0?_4Uyco-yn<aMn>@Ay<Cel
zF#rSUCee_B4qCQ9%ZH=UQ4R~EFV3(1SAn$PIuiDi@SAow2pY1@g7=9dq6rP^(Q7HE
zOOGs$c$QXt-5A)EZtQ{(X1cMOxGHJPA-^t1snO7&g7D2_5XnW)-19VnhHOBTpZD{9
zS1d%Z9~$xVS*!2E^hcpZsh%dQI-JX=1wR7*cUdJ>;jAbs8>Op+Tdt*7vY_>RG<3{d
zh&U42j@HZ$Z&l^Hnct&I4_q6;23A+M69xjidU>Jaq*>@}T=aLpz(Iy%2r(A6yLu0f
zj~|lJF%t%YeeWaIj8YDXCg(dP>uis=N_m-7MZf%wOV2-^zDog`DL}!|q_+azZt)@b
zAdB~~HB@D@kGj}JJW#npp&{6Q_!wZ;CkfK!-g7s5ckEyl6O7LTzHs69LJt;}0?jWi
zmv`ncsM;Y0fua(?1-G#+&G!Vr-X{pbWg;D&zs>(3#MX6LS}h4}mI&<5D10y{6sgSG
z9;Ip#2v3$}3$D}J!2P2>bA=m4vfW(@0V%V=(Icxz|B?n1OnJ-}MSWEXsv-C&9?U!!
zsw`$Q=yv~T(>4c_^~j%mtLXPDB`Jv5GOJNO-On5Uhfkk42zul1K7OZPm`i17+2iEW
zprN>WC_X@e+m}u7H(`?3eYrrwWhv_szYB%QQBn*ASLonmg@ZRhKPgzmqx7L?NlTR#
zX=nzbdx#c}l%$%`p{N41f)9tJ03D0SvR0Gf*D+3v(&3YQpDSU|dOFbc$}L^oNPFy%
z4p7Uy{q>q|MnmUYl-Rc80i)oP`=#H|djOU34jB%(FFrOJ9jbQ{X72sfvzUgdZW}P5
z#r0ky<GD~A;e;~spVa%wm*4d(Sr3LDu$PX>v&c(?NGoPrw+Hxzz`$1w+dS=l>uSOI
zW6cVf^x={9mOLI2K^2;nKtw;G;lExZst5~%G8wnVPdp|!S9k3}3cH_UC7mC=)*R!_
z6}LZmbKHK&7m3|uh{NX{G=v7~2(p|wYG>?i80{hz<TQK)ZHzO1fd&OXM+C|W@7{+B
z>!J-s<(}i0Z~igEN)7yqD<OF>M7_}=8Sp-8HzhX}`17H9a@a*l+&CoP!3Ye|6fF_k
zz#>V+;AqNeLCtmREGL*42;IMM=d~F%1W6KZKd1;oJhPoLbS7|*@0hyO{?JO>;!<`d
zd~8V`s{}*khv&ed9;U+0?~2Csu>yJTudTqefME1I6((kwR`l=;W3VvDWJR?HPFbzf
zOE89x>MDJTWZW6NvASYgMIi>leJ3{UAX?F0DNd;(*zQS~W7ok_5HYUJxHu-o`>a7#
zgJ0NU3d@-CUYLN4On6BWb)R`R3;i8H>PCJ6I_S81y1xD>`rZM950?5Non*K+?MJg4
z0pNzFa}3)xDLGh8i`c4W;wxu*kE7s|5f~PZ@Fd|f%LL9nrN@**1ctPYV^Z%9<gBmY
zoo?D73xq_i{+5x0As#x~6fJ36YKE9^((g^wr<aCuyN!Xqfu34)Y6-yL{?vqu4&wPF
zJ@?BB_D{OOdM5-nfW3DPmA)@#ve)LhuE7vP_iWnVz}(}w@{OwzJMQ?k+X+Ec6B{v2
zT8_wjHwC9Phja@;SV)_*i(`9zk+O0H!+q8yV|jEqk7!i)$PB~VI$yxk!Rph;QafzZ
zxP~XZK={6-YCv`9P8@W4Xn&02PzpqWE?r;nUeo$B2_khYefJ*{ki$wGN4hpUF|7~Q
z(m!?J9om~fD?<{fK@TZPl<G^L)k~#*3c>l4L1%KX4ESGS{ZUg-ISxqzS5V`yIjrdS
zQrD}n+7v{)6{+KzwV&-##Csd+9Gd+&yt|c`CM_$|IvRMF_K4y3N0<~k3<_bD^ux?c
zH=}qe5+z~oU++YRvQLjL7>hGRw@aS9f5t$jzT<3XV8|?t(Enmx2oE)P^k1I{0(+Ub
zJd0fY&Nf{gsJyCgjA7zSlE=3H6_?(99O;ihuWPHWfH(n8$CGUxZ62eajFYYT{w=`0
zZA&6Q@AEMJ24;%-j78&;3-l1ieTPqe{I+&O;Lj(da3r81N8(6Z-?=Z?Kx9Kmv-D7O
zBPMa8yuxzqXl>m5a9s;E_o3B;3=OKG+>=fpy1DO&6k+7K^COZgi2Gf^U#8A=5(1R1
z30WEF?j|kp#{No#&=qMxfj^Ipo$dEuEj;yOqNu*(Uwsq2j($H@&xMMJnOIRLk8vrd
z2k^dbn-(rr=Y@1y_u}FmC*^kCg;nT-n07GHp=5=BwgG#$xBJUa94~;mL?{sg6YUCY
z5BfK(Qx!4lAIjlhBt+CkN~*3nhkqg@Zir&>gXlcsUOlEv>xW{Qs^BNMmv$pGBeKjP
zd|_wf4>n;4(c@XUA7Tf@zY>E#x_yAQqF4tiG!!DW+Lu4UiBSU9OHT@v>~5JOAL*nO
z9Ox4WVXePw1r`(L`i8jd+uQjgUL4ZdaZ@=-FJHUWPM)(U5Zyj*7dkoM#*2Mn>&$8h
z1~R3``?Ec%MyrZ5pX|Gh*O(oBJp`x5uhrOvxVJn|{Uj6x_=@G>Cv7}z_-tw!$Qinn
zt^%^~VBsuH<zqi^P+CexgIu6NCx7Jpvpmrf42IO9nc1Px9G`WHzV5gEUEd30d;FqH
zo^^_i)ZSRI_vap6EEG75_nnW;%#Fphd$61^h%|m-oj*rTentupy(2a&@?t`*bz4Qi
zfJwFXny(>Gw=BGKxpBLFxbI3gurE>Jey54k_L*Y5%ZeEt0N~6a&^7QEkQjW>1|W1e
zA7#JXt(F5sic1~W6?Y1|A?|;M5#p+E74Itt(1s`suC;3m)S3MItZO}GyTNa^qQ7Iq
zj^l~(b6j)9w3K*Ih@Tp#b(2;g36@+IAT=*;XWpY!rdB#K_ll`qe)z@qjrWLCqg>~$
zv-F$SGsBc>r)_HH#jtlCwPu?TbC&9bZ8k0^<1{y(Ajk#PP?$#Q=Iu0*$pvz!<q4+$
z?ygnKC{1sXf0Gg4Si)vn`n7*6eLQGJVFnDhE^8%vg?`AmjQ-gL*=CbkezLa3$iW@T
znoXB?H1I$H)S97~jlUZ-0byfpfqk(R-;S{$$t+K?3yw`2iT1jwr6=6{kSB|Nwr0`I
zype^4wLn?8Pso<gg|r{B03+NbLxr)x(@DnH{$HZy4sxH#KTp+Ldrkc-k0IPM?Yy-i
z+e(cCnty+ou_4GZkIFzJRoB&=+j%C1>`kCC0FEcbb#tqg5y5$~GVWY}jbT#?(HwNr
zLP~(-S@?zJ8%L0Ld6(d!+`p#rrIJA$1oFi1x45<J%UT3s!WJ`Y%E)GuLL0SRSGq4-
zrOtYBFaH%Gw8ga$GG?xmU?+q{Fwh73-x?3lcE`fx&|!*%&u#UlJU%~(e#+Zf)g653
z{|Mm$77W4HA{sS#>9B(Ni<G76=Y(l5KmOJIb%DSFE0PsVP3mpn?AXr<Entc7%p_Qe
zDlkvwXVa9|@ov^iGQ;{QkejXLRBm!$p-WVcn@4Q_LnU^V6vG!mH?9eHSlF(NAI2>o
zW#VMZ&b;8a>PuH>a?(!=k@<#JG}P0HXL`QwiVBi&Z%{zTbbt`=)i?R#n8ezOVaGOA
zB%Z9tR1(^QP=7!l9oR{d9)wL9f*_Spd<7Xr7<FUbNg>-*tF2Zz$A_s%j59-Y9pJja
zgNto1e_uBeVTR7@whzKv#u)zo%9<!YD~CS=J^n4Ci1<@n6_}=(MLv_Tz%8LD=th^v
ze*^-W2vtp2Pf34mxLu^~+Ray{*1OFJy<OmHY261BRTz@0?PjWPPsE<T2e;Lk653kA
z90hL9BBwO>j1Xh`9%n}$Uf0PChnb*B$Izsn+U}h?gy=lQ4y3yMIacDJilGB+3}|cz
z;zSksU|yHB&pT!PzDvvYv5~?S4&yT$yfs>2gJB7AmN-R2s?ZtH4qcR7W(8awj*hl4
zn)H$;yibNQq2C^->lrG20&O~S`o!141EgT5SPTZbUbg+Ra$noCe2#Ax0XLQ_rY(JA
z;d>NVmTeP1jEWZfWzG52teS1W_=E0=*wYTfZo!-4M)p7xd&6Ma@Gv<e@0FZM{dje;
zqRu-NL>V-*R6L9K^H^sa7Uc8^l#a^g9R&<*sN~M$KI6UW`OFnKDjhogd>k%d3$hOG
zY0>wE!tnrgn(9@@F2(?9v?!dsT<qTrApM&HRg4#)6>6iUak7X>Z@`Iw)GNp!+<8_M
zz;^Ph&N<1Aj)D-6VV<b$N{8(_SEeNc>k`(M`O=HJATuryL#7^qA4m5&x~AK3Xj-Ye
ztb9yH#-pG`p^pP&GuM(Q^QZnhxVh7BO~HyM1l6;j_lR%U!Orequ;3qr!(Mx0l%1bD
z;}_czwZ`4(^1AyTutHF2_Vr4`98&Q~A^fj!MWQmzX^QeOEtz3i{`5+7s=2hz878T`
zBpKYa%^P=+fCdQ)NOGsvyojn%47?zHvY(n6jGG&xqb6y_Y*w8=`AJsHXSwpUjDhm}
z%oYB6%0yA@`a~SgQzr1xFLrpsug=V~^Xt~w>Af_&K8NwpP=pTh%?bINI>acQo#7UR
z-B#!X=%T{PJ-2-Gs*YIjvTKY&eE~$JE&wD>$7{BWs=@zRiE7X{no)nyLK$-^h!<jf
zfQfN2d{heVrI)%fmmDLRblCYex&gV?(rR^10zHlvn*o!-LPYvxZM*uRpA#fyLT<zW
z(edkpaZC*H?vFYX5_TR}BL*J@`d(hp6;nG4{Mv`vRRROJn`+sxtQ`JN2;@mjFWmNO
zULyFoO(eLNx+c4r7<6!bEWJC5EbCe9KD@{eG&Jo3gItsoK--cyz{`$WcB&p~1*u}W
z)~0y&1LQ3r`NDFR;xOF5vbt&rbRkN@Y&>A)tS~DlP-q10nPhVLHlRT{^OBs5T>A4e
z1G792cz8TSo^k`v*k@dOj&mPf8M_@a>ANreaB@Eqd2!qOH3Z=JBpS5^CjZC=w<SVJ
zvKJX4FSM8j&Pr`Fsfc4Rjym-5IuMVi=n;Kx!+RR7XdNuB2oCJaTkEx9y30m^_W?z9
zq1q;g@U@!uMB2(f%ZIP$K`%MYAY;{vw0QU!4FZadkpAPrJP;xdxf-``ya)#wy8w6D
zKw97rL`OR!iO&^2wCdYG1jydg)TEvBMXzIqc(4n2zKcVp)U|3O;8FLt`;`~hC-JV0
zHdFvb!YXszp3zkX?@8nWRV_S4*n}g%2L=1o8kN4JM1KQljJ03lR&7jq``c2?`M`ln
zno{Ar)*bCtbNaRcQ7arvL=t9S{a$JMF6yixvF~{;9q)4v;rk&~gSA-oa<4*+c0FG-
z$%^rwo$CUaq1sw+TDanz$FJrBTob-_70J%UGJMaB$yLtOneuJYJ><v5%fFYop=asJ
zBUza2*#J!<nFQg+lRw7#(?SLOmx*v%#9;x}8S|H1%_2`YSxABN{dV^6T^lnAFxZ2c
z+5#fk?HH7kXFQ{I<j)ZUC@R8Hjz4memmdS+-(g$*g{F{?<ivmq>Db&t#;P+7B_kcC
zBEnC?C;IcPFF1A4tz`Ih-EJFOjXqaZj^Eej<xTOx3BB>;onwBoA}d6xv@@)@|MLEU
zoxt!KNrwP`fk{+WnPQ#DVgb=?EDl@k*<3rV#->h(K2UL^!z?W31#O*!=mjs399s9!
zT=fp3#xLX{<?i*7UrhB$Q6clnC=V~2nTQK>=<<!+tIuJTK9w8}J79itHAJUEl`+M3
z?uB}p_ERiuYs9!JI0~3q=lNWGpDI|F$RBm)xT>t}HK>$#Zi8yyx5Hd!HL!OpUV5~<
zK1h&_);FEA3^{|=!uju6<n;n;<(hk($w|(sZpdU=NU<wg3I3dfT@1&)Ydzb5S%CBr
zM08mM2dt+hKAQD};FEe+oniNUal9>G+RA<ZcpesYhZiE9Ls9%N-%{558Pw)ii60_G
zGn3ihv5G7z+&f<)&{pTl*jpQxq-nF$w4}4n<D4hi|9jPdToer_1ia_|O~|4dJpvpw
zOWt#sK68cN;Qr-IM*V66$wI<B!oce5OCA2qOJ&y0EPLOvmtM^$i}7#C!XNv5kV}U2
zYQL3<V%g2Jztt!a6vfQ0_K5GcV0pvn*G^eAXNT3%nQuDAFD4?wuUBVFwW}v@H4J_~
zb6;U}&AJ=$Z9OauJ+g_on0auSHb!7i=x=yuI?CMP7O>8!ga3Ny(W1h~(@$#mIcF|J
z{_$`ONblf)i9tR;5b$2y#UA!x-ss|84iDp>U)kCI0XvLnIoEw^*FQ*M*znA~v+yQ4
zHJmd3)=e<l#ed>cH`YJnwEUgP(!&zyz_@DKkH*sI)|;HN8n}07rI+?<@#5O{TCP}a
zJSLVord@nF$TI7g^fKSf1;NWgL92O88>ccn?a`Y_i&3t-0^-?$fY7dGL{x#C3Rt`l
zZaO(+*RR|6riI}gfZV1AHI1+zv}Zir9I&a@MKZRl#l%bGxZ?fCLUZb5(HqTyG)ah_
z9NqtdphgM^NjEpy_hJ3$r%w1;yNYf+Po4)frxNuQYmb<u<03A44e+)JB*U5|8}&BJ
zhtGf^1v)2<btQ=(rZQ8k2ineT94WLkE}B}>dR}iNR+vh|g9!%qra0qIm@k6q^Dv=<
zQj*&Fp3NcXk=5SdJC938PAQgg65jS{F}frZJwq>(G2Rv9X*k+-W5@;H5nP5fCKWZV
zU!wJ`UB@WcFS>h|IP?VI5LjO>^bv|g>){&sZFF1cRBju=$^;nS^lI9`wxsb1xr*8(
zHNBa5jh|+ljqM0`m<=W3!q8#+AK8RS5-y6|?1z-1HEyG0<~ndIfl<K&Uc3D@WHw4s
z=5zgQhnI+v`?#mWe^PKs-s3Y1eE*1Vwj*b?>QHTV7@f%Yu;v7c8ZYceg!B|?&l=I)
zUbTt4yWJvy+F^sYN!&H!GbQH8nRNG;0T~4h39g9*A8K+;(Z0y%yt%jv>zZ<{Y>Tn-
zZ6Er$_<}eQsxYCj_~N-2eG4;&rN5LDP91JPh@-Y7-j~T-8CtK*x*ng3Qur62&RtfX
zs8?EbzMNMmVj8^qtqJ71Y2uy{E?+k->U^Kop31UKx`*V7*!Gcw`~v59<892=n=!q(
z)O*niv+C2<CLr+om_WEV1wT6x#2oeBfIF_=x$G`<&2Ci?2X0UDDO|(8rR%APXAFf9
z4cDwwkZ<xH!T`sfD=Cp|f^Wes;LrDrPTVh|!Zh9!`ZZAc_<o3G;cGe^X*vvFI!zvp
zm{2#R_OdcH26F6mnRmpc-bm;F93w(2(qq9v74Jj;^~^Rx{&Q-u@n}|HPauMa1Q++l
zmMhPgeWJzr%#5ZYjwlGBC&I>{VR#Kr5CKVv3by~GSn*otNIxNFu^JZx=EC5cdC`8L
z6G2$&b7NRru2)6c?@_3kb>{rtYNO5zN2q*1=xeSRNV-TyM9RnI$u?T*LL^lHvSyGg
zKuxFzz#V@8(6<8&B-SMWN}RFNl>(X!I;7{#+Cc%Zb})?wi%;|s)}3F8gWBH>bjMq8
z872Yq#H<V`+K9O@r0Yk;!Tz#>?ZMIy&!aZn&%EDOf4zd42e{C*AzZ3-OD%D>aQK6W
zhIzz63+L69v!+|+f<*Lwy#NQ`KPqG>+ZpOQGYkU-kawj2)9?S`9BIRY030sx)=4k1
zfiXJqIlxbX;1+@H&EZ2tEz*zQkG~QZ(9qHT@8QutG<31a3X(7g4`2h9J`CL4zXSoe
zDo3ijz<_2gu%382H`s3s=*rcomdP*dNsT@U@+WVFi`yfjzyGTcPBK8T!6gKqBe=FL
ztSEmac>LxcFrU*U_?Ms=PD73;H<n|Qi_(AuumCaOHeRI!{_sA|KhKP4D!?=259p9o
zhzmPheANGY^FLE1ehrfU+&d1MVP2m+G-rQ*w+Q+hyjAT#q?u))EGh|V@3cvg^bESo
z=qaZq7a2+nqaj8@H?&l%QKY^XN}Y`PT9yC|(*!h|(x`yB@NJUNcp)n}*h|$43^d5F
zjVG!ozk0@p`W}!=k7FC=!FOmr8Wr<Cd_?yo>kCHfW*o7T@VxxI&cyI3`pTwFagt!c
zNnG95XzpHJQkf1%QDTz<I?5e`)FYFB@C^<taNsDyBd=@(l4-Ns!jrnFRU)Xu^w`LR
z_%^HZl&8kLR&_#p)Oww7ka|1+X~$yWh(5v$-e#oJkY`jKV=Gqt;U8mR#*VN`N#64n
z-L2a}R8xb(UO1hc+s>Ijt|hOEwztjZP5G1xX=o=*f&>Ljm_`GVu$kGHg0Cl#O}fL)
zH>(GVE!rQ_c1Ju>GNOYna(E4D;e0L$^mA2<JOY=Wgk?B&!ms1P{2}@-!mgWL>fc7<
z`TqHih$OxFj*ykWu}?DW8l$Fcb9L^hg}u3u01it3R3F?ty@hlF3NIS5mXZS=%$chW
zpRM9&beN{I5Ckc)0t7wHH!Z95&VMCcn@~vIl-TwmMh>Kn&B2>Yy%@gb@Y-H22ip&i
zHWXg2g8s5JN~JN+6)(;959}ACL(kg?d46rr3u0U0J1zFFM=)kU)$P=B*YF%g;-eRM
za19ml$K!1nNU=@sFjJ{|^-i7_pH~2Uxj?&jnv<3y?!RPZpN`H&Dqn9bb#HlLpQgL2
zUGaSz;o_-(Q~<tncH~i8lxI!>hp@r^M--Xo;(Gn}od#5BJ-S6XsVe5$db2m}2gB1D
z{jY2W6}frL$NUcz;NsGa@9(JD_Z@dXYV2E$UMF)&eEZY-q9`ZBkhF=T<r-}Ni)Gb)
zdyRt}n2jFwuHqyElR}9vgC_4A9#Uw{jM9dgInifUk0#!E^`l**Zhh=sL}VQN@Cy@9
zYM=oK9Agb*=}xjh;CSHw7dVvP0>}T)d4~4!qCLB9%vYv{lB1-(ymeFb4~doKKaZfB
zP0_;*fVA14Dli1{?M2s-$YMx~dpN#*gRjLP6p@4*sT`{3*Rnqa$ukl65Qk3Pva3xF
z1tejDf8KxHY99Xesq*YGVeM1a_>OPZm({8w-?jiSGxqha(HA*1xVe4a^5oAtD8~!s
zp+ZGgsv2(-`M(}?)B*r59x?7XO7UwsGRX%0vS(~u+cT`k=nF9eM<3k<lr~62Qit@<
z^x8=A?9feyC*Y$7*qRZCgQ{L?FrZ9OwZT1QjCn0k-sR^S5VU8LOCHCG(;m`@g1S?K
z4y=3UDhN}Wi@v15oN1Mei?{h#B9e-ez&=4>v?5RKp8nlKC=&_sI!~qRjo|Tz$xNMZ
z_nCb(GGXnFpT9+1e%AADMh2w`NZ`HM4k4A9$N%%uNq&pcGBqwA{yD=u{j+VjMtMbr
zfOvh~hCnNY_}?YVM`>_Lm?9Vfp(I9>AcP<qSc;$!vY^7VSgdz60`k!efLnsyD4qbt
zE=kbOLhql2G&CWQ@{m~aVd593<zw2$w~nWd#+H`GR&Aa{^Aamlm!q23(qA3EHhL*X
z<I>0SuHhQ+7&%FJLyMC%LMLyR<F24Ur>5C*Lx>}kkM1)oJC?4H;b_@b)}(TZTecQN
zogx*Bj>gR8GtC~#Ik_nea=+$U5o6DWXHXwfk0ypPj#88wp(ePCV9fQj^5Ppy>>QSz
z7jw0GsxGuW3k{^RC)T8YRIb_@F1yX(e6E)^Tb1Kos{3ZUob=7rDrsNUYWHB|La{*4
z^;<;qpS<HIG~1ca>Q@f~<a~n_?@PDqH~EIbMJ}ZJdJLWqlH)hmh~ihb5BI+k_!`90
z<BVn`@_9WAmG#@kuVrPlJw}JQWreP)WZB5LWNQgF?PN~1z0w|^mgcK|)Q(d~mOdbd
zA#PSJ*WvIIFsi{LabvTXJ6utfi`J}PDNH<HE3j%~!oa^CKO}c#jF&2J?5sycEEXL}
zn9KhVvwUwAV_8h~cwAjpLcpTLHljx6g0RY<l^|Dhg3uT?*Rm$}LZ)gYwf+MDGM850
z$1!BCe+(SN=gPja8L4Kky+pDl80ZitT;WY3mQ|a#6l|T>F@a9jo``btbc@__@56k8
z4{U@6!I=bdcoF9Q(Bj_E*%IDW+{srimgppq=+?6BX12>GSa~@D?TlY$$sy56eCA2l
zYXw4KA#c+OX?`rNKKrF|*{Pg({m$m*s<6n8bzoFiq79UjVD{y|og5ov>Wh%_7yY~y
zd~^MLkUgSpcsN*;AuHKcxw3r{g8Fdt>V8GNudu_}f&$|b{ThFE+y#X>@b32~G@5+4
zmd`q6qtPSvYq-$BjltB<>|@t>=);hr_u<zs-*zNtQ#pqj*UAZfy~}N*{5>N-#Spi?
zsGQI!32LV<_nm@@1u~7wu`lG0={z4JJzd0g=-Cpi)U?u%#@p-pG+-fh;C?cM{hrcp
zU2KuFY%KP+SL687j|QI-CIDe3LeEEmEb27HRmZeahzxbhtD$iaF3p@vF}?muE8%U+
zO=(*7;q3_pp+F8^>{;)s|J&SoAZM9~r548b&_^)_JpaB=3Z;!-V*<85{p*8{KWH9@
zJ@~E3`(hD=L5rx6vMue7GFkv)sR9)yM2{xu-OzBI@0PFXT8tzUFL`8b(cx0d?)5Qi
zsR*A>-%}x~s$@VIp2y?xPZqXMDT_J?JetGivqU>dq!445=Z~|Gbt(R1PBf9vuJ9o{
zj+bH&BhJiyxqbrc!#V|@6M_4YL1AFxmO7b>HLt{ET9)oq(N_&*D4#if6u-Y)-M4(=
z6OyQszAP&vJB1_QGl%sx|LTIV{&H$hmGqb0nJ~YeDvy2G0S`Cc{W@F4^{iG01F(1o
zsV|bVUAkr(Pw^_8!PxHUNpx!cGD$sF+msVGj$#7Dw)5b{cQXopC<-t#RNVGN=BQl^
z1bh_#Alg|nLA{&dC(t})pX<Nx3qd!tnM{zZuXbH_1k@WErTA5R;F*=3l3Z-B_Gbkh
zxtS$O|5#Qc_H^=Bv^_UWgjI>Rx^buCbH7Z(WYp43mfb&GYBM^Ls?xiFYSG>pxM)3|
z^_Qz_ruRyHkRehfb*B3*&z+xy2bF~cr4s2zGne%pmBA4SDyg*McI0-e^xkaR;W)9)
z^O3wleF@eFt#)yco!BE^h_TkoFt>cgWE9NLQBZoRI+WPNtB@7rd3!y;;AaMIsCJhy
z<!((Z>RnSN<L6A5%yfDG@9;!J82$;hUibtb1p%K61zA2|)08Of>#b#5A_fI4m_@kV
z%oiIO>V${eYFL|bC{T8a8iX`rF#3Lw<m~2WZB@l7w4GW^Tm7o4@Zzvc&|~DgzkgAC
zJUR08V>Q0qG;^|X`a*CY|Fwzy2^I(WW2znDLW7HmSsoS2h7;5+KR5<z<~6TIlmG1k
zt}a`w;S$ny_qD9@T7skVYL}Vxxc2%3k!=W`Z~g8&e99@Rb7)9sCGdjoH1L(PN|Gwx
zSpPJI3q~~28W}1+WhQ~HG=0wEylt8sSfFc(3)Q~`h5`%x_I-iz-vUwoEl^eF|6E|x
zta$<_AqM8P54^>Qx)2^TAS_br<zAB@zFA(siTQ9_n=hHYHdNNCNZtCGi%nLGrdVuj
zg49=|P`FnjzT~%-+0p>;L#I3>LK&7tDdr!o8dR3y>WGU3)-#jQn5!J!2w4lZrs8vr
z%6DQ|rbJqeb{z<5Ps`@NFOe`e{II(}sV?6tLqHYJv4O8<9C!W%+zGY+ajPi7R)x|@
z)awA{Fc%qeM_&~wg|vK3L(<n9a$`bg&?a&f*D>@hDw2`#s}n&o5J|qhI3kMm&zzPx
zXvk2G1C>N?i|+<QmLqF?0yW&85y=15?d6_R$jBV%URBIJdJdMw9t3>BISEhQG^k(v
zC+}MOUK0x~3z1bsoYrKjx0AU!g$R>P+-zYXw;1?OpyEiEuxYGb7=jmrrb#OF8x((z
zY0~Xa79>WWGD%?)4Fn57*xYli=_mzm!(Zjs+rF=~E7JVkd|e9j5u4eL)afN9cj@=5
z5w==nN0mfPYQg%*q~U#c`-<mbiV?#^pTVa0hf8!yi+q80)I?)dnm!&kvw4=c*9l62
zFm#zObS8OQk#mgq+*X5@`2Tr4eSu#`?5Q%E!9{(`4MfIBY4KjO(YxoDaznN7S=*E0
z1z`z)8|e&$>#c$=vAlKPTLko-e5s64Wtoe)<fyx7?6JX|1D%E*T<FZyz?i2PddAD*
zFSRZ@_jloh8ZHsf9fPOTauT~S@GkHND91FnJPK^{daV|~5lLh>Vw!^W0iMwymU`L6
zaaU(96_4TCtk%FN*jZ25m{uh(OlKvM%_4UH!{edFwU|S^8<ER={aP}~pRQ~c+o`nr
z$D6e0$1rL0c`e;(iBXaY*DLb{4u|OPV<}P(=Be9e<nup2e^ShtGm^N$x4kE1s~-Zn
zSZyDslCF#OxY5ULt~OC0p}jpLVM!oWD(Z8mzwMA_k+)RINcew*{@cI5Ry>26+fHO8
zfN(jj1PdAKcYD_Qj<r$&)#rj>d*6~t^4D(0)!}f2A^!IENzQfMWAWFhf+qHODMmSQ
zMkJ;uhs|Bn(^s)?yt}HX^p|d7(Z^Ekd&i$$*<uwx6GtQAeLT1>2V5u|hNe9f-!h=p
z$5%XGh}pmAt7;?W{mCB#SD$Y>9jXe~gztAdVT$H+^9On4$CRb6mdb5~(#~-IK)eq>
zrA;($l^jjup$_DI(R?eGcHx=hZ_F1Nr!;P+if6vCCsbYk-roj%G5TYCVz*0GXZNnn
z=X9u_wKH-ON4yHWUX>QtQmmiyRJ9ppsIJfbSswl#S{VxZp4eXOMQW(8IrIp(Ex^tG
zLNYv1i7%M`qi27=9f-kNeh3oS(23hZf|^ywDBVGt;DnSvotC5Cw#6D9c>l6DbbC9v
zXUwHruYafW0!#5#o6E9{3GY*xVFPPJt`yEPX!>=c_ve|APc@N!*VC8-RNQ76`3^kr
zQpDSVPqzQxPrd?UG@Fi039yh&fYT(jVsQHe6YAl|?dI(O<uBxNfS;$c#dx+r`%1yl
z<5=4Sg03ILPuttZl*|76>fP-=p*3pfHF}Y=eZCijA7K8muv^ZADPwV_In4|u+S#$e
zF)^>nzkZopfic{d4s+{ZosJSeiEy`vxW!+LEoxOejVdp9+o~N%P;p1>TXDSQloS`A
zu}WbftB)5`%5Z2_v(`obWQ~_j>P?tr_;e5$Kd*2b|6#z7iIT^wMy}c!ByA&lRqq=O
z#Nlhcm$^TjTQjt-#kSpOGH}J<KCUii7K4jFUze*zC5_B%7W3J7*#{Ue)|cw{(8Hnx
zUX`q=1WK&<1gt<&y2wOO`e+f+!4!Aam2OGNJev8vl`TaxefvkpFfwMpYyUIwE|?Fl
ziqv$hBNc47Mm`wJX3Pfo?*`gOZOA1|Eu9k^`67MggpMHKD6H?8kGY&}wlz7$S>%Au
z>63_W!1EAFqVWr@O0Z$FN273L;|A}5MB(CQnUH5vOZ)5=2!S_rVr^Lbmi42Op=pMc
zVjZ7S5hG~4o-R_hXszv@G(VNcL1zOA5;#(@|Bh6PGC71HT+G*W?u#7m)g72WqO`Hi
z@(C!(*>SwPc_!O|?&(>7uMWc`4>;ufjr(IhHgRk2+W3%BDGPw${A~FJevYAk9SZys
zw<JL#@n20Qe&LwcFH7TPT!)v-(<tLRw=<L@1S%!^C>GcS&n5)XC&<$5**R2oy*+c$
zSF|SWaO|MnaOI(aol~j{!-`LI(==x_6JlbA)CsZjsw`4;wec~y^Agt=A`G?mu^z2o
zm1ad|K(!`{n=jWRTb><gy5Ei@sty##YM+n%Z0B`u74qD0Errp(|EO`D=$ql5WlW+=
zY$Oy{qOG2OB(HOA@VmL%BrRcPw$b?I%ZK9-m6=qH(f16YaMI|*NX**@1Pfb5jN#9M
zFuy$R{1cy#P=p*N^{NRC1h76IZu~w{|L*>5Iq)7@9yK6i|3|_*LGE8ae?i8l%qB?6
zWU!=MF@>=@6QLYEs8K4mX4ZG;+Lg1$P4BHrvq@`_q~tSA7%U=oJtIQ_#rUi!LL2O>
z)_$7{BQjJ+dqYLI?J`WEIqMEQN%vx1Px%i6yNX|Cj*Ql?H>g`pUg0&f56spncuup&
znp8gGAo&C_!O*?{q4gA=N5id;$5&0}|APcwMF~Ip<n#02|465a|F3p7e|y`ZQ?9R?
zcVaGbCvrCjOiJm#XD8CZQ03hKd^pI5+Wb%;{OGu}cPma_3gMdTLKwXD4m!O+d@*=Z
zzn7m@{vbU{%UjWWXgS{XwQG!*@w4kEL>C69D1OEN3e2eK70CsUw%4a(g0H=he#1W5
zUt*xfY@Pix9?o-bzK@kPk7w1O7U;ruo_C4W2tcmhQbEgCG^+Tv-+3I)z>V<HU5(W*
zHW7zhnExq9R{1bsarVSl_R?_|@<3&;P$9|8>t6p#pH_mKT0&|k`|K>Zi`Q_F+#MWN
zYN1Ef34|Q@h<DV(VDaDU?<e{&#5|)8<7_A=-3%=71CVhkqTnD!FGHE6K<D3idKtQs
z&W{Up1^_3XtloWY&s&UYl&C(4VYOy4Z`)<JTR)x<UdT=EcfYu9Vryw9v-~}J5j$sK
z5|)T*=sp-zQ8b_~_c@@05`oO#kn61308LU(QZH}G<Wk?R-Xlf%p?~_V#K+;f-8;>0
zn_)*;U9#J5ldxTLj9qK`)L>A@BAAxFPDuRGtQ2Nc$r?rQ&wd$Hs}=`btC;kFo7ju6
z#jEX6!;||FH4i=KTq%39(0$4vbTd`ou{Q5pb<|_p0z76ZHxVkQdv@#rCPJJ@nUji&
zo4#s+^+2oEOXQQ!Zl6{4_;A^1N&*AmD9QdoOCRBSnlg6i_s|>fVVTmQ1p#=93K@{H
zne2@Z>r>7*Pp#v1_bc+SiJr~WmkAk)mqOE;-@9rVl3N8?5~A8=xA;kikKWLxRu*}C
z_AvyShfxFHKs{ey<v4&Aacjh-*d8>^AHZofKrdHRkqr~qF&~FAc#n}Kp+NVkj8?AZ
za?I=IzKIyV(<WPbxlQ18gAW%zs(dI8hx%3`&dH){3#JMH8H-Wl0{&En!T*)Lm?q!~
zDMr~tb-=#8LAbM57lvur=O@#=&0q6SV_$v_cV{GcKcWBX{lCk#cboyRK6iiSniU_R
zY5I<<4Mu9D+L3+vNlFvrvn~3n!Uchk@yeD+`PD50-yh}z7TIJ<e=GTNMj#d*cVvk@
zows)zk44kI9WKeWP5)J3YUGH$T4e5G$6x#bZeaapgm<<x<oPkM>oW+`wIK-ri)&7n
z4C5s0f#g5v-*TY9IEikeX6h`E9H3&c8~y!YA?y1^XBh<vk2R`AdTLu0EdSg_^wVAL
z6=Iw%W{UXoPZPt-Ie5cD%B`R{_T~WpHIJ*od}$$irU1$BI~QAgAc|Y?8^nJSAyGks
z8VZn(9kEL=yUELN6WkUk&g%+IPVpf~d}DQ*=(tIoRNcqTv5N&D@DC#o4iZ%vG=2FS
zulDoS4(*<79RLW}MKA;NhL@QC_goiPKqN%xr12H!it+NSi6NCYBVrkYiKnOW%k{gl
zPn0K<5_Ec6&p&Q?JK~C;Zr8TO*vCn%g|fccyXknH6@~b?=WTs(FScaYda(DrhEg5B
z^{{FwH_+%(baNZX*6WcFMLKTz=JeZnc5Pt$z#U*&$XrevWI6``%6d4X^|@6xDD~?|
zAwNwXeU*0FcNEC(;yS9JPjfyE(?J%5zxnmNwnBZsxENVtjiE>gL=tgO=oFcb1=%@S
zv>xV-3-u@yG$LgskJ~gwdOC=soZM^nor>~w&+=w(=;xFo?tVqz{ttR%)cSlM5L5ms
z)jclSY}1`Nnsp(rnZcK#X)lp4+-E0rfd1dxI9T;6&#;(F9$_umWG3A8T3(F{hCIGD
zGkO&#Q3SGUH2UT_tpo&1^tr72C6B-TD0=gncG0TLxuwgmkjSvx>*pn2Fhx{b_0v1g
z)u>s|b*+n!4$F}TEf32Jc3)6F>T6Rc7Qk4n^V}A%xNXZ}c>$-}g>r2LrgnE{nUVbJ
zK4y?q_m&G{x3;*slQa5>@N^vDGDNDhyh>dY+qAvjQ6?J0?ixy@0NY|}$HT$)YgB2J
z)m=nD3@1<;_17LW5kJJpyJQX5d#$@<*$K4h!ZZdO(CsP%H&1?@`yM~G_ADPkbMFTh
zVqbA-O67BsJv427JTjxbTj$YWH?V-Sl|B?!jthF@8lr5*2lz*`#@K%+-S!*CYgW=L
zO)!pLkED-np~jORi_Yw(gkn!V<Zg}P?Z2FU)xYnU(b&_3pe$C6`SQHvx8lbF($rZu
zdV`3c*4XR<?+>Vr!TgvgdvZ}snE=$ja9+Z3cQ6H?Wd$Ald;>+eZ=eXL;F{!W39IyX
zIqxzalz8%d@iL+$X_}ZNjeYS2>)KJyEr0mn7x<0dqEPpxD7C!jp9-zH5h}j?Ezw%v
zpZyZA`L~KQu|wiobPghq-YcFzP2^jADI<cFHS>YY-j8cNfO-CMXtZ_#q>G_$(UtAp
z9KQQHJ;>chqcP8W8Oms+c;&mUaI2x_vYab^Q#x`5!=mKKJM?%MDZ1wEJgh4FGi9Zo
z+fc~z&1{y|>vkCTBpLH*A$+m-VJCD`-;mEJi&^@9dYOI3)v|Tj@QcOOq;8w1pQuf4
zyld}*Ta!81EN*Q?oxM2UaWd39wXr=Yh@BmA#p_MHKrS8<swKvdE--kDeQ&5&c$f3+
zKwVG%X*+P!^|wW@G>%Jc$`KB{a6G{ND4+K|4>a*8i*dTAM)hgT&76lKgK5&~TJFMz
z0p^!Jwt_I=C*u&+-uJ5=j;9SRJISe4jlYH~DBI3hGmNV^Lh=3{)css^HdF-K(mpw(
z7K494r^^zQSapwtx}yeSF0}x6^If}M0SZP_*Jq<7M)6{ui-uM#t=&~q?ky@!^?FDb
z6?@w{dq%QArAg=tur5|HDaXFH$bXL3Yj=O7{%yCt&8k0F)js07!m+-rl@<4Nb*xZY
z!Uo4XRUdz(@FiAmZNY-A=u#!Npz#T3YIS(e9}@DyRrJuucBg!*b+htJEv3;_`fLfe
zLSHPu)0?|`g#Qi6dX^eCUg?(_7cN$2tvMFZ1MoE#fUh_;kx_iOR-GSuNzQ~HA6#J}
z53XGu0yK+LDHUQqg7}RUvlNik3$C=86Hoza%`sU}>pe6-=|j(3F^u-V@<<j5iXYbh
zjwh14M0=4b#!1!kLZMsQ2S2Xh_3&fj0nRhqWx!Br4|?aM8ZuO26t9pgVHGUAW82G_
z(i%VZpdDc~s_3jGlW}|}ZBv!nE|X1ts_Lo)uJjv1vA93U#60$qwCxVVS80i_N+zC$
z!d2=35J*<(&YJs`PIjd(tWO3CaWv`Ilp)h?Jyb-dgM3w`R+1+#?z@)LMqfPWGssGV
zd$U%9v-7K|Eb>Jtq6h&9r6E>;<4=8lwCbq{+M?HxzrAlCjdVzftaDZ&Qg|(a=?#ZR
zZrophxT|ohn9WJX;my>&?kfohFmR(e@h_dZVhy8u_nV@b=8bK<A?O;0P^+0ZK^;Vd
zWct`LLju}_kg^B;LvSQt#vsQ4t_Iu$Br1d3)ja=nd|!w%m{Z>*+Ox$g%0)Nyr88|$
z<3BM~76Zi^@I-}}f|Em8a<$WvOf>%}WL7Q5;Pq!r?aDh*lTWP$mhX<$hi)kGzk7~B
z7vOeVG0Gnh-GAlWK2rMV99}I)<LkY03UPQ({#9FV_OkHHu>*#>-Ol*w##yreF4Jmp
zB)!P;3}=;u6_(wXxZCpUCr#OJZ(DQr+1~cEb^f8kgZgL0*Pkz|0h%K&cG@4-P>!Dk
z*Yi-ets`rd0EXx#{dSFR>=HRv=RED@uD-8Edux&z391l!K`HwOC#4ta5YFlG#4&%Q
ze2JuKUL@La_b*ZdQM~gP*3xh7ki8_jNWZK%XiGMh4eC0?QdSj4d8qBV#*i(`7a7YF
zeV=iPq($(V=-<)XQ{d7gBr=v6YwF!#UD4CL5e$Kc++~SM8*S;YZrVfUsfZ}K(U=Bc
zljli!izA<5U@RKgv+J53>>~-S@$dJHbN<esv_=U7QNfmanQIo<vnTPB{}mf{Kfr~*
zt5y?>Uu)8BmmB*HlJzV5UZ@=c9i}@8Lky85XBXgr0GThv6rJ!9?l;b%U?XGxoXn3&
zCEm63Xg16%k+9F#8x{zdFTUyK`SDZW9pqm7OJo!3$QPQNL;LZcFE0gjgVpbrl7Bon
zoqJ2X0k_7Le5HpXQ@h)Tn>IF&%CSX!6Y$+`Cm(Bxs2OJotWEP#{V#TD+btQ;wXdmj
z^%Ck#y0i&Hl0dstj28}oF3G;_lp!sVc>lN?|H0{2fnw$7kcxwMsHfImbCggGYfxR@
zJZ+jpw9fKQ_lyhiDSzdIeERDtgGM<rmC%}2j0pmVk$JmoIBMl8zVUregD(zF7y_Ph
z64~%?-C&~sh0jy<^<O~yKtC1;m*%VWHIrrrVDDI|E*2i>tS^DQE0S%eWXgt1iDG*d
z@rbTPx4k`wl+4|o-2p4LyQb?+EUVjp4Ic&)oUWUU>SvTFQV^<55eQ6SRW0(H<zKNW
z)XV{L`jiKYsCM(fVFuww7Z#Fw4RYt{E?n8gu$7d#3Hv<bFTr;NOs~`OmFzY&sIJ^(
zgS(VEc=?2a<462~8H=<wQS94{!fO&ORawP950)3jt*510i+k!Nl6Y;$9V)9T9}BT>
z@XEKXZ9hV1_FM6KcrU9w$yy-ceygvxtP&mDkt}*iddlmpa-g}1Tb*^=l+Ix6!iBDQ
z_YhupDR$S{(y<Ot2qL2)k+ubzZVu17uOf6xYwj-b40z1y;avN9`&+H949v%!S@u0=
zc&>F{>%YdL`*X7PiCl!*4D9`0-$u2177t+od9w(@mcIY|?a&yu^|KDXb0-TZ%-!n`
zN<l~^nt@31f{;Cq2{>p=w!x_c$mA>Zc|c$54@ar-XV4o1@_i(YV_Ft6&$`YvGbKuZ
zAGRYP(Et;q4*GUz?v3_&$)x<^U?l|sy*?8Nsy^`Q6~y--QNAU}P;{D#HJ#gh#dg$*
zpY>yA)fh5btF87qRf`d`q`qgJmLJljW~Yf{&m4bnXC*o$YM-e7$wJWrKtubHH!s$~
zp#if)sitqi^FgQ3Z>4iT+6hZMXy#2w8;H^#q~@U8IfLU*Ym79<P2BQQI%eyA_$Z4&
zpA%1~t(ceSox*xw=>1uj2jSBV{SS>5%GSC+>xBOi9A~4|9qDiMG%V!c`Fw+%X>DRJ
z$+-CsuMAcr%@)Q*+P*LNx;+(m7Y`+Qo}2|QilkgwBm(d^mSIWL&gG&mj`Cc8+B9HV
z!d}5ukXiWhuB91eLQx06C53KIJyclPcHhUN1Y}#^mFzN~j@R?aD|1pIR{@<#a{`Mf
zIm9($W|fL4?>7u1e&*B_2f(IbDUDy@rzXRL_20;Vir$>nh%i$Zdj^a}PYmPI%uEmQ
z;DZfAcW9B87%>-3P@#_mbvAk0=3<%c9!4)V94GTJVbArse)!sMdKw=vzES-3MNLH8
zTfb#%iZ0>Rn{77V9;N$bun1ED{>Qax5jo)<p{@Z;FY+*$AD$q#({vOU8tBL7*t6c}
zJ$8+m!pUr2y^j#k-HYg<E8jGx%eAwzU_Qt*it7Lg0hrCH{n_9jlx6u3%JLKokYGZo
zRp{VG>l*g508J!|6Ia2&yvp~`;Kg0Dcxd)G0g|L_{C6s1*U^wtY#W#4FKM(u(pM!$
zAG)e}>&PXKwCWUB1X2<z9mz*A!%5m=ZhdIDotSi8c+dgtDc!W!0Iv0FDo{UDIEmD(
zW7_Bnr{{xjdr+E`LlV)U3o_!CZ97~%Z4%G;OZpGG?r}Ve=KzHs4WKWQ$=$ATAcNd@
z4elnWN#IGDzOIvQ(B9H79B&9%E;4$j`RhaIH8Z2wFYh_n5AUI~pZL$N-g-~~jljKQ
zsh)gb&FtImjAe-)>{klw^OfAVNM11X83{_K^)=Xbb-&Ps1W8bL5Ag_BJ}(%B@H=v`
z_c4_;G`De6X>yp1=XOhcxfHN~ofRBC4zhccGcF&EU}cwDi}T_O$#UpZHI9uqD>Wg6
zL|P&Rq=R-|Yo23pCW9Q;?c{#E8X-Q58^YRzQ@k5L^<H?5MH4I}&0?0d7z->ULl+4I
zj})^ngdj)cT0K$VJ(jSAcNQ)TD^3-=TCi|_rw=A4Lsqr3jV&>uC1b^i$TQX8xAGE+
zTtozJ#$^XTbD}F@B@btyr+Mv+ZK*|Ij07d>^i$os-yv3RTVJ?L+Rucwnw9EgZQbl)
zUNz7`8P1%4I)J`Pv((mRc7~G(&G6j>TNoO=3p*DB!Ey1B|1{-W>*w~s^#6nV?o657
zhERyw7o|)OfG?f3=Voc7b1p88G~T5<HDL;7<sl-E+OYY}vvz~E_P54LNToEHuqSKS
zYI#^6Rq7Tpv%Zm;1WlyhRK4k(rZq@VhrMI5*BHBIV{`yMer8Q&7Qxp`)V*dmyx!xi
zvN;fJvFNY!59Y>U=3v6(5CPOav&s`w_m(Pu5JULX{~z|=DyXh4iW0>=cyM<M?(S~E
z-7OH@-60U%J-BOdch>}hySuygCiiw%b@%&zRj=Ot$VVz+AND$P%{Av3V~(X4uI5Aa
z*#~OPUq_PdUzD?6dlD@cU?5R|zVOSo_`tp?ClgGH9a7VE4Pn(q@N-RW%2al%SA%t<
zz6HcFhhZ38a)&~{J#b5V&8*A_k-@4h=?h|YL-dYhf;#7G#E3R~=aWS@0;KXepPlvv
z33uhK?vBi`Sh5Lo>piYHP}Xc6hfvx#!?3k^jNvWhM-(e|<gN4Yp5Zqhob7w6^7;l{
ze<*V6Ci3%QWIW9!E0By9EhKo{hR>WsvBpq&FpjArs`hFH05n@r6x4OW!@7*eiy(tB
zk&>0FdU+5+sdRrR|Cc2JMHEcndcBsiYBxI*Hg_{&YzU|lY|^1giyjvJ*c}?|Cg%=B
z^2k)6Src`TP<qvPhZEE#-`Poftw~E{M&+zHkysokP%>(swlhC@(Fknl!fP6B^-w(K
zBdHhm^uj_xfdm6q#_4E#uRAtn2-q<&87hEq4OqtxBI_r0t%4~p+yW6d4iQv>@?SY>
zEX8sL@K+o%^jH0MOOnG(lf6R@Cjv+;rhpxSYCSw*c>g8Jbk}{85i`Oz{4}`}$~nxY
zg|C~o=>L|ia7dsEQb5T%{a&&@-u8R#WdeNe8n%5FN+S-Gzd^s7A_<rMr1M*F!?DsK
zHhw$~*PtHOD)r|-`3PVxo#0>!s9~2QT9`FI*0di&0J6Mj!r$Q5R2?=Xy=4A3#bC<9
z#M=EkjY)XN6;~!26m&AG7tcp?mnD@D^*99ot(ql=_oQO)JV2-m=Meh=JGO}FeT(^2
zZ&;@;m>E=?p;`w{K(*i&?esic{VuipXq_)abu96xQ&Kux2xNdDC_&s|Bh*r$FoJK#
zq-mtlFD?TRP8tsum@n+rXF(aS1C-wI?{4KU!1)`}WcT;{mC66-{M}Ox6r5zHfp3#W
z#lr&kRocorB=<^QR@_%BnQi_Y=X&zEboJ14oQ-&xt097;i3?+jtif_ij=%I_FzUs`
zI=3lDFN4^x<5a6wld`hg!GQX_i;pH+c(VEu++YC2OygJycG{(4wH*4lOrm-3)Cs(^
zY)@B1a~Ai%fH3!6ra3Vvsw5Av_R|yX5&g9GM*lM7sq6JzkG+XPYm|{3z6PiCLDquz
zApuCMW$^}3Ki{{<9S_ktUMy7=*pU%FF$2H%lQ$nNwjrq^vJ!fZnC50Q0KGo1T-jA&
zNe!YHS>Lr(j0jc>(fS)o;^+1Zu6uHqY(fV5vCY^~;louDqo#UncL`zN&wqdXoipjL
zN2$`R!DT{GFQC$Hurxl8MzgI`&26R)EclXsdV?e-rub?<1`Xy12lHGqbL1{2<+NKA
ztdw8LPXr9K@E*~5LLdgY8t;c#ZuY+`Ef(07+9`jO^1L^Z6<^fK9?R@p_IQdIcTF^b
zQ>bmYsM@qMglW0%&TF_1n<z<<Ahez-c2plT(CpF9EmGNj;L$nl@rwwg?53C--|003
zAXhag4F?*QAFd8WIo-c+Hx`VoUH*o%?uPQq--LMt-3+JwSJp9NZI%ZdIbTwMG8d<#
znaS=8(8NLuye{v32;&7Af=1Sea8$_V*v+RHX*NHiAGSBPIcPUDGho~CC*%$V8Z5uA
zc+uQ8xN_Vko5!G&?l>}Ng2|gY34nw>SyASw93ndxTZ*K-M#Yker_p1C@j5@$X0Ejs
z-n;?W;-E2VtT5T&*!&efRp--TL0ttU-i%Wg!|YgwoWaE}VNTpq^@L+Y=SyKfyoZBn
z*}Mn#1mA0ev-Gor?H<melF=9;sTI(>JNI{ikI}zf_%C)CR`qGC4Pui|YE^kaKTtBo
z{<gz*FDciWzWTG1zip?yoBKGj-lo_T3%iB)JYwBr1Aon^@Qt+|XrEDnsD+;!CDP_%
z--%tT!}MtWctCV==q$_78yZ>f^*n%>y4Y5=nvf<H91n+WQojo=Bl-2jcb&N{i%}73
zSGvm{`tCS+<4vm*fKh1}*{elE|2^KX@XpJDwXTr?j(3+cU9J6Qv_KG}LsIH(0ZjUD
zQV|xWMsnQCZf#0liEj!Q^_DYVPc*}}TFX<Y+`)tuar%`8zB3=DHqW}8_&{qcb^U~e
z#<B4_1h@jpf$t2Lqr;|)%YdR%*5<cNmH4H#(cu-xEzm9uzVgYI&~}Y^<Rkh$!_f^Q
z5hkvFJUkfxw~Zul`?ls`i35na6d0zkyOvh8vbX#y!E3Rr1ysUR3Wa}ww1b)0;U?ZO
zVO1&a{cOemMfoW3TO-($r{c16xx~EKutD6wpMJX$ty3~7W5K&M02`uuI%q&-zUjxz
zix})65lG%QDf+L6zlpLh#7+d>Gd|J30J7K;k)7CVSv@1Wa(BnewM^=KaVSR5x6f;z
z$zyBaEMe^2I<`3bt|bsUw9jaB^`~O{_FkN|-`Rrv*L|CSpqAO6O>d$L+`f_g7{eyW
zQ817TYm}_Vm9_20d{abz)*_{B>U+=9NO38sA>Hth%I#(i*ktB!{CRWjM#%R7v}j`Y
zXSYS6S#_U`Ougg&r_2qvnL3c*Fou=SXGLMB&QSvo<|W&%ROT@2yZgz=24u!a`jM6M
zZ&V}_WA1;`GR#?DcFs`#!;j4PLb-{*OYtZKxd@i-7-l`s`iT4h33c{&`eu!HgwHjB
zIwv**ph|tJUI7js@+=!4BPvPu5KFnb)Aa$<X&-@X@oULyS<cr0R5PHkxEwe(eVqDV
zouT(pVm%-hfOwIcQpF~pQb6x>>5y)x>}482#}ar~x6nSUk6T8*L|O%bZXF>B{Hf`N
zrbSvrz(%75lhb~dOaBM<jl3>p#3l3q-KQ>fwj>i9S!DopEYstH*zFGYvO-w`UfM|W
z%nHo<u5xijl5&9{+;Q(}6*nW>VL-8PiaF5<=oPH*D*&|u?<#wD3QGBO?2yXL+SQ{=
z>Z+xm4KCEy9#q3su;L@2BDg&4d(c`_Y?qieD5`C@2FSmkPKv_v0bXM8QU~UQ$pA-y
zx}U#}_1N#}dF{>Zxf|i6$u49&I1CgH<-@^pML?^#t`V;@KLcnTW4=vPy(7JF|02ER
zP@om}s^BWc%ua(OFF(`cq=LtrV}fY6`co4eUSnPbhnx<`0hvuuwsN!li<6Rq^6SIC
zLJK!@ojyRSbKUgX>t8ORDkLTEh17A3AL)~M`?KMMX)7|Sa)}HeI9!kRN;CP?m?-VN
zZOzyU0UXk6>TKH&LBptMh@jPORwe`uELNtWJO~yT1DXq-?{h~l_mdff1oz3hFZ#z_
zgNrkXN^Ard_|W}45u{YzZO`#(#D17@2NjQHqT+ze7?NHmp0bD)Z?1Qe<p@wy96D|W
z!b6Hf1W5+?gDlHYe^$M~Kc1XB-m*PJ{+C}^cFh9hphQ5Kjt7+K%otUDB|;!Md^uEu
zvRdeFTs^JVLzto0Pt0-VE_&>G0Pe3#!aE=Z&9lB9f0fjB*$#IWb~iaTXEuE{2M8WG
zS5I9*4$~ZTxSA0Zb3`X;Oa<Uj3Ab?90jaIbbjeHpdGB4S2xJa^c2ht%806eHS7R@v
zm)9#m_8l;uaDK8aoJJw2nN?_OJC7N<OqG`?`wYk*obmTDD@R@1lhDL%{qcikFp~kS
zhr$wb<98BL`svc^AVcSuU@j-tWLob~>M2K+lNrUTv<&~496(g~XQ@2Of9tmL2Af(N
zy|~;F)J2b5&xR@kxp56=V5ko1!R7Hzz<cf0Zuz*{qOJ<xG2?~<HP!O4oyoJAjxa68
zj(`O{f`t+l+xQl{vUTZjIH0eGS~Z(g^Ie<LhP2r4ZdVoY3ci#L{)Mxll_5?HC><A%
zv}-zST|f>)tdOfIFva=aVO@{EurBPs2TBUqQp0|{);>irpgd_xqkxLGgQ+?#`0=O5
zaR!>!E39vEO<N9$U^a+_)T(jL{<zH5mgb#f`vE3>eTp>q>t<;Tm%@=J8X3k%nGtJd
z|IhN7>GCm4r$i@jensn3qpt<~fxgD#h4AY-i)w&8U#|fhz$u=b*wSo@6r4^)T~6dW
zb8~4_V}o&GV640AN*qR`mj}W^xPr6cf(8m8AOc?|ZOC}coOy7TQD}pr<@MLfLPY+g
z>fBcek~RCAmi-$58vYQE34=<YpUw~+rFmF(e+C9sJn8CHT=fTw`1Se3m5uMr(G-F!
z|Lj3*1O^NVQlnqa|B`UJ*|2$V)UV#}<2}0X&x{VABYyfK`hOs@{L`1c1whYZ^aH3J
zj@NoMI>3yPt3&o!FJ3nCl@9vNuXieNxMcLwJ2b-8JQQ&v4o+Q%6(<-}hLHspF0<5F
zQ!~$5D}Lprc7d~KFiEG5JF49$Fe(luD4r!3Sn}tZH-|98Wb?JIfzEbxmHC>b`*+-V
z9FIC3^efdHle!nXd_2l|#P{9#rs<JHmr)|y-3Wpg(RBnUnA66eBUu2Ak-bZAe3K3q
zY>wS(`>C8zmXMWyO|9@&+ND7SjZbaN7}HX+O~M18c>151P9)kYmmzfkL=`lF9CH!&
z_re33YCXg`B9PF*#g`?Em++e>Lgr`p9h20T5S^Cx{MYZIg=sE~q{la@GCS&~5p0mP
z<?kY1-qp&iA6vN#KTtdE+Ll?3!?I8etdT;#X0&SU4fqfTawnIUWO8rFx&wmQS(=XR
zo_UziNkAz4a7#+(45;&NxG(idlmhEhfPZn3so}{#Y1^1x(UBMXM<<o}S0}|#3SnS!
zzp?JBCArzpt?~S5fG_IRxHTfO*LJBC;EH4hAEeAE*c<-i<k8A|db6h_)OboA##NS~
zLsrf{sz0D4qPMhBt(IwGl*L>7AiBz6&L63r5Xq;iz42$6Cf@m`(FN2$-?H0QvjM~V
zw;cNZEicpFvVH+m4A@w?&w$H;-HR=6MirJP)0Md{HwZd?)*eR~Qa(@0p5T?LH@xHN
zTBH!hdU4wSCY_Y3S^2pZuY4o2e9#a(1mOxFr2n(a&3N3*h5t=+7W+G_IsSBbT}t%B
z=YE&RADrMCR}Z1y$_%E9h|y=wV~6oR`tYhzO9)BRrtOy05G71jto2)$-Sd+12xpma
zgKqrJ8Wmy*n#TujVovARmqGnFk?{x^m3aw=6<+oR*mnhZfz-b$*e`&e=i8X2fauTS
z1=MzqF$8zjOzD{z0iOp|8upR`Bd`wSOU~1Cihmd2ip#L-T}ud*ZOg59A$)%Y_jbT<
z94Zq&=pHEu6!BIs#=eq7_5X-vY~IrbAT*aGg+RW9p6>c=d25-|L&<lDxeCpsgkocH
zm9g`fVy65glN0*=!1NW9=bu^}nP(Ew6W~YI0ZI1OY7^2i<h!1c6M*VYg9whq<3T=v
z{mP$o^bGqe0%iYK1R4_p6H&3WfQg1==)<g^tV=2Mq>K&D`02Fg0shf4737nPZOdPD
z$>>hgXc9oQb47silY7}+?Ok3fDFtj3YN>eZ0t!$Z2&H)T$sN<bai;kTpa<}dVg+$X
z{FXti1|||4F)QOc!BBAyOom)B6QE1g<2DtFw}E+|f*jiDA2Jtz1qsi*o^47%5tKj-
zNRpZH7U2F$GUI`Ei}m1YSso@OzfW@eJ47EH8FJNMNy;kp|6Hv=h?wW&J!64k{UV>_
zs}V^%)do5T)mu74N^AK*$=_q-?(Z@{qH$iWHe}!v{vTD{b!89}8L!%uSK~mfVlw6w
zTefGkMYjIGztX$V9RcN^BNc%UxQQD!#*Y$4G0Jn3y&COZ{kHxMkWiBXH(|f)ia%5S
z)fL11-@4+ysovg7KsN6d0J76WX#?Ei{r>->0A&?V&)~@WgOHv7uBHFGt^-)h|33J?
zrT#Ct{@>Y7OrqHjCy7qBl&5RRBJF1C)7-#FP@PYxyEm{gzhwSRp{q;)V(eK141Q3{
z5$U!#wG8{dK2R2<n|Z!(UO})DSwCjJME@(N_rI=_fQ~p}Zv6c6?-a>S<KHQgXBj|#
z#Scfe?gs)*1AVTM_%1u=758IaG3E$b0pKelP-qCMY5Xo@)%PF#20Z1As%NicD+EaA
z_Ks!lIX1}af;&*E4FJq22=<FuFqHuRl$Z8BZ#D@5kNHsgHU9mX%ano-C;Y}){ty<(
z>nws`h%k294j|F0?;}oK#=uALxrof;|NZ3zfgb~OcjWJbVN|Hu&ru{^S==@-Q$)xd
zjcJ&NP@sq^Hj1X{?{9?+sD2NAZA{d@kN^1L0*_hA9%29cOmDD+wNsW|Y{~Z*1pn^^
z|N9323(^1f;D1TxzbyLy(_?~&;se+f;`u<V9|sJtuQvBeY_tCJRwQB#CK+--mt$kd
zJW9cM!q#W7$!Q8HpmLDnUB)&ATDi|1puBwV=XpK|P=lcR4N?5&1?`A0deFw3#@v4&
zS_B^ae-14I&BYPUmCyh50{E}d0IHS$S}R~k;J;?_eemJ`6MZ}Ta3-gNhR@qh(Do;{
z#&Had$oE2CDf%BeQ0Z*-N*0LfFM$BCw%8Qi1_zYu_W+_!uy*%DT>+@gpZ+2wsDK;}
zs+mE+L;Z2iL-4iPPN<ail#Y!Xq2`ygaN-M3<4F4VBAF;$n<UGM$3+{;3iaih-H!{c
z*74h|c_|4M({vU3Hp@EtTJEpJ@a*lpLclO-{Ql~SdUK0TJgsf+NlAP6-;wu{7l^*I
zhor~@>dqH0o$w@Yop4?iE+3)3;lwT7jq_K&uu2REtV_-$_#HR8HNM--_qRa>;EpRP
zptE@=zTR8AWFVQb4_WImitS$8lv-iXXv%X>E|}yc2GN|AyGa!~`v?Kkj#6ePPfIR5
zU)nb}V|7hWW{Be`eNnKRC@w;Mv^rnwJiNc`JP5oD+Wx*jEelws7=j)Ov%my&?Er)P
ze7(YPc{9kfvq#vGe6=?rRhbSixeDegHJjm{>)q=1icZG3o(jG5edWx;*kNTAO}X0i
zt#F;rZhKT=XD(h~m^@jZcy3KgW0IupI(+?ScPIOtQaH%!<ari`2AqJR9{0G<Cp$;k
z3gxK5nG+-P_hyKQ3dr2q8cj?eVV~XZX&-L-qKC6C9-m!@$dh$rkL`K=AT}fa_LQ#x
zEJt)VSQ5{iX9dpXe#}$y$Gr)iN^en|7FP9lxbO_SA0zE=B4mnUEFQWF-nSI&^F<kB
z@9VeclZ5gtUd5_uV9sW<5m_#<>*5S)WE5O$G*o-`Fja3hsz4@Lp1|$Wy=2uV#$-5l
zv}N?-vL+CE`w85We40FY7#~T^X_){QCfX11Nr;|t3}QNLWw@m{oHA9{`=rpYPrmcR
ziW(bOd<$Pz_Kb%k1+Md7h(i0lp0p}x*G!)h!3n>{1Ap#xf%^z9HlM(sfcYcBICSlA
zg?I`z-wi}c<cTm$r{;bfqf$*d?6&IiAm($kW--bEo)IKrKY8W&^bB>!$_XN!6T-I<
zZFrrc5=yl=ntPjmKx{_6ugwas;5z_It-?@2C8`V~F=S?e3=>9C=$ij$kMKb(ZN)XR
zgM$?Y!dEo!=E5HvG%t;J_bPnY>A=)M`xkVA#9iWLrqV3o(=PnVMys>{%}HU!hgJ7<
zZOXbzVjg{6lDmW|nhHw##s#=CG3hQ~0Yet@OuDC{_<Mi<S*VI%I;AG}rOsQ=Q4pBP
z37=Yn0yiJ({evsI=&T;~W_I@UyH$&6<@Q_-dV8_0vB`m;ij_N08{FOMfDQ*jr-}Gi
z+!nnKT)M0nSPfO67GI2v&%;6S%qh1L%^$1e@H)5BI>|1pa54a7xKAMWmOCm$ibvw8
zy1;j`TeIPEd}MgxbLu^DeQXd|t!ssA)EsV{fCTmo#9x5$4F0_UC{S##D@B$g=IsC$
z<~&DI8UD-7Ly-{C!yqdTA))WfYnhXGU3CWUV}*{3s@WG-6F}c_PRebPesBS8TduSr
zUML;E;kYwG2P=+WJ9$tw2bgHkw0Yl*Iy`w7><kEtW$I;_AGMky-yg4hV$QeGeAxTp
zOQs+Dl)sUk>hNM6?2UJBcyl<0;&9l9bm>(E+i~(s@0*xu2AUZw3};TV(~8B=MZRZW
zdyOCv2!L@&&ACaxJYjpGWG-EFrukq+mP?6_VwBfSYC`2N#Y3c&)7dXN)|h82XPpf?
z%sNi7=v6wse;jM<QU98ZNNn~H`IT&TmN5#b>tw1rtZ7kcszKkKCUq*jQO~WomFako
zJ3rnF$iI}gOMugovRD5J$7xz?c|F1WF@$;6N$UH0`#tKDm({Ph07dYPZDh`uy|c@U
zS@j#IKV?66rm}2o2AW@=ssx<$qY1glfH7?$+R2}L!$rNMx?Tr6v!j=v0Q(<3M6V`Y
zWCw79II7}kEAelLWb*Wv2IMPARZ20CvUhL0@psMvv$!g}XWFM7`l0&vJfU3=S9>OF
z4Rbr8BG3@wy+<}}ayxWQA__=|zva52KR^*EubXDjdwaB>^nSA1hW<zZA$y6@rhWRl
zX1vSTsZSLp;J(L(-^<%xn!XjHjh+_{cw_YY3_?~4f(8rWPTtl8>JRN4E_9q0E?ql1
z->}U=5u7&qoQ|ypm|u8)JbLWA3nd$Mo|BPR<%qt7f}s>ADwNxL?E=4T@RS)Icf~-W
zMGuMJTRQ_I(E%-ESoPI@d7<ITM*Y6YHQ_BI)Lk2F&ZZ)iyRNXdQqHx(Z?2Zr2^pRo
z`?c8yMzz0jB}{02(6)0nN5amRMb|5=qpDfOH<N<mb}{>7WOVdH;Nr*tm+`gtS<Sk2
zv+B#ZxDpX2zxpI90-R!@eim1mPT8v&d=eGfrS0uL$B!p7QaZvp?QGEr{bOSHmJ^V*
zl8Z*)(G~J|FM^XO&=@Ni)2o}TyM(V#O0P-}^yiJ=J0*CH-6lq7ZD^dIWPvn)&_m5;
z=2bEf5KqI=OW2u_FvjIT!*H#gxUsPjGiNcN5mY5qiS-QJ;V|I8vIN1!ja-;B)SE~y
z5)wh&8fw+g3A+%2_3944T(L8qZ7x&fh6rkRq_M%yM`7OiN`YN>iU!NZ9&%j0h;o2+
zeibco&2?cD;EJwumE%o#FTB0o-2Tmj;4OJS0c#qmbW8DEcH(eVl7R!aUGS$)x;Dd>
zfR2nB8%2}qy5aXu&^BD1a$$3!hHflpvE8pyS*RP4{GPVoFQ@nlg>7_8O*t#kjBOnH
zD}_LO9oJ6ZH24&rASPMo4Kn(92rv1?@)V;a0=XwsQ6ZF+pu{c-Eb#+vX}qeE2xpJF
z?;ZAQqpvBpQGRlRt@8pJmWsPbYc<u;BZo=$<jv@rYc|6Hx-}^J7(pyR=F#idC!&-n
z$O#2HBr+tn-+8(066#gbqu|p4xNpdL7x~-;y@H*k4J|=tYUezSLbXF}+dN(N;s>}c
z0ybJ=ouRue54Y8wqsSc}+*`AuAFl3z%X~mt+K|rH@=V#UIl$+<@UE9(9wN^?+th^#
z{amFu!^qX!?&VW-el)M?{!MCC?DPm=)2%b{jmeX-Uasiz{G%=uqi{L==dpUeU96_$
za`zUm_Z1(Qa}9Syoi9i@P45`O8lY|5N)%W!w-Kcgm9zUkxxKxa%!L$4QG5(3a5TFM
z0SgYR7o2KHZxQ#+*J|X^9YFuNW9&JsS-^=%ms-1*u;-~TWgiNDC!{%>Zy`*#$)t^R
z)Hh1l-aK)}ICIA7nMWY{>{6Tvlno>Mn1q<s4H#ZQr1{;e*6BJjG!g!WT173~3_&-`
z8X?dZEV|<RL3y*@nF-7=Ox<$WSF??lc3fmzTKu1i5`OofQ(1?!Cn1OQeeFF<3RX6K
zU3Ur4D4hih_Bjfptt0MhS)mgtfrcF1Yh7hd_v-xi`Ng2V<x;dZOX%ZV(?seD4qRLb
z6b3>_;vd%i@S8;_HUtAAyy<J$2d6=^&Qh5A%lYrRJ2e!#FAdu28N6?^E+o1kJ&rox
zi|RI6s_A(>j+}5mUhjZQyFa(5pEDKOJR)Fk$3o%fi-!`MDNpJ1K=}jCN57k2w1zy7
zFrlrlvK$no>)BrdUQZw@kgI=L1*%!%LS%DVt1=fMZaDZRn&%%BtAoLiw})#=pWljK
zZge@wowhBeLotnjk0u~a(l9(f_@|>)M6dbCVrK{9UL8k)e#_%8T!;@ar`71b54MMS
zK+xOg;hHB#vBj<0CVaHQKawS7jAAFaRhlxNkEn4Av*kDYJk_~`3%^FXvTOZoI#n)x
z7n9-kd^XgpE(vTz<Eqx<A+(%cyC^kZMQH=5ZQ&3;P>m1L|J(^PF-Pzdf)A;9qES~5
z$uMIsvHnz1K8ycw*O|MMHD&zm&-%m39-++L;A=XV4c}^uPP30E->PO~!OSJC$l~bH
z6@`UyXT2Pj5`p!M)Tmm}IUL>h5Lb*KGq~ra-ucBZ6NM&`1%U6A-E7Kgz0Xu9K9($j
zXH9xS<pPxyxazudexFC%=tY9T5PPl&5xGfB=G~MoZZhluIeJ?s@m+d7)`Q>O9>2`m
zv^=kwZ!0qgO=zKr8QZ>XvFmd-v?D1jKotDUhz^*82?FMbUbA1`F27J_rqA#+-#kHa
zvtX&r$>JZkvT{1#>NJf{V#Ayr`nNzRgAQ_FvvZhG$zUOivsLDbIHYmU>hF`$tG-o^
zChIRM=+gS1C5$Hcz#|q;ais>>KJiGaCa9q$4U3JSFlbyH@)%ZkIA|aK<Lb3zZ-_k4
z=vmj6i(ZwipD9Lh#(iKM96en-a)sHGXEvDQ!Bd{Vu$nH$aHrKKV$)m}q$f_dt{VS(
zwf`(djj-_<HA-lGoSemL)eVLcB**dao@t@hxm{rx$+lMW36*x;&=mh%miojQ5y?|S
zD`)<3{YG`s0yIz1_K$*&h;$;-sz$}f&inmfFsRPzo&eA`l8wwc+x}BgI0ryfKVn)P
zM2U}nm_mGh+7hfK<s?x8_uGDUKy*cyF+Z@4+bVw*adPE(<mq{T?guT>V<f+Vms=E~
zPA%nUT}s?_KKKDii0?>XZtcCePko==$T~`@n&Xt>=1#ds&R7+dx~$Fv=hP2dV3?R2
zZQ=P^*GI}1b{zUaq^YQ~E@BNoJ-M_I_a3Rw0Aq8p5R;m2r-6J4!{0<c40Ok(xzPER
zsd=Q%@10jZKTQhMaePWpQS7V7YZMJvmA(gI-U~K2OAo!=cg*!o=?5L<%o1~Z?k$b-
zw>y9IjuUbJr47G`x71eZbINEZ%$2E0uG4=NhK)~k&id)X;*%9y`|<XJ!i)w4uAS|p
z8bi(xKDXE&mSc~bJFrQc^U!3GYR-#uqa*da{PnGmEEL%~t<R-&9=G2ws^#bjn(zZb
zHW#;oNks-CM#`FlJZw;nytZr&)^6wOQBO+PUlMDIBp}dVd1n6E?*<6CmtFxyaGQ6e
zQNCo5OhY-GO%!*Lhng)CjLPr?j4_1wPkv*#b+_@NtAA|Um^B4>IOixw<{#423IY~%
z@6$OTF9|oy0zS4HcTw8f&J2L??zwgQ?3Z|WwE2VLtn#HC{5Jb7EwCd|eI?Oiy-_W%
z6=fXq(FAP(e(1B?Rd!deN7o1vT6;b)`T450!R51i*2zj50hzK9$gWu0aGdvgGdsE~
zFxH2ov1j<{dp5*idvM$?EgNmmOm7YK7vz+=&cv^29D1W;!#oOPEd<kzik&WlAi$PF
zeK}5@_81Eg+I!T7A0;sL<H)KUAEdj!G7A0-bH?kYW=O=OL$p?YhE$*;d}0t%p*V+f
zdi!zfX(ar@`+DPksoOg^6uWKIz%6+BsMg6mcMnB8exY4&=7tgX!xnWVWKKJyBW+dM
z!5_z8x1xHTvp(3t`b|>77p{?v%2f9}-NyoLzJ_YxX60Q55jF1nRzINGva}~Ne~(_!
z4=SfoF3IqL$(zQ+-zgfWgl2TNvUQLv5<x=?sEaeG+X8pNKVFY?z5#9}Ot_{`rO7u8
z18ieonIV5RN^eb=C3cb_V*nj!HRDCL3K=qBHn!69#NP45Qf!MA1i}tEQF*_sCt%#<
zvv+)ZneMa3Yv&N0M}JMofk9Vl?tv5ST?dN2<T>2>_%d4;rr#F0v4T)rcVVMeV}tv0
z8ft64A3lZTB?CDvmffLB2>L?scz-lZSdWH=Cx7Z&-oT2VI`2O^dKT0~pS1rsmB^3)
zN4uKX)V3~Vodj7z-Z$tka0rl$SSd|O$PUT^oI7_pU!(fZb|IDPpuGSKwIIiXi@udb
zc)W0Xs$5x4EDg=j*-u3R^uS1Rw#xCvz3!sbOC_^ve(+@)Ym&U12x9qF%&m_5{=p-0
zqSP#ZXe&Y>c*4W%m)c^h9gPAh%#vTP0`QEt+8@9a4_UDv;}X0!HIILrDlM7UeFh1S
zIPmo0NWWodd&?mjg!g?g*B+|n8+<4qRy+jmN@gKbS|kG5u{$dYKlozYwAw+tdUwSh
z6#6hr&f@GAt*B<q;wStWu^*k6?!vG}&YRa@D)49F*4UdFUsuaYyEID0eW-BD&V2#@
zch&?m#jl1m_~;aTz|Ba5u;Qs%3KV^hm2<Y?2_=8}Yy}eL70bJqn5*0PeRJ70))xq>
zA4%7sW4TVZ8uk7}FcCpv1aWI}<@`7`PfpgmyDHV^&HY}U$*yuH5+H5W1WP&TYd}b%
z8EOy7@IldQ)3$8sxq^R-U=dWGYL<r9wnnh(>(vH~`;_UaS%sZ2{_@g+?$K~V;?*_@
z8~M-4OI;zpO#83-P@q_OS_wlriLW2^E%MChBg9!c0y^&+lg3fWTmpWL$OFoq#6@7d
zYnMqG(sZ*`04~BPHo?(+(xMkP3*O58>`y;)jk8=kEKAt=9bu_xnn>3$7Dy_rrr!cQ
z>Dv7wLmgZj`375<B9rmjE4_-VwRI>orL@64(-6`S3Obop$Z;)(F>)9Cn3f>3jTd@$
zS3I->#_C(Jbkg+Gb(HU0EloPp6*!Bath?idQ-$JePOOq$2*?0!kf+GLfSZ7rQbq(Y
zEbVt?xT~YiX52HR6Jd@&vXWaQZ;a<%Id`AE!hFUxpNb<D>{{+FM$uT6_Y{D{woDo!
zLe<1@R;iz#$#PC_{Au6(Ee*+{8S7rpZ-GJNC~@vgO>!22)Vdh8M94s|GwJ8S4(y@<
zlf%`V&qf3z;<3h4IQzHIG&sUiLh2g6)g3o!Mibm$lm-$|3*Jcme>T$#wG(-+@6geO
zVivH}8Li-mgNX)PCw*nk2?$FWLj|drmu*4;84<o^`4$Ce-lQZ=s9!3vpN~PqM>E}x
z^j4n@h{?ye=(*2a^ScMGZI#jJzlnKqkupgON@s7J@z>h!*+Y9{>bpR`yXqw9jr;<i
zke&VMaVe<D-4bf!xt(PDe9boNYTVXH+!9nz#EWuG8}npG6W|z2ZC8l*D9lxVmcfe0
zE5oA**ihYE^5oS2(2@84TbIN31q*?!;*Wo)r1CYe4@yqG2;f+68S@qyL=ME{L7zJZ
z!!HPuD@Fynl<8%C?2K$8oR*DR(TjaUVll})KuC&jg*mkCQeU6z+;^n^*#98bnOz*x
zgDs5I!@wrJCk>e}_v(X>)_Icf$ET<c2g1@7pBk{mz_wy~>1Pkp3}EU{(B#8`N9Q<K
z!D~E`Br%Qq%w1F`)aCY)OQ<L)w1EDF37qQiqi=o)qN~5DW4r%`R6|7?r7{nHv<TTe
zOO0MIxh7{_-m1WtJ$nJ*en>{cQaowKg0xkXO#zme9udoB9tjeqZ$@UIMyK8IG>kfk
zdJv0=%~Jg?{Am_u0r7hReWpS63zp*&-;mv)LX`~=ti0wDJ#IlFXv}tFO1^7C8~9^}
zYh{KElYT%ujbE_!A!UQ<LWU3{kamzZb&QhqZD+SAXb|e6LNh4OxH`}jfOOBNs~Fr$
z?Pl?3G(7Ja95s25s;*NkkjVqHJJV@`neK>2vkVadLp%fyB>CbaICCB|67uJhjA>tW
ze{q>Ip)PKo$vW$VmXtq1nzFM-CX8l$#2lgq2l{0RS$r{Z!2Gsk04PRnftG?6irWa;
zTxJC5ZXR^Vjp2yM3eFoL<%#|gXr{vYD%_vz>pcVfn!E`3S^1_V1mA7{{i2A|eNW&n
zpW@S>9wXT?XOM1h-JI~ouYGqua0~hFE<KsVj0Ibci?~=U?&qh$cPv#2ZpmAE%}B?g
zEMP7sF)Me9)6?xEemreL<R@-H8C+QL2E`#4I9o(!pya9+$dciRcVhpVF0NXG?do#t
z1cKn1Gfkrv_%p*0&D1%w3wGk%9l@YY-eS>y%(6}Vdbc4uUn1<|t@GkPUEp*R;^Vx&
zAA0s=aQ2wOHnqEfdwl_8ty?MY*xABxV@+>6_Ji!(43phWmgt%vMcX~N0)LSRh9<4&
zL*~??0(}wxUyoA=08RqotdQB64U^HsyZG7V(fn0b0Urz|M>BhfNo_(E?FoMKJ2C9n
z?4Bh!E}g(GgpYx@6{fW!9u3Cfs=$~mMUXsWcW8&fp-Ss-jzb$rn)#an^hYFOm34QR
zNzR~QAukKJ0cW+W%~>Nhn7SVw@}Y1)PbK*J93_}on8#jy*klg3nbm?@7DDy=&Rj@K
z&NEi!ZoVP~F{V^+K_5}~>al}Up`YZbo<eL(33eZYjg=00I|>QmSt*Zsi!~K>AKktW
zA_tM~E3$9&LahD(K$+W<-gXN$x^x03L(QsdVQp%?0yRkmXMM4HKa*l`D}j6t;$=|u
zV3-QSHkpb+tz`Q%?PcGJRrE2VXneG#nVFiijcyycq{Hop##`dwrf3uhH-&9)jvB)O
zc64#EJ$ayzhFQp2B-KPVoNcfH?W`%rK&WSD!9qd?vu!exI6?C+RA-^~|5G~dB7G&r
z)yF9lcrEh2PB+e%6umEl(*;A?)8xZF5-pKr{igUSM;bV8NNGv%!CyI@m9QZye}yBl
zG-y7VsCH=UuT~7^&oG<0<Orm^KqBGw86#d#bT^lK!`*UNX$Y1eh?G(@O0u^c!JRBz
zWu$4Uy*4MA0e3ujiRJ?XiG4`}bWNCv94ch}I^Q#yy09P^Yhm3f!Qc@F81X1gi>rZ<
z=T&Gra<jG2xu!W8CLH5TRN>jx3Rm@#p3H~m#W2K_VE0h5CFp!9PCIedT_^V7Qwjvh
zKwHoE0|Q5oP<Ny38%wlE215VW?&MVwg1_Y7V$CW#QVE8|hk?qNpI5lzVY7S?ihi01
zSu*HI{$1#qGuUE-%=m}^blV|fD#Z#r5hgE%j1xqFFYfvH*GrSzN|!3cUcB<CRC_<g
zWEgkLPbrYI<@an_>X3X8*C?|A$UWaUQ4aagn~{-;i8Ppv7&_6ks$xe=6#lv_`Zobq
z2^RO?lsMZ;i(rJX4ST59MUbSNA~88oM>}dAD&=Fe+F!C`bO2AW>q<4*P+~ZYGQ3Di
zQ)EFMnZCxLpDjZoiD)-FJ;7HUDM>5(?lS$$3g6t*_q$*?1nmRezQQnFC63_xmFW8S
z170vp6&sm>Ky<HrNs1#tY8MBE0G0<HCH{Nst1H~ij>*ZZFl}Siv*RFeJKVy;<mw?E
zih6@VNQpAt+&A>|elS%Vnk%@l5#WVgwFjUDDLn~oU0)n_v8d>iBTng$RNM#|C4}Zf
z(p7$*K&!T+(u|24aH8V{cH%z2WD7Dm)?60C(YeVpp5WwE#OyDkQ%-h%>78IOoApDB
zz?Zzmvtc4w;3UXV0c*Fd!Ovkp1_Rgob4~-V1~@Fokm0}?j|nqELdgWpyHzj3N3}Qo
zlchnp3=CifSOkuySx_j!gN@L{$2s&;^GzOYK2?o*Gj~#3Bik`U5%Gdu;u0arcJ7>|
z(;u!a@TCtWiA`T0vrw&x1dCjiq3^=+!>ofw-$qk`<IaMA2kgyLH&85)1y9j}W<~sq
z_|Z4~`#%KwVe^2Q>a8~iR`6hh0}d=$TjlD2f2wHi05K_$HJ2hJ7<oq-F`zJx+RySY
zQ9r6Zgasr@egHFneN>+8=yNlGLwIdmQ`Z_b8tZ_`IJAj!eBxN$X<G7H@Hs>VjEC2^
z`DQ%C?w3w)HBj{B(@Bt2;}BW&>IqOCQlb1~ESh=&O9=<uDUE(rF!3K74MHG~1e~Kr
zA$v>(JY8b}U8u5eh8dhF4X$ktn|0!8)e~*~vWA>WsYaPENWs2(iH$_n&$&HZq+V=2
zmn7x&&Su5QYp~!|fk!1W@6mYrXD|{t^%5dE2-_&%?r+Sos}Du+MVR;G9!&QKUOEnL
z{q4VjgN9lmTWBhc3o63#k4bk8+8IazPRopRk^{K%2aLwvig`ku&fwAhl>BZvU0D9x
z$>$!SzhPjhH@dUppg=X8<w*7vrIg2CWDa747*_M*2x$W)9GJd=V{6WhgFK2}WnHQq
z`ONAHCG)7a>C#S~mFeheMd;#Pr#uGJ&)UZa2JZ_&_WfQhjZ^JxwTWG03AitUiA77%
z?(#q-16?ge%|BhO>oJcV^l=kt$ox5Ci`?&Gb?jVDsmhoRT8krVfBBrwe*Apve#oCG
z03yzeHQMuu5zz<(0j_Nbd@I|y)ty)f!J{u1={@YG5Ka2EWy~#&p%R*B!M4bI*iz@0
zX1zD)Jgujst$_jJ)R}z_q=O)p4Lg=WtSX!G-l-rw!qmiWC@#Yobui*upcPp+L8yLR
z5snh1STDG>W~2QuZDyz#R0BKQrTjBxd)@+V#52=R)4kSwDCYAZlo)+T0xL|HTcl$y
z#6<CL+&{iHSP&6+9;*SaGH;25Daye&M<xj0*aH2W&4@N#NYz|9A)$^j;q>y&*Mo4h
zD{8*ten_oIyy;$ynjtb+ytli;IUx#2g}Gm1f`oFTLWynSu9pFDu9@BOt7Z`*CEz){
zGL;hkXF0Q5hS{eV_n8jI>!9t}#3sbaaU=K!htos`lII-cKrSqT5Nn8ygNWeo1%?JQ
z%GQsAiXo|{j+PbI<(p}&5I==p;76eX!@LC!{CI^KG_hp}wd_o2>py8(MkhYVY}Yq$
zIq?tSxnF+q&SG05xP)r(Irmtx;e3;OI-kqt5^cYi$C|h~$GZDx$l^DU8L=kygVz0i
zcD`e2ru{MM0)Lo#C!b%E<w+tGKc%E`TbxjGPiRn8Z{R@4W^5?xQ2`fmjFm&Vx$#fC
zixyYzp8&MLM;DERe4HMqQqakS-_sU3WT6fVWJ2P6XvUw&fssEoTerc2MPWR;9GZ!v
zB;Xiol%W?ze?%@#0cB12O;Wx3jv^z_K%Tw*fJjdS8Fk-kMKijDl(9jZC*WmV)9%!u
zJw-_a@89l&;hk-X62jIy_(s{GN}W?vS#e5(&~K`4dW#>fNrTn)IonCdCsJ`Id`$Bt
z?jw);t4Jnk#)(;BG}CpkGW6iFk9R=?0QeK~AC3q5z<!s--O^T_3#m~w)UnIZ4fNxJ
z_Q^T!@m^^&_=#8>U>&<H(sePrSuaFe{{buo7NW`~!S1!P{1|z{%p_0ZdJKnbs&v(`
zdJO?Q4?^Kfpx-o87Y_HR2cJfW5URLk8K&$yQ*32tp)hlMxiCu(gp7wxk5NIs5J*_B
zA&96@fXkQ>932pOvhzYg*8zs-;bRM63b+m{!&t~xK)XNDixq}y2_-?Lm`Sr-_|Y{-
zm{0S}yV%afVyRN7>l(iHmfy!SFzw;;XVS}jl*vk710HK={-P}qcq-uIXwA~^fd?QZ
zN5Nw!bHaMQ?ztu{*UXticx!~+Ju(_cDW*6O2(Yh(5CA5UkUhk%6zBKLyy*)vW3Bhb
zGE^v;7l)>d@<$BpmIz>Y77v>5W6v7|?GVz`ms_ijsqtN`A$Lwck~%gLYJ*vCCCI31
zDXum5gG|PfqjmnfzoSC!n?QJ_%&$xG-+BEAdvAaBQ?mILFq5kUR;j;vM!VHj(@fmo
zBfa3leQkmQt%E-Md1TWf7h?U)``HnQvKT4+FoTWEXwlY^&lreDWJl%*bM~XLbQDt}
ziyr{bGx!j!F<vUITl+68?y9*m&Z9&I%!9qTygKAavX&r{GbO$pK!E84`ZZVOpsH*4
z5?yQ8Xxq>wxLImfcXVx)$n1b=)kK>#N_ybhEL5W7OQr>)7T0wngcshK{Tfm18yTQW
zxBCvXI?Ij_JT}v97tB?@w0vcdBiv+Vh(#J#96`*b*QFlOc*}SG37!qjylYm%Q)bFN
zL<TtGO4kH5{s)tm+<9odiE~gDGwd*aMHfwz0E2hH?^`C6aa=VV4C)TkjFmFi{VC8d
z_M^)HNL%%-Picy@*Hf6!L-5yL8K!O6HWGlmvduk`#1LhMPSIiHgf>w(u<s@EOT|{4
z$qtgfJ_!-(qA9X+#V1cZtpCuq9CNhWv4{<BG)@Gf;ERu{=<Z|)g+DNCH+r9>Fj^To
zx*Y{gD?ONsi>m5r`$Ce~S(rr7Sk#k8E6K^D;$Vqfoi)GA^Hqc%ek?$0T$n2>t<Igk
z5k}M!*$!aRD!bE)pw5~JZM{`P8+thi9mEYA)Rlsv{PXwi^z#A6jFfy}YlXt#8XK72
z8beqRCLfjB<4Q->RRZ9vqbEFJvN!KXg~YNNzis6;w#HTGPWD?0OBHLzhqt)!3FLA;
zho#&Xfc<)F`d{pq4RG?`8%oZHK@f|(GuC>F;143GiBRnu8ulUh=Su@&lQ$%;4MU53
z-}40mw~f{O<SIJHEYlJx2X^DYg~^a9`&>9q7fZ3cpUO?yAp~SGRGq!8HqgD8w6E{{
z@uM?;Z~`Ls_3b5>$1M2cl-Dr{wO%t61}Tjh$cH(i^VPEN^+Irzut`td%iw)<S+B9-
z#zn$$8t}s2I`8baT!$&yTR#G3JX&v~$S1(dQ;ilP&>i9pP`4Z?o?>|4JGKwV0MMC6
z*`YTl6pQYnf;MIY$*<w7vpIhlbwsaf-OxATdPT$H05;g*FKBE$;L|7@LVnmNvzIUa
z-oI~t^>`>Nh3%IcRs#p!19=>+)XJQ{Gr?>v*KrUf)7}k0Cj%QHB9qZ0mlcasBovOt
ztGtsk3J+u3meH41@;Ryo)YmoMh2ZXcGg7emn-&sG;ZF(<k**<6G8+6gpC2~~LdifE
zEdBtR8h)-mdr&K+kOYP+_q)<yE;NSrWORkD$cH;AmpHJiQ?*2gds&T91Onj;<K{+N
z4)6GhUW-&p2xlRqA5nsAbpW#M+^$?q)%=+Wf+TDp$YuL_9zwxNU3~zagbfB+?clXt
z5D{MU!%@tZx&fh@{66J5MXv^@$I*tUo<7A_w0Xp}1Y|I>>JR0lLX4{qj3RY&oA}*`
z7fT`4Brvg9V?rRX>GpaFGad}JE>RB4(t)Ej(p^yqJv^nEcZ!Sw#$tCgVonyFMu}NT
zNn1%UtJCMNh<VJYZFlO-yyxCaBaO=a&orwym`a6a#wByUrOueiQbb0;+#U+4F=R3<
zmUl=x6e!CT%RHSYmXCJwNQFomR5~$8>Pnla3w}RIiDphWV>@$Grok+$+0{E?ViDD4
za8o@FAi6n^V<#nm12RzDsRG3vh+l*`L)%dHISZj9#TaN@-$pcl3NstBT@#f5mIUIL
zGuI&F;i9-kBO0`xnl4Yr;o{TF!}8dc10c;(!CVl3ur+bxY*fv<YAnM#S)5j;7D9%C
zWt!Ha5_3^9)B$_Sn&I9YiMH00BUEZVlH9qOmuJX(yMn6wT~ky8s;22RuMJ;l)QyG=
zS^MeK>mKFE1~yRz3;JZ~wT0*x2^{Jz8)2;?ZS@p|8Mz=+NZfBIP?P#l^`l=^cSqZ-
zim0n)!t#Dhrzml&>w1k!(z;_LFlu8UH~@|XuJ@zpY?=_GIYAXSD|*3UB|H6fJ@U^x
zhxmf0R=Or{=_=*MKjzLcMnJ9@$Gec5oKUecR+Q3r&3%nFi4Nz0)&*w+hrS}I;VH={
zGwFn#V`jv=p?oOv0838Y_Q`%lv5iQ)rM|7e0Z}kD>~c!^=30e+%NeQ4gUY(ANUN-4
z)^c<E;m(Sx=OdOtOxNQyetNx9quSnv5&&}G|3IE(&MK;(4%t7drXc9~k`!@FVO3Zf
z!#HA9{%c*c0~>|0E<(}J7}*1AZc&}w@<US4f>XV2Dz0-eAsK!!2aZwkR_def3ay)$
zHoUMTF}B6Sp4MT|E`<r22?(!=xr<ft2kuypd9BF}pdJ9sm9RpdO0f-NO5A2?X{v0<
z^n-I5+gk)E6WYXmB5!Cyc0?5Bl3Tnfz62>!2$y{O?KUr?bU3_s!yJ(~omkUFMi?A%
zR{OPR74ul23=}n7V@UEE+&7Ri+B=!5qa~iHi$?huL!V8jWTZXSLgi>){D<BylL&(D
zN8E!3^^m&c54GY4mgh}l>SV+TNP3%0-!}oJ-o7^y??VPFHv3$ayKz^SI2kv)FZeDI
z%Qz=v3FB=Q)|$kNObp*pt$zkA)lfG}n~>?JwVEs-0wQoFEh2&}6kUuFsLB!U_DQoM
z&t{GzKB7IsB@8j<_hg4}{<EtGz_(rfWqZp>m*LO(pYFc!;p4(igFhTaEj=S$T7SJN
z^|-}xuw}((Tf>a{aA$F|EyOM5YQPU4@u^)kg&|@ZQaT}a7ES#=ZfIl0yaP@l*h*@I
z^E*AM`GAaf`cY^7D~&CBB@(2sV@&_-{RX2>cRa$$>;eBGLlNRB$AA!4My#WGPn2<q
zk;Qc8%bmc8J?#hjK^jmV=AlzhDy&GWiv*vuUY;eOij6N*)x$+^(>J6*mEL(gxp}J5
zgT4G=Ptd#Ov$oTE?MYO*)e1=dydM$3L^$DLg8*WgS4WALz8>bERDkvcY>j_t#eF5R
z@nBR0u^r{G^<~PO@ZcugS-Z@&n#m^_F%#LrZxferZNo01T_>1Eyjgg7r_IuZ*-St?
z{zE!c6#lbtFiwkpC}|H5_qB?6RiN#seK3?X#5J?oGm@%Kw6vowTfo)zeW)5)9SRif
zw!15!h>|Y&M@Oq>2(LbHuh8KM^p}DR+dwTp*ZqJTeD8E*REVrFWD9kTwtYgCT7+z`
zIt;X0)TM0Ec0Ko8q6lyZ#Uv|7__$2KPGjB8t&%AJ*KppNk*eqMmRE70#uZh{!j#)S
zUi<O20F{jgzoyC{gM{{ZTi_h2FwI6DUJn}K0hRy7E8xyGMSuXNXq01M1~viTj}Otg
zkpP5A2E6pqE1UFT44uXxUylSh^TZDUSH$!>FywGSD{Q64a=f?G>}Y6oDy$gAma(N{
zWw-Y#vrl!vyPq#;6T22Lkz7@vdb2&_UpJ*bdID8og$>9C%Qc`9?+(C`WAv*bwmc5U
zsU|l{utXK2LxFbzcS-2O$fLT2q8QMVL<TDsWwKDI7Dc8_nn!hhyTX4vGn-gl;M+Uq
za&?`|+<WmlU0GQunLKtk-+gH(--73TCFHNJ%O(lTS=qyoBO<=apWUv%yT8c}p)MW@
zlu{~J(q;y&jANb<v`pQ5rr2rKK~}aZeQsdp3!WQn-z2<piC2Lexe|j@ywsM=QHNux
zJ;9R*5Y6ijs(*gvSO`;<{bmkkd9X-I&BkBlGwu;{_Gn83=_aMiCVu7m63_y*Z~nT|
zKF)hU>(P4RTH5g%44=NEB=a29n5lf?5}3}m7X5VgVba}+<(p;4q%DmQaHr9;T;wdG
z?b~g9k;!UBW4qQ>wx&-YtO@_q5B_N{4x9A&LWV^#D*{me`}_g7M>_?BqWtGiJ9p`R
zc2d-Iz9Z=qmT5mII>+z9OH8v8px>-?j=1GEzPyIWbfh@Cb~=5(9HLDAL+3vJ!M5ds
z+ShTI#X(P*G1W@Y2lFItThndPO7NBahwuDKTK0TgriHg;#gIhiWfYCEb-9G}k5g*d
z@efXPm7ek5r4<d$md^Go9wfew{nUdVZr<b4=k59R5D=SbL{^u%`CrY;7TF&3CamG?
zl<wPCLH)lT)tY-}!!{Jgbmnifb(RJy?AUTlvryfG6Syj+7NqUs#01Zg62;skFd<*^
zi}TC~<?38HWc?PV`_w<n;(a|{RjlXO^SM{G5K`WI!$Qj?Axt+=c#y+?9>QVz?5@&t
zrLHCS)pK~wUI}V+!tG<cQe~M!+unGH?;30$<+rT0aSQk`1f@){9*0Ot{*2D(h=aHz
zQz>j2lWV;yuSPEYovn1`xfXM3MmYl3ac>Kn+wBL2607qIOo7YnLuEqlyb?0U=Occl
zg17z31U31|mn#7qu%ZW_r$+*!2hl9(QE%N=q7p0oLKG^n%S)0CXFafHN}1rIH~<Yv
zc)EM+WNsejkP$0hhBw8V$al&~^UTlh^|4LgB{&Kz^8T6<nxs~DviM5v=;@>TnR1fq
zAd=)`Qk-pJqVtF+@CfT|ABIF+3GM0zPXY^e-b?+{yJ(NsBaV`Uv;}_O)l@lxENe<r
z?S%=BBl_ggx=?{9)jy!ngtJ~h-Y{OW?3(gc$HWYNU0!@a@%V^Lh4j07(J*<X6myO(
zC*J3}-F8kcrt(kQq;Is(Lu<O0V=wGYI?LMaAx*tVl8i3WZ)!U4y<tjLxkOY5NooJD
zaF<t%PYF-YElIl;$wH>u2ZdzdY)L(r%zThl?o@o|-4YC8W=Us7-f|o21m74c0eSvr
z4A{76gmQMQ_i(RwR6TzCcn!Y&_mi}qjY<r#ldjRxiMLgX`m%WJB8=0{Kc5Jt2QsM6
z_zoAdc;+D2u*aMofzZ0XVeo2@20azq^K#Ru&N_&+Um<+Mx7tF-YM8!o?UYKap>Fxz
z1o~y^>Q$n>V#aQ)L1v_%DbT3U?9#$jE|ro#%&4>THTh~rzL{#RgI@yrr5`r)*9E%S
zwrh$|5ZdyWA_$aI_ieON9{&A(a-naxybt(A6sOSozS7DmN*yxEWE@qk8Jzb`=B~Fz
zZL@2>3lpqW)U4bmnu9Y)o<$7Q88SqSd#s%ml|DsioWqg7|3`b@8P(*rwJR|QC<uy*
zs6Z@$bOiyWCFr(5q)2Z8+)D4g28f9027yiQpi%@RAf3=fMTiguLg)c05dr}cS_pyr
z#(nO&-*?XWcgMKn8+UB}B*2^XuC?Y|b3XH#PZIh{c!(jNab3=nf@rn1+d6qecB2i1
z6SktK6Pqt8-w+k$^Z8(@F?qru)#>WawqS+FXYTHoS{iA|%@@k;sr6~JYi<@bF+<HD
z&>9@I!O%l`ZM?$Fdootua(y0*>gLe+`BrQy!$e5bE;i0^QSEZn*SXyS7_`=mLi<u|
zRPnZJ>I-qGAObd&V%sj^=aK+6r=69MP%JjQctg4CjT3O(Eq*J2DQdnK3Ew9ij4NLK
z5hIXo$Z=}F`bONjPIw>2RqKG4FQbf}eKW~ajq*xodAG{S`H^twxKAl_WvzX&00Wzg
z$=eAWu)BqEzbrwY){34ip%3G4PQ|u`m>53KY0F!m(8#3LhxNQUTGHy<G15s(YD>F2
zsJtaGbf|I9BQ65Ac@Y6B{PcZfC9Jcu^6~)x=`V`pgkZkKmM!R&q|^c@G@D3pC()!L
zWXDEH5qJ*<?*VA0_1-YlZS$gxO~1U8!5>bU`e_|M6+F7n4&X<Fi)((583>SNE_&j*
z?#$Nbr3!I<^Dm6`qSXngJDpzbR(r1_l{?!dCGiwqzE_jzvfo0|T-A&Zx282YdMKv|
zPg0xP(6aA1`1M@+6BD5YN|uMtyY2g9x2gkE;&C@el<Y+-N7XJ9Ch_CZ^K{D9jnj5q
z<HYr$0Cx+gcwtNG)4^CB(Zmyc>d=`N_^lrZ33hEKQ6tHf$GK`}AMaL2t26EJ{;ErZ
zwdqb^rDF+c9yfpg5X;{0EiZF94PN4#DPLA#WUI32-=wG+>3i1#b48c;Sof{8Bb<+g
z1T7Wj)R!6Wlj~>0+*d3o-lGdbYzGW%$IZgHgl7dyjnF;Uflszw<;R=KVe*3}`&uk8
z(O(0d-bl!p@Ed*cG1kg}%G_OHye(E#znr`<=lACd7gi%UC%*ml+{}RFx+LU^On<OU
z+J`*2T<hB%x?b2g_fkV*^zde#VdvEhgTn%bS4NC&$8*EWSCZa`m%C&9YB5*Dc{{qV
zz_q8bDT`<42OF$cl5eh@z!Y-1%m4IJo+r0Vyp!YQeR03I$kUk(_hVjT49=!?ILr`x
znO$B=>jrcoT5&>cDRiR6#7~xL*;6H9)ZMu3Y_04r(!Z<|7ER|_(idnbOu5;Fjq<{l
zZ)`!`Tck2_N1KS~EqWKU`E|*1WB-#4ux=~8R?Zl@zZPz)IwZqRtsLG{HS4bP2{V9X
zwT%e{j39^A2OI2YWAK5`C#H&0?mtN@o3JNf1eRYb;v!&cTI~wT^$T^Eq9WUTm@QcZ
zDw5z5#(yJEb%<nFtP=Q5L)po##&6)mPl8MoCRJ|WI80|R^TX3@Qp)Z`1xYbqD4$uD
z*4o1LTPo5ro>1eu{rQuh&lItxLk#t9{JD5V)}RfohVjj+evy_mn&p*a*Fy}Z(q>=M
zkybAC<ciB4*_&r7t-DeqJ_od|)fO3BQXBmtt}S&BhLYME&#eTI+s)9bjeY3ZfP^U%
z>Z)`AzRZniMKgHtuvQ}c&5cRlHQ(i(-Px5eEA8n=78f7yU4!d=da6Z>x5NeOro~qj
z7x~Hj(e6ykcUrkcAO{+Dh#@|>H8JCQ{fbWfQBK$?cfbk&&PXQ>AJaYcg=Q+*fpEZ`
z^)Pt$Cpyx<LTd0`5%*=bCpGhZ)%}igr{07ll3Hsdbxlhfc@bK|t<SffzUi8yxw4;<
z5y{*9$qp_`sXoUB0xKo7zed!YIuEHt=@Lu*6_GN<emxZB*4q<j=9c>G_yg5<=f{*#
zr5qyrAA+0H?*{ZaTwnvmT!e_&WcxhS<K%-}DUWClwYY)ZcmP3Air{3UQ^@WS;!oEJ
zku(R5)uG<&dQRqfvJXT-S%$yzTf8yM7P$Nv1ZI1XOM3J+9A-;n2KIkg)Y`%X8LBd$
zC+U{&{hZ`AuX?|pvsWje@m_lhB6IGuVmYEJffvmA6#>)G*fO-`0PbCB&lW__<SNg<
z3vxmPueYiyJvu+RG2LYxtv2tG>MM|8LJ$eTmX#{!Zl@PDk%2`#!Mq;uQa|qfmd0mR
z6B=r9kPYQDCw({K0E*{QrK5lh8>)9}ru!w0Yipu-AyUDz`#Rf#n3T@%12&<U-5xcq
zg(hb1hS{0pz#ahKifyy&^O%<lm`66S2RF_W`OPFf7YBsRk=rljT*Qj9=3-S7G@lL_
zJ7Qknug}pE9GFXQLlR0cXPr;2i=d1TOIv8P_DqQM$wICatUf!y2C_u6%mD@~t$4^N
zfA&yWc#k(<QS<9=e>>v5GdmvArfS5Yw))(;tx2dVDB$BL7oV9j8+{w|z~UHS%y}`0
zT;ACh9uHgKldh&c$`tlCGmI@%;)@U-o&$>!k-TL`ug|VF#&2E2cxZTJfMQbP^~$6H
z3ue!;HWLLtwftHXawSVd?+|Toa&F(4_broITFRl-3t2NIu8m5aS@Ks3ag+jczB7Vp
zv>HgzSXT66pPh>dabFtwal~q^99s(Hsu2AC9u8N!>_1$B`K%a#4x`oATkHzeM_VTU
ziex_ybKaM7xB&ulx<XL5n;f|~UGqia5uz~L^WfQ;BVK2gz`$~bE<xmM`17KG4Zae9
zx{d8BW<SQ4Zqnxs=HwfOfnxl>9|s<x!cE!{?Ke5UbP4>pHp~T#`A!RF-72PWgddn3
z<<}yQT=>^TR-(ex{8J8~qWBWx<vu!9@&jX<B>`5{2|XnSi)bJJbx=<I_HVv=kQuln
z$p$)d9<cuN0b6m_#)+}Ka9jF5;6;r`0Gn#aJdbFv@lL%640PzYSxrnT@VK)!YzzXQ
zx(RF?UvY;nA22x`0EjBCnWqDW_YwHzj(q>-jy&r#@5{iP(oZ8oL(l6*=0JckGr_ZS
zE<dWTuuLvn=}6zMfq*~*J?3%1KxgmSewVcXfzJNydSnW?2PfbjH?t3^Y66o}<obdo
zd~|pS?9o@mufwPb{_lYR@VsyFUFhSGN<I*EE$TXt{}W(Lm*WX(6)$iffIUiY`enc;
zod1svSatp&Xh8TK%THkuz?6Wg{MRQi8+I^wCwN{+`QQHa>*9577%bnkc&y|2e|^<I
z$NBr^{{yy8YUdvQz=5lDt=}|_{?)#J8}V-k{`>I$GhY4k$=`wNe?Z{+J6QcoM*jaJ
z+v;9iE%!i{1$U`VK3J~oJc`%k5HfPB#hVPg`z-%|@W;HQ=l9zJ#_@q4FP5rr4!==Z
z|8}daX=2&kXF}wx_%E>HhSCe)8NhRBtfCf`>Kk?Z$}A^JnJHTURDBZ>(XR5l1wK`x
zk{HG?CglWJ`*`XF^%#~qsLcqHz4Px12R+`Xtw5HK`ry=o0(QNST%{{@9+n>tIvg+`
z&=(Kg+g|)|MDAVPg#XxLuSXco%6}HI5^U;6X(ai(hipv*R0z|*-1QG*6>iKE*BMLI
zWtPO9q27)P64DK-D5MBLrXkb(m9335>tHAT%Dp+c@R9ah;Bx};G9E$LjoWofAn+MB
zHZ~|2#1Gyhh3^$(M)7D9<}L~9F|RFC8V>v|xAJ38oAyV>jL3fAz>g^hC}X5Y;F>rv
zv;{Cvi_;2W4AnN8D&8&)VKVXaj&iBSRwJ%AklZ$D^#ofHIwz%CpJQQ6N8STA_A&_B
zW?zu7aF)1zCuNtR95$nT82b##T+j=*U`}yB-}+Amk$6J5{j(3C_W6H?2qWh+985UQ
zdA9S-+R25O`wc$0%L5I$716%K#mV_DuTatL6{zc+&wazlaB_J)+IoM7V7k>rWDd>U
zV+RjKeb?q(x6WVTJxY@vH-8!dL&H5n#>P^2g)Vst39z9=r8W2<`bXC3nO@XxanpF@
z(xWn)-5&zRhjZKmn#SCI5<ljwgpw9-iR|dtGB?LeiHylDrS<p2K_GCL%r}yn3aI2c
zozb?>(7Bz32$-ndnH7hcBghs6Dh((C6gW9KT@M+RjZ?w>wKzqz*UO^X_F$o$i1uJD
z3tm(haN~}LlToTMe0)E5Hax0(m5iNV9_I92NWYU$1R-1P)krsRCAsSb;)>9S_|Vyl
zcp-{sG!OI9Z~zfm=3D&d&{ro>$T3Le8v9Q6-Lz`u)7+)F@I4xy$0M+FQQH(_kg)_5
zLVl<C_{<#i%Pv+Lu(6?1v@_xlptL~9FwgXW>9~mY0-K^0phEhMjlj)@y1k1S75~TL
zF!60DFtqL|g}f6;seAh96yNaizSob26F@)>LaiBb0v!?mM&jULLs9bHoB~bA79IbY
z)ZV4B`MyZ?y<FH@4Bmgt&6L?8va9Kb3z@9mS&I*EF4c^x&JAd#<={UF*jU-QmXocw
zFn9#TJ8iLbtHr}_PRb_ec7e_I2Z2&T^Sq+5^+2r-1~u%_#w?9iUwYV^zcgc#IJzR}
z;ZmIAUn)#Zh`xpmn>r}+wo1>2-eX$oV#654i!i^77_)rWnz`w*?@w~py)l<u6q6rH
zj|*siqe6aHok^FU+OZB<u6>-rUAG$@z8wuO@<ZOMcU=j>g_Oc&8w+;u6=C@Falqhc
zO4wFYyv%U-b%mAieMT03xnU*_pT``@^DD>nz}9S-i>6sC-mXsU`UV!#B&+20^<rqt
z&fZoVG4`(ij~C*p-JOJJjrCXfqpC|+c9(cq48u1PW9Ji8m0TSS9N79&TAyk~H!i8z
z#<PPRzp_VJLF0PLquM3GV6X|wZs^mAZ9+JcP~_3LG~w(Xs;9bKv#3g>>(U_J!w;i*
zcqwq39fJ+M{dK)g+_JKzTdlF*`uDr_;yR(>pH7+@338H%`%Gfl3a}8oj=Oi#yylE1
zZ?L|CS)Vq?$-b)a{jG2t<h(*&iEPVyf00PbJMyvkz~5;4ieBQOS`qE1&L@Bp)=yS=
zIv@`R7+e%*CKh>0(8CU8>~`yg87#l=62?!4tRO`5`WRYHHBqp`O_QvHf;4nM5Sk`8
zN}=>5XvtBv)%N#gTKl*>=8{9!iVw2e&;A;7tc_`z60X&~W!RuBp>vk+OT)n+7##=&
z!zX+{T|zd_pBE)$Lzx3mg#T2}fF86(;;j@zp!38zF+>=B4o_rG5RLcOME0*r5B3Bs
zgV0skrXX;U!1W)`FP56NEE;FMo2m~3zDX(asEXrt$``|2YVx8WF7UG$&F;hg-palb
zt!TALs&Gn;P&lic)Nh!#Gq`Au3pO$I)KDQRRs{9h-;ezMgf4GG{g~FaJEwyR*}RB)
zzQ0qkFKNSTlLSTn;Lm%D_D(6|E>2PYb;IVC(A@PM-u+cx9nXj3YTbnW?fCr$E(70Q
zpJLUClEy>3^B#NiVm9ItYawl0A^7;P^*8fPV2&!ZzxFGAt0aNqv-#s&^%b|ev+kxj
z$w7BCI?`h0SjRjNL?Q35WPiLcCyJ2RnoXj%RJo!rE*zpPdxVGheHL^uYMh(^&Xe<#
zZmy?ob21)I6b~Oa?D{QU^%@r+omB>MCB!Caf4)_b?)U^Z5)Vdh5ySWU=I)4#nPpA&
zHIj12#w|&?JJHbHhXXBRKhx#Jb9@apw!5HVz9)N0Rsq(GQGycrp8octX<IP=S6!Go
z<I*v+Bp4GWeQfv}C_d=xqeB}anzRa8GxN3ggS_W^<^6iIM6rVce@^f81Y=#0hAodL
zwL}&zbRvVtQGI@R#e_o}JYh5hVq3vUF%sixrw{$=i!N^crbyJYLdkLQ4X=WtMZ&g+
z)uyH*d4<DF1t_uVLl}Efe(Rq?eU{W(s)fK!Zh-uLk4=-)y*~X^b0W`y6r5Nw6!pVy
zNXI%}W$~?jBj(w{)aw}8GfQ=2?m6U=BLRo}M7Fb}^SJz`4N3BI*GDV-rgcdw@@uK}
z&2t|&2HD|}0?kwGkJ4}ggvLI<Ouum;ubpXLd+Bj*P|--0U@qYN5}_w_<+Q9J6yn=w
zN(d9`GbEf_&MsE>0@9HCd_@<Zqn`C9vJXDyv5pjaetNKD;RU(7(M2vG#&Z01MjP<<
z))0yq`db<R6twe&9YeR3)0;8wf2zEdYUN(kJj3Bv5AIx{2B<7#KA@7nrZI?^L%aD$
zr)0Y8d2h@nXd-TfE$!h`Jf4kV)yYCDfn6O3g;ygwdfol2r`Kom{5rHqDx(a2g0DyF
z2oQxi{2un%<{^RamAJ?-i|6^)7-x0SBb+g&yuUa#@0pQB+@0@Sm}6jIr9MI#=`*f2
z9%Pdvktz-H4LBr4VK|7C6HU%ZCW`M%(`Pm+!kMCk#^C%J(JtS`l4AENIlsJ7_~wlu
z?;+FrK{I*vF2njmBaeglfvm#KwzUDr-9nW3q${VYRW%Q>Z^0de>FaOrs+XE?*44Kx
z=c6O4yMrfJpDACXKQgxoLIN4!MpaSA)bOMDiN|#L@St+b0G`GC4`SFl@z6!XyfaHx
zSXq8Tii^3r85M%FVP%@J7+#`-dg0en*S6;Q;5`7ogkP^v1{WQBTV(!49(+n`%C911
z;!1V7%Q2aeS`6Kl7LgGB6eIR?wFxu7=b3@2oAV5w4BhS&ahIQe+&5<<5n<s3%SSU6
z-o6>^bQ`R_$H5!68kxlr{T<Q)boCGljtMchySJ;MyrEMEIU1)@K#ie!^h+Rc-F>zw
zRXC0ue3Ei5VBAY*GE<F?4rp;Pv`q!A1Aco2*W0^U0Q2GEyc93X?$)%@8j?g5C&~o{
zSd&NZz<Fo)!!3Xt1VON!UFsCCcujOXS9UU4C+?z#pU!ySqwbH=NjKY0_}lGm=lb;B
zh_oXAarbb44+8^_eNU#cvw_O^-&(5jXrK4mn_o1sgPSCTjQfiwd9k4|e&kh;YwXf~
zGTqa`i=%(^wm8zyqFt1Zwz;&GSf1rMZq^$r@m#Otj;cD41kbu(WZy-U2=RRh&fT3A
z=dEHUCRU^}@3p^jcQbBX3`K5M=c#A62^%%|<a+Vv(Z4zzP1AuuaK}YO^S>W$Xamxx
ztc@Er^T|#XZW;3V2$0dI*BQk9y)0>h;&~jqxhbG@AiE3zJ^}-LBr7+$Sbf>4(>ra}
zg=;u9)Vq)kB_E!uQ;9*YWUjb}&Yb049BS)i>>520)G29B4Dw;#4}we1&I-Ol2q&$O
zDWi7;gu?oKZe>6_M#lw;V;5(W^xP@&)uECUUv^})R0%PZmPNEW9)a5<H7pJt?{smA
zdruj8-IBfqHW7tDDkWtV<X*#^X_p@Lwy`iwC*teNV2S+h03h#%tMAIzZeitTE>v}9
z0F8g+6uRjdzPDMPJcPWMFRbjzYnsN9abIdoM1XI5v%YOY4QAI0zH~=qanW7k885Oi
zAFuQ6*%;6WK*`*KP4~@9jnj7sZ5nTzrUeU3>{UhOSb607*Efczm1JuVf+QZ7HPxIR
zO4V`ew@T8s&Ku%`Zg=o1nd2|8gBLYEWKGc>UlOhK49T_U;KKJ(;TzjWJLzO}Qs!34
z$!xNRfkchk-u6tIWy%V`6TfP=Vd8Cg_9^{O>ZInvs&-`p*uma73kyL!f}eY_HBQ-F
z$07-jxO|OV(j_)L%ea3pqSjR$^&D+Pi8g%uz138fR(BA@^~1l=UpVuWYf^s=;7w)G
z52p>NOru>vokkD2D40a0U|JPAm>cz3Ofwd9q*l`+ECr@xadr@q8D|;V?F*poa~gbR
zfq=0AdttfWH9b3P4K?BCYalWy%FPvewzXD~lu(i?@1pFLEIf2~npZ@3e*_n1qBPPx
z?=zt|me}sodQXaMrE1uo=Qn@m#?G$3iBW*%HLLL4BX<m&{O(QEe*a16B`vuGCx59k
z$W1wgT@ImPLK+`dLfEFq+u%Sv;Nwfb(<^^OG=W<|d9eG+fNOoh<N3J@353mFde-E6
z(R9F!ut85S6_XhOV}Me#7-&%bl27J6hqlk`;8DbRJhGxLz+t<UvgH?eF3I`*Q2~}s
zc|`L)9cU*g(pwk5=mbbCX}i(AO2FWLV}12g9b(mm-`-8|=&9$=iP36~RL@V?eG^k9
zWF|zvva0Z&=-F^-96ZYTI75AF+(+xY1sA6pP(LBuja$WGwoxEf?X%iiDeO_?Qxd#e
zI<1CaL;1JcTIrvc(!HJKEuuP#ys18kuQbqA5A-C3J^lgRn}HtPDx|msCDBLOP+k+?
zMOS*mW^FLu>B&VroHdsBcYBGuy~8>MVvd^^Ct&xrJ&<I}Xhych+vOp41pB?2A)ryD
z``H&1BQez)MnxiTxGYnXQx1U*JSSzRDmdaq=Bt<6{Uu)~yWDY}t`HNQ(ofY9aDOAw
z@xwqiWvB{KBFtgi5NC8@S$pF&tTSkFtqEf<SW|JaINMvBq$YOUOcw^T1zKtzoT%Ug
zwAjFX;wRC$XDyG1gWLobYqFT=*_)NX_Pftw{rTC1)3gTztYG=;KABI7{AoWocNROI
zMUVAITUmY%0a-qa_>j|U*ifBl)3r?2NQYZbhP6A&S%N@Ghrf)W#?T*DULNQB;+3|>
zW^ly6@!A=_doHp5lE~Mx%3_)!g23rWm)74yi`6<JWFf0h#nm3BZxyc0c}YB~Nmlc&
zA@}$`)NjoiEDk$N$A)f1%w)Mbi?X3k^d$Q|ZhoreG#UZ>Y<qd^EyqFOrWdnocCA?1
zCfB7<`US*`)F3SRQW-jhi_^Cx{SMGc8z<<sR>(QH11q!0tTp7a5D$0TD)`n4)rl#`
z7x>alsAP0cdWK0WbUyvjXEX-PV=y_8Lh-L};vpoNJVr>Qq#Y0K+w^{!$a()F1ajqN
zrrjxBh~kq(!6LeeldkcFh8KOQu{Vy56#SN{;MRCXZ4PG2&*uo`zJyVg)j7;rkm@)>
zT8~tgNQrLq`aYwD-HAJZ`gH1YIPq`*1~*U>rMg^OnDvJOPR%DF^%2**BJ7J@mo1;R
z)CgEecGksswW^kddFF*f;C_+KyU?BQnuox>isk9avN9%T`OMG>?))cM3BMM0SgEIF
z1}F#A{poWc(46oMvUXugY;^yI9HFqP<$JqklmDan1q+P<ZTP#d1IDLY6M-H1J%u^!
zM?iE52ehVdX8w8AxJRiU<21fj5K|~mENIu<6y4gf%b%4_a!5Rfbxw?Ecdq#gx7`B(
zu{ehtRxmRFHpPPHQ{z53p#r-VxHUDggN5bit*;|yQ`B0eAgciP5wy)@!6fhNaO<%-
zwPj)HQH7Kzd%kCLkb-I6;W^CiK%3(zC!ppkdQkoK16K`)7?%B#1XEEw@+&8`#4c~E
zI4>-ihB$QfE6GczF*Czrzg!HW|AgOszd+Mkj2(QOtB*QIY~YB$FL!WWTHv~)P+UFG
z2t5Em0=%uyNy_YcF+Ao`yn3DBdfja8j5Xmn-*#F!J?)|Oq+hRjK}5S~nw9)LHY<s3
z5z_n7h;~*n%d;xrAi(A7`ip4lY_?~ixR5JYAQ}brH>lbA?zmXfGDXL*+g5LA(pxts
zmSc-jz~HL;g4|(^9e%~YiFetM4LCQ)@QuDEVV3~NAg^I(!!!sn_)BakykkaMdnn+W
za^W^Mecu4J6BOWLH~`q+pGeVw=W6VXp5*(ow*pjny&G~TjoYXx70I{GwAOJXK-WI<
zM-O1Ld^=2g0;+no=3s@fWJ%LTZS!#{lrf8)`2+hkM)Dg4x_(iF{$+}al@rl;{`lK;
zf1&>CCN0s$%n$h~1$ZzRDE*tb>(!k?pJ=h6fI8TtA?|w&t^p?)WPJf<i#?{_V929n
zI<0%iQ(Aec+tBc!c$6)xX5v_Im*qVOl<<TrBGzyi1mTn%sv4#IrUbFAgu4os#r%9d
zU3-;rmcb(DBG=Db->2#)t^m%O?kjU{V>ROdY609Ydb8W5|AUj2g)HQoFl_fhe6Y~~
zF8px4I1?UGo74Y${t`Ba;#>Rzo4R2gI32ZYd~L<_IN!aPqN;K&RKhz?z(^e&B5LDz
zJ|iwO)D{#Yx#QsyzS>f8Rm3^PiMusTb*Jk?Gh`w9mzs~8g#x(`OC+RKUyla@N1*GL
z^q*I`<WH~J?0&N0?w|0@<uQ2EHcOZRfhqz&Nj+oV5xgs1&h7p3z9%H|ptyd5U%~AW
z7ORnZOSrYJV$RMI+hT7c3K{@tru%Q~Gott%IS%w+fGGd$zRs`M3wyqxE1tIk0AEGr
zQBgyL6Y*0S7iWz~!5x?TwbOvNj`sE%98g$5y$P)!wYa^ET+0Ol0>2}XPjY-g13GYu
zQe){|IPYNEp~EN&2jmm};ja;hCK+v5{)Ei}lH!x6d3fZGp_8Nx<cj2s0_qdrFhBu5
zi#oY*v|LI?fG=>UI#<!^?v8D)7{IHEoJC*UY8qhq=JBu6TN}JVUv7!HJCC~Dk>pEp
zD=?;781AIXlHsO)%U2I$EVN_CawFQW3l4Be=SDDgoGY!q)n(TPfx!j|Hq<#QxR0Bl
z&azhrkd&*wT24|^r!hS$V-+RY!5!=ZvFNbv8xtETbhtULfPm+|108>^40dMGoRKnj
zYyl$HUB0giy0@kS_2~Z1Fd;Abxw}TOr@D?W{7_cwv`HcWN&>&fSOdVq>CbPqF9Ek4
zJcj{1fBc=ePW$oqshoU2^Zv>^J}+IY9;NFQ*TPIKDk5MQtOOx@utr1&VD!Ey7G2ZR
zC7eZcL@d=uwErBTWwieegp^x)F(c@87T1+7URcL)>^U$ciW?2zXWnhI{!Riv;J^Mg
z*r4+LXzLpGjtSsutrnZgwb#W9H*FIEDpFOv7c<+m3F+QyCaI?ZhU0a@XOCu&kWT#E
z{~^$J+RE$G$w=zCqy-k$V_Gwc+?uF2*OP?Jjc5Xg>HYuCLLPvQd3JfuLS98ycccQy
z?b->-<!&Cq16fKliG}QDpHJRP(=6G&Apy|MgtHX5snioz!f1D9@lkK=CZkSWj#3u9
zzfB2f3rTvR0bn(c%^ME}5<A+~3+3D<N%o;u%O&3rB6MI=tYX5`GN6TwxNEAoYlXSm
z6KP<I0Z4RZtN77<#Lc2(!&LSYly~9Nx`#`9HXu=#C1Mw}=Q0z62>mx9G6OxEn8QJ%
zhzGI$O%FE9gsz<yXunLGuf2A+6<_X-&sE!G6594QrZwyna9&z4+w9l{@Tpr<07G|n
zPllwSfIL<|Kxo~pojz^ZorVE{ScS<U<>qOgXmZf#K<<$Br&T27mAD^AYRGEh)?90I
znK%3pJai?5sOtaiqd#m)Zl5a0F;efZy(%9iZKs;#>}uc0Eo?Kd1Am`VWOZ<qJ?Iy%
zc`Y%V=Zo8$SP82y$xcRo*=AcXKo``vZj^T-;Tq$3@A9vf)`ks6apDsO023}<2Y7Y&
z*2dgYFQB|e1;N9DH%A=O5av`2kPe$%-a2)CQvo=vXN<&2xl5odH~^V^x-C||vI}oD
zbSVmwSe3rm;Vf?TsC}*F7;OnhU=u3Zeju%MV30A%$SgY%%%M}dQqR0Q1O@o+1>CXA
z1@L9{LI~O=8h3+~eNz3JuZ+tXfQgjJ_axoo*Zal-08erThtBEzB00U-wS}{9W}|&`
z$X3PO#r$SJd&o3>edo#IT2oV0_>P#&*n*^02oMEScQt6`mmvC^QjrJa&!*)g)wuq6
zyx$X6AUffw{m@wOfI^q5#kcD!;Z0)h1h1EUD+++@MBpCESCD)$9LF2{Eix0d2nOkD
z&}TI%NGvhPx(NgtiD41hk<0bOu<I*--W9%auQgrs>HtQG2`T=jLLX~L<>We@4Een-
zr8PY%{;`d`bvuhlo<41}Q1n&`sG!X6N-_(6=sZd9Zi3HchSpmJXqxW>l(U*EDr@&@
zEv${+>!H+@vLO`w^A-H?!WhGMk51yPI8tJiO!kRW7n4xyx%yF>{>W4~(@)6z%lTUa
zvJC$Nbk?~Iq2;WMMJf}2oT>*9bQ|iN!{OJ$JKv&ZvrGff!@mK_yO_U>%ie<UPYMk(
z=rL1nJTx0TsJlRw9iUaHE5R9i*XRLS%ZF9OvpsKSsD-x7Cc+ObXEGeJy*{rtZD#ok
z>K@+oX|ZtAwr#ArH1;rB=#U_PwN#<y8#&|x9AT4j$M!bB!b|-wKX8Zd`f|;{vy8Cr
zT^zt{PowKaT&c*Ndlc;wx8%9eeRt2uv)9Oz?oG?7_i<{7a&CjH(xpyx6GbHuUuKP0
zyE6Mwl2>}JsW}dixe|ppE2DWxI&S4ww*9Wsg5k0^3q?P&cnI{Zt1p2<M;0h_s9f1~
zo}~Q9qi5!UH%ctKW!=m<@dv4hDJmDFWtczER}8R8$^n#n5`nm8327Ei$a+!(<}+Bu
zw^Cxu9%L@qSxTtEMP30QGXBuQ&>JMrW!k$-kc1&8yFImgcKJK3IM=%&#U6xM0D5Y@
zo*A*?31my>nXpFpq3r15<HMK9LZ<x6n(-$#Q|9|Kpak@NY@64_F~8*`CxB10&Yuv~
zr-!(;)pZ?NkW7}f-2HhMs=^JbOA|rnbqseE&T=&J3M1^-eK4&H+HcrU-{*Qrt``Z|
zlmh?r56oMc!BZ%XI7BbNT%&cUA2zFk@_>VIpo?s?zw@f|rAO#Y-zk0xFt+-RIjN*L
zSH<6kdfNg~6M4a|anl9u#1asmpK5Jos1?*2H7x;O{MX&pqwYl4=os;5C5g{I5@X|T
z^%I$lZQp=2#!yYWGm+-dT4O=h&fESd{){X^h6@CDkJ01~_7q2rVr}-Q@5_^`C)Es-
z=Kwz1MF}U)#eL~*u6*G*ku<q0rrh{Kb_S#S)Jf_%(C0apl%ezlcKD2TQ3hc=p;*7L
z;KWW%TR1f-&#KjeI@%XHMvD*G2WWfmZYn(?Gill0;SILP-+3nqyAC8e?&qP=s^bs4
z!{X6Wc|Ir$t@Gc-RGmS9Do&1Nd5O#-Pf`rfBCE8_Ni0tFvZUT1#DhJsHO=#96f%1{
zHK}U`NyYCfn3)!NGyN>etf<H>WN(up6fpk2<fLU$`K1cnEyIzmDl0}U8I$6)Jmi+q
z))E&=IO<4BHw;wFOqI$$BVZ+nh{u6B@5?1TdB$xqyw`*qmJl41?7&g$H6$Pj<)ob?
zFYqY2n*oL?KtiaQa|X^=itNKxq1`Hp4n?+?P6pp;0!flxIl%q$`P{F^Ci9kTOmj_!
zuI^-ptb^~2v9H)67s4*%*pm9kh<lqItlp~5{X}uG(_?rQ2z?i5{cPB0oS2Fw$6Xkc
zL1fwI_3!(p<|NCu?NW<8OUP%EDOv`emjFusI#7Wqs|zgrsG9A>wiKPTQY|+HplGM4
znJR|4Su54?q9#s?R?zQYs;%EJOShXu6Ow6{ctz6Wzl0LJmz^4QLw9FHMCuO@aNR!v
zkP}~KurDtqe4t$$kP)2y)t}lLn~;B;8@o4|G8&nb;-?b~bWfmTwV6eY6js-4dhfAN
z5GSiYcF4(l6R2Qc(cKMT5GmaN-jOZ&>y{xl$ISv+T-}DPTaKzb&?r744p?$mBBRbz
zymlTQ;JF;Ch^R@Sg!d!#F#TFO`{=z2IBaUQlUJl5!4YV(StOE3=Pw|X3jFs9l95_G
zd7EL6n*_TcitA&Fg#beNQYUXaPXUO`$2sQV&gz9Y3Oe~r$8u|@LbybYD($O6*6lc;
zlU;OW6+>uoCrB#Yh}*n6*Af8}&w2qKLO>$FS_4h5NLFuW!5zZf(91Hocr^${eGR_r
zgKHU04!ttLV<!~kW;!ZZj|LJlKqcN(upFXw@P<`#nY=43dwODTR#-#EeP9D;+Dc7%
zk9vV-!Y8xU6k78b<D~gXMZH`Bd5AA{Z(BBCVnIa=;2hIm0uj2fkLybOq+A>$+dX8o
z1gHWT^ZwfPQ1iQ#PptF8-KGs|c-J&xr*D4`J`JwKZp|rbMkF5-4a}j;89ag#=gz3g
z+GR-qQr?23OqkD;-Y($~SHn|-venuN53PD?l~9cDq!?FKoEXrdjzz!en$BpwfFgde
z+Thg&z?cHpAnyZX#M#<Xm&e9_SMp0;9Cd07G2Ir%&}}1g8$?-2spV;-jZcoHe&){)
zpk9jW3s@WZwe=YzAXfOO_r~hh+QuA$#VQFH`v7ABdm}%#o5lhWrhex=jS1UUc+8@^
zS1AUYy9lu5Ws6VNO3@4Ecog;p`TEY!H)T8L-)0ocj+ZR9yQORlDi0bX%<xO_iLQ;0
zezA-y*-8)UmnB2$mmSor(jy<sFtCvE3neGM&z5+VyL>hAOH(&SMz%cO!}#0J<nD%u
zmdF~FE`qAXD%!rkbT`5)uSuD|yRo+vf)?;Ai?Fv@<al;cM%BGY#moRd@l}x|A-48>
z<x<kl!{VN+j~m_UMv6)!#f%XQ$8It2K6#u5pksK90rcQ}W_l>J%jaM;5d&yKugC%A
z6D`putZ5^kVl4Ete6iinxSq3Y(S0FRI+;PnL|GXVR&C^wg(B2*CIR>nwa8g|nY*Yy
z{t<gDqWLKIHQe=}L+=irGe*34TWYBtn?n=;n@9rG?_X+XmeR%LxPC+RU;5&FQxICP
zMP=e>Dv<WXAo1409~Or=j3<uQmE5CE^M(g5q+QHgbiZDv+`9I%cQBZ5x|A{*{P?H{
zmunc%X7BZIJ@S{51ST`p(SiG=qcM@3%6$J1ZS^LI`m=CeF8F&)=}ep|I>RhrdT9OV
zd*3+VEl)8KHS6dme;2V?p9z@<gm(Z&eg~xctQ0~IpqK`PC#VZk9k1>Min5;0fvpr$
z`KjuFAY)z}n5Q!!?;x2nC`}#;tY#E?QDZMl)_9;r4<m?K?}wKKFJ&&KRD{~4Q1X)&
zHUBJ2($i=RT7iP90hvI0ZNoAk{02tNJsjxybKyn6FtCX9n-D^*DS(`f)lUZ-^G=P$
zM#_J3mcGeaQ<i2sl4&p)+v&H@AhwlqWC5S4n69@4;H0u($hN&z#MSEb`?l4G0VRZq
zwbWmt8So!K^>kr6g{5^%<-f<%Gg#H!pMtI#e+O`$z`X^JG*l2z>;i<2o!_Wau`#PI
z`PCuE{P;uanF3<j*|f}m#4gQwpSe{5^wcZuZKtQxCi&7M0m;p|Z*GadM0D#9VAHxH
zILsqJ^!@7(e8c@<7SnNuAF>S?(2OPc!7GiqR_}u5Ay;kzMicyf>;y2U|1LoO>$T}=
zz}U^RZfSr0`fpm}ulfK&e>SXi`ycoE*9+!>9>W#6pfQ%o{?jcKSkeD{ozAb_|Mv?}
zK>(xj6+Y?zm;L|kf_82|C2jxQ&G0`?9?+s#1O7uxtVjIk)&0Hvzpekb^Z$MJ|6}y{
kJM{e>`TuVc-|GH>Si-{xKXJ;22Y?@~+q%D@Z`wZjU(8sPF#rGn

diff --git a/docs/figures/widedeep_logo.png b/docs/figures/widedeep_logo.png
deleted file mode 100644
index 2e7fbe6eb765aaf238395a8c79c0bc07d81ae517..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 85253
zcmeFZcUV)~*De~6Do7Cpr3FDjq$<6`Mnt43AVm;RdJPbINhm5C5uzZygd!jyz1M&U
zNSEHb^b&eax$(D6+~@ngd!PI7dCq}{hsDa8bIm#CJKph*F{g$-)>6H6k@+G30Jx;~
z@V*WJKr#jZTzE-AhPzXy=sphsTwJhIR(`CetjzJ)4P<TSXaxW~42y{)*VFA_4qTZz
zc>n(WD~gAd4U_|v9JdZ0e=>MNsm781DxU7jX6M4i!i$`ohRsjuNev7XM}iHDGscRA
zzGtL;2`6Q_;}@hlHtjXzw}L+IPC=rM;fU@8sTBm^y|TBC9k&K4hk6uOIQ5QoE6>eR
z7X;De$E13S7usYSO*$!QX^H;U=!M1|dl89>&z3UpA0lV5`9Dbhk^%y5Q_f!rN`}97
z;k#$Ln^npRcpOUKF#nZx?b_^v_#4WX?^i?L4RW7!UCVhVk|^!Wz_u1rOb_tB4C7J1
z5NH{)YE**~{YJ~n`hq1)kzdK1^eekRy>_7RWv1@$ot^B9O0Rn8{CEMDWM4NwB2F~U
zldxB^pzK1L9Sj~3R5J$ran^f#n7)G3*{GP@_;)XYsiPh;UJL8iu*7a#S(5V^MctEe
zjA{*bJ$Gf~CAQ#B5IR;%wj}D4Fsu;=dw#!}^A=t6NI`09RgJL|oxyDs%a;2t=Fa}S
z#yxHEs|FwFGl#-onPg%8pSVxmTzP9@qg^V?b;rB2O*J=;mow&+>K4(At}8mG!lnbO
zQxu<XlEn3vMsK7ysnhKCyvFIrc0MozMF2Y6@#V*(0alO|znyjUfNA{ow4p9==0^wc
z`xdDRag&<RwfeLjHL}Jg+E*O3RNnwHbJUJREcXHLxlBHa=)cph6xHU|VpuAc{2swj
zT79|kkUBOcg3WEOZ5#RRgExy5MCitHF7NXr_Mf*As@obJ9|g0h1GE1Eh_Y*}z4HVp
z@&eLIB8uKGX|4XS5TfQFzGx26OrxR>x?cR(9+=p<iK;`A;#H9Cd%!Z0JY5h!#|5h9
z%f*+k1ivh%cO;83=lM)@-h9J_L^OzDj=CC97|1e5<4sYk1oWnS)8r>itVBn{sqFrc
zVIbt)4fXFdFO}Kf1FsOP{6+cr-bXSu4$^XxXP2%jXFa}E&XV^U5*(>)lcxTd#~rvH
zM3yG;!{!=kv^m+A3rS5PFT}fpLgsC~1(V5y0{7>Mz2!`Z3|h$NADmG8lQv%PY7v@$
zaYZpVE&g`E$E$B6_LN?yUw)E%`?~%0Jc_50c@Ks&pFQ6CZ2Z}Ih-00BIXpYl$d<31
zMVgK+WI2;5EiQdDov;7AKVlWUYP`yRKzGLS=3RfA;Y*XdCK|Hbm&)jL|K@GqY-Mdp
zZl7vCcrjy9V#qymW#iqYZ`|gCCE+C-#giqM#`GO|Dw$;7wakmU#!7mbT#F3tYQDNq
zxj?g^wdlNX<^tTk<1JUuHUotJ9XDq%-?dA=ld~hS+qcU_a;G_d?hyqS1%=P6ORwZ!
zF@E*<3;5#I=Le#<Y8f)a*1pAjyZBOWS9X_vx8UvDw?S{iAH8~OX!ZKh>&_Q3;dkWj
zz#oc6k#>ZBysz_qB>L#XGqqziP={qH<#>4V!`>T3Jfl&4QSkQ)c>_-ip4Q|M4=?3C
z&5OvBcxv{9;)=z^(0i~bZQdN#ER!5+9ZEfhJlYu5=MM@p*mFFegbuITSK7a@H?rRt
zqWSz>XCr6cmRnRIF;HJIKRqSuaNvo%axHC&wwdOvvVu0!!u)zRwIEl@lWxgxtLD_X
zgF|dX7Q;-r2DxxOq;*C-w}zldbWM=YE1$SipTu6#UeWpTiox}q>Et6Ni`qAFZyskp
z%v4&8Y_rjf&4ydQu@1A2UbYY@kA1EGYUHm1wr499WH}<)b($;MLWvN|vkOj!lYhzR
z_?37@96jB7!S*}_z?R#qpHe0rQmgS?Xxe@H-n4G*1z&eLTDcO_gc-XACVmgb(3p-q
z^Aw9qYYrRa>wE`Z8(yohYw^tAu6?;@E9~v~P}Gton>qG+<#n3tYu8^h3*5~U@io^F
zwq^-`S9Pt51uA@3EKJ16V!(0$l=q@jtVg0iB*5~4_`ayIxxSO<B&$QOc%SgBGo*#M
z(xsfvNYBWm>XkqWe{1ZdfQ*f~b#?dq?$4?HRw|_nhD*<ejSdY(3Y5PX7)cl*6757q
zT&+90j}m4bS)J}V9gWGBXN{AM7nVAe1X$R-Xs^mIt#i_@Jgzcxo^`}dG!H1P7Pri&
za!ubIa*eP^S~TFX;YrYK7z`}?0507ZUl9}!Ve2(*Z(I#_-SsrucKf;%ogeKGxj{dw
zF!g2X-V{ysVc)%!TgeqR`bBj?GJOG6epTdMT;mpTs#Q6SJbh&o`?+!HaZL_Iqu@v>
zo)qx={m#iY<K=+v^AK2mpAKB#>?0@;^Z^tK>=9}!8bF32f}n}e6zt<bi>8IZ5k(Di
z7jrdpCWFwfl@Yhl;N*~4vJ@(AQdLp|GKEVe6dzufzxtZ4l%Dh#utz*qAYu9n+1Kh9
zNnhDRlimnXU~dt#(6c|ey30Qx?xtqxcSX(~G8FXrxPPXfB?H(N-+KGy#7o{1wUUYw
z;DZ0+qs7`qn>L{cL{wS~hnB6@w_9!8+nTJYixM{b$&EMLKYjgF2i9B>O!Tu$9jWj3
z;aSmL-d_f4w%u&M1$>D4O11cO|BY@n@1)L~6M2*XDvnr4DZyNERgnLxytaq7faaDq
zSbOrLILF?Vm~gUI`GrTj8wX7XDQm!cU+*1Ry|Zq!61SW$iz<s$w~eCisBxU$R2fgv
zQMXGk_Y2wx?xA{P*<9M-_E6qnCb>nCIV=C7NUPV|R1?@#-V}_=f{Fe4r^<(=tM;iX
zGZXbIr<hX(!3RRiLe9n+2+O6Y)U4#Jx;UjSo!ANi2@})edXP%fTQpx7pNPJxu~k`~
z`*7W`YrgSC_lx>h4j@cB&!4B;r++DSh?q=FnaxBnL1#-wAX_69vu{aM;=-NYc7-Fp
zEEd(FLbhnHD!ePyem+~$Y}XPQF%}UO;iWFCeiWA%vmAHid~tfnu<Z`t9lt6c%(D6t
z&5}$NUDfSMcY{}Yq9Cz}TGx%ZM^Bbg?jZBtI898BIG-0kFGerZ6tNx3cz7}*QT6Ke
z8#CL1(-Fhm!%JdgiBpL-elL&9vEMNzo<$=w3k_z+YDXSO*c7yIq^9yH@F?h#hf7L^
z^PKrWGfSvHt=OpXFw)sv{0fjP^lGRX>LY4!nyXoLBC-@Vf^|ggS6Uocs6;%rFkO5c
z{W-TzD!pdk`^Ro2V^ypL!Vp`Ee1!?ii(coENajqvBPr%9jXiM5FZbBdm`IqZJawY&
zYlAlg2rihkpU&+w37wcpPWIX<1oWd9!qf}nBF(JLoNE`{vv%_qhb=}~i`X_?)@|#z
z{Toi-bVbDW8l~ho`_9OMyN*X*Pfkr5Y<D*f`baEqHxzy<-1ZChLm!JE5s>%i=0E4(
zs>Z9Tz;a<-a9f33%rI*7jdRX+v|OW{0sMhnz*rYD;4}$UB@{r~_r0$Zz9Cn%RkfjT
zcTT}&u{D_~KqyN{m5d5%a2_zpS7xAqoj+_JPCrR+94#Hq1dTKd`_~~;TIwq2cjkGJ
zWiv4U+h?CoU9mhzIY)i2a5#(~6@0F<JB)JZs+t4A9+W*$&62$is=umu<ARnZ>0FQe
zCj~7)AmjpIAvp}=YmycQ4R$5QhVl6BxVjRjWm5&NwgO(6TrhXIH88-5;r$L(k|gb~
zBO04AmBQTG)AOsNG`p$&Nz6x9`fP+6U_uEnQ-7eUYX3kLF~ZRyPm(4E$|wRdtSbRL
zFHO9Wemw;h_o6^4bNgZtLnnRL-&xH8Fj81bnTu@00T03raH2QX%23T(Qxk9pcTE8x
zCAti_fV(2XeFKP?0i^%A1^^xru@J6xh;ILWh8O?{wF8j+e&!kO_ph(_xF1~W-@h+J
zy#bKp{<@6&d6h<tf0|@0?E?P#!b{v|03}^zH8tFCT?;oWD`&7R$UW-&RypnlrOQJj
zFaW@K>(>vFn$FD~0D#!tPS4QYP*X$N0^}t4!V>h-O3>TM<ySudS#N3FrIVHW3l485
zM`y6Kw;bnxo{+{}|GF*2$?=~@+#TdN4K*KgD1+RrIK%`+1%)}~FLH2j$huitOY7W!
z@VhzgFF8(IcXt<QAt5g>FF`L6L6Dn`&|N7hDIwu|Lig?o;GPfw`#8J5@D^|ebN$l^
zzMuP6U<)@p7k4|5GsmxfU%Ui)xXW>J{(2GN>z{F2dE5PYlQa1Dv~Uv?`t^y>T|r@?
z|EZa~o%R1qvtOV5)9gR<`sa1BzdDnC3bt}n201xdIlIgMvAFDi-t@nG_-CB|G<<C5
zZRKcq-wxLjjGL4E-Fu?KznlHnkN(G|Mt_=0ii`id<-b1pSIb{>kk)py!wvMsFC)s|
zl@<DL_x|hiI#ytiqsOnupE}#Q%isIm<iFhh-AY#I*Sh>`o&EvAf9~Q?Ab(L-h=3^h
zi|f`_)&PJaK<&Pgo;T5E168r(xY6wB^ooZWSWG1GdW7lf>-Uy7Qi+K4sjY6uu|1rh
zdz5A(SxF`H^!=+(FE>ASmFQ}{)QQiF|8_Gys(SP0$LH0zT`p@x5qA>JQ&21jkPxdq
z<T^-BkP$ha0V8WxlDGY5pKTzHN6$Cq!M$5Eb?3?GNs+42XS2^{A*ds$+?h;Wohp|&
zg(85ME)YP%0U-Lnm;Z;@VfF8(>P&R0@xK?mn?AnBdeXZ31(scHEj^1|b@ajFB44Io
zyG-Vqt%JL}EDIZ(rs`lpOTVozR5Wp_?7+OdDnOVSIL#7Kb*x%c`{Jno!nv0vHnM$j
zw_E2Au8PHpNybLrq+U?Q11+day(!qv*waJXkYbZcHIx?fbG3KgEWY7jF05Z5JZ{U+
z#L*g?Whq*Db*=kis{Ow7)fj_BhhVJb0m{~9W7Gy_dOF;}cO)<AYq@g#K*k{T`c{Oz
zO<j!xQFZQ-`b7eeYdO+9)S1q$;r{~og&XEXVrB;N;?nE^!>c10uin(UW6jK!BUKSU
z4gr<E<Pc7~mkaE?_WH4(cspa6dsU~VQoBdAP6swcp;TWY9>C1lB20a@(2MP!Q19}p
z{lVp<8)fH%AV<PW%&kZ|?X9iOEmR4&0K^w=ybrcN-fnPqFF&-gfj90;Si>96zuQXp
z+$}ml8KJw4!aiNSR~pweZZe6oobc{lcHRp+-9LxHtHn!U@tE$D<rpN;2{z-<T#0tm
zxk>1HE}fFj+|Sls@-KpYUtBJ3n$XQOCdZB(L|0*tR!`CG;CMc=CeFA@j`-1)qsD8`
z<FF@fGq$KRRfs2Q^$_u*xzeoMJPGO<bBRC)$w>gBN-$Ro5#bIsOz}*s!cP^M3R!5y
zTAwA<nWYk2yv-eN7e_C~pAIIq8ZzIkTf7Ixyt3_eOvL^y$sg=kkvoIKoVPJ)Df?wD
zbuA`$@m$kI$I(<tE$4p&Pd!byc+7(;95Pq71KM|0Q?Z!w=G~cFh^c<>d+eGsCu7<X
z@ao;^%;|pQv*I4EH-9WXr-#FsR;%(YeAjf>xQVE~Oef3l516GuP_5W+dj)g@y0+;L
ztg=k6?x)*zif`!Y*E(Z&bw{D#b4>{>4p}Rs6y;(_b}PJFa1xmzik0wI#i5FWzSw+%
zYY^73)6m{me}H`2e^;e`q4UXIiSd^;35Yq&Q40ld46@$?V%yrrhzE&FU&If{iKP6d
zeSQlW0h=7cbi|B?lPU7x^E#M3WaAtim2~$mETJRYSK_3_sg&<&gMa;(`kY3e$?#>w
z;kSxo=Z38?P#MQ05Bgx7z}A5jpDH3Qej=a&K-|KKvtd|3`-B7#vN{fZd0K-s%nyDU
zy9>N6iS#r=Z?)55Hnv8mQ5%f5bzY|2<)OdHf6nc)_up9qOC-%t36l;=ryrZ0g`$th
zozd_?a#bYjM1_!FLv+FJe6LLE*>ZnR12!BMA*jv({l{uetvrf>TMM(wg@5dp93Ci_
zTM0gxU%@W#+H9kG&Mu_7GBw^G3Y8MrfE9^BSJO(xCbFkR9iL9Ryz3mX^=n4)NE9`K
zR;b{pQU32oe^pDD;%7+xF0jaUj4*!y3E5xfM+0af@116wSqV95i0t#y1>@MrEkpwO
z$bgMZ&-+I60@3#Z1|vigD2AexD%qnVP{aBMXz!rog|&w!-2bdlno?8T{Xc*L5XU74
zLPsb1fm3IxncCBsM*m!=uBIz0W%gP0Zyi(5-awZhBedA3&r7DgWDmFeC5(GsvHsJQ
z`E8O3{Rhf5o`np5949!Cbd<gBm=BI($_!Z)>SdD6hwG~cm9on2zkcBxTtL$2HZFI?
zq-(~sFx9gKu@KET65T&tP=(lY@B%hqApM-L@yoQwz|rJ+io9z;@f%D{ft1~*{ySHX
zA?Je#a|-Ur8v!xTjEL?8ipRwoJ}Z4)PqzTN+1ms<K7^rSwnpX8j!(XS{;2$$C#$DO
zI``(>%x)Tx6TtcAK~w5879ggXYBu0xZBoHyVWNWg0PqZ~{-NUKyVEfmtj40%iIalw
zlB(A%<|n*@WAeo}RrcT04eC-=?8@x&vNEB^!=LWjZ6c~dCm^bK1@iNx^?Y`L;tF|U
zcw9n1^t*`jj8?Yw%;daTnfqs(d5M|n)16_VfHOY+@1A_6+_d;vE^(%b4K3B)DJ1)2
z){i)|7H(rFa%LoCB=5{ti9e=eaUYK^=$IcC^4(K!_wxger426V;qhX4Bk15cE8i5b
zF8Gh+F=W7vO|BM;(MCZo7RD%Ko<-YE<kb(i*01a4DIgZ$E_%sd^p^TMK)$lmHRt*8
z(}EV6W5kO{F;YBL2wox*NEhrOb-jVdsiOJcB>dG!6LknwC_*Aht0?vO3>`|92=tQe
zliyL0LF9v=dwoubg>_nZsE`w$1IPSvOeT(HcYQ<X2uE&ZX!PjxX%iTjplKt+tCRh=
z!C6P&!aBVul%&f58Y6ervip*5-DTJO2_FWOK`XM;m>t_yS5%Ktu$rsK#|4+`6pm(Z
z!snRr#t3YBOnDOR-~~CQRk)YZ<GC4rAV78{*n*LPWfID6pspka%gLfh;xT2AKZ>{C
zL59pW%tE)JvXFU9??|Qr#;>Ip&u2}yINrw^#mz}qli*FlAuLIJ^mD=ii8|T`9?m_Z
zNsGx!N!0^$4i7rU!uSxZ9_m4rBwAZh+O~XtQ#lj2<KsF<n&57_DUq+C%L8ALXOpR=
zu<-3&*ZwceroZjOlo%K*&veEq>GS6`qQ72aQ3XAWhT58?L5G>#-_Clk8<+dk6g1k!
z!6svlha^xA+`X9*;qkeq-Txug4(u|}=TIWD2!yq)+`}t|CY4)hHRy~~g_TPLS0Gbu
z3U)9sJv-EfdV6{wFeAK4LF@iBhn9Y)V>wUTXkVuuxPrE*pgjHL3RpvAk5|J{c$?qi
zY`0gS0e*~voJ7QqOuL1yz4%3?j>Q4D66wDKM!1(_=hQcVe8Q}l-R=WdjV?dS?^Uck
zd@%Lg#t+M}Cq*A-E&c8_oZc$(sUV${&!1Ex47ZhYu1oNb(gFlgDdGmZh{gq|Oxo0j
za{(T0i3h{=`hA~yV|oTFZ*gF7j>hHusG+dqyVFFX$DZ!`R-)5S8D^*F<Csk@r2p`W
z(fh5m%BT=L*JRMm8Qd4@Oj(e{1N53~P{_)*|ESVclkn!mYu{Wo?(jv3zxbvQy<=|i
z^NNv*Uw3{)PX4Y!E%aar)9#F}CVhhF4|c4Tf&d6Z6$AuzkV>j*(f{$})TYhm5TP0D
zSh#{=&_l$ST{9aD={=)6tCQ!Zw&GZ0glW|t&X>b%{Pfz{#K5J!F*8uTZ8}Ok!!Z4=
z_{{lv+WwLrex!j-a};B`ni6Iw5!w(>U(w0I)(yTEy`v!}&)xo<N^$#H{ZOp+PTlTn
z?M4d_2<hcs^bp?`=Z60G-S?67Hvb}7Ll-BvbFMz@!NFLt!`nckL}EKr8spMa^^$WC
z`)xBZzRDp+?8R`K%h64a&rwGT?i&G~v+(8fv1Mo4-%Qc;fMJd+gC)REQVI`_H(GOZ
zBN#Kt&O$?x3k~~u!s|4{J6)zaw4XcV;|yd_5%*`l@n(a&In}{xTfTE%{@w?x)7ter
zwC<^du<s}zRD6F76}?1_pFm*Kdy4L(W3zJ<a^~!tsNAFBJ^IHX^GZO0Rjr>5U*9<&
z7hD0~__*8mP&{ImZw51DfHa*%zg~ayA?Wwon_E!Y%2kx`d=LAh25wy5fxT%w@#}-E
z?rb+3Yc)VrUVrqpSc*xlyioDvEw*hZ$x(A#9#d$5h=RRM<u+gc%{56%fv~@w+XCkJ
z|Bw%BP|M^oHiLXrb~~#;42jU6Gd{G@?Ya%J%ZG^-ZJzM>Ulr+dq(H3g9-K|%La@+2
z<sVPwX85u9BD(Jb7!OqdaR}!eg86I{1X*t#-yJG6?h>O4>p>snOv-7DXuQ5NG5rRs
zvUL12u6}uclef~w-9`;>tT9<)+2%^0<5wd1s9-)va-`sK4(!clrxT&WXgA;O)7MUp
z2^_EfWWRX0r<*5@61s^n2A)k}t(8VUksw|Zw1Xo%5%+pe9ZxbL_+8+3adFvsErjYV
zLr6VRa1^GYeQv}dpgPGf;A7Si<4Kv&IUMbLVsQH#e^9MAiXFOdfgG>#*)q!>Tp`8B
z)CexcK&M4#N*)q?Jj9W=?xQOCGZZ=dR3kR>J4b8J+Z)Tn+>>rUY+VDK8>-o%wB~}#
zWu}nDf({$F|H@!dCG73l${T`u925S9@)l=`oD29)T_{=E*&!+-)ytx?2U!BkD^Bdp
zi69v>xcW)>qZDiIq@F&=&Ri~<=?IZ4@8E{#Gsnv~hD&mnkYvKcnoPYZ)ih}evYZ7?
z43?7`iN4?v>y@-Ab#3USLpk_;KhN@037ZyZWZAV-E~j3*$fvx0<IG#NJBahJNo?7g
zW!G^JA-Gu6;}Qj<<B&1}Hq%HVw}A~S%_$)gdPHngW4Gb*Rffcp9s27p11=s(%8>f*
z6d9T@LfXp~RcTG7Pu>K1(hIWuW6R=x5{hD{gRS|Wcr0EcCc<u4#L1s7U@t>?bcjT<
z7Jl%n%3v~{q)L{(j5a9ar>!z0fmEf|9JXM}GkcS+;8&%Gqv>b;_2Zy2eC5FWI|+@d
znZHXF#8-3nC46A-e&Whiz~0b>zdG~>UI~pgyhA8aXb4_FjJuzG!-SukMQuw65v)i}
zWbF3ujN?mm6BXQho@slg_I&#kykj+yu~M*XTp726_=2kRs)Fu51^EpdRLpvP!T0ec
z1fX495F*|FiXd=a2L?<bJv>|lFk6Em4kc6fNA#JTPR#Tns<e?a0V};JIZ1ERsPPCm
zqo6Z42Kq(fZvaRhoL3!TCv0W4`tp1p@?*=~N_)N}@7F|3_}F8!#2NKk5H2h<i1WT#
zthzf8zy9K(iYSql<>ae2#D8L`A`TOi=LezkVC>0{Mgb@5bTO)r!^Zx^nm$?fOIEYp
zwAi3s7GUEMZo`rd*4}P(fuLTBwSM`i)*0*XRRjbnF-XcwVi;7}P4=rNKFOvm`@rH(
z)nod!t$P%5t(z;>DaBEt?Vwt4HS{}-joO!Fe&Ih3Zyrr?QX26m%~edZAwD{oIBD&h
z!mO(IP5XoheLrw~=FMGDdYYRnzdSinIi8{2T~*;!rUH)#IZRrI%io<HsqA44c-dIs
zlq7KW&EI%Z-1s2uw6fBva+P7<$!XFN2k@<S8pzLKP{8`+R2u=yzLYy8lfI{5cf6tA
zsLzGT^vwo@C}d}+J<{5rRTyM&Y-@8WF?F(RfmWkvfUE)er1oB+ldr4Y{%O>-{3~;o
zIt~^gID{ez4J|EVS26(fj4~|-X{N4VI>*Z28yjrvtPLdfFs9`8p%(m^?`hpOODPF|
zu~T<EASchgew_5%4#n2Qjj|Sh$dqpNgW{R@D%eZmY{3SENSFT-`kGOoC()y)6q(hN
z-5{}%XoV8GgGg!E0{ZqCr=ql`?R4-{8c`1985?Oy$;3-fVMpSk9i(ZJsxZ)mukS?+
zH|A#|dF!73iskMV>_e_SH}MkxgcCM)R<sObv6UteB>!9AH8E2-7R|@AUj8E#5}W$_
z`|GmXZ*^x@$7jf!9@WbeC+<0yaeSEYZMt3Qm00o)`~F08dzz(T<jT&wu0Z@s2Oa*h
z8v8cuRlN5SF-XcvVkBe@t?!mc2i;rIAszTje&P`eYJMeXq)w>epeXSsrlc$QUp?3g
z4v&Bt30)9TP`?U<VkcJQeKuO_s%Y7z_HX6z4D?Q&St_GeLSlI+Q=G?LAkn)gHZwbw
zIf(g#@H9sJ>jiO8FphH{m%wWnI!U#;ykH0Rs{Uo@;XA8wUHh0kP_{i!T_>0q6PE}_
zGjEL=&v`n?vIG03y)NQi{@_8N(EY7TV&wSf1t1P<ie`V9kI)v-HM(Z9Z%_3;xABo&
z2BY|RNN(=}bnSGy!S6Wi>`gR&!P7vvwBwAscb7nkqCw}!u$d@gYexG0w0QCf9hdG4
z)|f2&bbdekl_8y^<TEeOEk9cr*5#CKCaIz2*4IA}|HUA*UXcWysz^Ci=e1LG+a;FR
z8lwZb>u!BRWVk1)fn@gGQ<`rPs=9qLGNh9`q42ep?dfyqlvVTpcVFjo`Ge6mjMuvC
zX!3g|W&d8r=4!r)X#th;wkj~VSiig{3pu8Z#2hHrU5Wdx<i{w8QQw;%Op$33<ISH=
zS6zjj%O6mc8YDfpu#M>9a>6`$GPxPPAs{nrJtJ-wm$G>k`|hP7YQ}1!7<h&4Tse{u
z<qtG)5cehPohKMfkccYq-BxC*_IZx<;k3iypuy>>DAk9tXv0&=FHd5Q-7s;%HYY88
zVvpT3hTe{D0$K3O?!eK+;OG$8djqdWuJH#ITrnA3$|3(T5N`DGj&Wx*zXa{)PXfca
zK|1+iN9Ka2yuu;SkilDXP%H-BzLmGVDj$t^wjV%TEJl|sKg8oh(Y&2rkEoQ47WTSg
z7^lxArgZ<Nz;eg1g094Mbt9WY#`Cw<6(CUnM!W;payO~oujB5<;71gwsA<j<pcJd(
zBRdxzEPn&o#V(-Bf_D2Hw<|#rq}ffIGTT7j4$8}VK!Z20{4aFF%cU6c(+uHuIX|*B
zAiAL)p!@i9r(ZV7`=K^XSDAT+<E2Zx@I~fvA57_99TYjYpEUaDI=&wW1+E{3WGO-^
zjqVy7>n2i~jCSV}hqn3UuDeA?>AaX7hvB~Y-2=@hJU{V+UWMU8;bD6s7hb>$yAXRt
zvTwG{C19g$-+lb%Htf6!%Um+M=ClyL)gTzVD^yPQ4e2c%V64mak<d#ht_li~1Yf}$
zheJ4#oo)M_?3`|fJv)iI%DKi$cd)G~!$w2=1R4C`R2!v21~C#<Qy^Osprj@Q_XDj!
zfHCsB6bIfO%D0N4NLxR@e1B!t$S+Gx4=+ZYFxbYHC>l?E6!PHBUOqL1za3%6zH(ee
z@lTxS5jRuen>OG|KqE44G=${jF{Ps>w=plpN5H3*(RH4YM@>B;Ru<<%ug7)f_*I5=
z=mf^qg$3lsW0~_C^$dft=BJvQYTrwoZatD2N^SG_U7A1RX@apH^d)oQp(Om6m{FPU
zN%3V9KH#B7f7!R2lVJDzPW4}>Wo<t?<jp;MDp;qvblK$2X}K|q{F^p&msctw>pi=H
zgEiV^`8ENEP4dkW?Aj^HVdbY)g+8|YuX_f<W3C35nep$`LpepWw&NhJ0O*OD#G_lB
zq|cu7|M508oVT5Q>M0?3n`jUhlv<!w@!+EoWues7D&w0yi;lNO^apC4oTFKC#O;$b
zalRJQL?9C8U!-)worV&xb6hBY0umAHc9~a<)R$27k4c9<TI)-tWdm3-JI0^j^G5=O
ztp8vTCp~a7Ipsh~9B=;+PV*Ocv1DiKamxF&!OK4qLUw9%uP5cMZ&GdU>)P{Hq}(z<
z-OAvWhzb4HS6IRLs_8x6q(7u^-kHL+3?NvAgJEoDJb{f3xXh7|#Ge)`uFcOM#mN=h
zm#a`<Ed}B-_FWg493q2q0TkA3?3G$?Mtq_&#T#NZ0*B6-$6yXTI^SqDrAqF%4=xLT
z4DHu>EMO4hOB*dZv8)}l03`Za^j4@=HZ{WWJAO0fzmWxXa?|04%penTOq=anPX_0y
zU3pZVvwS4?M#mNCNzEM%@*}DWgqIG7Az=t100p-n<W`YH1fln?sn=1wM>{=*L6y3)
z6=meb5b%;6TSotP_e+f9%qhJ{#num~pTl2)n#uSiOpalWvNAOYEJ>J%y1Ac?=a-z}
zJ)6HaV8>x^+sy`Yl+_b~eWk&7>eKgP?`&Al>gm6QF6#u@DdtIvB|bfJZy_GLbTm6T
ze)h12`ZB&JUv3-_<u0ta;7v{)*ThvPvBgBwxlGSGc?HptRckV?UZ*!3RWuR9_W6q_
zX>2(drNXJoS)k`yOM*J`<OUWBz5iX`1rP^Z#<?GlQzR)KOKFtdh3|9%dqSiyZi?!X
ze{^J|cF5NaDATeU_A}QM&oW&}inDUa&TGfs<g!_u%-*Z3tY~3yJZ0Sd9V1s$06_XJ
zHYswvYB&E(5{aCWN<P!IWK7=$QdGdNR6wP^#h4X1Dul0j=t_7`9^9j{Ld}mB!X4@i
z6qeq;AjEPgPAQKlfFlX)fQ!G?u8SNhecbGZ`+%f|XQEEa^kOmU&)??zq+nMLXKrT2
zHqUn465q)`1h4e2`i#;Of)n}|)%?3NT!!fVP9vz|3hSeE&axQ5Gn&uW(=MCfs)c;n
zXWs-WJ_kG5ndJ$|yPBQIjJ??LTL|l)b}y;NZ!`apA(H2u_&0Y2QnJpBO|mt(W0ffL
zrNU66#v0S>+P1^uLc?42+@pSlm0dZ725guQx;-m<J1nqvU^!G5!AU44;aCgDGa@U0
z6P6cO6;_M{)|7&UUbpXsJQbsq8`n2lr%NOcMPbXchSv4;{rEQQDjmO?*}iE)PW1M|
zMn}~NfvfRX;F|p=#ZK^cZWjxi6`oC*VT>jWHVi!He>k8j`#L8wEvwQwwqs(R7V#Nw
zU^0Dvoi~(_8Df6*g6)>##P@Q6ySS<jtqVED7|BE1_%UEV{*CM9*2bXG@>~JTKJ}zw
z#kim}f<>YbRcXdVu>ZylT;4A?C3T0;&#huHW;ALHLRvl{Yy6h?d*+>SJx}uSJ1Z~4
zJuBigak5+tv60zd_Jr|IB#iq7uQ&-+JVpS-z<|xp?bRAT@-@4Pi=m`nWf>Vm9FNp~
z)Rb1m6xipp%_$f{j2Aonw$9O!H%ST7{=XYLK9KaUZ3WM*&Y~P--`;Jb6N7*p{gS86
zm1Aw=M%S@Eo>!y4^28{cCbu5k+x6R<sK7Q05jGiv4u7Tg@<*(Ma36HJx<u?$2C~M!
zsYV2UpRGEichr=)_b6JxV2)N{w248Y2$j6?>W?I+#}(8zEc!m+<Mp-MK?MZ`@R>x7
z2daatoJm>j`KVAT*%fixv>u%fdLKdK2K`$gu~{0d(q5A3^$7WSX@cvR2atFNI|Xb?
z-TPy0b09;*;rhlX#3211w)*A8^l_#yyg$X&ead{Pdl23|lb@ieeb^oCp?gd`5{eKB
zPcKaRS_f1wmM}yy99>jCIALYROPcx1-l($}M4_Tl#+;JK?4zA8ojLD~^NN{h5q4cf
zSwl0LCpN+t7LW7tN3lUy{?vQVD0b-k4@*|Q6ObO?v~7E!YaH!*c=rjm+J**x>OfhU
zmAvWf)aN%>P}l-}h+;?7qWLatKQNL0XHMo8I6*bpHg!>p1V5e|^u@5lX`i0Me1+J;
z@zmN!&0?_pR-XPZ@$#JO5nlwMCPP$9M&hmWJHVR=7QZ?LYia54Z|4Ykl*g|L{;#RI
z_tMb_G_ld3=l0JzBQ{(0*&=;n=cg<ng9-gDxt%vyj?)GV&IArssRT1!;z&dIIXC6H
z5E2_jb3+pAvhkl1snGe?=*2azSZ(dxC7ACr%wPHvZMm3Jsh?#}s`;ZIJ3v;%nt{S;
zr(7LYRSiHOn_~T6MFV{l>+eVo2)e;gJoo*;W3&)^6-wNrK)fz=>UFnE*N$CDUA!pY
zsU|||6YLEmEh^%N%ZXgT;ri^2>eyP+-y$w3f&=G_T+SDG@SzU3gCYyDT7loGj=lb~
zDPaXYE`~+We6AVhvcnWPp7mCCF3Z=jLv388P8-(SDu7WWgc7r%0!|2QAjp%62w3Ab
z*S^tkGLdfptv*YwxI)*doxe2qzIm5t*b<bjpR)dJ^CWz6X~8SN<{UlRzNk%8J?XOc
z4;F&{dwuW+g{&)j-AsMdQK{5E{It{XUY>5=nPH^QFyGUZf$Zt79_flOj~?+7*8P4a
zY@$>26aIGpkXe=@Gj#8pFamEmaO39OBN5nwmka<RKjt;vqN(X~H96Pi+4>`4><(1Y
z`DFn8$(EB-gA3SZ1%1Gb55D5zO-I%#G&cAOBY@<M+T10HB5WPpZ+kA&*R309zSpEZ
z71a|Fk&`qeULoXBI&a4CP(x4>Az@ax2ResY3>l~p*je*evXtb)BuS2c1DW>Rx;Kg(
zhTO)e2mRe#DwU)Uy7hk9Be%dNvGQ@@#60r0R;hiN(>26a$H}&Z-_a*70=u%PDyn~p
zkh!Zx@!J=034KJ9G0unUA;oED)fwbI0}6g6Sh@IYpzOmXofYxfojz*L{I0Lag3%%@
zsh{tb<g6WjcZzQ8=C9qmHD?z=f`=wH#qb%(qYh?<p$l|*IRlR%*0B?@8TL<&#A@VK
zC)O){%ud3i1{=Ilg^rr3RXnWFslGox(etb7yT46oL#X-&Qq~VcZ@9_!HjAjoXcLva
z>}$SP=G_2FPwBLWL9Kb&Q%&zfKnMuBZ|C5d4!*V+6mjzxazRu)1kLi8(vWUR<(%^z
zx!W`*`a1aCed@Frf12c;2fE=UK0m%sa!-yq-hx4W?6-pV{T#iHLnJDzpq`%tf;Zm$
zQ(813AztcBWt!KMQpK+YhcGp9*|_6MfY4O!p}BZ%YKzUYQlHxe0#N2S75V;1>`USg
zgBhro$fr|?qJYg^3FEoL%pA7Y()bQW)PI$Zy5N}vv;{Gc@W;EI6tcklb@Lu-o5gUa
zj%L><$ZqPkPUoFPH6yIKtiudm)XbygvFjB;&T8%__*7UtPSG&SX<N<B8owNh=C&lb
z;{PuB@iFv3gqEh!%67E((3*I(vXZ88??V#oS>Tqu242~YIsT%L@#D}keB3loBjNvi
z&#%400qX4F;3oW-Bm}W3RO`E?O=rSVPYw9zBn#84rXx}*ZLNRp|3?Or8XSSx*fAYu
z4Swfl-#}P5x;2aR*!4w<lBsS#7L<cNQ_!q&ktw!>Z*mfO2(}`;av!H8CYNWqdg<_d
zO~l|m1d-eGFepTQ=8BI&QmGS(!V<^5-uldJ+3_8MeX3%7QQ$P!!PDt@$LRp{_IA~|
z?z^SlD7*+Op8r(>Ke@DUfS>9$aU$B%dJp7!mD%`&uee;uQgi8@_tv{i0j^LrU15Rc
z0#~+IBC05fMEGo<kk8NM(Xjx8gyRzU@HhX2s{{fjTWigd1qgI@)4Vx0L}+^Av=l}1
zj3(aa<89L3O|hKQwJ(l+;;J!Vsl4S=qumy$$Z5cM1PjywB#Vh4>P%M0!^@VqapEQ3
z47lSY)c_J_HnxEMb7bQ|UtZ40f>fVuw{=pg?SAP>%<@F^l5IP*05wzZb!anF?$rt%
zGtZm+i%^7W#c}u6%PF4TKefc0#K1-bb{6upx!KF-h5~H4ONzFz)hea)k%jSbHwy+M
zJ8Qk;=kIqsXy$o%#)7s}nFbP+kRVM^_2oKAo1O%GY6l1kx{NzSXN>RxVVM*H4(7VF
zmD7s*?VPmoTNFYCv@P79P5YcYpG|z%xIEF5BT#*mFnFZKhPPLedLYt#a>}nI0bj2G
z{K7s37*&`*Is*p#tyu8zg|}wNU^XYFl%3djcMB4%9UzxHPRRAOc#74E^|W5^;q;Ep
zQkC3wKbI{&%0iX*cnQb^3ydXVLUv18;605)*qr$2<|LN_0(!o9KDelQ_|$=3wS3KC
zTy{O0SRpHY;&v;7n=Zk<71ZU4O5G?rY2#bm)gDph#-9^uQo)hR7UzsuFJ8S7QCuUU
zDs5O_IRYtwp`P{8oHn#&rTc#;o%6~G8J+UBVl_xmb+e7jXAuZ*i+VJy_opHken8@h
zIFUNNI@!|kXa21uD2vy~#-r~tRqd6Z$~Y3*tINgq@A&0GXFRjhSTPNI=Xv=s)YD)G
z9X{TF3^}*J@FQw{!Hm)Xf2<65M(9yfs-<67-zMT9J#*YWLorckPZrMTcSjQb{{MGD
zzkMHi(Ak58SUPMV&$rC+z?^fVd+1Qu<;rV9ieE{(Yl1=Oam)sEZKfYNouqlZ{v+|V
zpX@^BRWk?q`%{<YW7H?sEiUoZO@udep71=p%7eIcl;SXYV)*$Z9<bFKB>qePJ}>4%
zNii1fsEyu4$H}{aq#H~X^0JpwIprRqgTWB%Tyki=$!ILVsC3k6X_0Yh7wQASH1L~w
zZC!<0;~cBxRU~0+PVg#GLABtpitF7!wD_3>cg|`|Xc}=o+1C@9s?}dgl0lPXC1`pi
zAV_M#(v<`6?=HCVP?tAuSAh?~7u|6nXTa`ryzPzZ5>qB@1DbFXyFT?uC=7SNZ%p8}
zWp9})RW{JF&R|!73eqQM7r%ZX>6xP2e;N|?B)MRmJJy07>qC4A0@JR92OQ5F3LOAH
z#e*JF;>oV-K8{$dYLiR|)i1i+xWS5_e(MiGq9a=Tu&%z>3P>RSyDQ`CYvLj|jqk;h
zk6yWXXKSbLt{S-qI|MW~fyU0%fDau6sy+Ljl;Yv#;zN1D9E8`OxLkfN>aw?;=RwY_
z7j_#4)TgiTPd}9zmF?B%bg(WAR^?SFT+etOzth*Xs)A~Kd3-htu`fl#ja;1WSQED5
zO-ykTKoHrm)QP9oKt)a$kobAMnamXSbR`mI3I{$9zag>|QyU)px9>+yCCQ2>d6iCE
z^@TkU{mIq?#Jw5m6AM2H%;$jFiF5Qt!eO^-WJFk{f1lV6QrGw;eRhi`YL1qLCK}F;
zQSDR%x(2*G+Qp|J^D`{(JH}1tk4`gvU((qy39~Cs$cE#MV`o_<m#xcQ+=9QSUK|*R
zJ^k=6JD78juy<RVVfJaa>yNh`eLXNyhnC~@6_ASscA(hM4TKo8Ho_M<Q6(!qaW=cf
zmCnmO{z(~60zqAxxD{J!M3@k$jZRXj2?Cg3SB24HF(9_-a~d;hab=4;e1~k`>OF5K
z4GeDfh`Yjjq+?>*Dy1Kkw|a)-ZiG(=lnU)Z7e}_W8y(I&tBlOID$fb4C#-=p^c$bq
z8hbo}a0u5bBn4oKSf|lpHaYDeb;&C#_Ua8m8LoGM2GLFsC;7N0?1bgsropz!>9XpB
zN#tU#7$W_5l`-fA!{6!|FQVtx@tI^`6M%AT&@p-i216{N2hqTiK$h9Q^LxzkV_tm$
zYd<^9e0xTE6-t{XcDqD#2-#f_ZktK`Jxh<(pSUncVVJhn$&;fxi_UL9pEXd4IJloE
zrB9ndnsj6^TytY2)PI<JOfV5US$TxX7mD9R3*b(#4%gs?3=T@TW0jZK@bMeRY<eW9
z^KtpRa%aa-^vTRI+tMsu!TQI3;sFb*o`Q>?#Q6R?%{aw=C}=?XZCE1x1d5Mzs;LR5
zC>7H<(+)MQIR2<|+$(6y1AKiCo+HE?uwJy*gUtc%?p^X1&$RrWZD~eN1B;uO70eam
z@H^fA!<k!gBC2vHgs`-4ABauvq<Q6mD)N9<u^Y#_pPu==>UsGjGhr!4`tFikg#2Bk
ze6D8*swkogewxug;?-d9fW%fdO}vm-CaCGarZtj*sFefmA%Z1w@;oJwbfWF}LFTBx
z@4=h`YHl|K=_8!Pc(XD29gy*<ksV8oA=CbhzE#%r`T3#`C}k7l1xnpSH_g}{oHH7R
zGYxqX6Fe%giIvE#=HEN+igVp0{9~0KHln`gB7-5w6ZH5z)Ws5bgDtKpo>SMfSG#s;
zDSPh`+PssmC)S_&`T2RY2k4p;_FGa_A%;mpHPdX~u0muM!i*pDkH2trf2u1Q1g-@T
z+i=Y_SBCVj3cy*}0yb20$Y&219M9Z+(*z3d9iBefx)&fm#QaLU5_oQCch~!);LuaJ
z!t(*_y0;rrGjj{*KWvG_Q7Lo4k4T#w!Vx?GNh}XtoM0C+LhfHMp-v1W?JR5%W)5gB
z8zhIW)3&$Io=$D99)jT<n(mKCdJHmrfA(yFmq?v@=ds_tH?|#AC$SrY(FM>-i3v;v
zu8?GNdSXNToxr|)zw*IW#qmGRjoTb*B6hCR_Z@(6%x_iYWC}qcZYYmFO&IoMtG^^*
z+Gs3MyWn+RT$d@E<^ve45^@~Y@e}KfI|+~7j);V=wAmhC->*|w+*(?FLOAYu=U2g=
zROJYtYU6Sa7eQi1?@95{{3GNt&U{zBR8@<TBio8(+1bqC5tW5(k=di!gS!<4VOWl}
zpGTIWb!ApWWexo!Y+zI}TDZB);6Wx+4|EnISFD%_@o0Wa*kJgV#s2T*KiS>?Zg%*R
z+~mFFXBql0Kch3`3R=VNlb88>Yik*Ax1Gh0c2te)t=N5m`Bd1I<<elgnm~IVAZbnW
zU2CBj)xoc-@O58}4fVm=%)2Tp89}{uL#IDCX2!;=8w!laRB%ZZ5NtIzU}3G!f_Rht
zSWiE%a{H;72xdoF;Q-hM^G!aWFI|2zxo^1U@eZ!lEU`?B`-b(TC{$v`Sk{~Gc#YGT
zZqZ4<rHvbYR{k!<B45KZ|Ly6z#!jdieG4aA@7Wq(2zUR_E?c+py9GkLa9>vU$X=PD
z)7%N;!qP#&WVy72$NODdaS5p<G7IA$dPjH6q>Rb1Q_aLVdF`jZ!ejZeD0|{na&$HG
zUkd41z3Gr$uGQe(vhLOxSsla+lnc8O&pTj@Vl10N9-Bu@$(n*w<<Y@Xj)yB$Gy9`H
z=&zpg+sK05P<V%VI>dx4z$r>7@qTo6&Fe(>6n>HH!>kpRcBPdBYIOZ1r12qCeM06^
zanBd<XXx#*5mt(!V{psB#O3kbKCf$#gbQb`H0xK7-UwOna-V5`Z9aLj_Sd5qsPAmA
zcUz<$UVR6A2d^>v5`ccHWihN~5^^S1yXVmtb9^_wG^T#k$D%67pVzO^-gxhXocn`O
zH|;C9ne`}3Vz%~?6t_pgipsg_%4$7riqOF9oy`=ov2w*Qo<v5TXB8jId${3xIl)=K
z4&k^po85GqXv^#27CH$dK_x3D=}$Ojyn)>3%yIfM;kXwJf7sMVS#For!)?E+`sdO6
zUZ`bQ@tKYXZ9s+T#naq6lO2N?v}Q^bZ_hx7o)3yocws4UF77b;+rS`IB7fRibjndj
z>d*Qe;Z@cY9%8nm{nrb|N-8--;$~ky!647>$fxQDRlW3z&KQ0+=biL3+8Tsn2!BTd
zA>+DKufnw&ifK7{()L~W@hAR88)AlHVF)vITc2z~t)RaJ$`E<-0r{6|(Hec(C-1t4
zOb95S#kgMY>=RB0=aqu`8<;SoU@lY6tb3^YpWs(CGt>^H4wZPdt%g<4?^z@s4cffz
z?XPj?f36O_lP^EfmkzG+_@GrgmV1=IcsIKKm8Q>yg<R;88~?_ldQC60Wc5(5=}wVR
z6D|4^L<#!>C5$^I8n`QR>4TNycay0o(XXfvbKfvuQC%n34~w^VxhTu?<H}>p^939_
zMelTv*Z0zFSC((;B~n~fK+VT}d-2x?#+T0y24mhNou>>I8MJ|S_YX>M*(S=TSUL#t
ziakH^WJg=V4(n4_WaApNP|vT?7Db157b$yqp-7A~Oubg^Id}N>R0_Gt;Fs(7{myqr
zTV?Oe6$Nqc1Ov~~vBHqm2--!See!*tjnME~$llc#Oi8eCpR3Pc?+;!5yLSXO9ysJp
znLhgpgcN=#TWz28K&0E9SwaI!{p5tCiKSIn(5wC*h~uifn9T>x9_;J1#tFriVbNjo
zu<JdW;OkO{rKB1InUL{i__$;Zsu}mMYJM$O^GOG)8P#z%fA0%C)~u~@irsPAHVLsT
z9ENc1s5CjM-6O9v?{m6R-42B0Z19AAsbUh{`rQ9>EwOviSZ*Z6OJhT`mx}gPZ@*6!
z+CTk3|8c)vpp(9QVB!TUkEU+bR!f-=_8i50UE}UyCo*IwR&16|X>f6x10a%j;LS>a
z1<K_j&WC`77bdm3>()iy0V6|qxg5S3os68GB+w3yJIF&1A$C(;Qb=<Bw1VEz#aoz!
zw}`h!{)^lCCu(L|ndg@@r7iB}XxlLNW+vV3IYvZYN|Et!Ppuf$R6y#C;+!hGO?kRu
z`^-7FPP44hInG#YW1oH3>q&WueOXV{YlX8+Z1-_ktx!L7qHQygc2Q`N2lu^_*RDNl
z#$@cPFW@m=AN8gEm{l<!ZdqosHnC?1$@tnw3+<uZ>3a6Q&~4&14iq2d2ZhSg)XqNH
zapZ#z14?rn<hxc)z)@}S+uQr^R~iMyrq^JsCt-b?y-Q2BrpYH}wf4q^>W8_{>2(gm
zd}e`|oqW+CNImigl$v-kbS-dq@FQJY`vdvItINP;vjVkv1@RY{9r?*|Yvl}RRxtL_
zmm;=+zcm~lvI$Q#q)bR&OVZWxL(MaJL_LkVdSx?8ZzUJVpqbH;)==11-do{6LG(ik
zT@Q2J7QG~BG`hy%D1s%GfKz@v3vTUdjA;!oveA*xTD$dh^w7FR0UYuRl%j3qZM1ED
z<pbqDuTHsCNa{oN_W6DRv~+u7=unxxzs|3sOFPL70eU2ml&u@1Tfv+Yd#aM1mqLBg
z0y5QB5}UTZr6g)}_BrqekHG{_-P7GJ<nyYlDVbm%4mQIh&ei%!fH6r_lOgCy=&tlE
z3IF)UBdEIqR;*-f$IEah|5nt`5Xg{%j!maVl+LOGP8plzwi%34E|TwSi{1aJkwq&s
z62iAHv}jYbXrmn4aMhus59szGpz-PEy?wEhzI{WzXW#5TG}ORVwiw6H_dx79eR~-W
zW|pAT$qZa_cGAaSi2FQX#t+>7aoMM^uKw>8PQQJBrV3-rc4%r7r&jpp{!GXZA8n$_
zJxisFQp{EeqTv>@toNO<$KsW%+Tk+eL)WnUPfpT-XWG}(uH%OK!?5GGPc@HfFf)il
z&_gCZMU5t!akMoKs4jOZMCi?8UkTAL?tsJd;>pr8pRvW%QPBAQH#nqJLqV#@)Z*Ov
zf@4qqfLRxsdU^dzn%i)j(~Q_x{ow@7EnP9pRoGj4)DK!T&pUQ;>sLx#mV@RQZo_Xd
z%Gh-54=8bvy`ykaTe#kqg^Ha)#S4_=`3e2CBFTLA&8qhOM~g2KE3L;TJF~A~6BVP?
z%49_$VhPS}w$k93-j5F^G>ywPovd`<LN(JO)Sgdj)1=t+)-pZfLT48k9aZ8ErXjfJ
zG-{`$X^Phl!0Qg$a<8((_n19yR`pahD}aM%)trVbe`s5~M$w~l^wq$Qxeh;Q&*+Jk
z;gYH%kfBVH#diPM&mH_SkUXSERPC!R5|7A7gC)!!(wKkBiwu$x{dOL4ui=H6W##FU
zH-zQ3n6>i<<s=Jw>^x4wo0xUnd#9{YbkVwYIxm%3bXI;fi^k>+YF%*Tb+@<KWYYKk
zsZo8gNVg#g_kU6_xw+~#<V@tO<FvrRe4^qnc|#i!DfJX<juK@T0EeGQ3g8e<+Xj<$
zUVVzr%->aRU%RQv^Ze*bqfnAgQ6b~!2>-H(su<1O$(^E_`;3|o;u7Fy#=yz<@$1h*
z^Kw1Rpg><!ck1VLqvz;j+z|=q@QJ@qt#L;$c{Fc(Xu4N>Fq<bm+Kkm7&3w0xTBw&F
zvoP(KcxoW?74hbx!a*j@zz<?{&O0II`xkN~W53<yS)M0Hh$^JRH3BiwQpFg<=`-bk
z{QMzg4CqPSc$C@XX2f|Cb7jT;TX7i-PffXNtp+OUp|7vf;PnHm(U03@W?Ry2AO91=
zYa!d<2RGS?N7}5|q{CJ`=wU*sd|EQJ#Sw$>kU?$6Pn2G=lwMy;O&1Y{v1G7vosQE4
zMAsod#<Ocau8ls9<SaK7a7MdG{yQp>CsDcGUtB1FuTvI;m@McV>8F)A<Nq%Jmc7Q*
zotKZo1A3PQ(;Jv9d<q3HlzptxWJN<9JIM}yYA0C^n~S$x?njI5bbZdPaoAIjIBtBB
z>QI*<|8(%VvY@HMcQ4lz@VTRlra|v6A5nM7vk$$?M>S-a;no7lL0n9+ZZ4q~%#oM$
z_JQzubQtE3U2vFTgaUQv<J?0as|x!ep)uDK*6+sb>h76i%rTZ|m%Ag{>t)YB9gf+G
zye<Ob{!zj7&!y5a4ZDLuAKVFOTtHJ(xcsQyZpeD*MR%r}Xo6_MtM~1dpIOvI?YWov
z(4Xg-2fQ11r(QNn_kA>4klwAL5z&A=f8;l?)Eh8cXpA*wnU#fC?#UM$X-SMRiw5lU
zjV0D6TAd6hXIQ^vM_(s8Gm9cZ^Bgm+De*2MQOT_*A3O#&g?jFt<w~jaeDk+&K1mYR
zJL?#$^^2%imS>yoIs!e@YKykj=DF&h(Pklo+TUW4Mrfpm8i*!v7zF%yq`B44Q$~|y
z5_QOW*5PL~xFOXG9^d!|jR%{>b#9+fR-O$#uW#gypo-CZA|15}b_xMv!bZBt)aIDV
z-$3#psyK?cN$fhmK-M<PPwiT)Gi>lzV`;O{uKnI;_bc%}$`!hKo;{_TxKrOrTS-=^
z6#bby;`(}~xsqaCrRVH<wf_%k?;X|D)_w~g6+}fvMXEy15e1RnC6p)#NS7`h=^g3P
z5)}od7?CCfh|(bxX+dg~DmC<8lolZL5D6h6cgOR-_x}F4;~V3SZ{!a$c6Qi{wf9=j
zeCC|b!~1ei)EEZzC03U@X_y%nda~u6NnU&-=d&=ML|NdW7rZJWO*ATQ&>wJg)<JGG
z>IaFf7`=NAAUD?4uSjI}w>%V><x8GgOy<f=&S9E4noDt8V@+uM<U^4B0z(w6d0gNl
z0NDm$oRF4hY3ebL#z^Zy9dL?zgMAAL*jGt)W4luEW;{(Ka#dZ|m`;=RaK7J6v}Ye+
z>Qc)LNKV~aI0hU_u2*zrWj^nj6IrK;X^`!Z;s~jD+)Rnl<n--%o!#=Y(vyYZT%=L6
zx_YYu(B*hg{KrED?jsz;?WbkEgrGt{=%1R#se-7_8(x4fw1WYvDZSdMV4!lJrn#uj
zDaHvoum1Btz=={<gl9r_k5%QJ4S1`%h+$Cl=K*7uS8KvpG6l!y%VMi6QkR3r7KNw1
zfgK)Jryw2xb;+TJ-{Y@sU1tg`1N@CGYkN}a<1I+b(s;*RYLRtSJRp##u)e_H?>_+c
zd5uMe#b~k`FJGJ5*o+G&B!g)&dgMFlpDS^obq*Xo<gD+uWBay~wx&W@;vtljL2L*y
zDCIXW38Fb&PDFq9__FN2O59rr)iDj~r+fiGh=}x*TP_e0EL@${a%>6rg;Ha_otjl@
ze!4G|@+%rxP9auy6xVm~IAkxV7da>N7_wCHxBpY#wC<Hs5M>5DJA_-vjFRm!LyNyO
zmG!{EuA{eaY4&C-ZJvaY(j<ot#u%s%Z6b1AD|a|N$&LaMhR?>6v;70gwwbiKHwux#
zx!Zb>8|yel!o?40ikUadyWIBJ>5_h}5beX6u?R<CajN~T$qGPJqxfr2GJeQ*uxtzc
zJD89$JL=Q>V=RY3rN^VVGq#^R%5E1;8-gShv&b5SPDaXV<2%vwT*Ru-UC{|XAn$V-
zS*moHI?4FkK^@4om@<O}SM-o}J+ae`U%PrZZ8!3qbS-vrnYHnleS6t*qvqVXIWYIJ
z{V7bzwY4_w?$`vxfd0Y*Ace4_RF{3KcmX_s%-o=1@h(zY+Cx6mE1(W6ZwK}PCNXtn
zT6gY{#ovi&K5Hpdz}N_y;?DDaB>iYCo8%(BBX#R0k39V5RgS(!=!d)`4r#Zz;ikK-
zN}u%`w4}a7a}C$05?c5^5yGcA!y2}5%(uP<0I3(+EmT=A(Lidnu4ilcqt6&h&#j6V
zMv|$yEb|$hw<^H;M$x{xOCwMVLA3E6Q$wsLEg;W6;FXwP$^qx~g^J1HvvmWtcCidM
z;2(YV3^IqDn^F;hgZbb*fDTL0=#@Jx1b*XOPhY*OQKlh9>ctPf6-2mkLACkzva@w{
zk)9g@<B8JgLG-w%u6bFS#C-p-9$z^3K7Rz!;opIY!nvk<HVUuUk9xBHep*Cn4Gc_Y
zv*me^p?zp5F<R3xQtp2{J!ZVLc@#(F&@H_+ScCs*bMYFOMZE1!zKt%V=%37dWc}zJ
ze*mX*rw&8se0yDeeCye0jal-ULkkw9@7p>&fiGSGFycOh8QjU$yAL8`2ad2@^<8_q
z43i30{seF>N;DrRW~=|o0vHMNV+#{R_gp27yd3nAR=UUym0H}a0FWQ)eL_K@HcDAd
z5w$1KZLPK=_ZQwWhP8`K6z6-%mY)xzFqeK~fEI}F7JoZO&1kvv^iHYYm5rT9+Z&gp
z4S?KwCHeb1KsxxpoP%GH3${UtE^RyP127&wwS*c&Y&VLQ;z#(8NcA$~byLPTQ;Gu(
zHTh;x{IJxbF7_bxaBODqW<(`I!mZ-;QQ>{uVVS#C9x`|k+hFVmjIT076P}PNhN==1
zxjk-V2<w%a?0#8(C#)*UtL!3;q0)ZHE?H=E^lnl7kTDa4#Pvdpb=9S;+|hSJh*~i8
zFpn*|m6UMHo*H%7^GF=p8_#Bby$)G8e_J<j9~t)}aQK(0W6|r7{4BUU`skOn85&H?
zh!!j1IX-)dQpult+04@j!c+VVIu2|i>6eaH6A3N&I(m{P2Hw^ipFKUX&*Rw~llIJ~
zqK_j@K(^{uRZ|<f$htL(b~f}ZFC|V}RcIA=la<Uu#W9?~%f*2Nj<gm=IE_{z0wb*y
zRhqds#Wi6hMu+#(F3jXh?*W2+Qwwk$uTf%WPYIYHX3oQy9BBm?pqfVJeD@)Y`r~i$
zleEun(vu;`4U67_4N&GVHjE3WH~6#C1Q<;~5ZZ3HZt_Wumy=Q=YHIEq;IT7eBD>+0
z&;ynICKaD)8L!%aTo+((;FsfhTYxup{f9aKbSAS?w`-D{d(=7vhs^TlI%-`HPDo7x
zXmQL;1pJoVb!YvUO?SB9h<mu+jIhA>$v*%(7&zzkyf4SyN!vdW!!T`?d5+nTa^XB3
zk>lFVBy>cvj)j~8P!rQ)f!p(42dLfc*@agXPwXhSKY<7IPZ8Z?gPKPvkNT~h^43}J
z2zgB*mg&jKF1FS*JSi)=$9!63%4hr-#kM6FraHXu)|Im;p&M<kO{YIz1>dJ%U8H5#
zy!_sW!;z6`_k3esno&ZZNle<c<mmb9*+$Ivry=`%A_FkJO-cV`dvze>KV;rV)`|ML
z-|vl@g6X{`yifMhsSz3#OjgRT5M61C@7Bse?g8(1G1n;p$RnMcclNxx))#fxBC6(?
z<!acrLwd{CkLBRCWqyt5kG7+MWeBOx@*@vK$AS|4&}4>&UM&4F2F%L+BrnhT@3KPJ
z1)LaDsmA%_<Zgk}OFPjZa#uwDq`tAK7H#m8Q39s~RQ%72^k(3?fQhL^Ja>t$`&FK@
z`Qpfc<hRUw9}gDSc*lDN!`TOU0n0T#wmUBG6SxC83G}Eo<XElI*G6$8uW3vYG~VLa
z2?d!2wS>OTeV}U11~eUFZdt8?{CI$_$P7pP29EUwk;WNrjQ<c`G3zQ<-s|>PVwjLj
zu7+yb`X#`QZpKO_>ah2$$m;r$6u3xfq66^z@7dR(Dzu6@`ZHr5_*SvJjg6ho3lKD{
zv`sQ^VQ+g}hfsCE)6!R0$s70FxPCDSqepTR4%|Uf>0YyQ)xWkMtq;nUZ$@`wYX*B#
z7+f^&^in^%R=)RH6({~;xII2xpznU`mz(PyHYXw1F#UjV?O}4C-d~xi4UazFd=TGQ
zdk-g2vtVUGTx8?%;$fN?!oHa*njU7&i;dGrOe#0RNKQ6p*7k2U>COSPE3duB>_e%t
zOUr{S)ac#F-NeII9P^6)7z>93H3?f8JKpGiw+@#NJacRDv2{f=GpfU7mB4N<vCDZ5
zvl19&z{_o9Ef(Rf6j9840qKOS-|j=}HgszZc7+D`863(HYr6e_ShO~gpuGj1lK;z3
zvTxJxQrFbi*rvjbtdLen?ANXnP;;Z#r~YtR;?Vrs&R$5ml74?fy2eMyN60$Sl!h<2
z_#Cmc$6c&!ShXSJB^x45ufF>wP`aNiD>ESsK=8@@vujo6<HD%f`r(xjnI3?KtaVeV
z_ZpMwajz||2B_l_MR@A`f~y2lU#b0Gx5<)#Xyea1;Y-0p%y$<LXuSA{=(y`VDVCYO
z9O)A=?83GJ2Y*emGbVWQw^Gu^*-JoU70@3sOaHZfo;YK5@g(`Ynt)j%dnwu;F*QfE
zwHURW+&nfLdJH1WRhz>hlmY#*zo@t+(l=5<O+trehyDt@gVIgu0^lE^U^dNL4$mwg
zJDC(3sjl27sM`>fi`GJH_P&F?VB@lOhSsB%hdztPk82{=VLEH7Jq0{`bFi93AgR0-
z(lK{Z$nS+M|KI#t?Rk;a!vYgl`=N_?9Iu*K!%(q}viGCr=ilZ(Pq2UeCPsKFY(JQv
zSUe}SgK%LxN}Z6t`yIZEDzz=bnvm~7N(ol&1%u^3UG8zhtk?(+mDfg6Zy=+nhO3?B
zMW{Y5?%e$cu!p0XJf%{`O~|M{p#2L?krj6Wb2DB#Koc@K)G7oJ(@_;VizXZql}s{X
z$_zl*R9X$M_$#%1P5N6(mWuZC4Zhp5LT0D#n|}oyrrlfPH$J9_=*E1bukp{cB{>D|
z>i~5B`r{7;k~Yp+0+N1U^M;tnVr}g;c5BGq&<*^KJ=OHRb<ue3a!%>XYf~L$bU3hF
zk_h~wTC-CraCH2#Sy>VOfpAj(;^(z<4PS2)V}LWfO7goS;&H`|LwegVwHg}GNz|=i
zyw1D-I%Ib4S$VSA)F^e^a1vN4Ix@dinbtbb2_u?Wi_dh4x%Y2Hjf`D;rt9{GrYJzF
zGaQP7dag<+^&YVONpErC$TQSeyGE@zO5=j*%DiOC5RAM5N&Z{m2Wdeu;E-Z+tKD?m
zt-sgHX}oiDx7<g^@rGV@e&x`FY1kXtng-$H`i`>UbW7zSC+A`h6Yg@8j2zuf(b)U|
zC9TDp-@a6E2iVZM>K#vcQ<=ktXhM=S_N}%fBSz<M$;$<bdT$2ShXK9^ZKV`)qB2?f
z01h;O6innD^-Y{gSx@>A1+!<y2q)%V*iV$;WZLH42RS^FPWbkKpU#@*Ur5zh7b=U-
z1~@`GQ#T|L$_Z)02i70LpBZN=@(99@D?95I|9%h)X9l>g7f6&!{YRJKq9jx%ej@oF
z!A5GvK()RrA^0p*_eShdKxxIX|4c#Eya4&oj~Y%~-;(|&TO)VpFa^hY5KiRhA6*J;
z#31D>()wsIF)-()@SkQzm*S-bKOtFXJKI(MU}Q(i$TeoVV$_Gwl6!KjC?8TfCP_P;
z$9vf$Y}hoFs<R$sQ;c_~zuyJ1Loo9Zo;ANIt~GOi4$bCxM?)y?UL*B|US;nCy@-b-
zt`VH^Gy+|}!5gSR`DNyj1N1<xp9Wv}k|%d4FTACtkJ(sRn{ggbZI(6}FB`cgvrbW%
z-9hFAzck>C)EoIuPbXI){tVm>!iyT5eB7R#kW<&*7JffNWl+B$yDISIEZG2^7_jAL
z3{iRfKEFPN(M?8k>OwWm-(Yj6{L{Dc#*noKb+N%Q`D=-8ch^01EQH(jJO<>!n4jQn
z>2E85N#$gF&g(2)%kEcdF>+tFp$68o=(_*}*;fT-q_E?MA|Cu+@|`5q)bk@mdR!8=
zBEyJ>*Xwg8=vu*+fsR2QP0cz?ny|r|jJyzr8Z+F7jree$Cqf&Q$lsBeC%!Q|+8<W8
z+ml@^*Rd(tYp)`en-(NQjI0-h!1_CDI1`#)no0jGPjSQWUzX%puclXmsRrQQ8MU?9
zs`0$Fsi6!ri?GqUP!5s{J6+*JDyl3se<NM)WB}cg%naSwzrN_Qa@qVdvLyCLn;`Hb
znWRPQlFl~mABNrN8C@Zim#1#;3f_NH(PXZ9U*p;Vr9V#=I?-*jH#c`&)bx_`PJ<jk
zUhgk*uRj3*i)Q@&Ot_}fYRKR}z#MI>;$rt}&xO^VkcFti8~aPUdhDfdeeUn!P0&uT
zu${;~O=_jzpq9r4zW-bht&!z~a6;06fME_Hsoa4xy1l7>v3sKzXj5+MHn#s>Z2oRi
z5g%mnA7N9()EUFG9U+Ukw`y8JQ1i3`yi4A?*E|=ng$X$A%kbCqeDN3hvwt;e^8{bB
zBQVB`87dMnr$d`5Y!%)5FLgtF%F+K`olYDLx-i7Uxus_F6&GFlq7Ng1mqQhf1=IkZ
zRea`r<Nd|rMo{&Hl1SgW7;_NQ{7(7ZJYv|BmB&9*1DCId&JH-{DM_0jw3-}r*~rWW
zi35tK%=`n9uck2(Yjq8}!KjMB>vjZ9jYnE@^E<9aHbv7$%JEnJ+5UT1Us(Ukm8z2_
z87q%l{(V+u%Os&mFC^m-0w2X`Hy@5#wE!B;;4g~7!4Gf0vc+}XXFranj3aD`Yv;3P
z0jgb}{Kn$$XQcPibp7}<UJ2MjM&mp=Dj(nMefLJyN95uX{`qHu|7MAA>;eBuoqn%%
z8PmM9EKcR8eN<uEM>ts%I@=fkj4tsvqpqHz&A7p2@6#LQ_xr9b+7ixM>4^*3B`Q@%
z+76igJc+4S$g;b9_cLhUXpL_*yHWx=zF75`K@Z5v^heEJirzs;OZ_}-unHA;`+j<%
zUZw1H2%=9vd!*nu2{%0iNSLQFy{H~Dfc@5n;JtM4hS=EO@jzYy0^8<d*d_oW_VCax
zVPS11zkjjg!@&dFCVr{okR~RG*@KAz{R@%DiI3^fQ4-e6!rY7OMe>SGtrAs5CX;dL
z<C)}0p1eL@ma=6v%LZv|TZhO|E94aS9ZZy>sm1j>M&yyuqY*8&@Tknsgph~*k##Oa
z{{zM4(G*}W(d~J2cibTT+}Y(~>x!ZhY3oqed11579sN63S@Lvk`us*OLU*Jj9<pd5
zLfukOL4Y{@3HXz_|6cdamA8@Zi32fC40Hh9&t~}<i+OT~(weLXImYDR*0lx~j&GP%
zy_mm2eJC?aXQo!>CNz*6rac9lJefI!*dhYJT@!~H0+}>tf5b|#BY3;Yqu<Q%1ML`m
z+vo3Hb>2al-`xGG4seQd_(p>-eKkbz^or+aui{~5N@klP%F%7&rx|gyO<QbLpvbAL
z{4+99XBAlHUzGjit@0010f(Yj3R1%KS5gPk+E4Kvs^%OPW-EvWdYhED&WC|0xo6Ef
zZcrU%0*_+yN96{W%%$r#K8}^9b8OScr@S)Pe$J}dxsoEvzWrU{NAH~Cp<_~%3XCtg
z+w_U~q<vGkF#wRVl?U`gEbjsP-OlG)iDpUYihT4TAGVx+j}Xc*viR8A3ckNGZ+Fn;
zojZ@kiV+HwS|KfC(!{Z&|B%86?f+yAVarvPbgj&e<LB_88wXxCmYoNe{{W__fB+CS
z2tCCh??);EzBSJZkSlSRe8#xiM3H4{dV#FtbLhI<nW#v*{v4lN#soX{F96{pp9Iv@
zDyUZEV+}D`*?x*yhp;Y2HeY(OMZ%~iwgW9aTdyzrlLb??D&STC=aAM#Zn^2;Rr{IY
zFaCWfZ+}}B0!O^;(-#vVi->5aCUvhcb573YhieTloQ74zcsfSfmU8VIRehpO*tL+J
zT;hbtm`XzV<;7`H%HJvGST1}PqzbGMs_XwE{x2uJhu^QTRNso+Fb%SFPqfi$x>7^%
zbBo1%zm(eI_=6Is?qj(i{5!~4I*n^UK==Gm`#G|s&r6%9e+Tex6*#Toc=id5Q<g94
zrUfPpafKYdYYLMk+ee(MFS@EjnyU%E|DZLuPnRU5cYx_MUff@QFt8L$)E^d)BmDcP
z>Z{g&mw?WlM@m%}?lo5JdFfIGK#d49+Fe^{v)Pe8Xa}`)89!oLbgIkn%HQ+Ro@d8r
znwJZzy<;(}<2s@s44J#3Z<IXbJK%VIZ*7?PjIZ=|>5=VhK00$F(QpDhM9v5d0QNq7
z-t3*Dx~u4Q0JI~-X0E>kYpwIHkKZzBvi->9lK5OW2VHEIZmM47t8M_~+jQ`l>-(Gz
z*qyIjW08j3b~^q+$iHiB^$~~$<I*s8wV?Fe-2oZVUj^YdK4cH{wdd8#Bc*}NwrLA2
z5VxDUVs1?)cmd5^K%WLUAE?gye`V=Zu<h+eHsgS6c0I=3w%XJS`bzR^GpOxaan+Or
z!-(LhWFtnp^JQI5YYjsD3Lt!w1bK1jO7nMM(_N!x^9o1@@uOM=@mD(FI{62?d~@%U
zaa6cS^Rk_J|B)rys{i*M6Ia!;od4eLSoh*~#2IK=o*H?#URhRJ6B(3Al;9_R1`H?n
z0P-;`Q_K)BdH{^YP@-#4`mNCq{8`VAlM|;T$4}E5KfSPB5HE<o!L^^GdHYnF!vi37
z|0+xmh*qb6i-}EGT@7)dI8x%$KMp_CCq!3IXj)jmx6R`S7b>#0o&Cq&+6_-^zIBge
zHUq?@!D6J<=o+f$iZ1(^iuv>+V%C|DdHZX^eg10#4g1H%(<yEJ5&aZqrhw*_x&{R&
zx@|zQB=0ROp{)L`Xcyji8?ELw<72>cEHkba_cI#G?iLA1jDuerAM(+mXKXb+HLmC-
z{lYhI$tkF2<}#zFVnUnrf|^3!wgkCQq-1=xZjo+1d3Y_kjQj&o#vKRE%_1sUxG)<M
zbYSoZCx*gA`wX}}FD36;DL{y@JZQI*yPnH@wSdDjgC@iB7EDAR2iMupo?3q>>3vX>
znCm-cZn>ml8^J#W3)5m}F`(y3q4Zh>o})lP1FFJnk4*~$JUOIN4_;D@jwlBB`Z&zx
z=eDJ3H{Wd84g}0HttFcKaOW7hB?HC@c`-%0abhSi6|gS+Wk$Cyh)~<7IYgQt2_jC#
z^0-*6yElkUZYW&j4g#3bbR*H$@-D_d>I5eQ*ZJD2o0)wD8TzY(B2mst`vI6<Xp048
zT(^;}&y*guY+dmAm+Pe$IlQ{z0xKSjVvAyD(li*$9NT(hyIGQL6Z^V*0RM<v_PW20
zMO_rOQWaP?VFQQexuh>%o@WPtrGVcwk+MgR$88%Fw{0cyC+?04LTb|Y8LQg`R(8ea
zIZpkHFzqOg$F~VL2`)C;U!Y%~;{`yS;5oyA&hbxE=^@F<77Ema7V90L!A0NO#`UC&
z$;jBef(yFbtDkCFLq-K_R`%B{GF-;=H!D+S(y)m~R<oc0o6pY<>l(c4_pEC5zv!-u
zUHXkY`2Eh1c0JVQ%!=|aUuCf)=NGjz;x&Ef4FInXzz;ZS8noS?3X9S3w10W}5>UAC
z*Ee_p$o4z5;(gN2`&z}oF(!{OlzLDyK(MjiML}~BqK0m)y3X}3O#%z~VJ9(!UDn!W
z!&$Md;lr)OuWQQRw*#OHBlo8Oxn!+5A8th=cWl)!&#ut+IS#D)eJ(cY?p1utz&Z-T
zVfCB;ajz-bNCNPHbhYUC8zt?C5!nB~LbaGP%4`=97nhUC6-womL(lXE^BWBTvqa6M
zz+oq4UZapaATZan=Q8CP!08{Y37~Dwnm04jWdSR<f4uB+@;cZADAL#cThhR0&FQC^
z?OL~grZPZveV2goJ@PVvDnB<J^?2}5w7q<<!qkM9#`*je^2|di8`I|8u3SZcycWH@
z_PzZVLYkHES$C8dKo6<f15w(isW;DYMUoBejCZpE^}exz!OL;EWM4+&V%W&w%+?`#
z1ko8}GseD2Y86}8;9di(4HI6iC8lYN*!z3#u_qYa<lOXvR(K&p0+jWSs*iGyR(LC{
zXH=_E5mG$aFvsfkaDYu-ne>a6N)T07Azp>3EM4EePye_b1T_#?E#SXYCw&y(O$Trz
znx$DGqC?tJY7j!kPczmx_3rEH%dv$>@;QZ`v-$*G1n7}$-e5j!3&cMru1#GwPjy*<
zH-bKQp~zUO-V~kEaG2|H8HCqb9t5Qj;>Ott&@Ixiz7?yk^;g*`PjA~%fF~>s|I6g0
zzQCrL<L_=~E*hV6{UbT@3L`ePQh@Euzxoq5tLIZYKyh(AF2rfCU|U`iBVYXJDb%%r
z<RxybwwF&ky;nB0*@NERebm%#;laLUY<(71U@Fq}E7jMS{2m%pH~b)7Es@KA1Lb`g
zG|Ng3=)1WmkAg}rAKM0ku%THqxaBRvqr<?W4Iuem?v)wHv*)^yD6+C>!rq`mihY#_
zxb>p>-T7E44^BddPE*T2zyABO#D}-Xd{-_w$3f2aW+>0JP?Eo{I=#~$(dk816zs%G
z7u%^zsyj=7ltei(&(<@SGBqxLuKsv@dyx?}ELLVB3wdj&AKErwbjIO0hn=z#_~8t3
z|7E_A#85%iO7X5xoON;^g5k)#vpQ>}kwiD)9f%)@vj6mx0U}1De_NJSCMq<3$hko+
zX!9OXYZr-Ye8WtUJh5vK10BGcSKk8yMq9;#aQa)XKY9t#YIsa;M8h;`>L0_ubffRM
zISoH)w3gcT@<L6eJpUk9;#sFzf0M#|z5*a^G(QtY4o{1$i<mkMpIudcW`6hrm^R1D
zt*Ub{O*fGuq!bt;VQs1tU=-lfRMME&Y~ppQvpWSbqHM#Pjkz1}K8V|n0P*kJ@P3y(
zH9NeWd6-#}gIBw-b~4#a>3By7tO8)tec9Vs_BfE!NuqhhRFVs&mrU2{?T#Pfzr<%k
zP~TAq9i^KqR8&b$yI-#y-sJztP=d#4_m(abbd`&Zrb-U{szj#%jfxqzS%o(qAJ7T3
znrymG=nEYc^cq_P@PIu}QNW~5plD@|Q2=+%z)19Zk}TfD#&-%Z5~PcEM7pCx-uLw!
z8;VM57~=g*%(6?nu-w2G|77fTl#-P*8VM_B0S=>p?b-^qy2FlYm$`JK1WA!rn9?%)
zIDMpv5PMi<vBJ&nVP0lxLNe1PwJ<<q_l@2<zfMae_h=x;1rHntm0s9Z(X#X*LP>i?
z4GYR5!N{M%rqF`!r$yZW=!Zt4+A(@k{U=k9h||UebO{?!&+P(jG6UH0K&!S+*;{n#
z5gvScdc-VA-O~NiR1z4FQ~*&F98HcN8q(16(&GjM9eFSh8wFl%p4Z>d%$U|ZzNS3k
zk16t+?a)69BQ-Tb!B@6l&Sq;$AMT~^v-iZgF}IE9Z~@rfJ8&-VBam_ddVCWp@pL!x
zD`z>OTk*<Bk(@A~=0`(IAUDs<Nwlc{A$Ceu&h>E>MT^67B_a<K5B(EySaC$)htX?E
zjRf)qOJ3<2i)p{<`i&aW(Y<1k%7iBLg-qn_z|3q-H>~D3IY&QMyU%e_d9o%&e;NWF
zr#`@06A8onwVtz%lROzKNcO;dxMtT<+xapsTjDfL{%Z>D<>OR;rQ`k+(ACbcH1fkO
zxj**g#w1zkEJ<i6?cZmo@h?DupR1o5Iw=*;uy!W2D(zfRS%{sFhkvY(4IC5s@Gg*F
zl*PB?m_XQyw}1?znJGmzQLLcLsk3@3H+(&7P#!$~cv>3*F^Oj}ncQUcY*yYK-jh=(
zf`_FGJ6DfHdZzHlHsu=o#A!8jucTEY9sj3$@=4RTe~kT|?%+ZyD?_3V6N#Ot%P>}D
zv)<@_9iD#<-ZpOS^O4EJa^<!4(#WXBR8}$lCZ1!x6VUwV>etolW_iT~2Hhb6Yeycq
zW+tArd=YR<gmi*o@@GX2<^fHGLo_|I?Y4;h9#`1tT@ia>z6LmljlrZyxyb0KLK|S?
z><qoX!9h1R0jxC33RY6oeb>(Ix<Kg<@)OlDmS60pV3cizSfYxJX@}h#50zMB#^D*q
zkb}TzipDSL!UZ4Bb;t(=f#_vV&p&1pf*Bn^d+DrO^~JIPV>whC4*^UP#E@~^jQhol
zPPI7rv3W#Am>B`>mi1f36}htWsDnNXg|k4mk;5y+1@U0-1*I_($}fCC2fYckqkfSG
z?ocauy82`F$Xr?O&-bugVOUA;tzRua=FOy$awswJ9c=AsXnxW#aX2jsp`f1VI&^k_
zKmaj>uZioI+T#7+7D`r6!O+xkF~K(}qHTa}QJVM<b{exD|C}vnR*0e-FvacW8I8Go
zC;SEn-I7OlFS#fwqvI2MK{B8fcuNb;Xbd!4hB@!+f|zZf+)STu`=Gz@yAd?{(!&{>
z>`VxUR*-+P5=aW-BpG62;r`AIo*Yebb@jmL$C1bk4qlb<C_sm+9(dc&ap&n;46uqj
z+GG*0wV1l3D$Sd2=<3}vF)s?Au|71$>7^7+Ew2|_P4-C4^1w(rm?0Osg)t5}8g7su
zUEE7CSaDjxUI~0c9q1%?9X-*oOyBi1$Wr!R;!#>QvNubCyj=7eni^T(yYvaZvb6OR
zWb_@Pgki)C?++(LAsW<%MODd;Vh)Bz;YumMGrSGR$AxQvt-@W^j{8hR+6PESa;n&I
zx@YrTMg8?>JzkYcVgW#RI_1cC=$;*d%mmynzWCDrqBTnC4@Mxm9+ot46eOPXNLgwe
zk`7%uHGS>3J3(&S9VI<g;x^;5*NSP8>kNHEaleweoP9h{{Ul#MNP#~ve`z>4uHSn4
ziLwd~HGa5u#i|&+_!q*hHLDdnpX_AaPJq_`Lv)ItR^+j~dMj^jk_v`nzWbUpdM}*$
z@2=|z1cH+B^UAjFBHzhRs>WHML5Rvcxd82q^T*oXq{d_Wed<u#9Y@(x%aqjN_DV{M
zLMUvm$420|XBPG6L!W~!5?^*m0>WN3DBN~uW&D}YOnMK{1N;Ra&Epf>OnSJm(Q;zD
z_7otQjr}WCP5eizFfZqN3>guY^`qw2T>noEF1=Wr#4vO;SSH%4YD+c)6;hn!)%qzv
zZk=(F_et#iy>bt#wHs{O{9VlL#@W1;U-Cfw(5$56mg?Pm{?P@!AY~PD2PGjO;dD%T
zH0~p0ObT}Qs0>Xin7Cs*8@7Fc2w=@}x4uQ&`yPwK&IpGP%|nw%;Qs2prd*xCQY~D7
z<LmBzqFFKOBZIORT0VuDWTIfLnorE&$S9b)li46EWE@06DVR%~4WIJ#7Zwx5oa>H}
z0)VM0rl`v$b1NQ0vaW{xvZ+~dXxctOI|=Nh(Z|DJXXC|J9n5k`@w=_}?lSywKmsnv
z3vo`59~t09^Zb9m>tNx@NgU~)M0$@G71R`X-XboV+UBe*I(!j=!?M6XQ)@m*vjq;l
zea5j-eVqev>x0Vb*AtmX#*ti^Qo%&=;NV-Uz~WFQAt#1hpSFayJQ}+zu(8(nOA~*V
z`8eeM<hhksySXQzJ2|wh4URa7h>nF0J*|ubF@x1(>)g0QX>NPZR{cUE^Z!oE_T;^;
zDZ4o_9iluH`y7R?7fdmgrAVD1X`Y=eaIN2h?RPZ}th(x~+w>M(KLG@`A&ksCnNred
zmghBDr*X9Qx0kXbdPsF5a5^9Ue5}Z`qB*Txx$FhZf;A?spj$3YO@Xq1x^^t{nVMeg
z-cBTPe@KVJjely8h3p0Z;KD?;ko`z#ZmM(rL#aV%!hT}PTkB@W=U0sNoCu4e-S`Y+
zkkRsPwC%`C>EXa)hYsTqfZybByI*e{p_A;huAz*77^o*<(j1aCSmvKBlHa|k4|!|x
z-r4kH5KtT-_4V`7npyH9e3!i5IKC`9nNR(R3~`h#i?sS})eErWj`5BW1F#W&zx2u8
z11>V0bGbrZs#|uSPgcIqb3lJXN&-u1r6e9b+!AxS25E#;{I@druNSd;pBDeJWYxb|
zEcn%QynBm%IeDy<eY_prEd*G30nO2bwWnLi<eI({V4rzxYEv`?$QEuYMb(%89H#%(
zXY+>Cg@wwE>5wvkKS}FaMM<M!&3(x1nt-H@*W7k?%P;wf;AZL~3&v^RU%4i;R0c3z
z0kuEo)C}Fco?46Fin|3!;bHw0vmMtFx95HOPt%xlBLSD+Qmst?uf}Tto}pWpU9a0w
z?t)JE8iguvTLhDEzNAKq>FC_yQU1{<KlGS?TXyBtBJ4x5)bRCFkjtyeYKdtQ5qpj_
ze@WC<;;c|T^~|cG|9zZ*%%-F1XF||&X$JuI`Y=pwa?v~D2ue~`Flf&xhIV-74PuB8
zGJe2ABYf!DY}Z{2Ik*AIBO&3>zX~cL<;Wazc{>XKj>BhPz+3BAu=2@;{QdB`9{hG=
zr-#?JfX}wx%G{@`2vb}rYbvGG+dV3vZ;cT~%2b7v-mt2uSmn^gLDR`TPgIa)ohYrq
z4(B3?(kTNh*B|e4L#=PwsSto}Asi>j{bN{{2GnzzHOL~?72}B#<+5I~qx42x@Q<nL
zQ=Y}DkX3<ozLmTL;B#R9-;&>ixaIccgMk~9t!!_@4OD-5v+lEEorobAGRbUiU&t}O
zHO{LdePfLD0PszW`Z!nW0c9)Vfff&jqYlm=yi)ecNZ}j?v+Z&Py#s*8IF)i<GJKUk
z^YjG2;7*>uwzb;p6zQ?f#&q=mtS|$hX+&G2^pLX=50J_lfF#7?t0#@4p#%CO8ok<l
zz@Py|fR%WeF!T_i8;JOXd?t$ODL{$(v>DCWwp{aH#!Z_?R+#V6OpZo6Y<J3i9ywYr
zGj*5e54nl1eX;k7R!Q9N%8tI-SQvlDCk^nzflc<-6f|%h1@*9$0;HMZ2frJLJ1}S_
zvaDs<9e}LK9yVL&cmLC3<4=v>H`U8KqlHbX^26EEO*K{Mb^#t1;GUR3SZ)nOKUGO|
z-787A3epKX6H?a8_%HDufA&kr!GZVwmuqwakYze-Xh)Y8_AFfB_-nwNwFH#@%B&q9
z*<6EgG|)zZD^VF&7m9qAbuC3-`BkGz<yv4_=1WWA#-?|iv&gax&_k`GFrJgcD>9+x
zmQ|we*y+NO5igR$>&DF2$f+_ZrJfMk>8D7%!k8oLQDm*5dC_!K8@9X_Q5yXh!n@4&
zUN6@6{rpsD>?Zp%ek`jMviG=6c*-G+^ch(g4eUaP698`jyj4qo`};p{|8vwTfZ99X
zH=0}-+JWB7nB<m7^|L`42;kVud%z#yss5G}DJD{#iezjM^{aFJT(17s*@$|$M5%|8
z*c1dD9O2RUWFw-kLJqq$cc(n69~}IrAPo@HD{t=^FBMqw4eT3_%ix>hV0|TB1zq{(
zK&mu6a)#<-!@BS*qFpXoiFg4j!-~6nC_O2SZ<_Ip1e^uCPfeSPHi~?9P-%DJKj_~A
zEQQx7xU@IXReQQl`Vn}9t@8RETf()gm`aMe+W1wD`MNQWjZo&37{A(|<+VFI+T`02
z*}4$a$fny7a0?~jy|V{h!vPb4JOFPOu}W9=0Nu(6tDPKvxm(W2+8$wz{wg+^v@qW<
zCm9>!Yjh4ms#)WkvbU)>NM9R`P@0<F`_o?c5=&k}NH5Apkn(dpAMP3(vZn5quR&u2
z4x9fu3}n@hxYL0Dost|gzdn5ZYS-t=#^BgbeT;*>?#pSknNJZo3$3On?m2Q_NMVH%
zJ}LaNMAs9TY@|vdo<SGqdrJ5BfItH~L6BbIH?uq#2!W`*)HVY|z~Pg{y+1~8ZvJ+S
zb;9spxK6?;TsN<~wHFhZkUAB@J^h2%%NOQe9&`r<_<(EYnRc%TB>$3|F9)KL0rE`|
znVBkpS5<6#Xa$HoAh#R&L$LwC*7&1Ylf3J{?y3zj{uFT}Gj00PO&)3E<$fnn3nkm0
zmHrEvmX>lyZ>MMI4Omd8UaL(RbQhF2gkoz#9pwiB=j>Vy2H2ZaR9vB4`Je@<8=;{B
z5bxsjSWgF7z&%jw1w3~efKWo{Rl7MgyOJp|*2NBlW3)l$R0nu+MBUAk)$2MeO9XHa
z=3UMMiV)Y0E#7)MqAZuKJafP`@qv|qQ*nZSY8A6TUwC1^m1M%c#LaKWEt7rD?bcXV
zzO0r5!qtHS_yRBYh#Jmn&nh&Su^!=kjKgQwYDK+1&yA;zA{vIuEW#F8007LV6quIX
zHRG#_%+n9Pn;%%TJWQ7n>+}b#4+z83U#+b5PspB#@hkF@`9?!F$vuiQF+m4mXK*u2
zb=$Oe3XmK-R~duhw<?HzoJAFLjLNS6bCr#o$ljpCn6QTM+14O=qHY+1g`#7JpgT`Z
zxTeWpLUoE(#vkuHND8rrnFeWjr2@he`mp4DZ4XeiQ5@$L(Wnl=8<gv{`xr0}ro#7N
zjemVI5@U8HI{hV2gdN61VuYqUHFU?zV`X8x1|X;gxoDk>R_DqFq<>6sx5d#$7(+mR
zlfYJ2WM^FjW0qK2^|#*>1HZ2mRw?T5$n(!@E}z3(xIO>$`}VajvMRr7Pb^;(B}JSX
zQ@>4-{S@BHi1GP0b>{W_=_G@)lMSM|T`9@c=r8Q}gH4sTZE}LG*VDKmul9H{8zbc%
zk*`Z<a-h0FFk)&k`Lr0d;h7F18`l&GI2>)FXDX=S`Y;YswCESA05>OoAaA?jpvPSZ
zRRuIk>cMg5H5u6En(FPKy8l$vzyqxwmD^FQvcw~fZtHMD>R=_*S4TMN7E;@VC-Jsi
zH!Vep=E3Z|`*ZT$O$_@`fLMHAtoyA+f%9yJAgKOYL6_^-`-y;Yz(A^_`j@@(-M@gv
z9T^Dy)I6-F$}us8Wzgo6>FPrG5$_)iZgc8irHboNHT3@Qz47dD>dW1nD*$KH{qy*V
zlHzaHEzk^PE1b4-7djVub3aS*#$B-n(QoB*Q>w~R*~i^K1WAWXocxkw*=wCRW%80s
zQ~YWo5-vFw34}waH_s?mqxjj*Nz3^FzRJLmeg!W)r}>embqYam8XQI5sp9u6jn4?4
z8j}Dd;K!`!G4(&E00Ft=tfdDZE)yMi9(Z8JUX|7)(7&?MPww(+RJMjGTFyTSpJ_-P
zloUD|d$<mBHe8!v!I+5n8@din;X|h3{AQBQm7&cA$eOv0k%Os_--<=$E+sSmbFnny
zI&bqg(_2RqPY<h?nTsek8>ZKZziRVhT_S){D58=}?*aor2ke%x00F|KrjqS|gP7yb
zUG?@8Qq(K(X5waq{Px`=&&|~Tz9~4KUns!KJE#t$yjP9zk7ez35T9{P4#552I`$GO
zwJIVgZDt;@*xDJfzIO7M-h<$WkF(-uV{()!|MNNk1i+?<W)Xq#Pbe`(FKiWqYVbpS
zn4E{0^NMKuWy=+>->5Gn#-55Y)QuzK1oF=G2>g}glPb|c$4Fh|bvb_B0m9rr10lCm
zA;<yj!GmZ05T_SjP1m+%#)5^Can*~p;i7oFXS?@dA?C`DaKEX`mddGJD|;D1y(w6y
z5g?D#(6i(NjsYV`Bqj`x{QwRFlj0b$R4r4v?9WA(FGu$7k*kl%%X>ZFdbSqJpRilP
z^b_9U_V$P${fgMG*(SLW3w@OB88H!5V`VAj!JFwBQu%$42*-4x3RFsnAZChjHGa!y
z8ir~&FW1RHJtt23*O#Ldi5z!!P!QiTNRapA0jw&>ylsJj+Ie#<@K;G%hRqG5lThaK
z9iR?SAV<f1%eTdoi=Ex}T!Mxz8Qhfv`gfY|axpvEhz~bA^lCB1*WO&;&K!I>O{4f9
zxu=s~Gf`qZ(?u`t`&R@62S)?t^d-u5LM>+*0ah40aQIM=#m25jSIoW$!fTb;<Nt2U
zB5;}(rU*<3&(o6RkK<~msUOc$T|%zPesJajBt%Zg{<lg`?GL-dzzVgMfTkax@ko%n
zmcQQAc<s@}!ho7l55^xH-%AZdc%sNJPeR^48P#X)W^l01?s~3^^aM9H>+4-2d<X8-
zYXB-;Y$YGQV9;m3U}Hz<E2ycYR)1M}(6ijGUP)dMczb!tZ=LA`Nv7}3X+YO~7EVuy
zN8W#<<|y08M#>(*)1{E<4iu|HacE-@#g%;B*$?u=jb|XDeXRL<ZEBDwT5o%_3)<<7
z4|q4NtsrYzaIo%(ai|IecW&nKAWDbB0xRb$Cc&ADzlYxSjjc6-C$}d&Gy8qR+&Z06
z`*E8MUfqjLQySEoZ5MX?#foqL-@n#`Y+~tcsfSW(R!09`{@PRxTq(3%?>GIz{xddH
zz?GWJ1diwzmiHFh1g<oPN~F>+{C^p&Y0!-9vlpe(igOko0$(yv;8&7U=~6a*KF!hK
zcHHr>3M!}F5F;-9{O5S}%|w~RVKz2kiX@K3p(%r(JQu!^x~QsP*NB%#8ck~I2e);i
zAh5N2d~(R=?<(?%Wm&L~h~b)B9#&<E!*zOkMzjQf=ak_zb3K|Vv}U$Sxt5qFzq0S+
zw0^94q-J-N?gDFGkq4Uf*31b(YO(T2OFi1Y;^3q#uB0@hzt^8Ew+AZP`poP}Gw2DN
zox1L<%Ml$B6Y}hnSK!jPIyHH;<s@{qfOvs;nT!Sh#`&swrYI*4)0Lv(NNE*%&XzII
zL8@@Sujm|y1oy9&W<=N5p@~WRfdlE|3_?R9-tlkmTz)|9ws!)mX^{CzA^C15OiSs{
zR@D>D6J0v1Lx9cx3K%zc>z`Dww!3BriYxxBClPjSXv3mcOEY_uVt9iVWn7cddU;%L
zTglJbN(t6|qoa%(6`Hw>J^$D_s*95YKQJCR`R&oU#m*^>-nNL>yAHrXjm2pD(_Uat
zPrW1$wywS1L1|ARudvH~*n%=sp=2c>`d%4HfvUJJLIxZiJTL8c`9}CPW(323_Vi3l
zU;0dt0MDEJJbYQf><P6nhXy~6xGZ(tTNA$Q5x<+L+Aat(#fmQ%T)NeuxcSIdyc<F?
zF_&%v3_<XYb95e-9)d!k1AHn?0+eiA0Rdid6gJ?W`)H}Jmichq$VBsQ;UulwWJtU2
z<+M7rTm3G(@=ldvl%;f2kXdWK{>6>ND8)&jTxx4NCQmLG8HPKxF~Nw|5g1wyvO0zn
zB{OwqGJ9?yI0mlU-j2eC08i9OpkUh<^NE(^`vNdg81bl_%JIJHw0QiTYJ|Ny7Abvs
zE&K8ovbF)Xz85zWfasmZ>@{fbf8w<qe4Qw-?snv5trtO3$^qe>Ua2L*jn@+W$xD31
zUHO3HADo#q(M-6tozeC{Wa+lLmWGXNV5Zd;IJD6hoN9#eePL^0Fk7iy!Ud3D73`JD
z^u0!fBM!-{4BEnONyQ=KkYg>YyZ)CKKD9}%q*7D;V}{qleH{<0MR3+tcqtE&9O$pV
zFj%mgUz$Dm7!VULt|c0^g+IL%a4g<GfMp^QPMysX<Zk%p%sv~b-53hE-CNLOo!Xwa
zniQvR8=zNhN6N*yuanp^>6R=mjw$hq#^k~JAB*WH0RA`nj5xI8Lh=#Q0lZ-rg8xhA
zrxfqsJ<u?e>S6Qy;pP{fhgSTE9{(u`Achz~cBq=i#NH0*Z;K#h57C0s1`%Y>fMb($
z+}UK+vF`OWL}gjnbZy^wuSNgzZ>Rr$@BhRjmhf8cBahgq@ORdB9z1;PSn!KF{kB-o
zt$WK?d9IljSglXiD&7>-b8^aQ{yPa)x|q&J$$eJ|aMkdmXs-muMUDN79oXaM5$P6^
zIT5Hm=r{+zhK>V`zeDP}xAsa-)bickKD83Acd#s4wb_%r-BXEmLXVU)NWwF=4!`!I
z#mt(;dX!Tq-xY30yhAHT&ym0W7D5h3)b)5C`K`BN)1SSr5e>tmC*XR6ZeuFtEUA;f
zm-}r!J4L_LRrG|#w<qX}Q_kJdJ50Tc4K+4Jx7!Ylu-x1|f$2C27$0J;rue?xU3~J%
z>g!eYIHXta7n>v}V0Zt*J93{B!h1UA7f`DB^J8hlhhzCAKXI1mnBkOi=<m^?;`0SO
zVhD$b-VJ}AqYNA=`Bfby=(xFzuavm#@G+C%TKqvgb-t{~taiRW-D~i$)%saoCkZ@&
zEnmZ}$hQEqDa7JTUMpA7qSRo*+4DRPg|gd06$S-Vr+Y_^J&^UI6aza~U=av1%!sW*
z0^<h6s4zRjhVF&fX1WU(P?CX{RY$woEF@+3*`w?*_t3jqwkwULFa61%_teuIyozWx
zl+cK{ZkGsR=W0(zty<!Co~CW^^tKJn##V|@*GKg|lEiso!m`R-w5bQ{0!__jKY(>1
zBr?+SIOJt4di{jAxfip(NEO7s)s}t5LY^qZ_e;Eh@m+@3Wh1LTo%fkhwoW&VcUYNU
zT#843MyhcLv6JEE;?N{$3svYebBDg$XNo`W5?cp$c(FkIf47MH>jg%|ZuR6eEv3Mq
zPxsh}_aRaJZI(7M+yZh(MX6edzdL2ZG=HF7BjSH+9!-$dMBs_n&jLY#d}s>6l)FEW
zE8?9YQ>XZb{G!{mJ)+*^bMyV1>dsE6_sW~@+b#1rg`t=Vbnp}MR(dG$(5$A9_5hbG
zPYoZ<I#Tgrdx9J;BeFp{FMI(I`GUZldw~4+RiXB=cJv3ai%OhgPPgiklJ^a3b5l8i
z%^I)^twQh<-$)hjF2o^cAPZ3_gv(_2-S5;|zsFe;Utx1A$6VYzho6re$3Yta%oZL4
zkG)2FJ9JGuX;aVPW)ID8v~J#L47v$esDup%25KJW#5=egEj<+h@z}|yvA^n1#W#Ga
zTLmLWyUI&pxtK&JU|hKj5R~xQBhSE-ylj=_xz_vdpT?C&UPVT9r^YXuU1UBZRVj0G
zw_AC+=QO7K$%!ZG=%|}ElLg_EKh5NY|L*-U_P$N%C`s%Z<xVu?E3QfI56_&CdrpzI
z%U_<0^I581Dc>+^_S`{TXTRj^=<#qN8Th>ypvkQ#TjD$~Xf{^rA67OoVnT~mm0wFh
zKjNEWeP5ExDO}{J`pS2b5m#ZcInakyS6<DRL4_*@*@3b$mw3&bn(%5E;QN6!^aI(}
z+TqQ=iV9Y34+lUpZ%Wf16XUXswPH;g?ePx!Z$(26i^SajWIaR%7QlqX$hhdT`r_Z)
zydd$*ObOEdOnFDO)uHK0Qc{+{bj4U+%{poO%JLhsxK_k&k?_yja%6F~JQCPq`&SeF
zM;c*Q>6fnRr&Wn<5^gQ~9JbFswC#CRFZ!;q#~5SU?|v$&9OF=0mcNG2j>&Jn^q<mf
zPRR64_>o5DvM+7v@E_h`=+VOXhv5JA0w4?@y7c8JT?IR}4Y(*8pa;@ZYQ4EFt4pC`
z-+;J_#TOL2?h61MG1g;m<D;xa-w~Z632*E_AO@oyN!L6-L?4&K#nwe{S+isJHCWb7
z`TODJMQp)l(c~TGicg>?pMZ8b;!r=@n|~^1gz5l-F!ov2W0~EZ-tUCgQY5JOU1zw~
zWuit+s7`sEdxWB{^71$Yc4t{&j2xM1fgmrO#1yj?IW8we)Hsf~m^^8^NVIsWJK#8m
zBtD7&I*4>s8-S19F{>G2*o?3PGc8KU;SQs0d8XR9H}*SiY9YOQ-)gbTz6Y&=Ab(Mf
z*mZ-4?VyAQ^!K0r+b-dNWeXX2?mIy>%-J0qDRF;SDl%Mee&cvDUEzmoWs$#+0I04c
z>C?zN>mku|`XWF^l{KuNIQTRYC@`oZFFln2UH=OpDCbKFpKULX_7jg@q(tThkW`dv
ztgteN$WR{QA3t0ii2FiM2dQpG`zy>?=^?u>QaI%Z*KPOf1Nl7Fr73l$h_-4GKfV9n
z{J6_?SOo!&K{HvNOi}x4x`MQ$--uO#2QrJQ@q)B0H`a8~4DQ0O_t#HB_3<uI01)~f
z7)k~YV)ge4)+e$=ImxFjy{)Zd%k-sLureZr<aMEh9)I1b6GWMbZ~7nZL24X6wsyZd
z&M}MOg2ddIl5#SMA)jB2*0^|Z&5{i7qTjDZXUhKFz`A(Eivx?W#yL`4xe3RC<Q;IC
zdBn1gMWGz)y4lGc)@W7g<2!t%HQb#))6ca2lKHG3Z3tD&SKH(W|3qNOgo<*il0&k?
z1O0#v5jNJDI^=z}4I}aCZ99vEuW3)tyW3;Y*Vg5{PcqO|#Uh~NNVNK^<rkW1ihwKE
zkLG&7ESBj@`*|F)edKc9J{3lKXC6Gm{=UaHekd|@|81zy+d-r7>cSNKQgsgvm?Q+b
z&Z-%q@F3BxY=SHt;WzRTE%;qf4_G$ixN+Zutpt1C`fsN5>5mAvu6ofH@WLRP5$$iX
zYRK8am&B<e<4&SyQF(a~uy6{221AOXS1pfSfMB$A{8`z6Bk?ONfA{mF%N$$C;Be@h
zSgF;Vea7HuzsNU%39C1Dzi`RL@4Qx=;vwdYg*<-c5m;z@@4Hxl!^K$WX+aXfdr&$e
zbTeADGZ(vnSiO&1r3B-{>BI#G{*(`ZlD75YZ6r1W2Zr+2^9>xjg!>;pvb9?fESR(h
za*xaTKkoy6oc!1A;>Twg>6MN2r<QmE_3kau_V97XYAkDnS#a~kSDSXBrjPDV3Nz=0
zA$zzhiN{)0{gSz4WPjMc^MwP`PA|!e{~VCx_w;bC<R_f3knM;L=?3R%oB96ece5Wy
z^gQId2SYANCVKulxT|NOb!uaP5lwRQHD4XL3&QgN#S|dp!_<Qd7PL2VEEe^1*ViY<
zwr31szpnByLRU4Yk;hg0%zbg3kSRrtzd0c}(N|wQrTsdti({RE^h`W0$3=q;FHk%k
zK6kdYli%CCc{EocN{2rHRl|P*liW6uBr{<NV$)4$NUv5my&=R3e8i9MY&Lsbi>me3
z_AYR+;`gxi;Zco9d~9BtHF|K7vKL~T^bW!wIM^&3>_F{a=k5|^@?y0=T=v(|Nt&K3
zw7viF0=>Mj;KIc?P|ru8D+RUn;0!Q%=BhaiSe64A)HUbuQVwyXQP2lj+LMU|5=953
zm#EPc)V#nyE&^Sr<96hf*)Q>Xaz!$4GZjc{CtMIcNod>uw-ZZyoF|XJc2B$D(RYzR
zZrY(dZIxe5t<=MD9(nqw4gKx`XF6<D^*L!xivgK*1y(jy&<j)Y|8$~dYSYkfFM~N{
z%UbLqkYfM;9rOQn7V15dc!2%o@ctS(7bSH&L?nOBp-Sg<%-c_yoa=w8gU}u9|0At3
z8v3oYP=?hXT<`_eTLU>2d~W;W946VP_L^D97t3Awd&?s<#j}z*^Zi(y#E`dyiV@7r
z$Z}i(@3H^!16)R`DR7SNt*igSA4W4heh8VvvIT9Nv~+ut@i1JsAUA0@Wq42C$I|L#
z&c<GWxzXem+inN{G%sI(t^Ct8?Q^+iO4j?1j_B&sk5VW9>?@U@0QY7T9K;=Y+6(&4
zCX1=ZSGso;l5EVfUMT`1g1C4pLS3NALmB#jG_3Y5VlW6r^E%8L#a@@$1}#qpZN=k8
z$~{Y2rbTo;59eOYH<SBQh392~|F4b>BXN`$MNqIX>AK+?2$lVwN7BMSyh$mX$p(<e
zKy{lZF<AseEr^$(drM{@Utviu7q*ac7#Mr<#pwXmk+s@WKROm&E02Gvi28%O3t2D%
z`@<?7*8rPOmNb7n2jTl=faantgF)oe2-EF_@BI7?>kloYQ9Ru&^1o1JmyCqD?*fel
zTZx|I(KpvQ_;*kY4EBRsiHTb5bl<*A(bdwniCl@IdQPCu&aK3bat83v^XopPar)AB
zb`)C1aM((m(|dRiS9I|rtbwI^Z?Eu6o9Nn-QTtBM9V7IYk#gZHsNf}zOQ&J4=B~K#
zT+}sP{jRQ<l&60>DGwC$zgT<ku%@!MeN<6IMMXtHq^SsqH0ja;7C^d)(jf{W9aNeK
zNyLt{p+hJ^s?s987ZpTmsG$>;5<-y@0wg4GR-Esh$vE$KUFZCM=dYP7hMl$dUh66M
zeLv3=>}|(=q?zJ$x>w)ovqy=iVmW_`Fp0II4pXyhJFpcP;4GI?QZ@Dm^8JUiKOScV
zpJcngzsrQ$hvs?u3bLPj<G3|Q4Cz{+>@Qb;@~0hU!|-u_{z!|$zj|c0SFSHy8KRak
zeNek>`Oc7UC%%N~;kc9K;wii*62QHhnj;Qgc=<`QY`1XRu-11TO8xiv6!UxN5U_s>
zIjQantbF<S8W)GVX_)u$+PUd>A(Sj*x9N!9^BsQKZnbeq7=m)2FSld~2vBWz=BtOq
ze8)N(?r1b|im~sEaEF6%IiTaa={4Do|5+rk#5i6gM+E}waA-`pJ?-?5Ex-Cre2&eH
zzj}gxTW3E})v#7ivd*RS@ymRtmW=c)?5j{=m8=&POnIaG0H!0cFPlFl_C3^8Sd;~%
z1=|IuLA8>M?4#&yPw4y4J_`ydfxX~$t)lvbM}O&(u%F)H(MqS-Hv0w(jJslcEy;VG
zALul{R{$`lX{@EnWUr9H@OTVVb@Ktr{FP$9z?xtAk1_N<JTzvh7vjX*QDWYvm@mKT
zR~|fPrA=K`s#jK<o7u1I6(}xo-yfC_Nd)tKqdEGw-X}&=>tE-_nIKB_xJhlNY&n=U
zsdD8z*c0R%{wKhcy(bNKeq{BfJ?S2Bl}0}^A7_0Ho>f7Nej{HRxZ9O=#Au)<EW~9P
zGP1R_W{j{G(wy$G+#v1#GSxkqbH(Uq=}Z8EdHxlybzYhJI3=BEm5s6%_Oe-ASL?^b
zRPj+vuX-c5MNF0XY}<y;i$f<Z%~i+&Gi;6fUzM3;H_?xI_Ugeu)nlXl6^I!d^TntK
zBa>TPDy>?BkM4aV-?2X(F-qP#GT_#w)gAa)xBXLa(CSO?%KVAwYzXzSwHbezgk;7S
z564J|NKBKe`=@w$o9+=UPy-V1va`J3H^_H}+jXn8Lk&@yIvN@+rluolQj1>}Zf)c8
zhBxcD_5G@dd8GH+;Xt?~pE{^dERJh7Bi;c|<0CB&A6~7B@Ehg36L<WNGGfJ@=n`r-
zr+nV!xo7u6jmNo3qWd;r*l;=N;WUEQCpxsX60thUqlB0Y?QEW*MyzC37J7V$VNrA=
z%~_J*cSFS>iYljl0+uzBh-JfZK4$W>3T5;>1J%3?lKlvvp{sS6sXHP<hqWpZIih{u
z8EelB!r6<PF<9YOC%iXiRw3^r2QoeCa+sz*8_^<MFX@yI+ApPU0a2_5|8O1($-M~_
zF|=;s#6Hb1ZM*)OdS4JiIgI+RAAeMJmu`E<KM5ZN=+Q<R$^Q0Itq8@K{2E`w!Jidy
zQPY0SsQ&O1T-rgxWu;4MOh;<%2%@W~)@SrS2b;}Gko(*3)(?ap8dW<VJhYBck`5>L
z)fkUxQL}GWwoWNG_AkC_xoP7bU0^&%kclzohbvRw>u_y5!P1<LbZMga<g!9diCagy
z{Oqr&@sD7@it6reFeM`2vB;~HG%;PfXgSd~cw(xx%u9-mn;v>QZv&jIF17okFO=?;
z#-JJde)7L%{)syJxoVkue-ZzRq;pV*oUL2<2%9AuA<-P=#;aNH(@-80&cQ8y-f6_5
z`l_Odk4V;*GBNi|!fKEZA?nQGiLMGMw@BnV+p|v5#E%2yu|?P>lZvS{JTPA+V1|5U
z+cqKtlGyl6Z#CIRtJBFa`bXcpzXZ#DwE!`O>6+Tvj^5Nw)AihT#hl#v5+9CRq8>lP
zw``=;f*^e}ccprN|8P=sGM6P!t)};f85!LqvFjk|`+5dk^Q*5XJuig3Zz;AL&f2=C
z2o!~=x5U85d=#xiTkG4K+am()4KlX}{xU*99d~Vai4Z?)XJIOqPphp_|NS}EdeLKx
z^=6M*n~xeF{Vq;&-bvz=U$W$I&QAjwFDKZH+IBsTa*K=nAi?pY1wdap=)9EpknD8A
z`>%HOpl|b%3==Y|^hIIh%8EIy_>haa-gJpsiC8UpE@iDh-58fq(`Yg@yg5$I(foo)
z)26KENx9V^*Td;@)kn#TbBh5j8|c?59s>UVY4Z*i`=o(wZ=YbQ(w*fi)%m{ontWPO
zA6mBYOMt6OOC5A-u>r$#Z^-wybLh~$vLr<2Y58e87lY1`Vad${^*q}-t1=6#<e}-Z
z8}&Vhxrv2x!g#lZI2a3@9Dvp>Zt&sNUHi{)BYKN(R;H8-vU9E_9qcUj4~GxNVUEA!
zzZoJR4Y2ypnB8B@pu1(c0zw6(fhAoKJ&h7=SlpzxKX^H%bgOk{byJCEpvIqV&oTI=
z&>q9^z|p5tE49h^_hoR0hwY$xvu@AHsfT^ih=a0QZ>}|hbmbNwbT<2CDD7(0D&a(N
zO5feaGQK>V0NI)ra*Mb%UO&=VjMt_;M5(-ug0^}<pWt;jcwO;aMSo49I<+H^`^am;
zQ;4j2aJcjfN6*)@xxo%d--5y|n#;n~KZA!p>Crr;{<nP>KnUSp#F}iQkX2e+*O*G2
z84js645`gEsY5LCiMWs7=GKmb{WSt<=iZu^4bp^=<|X<&*u0UI6%#_M%+x(V+7<jL
zzW0-+pr9RMcUidLG@95Zta_0quZOcXMnkqgf^>X+MrCilM{JDig+hs8wwL4ZdAV?*
z6LUHC;Twux&jDq)`_|fjhN9A6z%Hh-XTBxx#5ct9J;&t-TAWPpTcOI;23iP^I*f7s
z(U$gyRrr29v<b5iZ;#l;4rLt~olcUMX(st?dL}3RoY+($iAJ=GW&7K7;yyxp4WQ5w
zE8%PyBy5hjhF4M?YhL%8H9Z5~=Xwo=n!8cSJ7B>vSTNNx{@1_^Fy}`-6;d=OgMeiq
zW_;_0OT%!7Q$e=hjp6x-rUYc^J6f39+sF1U<yu|+YM0tGObH-$QmxQj1BAJ`t7>ub
zrX7>35%4b}NanH!u0t|^4Q6@Lz`A$l&L-5-tj$R-6Y_e1#RK*tV>%b=U6XnhCSzg#
zX|WjIc=t=JcT%sCJd%TXE(QYg(PQs0bu%AP%wJ3OG*ZX+d>ryGzZN!ZcAHxxmiC`M
z|6nFV%}aIi9J3TxFI38muW3qj{G73qNbt&0BPi2rC6|!P!0Z@Kt&wCjCf`~Dd@e~w
zNWXg(d#THB^i>!6cCv@l;u(HcSI)q@3JhC!{nV1tR!wT0M(m5Sl^Q-mPvfkp3F75-
z_Nv#C%#X96D}{Y!20mba2`#eFWO0rF4C(FtQqpY;8B_yu02wd%sgJ!H+rE$4=ILh{
zyy8FS;`pAq&e2jrAb$3^>ge~5e9MGWdnM6*rNz?O2>SY)>4|_$)DF3cUHW_MDXRm>
z)j^@#YFxedzWS}7m{I9UC4N-G9sfpV8o#ZK`Gaxbz~~!wJ*l}fNQj)J6MMN5xjocv
z%El`A%q;8Y%RAvLlopGV2{r|qwbE8m&s;8}!lV|P?S`-MB1%2HBrdZuQ^*bQtr!l3
zNhi|?roxEz?-hS5+ufRXR!Ar_(84UYIA+#3amu)|tZa!RY)PadM${0t!AsCOi7a}n
zP4PN193tiPCT0E5Q()H?v5Ks&{xEK}|J*q+Xp3jBPlJ)o-DvaP=Yn~in(Tc={kZyR
zufmSSCWTqGqCn`>o#eZ;lE}h60B%hDbT7oot|{QdFW);uRe$ObY<62&_YVrg^DIAm
zIQVYcr=j7-S>H>`<r$p)JleFFP_CeZ)J8#q)^#B_$D$|lk4Ju7j`|V#q~z{>?O}jJ
zzwZ={G%P31``u8JSPb`G86h7$tbmxjz2l}2!=YSKVoHp9(UYWFZ!s%9GWz&GvI<Qs
z#4>KGrnfBY?bhPAIP}14FElIO4c7ikXHm2^R3-LHS3$x(gVv1~_tavO%Dt88QF`;&
zVTZY*UtjCnuBNWY)hTswyPC4wBN%Rp;eS0eo-mh{SB)l#{q+<dU13^q5qsqTKjwLJ
zI+?GDC&sA$=tWzvn?9Uy_hzw8?s%Tk<#+7|entP#eLPZ1=Vj`Z2OWm}=zl;|exrCk
z9WHpkapv8;1a&klSnOY0U*UA>E6R#w-Aa1N(T+F-qW4CD8A9z`llT;1>w7fbSs{ZO
zGs1lWWQQoyu!mNtq_c#M&#9h2t@$i7UtFRE82fW7l6n7qpbiHghF>s^qBf+!9{Peh
zIpeQ|)=5|1;W!dbf^5n`Jf!R+hdx$A@MD%CmU%pH-yK2)HN!9ryVAYY?e{hc?c}T4
z@$=1GlKJ*k;rhxt#jp=zA3{@GAq<|_MmJSYuS8Xk2Z`}06t`a+I8TbAe7tifBT13u
zI**>5b>Jt)&EbiB32@=8ML^%6w*Nc?e9uq#M_nJ~f9y7hx(q`p{fPUFHXt7F&J8M3
z`)kn6-;=F?+bKWOz%*iM?QX#F_hDf;>EE20#6DQ-N7MciHv>Zz{iOb+ZSI$g$@x8s
zi<)W^o?{NSidznyh!-e1nPEtJas8*&1kX&v&^co|(7x?ZpsIu(*y1u-jaq;|%=lPr
zFZLl>@EYvz6$SbqIaePiD8%0}y}HuY-EX_S6?;$hfu-#`exC29r7I1m=d=jjmc41E
z<Svf5_XqXc${Z;V#GXJck_z2ji{YAcU}~DXZt+ab8iC~yS?RdJ@C+U|k3T;0hatz{
z%iH$Z47(A1X36Ltx72BogG-H9McLLgs$~PC(g(Mz)Dg&h^1^=0fWB8Kx#d*U*N|;>
z&<bwOV<TSHrnCBg@(5q%N0)Yj-KKt*li{KYXHuC9%g$xV$*6un%=EI_cdVn2NksK8
ziI^UJ(_6oC>6e8$pFp=TlNo>8m)*&WEsOirSY1u8-{ohxUb#T3MrqX(la#^I8zyDb
zA|m1i8cxxr(W<&)q!;_)uPFOU+sR;c1QEL)9>qmIy0$g~Sk-Z6X7IvxL|8LS;1~^2
z7uz2r-z4hz_@?Cc@EJ?%3vZg&`?IID{8n7_!zj$WR3DPSSo?5>25>AqszMz-#PJ89
zZ+XC%i{;t%QCr)}=BQbQ#T~BT+^OI7!@9(_^kT_mOo4v>PmXMoK=bokdKcdVbrpBv
zwUN2(=~oB~)X@tAlff=GC=#&8_Nxn0ZWSQZe$vUx{Zmz=<_gf;?M#vUez1&qU})E0
zrr}9L$KHTwmeq~O4(ZB1TML>g&Ha8*RJAJ-`{);HAAVeBoL*O#B!mfbnkWtM+WjY=
z;`eOfov-nKhPKlK+PS@rpUFk>S=0UmJy9pp!j9}D-{h0#b|t6Er%U+<&LgO|hg?E4
z>l9!2Dlhkv!*F?$^e3Pu6gog0Khi~34o$9e7}PoL*@-Yjm~Ma{*#BmJ*WV}Qo96Bk
zYNkHh+;DU`+9GasYcJH6SA&)UCY)`pPncpJQL{-dD~?42DfAC|GfHhmnG6zpKyZ=k
zx4jqVlx_DPVsnwuIO>Yvu7@2@5xEn%a6omIAv73`SNC7SV7A%vNq37c#>Gz-dy4qK
zvyv9^3DSnln%-tjRBK18mEIxmA@asc;IE-9rev2XXcpKd=O2D)#Xd0HdsJwYEc6I(
zbF3<3VE9x$@q#K<V#EIEt&<sk&sQi@F0ee_^AZopzV4i+V<6<1Yet@M^G~*T1}Pwd
zviUKWscJi<9BKbZ`B;^YWfFak#z3pDm%HjgV8#wUX_feg8B-^InDVPud95`bRg9Gt
zPZ_Guu#}tqyvRk9715WXCqC))#OD~`UUq{G+^wYc2aa%tW6(h0E(^nc^F4Ezy#$+@
z)XBDR-lnyttIWtD$jE#%sB+;3(976Q4`+6CtYMa-<o9^6<~~eHlGh!{JfIJ`Z&A^(
z7zl8zM8$)#tJ6P;%bvpv#n!D|<j*9lic}(t1h&Igq2D(&3ir|Hpw=4GUnA&G-M3>x
zK6<ng;by=uhoUUamHwbcxNF9~)s}TScrUS6i!#!SF-kU#UMQt|-+CK;&%x?4L<ATV
zXY(C@uuSw^5gUVb8#z3fcwvTFHm*ep1=AA#{;26XKfLF2UdPFW@K-{FUhJ1!b~n(4
z34SPDgdW(-`Jtj-<8yPZDjpxKwWDARws-Vd!<X4t{9R)W5D$detlGEhSs<L|KVPF<
z&vJs(emq$OzpZ@@#y?MFJKnkCKRDFTKIC&_ySItp-A(`6&iYqG(EXNa<exK`8iMyZ
zK27DinsV-<Tdt6jn<z?t>V5r+RFtbHWsDruG;<J$WgWt_(@8bubl%iq=9f~{{C4$@
zGu3}bg@-HH0@+kk%f%yUOn<cYTy+nMLsKnR1a)a&a^hbdgzNLL-F#`^HTfnMVs7?s
zn>2C@R)kw@R8fN7Nhf}?jqWsUkIM$BQJjE1<0KwQ!9Vy~w`08YZt6UX#;$eewQ(Rs
zg40Y2%$hHX>%xr8hH7N$b!<NgBdt#L3O9#2vCf#D?IK_1O%CrRGx@1MWiH!!eMpeO
zz_{#tWF&53B;<xHpDGuaybs}Mw_whGk1F7&9wJ3uI|v3iJm=KsbMBdlf2~O4rJe2R
zjRL0?vyhF~hV9F+u`PbYvL&F=VR}`o#~#P9Q_-hWkIQX~%Djx1J%{e_5nUL1G1m%^
zEpt(GwH3^8*IC%`$rFly8rXoaA_0u?<~la)xOvz%AFX|0%eIZ%m2;Q9FT)ijh5isW
zu(;6XmtCxuZT-{utNlu)%BJV=6kUCiSl@~nc%0Eu^_V4#%tNdUejs5yJ~f2ul`zdK
zv&yPzqyJ5TnOmivUz`KNpxUZ%_+i3|1ZFGAr_#1`N_xsVg{HXhu*td2zO}k$J=b+@
zx@@Kb=5G~C|F~Q+t_8m`k8uKeqoZG4w@#`-Cw9gv4DM04k7HP&!s)|NLYSnS{+iQu
zhbDx|Y{>)WW}+EzTPC&lKce8sn^|14=GV&&M&`3!dfpIvZ)BRwy?~^nI4C!WddiiZ
z4}!WXFN|Kn*L+;-F?#ll!9H2(t4AGFbvE5Z4=T~#r93V{2-=bmYarvs^&8&mjyD9E
zFuW;VR<mQ%U*i|ZM~9J~D-3qZ++|~2p}^7+R!57oD?gAY-)Z^MitDrZVMbQYPCGjv
z^aK5ds&f;i5c0)>o>nOch4!y>j{_WrA59csP7$>WT;Nf2W?0o|$P@m~IZNlv2iw?v
z!5pJEZD4Y|C|y06r!FwY?n@&B7%@URp=uBaZE?a5PLJm<($%9C>+VjA=lP*WYTumQ
z{CmRX;R=?(BD1W8iFCbzlKa^Co{GqD4SB_?LBXuERu~@<;PE)5^mM$GG@?p9Bv%Ph
z1Edz_S=OU7Dyh|o5eZYKWSk9nbY<^ib~9{n-044rzDYTWlBE^R%Wp~`5(f#iz$!t0
z5^LnAUGUt1M!Rmkhiv*;cEFD+78xGuUp57GKW&4T)%bfc^^xC#9)jF#XLt_7L-aj+
zUC434H$R{C-c-K)cB`17#ptRbt6(4WPS6YajfL54rjyhlq3#Tp5Bp#bKOonA!~e{m
zJU%R}yX!FI=WxNDYN@%)Is-M?nIzV<wOi=6hXJbIKqh7J{>@j)`xGuz=|RUEyJw;V
zHYX1Nre;?%+O>=9(Fano+^5<M=8(FR$wI96%xlh$B&=5^Wms~5xF>*|!@xt!W7cjY
zxY3@!eu+qJAt?pG^k;fGGD<g9X%%Y$o3%a<lVLC5KFPRb6v5gB7_wMl|NjhS1@Uji
zuE4cYL;XSOrt6vHVW3w+Sn!?hV>__4X`!Zl!!8QAE)FIc%B0z8L?Jz|W0|9}GN+us
z7r1Z?a-MOoggrrA@W|zQ-&^Kf((87)e0qedGKms&%WhYFjP_+|F(>iiw3e9qgw&SC
zj7w69<M2$>oy%Lc%{Vt<J74BuPtUE~Icfp5?vL*Ywj6_<UFi}GZ<)5&@kT^>yz8Sc
zM*E}t8b3vaZ-}PlTn{*(UUxe$F<vY;Tznr;_?ZV;qb^5<>vi9qdwnqYkyXDizc!}h
zEm3QH#_E$zf>kEbJ3KDi>~=Mei1PElQvAb@1OvmT{A$h)8_tlFoo79(5ZlNV@e3hT
z<k)dtnh*J#LnzOz!_&d;jMM2Y*|L5<dNMy-IFh+9D)Hxd+VS1q_Go}GTHPJHZFExW
z_8O^?QVYn_uf}h>ptE6(BYZA?A|Y#EcB9W)7NqPVefP~k<wSl*Io^0J%1l}P_Meck
zo4f?CldSI*1i6Fe*Fy~3$z3D0w$_NhM>k6>rRkG-O<@+nQmnjR=oPxOE~9j>sdd;}
z2=&majP6F{M4BZa0@>(mP0S?ZPo#QM$axRkbwSkyxc%RJ8G9hb+pu=xEqaaC3oc8(
zfeY>8k+3lf4w|88>WO_VZE@-Y8?CMnBQ}-}hC*?*Wrb)L%hr0+=dKCQ1n55IB!jE&
zJzM>c)4o$Oy<bC4f3Tz}_ejM4?a0unvZc+>%YB^tHq6;_A#Xq%ms-30ktaZVz5!7*
zY18u8!lxN?ay3KozV++8BFlJtYQ!^Cak>!0P7vSM=QLckKdlB<gfx5fqS4PX`42n5
zdWW?E9@>z$4yq1R?TWXGe$E%KJ>)`W7+P@lWS2(D*`g+_<gZ}-o9`!%a!f7EjSy<`
z*9c~_rk{S?5t@!3I_FmJd{Mr65MN=nmN+*TfX0D4N)hpA+$JJ{Dm$-Vi7v-r4|<Jb
zmy1%&-nCDaogbL4aa%J|J4|!hA{}T(SU??T7hPHhhD{viS3~C1Y&tfz;{8$}u?w!9
zkOJ}MZB`|RK(~5dV}>YfDR*B9S|OGkoRvvNP3My4eB8LioAR>tXz`uG^7r=E-%`u9
zxtGy(7AdC0Ju2*Qj=XUKm|EcV3)ebhR+I-jm(%4Lp3>Ofk`{-&cOv--I)%e`V>Qwu
zgX<;ELxY{}9JxZ<-|nh&2;QT!S^iq*wx&l<VOY0UtDK0~Oz+#O+~$%&LM{ZPL{`Jx
zkj_x{iyj|B?b=cpgI)Dg9A~^n<#$3|MP5s@RH!uNd~MQIZH{?*&hrVa4{F95x?FQy
z&FS{HwOBpN?Z=k-wG=ALte?lgOijdcZByBkee!$m)_;QLN8uh(Rd?Eaa|m2udOgp?
zkZo`tUdm`(tJe2XwVXwSKdBsoqt~P(R-dm__s2a(D{Yl?Q?D|kf*06t@m;pUti4?d
z3Yi;%7)_mSB)RhMM}4hG`G56Knq1`=@GG7SzhW6WCiVel%G1bT+ZZxT^;1V~2a10@
z#fUm7?43!J-I5Q@?-biiF7L31Sx5~R;rvFdu-V7c#s7c!sQ&RZY}!tAL40${Gl2$!
zG+$gefp@D7Px7@eu?%}KE+0p7YI1CY<J~;XQ6?c}^S=MzKH8e~TX1j>?-?(G0%;2l
z?-|kyu8Q6K>X&U_It6~}|Ln4VuLqqXg#y^m_f{LmeRNfwt!I1K)I1t%x{IH$pgN#2
zqYOllxNgJ9&JSmSfDhL_IV9y4|403x)Gvs3>%{@X<A4j<&mY5Z9e1r%@l<hE7bzE)
z{F@0xL0-!uDXC1j*tJh+@pcQzqiwP9oHt+K!2dhZj#r1=Su;;}5@WsOQ{`7>s5QkI
z;I+UDPYL3*LsX70enO;{Y<qmG8<P9`Fzwi##r`NhnlR&bloTOk{7~|N+ryJh$lH&K
za!$F?JtgabTs~XP@qlLskz!BA5KTx3*BkC0VLJ?(td7quo7kU9Y`9<Asg;wcFsS*@
zD4Mv>Rljj;zoA*Quhp78LPyd?L@}0frsV!S7jKNlJoG!Ku+qloJg@k?kC4TR<aoy^
zgCSIvkTU)&S?xV)2k9x>oaw9a%9k@<_QlQT3V5hYR}fH(6{E*A7FO2*8T)X?u;xr^
z&`V)=c$JUM!l>>h&%1i*r_fs(*br~&Ui!m17&a3n126_LVAK%_o#Zz&Mz69CFdWMD
zBTKvda4X~W5d)^tRNdIkSvJ-O=QFkh(mqY(lT~{v<Of_j{1<85<nlV9#e$Y%yhn0J
zQ7s%c3jRb3KU$qq=WA)YOU&cn>2rgm_uX8t&fFKlhV0#u>$@m6qbbl}w?2d#yr(PA
z&CuT;4nkIHOT4TuFyTYJ#1uNgr%Xs<E~`xm<!4x=>CqWs+qv9(q|#T-D_SU9{<asX
zUj`wu#?a^Ole7)Oe6z>2i;MX1FjO4bcU&rJy?1g5*l3Ku){<OtX`R~wdJW{c+t!;c
zDO&|gV6*);TUUn3d*!C^KzMJ_@b$QAJsa@?#l@9*PxH|HO}|wmoXYqM=&g@Q7~eb2
z2m9fgDCuzX0Yn&}4yET*I*upT;%hyifetzV7QYf+#qe?zgj2Qql<rve)*=>VDHj)W
z*NqX0>KKCL$_Rw`U>wkf$*x9|)PV<jN$olKq9%hnh<omq%~-;g!!V)DqcoV3E3ZkI
zEMLL(w8=pLQS|+*%|9atgM!ufWQV~=YWZ2KZCI>=-5W_bd3bjDY&3-)>lHtjlq*AD
z0PfxEtAQQ%wUk?IR(aU7P6FPpxl+8}dTIjw+D0Ri#$XFapQ&%^GV?EBTAi3Ek{o@u
z`QkzuTa}wy%c2Wro8D*W>5D%lLFkbz?>j=2{j$iHB2Wc`%ql-bpY(D1LORL=AcBC(
z`5h5FeI18?$KE~sj^tJ<EiTTs*pwS1YdF2S=bFKQV0lya?Rvq>uniI=;dFp3Z`-%k
z7I}kEzNmYSkLT!4v3X=7-Mdb(E;t}SWA)4*^LrR}2jA{l+jx$q)ps`Ax$Hd@xCM;+
zy}?wUtvJ%pGx;PSM_~E<E)E&DMCtkKL~qV3DZsjj^DlrIm|w<~Rw{7em7R<5Mdh9!
zhHa0YEo4u=V_|?&jG4U8*l?Q)3n8-{bFonz{@6#bJ2x}K2GU-po9MzM*KXceY?qvr
z`fg_*0yz0Ov&q8QRw<{aNxp1>ot5imJQI}4@C%<-_KfV!nP+EMcGGUG)0^y6muIuj
z{5$YoFhm*!fEKM{lO633DZJ$T=r7qT-d~*x_95G87qSsO!p)-OjlF=1qqjcB`WWD}
z@M0f$L2kleHsJz+Q@$0^U+^WJf@*Pi&o)rrHv8GbP|aLdpvUoHCepl1xS7-vvr%)i
zu1vARo6k2{24A}hokqN^0$Idiq3Wl9N6-~JcS=4PoE6-g9c+!zhPmXn>Bx`A-5-wy
z3MY;?v<wtiq+xUTXMidt$S!L~No*40<*)QwHs7yy1Nmn;$D3L`yybB9*XG54hmnC(
zr$;+P*rp$AZeL@YJ%1w!4o1D6Gw?GtwFgwnk$u;fSjWW9a3y}wj7z7t0X_2hwUkr$
zAV?f*)IBC#zGb7>2zY@^z}pBAiaZ#=D)R2CeQ!>ta-Lq{ARl$eEfRc55zYBxd#6{Y
zy<^RqoS(WUyK8d)3H?36MzDv>w5P|Epl{QO50s!H&=lLig@+mQhyQ*@j8sDM-VY6H
z2BwLgFaCZ)jM!1q4%Xk1AGFXnw4wc6$#7YmPqlSQelMVIuulQWK#3nx3I!2z3P+x9
zoWq<Knt#CWg?D0+ca?|3T6O-K{hTa`AZ*TxZI6j%E)#Vsl4Q8UIQAvUEgQa!k;S4Q
zHOK2cZ1C+A?oImzVnPXEpSd^W?#No32?>ZJ^St;=Os!U1cETQJ0C%eZ-0hhQH7LZ}
z&oWKa<J~f=?^qf3xiS+)*9-Z`-*}vDzETf@=S&O$7!6SW^Sj(FDTTmz3R<@0q;i`$
z91T9Bk6!Ev=c@hmFuB6Fqrx}5PoDL^ya$D@m0i`n7FNL*Bpauc6!R}OXULMi$K92(
znQ)Xo;r})XQmNA^CczzrwZ<5}1vTV*Lw@1~Ui=%Dl01EwZ6+meC^q)%qO@A$x}V%&
z-8EzA2@aooyXqUH7e0?pSfsXmlMvd<KJF=NKINDnujQ!8j}3Ku;I?c$lPh0t-Hd^9
zjX7uAeJ<NG0w9wGLbkU3YXIG!HXHi8Z_|^8w!Mv_y!Ob<1e3&T67ol#l%`jB=Hll)
zliC6vuHns^?}CIwlcv*PB`qq`3TdwV#oqlAsc0Oozr>DpcF*-rgoSw~$lXPRdF!ny
zcG{cvufuKb0Pf62VkKnGl{RT-t~-Rk7-!D;p9}iLQQVuG05T=9r}Za;eYeq9H(a+5
zX4#w~)KOIYf>v(EMtb{Pfo;6FhIDs*Q--YJt5cr;afd^n#e(AXap{6t3^h!M{}VhF
zb;klSyv+8i92_O_HF3$%A;3G5x$K`rT43DKcgD|^Tqx(FF!J_}Zs)gFVy}`2y~UzK
zv)OgmKN=pZ!-l5;J=%?s?u>H*`2g!ihOHnECI>4kalR;28aVRBH9*7bfa%))zxUD7
zWhQTe9(#<EeJ;wqODo}%qm;f)+)bDIhdR}xM&4|F3c|#~(rwtrEsge#0RF6>ZvAzr
zEF0LC45%zQ06G~stk(KMVR&6&iB+zwF|d92m6^84<+$nVyEd6+;)a|P9Fs?}s}`ED
z4ZqXfx@w;G;h+@4o9xs-_6}0XcL%=z7bMT0?fh>=%$dXWV%5B(PUsjir;}u`lVSmL
zW!RgfEI2cB{ijRp4OQ(XZ}m1x4tbx;?!L<$F2wrkp#GjXb~}qq<@MfStWmNio^&#m
z`04xpWYZ)8i1}^qOEEA8#-k>zL0Y6YFHa=|5sg?ORI0ePNj`ASak!r^W(90O_h(Z|
zgo&m{unDLd54ap9m6Uv<_ZJB`Qxm!G5PJd9Jei86_*1GIFia!NhliN|4qO{xfL&sa
zkd_M)cTFsTQFF)H@W*fU99gV<M+BEXy_5+@8femWx&`S2?8wy<YrRr#CGl))bU5ok
zGXJ$DuOo=bZ?czt7$2=DliN=5;f$7hXIBYa*+Nnjxz_x=rp=FVvPI-9@c1m0pfoOi
z$H$p^D4t<q(=N$tZ;8j1l=!oV$5%)q<Nr#kPw$kBrj?g`bP_KRvEJ@~QD$KyfUPJ!
zL^9=A8o^@35-jPuF$>mo%A}aS2X8Z;-XO2^!ezR@Xl({$e8gQfhCr(P>^Bx~@mvwN
z+%FOyg(AlC%~&g135Mi)O9xhedgE)(G#_mGUTzPrQxb)5mqq54<O^E*A*!=%#<mia
zwr0&{YK|7wxlO@J4d9#0^*!T;ZGa5>&Cy#jrNmF|83sPXp=bORDD7iYbFq8bQAJ#y
zFV+B^fVjkY2+j8BqDq?kGg#S*T2vNX@U}_WQn7SFIUp@D(W;C)Ao2;k9i@P`Pd4%E
z96`axn{CX)#)6k0xbhC*7ciTX8!-)hiS5pkUy|JZ{3K5cD%TrV88)Fgyk*wuQ7RHg
z!V~1DPh@YJ0pUOWxA;^OlW@UO4wiJe5fL)iJuufDH$^p^+dP5(;9eZQ5CD;knWEZ+
zO#IJ}+zctJE#j%>yh7Vt2@Y1CeujP>_FKB9p{|y-o}uM7gK)*giJ;~tfra+X;3wF9
zc?p;uT7~_Xo6|&l{4}lZlY4UTNZ~D?X_Z<vOJ3pZWKfUUz>$b+#|@DF3-FH?T<R7|
zL$b}_up@`QoB9QJus)>6j!=Tu3bOERi%tSWvCgm)^l^{^LUPVPF3>&XnFH-#)cVK~
zGY-wZqN3N+EIyW-H|Ga4#mF~*n#JKC9mfY`L>U6r{s?vAq~XSg&L7tS#b^Eel>P>)
znTzn*!n`%oa4O8(FD!h86J8<Sz1yqK3JMj4Dlbdrf~$Np-8(-cJa~ZeO*HLULy;we
zi{(6Wj;+7Y3ER#0!Cs+!dfTy(O{(t|&XnSJxV<5S6bxLqDN;o_>tOV9T(u_t_z_w_
zfZFzr5H;3s2g^_o(m5E;>)5_WMc8uVC{DLZm24M+T_udJAe1gnlFt>#L8tJZd~UcP
z0=Ckfyp@wx*6H1*FiX{PAv<o%U_9r{EbEMt%EL;hCN>7f{KHFpi$a*oqISGL^tZvi
z@4;llW@%BW;3u*ZWm_K52fe-)<cl95o`~6Q<kJ6{w$+$(6~6M(BMv9ZT4BHo^BUf1
zUW_Wa@diRZw2|+x6bHhd-zfrh_&XB|OkdHOLt?Me!RGV%`aru?*t!mB&MA<e-FAbM
z&WCT8baowsIj_b`)4?!xyl7(gz)+qWjxb(&qYlypg1+C7HmRM4C;JX$6L){Av#OhF
zo9cs_yB7|x)b3JCo(ORZmIUAc_C<clOo`jt)M^U+7%|5!x5IzID=&YzCMIjE)jk*#
zD_Km<rD{4Rfu#GoJ&M6ZND3T|dV$!+2AL%AeqHa*v(XN&3CNErHR$M*C3W2U&Rm}p
zmw^&Oecz+c(YQ*muQ@p+{3@9IZ3ms)n8C9gRk3bl8gY8gC=~l<vfFlNhF*Wo;WYEa
zp5z$GGU*2cS+=xXlFl1SjCOMe&v)TMz2VEjK(Rga<$X8oVPu{f@1Ep1sW><U1X@SO
zy$^w`&8{Vd!K?pNSC7KfG*QgSdoA}Tnd)gH9GkvE%x8n!tL)`hlyE&B9Uhso@-Kw=
z$rjJ<=tdCVoF|lHCx?iLv#JWOI{E!92kzvSB@T9z^}T^mEd6;4BctHMloIWjH8yTw
zY1*M;+^tuT8MiSFILu45tUZ_j?qahSu7no@i7ScSP>wHxfkNGP51$wmcKDTL)m%Ta
zn?fE0fO5dV#)bgP!2P?D3{P0$^z}GD74pq=TdaxeVzA^y^o<y_eII9G3=4IQB;egK
z2$0OAeqo0hzeRe=hUq4!Fkv=<q5YR1hwH`VvpRXt78!hZ&XGttR~xJBrc2X;#E4Tf
z!V|cWol5OalDetd5f>G*I&QA66?V}Ro^+ziPt}pC>Mww1{B3RJKrOYT)L+)m?c8=%
z(8kQO@?>&vO46{YIO)5vVB>BDg0Ph_q^o4JNF4T150!KVKL)xM&5dv*hj)<+b{KMg
zXz)3CTA4APKOShdBPhHl&o;)@!~3f81m?ETfL3sjn$O{n_67nY90TuAxA*B<J6rrw
zr(FN^?gqz0pr55HmG56#1f`gsiwO@F$0BgY-N6n>{R&~+0atFDp9_vDR1zMrO@J#>
zZ(xygHTk2uNXjJ<Z11=7_b8!suuZXBbGNKz-v>Ik$IDx_v#@|V0F|xRZO{D|0}o)1
zEdzM?pLEK?-hh(&q8Z89#-ve%UpF@2#0Im{Vn|b}420(WP?;4VG#*9X0TD-ZXVz*9
z*VA#uJs=~fw+30{f|vU!!<(*;dlIX;71Q*G*%=W}jv=aHu(PgBQ1pQLt6<8<)5TK7
zEI4jrl*v6eu<upv02hI;1zdy|!~*X>T>S}SdKLZq4WAm9rfi_LV-t{($PZ4J-nr|D
zEe49go2O4~zJtDNK<fyopQXtzf@)S)Tgqy*lv71K8@;D7IS&p$^;t#(xP`Y`)(jS!
z`=}%9slszNO<!UKBg`a%<p)ra1Jt6!pzzjrem3nWiJ27+;{R@Ku|3bDg#uCurwmaN
z7Tn50`oH?ofb_JnnLt&4l1&#ghW^3Jh=2sCTQhZkt`u^adwR{y4(14Lb50V@pSy9C
zWU5J?{e<or>@k`4ayzLXEDU>SfT}U6#tm`Pu}litSC%|x24@-Z4LZX1w~1cT-TO%D
zE4BkG+gXP!V0|!zHftWd0Z8ej@nn>oRGUR`8AM{>?gIKxd`-W$#{yU8&I&GxZB+H{
z<aYg;YsPTzYM0g7zCGT+PJ6-6p#@zMh(#$4M)fxWaXI_u(j(1<t2)C=>JBNr>~*X<
z>I>~~b#Ae6zo)ndF3L}oX2&~N7O<~NtUI`TM3z92Ugp^B)?yqXpHewq>I4G0e-eq0
z6a!aMt>J3em5Y2%<l#O+I2cQ9N<AY_U3i-WV1O`GSNN4?a}aH&C})NCu5{D;V*klJ
zi%%RZE8hK{;W6BoWO92Tw)Og>R>Fs-q<lz;WZhAAT_<md>G0w!O8CYUsmU6gSKm~W
z(<iN7(%6j$sfLFZqIv-*b&AeOjrx~o_{M}Q3>KVefis*Z_e-aQzGVtpoDI&`AC5i*
zZ5b#rF?eIh?&Fct5J&xyxdl4XXq+%t!*<+u2~BI+gM=s+7#;$O#-H7O3*T3Xg*585
zXAW)uQnjw1>EcjNGtzyWP;{)k*ZXY;t5BP^nD-k8?dq^JkStuVJfR^WB}&e0Z$CD%
z%Kv@9!2g~3#n^N)C(pi!<ShZcI8WD`o^b8WB*pDEG~zR4Ol+Wx$4}ccd0_4~$13r5
zkz>Y@R<fYmy*Dic6TI!ebuGO6O7>~Xif5zTTtM@lK17&0=-h+p2TeSV#rL+r;@iC+
zQa;=`5B{U{LeFs8XL7apV2k%dBn5NiD*%xyhaD-_7wz39W%1iQfk2jy%7xXTRQ;1q
zHZzT^KRR;o?=5=YlZl3T2k6?8R?iRvIefhi=o2(hz0~U$Hd^%4D=BIFe5eq=-^ay>
z<`lRl;C*nF;#S%Cf~bUU48I>Q!GSI?0|?U7Gn@Z3qVZnQ{5~EA5Fct-f6(=M^}+8Y
zc!U;QMk5>qQ8u^{;&z5%XW!~cBNd1L^a8*Zf`||Y;NWcoB|Q4IJIOiVpT?<9<3>tF
zT)lN_z%YbN@*1CgeoKt>9dABjO3@xZ8GE&NwI#u>=K>(N?ZCcqn4z+ArHBA>`!`uy
z_!ADzDvgLog8BO6e62h#xy3C`sAO1~))qu)u~-sR05nrh48={XQDN->x<*kjb6N)+
zT&<$;=pgf<S;}MyI_`h{_O>#qG~Cqfmq@OEI%!RD(7_+fdR~Fh+ktrIg-5^?-7&c4
z`KAT(`)%sQ(=T_)*PT(?alV?7n>@W!JcGpQ+Bs6?7uEsa<Pn&AXc?y{zq#OffV3@9
z9}udQ)!mnK+S3PB+_9BniZacc8@hqpZR1dBkShT@o@Y2RzvxHuYpO~SC+XjQ4GJFi
zn)om#>*tv$r{MtyOZ3h~V|NP#tdt|x-d_a6A2)fbqu)oa>(k2d!ipi|m;EUpes-<)
z#EnUE$}=1Pte3z*nkDS3p4`)gbw6t17`Vb;#{pbEr=r!Eq1Ky3kY{AH)iL0bS?n8l
zZF2=9<10GdD<2A|uXu{YO$Y#+V`Cp<k#CJ&7$e`>KcUv`HCMjc@*Q+l`m-*lvZ`XS
zS*Q76Wx}~Aw5!9Ss<-3-0wAd*y&(aH5Sw+M7~6NZWj%dRu8hPt$=Tm{9P}1`yXRHH
zpynmpwN5I>wO#D2zOt3q<=HJ|&G%aM33GmpVs9(Gif8qcDGYrYTu{80AXm&YfhIy{
zY)~MGpGzpxVlcU!N4w5mB0h^(aV-&u%k*nJ3Q%SH(AA?#=jZIutzZbPUq&^kqgDtH
z#C3<y)qR3zCp%!GD09cRf(+ow^`v>Ueqn=;xZxUuyc1NLq+D3lC(t@al<F{xWL}4A
z&T_o3>PT&cYW>{MPp~trF+P&+n8n9s{tx`&1I1&(BgE>I1+W)!B-iyHbt}~Y84Dvb
zb|gT`t^Zh~7W|)Fp=@DfVPgIN;tJIBZNTII(<C!k=kZ7YjC%MDDLVc2JeQjH&6L~w
zdsbEa$R2xt?s8I`b&tW<fu5%8YEt_jiAd^wxp(i)0}6VjSFv-j-KhB?1ubpumEq}j
zAJ&<)y)lRTqrUNevg!Q2OKSX{i`LrESdhX<N6+v58@c_)vD@~tP3))E$*Zik{FJD>
zZco$5Z?-Qdz|K>toGqYa1Wl6BUv#ukO@OH0(M!xC<nICyB+zyJZ<Q#AgPk))_zP=H
zjuuvM$du1I`88>n=T(&IMK|3{p)5t63nLi|yhNk{DMi0}h($B9?}X1iJ??^O+Aex*
zbOpb;Uj_P!X~gg4{7!~-`s^-AHQ6GI$`_jT1$3LJei8LZFOP|oq?cT0MQCut>U11C
zv%>&dd>8!@g7_FOuhpfjbqT*p?4B46Ta~y^G`%YM<f{b36aRTlo%P<Zyin@*wkd;w
zlcZkAELx@gvOMttpcSS(`I6-wpe$3#)5z48Jhtp&p@rgmYpYRnHD_q9%Ei;3%I~30
z0p7{D7$~;&zk~!82Y2f8t?d5fWbP48m~zQ9%99?5_)KV4I&zX9&f3X0i*9e6lw2W}
zrq~19lwP0nzIoYx3vhgGTFs!}#OQjk5%9MNHAxU3{3Zk8-8aJ-7>B#GiYXJD&x`6q
zEFUbUD8EsOSQ!6t&K(GE{b+hKyRk*$U|eUm!nI(r;YI6eXT8K!LjDyqTp2JxY~AG8
zje=!x)_xPjvNMs|E?L|)e)c9*F<Uq#<+Y}fTOrfUy#2_*7jfY29Zu;ooUxy0LAnk+
zh0uOUv+bs8s_Eh=yLR3&j4x2;fhv&Sl-{9xHx^_RL{j=kek5>ouilGPXqGTxhUpPY
zO>sbd+~$A;z`44t^|h|-2^J>Qmt1+cxeT`TbA|Bfg{9&zHh0GE@sGGU%!{w*ZPj_L
zC?%3?nQPTFi8PHZ>|j5D{(l2(2aa}GIxcHkIKSk4u{EL4Ymq=L95<mJ0_DRg{h&v|
zFra+Nd30U*UoC>b$a(=5E#4&QzfRwdK%BOEu6^PC*u2ihTt|s0(dP4-mTQ?nHlxC7
z`L;FBVp(~aD8g2rAcM4e5FA|AI5{Tgw?-B1;)JCUZ?$Rnj`}0{*rSwZK^AT!>wcdx
zHodf)x}#3PsO{C*8*fEYx$xs@fz6*(>EH;0c;_0axm0a#K#Goj4-KR2tYhg&c|J`M
zYyW@s(b&f-eN2g`C>F0y^LCXNE-oxpD}HEB2j9y8#vW%6K>K?x0FVFYNAiQfliY5Y
z=VLgH8oR4lTvR<>ib+4NPgW1^oe!Y5?Ml7r=aV-c1kJoaD4uWyk~eKx5vI4V3`8$w
z5zQ6Ku!fl%pD@BB^yaJ3f(OkMWhHVcb&n2E2JHD{Kfn;X5ckczg?hexwBhjz<M|rM
zRQG!)^9Ue-%xMUyoF=keAfUJEDnaGvHBZ0!7W*M|#bTIK4l(B-fU@)K2^Z!R<Lv8y
z{7m!Y81#}GWH|C$$9Af@Ze}Ev!?4?~qvP=R(0%dY3&mC`p(7)UGnDb|1J%A^I8t?f
zfx~9r_E1GF4Wu6z_(vQnRk+K`ap>7<>MbAi82AS5fB%Lv-OdZOZZZ7sPwmw02w&88
zPCqOFmAxeu2!ar`0@}v;XSN7S8oF~)cuMsdWV;`-xdAbx*)gf2AqcC@;+KE^W6ny7
zE^2%BBIDn})RGbX<Lpx)<9niYi(%6y9{tMxB+0}xc|E)?Ij+MAg@nOUC^4_eHKJ;B
zwk2{70F(6O)5qrChAid)nF3U#Zo1bEQ{|&;^|Dj+2=R{&0$3cv7d|c3P}Cs*d!!Wz
zPkY<)JfPOw%00(!(z6m)3-+%m)4kOKs>#uTtUNhO&!72i#BN?<!N7AOv{`!Yv)<#m
zl{{2}6F;uZ14Wd;xOs9ZLDA(2t7~Iq`!TDeY2Odl2cpLq;F+5ECqnFgBS^h>Duzlt
zXtUWw&C(_R$?sNZ_D!w9Ei<bqx7Jkx>D4}=)iJ3K5}=L90^Ss|02{t@&2!P=2?*eR
zD~_{e^0WP+ZMjHn?1ih9<Pz3I^kGID?YrSIZFz>Wm62J7CLFaw1v62iQv(?k&l{qT
zamVoq&6`O-!<&}3`9LR(Y0<P@--}!ro-_z{GTu`+O@;I*<zmq>cRQN{a`Zo}jGq4i
zs$(QqWo~^3I4f|P$Q(lcGrH5ffwx2gniq(_|5d*E;aXORkc(eL2J`b4rx&wtA*h$q
zbOz*O(YX;Pp*H)QC-sGjR$B_~Zd_JX4(eoo;04jc2dzwLa|}vowH{&!A#HR{>Fd>8
z<d9d$5l4z|nFw<j9gAzLRhtDua(+Kt3{K~+YwO8*H^}*cyWMwg`<d$NunfZ=Z<VgO
zK>T6KD9dM^aW33u@W;uR*-3=~%;Tzy9sepM(BJ!rDRtWM=#Tt$e2v}RViZL*Aw{<n
zZM67N615oObuU!N^oMF(5;T4MN|f72LDX7gr`6hO@$UKy%X-Qf&|!ps^Ge<5VuKoL
z@r4t(hvzK?49CS?{Zt)3;a<3ud6!nAQOrQ!8T)!0OWmdja~pfnejY8_{#|fwwW!sw
zEZy}?LrG_DB_VOG1Qfjk1duk1xN`h1HIh7iowI8;$6N)CIA++6nL9g1tO);sPQkWE
z1{FopuBS!cfL_=izvGtE$HfPHqkS@cuT_A2GK??k<(rM5<a8h)oq6zcAJ^8!=k;eO
zc3w9bvX2UZKr?Ak;u6B%ejc}?iuutsY9)GQ3TiDYY9f3%SL3~&4RFc(XxAVz@&Eux
zk`wCqA2c78Mx-kMK_v<6sUlW}owa%RbyIN<5AB#$uxnZAi`Id-!jf?lJIfFbZ0q*8
z6{d_rufI~mwmJr9iQZY)u%u+{wNe(km4zmQzJk>xH_O@q8@zAbdgx7se=*Jjhr51Q
zC>L=I+1?T_&!~NSZ~N%PErnm@I(PL$4M31i4iPG4vMJ*acUm|hDFea<g=O)RowhaK
z#*=-0y^%l*^INH4>=U8*x!l7GqN#3``5O~&t;O3Pw1PVB<OvX+<sBs4`w}bn&Z`q#
zIzf(5;zjVnwwY2A=q<4WvXdd@D6k&}a&L101)3_c%@~kGVbQ#dBvFdG#ogXvq|l*4
z?_hSRw~kYO4~-B<^zwla1~Am;@Ae2IZ8$<^q9!PlXMvD>4|rkQ>Ur<(dd#c(FOGA5
zdw&!Z8+}igX2^xa_c^NOt2}9Dn~ssNq*ZC_$9+P^;Uvt}+CdWw!&%`r2gj43K15g!
zh={p!CeZYO9-#gjA^1^-Zgc;MvW*<+ntVt*IYSYDFYOXk=G?si)&3&+he1ADJv~01
z5;bn7Xs>Vq;J)fm+V!#?-{cHKs9M&ouk`md6QHTa{~o}f|B}Zws?^lhW~KXIPRSdc
zJ6oG!X_fcUNG(-7;Ar_$zoIJjvhdN)=LhvUzOE7yU=N?>sPQua2e^+$EnX@hE&(U>
zJEEQ{zQawv$sYL?Fod_Wc?F>LmF<OPZ$N)mnDvzPw3%g$;=&0srNcg~^j1=D7-8=G
zqIC0cPP<d{*8vX+&%FE}Nj`c|Q@=PxtcksWvSuCJ6ZG~xjxVvJRJB%vOPvRoS|KI2
z@r=ZUw{BKgZu%yS+|4!1=DkA0>~8;(=L+nP!)2mM9fzfsR{}bGq1r~cbDQ|cJrtCZ
z=dT0@)O_?ga?NKK1K=XSWWRF6!{6~}nm*h3T4?~B+r|9#{TjPu5X|KT<?&}axIrMu
zyhcKl9Uw_n7kOC#rrdKv;eOIc)cA2Yfa?B*wQTUZ<ZEnM#B9R@M^_XVS3eU25^7ww
z3PLF{&-@aCh>M-*9<J>mddI*o8OzS&YV<+2KvspWBhq#04qy2gZPnFN9IdTjnq{FJ
zH><s1t=aLiTXF;4Imq5V^hQlj#eMeFMyHR1WZYBe71`m`mdh3Dk687anrxcijt*oc
zZ97Ww&(rmMTr^NKo*lgLX683?#b7rd$5WlpXV7~dzs;AE7xG;3D|}}@t00szqrh5!
zo@e3%kYqXoP9fxhvnGlsXL15;e|jR-d|t)0mD?4+VZ_Z4y1DPJVwRDUoX(_tWpx(h
zhm)h-<@uIwwQRm`l8dK^a&Bk~ddj<Wm+{Dz<gx)7eSR=2Ws(PA^-cIY!9Wog+1d{h
zDr&)f;+~P?$A(xK&Q$H*#@Q=Qk^JrbO5#@^72_c0o5!xubk562i4u**>wKq7MLNh=
zqhe%`0rS2lsLG^;sBv7b7<jm6ZmWvA={wZJ)GF5<)t<?74DJcEFK37d6;6G7bze?{
zZ9b)j{B{&R#FJZ7HV<U^Wvj+9(u?f+1$n>Tzs`oYh*w>a7JgthK+xas4RcvrM9~!n
z%NoaiEvonx?jx4EWv$lT`NPt7^!{vOW2>g#H%=?Q)#5)+CaFiUe#@*+YmtIIJWYGL
zTM3$)PJH=e7G8I_4$RE`7h89Fr=bo60%>J+nbO*x`W|3@xI1wN`HuC%&DVCf#>?o9
zV?pfO)75xT-*j-bW!Y}{syRaG82zS!2oP};r*R1E(cj7k_63$b;RtrIIJ}(p+5~HR
zC!-+jk+f;b^xWknByP6#M*Wj#xB(ym+|1GiuP7VP6v%O<JxbyJhiL(>YPiVpTXY%Z
zKd}b)%co45oh@5I3f46{{mLi*WD7JTlJZArNzxQFBj#pOz%WntWpb_yNmR3QC7klq
z==$rZc*o6W)Y%~YjjP71?}duHVa`uMX)l=HP6j2-tV>?|phr}HqXmN1&A)ad4f39e
zi&~ck@qhQa6KZ^0j45V)PpgX?XtP6a(INX?H$680WS0J+1V=}0S#I~Hf&Cy@qUB48
zUnS+R5wJI6Q%c_w-@GMxQ2hEE-#?IeeTWR;rr%^pxqJJZ%>5`5#jO;%F3R!i;<C~2
zePnf|<RmTQgk&Rf>o+w{13r3yvQJ8X1C&v>nA`R224&QAGs#77PVwK$sDl99`J;R=
z+JS@&d$oqW8bdw}vGVT7?|)C9h@iKI>c$)+0IBF?BScZ7*OCP!>j6(Y0puPL&RiC(
zUc}9?-Kr0t(-oi?+!V`*kr&AqcNJyT5_`KrDg{N2z~ORJ_KAmXsRPXfH`4WrB`*<v
zAuBYN_;c&e_CWOBptl1Tev_a`G9j<WcCfnW_!oB6W12k*rz#*XDQU*#DF><h<FaIA
zPC9pceakc~IL|#KJdmzu@JlgvU=crY7sPQF%jUTljDg1PRi;<dD0swYwt3Rr+sqFt
z4U$CTrkvy!ux(*lYLOUF6Pnht==iIdEXMy=Gue{P^%CC~C?L4|XZ`Euk#mw)W?*{_
z2h#!<geSaL%oFoUCirZ0(zZU?n}<Ayz80dlrv=XXYLtd^dUJ_3Fs%o`K@b3@wdTK+
z{QSIn$`E3A`f8{syS&=Qa4QYP@c}F1QFgqZGy0vRj{P%S7W{nDG78XZiY|I5(bF^m
z03mISc&DHw*J2Fz{cW0d?1t~#sow*MMf+o!W%3Wl$fZ$kb(zVzz!i11J&#*%Kjk8^
zMz!yQk5D24gk0%yyPF;|6<>ShPfQ9(j6MO0(Qo!He{Z970>@B^7sqy7^gWN^Ptd6R
z3j0=P%iy>0ojdE#ygF<+s85rfYQF$PJO+d)8PIqA09{}XngTDXj~rvjc+|gf9$l(-
zwogmT7dv*#Qs44{C_GN2p#OnuUWrltL(nGqFls7&=2K*$ONARHWLMlX^X!Ell@gKG
z(FR-I;H<gQ-co7yjUde-o416=lg)-+HY}VZT+!R!FK>}7$thg;n(F^j)F~-+n8>b=
zRk>sAKRGWX;nVg7+xi~r*sv%9mVH10EW2|y>T^@T{T=jW7j~N8R-XCCx~Vsqrm{zt
zZ>9H8r*dI!SE}3Ywq*_OU3q$gSlV=PmNinI7zpi;eWW+q1sbY;ikCI&9(i>coS2Hv
z@=V_saUhnw6aiyo6<?~0#l~nOIiANdms~rJGHH8g9@Jy2O|_l6`0Pf3j^e$9B_NMM
zk=Tp4sDZ#m%QR<vd^kR)3igqVSDk1B>wch!`>mozbGPk-yix~0T3)SNiE)-(X;loU
zCpd=_Ne5lRh?Q(5%2ijQUNjuyreu&dqNF;!g-4QIxbg=@CX_Z$(k7M4`7hsUsD=?R
z%L+w10o?Rkow5GjNo{jnhE<tZqxU7nEYabVLElS#i%&9K?-57H*}W%I+D9qRriyT#
z(QE7f4{6^S*5tOWt1MKIWko<l!6i*aK)N6h8z3NHK{`Z1dXwH0QBjem^d6<Elu$z_
zHb83VAw&pD2|Y?e3naNSuCve1!hP>O&vSkeBPo1k&N1Hc7UQZZvcjUUC(aZ`YqTel
zYbY&UwoDLQZ6p;Fx$(?mMPPlI=yf-92a4-%nFPyXva#QU;Ij5L&%oC8;|rSd2<h7`
z{^dVzOvU!WGokQ@kCxn<;?<Xj)fKLO!p!q#kZw^wKY@7`Bj&T|H#pIc7PN-mu)1g&
zv!y<$kbX>6LWWt&7Jd8q$ZwZh|Bd8Tp133SZ-m&#u#eAvxHqF5CC26d$pO1fk9R)^
zmEa5<J%6P!9dHaTwVyq5&fDrjdiOe0Ebx0ZP{gq*Zf9Gm4I86j1-~R^Q63auZP5ti
zG^6z+Rr%z96kHGs$(rG**f<^H>4LBb$LPK$Cn%`6QyZzH4JR5^n*@I235)FKx-c1P
zVT75a5;gwXNGCYi$AgOj(vL~Z>!N}vfy0Yv%1n*TuqjR5928YvBPGxQ3}RsB#!)U8
zE$@>o($BR4cI%IW86Og*FX@C5o_Qn`4FM}ru24Useb1jLb1Hrb@U?+R=GXkQO1@$8
zGVlP_S6MZGsx{0i4B5s8M?A2KP%gj0__N_D;Vn7)SVJl!=7?H#Z$ZlaO)tK+fz7T;
znAL)@*Y`DI@p8d>hBNK5EANot{TD2P!L<nX?@5;s=VWFP4~*Xdri2#-<%~Xms|#+<
zhY7?i1+AR3AMZlNQHjOx>&P2YYP>^@x{>}YR|lT^hjF+Iq|M+<25Lb2<N~l@!`*}<
z;%abqZY#9gFcqw6n&~k8cAg3ZkzI87yleY9+|F7bn61N6L7RujZJn;D5$F82V>Si8
zOqL1eT1RW7%K~PFIbMh!0(yPnH)RGaADDFD$GwY21XT~KJ;OT0<VMvba4H(qbHe%4
zVk9EOs(jwe-O*2Z86t(TU1@dlasei}lQkXcrmoSQGAlKzNjUL>&Qf5kn?#h|xL2vn
zlpiix*dYp0N6po|AR)mx`{5KD3uDPV_02NHLQY(d08rzJb<kO{4$b2RwVuLd5Mce=
z8)OR<@%?a-k`}QIFKX!uX^c=6;0=p0ULkrDG8@W(!hIHUaL6W(rc5&t&|6hqSql0S
zDCak5^9i<aX5I^68qQ;8T82pl05}X3eev|)4ED82c1rmitQ<2q8Dd2jhC&C4%t(X6
z?4FgY_93PSEs$r>T8@QRB*v`3`&SwZF`x#A<%AcN2}{xo0U2!z+bxQ-A5lo$H4Fc#
zXhtjCgOtIQXYOBaVuP6;=)Gd3JMiU*j7pf8#$ml;6#<33A}xQ7-5C)oD$q*5hAwOf
z6afY~fqC->Shn&o_Ii$+3tAwTCZi<y^XFV@{NTN=6~axUeE)u=_tIlNSZ~j}G*elv
zK8m)ZYcm)$00{!Kb^)H56@V@P@x0HYy~5ggak2TBUe)WlbhR_!`wAj<wQjr5>2iIu
zge3z+!%<Kl6sb@VSuq$;h6c(P6<k1ARZf?>n#T!N!Bp}(yD5WA!XhY^e!V+@=&zOu
zv0Oa@2i~~z5A1ZjAtGH+Q>afg{Lbm9Ur^x&%C$^b;Y<F(+JREg(|ql_CrT#ExIRgY
ziT))M5z1~>CNTo%gnBK}_A4C!ZWOnLChhNL@4R(L?~?OqW=oj@(N?hqn|F(#I%dpM
zdb*kGsN9w4JfWJt{3aW~Z2FbQMP7dAN#ge-;ra*xz?^4`S0A&h|J-Mw&=v8QPj6Y`
zvGuoTt3hG?se;3D?T$(Ksj_TZAhg&%D4*)oS15_7iVpbPH3^b~>b@!_9a~scvfv&@
z7MLyfK(Y44&>~tb=BpjzsjJku3f5U<uUe9QF}aRhKq;o*3te!^n%~EAF2&eYL+^W0
zwY9p=ivf4N1fqY!B+x7{sR_4J1HjEZ_~GP6^fj_Fky7Vy8sVjnQDNKwRCba?mz<DX
zzP(3RONV1LoYpXYHHg>-2pcUi!>rAe$QTmmpxx50LUolIPG1$c&;@BWB4x(_F;pR9
z|G?vY0H^Y16o}+B$^5%u{;2|*_Zc?&mk+-+sI!zS=yv;&^u;3KdE>^sw~Ul<Tp=$^
z?CfBc_Rm~{uK73{o`!~B&S<M-S5f*`z)XYKYw7*_dSE$^+c`G(ak)HW(Ukn-p*YZ&
z__7J!W?2Rn4K6^>w?T_$&4Uf|n1!`O6eaaYh88HSuf}QFVj>;0mpA5_RW+n{=>`sH
z?MQcmP=4Gj@}(aV9P?`4r1g-Sr?JRC31EN>%rN}@z=7td%WqIa<9{pyI2)prXR1{`
zpP9M3>Bfg$|LOw+ztO`3BnexnYi?S1+;5^Lt1w35aeeC(sLd7JO-93-ZAj!b`;f_T
zm-PiemHzJpltLtK=Id1Oyd*jVMNl)qmgmxgmg5SM61@Np*&@LcU`stYqgT;T)iZ&e
z;!GHKxt-_T)lSrdxdKP#j#eY9T(3sWL4);58V_vQ1t>pfR(s<&qKo=-y}Rt2T9FIy
z)NijUGPpA^4PAqAMa9*<%81*?(WnCNsqrBvf)24hcyt2myRzmZZNnt&e@bXzMdDsO
z5PDyg+MT5AUI1(wN!ft?kw_&qA+arQf(MP^k&PVz=JM-SsT7|o+Tt>4SwRkg|0+}k
zer=_WI1`BKha+LVb92vCfkDPUaTyjmjK4XsBWqa5C4Oc($t;jd@p0g0wxt$#<(QsR
zUzUK-qa+p3RLe%{oO_5VxgR_E7@^_1U}#xL^;mb98~4)d&Z)H1qoh_4%sd*KLpm`V
z@Gu|q5#P!Wjl=Xx5{_g-bpDDZ&iKz78%_O`+SHs$+kv`q4=4TaX5zgr=cdFhIY82V
z5o7zoA>(@gPS}-u8775#V5`dP!@NuBIR(0M1Q1sONkMGPDpOK$HAAR?WN`{CGXI2Q
zeAx%9_-65(R@V!39eieEZQh|XpKR8+bYcSoA6@#X{DfM-Ze*o~LASdDbLh8)RS7={
zwmJy2$_gt|K!}qX^(#<j1t75*FsqhFbEF1|g@T(vw6tZlP22w^&MRNd<Cc}9xdP77
zl-8a`?nfHR7D$Rs**@uV2MK^j8ELb1MZ&IA-zmIkNyPYwI<X4Sq@(khjC%7zRB$vy
zZ}aOQPO;oGFk&VV^V6-IY+h=v-U4ovvoVrJ;1Odd%x!#`c{kX%39_6zEBy=H+YH$5
zEtk9NZJRF;RPf5+rT84x1?cS@uwyJ=rhawjPF>-dN$JU#NEw&uq*&Q(I6vvC@v{sf
z3BGmy4H}g`ejUzuv3C=B=8dbK{sa^*ZtiNG(LAtg-3-K-1X)3B!WMmB0a>r4e`$9+
z0UP6JqtH<4hhVY~2R|;W5N-m1r13=HZyTP~_3{C8=jEu(mR-I{kj8~u;suq_a3>Jo
zr9Es;O%YzdUEomG7^Z_biWMgUGVn=YMd$7{HmZ=v<U^bm0!~g_WTKBR(88-mw{_>8
z(#<I8uR8G#!_BXE_bBls(e<snti-LgfiS_hj?H$zrloJy{b@iO*tyYC(#X^X3cV(S
zs8UIwzuLkOlwe!-s8@8<+YtYjTI{eiRWEjbbWl4(t)?&!Oa`=QCKAF6?>;fGqc~e7
zq=Ysqc&^_(>NR`{^2SQ-DXidjd(^i5R}P+V#EBdq9<E=##TL%a>ALge3B8%Z<>z7=
z8Oj06=||sAW`|^xHS<e<*z>5Mb%;?*%c7ZE@vmyh(AX;*>p!L$UcR&Ii<@0N_(j^{
zwEzab)_*W4kx<^EEa2yxPJ#IC9L*qjQLdJ#wS=GizbyPN2akV1h3^n&x!2r9$B=`R
zK~f708#K2vs?znp|7mX`R9>Zyq7|<WE;RV0|L=dI+X9a^Ec?&<HB`BgipKx<KSLf*
z{!ZK%9TS)tb-GT?il7BO@(yyJ*@w2Ll;;=7(XAqi|DM_uC5@^VAmoN=`UF^TG32~{
z%W5LMBhG%R)vuzKr=8oy-0%kr=&)(ZX5~+#?CE9XPw1fih<?$dz9c{Uwhh8yQ~7B5
zW~N#fuDX0YhfyQ<V{tR{wbsK0ex1)O%{uGDJ5RT9qK=)dX1VX_J41`mwX2Pdf`oM=
zbJE-##ho2BL}_ds4RnWRRK<|y$~RiZJ5Rbd5Js`s@oLm?AQYMp7cObaTU}lr6wuS+
z)tZ!i(tt#^&|E8*8yoc*ns*}fz#A2<2F<1u?nLZ<_2<7UhaYpCYfm8b(dQwwv~ozZ
ziWv1MQPF(ErVyWn@o+1zgf!Pm{%@W_;RajacGt{H1*K{DEJAum^C108n`;m8TorCu
z$Yz8W&@|GDXflKUd<uW8u!YibuF9GwI33efw9nkxrecy4)$<s~>Ncy(G2Kj^o>Cd*
zaBu1nf|=KrBxl3s1zefRH={Un4?+*d0cf!q=f_ny)Mps}VssToi(haCkq5zrjmlJG
zIjtA&JbJ*V7Z`gnZI1Twn*@Z$G~2mk-&Iig9B@friWS8*IFEq&TlN$!+kX=p)#m{T
zK~UVuU(RF|gLefhj`KH(EaF-Z>)61Kz22l``G+;F&URtEF2q`u#*l){Ip<U+x`Y&(
zFx>d{=$63eFD#qv9@2>ttSX8Tnj=UGHDK&lG#Kdf2B4aeH2BTu?(qW#z4+VbrJf(w
zU#(K;8|*8|8Vnd|IpBGS^?DaArrB4+d}4pQ)<(2PkS&$OJ!Hd6+i2*a97t<GeNjwY
z{XQAP`X6uzBYSnFLGhpqVO&F=WffWAbhz=#p1c0sGusV{n4z!bAV#iwUXhAa4$n~h
zDI+Xb4v~$pZ8_hyOc;ieY?|%Vr<RjasA^QT2xPn4I1Ne6R5+<_ch8P6NepTD7}x>Z
zmrh(>rzjr~l9QFIteIC_D^gA&oYQ^idenS{^_pa36H>TgRLFY-(V89h<fen`ho0bE
z(OoXhz39@12tF6Po8EFO11+a{_2lwftm2yA1;9zu^^Ei*L75>Ho|-72X(AKfFhp*z
zl&Oi6aByfS(R}SUE6pg5;3*MPf$QcK3FyVD$qO~>?mwN@uQp7&3(&asymZu+kjx*$
z8p@&`i>1Vs7i-%j1lRv6$EP<<!iZ+tB^l#dtm^Qz1NT*~emZR{`lO=Cd^PsyX}FJ{
z`L*Ls4c(CON6-&1phbwWBPNkfSNhmG)6aMcc<2e7oRId7>e_Mg<^`gLEOwU3C3X_~
z0PIRKSXP8Z?e|>VQC&*J28>T--Mk<797WnNP^=ilSvcEOD7YWnrN=w0#*HOd#@Q1X
z%nP1Cye>0c|6(?!8n&W}c4i3EGMvYO_1n*EC6@EMhjYz2!`8tpavH(syp*^@Cmb&!
zp)WxDR#gOARAr<?Edq@vK2{!y6^l7jI6!kw6}GYk8?5^<9y^nb;wdm#C$3f6SeT#q
zu!Vj4!<x-D00(Q&8*$@9xWA7eC`}%zz~9b7*WeV@oBlutT;g+<Z%wCIR2-8NC(hJD
z4yxE^Vye5AYwOCozz8;{wUMQAb-VbU>$+(Y{{8SJ%f_MNvYT!zt+naTeX`;AdXorI
zh}$y5spW<33&{?$vNP?Zyw-9Odd3OXYLf)Az1T9e#6tKplsZ+j5qSl3I;tOjH^#R;
zFeTt3siHGnXtGgW7j(0;H2P(wbaT7V-02n*Dxcs>?TKZ-wFEx}IuE$u>^k56YrKgz
zC#rrxLFyb?L8&nIO`hwJ2%}hK{h6V}w6$ets08MZZs>`#O6EbR>2p)Z);d#;J5V#^
zmuHm~wED~e;c~iu+Irr;C2^&A=_fn>wg_$?%X#YLa45Z*2{Fe@BA!U0@rL>E@w-wl
zZ&1RGU4q16fyi=2P&mg-joe6+rxwEOW&%j(Fwt_2&?GcjeMD=}mI^^~%Wb%Bu6X;6
zVz{?9?B)l%_0YUd!-3}YlLHaTE!9!<n6H7F*2{~Vg-QIpdRf|-52Kp2jEp!BDDor#
zXh!Go%WqmXX42WsRUz>#*D0N2yJYw9K@Em&S=ss%3s<4mxoRV+hI#3Rqg8IsA>SJZ
zWhi{1=r&~fh$+(HW==x~y&!dIBXB^$HW5&=LE8EHP(UNHfh;Mq)upiN#4DjTsql(w
z6IbYVPwT=a$2HfB_)W^Hb(N@?#=HBD>7xk*Rh-fw0N(=}qpI^d`zV7iyOj}^a_V&e
zONokM$15Dgaj_g#J&t&BiwX|uQ{}RSWhYl3_D3AXx?*8eiG-by114S$DsTOF3dl>+
zsV(V8A>z9t*6*w`b!REWT_;p=pMlC*<U%cr<I~rK2XQ9^p{9)qW~PO8xws+mM4pQb
zh5Rp{Bww1IL32U84%-r5ItF7(H>Faalav$f8~zJ@7`0cmJA?UtG={(ZD1(X_6<)}2
z7XIKHE!4nrY~8NDQoU=qbFlY<ZBH+@>WKkE^KHVIco9VvMUngXR7_yq=u#ZUX{ivV
z=@96Quy4_3D4sV94=BqNH*c$J21Uq`==mN0Kug-PE}<uHDN0IOB>JdP57iX+5mY<c
zBQ5+kP$5FF@3Mgsk%Xr+b<VQ`BzQ&UD$saqJPB~4ZIo*oPI!$h;8&bs!-O|cgx}2i
z(;v9MSJ<EA8U&>Tjr9=ABx%@d!byJLh^xwdnq5+Oz;JYw{8D>#nLk7d-&}2+G@fjM
z8W;j!LpfTRYJtjy+lYz1jNsQqw4>mm2m)~+ksUv91t-LE_S!i=hdrRr7Iik7tU^Bz
zES41o4_gOAV@jp)Q(Y8B-)3;My~&($%cZ!m>Xa*j9`qWQIr5?LpiG%TTfE;iX?5-N
zoYycSsZkIU-i{Kin;ovlm@r?RXw^~vMfaMuTA3$qWpPur@c{VnE#@k=?Dfe@+oESn
zJ8jLWd~8~()@ki1kk&!+s0b0y``J+I;@qXRJTwLVa;{J$2$FEw7^t@1FjA(<uVe?h
zc!5B<jW>dL=3n~P{m;Uv8kh5r5HTK=pnfg>Ck2va6&!sIO2sObk$Thr*aw6baK5-x
zmwIx81+K5=*$!X$JoxC=NaLuArY{oQ*)AQkmTOI`%H=LyD0M|JQyjeFUUUH0ptxZ(
zFVQX-Q`C}G3)qQ~^Hu^hAzbpn`5zIapR=;)555XFR#>4QPd1OzF(JE^S5(D;4I%qQ
zo3F+w_iP5xg4Z6lT=e0K!B)L)T^-0l+M))vkq0(xZOkd?tfMU-D}zFI1}eS-vEOxw
zqL_Z6PcU5Lf|3xV5h>9wGF1G}rQb^bxD=awvPsP35HGFts14P#utoiU^RsNW97T$3
zEkzx%IkURlfSQi6r6L0sOT+0Vjprdzq}zD+?-QlN&rP@fFa0@USc~aM@y}Eb7N3J3
zitJZQ8am%jMg2GS6S{i5oBn(R{=tF9_fsK?R~H`Vcx~R%Wxmw4R1&z-4h;W)9;Mje
zJ<xJ_;Dw0%=FgeU&4PGdymcz(Mp5Vg`%xwqr<h2JEN|*15Q}9>^Gi!xP)(@hDKA6k
z^8fcE`$*9bicchhSe$75WkO>pVaG-cs|1`0-h2WyQBSoy)K4U3TA`0z*%MG7d!=p`
z8lx+ZAh4KP!u&A#SG^6j>!<$7@S)H71-XJ{;&=65wS%W@k)&?>!%Hw}Nu75^_09eb
zFHQW?&(OXZQ;sJ@61pqwCA>W66&&jUk_{A_HOK;CA6U)@FITXQZ&@I<UY)sNrqaPF
zBRN_Ta3Eh`ZQ85t8T~su#8JZx7|K}NxiAN_sc~|YdOzko*`DdsOr(J~%KR&c1xKqs
z5E{GpRJfcZRz2x)>du+RX4)D4L--J#KWP~+9y4+F>>Ix3h$W}XePTCy<-*z&<)$d*
z*w2?d7Ux!!$v6$7SZJ4CmdJRUy>~jkbbcrg@$%giC+ej)j<xvW;-GN;HF75jnHK+{
z+A8x7d0Tr{Zbjve$W;xv*HUcWSyzkEIB2#k>fW*0lTu16>D*J^X)$_ykhHj8b*QsI
zh8(6SERQ%>>;*VWeL4?;0*(6LSKfAxuP+|(;md}DA4$zAu8$O#DOR=|uIht~o0h%=
zVYCvP9d{pBx^*7a<8AY3Y_m~U-{lk|;|4rF4&lDosOS373!+IfGt=q$o_jk!t5V4#
z@JNrkahtT2>6Nn8UZv)08ggZIqj5BLE4JZ(Dzy6K$v?ska$0ZA&Q+`DNiH5mX6Ejr
zRc?EPI0OtP$_HhF8o(Y9Fp{30J=>BEtWG{3O(xnDd&RWXwSD>>4SM*1u~M8H&X-P)
z;$1gPL^mbetvZa}L;Zw2<DI6svTQRDS)x|}&aOJLHP5_YDTwNUH08}mqpWuI;mDhA
zG`)bwpm#iAh-1Y__Hca8-=>=0@;TLf_<NTiHSG%Uoo)W;ZD%I$IJ}_xsuetb9zuZ|
zK;{}c3q#z~?Fr(v4X+KW%?&?zp8nRu&C=D^t8G8%;x4^?8_RPn57Ym8@6O~xLjS#v
zhk~gIHe{I3qD{KA>0)gsl<Ve5NERk!-YbJ1_PC}V<knf|0JX=T<$lNXcn?l3EvBk&
z(2ELiycuNNq;pCIaoy!uZ6_Tu`*lO$uJ&)48Jh2vMm)Cl5YxP>>f8iuj&U=a?m3jq
zpbWDQulZj;h5u6^hBU4Lg@|EYZO(vA@5^V<==<Og&m~WLrQ<ECqDrH~fum99?@uk<
zR*Y`7dDaL^FUB{6Vw>${xN=I=|LD8nIe=zBYpYYktF&o<;f278Bk4A%!jR~QkP+W|
zBbtnl>(b;tWx<9NM9g?#h0`rdd}1YLp%jUaF2qmA<mHAQNK=>zsS_+3E^Lx?MvPlS
ztB#pg^)N`Y7DV5E&4;dsOjlSOt$rM2vpU||T(&bCZRZ>ZCrRT=4bBt@(KNQh1`S@z
z&=qFSCi>iyH(lFErv&#O2`Md9TnoRH)|b1IiuHH1HqHK~n&5Rp%DRVq!<o`lHasT2
zJL<+;O9r@&w~Q$B*z2dee^}!_jV#p^k;-}MCbQwf``16038RAD;-VHx=aVH7csoi=
zOX}@tP*;X*f`8t#2|9D$uAQhzWrV<fTA<60`W$W->`TPBmX)d{9xI>p5}+}7(f(FQ
zskp6G1It?IFTo}<G&<`|w)^i=^nK9&p`ard>7cqPnxQOD$GCzT?p#v#ZHq`B++b;l
z8r23ALo!{H4m^A|mM9+=XbkgrT_e4e&6=7_l3b7BvDpw%@(7GuL>qa4a?HGCWqF$5
z?A39y&d*|<`D#If|JjTFC!gJ**wRrEWb=~wk#jQOll~_!`X7E_o<zp;>EA{wYhuGj
ze~?L->O#f!O>;$*)2b#s?g7|1j=U}vb?!0J!VAPzx(H}#GBp2yIrH{v-s?3ZCKMW&
zA5)HQO-+oNJd9K?)rTKAWqyyh{bkY#8=CY|!MP`gGw%pIOvqqa;p-FM2b#Xx@rZ9v
zPxI3F%`VcI+^S_uBCRN7&Zf3%rJ&m93{5CyQysQ&6mzm@U3Ka5t(a?r9mA&9`s!s)
z*f-5HJh{utc!B)Q;pdjCrz8U>RX<^lIHG38sRuc?id2)hyURXC6q}$7{xHP(;K~ed
z>7J^KqC4w{?cXfiM?UPDz1K!L`m!z7w#WOt1Za8El1_ZX`Zv5ggwZ?Wl{w6R5eT}y
zq9z7J4{l9EQu=GGDTP<~v~xLTa=s9jFn*$|l)^N`DUH@Nm&e84fo5=5$>pb=MrR`i
zonbTQ$0t;i$d9_x^wGTr5%slK9&Q~4H?dcA|A6e7%+ABMVZ)wlA>UOA{4BfXyqxHg
z4Zr&b`s0V0=IU1onVN2c<uOcOc8V_zK=<E~t5IF+^JP185^rE@kO^yj9CYDb5K%3b
zfS^$GgD1J%^DoJNj*b6O6v*$bTU_bCap)rBqc-vfcbF^fjqT>3{H%?K{uS$u^m8*J
zP8~DN(>Ui;1w}%ZmkfMG^))#!iPTAc*(UZuF;;5h$-ns${-E@DaB|n`!zOLc01)nc
zCP}cUv$TxjE`Orj#OSeN`EApMVftzG(qc2Zk-N_i+h0In_yg|#_6#LOIBj_SbcWJL
z<0Qa_-qJ1Y;qbqk_1fV>9=yT8_2*8sbb%!k=E+B7y--Frz*d|4=S)0?r=jPj6O<bu
z`2xAv_-@DXX#G{6z!eqjNvJ76ctrJ9cCf(4o&e#QLSjCDi@$Q2-WKn**UX{Cb9X*E
z`Gd3+IpR5>7@KLLv$PLr#$g<n<`rBkQdfVC^y)!}|4B{~7;SDTS7IOVI_W%BmAX1<
zAo=#LV?o$qY2(5cC2jax5rw-+rX&)8aDBzvfB0+;6f>thMP#JBe_n<-)2W1Y&Pd}3
zo?THD={9_;E@@fu0AI^#6-?nPO?+Uu1D5pywH|%p<+g3=e%e<q8ivr~S1uwQWFHND
zwN#Why)p01{HAx+3LbRHIeH7SE8Wd}nw=rF#QwW=k9r$iNgCpq*n1_QG&a~)!j~jH
z;Y7=U+wDYW#geA&hg1PW^(HotYO&xaAFz_Dc&*R;yym}1uL<tjPSDEBti+LRp~eTl
zL}28%<qoPZJv7WdWl+gu4sSoTX<Hwu)FdJV5Xm*Q0>nq>2?@}(ojB{RA8kmcH}(YF
zl_RM>#WeX*=kZBnGrR{~@(+~#!BSe7AN;~ufx0JQW7t6bnp10&BtCL8Iix20k*zaM
zimGJf-YU&?uZt_yg^VGeHJ`f2J7$Z`QT7~F+YRP^1?!TzSHJRgdM&thHydBs^-q2|
zQsf0)7JWuhVJ{<BUAfY~-8JwnJ^k=*nWa8R`uq1gVHY9Wdw!fqy{E7FGAcFh=iy)Y
zk)V&#C87rdn;7tL!7Our4+|&(_(BQ>_AR}g0XiHtQY5XAHXW#t<@$g)p<M~?+YRsJ
zgqeO@m}`+^j}cQdPKmQYI{xkK7v0qY_hqJRavA(CHF)(kGw5|9Z#M3AEh_zD0cSQP
z?wd~LfNy2uM4I(9lsE9l`Q8|LIh%4{W$_cCVlltY7t{Ft0x@`{V|{+))>r`KxA$Gt
zyf7D^dae9z3QFbXjvQ|d(;gneWyHLR?`4J{@8-MYspMTDvnA<qI}yquO$tRngKuzV
zi2n#)xwm}#Pb`$!?3l*Bm05h7JzkmIxgPuI>fQ}`DOzH(5>L{(gc)=RNb7OGdt&4Y
z?TvaM(1y{oTHm2Uy*&ShWqN8Lsal<6wFS#D@ejV+ulR2N_kDU_h;1_SE0i|hT=}&m
z>1e81xdhdVkLsMUQL+PpCu9)*$lf0P@R**Nm&9iW&kuLi?THg*pIUc#OO17S&H0O>
zxrpmdXg*_ro=1e1ooBhOum`{F<6XB?rHOu3nxke{j~sbD_uTK=O5N+7HIlV6`yexS
z2cP6|I~t6hg{H*qiKuW8v@dr2I|Cr|%`5VWs8DjSZjkZ1|HFmTvGcoFE(2cn`2BrH
zL>Ung3A1GEX0Y%)d+JV-I>r6WXN?XZKTP?8HihIWuJGt7(Im+{dV4`n^j`tKG-Wx@
zOp?Tp;Pwto7&+UPHR@#l+VcyR{^ETHFV{NnOM6#3!kKgQ5~KcpOmJe8zAXp(9pcs7
z*BSszjeDQh(fS%hO%MU%XVAUQcUpCz!}yB9w`|tH%Mp%q{PA8+f0a&sH>wL!?lJmf
z2GLdcIxNdnLw%*;7+U+g65)%*)0G*VQx$2UOFQi22J*XMZ`-vjhZV}Lx`vOz&gn3N
zOm+g8BVqD~ItVi=GB@$6zikS8LO0)Nsd&4q9oe`YGg)HN{@Se^07v@+J};eLDi_ma
zYDZ(7(X*wS(Zt8A?zbz>6s7?1bES@cdd`C~I9>`}Hqj)8y-1<(!kpEoJW54B_{v=O
zWdU=5S%j6#veD(fkDp)zmMafMs9)_06sbyfF9+(DxF7MSDGZq(697o?+@-+#>-k<c
zH~a^tN>`1>FA^8NK7Yzjqo|^T;4=080Ji++JA*-hW9c`a%P6y8|LrHz(%q5>G6l%C
z{;DxWs`2q3=5zVh{BDJP-pA!rj$eIB&EKO}tl(Jmg?h;z`f)iICfm=~5Nw|wO`F-s
zVPu5~0lR9lfQSd|k?VdFU)E~;2%FU{DK1je%VzGj8k0GzJCCPO_hY)QcQu*i!<Dhy
zF+o4lM1F=joAv8_8Z=km>@}IRoQvk{BzfrjGc<Uo_A5{$Vb+@yOIZW<%`>a^0xdf5
zVs34|#+`PKXTCqbRMXhOz11BZDKfSrsgT!fI521;8?KqiC$6T1{f3oKaNGmC80R2+
zAX!m-S9^fpFW8uA1cp(1@mr|p%lGzeI|p{)A4QGdj<l{+wz8!v2~Jr{sE-2oCgqM2
z(A5U+T~VAR&8J6h1ZwaD4Tcq=SW9j6I#FckJ7*T_)J%!vBs5WQwVSr;30)L1t8B+=
zEm^yTpC%#bKZ1-P<#vC9odaORV2}ITshg?Ib8lTdEd=1bEv$}xY>po5H_Gi1y0Vg@
z-R}$(H|F#<?JJLLo^C|;q0^yjlGIZd#Okvgr@R~Jg~d1iDAwi%9;+OG11r$beDkYY
zvq%=Nyl<`Cyq3!Mih2M=z4y40nqp(Uq&InJ14%KsJ3KAT0V~@F?7v&B`8OZtaow4k
zNLN-i>8h&(L24H4PcUT59hvUOe}`+fkEG9?ioqCmCtbPS#Z?uIb%XCVi`6;}S$PfF
z%pAnrp=Hm=#-zk?6DRKIxi1KULwE`?V*VY7iUHQZ9y`L)n}2<LLGMk-gjUa>LBY(a
zoTYY`z!L@ZzLH4;&Z@7Vi98{5AUQq#P{GYGL|N&<$dp5}Hc&8Z?q7CnPCr^OG2Zas
zBC-L{$sJNxVVw{}Ux3L<7G1HOdnix5qQhGxs@Y#`5sq?k?tM(><k-yYHQ>cO<^$`G
zeP=5t*nrvp#m*=1`E4NLcjA?JM7oK2Gf%85E<bSZX+^#`gK$Lv%AKd0$}5&IBT|4T
z&Gh6`Y(?lz^{>mai-E3PA8L&X?jU9>>pwGog!H1VL*!sjPpD$AJ_H_(fvjIO>cQad
zfQA>1Hxpb&doQ&$&N{d%SVf?YTFtF<WmBFaIF8Qb??1BfTr6q5J-ztm`QCd2?J-El
zf33o~+iB9c`aRw2joUC)zhCs>4}`q0DoxEWlpPb@2g}j&Q$c<_&Oy=UZqsvV;H14%
z_g$9LUDZG-_dLx@>8mwdV7X|yc+`WMVJj}oOkd8e(xIUJ_cO1hRH8WaYzp?^jGxXf
z_2x_Id&$whiQo0_LBB?v&i-cC!_?A1fjURQ^=~IQiKA2NKLUUk+$}v3rsS1%m0#fa
zyyH!*aPj#YS=;f1B%Zmop*(+T#8uo_m558!y*C<QH#QCGXl=goT`s9{GtPPVI?yH_
z&8YI4C5Lwj#986(L>PX|tl)C+=3NJKSosSy4*%DQh0g<j#vPNnk%(8NdY3o$-+rS+
z!X;TvLobHwj__K&L;nTwO1AD1@$&qr;P^Z-=2!Nv^x!%KaOzo#r)_NL=m!VY%yI_i
zV_@Z{-E_0gD3eL1xz!Fly6}tp!$0Q5w@GZCh|;!|?@Ukk)=SHoUiidR{<Qn(KXeUp
z8fO7~*K*mNtlp^ZcrkkU8b{IaP1~9z<R0Zg&Kjo~!sBh|wo4$QSzbL`r~P&JJ<X1E
z7aem);oF9o#^E0$&SrS>=FMZWShuMFUP4KL`t@M|njg3DW$^<$i+I2n->8g<P?R+J
zh&~*Lu<ASO9OM$0Mvv;`&GwrT59t`{zL%9gCGJ)G#&tr$@k!<?ATsx?#|JcuzvtQn
znC+G-N1Cl5%*-UY<p$L;{I<&;ll5t|h~gNQ!vxYP2gdi@P>cqCZw;pP`Rypv^=bII
z{PTsk3exi*<Ea%3UE5ToFBqA<jWlXJ20PZJTOcwd)zsR>GI{|Xa3dzt&~JTCSA1Lv
zr=$y#`(x<*LYpd*hfaH0^~ie#HY+$XBO92h$PO8#0+Kh^?-IY~5BMC)=tKeEJ(L1j
zn)!)?FpbMWsVSnt;EvC|h|!w+0Fo2)Y0>aJ^y>QVjSLTJgJZf>A0ps{WIc1WxAOQl
zmdrv(zRWo-cpiRRgzu(?(pSg!xe~(_?9~Zqn-gZcXf&NCIr+Wx#gnzu3ULBr_4iH@
zlT`M1Y3F6;yQH_2o-tH@piHZPi8-uK?>C5vBX`&*ef~Z-<~Ozea+?9_<(>1KFBVxo
zY8527F6YAnpj>D7NG421aJkk91utHPubFZ|r?onPM87jV(?>7u7eD-$jFzeWGI+Q(
z5>%aiDQ=HWQ3Avj9)H`MA8}CJtxY@rVD`{aW4vbyJ%xYMv)RWS(sUjS+LP8>nS<9B
zW*R6c!{*3=6jg%pf9!A$bkk}S#Rola>S@Q0-MZHJ()bl>xRtYd{wAV7oSu+-_O{rM
zIg0RyKfk_H<)ecwb*d&=Udcs&Z$Ejo801Qfd$p+w$zfJ0exV3}=xp)Cj^c2*aj>ew
zq^EYF{N9jRdzt^KpU<h3k9iuC0>RWz?zbQ}#D&`(0`-!V)fQ7xVro@Sy0G;<>xVOt
zr=Vt_Bs!(<%j(Rvi)~UO(?`Uy$4JmhB1UV%^IEa3pH8tTg4#4xzl(|l=WWuW2oY|r
z)n`kW73U6piOmTIN>!%%H9|^rCRq9X@{WUBs}X6DlANA+&fdgWJE~>;=6LH{nJRH8
zeb%1y%VaMSC6RlDacvhcNY1}=WZkSH95mCnZ9gF$g+bj60>TwL$lu7p(1-a<noJ)D
zZad4Ub%Y~cI&A8znRZSt`Y};lxOZ5irgD@0=8=lv$&|{QzP(X&+qV)%DV&>%%W}6?
z!Qs_$&n8Y>1J>JIV6P<<dS%|hrw**K+)yyd?a~(_mLJMD91{&+s=KM9$658g?~xM@
z5qDMj)#oW+YMSt-BIVYk?666JIv*-MF3Kx*O2&)&82|~%OuKRJqwu{0ke#+NWSq~8
z{c0$?ffBfCw}@(E9~j|Ab?l1z6PtA2i7}oUkIPuSEY)Xxl=AYpDSUf01I$<QABzdt
zPu;Wem{cfQSoRV19v#XiH-G1w2K0vWmhmiKn$8yL&7CP^(04*FBCg&fg2y&&exFQ4
z4E=|$^|M&h+bT2>&OL-~c`vgVKeYmFXuHe6&-1jWT!F-G)0;uWRw5R#xp6cXLN4Q%
z5*!$j|1;NE>BJcHr*?#n^wnFMdUNfY=UsecUZSJpHH&5g0c^#_WG)VNy*UDQuiimW
z8|gGbNqpc4|5Ra^J%z*^|FGiaOJVic_$##jg7#c@`ZZmy-s(F&aBV|I=%dZEPhF0{
zS8^|{?-T0xr+$*Qzf&OSWStF?8P~HN^$e#dk<P<R-}GJJt@;0M(Ey^fg_EMjF>%}M
zh>rfN+_ti_*gmJ+;?SuQ%FW<YS99kLodJ}D6OrZlb5n(OC^r&bDFtezYwS;~`hI2m
zPS=;*&D@tk++b5BTza_Gz*mXQF*Np;!`%-?Pflt%wRf7mdG`Xfbk&?_ut1Rx?l|>F
zC?ShU%#%nRyke^#-6+<1gG|2xR5L<qnAHKl+A!p^%&j4K7q}{@%Z=$5%x4D{TKcn~
z?qeU2S8mdb@nr$e3Y_V?AT)+5+Vmr|;&~puJ}<Z^U_im~?-F@^zKtproPqBh)ldgC
z-YtsrU7<@4v}!M6qXMl;T5|$Ny~m65V!rFr4;sOp+e38r)4nB@=hsnFL;~cC*3-nL
zygVuC={Yk+%zangCi55j+2A+!^V17f`3@H|pBQ$cJ(mir<Na^N7M+=XXONBku7Eeo
zgMh%kTc2fTZub_p^2l3Go$*q!eDzKpS(~SKlT5$ctxOL7;o}AkCNfL#Ew&4qch+yf
zB-Im4+S82O-r3*cm3fDKx9R4E*hl?@$+|9fIk5CjdSnrm-?jVRc1(KwPQC+nxJsu1
z2cB%^Z`rR!F|*pya{9N{KuZQ!=5}u(U40;q2N#I;18)}TS2bT~{i|!OG1h%UmvD{s
zCB3?)F^M8+?KlH?`9o;m!;~}NsL(vWD(>ylw)uHqUp2<Z&B9I}S^VfJHRbOk<cM-S
zpV$+Sf@!bBdYNDHT*n0zt3Of%FgSbS2rCT!>_P)>MQ_#VkXyUb_exRfgy)Fj;(K)}
z#IsETp)|p_Ec4D~1lK%1^b{bym3KD!l=+tx%I7`s%{KU_%xHi+8<rVbmX;QPdH2uu
z^*%RS?aZQkz0HX29%I{)!c}!_mD|LKCVWl%4>?80E~Wt1f8tx2nW3G&QbTe$e@L!S
z6WXcoFyfdvkv{|Om$|QB0Ypo4Bw4al*Yz(hSYDVy4(8mQKy*BhXzR|zQ(r&vRp^24
zKP@8dbnA97Aq6Tb+w!WlBSqPALUA@#UaMmllO?8l;IJZQUydItoE>lakivSrc|teG
zE??cId@A03u=0L{YvsJUSxN0Y3`H@RAFCR~h)ia;x{Wro|Cws>3+0Rjp-N1~)hsz+
zLX~iEYzLFtzon`YX7dTH32ypwxj<>I*iIniZN1spq|USGQeQ108X*SZb9OjuCh==j
z;rM1bgqj>jK*UY0?T!G4bcz&-8NEma2sg)>I+nuH|G;+X64#J&Hv!s;#+)yHD6i@e
zEhKH$t!lw03k~=qr0+_)-v=XIk!jN;e0M*ox4!^G_?6<If?6=W0w*xD9G36moj=nh
z7c3{h?)sba$<O9tr2&*<aywRSAuftbo|+WLpOVb8nF`yT3rLLz^`5_<R#w&mw(=K=
zVV%?D`MLG;u$2X7(Z>1v-sc~cm)~k=MbCZM*6v>PR&Ouv2)K&ltfmP=;RiMaDK>8-
zQMdCPVwIJH^`n_LM^8ZRqpqhjz=hBr4{kuwt8f<3%>2L?u8NX#t$s6tcbA;aW1Zfe
z^fevGEem;mOSxiw4iOEtzTENM3)`&;-|kw^XB67={g%bvLERBIU%F~x&x(?aX>%%9
zSjTtW0gJt^9pP<nWKB(WXhyiRp=DFC^<BY;_EM|QDdXZ%4L{2L;y;2>@QHTVe-H0K
z%;=@}2BtHMYEssQRtIhL`-k$ihMiLA3Y=_c>E$>Lxo<mwb3yE`BXGU6<HQlBkRLb$
zfd0TQu#c7MBEfm4wcNX)m@yZvw$U5m>S>D}y`xhcGQk))M~%?=iw!tn#{I%U_)Df7
z=@744%XUqRzMcOChlsOf6W5FBQVyV<-*U}~vpFW@?Wk5%OEE&|OGuC@vyo}MUdqL1
zhP{Hj-UE`i#vbTw{*x%C%wGL1P>)vpv{paI^qS%JhP`@UuE$6p*-m5mi;n|mXveUb
z37ox^Yu>lkzp|;lCL_<0>Y@Zgi%ZkeE<A>^Ke)c1bjYhWvb5WDC_9c7(Gy5L^q7L|
z>WtI~eA`H}>ibNwO>55Bn@M<XRWYd)f6X^9u#!m!;A%(AP8gHpikn>kZnuRrb1Kw(
z6~VjLHasuk0<||BVNo!C+Gky@yEngonyEstS@;F-Fk{}a75rxEz=}kRXandijpeSt
zb&8!FIca23=czJYSKDN#jjIWm&7_S{XlC>)bb0g-3A1S*Yg$KaBj!;wacML;5R&TF
zF@`mvsH7Cl6xG<cdSx~Oi@bH4OeMP(Dvl}S0aLgwF(}7hQA`8aUWtV-#ku(Wyn&a$
z-Zc4)66>m0-2>?{Nfy1Xt1JoAIPC=@05OL;(>6S>b!qox#YQ<S^(rfCbf4e#v*rqM
zojcr_^lA=x-AH+W$uhg}c@}Cch(M<#(duz|I+n;B9$nYC>_q2}Z$gawWl)r-(1a+3
zxKQ?x>VV|sQWsD#R16h{b0Hl^m%Zm!t(l98hJ&8PYcCQxQ1`bX`Cj%!WO&|(wYLRL
zNp6ElIp)U-xm^V}4(h_!5}GVGXMa3`6{fx9)l;nPnCbX_J<cBK7`5{Cpo%-F)0b7;
zy``mJ{uc485So6T{840&ohfn$LP*myQ%^B5Dd!;VoBdsvY5T-M^8SxMd%Q)3lF#OI
zakt-AOhh!~b@oyQJ-P$34$60`B2RN()5nk{zx}6WY9CubXrk3UFVj!E_Z8xj@Xn%X
z#p!rj;xxLXFnKp2I_18tap23MPIw}EQf9c3`<D2j&CDj?WoeNZr(9EVHm)PzsBEUK
z#&W)5;fF*NGGI9w!7r|Iiuk3&(2F0|f(?H;``dgmS^v1StRioW;1{R6iQf~8R$jwT
zu%Vw$g!z>9#$-H_1chba)WjV?AZLPP{6P3QhC{)-79Y@AdcpnG0-^QSbsx1UHUztf
z^vc_{NRB6E8%Tj$6F^Via<A_c*)cOrqJd;WBi1u`JQYO=y76Hgqu6kNVXt@=B<A%g
zc=GbX5Bz|^6+hb)@vWS{MXlpfSJYIH-z};}o{uc`5Izt3Mhqm9kW1!$hmC{8^UO&;
z?W@KLmCIID!P%pXb3E_{VhZUWc_#Q;9LR9Tk5@Hd-Gj7CLIn7au~hOfwNU?YTAY9z
zRf)6yJX#YJ4f41(be$Nck)gQ+13f@jD<BRn(ga%DbpRT0xiy;Q)r(i}IPWQ4%H4r~
zH`)+hgi45^lRA-o?UA!O@Z|>*LaD*-dYNhrxhtEWWsFiai7D@&|NU-=?lHg1puZv=
z6lS#r9b{jM>CO+kOl6dPvEmgy{pPjXO3TpKaL`vADOGnV+B0XLsL>ZL@8#K+p8j_-
zv$wi)ruz(Ow-6{Ew>W`3y`cAMGUrPy|JzRmuJ4-iOu7n8s*p!w<LH-?s-B4@cAnfY
z+Vd7KG->=YQqG~)aZyjcGB8d09Ao@*a0!vG9%Jk^wRUk|To?S@_eqU|Tiqn8{<`iy
zgg%OKO5o0#5xR5q5EvnCPi;@@p76T*xBkE@zD46TnK+cGh9QkIl-mrnUfu~h!V}7J
z3BeQJmq*m7ZKDu`$?1$La?qk>R}^*O8b3q1A$`(sahe>607DUoUoo(^_9D?=D{-QB
ziJW6O2QazNrCsZuSW=k_Jp@5*_Ed}4^%&7or$&6VpV|f~J!#$_6CzyX^Twt+z2*Ap
z_3j$0T|HkNRDfP(Z2`(%5Sf>jg|D<zHrODB2e^e7E070N{gD5G>YK~1e)GupZ2DM|
zPP=pt_TCNt^n+TBGR8Gm-WA0&3qeuId&=?mvX^horyB#~)(-8rMHX*S3b@0x`BcTf
zi;U@dNB@3%QX{<!1+Y-v7p!*^Ahw$?Eg?tKi8-llj1_lAo217}tHgJ?PfzHuh0*UG
z*oRK-9%7CsVG@B8>1V=4<pW;*+Vy~^2%8GA+_~AF#IzCti|s$9!PR?0F%@f4F#-wV
zs>7ar;R^YEI<gDoTZ7>W31I(u7kV!m<bG3_m5Z}}*ROR8I1gl#C6_J7H;AbtC4B#`
zTL7Ez(E~c$;vuv3k+YI!6$yzq6AhN*P1+Z4M$a5->VX~t{3Sf~lw>wE!gJ19F+J{M
z{n=BtQxpYkMUT@c0iwbuFcpw@mZDOH{7^3W+wQi9^W65=uLBN@Zde%CwZkMpkA5lF
z{yVjfTyCdnP6kWrz1Jgh9yF#^FfE2zWrghJdDO-AHZy$A9z(8Mf%Fn;fW>{_7Eb8Z
z_s}_s%YD{Ypxc+N^+wAsoN8?FoUyb9DnKCj{7Y_L7%sp5<;tm)M0x)+8`nJahU&yp
zy9?&Wl{JVl*Sgn%XB%SNh(uTQ`}`j`=(?Qola;)-j=HN@{fOz+5W_B|sR#(qrJ9O%
z4fvW`wotgsW}a6lfLgq!Cj5cP-gZFQw6`FxP=FEC?;bwSFIYXxJCH!Q?yQZH{Zet=
z@4~Zi_xGLAfCG498)WW;lbtv22&L3gx@9swIl}*Q873kCFzVLA-<v_~R(8*kKL(tp
zbY}Mt5?dp`2YA-wE%MPEh3gt_#tC53zGcKJ$leR*q1-LUto(KS9LjpWe9yOiO+|+{
zT!d71v=~K(p#Iu?UCm>ISh$=&HuM}=-#>;Nssg_jGJWW%xy%ag$&aVsnZLF#<Drd2
zvpRmidTzCk#fD4F&NA>SN!lu>i%c2D7wu<;Wh^=O_5{_HEmV0~Z^k&C<;J%>o5F$j
zL}@@0nI9c8@ALL4J3jXuj-Lf~YUf-tSe-!*wA&#TdP{DYSWlz7Ww`OVce6`A*PlY~
z5y`hCb;YkjMJZXzK4$L?79Wr|hryz~iHM)MahMY&tWv=u@7pwEqaT=9;^CLRX4{9o
zLMcu}J78-MdPX4WMrk%l_m0uNVdv~e4_)j_t-0H1N9kw{YxSh<j^riy6+4naNQelS
zZ1L&Etj&)c9MMpRJzDrJCsyZ@<Va?I*rXOMBG|OVG1x5%-AC!MHuoY=TDg_vGL}j%
zTi2Gj`KOXF%!<*ix%R&P)60E%*6r!_Zu-#!<k8U(u^u2<5r)(OC^8HIcE*23-MDGK
zlDyvb{$roxdiY+o<*Z4I5UMEhnlA2V;|D)A+eoJ#4^T(F{0vs2&UEkHI?r{num1vM
z4&WHZBy;>SWBd<yUnj@O?SrvU+XJbyjrLGM^8NIu2)2nfX^QmG`e#oGC|F@e&!Abt
zWcu0s0>LL1eVWVwvFUV7F+ZX~>xe2=^SAL)?1jMHANgO_a}0dcGuPHX;|gOF5g+&-
z$4!AA{(zje(}m!a=9ryE>J0&9ar@SQ)pG#iWmsjz-N9!J$+HaPHZ|E?7v6@M4d2w)
zGm1!bDUie7!qL-hbfqhm{Xo8q$K>|vG7du=j4FQBz1e`|-)whGv(cffQ*A28(7*Fv
zqXV*IA+anzg^0zF)o7OA08L_xLxcnyqxX58dPQP;fOlY^p3<C8gaJ~Nfll0+02Yg^
zIOMX;#i&Z=VhM;h14})lObSvlS$KP<%Q*T(;KAb~EG@^P#;b|e-oB$g8A~aPgAvkE
zx70;(6K@u3B+>D!H=cuY)Y>CBW1qOxlXF#0+`gmL6u`TC!5O+W1svw3m=Z9J<G4L6
zH-jEW4tr`<%-Ux&pQ(w|_A(jf6joHcv(P}ZQ3n0|UVV5ZsF>#u`2iFYzx?&I3e=yB
z<H=M+8p0lRX_rVI%wz4|#5o?bcb6U9$5oYly`7}g+<stAst>I#Ts9rByzDoI{SqmW
zVlyFtxqPPOaLhBhSkIInpd$WoF0@PPb`*Bwy^&TLsgJi%RFI2s0n9XCM1&-m#Ql>V
z!pHqgsYhzA89#YWb;D%<^vI0V2>#TBg}R?|PkVfS1+v}X8+--KJ2K)%F%t8G$>5O2
z)RssslsANf@4U`_m*tQGRdmJizSL{@Sqaer+(sEb#JhY^7U%>NxGN7W3p3*P=*i|6
zbZ5~mP`978nW9jH`aHcJ^5{j?DXRL>sa1Ok25wcG;ia;^uvu7C0KV@Q_xmKkzG|;{
z6W__W_(xP8J>h)SKyAD3wI%6@ozzbTWp7o0hG0TI@N!@HSufAw^rJ<@1{!nNYm-R=
zg&2f$tFK?sfj2lg(T)n-)N|h10ko4Y-FA-Jv{t|E75uU-uX~&OcA44-VsdxEU~2xJ
zJ~U8IKyvaE^Bpre{9$;zlYAKR<zw`cm+bg`A9`5Ed}Dd+)>&GY4M)^X#UFpQb4J@t
zVnG>V%H05aeyJHvT~j{@%J{)6$W~yCf|+iSL6#8s^KEQve3Qy;#nk=Y;rW!`hP1mA
z&Qn%{>W2V*|M1B=7R`{qT4?>~lJ&)e+HK}E1&dGD=wA)-YD*Tm@DG0-PT03u_+p(`
zPi(5A^+f$kFVDsbV=bl`Z<{Lg?PBe2z#26v6aRPJ5Gt_${=t*@isP#f^^DMObJJEw
zlje@p4LgO=@r7epif}7)(D%9{{n}B*VyJl|7zEn=SHN4u`yYP~`#}A@C4u7dE6Fnu
zY-^ePXLxS2?E&Mb7H(6wT;3I2W9$~Q73MSI*}W;ae9Li<0)PnY*y5%p{+`JG>S$h!
z&4~R*k;5POVvrRQpOdmPX?F5wrCriJGph<I-SZk)aon`@>6AagWFxLp7OYtLkEMTM
zxT3_7aVCcApGk5eJ$$mL|NWsFs~Ponj`|nx85>9jUa+B!z^V7>XSRu8(k5VEh6Dq#
z{$JbBg$Twb5q{0o;Ps5id5rPYy_82&9zU6|v)a5vPe9f;^7F0;`HoMQYR1Zm5XLxs
zZ=D-s+{2pXo;dzPoa3{AXVSQJlhQPB5)pVj_sd(P5550i-XdZ!!MUpfHP!V^?(IbX
z7|b#-Y(tFCKsl0IcniQII5#B52NhLlF=|~q=~dq{Pc@ncE+rIc_6{4+b76j>LlG_K
z-pAe%*p^_)in%5~;N7<F-ajLn^hv1DGLnmJU64Ah)-41Z>5B%QAAhax3JE-}kr%+>
zP@BOyMX7N%6IQNz49K-@Nhpf!Y<bYDto1#p(EB(AmQ{`mfE=DQqg(WmO7FFsQj7zC
zjFconJ#@6WN9u`zyW5lxRmP_U0*=EefOROP+Ua9z|EuE$mFsIV6A&E#50ro}t)=nD
zJgoJyXG7L6A~s`Xoog(SI@M~;i7WB<cLNyv<93L)P21p{@vems2ieQo(j7Bd`x@vI
zsd+FYE>O?;P{Y0q`;#*}GWbHKm(x^?lP(Gvl<Z+^cVjsjR#%QgQu1t)c03c*X*Vx6
zf`;5IT5InNFl(D+B#GnuFEKiH!T2w~{PYB-GdE-`Z<#O5Z0L>_Owk&ccYXfa7+}V}
z?*qlLzVN+WEZq`l5guRH-#zd+WoLd^T1Y@iG1jE=lxA96bf@CL3?cd1%Q8JX8!J+q
zH2LtRb@M~FMdo!cdFG+R@C#iAOyzG9lAw}`J25r@MpFe2uIoDwZj}L_Kkz1?tiIa2
z>x{Ib!d0cgEb<jk3C*<26(w&cS6smp)rOrT6M5)q$&S&7@PL+asN{LTFNsi|sj>6u
zr{bJ@nMC9d?mGZ-@Gk5xFiZp-p#L;XoUky~^LH#t6nb{OAyHw}!uw|7<3ho;`yV6e
zJG#ocL2E@whh94C+Ww|f5U8!=cr`m(9YLPpeief|$wbp*+3G=lIey`QJSln@$KqD{
z-jbt`H{;ZS(td>tk~(ocQ5&&GcTY}BgaWz>yYSB(S6xW1?ev`I-ZKB5iH|lUP|YOG
z;bV#&FM|;%d9pr>9NcN@|BAfeOxj0WZ)E#V#Fb|kEYm7?JZ~1WA6bGn-NH#t>U$#Y
zmrEgHPq%;X|KLf~qzZ)Jua|>p+fowOXODw=VJ6&O!qg&I@5rwWgRI(=LVSeu`oE3G
zzjxMX-sVFHj>XCsxWaVa8!f6ghg1!J#rR4*p+{Z741RTi^PpEy&52;nC@a#>9I2gN
z_a`x{Cr?m4ozMJU!E&2!8nNjYHVOEm`6@Ng?PZJhbBe18xsXki_W=djtVl7g0m+q)
zR%hY_2Ti@s093ddcluS~Mo%Hngg>nO;o!}R-N|epi4Q7W2um$`I?_^I?3?5|nT<YK
z(diOc@Qj%?i+vU;*}&abv|2m4QII+uhy>}fBzPfka#AOUxE*tDS9(q!EJXz$g%_n{
zzMOmRi8E@Hge~nGzWUp(bos*SU~NwqTG!CMVtM)Uh&^`J+RNeD?V!<z2FE}-rbaW7
z6?PpI1;pZA2cql6qWYJ5024avld6z9CPv;L74_SAL?|@%Q2QT!@ASBCxxSt!Y_>TM
zj`>}4BT=g*>zswCN$mG9Ym#f)96(Dy0z)|2G|tMO&F6lc6L$-nV+vYrW_6ZI7~a`Q
zE+fkz8Ar;^S#!y^h4LNvQqb=0V#?4}nr&CB<2qvXLwY;Kol9VCz@;e~mh+ll0Bp;(
zZUO2>YHK%9BIUVFOw<>=^RJvt3&8JK5mSP(Y@GzXJEkFdRj<R+s$d6*0)drv24`A)
z*JsdH?3(JNW`n@wniThY#pA%l#!>Hz<W+&eYiG_Isk??CzK4tyv;@m{!8#VRZ`56b
zt8e6RyAMXr{h#)(H6H4;4R2yIuO)2~D^WSD!<Z1>acD|}VT?o0#|<&0)+m-3wu6$a
z7$!3gW1TCSa!7>1E9*QMhf-~lnTQ-_SZ2b!&t!XlvueM-pLRYypZ>r9{GR`PKlgLp
z*L_`EH6fctSj8X{=1z6^KYFTgH=vx=CG8Z=kp@Y;V;GobqATFsT%6HEj(2~7GW;UB
zX80hiHJ%mqp+m1H)(5zUym>3hUS&lcb{JLqT!z2UxYeSk7}G%L;!uCe`!$A{wX~g~
z&KlPpVv90(?V-quegx&5uH1^UG@$&A66Y^|6xqP+oT$IRXA!DS#<;`*!BYkJqp$Wq
z7MMu_AaYDxZo<x;$BOpEAdA&v3q`#4Mn^gAFH#yYovEOV6#xPm_SE+Zs1Y}HNxf$s
z+0sc{MhZV!lBTsy(q`p3UgMzI2RR-gBtVsx6Y8QL4={tL60Vb{;?zs@Ft*8)5dD|!
zZ&jaFpp&wk(HZsqR88*Ka;ZXy|08p{Mr(IRSU6TBIzl9ea;<=^ZRm5f@eycgpUj()
z5a2Uc3$viL3Rn8<s)E?~y|+ovL$Kw>4~p|9oIKx^P*OUzHij|GmHfnl3>N}y^;47U
zRjLbYV0`}*zuXil{m;fnlSB(|T>Ca)wEiFsQZ?Rrc!=$c52<$k22{i%3q?5R7Fk!o
zL1nq2avMGwNl(Vy@1+io(YCIn>{-p0HxW3z&R8oV#8T5jF!3BrN>z|!z7p|C$%D5O
ziP_bA@Mk`f(PWzD>=6U)RpO|?!S1OCBZV07?Mi~ENNkz}g&Gx)`}VZzmH~NzzwIrI
z?6sj%2yIU6;Hb-bQOsA>7=!uqtHJ2lN!XeB3rK{MVSzXZ3jjKQ67*LyMD&e4Qvpy8
zK~b=abH+juvDKax5OumX*C!qD^@m1KA|_3Q(;bvu7H3L$Eo$s`*t`dD3M2XU`xEav
zC75Rg>1doSveJf!qq6cC1;_H(*!8|Bdn=~h53672u!W!Z&j|A8#+G-KqVpL)Pq%p{
zp&Hr^Yfr^_UAxnRMi1I#88TYs)<lGGS==2Y5Xf5L_GKoK{S%oXqu&5nXDc}JNH%hY
z8w4WZ=bQLLIif2P4Juc#dJXOBNh;H=qs;re@{-q+9T{I!RTmux!{2L=G>wUP3K2g7
zglx`&T%ObfiUHUci-7|-cLuqaai;T@61>osD{2t8&<-s$g)wwDX38gw8RHqzZoz9&
zEV$z1&{_{JMXG^bc;;>Hf+cut$}p$kofdl@XaxW2Zy``(L@M5#>!Ba%oU1Ohv2(v4
z+dg$At8A2xU?iqndYr60X3K@yjMdyF=cg|KRhWav@!U(zPeVpGMvn@@cgO(h1@JJt
zBD~pv&Q=&JRIX{3<I;uB#oQow<`u;&V-{VtFb3R^ecP#;8H;J=8K+CgZgsS)KzS0|
z@+ZR0Ugccw{q0?hb+ilTbMgte4V#(aq{el5%AOsm4z%retFenEsYb`uc5!c+rR6Hc
zlb7a8$x#=VW%>@0_86(Y#Y9hA0>mPBfouZgFgi>6D+REjBV#nll{p?40XmO0c<*$@
z>mId{Fk)IGDvXm_K6rKg;ufm+fgIfW8e%=t8JYg{nhvNvd+SvA-5%o&HCWynnWr~X
z*8|kTbmatvD&|!3>Pcw_7WAVzjRntx{cxk@jb5rJ_#MC{-H>9Sx5xKX1NR>AihK|R
z>=9vBiv`+Io46*RbAGVeH5|?h6hA?Eb4~}!eaSowYMW}w^ZOxC?(OLBLX9N=m9YGZ
z7YeC_p)bu?j8wh&pa;Y8zbFqem+6?i3ak6#m%Hu>L_Tw8`|^Yur+>DGjzlHd5QFv9
zYc8x2=U(5E;bzfTN@Fd>*^V_-BdV$2Oh`YJRK@I@06XuhH#aArqpBYh0VVeoF#oPx
zYGV1?yI%@+6bi}^b})fUsZ!rCl9pC1!Z~batje~vk^0|L$pPA<?gc>=H-}^VZJmd_
z_@xq~V6A{Xn_aAKIG5=;|Lj6x^A}p8??orft)e^7W`+H5*@tD$>7I8rcFSh4$F)|Z
zre(OQ(uQ3bG}a;C^9{O)ABYc#2$E(M5&wocu@SSXIuIdt%nRdJbYOS_x}ksNV$~eT
z6M2L`1>aJ*W4OemD!$`AYY6N8^tQJFxPE8=slZB2khpv>IzHpJo?!DhlOH#4kdyjH
zFP}g8SCoJGz)k5)WagAlqdf&(hFKtzzbQ?8=o*m^rOJZmLDu}syQBH}8p6-lJ!$OO
zFhFSHJuy^merQyQ5jg!;l7(6HU^t$T7BVG15nX|-QhQxot`9>$2#5<Ui9;b(zMnT5
zM%&X@T)?-mW<~>42X8=i;JZ8}^!{CtRtcdzY)aLBE`E~rFlH;o^v?4V>};_NH%VeO
zP2!G!MYVH2PN!5B7Np~N$~LatYI?B*6fc0H&M#gP`8RcqCbl>)`;~~Gl0tk_tY}a&
zVK=eX?#wWI^4&O86&1X{JRx0<I}y&hyq+ktiq-!Q>Bn)scY%a=?AxWJo<#f|M!EK{
zQ)!nwlg=?0YbbvX4D|+%q|<(fE_|t#3KN2FOZCQ@njH(@$Ejb5_;%`8v!1hg=rd=Z
zh(3g$qzq7t{BfVo`&}rN!R`)*!$YZi*}(6(K92AEY|+b(W#pXQ_l2<nMl)8(()kTi
zV#wP3m$W@@CmpiZgLC}`3G2{Mc;-4V<y1<jFmHHjFpzJ>oG1$!u#}IhJ`0`+Jb^Xj
zb1?v9OxPPM5Nn{ZSw1js3gsyq8x7cn8l)))cfOZ!97i>h;=3TyAa(6rExZxsG2lhm
z3rbt$`AZZl_T`uI5YRSIp0gUMwT-=X!s@=)#nc=Dg!Dzikvce<2NMemP#z$Vn~=^v
zB2HQQykYA`B03DNqQ^8#7!AMa4KhytrEP0c%PS*yJPr}<0!6z!eh9~InBj6}SW*iA
zrY(zrBn}v`n`}gn3#T9H@_B9ToMgA=__8B0+m-%Pc;3GPD*1yY?g(j`FTdeSCzT~a
ztXLrHGy_#-zO{hkIh;}pp0Cxh86Q=o!01RQOso2&wFj@tqA}4WYdrYz%)r(4WRcZB
zmhevjE_5!xJ3AY|qoUDoWk3d_Wxh3HJ#GaV_Li@|s^H<Z|7Lh#w3Wug;Lh>EOJzw*
z^?BB?3Tv_}V9H&;p}*t@@426J7{H^(eTDY|>he0YNQn&CV!<rs1wlJb+qQwKJBMGN
z>pEN52<S4Kt%du9poqlc10_hDd-*Rdq28?>>TS5+2B=$f@z;Z26REfl%vJTG*KOSH
zZOC1{R>VptoY?k5ZnfEuip6)co40IeP+i!r^iv%mZG9#J%@U8cG+i-xO7PDb9f-dr
z$2rk~)(x_E{{VfUHrs3ww_?6#BZ&Z1N6m13S*CwnP92wY=qTOeDnFpxj9(3WnaUwa
zKG`3SWof!WySuEzn5{(5+xu#U%$_^YYyq0t@@yE5k>y=i8CmW3?|*Uae<wM+bjqH8
zbN?cBQ1Mk}$!<hZF&~2{y)*JSQZ{4Po735T@_ZgRfFH53ElDXUDq^RZ6OZ8VqrH#8
zJ3>$wn~W6<Xpv7Mk8Xv0=GeBQPw0hJW%d!eWQG}?AbqgbB%R%&WkA^dG06(o+Nkk8
zM>AB|%J;lq;jpNz{S&Z{;}4cOar+P?f!Gr04ga(Ji}bfZr2MlK=z{#;mofMz-Zo#?
W@A}3t`E6f;pOv|-**#O=i~j{8Q%T4G

diff --git a/docs/index.rst b/docs/index.rst
deleted file mode 100644
index 32c3a33a..00000000
--- a/docs/index.rst
+++ /dev/null
@@ -1,160 +0,0 @@
-pytorch-widedeep
-================
-
-*A flexible package to combine tabular data with text and images using wide and deep models*
-
-Below there is an introduction to the architectures one can build using
-``pytorch-widedeep``. If you prefer to learn about the utilities and
-components go straight to corresponding sections in the Documentation.
-
-Documentation
--------------
-
-.. toctree::
-    :maxdepth: 1
-
-    Installation <installation>
-    Quick Start <quick_start>
-    Utilities <utils/index>
-    Preprocessing <preprocessing>
-    Model Components <model_components>
-    Bayesian Models <bayesian_models>
-    Metrics <metrics>
-    Losses <losses>
-    Dataloaders <dataloaders>
-    Callbacks <callbacks>
-    The Trainer <trainer>
-    Tab2Vec <tab2vec>
-    Examples <examples>
-
-
-Introduction
-------------
-``pytorch-widedeep`` is based on Google's `Wide and Deep Algorithm
-<https://arxiv.org/abs/1606.07792>`_, adjusted for multi-modal datasets
-
-
-In general terms, ``pytorch-widedeep`` is a package to use deep learning with
-tabular and multimodal data. In particular, is intended to facilitate the
-combination of text and images with corresponding tabular data using wide and
-deep models. With that in mind there are a number of architectures that can
-be implemented with just a few lines of code. The main components of those
-architectures are shown in the Figure below:
-
-.. image:: figures/widedeep_arch.png
-   :width: 700px
-   :align: center
-
-The dashed boxes in the figure represent optional, overall components, and the
-dashed lines indicate the corresponding connections, depending on whether or
-not certain components are present. For example, the dashed, blue-arrows
-indicate that the ``deeptabular``, ``deeptext`` and ``deepimage`` components
-are connected directly to the output neuron or neurons (depending on whether
-we are performing a binary classification or regression, or a multi-class
-classification) if the optional ``deephead`` is not present. The components
-within the faded-pink rectangle are concatenated.
-
-Note that it is not possible to illustrate the number of possible
-architectures and components available in ``pytorch-widedeep`` in one Figure.
-Therefore, for more details on possible architectures (and more) please, read
-this documentation, or see the `Examples
-<https://github.com/jrzaurin/pytorch-widedeep/tree/master/examples>`_ folders
-in the repo.
-
-In math terms, and following the notation in the `paper
-<https://arxiv.org/abs/1606.07792>`_, the expression for the architecture
-without a ``deephead`` component can be formulated as:
-
-
-.. image:: figures/architecture_1_math.png
-   :width: 600px
-   :align: center
-
-
-Where *'W'* are the weight matrices applied to the wide model and to the final
-activations of the deep models, :math:`a` are these final activations, and
-:math:`{\phi}` (x) are the cross product transformations of the original
-features *'x'*. In case you are wondering what are *"cross product
-transformations"*, here is a quote taken directly from the paper: *"For binary
-features, a cross-product transformation (e.g., “AND(gender=female,
-language=en)”) is 1 if and only if the constituent features (“gender=female”
-and “language=en”) are all 1, and 0 otherwise".* Finally, :math:`{\sigma}` (.)
-is the activation function.
-
-While if there is a ``deephead`` component, the previous expression turns
-into:
-
-.. image:: figures/architecture_2_math.png
-   :width: 350px
-   :align: center
-
-
-It is important to emphasize that **each individual component, wide,
-deeptabular, deeptext and deepimage, can be used independently and in
-isolation**. For example, one could use only ``wide``, which is in simply a
-linear model. In fact, one of the most interesting functionalities in
-``pytorch-widedeep`` would be the use of the ``deeptabular`` component on its
-own, i.e. what one might normally refer as Deep Learning for Tabular Data.
-Currently, ``pytorch-widedeep`` offers the following different models for
-that component:
-
-0. **Wide**: a simple linear model where the nonlinearities are captured via
-cross-product transformations, as explained before.
-
-1. **TabMlp**: a simple MLP that receives embeddings representing the
-categorical features, concatenated with the continuous features, which can
-also be embedded.
-
-2. **TabResnet**: similar to the previous model but the embeddings are
-passed through a series of ResNet blocks built with dense layers.
-
-3. **TabNet**: details on TabNet can be found in `TabNet: Attentive
-Interpretable Tabular Learning <https://arxiv.org/abs/1908.07442>`_
-
-The ``Tabformer`` family, i.e. Transformers for Tabular data:
-
-4. **TabTransformer**: details on the TabTransformer can be found in
-`TabTransformer: Tabular Data Modeling Using Contextual Embeddings
-<https://arxiv.org/pdf/2012.06678.pdf>`_.
-
-5. **SAINT**: Details on SAINT can be found in `SAINT: Improved Neural
-Networks for Tabular Data via Row Attention and Contrastive Pre-Training
-<https://arxiv.org/abs/2106.01342>`_.
-
-6. **FT-Transformer**: details on the FT-Transformer can be found in
-`Revisiting Deep Learning Models for Tabular Data
-<https://arxiv.org/abs/2106.11959>`_.
-
-7. **TabFastFormer**: adaptation of the FastFormer for tabular data. Details
-on the Fasformer can be found in `FastFormers: Highly Efficient Transformer
-Models for Natural Language Understanding
-<https://arxiv.org/abs/2010.13382>`_
-
-8. **TabPerceiver**: adaptation of the Perceiver for tabular data. Details on
-the Perceiver can be found in `Perceiver: General Perception with Iterative
-Attention <https://arxiv.org/abs/2103.03206>`_
-
-And probabilistic DL models for tabular data based on
-`Weight Uncertainty in Neural Networks <https://arxiv.org/abs/1505.05424>`_:
-
-9. **BayesianWide**: Probabilistic adaptation of the `Wide` model.
-
-10. **BayesianTabMlp**: Probabilistic adaptation of the `TabMlp` model
-
-Note that while there are scientific publications for the TabTransformer,
-SAINT and FT-Transformer, the TabFasfFormer and TabPerceiver are our own
-adaptation of those algorithms for tabular data. For details on these models
-and their options please see the examples in the Examples folder and the
-documentation.
-
-Finally, it is perfectly possible to use custom models as long as the the
-custom models have an attribute called ``output_dim`` with the size of the
-last layer of activations, so that ``WideDeep`` can be constructed. Again,
-examples on how to use custom components can be found in the Examples
-folder.
-
-Indices and tables
-==================
-
-* :ref:`genindex`
-* :ref:`search`
diff --git a/docs/installation.rst b/docs/installation.rst
deleted file mode 100644
index c63af2fd..00000000
--- a/docs/installation.rst
+++ /dev/null
@@ -1,44 +0,0 @@
-Installation
-************
-This section explains how to install ``pytorch-widedeep``.
-
-For the latest stable release, execute:
-
-.. code::
-
-    pip install pytorch-widedeep
-
-For the bleeding-edge version, execute:
-
-.. code::
-
-	pip install git+https://github.com/jrzaurin/pytorch-widedeep.git
-
-
-For developer install
-
-.. code::
-
-	# Clone the repository
-	git clone https://github.com/jrzaurin/pytorch-widedeep
-	cd pytorch-widedeep
-
-	# Install in dev mode
-	pip install -e .
-
-Dependencies
-------------
-* pandas
-* numpy
-* scipy
-* scikit-learn
-* gensim
-* spacy
-* opencv-contrib-python
-* imutils
-* tqdm
-* torch
-* torchvision
-* einops
-* wrapt
-* torchmetrics
diff --git a/docs/losses.rst b/docs/losses.rst
deleted file mode 100644
index 6ce8e769..00000000
--- a/docs/losses.rst
+++ /dev/null
@@ -1,65 +0,0 @@
-Losses
-======
-
-``pytorch-widedeep`` accepts a number of losses and objectives that can be
-passed to the ``Trainer`` class via the parameter ``objective``
-(see ``pytorch-widedeep.training.Trainer``). For most cases the loss function
-that ``pytorch-widedeep`` will use internally is already implemented in
-Pytorch.
-
-In addition, ``pytorch-widedeep`` implements a series of  "custom" loss
-functions. These are described below for completion since, as mentioned
-before, they are used internally by the ``Trainer``. Of course, onen could
-always use them on their own and can be imported as:
-
-.. code-block:: python
-
-	from pytorch_widedeep.losses import FocalLoss
-
-.. note:: Losses in this module expect the predictions and ground truth to have the
-	same dimensions for regression and binary classification problems
-	:math:`(N_{samples}, 1)`. In the case of multiclass classification problems
-	the ground truth is expected to be a 1D tensor with the corresponding
-	classes. See Examples below
-
-.. autoclass:: pytorch_widedeep.losses.MSELoss
-	:members:
-
-.. autoclass:: pytorch_widedeep.losses.MSLELoss
-	:members:
-
-.. autoclass:: pytorch_widedeep.losses.RMSELoss
-	:members:
-
-.. autoclass:: pytorch_widedeep.losses.RMSLELoss
-	:members:
-
-.. autoclass:: pytorch_widedeep.losses.QuantileLoss
-	:members:
-
-.. autoclass:: pytorch_widedeep.losses.FocalLoss
-	:members:
-
-.. autoclass:: pytorch_widedeep.losses.BayesianSELoss
-	:members:
-
-.. autoclass:: pytorch_widedeep.losses.TweedieLoss
-	:members:
-
-.. autoclass:: pytorch_widedeep.losses.ZILNLoss
-	:members:
-
-.. autoclass:: pytorch_widedeep.losses.L1Loss
-	:members:
-
-.. autoclass:: pytorch_widedeep.losses.FocalR_L1Loss
-	:members:
-
-.. autoclass:: pytorch_widedeep.losses.FocalR_MSELoss
-	:members:
-
-.. autoclass:: pytorch_widedeep.losses.FocalR_RMSELoss
-	:members:
-
-.. autoclass:: pytorch_widedeep.losses.HuberLoss
-	:members:
diff --git a/docs/make.bat b/docs/make.bat
deleted file mode 100644
index 922152e9..00000000
--- a/docs/make.bat
+++ /dev/null
@@ -1,35 +0,0 @@
-@ECHO OFF
-
-pushd %~dp0
-
-REM Command file for Sphinx documentation
-
-if "%SPHINXBUILD%" == "" (
-	set SPHINXBUILD=sphinx-build
-)
-set SOURCEDIR=.
-set BUILDDIR=_build
-
-if "%1" == "" goto help
-
-%SPHINXBUILD% >NUL 2>NUL
-if errorlevel 9009 (
-	echo.
-	echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
-	echo.installed, then set the SPHINXBUILD environment variable to point
-	echo.to the full path of the 'sphinx-build' executable. Alternatively you
-	echo.may add the Sphinx directory to PATH.
-	echo.
-	echo.If you don't have Sphinx installed, grab it from
-	echo.http://sphinx-doc.org/
-	exit /b 1
-)
-
-%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
-goto end
-
-:help
-%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
-
-:end
-popd
diff --git a/docs/metrics.rst b/docs/metrics.rst
deleted file mode 100644
index 0d0e84c4..00000000
--- a/docs/metrics.rst
+++ /dev/null
@@ -1,51 +0,0 @@
-Metrics
-=======
-
-.. note:: Metrics in this module expect the predictions and ground truth to have the
-	same dimensions for regression and binary classification problems: :math:`(N_{samples}, 1)`.
-	In the case of multiclass classification problems the ground truth is expected to be
-	a 1D tensor with the corresponding classes. See Examples below
-
-We have added the possibility of using the metrics available at the
-`torchmetrics <https://torchmetrics.readthedocs.io/en/latest/>`_ library.
-Note that this library is still in its early versions and therefore this
-option should be used with caution. To use ``torchmetrics`` simply import
-them and use them as any of the ``pytorch-widedeep`` metrics described
-below.
-
-.. code-block:: python
-
-	from torchmetrics import Accuracy, Precision
-
-	accuracy = Accuracy(average=None, num_classes=2)
-	precision = Precision(average='micro', num_classes=2)
-
-	trainer = Trainer(model, objective="binary", metrics=[accuracy, precision])
-
-A functioning example for ``pytorch-widedeep`` using ``torchmetrics`` can be
-found in the `Examples folder <https://github.com/jrzaurin/pytorch-widedeep/blob/master/examples>`_.
-
-
-.. autoclass:: pytorch_widedeep.metrics.Accuracy
-	:members:
-	:undoc-members:
-
-.. autoclass:: pytorch_widedeep.metrics.Precision
-	:members:
-	:undoc-members:
-
-.. autoclass:: pytorch_widedeep.metrics.Recall
-	:members:
-	:undoc-members:
-
-.. autoclass:: pytorch_widedeep.metrics.FBetaScore
-	:members:
-	:undoc-members:
-
-.. autoclass:: pytorch_widedeep.metrics.F1Score
-	:members:
-	:undoc-members:
-
-.. autoclass:: pytorch_widedeep.metrics.R2Score
-	:members:
-	:undoc-members:
diff --git a/docs/model_components.rst b/docs/model_components.rst
deleted file mode 100644
index 00a3a1ba..00000000
--- a/docs/model_components.rst
+++ /dev/null
@@ -1,71 +0,0 @@
-The ``models`` module
-======================
-
-This module contains the models that can be used as the four main components
-that will comprise a Wide and Deep model (``wide``, ``deeptabular``,
-``deeptext``, ``deepimage``), as well as the ``WideDeep`` "constructor"
-class. Note that each of the four components can be used independently.
-
-.. autoclass:: pytorch_widedeep.models.tabular.linear.wide.Wide
-    :exclude-members: forward
-    :members:
-
-.. autoclass:: pytorch_widedeep.models.tabular.mlp.tab_mlp.TabMlp
-    :exclude-members: forward
-    :members:
-
-.. autoclass:: pytorch_widedeep.models.tabular.mlp.context_attention_mlp.ContextAttentionMLP
-    :exclude-members: forward
-    :members:
-
-.. autoclass:: pytorch_widedeep.models.tabular.mlp.self_attention_mlp.SelfAttentionMLP
-    :exclude-members: forward
-    :members:
-
-.. autoclass:: pytorch_widedeep.models.tabular.resnet.tab_resnet.TabResnet
-    :exclude-members: forward
-    :members:
-
-.. autoclass:: pytorch_widedeep.models.tabular.tabnet.tab_net.TabNet
-    :exclude-members: forward
-    :members:
-
-.. autoclass:: pytorch_widedeep.models.tabular.transformers.tab_transformer.TabTransformer
-    :exclude-members: forward
-    :members:
-
-.. autoclass:: pytorch_widedeep.models.tabular.transformers.saint.SAINT
-    :exclude-members: forward
-    :members:
-
-.. autoclass:: pytorch_widedeep.models.tabular.transformers.ft_transformer.FTTransformer
-    :exclude-members: forward
-    :members:
-
-.. autoclass:: pytorch_widedeep.models.tabular.transformers.tab_perceiver.TabPerceiver
-    :exclude-members: forward
-    :members:
-
-.. autoclass:: pytorch_widedeep.models.tabular.transformers.tab_fastformer.TabFastFormer
-    :exclude-members: forward
-    :members:
-
-.. autoclass:: pytorch_widedeep.models.text.attentive_rnn.BasicRNN
-    :exclude-members: forward
-    :members:
-
-.. autoclass:: pytorch_widedeep.models.text.attentive_rnn.AttentiveRNN
-    :exclude-members: forward
-    :members:
-
-.. autoclass:: pytorch_widedeep.models.text.stacked_attentive_rnn.StackedAttentiveRNN
-    :exclude-members: forward
-    :members:
-
-.. autoclass:: pytorch_widedeep.models.image.vision.Vision
-    :exclude-members: forward
-    :members:
-
-.. autoclass:: pytorch_widedeep.models.wide_deep.WideDeep
-    :exclude-members: forward
-    :members:
diff --git a/docs/preprocessing.rst b/docs/preprocessing.rst
deleted file mode 100644
index a068db8c..00000000
--- a/docs/preprocessing.rst
+++ /dev/null
@@ -1,22 +0,0 @@
-The ``preprocessing`` module
-============================
-
-This module contains the classes that are used to prepare the data before
-being passed to the models. There is one Preprocessor per data mode or
-model component: ``wide``, ``deeptabular``, ``deepimage`` and ``deeptext``.
-
-.. autoclass:: pytorch_widedeep.preprocessing.WidePreprocessor
-	:members:
-	:undoc-members:
-
-.. autoclass:: pytorch_widedeep.preprocessing.TabPreprocessor
-	:members:
-	:undoc-members:
-
-.. autoclass:: pytorch_widedeep.preprocessing.TextPreprocessor
-	:members:
-	:undoc-members:
-
-.. autoclass:: pytorch_widedeep.preprocessing.ImagePreprocessor
-	:members:
-	:undoc-members:
diff --git a/docs/quick_start.rst b/docs/quick_start.rst
deleted file mode 100644
index e21d618e..00000000
--- a/docs/quick_start.rst
+++ /dev/null
@@ -1,134 +0,0 @@
-Quick Start
-***********
-
-This is an example of a binary classification with the `adult census
-<https://www.kaggle.com/wenruliu/adult-income-dataset?select=adult.csv>`__
-dataset using a combination of a wide and deep model (in this case a so called
-``deeptabular`` model) with defaults settings.
-
-
-Read and split the dataset
---------------------------
-
-.. code-block:: python
-
-    import pandas as pd
-    import numpy as np
-    from sklearn.model_selection import train_test_split
-    from pytorch_widedeep.datasets import load_adult
-
-    df = load_adult(as_frame=True)
-    df["income_label"] = (df["income"].apply(lambda x: ">50K" in x)).astype(int)
-    df.drop("income", axis=1, inplace=True)
-    df_train, df_test = train_test_split(df, test_size=0.2, stratify=df.income_label)
-
-
-
-Prepare the wide and deep columns
----------------------------------
-
-.. code-block:: python
-
-    from pytorch_widedeep import Trainer
-    from pytorch_widedeep.preprocessing import WidePreprocessor, TabPreprocessor
-    from pytorch_widedeep.models import Wide, TabMlp, WideDeep
-    from pytorch_widedeep.metrics import Accuracy
-
-    # Define the 'column set up'
-    wide_cols = [
-        "education",
-        "relationship",
-        "workclass",
-        "occupation",
-        "native-country",
-        "gender",
-    ]
-    crossed_cols = [("education", "occupation"), ("native-country", "occupation")]
-
-    cat_embed_cols = [
-        "workclass",
-        "education",
-        "marital-status",
-        "occupation",
-        "relationship",
-        "race",
-        "gender",
-        "capital-gain",
-        "capital-loss",
-        "native-country",
-    ]
-    continuous_cols = ["age", "hours-per-week"]
-    target = "income_label"
-    target = df_train[target].values
-
-Preprocessing and model components definition
----------------------------------------------
-
-.. code-block:: python
-
-    wide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols)
-    X_wide = wide_preprocessor.fit_transform(df_train)
-
-    tab_preprocessor = TabPreprocessor(
-        cat_embed_cols=cat_embed_cols, continuous_cols=continuous_cols  # type: ignore[arg-type]
-    )
-    X_tab = tab_preprocessor.fit_transform(df_train)
-
-    # build the model
-    wide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1)
-    tab_mlp = TabMlp(
-        column_idx=tab_preprocessor.column_idx,
-        cat_embed_input=tab_preprocessor.cat_embed_input,
-        continuous_cols=continuous_cols,
-    )
-    model = WideDeep(wide=wide, deeptabular=tab_mlp)
-
-
-Fit and predict
--------------------------------
-
-.. code-block:: python
-
-    # train and validate
-    trainer = Trainer(model, objective="binary", metrics=[Accuracy])
-    trainer.fit(
-        X_wide=X_wide,
-        X_tab=X_tab,
-        target=target,
-        n_epochs=5,
-        batch_size=256,
-    )
-
-    # predict on test
-    X_wide_te = wide_preprocessor.transform(df_test)
-    X_tab_te = tab_preprocessor.transform(df_test)
-    preds = trainer.predict(X_wide=X_wide_te, X_tab=X_tab_te)
-
-
-Save and load
--------------------------------
-
-.. code-block:: python
-
-    # Option 1: this will also save training history and lr history if the
-    # LRHistory callback is used
-    trainer.save(path="model_weights", save_state_dict=True)
-
-    # Option 2: save as any other torch model
-    torch.save(model.state_dict(), "model_weights/wd_model.pt")
-
-    # From here in advance, Option 1 or 2 are the same. I assume the user has
-    # prepared the data and defined the new model components:
-    # 1. Build the model
-    model_new = WideDeep(wide=wide, deeptabular=tab_mlp)
-    model_new.load_state_dict(torch.load("model_weights/wd_model.pt"))
-
-    # 2. Instantiate the trainer
-    trainer_new = Trainer(model_new, objective="binary")
-
-    # 3. Either start the fit or directly predict
-    preds = trainer_new.predict(X_wide=X_wide, X_tab=X_tab)
-
-Of course, one can do **much more**. See the Examples folder in the repo, this
-documentation or the companion posts for a better understanding of the content
-of the package and its functionalities.
diff --git a/docs/requirements.txt b/docs/requirements.txt
deleted file mode 100644
index 03877a58..00000000
--- a/docs/requirements.txt
+++ /dev/null
@@ -1,20 +0,0 @@
-sphinx
-sphinx_rtd_theme
-recommonmark
-sphinx-markdown-tables
-sphinx-copybutton
-sphinx-autodoc-typehints
-pandas
-numpy
-scipy
-scikit-learn
-gensim
-spacy
-opencv-contrib-python
-imutils
-tqdm
-torch
-torchvision
-einops
-wrapt
-torchmetrics
diff --git a/docs/tab2vec.rst b/docs/tab2vec.rst
deleted file mode 100644
index 8daf940c..00000000
--- a/docs/tab2vec.rst
+++ /dev/null
@@ -1,7 +0,0 @@
-Tab2Vec
-=======
-
-.. autoclass:: pytorch_widedeep.tab2vec.Tab2Vec
-	:members:
-	:undoc-members:
-
diff --git a/docs/trainer.rst b/docs/trainer.rst
deleted file mode 100644
index 808e2882..00000000
--- a/docs/trainer.rst
+++ /dev/null
@@ -1,23 +0,0 @@
-Training multimodal Deep Learning Models
-========================================
-
-Here is the documentation for the ``Trainer`` class, that will do all the heavy lifting.
-
-Trainer is also available from ``pytorch-widedeep`` directly, for example, one could do:
-
-.. code-block:: python
-
-	from pytorch-widedeep.training import Trainer
-
-
-or also:
-
-.. code-block:: python
-
-	from pytorch-widedeep import Trainer
-
-
-.. autoclass:: pytorch_widedeep.training.Trainer
-	:exclude-members: forward
-	:members:
-	:undoc-members:
diff --git a/docs/utils/deeptabular_utils.rst b/docs/utils/deeptabular_utils.rst
deleted file mode 100644
index 25f8ba4e..00000000
--- a/docs/utils/deeptabular_utils.rst
+++ /dev/null
@@ -1,6 +0,0 @@
-deeptabular utils
-=================
-
-.. autoclass:: pytorch_widedeep.utils.deeptabular_utils.LabelEncoder
-	:members:
-	:undoc-members:
diff --git a/docs/utils/fastai_transforms.rst b/docs/utils/fastai_transforms.rst
deleted file mode 100644
index f18d60b0..00000000
--- a/docs/utils/fastai_transforms.rst
+++ /dev/null
@@ -1,19 +0,0 @@
-Fastai transforms
-=================
-
-I have directly copied and pasted part of the ``transforms.py`` module from
-the ``fastai`` library. The reason to do such a thing is because
-``pytorch_widedeep`` only needs the ``Tokenizer`` and the ``Vocab`` classes
-there. This way I avoid extra dependencies. Credit for all the code in the
-``fastai_transforms`` module in this ``pytorch-widedeep`` package goes to
-Jeremy Howard and the `fastai` team. I only include the documentation here for
-completion, but I strongly advise the user to read the ``fastai``
-`documentation <https://docs.fast.ai/>`_.
-
-.. autoclass:: pytorch_widedeep.utils.fastai_transforms.Tokenizer
-	:members:
-	:undoc-members:
-
-.. autoclass:: pytorch_widedeep.utils.fastai_transforms.Vocab
-	:members:
-	:undoc-members:
diff --git a/docs/utils/image_utils.rst b/docs/utils/image_utils.rst
deleted file mode 100644
index aedb1f61..00000000
--- a/docs/utils/image_utils.rst
+++ /dev/null
@@ -1,20 +0,0 @@
-Image utils
-===========
-:class:`SimplePreprocessor
-<pytorch_widedeep.utils.image_utils.SimplePreprocessor>` and
-:class:`AspectAwarePreprocessor
-<pytorch_widedeep.utils.image_utils.AspectAwarePreprocessor>` are directly
-taked from the great series of Books `Deep Learning for Computer Vision
-<https://www.pyimagesearch.com/deep-learning-computer-vision-python-book/>`_
-by `Adrian <https://www.pyimagesearch.com/>`_. Therefore, all credit for the
-code in the ``image_utils`` module goes to `Adrian Rosebrock
-<https://www.pyimagesearch.com/>`_.
-
-.. autoclass:: pytorch_widedeep.utils.image_utils.AspectAwarePreprocessor
-	:members:
-	:undoc-members:
-
-.. autoclass:: pytorch_widedeep.utils.image_utils.SimplePreprocessor
-	:members:
-	:undoc-members:
-
diff --git a/docs/utils/index.rst b/docs/utils/index.rst
deleted file mode 100644
index 72df9e03..00000000
--- a/docs/utils/index.rst
+++ /dev/null
@@ -1,23 +0,0 @@
-The ``utils`` module
-====================
-
-These are a series of utilities that might be useful for a number of
-preprocessing tasks, even not directly related to ``pytorch-widedeep``. All
-the classes and functions discussed here are available directly from the
-``utils`` module. For example, the ``LabelEncoder`` within the
-``deeptabular_utils`` submodule can be imported as:
-
-.. code-block:: python
-
-	from pytorch_widedeep.utils import LabelEncoder
-
-
-Objects
--------
-
-.. toctree::
-
-    deeptabular_utils
-    fastai_transforms
-    image_utils
-    text_utils
\ No newline at end of file
diff --git a/docs/utils/text_utils.rst b/docs/utils/text_utils.rst
deleted file mode 100644
index 1ad57478..00000000
--- a/docs/utils/text_utils.rst
+++ /dev/null
@@ -1,15 +0,0 @@
-Text utils
-=================
-Collection of helper function that facilitate processing text.
-
-.. autofunction:: pytorch_widedeep.utils.text_utils.simple_preprocess
-	:noindex:
-
-.. autofunction:: pytorch_widedeep.utils.text_utils.get_texts
-	:noindex:
-
-.. autofunction:: pytorch_widedeep.utils.text_utils.pad_sequences
-	:noindex:
-
-.. autofunction:: pytorch_widedeep.utils.text_utils.build_embeddings_matrix
-	:noindex:
\ No newline at end of file

From cb6c384772030c626c314104ff8cf86eaf21bd19 Mon Sep 17 00:00:00 2001
From: Javier <jrzaurin@gmail.com>
Date: Mon, 21 Oct 2024 10:39:19 +0100
Subject: [PATCH 14/24] temporay restore docs dir

---
 docs/Makefile                           |  21 ++
 docs/_static/custom.css                 |  45 ++++
 docs/_static/img/widedeep_logo_docs.ico | Bin 0 -> 102115 bytes
 docs/bayesian_models.rst                |  15 ++
 docs/callbacks.rst                      |  29 +++
 docs/conf.py                            | 297 ++++++++++++++++++++++++
 docs/dataloaders.rst                    |  10 +
 docs/examples.rst                       |  20 ++
 docs/figures/arch_1.png                 | Bin 0 -> 58023 bytes
 docs/figures/arch_2.png                 | Bin 0 -> 51944 bytes
 docs/figures/arch_3.png                 | Bin 0 -> 61499 bytes
 docs/figures/arch_4.png                 | Bin 0 -> 74000 bytes
 docs/figures/arch_5.png                 | Bin 0 -> 76885 bytes
 docs/figures/arch_6.png                 | Bin 0 -> 123034 bytes
 docs/figures/arch_7.png                 | Bin 0 -> 57736 bytes
 docs/figures/arch_8.png                 | Bin 0 -> 33393 bytes
 docs/figures/architecture_1_math.png    | Bin 0 -> 47788 bytes
 docs/figures/architecture_2_math.png    | Bin 0 -> 16848 bytes
 docs/figures/widedeep_arch.png          | Bin 0 -> 157281 bytes
 docs/figures/widedeep_arch_new.png      | Bin 0 -> 96559 bytes
 docs/figures/widedeep_logo.png          | Bin 0 -> 85253 bytes
 docs/index.rst                          | 160 +++++++++++++
 docs/installation.rst                   |  44 ++++
 docs/losses.rst                         |  65 ++++++
 docs/make.bat                           |  35 +++
 docs/metrics.rst                        |  51 ++++
 docs/model_components.rst               |  71 ++++++
 docs/preprocessing.rst                  |  22 ++
 docs/quick_start.rst                    | 134 +++++++++++
 docs/requirements.txt                   |  20 ++
 docs/tab2vec.rst                        |   7 +
 docs/trainer.rst                        |  23 ++
 docs/utils/deeptabular_utils.rst        |   6 +
 docs/utils/fastai_transforms.rst        |  19 ++
 docs/utils/image_utils.rst              |  20 ++
 docs/utils/index.rst                    |  23 ++
 docs/utils/text_utils.rst               |  15 ++
 37 files changed, 1152 insertions(+)
 create mode 100644 docs/Makefile
 create mode 100644 docs/_static/custom.css
 create mode 100644 docs/_static/img/widedeep_logo_docs.ico
 create mode 100644 docs/bayesian_models.rst
 create mode 100644 docs/callbacks.rst
 create mode 100644 docs/conf.py
 create mode 100644 docs/dataloaders.rst
 create mode 100644 docs/examples.rst
 create mode 100644 docs/figures/arch_1.png
 create mode 100644 docs/figures/arch_2.png
 create mode 100644 docs/figures/arch_3.png
 create mode 100644 docs/figures/arch_4.png
 create mode 100644 docs/figures/arch_5.png
 create mode 100644 docs/figures/arch_6.png
 create mode 100644 docs/figures/arch_7.png
 create mode 100644 docs/figures/arch_8.png
 create mode 100644 docs/figures/architecture_1_math.png
 create mode 100644 docs/figures/architecture_2_math.png
 create mode 100644 docs/figures/widedeep_arch.png
 create mode 100644 docs/figures/widedeep_arch_new.png
 create mode 100644 docs/figures/widedeep_logo.png
 create mode 100644 docs/index.rst
 create mode 100644 docs/installation.rst
 create mode 100644 docs/losses.rst
 create mode 100644 docs/make.bat
 create mode 100644 docs/metrics.rst
 create mode 100644 docs/model_components.rst
 create mode 100644 docs/preprocessing.rst
 create mode 100644 docs/quick_start.rst
 create mode 100644 docs/requirements.txt
 create mode 100644 docs/tab2vec.rst
 create mode 100644 docs/trainer.rst
 create mode 100644 docs/utils/deeptabular_utils.rst
 create mode 100644 docs/utils/fastai_transforms.rst
 create mode 100644 docs/utils/image_utils.rst
 create mode 100644 docs/utils/index.rst
 create mode 100644 docs/utils/text_utils.rst

diff --git a/docs/Makefile b/docs/Makefile
new file mode 100644
index 00000000..c1d05ae8
--- /dev/null
+++ b/docs/Makefile
@@ -0,0 +1,21 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line, and also
+# from the environment for the first two.
+SPHINXOPTS    ?=
+SPHINXPROJ    = pytorch_widedeep
+SPHINXBUILD   ?= sphinx-build
+SOURCEDIR     = .
+BUILDDIR      = _build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
diff --git a/docs/_static/custom.css b/docs/_static/custom.css
new file mode 100644
index 00000000..cbe8279e
--- /dev/null
+++ b/docs/_static/custom.css
@@ -0,0 +1,45 @@
+body a {
+  font-weight: bold;
+}
+
+.math {
+    text-align: left;
+}
+
+.eqno {
+    float: right;
+}
+
+div.ethical-rtd {
+  /* hide ads */
+  display: none;
+}
+
+.rst-content dl:not(.docutils) dl dt strong {
+  padding-left: 6pt;
+}
+
+.rst-content dl:not(.docutils) dl dt span {
+  color: #777;
+}
+
+.rst-content dl:not(.docutils) dl dt span.classifier-delimiter {
+  padding-left: 6pt;
+  padding-right: 6pt;
+}
+
+.rst-content dl:not(.docutils) dl dt span.classifier {
+  padding-right: 6pt;
+}
+
+.rst-content dl:not(.docutils) dl dt a.reference.internal span.xref.std.std-term {
+  color: #2980B9;
+}
+
+.wy-nav-content {
+    max-width: none; !important;
+}
+
+div.container a.header-logo {
+  background-image: url("../figures/widedeep_logo.png");
+}
diff --git a/docs/_static/img/widedeep_logo_docs.ico b/docs/_static/img/widedeep_logo_docs.ico
new file mode 100644
index 0000000000000000000000000000000000000000..5e699b7ea222f5e43e05c969cde6c92a4b110b4d
GIT binary patch
literal 102115
zcmeHQ30zHS8$VZ>Qqon{LRnIZ2$ikP7G^9%#Au<0GE@jjQTXgF-+WoJkNJi%W7Ib?
zBcpCn2~iAVNh(YeSwhB@(*2&3t~uTJw%u;&dY|+AJ)Lu&cYXf<=UvWw-_Bte9`gyq
z9y~^$S=xeO=E8Mw@Vk2>RUUI2bhfq?_rn=RNsGty>-X;dumQuEI6zAMihDl8T=nNM
zd^ofDc`7qZz-At^Y0?CH4K*D#$Xa9Un30p&P*zmgqMa5WX3OOHES})N%lOuB7sIsZ
zJ9ea<!}820cjkvUc<|0X7g~AD^;ovUIVx?bI<MK7UOUzd*KV=3-IdWT4~2SpxtToI
z(YgHNc=O1fp$|8>jrn=n*q{E>p_hx6`_+L{b&b~j{`>kbN`u?*4rHyLqm*{$Y5K*T
zY5s46%^YU=HDB)+nJ+3C6L|{?C>!b_yclH$m61HW)vgN}hiJ`dK_*v<U%i^$^Xom^
zhL%P@-1OJ#J{GE@JGB2k<Y(taT`z3)z4dDS^N3~bhGulqITe_4a*>JPeqiBfQEMhJ
z*~T$?pmTsthdVJrg-LlvZ}yn9v3`1Qv9s%rnA!J^Ypgx7#^H_M_$fXC^Af+CSu|yz
zf4cCJot`QaVs>^={>9zCJty?>^RseEpQC4yzRoeg*+MmQLF95%v*qXBG%wUEc)9ZS
z!a2&!+>$*lE*&h&7wF{|9SQLbz42u8Q7bp?^M4!_W+%+Mkm<V6bJ9B1;Mk)PGv}UN
z=egEoo44gQ|6yCb_+{3g9h{c!pSxs3;q5Kq-I#B)XAH~uYk>WZt-qy<t`D+|y{Fet
zb(vv-?o-#;=(peGeeP)f+Ceq*<ihRePt190;t{ZFrL*e`VM^u*yCLz9qm(>ey%uOL
zw95@(yyFhXd!5$^dZFtc;;Ze{CO*z<#iQjpLY`@E+u%s3$)Rn&w=6?^lj1^E*CcGX
zbji;&Gh%RlN!g@S6Q-+A`tED|_TRRTvn~!b>lx;ea_jLfKkK3U``Yen&2&u};4$sw
z<cF<th5}A))m%fZhaFEIylTh;Z65#U`w=HXH><DHi!2!YSbdz8)%<Liele$o{5i_D
zwk6kXBG3GAQTSOW%aZKHW{$?`MtciO*A#ZXpc)*kdE+PD$D32~1b$8vPJSPdC-@`$
zU|`{-QH=7RVV#TPOm>(A9m|-y@xPwaQp1kMtYkyJ%K9N8&MmFDG#(Ot?K}JV#fNhQ
z9*4Uz=1<RM+-PNCD!3yI%T7Lc?|XHR+eh+tM@&|=4NdE8m8E5Wr9;lp725g7`s^H^
z@PjaZL~GvRMYD~qvV`lc2j5ukcJ-44L&Hswk?B`yAur+&^fe9Bjj?Z~&1ganzIng%
zb9CQyop&^kpQzMqQi+y8<)umv@8w$Nn!r0p>CJ)1drV$;Ik1tr7W?Pr*@i299z^`&
zY_~G0SI&Sot%G%Tcz)$w6o0_y*Hd{<Gme+Ilt+x6^x%J+zDyT}xy<nWGBNH{k-)<U
z%8&Z{K=koj`Zpg1*!!#*>6ZFBZ)&m2!Cm%X`)ePY&s#ovBr0Homb^V8umsDS{U^zO
zLI1N3dC}QF+b!&8w(xR^z>n1zq=wvS|J~K-;_O9(?Tq8iif&G^ObR!iua)n*@ur3Q
z<8R$kAND!4**f1sHT}@~@IBU|__c<CKA)$UFs<E#UR=|)*cq524BOKDhnGeFO(?w-
z0<8V`*AbqvPszoZJ=+Z#vh2>kZJJ+K(Os{ydv3;FE8)bMzd`cROmdv{yazk(IeD*r
zUeaZ?n{(R6iG6OaT)w-oWJKo;jN6%+Jr^9G?Y8#>@A~$L5$mn3@7vqtop)`n8XUPU
zx^VGBGxt@V5$9dApJ{q~b+TwR{l=hq7cxHS<6KY@6?~U(dp*ow?^F9HhWnSM+_StG
zy1di$nd1hUM~@HpUil*I(y8V7uG_%j4;L3cO$$F&xFtMu<cqQHUUM>%G}k!Y$vqI>
zYL{h@`NXn<GbT*r(J5a%-CR`sca+n<!1(B)hmYLe`&DP*FLsYC-+E;ppC|A!E>qzJ
zFG}CJcgtMOjr~&-bnUPF_~`lk<OTixdoNh9V(#l>DlV~qKl~-|kLbzDjA-eDV!hp0
zgI4*EwDw&v>g1_M2NqlJ@^s27Ox>uF<TlAs=;CHu6kRYn?9Kfw$DD%W+c!)%zaO(U
zEwdzf^P$J9-}=5VoS5alr)1={?HxVt4H|g%3->*X(o@c!yxB9tKFMLl#LJgsZ3cUu
z+kQpU`dCob4ae<)h4vyo^YZ*)kI#f-Z@<d?^~)P&LCPsn6RutxcI?`Kr7Qdkj?I}`
zY&zm)hJnXgFK;i^5VNUE(+9<UG3Th2r)k{Y*c<M}myT*IJsY{hI5saOB{in6%ae7M
zCoEc;on7!#u0W-|UdFzJ^((_WxOX|6D>%3%+~fWh(cN+1ug#uvx^?WymZdgjKAy%U
zH~ob=efLK_6Z~oJvTEEqq4Sy|hMBF*@JGw#wCtd(`l>`>KZnqBFi4~u@Lp%=wTgWj
zWYEs`@4I|?l_H<IVlHwby&-_y{>+LoH}DDR=yLi|hRglD+s1K)kIh}m=DA;tX`9vF
z@!4<ev`O-?9aXCP&F$txH3K8ZvteULO&A$9eAXHxxDI9&kcgd(y*qe}K|7{gDinA^
z1P}p401-e05CKF05kLeG0Ym^1Km-th3Ix<T#We%jTnF)}4m`+5r4D(q<&LW%zaDkW
z4;C=+s4W73tulzH4w6t^-jI(*9r9w!-Bd$<BkGvn>gw{tu*wJkw#a{FNT|9){;8sy
zFc$eIAfZZu{8L3YVJz}bKth!Q`KO9*!dT>=fP^Xq@=q1rgt5p!0SQ$K<ew_K31g9e
z0urhe$Ujwd6UHL{1SC``kbkP^CX7Y?2}r0?Apca+O&E*(6Od4)K>n$sn=lsnCm^9p
zf&5cNH(@ODPe4MI0{N$kZo*jPpMZoa1@cc7-Gs5oKLH6<3gn+Ex(Q>Ee*zM!6v#hS
zbQ8uR{{$pdDUg4v=q8Ls{s~B^QXv0S(M=eO{1cE+r9l3vqMI-l`6nQuN`d@SMK@tA
z@=riQl>+&vif+PK<ez|qDh2XS72Slf$UgxIRSM*vD!K_{k$(absuajSRdf@^BL4&=
zR4I^us^}(+Mg9p$s8S&RRMAZsi~JLiP^CcrsiK=O7WpS2p-O@LQ$;snEb>o4LX`sf
zr;2XESmd98genE{PZiySvB*CG2~`T@pDMZuW08LX5~>u)KUH)S#v=a&BvdJof2!yv
zj79zlNT^aE|5VXU7>oQ9kWi&S{;8syFc$eIAfZZu{8L3YVJz}bKth!Q`KO9*!dT>=
zfP^Xq@=q1rgt5p!0SQ$K<ew_K31g9e0urhe$Ujwd6UHL{1SC``Xy6}gv;^@&iBp$!
zZc6xH4R(r9;&hOIs{H`@r@J3O{*nJmLjaEdRE-DN{;~aI`={3ju>I4^J^TEN{3HL^
z|I_OO*#7C|p634F8^*p&l(=@4I)AvSc>b>oHs49+!M%t4)a!`zQf_+5YshPG9rN>+
z%meQk=F{ov+D3ztC-b`r7V*2qFKO8&exY^`!Bj(w<em|`?(z_$j}HNnNW^sLmC#+Q
zyWjxu{R-L<@f(<W=eRP>uJISz_Dpc}4Y=Oy<3oW?i#(TPF&cXDUcj$Z-lh41rhdFY
zzuy^Rjz}khifeVRAt}v(`!H!7*QA4VxgCuY4HVaUtT}M#UlY&Og+uyV-TEi-5lO{E
z0LFR?;I^bXerpp3W5KSFt=WnP0L3+)=#;^0r57)#4Zk(TQ3oGtZ<J_AF`No#I?UaA
z*TidG!eJcPq;McmTw$#qf_jbnlD+}Y4!`f;pF=UCie)-H$M2Qous-*||I6+J&ZsFC
z3iQ?%xQ?k0-%`?+fe&<|7g5Es9Oi7%QaB{m{OS;g<zH8^P@uQAt@Ps~>%eV|e!=$t
zBPU<P{$D{{s*ORye0lN@zZLyaQ20hctjB8Vo1N(lZMNQPKVl!!Hb_{5s45NuKPD6~
zu>NyM%wK)ZY~PQ+1t|{q;+n!3pby+Vs}J{VT3G+}V}C!!L-7y*?i_)`QdxbVWtX@Z
zo!$w`if1|2m_?wUO)3L(w=aSJI?n}Kc8)u)Z6F~2PaKg|JOsc8MnGH6u4O+Uggrt*
zZ445dE1soT<Ejwo*8hwe{4U@HYXccDFDR4r1>pQI2>Tj-H||zd9;jD*1bhN+@XSZ1
zcF^s8+P3?Eq{%S1x9?(l+OYSq6ddOjpI)>85kLeG0Ym^1Km-s0L;w+>9syXt{xC<d
z(rd;hSm}M?c#n9XKVW+hcL*P8e~?%);;2;Er|9=$x^nfzSMSb@N~QZU<{!^y14<cQ
zrSMXQsdWFEVS>1dXEop3NW_=-eUL={mF^|$D`}6Uy&?%bM8TTe>_wtno67eMb!P<r
zXbTby!ejK|FC4ZEV*qbKU<D6E01;?<2*7VtJdjaRWEi)$5Z+8m95$V~M7$PN#2d(s
z*Ho@6?2@)Wx6(i3K)g>tq{U0-+5&O9RuM0e8?UKcLtR;3rP+n{b4UV=kbmiZg8pCG
zPq6(<_me~pOiktz`#<deas0#aUy2Xl{D;#^<NOEbe^S~A``m!@zsl_d=YKf=llDBF
z$Z?Y<a|!JipZ}%%D~|shFU9d6$A29Er9LO&{HJn1f%6~f<4Gb1rY3WV<3G-Ss_55o
z{=;ddIRC-<PZjM1$N$Rh1jm0I|8f0;^npZ9GifrHIRC@>UlrpT&i^>ARG#xc$g3BK
z<p&le);y>~qp$x$nTa6M#)!t$X`ubcKl(qm|0?>T1{?pNykj6wKCrap^ncRolEH?H
z52!1<|5aMNU&V3QGsMxWBHl4>ybpB^-{@=6y1QULyoc$3@Q$XF@ZF7la6bmV$M>PC
z)M(|x`$~I&%{PG`Vw?pWqTpL(EgG#dja(4Cb4vs4&XQ)kSf|k?{+o~WJ50d+3bCEl
zoZ0VF_HJZq8l^aR_gYFdcE4{A&_0$lN`)G^5U^cpyO)AVXd_pxQHlfpYxT?l?{ls8
z8_A7Q*#;;C`gb2GT)+4Gim=kQh6x|#cgWyf&8pD0uT+NtyuWh`{C{i%px-pIgm;v7
z1PoH8VDJ|7Vel^6@-=~`QQL;81-zqE32>MQ^S^B{)*XT||6kDN`!q~xm@6WH2yhXo
z?)?MLI;oVu%>mDK#~~O%<^z6pJ+pjNyh5X_;?)_Y6|c;w>sg#$RQVMhl^JK>6VVT3
z7RdYeMD)P-L;xJr9xxwapUnqKh^iP6&>KmFb>W0*NrbWaH0nGC%CiHJoJ+H=UVcDW
zvP5_UCv1j97@JR{eh(bV!}?d|5Ahc}Xf4j@KSbL{|EY!Fh@+tYp#O+{L6$T65BiTR
zej`qU{)7G__61qa=s)N`viOZS4f+rIkJuMvIivre|H$Gu;xy<#=s#j#kmcME{sVa`
zfw24J8&b;OOx-~B=R>}dYr^ur-^B8T^(Sy<u}A!C_P>Vo9~IzWaRZ7k?Bno;{TrXc
z{*BL|oFx)vX^HzX(6is@=GJ96Cyf1Vi5tdVtBAuJ;tYE++rWOC-=G}vH)6|R?@<Em
zL)K}C^5mH(>=9Oix}S!+6MY%z&%-xQo5{00dFRFN8gIqHov;D?$wuB~$}>N}c(yd`
zIXVw{mM8DLpskMK)Rn*=_GC?zcbW3cuVNpe)V;L8O}36n)89i6dqGX0t~nAs<iH-;
zzVgJXA@hQLs%o%@ZwB~F0N68w-&LU2kmX_Cjf=p$T|?rp`IMh?XC=~e2DJ5AduwrK
z-r-UP+pbuAN?|)!ZO-EOBKX2mIj#WtKWBFHiav-f$oqX-@T>O6`aK`$3tNM9uMkZz
zn|-d*;^@TxFWg2_$HuMi0eP+l5s!xp+5F&sEl1~8q5C&SH;dKD<A6M|e8eC9@0|}|
z`-4se`ycFoIb#iuKkwQEj=%qmle6S8_h9pob^ZbA)IfCWDZF}=KMT@|$4yz!?7J0Y
zntwsM4_&?!)ZtY<g|$DBlgEMzpIF7wxm4)3aCA<rPSyj~VOzkuYXq!?WP+T8^#oH{
z^O7YUl(`dZ;rdKItY@gpl9w!LpzM5!@<mW5TUqjwB@L97CQ*JFtm!w}{~rVQ!=U`<
z;yOTmHk3aE%VLa-X<By^w1G9ziLi#Hkndf1w@!10FA~X57^WGlH2fo?GWe)*87xz9
zMWk0=G`mg#k;~UA-mg<A%eGDde!%`8QcDnJ5OzI){SCSmNcp>RYN8n8C4!s)u>)b-
zwHe4ckm#Dyvf&U9(|@G+0k`aTaamugO4#2^tK=WTqCvQ0h2%B6R#aX3sSqAeg{*|W
zZx_gR2%iq3R}rrRoM(Ydt<HZT9UF(OPcd9;R~J`W7^E2vX-h!vLjNz*IzEh>9ib1c
Ya_kfZ!`fyogj>HdJ%(XLWWzB32bp(QjQ{`u

literal 0
HcmV?d00001

diff --git a/docs/bayesian_models.rst b/docs/bayesian_models.rst
new file mode 100644
index 00000000..2eb95c60
--- /dev/null
+++ b/docs/bayesian_models.rst
@@ -0,0 +1,15 @@
+The ``bayesian models`` module
+==============================
+
+This module contains the two Bayesian Models available in this library, namely
+the bayesian version of the ``Wide`` and ``TabMlp`` models, referred as
+``BayesianWide`` and ``BayesianTabMlp``
+
+
+.. autoclass:: pytorch_widedeep.bayesian_models.tabular.bayesian_linear.bayesian_wide.BayesianWide
+    :exclude-members: forward
+    :members:
+
+.. autoclass:: pytorch_widedeep.bayesian_models.tabular.bayesian_mlp.bayesian_tab_mlp.BayesianTabMlp
+    :exclude-members: forward
+    :members:
diff --git a/docs/callbacks.rst b/docs/callbacks.rst
new file mode 100644
index 00000000..e8e65c45
--- /dev/null
+++ b/docs/callbacks.rst
@@ -0,0 +1,29 @@
+Callbacks
+=========
+
+Here are the 5 callbacks available in ``pytorch-widedepp``: ``History``,
+``LRHistory``, ``ModelCheckpoint``, ``EarlyStopping`` and ``RayTuneReporter``.
+
+.. note:: ``History`` runs by default, so it should not be passed
+    to the ``Trainer``
+
+.. autoclass:: pytorch_widedeep.callbacks.History
+	:members:
+
+.. autoclass:: pytorch_widedeep.callbacks.LRShedulerCallback
+	:members:
+
+.. autoclass:: pytorch_widedeep.callbacks.MetricCallback
+	:members:
+
+.. autoclass:: pytorch_widedeep.callbacks.LRHistory
+	:members:
+
+.. autoclass:: pytorch_widedeep.callbacks.ModelCheckpoint
+	:members:
+
+.. autoclass:: pytorch_widedeep.callbacks.EarlyStopping
+	:members:
+
+.. autoclass:: pytorch_widedeep.callbacks.RayTuneReporter
+	:members:
diff --git a/docs/conf.py b/docs/conf.py
new file mode 100644
index 00000000..bcfaa096
--- /dev/null
+++ b/docs/conf.py
@@ -0,0 +1,297 @@
+# -*- coding: utf-8 -*-
+#
+# Configuration file for the Sphinx documentation builder.
+#
+# This file only contains a selection of the most common options. For a full
+# list see the documentation:
+# https://www.sphinx-doc.org/en/master/usage/configuration.html
+
+# -- Path setup --------------------------------------------------------------
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#
+
+import os
+import re
+import sys
+
+from sphinx.ext.napoleon.docstring import GoogleDocstring
+
+# this adds the equivalent of "../../" to the python path
+PACKAGEDIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+sys.path.insert(0, PACKAGEDIR)
+
+
+# -- Project information -----------------------------------------------------
+
+project = "pytorch-widedeep"
+copyright = "2021, Javier Rodriguez Zaurin"
+author = "Javier Rodriguez Zaurin"
+
+# # The full version, including alpha/beta/rc tags
+# def get_version():
+#     r"""
+
+#     Get the current version number for the library
+#     Returns
+#     -------
+#     String
+#         Of the form "<major>.<minor>.<micro>", in which "major", "minor" and "micro" are numbers
+
+#   """
+#     with open("../pytorch_widedeep/VERSION") as f:
+#         return f.read().strip()
+# release = get_version()
+
+with open(os.path.join(PACKAGEDIR, "pytorch_widedeep", "version.py")) as f:
+    version = re.search(r"__version__ \= \"(\d+\.\d+\.\d+)\"", f.read())
+    assert version is not None, "can't parse __version__ from __init__.py"
+    version = version.groups()[0]  # type: ignore[assignment]
+    assert len(version.split(".")) == 3, "bad version spec"  # type: ignore[attr-defined]
+    release = version
+
+# -- General configuration ---------------------------------------------------
+
+# If your documentation needs a minimal Sphinx version, state it here.
+#
+# needs_sphinx = '1.0'
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+
+extensions = [
+    "sphinx.ext.autosummary",
+    "sphinx.ext.autodoc",
+    "sphinx_autodoc_typehints",
+    "sphinx.ext.coverage",
+    "sphinx.ext.napoleon",
+    "sphinx.ext.mathjax",
+    "recommonmark",
+    "sphinx.ext.doctest",
+    "sphinx.ext.todo",
+    "sphinx.ext.viewcode",
+    "sphinx_markdown_tables",
+    "sphinx_copybutton",
+    "sphinx.ext.githubpages",
+]
+
+autosummary_generate = True
+
+napoleon_use_ivar = True
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ["_templates"]
+
+# The suffix(es) of source filenames.
+# You can specify multiple suffix as a list of string:
+#
+source_suffix = [".rst", ".md"]
+# source_suffix = '.rst'
+
+# The master toctree document.
+master_doc = "index"
+
+# The language for content autogenerated by Sphinx. Refer to documentation
+# for a list of supported languages.
+#
+# This is also used if you do content translation via gettext catalogs.
+# Usually you set "language" from the command line for these cases.
+language = None
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This pattern also affects html_static_path and html_extra_path.
+exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"]
+
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = "sphinx"
+
+# Remove the prompt when copying examples
+copybutton_prompt_text = ">>> "
+
+# autoclass_content = "init"  # 'both'
+autodoc_member_order = "bysource"
+# autodoc_default_flags = ["show-inheritance"]
+
+# -- Options for HTML output -------------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+#
+html_theme = "sphinx_rtd_theme"
+
+# Theme options are theme-specific and customize the look and feel of a theme
+# further.  For a list of options available for each theme, see the
+# documentation.
+#
+# html_theme_options = {"analytics_id": "UA-83738774-2"}
+
+# html_theme_options = {
+#     "canonical_url": "",
+#     # 'analytics_id': 'UA-XXXXXXX-1',  #  Provided by Google in your dashboard
+#     "logo_only": False,
+#     "display_version": True,
+#     "prev_next_buttons_location": "bottom",
+#     "style_external_links": False,
+#     # Toc options
+#     "collapse_navigation": True,
+#     "sticky_navigation": True,
+#     "navigation_depth": 4,
+#     "includehidden": True,
+#     "titles_only": False,
+# }
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ["_static"]
+
+# Custom sidebar templates, must be a dictionary that maps document names
+# to template names.
+#
+# The default sidebars (for documents that don't match any pattern) are
+# defined by theme itself.  Builtin themes are using these templates by
+# default: ``['localtoc.html', 'relations.html', 'sourcelink.html',
+# 'searchbox.html']``.
+#
+# html_sidebars = {}
+
+# This must be the name of an image file (path relative to the configuration
+# directory) that is the favicon of the docs. Modern browsers use this as
+# the icon for tabs, windows and bookmarks. It should be a Windows-style
+# icon file (.ico).
+html_favicon = "_static/img/widedeep_logo_docs.ico"
+html_logo = "figures/widedeep_logo.png"
+html_theme_options = {
+    "canonical_url": "https://pytorch-widedeep.readthedocs.io/en/latest/",
+    "collapse_navigation": False,
+    "logo_only": False,
+    "display_version": True,
+}
+# html_favicon = "_static/img/widedeep_logo_docs.ico"
+
+
+# -- Options for HTMLHelp output ---------------------------------------------
+
+# Output file base name for HTML help builder.
+htmlhelp_basename = "pytorch_widedeepdoc"
+
+
+# -- Options for LaTeX output ------------------------------------------------
+
+latex_elements = {  # type: ignore[var-annotated]
+    # The paper size ('letterpaper' or 'a4paper').
+    #
+    # 'papersize': 'letterpaper',
+    # The font size ('10pt', '11pt' or '12pt').
+    #
+    # 'pointsize': '10pt',
+    # Additional stuff for the LaTeX preamble.
+    #
+    # 'preamble': '',
+    # Latex figure (float) alignment
+    #
+    # 'figure_align': 'htbp',
+}
+
+# Grouping the document tree into LaTeX files. List of tuples
+# (source start file, target name, title,
+#  author, documentclass [howto, manual, or own class]).
+latex_documents = [
+    (
+        master_doc,
+        "pytorch_widedeep.tex",
+        "pytorch_widedeep Documentation",
+        "Javier Rodriguez Zaurin",
+        "manual",
+    ),
+]
+
+
+# -- Options for manual page output ------------------------------------------
+
+# One entry per manual page. List of tuples
+# (source start file, name, description, authors, manual section).
+man_pages = [
+    (master_doc, "pytorch_widedeep", "pytorch_widedeep Documentation", [author], 1)
+]
+
+
+# -- Options for Texinfo output ----------------------------------------------
+
+# Grouping the document tree into Texinfo files. List of tuples
+# (source start file, target name, title, author,
+#  dir menu entry, description, category)
+texinfo_documents = [
+    (
+        master_doc,
+        "pytorch_widedeep",
+        "pytorch_widedeep Documentation",
+        author,
+        "pytorch_widedeep",
+        "One line description of project.",
+        "Miscellaneous",
+    ),
+]
+
+
+# -- Options for Epub output -------------------------------------------------
+
+# Bibliographic Dublin Core info.
+epub_title = project
+
+# The unique identifier of the text. This can be a ISBN number
+# or the project homepage.
+#
+# epub_identifier = ''
+
+# A unique identification for the text.
+#
+# epub_uid = ''
+
+# A list of files that should not be packed into the epub file.
+epub_exclude_files = ["search.html"]
+
+# If true, `todo` and `todoList` produce output, else they produce nothing.
+todo_include_todos = True
+
+
+def setup(app):
+    app.add_css_file("custom.css")
+
+
+# -- Extensions to the  Napoleon GoogleDocstring class ---------------------
+# first, we define new methods for any new sections and add them to the class
+def parse_keys_section(self, section):
+    return self._format_fields("Keys", self._consume_fields())
+
+
+GoogleDocstring._parse_keys_section = parse_keys_section  # type: ignore[attr-defined]
+
+
+def parse_attributes_section(self, section):
+    return self._format_fields("Attributes", self._consume_fields())
+
+
+GoogleDocstring._parse_attributes_section = parse_attributes_section  # type: ignore[assignment]
+
+
+def parse_class_attributes_section(self, section):
+    return self._format_fields("Class Attributes", self._consume_fields())
+
+
+GoogleDocstring._parse_class_attributes_section = parse_class_attributes_section  # type: ignore[attr-defined]
+
+
+# we now patch the parse method to guarantee that the the above methods are
+# assigned to the _section dict
+def patched_parse(self):
+    self._sections["keys"] = self._parse_keys_section
+    self._sections["class attributes"] = self._parse_class_attributes_section
+    self._unpatched_parse()
+
+
+GoogleDocstring._unpatched_parse = GoogleDocstring._parse  # type: ignore[attr-defined]
+GoogleDocstring._parse = patched_parse  # type: ignore[assignment]
diff --git a/docs/dataloaders.rst b/docs/dataloaders.rst
new file mode 100644
index 00000000..02f9515e
--- /dev/null
+++ b/docs/dataloaders.rst
@@ -0,0 +1,10 @@
+Dataloaders
+===========
+
+.. note:: This module should contain custom dataloaders that the user might want to
+	implement. At the moment ``pytorch-widedeep`` offers one custom dataloader,
+	``DataLoaderImbalanced``.
+
+
+.. autoclass:: pytorch_widedeep.dataloaders.DataLoaderImbalanced
+	:members:
diff --git a/docs/examples.rst b/docs/examples.rst
new file mode 100644
index 00000000..489560fa
--- /dev/null
+++ b/docs/examples.rst
@@ -0,0 +1,20 @@
+pytorch-widedeep Examples
+*****************************
+
+This section provides links to example notebooks that may be helpful to better
+understand the functionalities withing ``pytorch-widedeep`` and how to use
+them to address different problems
+
+* `Preprocessors and Utils <https://github.com/jrzaurin/pytorch-widedeep/blob/master/examples/notebooks/01_Preprocessors_and_utils.ipynb>`__
+* `Model Components <https://github.com/jrzaurin/pytorch-widedeep/blob/master/examples/notebooks/02_model_components.ipynb>`__
+* `Binary Classification with default parameters <https://github.com/jrzaurin/pytorch-widedeep/blob/master/examples/notebooks/03_Binary_Classification_with_Defaults.ipynb>`__
+* `Regression with Images and Text <https://github.com/jrzaurin/pytorch-widedeep/blob/master/examples/notebooks/04_regression_with_images_and_text.ipynb>`__
+* `Save and Load Model and Artifacts <https://github.com/jrzaurin/pytorch-widedeep/blob/master/examples/notebooks/05_save_and_load_model_and_artifacts.ipynb>`__
+* `FineTune routines <https://github.com/jrzaurin/pytorch-widedeep/blob/master/examples/notebooks/06_fineTune_and_warmup.ipynb>`__
+* `Custom Components <https://github.com/jrzaurin/pytorch-widedeep/blob/master/examples/notebooks/07_Custom_Components.ipynb>`__
+* `Using Custom DataLoaders and Torchmetrics <https://github.com/jrzaurin/pytorch-widedeep/blob/master/examples/notebooks/08_custom_dataLoader_imbalanced_dataset.ipynb>`__
+* `Extracting Embeddings <https://github.com/jrzaurin/pytorch-widedeep/blob/master/examples/notebooks/09_extracting_embeddings.ipynb>`__
+* `HyperParameter Tuning With RayTune <https://github.com/jrzaurin/pytorch-widedeep/blob/master/examples/notebooks/10_hyperParameter_tuning_w_raytune_n_wnb.ipynb>`__
+* `Model Uncertainty Prediction <https://github.com/jrzaurin/pytorch-widedeep/blob/master/examples/notebooks/13_Model_Uncertainty_prediction.ipynb>`__
+* `Bayesian Models <https://github.com/jrzaurin/pytorch-widedeep/blob/master/examples/notebooks/14_bayesian_models.ipynb>`__
+
diff --git a/docs/figures/arch_1.png b/docs/figures/arch_1.png
new file mode 100644
index 0000000000000000000000000000000000000000..7d829a0c3cb72139afbfaae1470cf0997e9179fd
GIT binary patch
literal 58023
zcmdSAcQ}{r8$bT`MxjE99?1+Dp%B?*%cks+y~*Aqg-T`bRW>0znU#{gw~)Q}p1<?<
zd_K?TIlkZD-@oJdISzTpecktUU)OoQ&e!=m&+Db4yyPXk>v$*>>XNjSm@*252_Mm%
zIOpI;4!OJ^3WZl>E-I=hEh<W-Xm4v`Ze@%@Nxh7WKCh^vam{ztgDvqn3iooz6?dXE
zB4L@i_q+5=)VPE)L4GL)k|b*SEI~oOmvqt@K9dOdy~1XGg?)SO*=@Y0RfDeqO{c!p
zR<Wn2vqH8jBddunD?GEK6{a)%C<e!T^#puoR0wrb)bst%9EJvyF?%Aum>a%9f;iF9
zbS8n%u~4cRv$aiutSGq_msWbu<IO`KV(}WP*XL1JpV`w0y^i+Vc<NX+^O9~5MRllD
zy%J`%j-@9O%PK-7R$=-qJk!<a8PP3L|Jah3IGe&6*eG;tb0jUMaG&UQ%MD%jg}|kk
zMK6NbuD%bh!Hf<LmKAI=)7F1J6r_~2)-H1to;+2%CmXRpWgl+LUm7h<?TS}iKR;r8
zI{Wm<woxitzTuAWc^>fzvVgXWTP+l6#s|&n5r19fZ+&o*c1)Wo<#jVNHhTIpcdKW9
zX04UAyJ8*ZE!=7zcA>$9AafEUw-*Y<v>dV|%+eEIlT7lB=<_*Qlrt1A+ZmlWsL*`K
z<dOV~n6p*G{Lcvo7X^E@*me45J*thT$uAdKhCL+ul6RP=e)XATitRP+v?Vs`Q5m%F
zwY&<Q{&~&Sy>&pWY3aw{N|E}+7GZ2P)o;m+_9oouyjBUesg@zaLDCHS;|Cd}u4pzw
zUqM+y+>gEjpRjHd;t7}O&)1V<Q^ZafUwL9L!xjGKajb9S<@J`?J9)Q1TA^aH2D6#h
zS*?mlsDH+i>d<S7ek3qUw2JMdi$Zs3qkNlENfSp|QCa&a&eHX?umr~6VmuhPsW6yQ
zFdwDhI{IC{Bf43PDPMP~b$|fNABXS)37SF*gH#wD<x8y3!hjOj=NLH0RA=*vpvU+7
zID^@_%sMwaQtk8QU%l4;+Iohuh8OHhpnvfh6~CCDSqM`a_Pl6D2yy*0Z7Mbf6rR7q
zdn^XhD<aBf*d_Q(&ty{Y%-B>f9HSGTw~4B~m!G5VIe$sS_C4DjbodkAj~HZi4o^s0
z0*?RSxMCz+WD9Wk<LP=+8(-~v;5hvrzbAPO!S1($aV0D&dmPM8i55bV2|uixPo=KV
zh&-l#Aw{bEhLb8!iqHPqym&xp`(HN#0-hMO+>!rFPnI?yP$O951>;XueTzGCF|T7P
zV+31pX859>J!*(EATPXOb&-p-5=+aM<vT@PQl0L%(FS~dp>ezW=h_HO-);W*F`@jl
z`RR|(iJv7#c(iZjQ*ys*n0&OJ;cDu6C+T(j&uil@;{<=&CsxPLuO;KyW3fCV{u)mx
zNqBh!$M`w#H_~t9-!6RnOUW-w!IAbH(SE&gy<Giq{o@Ii3DnczG%0pko^Tqe?6lG}
zdu7b|E7^ft^6(|uCc>&J&rmLrue}m9LR=wO(QZUnggfYeLm?`RJB%Q#J(MzROo`zt
z<|{I~w_$RFQrYR+Dg5c4VVRBb%??xh8@bcW)A3WQziKnCWFM$<IZ@w>)|3#Cs1_%G
zOvUsUgDZm_@nc3+rUccw$A3LaW7K3i$85qt!raHG`{;)1H)c`>H^%kIjmRBF9mR!A
zYEGTF^KT@zB!?9$#mG`7=K>t@1tN9Dgk)D!-Z3yU4l_tHvOUUI70erH7XItpMf{tI
zg)g5*bpF_5e`rYJ5vAW3pmC{qrR5-tUF(OOc#3jXSC+lr^lK)S-iP<(oL*ZLFA&`e
zj(e$@HFTv^$0E8Xz_i<xuXU!CHNneq=cbeTZ!rN`uZjb~ZL)>b>v7iwuN%A}zkZD~
z;2vq5O<Z<ddYma|V2PO~tLB3|<-B4|w%8_FE{;TvwyIw@Iv@A%^hQ~%_O5hLbl>fH
z+0)o<-sAS+VVChd{e09y+5Fo%^LgxrypG&{k2HlI)A`If+kV^rQ!{-tLhDbXDTDeQ
zia$aOV=UEe2MopT+TYW(Qnhw$|I4(6U7lmq+;bR<tMH!m-E7VZW4!pGSUfqGIOn#n
zZBJuv#;FKo4hLKP<Z8Z~%e}~98go8oHkLcSHYt~z+h!%IV%ecqYd|wiYpLATD%)t@
zG}d&`V!yAc{*~z?bEoz=gJtXLp)WrU-n_|vQ)d$0im4T6owDpc%r$K5yyJYO@?0gg
zhrl7$A=9A^Nh0w#CKH{fb-bVXK04o@UK{LeyF0v|HAyLNB5yA5B=5g@aTa%GIk|uJ
z<7A7|=;FxYH%Do^!O5=0oaH8`JZG<&jK!ZWeB(RMtXt%bDGh3!%k82+C)=IQ{h2H5
znoPJgF+Zs>H~1%gig^CfpY0y42EM_{QIGnDKGTYJ9=(qRwW}m!HgC@bHe^o+?+qU2
zZB@=G>=5i0toe}WU!lS^yYw9M7+rpD2W$TP%V%-^B6n`NTCRl?eh>LUT^lg*K=c8#
zXmX%NKpu`ILFNlKyn+j-gnuXs87H2mDW!21vAnuYO!<gx`|c3O7uqk}CN?ciF^h#3
zh2I9E?t5yhj`)4<+}GVKZ>0<>f5R5Es;U23fBsqSg<g^jHva0MFF}=a>S!)%-uJA3
zn7L!LTFK?6XdlzPUE_Bhb}QJkIZ{3esnbxTaSfeHPz~dZSxTt7=f?2kUixig^*QQ~
zgf-IJa=#M~?y}rp)*o_pcXT&AM}3oeGEGW4g8oZPc5DS_v5FNFE8#Ayvv!sBdQiQd
zJ=KLK&5+(OIm!4`GUc1|1YI>Zq7#}D6drzvx}lZxS+!*ACwsUtSyc6}?Y3h#9upp0
zw_&_xVoSBHJfnh<N<7WT&pFnmD+@=xn3kS>Js;`v<g`3E+Gz@CSC7BUIpytN-?!X2
z^jX_~Un5*2tlUvOYJR*zc~=O}XK6sc`bk5;>Wiaqmf|Gh#^QRf-G20Y7ae6EWv;)G
zCgbB8v>qxYE!QZiuzbB36?f%F*htt(o}X5{Ol<m#^wn~fA?MPT469w|F6a53(GA~?
zC~|i`uGKxwQ01q!UDZW}GSwEMy$y*+i7w6+%N7H9g_UNz(E{HD2zCyZtaG@sSce0L
zql;JfPqL1eg^IoN1yY<-=IYA*KloSQdvfoZ<+7#aV$UL1@AvJlgZHBAfi{#o{3BiZ
zeV@uhJJlVrZiG^}?LYj+rXzX$u8iD}-12_DPt^j}ve*-%o|2E|X*}r_<#c7~n`vW>
zWsfuah-%|b-Pa;^vYWb_GPx6}6B~s%yt#Il=6KdC{%khR9uAX~3e*rf{2Bex@tQpP
zh)t(r$!q@sU*+)<@ho#@KCiBT+Z$NWFTb}l$jZ3oM@p*QK77wlosT-siyFo+*W#OX
zN~)N2pFPrPuZ&)Ox}sA{=~!FLT(l_S;_0%AdE);yaGqd{zsZer$5oD?EOYJUxA3CO
z1%+H6j+4B?{`_6XWYe)XrhnQgb~g|7j+3?z`~8k)BW8`+N_2bhH7_My-=O$%Y=6@1
zr#7Kx_#OLH`ADtTGPS?q)b4Qdr&P<YI<m(2_3|I9s$=AywwpE9oJL#ofx!fZloy1u
zy>s?k4n|Js4w_#V(&?@1HP@2*SRSvOR-5u0dTO^s+TGH^z#_L(zJjtPKnuUb_H`yh
zbsc3Vxhtn+pZZFpPhVyptVkDP>g#c%E;nP~e8#PNi&v5^eylQyf%E=@`z@4~R`XBC
zQ@+7mg`jWLsGW;i=Z3KPJx|)l-ewD)D7c+^og>n<t{a>6!TcGBGqE|@unHvhm9e_C
ziJTmY2|nYXurRKpu;CL1d<$Zb{qtD@gW-(%VxZ6{RG>Ku>p$1X!%yTZ1iq0mfB(b`
z3qYNR|6PG^*A(=Buf{~BVE*?rniKwp5>XMAmWH1yhW5tBHV$UCj)vZ1$h~pxq%<5*
zC=xp48$()|b`9>|XRfO5s4geVYiMiD`ozfAz?jw5+77u7ir<wNK3W?)KB028wz6^H
zbrqmFyMh-!Bfn;&p*p+7(Ncg$T~3io)YjgZii`CQ>m3?FJSr+GetRPmUS%<f{|tx!
z3DB51I@<BFvAMXou)1)t+S;44-R0roVY|c5#?H<HSFkv^**HFNWwCL%`S(r!dmk}l
z2Sa;vJ4bU{8!F_!PYi6G90h1-kO%$q_4hoDUCsaZBpZkSycWD58}b*nyR3KE{<${{
z<wt(Yt7z_OY^5P)ZVhV&&k(%F#mRqm{r~ls|2^?vBh~*mlAG)PzeoP-FaPJrM-ImJ
zqPEuXNJqi{ZP$MW|ND>s8OYCuy!pRY;_qcX`z@@qARa&4KYJ#KmtnHC2ithVTuea~
zeu9x9Uto+V6vN*?k<ZEx3;Za-UW8H7Vj`-p7)!r#t5he+>egp%%+o)nUroO%^O~kH
zOu$-IM(ouG9O9QG&uNL37{d}DD=9lOQL~hN2x1UFq*-uYJsO*xZt*GObBV7ojT+nc
zjEOC-S=~*XscGj~b(4S9Ap7k(3XMbM`=5WLsY4#1Lar{$(p?rtVPFxS{qqTph4Q@o
z_vf=;NYP-ReQ7PMa$fywECy=qzXOnq!lZmrR4=c`=g|E3-Z)gks5+N_?kbG(qrF0f
zlKSv7Bj<k;z#&Afk^k?}!c;G@a8O}ch1pF1nk7u<{Qu|ENx{LQ@Hu)K%Wa#~5_j(*
zKJQ#g){Uo|Gp+WGbkV=6cSo67G)sb-?hY`baq4SP7{W*8)c4NbB<QK1B&C<?%I8UZ
zX*s#b-7z1%HlEpptO(bM`j=g)k_3&veuXfqWc)IY)**?U{G@_K7-S18yk{dM^A1^I
z!bvoNz*fkJL%TrPrjXl)QN>K0<p<3PKg{z@{se*y^Yil?8_jIZ#A@1?dnXGWV7w4A
z-jYKmSUB{8=gI!;Ml1K&FG`;y>PI=Mp(+Xjk1YB=YD`U!`<~(*t$imbvL5C-e6O4?
zW8ZSOVXxDbpJIQhxP7TATtQHyDS~mbJb^&`)*bI7*ZINXOo8#ub|K2`wpx{949s)C
zz7q)DHa9ol*crBM#=fRKetUu#41o}hs=>mc3Q4V7=*^uDAhQWk5Zte~O^k@R>hV~K
zk9)c)a#M>Wo4~T~W1)y-V;J4^#A~*<JvOOjl*=JL>p!j)c^|u1oE$8sI^X#HH1~0y
z(azG~&R?cUZW*`LKh>?e9_}oMPyBd2>~`TAYXrSq+#buUXeW-<V3Bx^S^+Dc^U^1g
zx`9H&*E4l94A-%ed`|W^zMiKX(AU9qt){6%tUe2nd<+wd5QlMX*tT}?ZEB50pk6y|
zf0mMH&C$j~jdGjB-x*IQe{PI=tiE*ax+&l^*CCQ2pU7lAR1(^+=OfTn`EKvTG3!UK
znx1u_n~|}xxaVmN-_|9R<4O6LH}kDT$1s=3<HKDKTXs6sTG7>gGJ(wkk)m@L%#S|?
zHPwdOMILPT>!r&l@)kMF%9C=K1drG^v;28%N{Ysz4fo}Fcs5Ne+!CALLx$cL1ZGSg
zv>)wn9!JIBZFP=irQ~(Y8Y`0Wx-%KId{-7ln5`q>B*a#|6~<Cf(a4e0(3x0nHQ2RS
zU~RYA%6&-Qt&Oq9%8lVW#x8jEjW5sS%H8j_R(!I_g4JgD*C!YbR#j?m22!|ZnJ)I_
zQTOSSz#LLhYN!^rn?z`wy$f@s#sUsz-dBz5gD!U!=n0(?w`7)C4>z^Max4?3Tq5I=
zdU=JeYH?2SFM$hrY8T(~Z%D0rdV!^!4&>q-)Y$y~qWs0t+{^M7x&78`vd_@2Xu!q7
z@7LnEteM=2(mBjYh^seJ)aX2K$@yd%P7?4f-N}BG8+8N=^ue??`nCp5Af@-`IYrXy
zSC=gg<};IxCV#$hu2$UTx~)|vyR-6}Ig(ke?3CxQgPLN@=d}8GuZ?dtXx?g&83krv
zSp3RMV7;E`hhyE*4DpP=1Gez+++sm(lqf!rnD^<i%PQ$il`h6WiFtU{;Y#JstG4w9
zO54ETtVi<U-fWKdUynMZ>^{Sie@((tG!{!Fe_!P7{Ry1f9}lBhBWq7jhWS>5Ioduw
zUdZ_#yLGWbR|mW+c5ZK5G2A^~zbC3(z0lx;abH9<PkN7Pc&TN7laeI4ar?L2*Jab0
z$=(*VLh!pXCw!-m-`XnIf07x0dyZRXy)_z}&=k}CbZac4Y`H0tnU0T7tEXq5Ez|lz
z>*@Y%a=Jxtj&rBsI|2IgQFqI%-k3lXx{*F4Sc$+%0_C@cHu+d2U}?rXNYVWgucB!F
zXxOR6Hlx8Y=cq4R@%=L$E{h(9+WmvCjvLj(vG~bj8@c&knkr>?<GXz_M5oIWy<Ncx
zmk(x>Jn~IfMk<m39QLgXobJY|KP&4drMLc|)1PK?v0?=4s`LK%SAQ@{&ez@zzp|?o
zgpNzboL?M|Yh(PyX%^BDyj^Y_!)YGoQtnoP)<^TLmdRK2&V4Ly+lY~`anY->?Rnh4
zJX&43YPISy8(gk?2{%F3^mlvHMV8_*vf0Xs2oXI|IRKBe2CP``!wx4o-}$?{!q125
zIo!9*D!l;*1Sq|>LL0EqBEz}q^ZJ1!HFr3Am74ouQ9YM`V&zkGOYhD_(&`b-R^xgL
zP<rn@soBXMp4&MPFns{wNR?`fjfD%Kss~-|v0FwSaPHFesksz#Vh5G?I;-H@+gk1O
zJ=5H%jyh3))Ny-)b@^f6`wMOSUezmK7LVRed)lE<Ea4}a<de;8wIeo`HmYTZUGkBQ
zW>Z!ZUac&sJ8IZVJAN2-fL7JU9Hu49yhEKEECn^(rft;?uj<Kep+29U916_TV0w%M
zTq1LKNz_8!u^!@l=e$$;afT_F{6X5`(`UF2-KDd2ewd@jJht{1PzmyuS*W_Hy4{$q
zXbOkYZxm|&oDMI3)o`nkFXeVOI=!8(ip#w^`iJv+aBT77j>ouFCb@^h&k>E^hYMJ!
z>lYVX9=x-*=_vCXXUt5P5HK0v?XUEIP}Nx{wrIG5@vC3ArI*XLqJ(^l&t*2z#oduX
zd$7oOnf+LP+SL9;AlYkYaDOn^>9R1DA2tr^%IBxq*U{L-pXWZjwvznLFUGLMn!u4W
zlb!0lNX&^^`z`IgiUF&%>=^t03I#{ikIG6t@BJUc6?tj}NiH>;87UaP#nm-mo)XmQ
zxcr{JOBz>2F2T|AlY~(lmGU*n!AWeDd~JDdG4h7#;LUgZ6Iud?HFsbL$E?Mx?%3t-
z%y$hJFKo^cU$o{sUfGv-$8Nf~Sk%l~y`}t)RANfUD!|6A#g4X87V}eGBQDCZ_IgWP
zyg_yD-0t^m%L3i%s?D40CSXUST%F9u`>`z@Otw0h<=w}EYjv0u9I~u-N(1dPu_)){
z>E>vHN{S2c)@}$b$;``UM7xpL@lw-cn+;gY&X5ile%@$eKJIhiKy`2o*N_Wj>I44o
zIyX5l4^6j}6|7%JZ7mP?M~%&^@`*Zp!$$F1N%7SWJ>6NB<)d=X(2eG{9UY0C_IhK4
zN@=(gn^bk^jP|Wt4QotXwdQuHec|FY+xjc?W_5F`7gd`TvXeud1oy03(j-fVB@QGU
zHZPvDjcqwT=WUF)OU=R2ZOCphXkF>FNMh6L$>Lzu%b)p2XMc#A`KGV%$OS@Fy%&)(
z1{MulN~}*waf+$wce(Aw{_;f~*FP`3b1O(ubBga2!=>pKmLt5jd)2&i>n%}s!`9`4
z?)S-c97Ibc{p0O0e4Up{t(_QTv&=`U+?@A2B`L{E?uTLcmW8h#)4AIQNk5vu>$Z6O
zHdo|rRR2ibGWXcNzRkvVndNxd+lTU|HHq(4<JOtN!rj)Q%4KKhw>+1JM#h(aKgJ}o
zCUDraui3f$dE)|G)%fpB-Ajguzv7_w@Tq)-Lwp^+o7iyQSx_WfPQZTK{O$)z*!F&3
z(Sna{)p~$l;-mnoW9x0(-Z4b*!FbKYFXu8aoC2)p2ToaazF-|^eoZL09w*(FI)|#b
z#O84Q7ysA3s9SbsA;sE(1U%EdsBbP!EU3AG!qnefT^M!mgihvIRP^z5#x`EK+ZLU;
zCNNvdS37==IA=v)shTBcpJ-ZiI<PV4QqP}C)4#2Ofq`{J4;b+JaLjd@5L-VI7R>>C
z|0L>5+&qTG2n5yk8r((QnJ27HEHX{@>MN*Pr$Ct*C&c@0&pb}tx>)H<<%Nj~x9_$b
zcVzuuZ6Vp`^2!KB=qg&e`qXobk6V9HfSN~+{zpb$H2&sB;s_pUs|GJqmiIYhZFpJu
zC}Dv{b^=*$qH|XYT~9}zT|<Y5xZ`0${)gwoQN{puF+k~G4(9jZ`8~kB|DAQ~qK|xd
zW8+FIEr(^_?X#(^RijfFsl77Aa8R7Y8Qs!7OJZ%8@@p`B`}PM6-<2-TuU#Q$Ct48_
zs9fQzxQx0lYS6NY;rV*PtJY)J#<u#FLef%Vfh{txfi?}#)*jUuY1_wr`gPovp(fjn
z?P<G@=3G*BDxLpdP$cI5(+}k;kt%o($H*?I<!Mg4Fx*0Qka1#W+O=b>>6E8v_}YIt
zufNcfJ<5wcN(fHClP>Mp_8^eYa#@0eYL)`k&{{#dH)L6`J@=RQ@)q_v)C`7=v$uRZ
zzugY;LN6}L{tM3tj}|%8rL1D1%-}z_6P>mnCI3#@z7HJT)6`>Z@%V=P_@t4DJ;@hR
zlo_5Ti(1x475V**%^cI?d%d|UD&3hgPF+-O($#U@4tF-bu=R<e{OT`5Hj_}6Mvo&V
zE8NY@D=JmuaO3{G+)D`YU&=YI+|M6q{6o1Qp)xj3vnHZn)6gmZ(b>60#nH=jc5hS~
zGbgblk<aB^UZ!c4lh1*yZ=H1alLhu_4bHXqiZk=yM)6fY%P$6#ax4it3IS`z<rfnZ
z;^QtjcMc^Vs74tx9lT9&5tR@T@<raed47aaKPl&iDkXVxzEl4x4Jm3*(2Au8vo@h>
z|G?`~mBAm^DMuo^k`)Za8?#T`7v4`EZ1OG-7T;=kbZ@Qec)oSiZ>&3kYvWk=+$e2g
zH&=@!yx%o=zay@U0Gz8*!nR9Jyp9Ni8#H#5B;&T(x^Ft`RN-8Oc}pNXznSUr$2mE^
zEYmy5t`oSmcU%9UQC`Y<4o8E<rXGC!yi+$-EyQ6<t+zfdesNl?`r*EiomtgDe%~$y
zg27Mj$h2<hF8nCx`=y(65jP<O_&qC5!td-jpFW><*hwuJH*r^4MIOU<wY5%O1C5FD
z(tn&T6KSP!k9dxI%yW_@$~mR9`y;uh6MjMhGu=VG-mUoHw=VtKyYdGz0yeams7o^m
z@vYknezui0`!n&QnV6ObU3HDEC}Dr2*``pz&$rpSUlB9&pQrTvB<Zg=rdk%)D_xAi
z(=PvO(Z6o1GE>dHL~e@|PZsi4-<!gyaj7-PguT&2>s^7jc*v!fx6A`a_dgc!I9Vlm
zZb+}XyK|%orZtj=>sC5juNDDj3NHO}i_4*S*luPyq_k{C!u<I5+)}A6YCJ-T?`@kM
zZDQtdpiFj_={Kk3Hs|ZY_nlZ&GJ<!XsK7%0=(b{S6Xcw1%Faw$5Fje}s*skEfJXhU
zb>R4<^0p2o+^<pNSt0Ox>0>H}-ze3>$mg8Rz)NHqxd(Eq)w@aM-tOtsCEfCybK!AL
zUgY*u7w_Q`qvrlP0XFGY-p_pZK;!gqa3qD?ri){42;cpN^(|}3TPl%l-S4Gq2sj;f
zrcF>O#;%Kb<yNuVgI6u0ho`h(cJcCL#N{<MuBC1E{Ki#w7*5_?RqQ?pa1VH3GhBQF
zuuu)K@ab?=9JQjEpY?E=n@#oh<J!0HogW}PI=)!IZtk6f@N)2c_oLFuNk<dMfoDmB
zJ=FN}v%GgGYm2Kla}2+f5-l%%ck4ZW-wTLoo5c?JF{fhZZ6W!<&OsziqY!wQ+`ZMr
zT;SUNX@Tnt<!=eTWbWUN;Vj=f-ita6jkKw;$>q&!+5T1|-gNB8C8B@+n@i|I;`+3o
zsIMbs4L9auq0s5kW=Lh9e44HA?EQI-#b`b6L60AFc9Va;2OfG+a+-CRUYLt~zuCsu
z&O84pv96KyaP>yR?mg|w`-g73zg?=i`*?>fOx10zTIU--%b&krPA%M7#nb1OQ>JU(
z$;??H=l)2v;o#8uptOd!UV1lO4R?cm-ixR05$Q2^3zC!W;ib);-<&*CWu_SP16G5i
zaTzmG=`uK&>BEC#@TK1_@?DS(lx&HWygS+Y!|3ki?rdk0>HSAOA6hU|m)4zxW`YZ2
zwo*lg<BQw6DjZC;SUE-{TGr0rC4YFAT;(oz$k6FCF1(oc+nn5tuPY9yXKokAq|oWV
zsq*4f&?-H=JAVVoD-3vj44yr!vevG*Jchj9AYfCSJG=r)oDtd&^(nYatmRTn6J-fQ
z7P&&zMQSh1Sop;}5jNm0H^@gnKVlV|P!AnyIM`nNq5b~TnBK8`EHj2dzW3A@4<hYh
zYex?Ug*V41Gt*-UE9$3t3wNEgI1(PV_&4Gm>9m!md<4DdDWfooFuGOZJuodrsyiqg
z?KV@akt-djzKecEBH<MaG&2?wNzZ(rMF68<Ga0E}K3ZH%;BlzGSj+q?f8ZP#oH`hs
zA6aw`b%-p)y2hTNbv3o{cD)utMvY46yqI9=t}Lb0EY*q9e%)|Lzy}Ru|A_4@XsP1Y
z%fmzSfz(cX*=OKK<J^bgZ?OsrnNmdO9Mv_uGvrb=#T+&ZFQHnzu#^Pw`2ZUCh^bIP
zV`ZsRI8@^|$FWfKlP_arg#;hEzJ@JD#I+i@NzL^txb!n|2bqo2$I?+dEmiLHpDZNU
zl87WGWdVXsnvFNpnc#CKnT1#(6Yrbox%Q8)MwJco3j4C34fFbsW+z;PONgS?daa`O
zAGHp2wKNhy>V=uijsISRB!<GM5GxjWum9<KVVbZ++40FuH~&%SXyZY*Dk{CdE&h*8
z_y5<4g{e_FR2rR@%J6%r#h>;C8QENJ_xnww;c;?Lnwf=h-dKzsKRP0;x7Ft!ZK*2D
zW`*`foDwK!!c-VYLi0%}`wFbFK6cS*=n~5&t;gZ{u<B*e^`-r`mYpf0TjI(x$+X#l
zNzB>PancKauDwnpX`dBTCr7r5DhfENLH0d{3y5}y;kUF*DZTUa8r|Z%d7AUi;`;9d
zE2V_gyWU#d$|mNG_Z`2Hj9qoM;6c~G2wcq>n_hz1JSSQ+dbpF!F<sYA@X~a5!_HE|
zN<${rJ<NOLcbNOsG!0|+<zY5Hgb@=4<-@)?dJXa+4|MWd%G;c-BPf5}5WMPQD06og
zH!b;&rZPe0uQ%TvWDzs*^TZ=WO%Vn6f;IT5UtU`p*5^GbmoCHKe(A^9d-0Nt(4V`~
zJXr=8BLZm7>}-k_%qRTfCUX5>BpvcTGfg(3l#_Y`yzw4b?D7e;*Dv0oM$D)_17>1X
zb|D>^*MkeL?9+7%-eKE|F&)i@MmCz$qhY#}N)EQd62TEc24}bakAH!w3V*zCV}LQX
zSxaW;Eq0-<vQ|w`ksMv>s8#5e=){Mrt5*;|A%P__GB;m<C3?y-K3J>49<8HlMCwiR
zBJxpD{zpF7k3BhF5^2tj-e(Fi9w3RLlrR_KWFO%AawYcYB+H_sBf@1-amUm1MPgt0
zZ*Q!0sqR1N#pB6QUJhDGPCR>)tFpeR6nz`37w}eB-w5|n4AxFrIn4NIblzZlAo-N{
zv|EV8ZAv0kt00f<$MrY@^YojJSkC@%HI<?wO!8S%Avv-g)))#=M)_uRgIY5Zhf<e+
z`{>Wsmb=Nv$Y%#~M+bkgjU)5#K1uv~dghqgz#;8kB>E$wdBuer>c7>B`rb5o{rF4s
zW+v#Z^(9PRvd$@?ZIZfwUrq>@V<d{6U5;yH>(xKH#XQky-5aBkq&0gw_AEn6;&^{1
z`|GuO*$PG+n9jdJ#Xz?%yh5D)TL0zheQw>1DyNgvJh>(jV%pU8{+Y`dfejD08EzqC
z6ak2EdlOU<TooCwEAN--`+035gK2;Mh%Ztm*X~VP@=?`hsM?un(OjiMF+8xsN7ikM
zetfW0Y{$ntPJ=}-;N)uB=4pY*tE+s=4$?fIpV-jeKEuACPhd08MFx@OQvA!Wg-)&|
z+*^pHh4q!y@H3Z_Qx*e@K@Qr~KObnA+0LKY`<|LF%GVZ+EbmS9pw?(u%QTBY8t0$a
zN&{`R#k;}o4zlhY4-?;2ofRLh`oaA#Q#s%uzR8A;eHbsjQ%0~^l#{zCma|z9DJ8I_
z-DtMvm%IIE95KUxA!?8`M*GTi(P{C)T@9|W2jWGWt4rkr^UbvBvI4U=3(oLW3D7o(
z{Z1wFE*ms<`qP)fO8>-GeOwsg?VLNZH11;3Wt+bxxU5-y|IYup0}6HFjE6$3Y)LKF
zg`i%_T+M5%T@j?4RAhACq4y#x-p<rVW<-Nvyt1Y8HE{KZeoHL2@$o+K4OWAxgZnN_
ziAuKlknUXhXFjk2K{TwD5@+)vZ2s=^>ceJ%#6`@eX6_eGi1+`3DvQ|9&}zj0ut^4L
zvjSXq@)_55I)^@G@t$g)04%1fhxc~o;2oaf(i&Lh+ShRwWPf_lNyk0|>!;#gBO`Cs
zA`*QvZ@W_p%M_+2{Aabrq#>-xd|Dy_Z~gUJ7!ZtZGq=o5qx4tb(^OdP=KID0X%ud5
zp}palh@&+!0knK%QK*KnOOI95CrEYP`}$QP<IV@4d-wYdGu6nL%paX!(4s%HE(QQD
zvCGjqNX)9hy|I`7+UWSi(<Ya=(Vm;{RP$>|YC+^ycoK8i7kffvyJL}nS+SiB$oQEx
zq#`ZbXjOgAoo8g_h*=n{iZ1mJx*yugkbRj0D}Ooqt%m>u%Rp2A^qa-RzDHiJaqx?_
zT69`O7SoTmqsB)U@ZZA4wEkdG>?Q0@Vko~D;(^n3rHR4QOv~xlk;t`I`MUTo?-BIh
z{(HY~BVoL&tA_~gtE@kEK3%-%WP@32{lc*zNn@a-&DHs3XLM4Dvb|!lfVi?A{xjqO
z#mEE71)adC`muMaqv<CacbEgoFzSR>_lUnH5-^(EckG`Z;o*`*cIaQlHsK}QrNflq
z3Ur-@?J&D}dMmN-@4|km2pj}F?*J}gSt{lLO0Cm<I-SIuq41Ske>BGs$(63|=0H&Q
zlVLgW_nFiD50-r|pbxJ-y4F5QsdL|ciJU!sYF+tbW847qUv!8wUICn>`g%QL0qoQh
z*T~x~-^liqp$3mUy(=S%Pm*2!_>PQ+Y5AF}yhUO7-u%i8t3Cd#&M%T%sxQD)`|PH9
z6Wd|l6WxVscVrM6OdA95|Lh(^uLzK<E58nmKi^PyQ1c|_l}VQ?IQf)DJ}mVAdQ;(@
zsB&5dC<(p2;St!9#Vx5!P|LZ4C-D+7dn_U<l+>ql>AV0Nrs&vV<1xv=*asdHAIC#k
zZC>4QY<nCUY#%y$_fQmGix3B}6$ka_=HEtt2q@W}Wc%vi(_N>kh_2Ff^~nOm9NDV?
ziG)(Bu&iR%8Vdk<cUeA}wo@?bd?{g>^rLit*jJ!X@938L=(B*&>j+lru*dr((n3#U
zOYpyYpCPy>PzBFBXFO3%iwYF}bNDwHY-{#2ZtGU5L*>n|%h#JGRxgQsS+rr~r1n^2
zYJyLkqEch3sfO~QMMbeee(=?^trbJ&VD)GLR_`w3C)+W>ik%@#W4HrSU?k<XNl44c
zi2HLoSmg8d?!@4(hinv|=8K^k$+X{up$fIcr1$J2Mw}0qJ9)4EUp`OBf~0KH@m$tM
zf4;q7H>k&%A1*H#dyju)S4nzqD`u?9Xzvr7%2Yt%J)?z@-mY9VHmGH_)QTvvt0wUx
z*2D_<^%<-E$(eBKeX?%~T4BZ9;%l}VhUMXMf}AST)$4<&=5ikQsLY<wv`a}x2LH@W
z8%xPF+M4U^%6Y`>2J&ONQDfLkZOerKu#e9G8skht=Ydd*UE71&XS(=nlCaoVMnwBd
zaGQzcdCcP=PS!YjB=NSH=&8uU<vhC7mD^hP$glKPy4ad`r1-TY31~r)>X+C@b|=J+
zwL#vntHwpJfHtQNl8H3KhPP!m50SDa?yUsEvYbIa5}BE>j3@olB`-)XU6ZmHJUuTC
zc}4NYbo1-D-Ylh?ojI~z2<_+x)X}y@<H4)HWI)QM71L2#Y2L>NAN+)M+f7XK&CK)B
zEz)9PDZJ?vo!=6UlgT-Kpa!~=BmMYsRJBD{nsj)IQ;Z+Xg-X#5Arr%ebnuywc6{B{
zs3S#WW89xWra6jb8q$~MeplyqJ)vJGKT2DxQ8Oz8VoA#EE&jDNp%KdO3tr6~Uy0VX
zfB8|Xyls@bGhe4ttS`j|2`ZrgrB@lb&x3R&M$*BVtn<aOF!3Zwz+)(w3+lQ1`5S({
zUbL3K=R9q%DUhW-s?h1O493$kYE9_*;$*2&ViwB3Ghn!L;`@p06Ct>PFqHxzotuFH
z9rC%VZfAMeI<6i20?9z}DdxG0IJF|5(=|MxH+oyV;2iBln(6WD#IhFt6YgDEq3fzT
z4z2gtofcI2(qu3(4ws&bJ%2Bc%<?jr&ehADfdB;c#Cy}xdRB4mNCgcFi)`e)jtq{*
z0+mrsCv4YmjVe-+A9R})rrj8!YM0>sn4^#^xWnWt`0!7pnr`S7I+-aqkH=z&Yp=mp
zukDUQS(7S+%`g37^bpzOZQ0U{eXAi(dkxQ`J^`llkH<m_HtG+W=I=>JX-0ZuGtEFT
zdHGHX6ljJx-%H+jD(1KIT>_b55eVtDhuppgAgcRekGe0ic$De!eTXY6GLhS)D!@@V
z|5%qzgN$C>wP0n5HP3)sT4+7?0r?SB<Rh31s+5^jv-DeY)e3a8jtlQ1Qwl}`#=t@k
zDFzlE%bh3Ru;zPnqkD4I+B5d~7qU|Es@p^O)^3CV=b`mjJPLU|X>79a<eIdEK=>#p
zU3G`}2~^Up0_fz3axlJ<nkFdCc@;sG8)6|~O;~nmHsJ47P2q-A0t$Y03fFNAzAeda
z&nTuxIUR?WD*Z7l<p?{pw?v+--=FudVP9w^mS!U=oLD?2iKE|ws&{0xuDkS5nKfCV
zSMg2cww*}6RQ)2G5)Hzq{^`lF{corK!FWY*MG3c!##`xB=7JYgslrJaHW$mns*QF1
zxLhn#d%m+p1@m%jM6BPcH6Homlq*N{#dY1l!~c_@>l@rNb8-R0A9<sW3I7Qp=W`z5
zOiuRPczKbWS9PUgHtB=fU4rw?M?2RWN4GRPDJxNuvMggt?DF1UFCI2B)b+UyPbK@D
zgtjH}HE+##FJJnWD*29H&TMo}{Q?9`L>_oamYf57Hj1@nUeX%R9i8lbAf4o~!ZQy$
zJ<1i_rFjtY36#MGn^%h6e-sJ?M~n_D1Z2=BmAy|i%6~h@d#w#Y#|AxH?Gd~AgX5Qb
zXZj78>)z8fkpIET)XS`5wcY2_KbQ-ew#Lz!COSo(;<0))jaI1Rd;V79KRsGd#(hA^
zmF{XRU9cIKMq!ZjM6te`NP+(erk}){>w}!{q3G>;IokI-!j#sjK^37~7Gu0DT!)Pz
zx9s+VSS`qzcD5~%xzwW9{{7V___5p3H`G0n=ZG=x4nEUPly#;Qm`v7n3L;&loR0tS
zT~*b;1jG*MAFqh5<2oka2?@3g-A2GKOa<11XE;$4)`Jy|RB%TeyXBP2ZFJ&iiaqx?
zDy;sD8bJJO#r%Fy5yhmc;+fI!stP&q62-IjP>{_W;^|cb&zZ+{{CxBJ#NxTa)3cCP
zp8#RJ<sL3JJfj>t;36pvG{4l6A8<fmn#<e|R(v5)SrdUNB*ZZ2Q}bkJ?q!Fr!;svo
zG>SpZu6LD^%J_o!G9$I)3wABd;yR!-HSB@j8b$rh1bx8l_UI7`dx~oo*ABoogm3tc
zLcnNgpin$|hT~b@IPf<kTThK5t{U2;>fD>3O~X4fjV?*&+Lfloohr-RxYp?<>24ZF
zU-BsT@y4KO@*vss?XT8|s*6I;l#giByxB3QNq{)*7jq2&64by7&#g|$ouyN#!y4AK
zxG|=f?&39$?)U-a={Pc_pmo-l)a5!n<()UCD;<fQsdb>+?l1{c&R98({_&Mhj>sM2
zSTmyGU;1`a@-3~L*^Lr1aUhX%2(9gpu7>5mfiu@_bgy+b?w+O5##G}Deb;>kXTMi-
zgS(a0Oyh%<5=N&a>qg6$8;6cpYr2}GWHik~4Fcmij(e8}Pa^!^vUWT*Fi%}tvKrp+
zzb$f5ukU4Vw?6U1Fgqv$8?2H+8N7;SdIs_eLQ8aWG+VjhF;t&z-Hx~WrxP67UY61*
zC*k1opX^O`WhzkeESGd!#dUng=ezyD!-N~bJ5Mn2dNmIvO-M#k>Z;r})4UFMK9GyZ
zZMsdoyd`0`y2M0+&H1ZxAtyHtng+P!%)mMg3S%M=uMWxqi%@kN<N>K5r;ekY->}=W
zFI%M{WB;S#`v)IMBJIW1g>CU{28+_u($WT)G2F#{k#SP=Fl{3X>je<nACE&rP9(Hx
zv}e3b;C0H0V72~P%|RE56zR-{eZB`=f*Ndr6!4X>oD)^ZL&CJcx+{j0hi-<bqGMxY
zyYjTS-9SV>YfD<6Y-li!YfrN%VR-BeA$3Ql?wXz_wY$Gvdy+t^E2r}th{E;2T!oiP
zu<0NSG2ktYRCH(TA3+RW8a(~#e_nFkY>8JFYikDTUc#%FI>??=diC&k4qUusaK?}|
z;PIha(O;;2eTny$%f}#u_2#NI7q<&q@_BYpjpKlIk5+pOpRa|;)}~8MBbSx>9Z;I~
z13Z<cj0xc<1$AnAUdDW<KF8b90?i(PfbP-j=P>mA8Ynzu4|bNB-;i-fagVxwHwoO(
zY4bpUSqbp|N;+Nkc{C0dZcDPzSjPTswfqFwr!MJR*m>VCVtHb3z5IGGH4fb+!<_HI
z3$}Gy&W2JWLqCvaW5H0oImKiO0ry{xx00>+PSf@;UY&4|M*~=9mss>RjZz7E>{yxi
zWGTt#vQr{E{1A5PvlSN}tlJ-~mD*EpQg)*tZO`=|?9kKD<r(nZKtsy!o@L(v=X)p|
zm6$)!Dz_oG7cfDH;yMFB+`Bx9SYV22z8U*f&;U05;+pQKg18wD0Re%jFD+tJ+<t)8
zFyDt!w?pq5{*;0WzWwm?X~Z_QA+xO)=TMS@+*3}!KVOHdb@p(zmhB*gtr}hTT=)q#
zr<_@)e4-I_zjf96_>gj0{bl~(!t16TNJ+jxcCsiZj6o=z@(I1zUr-+FU2}Tk5evO5
zj?e+~snW;;s|pQKFiOiN6bcTF&~Nvo;10QChsfx3l?cG-gdFVjsAlj6jjwNV9Ry2j
zzEh8#rO;RoQ<-9u)}yFR3_mH{;OqBfKpej%7spv>{s21rI;584juFri2U@F`Hge-4
z6y+J$Rw`FzhRdwep{I?j`<=<rlLu6B=<ZB~a<!No)og?A=M*j@PdoW<93q+i2O3y8
z;>XV5ADtwwU>;M@b4S>^EabAH&2uo9n(8IB^Ujx89cY``<CKyTPMFhMPOAYzQp8m9
zVZSGGDj04<S6m&GON`gXzuNzKaqU%7C{&&<-od$=jr8$B(2$GgYP2k<HYvs?HpjUC
z7eY{RV8b7b7ee3T)jZg<CFtdu8$1O-ZhDSOB(<qgB3IYxiFI<csW7~*_r)<=j%cXZ
zR3SP2O!PDWDTm5=I3Nt1;~k8)>d#M{`|Kld=riWEyMKF{1-*i^^t)0mziLCq4f@!O
z>wGZ~_8UgulbA7{1$2o39`C7N2o0j2Z+?36{apOKX>ZP>jje8lW?;vQ@*?+{(9~G4
z@2jKL#t<C3-iw}hgO<n_obRbqVvw8-N_d1Eav~zI5D|4yK{Vt$b;s8K8BA&qmj!CR
zaRcq(j_0182l03dM>|+-b0qVud+eEPKqv$AkHFi6vGiDgO9JsTn=rkySZ|B|d~%}c
z4}>^Vw0D5mB0Za8-TpODPqAMrY?vuwLex}@epq?!)Y&?KV|n2~C5U92r51y2J`ke6
z=e4n88A#|u(gh$YArS!a0}7=-Wd|7$DK7<Rq9*}7cLr-qq6z5*h3-rJZqEf`h>#P~
z<3JrgnLoIv5|V*y-T6VBpE$XQ##i_m!d6`|RRoYrAdDgDq1(=&DX?Hx_NNnKoyA$Q
zIQ6N%&L^dlwN``d2!oscNoE_|V4-LNI4lgSjb){p^IVu84|aMuW8V|{gOHrW{`7da
z!SEgf2UJ2Krb70_4#-deI6dumo`Om)E18gzP`K_A1{%fP!$S|XF5IJ&!&R>Cc7UH{
z4cPMb=fWRwpp6oE4>b1`w_ofyKS&IvQKcM70eDY*i1-tqIvLo}FKO2RMu*x?f!!^;
z`jGF!NE*D0WL&af_!K;gAB(4;Z}3{zb9w`2m0MU7qOeDTLobzSLPF4#o@;*E-uu%?
zH)8xeU;!_#h62+I-<x-go30*GRpva(1%eN4vQJ6Yk^bhZu>iuMN)A-O=!lV?xOBiP
z<zf@L;r#;D?66gX(VHQk&=*K<!(wlne|4B?355nSS}UDwaE|(a<Z!C@*c9$_)O=ST
zBroJD?9#QGKH%n#yg<EnygLS17SC-<;^nFi?Zl!Vf80h~k{7nO^QkdtjD!z_yFD$K
z>ZQZ!2cHlEIpZ`J6up0!nFkAkG(kMQ<Nn6<2W}jf)lmT+CV3WMhfc7@{qeqNs*oFK
z1b-kEYlc9*H0UE2`0F`jEJ$1BnIAwMcj@~o=drbnj7(9nq7UGA5xj4N-=F{*C&+5#
zb0_}Oj$k1odf5<g=x?A+A#AH7-nK>~$$f!wz9&0eHi<u$^Svh@%sB_<tTb&ri$k^c
z+huI9?ey;)5ZZSLuY#gNowED$;*I-?ZGwkZ@$@BtYt(%P&}T}e=n7W#jjZTAz_!RZ
zG!#cPvsTK=L^2t394FW}G4gzDw8A1(^n(!iXiLPkfzqc+j!3h@-dG>eb%Ma8#S$XO
z0d(=Iwz?j2wVJ-Ue$Q(5G{*2P1_(48RBO=9Ib2uax<0Wp>g&F}F#Ux2F@o0{te`!*
zF!uoi<HzHNMrd~ZI+Qu|Znh-wsNnNVh49-nS|FYV{l`V-56W$RTXa9yg@*4s1g$TF
zo)q*K9a4gk+l1Bzh0d)c=-?=u7vx}P_XVba#<3bIkw)6k_79*9Hx4R)pWmZAo)97X
zi)0`J&w1eGk>LoJA9&jAGUipzzc=oWJ#Zl&dJ77CGjMx-)Es8pB1Qbqz0ANhM)(9C
zGeik>kuuN*<7ZUbRlNZgxgwvy>=falIIus8)kR$3%M}L;xyzBI20vbjo2@RvN`xuF
zLrJv~d2bSm_(8wy5N9&)W4ECab9=CBNnmbx7+5zjyj(}2p%Wp45S}W^e}?{nhVBh`
zXo%k3K6q$B9kjC>6-P!eD1F0M%F-&&cOkWfH`+iKRBgP{cKPk3T<z|3X#1A)eJHod
zyML*rRCs7du@AmYte1cJ`x3cAS8rDaF5Oxi=Ji~7+hmBpl;;5>K&1a3NKkGrNct+|
z8cUxD+?jVvUd7*LxU5z1$>E3Btgntl20)pk{Rp=Djf;g?6fSY+FaRcC1iiB(!GF2w
z?gR<fU~$=um~}t5-}I^(P=|Cn7=?k-gIxGmxUVhmx5||-4<Vh8%3M|liuvKzc)YjE
zuMlc`Xp@jw)n9_^uSmdpJ&6l@MMWrp;YFnc?Y~pG1vL|XVx5L2?a&9~dwz{L=*^pd
zm2ZVt&Wy$I4N&-=Y?bB1k#d7eK)Wsc-irnTumD&lwZz;Nl_JvYVX0du459?n-!o^H
zd4+W91K4(D$dNJXR(*!PYRQHE0=+5YabJjXE`CVfy(Qv#&!PvA<{i&WOqFd*9Q=h7
zoKCvCNA;@T5pEUiIM<jf5N<2U58yxh!x)w~eSMkH&~o<POHeXfIen@2w~9Q^k4Sqv
zMR0Z<c=~`fnM{DMCHkgBaLCE=VKZbx*2(uZ%d9l|o4)x06FWsiEKVfPFE?z!kEh@I
zc7d(sRDJ*r>+={{pZKU()FD2I5rZK%!{GwDL|&(`#_(GY3k`m3)ZQ#RQ^b%iYZwfj
z_NCb#mCnlxe+SU|0N6sS63lc2wZM%E4JG;d^x97><J#e<K!3A1LdV9@l`gAM2#P`b
zyY(wYIpQWH+5XR2k$p&Hfl@<rW;0~&>5}g#vVaOcLs&JLZIu$74WRAU^~mEt-s%nu
z4GpC#O}!3YM+k5JEUWMm8khDAgA2a_21mg$i~(v)TvdUbGpHK@humSRgKleEpj)16
zUkBjMVUVoRntxZZ#!}V)`V6}Z)G(g86#u1;WB`5?XzRGQ*I<o)0miZc&n+Q+2HHwQ
zU1>T%Nk}-nt~`5-B-K~!Evs7m^I4juIm<XVJeCS^bq#Z%g_~&&g5pDj_c_jY?TmTD
zS|V6N3t%HvbT0D?5VX{Hf=n)>Zke3cL#ieqEWlX`OvJmTggG$p*^X*EuZ)-jPT76M
zmR9vZ4(Xg7n1NF}u)d&vB5H8!-X<I;8T=z8EhhZvvhcW&A9U~!cz8d~*JdQTB_9Dj
zlKSu_(!eT1XfMg#yF5_oQn0)QDUlX(j)hdv^S-&@z9v=36cX#9k8cCo?SDZmWAi_E
zNyh$i&O2dy*dJO+K9y$LYkQ#9ffi`o9MuRXO3YQz^gOt|pxvG#v}||f_p<+Ke&q@;
z$iD&t<dncN^kH+bY_xL%e9w-jaJ6Qh!zifUiMxaqg5Wk9d!>3;w^Q}KyHFiu(`mA?
z?0&gxh{KA3L5f@tv!EiBIpa;OnSxuUZ3#T~K%4>)CI>h}><YB*EcZjG)<qGZ7-Fz|
zR}^6^23aH%bqZP~Q$Er%ze3_I-^zWr!{zc9oXMVSXTXH?P=T^WfEUl_lHZfB!^ak1
zh!2526!!JTJrZ$ossePX#G5b>dmCd-lMq4roVnm8UV}T*O#tm?WDu|+F=2O@k>y#f
z+JyJ<!@2@{oANV_3CGCPv^lSQ)WUXax#PY6+_a}x6Wf_gsfPg%ZO2nDwTR^3Y~k2>
zW#GO0^#Hg*JZx^^AZrAkpXUW2k4P&E$BFySX{aOxb*pI&@gOW0q<MG{NxYy#$#|~g
zJ-gkw@YoNe8Rb6Gx6&PuIskn~R-;veCc7HI&tXL++;^7dD`R$%oXQWIorA+VPSMK_
zNvM#s2@m2r&<;h^=(~s@Mbs)xO~~;1@ARM@(t|<HWEQQm+k0&TNWuP>h(6>J;6viq
zbZDV@=BAt(zI=-W6$O%UypLqKiRQt%ty%IbV_){+!(wwmGA^r_?{O_iN`uFC2hKWq
z+#8<?SBQc$4iDc79?X4PjSzrIg+k1~%ENPBjEWseiK2lBK8$2~WEH0%x7&zdgP4}^
z>Up!_vb+cutv-mN`iP7$@esDSX6SVnjXt)-I|>#ME+PRQ2I9RT3sW>g$OH}1-a>A{
zRM$hcNCA%>$WdG4?lBh`BJ_`;3ikNVr9n)zIDLw_pl#(c7ovC|^2Y{9u3<C9zqWV?
zkshv(dbOh&xX0kg#vtYIyh>ZO9|dV=34o*CW>}FO>j>)vG?(}p2P6*#nZ*rcCF#xi
zTlK(z7%X6~-S?-Xn~}_wKyWW*1a?kb2kR}M12$Bp7*N;akwR?Tj7st0ZH*!^<Xl<>
zG%~*af^!3^mvEpd#r#1*?TNb^q;N}%d+U?ZMAwyH#r{jiph~L)feBC(WCIXOstk%x
zo@<B3BJS}T4K5sZp2Ag_wPw!%cpBgkYgB+Jyuf{MOFQ1%qIJf8KVDyZ2&`yHF`36<
zMhYxmoAq)aQUbRpaYfPzKw)L*c7)*%Uf}T*v|Pa=w1e!p?w&h-u&}Ei6o`Jo(Fzvi
z(2ltv+~~SDW)z|T+@-qXbK)7FmZDu@&m}h}eJk-QX8H#lNY8<QjOTv!z%mdMZrERZ
zQv)-Hp>R;>(_f6FKai7l_SfK`%?l8cT~>7=)BwP>dfn_h+f-OJb_@LH5S?Mw_)e~)
zN<e7=Jw&(4b%m_x9B@!VLv-!&PCSfu-?r#{FQTU7^e8woWP&QB3S6vlc&CT|Cl+UG
zCjNdbk3+_Y_u(?T{ghOf=TCJqr5j-9$#r0Z!0`x#jyf`M$?xb3pE%bI!F`Eg(Xu`j
zsbFo~`efWp;!(T`$DddaBE~@SnYBIt@l0RJ0{>j@%Zso6Fj(#3a#pWPPyb<N&Pal8
z1TPD@u`J7Cs6GzNx5QkK+iC#vyfX7{`hbTII}xjWhKn2y+~^P`7#yq2R>{NzQDf%U
zV;r#uVDML7En)hh@jPK$b};l985uWb;%%GY2$gP~Kp`A8qar*+xASiXpGm%%^5n2*
zAo&*RnaKnRqmfvrpQ@a`N9YO);j-LQ@UkM<u}8Til-QXO2tZI{&9ugwc0adpF0OEx
zZF+etNj<3gOb!4vPNA}}@)xGYhHTo<c1r-{#HH9%VE9!b9!6*FR^C`qY<e~@dc$9%
z*rXB22f5IT%rqQYU+^|GC(R>ZNI)$b$X$Lx$DWl*FvzePQk$eg%aYQ-P3+ONCkIIJ
zz@+thEWcZk<3bOkJ)fxpycI%DZWCO%9F$N83N;9g3q;<w(059P1+`F#71Yw1Aj#ne
zwFTtpB|UOZ49Ou6c2_A)@1=aksn3Kdc$Sf+zvb}D(<%qH?A~3Fbli_RlO%<OjiA`k
zRcfi$$^1dUeJiBUupx{-g6$mopN^1mX#FlSU+pW<V?)m6kstWu!E*c7onrG-yhCU@
zp51(UG!$u7(p0mRPxbs<VA@JxFG2Q299T!#TVHhnO0OPIW&kj9+bq?hlVP_pK!rkL
zG?eOOet&T?hVb?vPLb`BEnyHGX%rUY86-uYUh8NfaaYkgQ~KM1Af23Ea0C2+`zs5p
z_&dW%MozxCngY=Ler?A<GZi+}#27x4Ytg`oO(;N6BLxmliykSH_9W}mkm=9t7tWr1
zgcZf2WfF*nvTb_?t^j1(h^PfveG77u`$A<Bbqd0P2M~xtZ%PiG%>YgX9`*gfMBW>`
z56%`A?<(Z!R6hQ1rDcVz5ZrGH=^9ax{rI<sZpK;Whnjb1NF$OBlz<<^wU7KhD+K+L
z7VdrymM<Z=KSkuW_a|@~ct?x*qZvRaK?o@uem`<RZ~=IRES=@`HMXeBHzlN!1-<Mw
zBllqnDD*yI2j_dJb%r9zq|KLk`cCR7q}(}xnG5nr3M^m>PV(hnnZCM7HB(^^#gZ+u
z%>{vu#kUK5;yrtd`xxZ9Jdh{?3Hn@O*7-o!WBFd!^8idkQgj7yo+)&_0oFHM5InCz
zCJOaju->1frq8t!;p7EO;00mKohQ$>7J46oq6Th5;nbt_Vt_23<{zM`3PH#9p8_mL
z;1M%H*$c#Cwke81=%WS-!tDp4uo>2V<JV1q8$j`yGTdhH1YuY~)bn>ZQ{uvuj|#XC
zxU2>`pkg=?baSJKbP;(nj+n6id{;V%NhT&4pu&}ycjGKY>QII7BM8^s*TacM^WvBK
zoSy7R=vB|F*~;8(HO0~&#D>=nQSd{KY>oPC1D9)tjGjtO`>D9#ZruV7Z2>IWMw!69
z_Do;F5Q%P2AR&)zYW7Xsj@BEbH)9n<A0vT*hVhfvp);@_;LI@hcT%<9$21ilJ1>O4
zAs}Pbtz0ODX9X!<#=bVFlmdri<+fu|D@k5R?F;9sqOd|6xJ>KCG;W;5M$)e5B!uCu
zVMu~z9vAnDL{*rL%`#5Giz!{##Ad9<OT^`e^VxDM5qA8Eg`70H8m|L73HbPnp^rf6
z{*_8Gs=Ht?S5nEd#UrqyZfh6-K)f-D+s*`?MTz%H<yq?!!^b)ZpHRKG2?fa(gqk?U
zAb^c$AVocsi}Cj*juG%Eb6grwYKmsFg;GYA5+)+3LYL&V7!X+Hh?;C&g8v`(zOpNi
zt=qD2_XO8Kkl^mF!QI{6-5ru3A-KD{yC=94+=IKj^+QhI9{t{N`WN*5N<E{fT6@V{
zbFRIM{oOAo6;SO{@kJ&q1s~JM1N8}@Wla<PI+Kv2{T3KiU7)OHk%>Jrv!su=^Iz27
z8Ja$RF$(neVhIb-850O^X93u~sZku5>o4UTwme@6K}3&#5r&h$I~sAUNL$^z?Kge<
zf;<an8ML^hpq<fVL0YQ=>eQbh4aouQ+ukQKlERc~$||=x!sLlVzWe=QyL_2WTSwnU
z*J^d?p<sdU0^=BT@F}}Hh`z%szjx=eA(4;9<8~jTznj?keK6Bokz8E!iOtI^2-b`T
z(V`#L`%XE){HbL|1&w6ZZ)Z9OHgz7P!UWs%UOt-r(mba`0q2BoIu9f5?@5A!1U8)2
zcM|-OPt#Zpe|X;xqVKbBGg9f61Z#<IG=VgVrgLst0QEQGh=gjuv)?P}FUfauvj<A2
z9PjNnQN%$s?i=`U((+&PXw>rG3`4iybV~z>m8BToKu<UZ82gKTa4rb|TiDC=7Afjo
zk!ud%{_C##Ab?Jc$_{S(v;i9?44}$}y+xuxl|NqRCM3{(+y$!N0>rkE%XhtK$YB6+
z8uIYl66%Ao`}V^DflC1(mIkz84)*Ou@R7j;I0zEne6kyK!zc^@vJt}R?dMHsf1eWn
zABY56hd`+1I|2#-))r>xE&3ct*ZYW~>^c6L;cbWo65*d0UPd1ID;|h-Me^fs*uw%n
z`&A$X02LPVpBHQ*NCLHDus<XH@%%V?Hc)}!X@kFIo93_e1?;&QVBQRX_tyZu|6_pv
zPdxx#TmN<Kb%0*)mAHSV{38y``)`MfEpI-H{qH^t+Phm*TVp)oF*g(?qm|cl9&fUq
z_QgR*>n)HVNdKD|fobFgjfIQ;@y@}@wThT@KihcHsB+5fQq=^~XkN{!sp%lCubiSE
zs>~kg7TRouEBZg_w&d*(s0_OBstO1|xjt2^Kxa7BgASMYEF%?*mMMqvjQs0x>n$i$
z|1&5uuE4q)iG1kB*Eof4P8KnKf5=DS6`^b0^yDlbz5ns!2YOu^&g6S_u3ICIhVd$G
z6vy8V@y2IM<6AAd3guY@BWtP^F(W?7im!hPL|xRQVo?GbGu#ytMzx3|={x1ePG{+e
zwE9*{-V#C~sE7T(6T)W#!24<nkLkI)@F=<KPrl?pwJFb8cUU~D8pQ1IsnzV~9sWma
zYF2aK24@0R;-+6o7%E_EK$c>;<&TsU+zWS=!*t4HnALZV$#~3W{5NeQUI!{(Du(N6
zaJKIPjL2jDM)5iNe?+nS6g1#@|9Vo8ZakN8teEeLwEyx`pGxb+-mY}-ES6o2k9v5Q
z{<~Ht5nX?pdmD&v{VLYQ4q(?Gxo{R4f63vrCy5sNJLo>1`1k(rzyD_%i)91uwo6Y<
zM8cM=sm*s^;}FJRZPA#;YeU{qEAW^no1$`7@hg?2X$v!b#fp{TfBwYpdtYu$z(E{u
zsTw;(q0)F6#P(ztii~mZ@k`Qayqk^tspv={EcFrj)90B7itV~DPqk*1Wv{;FZXP&J
z^+I8tdsl@x)SVBPl}C}0ABW<8EbeC5o){GE>CAcLZcmrurTS`9XwAKnzz{hUwa;b$
zfm0D}9JX0J>m9$iR4d)0#~>!~l9VYn9ZJ=uA#EImuj4@n>pCgt-g=Zv9Gpe+8$gSX
zPQbvOi=Em=M%Kz0@p&{ku8Hq6i$y|Z{z_yV8D(c*6UWiyJYdsk^Q6>Zb2m-*=={KQ
z?>+0~sGfh}INJOCH0#zu-@$x){JeyS^DwBrAyHkxMYrNMbvl%=#wIx^EIB+m6Md~i
z=65{*5zp^whPx8<&Vw8M%2k<udY9jxj5}hvM|^1WhY77aHf6RZa-***4j;vCJ52}K
zYo_r_4M#WgJad&qD!*>CJ-0v7FP!yiYK69QyDb?V-d6m9>t8s(Znnvs+xBWWj+3$S
zt1VUIYi@bRY=dt3;B!s?`t0@48C1KvXtMU%wW0ssT+hXPt-3Jn`(ZoZ1NYhV)T!~|
zso%f}E%(T6R$RGNq?n7)3*Q|9*$~lNEvu0HZ(&sj)-_HJxL<Vw>n$;EgT;v%7|S0F
zcJ^k1(o-8hpY9SSxRJ{;rjKm<S{=$1v&(r()Cps9MD9B#KAkOm?OM30E%?}W6T@w7
z#%d3B>>trmQt9S8N`AeK>4ey>c)nUYeLOlc)-&r<5sT33X3w{%R(>hv7G2Z*bq)Md
zaJ(XOx;Y*PWkpvqbZ?7rfQLBN)=R#_*ROi5;FlymnVE~F#cBMR7HR-W2ok{OZd=f1
z5xU7C+*~KIxKZkQFzC41<RuKvlPQ}@p+tm#+8C)6uNW2=GkkhLSE%P2z&!lO<#sfj
zvpM9H`q0LsSi*XSa&fhDxp8b;ox1usRx4=p`m%+?<a9UfI1m#yD4f}L<Q87J`lw94
z>fPy<;Tzo=u}5FIavMyO^3XuAx<k`uxo6`j_V_J5dUejCpd+)5Y3cMam%Y<U-g_nY
zS=e>!8h){d%E@Fl+-xhe^X2Ab*uzO=8{YBrcxj$O=_T1ONVbslI-^(pQoLhZmZGUg
zM{tVK6DPG;Y8!dyRhPu!?dR6V@~OuyOg4erR13c4mnm4zm*0y4we6W}KBg%nIG2G)
zckRrtUQXd~S(Vt!QU@vPL*;{SIN?C@zvV4Fz<ngS>&48uDjn)L&&}hCu6UJUS5`pZ
zi4)8}gJ2ff^jC^XegS7|+hm%HPP2Wn4sYSRAbJ#UX+E!Kf>evWq}Zs&^r@LlpO2oH
z8WaS!&i%jp1IHv^?uVh1Uw>QGcs(NDpKN}-LG+8gNgOyVz$l0%lpDu9)L5WCJ)|{r
zHMiY8e65RhRf~;kC|m5VG#xV~(%Dq^B7dEW=TAYDD3uYXS9y^06LN0zS<egT{gZq;
zUz6cIX0da0qfk(B*cLGIKFD)uL9HHsxlBG20Rt#=cZ=bfegs<r<J<XZnR5$wv~rzT
z=5ChV%<0ydHh-nuykGoilkS7i?0PSW<6%O?d3WZRW0rDF^HRB1mf+GSUMpIxkf|m%
z=&*ATc6v3_nmP2<4Qo-uDzUQ0?C7-}-6`eNa_S@_RRjWVt(XC<%dX&<&lAM)by(Ee
zAe5VdGTV?>Y(Z+2TS+^L!LVItDz(Pr)&nC%{%VNX1EFS}6ZhfdXV(nh*K5g%8-BJ_
zfm7ylV>8g`5tSO^!FFS$@5F;_+xA+QF!y7B@+P6?hJ@G5R)7B5&`o6O@vYKnK)5n_
zd6?#!aRu8G4MqBlyr(Qau5y9O&e}@s+?vtu+Vz7vC;7S;&;CuAG@j|g_5;C$pD@=p
zCgA<tZN`+n+wK~krMp0F77G8;_Q&hqW=3vS!Vlsk<vcj9Qgz{^vgmd9t-!jF{YN!F
z&qD&@^u|*<Uj{nU&2~rZa}!3bcM{S<Jvl~qAc2SG&F`uC`|-4CqB&;D{GR7XkM|8&
z;*e07i-L0F1F*OsSbXLo@6Yb%j$NO1@ob0`w^oYB@zzSQJfU^Ngv|k^VQsXDu~JvV
z*2oU613kwCw^I7G2MO|9pLt*8nGKE0UvoA0bCHf>t>4;~IP9)W_bpH&8VQC{`_?EZ
z(pM%ljJZ>A{r9TkTi39$CY{ruKxZmRjF29CEp2G+YOiQ6uK48M($u*oiR;Xa(@i7F
z1>opkjof6|aOaH}Y`3}mW?wx2ja($_iK#PQu*<T2OX%MN3CQBGq|V&oh0RAVVus;+
z6gji1H8@Kw_+YtVAQ4_$&cGGUxv-%fGejTr4Y(T4>@8a@x+ij`>!wc{9ibLG=^C9f
z`COLW)6uf7QSn0fp0>t5hi>i7d_c0NPSzI+`yy&Nl_l0`g)5J!b!90}oF7n++&N{S
zcst_z>BxX}<#feN$7Bxf`n2242nm<P9TA_aK<)g>GDFt0UZ%38nXws!)wFrx{saH>
z6hi<u;K$gVJoa_G!Urs0_;BhfIF%N+zDfmhosMzI0wB24g~-Bn@^n%Q;{N+M(*<2_
zHXlpUS&qy0<EHJAbawCn^Yt7v;p>P7Sy{k-xVATM2j}{T=XotqZ8yn)L~e2XHW@jU
zS$4TqD2ct7?(pN`=)xHDVsaGdC`kgn_8&}jqHg!S5#uNBr?gXboB0Meah;+aU|pQ)
zT1#0}_9)w<Dkw7Bd))i-_yj+cDtgeEj92rrDB3M23cK}W&3^dYZa?aV>FqPn5ESHt
zY04_mKmtB`IQz%m++baRlj`#ALDck@Iby*%l(eg|Mrd$cY;*s0?#L7GuqX2qoQdie
zw^4jN=a0YKRwUZzO0^)ykF6mRy8MVd?S=t@7o!tl^OeR@ZTH^y1}LeS0P%9viJfoI
z^T{DEJ9|_o`*Mkb*3MfH#Q$e&be@L-)Ztu3v_XSs%4XVy_1v*VW-OfQhYCPO8M%T3
z1ZHKMkr0Jp^qmXK^Y^*!IP8s40jmT1JYo9oOIp?*i>cWk7Wa`@@1X%M{M~}Wlgo;R
zWkiRCdXLuTuMP!k8b^^k1#2*xS5=4m$s-8TFrUMM_D~b;V^{Z<3tvoJkLqTHxw!(S
z;~x7L8MnB$B*F+qEiaG&$3?Sr(?Tf}=8QErT)$2I`<q;PM;@h9#P^f6b11a#zucy?
zZj)b+5_Pm1m$FGurPD0O84m3$Tg@U7H0P2xT!E}xN@v2^IjSg{<mUk1pp_yO=FUgL
zlhy%k90vC2uom<jJZ?2#XY=p6gBR&LsgJs=_^b<KCDyjVbAAvMnJbhBD8F%?(wqHE
z&*2?T{2Ye1>WY!QJlBJ;o|rs25AQlK$w3El1hNi{vkB)p-~ImThMAl?d?<N3_^NAg
z$t$bS<_2omn>^9}1LSgm2?%iS$)ga6o&CZ)_2S;fh1_ExTBB+N^tGb}T_dVpqaNgl
zHd~7cqF<A7<SP3o&5+G0xb26mfUI;sB=8F+i$a^R&G98_jIjq#vAeq2-Os|zSD$O)
zp=1w7?%W7ubEm(Uu`w8Jq9AFD`aZr;Ea=x@>^I%{DiI=R>Q3NvfC&sQRUjqXNH9O9
zId=F#7lN7TTGZa+Yvkt;JQBG-@sh7kkuY;1y*hv0gIM#XXwQD!OkAroYfA4Laq}pH
z0T!VHZYl2)^wz8<yu@=|ZwTR@qX-H9d>+zbv#x>W!atd$FL*ruV|TgUwtM3J6IydD
z@G8Q2iE$3dbFJ;$`6|;=OKDh{+5S?|#-X3r{)fT3yL7d|Yd>SM#=Y-}$L;4O2C!Gf
z@Z8~jabfLsEprS`0q+0$4HbA=3j}e){sOv?%do%;3;?{oe%v`P`L81)kJ6#q-v7et
zkA6@?L{w%_KtyC<lxOII#6bP7nAggk&z&zU&;ltc00;3|uo{4XkcWf%N%#Hr33`{w
z&F!?~adoMG)$1OKXmzEPk^3>do!MzD&1oko=+kHLyp2p+Nq<Frx}Iom-|yKoOM_$d
zjf0;HGd}ptj{Go+=4NvgTH>Z2kmEm!2lYTh?JAO@H>r`kcy5W;m9&XYbj|Wy9bB_V
z34SO~)IrpZarCPzS+_d&+aDdfRT77a%`&nt^_W}}R`m*W2&gW8Z^j%RMON}ZDL?Qo
zsGlBOJRmsELha6n`7}1#EYp;lwv(UI>ymVyJX1N^HpI&PI0#z$+`(-Zb<_);0XB{<
z$Hi|)XmL9<;4T|^n0AXv7waJ7b>IY3mPt)%q`415X$gIE{0XxbF`Ma$5s9@<nOOJ8
zY)V{+Q)_a3w5{46OGu8I70lbg7`20;jk7~h^bH3OmXA|s1FU0_YS8V(m62)WyE?wr
zb(#D_9`n~eOQEAvt<Dhbozs6Deq82U=8k#zJbhy-yv63^ozs}wJf>3UA4x>AYbPbO
z<S*^)>$QB`)yjJ`svq6%$m}O{OG8{}>T*F&iMpM6tae?+yBHk0H3s$ZB4gcNI&RMF
zoSSL2A07ke?=Bnp^ISF@FUB+dJ~zuEZH+mi3<o?v=F4ncTrA0T4otVFNOhy%=6jT8
zcC3-woR$=gTQN0PYWrT+GgfNwc!sv|=(q9t9%z_JtZG%nPdjE%^_y`EuIkkZx%$-#
zSF}AtTX1x!jquc52&_;OjSL-4r@Q6UZ}0m8x&q*20^s6F0@%lQjPC%$xf));hVziK
z`EHT=14ZFS0I_0OlFx5fc+l^bR?$s)Zoma0RQg9qZ@~a#KR&6-Lt{cJVGFk-A|5S5
z!X!j;89FdB%HaO^@GNt&+S6$AYb3<;EHprOH-B?`<kB&D+W66wRo?q*LZDi?6xUJV
zr7gy2EYi9#ZOd!f!&h_7=H(pGQpT6y)JyEfyH%;8v-A5qpOWB(<9m<6$onn7{#QAo
zpsKmUFVLR$9=#AD`Hq<hp6hmWX-j5hGz9G2A0Yu)Ng0GVW$=A4D#d+=?#n*OetfSo
zTw*<p!>ZNSM%?dq?w14_%x+(-y}i;`!`$*`z6pC*4i9uYjCu}wA*}vrYt*HPq_~NZ
z8D5JwU|j@l5i<d8J1rRga=<H1{fio~R_Jkkg&$=p6w3cL7)x+C_$tf6_v1N_?fIHn
z2?!@fg9pMi?7u-u_(T375db45loUV1*D77K7osmmVr!I>0`}bPOcpLz=y<vPHNazA
z-c{1IcD$%od~(iZ!FH2qwTarelz;<nc-iT+D0;12CR#ta*nMc^cTmp_$N5&k9{)S&
z&y<bj=gFau$5OS<+_~Ug^PwwOi2d)ODig)%ucBHbl6*Xh*DfEPZ0%%!;O}lnw6MiP
z0zB%s{Raa#<DCiI8Qu~@ZnoIhWy$wU?bE0ld_Dn~j$Rc><P#|{$s>>O-H^;5F5_H%
zuhV`x4wH&*C;OKFX;End?^^DV-1G8nSl4}C^O<Bt5C!XsOa=E(g}eO@z<u{80)q#9
zox%BT`sq6dmz4A-P1&lYrN|ozvS2VL?yhyXxh3(eA^rcjP!EtIWk!8kQIo7)?etb^
zuM|oYQ*5$6A2=UFQH?!q^rc8#iPKAw-6vXzMv8+oh4XtoFNEZU@DJdT{;1!uF75a9
zRLas>U$^>fA5^%N?HMLbI^y9I-;BnRov7^>&yulBpTo%BUwgm|vQCb|w8?50oFk~o
zvv|C)A9Oi~a(F+<c74Y5D|x;pv+Z(fip31Tj83Maza=VUoorVfli#*yZbZ0RDb)KM
zjqY+?!9~hF%qj{mhMc?F$8z+EsJ(+r9)oyUU(qVR-Phs^rt{(m0eNAXgXUI;h0Qtv
z??xU5uN(wvm5@*emab3ot_RM_7maUhLy#!s)~FSFc#JB(YNz(?j4RsY*3~Cyl>jhJ
zoun@a?A1HA7>}ck!Ui%!qK^Ui<T7CE3<$O{AnOY7KvDufg{l0V7>aNe>m%7F1Dd5i
zn=FF2>gE>z?E14EuvuTuW9#O!k(&2%Iq7;P2KX-^))h9!x9KgNoi5djhpJkY+WeB^
z<_@d*LiC>1oB>xj3Fu}gR?83$n%28{D)L5N9B$0?F3zmloBo*RtYwFH9B~IoA9JW9
zDbGrN*_nx3P>2cVhkzDB2$fBUXV#m!wPCl*OY;*YY`1ywZgP&t4R(zUdVMJ!wahqJ
zRhCt}IhSx>Rg3?O5uC?#e3&E!em_zK_%mJ8F3Be%X?i0f8bn7!a$j#x?+n98seZz{
zu28_$xidj5`>maMt6%PTp&n~?#Ws?^!_TgO#8S9_dIlFD5YBZa*#N)_o>ag`Q?B+8
z(tF)dMLuHks@GLMUe2xMFN}sgN$7X#em)QBgd5A8^eD<0?Yb9y0Ej#7ez&p-Bsc0a
zpg5&#_Hv6_-;hKIT_Ryrr3iDCfSqla{nBsMYbJlVy%0G}L93;=@41vQpA;XbNpsm~
z27SZ<a{1F4OX(7<ht<V!3$nf5(`*ZUD!NDIbc+EfI>Mw>2lt?LoLw6_Q5L^ztcaU`
z1V=XAt}VExF>uEBHOT?i#l!j~)(24M8=V>>${x#jCR-BN5_y>1c$wVpulqCW7j}D*
zJ`J+Et$7;Tru=e)%O&9d<O3kTc*!U5@Js0u@QEV%zI;`thV(azFF+Y<C*K}(Oa3hn
zL0p(yjs}c)V87du@A+9#*)f^odUQJ_;oGnSfm3l5fBRaiXZ3zV4Zypq<iAta3k!fe
znll*JwvlfwO2iw>8E4-o>7+IjN9_5|(8YV=_d5Lorvg(q`IzSSov7{(TmFMVmG4-7
zGZcgzKJ&h!3Cf!FB5=uiaCv?16i0es-j1>;-wR<Xi;BpAZOd&J+=a$b6sG_N+=dae
zW*Iu&AKvmiJIJa){5&#CE5T=TgFjk}A+*yJB`mtnTkRu)g=vpR0ifF@P?5)ny{fHO
z&YEZ0a&2$^c+B`;&i0>7@_|2y^b?KLUC}N{bSyYaBn4H)9dBVkAu*w+iDol4Orw5e
z@ojNzUk$ls=x^q{1_-RjTc19Bl()Tn2k<G(b#3j;5&2DBe?gRIFUl=Nz?z7vx$#L`
zBI;}w6Pr2AIYjMe&~iL_jfy2|u#Il(XAO1!j(5U;L|S#TJ!kSGg&>HW5N>RWlb%mc
z&uCTZ$gfR9#Uo2h!x}oH!S|{8ifWqS$0_f4oFc)Bfl}*0z$=^Y>a$VY^ri1La#uDs
z+_h7hr)l2w7@yK&@$Ed>NYpir3jYN+XsTRUc2kS*LHZSXLjajVpDd}rTlz$zkbw)i
z(=6{Fe<3~u(IYSI@N83(jWV+QKg&A3X4k;T3-)dqTzfe7TfFSXA5u-v33|-?qb0$*
zev?91hM%0zJVXW-@sk^zL5@Dngut6eHYdV`p#cnMcVeQm3vDMxPR&uckhNLr3f}rm
z4E3AD<=Jdl=?*6go@e&k$Lt40bkJ3@oe)VI`oyR4sO8YR{gM-V9bDceOa>Y`(BJsQ
z(?h{_+f_0qAh0)4@mzZR^z`Au${C=Q$OZ4Z^lunhlg0?Dib4NTo(N*DGp>Ltk}CE-
z7=!KT2JL=8c30Om=3S|huB~rVxYW1TrH)LM-?@Y(eEd^rsrCfH)v5759t*B6+w}_5
z#)C;3idlC$7t(79t_=!Hs1;doZeslb{#WjoB?^HAbRt2#9UgYibSV*Mvig9JMZnWk
zsWGn{n0`>o430u+ZZp{dhDjM)mCbLyO8RvkzcUe=6w|06&fsZrJl^sr75Jo;7&%xn
z`fEJu%<_Br$FL-brW5avECvOGI0{2)7{ASS4TsWfhuyeZXNP_doIdxK>6QwGg|w@i
z_)pATkA6`IW6WhyO*XJm5Q2eZBICft`lO`IYo?OCViZ@FR4s5)ai>0_(OQHE@q#Ya
zzi|s&Oa>5tn1@@a87z-p!+Q_eb9(P_!~jOwf{C1-2f3%?=y%N&kP1QAF}+bvzqUlF
z=XW=_LGWrrGNsFd)}xI@zJN?g#&9Q|34S@;H-s`aGEcFPdxz;}+PG0;eZf->luvcW
zFW;@FckEI;bTcfT=V}5~da#J|s8G=R-tT)*4Tzq9paBsYfDmfW)5gyuio=Oa3!Uh;
zY_w2*06LGspb7)hAEBa+W$3DhFC*Co7FhFpj>OL$Q6%0=b9^XvFoQW}tJ`~YJ)Sk`
zyG8{^=M-!bHi8|!@%YMmKv@o1i*0@;$H&`RGQO-(@DW?u5VL!q>?odsOJM$v=x&#Y
z^D*aA=h|8?C>bgaqn2rcCe!0(!(95tL!&y<YqLWiif>ofQ$ZU%k&2m%0QLv0!NIoK
zkM|~gQyw{ycQ@1YGfq1bop?)-h~%QEzG<DCqzZj+5wk1#5ey(PV|0Rdl1?&V%8v$o
znfuz%CeN}W#}2NB^xucx^E=5r>L!ia@tP&IQ*Sp+vV#IqE}9lK5Z^a+VVq~>lG9+{
zAIB!|m1<3(5FE+}D=w9sq^|b(7E0<XEs;+NO0P9X0)2k->Wc`6O&gwTPZAt+*$wZ}
z=9tYD@7c7-7H87PEa9Za#ajIJX<{wDpbtteK_T$Hd(jCVz!?cm{9DTlV1p+e?%pSX
z`xB*~MH}dwnYLf9c*J6w*On+7fQ)VZAjGHjEVvZ|0`c0ZM+8XVT_Y8}#tU(3Th8^e
za6*Yjn4GFLLAqTuO|Pv^IYRioB?c2pv>y`L6+qh=?I&KJif5ObHyC}|6r&3qy^3Kz
z>=x8B%5Oi0n6MgTMl)e4#=sgV!d)tMLh6-GUbn(ZYmo3bt21zIy#G<1K@s7a1};#x
zC}G48<<A0~Gr)cCLL)(ab6x#+z|&r^A7ZV`_^O^LR^G3X?jO4?)U{>upvN8y6Ersq
zc9(Sl19raa5LFcW)DF286-uKMx#WaD@X0}OX|Py|855Zzpr%jciCTh*1go<Zj}_4=
z6qE|W0!iszWjyFl^RCoNj5HVF!3)1t>5(HL3nsXEU)Ft2w98t2Z9DhA5GIoUu3%Vn
z0GZPpiV`qJ#iu6)Lx+tMz$OS=gOCT#axq>Q;em}gLDe`Q#a`qn2)h09&$+^)cje>g
z9up1vFU0<h{@bkg?;xm$jcCbYtNP$)H5sX3!MgSfp{JMF$w%HNIgsc$_RdUsKL|Jt
zCuPPiA;rzm(@kZUans{eJ~J3)q-DW?(k&HWd}boh<~#pPxc{q~NnKl444mwH4rI?B
z$CEsHE}*Yk^J1{%8c=I~Vs&Aey65f#pvGIKe#H#cM@U?8j@)W~8ETS7)9*Q3IJkA0
zH@}M`h7D#$tR2ODfmN44w{j)s!p=$asrQZbzFnB}Tu<Cxq>itg^tk!xiD$|W`yn9M
z*GhXj!?V>pF?4ZF+)u%8ea`qpTe8m%Mi4W{{K-Oxl<z{j9A`pCj!%4rJhfwg&64XF
zpzYHyGt_v1AUI$mBGxM5&~a8G6_0eAO{u-3dnY2ocsJACfd?VDv<uyj_7j3ns_+vn
zQ_kWqNt|Rb|7fhH9(s5|`l<ry3ZC8J?GoE(d`q-ql+z!d6N>?4*zAaIrV-`!%P_8f
z>!0<acFct)57U?j8As}RVHvX2(^#4}G4+?L6MX(>2F_v#mL6Ifg7S#wf4Z}@3!xp&
zn!`vJOhW*Drm)NB5{=6J%9XX0Bdej{*dq;p<LfyP9x4R%SDT!D|AcTeK2j+#zDy@W
z$x(oz69-6Rz~*kAQ9IYb0od5lWpf%xmMG%vy_YK&d%T*tUjyvQ1fihUa}QiqC##T`
z(l#|kzIY0he>AjGu3nRc<zlR{r6Ay&TSa+1hF`%IPu5D>#I7<}Pf$Z@K~pia&OPxY
zKykv)jApB#`DK~d=w?wk;VSw`^!V~2@Y*`#U}3}smYhW$0V?}PI@~txQnfDV^arOX
zT0r?|R>UH0SG0#T)d{t`VrL?%VjVFxp|jMeeY4Hi%B&T7HGCb0+m)lR!ROw7PdB#m
zUb+(S57?Ei?+arTWxQX<0ArRcw(niviXRcm+`)#f^os8ac5y!1^MGx9I?CA{wMc#m
zwr8x7rAU{H-d3td5ztKj<E`xm>W{aBeo?coby%5N1VHVGgx5fK!;iJ2+NK+%*D3;C
zJ&C!$?a&9CIZVrTg<$POo?3IuG#7N?8U_>oIyc#pR?o$cx6V$KI3lXIrQM1-84@r*
z@+v`XowyPj_R7UYG@ykIMs4VY@qz=wg}`eO5LL7CJ6=|WxRg)X@;5?bY0+m*oA<L@
zbt1A>`nWLF?I5C}!i}Q#G*CeWED)IV?#mhg4<!c;C16K#rm<Hl#JI@spd^%q2PgCq
zuM*}7^>S+=S;nA3AqQEJ1s~L_HFj*ga9<flKV)6+=Cc3xB#~4P@PNZa8p-CXc16O(
z5HZ)TvU>i5N-#OrA}#L8sNi2kr1>QE+mm_XA*eDr-XC=x(TE@c(8qKP-oayjr~Yb^
znzGUD$uO~+TahYDK>XG~J{Ni7^+217^LhKkXCRd<w}y<!M>V4c>!12w6fa!U08e!5
zerU=<;kIR(QJ&a$FH#i?=#`IM-B?i_>ikeW?}E8u6;X>&HDJCaVA5%6yIQY)(l?}!
z`tnqx2;3R}^0GAe?b<S5?JGQH7RZ1YW=C_d=&q=@yAk|KD8?-dM!cbifcgE6Loj^v
z>Tn(07-MvRyr`aicGOP^B@LHnjdTd^V*6#&_&h23L7^SeA4-^fP^ccB3;G19|MvHx
z3pU-3CjMcadD?I=81<xI%YhF;bKYd|YRvZ@1|pnt%$?o`64>geBdbYGLsDR!Z{h=c
z@|0Yv`8u}{?5zR<sv&u=oxerux1WcQ5u&fYulJ3V24J_rT^GyD<<-SlDnI{xb*(W2
zs`?OB@z@W8^ci>?9N4=pmn6ni_IlAlTi~8s+%l=T&_~KVd341S>*bRT6{|~cQ+PPQ
z#+s5g0;>FXe{ylFYY<+u=RXS!7D{IyL>^aM!v7in9TjA<J)&0M*Rk2d=wJfZjkVE7
z?QB7IsOuq&0MX|$zXbzk8edbs)^<aBtQ@gPZjul9tE^Tpa{rj?NiIOV*<J21y5<r^
zUG}U<G9{Aydi=Lqd_K3V`VL~d$kt`vyt$Pu?1uasD9|)w@rdS9#+Ov)yCqjrn+OrU
z^^-KTBmzJzZrOI=uLpfX0}68`?o};~%1?Se_N_&tU-cVTJNA?woDst-TPYSX@ge$c
zqkm6PN8P%6ea%o|@W2BDQm!iOq$6OFcbXX8y4`5Z!$OYJs7S&6nNc3S62mM9-dZ%a
z<9c!D8Whn{z<r^Bl#nOmJsi1=y1+zU6(r!~S2+&$KGlLhye>^C67c7-7N1k8#j*ec
zzR9cZ`$^bew8(khCRe4%a{G3Aqr$<T_JDXoD?fPHI`(2m6_stF^eo$!wwFL@zZ=el
zJeCV}L=^CV&l(?ruJL@cFD(y^5V56%mJFwj0>xD07Z#T$p1au()IuqJwX4z1Y-oJF
zZ{>|lXtWmb&1T;P!QWa~Q5j4yAV?KaH|Buah6Z0bohku?0h)><G??-uHY(TjrHtpm
zgWB5c&0W<nJ6w#WK3A~dC}2{tQU5&$jV|q*84k!Y+a!2cDKw;+;eM~$@xl;NX6oWj
zim2eg(}%8gJ(CxRYzVe3bu_|7P3{F|V9bnhS5Q7s*e<}(O|o7vY=qCAMgbB)boN`l
z%tdBycE6I=V0~n57D#5O_pxa5fSfBt1N2K~j>|uzrFz4<sq+#bFRQ(4TXoS9bJI5k
zXMR_;TY0V>)>OkN+o6xlj||JsaBj(OmsdxrE@tQE<@Cc>?R-o~KBUX??U_af2`%pG
z@ReWB_>Sk>M9+2dMhopTL0pDJjAV4G)`Kr@?Z>otQg_#*#9KiIb(`-1p$g~KjJL>T
z5`J^D%C^A|K$)LSGtb)HcO0?D``qiSk$dfGNdHK?%GeE76r?&^?{(yH75UDWGYo?2
z-INbd^`yCjOS;834d|<5dG=$$D;>-%Mw;OM<qG{>$U&B7L*W2Wiya*7twqapUGJC@
zdK%1Zq73Y*ReZUixD!d0XH8w?|6!8^di>f1ndCyO(!kD8W)OF&?dL~Q%$b*adC}Dw
zZ85KcU~neoe=5OKyulhh0HU1h&r;RkNNH_-ggE0~^K+l2Pgz95o<#QK?!fi!k@_$D
zFdju}kcEJ7<DGeb9Jy+)_fy)#T^-hwMf*&4>yQL5&J!?~b<DLsJd=(8_|Jn@_r)9H
z^+yQ+0rbp2WRWOchnu^U{ezPa3TXvwJNJfgKtpo?C;M!n)4FjFeIM7_>Zx;^v=rb^
zPJrt|mc-IsH<--y0hllC%kZ4V0exYEC7(4ky7%#$s=3mS2zWraq&{9wtwL=>j$XkR
z2+O?>{s<7mPrQ2{iVlaoXfMx4JD2Pj=vcLKHmhlai%|*4x7k9KZ@A71POOdQ=oXd)
zI8|~StsQrJ)mlJoDQ+k6)@gDddx004ar{cH%B=N@h-0Vd*wU039V5-91~baa<bK`8
z6~DnR98TWdo$^IwU)EFa)Y{)*$!~`zC^0n~8&4HqY-eKPO(g%>{MRSYa}G3M(TTy(
z2e9AF(D{RW7!klHqNwA@n{?*NO4<De3QH=-iD}wvi|{54YF9p44vhsy(weWI(PZ+O
zz)C9dqo1devC=T&-88|0KNGifD*|R;Bq+nf|48q@2Yn%^&@aSv5wbvj-|!+2{{G9J
zFgXsAgBTRB8F9d(2PX{pf03nsFN;nhppY@-qj?a}=`q{v6PLT+wBV2m5Kw`2vH9WL
zbq_0;u$eUS`+>GrmLm|szqeZT(HiJ&KHiO<QYeb3^^&dc5TU5=O8N{!>kWRvgY3DP
z_F`d5T&7zk{BwM{3&E~B*vA}D{u3ps?Bs8lY~&wI=28O-zy^fNhfCCl@|FraaAi^_
z002f3W@Ex^R0}rQ>cc$mn3R!kBTR#O{ngR(Nl2@qP8#SZnGkn`#MhdfqwfcyAE&8-
zq`gmXbpDm!t!QGik|`u0f@SIQ80$GnzpKBn<D54O*5wIvW+4#x%#A5ltCRx|Fk}Z(
zy$2m-*d`RWIBxLg$%qKwhosB)evf&-G)IEOAiH_B)NIc_$Fu-(!_cde@7;TZU9um5
z|83FAO8k{30?MqXqNw)QD6x<Wa!>fbt=yg+)Uf!QZo>Bl^u52vvXBedbN$Q$bDbc6
z`0}j1=JODbT>2%PPglX8i&tLMcfLV5tUfr1TGcy}VgdOl%1k90O6lu4IVqse5G7d6
zJOzg-*V6*2UXIz{j-G={(t?Jb(?Z<xO;5<%<7aX|uUC262|N^6n<~sgleJEYY*!UU
z=XOkvLgRz}pd`+8R_W?)#H}qi3pY%?da+y5R}2&qy7G{&2oLG8Z{u?$8oBW6cZOEo
z+_Z!gob+7$2N7S&9|yVVX=R#lK>G@g-Pf48Agy_PVQ!=qGFh3g{fQK+7gu!@ULKC!
z2L0yB1NT%UN>#8Y29KdF3$MG0#-q<9$iVbL;s8_gF`(0+&a4*`NU<mOcj+O0aMWqu
zTs=JOts&EE{$T<Qj5Bm-(0jObYkksvbl{@Nbx8U3L8aHSZjMC2AtMpchZf`T1vRO8
z&aGUYVD#Q$2?PZg6DR!3>r_WyCHr)7xQAVfsNHz(<;id-5>IV@&taBp3x&<r(X)U9
zH-8yWWo9h$-@6z42*zTG2A&2Lb#+-fU)AQ2hZxXG81sS%fs<FWPHPoS{W5-l+b<Ti
z(9b!D?!l^9l~qOhQPC3*qEExpp}MiKBd4?1wQYC`&U5LhIu}THKA__I8o?+f^p1Q!
z{dDELT^A*LblE_m7U4!(6r{jx!@1bOYJhkUg{b1^D=zpZIZ&SFr4Ibnr#~HiHI3qR
zJ=D>_IgNE{`#m5&<#4m%6)G*O8vVkG6!}+Tf9;hDIA%|^=SK{v0nXA57hCy9)e&{d
zY2^XgxZ>-yY{!k6#9t$em=z2~w+OZOFMr-Mfxed(!LrxufwttW1qe`T2eVyE_7{b#
zR1G)A!fD|K!=cu_rhikJ>*Or#ZTA@$SwsFmKATtr>FQlA!;dG_!x$qV2KPom#Iu4B
zeX#6Le-v_FY>sp<4O`B$aj<sH(%FjAfVIrd=M5?lM4F^C!pr6y?>OUGcpc{tRDZ>0
z`S6nh`WI}aImHMU@mJKGQQ~xaMYe6gBkHN))QkWb?J<pQZr%(5xQlWg`-pYm{)j#C
zfuzpx5(k15JY;qM?y@hK^TM!F7<ppisQ3QDtll5ndM!IYtYI~eU-?~HM*e|Szx+`U
zId37lwLViIEGMcXgSvQ+K)Z9QrYD}CHCWft&-;bnlKtE&k1)vC-fT+Q$_lpJ0eYEJ
zNTk$EnGRzbj$a+${Te|;UzDlKwtxeY&iAhLXC$^h&}7}*X8KtQf-{~WBs9@GYK`1n
zsqz6!Djbqzuxh*8?dp}_5#GdGM=DXXlz1Z@`Y$FF^Vz*<OJ`luh=vQKte)A8JJy**
zTWpXugwb=e(p5DqBa4O~Nq+~lm(SdC5uH+ERtWbXen&&L0qZ*C^3r>~-4P2mIzmwd
z7_6v4A7r}Oj>(v8JE|c&aQz!JB0hLgfsWbJA;S2oRo-Ri%#IC?Op?LDMKCZoL=y;2
z*fuIW+dV8tWUQiyt&^?Pd<ER!)aMLvai#|)>_4=X=ecoosn0p_!(qRnw4GJm%t10B
z>k0dacdaQ*w|RzVoHKD}7U9Dv9*baVqx&eZu8jzZg`7?^Og=tsz&(0B@kvi<OLe7!
zguXKP1_k#DB(G0@i%E7Ob|?0ft6tpJ*<lj{y7UVJHNRlmyoX*O*U=>I)SNF)9(MkP
z-^gFRcu?9~C}zrOkC<XpC+e=tEqKVM8j`YnalY#r3f3>H9CRmS^0o9=Ye|V?#E<vP
zn0v0{9hiquEVS~9PL>Q6g(F97+Vr@l7knUT{Y`suoECOV)ltZ_A0nt!wd(s?Pi}zv
z5}u;MNo$78%3zaP0Qesg??o!qVlyl-!MP}FhosP`7Pc+q^OFQ`_{_JRkhXuNcNG$7
z)qoQ-S(=HT6O)PkXk@zovXR^|2`<nmZg~Lat{nR6)7@i(T35qFqI?*>7zX{K9)BN@
z5+>cAcKh>SP8|xb;e`S0LP|>|{g!H4KGTpNA~?T~orlna$Z#7BtZ1COHV4_L9DYQ|
z*cBQeu?SPPAoY8G3Cj?rUb~m)%(YD*)BC5B9vqID`Nea0Dwscc&oa71oCC!oD@ev|
zoIc&h9R9?d5(XN;&-Q%JTX<@;jbMX~4-sQXOJiKYnmu#&d?FeBdEv%ol*IRkK+8YG
zA;SVBa7okoa#yPx4ol8}om4<!zyMi(Dz|p}-n_)=6D>3J{yEtQ!_jJk4g%^#1zs#+
zrIigBfDn+qvU+{ZuIUvWP_4)Eq4KE;g@nkh^i^Ja-Lo=eaHJlUt>gJh>R205wgGMg
zbh}AkaM<I$M22bqYj}G?j{D?HVj>>Wp(fB?08TYHrTuceRvJ`eeNV_RRm!xQbnhQN
z9F}-J!|bUY$_i;UrF`^m%07l{fG?~p#YPdL&;z*AM_5-0-BS86<+3h2gluHlr#(0N
z&U@VA+$VeDIC86hYHXRERM&PWB$AAbCe^r$n&h|SplL4JicRrapz83@4(eMuF-h5K
zbK)c-Wkq5rMf00kn&1tNB~(EOAUJr%86JSslE<V`21f;mVFd(&l9e>0R<kkLAlSv<
z<$?`4pJIKKHWb|k3A?2lDHzP(mHud>|LwdT4)}lnm<I>y1d?#CB-P&25hrOPejpeH
z7eIYVf`RjYt7=3ZZ$dYdTLiek&K-e_8a*MilI{ur*t8nS0%w;sGV+reW{ZL)${|^Q
zrn^uLzar%uD$4V-3G@K80DCSmC?^;<x_7{`bkBSc)o+)|wqk3G%c5fC+a5uKxG%R6
znE;gkoyNz<<afZ2cIas9`wa~HYXbKB6c>U8bSO9W;EeV^4Tq!(Ujm0Y)6)LBSg_I@
zoZDOkEKn5?95nHH$O37&$?v(FTo9X;CN@gm%D(2obQ&wL%Jpr=B?`lazex#blAwi)
zdg{>Z{9Kvgan9=D(qZvyPHcg_<GG?CS2Q@V6#jkqFdK$zD_%nR%*NSzQ5^T$w&Qud
zngkCYPzCl&h}B-d&oZ4Ma>#dQzXSBAnELbyM1=XCo(wkC9>51Q5K|XK_(8Yp+`fOO
z{9^kd@{{Q$k>&msg39uamNR9qx<^-(Yk#7a$#7VvMXG@|jMwV%5GIxfNgS>au_~T3
z<xOomN?K|owU^ZVaZewPYh`mB(LjirZVGw|)}uzO12?F}ipDSlk<!K|NgD0gt`Flj
z9Qd)kPR$pVApBVcDsoXu0Bq|l#?}iC8nB)e@=F|b8=Pc$f-_mU$=c!Ayx#mbVyzTi
z^M<7bn~lIvhY?Jm<jSr**P@GURTkDvwcal^2||X3j9HXcF;K83o!GvM4mHX_7j9!S
zbjG}}_j;k<fnNmE-^PH3V*M72R?VRZW^VvuM{{2DLjt>@1E~imSm%9@cJhi-Tr=54
z^6o=vUWjHpqbd`%q0m4HTUBI)0%I~7L9guknz<p790etZRAkZ0<aODOrw9S3Mo4Mv
zR-ZT5#g~QX@#s}`(rFcI3k<^vK4^o(mjjb6kOp25f!XKVVZ4A0qCbs<L7y~}-H8&!
zUTd^!bbZ&bIcG+vrYi$ZhRPQ3mj*GI^9)?(X9fod!65X59Us@u`-f8!6GSrLnZdBA
zWL~Cr&8$Z2+a)t;fpu_^QGYmie8WypKP9p&Et*jlQ>+Iy^8zC^Qq!CG#uZAKIC4B{
zgAZ|d!5}2=C<ZO?WEjjUe8+D-W$T_*g*+a*fIw4h`y&xJ3;H@c2~y<<@bG=nn{L$K
zi$B(~62aofN?)J@3<TfaBgpi0J%yDoebfa5&RgU-7*oIUohNR!Z*w)*3Gnf5)_5F?
zb4is^evKDBhBPoWuW!&?icU`E9_B6c5VY@+Mlhu<L9wmOrJSELz!O66StLo%xRh=_
zDtOYo?6ljpr6nUtf&Qb-|9CJjVQuci^op<Wm(f6x02vH9`frb#3L1mrfCAij@#DRi
zsDU_Z4V{xxP0jvVTj!PBlnvTw!|XUrO%8_h7@JpcEx;efFKD;5RM5uPhjzrN>L-sO
zlrY0yo3oIzI}hTSS6mS_c7)+|r)ALMK{q2^-_Q>TYPPu@&hSE=VlRyJ!uufg-{DJ<
z-iHxKAMQ`u7VyzGIYJ0z!2nYhpCPs%Hh6q<*?YKh2c+uC&~ONg5q7g$4BY-(AAvoH
zSt1O07LcJ5dz6d2?=lv(;np1V%DCVJ|9sj;4N+fm^~^RQYIjFvdE!-*o;KCk;ZTkI
zc@Ts8-Lo=z8@@m6ZU^DFU+}({>ts_A_AG;NAgV)}rr`Tpa_J4UHxk`T^DV!*N4(J(
z%aDK5OVi;-(XtT&5N_nN17$m%Tsi|x&oba)5Cey=C-{@pW+fnz&@<-yXP@|jhwn-j
zofLRh<qpZNE_nk=(N=v8fV_(U-kc%BP0dj>d+8^2+vPkrthW4I06+5>a=*^KEvg+f
z_%vOH42wa$dvR3`&s&Vu+5FNWr08TW{>aq6lJD+vjJ5s_$a+4#5>WAx9mJGeFLtHe
z=^R{d%=C>(fEwPb1hHH0<%tX$$pYoR4l%l5)z44DrCvFx*m;d~+Oqss*-XFkax^&x
zWN+Z3k@`!+;TzZVi4rUb;{6$v;9!SDigquGKNFeK9&9M?NxkKHzW=8kssWC~@aT_#
zzKZ}2m0jv%m+~(4b{(!1aE3c?2z(2vuQDV*Iy_c+*|{Plt&H|lNX=(SpITEf=Gb*@
z+O=r!Tx-FqhoJ3v0(~Wr@SPE!FAO`sHe{*-h)_EVlTsVpd(gRRzWExyV|YjMvlSJ)
zVsHRaptHWVej&1%f)5Gd&d^z#gWM6e7Gl))Q$AK%z!@aTkn9qhb;4>s-k#1(rWHLq
zpM98*`iSHCJe=S`K#Iw>XZ|t&bs4_G?zvRcUyUl`O@JgI1j%Q_VC?^-hZ9eZPq(1*
z$FihW>i#<B?0Rwu`6@GH2^!^KT)eaTwn`4-ESVY;1mzmv&3E*ovA(7^ih#yK{a$&q
zWc1G32e*9D@Z6H#8Q?gQ!6H{#xfQ5``?oj2f3U|d9nm{_mjG1h50GhaSbufxYk&gc
zM(ukRkr{Gp)K1da?E`6WS#?eiK|@eCFjbi_;6KYwNKNkRdMCx!!|anUcq!HY__jy1
zPIvZnWGsn?{#E>96+nt#uhCy(mmKx2gm>wnh;I{W0aeFW&-(aqU|F{qNXAQjql<B{
z%<$wcZH+Zu!IT-wZq8~1<QTgZ$*m%yXPhiX6DRR8j=rlr6*M49N{Ji!?|>8A<?~wV
zVGA7^6hM>2lr3uae&<r}81R;hR~%xKh_k<Hxd{?TZuf>c3JWSmZSr~QvpjyXj!Mrs
zq$%bu`KIlDZDNYWFHHPbJWYBNPtUO7K*MrP{}d+sxn6~43I!;CWYNr7*>+jHn+;a(
zqWRZD=etSJ%<bLmLdd5Fq?{Sz4P5PS#;KJ11h3CuJ_2n_cTgTSKUsWc*Q0k^(@ZfG
z009G;qqA2+`h(4b9(F4JaAb#tTO%=mneuR_lj?A%J=+JdtNylRpQ?F6`dty^-$8`1
zpnqR}(DAfqw?uxHsSnSfVf6n!Ye?#HIYi8gO~fe^R(O<+ue>RvE?)|ocD>$@_hgA*
zlHy%&VMze|!#Ts#f<WgUbnuANhME+LyaUa!Fbar@1inJ`)b&{4KGMm5bJuZvj(4oq
z>_W_zpH4kuMKVlQ%x9O&$YlJRl0WAEGxy>TDbWR@>Ooqxe=A|s5U&9OaNXzp4Zi=2
zq&n*%Yx2~_1h@uL-}MCbQ_7Ga><}YT%Rwcq4;*uXJ9_Zf*^uEw(YCDTib|pAMYtGW
zeOJhSBeQqJWoQ$X%h}&^bQy1}U%HAs(_LIO-a`!x!_AMwbh%z#x6Jyw?y`O0BvCGz
zo&(#pQ8+}nQi)0b-u8iGAL7YL<Szi{#fAd-xp+WouK>6$_+O51`PyFj1WpU+3RFe~
zugalC$Gpc?&25ibgurtui?gcq{u02$vc!QBRa779_BLf&NT7*U`MLsSH>4G-n@|F=
z1dp$O%u;*Epux8N-p4kCE)sG_R6<?H&HTEz6oMNHNSiKI=5-mTcQGG%7{8U*yeh5I
z;`4Qi<N-B#QB6UoUp@fdkAsO*O0?Rc!699AAW%K%dhz*z8}qh>@^#x%CZMe%KEdDw
zLaF#|kXWv+xEzXRQ=rXj2{}0%+c3oNHynPr9M|=`bc$DekH1SH_xazeqcxZS%GK|o
zj%vlz8Rn}%WkD{mR4)UA@>eS6_;fwnnW$2A;xZ6{LFa^*_6lM_@+&oy4*LoC_l9eC
zIb>u99a^61tlWNTT(i9|-znA|mA-j!x@G6CyLp&v=ifhVzR=DZ(T|#@UL&kjUE~#9
zA!2+6NnA+7vqO8HI#=M@6H&c!JBhsfLmfI^S3)N0!H9z(!gvAw&q1p32@2BVU?I_k
zLRYmoo>~I;5TCK^G=vSTu!OFrcP9>z^cVr6zQe8}jQ8dRj46V_Mr}AR4UwRiF#<8s
zgQ1gn1c?E+SX=7iyPJp|tEltK0o1=Wk?G%S!6j&HTPha(<JNlT>jNKua8O6MXeCRB
zdPt*AjHG5PpZ@c5IH^T#s?;E8M4fQk_^=)sfF3<QsVJ)KFKNNhy8Z?_#b?xKA?kW7
zrPVG{qkurqdglBJt+-FtEX}JyH~DVtBDyKf#bMPp{}<s>Q|jza>LKc3hvO-xrOPrD
zS8!1x;P1q*Q?DzIE_5H$we<t9BEizK!C+m977{!(5nyN>SJ&XOZRqGbm0Li*-a7dj
zEhbwnoahM$*h_~7)i@CVv0%ae<EovNnKQ5iov#<AOq=)8aJ*VB>Amb(OR2FrLIH+V
zuNVI3f!8SVnXcf}XH@Ec_@19`7;T9({_&H4Ls9l};gH12M~fHbt|uX;tHUqLrS9tM
zR=17Z+y0K&e77~<AZx+S$v94YLBkD}*`-c((EJ%zBb=1f-q8A*P8WU(p(EAC_rY)~
znM{$tt%t$dCvY*fFC`}){5rR}ZEWL}MyD8FI<fNYZ$dy+7BAJm#AFvQ*>`lv^*Eji
z4f?@sQ{fDs<NKuX-xm!v7oeH5?QYHOt!nb|pFGhp=&^r0JGg&Y<p3`U;*YRh!G*^<
z@2_il7OFdzHC{F-oe1tYyS(=z;he5GUBru_rcYceOVZF86z@ph20-i;|D`Z33xiV$
z>)UUuUfgpN9qa}yjN$C={6>rLRSWXF`=lSDpZ8djknn@H^w8$zGu>a?22%oelN*e0
zoT@#DP6rJ@tXIICkOah$081}w&~kCx(^vyULu=i#iyrrV-iqoWg0y!{jyt?;u3Hq3
zT(%ULZ<+%?_`eh%h}-W1LudkISwwCx#C}g#rTFqf-4<rs-=`gS!j}fQw*B)aAl_`d
zk^3(xxhd=oAO84ThKcb1-EN>9PHgcvWp-!(&n^DP7X27vfc@q=Av$by7ib4buAMLF
z5WzF}Zthr+NC?0|3i;c35@RQShDXZ+z6-?<io-<#+~3yKnJC`;A2t<Ox~j7Bzm;PF
zxifmXn0iJqao5-fl9GTB&}HHP0oW=T6i8C$&m^GhibYKKl-+3hW@z|QJ0fqT+5kg8
zmCzOG?B$D?Ke#~74D4_$D&)KdnFwnY>KlMWg#~Na570LzrxF2OhgTsd`p5x(L=$K<
zm<IB631Vw->*?MKuvPv^Sb0FA4%ff~JrMMG!9~!Rfp4(@PSVB3iyARP7V>sIduS^c
zQrtgX)Sf08<L&K&Wk7E?&)W4!feYFx@kN!CMotJ--z)}orc3B;1BC!F<a_~4U`%oz
zD(E_ua<X^aI&4)fY}7}m4{vW*1qXV&R%rKE5mL~`l1a<p?a#2F#{)@5>VMTm*kF)(
z{#Sc%71c-6><bU@$K8SicY+5exCfU63-0dj8WMsAceg;0;KAM90t5-}?s|qid%t_{
z@2v0cT%5Je%bE*dX1c4ZyQ-_I>sKSp`Z^Ye0=yscDmDs>2+H+Ed!@0$-_db_qcfNV
zjxx<bC;^pLL<K*h2uPmjI~Zh<XZ8eOfb2n#H2o?H-p^AqIaLm%1i8zTF!cF(bR9I{
z==KR$9<U?8@RWE7W&$laNS^d3OMUXc?7;xp1Jm9or}7EBkMm7VP{tnQE~NJ>@86`y
zU*PCMbzqwOa-JvGoU`MU3IdXcsEo=8`S;}id-DG;k$f@(i2vJE)laWTsdo_heY)9h
z321zMS$8r+(4*%e83U;rqrJ8KmAgUtyyE^ge9OV&|G1rGG0O-E5zm4$@z>o`r~3Dn
z`2T`=pm6~fM&i10Ste=VS&o>|wSCT*6kf%8b{cX;{=fc{|I_ZDFX%JUvyLGfpvN#k
z$@$^!+-CD{PMoa_#aZUqI@8D)Z9E;Cd_vdAvv0Xa)zjs2@u&=mefhIX6jy4cUguEK
z<FpF#SEM%WU;D*Oo`n1wEq7;DP1g6%SX>X#jikW1)w0w7W_zx*db%6UOD7*&6F)Hd
zW#8pZ2U8yshB=LfHL&z;tBv+OMGA@Z*xWOWlti{nnqJybo=IJN{jIJ`yQp__ybTkb
zqnP&U=m|U#rr@;WAFfIm6U<7N4#s&-3jh7rwA6@(=n_qd_})0w;nZKTH5`_$ikJ>J
zVMPNt9~T5&qHwdq15{{|RsWuNJ}o6J`_3~MExe0ue$yK7`2BWZ8up9vwT?_Yl+s72
zE)Q~8e;yI)Ig8@cN3pZk>yUQLThKF!9eO#g<5vh^w6D%fL!eed?OGGc1+^H24sR9E
z_Lo_<^jvGSy)RH5H$573HZ~&HsmH$Gr=Q+<F`T0~ZkUCd%muZH>fhm|u5UF9^?fch
zPDDP_&6N*P960<=*7qZjXoEoPEwagXb{*AgUkhH}N+&lJA256W-861vMe%DO>>;?F
zC>y{)=@lQOe!uK@4L{v{VGUv+h4<^2<)HMh+&&mP)=8c|rk>UC7U~QbMe<4_CAw=3
zaO{!uI;Pu^4xC?(Fr{uZ4!>Lv##T(QI`U?nykQHvMlh}cjsrZS#uR<-Zs61d6a+7v
z>6Ke^e?Ps0;XU;0ZHx@9Tsso-O&lgIeR0jcbT3RVLZC`-quZ*&Rj!BYk+(50zY4Y*
z0xb-A`;&_YCYo9MBt7jLOxAUV;A0ht92|X>JR&!Jl?g5EZD*hX2KXLfECpT(9o^{S
zod?G%BrqW1s1YuxZhYY6S-!d0@_a&-AW105-Riw!b;#hXR@T(K+huIndHgzI{ws~A
zeAObEIyxS@&1wz4uX>4zB5-|&cQl^C_NUmR5_`Z0?&fm&D5Xcs#omEMlW!)M%Fqpu
zd#FbPRr$8#9v3=4G)Ee)PlZqAr{sESzth-_l1M?31>we3Lj!p&dhv4#my@G$$fui%
zXD4;m?uw7MXKuXQin_cH`%V`(tq<q9hxloDMp{QXC1q}hm-mmuTcm?qM`h8=eshI|
zuxBsmsKyAp)?kjej#>w#26f|)>U`)6)8l1ZyEJ(v9%e~fcYYi$wY>LTYL-<HJV5DB
zzEiO|j629v<<C4?hJiOeyDNdrvvc@L*OQE**t{f4FK?_$8X05{dU6$Ye7aq8s}b9f
zuzdA+`qFN86G!s^#(t&Ecdp*Ccz^z<=HvB++50DTy38P}jV9UhQ$}XOvDT-1N=FnJ
zF!NbJ?a1qCSlu{r;yeJ6LSoT<G&kmK9~A{hNVifWm6WKYY}jRdBWyhAv=#fuoG;S7
z*MHUmC~T+&zwAf1(0VU!#o=muqphHPblH%ooxNN#IV#xe($Z}7wCsnt)O|F?aglLU
zD--F&5_dLL(=hGjz1lx_pej(;dh4EEM(S*<{@XiL+tsJNzFI5U<4FJ+YfGWx_I`0=
z_o`$b$^<6->N;@n3~`C6siYE%$cy{ChnSP7KxQ`IMG5kY_^Odsx*6)2hh$8k<!O`Q
zyM*{dM)ZNWPWe{JDcm1vqMj)SNLa5VJ~ZIFU_;U(d3ks(Str5i8Iibm<STt=kW5kL
z_h4co$SPxbkxo~*$-@zA6lrqFJ`RGnMM^g16H08EC_GN_`u5kT_b<ShzW0w!`JMMS
zx*~6!c3kRYE=WcPL)5jg|M=IgO>f=xD|)&1PZEbpUWPV{K#}be_FX@K$%nCO0RH69
zFvd)F5vk~%^VNBdVxwIWV)tp=+<fUd!rS|(C13g5_ik^{4CbxZz3wK4&O#JiY6J)g
z5o1}+2i@)jejL`!wJ5Y5F=tSY)|D`0i0(rS3U-RrskTBAe>Ji<H|xuhCbQS)-si%F
zjW|n=oUQMeh3?XeZLv9*)*un3ND@_Br@F>Hu6&18F8p)*Vh!)@AwyZjd=v}(<r5{0
zl@N`-4J|)uHMae3|0`O3lyk|t($WXMeJ`VzK|!bA?<^Md?=%MC%P{1&QaQE{?53P|
z)jY~XEN*r}(@pDMP~9;K<|&8ozk6w?KuWFZ$%QM|J&y=;E!nwg+1|TkSL<f;AeuFN
zZN0c)s=kgrIp~(~jWKHWV1-wy+evV__GjDSTr$}@Zu!JJlG2GTRSbU&zM_$}9~Kuc
zl@mp+Zp{z4PR+k_Dc>EifNSKyd!jKy<lN~ZbFAa*nU?bm?gCnLlZ?F+4NAv}7k7%j
z)BUft$k@<toQmwH>XHzDy}04yF6N7k^bq#pgB(G-pXI@EmcP5t>abmMMic)ggG%E=
zx5bgj8;=KzXkA+g*)*6v+{F_d{5F%x3I+n&&*;t4=(p(F1(t0jjF1lBv^+AtUSQp7
zFoO;1VX$kEmEqob(SVBsBra?8snZP?Q6LPUCKY&J{COkOeiCw@oi;<>CPvq52kAI$
z#@&_PQ0S!G=-=;AlyNu@Q9(F(--6qEu)tup&Zk(j<`JLe;5KB&5yo|1V!k)7XDDfU
z=EVRTd8yQ+?dImddszR}%}A1gJMfy0M?c8#WSZNOS|UnD0{RLHsC)5^)ZY;8;awCg
zsVpO`<`8rV%+7nr!}Wu!k0Kpic2O6*-D16<;px}AZ+laB?ooRQO8Yj-TjYPziDnPS
zt_u#^EWWO#AyaIvX^y-L%Py6<^Zk|W2cv{xs#;*LWG{R@<lCd!QWxAXL3`9hfA{Xm
zv*R<Zg#=m#c354tiliBEoO-u888vlRt$eHygc@g~bWh<fi%%c!%5MSdKih*IS8SQ(
zGO)vI-1W0L<~C)%B>6r!&k038!+QL~(bL10kILO9k!R!D(6r+XYTb^F+*&jLy}i*x
z5FwfM!#a!&cVYr?JX2a3LrTa2GrM$><r(8T5IXom67Hfh2l&%F|NWj=s{{qr^Ks#5
zTIljZR&OQ(F4*7fJ?{yMNd86n=p>8)2jEfG@Rl25Zw(E@;eP$cCHXGHO4u41z?Tc8
z_NBrXHiFY14&3=E(s~&QpVwV4QMRhEH?6HpJgaxqo3$^Kud{SAZ6+huLYHGPiUb7^
zQ{s0zV8fW3*JRr)R>4FJf4EKlQY_*S!UdBvz@KZ!ut-*_cl)*F#eB!-^u4{gAl1<!
z`a(K53=Gz>eU7{IA}j7PU!7YuAEi-@wjGRg3XU`Vs1ZQ5Pk-OU@Av9s(@$(}Q)F|V
zr`reaMP-Uo8opt;(c!Nj!r~);(h@HAxNXc_iQllw!pQS1NetDK8Frk=to?TSBiWx{
zz3wWS5wsVN)8vK^F=F-n_#>j{?lZwlDrbs10qv0%IXkn>2RD<1NMiHSH;E1MtzO$?
z%nh5SoMK<~(7W0|BkK`|1smP@tIoiSuXciW(#WE)hhEy#CTIeUCVZpS1*hy3KpPb-
zdtmRwk)LIoW2&ben*8V*cZI{WiU8%M_*$7vy;UX}(s<#=aR>24vfG827XYi0rN%bD
z%~yvz%8mRH|Kf32nYEX4nAyW2RTb>%KJDEy795hJBYaU2*1B&k<_dY-8e7mcYgC8%
zyhc|{-eE%CY>T+aB0%KIgWI{{l#iv{3(gSARVSM5KB)v|HPv<u8V3@Lr}*d|;}}Qe
zpt6@=zzrYcERECFq@Nd<WrjjgRG(lt?mO3DjrhJ3c{4^r8Sbp4&}ME&u0b~al3xtz
z1xrcY<QXkx7#TtZhB4;w<`R;NiF>T$PnI7?UtE#HsJs%eZL}JtCTlh5bZe5;Jm+p;
zYa$$GC*B@L_hY1yl#Qz}XpGhIV&8s9(&F{=!tmEp(D=HEVj&3us(X#*-(B1v=zE`=
zAJnR>_Y<~27oYJ4UW0I(4AAPR&S=hPQFkJ(w2iXeXT`VFxUO)*gTNplm};R2r6}+D
zomv}5#~{L{%%A{6?DImtxihDRWGJbMMoUmk)<*+Zh}99g&;T6f0coR%RlXhF?WZYq
z)SMfGtn5AA6K2l(xmRxSW#0;kgWM~l;rZU8ETyY1bM;DD)+GEG_LN+l0Ox1r+w>hQ
z{keO7A8%)B+^i_zyKN+R&-sJ&<=CN-a5&o9qs&FVO6V2^>TIHn6|4<owU`JpFt#Fa
z*TpoCi8Q+4aoDJgidbzlXev<7>4Hz>l|R#cX2C=V*TZ@vvZh&J;ih?WYAoOs6j{@K
z77A9vZ??GVbMefIf-(;xuexY^=^++7)x5emPlX_=6uQHZV~a>$D)w1y^8EeJ@VJ}}
zO9^5rm?#At)M9p~^w)C8bnbPH!*{|})uT7t2kM+_3#8xmk7gFev|2c-zc8Ea;U!2~
z$eUeSF`Y?W&GhQaT3SWEK6`9Liisq~C}7<{ccQ+pombF&fw>ujO3{*q%~V!>GXj-J
zP?{}ZE^vZMj0rcJW!qa_Ou?X+Le<NF{awLn#skgJ-qsTqK*+qX8_b8lK^L6V7?wj3
z=ZXnlzFT04jP2d6{xbOcm1DLMC72te99&c`1%jU!SgX4$Iu#WIQQ6?I%=DtA5aYNl
zoaDq40<fTL@Nh^4HvNb$UJ5^!D-vhJN?ZGt`eq_XAz~{aJktCT2)X&BK2Suly|(e?
ze1)`wS&sh5GbijZ!f1w|6*Z}fljKol1zAhsXi|m2Nna8X7B)^&sUXWoED)|{LH0=M
zmi-L;?1}Ib)bKHgqRuABW(p=CJr#a6mlb>aRJz3FcYG(lfEWUq=b2)H3*8nPa^c~Y
zhSFD={rJOp0%P$qLZh+wg1^AJQjq-{!fR;YcEa!RQMC?n+YG~DTa0rLZG%dsS!6PQ
zMNCoB7M&ae#=fZp;}Z%V0&*?Z3EBpLZZ?-S`I|i5<S<%+??ck-TwMdnhY&}}9&3g;
z(sa{um*yypuRU(1-^+vg>EkTjUnK3ecOfY^@(pR^FIQ=pD1dOpWzJg+s(S_fRb*;L
zQzll*5QmpI&dqp!o4iIOB}5|oAjnx!3kry^X%6J&yfQQUK0X;BIcW1L_o8d|B95SX
z3cn~Pfb_UvL9Qc>6fAf0*~N0y^!CS6xzFkk+E5EDxws~X|MXd5|EB_6=d}sD9Xw@K
z=ROP=)g(9rnQu=o^^$W;cFal))fv9u7tR+n_xE`yU;TjfN3yw<WiwGQq*c}3?<_?O
zrj6~XDKEwo7o$A?J#bJ)8b@FZ#{?vy5pVRFSHGYH$f5$#<EK*?tW&pKED6<H_}@y_
zxDytBntfBLWPxAE5c)drkCmI0{FjbAOW|Rub2c7YQU6ce<cu5p0S7LWcfZ390yX{J
zu}#r1E@!O%yg{i=ebFo~KX_<A%x}E;5LIWbyEPZVUF~ziOalL>ZWFa|cMb|c+0=CG
zH%t|hK9p09Bhy`<*fjcdEU&E~h-lE)pSo`uyx3sn4#G&rr3aVn5pQHQm~+iDG#>r&
z`;DMZE0lJSeR+~ikiB`Kn{GONCm}JRdH#F9oeQYGa0-MlY?N<oZ^oZ4%5&tXv-0)o
zwj~zJQFI@_Kky_keVl`Qoh(F<fzV+`H_fwvI6od!>M@#^t&sc8u^G|EqngJ^f%>i?
zw0WJ_ieEHv(eqREZ3F<__mxgqzZSGf^x@zXuY!wY0*Po$z#cuRw^o^*Lh=MU|HV93
zlAK8L<-1<;3<<UY1tNYdg*J^LC-JJVpNvWLlEle#h*N_pA?(!>a5yQhqP&fQni@}+
zX#*ldwijiNt!fn=4TS7fl`5XF{%>^%x4VhSv-G`9QfWngX$$y$%YY4i-xH$%9<0A*
z0J0nj7aamt;ME(Kjp@%dB-(SxSa$-7VfSk=M@eg&S<v@hvAnu-<FVm`uXm;HWdOq8
zPfFRf@(31e&a)tev3(9GhO>U3j`-DllEUNMcFTX>!`8H=lL|TlN!L@Z)4&ps#fKpu
zu{M?o;CNXjocj2y*MWGyGG-|J3-8iKzvzIjv&$4PPt<9qCIR)jGB7&=|EFi7&tQFB
z1OOa#3vb*3Lhw{WB*JU%?It%6)iIi6&yX9nQP`J5hkdjh$iqj?a(+^sw&m;rKa0Eu
zHRpw3v}z1TIbs5%;=UArjx_$prC;~ExkJYjj{`5MJ0M}mndq|Nl54l+@nCCPhd4d~
zHbvZwaOV20O(Pm=jm>OY^3}3O)VVjesb9V6dZSRmj=SyY$3^?p(9^wg)`h@={$$pT
zsN$y%>Wr^iJ~Vvjdq*Hh$UcVcAn+2qMQW_09fG+WB=ff>jnD+Ne{w(cBx33#VmK2-
z3gP3nY?ev+l9TpOFDUrY$`TkHxh{4BvR2Z~rq&TZkYT&S^_$um4Fxrb;Pq|YrTv%6
zL#(#s@|l2NH1#xA?4N>mDCBStjOYJoJebTnx^2$|keLDc3PGCcN#b0Dg0svDDGSNU
z5D*E-4*0~u&DSpXMl>ccls-UVELHs(!xI>E%!36wNqvcQrsUoXkSL}IWTg+ioD&;W
za;JyShJ^e;WxwuRx*<1B*j)U~jk4YMLb3l9=a;BtK05vEJoz{ra7&+p7H-`|r<d<5
zi76J#B@lsu6sqUuhfr>ZCDK@iN_P22#;=07DN|%24wer_>j$*Jx%aoCjGWGuDXWc+
z=VT^5rS(bM-Gu?u$1!Kz7Bd<HPmEeSR)QOqxyMfa_Gd(zKHpEf8bdw?Fz4f8g&!qC
zf^ne}EP%$B@KY1x0&(QmcRlyTn<NJxZeqnIg9lF`r`Fn?$PHwu2+wINwB(oXErm2l
z+<zLp<7FlDKYI}j70b%0sge9)QTd0XT%o<WrYWj5Epl7~S`wuQj8lJ2U0?d{&xN?2
zq2jqN-s{Dct03pWAVq1eC}|OQ#g%t(s+>Fpsd3=aSdWQO97+qo4t~jF)|)z{kGv(u
zu)D8ONvg$YUd5{4(bmskA`bYga>IotnXn=vA(;;zGG54$!Bm4@*%_m5Cb(fHO*B_B
zsvy^junXJ<={1GLmHC)gRjgq9eput1nu?5IZSdcbD2>yFYNJCjM71`e53JCqhSplC
zv&GSNP2O*JUFf%yNT!_If~`b7laGTp*?Ac9nRmve;b+~bjMbrGX<w{pBg3jHq{$vG
zbr5s4A$^Q{O)AmkV>S>*CoKPEb9@w9b#~r|me+3QZjoPB5=TT5@V}wBmrO&f>Plj|
z#FmT91Aq;KVdz2&`Q}8{UB=djMSkR4lB7XCY;XFzSso5-LF4I2>2=914{-IfT0sUO
zR0wJK45Dn|$;B057}8w@888Jbe2dKS{qAn0b}mov=IJ+^WXs}~z6eH0kBrcZn}vYt
zgo7^66#rdlq$d23AWv*2h`mJO8SRNpiPG7or|NS0enu3$0AZm*AgxdhQG3aCTtOW2
zkq=CWaBYg*SU2BgF~`oD=cYF^p<c=3^w~Bx_Tz?bTf5-v!;bj*E0n+PgMfQH31eqK
z$)o<!CzkhTW|OKJx-{H<IFl=baZWcWu<SWYnAhZO>)ksq_3<EFKL+EN&b(wdExq)e
zP*6X_#;{0<)~7tw&IQlm?O|8pCe^UgvmdqgQ&MK8XVkx6;1Q$yXisDp_!_9vmc}RJ
zlgF=E9wKZHMlz++1rf=>smO<u%l)mI<5g`X1Y#vH7)>#`se2`Fr3Xfm^^%`ABTc=b
zN6};Ax}78EH%A6!LCHh2g=p$w=~vYg9XH;f5emw4jnBH)K_<{U$LzEe@oM)Pt!90g
zAlIcavfY)Dw6N&Q>V4oGym@_$kXm=Q+cSh!-_$-08i2-c(?df{aYyk^r)Piwxx$XG
z1k<iAZ!-ualt3s|a*R9<$VoviHGEyUrb~h|Rl1HW3@4h#!w=t9C}xYuD0S(Zp@Ez;
z16L>+@>tfvDEkJbK7Ci%s$+vZu<%XZ3n1JSU3C}#H+SQficQq=-n9JuCf<mo{cXq&
ztQ^TopGohji<)B-*cC<7VVU8C_kPKUwYM2CR0>OYbo%|ap{&NO4G|8_JXzj8NmQK3
zecnhZygkoBOLB=O&oUZx9Ym+g1>UCCxbq6Hwt&$pXL_$jM`5klf_}UK42FDlK)7h~
zYT2E7vlON}42+%2FQXMeSm}>t&9yQ4uwY!w=c2<Ezbw1p4+-y~sQQ~&j(t9|w+)f(
z_f1@(`zV5dWmwQDM58uPCQ2bH&HkTLYf3YUaCx$Mk`cDeQE&U!=hnZjR-89DmH#6L
zV=n9;4MNSdL}jS9-!Pd7SpJ?qM)L^YO}bjnui00;&5&}}>=W7_eL*$*hGM#{uGaJ8
zK+WLRZ!WD5`@Q>EY50|bka=mI&`xw3b0{+QFtQKsp|7FBpirzUUhVFwe%#X(A7UjB
zRFCc#uNKX~FSQD_$p0>Rt$?EYM;0>=@F%ePjooj-RqXanNRu4EKj5Hv>EdJ2-0)sI
z6I>|Nsepd!Q-?aVAT|h5DHVu4au?`)A>01Z6>E)`>Rb^&%`5*L37|ngpRPb^mkT2P
z@v`a=f3ki29xRLX&y3oz?001xa>%{0CaB6($VWy}`xv~)SMU`B$#Gw*b!Suq_a$hv
z-XE`$?TTbdWH$UbONH|9q#@$Y_6a>$h>`xZRAFzeXY;ZomyCBv;D;#C<|K05F(ew`
zMH<^fsp~3WsfGyjjKdqS+`W1uv=+Di%6~Kxq^UsHh&N?{5H%Q@Q^8(PVk{J6wan$D
z<U*Kaq}X01g1dbN6I90vsD!4{yT8-K6qk)s79#iK)E>bA+T_5(5f6nEnJ9d9%GksL
zCi8t+)dPzmqB61UQeJ1~l6tZ3P>4?;1|Y|a*_GA&gYzT0s#eR}%@mDHaHFmq2z&t5
zvVQ|<b*=OX!t8C=(_<TdzF$P>`4RAFXp(Xjo+82;w3G$wKlCcl+7JKEqas36+T1v(
z+D^Aru9SvBm4T{U+N5WeS{e$#j9}dhg|K5Lec@46LjZabF$Ag2&8(%Aw;s}$*>e$O
zxS)VX`)%5ow1+RTU(Ome?PwE@KI&XmdQU~0fv93tlB8sx>U&P&Mf3y=Y5OwL<INY%
zK2%R2*n+kO(3iUM4<g=gl*2(^6NkRDK<UfA=|%C^3sVPJVbs@QQK2|sU0<{0tK*P;
zupBWDCcM)<v3`k)N65j`%_kO+0;Lnp4$$FMmyNY}T4ZH2iB`j!=LB&<y;e?A`WW8h
z0GcKGQYa^<i>xmGJ3VZq_k^4<L5>}lLJ2UqX)bI*Ccz)l?67jo^=rFJs_Jsb$Sff_
zOurD7sqN8Bf22g{V$_`(n=bj?J?<<@IwSCOkaxdkI2*wBwk)2*6K4ycK_sMpWANUg
zl;Vpq3yyhakj+9V(~^Cx??u@0$E5F|hceFm0aT4Xep{DdSjFBiSGiYI`{14XFZWmn
z^H5FvI7)@Beb5jH?;rzX87Nv9o7btMf-^_#I7IHf+3$pTBuXRxct^T>bn%TsoZAGl
z+UyHTF@XJwr2;*-C2mj)e$WhZF{}U;UjLL8xbyYX0K&255BKjHk=QlO&(`E=wBp&*
zY8xJjaRtmZ)*Q7wxOX2i1%|mC$_T8w_aN4%SGOE4p+>qf_f2Jpj(Y!Wb^dl7Zsu*{
z?WrO);~H%l$hBd*L9}KYU|E1R5#)KBBz5g?X-pp?^-6~s##l#HmJUq+Ft9)YFwo*Q
zA}HqOtf_Rv*Y5{<w76Gy;bE%^y0J4=UDw&YhhL|eT-~8H9=Bh}?5OT5Wzaq4*`i)l
zv!Qv0kjOJ2DKXOOuZM<2>SKR~Eb#G+8_>Xc7|uX-V5p2igDI_J2ISTB9bYNJe+|oc
zL3)ZP#qC9PxZkmLrhhTofi4yTx9%+Eqj$Gz#8>!V{0$<2E&@qWcP+-<z^e+4=>nB(
z7r*A^dl3!!;0Xrsyi6D90f9p2@EcukLLke~=2vo}h3H#ir7u3ShQJ>B@K2mj_VAnf
zjzzCFVD|SB5cHuXBE?euG9>)YSG`#=44bKf`X=?m?&wg{lQKd68FDZ&PnyxR;eG82
z#6z#!(-V`uY0y1$GwLuu6Q+RQHD$W>d(&$f3%BVzb+KgqWCC;qwTC!3d@W;sD>D<-
zgAERwQ%kM$=peHFpzMpl8<?eHz4toF(#o6Ka_pV9cq3mll;@K4vr&bMBukgV&1CGh
zEKa$f1^yYt36|7K`qOn6xn7s`(ihLZgA4i$JgGH@#SFF!-r{T(SPz0igD50n7jVMp
z!%^tQSVt$an@D0Br&(re1Slde?UX8Lc%v8O>5jr<hnZgK6y#FF^1=X*s7E|{cVq5|
zBuN5dPf>L8l^p#98-_UxSx^3LFq0VNkXEMPeGYo|uq-kN0I~5W!1}5#t)@DXCCHW_
zLdAEDOHpddNS$!zOXFP1K9IpXnq&~b)REjHLo>xQ)P`yB>U*#teyhXz=Mm%3HLf-3
zEndwV0WcW70}JG~v*8*UmM(nhT*1$I%EGS2HiavAz)K~*W9yERglCa*HBmFh$|=^b
zFev=2vpYD^-+9iAl2SoP&+&TuH>L^X3zi2WP!FV_Ceazpc~F_JI|ymRd~l!|(w^EE
z5V146MCVA<Fh_EAk-k>W776-{12>3&t=|^9UOhV+4(<uzsyMYLY4t3@q$u7fg;}t`
zZ{(n<GRqBhXDsO=N|gP};v>^(kznJl-FVlIu-L}dMXWyoA|EcFC?M4t=N}6n>KjN@
zbBffb$oWi$_yS0)_oY9{Oga8G6wv3K%@VPEx<=KqZxiJBw59Fc_lN<F0(T>=AM$a`
zRVgqNsO~G$qaq5jVIGnlY7BYs9kDjm4?kP64tIm9V#lZZv*eD(v9Iuloc&1XFn%18
zvF)6dFD7c_dpn14*C=+Es-LND=D+OXhbsXy9u>_x>}465HjsR3<U`C?7FnfMNZ^Uf
z;F6Oo+3^N8o8TgNIgULz`{6KGzP;g-(8IwdYK$93*q6gIp)i7tx65=9LeCzc1d<rg
zW^HeFu^_epJ%j%0J5w6<2{TbiaJ9fU0sq*0(G85qr1khWsIyV}Qm`T){JEeITLai+
zcvwdQmCl_#9li^0Ui4<TbhRNPi{}ts6B~R-E%b3V#Sy7BgxwHb>V4X(cb{o!xR5zY
zdjDE9Qr%;+5gl2+nakF{We6?*OD%jAl}0X1KSOvKriT6Re-uVq6l_7o3jix^<IkCY
zurQ%d3GQsGfH%4p&}K)7*7rpBha~8Od0jcZbEd7{iLH0Z?QMhDo4q3bB!O<mS1MrV
zV1)|vD$G2nFw~?M6nSj7tc-;4P-w~{;S{2HJh6UR>ur{&N^Na!1tb3SF@jhw9CqU9
zi==!H%B)N7YRLx%_W}0v8jHV5$sw?G6kdNc?@f)Bov)jk#u_pnzq`C|*OUA8;eInK
zS|rmH3-UhLOR@!b0W};cJmB^tMwF&0WCV;6Y++)S%Fi7Wfd1$?wExXGKxWv8rjnvS
zRZ6F>_`SmdYxQXj-hj5MD7JajYQgFcYLmFr)I4-Wx42PcFcd!Aq5Z&amKstdHHg<v
zB5o_difBk@9SdaI6~Ntv%=?Sgw^c7BT8>4vV_?CtN(`vYR;DVyC>;tl`~K^241nKS
znXCAOl`(5Y415^-(doacMa7X2JuzF9jA~8!C-kwxvTyV~j1G?;Sa8k%j%*bP&N@}C
zwQMlTM>?hQM`5cvo2%t01`A0GwJv=EOtBBMT7dYPiIi8nGylFi+qQj*-XAUd_HbNL
zf}D)~V5qglZ;>)nd-xV)2W$BIYXZTOEQG3WM9hWVYrO_%KU*RkI4PBH8z&Z-{Enz%
zixg{gZ`O)NDIparvM(5Fc^bK&I3pHaT(^xdZhp<e-a%9r`ki?f=O{rvj}%3%!hY<h
zM3qaZ$su@-QabTvXS*7Uj}Jk}{0&)G3Pck8qC+?mExhEX6zI%Moq<G{WqHvL0h(-#
zG9Nt$<Uo7(n86M_j~)oRg68^_G|Yww&FQ;_Q7zxGQr}sP>lvbczqBF*Fulg&_-$5n
z!NIB#NM;vDi9eJWm@Kw<Nt0TnDDD}k8Hw4%*4OdoQvkv(6hdfi0Y0ADJIvrqX!3G4
z>=z8=DAb1Cod$xRAgR_U8LXe6zwqDKsbmK!j+5uJNpqzgmvY9fBC#LRm`DPE`cSE8
zv}lJ9#t-}fbTI5|g})ERDOYq~P8sSa8FXhYQPB}SzV9elJS6*?*j%p<S^<5_;U<P5
zP!Y697^iH4D8Jxx+2fTEy76}Vtr&yOUdQg^`oy4{{(5JGhkN^L@bgP=w!1K{3ox}~
zhJnP0+9<cWlLzUM#ussbQ@s0it6lZRcJJ{LkD2hwyGS++9<pq^?jx8m<?~z`gZqtl
z*C|GtH~%bLVKp^revh?{hr>a6PwKaFss-wMfj!6*tZ>-w9?fu{PU0w+oU%j~3leaI
z<R$Fdz8;zn$#<Y1X>e+UNOEomD<DT)R`!w<=AUQTyn7fxHoy>a<iOfqnXJ+((`)hV
zwQpTqwwV?s=X-P~?pG2@>IvzAi$R!B#dA-zXX2xu`aWKC9Y0AFM1Z^`)&yhr4T|2B
z-1^`%@90hP+|_cjo@Yz0(W6K_LAKv`bnlz!k6G1L3wWoqcC7=NmNY<$$DJF7A_3TL
zNZZk6=^1lal2`Z77e`AYOqqjrcbB`GtaJ<$UZWZwq2mr+uaGZ8L1|mU&tmp8h}JqX
zc8_C4o#CBcImtM!xnQJ~tCI`NeNf8(Bz$Ka%z?5(TKZcW_4-W_ZcF9_bZsW!-^6-N
z`c1BdwJK~m<n36v=o*4eGgzAmGnfD!CF>Y%HC?m=<<b=NwH2n}R(Ow5G>Cs^a5^su
z2`n4z2277+&v7CKO~01++IC^`RQC{O8JJ#a&~%iooY=$(P|0)KxI1?%CjhW}a7C+|
z(fs=Xn~AZHCet}zODHaamOX0@jn2cG@p;Ui1-i2f-ZTVErx>^SL61GJL02~Ls_?|S
zZqdk9N{~GVp@UEW`^~JnP0JM@K$)sOv2lzHV+p!AN(%Eb#JOYvPw&PM3%;!c3yy-3
z4fiWHr=Vn>@?c?HjouyvcOH0aHd*+5IsP?z^8@mtxdC<roQ)5c(?=mbdgl%Zh!2&H
zkK7C&)_CUO&yzKdCw$!wtg1ZU{H-ghHv8)AZ^1@Js=uf>9s{sf?WB$1OX8sSC~y^y
z+=35S5HXaZQI!sYVAv1BL2v)OoeyU{fDK}`LZU`CCDA{{k2)sJqv*?skH8h^jS|!9
zWp%1t8&q_`;V&FR4S+*Je>3E)b<^#o%~1aCLBIP<gX`vGGk)oe>{zTA#sG38RrWiv
zJD;q-h=Q(p<BN+RBuU^+twj|P-5cw6?io&k#i(VX47Vt(;B4=}A50lnKnEiZZ~bB1
zq@#H^UMcoO>ha6&vkv1uqW40+t!@Z?Nj7aag+;8h6_UqqgqltI`)>YX!d|PWHhp}~
z2*nEii&e~}4Fp!yTZ{$adVP|zbn<3CiMWfG2hYtA&Kkik`cXA2TQ{Aq34*EKrL=#c
zjAVOz@os-J+ZieUGK=*&)z0`3e<-d3gLLT3^3qqNT+(g}VPQ)XX8v1zVebwBinyeR
zxe^W7NY{TPrdBQ^CPY4y;r+Mf3+|`qva>_PVDVj1J=^y*lF}7tp#*3-8tqFjxkhOE
zf`~b3o!mC2!HjUtfADw0|J$7XCF2o)f4uO*mYHF*31g%0QyQi=xn6sYN}jeny+)Ju
zrd<ya*k>fdKfqF{JWCJk=Oq3<!haw`qR&Wm`(6wS^P>^FcQ}kSXrCM{#f|GGNBL-(
zx29HP+Mko|1HoL{=YoizQjAFcCb47vt)L~9|8F$O{~wx!-v9z|qaBiikO3$xX)X{6
z@%4L;hVEYgff85?0nXj;`Od#CX$R;uDCrZV&9&z={eNBl^8V2N@<9vFT>sY<z4{Cd
zaECoR0q5qwt~MHBsJ4Q3oBwL`xod}K5Jpqrjr)JS4>qKaodxJmu5kUgt^mY5o@c~^
zZo4=C|7fBDQ3<H9uuuJOUHL;)(w^Z3<@$mo|D%atAZ5Qe>cLV6|DskHnqI*lS~yId
zl{D&qG+_-=W=;6}xklN)t^gcb5H-=wkQDW|xYxg0;Q}e+Lbd$|pz<H=5e@?f5xt5#
z)cY4Z{jXM10azdvuI}xB=M7mmIEa6<_uuUOM_m5BdjHW&|5gsq>*4>LayZBhYr9(Z
zV_|8z&PnswOjgMeN9<2vT2Q{Y_tyAN1r?AR3+m3?s->H$FyZz<Sd)YKniPppG~ztj
zMCJhd;`b#9)(f@r%&qs11h0*y$vj?zTXsNhR9k_3YMxe&rQ=CAp6tdze7^h1suXE(
zbVj{=D%a^rFRAS1{+!9}`A&A6;o<Lw(~&nv?n`$$UzorCblLMHSyKSZ0fHU7)19Qn
zQ67V24q6nZ)A&7HB>HkWmx7<OM_K1Rs$WCXY;eraP`*AJ6@uxHO$L=Mkq<=eyB2i_
zGTsb2N#57hZOY_F+Lo0-WaGQb1IOw2>ix8t-d`V4pzEOr^2T$eyf&Th4(gDX&}p#x
z=s?9yK;66dHA6U8GNSt9V_vMn*SE1WZ;Ms(p*?7jfM=v1mHiYp&pfO@3_Mz2B=H+a
z&<U8czxk;J{WBSmrI*5Kr8eaxrVI-fYs#rOmW2axeSH9j?pHpD{x_!{0<-No|E!EK
zXkjI)WOPKLMM$J6ls<@PoM`fJnrroIW&BVpcldL!jPmw16=?chw1L$AxCi-#12nDJ
z>-~I&3F0wl$U$2??<ZW886XZXnkyhz6WAWl4Xw7CRS@%Bi{Q>PAJ4IXK|V%70-!pK
z)ezr;o0_N^GIIRTGmy{epmLuong{$<39xU8NQ1J+`g0YI1vaSc*u*@D$f(z%z2tQy
z;FDRH>8lmG7rs>xORJc$*yy&mjXGVT9;JQSji=Xle{)8;e1IAZd{-@uYz7mcdrAG`
zFexSb;7Ps7ye{`+zy}7I>V8vf3??R0NQ8AAK*FthBZ(PEea>UNuNK|DGN?1B<5Bo&
zyZ;GkF<B7-SI!jK@yrK1iw{S`soaCbg&D74k#M4=NIb-S!C7+Vg`&*lb=vGp)bpiB
zaE_)`B=I<IhtD_e|LXYLp|=}$4_pU{Cu_aPf>(!&B@5)@a$cLsX3nXE46!bIGdr#<
zW+Q1Tm4*^N2dmv*U`^(Kn0!fd8sSwbQX&VZI4sS+12JO`g_!SM>ir6tC`Y+QrD$fT
zIk(5zX2egpk#TdU=85(oc$ejXyWGW4()Od)r?OmnaCC}wlb=f06Vup%m6P3)a!V(w
zb(fv|M46q5{EZB+!_UdX6Zx_xogYzed2aE_l_srbN-^)HL>r@SuVe{uffa85A~)Ft
z#+dS!TN!o{jBv2Jl=u~I{Hyn0okj&_k7iCMWLb88a?l2lFEQn%zi}Hl{?lo)))R3{
z+$BiVdDj=01(`w*`3%AODqLkcjP@-v;<21?qJGf*a?Whe%C`AprpU^4G=sQ*blpnO
zX}u5ArED2*dYt&3FHD9<vBFN4CC&xg^shS;Cne4Tctzd~oKf#cp+|S0-`p@HRG?wz
z3C(Fa^=S?+j$W0<QBDFX?55PR!tXdclq4+8YM%3~7;XSQo)z=V(<#fSHxfQd;8J7M
zd#vo6NCofog=V9xjZZojAKJI$e0Zm=k%QzYl?0-9T>E8vm13y20_AL@27Zw*xkuR1
z_nU4uQ+>{OPr9(tFOQamHZOSWA8)qy_}}4V1*sM)2owds80DXC6Eqc+5&oo|8%#fX
z7AK6uYqmG@J!9*>wV54x&i3~1+INr*J~B-@E;~FXjo&!jnCCUF%s>En9Ge7r6#ZH|
zHa$IoifBhbTwI(qk-d1#Yoer)uE!@qaE+C@9C;mC)n`84{hqB_9{pBgrRP=N{BiW>
zQO%4SYU}%Dlw92Or(Y25WnFjqhN=78S#th){P2{QXeR2dZu^zJu8+B~LRzj)Fz0`2
zvCL~ZT`3dWh@e@Tjzx>DT|w>HzBQ8mW<i5=-p4Q@i~mc^&8b3oLEN2o)6?S}6RN>O
zKb)B+)bUoPUo9Kf`_)g%IhQ{MnXT5yD&=$ZGk{N+Ph9)5iT2Wy)QQ>?hrcRwIokMp
zGZmn*l4t@dPE~dRzNJcU@FZ{WvVJYns&iM~L$c5+1Lzy!ek>Xs5qi4klGNjN?qK=3
zopZVD_e2a!z+Ol#2uG$M^l*N;`h~?dif^~bF>~bDuJwTnUo!n8k9MV3DBph5*@#t?
z(Bm~0oJHOti6cUWsseBRU00v8$5xV2B>mNy$50Cz?D%3MIDw6B1Vy}e`&DlWoRhBR
z%rdcPythyp^8vDE%@+n<ttCgrR}Gn&Zp%JytW0}UfZ>Z~Sr*a5-%g`87NnRmH7h}*
zpx)t2!m~X^^TP`za+Jn<;SF}ZmQc&A*@dFi5J`|ZUg2Hw0Kowk^xz8gg8^J`wO0nH
zIoCO{nYgb<?HUhiPs7J=ythEb$#T`pY;JziS_*u!IK~o09n|xCcpn&P@u#e^|GF2_
z#?pE}VY>DaKxDV0MqxHgX)xjVXpdiWvlb=9#eWqFp{lsT{*+I@@s##6M-QD<UwITG
z6475HOk+1sBXXigB^)}G?{h#%X)KBNar<d$%cp#Drrg!*T-)3Is?puvgHCiI?T#!l
zp><0rVIH?d7j@bDooy;JJ`DUy4jNKaUvLs0y8W!7NzU0_&q7Ik&5bnWU%m+Da?fUN
zJ{x%>TYpUGR0CuYb=;d`Dk0F%=MAA(fi*WC?+UVJ-U3}eDiYd%v>@aOn7p7o3sllx
zHw<z;A(hSkX%$Ryz{0#x&Ld}|tR%QB3611$6KpqihS+u6mGChzY$p(znJ&W!)u`Cp
z-E~IOEZ&Tz`CP?A66R;$cM1@v^LU5Fh1#JMqL@NJO{rr9ouxbPyzFy@O2ei=&FnDz
zGM~_UE)^@fZ63S+@pf<UiN~cQS`5dy*IF821!81HFBj}uFTrKJgo7gMXZYZQr?AiT
z>pbmcc%&J>(FF|*$+baOJ;`++U<K8Na35Xrri+m~Wy@6x7(z}&vp?16@E*zs^mi2Z
zsei4bmR<Zv`BhcXub94kpbK41?=X#;5WPVscrZ($jto4~tWMn}%$+#X10<f?*-aZ)
zgjh;IsX=y99sy{J2>!Hk*6>O?41g!0i@=Rn)cL&Zb=15urzrFgS03(LP66OU4M=`<
zX*7!T0j^(S1H8A2ohnh$(|@`*SFmO&qxe;XPPIS@0|HoIC_VNXogL{%u8N<mnn?pw
z&<1ki7_3!!nzRoOSIaL~k=9^LBssb6Y~9tK-1olM$!K-(DL)i7mch3D(uLxi&<xe}
zz1c|ZD6+fPLj+d!36u*c6|Bm0nP%yMO_xu50V&xdhf5SdpL5xf+1w1pEFLgaI~B=A
z(DLOHXX?nkc?*#axa4;-(hdM3Ah?LR+Pp3Vw3n_|8efv)-HEoRLp(R))m^VTQbtrN
zODTY&74{ahOdnT{dk<)T<t&sSm+xoVYB>h!1{vI$Z-t?~$OG=5GV1Q_V}SaH%LQ4V
zQ&0&+s_1uG&%Al?kuV_m&YxPB9k1Z?5pzbDtGgx~xW3&hm&M<AOWJyT2`0gW@rak2
zm?j0_m=a%4?M=B!0aJHVWeuAt@^hL}P;DfU=z`Z-CZq5+TCe*zOiPgbB{u32!?<TY
z`a|p6gmna?U>410{D8nG&QNE7R;Xbz)E7P;F5+=%a<l}85(Ix7%3#G@-px%!f7Tc^
zLg5Yr_PJlQ+-6KzNJcxyEueU2Ksp#ris3sLH_R#U5dDdcvzQHf97>r*j8{Sx_2^iW
zVn(k?kSP;~SIng87%2e_u2_eV<h60{GkrG*P({eS<i1AJi9+D^(EJ8^1r%4t8v;uw
zP_}}1MiX8SMEVYAwdlSF_@D02oCe23_hf~kDq2j81_FfzOok*`x#~?!x@rDsQ#gYb
zjaw7`%@%L`(`#E?D2&T$+22Nc#-E)s0R1YR0>8N<%19o=<kD>^Xslnd2;QJ+oKMmw
zI+FB(3fM*4{BiO2*Yz?6aSv945BI|bQz_;On7i#~@!1R%yN#`)g>OJxoN0<7O*6ih
z|D-(k;?xiCxQMYD>bgbnLw*HR*o+MqIy!<24e&r!=^WQr-CtF_X(mSfkUzau?_t}T
zYAZ%Nm-i$<>UB2^EIlE+XhM{a4QTNfXV%(=9w_lI66h<AqvKD0SfQ(90yX);zLZ+D
zUKzkOD9N&f#rFN9Iq11^e2CKee30FrVH<Qt;AbX__VQlZ)*Be;i0X81yB{QB_IaL#
zvyW)J3J3M-544;{T}hzgo@y<H)*HdT1Uf6D*d!BHZT1$e6Eisyq#NJ7&;DbAW!V>i
zXDiJ~hL5d?&3GUlb#+|=YV64>`Oprw&3VxEHSd>6#Ky24&@{9m*eU#)vleN@5xVSL
zz^0QJfAVZ1zk38}el|N0h0KUPQY6y?JOHi2`hG|IK8ae8GpMJYgrMPx?1hUJKbGza
z8^-``VHz_YHN5NZ&3oQ9Q={ybegA0!0Vv0lmw1@mZQp&s1un=VjNq*v>L9?N-fN-o
zysJe%1YkZ%TVY^-P-!dEnE|-X=YCj5^5023(ar2EEuFhhE!5F<nSQU+VmZjKC~)yk
zoxn-5O(SSNv@#5E2dxWPlc={%Y>9(kdrE<H@6&#2BB`rhI!`_5NgAaOIRsTsdJ7|@
zKcIsz;`jYCoG0FXfbK!y$!!T1ORpLF+%AU6L<v|T+iHNx=$5kVX&jB=lq~&0nyZhz
z0D)vJ{q<rx0brWJ-om}?%^t+EeDQYVAvy8<-Irjt_OHG-3RUHU8IYl5gk;G}hj3ol
zHvHtBC3Qfq&YCv1>Vs`Ven}gTIn{UWvAN#q+M6n)Vfv84pYI`n!^Z9VS5w&$Z9^)?
zCU0l}jf^<oicJ0)R{M_CvGa=`9+R1%7`8x8tF?Ba;ZSckc`L0G;LwlRsep=b*Ix#q
z@we-s6-+S13PW)=W67TN)deevQH(tnlqf|vY4T;UvKPRtj+;*9e1b25hXQtASIxF}
z?5$j)1dSLwTFYK?Jo(+jQS+5<9Shmnm;Md%L{ni*EHM{>J03^l^ts>Jp7--s<xJY4
z*Ru&5(xb1T$tJr^mnwZ=U3@V_Jo;WboTGbm9+i~0XpQ_PZ+&R(=HoB3a^Q+9Z}Iah
zr-g!T^Gx_s{0(7BYRWACVO)w7m$W7XIb$~TBd(%@2uzpP#nzY@`oSx6H0I(XCjpKx
z!Z4{8ZDFu4I4ZV9ay#N&1=qaffaro-aRWko@ph2;la74bFYTI?P5F3!=+HtT%0&UR
z6`MV>yI*8knyXDDqa&ZN75T?sM=Q_Lia`Ufl4eTxgol(>NR?SCaZW)B4$xY1s`$7j
zg>@1s+qyutXy%$OI?>_jH>9&UwE62#NDV-SVrretX#z=!bZ;50k{I;s!=s(ubbQO#
zxc`2|8t+uXT|;;`m1CnBnWd@1excGxD$$IuJ#P*4e;E_jz8(y+v<_LNYP(jS^ajMe
zaXZ}Cpv2#1ogK>bz1@*NatgX5K=+;tY7fvH+Mlu!zSWcj)&s1YU|>*W@4*c~=T1Ts
zombAqlxQCw1g`M=zVw>d9x9A}D`d2a64SZdIjydz(~*0yN|IT*At7dQx^#)H1X%kK
zBn84kiZp&0uBK88UK9ttJH2p<%p!Gsw^Gw8F-w%X<Q5+O?1tt3VZC);kCAd6-9iMo
zSTj9$i*x@N&zlD)Sm505oY#R1?h&#}EqEiO=Qo?m-3tL4y2`vVQgmCFTYIoZl5(AT
zvm~F=ACEvjJFtFg0k&HcTR+2`RD}=144p1MUV-6+T6%o6_k$*{cZ5P9trkKUWFAp=
z^GPkTP{INuZwS<$QcZ-7A}3d^&Wa!0zzD>X0^P=g$*w2gQSzmC`6G#CUA_9K?piBa
zkl6+B!b72I!DY!3Q@uBN*2wT1TIb#cc^06N`Gb|360*(6&XgRFLi5{FU9oC)D_&i<
zm^5+E^TFaH=g}RwE?*tZ;DyUg(}*AjVd5x^cg;AGjIb*L7$lQ*c>=S`GZrG!6^OPa
zdk%Z&o=i}Pr7QtDpijj*E-la|sfL2e>X=Sk2X61b$!UJUE6q}BQ=(LS6zg?}F^XlT
zymHx!X59vybfKy$J;;oNO0@vi(<sd$UoGinCHNXrwMWyo5M>)v^E-pev`-@G<h@1i
z%tximM*SwtTLw3%vPSs*q*|wO(Eh5+dR_-LKkhj?gT`^ywZ|h`(DTWUViNdGxR<GQ
zFUeV<pTL!o*aRvmghT;McoKJg50*}o{$Fp7nqrmht|h5v2D{y92je>@?A?AlNL^hy
zj)}nAxO4lRpT(SP)<KiKJs@6w5__F{{H(CUAcmezpT`Sm2zO}sp5!^^89pAjF6G89
zcTHq|JkH6sGK9<itSVZElfd$9$B+j6`ZV9Al0n1gFYnxv^6O}-E2|^nH5O%^yfb4e
zzP@=*=Yn`)IU;6-ibmy7{^x+mO)y?zokRIse5>_yKv;u8@Y@NRF1amQ1yozZtOD}1
zQK6@%&pt3ZT85gFzgc@oB;lJ&DBohcthd9H{uYO?|BSZ(ivh*f)#a78m^f!8r10~|
zBis|4=(0oMqi&B6Xb#8^y>NDU_!COH)Rj?fqPushIB&sXIMpgEi(<)iahOzyAtlfF
z@?Zdw=Ggy;pN4#i@U+d#K~NN%qWCjA6qmS2@*movYTNpws2xltK(1VFI0iyR1_f@x
zQCRp`fENdpBn`M6#KAosi&;Qc7#b_uA3`-on+0rRpVUF#Qk1U)0Cd}1agld3si3{p
zJlH!0^6L4inGEpngD-s$8GRT~1HutKKEL|6?f0Oo@c#3T9`wSqsndKPq4Q6>6|>mF
z0RPo*CwYw=i;h*fI(nD#(Kv$WwqVdCrO~c8mH&_DXc6>>8TXIu<~+wUq;nmlg`wHO
zCJB(s|F-`hRI+-2|Le!AB#8TuexLu}77kL?)4_lUzVml1AVG4^l%7gh!%;s|_n&W5
zW&uf+$1?wV3cL{ujyuU4GgahYPk}ECA<zM)j7?wvdnfD9^y+n)D*gAj|4rgQ?ESxv
bi^rtgcrs#eZS%JP`14jmLA*><Kk$D6r07h~

literal 0
HcmV?d00001

diff --git a/docs/figures/arch_2.png b/docs/figures/arch_2.png
new file mode 100644
index 0000000000000000000000000000000000000000..197e163a54eb0cc9fca5dbbbeac0cdace29acde3
GIT binary patch
literal 51944
zcmeFZWn5HW7dAXI3<wA^q=ewmA<`03BaL)Px0E!}IfyhOBHbMd(t_lO7=$3G2!fQf
zC@Ips``{h_`+YyWU!IQ-KagSOoU`{{d&RY`wYIU^n#!ca^u#b2j8s)cK^F$Y11}ML
zgaqIhp|*Q43`Sh<BrmV6Dld=L_Hws(a<PNKRAN)p&uQx!U&O8l@#o%$5s@cR1X7kz
zVs0)M{^8(ZCPLke2`{iwrZKd-8WV#hH816Ap}~yC;a`cv=Uu+XOWe0^^D?UM6wB<A
zd3w4e>;7wUJ=gD-#M0CYheatEm-jQHoC{ZANz8p|_xD?bZEe11?A^fPZDC_%2-DNq
z?W6A_VftlDU47AfFpUAf_Z-2;+lOJ)ik;~Ab1<rVUM#Zl>ET<R-tCLA?Bg)>p-#uI
zWS32(<&8|f8<Yw!9PT|X_rHCQl9MhXvoV%%8)J+QLuB@+vf*Jy<$n)cvJhI0UW=`d
zjNzv$Oz6Z*Pe@Rg>2ox*y8kssCvRiu=F#K)`K~?nl>K?H$97Up>8i~B#0@<wlXj;|
zo=5JzD(RYU1u*9%6z3SC21#}Xm`dyp`i)X(sifY^os=B2m8*oXmS?6fkc+(!epcS)
z;u@%p6X7K4kPwm`&xvuOxg8Lx)xajKPIE<d?q#0+v)dfcM6T+VX_0%}KDn#MQdBOX
zOiL~D-q>mOL|Bwbs6&CCqu&y}<(VJ*<LX3^(rEtfmHBt0_T>tDeY=CXy_RU3p}m2)
zM+<K*`Uk!rQ|McJJ^riSXl@6U*@6D1TsG84lwSQ_iGP0JD{7ps%<EXLjLsjykHX5R
zqll`o(zQrl6fx$x)k+T|K2zqL9fh^mP0_~>Ze?P7$u|d<1gd$fTwoa$<CRx7`CRI0
znBQd5nRA%PSCKj9x?~Qsry)L=!LWVBbh%S}u!?<{NYiFXat`-51qnDW8qQOIXHY=o
z9ZoJFzukbRi6ebKMuv<aM4hKWFsZ=lFo-lPwbfNFn4*>N7y%E?t(NQ)_SFoRBSASX
zz=O{0T=UD24ceW1@D1VwESVL_J+zcUxMLDe3I2+FSrT>6Ju@^v7mPT<rVz<RM{z^f
z5x?;Q&%K)k#E$&>=Z_J&_Xp(-3pJOSKb<4J;a<oufOu?uy$a5NyK7A|5Oq9D=nu~!
z;g7mI8|=?&cERvf^bE(IR4`*F*`My38670rix6*EX#hnt7mj50RH0zGaf>-pg--W@
z2)bHD((B@iV$`D{+DlPU);0qInzWYcY*Eq13C5A!Z}hG12xw%)XS8L=3=l3#rrk4m
zn`OgTcgclBl&%eFioN=Z373bn=$?9e!Af?<L!4j`<?wL(_3JrZ&wkI>Ex9d9lM-f}
z&n}C_z5Q;mS?2E$ERYvJ^ycD>-wfI8(A@gWxs7~6FXYvG)GxD9$|&+JLc9CdyXm?a
zyU%yi>V#Kld9x8v?)TXBX!IQS+?u;O2lGrQQ4wO3c+8?wS<+PErHi*hQ5n6X3BHsE
zZ}91Bay2VGGy4@YN&Q0R#n5f`dZO`&OIm5kV##F5Lys;ePwQ||;l(kqCnjr*t5lYn
z6-bo^CztnT_url0-+H=mWg&Zh{asfXMdg9Ms1NhCbQ2|Mr4B{LTWB6yE`Kf$>Ra6U
zJURNyw`dJYxJ`HnuGn+YTp8uIFu0`OeT9xIfO|7_D|MIKTzj>gS;Rc++yiA(<q54e
z1%`s}%TeAJq*E;vWYyOT9&%mbp5RjE<~MkzFH=3)kD(<XxxkuR7wefa{CZDF&6Xx8
z&1y8tn6%;7z(IwO>1z$e0^N#{3NOorcpkmaYS%P;;_o!9QeI2QiZ!YDO3`F~C;d~D
z!$$|n_lxiOazea!S$&MYDM+h_yf~2g&9GWbpG7Z2Z}WhW{-Q|KHM%Uftjes?EC-S3
zMn@Aq6S->L>IM`3%szEd;auav_IH<tZ++SQoOWmZ^REv~9|b?fe(L?`^eLc7ZNzSc
zV<m0%`AXul(+d7-^@pclf=aYLIjoc~yMJ;2a_VU1h;pr+Dj2u=p#3_@HpAJ-eau!t
z(CeCsi@vM(5Us<2kf!jIQ}6^*wC<X!V5P_lJL2rGnZz1?S-yiW2R$=bv-G6PClXxV
zi1rIU75gFVkZ~?!DN`)FEAOe8nA@+k7eDWInU0y1n65SZyHws@amaKSzq3Eu*AwSp
z;N&xuW%JXu<7?~dg9i^PAK>iM-{YBPxfc8koDiLG_uciSXd`H24w61Z9`YQz(d1Hh
z^Vpkv;;y$yR{4rAY>W>N3QlZRe7~$|uj!=eqZzSHvP87_Gylta)%O9PsUMR+y1i9B
z#=npJc=EH)r`k7UvFyhiKgpThd#(eTc9(6se49PeTk<_lmuHvjM!x58&aHemULKz<
zou^(gnEn09^sVH0+f-1`+fj!XLlTx%HC^j8({70b(Qhjk686Rqt9RO#wRXw=)NF*&
zSW%#f97*rv9V40vc9ARRV((=|+z{aOciwo6dX@B=xhrZ;PG0Vce15cXR5hV9S$QNs
zan1Qt)GSjS_nc>mPKikU)i`?U%LWX;1-}ZnvbBoYyAAYZ{HVKA*FBad9&DyR8QwCy
zZ?WC{{&Gz71OAwG6RQ@hm3vRmf2Jwpm+FXVjcHppLWnY7FXWrOB9>wLo>60-?G}6D
zhLrzAK+T@pk?u(n&RCnp|IvJoezHi$T2A}50It{9N_p*!mYJ(iovObzzU3YWUKRgo
z_0>PnJJ6PZnU(o_iHd3pM{7o9<_nPqJr^E6)E_=yvv$|bm>x?n^!YxMq|eD3%Gt#X
zx~wZ?Bb}GhbNX_$)QZwBnLcUJZ`^q!^w^Fet>fMA!Q%i4dkOb|3F4pB&W1bHw`(Tb
zh)q&kp13ysT0Q!V=NvrxsfxW?!!$^Eh^2;Y{kV16r+S}$-+AjWY-7K$6Vr*&{m?Px
zbo}eGpe%9N+L%>`_1mcR$fIs&MH)prMa%er*Iz>Gk1CJKHy@}nNQ#cTer=*_Hg0_3
z9RDLNi{f?iWb#RMxM}vy%+kox_2#QzeVYc#T>kiu_^#|uZDF_47y~6m*Y`{w>3VjJ
zbkx_~?6@QU`EBk|uAlFVpLfQp>)IUuq)T^8lkFa?xjqrC;G2k^NN-r*KdCtWDccbG
zOuE3gU>VmOQ54Z}&HCC!=bz5bKR*2s{ru|p$U&j}X0+SoU8%_ttI^u#N5e+m$V-oy
z0`}Fq`OTG&A3kTaWpoyQ7S_Is{Hb6~`KhtWsYIgmMKk;J((RJz-siW<M=86qP6Ibm
zb}Rcn_LYm}GUxWn3WtjRSzDIad@;M-yL31~(<I%Ax;s1d`a?Wp`VqhRi?xt_Imx!;
zHR`1+<<G8LNC!Ls0&V^FyNuzvnC4_-M?lf5XT>XN$JJ>Q7n)5amwfVGd=FeYG9PM7
zU-$fF-gVi#tKmxhj~jl$e(QKA5ig@x$fl+G0z`KGHOQWqZ^U*#t}kEJdKxBtQeF4u
z*&pwGhv^3nvqMaOwht_i^L`(G2|rp&S+e7AwD@$vgfx$Si>dY4>!d&2aL&;775=I2
zk>O|O;xBJcJr2LWQ5krLW9ZG^Y<|72Kg}5IzTN36a(ibbI)Tjg@_E_H&?ox?2a_l4
z2mSGN?3SCB{atin&d0w_I~=5JgUtp~JvdF_NJbA`3Ya??0uzgm^<{vK998B8>K0U<
zVpS2RvE>K9RO|4pEX82t{cyq-B3vSIW2xe?-gh`*VNoC_%+$31jrXbK_*1QzZf4jn
zi7CNXe5v4*q3Og*nG>ym(+~nm3s>CqQW)NwXu`Se?{C)uh>f!|QnlC6fbl?p7KVh=
z!|=f?IQWr)GyLbZ5}fM{`NClc7%bWehWytxn&3C|l>~mEd;b2$OOAq_1AkF~AO8Zx
zzpuuF72y5*HG&U(2D_mruc``u>)CqQ*}2_yboYM9sv8a7Ao5T#z6*oVutPs^Rb93X
z@cdCHeIsup4fX4`?yh{+x7}^*`21Zxpy$A({I7$Tu6EwmXn$81x4YN<rCH9dxDH-J
zZ}YRD&o1$HmS!>1&_>I<d)c8y`2_d`SY(LNXtb2qZTsuG3QGUF9sDKD;^^(|ah;#v
z&(Dv~Pngf$%Yk1|LPCOHK!{&R=qkA4>fHc0Z)^XnZg*M#KIGreQLwvf>*eI(?d0x;
zhMsF}<L=`v&B6i=^q;T4^R)AK`tL|?cmK64ut0w36MjKH0sj9y8{8@dy?b5T$=}Y!
zSi#8^$PA1jbM2bA)Y<j_pHKcf;y>;*`tO}$qW^Q}KR)??-)V5y&P(3i6%6Su^WW|I
z*RcQm@LxAd@k1;Bha~=r`RrXFXBlEC{{QTm3~?Xf>p-xLmz)%|^uceSWY8D<BKU{v
z?{Da}Aqg{oS3eAffvGCo(D#S0<(|u-d_D0t-+9MEKd=VDg1Cn*K-}clE&0P@9(Yay
z^JIq)-QibqMMVb%Cr6g}B~AVKwvg+v=RU=j-loPkuHpMfEL5QPSL2doQ14Nh6r~k;
zI0^$p{_|V?1{#LIQT-%gz46ZvFcerAE&_%CAO7c?g9Qe+DtZ%D`JZ>rUcj*q|GX21
zR965`RCqwD_dibrs~&<8V*l}-5d%YFW9b>n?)~FUEHI%wI12WUZvk>RA&l~xs5<^X
z-h_eor11V{CRQv66pZ7JO9t_O=L{}D!TbsSxnc}X0f~mGe;N~@{vS!8VH+g>oE{b%
zONarfO`SE*|4bVLo1y#X^k8sGax6?h$M%)}|4bVQQ1~Cx12g&m7od%*Rrh_OM!S9*
zlQ!S0uOBdorM*zdU)%jTp<gJ6PcP*!VDQXL;^1A&d1#4zC@jnaPEqu)Sz}ql1#RDG
zKV*{T5)eoVJ=u5Q(JM=^rK$<eleo*#<n%F0PEO8oyxvZ~Sc$Y|K3*M2Wk3z=Rl6%$
z|KGg=<L_<EeyXvQ9pBHF^y9ncHYuJa@gl7%?DRy$WlVth!i6%$cp9DD>po^`F-Bkx
z=$ZRq7b&?~kO+IU#|#dB-FJ=l_|J4$3b!5yk6DYr!eH(MN!elPgVU2k{%>u5TBBD(
z4t7$Q<q6sXO?YX(c7=7dr}10L+{_WL`xX-XAraT;^s&Oc&9@#Gu)SD0wQtD#<6}A5
z16sj*;LCA)@x49}!2*p1G-KRVu)S`xa<@?*WWzs}Yw10`Rqgoa*EiocNF?%|j^eLL
zHm$tLUwO?kITCl3;u4WW6dx{H2G3Tz81zR=&W9aso-^+VOf`A#QfVTQ<N5iCfzzuf
z()xsCcD0c_>7cc~p#3fO*YOv%EGF9g_*1!bx_`;YILglq=1QipDqjfPm<gwX>y@bw
zms#l$$K&el`bYS?j#yq_KkuJMjycm3KOzjQMU7<h2IejDnb`{#$;6{R#!x2J!AxP!
z<Nc+4@0AgDC*#ez*G_K}8GCZQ-J@H5evE7_^j8M{UTw60Pw(!q{`<(kQV~pSLFnPe
zhDXhkdv;+pZ05(?hnLI!rbE8aFMNJ#II2;Gg5BcH>Y9BNM$^lSI63&iZ*ZbnB)kfq
zrC)%72kvb!YvhPOf;Pz9dlC;arX^%g3<5!j#i|nOKfU#yrIU!)M!3-K$7r?ht@vy2
zx0t5MNJ*R8ma55YzSZ-}6T;l>D(}dgzsQ&Jn>t&{;A*sRjc#dNs%-O(=nIT%eN@7*
z(L*`d*tb{bTnS^y{o;*6StqO)`cl7At(H)o9L_tiDaKKywfU`Ij}2FYv<D>(AArTU
z>mX4AcyK12#Ygg+shDErhh@LMwJ8f7_Vag@fVK6rW4=5yFW%eyZu)&gmzF7VkuIWn
zDzHB=ce>qwQ*MxD<M1B-dEFv~*u83!_W9+}YSXRbf*TR@Tamh@H_6?L^btWivzh1e
zQfo=qPLjU7DOb---kR@~*UFb+XF8Vg-$0k%OpWueBpI?1Kfg4XTg)Kl_RfEBSM-uh
za0XBWB|2gIl+zR`|Jb*cdgb$PT_>JDT6XEB!&-(N8^p3$0<Bal!n5~#|Cp?g1lBw}
z9d@dr`k15i$CnyY<qY#&cuT-GhwF5w`QY;Hp*(XwI_q0H_@R9M66e<?W;)P`*3+FK
zTbaG7S1y)n=6?OPTPPpByuZCP>U=q5=h5xgW&skE50kdnM#}R;Z?=biGq#$NOXxdz
z6H6K0GPt%S#FL&iFn_eay|X*v-ml}o*zLMM;ojA|#2I#SG-B^y7Z6wtW-Ma+hGbl{
z-H+qRbtQufry8CUdE%b;SXfwMGI>smZhl`oh&+Fh%XzeFYZOSj^6t{$V9_i+<ogsf
z(19V?Hn3qz%;-^`ejWMY-sdrzDS8bGOO|iQr)g=1E9>JQb7;^9tRJp-HMb33R^Xh@
z()8Y2`z7Z10T=ji`@w4dvi5lCce3*|yh)p~`}36jWg6MMM3fwjYD3QnwRj(mY~s&~
z>b#+T2DqY&_vEH7i+dD&3z@YaewI+ND(O7R+vc4Ys%;M=AfdG)cRsdbv>(cIeLtE?
z2V-8Va!L&pE`BI`GRAU!f3)3YTEH}MqSeQFqJQ*HUyRsLlsLh#akDLZT3YXtQoya0
z>l;rS1Om2h$D=ZR13b)_1+LnV-EUw^^Zfp%cdyC$%k#CbP5mtQF(o?#y56<9Wq;;b
z?#>@vbDx^5cDe7-6r<T1>5%M_$+aH2*R(eI;$yXx1Znxw(j_<)Q6|}Ej!ehDnV~KN
z2enS*z4}SjmsVzPCW7lnA0u^LrS==s`rEPQQl)xJ&tO)wFIljsvBSQZV*KXX&o-SK
z?BB2T>B?{+us?cETWrl5>@_lPx#*__c$VMv7tu{Nvb3du+1$dE`BA~TPEXpJ+-wv~
z-^?baxrXd_&`LC?Ej$p5gV74zj!|5bOj@Af*6qpiQAb$4j-#gewRYG1HE)8MT+cq2
zjhsvC^ZSvJ#@l*RJ#4TP&8Na}bnuMn{X@0*jOwI?dKF4xg{_VZpQM_boQB+50n`mt
zqYEOuZ+ezngD<3bPu5*zUA*c2<0%Wwh-6FnrCmJfS6#x+KT|=YLX7#A&IC;D*4Ev-
zG14%%1s3~>s^4?IGY-!jhVp1Eh|_tEg#sStbNk)f`eWAW?c|rt%m?Y23KXnS7#tf?
z=Je?9_%JG(P|>+gsE9|Cu_^wdDcmYMwk-%o1;^#`)#-P$q%|8gwE1rqHQhZtIb1ib
zx#Dhf1(vdNv>GoB>j?VuoRCao^g^+bJ?m4dT$q($(LmzH6nr94W@w)4kJ1BrJLkOM
z^;np;ke<C>%AgDIFjsAiGa?-XT7K@0H;_u<v8Va;2Bq>C)W_H<j853Bq$%<$C;u_R
zN)RZ*JUV&pUzZs<Q-#V<nNBpjyLbG0Aogc{y7@WLP|6F+o)Mm>)rl+O)4-BVdW0n2
znhuB3I3V6hO$RLHI)0z!fR-!H3WLui5I+mY!il(YQL_jUlOOk&=*H_R3~Tbtr7VeI
zOykdgE^|}=`1~}zNpE*~n028(`14a3aB$gGT_csZ(iRmsVgBPNznB-_T_!SbVK-Ey
zr)!MrioALcRyE)_J3=k3(ROP1%pNb%^!}(X<>A$;%~Ublw~0+t7Y{Kh{aXg)u*qS+
z=Pv3k%BfBr9m8fhFRiK{kGWeJmM_b+haP`rk-F2{c-5pahVO~#AFAgAQco7l-RxrC
z0cTJK0m3JXQFZnlxV*5I23&E&aHL!$aj*YEe|mHLq8c2hCY^&nd-Ky*Zppx%3UjIs
zVl1C|tBPbw;<GVrS8GqoT-Z#abn{jINKcyQF5@ig<r>+dBd+emFgMPLWA=72g_b9c
zgk8UV7wX^59>kGO`hH0zvw2+8WX|}J$E=8c|5loA-P<ef1fCmdO`0~8t)|BDq*J4f
zyy`8|J&ShDTOJ8gVJE@Pc~T5fXIShP1eu~~=I&t=iMSX<ZNK8)skye?U2Nf-zKjoZ
z7hi1M>hk$9I^L8xg%4}<(;r>e%DryjcE`ZQ#~ben@5EfwDVX7dorkY;l1DuizGWlO
zWeavU@nR`m>_RT36L9|t*iX{6q(MDr=74W}Hni+*oT7mNt#2Cb9gckFdYsd7(ISku
zZy?L&B7eHgOOAFm2{nrE*Jh8EN2Bw}J(Uy_Hp$Pthzr0CV<WgAlz>2ZU+R_q3wRHc
zlrE+;jk78x)W$D9L||Iv8-4p02QqC9Ej!M`M3u%#2C_{hHV7sC(60Dcj~7mM1>Stc
zK?{jht{Y`Ak4bG;)5M=ac6g%eM6o=C7qdtBEEeq;g4tpImXW%g7Txo1Grn`An`@~z
z$Bn8RDdTAz)S8Q~MEr^KIH<7KDN>6Mf+9BT4DYMUVT^rWvL5N+&p!UPs%ro9Ym@Kr
zkq&HZ5LT<EGGT>;<DLnmYO(}z>Wa2}C5AnhxVtzY#Kg}(%Pu^8{C(JGV&*)&fH#(W
z32*Gy>w}Zc?{9ClfxzqBG3X9v)Z$`Cl`nlQfWp5)C?0NFqqqim9^4_%-gt$<(=^Fm
z)J4eAvOzfjA=-)pA?z^N==-F|%i5`}ZezapvF~l*Bp5^(<-M0+V{aZ*Uo5EDh4ET;
zPTqO^JxW^<rm(=FLl>Fo_UY?I3v7xsj9AQ0*aO5b-u9?`$&f?@=F$vDjk3>TcyfA!
zA-KFr!i=ukcacE9SF6(2n>V%PsLw&HmeIKWc3p={$k3q#)oii<4PCbbU0CgTlV|C|
zHF`IXCj%CA!hC=C1=Jtv8{)-_zy5tkf61yP>*8EEj4<OdfXxyUjJL9rU~Pu_X-OVS
zV}cuHJvOi7j3rqUf5cH=J{c6v`WkosZ3NDxe^!&%&$oxW^?7=6a7_z*L38<4G#Tsb
z@8s@-4~LK*Bx@HQuW#Dxyp=EPS?M0VpzobKYVh<{uatN`5REv^?B}7rMFVAcfw%uY
zj@Pcbs~DzjCg*j@??>et8LFH|xjy<!dBe}6p=<<hxAvODxlr!V?fn3soH}iPg(f1S
zYH1jWvV{7>fQ{Br!&i?vwTcq@y5qOMzn@ARa?+1=$ya-;F``rY(g(_bYT1EXP{Uil
zftx}4+t!Q}GKe@P?eM+gm~QcWRg^He9rDXxl7LBdDQIP+w4s%8o_qjcrSWQ~L`~R+
zYSYbq22qz<Uw{AJXyGq$aj+Q?sntLLlAW8%wDR4qHiJ}T^hY0qsbTU>&R9vK$=UhA
zV%>+!NZ5INb|(JenTQc_51ID)7;~w>D+=r|7csmmKAu#tV5?gPJ1chP<J8M_wr{IG
z&s`Al%X$$pl*1Syq}R=P!ZUT-zeDn{)o{|#>%|lmjCV2R{i0Nm)x<LZ;+pd=4E$1K
zjNr$I#oxADj3MK@%YO9xi)9*{YO;~h(bv{`yBf_kQWXiwm!BhWd*k0TncOI4<l$$Q
zY$FfMM)Xq>0n+;>R?;HaKY8TQyyj)U<<y}gdV{z20EmwIx6>t)Iq8m;EDhyP2l8Ks
z<Dwb4%PYN%rHbCWDZePD<J8PiDgN>tfvIHmyOg@168zw!<lV}5tg|`g$%**)ZJg(N
zVymMy6ggonnd8|B$>)O1xXoH>ec19|KWvuMM_{)NMe(lu_Mn1YT6*`0)r-nF_SHSC
zSPTz?;r;<gBHrr!>`r8qtlHgF?~lG5>Nxv2OG1WULN_f-oD$X^C2RWJWpqc1@>Tx+
zG=AL;$yOZe^5g}x3(b_x#dDQ$HJc-B6L{}a+Jm`aEf-{ucOF*>=WD!Nvmx$Nr<y1c
zuyfali{|WnGfDl3UN5w7nz`+_k8hNm`idvONHAmul=S1F1n9zp{`@k&?)S^(i{QMb
z%Ic@e(jnisrLx3KzY;cAo=k4)To-P430D>kh>aL|;k)y;ro_-9Kts&0C{-%OPzJXG
z5TJHZFR<8JZEJqno6$O|dhPz6=O2zJQ`7KUxG-8Th~$u!&kinqu?)NV14U0yZ{QV>
z8&K!0VcI59qdw>T1ps}8?9trpz5}nf?3kSBpY~RKhq!P&&4xwIl4_iu_-4iLBE|SB
zVZOvz<-|))f23+1XBzPR?Ux&dWFF`Ek0;t{kDgpv80%E}!go^!AcJPNDuWR(hYYdj
zN<Ko*M&OA~L`hlMwyAne0nZ~H9BqiDp1yQGc9lGK;pg|9YxhsXrQ!~~QjF<N7?hw4
zGJK4JaOVD8JRwjf+-+pf79#~8v#NKainqP0e=_qhbL^vg%Nt3w0c>?sIrlG8Qsl1E
zeiT=lsbTcAO5x4lCTD%aHu><n606{<`UCoF$-gQ3T~&C)(8G0X!-DK};rb`>Deq<x
z6WpTQk28+Zf_>_8jp98XKa0g(4Hj7^C3Hutj8(l)RALZcD4Z|-0I`TIZCk~y`z~oL
zJD1=n=8`DKIB_AT1Q9vPmfl@q#^J^QO_C9TLeX6{8yeh7!_0&F_SCl^i&GuYez%mK
z&n-xavDsyPvliKs<|N)RrpKCZX?U_IZhK<7AS6+3G^jXdRc|EQFL;Xzq|itbEx=H0
zXprFDz;q!m;bTLu-gy5&35#E4gkV>EoZ-VPm_3_J7(;rlE<C@dHEeu;=?kDul8r(y
zW>*1|`)`<lAwnUt!#$gTni6?mHjF~8WEtd4Y*HLL!$pb)v-RF9x=$>-I(gN2Ig=aQ
zpHSUb2NZ)kFkYO92C4iFOc<bzcD~r#KNaLJ{BGw2GDh9c-gWy%#IU-Dn9p8G5+IM3
zfGDHlTx)j1<zy@i?EYIyio8&7Cp;d~e{oVLO5l0ZCdl7#;3z3(NSoV7v&1IvG;x16
z5uf@uk#hsYe%eNBYnuNdp8f+D|4El&|L?VDLi#@}2do95f)r<egYfNuz${+=8j471
z4z)eoqL_4r8${e_E^-|YUE-MhNCd@vO3hr6MU}ktUj|Jd0Hgwh{%s;?zweOOws*>>
z1$AlnbmUTbhNUm>onDEnZ?Q1L5aM&}V-dQ+a^b!EmfS@+|7YdO>VNseZ<JWrhY-qc
z82IxQC93#{ym?e_?*TrQr{B}%_2_`1+k%p)9p9_O>K|2N!XGMj!l(KfBmkxW>a)iV
zFj1EajXp7?`N;6vb5x-L$$Mhqyfy1j64}t!OvHU7(f6h*IlVEps|$tD7~p#vz~T*3
zzmzlPRELr<Ov(CU*JH*oi;p!vGslRR<54lj;ZY_RA8diAf^QLMm2d#EpjAGjh-v8v
z?>_mdkhXNuT%*XO9XDUfEk2}Atde2aB$OohcY}6;`7}BtQb2M-3&90PP7iKhiEDYm
zyCrRDL7p8I(Gabxy_i=-Gj@5VzYH@Dfz26+-<x8I4TgqAry+T@PRhP0JFEKT)eXq~
zz8zl4qodNfufWW$&wn2Ouk};_BgdF`zGH;uS;UU=bf)f*nTp{$3TWIsnzhFq4Nz^J
z(A2SUrOWD{^Eq2RDh5amYjjUu2$IG|3TCfKg37nP$>RszFQ;2srDQRWvzY8GG%JNt
zhpW`7kY}bmV-8JiA$Ag)8jc+?oBlTbeW$sinb?I*FK!1PRz~p~`c0NFjcoCl)oYha
z+^qjj34@^qH$JgaEQ1z^x)vV8aDJNiF!p+!<<#t8SZb$&d%P=b;thMDI`wY+0w&<7
z{#gapnFukTfC!81BP7p6<AWdvrO_X(mMF#ct-z@}_KsinFKeW5lx`4F@l!@pQL~_b
zN#3D((Ee93+tq>N>Bl@`gdU}!h}$~4)bZ8QC~t$o;8Ixoud3~uPr_(b@;E*3d~H1D
zKKAy%to)t?P$*e<v_2VFB^t?Vy>not8t3niF(gp@vTsN-D~z~zH$h)!?k|~+8v?Ft
zF95GF5X|un8s+PTj?5?foGSRt`=s;+Je*3`@f?2Ho`T_SVLh}8t6M-A_O1bAz&{qy
z!TZ0n?yO?S&0idPzt}#Nje~Cr>E=+B#~{8tzoEwetKvP0K;UaF%+%0ATGVHL=i713
zoz(EX{o|wHCwY1^B%&ci?S}C=p39qm3js6bKx{{i*4kIW_S&M7IFFgCOBrl!Cic?_
z2-r~#u<}U50`Jzki_~WY24PV85U_pv${t+pN@&Eg{Mr3Q@77v^=t6)|oO15zX)0q%
z=QaGL%vo!Vq)U*NVK5qC6?aAyr@)fRIDpDs?T|5%zev(Pyq_LL@TOS2lur+n^QpC#
zAr%Rk+dqfUAZ(Y}6I7HMD^@kmqnc|PJU!(hPw_$FJ20%NrY6z+9Y7Hbz|*Zb3I<qY
zw+aZ>zjR>mVIA(@wmn2}t|uUVE;!h{ooFlQQ0E4u!vd^e*My&6cOihIv~w|InNw7u
z!(C)=b&I!au%z6L4-k53djHw8hahO}>DU(VKzqi#amQV>UP^YonV@of-=R2Sq^V@4
zz2RsyRG$Qz%K)TD1Jz^`K&|wdtpdD_E*<?2=Lra6)oYSGHG7LqqM|C_aoyzjz+j-n
z1nn^lVZjE2&nRZRJ9A1In7z_ED&N*N%kB?F+cmUI<zwW;Dr={Q=#+!MS~MUBhTy$Q
zDhY-VVZ}X@9mlWe!oMnc>JsGfOO-06Frv#E6LMyEJ78<oPYGeb`C*%h+P|h|0Y~RA
zY5H*MW9#-%Drs=rg-9x+MV*dEv}c0E9fBL$ho~fgc4uT@7Zx-En<O7tl5Z$3<-?I|
z1M;3S*JllRwdo)!!r+_7F63}DG!hKiE{~pjX%QGwo8l+5Eyvfnder|UUN8K6=&pJ@
z>Bc?wcZk0um_j2cQtU(S2#ZFA9xD?cmS<YlZa6KSh9T!tTGfG8@8Of_|Gj>O6v|?W
zAVJM2B4(GOb;n-7Ed~UAjrZl3@|~G<cf^R*O{9F4&}Ryd#qxmX>%WYHz{$W(++|JJ
zZu+3CEFN>@^Ek@z{_NkL*l%YKO9$5-)f;PbgLQOJ5K4Xy&&$b`!&{>G?W&R@MXn=t
zc_-{{3FlYQEG~_k)5$qu?TmJVhPeRWEltD~c>fi^6)?Fg#O(mAz~Dnflar?TPoA5}
zMGvB`t}5%xubtbCgtgT6{K-A7^(X=@86Soz4IRHWI(@3QQyYS_p8NZdhrlQJC!pvc
ziQpIzfDI`n&ZjPOK9w%n4~cmaj1=Opzgp1xlV$`E_uL{PDF=Vn$+-TijtmD-o$myg
zP4MR%^afy;^+1(|8$Cx?&2*FyH!m7xo@YX?>v&#~?KlC9EDv~wLclDK3}gy(v$6Gj
zta`+Bwl5%hpkX#d7kZ)H#4y(e{=RePLjm0GT~fH{V3kUyg_gEk=E=2^cTB|Uw{*Q|
z0sEYT&t>|Yx7wsBE}2c$CQt@;rp%on_RXAM<)Q=*Q2<${o+$^aVMQQsA3pf~gL^#w
zGnH>qll9v6rzCOqAsx(7)u_wjP(J@31mGp2MP0utKw0n9D~7+aaRs*L_=>I+$c8on
zqo9BU?9+oc4=-z3hVHXpHET(ldmXP#{wK8XOSFt|MpK;#C)y|nd1Q78dC4XH<&f;D
z`Zed#6k!<1xhSc*v~i$51&J9(_>VY<!y+x({h8B`Hb(M6j(3wBMN7+dwW+;Dy=M2e
zP{HS4*;Gkr$=BS>lgtuCpbAwSM}1`k6k+DQ&=xfJ&lZg?fYj7gyqF41I|9{mcPVjl
zEeMqR=*8S}%uZ|_ve@rg{JN<`wLA8!o=z57_UtAz{uf2313;#i5Q+zEvAxf_7M~^J
zL}6z(d1hLu4JPo!^*RMju(Jf59=|3fo@f^$pmH3|RTG74cRqRC+5GV;U$5tztLA#<
zij7C&BdU}a4vieLp+`o1g<m^EBuN=W6ApHNGPnj$p7~YOMh;+AmPj7q0n;NQT$}E)
zOgu6LMaEj|h{+v>)bW3rm(DlQ_rH^hx^%2Y<^;bo(9kGVn>LrdxVxka*!Pm`4>MMW
zA$WrJ_Xz*$ARho^9(2?!kk3fPeQk2C3_U&=u?1a$+1x?9Yr{n`EnV3br^eU6ibr+P
zr6kRzy-=p2eQT$8j)WE?AAO+|RNy`WyW;ko>)|U`RQS8|Uw~D#3=#_hJ62b*0=ceN
z@jA%7xxNLL-Av`NCr<MKu8Gmu;`riSvxql@|MFzdePmDeigQTcWsC9GMWb7Yd>EKP
zz#_zA5+MCnxDU<Qo(-oKGWMR+ea9|6b!|~N^k|Q-7<hoZ?izI;iOyk`sP^VK#Pk!o
zD-Hg#7@*)K+1h!Yz3KdxRa-Y${!VWRJTB+%O9Sv1C8d>EK9oB4YkzHa*P`UqWPAO-
z{Hc+82ML$B;Ona)YP_ehOFYk1Hcw6t-B*~GdmnR-Xe>MA&$>jaBzgQLK~#YrrOhOP
z7=p_{{+jBzB6he{HVvp4-E=;48`tHh)7y_1dN+;Azc_anB;<05e?dNcIy9BGSB2Fp
z*BmC3J;H#zZdY9E_FsiW4?0kNoTS12RVMWHu#<N0?STC)yOqVq=_mda-8Dm{wP9I7
z$&n``1jyc3U5Gh{&70r$3j}1G0)Uc>1n+sQek^}zxTVMeZL^g<ghbpz#x8)@?6`A5
z29>&z{_3}n4}R;n`w#rOQXN|yNV%z-L!4vmLeNQPK=R6*EVwOg?5M5#sc%Q<ah6Hb
z9Ys(GEC%ZPVQ1wdk(*pc9Bn4V^(6qU?4`f72oy+xqt@-)^_4|ojVzI#Y|Xp#@5m)|
zd28-l<|<ipPP`Us8cg0?4y!AsN?JBI;F$uIc!~2HpiKM`*yId{K6T$aDj-1;usH)C
z`fAAmD!z%3${Jt@iKn^BqkIBH#b`Ha^vx>0;P8Bw4L?RZYL_Y}$g4`=MOF;i`S0?t
z#)MIAM1aGteyU0iFdcA)Gf-l?!3C5kBN@j3du6yddUDycBhbgB-M`M%P3BuTji;Yv
zhm6@&F55voL-Nu4uct{VPrigqJa-+@aIC-GJD>2rZT*Sx_kvbtF7QAYGFSmbaOD{t
zUSNmogtggPyIimORe<Fv1CrHi;(d8r)6#ofJNH-xGuyNrNMta=+lfrlJkm(%kb@LJ
zILPM*Z^hc%6{8_#{2&QDMMpDwl?=p48i*s(X}a^-5!6gY><4I9zcxKJU9n}9IEpnQ
z-jOlR%YEQObCM;nk$XUu9QJ7ObDP&f92LA=E5B^K!9fp{(XZ9u5uf=BX<#`YY@Zem
z0tKjJ!MVwd*Bq6qr8~a7hwnwFS|t=`$I|}_Z)eqto2CDU(c(;4FHj;HhOko>V$IdZ
zebT{u+O0%T5#OmNhWyi0L;n0vc##lp#^wRHzFgU)Z3y@qBT8{lU<QpFncc5mE!a;#
zlsVuU4G2gUIC>LIEIHNlnD>a&)k7#w?{>`{i!ifmDc|nb2WivzTs3`O;`FiXrAEcm
z4HN{wQ@}o{kFN>PBL@_*#|K}Y8a|u~SYmwPB;qibBZH3$4JC}sWtjhX-QUdL>Jo84
z*m?Z2jr^uRA&0xA>z#+1OpDKh_q}eWa6Ni!&w>X@R)7Uami0N03j$&zGOqjZ_^f`4
z1+_}NuPvUe0pH%2H?jSh+8BogD`ENMF#?hErkL>c;03?WOC2y!QZ5Aqiketq@N+!S
zL;$T@8aywEX%D1GRw%^U;$yPJIdbc%oqjhNN;Z`l!_mmx6^+@8$_ECyHdX3YW}YMK
zUc~C=*WOfg#?|`#xb?)mO_W_L&unmS{(bfrLxUSHAk&=rCzR15Y!BW~29ZRrKkaJq
z1;*<i?h{e!`)$)r97I`Ox?$~b5I3JTW?yLDI@3e7HFdqqWS25^Gwk^HMS~~$v7naE
z@$G&Z=uLT2(E=<T;B`=hvufpteu=hnIh*nBSJ3M5fJr(F^kFRiAyA&Q(5g9gy18@E
zIX!v}ts^72)s>8(SlpVpe*IR$xBIs$w8IB;B+hUL0coRTIj)Xxj|6)131kHMCU`yo
zJ#EC`Kyke!TPxJ=p+Gx{0{_dcBbJVnHc?}y;}t)H3pXcH?rp2H6DO?u`NUVst#LGh
z0%fD~7rqsaMu3+D*<Xg7?Uxjg?<+Au5+wXCYHxc;wlrJIm*@rXC_KjXB_|}!_jUG4
zJ*j+8Op7+kWol#(_s9m>HB~ju$CrFIOAs{{9m#HT7x~OoK)mCaqGh6@npaIzIcp|r
z1wgCd$D;fKh({T)pg;SoPivN|o2QbqW$rGF3<*&?Ty+fajO^C*D~>t66*nO>#(+w}
z-V+Lye`$QU;`M2oPOo#c&Ci`)!mA{Q-A1esl=c&Nc%DT-YEcMc9c^oX^(8{ueAwlK
zPj$A+Jo*($R0v)~0tK&DqTX2K`hAoUhc~9>z&`e4aD-6f#oSXqDimka!NPvQed7ju
zoi^W}6_{|uhz3_aY4}+Tu!pwpIZbrOUuA26nv{T7f1+DTeAT@5)B0p^;Q6M7ef&5S
zCt>uokxH6_Lr3b*(=DO`+--I)0S$R#o30Z%B(1c|RC^`}!1COBWhuf+0bBEet-ZBp
z&T$Rc@z^X50<zL55CwtbOwA2tdY>!qS>!b8ZOX({5YuMLJvXHp=iidgH5WOdL4!)Z
zhe%FO`z3{^XHbUSGoBojUnfeuB*F6SekbTwX(a-aYn;-MI|h=*;Os!y;VYJdKtUMV
z<MbqCW%2G%o;1{L1%)SB{eCf@7=P8Y^75W4T-V%h5n~GrcC|6vy70YT25VGG65<|3
zyv{LO{o~6?=yLDxbV=y83U{pg>Y8$?MQ5;aE0J{2F7NT)_lNp57Sgg$7@r^@<y-|l
zYuXR5dVvISLnu7{B7Yxg4yZkpgPtp$Z>>HZW{7%WhVXM_Oe$i9T_*=uK=_Yq^;>rn
zMb?7a8}2^jipG`&=#V=qGam}&N8WStpIVFM15$LPn69GXGKG~`A%J|vpm|CU#0e)*
z<R~-NU5tJo&I2N%BL<K(c<;DC3WO6zK=p!4KS80jX|gd>($8&3NZ}631C7h>8`t0(
zLg)j#O}lDJAXa!GVFN@woH!Uf;bU4Y5=o+J^(KMtQ)fs};OX(7((Iw;>ClNJ-)^o`
zw7}g(^%V}Pq3p26b9VqwRd4~Aj!z2<@W^m7qK%j}K<Ya!e<<`?A#2<3d8xr9r|6YV
z=<>BuwUy{~!eg}-6cNZ1qQOw!V~!O<gtWP^(c26ik;l`Wp!C*XifLcE4XU0yn{O{4
zhL!5bL+0u62n7200u(8@lL4`oS_-vr1^sc8IGApBw{#^|;zrM4W=2ZYfD*TBN?>0q
zgS-Aq_X%8p|MD=XGI|8s%Q%ng!+xiVXVSot!7A9H+zOrID5?^uGs^8->wc{$7d;eb
z+aY#Z^3tR8z{aAHwe#KbSe0k=aEs(OoLc<G^$Kn%(T}|!pWIM<1GZcS)a7RMj<^74
zfh7VeZ|v5pp2|bLNiZN4umvL_+@pq1rA#$AXweJXlboR+y})}&qN68?6(na|%S|wt
zfo{w>AWY(LV3Nb2ilTvko+iaj2DCLW8uZ;LL(h;RKp$Q?@XOkTa*^}f2cxF$0j#F;
zXG~!h6mO}-h(YgJEjtLe^78VY8_8Fx+W!P0K*tr^RDO4ESY8K!%IVWz=%~+Pv?;KU
z=OP;*iNjDL+RIvjf;Ku1v)uVuMyio16bX6)MYtIvu~;#}BM^wUg5$o_SbhYqyW)0l
z^3*$;fU~f~0k%=(Wr7?K85MBy0Oe;v&Xr#1HsSB}Y0DRphNutJkQLk%D+KPb0Q6q<
zhw!!v!BPPwBybgI&PP#cl5_B%L}rjl?MvNVBzd1JnZEj^rY3}4n}yK-5v+}<`c^gl
z&u?w%ZxR{Rrh~U1T)eA%3&{O{RPP)DRE-`vALq-4X@XX(LLs|8@gAHq>H{ZS=4uP5
zgEqR13*DLSl&;JEfWu)z=p#&k2hF$#6pK5HPU#-joycXpKJw0&XBv>Va+3jw^Tz~n
zp*JvNxG&|`Vr4Rk^V&^!3<B(7llKGZgzAB?H|W6%Y6?OBHw-aIc(n0~Y~;N|I0C<9
zqDRDOL_9vq8Er*~0N4?9pIsk5cR3gXO0Y?J_eiX16Cjj81#XwW#+d+zpRx$}Z&)3G
zKC=a-UeM*C2YT*~p5Ch<C9J?;B~R8kQid9V<%4R!ej(Lg61<KRJR-<f1u)_$e7Ns}
zu_DFuCbd>@0O8_5qD~-|uT$G?XG;M<Qh_%mJrffj2+*L@G9#*+2bhOBq%(F6g(K*h
zE+msoFvyQiw)*);-Y;R{ct_3k6?B9ZD%fukP#gxmC2X%fKt7|-OKN-;cz%G5H*<*0
z01FpHwE!Rpa)g5&rz~NI3)dV!prfc-ND*&1Jl3bAS2(-@pxgnit9l?INpKaY0)qR1
z4_0n2LdipfIMJF}9xkrmKHXW-TUp$GtR=H~db2nRT>yu~-r{LH7b3tY>01Ii8D42r
z5KQVIK~DA$!j_L>tkvhK_jaaKfET#`o8PpLAbU|bs2J5|e*iW;+~WDY0`eFBn>KNm
zWjiGjqEo@FLzRKit!E*r5%3W5nNk_ouZ@Ml`&+$(vXjB9Pmi@tU5V5)g%m+h(JhJb
zrp&tTbI0`xb_Bxp7(nXdY=QovoDpktkIh3uH(y_Dw0TY3vY&8Gl}&^){P&M9dL4nk
zV_plj5xsxJ8l5>1T@8(i0b`QU!wmp&L=J);1VkZ=F!bG~yL72aOjz3zwLupS648h!
z`Vd%2!GEcIIb^<}wZIDO*f!J-ZJGVb20}f+W0hnN<%gXZC!Ubei@Ym7N}>h4)eHcR
zAVmZk<TTqIxwDXNnMIAi27=KCPz%Ps!RcHgT+YOSqG4%yK9v5gjW<;46e|Uap7S{e
z&!=n&`t!`2@&K?w{iNN1e%T7h)rKgl?;ukb2xWLBhuR<x2i<X|z9;!o0dh|bYs$1_
zk3U`6XQ4z1NQPhJw@?x6@LIUJI{BhOiXSZ?@L|o!N)|ZLIG`{3_v5Q>;JA>_-KY7|
zIoZ_G)B_K!AId_B^%N`4O%4DNo*_cukO@Fr6d}m+qAx<#7jdYy2zKuwxNH|0dbFv0
z=UeNwh4<Npz9%2cG@LKP^C5?=WHnNvy45%XYRJIXRO*&)nM}PSgnT&_aCF2(5bdtO
z0xHut@y=Z+0mK0%zm`glxMzuY3)o3eba4LitP~JUdad3ofT;j-32KJT69Bv&2<|Tn
z14mboEKt5KwK`OM`aWB%98|P%?;UQdw;&Kt$i+Qou-=q?Y7wF?V;Im9s8`K<8-P_S
z8F2HW^Iann#4w25`M@?=EdC7f$?O;KBY|9h66HwxX|@vCt(&%UV*e1J&%CM_07w22
zEL&%cknhi|ot?W|^Qu|boN2B(jr5??UXnm=&6g4+#s*NI_au4}0KX9Wdyq>5(FRni
z?p)9(Kn5$%jMdrJZx0E>K>v0T=%PM&Cd5StBK=M5xBFnG+9&3|D<hGRU-h88^EyHd
z$hZ#BaH#W+DhPxUYpw0u3m~OdO=cVUsC9C*|G=JQ4GuS_jERXUaPQoY;?T_DI=!_&
zKjXhSxAK)k9|RMeAK2;Ow@mC2SW8rfm^%~XjsQqul{{SwKtiQ<St9iefg5=jaoP7k
zNYMh`c4sKA2y*D58o)kkvd^XB!F`BG7_X;*mQ)Pra(?V46F>li?(HI~GH!^X7jeX_
zd@N6QlWVZ+U!0~^`U$rA2$V0d)7OZ2kcd@E!t>{g0VVhWbf;H<R8It87GfeIoi3{s
zXtSvi?c6PZ3!*}y(BJNd?#)J9^j^$_^7IJs42iFFx9(w6ao^uQ1i%L*W4*(t%<@r2
z>$9XlkwXb{q=RljJ7x~TTG{I7r$5TKD!9QXS^$ghQOOdYgS!%$b%mM-9!Pl3-M{pE
zX*dkp>SGFs+{+;XfKe6@4qfkKvLdjHQ2bO*MBKoMArmjl@=2bOU6x7taUAeX7U~{9
zAp%`?X1|DLI^dwgzZ@89Z|RMLW)i^85LPI{5SXj=?=B*bVsM~2UJme;B?_?=s#>!6
z#8ex_UkiDFNb?}P7_5XVQGGm5(FHICPNSk+8&J0-P`Ls3;QU8`Sv-^rz~O@wfDl1p
zVu%?s$Tw^P;D`c<c%aS0q}h#e@;-e-rZqL8KXjYz{dYj#>9~#ASCy7sp<DWvGT$z`
z$s9LVYYU?`Xy7NNOOGRj?FYEN-3Nf+v#BxbWgvhEV8;R~^4%nVF$w^$Y7GuUWgHq=
zAHb&0EPkXxO+{c|5*>jcs1G!on3xzORe*lE;}LJo1f(Gegf&?h*@7;>Lulnm6$0Ah
zBfzf}AU&a?r+*S*QXB_nKE>g&I6$|;vGe8uL*wmU3U0!7bAl}>=5PYT2zkzMhZLbn
zv_TdEtTa~oWFSK@B1_Eumk1LdL_<d4eoi!(W`Bq{M>f(EOIZn0{6&FFw;<;M>Yr#6
zsTLx{0rLjnhu;mJ9v?!`82FNZtH2%Pn1&-DwwwTf{R9C%K^x@c{U=HcbP-q?h`A`v
z)mB5rq~QP%tptf^ih#A;%F0s5pQ(6J88>MBcx-j{hwDD8AAld{cAIPg)Rr?yXR5%`
ztGG4@kUOeYz^?<80a9bTC0Y3>k^>QB(^)kNmmz37&x?trU<dw-id(nBf`b^C@`fyM
zK__20D#)RtUfX(Sac9WkS;ITzy@6h|=FpIXAqG^ffT?||&|&RM;eHGv9=jUN-Fw*2
zXe%4=01mg{jAxb*!~-WtMmji8lN#EFl9Euz^!KxX({a3M1%kYOwaGOQ?!UizpkJaw
zuF<b-4fdlGLdT{jT#U#`4ggI9NY??qyRdv(YyE0+5D(Z4Aj$x%2Z9!Gs!o8zA2CH$
z{L>cc@59>mtMn@=018L|t!jK<P8fPbG63+M!9z5=eM@FM1zfPISfNxy4A>&8U@+Y`
z?zTpbC<JKkN^+A~pXyM%a;vI<zjLqq%z47#mK6TGYjhwCJ{$!u2!v{uDO2DLX8L&w
zU4W{lf@xKd(2hp{^b0!H)mrxx0CL#b(W|w(;eI<_3Ys(qvMi`V0UVe%$n2b^+5<c&
z;~en*Yg!36tibt`PgTakeE<Zk9XSkS4AVxvBnBgNu?ht4I0E~szpc8$Ap%n4P(x*X
z5U+4|!SwTl6)vC)=))heD(mLUgj4_|0-acC+1Gy{A_Q$VA?Ufx0VM+UHKvqY(%XYM
z#t_v0hmF~i6OxgYgY!)v6X=91tY1X|l>yN%!H%s8VogjDfb2_klN;|4Xd?pEklN`{
zYHDC39{Iol9u6cSkoxQ!XMpT3h_PNfVc_)AleFl8v&@<<JnZz?5V%0C8-P#oTIl<G
zoB?|QNNu@xSljM9g;m((5o^%SvJSvaJRkz}wY=$ungF^uak*h9n%9H(1T^xbGQU1|
z{qe^~2_HPx2GArZVb&fPT!S!2DnK{!d0n=aMT99ho~6;J$P5$-jba2b?*R1v4o8tP
z%vZ5pmLU`?943DyjoJu<bbMN=q7W1(raH2p7}jV4$v^(#dY+o+HsYx@3veG6gd3eD
zpiDCY&Oyy*!U4)+wYE&QFd_Sm41Wjgi$(Y)HR?kO4xCW}$O<F_s$%(>LUtpK!BhYS
zK<yaXQjbH9Hs6j2(@oxQy?J31#3NwTdxk*;q8zLOcRq2gerpp;<uf-q$>2~+8@P?M
z0^Rr6DMDV;=Lrz822wj9b3w<#W*UW%8(2aFObbr9hlOdC`z?P62db?JfMYo*m)KnY
zcJ3#Z1rm!g#`rm{n48(bN_jqEKaK5M@=6TO5KK2%k!c|-4VcC}sn42en_-Q`4#2+F
z^0`j=1hf|r2kaiJpSV}5W`GWic-{cc?zc0^oeKyKh})?rh<?$@z5!0tsHO*kyr~e<
z;cQXYU{mIlGxClJReP*y`{gyDvv$#9Y#>?t5C=5lR*6pH3ei|%+9v&15I-~Xh+x1{
zm-AsmcR+>FkPm-#qM4tR@j6vQ)`9yiBP$QEEd=gX8B3RuyKCd}puos#!QZbI1`(mB
z6|@k20E*aXY>0*hHij32kq6h)cPr^EvRVBQv;~@%B5~LL%n88|QJexcJ%n?{Fn>|u
zV7*tETmVTw1mG1NoM>19Cl0w=9)l-T)#hz!Gr|*H6X%<t1T6=a?+P|tuP;x!4fr%c
zTcsgk#S74=kXtgyc}IerLd~pffc%WyJ<j%_Z15cp{?i*k7eO}xY7excLVkM7`!3Kt
za26u_rOOy0qLmp2FUT&FQ~WJA1E}x%k6qTU2qX@M^yc7jcwnW>>RFSL1+xc0Z6FYV
zBUgaVk0hpUPv#ebtnWX?G$W$PRv+M9r2cCU&K=PZ3PW3oBE;OMQkaII(vJk|*Cz&7
z8kn&i|KMz;DfYl|zy-&P@mRpE-()UOmeh6t(?)G@K-drzENnOlL5M2<@Il+%{$V?C
zS~fSplW9IAA(5Ae*1okFw(gS<5j}E)4xX*i;^&`PcO%l@QXfEZN6-QPsn%LPOFBsO
zbu8uEMky<C!o4h@1vRFyg&%o1P&fftT`2{`iIq)+n~cwNi2&pm*%5%pNpFDJ^`#U7
zOri~Nxd9K+I21xb&V{~yz2zLp8EGYbd4DetN8C)~RRzc5BC>~Q{bxlXI)oRTi2xB<
zQ^IRLDT0Ge=r#<hjXnU*#gXLFYr2TDV(za=W49fkoqQl@kh#e`7;dZuoEHWU0cH+R
z10cmQz&BaFyhooB#{yRV?<4>lj~sjN07CRaOc*%N=mg@O$My<~qykVD7(5aeOUYUC
z{r=&fbxQuyqa)DiVCoN)Upo^lyAS)QqmFR0GsHCTkA(L!3Y_`Val0d&7FkOGr5TN^
z9^c=Pg6#&7TO{Pb`_6Dtq;F&~8%jVB;0{VL8FjyhIDlFg9H|eR?8Wi{YwpR9*8{vc
z*s>UfnZLC?5F4FvSh&dTcS$OqP=ycI25h~(pYjD=3JwHr;F$^v&?}1#DvwwQnyBod
z)*b9Fvb(!``A1Dx+2zo^a1a)FEyt^cAOLwC-27=a&Wuz63hp2VfU<hYTVmfov0x{_
z_e=-^)NLNS$;lqB34?Tk8c-XT1-+1PP*sGHAqW-ZttMMMKUJHG@#qxYpWMl2QH*o6
z`Tn<m5?hX>;WOg`1^9tXy*wrovBEqG#XwXHfQV=zN;5dzMl9dF)w>XoFFF7aIf3er
z$9C+}S$l>EOYPkyO%VC4yYK5F+rjB3`qTs)4DJ$`@WAk3?FcDiC|@?+O$MAEx}tol
z{E&%MEZ=QIUjIzN3Btop_LZ+&_;1X>H+2U90t@;}g`t5zo)Lt>0ZF*{xa07Wlz$S3
zdIpB^x_9l1_-1IuFoX$3OXzW{H)Y__o@W*~zzcL#y?;)JhLAQ5?6P$>P1Fmu8PMh6
z)fjuM%Wu(c6nN|@M%HugY@;^#!olg#{?ZK7X1CHC5d_0CFC#ikVP1G}O2DTGSQo#V
ze}@b`J@$KI()7e^wZZed5|2rv<Gc#U63+xqi2!;D9U2VUb-6QALIjoXXE;7_YJ|&?
zLhU(dQX*j+Evw(FpFb7c!0XEpEC$pDnBtg2RLbA^KOrY1B`podJEsed90RBdTIDNl
z(EwP$3MJ`YKf0gO5&^hk;kI+tqP+#&4X7~Mk7%F{kglu~4&=+`gM+!MAkL&Q-5V*_
zT5szs00(`!ftu!uUV8GA4(_`>-sEifVz~KMwFyufa28ampQ;ja51?^~c>~l(KBilS
zssgB)%u)bq2U#icau|@=1NwL*x%QPEgcAYg=ur#n++Sp%2c|w5+^S9n!Ay|<5!N~s
z5L5GfIHNE=0Q|78{95}w>H{21fKi0v$+9%U@7Ls`>=3d`;+Wt*_j1kL_H+p^P)Y-q
z0s=JD`jGtrRLcQ{4(bM<&jNHH@aF|z(&Rh7ad_;m8B3V-$kKvY0OH4ODI81;lKzPz
zNP8EBY7fDG4>17!UjSRWgJ3IEFBY!xPyt&*L#h7o_Dk~!l+{HqfXND@$u5roQlQeP
z&W50l#2$-1CC@ncf7pBLpsd=jZFp1CAR!?wQi61MC`e0pmxOfZg(xB2-Q7rc2uOo;
zhtl00*So=IzUTd(_nmL%`~Np{d(SxAYwz=1d)0BQb*_D=3N6O9c<NU4mdVg58{sG=
zfrsSPD<m)>gS<h&Se~pj_-al~I6iVB5u|}i^(UF&8R-99AKgg2?<>B?jet#Xk1lP1
zC>ONht#n8xJm3v~1{MAZt^vIC>7Fvf-|6Z=&F{?#$k#Hn!j!h%s_4LeC4-?9Dyn#7
z!qH(hp-2S=aILqK8Q=Lq*@nmn?;d)Cg-0RTr+?rAU;Tjf5$-uW<fU>X4fygS!P_ok
z!sxH%BiBAYH4HK(1Zx3&0Y3ikPoV%p?$|px!DTFB{l1Y9Mi&N~@IJmz10^U<opJ6W
z1P_T!r=9_mq71&LCBPuT3ZobGJ~r<3f{#210GLn4;pahRag-3=C4=DrM1TU37jB>k
z2ofPv%lFR#aqlF3B5*|qNRJzRMveq<6ugoLA4!Sx`=*Huw!K##$LbNZk-(Jtz)ye^
zMlznMtbuV=-xyZY=TZzHxHkXt8C+2#WyX0p<Rt|3bN>;-e}wSgn$Ru)3)QKY%J=3I
zReRtOV@eau|JsMG)Tpgd*lyNMH#<+aIDqy;h320^@5p|K1@%?9-RV)I>vh?tjc!DN
zg+|BFv?U*7wZ4oMeW@5$y4(+^af?p#nDS(Td0JW7UB0VkgYlggwVvf`sod{brD9n^
z&mUTL>$d)sHk~+p$qCxtXV@;ijNwv~OUbQJE2mTOXOH1FVO?j{)gib2Qy$IObpxdp
zp~$*l3SIDql6CSQVw5b31S85tjHG2xk|{ID4z(Swx)P>){g-W%adCft3PoCv3!RDE
zFTHGi4hUTkyhr4^`#bUVYnDUW<?1WtKfCT{&Qoa4c45}gP*Z1O!xqj@16J<z4!pZ&
z-q~w9&8BV^+|9-3OGyeszr%Rw*Ee$tm45%$L8WD;?R8IIMq`+dInZs9L>F!PEm6}}
z(G-BGfK#E)FhL>zt$>od0fzwh1J1+J^Fs8LcN}L_cydOqH{x>%*DVV%&}htDq}Gc=
zghu-l29<+}gv!?R>ECE)pHiMo-`!K{DNpUhesV4mt+>E3HdO|iTcu=`F29%fW!Z9G
zKIYOO+Sjg7v1JBgvBZ<j>ZE;IK+sY(+Tz0EbIXX0HWol!zTWk4=6xM?H}&$799eMh
zgw=1)52eg)@L8>G{`EPL_Pt`$t)wDVei}XXoDSP+fYb6w?q{}mEa+Rb<(9y|)PXkQ
zXpL1GPd;-|;8g+hf4NA|AxZ0@Ko|_Ws%|TfRjcYL-S*<|D{-JFvknaxJZJc%g_tpR
z#0bJajU_+Ll%odmB}mz-0Ba9hUOU{Id{l{^-++qFKgR)urHH?UnlKf8<%1QH3;hI!
zM*<0DES4n7?wJ&?aV&WA;<04eb7cQHv=f#u3ZO%Cr+RW|DR+_p?Zs!P0YWJWo0j)p
zo87bAzBI{JHTZVES=O=CfoZxuCjt(_S-<b9w=$A1SJ8ysSypY&*LYBnkTOKH%#?OQ
z%uCI6uz)&7-E1N=#?c{EHCo-DSBA)n?o!E=I8<KJEul$qNT~NsP^Lz=1J)Mz<0w~T
zr7%p?cTIYKe~v2QGG5v0!{jnhx*e|(lFT=xA80qw5gt|I;E7qlyTAj7i>{8htu-&V
z?M?HpB(brAMVE*@?#~e1AFk+fWuO>#W<1<-fgFr&f9@$ZG491Mt(P)sE=hX@hzr?r
zvs1oAuPQQ8Q3Mat(+n%5T-lZ0Vm<Kryx%xodk}M%-oxorYy6Z`2_8G|KW}n7Kw0GW
z2(s2`mTLYgD~Q9x7R39yE_CPYzRR>_dwCg}6>i}rg6{8Z_p)cUJMhSsx8bm20jJXk
z@$Sx!?_^L)`OjW@PidVxJAIYq;K9`}oa5!3&xuB#F1L^gQ;PhrmbVYXq)Ux756{?8
zw1OV4&oO5)+=LiqY>->;_)~B00~hp^wt^K$=LSD_j<AfNwNRc6od5Bt3@wedcXe3Z
zGuaIxU+%xMb{<tdrhUc#MQ)FWHX*nzV3TQW**5Y`x}vZ?L=0ocYvM7u3Afa~du)v&
zU*UX=ZVP1@sL0?M-@DUYk)%0ONQqamBwT@h1NKSFE1(`{<Q5L&zB;%P&pp2JEvaP<
zdmbS~QBMUsH9*M3EZ-_vQ#o5g+Hwv&sOh4MhnYmqD~wnQo<)c8^;K54nkt$3k-3LU
zH{EjVzS)nSG*;BMrL9{v5XN_Il02Pdv6wWkqb7Jps~Mao;GX4~b1X4Ez6~c~o;Kst
zK_W<HOqUSQ$V8IwdA3u`2di(ee7Jo%IbX7V`fGac{mD4{F|jSpB4@}<-Gp`YMU*d&
z`EvhT<DkHbQ0nDV(?juPC(7D!7HvUqfAOguS_BoYj+Fp8vF*yEpGHLLR?dk+W9B_^
zd_AbQq1jc4;j`3{hi=g~0UZ5`-<;1WkvZ@(7}mH;2sl$2$%JID`#nFT*A>#2A45g|
zcGob5SLxm@6D^T3-|vbT_AR_^(&{iYa#|mjEzlmAdpyp*?*N6i;^7IbhLvQCC41*~
z!~K#T5tC~6H2FR@J-)nGqrvNny{~bDb8O$VtoFG|P)QqaWQXD2xpIN4)BIY~h@#hR
zwbE#Y+pbl8r#j_l=Te|z>&S?0S`^vG;vl#^)8m-x@itP1?xK@Kvpq#*nBHadJq}4^
zh7B50sN_bzw$iQ2lWVEb!o7Yfni4(cW475Aj1uP(;4$j24U{9919u8B*`Hvpb+LfG
z+yDt_8g{oT<E&->Z9`nast}xLfL8NdiH?*rBwZpX>J&Tc{S?;H7JPd`qfVg2q}y5|
zUv%QhdQ2)I>HWP66pQE0_2f!e7i_1NuLPHyAFa!k>u2t!uQCzrSgAsuqK@B3`VC*A
z-HV5i`j8g(nuz87E;eJ@`n{pIeLkQz7Crby&}iL*lxhQm+Q!wp&JoLki;`W@-A3D1
z0=TQSr%%P9oY)LcAx}eJ87dE`+l!=x^6w<`Drq)4?=+W8b*&QR;?w4a@f%fJ?-6gG
z2MM5d%cn#P2g>n;Z#b-s+Ifq+Ixca7BRI;B1@L4vy8N}$I95CuaCu^^)nj+gY-pC}
zA8d{`hqf<rckQ3~e*5&Rd3HPZQ`K%hlaxYx72I2zv+PSDCX?h4e-(4-=@ISfrj0A#
z&jb#(v8yI@OXA+M3biEDJW!f$TDdQ-jvL`^>kRNN{u=CUY8Tlb%<<2>DPu58<|)h=
z>#ZFc!=8^=b5nWHyg+ycKh{E27YG$k3fZ9>r-y|I<$Sosdx2NQ6u@<e$M%71&8uuP
zU~-^;Gtx|-6~FhrK%9KUWeoJb))4hA_A5U)rrQD9i*x?gTgby)t$ZBG<Vk+&=~Uos
z<Edi3eT&7pxftJOdF<cROPPuu=8fLNo`(iaI=N^yFsG*g*v(aOpU`31W4hg5quGEz
zunbzW_^+u$as2mY$$TxHU+YM9QGSYG0nPBH2{X(*Iik`$th$XNFN)YQS{~J8CA6&u
zu{!&~<YnJ5VtLaB22wx)cn`s6o+qZ!ma&YoTfw#G#-8gw{JbNnStIk_RPmFv(*j;9
zR+8j<hx|6Lj|UMEbzy&G9ID?KEsiKgoN4j|jeVzL8x}1O-}z3(g1Np$n_v7&JaC|{
zj9%cC9XM-Qj-BK_uWFN_g=TsI3y|X3!*EzW8JfD;ctP0}D&dTudYWI^qJb8|SE(Ar
zsHfNFN)_kV5=Y`QPt=5OlVc)AJs=$_;TTV%GJjhO9;E)v#TH{9sDX&F;6|p>R7IR>
zcHlK7^F#Ix|5-{l7RM65!+GoeJE~pP>ZXybP=v2lqgN`FrepWe(53Vlk3d&C#ePQ{
zt#}GM@xJ$Fw*yNTL<`q9F>&{S;}r$lmHIJGzk(L0AAY+f5VLINntM&l$Gh3T?D>7X
z`2B?NFP5wBROfn(xw7ncT0A9_m#W!w_ib@~xG)j?vKBk7#woZc$@z@bq@!k7v)4UX
ziqFaLrTy}12x-ksUH!$8_trmYcakbHsO|lZcoXC!r60-p!`$ZDrI)m6#v#6{J+1A^
zCVh8cgc-HyS!QbP%AZ3NJnGE0L6<n?8q25nUoaKZ`*fCO0Zd85A-5bMJm0`$@#Xt!
zQv#=3^h|{_O^(rb2AfIKSS06T=tY{LAK?L;2a+0}hP|#z4|&pW#+Yvwf|PiN6dGXG
zvBqo1pGmu9rI3|7nc9dEXJG^US$riHT7*3s5{gSlCs1`Hr?<*tVKWXimF|~#&Q$!Z
z9_cr+*01?gn?r))*?3UF2>~8t)8(R0h^^bz*2y*6&R14^9>ZfE_Cq)@?dz4nY6q?k
zO(#*9c%BPbB#jQf(vyF~oYJGtO%Uhp8t~2Z>4Da?hf-~Sqhyvtu|F6r>Bt82h3ni>
z0rAT|0!D!g^zDa>=m&c+u8a@jjlsD(#3?lmKLZaXrZZLHfu38@`gs$DUXdTuwb^Az
z03J7kBln(Wg&c!Uk}22R$SvwGjy*;Mx+WJD*pYYpy3HcPzt45J-j4BlrNz#*nKmu-
zbl$q9AiM{TT0#srN^i#w*U^4ocS3tRzM#2PDI88-zz%5$E;Z*p0a`9COt&X4E#T2d
zA5}EUMc*m{+@%rq3F-_Ag%hnXu|;aLjNe}YG3!|JT?U5tf@MtL1O#b=jdSAnJ|)oi
zVbr{Qh$}jFzku+wG~Dch=<ZK%0v?FW`enoX`Z=PdiHJvFRup=cs9K|lu-!g$F(Loa
zfHOmkoK>0e!))%b`?StQ+1{M<C0>==|2WjWH<z&)nYU;-`V~lcn_Q)J+pn^2c1i~h
zlkw1fUNIrGfR{%Q-j#&gZn3wCtF+y`3`gf%$EG#;1x}$lH8l|YH2mseRl!cz!G=#d
zC#UKK;8!>FWB&QD-HZTD)Am}fA*GLA+3V=?X8z)yfSUC`I{erV=Fu;e>lG7e%?$Uj
z=RHB6pK^biKrFktwrrhFk6BXHC)G-#*$|5RJDsz1+wqw%B^@l}8@k8nU$Z2*@F~5A
z&5{|l=;=midm?b<H0rKN$~y^8o)|0rao#A74P<wZ@uUr{YUEq!B=^}{LGu|drIXoE
z&$WbxF5=ExN&)>JWmX@(9XAC@W*gN8{c|b~a4M-4uS1_u!y7Kz$|v08^2ivh$SrDz
z?t*i0NUjGF?1L2!;&WVS|2Q4I{dE~lghKN9BRph3X_;qHymzRbpLND^{hcCbv^WP`
zM4hsMg^*>z46{Y1$a+_Y-p^BHCZB@ELrb2YgV_aP;Bcr8%2xiSb`k}uqfhKE;=z-&
z_}jl(spPEmpm*l@LoaIWR=iO9mgETOipiS7bNq;06dM;b3J>QBBM(kNZHp;T^4-DQ
z-eV8b#23H<6iB8vO3L;S0=T}OnyTo!ccwb@JST#wu*@XbCgmL}1Xk`ELh;VYY}9lu
z>~0(*i_#TXelVs9;C16Zx>PT2c~TRXv#(v#G<91v&2S`H+TEy%`3*K{WUbajxG*1E
zRZ?lWBI%F8N7YuTFLZ1QV2Azb*l%)_a%D5bI%!d#>m~{!@dK4e>;jT*%DZ&YXZ93y
zJjVuU!4nSvd<saoNVL1X<!{=NhSB%H06ZH1aE$MzevL<&Y1~w1aHIxiszNIX>!*t(
z?eDLaOp>!wU_jTi*<f;m{TDoJ(6iFC7;LVbI&42)$OhAXJnQwx=I1<`%biur$<eQY
zqGk5j>=fb=qK>H-JC^~|+ydlM+bPEsT9iIo8`bLINQriX%xsgRcfcAh<14piKYCxT
zx&yr?9r1Pazx-NJ@AhzzQ82Pv4258VIXU0~{N8~n*Jyf=aRZh@c-?Yc7yW<_9&j<Y
z{f`Oi&?md3^S#(e@!8nst5`GsFsnteQKz^UV7BGu)@kD?;`Ylb9%$f&J6dULGECI;
z+p4T<;=1_#au+RjA01pvvH~gbQNTY=K&YB`=ZQCj{^OWd?!oa?n?dnRM_X^@k6jXl
zc*oog%wQp0mT!`Mg^KRK?a@|UMkJU53G1EnmnF%*ZVkV6S-yB4ANr&Y1KgFHiLgso
z1G~v?bg%(<t2c_7C}e!%p_7~0eBRlKBun*%Yv^IkqoXXesV!Dy&C)vfEyqWH33|3*
zg*8NfI;*{qmg%o6F}hr!T}qz725Ot#E_~C21f)4NbxNJFlZ_o-(0?KtI%wg8ZJ$cU
zu=%>tf)!@qxtcac`P>7>LclF!_4Bm`Jxse#@?x+j-3SV;Y76m;VvS*QubAMyi`edJ
zBM3nL1+evea|E^sfoDxVfSY$diVbb7Wdl;inS{!yFCu<ssxH@_Nv`gkw44Zmdq3Py
zsETrBV|EF2&51g-0|lg+>(GH{cs%w>H|+T&8qni{<|!!rDOYtYywNBz%x>{|&>GK8
z9yXwWH<HrAIl}Tf)yX=xGK3Mp)jQt9W_2_(mS@&P0lx7xv7ndENV%8T)n2Oy^^_lg
zF(5y?o8bjw8kKZBeXys^t<Z&Pw{Ec&*U8+{=A|Ywz$iVN|F>_e<%DZTl2jYt=Gl{`
zH_y8FqCRej|Mg>F4VZLFTF^5E>g}6n<G9dIIz9}?!M=Z*XLI^;Z$93pp5bY>jd?0E
zkd|o6`M3XkW2mv&oXaIhbi(e!wsp;JWpvz{Y$Ar#$~7E@%+!SFYE2`J<xjiity^eY
zmA)ekbLqY?Z|g3m8|;DYkC1^2c6C9=?_LL|R-tT>xZ~9NCPOV%vh9Psjb>lCLaWL>
zmIxNCtZd9=Da#uufvY3t3$$*;v||ce4)@yXfobRHiv4F6O&|XH#lb=l1W@fS7P_;#
zXLa#<Zg{-?0rDFm005xSuEL+oz<<rS-A_ecQ0avBw7tvvit;V+tKR#5?7k>tbQ3!1
z>cSYQH`Sly=PskhL=~~$PD!D>Frh0A<g6~l#r}LLl<xN6km7OJFye6Nb-**?Fnm9_
zt8%#np8ia9OFtbPG~3`?=6RFzr5;z@TWCHvCXoR-0zih37Oq0IIri$X&pJpya@4y%
zNey3Lu*a92=Xw^va%ZwHKepK#XTNr2K<B1G=w@N)!?UxM?U-V-$rv~Kwo$qjmZld{
zzJ$?at)0Ada_)gzVx+e0P&KhVId>z_ayW1@T5qm+GdHjv#3E3-nn_8qZSUYUdp^Lq
zJJTtkDVcj~Q{&B)cpNFS&8b}^;M}HM>Jv`v^7Hs+LCEL+t+eTIG?zyhF8c_kqdDWW
zp!#AXW0RgK`b(xNJ$JDaVuujMfmoNtg6KPkX;Q-3`e-ZN7P;@ZOw=Q-95QmW>g~tQ
zBdSGjefF2=FI}wOaPiyM84=fEyb3QCx>sauIDTW!T4||W=hkxiT!Z}_t|7ya$L%Cm
zP(zo^<BW*aa>bRTd{$Z@XYgEY>k#qbT20rxGQZ?uP5QoR&`Iq>)u_PFvH50;RQGAk
zEhC-HkZ7e5LCU?mFVm()X17_m+_@QN*Pa$L4Q{K;ahT$AQS(Ux4ATP2I$;8g<$`TC
zhv}8J{T{5Azv1tj{;Y&4{+Y7uEUSSMXtt$#La*%$9hXZ_Y)4ctl){$L(~V9^XwVUB
z;)9E|b65A5D#iSTrW9(+q2PxUw)oBYOd{4)>{3L7S?h}1$|)7=4%rNs=<;2;IdgZJ
zuVrHM?f2)`JFMS2Rm)QkMHLE&O9e?Y#^mQ?jP^`1X;H$FTD|RA6HxUpag`wJ^x@-t
zcyn)Ahi%Jq-~4xMFZbXkwYvfC78-e7J{&<7R~C_%-8jI~L+aP50ONCdHIFE9G>G1@
z+X$safh-)&BqCyy!(jK4uQ*YzJiA<=20s2Gyzl8>h+m%QZYlS|DO9Y>dq_(J)bn|<
z{K&k-dpdoNQGl-0`!U?&PXA?R7{C`I_jkn7yNixWJWQ3O+ps7*TCMh>UQ$#dY-{ye
zo{=_H$W_Ay@7x#nF2JMh%GGQ!NZkJ0Ft;oUF0V%IU4;k+2{)}<_y~<WJ&e;Y1gflq
zsoOLfV`#e1z;JrqjzjPAG0n#G^1M7*=6mId_jH00f#7UnJx-6zy`@SW7rD_)!R_jw
zyb>H~*0k$3WqS!t`$XQ6-bH>5(D1gi>#7Ge#o)|P`c@^9K_4eG9JH`uhiYt*AxE>t
z*D)mk_tgl@wn}#4!NRN3?9;W#ajBHT==X>VA(LsI?grbu9?itbOk%_dw{t=`FMt3H
z(x{Gb{<lPN<&%9YI$z0`PCBW1Jhvj&gtp(kMM6zineZmMZ;`IKxfyT9K|o1;eSMNn
zE<xwD=p^SK_`R972^QEcy|vFpv-zzX;R3&g{&%fTwoS8V&&Po`>ITyg@o(=M{-H<5
ztNg8T%yU;o#^xPGoXJ!9@?CEfV1$gt<^$|&Vb~wYu8T-r!;&Xv_St|iV)16GtYc?*
z<gm3v&wyk;UX%)ITmgn1eh(q=e9PHVtXN64A@1dmy)+iXAO}-+0sCjb=EKZ&lVfvh
zSejG&p@c_W>~{|9pK2t`nvI`!ZZ%`{b0!^+Y_Y3g)RfC#xd^H-<)%VI_<|2+<W#dO
z226eO_<n7;jz*cGc8x;c=$T@C^Qvvyfa#DCG3J&!zql!$jrP1|Gi7--l7Dh3J=EC$
zJ-jW@@|nK<W@hC_7H5?68v#nSg+!LyKaLM(9G2{@=cuqO`mpdLrQt#uW&+|I`$g|3
z;B>TlYcLlKvU<yxOVvoOO>gSP(E#z*Q5feW7-2JK-<K_E&>hFwDzErh>edV~@6aTX
zpL-B29Rn;YA#rwhxhc<!*kSw=eEFH%dF#Y6lY)|zw%2<)47!au6Ygb+DQcM^Myzl9
z7YtffFOUe_$SO~CZYg$~uz>052I>4z9Mol3J?i4y9yU0AY)u|0Y9fz*Gv3e!!P5Id
z)rHaKFm1(i3x}B-;_=D=hV-i5o6pd(yY)?@*4qnLYp=`3gM7etSCUod(Y`)>flP-A
zV`~haM5R0jG)&*%OF(8}d6S@f^X0PQ%>BFXue3sKp|$40cej_`0yq!_WycJH2TDLG
z|Lrd4j)Ko%+)O71$DDS>2MV>Xm@*wla>RF+kThiG?1_uprsMnaWoX1Qr%j~TPOB@~
ztAV{=0#(TyWqNf3hgvSQ(S@ryGuLLj8YO#qi8#|9I+seoT9?v>fFuO|1Nk$Uaw1=X
z+1@Dw6o}QI)oxI~%s;g6Hm1Xpw=>P=$oWx~D|V?Z>8K1hTzvuJCsMNvcrWh%;7p3P
z?z?A_?g=kq2zj+=a(X5k1emu?2lb0jTMOz7^&ED*dU7fPC;(Rht1LFTFFDT9qg0TN
z=8g9_4tJ00mEEevA<{c?qs$2N8z&x)f1Tx@A1u73oOJPTenr;G&9j47(P?#g5m>Ow
z|C}FImkZYVc@ALf_h$Sx3^o!g%%fF6fHqkXT!0`}4?1LFa(J0+-{O*(qRwq}-bxhC
zLIV#pe`y!<MOM$Lv+KSTh0za$hmP-xx1(p_0@fc*lEB@HP_H;G%ELTgsiO;o1<IO2
z=2@;#{Cj{pQ8^??ZzaL>H(Vgn_{;75<Ikf0FFy-{CHKIL2mg2H>SPb!Yz-eZV6vg6
z02mJkm_3O&3Nq~^gdn_zbM(h%Pp)3-`rz9xBLm)B-P#yjHB{Br7eKP=19`IAeFmrN
zY`$|J13gv0PJv6%APT~rx)1c4b>9!aY?ZOOv3Yfag_`6l++U0BO3}ZVu|7=;I`s1T
z{1)Bbe)N^%-@mB2fpd`8tnGVdt98rUogOZGQFi@YY_NBB_yUJk_ax9%#XV&@&yDRV
zn_K+)*-qGb9SgApA(w;5Rn(1X>qW-;lc)s7&Q}P4r%~s-&7N{qI-$kuHLLDxhIJbP
zqS|-Q>Zj*VI>lQQ1{>lZnEGpU{pTx}Cr)`K4~+@jTCP(E(SU_tzVyx%-<a&L_=QTl
z2<=<lYkPWTW^pb%pK-dz%g`3y2^!TYU&#lwnbbFTvh`SfouwP=S~uH40=-%e<0drs
z9n29Ksy8JJs57oeu8->4b*gM{N9-6PAuuZIWZ{E3K*h;@eUaVmuC=Eg3>k&-Db(P)
zeh7wI4M!qzIy5x&!P!f%&Ka1``B>eWSA{4NRkjCI3+NlA224z?nv}PVd8_;A@$MSA
zttYlBN;x$agXzqi;=Nm{kVhAv)}#tiDgP&-*B-U|F=1R2$b1`Tt3^qZYmxYK?^1sr
zq#t5<sT|r9x?x|JPF^E7RL^ll=65jyqzBhAo;NoPRDd^1aDWaR5#}Lr4IDM6L5C>=
z(tW&K@Q&+<^8Db4!u(LW+n<IpgN@+F6~2B0;I&<6%6L7zpEQS-Q`(GmT8CX$TQaPy
z;WjR2aq=ssdg7$%9VIP^ILHj7X?{pp%};g6b4(PNx?3~*{um2#xNmuR{->QC9nXKo
zy2ZB;%#?6v>DPMLusq9qEyQrG%2}>!7CR^j=uh{J4mAyYcT3+@*s)ik-+nvBRg7iL
zQ2G87b481`^X<7R?*^)P-R;t{qRWH)N6P-0&zM>zVjuQqu8Q%Vf1Xr5=^hs9(asVI
z1(+!}9+MK}Z}g67@gq8)_B{jTnp>h9!NiFbd}CM@lE}Q%NHbnIO>{}N<~99_Wtvo{
z<2PMpkuw6=RcL1MbX55TBj^KdID2OVy!P+HBkDXX6ZG+AJStED;vR;chIQ#(1)n31
zD(vSlAVXcNS9F?y1FL^u$-%nPG39RQF!)iQGJeRy-z+-mPsf%(ZJEvejWAE$!h3`3
zZp=XR!7xugn;&WDMf02HZrMFZZ@*Tw+#=K9{f6ymRWee3#UJtc(ZS3PSBSyl0vI1%
z1K~eF;8`i2d_zQEXYYPKb@e`_UMruTi@sLKe(s>l!+ep)2E>llY*1MTo(AS<sUqm)
zAyBe$b;<q?)nMK;*>be-kk=#SfO}p8&GNTyt5&m&oWLfUo4qL6XLr9-W-`M2cP<)D
zTb3;j2Jx7k5<#JWuy)8RA&$bfgMz=82XFauY)Wa)WyDQM295Dx$ap-}A0CYK64cuT
zDAfI@ECb5ztCIHXo1DDv7w^pQftji1zTPrT#}$6wai+Q*y5(V=l5Ce0jyHpMbor(L
z`QF1^3r#;C_F#$o;-d0qP1Ky+!ynSpGnag*`@*O?b0^#nh`nm5Px->RvOvda^4S78
zU8ziF+DENCwtBI4(Eo)hegx`8V$GvKG?Mt_)gbcA)Q*E`Z+z-FEgNo@v~q?|<=@Ay
z%}Uj%nAw(dBLfK&?bRC_;x%groajHx1RPf~9LqG-E*@&NNQuT2`j_oXok$P3(u+?5
zA3`yXskn4aQakY-i9J+pUTp<l+&D(1sW4-O0PP>#!^8DzzWVXl4C)*;_1vlDe*AIt
zdD@Pj5C*bq`0b#MD{6n&XNq$_ylsYqAxQ_AI9Sjw^C)nixh@s59y93{NnoG+H2<47
zT?w|mAJQ4`i&;0iOI|^pxgR5f!NQ6G-be3+b(7pnjGYX5hBQtsRLH<#y~qw`?{hgq
zwIir9_E96&2a_dPO<ceqUg*iQ3&QG7--D%ffp~3v)KpbRU96<yQFVrop=dwwV2oH8
zhbb~B12GJY32v2{t~dPTYXk)MPGs8~VG}S6-g&N65{3nT3#|`a;2%H&LKrZ$?pr0t
zZoWcqGtb$0<4S4Jk&;K5BNZut3UUIpFxpeHz_4`(yh4P*{tz?IdctrW;9Xqz8losN
zsyqFHdnr5d!YJ`D3l2Fw92AOx>I$7XZ+h7$R5C>@6Wcv57{fOCa~*?CTwJ`yn+OI_
zDosPBMwyZ%7hdp!cGtPXTq3*}&^W%Ps(FlIipjSj6STlWMn5~a(=Rel89{r#%{UjC
zH?@(LNrD0Y<-w;#x?lz=2AGPQaV8VvRXJYw>tuTRaLyhH-8V$7I>iDPF)f){>?D&z
zyf({)7ey~8OI9zKjtJnI?wevHL%k^MRAJi%v%-eic_$+&z!xwZ2<EQ6`O!-X>o;=q
zbrN~WCBPQ6UCBbAL-Vuf<i<ISY9$X39B^9lh>ghFv<qd@A1Y6bTf69iinOfbis21b
zZ})dks%oh4f2wNUBfP7-4Vh{PXRl#-|D6gDOuj1v0qw%OfioOZ56zaj>F;n%f?K^k
zIMxbjU-kK{MO9$|%54G#35X<)O!6x+R#;%kdBq9wPuEWx=|XzOxGxsUM3%&HL%Ib+
z-=kgtoiCju4)MIQ_n;%OVO0;O8_`OzfG&jE>1D~lcZ50h4DF+8(I~aI*m^bkh0e1t
z%YpjRSuw?y^=#g^51S-lHU`5bB@VOlOeIee_Vsh)e;Ov}mE7vNX_R>6DBi<w4^u>^
z2uj_08-?(0i;eds|3MeX1t8JTPDk3RnE%|1&+9D_!Kp_6=umUSUG?$9^;%tiYAgUD
zyAiys`7K$SG$ZdaK{*b7&!=5!1QIa!2`547a`8<~?0~#08LNTeEzC*xS%YLU(1AM6
z)Svu2w;xU^JCm^L%?Ca(qmprSU!E6j0^WP?TzHAQ+{tR>UwUZ>zT|pqjYEzChz-xn
zKW#$d|FVfGy3;TaL6tfz8^6Qf|ISuP`vV?W2oUsI_j1o<<PB0=axz-WW$~Hkt6kg(
zn@%?G4k5OCMGphvKvegD{Pq}Kqm}36PAdQsmgJq>d*qaeG)dmVwYt;yFVOM90?y|n
zx;k3Pv+tba_EVbY5P@3~eOWlsk!zf6JPjq_6MMsh(Bs-5|2H?rV7Tw<Me+#ewDX!P
z*#nRIPj+2)YqVrt&+JZMC(bu+EH+n4=Sr#Y>J59QUB7t29Pbb9_LA)AHFqEZj=x6f
zhSgTSR!Hk95+xac$9PnOPvu~ALsu4&V7zzAo12Q1*mq~<y?*sJAKRG2fP0CdRm-4E
zc8r=#_e@+z6K;7J>3=a<tiJts;G`kgSSFbCMtv;}`2t~R2B+h|g^ce!%(Wz2xXYvl
zYo<hG)#O6j=2LvJZC{c`MKZ5bDw~{~z->syR(b6%leDR47a@fCO~YZpbSQ)f{+}V^
zP(gT|M<+0qb<|}JJ{^C^is$<UwLa!L3AWIq6w73jbJ4EDBMfhoB**)gTdT|eMC|FQ
z+kUIpQYU<EIg-7|n)JSECI=<?V`f#z%4<N-$FjD0ily_#+T`T|!=;l|5!aEZXGo<{
zI^<P%D*ce;2>R5L@$)*>d{f1Si~TsRjPzFB((+QpI)CTVAZ}^`7|3$BbbQP&i;bW4
zH+NS39w*|LSnft;T1gYCmWVsa16hJi%NGf{8E9G$;+?#b57DUonSKGr<KGX@AK)rZ
zxnR6+B<i-xe3nkFjMm2=NbgCkm6w&1gS~EU_-?p32X`Ju;UA;pC9LCJ(Dk_>P)`Hf
z%#3a`qu^|_n8~-fpC68ttWz<ftX!z2bf9wS-#Z5)N*SwVy~AvXV<k3lp@C(^`7GHR
zzW!79aI3RJ9$A|vmETVF<mUb^Qw7pXJ_?_!zs6n|Cfi`A_QDTdgjEDud(}=l#_-&<
zN7T0?O)mOm2#K^XXs-GzaJswxC7xWP10<{R+q^Qrb>I63mlcxFga`P-M^XJtH}<0Y
z5vC0sXdJKf$NYx1N29vek2*Tt>r@3A%Xqy()Obh>M0l|^h8)@sX{?nP7EyYAT-hZ{
z{l(uyoXyugBQj0OybCai9&!3bTwblhXOrDt++{9W{v{CCZXE7YcMV=fa)ybpd$)Gx
zFfztBlqD7nNp+uZW$DGnrB1s)T}8GT&}w1B6E@<^{7snON5<zfU3*B7+w-Xf`8F0m
z1^>Xj%tiVai<Pc(5jH1LtU4rsIFQ(J3hTE*8%P>~IAaz8!V`Gavo54tdta|H(wbpl
zo~>Q@YWV{2_BymQ-|zK2+5ScarK3SMHGTd))u&c7qKQWu2Edv=k9!uk`N}H8c_rPa
zI>NQuMXOMKm3p@}VShV&3b4|sQy}xJf1|oX+=9wH=G!}4il<3w!+iVYI$l6uiJhT~
zlb;y}F4FgCjoyph%)`^$B>;R25Te|SC*UX(10e#lMy_I$`^p+s)e3(!(mh&$;q_hA
z>whZ$noQZc0WG!VN2$9x**>*=e!7}=GW^g>ev#xl?jeAjk@w3XrjM+k+vhmvdGoTK
z0{dB_vX~~#Nt~Q@TykFKFAwLp>03j+GQ>!ju7?fh$B5a3pF;r1jO&N}cYLMOw2OV5
z(`g|0z(3a{8?*hL#|El>G$(VdzqUKYfH-Vmd<!Ct>Xjco-37M;Z#{Z{#R_c^K1&(M
z=RJ-+Xp{ZCG%38kN~Ll2%ftS@)8(5^@I9=iT0tRC4Z-CXnSCC`@Vn${dEU~Bh551{
zk171d#~B*D^)R(24x`8l<gKr2YD%LB@&fPuIkqBlfp(|o?nV;~&tz(4+SwXQx5ex&
z#}S7p?vb(lFAWys>CYHjxsiOG8?eP|csZjn^^zDIdYTKlu8O9aw1KmdCH3H_3V0|K
zk<;4mx1nwNlccG#{#(-Ite6RsJt4dZ!mkM*<YNBe;4FFC_+QZ7OCjT8Y`FYbZZ@mS
zMc2yUNTZOOgT)_O%!*z`FHOX01kxac{;A1zpJ04Pos7m-9ZPipeaed8krAxx*BaqV
z*+y2E1Z$2h<3DuTtfEtQmg&0XCZ~<2Ua7VRJ}@oc=iuY}MK8je9){_-rwW6VT34es
zS&s~yxW{wM6lS;x4PnB_9v!V(X$7i0@|RKQ&smuT-q=cs&}Esbr(D=Dn~OEyHE+<Y
zcNapEOz222_*sY`PDuK{B5ognYP|>qwjI4lhD`y43qjaMcPX#`*>1Ev-_QXG9WYjM
z#lVQ%^&ZSc$jfA?2tbxVL|-r*BU%dWb!>A<2x$%kzZz$fTrzRD?D?Jq?6~M&{1?K%
z_LF5nJhreS7M3c_Bt+h$yN%5H_6oTt?(J0c@3g#UXhE_?>Or+?+>Ca-=d?e5$|Jq2
z>e<S2+vUKHeA}3xhPs%uLN1COX2fegIEaU#v&(Ly?3`omkw$(oOmkIrw13I7xJtv0
z(}#cST^YITif_mRj@FHg@&rICgmOjv_%U20d5rJbE_PZxbBkZNHrDB|KBm?m7fJAw
z`<NG@&gTyp{5~lC<6b-4)k(wKnVsYuxv1qlgK7EWqfV~Y{*T+vuz4SSqk(T028@H6
zvh6pys4?STO{#<j9I=?28)Owt`6HY1@cE#`=DbD8x_l;MjyE#mfWFyH5Xm$D#EPWq
z9~%d%H9;F3HP+)8f`fM6&p_>ZWKoWm3$>L*Q6So<v<Z>gU|tHcGPsYd%utK@OBe`o
znva`?01mT6DF+G(PZpa__1diBL7r6Q))1y+{^hW$Qw=jS)m+`B{C+#I5w%sXr@j8B
z&qGwOBQKV@;cEjme=9|3um1A=U5Fypnolg#xM^;&kcNfRj*%P8joVPTdv>oM^sbmh
zP1{w)ji<zvVkPY5ZXr4B$}d98Uxwe_AprdA=$`eVav0QAn<})Wvb+?YQ$AJV$*YDN
z*#iMgW)>TS^g{dxdjAr;2QM|xc!w$vmc6rpGO18$&SepOe@SxGM=+{kOj9_H3rHHA
z?!O0RgLu;OOekN6^wJl<T1DuS1fqP)lpddX!h%}?L&{C7eY<Tje+jWj0>n#Uis4SP
zJko;WM|t}^A?)CHUGEb&4UofJL_TD;RG!iNK|<sci*Lah4OR+bS*P_VLIa#?wJF79
z&`OteT?$g+0p6<?3-)M%dFMCz_%Cw*tYAzHZ`&U7Z~dut=Y*AeyTcES>9d$H%Yp@l
zU$3~rZG5y9K<ApGZWlOLI~<-SW^Cr|T(9Ek8Cm%+((i#U)>6t<IJ9zawpED<xAwMt
zgoDsn`~x}ua}ZY9X7~R0E0m=g#gcfetmuoqyZzKQ8%pdk1wtEz9>3y^S_?DxuSxPk
z)La~cS7n$5(Awg2VCEJfQ?7jKi<hNL0@uT5w<dXi>8;_z@Sqb-MEum5{Y8-Lz$Dqr
zb=b3Ws~NE1Zm$UF9ycAx*?eY>lhG?+xOa^G0x0JnS8M9>$9Y!YpV!isYU=Lu#NmE8
zybR5y0ZSw%kEm#YRuBO~r0igmBnPXn-ihJ<r(6dZBxaV|2uUn*8RURn0V%Vhd#kjm
zZ@Zi>%}dEtl{o^%T%&oyTvJUFO}d<$dP>e`>1_fud3@`G%|N2g-ygOO=Q%LclDjz6
zJgL^6%FqMox|aL1PG0ldr~;sUxU1ZSG)t-7n|YG7n98JBx%xu;2h~uyx%fYhc1V{Q
zFCL_ET|hZw+4?>EujS2R0W;(4LupJqKhZUg!%Th}F}-!x(F3^#Uk#ao<CapTUxP@u
zQ|u(MKUSNZ6utUn05tD2!j@lvu1~*qGK-}_+eL3=r+W@*5BO2X8gw}d1sF)8Ed%c_
zw6Dk#n|+k{%FXW%jT}HK$KI;7c>`ByJn)}=zHHHEc!L}vd_eQ)OP*mp(o!2s>{XON
z-lYeK68L0Y?kIp1*eRIyWK2997k8%4HN@jN74MG^${tnPcAKHt4L>M`&8PuhwbJ6n
z=vTU`nX{iv=5n-?ctPueNW$&Dj2leIUkGQMv?LfjR+=Pb8v1osJtdN`dWR<XFN&|U
zDKStWM?k`#T(z(NqpNP<8rnkwn()&_222Z52TtF=m<XBbME$t>C$;pn<-odVF(9t>
z!ro2PDTz|d6jmR^vqh)(xhXn%F$<?w*}c-8-s!muiKOKhY_=$UC_pqG9<R|dylbCd
zuSk2@?u=I|Maw_7GH3d>dkmyV*6g1+>y^jU!{R`e4Fo93f#=2NIgHu?iA)wA54};&
zMJEOAY$N}0L^?8Ma)I0(jb0IY^{}!vTTfEwl$j~)N@_=WZ$LKnQ*B2#y~SE5dMkbp
z1-QMJNwXa^SEv$TY(otHFYmZS5eG6??LAN0D*PVIJSP{DO`YKZtG`-co^Xj)U)bTL
zc$$b7E|gb&@&2dnsYcnHj|a>kF}LnUh_{%;XD+X=xKKSwhr|c=R0kR+;TgaRgd)7U
z!FEhrBmtd_VW)_AXNCrc4eN|!9?y)OW;=W^dRcl%iZ3|>Lz<-ab&d?9LkPq6*63Ju
z9xf1{UI^gRj6X;PYDLnMwZqZ>x0=$%ZoeZ-KIAs%E@)XoLhJbxWL7^%segNuj?jcs
zm&)3EGHEmHQm87wA_PqdW{SWw@-SB-Nthgb$u6=H>VrB;ya*wkfrigT6qo9=F<YHY
z8XK$8IjvJQ8j<p1D+W7m4txn>%k9X(Fz~?Kg8vK!kwFuPGXC5&UPyf<Pe+>3%g9L5
z;pQC~RkaFCeFUe@40sJ-8rx!!#1q6ST2Z1`EA9dBZZK-xk;a3z4H99G#SyQiw{bC`
zq!Ir~`Z33Uk~Bg)h)*1QTsu!{8Y9$^f<r68BBvw<8E+0>*JG*sb}c-Ixuyr{+F0?|
zUjxUrA0CJ0LRQMCmQbv#i)9ddfF(U%51;wcp>Xfzn`F7#b~=Q|M7dA5MsG;y_F~sa
zZL)wm{LDJLjqkqB;Z$|oZF=>~;#aJSkA!?gMt1h&@r}~6yXDo4S~dDXA?CA6Hn#qL
zw5!{jX;d@Sdtjxg;xEt8S9O|estWk#vRgU;_V&#WeXtPX4Q2$O#Zvy4#fAc(SjysA
za|<)rEcQ;T<`COsP*RWSbo`5SeBukR>^a^&Kr>ypqZD@=Ozm~Qu|8&+e_Q#^RIXir
zIxcSaNDjW%R#~Mk)V_7DQ98n5K>+?*-`hpY>~ctfOemUAsV!Ku_!4$igQ+@VU&R<Q
z7ykLY8IOqCx4DNbpOe$_HCTRMBpyrW^I3Ysb`w613Jyfz6Ej}vQjj3Yoo0ooU2B!0
zH~9;V`TbLU_hvN)P{knf|7`x-qs?zeQj7oenQwlNaql4<e$`1S#}8vFJnucn04<7y
z{>>d3sT?Ow+(>V_=#}HZ_E8@a*T*HRf(d$-z;F*%<MeHOUs>Q$mSeyA0s9qo@axB^
zJDOD5{GxAziQVC`7~zN@ws_1*Y?nylzJg8|9FYXZ2;u*Oy%F(FR_6o@fy-ErP8#m{
zqxL6KgXR!{{8Zk6zy54E3)0P=5>M%j@c&U{TB{2(0qY@0|L7s$e==Lb5zEwZp`bFl
zJS_lt_dl7J|197?68Qfo0T)Pu0rrQe_D5qYNBbr!nb$t>ft(v;lD?cWXkde-IpBE)
zfYdH#jAc&+1E%BG>IsF!uY_SD#8%w29*f?#;9pkWee3x^ehdDpawLKdJ^)e-OU|93
z9z~9&Px(A2zpoN(Yl;x|gjc`p=?VHV<_91MyYrjHjfq1-A?VNAz!>@%GV~YIsWFqf
zLxRD}+6e@7AK+(@;J3{y89=A?`Z*cQ)cvuYVaXXe_;AvZ#N2A|=K6je<&Re*sKI;Z
z_xYK!tb)sVj!!L)f(^q7D*}EO2lcV1s!Iam<l~t_Krevw5y<IrFs1tp-h4*?i2m_v
z4l;Oe^OFpX!Fup=*EjHERqr67!VziI1EP<O7{^%pi)3Upzoalh5V(Duq9H{D-kg)*
zPU`WhxhQz=B>qo$Y>eP#I|IX<|7`m|Z~H$={{Lm$L623}|L4CiAjv{bFcO&yx)X9=
zZ8817joQ&KOu<kK#!8WuNW~@7XxMhGrDme1{>U`YKdg|kpn^?O5r(k7*nkzh;+-=y
z5BS^mCwK%$yX*bGH>0>XJ$p1~J(Z{6iT;$W$M`z6{U<vAM*>93eyJP2_dWxS>*)h!
zZ<E(gk`E(s<oM~P@m=rxnM`^%(sO3q@cVe6P3+T~@q~km|FUWO%iH#aZpoF>j?}f0
zWf!S$MJ|0*(~EeUDj^$cu*G;ou|vyVM6vg;%u*=TmYjVi?1V_3pga0wRtP@RUqw$c
z-lOdIw>>VG)D-XG>xS0$x8*!y7k9NPq!vQL)}_n5|NTHXCcyjF#IHo}hqCwbLou<S
z<A-6|YoXE!)ED$=P0R2bj1;f}WX-DHSMHh0UiFC+`>d}rEpdYGR!^9W2f{7a!WY{l
zTkjCoN)%1jQ^wZUuV4{yB>nBGB=tq(^G=S}6ch?$TJn<k*}Ucc=TfN=ge%gSq~ke2
zjQ7xv0!eMNj+u5Bq#M|1qtB$2_}NS5G@)QG8!EB>z}d1EGd_&O7r#h0&si3&H(eUl
zd@UCdT;4*})OUQtja>g0h%3JQv;(;~8Ne*gvj1AR^Kyt!x<*3tLz+=oC~rCr;vIKt
z8dAQ~GD8&jr<U>B>V|&j(7NU$p#H)B3>85*?C<5!5OlFkp0n&heA8vu|IkFcNBqX4
zN+fzcW_+Z&-)=`^Qz|pdOrtHewVl|kv%ef{Gxb>iGC(igXaB-zs;7RMtjT%Hc=)YH
zgGG5bru0d=mrs9Q&A>Se+RHRoDXd-o0|#)R*V!S(kVU_qhjX8-Q(DCLy6M`#8M3tA
zf2gKxyeYtzB`{SGqYLIHI3}A}liS7QKRl-CMR-xBKv_%E&s5~l`neO6_nWj$y_jaW
zH&^fkHlM&<pJwHmAJ=-N1;=_HmJN+FixG`%yDZGnpDSP1hLQ2O2>$JfEc*_VSSPV6
z^C8yH?q5T=JReLN9v=K9<0)eCZ|vCL@~!(ch>2Z0&T*qa_Jn2r)ZhYiUkO)oydS*3
zIW|{m$5AzxiV(R~==k)wLhtV5(X8F;O0Mr#D^5Qh9(p#nEX?M3&Dh;t*<Wa}mUMgF
zT<xRJ_rr}`pavW5;Gk_erq;e2IY~k)Eslu*-!!WbiBNDHEtS`%@fjO2<q<hDH<%FG
zLZW&z?|tqm_77?4o9Z4IXmI&8UY?UOt(Udwgz<2e#}0Cxn!Y*pjn9#6Q|pXrc|k#>
zXP*qT)Asz;DDQi(wnR?VuiPu%XMdS%jSQ*L8;oCaDwd+fjXGs)P$b>uP#tc}C_z|l
z7rI^d`*Ck_Shj!iY4gw2ams@zOvJc6`Sq>GBGT*h@fkHmpv$zA4Mf#SiHfaDO`&Ca
zKFb%uB|4SD=_R(~i|!z`R#IZR$I^sP_>l?IEi2FSbX10iSDv(CFYl#&Wzu9%39KnH
z7IU)gBnjp_-8b2DQwpC~Uvo&?hB&4_)NmMv^EV%TctHYw(RYL%Ym{PdPO>Gf@|hRj
zh^K72*u49%>)cXTX7tjfD~(;Qwg_SVNB;n42K!ZLcoydw>Y!&@@?Bz#A|*5k9x3p!
z?M4O_$uaFTUHgNa(5)w(m&yws>O;FQ$=Q6IWzxC#zs;MEzVEKJ=DTYigWurz#M9<U
zmCCfM--RDyTynJrx37|+|7YCs8~%vqH*hqV`FPCvGGpX^vUkkA#SQj&h7hzuvG%T7
zHC$WGtDtaVpO^J-SRZtu$)V}4-s;DmswMr!{%q-+MMH~GRXQs6vT%)GULHQvY|N(a
zxUpQO##3G`ADGus<Wb_aeUjJu9ApMVGE4N&iYq$<&>wi9^yT^P9+M3e>z05%@5R{=
z{??{1-InfQY0PTa2pOBr4>|qsuIoQ}+(}TD{KhA49fMdggU<!;TaAKSHznVHM-h+u
zxFOppGQjLc=RHcb<JHWv<6M{`7cRx`3#X17(B_+}#o1@qkBfm$`LY+527j3Sg)3CO
z$+%ThxlUHTFTJo#j3-=iuoA_zi%|iG{BG#}ZV}~NuoSO(+;#Z<NQ>}rYk~GmhKTB=
z{JI<1@v6MBnDZV+|HBcRE^Hr~W=2aYOLG&khqj>kWxs^M{{6(m{@UWG*BKAPn=p2Y
zo7-@KvCNz9N-gB@Q>GRvURjNtDP+<KHtbL1(kcb7Noc%^zP&GDwZ*b!VC7Fkdvu7x
z$be&vrs|u#$?M5UvY0$LJ^3xd_FqKV4eAWUgx^!#!q~TXe%ayv40-)d7w6hz=I>{F
z#mI3=%-v<ZAvdjG&Ym)0t408ubZq1lQ%g1<^~LH2-(ChHS1|>Fq#eqZ)uvEkgl_o9
z^7&vn1#Ch8LZvl)lB-?_!jI6~)>qw3_1q^hC0A!++t8WM7CGyNZiYuhtku^oni(^c
zakL(ZjtSgCR%bma5{OvY$Ip2#j)bU&nccsh;}>ef-z|>G!~cZ+6GUP2p?lt9`!MK^
z%cozOb^F^CY_g|EyT{R5;I<U5?q{KrHnDcm-k-^`lL1M6z4@fSQ<Xg4g+%;uN7BJj
zWJ$File)mkb|aG|W(1q(k)&aNZik7FAmz;)MBx)IZC#eMPrNy?!GzsPM0P+}@?u5p
zLjp>63C3G}SC=t(Q3<Y_GM%$!5X?>8q%KmxYI^1M;2tjuWN_;!1;^JIUbuMo`z{8*
z_`GG!K1tkOCLC01_symSlfYa5RywtXB!d&9@M<aa!81$OV(S5B;aZnX&bPEDsUL;i
zhc(NdSxSK;=e_cQK+&IYv)~zsMKW5LijCm#+{<hg4)2-eP7I+)fXLAC_4gDF(P0%5
zg^-_}RcLtLSyiafS-OX1AulBpE^mS><wE@_OO#W!>$AIh1;jgLA;{Q+5-G5G^d5rh
z8v1fyQ1S;SRn7~T@;F%B(Xf7->})Ee{*6r^no9Ly_#i5A(&({k33I$2Fb<Czi;`|v
zTm6!Ua?pEp`yq%ldF+z#T7LoH;Pvt)o$ZEwBa{4-j1^Y=qsOp1-sC77*qqczKqz>0
z=sko~#V%B!fzNiYRrOx2a5hU?y*7+kkYPl~vn6fneLnsOUTSChL`gsc27ZnrAhS2B
zxFJWJClR5m-AG@=Z$RjH_2cH$ZLGZG?U!aF3c@8scB$pT<lJ=nDQx{69(#&LEMSYB
zjQ=QHx7QzGJwf%sxx-`XZ#B&`d1NiuxW#S9fLE^_KiyPVvwEqkd7(czA8G#~*_K$J
zC)%}Uo?LR88!NcPdS&jp*>mosUXQ3mAA^wRew7UI*_X3n2-7Su8_@>KNj}reL}KmB
z$WusvY%r_v5+di0Pu=geZw9Y*e-j?DK9*h;I3Fp>XM96ovCVWVSud3(V@hKZQ_hc$
zQgbqn>QFwmDajcvIkGO=Gj|?`)n@F2&Ip-;Um(HPPrn_i7e3M(iBt+$ueqQ=KD@e1
zC3!5wfnZ@Pux+d+aYN{DFp0fkrV@@j9zo6B<)N2F*Ii7-Zk#8xia5W97Fnn0YlaWs
zUtsINK`<U1Ty%;gSQtwcb&`rB?bt-WV;ydiwaPJZHy*S;Q-Fod-riC+ayD#lJ~zn2
z2Ky&OX<QWvPG*^YR3=Z@CCHM$a}HsC5lp86&rtU3m8AzWEYQPDe_Zl%mSY#2`F!r6
zcFf(eO(r+_jOVuIVwlR*nYq*G=a)hwM?`FDzCv7JtGUw=QK=(=RPJ?qNWRgzw4($W
zu_jt_9uYwfOeQ=F+;sKUYnP--W(IRg!1j4-#e6X#7MoY7c=^8Ure$=Yg;UZ1Wop1T
z9ZT=ma|ubiQTKOusnhG-oS3@<1~$QF#-=n!+O4iobe;m<Q`hi)nCm@-+m`}HkdujR
z_K8M}DlM-awecz%b?wmiQ$IwrVD$|z@mMcw{%B^PUAcNCxM9Q3ZyvUmeGa1d)q?TZ
zz#kL-$jf%fv5oRtcIT575U$U2#L!zesD(K^-w~p#8)DuRng5>Xm=Bqq#c<IzMAz4`
zUo0KFSjSAs5+}+2f0?TO9t_J*euJTo3Jg2Wv>?*Z9_3YCaBhXdMUO54K3l^(g!aut
zxj1|+p&Y(|d!ZYa!T;6XSw>aWb$$OFBt#k{rMpx@KpK%!5kwFHi9;Paq`SKj5ROU-
z5)uN3&O?W!ga@U&K~O;QS#Vv?b$j2>_xH<tkFf`XF<`H~_S~!If6m`8YTMoJx5gC~
z#MTKzYWz@^tiEF2LPezl5etrrZ}l*h)q&enZ{I5`3W#;5^aMRdeuoa{&IFMpep5Xd
zVrSU*{DviCCX@xOd*sQNdKc}}v~y;8%ypBnBn>*{T2jd(@Y5}$ER9a-1H{kuTFNNQ
za6Ew}S8+;UcsdfUl9-(4qtUs#H>=ao*zD7M+%sYwvCA98=djveLYPMaNrA2uC}eDI
z?GQJ0UioTHnS#EFT->R$=i~<8Mu9DE2HMK7ez#WjoO`Q+bQs$uQ9H|Hb-XAuvQjSm
zVM_a~9mMuff%m1b<9*G(I<jpVKQ&cpW-U}GLhNL6JQb5w>t~|k0g$k|Lp1^}PM8T?
z-h_{7??amkyG}mM5qL1RbJ2hMvb@Ee`&j8EJu;4Jr}_O{AqJL=W{{f{$=|&sA<d2r
zX+Zs6oFpsx9C6jPvsx=9f*_qQK(tuhxS@ok(Z|q(ga>Ha*$-?B`Yeo`tqord9)*w|
zxnwm?8Qa7*+HKpiK_bkDS*!U1Z7L+u;@|1?Ul7A%Ok$HGwzaDKPEg7pRU6*^jKWfO
zgY_5_?Mp--t-0DfHI=s0${-6@LE`egwQ#b4-sGiMV`<x|VxXSb!7!O<cvf2%BQa%X
zv~W}K)TRexuZVK$1vxn-JG)K6>yb|WmRRf_OcqFq3S?b@xfRthYFZ<J)uWtpVcIE{
z_-QJBJKf9-`g~W?EW`)Xo&XYw-dA!VcGB(!*mrtT9?(>}73kbozatQFmI}PGg+tZL
zH0xGOcT1%*PoiDd5L}%%+}s8)6w>c64(El?7VJ7^;|;%1-x-u$je{|I5Q@A_C!bvG
zZZR|n%MR2Vzb4(JqIE-Aa~6P8OLC?()ga(X@Xf$P^~Ossg1uuoZgaH|4yk^9Xbe6@
zRllcqwzbx;3SnfJms>ss#9kyM0$+GIS=aT&!sf!Iqn4CItnn;g*Udd-QoyG*5IV*I
zo3%cuYHDHPiP|2KTWaFh8Q{%865PhkzedI1d{yL2XL0o*9rY^`2ppT^Da870>xB(_
zuqTX_o^HBpCIqtLHLzHw)_{X}%%Hf`da|Yd=}n>jNxZJ^Z@9b1gf$3#3d*y1B0P^6
zakmYdI5B^;t_#$>=!zu@D9pg?#O?fR_7^?s92rf2f!{_HKQd2dh^aRQ&Zyf<3+Gm^
z1CM*ke&+@Td`1~XgH`>xVL#0yU_7a-&tF-hO1j~~)lT%+h8X@HW5#ZAwRm+gcZE<K
z+nn!7f_Vwi!%Q3hP|<QLm9z8IFF6qV_btZVK%l)vlnBw<S^7Mt5)epX{U!34)SVbU
zLh%)~M6*B6Q4svb{%f^)hur34#!;Zky;frNQX+Hn^LA)B=J9pTdPWWRjBxtk_ToIP
z5zMN%lFkr@1r2QLGHfF5t1E@;Si_d(G+W|B?(I4e8rL8bcn5M}v^T%vjQZLFV6hm%
zfPy64WJA+pD|vJoS*4q8qdME2`#UZ_X6Qzus0uRTCu~*(3K_zWRk@lX*7+vf4Ff*5
zmL#P)P%?Qc$5C_8y``3Xy8jv5!g1dH=&r)=K~dLh;lR<{Q5HKLA1tU(-g^fvDP(w$
zZ^N&2aAN&ZtMmIpt3$JK$FLyQgRI$t0HMBGN`KAjBdlr~{?ii|aoVDdGiK3j!!Lo^
ztWK;OXmpb%#&Ewf)Zq*9XFb8)z22LYmN+=iEJa=)N$NZFFvrHrOk8eM*dTm5k!mk6
z(W|Zc(o=r%e$Z&!uUWW!xiw9jc!q*kbB5nqLjB~GwR(fpPH@sU(=A+P39!<_HfCFY
z1THu)!oP53?1htzYkU(KP-EBp#HG#?bTp}GF=W_uFg*HjHMkzkUpwX_zovM8*}&K_
z9OEw<9m<*%o>Y>98QFu}HMScV&gk4Gb-MM)z&mSKRdT@Rp>L?W_WUl^uY)evQ9BHX
zmEfQ`bK9HDAr87TPlZin9O78>7#X{`dZnA42crC%j|Dl0BpU4<v%<Kob5%=M+Wt)8
zbn^50rIh2wU+s%s*-l4;o7>(A;#kk_Z7{k?S3lAG{l-|Se5G%PBSTk4Lnkk@3q-*W
z!3IxK<?%_I7%qD&1wSVWq@b$z8;+e~D6G(R`tFL`Z;fM=Z92x;oG)Kh)iiP-D_YXj
zH>9{sl0qpECsY5oRUzY1dB!ZoF$q%P$#wOGC;q-iV{Q#TGau~s6{1<I*&sOt9xCvL
z!}>QCoa|bd2ql@^51$*J=LLV{`PIc)U!1Y)Z$`&!B38ArsJ5uY>YZCXY4!Z+4L|SV
zZ7AjI_+|pA-w>5@1+&DCvO&#hX;eFM-?NShva!TLRKdVmHRjvn^I5s+?eev{5&u0A
zx%P@&f|!mQq|8T;Au!ga2f+!LQ<;1bu|p?rfj=~{L~>PQPx;o$*m^t%!!(J^k&Vf(
zPf042(7JJYP}rB|Tz8klt#ZE`os;;Nv-_pjwOMr-zcwSAubW4sym~{8ew6TI<*nCW
zks>%wajyD3_5uFEbt^l@6g2uj2kBeh;Sk=5;t<M?uB5k-3K;HX-XHV7VT*rO?)z&#
z#cHL*!hHHj1W)c7hOQXPr=2YvB3pZSd=K=lH`y7b*5#~32H9?!P@T!8huU75l^W#A
zaPVVr+3OTN8uuuBg6K!>U8^rc+bj(=`!6RkBt0rEghP}S+qx5?)ulF;$m}C=DN}x6
z8#lQX3aGWbH`8#k4YfTm_@HR~t_O!KLRcheWa`yZ5Oi>MlP5h>q*Yc?^r;(a5&n@$
zxDEngSH8$`yZk13^*HAX9<dh9wlMk$!9jEdKfW->{b0L*NoVVYZv<`pOK|gr4&HMf
z>o=n07>tNjr-TC&sdV2Jv4%k5v9gTP_GvYV=IR)%(0l3DW3p(URQ^_s7aKq;2D^8g
zV!s7>T|(KEyu=pEcfR9UzN8d|)_mjD@UzF?h9B(L{oBS!7mmrd9)YmA`<f4m{Dkg1
zo6?l*Wm+do_1cgYcy1$O-h{nrB)S;z8=PB&x*N9Av)CkF!R~G8*rmay>`NJ)IRDDv
zL8jt%Es)Co8k&!EvvC{;=j9;9^E{7a*j}I9WW4xfVCSXR-FF5P;&Nrn#X!D!NymJ`
zPA#iai#I%`d8kLhJ9+oxUii12^;3TD3RQ=VvryQ<wSZJ>p08c@vtjW;{10hU+5?vL
zyn>y)tt@)0ejuwjx>8)Ia}-OAd0kI8#TbZb(GC({q-b1@RwS63Qfj@{+7S-%yH~Rm
z_nvi99;EKAJuy{0Q%Fp{!Yj~jq>j0OXTeY)yeP|AsGtJV#5{IparhZ-eHNOkFHI}*
zB=?K+H_`?RG9Q!+DUuZp!<~IY6T8Btdpdu9E*O9%omvlR*TIZB%$q5tgkZRM1x-p|
zjH!Zs<6CRJ^VG9cFxEMhs$NlW9;|6!anWY`QnC--?znBaZ+f(2`;CXl`iWs=gddir
z-ge#7@1aIyd{MT>tz4Mam?4wiL%0nU_pIFWCL!JvC&|~KehU0Ut4@yRCQ^o5cNW6m
zcnk{`rdo*EzpC)Ooi<cR>A8w6;vjh~XSI$KDe3mAC8i%eNo9lw-IkQ_%;eR!=?xUR
zjUL4D6_JGBLQYK0*%2bGT#|i=KcZ);SEVLi#<|?I>WNy8?x6J~hUxeoE`OIG8CyC#
z{8>v<GA;gWnhWw&Uch^}Iishz=&iHlQ!T{2VYD_JDma&?%EzdowCU@3B>JJ{Y)i0C
zn0&b;2eNt3U;Wy;|9dx_drx25<&fb2-R;~6o<LF9H>@y}QCC?sY9@^d#5z;iZ;xy>
zWgM3JKpM|T_*%Tn-t-4<GAUf+D+=quqKoo`Ee&pnCODy&r&7OHX!mawNw3eBfHZa5
zD`6QdF<keKmH3&P+ddSK-dv(Nl(?O7#-v&;BD?s;`NPX}+@reRM08KAu`8GMV*4`l
zRC!GTJ{=k@>r*-3LR=~7R#?VO<t*V;TN0>%KJsY};aOq5p~7j!l4Nikq6ai_GZBPB
zLDw<10;3IP{IS7?SO^`HqXU`IN8h5~(i_Sbfo65WzbZN67qgb0`{R~GcRIEUw$-_g
zVVV94dqvIFa^z>$(HCx+4Pvt5LQQgpA5SIxF5g~>T<34i@fi15>d0O$54+)YT<}X|
zrzQCDPRaa_?X_(z-_EOA6EVNo5m<b|&w2L_2!?`(Wa>rY@vfETfi1Y30ZGXV{xhkP
z8LH@NTrE6I$~t|Vi#U~@|A?XSzYcM`H+V1gv<LQEDH{Jhiw%ZYQSQh7`Dh}kTU8@g
z<ia2Q+QV93OnnkhqN0<N!8v<ftNM}9m!qO|m?eOqQh*FnjZOD^))H^`7tiZ86@nNh
zn8|gP7^<RJ{HEd;MU-ESG^XicRbeq*(UwQU7OV_{^eZyWIbPFgQC#Y1hGKP>qwk5&
zu@bMCQq|f&>Y1Qv*&=zXYiRNn^RTd~$@%uCb6#v&xb8t8^X4<?6uGk2>2HZub&=M#
zZVPNw^~-3dE0)EIq4sW{bu0;5jP!2NcCu>|q%&l>Zpe3zgQ@Enc4uvJsKN9{;e*!=
z95MH=qMDp;R+!iSd?qsYL`w$2A3@++#17-8#sldCFiR<^rQY0;)!!V`z5USf8xBkZ
zaBUe7HBcc7cCl33;z#>?^W9l@a=~$R*(*Pp*1A~4+pfN8WmapDZzdz=HLCSxNqc`Y
zkt@~0;nrKLJ`ROVrjZTbZQ|=hQp2iEl<f3UOiR<?l&Vl8^1QEq>~73arg=DCIfMOi
zU8JKv*5xO<(AfPEtL0A)Sz_})y{j-iq%ChY)BEbfuUygXpSX^gPI-6*da_IugIs*j
z8;*2<X3jVF?wOzaz!LB<tlYNlod3ZbsAlb?0>omRRP9b{jBsZIN1r}QhDY0q1#9W0
z^~)*M-CjmZM(e864~86XhkYxe2*4JkL)1KZw3Jqj?SgQV(y}N2%q~4%j;tk&Y~Xu}
z&mV9wUp~i{rLdFw1tqj{y|qfBx^3^)-46*HrE<B$2@OQWt<$tUb&qfjQ}$rOH`5Lv
zM*AYaw&rV3uvCX`-Hld^mD>$$QcI*{f00|T9N2WM1;wC75`?82*idL_AP7t-q&3Cz
zCAOSd;(F@emUBYzaeXdz8(x<Sp4*VQ)~zNjd*2W``~l74d$aUITqwCbHPMX>&nKk6
zjOG*RW8)#X)v6cDr)bjuR5o3Nmm-h3DMW;~_8yLG6_j}UIKpuJpP=xuC9+ULoL7LL
zVCzz~j!p1?0-#sFV+!I`H%??LM-ju8i9Frh=uvZuDI`pU0h3lSQmAv_Ms~P)8F44`
ze*(f{r`ge2s$0XF6pg13(XtBHaq%8MdS1vZA~C4NJ*D}%^MeV9odk-{i&%i<%UFN}
zAOY*Tcx#}k+YxT|p4+mK{w%c^$zV%~#~mxdK`td(+@25Rl!}zW@c%12g2aDT`s(iY
zwlZLl!<frp3cvph!DQ^RBm1m?!8#*;>8*zR6>RZGJ_N?5$GpHD4j2R%HG_uzXUzgm
zgiEb~I;D~2Qtbx<_WtyUgY*plZz~`!)5~|^ypNNq#C-xWxxqws>FXaM$$%))J<5^0
z5Yn*DD1%@N1YYcAyNUsBrY@3=dag@_4mtEWmW)okiQz>&?mrh$WpYBicH&axFO*3C
zTuS6XsiYBaB6d;eKNlceU57}yDfKd6Rs_`L^2&|?a)OUa0(Aen9~86$g<gX&NU*}=
z|9G4q=2Ll~RCvaUApEx?0Cot=kBFWWd6~B5pC{&D$fUSmDTn@B5neLjC2@$twZX^y
zr^bx_8W)xze~@$kj~dHhNCC0aOkijm?}bVJxnw?GsF)&HhnfB?0?Gg^w$psyh~nRD
z#P|%@=&Agx)qmX&3i2ZYMI6pB@kjns-HTu1xPguGnY*3-Wn*CM|C;%aFZEwD|JgSF
z+nWFAg8v;f{ut^0&!AzSfCAXnY{0T{=oej90+_$W%I~(0doIdv!GszLriF!_D>Iva
z4_=J9xV(TDR9v$}lNt?(Vh;de%2Di5y)+oRBEg8J8}vE*?|(%RVSdqr&`Nn}RoYH+
zo29wu&;mh<kDCc%>E-#^X_@P`J{RBy<@|6H@CHxG-bR{r|9z_rnUWcl&!mgk;(KKg
zt(cRN)b9ndnyuP(uTxEp0?o%#M=LC%i;3zc;OULG@C3K%!L)FI&bedy%`E4-nQVB4
z-L$@Du`U<DgvJ5z8>^GeM`qP@DZSUc+H)xr<7VkM0hGv5qBHPfbR{P8ccOp*LgFic
z&a+fz@I5Kj$bZU4Z&D-NpY&Xx%fRRNhwHprPX;g+zC{Tbi#`OCjpC0^-#w?|ck{Um
zD$2^ZcH?F8w`y0Iwd-7p#6`N9M*;G)3oxJQjP<;KjRqV9`d@s4S8Q?pHuzXFWp)Ig
z_5?wd(<K@}89%}tzDNm*T=fHFuAaAcjnw;<jpt{_Z-A(VisB4RMqNRu04zKwy)xTL
zG0`7wWFKUMuRU$>a2^L_c-ct5`2O#9O*f@ae<sl0wU>FA$jMYS;SdR!jOqYNoA7|}
zO#D|Xk#yJLk9n+gzGtHAg~)89_K2sS^d%+dj|_Tu?5HO7%gy88d93B{0>1aU)p9R_
zrE3_#MHhb#t0n^q-UC1rW3!v8pQSuM|1{egYL&on;<a8>WdcB*NBD5{M@ihj&PxTP
zO70#-@DyJK8GEic?|N=GaIAm*l+2<SW6th6>6|>_bG*s_l^piO0+8~=YFdKvKC{G1
z%q5n?DBJ)D?})Z1M9qgx*lwyYJ^1$|0CEBF97u6E@$yTZs~tG-_WC)b_yM5dfzIcF
zItCNA0r8V0J*9^CN9ntRGm#Rx+z9|_N=WC}%U`?EBPIB(yCb!q+U#BCb`cyh*Vqvm
z*2&;o-=9nh(VvJpxL8bx`UCLYs#t3|+tMD|IF?`%#5j!A^nRln9Vf~sEF%A!Z7B6j
zQ8c?v?sc}9>lUrp(MlHGFxS&;C8h9y6bp&7!}%jIHZj>A31JNYk?nK>SW&X4k%n}`
zqO(C{w4eAFOMGK)i1hOlI1(>DV`hSTh)g=EuK=DFy9at15FTY8hG~E0<|lTcmj)E6
zw{fJ;#9{>2ak=8jC=|mP);&BMjlG{6@oadUDXC(-N^9RY$RE~*NSvSSX)fk$2uk|8
zU5BKxEE9lk^Hf=;k6Gszr`S{y&vQb_eY9~VV1|Vze!a0yD;ABwg$(~zkf`XoldVvF
z-*QB{kW3%&lH+KgvDyJ&^*#G)8QK&9yYM~$PZ8>Q{b(P1iC>+kX5D#b!~F@%_z_Ko
z0*}*js%yxtx(xvyf(5nleeoP<v*YQnuj{_cX`aQ-k82zj3+I<kz(eVVQTGyuZ>5bC
z%H;tAX96Irc(%lj)+)di-+|KrkZ&3JCzI}1hVSKXDiiVy^L(l&P3m`@_NIJo#=^-+
zqfFhK`(`t|HVx-X0$9PRrio+8bZp>Yp?q)innBKto9#faEyX<nnBg%_qyQc+mt#vw
z7l7I0Ee@nluLX6GLI4N#J^6CS4Lf`Qzv_Fqyxnw8D7k~WGWJ`_`$%hf!{GHTJ(`~6
z+vzc(F(L|8o0)*|t4XsmD2`PJk>NP6Sx&>7J!AI**knx_Vk%%Mfm~W?q+r)bmLk!*
z{>mZfp4(%RjIa$Pn{j7MMZdhyy-r%w@as1LdAmGj8Z`a*WYMz@Ui>f3*3c`sZOkYk
z$l51X5iB%c%h=m}M1JXj#_dKgU_OlFC!<#_r3XG{4%X>`TItYx?c6(+M7lCM5I{Db
zNdt`J<}E^tRM#nYhiUJFa7$ui_AANOA4$F}k1EAg_C-aVv{M>fJ!66?s-TzQ6eHIw
zQC}V3TY-cuvO+Bau3Ne`1=ps(f(n1i>(ClWtQpBy=Xf+d1lYxcm6hziXvyo9O;<mv
zGlFiK2xze{T=MVYcp74ZrPVfWx%K!{&`fjeCUPF%Se<V2)nLzg{FdF4^##Lo3`R>9
zzCseeKFPIr2`XX_s?-i`h;pO=Jg`pxg<q|R0LteSP}IJLChgc4ij?A&uC+zJ$&Uj}
z())>baUWqjuK={-&r$%@{Okw4($YVV(-R=T#OcW5ujf@5x4{e>aW-dwZGVDCECN=0
ztWMnpO;4}t4Dc!)_WDF#?4QNE-_tSu{5lEXdc#R?zK7E}>6quD=+91uLSU)D@ulxq
z#@x2|MuyiEH9tj`;l1SBt8SX^{(CX`MHa_eaO_b`Jm;v}iK}s_oeNpUR|8+kY>FiL
z4}s-^Z|9z2G}p4fKK-Cm0&iD~Y@l@@_2C?+hM@bzO8GkZyrZ~Qdoir79(#G|d#rvC
zt6f_NHIl6B{W1^K&d`Ia6ZWkHW77pRx(@Yfn#kT1AwJ)|hjMpw`~wH1jtgT#5g|%-
z2L`^UqXHsNBfX7te)NlPYaKi#<a<9iZU!+bA}{&gUJzWNW`QOGapbUc68G6)%8dsh
z-p{Xy)k@^1y6B*PbTI(9eMOJ=X2sW#KKeYYj`$(tvn6~P<vS9FEHY$_H=^F1JUs5T
zKgur~(8#)9D=YbXA>n@2lt)3Od;5EBWF8d(&4$6~Rc|+J>!v6O-^(BQUfZGc9xIt~
zG^eH9*0h}?NcFDA7&7jDgYF%&b>dsqEI}Q(vfYi0L|%<YYdpE{SB$}_M_tKt`I_eE
z(QCnjEAt{}%@ku^&$E2^hv#sCDO_=9?C5+WOU7f#K{5|ONJO5caXfx=od-co(fg+9
zn4Cv5sNM5m(zVDDL6DN6ZujL0GThK&Mo|J}llj3;Q0Usl%L<YUg|tkxTKp)kF#`zZ
zBDbCo<)Ryh;V$jj7E*%7i-}jU{K&}Z!IlY1{D2k5XIN@j^0*hw+HBdqma<w(rbL!H
zq9d92aSGlCOIj7eC?O>Y?MhMFGa;%Kf!Rrf^=f&WU$b8IfOoCg#LLfjFTiI<mgvdf
zH>RQEUHHtMm+#2aC=mQ*x-?F;W_39EO}n|JU`64pId|}elvg>)q}S~<5D9Jd_cbvx
zY$t0q#fXonPVA3x;tkj0lrs6?y;SS8b9eP(jeDYV$p`ZHZp1?T%;*z{7WMDIYhE&I
zL;axF9f?MqOM9}Ea|+!MxL6BVf^Bd^!7m+!(4JRHU<(?gdDQU%(uFn92r%ZFi^Nuz
zqf)19<uZ@PDMO+jT;+ZX$kB31_(@^;T1nf>d3$&LhF+>?8XBgkqm;zvU~N<f(fIp4
z)GRwH%~MPv;?=#fWl4^3i#b}c?GPM=E+Z@!2<#}(VzCQS?1H3OR2LI?*rh-9P8b@|
z^KKKOdxB6qahpi{h6R}l2c25_=W@EQ4i^acC*k{Rr0_c4tkO&+BoZwJ8Jkf0p$K8-
z0@cNN=?xeA)pw&p@yq<*si-$>lzGV*b&cU3NT8CjzF{m<6w&KiIL&9*!wXZpLvicv
z`cS_QzyKrkNS$La7_Z?H{iSN1`^DtRCVSj#e$=hmKg1=^;BR$Dedr9-WQb&hgasn7
zPVBlpHTkeJA8BPL;fMs;qK&7CsywuVl|WW23QxWq9PJ{m5`)%lFmvRy_>HTmcQ*6h
z*6ex+L7U->zMmr&E7Lv=>}!&QDWDdx8h;>b>6f1u>sqMF`o=0kP*-AWp2J&kaM{0i
zpeKEeV7JnCFA(l(i~KkxW>C0=a$_GyJ=?5k#|=)%99z;9Na5hnyeE>%`pryCI?9B{
z%+oR@AH{Bzz@bXWMMTBaD>jopsUj(kF#}M~72KO(wTQMWYFbu+`e8eqo&07xTnI7e
zSB?@#$pQU`OQqTqw~PHg83te+BiA@`wO^QktW1M|2h2mj`|~k7=WSO_8uVSBapG7A
z3fnN}N|6g*|4+^CO8z39in1-}!H01IRjJ?EN%Y^ij%lvsBI2|TfS5-<q;nIj**ESu
z%!qFavMs#!W~b*+$awO=IR2vrzL7Uv5dw=C%Z_X>jb{?Ug;YFl(3dk4J7S-XPY=K5
zhfBsTNAb1^tUVl{&|_t@!Um0v#=X>DW&S#|wXlNXTJHR(nKAMW0ZuT>lsucd8Vk^=
z_1%?yW(%Z|u=MUA4y0S4x0<7i2s*$$e>{$W&aLo8Yc20F+z1Kpi{P>9A;`NW-m7W6
z8ASVBHtM=LI0F$qakEHA{DQM}$MmC;a&}eigfQ$?!($HBRVB$~=l!BB!D`hsbsz@f
zSA{o>&e)B?uaSqqH1H#Pl=Tg7MD}@>?6>hX%L>0!)qu3=2$FUOT)aR%8>~KacKhhn
zqg&w)12yh@OC_mZAPvKqvYy*)u;%0tK!tY7!a|skwHbhj3T(~#X!9OvQp~pk*6e*9
z+}U9+G?ollfn?Frz+8qK>+InAQ*l!8kjJShNuQs1`V7L+X3s;27;L3&ycHFt1jT(8
zj}2}!D$v8u?GDmRTnQqCtPlud#0{h8rt0*$-~gRodiX6k0V9<5{bm+1wIqS;j4Oo5
z$lG$?D@59VygS;#tN9KD)ZgJ-(zj$O%9~rDfMy7zOLxPI)`p%aVDl0yYI8!EyLUuz
z4-+6_84BomyB9WodE(hCheXr@Z4S#;YkV9~zgMjlIg_7>dg*3Iu?5E;z2+VA{Fspt
z5nNCTr(q;krVBiNR-kz^NafZphNo|ex@`6gZEwuG06hrfl;pt#na6g+f$YvvNW7Dy
zWWTBDS>VrE>YJ)*5=Ut%)a^Q06$r#F$}fR9te&|OX&i16f}ZIxOa>!7Qyj7lLq7pO
zYK3_gS4rVj)I7~@gs;j*DNC8?C*fyzlG{(pZ&!BE-|WJff+ms)R@h_F)hOQYZF6HT
z0S(rnIjbU;aZpO|$RA7&8*u4T;jxEF5Sb5_+~&Cih7N%sU*X#yzlgS}R8oBhMWO|<
zWPa9lv9QiO$8aHIP=7uyy%~5#CH0BAZQ+eF&-~I{!}A!#m=-h#!FO8T>3T${PLWH@
z1$kH8&OdR5ZrUwi9L1}tgQ$r}lk%>8R@G5UKZklSh(3TEO`o4F-(}ea15s#{qEPmE
z56>qP9oCUwE+h+yxtaGv;cu{2vD3kKyyezWKA_y-&91Qb_(N!Xa+f!x*H|?PQW86O
zciiGzgS?za2cLUsWRO-sds}<%%j0Wbi+*56#Za?TeTXFIgc!xoz9iid@dq^OU<Bm7
z6iVhn@-00lI*SQ7oE@aZH^NwlD0>@8G2!!lueG0T;l0;POTuyhUzp7Hj}efVMTz>4
z+p!7o*FjKSP1-*#zo0mFLr+!a2&hOpi+A>8DjyIn$u)Q$^xw(17|sdqN#dbw$F!oG
z5&LfOlfkniMG}MW?tJ-rY0YYhVQXQzVeM-9bX0ic09alJ^WoW!k*Ac+ka@=x?z5ES
zwr@9ssW=WVjb~F(y5}bst5v#1y7tT6>DX(7%9HP(2GQlH^V!NUv6igx2*Bt<skH~d
zN?hNKmU=myBdvtnj^kMCSS7yekpu>60Z)CbR5$Y59+V{L1QCMDzt^HsPd$0x9r=d6
z@^+ddck=1ki3&I&-NP*5bOK1)Olm=R(Cga<^uzs9Y42LBai&K3FbMbPHdtPnqiY;<
zg&AaN_*lxeSx9+q)<O7=A&jYj5^VW0pid})fJy=YqMu!0!M`P%&38oe2Zs$y3R|Oz
zTX=B~i}mV+KZ}iUMQhOC|L%u1p*4ZuIMXJBaS`~6H+AKk8T^d+RmDJ|T2{#Dc*FiT
zD`(Rf6+VR){?l;6;#K^4N3c7Lq2{NvX}h;f#d5DCI2a}qI&MpL!-|9|=B=k;5Z&4P
zp+Q5|BK-!BF;OxOdMHfTq*bUy4FIO|izoO&FXPRHJWpMsPuW$}jVj9=k~N#UN=?Qw
zHYeI^l+j#Ommm1am$tP@Ye`<keU2Y2PDqu4{1PBa3mzS8P|^%Ym(Vt3Q&7<NdN<n4
zNL|R7-B_%Ae-*AAZRDRK8lvi?&k_i9kUh?CRD>*A@i{e7PDSI+=(Z|fz`)j04Sj3;
z10>p;dOX$Vysw(OYDX(Le8OHVN$~CpnIoZT$f<=%S5M=e*C`Rop%;Uy664>=nY>8D
z*iqDe=Io*_LHmf(Ar7nbaJ3&`qxgbOXSh@n>}GE|(x*(L?qIY0R=Kqr1u+N6@~bXs
z`+co(^kRcMALB-WOx?(3T<AB-yOa>hvXg=#(m788$q2TXL=H%OukF`00Q>Jvg@JI+
zoZQW=6Z`1Wx^^ci+2j4qU)G9%;r#AZ7<XvRr{5rdbz8^9p%2%^SHlQXX5Pf5#lrYF
z1n*+QOgzox+#^+^lTk~(U2`}WDUUE^O%fM!;Q46G)r0*+BlNjOu4*FgBUuYBK}t3D
zyJe=IDb(l;)gqB{?xEdx2r5kLIr+2ZIggdq8uzhQIqTHbd{UkD0sb9fK~t*ok+7EY
zSS6LCLhBJ&?n|iA!PMx`TFEjC5oOOtbw{sOagx!hrJj}?S_bK-e*!pW^balRM7_Mc
zaSS!Qm4)|_w@BHp8Oalf$Lt2?zDXq(TKly3UOB#Hj`iq%ILmLaBfZH!18qd`tHGrM
z1ZKo42xSwMNGnQsH^$uHVUFXb$-#8XL>H~1>j8#}jk3y8=m9%a8nla@dG*hW?(oHT
zpBadeiWi;^DRw_$_y)T4JzPw_&wTA3BLiiqHithg6R;W!I?!AFnjcKs;C0N!sg}d5
z3u?Pu(e^_R7YUBNe$NCM+l&8uktXN2dk;#S5!(I!A14Q9%te$ay+uD|7zX2VI@IwW
zKM>f`1K&p*bL;#AuvdeBS#bj?Wd7H;Qmq2KGtO<RuArOcObeQYU$n2w>*PN8i}XN*
zC}`o})&9Qxt;s-h7nWuJqj>OIXP{m)>1OTk{@ri$r|;{ZBB19NewzWLk;*^Mx_Bbv
zcQ65!MaTWCZWopG^S$ufENk=Uf~WlJ0$Gi(L&g=l{y2XHMWCcX5hr_vDeC`Q-OGn|
z2?O_-HH*m~_k*%#7(fwc?wY-q4!3^{Vsf!Ks3Z~iS9LGS+93jmisuqMOCUAMrFH)L
zS@sy5Sbo2X_+M-ORmx!~I9%~fBX_(Em;a~I<Luzu*#s&7aX%>K{{u6h<5cfa?okn~
S{RZ0y$OCy5xnfyZ!2bZ3Yzip=

literal 0
HcmV?d00001

diff --git a/docs/figures/arch_3.png b/docs/figures/arch_3.png
new file mode 100644
index 0000000000000000000000000000000000000000..ba211fbfe5c8fe3f3215d8c2b763ee744c2961d5
GIT binary patch
literal 61499
zcmeFZg;!NwyEjaWfC5TNmvl&XC@tOH2uODb5>kR7-7SrPl$3->r+{>KclSFNKF@RR
zbIv=y@%;hs9%H*VbFO*i?}|Cs+J01!lX!&k3<U-T=8=@7s1gheJQ55H+$u8o7v}ur
z!y*g}O1Z_WR|-<EUXdy|*uJr_GKGPW{1_YmKtWj(*Jtk?bE-egL(GUL9#6BM3QMnN
zo;{}{dx#+&?3-aEfunA~7#!^LNH>S321j@_1d%=jkzw7B0i}D-=x1Q}oe!B+;@#bv
zknQgDUaIRZ&)RIQ*(yH_jZ=w6%42$%2(s=t|BD(nW22>na}gi-L!V$l<oI~1H$nag
zFe<rgP2E9EFtWX_ea}5_j;_41#TrRNAHZPwIgks5#`_-HJN;PwNHqaNdZpN~8)bEX
zpf8fhB=S_W*32(D&&|Z|DK-9w#LADzN5YzjFmQ=Iv6S$_qpwbS3H4YvgSJ1G2Lv-?
zWqxXekN@;ZMzGsl$H0FwSTSvXK>9j5eYxpeCgx(<A=;F`Dqf1r4W*)WW7_m?&Hmc9
zOEO-rgGKlOkJutXU_aV%FLAc%WsgP-9u|M!%iHW5$~;LgioC@56-=%^&yu_*D{Bvh
zW_Id_4Lqzu6Dh$KI414^@)eY9GC1^7i$Bxel$bm(VP{mzmB+L*xph<~&&uPGz{6(m
z)3jK)W#c4fZ4iC-yhoq(&_4a+7UR@A@zL~C`sL2iH+iDx-KYJjUHYU(1LwUVkt^-E
zZXSK(qTSnV6T9Uai^mv=4Wu&?xdYt~<BR*mnU{MfF(&YH9d2Ic;=92yWB3TlU_AWl
zBk&D@0Ru(&yTL{)5h8KoqUjSu2WigeFttRVF3f}8HI`zAuU0S#`4a{72TWGwIAraK
z_`1)vUVTM3Pqj)Mq>6+4tpnrJ{RKaDmI)^R0*1ZnAUi6Bc1Dy3mVp$OE(2aQ<Dru;
zCd;d%3V6BZM}6by2p^C!P;uahC1LS};o^L-4H#))#0=QRk?5LjzM=o|`87{tj+EEz
zZilb)p!%oRKHjk(>^{mTA9MpWKT>{CU-JmMY{ZRMxe?f{embPgG%zS1j4}~u@Slh%
znIl#{rt_1|Krv@lLA`-X_3wYBo+-CZ_UFMP5!*~=7Px3b-mkC(&5njRy@59i$ZoJH
zXv~3*3!ZKiI*-+V1<gM{=l3LPL_cdOomWI4bwGw66z|2rS@cDqu$O#7E}}*jAc?OO
z#!gx+$>)H(Ar=@pfJYb@XlT^SB8R6hLm3#P`AIW?wq3>GEsJbIXhMC0U@!72U!0$6
zN0Jdy8KD&#Cw@JGwh!Yk;^wqwy_VUI#|A?4cHBt)7-pZ3+S(SC?0f9nYEo;&r+IX!
zOGvmvI+j!qa^1{4S<*rW+HvPy=g}7i7Wd{K?586;ATau2|4hb^z`#62HudLi!EYgI
zL2bcP^v#!dqC|Rn(Q4W%+j`Tgw#c{$WB(~zl9iGtnq0CVyDHm334Y^ALC~=rcuDk&
zFsaDVREwAB>;_L`*9z7Sm{65JocKT}9~Z?Hg&s8!NfI@uNP`6*LO>NBB|9NmkfW2q
zpW_*o*OlDkxO{O~v_ij<yu8=hl>4OMQiao*j3Zu4TtK`*j7W`?4v)r-#tvJJR)sD_
zWnB$VHJet84vGE^4G#S%t)42ON(((cjXUi@>|yLFt**jm9vQoC(t|JwZHX!QdQpOm
zrS(9k#{#i>qCzrz8J}tBX{Tr;Xqi<@R0NBsdxY_j&>mBymVLC38EiXeeQk{MF3w;y
zQ1em6ZtrD2t9F~LScX#mP`-oyN+_N3$ZHN+=g_wmn@>4DC4JP&pL|lK`!@bhpxLk)
zU*BpUQ;L_<DTTAfjHrN&SM8<X3Bl%<XGzZlpBaS_J;P-W<iJm|Nh(OnNit&(sx;SP
z(t25}R9vCOoY*bH$(E|w|D%&|P;KmVB<}6r$nJ0A;TL~C{^=UF_~V}SddPI+`9|F4
z_l@v%iw(rh;@?GM@3Q6pm~G^(+m6|e-I*JhV_1Ki&6qIwt<V->oM5S8J8mrc!hu7}
zO2yh~0MD$KRgP`e!gC6Nvy4OPMFD%QDN6EWB8seQl1u;3e)|N9BxQlTsZUnzoINj!
zxVG5L5*{S1C2}P<r4?~;+3d#E?l?ASk85RXZ&$ln6_{+8C7MmVy%_Co4KY)-a2`l9
z+Ock!tZTar3o8g~eiPpZubpI_vEwntIc4i|>hh!>sh;eez!kz3-IWbaDs~Iq8(sTm
z-WtBIF5D~o6NCLPrVjF#NaWtgS;#rdeK<l}d$_ujKDPIDsn>aSYkI52Ny=_wX=tl(
zr`x&M#cMTptKF4v{?yO9SI(5gsL7?;F1{w+?rwcyy=-VHg?e#gNppQ-A!iwTLv`Wg
zk9G&&ME&f$){arL+5sN@ucb|UICD1PNI@M1E1%9Mu8NQA*X2*q&r0{baSWc2J~V&i
z4}Sw!jdY5z@!+Fh(gzV1YB$UMXpCPGZDdV>i!Wcjq<@tjq#0O@Y>A#1z>HFgdWW$<
zTt>TSpRJh9Ud|Zu44Xui;N-<5TODN`*BhJO?u4zfw`DEkaonCdD$~9-gBN;7)qNzv
z)nUxRds+rH1{;1ws3SPJ%=`_(b;0%P8gQIsyqQc3^jrzreMGX$lxkGr`}}TG?xp88
z*Gjh$&6*12Zjs9=DpBkS+bKUd+-cf4au`fC*2%tNG)kSw&ZJ(xVC3F0m~``S@-Rjs
zqaa(#mXwNlUYAgiSj%3aY(>X}amM7L^TYZexK-bQ6t!C`VkAmdBKZq}62%7kP$OY{
zN_UF<>#R6J?ZO(B%Hwv{Xj6i?hR&1z8+V>JJhtvrC_C7e>c_<<rPK8&TCp{S)>XTk
z*CX(jo}+)hQWeW;zhfI9FQweOsatn0zIb+Fd3fc$f05ZJ+$gN{yJ6PiW|!oJ5Q_Kq
zxIu$qN8nz-b&I7Kj+m*KeyDrf*!%M9g6q74Few5)&I#+uD*S5A%390Nt+=EoZBf%v
zx5d8N$<m2A0XcisjFT=^y}4FrE<-LGr?ZDXhjBz6e4KmdT9HcjO+yXkWzr3AUyXF6
zUZ=Xc)b6|;FD|P$KZ_S=5kNn^+_o;{%x9Vknu@R3ySUB2*%7LEUm}p<lCj=g{UPf^
z1BW38uH}xU<<_4q&XHdyLzkJa4uWh*PWh*Y3`W0IM-FN@ArM9qyI;I+Vb+zn`TU*8
zn8=d5#QVo4!j7on(?6A8EwXuXYOATf=Nx6vb$wUM8-3c8bmy@jb6U_n+?~gjN|xFs
z#P*)^Y<r#OpmyP?Ywc<Zr%Ip^!*OA@?RO|q{57*~?Y7s&OTPM>ZR|Dryb@kL0rxOq
z(7KtET!Qaha?_O!?peP|zHG$Z6vs_HuGZ#Tb55&W@>sjp9jK4rv)|QiB5`V}pfBGN
zarJcFgTMXoGiU>Sj=$TT{nSks{d?a2$Cl{wyiNHcZ?@awvaymgr*yNqFtddL;<KYm
z{hPFtt1;i}wU{+i=1RRkkF_49Jv$_>yK%Vf@l{_`H~xiqr*y48V)<pP<Ie7Csa>+S
zvzeeP`JlROPi2nC)Ap#*n%(4hBj^*lF$t<r!TZ9C-plD*s>`0xGAjK8{hlU#Z_AtA
zy9P6UV^5vlSUYNMSOg+Fr6(}9=y1Xx5q(?;V1}*>(ma$h3hsQQ;O;)=UG7Si!5ipv
z!C>~lBG)`@4o9iX5xY@df<?~E@}P#%*6wL{y5pNDk`HbngE>XhMw&$A_q-jL3oj77
zm3P1MLVBua-8{GE4c{Jwym++Ku?L#i5K|4QH?p!Ybl^KO3<B&k7)0<57Q6*v3I6k4
z9F_(K{(c+|1}4Y?2H{`V$bnDj6#?GRn!lg$QGqZIz+X?m+bsj`-&eyUWx)UYJKQRm
zgAq}FB_#zum5m)tO>G>_ZJlJF3j2Y9hjx;hjxaDdRL~n%N{MnG&>yu>(Qwj`mEkqE
zwPrFju{APfa<jIB=)my1@q(|`rcQ>WZq`;dj=XLH<o8$bg745UGdbz~B~F$C<QlRH
zq_1opOi4MJSeRJI1yM*zN%<X2-ta1kivMdl_)CD?+{wv~mzmks)s@MWjmg%*jQIr*
z4-Yd7D>ExABe;Um(cQ+$(2dc?k>W3rf71~)bu@OcuyeAowIPM*8XDO;I|-1JLyZ3O
z`YWfYo5g=|vT^*ET0lW&XomR(6ASZyXoIEv&?v8hg`26Brl^HAuo>VX$i>dee}Db|
zn)xq||5&N<kCj~P|6KWxng6*`)zQ@9m8~^k=_L4H?)ul_e@^~uAwM&u`9Cc2*Ua~$
zz|Mjw{LKIHnIOuRME)6YBcX+;ybAaPAcJ1800J;He?Ot`I+RI`O~f!T!Z1>zA}VgM
z+o=y?u*Aj!_m~v@qC_!7VMTgFV9{ud#6{Z(Sy1T|g<;`I^E3Q>GM=C{k>@9fy)b)W
z0MqL;A4DS=kwH_=z~?<TL~It6IFmY)mbkf_*Z(tdW4EyGthOP|gML*sg_06k7zPf*
z2L^!@1{PVE^bdjq22y6YQ2(ocj5cEk<L#6G^EwzfKbCo_f3Ny`DF)IH`2P@shI~lT
z+#cSO{SPgF*u8&o`q$7rhCvu?DePbF{~GgQnJ4=1r14(-&k;g68LCB&|9;i~5&(wc
ze~I+Ur$F=v(ti^Zrp3qgKLXxc!=Uwlj@Tiy#0HQ3_p5+S9{+cneEP2uelix9Z4LL8
zSoP)FwX|sISUPv5H=blgYQ<4c<@pr@(eTNMc%6cq`fMSNrsT~kS*!~bY=7+_Y_i-L
z>9A8ZCT(L=^5bGPdFZ?KRHZ>8yG4}K&ZH`xN<OJP|ABvs%cO-xLC{Mw7+<?5!g&5{
zyKaww4`JYlF>7|dPb~)$*sx^HX9f{Rp)&vY{Uz!tuH#Y|S9fvsEZs^^e0+g&Ubm)M
zYV5g4$xEP_WF$-rsh9!(V_(uI=gZOZ%SEM)QP}iqo&9`!3=fFiGAhjn$%9KSTk<ZB
z)_RgSEI$=LBH)Zkb(t)Pn2D4W6}&w+cC;>SIE~sIDZ;HA_}q(Gv)6dFyjfJri7j}&
z|H}dwvBV7i!Q&WA3USF=hn3}(c>Orz&jO<jZPY+-U!|wQcrnk2@X6-A;ZSiHBIrwM
zUe3PTYk1keyu6&CCux2B=c{JLZ-LVH3_3NxgCprw3p*cDr_8ICS<p)mY*v^J#HD-R
zYBDulA6AZhL3G_7SA5f(V7SQ*c(2@?%`vOY<;o^!JFIlCPCg*<Bn={R(a5~I?nLt)
zDN=u3@utsYrp{TMaN2P|AhBS)K&)-yMF$~;ZNo_%<^kQdGlDMyFm46|0tS+4ri32y
zc|!A0&TH8ZN<ym0<^zsLt9>gK1A@_1ujB9r9`h_?xC2{@671>afl=*T`E;hdG>=05
ztL+Ls+&y=9UehH%R3?^TvF}zBi|!X?i0vrw4Ywy_`aNmycJaL^nVFgEYOoD4+M*7a
zd6hC2HQRnp*E$X#x+=ve3*8(?Zv(33?{7S6uXgKmBt8?13G@p<u5Wv4=RqWi;UkRv
zFvX#p7IW<m1JexCczHx!pHZ|qk<EH{<Hqn;Aa>o_2)dbGWx+2@Bt7iJQ3qnLV@2cA
ziW>If6Y~Y?bWewvruK|?XS)HLPU2`;XGDey>%>B=ubMY9KXVVe$q=S_Y!>vYBxT^-
zT>Wv)Rx8$uoO^%8`17fbABb-S2e?mo+;<XGBBWLbl@GMuJxje)?M!u@rM<a4nL_!w
zk(bUiz<=~9QTn)txv7(L%Ic|?c-|lL(yDNlKgHTy9=c7}z+rFxh&1h0&%IYF*5bH%
z8zrZ!RqK#*A?UVm!1gFhHkmUklxrrtzHVJIAl#_+`An|j+U!_~uC~jZm+QsRfRNa`
z9$+bD0Q2zm7y_ab$rv0)Ey|vFX08D$Exv;`T>Vac?;EPHXB@0|)2vQ~oo932w(lEH
zCoLD*TY7_k*6%fL2^tBWjT@ueKYJ5U61x(_XO!{c^00^5w0`G%v}sSA*t_i4yhi(5
zrA^)7Z6ogEGUKeyb|R`t_4-n8y%JA%IKxtQaTvh1m#<>VroQWR%2t2<bS>SRiGa)I
zlOEsk?^loLR|<!-DkXF!zD!nGQu7K4Dk{x}OLCm5ebrF>^JXTAZDKUN+vH%et^C(x
zz68B@D_La#(dd0w6=$>W2-2U9sE%5y>$&I1q;L<mijS>QR=m&@xa7(yy8+yCGBSi?
z3a*$n(?rL-uMZ?FhO$4_a=huq+W?@?>Zn12Fl~Y$T|(ye1P%ik1=ftIa>LZ=*&Jq`
zee7>W39JOOl)Grrly<dDv(2$m+ua%GG2MWYE%dwtlD;)|1k@R!yX&JbXO+_?cPW8t
znVy%%-7&PQ7hl48=~x|CdnHE-RZ&;e^;1#}zjx>1rtA>)RrKRTJ>27a`D>bl-CQB_
zs^29dU{mtV-f~wo>I(f9x@+rvZn8}Xx8=i6kI?z|g)7IS0Gj-mlg6sF@rmqPF);ye
zM*7px@)5Ior;R7?crIsl$i@bv&zIR3QOg{1^Ape7z0`FGj@cBS3BC7h#Y_+A{%uCI
zfNwvlLD~HTU%;c%{kE5F;M0$zem+bIQma^fP2NX){uezCiadp0@p>K>(rw&+oQv$U
z6ugI6J*_vp!Hb<!woR!@iAfSiTkquVL5>6W@eW-#?nxIaEV71thS@+`E}4BZ>;uXw
zRRWfNBk`kYklfVPd8*6e3t1FIvKEE!cp8p!ty19@^o9psGu37*=<cGc<K@d<$`kaa
z@;bA+ry>sJGFV&j@G;bCW~H}1gg3+bx%?Ryi3CLFAI`i7)BIl^ebab%Qa?OQT-+1N
zT*89K>tx=lIcBgau%7WDOAMD9EzR?=D>wwEY6QRPg>qyip}GloXamK?#l_&^G&3Ms
z{UTWskNe%ZM2a;pMgMwWfqGjqTb4;9-xsXaVe9L<liKwDd~qkO@DTPquGt?CYy_?M
z`2@Cimj>m5tdf6RUv9bJ1QPf{$z6QwlehO5#F&Lp6r7U|P2&h9+WvK&KTKgC8Xd=U
zU3EqC{*2@+{+b?krHfs@6(Dh6jw~=)(u5L~Hl{G1Re(P#2C|^<(r2MrcHxv?ks(f_
zJHF0xnvBOf76BqwWCbNJ>;il_+f!(9G0l=>Dr@d;_Phth_inHDNE&wE(-yPX9c`I$
zj^d%=v5+oq<6&D%@(w$Hk`j9ek4?L49#SG3j19wkxHMb86wYTf!#3ALjyBygJ&0Q!
zIh7}$9?aIjkVQfKjL+-LC^}q&XZTldL$n+(?V^lC$gE4Qk64O@>}ncH#rZM{Jfpf;
zNd+F)WpE=&Nhv54p?YVFJs@DeycmqEQi);{qu;dj6l6S$)3*7@J-oAMi+wT0Gu#}b
zS*q7Sp1$-Ps&I?Qq-Y@&tzO6o5k6Dg_TT9khxp8>35Vyo&!!zw1>$85GXxgO9tn?q
zyS=%RAh2w@yRCmW;d!Zwcf2z-HonLCeTG&L=$86E`;n=&LI-ELF6m@N>Dg4Z?Veu9
zdk<Q)%ExAF1?CeaEWzIKDGhgD>>WoPMbllIK0V{0^O(~#Th!xBFJV!DjWEAVtOzUW
z+B$)AT|@hrFWc`C_>vR*#2Az@OtT>7=(F-3Ci>06nB7{dSod-ZBU<dvU!ba(KO{wa
zN#H?<&IUX4QIY_2GFe;Y?L>Kw`)<urIQ_8(XMnF9zEn&Kqx_e!D)*yRY>(E`mJHXu
z*>?15*lmX8@ceaJX*}}}`o=}dZ=d9VwAD5}XWv0oUAL3N@|(M@M1Qb~x<4)n(`@eM
z<w@bW;=9y<rGS!pp_I0^!J76p=UU1wTGu?2jmJ8A&)0;@fZ&T~F(EEEaS}&`27_+g
zp}ER*6t@c!#O{<@x#4M%Q2QAZ2h);Kx!m~xmgK1&ZU|1bol@Sm`4`6bvrS<w!nnLe
z#g}^*Yw65bL`|shGl4_8BzAiZXGv|9C8Y;bv30{uPS!s`wfHe2gzauKdrs`e93Wyp
z{AU`5hx+u{yTv@b+B4^8QI@4Bq2nqkEZl1;jxu_s0YA}&t}&`;-xtkkTs`q}bs-C>
zp|CH}=iQMGNU4}o&0=WNb9A9#JzeKpOes^`KH|Uczwu%HMOayNf`ok@I<5cy$xc}*
zQI)^ke?$DjNHnv<3Ja<Kd)Q$6<Og&?jZQrvu5-Jt|JTsB92TE{a55MN_*QV3$M{aI
zGi)jo)Rc9-HpFO(;QO97zPH<AtLqrWz@zQ^tjVcP!>T)u<+n3Ii7YQR-AK0mP{4WJ
z>$v)isj%pfL{a7Dkiu7ES?0GUUzo&JR)`_!YQQ0X(0Jw<BrJ(sGtA{AN~-B_8uNMA
z>Ik$;*dD#&h8UnWYOA;Bk|M*;ln-0;Z`w%hwoGamW(e*9D6R6O=4L6|hFs<~9F)-#
zv^1w`Ev49ESV|vXk$7KYo?L-0HAHWWdEyC@R`t|YDW60v)dsl*_QSGznhmB`1(dqg
znS1T{W+{=<oM{1|#rTE|>;F?tvIfR?{%Hccg;Hf%tW{CpIAYkG*O@LEO$p2IfXst}
zd`C2+bjF<$(XLTEnRG#cP37yCD(ZNYQ~iTGRZNBPd<6ARYAw-pY<9_)Lz(JmvMdoQ
zU*IvLrF_`vJuxxktTCm!bP5nD7u&GebIIC~o;lPZ2sPReEK2t{^}igfv0obR8QiPW
zYsD7bEs@#nvqFKNp;O<Hu0)Nq)Tc}5mCn!}dYf3+XDJmO!si+~ZnEYUQ^}C_0Vr_4
z$pKy3I%mYL3YIl{2emHRM4j`NIR(}G+46TtB3XzfWmy!rJ{990mG(tpq-%fX$9trj
z>ZfAgE;)fdN?RaLU|B0I%>pCcX-cjkFH~g-z1~4iIh^xr$lG4Qo8#eJG)A+9mVg6Q
zx63JAb^>O!>A-Kq2&*Mj!MI4fqp_DVg)Vhl=qbwC)rS&mO+|Z`0{}1Yf_1V!^iF%-
zop^<jEZDhIYI~-Xa3G;{2v91xuh;oT<$|uC(n|#~F)lB(cWG|eB++QD=e%vA>(Gt}
zZ2E**0a8p%Y99OgJpGN>X7kE+fBxs2Y655Xi^CPnYA*df%Mbj576bLOmcKJZoZgA<
z91wEZ=w$K?gTy9~I9BZ4{t16DFhY(<suD4q9*zC;=@64<Ipy0?|A$YrPG;Sf!Unf?
z<Hs49XWFAVe^inchDncwXZic0AfNH1Ze~7cwit47+KjRh+fbE$j%I{QR{y;f6<z@O
zD`<P$)n>|%!M-LWDd&-;wT$I2eVz_#R}(RLZWq_;!HeTM^``p1+|jKZ7KO+)CWo=X
z!10>K{T9^0;RxsCk@Fakpcf~IEQa7Gq@z~*Qwb(qrmQLGLX4}YM|5+YT*=S2gQAG-
za`)20(JW++(U6}&p;Hd_W-+m(7i9CwaFe9NuQI2RaygNzGWTvr2TUHN9V~T7qz{WF
zc$~-Zt$BgO6Fy5XjkiuFx4TiM6@Tipb(%j3Q>;<8sMau3*3w+1ND#AI`Me?b;;Hpb
zzQCGRZnAtpz?S97=E#>Ib8IR!_TIYsFzvuy7{__v2jcH`Rk7S-qaMOD&guGZPhv8o
z9VhZQt|sW2o#A7n)Uk}{vJ{O4p_7z+#*w!DaUlLjh$ztl^lYN?xua{&n^AnGRFN#d
zIp9FM9kx0aNpMqq_udI75I456Lw70T#5{#Pe79fILpG1Qjrh-MTr98<=a1u|*X?fv
zv|mkN$4^JnpLCfP)=Sj#$!&0~?1qdjc01lZcD`B=H50~)Prrh;;)|fFEPmW4fFmRS
z_NFgM5|m674!kP+a?rsh3#={WVbq#eM&{;qz5UQ>y}W+>VCUgT!4sP=e7F3tTuoO6
zj_p<UJ8=u|H-D;tU7!mI+kK7uQvbO2krdYeSrFlqLA@Iji^)ohL1N9mFDvH6)-ig{
zBdYCz025R_HYN1TGjQyqwv3i$YwS}WpHkx_a>XX^UZ0#7h-!&%lbaM@Mav6HX;gAD
z83&~INWH7glS_R@l=*SM^J=FW*MmewW_<#gH_GXFP=Y{?_l8=q))IBv!$>GJBviW}
z><ox?e{Zdhh?g_v%ep-IHpRkZL#N1#yHb}IytV!*d>t#^2}^YBNoxq~iF%&2g(h#I
zLR9wF`)NhO8-nK|U~j(fCmfEmcrE`CY(f1KeApgIrWDh+S@?V$?!-1TBd~u)*Y?Ez
z*~`Ual@PXc-4tEbRtqcKWa?Yk8~q4JjJN{*9(#|3wu$E)w=e8H3UVCR`rDbzCd0Xh
zX<9ApLWT{n%@ztTs#SAqP&e}`xV2ISpV)ikv|+N@%g1czIo5rp?U-;;N@BMlm;0hX
zSf)_ki~s)Z#!$|p3@Yn%N~%+9Vz<&)#D223fyERskEkUGv64GN9~|r2iQ~2=%2CRF
zQBP6);J@dG>+G)DJ|J|*&>tM=?tPN#ApW=&pON;vn}ky`f6|<0h4JiM)ODZ5&&%pU
z=eK%hS_#|_N@6c{nMM_e0Zd+s!5Cr`+AT34T)!sO!#f{)%7Fe17W;d@3LQmmisCbS
zaUp5S$I`!kPH+pSI3tQ<t*?bLH`4U8Hx6Nm5&yTQ)5j+a?BhAb5tDV``rQ#0WvZgu
zLUBVYK136G?*H6nlotYJ$)0EL$B~R~e<dZG)|v$(7T7EKCxJtfKMNi1$mc;YEzLq@
zC`CEE*z$SRRTjg~i;+{r2E_5Wx3Gq^&;!5$+5&|S3$lLi0Ulf<<va?!*#jJV7vjSl
zbiBXEG#K9!0gcsl0|Eq645|!NFK4D|rsKXpM&sryp1z<)fR4c52QWksX%~aU|M#&K
zmIREU0wLM%KhNk$5r9Hu$gE7zS<b)D#WJ`-GG0WsC;6`-7&tOedyqvU>@oj2*aq5h
zEQ#RzaQyd?9WpFvMv(nJh5m;gAWahvnknv=(?NgbyO&BB88o&pF-&y)|2b$61Zu-v
z6uj5tU(&x61}^j?T_XSIU<fc1O*qCG8Poy&OL`1AAUOsCKJGsU6$pVq_Wm{K|8*UV
zF9tZA^e64CLH_69e>&}d)BFEp%6&{<$yw=-*Hz6I)e4lhRifXo$;wTC2@%H>J?B{|
z3|09`!K0eQbC>j5uB}e+@Uil}Y^13`&yDQY6OfgLWJt>wENL~4;;_DYX_H&d#dogG
zDHXoRduiQOFd^<>NlaB+H7O-m)u7Zp02*pwFAPGU>3r)Gc~B}MFoLgdqOlm#>9phI
zBRkXQ+&7)&MrLw!Zu+CN?>xH-raOnrY0>T-xBc9Q<r|-%#si=?sWj<f2VvLU0Vl1-
z#V5&ZQV#(}eFq1sY8z_WxpR-Mf<|wx!VHCb>wXIbpeu1Rcm&m^V&wIF;ak7=O5Z$I
z)y`P$$*Sev(aw2hubaa?yyxQg8aQlvd=Vdkblw$C`!?#j?UzYUlwv>f&(&>@Xl0S^
zooV%Mu3D*Vo!ITkmpk_=Mc@FHDh+b(8JbdHXbNd_dOL@Ua^7lu_v7zRt&lY?_;$7w
zRWb9EI9|s7?;2~c#!ABH5;AHl=6JbNN+K^MqPx8-!$qR@pbR5jwKDo^v=+2`CHDP*
z5~v|Dut51{RJeYp%xhaFXBPsGKcPcIdg8U0xRMtFBv|&j`1d;i;pRKwq}$JA!~hQ>
zqzc~@IS2TLH4Uz#N`G_}J4Rj4+GYKjZS*bOZr8Y{KMvyD!1B2c6onC*aQ$7a$!jUg
zgi5D+Z?;Y{z7y=?Cf@2%U2KQ$W!v1e6URtm{#AkzQX(=>4<aH>S}w*pATWVX>o(!a
zuY&a{iOIoKBC<Xxg$DaHc|nGrZjDcf1@GR>w`8Cnl!f3BAqICb=N!u@B;TEdm{Y{7
zhV^OVRvv%lnM>qhy*z`i?=p1w0OR`=oR*phAmBm3j%|*ka^<N<KRHRg`Y7tVC3>;y
z%CknKvvj-s^q#z@D2(C!7Xw4Od$>gktxC>rj~ea7WlJgGQlZG;n9&c1Jn&}V899?z
zf{VY0b+Z~AGQ2j$2VKz8**)au!&vc}RwDO)zei6Jij8;&TwfKSVs#TBxy#vjh;9t6
z@lOT0YTFj(7UU&->WHR{bSq6(x}}r2oR+8%xax2RzB7)XjM}D)ziHwMRQ>JGm^dI|
z!IEQKSY|GJ)#OzW48ECji!G#K3Fiq)Q{_J<=qecM<$7;K11bFtDD8jqsM#Ap{tE)<
zzzz*!QHr&ISCljr%IyiG>s5F^PVT@}t8(vO7XP~4;ERSyqrvr>)7!%>@Gbp24pU4-
zn;q=ONmR8QkDa{_AKqsU96(!xkWdCR=YC=aM?d70z8z}nbFT!@C`dSNr9H<!^=iiz
zG^YHYa!??U6W~hCeqHUK^z-}lM?FsZ#}mZ%z!aSVw{f_xiCha!8FFT@2o~7{%H{A8
z1#rM8Am{}9<wU;}KUgTBIUmy&*q1>Uq=0I@kIxLis!PX%EugS4MzG}UJ3c9sVP>^=
z(T^1~9<QmqCB^=%Q8@SYlw|3?Lj1$(7SsygcD>974=^oC^BAzRUt`s9s&cOjm|;2)
zoKdIWW7L2Q*55Zk<R`fCL#lyrxONuBC^TbTth|APxan3aE|7C=V_a<NV|ah@{HHDP
z6a?h^n2;p#{qgxUwlk1<SX~5Py5FBh;mDzIhXNZtQ61@Z7s|aAH$YaB<TC6A<)KBE
z)H~J6Bky}`J*KHNRswGGL~6d$)SPJd81mVb&l$vk0ZL-r+zFtxo~H}{+)ONY@0S|o
ziBeSDFFnIP^_Ftjs;mx2M7lRjvk)-MkB88@AxVRtJDd3H0Qez|SDRvHudwoCSCYdb
z9O{qdXAB(fv%au6s7kDUiP8{oo^;-wqD&n2+>F+wGb(1lbaT|`bnm^Gc!q-qcQ1GV
zKqJc0JTBxZ`G;1wKT@&2x=#(WzSh>@6!V=-(cYpduxr-N;Yjzqw@kzv5bEskxo*IW
z(+G!eIHEiCg-nQEei!qF>ntcQQsK8U9Mv>{Vigvd60%BfKoPW(pDg0c#_Qwq&8C!Q
z4r42lV#fD1m9Ny;pLO)KxWE|&WNQoPdNzZQdn+0YYmb221T+YlhxP&Q-z;~;5{b1t
z4M?hX(bKBS(p6niTzKvN)mRNEQ67@s4ekhKJRLt4uPyFp=?r8!zv4iZT#Jd8?pW3~
zKz{!BT5=%2sLK5Ta5Dv5@@(PeC%o$wMGHaZTpet))_M7KKHm#S;D3|=16bc5uC8{Y
zcwZ}<qqdVHl*>vg+6}}`AN_>;3lT{e!w3x2`=agkigfQo?vsg{@#GdyJ?VZ<+3dxI
zP|sb&Rj6%2W&)+$&N$Qz;2jYGL}m@v*JM*)stGQ#>UKk=g$(p^TV$NSlD-6z`fJ=5
z_+|o-9d#JA;{>*44F#EVro(ZiE(God*7OXZKErn&?6nZMzPkZDX$TwiEE+~NnMWYT
znaueg$_7c*KIeKfhrs}J0+;#&Fg+}@gSsFeE(8=cLB`~@)P;MhEsgS0HZ#Ls|5sVb
zli$bwINQ`VkUEvkcWZU%Do|m*3N-k83zL!(I4(HgzGM195y4fID0+MOvmVY%LrE@H
zH)qih)ytgc`Mm+qfdSl%!ty{ah(liXephRCvEs`xVk1$t@=WLX%300aW8-g7s(?Xw
z0<@xRZMp9OXF)0xQ05%q>AI&HaHAhJ&Aan|qxv0Ll(Xmg8iL8h4^C|ylj>O%j-048
zw-jZvdKU%0dt72kfku*&*aqqu_XC9!@}9m3>r8eHR)RpAoea{|j6W^ZJo>yMJ-k|r
zI_vVyG*K)YF^j6$fz3Q7%7fDW`A1NT<p*2TKq(#rt{hcE8ExFMBvIlJtX_>&T4;7v
z>T5rO^dOBu1g>Nd&XWc~lLBns7t!a)7ubD)a@9auJC#mImu`;(anlDlzJA8D9SDQq
zDM?D6Y(_$dxFvL&xl@pwQm<=SV7DJ6*Pil+EgJsbKCE&8cJ|9Epat|#6gIh`j^et)
zcpySKkLQrR_>)Ac-=!FCmwFdUBFK*++!=kO23+KU*=IDrGkE`@9*yhobvR*;ID98d
ztNX^`KHUhbf&5YKmnsdSiREeVa&c52$qq&24}JKLWO+sc1$LeBQOfpL!k0nH_hBLp
zI;B6Iiy(s-aQOPo=|%SGuS+$HBrC;GU~BcqOixm4^!-iQ!UmSW4#kOfh(LUhrj`fx
z7`5L&OOC8#Rh6n0ZvZLy3!5s@$vymif&fs7Te4}O2&2UeUhY}tA<CYwR#)zRv-PXC
z>T^8GRnxlFOO;u*r)8-3_6PwTJ`LRC>U%Gy3g41MolRgqxiGewERr@u1XVsd<Uu2~
zQlkfNQKr@3z@-3YdA6X$4n;;3MzEFEa}PR)<<bBu_vtz_heCF?Jc2cq$wA&J7UX*t
z(!k^j4r=^R?Ig8EAa|!vY3uCZqj+#Mq$a_O(qGV6S}}gEYW5(DXh@OY^nOz!j0D`u
zikw^s$%Q2aszb%<JSDA0ULN99x`QR*=2u#`u0%nVU+;~}F#ak+1r))-M28H~N$Tsz
z-=%LCW~J$2i$pKmQ0Ty?%ea!vSC77;CL_Sc15p)jmIiGL{VzX7VA%18Sl5>|R<|Xg
z!4=#Sh06>~CEu=0x{6)DtG+QryAOaEP&_lX(P&{IM`$67-s;g^9KDnJ*=;tg^u91)
zbyT2_=ZnGDWtCalk+1xJ_Z@=3y89v#EFg0rbNTv>F!R7=ey*?_^!w`5Suiw>iQp<_
zlNw#FBozBs>o-st`jw~!$rlqQoGpx^CKG>667<^SSG*~Gk$<ftD55I_=rPko{SJ>A
zdl(Iwh@lfg7}zEd_vaiA<6yh!A0sS^GWvrSu_w1PE_NW;YGR%-$YCT~XGSin-q@_G
z*tt^KCA@k?`Psc*8lY`Aur_CbwhF>85WZhiAr_hOzFc6W)wudDdVAfNfcDXLO~sn5
zmgBR7zc8O!ox;0F2h}G+ovQbEKxPMc$oLiN2$m-xiyo@+EY9;);QUO<`r%$BdYFnC
zz<9T7rBYJ298>ySDHA-j+E(X6nm0)twRC-dVgaK4e?5-zpXWgkQiSFDg|-mLBPE*!
z+V1aLW6+=ij+*O{xIkhbbYL=13`!ZOi!%VbP+vrmW&}vE4Jf}->vn_0@Xw7CsIdP?
zyrdXle~j_K|GwYx69?ZbgBQ_(U3`zhUNJj}J_mBlw?L4#k`xS(A%hhGWPW{&-VOym
z4IhwX0$&LZTEPPCF3Gyjq15u9y+06y3RGyC7j(psg!A$MbKdQbilA(00$nxST7V8&
zJ`e&tTFNHm`tumTFVFYaeZWycz^0|WKMBUL0m;CwgJcinnSX9(o8bXjbW$Y*NWf%3
z$twKucSyi*(9uJhhQUKfz)DCZMEU?oC1hxSJpVG}6_k%qKtGGM`jh!@e9M3gnSiPe
zNr(UxCuPZO#e`-ofFP3edMKdNAdT<>f^2udpx%q?3z(oQwunPlnL-H#p1=H_2_|3?
z>=y=_8DVZ9RSUlB56B%6bYQ%&)r}Z(-~i-69F|__s6K)UO8UNeS`ei~avzpXEQ?KO
zV>NFJ*{uV83z7;8AXb5a96IL#?16?p_mA5MHv<-M#0Jw)M9c%$z7nLgNDzx+0I=`k
zNcE7g<KSe-K%M{F{lprWNTbLRhS>Q6SRk8d*rR|I&E8<*u52Ehwtyv+)Ijn6Kt9M+
za3TOB!7PiArF|=b-Li>ZV1v~doluwJa-n=*Et`O)&#9Egp_G>q18C(SNN_?-$e?2$
z&1QQ5U1Sol;M^#@10pN^3aIq#?ee{@6hK!JMAZ8;0~6>`o6!A&XAnl7L+L>zk@w!{
z>A>iBFIwRtvSJ|mu|5VRL!?OenPL55OYb4+0|Z3iZ?1t_MbNn+!+#c5y3c0X5Jtk#
zT<$UQk6k|O0GQyHkoLXT6_k98&TorYkgj)t_pR|yE{Gk5BveIH9=Sq-Ab|ZfDTZO|
zLukeVG8LZ@9VADDE`*W8JQrx<J=nGGzs~*!Z8C*v0Tbzm;Z_inmr&4^`z$emC=~<?
z_S*<QLr#tX(92-|(+LlWE&)s^h=P*~fvS@n*bQevnFEqa1{m7JkJT9xYF`Mr{LVP|
z0Yo+gc$4M{Iu}GrcwQf<u7?Q80Sg2~cJr}^V$KI5OYEDC2TgnjCLDi;zYT4f4R8S^
z-Gk<5&_o)rp<8$i+<iPm0bL^u+i0PQJ8%wm-Qj0@Z}f;1FvCM|-&`Xj00~K3f}cV&
zLy)OlT2&x9Nb3Q^i52<#qmS#CV51!`Y54bnCcr)B3E>1}Q2$SWlnHEyE?D3R7A%DF
z5JPPs%_Bh8B0us0B>HD4YOok&?xRKwikg<Zw~$ncApSZLOgAA9fTyF7;qN*y&>^z_
zY~XyFn*iY-D3E|~KtPOrm?h}m9V|qaA5Q88G+_ne{Tm1CI3nZ$CP1lv(U$|7;0HF`
zC!|HZ4{=IJ*Iyy`ea$UE_xWeUIf$JwDG!)&;GBjaje!g#6!yoYhGw#%UFc450Z_?D
z*abkLyeEAUvhH~RfHZStiyWj!4&*>8?s5pB$k46MPPp)Tu;3g>*n;cG59JYsXMpUG
z0tJ+?VM%~H&S^9fpv@&|C{Rz+Uw#A<mlcF;#KZ1C_Xj}#INZUY0p84z<e$Dj0Pyt&
zlDJ_*0wO@ZoQG|{uQtShtQ$g<5i~Il`ZfbglW$P#<f98%xU|7rLKAMlhVwj1C=ewK
zNl4c~?7Xj3-`ta`at$@?9xki!&vD~+B|p=lK42I}pi9BEy4!yY7YER*@dUXS`619x
z5-tqHuTLHzo3@zEf=mgFDJdSfg(3L>%<|?df#(7+oB-_%3A{T75KiAHF%mGJfPn~v
zXAsWiq0nuUGQ^MsAr<inZHz49F_?8y$dLgSSqF4QKC|3v0@xWyk+M)wf&sFJ>`+6W
z2GHjg5)GgYK@T6l<P|9qnBB;bP6QVJ4e0t28(vrV1NTG;8;DB+7rcC84%v+#U}PVQ
zIzWW<KX;>f!R(MoEbINo5~Lr2mUA<ppD&ef^QQ=)us`MR8ju&=fGy`~<xoES*Ihs-
zFq{2>4(b3!pfL6DKNtY|!M}#{#U}#_BTHr2gZ!}%RY&C%EF@nOAY@l3xR<yAKri_~
zwHrv}2f_Imc`2O=OvWL=vAqNeV?+r<k5$am0y*-y!Kr8$5Hc!qffr#20>|UR7$6Y>
zgv)izcA?i`lHno38-JiMDWz|_EWp1qkVD>H2@fF(;DM6jmoswq2QW*dLJK59<^>-5
zP4UV5|2g#kIrRTU=>J!QGLZQqqQ>{&n|+SGdp9p_QG3Tk)!Sw)LcTw-7P?Z-6uq*s
z(O^23%Xi+v;qU8h6p+&h+UI@ebCy|G5YUaNnhI#QyI1<zGxpV8rA;N#sLbXR=t6F$
z3Oc_&V<GF<4K}czlxdbPg>$SFkJ&`bB#p_Fx#EzB;hA6xss@4c2U3~A_OWeH)MIFo
zCTmhzPv9?hsvof)%=UDv#63!EN;ne2cOQEVo<wn8UKKHu3_w{KG8$FaXO1<NtC3YJ
zxz?Q|Y)t-fIRrEDCOIUha4Y4gL#n}pV>zTJJWAlqT%(ZHs_Xdr)4<BiStFakn!3LG
zSH?H7li?4eXR(Md%^lrPYx5nG{a3C})YAyZ-q-YA+WBsO`{i=DS~PY=5e^Qp4wlIg
zen4s`(X5DM(7@rKe6Vf05*sbOE21gt6G(OW2JUu0_mU&1orKXXGip#N(fP^F=~O4(
zm!ulKDs|HSWmqtxA9s#=rF%vCww<=%P4(^tKC>4YkvxIv_T;xykpiu%s5Q=8+b%ku
z^@wDXlTD%JX85Fp!1?b}Hp-hG-KvekC@AEI>%Y`{F4l<EHXW=w-9~F&d!psN$@5Yj
zGWd^{IXDwko)eVR9KNr;`&pecqxdd~o0?wcZgz^uwP=0r>#eN!=>g-AdER6farF{!
zOLI84f!I8#C2%B0gq0ya0WI@2rxeR4T#a-%!#mZf^kXMm-$oqweq5#M@EIg+6ntvZ
z%#o(r@V0(X`&~Xr=}e&`?juWZj$--V<gdHdXp&^TgU>>EbXAt>^qTFI@?>aHK4tR`
zQUSh6g)hrS3_qA0e5POd;rKYV;ImUj^9OPn&?CbJX=Chz+mg+EaWM$Kiw`vz&??QE
zq?YQWyc7)WoazNoXS~-JTywTY&^uTAw0lo@oeN(l|8j|L;<Aq8ag0y6*!D0!XEf{a
z+2Y0N)=nZp{_rQi@75`@Qqc38C^@FP=Ua51tKQK9?1*^_e-SvEI2~y*y!@_V;yZ=@
zrhVeBuC7RVp=Zw@R9abmq<+{7YKcz^6|$Q=D24*c(*D8@a5$(R*yL@ghDo*(r%!A6
z99M7U9}7A~EU($V7!JJM^DW6W%nU=5rNe2TIV$wL7?D4mUoMGCz8W~GF;cqx$RhbL
zo^a46|Hs`=&!_3kO?V4ZXc$NZZKToid30z_(&K4Yden0R`p>ODPwURUziF3AO4iZ^
zYPcP+BS=GseA0F%AA}2#5hU@}e>m%|=R3wUT^;tb$}7@79B?!IX8PK<_v(<^jBLfT
zt4G;)NH|=`hGZCCOT}I2u3UW9<BH&7BX7<Gom#&B`#b5dXR0CH<leWnQi2H_`f=Bl
zT54Z)m^r=|k;^9iiTy6It(a8>dxXotg{rGwe5rdnU6DyRRG8R%Yo$0jinng=D~COJ
z+*(QOFp6IghO^Fp9Iux~ABtNV6ZpuoOXLCP3%D=YEy9|8snw!RWm}OFl`LBXPS%q4
zQiv->rQWnhYs_4hs^L5yrR1*tn%_N(8MG>gG%*@d*{7<~G?|Zlj|9PU8O%fD_^^EE
z8M#tx10AI=#=kAtUYtjhEDsOhye`&KU5Oa&Gf8hCz&0y)*So`snONu#)-3wq%IHFN
z!Q3<2EkVK@W7@w~cj7Jmfyh!SbjBs-*JEovE3ZqR#-?ud;oGYnBdeXktA!`UFIW<9
zZw8FzJ!#gQ8#}Its*py*pOK;&{c3kb&JF{~$=aM25AGo{f+(Kn(au2m*!!EX4`<U}
zWhQSW;=vPg0`qltu7iu%0m3Bit-FVly|7AUMi(!~W=KBRyq6ZAK6vSL8Lc;lRD=?e
zm%eWO?8|P&?ivw=K2~3KS0XufI(ROdY^g(!AyrO*=2q?a?1W2zta^^;7sI48vs#gw
zCa;URMXuvIM=37J7JKRI(gYLHq4qhFDDb0{8dcn@0!z7mm(k~6b+61a%{cnc<lJU-
zUVlH=SSv|)>Q2pSq2%09snQNgqf&Pkcz)`TxtAm{ps8_rWE)Vj_vhPpndG59_BzxV
zyZ4`Judr)#=dgIb4F4S3sQj?J*z=s1{DshELb!^B=aUfvdNeu5#j%ybtChW`yU?;#
zBF<%*+V{=0gNH^<`4QYJ$MrH40;e7)o3W<z&Uu~9w;%Hch`g??wO8zqityZ*GmE0q
zUSz<jjv;!T55jSUUi=}-QkmYgdDX@!7~WT7Z=9x5Y+sOQlY8PKc(*@3b?PaBH~tir
z;o-kevqmwIgIVxAw%AzDSMy}QI9|w~C0L`g{hY)`(u(J7<h1U_-BFzF+3@wgh;@IX
zPT7$6=%<akb;C{e?s(QPBYTxiRnbP<C?kKm`>rw@72O1RFRtG8OSuhp>1UkzR1XM$
zk9)oyD3{9{+Q6$jSzAk=Ca<qHH|P$(`5fnUscGDsv)Z@$`s(SK<Fjc3lU(D{aN?RS
zOGB3ZcIr<sZ|Aqw?~d-;>qoGz3f2g>-^~kGR2A&%W&25pePUw0yG+g*y#DC#dwRO{
zJ%6v^t^t$0Zt8Oh-(cFoK-2jk^0)Z3rqSSU<|23R2h;B0wdZJh;!C6JoU3D_<=<N5
z3O1Zxs=tX5m&kP@zU~j(#OmHefyWl4L;kb+c;_sBCk2ye#=g^(f2sEUs)yAZb-eXs
zF5RF^)C&0>w~Vr*T3kUkwC;Fj>Z2z7+L22FuiHvCiJl{7Pn)zcPY=#!T*vq9VUERX
zWfJv0JZ3~sv~w=ryN`7pdVl_5Hj6t``x-GA|EJdFXVM|dItBj0(<)qjP7&g?!$w+(
zv1>vn1LdRHq69}oMY<MY)WW&^WKDFZU(L!ZamfShrrKvD{=Uv+KQj!#o(6%f#Lbl6
z*9RG1_gg+HZdGPcSguSC2&1rXuCq0RG9-hS&c(kTKPrvmu;ff=PvGn}isqF2%<qwa
zPmAZNOpWg_yfHXc7k~4D8jZ`8pSw0x$6+vX@ZF_Yk6!gI^KfTaLuMGy&do@6XpwWI
zv(-?0N%&Xuf%-HOmZN|z84B(6L8O||f-lTTKX8yzQ^dC_2E5;ROw4&-rPGS#W*OZ&
zUQ1Dk5-d&rDt%zEQ@z*!JP0L&zFvSdn#67DT8<K}?&MmCEh|G{Z10DC^mhG#(6F7+
z(c2GRUFgQLzh*eZE{Hu^G1#aiwuT*XR0z$7Hh8F=YGz~(c#b1hS+{0>+Hf9ipupc`
zD!={n##JjJ?OkSQsrQ}N7fcg6X`^U*wBsa{?`A2(J~NTnUeuGsj-l>467>B7SHH)<
zvbwDgvmVEp^&$@wl<Mf4qx|+EAm*YFt+U>1+1q7($;w%AnN6W}U?!zAyr*b}uxxs?
z9<=x9#qbBCqm5mqx}KO^5iS>p{>Jk?A~i$1P5dvZ1=`6<&tCj4k7>B|TF>u^{(Ls{
z`jpDw*Xza4EMdr9tY5iJ8DT@gWBiAFY16D^b)n4sExpXHwsp#6PJ+&KVRBsF7DBop
z4gR2$bdqzoazO0CwASjZx1Gg$_j+1<OYhw-<&hvs<H*bD*G%c=9`??30dfU27L#6k
zmVJwT(Ww?9%>2XE3Y<IGqYfNYZ`W%nFg@lh4KqTePd<gH$nmHhC|B%azYU*AQgh7;
zkq6HLn=g4!O~xly;kkx$%U_=G6ey@MzFX6Mmn2@t<S^1!VlxyOANOF+;QiZQ`iR0-
zLmMxGsv9Pbd+*9teP2F$3t}U`{WEI271b=$7ONETb(5>?Y!y9qa^&-BEAS(UjPvyy
zGDXFVGv^!W9S_|w7JT=si=nfmQ{cK<<t5@QOVvyLcjcdIPEuZ;Idy9;tv2+n2XR(b
z{nU-BG+1z$upW57I@tA;DZYnUpiqHReS71$DjjErkv9o5*@<$|H)vK$6R*ab>)l>e
z^g5OR{nM7_?r#Pv`(TgjhpWtAp6Juol0joNATT2#$8+JcPQknKDlag_?z-{t@Y7b-
z*zipoHd5DPZbyQ<j|icsgx;)ILH+_)Z_oa8yy-&3ouQ|)Al%^Ji|uD8wdOx1IZsFo
z$sXpdT;t|E*%DF}4@<M~_!NJ8E-!Dped9!IH);^S`^L~RzGo`-JjPs?WqLI~v%S>>
zRWTa;4ugzNoSpI9<&RpHY-zop!*OT-Q2p9Nk1k5rka#&httu-&@B^NLVHYRjAN@gh
zCZh@!8RHEA-~w%DV>xGa@n6!*8n)-Wobxp2w`QS-vElot+wX{e4a8br-MR(_$q7wm
zE(k5Yh@2lj91yfkk@4O%rtLZ=6zF=lQ|CSExIPybPV=c@>*dD8joX;Y(S+yc#`66j
z{k|k(-KnSPN;9#YeS0V<x;+QOi}f=-E?(ql+aV7|5x|cINWW#ay`YEsNvE?|6I670
z=hcv{t`3`_0;1B!)11V-yTL22`uO2nO)bL=W!1v5x?To`k-(<@g>d<vl-eC$qE~Cq
zmU~{?i>B5Ijogmm(~fgnZ=^#xrx%sU4d>;01`%^6eLl%=U?K5ZORrtJa$9&O9&8IW
z{60A8xNEdsBye#49MrplEi{Ll+W+)Qht-h9bn}730@Ycf#EF1(rNB=8o)@#nlZtR*
z0VnS<^D=d3Z-wVF9>d4u>5giPGtp_D9^Vt6vu!l?tc3L3Zb>oA;kC*=9I_CXsG6bB
zvb6~CEitp0J9w<zzmZ3@ErULF+4@%ZLl&PR)#6<TiFYW|<FQ#UujhkVr*4KBS@~Lh
zulthXOxFBj)bR3|H>1lH#nqns`^Fmn%%2BGC<x5CZYFFX)<x*n@M=&=*=Bxe0Ly5<
z;&k%({W|IW-Hz`|WjQBY3f<l!skKIui8ro;8-%<;*R***3-dajKS6r8?U=gGfA&;<
zeC#W8aH&vgi?HkX;0a8fcHfwz?dMY71=|?`i}M}kgm=lD{5q0^wfIpFKNwaE(0>Tz
zUlCMWHoMk2eJbDhO3&R{FVcNF*>%R<S7&%j^vujjJ<EgwsV*$vrnqRs-K;Qv7hxrL
z2<3wY>`GkhHruTy43R1C=D2q6#giln#M+X!-)GViElv*@WDi>AgT&~wCw9Gh8?M^*
z>C|0#COtpj$b+9Kg*IH``(~NGo%r8o02S&(*^*x5tXCf1P*-kPHU(M^HqzvhzVQ`(
z@AYXYZ?(P0*Ut<?{4=g|0{OCeBt)|=`I<hS6I9k>d3H6ZDpeS+)-TBoGad=g^hulz
zjnT3k(5AK+EbB6NRl(;<#e$zy4LGB>tEcX}TIs~|)XR>Psv8%r0ej@7B{G_&v5oSV
zDi3_0;=HKE;M@LYok^5(sdJV8iTJ9V=c(~!WfuB`<>=S6iZ$=PeBHJ%y!?5Eg-j{L
zC=@ELST#Iz8IJEsPRqvesnxEb)0@fzuYv_of9h6#-HT_6_$(BrHO8?HcL`AO1+6ad
zRrD@lwsOp(vW36zt+mR;?i02o9HBP)#_O-Q;&>+5La{;Mjt(h*S-Ua;)CR8$9`SMQ
zn0|NrBQWdieJPVYgaThB>zXiVK~?bSlX>G-3I~E~xW}MOm}b-E?Rv>e4|!$9mW-pe
zvyYQs8-DWM&0rJUEn`XLu>><}yS9H!MOhssxw|;7sOaN?^SJroiiA69(c#6J$nsk;
zI=5cen0VNky6!eRyDO|nPUv!HOlnX!Ksu_4+c0B|X?^nPPS~C0&8q-$5ovlfCpyi(
zI^EtYp_RLmDAa!Xi`hGyy&pCeSC>8T)ebKJ$p_U%j<uN8@p_)w{451MYVNNoz^drU
z?2R1UlGsz~?P}FpH63sYjEVFKSrD8e?i+s!;)GgHg=~*+!>DY6Tn%fJ1iLn}_@A}&
zgk!Q5pQSSo+E&AACbf0glveJl3hf<X26&`C)fWG83<E>34E<jKBP)82xEPjSRM%#<
z;3meigWUU^<dEgqjWYaR1_V6UP53&{`g~__EX_++Z%;b=K^VtGakaU8MSs^=?L`}v
z?sy@p4u^BoB{x3TYSv3lBi#SP-dn#_9kuI&0$aKf=`QIKlrHHOkdiJ1X^`$zQd&|P
zlt#Kk=|(yQq`Q08hxdGE&NXxXgE_x>A*{XP>F0j#&A>2IqVDp1bUq{|4N0}2EQz8V
zEwM_exFM$pAF*C?vx4sY$xOLS-|&36lO~N9k_P23pf3olCX-LS`j_3Ae4R+fUs*93
zZ#%R!poQYGMr*cZhW&Q7_qYl5`!^Z1OdOz7^_{OdlKr_uuG>lT-a%Y1AL3q#VtvHU
z-^*;p>5}3%z}6O_BK?LX0^z7F|HVy$0wt(#BI?m1_HargKBtt3C`Jl=v~`?H8q5Y2
zA~W;mD=5!Wqu4Yj4l~1$)eiKtdF&>WLS>#T1yZ!<P1%aXlRWfagOj+0w#Y1AElaLb
z7EJ|pxj*%_!7F?}PPQ(ev9LSfa$3I56ZJ|N-lnndtYT`KjhlDVhFgGV)9B;aw8=T5
zxl#UL&}2aC3GHf1upp7w9Ec_!FY@2;)!84!nu}E_T{2nrhFOKCfBz9y=6u_EtbESH
z`NxBQW^{yaRdJ=55zI_n9^RJTOCvKf2=Ua)#ee?iW_`1`I+dl{bGCK382QRwEx&Su
z%C1*OGDB%d)jo5PXR{8~y&&BN?K{TS6c{*c$}<zW)GFP`dauWxxlRB4*YCl%c6Vw|
zfj#|ZnXMBG;yw&FAt&Y7$n*_3UyyjSI!*X4I7%luoorl}?mxHi+c?E%o=7<+{x_(b
zC4!O7na!WFzvn}qarY(Fqs{eLO&Wt<xwah{H_hGYU!9AzvKm_<hl+r+hq_*B=?CX+
z4J!~<%nTc9l|f+zzH(!Eny7D}U)F@g7bT$;Hb?QIF#YvKD~TbYAdOOnl&htGjGGqV
zK_)UIbgeJytKXaGuP<3C*Lkpd-@ER@*}5H^9<AKJA>yuDuPZ#ZJp<!IZDvetKmBqG
zU54<?PP)<hUCu?Ils9<(s_3%NdAIUh>7OT@wc++#H$B2)Ml#?GDn`gH@$Kt{|K5GN
zPgON!QaGa6{qjaCS<|EKbPlOX!v$}2ETl?PSDOw@lI1jPXt!)*D)&kUCC+@4B^i<@
zh+=kX{i=VzT`Fi%Kgr?P{q_272Vzy4wb5i4DvM$<=E5I7oZ166t@3tJ5a*xov3`ux
z7!3X7u~i;9zVS2dqVTO54x*0cj<Gd9=hlyk%O1Wqt$W0AAN(+P<LZ<X6+yk@Z$pd*
zLUpyqzZ}eKnDN7}l(ri3_Ippd+`DBN_V&$r{9j|yj6MF7yv1iF`q5IWkI(wj$9((k
zxffF*husUDcR$sv*WX5~N})?JSMH*<w^oN5DvLrTHe#*0iZ~G2QU2MaU^CLQ{noUI
zrR-+F`{81?>4W=xRpQORHj5`SDD+pNjLo$kqYNo>IcQnVaMoGxiEW(u&%Kv0lwT>6
zPHfK@ju)y(q{f<>3|ZM7zgxKtuk0e^{D$QAeiX5UTr@Yt_k9x{L(BWT*3R>ztxA49
z8JAsqk6Gtee~Y+v`i7}2WboE0Srj+15p`^KsH_fky|!*&$=1vWmL6Xe5RO`$y!yKt
zb^6>wsNfkG!TNb#cYhicmae#T_2cq1Zc!<poVFu1sB-2)G4AJNqx62rK(}AQ3Vj-X
zw!pu)>#20#ven9g3F61)UYdz?==ip2uB&$b)g)?K7pjz1nB3k!WobB{3uuZ-Gx=6N
zBISO=hiKz%eS<}G79QSm&cltnrmt!58RHmpRNOJh<YeRT;q#B#F}E-NTv{?yU-OV-
zhKTCrRe?}0s5gI(ByGIhIyKjcPEi&)wcW{*X9y*aT<3Z%Tu-ST^gDi@oMl9vdY=4E
z&8I(cTl^@ojAQp=gr><aPlG_qiZaKqYnKJ5Ooh-@QvBT5;AqV9@^YOgkD;=z(nH~<
zr%3<Lnk3&0!rhp1%U4jqN&2o)vk)fSOm4kl8K6j#xn=MH>!f1Mif$5za3(C}S>d;&
z7Q?S&zBg)^@1|dxG*!;H?k;Q_aj(geMDiDJr&eOhn*MzQ3P{drj^B?REEqBn&3_vH
zK>A$!jRrI9Y3Xw{Q(~U)tcoc+hQuY3rk@r@Rg)#dGzpDH8r^QGCj&ZB5%iXdBh1o%
z=VhZe?A1$TE?m8POztz{Vjvxnz^coDK|KCEq@ix?+vo3c(yC9{ZvKSZb+){?O5e1R
zBCdS3x!>|4#YI)wbK;`2IqXw<R8wBa<>ptsE2XoIkB!_I-yP*HP5OGLTmr>@tEj2g
z0<mo;^JyG<?)L~SKp{J)c^tVmn~`ZE-D`_@bNH@>k*P4=baL#ly_Icyr}CehMTal<
zAh{JRt^{GEilA8r8NnbBh2|Ejq}~($^~(U3=;ldr>-3gQQ}R33K!Y?%!knjkdK@Bq
z>6&J(&Sa9xDrbt~gctPcgj@dyU2+RLr%N@Vj|cU4IfXBC5@hoWE(VMO#Vm2gHYMr@
z=|^;u$K`qt#*hQh5%TNHK+YP-b99kL26{d_cKH*nxOU1`B?c|Qd|i}fk-_tvfnrmO
zk8XyrfnyIARl|Ux17rp}Jm{K!Lu=Q(KMSd}wy>@gD##B~OA-Rzjk8X8d3Z(8uG~2I
zo9YaHFcAa`0R3eO&=|d-lu06lfe)g5g+p-z;37*IT1Jcjs{p{EenRD-F$-Zm_#0nZ
zkbxh73J4|WS|WipgpqOS5dlJn5?Qqq=^k2%;f)q(l23s(43$~RXotZs`C+mTY6B>g
zp`Hwc+B<-NR9OCjpTt086<PerVH<>}zWaoq0D7}$V2%G*sSE@#1i0t`I^hY1c9sGt
zjf}A&M}RHqc>rw>YU6;`_WvI7e_!!`1@-^dMF{00Ovhn8^L%rvJX`(8n}K|CLtY5g
zv;;H^#*p9&jT^LT!k)5dQ38gUhUf2wMfaMoZTL>d)x69^em~%Eh#}|!^=A>FUsbT9
z<KwjvX^MD2;-uo>=vZ9E9UJU*bJ*no$fbH4-HulQSyO`j%I_XP)RZcVSMVPU@Erhg
zENK9w3}rMNJcai~=36lEdITt!dc%phb0k7Ai3<KuXcQO{12z<NGJbXHFQIdu#{;83
zG}##qnj1w(TLDNLK&v2#h76@W09u#DwgSG3HNYwrje<un(-DlJ7<IvzpK6?6u3Zg{
zgN_2~9-7p%1rOVfonZ;AfeH<E2r)}32+Yj#C!Tb0G$Q(gDU&GK{e>p8fn=W1CJ&do
zwFKQ9%Rk?Vs@r}3Y=(Y5dYw6(D^E}d2&^Dlp*u%d*aM6RfLK~B;5y!d%kze>FL;xW
z(R4bYJ6i8-_4TEF7DR`1Mq=|N$mi}dAD#GlH;Y_}c4gjIOQ*ZbbzM`cJa80jWO67M
zGyW_M@Gl9-A7*~a+O*t$9P7}_mBX(Ciw(5Tq^72_Xs_N|wp?v7cc@Ih`)NHpp};<4
zT{C~PPY4Z<VV6aNjkJSGG(et+z=M(LU?RGU<q(QhK!gy_Y4TeJaD&{ghXy^}_BSPS
zuDiS1mXyXt#3X%&fQTO58_%r!j`aZ`3jzB4d`v>tz#Bh?0j?h^Fw@UH8_jFG@OH0Z
zU-<oGN!sIt$3uY{Ki>54OunU2%U6mA{8hb8=L^hTWG-(HNPp2lFQg42Dsf%E1JKxw
z;x|C|L$93v!M)^#&D^uw>+=^vGx1CsM>QtGWp}$Zi@e#~$Lm9hfcHjODuS2}42>WA
zttCODldS-SiqNnU;0*Q?1OvnE%z6zU<EJYv`Tz@-iF-*3kCl05ZW|9HJG*i)76luP
zR5&)i$(5c-dz*b*1YbIUbZr8BkBo?kB@jAzaYG;<T&CUFmEPZ!()hU?R%Pmz{L$e5
zrisjJ{e1W3t=H+FqZzAG(C#BMeacYtVR|%U(C`8N#{ztwDc>VVb}j;10g;pK*@Qv$
zhp%tFFW1b^Gl_XDFFG!;wfG<%o2@C`;15g}@<<+w{h?V(4nOoT*6Itv3Kq(9pcyA$
zSOSZ-F{}+51qRdF8Jk~hz<m27q0<)m?a9&?R>Jg($+V~2lP~fWG6134oeTN{vd%OV
zcvp#Ug#c1|K;U<C<H>FI4q`G`sQ2EAe7V*ef1F4|_48~@S%haB<6r+*n%&u&L@>)Q
zzw$dmLgqh1bU6@ai<mtfz}mz`4JV4U|EUz0w%q9`@vS`>EBi22W`rz=6&a>m{ozZw
zNmnut71NQEbtq~BDn1jeOIGj!H_8dbq)U_$6KsRskH=~vsiV?~WE4Cek=Ssi1kyc!
z;2I#LYLST|7h0XR$cPs@pBqc%9?4TA>4wDBR}>ngg{jm+ljBjXag1s?${E5HHuL<O
z^9?oHjXJq<@dtj$1zp@0gK;+V^`A<J>UJvn`g!xKtR^uFYHA^%;5R35ow+MgY~VU6
zVa@k<9^BR&pllTn2&wv6*cPlxo8bSF*>vI#2W&XWjckl$Lj+meNZHPNbJ`9L4#pHK
z5Q5fk7>ccUPhMjB+2LJW@3*W0U;_7SB<N8NkVZTI!q*&@L2t7<9C)#q6dSt;dK!Fx
zLuvX?vV_j2Xkh6NL&JVqAf){KuLHtS`oi?RPBk2#P5k{8HUMYlGVh%Dq2=}r?&L{g
z3{L1P>Z4W!tfXB<pZi;vy1n{shz~E?x_X*k`)SF;1dbpu2TKy+0YKvA4FNlaIawQE
zB4T6J`N<*l%X+rj+4h)793F@jFH*M9Q8$gb{FujHowSnRVJCJ>s~Mzx)W`@6VDyRd
z0EUW++CSsxhkzufub)2c_O8_z>f-)ty^ai^qd5RPc3(Fo0YzGIs`G@lIazz8z@An`
zzaVtG;7YL(;ZSI8z<QrhJb}S7w?DMV@S+C=*v5BJc4K*pcq09ORPd%6-72*mv#gXs
z8_ss(8)VQ)EeW=2;Mb|J0hBYq?iYzI6*e(E;Q0yxHc;kw1PO!l_5_c>a4Oh$<{^c&
zyz&xQ^to<I^UYD;XoZ<PWIIV+m;7?sqf;SRr6+`6jTJQlxH>@M2643Ujc-jF`0!*u
z?g+t31a`1`_3;5Mj1(RsAk>b{+E+c_n>PcJ+$h~h1f=a7l&F&Frv`}DA`g#|l(<^7
z>yo+E12Z@v?hO6`=<3$Oxd*#n{ds_HQm%lr6G<xtfj5|hIoVtksAijP3}>3#)Ow$7
zj0Dsz>jz-kyA^4m*dN%F!BYWShpbv=j}I?Ki`|f-0B9Kcli@$cq8R5??#J%g@Oa<2
z)@|mc^zXf?56-n+Y~Hbnza55Vv;G@&yqg7X-ly5;{_bcg0ITBO(|jbPnO6ua8xVLM
zjjJhtJ?5ZSNTLJ`LZ%yakAMpOKM-VOd^n-<Ghpl!yl<~QrE38WDGDM%R}O}W;4Rj&
zj_E19pX|*S#&v*f>ZRCh<gnpmaE)LfCE&F`4M3eoyTLDd9!Ky6ZUJ(;M4HH#?JUUi
zfFq2{7MVvJz?p%d)@aXWS-t(P@q^xF!7cx-Ll0d{Y%Icqoo@!v(F471FfhDKQ)a&+
z1vyTIY%7R~4iGAK=O*gQtuT9*`d5*Ib<WML=3ExNG%*EsI2=dM^gVXXikk?*RGcU!
z1mCA-dJFLMMN}M){}ddj#$)K@SF^&{V!qHw%k9oo6_;a4P-4qJFh_2eG$>6J>ssVB
zY>KY|#>GzIb{9zSCx9wzkr1PUcSZrS#O^S_GuaJDG8jRzfWWmHup!K;94rvPfYGCw
z)b}VZka;&=@MXHE07M?mWn?q}NNNY{e4xOcD;C@~@`vdP$-Sm?{kr8~5?T=v0~<g^
zGS>ZFCDY%z+87)5nVQ3Df`z+j18`|10F$c+IAUl34-E?w+BpQ(HH`<x`o~2fDLi&4
z-2tE%lmkS7W`n7Gqs?9(V`DQAQIGTmCY%9|9o8ZV?}wz-fB%xJxaJ#N)(?aXBymir
zu*tzQ9dr^aQyH@Q0dJ7<1&)ldz@Vi$oLop}Q1Dp6YWEiha*mwnWOo6jD<BUv1D1wc
zZAGWbGTxH&VKRsknn-y6OMV%sfjtajd~9+*je*=ZM7~(F;Qm7$5S!F~m<#b`M-c@4
zJfjURw%Wk4O}lHl%zjtR4rcdidk0>~6|B=1#z-#KN&o|tg{Fb6z`)WYkvT<gidh4)
zE%pyxGRSd3t-%;1{m9c7hby_j!<z55@vWz-B~U|jbfT?z5VV!!H8K#u*YIv`ZvO<~
zcNvS-3u89swt<=ddO1nQ8n8}@4TA?{AUsx<=0oYMU8IiT+D?mrt&p-d7z=151U~j8
ze%T-FOM%h|&}rsZoi~@qYekiVTtd0^dA=|(SNrVQGr*p{&Gy#)bA|U7A2L(2>;pa<
zWW3Su|BfU1VdD;;BGi{&cVLOI02Ub@p=>C6L6%+Og5lz6l^X|XdmLP>M9Bl@u?Veq
z;NAX))IGqx6`|{dmoEn!k)g%?0w_C&L;V5U;F#xD9W0(%6A>W9qkUURY#WWF{&-(3
zBW=#rRq|~%fa9U0uR_#2aQol@jTUmBz!DNmfcJ+hM8FFKYk*EwcMM@BOA>7gi1IJ&
zsh{s-r_;89O^$|0MM0dzmT2Hz2WUs1#Kz7-$LR&0T!q(r6?pQ$$Q#|hY|qGfEaLzl
z=@`rN=8dedzv+Jg<X?pp;2@tSjRb(lme;AVVZ~!mit{>~&W9cyMB*xzkS)ljp#h+8
zWl=2KQL^9!Ha#XlI9T{D%A#1O>MzC03veiSKSs4YnHKL`QxGISXx36&KrRT98RJ^B
zq3y^)+u>4z+{Y6!(IC;GbfGnfW){STv#GLn8<`<bzXa$4Ib?!Ol;|;oI4}{oTLWU=
z56|2Y#k(TO)ENCOps@PCP2I%cbPo+BLV3hDv4DQ@l@yzL*{ic%W<V9IoDj4P79&FT
z1OzwcyMy3zphnLvkL7p)I$aDt2s8fMa1Gdgl*rxGbCrJsDGY#0v;qfNV8783mG0mf
zxB_B$rf$Gj%KBo_v$Cfo1E`32bRr_X8wd}?!Gj0c9e`BhS!64q!dkuBE`2c;<RsUT
z#A%WP;@@9pD`3iZi4!iZeG;S&Zp%Rg{aLFZ8R-TbKY#66pMa5=!2d#nY9hydX$8jB
z{HaKZe$z}5BVbA_0Qp#Pcy0{fsRH9>_=rO8CmdmSfJ2fRIJXx9{_(jAM<7XAhK7bL
z<_?L$xTRLDUFI||6q3Y&Y~-so-Z_YS;(C{ifS!86oKg(ui{I@`m76n^fohukAP~<J
z4VfNrS(K0?BzlwT4~C0RzR?N2-2}l-EC@e-dwv8Qz95lnD)fcsY?V<qvR0W(8(%wp
zIZ~A-GsAp32i&Y+aX>qiG(rdpXr9@R3uk!5??*|!oYT)GkZSORk-1D6g>c2onuDF(
zf>F9qh2-b5P@=&f^DX1Ql8yWp2Phi%1j@LyePARAYh)-h{)Ot8G@2bH%mjpsTPRYR
z!t3WbzA9j3iGBmmtXUE#>~+;WUZ`dU2pr=9{jO#t2)}+#$O5zG+X4=CLncxb;%fl~
zJf=shJ$V*DWFr1?(9L@lyv~(N|05;_PX7<35hPR&r++m2Ad3NrHe*$@r2wM#^ugHy
z{{f7@76mnWpy(CKhBmDJ;}UQlIAmlHt7QK>u`8=QL~t%V*wN8Zzl|hFHaLt8T-c18
zROq|akYG1p-fOAU)rb{T-O7wQ5Q(@<L;1q~d!q55`-Ag9z5!IJ<|j*lSvL`o=1rwa
zSKQxS&zevs0L|A1IdV6XH&ck{8L}XtF*V;mh8TpM_8oU+i@<)*(6gZ1HGzS=i{9({
zEFKv1uiUaHC#!(s*rLdr2^7b6fUuNtvuQo{`-!7fVfqON#%<n@EfglIEN=}`!I`aW
zLtu`T;GtYY;HhHjOH}~ZV+U6jJEkNspkz`GM$@{H{q6yo{QdnvW-8UA|Kk4MNkeH2
z|3lJ6@d^z`^;u4kI8(fw98>1c0v7yf@iVW*$(`$Dp#~XQ@i0I+tTIs%h^jqizV73_
zez&-pR>yt5Q0Fozz`cK?l+oYckh*r=e4ctNur4?x>n!=$h!PIhPRK8efW0@GN+L%-
zku8Sb*>ZEH>c?~I8THW|*_Eq`N?%%0ct4aPgBJY-z^OM4g3m@kg~nyvi3}y@@Qbb7
z;1^2hqCW#tvY~{>G;ml@ezFnlvprFK1VVzogXIo7&62n2$2a2ewAiySHobaUkQK%b
zrU~=`t8)5C9FQ*4TF?9}#|X}4@FS!VHNrN*O8phV3eTXN9tYH<2RLq8m%TY1?8b|~
zVdanga_Qi3Ww`tRZ?6XN3mX<Fm-T;))(hDO*TcWE%^3tSk5FE{fI|(~@b>N(LV?k8
zIB%)HAIS<z<#(22Cu$$fJlPyWYD{O4@rB8>0)V<jhAi0ATB{$z>mP4ep@h=3C*~Lb
zfHE#6Tq5o_4!g|&vvT9a))Kvj9Q`Ku3eQUi#y3St<cAG_eb<BlSzZaAHkt(<2vw=m
z_JkrL7CFmTF0&Xo1VouC%Re#r1h&+0xS^QvPY_2AQ<LOzW#U>vQk@GZG2KnExDmXr
zPQHMuiu`x=LfocAg6Kym(V~>_fY5W2R}LZz$EB0&2Km$)V1-6;;){v}r{e^td*<!h
z&yS)7gWLy*RuY>$&gDQ>Yqr$t7sIN{yZu~57Mzb#61LQ8*B*#63ev$#-~n=h0FQ#K
zu?~<Nt36@2r3YJM0bBcWDwsrqi~PBDxg#VO&~|cxXsahfq=mePS-Zki#s0t;+y<@;
zlix~Llzd+Ts~O<<bpXk9U-uW9T!pV(vQ24|U%=L0f~_I@i1w4CT+lpn@jrM9#b10b
zd#QJwGWzvSn>TucCO~Ltv0uY8zbP87RuFL6i~EevY$*_!%I6sT979S9o_0wP4i#bj
zE4L)B42~a&7&cFvLC6-pp6;Op)ht}*?4>{dT^)E>hs6(wfc*X4ZLVgizIzLTr?D&z
zuZk(yY^V}+<(G3w+k~_SAcNT)2r1(u+Cf@kgc!g%W&9<f!_&fFpY6sXVi0$O9CED<
zi6Rap;#)&9y)?%_W}}~jHDp&hf5=c4!60v@nEV`P>PaI=VrH(f=a9G_o+}4CiQ6J_
zfqM<`M^AcCo@|UrZ~=m4#%U8_AY?J3zU`DK(XJ>cfnlwH`>cp$N;2%J>=$aupL%T|
zCL%4_;m~`y-RM?ed<vWJjXs&2?M$0>M^nAd6c2*!FU#Um`gKzQ<=`9g=c58@hDN~l
zE-@QKuZfUDIG8R_rIC%Lk60)6rh)^%D?@hxoRloEK94gFbYT0E2gaSyrBB12p;KR9
za=38HETz~skMsq7+?k>RjP1!#0ni2{6oj;G)R2gA!?`Mxa2f|XZH>!vOpTPANbhdN
zEsSMd7IN*|Fo+s0oiX$uNn-ec_(le3yBV+;F$~J&LhWZa$3xPJcVAVZ6CFq|ffB`x
zl;z%BoqLz*PHH$=91*w@>ZnISVyi-U|DOFuHk4NV@H9{{piH<1Tno5WmnRYB1b)Bh
z;1cnCC#hZ8{Ry&xv9;37*RC{ob}Q5t#0_PGUiP#xNjDLv)gcqWg?m6<ZS507`mg{2
z0gKkKS+1!)q~4T~fbiTR2jq1+Pg+e>K|GZM1Sp0{gNYi_$wy!roCEBnpedTx-<)UZ
zeDVtLv<DP$J4xu1eS}XK^&4x?YJZD;xxY9d0gngvT-(h(pG0>6xMH_APe4#54>bH`
z8hNEHJQ}b99BxOZfI2ZgRoF|1Jv%V-2ry+!a{w_lq6TkAf1tp7hTQt7^!-@g5lA8B
z{NNv-RAhqqi0e$WtP_%ELz<;+SGe?Web?p3v-mf1Koy<<Rmhj&PD`S*+Z?6aH9{`A
zV$*Ld&^t2<WC4bgvoG*519);c19)I8M*G65Z5Nd^2b%3JQedRuQ<t#c(c`7Scx~ni
zUR+LtC^wE(_YGPgC>sB!M^8wGfm_0*GPK^C%9h4z%A|SWq5vaQjuj0>Q6UT@e3nfE
zhwVE%f=R~T4bpO2kFy=9pjW7p3BL>p{v#AxzegX+prA6t>+GxGK`X+{%er?XS+cdU
ziQs>80>PEZZ5-<39!MC1Tx=#`30Sh3Zo-nwl9oGn5zQlPXpv}9aY(oFUq}mhUX<yZ
zkAtkJ2h^#=YYs@kf1)C*5av80x21y<@B>!PlB&$K2iFZmxTYZNdYvPKb4_;xE+3L_
zTECXNEBU2jWT~F%3bWX@03>-(rwa%VN(3@Z8w7Uq`q?2du8eOh_y&WjKGlA7;Q5q*
z{Ln!r6ofnsb_B}6jDbzTBn&}5GGlH9e8cPS>Woe5pk%s_PLHZD56l8GFmtaS9X>~)
zeFfEx+}HntTB+=>&-hiWVv(eLxb;Enko*DVmt8tq>zi5l27(8Ny*a)%ixgh#lWz)K
z*{jxI%e+7Wc?rQo%hK8HOmQ|b4uNy80@n6P8qHj=wXNO8NN5=u%EvIUE;R_)Vu2dS
z-rZcDHgf>CuOJrQ7!Q+q1Qh#%JX#bRR`Rxv8joIKm_{8cCOk!&%2-OaDnX%sst=?I
zx+7G0;OzfY>w(Ao0?G`%zHkVufNdRBay&c$W*h_VNJ04HQ+_@e_{NF6CgURgCLOuB
zXPNnxATHZmef7)ZKL-<kwX`!`32!S2MMc1)%4Z0B<p75bl<PZoZtFW4%8%gPItRgK
zV`Y~>DK8#mWl&;43pn9thHF9E+!G#A^PkI3qC5a~H_Mb#z^LB?LiRi)TT}wJ57dmm
zWkGxaq<BWU&p*h&wLJ*!lwJS{i#!f|`-B`&XIl9Lmdwy(dq|2Np%7C3cVEBHI0lA@
zgklXT8KuB7AhuYlm+plJ%YX_jQv}J21Rw<Rz`&k#kAY7o|6D+X^dOGDM)25=58gZn
zrKFyC=0tP~;kWwkn{*w8T<*A`-jM0JN7wep#;tYQj6o;iRp58oW3daH1U-xQ3ypP9
zQI1L>NcBG_dnslJ>M8E~0mZsCz4;m5Oh6tTskQUO;8o>;Td<@c<0iL+2Nrbf_b*Td
z9*IEuINcx1pj;4Z<Ow~OC_E-^t?OZyLJF_qg7>wxjo>Dw3q5`sc<-#^I?hQd`ZAAm
z8^4fD;61m90)VWAynHMr4c<AlwSXcK#{X;OKg)O{j}O_J@6)0ph?KcK<3#~v$a?zT
zmu5X6{QCfJSSU-W0M=}WrT0Mb6DyGK0xzfjm;XmUpP}PsbvQl<P==@iG6FB=A%$Sn
ze<`m54<<@3(UxP;wS;2aAvkz#nr~sfNLB)FxxL-BRsRwAQDCxf;cu$(OND2`6SpWM
z%Y2Z32vu%XpI0`*&fxjaQsdYhe_p;s@ojIFolyt_;TZ-@;m_?8L>pHDEkxkglIRS-
z+uPwj3-XKo`SmGEv@Z#A()Vjd^tpqNW$0jYobs*eD@(L0s0a-2xmm#{yujwrI9DUA
zUEVc>{El$~*hHJ`w<m~7qvkr`4{Rt6zwhO9*mcadL9V=$1x_s@AXbzSJvC?GC-{Ro
z(T5E#1=j?aLHpH)&m%@+@Ow@8Qa0s1bWlHV(2Vw03|pi=NVzIu;4YNlb2;o{c*lT?
zu|t4Ua~28mBU!*q@B><Gj*?_anhKLMTB5bilLY1+5k7Cr8x`S2BN+<<uyy^o4;K*k
zzZdZT6A8FOxw$(ot*{y}zcd3a;OikGS)Yzm2l>0%+ym~Y9*h@@8~60&7$gs3BsN9P
zhzEjJO20TfASXG37P3)Pr0q)^ev7LaYCi5}Tmu@KvcV+WvI(Rw+{kL~FV;^C9Mqk^
zVcU7_iMvcIZ|}dEo3yACtN1s#%;$47tjgzg5_spOm$$w4ud&8`P}{(Jx?Gwqy@gWS
zaV>h=wy}-Be~1?Q!mV$R_U-8+-TTq+>zW*Q$JhIxr!K4aR@lv?)-$i>k6BLxh7zNb
z17<GC|Lt?&J?LEEWgC|pq~tOvK&d3;j-Ko^En_3b>Ym18WO(ns__3*x-_#n5B|q(M
z+dWXTepLVOnth`CI-=eq(pg(j%%*<#>5#2w`CspyKkGSi`~T7%kFO)Z_|f2>7H+dX
zYN15s`0@9nu7fM`>lQ!yJ{d%H>k1{ym7|(94jxJy!h^i?OYnIdhNoW!ci~u0LZ*l(
z@jlT--T0pn$^t}no4h{PSudBqCOV3dzZW$WJ_!aN_r5qpEGEoSJX^9v9MO6SX|CAw
zW{iGr$O_(MJlKxb<tK=frnO2A^Nt~Gd(%F5@v$M1Jt97x1GG6!{AtfqctwxqQicLL
zGzkUORy5qI6v>9%JY_#~v^wvjZ$HBl)Fd?!@@of!s|D=Xh-vBG0t-U<m=t4O2Q1vC
zJ*&>bCKLUmR~<(BUB-u93<2~#qvsZAcTe3*Q`s~PT&!7)L+iOxAe)!9r(qZj<#?b8
zoktvU|C-~THMO0xl$gqz&m%NHdc)A>_DXlYPUL)_sHwWXYm#tUxw!NYA92&R`&rW)
z1w1Fg;%IdvJC|~#oRQA+qS+IMAU}f0C-;OJ`LQ1i>cmh*ZVSfA{_y-dtmE`w=9zsz
z>mSk=AGd`tQ(GQd!V>#n_~J{^(10DoW_~byM`5%^hVj>7hZMbC2=p5nWCW4%q%QwP
z7Fms=ZzJa%ZkDL(T$i)dw`b)(H{XOiTV-)0)_f=WY~ZpNuU`0CHjxgGR(CHo-6NJ7
zJ#z0}5p!RdQri1YvP9OWSf|(N$7&;UlxclWs^3O@pXO?pZB6%=CZx5SX-EHD6&In`
zFU35<Gg=nu5}#1OkeRLcdZ%0N<BR*6<5yZpN6^bWA5-_dLY5+I<y{>z$rqfirCYRi
zu>Oq>UyhtTCn0jnVsUpRZPUop@D0yPqyA^Q)#2ka29K6mHXl+xHlK_eiP=!oKF6?C
zwN%f<aP1f3o_7@{r8x~}mQhU|``6whhc{kZKj(6GR8Zre5wo*ydwWv*NT1zNhn$l}
zXM?Y%W(gYGf7L9=g!Oed@#<E%NJ{t8e)lQknLCl&4-2)H$qND3%Tc=NI``P>y0%5Q
z1rb5%{!SRG2d~X>s((_o`AVzJY=yJK-<rkK28VTt^WHQp{Lx8rH6M%osC0FofI1Cr
z$?8(A(hLbd28x)JCkBnm4HGsKX9AMFf?o{39)36=H=$>2_{@G!bG~V@(-Sw8*x`Zc
zZjNow<kBYar?Os(&7ht|pn3XR(vOx9Zi~~!gAR6K*235MN1xi}lph>foyG4YNa$K$
zLZ0$4d=@wKl~I}3FGM1Zn%F_J-Q~2^yVp_M`w1MP>*Qxhi(b`lZP;I{==2kAFsd4J
zBPy$Vi#ap#ZYd2{R_P}m6)zmlr9^Xx=9^0;W_~yM+2+`EN~f_keiY4x>(#7^Kd)U8
z8S?k|f<45{bT{(**ky{Q)5ea7?y<95oup#v%0$!~NrOb6`<vM-Mu}zvN|EAg{k&(_
zm`<?b=<_>|oVB91iJ@q3L>lzGDhgUF5uGhr3d9;0Vb9X=H~Vn>`)9)zr!4F%2Vc`d
zdbaAqz+ipfS>K5Z;a5$s(x7jt&C5&G^P}gdQabOdRM3CF*keSB*QE6DMYg^YGiV{x
zcS_E8CS>v8{ZO@-)zb0u?#t1T-1L?0Z#+BfJZx8FiZfywq?%jWpmg_Fa+Ae5QOg-W
zC*SUBo3Ha-+_RU75UQ1i>A1U$Z;xyAertFeYx#OB4OPFZm;z((Zpa4F_}#%j<<$mB
zW>cwrsu{c(XT8k)&vH$i_SJ%w^_Ypa=3ew#)ioa%UJnTb{p3J<{$m!|Ix)b*e#e?_
z+=yeB<SFL0DC^f`nnbqWpEOdt1U!`wk#f(<l=LgTCz?#Qoc~ZyTtyHy7s(N3+4vGL
zFpI8Y&7OUY9O5^>8iT?z!>b-nZ^(jk$bhiQkfZM~EAo}mqYhK21Ds^<VdXt09W9@l
zK)6OkBPZu~RkgVUWWr?GQoFWeleF~cpPNs9KE=<#{7Oy4Wx_b>G!$wl-fk$Go9D0U
z8m^ul`!!!hc)q9Y-C@+C%|v!md;>2oa+J#B@Zfe1d@`{jp^)v~Z7#EDuau7kgGc0P
z{klRag(Lq)4`Wj2y>e3T&x<_IFH!64_8QMVa97QX-jZeaZn(A&*BgEc$-SE*KitFm
z<!2b!mRu)qT1q^!P~#S2c=kxDCgrJ5wgoDO9QcA7rAzBc;k06-*{7DJ>Tk$7J5G99
zKh2=w3zF%1c)3p(^KTR0O1Hk_U%(0fHyc2QZil{3nRCP>VbIUDL<*x7|HQ;D`z!MF
z)a6=4Pku7a1PODK2nl(7?(trn<JMQQtM@HW(UKHhE#4uyeLKNBo6VUF+waYszix`W
zyb~ot{E)Qnp%qr&T*@;#gz45?vzi^VS8ru{enoLo^q9W~d+c0Dmt~<zZT2_HhI58z
zp#^_*&8LCu0+;icxoDvuIZE+5O60;!7bJxfrB%&~xQG{D=++gFqLYiKPcG$nR-Cjl
zq=hNm*0gVhjeq~EZoka^lbU&zdODPhMbPI&CyZU@$x%~5a<$z_QDL@enfCFUnR;Jd
zzh<S!_0MmT&HA>Qnh9P9<xfw`dVG<%?K;o>NJJ`P99JJ?W2t+^gc?c*41*!Js9<y3
zMf|?~QCggzZFThRcf80!JN7jpWO+Gk79-!>=S^{$=I0gr8(&UvZJVCq3-Am>B4TLw
zHdD=O^Fm!?p0Tao#z_Gi9tFWkjz2z}yTq&fJrCpY9?6yaHbwZq>`L{k`CfOoNN2&^
zj5pdPYHPM!xuu4AFgSU}ylRSP+ai~8Xz{#`A@WL9C%+BesTIZO=s4;`<^63Y_!D~M
z=*;vUOL0TN1V!rf&OS}l=RAK^i=(|&4#`G9bLes|b`jq4)#Yw8t497U#cd-ME~4$-
zRQ+2Y{g)p4tjrDH9{b&P_7b6)eLTmx8lLsi*}^QkZPX*AT)5glk8T)ec?{2gddu{D
zB>Qq{!RAipy13V5SbtMZcrVQ(Ti1k9f77k>7~2v1>7CU6`l*|N(yh-=pRVuwF!^Mg
z#x?UH2l`iW+Rp6;R@>HEQ7bfyo0PRk#`&A$-swb4wbNvaIf>*(UAQ%8XumgJ+DF;G
zQ$KLK#Wd<9?+b_?RMRk+_=`%tUC6{P9~60dVx7Rzol2APGnG#ZEp$kF0K5N@=v&^e
zbJA#Q)|Q_}HtN$(6)?@6iXN592mJUsN_}-1tg-s|v<07XAy~uq$35n<7<zi=&0|;U
zjz!_4Fc(7c?P7AppV;DhaBZV2+gLaAmODugu78Tg<Ud8LbVvd`2k+&nQrjO@BTUp=
zhEEv3|NUJ%%qbv>^o=^%1$gw;o+3JQL#Mg@S-yHlv==#qRe{n3PJ2azPLwXWghpO2
zPMW)CPY#V(c^WCGV7Q2caC@ht4OhfIWC~u?o%f=YJnXOaqYl8tzFeP<KI4t^-#Djd
zsv0h}Wu&NmSvQa6376(#ZJVFP^LII$Cq|}Z3;3TYeQUW^b6}xsf3(?-&&t2*TB5}d
zPZf%3Ib@~=3SmyQv-m!0E{$G^5mfLE90f1(QuGH+q6j~-Z9FqkM)R;BnA9RO@+^H(
z-B2yrdTuDe!HZ}yz9=JK^$NXhk-LU)*;Aa%8hmg)S1&6^C$+zZz6a6*E+V!twkUF6
zzIRrFXvA4EneT1;5)*5cN)U+-v*AC>wQ%LraQ$m#sjf<dR+fpH<Cfu%;$;L!1VB+7
z>;5R{^=Sv3gpV{U&Yn_=m%zKo%tkc^rL?$;yY)Zh%DwrWwNMGB)#xD?4Z~3~!Y)Xi
z{E^GX{@hhQ-8DOPk}iRxEJ6FGm{)fDU^Qh1d0OW+cdoODcFU5L(A9=5MS#C$6f7Kl
zUp>2oW_*KLyenwmYNeMg*K&gnxA~gTFs@>ECBBM^b|TW`J&HNejdj_-_~j=zzD8z4
z4)!#^wUYhlh?|XM{=9MTK@TrQaQ+UqKApI**guRo7K$YQ<;;4au+Mjmj9y$WNGxqR
zZLRsWy}a=-qyGISS5sVX@-j-C1tFseLKE|rNO+csVvE<Cp86p+^m?|m4YSZlt0k!?
zd#}n@i}q*uCakaU`WkhFUW?hv1%Elt{}G=UqVLMI)E_Z9P)*T>93Vtevo+!}x3C)~
z>rNxAtA1rQ<ltOY=3RgF!`p(??h73{rR&lkwEgGsQs-DUi`hFLQC)XPZcqa7#|XMp
z$+H^+;Szi?JhR;djfM?a+d6hA%nS8x^d<65?z?n&lei7^H_Hrv-o~vr=n;x`>IRSa
zkc-fzBo@7{WY8A;D3hLo?!yU{7zn-X!qcHrodT4?C-qm_&Vfu}V`=ftg$)mrw-QSf
zqn5ORl?;+~_eP4QJmJ*j6Xg%!%VMZZXv+uYjEsrO5tBKe9dt^J8opIFD{z;rk9$p$
z%&5$wC8e4vAtXc{p{i+M<m+h|NLY!p@l6LVdvYU3Sf^RNabT**D(1TGuyTz{HXZ}+
zBle3qPm{(1`kEbPlBaC3x_ixfGd_Y$3vQ%kfo&`JeZ493DHlz@$J^onp<T-GWgm0n
zQ7bl?RE1W1fKr34OeBYvuwq2`Iq9w@A*Huq_9j_~Yi^?AZuRYf{7`qc<-CyKGrFh#
zIEZh&u^FV+yo9)yFG6+5;%c(?Y41(W<yU0w_pu@&hnfCEkMrAMy|b4bU;~&yBlDRu
zya*PFOpDg*ZNTm9p#*w_*x%fSi^u`nKbgL6&hI#yeUR~luAWy9?W^ZdT9MhKH)@QM
z&r!B-@@PmvD;C3+QOm*5g~l?zGlRo7pR5nLZlH15-RtW6bQt^~p;r4Ndp2@zHN!x3
zlHl_<gVEmjDep`)ecB&Qb`1O_Ub{sk6K_bQB=l?^FRCZU|K*&C{4udOfOp`$;V}Ns
z`;LD`UX%T2DZpG&dQ$qPqBG(kUdA!YPifH7wHBPsMzstlwsVkGG84s<$tC|7?PfA7
zeZ*-YPDm-n;uB!c+ITH$ssp#Z%<ODKICR#^Z|eo3&wuBd4KB(_B(~}cpV2assHPiN
zNn{J~tWgjM2|I>gW*Rn`oaMffP2;>+ImffCo7F{;+;_HFayb^_);YsV;P4etM<;$I
z{*YL|Y9BKrhb#c)SLk+n`jskd!k!rUD&Z%H8U7@Ye+*(X5zt!%2bRi8z;OT>u@e&U
zf&y*|=cA&rE+zNy=<=O+@x=BE_Z?B*7U-VK#^!K{U0Na$bQs80h&74SW*H|<&$U}x
z?j0f;7dZyaXwjLC3^z%Nl$OY_p2qzfTM-<*&|mpzWXLKhR%h^9UL)NjKWoy%#4VPK
z4t=MQQ`gVsZ^~CRAvPlt=9Rgds0G`W{%sH7Z};Yr5U0wT4lpDZY>I`Rv@xru;iP#f
zZ!apLAlz72dtUG~O}1s1#>Ge*u&%ujnGmbZ59&U{c!Jpciu8&s@^@Qi7AMg|L@Ex7
zr*2xT2u9c8aK=~eN7C?7tjMJ`KALjPx1aB9Zda6nKS;z|M-7i2QEl(0{|jZ{O>goM
zNW6?b;U@6Fk~v#Xo8hiWPFypq&k*)g^7gCRZh{45rUWTrTW`<Ysxqtb4<<LOMjaYB
z$~@;goXw)l4u!@5%OJ}?yPBDm%bA&gNRe!dZGKPizLYxGKeSn4{om(fL`v^$?vWNM
zmn8daL_df`?Sr%Bm`%6knEqV6B2knj?UsKQ030YcVy35DI4E-Z!bs_%vStP0GT{?c
zT6DuzG!w}W7pu0is+2~CFT{Tn@0%cDF=!J`MMM{S+@c`#n_iG){;Kr<!Vw;{@7);s
zJb74vHVP}dn3dd9!(-eJ1A%w50%_-G7|~PM6lQC1tASku2jQ)a#dNi2)52j*jSi;y
z;_cIv_eMjH`u8sIls^$s=vrOf5>W&=i|@n2|1AHYK<wc`tUf(|cT#Q`2*)cKkk9mC
zHF#jRx5m4RS--))V%~0coBFgv*62|tRwkyhF*f(z2Q$y(_3M=O*2GYWmX48^FV&yZ
zOF$KDDbNU@#j>I_@|{`|%iG_MsIqlr`SPCE**CwEK!$KhxDT0V5yw+{pEKL{>*Iz9
zxRNoc?x8;iI}4vpn%fQMy9~C-^p|;`2(-3$V6{Uzy|X2|AC^btDEKZCm-mE2u`$gY
zU+-B61!WV$(~2>;%g1|kup%)TCiYDd^@|EatRJ889}vv3p(i{W5<`nyr-XeN55}0q
z9{xnWn3HdjZ`nr@FMN6L^O_pHIr-#^r?6%H%=8888e(;-#k&3uQv}_6G(_czLPJGC
zsd|JUa3g=q&fI(%a&!V_H!%k7NE;|S`kwMS2XOL#tW?S`IV1oymSw-%iZE&oYmTmZ
ztC$zGK|$h!)>-UKDuw{-HMihz=O>8Yu!BKc5OG5x2I%s4&~c;PKcu5Y$If0;CR;3@
zeJ{I)ic3Xh<eO$t-aLaUi;r0A-7}?~9!csw8w3|9u{cTM_K{dAp1+DKd@vJTbnJai
zG+~ouYv?;bcV&gednJl|CpJTfv)Jqq5Q*^-k*H$s8%9ofCcZg`&H(w}NlEhzO_J`f
zEqA9oDq%NmXiwnJ5L+u4$EXNbOavWk(*@l&T$*4EyMH!M==Hl-n2Rk@c=L`4I7oc8
z>HBn_@X`dx%-nad{W%oO7s%8~cx<wSO18DF)QGR_5Cq4ozI`hv6b0wL_5A%wI$9_J
znPdMMrJ+;fc8shgO_|eGO^c5|KH@=Z;2NJD>t|mVoS~y(Wu%kzr&Ab2;v*T-v{vtv
ztE@P~CrV{~(sU--N1flN)*cO*$c0B0*Oq;m4o%+uscS?ku|TO4SxBe(49ox4R*6F8
zUKDm)`!Q^t?Dut~UIWg8I!P*>X+tJE*F_Z2Ir&fM!q24?A)(BJD)9>Iv*bQ65rc5?
z5I0;;J<Ch>unkS02+uAzJIo$Af6tzibO}VXx#`wNzira}s*^sbbNJa(B5%*0gljp6
z=W0*D#MhC7d`~6ob9!#8?V@6TS?BxHR7v+r=J0+UGLg4o(+IUJUaD?%?J=<`NKUR6
zAIVCD1}G`s_)Zdq=6Q?{w%++y#r--b4HrOq_Hl}A(uq*F<;TszY0c}4P3hddf)W(^
z7nR2xYhv%dELxRubANn?`J@c(Fz4U?CY{6t54O13xYGDRCrkhq`lDBmqz5nFpp5Oy
z;>TSzvYRV#$6n}5cTdPneB?h3XX=$mH6(g*7%X{pkq@i-yd`Qk;cde;IXgbvPk6z$
zigqb4=R0y!#MUv+*0tTZAo<}$quj)+3QJ6^W#l>2h5tHPje+l%t4v*j87#ME-S$Tw
zE|fMf*W2X6(U5b7opL|ycN7R-uWu`zr)w-Fnq<=l(CkU~sm))Mk#NmDxem6E@aRs+
z{o#eRy@=6uj(V|o-9*)iqzLtQsJvCeANriBv=5dL9TbX{64=Mz8(ktu6%O4wN;Rk}
z4kmA|uFCGCdo4v*^R@QFN$K6X*~u!++0?D?$L6mM@7=m5v`&WMq5E^24;a6~Hj*V;
z_=L7TxcwEuN`3aZ=F~*1-j-xM+XB)eSBjDd-(kn~6r&kw+S%5R(>#B_osThiHL>Rp
z{iIaS&rtix7bdO%Dj(Q?KhP7B1&R5)x&lr0h5f~O@WY)FWDQ7pF2vNEasIz2&fN9}
z2pTA3v>g<-*@!tch_LV2IIuhU)y9nWU1rzT_g5y|X2$CEi{ZRq?Yj3Yc|Sz&J*xlT
zPLgzJuVFFL>3V~}E7@9pVuiJ8*9{FlsRj$w*q}S7jVIcI?&giOWrLNw(lje%>u~2d
zP|rvffUy2QO*3(I#22Ut;eYpMf9e%82<E+o2|t@$BwSadeAV5Ug%ZMXVb(>y%h}e8
zT%Fm4`1}dtQ=D_3WYD3amHr<(R8&j>Btd>&>q@R{zDNjap!3jL!;%mtiHcyrwRdX`
zTv!a@|InEWEI@yP$cP|o6wT$`DB$)Gv-Bw*?PYPukiS@fipLHA>GE-2C>g!@ZQXm$
z@ix3ohDS5Afi{sVPVqa(%d%4jzS>=eD*FKuhpvF2)#~e`Z9Psr36%Ruk&^i$FXWD2
zdUsvq2Mvso_*fq&0<U>4{0HqVY<8(_7xT!DW}`&<Fs3Vet<V1@7Fn9$FZ!-YT(>Od
zVr2-si_<eLjv^m0q~D&+T?yRvfZTb`wQnGO@r>OL3Atkd&oVouCGwtX(p;f*erSzZ
zN6U7@q-El2j~RVwmPFp`?898%O}GI~9sas*e2eLaS1Tk-hzG;U*Qbat-S%4;($QL9
zo?CW}ANS_C*qk&iUZtfHA=K6&OyaeWk2H@jSKj8$I`pP(P3w~wGN|Q*M7SNWX)j~0
z)$nU<sl(sz9}r+ndb<8gYN`bVq6xDbbWW(75?cAy(CG;x%3lZ?1p1QF|LLP*RW)FJ
z>ktGKU4N#9)-+T;%pKh(TAiFN`GrRf-Fis<p4g`s@oxCFt^eWm@!R>{*Y!$tr7LF?
z(r4sxoR-;NO?r{4g$z5sz3||T7pMC?gSEZCP2kC@rgI+Kpl(5`@BRD5X6~^?%fYX_
zKjVm8wo_O=6w;CMA^G23$&SxgMviiGwmew>rd=7|gjCo~A^GbGc_AF>mOCQUwT!Pc
z=XbK9FKkUKycyK`7TBg=x?Lu33<AKF-HW=Fln8%Cb8N(-43bibyPj{d6(_gTpbgeI
zdKWvQZ$IM$V@99wtkwE=LR`#$eD8cvd{O4H_QQG``Q;_ryxfn7RQ2^m)TRXGF@fMn
zt2s3^^z92pH=8eP^7L9Fmqf-;iv-((yKMtBvO7Csp~@9@)I;x<lHBr~ycPN0V~u3v
zi%~eL$idu-du4pagcmzf&HmYwxeSh0wv_!BJ8UAJRd1p!7ta+qn>233c&IT~^aDZ9
zs|WPH4mwQ$a^JNDHKOWSEAQcy9=7g)gdyqteaga4?%EEO<r~T<=din<fqp#g-k`T2
zCn{KU+_)B|k)B3AQdi@mf6sTD*sqbM2m8?Muk-@`BT3ahs=H6&Uk8Ii<EzwLik+BG
z^IV+FUmBM5Vzj;P8qE?7E&7-f*?$zTl_peMHn{9uy~p}!<hzoD-*YVrCnW!<d4dJ8
zQ*5QjYo}&QFj#_6OF+_?o3#I(VL)<#rPPLRGEVzDpRMPGxtHN$G3U(<O-ErA*i=c1
zmv=vS+|LSh7T<6+bjh$rmkr!HPg)GOw>5K{HOjbB{(=;kaZn2zM}-gu7nTU%0@1$I
z4{h|&%t<SUzUB&}Wv=;h>*tv@I*2EW32PbXsn6uQ21cHPta~U}IbM<`e;d<lMRwoK
z+Qnu*Y|wj>4$nDcCeqn;;3L!t?MYzBfBtn&lYP0(jYYz(Ma*To?n<CZ%_Q;HLN@r?
z>eebkXB%EGU);y``aw$M{7kn@yd&HcD9(bhCZ3+4g>rd5<kR>HcJzpfyU1PFGk$%1
zr@L>!YGZ#w1YFVX{C4El_g~0^*|9>+@0Bm_!g#KZFwuUyvSy=BhQq-Y7a8tD*Naj5
zs@z-NDz8^*1#um{iORcbXz_7xRFUSOf#LW$2&=8|04$nw+i}W*ec2Ng`R~L9#VbVp
z5P%`6r}$or5&gz-`2hrn{OUi-8Cu&HncXq5<Cl1Pl+>lq%&Ote6Q<eD`-`bi>F@hP
z2K2J$Nb*h~XI9m>Q61aXXQ|t~QQ+b8+R16;{uv;sa)mPFR*SN$Rq_2UBGO?wziBc#
zTVlHMt)3WC)p_rx`_uM%@a5TrZu^I)NaSKhX*CH0OD<7n6f&S0ter2I(Sw7So^tEC
zdFN?;>9xr2K5)~J-QccgM1UC2|00fIk!9Wa?jRkZzUU=j<1PJJw8jTGMwWVfGuQ(b
zEHigm43CE$CJaS1JIN{n;k@w~uXDzU=kznQxLrOyMZ{`4h#My)WGMK%x73med2wVj
za=fsxc1lihL1if5TKJEVUr5!vMoZInvcw_&KB7zb`zl!UcNYizZ$6k4_K4~U243=M
zf?`JL`1Wo2-Xdz5ZYwTr_&2PXn{ophGkK>A8&q=?gz#5bi_P7iM~^R8ba?y?Xi7S!
zJ+DMdiLke`RFDw}sRP=~s=|?tP5Vaw^xEmLA|<ajk#@78d-fTxA;F^tcY=NxRaLN4
z8D@wz;}-^m`+N9~WfA4n*EIKuaf%(J+T1SXb6Mzlvp#~o=dXus<p~XYG>6vI9)Bc6
z#`Y5NzHZ_r9Xjv!O5eVu`}t6y7%*!uS0$wF53zXYjV1mE6q5K;-p5wwCPLvIaZPIW
zO8dF9#XwDgUtrr=Jc+4#n$pz~QaW>q;Lh7^j~_QTw@SNW1;c7lGEJQGg6q2i$HG@~
zj6?R<1g>R%dad%7?<41Z`8bUXDg2}vNYbnqqe(&0EzG)avZr@5<GaJ|==QqTied=|
zp-SD?3eBm)MB}O?&S5<3mlo({{Z_wD9#>d&yr~M@vQ<tmVh71Y*<=BCW2G~R2pLdb
zd%I`X{6zl)Y(_?$=piK<Klk#=-H##LJ`-(U0=d1}n;M)6GZgRk0F^PUE4LCqO)95@
zSX?>gAt|jCo1o4>IH?6r_$c38RW0f@k{BBFrCTI@U+j{WL&Uo6We>V+;>RHvzy8tv
zfZ5Y)5~7#>(f?Sykh6n!Bp34d-i-h?k;7LrYcnl)qoSRIG<bx_wikq1ey7!I+f`Q6
zIj8{@r;Bl*SwCTaNU@F#VQQHDkpG9+gEkYLMEF*RzUK3G*)?&qP7M++V#DM;B~*W4
z4?y}R+I(-{=G^;>?yaSc5lF5o3=d43;tDdkT$P@gs(PSKttcrFwnlk_Tx;`LiqdYW
zz(MmN5^()|@0L<nPhKY*mnMFZ>*Zl=q|!FBZ16kUy<v}GMPI_-(p;P0@Lt3DepXhi
zP1s3>3A!M{Uiwx!dhygEf3B(~{9Bo}Z@gF5`aU_vHpYdvw^Fa~<byhT&uf+cAtc1@
zEz@PiBYIsG-(0cR%{F6Juy5@b2sf8n&2&}Ii!HTYz9FxCy+C|W)6_8wl|p07d<m{o
zWx(e1;Hoi@z1|FkuWKW91>hqdU;h|92&}WrE*a;tTWNId%L8d|nN7=pN&JZe*`m_V
zv{*Gz18zZ_QlLC}%gd8)|02jw3h(e9x@DG^|MMktr0;?x=xx2LD!ID8vE`NPC#bi!
zdxu9JeB(y9b@djbO6|yK*BdXmYHFF6vCXFT#<PRz$p1>D5S0Hma<VHu6@#}MZODG`
zT6Q#}PNfm7-#&KL_OGFXT}-oI9dX~W9g|)c?7zIU92I0mzbZ<{A2Qo4s82J!{*3Wy
zkWigsSc;rYa}<xg`NO4gRk+o-1jX_Eahr`ygb>-zHKL)#-!J=4*r5%byf1Y*Tw_>>
z1aupMgSMvv{k?kBCv)-W0PH|N(P#LhZ_)LI{2-yL>Ze8u2<s*PAFLLB=%cNvKaRqh
z&PJGSnW}$86CXQ!P9`fJ{jfCfl1NB>aa|R5+!Wa&ONsp-s1D^{HbLP+Ns8%<X@@mi
z;fRkE?}@gJk6AyI1SviD7cnaPxPbe0f@;ug&7@SAq12@y{{?-=DPeq<Fp{?8WvN2G
z3u#u<?9$BVv@Q8Vxgn26yxm)yhUOa6D$Sp0DsN4ohKCE(9w6fxGpeJ#HJ`BHYPgnZ
zTGraEkB_q!$JnlR&6w|N&c?_#Vpj2RWsvoekb=M$q!-K5O+N5D$VI+hhzqvkdiS0H
zqp}uzuW?JZr!S(Ol2@F1Sa=r^(OU1rr|na7SMEVqe|B}^PJxi~=~1Z)jiqyRY)I6W
zl1IZOO_Mpz4mSxMsLD_NU+ukRP#r<Hzl*yC5?n&i1PGhp4k1B<C%8i(xVyUqhv1f=
z!Civ826uONcm7RsUOA`EmwW5ht-7~fYJY&)!%TNi_w?%3zqOu^(^#7Bw{CB<Dv8oA
zJOy|HkzsG>QP^a?e?!5jb?eaR-jwD5XMH!=3psPFbwvy=Dmgh>9%4S@jLvMt*03Pj
zVN}uxbTJUTRqxgPPV+c}xJtoxifUuUp$~9~K*5sZV|n7$gXO0m<&v_xKk78yg2lzk
zd$=29rf>l(<Ub3y$eVXR4*5(}d_$kdAbgLCy$e;d+G?Sy=Pn=(Q6y<8dq_H>Lz7W6
zW+7Zyddsp?!1)pJ#!IUR#)f)C&Gy?qy>6$La}zy$9bxkg`rBW|MYzE+w<n#c)9AFJ
z+CEu3!yMk4x5CA!-qjQ@*f;7w9prxveGOwrZN}+bEb>KZVv4^p<u;;k2gkr6WXhkO
z(>#(tEOpp;hbcE~$dxWhR>N`Lx#Igg31MgPOI0>1hjY3o&|$n-{F`kQX|NAYwZFg$
zqy!aq=Xqfx*Ik8`-hd#lsy9NmuhmtM%M9%0nR9)K0pd21S%d7bK|f_&w_q1o<a|bQ
zfyin@Sx!{gNQTi+nV{L<?<zAu=J!YBNWt!}bZH#PcAg0u#@>8Un4ryCHC@>J8UB&L
zZPG`C`7WZEFl~Z(Uac~Ad=FI~h&Aky=b)ar=I<#t-o`|7@DLzTst+Y{_?H_k>HXLc
za<p^P>DM=G2d7Wr*o+Qyz%o&=g|QpV_drD{x(*vSJ~N1%G+17B&Sd^=X?@0eEAu_7
z--q#Dz1!dTiz+r)cp-7rbXg38`|k^Gm{8O=fBra@8`<`%zJ(8X5?t+G^~HBy@AoeR
zFSvQ1p1K?1y3gFe`Hkc#zgNdyb%8$zL}&H&qJoaK(qA1UwDM2R2i8XTBEUB1VPDW9
zv2Y6+gQhqt`Ar3WCfP!@|L($dR7ylsjq%5g1|{E8Bl*&VS7f@M$94xY+Ef`G=f2)y
zB;dU);$%&@OzGR=6d!o1WNGkgOW3i%l*IAy3VWQqNE#njTCP^Um17EJGbvVHewv`{
za|{qe(Y!*lAcI@vVBPV|)Kq9<ex-UK`BMG>eJA|hhokkT*tr$cX+nqC$HG|@IV*@{
zyLY*SdF`tu*>*^8?!^Tz0&nlwkWM>Y%fmyHpAD84+WMnAs&C`A@hR;ikIQ_CLJK6Z
zia}sLi1W;b98$F|G-LnqI>6mfs#)DFWZy@(ic`+o%rI!wqSItL;2U<(9&M%5z*Edj
zjpnd$9f%>vMR2$shU4Pmhg8KNT-8pzMk{Wf0=lWu$TZE<FZ^76c9_EV`xo}RZABuh
z)z|#ip(fXEeO7QNanP5x6B?$YXj12yEI?W`Xoo~lrjKD$%NH7t9E`EVWziBW_!j!}
zJ5p)lYxPpP2zaK5Stw6$st*Kj45IUmggX}QaTI2;SQ~dI^XkMu%apyT0r6J)m|7c9
z`e+egt-7+PC-zrk3-crmg7nU>lOp2pVhlT#?IZ*F5(QQV1hPEJgIBQlsjNk4(fUe4
z+Tu!Ba9}^!^C-z+YBbT!oEaTvEJk6Xp3665zu&#^B3Qc-=OwiefKQpX%iRgRzs;*8
zY8B|2YbPCqWeWUu0~Y0<fiGrNlQl6}p5c4lohOXbK}^OZe$vfFFi^b7Vp=r|@9pcn
zkL0n4oFxFSyQ#(2TFgzVYW8`jGQ!E)Pu%{ZCQUDyaCyj*|5~#{SA1>n3SXk>EHjmb
zzkD=%MYSuvekHd&r&W`6x4fZQkYnzrS#75tTOIi>&$HI7AEczfrOfct$CMBTPH4uF
z;&ius<Lw!I@+#BWudZ$(kv;#40NU(jhBGPn$CdeHvGun%NFB#YbWMD(Qmw{|9lP0)
z3CUV(XKgBoyX<2xfxIrlN%xoG3_YEc`=gCN6H^rRa}Uvx!M9@BPg0Yhf^!pW`Wbb;
z7`Jr$1vHG0a^<@NsnMA5zw-0_=Z{J*@Ar%Sol-5K9_3&!H-Gj@hMd>B_~X<Y^%@uD
zjn_9MHERt9U%R(4`6rT>@Dde9yqbHi`LiE}Lvf0Cu}Q9-CHP)#71jzD6VSw%=}gh^
z43c9kXf5pVZm3sv^R|<X_R3ckDIIL>A|8vM_J;C0!S<mV3>f+oT#>m7y#A?-TK(Iv
z9=8p8R<f0vK?X?yzsY(btx9Ecv-tG<{002!unf`<pak0dTp#yxtJJ~8tM@(Zntx~d
zUrYPB(*p9u#_V(ffgMz9rEiCA($4o152imIjZ(B9*i+oMhRQ8a+=Y$I8R>SvXYn+q
zD(^B#=pWUZ-2QP>j&b=3WulyJv=rtq5uYkFHN56laLqO?PLSH8Ic)IrSQA@GJdFfO
z!m;ENpu%vN`auGg_H~L44N3ggQYbMFO511$yOv965ls2Q+;c<cG-!iI2Y2GGS1VnX
zYbGmCezIoN8)Q3Fe<!_Fkl=NOHXsCxzp_%o17;;{C>mUgOV7`rEF#eDXB0e$_Cio&
z4Kc3BzFJF~Ca+}Q{GHimANJpwA`j%r#`C-J7otb+-Cv|v3IKsVK9I0mLc+&9WC6zN
z-AWJGEo{<{-%O)G%NtTC9!ht$YnpZ?ao!&#92V3>j2&k9+ECaSYkn5Wqe{15%a?Y4
zr9MTZbwKFzsy-3*e6FNLF^VrT?7E|-aKG-08V{qB1y#`89@uu;h4!iU7}iLZsvY-R
zL3e&!_kB3vP-3Jyo?Gc~Q!j4O65nx~wq;M^Z^~rfL*8Z4y{UcdZO<zg=wiUu`177W
z;`g?<z_qw{n7t<b$LuE}p7Cv_fu-pgwzNLQXpVtQmRaUahh=|z{jQ3U8bE_wBu2p+
zw?1QYYrDt@Z)p6=&?xzSIxGgk<|Zjz@kiKoXjBpP?~&e=1ZlD_@3*enhEt@Z8w=bq
zCn3@U4h~Y&4Zw9&jB^s)FJy~0IgmBV{VY?aTZG5CWAexP1U|)I&qWDdAc#1_w!nKF
z3{lavAOL{mzWmWLrups9E3)$z?QThaK<>Ho&*I+O{;>t*m<g7Y(N60gl|mm^`|Xgx
znWcGaIae`Jd^E?}ib`z{ydPfDZNIH159(d6B1xw@ndp7J_Iw(#@*z!qoWaRVy4t=#
zy)4=wg0kc4u+R8OZCE3!pR0WU{~xZD*nszs1tR-Xpp1VNX#IkWQBx`|4<4iiYJ}p3
zb-?p_Vx2v(9D}j+tFnC|ym55?4QAtI`K8%pzzH(4cmKf05wnk$>N?-=Z;S$?t?IWK
zL4)LTXt9^<uM|Zmj-E{mw{WFM?#ar#t4W6?``RX@?gR7dF)ZStwiO%rcdFz5NT2l{
z!zM7?RltV0UC~=_cK3Wcqef~C&-@`*2dZ4=A7vXZ2eCVb?Ah=s4x)rP{pIGOqWHnN
zBVx8a?4tW#_T71=MmO`@XEzFKc4sOTWZsS8YT8=u@UX_aH&8{cSo}znpGj+fGq>UX
zHK9%FMd%rTbE)^w0<Z1e-+uEATifIJ(6yq;57Z&x8=Tf?s(P0!-oSA9^z`qNVgV(6
zJGcKX2;E4xST)*t0C`+{z~F7m;t09C-cek9<V#iDt-If71H8>{Vm~3p*#>LKowo$?
zGev9mz?J?dT;xDqI}kYfbprm>+vi{PR<d5G6LqZLXf9*b1nSil9xUa|QX-mGk}cQF
z`<dRwD8VSbt(&s5Ow*09Xtj%3?YS)e0;yX5qIZ*%CTi)V*S~~u7Dl+~o#&F;$RLz{
zf|m0)`{T}}ly)<|WgBa`c_Z8P1TIB$S?3z~>g;C1lvIw#A+e3=-hXFDt`vT2u}5c$
z+v8BWrPkkk?(0cj{Dja{+y^>H&$e(x!@2@^Wu(k4{8kRpSl;eg9^N!)!qw7NC9zJH
zO?cQgNc`M}d3XSu6*M8#r{fZC#XJVhi`~;ZScNAvX#J_2d2EEnFDF;gh~mab#y&kt
zJnbio)Ea&Sd&hPk|4bGA-Mm+SE+yb{UToFzHh#|4r=chro2H8f=3JyJN1sA2TWqg}
z_{A3>4l?rBkC>Pl8zYU#kD{gU>1+7SUGD$mLSf+P?o}%m_umP$<OBmG=@Ni>{b%C`
z``P32vNcsf+_$+M70=fHYJ-8>R$I&CD|!>gosGb2^|~DH9@BO731^hc7(1!3h*BM=
zq@g!lmy-pTRjKE6-t%Q%^~m>^cRoNX;~+)G{WF#+nakyaNFuqluvjW*RFcBYUeeu0
zFyHwmU1o++D(<6OFO-ajo>$XJvDo<y3|dA>6m#&8oshlPzT!{MW-CpvxCkbyv*)YP
z5oJ3PPv&1u2Jan!V2i>NcI#!%ZpK0olk_-N_5gDlL<F_L{28#j|21IgwFkfGm{(9{
z2lR(m-1C>XqUFz~NAczLLMnQXxLAJ_`i(x2Sv{*3dS#c-ScUiU3(4_yTLr4W>vK)%
z?pH-m{*enHrKlZBN@sm(NHOKx^0yR4ZVGlJS~QEjC0;~5vje6_x9RPv4^gSXGn>Iw
zhjg_ciQ(GO?r%plRD6`4JZjWmF6377T<LQH1C{5#SNqDL>7A*C%l-n#ktfbixy<28
z?8P&IEt<OhrLf7rK~VuA7yg-);1%@pPmQV{YqVEb4zY6EeU$O<*QJnq_@`vR(bI61
zUZf?-GsD-Q%*KB)m}l>Al)5GF9P7HHVs-YEb?+mEBPu@Kn)DR%s((_Ns~lz{o(6}4
z#iC^9Q<^nkQ?DnFFp8{BMoUU+kNIchdpgfQiSYq52Ev~i<DJ<5vKjTCaH;>qi~WBk
zV9q}dzQ-%7=>a)I{F}IYt<@SzJP7r}N`nN%i~RO_oM0oMK#XAOtKO%`APDot(n|=Y
z6b{Hdk-#xR@fhxf1=3I;k$(_=|BrxSLT0!Z;`nW7wzw?0NX3=`xt;BOj~g)pqDaO2
zn8?Dy*q<LGM0Y)zYWs1tRu*}zvqT@~jV0=)U;MMu9$Wt2Akdd1hxvETi>40pTv9w{
z5Y{XJLrxC9r^F`^iw5hmfKtScT`p;l5bu+|_!o~cP(F!JJNFj+&{<@U=`5Dg`dQM0
zposi)oYea0|9tNQz7N;Zqm&f}`m7#O{IVS$Xf}*~W`iJbaZ12B&%bHCgNt+H0Fodd
zZ?2KxS1=~TKejEI%Km(h4Zauhiimhe@R;GkS*x$`1a6XpV<73}pUPDugUVgtXk9No
z2f`VQ?GMcVsr-MQ{C~Rr|HUyc`6wCq=O+O2m<8ipAdcjZl7JOP-o8^MP+8;gFHr)|
zT>H%3djWfSe7F~_{}oCe3f^26aLhQsLu2v8GY$*Egjnsgqm&j5;m|(;+>uugG%ddt
z{#}mpgjKhxqZz&E)nl0SBIO^8;<MJ=!GFS_{}TsppJ997*I4S@tHNoynZ|QEn`bq5
z7XP9_>!#8yJRR{uJapzDhUa3I?C9z<0!8BArBG=qx3xh)QyqEoYLR?rW*0M~*b+@I
zp}6mmFp8&XUSqg-Z9`aWaz5vY=uqjiWs<Ng*vIW0g2&yp84|<0I@5F;$1`p9xO>gr
zn8cG=f+Kk2TL-DDl}T;KvN?g5^68Fk8?gkzCVvC;Q#agpCsIEzVD7Aq9yE4me#N8O
zAi4PRIh9J}bt>Hza{}_I%L?+<R_&+bKGm~KmDq@mpi+b0DyuI87*{8&Of}T52sLMg
zZzHptV69C3f;)!w+K!Mv?8`|vq;OKt2fhu9jU11P>EI@nM0`vk@CFRA$V%?v9}tU9
zYrU6|e-kAfJ{VZtR^7cRFu<9rGx@Go<1)1)2J-icle({&{v7;KrPuH>d!i=#o}<*g
z>bBB3F{5^!tL7qb_+?1w1x<v9p?2~6dJ$`c$|z0uX<SB;xMR0WZG)GeDk3s!UYXXN
zEOB6kj2t&xOxGJ$?9P{WSogA@Wal*2?g5{TAoe)pQv~gHGS<LU?vt?Hg$A*Wi^Bo_
zo;Hk%vx{qQ1M&4l^P^~rnF2>DiE~xn#yi*M(v;LHvuI}b<QNysXR+M-&XiiN!R7Zn
zmLEl;ss@E}>l?=DddePdsSxeVhB_f}4zBC@DJP;EEj4A=7lYSpYX|&NU1gsLyC?<}
zkJY@{$QAEfQWiimF)@$~Dg3E;cO3#VocDIa=2fe-{hn=PQP+x*&xO;~C)??;+uJA*
zXP2!JzJ9>?LI2ggInhwYkNk&QA;;p6j)rfJDIV~SzH}kSJw^CRgB}_0s_F=*pJg&d
zwcy=u3u~j*sbBAd44*q@To;`qL$~@_Lds`|<2okEO64NhTvI}!KUq{;3mW-^##b<o
zOdy)_p*M_Q)F1Caf~D%Sab-CBG{&Lx+?4x64EdQx42f~$Iak>EeY^ck_gC8)N0QDk
zbcyxuL*b&Gm4OMPm)~YiFMf68N<#)mZ1=f%=}goI>sC#{qlO)qZcLprua&rF2-z{X
ztoJ|^Yr+H>f%s+KT5k3&xQo}~p>%h*Hg?$(n1@fhQg^#jrNrh9XPvxj+{%+4edALi
zSM`9XK>?kDDtAaV40h$;kJHCd+vB;4o6#vz!c%?SRh2~cPF{E&#W$7`0h0f9?fHJ~
z;jX9YD!4*%)Jo!a7UoNZ2uP(~s?YE9v{>`ZYJw~wcW{xZ4IrJz6(`C)K*KM_w=a{p
z|D=j%zAhg;+xbb+*wb&@%j)-Hpdlaen4Y?Q42@qO))jVp?K^r;Zx$ne1aN(Ic@H$a
zV9)sk73bTO{{lvzjMVje=|RDptk2KJ*rXK8P!z>3t`;1HFC+Xfa$`m(6u-<%1b#i&
zxR5%S>tn_*Uym}pCY~Novd2AC^QX#M+b%!4UB|-~MFC4l_dZ6d7P7s_ZXvi{U#|XJ
z;e9z~_+~hyuVn0Q>Avt>V;}||Y~VG>qwuK66^h$eEadEtM3SNJyx9D(?tUreSg$NS
zaQ0I^JBFK#HtS@KbMpkAf{Z`^Qq7sO%V9<G*NYE93~eCc%Ncg8>q1e~0_JW6vk~Fb
zk)MsX<)1M2=6;I&N_8wt`MAT?L({(cz~g#rKUL-kL~4~p54V<bpE@|BOD5IDJ))76
zI^3mFhnd;NS?FHb_ATz-n^kCS<ED_H#M06k4y!$b=+e^R-(`(o>4vzcIFk;_chu+c
zJhLK#r57|lQ{%hifqa|0DJ!r#tJs2FXBb5Oz-q2u@o}bRk+)i`UfOr;O_{*K$j@|t
zn}qmMeY{|v?|q0iLU>g+)Dv^QV>g#Jb*pbV8ORuqpJO!PW4gn3Vv>aCjYV>?#C_pl
zp0&{RDrq9>1`jiuY2gWPh5OfEm}?(0<EU`+pEPL2VlSER$Ex$^To~3aCU;o(r?984
zHr((PHT$#;W3D*@og(c?oRZp8guo}uj+!vt1bqU|H%9^ck>}gS8GF%;_0u<;u+u#)
z7dPA15dCxYpwoj?chMWGiykp5(fzsy*)101ic+DvGy}+B*21TZIGeZzIaG{=I+aF4
z7n6uZ1KP**FS;uDX|Q9}N*D#}WONsPqeLteIR?J0tN3@m5vAr`P7<U)r>v6}WKoS*
zn=eQg7Fs8WBR(jbpY!Iso2y+%eAqk;O*=8cteX8KlMr_I(Ozk{v*19<fH&iw&MN}X
z9Xk2&G`{qUv;X#-@J%h{rf!_qd$*yI3v@_Z!H9X#YL9)<!OJV{H64OODP`6-)<rD%
zZ&dwQT_jQQZhWhrImt{*tFN=F=p~DEK1sj8;y<?t!#2e?^>P)V4GFU^&?LtqMnd=N
zj}hrn7+vtbGMLIQd2?64?#23W?s1XU-E}2S&!^9y-zYTscCn?lpV@poR`&O+9jUy#
zx4}6}Zb({mJ~}G%L3A~#_;)u}+el!KzfSQf)<?!sGlsaz&SAa<NUb|jb0ot;BQH|=
zcvK^n1Ccl_r3ikLnVwoLKr-451~!Q<$$vAVq5X){BX2bjZ$uoKn7TSEq8(0bRN^Xa
z<s&#kW4#s1E4)Ehxw`^?xO`<jT&Jh8$C@)#jJdY-#!|w67td54mhH~?+mI;%LcbIl
z{}0~|)``#IDX5h>f=S%Fv7I*{P7xCsGs6SS-~3s=&7tOvF?{Cy>13$m7rM#r5NPA*
zq2?~9UeicLTNx%JOlfuq$9o!li6|$=Swi#Gr96W4&sBuHB9pmXjTVW45pKaLs?ZYg
z#c1uCP2~my>_Mn8nQ1RxT~0DPMdOd8<D2?%Y`!rtt?NG?H;?%u3nfHr14(r$@WaI=
z!X0>qFFF^;T*X3iRx)ud>b`~OY@*OOetgyDX~e>vN{Z~agqx+gG(Z4NQOL_bPh151
z1*p!TbSbZHYnT(B=-ax<m9dPoSX~LLXS$@n)|mR`vK4vil;@K|Dr|s1fScS=Zxqw^
z3zOYP4*$d6kWxb&ms%J{h=%(M4ad4Sn;V_y6`EoX37PPY{?J78h%5?;4Jv7=bSBRr
zzI(<{<{$_<i!q5m4}DXsD|Bn?Al7}IDaaV27Zcm8DxSS|_mn(G=)~VX>Z-MN+xf7{
zdjrK>(mC4PYVu60r(tLI>}XyjAX_YXVeFz=3rnI>2ABS-bph_y+diS5I>j@M*SZ-6
z_Euf@iU8(bO3T})k+;uE3t?xI@#fTetM*hxkfymC>?v840^PRMu$_nsIK6L^-G$yP
zt)MKI>=wN=*hEEaoFRdNacpOi)!UqgEmIZt%IKO_eLz#e`zwA@&eVykW3EmTdHJ2n
z!og$s)+IO9qr;bp0c}t)b-1Fz?KMRs@2d%eZroAk7TS#(_Xk;u-T3-_)r%`)f~ky1
zJ_J}={I~EZ&C~d4D+@=NH)lDWcT~M%LadQyAvroebrY)6Mnc&%nneWCN?NBR$)<h!
zXtzeG6OpLF+pSF~DI|VwC&p>J)H@t8n$Iuo;vkt{f2zab+qPEEkb*5v-MIEI>B3$N
zwTK^<Dj8G-1`wXopXc)#G)~NKwSB<S_sK7I>F1a$X&X)j*`#c?OW=^?d*bvlsI@p{
z{H=T|prHJPTQ|1-5G@4+1Xr7UUvmv4>lYjcO=BZ)AZgC;jiLpyqK9@Aua((vXDVjq
zvP9=5`#aAW-yu~-RP0OaZ%j)YbZz3AZ_SjX;0<&o{BE+ovKum~N#nY52-}0c+Lhh1
z<Zbmh{<y``XHBNxdOc2?X`re#ioUWhK(3?>sc@#a(wRnLP(yI5A25!`@tK|WWj<B;
z9AJ~}{iq_d`Y$^dUe(~P_}H2i_O9;rZEL1xRNfuZWqnUQOAP#X)AqjFMvtl4b4J{A
zG?Qm;rNQOI4>gSvD<tWV0Dm7uxyR&JWFsw2D&_%Cwz|1rJJ?3cALgHK@@COX=T!{g
ze{<ic4}?xfRlv_z=a3g>2f@^Yh%Pmqz3+*zm5-L->f4ZvQ-?fDV1(s%@akp5a$@pk
z6kb(23~*)gH{O}VsCZ#dICezSj(<qRi1UM|39kNVG^WT%Qj%?mhmbsmhBR(F?LAi0
zI(>YBYg&~$`~&(N`w#(c=E=ffx|JnwB9^!|lsSP?3aob}b^7`Zb^PMNSn7vwB?a^2
z6Q}BR@-xzt=1Lhc`M>2-La~Gk`1i+{Dtw^z(XM|!F|M8AQ(H(sUYY(xV~1$UqeF_(
zj@rytdh`P6*rT1y?#0}O`G-=5j#)txz2>-iCpMGGZK0z!>X?ZobEth7k;0QkH4Np7
z`Q(gg{9waRpR_mv;(2f9rjl{sluyGo&47BzQ5Yux?6E_0@T|oO6x+tQJ|8sqsr0_!
zIgTdPDj_*`d9OMRI`*mDN8+}8Dg-aBz9gkIh-fg@0HV;}hSzc`$|}|Ck(sM;aPQS@
z=D}>7XnBrHq^#X$t&`0)orc{IRN_PNxLw}2J0i!}i#)p2KJ2$HJ(m?lbTUVlV!Hgm
z=_-fpo$nNgqgJ53SIIlcy|kCjaG^4;*ZBJBf;PaKXw`gL*dEv>z@Ue3iG1!{ykXea
zA@5`ub63|6lPqDARp7D4UU4vx1~2l7^+)g>jEgsN(YqMcUpkD65%xFIXOeH47Zj9>
z(RS+xD%)#uB|fR}S^M*a5<s)4DHQua?_z1iE)yY)y`)hcjr`#QCs7=dZ%){J+kxbq
z{DR=DtARfvK}JnM8M5rJ{A}UyF6jN6V!PSCkwaGhZy{243lgUXTE5Ww@zi$t=?A`8
zzIUy#(v3L=rMJ|Ji-Lc23s{e;)mn-ItsQ^_eC|NjhP^02?rrq~`-`S0<}dbW#_W($
z=Cio5ciae_<VtSQ|D%l;qhxtM&xcDny=-YC+3;o~e20iAT%q|JpNil7Mv8Ib&^6~f
zE`r5ZKNJk(QdEBBadys{KRdc@ff&pR;`jc`<L_74d0sW$LnGAA@lOW`r9ghlt1GkJ
zxbot7N^#V&`MlEdfu~pwAghI1sd@~rF4lK(@ojEaU)Whw5QU|<*u=U)zyWOAJj`yl
zGg3+x^u0vP9s7mtOud@~#<lt70kOP5iTu_S>44+}-{lmsZ=phKHC05;{Z!Ow`L<;d
z09P$MaDWj<-L4U&D!2Ml21b*9MKGF-*v0#9s!D+v^<~Qa+u|0eUPK6yCjmEI69SBx
zYpO%VFD<qX`w@yAu8~x*G#<IC>;ySqvYQXY$iXi4rK00}-i3_*TtsIRI)a54`g!xK
za)v_(f4koc_we`2S&%~!-_A&oaPRRIp>^fI!plvAq2VS#1SF;&`LY_b-mG&<53b&C
zuzXd!$H=6|=F#5*Wh^UnEej1*IdNl`1kYr)+|sx2dTYO&MsaJKTH;Xl=}P8_+>(R{
ziXDl0emzgaDMGIp+hm&!{vnU58UkL#8NMtaP;}x<@GJ~jTPsArbwn;KY{hP)d{|<)
zOmeICY+Atq*NDR=bfMQ6w@#((_$=p%X$zDNA!!yC1RgUnI0LS)?Gt0T!a!I3Y+6*^
zc!qP`Oz7QpS-Hctohxy=*Jk+*Dwyu*LVs(zzslpg7l1HpUl79!>E$!~dDf0JKT4N4
z|LlZ?mTRGUIN>y>h9oq~walZisTQMtIl4CxWFYg4!kYAKu7AUW+id+rEv{u8y%RDs
zBgE;Cw5mXOWvI^8XqPWjDo65jNH@Qc{um#bGXAeo1TAdO)bkaJ8)_1kl2vO}OKWfA
zY!fzS?c<lLl!;*PF#?PVh{w^>;2g;IgK#1=vOeCt@*+VuDXugjJ##eo2Vktq1uyzp
z-JQx+d^Ao_DfN{}zrkk^-ZiXxWgE-(93vCr;h$Mp^lYqKs|GnH`4aRys50p|U$5UN
z!BaRVt>yzGi}sq>He9GY=jF#mci--!??XQu*^+kUG~P7i4S}33H=Z3&$p?c&`L;Gv
z$Bj}0d}jNmQ(d1APz12m00=#{zJ=SRen|pyBK6_6FI4)yVDI_Pfz-Y3zZ(=vEh9Nf
zcAnnw^gDSo2e5PV%{!!Z1I<$XRd;1P$X-y>&V5`uow{)J6K(XTk~L<ribVD!ox2ZO
z5Lj<Q3%22nlVnoG<#Y82oM}q}OJz10i>#r&baXC;XxN*R45?({;tvs?aFODFJ15j(
z!4n4)$hh`(PzE0+LD6k|Q4+h@3{CKy4cB~^_e*Vvy@iM}<X3D$d>1*S0gXD1d1S4^
z|C{Ho0&gS=BP1t!aSIXQqa!Ns(<0*Lwu6J|{ck}(Lw=8Mvn(2oumcAuetmN^!`rp-
zyVWcE<zTx#!$d%R{7i3NNksYCh^34se$jrXMrUp(?K6L8?Kmkrhsu1sg?2fxDQ_nj
zK#JG`l=OT2J87fhI)i2s<2>ZAD|3}nj?9Ep<)s>=9CjzEN5N4AL~^gt@*NHhdAVIK
z3^+nps2?YKy%g6KFchg;7R1Q4&^s1Y==XXfd1G(C^VF%!n5sW#G$F$<VN4ATv2K9&
z)<%NvQ)$g3n--!+^;*H`EhAbMRt;k#YhL(3d5+*!Yk`JRakIGN!R9oHR-qZP0k;ef
zU6>uA=s@Hn!pSj&wAPd)<?~Ptl4$<oU-xbPadZdocK&2>9?J~l4*nzIA%*$fP{Tsj
zuY5TPi$!Dv4V_)%<Dnx@nmFE`Gi8wcO98Px)quqPs+h6W<3_4_`-4&OxJcF~iDWI4
z5R^fiO@$>{UZ<5J7p0(~<tgs+i!7Obd=K}PWzI8d%}s$D6zpn*J+*tmWHZ`&Y|qR!
zQ$(V6@;L#bd94vv7C43C+Um0k^k&xaQdSnXmCD;aR&^2-?eHW+N4kUVtKGBpD5p+~
z$)2<)9SgE@UTe#^#0W(kIq;=-e8!x3lR6i}0?>YK5L&-%I)^%O`li<i?kh18pKi_|
zyTZSan(rm2SH$-1!J5|UBr55K7A;aJ<iZ0=-;fCh!yi|n$z%_#%%Ua|&WkY|`G&4K
z?_X->ggo23NK>zqt_dm}tc#6R<k*cEe#`0BlyCZ+QQv|pM0t?e{oyA&p6|w+062GO
zBJCe@Rp&>|tY5x<<vEGek{{BV%es;9?$mjde}EQ;w&UMFzozh#Y#rQ;RY<Rc48QXy
zuTX(?f9rUq{GIKMDobJN`OnZfj3>Jq;>FhHM~cg8k~*u}N!aKy5zBiQeDvxY1(=sH
zZYM)}1QTp#$T;I-^NP!k$(Tq_OzQV*?+NXoHb0U##=)4fUE|cKI_3<eKjaDp+r+v-
zJ`~8Ok__y?1gmPV26?!U@0S~uG5HXUiK52PXuH-?xt#_+z4uNw#git^(B0y)Po4*%
zbj55qWv?bVP?nAqbGLtL-K{)4D7{7CzjLHQ(Q^sjkv-C7goWF+2(-OIDjwL>%*!X9
zuabBGFFAp_NduU4fz4ALmb!3k^WuW0+M~RLf_VNPiRaiptc6NDK~SyLg++K{JJKaJ
z1j4N)5x6jH9Ia*@n@KT0r_|<6xMj$y?vZ^V0=fjd<#pfOtoTYyjJ|juJ2A6Qt^*&z
zT$w6U%8=Jky{*u$@qyRY{cEEuG9FcJMC09cv?qQZ7o;g2GBIsz#9or<Y_D1tB*UcH
zoM%s#%`wU>?IygjiYJE%j~d2+j^ESx8^Hik2+z4p)3L`B<1mgF)di<VhPjr-+1l6K
zzMh6H`(b3>3^W*Iqi0<qM#^L+lh`P6f+DV7o<Hb~9#VkwdF{+Tb+G<B2Vx?8IU>QE
zg&f52NG=ZSpmG9#?msRL4Sb-^X!l^9FXk+U5vKgUyhmy?Xt>W}{mo9^?+MH#4bZT|
z1qUpoWx<zJKA}L3OL>>+9om_v^jfwt{aMAt#@0&X&KfTb=EILLT}etd7$HP>5}OYU
z(;_jJPhxs(KIwG)Lgbra&UuKS$q13Db7B2CVqXgKXn2RdWaFGEIXB*ssguT+X`XNm
zt-D%DmgtAhzY@+yNv12VE?D$7(v3g$3eg*MtmOEfU-8Lpk0>JZ9li%0*EE(Ak|-k4
zbQ1+e3Ims=QX-*|wnus8Ps#p8Kip(jI<4}lX{r$}gxJ{0#%2^H=dg_+g8=)3aTniH
zq-fSsxEt?roY)TIy0_$L;r^>1dQ}~z`?FmUb!Wo}ILB_rnRyjl?t?p<?LA5lJQX1(
z`~D;;(mxyVZb>Mba!AkVk>Me}v9EUyh;-1>{T<9*&<V8rLa9DDXg(?CX1?ds+6?ZP
zLn&tUqSN}fRnxyG_Fa51N8~D$$U=)?920%PHK`zAz?QiEk?*dfR_>RRe32{?zF_DV
zH9>neri1{GP}eOZ$!M(247U7S(Cz6#?gr<<X0q><Itv%APIFVuOmM*&IzHIxHw{Zw
zb2AmPjQY^K&^s@N1RYf*)31rHI?<516qL&D2goeti<B0sRi~hdWi?1;Fru|5bA`yQ
z8v|^=P1d1OG%I)v+@bt-Hf9uc9ZTq)OH?ps#5_FL0^Wnh$cC}y81bt_rsCP(*T3j0
z_C-W2->n4obZCdX#tYRwDN?EL*dhrU`pyR06hpWOw_!m9_=NYBrGr+mkPgGtWeM`|
z98L|1k5Oouh3l6T#!=!T{h(R(^Pa=Fw#JT#^dQa=8tHPl-++aW^7{=6a{Ek$q-LpJ
z)eEYNlbv{7mh6>AlEZh<OMdob%)AYW)5>_-WhI!1_J>IP#E7c9(YFdW&29^>I~qGU
z>9<%G=Z*GAA?zv_69>MwBz)w@1{dZOs^3PSjcLq@Mef4yDqW_-YVPLRIkUaMg!l87
zg5-b3sa@8QD5ZuDIoaq8iB+$K!d#DRyeUdm8uti24>3~w7=UDho9}_%S@=o94e~Zw
zsp-vkZ|<tiA)S0l=^#a-V5jsEtF2}D=A7)tP+grZl?{8Bn~GD%8-hFLaCWkt=~Wk<
ztd+xg^+;ax28Y^{x0_n}l;n!$-C{TwT5~ihm}vDd1y|B$vw?Gn6wJZ&LK%y9)h{PV
zj)ZUT+C7~ImHo_JEW~<h7a3DlNm1=o_-eKm`EGxIucL_;=qP>O*Fd$#>(RxVHSanb
z%crZq_iFSQg8jkmkUi*FXKUj3_SND6pNxk^$1Yr<bP=0iPhkuhrHyMENC-9y)bTw$
z)VO%Qf;v79f3i>2{Z$Lsan_#W&6aC-cFgwpzMSQ_MpZKUc*D1g?;_}Rmb=2pM+$T$
zkVkpF7T?B0rhxV#8LHbKsY%A)jM<dF4XA}Bc8R{SpOYKpm-pDdR*aEh7%^gD#T|KM
z<5oldq{9ZAV~-(HlK+c7ON<sa#<=PI26V+oWN14$C38$^B*?EeD@rwQt&8P!%tJDu
zYYF(1&%3ngf62!k(e_5G{-&>i;q#b?d>tCCB{!NHDvz98O0Jp+=}T3nK&In2;0hLx
ziI6#zU;oc80Wy9gm7-akI2?B|9*0d!K<$$=zrqxZH>=|m6D|>?Ztn$=nrM&zjNf0W
zc={AUgX7ZsF}VZwUpb%XQ(nf$EppGUzeQ~OUyAtuqm<!TWdh0;fA;^iff|76_E6ad
z)O{0xKCtcv!m0kWgy<uH<fXLN)t|rgfCxYofn^>*QXY5oV+GzJ!!~lR3jRx2@OT>Z
z3XmZMQ|SLW37q%$gLU<&a(W<O^8c>`e-CRV0Tq;G+Xs-^>fb&uQh`!wm7iig{J$Lz
z2Kp#SXxyLt^na5719?)oz$!QzaBI*;{HPcD=f}emRI1|%^S`McK;Z&hz|n|@;5ecG
zR%VD4NI%b^L;cUv{`n<B8VH(>Q=eG>kx~7#*c~8i557p>e=C{j^J8%I-957o#$z<_
zKR+O4E(+{QK>R<8_2-v&d_d~d<u2Su{%-{aC<D;+TEt8^e;V;0<@4-%d>~f?0h~aV
z`ETF+r;-2D$p7ide@EK?Gp~yW;{zU#%$z$8(sr+UEk&%C6u9r+wfm9Odex)iB9v4S
zw7+Jf=XeqP=G7|&VcBA>Rz9UBKd%?K0w3SCzqSK#*&Tj~eY1=G`J?%ndi&JAx~Uk+
zt5prQh1FO%?ZL%>z(+7M;6VZaBcRYt(Re_*+z2AxKK}8K<18Ei)9Ot@4dBB6I;fWl
z)$;u2^E;GBbmKpc4=w_1B6?^X(tlm<&lSR50cmnK_Q^}1f8GXA`Md#ikA*|$#lP+i
z4xR(1U#X^W2G&3C4X%tX1@(WgpaLT=4)1Rc5&_0#Jiq>YIty^Q^Z_%`T!ny=9V<r`
z6pN6-p%ndZ`E#P10Qpd-a^7j@@C@~JbrTvsByBj&0TVFsus{R&p?GKfS3Pq*1qx8T
z0p7jS<!UWfAK+$LP#MZ)Sf45nvD^b>&iW(RIIo~mMb@3i<Qg=IWMgnQ`QJ89(?A|A
zh2Zlz6m#z6mjU6#gQ<qg(fcxhu^XqG>A=W$zSv)nH%MWk%P;o20`zr4Zl~p5d{BT6
zl#GT*?>ht6h<s;`N1Qr<v#?1p)|V4-TItQf=J>M<hmo2BUN9CBz{eHzA=4yb-2^8-
zK7$@|nckMgAi6@xMzXy|z!u<hGIQ@13vqV6*)cfz&CU)uG{2PXrargIdiPOB`>>*b
z`=x7{-HIC344b)3*Dx7<aReZ+EI`cF-yZ#0gGscNYrfF14um$LA*Q_yRV>%8i;oZe
zqj*|OBNLLhxI1f|B08x0n2h+djFK2S(<SkdduG&@#lqVCH<&z!0{x5|N1bGtHP6wk
ziY=f4{*xRK{TO|0y8#4JFGg9PpsbrZF%CSo-o8ZkBB`jbXZ=E$yn1|Bt2iecsqP7Y
z1e~aDKR32pd(e;tNJQ@ec3{Y#%<YR=<DFf4`Uj116=Q}+?#@W9{egv#UBkdE^O;{G
z00ir=eN$uuXF6Q&1Aeb*bQU0UC45p;A2X>KQ&`_j4`chvP^q-v`_M?pW%K0=S4Qt6
z?=_f=pU>sT(8T%P%)`QjJECOIyh+(_tNnKM94S(6yZm@Q*I!ovyCy_^vs)H61v(w}
z218ES*3aw|6mMfd{o@4K?iDXKLe~4!5$RTgo<w2zBtWuPP{Q56j+A)K#D9C}t?Y8t
z{u}WB=JqxnMrB3fpDC&_{aVim#6hx<jPxjggS{C+f7lIrBKamvEdG&X?fo8CnX<38
z;^D*bxYosll5PmZj9X_=%km@kA+`klldN78Z*h$c5&%x`Kmu?intbNjsx}flb(^6@
zfFuu2d3MvG^aN|t&un$VcPX?InnWo=HgeY&PXnJGAQnrO^2>+CJ)fZlY7+yL_mZ<h
zFx+?_+ZVvJGT|UHDHhbCS?cgBTaA?HXjnO3A~OdF=t;<|<l01#f7_wmyyM0q=vr3u
z)}eqziIgw!b-v;S`9{raFvtp;f(i!1STRxZFT0O)-^-phr0!=})3sR5Z~QRCfk<*Y
zS0vV|;35FPtK-Kl<(mQFhbPa`2uV1f04(a+ho(?2;j<jV)ftmANsC#_PWWVQdyL&z
zJVVT^)L)TdCun2&xocR9d6zv&sJaZ3$ZQ4$J@=TT`<elcj-;JNwiF;Q;lXpY7zC8P
z%?2(VlOlT)d^bXq0sG6)WLVr?z^@gIeXafj5-rSa|C|jC`8yb*uWwZil5Kz3bJ=b`
zowFYR5Wc5GerB&y8*)n1@dW|;)|-^q3C`|)8ttsx-8Ge;<xWAB@cne9X(EOGuFqx?
zPY6EKkLO94e3#E@eoK83{ZP$QolZjX$|03VG7+jJEii_$WwCJx+o$=KjDrH<MA-sx
zvMF&c0hBmh!_Zg*TdAy#cwm6<#-w;uPU3yY2&J~gh$Q=aH#^V-k6<%d4!|8Vwg3>?
zcLPPgwsSY*-QQR)x1|A);&*=5t}&whTI0eRY7`;#JB|%hSk`-h0lXDB;=Wjok`~q$
zbpQaxcHSV~*P<IEfZ|QqP9=B<0O2B*qLY{Dbbw47ma>+b(_C1$`ZXTlxL^Nj1hC1C
zMqK9tSt8%`A$VfAI9;s=qTH!&VY$lHN%!%vyX*q&K0x?0)N?oD4_)FKJBP*I->(5(
zHxUeH79Pg)#O_qx=gyNfDGo2x#-+3?3`aVMEt;S^+-PyUhlVZBSDx}!@rr!kGicZV
z^zPrA(o)>vJ@-{{WkO5B#-Ao^Lp|4xC&S7J*v-_wpypN7K3uL~>hceFVALm4gmU~?
zou1I`2u3z_RKG4)%{BSDW}@3X>#!2C4j%B=U5&F3=Up6NbQ~ckS@pcs2c$#oHiHz$
ztVb**DmIhRD6Ot@L+gzK{!<f+EKa1-Eyd)rS*u>(1zPo+{xN?|*RxL?_JX$N4N<PQ
zfa***U+G0zIX_z1TWM*&+vRr!0N}yT7&x1AFi6-;lpGF%nfoE$2<SbIg{W)q9V5&1
zC4}uR#0rPIJ9BqVT?buYxJyj{;#C2{t^-ho+H=>vjE!;Xc2iUd(&4GNz7+sUo?za1
zH?G8_yG#2XKc+!mr|1sp6n%)tM=tbJp!k#Al8rCv+f*KR8@Hu-j5h<4>~k#JrX`;Q
zn`@j3cSSzCwJ{kqyc`0xCz|XwfnTEC)e&k};COdHIb}Z`Xca-m-AzwTRa5nfgehFz
zj2P8kx^<9n`t`bA?au8^^-%|r|Geo6zv!!4ouwl*7_2WGFf-<juBQ)1+{ro=uUKCd
z$Rb?uo43C=!T;rWK4sv4&+;b1T=AhDPxGC=gk7Yi4t^Nl^_Qoy>JI5msr)+OEO^aT
zwN*Gys4a))AGvfE!_w3o*S@)hb8b|o)AS9z%FB<m=6_fQ6M5M29AK9v4Iyxsa4tA6
zvr-@gela?R&U*JrWp~nj9aqhD%6)o1qxgrq+tr%)oZSpV`Oh&ub@q5=GOnqKY0|q{
zI0r>3e2zx0>3UeNRt#?X;`;Zp?+b4TTSBdsu@UUv2SI)(ELH)!C&Kj?F@Y-_==O~e
z_W6KbaR=#xuGdQE#XVJ*Ak0%B1X_JZy0UVA_8gnquN@Y2A8q7KclC#_2{_@a;>h0C
z{hTz{?hW8VkzU9wvJ-a;jW_$XocE;8yCs{-C81AQ&(-nmPMK)X)T1fT^z08>23>-5
z3gxuZI0;{eR*SX1?gxE|>)0T+!LCX-?7FZTGFO71S1o{rVEGXwiEBwi=_S1x@B@NX
z0J(%UPQm#C6ML|GXk)QovmyMfc=vNu5+j<Pe_jd^dx?}tt*-vlP=(IonL#|bZ&cb^
zv3O{^Snkn?u}V`N+pw#A!U+OOJyYbDU5%_iQ3hB&2Vc|RH*8ZkPf0F|!?#<5>OmtZ
z$v1Dhk0xYNcF2h1pKg!Qp@@(`Z9fgXiMq6~6>adSUW-xvVQ`Fd(Xc<YCQRXO8M18U
z<EemRwN>1kM!l|z`RW<Mwvwe~PN76k7;Amd?K2!K@vp+sb3R7Cpv$afzQ)j&!Bdf3
zV0^6HC3qc$z6!}8S8>N@H`9r$&_qiz41D$d(N6L3d4;FA1o*(HDLZ~nBkJ$7kqs@S
zcn^-#N%h+k4{pU(dCx8T&1sFNe(4kri`Oc}Ib{jguY4!1pxp5B{1CvxlYG!N#y(JD
zth3>ae0!T^D*iE?<G?TGg-!ka8vVdFD(oFK;NaptwykV{6n;)88i0AT@Oj$LJJRHi
zROm=Y(ovz^a`m}Vh?DG*!gCH8Zf(gOtnyE6>NT7+GKpoqbLtF1i%sxR-D2-xAL@~|
zAFk-j2;!9N822%}XlN*7tUr}2QA9~0M?(o8Jw^8se1_YoS)=u!tmN@cVcM7c+58Z;
zSs~0lc=e(JrGymzQ)608e8_VYn!+c$?jOtO(cqeXlMNobXR0+o1~-Ebl8}Xda-!aI
zCkcVMQA8j;y)ZP&QS(nRhu(3=bIQP0ji#<Yy_BLrdd;9R(>QwL#0L<e`~in`#M)`k
zJk`|N;UZx2Pl;9-<#RD=t&V|FxbSqfb8avNm2k{*%T5I3#ImBFAV=$=`b2b(h44wK
zzwpTC&cJ2qlZxTZcSAj)#B?yTx_YWdPz=)g-n(#L!aqR(4QB8${$g)tB8-b?;0I*T
z{{#R_G0U^6+k$nMV;J)DAhe%5RiS)|fbpM>rQ~`xmVW)xELt&;mJ024?QSV8nk1yK
zZR*p-4@fz=Qf472Y{-CG>-2u88cVKtBU5&=oyt@H7-e-C`|Y3E!b@A+YXjN=M4Vzl
znA+j*ve*tN&~o!<OCDGRg{IuMv(rfFKFuc(@9<%Zb%Np%Zrd&y$(c3uMndBFT%9-^
z2aus+_iuI8eVa+_XCqQg3(288(Xb9hgrm;AjwbHIm=(vqq~%dmPU?IOvdZLAI16ft
z=+;p;-B8b1LxL^9ULDS>M+o9{UzMlJ`?ho2J&O9vG7i8JgsqJxjWv?iU9S=|c!3%C
znd<jzMZ$IU^&&BD{@7<&DG6}E;QscE;yg8zUHfM_yNHC_^x0W@n0L(3N7y@HFJhh!
z4I=RD%~XY@Fa|&dw~6*CGQ@coKEKD$!IO-dQd9ZtoF^H`Qq|5h1jof&lm`(_h>9HW
zG6rMU61jS8>eRh{NSNpuJq4HXcz#my*A*Ynb-e<>iI<(TJZi`5)PGBX-M;Om8_Ef6
zgGBc!-j66PCC&wKc13XloxoD?j*{Z~sYMPS;&)w3fTwCJwPZ{=H5Ql|!*({3ZtllV
z$ZWGI^I&OHh~QS@2xE8PWwBg3oOT6&ZzOe(ot1#2*!Q&ieU5sDy4&)49j807u{W@G
zdU^<Mq;bqHH1a3G+aY&ReJ#h4*z#HKoCWo9+4B|wGE%(Qx##)Mxs^ej|4JHdGOyDq
z(kb+B`-Hf{y60%e-4^Jc+mva3KXGoa7b=Ok{m?%ldjZs{NH0b$7rhid?S=}>sHUx`
zew4~*5sC!>BSpIKkzM(OqaP(qO-Zaql79Ki{(v;+D6jkrpJ(42Dvi?*>k^jzcnHT;
z3vM^?x}Blu<tWKc^K&;&3O&K1OU9X%`o9x=EsZEYS(EQln`$>aO;ON0CfadJD2GX$
zseR9TkFDKP8}|hhVNI}a)H$`Ww+vN;uIXjL^;g{@lC(MNTICF9hNDgC<8e}Eaep`1
zo@`;e3O(C<Qk+1~qqJ?X{>?2$x+6v0<wr#=meXEL#H2FCSI}$`_*Rfty~6gZ+Y2Hy
zkTjDgkZ0Q#2s30)X6hrjWFEndm7@UD@H0Fw?>2qG5Nm?N_G-M2(tV7WAMHVnqD)MR
zI^_-d*<ELHFC}&V8ps-DZ!2w?Bd8W?O;kqZ`^iLXXDD4n#MXHD1{>Y-ZRbMkm-@5u
zuOxz3yJZ6~RV|4vWK25mhUkA5UbM@qC74w&`Tfv7EZ)Kr!2hVNtn;A!aGtt-XaDWH
zlhN@7lLdd&H8z4<m{bx;PNUQNdvTK23=IyV*l+raLd_gi7?@N;<;@dxe`wTG`a?ut
zw7MdFWcSo=^;?U>&gGLq?0Ju;{&YOT%yB@cMF5WS4a-2&7|lLybK20<(Mo45995a{
z*=cQx6P_R8ph~>R*B8}yI>)dlAT=DytRBe86)K*3Zv1lMC!*_Mnjj~-1zt*DBmxGp
z5(@wa3&SUK*}l^HQS($~ncJ*fz<Iu!7h}{Ar>eK?s|~RIWGCN1hs-c`UX57-W7<>r
z_go!(EFQ_&I&DR&7a8uxm?V_8jPIKN-3Ie}i2}JBX?IF$vj^x;!DdKG-!gZ3R+3Z;
zKf(w_{XR}!53EkP1TbNLoRpKhB!}fd*-TOc_nm{_pXFfASNp|snp_T-n~D)gkvZV{
ze3|bf;jn$ZXOH>MC4Ki{4Yh{bOHO|%xM7;K?i^t6*+KOC;<NzUI?(QRE#u=lzsjB@
z@A@0qVLn=0TkDN}^xxr6{rVzqU1cI$j(t66It7-3iP&F1Wte6um&=yQd?}v=hsl3y
z@dcX&P;cLPV&ioN6EMSs<OUo7*s#R-cT4Tj4PI9w7E86L+;+tzi;DMuY8rB9aP&5B
z^u7#`t7R#wRjbU78~UlBpb(tJ-U)uxrYh<vaq4Fc&h;xYrrhDMTJ4(jsYa;2%^Av1
zp_rN$7tJBxo0+AS<Qs9ti*;J9n6KdT*mm3jxbV}ytCu49)gkMrp(a{1Qc3s<*V_g1
z1Byu_kw0Ez#A9?7Ijx;C$x|#K6K~KM4yOq7j$$3gj*Mu)|H^KpWX`N+u>_tO4*_+g
zHe~N#bzz!Fqu7t{&=|J!)OOTD>>FuQ+>Jg{HK7Tz%5khb>Z}@Pu3Xu~gCxjcxW#O7
zKC0^aJ29!&2D8}D8w>oEUxr=~E7)K3JV7uTtu3#;xYeD7{MzTVhTfRK6dGu2<CRbO
zypV7pK2<~=n15?g?FW2WGK#>1^b5PyodNi=K3Z<+KE$gze4BR9#j6KTL{?~m)nNvh
z5%KWd!6MXb+gD+#Tm#77e6H7Ur4(oE@8ungl#IXm%L$c>f1UX>nJIK+l;Z8Q%No7E
zA4`GuPNKE7K&2vR-uj@*Wa^FDK4WN(U(MZT#Iw+{oo~t>Y5+9fIurrCAf52C;h1<u
z+%M2{N-S|;uyPPVv79_au~>z)$k#^Qk}N-oH%+q>8fWR4v<^-R$8SxRq?fFdJX`BH
zKP8-5WEt-mob)R$2@%wl)ZTs?!y3yp9^KiO!aFmGP+Ahz60Ba3^#%X#`ixm&)yFOW
zF1y4T9^Y4YPbEY>mg|GWOr_~pzlhU8w%L#hGV~EWMMKCV4ZKp@#H5vN>sjDuT~lqL
z!b*`@C3#Za_X45rGwEb}2f)7Szso>1wl~_{^eVx*_~4I068sEROv8Gg?qS(bggKam
z3TZ?~GQsM0^sFU_ni*U(AkZPeoQZnerHdBRX`sIhqQFDI{hfiv=v(})4M37_@Zmi=
zyM`&TE<(hZdE3xE>A{21tFbYnH2pZuDmSPHTE{~G1@1J2fX}76gd)8o)Wa`X=q05$
zExF7y;H>&2`$3XB0?9i|y<rvO@!W(*9Js;mZAp;G$jAq7OQEz9@QNl(<T!c;-hEG>
zFCqNHbBguF1~j{{h?LJ2xX{(RKRx8HYcxa}I502!0-3^~&Ql!)s{iE&eF-j-0hLT^
zjjZ($-|J2HPe*Dv0_e+xqk)eF{-@WJfC4yYl{~|V|9Kj?euf~Fb~u9V^1m($irs<q
z9PN$5b4tX2|98X3n+Ya!{JX<8oCy><_!&~S=RfZP{+$riz)*%p>fi697YzKpy6|tU
q{+ny}KgIk{YyNSc|G(Hb?-4Jtpr;&WTMMDUkJx*ucZF}Yy#5cHc+5=z

literal 0
HcmV?d00001

diff --git a/docs/figures/arch_4.png b/docs/figures/arch_4.png
new file mode 100644
index 0000000000000000000000000000000000000000..0c59e6ce34413d78dbbd0c918dbcb73133311edd
GIT binary patch
literal 74000
zcmeEug<DkJ`ZlS6l%OCY%@9%o(k(5efRvPUcPcFs64ISYh_rw(v`R<~Dcy~9H+*Xd
z-}n5^`4c{_3mJyld+oK>6ZdmJ_cHLAyd)OJeGCKy1T1MOF+~Iflr;ndBm=Y?;GL|u
zR$K@O7++0AMW0EFijqIGvo<odFhoF*3XG1u`AkWj&};n-OR_Hl`t2~h*Z3Lu!cP~|
z_h}wdpyNIb@J`c{Bv9353JCDR(#oW(AQ0|<kIL{Km2uIB5u<fo@4H{?xfg{+{Q3EU
zkoDT=db0Bx&%!v=c%C1D&Y?gp37Y{SjG{Hh_ppLpUvDPvK*S4q%PT+-EjAWn<nN1u
zpq#Z(*XqxVAlL5PLF0D1ee8Kxyp}xZCc+&bJ4&ITSnn-chwAx2$Pfbg@$;Is2#ZY=
z9g%ou5qvSIu}@^Si-8Y5Ez#Tf(m=FrVRcjlr1-XIYGmPl(cSh3+H8OPR|3Dj3t+jE
z9$bqY8yqYv*lMDw>pL9qJY}Qv=}BbjY~6uu)ZwgMq#=J<tTcrSMoGicsNwm7?TK}Z
zRIGgSBjKAo;?u-_FwC8H(hS3+HnpgGclbM=oMoI+XG^(LWyi<P-RA0WE6A?1uzdZj
z{xL0j4G){pP*Q*?fr0Bgg%WCZSpo*>>F+5<1qL()kC_y+6mHuXoY^Z;e$3{Pym$9;
zhq~$189OH_TaDO#nl>HsE!)(<Wu}og68))r470!bjk3iKTK8bdEjr|Sod@mj!{?d^
zU0!z#inXpZ4y}Dvo8G~VuOT0k%<61KkInCpV3}<n#vLNcvO9f}MdX6Sg6kzHi;JG;
zB~XOIh>IcoO?Rn*1eG*?+7R!R-BZrU5S4hZmfM@{3y<;{^DGeJJ`d$EY%*JXC7@`E
zC(@$P5Y59eNw$dZhQuIsX(D*Fej-X9XGZvZi14^<Gb192eoTx9k&ztnVH)y_G;{~=
z+mA%IOOWO3u{s8EP~M{9-XcIEl|m#EMvC#etII@(Ag;?kc;jKcRS`~~*Uw2hlN;If
zt~NxPH_N}fZ`|AQLEOLy_QKJ{^daXL^EL^4n1Q+^niY1p!AFyvg$@DZtzJ3`9TA?0
zq6um#_CueiX&5Ff%C}CDl6_&Ks_F8J6n!_bM6A<U9w9}(;>|-OuD5?h(C&9Sh30~o
zgvsJ(Kjr2^rHQTj(|?lYfZvUz7H9uQ;pB4^ayvBSZi#kWf@yCQDqAT$N)Z)`cTz-(
zA&<%PrTFX!m&E<TJMTU4^LwS&{z(3wjx4pGzk0CxJNhPNU9(4WaY1oaaf0n=^L#Nr
zFPandNQxg=U~&>wp}h2B`bk=!Qm_4Eyct_pXwruJ1`OBu!**liw4!aBZDU1pg~TY2
zCT#&3*Zby~7n@lw#%_;Nf;yWBC!Hs8raGtBCvR?~qS>J^`P}`Uh%1SEdkf9bm-h$J
z50W3Ze%yQR{aL|*`Ud`CgJFYQ!)b%cG}APKZE%JZ8#PZPrBqHvS%#e=@)BN-|BgKP
zl7xvcE6dZBOB86X1&rQ>3PL*#AYajk-ab%>iQtOBiRcU`i<o#$cL(`BF(foXZb&L8
zQ!|Y}(=8&qC9%zZ_HZkAj$tlwcKugf7GBPgGN&U2N34c~fJBWri3<6{dvq>zHg{F%
zl^-T4FRI*okwLHV@CJhs9RWi>z4nU-%0C#0=v?VHqqm~>=(V2x$)<R$m2fje@}=a6
zLX{YC+RUP#1GYf4wwRFYdfEp%2Ko^?NqUwS1<Hc?qiw?XZeU_lB^L+UMs+tHu*v8X
zyou56_fy9zS!+M~%=WTTPCQNVbI)fxow=ZgO21_|<Q#*{O8($;1SbS)d>+Ot(=v<g
z^E2)>=Ifa6U`}#(*rRe(8xs?db%!1a?h^m`bU)#~;C;OilKX^@{WypctP*k(G82p+
z`<I$%Fl#)?SIjTbV2N*)<z!D*hgJW2(5*7C_dCXH{r6fIX)kMEU|&nGX`kyynI6L>
znx&XO-<CobO_xyr<agx`yvb1LGhWJGv>vb?I5*KX!L=+JPaD$hde#`GA7`#+J*Y3n
zYR92rp={~UdC$0=O`d(+)NKTXvzSAgHRmzZ5F>Fo9z)JK!3p*qW*bM9pd^qz5^T}L
z*~Xg7wajiDcQbAwo-45~C6|lKYApu3YG3zqP$T2zO1X<gj=_>~yz!9PVSj7Gd*c_T
zj-3g5tClsxm5oOsAvq!SMzI~pFB2@&R$q^Bj#xYGIpI~^sG@iyaEx;N@YsqV`R<R0
zMq0M@ycK+TPTX@FL)|dek<HIDWb#Jxrt*&RZ?`cQ(C1fE2iEgu+8xK2N0)y%NZSm}
z^elf_ZFS6da-YvyZgS?E-1D()mp3HSt8*&1iLFSrIbWPwEbf^}qMcrvQC}RI%ACEs
z^kQna?`1RJP}TUGhUR`_XeW<OUSZuj!Gu-l4gcnxx!{AL<NTefMTI?_{lX1T0$n_E
zbQ3IJ<Wr>b8+#~AHv@eV-ikb;bur(F#Qho8NKxlE{Y3N$gJ`P1x?et;IZpOF7L3AM
z=eSd(#q`s*8P78we`R`q|1Q}J;$7BZ_DbqXE+eb<*0|+jv*I6vG2CvN%A?*D-G|!S
z<sD=J<smEq>l(Thx=TK}w|*03vGCUfR0dQnsv&Vw@TN0QF>uAb>>!bwrB;E2Zt%N|
zxE3B*ohY7#)vG_FbP1nLQjT~Ww~|!N;Y!!Yk;!PNwn&kOTPwXQH<o<F%EY~@JM8k>
z;kEt^3Mz`33@Pa-n##DGc<AF2C5wm5xckgbn$?z@0S!8K<hNQi!hT1{NhW?GR-{_O
z>8X7Xo79@5AoDTi!OJfd%B4F^Y>|e<F*Uz-VW+M<Mm*N8BN(f9%~f~u4GKrAFf^ho
zzF3y6{W<xKZ0^?Imj}t0d-;aFld_O{{j_q?G5_%Xq50Oa=f+`rt#GZdVpq+$>FF97
zs}P3g%Ajt|t7gCTcPBs0#R<d>#dU&Q8wWhTp5&ZlZ-z({^KlMY4wn&?tCvE}gO+0w
z@ERjVBhK=@UnV|{&wQ7;Ud}Y^RMwtlvG3I5w6r(A<+T+<@|urx{XipJ(YCIq=4<iO
z8Z*(~&B-Up&Q8!(v%&o0DwF+Kfgb`mdq*plUpPNAkNA(omaHG1eLh_kD)A@~NOMYC
ztS^82@of#qD-J^QRde&@zGcqeKX-eM(nUA@t;qKHM|*Vpi^{{h)f`YBgp;}+%KTu_
zl05zJjYOZsoV&oY`VY#g*em?L(mc})o=j*t<Xh%;#zf0EmF#}}x`gxB8&P{Xt-Y<;
zT*(y4Ekf)bock+_Je$y|?Use(5rQ&-T3q|7@y4zolGqa#E$E8-;S;{9)0Mjm4A}*|
z+5)a2U_mR#cC(1TamkOC*0_HBS@3Bo<}^QM1iSnt--2TbbmsNKiB@M-?7HomRvno`
zT?xb2Wf5mL=XK<>x8MDja3=U$T_5kc$l-j;-U$2=`8E5GLarzKS$^?A!M;PP@kEI6
zR43{F_L0tM%I@)i_sK%kf+0((b|1C|R?7V?(#li2vo>$lX;uB7sOO3&s=v)Y4K$zI
z9M3dKwg0LoZb{rMZ(LWNAaS$auC;t@u(RYJjH6F>ODM<V%VGP`=o#dwEvOixv#HZo
zN91XKx^`Y;%&+gJ*&b~}`w|g_#6}Sh!5RliI1ts#i5Q{hBq!yyVp`6*mo(COVD{0P
zbTP874j01hHbk@v^!iYY(oFGFr5QxD^pCG;5njG*YjQZ}8_HD(_(6fNhxzixFe<;>
zS?5G(j^LSs>$&?4d~M76i3LyOCV#Z)?V09v0Ak-8s!1Ek$ss%huh9@t5bq<Pf>(&(
zTM&`>?`sJ}It1j4_mL0~{7n&1u0A6Ve!{=Pz&HGw%b&;*eh4?gZ+PI_B@OBN(<p0c
z$k(rt48Y$IM3h9OrNK`neLF)#D|-`b2j!M1_?753QtI{y2m}!LH=?v6^#-{AfT^;Y
zgPNQyufDY<^D6^uJws*}OB?uo5cpkq!Ana+hgak-mKIj_ye<Ni7oXq-ui<aAP?BGK
z#KByEQcdm|xu~_BAvq`WBj!hxf*9oF<otF9M!bq*5?7al-vlU492{(TSy-H%otd54
znXT=NSy*{^cvv2>v9Pf*fln~myIMKCa$&Nvr@Fk!^?k$)?e*<UZ5&Lkt;pf`eWhpZ
z=paBz2_NY1*X45>x|seml9m0{w7>*e;D2FZWq!o+_uk-Ae)zk*&rDqmE!4$KEy0?B
zF$6hT+4wI$|6hOkXT*P9srJv6tZbbBap}MQ^nWjXVQ*+BYHbO|bP)V!zplpp_aCn=
z<Y$3T{a;IQxy~2wf`t~u;Ai=}X@VF{4Wdb4A0L>CDJX-Vz{=oXh-}~=y33#N*FGcT
z$u?*R2*L=`Vj{{eh$~|^qm;$_>o@trlS*b8=>pIgeF7?qd?<bJP^pc>aBx}IDjz=d
zNt+FxrSm})d5#%C`Bd2s#?vf)x|=_iV(8b>JLOn+&Swm3cc@c!T;6}JJMZ*6pWJ^N
z7aIjv7y<FW{wazQ#tr0?8N=}r`5%83u15>2n0e=gfP_Z=fB*49MU@&=-^jwn`yYQq
zK%!oA!TPUj;P=rLLBw+=sHXSz{a<%Q2;|d7{_nxSN7bHrH6){2V^C21eG(Ui$xj~y
zBcd5pTfZ8%QH%Ovk}e&z$Fui*eoo~&6T+2YJJUqE^j6~b<;=-9P^3mt#{~?<{uzM&
z&OKH=UrbVdHJ`U{=SHmSq9sXPso$w)t(CS!(#m`w6`*EhlqF!+r0YoFkkLX;3b}gf
zZ+FOTjoU_GX;(J~t2getK;pXlTUD;dswtRgYt+8e_0C+`fOcfMsQ*(Flh5tRe5xAX
zES9>@OdeeA#EA}=)cJ&{#KoL+^}TnO2im?_j?DJuC@EIE*ygC@Cycu9k8u)gu68-E
z4nr0?Q@KR$&?}0)Ct$iyOYZ-ZqVDV{*L7w5&3d%O(AP%S6zgi*N3VXMER9u_FWJnu
z$CD}yLnZdM+F07&tk=*|P*80BR@ISydjm^TVUy_EASy(?8pcSd`7f8&YZujNeVl2m
z(lL_E^MqBtNJK=$h~NVa#BQ$j?gEdqdx>!;X(VHN5>Lrvi@`3fj@Nl4av7(4Bi431
zi_e#a%W|8Ky8RL#n^LBFo<Wyp()~#zQaz67<vq)-#v+1=*b|!j3$z%@%m+fZ`6pZ?
zCs&3_6u()hLtQpMN{@#SbHVsX_}+BSTAm*5&Q-0~vP(P3RWD=m|FvOq3SdL%IUDyb
zO^<+p+io^M`e4ymu5YP5A3SBtdfhp*KQ;+lxX{7m+pn*8&AMk3sy?XXYqWe656Up@
z&DvVZPOYqREiccO!5b+vH;xgNkd&;dIUOOc7O>6!aDKRuYS$^S-;jcL`}XasL&U%;
zHG)FVboSN9(EOIFjzq3EvkTNYA|!%tY7bQl7-N~WBDz0G%G7zBM(`#tT=`mk6Yv8I
zw{0R`_=9?RVwn7}INX}c`gKaE<3@3Q&@7~Pb8D??y_KP`TCdh~Ilo-Y+zfh5ZW~)+
za?+qRQR{Bl9>-ct7*-LKxiLhTAe*n4`Q8N2xg`R!)z02|Kpj&);qg-2^VG2=nXi12
zQRTGV``W7-2+3^;b$r(^?X#%Ji-yD@E0npMSkSGadgI3}XR+MCfJ~X#gv_tap(K&A
z@obGLpJXIBTr=2rwiY_cjw?KDx98>9%@ykU8zlPk)OTg<7kfr~vlS8x&vy$w;|)C=
z|McBvT3hFJP?b+!$}T~$7J?=5P-z@9DWvc#rawW2I;|!wT9jZBb4ra@yNbs~{mJ|8
z8|r?%*Aj?N+wO|~du^n`_LF9vht~VM3`T;QmWztNlKGqytbt9NcT;CI?f=9Ps*q<*
zWl+gm?Dq|1u+82#S?<q|ppwMq@i=i=2`-4DS(8uUr=M`!4v800&ypo7DOJqJ<J)me
zZxbxbO6lhf_B=V+?wsdPs&#X1iDl7A%054s!GYS&NUZ3Wnf+!{`SYdC!%-puHt)96
zEh)ZS0&)1f|K&xlZ1vRC)JIk>vpzZ}r&lNDeFo=er;2I9i0sR9T07bWFUw)o9uHr9
zF=%>l&g<+pQDUqR|JXDF%B4Zf1_DuznTluD&bZY?t(Kmio|v2#S&r#qy*IMr7iD_;
z7Jo7iy3^x*g^szTpsW3Ix@~hd-Fdxx9+Z=LjdM0<q0=@r%@1I+l0MJ*BAw<R2z16{
z2HMM~KY7<~PbC=|yj|fjBMWQPUXbl;aILi*`BvO%S{3NdjZa})&}(hfnPLemR4q)=
zZ+w3@lu`X_15~?tdUR+zba}^{RN!@-J(=rlI7+HB4N}MU`N@LN41b-?)DLcVYJ&AB
zdZlU2*m3I=w=L=3LVKF%pXzNeHv!Ax(m|tSjy1g*I~o>!vbxh-#!oo+G*m~;b~b+E
z@FlfCM-)O%H7s{W*T$-v1)YuwVIjEf+xGMA(9>7jW$Y~k#F5{oG<__!xe5l}Gu${$
z>SA?TQS)c#d611IFc5#wiQD`H;t;6c_xbt!2?3J^ckTJv8-I3h=jL6rCD~wzi~DO*
zWyz32`vq8zSyQ}?6L+LJwlUH{s9KqTanEqEQAdJuS<b*>*%9x;--L&UCw0J8<O`c+
zy-I)^(E9C8A?vd)&q?o_cp>8Wm@vKl`Rc>%g=U`4=wNBfX?c9Vgz8?Lg%!@;Z}z4-
zEBZ$++49NLJ99@((G|UMJEHg+3PPu)(0-`5pQ%hF1&qW_E4INyp4oOG>78-Ght!JW
zxmX=NeR~)=`+|v21ILX4;|1BW!P?QycMP`s^NH5HNsha+b-!~}KUnfY8^)#^gX9!d
z^c`WZ9k^qM6Pz@qg%^y-LqC81Jh-Q+LmgXT4@ty(I*LWgKe!1}3r|uBvRrk7rX%dv
zV;w%@!^UA#*-e~<JYjzZ`p?qJ3l^N)yf#|d=sXLUWe2g1G;5Coa|_W|N1(O=(v#!P
z6Yj<CO`fOY-OuimcKLn`ZB(COexb@|T;9}_xi6w)&r*Bv@h8+_ES%j~emkkQK+(R!
zBL%o#VnQ1?;Z@eKptf1C*5$m<FDib49qK1gNZ+imMv|L=Xc3WKW%Q_Kvx(RiCSyLm
zwkH8HpQ&D#ulQ4J)oJ-i2n_NLdx5QMunn7({~lk}%GvSy2lvrkaSeRug5}*3n%q)r
zIc?V?8_1gW>m`W4eX~w);ohods9B}_L)&a=u>lJ)xxulNwBGp0f`xI<^HV;*@RWw+
zT-JdVaaBCC$$D?>g)LKm*u6g1Mf3GNQDZmWSfRv)6qjjqe|xCqN-qoo!(RBRwVgyG
zKSNeP<UCwJlb)Q&(U9`75NAN`cxMU!xPq^?!e;8pia>XKP7Y6!OqhQ2{c~Q+0re8t
zl%_oCc@A|Q8Lz`<(-Ph9KGG@8kyo%nR|jOLy_)^ti!5J{gR0r`)@rO0LR4c<s}zZA
z(wilhwCxsO(L0w?PHUp+jYY&p%^7cZxMeu-bH1dGX6**neS>Ld$WKifwaBNVOIsRp
z2Btrj_o(A;i5WL7ZGR9tt5{AS>|B!_H6?>#N$HI^LDr`0VvJx(L~kaNc2V>@o!vLT
z-y~}gT+g##d_Cv}{{C1bwrlIpaG80cxl+P|&|Vk3iIbz{!eUmQ<t2!%(P%a$6%`4L
zB%z~BtI{Fx*nqdze&~y<URzpkH-9YUB`y#~0C4b@SK=?+i~P1ig-bwS6mgT6_+ic`
zv)^Cl`4>V<Y~{BNr{!bCXdK**ma>amqL^NOYv5_r!FN92_dGXlntJ+-hwI43M3F70
zA(>|Ou{X#$_`E6wnx-jr%LP?yF2l)zd^HN?0}6e^{+}d2IJxbQIolrnp3P`l9#x};
z%@kNCB`P$G4;&0sYT_lD&L=<*`jix)3HEIZg|$ZyIcXN=>2l0^p_@`GWeLyvj;BVI
zsa{w%2)P2IEVlf!qVx}ns6_~@I-qp>a1nyN-l1v|qf@J@>BMH3DMCEPco=Bb9H=lu
zUI@jo=#WZIOyD~&z><TsF!(|0DH-xH&)ek6wqdgFA9f<)bE=*ZjVvlKpXt=KH=bsO
z8V(i^_oay-<13s%N(&Yo<MSD`<xa*tfFGNWRhB{f#{#Qr-t0Fu&5ok(?}bI24nu?T
zesV<q=3v%R$qQ6juxf14PVGX6jTy-CYy5n~%gdXt<brwadCbDouz3~)zl%qo)IkE6
ziPLj&IXEK&TXnT~fBX##%v42ol!V)coHL%y{b+}K^XSdA{LPu+j3-=LU82UNrqc_1
z4PyXq>SJk2r)LP7BmLR?nGDv0*nM=ob5*B`^G0`qe;^4Fo57=*p1H1avf8JkrengY
zvXy;YW&Y!|W1q#Hd7LZ{rYaQ@EM!dr?G@ckn8a!Xi`9dqwc6~jeE|pV3l*Kgc<@Ij
z7e>)+!*`U>(rX?nG1hmc(E_M1sW&gKqRPE#Y=6R&jCNdDYV>rcSD{#%XKb~<Z1cxz
zARmcCAReUMwGvY)Hil+5y=2Hr$Su6Wo{7XAq-<*($lfx8YC%Kj8-TitKP(f%A9SKX
z-M(_$`Ze~Lm}sO+`MOVTRYTT%SEYACg*m_?X3R61SE-;V9Ml}dV-|kz;bt&-juc6&
z5Gv=l$ee*rQchSsQLHbSq@)?hH^xnVmR{DbS>pdVK4vhyU&|0b*S6fG`{SU;jJ5LQ
z)6w7gFS)+kXqMDQBtYhav<$U4)-*+`ICB=e(%*`j_vP51G>ox#^6Qfh3~A;FXa-g_
za5C9;U@HYRV5<g8tB-rE<`&kPO*5%~on6Ig_Q>DG&?=-ID5xx;b&s!j1Dr`t6Ma>0
z<vj~Ws$-1S?w{XsiF2*dGMkwRZXMxN{!n=)p6e3N+ZR+ikc=Ptm%E^*tP7xIu?!PO
z&fW7ay+aQ3{$M%EhQh=bd1|cNnoo$yM^o;|g0R{`EB8woyyvlx?jiJM&%(Zz<^wZ`
z2hBXZKV?VSq8SEAI(Jt`2KciR?=^rgSC4@Lt>VBB84+@+Zd-w-qQ`>(2PJWio`#q!
zy*mixQ~mf*DN7bxm8t3=unLqx3EaC$eN9VWJomdqamHAw^XulHra{?l%K_{dbaGUA
zbu#mgQ2SA*`47%BOp#bgy+kFKFHOl!pG#&nNj1uuGn+xa#20vhpEaCi^#t$SF=0@=
zypDOsdOUoPgFo~NOYPx<+N|ykEZ_qBo@felCD}ko&)}eS?5U#^t$dPAh0)KY?Sw}`
zo{Q}q?ge1<&>%vVglxyAt~gfx>Asbx4{f>J4mJkQgw9X42gTkohiH~tX`L|3Jt%N-
zjOhha(d&%ow$-z!E3DbmSeYqcaWG0-)=;$*#ivT3>&f`A49&oEPMEY$&Xyr6)T%N{
z7;thhwK6cs$gOhllxMVcKBl=_pf*jr7D~c5Pp6m>5LgCmDHK4s#->k!EOJh!dh*F2
zF~tXhk_~%cEkmQza;YaX6qL8kp%WlR%KfP16zLv%T|P0;DR|tTjxI&AJo)|YjQ?_P
z=GTN#r;a{QRV|PdLw4%tI}%BNzd4)q9BoYfEY4nJ{$5#?XQh~~4~DZ`v6psFFBC{t
z&l%v~Y`~XJZvE@L_P}{LJrN(dUHHm5ewS|xG7dj(VsN)X2+g!6I>l*{+7rE%!Q$ct
z)9^EmGPA~h&0OUiMQ}^VntZC@>aSEG&lM=|+dPmSXhiOil6o{-unq)Qjm-4=elQ;@
z5G)ule*!s6XYVYP8z@USun*yQ`ON}b&SnXIhf=_O_x<);n^e!&C>xBoPAGL42v?hy
zA91!+oO(~?F)iOYRB9R-UpB`piSJrz>uoNI>V7ijJn>s;=0UD%VJJx|%;IEZ;%pw2
z73Hq7vGbKN1~gF>z4skK;S*Y;Z7ld_|H!r;YAdDa4^!(sPP1Kd?>Sx^K2#&?g;Xa9
z2YVja0qffWNG>7nt&h(Gv)Iw8gu|su7PBu^=YPywcH=6mBHCKY7LIr@Z{4zA8HKV#
zdtFJd!!#oZ(@k8d=~VFic~Dpn@FfYr{U0K`x>26gx*a}%C%6V3%H{hxG|!I!<fw9e
zr~Rqz*nuXT8Px&OSSqNl=-<!SI~&VjI(4L3<z(4xU3W^bpa>uoesHCE9z*v`<vhsr
z4vTpe4WX{v^Nlc*9`jO?8i`1Wn%BE*tkt{yF9Wr#Na2gY*F;qQ?FC>Jnlq#$BdR$$
z6e|B@H|jjhwH&RLez#ZSae9Q3*gxC38mJx{3Bya|bGBN+Y(5~wA?9R+EsI3D!+}S9
z;NgDV`55=oR{w#2?ziOJ)Da<CM;*;ox9twDVq(FHg5T6JRuyT|<GujsJ%na{O&Kkn
zDsCoar%5>ZO%&cj!W;C=d}u5suXpp0qOPTk@m%I?S7l^@de{;BKh-NW1UEy0*K=v|
z8p8|Sv0=841}XDBy9gzWI*Qww=B_a%#ZWjm{@v-MB`Vll@bu_irKFZ3gTg>hpEC^G
zlLOn{x^FW{=@8eg*+9<(Zev&J<GocmO_*WMv?dK5%r}fXcJIBtKJvD0v-l=ymn|<>
zC+>oUs4eUdp7B@KaU%NTeklK7sr~7ZBmIjnQQVQ7(@BoeUxRAg`opOt8=xz{D|nor
z++9%YFrcb%JN#PgNMjNb$?h7v!m-9JqH_y2<T|a%%mB-X;x-<G9B9}`>a3Z~EOvjI
zPP%QPU3ZjV2dl<ku=r_^=uFet!STAL#3;k{G;7vap142t#i~-FBlDXWIH>6;oHIb4
zL%GUWXlTGK5p9WyP%i0@{<{IG6(v>~Vn$qua{PR;7>Nei_A#Hhjd8N*Tw@7n7n<0n
z^URGSoQ@8ISZc#S9hi|xUvSO$5u=Aylyr9(eZI^U2TuSa#nQLLo{2=OkaA~1QE#Zu
zvyR(&GWtG;S;kI}a&o!ucO=)nA!9imm|?lmH}fy|Ck?K1X(q^}FUOs3T(V?d`ydUu
zb|$3s(4@(9I37AB^E&qQ3u6@8S-pRS=M3X@rLXqsz2-{R09UedhM{|M`5-Q8_0bjf
zYiabbWPO2A;*JX+x?~h_xxq(~7PQ&$F2`l0Pa_sFVPYLa`|&anqK4;_cAW@;BT9j1
zs#5l`y`W&O62uia0bV(dOK!V>Wq5RLW>*ur{FOW!jH8I&wDiyC|MkiRaAwI|QA@~I
z-u|yKg%P}!F~TZjAosHW_1Kpn`T}KOfgK!wmWA^l!(9`wiuA(M%+b0@*XitEV*(P1
z3%;hJ@q+&`-1Q^;|H=O4od0WQ6)7hJ2%0-YDQ_D$-gXzKKKkQXfYI_$gknhS$M(}_
zP1DmSG?08->`SKti|&<rE39IBn^@}LY6m6@k+5wwhaR#L_Hzq);5C=6PFK__G~ABg
z@$&fzp;MwqZ&iHmWB=RMqc1YeB06~h1>#Q9c~v~YGe9zMkGd*Dl;A5nJY&SQw@q$H
zieqBUj^Va<Fdk!zN-L_y(@cY+5Sp=k=P`$7%H6;AR21Be8gPS&31rGEbM+P`41m&i
zH)lQFNylll{mGLHdn@@Ys|Qc<<eWOgMm2mgm~<sD>mQ1mp9y57-XEoxB@Y79-rX+W
zv8isD@0gUWSkl%P(Pu7*#z@v_zIWcFprokIEnQa}xHhR^=zaB`zidA)1FpSQrr7G@
z@+Of}F$vAqJ6Qd#{6{K2+mB^uF-aw^tr@rI3%D|OoM`&V)m!Ix0;<Ww`CfCI4>EmI
zKKYKSA*J?Cn(`Ht@h;)_zx#(q4T>CQTjG?z(^rH{R#b~nmVa}AedUY$vRv(G_bdw!
zPsKGH#jtQKpLyx#wHpYNlK{`{@ovt%_8l)7@}t2!fx|<p)HFYK!#IR)`pYbi=_e<u
zI+Pf>iM{(97;qWDs4+e)*Y4$AiY`XxBKdKzHsM>4yTc2U_;ftQt-EBq^5xg}MnIy!
z4JcbY2<-2p;@u7y78rz@uLOvtl&$?t&Bu>F(~?%N++t~oqK|Y$hOk|G{(n-$;4v~3
z$jv5}@oLUjcQ~fs=5m-0^b_qN&xxtDdp@^&K|24IZt<d)x>!td9uP=c&^P(7jW*0n
zc&FGDudK!#sZ;GZeI(K;wT>a8gJ+1jbXwWCql&p7J8AKHZivJnNR>><8UMPPN|>N`
z+;%R+PM_+wr1H`It*zhakX2bUk#szjT?TZR)<oOwZj$Svyo43OB1iO0@m$Y^JjIJ@
zI#}#XQToB{>}di0Rcm~?;Zv@av<)P@y-7}Vj85ZKZ`{8((4_(o&sjNydG~7HdRTCw
zhth#O+u}#WTY9%zAgh|`C=2F{uU-f`ySEu%4}*Y626xO&th6^*Z*7oyFX}5LIj&Y7
zE7ZfJw4|%Pz(VM153J>?GH&2%*nTY;*DBAD|K~{&5LrOl+s1o5c4cqC*TuA=0?G^<
zZ@cze--sB_NXBNQV=VL)Ds6`9i_6y$(R%-NWbcQ-MxL!R&Rkn`IfmFODP6a}1hh;}
zfios1Bbgy3Y#N!mbX01f-0dG%*nroXWZ!$4uGaA2gV#(GV&DO^+MaMost%1;K$?%e
zKJ5{DVU3IrTl7xQCLZ<ma^rr6C%s)58}rqqZ4!n13Kk$o8~4h~7Fzszl1q=RZ73R#
zq)iUHQ!t~y{p0l!aP!2(Rk+GUULB~EEWh$o-@g~rDdG36if}SYqfn<7O{RkTppX@k
z*tPE>z3^(#N*h;iy7mvN+W~^ynjiP77h}Hh4+Xr8bUMd+T)J|Mr>Iv%`!mb(?-qGe
z!z)nlpGwzGBHX~TA2{<zMMl(x(li?xyH!K|2id6JQZfF6zcc&=pW)#e8|T$+q+-2h
zhM5D^oSiF&H+C9Ro(@B&u?x$Te7<sX&HqJ)AS_mbWi5>CrMO<!AmN=eE{NGg9V2l;
zcBB*UMpuvZQ>mN*CUarkzYjzXjOf;ERomc|JCJK3g%kggYHuuCGW)4S_=d_fyZGjU
zo79gl=d(Glx|pvY(@Pi+Zuj+5^jD8rR4m-i1TCZS57;jGjcpBCZLfICF6H++Hi@Vj
zQ$?Sg^m<juCqc^;^@PdD>)PK4yyJp|WM=*Rd+c%~3pdSr*6OIxb8Gu~whO*VHjB?(
zKP&>0GjP?aUSG+-z8fgWukq1ow>-+1CBg{<SL2Kdoqn%6lM(I4brdXm3zmg%)UYYw
zD!Szm1Pt$1v%7N9{2p-qgQNplb>IgE!xQQJNr2<=@Fj~2-+L)Bu<w+Q_^<QS9znq9
zPDMk~X)7LDt?V$pu+6vxp3^yoj2iA01_gaxTHwu2l_SB|-mJ)oTU_vwyI@sn^Du?C
zdqA3xMD<w=2A;NgRkoj0)jv}U1gxSAil6_L!Pbu>-4-Ia-JGv4C|>lWM;_6K?M}>w
z|A(VKAJ|`;=7UGB87<@X>y^x_&*$I~|Huq^laei8^t1;S%<|aZ;InhLl4z#Z^#&k#
zi^0tj>F3X@%XF>1U*`T4d+^*Am+g!SR<3<G-)<Lf5ntW&VmcY+-BZ7#@IawX_J5w0
z6u8pDZI%1iYvoDEUCsS=MTmd`XMDRZ{D9Kdd3*VMcfF+0hyAHv-Nc6azQgq@q74Ff
zO4eHEpSg+%-YBS?mc!Ia+49l!k<BvUZj^jRG%gAGq@(t1*|Mjf-a0Jz%dJ5dzR70g
z-?_(lZGe;RaP-KPEqQH!ab7>C!T_oBE77`^Ug%`wtwBe^i-du@t~tM7&_D0e`+h{8
z^v-<y>4*KoR?xYe>y{*o2JM-)UIihhSFd1m4G;b`LBO3Xuxi}D4qCalK-norz@p6v
zD$j2#BjpP330ar{kdHt_z9H2_^5Ol=7MTi}kLvWT)-x|nrsFc%6#k)h3FTZQR1B#?
z?V1eGsZlI7d0yBcLVG=o*BID_q18}!?W-w-fo9e(&=@wD{S|Jv)C0*;&WVujcU<fe
z0d4t^XS4k(c=l$uHRQb~k(eNPTTk@UOYl|m)F@_}Kg5Fe|I=sB!X8@=E5F`fV*<n&
z`@~znzX^Sk6s*>oZN}ZJp*E1cS4Ya*K0p5?+av$ea<C}<WIOdd3RE|vx83{3RUt?9
ze9rt<%{QMst))yi^c!d0`RR|-R$)6cJM%#(2J}B=;@J!{bm}~MYTe(oz&Wvn?Z2aW
z3I738%Y+4$Tw8M|3TOb%E*I9tc%C0mY=IX~Lm%+W4jHFLt7^N7zrRBtv1sz~Lub_I
zI$oNi>f{u(ngxAyXs+l-(a?Z^uy?lzTYb^-XJd6d!a<K#8W1)ejMc~Mbwb_CE=bp|
zE=+C;W+{EVacv|AIrVdGv4-D$P}v+7pLNH-;w8?LiDfpDP>W$yk06PjIkEz2$L;IF
z(3J1Sa1}M3J@gLNZl}<B6zF2Nog8k@{rR1%n6JU%dUm*w35W%n^w^IQ+~|J?{7+>F
zBE)909x<NNG6CL2rLo<qN5=UD+MStliJUnFbp)HDhyY(=)1~%d)6ntP`3?v~54X*>
zMl%>mZ-GvuSfp#E!#DGRf*!!3(gOzQ%g7AUwbRyzf^fUyGC;(B=>UjyMW($pN_AFi
zqfk6NJW4qCy}SGcSj>F4gq5p=o;-`(_O7?hE1R=y$V#n34@VU6jkpd*K;tt%fBrn{
zTbh+oH;{<f7xw(-Cj48VAp*9B*~^+)RP+*Bb)h~+-vP`|E8MGyxvZl2UALsFoK|Nu
zW3?lEP;X_uIp~ApOBd8?nT07N^Fr`x<sY+CIvVTbvwlXemQUt|&Us|g5WucO1<HvC
zaDSmuh~8Cx_2xz*WY!Ghv7i41mu577AdAJg`!LQhE&-(#u#idiy}NRK5a_O`M|fO-
z1IJ7`Apos@m9Weo={%#nzKe?{-9yfB4_L1~mVY8DnkgPcKoh76SfcVh@hj>VR53zu
zu(VJmxpL`!gR;8&Yh!GVf0T*%obpC1Y()V#2SL7p`G}az<vpdHJw5<n(CmbAhpO*~
zDN#iaQ(Yf70p2Cym<KSzJf3IOfFbU-ApgjB4*JVupWJ)7uLF~=BQQdhI?)FID}VUv
zarzo;)YOl65fq{~O#AcVtIv*>X<BAXQ{vd$Aa?dj3eNg1dq2N0MozjZxI(@qa#_Fc
zr|naBsa)YP@6YQ=7rnVOUQ?x9pgF|A+J*jiAP)FIRa6acuLhDJr=<MBnC%7>6b8UU
z;C=eYUqt*aMYA1=18<&&-UA?_oV#FsQ#+Mlk)F<89UZ_3YP!MY?vu@CGPc({ChFBL
z#qsLufaEO+{s#0Rc%-CjyDp2k*9N~44U(x&OE1gSGf>k3aSc&Chj~o(`Ps4E(T*vc
zBEo#Y{jAb)dA=k=Zl@a^uk4ev&*qIkZ`1EK$w-B33zDStvn2s10gf#05bppOzzoR`
zq**DiHm4ii?-M8gwZ;Eaym}j;o}FwfM5e(d28$Ma$%ea~o@YwHMPpaWNQEfo{lrc*
zwI6VOaH(csaKMc!{_Hz24#7)m!I%mFso_PgO1xny5K%mu;4<xLX#bI`Q&)?EimFa2
z@AH@V7=@RHV;@Ve_bvzT04$B%X{BLXj5N>@bx7a?syhMn^Eh9ux*W~MMQ&5uy?hBa
zFmUhZ!;a`lpH2<Rm;;dq0<V>6WaG#|po|5rlmw!9(0*cV`@B8df`9H%D|wAAPc(ox
z=u8OuT)tB;gz6wUs33SKHtun3IaFpIdxu$5jo0xHHIOPrB=Oj5I6T7+Jjst~&^-G6
zaT7|LH~9ePsalBRnrZvnd@%`$)(D79J7YBv6(z{YIM5D{*SwJd0uTN{cDa9+6`tG#
z%wJ#P_WC^}&<Y3xtfrbo@I4A3t%CdCFE2!7*&gP{kNdU+Lm3hw)8%!64Pt77{f|-a
zJz$>V@BpR#e&&*$j#jDru@e!yQE+=Amjbg+EhFe()$J?~C;%SZ(8%`aA@k8+*YAN>
zl{G$^*L8Ob#$prFCy5X?Vk|L9Nf|K3+1&x1SkRuOeUtxNK0t8CTWr<iyV)x!B&Exa
ziaO(fyu5TSNTNWiiiX`776r6vZ81zQsU<@n^km8qmK3)D)_XN5Y!Uz3UZr9I?d7m(
z%X)3EG-<>bZpW+TTm3Js1N)1?<%y+MV;b77b9d>Ve-zV;J^VfcgOqqa4vX8KI1<UD
z$6@8(pMb1*TPG2sG)dqbB@hdFRKuIPMkSG+Co^O{xvDItqZP#mKUZK;6Fji%s3uGR
zgh#)eV~a0^dXvcBfQoMgvMU473H$1K?h%>MtF8cOaIO|UGT+%fW|G_4$-8T$cbY>^
zt*l1~v2GYXoXa!c4?=mjurIz(y1ET|9!59Dfh2AYG&t-Aw4CEnv_UJjV}Nuusp%RL
zdON@wH(K)8tKF{mM8PH#v~W5)0m76yuu66l?)&jajv)ERPEJmG>Dy^eoTkod&AQf+
zzI>*(c22#@7Q4M{OHJq|D;w+v(w-5)IPkCz^R0N}6jteK;MX+HJe_}yNe~VN@tt@t
zH<{c91?cvS{tUo%f<8Vz8P7jSWO=~bbjZeGfx*=C(;9)J2XRt6yJE1SgP#=fki0(+
z(P4*SjmVDHE;q3sgaNT){A(BE&Xl|Po&)99I;H~!$vHs_<Kre+e>b%XY-;I<E#z`j
zzuoP^q5-XQdN?$Lck?5FTEleYTYmLUkBnl5)a{qh2F(+7N-hBkJja&0CXxivA8Y45
z@uk~!SWiY#-Ix2B;)OhG<KLCP`gx-SdDXsCz}RT&1qilUV_koFurMPKY)!Lfo?Is6
z71}fzU5@TeAX|IS9KxiO6+Bd|WXxa)OB(VZKPM@kX*FWsmN%a`{Fz?1;*h2F?AyaL
z+yYg-(Q4Ncfbb;XTFOd3OR!9t07HFjGi5jUH5q+x5#@T40W82*YSAXH_mKptvc;ER
zf1tt7N)Rk6mQq~-2xvyGD<p)28;H#z$av7Qcihb9ua>%R8hQ%<c*qiyCJSLA$u9Ki
zo!!}4>h16&;B#KPNZ%myWdfK1S7GVA6R*3%qq<69hMYB<n8X*);2nhq&k9){W7V$P
zaQSBfP%^?}-;fI>WME`1Defg+nYou3S+Z_Ry8oReNcx;)APW0iMhMp;zzX5$_%v~H
zOb)IG_zL_KAalkvlfGdj=hBo8L|?e38=OHU=sJ)%ep${HRSSd8pxb66AsDnNgcxEy
zUj5OrPTOr$oc>vw*W66`6E9Zd7@0np$8?2j1ICopv!;)DulMhv!}g;f)0^ew@=4sU
z@LYKD*9VLB;lk0S-s}$!D}&Fs=Gqple^OlI><u{O;^9z&c2%S)U?d57YJ)t^^wSRu
zUrrgG+WQKZhf1;y+T*$-wQd@XbPrIQo!3E*y8J0ar6z(|>13b0KTq|IIf|8%BA95v
z6|RvG7jFUnC5N{TU}@~ehE4a}H1911^J&|^WZoKx2@6)whcBU3eJ6_3>agJ-*QJPh
z0IYtA;s_Z084v(%e-BPPbHC*Z+h-qfup<{|*u|1$Ol$18V}JOK{+5376w|i<E5lSY
z^J;(K*=R5cp9;uJRD>+rcgEhSaFv(<VFjpsep@{VzLqHQQ-h=?TJ-o*W#{!9wLkx*
z@t*<EV{d5x+$)ycUzqzW-RHOHe%rx=a(Qm^iICVhe0u;#Q@9^X6KZ}=_Pw*jBtnGY
z`tjCqFs%(}1hFeQo*Zm`bzD|yNfD@0e=_n(5{KQKV|>Z&TFHl&@f1|gUn;n-)V1dX
z6^_d>j4!{n_Z1!k)mbJ`3(>SZGU*~$YJ=5DF0gd-T|@^SsFWVela}^_RV)&;hxE;}
znD=-|$IsE32Niwn1!pbj=vG5)4J%Rl3w5lWw9jTE6=p$WcWd#}hXljY%A6ppwIA2=
zL9`4N@Cas&++>$dC3J_|{iqb^b(Q>Z$e4{`N#Q^*oiJAfTKlYyl_xe*i1>Cn5sjJJ
zt6$j;J1POIJ#ttq`QWy(_{YNcR7dZLjGswy8i!Qo;oV75^<j&3^oWW~%exDqGbUNk
zKSXY#e0IZbZIlPpMeh#>LAyvg%sgM})Rh6mpft?LYcud71f`KHVm$Mu(Qic8gNP}6
z$}2}HCMFg~Bn&!TU;1)YJ5?#V*DaBVIW59KQ5Ki+Wo@+Wwi*()zo-hR_$q^)fa(G7
z&K+t{T5yzX>=-|*`$y;(fd`nbDD2Lw)Z>qoz<4m_PbPa>33}wRUkzF)fgC9cUU#)d
z(a-o(VT^|PV-it<>N5&d;nJX@Kz2?Gmc=IFHNbNL+UMd$Ots|QVR$s2sI377OK4fo
z;I$e`7)~H9j<ii)%x8r<jqeAucC{K@2Hs7=;wXKQqap?Yb{P)*Vcnzw6sYuCm8J2~
zb6g_$CY2}-^UEMB%Hpm6cWtL-X@8x#QBpw6udm7uMuYEVy%7?;P5|zmNaO@ZEN|AJ
z&D6`h4vXaJ>FJ!k8>v%P!==(+RPrAJnHf8}%6VXt)EFd9W~e!sh-!PGGZrM{rywiA
zgKtl@t9@jby{b^@HI6<5I0fdpP7;0*d<Qjkfi###olCz7GcYhP9K<nM?Ki6&IWKZ!
zIt)+r4>Zhe28&+VO*fzeX>F<}id+=w?FzfO_lupIvn`Qxg>`3aGRB}}dN@|)Z0uSk
z*<gC-(jd>_xl?HWz0_s)euQ2LbRBI#e$OOyx)W*879*G9zNhZGlo^{L7Kn#-7%mJH
ztQ~gq;Eux|KGG;F+{#lgX=41ORqF;76Bm~T(W`@PSR&-Uw3Jjpc2&0fRq<F92T*&^
z819ctW*+FqwInY3mq1iz2n4+}(gUF4EXk9PbbfFKR*4!2i5Mek<k(8SoG`rpxa;R-
zrN(AEEmjhB^edbSPGc+q%<b@}Pa1=OaWMpU@w)lPNDdYbiOc&^8HXz+I|-B)@FMFF
z_~$H$!dW!K09dv&RsrnMz&v9cbn9eU=bWvC<^u_z1YXnhe(S)LxTItdto8zCf#=Jz
zG7KVM0+1G<!ed_kqif`h1Fz>EJ-TX$h7ouJla4!b1QOfuv@~)!A~TUI1vOn}CQm_e
zoR<~(MKu&qtY+e?|H>QWqgIe34EI)sriTNvtS4%XR5gsoH4d6?-@il^OQ0nhi_On`
ziEHYUk$6jt!0N+!1ob+P8v17w1zLPS78U>i`e3qFcMQ(OX#qet#KIwrx{@H~yMj~Y
zNs!0btjGD;Z&}YYg+y?et8{B+yxCc#h<^BDtM~bB&TC)_LNqhQUDB)65QGGz^mxhY
z*?hrfOu6tdRcQ>*p#THEVa2Q5@~elJ<JQ%ie$WAGgBl9bLH84&9c43WzXvbjOr8`1
zsbv;{c8O63H9tRp(ap}Qc1MvnFa_dK1HTJ4R9DU$u0sb_+R>g15P>wnBjU>grl$|J
zNZW45Ayv|g11<YpGnpr5J0jb|R5Z;Bo9`9|>E~HMRU--X@z&!Ai{PNc9Tx3TTGnp*
ztI1@5^Bh`7Sf+@VKtdVaVz5X=BJ=_MEkfp~qM}Ep!=*VnI@#zXq4fk^B;J6LvE(F%
zQ!L1Dih^OZ7eOvx!!V}%PtTjvTIom{BXuWm2*OpVp4R!D{H3!<{U->dCkO}*j)F^o
z`U*#Y8U|Fns=Q7sbS%{yK0_JgjaarHb#7H}!}w^V-z;`f(5vLd>3n!Mna^Sa&T7Oz
zdgUXVDyRk5&}Tl4ju3P|ihJz>^b*Bq2m#l%uWthh%M}~LPiF&<^PuX*BoovGpo5|^
z@iRf#IuHiK0mEB?-qO3M@ZbYdK?FGbG4y`wwvrddBJoX`7)I8T4RGwk{A?AJ*CFxm
zn&p2ZzV0O=fG^#}L8Du7FI$8!Vrd?Ak^O=^yx2Xoxf?ui<ftI98x{S7nt%mkk+BL7
z@-)t{=SX8xxSkMcq})2TSiiY-6ly;Q-0Wk^Y_0q8(_EGOLCj`CE?>c`HIo4Q!Y0gr
zsY?(xLvi4QGk^v_QDvkevpwH22gtZ^Q2k|O5y7y8y+J9b;|CIh#zKKXO9UeK6Tevn
zB-KJ~juP)(EWiAh<q23yyv}PDoTYR+wbepc@6;~GK{EmP3eo)y(M3}O^y=ZdU$`{R
z<n${z)F1<*2Hnxk4Zw`O<5e85KtAi147tzI-h{9`P`LlWE3k(Xsq*LM!RE{y;AW!(
z0?<sm(!A9rz%dDZbS$j05M<;luY3<Uu{;;mfD1ydo&^QaO%QgY_D3>go!|%tuZ(!C
zuvw41bI_b{JtQp=Wzpyk2$7NB-`=buoo7di!RzvDrcp>?1me8gTEkiuHm{Zd<~!tC
z1q$4+%6TZ4`M^sHPSeiGv&&zE({l91uV2B*jn-1rUa_FsfkK@LMt}F&aLLjBn^>qZ
z^&mnTcBP4Y1&4#^!HspXLc39tg#V+Mf^c)IrMS)$!J}v&h`f_cZC+Op_$zQvh6xhA
zi#Sjed_`%4z^3E2`3`J?xJJ1Z6HCnwUf{j9QONQQ#atDpZpd%XlTClex72JXc(`2&
z!2lfqNNBgWq7FS?tLiW!SRF2le86KzZG(z)i3z;C_yF{VDn)*~7#B%?@@RMYt4TMd
zD?BBD`rsnS1JC9M!oRGzH}zO?h)}J`hCqCymZ`3E(K)~fI=wl58~`@=@g~3LnOkJT
zNYSgeuG=3$K<NQ!FbAB`;d+vGrR}f)Vh&hLd9uhwMx~~;Ss7$4xw8ijwnTxLAO*0D
z+$ry=C<iq)HMGr@)LYFPevAmDCPR>VjlL<a;JbNyrD9lsq}~)t!f=a#5!c=R6M)Y~
z?6<V!69KIzL~{4lWv!$u2duBZn&q;cJxPT)cGn4<qcBpBWBvdqx~q@=<ZeyA#W|v*
ztlV|>2B%QefVz_!@U5Hs>9|5jwkSJ}0`5l+CFzZ@WIlKi0_S75p99>)y|9IK9pmJH
zFAa}YV8CM>xfe<)?lCwoGn?!@9uCNtry#Lp0$CxO!7p5T?P?c<kf?N*my@6jQ~dml
z0)8qF&UEZM?eF+|4WQH>&J)99&`{r#b2)cVbpdW0pp144tEn>K#S6rP;Ee!S8$w;K
zsHux`B13Z$@W$}WLd0ni_ki0**uhT<Ue_t43P!-u9alp<8Z~8_`04I$spSYK6sV!`
zd4l|Jqdo8jWNGtSWxh@<-SZ#R8}KLzPGj9l^1c=lRl~~#nJ$M5^VcQu7WK!D&(zZR
zv;-mxoD5g0lXnip!p8n4pC&fFj|5;8ezIe>Oc4)mQKYc4aulOhWncS`V~~#^{QR15
z9w@>EaL=XqsQOndffn%Fp2p_Ji^}tiIt}%AnMX40>n#k72pYMBU*H@VB@X|Fmk`!l
zwNtT45Y*twTZw9O;{2>URZK1ptwKi^Xl{+zgMgqj@%4wwAq8;cECXO+BWI2MeW{?!
zk`^Tt4jQ4c!539EIU-{6?j~Hsc;r}jx|axWtPC76!czoD;@sX0E@+7RXrRuOe!aIs
z|FYaF`f2n-X!dgrk@^p)4mUOA#+N$%YQsP7iaDZNf-{jBdX4Wr_}QLZQFr!WQvAj;
zZ5Lw;-l+h`_zsb64>qRkW`EtLS1m}A{<ugM5YKMB_sYP*lF%ERZ}<g5kzr?wfD;kh
zj2L+%5;$6dpN|MNXu*SmVu}A~@sklzKFk;U)-4%ZzNE_rrf?b@aPRQ7AHZEV5q)K>
zYM_0-YYYqMYiZzTxptS~v89B+-{=A6fFzt;f7E6G!iTsk;Jw`MppH@Mx+7{S3L3WF
z*)9e|6unweTD<o_nYk)G5a90K6!yJbJr^}VP5OJAUg$XM`H<j8EOT_qT{iXDU(SGY
zgE0WGtCz;$-9#EEhy!%4$+So;vPzC(<H%Sq8IlJr_#a#h0Ad3C5wO$_Vy4SV#UYX~
zU#9i4MDrp6L1<{RfZYi0L&7MOl#~WTUkxwL0)nCrqEqYElcPjS^w=b9wlh^oSfb_8
z$@4V8etZRak;i7zOUO^59qm9_*!isAuoPaX3`Il1`OluKN+n0>Pw(X7)hSaJIACPs
z9=iyD89KtTDjjjGqJjsL$ULrF8Ka>0kOgeerZtR0^oTDF6{Uy52DIQJxk(@CdQ}7O
zSuzQfxm{H5Rm)QX;$&n4{9qF`2=G1MEMKzUS_jr%&V~Ky!WN1`KolPtHM;>9(;w8;
z1JS9$bVDGU>y`o3bvtWui(W32|8O?)Oo&?o_Yo4HG9U99S^<m%A_&2w89j2e14UtH
zV^YwWXipW=DbTKY2nvH(piGqpSeRPBDYzChr1mn`ZhQt~ec0-`Xg_%Ip?I8ZN&*Ls
zZTu4)1h~;w8I~GqaL%cmFCiNXW%OZM;4Fv^0nGcKm}6g>nqneCfnw9d&aEdydTT((
z6JGlPQ805)%^PQsy)d6#IZ_gEB(JJWOn59p!$eGeUJh)ur0>}|H-qx$F#5fTI!{l)
z#wzGyMm$GJst3);cD5k^qkaY4rb4oE1J^O2|BqTlBcGKB4^J=iSs>~qRD29JNrs(-
z@S?bMc`^yu&X0U{q4znS!cqhn?mkq0)b@(cX~o1TXjd$?G|;OGWgeEy0MZ+Ne)EeG
zBROXd89p@fFki2J-rqznN1n|<p_%~{<#3jEX%t#=hwd49AcN~g&_+O92BVdn2~fg0
zy?O$q%2JyjzUW51=*c|Y;LwWe;aqGcKrY|z3BAR{^ae-jS^&dH16mb^@fNzHiV{<j
zNZsAtA4}F9B!{n>PjXn*aK;@x(5VTv@5NaPuljR<A5J@c3NKP^H6z?FK+H698dKGA
zhp3krMT7SDFJN{V4of}Sp?+()QmOSIhfo9a&+6|7`6;fpsp38&N)Mz6fC?fmtE6_o
zt}<&@=sJ-NKvpUlEWcR{_59|%YT`}8^WS)17Rv=(p@P#jCG3+$uYQW1SM(UlX{c#B
zqqT=q-F%96kXirZ-K|W=<^C-Iv|0cGH5;o>Zi0d&_n&3UJl03R1xUd|o?_-xZ%)mo
zi`0PY1tK2OMiv`2eEq&4fm9s{jOG`R_I6+ffK#CWw(uhRJegLhF>i=g^ake`<1)rJ
zXPN>2;m(Y;B}GDWk)8w~GnQ4q=@&TpCyxJG<9&?>zV-G01t4NRwS>11xHJ}V5w1&9
z%#a6p_fIwV1xZPcg#`Ls>}{_AU(8g=zZXcG@@RLNZanpdEOT*&7Y(9Ae4f=uHgL33
zrb+4<8u<npq84v1vHC@G{s@*TR2RQH29i4{2kyyU-d_#SR>_%GH!mc3-c-2w_*5F@
zYoM<X4F~d!-j;v?0SD#dgo2S4IN2l4G2jPE?|j!OF$&?|`bAX%T8(6Ai^S3Zd$YBf
zfbRUOaW%7^jKKJJ*~HYSsHlhE>CkRpV1_Ob{7{cbjNHJ$Ua_bJnovr3^*Z&_zs(!@
zW(GJwkQEcQ_9r`0!G;_+07+wH4Sp~mIN8?~K3wZZ`WP<WaJ>M)WQIUAKfj>HCDn6Y
zUP`K+!taXYae{eKxYSpIJ<5Ay9RQawZZLSiy@}Hbbnor$@7iKnWB}?0q!0)bitBBK
zYt-MfR3-s6XcV8<Z<h4v)e|$81#sq}C=5=^E@BRYoofSCI0VEWqjGz>X}~{sJPF6c
zfFq*+0Lk^DNE$0|0OtD0XJ#gHmYwptwXZN&qwI_S>kM_2JV=qn5S!Ad2XKNdxQbrm
z`)z=#GSnvluMu%kJkRnbYM$#XkCYdJ68<S0pz7wGpqGW1cOP6Q6GE_X5hCt+fBsB&
zxKh$NTeCkN2|r>D0+`(MGm*D9TC|+U-r*sA!!v<*5Efki^wGnMhu?!lj7Uzn6|Z-{
z05585e>hR;m^<4Z|F1%Z1nnNx#hh@!4jtmA7QiE|Cq6u(!^uN<mkb1fyQgn6K`k`@
zAr|y84@YbQ2)!L{0SZsb8woW7NCnsoTcaGxC6R$9$kH3+rf$YSol1wC#jnEvYA0&A
zfQCJdt@Y|lNeB4yO@Ow|_Q*@gN%$}OC;&d{zjrdns)85gKxP7VDN8Q#>vGDz6|#W(
zvq`vuoi^1?5#ILrAg~vQN<|HvaKpDAP-1dlwek1Bss9Mz+cE%Lz~ea_bt*VWCxW{p
zAaULW2ZMTvJBz@)0Lz~T4sioc7BH!RDj02lcps1&%Ahp>iM7W@LS*RzN58u7J0D3_
zp9R~1hpV|zasURGD^VSO+QCTm|FQR;(R8(4+b|+Qh#t~(B1MUkkRT+8P7uBKkmzOW
zEkT43ov4Wx(W3^@Mejt3E;?K9?VZbY-S_*9Z+zdM@Aor?kufrCd#!bz^PKaT$2{g-
zfCb!#p`5BwlJoQ4l`~bJrx-$VzKu8=I&pfrE7hY6TYPE2eCWF<!Wx+E6I{7h?YdIM
zz+i(tayO}n+CJtr5Ke~ByM{K8`wi}76?<oO-0~m#F^&%<d*{VhS}38CWWR}lzW1u@
zM3`r2Uloy`-$$R5M;`1kC9lg)7G(0HZFzkE26f|Ko)i-uAL-t7PBUO&XWdL{HMoA?
zjIm4;aTa;%j!5zRF=;7;Wfs-EYQ}ZDbLnuR#i2za6*d{%-DMh4(SJm)Cm?w2&{@4<
zja=D+bg<H`e1BShxX{!FN=Qzi_<$USl*9rnEynXQ!iun{7m5RYB5cE_yUKY^nAQy|
z_JMgQr0JhT#-Qae71EbknS7H3CtnYq9<kPbWb4h5Tys{L>p6mma;iVwf7Q19D7!tG
zueKP<l!fl`X!2G7PZIc0Fpb69Dx+%vCMN80j0uSl6mrY%ma}CH$RxlwGl2<4K=g_U
zhKlg;7!C#!4ZP~XqM0n}5Gj{XHQxLNRXiQcuDI77<A55qkJ?Y3gn<uHfVXi*9Ip~%
zC6KEAbS~n4*;V~odL7zJoxWWk7KKcNyvsxIS1^s`gQBi>R%(wUYc71baa>$NLc(ei
zE+iOWcx@r4G?;n-Ii>FON%xo1WaID9xNU{JWaZ1&NspCz*&Yu}kl@wxdi0QDBnY^)
z07KlysTQAq1`bn{Cm{bvNXKr4%Ln$*`6!Jghsx}L*#%~?(`TJnUKn7<lu101dI+Su
zljpQ)yqR+X44EjZ8VIM)YJ`VS^E?LwR!qseu5v!oDANAv%;|A*#9dr#1PcJA(XCUH
z(RMco`Wwf)<FUgifJWRB6|m^FXJc7(|3~K+E<YxW8PV}LE&>c})_<q@)BQ|Htd!5H
zQ%c`nOk9d4p_?$m%qGf+Ed>h_DXiwl0WUJa<^{tXT0l4@z0w9Ju^Hx=Fhh=v1pVky
z)6ssSm9vCyFBov#=?@hV1T-+-wptm!AH!prfDlaJ3>W}Nhrz4@@x;e&NV!`=yEP1O
zjt%B<Ob`8LS?p`jGby;!nIOx3o4kwarTz5v(YB|kw00=Z%#*5XrRl)51oGlq(=!j?
z1w1VqF`C{09GeC{XchX7^aUYGc55vGZDqDNuu#59M@Ls&tha_+sv%@jNCC4}5P=M|
z2RfiO&(skkba@t|j}&*Fn(U92*|)ast-!1mZquH(kBU$)*AYbuu_J;HRueesA&B^U
z{?3Ji7an7j-uxC^S5IGVGIfxQHU>d2GZsR3zx>byDzEpylcwONuq%MCky?1B2;naB
z91fv4+<l76F1iP@lucjnlBpyb#LO^P5x1Gc^AoXv1=4pxL6Mj5Dxc|y4hSpQyjGy;
zl;9e76Uzq~m1#{-AU0pzS{)xh>UIUCN;Aa0?{l_T$R=4}j?&IS<0)ss8sbu6(o0XF
zsbWElkHx3taoe*E`k%=pmg)pTX8&#nc>5Qs@d{;jrU;pnp?4OoJw4~n+Zn$CIZG;J
zzs5M?ox9>h$dT*8I^gp{KU?}%u9l<uFs!<lXWIstnym)@v$EL)B`ny{)B_TbI}a)e
zMb%2j@pd06xWl{|r?na!a4)0HoV>h)IV^$KF%xl*V4O`_@hHOifsSrlg23*~b{L6C
zNa!F>fM?kn{|3P+hH9pWVhk)iv5SfSOcFT3j9lpNGm@KdA*l%%pTf1hv=wDFs`~}-
zDC9vF{59ZZDoMJS7RvAf*}{y{jmJmMIm6-qdmbE-*-7>%D4A>ibXGF!`$~tHGk`mB
z@ZMpTo?K@*mxT$Or9#OYUGU`{SRgq24nZH`DiEGJlv$2?p)Vs0ghE~9NyP=148*E0
zVj<Kkq+u$L50|;_z7-}wRxA`t-Piz@$awZ-XXOtQ3O#}-Jrs?Fr7R#AA?cMLa&AiH
zMEo(ZFj=65^}J%NF`}OYh6!Q6WZbTeGRa1jA)#ya{Daqny2jb_G&L~PCT{f{JwoWx
zUd95E6-Vn!@GWcy&-9Qg%IY<MMx0!SirwVt|KuWS5ND!l76<`GS>Y{@{rq!8DgYVu
z5BK9TpzsFza$ohN;PKV|__!D7DT)oOseWW`;v;)W7{g-}@)Z}Ukch`uLypwmF?dkk
z2=rEa(3XJ9PB?H(QUDng2CkT#MKniIxROfr0>WABB~}CAlEIolcu;`BrbFLTEB2-W
zjPb@!JErc!u?iJ}gjrM&YJz&HdGC-cN<2)+&aQBNxg3FOkohs|>FFUQE-w6B2n(iW
zVF73R!_NTw!<AErm`puM*XS5{5PFywR0;Vr5~JCXxko3*hf*NVnu==$^sUS41u4A0
z1p$}&FgMrT2K8<Pw$Vbj1UyC)I5<O?E%0TAuQ1XCyi{9GA>5gi&a~zk+x*c2l9Gj+
zDR6^AwANmb@<t2pfJj%ko5=x6A)I{Wh|Cq}vX6T{{1B2ONgQE@RC^jnSv?CdaFPB2
zXPh|dA|llTZU7V-<|C!@+@|UG1?+EU-`wOqdn#Atsd&&^@Kl&=Ymm^!pmI&xqUn%{
zO9<>5)hoKf4Q_ca%G2VCBJ_CggI?tLuGJTWnCiKebbSM<r-Lms@>q^>+dH~!FET+*
zgv;o0&mc?GKhy!}7qzZ+l_L8iE?zxp75#_<G)$rFRYwYdR9Rp)ObF0AQoyN^mA!2Q
z^PfVJlrSJcS^;?@UW_6*at*oN-80=mnp~pA$)%BURPIpqQz^H3zsk?Z;RCPa1Z$SS
zlZFF1w-u(A`gdqKyb5!Rpwn0wUADxw(4Fk3hBT<6F=zzpglY56pX_&fv>@~>Bn4>?
zjC6w|yODdw!=H~0`zImMirqq(hc5+d*SKHOf}g|FX-Yne2@4=jh_ZNk4eqP_;yld;
z;(3w0?1$}KTUH#%oTK$S%J{A$i&y3O+rFpW&~8BVpD7ZMWS`|BGM99+$2Gtg7F;)n
zWYC9)#mNUbq-YqzH0Q3UE&|wrau5_S8Hb#pqV#|NJ`_qznn(!EfWot9aNH!FF8i4<
zSzvd>U{PrYJ^DxcjY%Qm2{^GrLPCIpC52KN<kz7b2G11fMNB4@a<v6wGMykl2FpqX
zzou}f!3nJOl~;81l>!lwr2#htrKT5w|7cBm@&-w)&H@|uxBnK=ZtyusL)nA;Yn`Dc
za;>uSnz1(RF<fFuQid>*K-`KU%sUixBvFUlh~pVVJVN~HV*@b^5GEp$v;(PmhzBA8
zoS?-R>9g%dg~5cdgyrox`1r})q*w)nadh9C#5twHJ}WJ2lm90TVM*<8cY{!H8?eHI
zA`h4`FxMbHkI)pjH^2zIY}v1Z?6Xw9TB*tigcmmJQ=-UFG(>0#6J>7G(a~8zha1@1
zd#q1i*)@u@0;vVm!Qf3Ai2ae`NM2zz2>fo5-IV?a>A-uo%xus;X85GXs;`xYr&?0g
z(0O_`0Q6K__9X_%)1I;}R4c!<>e)xB3V`lMe{tY=#nNNbTz3oFF|{u{&c&ntA~W4U
z6D}1(FIq7SgFoRZy%?<^up$D8p)RRY8CK2GyshOS#WK5vr&qm&%DMx=LGj;&BWh5{
z4bk|7o|uY?6`78Pz#K%B5@|o*@$yk!4p25HoCH%KrsKKzB3r$`&*%V<>FD4c8RRn1
zujgEIa2VN9dkDvuiJSWcKsICJNtit`P<iic;EeeHEp>U_wSu8C`+L*DMo3&hs(&EY
z+MmgXlL**2vYLvQK^Vh}@jT%TUfl=r-5%F35M25A_Xj^j6mW0ONQpECJK==d0g8SZ
zAe0$ud&+b2#n)<4;0H+0u{6LR=#J|=1IZ{J?(%rm_|bP>(;mixO*JsncAB|^97`)J
zD=d{YVlWzs35df{bi%#}_1o^t?Qo6nj1;f9Z3Sb-6ca>VU9=p`tEsRahVpU&$snM`
zx%TJt_{bh+L8NOPyM_g|Arw9$bOV_bBKO<~gdF181tRJ*?PYoS7Xe5G4ZJ`$A`Tq^
zSvpl?!D3J5!LB_$@LtAGA{q8s0wOxDieR?+)mKo*Udyc^MpCzlXJBs@^(nXEf(f2z
z{T;~e-ghMJgX*6V2AUxNY+<}g@O(%%s{9=;W+MK}$t|F6Vm-LH^Zyy#HKFANFtG(x
zZXznrmWPX^04kZdqk0u?GD(K7S=`$~Ac(3EoSyR*Safh#hv$KbVMmm*M1n`pfh;vP
zOW#`gqXM#5!;2l|Rbq(a5F+j>b3jC}PzINTg5^0q@j#@$ye5U<@^SaW?0HW0(Y~iV
zC650spagE*bazt0L#Q))0KBYbnn+~1JowY41ab3_QQs_U`vO@3NEE@|+k1FgFUabG
z@4)i{+|~>Q&Nc!qDQpqqGY2^<qZ;qU17e1U)>Ci8S=4&^YI=DLp=ufc8%s;vg@E0I
z9Zp{CWd_JQ1cCfYblZu^c+>Mu>8@fGI>7~<PlUw)5Xug3et85mFK3<_SY%lh4FsFQ
z+Pfokh9Z`z$)wxq@$M|D5+Ld)HKcfA=iyg~@c;i~-Gqj*?UdUPMQQyz5=4rHh~!=#
zhXG9-<9y5XRYWHEgB&avuJB-+y(mrLht14#OLZwiW>gV__`y%@86SAzA9f3&;1WfF
zLt=o~#ZlgS&_IieG*^m)R-haPJOaU^Dm4L#6;K-F-Gbi{nLjYR51Py75?o|!>*}&E
zvvk8gj^hGdD~4!*ky=`8G#(;z$ty>=Ohd?nkz{TNoIe~wVWjQ?sDb-_&<eV44|3O2
zglahzFvP|kd^Q0M7~I=SFv!W&?76Al6vUn;!IO2OKUeNSTD%sBpr}+pdK<s4TQ)%_
z2|7mh<(ZPNIvJ^;a22cpmeF@tL7Ff?4+Hd*B=iLc4D)<I09FEsqvmd_K_ZJ;f;%6i
z4@w|toTxllyxR<u91C9!bSxh<DLg14?#0Ez<vkDj4d$}VElB*BAhoDJt0X}=2<Yfn
z557n9JAFrLZxA~STQZ<esl7sCx>1<H4xalnoYw1OcD{k0g1G+yEsFyyBR_hNLV?X^
z-^?+E=yIfoTMzr{4zj7|OPsFpLr8WH)^r7ngM;IX<<X)s1&DFNCnj{E%K=QE;us}(
z>(;G%7g}yXxJyDP#H|%0D_lT|P?MJaKpKCHXwZ=00168q-@4k`BTFF>Ft13;uD9_W
zzZ_SJ2HF(VT5+U3&#XFwg4b~^Njm!86XUjMy|J&ip^`rzTdPjQ4y;>-MKC@BR4kD;
z(May&Ge8a`24}1cGRE^ua&jto6L3Qe!Th0r?aw@Bii$226Vl%8gRJc@4CIutwEai$
zt*hq+HMM4N7Es>Qao=wU*E<`aK$;7gB}GtN*O|FOlZ*4*RURw~;;tEdM63^cbBN}H
zWG&Wdz!nCYDSS5s_Cp*Ojw#Mr4GjoWBPa$sTwyaf6JPJ;dZOubGWiSn1JJS>P!tHO
zeg(K0lo=$<fDx*ZzC{-6>$q5$E_g@=gXlhKT-Usit}e;&TY6?6w&uHqp?+_{PVfx!
zvd5Qsq1^*22SSJz0udR@8GN>dZ^FPF$JMNKRYMH5rRX&bIo$Z$%m5Lb;&`+Wxco}I
z_u2@>b_z-AbSjvhHd)tIR&@=e7KB9beVEB&99j3ka!m8-)29^_I8;b*WAg%Ghy*;4
zXw;GlRMWQaB_c-_`}LhX<MwL>fervHNiSb!0<x3a@}A(y-n6kTe(d*qui<DQHb*nn
zdmqbzWDnv@yUidp!^F_>y!tJz?KpWpLq6G83-<~$oaYy3Wi*#v3$DX~FLb15@)J)Y
zY`eNb2#(Rg;4~!j^&6xG`k*u}esZ+$Mogkb4=gVj+|YIhmjyOIOVV}edA<mSXMx15
zN15gJ3+#_T%^n5*$hj2t38{mHQg+2u9W+M$?sr}Nxf@#bZ)W(KdEy}60Z`1B`!Qgi
zA}Rq}=ur)`fgE!_q6=WCV+dMwvvn2$^F|8uAn5pr1O_NUFaoC<0?L+8sV~*O8{*C9
zbMeAcvtwqP^Z`pb1OkK?z)4qmGpnC%PcHsEfjR5pms1bWEj>ZTRFXPEOb&xel(PW!
zBd``1ps5NP-HgL=9W`}9Zh({u3gU!#62B^@2W9B&0_i1BR6l)S2ODEb;<(YwYy?=|
zW^JOTP_yIy*#UZR8B!nLA%EpCM?{5jrrL8sKP(8}0pl;qxL5~0gmTFF31?ZM*Cq2p
z$_PNW!f-DHxTqJt)LR^o|2ujU=9u||#9^M$yG&Zy1bZpe{ofI7J@k3dKm=UGZ<|V>
zDfvOk4>eWD2+bq=(^6&5MwnQ@nxV=s=rDK+y~l<bb~9mek2yS+7s~_xr2(6Puw_4w
zPa!RTJr3cv_7rr1DgPoBTX?|)5*)Xy7v0XXPi#+QJP%S(-~<da1W5@Pe)W<22owYt
z`EaU_jd0;_A$GHGFI>kIB<s=<{|~cwL(pCGj&jy5P-=1A{{m99d6Q&1m@k%B;EA@#
z=F_HuA+gXBuq<6OcxKQew~$I$|2HLg8kl@4-KzfSGZ5-MP<lJ*v`)S+L#Im#Sj6E+
zZy=eM9WH_-fM76q5MPSq+#tJwoV)KUNtJ6#U4#^b5FRXbKgFo#MHbvqald~RiE_B}
zvBHneSt2AY6aWfLV{fk68KeMZ;_^jm7|BN<zEOJeM9|RzQHM0h{j0QT`uGIVUS8$3
zN`f(y0_a6=2jsx#;sZ-81mDJhP*(g~UBcR{iG*j5v&@uaWQ+tJ<H>BTa%H>J_a}lW
z5*5%p@dy7+H@&COH#j?~p91lTabF{j(EmjK5M~xdk5UvFYe8vE8WWXm)QEWhy>cNy
z6-JPNBZGt;e(vnXN+9E4JQMMb00rOl;xXyC0nz+D|629Ao^&}HQ$+IJ65ibTOt=(K
z#DB-C90?(c6XQFl5bZ^)DxpT%W=y7DApkT#(KDfU@RdRvSY;!ENS*zMt%)KQ?fzZf
ze^+i7|7DK_Fpr!tMY-Eh8$wkIq-e93Hi6LiQ6&tJ46_RO*SV6CplTAUU_``h2!3;x
z(hu}n-DUPjE1muFNbk99Bm?|S(v>R7s$TdJ7zXXd-`TI?tKa<-N{g|;g(XW&uxgdP
zuv5$a9R@AACG%+%bU;7a>p?jLYIdZk3Q`EDo@j?{P|dW0!m54lUy>d=vp^j={QNB)
z15$&syB+#_wJ$z@-=K*N6>p7DHz#yS*HxwCA1Q&->$QUbJ7^sKQ9E>R+=q@|T{-yr
zor=}K;WR8BixJ}N^v`(Z;KSdaa3-zJZZ8k@N_Upr#z#(ZEG~3!^s<EYlVX%{`~SO6
zcP|1HA7DY4eyXw%yYl(-=bkJzu1y!df`)9zMi8cTA`E2~dp3hE9%j7Sm(%rN8bNKq
zyOs{Qi{n*QP~U1fqSy7O#Y4JS*Yc|2Y1(KW11O2CIz3*cMt_(+vo2!ss%mh2ZL8~u
zzOAYe3X9#0Yj#D_utmM&n5<wOGhkn8Cln;KSf-SNFi{wRKR|c8BAR7T)qrKgq+$t|
zWKATm*~{?kDAf32lN!hG6w1STv_(ydNKtiFFyrr60?z>YhLw_W5vkGn(^tp{t=N{K
zejjTFT^!^@mbM0VD`S01>dn|jy0;nzw&P0ywtsr71w8$0qZ(c0mVsfZOfGMR(zHUF
zbj*}<bmy77gz+!cG^d?oPSq0fEP549YBn9>q>N1rc<<l+rS&%a7{??*%;fL=_(`+l
z4SCMcFmFRVQV}6+CUMfYVR0mwlQNere~<ngeG^+Azp`*RoH(*vkM`)??swPHDl!Sq
zO3|YWE=Y}H)ljZjgQF(~Bp6E5<+&a)hSxhhq0)e8q9AWi1Kg1^`vBlwWFgMobjUCX
z$pL1*4>GROZoV{1d3pG>5Xl7IrH+n9Y7~(Bczp&|I>Z>5$WPt}1kk>V>hksV?ExCk
z$om`6Vxe7FWIM(7=cuu<FqttX*(TntV-VxActKXv3xy1(!#=3^EHqH+kgt|O3l+VA
zzHaw4nid0#kiI&h5pWJ|#~{!K{5btLl+K@B2f1N>=dE`@S<-4d9+d<Mf^*cWSabY-
z$6<MpWE?~62?lofO^iv|1@sdTI0|A0nb!>IQL?IwK;-_yGwvGZ4D6LZ^O24Z16+uf
zcoTP|$|DgRh8#QqMd#aSF$~rVXV-dn8yf(v6!h+<K=^;Is(NJxDxy;pASZecV2(`j
zUQ#@5(q&9sl)63hwPZd_*-KS+3kugb^sxcZhQS6)fY=QYOUQ!<w6lCXjDrQ2jKp2r
z7zA)9anuRPsA?Gw8`eLMhRICgxHjSCYKe=77g9QaN6Pq-Hx~|F8j!nu8MnXQ#jSV5
zB1Y>U-NnF=rLC6Ig`vZDuIf9PzJ}ViNY-==SH~?MEaVcj6NwmdRxFyOAED{sUDD_Y
z$0RPUCkC#kXBsvF7+PNfjFo_codm$72MXviP<Q=>!jn0?(cTk)@s{uB0a<o;AO>^H
z70^|@bBJ^O;<~><S`~CIYjZ=D{A|x*Bk~%b0Qjq)wg8HgBrz}<xib7Dad?69wihZ_
zBK3c&JOr8K65=4D(@B{^k3p_=Yd1Hs)dN_y4zysOwjvgI{ZvpdbwfgY+hU}&FHcX{
zUq=Nx)MPXQjlJNeVJ-($V}oFnx-@hqhFw8<IbB3wJi8PIRtJ3^v|5s^@oMjkdz{LE
z80MoP#SNm!Dvu<1;IG0MwegSKygt=iVcp<&gO>MDEB)26pQQZ7px^5Q(tMr5f6}qb
z^~{DvJR7@#*lXb&g06b?|FJPzc$#H)5~ONYPXoZ+%fre2s#Wf=;Q<u9z4?RT^H@Rj
z7?U0HqdqN*;z$hy3O<b21l{>z^MWpt-x7irhjgH4$tn#O2d7_>!9HkV^L{4sz|>>+
z1}HMLt;&ysErfym+z^`p=w1j@>DOzgQ)HnoN+D-2Yh2?YCJ(>$7Q?93S{s~h=7_6?
zk1-9^1EDYv!2xnrCRjBv%`37O+Ht_3yyofhpcft{&w)1W8!YnH2ssM#UrTGpLp&Vh
ze;_(Ht%o<;Y|Mo5I^G~+hnWMPN~qh<VQ%9Jl`7`SJ!fb?)2UAlz?jANysR#uofpY7
zkmLK;2<928uWo|RU~4smb8^Ft=U4nc&mly6fp}sN6&E4E$oLkH=98n6LeQ+!PeH!O
za)DTg?1|7NFy_8_oW7^FSVv;P1}q<LJPd`2&$l?123IOzok?3|H)oTb5R=m7!_?jg
zW0In9hPDqz_pd9LtIxqfM=mAAa8@Y8bv%mqE$EG8(^9W&h2kv6q#O653Ad;Wa*<`q
zK1nhxXo;cZu}w%Vvzo-Z%&H#lyfyy>>^==V(W@Rw?sLf3^)XZr{Q#m-3`~{zM8WHy
z;{<m=Y>x$^=5!D$_JDCfx2kaXBYQKm(X#vY%Yz{h{x5IiOc27oPQt?B`(g75(Jo>G
zcP0eRG!5GLiA@5swSShe_=)Ncd@X38LT-J+6xS*=5$V@WkpPF1I7=FL?q9Q}CC0<L
zAr(nQ4qV5xIO6vP_M7j&Al#KrfcVJ8bZ-n?5g{-IY%42VY<l4d@VKG=rl2h;1W+s)
z%7|x;p|X?L3T(BZg^%q&EA>UlvvC^g81fK~Ucy@f<V(re{&lLD#Ezt1P<W1EffqB7
z#tXPaLZT;2^~f#>>aov(s7$bX81x1Mm){HHZ8<YWtse4uVA$W4^STUDJ+yiI-MC%y
z266nUD&0~~*!(0f;%r0&F$t5jVC??7^DOP~Eo2+de!Q3=Q8=fbukYiCR4YOt^%Vor
z3q#NfO#B_HR>6?Nz?6Zg)LUrKomEXXF_Z!W$GvaL#)+%sXnqE2?>$Nj&@|#JymB!T
z{`uO8Fd>*HV(2<M_@$G7GqBS(kL=7kV3nc!Hi%l_j`bRE`60OYug^a}rJn#$8%SJB
zZ#80VB7j`0K&Sf~1}RLa4X)3f_MD3=TKm8=_HoZXTJ$2V-N-fMyXc?LLXG(a=;33w
zgX@iWFaP%fpYYZe5pjz@<rwJPZbK5*2c;!{AC)}anwRd^EO4-#Nj<I7O{%^qbxV7m
zf8=mDyJrt<5D5`s&`=jBt)|%D{kuM9#0U@PDAWlm%*rcY<?r@k{pT`>ws*>>k&qa*
zIu^0KS`i2($FlEP6mQ|)T**AG=3?=8ygi9Qv~kl7M0(uN_43Gi3LA-8^^Yi49E)+!
zpM8}OD;83kl%UYQ8Ni!twtdO$KSn2w_^u7AyWHI2K{SH0pqfdC;+Q<BKA5z5Z-Faz
zgvXd`K9?vYgo_j`xh+O`KXDpe)PlD(cxO~XAIXb!S*nx=jeKXv<|3IRjDiH}#1-(|
z9iLbJ-w<zp_G3k{2RqQjslK`D0Ere8g*KEqZ$Z7z6B`aYv8R>a00#C5uGP(_+Jxm$
z-5{Cg#a95;PKM+9l!b}jKYJ@AiUqfp40_{kpx81%uy;q9#Qs0Qlz8n4g7ZVSf~lAX
zUp<BzDJf1a%qs}LqjZNxMVAt{{st0EWp$EM-EpmkHyQ#c19Tg$DIsj%wipKYo(kOF
z;g5^u|D2u|3|+Z8f(Ia~nuiG}_coN4ZuI@fXD4`90}1uFTuodY7b5-#XhIQzN+~Hx
zE;OjgS>wcEVI7cw0q7)t5tWnh><tCVSS_+X;i0YX#%#M3L<i=Ceu+5Vb;Yss>Hn<J
z*QcIQO~Ap#Lsv|uWW;l#e{mKDW&q+<Xq#E!mohh8;#8hkbK+BMsCEfsdPP44mms}h
zTG%T(0m%(Fr0sa`oI!^3?fGXc>!UL8AGAT<0i+KQ=TohQ?S8UvatDw<OTpyC@Be#1
zor!h3k&sS5MaA#<Tv;5!JD`Ubht{9&<dB!=|Lc0h*|A_KN90T%LdMxmgD<71LFOep
z-zU#TB8}G-ab0No*lK1@M9>p5@Z|!jP}qMQw+9mXBOqEYz7wW~ante@YXFNxt*7d{
zX)x5UFd)b@WR%+WS~~~nY-b|7Hx&-+9J?(tU#KR&Hz@fZQ<{I0B;<t^4u!8_U^=KT
zJ-vJObePyrgR-oJSmi~j@J1?JBPw<FSlt9Y?rY(d3*F{S`5Z*|=JqSEkw*_G&3oPn
z&ABSkk6yz?g*@}cj}!6K`kD?f7>EJDrND<PfEFgc54-Q56F-56jSw)chByRQXv4VV
zc&YsUb+G>69baPk|53fUwwCB;w^puIXuI&pjH7~xNjz{66{$D%R0E<z!Fev~DMqZr
z-PNdNZ=j4F6pG|kh^+`PuPg^G+<8F`F^vREnZQBxTacNj5HD8Vxmvm!Lc{>xee}G{
zhe3@Zp>KHdF9^e})(w6~7<7h{9O18a)=)Ln#LwSgEMwe1z$}74vLeL1>j0Q;Zj0UQ
z3^lcoc-G`-MkPH3fBc>-Pe1SO7~>1}3uK;D#Bca$WB@$miB!L*A~vt3x?nEl<{JPD
zs(%mj?wew>(tA6ZzZt#~0fJe?w!3nuWD<BVB!=PO>6W$52zMlatD1j~c4VP@7tF-d
z^BkV))k39LHwhg2ow2vUB%a565*k!32FeQ_;#L#p1>aAA^PYtB&kYJ3P-i3eOM7;y
z_O~@T>cL5m;U>S4b?Ggu#B(f~&|nT??&SXpgFD)y)ombzLnStzd>KAN$f#@7rJNme
zZ%}E81UZ>P_E?VskH2v}J6>kM;H!T8+mRIRh#EU<5R(|aHSbagj<wngJBwiW+kLVN
z<z@oj2MA!474pm1vH$fo(G6P{XRz^_9Qc#2H^{Qr##6fmE5m1aKYB(y(I_&m)m6zB
zdrkw7-H$g93)69ByC4|W{ZMsU0`AC}I8Pr3LQC>$_eJ;&4rw0e%go=>SC+qR<1T{B
zO?re$^~=5gYO?zdi6@mV%`1L9xN9OT<z)<9l=GH=Iws(FAuQz!=KY`W-e?t3KXHS{
ze!-knH<<g#ps({W7{;=}2WHs%`6sYnkn;priXqm2L%g~rDO;UOFYkXh>wh;3j{g6i
ztp9%}%To|*PVVH1U&(4RSJlxol;x=?yq__-ZQn+IazHzUd_nQBoZ#>c;w^#Q5Rdn1
z1Yquoc!&FD|F4dSn1FAhcGYBIr~f+jhd?qOHYS<)I0Yl=x%~0tx|JoK1#dbltI5p*
zfyBn+yw|b2vXOVrr#44++aI<3z_&ZDHQJBkb>fsv<O*1d{$V{kV6ZhY<|Y|-pMgm#
zzINn8qTG{-zjA&FQ&}Ez{$!o=Ro;__tK(ey9lvvomQ2w}92;n-k^{khozK4+#&^d)
zbUwLx=0qv~%GOOo5YuNnbG?P$;5mL_Y-4)ql2uE)<hjt=bPnh(2Wtjc7k6NL>)}YR
za>Y61j?9z2A{Bmd_jwMs(o$b^&s@u*b;pVQA5(>aV9mqFdk%L}TSJDnV|dzJs#Hrj
z?;ob5FAergD!x1}6y12}H2i{lt6YtoPxi)u_1Kjc`Ak|T({g5yQg9+Nbo!^SJl<no
zo9|{=+|ay#%Dl%*$|w8!+U919mD|Mhi)>W>#J*E9D&@n&_kYu$=0=jPA1sH2+0Axw
zd>hJsIOF`;^EW}<An#e+W8UFjBmj4N{_NQ<5&70wboiDRR7;ohU79mDNE>xd)Jq$E
zRIaO)Q8!4#;^3I$j67rXlzMH7|LrB)&WqvYC97Q#wv-DyMU4rL>!kJRC}R)VA^aKp
zCz}*#?N>@si#z74k2<c;{$wCU?^P5V;B?_f=uF9|cv;OoCCRu%fxe*pk{bOfQ;*V!
zX4k0aMYBMVVAxHjpXF9rjg`@Rdizc6rYkA?^;P(ZuVaMqNSWXJU%4@0e0Kkh_FQ+6
z`^P`Ke{-U=8g#&U<6$>>c^4Yv;p|-^_@@AWHUgqu&0jy0Oi|is2O-(Iy05!O-E6c+
zyML>H#nV?FyUnrtuiQ6eaQW2`X<Ok_&UicFb1+ZmO$y((jlx>36HfN$1E#0<*~1cv
z(fn2-`@-UfUR$k^qc2LInl2J^^0w^uu3wl|uVT+%k?eDgP<J0wX}oDRbo1;US#{2H
zUxavycNR~J?nYnWGagoQxt`UJ!?p6ebX_Uhw4{SAT4nLx4o(<NLN<k$?l9(9ZM=l>
zXt+<|@@B?{b@c$*Z3Fc3ck_#=s|oGTqGS`3adv5*2k5zdHC!2JlS-Voj%rw2JDl#a
zhPnuaTb}Wq)b_|``;}f?6s2r@YTmeex}aRb=eH81SEcL=o~Q8--F7OgW+HanS|gzv
z+9x}9I^rHJKjc|g80SyDD0YQt?e9t5R&b3UK7Qh?O>21xy30$3Z=okf&<B&f(~yLV
zn2EEND)(2Z_4>EWYcZR%Ng|z$OvN0$rgcA(nHO?nOK#go64cYHjO?vz&TU+16y&d*
zqb%E*V=A+s`BZ;PI#z&d=02|zyZ-}rL+ib^+{WAPU$Ao=UFPKCLJmjDoa1}0S`#OF
zq#Bl*#0+f8<%}mbc0M1>A77}=5PZj+G!x$xJNi{va#zvKBG`PJnrdjBEFwcC;uGP@
z<mL?>KAMwFBB}%T+<Mo)<LBH@OKFC}x^*7zOg|Z*qbQfLR5nIg^|F<@?o?FV{SiR<
zWuzf7%9j0FmB|3RCWkvt!Bumn1EWo%9-{Ltm(%So=(r~mt`0J<GXyBPJ*<6TJHF=9
zmJoe4@Nnj?lE}fJ((T+2c?RiP)TOSsG-^hk=`f}Eo${yXl%O9(U#}wa7>yX8BrUr0
zJZmt&0!>^Rd<J5?mWW2KA~`2L0?vZge4p@s4Xp`^*`ck#z;rb<B-p`F7GBUWNYh$C
zuWLL6G*tDW&g+Vp5EEbZ+tb$BQSu(WB-htG7PObMUnm#LOa?XZ1bLLq#jf5`$@lJi
zg~r|NVZMlB=lQ1Rx^OBl^8UT~qI~>gbFG1Aj3fE;_jccK_*&)vv9J6=a@XPY9ScU1
zuVY0q@2<|C%<nlZNMCX5w(z987{6Ug6<~c?V<36>r@G5BkKopAwP5r}sgZk1_n}Hx
zx7MMlFNw}apKI{e)q@+`CuZUM9O@p=ei25GS~9Mve*eAgD%<q2T!ikEZUqiY&W|X$
zs?!z*r>2T01`CIx<2}iKM5A(H=CV&WetWo(l=POzJ>I*1DRV&SZI||iH@8Q9drTQJ
zGhOsTN*mks9-qGrm9JeI(MAUvt16M=huG2?k1l7Z&)uBFR;CD2^PKs<HvRmO+uAJw
zol0`0g|6p2>w@c~H?3G1P%B5L7aFVZSGK(J3y-`TKbJnM4gNH$@jSJ7TVN-aW~}4X
za^xAsU9lAnSr=vVv>)fhq;6_#R?M|VrO18`Hjr???|8%fc=7s_(E=YuuKiENBbJ>n
zM~ijqq?Dn#Gj8pd=;F!D9HVj@JGdjU*^@k|`6<Hod8_I=e{Rv3@i-*;4P-f)zf7;8
z=ZsAp5`OT3Ze(MyI^V0kmi}n)&3Myso152(DT3K4=e28&k({ZT2=Qr&8t2VR-Xk7E
zKnJI-5Ipkc2U>&Tt6VFxv8KCdVb=?oO1NKN;#f^x)ZS=GZETwiPdC&Xdf>y?anLd*
zVW7^htv={s^mTGOQn<sZM`0*5SB1FSa%TQ&I5!h}QT+7A*Z1P3hZTB6`s@#Ow)9DE
z9d0^_zSXtNWZ>KNr+l7Vqvh{YGk<4UrrsZ4_3PxIiI0v;2r09iB=gRR**mqyzZ^1t
zl?INll<;<&_G#Rn?4n0f1u}VhkE>QCjC<5pgFl+@wCL--BeN!VO~tlP;AI@y?#^f6
ztR3AEeKlV?N#dYCo?AbLiVN3UEzxM`4f>hYW85iXLX`N??1R8wk>TYBOHB^b8zjAq
z0ryP4$%M1u9UH$hT^4Bc`PsckrDw^ztA7zy5geS~71Jw@U8C*n8hBotEtlIR{u;qN
z%OCV9K~B#nwi9<*{vB1J4rN-NWAm+{AIa$=ZPR!Q#Rdnx_)6LiRe_Il`a?R79a;<$
zcX2H0=E^BuRw=se2V=ETS4%s5v{_}=$(U2d-S^ZgPE_ml^UK8>I45)Bx5o^;Tn09_
zr*1dd$Nt__%)hN*C@!3D(2#alkMf;|m^y)nN)C??-^s^2ijVGCIP#&jem1NP5`;}_
zb}CrD2;U76ptdFN5Obpl3^AA${6pz3WlLcF5#MFVm}^RTtitt!yZYfz{y*cKlQLY9
z*T(XhMXY9>`(3KenSc5rErQZ6s=N9%hQTs;&)R>VSHoR>pAdJlBQmE(7(9y*EkBr$
z!(KTyD)`KVPTiRA^@5A>824?|h&DydN|&Pqr?_yxJ{7vp=gEBY8TW!{IzdQa!?)lM
z%efVXp#ir?))l&gKM0-BD80Iy@G_S>TlCU}O4d1kiPGt}<l8t7X_zzPq}`s=@AIvy
z8xpM=ztmK#&@k(K?N2_hv@)pL!E*{bM8WmV?lohpT>G*7v1u_=e)kEQ%a_FEn$2XY
zcfrAjB$`M|V!Hg*FV^ABs_KiGTdciZs*jwtDnj<k)P2R2XLc?z)v*O=Rn+Irm;{Sn
z(-{ym(lLC?xB1W(=fme!$}ETe63@wyepgjX;TPqs$L;iEU)@^UdS~X2%Xb5fdUr+h
zYz6a-&;#73$?M-U_tSrh>e>jR*+$mg_~foVbs}IFWMe72lsm3fF&TY$C{<-u)z`G=
z9QQbTb1(kYksgn8hMs-!^ijztA3>cl%K*W65tY+b*4?FIfqE$iMHAUo^_*57Nz-?R
zmX&icjy}5{-Nl0I_gS9pMBg8oncN2$`l;vGVEy?t)5x*%+ZKWSYpXp)y7o?fgVP_P
zsy1v}s+2V9<tU!C@?^?v<Jg6^N>8#G{VMEp8M~$LWoh4}sZaEX!lPX7^u^odH@`@S
z@u)8NGc&q~eDX2a4>J2LCW2nn`Xqm8+M^(^^ilDQ%Z5wS)ue}?19NL*Qua}aORcN-
z@ITA=*-?HDZM;Z9?)N%wVX4P+rC@`KxK^vW>>LkL(}S875E2ZP$;V<!T!S$f#=|sK
zIMJq!(&fAFs#|04^}1^t55zy{MUO`95nz2I#!aKD&R=RP)m~nymm(^0E0V1I$d!1(
zw|YmicR=f_^kh(&0>StXq5AaHy7YItbypv%xpFkHd8io;31>{dtucI$d80L;V8CXb
zc_bgFzcCjTZtIFCWJCMuR1dSjB>&MAUE@^e^`Czn`6p**GW$*=N<@e!XDTa}2UIwG
z_#P|gNU)mm&gTynI&5H(n2r2v7cbuOxm%Roh!MRv?LLzl!qWHMliRL0EBzY<zW!=e
zpU7K|Rikd#bEwH2#d*K)n*GPWZkUAlWEo~Cek{MzFc<ywvL))%UPyiI^h#4?Q+?@S
z>mJpDIg8iqRA#5lGgp3kx;AovvESma1w^<0F?Ja}|3I<SKri&XB-?nqx&_O6{KTYI
z<ND^a`%A1(H=+|&TAYrET-+^U$`$za4_On5*$AmzywI9uvQsZ?DkOCpj<jYBh6v;a
z#&kZA2e*>Bc#wn$zVb3z_{o$|`z`lSWbX0ZaZI`qQ4^vGZD3B2;)n8u74ea=NX0j%
zd50GzKfr-5A3fUeig9QwjI{*#8_(Bku|wzX?P-SXZHlvJ5YndFqBgyfwq-N@i8?*H
zHvUK8+VB4BeN~!d5(;{3SL4^wRtI_w&(_EE)@xZ<*0K|e-(8I4iuApfpyWr8o4W41
zOB#i#BNsYxNb|ysNeUyDaN&v3;^}pwz@M&TcmIfE&Dbt;YCme$b{<OTG#Gn4>|w*}
z`T0JLi{^Tdu(tQS*0ZX9vB<YJHiRBu{BDor?@lpm%t=1IR7u2MB#^)QvCwNJUnu<R
zMbt{;r7!dm)a?6)-HUz^v4$CW3v>&)9dY$j#p@RQoF@kC+(EbJ&=IEEb8I7*wE5Z@
z<na;<9C}<O=o_TI`K_qxYTp!evMG|wIH*ssSawqWEJ}4pJGnxqQ!F@O^s&gZQofyO
z%ysSQRzb~Oru9(kRn9_V9Foerg;`6z2Em=+swo&B-sP(5)g0tQZ?QK9mWG_yT;F!Z
zL&eTzC7l<3%Kw&6dy4;yb*EMBsL3x<+nod>_Re7ejb_`CQx4`Ou9<d;V-;yD!N0=}
zlxS5$9x{`WQoD$q>>*=Enlzh+8aplWiA!fYdcIWA$^~I6*TjI5T3OYu8K#kbL&ddn
z;sXCfo9?>v-Rng!$2hE>)SD1L_Yu0cf<ZB1L*8NO7;!9P{7Wim`nR#22cF$(dlNZ&
z#6VqUX*+{4Lsu@YGT>Ke%-`z7MlG+XfZji0e%Jg7sJn3@)Fmrev~Q|bc`&cf>*q3A
zP6jR2P%HC){$-G_Hh35yn$lx`M`DEE*2C`TXgzjyUMksKlb2BE{Z>ZZsDtyWUg6=A
zO!z|&w#5<_#u4i=y^AQL>o1aT(%A1h$tF2G6D^&d=@R~6)N{gFTQ(3=@Xh86sqB|(
zsTXQ5HrbTlmaiILPcRZ@7H1KGE3bCCahp`X44#8TB}B5~<XXgosY>e$Bg_#^xF}<{
zy@cid*?6_6-C)b!@ummmej7hk!^_<E_l_rz(GBG2J0-6scz*ahxx;x!X+-)w48ok(
z%F(JAhfW0y46I-YktZ*t{XQ(0vQP(cM+(_`<F5ith`VJt!4p<NgfmkRvFb^T_U=(~
zyoHoqh}D3Lou-g}GfIQsRv5;i@g6x$E+@XaY4L^+V--z?1zgCWv_pkeQ+RGfVD+Kx
zokr~G*HINIZ05z%leGJ!XkNu{F)3eGr>se7%>NuIxeK7d1zMh%&+|_)UcMo`TBR}D
zvvy&W>8dyTWKd_-HHVPm>qAm%dDfj9EqIvP9_gu$m)D)T<wwZuKa7;Wm^yYEA0Sr>
zO_P+ZEAHHV%#sKY-$#Pt)(VR}33?;)4!!kpLRw{<o!;Dcdz(8hpYu^uT)u0R%oaC>
zjq6(t^6wMZQVD*HLfLMd@91#eUv^Tle@LxS_e86<q0%;f?eqhzvy>oL#j|v_i*x<6
z?s7Z#MoNtDiZa^7)KW1gO^A?_T;8Oz)wy$6NR1lk(0LGP%kyD=hr03EvfAPXuDzh{
zMO4GLN|rB`hxxkZ8VK-UQ~l{P2P_XrF%V`p>3pv)gpbn_%>OTQMok8Z5^2vijw!Ho
z?MCmbl{%_czU7mp*S(oBF_T@v7+xU%xOqBgU#H5kAtJ>)s^5Z(zgYIzwLZNu9_5j5
zLd~2qv8PpG7%UYlfBaRo$Yv~E{h-3**x^&(48xDebE~!(?yLImxjRqNX}9Axc@4hq
zJ}kpW%_>M)>@NLMXlpUQ?pVQN{9r-|4~4E6`V&0vpS@Wbxt!~o6&q4Pr?c<{o$mBn
z>{+R8XGaU(Bi@}v)YZ`=t>$~JYDRy8Y5uyYKK{crvc77(N^03Ns(~UPs+W9GX2F!W
z&4HSIK>8*R7p-MSf<D2)z&JgPznbs#pD`87@#ibZm9GENDZ9={ei{EV_5$&_2n7*I
zVo~DfU3axedq&0ayi>T}dDGH;##O*lpic@9FBGlg>HIOiJQ`h&LOnn)FWb5c4*GN1
zcI+n(*^ax-3am=lh)I#Y<wg%&eo~5W`2>qm2&-F|_NGVbJImMlfyat`1d-WNzr`zS
zgC!4tFkG(7Uv+0Z{B0Q)ZmJ*=mHRm4#_sI8?&0RQ4;!WdW1)}S)UOt=;M<>6&nC`H
z`_g<{A)*Y5ytG*vLNm)2#(Dp)<nf6))vhYHW2h^ilNbMqo%O!1zK*@?{@&1c!@t}T
zVH`gR_S_>0a(X3Ysd-r}cT8E0bTXGqmB&jR7qeb#=4lqQYB76IKm1G+$tV#jUi83H
zsv*susu62c^}cDP>%%QB``F1ploH<xP5CVzrq8`p6Ew+4>HH*d!8~rsni}UZi-yX{
z4=Xk8rGrZT-QStMPEYNvG<z++?;d6=%<uQ~yw;RUd>_+)Y|!r)9Dau-JMGfO!l$e8
zzRhXuDj5|+o3oLJ_%l&~55G%(|J(c*XT_;>V?c0YIW|{3-h!rdWOXp%4d+(}eEY1q
zmS*bCk+ROJ(0w<-kS#UNmL!_FADWq~aRLqs4?1+%1X&MdI}&-5X(CgaLnTMi?Wvlh
zQ8yX`A3VFi$XIn!s@U>72Uft`JiNjBbvN<K)sxp=jB+tgACDE)TaP#yC}g^SH)wEf
z-k$i`W4@5yk$&S3>%|Jj63fv@-#2`c&1Ri^LD$XaJFg7O=c-6W%#_Cj7-=V3zqj>s
zqRBL<0(PQqvB*B-Pb$}3>QaH_wb9>egL#P*5{q^ToewKF_t2EtM}^#1hx*rst!N5?
z%d;?M=`6#}(#P@a0^NU8tyjK*$_m@6#QueTt?H{om`$Elt9yKjoHAzj(Nm8r2Qf>P
zX$;T<uSo|f4|!e(4-g8u@7gVFJzUt_D7V=jze{aYHn(kZI<l_Vll+vyUZOSQ9FxZa
zvj7{7YQc{i<EW-J%eaIgv_tjYu1gELD3<XRx;&+Gc)_`VDp7c;II>P~mB)~rfUjEq
zWACp-F*)(Xs4*pvS30j1G|cQA8yB*c(gYo<L$s<|h})G#y|Z0WC$2d>y*p(Qmb8?z
z$G6$<=d!cEIt^4PUB(C(*vKZdv06fVp7<_}cZ-*t3w>V7<$S{_5L?_6`>9+tq2O4S
zexC5wyhc%sW)-|0?2h(xbvYV(iO`$Y3nC0}s8(~FRh-1|Oo{){T4l!<H`O^lAVsq!
z{-mW5*o`z+9aW$oC;EhYA^Fzh$&MeKCWNPSf2#JiD-(-EP9CA;<IOm~1dwNM1lm66
z{Uhj+=^x(ilDTx46~m^sIWl9-u9f1Hr;2IHXB*w<bR)04h7j}0(|Za<#UE0p`0O<k
zVy&(%GrJ39CHQX_z}rvQPqP=v$Aekz6%IRT-cVspwprvq-F-);^Nr@~f!kJR$fZXu
zHLT?+YXL^73(@EUy{JyBtIK;9#q9D@IVYQm;W^q*BnPr46o_`?{UrH6i&*VWJ8%q+
z{%LR{&AW)1Ifyyo6?IXV`}53T!Z}~|jlg9L&E3w1s+1)z%cl10JVjCWe#|WAD?T2=
zEZvEsaS!EKt*<2S%Dw;noNl7f6>dql%m)#QenHL2a+f1061RC;rgk|3Kd-#+4}W#4
z71DG6PLsyD;U6Q@eV>zCENW|boJ&H}7hhOE%v~KECEWY_6aC#}l%hFWAYf}iyY|ta
zpc0F*aQEB_=DeY&>sXzCxVjfI24+?_qf+)-J0+6!;<wOids~f!rS99G3R%^K(Ib_t
z><@lp*97HwPkc5nHxfA}{1%v&TrM?v(MX`B&dX{D^A<h;w=KIEt~>c0wjHl|msP$8
zPzg<T<XV^*lhR_6;!<N?M9r#Ra~heHb?v2Y46__|+bo@q-6=J>pyAq!Zee)g;&A0q
zxWRY#13#8BDOzYXSjWA$;<LuLC^JWne&b)y{jQKDTKO&AEV+J5kX6f1Cc<^QwA}VE
zXzS~sc*m)=dR5=_`LS8{s)&a_e~J5`w8JQ`_1ihtOgGfe<|@A$8=A&``e#YtQ)}<`
zw?}BQ`Cn5ciiQnue|*Amvf1{Q-wpMp#m?O^a3Q+L-_9{joBhD0`qq$EsmA((rW(KN
zdo$;@w~rMjhDgn>(+{L5yBV#bH2U4WOeDqI(%m$P&reyGIJZ!4w3o&xFwM$1&3^b6
zO0OhoZ&^`V7ckMlkoL{Z@qL{lOXc5SD?gWcE;5A_Stglhc9G02QOYHiRvU6mLW%pw
zr6b4l=X%^ba3wNkI3y>@(f5SUv&+{o&7E^%GJa~g4Xts+j|XH=6$oFl-x#ME*9c7)
zw|JrDr@|tDJ_%;#xL{tFF!=;o@OQ*m4twj7R}4H{YpKQy8wK`Ym1C_Mc~-1D3E=<v
zxZ9CbzfPWWO&v4%c}6T_#9w<VN^6L%u+q_<;#zmj>`$x}j-`Pk)l##{B;me_bo4Z_
zdfw`?HI5#U-sO8tmv4h=tR<7(sU`eV)TsQ-@!RROTZ<>{jF+`B4{OA4UH6zHcEG|L
z+g&IyR@M`+(60K-kHPnnPHcnUrs~n^(RlSInx=@VQ?d$Nk*CK!hT`ePeC*dU)6Qqc
z6Q9h<H4$v>2VE9y@(D8upJRHPCyi=wHc?vb5_$iE#Y9!nvA0HwOpoIu+G+U-lRouO
zkq((}&g~Ff0^-`Kq8tuM!(K~6<<fHN?Xs=hlS+oyl0Amae{&93l}miAE?g(X8#5A6
zkq#lxFc1~6?^+S`@Zoce&_7t1jIa92S0hEPSRqHvduQ#QbkyxPq&5YkVXfJW*3v(J
zC*r+TmxyP}o?T(<u{FxtTX2`BrQ_Oi(fhU>{VP5~j3wx^1@YxhGsXAa)QoehagUOY
zaY9HZMYaxVr!OZ&h+A%%8u7Ym#uL<e7%yWukNA8xP{qdkh=F&7<Q~_{>!ZvZ!D{Wg
zb8srsy)1`u8fxB|1XvF5rwJs8F{Jpd+}rUZC$^-P$Iddw!JvK{Qkl!GC^$H&*7vHP
zyg8+TZM49m;HKfWmB0NGLAlDzO+t=TO)Az0CaZ&kaZSmmGWJ3CnK}U<DYX)_b1ROl
zvTCY#6uun<9GEgOnn#yN`}?oG?(w;5DD*CfRH?P~u6mY6y~rg8Z`%7B0j2?asg5-n
z%=Mf;U-wmi=pWEU$+>+?YkgvlW9{T;X6^k!|9#oqB|DF&(sDyZ#$j^#7A7c~)P__o
zW7WTZgda)zt4swqJ8N$C#Pv9Mej?2r$9QTqv#YJL?lu`uw7$8W!1{M#K<9TslmBtm
zD=f{xVV?K)0lc-uIf444Pp#C`odUAsE5(U>`yW0Lcag*&$Fk(tVzGa`T9Mu8`0VF%
zK^@YkKid{KBm-7ThGx{-4)@+D%q3rON_vS|_owjI+V|<Js16Ldl3<j_ifdDTHa@EJ
zcWxT`1KB$!0gbyf96moA%Irr({s`Tu&GnU$*DrCu(Mex%jwB$r3*Ts~E~wL=^{}ZU
z;`)ulj_d8r+X1^ns1C|+=3EXKKIPB1FRV%-SP+jGc9GetH7~LFOHBvQN`5frc;Ti@
z`Jrbu&q~N)+UQUAA97gAbRVxccq$EBq)|tY;jpu_JoEA?Y^z`9R787x@<i>-OI+S0
zOhB!vd#}}yi(v#<$MZXiI1T-1yztAL<Z6M*C4XK2+r5rAEi}t>_*d<6^GZMY7~|(t
z<%|E8$=2j2LYouHWO$8U=E!C@3ouC$kC3zxw6m<RW;l13X-g$`|C0D5+tzU{DMV`V
z>W`i=PyVbu^LM<*L9nxz*0OkIFHMV?&TJ6}xpS8hGP+ux75b39w9y|#Z+`p)g*h@X
z<n^Co87FP>yUNl?-%W1iwPB?c5csFHK!Z%?of2faUzz=@40#3zNrQWT=3R4P26ro{
zRzU&BYNLVQ6|{r&Ab{R#CXRLf4@m8w&sdCAs6_<qd>Mi18aO{$=C0?c7fB-BeYbWa
zi$5wrM|%o1&MAPzZ|*gwIkPVnB<xNc$k}1!#N?{NygkBXlb-KLO6xVxq-)$&e2w+m
z^2s*tc;R8=-MAn7X`{9?lU5tMsaY@Gd(rqb9-GK3><)HTUOB3PjxS=uz>%~1jezsk
z$FZLVDnA{nl%T0BB5BT@|H|PlGNbFM%Gv78O(FT2Y}ez7{$9EArbyoV-2=0}Tgi%;
zG}mDsB{UZ{tUZPfQKVn3&h~^lg=&JGV;0^$_|5^#*!`SBX~0gp?+W~1oEyvd!iq@d
zYko-If^i6l!Z7eY3h`rS-P@^~UYUuS*1vyM$sS%!r2vhR$%<*gHPhKJ^;A0-URm$n
zCD1+Lq@1rWnmf}0!w{fvXX$=$6*Kf9u(DTPp<HU4KY(#D63`zKC>!&**kMH-hIsE8
zI8XjUdiIcc8*IW=A00epy%WJ;87Hn|A|}i?KB)e9pv{7TDfPiX`QJ?TCO0{L{rv;=
zZ0~o61+Uu(%x~{<nPoU+-6CJy`XlEuv(}k^_kcGFv<55950kPtA6ongKrG1&`(>U+
zwwNjngrH2ayxKt`4!kd+CGeW8hscQb5*GKwOn4t!mgbM1C3~cE5M+DK?bE9g?|g_a
z{Y(XY?YG;hV;g%eIba;#3r%ARFtO*nqVJm+n8SG!Ms=?{OG(F$!kDiC{>nI5Vtl4q
zI!xGfUMz5^7g5$n^v+T``{@o*yD6@37D+}U>%a1Qdl%oOK2&~a{FLVQ+H3!hDhADs
zy@vbD1p+NK?;pok;&yDE502gROvR19TFBauk<H0HnEjg}iwN(dXFr~*-ZD(CE`YAm
zp}XHe>IwR6m5SN*U1YS%49JJI+2OT&snE50|Et%g?cXLkcmdF+x&`!|&VVV0a+@mT
z?jJpjxvRnkLhQf+DR{=TmIjkN1?Zdj3*V=0>F4m}=Svi=zk?9%mLn_J0ypVB1Dxlj
z1<hglVv~g0R#lecnb|jNm8m_-q79oU%CEtTYFIelmQNoNHs=bW?YMDT3u%V7GoP2*
z(&iugBKz?C|FI8plO3D4KZAa@K8m66oa^U$cmW9(Be7U@AfPn1ARQ6QA^B21)KU1>
z*DmpGDM)%+k9{`$vVuv4OKmONqJ#y*l)`2eOKousrro&S6!%+o>6WMm=RJn6QD)Zl
z$xl=?5=ab%;cU&IS>%_)wtA~*bk4I;N&@jRY1nOpnB?f!8~2`<;%7139y$LjSS6ML
z&glach54K>R+Nv5E4W~}1(4;Qn<7u>q@YY+Gn{HA8OE%9u1|f2Xc%;V5hhttHnP(f
z=ji;r>GInmvY=7+A_Yg2#+qJpCTq$>rWML=7xQ89oTujgp$_B#PkvNcxl7CnzYh3Q
zV)3f?5vnOeSWKxc<jCl~c!C%@IBIBoj=edi&O^Mc?JCo*@NP@dY0k@D?cHX79U`e&
z&7qkS<_)PG7h4z2;DvlK_p#g{os-Q9X$w9f^MR#N1segYTn3NGk%x?bMjui^Y#7MZ
zyv%ZBb@KP>{^1A{Wa#mRov|9!x7EZI(j->m`tw)VMg){|qIK^qeKvPtR2uHH$^EsT
zAV%{s<-)qy$>9>sL#K&%4s02JT{bv^qgkL_FW>9NW!TR6kfv$rR<B!KVB_eAR|A&^
zHa+4Zt2(s8$ukt2wRY!1{0b{&fDmvY2|6}#A-tHtBp{$#=*cMYNyI|)CtLY|MW|qK
z0`b<vw3Wy5m4yi(5<z}r+(l?}8LN`&(vBy|#s%0)`DfL43_M^oPYS$$!FLNfY1oB*
z2{`yE=WCErXVAQI$+8!w>mos5@`n)77wK^Ga2U+M1mkA7A6;JKxOd@Ia^DwG{Hrim
ztUJRq_|lboLLlMScT9xQFIV{<@UDlO;#{Gh{Rg8)d0yr7tktUE%|>B)|K1~VHq{S*
z^?aN(m!DnGqg<X=wV{Of{M!ONj-P)AWUM**yK4Iudc+%ybEQ(#?5EAw7RG~`_o5SQ
zAZAAK$JiJ~@EjCJdUr-b!$3T1z(7-B(3=@O8J5hoj#r!+N`gv)bAh>q7PlhCGvW2Q
zZ1InowGyQqO%MK)L1_k1N#8$q?z8*hbnM&su&+dRm*{WvMLDN9TMG2p@}Fmrf2xw9
z-EEtrs<y&<?4(f=@sG1x*WLaPdv6_8W%RX+3erd`AdPfOcPTAl&`3*nZn_bqLFo<w
z0qK-(0qO1rrMowtwZY%H-*@g9_ng1)IAiR;_IP8xYtA*-oX>pbn%gx!QxMU&-5;$r
zmM9ZExjjOPZXAo}Qbp4pFSqdGEm9#jPxfFa#_^SB)+Z}F_!O^Z$3za9enkbul(4rY
zatdGg^3u0!_8jUmP*d@=&Q#ztyl#rG_u|NyL`ET(Pd~u63<@>MrP*OUEJ8{f*xEHh
zDB?OZ>>LH9IE5+-`*U@7C(?l<TV9=RWorL^MCOhCyEKWh7u0y>GuD6>B$S>+rR6*S
z*@myrti~5WfrlKZt$Crj2WTzIb%0Q<x}McYUjvx;lq?WX2rTC#SqIbsw}=|OtW+^o
zK%6PwS#0R_krYf>Uff%c7i5?&KQZzTeOoHfbW|i6MQDJR68kXOI}Sus0-E~NbFCCt
zm993^RaesMPgE}g%VYi&TiVMX)3>i+$%F?%;tbe?w`(Jt;B3lUORAK!v^D$v3xVH7
zEm`nI?!nFRPgI(ls&v7(#|8CD%@UqlZK->iaY9CD3QJ6xTMZ85SoVK@xyE9xBv;g)
z?5{T)`O->JLLCm-eCa=)T#R<oc*cd96aN*~$V_=${Cc9S#AZjcLKctSow20J9bJf7
zgKNL3UGTi`P}l3TV3IGAb1HsB+<UAp*ZpNplV!|n*%<0*4@ojL9p|M&ceob~7W_^W
z<$P}qP$B<dq45OcH)Oukdp$JWT#>!mNzyb8>q~G`&J(shY8P*$wNEjE0}iBaTu-Ie
zJ*RD7BPY(96s<(*96ypzrfkGL!&bvSL`;0~vgKC>X;M-`GZTjM@s0}(*a#ePO97IU
zD%`C~jFUmCClS~UbJ{4p?l-%`<I1tbZ@b7n6D5<3%ilUppjBqYh$N@i=2IUu`0dhW
zcyagQBGM@Px(1Z}wE7qxuEAMp=A@liP<EZZXV*a0x#IVTu%6R6vzeph-|M6Lo{Nbq
z|3<Qnt4~SvH%qhSg=})tH&-JT$Z79Lpx|2Iq-t{F%h;9xC_3HWFRf0R41?k}ZBdDx
z4AvGlcz>_AbqS)ejM_$R$7WMY8J=+Ox2er`vAShqn*QZShxELojyGO)V=Tkor1VoT
zpI7tJsd{6y|HGxBk88ODcaP%dt;2Fmw|RX+#N6NUa~18rCRL4vol#9IV(8@NdM<pV
zT(F1fCVIHJEPyeYB6||a!=t*Z7~09@w@e%vA`yLo{~?IdiFAt3HRT`jnHMw^3x<aW
zm_l|&@{|NQ>Ja7u&f|(`;A^$~MNKGsNIxgpvieZ`t3tHe_{ktg6{9TF%Sy-`sz3cB
zDcE7!wtzV|%>d236SDoYaF?k5eJI1%d0kWb_(7d0h{ztJB%|;ypZDRI#%2DEmO1$g
z)yz$8!&LEuKLGJcS{eF#o!A?y)T~ahw^TP_((f3wl$So47iiE75V;ucbL}gJ?!2eV
z19oMtQv2}>v7F81YV5NMfTZ{e(`32W`h4T)7qN==097^lOh~}D=J8oxyA&m6Q052c
zAHHl;{m$+d+o1%Sf`i0wfc2-1^ejZxRVpbgI6Sc2t`*JKR_$heGiSI|y_Itz`5?dy
zSP!ewz`)-xh;}I9`(|Svpiag40LIg-GbZr{FWd~A7u6~vY(*>HuV{EOqF@E=)Tktj
z1toR9R$-VK#{bm9PmRKdZ9g|^Vs+JbX49m0xPc8(R0>WGVyHe4BW_xK)h@~G_~tpx
z&&p5;9RJKZ*ZSGwWWnf4zk9Pp)SS#V+yF7Nnlq@*1$!&ek8+N3kDQPq0=1(pm5Ep8
zmSe_olA(B1J2g=7y!!X4Z_B4Z+7MMQ&7`oM0&>++S3Ih&)NJ^_i!E)q-XKnQ9he_6
z*y{frjvm)HKjTNF-9)mPHeObO4oMi9gUmk?ClR9#E^oHJ(qLgelKIvaaWjX)3dia1
zY-@piF>hLmJP-we;UuhgI#Sp@rD=m;itLiz?f@o&dP(+fCI&uduSZzwHtC31kP2!a
zW|xSn7=*aftm5sRc{Qtaaqs4q6MR!w)S~1Lzy4}}v8g+TR*IO;M6(~czs}dZwrW;>
z^Lg65+0kiL60;PZVP!;RG&fCv%{Ds9c5|l{3FnbPf;$J*oc`7CG^gbC>%>=;&bw-d
zvvd0vs<p&dZp17)3vA{EcJ&%|OldT^-K4qf-n<TFBr`v>lAql~y1(<p-@|4r{quW|
zZ1Pyl6czZdFSV%t?koDwdmA_S0aNh<bqa82_sBkMr-Bt=o7-vybERYw?l<1H_jtnH
zTMW8C3H(J=PHod;3e;FdmaU;XB}R%R>~1_I>FRLRrzfouLh4WO;ZxYY#Z+^!7-aup
zpj^gGvpCRu**2yt59h3893WV<5Tf!vn&U$sGgiFI3l4Lg55dbdhMib`?nzJfq<H^4
z1EBWg%Ix~*;?PyqAO+@S%);J*9iF#UPtb$(mczmzMzLRJ;7QyktZ;&}o80D9KDoXu
zsJ6`5>-^g$4YQ0*GncBk7|x4)>;$l=?>EBE^RsQMWjgR#OPoMrV(_;(cW(1>q+Qn9
z!jBK(^1Ikf`E_hz8EODU-dl>(<|r4Jj>TZAHLe%T^ZL|nG9&+VZSUriq#K}HRH4%H
zD({SvEzk{IUTIWa3oHs!F7N#(KhZbA`H|kYA9{O>22sCsnj+pEQywm_VtO^dM~<Re
zF+LhL(EedGOC>i@uxRT#s9Iq6^Grz7eniNrK}RFr%i4D33EwKspBPrr*kgIiOA&IV
zqq9Fu52!BzwE_(yfF0_U^tAH(#ABiz)oq-KAmB1@!M;x`m3^B)1itcNTXzyor_xde
z&$VfLgAH|eW8L~Xei{4B>fr8dB9yqMkNn#E{AUWcl4AR`0ohYmAA}Xkdv6&_o{6+e
zj0@oP1#<meXHhTMz8vQ3c(ly}>@60P#-b3Bnqu!u(OE-x)gh@ZevaGsq-P*+6Tw#4
zeHaiR*fI9fQaHz#*x0=&EMygYedtG1gK-mdYJmQ6SkAtPG;8%mfdmtu>nHv&7bvKQ
zByr|tYT?Te^<Ve%w?4#9^TC#^_jt1TZ8HUV_R5dcmE0Q;2-zo~m3_Abvd1c3Bs=xa
zAG=q`AMd@l)o||qm0?xY_1{8^pt8lSFpcH|umNI=OfWNe!6!H0sOCb53P==17NAZy
zPoaFA|58`oPa;KBlr^cB##Zcbu%dkM?xR~eAJzGe6*K^QvB98kgBiy3@Ojlsq0B~9
z!uNTxnoiaK;zd@4KN@AxAmt<nLakc8<v?oL&$TI$Hr=lJq47zR@$psOO|;sz{=rD>
zOiovotnrE$8Sy)s&}`f~+XZ_4*!M+op<`f!XSm!#mzpN&ZX)D*qpGBlA5ia0ydiVN
z{FHRFl)YGkclvsM0zr6hf$S>tOTOm9`b=se&(98g&#|(ucEML=@o7#e5FOKO=v;J6
z$w79618=c*eA&L^li(6mR0vSi=qGp^auL@@GJ{V(lb%1Hu}#CBb6EedyZe?b-twY7
zP*g94aoexcUJ%&*LT*2#`f)dEf$etQ-Qnr|=oxdtFlcmd+9{?8u>7qiy@FRt6p!2H
z_h-+0_XWM|l#w`VutS2|QXiu7%HKz#oT!3x@d{?VuVMK_b!bI2?cASGev(=D*QHqu
z60Foz%X{5{Ol!x*RAZO0yyb53{;Ms=vQXugLaRT&pY|pX{QT~+Ax$;Dl~Y@*SVPF5
zc23j1Ro)D(RMXaK(sy#r@-TDqhKg#}wu=b;OfpaN({Nb)s<_(UIk#ZA9#K7F@zvO=
zYRV=2q)$l19VCueQhe2&_9OOa|8SUmN#?cg9R_(T?eoR&D^q_6pbc_bh&7TnRa(dG
zxkoRulgzKrzRz+rFVY=Bi)hS6XNBu>&THO{SKp=|7nGZE<a`E>i9qhYfyyGg!jIm-
zcixO|z8`0mu`YNFr1`EBPef^IPGd{6rSh|J3Sd29rttj&Gv9$z_FH04DljLtMcl<l
z%tjMZ^Ppri)zVU9Hj$%5?X?$v>M5(+Z$?Q&Lo!%ia_Mu);u@Bz>Z(l4hrXUq_#2o4
zb6D15k18Gz?Rdc(o<(7!1j0~5%0G+?%2awmW$C1{@B%<Y0;pcJw7##W;P!S~_swpK
zb8R;jKVIM7i{?1q{@J?d;kG`_nBEqXM8bQljaV&S>mHEd>u{?u^Xnh?E+IsN<d^UD
z2kydBr)Tc=7VuLTbs$5WB$dqUYENTqG!RRwL4vB|r5>wFkPZS>nuN)}`(K+8$XsQD
zX@(q(cKoof0tebsy_ij!+!5fP2GHlg9OvE52wN0iH<8Uj(10i^@S!2WhR?aqgb<`h
zcaybX%K9M0ACi(*Y+6`=nLqs=mZKr{+tkFga@)sISZfa{OxQw?#z2MF0x5?vo?8%O
zo%sJum&4;zA{WSQApM*t!@9c>_WE?<cnQy7dTE9Q@fqXWjg)C>?Kp7qxJ=F*)_h@O
zDt}ytE%ITN$=9xkSlj&{`_^?{rv<nVh#y(9eJyCt-wXN+au1aLm4pyFEV4J<6d@a4
z7b5&WHhpmIC>3>v@E?fJKNlt%%}E{2uMtGWM;Uh2b7Sie&ruykVi*{>tRp3Y#a=^W
z9XHx$rO77UO#adokC6lyE0IgQUFg(c-h;MQN24jNm>@-x`=WpqRv_upf&dz1PomZ)
z?rxt=)tLEYODd5xsCiDnET55=g0-gd&V(Q`r6AWAWW*@h5K@c`MdJ5GbeaF^+-fzy
zceQ)sQJkY}0j*xrkwP_~6q@kTJWIt5I}u8_4H?&CYE~OG)GT0~6Q}mbR<h6-@@~NG
zKCvLD*AydqWVy-S`A&5Irf%=JWG&gQf<z~f4_T=k#JKdCBm8U)bF@`ERMO7mugN_5
zkoVilOH`t_c8M)c6DUM~X)_KsY4oqpY^xQ^&1P{w&cpwV{Ij8H6(P;pt=CbF=iB(V
zZ+(w7pd=#0bnj-c-iE6E?xsvr8TZRJ{uM1L#k&`i0Wx$HLyBV!NbFS_N(YsNN4Lap
zFQIN%l&7eNj|baZzt>kH@>>@gVylK)_!*}bvzL3cS!!_GHNn!k4Z<BNwcA=%-*EA?
ztuOwjidf{ntL22p^D_h@z*eTp&>GRhLLq0v#Dyc;5r^-Y4G08o>}{!!0VR&OkF%JB
z<GUe7)F9xc3aJ<QTzwBvUwm3_5)D82C8{!s>H*2aGn=Vml2>SIE^?hIoQx!g?cT#j
zP}d7A?qsy)Dl!%=2@dmfIr>ykg5q^;7I~8gmFmytXBB$AW;b179je5>R}6(n0@qJr
zfK>3Cv9+*fb-r}b!9_1G(_E(fO_!k4LB735kF%1Z)y9-CV-60<%Y>}RdtVDdgzoSA
zn^)H><Zs?dvj4pN{CudbZ|f+X;ERbmW`802N)j6?nJ3{I#Z7CnLi&}t<M#?%UblS?
z1&R7>1y|wXhII;4{t__@$HrlHmyy)bTce)Tm)clttrxgLzi+4hCTpI+zcWwG3()}p
zRfvE55IK^|8D#|Ak?*?~51D3y1=2)arR>d~<u`k-&f%j(VPoRtvDI%3Coo#=j2UkV
zUGYeCuU>n7u1p+xHQ?c@eTApv5nO?6EI$Fm>8drE?hwVS3X#WqSrv;^RpUVMsbYn<
zNj-pn;x{<f3)wmPl{~Zm^Y}=v@MAgfeIp8<74xL<=bA@6N+=6N5HA^RW8dP<qsp#^
z!?C!z7G7)X5lzJ^T+~#lx>j}B<c?n*5Oi=0k0!4yePt^*jAx7?{Ps$Ay3C+)vXcS3
z!XuYw{t0JU%DcUwz0c>}=eRdgr*XefAi!med#e4ho-gCKkzTHr5zE@}JEwwtTlFhD
zubK4w1qe+Kl`zY!w~eQd)yt&Tvx5ahh1{R+Hpa@+<z(l-hO-9>?xyKIG7a@Wqh!_w
z1cE+3*!3B*t=iIdTE;we&vy}DcddDwyzD=<E>lO=poZ40uFCW4uhb_MfGOo82m|^6
z7A6JC`9l+A6zky6X&hGO{Iy2$_6a@}h$~5}WP4_-H|rb~vocP6gYD;ihU*~=c~Omv
zR(8R5ww2E{Rn9J&UUKY&J2TGVwh67XX-im}>3FL%<5VP2Ea*pz3u898Pfwkz)xH$E
zfJJ~ce(ik^CCSw=pJFq6FE%OhkG2eFcOS{>q;iFAwHZb$+=p`I*K_4ndA26g#rmDh
z62OtR%G@d}xbv_AszrXlHXNuYd_V85h<?Zqd04nyJUX@H0=(*Af^Zh4-_9+cpQOh?
zq`Y$Q*NDM-(oC^p?wH&-&ScCP-r37u3v%tNw3UoqS75P>N$YAFRXKwmd`?OnbL-L*
zm$#w*g_b;4kiMf+FZ;<v8$S|~n0dMxjmME86$WS_3TQ%FFOnkm_S9>!o-O4|rN&|#
zQ_qPfZ0ib3!O%HaB&8tb9Q_3xuwY+yu@uZZ%d{^V;8U<z5-wnUlHS}#`hty8{z=;_
zeSHb$@_f1g{+Y8=%}z@1M7q=Vn=ssaoaD`sZ9skE<GnV8ly%8xAOx%)pON*SHy6g8
zx%!PHC<ny9Uj#Blg6XCr7C~jAN!O895E?PTme`>Ab4Ni65YN(NWHx3KImPID&oK-v
zIE5l9#n{lXMP<dNPXv?~qoL#DTI!Zp&lC?Yle(74?o@-sFGMO7GFjg%XkO#>xB6|a
zu_W|MQQA?U2RprUYujEQty_ojVx?*9uirwyYzua(zixk!tM`R*2C6ci;o~|g5)%|f
z0gU7IJ0r@{&Uk<y<~oruUnqn{Nvi*3&n$g)7~8jcb`Oc0G;g-xvdr9tJkQm?6AGFS
zPR8ZqF^R*g-mhKayS4j5i+moITn-T4yT}!;95kR_oX0furam^A_kRVfr>(0d)cObM
z67yWR^@t>k%rf(wC%0vJQ3~InWUXMORZUp_qBwWiWC25ru#I)(#@Y87d4)vJ+ubB{
zeKFzoP0f^A@@Dm?`v%?%2J>F*rdhN%EXt#9vjZf#izA1hz7@`=*<_prS!?GDA(7Ih
z^1v|BU)NH`PJh+sDFOCwUUt1D(|YuE?6H2{tUV{%Ka^(KA2)(lx7kgBHI+XPzqF+w
zUmgT|X$Zd_&7?tO=cu@*@Z!B|U-5EG0U6d3fK+7ydSnO)drd@Rd=z~^uIrhANcheJ
z-N1UMbZ?sDYp*OZ&jyQ6h6RP3b->{>FYLdiP7s-~?-lE2@D#9a-IOy&mpySNndEIi
zg-{VYDzl%!;#$Ns)=b@%iw4i@xTCOkIQi4gF!=1X65F7~cJ)UO$-Ld<l9&^P-WEiC
zDSjiKR`gmCP&~e+ub4c(4@*O{kOLHG;BZUvVH#MaN%7*J3L+Sk9xDKUBIxW-SW5jY
z2-itwip=uC3Pj=7x0n~j7StQYwz}FeLI0wMygWcpIY6o!)cK5HKCVkKT(U`VeXD12
zlNBV`)8cD_98YS;l5HoUfgkN}JDx^bgkg4g6ez%Xb-kN=sRL*Z(A(hIn#;a%z~EzH
zluGv^0Y??f?`#Jj=aFybLGNdL`fr^cI>_5F%fZY=RqlU`g)iZb$0#ev2WNVUP_6I5
z?0+El_ER@t|6dGF$<ZJfAoG==2^w_A#{gDaWbU5@J9U+6z?h0ShU$9iD&DxM!@??$
zj8EEoJ;}Db#b&$;T(gYl_%7<^UKhpJUs<oug|2*ZdJ-1a6PgTFYe!4E;Yk30^4)?b
zHeGc2M%}BcqQqj8Ws@)1caeT*Opt#+HJDPeyG5-j{(n59OFxTsjbu_)3&gdXt|U4O
z9c;8~jxZ3I8k@!%ni`6rqocLPNf&4Hs&(79^<TS>1_1>?$);^SCK@CTFQ#J{q*ebk
zd&R91dGo?Q*-Pmg-3~C63bo?fT(5?r+<}YJ`}}&L0Ihu%aF-WV$ZoZ+B15(Egl7k8
znrbui^4d{qU>;C9q%o}~xW)kV6p&O}wMQOI>XTq&6*>_mSYy){8gB6_Jfe*0=N5}2
z=C@%-M+n3F%pxzjCL$3@RWAXxU5F-^>XFD<#wrPq;~Na78n1>~C;g6p(WLS%%zeu?
z3ax#=y^V>V)Z&MCi6+VjymtA6a$>cTS4YYWfEG$k=6dc?BPn8rzx{?#CRND#*Fzlf
z`_1FokGNRa{8HV`FW-Mt1hPq97?JB4Q4Ihpqz#+P`n$kK5IU4?4~y+Qcs|@&0y0yw
zpK(&xKOKF2!dSE;u#sA9zj3~5U&Eta1)$5Aw|&`kDf@4ht?E3ttLM%;s76*QZq4PG
za?@GK*j2t0g*HgF&7U%<8C@jru7k}nMnId$jHftH=zVz<?igUDWwVf#sZR7?$Pq>U
zem1=~@QvWl;U%=F#K?M|ee1_Sl1s4LGi?D+MGo@?582N&PVXN^+plSnt__ez&!i@Q
zdt!C^*_0}vYZ0>Zj_DWX)N`#E8l~d&4U@<J)XNXqUjg>{wD;=u-xjR<`$ga|kc67f
zl>>-A9NNW0j0Yh4U!BF$56d35tZjrR%MA%M91{ehpmXIYrmF&mE+ar6lwV8)p-aE2
zcDHw>NSQP#^5ErkeuIu;TSxvwNs)vvi{-*rg=6y<ZTvtA@8dn&>^>8=e6t?}N~tR6
z3h6Z2Iejs*tm*u8YJVHEB4PpYaDiw_5kTaSy(xx2TD!t^nV;-s)Wud8`@9P!ITzB0
zw9rJfM(m~myl7<5*95PRgKG6Fai5I;Z@1ay=8A5Fhg4cn#OJwEmV2mVRs_?M@GkES
zu*B~Zv@k#_dT4^~wLj2VNmsl(C+S@wY~yGie_G+SOa;OokD~9dW2})Y6XtZ2?KPEE
zWFtk-<+eY*y=-*bno?`0?d$z?oMXD4_qFF|F{?^YFi&1L{dA*q#YS{)T@F-q-SbBL
z+wq$)q1Tl@UO(6IT${<bwI}4Yb2<c40ycI2Y_ZAJ{b1|yAtLW!nxotY8C)arzlY`C
ztC%|iZ$XZBd~7~%TSaA3P?6BG<<(xr1oVqce!5~JZS2_ZgJPj$DHUkcl@Q2nd+<A!
zYeucic$ckEDC%u78=U&8MawDGzi&_dO>qyGkq#nh$--0XiQjvO{{KA0VT`i`RL`82
z3L0ho>}1DiC~-B@LoQ0#qyTnSrIP7~IoMYEA<CGg9Ttf2(a%;qa!E@C<Yj9eK-1YN
zZ7X0y9W_$=>nvYvmgWp#FnaN_erTF^dNcJV*I<oS(m8p<#**izP21Rfg|w2~N6SQ-
z`VH>iLf<E$I@l!asrXh3fQ_~`Bq3;XIbX8b^uMt)j-WPTW>8QJ(2pJ>@_JAMJYlgr
zmqY6@Ho(ykQ-9HdBmhF}eQ5H!-UpU0p@3V2A;_N1?k@Hq&cZSgj048rc}JzXhX5Al
zdYaK4OCSS4Vt9Da5|*AixHol(k)rz&FaXvI#$vlx5hLUn)x&!D0Ur?L^_XCHT8&d$
z%8LQ>fDjN#0O&3fly}-zPw(-1>H~PM1&lbQmAzqCmC;Jp?Wc8zz@x(foT{nvvul<=
zaKwNAHmQz5kfCt7rxN&er$e5>hTIGzZ9o|*9wnNU2ElQJ0(86d3GB}5Eo>fI3a~f9
zw{?et-7G8W{$n4D&|7i<4d;g4`2Edst>%zkq+u%y3>QCug_B;oGd)7XFey5oaKe-U
zTj<`8Y+W4`gyoNXbYZ@S*AwiIzqqE_A0$cwIC?4scBhsT3Naowm?603)lTpcZ)EQ6
zC>r@=o<soZ62Weq$aj6uVp`f=wuBD@vM*k=fVIT7`yuT&1z1Z0SBc+&%>jJbJ8s$1
z;b!U`QHu?xkj-c)FO7+I^nHFk1Akgd!tVSo5D<<U3}(pZ;LrdtVd$yvH8~IqEm<B2
z00VZT#h6er$cwG2+lImz+8&fJb%=N7P;!K+11a>EDoh>zMXCoE$BK6nAb{CDUBm!N
zPK>kXV#YY&&+2EeJ5@>Bx)3D542fO101lSe<9jSBv<*2>fffh=33em5Ihh(d(T*K+
ztt3o2ny^7GbN`iKBAoyV2reSLjG2H`{QfaeTy9+H?j&$WBPxLhc||05LW&9$@TV3I
zxYMQ(Q{NE&8JHmokA?)G2>*|J*hULfO5Go}&k}J*79wJt@!`+iDTgi!a0Ac3rvAbs
z1<EHOEm{!^M$-C^NFsFGfOjVWN#N(~s=?|1l=l~Y6r}+enLgiPch;x`m9oRZfM;wG
z9Wa;Jf%nRnbs(<)84SKi9OfW}Cv}heA-R*FTi+=M8w|L=9#P!*gb9={D?5E!4oGF`
z-U&n}^IrG_lK?ni*m&ry(C7Jw+*UHApUfav)1nHybGG2+RDutfArydffVm8Q@h^zE
z$SCargP)*)MdMQatLEbV!V!oycgpd9mnYzU{huw*mtaW<!UM18jnxl_&4}HZE+k1h
zz)c6pxFt6bJR-`}ayN0HAi9uD5j;?B37GPcziH9X08-4S0%WPJA9H6Ixwa<Pti}&v
zN+9P0iXCP+i_~)Pfc5$#{PPM%=!4k_SD*mDDpq4Br~wl`3EHL!6hs0JyzI?TkEKBt
zc1CL&{gej%-NitI{T}Q-c;)zz2}c6{@tz;b=bAem_C|SKdmMy+9OJVO$aU8PA^hN_
zcT%?oIz9DWdO9rw6(S)EuS(q_J{!{5CIC}IkPNil!D7)}NWHrj3{xINLb2{aPMj}&
z+ESmIH_#fO*bYS`6dOpwo(EbC^PM4M(AeA=a-e58(qMUaI6Y-bc>o5C`e5bHPudo7
z!Sd_KD!^f+fa3($Cueq9z(xXoEo894WRp*a@PWavsN42^MBp=>Eqyml4O`YSJ`cKY
z5gr_J{rlFb1n{lI5bG~I;cePqoINVJlKq`4Pq=1*xfO%A1JC}Cc?1(FK@8h-CRWdE
z*a8Kw&+U|JqWqneA)>@eU`c)j*GijpS4@NZV80BoTwA(ZtP$wvror5h5x|pFYTMd^
zlp5Fq{J<#ke;f0FjpT0Fk3f+1yraz@M;@?{5FliQY0u#>XHYo}Fkyo(C5DTDI&Z(J
zzPlOR;XwyL0f_3ar%%DtC8b2m;%HHAhj>sU;_s#$0dHxFyfwAE;Gg`(d|*2sN>sd7
z1fW><IiWH=mItHpm;HPWldSJgik31_j;l{qz}|z`QTV=w8Dq491``9^4BPmG6AkB|
z;Q5&R2Kaa17Dv@mwrH>2h6Z~ZOvY0%GeK;Q|Ckocc^g7fCp9J*{-^fTufUo|i5cK7
zO{IgU@cf%}YF03hpZe0jBBuk^@)GAdDT{=Ue|4aMzsenl@;MF|KkX3n$A7cKECW;q
zQ7}*iEf`!6+g=LsmNLL$Fa}WA^if>ER3@EaU*>M&#o(h~0F}u(dA$9H38+j{j6qW{
zP)WgO_2IF=kWu*hpa7M;w`zBv8w`WYv+5&bc`3m9cz}jA9rOU%>iUF6>&4xM0fxWN
z#ISManScqphEZUeMZ(YU-vBNxUCYMqW`@-<rtvXsFfBmAydKgE{YQP@z{c{$(eW|&
z=Qi!fPR?9BvHs3nX9BY@GrkMcnQOe3vwJ~FaKiL!?)uD9Fc7rgVbVJF24cFTznkL|
z*ra%R4&eW1QQsj7e(BDXV9cz-J<f!<U`~PqkzuQ-Wdb~T_J{j4-<?2#Lm>yIP)$ha
zsV_`p^zzDjo4^$A3Nmfn$-Ew%Qfw2S_CfGo9c5a)fyKBn>0&BU0E@KhI;aXg>H+fZ
z1Gpo5*aL&bKsZcQI;w8;@A04~&<q?52Uq_H(=bOwCeZqsr1GVB?D^gEEr3Sa?B43{
z{(C;f7od>=F|7NkgkU^0NKSE2dBFAUACnMaI^|;oBvd1>R{HGUVgeN$Aom=Rg7`53
zXaM%Zja|HWFrJR}!I3+o!49rhdbGv-`JcuV0k6r2`^vZKuH2{9ut!h>jNb~~8LjY6
zFqE`F<2!s@-T#fh^f?$m{vL^H|6`yrxuu6$GL%4L-nxaH!p0AFO)&XFarfIa%n}fR
z>!T+Oi)>!N#xD`PNtFb~6TBy)ng$z!Jh<KljhOAuaDsPefd!=(YW;yK1sG8WzgC57
z93ZS_5hGY+E<-^HrW5a3$ML_f)zbxE3tG+$&v1bKVCMMEh&ATSn~`BlTG#^O1JB+^
z@K65F)EI%Ol+Xy07I$EnlN6_ucEIB|m3G*#*aa61%lR03SDp#YD*Rg-&nh2YJe~>%
zLaINdpIv<cmK{3g-kpLe105iLGXF~owq{{MpakFeQQhUaAL{Gc(4_&xaXgse)tCBp
zS8wVJGXmDay=Sm%?|)(^z+4bCdXhh3*^I3DpZ4uN5DuiZl@vDDC{KN#BENLDSOA_A
za2#cqz6U{V@B;<^CLW^wlLc%^-u;5zfHldw#i4ir{tyL&IL0HaxYvFJDIX+vy&D&>
zQHq@akET?boTi%ZEzF<$l4c+bbfV~v6BsF2ctW;)H(neWa0`MnQ6X4(QTYhJ2zNed
z>f<ZI>wudM;DhpDf@tvpt~rHi8!57tKiM)RY#awKfNQn8?)=F<#LzoS&m<GfDvmtx
zcfAo-O~(DfW+oQpd*lrMt){#4cTKLl{V;&ilrr80>oa%3dfOG4BK1_5YhK7{3~ht$
zX&34hur0~`KE|(*UaWWmd$tUCpH-Fd$bV%u|1Z=7;JATp?vBa4Gkk7|2qpmm7Z$jD
zPyDZhahT&MU^v0%0b6U4+^>K#mLlIr*X}36T3}YNePMBcss$~EbKxn<Hq0e)f`v)n
z2NF)v$$|Kn4f|uj#O&O|Fau<sgR0;|B*3l<u<N8d?E3$H+W(KFmh+RscCQ0xy4=H8
z--xMkUK11vSi3|ZX9wNKtS3U`mJGl&blz1cOdw>!z(jpdpdv+zuo&>aATb(mKY&xb
zy#gPZy`3&C|Lk?E2J-n9{~ymngQ#v}PtXmc0%#`-;=9fS3-FGjG+Ik>fK}lcJH&T{
z_IlgfZbsVQuCX)dl;!61M=>lksPZ3P?|nboEL)^Le;C>887XWQYmWGp%sch(J1=#v
ze!G%(_v)SvqvJ7=KtbfV{DphSc1!wWIIIXp9)zX49YrN?rn#oV^AFUy@JdIL&j3pw
z)$93Oou}5GXXW$7BdxVoP)lut^QGt`A}|;-H;OHMi`mAgD$$gMm~sQxWO6rhG_U>R
zl-8qPnJD!Trv}NcOIhLlBqgaNbNtU)0MmI8aGo^PLSZhE_%dtxHHUf?FRZ=-9p7`|
z&b7Oth6Q64!BHl8tz`BcsQ92<3-uYA1yW&ySvOW4!~EiBDtE89yZb_(;X<lEJk&Kx
zpBuuf5?#l$YBPe~#GkEZxqQzDNEuX%5-QC`rF1+`RaSeE|NI7>W0dQ_InyfZpnt70
ztXnC!`54Mu)UFzhCXc#DqyoyI4(eGAw>a^x73haHR-n>%;YUHL4UVU0gn))a1@9{f
z*1;j%f-*FOi^hPC4Z87Si`Ug~Q8vp&KN|r&d(fmc;t-KGVg&O!9Lvvm@KGE?6pImU
z+EKnUOU`#SF+9I7OfA~lqUYE4LpO7NqmSndajIl}zkD;fAWW><Qw9}vuM~7oyUzI#
zJ1G<)J#ue9rTv8u-2;%^SI&KFhyrHynvVu`%k?^>k9L2R^^Pi*fDkEFV#Qn2XlH-o
zdot(^If*UvzZO!t6EA!PN0Q48aSA<M694k4Jp3(adHVr$8<fg}7eVUMv-(1j1<oq(
z{jJ>64LW>MX#a3zZ%~q#4+Fg*Ck|4Yk-_0QubBBROSCW5B6jABt(~_HN;vAa>&*p?
zJ1l!*j;*BubGe63!(o1Pig%`^a=g91rYE+Ut&gg+ug5>|m2CMMLs&UlZyP#ejt64s
ztKKoQ_ea{$a~uAvK1C<yLH|hEkId7gn{NifE!4(uyV<$d(qC<^cV3ape2d=_IMPe9
zS-1%#xo%ykb%iGF-0FW#%5{g%Jn2bgYd(@`-d;d%#1D+msh?nruka>OWg{+MuTw)4
z*47xw8T@@Jkn$*x)b^&u`f|VImHl8lBDHpI(NX$DUXb$*z4W<N{G;i;n48XYRV&5P
zkzd$vVUfNTD&*oBw8<j$y^zBT>-h!-LgwctA{KiMl!$}LLGfaXZc2T=>0VI;3t`b8
zh5Za239FWd1{Ca%hD0v6+cZFRY9r4B{4dvJY%8M=tQ<wjIHmQ>BhQWccMi!22s{e>
z98=0iR$Zt`?=~(^kUVRa2CkGBxl5rqAI8~tCj8z_PtD#=QKbULW$7nrSk!0`G=G}g
z3Q)@xSrf18HO&+Wn!?M0gYz3f6%@75LV@z(?T8a-NGE<S;lHWT8S<o26#|+|zOAuN
z@mb=oi2PTbc9AtX2O4<9fEGBepyaCuv_t3xEztH)%b#)@%Ydc|;CbuRyAO2f5k&_N
zbs@5oYG`yWtBm2ZvW<!_F^;vN`{YkAZwt%!ySZ66-{tjDlr&6g=$K7Ruk$${K#Uzj
z)s=?hlPcpUMM+j-Oy0)R9zp$j<Sw@B^eV1)b4*ONZoT*+EBQ?p7m3@qK}H9?4?k|g
z3tx=Z==OfG?k8!>K8DIQDobxwlG9@g%#RK(vV2A*C8Ez9z9f;T?}F0GePdh@R&O^$
zeq?&ka8qvfor{=5n>FS3<<fy)=~FkkIH9_+zouUr4!W+dD7|Rm!JhBK5+I1+ciUjY
zG+lOP5nt2y+aF|He!d$g{%Ns$^(wCZ2%$<#k({5N4s&mKfh#ALF0J_SnESEuAs$MV
zZOuFPhz*QwW^a<I)i<aOhy@D`^~x@Vk_|RlLab<qzA`Dan^-aav%EiBcZo^kizfXV
z)A(xD=;!#OsJD<X&N^BLd3QHzXQ2v$Uh+VgLU3CUQU<Ii3`{J&8*fYL=MvqGU)aSy
z^%tHx^+`Z@)1w92+@!#|(`w;24}vC3vZ}==w<F&{!}@YN!?{JzW_0%22L_Z9FKxbk
z=!r3x`&J`#n^ve<RiGzOUZdRLV%Pd*SuYoq?tu;;uNoF`M~!Q2B2TA|lGuGeS3bmj
z@HlJK+adz70;&4lf-V8K??!z8wz90Gu7`>P-_Irbs+2Q9LY@luwy}#y(R^^?15_@C
z_7!cXP7!Nj)nf0#ivMmn!hYijTl1dRw=f%BT^`o7Z%}+W)<y#8+CkJyQly3_#4*cN
z)lja(lF+^?&g+^t6}h&yt7_?m4(9~)jILyEbJ19!x;feF_Y~FyQo{g+dFv{hWWm^x
z=_eEFcM)^;XBYduln6p$TaU2X)2igq8m5;YA+QPN%!?Mi4)twWEuBrlV1@tco(fpK
z{`tWICcSCzLd-prg(T&sD;*=2>SSYSQy=L<Iy#9*8VBaEV0jg>4s^&tDe=P2lOX$u
z^;mL-t1m>Qw?h(Dqd|sGuIbX<HD<i7oL;-@$rlQguP0x=EqZ}r#(fr=v5?Fgnl45k
zSik=6W8aQADG9u7E=k>5w88q9?TcYgK@xAvq?2Uo!KMl7Te**i^3-m#08#b+EWON%
z9nxAKhva^se&s;dWOg2;YPn$QG8*v&v03XNvEjyA+9*L+1oE*h^>3uy&l2FuBiFwL
z{bl0%;@R9At+vL#t%L69I>#6892OHru-2F*@n7W=-xljM7U(p&_y6whA>p@&bs}%j
ziGGhq32U|uI_#V#4W@qveehyHH+h#v>O|V4jQ1W$OzS<Lr9tt~`l+)iU9V*QLs@{z
zXA-1X^xxoL7${?1+RNL}7&s-JeHSW%dQE^+A5z!0m5{$VsPD9S7O><RIDfkeIQs4O
zyJtkM_J`B)$D;=UL#MXG<s(iT=W^I%`+shq4s<dst@(EfVw<C~Rl9kYDR6w~U(Uzv
z=w(^_^?AnXs&4G*vqkCgH04o$_U*0qZ-vyJ8<txF^2ekn&WKiMTbMmw=bjw4(@ib}
zK5~ziF=LtERi$mX^0p7>_V;b{PEC}(U^ZrXiuwbSeh!5JCR7P@$m?jq=J&5zQv_Je
zIE~{V$coWzgH1V)jZ#m}#&yj`cWCJ$Z#MbQt}ewyzSGW8F0^!YruSvjq~kR-mRg|2
z+9KDi_+se3wkal~SAM(j=WpWNv5Yu!*S1>8Pu_Hl*uKp{8UH;)l5RZr_Fv@N9mNh5
z>+fpO19T2bzP6Hi<TlMkgJU!@AIH;aE>w0`@i~XKr3&I$>?E^K`D;6%LDD1`=3F@r
z*HK_bHIOGVaS{G|dQV{huS<=7NWOy~#BKKIa;%_)MXEm@H`YQhU3J#IlF#mClCo-Z
zGSL|cUHo9G>I=QC%0FNCS5jkM|Eje`gS?`{jH`Poq71JZd6}$iDH4hFl;mP}vwvi}
zYGg}3`7reB&*PHXwV5WjvLy2`|M^Ypv4*+6+LTEMC?$0-8gob*y7>OG@})KfPuG`U
z#RcPzL&^+l*_k?$ULO--abQ-UVC`&ig23Cx=xft6w|HK}rqZATa2VvN+=c(|SXKws
z9@qVsN43>knnz#0J+fIQiNdpCq&Qofd~an@^9g+~lvh0=&Cl$|Ax+2QsndC0#^=UT
zEU&2_+vpk`f&%q-3d=Xz#4-aU!qjS&4>C}wL&Uz&d0$JqhUgCnAPwo!5Z(qkTYf>;
z+-dP<(;k}3cd+OS6D>jw80?zu@@CV0>>{)UC$;Pq2K5)U!bV`}hqh$D{XNY%GuwHw
zc$E2F&+$blXAq&oxv;7P;;UWVzjYbiwe{KGnI|$=s0u`}XEXke%=c=w>V71|#yNOw
zhkr-{PIOfYtTY=t6jlm;c>-K&F5ZpzehHYS$8XlN$OI*+_p<{lH-|Z;WS{5tT+T{u
z-l%txF8VsqManT+_42)+rzG08$xb;IuxV#ni|@DK_Ft+*Gk33XCY?#_2&$Pg4ixm*
zjM!fUuW{#IT?q@(k7in=Vx(Z1>&Bx>d|HW@owm@M)4CU~=rWK9BPal-&dhu|AB@CW
zRA%$9x;OCuUES+RyHrBzv)I_*x2Aj^sbotu`=i)WhexTPadCVdQta?G_w-6kb&w=a
z=-Aq#Ss&Z5$!<|;y-uJE-H30~X%REemiNcHMY+=@eCvKVT;JIbHJSxzm4m1X`{DB{
z=am8P_Y%+F2Ta-R<15nfa%EP$Nk@I#Wc{w)o#vx$kcgNrAKh5Z@G8TXsjC5>o+J71
zQUiH)q*B0-@(F&jM`2sZoUB#X(1vjWU9Px!5l(1)kk<BKHEn5crgmTA<@p;=#pY}7
zz3Xh=G*yS*`A9?W*<QA_isF5g4wg;Fn(C`pv#75g`5QbVU-1YJ`&PV>E2x`i^OCxK
zi#@pJ+_^;Qq#e%lnGWU_`f|T;t5NNrC9|0GXu3J2p5lFQ-T|b*lpQNA3QKAr_D#P9
zgZ?o8DikqPHvXxF_R90$o0K~(*BC1~4sP;zDmdM6hlu`le4V!}i>L_VIanfT$$C+5
zVuyE|s~l*5{(`eYWZz7|GeEB-s{eVw@+iOrWk{X$G?3Te!~5jP*!(pQVFQ&VHxo)e
z!qK&qzULL;wM2^pgjBpX?;&+0c6iSf(BUlDo#us?!wGV!>%Gq@-aC)IA1c!<s`(&Q
z`*N=BQz|jB7i8%+l8|I@+;g%pYUjJ6@gPu*{<tRv+w^mRAw~P=r;~c!sUY92-vm1l
zqEQ7Liug(0a1qT=6clTF;Mej6(&6bw09+^Vt?BWF1@g|-E%7d7=2gL48piP$<)ru&
z_bUah94t&GrMQv;uk;`3-6CGcj;>#$>|eBaWtuR)x|Utk<7|DbxU8uA_#yi1CCIlD
zSpwP$;CVdKb)PyY=NX*OGnm(Rn!7OmpdnfN@?nxtv{dJ1wynujs=lCU7iO^fvh`vv
zH%DnJIvVRrALT%bC~D!n*n<`V_H<f`)@=-?SOanRVC4k2*}vPMz0Z}};FScX`GSc|
zJ+~biI3t1qd7o3b)g5`1IUILm=={5y;`3cx)(*tphKz<HY{JJ2n%MhpzpjU0{+a<%
zHtz28Niyr&*FPz5mUrReCb~SJ6RS-pM1-UHBB-RvoVD2=oUREQ7-zvpIWHb4B9H~f
z6XFWRU3t~Dzswj0r_Q`?#Y6A2^2b@BBN=KSIek#Tt$Ff=h{df|`Jv6jtH#SBv@o<X
zG*9$5$f8*%+qUA*1M8p8FnSakhpF0xiA>usWDB!C;hLJhUNwI@QXAuT>^BY?fz~Fr
z2NFR;)Ll^)wwlC308xDBhaI#)^8mh$3BKaC!TOsa|0&Q*I}JwRAv1d<g@f&(tl`EF
z!@twEh&&)eB`AJwrkjzB#ngV|XYWfpS6u^Y`M+MfZwWASM>6!7qUTnw3i_&K)SDG=
z?7iv}R2$G;#$2?JFOh0HM__+`IPV`%Atd&9%x@%17wP+r%F+V)zz0xlZUXgS2t}%A
z`|={WTNg&2@Z64+0p}7e0KoOm(!`cEgK&AI#%3-I$3$RvwkDqP<$JYIM-lR$&Y&T0
z4(LAivBq?^`uihdo?i#=GmziV_VU0gm}8=;q={0|F78g|Fdxne4Ex`Pl^2)Ls)q;e
z&YFXS>5P61@u}R>=-CLf&}{R-*q4X{?bY2cFw}RRWj<zI#w<H0B9Qs&wMeXhex^Ez
zv#){|Y<(`Gsjl#syEjzbdkCwcvjLE7C0Kt%CYC*XrdsPjX4_?bHA>Tx!xjnG5Ks&~
zFWJb9>!i6hW_i9orRp@!&YT)fQ@Q%;wyW0P$NxdRdeT!Hwj_&?=DlbHtO_q1k?gpi
zUb)VOSj$)o7TE1e7<`te`>gNKPF2>**BZ1$bpA?vjB`7YjG{-+Az~nKb}BPbGhg|3
zil%_F%Volb2ROgpMb-zwRlbYuj^)+%EoUom7?>0YeQ}TzApJFMqCYu5fLz`RBr=hQ
zG2Xq$k$>WCdn22$H&bO!Ni*d8GISXI%Y$Hdv84c%Pv0uAYku@!wysb7unqF(-*=aR
z>W@8aS?PAVCLn+NE|h7^hSkQ2eAoVa)9OCC1yrCSU6eJECcV=AopwZQ?vV5kUWzND
z&-fQ7AJ2kF2Fn~?2w#tR%DMi|6h5sFw||dpGa??&n9mDQKY1AH-2o%J8lPBG7K)H(
z2XyLNctH}o(4#N9OH970R8nLG2?Dd!uH7Bbo86j*ojH_#4tcbc?7VnIjk{P`UlVrk
z18JG<bcf`%2VtodCarWjf&QyGphw$Va4<?#n_o9Mf<RGjh<zwNOF{4VH|@;?hqtP@
zOnsm|=|4m!8S#H2GK*T5vECr6Etu<1<}I`z*7>4UUv0AUyxBTMY(u3`l>rWaePP6s
z>^~*)sPF$=n}K61D$rCEy>}mMj_cx3kwq{cem&8o|JHDc{xmIZp}x-J`cnPW9lu(%
zsSkxb@J|)$S6fs_zH|^8AjGHIdidrgmYAKjob~!5Hup<;dCwh`<14Ml^=bZ2FX$JL
zO?7_w=olQi*MU2Z+&=|}!!-Bj&~<5;+i4MlfRZUH@4-7;_COTfmVl-?P_rlMC9EZ4
zcKs38>7$ps3`He>-u{MnZtl>tE+PaZ;bki+`5wz&svgtVEh7V~GaVsXC`chS@R^Dz
zVrYs9d+Igq6x`e+b@HGvxqd@D_(>2@u_nS!BA@&q7f<#6CuQjK@6spx7;GBAiKW`Q
z49p2$Azz2I!nM|Pawee|y8TfgjLsMGkNcaoa?Y<?DxVx9EUono-)t_<fwe6fR0*$I
zmx9>;R{q2l%D^r8!=*E6#QxWd%~^qJDAaJbc4+P60J$BSJKM~L19J;vdwuW9AvBiv
zPd)~naEG2ec{k)i=+Cfu51xgYKZ6k^g}o8`2XUJAWNBp=3pA=ig4swwW5V}7Cv=;Z
z($PYv%AuqWQ*=FOl---mB8zx;_n@-N1F6B>XMm2W<xg#)j|dV$xXlDNRS$1RSF)RR
zg+g&>7DiP`S+31Jn~9fiw6K~}vExb7wt8anhK%Pe&k~$gb;S4r1#c9)_Sa!RQwdx3
zoi9V)l){?sY?(j9b(<GT*6fO10~HRZB7rlI`i0NOMM#4Nky6gOP5^GNL129%RI@L{
zjJ%NiZ~ESi;8I#Q!9JT+FPYFsT8Kpf{9e4<w_+YFFU5xQGqu`LmFo)x2Kjl5>p5sa
z!=b45TXaFa&BZ=~F|_9{p@&yNFO_$m$Nu^Imme!dTSRx0!)lUW_jxtDFu|#~Hc}%r
z>j=9;Sxw&6NpZl5sChRl$!*6&#{}*W*&sgcdE?1E-6)gz0ZTtsVVf4q$-9Wb*F}tl
z-6AdItT%ASbbqW`xi;Zmox8jb`IfM&>++g3sF2?aWm}eD4;vZF4rhM!AXmjGkH=^t
zIpa652<IF()7|8`xssfWgM3st)3c2y)f90^4Y)9r#mr`LG5mCnr19{JW@)=H)*1B4
z-qKh8%j_w=GVBl>Z>+)>t$18zpFy=;+4hc(2!=D{+4ACl7QGDlEY%-{diOs1xjMYE
zt|pM`v55M>QX~daJuBz%2E(4}f8MCzWGB9s;_fIS`}OF>`ybBftfzgbNSRsBXHi1O
z^gA_*gsvBL<Hy1nN!o3)L^OjlF7P*BdJ3ramb7i_i6b-bVE3-=NDP}@nYRc24eJ#J
z;NhP$a~vnM3g72UKl%Sqa@$%i@m?b=$Araf+oW8qE)i>Q@0Y=0q^Svf9A0@`OJF#6
z`HG9{@_wy)au=5{lT0fpR3$g;|5y}Xrxep04jixO>RA>R{q`q#Zywij*}6q*T_udC
z1A*y8{>k|Y*W{&EFDDA!kAW+lBZA2AzCcgUTOoYM-_TAT68jv6`J`{WB<hDq&z_3-
z$1d$6M{+!pr>h#*F56E7H8MZCbLwp&mVXD$1=xFAW{FW$_ZS|xX`d%`t_PoyxS+Re
z>0o(!56wRLu-|2~{>lEy*Y>q1plqM6<7XG5;dRcfH2JGeBo@+5T-?2rC#glx=J?q{
z4i|<7zgkvJF;qk#xlRxHFK0uSmeNwLo3gvgGP*|TF9BQAmbKw>>9ByVW68a8&Ed$g
zzXow3Y{6jb&t$;LD5&ZVaYL|IBHHmCLLaCCmJ0X+#OMFbBLK+=B5DnECXmUEx@de5
z6oPN$fZDdRwC-5ZIvP8QV318E#b<0t=~eGAvf8*xXq?!}qm;_(-wU7j@J6+sY5(Ed
z3M017zA+wr(vE%8fE1#Oezpk2r!~du^eD>#0k-c@1fS@Rn9M9m3N$Re#-BOtZWRWn
zHgQzJTY5}hxs&LJyEUq6f*8MeWyT-NAcRW0{-}fk>Ab6x^Xq(vQAG6;$q77P!BQYy
z3bU;a78GF?9o>UM6*5dvVAibF>=^K%t!e;uy@ruMa1>LhkaxdWv7O9+G|K%q>!t;C
z9u#x3m38#k+`F#xH?kac7CUc8sU}EO#5bX>#m%za+%Cxx7lw&dhzJM9dhPL?W5fY^
z=n$*E%~7GThzsTZv(IsL!R`a^lCjk*ZnpcsOpabjj*fB65jrlVq4Un`<u3Q@;1vD*
zgb=O3vX+VQ9)NQM!HU682<4YCjWXq_A^TiTudmr2gJV@NKE~Q#d(>&O_2wxxEVm`I
z7CoB1S#|w5NkPx3UWY6h&b5r`opabtgj)6h2X81oKvd4g@X~JgcpU48?03~P@o$5-
zcIq?2e|>(Cu2_jFbIJjsUI{PdMv$+#Y_B&QvEdiApsA}rpqc9d(%%!HRVKqY3qT3H
z6<VxxHz82~VW%ue)zcY-djbyU1VxXgx-Xy#F`f9)2jTNoQ!FDoO}uwqkD(=;A0k0n
z35^x@RwvNlpyXjC#{c5u@^O(VxDgWvV^jyHNj9=M+}*p`Hbj1FjxgEaLrG-VH&%5=
zoaG_`C0)FI`BL4gPBdiRf_zoJ&c@Q$BwHZ~2f^3EdeaLTA8YtC`7`u<Wqy&p^ov5x
z#I^f*c#ZJQezp2YDo@{B>o3+qpfDG5qyt?<0S`loKzM)8;(6h={bnDVFoE@(vdUA7
zFcJ3&R+{GhnJh)4foQFDQO`A>qZNxeNY*e-Zp52f&OnwOf_q=lAM&^%C+bFffN+#A
zsPQ_6C`!kCL`TO$w58KfiJ;JrWllU+`n9b~|M>OuTV8BaJk?1@Z_aX$UX(=c3>HOq
z>R_n@A2j2e;@BDGwZSTe9+_~rNHv2@hgP?YL9ivOQ`VS#k0~IO&+Dj^JAaI}jJc3}
zVHhOJp}qqH!S*`4>Pt%aPFxZ<zm8UplU+W?s5gra+p4{;ATnzsHNbj?METZ(EP5-a
z)ti(_Buh0lgpDWDVG%N^8rQkiBh^!6o41Hg76p<ZlR~nhYK0uDeE6PEo6dsMXe`q~
zmakN0pxX~n*H9P(ra{~&IA`iMx~J%O@27!!ml`D5i8eHlEmzMZFxd84<hW=IE|qd?
zvq>UYaM3ld+RG-l_h<t-#UFV^N$g$gNsp7LUg=x4Hcrl*IeePO*e`e1ydb}+{=_!H
z74ZB<wHBnR{AzcD*}@mqq|E-t?JJ{@$WM>sb-ZvXZJgi6YbHn^+s9d?YrtA=a9E<7
z?1ZOOZ#vQ;Sa&s5o?JzC?yDqYC!5rd?))g+eR*<3?#Ur{EHzi+y6-E9i+|p3S;DhQ
z(5+_F9J!6o+QqAa-z+=qz1J)YWep}kkN|{V?6NjenLlKfm#>7Vv`xG=IfWW%T4v#k
zz6j`F8%4HKE4R)2;gI*NC|#MQdDbDeUP5KDiQUE+Gg&f9wOpSqheogo&pDNy@c3B~
zVQ}4kSx&2g%$M7nc1IS3Eoi2|--*x;2fOI;pHpza)mMlCfs-w(AXvZ0T>ZQPSn<8#
zGkgSpoV51Uihgo=?CM8044flL&vz0%rBbMEC;nzE$%(2;{Be?`Vbeq5YOC8;VgF#-
zKC+L;qHs9E3i{)Uibua&mkd<*5+E?n{W)*VTYyaQXsRQ0zf1(gh0LPJME6lW-j^&F
zlfT)>ZA}2ayhhAoGvhs7srr#STx*@w89rC;k1Q6tj|P(ZZYqNvj7_UVV$VDt0uytI
z8Vr2fR3iPn<GT0Ra;DGh*O#NX13|n|!^145!fF=-klDp|SqfxoE!@T0<+Z+Io<;x-
zMNM_-@$fZ-uDfwqa<4GzHkbWGlvD!gKRtbML7Mgw@A@ElVKG46=fWzuxf`1Y9GbSg
z&yui+1fy)=u*|U}b4hGG+ROhjK(CjxkY{_|A7EC>@V4M(y4OYZO@;z{XNHf8NUC9*
zctVR@T9C7KOn}gyBV4Pgk#eKurqZU_$L9%|BiBhMon!|{2%&aMRucJm25?LL0dKCK
zkC)DeVik--{QuY?-?H?r<G7%TE!h)1Iz;~J9&S)DcOg{s%YInxl|nM*XGQ{VzA)Td
z!nQyyrR3&BIzz2-mHNFhRJ+&dapB~nc#zvw+<b1rw$wy3d$;+1k2Ckxr@roj2Hb6)
z3zMS7x)FcAuNe8X2k%F?-jt2zG5POCsUWX@C%UWfQU6BBgirScJ054z?k3tvN&VvC
z`m@XHRLAD{MA*T7iS{pCF)+?vkDTr=fZaA#jjO@zuhaJV$bP>`qyNwVQD>jR!82nN
zA{d8W+`pArQV0-Ia25kC0oh0_0ahXQOtZG$X*>61WFwS(x?USfUy|c0OBpQcHAFYQ
zz?>rLs@$aYTq90_gn4kxy6C;eFx^0*JP)amLa&NV#t$@yD?0P6G_sCcUpl#b{_Vrl
z)ho8#-VVj^vH#KDcLznaE&U225(OkFC?Fz81_4nrN-zK;8Oef@qvY&}NDfMrEXW`k
z$;^->Dp45886?9Hhdl7s;5qkr&pqe9SM}<v`u;f7E^5#0wR@%RUfsQZy~aFuLUngA
zR!y9VO8*wdU;dD{@xAXnbl*tHD*;mTR=$*4E_x@CRqOpoG*aH&zGv1#bqD3bP7};%
zNP)-lo%mCeirE8km7CnJaYF1GO_5FaotW#6O-QZS+Stf6T|O6$kx>OD)dvZmS$pt(
zG$bYaD2C|eqHZFO;8a|9cF{Vw67<k+P+w5wCb>n@_Gxw9TbJZL>#itfe_`i_ol+_E
z6CK$}w}52t5|L+2C~7>S)T9_D>%t$y<EBGC%-aa+jV&aiJdYc^ovWvk7xJp&>>6uZ
z%$&ZH&B0>Rjr@)!3uUtO3c}Cis0rmsTPbbz+mXjtG)7t&zdyp!nJisDT6?z`%3$L!
zp?!GYXXfP!gDr*a2KiOoPuCbY_4Ktv^?qpGzSt$M5hSNWw}Vle7`tqsE$)!0u!M&0
zk8SN)_!CtGSM7FAcEoPd-yvwnEJ<df4ovzfUQ#EaUmd;P^|m{ZE3IFSV}$t2lJ~c@
zWCFnao;#-%(y;svZ$%$Caahl-Vs%{+AxeSrPrxSD+X)!E^<gt%;a&~#03oOq`CEYA
z@w8YWxiJa<>K0x&gX@Re1uuw{bB320S)Rq0a1dOZng(ZLXzSsI7qx3kDO82FB-*w+
z_<uPioyQ70-R${cd}xMJ|4Cd;5Vb{wTEs)^eQrEF0yBc_h2?jCVsS)46H)s@7mJ?u
zkGw3-5qWR*;kgjq1Ee(e#16(+>F-z7-grsocayn!8ZE5E7}F5Fe2Rbd%IpOhvH(Uy
zdmpILTMlcHJ;hBqnYBkrltcUZ8sj(Amu?4<%2b@zBzQtD_2!WhM<}O;pzOF2`bus*
zl@+jqR`#A`gqW+#2igzFzbx;wR%EGh+B8|lSvig!U%BoiO7eX9>Nfp8wO4@YYO*|0
z_P{y!cKpWLrCV+p!5?1*`(7EDlrVvPofIi1%;8mBI7*&=?pk887`GAios5o~2e;;K
z>F!&t48n2QicEv=HN3kFSD$oKH>z&M)KTSdEsfg-WCYMd=kRg&BefOIi9h*7snFfD
z85G6pC5OhmNDv*_K)=$8E%3P|Gfp}TW=_;9dh-(#-Lk|>qnlX3+^!^6V3!rne*>82
zZ}-<uAeo-nOoss1$#Z5*TzPRfRcd<0?D=+dA>;{s$A^3FtM@a?br9{Vz&1$@G2e?n
z?>jN$i#M64(x4Ils9%xQ(e>d??xbn;iUuL_iRP009F0Keh4*Z(%%2<1j@hFY8b4QC
zbf0zYzRi|9a2jXQjr+yNH*=T7NUnGq>L1p|8u^A9%_RA1UxFitvSDUthxiu#BQ3_N
zHmf^qj34}Xb&gb5qoL{fKtbtBfySrHAM)@W?GEtG`7LUqKI#rDw~sMuwxz!QqzVlc
zL-A!08&OsblIygKOMkhDEDEDBwoSL7?KJH+;t^aBR(7@|VRH4mQ*rn9n#SHf5^C|-
zZf-z|d~IWvm&K{1Nr8bEBUS`wDY>X`jNMJY6;|P^Wq2C;@JbIn)7CUBdZ9W#=UTJ8
zZ3y+-q)qJBa_{Nn4_YMkIvXzWiRB)9&usvJB8XDsaS!Dyb+LoBs8@42%0nNyiTHhF
zH2wL4#SDoH280nIe4g8*tEYKy6XbS|yAVLpBIlI5{2aUk)*SpjuOmuj&B+rWkPD2@
zM~?%(+aUXA279~wr?HMunERe8KB=>h89y%;*E!HI9FMv8`|&SWXy^-Dy!J!<5<k1l
zX=zVESHu%IgLmUNW@36#Rnm4V(A<N(zG$X1M@Hf>t{+($DSs{WW(GBbKH)Wx)Uw^C
zG-K^IU{;g+fT(~!fQTT3u;9woCv5MZqc;z3=DmTE>OVrP=QU@~32hQWKC9NQ#Om%b
zDC5R&iZjX77xPup`0x%8a-M(hQz+x=$v^i}59Tcy;JnuUG9X}XI=_m{-L-weElJXz
zw2=$%@)=r#<TG<ir$@J4;&qv3CPn?Xll$?k8u4H4Wb=06$jlcH6^AeNJZn$WFs|RS
z_o1vwH{*rfocq3icKl%X{3~nMohv@@_Q%&^X<Yktj&@O$?bE=FG>l;k@c_qP;>(Wt
z0`P7;)}{!D9U19VgT%{;B(koPQ4+<XF7m){7BM*w9_t4qg{xg0DtA7yWgLuzzV4`3
z8*(Ae)MQSr%}Jvhi@<`m?<fJ}=jDwDdyh|%I!i+NMZq%nof4b$flj)23Y)`o>3YQp
zlvUzq4$mf`^_0mStMdE;DvdFH#j}c$MZIH=S>#3@9gJhLOlzNw$vLehv?`(=GGGzD
zTHz-Nz<H_v1+#Y^Ot1LQWI9=?abppq9lE9;A8%6A65215uX3x$J91o?gbg#3z1`%<
zT|9LRAn}D<Cpay-yV#uP|2r5oIHE?rs&vqIsEy1(!GW+ae9;)^Rm;QZ*g1V$j*RT@
zfe|#H;sK%(doG6ZNv`tWzzctEs@Ayi%n)&Acw4QH-g)av^{*fE*5cW9lu~FE0}5e4
z+zdTV2ulAqyy7ne2h&#v0UQ}hfW_mLX$zSDdX_YR0XjA38th_o9vCfS0G`~C>}P7!
z_Pj?K_N+TxZ1!SapTd7TL=7xr1HLDq$|Ujq<UF7_0jU1uFpx-Ws^=>-$b={q!KzXX
z)AoJ8aFAEG_Q~Wv7@6&#8KS1)HKv>Zc*%Tna=JywhyS-}{h#Za{BIC*XJ(L)pj7>}
z42u+|!Vx`mO}C>}4!^_+j;1%n!NQBVU%XKV9mZbJk05=4#{wQtian8l4(G_V__xEH
zK~Mj<47mXmJhzuImlOyE6&4RFo74gO4XFMTe2&#jgrJmy)bus%i{vEW!%OKmJHOTR
zvp($qgYyLOcBTe0Ru_K<uK$a|?>Dcoi79IWrh$Jn;n(JX`v@kGC_u<;gn!bFKZyVF
zkw6lZm^AG?kIhi{%Zo`GH1WJlBgx;^LhcPp)ORIb`*T-+Z)k+nOKbu{K)}Vnt^%7~
za0p8HrQCjal8Nz0`~DagC>{XUKP0CR05kU2m((sOK`iMI^LvWTf3}zqn<~V_Z1M-$
zKMe|WjS|e8_bk_&zZs4fc;`WW8jrM0`=7@8myeTdpu}POW9GkUC^&`x9!Q~j84w5k
z+gj=&pv3OGVwt}hi<3s*ivZJ)N5OykZ)^GgXAtOdsSEG7z9@&&b^f{>>@06>;McVm
z<nGhTWJU4t$ZBTG-uYPSI!et>lqDPdmZ+3j`MgT&11k0hJ3W3x)CDZj?ZU@<eF7${
z0v@XlogSTu4zn0>%jy+@a?{y8aYXgb;anxTnSs<qzL$e3i`0u#IJkHO<V^p%;Zu>C
zuwi*~`Pc7$dsxEDWY=F)DRBOe(!alUft|@t`0bZ-|3>QP!y#Jo3d`}4i-f;a^4n`6
zROA)(_n)5q_ZFNyTp=bHDYZp~`~^WdB7%`nmZH;tYB+cex_t%@H6-jJ_ZI}Of=6=a
z2cduNuha{v16*8}l?2hNe?jmKcyu*z`M37|+JjUt94KPDBk9e560Exi9=(xq{?B1z
z`d0IzPVZp06XYBIxi-TOFX~3`F^m3M$#13pnXX?Z;GgOGm2>`)uHWQEfYYu+sQ>A~
zaxtW==K<$4kCjr*vW}a>?aOCwi{jws5deKszs&OQ`owSpH&*xg{j#sD2~ih4cN-Wx
z&)xnJ@zr$p9xmQ&2$09bEDqWSCsIhGTZh~~BOLWp?Zgp6{lx8d=F`JXTt=NVx4~IX
zq8{zW3<8wud>M0|YoE^+?`_VRw(%gAqh>jaQG^5|P6bNQ0Vh&VLZvm}vqkpm(@$m8
z{v7JJCwd5Ib)J*homiGU-h()I%hEneu&vqSD#)9REuV?eF_GN+3MtBzKxYkl?TiY0
zn(ENH12yA!#C_^#`&a=Ld7tmanP_4pLhmt9x8Jx=2p2Y-Ym?ahOcq1!{`m~;Q(m9F
zwmHWcn~KTTg9YB`9ILO_V!3oiI~M9IcFlpVPH>(IOpy|X1tuu=Qh*$fwtV<vwpHM@
zm@i7AD<xU2&Uf|_-DcknX2j|bxwZ#!=EqfIR%~rYOs)aGY`$8^cRasS;w`pHet?~s
zIxA|exY~v@D3c{bk5lOtu!6ka!0Y<N=febvuxND&Uq6qk+U&FfbJw0<y*<R<<s){n
z2AyZyQFA!MD~y7#H`AN0A4P!Wi0?7BJI&!oUu!Hc$zwZhO{D8O={Hyo$}cz5+qNek
z{$Mj*Dya1?8+9IjMkcma_lh+$z_VsEY4xe_xTnMRS_7FWXF=8T?QG3zj$!l6q#{|G
z$4_--;ghfFizp#t{K5V6J(Y(WZKm|rCCX(xV;-@gm+_yS&->~Pen-!=jjLFpY}f`7
z-Ey?sBH1|=Z&Qu#TYlH&X|nw`wu;4LGhsYYe3B4i);>F69L|ksZf+Lnymb|^I~Cl9
zNxRapm{Tz3prA0co<>{!(NqsvPv*VmyPW9I`PHU!CT6+z7-MU`5#L|MI_x}beKstC
zcE?Y0zdO8jIY!@o!75yQ+pJY282>`~sIyKbcPmZaq6!?dJ=_Mr*T#6f+foePUmc`O
z*=<|~{e<M?^o#F|2zU%3lQp;@GmPsJB1gN^#bRB>!qp3kg96FxA<OWk{5J1>WJ-(^
z*f;3%LODE)EPP`El~R7(!$o;rVq#+|Px*+`ggE}0rZ6T}HAE+3+`i?q+q$#}hHXi-
z;dvXkCY%<EsNr7WPWIgRy0<-SJHA=4Ys6iU1H{ecN~g>VDYi2-v#W6s?~Tab9n0<x
zlg@rydta59%y|=mxwf=ZSb<zS#ywamH$dw}qX?(b>)}40NBjM?HA6R-B|Z`K%*Zlj
z50xiF4lj>0%C4EP?8!@PygB?7?qy{x0rA;KZZgLS^M#jJuUOC6%NFwSx<}1$<`&*D
zmaxn#)zDYy`4-nT70f&4s^H^hw3lOmu`p&Ze->3vfFf;QjV|zfcJ$?H_=iys{o^%%
z$se+(N=3>DrqEX5e&dI;!ftTW;ME!CTZ6?Aijv$gTlmqDlWauE+-{a`#d1+wwk3P>
zU>lkE);B2Ha{_u+X(nq%=spfND(LvGiH>%h&fRuLVc6iesr&_umZ$kM{-p>S7e$di
z3Z`!?<d72aysjINrOY{VFQiAO2-axP;)53WV3yP^^O+4b%I*axOM#3QH|LpX%^2<v
zJr{%O^qjw3@&C}2+>hBAmlQ=+Ot(7m?|n8&u+}ImJU|!tBvF&Hw4!ZY8|Yi#L-9L0
z_d{(xuMPu^(dHGL`sDUS8H(ICBdyE}+SgTdhq0=_Dk~#~BX;RH%iwOwrQkky{!7Xs
zinQGDGzIpi0VlvrY&}M3y8hkR+QSUpFd9qQQ+bCTNltxVlQ*I=)ef2%?HMmFC8$tv
zesvyv7Q5l1Y!$BYqWcbSUkkac%ea?|o0je-ZS8IojH`%J>DHBD@LijAPCnei8QrUC
z8q5PXtGqNlC&$^S28&b@VGpKpKs#d4{ifZ;92v~tVn=<BG0&>4;0rKwr#-qnt}cm#
z<zy$-O7grbs+M&hksheZOe^#SCra)WBaiQvA6$KO?VG4iTW4;-e+bDKzQvLBS4@A=
zM}9m$WXGy~xBWF3f7`q!8l7BEj<hDd$S@~7f-PcK?PEy_$kMUtBrkRr;;M0lM;%pr
zvvcMp-qDhcyYp-xZgzF0Y<*x;=gz<{tvKk3tx;Pk24sH7KFV&LGdSMYtXaxOXbgwn
znhLPc!GxQ=QOC<@#UyVg&PC;YT^yF&eAlJsCIfj7i_#vUejo6BoXl(fu3MsH2IJA1
zzdA#Ad?CTB48_m_;vNeL?Wl~U0U&6bwLr7>-VQ=#i|0`WpVBMMI}T4nJdcJ@SmoaI
z*)C4W>#N-^Ey(L5J1}O|>FC!P@q1X}ez;=mGwe_~Y*nnN4!asIR%Dvsw|$1Wm8;_h
z4pbD1I4b{8r&k;B3>OcdF!6aSdoNku{FsVZQ%5jc<DUCcUMwv{F~@rSuJyO@qc4af
zUxcf63axSCkEXa=;(B^PY?F8Wg|2I#Q=1EBoGEZ^;@p@Dbg9`MveaBMHha0-JrcFp
z|41y4H=I4WFZM2>RpDoRbq3{_eX@CX{qDT^3|=quJzg7$9<rUOEYDI2YhG=E?_?``
zqOWZ~x+rrb_sy+O`}kjW{t~O_JjgLVW$x8wkM=VXduLG#?jilD7g*EuVc-_?ZDZBw
z@LjT{(x`Kqr4q+o_s|65K<olL8&)SezSp@ZI+f~s&Z26KU^Zj*6Kr+O2k3A&xb1cA
z&oi=?rTlv`vE4?Vl{Zq1maF)Ojb#OC?#%_hc%nK4#-Pn>s$4FyT(Y;eVf<ihXWN!1
z)?f$TuRa%OQ`yk-^t20h)$b)0q%)_=8);6Q1ww1dP|KUvu47%=8@FOE`$Ij%UK2%I
zYw3M_Fh185=d3Oz<V{AE*WZJ>Yhe0t!X{c>!`w3^(W&pT`sgLQI-I+0M$=os=~1j*
zuM?p1m7Hy=i{FdUv3a)&mPfa4v0LE@uLJQ>%qhF6;r1kHw%#MDP5$m9cp<{niY}Ww
zK=;o$RV_c`Z3{N(-s5k%gxvkC(It?EPB|Ev+o-r5?6_LJGn<+0ZlN!Dx-%FG*-Q(x
z(-%Z`GP=*d-+v<R)8fV)zDp|{xA2hJs%L+TZjL@`3YKlwm(b$x&i}=n!GIN*BX0U6
z4DmjTsws2=ol>e27;aSk%x225r&}@IOdx%cw1GZ*?PjNF0UYH6x^3>%wZ!Ya9&C}~
zb2!Z?J{(WeL%SxUGt%UdI`5uD>zgu*HPZHEsf3-H42#^v*n@dfl(Q~8Ers`e^f!7j
zyclsy!dtNMK)6?DWZ)qlL3_xHOmuQzSG#Bz*1VB2ls+@Xz=cWXVa+gWxB2d6Bwi?d
zeV6uP?W)%*XBRd1!9D<hB2$jk#OFETJ~#E{bGhKpHTvmy=rLIg74(8M`d%aD6{hqt
ztn?K*<x(LY_0r}kUfb+#stVuv%wnHo7BW-xqY?8jN08bDal1zIpU|Pj*X(HdU%=EH
zqw`uX-t1`yzGz28t&PI5Z%Vd}0<Y&JaSmDRR)(fe=<JyMWvB1QN+WBUz*6k6^%meY
zxn<GrpcNdnADJ%g8zg7QucsrHaWNWHAN<)6=@ze_UMg2|>S9dVf+J?Ysb8;rL>$-E
zchN@6yNrT~&kHxX&qG_%YnA6VCDyWXBx-z0ww@rWT}k#J9oL28)&1JxVE#q?XR1m+
zBDN1kSH1g-$FYpT5&mRHj3VTr8`X3@`UeH^G~tNs!;L6;!HEmlt_RC;FI-ZLeDJ{m
zYYOCVb2d*r%%L|3ey5e=r8?G-8-L^73KZdvAsP|tHAE}{4;#(X1*2Ex-B)um4q_cx
z|NNLlO}nftm~6r9{dPr*c>ca&sErZ$ncdVVT$eYHMU}FtAkNwKNnHh9<r&IhW8K#<
z!312F?kWC!A#Z}J{oaDk>AS+XiRi)i^_dqwmStU^f!|_<m&=ytgKc~KX+k}&mfHCn
zF-JTw3C6fsQal2Mf)h@Ftcoh2um%L*W!&!IYnT|Ul_t4H1xBO0fpvzmws-m{W5EFL
zEYcf;Q&ZqM!`WLlkh!p{6$>A3mQa+=S;s@A4coxOX22?ISZUk<1lyi0s*@s=20xk&
zPLqjweP(w`JI!8>uKsFRpoaVOyA0tlMf0+=C?nR<A6M*REv#qt&!Du+`a{k4;rtk2
zy0oX?xAmc&oDbTEl6j`?PXtP9pUrTe;K2tR(b;`QSfz>qxCiPI{26<Pdz&p!g}_K)
zZBzYr8@1>wdR8P~=nM)M8g4ab(M6vVT`sgZzCe5>EYv<b&Exq8!M>g<w@tZc&wxjR
zWSKCp&O{@(`rdM5Jwk<aDFu=U@7NaCOy8c5b$S>HmKotibR#8X`L-SjrQl-R(TXB8
z33Pd?G3!m!+lM@Z4wo^+k)$KNj27DlJE=4C_+nht0`;@}&3Ymdbh3xQsiAlN5@(I;
zVtg<60yBv4;F*ftTxb>)AB~giRmU6cQGH+W>2j3H1P}a6a~v^-J^90TL`t%K^F<8e
z2b#D29g0JAcFKa2!FzWyyH}UwoFqr@X_xhDr9&1u=n!Zen5K4JW+@O#fh-k|AosV&
zU4BMJh<V6KNma<NyBCpQSsQ(JbJZQ3Rct@cOdzyu5Mr(g>Bi;Ndl-@;u{=<+lAV*E
zZK`}Eo?NG=pcLIH#ASH#qS8$}%@i$;=2@kFv1K9FoC{IdVHeYug)M<!j=|}3ZRUaO
z5sh3UESRPML8$N^DwJyYF1@28*YZ=zh?ZGFXGMk92nnsTiga&qT^ev7`(Ju~FB+m6
zVUdrAx^q3$fVTL0=mQKCVK%A-$;#;Z$iKPRvYT)Unnk%*BGD$Emjd63OvDD!QuUyw
z;!BTv#pRG(dG<n}JN$!B;NyXW*I6TZs@Cc`!V6x!<~?<aJnp`Xw3_10!KXJKN412o
z*|`9pNNsM&x_tC?vSrau;HviyBFCZehbM9TIpErG)MD56#0xjx373jpa+H7~q;K>I
z14mOROJN#LHNrP<Ca?kevQ*hbJ1R@7r`Bmh&j{l$bv?`pZzyAeA|fFYmHUNZgANBs
zE0U2WWNw_-EtBI=L*pBPsJeAof$qA|kDrYX6uBsw_%J~Cm{tOAJC<<3KQWOQpzc?F
ziFqng*ID?Ror%uNE`2&uOCj89;`O#ap0($0@lqL~Z*((CtO^VpjZwM*CZ~NPzSUv=
zy+7<Oy}?o_fzjKlb=dw>`eD#RS2D{HUiYUN_TR3q(cBemB9{TKvz2;Sai>Vm`oK{%
zy-mdvH-)XOAPrAz7<iNmACoR_rJK(0pu6bB14VSpsh0WMC?NYo**Vzb-C8K4;}V&@
za7f(sBI@))3?eo{@6+!it%ycOfzL7%qQov0n~yklN`oQ_yjCnRO=Su%>bS_)O2&};
zkJoNG5}qg(La1Q3{-Ki3XJY*b$&D{p#Vbd=4_4elJ|%g#2%kY2xuy#BX{8U%88aSe
zK^V-h(1|4a9POFgw=e08ZNzYdgMQug0sU=W-UQD5>|C;k4MTw<Sb98KM926|+E>i9
z$VVTly1iCpy0DkxzL?cp<FWin9?4GM-{fV0+3Eh&!Z2K6fH(0vRMiD6-!YUrjp*iG
zXY`;eL=v;LIAr7sd!lEJ{OSTM@5MOSWNeQ&a-G9Y_;NNfA+v6DYk&JkK0R{F$r!#;
zHoOt7p`EqArsKKZ)b?q1Y%_W1$2!Ygz}2uXng(8Ii-VUyG29jL`rH^9b0nw(L~~2b
z<ko|JI=wH>qfY&Yz?8~XFw)2vkN|v9$h+@N)`mw=!givAR;q^+Pb@E(3KOpULb=<a
zgwHhnxEB6fW<!^)ZuvQ^y^lsnlz~y3a?eMg;8hZqR$Jsjk5$qszJfqPJfqWh<(YwT
z{`d8%F<uk{IzAY5&ZaeRirw%sr?id45c5!~&rKC!^U1?ei%`UX?Oc*eXV>kyL@`#X
zkQmE_-GQ6g306NCBT?K%UzlfZrz_tHY1)~SJg(%@{6Q<Q*yn4Gwha2ZHFd<-NzFaW
ze2?xKS>!{W$`4$3<7Ai0<6P6&o1S{xdmdUAje|MPSplIARaF}CrNzh>xuCr>PK_Q+
ztrtUCwnrSg_&*OPlh7DSPuSqBnan?eB4{R^jN@x;6xP3A^NMJ`^->XXy9}r}i<C2j
zK(109^GAW#(=KXOhKDK}G)Y(#aT$2xFP3b9&fN0jvg_wHepfd_wN`S(c9*jgx&K}a
z04BitPD1=0Q4c!^-M7T-c&V;k*YhTMqnd&)NtVZ~n*HFrE^7;m+sdl(L5_FN6tknx
zqT<6so$SL1v_jdMD!hC*0?|5FpPK5dI>k2I#a>1?goUT-t1@jwsVZfKd4T!vAHZD?
z6GeSNKJ~H>1HPB}L}}w|vnDMv{hNCIv^8sf@7)xf)9Xf`&X}NXK@l-4OC64+A|f<C
zMaCn&h!(je1WZsJig3k!E1{JxWZ)A|3PtQP-!o9)Z*o`t#*?ozkzc#bMHx2?bO%|1
zBxK3|VBjQ{CctYqV&8(^iVr*ptLWdGmzUHI%c7}WF&-_f(~Dw$`GUVw4{m$EfZ|*5
z>vOaQp6g%g<;arF*kiTf8Dh+J)B-upyMx{0<?LWF>F(iCY>wCn=&z`0<E^b)SZj9b
z_PV7_(s%kPd+ofl$L^%RFhi_{_rdUZXa8f<Bz{)@nDIUx+tGm`8*n7Vp2|aRbq2~E
z&F(w4k3?qUjI*oK%T+Ila*YC)*-cRD{`71{QKRKnT0qQO<Qu?wt_DKajDd=69}&S0
z$+S=1mgJ9A*jF?gw<6;g{L3qOC1&EW3h8bRB<F{sU>tUuw)mrhyJlXk>qiT{NT*(3
z4`19+^Kn-_FgL9?Y^v6q(Y>dS!&v9PvNS)w5QT@r|EdKsL=k<jz1uS<h3Q4QLzd=H
zh~a2p^BGQ(2Ye;Uub2utM2ur>yg^vt+$DJ%F}m(*Ay!l{jcIoWdeWbfq3enG0pL^d
zXTrl@uEK}MfDbJ`oInGb5A^8!k!u^bfm6|%I;AK)`rUD$qOS=v-m7Vl<g@BA&WWfR
z1OCtH(*cSX<sNZpUwge;+OIEat*4UEIa-3MIY6~dGf1+I5{%ouCHM~ZR2J#D5A%SM
zaWu{@e;UGnZeLX_@7^?BBdHY$&rlR_U@C7+P~+6aK|e}1-SW%2Hk=mBqpcLsUE76o
z_R^y>9r4XCgCFZH33mglG-D}f{nC`Lms-%oGhG7_2vUobUrE;D4&%%U<EirFOXzor
z@vXhJwZL^XQ3B!E&)a~r%0Bs>L{$}{rBo+uvNvMLpni&AduxUc7-hG1pCb5IDyGBK
zBP@zLZgQqn$*gf`CpQh8qKJDSo$D&TQi7!8WjXW3D22d6)?w+wgb=P5#e6o|mWY{L
znpO%>7DJsN<gk&YR`PPtW3@WR+B^koH7+<0N#g5=&UUP`u*O%&JFLS>7GMb)Ei%Lj
z1H$n|EtD{V`ixUgTV1{>D0N%#C3mhMR%_I~TNr&jD|!0J7GDoLDq_Px$-t0p$P>?F
zL>Y6Q#PzkvK~(olw*wbcAKd7NYs5Gj({+lyla{vyypk<yP9)RYViaJ1+Ndn8gh&7C
z{Pz@Ke>LAWqqAh1xjg#pY{rE)CU_>y>%bruL=B(k27q8J6hQ#bu!OYq8YDEo4sbFj
z2`RA^4<DdmHP?x{jjz4}Nyk^bOog}7)p7$gf|X-J{#d4I8FCU>Ht~M}(6`ED`DyfE
zGfY4&9L`VrI>kyDX69E0SVbNBkt#wPc)i&+1>K55@CdoK%DB4sk{NZQK#|6_Ok`lA
zrVww6-Jq;W`A|{v{-Qo@OY|lBxlsG<v?iSpvN!p*pn(;g7d8!>s5NuL9<~$|))Vbj
z&Ls~UL_NRVY@=lyD&!t^-}h^?7VVqajs(pz9sseAZkste-ip1NBwS52f)0MnDrK0B
zcQ~KIaewb*O91M8#?!xiX~k-k##Mo;paLh>L@IS27D+wISIle|Ni7{|wNl#O8H5Us
zguhM8P}9JBRb+HnoJ*kgOtz4f6Yj2by&z50Ov6#V0~?{vB`|!}q#!+b{vj8?U^q7V
z#`!X0MxrS-eGr6olVv5E46_H*!zHx!+kibPA{Z`BA6%!cL87g#ugdT6sH~ME=c()j
z2PZDX_gtWqqXtQxHk^jLwJ0seLIZEXeVbRiF?v{d$Sge>?MoB%S*bY9NnfLVHY%@j
zHvTb1JAH{|*Zy@MSDHMW;<RWzH*VS3(PS@H_#-@_Vi57Jxj`7dcG*P5F}x#<#9o)b
z(?A1HxY#HrJy}&gsY!y~s!+PiofsS1x(|h|#=Wazm}C99r`F6yK0e8oXm>|L@Wz**
zX1p1%k=?KoS{z&qm~9Mm3>(V?%;Ju&GqyOCkYG3L-KiM8lfB$JuJf)S=2--yU{*b(
zswT?Hx<t^6*d2s*j<{i3@S073N)8)#4}fN^#QSD{nB~>q_wmf`0tqFZIhS&GBEKOz
z#|+!DC%_9Zt-wu8J`rcEf8PxBi%AvH?DiSpWv}czps{Az?<BBG<T5p=t=dU|Enh(x
zc3WAI=k4$Bp0F+;>I-_|P}pOFytdRVm|)RR`fmh4X}$C06-z4C9K<Km_bYWNQ#?GB
zh-x&^o!^N4@$lO<Ae%hBu%bUx!y|B%13eQ^&tm#hTmPx**SY&=0I^QgPlo!31OFw`
h|FFgXKDIa_d+;N1L7DoJ@hR|kUsg#b|Bm6y{{!c{5E1|Y

literal 0
HcmV?d00001

diff --git a/docs/figures/arch_5.png b/docs/figures/arch_5.png
new file mode 100644
index 0000000000000000000000000000000000000000..035af8a176b9f84993aafc9af7f8dc06ed4ad816
GIT binary patch
literal 76885
zcmeGEcRZHw`v;D5yA=|5gd`!O?7e4^k&KM&QL^{m+%!;>>`mD_B%3>xvLkyJviIKK
z^TO@*dVk*U-#@><zJGi@9&&fNuJbz2<2=Umcpk?oNLf+lBHmRz3=E8ma<Wn?7#P?v
z3=CKn&N*;}Wyk+J1_oZSxum4BoTMbRvV*OOxs@>nhHOw|^m%1fEi&YiCwt;E4BShh
zL>|Ox#1i+W-)%84(cmKP1^T5tmLb(JWDN{NUers!RZc3=^Ael+B{s{nKMP*d(&NuB
znvRe(R<TD%Q{uLZLraOSi-J=l6{dehFm5>&XeJOaV}#N)MLpXo=Qesg9<zNLiM5Ih
z6vK&*rZ)+A2FFm#n5t_EV8c*oacyPrI#}EDCAnKm{pvi%Wq*eo;;*9pR_&dt{|3?b
zV^Hrss96lNT7etfj%B+|ELCCZAD-#<#Gm*k`P0~sK{#s?TG$w{*ycz&EQub;jh1Wr
zT(bf5LB-Dl*)P8fuEmND4!$qeWTtEQY#{JK(sKK~{qW?8y6yWBI};A!#v-NBax`vu
zC0}QTjE|=5_iY<xqZPk%NSqhE`-kF18~%FB^)%z%X3dBzmql9d9Ht%6Wy*TfX2wQO
zUgB@{D#)y}vi49$@!Z6%5#$o@PY5(8ed7LHsf3REJ}I-@pU+7q1y2|Xcvw|3lrGsl
zIdoLL@jg>f<_ZZ<tCsoiLvFt7Ts2Zx8JZ2KSM8I7=2!<krF)V$nJ0eqm}E+AH*K~h
zHX2YrZr^Ts`F65_%*~^<PpWDDTmNFQ=AU&$Yz_6WOh$VXZghUDH2Xx$0HU8f!{Ojg
z2DuxI9f1_PkHF1Cihh8zAn+tU8P0sA#J(Q;$C&7m!#%$6kcY9z#!D+LQylp$c~%%P
zS^e3}D{NN9q%;k&<a!L+l6iz?iB_>4^ii;%x){i&RPw|THjJzt44%@Jw6FxmVJSgO
z7HUkU6fE@=TqnOv9Fl7#Sc<5Nt$l>>r#OfUq_FF<nB)?$C?ts?>n)7ChTMJUm{2w!
z2)mG9#%`IN%S5@`k?Wo>`|Q1ZW!)ch87~+~Xo&AmEh6P-7Rr={JtLVBO7hiTmzw<+
z2Hw-h@8Gw{iEgWyVSglG^1qjYXU48};Q*HStW8qmo#Hf2*ZGULZQrqTz``F1<zZ5w
z93PRkyg2xc<A#}l&;G*kx0f5OE`i3EfH8(`5iiPG!mawku?KK!2OO*p=@ta(A3r#)
zy)4m<+Yf1;%aW^v@KEQ=3OkU^+<o!3{mQi$FCIN^;ZVF{aG&l)fL5^9bH)ZWLkkXt
zm{&1XF=8z^e}$v`)xXC*rYyQ<h0jM`1=m5cez}fHLh09!d?zpzAF~rU*M=~Cv-a)V
z9~Jv%`)}om<<difx;G1`_+Ne>S6|6+Gxg#~dez=QHs(4;_`Chj(%AXsWE=-Lt3S!-
zc!Uh%(khPeGogC&ddm6>^;aJFWhpt)og?1)YW!8<>%rHDe^~!u*axS{a?uHf-;m8t
zD@}7y!I~k;4p>(NUovgC+0+zol}Q)qE(Q*fRESlyKcO$i?SFbrDJqOVj4-VIEmhd)
zgIkxeUQ*D%4pZot%}&=%5lQz7%WRBqcAVH*&7EYPjGtKgQI|oKy{pFOOv4+kEiEct
zbC>cVHPe+_Znx}69x|#iC8$k5yrQ1QsLgba+2j@}a}T4w`ZcwBX7XF^j4P3=k(-Qq
z%Cng?JbH2GLu7Ph29>I$C{o6!UpNtnM(Rt6-(O03bBmdA@RkfCyLy3|SpHD6#FcaS
z1hk1oLG}?H-?q8rjYvJC40~Q^T`XB_+0Ek8`KE9;MJ1~<%fVps6_aYWJg<WDD~pm@
zV&34mAnmLHqEbDJ=&l#0zf6T&|F*Ivcsp&<I%^I~iQf0F*cIELm`%MJcUA1_;}FWL
zWIQi;$>VI|vg6X@OnCx6nrX9X-^o|WFVSX?ZMx6Lov77T{o`84!`{vAD2t`;#h=%I
zadriDHU2X1a(^%1X*|O)6E*v3=JmAs4EAjP&)i<mG^H-nnapY1UfbRyGea|k^@ov^
ze#4*2-$IRIEH!QWjHEanc(tw6tex7gn6_{!a*vpM4Z`_~c;z^=c`A(Y;s;{!6kOw6
z+CI10$I!;9ie?T5TQ%@CbLR5Taht}RkC}?)kFQJ0<>$9qjH+00tkdb!PScq$bF<2R
zGGiKR+HbMb)AaSFsk*syd)(s%>zaYeZ@VEO*&!&C=vFM9IO~)Jk3qgcTbE52qN;OM
zG@hb+@I9tI8`4CQdL|P+dz4VQaGs06<Z^#U8|UCk);N`-iK4lpv*Ob={3+bO3(37p
zdE+h4BXdJ@^-gkj{o|c;ISWnB`7YjnGv*pxg~vAity>h0sUFw4l-Wg>C)*uO|DG=D
z98b9UXJ%Y$y8n0j1j&s0?~N{<@5236Bc5Ns_n20+3mW7V)-92a+Ppp&@I8Anc)Ncu
zf4yp2X_IiPaM_pCkcb-B?BX-516bL)P58|DApf|hw>fUQSuTeozJz|GseAF~j^rI?
z$>ad77x_4rgqhFT@d_^-A%0&kV*F#D_8^U?nDymV5-N3y4bB1XO1esZ6PuQ%n7Ja0
zqWZol0WV#(A;0pD9sRYkR;s|V5ca?&ZNqZI8UNf1-J}`pA~k`PfmPF*Fg_ZgcWl3z
z`D1iiDHSH@9@4*F7I7PNFWk1-S2+wtX(``udpnV!7RD1ZpHR)~e(M`=I*YO9G)*3&
zR&GOKIB}PgRbatzz|F(S!{{6hEzNkEtXu>`WlVN#1y6~p6%!j`i_Jy1+Il7MtAPXc
zg(mIL?l1+J_*4oN+8M&m+H27XO$kc!@1w5i<dmy@TyNkCH>QZH`LWS<;4Ww)XzM<R
zw?Ja4v7Y~=aHtATJF+~-x^!`NzZ=WatEVfEK3_q{le_&!A>GnJ<+OAD&ea{u)ji+k
zop-eowGt{nYeviu7O6PJ@qFj|3~L^Jf3fs@zuxjL>0RTy2Cv+|_4*X=XYXgOgve0{
z^YvQ~l#-WeeXOv2H5U~}^et>C>@eR?C;nb+`t$UqGS&f?(v}RXEtgK0naz<^<Z2Y9
zhcMsLw)R^U`?}7W;-Y&s7LwiH6ZaEcT`CqV`tpma%(kLM>qQAScjv8h__Ejr0|ui@
zmUa%a4i>~qd<sNUTvDb{Wl!Hft>JydOJ=!XX*t(5$JhO3qjUG2<Vt`I)uzZ$r(w^B
zvbP<YPVj4Que<Na*R$)%9K88NX+&u$P~clV3ty0WMBMc;&pb^qy`qf%Q~FxkXyd1c
znLWgHaYr7@5u4dfznU`n6KN6~#kqa>w&te=S1Nw5HBRjfl9r0rA{>8@eEa!|GJ2m~
zuVUVN=Z<jI!92+nb7p~%zNmW$5NPG_Mh3+ve#N1WHSX`f6r|2X9ppz15|rr(PdO)5
zjC)M&>$O)!FWE2Z)loUsl`t32-FEeIUBWti`Z-{Ra8#tpooCZcf$&r2a!`GEaptU2
zt}pjteo=41mQ%9nXo%_W_Ul_~y9NhI8+*Ne`%@89#_S*Uy9l%|CS6^<UU}ef*zBkA
zN5kj~_L0iIMz>{Z@Ao6Sz3~RwmLDjJ#`u-8Z%b;UlwP)LwbndO)@K5O35}>Oh-drc
z?6m9-9n$YMzbc|PSTSg>BlopDSUjpR6*2PCZHctIse=ipv{NC%uqA{^1YsjxC@?zr
zvy(hjQnHVba<HSI%-uz~A}m7#evC`am^kIQsMmNO)9)Urj$`7yd+%`*Lr15%!Rbi2
zKUXQRo(5wRU+3HawuskZ`{?Uzu|p;IBkyy>`qrq?DPOFH0GvN-<KLG6h<$0SDQBXf
zfWZVl<6yusuVP?>Pnh6a43pyQvoz){46Nhp0R9G;W57@EQ3OAsuTby}J#+FCE9?cv
zdGJ3X@a>iYL*I=JOTj{ahGl`@Vcb@gl#>HLRgD~sjcpvwY@H^STRwmbxOTEyju;rE
z^w2k^oC@7C7{AwCP18wJ;l7ZOtu@=DC$^7`+1#w{pm8un+=ReKYh$NJ)Na;RHjYAW
zqBoB35CWf}%j`F(k8g3Z6uqITpiC`k>tIaH$Hu|NaYGD`nwna~;faZmij?%}<KTay
zH_V)z?1b3aU0q$-T)Ek79ZcCd1qB7!Ik?!lxLCm*td8zBPLJGJZ5(M&20@P_W$b9=
zU~cDRZfipgjr-`at+SKpjT_L5&c06OY3yeH&zo!<PnQK2$PWF6os*4&{cLRTs0egb
zNZH)Y*h)*v+#1LXyhDsnfJ5Z?{{Q;TKX3fklbZiL$;l<a_3uaj^{fB;sJf%EgQTrB
zc&C%tKl^q1@V|d~`k)9qwDf<8;zZ8JSAn3#@I=_pHcbpKc+J!b?Bg|aDJ3=V6KEOq
zg$aGWb@CJX?EKQHXTKW*Ljpri>b9C2=KL^rf|~l#_mxpnMoOYiYMfgshKMk{mv;k+
zRuM06l}~hb4zt|AxpxmQ5cbla`bJ>Nvn;2l)bx}(YjsoGb>oZ0RrB+TL$NzUy{}CF
zjClU>mZm!JXcNWc8%Gep5fT`f|MkzZxgUnVMDle5p#TgR4mI-s{DUN=#+eBCzgLfM
zLQ>%%;Jcrxf8G7BL7^Anz;-VF*Kp8-W9MOTKOc@CPM7|BHsC>j2_LNgo(Fhv+YjS*
z3Kc$=p49(Xh6Iv2boBqs{q(;+)Dnp4^H1-SUHV_^2g~FC|CaY(JMsVK-P>0(>`ap@
zHfg=8RrL6InnHXJJ;qdfa`aS^r{Xz+Yq!{|m;L2uQ<P8D(k=<;%^C|M?!Oi(j}uHO
z{II^ho{ob<xpr?+IQv2B3;6+F?_DSNy@fJ*Iyxy*c72wMS9$E@L@&QQ)ugsd)HqJB
zY199u)`qx#zrO^?kJfrS;|N&xu$cE3r^^p^7aE9@^I65f&!2DJ-&sqS3uj=<e^Y;|
zfYg(~02s^0710xXi)t4?V)5LZe_MBSsDnqsL`TB>FhV8$UPTt)%bIA^;lbWyXS)1u
z@%ig+5_p$yA*M*<TN!l9Y`RAyS#=(A88=;?5}_Kp$>BV!=Ds!T>ipdCZ;QP<j@f3O
z>(U69!-Om@9$r{s-Jy1p`(N_4%Y~=*Wk>_8?ZU-#3b&6f3N?yNa7}>Pwq_%0fB6fc
z{Vf;_w$4IJ;_#$7inUG9)LX!9Ifa0V`^gXLjoEI;hed46GT4rTpRB`GGZh=-ZEI(W
zJ=8w4*Q|%-=~k%pJ}9+N@Be77ny|Y#R6#q_nf`0^V0-)xSXQLK)Xvy-zYAn+vOz@j
z6RVTWLih<>#xFgY_$(egvP%?lesZw8A@ot>c5NKZ55H6C_YhIzeC%=idG$nW`wzz^
z8pDMoR0^54Rk@q5doADInT$4=*j^bQn9R;h3Ui+83BsF8@UGq*uq=MkNaQ{lsWGwE
zF7E2GqR6hq`q`@E{d4_8a!SgD^;ILXE5bNxqeq8(ODY-n$@l0lUAnZ{dJ6}(z+&T@
zY68pop!?jf%4NZ!(cq?>^X%6k;?*CtZ+=8FYtV^^=#F?U)ugdaT|4!kZonD+#F|zM
zI-R)dg)EH4SM0Sv;|cP#OYbFlZ))w`rIij274zPmZkK$4e`(`Bmnjw8FX&YRAxzk1
z{#N)+rB9<02(!%PJ_?eyD0TyVwOiFrGb-j?S#M<qxSXby-H#5o=z+$(H|fvOK)d!2
zuYsmJH~2@LDCk%jjOnW~cTm@AJ*QEF?AGG&QnB*q1|aT@&aq#aO7gqwGYh?0^u<iU
zD%?1ZQ*G8q`ZX2ad!Lfjvqo}LCV*Y2+HSmyx{)mGs#r(GVTjc4(ahK8eGtzY6ENVc
z-YTV5a@mLqm>${2eDyU!haZxJqICQ=LkM09o4w72Fm6-D2l8g!c@58Ae+eLLj^#F8
zcg7Xnv-|#<(qZkd0;A@KltZJacTqlzgJnv-`<sKdcb~t{@uuLn;bHG6OqY*#vA{4o
z=VuY@v$w$C7)JkUf1}r6lialNBCI`0l-aVkpqpFd6&ZU$bQn|aSAvZy<!0S(ZrtvY
z8{<^*{I)%Hv0?P`_X>0?{v;fNCntOIbvBonGz#+aBXYI>=9#o)_h*z^bQ5iORXrn^
zJZR+~o!-ol3zt@lPxf3C+?)Tty>%;`R_67OgRRj*(@91$-YA_<RvmT80aQMkTY`t)
zZQDb&-kv+tsc+W%vP>;AfGc|UxZ6P0!s=*$O(`fS=#sI;p?z9Ps?3{v9q;dMuoB-?
zj8IGz5_Aymy>SwCkh$2wzGHvSLX!xSp-6L!$v*d+R-8dA>6kD>onVBrL23baeWXC|
zL~Fc8GKIL0SHe>FdD{7sQB%0MeH*{+$dHd>WPBeLt!9v8@-Bz%h={j$YaFk&f%irv
zt5HL+{i239r;AO^oY=;<LEF0g!u$uR(*9E+h7XdLM(cbN)_@dWtI9lHNaQq<PS{h~
z`~8i~pljnQca2p;3C6~rlGx5y{k)Z63D2rvj;=%Rd46e>WnHgT8JEZIOs2JT9~pSY
z(>S-ly0;+lpsLR{ID~P1M@jsEXR~*==F&j_U|JJLRKnb7$c_%@V2VDmkl6MZR{SNt
z?~Iaqmhe`-aq=MC;m7eDkNj&7yEP0-y@7u7(ITqne2mTE*0>|daqN+)76$Qr*4LOC
za6VW|YnpPyCaVT=nz--S4ZgTYag9lJICW9I&hg9`q9p2%CXDj=I!_%2V%gsB=TBwI
zYa*L{MnRiu*rW}Av34t;Um$C!Xv25Cb`hqIkzFsb@jb>nvzaf7YoJbbw9IB$NKx<a
zd2SpvYZYHLiJlLSB)toZ-UYGAKWA<r`0<x@v)2>buYu*Zx9>jna%Qqmaan83?~K9+
zey{q^uXV@BnHDya+qO{KEsuTX8?Sg)IdU(OssB~224cJ4G<haeqqI9F(wsYrbk3ng
zxuY-P8z~8iF?+k0V<GiV{6Zo;*q8cxD^{IPAIVF|oITYwEnBX1FNJ6Yx(Ih?YzJ^!
zc9^^fW{Ti78JrpQb5F+G4kmY6u`N08$8`B&9j<y=(c4!9&zd8dO$)gXm$nN0RhFOQ
z-BLFQHiZj0m2vL%jIi+%%<}i9$`HmEsf|YQT0ZnvJSey(*aDa$jl#tr*1E4wotO=x
zd@tE+NSRUnEcz8;&-Uc!Vg;PLc=hYyZo$@H{9He*^9_R0ozxsIGzxJ%9oEX2vJM15
zc)wp2J?k`K@hly6neXG=n=b=FuCI=NG>r#`XGx=wX<x-Tm!qXSfE>O#+udw3Kb<$H
z?o82a>)b7e?Z*_sV=AA6!QnENY2p$$CH>~Qw+6ooF+A>hYzcF#axU-#d`1=J3aw^p
z-pX|~eF1TAEm*dCre=E!ZMmb!29+q}E3F&2jJ*}Hi&h97rBj$Ro)0wqCEt|qTKY0?
z89+ceKPo)lDt@$YL`BT&*z@*_ri1m8<^G;pMFaPdRm1Aec4pYVh*cklo=K93dsy}B
z@Y4V~<^uxgR=g_hsptPn%<(0#Y{Y|kB(|I?B+<ed_4A|oFM*pP<nWxy{30x>l1HMR
zn;!$XH4624>v{@DuEJgBdKAh=@q1h7tH4q%1dH$*)R3gYYTUQGT11@>cczm0>x%1y
zfDE0lP}KJ|^F2jv*QE<O{_XqT>UKT#z+<ho+<tt1Dr8D3&R|Xh+~;R-r@1j#E&KhF
zf=%DG%w;+;tKrJ-5r%R1XL7paTWU|9q_y_>>5PKFJy?EIb*1s6`rjbdJ;l2#LujCr
zjiAQ4kA1xi?V}vwckbMAy7JD%lHOTlcRF=f$Vle5K(sp-j+(p#u;rtDM^Awg&#gs)
zXg5nlUxlOTFd_Fc7oUDnb|`l?wKBFB6GBD)39(R5o>pr6RCDxF!LK%aShv78pTjf{
zmnUv+<<=nL+s~>$%Csmf>X1)ZY9(wiq29C)@hzMQ>$@-DXfoN%US~x*710ZD%)khP
zQOVVSCLTWjqS0K~J>KgL3^xfPFCG1<aGV-wRx2?QJWX~`LI7*^kYCXDKh|U9C|(SO
z@Q`IHRw-QLInPS+py@YiW$XLz2XYR*R=$zhH}niXMFD3x5H04r{X4*8?LtZ63jYPp
zXvbVrtWOr*vjyTzUA#*n;$PZgZ;#bPCB|c|w{W*lZ+<o4vP#lue%ZR2L(REqUr4<S
zB2Mp?kT8yj({yN!LId0PqIjqFtp>Kwe2y9U{p_W|+Sp|cHeFlQ;_og55BocNc5zdu
zrlt~CK3hUhUJ^|Hx__qjDNx*28hcKr|MRn79A{+LBe5#$i63w6*-9QG+}5WbcnWX>
z)F-g8u}Q7$+%Uu=W$pitfOlLGTdwFS3Pi(!i!Gsy*%~Bf-^3Eg;SHm1^e&=Bqwy4_
zS*FqqE&xS8+>&QKdhPATgrnB3$1|7~#V72MWv@apm8F;0dQr2*Ugfv(9r?=e97$1m
zmEq*Ryr#6pNi^mue+V#7X>G*oQ+Lsb;D|QLA@>cUBxBS4uq0CT_}fc?Xe~~hU{TuT
zy`{Q0zw`}oVEontPY8FqY#RPbUnzPj(LGcD{KDK^SFvho1Yfqe#V5V_FT}S~D%3tb
z%-1f8f9&hKFu?0sXWANP43Nr-8X{|!&vYX@F;UeqOF=>H))miaqtXi>XahVJ-{Zjm
zjF|rq2L`}978lxmNp!*h1S$()wutV3_PUP(F+W_z-$rkO$l`Md^vyE;99t5vCJ^2C
zwo~sZv8VJHKoavC4Wk0}_%PrZlMEeOw9MsNAFq8a`23d})+!wz|6+g1_oyV>bf0l*
z`?g`KbjbRa#i!U*;L{jB1qBOwDrnKU#8}^N;@Yh7-rMY6U{y(zoiAKi>R!9vIvQSK
zS?t~!e%tWNi;K=xL<8vqR>`!57Nmv%Ejz^$iftPf+p0gZiDJ=O_ihTqI!$Hr=mAK~
zUrFIT@kiU~D3FXS$jhtCk{&6?lZot1gl{U}ICT7uN6M1bPXCeS?tCaGu*c_GOk<3C
zl}`J!#QP!;Sg65E1u)5>m)GK40uws`Ij)IweMPBanS7Cf0LD1_9338PQqyNITl5zj
zSK_OG|NQj)c*O5{!oLh;D^Df3x9=s3)wFt%8tzlu^?i6$nK*k@X3$(r{DVbzj;phr
zXk8or1kgG*{pvpbg1C(<N7ExURbqv(v2E@v3U9vhU9j)whN)9@vxxnY6y0L4_`WgM
zOI)Ld{UPUmDdYY9QS-<A-wyQ}M(@)Xd2U}nUH#Bakl2e^THHC|1N`XVUFWDxLOBgl
z7&R36mHRER+<cCol7tFj<=X*qTA6K=?a8AqJFCXt+O1}BBJK_erb+Z9@Ym$r4f;n-
zk<1I91GMofjhmyk)n{E|7Su~VyrvSjsWTb)Xuj#Ow=rjWG~$(N!WOp`_b3QYTWvY*
z4&9or%Iy?9{MNay&Q_pr(|^<jEf^huht<-{F(AGGGqSz<_!GSk!fkimd=QIJBxO)L
zMFMj$Nz`+G;lb@VymqypnF-@me_I+;QmErA_VPTgj{tmbP0sb?!`5gl7~8p3!Mw|=
zzHfXNBsT+UDcQSMNl8iTDz58Y&8v5CWH1Lnd&xrQj=}suX>*jIqe(p8XQc#zM|DyQ
zuzVN7hw(46YWX01$d=7NTb60!lj8~EVsck{g5)WULWqlid*CElptI8gB%a}Q&v{PI
z^3-}NlKEK7a1RBRvF)*^g!8@G3d8_ETlty{e<2jN5|36xxGfF|>VO1qp*M%z=8f2H
z7C{0?&K-NL_#CAT=Qp!48g(=BYBjcUkqn3LOd7JeT0PZ6%{F;eGw$<|#AG{ypa=GN
zFHh^mz`Ii5BsOE+nc&VnZ#5BFP5k^9IR`I^<EZt{4^^nPGjSI__?$8zTR${*$6NgI
zR~%>a1f$_f=gq_H*k~n@m1nFVA=(u#z1*+sAE%7SN?Os#)8yPcurSHk%{#ot%jY0B
ztIR#yC6MKkZLczJwrG`Kt12I*PECHaJP|C>9o3zyF~@fv4)2Q?t|4$V$q{60Qd?Ne
z^O67S+BA%xyQH49)B8wPHqaWR`&*R>YiJaJ2~YrGiYomn3ZP@Lo9U$2$y7}I1D9e7
z1zBP=tBA-Dsju7zt(SPlLQd0$qxYqTQ=_<ZVihfBQpt<6r>f|kvrKEVIa={=xAZcp
z=k$T#Ogi-t3g_tqBz^d=TRf9IW*Ot-N~8OV3_rv6f|;G)Fv$$3-pO(uahEB3^dpoe
zXt<zGLqA`;^yM#G^@s27Jh!6qUVjj_CFIcQb*Q_a{y8VO52!<>FF+OWR}UYl%%}7^
zsg(5nZBMpM_9H*~3vz;QJEB?3%K&xsa^D5GDnGHyVPrdC$4a6ksjpBj<ds(2K&j;d
zg<Z^6j`DqL<hwiL&j7xw>g_btuzCNW%tp`pef<+&qc#12xXG*}>?^|ist4-w$?94b
zJ%$bh8$*hYQ5n0IF=q~p5s?ySyq_$3{#16<hUAz=T7}nyoMeMhcY~GXgA+}^;V>-r
zi_vZ)j2ObMH1mJ!+fWO(TS(gaS|@Rk+F{^(kUpm|d4>B2oy_ZNE9}HZq;b_N4HT7n
z4P3gMQFmWn;s`2=&<YM=SWm99dAap`HnS);=t>FAw-{#zX*cqvmlNJRQIc~GHY3%H
zcVCf`(87s~9%`=IxU8w)R7$#R6kF)R?NUi%WIijZW4AG@Zfyy#9caYCT5^BG&8A$3
zZM2zkrTOVMoQkh8Br#*QpUIsCbAtgR62xir;K--jwkd}ER)f_KPg2`w^Sjrb>sDG6
z4)zq<3><R|H+pk&REI5D)}jEY5&Bj)>B$#y;WYSDtApU5{b$_dH25!T_-)@#sT2BX
zJBOP^J*as#|BI>-X~vZjig1#18EH2gq{SbLkSPZsXPvuq)Wbl=A94AcC|X6&0Ttz^
zR>eLQ^ESkZErn#WZ%ohkmpo=ki~?zHV9}Rik;CnAwwaOlcVDGxm6%kr8O0VQh0KM$
zymk51=_cF&PWQ4j-U(%df$2+3jk8{MX{+`yCw;@B!UN>fx&v-T;Y5`1tL@}Y=AxI;
zkRY23fXW&g`qTUYNre}4JEcf6=ZY}a-a8CE=`&V$nGa0XmV$EtoqI$flqnpuOpCHZ
z1cyr9pVJE|(*DC`S3}6dGT`+K+W$r4hI(Q8qb79h(YAS(yc`QgpBXHsR8r>4|D=Xx
zYcXycuI85${fjs}-3*Yf7m|R554=nLeJbMrd^?>1lA2@Zp)Q&O`j4BCy?DUoF8u7U
z`)9a+eF|fjK*V@47yM728)b=&1FLbFHWNTw%zwPN5fXAOiN+~%KSg6`Ehq&lr7!Hn
zbM1ev$Zz5T0?sh{lKFoMV_6$)N+G4tRrGfL$BHCY!K}h<zeWDXRrK54@`1MX5>Q^0
zIosniz5^jay#r>|SoY`MzeMo=|MntjVGnx0x~w5rOzT#rQu=PN1xgu^#jn%|!Th;{
z;KMgD%^mJZydY{w4r@q0f46Jv+CNiP1j2I?V#GcZo(~g(n1(aDhupVEzD0cW)xv<{
zC7ocSE0}&nq1pa$lUy3VeoJ!S*+6)PeI9<nKPoJ%47UF^Y3)B-C|e-WGRaEhXp;1G
zzedXHZFHEYPI18noJKakE&jKtYFk;uo~FjifLH%4oe0!k?)N>n{bxVIE`N|~*-<C*
zc8d9`yCc}hoaoHO(M(LFmR1P=>+$&(7fS1!3ISR%#`u9?eYm@v(6)=Pdo9tDevzoO
zuB5Nv=x+jyl7)V(R)(&8LdvM$PT~C0gadjHP<3EK4?Op0MA4&#T4CyZ9VK#3i7&!;
z^C@1kpM37;9xuQAbUe6KdePOGHzEYR-`l}pzuWQYDbY>>u6q;VcqkvVpBgDUAzqM6
zM^7K{Bs`R>BxJIlJ$=Gv@$AO^d%&6tU&Oba?EwK^-+Q`QL)#!Rv7XNq!=CQS?-q%4
zA3s}}Mj38Byl;)(J|r20V>%2w)XvtB4^)B9VT+l_(Y!_{SHavE?3SEt1<8%iu`1f|
zEgH1th_M1U@av2G6k7Iv5x9Lf*zZlep8D$I?Ae63{8Nu}OFcrhKjU7I3W;bJEaxs-
zlk<QJF&i#=XM8pf8MxJ@DZ15O?S=xBuUpz-#KWtUw5xy3-trDUeX=w%I&rw4uBv_c
zU$18buMhunFZ1m6m+_=VBLWR9UoJF#x=cD2JJ^D-dsp<*-Aw202bVYUao_m@;l|7N
zpJtJu4f#U~5IOew=L~0yx_;@yw=3;%D{6}XLtCLg0mxy4y|N9z3VpsL^75fR3A&&0
zm6AGVx)lwiR~a1Te|BMsC{U<R{{5Sf)g0-Yq_DU|9HyaPeF|=AJ8w14BwPq-iO2Sg
z2wF>)3y}ll0;|134d>%~Lx-5*U6h<AST95>6=5a4>v29E=&Fna3W{$9Vd=VOgXzF;
zu*-c_n*8cA6;bw4GmHdw=xYn>6r(mz8ek;~8@Ak)kvyB81}HvO-ihQoyTDEqxSJXx
zb;R3jsXZ@?W9Gti%fJ$sjr-UB4`%f2#-bsM5S(|qik9292Fk@bK*}ojNgmGY6}+&B
zXUIyU2BP0QZ)~HHEpI&Tiks7&J&gxkphlJo!`TIT*xSM6sF>#X>i4CL1eiy4u6|m%
za<b`E)$a>=&ccS>bFeqeYK+9_moCHnVqRI&Ied52|EXd>9gCE6zBSQNNY5by$_T&F
zDK+(237D;z@JA<n^n<e6)T_OwbbmW5)7Xnq{atPF;Nj<}d1tzbqSB(eKWDc3-<H3v
zgmSSild@tx8YLt3o@zn-oPQj5(^XC)!{M)t{+ST#K!s~P{#`~YTANY6um_%_QPZlw
zj`r&Pmvk?|a%|3t{#5j*+Wp}#njwe24+(`Oy*c@rvkMz=s|Q3yoNs_3X%|`_jo#3B
zj?CFchF%NH*wq?W_+Y%J^aZW(nE$n#7cP9&-EZil`&vn1QqBeE380_o;BCmDac+H6
zedZc`Xn^66eYt-VJ%7I=cwRM2A0<snn4$Z!Z{7KkQ*%HbLAC03&0nf$ZIielWyMJU
ziDHj6^LGJ4*;$0zNKQ`GJ$Vw}3~{!<_n-}*RpCQlFysV2P;h()xuuh*eq#|Iv$ZWV
z+eZ=BIl(2qiYCu+wyPocN61Ne=KT+e0|mD#q!w+qf%M&`g24NP(}o;yTZU0E7!M|T
zd+pwU2R#$AvC$VkJd;@Octqr!l}tbI{_5{+gmO2fu2&YEWF~jS;S9?Bw-|h}-yfQ>
zcKEh_f(P7`l)y$r6h^|MV~ueyTt!#YaLOftr=@J_I=i4LMeX3p8&t8M`dw#y2BC}z
zH_2zBiR5!1@;`U>u0>FUbBgBKIJ;m&D&V#K;o0-&awR6MY4+m{m79T7uYwkOF7cPq
z**HJM!Bl?*>m&>3Q78OneJ%NH$EiSYTmStz-SSjn#*$&Zg$B`&zPxCBhD%Hb*kdmH
zKT_tuGC~>x`(k9QF8O-mH!#tpz;I+8>(a7)dW=Zbg3$q|TnDUW_)pha3q!dq%A}+b
zLa~69bokpsZVrll!PiB#S0@^c3Rl{8_kt;fZ2H7=`Dc<s);d_JUVI{%IgHcExmO*Q
z5*+raXX9(T<2m#;EOP=qprTN&e0uuAL&jymM#`F*W$my3Qd<2=tRxF4Ob9!>J{r;@
zqkH@+Xsza^KWcEa!Q9_e3=WqY5PFc59j}d6R7^skj~*C_OlKENaT7q5ZFghtCYA4g
z(U9-afrp&kTu(kd=#u%&w3yfaqy^^*X+i^TrBXqJC=008McNEk(pyAP(;1u@g#ImH
z5YL}Vhogt}`|(3|2h?)YeUA<u_BO3&0I7q2O+cB;n9IMG?sZ^@dx$A~RafNh<xqw<
z_Yc#n^zR+am$WzeV_$%>DY<ETyfY!#03j@07uSAsDg>WfA6yo+Yw+DbHmp*pFIt0N
zE)_)77#7&4d&i2upHbrx_LEy|<~i3Q$hl2^fD&n@sT!MJr5ea?BUt3w?w})RwF20@
zLFwSl(`5jT7E1prT^5*_H3}lmQ+X*bo!cLBY!{rkv?G5pXvZ{Fa%+3`n_qv9yzI9n
z=PgiDpb4Sij{<yNGbjnC>s7hb4p{F#N6(W52<7`zrQTDapoHNC26b#u0k+8K`@Kr1
znKY$jF(Cv#F0Oq!)&t*K#*R#J`)x6UYMNWVI=E~HMKzY7oc9JwDvLcfEX+r$E4<b_
zWI&1~z2`o1W(gx~;5`epvU}+Fm||P;#k1&^D~o?w1X-3)+Er1HkNWj-or*G46pRnf
zduFb?Bv-PRf5pLN8e&#Vj@SvMk$eIQMJhRJj9Q<pVk20zxY~HV&aiT|I*^o(%xjm^
zR0pYa!D_TtPy9TjD3BITPv*3_x#&R<v10CS@+WzlsXI-D5hz`DI=3C5Tj{`_9Oimh
z0WYXD0m^I&%R>J^)M_~}dc5?jPw0JUzpxJqpPitWfl_R;F<b^z*%$!35QaozOs$Kq
zh7IkAWj=^%6`XXpX2cBp)PVgqbUtWPLX`aj=yxbKZYG`S$&c4hN(`Iq3wnscMSEUU
zD+p^g=Q4%R7pQ3<HH`%g0QgrpBy2`&^IeP7m$;3aB7RP0@3Q^LEPh;dD3jB)ycv(R
z$V_XVw0n>=tUUhyjzp=S>||4<F(?g(iTfUUj(o>H3qK?v9_0%#eMEPC?B5#$h0pG0
z<kr$Cy9L=8n4%nLKF~EmloqB+B`5qTUafgImI9-sEAKkrXftb??9E_3_^CU-G;4~l
zr$A4*f6Ia9EDn$WTgVNDFQB)kj#vP&{7y5ScL7tK&Yu<o3Yi2jy^ELD+P8>y<P|hD
zjAm9PnQTH)R=n3NQt>P(crECFYl?)LN8Vf)O(=UZ_%6zh6a9ivE|9iHRM?Z9U3lyV
z=$M(QR-ovL?HqH1WrYOr;mv(PonWc`Py?$=rm92dZEHEH6|T#zt((6^N(5BdgBrUz
zsK>S>iyHvWkA3ZLYq(W^v7jyi|C!PzflAN%Ga+a6o*UrxS@CIoF!Tc+y|KT<G#${P
zTozroDnM(*@ZzOXakckK_gy?|9%gZ58D_zw6C!dlyp}(o-|hF?S)B|AHTx!`H=<EF
ztL5Zq$6*%$qMC3-Hk$L<*Ea;5r3Y4MvhSf*lGmynE+JtwsEWF`d~-3Xld^LEku^H%
zLP!wG6S;Lyq9H8%lZMeixxLW?VbR<W{Vr}ey0(LX$qX`9ZFeU#ic>MU)gbYLx;jTe
zr@=i?sar3ZJlv`~()!cy@U}`nB4#bOdZh0y0Z5e|+-dTi`N1lu@452lE)*N`v`VDA
z%0X#fy)#oOc|tVdEOv_kj(wWZeHM-QlSu=EgD;nQIgO$GAeUOhyo|)ef|?~t?$;R|
z_M2H<Rr$iTU@s$ihsnp*ib!QRTRoe8)FJ`Y8JU1IyXBwjRm_#BQYbw_hdvDmmGTZb
zk)Dcdh-Mk;MHpK``C+Z@ueCR)$%Uu6t^8Tc8u%@A!&U4iCs~hxS2F(yBBmtincDV%
zvBD8(o=A6J`%84`5)BB%VZc@^^VsmtGEx;l+r$v@{*!1CA)s!akpoxI%A->JL@rjq
z{(V=j2D=|~{#xWhgMQHV%%$-Meh+%gBbc>`5G=szTHW|kJlNjPy$0<mO}T}&OtV^{
zq8{s^AZE+i*?pK}N_&DfX%s|>p8rsOmU+s`jsfCux8HaC`<rNBCvtT@2WcRP5fKrE
z{hW;2Yol^O5HM{g6{*^l>&$ltOYk1fy88hlEBxClGWn5e_hA+O$TO{#1IY?Q`WHd;
z>!@)tH>}Ec)~4h&HDg8uA6k=g0-bG46m}(Y)mm0tcMj^)zHP-%sP(}<+g3&5#h<Xh
zjfppIFM5lO1B@^Mgx%HtTy4n`wAbvmDZ|c;4gh|X+L%2$f7mWW0a|w55u=;vXJ(Se
zz4|X-(g9_yf8lXevBHf{8Y?yQ^%in=>8(VellE_dTSsa<(*dXYK8Z{a^k6~WklrPD
z6X#qc+S-7>f{BX3dCo$i+!zQHon^KLHRR8<7X`hx?VyG-rDwt&GBUDV@5DxUu5DRa
z+O7NsDgTyeq2nYr08PEzSlvXwqYZqt{<g%@Y85}RH;()#8K(o7Acp$FKOn&#yUTGD
zYQG>CuuB;s;p5>^+ud3!jP`D*>8ujJdvGWoB*xuaCA@N!PuMi5;@{j>bl;%sFQ*08
zTL251KqFeZ*UoB`ImJbE)_w#zkvH}k(WeR>is#2`*&_=G@b%R;5O$9BSF`AG_N|Wk
zvVj})4l<%a6(2|HL#kbWfwV9zRHio&u4(xtDqp8;R_{5xZaMX@3<cs1T{(0!hXhIw
z*vMj7#`V)JUlzjL&@BUS;)jNTH$5nl5%H$x0zjnSOBg_L&G{P+*4(|8$YjdeZajXt
ziF-W`X5A&+Pk&c+NML-HJ@<h_XSN>r_?A&MV|q9v-%y6)6qusQz?OJ>dd*OtDnXhA
zq_|w5_)jkGQ~l&e=xwl|OjG!aXPTBST6mRF`#Qmi&uCq19=G7{v+ZnYjy0j?OX<`0
z@yeDE^Jcw9j~}mhI&1_tI!rb-f}ogY*75!|>6eYOKn2x!3GD6Bu7A52*=iCFI%m|x
zzb_0`IC}JF49xJlE)LZWEqvW0b(FcW#S<Rm*i2m4aK{wp$R80JKnCciQsvw*o2Fg^
zUshJ_j|dX{90e(TkD2r|+cB2^pW6KTxb4saD-^^zIS@zbK`R?n_!*rseChc(eA*~A
zWcb<M%f9!jx-aEz@o~b&{(Pa!vLj55!-#%3`Lg~El{g!Oy8r{P?y<GV`t&?u6ewe9
zsNIc4d$uXavmIik?44ZLu0laE1;8yX&rPewc>coRE1Y+<KUvKzvAAxC|4pW%Yl@*X
zt8&~ea7^02L2d1W%V&al&ct&dyj9kD7)YVx_WJtFuXO2<tHov=H=wAbs>g<X<`{~=
z?ipDwqbr2_($qI@++k2kVtIP*BGHEa0^U%auWw@vXX@w`fuWA9Tq5Uf?;tV3Z{RRu
zh!*lTwven{5luq|bu8uLOLbdi`Cp?xd)83r8H;|kW{!HUNq0sk8(IbL03XMGjVBJB
zhGY{fw;QW|9tE}0NvKX=g-OQzkj44=ZpZK^OJRBoUyY!hDbF|SLL535aoGg*VO;9V
zvIhkP>szC~(O^Z5Z)s(otWS4%Oj&63%I%<KNCjkg$l11z<`eKilMR9FR9oU*Y(GNL
z^-uzgWYAG)rE&}5=>AHIijskotoGg`+}FdTF>CI)Or09Al7}wy((KjR3%bJ@3KUwE
zPT4$Geaa@SarCj=du=T;?AM4+7lBlQ^5m}^ooKWsdv9#EI~P*_SO94t^&jy)@)o7#
z5D*Z^9u$bak2x4}K;#Wr&4fhuPdeYj+%;C&F31{a!q#}AQvj<80(fIpV?e=Gew&2#
z*>3rlml<z>cG7UrLf04IA)|q|yJ5(_Ni)Sxg=-fNnE?oBasVAofF9b;9Cc=LL5H-7
zrpRV%>*d0iR>0{FYjRcVKW4KI1r*HP_qGDkh=`7u6^*;VsqpY&WYc46F_CY1jw+&F
zI}6R^f;Kr8kXOEKj%IJ}%2w@fVK8Sy^NbQi;QexAcIb4cTEh@vY7erWg~2lUNG5ft
z&GR~_rY$Xm@_!m4rTsK`^O$6wN|BwwF1oA7UH^IRXp>=D3>T-3jwnnO#udSFC~^yP
z2JdQxK??t7Pr9|3>tX_r^?+JWp#eJ>wYPZyIJvn_Ax1PC@1+C)_*O&2b(&*ha;z6-
z+LrJZG>QI55Okz7sP*CkHBa~P5dQsvXKi~gB%GW+t}hH~*iyg0W4*DD0=*M^(o@su
z5>NSEFdA!%+DYRJ2CRTIFfj0E$sy?0qI>)Ht%oXNg9Q~#&Rtn)(IkV)hvh!Z@BHl|
zA%WE2)vxpMF6&W3Q<`HnKsF=7E+5eoJv0~t;lDH97s}K0y;knlSOIH%zYQ@~GH<R&
zHiS?#w<QXv8`Sx5*^Noy;y;lqZ6^?*JgGRN<iK!>{%KdxTU`e=IB&GnK`{fg2|HtK
zgN~Z=I-0IP<aszq2UsbEoI)W;w=hs@{JZ|S)w6Lw`;^mi!GC2fq~#tPIrx-9YUh0|
zW(u`o*REb=`uzDb4$YV=ti-fE%5~IRqu8{aO+Jd{Ehzf$C1O&Zq9nfw2v)QQ<)U>G
z2IGrY_}iAq44Sqg)~4F!fl6$cP**ri7E8>T!C*<meKnqC`b=K?+xFx@zpJ-KJd-T(
zdC}>aG4MvL!aO`^{_gPw(21oUKvB29`bsTZ<*qIMdl$t?2@F49Y|wpi1In#H<F{-$
zgTlo48}X?7v0R_gi~~*|7;E8jH9F)n;{lzJ23|Mc6!y0{T1oo#HAdk1KRi}7Cx%5K
zuJBq!j7X$`m|JYs&jq4fF#W(?bkPj81~wpx3QHT!f8*jvNlE<xAQMVHiO9*J){u_h
zFFY8iL0OiL74Qe*KP(Gt7<voz!e@SE%0mFN)~^KZpvb^*KAbI1rw(d~NVCSHbmZdp
z&v%lmo$~B63b%pL|ItfKDaCQqgaFR!?p$FrP?B_`@2^4PVBVRwVwanRcG@Bk9WLiR
zi{2Fh7)Td>VUF>SdeMUXqK6wc*Og5?bybWS3Cd3IQ&ZDHHyxMDyr%p9%J<2VcCl~(
zQyA(n&gy5$fU{c9{+-5sGGVt&<iLdC^v`!f+p6-udjR_IpTi{}*lZ){uz<5Lr0^jg
zPy}VwWq`6cJLn-V)x2PiE~)%q&I}jt>({T-3KAZh^NgU+DU{E8;78ykS~_62xccn?
z4uv}SfzD@n&<ykhbS!fjqA(6xT2Uxe1lf;7wD_EX`1JXo@LV}T|D%ZQnapH5fXaM!
zCdsn3OLLwD1js+fBk@XgT!&%4!W*h|mLGASq68N4T4Zm&q!_dVfY1?eS4X}Ct-1YR
z%r??=ky8_sRT%>X<}iRqo2%W|=zI_Np+f<H$Z!wavIE<ez`VTjooa4!3#fKy`$exm
zK{pe0Fw;-`5E4lb>9HUS&#7}9B`$ZIdhK&=4A2#HOiU1i@;;iu<VWb|Eg|_g2*7Rw
z;C*krA!HJGS*}O*aOg}zO-eTp`oND0Er6s~MHf#~D!(VVP+$DgxxAzYo2$+2b3fn7
zzP-^{84YS-C}&tSyFpYGt4<RzR*1jgGHSqoK(T3e8uO91pq9G(x4e`2WUVYTA8bd(
zX1cP&y}&^ZsF`1s7VuO9QDzh&$Cd^-nUOS^g&sYS^c<5cOzL=NsPZ-l=~UO$DxV#5
zN65@$yR^XlFn|Fic*G<ePV;>a{GUD*9p;&p!SIuU^I7##1JPCvRlBQ#)YQOpp;W41
z!tzw^%MZbLwv>VcC#5kYHuhrx2W`@7JT}6h(+{8p(|CLNH`FM9;Zqc--AH(0md3up
z!|>f-kr9n|ZHeIo2@=r}ml)cmL_?M18B~YW$s6(*J^-Et_u|D!2!sKEl3|Jkjth)f
zw5Ea5za1eMO$nq#_Y(vjK<y(SHwY40MqzE0zxst%zl%UsMmX+$Jl&iN7qZBuEY6Jq
zJ#o_H09llIPhRYEo%_E)p;iz!tK|nUdK5_D8iA3efqc2KBE%A{b@gB<((BjaUYtlT
z5*zddrJCYrn_^1K$jAflk#0L$8%85}j+sCH6AcoH1v^i0EfhjNfC^xQ;3EijvLFOu
z(c3Ye!d6X)mAaRDTj%5@%S3+Q(8WD~ekOmsrCn`eF8l%L;xxP8_4=J2aiH4%8Yin&
za-ht%(DVSbrLOJrf}Wt)mp_-DF4yo9*pXNG@YWON21NniTpZ5jM`%{L<VS$BGz<!H
zbfAS{^+m=nX*fbQ9R*YvI32dIn@ZOq;G_mRiX!P9gimMppTskg3=cHX-DSUk4;T&L
zy&C~BkOq7l(dEk%&uiW_0)L?5XW+AQ7l0ua)Apop3RtOSFI)Oud#cm%{`2SvvJ40K
zmmk1mf~Z5rt{(whpy~@kDmIW#E=RHbRD>h!icw&C6CwN~p@p@(_v#em`9e&e3iZeQ
zIgyuLBp_dMpD>c0I2U%ZzH0Z9XD?nfy`~hN0Dh*Y#247r)MU+jfH%kRF~PYa$pC^#
z05R`@P<(fRV;}T=38k7fp_wi+pn#Q#==Uci=d!dPU|r|Pk;y*0*_UrAOMuO!SBU4^
zXejsz8dvO!P?gSeP$$WImrKclju|75`o5{FpsR&xcR-b##O<{07|D*C<@;8n>L^D`
ztfQl&Ghc@v1RFy6Lof5tnF}yBgn;vG1dn;Al+W(W&4aBFfR^t8+Tnu5ds(!b1APXD
z7T6V3CyS**Yy)qyVUq%w?6&~+`~XLT(t)Ok{p79$p>GiZh@37O=oC>pXj3e9TY20V
z&7LfrlY!1U?Vy7^m@I53ehSqKKLn>bW`-(C^7N`6Hbt@KCSz%qm?#*`F&u!n%#WLe
zp=`|*0cc^Za=UlSTg3dIRWF`wY~y86%4fJ7ed1Ba<d2;i$P-@0)r2#uG8~|e2q=YQ
zm6b!;ox^LXkpTB>E(~TSla%`&`5a{Vfqubzz%0&d-$6H6AfWz|TAbw5ZFuAt6LTe&
z*D`8-aaadr#c8IC_!NBi9VUMK{!)_xYAj<mexN_`hmLJcx+!+;_McZ=u8X>jalE-z
z@wdHCZ36-DC&zUhs?#ym{Xiw+!5v9SV_<D6H6G5fJmz5$%o=SPRq}PfGs9q1Udw(y
zE@Pmtks6ptB<ShaOi@RdvyPztn9-ulspHXoZlt)Z&v)lWLu4fAM!f|>Vi-tQhm|~(
z_%BJIe!)QdeM$B1Y_7xMzWamEhQKw74$}cA3+*M!7DS)}m+6_~rWa)41#s*}hv9w<
zHH7ephE6o)4WRKydaD$V!2OS5P&oeE8sA(t>cb4VEP#QQw{E-&xzu_Lp_U1OA2_;V
z>T%qlGiC}i@LG8e@wcz9i!xoL;EPxp|1J+&==SgXE#5#PBjH`Xp;vg!LPsPfeg{xR
z0Ul4T=<&C_E~$aj#`1PRMegn+e%(ISMPH(AaMCOtbb%Fv6C@9g-sdMBtBUW96ma;?
z(EgYoc}2<!g-?_AB-X;})tBFN?ysFXIZ80OQF3{Y<FUuSL)y~AZQZq2#%0>Z0<ymd
z2*mpP`|oQ$FQ7#t)v*;51l}c6lv@w-&2*#&;8Teu)EQ_we=s}dkC93M*Y0-Nb&{XV
zjaXTJ!K5vLvBq;N3iJZXjCjnk9^6L(H?v29eE^i5ofxDFp#Z@{Y0`$`Uog=y5%*O$
zEp{`u6MY_mNKw7<yOt-q^5KHjz(+ZDtdR=G%&x2llI&PM6K`ear`#d<>h=@`#R^cB
z2?MIGRqa+Ja<Kku<fqj@;7Od&fMVL@hYquo>5)QqzCPct+Eph)#-gbV&U>&H)@;U!
zhyQeiBbFWf#1FS6uLAx&fRHNc60J0Ju&t(AhUe7kEx~Boj=E+fCo^@$g>+@<P#C1J
zMg3O01`*w1V}L>!b!IhyAjAlR9cU^yAN-W>wb^glnQ|KoGNUJrVc!NTZ+o6D?;~)c
z=>Y)^C+53ch<ZuFoCZ|h0UTx#wKUn^Uh&|g76XcoEX0Lktu3`mmNE^5E|z~_8Bjdh
zJ%;gPa$sAPW>xu5X10Y;Zk`l!o~0Z0-by@-{tYS;GgFu&<9v!xu-9jWB0GS_LAsSr
za$@_-lIGnxks4b84JS~*@IOls)KSO*Ot+sYx1qzJjR}G!K1XfBOTeQj=jFOVeRSIe
zD8P?G0BTqKSr%o-moH!F0nD!*eB^N&t|`Eg?P277k`r%0r$NoYAT72tK@6Pmn=B6H
zg)~8M?ywQKB8OLFQ0)d~3hx>v2KbpBK-_3x{68RS5Fj_PU*bZiR#64)c9>Q1)Wgu-
z_+b8v0YWQ48D=KyEH;}#YLW1x+Y7=++iy^SYM2C}>?dWCuxml}a{YzFU+x|!b#){$
z7%DMP?!n3Q(xf22-C1I)1o_&mfN?+*Xp2)Y(;M3p0`ZRJ!YoSCpga`BIjMyhF}G#I
z>(OJmPZfaY0xm_D0q8H6_9;J3ZSFT%X4?$T(X5B@g9!NO(W6d4h07(2d3W+VK&<?-
zs-Ljy;&kjU5bdJ1OD$%OkYU2EHsakWgD3M5fQo<R<gO>EG}OWmoaohsN|=A~$w+WW
zSu{DiCsiPIA0vcWxZ8n@h8}d;L&s9HRnlLIJH?CLKu+YjMPgv^-h=)Y0Ca1p1w=m+
zA3p{(qLFMHl;e(liqC<EJ20?ywiwW|tmK1v(q@=~y*b}c8*-I544gM?0?uy|fO;8p
z6?+jhOSlnIA1l{!s4vA@2K5C{Wn&Bj=f22+b4zN4kV^0$MyOP?Ebj*nIy8avOY5m5
z;3V*VCldW^ATS47?bD-S$fqz6tyddwCd&eOL5F!)K`o*gB!3$YeCJ>NIcq69_T>Mk
ztCffh9I~NWms|knlvzRk-_?u+<WT(f#e~}^GWZR$?K|w#Sdi4j{6b9@b6>>)wOfK>
z!Qo8|NJL0v3ASRA2qRR60KL0%z+*oFpm}vRw@|3+YLg@?4nWeLYh>D>7~BH_(#OTV
zpImS+(4F`5{P{?>V|#X!9s|J^N+8KOA3w8^0C*W9s(><2#h!b>4Ev`<hU5oM-eYaL
z!v$Wg2!@z@5q$308Z{png6daisilUX&whE*0XPFIz^I&pB#FNlkFbNfA1%EZDKX*y
zFg%8jfG@Z}NT6`ciF~L<0lP|r!o{O6=NL5#5_|~8ekqAtAH4W=2d)b_TPCe0;F%4e
zpxXU5-2g{Z1xz69%bnC?ea>TpTFxOt-wO!zct1tlK76zt)rVnW<&QWG1pKQ(XUr1_
zrItTq9S8FQR6ujHGrM{$%MCclrY9llq1Jn^X|g%md7J;q4@4u_k&UbJuMo@6z)Eg{
z<Qo%Stp#2mFZ1RT?bv_huP9aM+)V^HXSuUbHkt;ip<L#jbc{Nm@-pKjYj8AG!DDgb
zcL2qLs)V+Nl0y3hk`TD;-LaHT`r`mFg_j^;f5&5eng*Q1iUO=wa}1{u)LGJ&|Ev?S
z4Ea3+a=Bg5xAIRj3KA&ykM@%?3dqW!&Io|JBcbRZA%P^o0!aWPz_aob1hs%xpuw4t
z0S1L50C|BJG7Va!L=JF%s^7l19$RxQ(+9wsK=qny;y$`@Jm&AQKEtr_wk=#|$Y5=t
z;n^vPo*v{cz7SdZUEr5IoCTEE%-;H?kRLzagXk+UeFL@-5WDO^jM2->cG(Z}<9M&i
zFe8?sR&3iV^icGB2pwtfH2X;k_tFP~nF@#@I4%v*$&7)?LUci_kaNxmrW@d8Aghu$
zG&FRunF0q|Ie-x;;6jaC|6;9Bj6kG{kJ1pwa<nB0eEm8K^j^q<28~JJT06m6<p(r|
zpRD?;OXWXOV75bXw<mc3z^^^9{2JR9GftR3)GhJtsm!nA(5G@UCF;m!wo48=ft{^a
z_33^bj~wtA;gxqPC@_6NzwG*!0mcD9X)<6!lqBOGAZY^aZiY<D<;T5=C}9Yj&;XAL
z`8r0Gv_L>PN`vS<k(l_d8%GoB{ji|D;|cU(7EJyj6%A0;|G+DLY0!42OV&YnA8aOs
z3?SuzwsV%R4QvA~c<{BEJuA>*>N|)Zg+L94phg&JtDHNKN-QU!z`g0<?<Kr9!Kwv+
z4d9q88uL3Z(o1b2jnV)!_jY~)wic%x>ONI3jsO7@7wjzqvR>mvA!t*Qj>rI!2<bM6
zbWpdCTE2F-tB|Y`;zJy;if-v=o4^}&gR%0(?tCP~{7=#;0|m;M#wa8vm-PTo4c0PX
zP9Q);bmfYfemy8JmV->_CrM5OK%=$L1~jM2k0L%KfPp7pJUe&nrqZOTg@m*-l#<!N
zNy%`?H486Q(?Urzs9k4dXaY9HZvyNngT~7_;1d9+2>yl#J(v>(_%Ix(pnklP0updM
z>!}miQ7Be8&vw0q8X<rz^>i51fgR|AkW+r$3zFL+a64}7G;IK2{zvRhx+DOc9Mm5G
z^+?=(dAWna57ZNWa)-Qo2;bKO3mNsrTnA|E8gN!lYQb}tB&bQi7G<NxA08`a`!R(D
z!8(uxbhaPP^_OIVvWH|Rpu_uLJP?K*LLz8SRT>3P(t{@%AB+QrM)n=hE4<(37r~ZF
zph{pCn+5>+O+tT*0q}`)T1bFzwyaXKfS;8DODBOs@oRv5h5ml)2EmjG3Z^Ue+{d1-
z3fpCABtIDogna32<#+xN#RiJa!^8Nqk{Etpfi)0^|Mmp$p$2{$^-OOYfB;?S<O9(|
zyyI-p?h~}5P+Qe%hYS@YOk;2YXSGMic830QTfwVU@aR8_UX}+oUw6$#C<K`a#ulYl
zsyW_`Mj~LMJ--1@elP*5jH`=POHC=apVC?O<TX|%L-f}_y)w%lV2nblJ3>e#KUkH=
zFS8CvoTzp<C@2(0V?uwsVigeQKY&2eK_=sqA>n#y8T64zuv2iv0gb*6S?f09CkA{U
z;#88Q<7**jqkqHh|6%Vf!>Vk%bzzuJaRLfc5EMaDq(kWrMG&N0T0}ymTT(?71PmG^
zq`N~J6%Zt(BqXJzyK9fz=UMAp>-hFQ_OJbKy|2$7VBYh(M#nkE4N)vUAt3_Z_`9Xx
zHZANuS$+4j=PvqCAO5n_2geCvCm{BmqFzI%^uI4KVIo9|U;nc69W(&vAk8B}Bh8Je
z7-pjX7HrWV{;SI`Rqwxg0vy~rQXQyFIjxKuAkG}>2jR%VisQq*qYghJhVd<s0Yi!B
zPKaV7qVdo!>*jw0sE^zGAra*)&N0-g*I^xlJffhPYK@aiA8~>A(5b7x@Yx<<BD;t)
z9wQRFB3n-iPmNbehaw6@grD5jK2`$>J!Hp`MHUdfOam$)tiE2-31}P(DpcdXX#r&)
zH$+oO=X+lPObx96(vKc{gbO?<RaI5tLv^}wsOY3X3FU^x4_c&gXny%tF2paOSCX?H
zYT##gVX7o?CG@)3t$JxV0r};#F-WkyTUifDT{`d)g0^)8g9m(^P70u2-?HhEKp!f=
z*VZ+~j?()|&ckWcZRPOcV%Bi62kQ2Jfk~3`ry<x)6F?z(#|1X6yG<ci*dVc#mk*kR
zwpu|Ed~djH08)7S|G-fR2tx*wQcXb~eP>T2Gf+ro-jyt?0M!VjnsDYR<P2M$lNPvO
zrf$P2`0~OaB(FtAXR1sUI<fx^dR|Lg0Ckir`>?}BK8EknHP}7z7SIBF()~yrcNI)Z
zR2UJ0v-d+%gBAV>N~jV=!^yr|upkBSRb_;ecK}(5WRyrZ8bJpkeHtdbn%ng>@S!97
z-Z{|X6gUevpT{srHW6?_7}GTUSEuj^^(*Y8dX?&tP#u~it-#Av=x~FrjaohD34dOT
zG#Fwh<@14WNbr(ZoC?5|3ZU>RCrRz!LYLhwTFA^{pGVrwda(I&fZcfjcpn%7CYKGI
zxCiI_zl9;RjVL6qLo6-8LWwi=%JY!S5O7Z6zt=0E*y@QCm(Jk5<pjd(!L>$O70oca
zRX~cm6Vx9GFjH?(eh|pd2TMXlm$NaD>^;OlEr2_Xd-|>dAi|zV2s-;Bj0hMS1P?Gc
zdv^^(iv=hwcNyNwU-$j)f{KEVZ1Ck&R~#ZDq6nz}E5JsJtAKpCGunIyZovRzDRJ4l
z5*Vt4EH@%x@JG5TV58p8enmWn?-(Kby%IUtlr#lNb9as|4+Vf2r@lvat{A}DGCwz<
z-<squlF1JPpm>#9G7bVPbA%!4?~4%4qPZ{tRRN^r7%Fkhj4U>SXvlp&Gu^Ww-3icn
z{|=NHu{@Y0eqz?dMESZ|ZkX%YzJP;#`Omyx3{^fN_S(As1k+*Klw$V%JDz^N({Ni3
ziCer_TqC5lliHQSg&dXiG$$ZO5buZtJsojEX5j0(%O30DV=#92IW9mBAdZX-zoip%
z+%BRLaWU-wz-jPOgV{(>NrayM<5d3@*`+bDbvvn+sIIM0=O`ZA_a#;$irp#r5hDGw
z0}IOSv97`$VU|60RoO4i=SA7ay!#gNR!H}yb(!Bkxk4PtLAx^&xoj9|Pwm$WHae{%
zv<iYNIv{dDMa5t7_VUZfU`B$v&6P2uULIUz+YeIEfm!7P@7Xy}<P?Ymxwl@hFq=&M
z;YxL_YuxWBvAj9e>QB+Mp?7nUF*DkoZ~knl&#{S@Po`FB%29FlXazlHxO&>xdT-9R
zrX?#x;#ib11nqY^=tXOb`o$%)tJ^a{(<4)<7pWB&dB+&WsSsl%s73$^2|~}UdsiRn
zckBlIE!IL3vOb&OBM*bMX1~9_La?YOe?)5l4>t=Tk#KWYgW7&|ozCso``zCCY3Gho
zt-?>9iaRSAm{Mog{^8t8$Ts$N8q8*UTE83o_?Jy(TU^a`87bq@YF$R7QR}s!5+B3r
zaWNLHP45HqQQC15I7?)K8Z(>)OnX!T;TYnSyczALzMY?gA#qO_R#mHZ`=`IU*{$Gh
zCTR{d{ilcQloLgiR6PB~^3jsOIhysD&E>tZv1TJ{ZY{0cQA-I4ADc5Ux@~q~u3b~;
zf!*~U!@J=oX{zm<;Ba0JCLoR1<0_!1^<PICtJNhrj;;XC00~p2rKJ&QFj9%0)K#*q
zF1}d;gCn9ap7-bNb+_a`jXkG^(05`7UJi5J&hE#b){FLoxS!}b%y{X+q&uhk`h)P|
zrVVtu_mA5{<=KxVY-?Wy21b-Fct;+c>G)&*2T}vg`CiuS6`uS%i_V=ZOw-Nn2h;1*
zv1(<Dr_}aZ&Y1OSi@e`@UBA$8`@Mf`G2%<WNd%OiFBKoO%KM#hxT>WYx1!F&@Z&8l
zx3F-N&O1+)euE^>q^kHU8N(;L8zqHTMRch1HQ1FXO??C%HscPDn^^{{c1kj4g~WB}
z^TRkl7!i=QQ5+PZ$6&{k=1=c(SqWk&go~<GJ*1cvs*NV8*79;+{h@1Goj=f-ZB7lC
z5h-Y(QJeEU)Xp0XqxW~5GZx%-a5zw`wyy!xwGf;M(+rDX6kG^KkpS$NvA}LZ8i_9I
zj`tN}1Z+9Mq(wk;Y5yXs1|q3hNI-?hcqtfbd+p+K^h<AEu2s7ekw1|4B|gYJ9r7Jt
zDnYl~@)2{Ksq!;D6MbBD)NkK@F^r5i;c8;pD;Hy$O2wO;+I>Z6U#m^&b(Uf`YxB>s
zbli%mkK0a-uE!~2GVbUA0>(#u%p3}lwh9i#*uGJLv3NSzSn{P>vo3<=$Tvs7%OF6*
zUg_(>cpmNQe4pMs#@X1)?LMOyEgM^{&6X{#eIg|Z`|JA>JS#)5rw{kCy9;}$bc>uG
z#T~Q}=T~c};t^Z3v*sxLiQ=AnH22Hr)o>-5i9{~{s#>SVVYgd^Nd!B^;#jQhP&J>y
z!}dW7?=b%am#}$diK|;iml;G{uN3>_j~;qiWu5=LE;=sAyu1$r#%VbZg=#RVZxX=q
z(n5tAOvM|>s|TThqmiX?2f&BZP;}YkIb8`pjT6aADITkx&JLFBeL*|fT=FCK#qcLm
zmMWhsbXKn8gN{CLe%g9pwJpufG|$Nl=HK7+(0y^v=02BN`@`(r2NFV7i=sKG>Wd57
z+)s0yD({w_4<Lw(@cF5$p+k1E>o$Yuo_*B3zffdGAU9%8^RxIz?Y-jf)XR)zk5)rI
zG^bbvO9>CFb^gva$bOxn`^qTiP_xrm!(?1J7Vl2l-B&Ba3mmY>LsUdiP%hr{V}(YR
zjz(<eVB|vWfk9`zhTo$)@%*n$KPPM^`>)MKds@l<X;lvY!NV2lRjX(-782&+{Jn*h
zf%hB%!E47ArsCR2MTzAe9*Vw2NitsVvQ2YU9^AgXuEwAdE^rq~E3Yc9_v>82UYPY3
zuRvm5#UC0_fraGm&DdXJGNF)tf!;^^EzLAO?9lXnhhSJ?xzuLe@E$(y2pBt8Fm{zU
z_ws~UHRc(rTxcTa)W^d12n7~!R-WC-P5C0yDOcqx9zl^{{pM5g=ym?SUBlY33T>bK
zj#$~6i?>?@#*VMSs)_e2{af!znu^q&GbGpg(k490k_aZfe_dB2xY%konr2R^mC>sW
zQCD&#1D8?vV69#u0og+D6;3M_YiX&(ds+Im_icro_uo|;qeFvc1LR+vJrR%j{k0`%
z-Ko97!{vUgR^Rmdxur$tCZEZ3Qsd!l0*tQ5u1*8}LYqR-++{LxHW|lRg>%0a!mB=4
zNCY!}4b}EGvKg*mSaIg&U%$NbB7QEDKt*c4rxb6&F<F*3yub#(bBNdH6f3iaLfnG*
z!OOP2lp1~hA<Zmn*DC~By059I5ub|{?W2d09YCfDEpLKRJd{HMCZtSF0ej-sxCjKM
zYIa6@R(Q-G?uF7Eo&Cd#T1G@P!8s=;Fnsajsq(?w6^@5r-6^;&qN_)zSZXemSntI*
z-K|%5*c`uQIcAbZbU12ZsCnjD7G~>io%N@&?o`GL*D@&{v9hZ*w^1ymY@K8+?9BT#
zDIdyLZbjuwtc>v7=8`B&;2n)BK2J&j9iUPA_R*-13GZDk#?URc&)*ah^TjH1P-u1L
zi#QnHUsSgrO}~cmoNpmfOI?3v-^wGg$t_)~|LOSPq_vm*oVe$dEsJ4RHmAp8#!Pqa
z_l<{qJDyj#-J;J@SL#dftkow>mS``=e)iyaQ#h&p-Eph6McZimrC00mD6BTRI3N97
zr0iVJM1oM+y!&gWkmtYhXM>SMV&trnGiJ9Otffz=L=BWqXf7h%XNVMA+-Ua<Ghw^T
z-$*fneQvTQ?Fs0<D)`9I)!0}+cHK*??!o7zvM7q@sJ|r`Y%fwV4zAdyk7XyOOHJYz
z{r+wQUY_PXSHuzJ-J=4_1dV*LTjd9<$M|X`a?4fw#xGtO2#cjh(!^ej>P%Euc+;mg
zwJWPuh3C2=WYna%E1#vs!V=5B8S0ZAe@>^ujp^uUdV`0=6E9vkI)-*NPu|q4aAl`Z
zZN1?n{zGgGtqPgm@1#73Zv$s##LCxl%Suzsg0f(Gl4N_(t1v5MW7}W5snyD~CtToH
zZx}_-vDa;rMZ(^@N_@#Ywxc=g{c9|R%7bIgM5sugpE8%L4_8QR$xzu8DX?V5={sll
zHs0E`JaE(c#NrU+Dj+d#s{%4_?`w_@p%VJ&dSU<zKre6^>NkbbaB<aM+DTfCx1=bA
zh8@z5^LX&G=YciL+qYds4>grDEv<jN<!}|L(brxgI#g<o7M_ZhD1YXyl^Bz%oxFdu
z$mFra$j=t_+P6PA9k&jC^*{U-x!trZH?q$#{j2oJCePu|+&NPT&AH4SnaAR4Cwm0@
z6wceECnwi3X$xj}Zf@N$E0t<=Y~oq|P1DuIUqf27e;lxNw3T*gEnhW3ofQ2u(?Dxp
zd_zvUG4S5IocKr!>P%QUH$If2TLE)6K2^I}2626D?OGfO5e}gjc(VgHV#<xr)bteJ
z%gc>Fcj=_ULUhc&Z~nvnf@6AmQ*@(+;UhdhR(4T)LH%Aw0wMORP=ZWqNM!jh872;{
z1PPJ>yuL&S<xsx(=CIqa^N|5EtiF|_mwtIbC?-TA^YUHZw!cX&&2!yo!o97tnHbgZ
z;ht7@d|x`MVSC9+Vb*5jLH88jNB%gK?>GCpl6Ryjcx=Lna5S+|Ki1|K-fu3~5r?;O
z`5f~PdEbdgt;`!|yZ5lJ&&9bef!}*afC>ux!`>W{Ai$-#uF2u9sbEAZ3=7<9$IAWA
zQvVQ4y1%l|St^yts^af!)w1!qQKDtVC+@X+z)-mE(o7Ej!|TMQ=jHa>a@#_8JQb~P
zr^7n6H*QDeUMspp7b}oTbo;nQfUghme1B)R2kJJfbbhA^$>k92C<1&U=eYMRL)9w!
zhmS&xXGXRYWKTIrAz`~qb>bbtRBsahhFgU-E``v(Ld}zc!Kg9UV3N{P`Fqtur8ifS
zye7Bi2C9y;&N1}b6u|<y?f4<S-HlX(#q{rDKlQwSIZChGJJ@QiN>GM<IJ<`m$<qsb
zRnvvCK`%XQ4Ao@!9Q>i6a~9w5x1yBsB2k8~ON+Ul*}Uw+<_!yH8+kW2_wpm2Laz58
ziCuo0<lXC_94a)7b6k_ipm+v`V~rRBX?pU5yI6}t+$15)ZW$O3+fc3=XdFmjcVWpG
zBS6KVrb`bf)psrLnZA=Mi%y&oOAmrRGtzJ@C;smD%sk~mw2#1s3gy!b1F!PO670$v
zdpn<=2ssQh&7Q=hna4!OKACEZzT=aAneyP?R-D9Eb=|IP(*2~5E@f9%y>C+NdNUl%
z?Q?~`&95Cz6-XR7Y1TQ|nTbt#TJL>I{JjZ-*3~_0k>a=%!x}ye?}5|AXFeA5#rVU1
zqmyqzT%xQ-oGTCGyYqvMVy}J|A4`k$On2g^kus)ITkwJk_-ND*dbFnw@G;;_WFDxx
zEn%F5kKz9I5?g}j`l~h3MEx7cC<LS=af~vx9Fnjgt^OT29~&C-t=HmPIpWUxf*Wr$
z#<_%rZ?9imX70}$iI@#$EKT6pckKy|E*6+KpKly(+?i>|w1m|JfBia;@}z(N#={52
zf?h9cqU9EYul35*9B}+BChmJu+N;AM6yg;{XRx^6w((utI`X#S@yI#1rT!Wjv3xGZ
z015}Oy<7HsmOY6N$W#j~GU7y?RjG6@Q$4}ouIOq{eKb2e+1tmrs2&aB;E{ne51Z|9
z#rtt<1?Oi&IWj`chDDFb!M9xOE<YiG7Gv<}8yPXSA6;K~?^_lmJJ?jl|Nr9^Go-qN
zbawwbSK8fJ4}N3V$i)Z?-wc0}BE>y%q^7ZLa%6Kvqn18uXRz{N!I4D&vK7C4JG(kh
zTd)1Xy!D+(&RcZ?&*;hcISg0ht94FPkB-(zFQynCsTSFu7<upnaY&qfcb202kK?>g
z@7{gJsz2j}YK<S)m$YlPNJd3m>7CtDT~2GsLS#bF6rabZ<FY7vE&4_(kIVJtG=b}?
zo!``xram|7T|Ine>-40Tz4>bhae8hzq-a|{m|vf~D!h7v)zi7=u7o{%idw1UV+57|
z>Ur&cX%?fiQFwxf%XmHA_eHwIN=|QVg~Jw+d+y-WiRp?`a&lIWXvm3!9M##c;v<Vc
zUO)AH$rk8KC|p7NvWh2}nU>WeNNq68n*GkVzyS8sRLVQdw7=Sd;wc6nbj3$sYz^s%
zo5PjHh{c~LLD4I8oTYnncX^5QtF=yemY(b0&mbjL=am!j&e_a^&qiZe*}5yQ)rS_%
zIepp(xw#+SILfW#vE6?6VXr?X_BgKa>vVHh6WeLo5drO{vE#bsw027KSfdSoe(pGl
z!zG$c<%H#kHIYEDv9;A>;mR>H3Rt()z~FdN^pM4+u&2HUh{2UG6^0n|BEI#nGDUoR
zd||tBUvOdh-Q8;EC~ka}xzY`}xggRKT0eihQCZ`0Y#XR|?pMskjYgW|jOJ@ov0OAK
z%Qq=p7x}0G=+bK>7L$BM<a@ghWgVT{h;x68KneHqZR=q%*Od+`3IQ6+V>bbB{|N8h
zPkTk@|4`J)>?Zge(ooQ8*VWeXp33?8RMIq{HLE9>u3vOxUUYJ-Njsv-HR#dsY?wP=
z^4d_gdXq(CbKUj$vi%G}2mSJM%Hj5n@oU^R77p_Pmp7WIDEn*DUy;5jAK)6GEhshI
z@j7DS`C7FVaU5xj(=CeOZ9Dg5(ggkC#bkn4T~A|BWLtkQwsSYQ`RD7CBZ=C+sKUXl
zOd(sFU)lX!y}9<+WREpCCSTAuF-XkHq;_}@d5&&Mu*~`7kDW}~@D%Xnk?=kJUhCy;
zSzA+J{LawL=0Q)|VQ>lr3Rky;_yLs-@#*k4OQ^TRF{`MmCQVH}00fW{v`Vz>?F+9d
z#r^rJl>y_4!_c$0*n8f8{Kci_hI8g#xbb9m!;;wcqS(i`!P=1&p(o<{Sqmpc2h;7r
zD$4>Z>^(1njk1%HCe|04`ZX^`R%acoJUTjYyHWNba_h-PpdiB`4|P+x|83u$pRZ+k
z592poZnmBboqTW?m`qE|nX9i`-eKFwZndr3vZ&G)y8O#_%wEn%q{a3{h~hzW)3MBC
ze;iY3#omiGZ*}az5Sg3eLHYL1=G;l&0*^NvgXd$9OB2(Z{muM_b2}y8U0#)Y4jZ&T
zxbKzDd~X`NrJuh3MN?vpFDKq-o0@D!`Np%JH0Ripqkf-X`8(?Utr+Tj&r1!nahv9R
zbB#~*4n@D|@ksn!5#Ntw6R0~JuJ~STUv5s#52IWq4OnJe+!aWGJj{POL1~caLPln$
zs<(G7_|e{78ma!MI{c`pC<}2{W05eJeldg(IUR-t$7)mM3ECK1@*7KrCOr3F;xpH`
zFFUDM-H@-ZdNF3#x0<(iudmO%oo)Cv8`hS3sJ~jcV2@w>pnOep`b$7)thVrFZ9c<2
zzM1SL%5MkFtUfC4?%}W5HeQVO3cN9BXS+zuxBO&YBiN?8P<yi9%cqaN{*kTJWcj(e
zqwSQB$s(2m?MH2Fgui~p?@%F2|9$CH=~2r_U9*#Y4h@ZntH%05$k2LwyqI2icz6tq
z91{={_Eou#OJJAj$?rkN&4d)WOyql^qUnNkXM1=vPUH4NWKrq;l^1@fL^@Dzo8W%x
z4UBL+g8iH82S?D&4C@(6G=#7t3!JkylkynUVRJwp3mg_<C4m$?63?%R69QLBL3CDj
zc6<IYCv?Qu*c8RWBO}wHzJ7YsImxcKF$Vb)NpNhq_Jznm80}L@aWwdG=zJym4^Sca
zYYWB1ygVhq9%(*&_;7Ls?<m2Y4*~05xOlODXr>y$@(du`5JoqW0Hjlp0G~X1C@q7j
zK-_ENmsCFZVO>3J!VZ7)CJ|J^>hp7aRu-17H*~pU;a$fgDo}!?{Q>RMZ_P;W6mu5}
zgxp+Zw{iT4prYyWGV}w|a~?;tva*JSg{8or#@GfUP^);i->=5WO}UU}h!Rr&iNPyH
zYO+vD{=m&>b*r8aewatW82tB#R#sMRW5;Z)tlfP5`pK}(HcA`Tg}EaYqdY{I8ZTL*
zaMA35nk>P_-lX%zk)ln2(*Wqu@5KweTo~46VPVP2$>{}vnx=|MB2>2Z?3{>@T^2fU
zb_+WG0BM|mwu;<;&Fkl@jvV6#%RCs}W&+ss*<X$O4VopqfN$PA@(!jF>MlG|LBNGS
zkbI*`v>Jg0mcU7@r9Kk^GsSVA)~^m0-GKg&eqv?iy}7wLm|BbW*qTp0JoKyq*kWK{
z;CjW`C1{prBYBFLVGJ}9xq<U9aKjnTOD1x_6UrXAL30v|QTxln2zJ$n57RmlB=UE6
z-P%8hXxrP{OXYpDLGqhtaNaAt?D9|{(}QOW-9LN5&sQ6MSaZX2A1<MQhKTR}=wWhB
zPFpph-JLtH@QA6cTMC6Ra0g>T2>bM2`_Tivq=4!~m&sG<$6*ng<ZR~~BKRR53ZQdZ
zYHIHQy`D&S^Tjs|4%6Kz<7cg)_8=aJq};b70?7?Q7GfP@EI}=R`NEd>dchi$ks{W>
z#wHK0+zYFI+oO4!tLeF~U2AeFRYP{C7{3CcV{D`Pt8t))A5FlWZNsBVPF`nVanaVy
zKYsj31mqGMBcqI_rsj;IBmn_ITkO+;0)2U48#)Ipcu7YL14M!%aOaWhvXxLpM{YuK
z!3=U1FErA;PUuXBIRh)`9D@#4!pk2P)oH^K33>}+^EySm85$8xNF&1Ds`s_8DRAYA
zh%f=<+u5gKuV>!orW1!ztv-MtwZO8aDSgO-%6zuMgd0MCFZk;v5lU7FvY)^sh8#IH
z{QpTng>{Ccar}PZ_xAN!uKjLoZEfv_zH|E+4y^CU1Hjki%4Y>6K!kB}B=)}hP8{eB
zt!K&X6d{-*;f05-Uq@O@L+24BHVG-Irj}OPD^jKy*HhHI7Ahi7R>_f1+rvjfeQ+$G
z?}~!H>)EK+SD^3e20}ExGywP;eRdT?!@QJ~tB;2(R0T$y-c#VfrA8aC^dL|kEIs@o
zR;uVM$Scya<4I0aMK6L$v<P5Pfj->8?Qs|_ZEbJY1kDw8U1pzTt6y7N3w-seP%-!d
z(vBRs2Os-6nIs7EfLNb%0n&Fc725E_n&MO*kYUNhq$DYr8B?>e%7XvFU>onoLs}SY
zATTE5|Jx$RO3}jTBAl8V4pUz}097Jk$WaIA#9e?h0+_|o8a?p@Wq7MX_n)rhcFJck
z3;Ow7ZWn_0GDeEYTQgM9CyPUtb1m+vilK;*0@6S|Gho@Jne1TZDm69L68d8EBUR-f
zK)eti%gD(1FVp5A?F=^rTJnfA-3X2SlL1OCdwLKdCg}88oFGMLAzVQ@J3E8uw03m-
zL|9mb>Wt=*g}BXG_L&o6Np1neiu>I@6(tf#z#+b7s`8*h5KzP7A7+?^d-@LxNl2Xj
z{Gbzqof<QWQUEi*(5ANd7w`E4fFZ_@SpVAO61c@c*|`<4@83YSZYs4tg<Hz{t*<+5
zwAMk#+Y$zn5IYR$E1PDuOk{0^9g^x5&&qE7S9%a5g$=jek`UjEXh7(u$Hh_c>{bD&
z1!jcxH=1a#U%zf>kwA^K{}$kjas_5zf;d5+Skk;RbpE_u6g=|_-tA8q+SLv8<;#~#
z9H!+{)6%lQs6oeG9aI5^nf5lwjsMQmfT72ro;f0BeHAQYqN};`Eanz;F1b;Es9<pD
zGh4xv9UFMlfI`z4Y<TIfzK+a4eSV^%1$u_Y?iM448~uM20ug)-O-&XK4n-u*gUUqm
z)2ArKcu`%*c3fEB-$UAmu88Y>QtJfPC<#o+*%}r$Xo1IqF?ig)iU(1$%s4>AW;A^U
z2M2G&GhMsJ%D`|FVWK=?FJi<iWI+tXic?5IkP%h5Y#svgod)NkfpgiF)B}hm?fG*w
zh3jc=Xi946>*r9~R(ExYsUlMf;}$?T+0fA6fJ8yTcb=D=eS>jB)S#+3-4RX2VF#?=
zXq8~Vz$j#fyn+HBApc}{Zb2i~*Ov`wXK?*?4BUbcSi$jiMg;>p_{|(hL&a#fy{J1q
zHI<Q)a;vbg5IQ6A?xt@5#V@-hRfv505q!iz<lbe(jezlefgfQ8ZiEkX>8FMpU|_77
zz=|4=qObDtX)-Cr{eS_zwCZZnt;NBff9>$NB^+KkG+_n~kAct8{uqG5$5RmCvsxiX
zV3%$nz4n&b4NJh$RIbz>v{61A_SzYVxRP*+4(uof4X3??LmrEC<bvV$o@e1h$ka1o
zq0j~R^)(e;-OQ2_-duRyi`V|9s)<Pw%vw_o&|CkVcQQt>7Nw(9aM(DcXLd-@jf*0J
zpvvh-0W%qHJ_Z`~4b^+ifDV5ksw^+BcKh}Vm?is7hC*B%45h(mh6h!JKogN(|BYoT
zGTcxwpdL=ECBeQ1r(xRe!RWb)>S}!$moEtn3gUxd)2avs10Wv`svYPT8?<T*!k6d;
zx8L@1LktHiTuDud9H?r18jN=vjvcH;soa~11^o}D6yO|~9v}wEg&2qb^v91M+jd@4
zs;&VCckt~(s%+Doz_cWR9Xk7gkqySRuqYVg311@{hs>?9smTf;D}Z{402t*nonQk*
zegRlei9^@W7d(K%Wg@f-5pnnEg%z3;h*2-+UI55dHmXVU5}Q^Iw=JH-@2~hZ$NLMd
z-@mIPfaa0-e_vXK3CE$6%DAd~2HcTd$S=el;h?s`mX$P9WuS1C@sS0|U`zq$*aaYA
zuDY@Os^!Y>>Ug|R8vuVGge(2@+o0nEbPjf$s$4qS6a-8i2n4SpnZI4{hk{^OUQrPc
zP}#k(e1eRed_G*;(B7B{iS9uQ9^gu&n3O|@&cj`1cmLi6U5&73WUxmF6@mW`BLmj-
ze8Hs3D=BGtcvJ!>@9UnU!u&(bYjA*T_{a3)O1K^Nt{vfd;1_s7q>}6(kU>y;(-_7D
zXGVO9X~c8P<K1tA9ltDSXH2QL0U;N{&54V&S21<hWhH6fq2ear_RbD~QFH}StWtit
zkG0E?)YXlKNx2fh%)|6XTQ&V^VKcR(g}^_1Fvj8QQ38tZ4Y3w1^+KcF&ylfnziS{B
z$eDa4<<K4RV%dv})*E=VP1-)t9Lh%~ECAO`r<)-24;#UW&`JGuB2R#gzn+L|fJu4_
z%Je!%LRJ>S+jHm6-K+Kd)ZN|fv_5sG2DU{Z4i?y`-@^b=FJg;s`o5{Z;+R5+9wHg1
zvQ>ETcO}&;+BeG3m2%QChWFyE?|BYYx#m;5jpJ@uA=}(t-3-Y8ijgEO73w*9Ld8rG
zy~t26WRsw)f)9QgnHl%yNuoznF(5cDFy&UT(FIxY4Vldr%s+{wdH(}hTTR2E^Ang=
z>K{EnPPNmKGY89AP>4w#el3Pwt(JWJIM;n66#R)nwVP^7OUsP*)8H}m6yga%T1HM|
z@E67D=5HpN0t2pWn1W*2Ax(^`Sp`?jbA4oSz5Ly~8sLDD7aWn1ky+OJ;?hYF5hB(D
zM356Cz^<i;6T}uo@UPgS7olop(fLV*41!8z9Sy=k=LId`;9c<T9<XHVu{P7`zqW|h
zjt?G?jlU+2>L4Gda@{^e28KU_5MDAg(vq!TLx$y5U0lkL_jXNY*F{H1d(3`ty#TLw
z+8wn(^blZNK)}&B5rZePzK+8p#c8k%5CZ@=ZabcBNq~cGZEgK!U{8-TFi<f;+~Cuv
zPxF95VA@Xyunb}Z2(Tm-u4PUyeCg_iI%bilT0pW@@R)K`3J{4fL=f3I+XLWf-iK|M
zHXhZ(V8l+&nc%+|aljY7sW;e2p+v`k{0Ll}fF#kkJ`{97;@;|6+)hFOq}+w9=3eo`
zU}VJzyqCrbHmS$E;{J1Z133J04AThh4}D+9>Q9&x2%^y55b7RwJvGO;X(g7GX*D)B
z&S?8^8CGY&B)YPtrDb4n@Kz4~zX#BU1N@EN&Vmk#pMkHJ`~_eC_}3~CfZTQJyldXQ
zd#3>f6XZo|F!R$5<BOtQB8q<*=%x;zj&)3bw1Tk<dGb$Pzz=*go=p=qz*hkrbxK}N
z?g#A0ar(|t9#wa07haH6@lgQreuxPc1MY1*xe-IhAB>>S3nxUW5ks25!Ny>$;KyU$
z!OX^{0FZAjef_u_JUo1kGk0ZVWq%bJ21@^Z1Z^)+Ac@>e486z*@}KXn0>gC!2rkZW
zBk?l&^EDMCqkI_5Vi6NF81p%)dDs?}1k&uhGAd@>{J`Wd1JR4jl5)n)nJT1U#mA|L
z|5|Zh1pPcBpe`((=wA<`0Mi?LfSeZ(1_X$Aoe`{8HxF8J`*%qr7Y0k>X41O(#|&u5
z1p{a2CC9CRh}xvTz0g$bcJLyO)RYtph=}IL>gvcj^)n(3Jz5&Bs0#mM$F6`K`%^*3
zMr&*i4-iof7y?7s4G(x5kJBtmdlpq%S_-MP8l+dq7#{%0jWRPcfsN<Bbm{y%U~i6(
zB@ujNs1YQY2&(88SO|MR-H<l7LQk5uGA0LVU3c!>5q6k*8SiuK4qE}_1cvRNF?swe
z#Axpja7dl1_bI`})(b8F<6@s91BHi8RJf2^!du`HAt&_$=uPF;Pd&|h_p*?69q&vX
z|3Q|c<E~1|ji*?`aD*Rl^9&*awByLo+DcoDk^!q7yzRiu%*+z-kKmPg5<26A)BzCa
zVA)&(ZVQpSW;6z0c&2$2GhhI!xJ>#H0E#dFvx0X8n~|U253Tu}*RRhaNJ5^6Z7NDi
zAuvn&(@%5o-=()ZgrJN*O-_Fv10`)c6P6)7`cFr%N8q{Drw!J0AF#n&FvHrqva=8(
zvRq)6d*Pom_>RCC7WvQ5OVO498I!giKxxD-60vAAcLX>u3VLj2z-V4B;5oAZ=DQEi
z*!&&DRayOL=<m#iBzz3y5&78SHIT>hS1K@hG=T6ysV9*fJ%r83&E;D;j)e*jlxw0*
zjOFD6@0*(b8h93oMCXW7aDY<$Eeg1TQdOCTAZf=0)T_>hvSrg_Z}=f=fWQ(XtTc3+
zIL(8thT!04nK<D;CUG20A}U`o&btN<uw448hYNx-Kq+5(N&bu8I0;#+K}JK9&sDoC
zT|VLxi-e~z<wD3d4sqyTe}w4t2gg)4bPQUsIri!<5-gN}NwK#4%!aw8fy~LZwzc&F
zdfpkT$}FItwsl8kFtc`{t5Wda1>`p8X|LLSzdCoy@8@a3W<A#lO}JS-geFstzP^g+
z`GtioKuEJdR4#G2`fSv5Ly<$LKoVLx#drL}{&g;p<>L<{Sw2vtW9{EE2|R(v*Ovob
zd$_K!=PB;B(JtWeCn;cR=fD{z`+t4p5B!IU^^buMHnGe{w5#gy=>(5e$W|oB<^4|1
z;NYB`?9ELD{l^<t7eAb?zMx6jWF%dga<f(PYO0i~WXjbGS8ri2ys3~kdV@DN<1mNS
zxXK9sM@z{|w&Kj$XrxTU?hM;T1P|6!xo)?0bchDfxH5Ib?IsLHJH~r;x=)H}p@1Kl
z=g;e5B|`?R>Ene|to1F!ncz#;pMoc*c~?ZMj>W-6{uL1V!6LU;Ssy8_BgtjOmTu#W
z*;kbc&J&_&u^2x$Eb3H)e+4a;3I)54l3;fj)a7nMb}TD>_7E_^%rFnw(1Mc*-}cWx
zH%6Sd^=y~ppdy5=cO52ae<=^YkfX<;Qz6m0N{AZ8u<I5*hYbI2b0|$Z=ubCH&9i`~
z9Q`;c`0q1(CEwsnUd7x{xq1e5jlp-NqUAEW5f>NBO@@o{OS2-n(A0{C>Ev5b@Jxqt
zpL)a7(=5WmF|CnYw$rT1Hncb><e#xT3X6Ytke&Gg1|5u3J1KSZ)pJQ4G`k#*bmD1B
zGR+$okpW~C7)7xJRACsaCMy_7iBg-97?P`U#(7Ua^8UwzNZ~=&-9IwY9^;RT*!)V9
zZIlAYIxP;}4`R8OD3m>2{i72|n1iACCIj2g(qYD^3p^Dwpjz$k5cU5jZP2f9Zat;-
zyLE%sSPius5?f-T!VWA@$NFL|0v}4!<=i%gS+ibP?Fj1v5~7X)U!}vUk@2Q4>(&0h
z50TU%@s-0yw7-yk<~xR7y|P0|M*kECGw=rIxg;gOa29*$UzUS<^TJznfaL*pXk-<t
zI|aSjzYNFm^F^q8iGmZyuYlfn#Z8CrA^~vQm>Z~zcS_df=T*Dejw^Kg(D->yE#Z9^
zmN`*4FAasiyQXN>PLQniFIlinXj+oz_LE6};03z4p3O(T&za#@WsE?VGR4rqGePyg
z8-p@orJfTc5%(96aT$U~^~R~8PQZqL{ykN|p_j6Ar{}|RQ^F(Kf6_A(V|;z1F_IWw
z^+$>9CNkpymhOcyp?eSv=$t@1H3p#97qy%vO@If;L+V4|7z2fUs^%i+LEw~fgDZg`
zXn7oI<HWDnv2FCgDLaJ65V?d2J6j=6a1|3rD~0-1jEnOxNib)C!=IDFRc>QnioiK0
zJBYO5Avy^_)rIowR+p}#AZq@Ysh$qBtoZ{|oV}IKkLXYU$L5z4qNVr`%HlF$B(HKm
zy|E7UI5>=`)8bnmxXvCvT*rqp>G?I<0{4xF*UupfYBiFiPhS>sVJBnP`GICc)`<U0
z6BK9=t;+qE#Ha}woUyf4E3PndxTX#Yiwd3&9{&8g9_C*&5f1wtd$vaNqyvWmL(QFi
zIuN+{kL-`%#i94(IcNRM*Aqp0qQf9TI`#J#<n-9J`(&1-epu#n93({Mf0Y_$(tZGy
z=U~9R7l5U77+5*M8<*{0^z1*Bni)-t{>_hrYl>VXLcrk^E*9eh=lN21PW2f#eE}>!
z3w-@L1>Pm80$^u<+9-H;2X_{3AOUKI!57zeZwSqZ0`(QZjXs6r*iV&OOpm*HDrX=o
zgA}N97j@$Rw*V71=pZMOgP{_2Q^`410YitZ`V|E_X9{3{g^A{w3*_iFd7M)?#bsw(
zPlK##p>CYPd6>)7YG#1?b4I89u__NRK?wa|`xGs@)siRW@<TKcjrd*IQu7fuH2eT{
z%z#KNj8^LDKZKfLgoC;N!efNHa5Mcq*w0I;MK_Rx4OeDR!okiW2YUhL5&v6-%$tYY
z80+Y&r-7PW$}MWCnA;9=r_fJtOU@|JTKK<aq!T?b#eLp`kbnGkJwc~;Ijc#36Z!{n
zf&FCiJp9<6p0=0<hmKoG@oCUsu@yw!Fhbq#L!H7&mSdoS!)Id)J{OO%KKAqY!{qDj
z>-|Cw7uat2&y|K245c4~#D9L{+ZO4oD#(q)9mu;5HnD&Jmre8_x0^g8z#Ay>$Nc35
zc>JP_<U<Ltb2ZD1y&ycaXYs#z;$QMV^*>+sKV<wL0r?+2`+sCXHc2Yn4$mIdKIZx$
z?(}?lxKgpDCnd$2OI*R9H6n>Ox_eHMy8leHZ<{yH^LknVa(HbZ7RsKaZ}vzqF1j_T
z0l2;bHw1|gFE)hESzYYUqX-##E6Kc%8>tkZs@H$w((r-SL|XnWCtqs&yRPYqjgH*d
zlOK#S+!OU{ju+e-_DTk{dX%F1&!u+0ZL~ig`lR5r(XCmS!0s&H8g@X@EaW`7Igz|M
zKI$c(AV!#aH}baFYh?)x)!0rfO=0}4JF8*c-g{!dR_>fVuwL3-J`>Ypo>Q?i{K#N2
z*P++i$#v(;0P%I)=p!7AADxZ<17hT)h3Db;FF96>KM*(K(CV1U3ey^sp$gEL{D_jz
z#S+yolS^*kKmSOxY8E+B@@NmQ-HK8BR;6ctWTMsM%W7%n#Z_PToJaU$hRW(Db;kOP
zRGRb$sSc`>(hSm+%&bH!-cJs{o9LMfpdf7Yo<D18Ii99_K|%`E>^}NuV<y09tL?NE
zZ{n#MwfwB_LW^!ptGjEa`<tm&qYM^tuM%j*gGcIylh5&fytj`f%wQN>V4jc4wliea
zNeaV$={&ZNQ#x@8vnqKc-AUeJZizRcBlP!n6K=AJZE(3VdfE*};|t$Z&f@eMEqqwO
zDBzf1o+v-3;9F5n!H)a8gHuEM-ozjctIM~3PqcIwu}Wj*RhD<E+mZ34D$B$Bp?M#w
z?v~qp)gcfM7d-EL`0(m9dsH)Sy8$L=SM^HPhOPqKL#1b%5$#Y--c^$!H(gXchryz*
zFr$Wy=?Zoz6-Dztc^?-EHa(7}e{2=Gk=ZSwXiyjIBJ{BNLu#^l_fY1>t2)AG0-b4J
zTiy+wzd6vSi+Dccfe|8RZgAow7{v26-UvH|lPmn0L^3GwQ)mMT?efwoxI!`Ebhwt=
zcECHDb^eQfD%+<V*vM7hZX{Dl?s_>Ge{rFk897=h)1iD^k5@Jv#Pz#^&<_Oqqm+dA
zO?KYQ1|vG|6*ifSrPp-xX1__L6A6h%L`e0v({9GPswN4@t7pGDa!NOEL^ts*aQ?(&
zS4~EF39j?7X?4AGem6)AL4S)LUN{6s1thr9T)4@C{th@e3OLF4bZx{7&@V+`W$D)s
z2F&@06onnWDgT6dUPLp_iOzcPQ}HCCmU7R3V5#&zftkB_`hb>yeOu>rB3pV#*RuoS
zjJX0%jrFC#`P9ODXz^qn+W5UCuE2|2)LRA{r_H{-?X%C7kRDGZHJ>|t``&3{*HZqS
zXyy&RJ&M+SX(jm&s_{1innS63XnyT<`PV6R)rh#hO6RXr$$Hbdbs_X*Uqnn}*Up$u
z^Mj$s?AXbl`lcnP>CMv(zUj$D%&ZJ(di1*W+d3{I*9#1K@%mxBngqYa@kF`=cV_J?
zk2*QzckVvC-^Fh7F`dfZ{aJtL;X>q1r^(M``#VGD3Y$z{#0@fses1w1sPLc*YELIV
zk$Ls*noEwh+uDHT+m5<^TeaI}S6Wch@*Y#EhMfuewS+z&(l%t3yWu)7eSVPd+=`Er
zr-*k6$@+BVNL65ea@K28ek7~c!uU+P;Jq_6VxDq!k4G4OwoR?zj`<9SSX<6<P>p!~
zun%l>>>G+0yfR$zEHA!>c{zMG=wb@|hqdQo8!L{D%wrjApprLuMRZi5UUIXxKS=$y
zKi8OY-AT1}PMvPa{3G#&w#Q#~uImLp)argxsA1C86e9iUah!xls|j6Oy2F%@p=I{f
z%|8<i!qsMxpY`{|hie&Xzc5<WO0E5Fc2tlW%|q;>r2Eo~FeJC3Wkx@!ML$)Olpwj@
zgM$-+<ISscNe2G`4)%5*ft9KWK2{^{1>-}1^E#7)H*e?gqcUj{KMQY8a>fi~l@A=b
z>DzjI>dw9o`=xc;Cbv%c=ze+?&k>U#+)0sYcsU!pJM^TF?7T@iR&3PTFw)A^a-!mA
zB=#G9Q(Ubn8Qp_OhT6*WT;86nd4kSv*^f=>LNc14t+`Kcy2q^AbICjqUU)|p^u(4w
zTfx@VX2wB(WKOM!^Jr(MZk|)-sPgsk@EV&}YEI|3&!Lwc>JCX{Iw+Hu@wRy0c-ISv
z-R|iO>K9%&KSR+O$t?Fi?7+8Cc$V8~LG+?Ir{lq`o}d2Nb8E@69bI4fMf~@S<;;J3
zR0gP@3H?z}`Pjc;NY{kBW<*jr)osa{h+Wuq*CPs+8PE?3h&BJ{GB^{*b^cfNlw#(P
zty<~J9Wk^X`A6L!)2mB~El%^}2^xu>ELq)Ieu7Ot@$&Bmu0I~NUp=9$_8y<&_X^M4
zjMP_R)SxQpwHOOyh1V>=wf3&Aym#i(J)|mjWjQLEmyPKWk#}R&nj~X3dN*`8L^sIM
zu+F~3Y*hZY<*o$Ry@iT&L9Vj(xSc(>!n|un-<xBYKf{cXJu%zII;}KGuEYT1`(G`^
zJ9kzU%!0a_yJ`$_Jo+9nmCrHVdgPqI;nu?@%`HLc_SlTCAcMxsE>%uh?An3)-&lU}
z=gobv$h3*?krbfg<DS<iiZeI`XuHSaI;eV*`mGS!>3%yMlujQ>eNAp5F{%hZs*t8W
z<d$0?R{)U_ar#Aha*|(J*l&hjgC17aJyDrqVv)*PIqMFu1Pc|<aq2Fdqod~x-49d~
zZC4L67h$jU#P?&MbfPUQGF9Lu4|gE{#&GLg8Qy{EfuTw2reR&#S83A%l%7Z-13tO8
z+fs!z*>{zqUY0)t@m~T3T{Ir_SQyJT#^eObMr<4?Cq6kDNiwQEdAmGT!z9|-F)uD}
zUEFtxP;v1Rqt7Rj3)jtjo3|PSu`jU|k0&ngD-`O8l<jaAI@5}@3>13=6`&um6`U0+
z?j9TqeRW@kMxU<wcDC7+_hh#!&+|x&?OSoYW)8w1UpiWkH2dXF?^2sQ5}QpV@bi4#
zS!frdXW!PK_1rV$v?<Z_<^GTRHS}N0HWIuwW4fl_&pwx-BkUGh7gKf(=NH+T#it`$
zCm!Z#+D@U@G<2+ejn`+bI2#gYCp1(0#re{wM{f^I=zbGDDbKYK;dE>uNM%tiy6W^`
z*4rff+s8{mv;JeAjgv#VBlpo8R}_4(5wG>jC_cOh6qku<n`7~=-gR0!Wc<A%*E3<z
zxCOiru`GU=@;i`kCVoYLjXZU>(Vf#g({=R14ujr<8P-jDXI*ZsIQ0|j0~fw_t*^I<
ztWU??)ZN)g$QG~|8_K1(we9&R;9YvFa8&FX+L}{)WI5MjvVRQ!e1bVyVSxS2w_nLF
zTw29EQ|dF9u+oWiiZAaX$tUeHVp@J1X4U8MP9mm7c?cT;qU7cv@YTA3q<$y@XL%{u
z3;RZ~-uejVILWm4;KST=`pUvxKDo8!@9MXNGNfshgRi6rYW*?F^)Cox>(96b101hj
zlEz}c5lPQ)g(MnIOJM2IuvJ$Yea2RrHV9k8_Eqpp#$XbQ=Hn5Sui7UCrPM(Tn)m9M
z(tc$%Z+T}q-%0DOJJSC0l7miYwfCE@*zK;4)CGU;y}{4+VVcqM*<|8RQvGgdZl@JQ
z{9dqPB{vP%<Hu5m>55LWgwuq)JnvPK*YC)VrBi7)S4t=>ac<ZNS)Wm7Z;Ck^q;acK
z?_-Vb1*+m55x&aZYM<Ho8;XauSey^a*73zpXLe|~i1;Wjv!K~#H#($g(<!ez<J2VT
z&^Z{Nw{@%8{(LX=ldI8<hCQ{$<P{^{5|=iy{?!c^jj{$k&e=VKcIl51&Z8fb)aUk3
za`%W%HVM}*OgjzU5K3cmlvZZ@_^WC}|H{Pb8a{^yiP4we4+C_+Tp`aOULHxGTRMYJ
zC>&ojYC8Df?KPuG>Y*7Qh2fjlZ<~9(gc8<isyV)DkZXPy@Q$6|vSQ#=_peVcevtU>
zlj+OTUsjkp=*m`2uYWp<yy?&FtrC4hG*gqtB(sG))@r=;O*8K@{jH61@yB=j!cVM>
z3;FlI*!+I-jk<QQAyms)dE#(nc70u}?Xy)f?vnp%OD6$2^BL>idkboo%@0Cw>Cp0z
z_^+Nt;@bu~2m!HiBu(*wgLpXl_aQ1Np=8H3#veaEH#4VSfUaQnJ}S>*^nDU*N`rnt
ziZPS$?LSuUmoa_B9&feRQ>}U5w{t5mALBB8xz7P!^)2Z_a>MfgMfZ?{Lfil&xo98W
zNTn&dF~b+dk#SFXUN)<u{!vn)>wTip=aOZsmo>6WUnVm$zao`7{_@l!{bQ;BX_}xr
zE4TPvTP6~+`!mfy5kIg0+!N~_Q>+CK7vI@G;LDUsxU^=qRB~E+*Pi^ZT-*)Ce%pK4
z7Ose~8_p9Y3-dQ3Pr<W^o7OJWF<vm-_M$8Kq<m@?9p_)q|6bx^kK+f@8E01AOA67P
z=Y)dvsAS1%-KyHdj_-3wjS}&qi*AoUo)#g=yt_A(*f+z<AikoYqwX&mqFt(Zla6Sk
z?4FT%gAzF2KMxw-e@W9-2xn=3ub*?`cGPe^n8BZ0%;m)4eCSDm+5C*qcmGD4(A4*$
zKi5pz`j!-azH%0KuTjoT^w!?fCu$@Sa2T~PIqzZJNw=0Pwe8udTHcr^ovS6>AbQR5
zEQXFdio?GIQE5pO5<}uQ_TBv(>dJgJ!Cy^@?rk`Wa0*_kQGhO}W%M2Txr@e4{u@W;
zhOmTZX7@>0-pc38nIMrNL-kp{KQTOs^{73rwY&UL>FrV0g5A(3>t_X=_pjeN$A0OU
z+mvqF#~_*@TgUq-O-D0WJ11Lk^WJq$!JB{d<`}L|7k2p)ZOYd$Eo2swsg-Mwk%kbu
zEeXW4wrV{WBB)Nh2BQdu$HvCNmx8$Cty<JMM{f3>G5C5x$0g#o;w@Ew$+A4EYO@eK
zyU%m0y}wPL_>ek}A4k1T)W%o~<q?zRUEpa9mnr!0T_7;CE9-Q6`Hf=rw|cELYdU+)
zS`O9M5{mmWJU$d-2W*$lUTnTjWxmwA_f@IJjbN`J;bKPQE9K+DGKEZ!uMuw66g#(j
zhBT@@-18%%IWLMVLC8uy;w7omzc!taAA3hsW>oBKPM17p&el^$w&&xU$!uv_!nj4}
z*cD46>l)YjAF5^}?Vcf9_@&S5Y0SBjPoKJ0iSZLneWLM2iCTIbV<s=Aw^Kwm|LcP|
z$)4bes?T9(43of?K7Y$znCUNF%!4jN=LhRM;R_|M?aG195AHP!e+&t_YWt1qlr$!6
z*iDWasReJ6Kw|krkKZSscp5R%qk8y=k+!Q7F~&D0R>0YDX%@$PyM}G`#v%0hL@=$H
zGrIh4zALQ1;SqJ8x*)Y(eibWPI3J%sc|&D__c@2ZxZWkjtR8$@a=$%8c)QnK$_&8>
zx4T|!Ud%r!CoSJ@d6b<eH@DWD=0rOP4WM58Bu<(aULFt*mdWfcWqaj(rf;%gceuuZ
zv|VGq$*=$V-S;dXWQ&Y$6D1q4@vD_pcdCW1T=R&VJ(Y#swK}re^gh{+v43!-mjS)L
zx#HJaVrNV3^Wl4&q2r+aS&`ClmQU>6p8_f{ezCQx>|=OJ-4{?oUN*^tH6QiYzrSH1
z4A0PfE6=ttQy4HgAWbWsc#m4?eeUwG7}LhuTKf~$(dRx%7r$5EU3%CwEa5o}9?5t*
zbm56Je{xel16p{;O!v<u--VGc!QZv#TpJIr2fa-WXBk_x`zi*$OT=cb#v$uq?#<$N
zgyd)Myr=)$4_i$@VylzX9~XHbVYFooV8mgmoB2!v)Xy8k>Y(04)tWmi;Ph)XQ)9j{
z0p)!++HkgCe}9x!D4-rM9YrUr725g;-)|9L+xhbP;79xWUY`<X^4G1-XAt<s?asG2
zzWVTD_44zZ{GCY?TfYcrFWi%+ZLae^zVTSSx~B+dYOyi$3UBZHZc>5AZ?^_aM@Y}@
z6<zwD3Qo$}WR_|vii%&#+Fd0$wl(SA_$!f%2A}nKUN4$T5afLRh=og}Vc>W~_<VTo
z)PTp0aQds)<f44N)qYYwSkroe{YJ6w<l^$8VbnE0zUB`zXTEuVVrM$_qcg(vhdvBg
zI=dE3W7bxXKed)?hh@>}sZl<?Z;=*h1b&Oy=U1NX63mOui1G~>1=Ukt%xCTCr`(et
zogU6GiV$Si`k)NN@5;)5MS~yL>2IZn#2UsK_sI63hvMs-fW#Vy>`#|1lBjSXPJOb`
zfY5W<^$a`iq6z-^=<>iB+(ESTAh8`eO83@;Ww3R7<5Qo2d|9^~rZh%$k@}fFON{dD
zG)uwaV_yCs$r+E0b-sWeZ2axR_xDv@?w`=akozq!)ki#Aq<!8mG07506%#rqC6isV
zOwFfsP<LDqoaQ}br2jnxpuSI^sK_MD22Q6d6`P8Fu52!>#AYaQOndaJrNwL6sE$q7
zs@8~oCqaF1FXp2kBh{P9!;X6G>5DW*HwC2{xrEFVU3m%UU$4k3X{$r8mg%zYXirE*
z)!>y=jWr*Y3<$_$rdA$4c;!0RQFp>Z{w_Sj>4SXCv?{T9zMz$f>WG*m#)*k!>PV#;
z{Z#)n^f$r5z98|%UJ@9dA9C$Dg`<P2D0<F_!;Y)Jf}z(z4Sa!Me&la3|NQ>hj6ULV
z51U<w>xHtKM9xo2(g+ks&;OdI;h=ldxi;iejYmt!VEyUq+%BkGY1X$l3c5_8=~NrM
zGNAHQV`9S<cK&!8Cs9zCTY8)RWLR=LKTbUVu0e>ddV6dXZ%Ie`{agu-w+ijw=cOSb
zI(nY2??TIGoSZ(Z!{9F&zL)ym+_GMehLE`Yo|1g8+abkE$QQRTjaK2O^x7Xjxn4ft
za{Og9ckp{@@fYIRjq1l?*l+Hc3p(?KGwOB89^K<bXdW<epXvWMUaOn^@7?){*>Mj^
zQ6Lr`V<0Qx0XbsO-Y_eas+^FmXFtNZ)~qBn@TorYnU9sy`R!|HVUcqad_$ZG!Yg#G
znH|z`>3ELu9ZENQAC*GvpwU}Y|C@Ip^9<MA+nv6Tz1u+zo^r0nkFq)%=P{w>)p!$U
zZVFeWxF7$yV<OdZ#(hb`@RSOhjsYgFx03kEaLIc$7NUoEY&}(zjXw8Pw^tiZUDhZr
zT@}n>jA)u+zg7CR-+}ydj@Jn3qW%m-<j+sN4ZZE$V|Cdcg3#5`4^^MvZ%)dF{%L>c
zoaxd}&N(N{Mfbb9I#A)NS+gAIuNV2uV2?mh<h@4*m64rZG}4J8Z)sNoeludHSZRfj
zo+5XcFB#smB)9D^B(&SVE=1rQ<NJXFsELH&tM%s3c(VATq5;~GRR-fUZ=4=e;`}f3
zW$L#Y0=}vniga9_!@5@E`+0tt=Y3u-wEO5hbKDD!<Cl5ypRkTe>-`3p;k6NB-j$6t
zjZ7J_EU3mcdw7XX+c$S_1>Su1>g(r`Ctea;t4@n@yhN1Wo7z(N)Mp4~2}GTB4LXBQ
zUo-Ss=}ltz?8u%U9CcTB_AlA=lijur(%iPEvDeLTrn$7Gt&1)AYmOw5qC{c7M-aKl
z$w{&-Mqk45x|f7Kzh;odt^7KpXYulJaw#4QGZOa$_lm9?1$w!7eu=Oi1)EQsem%WY
zr;zB{6IL7XG!e2Km3vpcmWCN8P9OX(?D_0|cdGh&nB1#mvey?-?i`9`tq(Y(<J<E}
zRDA-<P07^+ypOINeR7m2jUG1YQN=_8g|*o5)^wN+QBv&U@1XLAgZl+v$AtS*pn07D
z9i%|YHYPf#T_*HrlJOQXs1P!-Kd{xE803mQ-%U(|dN_Y3qZjIKr`Jd*sODX4WyoTF
zlK-lGUe6F1Rj<`$wW^Jo+I^JP#-Ld-;bjgS{%qp>E0Cqalhg2UH;~~yd*GKBw*tw=
z?=&8)GKjOY$M1Ve7`R1|oq4(%uraeUoYa2(K6yTA6#e7N6F;UJY7Rb27k)ovZjE7z
zl(o6J;3-{xvKP7=AVXyRw(${<<Dc3GPowZST&^DDf7Qj5?~U?Fbd@?@^fzNBe3Z$P
zMw<F0UBQu6?NiRfJMOu+%jh+0gd{e)9i=ab6`r^@-e1YXeu%;qS;^?<k$&l7C2dcJ
zS{eH!`)tf|((cli_+Xxy)@$#~=t2(cs0Q|P$6}&4HQrr%t3u^;K$?O5M*Ysca%s)0
z?p4^X;S|f#?6K$_Yusp2XZiakI^RtTh5Phtm<C$zzME<ZbKZNopBsE$eI!uUBhEj=
z^2(sIZ<D}+lD-R(K0i4N7n^!UpJ~UK(3UW{*vYPY!X)*>ApNq#snuiOZSj6Xjh;7K
zEX6dwnGHqv&dnh%Me+?&)J12$EWBn~h6}bbo)>D-FU0&N%m|SPY1QG8oR~Ef#xx}5
zL~}4vxis~wIu;Z<#o<_-f%Ul!XC9W)->0|7)2;+^|J26NK#sgUe(2seLu8+3eN>)d
za^eyqI4{!YWyLLPY)&KgosV4WW%bwti>u)tvx=h^{CqNoD~-dkOoQw4WFEU$MVMHi
zw%eysjkj^3+xQH5#mqJ(lW2p3P;b4ViSAXeOzZ1gZ@#As^Yu+97JccsbenbY-Bsmz
zZ`MBt>h$|v$LD@!{-8|F?EC61-jnc)>E7CEl{`;J07IqPtC-Q6U-4Qwa^^0V>C<qs
z@e5XUOShhUR=?3nZG3&J#{2PH=}uO*>Yv4x<_l6>(u+HKB456!Z;>~;pCvT8r~XX0
zekeuff3f%0QCUV`+aLmx(xr4L9nu|wfOL1afJk=>2+}AmEscnDr!>;t2+~M*e&<Gi
zGw;mzeY0l%o%OEuEdP-Ext|mJ?6c$A*DklY_CPLm)m?9Cn=1-kA*{kz?a%;@?(7!f
zv@SDqy0Cxh+|O;I;;#`kKUVdBrtR?-E;76W!9oM}q^gS|{$2ZZB;<7IS0)%=?Sy_q
zv1W$|>kkzd@Sdw_MZX6ZlIj^BGc1$NdL~M6?))DAa^N2^Yq^=9Br2eRv+l{NvtUth
znHN!@*ul1C)U{sW0tdw|BE8om6!I_*J@)v!K2<V7$#F&XA__f7?;$|}aes0=)(O*}
zJTQLGL}bhV+6)GUAilK-uEhSoS7H_!dA&T_==x^(o1o5t7i_<~3G+AymzA6Da9`{n
z_Ba~%i&Q-_Euv?_D6*HDrJua^{6n3}?;xuApw3gv9FeqOBrlHA?-19%i3W;GF~X@8
zaZc;H+x+YUfpoZM22Esh+l>5oHQ9sK?+i3jk_6&-SV6o!c+#1B<zrRM<h{YpQT8l`
z0D>;H`vdDygg6u@Pezm#W}k~wvls}cD)!Udx)#XfQP_p9bWE*j?3UW%n`!-8hi8Br
zI}{LqGBJt)0(S`#ewS>}9$+@TZ-v+bVBaYhK{><+FMxcN1|6p|W-3i`?K*jXX)^0J
zdT@do!H+Jxb5c>UbV<64prOog{azDMU)-g938+T=U2oEb9>X%3a0DuizcmG#)OrAf
z8wcpuwd_SkZ@q>`ryMv_Wg`RdQ&FqZcM8E-WJ|uAm^hH*@l#UdK3sG$Lj>VAKv_71
zRbYN1HP;G5{=5dHw!#x)N{x`ChhNRv_kts5-?ny=1u7m#0p^x-P*Oqy<$=^u_CSF5
z)oBiZ{8{IY6sXWpU|OY(xkn%UZm1LmY~yr*oo73C*W_oSmi0sYRZyT%DZeSi1=yXp
zK*IoYJ}fLO4$!Hm-_f7+rSM-o48YgJvUT}k!v67b&AbJcQ!WjZol;$8{V!+;c7dyD
z$`X$g8f!hvQoV}TSmOmMlG~~&QKjPgppS}|{vDu!MwnNQWLlxu__o8o2wJ)+fo6PO
zoyf<(0nB-`pF904UY_#}U~0?<Buv_Q%U*9lEx>Un(Zb}*l|YOBWO2}AFw&89D`*2T
z|LHz5WW(KJMMD{#4Fe)g>WQ5NrPx<V3<e+r6I*Ub+BephAG%*BRn$3{Rb;`eoP&Q7
zA}G;=?t9}95_w0sQrg2?5*y0Rvi|UW>Y&lAHe@e{QwBZoQ_7Bk&`@efiNB@?mvO^X
zn?wN*jU~W{HD$qjeFMym6`*uDeKL+t<voBoud&N#&}ey?VvT6fYL#Utif%K=XRKR|
z<-ZWPnbDxtZ{Rt=-86g;$l`|qC%QLv&uHM;&h!d<q%h4e#(psNGK4V$sB?gN!Guk>
ztLli~nl5-7HKCf>!(Wx}_?)39tipwv^r*<SJ@!Ftv+f5&>XM|1!%~CcEh$IAfzqzr
zw_yS7dOxV_i2+g%%4pGt{$D4NPAx}R=YEu{kS+D`X(j-tUxnjr^Z-Kl5g>-%`th_%
zly#2l@`@x+Kwm$9(1C23eGxR%8JVwlrSrKy7g~Afzy=s5oIs1*^n|CcqD6l9rMIGh
zW-d9P7uuC<<aW==s>e1mfWc%2TZ0_R&a1yLnEqW_1{D7{Ij$g#(!oAt)_bPSDxJjK
zIh!7yix!0GcDb(v<%uG<cJ#`F<ah1@$l>{-HGRa^WLjw59>Y8qC%gb_qZ#D)B<%(h
zR&b9R$v;i@n)jWLpKvTDB6MWZFz@&OJg}Up>MJe0KCgG$rTXQo3WJ1)CUhE^1UZ#v
zw1FaLy<XghsM_2&I!OMzhKvm?P;Qm$5llE`1n8*^|5dc*(V&kk+G)dQu(#r1ea-|j
zSCDcMw*7)-pp;Nc)?YM`_HpikIunxrLif2Vv1o>G3MNAhBsK4Jl=li=ARP}NSb*1w
zh=f4{w!>eZW&{89w5L;TAfAgM+e3d_LjIe^R^ZXG+6K0&lidn>o!1s|smh0)hx!>i
zp(d!lK`oyUZ?jbp@3Lp_=Cam9@iKIS46@;UzM>t1ES4GJwmzTlF}Pk<JJ`f&t<!Y%
zr*9ok5cC9m-ha%K64y!_eBpXmUo3khC~I-YBX7x;m#6#CAp4lN%uRQ2)L-HHvYy8W
zr00`|EkmZ>PBoL5`EUhO-HvLpDb~G7BEA|RI(xfF;nHq@GADky-is!kOwXG$m=<&x
z%lH!H1nncYrd$YM4d`{k-L`gz=Mm}b%bv!H{koQ2u%K(|ykxq&xnW7YItp}P7FzYW
zcoxNJ1LgH{x$|T{C$kBDo{!19lF(%YMo%M6jlQ6x|80{)Do6??vaR(>4>H(%28qiv
zM&heHbW)0@gt2|y;cHwLAAvgGbyAY4npEag0hzfq0l8Ykpb;w$F9d*UNDzq*+v4PX
z1k;e!iVWRjLr)FkEqwPD8ZALJLb6#nz{mlx-RrbbPtX#qX$*A!>^BA2<(Clav_Re&
z`FfALbr8V&tEj5RPSDMFC9;-Jpa7=WoQ;jw#VU1C5F!&Wjb|pwzkCTVmx!5il0S?w
z^fEjEXdv>XxO-fMNXgWXGk8?fPr?_vSk<}&TFgpEXqP9!Htn#9+!hp!?Q$Q^Q*Nhj
zP38EgW1hMz7tU=77LJtpYdj~o<rDuzbqVD}beFRSqT?6Hru<Q288#hKTwR!FMGTxt
z<XLE!i9CO$F8D;#raQJLlO!vTPoSk@3`d}9<Xc>HGIv$DNKbxyN%m-S$38S~;#<gV
zGU;6Xo_%DWPV*qmu|tlA-~cbGe*}U{cE4jho#>JQ^MO9&4O=DgV(`c8S8EQ}`|<M}
zzgF~@r@B*71>tM{<@o7l!)gn<Mx>Z0`S#%H;opLM2-9N@DXhITBo8Z(9+kXozIknI
z#}Slt4NK;Bp_91`xy2z)ah}wPw;7nY<10@b&A&8RHz~RvJTZkXvlq;TlfR$C?1v)I
z+@F+Beo1LJJHsxZcDv2`4`>Np06G<pWr`tstdrD6u1`_MJQO~S@4Ene6RN*;>RoK)
z%v<@o(*i&vCn?Z}BO5fC{BT)iJsG(G$`uXGGLr^C)!oY`fZ)ZvaPa0`1VCr2PK=3G
zP%nV|BH_sUe4nB;&k&z%8-TIDG9umhWn{h-1-r;aaSsz5GUmq8*+(n5K`vb-(DBAA
zxTepMPLGGOs;PWV?Mgbp<LB2@5&olaHF?MQbpl1hD@fTwRx%GajN7HNHFJ7nR~0~3
zBAv$2s$7WlOHwf(e4@iQhfp|q*fiMhmO-*SDEcw4uypBVUx7ZQkQ#+zWrj@R2A%kG
zMvC4B+j#}2m1Q-os15;((yfVp-`AOxZ|%>|E*IM=iGsaK@bYjK@~~&G*H@*@3hZ(;
zh_ALfff<X}TAu!$?`O>2Ll5mzA%UhnSH7EBgJaFjeb)N7VVz2A>uH(<hm|11vey+x
z5amG7L-Q3Tsl+O$+t{$4CD;L15-AST?aiJM;rRDYZO4{IrHQ*qKb@pY&jkuJy~;Sr
zbZA|Y*^?az2DWL7>RPj?bYyPpd_I3ix8eJW+E4cUh25)6O=fDDBvGWG24<>~!4UFb
zD4>^u?U@U%A{rb}OajiO0fa3&Qxi@pCjfBIo$-ZyCUDLA{Jr>2s-p^&&)FOv|Isd0
zLejEne*9H-yMK>%5o8))gN%Z)feex^?_2!h<mWm_Zhy?tGcDpQGprO#%<M}Rlb?u+
zs0<#g*O<?5p4oWPi4A-myX`Y-S8?FouNWyvKh9gMy3LXmSNqzjUauWd0D^yJMP-rY
zk|?dAXC<liFMn)T6U0X}1?JZ9pt@&Y&j9w=YTcJ&1D}{Ez6kGhepb~g_3^4#3CF><
z7r9trgs1kRr>IPxQV5CiN)xZ#BtaM{*{=GqA($ZNl7vq}PZ9la!M9|7e?Rx<XbEET
zyhQ9toQGNgh|?SEY!CWvF@0bX(&G5-p?<xgS2%tkW;D!OKl!*fPd4V(^pd<-#-0)B
z{$5cOX#Us&)50Pf0uNa`=&%4a!%rW8Hj5)7+;UnC0|Tx{_`jE}p<jg2+6Lp06XRi3
z;;#1EJpGuPqGZ&v&#5$=IBs15FK1%j%wg<oFWF>XH9xmwZG7I)X~9(NTm2@9|72ZL
zpor*6;lBTrh0)g`qHoq6rG6aWB^X$9O!n8N+j<onHSu7l%igxUFCl@=^3YF3=(~dB
z@{zMFH&s>O6<VQ$=VP9j+PItOassH4(lxzpsT*Bm9#ns{OD|~<a`>?-P!iGYyxO9Y
zy7=MEE)N+!#gGzAM8t3eUzjttdw*NEy9qFriIsyy78tbuI9mx(n=>WZpiW|<`gas1
z<V8tzAm!`hw0_`$plczHnXnC*aDFiU6nsFA{vd1qnzp@?fQLZ@(nU=B{Q<OOKK~3v
z4V*y9r}!nB?aTWG#;F~VjB^1%EALbU4B8vM2nnJ%+ZXR@tysV6H*OTtuU0^7*d<q>
zGe@l*_G}(?qR$Eq$5KI1^fB8iAylP^X6o-G(as@t@wz=IH_kj$3U3*+Jf$T-mE=@@
z9w?&E9rI+L?>VE;u1<ZNKI0eJz0^&<#z`$Cr;;#jQheFz14$6HRXWN~+cIMH=Eb#!
z=R?`QLvuMkI2+#%HCq%jlWTieVe`wXH<yQJ^I-nvS$?Fq8q#mNIXLcqN`1)~xUdG_
zH+O10K>rqgrxN0NCj-HuYo48?5hB1WOfNlxoTArQ0B&heq|aln0Px9E<?~9`0P|Oa
zb8jy_K67E6kMmTw&1B(gmmS=xG%m!nm_yx^LlUTPCy;wHt@Dxv1TdB#a2H(VI9*RJ
z1_m#}uF5EV?n&n#U?IE0D^BCk*h%n_oZW=?Ok8!ov7%G4NhOJF2a!RYsnYz;<z&y0
zFwE{zGmN)0FFsqaXl$L-Yu*hA&ex#$12A0vSMKxAiQ4n-12|DGnH5OCEoX%X^d%%+
zNfdKm-fiovH3y04+m^1@2$4>%_~}A7hl9-RQ|Cu8Y2RD$q3NBAL1=H^tq(k><N&oT
zIP;_u9yqY1f32%l>%8c^3ZFG&ZGkt1w82-nx|Mbdfh|mq8bb|(T}8uQK4rm+0Gr|3
zZdB6DxN(6~NShYyZo|t7CQxQZqBCW`R4^d0IxW~Pj%2#mu#=M@*6U=s5rqAaF3xzh
z)0%u&?+C{}8H8@SIQL#O+s=Raijnx)gV}TWhFevX+!HNro?m(_j->WG)hE>_qso*W
z6uqr-{2w~heVJg3#2XYiXE?Dee?{5f`y7Q25nv=@B!qyts#6$(_=(yK?8%S<%ni)V
zqZn8q+KouvJBT7k6+?{>knyriCR}8FH(#_kmz!e0d*B9lJi92Qlhk7rG?mw1K1hO4
zm#4;WyAD9;a2Sg06||<)qf2`QRY=zjSawmvr}4Q3z8|+?__*`jIuLVbZWKcm$#k&k
z%fcRGZfBEnH9hCX5prp@G3U=Mq0lN8Edmf!Yvf8-YQSjz>h|WAhp)5u<0yI%X19uK
zaMNu3E`z*IjeuORD;{iy_T;l4d7~KY_@YRv{)xvbz?9FYH-Rz4wfqeS>Id*34W^UB
zq=<0?K>9<2m~<O7b}y4iDgkOBmO>K0aDmG_!n3j9w#fz~_2MQ`k_`YgMpF9Mu&@wC
zT2TM)u6{=FijmKEaql5<+{W?1Ce?i7Cs9^zIIy;Nj$w7NUm1u!zH6$MO%RbzdhjnX
z+Ics<r3MCUhmLC0N-45`12sj7T({B2L@jBW?S)&0=+Z~5(DRcH(NF?mp@^BrZWF=^
zQi|xw`Kzz8aI7~I^Bx09qrfVn1BzpvcuF*e2~W%PPwDZ8XPJ(~3+g2Krh{k1&N>3E
zHx+%ib(@Q!=U#S8om!*=n&j|xOKg84p~``{PM8+jk#69$w9rG>V70VP2I(bo23bZr
zh4PSsVVA4$(B!-X3S4|CqWXq{qLk4;d{b1#VUu?3OsDbF6?-gnz@zkc4dgR^cz;UO
zJ7%0rBo%TK>A+?A)<4%!lcwfV@L0dOai3ndTGG03=r>6M{aMj9mY$Gv+F_%5$`qDF
z!})>kfak2e3&-IEhz`{qE-nJV4r6aAPyADHwgj0xuS4EH26yF{{pStphAiDXXh~wD
zMtb?bXYnetcq5rexgsV8b)N>dWa_Ldh32X74jgp0t2b^AMb#TuB3;}4wouKIc$i>)
zRy->`I)jecl^n4v-?2cy&a=03{?x$-ca=G1(th`L-jgF!T)~)?n~TD3Ix(rjyw|HY
znw{Z9hy6#)`-%5QxEns!7PQSiX3ifvSwwv(jIF+-;*b7;V5MqWe|<MEEWc-dEPOl`
zT##6Oj-czs;RDJpFjCOLM^xlJFlh!U_^QshC8-6<!xQx1VQz@Li{Qcjs{*?uShvmC
z`7v_l8Mv`LU2k4t!K$qj=An~NoND%EJwarm*k2d7-uRqm(tg3_Os_JZW<x7=_&AT^
zjdQib9HY^nJTnQwdX3n9p+17iJ(3q@L=CI^q}P)y!R?iJ$xUJp8DjA_i+sU#aW>sn
zoMYUR$R#6TpX{rZwCD>fs4j4!ZaFTZrL#|U!WK1nJFbl+N!>89*tO!Ja>!Az(RzdE
z<Zwq8%c{Xuik{&yT(XVJ(q(fMHqVz{nH@?tjx`%aDwJi5D3=B7`LG+OLDSq@VK2I>
zaiB*7KIO{F7gL`)%4(RG^sjtA%eG!;L5}%$^%0qQ;{8B;7>J+ey2dqVy=k;Rpzq%j
z`Z$$|7(5jyV!_bkVt|fkeS6@{&0_H7cW;Hwy5)eRD_J&xPh3>A?e|=U?=YH)A5Wsw
zu;?aWg$@z2q*a|4d7!yeJK!R(L^(m4dIa_E?$O^unFDc=PAG-m6ta>+3hmK$f$+CV
ze5lxGJnfe@(UTh(SA2oAHG9rL5m&Y;jM%>rbrGE(zYhRNS{F%JU_3!Q_2-~jUho;&
z2uOsPeEoy_U}K<oHd&NhL#~o*<<B|tvdX1J8ST3n?>c~3PUK1fP<4vPoSu}lFFde1
z6lIe}YCol8Iam+skgDHa%e`Uk_MbG~sOIW?5$us!j+sx|2O#Rktr0;?=)8x`A~8o@
zJb~hzX-DhtSqMh6GFc;)r|O>DOoSf;;DE7>K-G-Nwjk%6dczue!4aLFpyLZ-<G#8_
zy-sU5p|871o!zb2!(50pzZcCm#!amW4~S-XM6b}O!;hj$Dj-|`3{-fQJ#f?+;!tFN
z{;8w-m-8%ESQ+t%skCHIy1d+*f*y*#Bi>kpDESV0__D;=5^46}yp?d?atI9^m@)M)
z7#>4<=We$rgr+iqB;fixkf)Q<I-bB`q$j`^MJcgkz&cb6pP5JjaozIQ00jUbwWI5(
zk_x0Ibx5H9X(puR;qU6IBEG*>rFW9(5n$3}?b_X7*$_Yw|9KP1pO0TahRem0VT|1c
z9H;gHB-Y<J(Zd*aEYLPlPDL^N)j{?)csK@KS%U;52sr$qiedm$A6C%Tm(_$js$a<n
z%gc-^TGZ()ChSul{&xOavQbwv34(80IiE-KFcr5fQ|XLB&`t`G0(-YQ7SF5VnYzp_
z4qY{X(>S<!K=S(6Qjh2xl@jl&*FIKk73E#A>L(3H>$$e^BWF14JYPvKh;{1XzUfIJ
zr4QIoqu6@XKX6)ZNpD+@9xOdsVK^xq=8x%*b&$;jNPujI2&-CM&%INUE!B#ur7@RI
zOzr1(l%>xy2qFJ%&&7I^?U|-vSNa?^J5?~U1?84K{)!cC*m!R5K1YQ01eH2Pbcso;
zYqIfpw@U4JOT6FsOoM{jEQcz_2L*hS3m={`>*i(h)fk>mQ5$=4^5RAIcj^jBGTXV{
zG9V!?PZnv12a7oInyy_c0sgL;-SEdGCTwNPD<0~jXXKeR$x_BJ86O@ZqEUnDxYm}K
zROb7`0_zvXBo52~8BU9@aYk^cT(8`|Y5K6LFISGJ<oYHbHtWCpNcaBXo7ekS6M}7u
zIlv?ANcPtL0NP+R(;YCA2veDR9%wQEusm(tNA)+q=F(|4N(|6Pe0v{r78xTQyovF0
zvKy6eo<M52TD0M-aOc+Jb(a6h*motZ6jeFVxJsr$0RqRY3l~&0K@@Uz7Ho9vvuRi|
zJf)e)?{5AXCHptWdne6th&{-xobhDI{_s9_<#<#5GLwO;+6Wz6;T?RwusXztVxNLu
zR#mCwU0UgFGr~vmS4d;6k}{}xDWF;l;diG9KC;#;aSms?`9^z}KJf8{?x1@30<6wC
zFalV91qS9}@+(9%x^EV0N<Ob5K+_>T<ijM6;_W!iSmSeeQj%8xDhW@CbEwMx8!Bot
z<I6+7;7z=+zV;sDTnGLxrE0Bq0q=7b>3F_}X1%HNIfB!@(UWwh=16N)_knDizk%!n
zaZtMVzq(GmX&rVCJgCfw-Q%pPY^D3Ud{${;OJoK*DGcb}V+ni-heHjV$B!T|k(YdF
z79d{QV{<uD>6nm$5mu^>_H167X{(mlbs-6WlP{&T%tIq|$6Csh4`Y3jPQOhrk7^;6
zTnOnTE%dQ>q=?0@J1PS!8z)N1I|yKO5V`+k07Pc}<BELuec6V>UTlSui4yAgJLRaA
z(aV2P$-7%tXSDes5fuXbAUJ<as4nJtp)?dfoguzx!6x6Y^aseLNY6>x9H<0KRzN9!
z2wOy$yy7}yKo1X=cM(F{$NO8yJ$cn|${!3BG_=_^iRv`p=!Y0^@^j$9X7UtV8wFtN
zFN8$UXtu04ui-qC5tVe=B|v!mrlEDa&5>}|Xtuh@(q)g<o@X1ib|a^(BQT~XlgyQ`
zU65;T=#bD0#Ci!&d@M>Ea9ONRj5jHd6-%W}*W(2GLOVoUF7v;A*-PWzL!^g4^tk*&
z<cu#>`k=pD{0JEi179TS4=-c5<CI<Y-qxm2giuBAJCk2>_Yr18w?!f&0}mvBqs72{
zzXy;t{fM@y`@XpIGZmRLsO#`+_2EPU;D$ES`d8orn*BDcXjB64Z{*bVA1;6)jQ!Bg
zk2GUGEq!qYIrvJAfdTDC4eE7e^1S>KJLmncN&`pdgockt->LN;%BmH_l*z^tQdew9
z1^Zr#|4i9>Zq0L$yg}?XF>`zx099p(hHsa?DeHSwCPOnY5<P?Ew0El4Yw!)el6bP~
z*Q*%uj*X}~R3kYVB-2dWeSR#c2!md+370c!>&1tUPw_9x*t=OHD1)3SS=`5g4{xu#
zt&kx~&cfNz=WyVly_5ejRlm?b!Fi_9_{-A1aZ<wgkMoLuVP&q`4?nf#ko?VUp2YUa
z{`D?sAn$@pQfLdtPX<l~X3+)~{NuIbH31wb{%W$VU1^|VG)UDKRjwWYFBj=W<Tla2
z3Xb0(Eo*x2bZOKnO0RvwDeksS_x-y`SNBKf4wpa7OdI`cth390hwE8o`i6H_Ozm%1
zT2u37Q+v1Qv;o+Fi4=cDq-?OzlGJ+E)}SoX+?gzgm(%U6SB|6nNYR7dp7PhyM?l3G
z3_Sdy9etDbF+=>Fh9HrkmXbPj8d97iY`Gmx_A0a7hSji@397#lI*&6M&GFZhhiJ#3
zuB(m8?zk1tNX-S*DTm*eOsSGUmPZBep%R4A51FR~=z_Hxi+)WMfM@CN#xp}f1J;M=
z4?)%1!W$S13On5+(zX{N<(M^2V~@1?ZT!P9-JH%I5RTS_(O#}^n=4=LXbBXuLO~Gm
z^9YP>B*_S?K-*a7Qy=|?iJ~KM1M7z@q+D`X*aROef2Nv8DSvYEI#iusu;swIqH<-z
z#*c{nBv0gz;;%#CrQmoOjz?z-@|b0(M&9&_nX!r3e_|QAZ2qN7&}?bh)h?i<z{)b-
zFU7z{(BlmT$qj-d?*Eh<z<LDp+m6$^P#Z$*yspM)d{9xlL>mLH>erH(ijXVu@q1UE
zx^RmDh21E(%;v2BC;0r*s|5bsQ@&3`iw$|pa_0ogn`Iz&=A5FOM@B$vplrW0ylAgG
zzyAzgf^(C8f?O8hZdZMa13f;h9D#ITn?n@F{+M~%K?WWy9;~~-&jG0^N+q-doTnK8
z{id`z6Sn~(k(&iiKa8oIS77}$3%KfH6xBeiVgKoU4I3DCIHgUb*YP75Lu62A<v7=o
z$r1!cotYPcofrgj`M}%~*`zl!kL<f?Ae-KKSdb`;DTN}#KA->2O+sqqvHlZwyu=k}
zhS})bT8C8WF|Pn2;^FZXWdl*e^ia9Yt?4k*$=4(yiG{N6KJzM!ziYs1TU|En{)*h2
z7|rbG?;s4`vUkzOX0^#MNqu?3BUG&LH$8Q-M>d8DQ29UqR%s(ZRoXyn2tw<N3E=p|
zXZHCrMnM+7`A;Z)K}UAs76BWhmr}>aa4C(qk<(Ws&!m&vzjPO#kb0uM<!y>uxn8QC
z9hw%7RB4(F9i<6%TO$Rf9H25y{bh|_EY%&V>Uw$9Jd5+RFZEtIRG*b#vL&w@By!^6
z65`<2_twa}+n6S}LW_OG56z%RY2Wf+PSC~yCo>rY2Yx|uo)%jAf0}*c34rwh<9e+o
zOw0fhZ!ra|XU|cAc>f&1?u->v5MFA(6@LBl<}@oqFkT*e`=&Ob&NYa-J<V(JvVv7j
z<F?4im_hvKl|6yk&$x)oZ?~d?JLkvG;Gtp^b%ivU0`}~jR-HATZ^%w!C72s2>Dtsn
zxvSUhyv$X_lK+-)mtYJD__W6Nsz&radD<Tnl?I_hpgX6Y=NP(kgpYBRV9TKuWMndu
z<pnF~*54p6Zaw;c;<O%Pv|9C~CSY3LV~&qQJwoFn)1<8P1_-X^<7G2;SOnL3x`+y2
zb)~s?k=i+{*4oaiCr`Neg)9PFuJ=kBE5Z6^-l&Ug2j;2kuLfLRVfu-_U4W}W#t{|=
z^sGlAp8H*?90h9O9beMe+@FTS8NK*>g=o&f(mn-2mehMG=)n80QXpU{rU@%xWYon3
zmiN!p+x&7au)GzunkV^bJ}~S`9Tj;WpXRQa)Tw}ql!x@EbNo%>3UzKf9A+xH>k<P*
zVjp68td@=d)E9K15Xwq<c%(SXf)Ga6ATaMToyZcTVHjET41S^BJ6M7{G<!5SO8Mh&
zDF~2qEZQR~n&El9QM&xogvutlGB!D4^TlSFnR5B@`J8~o!~Q*CeceetEUMZR;<vgr
zq6N6tKY%C1|IKKaeJg<Z_?%S<6r}`q8J|3zj0x&9@x(yIXcMa%$bm=3Da<>tg!>zH
zx3PoE4248X?q0Om{;Oy$Nf7}&7BVXTTL8tf?Q#)It%Q_Je@CI@$uV4F4=8vH6mjud
zw1vgu|M}`8<r9$fdn)L)FM?!}&O|7=nbx!8Z`bOFe7v&rn1+gR0{}Q^Qg4m|m;F}I
zb9!PU-|99L(!o230#_4f^kYeSGvkQdffK#-t&>%q`vJ;)t+Ng>!;Sp2OT*U|rNt-f
zy#+@c-_@<_N<zGdr!sp$1u_y1wN37E9w<uWW9F~1M3l4Uj@Hco-QxTccyx6hYp^$-
zH@1=jOntAqq~2)8|IO3&EpX*O2K%k{w?Y-^-+nhjr#||-3y3K2?yf7wwV@2&Dwc{h
z0VWL}z@+ig{KJ@!$n2AVIJzovarLcBZ(tbZ6c!VOn#f4@XH<^jEB*`WGK(84z13fQ
z&G+3t*qAjh@#l>&6d)Sk!+SA^-(N!NaPhkaRHfyj88Blllck&XwT<jGNP*~qxR{mi
zGKllmtVg9~2L;fK?#-Saexu6nNzs>8^C>n$vu|7p>_pHx@E$gb#v~aZ1<Bal89kxg
z$;FaPQtWgM`SiR!i2+P>(cN1DDy~E1H2R;`By1&6Jyk6N|0((33zYN>(+XfG0F4ZU
zcW8SDK?T5(<%0=HWb->rV4y(Z{N|2v>Ag)loEc#QRmN(<Ca%b%+t7iq6$aCb-*2gb
zQ<zx5*;d5(=85aY7_W!jS=x6PQ;fP>_%u?lUnMF|8l~_6&UB&K!GFeaSL+e*jW*sL
z18@6f)t1S2iwK;8T9elyF(?CnKf<6MF)*4qKWhBoQyzw+-lsfysVBHtLmFOZ+oiUG
ze}%NzwK+oq2r2%(wT?^w??O?9kb@c|XuzC7M0s;A1qFnKi$Y&>K>=Yj!Y67_K$!mT
zo%F(n{wV^K4`_v;E11_KR(T;0{J02+BT|8;GPVeOJn<BYymIOwRE9z-bXgP)A`qJT
zTQo1izo?bZ4-jygn2>Re0`vbeU_03uI0KM#ffV<!3_HXBhd3zoV_|<K)N@`2%8ZFT
zhHn1nD2V_4WB<L3m~{UwApiTs$p1gi8N>=lea{LPD(G`Pm5leY7|MAd`RlhV0Zs)N
zq{JWymyQYB@4%f&4BgtIK<C}s8AUh%B{TI26tR0&@mtBkQ((G2xPLwc1|fjjmK>Mm
z4YDB&W|ds<yMNqLes6&e-;fA_Z;J(#AkS@haEX8wCj|JWX7Y-aR7}9*hK#nq?^P!F
zA#+NgQj@gd%ve8yAEC0OVu4^bA#h-9_I~eWL|QOlAH$qKKvsf49pD21Q-Ff$^^Yv=
zQtaotNa2+)LBk9Ra=!!-qzoz<=(|X)4}ifs2h)9Das|F4`U>X853@~jf&vhkJ<mRm
zsntSSdS)0OFT#8OMjlwVQ=%?HUVi|VgUVZ`kut~(tMFgAhXangA&^?>8iU`*e(rpf
zVFw0GP96l4cBA~AQVIGo==V^e#DF+^W;8Y+Ig5qClX_3~paH(Eq+ih(`umU!%%D#r
zyuIN4+kwhv0hd5q^&dqCZ?Gs^E5SufqA)BE!Z@B5-shDcV78PC!)S(}K$i;m;6r9Y
z2O;kG!Dp~S%}NwmamZl!s}th$(C|e*gW=5R+Tq?W4?xI?0H2$`f^91wiaFuq5toyI
z(Y%@boqxa4m}xgP$k6wowQPdc(sp*XaScLXB4*sagVBer!MgvO4?IXE8GQcI`F9lL
z86d7B^2}eM9t+T1RO3r}bAnMok;Qu$+X(cu(@~bgzo(Zz0q;xbU@1EY5R6sWtz=o<
z;<U)L_^OB8gDcf3{J@dZ#y<JGgfRXs!2ZMJVb?W610-a{!;$$x^002m8}K20_rwX0
zke8;(-bcKr{3m1pFAko68vTlT2dw#{dTKF$Gt3ql5d^jD`}eO=gZJ0YBIZDJG|(pr
z%rshx!18kvSVQ(|^j`DFU{xN8MTFl^mIDl*TTUf|{NL~mdB7e$l8k&LCkhr(iV(H~
zi3vO}+DBFkjX;DJ35*A2tNR{Y3*PYa1z2>)3OH#a0U)@Y?2?hxPyG`7kcl3O-@|Qt
z!21&(J?{C>8TkOFYq;F`VmKRA3en=RU}F$~=?2QkWTrqPz!U%?i5T@v`Tonm;k5%Y
zAW~ELxR;0#tWSVA_GTa_Bx7|Kif+&ZLvVo1uLPnI{6~hLK{EWo{N=M{5kQ?nobd~L
z8w;$zhfOFU^oYnCFrcUi%z^(5$Qc^Ye3f{=7Y@L@e#RJ>2?V0RW%*k5eu^nzzI*U+
zpZ~Kw>_FtFzP2jqA%f|m<4%2N2cy}UyM!zmKRvKnwv+LajnLMCj~~(v%-lm4h(+)l
zQ<O(N*r=9=h%WaFj}OU>IQy56|K6M-1r}0{k>@oWn5WH9cfRosIC~+fSjP89a~zDr
zwh~|58~QRN!M<Q1=YuG$#gMR9&frvjpaLUVMnX@xpXOs6aF)V}FvJo6$w4ZV8%D-V
z2xbBUH=1>#b$brP7*jU+6)N<?wg7OLehtghKKplAyCIA3QQ<3gVo1({)kUA=067-n
zGd8_Ho4Hg#&Nh?bAO72@2S1>7e{5xH$_B<SH$hG-{~G8TOv2r6Jm~$RQZT^wACX-6
zH(g9^NO!~wJ7AE2@no7rrO3Smx&Vh0Aruxm`v_Pq<%{G)-3a%yg@Km^stg8)3nkPP
z2|WMaP=eZ=2FTg84$|s9@zN4B-*ed*%KzvZ$lz?t$x$3I1pE5<P1Y+GMrgkGbd!*-
zV+Mk@-4-17Z>z|^Lg#RzKe%lhkYOq!3g}vq0KwGEV^YF_<ZM3ytk3O7QGvgw3iCTD
zc>mYQ-250{pvx}1kV_fmAzjA(yzssdju33`q~ANS|HusvBsbZaN^%%rc?6@E{74{G
z)}=0)3I?zv4-{qn;)8tQzj`<olF9XnYS(!R_?h|}rK>xz^f1)`wg9fAC@_#yYsaz2
z|5gSpF8ID^#{3?jHNtE$#N_^;o+IagHzz@sYx=ttZE-D#EZq#efS%<|4BNl;f`sh;
zGnEGP#Oxw040IaF!CF6n6?6JKX-BLUm=@hX{v+VYL{MD{*buFwu(czA!c(_a8UuNF
z5iZCz(1h=OE0}#xC`B&v7-oqYQbhhONZrIR@aov@2oL^)Q-G!e+2?-0M9KAGPwU*3
zbigm&Ku*+g!ay@FK(<BU7(?>xCk=TbmCT^z`xwp+0~)G`;ond_!UTQ@8YD9BApq0!
z!h3)FKOkoi5>vNC9TO7<Upr=hi2@!w{mWiBq<@BO1rfu(|Le2){}*^S1bT|A5)nKG
zkT4Npv23suhAe;mra#UsnN{E|Mz8Ljd*c6!sQ&LC3vc`HW&HmjAQO~!t2@XuwH}$~
zIfY(R{(!7E-yI$CkJ#Tpp!Y7-7%*<gW&g%f;i0I?|At<(RU01tjg}Zyvc~w=v10>U
zrL628j=^6U-v1JcgWiG3kkd?e?=(jn-aE}W%<zWDn9NLgbq)~k8J|b@^c!$XAbE$)
z)!&ejOrNpq3A%=cYJ3yxQO-s8&NQZLErqM!7%6C`?JIYuTbE(!!sc&~zw<u9zNz@l
zAek9OHYdlg&V+ddNY6CiG*Uo?P(u^d`1C**V4~ljvqIs&o;NVFW#eH$0PiWOpMcMC
zh{JEDM-1?v=3YXe6ZmJLMzGNR(1dl)yI3uctQ3v49jTa`G8o*XSQ?u#U!=9!xq5W$
zbEc$v@<qP19er=qlbbP*wb=TDG`OwysvfCtnnCob2t{$x=#XZ61!XeG3%W^rjA54i
z+xOu>R-U+a@KL`patqv#Bya7}4*|^~hc0&>aE+>U?*_|OvRUDO9`6|ZAB9co@c)5Q
zmzqd=0FvySHqA<_xNg_UZ+euAS>j>J!RRFUpms_5NbkRvG~xQEY|jE7WRJ6X*POL_
z^M1mZJ<uKPWs|PK0N}c<1CLj~C4tzic@~3vG2p~QaWg>$j51)K4TxvCeDFf-$$B6;
zeCLx?3Ib6i4tPgRGs=7V)vs?K;&NfahGlt-3Qv|O4u@9e4{eXb*5ca#h;C^u-rgA7
zYxam0@SaFfEz=f^TKVO4ud>kZ<B^_1k73*_Oy09gV8Fb+yW(ad9u4zJ6nvoxTlk0^
zzVNB&o#lUl`-@OgQ3d&ByDP7^g8mE2pgSt3%H~+XCrl(xg)M<@cZUV-Oi&<&i-(uv
zEPcjm{_|@kY?uS+@S&aOL`KABigdCuEXJN15`vWc-00m}tq4XI876J%QhT%K6@DsD
zh2&G0+jBCGFY8^#{Kh|ZJ@{0(B&naI^~hPW3$IK~&MaBhbz$t}g3^s_fsVcr#~>M{
zcYz9);?D-ceD{~5>qYFPW1kktH1Jp7W`1h^gVR+ebcUVd#n-qZu<m%bOHwM#mBn@A
zxkBGKeU3waq7ElXoHmhi(;*spV`wW-<C=heU7;>3)8u-_W!7_(^*wPgafLTs>mcge
zRrS~0K8wgOGL{ju{ue>|Mbc6#*@x~V=^SNbc9(;2$BT>H8%YF*Du-j1o5DGAQM)%u
z@}vu5bfae31Yyn8Tg29xSTF7)W;!VT*Bt^?zBGExZt)Pn?0WwfC*p{C4|63&Oid{~
z0+(0B?e!{6;S}w=<Wn?x8i^tV$upZgM@fv)#YtwBkH6_+5v?BdCY6L3V0aZD+cCa{
zZ`zrC&-F8P$KKvwgF$IB$yzWnc=Y~^^W2~Dy$cagLeu%kBAggzN4q?v>B&&!Hv>MJ
z!hryeCSe{CB;3cc2VJ02r>eV1zp+k%6x8QsfijMBP|PF+{!hLCEUISF=gMY?X?2S2
zo5O0aP#8a8<F*=F`au4Ry)2yukc2DPHeKWnPJwEs%=c(SmrxxPs45!)ot}Ojm4S*T
zB~Wta-FdM7^zy;2B8pGSsP9f*8YgO3vR$QlYJpL8O!C3*^%;wJrTb^}yN1?PFEG;r
z^X*;Lay8e4m4+XCN%l4-j9QCUq*8m{Ty4(=j!FA1kJ_GyzS^m_3x3RBfW@cQEv_+;
zrs~Qm%)aj7|Gwl#<tSw#p!s^Aq<?3=m%@}UPh+e2QdQW2w$yHm)aS+?Pi}MMB~rq`
zx+eS^uGfTdg=@2I6Wu;7(bTp>wzJ&IMOX%-m8cdWXs_LpFgc&TN{l8v3(XDPt5(AJ
z#SRZy9&et(AGXacK6<0z$^Fgl;^gxzhkk?G@fN>azW#dau~X;Q7dC6CG?7EoRETi~
zq92u7Ckx>@ZJw^B`n{pBOLrh+dOV&waP2<y1@oF=oIm|EQ<|*e<WeT7OZ+27;7Ub{
zNHp<ud0gbq^jeK+SlRjRG*vK?Xa=0rEAHnoNN7HQyC|Z5oaup^U1tDCmFnv`_Rutz
zLzGJIXXSnmDW1PQ3lA?cMTfa&VhK}WRK7p~l+ig%I#uVP*!#8gYK22Hl}*0w3@W*g
zY-Ue?H<H;=6B2ws@21QyQ7ki+Vhx(mND&zv&sPd?yR0lxY;dz*rW<K}1gZ|RR{IjJ
zWIyI1?@i;FD6O#@GlG)4kcZLPqj$9LQ5@g9;7?di;<c_<lYD+DFTUud!)o|yv%B&u
z8~U11x#v<2Os9|X=hN%&vCA~CL^>9{f=6gg%&ORaH*q6YF*EAj2%gDUI+&WwRK2NU
z!<T+_lW@KKeQAa5(FM6Va;Qft0-dojMuN2k#xAY7^FgbPOf%NH!zzokR{G$2NY{G;
zPHmsHDU1mOosQC1o%kOVp2>^4%QVOru$yM~<wYUG;jCq<nOFJkoV}GYw)W&=Hf9h(
zNA%IAN-6&?Zo22LqX(~W#uhA!WE%6aFNM#Wn!d2YrY#S9_%!Y)Wbbpk)Thx6IU3!s
zob!F*-nI8~Km}j+aA`xGlcZOueq})ASnU8t;7f=TL1fo;*1p5V)|lm1Xhzk>`RR$}
z`q`p#sdQz*O<>0hIie5lG~_Bsx`K`z{Uj3|6tsB*(MgB63=FcX3Zx6$Ztk399I3tI
z*1L_-2x-WY*5rb2V+Mp2@LQRW8?0GjkVyMPl;1rAoidD<XIioS1SM{3_E8IO?LlXy
zX7^Rfz?K$UyiE;Ybf4>-L%iYrpSisy6h1yho^SG=wMY?B0F>|X-wc{}kIG(tz#aze
z9^0cXgs2eg*u>sHa$4=hOV@a+`ZA*4bx-;F{J^^t^|$HT&!kO2gfGM*l@anhg~b<v
z!L&a)4s_YvW9R)wv)*?yfdjluF&3W5PBX^uJasJI+)d4x9d+I(aE%$^>EquGO}VvM
zofCPw1|;Y5@Q=nuNjXta!pheTn3)2<2V?A_qKe}W=+j=JoTM~rA%$Bd@1{%ep&EUm
zaLH|xkvYIHyCK-ijkRRw32k3qAJ`*MoR199?fsc$RHDriwmBrguv5-5sT9hw+3qY*
z>^>D77fV;|N#|Xa!r?S-RDxbBN&)Xc(Kq%wUlU=#Va&3t@PZ>#Ec|<4d0_>-F(U4p
zSI<B~#}JhwT5WfC9CfJjD34C7kmLAZC`q+uPDk`WUFYrKWlnEJ8(o{ggnI<+Nkf`K
zIe<Uw|EMxsR9yJD`oUMP)G8V<y*xP?%Ejwou^z;ymtsQjeKGi(ZmGp>Z@}jAHY~>J
zXiMB`#G+xvEuVY&@DRf}>Et&Cg(61dl0n!5k^Q90Pm};&lX?6!QV%c!jPaPG0zwTe
zN5Jp{Hd72#Y-W?gd$1j>BS9MVBM}X=i9dRY1%W83&f-C==6cJ-Q7Zj!e0}d(3169K
zg%i#~OjQ{4+dC1j_0_^ZC!c<P#a*I{*=GFFl<xIXnK$livZn-!KSLf`-UXEq&O6OT
z+v>1NG(q}o`=t^C*P-<j=`eN@8kR4Vwqs5oDlKl<SSyn;PLrDDO50dg9Q5argD}sP
z3l2l=E&{~hf0uoCd!oeFNUPpwWS~4fBR5XDJZcDePgR(((+zI@2};WZKlB}|Ypy^3
zQqU=jKJVIP5`ATg#pIq`5lh!~6svGhnDAW5xc_Mi!K+mRWlaGOC03?yvj;uKeb=Z&
z3SGY}dlq0zdcNOv?RALe6vPJcYsH*dFoAwhWFd4#U9pIx9iJpOtIEh3Say0*1JbUx
zxs7@{9d9rEiNChMefZ*8J<MtLr$dMP8=d)z7H;)13++1--V)~q=OeyS@!Yy4hR-@j
zw=o7)*esNvSEn0P^L05_Cl&UZMIH2#hPBL6h4jC$tjmbEFINT-C16|<`bXgn+UwD;
zWBqBh7JH=cE~2Rd-y2Q#y#a=)M5X8Kg~X~4eYPyonTzlNpL~M+O-`bX1+_j-z_yoi
z;_oim4?Sts(GJ2M-wv>UTKfA>4D;Sb^sRTeF7?)aiq%Y_9L&986fJ6w=k0P{FfWQ*
zba|u7_n4ra^2c=Sc1_yB^*Nr<H&Jrmb@{s{gW7a-y41KZW3=d>wMQWj-%aZZ;l6+h
z1}^{E1MOx|`q5e&wr24eChYu&M3cpTg}Ma)bD{1HHRfw!zplpg_@1XDXVivr8$*6B
zU$sRlXO5kotn(3U6%7h!o|My|^tDR%K7K#BVasJlA6G)QQq#`+?W!t9bg9PW<n`oM
zQ2R;*MG<-}LC5xO=Y>i~@;iYzx0xniy_cL#A(IPNHFgelB}jQ>Xo;eDgcW1547j4?
z)eowJ9G4D#V)y7M7OEbz1?c;pQLT?oae=l0K7Ok~BKWd1jUUAm&&#H*KMy;?TJh<_
zMHbhe`>&EtHB?WxSSGKPZ6zb^PS)&2&APn!+2blaa8*CaoXg@-MULN>BF5EGuaHOA
z$4Nl9>ee8yt=i6S{I>bXd5bIT?G8!EB*XRok<^xvUV6g<W<o-cz*!6R2ve}H`1BF3
zld$*|cS&vxQz4zT1a7uM)|$$3-X2Qk2(R%s*pB?JuMy1Kn)&Nmw8q;4`249qWI|u{
zZ5!@ihPOK5A6^ZReRwE=o*te`ww+Z%nKhdpag;l>Ly;OXx#Ni5z$JEtEW_dZ%$VZy
zLz`0{or4>s<f#Ke3T0k{L2K{*lYuse=eZC0*0B<7^b>vuYpray-gq)69M$0nFhvkx
zt`%JF`uoPL&`s?w3gU~_(^H@ApiOB&g+tpy^4kltNA*8ON!-vr9ij=H(#c-YtLP5)
zcp<k}%KGagQao2QZjB<_0H8Ji@|&8aAZB_nXZ?_3C^dxlw10mmxEyoP^TB|qK+?G*
z{0)pRZBGQwYhlB?wY@m|8J#S}XN3(iAMhB2&h`_QIbzTZ^}i{Kg)7Vt9tlN(#*B<l
z!wKD*hql$|94^lqo>)IuMp8|U{Vs+{ns7;^n#t~6rOV;5V&C7#7e7chfnvJXwO}5e
zB#!hOg@Ywg_3I}j3dM3>G=5s3pxr6<_fOF{Dx}1RH5-J;N=6A@8QMxP((V)RS5v+!
zRU`GxGDE~-^f*vmX=Oln`}*O#&%jf@EX^)^o{SK^h6xy*!&2uI!l}BDyf~|e`T>T9
zKg!xiWv5N~e3L@kP{IkU%&gh);jWm^r^Sf*znev1K5#FF<>EJgm>%@<);U10^vDB|
zgxY7_F;lttu-)a&!v448n{cJjN^Mf3%?{M8L(%VVIy(KC3K4v&C&l(5Lco1v8CI9U
zfpE8*55jm1vd%ANXKb#N{#1GEg|;@UKeAGlXH@N5Ca7i#dnrJxQnG^jjt+i6xq<Q9
zXHNO+EKbXHwW!M)Tt$=hp}}p-m*w=!2^6R5{my3P9$Zh>T~;d*+*$QpgE;{a>+<xC
zr)wY_Hok*<=9~d{aW>=<)p}!dMoBl0x5Sh$So%i>F4vTr&aGFN>I&O)`nuNWxhu_l
zx$?o^b~s#ay}{o-t|8N6Or6fFtl6z?%L&OdbkuW05%#5->T|X2s7Tvmx1~^|h;E(~
zNKJwBRd7R7y4@|sqteOOSH?$Pb9qvAim|-;NCE+3PT0DS2{o?cY}_;dY4N2aOUl5k
zA0BZ*$oI7=?!NM7*S!S}yQNQ0m<F7Ht7+Meq+y#euzvF9?9Xh%<=I|I(#t&g)YoJA
zO3CZTq`w7tZHtFMBOTBqdxS6N>M^_dgmxyVEeHJwi-G@H3c7A9f+jZ{qaQ*-GLUho
zU#FoSlV3jY$42$J7P)#LkakmO5NPo57hUc=-Ve0m7Vk6tN)%K$G@(InPnVR|0ZC(2
z32>>q7?odt=s4H6CafrNh_uuT(`@gHss5h0`xP%?Qrh&UC1DVd=WGAo@s>3gZaEaq
zID&iAb}GY+>T~I+Q;Ih_fA^LS-$|kV<zglyE?=g1_)^=qiRsn4YRnBne@3K(cVBjT
zKl|!ks9uPRRwf#w*1U+&Ctw~oD4G2!_R4gd4}Wq>+kWiQgjt1Fjm2NVqF2JCI^{rp
zj)!D@GnRPnA-2A5d|o_gB+9rG-T~3rl9@1w>+7oNRqy-$nMa(%?YQYWkQ$p}ARG=s
z)7c4a?06qqYS{J8crW8!W57e)(oAE5ucbz|QvtM(WsC_pr=%FMB3P}uZ`b*+Ey`U7
zcLfLM|B!lCr@J>Er_Ii$&U+8qdtTKq{@j4O2qf*Rf76UNF+10YKhH&Is+^n0CgkmY
zd0M1LE~V!_`UsvrB#*g~a88}zyH#l2Sz1GXFY^*jFcnMV=tEq!VAxy~tHPr1@67TR
zxT><P$dJa`9unyE>b<`4YpByA`+lMPjlub>G4gq%LVbm)&RzCQSo7kyiLUj%<__Iq
zUxn&gb8Ho&i#OtxeW)(vGwFW_;2NX8h<EY6u0GL2$Q-aJQDH%@R;k*K_O7otz~@8`
zqc)MEA4h<-C3eex;y7wYkl(P>gCSkV#ISLF4Od;GHG&&!>(p~dIh61C3n4>kr01&O
zl-*C(&$MrYp+6BBQ_$x|j{LjsQ?kLM<LQzoHo<5_Yv#q{7{34)4Z9?XbL{4HxkavK
z?<>_7h4NLa=Ax3Oy4gK+e$}%dK9z_CNdX`{WJL`+!QWapmv49h+8Aq>8OuDYqC%@1
zNiy%TKd!*ae%s7v@W*vMbvCk5U9IEKR{_xKl3-}uCvRp?Aem0HwM)-+zA}}G|CVhL
zj_+;Uc=EUpM##W!5$kE(<4>w<-^DDPwOm%G5+u<C+5~(?&31j%7jI5%wSJft@Y{Rl
zDEW(!bd}C0xSXX*XDd<bO7;~8Al3vvp8u?Oywg?snWXks)I5}A>-FaVh3ECB!rU$-
zu>PftLHdcSXMXV*@RtV>vGy)MJ_{$ncW$|B6!ewW$!?N4d6MCSl}-0uQg`fN^{gQu
z9_m_TN0}4vV|Ufvu6tx1&i9bfYq*3dxOlwBOt!<j`!)T3UsM`C<LJ`Y%W=x>tOrJ`
z*}JLrB>e5p4<mNj36TW)&3UXO3SJ+yrqq-Y;yxX73r;T9FcFSVc+tpZH~r2M0ajtb
z1@Fsl)%?!!)(tb6UQ=J_V0!(TgyU+dXT9@#fx~#ByXSSSVpXV@HZIi0aeXviTG%0n
z{vu}ZX%_4wy^A;j9>sfgfhDCKk-2;O^;zTZeKkA}Fu1=SVC<FHlOjgmsswi_H#ko9
ztTkb+*SwjONxS+gXQws4dSWZ`=~9>LHmv>2(Ozog0HMJ0)Pnh3BX{jjy_@>a`YR7Q
zQx)S0{A+H^O2di#7*&gk%4Y`sHZUyz{P`g3x7G_!YpVe3ze;)Gwt$j;a}oWU_K>{g
z%x`lM1#IVe0*S%x5a*h+f4$hw*RpM!W{q(gtj5XF#!sEj8q_zB*au8KA4b@YGUR(9
zMm&Pw64E;BllfZnp7&^%pbz8O!NruPL{d@YY**@NNW$&#TAJj_0+}&MoTlAWP&MJR
zSM7MP{`T}8`tg7FbiwOXX>B-$px&@et;9#;j9=<5y17yhH*VNzz5uZn8FLIzMUU;1
zbfv$@U6H7=pNt+})U=y@bv+h~_X_pQ(5JmTuRrGJ!zGN3k;01+jce+ccX#5#AqsVy
zn;+ly4xfyi>&(YE3-HBLcyqZv3E!G2Vs0A77R<%+y$s&ABJrNrp-s$QN@CfP_gpG;
ztbsAH{meSicouV;v(Fl)6>u|!RfoCtPE+n+J?riKy3o1DXSwkcrxtU)d54^d?amnj
zpgefq5o(k*JmP;%bX53<QkPCd>rY3s7umSLWYBKL__{-qr~QjfY3B>aVa=z#$&4(%
zKAq->#5_#p$0PRX^~kEdlfy?=aQ?lNjk~dy0cZ64Uk^|@w-eaZ@?8I_gq8ohO1Ld=
zOk#VG)m74r3?Co%d&Ql>>vUc0<CpFLm_@i;UxwF=%`4Tq10=6>6^d)9R7<E%bVq7@
zZc0497iJTpMdU0fCUYpVqgb2&^7X!o*gnWHtD_+_4Sp$|%&64mI2Yen_bZU2^*!}W
zciGENZ1Rs!UhcdMx4)Xil*@bDoa`yp7&m53r+tk{sm*o%i#y_da41908Bfm<QoD`L
zpEAd@g|QCSWolF<y>Kke;@4c;78imUt8o5j`abcs+gU1llWp4z*9BWIZt;eOo<3HX
zbEv)8-e#%YO<3(uaUtKITlieVy33I}(@S(hyO{ci{`t5W^JdT|TlWdZ`pydb{^PkQ
z;S6LhgoPU#CX~LDJklK>|KV^6l)6pU;;c6>%^EfXcG>woOVSb&v|;^)zLX1Ko)6FZ
zy3FTQV%*ea!)UtqP-42xL|P|Goq8);ip(9Z><<PXZ@yi4d{(1n8Nl?jbW(7OAUF!;
z?QZpQ1=sX;Y*;_O=NX!TH4Em3#ZIpb`Ex$09ttsAwrv~k;xIcmQM}vzPdFRbtBU$r
zHohs&y2BowabQykyEUiZ_jaF%n>=Q&p6E3HL2~ujOh=vY(~#|_4ij)r86&yB$-jkJ
zRwb<eIwL}!1Vdt-ZvGRVR+#;NeMqH;cpMrBBz~Z5cI^fX60LrG&QaCSsi|Sh8sSUE
z<kR5Aq{71E&7E#`$~#XlZ9LRCQ&P`p>9XT#?!<mpP3=GCUGaX3@Ooe87km<nkVnFB
zv>H93zRZs|&+ry;UJ3tXkWsiAara_7a6alY7+5ktmz$}YStab;{HZs=YT$cY`k6pP
zi13r%Y8P1(H{lPjIdAUIw;IB@X<_;*4I&>v&Ol9y$YUPEIcHBl*-qRfwWMj<<+SM}
z&90luhKOm#w;3(>1U}YckV@W>W&0}eV{~sYeb`%_Ka23NMR=cu{v#_2i}n$<yT2_}
zJ{g=J2fxMkvq~Pm8!53lwr=jU{oEzLkzv4`R=+NBhAKJZ-(te0)n+~<w>4yhUbB|@
z(a@-ZApwDJK6|p2nPEHXi)5YU><LA#lWESgJgxU64aLLZMf4~$wI9@W5Pv;l|M?oX
zeMB=1==DmLKPou_o&!u3cYTzY8|C91IENXY`lt0<LtlLSM_KHXFU~^3Y@_BS0>*j^
zy`pE#QQ^0;O2V2$JC(;0LrLiBx=)6kPEPHwekgGApIGz1IwwfK9zR}mlF~@+WKfD`
z{-#QVbnHyaWWarTsm9D#UQUfZ{Kz6XI(PV4Uzo_3fgmD32`QSrU~u;n{g3v(GaAnB
z-J3Cjh>~d0rRZTq841Bi9z;Y45xpdO86vu27#<Nlksu*@P4wQvL?k-V+o%zB^j^oi
z$@6dLoVDJw-Y@T$$A_C`=HB<du6>o?RqnlOG}7+-KBxE}1mRYRpCg6ny|)j$XvQ{N
zmY>!Z5@#hMZ{P0LnQ}9D7Fl`RVm&o%?MFj8<%~@!#526rlr*N&o^FI`kNy~Qsi~a#
zcC?upbWn@#dLk3XQ`fs{$Q$k{T9`R3?YiEyu|4{=-6Xq8+%Yj+%XGnZWYdY$^Ge^O
zmuxA8P;7x_4ek;BMXd?VsR}&d5A*D!@aw1#&oBqLW3{1(Ox&NZVRA>E@o!r9i{Xjh
zt7SO*WbSAQ%MppXFoosvAk;0>+0u40pZ60+v6h&Ck^MKkHTa@)JJR@t3RN3DMXmy#
zvKJ0-f4c_(l!~r_K;HwOT>ES$wu;gNU!3PLu|*8=Y_u8u{)F~0261GQBtIP0SAMY8
zjD0otK^$eX2a}92@JX(m9x>f~9+$%PR<xv>?RfDzZfl<hN2+f=j=5s6dcDe2Zq+6h
zgJ}2lYItB<Ihu8lRrNYL<{AXkBHZ^~lSxQbR;=+>e2ZJ$?#K_m(!;8_SUN@7E(;kK
zX*sLX%8~W;9h+iOh=glu!?Q=qErZUmhQpN#p=ihAVD7M`i^zkX#kRUwcImuc7O>v@
zcQ#-?Uf(RL#(r6h_CHtsDz^Ah86EXP#=_&gRj=|hl1{$)Uq`PfUfHE;2fw9I7nc8x
z_Vg!<<v1hR6gZ%}9IpZ&zjHlT71MQ;^1^XlVMjb}$uVnaK1&|`a5rm(CKV=AA6n_?
z-V7HT`u0w15&E*4&B?WSaJB`FcL<d3u2JpQ89kdbZ!{U`gSHi4u4$O<KhiSrKz$Xn
zZ{l=$xUEp9WInicb0hc@FUgk6RT%+kjb_A3?IS6VR7etz<9TFjXsC35#JG2R=&hpy
zIL^Ae)w`qO^POg>%^MZFvR^oOwfeF^tlh(!u5b=V<L*n^H6v=i?|UV5cG7IH>!|Ke
zZr{$hP?-Jp1(7?<uB~~MuW#<nSW1mt+-JChwXotD5j0Ndo~h^0Moh#rB|KfO{%IHI
zQSqwFjG~X86!u`;T2(svyRES1Jx;cL+J~M+88!nGnU0(WeT%7HqnPSk%Q4inM|lLs
z^+Vu#qym@v)IF<JohYNmc*`<bli{nZUEY@G$wOULMyI|#vB>t02wyJxIbLfhG1Xs?
zNf@>mwHkfC$v>@EI0V7$e_x#Fn}x@!>gjs7EaR=#iuDB<jt6Zyt&4};rE#m~N?XsF
zz6%=fm9?8)M*Dk`U}`yJ_RD(49qDSUJfcsuIl7TOH;&aet6>u;$GhY!Ty#lDX?kwD
ziCm@xQ6@HVWR4d99Llfi#oo_Ad3D&!DTj+wOdEqHn8&+|4kFCet%bmyuMgbtIarn1
z=$qwyzsF?M=3FG_n$4@7o^o*Cg{+^(a$~6Z8k2#Yb?Uh~MF#Ux@24w{v+v5^JXB*>
zO?hYDl@?>{CE6UWd|a$+HqRhEBB3q+V9TW2dlv$C&qD3ngaE@;lbqLh_HD<_aY*0e
zx_u$8S3vu2Lm`}1GhR+OkH*UK+ap8?1eui0naaK;w+!P-1#R~6#?Vqjo2^T!x(2DH
z3;dnQ@Abw}TckZkib*tgzqNbeci6o{U9)?co*e~ucKzfX?}UlRvMFGr%?!n({m?gq
zq~%<47qK4NlGV>+nKA1t)pcL(eVs_#T1Z}1A>zyKNS2SMwH4_Z9@<qDx8E$s<qQAe
zsKBJ1o^v2dn8dZ4AGtcIwKlr;-X5nil$_PZ?*E9yIQ5W2`@4lYIZ;9pmMev?A4*{x
zsvfyv!i{oPm}NV|BrgY2ChldmfA4Yo5xul$%EQAE%R}gda>WMeiGh<A#PiY#VfYk7
zBk}Vg+u0gR*yi;r8;^u(g{0`IkLIlQL^cmwEm%c~1*#b@(E$lfl(1act8Ss)?yFdP
zfg63vph0I9@Pp7Rl9e@2y35hIsoF#V&P4YoEjr{-e~Fmr{dj!ca+t9FX3TwP^p}Hm
zafSFtKL`|Cg7xy+8Va0wFP$$&W9XeQWHozau#h)=Ln>rWel|>;b#nwKQPc28#qb%5
zqcn(0UP%UiBXP5KcWPAfUAEfbKur#F?Bl)B{^c9=?uVZ1yvPQ%o*U@1DRb@;Jat{#
zBev1SVog<xvaCcsXiip_Aa=`TnzHPlKe?;>%;;m^-d1Q#FC4Nc!d}gCE;2>Tsnj%-
zZ`3{5xgCF8ZT`$>qsgV-Kq&SMx;ZM-WvCCv+!DKK-G|><Tso5;w%kD`TzbvEe<H2<
zc1f*9uadD8(q!0>?40Y=f^}ts_AQ@dju|Iu9{+p5TKAz^T(hgz`00OXl;MKmI@yKi
zx5heQTB(7F^A>A;i+Zu<Dhebu356Sa_A`QEtjTue4=7;^MyS1@gSKg<o#$_IzJ*D@
zR^(D==Io=UU8K%yIJVBI385LUqe{xoiM$gyx`=js<YJS0kawdo{X<NI_D5qDiG~*q
zGrLPtaqrMEVbqol{Rc*37`(x@G+rRpc{^?WTYF60YfBEWg~_?99R^9FqW(VY%_C8!
zBT`LbywmpZU|;oYGZRbu;8Mu2e%)aZEW$~QoW=+J{0?LheeBt`ewlVuE!xM4<W;&o
z(&Y_>`cT+?-xk@=u{8()%q{c3al|V4t`N6(Yq0S`$bTttwXP5ckV8LRin^3>eOTNg
z3D$B$qrt%Wg+6KXgBq8dy}q2{N_lyS+KIO?o@&*%Yw!s6q(?)L1+jcb2z`mVuf39s
z&#gszW0&UChOvikhV6*7pJ&)hN5dFisUsfve^3-_ED5iwwJA`?wtv{@aOg>zb!$xX
z&bBM4mz_zvBw2wyI;MZg;)$TynAuR<S3q(|Xraa@-I%HLVimjIA<w4py}KOpU3<>L
zA_3{TQQIEfYmzsnJnNYxGv-o0Br}4P!5}IH3Nm~@1#gQLyYz*Y+UU0nr|TAWQ$uZx
z^CP_5&aISZQ!dX}1{XCGZ2dblk2eynDHo4@d?p+%%1s0R{2}3*Fsy~l+8MjtnlsnX
zuxe^#b@*i}50!KEZh9Ed<4*~CRirl6dLb4ARfLi|U)!S`ZBvoLZ+c*%af=GIcb0ZP
z#0IDwGe(6U(;TcQ9EWR(Q-n99<~%a{q+4=?bs#mCSbi}kdA2Fvrcr!N;mwaMm7=kH
zf#QzqF$X)zC1{sBs|uz8+foFt<B}&4x<*hImkT3A;`d^YfW~H%mwdJK$u1$CeTW4%
zctJAl43w5H8WBBCa+Z#~S6>nHNlD4bjHCSh?S<-<5I^k&EOvf5@SV^Bo0h2T*I&wX
z?W?ub6%&y~uO87i;hra=;j3<+D%S_cW&+3@vM-YN!q3i8$=OIsQoizIt>x3a$9%VC
zOc0j%z^h*MYw|U$RafZ!vv=)m$ehDk7d6kMLHJp;Ck`Aks36(zqj-iL4|#C9wk{k}
zfhrB?nFjWk{f7AQ1&AleKg6`T=FIoF-JD^QR9T~y#`S!QNfc8udLF3jlpS9``g!e;
z&Z0@Sr%JZb9Quhe&QH6gtY^?}S^<+h>Fri(N+0etn?5jGYWjH5XS;7k!ZwLWj}e1d
zQW_GcF_0Kky~4?^uS{Gg+?r?STKcB-8nz=tuQYA$LsZ{(b9H`TtAu?j&#f**(b@gz
zm_~=%wos`Ja!Kx^^r;Nej8uH+#ex`VYV#_a=i&F#hwU*@3kyxQYGYDgZ(1`78PjL=
zp^V>uXlV-*oLaof?q=Zr{Nh}nRFPKyassd6#e*?ezLKuj*+IqQ8o`2!1IO9w9R+7^
z(yilXgtZw7Y2{Z9@J)q88kVmeF+@cC!peWhKZ<8EZdml%)a70=Jodc8QT&-){O4{d
zjXe<ycRQLEk)#FHn=A(VbNOv#83PsFV$~plkkhN2YUo+$etB3!j3zvxidTSdH7>8U
zTItwxB(;iPprGG6Sy<RAYr+?1jNVRu-{(51bw&@RFp8G*#B0S+WfV?JiQ;pLhUwEf
z5$P>9!Ibo+<;KVRaj;v~XI5(7qhiHzdB)|@O6H<y0s)t?aiO)s*Pw)&6dtvABb6O1
zvXsJ{*()2XA#2@MTe35c*Q8Z{Z+iCL!qFOD*bry7x#yA^F|1g3oeClt6aJYg1vhZu
zRyqp74<!=X=e@8+#ExOPS&PHrpK0iO314e>N=Fx++VK`R4Yxg;5JK33S6>uL9w)o+
zmiUOCPkTGLKhZm3@4lrp?ep}lH((oL4-lG)M-*!qju1WSo|J$BR^^@;vvL{1XnHNc
zDFviQ2P-kuhM{R4g}6k5DTV(rR*vcqcLMLnp10^4&$D@OP+u`vuQf19qW!*}lp#BQ
z3to1c@v<*s1_HEARBrcpoLG`5EYd=boU2m+lDj$t4(+b<=!Jo7)IQZmVq$18H$Wmp
z5RA~KQTL!buS0cktD>}3)7|1RS5)+e_y)yl(?MlL;ydQzHavZK**S(q)M~T{E}xU}
zSzOBgH`+`NE$c)nkHXY&$#Hv~t}>6iZuqjm9A<Pa-QHZyLKXHu4C*MtGp@=(X`6Bp
zfGLXc;{VG@ZyR;HyOdKin>38c6q$V`iO=9k(?Kt(wl&mHi`woCmVWQ}&?BGZLRtib
z@?O|U+*iyArL|%>g?~|H6So$aCQyi@7rnkjkna}&pFtlpqc_aj;%7duk6rEYHmhl7
zkA%|@v)Al$iAf+&q1D3+4=6DiU`Z}_^3nfqWE=_!*)sZRcOH>MaXO7)|Nq@Ip4P<N
z&<At9jkyMcg#Da2Nl-145kvJ9k{DzUz<zKyBK83QQpaAB0M@h+3h)ntt3YRkyMvJ!
z8mrEA66)TlAhvodejrbTd3^o^HGy8~j#M;pEm^O_T`F=e^=p`GSTb7jYyGI8Vg(`$
z5B(`w)c=3Uq5+yf1n5)-E#QKsJhOg85F!4~vy%Q$i2izRv+^Zh#ETPAPP{~;bp@;&
zIUqj_0)^QAX)1Q2L=Pb&=isM4=Jd0_F1bR0JEgdo;yXkbMp`b~K}c8=KiBWu|N6>J
z@b7p2iW9Wp`IviPM8Wx0mXqQCH4Z|Fl!b$QaSl_+@t+_3<11;r2rdyB6`R|Df>Sd7
z{lZla26e)Tvt9fjZ=R>1CBOe8GWOqZ`R6cAxj-~W;8bKU{_&{)n3pFQxu-ad=f6J`
zjPP6#OsZV99&_p6Ncrb5uM)t>o7X;4{0}oG!i=1yC2vFaJTv(ZKK$e6zsddQ0wMlh
z-G4~MiDdpQ%qK?dw<`awyZ>f)5PzGhe;N3{E%U#5vHwR$^oVzLCe9)DXlH!<TG`-J
zi$w43>oT63S$U9tMIuZzBk=Xz?XAuK#rM%Gn~YRAS{!W+xMg}Re9Vl+kdQIXZ{E}f
zkV^?54{5~b&{KPcI8WNd!}m1XspHfG=5ncTo@ALc+h-Gf?9m%><L`_Qe=@0#r^y|z
z`o9-751t>Xbm~c@<b6d$48iPKk%dD}?EuDrg8qo-vUXTq-EW$~0kE(0Cz*`B3YkSr
zeD{9%5KI$1W|)h+6j`+{#bC8SW3D}i&!MB(%IA&Pd@MoAhpg$mL^^QR8Fp}LmfwTg
zkp_FwUD7S0DDw7sD1C~BTp<dzFA|+PZqEFAqvl$nCcBq5WcEh2$lo5(Zkt;g?XmWq
zsolV-eYyz3TA<?kkO)jub)?A|7mE&a`x(R=J79!rZ@b>eHUkO^al&Mxo|{%;;7F4}
zZ&=k_%1l8$b%=LZzzOP7f`W(3!{wF6F6H>2Fzb~z=CW};1$1W1u-S3?1i&TerM8&z
zuakL2J^GlRx5guTuqakM^1StZ`Ix8R*1ojHWE~l&U0&6~r$Ma&LpQ_J9WqitS(a8(
z)4)ah%QoHv1?;C(T_H_C?^`=Hvb((tqT`LdIs2g}r*?nAA`V%0ecb2RTkB0gRsuLx
z2>1Dza4?gZh!zcj)m98WZxw2%mR>Zmc=zN5b$mDYf^8m_;F+rA^2E^WwL;slZ9Z~;
zbABq4H&WnX8P4g4j3uLv332gk0;&qhiCVm#Y8oqU9jW`Qj<nqqpKG~2>Snx=ZG0q9
z7CTuVSe^I9b1koWS=6*K+y&2n{poyqjPQ`+o2;D5nON@Dp9!}%JMYW@3dK3SRxx`k
zKF9d`fT_~OTmAaH;nG<LUr#ziv<ASDLp@-*D?Wu?kdi%Iy|WYBp{nas>$TaXlqYn!
zA@<`|kJm=Kq{Ym8OSnH7&2~tAW6jPOu1#{>YjQ*Uxbt8x);90DJncJ~?ZNuP{Q)f8
z0ee*D0mRxMZXWIa-aOiorzlzcwJ-9~y*FnEY1+Qj_Ko<s!n1f}LWXH@;Rm!{hii?P
zBa^(ENS1N~_sM|xYAR^I&Yhz;aO#u5peqikLMUdkaWjC^>Ks#Nfxz*skoqs%wjs+z
zb|YhCc}j*4=O8PGn{1i>!xPy*wf7>b6JJPZ&gE+n64WEa+Ql^sT7|3s%-5<?E0M>3
zIaBu4C3S1g!BEalB+GEU=Yh1)?ljDYS^vl3#t+WH36rPnfo#uT;yTW7lHjI<=DO2W
z>|_LU3T~V5w!h4^EE}>28O=EK+B~en=Y{0f!+Q6IO`bgUj&rE>Sl##`chIN8aSS+d
z2F}H5Z|m>JosVyBg<;utQE<Z;WI?eRJ1dl`JVKhVZVsQ*EAoz21HC)693kwPA{WK2
z)SpW4;0IvshML+95Hskok8-<S@Q2*%HPy)JOIJjey=`HBipBR{pTC${FA4>3sdvJb
z&FC)Pfy={(ppGjyqff0(FYmJ+Rmp%M3pmlN=uEFBkPai!{M!9btrP300t`az*#zRa
z<kwo~9U6%%ilNV1#hJgqV6~qUB|6^Blw*#2HW|GFsI%uwOB-?nFX31e1J$f_LDF`s
zc0BR-QLNXo=Hq5y$5FyQo6IDj?00VMW9P2(6{`;Y$%0lkJ>Z|ZKl>}Bfgp{~DL(93
zeb~6WgFPOV(N>J7YO2I)>zXqF$=di6KTk8NWaD8_Sjbe2v#+jPQSda>4Q}D;eNNRZ
z?y+uG{yuzfy%iRb{nY@qR665JLj5e?WofwYGhrWZ5v^<C_(OtFP;~X^COGD+n=cin
zH|8<Ro|<4>JNPAeZ@GHKpxUmW?hLvR^@wo}SZU*;Ywf%({hLF<Ha#%xVYj*YC*)g-
ztbx8f0Z*UL>n(yoW!4-_F6msa9Ftxzh|zQDo9e-TZ8r5c-BC0d(YNi_wRFJ_zxs>y
zd`w(%6F!&C=njAQVNV_Fy5Q;<Y9&C|vaH(5#HUU0<{)o^`{j4u+e7vD^U^}U2<BB>
zHn0Q>LuL0$FlQE|jI=wmwewfEKwIIT`3lGC4ezL)S8he?Jf(<PV)Q-&;z)y6SIemQ
zDF#7W)#svG_%orySXjiZ^}9NOMeInEb}apYj@o0a=-8p-=HigpZlfIG{BphP525CS
z5@PIX>AIZV&SC)v>*sUE9B@0G3a`v~*RwI&`=*{2TS{V(>6Dx{x#PVb+<~}MrHIuJ
zFR0n|)>+in1(W9B>B^+!&<o_(eDL7HG<g$3ewAXxeJX5{Nz!Gu5GzWy&9rDz+V_}p
zF~4fkcK>ILGhK33s__I8E^pHKl49}QZEi!<B5aEx(6Qh>)g)n)+o!l4PPqtMue&Fh
zOD{a2es2E)hjvP%NioF;93J<DD!vmO!&X2Yud+$$Q6lV;x<9&to+b3Ohhonc=#or6
zKdx#odugw?Uj8EcrE7E15ENt2QwH84OLCg+cv1t8OERmTqm$-Pt4L_);hR63{^WYf
ztXVNdN?a9oX;+53aj}~JqhO>Q*>P4J!`V~YzCq6B@$2=ZM2gQbp^z+ZA_*J*-LM`o
z#ODm5w$5Un6dsqSDzk9ewsF>JYp{?E_c@RBp6u$uLn6^UJ3zZVyZ1mRx>aS91r7Yx
zTkK*2aG4#G=w7NyjgJzvyL+>VfulxIn@@^YhZkpnUm)K+x;^{(JEMMy$0$tQ&@~iM
zo=`jRd98sJ!%io1u5;nQw>@toTidHmh3i=}cTA{v;)vV0_Y)i&dUulHM*!2t6%bL&
zAz!;+brtFOLha1ryfP*;=-9|k5=YHG$CEaUNLuI$xHE;YvLhK?oxhwM7aYzQ_lfPp
zv#%7(H{fILQ(rPLWHCmb){A83B@d2bpG^iZov9kq+*`=b_tST%ktneKfnk3MFW~RO
zsH)JQfq9;@;P$bV`*i1c{zad$v)NS4fzlG};~Y<W#&xo)yMywUB!(a0!&a$ZsApI{
zpq49<4c0HSGjCl{7iKbaeqFuSw>NhNHve=E&njkd-+knmu!BAN8QBP6jI6;d&H*dr
zA+#KXP124#pS9z=WWzC7XXOu+rqOWF`HdCX4%Mbgg0RIBgY+EIJ;#Pjdg56{P}x8X
zZH<q2HNQFn<e28Kfc*i6pgptkjYeRr6=LWDL#NEUV_76GMNn>D&A~UTBoY31S-}x<
z1ov{7IY)Fv$j*Fb;tM&p;pM!-#^iu9yp8+ks&PJ87s8}l$FIwA!}-VKkD$6AYH@cV
zLFD5i_L#8jwfK!$IlE#S96pG0A1iBhQHCeSzfAjC*2T1Of&9@quDrp#x=D|g7u{?p
z*_N>f&bWPJlJq{LZJ0zqxm?bng7|*c<~F5p#e|P^_?{pO`RD0qeTcNbmR)?5sB8+>
zj+<)z6Czy>eiYnm1jHw+&f^IElOsws<y_P~g^xeGklKbWTIE#`KV_u$M_D6~P+EjG
z@o>d)Rl6}soMxoV_>-C?H{<nnca_s?5J`Sg1XMNq6zn65WZ?_r6`0SSO0y@;tyP4M
zQaXx*bjOR`)u1L)YBTK$XkV;4ka|n2aYFT@$k1`6k*l2O#MyW^qlyIf`c7E5cZt~2
z3;ceSpze3A$Q^`9i8lqbq0~e+<~(_uWpzbr3huG>R9YWWg@1ORtBj<v$`0W@C+KhA
zz-)O=E~`)If!Gyp22S!vGPRVWDIvlvbM<2nu3qR*9gxUVFlg2URloOAkG%_bk5+s*
z=k6E~_slICuNRV#HO2?)XCQmy)kX(D=ZmszQKlr|<y>BG_~Zj57v!=AfsmUCp058;
zZG9H|2|DGXB;G1o8&8ihM(=$yEMX#lM8jnqQJak?a4*Y|vL1O(6W4<-S}8A8{5NF2
zc`Zl<2T}UDT&2yjZz0@Z)HQqkZKI#E^bEv>=GRX0)t6}-54CNuM(MuE;R?+h=%x$E
z)N5bE+rH)?y_->7>SEj|8VIJgZ~gc#mq;e^kMi8bDx~*%GcQKtU2F9p+4v=8-gtWS
zE9=gYv<p;F8xf;)x9rx!U=Ff#7is%@(p6$iTHf*_KecFbtpYbiv$&`=d2`plkEpWB
z<bcL4Y@^QzpiZZ+TN2TKmi65*UI@?q+4xCn?He5~J9ywcg6j%#5k(VE(_F4X>1n9I
zeAX)E%yEsu)OVh!Z3w38fY#U;6)|A!<K4V0eoX^~dyC~@;#;`Or7pd#%<9xGq228G
zRt`wq#iQ3HhZS!UvT;u7Pc09r#1)gqjg%RG$R60Gs}6q#4Lnp7lo}`5O2|`2txlw1
z9BA$;jq-CRXbX<=@t!Pij7ga8L!(U<wUX*3&{M2cdylq;NNpvr4oNLsGy&l8+2Vb1
z9Ul}o3wf`C!^iY!0p>g1WRR%GYCw@2C|{|XK#C|>j%!ULPD$CK<a4Yei|Xy+`3TlA
zl+Za|?qDn#p6I6+Md*3w+m&)o-sc*NPi=P@p%h>gA-#ZuY9P$2j59l0{6wG5+EPIk
zdH1^qX9({EWo*`8rLEV~$U$3%Hyf8YB`)bIHK2F4E|70txRt>V46AB8JwBLlxsIy<
z`Na0f*Fj$s#~pjM-bbBR_2(oBCJJIoLPmWKA4KUXp~9tp8+|C(9@%HaPXiS=5Y_YI
z(Lk9}E0#|>Kxyx)`PxC~Ua7tsI|%xnR%H<^=)6rCuyfI|cy+;Qm=bEUuIz(NrI5Gb
zSCcz()@j_w=Nqdq*vExKA>#mZKQjiF)bhJ`pJ$?b??a`4gHE%ct!VVoe!vzLyHu}g
z?|l1|=#@9yN?an!FQ&K1X~}bxVD(7vf<NU=re1@|NAzb<&H|V<WdZ9CEY3(7@ab|~
zT%dr416+N$I<VL@%^d++tcEj&(j}^k{bI)c7)^AU`~;^R95bn%=DDQ=tP2Q0MSx))
z!u}Wpr85wC!698~0lbM`qOQCwsUu|ZZ$}>+rHIw!6?aHY!`$X7m)|%$Bpq3OsIzIT
z-H@IK5>~vszrieS^=e#v`+Dr#BxOozy_7eJb^r>nk6v??*}Awf)g<^T^U_E0UmCL=
z@%0-OE42qt%Y}?9`t@upk-c=Px-Y`6(ed0TkC3$PH)$ZZGYd&8he0s&x*3n@M$3xu
zW$B6<de!K)F0)GnzsOA;HP9L)x9cepr04<6%Bnw&>@zXfJ>NvR<@H9;wJACB?9QI`
z1{=subDf%vo9?X7$xnq%f_yiKW=qC<I!c=wP1u`_)RWzxOGTKzJy=<D#2}~wOBRb-
zU*JDDHZXI$?DF3G;0~hJ!yb49*tEe|r?Fq(f&FlIDgxR6UdqL0b+w*`%rjK53o&tC
ziLJ|`?K!tVmUOOM;ypU4PlOXsYfui}PI0UbHOx4uo!dzTVO?Whrms=DBLT#hf|6Y?
z!?mYK6fN~@kG663rajLdMO85##IJxjZW8WTue!HY<bIX*U4V`30{9i-%6Ic@71DeF
zi3bvXj#1yPaJAkU1S?LaTkPJOT^O`Ai>TS%g3WG~!2AJF=MJpJn#{)}tw+Wnk~Eei
z{G;R^yS&HuWovh0eU3Z-D7Kj20^EwKz3~>c3Yf*hd&;aZRCyuqNp=w<PD=lBeY}0^
z$P4U0;fCQSC4$xN{!FqR$eY~ptl?lH?_K*0CNA&%*aY+>U8}fvk68}w1;SaTioza=
zIt7iHWJmz8!k!h_<<V-gv##?x;v7kyclmWbPN}>7syMNfHSBI9*WFj^$VgOIv)ZJv
zHAc4i`UMLES6_;?iTL9(BiY2GC&vlfl5R$y)Wmp{{GGZ?yMMi?lIyK~+m)G=-C@WB
zwg_;V<q|WWwZ8nSBBhz3w1yB;JPzW@`T@7WCxI0L_o5Ye&2?S+v_>80WRf->b$<sS
zew62;frdQ08fZDf#B8X7#6kgPt}M#1T<A#Pm)tuEYSWQ?5x`=FiU*)u;z)0L=fwJy
zdSp&>(+-|M=^Fqg93Ze&QA|D<URoXDdM9YCrmvk}qad7Z2+H8$e*VN!nMhIoQBBxC
z2%9tAJn%LUY2mDW0S)f3Va>g=UmnR58SJTVeIajd-_a*K#MS`VnFeIG`EVe_gq@#d
zN8}@i(71DtDqS|SjHh$bgk|~?!_`nqJAm=RCS9>dOAe-DSLF!1Q(x2)vb;mtZi=qy
zZ2A>lORejyn5y~}sUpz-5FY}{S5d9Ut^lwMD|VHboxvb+cT2<1UtFTjS^QEF`It)`
z+qp|)W6l~Ka|;S_d9ub-#ccWEI-ERfiicKb%=4B~QYA1GStDpOx<}iSTBoux4-<L%
z?Pa8qOw~$fO$UHl%MEEgZYZH_dh%XfzN{=q$dPx5&m&*;AUcr1K<=3J)!%*ZB}?~X
zAldi5MMvN6)IvC$b_xI`kp1TCUhJN|Yc)V{9pf5>HQlVu#T7~HYh_);IBvN=TXaQF
z#=DI%=M=RFAn%Ja{(P*DWQe1>VczoAWVv$I6*ot?$NZ=AbhiqO3jf&Q8dCq`{e)XL
z)o+Jzx^*UoaP+m_dH+H8treI0N_NH9i{J%@J!8caH=gFxN#JzX5OCd$Nvfc6KZ6lM
z8kp~RXr?>acJ`kzMkIEdUGk!xbFk-8B%^jp-Pg1)=FZi+mhpY@Qy_R|HT<w!uCZJb
z)bR1H02K1ZOeY+%arTe=O9Usb{ktc4wf$gurU~Fx%u{l!^_SX49^4fm`{So=>@DHQ
zRvT`WC7sWs@ksGQ)fQp1M~!_b$1MINY;_TQ%)VfBIO;YY^_gSmDhv6E0*(rNvTS>I
zUHI-Eklyde{$%s2V-(0S`lD8A23*$S#jQ1-B1H3UlUBZ!;2%H9ffWl^I39%SZ*L74
zlT|eiT;*Gjq9euVL4nAp^!p-#w5yiqT4kg+yyS!JOMH)@OlEybwauc<XypEc389I2
zD0V-JDQsg-Y{}xiIisY!@}5*6C))kupCPO&YQ@`%4lUBau{Nf!i8v=)<?~(qG_`RM
z#EQQ?<+UqJA~n8F9MrmVKK`TN^5>R;SrA>`mjfs!a;cf;ZrLQmi4+k-yDz+?gl1?-
zZUN8p8KfWGW9Dn$+UrNfCH33u%b>Q;`5Up?S0o>p->nldZ?Lu_C9U#OzaLRP>Z-qp
z+4YU<fB5SD{4~omZOk3gjues=p7tB*(`e(CifWLm8_l<9>7MV&RNIX;&IFJl{nNpg
z1Z`e-<(A58jr}MNPHp(0iT^WJb0TZQ^kqDg2<es=qGupZTr;MKWeYz^OvXmr@769c
zP+_jeA-QcbvQ}^om`MOd<;@+6C<G2x1x{Qh5#+aTjzFHzQU!TQq&?M8FreYREi=^h
z=eHotiY%T)zpL<?__Cix>U!&D7T05sH4Gw*?!4q<|9t&X|2M%6e(G)pSo=Lw@UG&B
z4lCM234%HMbI)`Kr@%SY(BCO4_u{3oq+X!Wn;;+6*&KQMRuz>0AP=clEB+*I=Tgrl
zl04rRTqyf{LVL6mEC~^=6MVs7lLwL6C9uxF&U`AxMHl@7%2O&*Fim2hB2@Zluw8rf
zd7t^?_ujGUsh@Wd6Bsg%i&i2-q>MUAX|kl<;!!Bn6nswSvCfl#NMxyTm}XY!t+4#b
zh<w0K_RNM*9p^5E`lZ8d!zyPA4iLbuYl4i;_V=PzVf~^o&Jk_XfUFZ2DtzDwGQ(~^
zT938S9B3@zHNvFw;tzmpg^2AhcSoji{FVk5!7mb@3&$3zh$#+l9~Vn5yk(;tv{!C$
zm6m|hn%U7d<dk&ZYj7RNnSY7oi76!JF>Q<xgRh#~tu_wk<!berS7TXl5(8e_LzWF^
zzJ<n9w4CIc8Spnisx*wsrpjfJtH1(e%HW?GsqJku9>gz%rvq=sfWPb6b!H)y{f~b3
zH}T9Z*D8NSHk|nyN@d24Od+Z$B>5?&9k*s-H(+4ukLJeXLS=ULL2}oy=+=A?yI23<
zqP(dkUbI!H{6k?~U7D6kH2!efx|kuX9pc>hw3;s_M{|je^{NC&96YePOtbe`v~Uy?
z)*+7@a=Z{m70n_YqSc(&XQ7v=UvAIc>H&HZ$nUFRDs0;}Z*gpg{Z;s6r|XnB1P_&f
zTh|T#g1T`I6bk85o^<eV(duNg;k3BFsOo7M<eU~3)B}hH-U$@wF#JyBvr%{*AMG?y
zk3MorQ3-z#a7RAL8W;h%s;0ya=J}ick)_72li1RO5d@k)a_bBKRnPeKSE4;2A^>4R
zDJh(J(jsu*66kS;6G;K;_p+T}g^cr{79y_NH$rxgtDq7GUfTF)iSn;vC1!>ReB}^1
zNN<TK&&{ynp}ufBy??*x=>Ywql(bKH5N5CMVM7&ulCb|*vGe4wU_Q`n@iMpL=3o68
zfAQy356S5tfcqz7KhP=u*QbMV_)>s@xm+eS`@8t^A2)KT-vFhG0NdveF8;SCg1(QJ
zAlwfbT=*dNTfKfi?62|?ViNF@F|YaNlm9x^6TVyqJr39YS>gGQ8;KwRbYSr+cxL}R
vO=<j1=)aY6|FJrMujq*+|6fEGb4=nez_mhlBb<>4{3t6t_#^MOiQoSLzo2J-

literal 0
HcmV?d00001

diff --git a/docs/figures/arch_6.png b/docs/figures/arch_6.png
new file mode 100644
index 0000000000000000000000000000000000000000..3969511041e5fb8074316475a6bf74531368431c
GIT binary patch
literal 123034
zcmeGEWmuI<`v(kDDgp{Bk`jt^3xYHV(k0ypNJ&X|OGt=HHwuf85*CeslF|**N-t{B
zz2F`8R#^AG@BO^T@qBr{{5cL>7qjM?Ict7r%#5!U<t46Rkzk>qpj?xZ6jMS$xon7n
zg5HIB8MNG?x$*=B1*_OxR8&z)RFqQD&c?*t(ijCr@^y6V6-8xDLLaz0Yw|0Ut2f@>
zbi0*tOIUjG!yXO8-K%)g!M<sR5(Mhcn1h3TuIWPPDhPy!LM}0eTw+@EW5Q~K8-5FH
zJN3D18Gm}ZAY`*P4o`Mo<5`%jG@a*1p>rtENV?95^7d|9%&UV64kN>vxP1{H^i7{&
zLCn}#YLlQ>7$_>43w3QlEGTjv&Yd(Kh^-^<+fQmKL$9FV``O(S3XS#Mv~~D0|C)LP
zh4ScW&Dwj*4GevecovacVwI+Tky$R!{ch0``^T5O#@rItyo7=l-yVG*U3f@zr{j*^
zgQcLA*Tn(BtoR?oYSClE!ej;8%ygc;8V!D$vfeFy9GN;>w=WxYFl!fS%wHNSb=L(8
z+6WsrK3%XqwrQ1&m2Y7azQXflnl!KrXS;(u!}zdWBZ>&0zw_})2I78}q$gEYeC*r}
z?oN+_tU608H^r}<v{!3*9te#j1)CE*cMVX0-sg}dV3eBvmSR%yoTh-2St(QD#>?j?
z_R9A@X7NZ6-RA7nH2-zN!A1U{MvR1}U7vE(Hud!~^O(E%Q0gw@?Drv)EV2Ez-LB+T
zeM-ab{f>}#bIpV<Zk<2G+E$uI)`~TzxAEd@C?_N`yW6hD=68y-&UTFAjSy$rAs%NE
zyP&b+`3TD5UCr|m_>94Xhb3J04Aw|?i9CMV_@;rKG*@J}TD(u|jg5{4wtS{MOO&|m
zksQVi7RzFSyUp>$x-?p%dAMfDmhrvRF=#zHC_ZiJ#L1H^DA@-noTVEX?~~{!#CTAd
zC{Y>G&{fl}I{4mT6WxNM%YVJr`2!cj9}^Fo0F7J{l~@=p#^?4kW;&E7&p3WuX83CT
z8F#>^;U}Hh<*cu+FNt-olz;PFC))NyUB?RZ!F`6~N69bdYxb5Q;}T3X^X=_MKOIU|
zIutB_!w(pA#5YBh%r2E&XYi9w!!l!4!A77Zzv>cI{~*73ci_r35t|RJY-o`Nym_dk
zU+oPDIsy^DFkMiSa99KFe|fl2>0DQD2>MC0&+kE2i@R4}`12_Sr5z@EuXqO@!L%<1
zm96B>dm?Ig10;!+!Z|7PCHd?KVNU|zbranQ3^Xw8V3Q}(m%Setq#33eK;NwL>;;=#
zTxeW%oL~p$JYS5TYD<D4S<xL!94_K&3~e9g2J){dU-jxITdqG7`uXzF<t{wah^?lk
zX(iis+op=-3h{9s9ohm4?vR!l)s0LSQxCS3(C%i!pUyvVe|1m8e_mNn#k9j<_PhNp
z5l;f|#wMomE8cqIda`=#dZMSk*$NK#FW)+7G;Wk@L^P^RGf$(~hGj@TxX%-LPckQ?
zG{a5_9d<J(Xj>lqNOXy?sL0cmix=pu1&`mZ6s+uiPF;L;#Q%;$%zN(lxbM5)QM{je
zN{5dgLQ4JSz1)ao4n!x7AL8*ot2MFRe)eE9caCu`aTfl)F7sy2p$eDd-G{MS;sW9|
zPsr3L8Hngy=w9AdqgP=_Qdv|ZQq7>(Vz|s`LPx+jM6aiMN2Q*Tn9h}cBYHD>mtI$K
zDeEq$Zo-vt32ling=#U<w3)>~hwB2-dSXJd@U#dzM*1;233^u50u{mh@pfUN%Q)Am
zl8av3M)fxBKaepZaF2O56sUO(y4G=+{Xn}(?n#<bc3-xg{#+=7@}SH^Imgf!(4|`s
z!xCO=Wslx0)qN2=5NO(O%GWvH$&%#huuJ8rF(D=(>sfgyxI?;>PLe<(NMaaHMncFL
z_>efkIw2<kl3>aiRAQ#ZqV+gmDIcoE8s8?%#gVMp_2v7WUbW%f!I&5D!L=Upe)fUa
z1Fik$1Fj!s`ix;Tu$ZMX*qcRj*rlcXp4?&g421zxSk|J=u+8wP*)uadtIv~ZBhPvi
zo8B75S!me&FcM?8d#GipV&%|HWZLmSo@3J7V+@0<=%EyQ4riq?R^n(pmYj2fQ`fgH
z+c>HOWr3`*Fw17HcJ^HEWe(H0D{%|)+=+E5x!l~=YcZ9p_I27nv@*0;%3UmTp2JMz
zO-EiF47D|en5vpPb|)CFTGfnJH64bB=Y)SXiS0z!POwT_bsOUvvvJyWx><d>`mVdc
z5ylb2ku^c`?Ro|iUE8m`6?}P4kLK1#db`-iHnL|Z<W1zw<sIeyw{RA&&ab8p!}Df3
z94D8@m+Kv*UXIN4Eq_{VbIf=0oX=ctcINxJ>u1#=Z%koW=T!bOwj%ZA>Ef@&qQ03V
z+G*H~=HkdN$n0&H>aU#v?H0a~>Ph#;mLb#1ZXW%-!a6v?l=Yj-K`l9RVf!OT`P<ct
z3cI*_h3nn~&u&s)HM{l-9f4MUc^3nA<+WdezX%(xi^X~*Uc=j_yLExnk3}Cdilzo>
z2Iga0;ARD|VijVa;{75oqMx?Sc$&dk%p5{;n?jXzhkcZz>V6fsiFHR?+;Y*2qWT{(
zk34i##(gV#5A?RmJ1K(8!&!sjTF)w;!TfTu2MID+`D=o!f~yxb(75jMeqi~<$Q`HM
zNhUXYUyb_BI={=9YvI22vC_%gubPVYT;9zlsl4ZmTS@xz(3P&~A%w|TWASbtUai!Q
z+(hyrJM*K}XQM7|4sJ%5?^4~J$&i$aqN$3@iLc~@DqAwJ;O((E>3p%;2yWE3qr`60
zdOP@DP9ia#REY|P+gE!hHmNO1LFQx39qmsQDka;^4<e08V`{$dbRk@MOn7Wu$FNp!
zTc~g6KQA1w#?p$e_+(YOwsbs*Zs9RBkVl;_r|r(seXsC798tCCn14WWV6l1Ry?*ea
zR=8GJsi$Vr9I-~hE`;U1^5a>JK}#S!;JDu63BeQNC;FkTO~YQr$2rGY8{tx<d|V?|
zqou^<nkAJMq02D|H=Ev%zdy<M)lQU-hXg?2<;<f_r5%};dro~$u-(Z`pUoIDH$E=-
zzScV>+q%A*;v(sq7ovkL$;ZjgPL-=Ke&iQbo9)F4)C=J59<Eq@;>u<j3mS`s!VgZe
z5vxK_uL6NIr?kbd<^CW2YaSXrB(zwyuvi{g<{E6+={x)&x)EeevCBW+_iX5M`MX{X
z2aG%K$XyR)>RELq5D{f$Mr0O`3cSB8VXTT7+!`p!Gtc0GRF+ehLAElcTFcb3hHlj*
zoVu+??dG)gw`FlB-%V~6;_%|yTUq4UsQk6nx^Of`P%2Q1Xa8%msV9^y_Lx<-a>euD
zF<&)e<@N$&RspY`fNMAqXw}3{CTSVB{CG)?>&J$IbXW``KW6NDxi;T|V@l<W+rqJK
zcXce>c1^dA!l4ezSiCIa?BNVYKk@$-1jC);Z*%3`b&<m@%UXY3A6cBWq>$^)agtv&
zT(IYmYC08e`m39KZ|hJWk+O3%?0dWrwP4IzqBn3|>skuQCV3UY?xfvUeOle9;nJzn
zvHGAz`f$tX%cGfQ$&T+|Nm~;)%A4RSQ)C`CTeVi4&$nSgVYo&V*g`p8pAI?>$4{sa
z+e3?}^*8j}>xjKA5NoG3ru;@8Ivvq3X|+)?$X+VlM6tm|6MlWk$B7iB?>HyLO(`ws
z)JF>K^mW$Znp6?`GktE98||o=6<5E$!770~K`76lVt)AOMvJ1Y-QMhQ$~Tg$5L|y3
zWfw>L^5`XgkCX1HH#veQ3a+P~mv8A=eVtnHMsE(noZgygfrB76#8^YhL{1Kc0X$=(
zV4#wqTmnz1;8zfp^!!;Il@0~{uXZ#Plpu2yjNg6a!ROh}Tkw1K&flNt?*mb;fWK~n
zUzaqrKfNy-rlJ3NM(+aOp@=ApN=boFWg|OdV{3ae8;7@F7=%H?)t8c*_9!R>)Mvk_
zQcCyN!T7`GDjE(Na<aTeHdZVK&ut8iSzN4Mg5D@7{4TuU(aPAtfYQav(%PQaMd03F
zJ$S+MSu^WB%D=ieSP0zHkW-`-wXrj%<YHlCVY?@YMM+7?Z};4US4m9#_v_#<fqP~S
z4lj9GS)HAoS)4goZ0t-~*?D+)SlJ%1K6t<kdNA9&T00oHFk9PG{XNK^am0-6jqJ={
zI+)v7Q=W}$U})p$AaL*A*+l0*f75B~Vtz4`wf%3l0E4V&->|Z?u(6(x4PNCxYvolm
zcQLlq6f?I1G6Qo6@^EwV|JDD$zPXqY=}nD`H`%#3{^M1oum1U}s=cwDsErkv(?Re;
zU%%%?{_^*W{H$lJBZ=Z~IserP1TBcg&w8#jK`giV!z`f3JLX~vD&P}X+1U^30{DmS
z@6WU6a)-N)RE{Vp!YEQ=A}TJZD-)M0o*N^stWVWwe`cD%Wa^{O=M0w4iFzd&>u@y~
z3+pBUrqWF;$sklrNp5dpd@<n+(wMdJimlYWpE&mL$<b>2wSfixZeCOQFB`6=sS;hz
zQ+p-1L$rTj;!z6wprE2*{P)8rmL8hmoT?nOtN+?3-iNXo*Y|(F4hDQ1=|f3FtI>4x
zU+2JJ_d@?4Q~ynjZxRLuJI^<Un17vsGI)dZU&qFry-yk3dR49HUnf8n37h)Yv4tsV
zQAK3sKZj8Nn}9Ge4UaGXx7_};@La_-wDVJC`k%r*>m-af!~L&g<9(MB#*4g_XZY{Z
z_amzN*Rg$s<<b1&lAhg``4{bC;^8&!U;9_hV%XipYn;W%)%Xv~$Y%AOi4YPjsGLuz
zSSfBdYz)3p$+=cKtQYDytjm3!Z{_oTGk5Jf88WPMp_wrODH?^x4*iD_d_}={!!sZE
zX#FdCdQFXh^_Pui4WsafmldU_%9FefVMXmM)s3GG8m6a1S>J?njb_6WbwvJf(lY>9
z%OuxJrb3pUZz^KN)OP*b71O&Nnx@IO$1I_81$04o>?H3v4m@RdSnPRpdc0)>?c{1N
z6%4rb$7}}5eJC?)TO^kMs0{=2IhyzB(bUYhD->bsdal&s5#(g;%;h>vU)Jj#Uff?_
zOpj<zw5g3B6kJHMZ`QHz9L%8(czk!Q_GoS82cVZ+SwX-tYpWI~<tu$w(#>jkt-IQ<
z5!2LUcTDA_5nS*6hW#d1uN$I)IInXHYk6yTOADjr5AOfr#f%*YEc9B%+PT0uW4;<i
z2oUhWzu1pEjaem=Ob!jjAQ+Ff-?|>cvfvH{TKeSP#~Vz?#5{J-^PpY4gA82dc1mn^
z-2!`!*sNdh_*P4lt#>l~hV{J>71JS%#k^{o#?cjs-7&twy;j=PMto)2cVyKwZ}<%7
zqSe<NCS$+da6#-08~I)$2ri%U;=?wQ7d)8#mQ$f2gEdooI9S*$$2}1hkV!8o$MM|q
ze0}B9QU;sXun!ykF&f{?H(X8*7NA%5VH}rO%}tv8uvv&sj~9eyfIK?giU2Lm*SN>=
z=$UeM%MRufTc~4oZjxG880fgcAcHKmOLGwFw6f6`xt0exS8vg%)Eu{=G5W_j9DfKR
z4{0LM&hu#fQDoH4>QC4hrFq$tw)2&gfiafRTHWc1&PmGlP|iY%J4@|)0Ba)<-Ywds
zmycI~vDWCW-TAX{gd?#q4HF!mIfniLg0ziGN+@;*bFrqZh}z9IMx*@FAqMBA1))>d
z<3$Ofwpt3?#vAV@rJk<J;q^n!hAB;en8R)<@E3XbQ)bFOg(l;_VJ;m|diS_e2mw>r
z$hYy#rLp4K{7yB~B6MQg!PbED&Lke9KzdGs1po>yi!VFpIF}}Z(I+~)%?j(^6qlgz
zbhz{KE9$LMegTb>cKwt0%yPTH%9^8;<uz+*n0uq^eK`G1Vl++YDHQl9ypL_Ua%onx
zt_-G&hf7zF#Kb&y-x*YgB1CZoIJMQx2UXR>w~M#>h9)P;&1vKlAj*|ByQ3EGCX*k`
zgmE<A3uU@V;ju1~F+zE|-y&p~8E+C&@=85nuzzBAHd3KEj@_tuxypU7({pcvkiu)<
zphIA9d?Zzg*c_H4LGF70$Zf)L_z4z7>BzcfXQ+>QWZ`2di`L+kpAbr#@LqIWe!PBp
zKC3U=eKK>zgotZ~dR_-}6(@(wP-6M5ZXxfoDAnR6bD6r(k`6UDS~)j$#o^V{6ZmQI
zRL5g%W_*rre*3Kcy-9Z)kL_NGIU~mW5+##po&E<F=FO4`R;BX^7UFJoQqJ0&wSXt+
z?sW=5ex<}=UFV6-mDGa0`;l2GZVV>LjzgAe(soitt_#V|CUvRQF&g)l?T3#(ZFqvN
z;ebG9(!_0R$PFrSS*0QvT!myTh{Ed;FE8)C_?Qz}X%Ph>ghK}~s1fG6gH9X6RL|Y`
zs4?AhpzXmyYdZZ5z3~TO@F*2c(VCn!@k35)p3()91xz=rSZk}}7!+IdIW`52B^Shd
z?v6n1Tgcrlvu0p2?3VOX>K^Npu;UnQYfZt}wpCz7-<-0J`eN<71-PrvjKJ=fjrWnZ
z!Ka+eB?s@Nb^ulUy`HeIEY&OTb*r06EF9cxiB_I3cyC6gI`DdLG!Z&~O3&IoS#z#i
zc9@#=I+*9KiyzcZf+^C~V$+NX0oAuP5lr1swpM2kW1uJ;r@w4D=Pt7<FP@uUruBiS
zOhr#?QlShs3isNpo->N(Ya(ngUFZffXloepV&|kUr;mQBv&nKa5#_}Q;mYV<D~pup
z%~8ZT;*EY+z<k*=V4Baasv7cg+_uGd*rp1W8K7-l2~os(PE#eMN#Os+8lIZu-^yNU
z6GY||mth!!YWRUD20z4iQp`$j?zZUn`cCdCektQj_r4mKv-LUdn+c$D7>1nM{1E@+
z8-&v)G5YEf6yQ-BP9w(cQ1z-Z_Ri;$9PG^z6cmJdE(RV)V^b{p4_fZnKj)AQ5s_U1
z_%=6VPRkq-Wzv?x-036f7*#fPvRgi48L%x-<0DuzbB`GkWg-w&#(8VHgQJ_tli8bq
z;-qCr+eX*xuB#jSf!lI^Z07*|WsVKdX3W91R<&HPKTTMtQOmj-Wx|T@A4exRyXL^+
zDaB#g*AZ)ock2uo+85x!W6)j37}?|8qU<b@)Mi&~6O=iPG;vX>ZGbp<q@_ZL*@5M`
zWN3=zz}s2tJTl@X&6h32WwjVA&fJ(uVTPuZd6{AY?ERZ8VwvSUsC>rUV#qh?ESk&_
zhSn^i#^+E0-ee{(so|^<(i{QZYqh5cxI_Z+`<QAOIFpo7F`&R?RRQ|4&oZ`8?O0)^
z&w4k0QZ#BbD&|1isOWGhSAyNL&7?2ms4py*v!5EV#!$|QejjlX@<GSR^N=a<!N^XI
z_vy)A9kjjr#EY?=XwmD1a>7LP<c&x#eZ<6d{`Ieq=HrZt+m%!MZ61b8kt`U-MTILD
z%QYh=q`gxkwHX@DHP>T^mLDqsWB&t-D87e;8t$v5P;ZgCUUAZ$(I0&{{tegp^7<Vi
zn%5(FsWOY-8Y;dm{jgt<(AFTZ7HiUoe;WBoQKvpfQO7CQV-=u<atfjdU`=d9tkYq_
z4P~7ICJUKiVs^+;zv;#|7bm%d*0+0V2{*R<DUQ=#j#-r(?__!r#-6r)TC)<DwRJ7p
zwEQq#ZH(1KnLj?x3C-b*TvV?+R6TGp&ng6?Xx?f4n3d02Z3gVQeH2jx_bgPzA;B^i
zS6C7~TvfF;7gw{btXvr%t~=RzZ0ic$giXJ<3z&XjK2kZYWI8tkN$v}aQ?X`WXT<lf
z>C0K!d%Cnt?!NL=P%k{$a51)&elXGGja-|W&Sw5NNZKKGp>Pj+eDsupEH{MLz*7lh
z<1Vf-%No6f`+sC6Y>(w2wD5)J=#%=nQ^p_>Pupg50a&+I+S#N@K7u~UT=wH*zhfzm
zVc(`!H14>}HNM3(|2Eu~DS+yIzgD16{o@dOQ$iNr^y75Q#6#M6kue-Hs3KTw45HyT
zC2hsUooXK?W;<jbYj#r7lp1Vxa<j(jVi{JQ#B>QBt!!4@Hl>a(WL-Tz9Ij(YWv!*n
zc6fBebcvn*CCr+qI$B+ir@Fk(KtHSA{Xi*+Ue#LJHb#Y2-AVAF^>{`7_jgoKcV~bT
zU9_|_+O1sidnGI5&+*}E=P?B@AztIa0g5^KIoB3gIHX)m?nFr@jN4_*(2fTqju!c)
zYFX_sh#T=^?=KI;^u@}s)|4E-{8DjW74~v>BwIw**1Rwe0r_e$ko`0O|G^GHV}=j4
zz}qSFtYlu|s9F%#JsdfhexMH#o$G#ancX~zVM-`!l1xSCdg04{Iqv9X89sVT=~Z2i
zjV3-aR!`TZoUFE)(VfZnG720b(j1{&Kc4w3*{U9V0jsFp>JYeob}3_m*ks13M6{2b
z<V}!ynw+o#!c_Y%p1BA8D{TMI;ea%&8hrW5jQQ2i+iyLNR?Fc7P&XXI5O_MhP#8;X
zR)(@WsX(ynOgML&OyTV*Xk?9tV7SboeG8eh!;*zLPvvyA0T!_D2{CV_Y_+QLWt>RB
zlP9yg`it#h5P;?h=*BWJCM&cjM2{x=CAH-jQfBV!rLRoBxWcZ$zadsBH>~GwWgK}p
z?!Esj2pbY_0`Y<vk1M@&TFx)ub@g^8#Gi_eEHw?G&!oejg7!~Pzd3h*oJAnOetkuK
z_&mTSQU6IgwJ*N=ldF?x(TcIAdWNuW|4CN+5_|AjK--Sn(#^lsA-`(1O-bWu0z6RL
zVfpNP852(#4*9Cf>}O%eNZR(>y7F#(Iu+|d978wvDS6{$$lg|l^W~kAa3c15%j#NQ
z1$Mcrc#)IU$}g0)w5*iMIpt|nJ?sp^9}dw)fmV_%d~(O~3>_QC4CkwmBliT=fw7vd
z3$kkg*&8j2xaVXiD5jO%QAyb7>)$<GE+p(b(&$5<1bIqczwr)Hqe@qB<y?$5Z)rKP
zU)-2YUCo{1agY=R^o1@~cLm3rrnnX__Z6#~(-%&jWQ@>6Ycu=B^(pCE6jYiRE4LlN
zYxeXir%aO_)ec-na{%2-ES)cSj@KLdhf!@<>yGUR+A@6d@tap__9i1$^xWX`RnC!e
zT=5x4I<kL!U(X$o7tIq%)SO2iW)FP(6@+Szd-g!Uyj12K&5Cjg_dXqiB`D!V>Sati
z%D~a+1r+6fNh)Y3puQ_uhKCM-Xm)(%FlF+FeJgcr)bi@#^sW*f6j-tISW;hJ*rbVV
zlJlgi=7ITT(MfIC1P#QwF=|Qs`MW2@T$a_VWdYl7VrPdR1|JNS*G7{mK$&SA885M)
z<u2vrF(x+mLsQC>NZp}%U-o|b<93Z69dvuwZoYICs)1*1yH|nOuDzvSp4mSxoO|+-
z#c-bt>qL$mI^w;_H>SGdP0e_0mWpR}p;88;CH?Y=6R}KI=Q#I2k`FwW%~o1$2qa%g
zve!|D6<6ON;C35sKy|IN;acTFSC`}7<Nda`S>=>80aszDM<7%kEj_fZi}j1czx-vZ
zi|>|hBwDh{<Tj9DZzT{52Vbn-(qwOMFYGuyLQu==?S!_=I!Fw77^wy~_GcoT8PZ4f
z-LMR$zcrji=X5pLnHG5VXGpy&J1qNxPX-<V@-9`HB_%g+DSEsd6Fhqmn#H!uLA|Q1
zuiSYl>?DZLtM2+q@0ww|YTdp}#-3A9QlimHRkH`F8#{v6P0c{)QDltYkUV_xamrkQ
zTlIvce}(n}I+o8e37d|cxbtL21fy)q>uA$|7R3K0>uc3;6J`ny6WI@(+SCI0@~D!-
z%$-b93xa!<-M1&bj#gJ34ha#oDK3Ut&U)Jfjdwh^d*==dLC(18zfgidnCv_`QrebB
z2Dvh2PEZf+7SyhtA5d0c9(MnJHz*2t*}jhCz8Xl-ApKh@+>?IM<G#vCAj}_81u`-c
zo**~Z+{j5P!5^)#osk}t+ci}X=^(ZF@>G)bIcq7ErxVBr0G!q**>}b3at!O(-zN81
z|7Pc|^!8nSD1OQPg#vB!`JT6TMX>-r$U5=X;LfpnAIKQYet$RTmP_22p~%eqF81)f
zNx#aQW&P8`;m24}eU%{T_-tWs!AnIG5Dke}sRq%RnAP&ySK#6F<4h9FvFKlU0BdNA
z^#H~a0M=tB+k`{EvFC(cJ9GPj+n7{c4d)j9fd^t)AHJ%zZ~+Gz!g&3S{<#+C78uU_
zmfVN6dZl<obbOnVTfjr%oAf9v0tQ3flA=vksimnY@qk3DG1Nhyzow3%OI>CTb_KZ^
zHgH%n2cw(~M)Afmt>pW-6oMWfNA7^&4^jr{-bSe_k<~g(SyRCMUTtk|BFO-Cu)(sE
z_DptL%E6}pEV<x@ZKxDaYWZ2DG7LYMseIgSE}owL)u*8$WxU9?sTaBVZB~+2Yb1>s
z7AHG}-o{V{X@uN2S+{rYuuEI8{ci7*EeG?g)j(2fy4YkBa!;6yKEqj(I#N`EuVsEU
zZ_O_bENPIO7kYRn#7EU;37zFXS#DKcQXJBKk20Jfw$d|arIpxrX_-ujaQ3oVJ286>
zv%0RM>QM7G3*`Cmuv$XADoO0~ssSY#N37xZ?(|G6xnKIG8Vni&Z<D*uzN=+%PB}MO
zsY@8_a=f3bo*?0xP3?h+0Z6o9+dhH<_fuTw<FU`O`47wQpF{Mrf~MV9iX`p|_^=45
z$hXr&ykV&^dJF+~U<mRWD_AlGeFt6i(?cECxoGUYaeIX<Z5a_+tgtTNWH$Ye`UDsY
zDh%&-YQfB_fBIp!aa`{c$VqZgz(NJJmh!x75_fFH6v|CoB8S1>!#PtqdA;cpgz})K
zq6bfoiiGS%ICx6PGEj9+)9>(jo*>}f@Uq2iwz>yqj1!RnEWJ+-=7=SAZBBr(j==Wb
z)+g=^X&1gVk@lQp)NfsqZ-4DLAq@|tA5=aV8LNnDs|4vVb|`b1Mpi9E@g7J(0-PqS
zAe0{qA+4JpuMwjbg~_En0j}oFO8VLfbCHvm`W;P_wcMDaqfI)uO=aL#44q2S-a$yf
z$k-E8rFSg!lj~0i1p=ofjkJn94gQVBTTeZHk~Jybod+(8PLOB~+SGKV`{{A){!<-6
zh8P{YR=Pk^&&mEBzwJ~U)|zc4HE)KjNB;Lrl5Cp`VxQO&h+Be0U$Z)--RFs|W+t8B
z&j>?dye7gLQ*WB8oOa)VZjjb8g+P5jK3cvP^6P0V<KbRu#T{+6E|A#}0gSc5r>K`;
zk6N1Q^!FG9H2{hx<K_qDhCMf$AVr6kBDV2W9c4rn#_~;UnceQ6Y();4@lCtMY+Z}#
z0&$)u-R@tt`9R{Rni~dkySpJAPQ_C;hjO<~o5gIo6jPfBMY~o~9K1J@Wk%rTLf#8X
z-(c_;pExG8bxc?aY7duCPh*ZOv$4<D^pc)1p7Ne(hVS{B#vl<*v%u>o${%V|!;=&-
zv^6hd`ohrlIAnSy#Vddvy~q`yM*6;99MdbQR8TE6mVG#=f==Hia6W?EJx~;T?>lhq
zLi(iyAcA$n7??`@3evheO^~taoUD`(ZJU~hwV*b*>vZ7Lt?SVHWFr~wL7SUf(V@vj
zYV|ds^4<9wp$C<5IrcE#^O~q|+AeVZqY=IY+bnNmc4gY<@~f6U)#yKpH!TDYOu_WE
zrX~+$?D)=uW)4ukPB6gVxhUTYlVL{r#ep^11d3|QX-^XVlqApM3Exp{Ov4vKm$fb$
z)95h-L4+a+pN<2tY4}->d<y?15=lP7%HW*@Hy6h@e+d7xr2h<(4FOSB%;6HvpN8|$
z<@ZB^7C_m7hFtq|dH&-;m>IxD>3}--AIAPU)&H&*BlmtmPZ^v8sjd3AO1p0+&==><
z`qFvT@Sk%ig6v3^d1p}WpZxt{hqDy$nG>Xb;Xcl7?(YX*El`JikyB`WVI}{`Xj;x0
zEDC(#mH4+g29$vD%6K}Sm$v^|+?Qhpmbmn5kmq0LhygaFwUnFqPyYUB$Ls~*QyI7L
zB25109Hsccw3YNH{*(9%lK)?sL|W-38V{m?tMDhXE~m;tchf&weIK|XDqdC=I(fmZ
z**!qqt!tSDNHi&Cp*4Ml%p~)r&*YD#uYYT35{h{8E4?KYwL@Zp+vR&yFV`GeFV~=7
zlTfJm(RQx*Q3>N!v#MdcsB;kr>jEQQa&zHCGGbrx+jL>>0=#;)k9hSeGl=TP$$HtH
zHmmvszYsO&;jKcg309$2<W>zP312qHk*wlt8d!Cg12n0T<WmwN+@f9H_{uaQ^v!VL
z8@jOetTC&GkDi&1M&B0^AyDI??^)L&=eq%*oOl~Bs!DYGF_Ktwu^qmgl*g@aKkePv
zP4<`W6`YV}pWV7b=pVmxxl}LuJg>o{>=6R5pC;*?`&Xb{U+YU)6oNNqC`LGVCo9Uv
zw)`DGdp4IEE!NXM8T)>4C0BX}Nvb`#U<9GJI!FYd_`b%#x>ZcfHj{dyxq)C!m3641
zxiTuz*wcMVmw5M<mFSB;jA$e#dNjep9%<l4w#99F#r@~9VczRwy&FSTo_gA$*rQqc
z%z-SnPfNn%Y6u2%BhS@>;)?}{xD=@S9|KVLYz#58pZMBH7_2aQ>T30GSv8o3f6%X>
z0|TXJt}61JFMMA$P)8^^WEDO|TK9|P!Y#SpuWfOypX!8ITGqjHU;(QQMn7&6iXSnN
z4wn!!(EeEqls#==wWyJMdR{aMr%j>V(mNCl^LEJ!201s0dG%@Q?tVYIMN6T4EyL*O
zC`k@%RQg<R073tW<37SEXt$b^NM|TU%QT+c?SAn|Zx^8;mglV4I6y_UR{Lu<&T3;K
zS1$;8TXs<w2(vl@ifmOC!Yzs1O%+E&^d@XGB+?%rysA8j3$8RJC1{ALq!~5UfPOgq
zfV4MCayG^yRW<@?kXaf<ms273T?$5tFmKv&-c34}k9a0Sr-Kh1Mu}>VPoIosH*IFC
z#Us7qKt&l0yWDgaSuJ8M670Kcb05L_yB5%Egt=9xS0YzxhNJ&0^~S0)mr5Py`PPSz
za5?}&635vRBoOk|xzsFFY%1$=GOM$k?exntfDLjO*ikBhqr0Ecr=E&5{VW~W^Sz<o
zi`2xKDcrKG-csS6KdcxIw*i}Q1D_$M1gjkR+J{TX2KC?D&Ayj!FA|X&!F4#XC<I2>
z$l?a@XdLsb+71l&rJIb0M}xR!g&rb71=ShtwgV;cNM<^YLBsD3jC$qkcw3><H}tCp
z(pE%OKYQ7qZ=*sA1(A&Jy9n6vMgMdUsmV;%x4vT7XoYO+Ntk!}U0ybxyRt*yKJ+)P
ztEPmN8E`xl^M3gLd@|wxHNOs+-_a$^d$UKUt-$F+Bw77&p@`_hkvUHM9_^rY$|jO0
z+D`==`U)2j0T@{-Z6=b){b>9t(b0<ucNNB0Qe;-7f-*SzJD{j!C_dm>_weoiGXbBl
zWnDJHh)&n#RawhNFR5|7qA#^t{8$%pH+8INdyw{gMWUjq0`H-6L-{GP_6ojMc;^w2
z*-YsTlYOTG(_5!n4-%CfTRBcRhoA2RQ1)bl)hJHJvx*c-q`g23P|yo0D(}=}DpV|e
zkgKam|BCqGgXut{d-Ex!2@gw})(7>1%Wl}KKoyOVAW{*~u}_SQc$;XqQuqX$g{o{O
zcxOrLJw{hQU?XS(J8BdrNX&@NSpbd!gbGXF&5^?q)muc)Q@p(!&84c0g$b8b1KEZN
zP3yx7mSp3)wMGSJM%gXnEd=?VMV!kK#rG5N6QAm{@sO4giCP-lUBOD@!{810pf2HM
z##gPpf(}bWMa|kxj<*5_lKn4VuweEapd(d8O4)^fq+}aG3)nlkZ3NddDQ?ekLiHyB
zrm)eVUcow=saG|s`}Dk`isBmsLY|WC^=1BZ)=&YmJz{P?bsnzr%YK3xi4w?S_W~#0
zKDMg3hv<ULa260`Y~L90x@f|nk$HA(bOm@gZvjnXm92TL0Ja=lQE@nhC?QjAaP#>V
zfUuA7AOJ+wbQL_LX~XH*)=sXa&y=<=e}t9d7~B*FP~7~Pnl&FAJN!?81>71FFaQ%`
zDa1vS@H1gFBlzKC{7LVZ!h334nOZ??IjmOcF~PqW1r<B3>CZQPeT3VAm)AO>MLEZ2
zA7KmZuo}-bI*(%yxf@G&UrC`@DR<m%Q1>jzBtUipmS^*lK1JL;_YA_%-eUko1qR#N
z$n^;feyPxV9G9oikQs5q;)`bI+W+5aJ?t9RgX=5gF_rTMVasVAuG{0mz0m8@ssieL
zTukB~j+WgE$^S7Ohz%oU=Px74JV7c=&o3^bFc)1_M3xN4kLKK(ITJxFIwG#(jkNIN
z?)z%7%5h1BcODtaM9wkidHfm5mjJAVlAmeI50F%ndiA$tj!my<xfEK3R2uGo>Ms54
zQ*dE!*=GgR!%XS(gAJ6zX}%cElcmprI?{Z#lJu3w#69l@7Vg8385A!@|F;Q`_QZQP
zmW)%$%zoAEYI21c$mosMqgBbDD=h60C}Js*;XF$MVa?^6M5jM|*?)}~kz_X=JaqiN
zh;F#(C$XUJ`~CuDOVR+Pm#*l>UNqtD%RZ~e&T85}T{|4{C72|w`kur7yh~S0LwNY2
zmx?}!xILArk;58cu*TLRikw*|gqgbva~jn5)l1}RRJ?A}T{s0{Y~UxfUiLU%G=Xr&
z_pyYBUHkTpnW_9oKm4jivt9#px!pT7>6big{6`(AXa}Iiw+U6f5Ev@jwR>!9wT>TW
z!YxQMN)I@pXOY{F2g9?<6-pPxmV5?W23*lcq91Qx<ypNwZR5Vgp1Jv(57=ZBqVtp0
zkZTU-#TXx9av)TGm1wDRG5ZLMUd0!5rEi`TR2Q?2FOC@xX@7T3f9JrAlwCr`Nk8ji
z%2|+DwJNKRAj5eWfb#-!GMkAZw$*$S(Xq4mj@jAbVW<iLl7XGx0su^ZZnEGZjgIYC
z6Nk}p17BH2Q_qv`VBkGIP#+caUkttpFl!dcbisHad!LD%-h7^(&bAxfY-SU!HNxdF
zG+q3>Tf8J9Tj_NO(ke<j%b#)#Qjt(Nh9GWz;0F6iYidc<x1NLR5qGyvDj<h>NP*U6
zWnnyyxY4D17pp(5;1br%WN?vl2O;}Oa+S5|APM(bywn!i%VpX8>2?m~g*GvG0q*d+
z?;~SiC~%m&!|W$T=vt2r7X4Zp-}WAri|IQeIr@A6lcn#~S&>AxL*K*B1(ufvJ}4gw
zj;TY`O2d<c!1783foPz5j#v|tID3+TG4S1PL4p7?bKf`I1+Q)Q^2Y`){~aRgROcLJ
zBW2!8AV5*ZO;5XaA#==u<gfb@yCGM*Q&pT-KZ*N;u*?*MWsNWJAvj2E9)ja$+o-dg
zNCY!8^laFMA&%^xy3|#8sNb@(N+$^YrGIE@D^*ONkgZ;IkzS;E0f5`#<I+MFQ#eRj
zj=7sBp-PSW<rd@b8B~QFs)$aS_|mODJ^7@PW{AYxq6`RbW8ZEe5nfVqmUiJ!5h{Dc
zO1J!Ib~2+N2W)elB??7HA9gOflE{LWpP^?~UNm9i)C#x6Z5fS=Y7;WD;CRic<`e-t
z_OdSNt~FMbR>nl)69Mv!?WwkCq_;~xe|^Qi(71oPqS#?b!x{5<>@88%k9hXyyH+H*
z6I$m-Nrcm+&Ju(NZ>uf}JwC!GXk>+Y1T*BvPGI>dY!>E`JS>R_HjvT5Uxiv~g!e@u
zbp{9RFxb(B?Xi&v%4Y;wk`>63I>D-#s}eUO71YK-)HgLb*XOAXuMRxA&|Bx3H{ZQ6
zBy`b)@rgxoe|C=O^d|wx(wJ3Qt4S`EDb$JM=uX!I2+)2QqI?-?vdFuX!SOi!<;WOW
z501k$$-2yg<&i@bDiiIgLxH!d+}Pua`}*$IG3Cd7YohZ#4X_$Z&o+F$a7r#92mm_)
zoyWFHnD_JUMHj?U!tP(`@?Y6%)Yec58Ipe%PJ;qlhX&ao$sdpCt#AwJaPzC_mAE(M
zaH1XrdQFaJm8zknErIM&m<4VL=H5jtMfo-zoMV;Hd2-?0D0|${0EQ8Ops?VP=)6r6
zb$jzXdXFIgsHqk!hzj*f6za4D9qFQxz~p~~beIk|i1timUFN12T@Kzax-1kJ(rrn_
zV$*C`90i-DIyIT4T0w~(EYII)Fo|nK)46bVKEljr5TK}z96slR^w-w5wU{q`eRs8Y
z<D}u~{(8}LY6}&OM^8<|+nkS9vuI61*y2aoPtvWbRMS1HqVJ7nEuOeZTwpd{wl;_y
zC$5JFA_<^p8?DJFlkFD*gr1(GgCCS7#@;?fi_N5PAv<^PX4K$4#_+OQBu!sn;+gLJ
z87PB^RjiR~5&zjC9~{^<hx87qBU#&$Qd)jKvxWKx#q9Q7b9^MrLI-AnGv1Yqtk>)i
zplgOOtP5CIj0Fy?>JC9TGC)TI+2wPN9-e37Jug5nHBwu|`MFi#an?XDNACM1V&{ej
z5(>Y#Pfr<wFVu>Ph7NocPDiKi1sx;+C;KteGr}*LF#o#(sHFpXO$(t+AlDE6RB<n`
z`a6KG$k;qPBo$HiC;(XtebBjp!!%TYkk=lWC7kaDgJ9YogbE23{5yf?<O-kFTa_9F
zk;I7cf)bQJQL7j(Cct^`Luv9hO;z-qaS%$S0BH!cKDsbLa8MG1JzOZz4M~Z@{=l7)
zc%VGI@S?)3KxTR`laS=^^Yz(T8BA1@>_R{<r~nn=)mj%+fI=5==uz}57qT$p1$_5k
z%ejthn`U5qaZ>ix3)S^_gKnq!I!Hkaimwi!zw$E!Qns!Oh*&Zt;`BQg4~nnaS>Cg`
zxO`D3hXRfgJYVe~XJ=Jsh=jm1r#zRVFDmGES$07j$#gMQB!IsplYm?>kSzfy6L=HC
zh0K5wU?7IXI{BP||2_f?WZ?~Z9g^N+zL29LGZ?J&e)NB+2^EBvmoXQR_Ij=X{hIrT
z^ob#nmY@x6Z+s@#`CI|{XT{;wjbV4BUQ6JJy7CqOw5#X6{^zmqZFMl1KK<|oPiFjp
zvYGIB0%XK|4j7;zRh&Rto4_~ZeEodk?tqzT0j0e7P#}LnhCda6jXS2L7g7Xk^GuEr
zobnfKW-g$m0r$i8izdpR#WT~O5LvxgVLfNgz?9S!DK?;FqXXSU4Aib(XfxrAiuwYj
z3o0-$J!mL{4?60Rl4KLG@;S$XmvGNt1hM$p8R&|;{m(95MF5-HPHYRfkRm4DcVHTk
z5xGe5`G7y5j9grV6p1u5bHJ1gZe7>8962jNuhuIXdq};q&&(YW@s8jEd5N;Y3Qn_I
zyQq%<I5<N+RX-J+b83BgL2ftjE6wcYc@rvHJ~dz<9~@=;!+`G)xC&4*<g(0+)XNdn
z*>W00dhh?~1&TMIo61!sc4RjHn-0a(GdrfBVm@~qK%1=qN#q2-R-Nx{^z?yVG2~v!
z7n&1JBM0>Fuwji};0+}O;9Rgk`h4Yp++Vm342H<GyL(QpZ{V4p#^0K=oj0MPbpmag
zm)#hoxL_a^z@;Gc^EnY<9^d~q1b?ufM~rV0jbvxSX{LaVZmzD2HZup%k{*$^d#*W9
zp;!k@1yVdFyQom~fXE_Kf`k@Ho0M#iK(}wUY8N5f|5y|lm@2?T8Q4}XbZ+(lOc{AC
zd+mY>${rUmRx>FF^F>vxM;~<ahUokUoBvh%1bcx19G~HFA{7IDht9xpcYj&nj~qc<
z3Bb|aV+VO}<T42LIloex-}}=G#rMBK3E4=_l%@~ptbLV)q;4eNb2bo;BFM9ZAC+H#
z%!L&HHi$!Df7{fve=1g#xW2Z8vrj)g_{*aRBd~Ce<Pl|Z?SDkg1Ax-5QT0zof{ZBt
zycF<{?7^2G7@})7!(X-IC9sd7+?}5Od%s=y4PfMs?d?B%K8sHQHd2M(I^(w|1AIqA
zrZMpw!vBk4pcwK`ZD$L^*A0kzU~AFrcO3lB2-FQM+-=Djj5En_<^UU#x}sM8H)p=J
zfI82sJPN-B_+1nrP0-(BNJ{~0V~3x2^mpL!k7~IuIrv(j@6liRgB*y~%=>%?ng3oO
zX{>-c{m17>v<b94`{9%9*|*te<}VQOrtG(J{!g?RnExkQaR2WgdDH({EhgUo0m6Sq
zZ~uq2Ap7|L7;Ak+@GGW29ZmO2w8*N=JjU)MpE^E#aEq*v|K)CFd5cA1jkYV@*lAlr
zXj7j|Cr7XCl>{;1*6yR{kpAt*zh(euN~7BGr)cK;0z?m=K4`=m^CLPeY`cY0!U**}
zIGQ8i!A}U{sbaM396BB@ggK5(n%E317uFF*2yNJ<P9&Km&>Iy!Sy<VsQmdduc+tu(
zey>V+{Ej}8XvnIZ`?Tw|ZENaU%HsHZVgVjua^0|43nQ+Fb&9|YBzE!>f7EK~exj98
z_r!3*Vg>p?&RsoI0=_MhVczX;cjhDf5V*-f#k|}HDXzPvKNj2<nxmUE(a6XG<gF|$
zEJCfyw5YwDX6M;kR{EWJ)!SK5_ZO5mW|a#Y>#;pa2nLIpT?ajTshz<s#N+|f4Q%E|
z?);lQ?4EkVD}?+z+qvvhkB_3OT~B!28=Y%wln&hIy27~;v};Gh{d1yONlu~}l*K|K
zvQ@i?Jt3B<BLszB+$y$V+x^Z;s$laj+d;<PZuczHPZeYN6YR`_WUdvp)agkkF~-#R
zVw}s)phBz`36Gsv-j_Z<_ML3{=?|x0@FE0GV4HIK`ZlnN&0_Qy!#ykyyV>0$h&bj~
z_jd}*!r-gbda=;XjnW6fQXG+iIOgy&aGS2;S<=gKu89;2h`+??VH;h*y__jhn+lKP
zgZG|CPd@k3v@{ueBTVc3;tDQx@wXJbGhw}ISk=Zk;`|?hCj#t&ZmYLB{*Li{UjXCy
zl%v^SJn6ZYuHlj6KYc(r84SxLB{oHP#cJ3jLk#!VKg>DldNtM9j>4_O?7PO9!O7gE
z%?&Mhi5A}0WTI(fw&5d_WzA;+s@KEornba7INatXp<gnS>|DMG&X*AJ*tv9bBj90_
znF{v^FYaag5&&9C7QZfY{T+-d8Ub;JUrBQtggbwA-AKGu@TTPW$13;f{lu&NQ+@ri
z2?~P56y}<FnkMm?{Z<s-{49Na22F-WO}7aaZE0F$j3z^_P<RNQY^`IDyf?#UV$)XB
z)3VO1W{!?;lOGdLBnRi2H@sGg#)d<nL~2a>g5c_HGhs|z^>Iyg^3lnT?D5uQR`HQ(
zlC26;-T0S@+UEi_7PDQ|<g9`{6a?)oTL;qu3N4-G{CBWf`6WCYJDZ?aM$OjI{`_A6
z$L2#l`Q-?1a9lG}!+90*v8Lt+jKM0J!;pqL*mbvZc&2he@Y-;UhDX6`t*x*A-A#OO
z4}5U0(xO9f%;8qD!WF^emzMndJ@!pSuI%m<ZR7UcMA$5B?o0;(l~AAbu|$UJlN;U<
z&4^PkE=JRxL4r4PG0EjEQzA`P=#!owz^#zVg$X{afX6!CJGe%xg{Sw9<PaMlJq;tn
z>wcJ4`o#&p)<L?KucZQj;v;(D*x$w=91d3DLvo)b6(O(VS8<+e0>#wPWj{P16+g!X
zXmWl|nq*`UcCc0~o@lzBemVNi)R@i0FEA@K+aoi*f2%NlF_{xqrK+Y!_vVphzlPDy
z69$`5*ZC=0rSCmsWnYh>O-8t<Rbw(r!O&kcEzPCF0&Kkpj>DP{SP(N_?1qh|tj^x>
znh2u=V&@LOO#Iy>m(9|-Bd*bBm3hfW-lxZ#MNbG1bjF>#pC-4_rQZ44CF~z3dL3bO
zXJ@P35CBtH(Aw#o*u%oZxdy6B)Yb6A(nklcR%v)ohZ*6ax_ghPv=8%_Tk)$ry-#gS
zP=DYndzh$Ju`u`U>|A^E27f@+8r&pMz2mrVQZ&Y*pU<yTint|TRkoj!ZeKjYrXX~Z
zNVl^wmY7dOm{M+<x`SzWc;C*Kvf1W$ByaW<7;otHFV??v%7<q;f(yzgWrQ5}DQ#`>
zZg+A0i4qQ_(q}gMu~`iED<?nw9?k#AEne!Aea%T_46Y~zgR}kA1$7(WBvHS|Xy@Ho
zNUm|uqJa?e!$iDCyQ+V_S+S3YEWCa~7|$`La0=Zwv~L!<;srgfu`^9vRd{XRT}PFO
zx*cA~O19O>XTiU@C}Z1MdfLS@W>7si6V_j+T?@f)jJj@Fabm}Qw9@~?P1D_6hQn*<
zc++qr*Qg-ac7~=gm1UB)w)E)~2b=WX-mYdS<@~SHTB1<q14N!?PGN3di6T=;&&ozU
zwoUVdxA(m*>GmH2hdqU~30I>JPX`^n2+Ai;BniYJQz}|rYX#L55$<qz-ozXBgL7A!
zM=d-*lw0F2j=!)VCoiAcFk68R^KK;S+ax1u``}E70zG%wfM5f=NQci4?PVGfS&Q)U
z$=@loF_3P2{q^5j>oYuHss6#q`l|G@KRgSe#aMct8&Pqt^Ih$sJE~hiypVY2<-D5b
zI-X5gxCoh@nuGbHjWK1LU|lCY=rGrgns-iKTlZ9oo99Z=9!0UJ`L063C6!Tm!MVd9
z_NfZpkiA*U^3E3{Ic?zj318i#rLmDd_5Q9BWdefy@--#fgVZ`g(V*+E2kY#_2!!u!
zwKE%?B8C`m3uva6Vj6n6&FW8oZa7V<p{46&A`#e=ofFJ3t~;?C5h^~hUhgz+Z;|0^
z<>tvsn~x#&N@6yO>2h?yFDB-y<C~d}a~gfOv2v@$G!=40v$qwkKKXjvU2<pKVdC4y
z+(Ux!7IKPP*vwACYTI+(BSJ%`%@1nSsDfe9Ev=wHzrpw07yawf_BPuvIQrETv<6G2
znGxi7em`Z^r;XP)*7qD{aDoH%;s!TO8h)4uv9`eFC(MP(mhw;5u$V4Un~y-Z1(OQK
z?#bMV>)e%u>aigDy+&Ydl=H0;5uvU^LNZPsU9P(zlyJ-KUx~G`c97L^9ju#fxnhGK
zL{TL*Jbh)m=0Wg?dv_`HWrl@_IZw%Ixo%DWuU5?j%h-wT;C^p!Wm%5SsEs}-ZHtO>
z_@auFg1!>w`xx7!pZ*Hdi~1h4*Y_uf`E(PS-M5e3TdLt+_}u#5-^JRy9IISz)+M<e
zh8K;cm^R~ZgoWTUTqo{JiB=@(2cg2lE<yWYJ(=5$+if$?K^<bB>~|4{`7Dd>^&PhQ
z?arG)34jDw*zwbp^Qk@5xg?C_;gbx)ZwgD`mL(8U<Pv=Vfw}k0wb%&wT8T;7m*LL_
zqJ1B{tZD|QbW!`ei|q+c?l^Z06~QN$WgS{0ygY1=k7K<%V;J41-x)!-1TL||AnV<A
z>*86IG&Sz?n{|_k+ISpK{ZE~rA*MT`I2p6-5|b$B``S;5Np0M8)W#PU;(5H*5j#|!
zUh$4Y=8U;<W_5N_P|ir2n%u1An3#GQndu$Q%esYd_+DZD(N@K;=Ae%jKka%VSeRId
zosBd0`oC<})Ax?SWk3;lm7rE0ufN{EaQQF12M2e-MvK6S+wWq;fHWwUFdlpwa-Z#H
z38kReZCUZ8Hxhcpr)~X0iD8i$6UG?qx?RNnj@;vP?;T;gY9@7VAK#O{l{c#DnsgTG
z2hX|rfFyP{O;g<k`s~cit7q$LjQVXnbC#)}YNh0Xog(baKH0b64%VAvSms3S!t;{@
z({RqvzqqEUS)i`juH6)PWIi;|#>h-GW?iFYS=Nd*;;zoHEOP{#;DeFzn~%A*t5zi&
zI4)a7G;(y;#xNsdVtjI&cbiG!EtdAH<TEV_!9x1QHUultZ8cj3ErxT0-U*&H^KJfa
zp4{P-nHlrY|Ex}To|V?uHU{~A+khSr$`0>?$bo{XR9E|zFUxSKdy41gt2p-pVp`bA
zq~F_Gzc`iNBe?e+??($|6+b`n&A|8g-_=Z|5BSJ&ofsw^1x*-*&<93em)joEM(DEE
z&YY5mtJZnAIPH(-df{1kTD>lab-=nwehM#QKlOkPfeVEu%#aZo#0U*-Fnx5YNnFOp
z!E*GT$#uw#eFwXzj^lxE(ZKt3nV{lsDlaGV<Jr$bj=V$tv7EKuUl#kH<z-nMx*vB4
zjg#<s?QO4YMk@R$i0plJm8__tD>Y@=f|wuF6q~g7c|~Ngacc|ya!~&)7b-FWfX$!H
z)dF`N1<2a-UrMUY$JZULiNT3sxy69%6Abp~xw^xp>Wu{%`0IWzXQSgI--Tjj_the&
zv%oYu)^QY`e<DvvP?xc3nrvUbvVLh{?S{5xt<tc}bj3{J8#tMZ9@p|LqP*+73Fcsb
zSlqYkhSI|3fgR!80#TPt7IfFW-%W0COgK=5!VP&!`<3*)58DNbVC50r*423xD&SUO
z7}IU`-UV=#{#Gc{apmhbxU*k}xgYVKIPD-jCwaFz`WxdG<XP&3GMBG`F!)}r0)WK$
z;KtuxBke4!$Y<|z`@QuvaCX*eQvCVX)q`WSBNLe?`DIJ>w#(r{DH(Qd*pF!*2Ggq^
zZ?w!9?iAj+v3ESvx<W<;)1?wPTkjJB<_Dj^l@i*sSm)8K1gX43tKLxx87*SBO2aM3
zb-LF5O<NbXAuk7F{*{Gc9eP*i^+~TS1;glQLI(Zyf&Ri~5A4-g&$`19cnhxuEBLP@
zhPyyCNCfq;E+^u1@+fl{Wor5iS>ElcrZ9#>q*`pM)_(49l7u?d+&^LohQ%~}lLjh-
z1bY59+yKBFGv|-0-`l1G?*MaeE*oto*liaUhQUwQ>+{}DrIy*RR3Jzu2=>RfY#J?_
zVoSk=wh|AoZ4Y+|-rB7<`fvIbMshs(SCdYT5)A<?e}9A6TB9-=F&3PH9fRQdCX9?}
z!U|UF;LUrwMr!JCRPfGW>TxI;NNNOWYODJbY)5UJR|>M_TZXjYgtmxlxXq8TyT~K>
z9b~iYVIn6<sZ)%4?yD2>a8~mH%K0pb6YA@H$KC99v%xM~(fzzN+Vf}m2W$Ru<XJxS
z!*jUsiB?yx-O+5M?MU9c528UT<XyY{H|-en?99r%f4RChmvd_u^*=m=-!F*Wpp9gk
zLi~kPs6>{%s|OF<kI!&<Rle1WidE=z40x?7Yx?m-*EhOQ$olS&RSVm`jWI_c(Aw7d
z7E@^PkhXJ<C9Kq%b?MhL`q`tA8;LErxLS2Vxp%Dpv4i!GeP!QPu!|v^qE9xFtoJg7
zylrZ)(DYtShF6{EP^aE;ZGG%KD@*4fIgEM_Ms}x;cfRe%Sxi<=YprjMAki0iU|q9X
z$G)&&5)5KkV_Ck{?~=GR%E9kV<Qza;U{MXOxWkGDhxqx=qTxrI>z&0%{Q>kil?O3F
zMxpr%A?1^YY285we5;$)k7n|;4FieAn}pVQ@698QnMR5=aS;mvOp+Q7@49645j$;1
zh7C4kiMmkWrkaZ+Y~pu#m}{aZsos+>De%m#SIoqS8i6>ob#yXYq=RQ+f_!Z4<%^*p
z8J2NAF2qo5F@m#eYIKErgeltr+`fur8qz*h-{_OEX|ipxB-1}ynvQ>3wcDDd_gY|&
zhp(V=cS`f~RvUI*XET2?qZP~<I48ktyS3$Tw~>|fcT_>9SrjMm(2W_<QzG--K|%L?
z<tpQ1K^drqny&GOiq*{2Z4EW82h7aLTp(s`3hho(SyP_7CULS-yuV@K-4O%s3+f&U
zK$sC^L?F<Lj+HN|>*Dqz9@g6vX8O`|XT<W;HlH*+i+F2uY;I~#jGO15+b*BLo>2}~
z`r|))50PN^EMDyMVeK)5-KOb~6v5laXC$qiaE;ya$J_nI^dgAp$a`J-3_1Y~g?1On
z+?PHs!^Y1{UtE5ETe#Qhq-nTOIsGjtrMt`VnxV9ek1=iN=~m@*`;=*_=b_7@P`9aw
zJa5WU-I|TiqAhWb!~c)Hw|=WKdiFpCK~O|c6r@8!q@*OI5u{PNyE`Q|A)<sx2+}1j
z(%ql}BHe5nl<wHTrn&1y@O;lX&wcLw1I{lzz~1kAXU)u-HEZTGvpUyY_=qg?kce&m
zC7NyXCoSNRN{;B}vOHdxYFOs*rJGQUqYvMM4yEPCBi`h0^nG_!&7UshP8h5(skZL*
zpK(~=A{<z+sM!^rl<Z}BF_teTj8yU=RxMeZ?=~8Qq`GKG<pM#;z)$8`N-X<5)5K*z
zc2Y<+eB|)WEVTx1uKm!43%%#s&cVcl7dhmEuBshkX9ztwq1AES>RcTLRm8Wu1)cvq
zqm;`MIozU&j5VoFpIR*NE;CM_&dORl#yVpu9|c`mmUf7B+%nMVs&{mef)Ve+=|4!N
z7~R;`aobG<Evt}!y2fq=-BVT5ONB{c^wk8vnwaTWwLGjT+!QHknUSvz?;LQOd#Bju
zZ7KynvU&l`oA0dzlZyjY1OpI@mN4=%XLk)s^#y2XPa%H$8_saiX^}SY?;g4qaTBMO
zW?=L!2_NQ9ok*?LAbHh=6nLXgo6jbX67jN<yhPphq}_j=<^|WsFrAfd{=qQ9nF1?p
z>68ODJ)Nv|$NZH{T%_V?pYIW8L#^7L9{)QGPiu4Wu8^q{?I)UPPS%eMPiz^9vl2>A
zo=Gcb5Okg`wnR6zW@)M3qVMwr^}@0w+l|<U1^h$I_aZ!LqhI!KgmtF6t!LvI<nx$z
zgk-s-q*Khu6Kq86a*>YPMHtQiN03-hz%`BgxJR~lg%;_}Ts@mVY>*RCjB&Wp;B4~5
zFss2>=b*w{EROTjWKcx;-d?$yMQiT}n~kd0ZUJ{~rH4A4f>t|PB$}Me!(M9-?)|f-
z8JVMGlbmcizu!tp+)}PtFuIl-?qp5W?d#?A^7ynP{^>-rNuX8j?```gd9Glku30dc
zvo1bGrbCj)PUZH^sP#74_KeQ$D(wgF7|oXoOW&%wXdY_Y)Sb|e+;Pr8^vkdtKf$wV
zXHsD%UEV`PKi9qOPws5)vHzQH4nc2pthwdMqpN1HzRmg+n-Mr!)8G4@{_a(J4q&f}
zNL$eP`{(O#Lcr;|8kuu+)aPJeKL3NgyB4%iI|T>bl5X$LWRuT)hr`n95@BTAID~zZ
z8(x_hGxB@~4)*l96^1*DvBelW5f0wjsT`J+o!=x&wC3YohRCJ_6{`i%8M7x<bv?QI
zWa5CXZ=Y~wtPq{Bcj8L1vroFt)Pb_l_7>T;sHh?L)go5b-2p|D5`TrBp(&<LCAaNt
z)e;J`^(xbjpEU1;g9|f^he2ucZj5!gdC8P`++1HW)4Lbu=CyZ^fswq~EYRI&s5h9r
zt)*xRP7%DsxSbbJifaHO#IDx2e2J`30s*&2oU4!;3o(F5tcL!l-3wIoqBw}hV80i=
zjLnS$wp8KeoJSaz0nSm;^lB>4-w(g(fqeKiX$}6%@BW7~V1^D>4#S6Hfs3>A1b9oe
zu>B=2Dy#+o=3ET!L5CorL@>;<l@N#jUzp8(&=9agPcrA?Pl^{%_2od}`3WHpf~vfG
zIrRL(;UETY;d=NF?c)GoH?wlNsL-ch*uiIs*a7nw2Q6O!TOedK|5v8=2uv)JXb(#M
ze!d4eWslnqCH@9d{zr+D&^qAec%brm7TFoV0<ud`dPy$Ss1Q8rOOr=-p+^6SpDKL=
z+@E{m>g0bROFlN>s}rBOFVSMZKs{B&)1n)Y&Hvm5K68FT5qAOAxfuoE8HMSCm&lzj
zAf2mZF$`tDeN;eVwe-isl?Q)a%AR2G7UH}9ms>{_$hbSush~ig4uJlfnO)bOIbLLb
zzk`$K&8SlM%kM^liAiA}LGtUl0e)AxZ|r#iyQsd6+vMc?=d#X%!Ub@1z3CBjwD;i!
zj>f>qYqN_`$YV9|==X4yOO(rh;-oboq}z?O+|a)|BE$v0YV(5R5~DN=;G?aHYUt3X
zpb6UlDeeC$?f)q)Sh@edEUkFsn)gocaEwFSZ)~^?HbC$Q$pWLsIj4G2#QNeXShQ=a
zgBOs`(qdo@^q-$)#-HImUG;E3hNAKZbMw~2US{0jrPqK#VqctnbYTIO0WL(5d<GcD
zb8{h-1I#p$O;bw<DY+o`ek%9+evaYqck=;3EW-@-g-zqwaGY}*I)N$UOjEt+)Aj;F
z3tOKIUcf?2zXBRk#7}{98p>48OAqBqiViXvnnq05i5`gGwEQ9f8p5KYSpdV3eSm9r
zfxLVH8G7sS!T-YJQlNt7s&cEK%MUBmmlCuKC52jg`Ic(2U&l4O=2Ko?o=f~rB!#d#
z4d?F!K+PJkqNTHiE0>TS2vx*Vzi(hwHmG6`I=D7{&B-T<)_$c9wKwscn1_%;UqO5>
z`%%`9OOwe7?Bc*%?Tm{tQBZdo?HaMUnrP##?%>hN1b{xDtmR`iHSRX~0PWJ#1Z)~D
znU?ki&Q1f`UD-tQ(*8Z}dA;^%y{2d-zvkyV0-gI%s}U``@93<cgUya}JXDYHOBU=C
z#CdNnfv>(`2O{LKxc3j}G*I#_uVONeN$iP9N6av20XZ{l7#h5{x94tC4AFoJXu!X3
zq<taRIMBnQe@8Ue_a6;)*FO+*F^4=|ujB@O-VDB@6Lf%fN}x|o-NYZLrNK2o0lV42
zqkmQ#P|zBz^8MccyUd%LjW{j%npU@Q_%?<1<YGYAGCGiZYjd|P-vcCwKqL@ZX<y**
z;#2?$rJ^~8|3;|04)sAP0`2>nocd)6Ko4AU1JEgSZOvf;+8XeH_+^KcFU9`_(xR^k
zH~yg)zqt5B>$uaGkC?<)Zv7-S1YLCAgU%UzAW4{cK|(4-0;%i71<)-HL>l{*8nfB|
zgM@JpL}w9ryalwkn0bpQ7eMMVUR2QzwGkHTgjN<gDmJtM0+wLPl!G0*`1-wzKLL#!
zs8EltRnW81e$s8B8MG{Gt_1yJ4C)W&bC4#M<lre0+?nT>514s?In9KokX`sEXqY=x
zcAy_y6KLv52x;XiP$PQ~`*{w7!B||lK{dY5A*6iWYG#)}RK3q&Qa)`}|Dea9vIz9C
zOn%3VLFTFlwIH?s{f^-0b~}qM+<FKq+i(KuPN#cYi5KhYf1X1D{6uv6<8!h=&;gJN
z>XJKbmKI!0(<&Z9GSZ?g2OP%Fr(k*Sa?xsBx-l_Iz+Fp!U-hpf4A8(_y_h--x<wg+
z-g)*z>ZZ+H^;^N!HU&R{&VJ?s^S0&v*CB!kn3?|jGwJ_T{^E&Qd%WFVv{pX$6EsL8
z&^_#n27Q3@p&n^fp*0DRo$~}v&N9rS^b%Is1K~WqKWSb1OB65Qs1*~hG|SOOK!4o3
zfUR1f9W}G_gbq{@EaVM@c@pVB3JDVOzW!JBP>=sN&`paRbeB{;{*B`u02;S3GXrU=
z>G_}^R<@gi=Qu)u4%+{elD=G*{J=iChubGyuvQ50;k?mjmrFe!=$WSV<m!AAp8N+&
zQNID_cut7&2w+=cm001I$H`@RU~#<-Q^o#ECoyGW9#bkT7N4CSwJi3gm8d^v$SoAP
zP=}vTE%ExW`DM?7&mh!DFS&oQ&VdUI^MRV{#G^X`&HM>^(j~uptSHN=JW-)}6;S;c
zoNhaW=Eg69o?jp)o_u&0^&c9!3H2o`pLjI}66i*tr4M1z_iMyE4rPjfMBx7k%Q${W
z)4BNme<H+>1z|3zwu-w*`{S&e^dBF7WWynRBr0|FBNf#hhXTtsK4Y1W5)auU-Cj-P
zVA9aMRN(kb9quM!LPgESzff`9?b%g@u=A@!9<0Jw?7K#0G&Y>qy-wFQ)GaArUqQuG
z)0qa`=&N;eRIhf*{=}lI6UC@bRpl^O>`_Hefr?2njfL{&zV1VZTWGjKS5UwYib_;8
z*6+Iu%a|0<0ML&>%v|*n@#UdB2Gj1ONYKq!0_<&gBk{7})7QZA8YI9o2QZ<B!3ZMq
z;KZ^snn3j6^6@AFZkP20r=`B;o5CuYGK8^DUGncTsB~=4Wxj?+C_u+F@L*~?xFFz)
z&o?aGCbY=}2CB;uFsZ3FLA!J*SJ%qb$vRDq3LB=GpTQA3OZ{{XXz|c<3pa3^xGSu)
z3I9@z`8MQ=u*_m9C#Tr?q`(hGCcA@eV?sf%TBo&9WuS1(f$!gkgI3E;ea4w~5FHJG
zj$@{dgWG=@aRucJ(6K)i!|;z0nADVojtg?vv9KbXSB50>G|FZ{^RsA1^<rxKyYbL-
z3*Zs%hTv?<3p%)UK_7qvYZ>f+gGysDYE{0VqN1W!%#>j>Zo8YU+fbj;F8v!!!$$~A
z15F|Ruk4`Lp56v|!cYd&f78H~5SiZJn(0dw!%{Cc3RKEbku~W~VyXgU{dxEV;&}><
z5H6m^nymH3E5F48fp%G`TD-duC#IUlH0T)>#_PQNYpFkLZZ29k%I7MEMs@AoTR@1I
zQGj)YvsKI&Q=piB3RGlD-GTN}MXvZzqn(~0b(e<n;=y5E5;VgTfP@MV5d4xP%_fi*
zIRR#hRIFw5Tr%Q0U<8k~r0k`XQKYeQbsKbk{P<zC(3RL%Y%Cr7SpSXDudweI@(?Nn
zL%f+l)gr&pEL@2iK-nlZ&(%xHUJFs7s^)8b48)_8ie}cPmQUhiGwmYDR?1eOdR}-P
zdhQz&xEvsA`rhRjg#}lHKX{IXg#U+n4wsmQE}^G6{s2~3RA7JlYHJ*4&Rn}J3dF%2
zOxz~544Q`+moE)-M|l%7FlJAE`45N+&@Q~_t-pW#?-nPy7(Z-c+G4JAy+r$iM-dPx
z|IYb*AwgPJ2iD8#pw1`!e-zbz1N5gW=csJ=J|~xqe56dx#-^}di;!!Njz@*ofXZEp
zfFgsO7ykr60jii;=@ACD|0R)UXhV6LbY+(Cs0ccFsqQ3x<IU;sfvcV8&-kQ+mxkXl
zczQ{)o+l9LO2uuOXaAN5i{0t58(uANaO9*SAJI52ba56$v2de9-(X}G!ZqBwmg;&r
z3n4aWK%XY&%YR%!y^duyTqsP$Wy3gDZXGe-86VVz5y5Ks^DT}Z7W7)7Vj#|N+2)YT
zM_wC)e=v@5+`aja6Xo9qb5xDDXIuNS73o?d8KPsc#24P`3qwS1CIZ22F!JDC&H<MZ
z$U(2wQTvh!-+rPV?X5*@ZrZ7rm<013%to+zpL(=LF~tX4KI;Xt^g%;o7N;xp8T?BE
zw;F|Lj4#9UL0rsQpjgwLRaBQ_>sk8HNJmMisl_Z7l+ww>=uM7Qi`*pxWJt&W^TtYA
zer{d4m4Q-cGfCIIp$2P?+Lbxza)}MgPS3VR^i?}ssu$|{DCKL7+!T8Snxc>75q%<n
zhKU7lY^xn6hUWf_5L%&T&n@%`fmqr((9zFl44I)^hC+kp8|I^>*(8EqTGcKa%(ivA
zQA@WCC+;Y+L63p}539hNS&s85mEQr1qU`umZ3UPiwT9q_mu83plpkObF#qaF5zPQ?
zrWI13T|Zy2u#Gxl1{?x$@Fr9sV>~+t<^X64dMJPpEDlsLyKunu2#tv9e>PCy?r^1j
z8i&oen%CKhOJG1i7`<YqM5auvhby`qWMYPe08_37Kf8Tye>}jjGmZ*!_eTMtDHEpy
z5EIW0Y`_D%-DTz2C)RP`MhRhu`40LL)9xsX!LQ9Dq4yw#kpL_2?K5^oN$3F|HSm0q
z(0+i>C{Vq=`Ig>)dX$E?w=u;c8^`exG&dCgs*?8zG`3AK>Q0(?>7336jr1NoqR`ab
zb^Ztlf(6)1h$>+Jua%8!<dY#C-3Bgtpcs8%GG1wK!d*;35ZDfxithvz0oT2VU!a~7
zbGbwDYJj=+c1q7(S~Z_^bh$)c8tbw0NH@fMD`;-|$Z@{oqsPHEjVA>RVqyvf;PZ_<
z8Paq17yv;v;s0hwH3y=T#3RS}XO{Nx_m^`D(#`ub9cF*sUK%aS0j-&3`ZA<VH>6&=
zL!yrc9H}3e)jFTmD=@1fY3Y}8@qlZ?=_wCsFLilZ+6OpA@_snv%msCOad%mCv{KX3
zGGt=e>TA&!M1bfMD1nSVyh_JEABi4VEXh~`*n_aQKu7NgSwY%yKEDLoLi2uFX!q-k
z=Zc2|(ptft6Vs&@3gi$ypaT_az5OiX`6zpkDyqU>YP3VDSReS1;vYRveF)i2Bx+u*
z&4MoPk+JNiQoGB8o-PhuZ-Mr3zXTi&%#%tye`7er6{&zV%ybE`?8LNIKS0_ExQIEd
zIQ`A%>YdxSDH&ArKea_Nw7)^eqJADivX^@^<6NKV0iPwT^UrrZy?a2Pb4t<X>;iy3
zixP!f{L|;_*FmRLf<Ax_sq#1|hMLuay*23VD`S4>O+r?{$P80?obvz^9sKda%y{{}
z5cuVafg}B2zJTp63^hwWIcjfhOxnq*R4G^OWol|_x{wd54<jlZV(}^qu%Ks8(y#qx
zF;FJSD#GU!VSpSNDsC(2E9VnleN<>M_=x@P$ah~1po`3G22Ge%z*$+<AZ-f<(?)~5
zE7v_OLg@Pxu7Km;OBUKfzy@J&RHAT652mtD5CKaY=m4G}pDdt|E^)gbD--BoN}F&U
zWCu%uOwuR9*#B}84EfW`T0D>mY?X#f^C^%sbk1g>ZS-=h5vpKft|(pa6J^jVmzv$A
zBdjBqo$d^6H4yp^t}37>|GMR0{Q#4kIewDqhjI@rmWNyfqL<S;e()H!X9W}{*r+|K
z6YVjl<<L&|J8lMO1Pd^Ul%YQZkXaNG0(-(f8O5|BkXN#U17zaP$Dk-g<;9RGG3hjF
zet&NpTpZH|ghXxHmEgmKrV70#ff2lWb|}N<ocQb4ahs^A8{N?qfHM^F=*g{r41UA1
z3H%)?-)q>jKi*-ny;@Z>AN;Dki6wR&dJ|~5(Zro6mW^{Rdk1KfOKTFqq}Bwp98p?5
zw~Y{Q)I_HN7y60}p9g;U@ae|AM-o8ncONPJB6^p07f|rJA3T~clsbTME+^0@khS%t
z0XbSYU~z=iqgVg7-8U?S?z<M|ed&S62Mfux9|$=uK52QM?0#ptPXo<7;SreDKvaef
zC57D%xXW^<Po_1RC2MC<LvbA>e{sKXD(^&pqf0(zBxhD*$dH<k)ZtIsYBk<T7%L}z
z@0f4U>>$0SVLX$Ivv}i8%xecn*8k<93QYr}!sfnXwlq;wahFl8XXj1t<m11q*bCf-
zg<E2}!Kz+k6HUUs8Ld_AB(`2f7dq~(D^SH?drMF?rXP00>U2$>$8lb_TUoN6MU%F{
zsO=MF>&fu<Gt{TA$f8nd++(ov&h*1RF=&3HBj9)2CLgI@(+gi{i$|)QnnWnQuW39L
zl&|!1+KC<sqiM%hs2<SPd7AhXnQV3JtlLp7bkZ42auU0<N=oClZ4`YKHWgImr7_8I
z1ExCfxi6Za$YHS=R!cs}({=bd|F7mb)`NA97>gmgI77VdL@&3Qc%-kdU##DF%?(_o
zV?30r?g09{kJhT(g*?o{K%v*ZF?B~Ft<n<@j00Cx`+#S;D+2t=L=NliFE<GXB(`U}
z*2|b;$cOT#m}*{;fBe&&|4!^XV^BfCiNqY&Y|TbR)?(?D$8I&>o>|dd(wTxp91=cy
zLEGU63*EY(uI-ZVz>OJUI^O84(gI@?Wh)^P#gpBGo<jH?D{-@4tJ6%M_5J<Gs%Jag
zN=Tk^_gea+2Mx6$f`9(Z9ot@cT$Qz-$eMhz`;%O_BU<;w<WzjZIO(zZJ2PB)+!_4e
z_fS9psEIp^Df>F4yg*60Ur;=HB&AViK?6<-AAzSYy*64#swVvvva@Px6t9Ip?7hy`
zwtc&3CsjS}YDnug`#aU4WQE}OD7)cNh-D*u;H$cLl*d3no0;3T^CwY%xz=C$zxyUQ
zk0T<Y+`B`glD#HplrtKi%e6KKMbA~Q^tC3s+3b#HlGh)-dOcgmzJAF4^qO9?<;g<(
zbcH~8IL&t3ysqE1>Gwm`ZyVm+$6jzpx(Q|$!?ba_&YX62`saF|p9;Rs(OM!S+z|&?
zC%8uN`8-+6YWrl)VEF3Pv)>MHr&@K#l}AcExC(z@v6@_^ff7gM;p*gSg{n7j53q8O
z38DA}bOC|bjizWMV5gmVEe<Lw{@Sx?KA%Q^9BIUSr}0R!u|ko7*a0vkvwLH<E;?e_
zw*lAmoPDmKvu4SkF=VW7F!54%GgXW~WX~**jN@F7{y^B@g6VnOD$i!#7R?58haniY
zVz7n1W@ag91x6`ty%{}qlj%TwQA~~`>FyMBl=^erndFpK1FlU5&#CB?MnU@%+p=7T
z+3xE_Tx0WyikynpueM&%@WTOYi&#S(IH#KkgZYU;WDJ|KM{dLM=1f;ice{Y6Sdx3O
zQ-`T{9c92)G~<F81qZM#B58@)G#7FnmY4>WA5TQX5}&%ri3bzW05O-mT%vyjKA(OJ
z&1k~C>m%z1<2_QZ)v?He_@u|JkpoE_1k76D24ar!7~hjxmco>Honlr>6}D2S9KeND
zZZ;uq-kUv=Bc<jI$K`YK?9>_`sa#?ZOLm4^Ozv|070kOtAHwnIx};k6<g2}!wOLJ&
zRwHcfbr~BW2P9MR$y}?EyMk*^S{8nN6eD}6`=V_;S+FZVnnl+h`*yh~IHl>$AE=%6
zlc^u<;?wK!g(~G}v<<=@rY&)%<9tLcbkobLHcaiP1v*lZ=So=Prtv36c~?o6BA@;K
z;qnVlS&H`lmy;7D&%xjzS@j~{1N7}F{H)ng#percmAganGNA;>Ti&aJlt+7{T;y4D
zy7VOr9+n}=b2m2mqPOR6RPwRA=Sv}ua#Rc2Scs9a<vltI1S~oZoz^ZjIoeo~zeTF<
z@HKQ*IIIp6(k~xQ&b39#b|vxKl9jF?WmdALCerJr4@plA4rtjcB!2&>noYsLicgY0
zjd?)(N-UhXX%Tsn?9Cigd{T5)=~2Ez&>FWZuDa-+JpM{kr9Ve}C5*Chcl+4!-XCjn
zey`m)>*9^j8Hd@$DF^9t;^3{BNU2}4T*mLUOSzMGZFgIopUTP?%(Qe@&I|FfSS*}A
z9k{x;djG)zqWPq2Y3jJonAo@_q%qZdN?wJ52_0+y)L%p%PDYztlb@!)pzEFJjNQ5)
z(-G&j=8dCUY4@^t-pqJC^3bIKvD@K&Ml*A-rnq{8EjlYowIQCjLVA~O<6a${y6lU;
zAOCXRRGmKq{>SY82Du>;R*g*KcJIiEn$Wfhmxag3FCY+3lPw*$wi6(UkN1>PPa5pn
znoo$|U3!dkHxl?^D%DO)RFL1WJ=YF-M|bHJngW$G<e+WR6Acx&3G6BA4pS2^cT&t|
z#BrpZj9Ts<P!Bzz$xeyg5k3XRqjlItFqEAx7+=I+-Q7{>s$1<gzm9R6?mi8IV%X9+
z7e`D>zj2T0fni?V-b?buAy2PQj7NXq++)eF_&@1zG{nQ-A`c6>Ekg)od(!p0Yi6RF
zNmMsRwlfoWwLZe=?Yk4V5u@LA7VK=LS1c{w#Pk|Q$LVkYr&+gSj?<~X$_Srj^|$+%
zHGX9LHP(#wS_5~92~X8qBik!>1^%m}jw#!CwsHRLyd(*?uoiT#-O_n@GNm?s8Z(`9
z%`2(VjHc3%jcd^*PhpU2Zu7UIeMpb~xQ^9UCFn!>E?wT+Ghq#zxZweU;P_$M^JkT#
zjCE$9sbB9z5Lv-gqd)=Qiq72bS0BZN{1Ss7!V_QM=J;DNGVxnwBQ7U6zA^!(0{i>P
zv)5QhGvmF-SZsF+SZr&n9?#p#HGU{ci+|5s+3<i!Bv*ZK%2Fa<|3TER6<d!*<C%P^
z_3?^dFhg)km&Th))i-HjW|u9f-Y;Ky5kO=$Tk|yQ1JUHONBk<|mRyPL)U_krJPxzA
z1-tKV%!j{2)X)-sawGlJ<nPab9k$^;tiN2mndVF@F<$5Sfk(>!U4K>lPR-FmvUgxt
z4yUt%CE{e8FV9aIyK$1Wk=8vQzG>g_3)8&+>(_<l7$ni~Td@xj$OTXG61k}_wg(RA
za}hLx@j?sVCXuz}HG&;6Wo?|@a4S5!D)>?kS&pcbMNiDI2yR5otufrf3b4=DjfM7I
zoSR;YqSc?B!mi`sxSYIkPA&z_E`NR?oN<4>dDhs=ogSxj724{Pg|57bDNs43l~*D+
zu)?oK#l*%}C{CvdSlX2;{cwLbyDCB8;Q$#u*OZ6v>bP6e&ScLO>e{UbY^Ew01yoDE
z4%i`1v3PZ(wNXqFx>Dm2@LIUHRAdK39jsZfLcJd!kDAS*>_GX5y{s)jpkq+y_926-
zw#JhJ#(vrg){R|q7u*@0mlbku2t6gb@>eXQmZ@KJqFW1ZC&wh=l6!524-}|=uNyHf
zpvN}Y5iXWv%*Syps|Y<UJ{`z*OvMYaq)<*f^qfbrUY(RWoIYyJnqq6&T~tvNzgdT0
zevF)=scP^bKk!^Z>BpPTBFAt;mkScmVZkEEW8WS0AF4r;a0<%<r?Bqk&ye?z`ES*#
zWd8OQScGq+?om{p)q6Nh)g3&8d781%SPN<&?&i^5JrX2ec(hJIYdO@B+PGsL`p(oc
z+7Y|oCBkFcox*E+S`shU>d5+80@;0HuMviO<ivVEu6wHGx0tI(6%1RE_0_PLWKf8=
zr+Zt<SGk*hAUQFYd7bVfljesbskB2BJ#n3VA>8y=`S3kdzN2voc8r%is-d)rK@u?Q
zFW(~>RAq69P6V2`?CZJ&%Vztr=WF_^W)$aZ()3&WvEXq!bZeu0lp7yxOx$NGJI4!%
zij`d}lMf~*Bf0!np27-NVD#Hdeb}^%J)0&INZU9a{&mB??XQg-78`unQM`2={Rzw8
z>XFp)>#Or#8OS_U82lqT7Cg49goAr|udkmqdf>plO>igfdN9Yn>rW6ByXL6tRi^%G
zo!4NpVE<UTm&e0^#TG}Cv7mm>M_oJL3dK>rSH~*ZyNGojmdkJX2g{%|dQy>d3LhKJ
zd2{Hov@hpL{J=)pWSR=|#>y^|=;#pmE@Jy1UOPgkhW@AlH0aa-ynm~Qc1wLU(CvEZ
zss%dXEV=?9X3ggKW4Q3qUFZa8mkAX4J~ke&Q<DC12w6<nAOBeM>jvZenjAOP4qMxG
ztiOn&1}9gFjF#uenZ@3zsfS-bL+T$l&CN$29KM8|nXynH*=&An$6yrj%F7`#BRgoj
zDa0ESD=-@LP~TheXvX@daVFR63h4Aq@aYXEQB8T&45ASb>Fv8qO5&ExA+OG(l>6Z&
zK2_mrXo63YRxdtQj{JIy<9l;q1(9?CS$7k?MYK(c>;^fux!0k6A;Wm3<~YBO)Be}8
zsXB9!P;QJ<Td~gz0lmNB4)e7mmTKId4?X3Vbmmc9MI66W`rc=HwMzDqNJ2Tsp6Sbn
z6VG0Xm3oJQIR7o)oe{Y1Uc^&voL1-FNuALZi-Y&F^CgjQVl3o8h06=B8_&P~s<UFv
zo*w7-arCOV?^SG64&0teCaig9&N;aEfPyF==bjc{<G89${sUWVi`gpmC)SfJ$SBX|
zT$qat`_jPg?3gv8fqRdJIz5Jxs92oyZ<okUExJ^U`+6FW?TON=ZLWULSkvCk{!Yu0
zJ#*vRj%^P?yEZy+y#6Rh%sSk=p6Bf7Wnw4VUN`4`>o>vO_zjFK4ZQH3$IDY2JcK<3
z^;M<Ul5hmJs|mwj!sha{wlq3iMEq-Wc315jy?auo%dJS}jk6m+nt#da(r4jgEHNz&
zCG2|Xn_|H^$;Mv~w2Fh9K{F+k(4<ImI4z%|%DIC?*8;gXV;i%1IG5D@tA{ce&xk5O
zGRzMfb65u)^7mvV&@ax@-||t%Dr~vx^o9y_SwQ$DVx+`WI*L&;RNWeWs9*!s?^_fq
z3aFdI(W1DD!Nh@6z>*ZS;~tPmQquG$tjUkIA0w(_-olV<wcJ8oEK|lRGNHYBgsy0%
zWn`r62kEE99uZzXl~pnFQOk|{YrDmZjg~k8{Jt^TW8y74?|;QNL@Y7=9yw}0X+L}G
zuG>_vBPEzc9bjLPw@1~ZKWbA+=?kA5=MU%b=)q_&A~@l!+w&T!p9$krODXJ#85M8i
z>w)i__V}+I_V5?c^a#s{7!wLO1iG2!kA;#ZEbeY<!mE7muSp^rm$_=)Z<@)6-sBRG
zAtN7U_{FZ(KS{xBU&Gv=Rr(RX^HDr<-FqNUtu<*r4QS7=*flZop_M=RH4C_ow%FQ?
zo`;?k@=8qf3<4X5d(z9~$SK)1)3&(Z>wHL;Bg=v?-~1#RA`w?+V}>2hIt!;Cu*CgI
z$6g9)n48(_%7f?>d-pL9aAr`bIDH7$;)<OcD5B4dj$hcyb}K;1*`rY3wwH%*zKfT+
zpZ)uzp9JrRnV0Z!Zn{u8R?4uh$n-k>Qq>R4&{^|Ga{;=@mM7h~X#7%r5Wy~Pcanic
z{vMITY0(O4Ab&y79@#*{$o`xAq|7gU`&UP7!zadmR=PX7;c#&~?bD@-4x7tA#D0ON
zfoX6jT%R9x4}_qKyonXAUPM?5Q=mjH3zKHKQ*g6qnoJ_EGD!GYHfp>u0^guOgB&%d
zwcF<NX073K*LV={(z|K2_;0E4n-<@Vr@X6&>rSoIUgxci76_184<Fv0dHQ^BOf!D-
z`NGwu(dHvKvby0&|H~fkaFz2X9x)N&jtTUd7&j$8ASLvztv|_kNUkx|vt<}*OTCFP
zT@DG87aXIwj;_u7TrSwuFHV2x);ckPEvuJSO@BvVN~85sp}ob{uST)C_yQeywo)t;
z+_X9V)j0WQjp@!SF0pQ0rEiY3a4MZAiMyLg#TN@bMzN^&Qhie7UX#@<gmq!}S5GLF
z)SRv@PK7Om4<M%;r%G$(mq=KXggdZ)Z?s_`yf=ybWES2mO_<t*gLAS9kNavV;rqOb
z9G8Gwz683iGwb?kh;Jl<9KAR7=hb!B22&d?lwCiE9aoxsUZCw~e;&%B!x(y8Z(a$1
za`M^uS9o29`UYY6&k@#G1`b#0WS$>8u1Cv3_1nlL$JF0%_pbGC`0Y*hezR$;rrcJG
z^htI6?t07k%e7r%op|~AjTuE*p}R+YF5`pWZ)!eq)e|jn^Ixv?^X%u*+acSZE3QNt
z{?(f-***Dk>l$mPZo<P7s)pSgVf+O-J|AyIXF3inx^V8jc)RWic85E)RMULGQ(q{f
z=e&%IVPTQ*yDMpwno&ZRuqqYkXq$8;EFG3jkm575ia`N35iN=<!X2L}$t4?55S=Wo
zXGf+{ggOi~%Yo>WlKcTfl#V;!le@V7=s5jx^`ywJ!*XJCOH>jlq5(ERz-EfPMa*Bc
z-*|GIx$pC-Z;{Jw<ff6kwx^z3zih0`ei&y<kcn=$f1L`B!4CY0*RAf>$8^WWCLOs-
zKkt6Gr%sYqYSr}x-FVoqoqBaR8>;-2`VO9jYoL-gEgln(dLaalg>O|Y=iv+*vSfoP
ziYhnXGH_DZahcRKai=_XHgZ^%mQ<KJ<HksL5c~Y*{#tW+)4kQ~qP=T}M;ngzQY5;#
zB|C^<g_5qvB^wyMi5b5UIL#zkj{HkVCI6Ye-&+I2j^>*`)YwBmOcE>)(Jxfd&!3RM
z6P^&&W|)NJ#6%o#1n<4=C1b5NW1&4sTIiS<@)+W=a_Wj<PW;+F{8HBN&Fn&Q4h3SW
z;gzmEQbd|@cqnDa$?TgrHrXB0?T)&;5Bml?7zP;su*UM_CBmFLEDeNZme>kn^!qrB
za<^RrV+_Y_yOQek=HK;oYkHC5XPJJ43BI{sfmVRY4Fz!Btfhi4ZF?#zxT!3U58Tk?
zu(ztdG*(ei3vThnXQFRD7k9vIOb{43$T{67Q)u}_<HH3c*y@IxhoJ39I0=5%aF%Gc
zfz|E5aQK?t!Ni|i)WQuP2obiPwXq9dIb?XH?Sx59dV_xN4q=%_TzQP1%ErFur^Rvd
zSF5+v5Ya|mqkOM+?;NiZK5m{|+T{}X#D&4amgDS7zgG65T;_DRrul&67lpX!{lo{N
zV^ZF01$8&J7uuJIQkQ5)ie?HuN?q&dbv>rG(%qY=?>1E~V9|%2JsGJx(O|)pe>>}O
z#*dU=I}$=?O~Pwk@pu|ZUemIZ*=M)jh;Avi01IRZ=+$B1eH-6(FO9#`DS+kVH1}}N
zg&k)Agj%=|u^tFzL3XsHtj4P_3;mueSGXQZJ`+~DcNEx_7@i=upcIJN@MKoQVK3A+
z`NTU37hBDFWN!?UUW2+KHci|Y&w7C$78=_lwjnDU7aaYi%5t{VB4aM7J0ClHsCd^X
zmen#cmSDcSZll@ddiM4##Q|uKKx;OXWV|j0`(ySw9WlG;!BZ67^d=_W*0XoFVXM1E
zfO@i<mm~JpOZ+JjT)gBPE4X%k;7r3l`cdN|4FCEadRO98I&f!HIN_UIkhi%*O3Kt6
zL{MQDNKXXCfP_9kZXhy@)4g+lo+E%>8rAj=mWaY3g$CV}PN7+zOx$*c)$uNlqgbNR
zX=1M|US+j}0X)<*EJ;vxa?iwhH4r{=6hc;`RE>-?oWITkbBJ4_G-{y`Ur{VHQ$<Xx
z!R|XRDK%<0yXuy;$o%}UG-K6ox7yo+5l4nK`c#7aYjov;+`&@I_rPmL?x{dDqw#-I
zo*xByMI$ENa&jH`dyuO|h_z*Fh3w_BD_zmTGDmX##(!p6IyV)0Zz`|SWK5k6x6mOS
zDc;=g-uNEGL>6to!GVmE&JxR!q<s+#Wa@6VL5Qt9K)shbmGcf!-Aw$!;^EidU#_Jp
z(w2vKet|E9)2x4BKl=c`)w;hq7A2T5a8lQqgmHGCu4<#Q`6fmtWdMw4R-*t>;k3%u
ze~+4p>-Wblru3kvxqWuaQ~m6n)x&3%u!1HswoN&Bf+7Y^5(fggxM_bbs~b2-N9^wv
zV`yO~hXX<2ZKl}z1AO)GK-}-%5askuU`tI)AC&#gR9ry`x{<40qXGdNKpMU;Q<m5b
zIpLy>^XZR(QIXjeJ^s1KfWX9F;RGr!75yn*Zug>hkY~h{T=O+s>wI|-$Ys1%t!<*l
zlxQZ90@+WO4=^u3E)0$^5GxEWkp`iF*}5%ViP$4Vd$%P<{2I9LFUgi;2u+h$>0_76
z$38L9Xf~W3R-A6<JA10S`7<|+Cm+{E@zFh$U##`HVNh=K^dYZ$E%QdOH8bg$;F2;m
z2ub@+9sYVN>s~bDPb(QhqF$RHKf2KXmki&c#1-yn&*#ze3)M21NAK&Q-}B&>L}%r7
zd4G#NQ+UL2_t7s__&8gY<CKBO%|u6utD*Z*tQ#K86Yk>G^PC;j-sH*Qe^T&fo|+o0
z)E{}(#NtUJ?YoZ<BMS`@dA#uXtX}=oapVJ}he8qp?z#@sn}UVYxx6Z9*RFL{w`uS1
zLQ@=N|Dx!hWl%1-|7&GF32E1yT5O_x7PGq!Oc(1KuN)W3t0e`O)&jwXH39180)UO6
zow<T$^#J6v;xkh}U&Md)=zQ9vS=vF)n~vXoC#=>D(U!pDINSTY|A~@s;<CItl#y-0
zM0*&}i;;WySvfj8LL=z(4}bk!=n36yh}KB0yWA|pfS8l#3PH}V+3WCm6+7~$Zp*5D
z<&7MOmkXK?Rm)$)BHRWsPOhJ@X@>UCI3F$H_1{UjUk}{NV$T*zDC8kS<ZVqxP!QB~
zxT(}xVGNEf3b|%BBh_wi;X`MqHPmicPET+G<GJ#v*)w(GiGSn^f9P6CMbs>!O%Qv#
zx4+fUxhMC>_vp9HP;RU?Lh2@0BhwNIA@B2mu9URjT{DpzlADa8{@zqXcK}Kg8;UmD
z)0)d~e!FWkynTd1?>5yw|K+x5W5)A}I33d-9Tg&}r9bUttjQjSB?H+mOw=D(uEaf=
zu{0li^}fo+W6HbaJH{CrRY0%gA5P55^=hX~CP#2m%b6`pB4tiyI-!|NX?1G(Dvuo%
zscZXDRZD+T)0fDNG=A9Mwtm7wM18_^Jwa|;r15>=TGG6AGRxRQ5SlcppNj5&9;bS8
z)WGij98~73cJ`R!wUlwd37;4Q?yR$`0t^T<#wpj$kz|b1JEU!L(cnZ%Q3o4*vN3UH
zMxosByG(!8NsTQLqrI-whD+J$i^R#;XpLj;q6f>mn7`G)j8g!A_exbo%|_X%z)C&&
zr`(jQ6D{ub)s16u@>49yULH*2JL6<MNWvee?mWzXulqsbF=6t@0M5nH8<n}x63*O@
zB@SyG$X0B5`<QEL^@O|~h2<{6dGbyoQRp>rwnrtsgYys+i=sknYcD`V$8GWZk9^(F
zx^%p)Cp)n@&&c`8t#s6XHjGha3WTZ>Q}ed?p7wbnSVYC>28YO)4)tDmx2`84X!d-~
zNZn=pgZlJKu3=l5g}@^$?x*5d5kIe`qmw^<0G<C{mOTBb#j;22*^`G;3of_|T0xnO
z2e()zDtRDmz2mAxobEGq8*hatXV{#)Og(a1+Xg)m$ZVE#8<gM&r1Q@Tp7?xUE;wr8
z>nH_xsea?^==IH~4BFYvujMgX3Xje(3as^;`SC7@O}{T1g!9U_NB%_5?AU=EfF-;R
zUunsAp*%17aZ9#XAPg2LVAtj1B8OOkVO#vs<bx3`&d!sLDzozwIR$VhvOM!I+HOv@
zj%9awdazPZqEN3TeDG>+g|#u$^I*_c0eSF}6=9mIb*hoD`LN8fY0-o1U62bGUhYZ)
zJ&{ayLDcgFK8qoT)L4Y*%Kf9GjUe}$?%M$rCcP~yVhc^Z#<PWXOOv}A!x`7Kt2>{F
zIxUsIoo~0t_<X3Hy*1#e@cEnd{06gP=8l8yQjKhl+cg|_X6>DgV)wj?PJT+o%$sbs
zHUw^&Cb7VE@eQo-_xwXto*QXq8&=9p-f<?ZgXs-_>rZrru6z;qJ&}rqn#bPkPK@@1
zBgRtQ<_!8ohC_y>{@jKpjzqAAsH|Nc$_Fo9r!E@gcKc7yOpG|Mp@!gQELgQWs_`es
zh0};GZPPedp-m9j#5U@0b!XKTa!~i<X8gGN&dG&6clv|+RAtE7sB;%t;H{}AX1|{I
z*?#kA4g2kmr&Qy=-+xB_*?R*;8`A)_t+P4dGQWOd&~Sx(4iF0<p~Y+BLL1BHR?G)3
z4@IfvjXrntHme{FWkP-zrRyv>a!$fVrs7=R^%)0X<pmvMztz4l5pvw+)JE4OJDuu9
zUxmcCM{rs1n5t(x-@gdh!6B7I;_8{Ctt3CyZg|waM)cDZVS6MSmbS8c$a}r~R$gRJ
zYNH&#G9oX=aECI(?<SpG0w+T1xZFNQ6^D$!o7fk>?X?e92fZ^lJ*mwiG7qEEPPm$F
zFMSGZSEbB{cF3YfR}J3;Xyud1@@S<*%H`(!#4?YH3d0^%5@1V7MXxa(er+7|toy@>
z=ozomZp<}JCN>vv-y?Cbb!k`@b;90%a$F40t)7b_rfHYuiot!Zn!8^c+Wq6cODHyf
z!L9g=M(wpR7cHk@HgF*ENj-YHHNkbYmP|bBNkFfqxm}h|(6x}m3Wqb#gAU)l2hUHV
zjt^|uDTCJuj*bde6ly%GokV5iN%;L4kgUH$>)~X#*XOFLabg1JCXy!{BNmSz$o|OR
z{WR1g+=zj#!lBpOJkj6iSRbvJ)g?J{!?Pnnv^unFMB&%2H?PO%Yng}NbQ%<^&A#!t
zKf^;zo`mj+z`6$09;95K<GiwtXe-(Nxw0$!dRahDclQ`al}K+?L2w265J@^R*Yi6f
zb*?WXiXlOL`=vZWfay#owp?MfG?~bD`1VolM)#2p5tmzcZvBp}kxW%t2+{jR4|a`$
zRI&BPNb!^0?~#5(hn_tR&BH-8{7!X4ivIll4f-dVgRpeJuUqJ>)n{GW&(r@Hp3LoU
zu^;v(2p%zUwSt6v%u!K(|D?gFC1Z-%naZxM%Gs;3RU#vDp2G^BtkVf4k1rA;{)b-t
z0G21qV<-VIC2yS&if$JHh0VxsTQe;g#_dtyUe9ESGK)bYka5pHlW3pG>31Z9zU8BL
z9qn;I@1tdwf|lBnf#Ygz^8G3TPl_xYjj=MfAG={?e(-|IiH3k=ZZ|QR4Fb~lU+XQ&
zFiG>XU9*$szr#FmQlmY$atL{)VJ&6_BtXAQb5!Wf$!^xb>?$jokze^8bgbCLUabk-
zDN*l!xe)8X_PY*W2WP?=W38wsYl;t!tEn5{#|WHr&e}Z-?WOUeYFMP3ax~+JX57r_
z@JYy$b}I3Wdn+bn0^a-Ir8DcR<zTuVwfl5S@QS3Hs~bn;&no>t_LB+G-IDK<rmZr=
zNz*BK7T>zc?@|x4^(ILPKZ75+qciE>o-ta-n2Ob*;hlPWO}U65abu?Oon`0poV0Ev
z2w1+a3p(&P+-c}bKX5$&NkzP{Dwjvf+stk-i3i!cPK3y>9Ca4)d+Uh$oapZNG5#k+
zQs<GdxXG@!Y@_W`$fr9ku6S{kEcYj!5|+NF)|KFQZOrSCA~b;ijwf^zmQ%E|yd~4&
zjwE(o>~ZrW?306Qd!IPd@+A<DNZz=&L2H3Kg`D#AkkZ}V$o$DJ8o?*y8-tFh{<*qv
z%V08enK77%&uHj5N3!0J^z=9PC)78tcXJUL{vdFX=Q`1ZOD_1|GH29=J&SXba+-`_
zeel2_f5ofBWr`I#02NywQ6iA}3q5v!{K;K@zWvN=8__j&7dBFYjHw;)EcaMm&Yr6;
zUr*B)CxB1o_zAAtaqkt;9~^hV+B=;85LZ9mbBpq-e`zs9XrfZH5MjTOhD_Z{G>Os2
zomkmXUhpPF$~oc<yXCtyz>mx;k{mm5jn}Ve!N$LsgIp%3Xf4g%rbQkIv@rqe{5WV?
zUL|j?vwM1CvBBGysy-L<?UyA+D9+)@doxzb`J>$UQo4w}w>NE%16q>2yrgkwO!ef*
z{~JgNL_<87Hz!=CB<tLFwa!qEDr&*4?tKm3<E{LjUN7a&VM4g~eXd}zu1hWX%8|NT
z6s-Kt5*TN+m!G62cd)SawI0KX)30;-Lg{%waa_#Xv(6E&&*QrJX|_GO|EBGy@R<pC
zukIU59Fny;g)e&fXbZ{q8wDsqI;t?Q#YWkaNN;con}{bxer@;w*P$Bim+)Rn>IpMA
z{Ze_0C(9?k9{-VR&G^S}v#aKH<pRHk{Ev9g!n|<!7byBHZKv$jq#|pd@M&rk+&gl`
zCK{;dI61g#L0m&}dYbz^e6Ozu_ZbXk_F_e+)O5l*znk7%7h8WtKydUeBhmfU(O1@(
z97rC1-Ccn+e%r3Z8a?-*CU{Ph<C$KApvV3v`i%@})%4FRu!1`M@RBcqlkGc|Y{&(~
zjfM!vV@kL)vNA!dD(yhCJLx2ImSnyx)K3zIBy%Y<e&T35Ko<J{w%1M5`FNk5s6q%>
zJUY2gpcG6VLNPuIeCuzsrO3+71f8m03!=Zd+q7RIl>l(HBGN*zKZ~NCYq0|$;<TmW
zwYVw&Y@xcBVh5b}>ipD#kXKZAFX8!D0nQ5Vg$=VG1Zo4|8M)hP4PJEs(cK<G`V_)u
zv)PPu!*PjGP9x(KA7?{ga6Myi?qjI9Wh{Ed#|V?d{E1HLd_HTVgD399NRDd2Uj32#
zAU?atM2cuuN7EiJ-EAyX5!}Y;D=5hT<iGzZBH|oE3}MC4^R=p|W?Dj{Z6>O#XgRu=
zK7RbDP~~8pr5b+-eJO<j5Laz2-+9|E;2i6<OB}+A_iZK2w@SOQZ?4NTh%i&mUufX>
zPp97BC~z-iX6&a)O64pfa{vmJ3?gfx!UgDi+FREdE+Rx%eD=}zR!0oozk8gWAQf^{
znAq>C1M5696<c;beb9eBz+U{R0u~7SO}CUi&9L@2-(JZ)$#;|1!i#ogteA*Wj(f{b
z1toqEqHb1ytRF(=-(f}3MG&Bn_E91gFM(GGG<;^Qhn;bpj<N%>lO5Olzr1{~pd3zg
zKKmvB6b!WdK?HKLdEc@X7iKp<Dgq?18vx*!ERm)aFk&`=DFllCxPjJ31)y{I{2`Z>
z5G_~<-1a{@P!JcOc_T2Pa4uV&7I^%P^LHPEu$W%~>iKE=Az=AigB(RuL5TOaqZKdH
z#dYXWOq#R+&KlitdMNFVTxSAZC;if;qq`;}Q+LjJ@edDsj;Rs_u{Odr2_{D9a2cEV
znd1CAKpXD)0MttRXJKC>0K8ZDszlFDAhXn$k*C1d<^w4et^~l03RyTZo$rsA*kgat
zKfr*bFYG)o4FwQ99uxwA8icV07-GpFd?uqGZ?A{58j4RPB$S*7goZEin&$xGK7g-|
zcqQ8P`n5lZ8ev?sJ^wF=*gKtTRRpbUIJ0)O1kkV!t;ax*v9)n6LLhqVWC43@|3BEn
zj|L#EbP%)}0Q04Ti8zhc$EzGb1?X_v=BGQ*QR!O)z&4)l&aIyFVF;>JFsXU*XbkXq
zxP0j$H^hGMx_yypP{0!v5P&HO0CgY81$BNjHhS7?be;2N`Yu>XS*6sw=OGk{KmdxB
zDjg>9p@vEZs0}bKTnf9?Z><q@8M5(DwtxGROJzt!o>xlLAKB2UMDm_1RJs^YbS8I_
zb0EH+Dj2q=oMf*23*e0yv7f=^k~>3F$yK8SN*Q6;5+YfkQ)krkSvd3ghqL@&vF8l5
z0!z+tEHiEToP2Yz93-!j`BVKjxGR`^3Mv3mg@H=7UjU2@jL{t^GzX$R$)c9hW#@~{
z1E}o`p19l_i069jfS`EJ`^{z5fS^ImfbC0y<k6nkj6a0niN@{t{lq4H>5}2}inPfI
z@e$|z<5mND_dQs8aP=W5W4fn~n0Q66PE|1b>tl0>^5+iBVMs5@)}GN@|Bs+Y7?BSI
za{5HCn68*57tU@f7t3iG1r<kR^W1AWANy@8;P=CFUgz?{ZvbuB_D0N7TkuOOMNLI$
z6kk<|W@YJJY!qU!zK1Kf9#-E2GRYl&^y$3122xn6Lhmy#s15*VSM27TFOERi>c~N;
zbMmyq9UwRgB?zvTks<muNz+S^!u!mVxs{iI`IEFw4U=W~5xRr_8?l<Uq`H<4nuh(=
zdv3)VOF}wHmXx@>5H7fXXn+G;^2UmyKTmbB(s0erYcqgm&jQ%@_9SdWvOiPysy14_
zKg8o{G{9rJQYz;Fn#%x7(a_HmZYK#tx2gqpdoDEl&RQJrFkw-F@M7cb_R?1CSy1tZ
zWnf;(fqDxc*gd-$g)%*0#n40(EibLu6dDBU2f#YKwUg~%4?=%|$Pu;o>9JQ08QnSB
z$*vMew2_#bBL{d)#ik9CH}EU@h0#D@?u!Q7_cbe^*%*L@+?Dp(8jk@--lTMKUM$sb
zET{m(&cUyHi(G&aicCTJ<P6_pkZjO%RQT^KT&KtsKEZ&}D3r=BkARl9f)yu8d$)!%
z45V~oW_gMTE>@hH+B7)nMN~|AX<oZ|p8~+9BffljNUvF*`%YS>_gpvs1;|<n0K~53
z)dX{u-S6<b3~`z^4HK~S7{mfgTlH=YjW+Mc_c-???S2*d`pdOt${x*CFnL$|h1ES0
zdq1w<rCO_&gvL;4j_~4fezczOgl4@;5!5V%PunLEdrcOq4xN>1$|V5!mD^_zVR}bz
zuWs$y$(7TyV@Kij29{(fOtlTyH8t);j?KY!&%(e#wtXg2TEgeJ!i}^M1C0b=xM#l1
zr?dUH8Vq23Bz(VtIwKlJ#ts0jjqgZ!3@YkA5`*M1w(0DPzczypu$u3&>&ji9forV)
z31Y3Ylw|>1JC1bk9sZ8$UBR2-4#rrn)`BkUf^K<de1u2q3$(AdMi&n^7augwcA{Ww
zp|tQ9aY+tlU2AVEsAy6b1WC9haBQ<5g5~$T{<4lpGa4aiaL<3Tk(_Uz%osN%3yS-2
zO~>ktb#G(8j`A2+tk<rS^$S*`0#J_Nkqtj$wqGBzt>@nP-7_58x`qO74r}86%9Aa#
z2^QF!scYbRrN2{9p!|GB$>(|KwA7R0JMO-c$7cBRX7<Z8@kd=vlz-O*HMBQk-ilv+
zgbVsM!x?(n3<k3Nay_cvr1C42X0>)DJ~ujuj-Ty_MckPj=4_bCy=AX<1XBC)+9?6B
zgYkP`6P~&VhvPMIShEUKrK!}oFQ-SCT10*nL3eB&vc{N07!!9cT>zNulX>R{Nt=w?
zU!@%b1SW(6&pxtpt8!W@hH^*h(d-MI+b#Jn)Q<BXSX;o%S|D6<CVh4QClE$%@2yX$
zmouZVoBoEvPS63`{?GvH0hP>D`NjPD_=dd@Dh&V|$EA9YNK)JvmO1^w(poPVA|1Sp
zZ0(cR+56L{-|$mji_%(CHFC@bLAKpt*>dLjP^~m)bp*!PN?`(!E`3GqGLA}4M$gU1
z7k|g}2ges`@zyz6$AfZ&$x?-mC!RC640b#LinXoAFfR7-b3qu{5kLb5-m=d;DxvrX
zX)MnN>)UzEFZ{w{3-O_X@(fPVkJ{Bvl0QR889?UzgDH#3-;bvs03$>$>_EhDz0x-(
zRm?#;adfLYC+qvO);n%>?-NH;T1)8=c~zq^ZaNxexl|yFvCV5MB}mN`kbT?~zuj4(
z*;!~hmG8mdWxP&5(Y_Ost(jHJcvu(@f8~OU(3-$7`d*u_11jwVshg#DIaZt&i#6n<
zw?@0ba)GLO0!-5H3<1}vNKW$wWQ8GRl6W=^DrQaY7Xv$xtyYu@Dj1&Mz-Ns4b^D_J
z<QoRCv;so2cUAE0Fjxa4SJKcY=0-;6YF|kL1TGx}6=ytf^5sePBevE6I3HyxDZiU=
znDzr6a)5a?YN#u_?kye~5SFj~=a&9h))dC}P*1Z-WpSeBb2sf#(x%$cnvk1rB_HAC
zY0u@Q^J~{StDA$|CDt^y{MB_Zwop)Nz#16NYkpYJ4AubSq|n{VXzVl48-N(vV|+1i
zq~Q75#)~~E4xsIUurFltU%x&B-#@nPxQEI-eam2-$+P0wTS20YU1xz$Ki)aGSxZaf
zrqxL-^9yQtO}SIdC-9z0@xz@pIC2<6`860#77JoX@3=-JP>A~r_y6v3ish;Z;`nl=
zmrzpWf4~30u<Mvp)~1?Xu|oi)8z8e)Y}7Es5Klqt_kjzP`Ulu+1Sup_0PBucg;L3P
zVSlL^?Y91i5k9es*;#?h-ymRS?CtGUsP{OyEQq{<S`T)ct+Iy;C9hvU4d}HK5q&?)
z4B%TkM4-$%I<};NHDcnd@>P~rl>_H|U$>yDFqP@6Fk+iuxFyUfT&>p)R0>9DLzM~5
z<ce1NRcEv5q+58jjRq{tM-ASnH_RYk>U_r&mLXwQc^d~d_%CJLySpVoiIaVJ&P$sB
z=hEf$+M}3Em}5YRj})k=HeMdchH4@**NsU2RtpPR0M05fwAkb0W2)fJNTxQznAA#D
zXk`x+o}3=~j|@~9I=h)VDRxi%@k~yj`02P`xcDoiud=t%zKhd+Rbgq;G6YnE2zMnV
z&@T0#ojo>3W3{hTyM1|{hIa#%^SU<w*VV?P2y^g(3c^5f7`0}(6&tX6*}yQg$^W?h
z*V65a06&M|eL|xFstTsTI5kvQ(I#I@ZwX%ur+T*$wx%gw%|){x2V{v;Ka+n8>_b!A
z^RKUMYb00OcfqSx7~O1-2^8i}kIx~$W7iF8x%YA3L$1c~^1w(aL`~D&z~7fqgt4I{
zUd;qnso3MdAzQ1exFcCm*QREz{8QpvsE`UQs?7|*QWj30XfZSe;wC6PbUN%UqS2rx
zMk{YNrySp^*brV$ni=c7;(ecbt_^>4G#~NRwd?zzs+X4uA}vRhOvE_Jf~SM;eHZ?q
zd!0<0T%nu1-QH)~;kD?z?L~%J8E*YjY=#z^1#4)mIwT&LVo)KPmGkoIb6{r#|2SA&
zF7f(r;CcvsKZ>W`r?npl*~LMreB!b12;N_d>kZb|zH}7g+RJaGFVdYtVp?*r!<DCw
zncX&Om7kok4Gh8##Cyq@h*F(a2npF2q*uX`Aj?s5{yYFHuf0h$+(S|@2!Cue2iHuu
zZ-O~&&kEkz`_XuZv&Jx-trgn<pXES(d5D|_>~N9!+DTC_8yjhWeRSCRV{AU+UfdC=
z!vv)JX;2p#4O}mnb~MG$AYYu6hv0boK2I+31pG4dTuTlrbc2k)Bo^?p#6g)lHB=c2
zRaq%1efobW`|o(H-#307=IT-;MJg#|XM{rbxRf2Tvq?r|Wo2_UB!r9*sce-Ik?pE9
zP|3(jTqUBERVeE|&ex?r@9+1%@5kf!d;e9vIbY{Ej^jMW^LU;+M6F~dHuNJLFJur2
z$WJ?xeg<#GS*a4cbtLa;;6(P^-O~Xx-}04JgX@w-{Zicq>WAH8*qf^_Q%UT#UK>o`
zU-$XVO>3#Os$slrt<4wb-=8W@<hG9R4c0xYSJHoQYjSBM;Fa_DXW23Hw`*pe>m2&l
z8@iScplIh;&Y^S9Z(kC(BjYMWHR299EmjOa;z({<Q*!Qkoz-E({bb+aDaAGD)aWLM
zYNDXip~A>z4Q+e-b&2($ldv5JE;)*Y`kTU=fEPgx^=5+SvzC<0e^v(L5dgtz0>N6l
zT62U%GpM5D9elf*J^(!biB9(_oy>af{DrpH(!-p;zL_K}JeNxSlo!+rrLx36l~|6Z
z$r#(bsc&<t!7l(>-yf=t$_YgldAYG{yF~epeZFcw^*evPBH3;-)3|nOLEihs5Sz{W
zhtfK?!sZ+XFNz^o;m~`JCdGT}iP4}}a_!Qi9FxM(&LmtqU&3>A$nA4Y%S8?851mhT
zAF$~@>e?&xt@7qJS{bCGOs_Th?xRrTX4P$8iZ)%0B|%4{Lvs{U=Oa&BLH*ki6Et(=
z3)>=fR*D2LD-N%G+Pig|?KrHDgIYogB}ay=`M1lGy?jfZatHd}SYI9`KQ;m=tRBEZ
zs&=s7nxua=Ff&-Q_R24wLs2BqO3`~WODi9%M1FiF%RVLor#b|Lk>{l$0;O0Eir`Q~
zD_)u3H&XzLr``%*HQyuUCeT%I{2YVuS)=lJXAO!z;r^QyvSR{Wi;=`X^NNO(F|tSu
z!lEZmyA8ybFEw4>R~6IQ<m+be`+e<xpi(4=^^z0*n-%AD09$&bYNDt*92UAf(ux(F
z;Y*~W<mr!35v9ZL<+=7er`I>~S#%&`SC}VbX)P+;F36KSIQ>-r05-2Cl*C9$O}d-~
zc7423_AsU2g7Y@cHtctvq-(b(!i%gdEu;=L1aJEJ$BNQnPm}~X%)JtSffC`u+D>&v
z=Jc7Ov0dBrMJa1`6ZHbnP~b#fs?JoZ_h_pYYS|F6=pL)gUc?TPYd@%~x31nLY#1xB
zm>~>|(?zf+$7N;z@^2K97KX#Ip6rI;1?>X8n+_1FCMtync~N|kvRm-;D+96HF`dY@
zhC})unMqB(4fc53mZ9~q3wpgFdBK~b5q}-%o7JF!WIyVr2J#9S_x+=%DHA@A8hv#_
zOE9bPj}+iC?2*jFlSLTXf(l361yn_sFJ8LT^5@sLys(u&;yz<KEs1=`ZpS?SMVZoc
zF-*x*QFbpTi@?=rKiQ#|+zIc09;$K?+8j;sH2i3MS5tv%;n>)#bVmrQ4NGklzwNrd
z?iMC&2g96v`hgfWh7pXB*_qh=lpDs_<w6^cB|9P_*lFT86otTHOg^r>Vtn^RsYlS`
z!Lu*e{qJvfTtCV%7-!Mi3<5leXrTvrxAP;hN_vPfbx18WoM?0^+#AM42wd|{K8=Kw
z8S6xE=`C>YUa;=Wq4aUn2EY~sxK3jY_kW}CBIgL06b}E|k+Nv7Xp-{UGS7!vKZ&CW
zDS}4ozGmg0p2f)L1$D10iYI*BbDI+8*Ioo<lFsRe5|vxh&`b(z7%4NEXECeuFGktY
zA0O)SLYL-IJ84S-*M${fbq30ZG*JUL7i$l8-ZJK(i!H^+KAcC*@OWQQtd}?<;sajP
zw($U@WKvRX!^~&LUJFAebQVau{?aCZ#ZLx5D|axK2OIWZTlIBl6n|jsqraOX6ZiL1
zs`q0r_Jt?c`h9B!0vp~lJ@<Qr1^6hCM-n_&d1zICG!xrOQ-p>MqBl^=KpyD*#{%o*
zC&V3#V4*%gW3f$!+q5-TD=Amy!bW1f0@l<G$&E!A<+0}m;M;<oU<`m$BW3ugX`o2I
zHn50|vOPu2GGWWrbauM{5(>@65)F0$Z%C?5I=h4N|NX%aeBkrVz#mPX*bT>)YZ^Dg
z7=?PPZFZSbl!37phoG4c-TDntDgrh+1;qH0=*GhuSfmKqyeUjX3IBhRodAD98FZ8k
zcpAx&1TdHrD1+|MlIVXI=SWGbk``b+jnbpB*sk=TT?yN3aKsYr%H0})1C;r${vXM4
z$Wx*=Y6-E}@RH~MOqw7LlAjVLH9^1b8r@Y@qzD0cOt)Zb7BiEXDT}h(M}*YJ%etvX
zVa4G2<iQ%BV-60vh%Q5E=j~i<60ohtu&uXeea=$O7CLo7QFZr3+f&9MpCQ8UYNC7)
zjFU7_c_xFAADqX=#w}m|HlY7az3jnl-AQg7eH)o)L2G6OLimY6b%rvx>B}UrD*_@G
z4--I*+Ah*De+Hhp=@*VN`zfoYm9dK0bYrWh-QCL(8)&LMWy=hTw&SEAHTXb<$3T@b
zxez|DPt9~LA8IQPSUl9cYh1*QTo-^E{rUPc6Ke*C!#bh^19A0`jajHS(RvMg6L{cB
zEjMs}PCZr6c}ZDyuDTsGsUqSgKu|P&es{t)Wp!mKW=Z*op^sABKYozGHOo(5-`%rA
z_MnQ9+~^qNl$}5eZ%cDNKdY;KX?<zCWWXKV!X~3X6!c9*(By8is)oifIG=`dpZ?LQ
zBy6VUIGvBXdK9F(5I6Yfd~5qNCR%gY`ULkNAuMIgxEZ%s$}Is1R{CrG?mcvUrw#8=
zS+_uLQNZ}7++Ukbg%7yRO64F5#!d-swvnD5R8{^4@|5@(WD%V^5E<nKFZ}Rc`u-@R
z#txV}TnmKsP5&M<k!qWy`&-Q&Ow{;#e6?rb*=cH^DM3stoTo)8fgj;EmY`;0Vw(N_
zXjmH(x_r>`>`^m%onW2yjdDLn5NPS92@)6gWZdN@GgAMs-0Grvml*LH?4VnZ-42X)
zfQs0R-z8$f3xxpby3_Hy*{{daP+Vqk#8P1%D|~H%OaziGY|JwDN6tgAGtItzl!8%&
zE?zH7K@e0S1tBv2lbOy)&RUn#hzM(%fZHy;*K-DGFP2a(6u)&3c8wz(1gGG9QG4Zj
zI6)*u?-b4j0vl*OJ%l>)(XkD#o&9*B%V+cY&JZ|Ns2A48s)0G+{8AZg3S(I!)q;p{
z<wi#pPG4Bpk%HnPK_s;iPw}Ew)tta~Aij12NM9h=0)$mlWfq3)e|NbYR#!YIm&`y7
zTt(!@od|(SAPpnyxNB^gB8-bFb`Tvpt)>F5Sl6P;O%SnfL89&PwiBO{*R?c)@C2vu
zR;Mpg1oGUvu+2SgEZUd;Ph1tr04F#3qWKDQ-kIK#L>`T;e{<1scX?9&%JdTLG@(n$
zT}_LLK)wOXYNP`b02s?OyFwpb%B&5!{*!aog7y_YIXMfDhF?9#nO8u5F?&cwJ&XpD
zj?09T#O)`A3VZcfLDG5^N`7N&A^}*kI08tBkX43~0`!?t3=s%Xg`2tH?scsPCb4aj
zN3jjnV*+tbzMjiL+jk5U%;vkA3wCNpbjW}a>ud}S4uo>rD9{3412zQB)#cwub%<Zr
z^&-Lqo&LM)cYv=YApws6qJvpH5PU)YcBVk|%lf^~fTe;P>`*Egipz;`+q3|;UuvX;
zH|8T_{(vQD>YtvZ3NDoRsP_DgQIrv^V2i5x?^CuBR2ZgmYYVHm>c*{X0%|e44*NV#
z7O|8nbo}(`--(65iN#{%R8Y+PVQK2Y-WnO<0|E)`5#5;6NQ%T|NPAu{D?13C;7RG5
zoKQ7N*2igY&qeOP;fw(6v&0UIbUAMF0T|_t>v7i$5!N7vlbwdiSma6Sz@d;35d|Td
zuHgB}^&Dno&NrRE+&QFGn9W11CFO{2#F8mWSg4WqfFSmq-lR>bp*_KE+RAp&=W)od
zcP9*sEe<0RBM4O_E~*vEt)Ew)8W>l97r|EWDLi009&|Pk1th>Q#O^T*%F>&+FaGR0
z&d1GtMAW8U;MDWjYY;Ub>fs3a$IEeFn6yFnTNMKz6n7V4kDpCFu>ezm$iu=zW!HJM
zW;SOkg<8n@j&m9q7|gysx-GpINNAqdi3wAb83gg(W{|<VB1db$s&c_>=vvCaA6o{O
zb>z798%m%Hsl?%sxvLX2KGS5CiJ^u%bZR9~j|`Z1*txzcPCIuvbc?!B0d^d1Gnmz#
zy%!IIYZwmR+1g$o8j4=TMr>7{2R?(g2<i>F`ge}xv%K1{O(A0Izd?}X3}9tBE@99J
zhm5Rfp8w-|CF(Qvl{>~DS5B;HnHyF9w37H+u1XK>7J65ZwJteieHcd$^RBn<0-Yc~
z&vOMv3DAirSC3L^3z=L8-3KHmx{Hz=Uf(eJxgyc7FcKSWK)-HwXaiw{$mJ7M0MiBC
zZ1dMuCo*{f;ZQ8kjceMN;WjfM16@_;eRB!(#*mus66^OD4+2$X#?Bb+^uIbzG)AV~
z#ElIoEmU^EuBe8HUM0|ik8#m!#K;f(*cx33(0w=wl&{xkb1ffvtk&&zrf3}4iu1gr
z4X-&L`<O$Dv=S84xa%x<*tsg;eK7>EQT%>_8&SgfjV3VFA>I(fJ4a|Bkr4;2?7Z_H
z6mlSBb=bH-YBVHyQV5*6aARFE_AxTY|DTm&e-<+!CzEvPR7Ip!)^+|V@JveUuU=cv
z1y_Tx_1!@Rp5rvX=F^Q}+6%!3#z<{>c7*g1gc$QbFe?y_ar{8wm}lD*W_X)G-9<lk
z=Q79a+NBz-s*8Sz(E-NUNXbg(nt@V9x5&UeLl85@230+ZQq|%RV;iXWZ=j&bJ3I0m
zu@cry<YydVxp?ERl53)k&b^V*%1x#PIgUH6E<}v@j_%N2haJ1oT}cwc!4@TNWtU4M
zqeSBDWJSA0-s$AO0XHLU6j;&?PhP3P&t8U$G$g8}2-Kn<R@IVkzaDaz(6K3nn4voc
z((5S&x56xdtCtH1E6@M+o)~Fmk?dF{8J)ztiGl23ZH;9p4+l(dN3aRwkJ<g69fdOo
zu3Qad0Kr4tVro$RkJFIR7i~-fodE1ou0Cwc=Er^PsF(>G^S4?{_9cSzhNYVa9=aHo
zSji&BNc(5evJC_OaKen5#$a!*%$9+dazFrPbN=d)YmBtOpO(gW+Tkt+1xk=2PBpFB
z+>mtP25%<?LfV&bY<!`Ae!9pDYOL=t6>cjdnV7#}&ov=VMtj(vBljN{iQ;ylB);ZR
z{VkYie4~Smjuw#23L^g>++9luLwU9-FPHt=tbOI5R&b)j(ww&1MU6KU0X=``%LWtq
zSH(ubip&W+Ao1MJ!6Evsr#J~13tQm|8y(B=^!K>*k42O7|AAAOkU#*J{o$EjJs|Zs
z0OW($zhniA-mx}^%Z*|#E<QvzXzlMJ1zZ*AsAHQ`&fEX1Ar;)_pW_CBK*Z$;LHR7a
zVbTIto$uU_!A3d7FMmA)JllTIuNQ{}cOD_lzz(quHybljAA@+wtlV4x3#JISX_S6{
zW@dhZGKVPCf%L;xU#4n~e<n~K4t}7dMc^Qg3xr4ba}$n=lM(O-X}8OK46seUM+?<3
zy+8Bch0)Cb%7D_2h}+UskYn_(a{+v!FqdP}+b+Qk5dg~m?7|ZvxFLX|GIsb%4Yqc;
z@_S0Gg^>{jMc0D;c0^eauA>|8?Mn4tFC<ant~0~AkKVY5<BEXP;NYV~Dtie)OaPMM
zx!C>89hh#l;($8kdPj~%+r838p>sol1*+=mt*7D@okqEhr2d)>QX;xS9V+w%P_+l7
znnEqsgz}G9uQOwkW@bPLZaOWN1{`@RUwcc%AnTP1(XqvY!ha=w#<F@iF_B@_9gNg?
zdpfV-XqqRWo}j-fI)t<uTC|rDJMk!b1jh-1Q_>|}5+zT+C2f9|y=DE5h=_1U4iAQ*
zxOgATDi%f%^pU%ZLJPE!I4-Z{#j`-bl@bE{&9lJz`x2(|D#3wRE!F#Pr!x=0PVavt
z>UR{VOa~uj_Lp>TroIO@-mKx0z&<n?!VSFA(xik-e|cV}oS(K0KpP<(T?!8<m}qAE
zM;XV-`D4x64>$)Uk)l{a<W;UFVW0_81xjDdpLYbCxzIG|s&-S@3d&e{10S$9oPM&9
z$!4~{yG%vB58djQ(=s>161YK#IA2*o%f3Bqb2IKE54JrkG+}=x_M`OT_xF#E0WYQF
z&+!f?o83#jv3u6z5-MEJb~yRUfeW3mgidtwQuf<Gh8jSI08)z$L`c9RblBgVi1(K_
zA}FGSyoQ2i-pBHPn>`1cJ%PB_Xl1~8y541x{D;#HWz&KmGUrh`7LM3KKN4chgG9(R
zloUZK27&TAWh<Nq><jTB^5!TAn#T}O?%d*60-n^`CMv46wZ@apZ~nfYJNaQ#z22{V
zq!~}%BL>?=gsvIyQO~>6*?BwfqfM}*XA4KaaIiyNYU0I*lE#L6_81zrYIElqzg)gG
zHMQD4Qq$X@&>q^udTTD|U1-*q%(=Z&!)L?VWrx2k$nR&X+C|~~B#@rr+LhPpQA!gL
zM53?*RzCQw^3M3bl?UVXWIp1J%sfP@3tkHd;=)+S^`i?s5r23Rb`zOtn|U{8Sz*$~
z&9Vr=4|c%;FDS%_mJYxT;td?+YFP&b{~ZeLMP3ra>x8sEU^HgqB?Y#rTvah(oW4dy
z+`aFxlnF|d)bo}(v}`Z3d{lD!U_Q7LW6DePm8oB~tp8eI>izV6!%e%HMn0QnyTLVl
zfD|ZfcA*#kBu0%pG?U1W2otG@g|w3&>P`gBPYOa^^Lm?TosOZAQJ!0Wxq()O>c6q@
zPngM(P60M<;4WT)#icAt-Nw@{8bIIQKFgbONUKG6E`Ev;;}sOt0#<c-3BG=y{zKMN
zw3y-9KR<^uVALZjVp>2<5-3g~EGdrC0RDoWP0m*nwu6UV&Nl&^wJrtveBgnN6O+am
zVE;E6Mz8DeYtEIM@>Br0g12Ap?CeHPzE=_H`PIF7ro|TJKm=7Y)wra?I0ml{__(+W
zNZ}6c=u?)Dypkk|lmo81oi9zg?$G3zfeUL4JEG^#+IlJC5e;o~KwW4NxVW5TS0dEi
zv><ZTVMpclmX?;1l5LkXIP>9~_N~W0t&hXWIPx~~jv3r%upK_AK2LUp0c||2zoO8*
z+Sm@U3+HMv2Li$S=ll$hL{wj#`Si@%VsG?SLMEQJxtZ>U8@4_Gns7chc&&1h(dz=N
zQTpL_M^#cVe6N0;-`cB^^xm^c1Ty^BkXA!1ldBQWfdbhRy;q*_FtWgx&z|IG#w^8U
zI>xB8W*79?@X)t`NT5~en5MH~T(K%4Ka_;D5||ziU(=czYH$QDOT$`UmvqJ8DyzWJ
z6w&*1YU-RS@a66J<5k#7>2`85Ch3QXYmlzOKM(e~CbLp6(8Cwjd-zGeaQ94$6LWKU
z%^QL$CEfb;)#Eut0%t#;4Q(i-!kvX*XFlN}jwQr^1e3V?GE_1MBt<k>mud12YB8AZ
zjNutW7MMKA5qEjpzU$x$YC$CyMG~kITQd?TZ)Hj!9Rnmgw9PWPTml=>X%ej9*qT!^
zUNKIeN2>$q6@df~9oi4}Iwd2|aQY&G>E!u;AD9KEXrPW776C_5^6~ED#&O!53AC_j
zSM>KE$239}mZarNpP-B@VTPlv<<B9}1{{H--M=0d{Ma4<rHNddh9K|pw_J!WvAfu^
z*Lvsehlo0q5X)?NNgTUbn=K*;I`=PW?W1l-E3-;6<icBV!9nbNt;&V$fhR4ZTtx8&
za^(@}qWGDf12ymxLA`;jE0K^5%i^u2d5O_ab26dBxTBXvTMgsJ-ph!8!b7%znU}cB
zZAQtl>SK10P%|vhV@J6$-yehK2uup3NnR)T1VnIpyI?h+d+vZS0P%mY9Zo~oyi<)-
z0mCZ1e`vgg8gj-_XMN}NWzhbwU`)o1p-bo@$^|;>#2Yt+fP~B^{8=G2P7iI(!(De2
zu)$^+VX$qEvjmb68th%B@kZ)iS{O{%$N-_B;4DzN5lKd+Yf2#hE2xBweoi{>Z>uJ7
z2<HL6JrZe}P09Fnz%-fN4LXmK=FnJ}f2FY*#-jKX1kwjgFc9B%hLn4*-Q)A(5aktF
zRNk!<R-0h*ss#Zkc*Ho0iCNSoury|ciTt=MwEBs$Ja1YVQ`o&z8IkAF8U5ISV$Hx^
zuaHs!z%&)moo%ennIGS@>bv3xdEu~hi`6J8<!sPRev8E13lW|me2e)-P&15!Z3JN0
z?Srrn5p^^ufd{ngDNy*r0$@mb{3v0}(;(@>S_Zo<k}(o+P7C;3HlLOp?V>u&#q-9E
zXl}9qoMfEq*3BrZSZ!nn2oscf)zZ?scBZtUXIaD``=Y$?(IO_Vd$7PqfGEX`YLqI7
z7;n&IOoSN7%3^tcktB|e1@v^9|27q6PD6lqEV8bv8c%j$m-kObC;BE+_Jg&cc6`KM
z_@(Lx?*zcv11-vhX>sp#2*-!)hjpJr^)-lxiMur<H(Wxl(BS^tUGcxZ^|s?}e1FEo
zMIr>xlF*SWOw-ZuQ>A~wv$Ps8Cm9Pjs2m|Xy%o&6CJd9NT9}0L*ko^_XVZ#lVq)TF
z2=FCyPQoh~Vqj?f0*eKdaH25zk=c#8ltN!WY;Tf?l>ZSDfHdrv2mf@l6PrPov@#WC
z(WO(eBVfMYIS=p|-tZtgf|z~=Fs7fQqJShW3yEmWU!RH##F38riY`QZW-z_7jir#1
zfd}R6Kk?}*p^a+8*g4VR0qu4gIcDR@Hwtq)slCykc*tCEHXN=xcQFJlV>zrPp{NfW
z(@u~BFJ9O=ZukT5k(&xgsZ;6dXIq6aH3U-kpHsuOfPna3zDM5*1-+clH(k})i8%vE
zc6@fw?c<CfJ%Dpydw&^_?||>e9eQw<&jhIrGY-lnkJ2OE{`{Ok0l${6ZVD6(gR(u0
zasRO?aB~nDVFni?n)%U94`TZNb<=}Ah0fztpyF(S3UMyPZk?sJ%Ca-f%+{DajYlF~
z_@D?eHbxK#utgE)Szxdmu-Lbmec)zh3&XgzS-IGqW@ERux+3mHCQQ7$mx><2f#c-v
zwFxf%_Dzk3ak?JPy<PgyPR#3jiDZ;tAOsR5FVK<?u=hZiaUVX|tbN@VZ3W9_)UFOg
zIDml}n}Wgz0=8mT5QOP?`;fkIV~lQ?0Q=1d^Gkz&e1bpagJ+Ba(Mu~ln1~_*au~75
zs6ZPK>%6v&At?B7MXBahZG#Q?Wu(f-i*>EBt#^Y>uEK|d2&lrJxQ;x4qOqz8Ow(9a
z3nd-BZjq|o-y;)0J8-5{3o2%68^l4G{V;<ShGZ0T??410d&)S{?qASDs|u&fk5lkh
zqawP2C2f&nmq&@$A1r3oUU$%Sh#o8m{4-II2cg)H(jA8hq*5f+1GC+TgMAvjqTS(-
zlq)qj_rgDy6Q7I1{y%`_MbT*?&lo0!3Z+1?dHunu7ut}J;5~fUL=>K%SunW>RcsOy
ztVQEM00x^P!y7h-R$b1xMiY`@r&Z|^9YFNqxu!!W&ptRr*{q$yrkdW~<>(EBfhu?5
z4I8&40MM(Z$-bM4x(QY=v&2MLLHiXB_IJj)&gcxD8(Dg4jymoLLKVoYLq;4s`hjzF
zY%*SijoUad@G#^jS|Er*nC5fX)kEtb0$X!tZx80-1|UD_-23Y{xESGGyhDf1^Fi86
zCLb1hYPJj^`llmFkWa8ve;fjRd#Lu>0#(n?$wqoJ>Z64;kSzfV@KF;!3qP6zo2CrF
zadl2^7>jA<0hlreBREN^k9|;2VDyS1LD>{lGZOb7=U|Qq9KUmU(9jp9W<Mg6pb<lP
zq@tc1<U?3QOOY`u17=cfkOXY3c=!<A6Eb2sCG!!HzXi^q19)!U@Jh`~sL_qyuXr<S
zxU&eJ7r?`Y(e;oIBmIf6McR~?SF|5z09J_ku=e&NR7AjBH(w+^%SS(8iI-f=R!KuK
z`7Nc;T!^ha1m#9msfd%X)h#7#OPB*uZ4Nt?VHsb}lK_wV9iltA-v}RqJh_F-RAAj<
zP9N&dwHX~(<jYaNnkZaaZ-|_Cpy#l;%k8H^MMOI0!1weEe31%ggKof4L{Hs}6jH}b
z3*!1X9s=i>4B<YyOa{59QJW-4#PZ@ueCJ&gVOE8JGC;Vui99QSA~j<XY>XyNKjw}6
z&j%!4gehwYD5vv~En$+4lv14ylT5($?Jewb#t`Q5!~0)xuQFwgJZdNSOO4|Jwn8Wb
z89$O@FAsK(>OY>}CpZUw(N_*&YFW|jsXik737MaoeI+}`4z)}icI{3;w{%fK*4KQj
zRu=`=OxkI!L@dc1kpaI}T~sL`jEh5#MU>NRM%qO^*vP3H@CsE##0k8ip&=r*clz^y
zj+q~j^a`I+?ixVU`!Wv3nJj#OQEx}QM|Wcbji55_Ea*G)Id5^WX2&83TP{<h8Hp3&
zHr0fEcX)iB#xPHNXZoR2k?D!A9N*ophP`r!t?1arr-*;DD{W=)^6Z$U?XgR<EZP<O
zJ+=j}z4A9({>n1@;an42<=0l>=r(r~QTeYvN(9n5*p`sq&&Ionn(%#9p_i*rtAWRl
z4)q(HLJ=Fd<cLDQyVTa!|Hq&pD$wk|6=FCkc8yMGdp~zX1Ry?|W%BsSt>Yl!*ph1N
zzU9msNQk+ph>wTaQozZPTRons?(bK{O=Q163^DxksiMmS%Det;kIq!#mFn_8sU`Bf
zpd&6k2%0f|^u>`(Rx_NRwV!?0N^Q`R-+FUp^k&{LTZ-vi{-d_EYku*66h<~(-pYPG
zUfDc#W<+QB#C`oE5GP-okW*46J%GiqVb5z6Z0_rQ3l6=%E0Z9!2L$NDd2HI7A_VoL
z1w6D(@t`y`de#Rj&V0zK)y3LA4l2z9PGlxf(<@Z%taT+TVfT_7?;kb0R8y$R^!w!N
z>9YANqt9=J%_l7{c1bbs?qJ&cz~ZY^8mEMlh(wF6)z<^RJ$}9QHP;Q9cMr_|-A!_S
z|0p)~$)!_kTPn@)_G51>x%Nu`pzU@Xk2>M7G;h7#mD6eAM`&WHT?1+LMbLx!j5~&R
zKbyZ8xafx%jtH@H0~fb*k-gwFwt8|#Uyjg#BMP7(d;&D=;O<IHPUeMxs^Q`3jw6v<
z*@gDs^i4b`q=~!^=q0hw9!Vp?tIXv_B~3-C<Ft`v$GP^Vj7J5TiGYHB$x^Q^WPQRg
z>zI4`j|$aB*}4Fq{@P#1S*2vO1}bahozw!$p3?@pd~_!Ml-OwTwfh#Qgi$B%aBWBJ
zROGQM^fCCV_;)d|F7fNXo?HHTxv8z!Ntn^+@$XBcKfC6fbvo;NGYt1N4z~r(TfNcg
zo#VpAK}0svmw(5O7G4n{#2Q&fdUKJH;0}4%x<YYMbeD|pc*2%_N|5oL;6??!_VXxb
zVP(Y=4FSg?6wnzBsq%*zO!zw`YhCtVc$q*v%xs=KGR9s?(0M*lJJ$5pl%vo_Nxv~M
z^<k2qvIl*=?eDY7HGl5TZJ0Xs>&XN$tvmg7-q@)Wdy~*kHO@n@N!v&CN@%OWNsEoy
z8=uWXwucFcdMKbT4(&uOk==Jqj$IeC72T$^cpGe;8UV=0KWsLV0B67o4BO|(t-4%w
z#N9XfCo58vuPBL)Mt&396{@Fgn#1mEA$jwMHqbANCL|r#Q=XljYFgNsW$U9gQ85}T
zp+|SxMG4a5(o_2i4_U$6BKssPMmqNN(o_lRlO5&;lS!MH8VkLw&lyRxf0%i(Z!N7u
zT5hujl%(JDIcIfsC(pDipKF76j2tI2K71Z)O=U~Dr}zHXpTts!l|RF(U#^Im-cK0x
zd$BXSeMokweEI2=F6;bvc}_Wo_kO+ClfyQ!ME;E3T^Tm_z0~n(<nZV0m}u=3OHp0t
zp!@YatW3;$oMX4b=UZ89JtW(EYVz&rz<vA4<LHa;J+AD|5~|+rnCi5e$$i**?t>WM
z_mwSqf*;-Jkdgb9?qQp&NjzoJb-&A*x?y2;4`^9|)mgPeHNW%4vw5DDTQpi!YC&|&
zY*sv~G?B}!_WtzDJ=>Q9m6ndfs~fUkO!I8B-P;$(@;~pq*%r2X`E?7_98|$6+kvH2
znA0GPg~C#E?aZ~Kkc!!V_>A0AZ~NAf92we0bn836R~jV54iQRM-%jUnOqWQP-TG>5
z4ea6r2Bu<-Kd&e(k892aI`Me!k<8*XDrj=X$?g-I;yGa*7{+;ZPRvzguQYpLq?Vo9
z+n7YtvX4uimuW(`_qQ234zz9i7M8MUs+4n%B}8avtF6GTkiK5t+S4;%3|AhX^bW6_
zo$Zmor>3~8zEAJ&@Ci}t{L;qNyzUa|A$;8V6Q>!+`so-d>J6I0*Hns}^49+BwWXQj
zm;0W&(PSv(+ZkrxKdz$@zTdt}^QbrSTL-^N0pr@g@5&Hk?M!5hW5@D5RKJEuU(y?*
zPH?JU2-q{-sdnpVZadFro5-sHCUOetLhzjX4Y&S-`~fL}#XN~Ws{_OP&uJ8QMcCc6
zkk(o#*{1nmZZmtOjq>Ra9{I8CYx!&n0Rm+<3&FR8XtZwjPZp9AJd7vuZDu>7_u<}$
z&$xs$Yu=qFmHECX+!(*wCnZ?ME@mYPuE8jS<_uF*6WnBr8A*`81y^Y|a+TP4rXg>s
z3Hi%WgeA-HY%~XcJwYIL%{tYpp#sqc)b{Mzh6w#)HZsKB?8NDaU(3mtVoXuAsB`d$
zXL9)Jv!D(pw%S)(E?n`7-VP6=t;;{ljqv)t@%Fh8Jo@7JCiSxVrJ+4Fqw@u?omT!R
za9hircy#Vi_<MiNYwYr?Ql&;t;dZNECG8wVWyQ6RXb=?oXBvH%)MoR?17AD5uWE9b
zEUfLUf9Gp7rW_t@3Rp{#n{`V2FGSY6Wf|r082dKAdgY><XWGd?Jm81df2RH6_4t*!
zH<jv-2cdO1_*9Ef|HI$sB)Cui4p!`<$(+1-c=5Iht#wJMacitp=e1qHN-e}_B^Vrd
zsO11-jtUP$fQ!7#!DF{(a&r778S`_Zf7H%xHm|>Z!9`xaDK`AW_NWI1m)kO=AD!(y
z^$E;+(z{`|D)pKW=Pfn?C6x-xcF$r>k{=&sX-l#mOKTS%_PrvzE4(Tz<>B;b${4Hm
z>x?1MQ|dj6u1y6?VrBWcOYioKK5S*%m-JjZ*+cQmjSibVA%YgUw+|n2j<E`OoIX1;
zz@+iLeUm6M%bfs=rR4>;Md#CG3oErU{Xy>M?ZwAG00QU`4ro$Q_S~YWb;Ua9=lIDi
z`cvVuzB*ij5CzFtf}T~p-+#XI5_8l<{};81xt}56FQwN13S#luF!*L*tNyqg`$w-I
zvIXz&y|QJXwF-NeG&^@)`SZXT(ZtcIUx_O{l6ohSgS92S3?^y~XIJGv#Iy27^-9?5
zDOHxxS=BH2p9_?9dAR)N+pYXY6ORUCkJ`%k150+rtXw%fCwxDQJAc)TaXOb4<VqT8
zoptf+w0oIRs>pX~bDko}rog1G;Vab{?GgsV`aag#?{~QprKhgB4ER25yScDMTm`S|
z*&Q|DBpJiMJhT5>?$^)@j$ht{ORzJ|eZL{XznJql>iX-cWar3t;yH(XhufR_>{^Sz
zZQK@MC0EsSLivsF*`u85D<vg;?~=ox^!@3~HAwOGGynCAAKu>*0!H2OAlHMFKU^>9
zlGLtRJgOHiD^(oOQhw={oYZh9bj#_3h8o?Hh36__0g4`*P&>Rw#Pr8k`p*?RT^_3i
z?^~_?cP%CCre3nJ4B!12<pi3+E!3&cZ9(<nm#TJ<PVH<6K8G4_AS0m;K6$b1+tQZ~
z$COacb=~FuVa*Qqyq@0T1|d35iM4O)fp5y{RMU#LN|98#-?v+d?=&y+d4`Fc`;f8g
z^ls_etYz1$avdZ+cp~j{Tl*il@Vky5PZ8blO;-VTNbYc-)^FF0`zJ!)TB^r|9UW6x
zy90C^3BG~lwL4^T?$GV}KB%PCtyiE+&(V@Yopi=?+-2VOr(tl-RMx#N6P`t<waZfr
z;_YD%q)!j}mMh3R76t0Hi0yr`hb8#tYUKs%^s7-1E;z{HGzwh0BFv^cAI!P2$}R6K
z3hLcC`!!qd@x9K5I9^7d_75l0zvSc|n_lyb+;Ttt$B1S=fADCf*yo|IM|Fc@So{-$
z@7kn@em~U^s1O*dAicoDth!A8@X&wrkWIk{ONXnm9aL90x{6y)5@Jj0id!AaKfim!
zZ<o*GD#_@x<er+*F8)5X-AVxL+@-7P*O)Z#gPm*slO&G|LrT+oy<iC3WiKKCM+{Lq
z0f1DU4_%B{l|rha-$MT5Xf_I)43S6SUOmDh-%W^BqH26uqpN$Kbpx%xXnYTwh|ibe
zgp%VQ#EvhNoD9$WQL6Xru*Z`Oetj=8iEFMb_~xe~gA10O?y3D50eE}gJrAEaE1xX4
z{q>t5u|2PYIY~|9>CdU9LMxxpZ9a3>8nwxS?^|@#@_58H{t6$20^zKDXKw!vqo#0w
zJw?}x@o~3bo+xWPO?LfMy|J_UV=?jWKs#s0rUqr@I!Ch4(letES*gSeERJ&AV?#@q
zTZMZzjO^`cdZuvU`Ou5idps(VwEjKjG-?+V%tfw2(m>ltS-D>9@Q1IOAQa^V`W^hw
zry<drSXF%Q#>BK})8wL2$w2#Bi>t%i@duuo_o^h+3JwG>6njjxx;4>lZ4Rk(E!!C)
z_u{@q(?{vRUzK97yB+pS+s~kMeY_;^I|64I+!CLhb>Zy<F9;)FG11$gQ1vVmaus9Q
zWDVuc{Wx%mp5;0`_R{A5=)<nm`W7fmDnL@|dBxSeuAF0)y-ngqa)QlAw;4ZFKf}g8
zXgf0_C-J(kVZn+|tf`yJsNzH2f>!@M`L=jvcZqMmYWPYUGJnxtC98g7iu&!c;xBmi
ztYy;oNIxFk+nQH@{yx}HtIR7`o_oCXa$a!irq8u|gQI(XUwLlgu$=bCX`lRC->{h{
z<*S#S&&TWwu&!4Xp1Yq$cD^?LlTU774E#UsrK)pnZ&kF2-@miG*xa0Ha=#{>zxcD7
z+UtswlcAGMO_uHD=F6=tW8ddRMvA9s3kL3Wu3Y71G@5&$ovyT?E;0W0N0#qz)$LIo
z7k!fxH@=h<I3*^REhIY;RAlsxzlOa0Wc~%QDSV~UQcu3(LHeKKkQ(~YPH`9NWYWeS
zdWQV^rb&50vxB9RgR5@ScS&dD8GS|=SS-6He$j@!p^rzom(x!TMez2JmF4%Z%7tpO
zIm#~$L<0OOg3wKa`u5SYi_q_23s8Ph=tWQJ(vuhY`a6LWs`x0@a^9=hO?b6<#Z#-|
zde;%}>Y7=;@}I9?X)sUrSF|gzj9vC?y7922sIWa|BCz}AwO8WN8l3!#zj>P0<P7wb
z6`T5q%ndcA(qAh*EHv0pX7lKBgg;DtmGy3Lhxx6&bF-h?N2AN1-Jj^$=wQn5&0O~8
z>!YLl3T<r0ni6*X>~J2>^mpQ^2pzULoNJl76h9z+5A-fo{ni=gui4uYdpET@SHbpy
zn`0%po4Iwn3#a;=*}{lax`JPkg7UdbZhecEoR*>fz+&qcALRe4_nveHr=8u?Q}r@4
z&6(xl3$Jt@m2Ehrn=!~Q_lzUv1&Ye*iA5Z^4ivbD8W}=fWdwFiwvova`Kbm>X#m=j
zbWQf(;QT7>5Ri|I&E3%8lh_U@4``H%nsLjxyeBGfq2aK1+jp03oVTQEu8c>;-@o2<
zyxlp!z11x=rl-qsa#6bYa`%^A46Wmuj5nX_6iCPqZI36s3Z5#xmf?Bq-j~L3XKg_2
z<au166F=1>Zln;p)*IK7ZDikX^kAaGV3(NYUh3|TEyk=;pG;&IGP@k-{&b|9ws16b
z))*aTo&K5QwAWAOcXyCeZLw7G-PWi(EgIi5OgPo0>)!?`#c#YKS>1K_*|kk?_oYPr
zQpx+K-Kw?!W?H|{StCyVYTUiXAqU-<g$Jie)7#Gcj2HE6>QM7&80{oE?J1KR{khR8
z*{E&oT<e8}6SVi+7LQzzmHWh3afzVgB*d{AONhO^H}x*pe{#_>RbXgOEtLVBhcU_w
z6*Li4B5I?(NBxXCg3!X7ehQy_u{@<Fl2i7}sY4g{MA2pSxb*zYPqdngEMv6^9}zAJ
z8n&w#J~lTHt#>ZJ-MEAN{6*W6gKTliO#`^cV(xbwcvozI-jYph2tO`u-mYT~-Au<1
zmX_94<&H~u`?{=OPlH}7&&u6g26mXtS6S@8IiOu2NpIu(^$`2-?DpvNq|PS_@0)E(
zjj|`YPrv&iyr16I%{iQHG59{=)zkR6WO%4r)uW~+w!R|wjNL6S+v>xNr+SSvEUdi)
zHAdb)SZT;^KTl`lYoWl-+LddtGyR~HyIc+V*PVKPXcJ7lT*bvWa_4!j5N$Imb%bQL
z0KT^q1Waq{G0JCg-o-(MqYBVW8daXdqNuZFZ4}DkcLNg;q}L8IGy(Gj+QW=FRP$AT
ztWr<x@=s=y)uZi)H*}iv+-lyS`+s)&!p_l4Uo2wuUYLwU$Bk7SYs(ZCRt_7$6Uw!J
zemW-qazWYc@V4sSO&?y^Yo)IcyYI@|ypb8%@%ww~LvkBLim9Ecev<u_E`F&$zs*Gj
zewZkd6?f>fm-zGLUW!Xs0`MTJ*W5kAQV}-WtloKh==E6TiZX-3VHc@q^$cNe>%yi<
zPk9_3XSc^bF3ot6{w29@38?G{KkA)tkH5?<Pi0}2R}KzcBhu%ciLY_WjVtMLe&*oZ
z(Ls<CD)gVdDt_uK^9$W0w|*y;+70})`lzEj|L(4?x8v^uoj+ohIahvEs_*QN*;vF8
z^pn-R`gC4q=)6n0<I~;2Cgf73z5J&V+HA}}59arr%dwuwkFDtK&&hZr|0p2@A|F9x
zfu;tb&rOg(%%6Q83B~b_xNaO#1}x<j9w~DuE)d{Yu=+&=;U|7|X-avbzoK9Zhk_Zr
zr6y8TROJ26M8m^QRcyZ@^2k<EO10whd3LLg%SGE*&F0d6Wy-B6y1WfY?9aEk*(&ZG
z;4V9wtT5IP_j$a3Nv&r3iA3JH_{(wHU+!pRUHeV1RGDRc{dmv)J=|*1I}W}d7ud@F
z`pz-aZAH?;#cPM%W`Qwzz9i72tzh4;!|k#if(@qOa}|Nlb<&T;g)P=Lcov5H4M^Tp
z(!I55u4|#4&F4|rhFzMEvcxoJ?44gm`nmYM<$U6P?w2$*bMeT%7`c{rCvJLnc6`Ym
zw04hLo_{YY@zs%dSm<S=dCuPYU-Pat4Xzdhje^%fqLq29B7lqIDC+gfp6U87PT%bI
z8zO(BqPlTg2J!l;>ZD%>|M>X%NPOsryTNiP+`ctQIou*6J%x3(H-F^-tLz`Yu99y(
zu?UaxPG{dEm$|9@ELN%1$&L08^=RIlKAqTgyS>|aZ-!JiRkTu!W<0$$d}K@bn8`%f
z6yHS0FKU<h=I@Yt7p*AP(~>&b`JSIMZ{Mq%UD~b+zE{ni*N)$!&dmvvOfUjB+u`H#
zym#Er$qqG*PzKi3=)kb{<r?>jEs_w39S)sMCBz=zbHkOJQIWdNvv*410J{C^L7AG5
z-3WpBRr>#c9{{{HD{zeU+eXzMY2%O$N7bO%Qw6?@{kIHK_fxv&(}(JO!)-Qhv2f77
z-`?@|a^ttt3=IaQ(rmukq6~K95{8qG1F;LaVL1<};*@OjJ-%m5nv6}YY}+QCv(NY9
zRlR#Azw?iE7C#zBfzROC1*#6`(D7{PiM1uqrsdToWp9Hk+_SGwT&M_{zjE{W%&O~$
zhjHmN^5Q?_od&nN^gBh;iLN{lDGsZ>8n-GK?k?S5?Q*#$JzQ(TC-(KZGvU{cUf$aF
zEbmR?aivM&f~+9#=e)X{>Knvs#|tyJv+TOo-ee}Ww6IT-dIQ5l2NNmwt2ru>;j8^;
z=6ebHce>^>Cf>M7gz{fi%HY4}I8irzgOGUL=>3TcmU7&h8e+vOPKC)rfu^<A6?>yP
zRDV<l->(1N)61{Ou$X2bWO`e5t|z3)x?;2tuIUrwt+UpzUJ87GQlZaA^|iv%(5_<_
zgq82JdZ^Q;Z#loZFe`kkZDkvS^zZQ(azUD1<_jaG&&`#C&w6C!WO3zCA7%J_`mF(r
zdy~ZGoAx&C!HvJp7W7E$omF0dtM;sIhJihPJ1SW|W+2SdE*gWga(r)=D-UQc&Clmg
z;OY20sE+hV?z!Xu5Lj=>SEwuVz>Q9g7l+bHQhM}$rLHe7;|!Df>dmJwRVsgdcdqd?
z<C8Ff+OU##<C=o6DV?QLJSfloi>G3IJbruL`)H-Tqo6gZe5L+XCumk(z3%Ar$7Sd6
z`tr8cPN_S{?olat?fQ1n{P#?OJ8|XkqS5yuZ}Bi4MM0~@Zqk?jz>b^+g^4p)tEw9f
z1%H!2WWCU_H~qTFux=<og8Ni^$;3p}yDtgAL9p$5sqVsWNbM(`&G|-ie$#}hg{Nd;
zUVB#h+_J`4Wr>(RXQ@K}M<qe?^1|)_H;3!Q-5*Rh2(rt!Ur)JJ+WUy8c0uJs?VzV_
zub*|Tfeex1yv?hdkDln<%74rf+;L4uqTsNu^rf^C`mN0lQn+VB$`ws}3bpSlnqCiI
z47ID=IGr^aaBHTi*1k6fcc}Kxo{2*CmE-X@$!ZsP7`_eLtG5+enzs4)C?|3>j9pPO
zdTt}hXm%%rZsPs7lRozcZoLl|J~1IYCARb?EaS(Nvv8Iak6UNx%e|hEQ<}k;**-xa
z#vj+`02N1Sh^eSI+aUL<pQW!pumUg@@TlkL<4yaL0luGZr!#NlfB-g}I`ESx&j*J4
zuwTjM*%0=7^tkMo%q|@}6%GC7GaUx|_e{QZ4jC`Jc0Y?{d+Dr0O8mynJ(hYwM(PFb
z^7l?ZzAvS_P_uEbX=p=BtaeSd+suItMic&<gK~1rzz;sD9yC6l_)2{k?;h5DjG;Po
z?P)3SS?%+NAgP#1L+$nHf*h&X=?$+nHi&=gU2_WYz46Fv7sF67)D~FzOlGsW4JB1=
zUmu;bf%vEY;l#NSlpB^{S7Fu*?CZGz_Cuks(3p<chwhqUu_6%DG5AOEb03Y++8l_`
z#bJOtxi@X&NCsFvqO>?Vl*#}_Qeq*<`;|R>bd=JbCJ1`Wo|tTIAaBRP)7#Kr5-|+M
zY_a%R5%lh=>XHF=m<*_;HgM7NpumJG>?~GZD&3v?xzM-%Q_GEr&_F&#E?3b;+zcu>
zDxdL&f+G*iEY55iYWIw^=41dpr9fw58*~n%XMH<!G`5{_P0Zw)YLBFZ5m0WrnT*Hk
z<5mc$K!YC8{{IcN%jPcg99F`5!*(Ae07%bK{c$q94JG7~gbMMDfK*J2TI|DW_M|+R
zhSh-Aj=lebZjhZ}&0!v)Um>Y~lJ34ugGFLe(kS4H#oI5k%6jL1hAz1Kl6?re6_pNS
zrFA48gmEJxE`V<j1XcojaBooE{gwY6W7+^~*vekZG$1M@H2*`50p2wafrI?M8rJ|-
zBadDxY#JN_Iyj;IS15-75p3)+)W%6>Lx<o(uYEq^7CMBY$|9iuxhPTwTxbYnS^ulY
z(y$@OFA%{^ygoiTS@3i#+O=0OpXSW8EvSfzXG3`jA_Cugfx8U-(%bTmlJ3Fk>j%oa
z>8bJDyUgGvc2v6iS#$7c<d$oI#8A)Vr$b6}2pyY*#dZwU<5dQ~zDatk8iE_+?KAsD
zlQ&bBf`d5|L~o=Fr)i1dgP>a<x!zX6-z$-D@M$O0%Cq~UnS2rg#Vi?gKv)e<=s1Le
z73%$t2!^#)7Zd|T)dm5V@_KSHi~$K$Bo|<l*+z+qK+D!ych0i~5NXX5iRkwkec<;G
z?dPY!G&jKpp0&%-=f!72QU7`W_oEYx(|2LMrBw%hfzXv<Pob~(2B&w3$!*RDJiy7k
z$IsDz9fAGgO}_OCld|oA$>3L+c8G}xhJUVp^j$y}=Ygu!V?8;O@ZFt6c%)g=&@d4_
zkBz$ASrvmG0zIkNUon%0iJi^i=Rqe+y$M|>`ls}F;0F=Ljvbijn5F@~vb(fW;1?9b
z%kca<X`H~HmbCnPt_?wj0H{iTCR_?_IUCf$pC8gyfjl_czQ1-YLK^8+4to&|&I1l;
zq-s}AJvOmoOQ@{CGlw&ggV4pyPwO+5?ySz|jv`3>c2vUVnsmV0b`tDhj&vSW<r@D1
zB#I#mdPLX|SOsa1*+G+DXZ6`r@cdQ$mOYZ1@B$7~&#D{3jLR5;eDY*mI)X`SGbA5%
zVu@ND)Ke%(waGB#!4D!uWhho<1@A@(Lkpqz2D3uv&5QK!oX^e8)hsg1%TAQKf@JK!
zz%ck5RL7I3e9Kh<Q3v3(4`In1)Q`~No@q0H*SBC|(7%F+IZ@l4?_{3=YDd_g6<}}X
zQicIYPQ%LLJUBLEZH)hea?}G(=y{#{vDb`VfRx(3pF&R>MYEMnhI8f5F9~8QboXZ%
z9PG6|XT2_AWEnKnneCg=v%s8;o6xZm`04TW2n}ob9jKqYVqmC84|gDlL>bo70BGFV
ze&<_SaG;KpeGd|q!&l3?PQs27NIQ|i>CoAMu|smU!r!vDXnWHJK*=h00X@h}MyMY>
zx1Fu<Jtq2bA3N|YcmRTSxcy=cyt&DvmxM{wfANs;d1Sts4zzHSt>L$a+$~)rBa)Cg
zs__#5!UnJv?(e0FY?~Sp6R%;Vt4X??-(8@skeu}GLk+TCH(`w4!>lwE;NE#KYXY2!
zc-$(|#M1fvDq0yen4poxd2djZW*j{*6v9)wEeP|2pgCZ-HP($q2^0Lvjla)I7=R^Y
zR?|s5f(Qldmpk5cZ%40)BTguNAi1WsBy12v$Ugu(Xd=)MAD<e8LG1o=lOPlS0h6M?
z{SP{`2BI+I`QFF}zg-aI`P!e)fc<8ozi;;72-s>zI7!rPiYOldAtJmU`v1RMqZw+Y
zOn`c+Hqc*whIj9i0H)KzAU()vx(Ad00BzWif*VP#NP1~6BCS*vkOE{Us&R-swuRfF
z0Jbr}H3={Us%m8Do<Eplhq`aHw6zm~`@2oBAai@lLoM{0p;s~5e<^kvjI1Dv0Lp|9
zn2ZL(Ua<ESABqm+u7K@H8sEhkj!8fCk0A_9%T4)5I+-#C9AS$e%jTU=Ap!dzWETEL
z-VM7RM|=vcH!pQyP$!`fFuM<6Eao#OJ2eo*-fg(i8aV}!mM%vu8wjx~yEIvmv#yv~
z2&i%e_|m{c!i5MsPsbelnhB=3<pnI97tqH#+}$mJaeuPitUllY^tlkATFVEAkOVlj
z=#^)J_b+0%<up<`G1RC6hKTVc?4ORQcnI9G=Cpka%+S)C`D|ARGA*j8e*05b3+lJo
zyk)@)9;i<;mR;Voq<&T#;S8baNLA1;5g9ZEeo(_x4BCE}aXbR@%DkO<FV}woXaDg!
z4&s$oei~`%>0Jjfr50dx9Y_(>-kn<eFI~fBAV8cFvZNIo3u>{|0s?l+Cg>Kth@?#C
zpcJ-YplO?Vz-AF1;<s`r2%y#k4#l7}XA%Mt37sfmEB>s7iY?P6u-vICk;9bbBD{5v
z)O|4aI&i2Ctxqn4H8};^dT1j*kiyeilc5Bfj4+TI`$Bo+73b4*(hjO@-XXvkgmXg#
zAd3iab<6TrEdrOWqLRXJRl(SQv;xnKR?oCg9vr`ic+?k&*ct#e*8)TW@OJpVtEQc~
z2*SH~6h=}WNVmXxuX7<0nKX3a93;)HU^*}O?BOzvB7*GhzMaSmix(|B^<t|_cj2`I
zW@QA#*?@YYpx;aSC%%tj#&ltyG_bD9h<!+6AG|A)z%e146cv9HtbYbbVL9Ck4_I*x
z2_VD#0Mtzdtf2OAV`v@(llI|U(F%Ts=z0r5=jqI#=ZD?Gv9N_(@vu8aq{b<)#p#a+
z7aRasN#I;UBYM7oj@txXjNbyao^}@+7*@HNBV00|cNH><9#F_?z#=^*A}!@4YrqP_
zfEg^7pb8^!e>&w1l8E4aZcoq;I~>Q5GMSkNpaVYzyp9%NEartyk^W<`4VZ*6Y9>&z
zOw*m;ZwU<wq~A}lQV)UDh09D;{7`SamHGv`rrf&p55f)Mg)sPXzJ<QudUT_A2x8AX
zTY5IUvrNgA!UU|5iHsYP*40&GiZY9sQ)|A+i>3)^&pB@j+VcpqpWHh#z<r#jk#g(f
zgogqW0bD|G?Jnd(?8VSg?WC+SkAYbUoR4u<r2W;@FFB}L=)4-e3{DvsgCJg_`r>FS
zDUqT|Itz<t>*d`+OYKQFK4*L5@ELx<;3uF?1}H~0n5ej7#1Z6U)S~3j{z>CKL^l}c
z<*vQ*=$1dkOs;3c{6OL<@DgM<ZoP*cMfmc}Wz@0@0l|KF6SyKh8Z5DW|M3Jx><}5f
z(3Wj=-4cC1;wg=ctxP>p02&Sf_cm9K;ohKS7OvRS8#fap9^tPaFy}@uZXyI56f5(9
zme)x&_K_=Gv`9Xd50Gf;F)d)sqWTguUEj9{bS<iq>dUAZeUcI!AqWt^#7-sXJ9MK%
z{X+;J)UGfWyI$@bjTSC>X=*WvUFQ*SOIAmm^ZuqUh0o-J<23)S;)1^Orj2a_$QGIy
z)_kELFg==Ts}24hfJfDDNPCJRA6X%AHlQruc%AE)Nt7(R88*b2`VGq+?FXn$hjAZ_
zG;@|8x<)i1k;mV#X$pQohX^v?62-*xN;B=rCHgi5)~tH(K`*frEDWrMK(dB&th?6$
zoe@`dkVCF%hR&jkIc^uZKI9?YEwDY4pk~cOX>5$s_elsd4Hg%p`b8Ynz(L>*NZ#P8
zM(CS>(a@DdA7FXD2{c>*=JtZw9x)s;+M>Dr;+KZGfz^9@4Nka!JBE^RA20@U?3P7m
z1ec#%-cu;3A-SwJfN`X&_Bg5o7%Ec?14F@FOhNW55<h}oM&*GfnSD)^!g1{AtMMtI
zBcUf(5M?52rW*zJt_3??V1>bkGpF~WzE#O^1~#MtPVy&5Ju;NrMM#0N9U_EP+C@Oh
z#wx$RX~B~qLPR{h@ZhE}YQ@e{3~oiZQjpyL9ApMMNQ@=ulQ8KwcjM4!6kxC&kC>aC
zxrnGF9w2xyy-BAk=@dr(fRqv%d-);Yyq=uAqXs-9Mbl$<F$IVqjKc|B_IP%kcF{Uw
z?$U5P)mIoo4-{Y$>n!=72~BV(0>obnkf|Xnzj~dt#TlMw0yel`FP@zu{m}NqGHr8O
z?P~-f1TSOT{36{7&V;B>BO`WM=$Stpgf2)~M>nqsUV#^E+({H<q7E57<rUv_Dfb(L
z@WL_2T`m)J-C_D>MRFdFQ)HUXp!!1JglHhBs0--5Z2S>J6TTkqs^=NswTl^}G|BMB
z_;nOl|0mm_b?3+>ZtWy6%V@aiL^L^9-LK~fZlIdYJ*nOBKZssva>CE&VR8&;a{9as
zFu8;0eptD5=3W8#k3Qn$W7sS8Q~!J6X3BYXfGty!i_1M9u@9EHcqgR@5I)sfpyU}A
zG3DH31QZ=|Ip%sEOfPg3rAnUZ(Q#_e5=jv<NkqOC&<X7}xG_oTM~yfE$GPCzW+;Zw
z7K7vblb?%v#Z|dMI<_m$1C0G27@5pPJU1MA0$d`Vrsdff->D(%TL14tKr!-DpP}gQ
z<aWa3P=i23Dl9Xg#;*vhJ;?K1`CiMb;ymOqnA{J~qpk=#jyT=dd8Rd?-5UN=YfTlq
z_5EibI3!4&nrOYKr&6!yXVI#P!rZ)%VmDW|Ak6K?ja*1L-9cWGN%VKd=`<1;L>{=2
zW`Uwe3lc7lK=6ln_~YLte#MjubybLxPmO5br44}jTw<|AT<sAVZ~}ETzXarQKhRY_
zRF(6Y*5Mc;!V`CTwHrA^I|TFz`8w&`@Q&s2s!sd;*wpaLLSP?lEwh4n7#op&eBFl#
z20$^vJ|e}WbwHg$!mLUfqzgFcZpAuX;ZEX%K7u5G;vD<<<dUM9(Orsp1fi&oh)203
zG9OqDOi%!=%?N66$uxH>Z8bukE+L*e&`}nf0F|tOPjtVbL)f9O>?E`mCjHNNaB@DO
zQGn~0_M#B%nXzCM()Le0uxCndJ_Ar?0xVkY0+}6pWBMx3q(1ccV3B|PmT0e)&s7K&
zU#hWKf369FCN*>b_)mf)nh=G`7JBM)^l;CeTQUd&S%wS2aSFZKN@9R}Zt$&kG855l
z^8w2$9oOCx5r|bP5St``lDI%PUOBl=Vkro)Y*nBHSjz=D0PI@a{$?1Bz}X0PZRDFS
zcr6KVr$q7q{odxu;U?RlUtjq)Y}%SkFIN9H_IyXjL6qzV;pAjwhm*`1(G;RxRDqM+
zhd5v0B=K;P80LEjg+@24a`|a@W7z{_Gk|-e^R~(cEV0y<y6rFTP*gTd$)K565U&6t
z_GE-X=NDWHelR0zj8Nh@iqcgj4C$cH2Z0$;TPjK&0C8F#_>gZY69Tsk59dcMq{U4(
zfb-*UH@gesS+Wz(Zy+Do`~TOim%xws9*a(tX4TI;-|OA3xbPA#%v{ZD+|z%@ibZ2#
zL8b~C%gq=GNeSFZlV8VjrfI8D@7kG+xbr*-{0E_f*k!Q#0i*{tBDb(|gL83VBSm@e
z_HcuTV5x#_;9E@VpEBGDXLk|K?l__bfki@uj220hBmzadCA5E>QExIX)K@{+cxd+f
zHz77Op$+2ZoMa|6A;eonh09`s38m-fa?@5rgb{o1M$8lL1paCCB0{l6r30v7XN`|D
zY7NQ^_;lspSea-nX32g|GBq0OwCY_r(yZ+;RuNFE(^jKqxHC7R&vSEnlS$M7Ai=t)
zqFKa5E1X^@LY8rzYZ>4syTH`EAv;`-hyhbm4UYl<_fCKzn40|=DcW4*05COip4^+G
z2scPzoB3pA%~5wg%H+jzC|b;ZY2Su{{6V642@&**_}fW*(7{X&`H11}ILYR4Z)dq%
z2t`KZfX)Dso&X!p1&5w}0WUyns6NmUG;fx)VH<&Rmq9vZ2TX~$=x{Zn4NP-2g}^Vs
z2B!2SBW*J`SrbjkZP(^YFeTJm1hh)fCRX~=`rHRT1lY({i+S>IEVMkV9#X9dLXM@0
zBO=5=$d|Iyi+FKyD1xfyze9jl7)0`hVoPuWDCwgrH#!E&Z9yxe3eE(j%a#U|&Wgfh
z7Q#mg>_72Z*OAE!`Tj2?X=qe9b;$s5JlvHqvBLqV&?|HE^R=Ow^dsdyM9DQU$G4x~
z7C;nqbC;g(jrs%zbPCO=q^0QKe_Aba=@5}ZBDnX;D?gKbzjkXu^BT{8*;Cl~!F)d5
zbMX@=*$(7js|Q;&7>ULmAO~$u1)a3K2*v#*BLc@skpsjk`ThNgH}X>oDBU^e^^}34
zd-@{9qEu7_D)fFX<c5zT?*wBgWkXQ|mWU;gdO@E<KP($_5HCWoq_ta-6?CV}<Aa>U
zW_69H2uqGy&Y{94MCoOP0LBhX4EvxAJUMY0dH7fZHKIiL;yAnyN5%QSUYt%m|M9E?
z?gJ060c~I#kcy3h3Pk=@LMu*KC;gr)+=%lH*168UxojITCw>_sxxZn3*8;9-Wj+IS
zh@t*baKq|!ZYf#6QCe~vuA{=N9%>kZ#T`3lpoia%?&=wL0v+;jU<2D8@2OV@0uU4m
zrb_<ioDL@<rqVr<r#X=eumuwI(10j1pGuX0*>$jS>^T47Q1F7z?MO3(POc{)u)MKZ
zvB)dYfjiOp{|cPlP@B;)=<;w_UYzkBO*x^htsQL@G?6#&x{8Iju=z1)epoED{Q%4l
z6JGFJH^Tf_2?5XwdKL^bGmDe>OdSa*b%Y>&R}P2ZGThjpu=2AD<p+M>yz~df-EaaH
za01rveRr@CtKsg6l9y#g-+2Kd5~9Ol3R8}SylgY=bq6MWA|wZHOuT?rXZI{>37}L3
zl5BCA_1l8%jRqQu*s%}L0Wc7T4xqwu(R$fzr8I+n1xuK%r+pqYX&DH)P`>l`^wVvK
z0T;cu1RTLs|Kx%i22*?+WlT+!0lA1mmJDXawHXFuihd2Z*<CbPtCl?srjsG13zq~R
z2);OV+6#{0Eqf@}D&sVY)Ua++LM(Czcc*<u8-EbFgLKcw;ELM`ro!MPFB9B5{*!%N
zz>I!9)h0*_*yT)7lTl{ykD9+WHR1*A`-0g8Ak(u4q34dQ=0%KN4o4xnm3rF;xD;-b
zh&seY2N&!#G~^wFjv)ucM*!WQF1;l;`)<ms|J^JR>biJ%KdbDKkU9i{oL|9YU|R#{
zs0v!d^M6J1RV0WK(3396TdG-}ZG*<0cgvN%a}+*9;HcOS3a~JcyoZZX7!=?^BA+$7
z^MhfmiZ9P?V5O+G^K!ZG{TCcCAlE=<vIVSl9&i}KOJ?CH^VV)s2=OM`<{=gc<ColZ
z7?_~l>r2dfyBD|vZ`wM}^1}5-3x1LgqNs?#u!ty?j%Y0ak#VT~e24Rsvh_PQLK$tD
zhP=@+d^Q(s*`DZJL_(zsQH=M`16mMh9zn3N+MSl%cKH9M12}>w53x#nWM=vgxQzcD
zEFKMJaLUL8&ka!|6BlP45@7?lvDqB;6CprO07<>r+^h~Vfs$8<R%*Z{o?X62=UC)l
zJH2=u?DQI9QRb|JO)nRI5e91}xPf{7|FHL#K~=rsyRg!o5`vVJg0MllOG-+*_m=MN
zmPYB41_?pBn+?+KmTorP-OX9}JLk;2@67x8{qmnZ<G`L-vet9gbzk>-uBW#rQ4LTb
zoCZ8AYQIwlDdIPk-yO}9o)5rR%K}@DfQ_lZ!736@Y}gZO^M1mn{|4wW*ua|4h#CQX
zF@UUqH8%>A8wGII&o4%Z?5mTlu_oEt{-T1&t+z)`DKj3)cg?4u#(Ecu#?psQNAW1a
z6>X#wacVk$Qxf>gQCI7KU=^<s@ZDJQI0OV=e4tqV!{^jX#DEQKpXBZUY5^FR6blt9
zBp`G9H7lYBjjMe7{)WLN)H%)x^cH{pl?;ym)>ER&G-**?=|w#zW(3Ar_RU`)w}LmA
z^;C@sz~;gg7EwQZI|sM5sg^VS{2xYbUJDe+`GCkHo|C`(83~f1;bMgGlB<6-NejKE
zCREmOUATW=J^PN5Buu(s*`XsyJ(}DKJ(m%nYV8aE`=662D_#J5YzY`d{bJcX-#<0_
zdh~eF4*>vp!9m9W(i5;n6)#$mc6VNz@ar?4$=4%B0K0sY``_gvl>6hrEb}$~kG8N6
z(LPUY-6L98ZW~aAvsS;R0syoZd1b`HJ)O;9Vzo!7W2+yST<S3`O$kRobo@kp*A3+7
zAoRcgC&@n7ME~#NB8h;Gr^Kd>*_Aui$L}@?^j?v*GH2631l_&)@A7DA0T<wM8Kf=H
ze>WxiTLj^F3j|Pro9;RJK(ci3cXuYEWR%xBZ>aqK-_s>wT-CyI+w8ZW%T)s0paH@e
z$$twX`dj_KiyL-gepEa;*(C5gXVsq!u2?jCBbI-C-*b`wE<aiwQ3QHeYJ%rC;s4zf
z-chtDU?^Z`F<<5{K;F|`kD4+gq!*n~L&&y-{$B|J#=XA#r&}OB&)+K60PT_d>|XM}
z1;MjV{O{s2e!71;zA?3;g8Y#sPG$;p2=iLu|5o;Q^naJ%x!0%yydj`(@bLP-n?kyy
z{m){4QH++rZ0&!{{XgdZpXmOd=>GpRnM=HwZnBM^IVlPM7Th4cmbU+BmsRrU7?}>a
zFyz}_6O3h)!9a5trKY?`_9w-C^+xS!5d7&WU(57LeId8{C-l*QKVA1=c74zZGCt)I
z>09)>lxYnp=;I@SyfG=(V?+*rfx-B;E;8_+vxoD((lhq|?EsX6D8#jji8B%$D?C&p
zv*8n%$uKJO`Z$MT{2ZBJdQ6;8%MuGv*MIdlPky;@+()eM{rT$tz)Rfs+92>cYHoA?
zW!z~O;c6Txg?h>OzrvdCjRcVN09?R&x<!K*p$eUNGF_3i`{ea~Z31pTnSc5|he3XQ
zN%x&h*%rq2Jw1f@4Z7}z)Ur1DR1SVbDd#TVuTkQI{zzBeEBSpR;yc6YXY;tw0@Gc3
z@z$}u(ck4Fd>CF9k9d1A_;NFYzTgFux!({R59v(PX89eTllH#b_Q*boA{lr_0~BC(
zNoo-Vjc{Rdb2BLvxhssIIu&mJuR27%0)T+@yAB^oKm4-k*!^u#lvcw!(Y>TOORe9b
z@bz^wmBifDqwactmgb+PU+!ABycVl`Lel-Hur?-oW<6FLQ*2q8k&W{#YB@7jHV;k3
zGKOepE8<hfyLzu^=cfFSSJQsK*UryIwH;@*gN-slFM}BDx0YzMFK4Ezd^(9|Z1lZ0
znca)heU6bGZMQM$H4oy&<$_A9{zDb=b*Uj4Pk3w-sPtp@;8Lh^&WCxev_*#s@0$~W
z6oifH06G)fB5L(CVOB|2LC^v>1bUq<Z#lzwqz#)Y=u6qmB6KeY?bVcf9pT6xz*#E}
zezwQ2mlmE5-68}lB^AIk0B(K*t7M8q$hMfd?eVUd?%akp(Q-b~mcp)zm=}QYH4S-|
zljO!#4AOsIy?2}#l&%$HNb|g*h0^eyRo3N4Z9<fFSliag2qWgJT%p$BYQ+N^bzRn$
zqKE?Qc`d%S3lCt8qcm>9X^EFxa@x?OneXe)NVbLMPof@P*L)w11^X0nW7(`e+Nn2O
z&5Z6w78>lY=0#@Ug(+WddPZ`;i@q>tm6Yo;^Q@{b7*4|7gtBTb`a<WY@}uHYyQfU9
zkAkPxYaOcBqf6(a9slCrOw3IrVNBR25&~*^zs|p2iTdR=I!x$XJIx)cY%dECDhK`A
zc=-g{<kiw5x}MtKuBYXd(c5pyaC&W5vJIN~)0`LdeDU_s_Ifo5z6KW=lN#6p+0N!;
zC9WhTVQ`A}h{`8=;tm8VapM3>mkn&-Y7WV5>Q{H0{{ce8<e5ZxVF3M0Zna=n>N74Z
zQ2$^mJXSa4=jrdf>p~Wy2#WZ4nL-7B_exmdySyj(3@-k7sY>s*7=Pcl)Wm=L1i5$}
ze~m(3RibSI&d95?UH`L9=5YDD+ZjoAZNj1q#f7`>h|`B7L%*b<_(Nm9>+!qx;vro1
zc|pr*bpK@MA1%kFUmlv%=G@X-L~hr=`2WFEp7#uav06!?Qy`b9GFm;$KObs0eWCfe
z6RGvectqfR?%3zuP#-?yGoEsW3c8Vg{-<1%yj4R~_|&4HKVjn7qmQ07&%U-)y83e=
z{OCM|AsL^`ElnMku(iLSs4Hu0*}jd6((5;<>n9Bx$sWC5l#}I==N}i9-~Om6J6MQ_
z;IWHoTk>-?>}t57XJI|<QY5je9?Y!JbN-E22*kkF^q{&i!$x!fmE6Jc84XuIGtRi1
z-5i0u;!A%D%nz$X_ua=tz>juoT`T~MwMqoLjmN{eZ4&2pMBd>#W7u`c8}s%0+5yT(
zJPe_Kk%W_Pi-xviFAy7<ZHwt}g(f~=WT?_rR7VMVxHx}H5oc9~abUb)*a9AuIsy+u
z5QhG*|9%?LT%Et}+}<bnc%ANSG|YOK77vL*mMBODHlitE4Lw?;8ahwkMJco;HwN!w
z?Yto>)~s|BItWX4D#hm&DZd`M%PH6-SIx7ptVX;IaWqoc99+2Rl)84gHywd&W_Ua_
zI=c3+s4NXl3w^pj%I>Uk?M^z`jZQO$FFt-K&|vmZm@yJoGP~D=_qRX7pe6qAygZAc
zk9*(ClgeT!()_qk_$j`l{4}f#axzlL+OOWN&+s=)SKaYG9hAoYvn{dbckK(^yS|0I
zb+%O=`O*EAaUH|D7<7iP+?xpR%W6A_@q4%$6I?B@KX%GXSWj@k;P;ZozOZ??kOs=l
zy#MBe^{qv<>}FPmUw_J^QxI#h-tn)Nz5Da>?}t-VJcE-i<fV-Xe#99%uvdgXzv-Ir
zvbc}QL!yhOv?qPfesZNhu*!NiDdf)?Uo2+_qDoSo6v5<2`RdNE#$3l^rL<Xgg>@;k
ze&#0ZjhI^2dv5oh+<v!V*z!=m9$e2kNlLNttA5D1uf50Jg@0x*E_)4cx%_Go`kp5-
ziGu|ql-YW_g?W1Sv-q&-IN4>gZe&%<Z$W4L<Kc=`8Sl<isXu}asIF}81u0EI7D@Bj
z>*CUrus{0u`&CNAH$Gk$seYYA2i5tqefVt_1pgfDFXX=u(xqCjpMsx*cdn65YzD9u
zx0TjJ!}eadL9F0oRqN@N+k2?#h?$P*g>__vKk0o``PjQ{w=2QxlY#3=^Pr*3(8igj
zsB0Km#t#vhu|u;ImJ2sM{Q~&8QcdQtIp}6c9^2MLucqZ?f%l(=$a~-3jnb2FFs}9#
zgAUJvQr48y&5X9lu2Y~o=(dr~mH(#x%H$`VX6D~ap&S_NOqFY(`s0G*H}yq=byhC8
zbl@1K@BP>&ypLv=f2X6Em3<0P#|;YpyW1$fNnF_~0HGM$4A?o3+_=1q%7~k@9CKn7
zhTlZ_c-Z5QK_}l<x#{#qL0{D7Z`YX-T`k9}n;H)s=s)&ZNMg!0df9SM<y_Rw+!t3X
zZ*f)l7PQG^X>uCb_rVJRy)LDZ?ftW>(badWh7dSWO_^#y)&g0}LOUYrXcYY}{Li6?
zUA$`P?mP!fWUi{|lcF#^<S2jRTx8P8^t@rrWcVcU<mQS`!^262+$vA;$nm?2PuY}-
zHp|W7{+vlxibemqr(%|>?ncjv>Gyz$+pTKTa}l9GPTEsH;Jy{3dbfIx5`0ecyXRrx
z)vfbd#rmWLGs^C#E;_!_#p6k<xk2rzos_e91+iva)Vke(Dw*iTAM})sY#nT1Mz&TW
zvXjY=Z1$FTuj@A+?a2MS-!pi=xf&VPcwFf<+Ul&$lW|TWgzGGLvfE^YY<L!6=?aUT
zjNDQ}^q2vi4MIG2vv=I;<%Ku8zd(I>{m#YBtaEKQ0%+Vtu*<f}5sT}~sqP$>TzbFZ
zhWjelCz-YC(82xg5{D`>VYxhu`|APBpjaGOWCdL6+>X~jIf1ovJ@;A;;d)n%(372Q
zkspUsi=YKd??JCAoZ%jM%F!xwyE7CA+0GP9QW@~J7Nv%y)U3y3Y=_GDZzqew=VGV{
zTjRu7)pi=ys7*e}UmH`L0nXn)#J(kz(HbulKnmuh;cJ4&9~w9O=khG8Q>BrIIPrc#
z9$Y-K;Yc$zH(rf<8&+$77@|{QXMT+*YdG?ABv(g|ptEas!qf{+5~TLsv)5aTlP+r)
zAzT3>I4!o|n?9##&MVH9@ZhSGiA!VGOS2(o#EX!m1wG|dMHw&R_&?;WdySeO3ZCC>
zb<_JBeLNjiR)n2X8nJ$MhTNCb5s|W(-Rt%>yrbkAOAW9o+v@%f-C{2xVT+V<ft{gr
z2jxTM|NdPBOIKtVAg`Rw{~+<X$Qh%H(7FHkc`g0k<K5%UQ<3}ue0cS@Ko>}g<!2<%
zzr6$E*jScu^qV~*`|(}W3%%zVbrv7fd^gBwubg~Jc}$9HOBsm{n&}s+xdIT`>&HEQ
zry5#>nLnd<@3*FlQS8p6ea>k=`glK+vdH<k<Y5&g0aM;UaS_>VOl}t;)#YEMPvQA&
zXwSV-yC`!=%Bk9}Os++BciIa%@rGBPTw5KQ)#*5M1`ETFtank>Una5gOeVDX-qNo2
z5_p%ojSoEf9bFy@n8<h9ZuC00&uzK=-nCLo?7gkQc;G#1nlQOcJ<Po66owi+rnu~r
zwroY-F!o%&pNenKlD^Tg+bezyJzeD8xmkO)n|)bYa#-W~g`T%ODMSm#nGts(uX=wM
zdoWzGx}lLWU6O?K_~YZ0i>~0?LDBRwTo-NOORkg!qr92B!la&WV6)JsS9Tvh7P&-b
z!!jDLW|@CZocT-B7mG<lQECvG+sU__hk2%oO!dpBVp_z;2zDxYe%d@PDRB7cS1VaH
zKEw`X85B*Oa%8@WJ3xh`hA9%2!oES9b`-Wpy*<{jobKkYuAo&r`_b}vQapdXo<){=
z5rITFIx!e**lxGZ6(Y-q*YZ;@Q!khAK>v`2y-v_S1vS(s4$I#DTwkB_DR$4+-G=&z
zgC`*M2_;&Sq;r5CPql_LZbZpx-gV3FAYqB}+s(A8Ud%4F8cMcbVz2Hy5*|4-X5VqK
ztt@N#Cuc|3vA2C1izW3`h4?{#@+7)&cFht&k|En7lf^&He=zEVKQu{Et(B1^-k8)_
zC+^bO`QM!YSv3{%!9rM8sebruIS6+7lKIy4Ffp+l%6R)Y$4EAvd_K;0ssoxolWKd|
z*%=)m%A4tG9bs~8a_&Iy+6o?so(zl3YQ54pn^N_}zr5ZZ9FifWK+eWBUN~7$Vz^f>
zCP@<w%}9y*p~Gukbp2DkQcf-vqICZmS=!k-oYnKVuvexBA~^%><B^)OSlS6F9!OT^
zY*O^22X?}&S)Rz$n{K!qJ$f!^Lwx71!e1)qK5sWPVU2q%;l5qJ#b)5mB{9I3c1<ik
zXkmBf*>-~y`h#}2HISeqkT2NxXWc)PM!}YNHh(2p6y_{aKyB+fd~fY2M<D&%<!t8x
zY9RA-ao*MU#ZEfEPgaBNQep*l>I(N!bb1fO-fEp;^*e~;oZ2hi^Qf5FGgdcKgpKgr
zF!!EJn$tm_jXuj})%jy}&^G)k`X*fD{@k>;(SpRu6mi$j^LajrVbhj#P<P3u@HS|x
zRj+D(En}j_+B0`)lS?G!S&lI0nSoVm&`%DM<X8dE?7TL;v2aQsp}p#BhI{vWSh)}2
zDF6E-=`MtQFX;|`S&zerM|gNB<Enk`HGakDiPFlm6SjO>-k^oEHwU`$QENNhYD=Rm
z7fw6S{^Ck4H8yf+cjL3+XvGhYFKvTB&$B(AHTo`kJS02w^iFBD^-zsf6!!~X@%OVv
zKTxPokvZ7hi;GkjCTq6cIy6y<Y`cW`W?|Fg<(`k<yVP7GYksej(u7!7Etn3!sF);c
zFQWw?RQtYg)*R~%3^xK7xO(;6lx18Pu~F=oyNs{9`;$T!N3DeKGYfO=#%0P)xP9-T
zPo2K3gkQ3&bguN!?$RS*_xsi&+TC-qPX+ReCXt`6t&f{GbPfb}Z=1=aojiOxUOAN(
zTI}Eso~8V{c|`~PT7Lf4!YAMVYeeqFIjRc0^)bb|DziVQ`uP@ac1<Q(&`8rRZr7Oy
zOVb&Id+siIsO9UDpHZoithD=9twuRden_GUoj)ly*tPY6ea7ydR{@cQen8(8!TnK(
z7F_diBf*7jDadE%+_Vaj)_^88^(Cm5`<$VA-5h>E%xE9npFI8yr$T$s^Bdx7kYeUQ
zkO|D=$B;_m8q`SLrG`q`ikP`KQFT4}vtl&#+eA^a-=XV~q~DV3@O`~5hpRPLuJajK
z@{p%sd&7X`A1NI)LHFO*1<j{Y?qqd%5XZOQlL{WfDxLwK$$dwmI@8p<=;YfMjZhq(
zfO%GUWEEi#%ft@IUTU|ITH2DWxces;A2f(rYPGTMxH_>o5o%=(X7dEN^fTe*^7uEy
zRpzXnAmqC%D__q@Ncday?G)kHT}B(>KW5l49;<hAAd0#$(ic|*MXDUbJMZO%wXf>X
zUvHC)5BYlJOS)H$=59<`-AHO&j)mt6kclwaDP))hklZjAH%W}zHc4mqZO5T8;u#(O
zYU>+?n9i)T7)2)k68aVBvV2i)VTU~sab@TaZM<#<XolSmyGP~DDR>rCEAuek)j-|r
zGb$k)+9EqIUhXz`+BnbAkt}u&R@um`9zfT0o{<=~hbv;TANki99UL!~u7yY+;(%=e
z%*gust(^zL5gGdxt=yYgJ(qLae8)YKxx$P=1=?cKq5M2`aFmBx5;-zGOAagh&q&|F
zdrGzoun6jrx2H|-Qe$jwH$@R9!@nmj+C(99)fHKsG&X9uI0>46+TB>&QOVU+tqupm
z9WxQfNh+Bfy*x6Fg}*}*q^6@w5=)k>XQQCa-71UAp&~2&ZYCT|9W8TU&dPsi8~g53
z>v1N0G5s$&@{OHvqn|+ILA$`7`u2mO<xFoxNOd$iwl)2^jSYg5XW8Fp%5XQ+{5i%+
zyZ?*L!%|+yJ{T7Lg#=BG0sm|yOlT|v>LK?O{jAkdIACtGN{?cA)|poVbSt`OQO;*Z
z9=~MF92wLOR0{$P7_;YoZurBm0!S_6kQKCN#EQae<nUCVnK9*az{4|nKQ!VN^QD48
z1+HN?=uqn;id}K}=6Tj<-z~=$(Fm0H{JK*_WY#12f;WgMOOoYD&Sz)(ljku}SR#`-
z7B#JcHeJ9G9_+5yXI^H5Z|{eljZ^D<HT_1&W^zSFD+}_g3MOk$>k!#*TBS6LZAuQC
zFWz5nJDh+&&SKx=Ls%2yOU`mDw-N@(-i*cGyllG#5_9^wk2dhb(wpKGY~sjd20M9g
zVSAxGk*gvz{pyW6U+;<S*_to$e_Uf`65N(2v!xUHH>4G(gzcMCB&pNyX@MHFm1nU(
zCe>e~@X~QS!RCgRBSXA=wA0O0J8yM8%R&0yVdcwhT1xEsX5w3eOp;>NyX_p+n(gXo
z&WhYA;)T-Rg+B7rKpMg~^V;>4DR?vHcY0$73A9jPJmY>&0rg|=x;f7G<=)xto^*J*
zSkwP6W{m}z-2b2Od+)=d)>vcE{-)uUIa0>XcHkzMi${w{G1N#?v51(h23VC~i=%Ii
z49Q2MOZ^)t-hRw%xjbY0OTmld6KTyCIdc|AV$yh9c+R&QWxfV>Y469azpDd^B8G1g
zFv$d9G7mc>XRewepQS3chVO0#IAxO3aYhbx8oxKTmyQ*ioG*P4z{?eYtDOpa)JW?b
zeeSPz8jieNN;}woAJ&luh})z>efuY#h1C^5c*-K$k>R;}ZkNP-EO;JOVEfAgf#PLB
z_81ZX6>pGr3Q#e@XMB>Lx!Vci$n?AoKjx<WQGFJi4XIAwz4W0tAo#;!c%k(u#jmCM
zXFE<iZBDA`LCYT-`1^C)AYv}wf^dEKQn(cGyFUXq@Ak<-N$_NX<?2|Z?+9ckQs&?@
zLLjXs)Vgl$al5bN(yYzeH+Irw`6`!H#yMf~ThbyY%wu#@$fR!iQNu-_-HJ{pbTcFk
z(>vo*7m?kQX2QewMpWB$`@<>6Kf>({&`9D=eaz%!xd~ql9|eH%tH(DEe~3qRK;qb4
zbSnaJmb<tHvq@gRkr)TyPSxY&lYDDZfci;rcm}-^3n>X&$29s@^l#1v2ebIzlEUGg
zS*o<Qo7_x5ZG&e_7h8+FcfO*Y=kHsS^V>9EKgtI0Kg!#)b2EPXqP;*g2$ZC)2p#@1
zbM-pvmzp$>4BeyacpwLastU;|dBh5KDVtQ|Zfg~il2DqQp;2Xhd!GF`U0Y{u{6cz1
zIk~C~+=n*8J5Xbt?=Uv>MaG0Fev=baCkR)rdS;#Pr(}om@Qp%`-K|#?^ocWu02h<-
za;D9^#e*RR1(Op{WGgf)SyJZw&aJ67c;b2LzD)(fb1W(B-xCel$j+5^ej(d1czgm&
z&ZKxV0z3EpvPgmk)__)b$RieHZL5`Bj86hU!SC;ky&6yTU_p4&z!^@UeB(Z_2eDfR
z?ltNw!*h3<6m@F{;<fXkh~5Rpm@g;(Uw!AEx&E7ARb>IKxC!B#qi(NM#;<=m$y$Jr
z@|tDv(Sm610@5KSi<votnLK>OqC-D<v{fCn+juc4C+%RBbEg-crUUdD<aH{M%?R4N
zAei%3jqmjY#C=;$o&Os{HL5v^>$js|;a_*#USq9&+iFVu>#9BLDLNFx38(-W6<cC!
z2teig;0zptsS$#vt%{4RR;^Q!i$4h9q8#kiaw?g(H2Q{^#7u86a&qEw2wtO5V<Y5r
zNj8BN-ck_6<)F!g{Yc;>Abt~uFly+(#sky)<H&sgZFX~mWiFgXh|I)0nLaojv<d(`
za#R^&hh#7<vOvY8`3Dk@3VHD`7Uv2cpQewYrW}IF1k4<!d+nw7-)?ZhueIX4_qa^v
z&4>+~ShDg+e0csWJ6$)-ob)gC^9Sj-VLJ^KAW^h&M?-T2mbS2jicL{+5Hpj_;^v0)
zGel7)D@5P6$i)!sR&=%f)xu1HO&0Eq<)U1YiDMz<gCUA;!#Y8i1MNb$YUz7k`Uh97
zL8L*5$zx76X?TG#j)7pib^5N)8l?O}M;%4u15m_>DpCWq(?y>&XSED@NdLPoQBE#X
zf!8Hn7baHfs9eQX6mKDyp{vQNHkA9_waI%mB-!_aIZ&BQ#(hcgX`R{dH8_!g9zK)T
zaycI1SGdelM1?Y%Q_zd8u2&LaV#!6sV<=mKLyu==;~%`VZf~%I;m1D3t)-w|GEX!y
zQ##srDSYE44u7T!+QhLAxH?Wc5ZBQsG{xO#=O|lxt#TsEH<P2UnY(^Bqx!JX1@(N8
zF@XhfQPzVX%u-Y^??pUAZ-qRLHV*wH1!s?xv_qx-y5qSi!LR-4u%2j<B-~l-@s*h+
z8>-tgOLk25jatQ}5r6wX7NbA$(Nx<RoCK)faZc7@(a|#o(b0ENmyq7D$Q*Hsy<tbA
zm>KtsdB|VOJWRnwdXHvuM2FzzK9@##bH_qemUru><@U??JSYtHX?<hU#y^%qEHETO
z-n?8GWR%a$!!u&o?REVX5keKcLQf}(KnI+QoX-D-2s4rdJ<SY_9Z$`cMNek9;~G}`
zjAl}-P^2aod^HzePSce<6_lK(ms-yXE;Spd&*0GmU4=@*2?-7HQ0r!{+}BxsC7Zq%
z=IU22=i|7!xa<8c{;e7&#<%p`-%$dOji7~t8sm2&d3CCI%Fw~ARxg8GnKgLEh#1Fo
z_47zCjwM4zxiK2MRtcWSTB&?HRte2rdbzf7>p-+3(&F=>XCKn7!ss&J(u)u6Y!95$
zKKNvbW1UEo(=66rdn8X>{rX0J*RYd(GZu?x{CW5dZYFj)VTtBiiqz!cFC#*dcV)!g
z>FG-chaQnMU35eQ?G22iG(CG0!Tua2*{dlJyuK=}ka~r1u|V~{#_=hy&?>ed(ys1<
z7WZ^BV*DAmqTyU9Z&7FJUvSU^Eb8L*wLjx5=4|-i$Ms-P@243|zu;t<J1GPqCsMs2
zSxFgMKUFqytW1A&-Q<Sgext2s>XgnnXlaN0!s!QmAHhOD1bjnku)O(*?t@CaK|~N2
z$G4(>nQnDIG#>)LIPGULOq?^9yY+?k$`sDuw|1`vsKB#FlTr|L9|ot7AH(<>%*w%T
z_*0=D!^9q|5>H~p;neH~tD)EeNbpS=tuz#?ad!rZuy&*-VJngGMTN|%AWJGr{i8P>
zo}B7)4DtwPJcnyRF?m6xf<`H+XLaV&Y7|m1cLs5Bk$i@iuf4e$BIy(3yjnM*AqB$X
z;+I-{6Q?zcJc1)bKGGQM!pgxHpMOP2nag$0%X~qS`Q7^DKqheF<2K{i3Xc#xM5Ty^
z8T4@XXv=9?kY{N30ba)&Y5x{*;t_q<UBfXs^b0Mkt$<$;7-GD?N8gXZ-ms`8gKzcs
z?AkK_to)%2OgfT;7r3iL#aCuZ!!m(*lzePR_YN;C=6P}@aZyl=78+4-!gK_gQOsRW
zNy<<7&5SUW4H|o<L#z+@)k+}XI}7*Ifu8!W{JsTAZnZtQ2e-bcj$0B+kVWS!R4Gu&
zel9$=l5*W!6Km#3$u3nuSzRM2WEdfcr-&OqVx%$NFCiV-=@~LntiO1nBhcg0J`o9I
zH)5<$%jXP#PEYvRH_>r_><><Ebqf*_vS88VyFOly8^*&@?Jj4z7uiosg#CmJE8e^{
zOukJ+vC&^TnSKv9LSvU45Gj<@Ic=1&5t!H*0n1NVeIEhb2^4)10DNAb`Dw-uv8!El
zKxh~U+G2lCq2Z(8#^QSsqt<8`SBPVwRFW{k_Q~IyB?vPiy==LNv{ynWOwIQizPdbD
zM0(w|<QpU%qKQJ>%tOq~b0X=^R^k=Vs;trvp)u{eGkFhP#mgW<`NRI@ASFV~(VYhi
z{&V(s4QuFxI93i<4NGQ4h@7DhhzG<(l^v@rA`vD#fcQ6F;^Sp=QZcC4myb>|%TH$q
z+m`ARcRxB!KRUZRDH8kH&!vg8%R-K_P1;n6Km&^NS?01pO7eao_^Klysg(v+;wY4j
zfP%f{+NQz=zHDL1^az;ZC`d(<Dm956gbtV^X=5!a0w*maC`<e<3{+7b6P10zNx%&Q
z%r9o$z-WYmFOFM#_4&&ju7z4V@MSRBazBCr5=X~TuFb?avM)NjSyWb>;!n#)VoXS}
zAQR5sU3-Ca1f#h7AM4vaX6OU5I33Ld)u`AvHTQwj(HHxEr`Mh@FI{nc><l^fBE;O@
zJ<M44P-nruy3q8X*#r-?<%vTQZ9XmRLtp+9H`>?CuI!-iO3v;iM?FO1Z+}d`mdo;#
zW-)~v9fuAjHQJ!Q?)niQfP+1Xw=NVg4bQaLN;^a+uJ^8}*%6G8-x72O8ztfewhcE(
zL%uutJr)ZF4fL(554sa~+dL`~rFLreW{5|S=Z78q1p}F77r90}EE{TXCp3%cM!7ue
zWJ&e*|N6txpAuC<UsKnY`4B4k+4eOGKUKgfCZo4ctZks?LLIEMoKqFEHpubg6~360
z4#$khOkK&Rg4c70Rp&c)aMjT0ZPnzkvvKl+<)6cspXVSn8wI6a#`ckuH?Hyx%=U2o
zOzrJ+`z4}7t6`$zVA2<5kvcGZt1RL24r@(o@L+L^SL<xalw{}EE<uNq7lsUVd!@Ia
zBVqr`$a44!II+Q#nvbT7il(QJ{66HKY5aJS9<t5lTJb<IIko+8ZQP&QTVJ5<2;({7
z78cpz0c$8`g0vmYlEB3U<e>|~kz8MgXr90AYM{Y@DFVsG{bR45>{D{_MM|ITuYkxQ
zfpcW01tH>CAEP80*{iV%bwcacQh1nmZC0s(Z~yoRTV%f55)s3dH`)+Bk!9sfi7jC7
z3rA+cKfyAHBRCCXKVbKJIKD-%eFKr$AKkx}KXvi9&AL)&2`ii|5&LM(Mpc=OPdl5`
z_gN4$8BA(qO6dFJQt#Yf*=DLBQ9R=K(-70{wYS8+DWu7lt8g^m-(g^v1CIvf_42dz
z+F+#eN^fMl(*c-$f#1V^s*3XcA+t7p*V0T6zf`+>HK|{2``U821Y2*q<*7Nya|``>
z4N3dm`Bm!_22IFCM~2XTut2o^{s<UJdh)*n@TnRy!5`#P%sH0m?Pe+BFJ0gER*MD&
z#_nrIBk#E0;5quxPsVh2VnR_htw{=km|#kUv7ZDoqD|KDIql_vpHX0DGbY{<8CD$7
zGMrP>b1x604ek!+(h`VAj4+^;s+;}x0+Y6;9~bIaIV2p^<7g}{nc`Be8Zrkb+sIAz
zbrD0)auM}6-|*4U;nDQ`5)i)hF6=7m$vDqWh3YUgQ7Qt)-27@|<U{EmjXf_2vr-`!
zuc>`|QFYMSE`WCUf*_bQlBVVHLry&YA{o0I0rhhNG|=+`*Y_QA6RGpG2B!psh83ce
z#JqwHx6<dGeF7h}4aKp1@7%n1u!Wj2tejtoxOdsnY%~M<<i>$fWOWYZAR3fR{#HRe
z`&|6o#+YxN^LpJzIFcPDKPDxM2mBs0C#?RC@5Hl_5B1Hp{Lc2kmtJi-ez_|?bZh4l
zyHL!7zM^`mvAjN%ks?O|!k8aihOWt#U~qG4s~jQ`6HBHaNUII)H3br^h?3hFK45(a
z<MTTZu~5(ZcJMi=tN<zVavquTG$+^oXQ&*s-1sM+QP5rkjT{3D<+aXkpXSTq-Zx#z
z79U*Gp3=FV-47yhEC85J=0Y2KUeD_)?$ZVjfUlUlKCJK%EK=M}cG+<ZEw<GgD_d~W
z7>LegT{688k61<zVjCn%MD1x2HlqJ)ui=lzE_;7Lp;8>VH`gLp>fsowb8D;?hX&X+
zMt0K-{=$839<sx3kt6PrLKu{#2>Y#jUUd-x+QetzCjjz8(g$~=4;EWc(|pZo{WBNb
zH+DWwVYW^g6*fPC#FUXqM1~<);<y;6w6cLUv3wvn7}WY!@CP2stc;X9>z<pga6d~8
z?}y0j+LiIV_AF=TXY%hj8EalX_?9W9^>}O987WNiu4@M&mA`f;x&A1iX&X~HOtAR%
zV31MHl?x|E?M4B;z^4qOfNMqf<5$b|c~*#k*;nsrlLp}sJeXD%!B_E!9=t_AaTCrC
z-H#jpYQY7~(zs%Xj}G-Z{nwyjIr%U#mLtSMNUfW516$C%C7;a?D|;b!8k9=~%LPr)
zbd(L9(%}pDq2Ep$i;PsUrmL%jtSXO&%H3rqyE|0hd&HDzP^!&LU8vrMc|`<c9e0wU
z(<E{GR;jRT3G0L+yO-|HoviK3Hm{JwjyA`x@T0T^xV-8eOE>+W`(o+P<g@x>b*hU5
z?tHoaSjpN&dR(b|nQx@76;ePoLHf5a`N*~<Q#lwBZR$$}_Qs0{ylTf2>64kQmnzv7
z*c86)TOX+DUW(7Nym~JwQuOdQ3PCBVONAPMuA*0%LG2B~`Z`=370^tVu+^;};-3U>
zPRl)M#u~IG=pvhXG^<p?#QH?t>HRUqr8p_(Iw$-k%zP(MsRpB&5_IC(h3L-D3w?+*
z-*jDd5yS@8kkpVYdmfkI8JdGAr-r4y@XP9_-CF;z-+gJFXp4;?=J72zqB4jpuC?3S
zVWt*5h{GSt`%5V4Le<fLnw5=b$o6@mY;9KuO<d+J*7r_Aa>|(p1_0jb7q@=@%5p1H
zm<mEi!VMFVxe}`PnbAoZvG_%)rDPrJ>Er=`+M~4FT;Emaz7k6cxC*R1uR5R}3uPJD
zr<t1?GhzE6bDVxeh)%Qd_fll=_mnOZ>!a-Q<~KR0*#5-f1yZHKao={;hx_O{OSYDH
zLA4EZ4&tf0V?|`9qQ>`V_ML+4>{B+^Kjk}jR4T3{_~}S*)U>8htdb}nj=$6>iz_TP
z+fFF4dF4Q_Y;E4c*IayF1q|nD_&E33y%J3}1>C@|+^cmyPq1-^85XZnb{;`p=1+Jd
zAVWbCrM$9I>N$5xCEsJMxuSl*m{g=L%T!-huALhl)dmfS7Wvhg_VQBfP7FTiZkAfu
zx;%WjSg4c>;Vn{Tgeod5n^7f6<u)~S)F_9sqlFxEs7cXANJeazEVq7Vu+e#|vCY@G
z?%!<GRJQ!<Rm5yu?AZ8^6=fmw<i|{bj$_h#LHuT?M;>@1MW<p<iS}CMz)xt@spuZQ
zy(J?pvCew}ZqQpA6!~Kb-0%t7Y>rd;eI`!<;2V)&c!%>c4*-Dnotf4y^LN@y3noS5
zx-a6IqWNu)6YkLA#Nd+3iBSHc;gQy$$1e%Sfb@b~2cFy|;?8Y)^!huS%47@Qy3{$;
z^Q!m0sS_DDgS&bn+JH(#@m9lz!R+xy6K@3zeH{?bM}aQ*w!fhWQ8)}b&H~Wz-;J%V
z41RAlm}Ou^ET0*-e$#->4koemS>n=uKnify%Io$Gu>}5#Q?G^R0iTdjw0@r^Mky>9
zY80r2HQFDvoR!UgD3=Q|d+qD+NB7h~6_aKQ33#+=n;B!xO*yKwE5=VWDBP4ZxEF^q
z6^-sB_$r4-(QR9QK*ZL;Y)LGBw$Y4*yyKb@3oj45BDFb+wLM>|F(Ot^GkEDoBUU#>
z;&m-Z^|ehkx8;^!JNAeZsTkDE>ook>iJ<S2eAMoCrabdxK6i6~V$EF_>Vmx{_WEMa
zXRV^S(nV*e2`Dsa=08!<ox;<Fe>{kx?gNdL-XYtfo4YU#bmVk_ZfbFaB>0%BU{>&{
z3ADf09s%In(n1atZ3>cnejPFZ2JpDnJ$e6+Z*U)~8uB-3<7@Be?ly+vn47_jNQfOW
z+qN?0;#-PNX}m-jG$<bAR#JxK_X+Ojuav|PHY3iq{}y^;;~2rUs43daIE-7lD|fqz
zYkwM_Z$=Ip2i>q--rwwdQi&t@!szc%gnlsoT}EOkcF_aS#Nj7*VQ2T9R87Pi-p9M+
zf<KrZOP90+4C`k%jv<TOj~w)6S=Qj1k5uB1OANI=y~|M4Gdmx7EeS1N`K43&SUkf|
zDAfgMv%;4d^;@!j$?~CU52aJc-a=z14%H?ZI)o?D@aSE5-%>2S8;*~*Hhq+!GBjY5
zg=P-U=9fZ%D!ug(6#LfK3vzYXQrs5qwl={!?A-QjYRL{KmL}@aQ41c&0%4_?Sm|pG
z)am+7R|5Cr-=gml{U!D|yySn;TYKfz&PTiOa0DH_>1^*QgDIWfMt`+T8Q0+{AQA~<
zoe<y|Q^p_S5pUVPJel((Ab$Wm7H<|6rkvcu+O|E#!#wQ{?CV?K2a|HNv)KSGMo^(U
zdsZp_pU$b9XY7z`JT*KBxRk1}$mu-{2S;r3tEquEy9anFTDNFZXg*pIL6IWi+n-5C
zkPDqP*b(TufT5ao3F_zLz@po4{c(KK8bkq>xdaSNJ5*AEb>3@W{Msp~x6jlPFF#KC
zr3@)=PjJuBYo%z?^xM-ei|s3C1x?14;Z_IR*YAD%N>$4S@Iu+*$4Sk5p<DKXHys=&
zhJ*YxL<TWD1V|yc6Q8dDZtDzpASgFS|0j*gLT(53)yEK)bHUK4G0|?Ei~#$BN`>bB
zKcH6PAtp<~Cf&H`Kyu#sb|3mKO{w(tE`k*&C5`Gb>H$*p4%O>=FyT;CN7FOBYVhu)
zUZ!_xj>Ff_JvxU23g@i_0zx?AU%ue^2WwV7E9T2yt<>>wD!KXWf9G(Pcb(uXlJ$w>
z^)ic57{rz&*;R^bj}BfolQS56avlHYl?Z)x6HbzTF?SO*yrOYJfGcq>+}-U%tljA?
zQ`4fusng$n@;a;7>lmv!Ji|<lYN}YC(b<^+8;B4b0@Ur##OsWK3|7Ykw=zIrx4p8O
z`P}ovjO<ZSX{ORUW77coSch>RJKcPE8iGfK6+Em9h#@^~^s2L-$(gG(AITZ5GLb7(
zduxzy78brYU<*hK+%%@~*iFJ%RPw@}PGu@mlf`S%C%Kva0sR3zI@SqV!`nyR4?cc9
z0w$%)x#dv1$O+YP!d+}_6|cH^ZM<$rcX~?0=`Cm<_nD16lT1qd*E*`4NOp(m#IcUO
zcht$RN5>oS#@!LryzZLkJuGD~Ul7mK;^Bs5r#_<s`Nola@JGRS!;I(b8Rv_fIY{nV
z4k9ajVUZ8GUHLzagvSvNHy-eAtd0)8xCMgnw@!Y_o!BmuEZn?5P?DQQET@?hIpz;;
zUAC?*8`ubD){l3$$7j%^=Q1c>?@X-GTAiuygogS~mrvEUyH1%9x@fUlhcfwaImNzH
zUOe?H^v#d`Yv;WaNs^~h>{%9u+r#JGkxNO4<U`NgBfWfG<$Y(}E)e`s9{cHFj*^K^
zeUAG?CzpV5F281ba}u``sg~)GOKlI<Z%kC%ktB`vpeRf(q*^*xB~BfctSUvHFQ|EG
zBJ`4jJ=$Z{vqiVMx!6qt-34vihOUH4r>x(~)-cBbCsU}pDH>1y4G-hpeM`}6J?al>
zd@olsB-Q)2JvAj1?I5ih3@wD^aIpw=Y8rxPJ+v_>cDhR^5o4%#SBx+2!`CRCGHyDd
zar2U~yi`oN_+P7fJ2)cSX_;HiOICuPr)Aae$Bs!y6<6>|(i_zUz06p4_oWLWeMcOr
zT`bk{g=YT6+b+@%hL2+yx%js)XjLU<TYfjJqSB+>;a_qC+>@{Xx&QhWeBzNw45d2q
z2$3+8JvlhU-#Xt4i7gkmdiC#HL|P~Q{1(uA6Brrsa|~CKLnPCP>{SDIo8&C2C>v*;
zC@R5sBoj`@!tRrNuV7f*u=voZBDi?3n&4JV+~yyIuVu|I_MAF={3^K1oi_gvP*ELL
zCp$Pey#M^=@=F9P98nMuW6<0l$qj`y16xLaB9mn?mQ&_IWxhpS^37t}F}U)HdoA12
z?vJu=IQ|6#eFWS!Z(lOcmv*^+=_|x9Nn|1~CGBMGuMhI+P4LK6T&o<G!P>Mx+<8m8
zWY8i0WVRv)6435-chSvwWeUrZ_M*OKHCc-m{~__XKUAseX{d=ap(OLp*?vvVs}yO=
zZbj(f_Uxr*8&ly$usLG1mwOJi^~<0HQGy3>^!}rISu>f*Gs*{lhJ@#fJZPcmEPkwQ
zi?Z_hfv7l5<1=cH_=F;~DFOSCD^*tv_J>c7OI($2x|Zbn(a;0fD3c}0%3h$;^i;D-
zY0+S?XBr#2!2!zvZ+q%i%eW=H*7Xw3Kmxc!1V+4w;Xvt+-QT7JtG$JM0FlHeO+83~
zinleeF&#J!b(Dr><4*?jL=iouX6D7~?kf*k^q4Oa=txGOoR=^6B{}OU2Z62u4;?{c
z8WV#()_mrTJ40~t64_UAdHXwew*fRQ15Oo<_lt?Ip1UT*(RQ6aao$Uc)TC*Eo{@!i
z<9nK@7O%3Yi;v&my#>yzZ*-+b(gikL#;*@1F{Y>jr$9c5;C%&4c*VXU%LH0j>gQQV
z0hJ`e91(E<;#MC0^o{|zAsgV&fZlXX=Qq--#++_G#4j+l^66n#{<<B$gwyjBOS;|@
z8RE4aEgewR>aI{o9wzBhy?dkei@6agsg{b{ddgv78LU}x)hSHE&wbL4pn}F1;C_mh
z`<w@wRqs=5^wCr~JZRxYRLox>-%Omh3K8yIj+@2js=`osbuOb-O?|nKe8&CYFGo*Y
zdy<GZ`pf}Ssi-^rk^eX}59S=`irvzzwfv#e!ySV?EIaD2!EN2120O->?DKBEwr~AW
z!F7U!0EY~IO`P1kpY7E$FGr}?6<vHWP(m_x(gL4W`YMez@7_*S=C4)$+(F7|F;(+G
z%_a@Z(vq$vF8{GbPO3?wyC?J(G;e}$M7nJLcjM@f$5Kjf<lxmyLj(=dNIa618GZWS
zQ3Md=e>^~hC>u8jIN%64L@<SpDw$QEpU>+;Cw-R=ba84tS)#puwr%Aw;Ir_ZL4l0V
z?I3;XP-InS7SlT~((V6f0R$q6{3D1&5Zb3PA2UD0ip@cLGpz530CV`D|7VW+`%dtC
zv=>IHE^fLF7`WYrEpYEvFlZ-Xnv5M!yoY<ARm9uJMuA{HJUDca+gD+L%af>Cn9vX<
zN;w>rRxy{4hIonmm}TvuAUp+kcV7XsEvE4BJgGV1N;$us`Tg<&xXF71ICVf-mj&al
z={n{P;Fq?wit^{!Ap+5wZ)t09YA*PLC9_GeA4OR++m>8#)=L+ErhxH=PxG|H3i078
zt*MQl(Ah;A^+UR=$RhW0NM3Mf{SIi?;w80M%J%>x^f3l94g%2Y96_H#*aPleJm>mD
zeCVc8&P1{NGSS^p;S>A&58i!6`()_6*B=)K3%eT=rw#J}R;{?ZCVIfV)}9!Ng4Up&
z<y}Y53(6-`93`YTR^@iN0puDGfV4mW=Hd~OT7eawc-RqnYEm!!;d9f+*QvVECXCMs
z!jPUb?;!)@jhYhd%&)<<H)<C7LR3rVNA@#}X9!Iqws%rE2Q#}&*&vz?KqB(}!9Hb`
z(>lt2?eUTmJ)P4ZYsXI>-ghlF(XI81OBX6<*9H35q~u2zTaN<*IzjzE1m9tZXzH;L
zr%7S*_u{-76<3_!%O2QWCtfAKw?JbLL6zO%oMLX@P}bj|V4lPvb{7vwHYCQZkQo=G
zXQehcejD~m=2G}N4E#&(@RuJYld1~kJJpV97oUT>6)SpP7ee=i7Y&E3wbN>@Gi|G0
z*XPZEl;!&1tWn;=9e&(xl+_|v!_2#`&s1QMdLi%fi-6~lPB~P=ApzB!6#~s~33#Wc
z7j`CpcT3L@dXJ{tb2?EId&fWntLTgGj{aJ3IaSPGe5TRVj>HZVJ|ey`UH9ClP;Z8i
zrz|)nY}n_X%=gC}OR<c9)~76ib^d=Dz{tGnx7MlUQ)Gmh0`44cUdD4vGBWS^P5?bd
zl<vRW@*&5>=4j}x!@cooj=GMFxo~Cuyz-f_W})y)PTB9#S-F=!=<MOr^6~V!rsmJr
zGx~kjT!}&kWO|@@noTTedB|9+uY{$?tw8W{I4hrRKl7pDc7a5|GH-`Yq}8NEt8Tq|
zaEA(*P;LRrfJK8c)lr<Y9^tRU_WPl*vp3K0+^yD3Nd@tq(O2lT{@{2wiw@TB+Wwz|
zNv!>Ceh;e*vrClj2-xLgig2)o$D^8MVoZ5#mUft|kB_B=U{e~gpFL$xPZ06($?&(J
zD>%?{J75kH13fIl9K7HmCSu0+eNnau?tKNsQ`0i1Uamz2Y<XtA?7G-8R&7Y@-%I#`
zeEhVr2{b6jvWqrrCsjb>SdK9=X)O*qWtqpJdCnXmg%&<oB9An!{Jwn&NXgI8uf%{P
zUvJz`0GwcM7^jpX;@7HHs>9W9g&@@EysZLA=ZeSG4`&Ey7O5A1-{Ptv`I@tg{$ztA
zSs)0OIXd6)$#~(@V|N?IJwu-k`9gFNARb|pI@c8N`AZ9c%$yraS3pVe02%5j)ME2h
zbK=L{%VS_vJ5D0*n`on@&s1Bs=?W5eckw8zOXu_bNQSlh1~<uT)aBtUc6D?+MaPq>
z<HNGq!W2gPGv7PRNaO&~!*Ww|5c#U9M3}#sEDR)UkIviUL5oFl;Z=4cTD8${k?!b|
z{~^p_Vb3VH_Y`}53)Z&7rL^eTmNrLZO0zM+YB#8A*RtgL(ANra)_1_mo@DBsaLb5J
zz*}Yd5*^vjrL*heSYfI#UqT~XCc_KNTc?>-mfseN<OK>>`Yyg_=@L|_Oc<v~p#Drh
z-|i<N`x~g*11Imxt$=lH>Iu;OS3Gh3Kihx-bqRW5;l<6|m6eqz?N4DCvR4J5_4uSX
zo;6crIq?w%n{4eg-67_pgSuVm-?~PNR$O$n6-*y8&0^lxqRV|ip!;n1U*W#+MeR`G
z4@1Snk6F}MMAwp8<f!#p;kw>kqXLU#MJ>M~Kv`pFJ4^w_4lHxRw;#)=0$g+>f?Uk-
zMzJVg2iE(&115XiijBh9u?5@HnT}Z)LZ)=UG8S})3CTrtmcgjF-{2-Tp(EavnqoPg
z9<j4x1mih9?C}k+S7FGZ9Gmr?xTsV#V1T%1gj$ld!;VYBl_}9Q7;z~QU$18++(-zJ
zg}PkT$dur(GX0*2#N7Ndn8m2X4Eru;sgTN7<-4fTues&AeROONlheNq!3)H5JT5Lk
zpBxbj=(_;k9{d&KguHgpOX;i+i~LJa(P17`*EKXu{W~g7EFkOOs#5uXnr`V1CmCF7
z4>UR581{VqX(UJTCvYIEN~5F6BSvgT@lq^ed$+r7>}$z0r((6jIP0hylK$5#$p5aF
zqJDAF!P5C=CL0<S7h=c`6h~``-xywEG(FtIHu_?9Ec&#Bx^zLIJ2mF%l&>zmJq^W&
zGRwIKx1=)8*Kw0r-db?vUACxS_?>3FT6j$RM=QV2Z{EEt=*%A&y%ZV0jn-sc<+oj&
zuHCU}Z(;=%euAv4^?_*V8M~5&<f$!1=a1yy;u|hyRC}FEU0e1P6oKKp=SqU73Waza
zN44KAp(%&X=Jk^t<-@)adD2)NJwAL!*-yN7u<`9&b&)S7yOv$SNEj>l@>J9vDB%Dn
zjJUb+Co0ZqE|UwWMKUj+KXESM2~bc=I4~P(L^21Z&s~)8J%u}x_`io`h#~$OYo)39
zAS1lPqd3*Wft0M=-4&F~Ma3GYy&z4NM1Sp}S=lkV68KPNmESrADJNRtkLi+X>?z-_
zm5h*!BQ#vnM!+Sc3(}G3<ANm+(3t+8S73@GW~%!1qlKwc5|U6Sf6ug9mr}zLFxY8#
z++q%=Y*cqU=NTwSS`V3Q^MaDkostG3{z$k0r|~o?V+cRQ7!RbVkLJl0cl}a8s2qeu
zCu1nUV@MV~#?%>YwE#oq8Yn2FU%tR8;VU23HX@+*Uqa)3niC^kE*{we;j@YV18NUW
ztL}NSb_~IPVe{4^<xQK}X#u1!fg2s+>!$dz)C4eAxv-smD*^Xbf@l;gS@nMhB{_so
znS~6!q=&T6>GdW>X3%4X%8z9CFJ?~9xy(;VX(X2O0dE8(2R2`RY_lA8(O}8P{u}Vs
ztkoZb9W?=U=m(F^?q5CnqQoT4X3qF=i!wJ8fzo^GYtQy?FEH7Cia6upt+%q2Ot(Rt
zuQHcRf`({mx~QGC;7MtN!A_6cC*uc?nZ%zX23Uasop2Usw0OkxuKX+wQtUJhp`Gw<
z|Jq-6;u=rWK0^e>T8|nM;x=>dS3v0-C;R~oyJ0X!{`a22|9<B0oIjzIRhq#WWIDx>
ztZJ*u`ah4Hx}g}!$#_l|<$iezm6rTb+$g)@(6f>$=vN9hOsoY>`R{u?o3eW<<*UH#
zS}PvKYK;0^|1#Tn$TeP*5_n2XlsCr)D}s#&zyY9Qx*J}>2EaJ2y@}Z-ysfCy&%-l1
znGbuOQ)!g>@P$)AD2|{DwWG_`S}J4YHJP;$b{Y}k#sEMG)9eeAt$WPcP)#j+=3S|e
zt>p{y&HqsibLcdfqhO8*v0<iKM_&D{lfXkAt^_1MzJLCVISAnrjzN;49zg#K{@{UF
zb@gQ8&-nj<7il=zm}2x(9vnU1PIcT~k$kHIq-gSliaw}vj|BYj-F<NzGYJe5V@a_X
zXSV1Jsi@JNX>z<<!f(9nmqx-Ifn2ujM(D=hbve--Q42k~Dir>Dxa4^$j`aaQjBwj_
zXvuYV+H}EWa{ybX=C=oA`%u*^&*m$TS}iZFXuFw&Hh14oRYu=LCMCviTG8YmHfcMq
zDx~ObR<JiYn9ZE7Zx{<1?qFFnN4FkD%s~Y-O5%@)y>^VwQR<U&U?ThckDVbD?^|re
z)4GN!nd*hK`dkh={5DP00;3SvS@>v7mt*>?toN+@9pu2<b=Gmrz#+sNc-9NwmEv9@
zpET)Th&B6H9k0T+@mHm~eFXrgxt-wW{@CSTJ8dpK82~s9|9?(12{rO20G!4e4{(}D
zS7VkZr=gxlDyblOVc4<ojeFS7aksU~<k<2WQlzG$iO$}$a)Elem#B;_fc^oK4kA7j
zay@A5*-LZdeOzSnQ>T|9?jYpMlM2iTz^buDR-&}2(cYoQp)F3eNtkXCuKMA;+Q-(L
zmb{M{I){%nF43)*DYJ>|{$`Xs@cgOo?PMG>M@Qy)f*wHSF1(x?(s@_y6%ogr1d;=i
zO;N4M&dzG<!?tp^|5Lif(Vw*cVAcqX6C2%^bMC56QTsitgx6)`yItmZm?sAtHiXW6
zC+Z^C8k^M5FV^LfOl-?9*NaoKcJ8gOo#din1zR?&S-?vM%%xPa4Z=P><ax(iwq^Df
z(*KLSw~DGd>e@#UX_Qn-Iz>vPq`On3JESE9LAnG1De3N(?rsF>?(XjH|JvaDo$uzH
zn{#u{81EPihp_ivd#<(Sn(@qM&7a$yB49zi&I-aGCKbPiO`EAR?@8qBA1NSZ$Tbq&
zxmw;3j~R1Kc5Za7b(OLf2$+=INT7B(jb*4h>^b~s{T=C*5lU%=Ww!;cJd@12nO4#|
zi1SN~FW^f~Zc$I!M-1FCY6@!mDlf)Bq{bi3B+EeL!gP<w35m%jXch<Ht#o=*p7)!E
zK_R9`yF=J|!aucf=fpr7E(cwPjbd)E6!(yekn4EYC!=BNq_uUeLhvs5Ids!!-?ysw
zt5oSVtH=QhC49j=6>g#b)@CdV?J3-kO4y3~s4cfpZZ|>-V~t+`Alfc6KvO1``P7*}
z5LJayXC=b6wS$2<lrBr7db}12uGM~AVEi7Q#z(kW;asv!_!)<gaqWAxMK=Q)7cnjI
zOOPn|#c{V$>_iu`R5V5(h4k!m%Ivq)Qku=hAFcZhJDR_+&u?=2BW84eqkg5}qAqiH
zR?7p^ABz7be(DLu)J<+5Ng19HCOHD-=oH)7^hF148xH&z7c~l%+lKw*GZ6_$ILvM4
zOJDirIS+~JN#R)#-wVO6P@1?ItB-~rSs3z(oXCICBSZo8+*W3>x`jA}L%mLSxpGr&
zqc-4BY8lU^R9(gs<hF1-BAbtHW555z<r%4RVy#7pw%vWUFuSjG77D?rf#7=%a8oC1
z(RQj++;(ok!^#xR>DOGsl&lS_cK_}H8YBS8TIq>+YUT|8a7^Alo*|EVs=8RnW*9SE
zpa7=|38Vi75lNB0gJy3xc#4HBcbjz-7{_<nw&e_vePjl8E1ezM7Sw!UB}HVS7_u0`
z-TH0}W(w+Q_ZSqVv`jyU{)#Gu(S;(UP>*uXfrVPe^Lj;dXU$H1Kenx<+XQlr)c#l8
z>VNCz^vfo6XPlfuu2*^+^6Ldmao2m3a<kk>lL8YrkFJ{8?H#$aG6Jv?d&XOk4O%$4
zSDOlm$~&I-HD717(Y#Ez*5FnQDlV}*({re4m$W!f<aLr!dzq8U&XP1Dy4DtqW#58B
z4KmUr>sk>DlFk5|3oqr)l`7Yx>2JK%D1y%%_#JQ!y1s-3WRH`mlEASESiLH&b3~Kq
z6FczA8CZD^sVOf1y7R*Majv_+mH=HyM2=t({MzHUV@GDm*Bze{y&DZ`NI1>Zr0MJD
zmJ)*Hb0ST6e{!?*6p4{gs-@h_dk{f+G9i%xek}eDuILV!c@GRL2)7X7M0)WQ{3=7m
z=d^gj=6(vI9b(Xa>H)^M7{_9C8J*9%4=0ld=QpNDfbk%=mv2H|0<hkc4VlkTjl0T{
ztY=5B-*=z#Ur#MD$ws0Joq7d&K<FF(+FIBf=UUOJlrQ-&2m}hJL%HB&qcFGJXr+R$
z1JO(xEIfU>E$8pk1)^rt*<%E45&)`HNESwYE!|*YOYnWYH^ik8q2MW%eH(d%oM`yr
z%3@lmx}TayMIgH+f&`jpQJ=Kk>X&d<L4nuo$1LrvLh0I#x8m<UHGHt1wJTM3YUzgC
z9&yJFyM-WeJIdP=uhHSh^wJ2@>akSZKF`MYml^#oJV>R^nFowV>g$+=L6Ziexgrr2
zh2t+4VeXAE4~&@`7eRZ?_rO1Th0!0QusoUam%}u0h4@LdxJL`GcRcG?t>%El$f(S(
z<O?If2gzML_1+<aCdx%?PmxenWptK1lDgq{;kNvsTkYtOI**NPUy-PE_cFb7O!Z|>
znwUO*doJE&`FIpu8%gWl27rUmf)J(Z6A!g3{5RyCpbut~m#)`Nhu1vFd^a;tj<9Q5
z&06SF*YQ7^F_Y;o*?3&<ACda3qThW8Yh^O*O{IGN3Ok;|SxWkXC+8becG0;?Y!eeG
z-|-L_z;{DMk2{^!1znRbFg!YYH}&u1!r@Tf(Y{OTt!ZBFFGa{C<b*UC*^97jrLn?b
zQ97EY`lrwS_D?t399}s#BkdAMkhFVyy`m;xP>2(C3TeUqQRy8HgcbYpuJkPkpAeXI
z4EJ%MRj}<7)eLpUx1<XOZbeqnS=NdrGFCH`Y&8Zo8dQt5+H>SCMYd|brUt#$-&wYi
zh`TGf9A-$jA1#qD{YEpM1A9yLcRg%v^R|b*MB--4<c*r-n{kKFU#T@kMMH8@A{kyE
zN;p*7%#e751LLA4=PNGWeL&caDCb*@&rXEn4U_j4?}IPJo1Z-(*fY+IX880TZj38M
zdjGu;YmYYN3q7t*D_*&OMyP?hjt3vI^867FKm5UDs{X#yrb2gz?}?_=TrCW9+c4GV
z*3BCYU<pUPx91BYc%L4u-@8GnkI{xLOR@$$WQOUqh{ePWx}KHb_r-*j#S)6yy$rPF
zAKy7ih#$R-=MO!sMJgz7eC+`%Bm4$V=0*7yZBrf(m^=y{@4=!9kv<1Ja7>5C><Xyi
z$985oXyd3JqqTlEkLwU(J|}d<EoVasppxm~`1@oYFwiPY?00|Kt*`!m5l`9_$;seS
z-gBynp04SD&|=C<qyEO-PHmL?eaXr~vk{Dx8rtsOw04<drY{2({C@?cFS9<+kr*xB
zS7Aaw2KXe#uMN>%q1XFwvzBBNf8{w?O8`8~=ib8^8d2y0_IRP^_mk8{hwe=*^(XJP
zfD-?8hQ9z{4ol;`e*<cEq`0N?<mcR{*`MSRURB+?HCn`_*D`+RQdvW+lnJT^PZ+*Q
zs6i=?V$iC}ZiPw?3l(3D&}iu?Ppa69bi2P2J({!ip8@6A5_NCf1Eav``#WwLzyu_v
zLLnSGj7E-lz4Q{3qj)`Qgbc2q>!j6{z2qF58<Ro(PUB4(zbi<J&x9_?mEZ`2pFeJH
z@!{Q-I;fgAalbZUxN)hCw;|Z|wwa1$d5-Gg;L_gpI|mtNu|usFC!1UM#vzoXZKWa2
zeUB}<QC=O91rm+caI=$8-gAHZYdL?Lo-}JaDb)FaxNutwiIM)r48mj39F4|;RZ0R(
zsD~IPaCJg8LB&_jD0zQOyRQnAeX>k9{!nQ<S5`Q4eEz*n>Wj2G@2HvCb{Ym)VOz^+
z##6mS)T-77Hkv-qeK8PF-MwjhsR7ew6*j^$2Cgs~wbs0Ui${K}PmNZ&K;sKRBR`#|
z!}&65C#c<)6)}E}vfLjAP-W+{ms&gXy!Upq{()TI=2BbGzD9vat(M??M%)h#k|M{9
z=@#9%Yy}wEg;IR8qn^KDaY%K9Evy>U!anAcLVs`!zB%LXC%Xo;U?|%Lw8*a|*pHy6
zkVRQJX$VN6qc27qR<@7MH+fe>erBZaCEK~Ke+CTxkJhuUrJptH)coJ)8=+XQhQy;C
zub90&O=po(pwVoMRWKQDvSskb!A7$1@_m4{q<r!GCn4^hFCCJ0lWKOTxbOfgU2fnq
z-i@QuI>|{?t?krKOT;@~zCmFa$gNW2l^1s%gGHnTuy15$5eo=4_NsnHgFy!I2_yoJ
zV#XcnoRj8U1xVCMK=BVUj%}_0r-1%{Y4NEAAnQapNnl0r%Em`>Yg<_R(V*ElBQk;@
zR2Hs&P>udI!zc*<#hN>aA|-#@{E&lzuo<xQmkd!yi&F#F9bDnwwB3N18(h3pGkRJ$
zG3`G?GdWsH!gpJ@tImG7jlPRGV!H_=9!FBYzFbAr`shaVE;7zOuuLJ}d*e(YgNsL~
zCLjWJMi(Pqg>k)b^m$eoM%=)RHz@3|9d*FTB<S1eYC{NMdVwFK7P<So7n?Y%A#S;I
zGBUo@XG5iGZI@p+6bO#;#-o@tK3D@9*6fSA0olic>&gnpKn8K*wGa)QNZ>Dv41Q`h
zIAu*uQ^{C#CO1iy-aH^<J$qje<~lkT^1aqA91uvuB8TB6t^(Np`e^>>o?@3wZ=+ew
z5b4J%@Ch^>_|TwPtB8z7mw6f$U%4{8j<)x?rMCXl`0v7UoFUys97v+S%|n|ynI-X&
z-R+F`<@n_XexojgbBCkKc?0{|Gqnl8z~?jG)U3{dR4S?)NA%_0jN!$@==1xoM?&=9
zrHjO4iT-TVkTE?o-E_=c7Xh_~_=~}#)e~<HDRq!zAm~ZH+C0kWE-@H<)0?7m=o8R-
z^A+LIF8b6hDCdehEjFSFIxXfk2J)FoI-w)bX-Puw!HhXqi7iM6k67M*u4XhdJVy?l
zVDgHF_>#gxL$P3_=P<V+S!Q*wA@q)KlvHh88WrhvL)AH7-zZ8hBz!c|QFl1;Hae+)
zp%?1j|B;&X!G?#WW-dxUq{_UYu<^gsLN_zrXL&3H(;tyZt1Rtj?T$bSs(0q9I^gE%
z40ysXsPEG;csRCp`hWk@`SR!Yrg_y`tMGh$(C<^dflmJ4IaouqTIG4$CtaOHouaqx
zo453m?c0+E1!1Pk+psjB)UNq?1jEcfq`**C?&V^S?|@`NIlJuiII=T9M)0fgI|NF<
z3_kB@NUaFej0VgG7OK(wqQ#&dl`#JFZdmJr0UYS}+m>X%riQd&1pLQC-8nt}kY#p;
zeqb7DE57gao%KJ3d3R2OG0)_MBbB~&e~Ld_$bS!q+2=^3ODTQzH8dXWY5#rY+&<-U
z>^a(sk1=#~*$*KS$}eYJVekIILuRk$gdugE3NnaO)6CFM=fhX=5BYIYiq`-)CNi>5
zJYP7yM%m{4&NvjD?ha|+$x4H$XE~E1yg7jH0<G-U|KeUipCEN-l0$KI)h<vYD=XJl
ziPgDG!)f;G#fr=L7k&MIQPtH8Jpj2uh~0&)+2yfc=n6`|FbH_tx}Hhbb6Ooc&o;m7
zi{X9xQg5vHQVK$3Fa2v{^rZYu+P3;WrUQm#GCE$mDTg)qtVMMn(4p@Ed)B-3jukqG
zZdaJN%$^Rh3KN7&0eM#o`UKGS2Nke;RAG}~Z4ekXh;%@^2HAQNYz7U5lrA+0)3gn3
zkkR2;*x!IpQew6HIieYxRf$IlKjGfW3&Zk3AW$Sf4L)#RDYyO-ogE4tCR&VG<#ExP
zD=_vS!tXdpnlzFAn^E~0@Esaql&UE@aAQic_d;v!RKBlU?OeBz=nP#K{$3HR*4nDW
zF4oM&%#7Q1A-LJO<>(E4l0pTmLq<(v2yO*zvKYTJ>?S?uzRdpOzkPhPDmi5X3;$N~
zdFiM#L{*MWxnpq&d=R*-15O$hasX6})V-|O^;gh9qU&^X2q)`ellakrqBwRcDgxJu
z4QQV6r<Lep<|Yi#DSynwn-ZD;N*gZgBRy&+hcaI;cEC9^4<LR3@el*5Qo3baI@eC+
z@A*^My+-w^zE||^Ld5_WbxLv;iVX}_HZ#%s&>~5|?y!*^4%m-cn-fB!PxzRjKy|+Y
zp-?jD(NS3EH$7C)(I_4wI#1HApPeA0Gb>3(!tZIC@_h0}W3tUb&D{fSnMOAu8r+f*
z+v7wMTmOTY^4ChUHngx?n@XlTje&>87OrmH)=XYNDr((yJ1BTung>N^49~FeQUbtn
zB7`dEhRYo^2K~x$CwOe_hY(-zag?2DtdlY5nCT4h0UB+)t2Hkmfv8GAbKpan;XBl>
zHv~{ed++fdPZP+k>_V)1ymG>c2@sy(SB)XM<<nbSQ+-4v2ZkD)PJSp^JKbdMXnIzy
zLyQtH@HiBQjA}kyIkSAcQg%~DxvxqR2$-j9K@I~)2*49z8Z-x7ki1w35dv_8C;Qgu
zS?kbk#nNU&!!s2b?mc@xc~=~izoUxN>a^NCM3FxDm%d=Qo`xkvkaq{$#;4s@-W=TM
z3$Peb^&^hUXma@`K3eg-x6;9Uo#AWk9<f7$9K8ISHXH{{E;s%L%~D~fdA`pAU9oIg
zkVJAB!<ED90Yvc@)H=u=O|M7L@MWAsn?eTla1%(<&ew&AKU1m<amGdq)pwt)tFJKh
z3N)%yxm6*7a;slP?=LMSY0ZTwG@G>x>P>~;ks1wO#eEpnIn9dkCoZ3QN3Y;{uHRBV
zOn3R4%>j58+!Kr<HZa{E^@u{2o9~OT>dm7<L&0=^^tD5=Tq7fH*R9al$0?(8@E&Kz
zb686yy47YXDBQm?`KnKm>0GU9Jr^rmL!V%bZ|6b@Zh4;J8m1WZg(x}}#%$J54Fu|V
z;~XCcyjxU>qFAN?z(Sm?4QKnmJ-+bY{lE<{e`H3(@G@*eyuRtoM;ttbLdU*+{&M5#
zko6Q)Bhxv!ZL!u*)r*<;B#hF9gZgso_~=h~@C}BZ)jj~vU@qsbbV&IBGUXL_4tyeR
zeS!3N+Ea|F*Z3hOk`*KdgGKx-`Xva!M0OfVm&rqJA;P)lk}4|1uXM}TXtCrRX+5o6
z-<d9v{4w%5EtmyF3Xp2E?Oj-Nm;opgI5y6Dey~2~{#Mne+lnSGubrg!V^?j(&Z6U|
z?YjmCS(m)abRPNawpNSv+u4{&gJ=|ftGF$$KfN*tHor#dvHKFDMobFFzvOTiPT#Il
zIy8Rq?2vtHaMTlZJ;?uH8%a6l`sE@*&z6`{>Jdh#S7w|aOCS!$2*kASTFA&oeJ!a0
zP=6f5Na<<Nm8dun#U~-<dwG45G<2B5wzA~y@A<a11C{Vl-S^UBR|$7AIjwKLDhkl>
zo82HeGpH9)!aV;&`+&HkKn}TGwh?Wl$5XT7iQj(X)X^R+E2h4<<5`k4r#@v-@QDr@
zygMU{`ZfC2C+pf2Nl2(zY-YacAI5PI*NgwI0-$m=5`Hgo4by2GO%Gwd#Om)0Rd$Dl
zh7q8Cp!|lbK_=sRYnPN>OH$8FI4iyPMo9&kT!qV~dr7gxaf(XbQVbpv*Bpu!_7@t`
zhfZ>fY+X;$Fe0o9T`iwuhKf5(-!P2cYn=0ag_Hxb5#Z$WUG%HdHe3p9vdO&Ck}BP3
z*go~Wb{TVdog+{6^-2T#3t8qr)jBcTRRr9tDdfT^Pk3lbkkY+<H^&DJo{Nf<cTcmw
z<=SLBu1N*%F_U^H(U4SfD~R3N$x>|Se~xaHQj&_M`J+uwRcPy?KA#kg_B||2?K~pi
zUwM1`Ko-EqwpEPFP+W!!?8~_FxVJ&zGD279yek`hhG>9444WrNG9g&!XPQIThtmZy
zCO@Qq<JtUKXj1E8F$sN%WT*4Iw1Ze~XZ|q36~*v-Gs`_-K>y9~$1Z>C@rR*wwSQMJ
z2-58|iiu~mgCu*U`WTD-Wci85Kq!SW>m@*Sx&3>G$>7+C>6uzm6L_pO=G_2pzw#sQ
zU3W|x)yfR7(P?o9O;m`Qi~6>?_I<n@)7+A_qa^+75LoJS0@#%#Cbv#ttR=&Wxgi9Y
zYK!T5$Fu-9DlOs{Bq>1vG%e)RiWWB_*G}YbXLoI<f2_?-xMaDE27ntdE#pqj`W~QY
zg%E&UNclt?rVpQ!T0izHW*)9Sx$oPY@N@seTEY&?hI14MjUw?`WH?&sw_%y)nn;l&
zW#8u0?)JVR;J=+V(e-~V7!^IDJf7whbD(kpjw=b|xQZ+47vO;7T1_ckBGBHXb}U}6
z=vzp7rKW!yiLE{od1P5%5bW|8+=%`Rcaz^jm9}bTd-vn@yWDN7$#T=QBb*QPWXhNv
zypJ?ijblQ_zD3M&db~T2pMUE3{2AWE^~Zu`8e@#TRxpXoM*D6SymKwQc1lPFh!jDZ
z%3YzHwI~s1E87fJE$CG;j-Q+S4fOoA)J_;>GQ^XVewUfNmaRYZ{37O-cya0r;g~wt
z8v5UyTY64>$BYj2DYt0aUbL32nQbL7LVL*htlF>5M?xV>XpemnV+}GCYNHNL@g^y#
zxvktx`d@{Hu3KlEgpJ*$^YQ8u4N4yp^EHK?Iv04Pemmah7dXBfO{OD?DIJ=g?H-TP
zhez4m4INF28h5H&u7MO=B&=d~SJA&C-zi{Laqwt<bhy+W8gHCrlOg=*f=h{k5c%lo
zi}Ks99|%^I5F@~vW7_lb`%^Q<@$^VVghUcxK_dO{6UGVO0$}}QdlEtR>{=Fnm3I}<
z=D7YFB~7Be;njF2?}`@^qLsMAdQ&d!9HR%N7vuZ8Zr#pkI{#0~THI@IO%omX`2;TB
zjl4|xYyBFXGH086f87<_dw^h~zV+WGf8z~msR+mR7r%Nkg<M;X(2cnZgGRV>GPrEL
zHSpooi7R2Hp%lt3O4KQ><R<n(@MqerkQ3#L4k0_?@Eg>cg8GA<bPIF{U~$Ahy3Ss0
zDXd^nIkE#F)8}z;w;1BkCqf@SA3ztE=uHa24w@jjFf_x<{R1B;__3RmHZzD(T)=x&
zbhCY^$_FD!1UBeSZixkwiSRu-73Y4)E5srY$Wsp{V)u8m2g@`IA~J;VVDlKpvAQv;
zasff7!3inS3@hS;d|XHJ@$(g*9kG5s5+s3^Bh)F>wGu(&s~r1a6h5FDW{-(PLH*Mi
zh@?;45C*g^1pUS#DLm98_w9fil6`vMK#qwox<W-R9Q@e7sk-+*+P{gLKc5bPpr1wE
z&szSi&@Xqo$InlcY^uJ7EF?la1cf}1KSXgu04qr(QTGEPS;6BbRh^+&fl|CX%nb=Z
zu2Ctp1R;y?SjlNU4}iR)_W)yx$W0dD;j5JXn<^{S174|nj2-p|2wFq{_V?ckfo$dR
z^ZMPHMk!zbF=8voUN%Yo;k+Sf@?dd$eTWB<tV-m;B`!^IHqk+T2ivsk%^Pn29I^<%
z9+hF93G&K+mEph2;3@Fzzsdk1DgVE-44-M*CQ5`=UUy(t?bU2Ef2UC{c%KOS1e`dp
zAYdg0Izrijp?&||{UrefXJ~qTk$!HG6fg;PAiUCDAJd(L9Ox#VM{m9}JVz`30u2O0
zuz_u)fMy37#tX_^OYH4C%!h;P0E{c+d|Mvz!<hze;6p%=^F@3jWVnbAz4PgPR=(;-
z$Q!LVo=kCNH9pDXg}m_sV$pbI>K<(9!xv7%yl#76d#id4g;*4ZPW)Eu;i@S)S~-mI
zacae;iWOv#B2YN-&ZxS43;6}|<MEn;kAza<RnVWHXHV?jk+vqtQarp;ln7;>c-nI*
z1bLgU2oQ2`!jzWjKSH{Xg0EANj_>z31wsnh9v2@#tRf%u6SU-BWGF0#hf&}`dB)0W
zxpeFCHSn)1W-{6`+Z!!0Xwsce2Xv28SRp~CR>r(74jG{t5>W*a?NdHV2xboV1c|ZF
z*l6sX5X?j;0=k&tKNIW(UFJ|>Ns0xSCBVk>gB8K*Z4!W(#)qGYcqnh2I5sqYBL*Pj
zL@u|=gHa{+I6wz75~;6ALB6h7$o-5oQZ39D;&&c?5U|1bEs}OS>xV*d;xbO=<-K?q
z(H#|hA5ZeZgD?H%jtvYt13A~jtRLs61=hr#WD(G~;=u>~N*8e-*LZ@IFAiIMw{r?2
z7he@{WnTWY$B*j_SzQX~7kYX*EUHmOm^xHsJHD4tkCMcJwJa4EEW&{d58P@QHmfBu
z#H|AbgXkC*_`Zo$Toqqqk|r6Eva%Q54+}_+M*C!{P6zSmA@ByPH-&6e#$@*R8W;%;
zAIz`$aCsJmtV!laD_;EQe>SVv=0BT-1_cU`P&$Y5|JH=~Ne2elxrt1pKOY6$69U5J
zmbo7%Jt*eR%7CXHXz;LK+n`#c(jPBN{-4G4<pGmRE$mfULcjo%^E|jTejHK~abQjL
zXB_H--Oi=}sY$tXhbN#=JZ_$fn7crLchP0`1{IjwNhOKd<3cYG0bRL&fA|J63gk>6
z|Ie9#d=NYp3!v1HKmj}<2xyq%e>LpC8U}RH<G&jA|7{zF@IQtIqX_#6O5Wsu%ykhB
zIODYy&$=IA{s&7)1x(7$2p%o+KU)C~{)88B&7X(RXyG7#1D1c+yZ4nZB=CMT07bCC
z=2aNOVF5bbqxE!$*%5YsYIbId8u<|J???duQ9##f4V?R<37YV8H}{S<aoY9rGk9<g
z*O=M1QMcWHF@PTzLIpw!%c>WrLqtLo2gW%hz2E46?3WNS_`<;<d0HySXyk(K0#fhX
z?H`y&|0BTv{tqc1cn3`Kamd4PC}4&y{G+j4o3N(Ojd;94hG6$dmTl|f?1=><st5vc
z@%o&RAj1m*DjMVpi?N%e%7f9bk^tNg{)t79Oq3s385Dp)izOf7HM@C}F_5(ob3y0_
ztr69bn1nA6{ejZpS6=z!6in5M76R7jFaD@r!dF2C{0aa_pxOK@FzzWDdIQN@q=4V-
zagX7K2+)S`C;*Ows>*Z7l7qU!FKc^{PSJ5`#L$3m*m<M@`ObceWdXZ_fC~J}Kgr0^
z%>EHvcJp;zz|i=TK>9_MD&2zj@MA5{q9?(sw&6b}rO{3iswCh%|A+bU0F^NQV@8W;
zVJUrwY%s{@)A0#GLWzv3ye0vJul*<dd4gdtM0l`zq>qw;q46GtS8*~^Ap*j8K%{?<
zWf4u&=TZLlso@z|Jx0;T)mzgt0Mr{X;vUHAJxcbAFW`|7;ujL-J^G+-Gy!nbK9H39
z-vYP&U(k^%NY^|L0D%+XQ8KVGB9EBl*c{%N5Af>65b1-wU#<NFoQKB+=Ii+f7*s3A
zyaf3$0uxxhf0B`-k^Nh}6$4!sFgG#C)_47P>;HeY^;A00$3AwEjWS8UNc-!_y_0^U
z9o*KD)FA9=RSKZE-UlkDl&;qc|5UX9vu)2z08~^6GuZeI4g6SYPPvK_&|A(&aIL6y
zy5k+hC}095F*c=(jDVa7Qqe~(^E>0dn|8f)Sf^xnIr*hgWB=i3?o05Db-!RJOMkEU
z%%6{gBE)y6RI45NW~<eB(iwK+A9QK%8VIzz7hUGNlCpv%YhGD|-+Y^8sGPYjN^fNR
zIFZ;HP>(3JIVIF^VHGg$cb)W_Y+qVy&z36|1;9h8exl3~fQLCaqDGth5=I7}gE%g-
z@Axv?+Ez{Mw?d9f;xhmGKU}bnvoH4QF}0Ct_<7dSncrUnF{$qNk2>yL2JhM<9qguD
zQK&|)Os{QMyHvAX5JvT;k-D&KkB)7%XWNc`B4HkNCqmHan`G=)+!YV1ZrC`2{qr!s
zD(`55E$&#R?E0ItWs5=#gTK+LZyINXyGBgubL6%6ZFw-@`;u;z&-NyWsZfc+MZ1(<
zhBHPE&6#h1+OIe*c$2UCR`GUr7ab6B$Y*?^APxYEgUG}Tx^|2<?8#~OHSMyO<y{E#
zCesBMcF^}I2F=jh+RBZ2FIXgQ<IU364@oMW!#{&U2|zoIiDE^SOfih2uzFu|cplC#
zz<+>*A!dTqYG*<AXuVUvQ|XOcGd#KGk5^&!LD;|Chaahu_H~K}?zl4-!sAic99Q44
z9RwWSehQxt7dyJXBRVBOjrjdKzn$)=inD4qWM3&tI;l4z)!_Hv+B6;{dZxKuHV-Q#
zMoB8vNYv}{FNlsYHY<^}{*mFOZV4gU4YLJ;sVT{)PSU@!v~%{q#k4w*z{S-lxhU}J
zHT|p)-HzmJy{z(G=S~rHxip<4M`A>`L^Q$%PBg^w-uOj|+g!Qoz+#t;6UlDzD~F#c
zG*37}%p&A0$R{!bCD%4IX}oc`zF9|At~BlCUrp}vUI^?t+~B6E+yoFmq2Z7~&(lf`
z&a4-E5okbV%s?x+JN|t~kKyJGEyGU)pvk^E>w;u_I`UZ}C60X;pOR*>=U-5(X4sXC
zA3|eTJ(_TwVRhTHeclY0Ki8`xA=(sznY2Si`=1;RHrFXmPfawG&DAgckj$8s@K7)K
z-pz&2+H$>kWrwMEkY;z=vMHafLS22@K$?=BN_UJ9g?2nUo||sDyS@V0;MPuydPB+Q
zLY;2sxAbu9{Rt8Mlqlg$RKZ<wnj$qpqK@AMq|()iSNkhljMMAB)-h27=Og(uh-E$X
zuy#Lz>ZpsTBlz#DU3l@!n*6*cFH^EU-!FZ{JsOj|+EhYdypVoxDMoea-2Qs}6B1Xv
zfRvXO6jAW=|J@nl1EK8Jxf<lcltFi-e2zQ?k?gbC6bi}6jdhE;I#u3Eu@KUsRAJ;C
z`3#8?)RcNLtPc|mz|C+jB@^nD*CYHcs++`sPAS5Ue8JoXUPoXYc>UMn@Z!#MpO1`S
zqEzwio|oV1!4~U~q%_Cx(-0S@#8=}7VKDK)pFplyof!6R%*sbf<Gw-Q9ouYWv5q?O
zLL^TazH5zSQTyxR`$aG^FEqC)!MZS(J%&SCElHJN^8S1G^;+7ZChYffSRxmR-{k?{
zH18YlZ)NwKtGwu=wv#*BtKIG+6SQ43joJ;nB0u);og2}`Nk-9c{Zi$1EBz33S0dYY
zcN}##c5&LFX?>Bkf3nbOW62ycH7nx8#kg8)B!eH}GCkf+U4GWb>lPnM%YOX2(7!f(
z;rMH+?ag|R$!>_(TU9n}+C0wu1G%YM+l2U!2JN30T4a^Q-WhUjA&K~maL}owtwuT#
zMsgB5OF53yzC2>lLmMiPo~*TXTKVf1ccN7%Ih5H*qGRlK|3%&|U$bDGy2a_&$9h?B
z#fh4>xvb$1lx-u!zC?BFdB>>azQ&!&HKaebHYqJzF3w}Q`1c97qA1zhzYb(%wkG5(
z*$9qOX2NNb=Gu9)CC6qiHLp|Vn%qh6;AHUcgol2kxj5|CDf_##8Foglq@juI)&}N|
zxUcNLDbsfB%-R#=F8tUNDoK6n+dMy0IKQv3KTO!n`JA=dvR3`RiP68ZlbbhuJ^w9U
z3N6ENjfguzh8ZnsIk<o%7IMB{t%Q<iuEnaIdG0K#^D)Gt_8y%e_Sy6cJ-%i_YL&S0
zY8xZJ;KnnjX^q^fs`t2Wb=)uI{#Ml5?Y`l5JUNKb&Q;1U<6Ayn83m02#0QJl@OQmg
z;y+%!pxG6c=>B`1M}d}CUsO?+;)J&=yrCjuCbRoyCb#2MLeY}=bSdmSyjY)`gxp;9
zE+oC?&e0)Hz?Ks$#Na2%V@W_+8|Rry<44k9l$VM(-s2ga3o%dXB~+fm%sr2y*E*?T
zaB5ILi6R_JK9JI1GJR7RhMVY9e<qrDuldfXf7KQr{vd<7SpTVqimuG%UX83;;H?5G
zT3qm}X%#Ac<KeEOC^fD*WzUT-f@^t>^|2hY^XInWO&uc4g_+&{sy>N`$m48VyoweL
z*H8LCmK^71yer~|rD)+I?)}jhxoFv#_x$H&i~2sp8-mRIA=Ado_Z5tCsQWK`rPsU;
zNVDATrHgp_ZCGL@b_>#avOd<`-exkCH+_E=`#dgz^n0=Cm`coGRs`{l2m2zo{kIWl
zioTa@L}Sxv7$mQ;au0rTor`?Lojqcq!-!^}m9GhHAo5IOpN#YSPcMKF*IV7Jug-^4
z&k&cZ){VR!^H)#po^qr5d~!3KA3M4%3$Z?Iaj3i@)=~U?k;tXZ+h?S4H~Dl5NubH$
zZkHc44eCPbOgzV`yy5yLPuiw4n{>FleR(tOA$Pi@PLMv5`#kIg9CtVRPZvP>c-_^>
zRfkxVw$j`7lPu*$3OeQI`vN-0cdfI*;UfZWdS-4hRJuC9h@eZ@bv>II=645tAVEYN
zJbj|gVx!H~Oi`vHjHk<HZ0}0(y0)!>u9TV;0m5PXpX2*YacBg8ySv8H(De#WHLpI^
zg&^`I7^e!GUU_y9mvE<$s?R~2TJt-IyXj|=@3)GB_~t+zR_|Gta_O=hH*Thb-%L#a
zAjz4(>83|$Uu!1I(EIaKJks{DvBoHQA>zr(P=;iNfH#u(9P76|jU1(XwbXGyE&WNE
z2mFfs54N#gSW1Sw>vf_~?bF&p%R6Q7q`69$?@0Xh((itEhQJaz$WQ+bA6(W<Aw>(Q
zM^MBi4ME)%CeNPUj=4i%+)z$-^O#@8@t@8bL%!2J#-q0C|L$l_L-|C1XfIfO9H|{=
zpXM&UUm{TnGym|`$4}})tIcb!?%RtaCr{3+m)CaA`Q!Vy=le{KEi92g-e26Z)$D&S
zZqiEHJGtEXXezZoUD!CV?Ao5LA;d00Kl#Pl+}dF6@&F_yfOK9x_2VLK_u8-p8)?j_
znS;)QJde@i$?VRhg(4G`KT@6ifv`916bZ+IWb%2crq&z`jZp)GQ^YrfA$khd8`~?d
zrZCt!5|Nf2M=#Ju<G>r|-cfwl!uJW9Gqr%i)Y&DKqZOQXxaRuj0}QI7Y!V?H=S@An
z`<$q4Qxc=&A%`nTobRY~B5t&3q5?67r*5)5am->4W;6NR1aB`+X4^-n{GaxFi1Ocb
zOxMHBJSTx8L^%j=YC^?nHro_k_4eZW^_zJ=^KSolWuRNpUyDLKS(?IMwo}V+=KlOw
zN#WSPz7Q2W&p>06?oHQZH22a-Voj)P+)P(?2`HgBt2Sztz?CbEPu{`t64V?FnXIr*
zu~5!q8P;!Bs-`y5s-y)sZI=FcAFsoFcSn7V8Vd<h=kv67@FvVyzeY|c6STcYAhA~_
zk;2s=o@xpl6{DbO%&E^0xRrCE#~IYb3fRajZJkG%ddUlOKnB&$rnogSWIjh(c#f|3
zQtzCsxfYRRK@Qsst*ysTea}IRqhD*E<!_E<)s<|rz_Yso^ga|sH!2lDBsZsnB@dsm
zVr{pv-66|M)JO(*Bk#egpMgXqlubeMz8+UGny-b7Z`_Hd(m}0!5xFvJsrL9!<y5zS
z;B+lP2MXLJ<3un_$YpuO9@Bo{K83T-x46o|-P*!Yrphiq!Nv_I%xieog8L&cmR*cm
z%8ZUm*Slh=y_t#CUbDXS;`Lm~)2ap@DtG7ORSeweHEHwKp7Vv;2Cc%wP8pRVdEe%h
zK!$bckuWG7*~L>alJ*(Xvk8>;wFr7mHKhWLm`}q0dc}wz#q!BU(kRPBGH55Rn1j39
zQK#FOS8xsV>GcP~6gM5l4A4e9FiwN(y+-A1ye*OX#8!6$eKq%n0|G>uhFOcIo*J0T
zXuM>6kfs|ROFwo(%CJcUam#Tp$=Y;06}F8m$3GapjWkJot@Ybckeg&@xY$MG*<05M
z2c?#Y*e_^diQLx$?d%%!Rrm0I1>DaT4Xx6!Pw=|hxz}V*a)xs6ZRWM))PglvSSncE
zRIpW<D+*(yotkZ@@8vT>;Uyx&zKQgW>suFKW<IZ#DPVaS5{LeHJHn=1m&8|UcT8bF
z@aii1Q%@{eGGzzt9eBE+Xs=gw=Wat_Qg6tIeaaqrZ>lF;NEnhH1?b;ZixRXP#qKwp
zHe#?>%vY&HDP7<axZ%(pN9`q0$G8;p6Qh1T-N*2Kbufloc4SnECetn@_j;mr>?Ts>
z%g3+yPnE<r-_2~;-FJ{VAqbR`l;c&35x!aFksJivJi^kzDH+v8wMsbcs<&?s?5Yce
zOk7aLaXY#GqRLX)bKxQ)$LJChrL+l9xA6(_ixt=Y3?C5bvQOkqdMiYr{W=dmw<+b)
z*MWZ7Llqh=;_Wn)p76S=*=z~37Y=@pZFj<umIDHAN*fwIL%~UcqalAxtfgc=J=x42
zi(MW{Wzphr%#rIh-qm5Sva_^P8txp2aa3VcPb9L0LbKCnnb$`B%1Hx+%-OQNeXW`)
zy9>Lcg%-HG(kCZ0rx~t=h->Pz6@gD=?epr{*f*QeXWr+F+Nd(Rq+|Vrb2%)HZq4L>
zKV4*;iBuWIHW)){fr@$<1d8r$!iILrz|*{lQ6^DA$6GorGE_WSfxlT`P^&4qyY3)I
z^t~5DIP~3T?f+!oruPf3g`>rSlpkULBaJTOg|gb|5TV_g)8eVG_$#q(-k1X3;-FUo
z?0ASv*T{iWc(o+gZM1C9ICL@KDe+#&rzkM3|0U~)fO;N+$K@P>OUO5?MNf&a__LGp
zz>|Nf+%6`u`iD5~#sCG&Wh=OJgo)#gBTITVf!e-i&>1S+7tbE{Nf`JRDx>~HX~@Yi
z?2T2FN#aRZF*g}X4*@-~s$GZewWORD#oJ<Czy*-^oPV@%1fEg8Y-%U5=0`tOhr*<8
z)Z&tqMf;x2Yk&HLZGWuE_-IQ9((B37#g2SG3nYUsPF$Rz=`uz8`)5~lIV0aDreHs6
zMpRW^lyWHShax^b*}p{@R<4yM6!l7EA(9>p^R=w6^d;10=2G~Dc=bke-8G*IsrH@X
ztLUDe;sq*MufjCCM=esQ1_j9Zd7rE{rGM3Sp<XF<+=Ee8x#c8s)QCFl|Jd>M1)-1>
z_1)5|p-;cQ!H{yw7O)&IR-8U&-31jiLQuX>4yvM$c466cNux7;zDf9JSwu3K?jqSI
zFmEtX5AHy})MKT+lAE(U{XzH)_u`VLvYhMS7sI;D2uXL59V}cxAdIX>=AROiU=M7(
z%CN2c!kp<UI9@!_N+~p&CG0&b^^xwsWFtvNGB*Y6i`_?~thXn$3dc-60#ewki{BYk
zZQj8RFaDS?>Lfg-rW?|!j{}M8h}nHrf&wSEHcbwE4>%aaJ=lhQx^jE7gUDP&F)Md9
zvYDHztyS6cL;EK^Fvrg&6%guF!5t;RG`0jQOlaUQcrUJLf}b5<(O0Fco5^mR>Jd__
zEB2$jT{ruzPS=KWQ`{#T46A;lC22Ib5L99%Ww~3-<UFg(@CB6@?ttyrVdl;|)8#K|
zj#^3WwabNQOqtSj+!ub$LN~L*rrbD%KeL|#;=XqOYViU@&AD22Y{WXhB^^#bNfoNi
zNiy@8dEF83vHNMT%5_QvRi${G;^(XN#gh3&Z}^i>NE6Hm=+P=r*H0Fq1zbCDwGm+w
z+lDx0R}{_k2}0PWP(zPw)hX;$#e$OFf8~QD>i&EiuTL+G{ioAk`pSZyoMEPPPSp{(
zsn00`lV#A<-iEZG6)~oyYbU&Ogu|!&aiV#YZzkYpltSHRkFn_bmNghq2$FO_fs$Oh
zU;L~EO^**I>=q9-dacCc)##^^4|EumdO9Uhg>S*N?UhM6bcA*-uiwu?{`%bdi^)}v
zGXd`VLI$d(i;#ZKH#ZK=7z~Edfq8Q@lPO-eBrTij4~^9FB@*V!2I!IK%cO~mk&GVC
z^cBQ371A;6JEDhM&Ye2zND|7Rn2kqWD~pcWC$I`S>?zw>dJ#Q&DXG)$y|Ul?2I-U2
zz8ol&F(GIm)GxyEXQ6^9YgE|u7`TS=5Fr&XIhOx2_~^6QJoAO~4ymRbeg0PVB`G4)
z6_u?;x@IJLBLPVYu;@sPL}!7+x(yhZILY*tZ)|g2XQK1n5cJx~Zf(}WG4x8zD5ft`
zN2v<#^s%_PA}o$=QzqJLK$jK6!6ko;MJ29Ap+kI|nD)=?`GWN^NtQ)TIy`O2WX<l(
z&Kn}Puo{vMQsFtHGw)Zgyr5Bqxxxhk&^a=NGGdP+ulNbY-fejnS-W8EpVhofDO^gd
z6(sWM=;jWNu-4Saq3LOFsS!v-q)*~h@GVJ)Ne%x>&z-cq&%-5#_p&pcTiMyqQ+!o_
zWOdM*!K{#^-wYkjNUjy8|2O(}a5k<f;^8fKA+mz62li;XOyE~mw7VkL-q!wxZ0>Td
z;|fR6s7@E;!AD$n;iU6J)R%j$LdagE9_NSx)-jknvbjbU!vf;Ri#Dr+M{{O8J4%UR
zPq(7RS&Qr{2j9Sq?K{dN)QK>nlUB1RgD&}4!TSs<VaP3LhGn|uuiP+N-0j>bbdQ9(
z8Ie!VGcI|&W`vB{$h8TWm(`FM;SgQ%KyNvRXVQjpD@GCzde-}(X9GlLoU1lb2u}r~
zKfCj=g{~)O@ceb71<kwai}HTy+M``#>$>9^sDe9D0kd}P<#zg@BdCVlE$(r&PVm;B
zQx19-&gWqzmJMOFf%B+0ulFP9vI_-YFUc1&Th(^p_#d-mA~M8a$zl%xBA{)0c)?)L
z?p)4nTxS<<8pZwWxn!aDyuEZL(R@L{*^oo3H5~z7uXAYv)p%@or}_3HVNAn(TvnuG
zJGwy5fy%{+XF<4vF}cp*QnbW+&3LlWDHr8_<+965&6j@z!|QExES`7Lk=6y|B1AzE
z1%Z)k1~xKtmzQdlN+TTdjJ{GXJIUGg@5$v8S}k9~=*?HZVJz|Oor&*TuNy|S*b^PH
zgnStdDV&fPRVI*`zJx5qKtE|=!qvV@d7)RM;39;QeGWyk43gZD;}G#ZeY=Ci)1=(!
zNP~>tbOWvNSR*QXye81^<O7{n1OVSAK<E69X;-s_ug#y4-NG25eiA_~EX4Gf-zP5F
zH!f|`fM+S_6r0#jH<9&<A<!uRB+Q}2l)$4hWL*kvxyH&(4YaD!gwI;*50YO|=Na%|
z@p5vmwWrrSXoG=S<~+}!B#6iotLvA*OF)>I>hAK#N=!Lj>Oj<cM->H>M5ONcrw#$T
zc4C%z1m+oGBaK|+A?KaltMnF^i`$U{{yej@q>){Ar9=l7{*AMqFo6In^gVd}<49Un
z(w8)pQe$EPD>s+N8K$GZ{Sssb4>KgAB)pO48Yg>W9@|?TV_5{688bY?2)PCgCu8fw
zcDza2nhaa*;xKpAXPpYi!|0rQ|FUX#?;c=4Y*yu`#omN7(mLlfl5-ueXgKfDsUFjt
zKG4fxP>QP~g-eysw>{k~y}<PIGeV|@Jw~q2MXZct*{oDTsK!?+cC*TSyhyjyHZ4L7
zQWp}SMb}sj|Lcu7WFF#ro>`Ly#T~BdvHawjVn8eeEwy?^4Ou#{@Jp_+36kFa!eXn#
zu|geNX}Tg%9r4`C8BUL|C6XOx(i5a>)5)aM09VVuU`AH?v8=Itc^h^fGmd9g=*98|
zg<N6vzV9w=`7LRM%j@BVb_S#+Zc={cW<8yl@N*G>qnCG+j!hwgruRmRF=B~xWLL>o
zRQRCZe!j<BT5jGsL$0pE{)$zkO*HyHZ|X=bQR9}66CKYpM$OXf`&xonLF%-6P^mp;
zE2Ki`wRA|;Je-HZ!Mar{U)pzcRLfmw!Gx92>i5Hi$}FU9o79$A4m=3s>32^v-aEbp
zho@iK#iE-D*%q0a^(K|5K=E+Y*1GF>;=@m8NqCN;isRh=PdeY%ZJQTkqQ>-hnc^=@
zI>xWAxu`!7&Jz)6(bB3seU@|ZarN|@@0|D=5kQJthmS!XL$g0O*%`hwWg=+YmAJ2j
zn<s#BAwefOn>zlJTC+5!Zqgtf^w<^%kfjac!O830o~bTv>$t`eTYhZWrMo!&(2}(E
z`Wtw%*>6P}TxMgr(Ef~iF^pgeDiem#_8oGkkZ`o85~oCBh2x58Xkp+I&0c9U=y@H*
zpm8}2h&rE7R?NU7PJH>adz116Tki{6(DgkdjE*SU{!{VLC~TL$pPzGS;{LGWhM#or
zu;m8}i=3UDrKem(f|NDUr&XlKeoE^bpQ<M;T%16hojk$!$5XRetB81OvIzTt*7tdF
zYGVB=@BE-DTc<^6*3O#v8B-xP`Q*FW>p2&@&|7Ktd%s{$E$1?es_r|?LZ^}EEDOgY
zXlO1qJ^OowNM*DJ9U0V8+7rc5+L!&e78<dP46X^|8P1Xg8tV4=&N9lxfk&QvJ^Skl
z`Pd^r`EydV!$-`=CgCksO>47x|7ZqrFck_1qVh0*BNjn*KOq*&SF_`_yVw`SxGfox
z(EM3*T=Q+GdToj+?K)f;jgRg{tj&$TU&VIimkQ+f8E%P=#mIMF=Vu$e95KOj4yCG3
z8HcqKm*5b_?3g#|2_s)-l0!3A2H<=e>5|Y#(L7k`;CH+5o({TlwwT|Yc}n%%*KQ^P
z6^kxapr9WQxoXpU+%z9nkJ&D`0sr-%ausVPvOCVT{q8tbra9kTd3xu)Wz$@m>2^51
z8~PmYA<5u5v&Po}Hsya<H1!GMnM{6Ylb4S(BL$A2Q0l*0@{vE4wHGAH^QK`ApLViU
zP3?mEAkNY&GgDAnn`ClVk}z(&{3jgmRr`9F60wL2LoT#?4LV&O)WSK%-e{Bwf%48y
zjk>>DU*(XdS*g}q5pugIod7YVJjFC-Tv|`<c0p|q`+B^7rc}&Q!G8IeS~tGy#iF4P
zft}b3WjgB@R)^D?HoYvfc21{TigvZ(*am(W+U`fBiSNx;VLhTR%u*<NO@fY6DF|dU
zt_2)E<S5+IM~M)N<lm2|_q!!IjPTp3=-Li{$>=lmVD+zwJlHCbr+$NWoS9!{7rBeR
zc5T^cH;bLSDw%tU%`7|b6c%GY>QIn6mP!!GD7f#`3#*_Za@IIW<f4OeJso<dKRG_w
z@3z{GMXymrp;SOOBg&IS^Gn2mpP2Hij0<wu%dyEOlwX|e%vmSXlJI-$sf}C1nl5j)
z7M<Qxy!0JRR>GSxZ5Ez-^;~K0SGrFJ)qYtJf!v+JxICX`$>x*pq1?PpLg<|R^Oj{}
zvAYl5i8a$#B2P_CUS-MHR~Ux?HNb0ALH!->NMsW^x#d0&SB+}P<;N#grJlzPm#7c_
zfrqrNGj#vfAuyL4zfV2r=p<(hn$dL7dLTAmvpH9Z&U9AZgyVi7p*Q^6f-(2j+{Wg_
z;2bM|4gT*)xr-mtobS|@T278f^udqY>S5+U#dIZYlT4rs%)wNW(-2+gw*$J6V<vaI
zAt#^9Ia|6s<cwTV+`FV49Pcuk;}Z?vuDJl0bKePPpnxUH(ip79p3fIhvD<C@+%87~
zDuQxePbJ@5l<P=q#ni{n(z4;^lhWsLXf;19#*nWuiP59)f{#+KqPSOuH*J1pfI}y>
zX_k`FlTl|(ZRB|?rcNBl7eGYG*RMH}7W5|BXSjL0%ywBo6jrZH!=9`2z|pbrP&){h
zbCCP^S^w~UMuQohal`c$&2ZW{PjP+Hg{{&~P0HZipNvR_YE}!Pn8YAHJct?%RM%Y(
z>8@Rs27UQ>#=9(Ex=4TRvvzlQRvNP(JGN!_xvzSrZLnj{jIdPGv6s8QZmlJYxA)aY
zVboaJ?b+Gq#)<{P0dB+^CimNm770Q1ORl9|S8;d6VFq(KFZ&OkR(zF)5^X$atlO}T
zQGCTceZGie>uic=zmnNq1JBqy$r5vOeXcTt;(qht6O?g<d4aurVfl<*oJ{}@$>klB
zdhh%cEHDU~N;JuTBGu@vT(%Nl^}_nxD%rmb+D-Gm5u^*0OuaiBy-kqqMfbfNV5Lwz
zH<oxRGH+6ILObhIGjFG_-9MrKRM`n!)lIk^_<H_LL0qdch+{N+AgZ-GY@Y5-yvw;&
z34z`Ht726xS3RO}(G6?OnwT&PXHq+MUcvHlmN9+>{=1ge3PbZG^O@A=w{a(zY^yAR
z0%_FO9W{s0Tq^#R;$|`s?I>LSKCbl_R^|$&b}Fy&R_pnkqBfS9a5<8*$q~23vvT%3
zt}I;V<Bt_Fw|ErCQcP$zi|Dm+RQok^cGNW$>;^aCq1P`HbLjC!iAsa76Tcjmmiq4<
znyyZcbNpsaaILeQuK8wvy1?xb?e59ei2SL%C#^2~E(z-YNon@gwEM|UpQXQzKyJHO
zPtLWQ@1ZSO(e6c^cQcDrv7;yr*riDR7<H;)P~O6IdO7cO%3R3JJsG|=#A{k_J^Wj<
zx?<*zkyq7&xeGinG{q*{6;5{2L6d2DbbUr}tk;|d?)hE;e3@H`ezY^|F#*%%T6gbk
zg~==&=f2Syu5*eyC)dEoq|yRrZo7$T>2Z51{m;MJQMc-QOt+((E4c`!^zNobJG6D%
z@y})1KA<*nT2@6&qr$rKB%QhrUf|+&2%{F4H@d}qbG2y`u8etUeop1cmj~rlk-O&X
z6Hu3uoml|)eLVR<Vu~TKmvLQt?Mu>h#r8rC=kQZvqR-Y$u^02@f<esZTCPFe!l==a
zs`a@}h5S_`rW>b5L^esa`v<5t1FFia=5#kjTT4cU*tVr>X$Phr$&m<)%!@b8hAo#_
zRpPaTEN!;Asi=6`MiY1Hv8Y*%YilgP>0KsPX=0D_W_ms|-7Wr27he`!!#Y=)A3_<x
z@n>W4HY9D|dKubS^u)^Ypii?muTq=Me;QgWpL>daZg6)Q^x3zd=Sr=7#cb|>3m45D
z)V(iS#}3W-ThnUeYBvBbRZD-_e74%SyveiRUV2nX?)=@=jD|SsxX4{@loLW$aRds_
zI$r6)l4SLcZIDId#=HpX7E%Yz)kXcL#RA)N=t}v@z|Y@UF+F*NZD+OhYnz9Q!(W}K
z>64l~dwM$?Z#R9scTx7`wl}9-waSSY_L<@W%{;g4d$z(P`2<TBKl|nCj^4Wb>F5u#
zJrWd6xICyPFYp$34;wCF?2E>nBLjPL82LBSST*|scT>M;ekHOLlN`G5y&H)y>fN$<
zR&84`F^TUuCoQs?;`%m+WgR0yi7{(ZDR@!GX|b4~^b)BcZvg5XZ-b}v>$yQ|@2Q}h
znWE#2B5G_zt+~eF+FZ<;CvMrT<&%tT3HmS#9P@S*j=!rcD~UM;GyYvF%U=>)7A#*r
z?Q?UEG%@dplBCIb=hG#e;i1gE(iLA;(QCH^TcWo1L90z0)1)`0?loP)|BYeYf9HBx
z?uZk(iXr1zVXG2MF@rPmfllvadR==Ki?Q+Mrj&X#!TG)Hy;du&ah3Na)@QT6s%7EQ
z0X@kJRRe8x>(!|e&VFb=r1TG3Xuj0%y-i6{?s`}Z17v@HDV)l=(mxgF|H(hjy_Q<K
zl_a;^ovr?yqWnU|YVeOHCMP$aQqm7c<2?9!&fnJ=u2ff_kF-t&wP(Fu?iMXIzisGN
z)_fd#stYUBS9d*H7m2B+ifRToy%3^pbmxu5dOYbiMP;roE`56StiGlG{GJ*u8H@2_
zlKo%VH;$u%$#-cMjJx+W0)ut!QUlQhj=uRmlO-yS5#>7z?Z-9E0W8r`--e0_jnBW;
zo7<tX4u|svXuO<|<C7Gh9!%~6<tB>pSOL`DmtT$dfIB<w(#P!0Znd**e(9nlZ}wB$
zd_dMaP*$DDK45ZpBjqk_aDtknki)0uY=rz{Fa51fA0F>obT`!+EMl3p4UArRFwwHe
zzU^&gaWngKyUgRVNiCmfxA?COhAir_UvDW4cf?)3-;6Bvh;V91Qu2OSrZJB9whG<1
z;^YZ4mG5Vfr78WCrP%xV`aHE!KO@kQXr-MDsqo{4RwiCj)4dDL#c6L;Qmr|51%jjQ
zP=bhN>Sgo#T8!nB^Yg>bYs**M3a8@w!lyiT(f83)W_aErrto3K&kuF-uc+#PB8z@r
zeFm!&F=7zqs*|nf724&Zh#?nKs((NK=4lmf<Ndce5>DLm&k2>M0)~INhdrA-Ycxh&
zL`=GGgP3cH*lx~QernTmE<&-r&g^0=mmjS7iof4@`15i&Y0YY0^yj$g%!^s&(GtaZ
zHic@wmy{zc>7j3aZ1CJ&?2bp1b<7ap`>$=@aebYa4y>{Mcp!A_5*7PfvEXp;?);L<
zuvfaGG~d(0b{@4%OLyq(R8U4dkrUrK)Txlj{<kw}M>>W_bo!^bNK2(|t}g#fO}5@*
z)>UDXKeQ3pFnSW&$aDtutHP4(m^-3V&?Yc?3NacHU8e!E`qgFXj{m2<w~UIiYyXC6
z2`Q0~wvg@+=}Q3xl<pV>X@(AoAw^Ic1woKhdZ?kh1f*f;M!Jz2V2EdrKL7Q8c(42Y
z^sZ;EXRX^0Ea9AU&)!G=j^94cb9;IuoLx%#T=9f+nsY!K`d?KDZW5wqud*$TE^2t~
zQX?*wd^>Wqd+REcEx(^`?IwM<6FawwC^E4iMi$&Enwn$VthN-(+wxgTs5N3Xc+vP=
zSfSk4Z!PgfRiLMG5vV1S|EHEvX%in|S%K+B@{^B$*(bI-`Nq(F=lFI<do4;NY1g|d
zv9%)h_p_jPQ5%h6-c?0>t4hS*)+`XPM!A*P%&Uc(Lw0~BA9*ziZ0ybS9DYu5C+8D>
z{)`7-3l{>{B}QIzsIzIGn7pz9uDlUtO#W7Ec5%LrAqt|v1IiOavq+zVAFKBJV-O#U
z+2X~jp3)i3^@pz4SEWReJxAWQw;JuuAR_)__Z4@y4Khz84_{h*#s=>xin)HVItL94
zysG-roCh|W@>M0R-t8P1j@g1RVPCuFN-G&^LzyXg81z!>%6OYjB&v3CNLaoHC1V!u
zIYH0vAM=D&VM=|>%p;Qd7qi;MVJqs(QY?-jz0k|C%)amf{Q|)2M4}*mKZ<_Y7#1N;
z!JbompndRqfGsb-?pNnf@A2>St-<TRGLN0n>G38T1ld}#xi)m{><b@l80<`ovcHna
z^s3rajD-}|j*G+2nkD2?h;h~@b(Pu5Rl<~_=0y9ZU3ZA&(w=vB>cB^jB`>EtZ5#Pn
z%1zaqxX)3AO?^8Z*ImZs-*`$H)xR1qSuvlgPbG*uc!s(LTOSxFTd*DLS6u|4=WfJe
z=PywZ{RR8H%mN3*zQXFN7h{EHuygRg2HwuwN}F%&QpZP*Y-e=6PGzn(FBm*=R-%Wj
zI7sYl&Z4gr4yT$WQBBaDlp-3sjj@NExhhiK@ktGK`kgvB>S=oB)ivf?_>p_$Q8|+I
zg4;o2*<X+zyRW)~C+tOZU$qgWxbAB4KjFwL&%zmbFh$H&)0jUJjWbKr(No4lRGz}=
zlk=lEaO!(3C;J*YCf0!4#`A(uQ@0fTRXS@pkMxV!_RnW4Thfi?4TB#ZMa}A^UU=Mz
zQ&l(Sx4NaVkE73Xz|I%jSiw1g13c$@!KXi-bM61}oFSaN={#m{pZ&T1qk<L%rSsP)
zbSr8pQ|Y^+eO-7`wm{JZ>Nl>A!!l<+v&_!Id}Zjaugr9f`Nl``3IFc+9aL1q>Sp41
zU%$H3mXEwm89VJ|q*4&^*Gm!d#&dITj%8g|?>`QNartBtTf#yh+g{f>ld}td9vVJv
ztp8NT16{p~_8C>3>X$y)tnS8c&f}?fh8Abm<$m}J=5J){i1CywsH2{qnVFbQ#Ld{g
z*Pd=A!WW&a*qsj)-Fb&X6Q}n1bo7|9=&dPe@NxvA3nLgeu_N1Q266YCpTQp;M(Yi}
zmRbt+QiJf4%Rt1Cx4j*HNi%z0_9~EHB?;x;?+<a9usVf&=Ck=Hrzw`1HEtvFdkH!t
zw6-sbS?5Q@#L#$MH#|G!A*cV%n?pw>Rw3qL24zo{)7=jDl*oOTG=7#-#HpHPb3`9b
zgzVPrv`q4rXqjM@mUy<qvm3>kb}dC&a}CICjCFpxrKSCPa6CEYb--I3O&aP-Z1cSX
z%4Jh?so++Zw_TqgC399ArR>F@Jt}5kQZ72lGm^Ws?{q4>dI+kvm1)TS*<=4}wtjkA
zXyeC012wccBt7UN<~Z7_6lCPw$V6mGdAGli(dSQ>Rl{eCozLUlc$i0MO9|4rRfz3!
z2$H(EFVg5Po03XqQr5498Z4jCeC$%5V;XLGJ*UYlcu=YTjUUYh)1i%{&+qYgupy+0
zWi!>#Q5!F(Zd_K6I_qBP`iQ26XMb_n*w998J3j+31pyB>L6lD|6@J^*Hx`|2XLS0#
zy{uF8=F%@)(zI_}=+Oyv-|KblF~RIeKQ_ttC=TL30RQRlJ6s5kuCO6leRjTg+m+S%
z!(%RExW*ox!FOPzdQEYtYssF~Tr4E32jWcFU16@t6G}(;!^<@_CVZcV%_6<})Q5!n
zTgBGq8#Hatts-D>S5OqsIK(&y(DCH+z7;vPuQJW&OwTpl>e_^wrt;Wt8ML>#ITWNt
za2lSb_zJ)1nc|Z%n@v0e=iKJ*qUrlR^d;<7UTNr}h*#FcGgJ*sj5kmlGUJzQle4O1
zCKxpI@`0x&_1}s?TfBT^CZ|TDdSm#pN=IBxl`mc9vBKaYu0;Q;y_ffO$qu_$9o@HY
z`^$FIJk1=in!qWq^^nxF^;|7Xms?APD&K_V>K4~WK%J@+Nc-M(Bq*H`j%++Ji|VaZ
zGt?byx(ZdYsfGAH>L%#3tlA~opeY?HY(B(&j_q+FtHEVfZf9+!;Ww10l}=1dtR>=6
zoSTkr-5;Usv`JL-RR6jOAJ*aEQ(C8+n&AUJe|v?EN7gf@jQ{lRUXp*{Bb$iQ6KAdo
z69Zoc$lz2?SZQR1VnPnFp>JY~F205E^|zc~^L6jwyZ?&i9LQJ10kS2ok2|Nac^_xH
z7#X#`PWT59K3g@a$bnngeWI$^-eRUWy5;p<->+8c;8+zVKI_`|TLC*sBWATO?!+Gp
zN1GN1A?HkPbe(MPCH#ddcStZ6&U>Q2mar9j#|)1mM-yoIkyHtz&T|qUt6FyEveqy5
zM%@R@R|64IOm4E-M?3Z*d~c=;E>KJ}bffCs{G<UZH=btJu~Hz7Ec4&(tD{sO92s!Y
z2`j_uZ0l~yhmUNB*c_fU+!GP)5QRm;(=*L<DLV^g0YGwh`zYniTyTF=d54fy&F^oB
z+}Jf(38j!_)d%x<4g1Ly&`!9PxY%qwMPKeZM)$Fn+0$ZnJf6OAVS%AN!8U|(Jw<g#
zV9#z!v){kn?t>dej@}<yQRPQ=W&_ReoBLy=QeBdwmtA$(9d4g&IIH$+7!p(J(Ftyq
z)8!3h$(eF#k@!RG!W#)ic_HmV<$3!ECm;p6fP#3>PqZR$N=m!I+_b@D{#-G|FJzqV
zmwdA=Z{JYEI9er0uW7jWI*-dts#0rc{Bwvgf4bbvmul)$@&JOrfZcljS#ISIzl&vB
ziHBp}D+~<56?f;p;M17&l-bwn<y}mUC2@2F$(G}RN#wcN9W{U>OGePQR!%5n)2jcL
z7{aKyC%#}7yEva{k9^;WHyF^ong2r41Eo#2=%7O9tntSi`D!CwKI5wN8<PvW^dj@D
zq^T&Z?EgARFKo1v92+bu79;dHGWd>IcTjna%nwFZ9717)>>o^L(GcB9hK;vj;=iA<
zqbhJs?-~w5CdInp^q0#4GHNdAbc)-Q#;EG+>4A~USaXaMVfnp31YK5qA>lDK7@hrg
zv_iVN=|PW8AQt)P7yphtLe}VKf5n(sH!W_3M`0BR$ONiAJ<P{DqyMIr>&9w~oH&HT
zm6*A~NOX4B-W2Odc~^vE;goYk{~@kKi*%3avGMXA$a<>(y+Sw)mV#lJ3~KI->*4`P
zy~W)AzE>!p2v8=rIL!HX{AJnxO7Hc)^J*HUY#I@e-t#6iZSZl{$2iuQ{43}x=ad5k
zJBI^G&m$ha`pm^<gQKkpK`^1LzjB$ZGnudjvP%I||9gd1P5JNLHtQc=LMUai0&ZY#
zzc1Qfz(xq%is((j2P~Y#n}32p%+KzxK?+y_Y=mki!q`#Tn2%p4`s27i2k?KSgK5G@
zzNuRcSH3)ojJ#$$g0RHcThd=&usQHDF9K8d0#pBc#b1`^@7=lIxdLJGSlD<<APiW2
z8ka<TVQmU3XPYkp(tNGKns3g$EYHU0K?WrEg;myb5KmGE^KrqyYGZp$dou(~^N0td
zwYk6%iAT%@+Wjd!cl~9de^WzF?`B**nED2hnu*q*D?laxKPX*3jI{0bnxqR5JP1j*
zXy=;^+WD}^Sg`TvxdD0eonTjsm`ly%n|}`*P;4O{JKx5Jt200Nw5;i0!QYU8y$1I2
z=l*~Dp-;<7@$rbue^C)Em_qhne+0DQ5w_h^P;URX0si;jqTR4@1oRtm9{T>LQNZv|
zRs3aDOw0Z|p?{b2e<k!^8~U%8{c8yN6Px}UhyEL>|C`JHlYjg-x&Ge@KX@K2=4xht
zxj@a$ahDHkSfs*^1q|)0&ATV6AI*IXz`ge6GsJhIM{Le#<!rO4sB7>=0)MmQ(XeLB
z)XsQG=K5F>WJ!|k4nT<iFMu|n6+jE|8B|GrIC!`=*ji9I>-D@h@`y6$8A!Dvo|E(y
z$S9f1s`c6WT0`rBG`W>wO;xBK0r!km!2Ni=mY!sBWI@%h4;GVU))9T#99+z<19BE%
zq!|62J>-$xKNw@QO2CR^q+j_l1ioFChuE>G=m(-*`YvWVWx@gE(kD&YWLL|dA=8D=
z69s=1)E`gsK*c6l@NQtB=R>C>vFIb&aQj0R-;;E8wWX`e-75{L+dQ`^T#iO`lalOf
zICiUc8_@Q5X>kNRxqUFuiIb+&Yr;U_CJAQEBHT^&4ewN$Ya_uIe!I7Kx|*7l>U+lF
zU<;ir6L9Ghx125G_DcXq9Ia0vEM*-B<!KWeU}%C=&mdVH0Bm2+t5}|1YwL`;+pDg~
zWbS);YMIG!MZjPey>ZKdC?FvR*M1}~?sTVim$2>00zWBOagHo&w|V}XJpG^Pfk@=M
zycV19>GJ0+<63u8lh4!uiTmPLjp-KsV|YWC8g+J0>n9a9-eL0kpdw3e&<Qg2@->a2
zod7cZ@ii2q4LFxD3x+=S?0&u8N-lH$a#YWZbPV%=Ag^@_>{uVMmJw$0-rr2Vi3ps7
z&=efK%tCQ>iE8X6dXPNdxBr-2>b2;kRi;+|!+->>>Iphv0?}V&<GV*~be*ksKGscG
zBqd&Be$&KVHt>&v2>mmlBY)a?6G%NV^pX*rT8cVbiM_`HO46I5#BDcug;3KjNf*ac
z{iM{zdo3iieSK}KM!c3L-rFLG7Gf?s{Jf`xuhjvHC+PEv_miavec#YzO-Z%gqJ^m}
zHL$xzH?*B|4jHv;jK&I$BhGS7UIySB<tHeOnTZ3r`C;!IO>b^qajp`AQY{x}LAHm<
zWBS!fq9rrxt8;t!t(95tT@lorpI`BNUgMgCXCK)T@oA0t6WWu`sMH2Hw$mv$R7<3z
zLS<E$zK(^k_>M>5@3%8JBmkw)6;!W$u}Fee2+a8$L|`mhVn{BCpqkJpw^c`au_H>%
zq5dfT^mhkK?~pPdRCLbA1(~nQC)B~{^2tCX0R>zusl=><zCO=A_X~4j?M{uw91JCL
z&?^QhHd<hiB0TN)1AWB$uPG6$Dc&`14_+i&m8o*J4VSrG9F2)xov+ID8oBgpSJj{;
zZHC#$s1v29ox6EfN;;Tbkm#dL%SUb-OMrP_D(!o;c@pJWRO6Ia{jK6^CCQ=2$)e%n
z*ifu#yo;;I&V3H%fs%Ik9!MU{W4h$jFNLbIANn*V*v)O`KI>_T_}$JxTX5Q;I5GrV
zj?<lf8$>QWp2H^p%4`4UgNRFx?Z&IiopV}Vs>|td+3=bS&$IRH8QS;j$ZQEJ5rK<M
z$d!$bzBx5w%)~oxdvtl!(7|nWXt!TOnP<6>2bLmY8($4H-<x05uu&-Y9O;(58S@t7
z%J;CD`+hqX-ZzfgBEccNceklN#TzE5OH&Q%aw(#>xB^$`xvLK;p1B@0fD%4gjGB4H
z;@E_3(UTx>^rWdo5Bn=biVu0fb>V$^x;ldpbfh38p$H|i9o|Dr!_J<}c8WY4CzIHs
zxISvaNs-z7qpX0yIZBCGkDWCm<mCI%&j+b?FJvxU4>pS$X?U{O63*l_=_r1aa1J>B
z{y|$iA9ef~I}J7K-QQ1Uax`$2Xqh75ciIgJw7%@EJLt2$xs-Fy7<egf4pn(Ur0!Ix
z^5pR?%j8$XW0pq5*GykLTE9mds^OT=zg|V{AF5QlRT@@(x{@@*jbA?Oru0gKN*BN3
z)2!zz6B5`+i~PQ{m?%Z2e;AIb(=+TQmwu@+{Gea)`felI#%VwLbt5RK)u`B190QVb
zT(Z;Xh%hnZoqtz7h-N?4ex`eJiofko?>YrG*3DjAw0k}8q-4f@DiK~W$yq@#QD%+o
zfuUwrORc9W;HMojmy~^iZ;Axvm(yc89dky)R59Vu#Q*A(<_iIc&+p*;HU8I~6pea&
zHDLV=q*Y4)U<I5}&zoFLRYMDjp$lBW-t|t@clqiMH8SUfy|+q27-;2Y4svwrIB$bz
zVvyX8=lg}{n&MBveUmrXX+sAGfgy|Rc!td82O=7f?NYQzLC()M)q!O)?`Tcl$T@_?
za*ADD?E3M2*?F*bQeGG?+~%M19xMT>(EqS<#@$fY$uO?Zp=yCZS6#1PNCXB_+Hd`(
z`n-&4g9r78L&}9X#VPRIjLPH3jZ!skoAH%C)7LI6284REQ@pzHjit}`9v_V8LaJ%h
zf{hu;nH~)@y7UOd=&x6gQR^Bl><>769EBU!nrSb)ZByk&;%|M%)`%c6f*p-`DQ#a)
z_|mw5$DKPT2)+Br-o_!GF9doTfM!F}s0xR<hUU`aSAKoIL_HIugyIradi=04V~^g6
zkozT`;S6HDdZV+fE=hK!&+NfEx`9S)nh(Ku3RpUr#CX0%HA^2lZjRFki+exrg@a~6
zIp#++635D~nMECArW{*IXJi;25{|A<SG(}f^jC@<cS;Hm66j6WRQ$BE596tD%-V!k
zn(Tl`;u4+h0}YECm!ttItTQj46MY(sfHb*?KBekGx*D=SSur#0!Rwx2@Lm$E2v#c&
zhwa_0R(X>F<=l)P`z?HLB1GL@ib1U#T7NVuK6S=cc|2*m>N>Mdcc>~Y>Q9{nA02cn
z*(&X>SzboBQK>H3DW|fMPu<~s<G_;kW97$n47K+x0GC{nzl!$MJK03LA-CW+QYj8D
zdXj>G>;#!#Jh~hqxNtdEd@|8m+<ac=4TnIAY936{d@k`kDIFnr<gA(fqL*u|^>kc`
zOk5(`={5LX7Vql|l83W8&Y?Z6s>8o0X>v<Wp*8DS;%Kujz7*8i@sz`=-GGeV7=iiz
z%4REYoX-du#~Hzj+LC0rY;<Kfbe+BcQnjm|y4W>ar*ewP?_5pwmr<G>^_S#fD?XOC
zPp>UU`bwm@Pg<9lpS=oU8cj}O;{)81+`NMy+Hg<I-mtCBe(?D14Q@-8Aw^zYzD6cj
z$IDaoVU#Tc|9eSanG(ON%RZ%wS7NE3N$sjvy)O}dOj2HD{de9-o$s~cdNdj><in?_
z(dbsoZ*z3pQ<Z#)pHm>bHES8-7wrrJrf(r4yV|E~D8mKmHAIR48H==}gQxWHXMY>>
zXdAsI_e5&R=z$<RLtC+3D`ua)=49yY8|MKXQruvKut0Oyj|k%@_^0Dfp@km1v(4vK
zIFBLxP%g^KWE|F->bJ<^SFM4PH}&2tUA~HGy#4jfulM56xd-@M=aN!{2xIqC{g|SP
zA4<h_P%`>zmFYH+EM+Wf(Onnk2mPL5<5nI!AoP;&$LV6ink%ziL@{ghcmGiUZK#(r
z{l@^3)c>TdOx!kZfG0j>Z0LC1*AhGp_Z0#63N%x1Dl{rxMB#N$@lH;A<8RraT<x{5
z(xCHA0n4SP(g0fwY~g4*cRqj?Dv@!bTjaR$XQ8&!(?F}K;9O;6HQpSR_sSJIPQ}TA
zXAAQ&U0aSnBDjj`O4%6<NyLjx-|;!ztouzZa-P5atK1642Jqqqp<_|M%;)LN1F0{Y
znhz-!SsEtnD_km2A=Bw0%wG^9B|*;5$rqHgY3vGNcK}eak`7S96<83oMD&V=O$Y1=
zl^7JV!h;wj-xQNPWoJ+K%VXS5UDc)!{oQSAEM-Zoi!)jmr(?ijSqmfKRLTFP|EiQH
zY});(M{8AyI#GMVK#nT4(FDqM+M*rQPn>~XK1!0K4u?8OSI68{tohJx0HIFF^!3I|
z92aAO9jcprf4JQk*<C?KoRYmkw+m9so=siRT@0G`6+HUUJNGbBa=)`PQ{AzhkBU0<
z6rhS3(#PH7D`q^FC@d@nr)N*(v>e5!ie9Np=hfRJZCCqjjSI;ih{(aY&fRZfeNd{I
zu)Z~;S2ojflh<|l&V9~cT&ld=Mw}w|DKt4DQcw4p&jgu1`>-lN%CvDS`G0ET2YVpQ
zCU5e_lCU@!!Kcd%+HP(eSq0kIpMYlDBQ4%wT!BZI%khT1w1rZ?Si--$jq3l8G7Eq*
zpBBaj*<hmuRk7*t_hA~g6ONnd-MqCu{hVIf{%xyN_P4$J#8>f<9+TGD8AlmxYa!|T
z8k@In0OnlL-&+cGR#tfI$f~d$VUX5v@+-_UX%2v%p#8AP0Z(Bc*|Z-dTJ)hM#!T1z
zR^1DxGZyp4h=(p^w~KL`e~aCtKrBW|5Kw<Rx|qEpP~-28)ZZ$toH@FN1Q5o%@fj+w
z9|HwV?mP4j2ysjC9r3JFKjkP0-4-gH$;3a+4|ID)Tv=FdoI~B`o?2eipo@h+(=UzQ
zs#EH#=)PoqnmKnrkMMb(%pc{v@IKzUHGSPxGfJsqZsKKPj;zW<H7eR3^@a{+?}Bbc
z@+p$q>!<TU<bn1BqsAUHIWvyPovOvimk_(g5AxkWicT{HZ)<O=A2e%l<boEg+{w}g
zEcUuVG;BX0K_lS3rmB=k=L60)HLd_UCWNn6=Mxe9uJ0|f;RY<hAB%XhzVzS2#N6lD
zoZ|O1e>NYdGY*?-p3;#upbyXh+~Ae>Y6)@ka_~tggmbwJ|8oq|UvIAMi#cLFxUKZX
zA+l=ZAd4#W=i)prP9o*JNq@aGJWPr)R9zKIotnj=@#1)1Y@zY;sHn_6VjDP?5Frq}
zy1dcV^=H(ot+^>&Ptl8ZDy$0FVS5J^z87cs?-smsy)=GT*tiWvf0K=Z=<=+ReDp=E
zq29%-i=owjMjqT)de!0i-r$}Xz;G`ULEXmXoXFr%C!p6JIYm=_oyyh_6avuH#x#hW
zh<>>8GLAq8lZVk>NTDyyV(gg@O6-9|llH8sjp*uX+{TM1&D4#_Ov%!G<glE36qyW}
z{f`$HJ}H52VfBEAp8HGF;2?w5GaD@~#tV`tvm3m0EojYWtSfo>g~XAoX+dPfMLV;V
zZL;AuJX6j0UFfNvHs5vl#o{j_Org0WNt4cx4M5gYHGd_tpIXj@WBU>ZJwj&vX<yFr
zlXb7UzH6L>eDK8zF+@HY(F*GhlT}H+j-<?S_*0J{Yg$?Z=q`>UmGVBV_$}r!-UFN9
z+Zg!`Ql2)tM8FxC{i-nn6q-no4fjG4J%Gp=Mbe8lWjo44w8w3pYHWJ>nPjy4j|$5)
z4nH)J*2_$%&(Nq^&g6`dzi&)+M*3~rddorM0+evM3ody{VU4#6>h`-z6g7A$GT8#8
zS<2l9S@+eiZf^bVrY5j!i81D(DsRX--Ki^6(Q3+y1?}}T&7dN@<vsmn5HtoFoLZ^z
zk+SVZN$iw!1SBXjd*17m;jEdB#SuVet9mpGvN*m!u(D!Z`M_1k?#Z0^9#a`}%o{GL
z|GaHbUt*r`t8(JJ*nr*%>zjU|?LWsS=CaZ=zAS7OC3Q>~f3*1o*2o?QN?(mfJ&*s?
zvJW)VBfMe1=Tz<oBsd1k;|TnCry`ejALHADkKN1n20U~kiWte-hF$Qj{dV%0LP0qU
zaaNT&(t7b2C)s1xv-+c|^n^C&=pM<X#}h5qmF>zovX$M6Df>&ueG`#z|Ngs$+KuGJ
zWi}(6*|7J@c{v4jqoC-ll20&c#lbVRIa$hu=wF2t23Rj@RLiQI$TjvCO>d5um>r0U
znYM+#gml3-R~2E+1;DaM^N;=aImS}G%~)u1y{V2S)=}DGO5#jTZIt38znC7mS``B1
z(tMtUXF6Qd>7Ym+M67vouH_P*09Y7;7*B_}T94i28pHt`afP+XW8nB^#MB0Rw=ni)
z0PKtHv)26(c;)OP_T6Eo$41{tBXn20e`sJc7?W_)PL)|xaTl>X%qt#@e!`k{+kiVv
z76x+Ms!8ffoY6B@w(~T;4G-rp=<W(oFu%fyBQPCCgkD9~g6et+`VC`cxNsfTmD=3~
zm;Sw$MIrQo+6g^qwF9p^38GiD05$Subl}+D!??%qt`lXU#pq|{;NIuQzkIJ**R&^$
zIunn^Y^VaiBu&*6X+7|3+hF731s&%(D>?n(9XFTa5sMl-sRm(hKN@tNNQ%z5B1<>9
zt^g_!PY(8mfcY7{i6fTfXa#Ut_&8jK+08IbdXwa1#<XuDkXK7DW9Hd=J79B;LR~<I
z(%A=U2|mwpa0G${eZc;w0cBe3W9K-=mKm(meZ+Kr1K%elP`?@tSR(d^$QjFpp-=Bn
zASrm_<<q3`s>WQ3R1cKcC_ZM1PW{6Q!{l|Fh=oS9eZ%RBrv$z4&f;edlmR$3_aZ>&
zF2w<$N2;8R>%JS@gH}vfh)4Wz11YH*>-ouqi4p4xa1SCJ4&t>O?TC~xPIi~$3Cq=+
zX@jC#>097^RYfvO^CIWxqr1x0y4zg2mw40q<quM|GI89m-!QXvt*);$OQf4J*BXqG
zygXUV@<#30pQXtKg)=cqJKSO@;D3<vCL_rLZ73}2l_b^}OYVN1+Ki>RO1>P9$sVyI
zAL2{oZo6O#(e(<?iW2yX5Alhj&@Uy%CDrTRJJs1Gi=S@tEIt_by%Vv$I|%bxOAqN$
zV1O=J#`%=b4AL=JId+ea3S^meM9Ozh1tKwg-Sf%-I<s0gE`xRjM8vkv4-5~dfM$v-
ztlL*IyLRA=u<O2Imh^3frvjwwgd~TCiRTCrzldWvKTwQsJ57td^YrDru@WAia5(i#
zQ07ZzZe|*t0sbR`M=u=u$kxvGp4l~7(F%VKgI<F6a2UIV9c%kIcJGIkkOymefEIis
z_Ol{X07QJJH=?_k-S|AhwOMmR!y4$65?MPL&aG;e<t`nG=x-%v8C7o3``64G_Y_yC
zQmld|6qajL0l@%Q7tempB~CZ>pd^(VZR1Cmpsi~x;^2GVd=r?QwViIVz0^#1s^>~#
zon#9Oy<r7-UwdKuvEO#-5Gtn7xb__)!e<U{C}vhm{vkQLR2OYyXt&Z^c_GMvjvg7;
z5%WYYN?XZbp4{rIWqb;n0j_YSHzR8OoQqUf4RmiUc1vE2B6rhTUbC#+ii0Rs$cI;h
z;**4SK?o2UY5}C8<4uxhYnk@DS`BNSPqj+nY~kDLZM~(7xYzm{gws>}B;JGp2r}oo
zHqZ<!<M9vQ&X@(jXk?dp3YJh3R}jA4&IQgf&e=|HhgCF3%%gozPGeJIDs0F4c!azT
zt?t@>Ki-~Vkn%RL{vJ1Nd6i1qNhh&Wg0{jHXeYZhZ@FAQ`I{qR`wO>&!ATG_(ZSBv
z7YQDX-CLTkhc1}fz3hZl#PtTQJ2|W*n30)rJ?ZGR@PZQ$L+m&V70VXhF%VF?F<6C4
zo_O!pi|MsXpRgOSN%E(`=GBE8V~K1iGju49cUiNYEa)eBLgy6j^O$atiaunPFIN7(
z=Wh%sF}`vue$|eQ#Q_-+qI1BZ1$i!rE#y>>zE?U9_|n}$V47i3YdvLO3oWjna%il#
z5@!e{TA{6YvQH9XtD0k2?)+166Uv%3onha=!yg12cqropo}+RSR8a)6;6^cID9Aen
zzC=;IxonV>;(N9($4swPDzLr~!a}xR3`paT!G_tfrP?#9XQllFeoFTFCGh7WJ74D7
zK@p^6pBbA-&y8Fa^Eq)8OZ$E`9c^!HMXC8-v>Vd^HN9CqmhGmEyu-$0T@fVfhR115
znQ8fY!@bK-xBhG+|G>QhsP6<9t`L?0;sqN&ALHia?q_%bGdI=P9fj^QfdSzsf$$UX
z%DN|_{wc^RfsHRJ#wzhj-P{P3;;%V6Lf=QtO)dxrMD!z4yRmTz@ccRfC+79yg6tg<
zMCT*~<the6h7%BKUVUnirIUQ)_%Lj~o>?J|zPq)ei*SG3CLiu>gQJ%q%sL{4nx>NA
z*FJeQcsqnKR_smUzJ`Zh&EP4F$L-_rJ|G~BiwkNP@`Z!_5+-!`<nwd;y&@PEQHzLv
zX)Xf=DggJfze?=ZA*^n>h0UYiYh6XD2hfNl9!|aACnvWlSmq>ZJGm(3$uYV*lb6up
zmuqI<PMsrn!`Do(+xAcMrektgDA%j<dH9sZ%_zA0;7=}72Dz*k@>6d0#c~>-;$dbE
z$1`&T%a&Ha%{WbjWq;qqq(if42B+8)C~{Q8eKTiEgJlRn*v@xYD6gF(YvXl1)1g=k
z2|_K!v{>2jlpm+N&D}heM(V+HUWKgN{YoO>S;4*8+9_tz^TMTiog1M<9@aL7`%7@h
zdI(_r@X)4)8(KB=-6M%u0SWXn>0WCTk$HgH+9`B!#eb;~$(U&9K4Bgt<2vJ$fg^C|
z&35^ib47RlOZ_QVzU-Gl1f^Os@f!zfJhyKOhuV5(%0@U)rbBb*<fW3w8muG}@(XJ?
zvtP!na7?nqwSW%GN>FR8VB$-%!Rxj3FbL$JYh+ye5|F|=eWI%_Q(6OZlzUOqXPr!+
z_mTcq7`AM%aI3hLEFhRzn+d_ZZzW@Q(D`zYdpqRP*ze*<-qR|O{Eby2cy?4MTdwvs
zyeY!2WhGM~;zQ44<c0fE{^dT@Wh~j?(FAXApr>!55{`hOU5Pw6Me1;|$z3cSVcb>i
z!c=}2Fja)3gKq>3bmZzPp~Db4Y8JJ9G$rjl&EjWcK}FBgmZ3;*vFFIl_LTRNELpb!
zCGw0&?dL*3Y*_?+DQz@!;RBh|a!SM0C;C-+^PH=l%u*4f$b7q_<>-dWClSAO-Z<v{
zSQ%6I!_*)DraUdycK3r6%_80n@pivCMhSPj!N^ezkSiV`Y<ugSP%;yBYD{F3t(uxb
zQmIwFT`?61!i1N_a(^lgr<dsXz_&TEhCBL6qEKhPy1jp1(`@9<_lnJhCwH_>f-N3R
z^y+|AG)nAk&e$I7rc(f&fXWwP4y!?mxTsY3HcG{|y!G2?47i<6JAV{hln0}aBgPd7
z-aPG}e8)Vj6+3)C9KyFT68T)?Q!mIxl-Dbk$9`io!A5&E16XCA7ZKmN$;J|@p1$c&
zCEd7}U-KYLxuVgF7LSCF#dod(YDinWj|A?)9{LM4Q$*_v1Hd56T@kr;!tWWsAStZM
zCe6ZxAf9ZwNtShL-df}&2XFB?!E7_%yDxh`WT+G)WFx4|sa%*wT$mR{UKd&3NxFee
zUQW6avCY>TBWD_(<0Te>rEDAC&#SuoP3pa@<6^DbLMi2ZD^x8jS!xJelk<L80BZI#
zbXngSx915~01XG&vpTe1Fo8^F-*zEt6MhnAPMISW;K<q@iM>VOJQn0F!K~63sd&Su
zf7pdwsx4~svSF6<X@2Ra6sc1`KrIt;`z)Kts5Vlz{ERGFkJk>Kb0n6TLzuPUAzGvg
z*F1t%HyXxe95fu{^2)y3|I)(Nqb@!dl4uO~T6wSJu2Rm*(pd1$ve-<c#?PH`GudmQ
z#LT241+tvqHQ@`BAj`^MgZqbb7Uv7raVl*l=z<LcgAK*6p^p+2ShL7pWRS`EZMnb9
znI|{!8+RX=nFa;ymOaiTw%0Ij7=M6)zEv`vpV?kfp3@}4Qq=>Ol#k`eWYDnuHId9W
zU1_iRj3i{0)8dC3#)DI^abQEVh=!uj>AKqxt+b|9^^-i_(%}q6u}(*6uOj<C*Sq0L
zXDxN@^tLzQnHJv~k_6mHvUqC`2iWttMEx{n_L^{Az}_fg^RZwZx!@2kbGZUXTp3>t
zWysiwyU2?OgKuw1%7FJx#<%yZgOb4{r3(`Lu_h`^iFh=)wgkC<S@$AEasn>sI4?^(
zfWTEyG*@?JTreB*#u13IQvvGwgx|85Pd%G`n5aZXTC_B;n%-5=qVp-ZozpPGpg&Ir
z+F0N_t`JwBheyGCa1>OfrNIm$a*hZ$ApQXu_lB*z(EH9W@76Dj<q$)q6?cwI?yP#T
z?Ag1gt;<-von06E3dy>?m>gU>&x1p#^^BD<^tWF}aFJJ*&QH4cM_O_QW;bXbwZU|3
zQI0l+p#EE;C1@=pUs#PhGtQ1b^qm7$0o82We7$c{nf=Toy*@t3NzI@F!!mfk6?K8`
zi2~}R#f@0TmXSW<nbIFV#Le}^NYUv*zf5ZF(Ws#@l~_c_(D*m20_K8Ek}?CY9C7o<
z9|K7?MpZo5v}dg2l)*+F*mv2X(Uk^psL}Qm`Dw+o2D@>;i%VQYL~DEMmXJjo!P|4+
zJkOkXWR$R5`4K=9X#M8=JL?<$h@o$M6C0H#y*Fe4xK`42!4@H1`_4K*r7B5zx|jQ_
z#}}fus=cZ^7T#{HuFT5hlKX8_Nn~jUq=ICESe>x1E#F+S=y5UhNm@L9&Ju;+F}>l>
z=GB}oqOTGjyl0ddqn|dUh4MtgyBMw(6$6s1d{fFJz~W>&4P{<uba6KFd5uKfA2HR$
z%GnEvZjVtT6#@L!a5fg|?Bt-UUV1j`wo40-sg}ARCuQh+jaT$I<+mw7&6UuS*H-(J
zEoo_($)UQda^WugGx1PfkMT+Uqa8%eHInSiDCtRfK5^&5x1ArKLU1_de9=&T<+vi)
z>;jE|f$U6ZZfRBOE`-`gz0%2xAS<0@_#gR$Ln9Gh>m;#h$H(?-1L=htm7MNi?|Y#d
z@E>9hgt+(~zhYBbb7Rl%gmG(>spc)g#$SbK57#>zU8*=t;9O<TAZY1{0%&eK?loUy
zK$~@LgC^007hfNhhhCUI_|w8!%JDMjq}`qBrJe0#GsCmQ5go(caaP$(emiASGnO(s
zhGDVeki-(6>k)9Kv9!9a6ZUq3*4~~(p$Mp(wVkr2S<;WsJ$Li$w55*~-<34GRr0y3
zRL6R@^_Ib(g~RVH9RIBNN8niTQwf|Wg4C)F9xFDZ<IaC(iF?2AF>eL6!>pjSZ?Q4;
zE!b6b5j@dy$+Q;hxeM($4X|xF;0@NUJ4fGP31OHRds{84ktx<OPgqsL{o9|CDXuL7
zPGPYXea$m}vg=yAdz^(`SbW`jobSMCsTmi8Vad!zJA%4CoQZ_Z-HjpIsWJ0V0!a`b
zzK!2;aT#hHaLetsENd3;3$qKxC&R57XX1|{@j&Sg%9lfYc0;6?Wf-PUVG;mHB#osv
z=KnBawUnh)-Koj;ao)oSsJVlks^F;*iS&fi$KLWWlg+G26UT5Uk9IL03xZdcm2Fd3
zmPEqW{)A`iZLtIx0PIol$xav?WJlv-x)ZuU3k08X+uvpkYC<sf-LIeqv;aH(_TZ%s
zP2={R`?Mz)9}SvvP<T_MUz0NWE6|8e^TY&Wz*J61p?LN?s~0Leku5JU$)0ey7|Yz2
zG*!}_3A2r;1_T(a#{CXlU{%xb-@lh?+4YGd3()WOza$TO%(m!w3%#w}%nEjTq)+B?
z%{nqgGZmx2ZudciVId#@ES5TBPH+7Nhk44fzUxktCt7>wH(~xck&NjPX1l%Gcg1j+
ztW@aFvIvJ0uw=4qw%(T1V&rrel<Z<S5!!!j>;oRV>i#p|iz8hG@`XgAp+mihpF&SI
zKnH!{EHbkZ=o<*ak3!r)T&)ip?=#nAbbR0!b7x(Of1|QR^;b3{HvwB9ax@_devOU<
z&Iq2EA;v5VH19w7VR-^dM1jTj2q6HEzhZW=d$%SKLBYcAD(K_w=au#|J=oV6UwR#`
zQT8m7;sgVlh*<tl;>Ku%a5x5ZrEILdc4o&+JsgZqN5}T3DMk3H|Nb}0&p1L5*il>}
z(9Co2R^+atrW=IkK)dghF+}2jZqR)SxRrm3Ut^5-&-eV>4~ozQ`u{ir=E98|p!J1<
zd*WYvb}+NYjEZMkMev^|0_^!MrY##!x#B-h1Pl{ESNQ+69$D7g!03pYV?pBix2633
z8xV^vrq$;^F6!xj#qw{m`L9_1)nxt;isgzte%mBs#Gcv^3;cT~|KdsEKQ9CS7cd3;
AqW}N^

literal 0
HcmV?d00001

diff --git a/docs/figures/arch_7.png b/docs/figures/arch_7.png
new file mode 100644
index 0000000000000000000000000000000000000000..c77805365e34413818691fb212e700357a380932
GIT binary patch
literal 57736
zcmeFZbyQW|-Y`lEk_rMMQX(KNBE2aoX`~wk>5}eNDG5oDl5RG=r9ry8QxVv7ckFL&
zpXZ$Oyx+a=|96ag7z6fNYtH%W`KvWIA<Bx<_&5)6P*70tWnN0CprD{rp`f4*Vc!Bz
zsGvEDC@45k3vqE}8F6tMWd}P`3u_Y;l$RlK30P{X>qLHQ3rA5yM0cWyJczT1MP=cr
zS*{ecx9`Y)^8fb8rHw=nm-&&FhLgHPOBG7F2CJ4921%W5l%$wG87>d&E?Usij^J*?
z;c7CX&vk7;gbN{rNaiB*8~82rcsh?+G=fj*0p;&rS$SF0&li4pI^S^IXb#S5h!o$x
zz5n`I4uXBx7WG+?-4tHu#px+ReGKD=o-hhgh<yP~LR#SdD{K5c`Vk1qoNb>igjeZV
zU&2#U*)lXvpQnB6nUJS_`E;K;BhT8TF5eVkistOQjX0rvy=`i?ayVMX_uTi>b{pE~
z`^H=3^$rfExKBF{0x(zUTHN@A1&{0a)_ubg3xd0svsv7ZSm|Dzkgdp;yB{J32lt!k
z8FhK$Tk&)&@hsjw&nc^o64b5`Ct+2@kGyaE`E{G`d#<-F_QrjK$8uE0sDmPEWxRqA
z>rbN+{(6$h5ZkT$xQ^&X9a;i{I-gk-Haf|TQ=K4CRu;uKgLfQA_HmyX5qIzz-lG=o
zsL|$3Ul#VksZ%2@&NF;Ma6maok;mt$xZ}D;Vo&0E_{=Df#B42mlH;CZhw&-{vn4V1
zHF}5luX{Jq5JO3J8JDadhF>VNO4n99pPa0|A#?Du%lbuOdhp^==lvfv*jjgj*0q9f
z5n!N^{h*~$;<G!82w1f4_R4=EL>JlKQY+u}EA+fNMvK6OKoO-?NYI0RVUB!VQX&Ro
zt2>OjbdHLiX;0yb*_<wo!n`=$hg;@SiN-fMm>_c~ZPU7qf$=Q>=cyRQQOkYS=35V&
zF(J)ZXfzm)N<Cs;Gwji#JBZ>qrr(Kw!5RBQU<xCy6y2pom=8NB{jOrO_IGT}boAOK
z;jVPD)K--GKw1p3m)MkVXl8;@jY75P2(T&kU!X-jef&V?g-Vo@#HX&P7jOAhLSt#;
zx)if+on!JxBulc>rF7|MQLIZF(4{Fqloe0Rqq6*{C^h{A3t!TBnA}`I0E<mDY>3+&
zQzx8Clwzm?u3m{7mpM2r_HZfo-B&*GWc-Nm!&b?qXnPAjCfmCu7@84<9pgX0+82MP
z;N#!H+@VViHe1qmqo~IczE2&rw!F4P_EU42W&d&y|LpLL;>`96<%Y#yafn`vc<#2|
zi$}C6PZN~opD=FGKBUv5ak}eqdpB4jgJC!sE{n&&aF;6tH@GllHu$)Wt4+~}&^}7y
zkwr|AOllT3QDm$%GlLB+B~1#0KI3t;a8z8^t+!YXH2jII3><U`F{d#~pP?~n(LqlN
z?x}_f(g!^H)|Xq9y_NG^wd9otleijp0snZ+1G)Yz_G}I1%{;O^!aTd!*w|08bFmq*
z60&l!%&`Pa`YIb57cW(FwTG|FM=hSsJeV<<A)nEop%6&1l{!sl96f|v!MTqdkED+X
zd+<zjO-jl|-OC)}90CuVR}z@-k*JbH^vK7d#W}>$F;A))<f*9jt6i(P7lf%5sznwq
zXi%yd=S>$%=B?xvYs^Ajw2PHevTSpxzOgHFs4QmtC^iK78{g-;y6@HD&|%wwDa8@f
z_1w%MF|ki&Rm@b-QB+*NRff*MI8`dHLk7QwL2*<*rZBO%M&qz_m91Zi_=w>h12$vP
zS7qLx7w=DXh^xFR-|Nup9P8-oq*ZFqpw4K|Y*tZJQEx0c7dd4(t!`LP)_glOdgPeV
zDRI*l9TLR2Fe>R>?3C$D2HzR68KoQ>ox~l9f!hoV=AOPVR9%!7QT5HdVY>D~|LC9K
z-}O4P#Ujv*6`wRHuD{@QiW&a>jR&_VdhlEyR8tU;CsMSNvvC#hdYH>|n^NSG+da4C
zq<YHvG|^JUT*b;^FxIGtr~m1^^^E1kLf;%~i*>ekrgmCQJpW{UbN$4XKj!?8S)&Na
zs&)mr4Eot;y$(qxK_r$d)^TstO`Q85RJoLqYTH%pBujkx%_9TJ_ZYFE_=?}ju$VKE
zGpjMm)8~?Cleyj7%@7mA2u+(4I4?}8(664it-CzgIH5i1-AdXv6x8D1qbzu6D=<Qd
zPDv)n!{6vo?m{GB<667i_bzp#ex#=TJS?m*Y%F^qdvUytyWP2P(}pL8vfH4)@we2w
zLucJBmmk}ibzz>Vmv2s+Al8s_$cg8X0%zKz<Z@5h$lTA2gO0hmb73oSzm8)P=4OYI
z*7Ucpr#h#kmnu5fm^!0ly7`;k*R9rAj)hh)7Y66;(ih2nlYOInLwr5`JVf<GO`0=B
zGn<c^XPW(K*1lninFZW#NpIN<v~xWE?q!SbO4G*u_D$EhvW~gDxe0vvY2i}~twS{@
z%dRneRqFRiAH!NH=x72c%4mPR$#Val++~;<{Oa>7^;b$6+nKnTy2kf8D~wJ$>%)%2
z^(j0>1|BJrP7*<hM1&d{ZoC70UuXwF>>YmnXe*Qvkjco(d8^{!b&&eI#R7GdM`1H%
z2h!knQB1zdwc7t3qH1P((ic;qX1Z@mDMOp}IBQGkyf17?{qUA*O}?%w)|t;$|J52A
zb&Hf((~znbgV)#9M2+}~gq_4j+6IwU!A!{Vhb|>GCXYDs<Qlp|Zw-lwg(AD@+tbJV
zzmsE=myMk5<Ik?Fucr@9U1f{t+lkH}p7JNH;=L5l_H_6>M_!}gE^p0e8Kd2$(N%lc
za8P(4(ifiQ`8IV1vR_|)u5qZ@@2hi1Ywr;43hgE!{zdDL><{y}{J5^{FG#c5Ew!!d
zznk0dMt`Hfj8{o8WxKXXoG&jou}ffNwPPP)(NP(zcw{bVQ|3@sd3GWj87Jt|jG(zl
z_l{dGU@L5l$CfRLor#N2@G<B&^j?v^q-OECs%EfJxYS9gnw`8m88SJqv)kZfn5d&I
zrakyEpW*>eIZwQ`SDnQ~PVb81iXe|FZ!b?T%Ma$S)nBZ&VJnU%QYM;lcIwv}^TmbL
zjf)IZF5Y#6C$*Fwg0ik&7cxhqDJf!t6b1bSy{>m%S9U$h1$|sM2e;C!PT*Ti^M@u-
zH>j84QR5+Z2{-qp^C`4g(#oXj8Y_ny&Ik7`%^%3IHES^SUZ>@IEO@k)5Nl{lNgy=m
z@ZLLXCvSbYxbuFe?1QL}=lJO1<lzT7SvdpCw4J(Ng}z2!TV8H15PxJWP_#_e^9=PU
zfRDjOF>|p7Uk=psCHYk6nrQT1JBF;;rbY@Z`p_Qz$PV)-8?18BQ8GNJcluR+xw5P|
zrTG9_t7l{GXe)S<bZFO!bh0Hi238yIHqCdEN8C4RPme2juk7^dyBplASn3N8HRg&1
zVY!R(i+AVN>yo@r#~ZFZ#ZMC;G>|9mc^9QupLa4h7TJ8E33XjcE|;z_p$K=crl7^c
ztH?PK;ZstfbP*<B%e~5+!Sr1T-`BpK6&8aYX>zHA4Nq=Vw+D+k*bJzyd)J`V{%!sX
znj@Nv22h{OYtvobTo_y6=fKE?m3j&f!vnF)*2c)K$l^32VIj(|mx)I;zrt1$R-x&J
z)Ly;kCkj7e4y2?|A65@lKKGmSM1{+sd?G;Es2oo8kcb*a_}vXav}HP;27gBxJ4cat
z!w2c0d`M-H@S3Ze-0x){)h2pgW~Nbie-mBBMJe1Efiqfyc6~=l(JrA`<gHT#qGZR{
z@_xhy24W0>hGlsx^rl{#5M;2SCR#G43JNI9;2IkR1N8w4Cb&Wczrv^#|Gt(&eS(7i
z=Q$b*N{|H##@};5^84o}>hJr%KGCB;qF{k1MBvx$8`{5Sqf>oD|JOCz5OSTEs<?~{
z_*6A^Ffp-pG`DkN*w;4#4{pDEsqKh@LQ0SPMU_!`ya#BSuu#)-(o&EYFt)Q{fAiMP
z$b{X^<{c6a3dBtST-um8y`gclv9@&-a1(m;XNCZ{Mn2|vMDu5gla<gTEd^y7aXSYS
z8Xoqi>`xyF<IvF1Kpfth3aCg({k<IA2|Y4*a(XAg!Qtxa%I?a=Zs%ae!O73h&++sb
z$FpZ_U<R9`yRFk3H#S>Gy1#(@3yy?|qp^d<J0}Y}TN)(XH%4~OPC}0!Arbxi=Px=<
z+${cs$=302wg7`1$Po@s_NN^Gh7FcNkWU4aE!<43wIwWUfXo06Vcus?A%EuoKSut8
z@gFO-{;`su@1HCGG4g+}RChFS5Vx}dD4m4=qprUf|8wx~g%A!T=l_tzUt<3A6v$Z^
z2g31hoeAUgk{o^mYJ6xRp`->rfsrA9P|d-gCx3k+uXTR+-LL_Li=xO#h^e`uZp~xG
z-fN%f*mv#b$9}?0^!R~j`+IW7?_V67gK5acFeM~kp^{*r-hH~49?b6<9;|(rjwbZm
z6B_1#-+e#4j^k$?&4Q+f1oK;)2KRT~?`#wpa-SY?&8w#nbB2}D_@SU-|ND={y)Ig}
ztXByccSTW9G4A~Jry1uCgzwJZH-8?}Y@=aBRuWgB|7!~JDUI7Zynh1%AVp^|v8gN=
zO|kw19-BsVZ|VL&K%@9q;@*LbDE1Nj6TIJ23+MkNi;TuCE9DF2Kf$B41RDQMBOwlC
zuNnPdLf5Xlm!pYwr8hxI4nKiSmmxRRqhQYGa(kpizpipC@CA}9X}_hPFZam);w};!
zkAg?uwtioVNxdMp{&c}}q}pb-a?JY2``aV=s*Fs!)kToQJ}!roX|r4tk$a&)Sa|^R
z^1m?O#-oC0-NL*7TD-F@gm~qkmqW>F+qPzfwB~ZBE?YkF*}?A^r5;Q&y{L)nSA1yL
z?=}6F?k+Uuu3%>=EQN72Mom|k^^RoAWs3#gau^cBk_jVa8>tq-Vb@#AP4kH+VbPe1
z+ZZoo^xP_wZp44|IaoeTc;2UaxA9VTyg>cXdoZc*HsokX9DpFJYn6#%$QE+qW7}M8
zYQm!wP?JqyXR~WO&q;Rs6*8cCB?+ijVG!MWNluSR>(^ZGe)xOF3K|zFeDQ=@N>MT5
znnYcxz;dD}akoybAyLG)5nnKlSv^`+k)K|@J*W5~J{p&4U(}*6LSMiM9#ws9-eQ()
z|FQmL#>$0Sn#w)P>uSIA`~>EHXqvB_9&o)Cqa^&9E`smT;%R;Nr{;r|-fWA}oN~)?
zW(r{{uf3MOt*I@^*0tM=$uy%m3Kb$ma6&8(%=ef;H7~PKTIk}w#(oeD#w6u1uvXGK
zhTg)zUa8lvFg+O8H0=KI0k^pCO%Nr6YOW#)K32W~d;Q)Ecc7(}qKbj&1OpGI@%G$$
zNJI8Yc`thfY}9(Tnr&sJlk`3X<mhm>31I-HroowYtF1!_3>e%bw4dF)fI6A;P44Z_
zyDnz0EXuqI*QqrBTxmHjs{u7&%ajU_-kGl>s}@+K1*W-!*#~{diZt>OG_zE9)3z%S
z<u*K(@cQeM+4CN*)1ukkqf5cvdagy+d=KLtIoRdX7CLpwoT}vR7{=L)uPv<>^O2-J
z7wfsw*HwhI&YSUeX{({qN1nux1MkkHKI|_ncy}F>Vzz_U(tL%v1Qklju21tjyA%bt
z@1283L`5yqNbKF1-+#Tip^gVKizXg296C-=IrNouSoA*a#^;?Y7586D^(?7B`xSPg
zQAjTr|FS)lBy$taCVf#@XFxmczLt`iwi%|7#4}yUaN&WtK22a&FM!L{!He~M5!Wzp
zp?6E7o=IJ=q$%kI_$IZTG-w(|GBn>9k)=qfbqbFasLKJ}b7(jDG{B+PhCb)nV<q~!
zNAZsvEMv^iR}u_=4~ifvH|ZbL9rW`qc74RB$nv?qI1$OkYlt)5nX5V31|p(ceLQHe
zf40|3n4-S5*l@a_+ZSTkI2i4|thYZT{FU1(UJYuXe{;4R3Fp{oZ}_!8Y3SQsx7%n1
zea&q#(zft5Qv;LDLl15vE|4S9dC|s^=5vuRWcroYj@@}<to!sB%x9zEwf*Gda`rPw
zm&JK}w&N(&&p<(N=lilmC*xzCvDx>pbr&60dO{|&b0gTyEp@oHOJDnM0n0fl2&^?X
z^Z~|la%M4Io^9ip;Z1)z?L4W^xZD{D^UgHR#3?WqyBzjaoq>s5?FCvdHtLU!okPFb
z5P9EhLYsP0J$Edc9`HM49c@i>4O@DB1Rf+-{@d@w4{X32`TJt_HNa0aRU93tIT`J4
zYs<q+H@_fqL%+n-cx`UgZkC+XUG6rCEUsO2ybpuBtV(b#o&xK(I1*%oeM9tx=rwL1
z^%E;W-b{*|Oc~SJ0Z*0%I61GLtp1Yt!DUKO<mqVSjC23J+}lmOVjVudvo^4m%LRGO
zuXfa!<EKA^s0U#B<#x#|RImjPtF})orOs|g!_t}Uq-&u;w_}HGQalQWhoz}0Ie$LV
z(b&SR8g~|S*}8vee^hmd+hlMZVkrNz%gn^~XGLeYV-cV8P*X@er*Vgb{P%v%k0n+C
zHf^8mWh!OfwgumHxjq~e5jkkVqLz?HnSSG0^_=nKHcV+%+<ELm-}`vHB$U&i-{^;!
z1T?w*D<4z=9#N0Dxh&}!vvKki8qfv-!e0&N@s016+Kzwu9Z{M+G$D_pQbuMhD!14E
zHsi#PxwOVhPRCT0R&)pX6P+j^&-cvFh7f(@n7Y*qkA8XO_(E$p3cM~I7Wi7kH?t7y
zx6k?4l8s^9=u0-*9?#57;SM|@H*CyqUkYhlF3*<s!A6tbH1%nLZNhIGSbHB=)u`N=
z*JL)}YcU@BCU%#rq;2PPDUdLNR9|h+ZKr1K(Sv7|C94H74eD~e_Afq>_fdEE_lL)?
z)jwf&<74a%2?%okDA^`YN5QFSw&lb9@GFbaNmWgc5ki1*a^NPpX|+K9sG1(%f7LQT
zh#-0Ux$+kljDd~eCdIv>I|ZHZcV_XfV5m+_*C$~+TsLRoq)~0S_!2WzTpg|IOsgA4
zl?F3}9)0spqY8H$R!XJ9ADjpZ<;<}TG_4s&@OSi^7KH)RyT>HD_ar))<c{dKC;q@c
zPp2+@35q1CEjYw2cb9c&JeGX`F~YSM>}b7B<xy^CD--f%HaMdlxHY@sm<@CSKAMY~
z!;Gw!)@cKTsqrW2CZFp^rX6MQkb;_&ZjJoXrF7b^ng?0N;zsBn4Qd=3>p(~~PBJrO
zB!(paNG_Bfka5~wXrNm?sS7qJ8*Usm=5QHiv0Xjkznu5IIZO+#(Pz1+s*z+jvb`){
zwAc$XOkPS~u?*%3EY2*1@Bgf<vYIRz9#q{oTIq=!79M5^v?X=K)vYYvpS7rQop;u3
zn~S)QP*30O=Mu6`bl#j`a}sKl0AX(q<S=ugWx{CKTQp0_S6}j-T+i4~(79rwh0<$)
ze>4_}c+2Gm=QuB8{G!-?ak5PzM6Oq18h%)Iy*arwoU25w>$~=Lp$4-6un03-5BZ*2
z+L_uWNaJDhqDf@Cax^(I!&j9Y;zVt_T5dkAxECxE-&etrac$Z$DlzG5fJZG1xo99?
zTlTU>2(*Z!OJ&R+_C0TPnN?GI{W>+Bs5pjPDH^Kyp<%k#*{ZGYjv1_B^{W+*?)bQC
zMDbTLCyJCpG;MMa8M}x<p7rEU;1P(4OU;5qhd;0t>%yuxQ=CR*&NX^X_l9&WiP%hK
z7vSAXeTiI$#lHXpg_<0*1Zx<xcw>>~dCaVC=HG@I$8{r!*z6jP<F9>S^DeVLH`Q(t
zZs$(ziwm4|%AY^r98Bqg8RBc|7;<rnHV&rnn(nz(8ZN_}^>8k-7i&xu%-+eZ7*S1;
z6~`pbHLDMs{_=xYKM1ZA>HFrUY0?J00xny+cdXF&Pn;~kz^2NN&U%RPf^`WGgWd9V
z(ZWa;<A?oa_WJt}Cz`qmb%f_ioaVuC$zouTV$8;;!TW+PMU<@DmE1l8`fDeAl(Y6=
zd}lB1<4d#IGZyT8JSxfgjy0z!_g~e;N>^-J1E(HI@v4US>}7JVT9z}y*vn^g1l+E!
z4hM$?+La_Jb(<uh{j3^y*JLwB*&Emt0y7fX*oGM-eN0U&%8!SQ+aSg{UB}v}K*&!@
zP+Bs-s}u#Hpkd-vT1^h5=D|6Bd|(@BI28{f=!bnJ3k*usc<%A+Y|1!tr4R~8=IJ1R
zTN0S#Jj{aQ@#_Hvq;IV175Z9Mb!n3a8f9J9Gq*OvvB-I)Ei1FvO%dlz=A5eIGcN1f
zPGxe{0YMQ-_8VES&>H>9G^Y&A6qA1P(%qw&kJ%EiE|O|OO5ChjGN<vpQk!!EBysY7
zORv5VfILLB3?q3~_b60KEX+OUg_CJkf}s!pqNM>IRhcHl$#Q&(9=k<6H`ZBE(Ycz<
zY{|)bZ-|H2dTOX77q$;Cmicg<t4VB>V2wezHbHmj6**NPVF=Z)<C)nBA!d{knIxOW
zp{li8EBG(~iSNq@T;hJ;Ht-|6Laj#IY3>sh#afDMYEr)zGLwFy0!Vs%mj#&RD>PyT
z`QX$(lIJAWTYlrzzE_-vpMeMSZ`zl8Gq@kD56i01A$i`h4Dzq=Y9^m=WyBDiueM|7
z;|-;T>lK`ZaloJ1JKjn(v~MT5OPQI@n6%&03OO70F&N^KOD-q6b`Fphiy`1^8esgY
zILS1aZ~P{Hp!PyDts!IEd@S$PR_E}ffI;VNy@4WwfrM#1DuK9)36^12OU3KCtMS5z
z0w(40`J;EpT!;7I@^AEp9IZojNjH&6-99L4-0m*W`T1F)rsRy(s7ovLER#`g&EzN~
z!DW=Myq5>K_=xadN8oes6W;kWADIf-l!<rWaNX*}Y;?&P9gkTYFVx&EO}+U6U0LT6
z`bCzaq0uxd*0oO<!=)r1nfjf?rTpp#F2~ff%H;FIwa(o~*}ExX!+<~j;R&p5eq;V$
zfk9kV1trVXsRL((hxg7XXEK)wE4MEUnFFP_#2*j~JsEP}`9KBT*ED|^8E`8h*XmQy
zFFwMbYJ`-qS92!wwa%C&1^!CCqcRx3B1Z#wLWS6uu@=cH@zdx4gY6_VZuc}pkAd@7
zMx}hZll+q4A>YtId=|wOh;?_@EZ_S;8L#%k4ETq3T|f>AKT5d>+lCKk9?n-7XzdL#
zS_xkIZdXlaFzR?%DP%CorYxn;!l8Y*WbB_btmeMk4__}h^s)^PrjV7XmJ2YNc^Ks~
z)_RO=jAa6^iQ2BMWcOhhi*+Mo1tgdP>XrrZ0RhrmG>*CK`z_+sm6}ikAzjTB71xuk
z>0!(Dkh;_&t5W5)v-X;2SxQe_O{GWn@h*yR%pPdX+c5_Q0Y5;5-k?J?3xzgbSRPrd
z3gExHitb-Z|2bEql0yh#9XJ4~ykvoTA?ffCr67IffJ#n<F8`n~(+)bu@zf}b?hNAv
z-aS%K!_tfc)vJZ6P@&P13=s=^!@oMB6*4A0c$|EC0sBT=fXrwnN5!fy-tn6CiS@>_
zWiFDCLXK!H21e7I4KHvT6&DK`zk`r8`*ZSQYU$_LD<!Rz*+x)g3lEQFUGz4^uUqG~
zv|Ct#nk|zspM}*8{wAY>j^6~1_X3O3N%^QJ@MDP@8zv?p+amefoiRFLH#q1EUKU@E
z&uL2s1`|MBZb@veiGRc!RzG9|B~N&@5|owF1D<}@yS%;x3UlFy9X-aO5|GNYU+x&*
zKJJNWoJ9C=qa$#Ed+<LA+>ySNxg1V(R}dN1Z8oBQ508xlf^t6F^+i~177fO~s>>i<
z>q)Ba<{Y2yOeXI=;&OKVVgG7{A3@;ku_ZAm%4F%;C=KFhC=oCr`nFSS8f9yLP@PJ6
zoNOiTzE@N7*%-?=POG`&8y2Tpnu@V8`tA6AFMr6$toTuaUPD|Pe}1HI>c@(PL(Sdk
z$r#sMyfvusnN7zh@&U^eC^6(Y&3cjgKuAGFKPc)*hYN`?*;I4nLvAvPX3hC)!&8r3
zcIKuhehvukvYI<y)_APnt18=`D1Y!J_f;gdNaK*ryx00C>fokv+1_Bvw<VW1zrr|(
zNQ)VvWi2E2BZli~UI*Q9lAN>f-I}#jug<`aB;Gv12Db%4T~QpfKt<BTXDpa}-FCxP
zWSH!*;C(s|_p>q66}ml%I*KXOJ<Gq(zM)a*L{K|n;aAn@;Vc7vNm2z@e@NMTmx9O1
zvms8tcck!9J8+C~jwG@O{0d~JV*O6n@QZ$gOd0ZqiTEi@CL~cuz3+DFml$(SF1ir@
zyUgWYjv4f;2^yTsx7+%9=vLo0cG>*W974!(5q_<>oOI~p_-&N4A8x472i+QJ^!C){
zaMX?YGArMfJQT`LOUm_h`Uci2WlwD`*uUh;sp;8vRlv@`+Ftys9=5tZRQ-^PtzZi<
zY@{gA{{HhXo8+HWpqge~t!COZH8mn~EL8f;{gjZYvYO$w4kNK}=o|wH3%u-gNknkX
zylkIL)t5r5+mRpCwpi3nu5K=OjY_(vmNg@p99P8q##0}K@un}>X3i#Ys-B#&4K(qy
zX(5U?gq?LAXOq8RJ%~TkVlU4q0<5Q#(zv~^y|jR!{Uiw{r1}~6ZN-3iXvx}^yW&m1
zo(YlTagNr{%*WeLiN%A-C*Oz%-FROgPqxpkYDH&E-TI&5<qqg6iEWK9Rdz!n#rulQ
z%U`#kadP=}(*`|^j_(k<J{ooCrcapiPModY8t<sw3jWjcp#yEFr=WgN23g>X>Fq|#
z%7ubF7=qGy?3G0^jC{;CFo>Kndg59cbZGu=QGvO9Ev<b^bf&8X$`6Lj|LSx-1L<>D
z7SH7i6x2a%xTtMIf&rF;nPq8(*#jlXzgk#b0ibn(3*RCFg#X>?@-Mx82l8WLx$^I>
z6LP?x75fgP+;4#WADuSrB(T(4NYV2j&;ASUAF_K#duO^_rP@a8@hmTU{I;&9++)|k
zFN8O2=}z?BOsX=namDP}%saH%EaM5u4&3^&0|kGYk8gn2dY=!oU;*k?e&1YujQiMi
zMYG&_Wiq<XGDkvw>RrX}FLzXOI($^BSN&CTR>f3m4^nM;%=G`L3I+5i(dKHN<0EY`
zqWMHY&ZKt)X3}eD*HeAmL}C_W1XZnd5NFanRbztncB>TpU~gZDAo-)k07lV0Y=g9s
zM*z|{EwtH*ffxl`#_o6%>R^d#GOHvaXsoJi&$tB($pbyw>_Q$Edrs?MWJ`9*5H%pk
z_cG!uGIG-s4HZ71@TFqYMVuE&%re%!n$`)DYiF46x6#6nVGG^<gUjo&=-%EdvsdZ>
z7Y(_8SG7X<IxM8zV)&lp2Rh!wjOaz4+)rwGz9DMnZgIb3OQO8oV*zl0%N^ias|&_?
z1a7-=f0vw#O$^qI_bDDrAAHCzeXl~|8q;$*$7E8lo$^D42?uG~&0%O5gd8V0&R{QK
zs5~h>KV5qpSX0&07nJlvMO8~Jzg(qz&O^O%;hc#zHsjA$s0Y2rJ)KBpY9xK?Xsa#-
z$07?hriC6X>hz#B=HRUKxyKX#&f&YL%AKGee^@F9TemAh&=c9Z9AUmQ-Kae=NcH+w
zl<(Jx>#W+n@~Vg^g}UR<GtMpw&cm@FU&p42GC>WHju-TsM+z>VG}wCf$)@QE4<X7G
z`?zFoL;WlxYc!dN9-W~#m$b`tmCwv+=U?6DXFye5$=seuhDJDEr*@XvPT;S~cCM{2
zYKESEOE(_l@_iY^wb>Ia$R5vJ{4sI1L^j)v<KACb*#Oq+-_smO3U2x1JMMn&_pML<
z7(J@{)*AjDCM{l9aZr~+AE$Gqup7Zdi4-A?F-SW;y{e!?UN@j6kqVhm;qfsy2+D}B
zhr9@?ND5W(l>H#Y?yzw1$8$7y0+Ca@QmG>&@k4(btF`X0P(e4>9=+TQdtY2!N*8sm
zaoZ|%>dzzUN$ob>x&ST;Y7^K%R};o#ND#rqL*zvh1K(eOJ#_yx9JWYGE^ts_SgV4b
z@6q*`Yc8`MMj^5NM_D+?WU;3;0c;?@Q%s((LFl$R&)Y9a!t>wQiw9N6e7Hz>DR~$r
zhRmN(P!YhhUd_6WJp@Riwkb$^H#peuauVVm_5R{<nsP}D!qE_M3XLtO3^M#GBlJhL
zzd%K3Pg9T7?A7B^+llaU=X1u!e20)1(gvG76T&_6Wu8#G8Cl6cvk$-?Jn-B@7C?L9
zqB|Sf<tLkpjrn%Z(pxRwj-xZaOx$2gf&FQ5-`V?&($XrOBB>5m`fK?+S6p^am{524
z{GOi7$EKY5UhW+6nVBufqdG+m52=T5uYvAbc)S9E`M7DkzU-hc_qf0`weEBooEXm6
zDtHmZHVEmi_ee6!R%@S6`E`!$q|hu8gH59(5Ne6sG-YTMYa%pBq#~4;pBW8vpF91o
zQl@^AM@afs4DfenF9A?wcr9sz<c03e=kc^urdeK&!%8WE7ePGV)y@S}u7frE3Lh%L
z(t41!2=;IA%H#Lx66K%W=|88w*gaz>;8~lxjzg)KSy=7!vS(~+(Y@F3kMyZQEb$tW
zQM(IJ1(QrY^?h$`aAmzEo$8UYGn5|A*3^lqlD~eOYRMb-hny#RpkDh<xQL`Yl~{`_
zv%y;Tv|?=|o1{kqK6D*hG9pnxdGNmRZ{kA$@r;y`+(_a{f`6C9s8gqHbx#*PBN5p8
z{Mr;&U_Uk+t(t>_bqi4I=cfb|E4==sAOx_H*F51X9OKM&=|(`8nN9~iRzIO&H?gAJ
z;m2iukCbARmfzCnsSip>*h)CoI%n^#uS|+^H{!#G(osk#TtDOB&v^C(^!$1CbB`5t
zuckiQAuVq3w)Gc52_D3Lf!Rh}F_^sKDk#oe1>YsdWez|>R{jo(_;=KeNCSh=m!8bp
zM%Nu!DUC5DrHiBXj6swOsGfEZld7TpF#&U60>e^8{$Qn>)a&E5wceBPCbI_NX#z?d
z<pLw2e4i;tZP+&?gP@QCO_+OwAyr5^l(36#vQn%U50o@Ee|-RSWcQ3mZ%8+$s^hvu
z{Gr1OfZi|GA_jQ6YMToPBV)R%!_PO$M)yY#8wT5w2+StpJUyTHwUyw^Nh_|0sw{u8
z!1Ng;`-5r$gf6#2kvx1PM|sUXr;$@X$*HEXsaO`QRLzCB4zk_jrTdLV`nyV|ZX>A^
zJpfMSH$D@xkJ0r7iH=(n!ee6XYq0Q2aUEwhwDZ!S<dL&D(kz4T(zvy~-b1D;dw?<}
z#h9-6I0LAu)JB48NzNGGLHMVwu#`Wdf6M{i8!bc;0hawg-7dPhf^Zd%M}OKL>=k5I
zw|50MhyM%M8K|HP*sp)UYUX&b<a;cg{cm6q`-;;Cj4(1Lw;Fw=m>$phubiGPf7wYg
z$c^|#s-*zk&zMWsVV!T61A@3}m7qICzfvi)U$hq1>j-UpKuY`$aOA=0KiR<Ll8HZC
z??S?Hh24n*36}<}hYd@vMxfKw+II^Za4ky150vBJV;v6Gy;T!U(<xEnFuQv>=HTtY
zkBbzip9rw#8#tpS(wZeOHGF*<6Q?&$;g?SvpQx9mCJt}ZJmpE@LB^U!;4#mtH}jCM
z`P5&x&b5@EM^65j@tWNMj-NeF=UuOM9wFL;NH7!F&sj-t-cvAB|MjLwT6b-t_ltgI
zlct|4)e}F{dA{u<Q7;12KAW5yNOvRQU^3phdV96TO5EbJYEHozUynRfvMNLDcJ458
zt|3T_eYW{G03+$7*oWI1bEYI4;%ObAn-K|ygm)kXz#4y&OjWy-6rDdy9xTms*^^+>
zuaa~VOp8a7FN`F=aygF&uw8nR#2#I@F{>;Vn{V%@W+TL-W|LEfv@T^pli0e<4A>~?
z#8b6b#M-=@9m-X8vW1sf*4uRA<E9Ug<YGvI_nfh*S(HE<FZ_dxt_w0jRv~=PhC|iH
zFyCVcV{{@}xiUct>4C(6G%$*#EeL@$WNBRc9})|faN9&+)X(!XI6+uJ(m}lEG6F$r
z>pQURr4u$baTNb3>>|WZg6V@vU4L7fn%dl#D-Bq;YQvu+BLsFWFc!Fw{3TejhxWO+
z@NGX?LiY(90;9Ybxjij|6yuZ2xzVPDA0Q3pJ(Ae?)SMJRths+NLY`NNYu9Gxex+`S
zLspN^X;N-1?6wnPm=8$;9w6cA(K8zfP)=Z5e-TmP;qYNlwgEo&gG#;yRJ1Cj1G!<|
z0BOFjTC?B<vQVNa@7Fv2E>>Ogg(|o@ZazP}W`?s*-bh{G&UX4;q*SMXir$EznTvqg
zvNYv3fkoG~71p}EgOl6L`q_zULVL6^{LJUIlxkE+t6lO1L?E>Hkck-f`{tAWUXyv4
z{@dA)?Tyw->$09N<Ip=~^AD=LwSAe9mi<4xtRb%TV)=^+@5x@h^Dn!FF2LKX6>N#9
z$i~ffYaZv61A0)<Btf_h<uSqs$eI(-KF>M3<P^@0S9rfdJR8@YAun?1QQMdF&VT%8
ziwJ^1D9AB^+?rSPrQU_sUl4xQ1_uGToBBSdj{RHeLXxs^$4)}|50TTsOG|7jdYG{s
z5;iZIZhRvP#VP>_rPofjBvz!R<*;GTwnIQdfuugty<&`jg6t(avn0||4DL-eTIAPU
zaK5N!zZ@bSXVG1|?-;l2!KD6%Old4g_K&B^0<YLV8yLzWlOzaZ>2ccN+svA4%h`{T
z+WETcvd7sOHN^$i#jO?PzAni3`7}|Kzz4X@O(Uy=D1HAfm+6C3&vJ{O%ciS^X<RA^
zBkEsYez=t&SCbOSONhxD_u0Fg7|@P_x(mW?{V3TO22ev+fM^?J{fY$`WYQ{W5n<o`
zsBorYailx1R@D7d8q|%7<!mL##w)$~BjOw2e9v;Bc}z&O8pv!t1NLU6<Z9*yX4CqE
zXdCL3eG8N#{wE?0Da^S}ABf^wrT#2z0U-gSP{aZm5~7~@cTK35+g#hZuy&n<htZdt
zRHcygCX&mUWtAi13jdTt0xp?OD<7j(tEGB2M8*Ds0gz+la#;Kc8Au@j>81jcb*r|@
zyAN)c6#SEueLF2A9^JLmh|>9!lDL5`@6T@AbQLh$&lM<fVVlAd`99N48w{*0q}w6~
zu)qM-*1ZCx*FF;k1A&Sep~!&*;5iI7@Hddb`3!jqBdswb2RXn922$a-n31vfOc4y!
zv1k4O1EMYf)bz30E^^X8{2TUL8K7Z$Hy5O%IRmN8&qIQ&3*>?vupp7t%oln66hLi(
zOY0z^z6CAw&zzQjA_;~p*z9>bS)9m87hu6UuPGt&`mff4`V5lrtXE)xlwK&(q|rdq
zf&Ff6<T=ne4JFukPI)Q}Kty+taeeRd3n|hvnSkuFVq<<F!{ZPzix;Eg&+h@Vco_~n
z5qCr^lIiyf0CP1}ZUYiqGywD%bsCv3W9Nf8S(Hi@$aFOtB+|0Xoi4~Z-vM<$cYJj~
z#K8Uznke5HjQ=`u^LIZ3tbIo>y7#WVy$4K3L30P?0RiS8)6@i)ntDZ_z)G4Z5Du4=
zk$dUC(@{|WE1Cb=%zusG|2v)lG&MfYk$LNR7VK#HtC$Q{sV_DjV>a>ldj#px<o5B?
zl6^bJehcEoXG+(TP*GXn)|uX9-2rYh5xC8_l!}f*z#Z<%YbxKxm;+hMTv8b(aC(0Z
z8qx>?FT3@<LK8#f5y&EL(|rJ5wF^L?-F)|TlN_WiX0-vKG)o2`hAezyL&o#~tWFfP
zX%HRXcBV8uiXs9&UiLl{i105!^9ox#XjE?yxOd1Xi5dp#DR8o<rq;*^15QZZy3-W_
zC=%S+BJqy}4^)1<qy<sp8Gz6wki6<41&}0`e0zlb9hCOpd5v*_qyBT!QPc%cG;MDN
z#m<rdY!XS~h=KGLbZuTt{Cqyi1&|CnVUmb7H-Iv{fm|2q=`jc+P%-}98>68IY{ouS
z!3Hq~M7q1>1i3Y-fCaZ4mON~5Rt=kKr+*4r=KeXN`JZ%$3CziQ|M>-oYi|LBC}+O@
z<4*tym%f<fop+$%duP`{ii|ACqn7?#fT!~10fW0QK|g?w{?k1W(`Z5J{=TTyqQnfa
z0D2S50e+r9s-9fc$Z&&A^$`8Ps{60%{{Ot|{N-`IuDT;fGS}!1HhQm3B84o?JWeg<
z7c~tLk0?vLl~a9p7%WD<+v=Rg_58R^l|PoniV+I--jhZWt4B+qJrT54`yPjcc5hI`
z%4Fe6<abnHlI;3OcBs}Y!mtU^a_Z2bE&lfULcwA3o80f!o3)FkSDb9-zU}A-zhfic
zf|EVbrwa|@UwLQQR&{q9AG^81?DDirfm3MNG1MV@2i(&z<<(DV$KRi&b7hx>TtClZ
z8K@oIYS&tx&*w5(En0NZst~&taJX>~+TG))zG5kLT4$cFurQncZXcHH-a&gXg6oa0
za5zxMwX)?HG;?C$(LMak!ibPbow?IF3320D<;1Hge{{CteNC<cv&Ek@$+8N>r{L-B
z&LxOe5=mmI+fotFX>_dHsyL-upV9B>p?Ayr6dmxN8099{EZ4F;D5BY8hl{8dC1H+@
zK|isJCA8{|*E_#RGR#P;vYb|C)?}ylPAegcNj(!Ssr9IgWzm%4DfVTcuUj8+4)~O`
zcS2y2;`YPm;3NsQz30v>GF9(h=ykXq@F<KGoc@03@pAOhWbbbrYA-(N2*IC*X&%Rx
z-5G|AZyip_=O@k55RW_4_>?P~1_Tc8xSy<3WXUIv&=<5`r6FQhJ)J9Nea_n(9A0mB
z&@(gYQxqFCv{b7{_6j%`bSI4N##|gXc*~Ku(CJH~uRQv9YqEJ1%>ak$bnY_xi2CWY
zM;wQthQ{^bd$^(5FbiQP6{0ETfN-%%E<(V8V{NnYT0wem$k6q;ZA6>O_sTo*u+^wr
zVo$5myym&}T=%qdx^DtU%45N8wd^71XXh&?+jrO9=Ea{GcX+K&o<6Mk{VVfX&E=<U
z7NU?YTAPcRZ^fNwf_C_Qr*<tilF#l!8m~6Yz^UD)EXlBW6U6la!dyA|g59uDYUbgv
z5-F>e<)!n%azki^_QV9O+uQI|${<jC-913}TfNi;`Y~>1Q|gKGH24(YbJU{m@cS+T
zmsGspl{d*bd8JCzHXf1e^6ikv#&`5C<QL0q`m&b0^)F9yLMnx%9p>=@jZkQWw_&y{
zraPiR>~_;~>k$t83l`<&w_Z)IotO!%t;8_`uDcKy=%UzB!Aj|6N$f&{BF8K5T*z^m
zp3C{m?(Xi-<fFU$b8(4NQg<On+-~8bs~`d;R+uVbXXyd@GIdXbOt!wmGHf1tO=h2H
z+N_~dzP`NhZE{!$y?OLC#IQc_A!Na0nTLP9?K&f;zN@i0M?u#F>Rs3S<3q0sPeeOM
zi#XJLd$u6A@7f}yQpnUWk>@yK%k?EHH)sc=w02@=fkODR>FZEwniQm76AaJ&<b4k4
zJgo*pNj861s_OP`LJ5L-niS_*B+Z_W_?P)!SIKq=PrWMHbaOcJT$8lj;c<7}U_G~3
zAR4x(70?~H-C4yv;WsnA$Pud~oWfG5S)AZ~lFt!DJ)vjdyS!->zOMzdjj2DM;62g!
zEFh?vKpbhOzLl?3;oRP;u%}hykm4m`R+b{OQOeW51s#c~xf$Q*4<kbu?+)7yd!0@}
zGoUb=S$2`D?^>&09FD3VI$O@X)wCEd_XnrDxoZu!a4DCcF88Jxi<CpbxfeDY0-NgA
zwGiQ<XA3@Sg2Tk`8p17L`mWJ2N;6H(-j;?wm-A4Se3cqFbjRb_Rd=pPk4W!ir~%VR
zp-mYKjbX-4MmWj3#6>UFwQ6~jBWU~SM1I~g#F_DjdSeC-t%FOEktum4a6o%}qIfox
zyTiEuYxu#$>|)Qa#RT14DSX$@s!IILe2=N*vSFGQ$|EbuhdX5m;W`&#b@uN|!RFkB
zn$WkGycc43IZ{0?h-)gZvFqZ=Iy3pk8Qu7Zt|VzHK%Ud*QNfF$!n~BdWWPEs{<~_j
zDcW(smX4$GxxVFC9;-lysM-GTx~lHhy#CR94Lj748blCYqx{8!n~QLyb_H2zd}b4A
zC%eE?uOUjZUIpG{l1ux>N&UKYzg^d@+X;ssBJB8frs5B-aMQ>HzUysz+T;dFC=D2s
zrKGH>JIZNgeiZEVsc*Fs!5|c(6)K+Buijya7qY!%tfu8n+UFPV=cB~GyXfucyQO(j
z^|7`P>i%>Q19d<axV<>V$<(*dck6vpWns6z3MDy3TtB(Eh-{A}C4a)rs^_-<&XYpf
zN&GOwsNLb&ubaYb;~-*Vk!tAinLt**LiuJ1_sVD_R_{3(-fa4UXH}k=@8?M9F2RSI
zt7dy^^N2*uLu!YrW4cI%s=>>5Km2Tq&_|%(>x&^5Q=ZjD0g9_>KC8s*bI$c}(cLLY
z1U`JcT~$$-=EV52_EpTL%NdNcDf(k&n$LY=guVZ+b4NMzI7b<RfbrUp(870}6SE|#
zp97@2V`htQkY_?_PA%$oG3)O^oR+>5&`=9+$=0|x$GGrN3;B-|Pb0Vj!O1SuPkL60
zL~DK8Z?Q}l=XUjKHV5P~yuWG65ZF3qUR`W1o)mPiZxclK#4%4jb6%=m;JpcOf0;ON
z=Ux;1wyresL#OFUz4al(R>U(^SmXXG@9@)CLq!?_#DlLo@k&)wH#qt_sV7w&9AUjI
z(1Fq>6sxndtr%LBD$Dk>d8tJ#Avd^s-uwxiM!-*t(rIhp=8`*BeYGe0qa>v@w6`)p
zu#b)BQbPaegCgc^`}-GW+SS%oaP(s*CT6!QTc2Z*DGcnCkDBN&>JSf~n8+r?pDv`)
ziy*ENgqVC!xBVUZhR;veF$m{_Qr)0>H46sU0@d4%JV%z;quII%swQLTlv|r>*Msh4
zb7q0C9l12CS96lkp@%+ieJ^*V&_@$<sUOW&+wp25OndB_t_5|I<Kv#V!L98*@T^`>
zkIGYc2h`}~!i26T^|nXtDK6-m$<h{$p^EJjxvps;<yUFkCv*)u`C$>#)1JE?Csj9*
z2K=88{xlaRs2nb{%MU!&9sS%*yHi%D=?(Ni`)-S5<K8WhG2Hh^<hiARbgIW<yj`wb
zzH$4@B{58@!qxt7j`9dHV75*h5Jf7V-}=LbqKt~k4Afp{4cB^2uJNh}$J~Y#DZAnk
zv0)F~NOq5;eoQj_3Qg6cEZ0@osa)$VL>*eXrmEB81q8LBL9R8#5w{<@`hL4D#tsHk
zqQ{}~*qlVgI=(B^Vsn}n_<NnHArqDj-+I;g&^7E@@@%6pB0-wL$-bbrb@3Y(q0srb
z88`gks)Bc>P^S#r@QZeVsTaRtt*a2*$?|)saqbu!IE=@pZbPuk=ph(&vmv}aFIIlJ
zBRKu+SfjwjY>tiE`zo-3+j^h~E6Ziua`?LaR%A~c%NN}n+lY_EY^Ou57c(ds6};Yu
z<E3zye79kTCIgp`6&VqUQ{EP@rx48x&+HbcN3vw}-F-JwhO0O1y-#XRr%*Cx+pq{5
zehhE;ZT8zZ^lr1-dw6CNM;})gFZabZ>g+6dZoffn-#Z%WP{U()opEY=#oN0!>KlDH
zAk?Qww*{KzP2l*&@h#`uF`ycgZ&1)hX9ON?ILn>PGeBg&Y^S_yj+uKeB6lSNH!_wf
z21>T}*1BboP-cyN*{0dtcbfCW-*SfOqW4_UnH_fRI5^b1(jjcNx>=ak7cy#a=>$s?
zx_GHsvk|(SvA8I5z+LmR15vT-d*&0{kiv42?fz5GM&=$Q`EvMJBmLa5Q%dP1d7GM>
zOI_c}w8(PI-0a-Mpas_dmc<PoqHm4uxKTVPP3|`PVL<niH&M4Z8TCSE=xFfcOm4&)
z%UpZXuUnCZq*wF`Nv>7&ou@TEd86di?nnJ6mzo9}93CI~w2cBg3pq|<=_BWgh?7mk
z`#Hfalj-SCug<Nyjmr$_k9DBW)Jj~+@Oy1jRfMm@+)pfnb|9~GQ*=p=k>BN5)m`lI
zRYaVU)<%ld>qhX^e<&f<PhK_@IkO*tqouK?_AI4y>8|5xvYpzzZdI&&M&%*4x2J5T
z9H)#d#K-5K^Oz$;`oPk@S?Y%IZm~%1Hd_W^NYe2(c7D|%$;t5#%;jyGi3Gz$TC-%I
z()ueSky-kSUgz0m+bXDUDzC<gb!9Js{>uA=P0v(YyEO-m@2fLdCch90EbpuX97mF$
zxXrnZJ|-9brS7>IrHG(tV%4jZf56>1;8X8#PMm>mL(B%wV~1>Hq^&~{z6gHK#mR0A
z!W2AFCHZ4Zu`0_r>!*-GNP2^N#r(l`CZAm>1;lACe4#q4x4@Sn>3YYlaJJ;Qqzv|E
z-o|!%)6RC@F@9=^({3klq;2j6SsmYCvdr&$b*ZzHo0E3&1>83cRq#enXq50O({j7E
zksXO%#es|<0kccp57R1#nG9IQ_4#FHS<>{Rz*4<baqpIucX0a0aS1u*z|N|aju}oS
z&#hOwM~n4jWC1~Bo*x80h=FABqd`u%KgpiT-V?)hG{MV(Bt(Py(Tz((Ksqc0zNLPV
zWtSqLiB)4T|K`)xV%F)Q0*wEng*VF#>Z|mXr^HjhqfpJdGsR&h<zS$0F_H6rvBd7C
zD=DQ!5#v_@xzEBBkA0X}-5$FB5cQl*xH%}UK(uiPgvB`SdcGlq^l9u3adLT3s9a^B
zLyt*eQ}l`8yCv|L8Bcx0mlTUWCQZA=nyrz{>}wMn*Y7&Ni--}h?=8zNj_uY%`h(Lk
z?}W&eDjzy^g`Y6(kbTwulIl_J>J*m|5ad{&E#PL3nY@pG!HVb2AZ1VLOv%04q*7r@
za=x}JBWSInwfFsZ`P~fkUT5o%!d#DsRf<=$Ghn&%0!v;s%O_z(#YTZ9<%9e0aBc?I
zU1!OE%|$rcR}Z!s21ql+PGg}7f_DNg&wmnQc+oAz0Xv<BvoX|q9Spw~jGDZS<B=3i
zaNjj&u%YqP_@Eh%n6Z#@u6?C{apI}BLva&||L_5Xh)rNX+$?uwI(W73=Jcm^ZR%m+
zX?DUxHYNFun%0~Tn2|zBE(5&pT!X@mh0k`aV2|+_Yh}R~6Oz`QV(_VE6e)ir;F+o`
z(`zEclp;&40y(RQJ~o#JD4{yQ%#vBO1xk$f?FboJCvHbvIK9LFH~`LJ|CBF(ccL9M
zvoEEXf^+G)bAH+*$Km^GiOu^1CLyI-N#G@}U^jGfNoUq@QqQ{{0ukQV%2qSjpR!7$
zauKjjVb?{>d)m{wxs_?=son>v{2RE*%zZ6XD6o3xm*{@%0%y9r+b*`K_IqZ7519nS
zV-Mx32WYq#=Wb=nZ51<EX)1v458P?uV@wn%91j~j4fkpA-CS@nrB7i$3ZJwK@!6?=
z!s^D?)JHW{TLv%Y{J;imWq=^o{I=X)uuUDoh0jjS;8xt7Yi@lr%FB!1%RXz(a3jsx
zpU(DyL5^4Ovm|^g;-F!TOGCR<h4PmIdX53|_fMOcJS)f{$szYntj3iauRYEPwv~?j
z4epMy$a+xB7Wk%4y@~q~5JWtL^YxZCZlTu7Bf*`=!)HupX89nt38vm)r=zd#Qv1Xm
zZ43`gf5^Qdf<VS8)fgl|=6c7OM*%<8PZOM)U(%!$EkRFpj;BkBdr9W~JvoAZC`Q%n
zsoRzfd)cOd>R!jGMr%ikgxS{zZkw3rw-h(Gqy%hixwsZymykm!n(mQ8Y#VGs=Y)P^
z3Nn~<XGs6zWjncidW#T?v3mC-frTNwLc<BTWx{RMZC!mI!?RNFs4VZ8Q;`ehh7{L^
z5n<Si+_v$2`xN@57V%EJ^(opE1*>)HNJ%pp$BlUS?rTT91@NC~^y9COeFY)+%Sx9x
z9mX~@Mt@m8nea8fn2Ve;>5e8Tmu|J%mj0kmE2q8}oPzzrY-%KNwf;MJtvbar#`LdO
z?RZ6hSPn^b{FSv#cSIUlWyL!aLr6JQ&}(c;<A%jW192&ZMPLqJ4moRlqwq*$I$b#2
z#nV*mY$p(&Tz5F;AvLq7n#$QF7Gj#Yjahc-SRGE;d^MnpzI{lTygPU0?6We#XuruN
zGE!rHHgYFN?hRw~zO$R%jJ04@sZWXXCvVH|Hv+|@<eQFKMAr@_O-7sBP9dp&#n*cI
z8<H+wk&3yO&0W4wulC(Vcf+}*bk&EGc1OE50wGj9<g>2(%XUVQof3h)qx>D>X;vdM
z^20@)<y4&>5V9(GAn$QYVlAJla;NQbUrA0{`=BNF_Vu*G4;k3(_%g!gLaf#s1?52&
z@_zxqg9hU0yo&<|aXnre;c{6eE7r~K&k&o;P5Cma4cW}G4>a+n1cU#7GrE@N_W!zd
zBBx?z6gJb!(lcm|$PAL%*9^~gX|x(XNZDl%xAVJC`VT89trMqEshh<MjN`GUU5*kB
z&uSjYjFmAqUwFru>Psm35enyp)L*aKoJ<;e-*mFU6w4|!EY88#8wXxE^_9V<P<t-6
z&{It0sQq#^O^@34^K(Vwhm9Aaa?>_a-p%N+X(p5{FGSnUQ&-QfJJMFB*223hgz-wg
znK4eq>D?&F1a<}(Cg?EEr!3rLpr;+@A6$f;sZV~2&)@AY=LR{)t+|7&21-ThTe+&_
z32c|U(2a|C%|B0@j5^og9T&|IN-O0l6`%wYK4a>m#~PO%?#fG%j=nH488#=cpdH?m
zgd6KygvF<EnvwCHX=lxflZA|nA-0rb$H|hB^Eqs_z-0*x`&Ucb$&EGVj*eKm*G3?}
zp(36caX1u&JpnbG{Wwh8?+V*2IG$c3sM7qvZnroQ(OPEs_>9_&ff*l%D`(NlEPnKQ
zv)DSjU!3A}Vbhn8M834<`ekGpsi!M1;nEyK$PXj2S(9bQ0BN4FX-2e1prwz7;l+w{
z@aV?Jk2#0Cy&=%lrBtV$CMBl{vnTM7cy`_D-=PG3<5-dF8OEXoiBG>V3VY~=r{3{C
zZ}%R_`-z~UGS_^L7;WgJJ6IC8n}I!`Im)s*ISoU$2VJV^g=*OIYz1PJlISj`u}Upa
zJ#e*b9Id8^xZHH06aR<3w~UIi>)VF~k(3q`X^?Jd7(zli1PMt&QRyCrPC-CGTDnC*
z8l*!)8U&=9p(UmJ-6!7ndfxc(|NgFLEkAf&&Yb7ozs_UU9NL$y%@(t+#pT+(2!NjV
zVo}<r>)de~MA>E2E-Ke-*ihCNL2Bp6MWN%11;escjNSe6od=&xRa$CACzN}YYYmg-
zd!5+t;pxB*%$G3|C$bJuoNM2$y_>ainW#4_=uNxMy;Q!IoBD=CfN3|*vwB>yQ)g1o
z`IMtdPB)y?@<wrQI{0R|SMe)3JYG*EtfukB@Jufnn%Sni7~$A@ViO-z*Iv`&TT#Tj
zd8<YjL3h}_)gxZ(KT9msU9mc0WxhIL&fj|JUKJs8A11z<b#-&~*!^W)X?uXR^AL@H
zX=AU+)v;)e<J07hNT5R6-a>%K45QazZdaPe1|{4o&?^`J@9*6r{E6Q|54|MzeWkqB
ze_8W2tGZp>CihwCC(~TJa=NtQM2oQ6DBs}*t)`ag^NN{cR3j{^vx0Bpe9CH%4z&wf
zNGW5!sfCX&j~V||@IQ0TEU?5}11YnoVofTya1YA4AW=hv#MQ#P{^Q#1=BczeLi~E0
z)<CZ!9o3?D-rt-y5t*K4u1|hQkBt!Pr&#x#ju=Y}%c@zOf7Uk>aS^HMjOG;k(YU*4
za)jN|q#xSwIIC*Psb>4$YF763PxOdI1CtLk8AdGC2W62*So{5}B7erWZgWM|t_TyH
zIsKIZywR>c@4bEeQw7nHiYYp^VT>Cf;%qyY)Zg#I(EUR0t-+A-QmCG)P-E=cmd##h
zpa`W$p_*ERA)dsynvZ0j)-69aAAZ3TDv2-@8!Xfo`J~_Tk@D?5eK@PpAFWK<<N9|I
z<hW@}sU^md?Hk{$4eQiCne~j0JM+t|kv#Imd`rB~-Og_*uxZ!c%JMvoYObN96>ed^
z%G=<On{oUEnLY5E*<w(C>p(jX01B|x<_FSvuJn)dq-<&~W*N6N*eM?(JNh;q;uBu1
z)Yk(?F)jL=qZ9Z>nqy9DoO;Mn<Yye}9HsSjMa(|UWcpdUkX_=rUOazR&@d~NGr<3=
zWZ})A@yA0rVM9xqPUy^5a9+}uvth&GpVhSM$2t!A3P`a(j4(16{Wq**WBQ$5m)3dg
zd}{Gy`VI&2)`C5Xjtp`b1O=5wzF#$S6O;Os>WjqN)X$&VA_p;)Etn7fX6oP^u%_ch
zjp^S$ByINdbMz4P(m$!%O1phD$GhN9j2w&bEdKGlYOFcEBh68k2cD8?^ruOp!St)d
zqjzMoGptA1HF}P+vm*~FS9u-UMvlI>W*$ZN?O^@C3nzXLVIFfld|;qBy(-2K)P7u-
zN63!yj)%YCl?@bQ5ru!>y1)sVkX&2I2%ZoC9nCt7ac^Bv_&3>doutGBX*^AB<Wn?V
z$>LPQNv{CUlV$%`B9l``e!r(v8>mQ97`~;|jM(hX#GigKN5h|UA|b*3406qvl!}fm
zVC)1_$9q`M8+&cb#H}YbJQfq=1ltWqXYmdzWI-qS;2Xxy_hWHCfp18rC}_4odl1HH
z!=_*+2DDov>uDKi)S>RG!f&wS69M&9hX|kFMnUrckKWaCc{XtiBwnb$6ZjZB0C#r}
zg*l$y@?8R_?;_VgMP&f3%NUt3Yl7VmpkYZ*`Gg+~lR={$|NGMaCGY<V(*Lhi2Fyt`
zN27vu&`92XA?f_*d5c|V&sf0!rRfEhf%{n88XEOPeZ*}$3!AVkZg_n>;}ZWY>uJ}E
zw<#p)ql5RL2V4W&aD7b60Y>aTVK1$$bT4&9LvT&;8aayZpVNAm=c=V~jv2b7^d<{y
zs!Cisk%_rg@V)y*bFEl^Gb`U20xTKZ=H|^*pXz-1fOvF&ifGTb7v-RXU=$7Cnq<&<
z>GOS5QoytglZ@{jKxQSZL57QFza2=`_@bO}t1{{fG`evG&3+dTeB}Yd8oDpR&hX=x
z&9}$|ZIXW%=nPzCdaoqB4b;t^|H5TduUiW$j{Yw|-=%N@Ahj6nqTj5jfN4=+H}zau
zpy5IJlx9seF@^@WLkKKNzRG51si<xnl*8F_Z7@(c6pAtP;?X0+DMh>IfB2i_roge^
zS(=lA0hxJ2&{}pm4@`t+JGCV-RQ_0N0T`xe0qe=mlRadPR)O|fHGwX`_L+uHS;K$1
z_9gOf73u=;nY>UUT@vtWNtzen9A$o1(0pRH!pr@YK8FFZ<8k9PHm&?eE_0q$ltL-d
z(a~ST(3{2q&XvA2CH?SsYkl`~E$~tQIXgXQ(G3C3i2ACgV?z^<_07NYG;@^VIUjpp
zc|f=vW=oxsW&8c29mPM-i7?lE92+d0_WJ}wjj+}M#?`@}F*VlbU)BAF#-SNp$#iHw
zd{MEK3YvOE@dXd+1~5o3bGxhos1*d0<uGmEVfJ!hVXD2%tY_)$PjNg`nuo?iK^yj0
z9pSOjsv;~N1Qid!qdFnKCH-sjX8lz8<@wRpUw}|FWjrfrGtKOIepK1i8P8?hSkiaG
zc?T8XNi|YR?+>d<fZbCAy4E24cmV12CH-~C&CulsFdN!tvHK8utY0&r3sZ`Sa}*ix
zn@#|I_0rqbcCx~p)PiW%&}O1cHt;SP9l)3%!F4~=I}gC%4Ltswz6@^T21vc3!UAph
zFvm56$1ed$mMWy6#aIzXYaniP4`<ys-KjDSH#D!)<@=)FS5jPNBmMEk$MM8%0W5$U
z{AnIU;QtOkfzjDn0t5im*equNJJ9&+zcTz&8u9^Z%<NZ+%1<nsU*yxoJ!Lw457q`-
z#V_kEM)Er$>BW3R3kIoBBUGS$ohnqY`Cs9S1Q1K{m&bE+SD9XG89}uifrq>_C7^jO
zy7C6#sDZ9i-DGZCn&;63**I?<L=r!+MvM8TX^0i<oqU$Gu_<cEv6yf6g-`ojjgZ9x
z)UvJ8M2!@LFIUsWQ26l)xao4muxd0|!<Pp@T#GtLF~DFSNkYJFI-e7Wu=3jTfc%yC
zh38;TdAXN&pn|FBaAQR2yYBaa^k#tRw&aUJRr1P#PiAAB0yo^OC$8@$$2bY32ev>D
zn$9C8!KOse9qn(rCe*6A!%HB`n?nx=TJ2i>^LZh&FNB_Tygjp=6K@m+ZqgzCcKK^o
zII9Kk&00JGH!v@F&~PpdE^0KBTHI`ddk@gT9Dr#6w{yr*j2$Xr3h51OCbzApHLQb^
zP?kIP@sEw0BLE~oY*>DY8f2pc!9$=h3v4V&VD_enj?HQx0=@4hl%p`gh}Z$3S6*L5
zoSj(G@!fvw7d`#>fEM6QRN+$r(5Z%{rKM{2;1Otq*bW3!ErSCUV6A6}y~KQ&kqm|?
zNg=ILjAcDSaT+sn=U~b(haXGrIj;1Us)=6%oR;OatcdSb##;b>%joMvXNX5IC=Pq>
zqD}^Ql+`?C+hk2GQU$iH&u=vYL@Hyj?vL#FqP|Lm4eXpNfQ;HYsIwgX7MWyIs{ruu
zXn-+dHwm_a#>3InK;^eyJVzFYF|99Xehr5s5diuuM>dr7D1I9-=gqsF&7YyDJ@H)N
zHV}&3R!kkf_(^)Q+eV(zH`bdK)5n}(J5xwl6rlMufenVnnedr4U;?Zr%G&!4Mv9Fk
zSagb#=aQaYYChTpfK?My#a<^17&Y^LIGaMB0K5%{Y>wt#h~H{}-xB9xdh|Y$acGUF
zE9B?gH2^_EQC0l(^`LPDfC@{_nJm%g!m*mE9SJ`jER`_pjwyZRBjU2%D^1@pZT=hV
z0S~bPHDKd<4<4e9Hd<uJ2Ig#%K2nM4!Dn6Hiuw<>-#4a+Ss4uH!v;OdV)(|4;GBo1
zUX=|8fDMcH#}i==bI*o$%A1n_&ux!zD#6kzKGqFVTZX;&<zajZt4rr6T8%<Ie*BS7
z`6o~26W*qlzVbT|+88ZNng+;i6mni`LL0z>vH*v2GcF@Z^A7bwgR+9E&&&OwdjdeN
zSuxT5VtWf=*8vmCbxx9n4yg?~sHKugz5)AX^IRwmg2uUR4Zu%rHk|sS<Cf{bAoVK_
zRM#KC_5f@6M;m0X-ovJ3p-svES;g@2zGM29USu`daq~0Y8~VSt-flH3BBGO@d0s66
z7{<!-tn~+(;+|*RqKWaMVYL9NMg!<-_Q^P~t$uXr1IBApRmCuXSO)W^Wlq_MhOh0v
zbURq11$c4wo>aZn_Q<8rufq2fw(_7!d-ea>phIO&S4&RiGmo0KYsJk}O9q&HRd*?b
zJ~63cloU72W>?L5>DM}~)8UY^e_|`H*Vt}2Hp<h?4Ie8|xrAo(O`*LOm3FX@K)~lK
z?6@)>RPPDo%>fAT6iSo(PJ_#@L<M(`N08bK^2kD_96;iA0ze2R9Yg0>pk*gtNuERA
zPadoV$Zxjff+xN74I>xqHguUWQ`^YbDdq(BJGP*>F)_OC)^W{TqJYb`f#}IaE51Yw
zKtq%|I4S^-8?)_8Z-lrZ4h*|E-YodruRF5{B<?wP)q9-#-6T2&w2)obtF*Y&Fi|m(
zi%QI-8UtyUcA=h`Eyi!(HGs|d*W41?*9tHM8g@kQb3{)-1X-$1V3E0iD^6b9dVYrU
z7vT4iM2!wBOW=IdEtP`ME(ZxTn!G$kiM8wCTNl%hHo$5aE$~tS6-aXdR1?DS@775l
zDMhs*MXsr-0hlgT<I&Z?Wd^?Y0Z1e_rnbu1695J}0Nk_g&Pe(kqXst=bILwIVs6O%
z{OYM71c8<Zm?wg8G*ACy!pocP30nMS&VI-ziivt=r$mYH0x&{~c^WyJx;Zv8wOII7
zccpfQ(9!(B1RSOp^{^D06cgl1&egwTP<yDu0mK&>k7+=%_tj~JWBQLDB)@nR9|{?H
zUph|!#KNsY{~#*2cXs5;J-`wZ0Fje)q7MVsOl}^)4aKI4xp#w0Z1NIrH=f(%=~d?M
z^LrCw#x&z_)zd%`l*H9Zt150`dF9C0XY}gU(2fWJ1>ld}SyUVifrT%qNVKDexy-uC
z;fbHTJjb#7Gg>HZEro2QEwPyq5rv!ge&*4rgIkLpLHJYHa!7Lqu%QALP8JHbGr?jN
z0auAx59v7!?=q*#<Mi**>@ff)ojjjIjxCF!J|R8XuG`uC{N=xq0h87{5E*<*Ql0{K
z;w`WfY0eKuWN-OiV?n@pM@LZr=}-ZTy`Y{e6^F0Cd_waLb*joX9#EjOfANdjld*~E
z17Vtn-8ayARzUE}W^Qj%qo5h!texO`7X8Rph|UIZF-P$;An@sO7PbPm>`ok~p+$!m
z3;<~ieS5)6{xlrGB&G|*!)gB?d%bj#@-kHJn>^$qz)#v<Z#W8rFzgms;Cn>y8BDx2
zu$};{KP3<~nT?m2K=B)sZgd2se+S?gL$<wCfIW9m%ij)s$5%VwEFZ|Wp89@tf0~p-
zFB^)QXvR^K1Z_!EX0~SCcVdCx>JmGirl%Bf`jniU{8BJ^6r5%iSh3VUM9>y-DXN>9
zG<Dn;e3gP1<zFb}z1m51+KO5%L8uf6p`31xL;&DXg(Ly%*B2*yLjV!T)T)aWQa%gt
zkUaZrkRizXH-prHai-2C9=N40fH7+AT{>f)n*up*85$U>8f-5BGCF0Z?btO}XMfB<
zz@ky(u)KTgmj(#z5<$FZd9VrW8#a+{J)#uEP0~hQr)Cgpq$gts6u$r3@Bn(oc}Kg*
zfCYq*u>dkv2EukdyZZ?wHmznX-=KZrYGBT*_f4(Ak+Pt`<u$;xasa4}6TV9TqOss$
zeORs=^z(l<Xz+n`&JzG{wl$syU{NRLdE5@(<rz15A)dSw1_8MisIq7qVawg}UBre^
zV^eSS{U@~<Bu+=ieR$8saTM%P65V7&9|-A}0IN$F*5-hg5S=Kkb-e+Gw*;(Eza(!2
z^XPXmzv)#f+BBZNaTkr9{mbL9BoELN9Uqsxg@gHbz=6<LQ0@WfR_Wj05k}4wL<!g3
zWUYrbsG`8a+N1W4pTPfrgt8Sgj4CizI32$J0HjIJ^E4O|=q5<P$L{@_&xjUzN3cF#
znst}#aa%ug{OeLHh$xsp{Qbd$9&>Voyg;;Y6&PwJAm?9GMjZhjVh=)WoEFF)=`Ztd
z&fg0FAf%)dV+|Cga5fwawXy9(D|YJ2K#_Sebb<4Zy8jC!VZF5n9iNIF#3>ctqRtzS
zK#Zo?Xr~BdGzMoaeIEp^s!jxFr5#k3)YB6{h+=kvPzfbzD1cG1JK!ypq4|UA=A7lY
z+MlT-%Gr2gD*D+u-Vom9*6<aa_Z}xfu*_=+M7YV@dytLxtP1c&BSY$ai{KK3%0L6w
z+z2sRz+dGHn*nz3R%<t-S}Hq0Vy$fY3}!p`@@BAdfY+S7b9HrYyAGTj>~tv_Opy?i
z0aQE)&ewd7OB9@S9vEA{;rI$P6g4aedz8xS#N@nZ?PnM{c%;)&J^<Zy&rf|0(wJ#1
z<s<<W$Tvz{9z+`en4HHRCw6)IRYi4Dy5wNL0gB1Zs)pa84Y8#UEdDAbH}=?jzcvyE
z*%PBJ`5hw&wBj)XWWhwIMt)^TZ$)5<?2{lXALk5s5$qN>Ujyj$>uWx^*}MN@S8t}=
zsV8Zl>Y&!B&92ile<Fp&z;KPpJi{<EGs7`c7=TLm@|GXAJTnL7(2VDtYoJTK=JBl9
zYHEr*e8={pbK-MC%4z@X+m$NKNWTDjA%a^d1iQC=gYc<`ntgAZFSZ582-%xH6os>c
zICub`haQ{YmapU^yN%y07Q<g~t!HXuu}N5E0BCVKu(o$hH6j_o2X4_2V71WSQbbX>
zg+fjM+1#%nB6nIJVz3y@yc0$t97Xg<{-u91Ee;LXaVFILZbjz2KJ?yZy86l1R8=kj
zYJLWw&ZKby&%wVFe!LYRhsI|2`aar)%KXLYzJ}d5<EGTVOI`F$2oQh<U3uMIyp2MD
zX@?53hb-VF2LUD{i)_d}KC|w7`QX2Oavyeq52!2LYLRBJhT))D$;<-;YX|ueAC%k7
z4ojU5AGA8cDOnU_m<75S%1}{h<~e-D+>g`?v<ue&84iGF8w6Q1i|5(F?9<!V;7|gO
zZY`L!O3B7aHd~=KU0>B%=~9R|eaP1?WNotgq7eOavxCkH5HpkjakxREk_t;WJKj+%
zHg00Pk{kryI)s!XQP#odwcjnw&rfc(wBF7CK#XRkIDfP?<-pv&IZ@7HJz1e)_q@np
z;!BMzSoJ__QS@2$*h>bD*)feGgPJAv-o^INAb!073>1PAW)w{PP*RF0T-1Clc2Lz&
zrD||o#oPPDYSh3A-yF}6_FeV8g{k|g8Axp&a0G3REr|lwiXd#wlnEw!!ObN97}G~m
z7gG`u@Sdv0PZEvB?&8GWVy2cKd^ijombFQ@5qLyGn$=D^Ot7(!1@pSlErRo#w4j9m
za4-X#I+NDGD-X0q)Eh<+P@xz}%2Qzm-1h}VpP}7_*5!DG?TPtid*F0bBk5$Ziytl2
zKy;+BTk0efma|&zPkEjxgLMW%c^0{F3V}|tvRkM$1P@U#BYt5nh-1n~q5>r8$RxiM
zanHn!v0^pk&#<gV3Q;nWM4jO5K4fT!oy8i*RgDNLu@8b&Tcd^KhwH;ZrawQL|3nfn
zf!p~I`0QeS@{@cF!WK+}8i)5F+fZiH{+15%n=Y<MV4Sc6vT1Q|m}|GcqOvhr$;M$&
z{ps<WUU^Wp$pU&l#hu$pgz0mO*k`xd9UPA)zAl;P75d@!OcF5ZjNYe8uYpr~f`JD3
zrOZ^~c@7tj-yAP}^}IlvW6_lcr2KrBHHcf_xO%7rR2HsCa~c9Ww4<Flt`7a3nY#Gy
zI1Z)ZT(w*PYJSH+B$5UQkUKHr>Owe-j5iQqJ2M`i&(A^CBV5I;6vrhzG&(#$pAVX&
zoh+C_goxdF-|1;f;JO^H(7jDA06tst#XK5xnCHYojw^Qo0<g{jGD1Sb7xm7&mEMHE
zI?f`yFz}LVaG_lZ%{EL_6t$Nir~jQHjj<?9S*TYTRiLx?H_i?ag@%9`4aCI^$9F-=
zEOGkwYmhtL0~It$1*#zwpfR_B2vVohVq)sXHiJr(*_|s8jjLZ>oGzzQP<!n9YL)vt
z`QO4EPj0?EKVG)bfuZra{$1j;UzEuQ5k;S4*ru<KCzd2)yviz)m*5vxJz`IaGguU{
z`!eJqKkJ??2&0*Sg4R+~_y7V*rW1`<*-}?pO+=l<>ohEm6eCF3A?syH=(Ag*?Mhoo
zGtch}@cmTqaz)$@m_fAZ&#00p1H$0HB2;L=%H8taeYAUaXqhPD%zF4|EaK#q;#{NW
zvPQkGCl^fV&Hy^tJ(olvI&d1FG(ScIfwL?KTyg=Xq$V)%OZQtsNZ9lkFXfqmIA9{^
z>`woh3CnGP96nYw6|@BFZ;McXQ_cgzVqQF!!hX`shFVY0ZTqub6lJc?B?591I>jyF
z%|<2xZy}4x)LdY+x1m#ytqqiNNuwU%gW~Nzr+1|xC<R23+59WFZT-|INVC#rCJ|`#
z6Kc<c$Z1!YPLY8?4@22)RKOuyOqlE6fIvJ-R#}W<1V0E>ET*e@fE~p807L-w6ELH<
zNU@{PP-lrGm;Z3^oIsX?7-e=ZIUapAg0nsJemvXJLRM&s;se)G9Kg=Hm`8sJO|Yyc
zD-`eqg<Q5N_^rkXT5;(%lYBk`R<i4sG@E~BQIm-vVIyq@WI)3N(s+vn^U-azdWL3V
zb^v27Kj*QJyWE?giC1t2iXaVP=vPl}p-DDtqZ!;EosTUMKi}llUnWdIK`9^tjI-#X
zogs{C1|#N6*jn{5OQ2k_92EtNgaM#<!Z@tQc+fqI&;U{)OOP?8umhS>a9zs}Vaxzx
z@Mb&Ihz>s=E;JB1a@7QnUR|E&NxjF`@BxzgBmOG^I;03zYLZhhXCS%`t<SAeEzarW
z&~GOL!uZ@Lh{8Zso(Et04UmWv#XUtB)KZffRg>lUEl1HPfMkI`x6u%I22<ay&4KZW
z;W8E%KCM^JkV2Pg!MU-Cl8BRNMqmMnZvwgg<!Z0X+GLr2B{^Vq#{!sW6``k;2>Jq7
z+S7*thLP!lOmL~ilNUQ8Y<HCDU1$h^;f0}o<^FV!!0wUV*?~C=pMdSG@*Nz~>;&Go
z8bFyS8|EKC_mFH>LdB<o(HL!oFr_dlL<KM^f0l%ziTKiHQE(V8@F0(jKyI3sbiS(t
zX*~?~HE-!b`p;)I&J8(k*>H-9wp=<3=tLiM(Evy(&1&x!WP)N?=!44M^d}3OK9RI}
zBFO{fpuKrdhyXMArpDI~sw>pFAZ6~Vp<vO-#$>_V2A%~l|3rQssF6dX8~~qQ2L_<6
z31EkRqW1LoNc007Sc>M>0wF5JG%AV~8u0V9f(?4rcC==_pPzZTvVpQveg{(c#>!-1
z88o=Xia2k`{MR;nMuPN&BqM=lpXMbxI1cYuI#mL0c^YQs4=epCY)vI%9w+J{__JyB
zpk9am%17`Yv%Cq%%gjQP^<6-$l5gB39!4n|7saTuQM^Wok12xy2AJ@Q<d)Aao!!Z<
zNlkt?$P>Q;XINGz;eNEa5K95Oe?w?V0>s$?%btY}<Sc<L9|oy-p6mX~Jr?j=GAO{?
z(F4)rMwPK6*o|uj#4!iJ<$PWj2*uCb8OidqB_`6gS-#%?jzo-=EMza$UvmzM2YK?5
zbbpcR0Ha^UgDw*t<s6ub5^pm4AQYLGpeo@&TmUM+>d}lU?|@5*VSbi*OOZATgeJfe
z&&$8uTWtRgevhKoaW#Be(;VbUWlU{@_~AgeJ^};C{S&KEl13Sg)38<s3aCKLFa*W(
zP03KrP}!c28{(Lc&qGvp5kzGHDAK4W@|nK_ynmjfNWzcbp()w?6iATO^Us}%^MwER
z5T@P=34*4tASl1bZ}GcXiCqKj=1c^1z96-i-oCV7`$enQ<Rzw+|C~kUx*7`{R~ZC2
z2^js7&9-Rpommd12x=HYU0bzY$p)Cg3sMO>5HtWeR9AzdvWnMTL>9l&Q3T*x5!7lG
zLV)3+_i3W2KclIp!W8PIw7@CNBIJOVy3Ly*PjYjS8v$gdyS0#k$wx)JhskbO8v{Zy
zMSBpxA`;{vvOqcmsm=Ra0YNwf$sq26DDdAuCuu%~?>mNG>0hYb<$?aqCk*R%-?P!b
zHU3*w9)xta0og;azLs4QP=(--Mu1=vM8S;d`G0w$Q38SzoCpQMS1A8|0Pf-bi3z&T
zZlq6scSpw(88-TY2_`WN(i0z5$=I!+-CvQ&f9H|ZT7Jc74rF?r&IWu(lt|s|oEElu
zazvwc!)baHt=Nc&F5Y{0TUrSi5jTWN@nN09U&x^EK>5c-@Dk-A0;)GdveV!@@JH$0
zN79J#y{#vyH#e3;&1a>ij(*GhARvfVdH050A}CtlsavGw!f*c*Oz@I~Y+6`vc4#%A
z)u#N-Axn@`{{Idgf%aTWT%2)0of{Kh`tv*%L`o;rd@G57pl}Kd=%BZtm+&e`BR(tK
zw};$(<jo;XI;t>SZw^h3^)=taLpe3$B>`0+_YY#|=xb=tC%iE6DTVR=aR<EQO;q46
zcQofBIxv*r?%<HE;kNjHZw@U^q^@;(@);)mg9dij#P5d2zy8r&#Nj3?xP=K`()$Q^
z!GzJU4HuQ>CAd3uNXj{?rJF;0JOVu1<8cl2dxCh-0#Jj%(bI`<XzsKm1b7CCq1NeV
zg@-Mz%C#TJfxv|SJEUl%Sm({5Pvim5kgxvUz2m>j9e{O1V@akPny+=*-;jdlMlksP
zS1|rhg5i6M-tK%u<>B?nt^BDhhGpg$(PoXHB_W8>LgK@79RyTYj3FnI&F-ir=d-*x
z0|XE|<q&a?Q*luVeVY4v#>^4OZKumclJ0YR+6|Fc@@Sov&gEYr#U%I-{R32+b?UB{
z7G8<C9NaVOON-8w343_vKi_KmkF!lA38#d8>k^Tf9UoNN3p^Cb=6WoyZd)G(#B&)9
zC7Oq>|MVXFsxh1x!|BdA?1(VECKNe5=BZk3Q;YEl(YgG+RXf93GX8S#M=ZhqcDWs2
zdkA@l%|wNk7bMS8<<Yx7Gx$?UsCCo=UX1}qBij=0e`6-&^OYjDCGNG*-#;{dJR9FZ
z{}Ct|qz%+K%rL)77Nex?xkjZ!_sTC0hXwEDh}bWcucC(P)twVCt~UCO=lbAtsz&mE
z-mp8mE1yv$n6GV^+l*~FEnIE6|32OqXq(w$d7S;lE3`wiUS@AAMpl1C7BV$3Q7ruN
z%aR6`l5-Q}s`XmQ_@xrlz_WpTj%E5+Z6ONf@ZyZlXLRmKXeY=2RCwVP%6mC(QEkZ!
z{te?nHoHIF&fS;Ko`hW6NDcoAl)KYr_*3o*mcD8}urjxHkU@NpFrcF)WBA>lvK^Ku
zQ8Ijp<$H=Tu+n}M3}2-iYX^Do^*}Kja17qNzaO+%9LIl5ei8)V$}Kz0A7ZMM;Em~X
zuiE}KLC>_lFoR(#)5#Rr(i|=N&l;q7HXD^@?ap&f^e;bDSn=0Z{i3|{mN(2Bfr&rX
z`|W{Xyg;v^nfrqM;WWd>vzj}g<{a|y=rW}Hciv(M>MDvNbcQ!akgX#>sx~=p?dK+V
zg~?TSjm)fU^LsC0pn7gCOOshDlX$}orFuMDBCG|%?=612nB)?+Qflu6KO9|N)NL+5
zI7Xs|kcuS={#X#CfUH|@FC8RutyF>ma^qy^rgs<2j%i^R`vNz9<BEl=>F3{zbzYGY
zYeB<0yBFcXYH4ht7g9FQ5y`?ATzl7QRIto%cV@GCYs$&=C>IY4#+*(fZXe98??bUh
z_o0cAVl=;Wi7WDyQK#Q1QwVGES-!IQ`Z|)ybo}?T+LIkJ#Ta%Tv)+`Hs2qg_($u;M
zwQENMqbA2oQkAX822_G%<4=BO-I0%hVKFGgDvLUwEi1fZI-(?ujw>GY9PE3@W<;Tw
zDDvJQ|1jk<Px6HYQ&9`OOR1mJ_Be~k-WaVLllI;2Pi$;CeW{V4Y>;_p(A%|)JVFB0
za~_-ZCrSSuv<<nPTH<xSfg6&<EuPc+`R88Ta4DA<6<4ZQt~WkK26=qZ&`grcTFQeR
z^)jbCX7?Y*QpsOEwd-#W$m;f)^~HzM$%e~Jo;x32c(<9Qa>_Aw8*fPW>3{a7Bqs?w
ztISbr`ew!4-}9Dp$Y1n$>DRZ{=9;zd7hD48QutmBV<hc#ET%NsEq+7<asA42$7fd9
zSc;+Zv`~!Z@b7SN_V`e>BhjRG!QR(SjDx;wKnm|YTH9_~KdMZ+$EIraql%m1{(6a@
zCb0iqEg>z^=XC)yK{KmOVING7C$>{zh~Iq_cI1zgW|PbM9qL#(ryK0`zek?TZug14
z>xtT7{{9ZbDOI4e%YD*{jO9(TuId<tT7BvHMOD|p@{y`K!Qh<e(XT8Fe3k(@YITQa
zOpYnd)?e!~4K=@B$bG@1bm{eP3fF>6fOqWYw8e#=BPYe^L=i8XoZ4AqzcjVoe{Exj
zAy|`4enl)ZPG5&Ff}ZoB^p!^Jim`eusj-lu3#`Ut1vSi)SMpkVbbIgQ;*G!Pvst2T
z;!>Bw>x9N_9zDLJ{Z6e+Mi&a!;oSGjbBUA>pN+&Bix@>khfT#7{c-9W?&=un(Fip$
zoVM9h&l%Aq9SmLNn%=f|BR{fMqK;u%fe$=G@R1J%d+B{Q!p4`2sKH*%3D-ogCd>DR
zU&&_aA4D3x-7?x5-8B?8p(;#kh#Cq(-`=g{rO0v;InqX0ox_Pa9$r=6!%-y+juzTO
zYNTv)d=PeZFl6J4#C^FnyJQ@vm?4mdB&cOV`?xaxGJJ+lO%!*3=gRmx#vN`aqgllj
zFV!NStto;VO(Oq8E%aGO*X4{~z@vJ98IIkaeCl_n;}Zu5gTz%O>^=pRMQl0&ulANV
zJY26`+skMVMgBZ%(BYFfOy>2|4o_1eU-f2_O@Wuga~Yr4xLR-VrF&~ZCc7jNm~8KJ
zf#z*u*++E)TXAphh;ha7{;f-h5m7p;;tFaVRO=ZV_M_FLEJBEHKHAtvO}lr97}l?o
z@cwo0%yk((|JPUaE#2}LMUCx*a$%%e*r})fxt;g{la+Dwwa+3$7*2l^sX~DuX8M?K
z)96v&;M2zH!FHW3{O)_XXu-rogJRx2!J2v+Vp#Y#2$iJ;j;0G*9p8D0iU5VV*gq!w
zOV)7Xu&w322%n~v<Khw35WFkXW|J)02+qcFs%$pXZ{j%dzCBFOHhR}99(5Y4Al`|g
zJ!`jnXvD(m6%|n7@`tmQzVWE+)!zKLkQZ~K3>RsbIsD>%68Yh#!+kDh_q$8MCqeDE
z1*HsU0PR05eq>^M+nFGHcU6Hd{+pP~Cf}U~FOE10ub@CLQLO`5wlnqyGwdYrA~T9B
zmjH*1!MN~#V7pxLX;yMeVEnF->pU}V{#5W{=rN7|+NmOI<9wR1hsK4slz^3MJHC+v
zw__@W>iRM<sWAIX*G}Rl{lcg4-1pMC-NldVYwvMc_870Y8sQEU@5P^+Tv_M|yoqBE
zsyT0T{6mSoEo?$a<$YNZ%J`j{J?l+mq-@NeI+m!A)UO4Za(}pIh#H<v@ALdUPq1n5
zZfv}UrJmNkU|aLmZTORZ^0$32{sN3ROE|t3(J)h#b19mO*-W}?rIuq!mc&d(L9I0g
z3B*c=V#{qfHGJ;SmX>b9b9*9T_R`w(d})Dx!m?Reiim2ue0Qq0kcd)^{VgHkR?6D*
zubXnJLZ`^Li!ll*MSH<qaARUBic}=~-xn;Cmq@%CITGVg5<vma=anC=TvEop$9J6B
zBwCeop~(I7ix8uqVh!i=XP&LaWj{uxJ%sxhM@vU3yk!j~-p0ZXAHlTM3}q~HxnjK$
z4J+X>mM`l9mnFLGlUC{qIr+-W)lP%@TlL+z9;obu1j0nO&sKSofP0M&%jc2k9m$Mo
zO5B|%zOtCL-4mLSTydPOGLhUmDqA4eXdSiJ4Ik`UhMAo1UCK#iw;|=$!`~0>{uEmh
z_BLE8?hWsMhInVpth-r)+#c;p2_zDHIT5BS*lPay1p*77GzOO4=UiEg!_=>_vr+h3
zPsz*Z$$_GCOh@X~x=~&YUpKpkP#OvO7=hTcu;3nB)g~;BUFS6ZUQPA1qCQIQNUzH`
z(G(5>ZmmgAt0O<O(80bqQ$F#&C{Z~o`Z!lFd4+2}u(a?(bkv6M{KM;PWz&<ig8jqG
zvBd>XJ#??rK#5Qzs*SNXPWWz3x+p9UYX}EzJg8X>u3mA-OJ3&#{~Yj=VF9Ji*6JDI
zxF?ac=+@g~!z~|VR4pwQeurIS@7`|vwq=X|l>F^Dl^E+hXJf7$+jh*m-!B_h!iZX1
z9e(}Ta(`#_$IJAq2Wn!YW6mp$+h@8rO8jZehde+9=Xqpnkm5x#1wZu}95X1cO`zQ}
z`01KhgmfWdjc6xcz2*f+{!C3Csm#^iPuZ$w4?9bPx0sHc&*++s{X}Dmm@b`D1=K&v
zaxLDAYogoTkeR-fT4tTFDLV*z*W&Mv$9Pq*d!K#M(^{7>O>Oaf5vQVHUlvjQa{)2i
zpus!X*~q-gULAhubdM@^g~~w0In_<uoHncAcLk~E;4(+YE)i9@tD_i|MlsHw4vB!1
z6`H>zkc9K;ljWz}b=C#mF)jX0lOFtEa_d(jyaNT+6g%4M$!uL%@I!ZhrFKcr09Bva
zo!l{Tz2?^rZ}Gn-t?i>=ZKs%9on<AZ5^L}9ISyGG_Cq@;h7wXtZ9=M&^F`Y>`~tF;
z?Ye9J7wy}1Uku*w?v4N=XI1E~RCW0rJzh_W{}xRzL?Ka<I_~m0@b?bgeXRs<JAX^k
z+eZffXshEx-eU!u0D@w`XMkOu!(Pp^eIc8?Q;I}q7L+lZiRm}Js9z&mIwhSx{3=Wp
z*xK-L`gU`xDG2lPXNJ7_pCN14QyF0=N@tnUNlw?3C_BbHnO#Zf))G7vouwFZ`$+tS
zANxkzbG1sv?c8jnq9FLZM~vJLF2<*??C3O>TUbcE-$#$bm&$BZo%}9c4KXaa>h4Pi
ztG>TMwW@V+wwp{uiVu5kbuuX^Jyzi%bY($80M?cddWbyx?}T>KdsE#bULJbZ-{MN)
zAF%$}-Snh~0-t1)J1=da8VGsged$zfm1M58*oB5~&r27s+v>AhY#f~ph>7EljYE5$
zA(q%)e6CgY=3NTi*_kQRWXH5i!j3KPmyrf(`_F-o#s!%8<DPTxF*X~;HXFeYlVUF{
z6wDkyf_>+_OeMujTH*VqNY2_(AS{Kbjw^Yv!rAybwC424NjuNuvv_fLGcFfAYjgmv
zH=|&5?;4E>c6DXV6=pu6<942IWxq%%VMr+$j<~}*78r@_pQP4eLq1ear6!1e?`rh4
zFgz@U@oNwxjKZCuns6Y@RcAGHcU~t>=W?;jESZanerv;ngvx)z=^mREMYQw?Vmn}^
zMH8{BJ6fpJ6y1Oc;~W_wBcb}CG6@gk`nDP^ayN1@Vm0Ni0hNc^$3KpMI#i*tvWR4E
z7kkU_c0RR}l~dX2=#JGQPWGK9()g@hkac;#(uNaIZ7AZy=c<9<cJbocGCfXllyx|i
zw?1|#57axg0AaX+Qcsm5CjMOYso3u?cM8nc*Iqm}8lzR2Z1p3ik|-RP#A&m@eo9d+
zIU?7K{6$f$*pAxrp<rlHiB|mKx{)+7)wE-?80IEo?EyZmqjMuB{?*VxfqW^FCq`Jj
zlcQ;IpCV37-z%E90o<29=`gdkX(xVP6p<RV(xQae-C%S)Ycj9jZ@W%54Mql-E*Guw
zn0|PAWMgXowgPogD@EM)_zSGUBVn&c41fCe!qyvA;MJ1xZ9ebz?_L=Ft$);^HGsg>
zt|F6W$<SSy7<|7B?4JESD&p^)*l?_OectKUQi!i?wBw#g;Hu_Qfv+l7k&Or{hJy-9
zWrc@m(C%(n(wy|x<Svfq<u#jVwW`%96>0LW(Dzm9hofJU4t}!9gRH;+XYq(!pZ!O7
z7<Zq!^z#!-yNC(;iXJL&;!rVuisKPX6~wMM(Pf*(i_J533i>;KmTj}x_(rZ4b~F7B
za+^{7-I~&+Y5~X<r}DFp(Zs=?BvhVPV}bqD5?cbw%&-6q{P!E@uKL1VX+a#7R3kIy
zLkE9(%9qOueHmd0I>Gqcb*|@4E{f%rr=qDrS_q+^Ia|Ta<_lC$;PtHIpKQN~xJ9D|
zzmMVq+$RhOdP-;KRX8+-TW70`Xx?thy0)L_+x|m#etnyyrF9IB9}r||ehSB1SRfZ{
z3KIFQ^C2kEwoN)W?{=P~`KrR~Me{s35WmH#3GQioJb7+AtGB6;XE?5f5iWCH5bm{K
zC_CT64@X6W4OX3bQ^P6h_QUw97XNtG8KE8W(meIoi4AYVrb^06F!7D}b5u(891_IF
zW>Mr+^N;rCFBK8}*rKUP8HEC0c<%nj{t%SE_lX}4``taP(WFO3;eE*hqd<<Z;-sGR
z?a&@anfI3(G<aTD3emv$h<x`gNg!8sbFIs%vZNXBq;f$y)&Rgz;{qiLrubLkL6%s7
zUU&<HBJ>R62tPz!l=J!sXlfVZ&N{XZ?%{RS!#MD%_bQTY;vK@)CfnH@_$jG07R{si
zI!-EHbp)?ppXXCu?i8F&7OH#mFZ`f;Ihs~5!}wSAov8XYTg$nu-K&{sq*g7Wk`rD(
zAhX9x8_GCK?ROkJ|5o+dI%g`X_Lcr&v^#6SD01+PlNzgDUW{fGVkc68a#s1SjcAde
zr*1}o*PMN3z2&RKG1}F>J2ixaljwev+>eZE5z`U2jM9Jb?S<_q^qRd4UG`H4E$?n!
zZ0GurTeLUergZbgD6zQn7R%^s#~d4zHJE%S4s(=KQ{W+<J|7@faV67N3m<b!n{fYD
z&u`V|RQk<^aYkQv)|-O}6F*F5{z6X{2t@04-PMQ;U)@fkw{bwwcPV<V^vBK}*i~@f
zKfE55=LA$mk39b=21~{lXL1E^tBcr^h}^#M)U(%CvlMTYd&C`B482O27D^2_GllDR
zlDdAu623p#5pY0C91S#J7vyr1Hjs7YJ=H(vxvG!9eZ5d-IljuqYFL-qSWB|gU;p&m
zGsB`M&NVWH2M5>Jz>BL5uXZY@e8Kw_-+Sr0{IsTk*><f%^vizR!Ms7$Kw@|B4ot^i
z=SfI34b|Q(PA+~5!Ff=&!<p1p^)my*QcJ-dn(pxU90NX&+GM-_RfV>v?!@2Om!xNe
zi!9p7FYN}1B%&#cIeGJ4ZEKyEYpv!TZ;#kVHNF~9cKwlQa~!8Jy~h|l{dE~@ep8B=
zD!f6mp`G!)aQmD0o7=P1DzLl#_T!(wKcb#<Nj7eEBtxtoIrGNtanzU(D$`z^gf=qe
z7FsM%h9bL)n~E;WEa>~lg<gDtN>8d`m-DH6ec5*oW_p+Z3KS2uOLqzt!&cuYt}d7o
z`U(>NF`V;GwHpv)pN@W3C8s2u9YViQa{pRP&iMENgV%tY2h(_3@s3208OFk1pft@T
z5dR;&FON-*>C0wZD#?Mpo$al(Y!~dA+wah{aopW~VRSDj$YVcMv8exk9sa_C;^05?
zo<I~5PeVDG9%_&iOdff$9PciPa9RIsZ`u|3dst2lhK;ZK>4-li!Zmm*J5Tci`UTG5
z=SJGrzZx~u4^}CgC@CfUb=rp>4%8E>F2;1NEzF!d?28=r)IEtHmM*18$=U@!?b!KA
zZ~J2(t~nf5l<JNN;}|Y@KT%<a%R$Mnnd+d1zJuwuvqC9c<?GH+?IPtkeX^;EWUHwY
zHAWiUx=QP~<#;<LD5O;WWd~}8(SY{f&)2|?Ps$z*pdy=w??18>(5`>PlRGtNPao&v
z@|LhOoQb7_1U1+_b9^tV;yPs|c$Tbf=({abPEQivwH?0N7<SlUY&8Pe{x&nn=Z<pE
zf?=1v8P8SiT=5(Inr;laALr@2+k0;#F4$z5Lkp5jQ6+N2g2wi^MdbvgtiB+SfB91|
zj$!Qz1Uhc(V%`-UGX|8cLCc@tzbv*Wd_+to&`HO|*U|70#`K|`wds1@@lD(gr*}#S
zz08Yy&jc=vs<9(d=B|~?+mnf?SUbX*7MPUcql!|R*ypgeYU-PCcN~!$9-h}VQb|K*
zLSjU*N#7?Q;0Ga{-rQ3<b0|(Ap=69N;NTzkEUjnUY`hwh<*OT@zO0_fs>zK=>#j{3
zn23`LekePu7)cv8ovpjSdM5hqnTyExRa@7DKTbt|D!xbj!9+h=3Y`hswaeL_5)KHm
zomR$ax9~30)sAN(wd6GW$=4=b+?Y@;iAV2XjWEBeMyY)>X<B-)ajda!Id*D^y<^^d
z?qX+<N-;pZK;=J^^j)+Nz0EI3Co6xlEL{IMhY_Z`#790IOd4^bC$L?4QXY~rR>QL;
zA|qe7b5bYEe!oZ5$D!50ahh%KS!G#f#;G=JSIj0`ZA0k!(FZ&FSi_IwJb5wWKavD5
z^<k#yd-snYJLikMUtSRE!`6<*b)NlY9jWV{6qdR|#OSHx#p6O@nlpnm=8QHsl<V$V
zopiRjaOBePYr_iGX~R}Q*a_LZQ?`(h#|s-?TzH&6nUAMMd@U+M>J2+y7&pG|G@@+m
zXeoJ@1Aa%mpfZ`FPju3oSpeON1KtC7s^6mzrB$<8R0~xhp}J5u`in3~y863U9}3%H
zq)tRS=nJ?Bh<RSfxS9P(ER^K55YH~ZHwsd)J{3fX9BFt@NB@_sy6aIkj>dl6M7|e8
z!N-{ijd|3@F3yeh*X(c%kHH`Otgfj_2d@u_#PqbCKwzeX3#HC`g`x8BwD{B2XZRVy
zRCGP`gno{&CD*tIhd<n%cI}M&_JE9;@^{dYC|SfG7~5_&GQzq?3#D+|)(GHwJB>(m
zzwr)fxy?P$OQet6Z919n_dLxJt$AS4hHGj|;muQHzm7dq<E0d!qI?n5vDo|pNfm2(
zzdhb`k0kz%f9}lBN(9Aw3|pE92O=MelIxfQac3f1^(b_jhgt%T+>t|CrM`1ogKW_<
zMd>6OYFNiFcd?(>Td_GhP*}9NMzB9j*?Uie*|m4y$tJl2DTKzfy@%OlPgNXxFheAU
zb#dqCRGIz9q&vTt%kRHW59M;Vpa_BspCMn7s;q{m#2Nc&gsh6hF@;v%A4scH5qAlr
zhxIot)0#-Bm+e>R*XQ!t8x?;j+Fw;m?J~kA)q6@Ax|0-N+;f~3WV9#R+rv)oJX?SL
z6j_ldff=#O#xw3Y?%7er1&R?J9za*!8(rUR@eVphJ@<H^Of#t#Zhc$LzID4PBNHkp
zAlz{5`v_2U7EG84V6sA}>@Z!Ql094DO#QDom@-0=C~+oG|8L^^5CuN3@Ml87%|^24
zTlbkZ8xOmmLpfN<Y#kvt%hNmexMc}<u@)9yvULm`3f*UzC#L>E1oc{o*#d%GolK7V
zURhRsHaRZT%Did2RnCU`C~~l-`>=-lsNEen?KG^H{D6at=D|oOi3$}~G6QzGBRDr3
zU9Gkr@NPCbD{9@mu3wbUj%5Tsv+Vi+=X&CS-MX|NXb{#jLA^V1v!BwEhzD<%3)RLw
zZG(SEVCwq+Yc+-))6!^mvB^!75@)-j)*?lgP!e<>VS^SP0n|jzXggWMYH5|TL-+f!
zP40unMf38>zwY22W4uRXP)oCy4r<)7T~cuUZ|m;=*t)Z0v^(vUO&oH40OI(Rp*KAb
z`tyDu8)zXWF{;d-flNQ{!G*u%I9Bs^P>KoS=S_<=;4KSsg65?rYN%XqdSB?6M)HY$
zUPm%v46P1n(#mhI2@R<Giua%HD~#A@=D4?=Rp+KGUQka+NRtA!K3csubIavahlV@U
zm{t7{F#?w8tm?Ix49amrs01Zyq{a;%3dECmnf{_j>ffqM^Ex&cVpv?@=dXYL6n+L?
z5KI8Q;3|w9KGy_m<rNse+|8ywdRqYX_f5Fi+U7A(n+m5C_t8}|?++MPw;!3jC;AOH
zu&w%2O)!p@D&hI`mEKX#EBEURP=Oz6UXEsVUUBkGd6yjywfr)n7Ix9~V&Kd%b&q5)
z7F<GIvE|(9rVBJ1k1KVO(A?$U@lHJ@{IT{GYiiT)paYwbY^S|8G7(T%^_(q2F#~wE
z18Rq2wY)*sIhOZ1HM>8WGaDW$H_F{ZkrMYj2tXhEBXeIhIp;W(<dYaRA<a}=e<oCQ
zk%z_r4%Mbh&aW08d?;<+#xTtbNM(fj==29=KKRjC-;%i_MhA=(c#v(GEK_mMKI-zQ
zc$to8+UhSV)w&}EMz&`CSo!h^zjx9}u1VJ&4Srp*b^C_sB*7u5Zf3Y?Y=a4k1gA@4
zU?#kU&@I?Y&VBVB|HxIksn#^!wUZ<CjZ|?DsiDO9`YuP~H^>cS)ebLt2&iVu$)8sd
zr@-u<f3u7A4J^D$1l^t`&qtUU0nngCZ?7bn9vG7VhU+}&6-@_dn0DqMQw45@08+$?
zm>Ib<71TujuA;EM$b#{c4|Nsf?w2rKU3dA*#Z)%C#|XO!gb?vte6XLcfk&%ed4AkB
zXp)WCksu--o6CtI8&V&=sPV(?7(nc8*3?U6%+R<Yj>Tj1G~Xkxf>%s`26K>)PSzat
zw3bx|J+f&zaX`eu4bPXm4q%)D@8zXI8@00M?=c9`mdYzaK^XMK#q?+TSh3OYT+>$0
zwQ5@KTeVb;vWmQ~x;aBs<OfflJaMubm~+}&5_UUWBw1!p4jNkSl!>H!KYP;TwTEpe
z;;af=rIYn$0+VKGAyD}_Sd(!i>h;?3?#{XS>vE$Z+v69lEH3s3ZVyoJ<E28P^#=Dw
zaD&MqmnvaPWV(0%Ut83eArMhdXLogC?sm15aJT$j+nAF!-{yWSyRufIcSZ*>$5SW8
zH2QL{Z3GxJR5`Jrp`7siwO5wh)@%RVd{2N%4xj$vu^mp<%s`YNM&-i^L%u~#;E3nK
z(?)ZFC;GFRiWiluB>PIR&#{i*_Sfq(v*oAVdrxdO%I!W*U2exF5*B+KHK-Wo4}SL@
zzT7j$O}}iq$}|2W?DB{~Atm|e_YI}`haXpTW!NuPF+wO9%HRrDv$YO}s=JL&8}-A^
zq(V@yn=UQZp|&Vxy2&$w6fZ@w&U-q;(BX_@SK*oPF=LpnbMIG6?UF=;!f!SaanCar
zu0NvtOSD77DR183K$L0hz^%KEqujjGCl$iJdt=_^UdFftD9M5j_j=R`Ds>dnOc%7M
zkSp(-=GE>s;GKJ&N()rf5GHf4*Ewykv7EVF6Ruwuls0^oZ;!GuSv^RS6dkoD6vH<U
zcE_ia3ukxJ$0HK_HL+LugpIN0Fp-^1*q>^;hU;=u@49?DzUj)6(>d+{{D3>h*7IxJ
z(A^XJLA}N9Gtg%=d-&(jis87UdT)+jm?@BHfmbe-n%8|m?=r!oS&Y{I4?J$cbygWP
zwXjV{L&L9~<pYC3d-<MJ!v2ySb_Wx3@*wy26kHR`f_>O|>mgBakj3G7`nO%_tlm>P
ze>6W_e|(<votaVX(c9mzCC}&U7u<x1p<$K;6{zEWMpgCZwp0iTJPi{rku8XJ47}Y<
zyFttEgP;tml3ZVQ()HZ9M1^D81ySlFt;dQxC4&a0K>v%8MK*`OWh>xk8X=5<R|iC5
zx5A~fp$6KY)o)S9Kdlqhf}=!tuJ?5Bc7;6TV3m#73)ie*XCEIgCoi$0v)_@k>z@*)
z0tGF-hoDKL44O4c*c?=Ew7yQu{kF1`qowH&)yfj-$xtKq*8Aof-&F4?bgWy#j7%ln
zjx_1WG;uqQta!U`dr!h%D)Hiv1`GF+${s$ig=T%+fllKSXVX&Tnr^nWad|@i_lqHw
zbm22cu~9c$hkM=9o}?e)4PpDYAg{#3TX$m}p~-`)O{tB&UmicUEg$U``2Cxw%H-So
zNPvY99j)?QuXO@rDHC<-b&i_3)vnZ^pklo+2`JX*NSR4E)KX7(jX1`C1^KAGy?^7<
zDv95|;sOgy=o#5;kK*lkl}}+_+Y2TOKct5Oi&DSjCdBhv8J&3FJ2>O7d@%Z6yvRtq
zsRN_8*s&&1!{FnE4Wqqn*h*2bE2UvKe^jn>ZDo2$*k!o`Xlxh{uQ2>!hQ=tI&mFex
zPjI<!=B&zZCY-L@aO<cq2=v2BKy{k}Zk*+3T#k9;W!}WnEb=QiCt<H$ZPhJWdRUXW
z*ohHO#(Q1S8JA0E5e)x`Zlj*@X1+Wqn9%mxDxRz3k1f#Q7`I#CfSqwr$xXD$xWSKY
zW-~xZlQ`_uW!?XA`vKp+U48~`ZTEhHchQ&ey~VZT3jf&={jN)Xy@(A}P&>;sjB7IN
z7L0R`l_8QIR2=yxNv(DI9Ipnu$Ktu3?Y(1xzNzbm;e9(Ip&N}|dGKFbHYW;dT{~&V
zc%^7+H<M?WH*N&QmX*e+E7+y=&GD0oMXKWqAqMSli#3?A7Y#iJKU8bxQ+>891&5DD
zBorl5;fnaLL9I^nsqN>j7$!BQB+?>Us?p}{EDJN3YX;~}GraG48zzy<Rmbo0x3$Gg
zG1tUtXO!|%1ixp|m3P`bN<jbVW$mCpnV8&g|Bg3f>L+p2nY&p(=GAzh=rLi!4|3|>
zz^Myz!d@O<?)THiefl?iMjL;7n-%<=*PVmm5ZsEFyZ5B?98#2;9q9x4H=cs|3ZT|%
zST>R!?t;FbUsJsY^C=9)7-hT62HUSJ1T)mWo{k0bzu$O<ILU@HSisU#l5|KZr*v$;
z%n4p?d?%31`BM26zRDOFw=_bxLTdu!`?#80k+%C3W^uer{{JB*R02O95Dn6mV*D3y
z8Q@hH`1lBt{3;c@mfvgcTCBlQ&|tt9cwdU_aGunWPKxPQ8u**96Yg$j_oy($Llu8V
zj7AhQBCY*z!+!hY#!Up4kx>e(PzUj@7zgd<+V#(zf`i=8U$6E<4$sP3{xg-axyI3k
zgfsl<Y|#98*B6I}ZkjLl#;Gg+k5e}WCAss}nU7yfZ-kWa5Em!i!(9_)FGPt$;@gw@
z?4sNnELyXIskbW2yZNA*6dh)-(*!4?0%fptl*&2w7ox96?g{e@@;5am??<)31Oo((
zd%=Ln2F&_X#yf=Zr(*GZj-JAi4>r&1H*y18enh+7sEZ?*zhpFo)Mdx}M$%k(Gk@Ns
zMl!zz*?s<QsoZJnS|N!?EayzO<7j{)yV&;V7K=NWO2{Z!z0R^aoy+o4`YG&c@}$*x
z$#p06mm91$c%-=KgI0;+SiZapI@~U#W4J5RWcXdr(9ZHuWqo2aGJFAc->mum)^avn
zjU3Zr&%7Y%!Is0S;izkCBfu<pHsPj~2TF;0&jY;@Ztr~B?%K<CPQATY8x_aYG+9Z4
zT<OlRs&~P(E$cjwXuiXEBZW0V|CPcZz~!U>-3!?ph1t~!rlm^5Lg=IHQ4y7&hsO^f
zgZrnyh7*c@fi-cWb#3H@vY00>+0^+AZ19G|zs=B*U%e9O&sbX5?BRyRn>g)@J7kG9
zsuy`Vmljm*&yoxwudiE^2yBR>hmjvL4E$wnrH1%emf}5TJGsgp48^c&>dEa?nE$ZQ
z6LE{HA9cTa*Zo2$0wzwoBC5R2EtA%~rMB<`UsoKCXPtZ;=0RG#t@TWqZZ%M%4vO2W
zqVIR7^n~iz8x4lID5g!C3B?xbYi-^#PBLxxET<z4jcm||4_QTC>u|NB`)xf`ul&E-
zd&{t>pFi#w`H6(mDcv1P$HJl@D5=s&2q+;)cMC`gNH<7#w=AG^NGsh-cQ?yDi}(M3
zpL0Js5AN$cIOqDwu9sfl-DP%WzB8YhdA~oeQHS^Lbl0toRIs+k0jK;C;#p;Nxv953
zEcwZQ!p3H;I&k&@-6b@{{HqpY*n0n&tmXNSDXc)6+7FBF7oMY7=Sx7}rZW7Stu2*^
zr)_6eC<`NyK!9CwykgMopP!ixxWj8C9!s;tFa-;la+-Tq%+4FT@Cv#VMhDx)`UkLS
z4rR~&HyK|$llPa7BKlnMPua`h*cZ+YoO+3It8!nop>H}0(V83=c#rd@btK|SvwKs6
zcIN7VwqPdOhponReD1reFGNk!4qYKytg!N%VP_*A&^U<AS}~d|zz)Vopb*NBAW_?E
z&OHwjs$|)nnR+*P%>#m)89)B@I3}fas);HVDg@jcHqgjN$qYu~<j=&X-^$=AgD&sc
zn@AJi2pW@IKk}+3`T(!x9Q>?cx?l};zZJ&*qWVwdE&*cqS;n?Sp1&*LX1NFZXA0i!
zC_LNb(;ITB^eZyQK=^1@NY>Iu*dovwx(Q>#>b(|wZLE-~JL-kIGcE;+9~tVoO|M+C
zGE;Bo5g+a^w88Br;W|J=Y?z4{pY<{gTXR^T(@!)ik7_;&_+97a797Xg<fw%}G%aXo
z3)>eonP4cN^v8=O;o+tr)q?A}%bVg@;V&{|-|dd3DA|~?%$Q)4w)SC}=2Cm5fi;(W
zTg7*==F6ZnO*_uF!D`%GsancdEZd?tyi>Y{6H~j!+;LJPXF7JAJrZYCWFAA=Y)lF(
zFb>#C=YE`%Qre`0-*ng4hlfgQ)}It_m56A!aS@9payR7$_9C!d|E~noD~4<llN5D&
z#2P+KEHWWtFx(am(Pe);&1Rctwvf``(u9s1GVF?1CAk`&mQ$t0N1iCw5>?EDaEo1J
z3<OEeU*8qu@qw@SL!nV_l)Otl1R45VT!@Qh1En!;opf}kou1kBw){wuJ^{=f5$!~p
zSr6L7I93+|sRdyP5*M&?jcj~@6n(`SzOIj`z~tttEAhYW>(j{5ROkv0W+C-^WtSbo
zKgTF;n-F~#X?|)ZoCnhQ+AW5F%za4mij*h?fvu-92K`YY^4*($F8E;g?7rq67OE#b
zO2aP8*9D?bq?R0C3GEB8&ZJ-#r8KE39Oz4De^CF*;U!YC^nTTSip~nD?bbrxW<cBR
zZ348osr8!61;}?A#0a6T-@OxpKYn_+YuGQ3-zrtf87n@;tbtUUN+K&52-`7IQMNqy
zEKsk*ZhzaF73wfLQZtVAS2juT?{>@o({>9|nf|YYp8EgTTk(+#m@5ovl}B0dfTlJf
z=-W5iO_O^ff0h4_q)}?-fXkvn1Zb3#NFP;@I^TVHepfGn_8I7`v~LCPjC)pMGI(+T
zCu!MJHoDLndhec2_4c4&w{99b)Us<}ZHVeax)@>J5sg0>yn9&odhW8_Iv^@Bf1D&K
zlI_3w1U68J<uJ{vi0`OM)UYePyNBF-e1Px0Ed=}o7E0hAT|xV@g+>i7f6`!nj*s+8
z#4CcbS+oSsVF6!G(M!4KnZ+324$9SR==pBT7Kmce4BQ>>?0~OHkmv0?9w03|z=ZKW
z2}XJ=fJwRR+L%!0N*&m+3_ibpO8)T~N=$fhFd=Z>fvzQW_w6M%uv8==O-L?x7~DR|
zqQofA$6)?-_ZSm`vL!uQtVIPC!x2p|Y|4R}=Wd~v-1PuiHZppYDJx)bnE+0@fWG@=
z6Ze6M%^kH=;VV7TQ1K`Ulx@NfznK^qDmBxizskJ9ms~a~T$DDxEgK%FR^Te>UO_sJ
z^nP^d&4bH?&hLGQZXGidplp52U(SjpLB(KY`|(c(qp;JwA!gzI2RVlLWBfRw09Fn`
z7VymuQDsh0!%^7NesB<GVchQNGJ~=S8njuVfr?RQWy%^-2W>iDARO`Vwru}jZ*q)k
z9a|I>5mIFC>VKR6|Mlia-fRDTytM&+iiAg!kp|h*{`)BUPe=W~nasS~F~_x?i^TSO
z7TbN7fJF*v0{>I-?A4_tVd}jpD^yi!;FWu8{x>2k)@OK!gZxMs19C&1TW$8-e{V00
z6l5m^Z$!?bpPaC7e<ko96&wuFn6>~T8s<xUCUCj<@4B?cGxVznvXk$HA*cTaRhLzY
zmR2#nJ>ZN2el6gnL7x>sUPrl;0$w2N^$impkD6KXSv0s@^LP0;ej)I5<@PQWLoTIu
zUoWRvVc_mM{{)6N;I=|;m^+XC{pP?B#!hqniJ20_C@93Qz?UCS{ChI9Ifjvdw9ElV
zHLyv*<YSzg=pZmS+8nYfZ&#`Fy0_PX?Hjz{2tCmn+{^z~2xyxB(+atgd^?eK$Nr{Y
zkzMsW@A6F@jK)x!=##?y?wnUfW#^5fyy0;4dY<X|z)Z=Tbsx}86qx^73B$m!OJz2G
zLskRbdjDVn-_f70!+LvIOZ0p|@4Lq~)b)6SDZ4+~m-pR(RFFy*=3`1r?GVY`?@Vjs
zO}eV-O6DJiyn@YwZccY+?Dv5^#p&{1`xoMXVPlVQ-lZ=2^2Ij3?S#PblpkL{=?|?C
z^UYW;v7NACCtqexIdJHLd<4p7KTp6xz_(7+DVjB=rMaV@CjlueNNbJjmxT+jRnwg^
zr7EW>87$Z9$!_JW=Mj1D{#>Y*9=?=|=Fc8!EKAi9@rvxfar2uEdKhv*%tJA{hBr%l
zr;;PBxy`N0S%>Dk^+DO2V8ixM<?5-U?17EiY}Qwq^)tBP_XkoXatP58Sri)FFC%#W
z1dW)oIJw{K+F$+Hb&d7ZZHVVi7JZ6GEgESpjxj)uJC3mCa$Uv^*mjb^ncDr;DrZt4
zAqv9fJ#}l`&azRp_1WT0UPqEU6NjYP@#XMgG&(HqG=SaV(X609PS|hMS9z2JCq23h
zpBoN%U+^lNZSF$AQkn#EhmRv|cUActnvv@os^F8wrRYjYkb!8AIz`;|CC>h0w1<E_
ziN}RY)I4GSUVnb;W!}{4=ViiND-k8OVPo|(`{VG$$y||Zn5%?efa?);8kt_pKQ(hk
zHscrc7R`41Ehl1!Wy25s0%+toAna+KxVtI=JuYi2F(E`^dZ-_S+_{NaUwz7u2`O1M
z9T=D!S&6C4Yt2I=;xRD1{tm@K1)6}fNyPPQ*HY^jJ@I^L&<e7zHt}lbl#{^c&(AIe
zjzFTBPSJlmj#J7dF~#-IpO3IqS<X6tciSYdUK7%4rk+L5b8l(5>+aPAdfq9KKQ%!O
z^a?5pS{jk+zw}QHlu`7P9WG4@jOYH5v-1FF8{VOeefsfz7JS@X@+SL+t#ypY-o1(>
z+?^-e(Jjlf#S}y=PoADv$5ZSKWi-PVFZ<TLOq2s>V<-;ZwMTzyyT>txPV*`SN6P*t
zf<ZRN;yk#^ew|boCso4p`K8m<a}A#4oU>i#Y{)aH06*#H)e>vv{^pyloWfP*ktkW*
zrLnX7EI1?We4R^i7YF(^8L|K5e1{C3s#u!R%-x{RG$CljVQv8g%r8I#nBboygvTKg
zFfZAhsDSINfs-bSU;4r>(ruOUIU;HuTegPIU@zZSk!&^Q;XdSks!_y~!2F{6`>*Ei
zWnW8l`Ub33v!zMJ%2xb-Xu5deOT=Kj^>&10IuzZ>YUro#oAuau6C#n<@8lx5`Id=W
z_k7&aEcKf?Q(a^pDKcLfN7zO6Q;K}U<&sO|5^B6LV5{vE_c#a~rxEjH$vrh^3vc+M
z&+bo@6WMvCT=9E-eP${D*v^Y)y58m)0(-j7J#{t+KV<k_Kie~_>ASoBXROpDj+$57
z;JTV$2-}L$a@I6`T14BNl4<`~Prwau;tNZYw!1lN1lk)%@p`DsrHR*$bOsNm?hXK@
zUDJ@?cEI-O*p<Y%=N#gm!KDX0=ksf%*O`CO69~}Fq89U)2jjveE`3D9-z=&Xg>;^#
z@2-SJVT+G>zi9g)qK}=z*6zL8h&e+(7<SyN!_$y5Fn3JD<Sdjh<<PXZI$F(NtbBj5
zqdrqUNrdz`wOolmkXKl5G(4Q<%vZ+!82X)uS<KCLx+|Ol8ey?kLoVDGMOs=KYN;=u
z6X$kEN4F!{CI7k$Em}$8yQ2fT=ahnPk{9z%qp~gwXXY#VRQcR+5trU*4k)(PQ6(kG
zs=ZXQb*sH(W`+Io6<h9`-TSYc4>!CPFJ3e&aYHMd(2tAtwIvES{gzdWo+{?9l8Y>)
zR-ZF83|F#*r@&aACx3BO(p8)|@LaP`QPvc;V+htTS?oT5i;kr?2$hPQa~!=Oewd0@
zvr>0wCjY})qqZ7_Q;v+djQP*23o$QMp$mg<e%JHGdKDPwc!AKSj9QXotYO=Fj<6Pg
z4e9<@K?H@PJtyp5-%Z5XE)OG8l<gv>;wp4_R@>ULbr>t2uTX4E6p4q!rr0zK3~y(N
z5)2Efr_Mi48se}ln#Bk+HQru^M!Ig|*4%L2*AKt-Lh8$RN!3YtA>^*x%g7ud%|5GI
z&T!{1!a{E&bL5n#QC9TO_1J<~i0l}W0xT6T5Z|W87HQ%J@boipzJ@P$2=Hw76dLc3
zdG+nTEUjrZHqTXRHVLEtrsp8ROkEQN`N+H<Q(I->z5n$QLc4#_XsIW~GIakRvD(4s
zbfT$HH+G7$w5!qxBJO4@J;hv{>t`=sXJx03jJ=d8--v|Hv^*AHZ#aju#hmf}YOT!(
zDIEzowTKh{=ZX?x2=ScJx5QQe+GSD;1Oeh|g0P!a7GY?AR66&Lk`fV8^G}}Rl&5=M
z131pb#D5ta7WAThdYqfHM=;wo`hZ>K%A)VwPOVTxN^H1;Nrmk~?6K+4&Zg!0Q0l|!
zU8vX9Q{O~2S|zd4oQ~br-qiuq0iMnG^#?i;Hs&XFT-M)!hpPzcg^11w#@w8iiWKa1
ztFpS35p~<@<E_x`;K^JhHeU}n+3)U1oBQ)dgJ)ewGi&dPxi6+beJ;~`bm#J-!FB^r
z`?15<2mM}RB1^CWXZywZG+RAlp?O)a%iO8Uu0@LuZ$Vz2QOIZ3-pkKxvndp9yU-iO
zcU=(yGTd`)bCvVME5&>B&F0jponbdh&NyT@+crGd%|>I~sS(S(+nt$2Qb$Gk_Q;K%
z!z^vk4_a@3D71#~YSJ)TTEnYp?t3TrPWF(IB0N7YGRZBfZC%C01{Agh#SOAemnhB+
zjw|W+m|tbbH<O{S!cyQrTQXFtQ7h6l%a%-jJ1p8Ze(KdfyZp_?L?M;-+jNUrr|#67
z;_ynqc}1Be#Xrlsw2kJ?=wL(l>!IZBRbTlM<uMm$EU~eB69r~cW<ciEQL_7Mw|zP>
z_(1bgmXO+=TK1^rL1MF3n$y9?^8F8|6PFc249Y{wrt`2q?MLUE<9W-eoI69q)1zv3
zYnz`wE`8<oQkU>eRR1cKRS`mG%Mg2$Bi=vAgB!P@$&^M!1`B+0`l66E5%1BKGb`0;
z*4`k8WiqaQO)Hxc=r_-(V+GzVD1RScwwWsKHI9-RVIRE=PCQLnB2pK7CEjV&>A%dM
zZtih@wwU%Or{5@UO=`z^yro~UFe`BWbbQE0)Xn2OVvBC!ihq;u6>g`Y=i4aTirPSy
z?Zc((`Q+Lbqmx0L3KRdhC#iG@N6LUt<u8}6_JW=2LtyLsSUdH`IzYgm%5HHpwGdc#
zJ=+W|p?iC&8+6wTXNm?wVYT~LToS8*frwLnC|#I?9f7^~Cm>B!piK&cWP@v`!%CgL
zk!;5{rcNV$dr#!2|Gv8ORgDc==<aJ}PqnnBWj`XUu!x(ZZAjv1?Ptml?bwWc`*HIZ
zG|Vxf^*7aPPmaZKM=ZPNQlLddQuZ4?aLmn<-p89t)C$4|62F361lB~K;uPDsf63e<
ztF|S5-={LZU$gabQI92+@R3U^ROwVRdataA(oldcwE@228bZy0cn!`7hVnmnK1STo
zuwR*}i4=7oC(xx>U0Px)ZVWwlO}J<^SUl$lvz&q1R9V~Ox|j{6jU|&e0n-w7c>ZFD
zSct4_C$@uSw3lX$=yCj2D4gqBvwCmvESKb*veLD7q5hFP%TcD4N!V9$-b2rZHD=>;
zVCTJ~ch_)%M&tOZp4yoM!t7|pN~$}b+FjZ{mjYiTEU_t+&_EuV^ZXH~F1xmpC;U4C
zg5faAcU!2Roj>h12Z1TmLi=@ntJsMDv#9s*_veTPJ-?%^7#6d`y_m5lrR$rX#Y=u#
z)RQezIzD!@Sm9`y-+K+@e+uNetjUH9CoI8ZcN(*IJNNcxKYrLV;1wy#F*tptTKs}E
zmr+FUAb=auZZ;klB@0d7gGC28S*d(Ivy#lTtq90*oI)6Ose0v(Q;k%;8&DLn?;G;%
z@S!MGkv7t`<7wN=;yG-O^U2Yg3p&zj1X*?_JrDv$Y8)&5SMsX)@>H@s_U<})o!WZN
z@zYt@R{k;~_ZHJ3#Xaw0;+&Ow-9FDgxLmjoYS@ML&<U|hra+BruJ#!*c_J0)^E?|=
zypJdvVNWWiGW|H$KRBdSFe0`~=!S$#Olyizz-5h9MmiTqZ`8RM4?eE);S!9PdRktv
z(Tn3D@%@>NzVE~6+3ek^!pi%FupH!pymG;3bkL=T?W-z5Ro6S)-JP=fLSoVUSV_hs
zAGx*!d;Z?dR!!7Oz0AjPQUXmPS8baw(ywIOqF_Q?mx)R=6FdQxbrPd<Et&eh%1fH=
z)P!j_nFl4|nPa8;-8_}<M&pqmqbuLq%qWA%cKM1hHJsP+nEG0j#(kz);G>qno0BYS
z27QVZ_)>Q#;m*g2lWT&>3R^*4qDDOToeg(<>PZcGjRl*XGpM1kw&yuU>eoJTypsM0
z3q#d1bm!E2cy$tdO?Dpd0#_l@YhX>418eUY^sY~uzN&~Xh`H>;((1cVz)Qld8M*cY
z4wrEn+?)IPEHc@9*2%wT&3@5e)Y(#r^Q2Spb0p~D@D(v0%&A&iJA}O-piIbRSp`G3
zVvX}-^c|bulmky(O06;3bsN*;Y1GBrM3}V6_&kv!$}HUwmFhk3?Xj-blV+xcOMFm&
zj%C*!!VizLkpd4gF8qHGHJ*GL`cLggOHG!tUj6gao@Y|yL(`AK$-Of-L`zR^>cQYc
zJ@$T3jms<vK~9qbix+*9v~fv)u0+Gk5&DNasx^*wuFTRuO3g`!!k<F+fy>Ei?)~&@
zu0Vayfa(JFraG%=oAT#ftbVv&DXabzLWwRVU!b4Sp&@DA{nbvAz*!bg2dOIEhrJ%F
z6Yw$%=RKOLdCFa%#Z{{C=i}m3N3qd-;%;YTQQg*t)5DUvip0IQ<q`!mNyN?0+z$qV
z9xpoY$cCO8cGZhixApJp*U+qyY)xiY{@xhjJaBEmx65X=K4jzuH*}To3F7;wn&LMK
zmyQ;;+Ad#M5>L|_pKzNLovR%eB6}Sa+3TPd&Cq~}mJ^Cfod3P9>f(HwRm<QmWDuIx
zJowH!9T$6dx;(@pUoXx;Nt{{RXlky-H?8~fwRP#UOlqU23YPv`F4O)j9?fqvbV~lH
zNLn5pkeYgh>93|+K#D%;7Csrt*V!gf3b$0@Zb+ux0Vksz_6~*T#uHl9VZc|alk%{|
z2&~eN`o>z9bm;Voe%PQ^Y#!<k1k{$u0JFxm;60x44c9j(Ie*x`zSMK_&?p97X$O1Y
zrva%>cr<K%Ax&`E*01hCh0#Q~PWcG@pp~+?v_(pu;8e04d%jQ5MNQZdX4+b$uVk(l
zeT{U&{`&f1;m*{qPGZ@0KLjQY1~!=oHFIPvnN7EmUM7>_VA`nf4~jEv-K)Z%FEbYC
zM-OjY`cOrn?<VwXXP9%6m>{uATgr3&WZZ%w@=qZ3qZFN?*_LTu&sT568Jq7O@>as4
zKXmB8^7RaQuJrEIx;qm#vowo0vvSADF2A?$ES}h0X@9WrQ(RF|LXIKrPubM-g3+qN
zDZ4&AP$}-@r<C$)<PP?3me!<Iw#(TGCWtNAE=B;*Sc{16uv-Dck~paY7Bjux_EztS
zZ=QOR#7_g8hLYkRQOca8(_a0!mZwgLGPxb2mfe#I52tD7H1j(x?2szwGYt+giIb{F
zDEdMp`N*p9{jl)U<<srz<epF1$-^3`v-}<8tg63rJF7NDD`6On_EE{h2JVXKc}B%8
z94$HLPx={>#f(xssy89LaG0U5C&w-2T@q&Txv6svj`4FI2m~5|s(W8^V-g_Qt>uN0
zabK6S--U{PB%s4ZgLfajYV*W3ZlJCX!NPHalt9|yD{nj!)k+Y{U^`)DZVAqBODL}F
z6A)LjebD^$pdPVhX%agbOa*&5Cj2<uKVaV3R)ae^Qf%7%lrrwEV@gu1XO)2m1oH9x
zlbcdQ{H2uhUjH&=IwU1GJ9HTviWte&dr?l>t5koA4S3)70Ztz=jaMGIO=+?Hpo+@I
zcA0<wIoC>J$GLYso+bFu&qj|eR72r@-#&~HLE$v*{px<D)g`XsQ#WFljqSNC5?M{;
z$Yb!r$kd=!3IvM*>g)y$xKp@OyX)9K+o#g)wdS`1aI0lP3s$|JYPUZ6wxk1_%#4aN
zIo-3U0$_bK2<9Xgg2@dH6p{{3iWmi+5}As;E%?%F6O)9)u&VN+yqjDswe&}Zbdq1<
z`Lz%KN)mKgT>0jz!jVpK{g3MG;yrytI7gR#*<d}B<Cr7WP5<Gei{D&qBfUwm<HRQQ
z5^F{`9%<jCv<8EHqVkbb+xqf!IG@V_ud<<W=Xy_SX`Td~V0nCIB%8*aeJ1fz?h?w@
z3)f#1zxQ!Qwq!~#d@Gg^lGM=eUUB}_CG0{vOZS-yM>BCUNxIx7p@}aS?UW0((Lgqi
z>(Y49c=h0$iG6pBRMMYetDO(b^n$okTt--rYxmBS+j=jHtak}`P=4qi9ul2PbBFit
zcAR_R_S#=Fkr!SYm*l$BG_r+v;_d`37(83*aDlnJy%C`}7Lj!W(g&k<%cbeF;^8_%
z0mp}itH7u%t{)nbOFw(+t6{!aF^~O7Kd_Ja=Gp`|dmwh|<9O=b4LB51Wbtl_+_q9W
zmEw=tmZ3+j4L33fif--IS*6(D(9<<1ilxv$z&V1bhCHKg2wiMkHSkD|6ZAFd(rxpV
zr{=+UY;G-fUMuLiE#Yx)I{Q4rjgUb$C<FcdCr+m<(tW{o0X^=aqM4<Gm2h;J&f{Hy
z%C}=)a(5d~+!1R3GjY=b9pvq$QIDbD_UXXX12C&I+|KIme`j^MR(a?ZT5{+3VI44j
zkp5vE$`C33(H3eu3k!7`TIrjvyVK_OstisYBK<<8tMn39WLd%gaM`kh6T?~fMc-&+
zbzX|+?(BEw0O$<N_Lbgh2enmICpPVlZUStdCYJ}k6%;_=P^uPP^$2Ei=Vric(WH_?
zt@utgjfgU}`HgjJoDlw!uA8qNEZO{&t&mmC-{Pms{)T9H{qEGas^q|4PG`#=F~ere
z9c-|p&=GXlv+gjx#>3g-Tkm)S!zlMQ>Kq7St2>aO(5bN@V2lj4@VbxdISMvqg5q8n
zKTa2!?YlyHF@yBCwU3QwmJm#nBH>=kw#&RrGo2&Geb2&aBnF+FSI=o+BLzm8@vRq9
z&eo7g;r9~8GB6Xgn~&BapR3MkFHhS=b|m{3<~Thl7~Kl0&LCVkKipkP2us;dm40a&
z2At@;&a%^<*oV-kDicM*6(5SJb{F1wRF1=$a(8E8E2Hq%5aK;No|PYALih>^bR4?P
z_F+Jbk`MJ?b=`mcryvDvyZv(i9I!Wk_2Z<OOKluc%jque1EvEUFdgI${%x=H7{Bn-
z^Fn5!c~2cdDZ%+_)4K9sp`*6?Ro{BB#e@}Un{5${p*wqMDWsuVh@PIX&r;d<w)53D
zCEAk0_E!`@eg&LXRG&DAxQl72g~Gmo;}uhUbD8a~Utxpw25TS1><jLn8_wYxV+EsG
zd@(l_F`cRR5j{@UtLj52H*dDGZDj8)&kQP4Ua6E6w4KjGeofiD+Qqi{G8bn={kk;?
zE>hCbUa0+NlV7zYLm=;kOMQZV?F){4(|Ym?2E;j>lzgzR2y*%xp3W7t{yn*b<c<(~
z2iBb*`kB=S3w_6v4-(kg@5I$U1he{pRQ~8Maz(LGp3K#ZF-bd4$>H~I`WExTI!cUV
zCFae^CWsGS3rhyuk;zW2giH0cp=ddUQ&sk&edNP>B}6M`G~r-}s>|N53j7)tzGXCw
zI{XrmRK|f_l@uJ`DtXV#8JHn+F-dZr$tycf*NPCQG@%&#8UELYtx?}y_?{?g9yl2H
zHbyGCh8~{nm>X2<d-9xJzx`PvLc)vP5E{Ar*)i0sKKK35xL%dM0_$o$TcXkBSFqhy
zFk(|S(oi^~v40!m@GLMvgq6##%dQdbwFZ}=VEav|gN;%DcJ1;!dp^Jo0-ax}md}WA
z*E_Mu>Q0tI_!9|O1$IXD*7YVn!xEC=@5E-OCTMSc^wbXWWn;R$dHH4Nk{KVlHgQlu
z#~wijk3Q_%?}Q3Bi+KMNoo$IzzauMEnbdo=<k2f*ar!&?_4X@%rg`sergW1&t71K-
z@mWnfn-ceQ){n(KeU2uy+A=rrdl8;4rz25Ys*f*@k<aFms9-5hlBnOQk7EZvKpTn=
zV~aKId$*=*{eWiFs03oA*Tm`%&CX`b9~QyfDeKrKPrX?YZWemRLxY<n$KXoy1%}R<
zp@laMvwd;p!9<kI+uJNKb|xjZ+C0Rus@$7ybLG@2p7>LQq->hP<=Ihu8?o02*r&xA
z^0MAPJNyprg)}qAP!7oFfLM}o&vDOf-Mvm83$HaIgG(tBqDb6Z8%5UZAB}G@uVAiC
zSM2@rTL-+Esg$_wi&V|}P3})-OMWe|N$7yXtVe0Lf}P~MKsU<gweV2L!pnKMv+~Ig
z$ln7#m-nWd@zqpPE}kgL{!TMoiL26_59c%#veJ0mEM!W?nf&Wv6Y#dtx8Mk`O!!v(
z7f%cMm3e>4iuz641Js2Vf0eQWuWR-y>*XuuLvh!N$a>ntR&8KO{KSof9I<2rUI8V+
zbR1g&DRB0e2dD^Swo=VoMdTfDMkh;JGY6>&e0aAd>h_opKQ-H8-{dfU<UF%s1jP9w
zieHy4o&{4TIJ*p2R41yGV+nAjs(`ajkaQZUh?Pd>P?NXSvKaTYd&uC3_fqk;L@#wK
z1l_55q+Aa-p1IoJP@@Ii5CmQ`6Tp&w4Q828EJ}iu>BF}I;HOAQ?aox4!R1`V4}1*G
zzj-YUj)l<HL1tnAXu8e!__*8h)f4ABx}Rvsqe_gwN$KznAI~!*Pavf^Z@*9e)4$@R
ze-lrF#E1V|;>j8kAe(voHzn!qVK1^C%jl6xj#%>H2mhDzY!p-y25|6Njw)sQzcQvG
z)g!@3&57oR=X}U2|L;Q+B<2rj$9m9>hHS0>J~)txoDK8o`u^8JIXLEh`xtpv&hd!x
zZ&uy^9G4e82cnMPY=3^_x%Ph_-t9=GNYRh4I{*FDe+&I@P5-Bd{M$YLy%BE*kbg&y
z|MPMBvs5BLX2=09PBs9vyaK7mky-N>z6J-LcOh9S=~4l#O3A`ihV8-DpUND!wI>_h
zYCBDG6_a8CIt#u}%?*@3nn2>`brwSd{wnW`>O;;V)NkIjUjaanO1WYCyVQ&0O<7-b
z+!{3mst6x|Fl2zt{wgP16M#cxv=I3_mg@~*VO2sT8bDU`wq10<PVl}4!oC3ggqD*4
z8aK!U1WX?*<<)Ax@EZ?Y08&L406gmG1BuuohbyrCdQ3n<^do@d6JEXypz)+#L0wqz
zLt~+R%K>zc$e)^vD4bJZF!gi8*JQYZUw@FmZ${v-JsG3d<o*I6y>bCBUehcc&1Y{f
zdO@??a0n14X(5oGKLzm0Fo8aC+;dFhVa;3&Mt>t6%EAJ6F~*h<y@mvU*X05RU)S)>
za-4o1;4I0_-NnGouG@bE<}38u-;Go13-Q~tX7aJKLh>Y}I8ZO2E&KL#9v{mt!{zR-
z(!Pb>zOKYdz@nK0KiN$R%!m})_&;`$NKH)yD9CF7!(sD6Tp>A5Acud1MpUmzD!>v9
zu8-ud10qGVgoi7;MhT@_sa^_5%xW08)O>@GKU^JPK{R??3T7*%Bw~?rD}<5@L~=mT
zNV@?!(`>9nH=6MUK>>ghJ!k;Dp+)`_F{iLdCV3ga=E?@Z0u_*UUR;LHVMA4<D%h$-
zuQ3rxUAW%gP}Hh;XM@!mwF5ny|0mac^2N)Scp%Aq_}O``QcC2ZG&R(dQv<y>fse%l
zpy>`Y4|mtcsylsMZZQ&<i*-nzN}3l!B!*jeQpMV3J2%ZU1Jq+Le6;w@h}bR+zH$l}
zg^Y+as?xU|%QTUPV3FoR2XgYK7r&_;1{?&&D8T!D1t_kpfPSG`swXa&3UPTJp5w4N
z+FkJ*vj4&S%Cfw{^_X2TO(G?XQiOi)NmIw^nTH<;%l8z#w;95h1Hg74kRx0P`4oE6
z(~`?r)6!z{1Ge#Vn7ZYSYzPTgo^fB|g+qCPT3MT47e;|*dB)C|fW(j_E&)CB_}*kF
z;6$J!845-aB&dTR)T*y=$_n6u-U-t=`n0|N`owHFYo9FKbm?A@2$to-xsR!;+XHwa
zjYOJ|Rcfd3nc+_#G^sBv09fz@uq5saLgt<)@MeFJD*8ZN1$Z>Hbg_^Qw!6>D_Av}H
z0`TekGof^C_E11_I9==9Hv|m4Y>@F{dfzs$Q-?cM_`{fVHYvYJe=;6?p<?9f&}tUu
zlI=yu?Zef8$IzyT7x)8+x#mbDz^?brRP7yEV?0_u`e(_b47BQ8!xXmD^MK>x{Ib%x
z&j&zmNEy{_hIefkco-Mb1uiD^0b<J+5TOYen*_xK5CEo1d<5?1w{%=u71CbRA*sp|
z=ZtxP;`#*1==DsYSa8~%Jthk<A}3&yV{8o+f8>Mgo2uU=?}+0lWyUuwJjNl>W|E8k
zdK1F4?-~!7C2M2Vm*}|EM`Y;wThsNG50Yq~_<RE;k<Xca8BQ)xK+SSUEKcyVW8Z5#
z<xQT_07Yk<4i>3gJ?%t8UHbIA%nMeXuN={e^rScu_XCJNC*Nn2lapi2rhIyxos~t|
zf}P#<IJb&Q?!_i_E<0hasM@%3chWpf@A6t<d$!uR4`RMjIV-;FKK2V@^f{o%Jni#@
z7=vD&wSk3<R5F!rGZ^y*l^8I?Yieq;ei3{HNWHMWH97i0_jEQ1K%BZi`YfU+bN89N
zd<fhPV2sAUED2K^xAjsb;g>vpqvI{fNBC`98R{{K*de5@x?bPau1uwaBp!FW9Sf>>
z8#caI?^L4_;+x*l4TS?TYTG~e9$2T;(J$B>t-XrtP`C{0izYScRKN@dt-o8cJ5kxG
zdx-msC|NTTg*`;{3Q$1ycsc<G&`yw%@##-e>WWH$-qk#<eUf6=zNy^Oq41<4&L>UG
zX~%xOZ}H%5Vn&yc)oh#b#cK<QVRoeZ<p~$)EtQoVde6X{U2!@>iwP|WtUKVbm6<~+
zCnqP#`7#95#uvI4IcfEx<1L=o7XqVsp&o(+dB;9lXS>5mY$o01cMB3M<K9)znOEX#
zh^y!XNRHrt+s@q{i)+f2X)wD?Dha|=CtJX88`g;ca2LmvTEGfH=iqs4DCS4Eun({q
z<4TQyp=*5w{+&#vSkAl2K_F(l{e0&;JwN%|U51~8SmYuv$s6g$JSssk$!##uni)#W
zbgQ45_{nlL5Maj!%pyq9I3WD(Jw?RbOoSR<7m->~k@E1d+S*+0Wvta_rUj`~)q{EJ
zjq~?$Jy^8gyn*k|H8)tOJ8#W2jJ4aK+`+hFlAKbWDg{ZtF~hZQoT{#?*N%$KF~XaU
zAsl~<0e7OL(u#!S-pBa%Np8jG&Sx!I&tYAAfXhT6wxNSST#GrB)>_Sd&zCmFciBkv
zvIV>|4HCFbdvi49WlN?hSD@&g`aGw06{|^W_!{>@x|`|uSguY01-qL1Zo~ok;Lz5m
zbkMJq7j<2VfvYxIGB<cs=Sl~@)AuEGOws+~TI05X*2i}fKcjqver#^q45fvr;!JwJ
zTD!cnFm#$`GebVvpD_B4=(B5!5g1iU;BFU3>-2oxPc5c~JOR&(eFpF~QbM)wXEQMu
zVZ$3)-%GK)_v-}2sTAyDlYHXuPn}KQby!G{rHR-v&jPGo(JPYc^G!Ta28dFM=<-G-
zhyipQ$UzR+(Eh$8A*15;2+azGt*WLltZ>(X$LOCiF!)98OnOdwUf(~e+WWR`wd0Sz
zFf=^?^}IneZkCasmVD;lzzheZAu%!XzQK&u!L+teVr;EXwxD!Qie^vGiA{1@*c6|C
zi^iu;OnujDFnvTa`i2w~fIqkrj4_%az1!Ep5@HMx0AyO;G9Pjf<d58Y#BEP#V~Qq&
z9}Wrv=1b$kK9Re$FD1QepuuLBH=ZJ{a?XP%q*r``TKp}U-PF!C-$BS@OQr7O&W?5L
z7cf`Owej2--bYQ9KSc#`9*kj1s#(*S4XsH3viFjDQw!ioo@f!~9~QJoahvra2uhS}
zM_82rm^8#`dy?4vQ{x0-o9^zHy}`AJ_l$Av6Z6otSsggtX$PRxI=m-#6QLSBg7iu+
zzIIs8+hf%xjDG(Uln<}7>iU(!)Xgb0oy8CRBeXb-2`ro$6OUA8V=MyHMm&R`iZ3j<
zF=$sBD;7DB9V7V3=g~vG$14D2u-MXb!s}ou$ckQs8`RBj^IbK0>+ZK|3+dGiC)A1Z
z*B~;`(K7iJzc6es<J?76+88(C&Eu3NgHO$8(r>?<%x9uB$W%dzo5zh2K5fK^imEFE
zYD;dqa12#$iYCX1s@wh@F3Tk1QLpirXU_a^m4V+bfHPXcM|MDvy>#pcaCu_M*j%)w
z1i$+0`~4@dVZlL+$`2RPYk6uiEggkQT}lg}5{RRM4D9poa?YlS5yjy$I2Gw2Y2!r7
z^hd1xp_~j4O+4y}+)k0Z-vj=3XssFX#V1p9;y2AV1!{hs%@<4G%ch%5vNJr`uNrxt
z>J=|3b~(DUC?=}r+q=R`Yo{wR9q8XX3ggb?freV0j&%aVARQlf|7mhoN_p?yKkbuo
z6w_N0)T1&Qn7G?!hPzJti_gqwJubusOk@)+==GC{Fqx9n)2KW)v&9uH9#$*mveH6=
zyYO&7mHf}Kfas&w7((BmSn!UY>_-Vjn?E}x#Yzi+o@{GrsSiL!lP(ZONEJ)_+EAu$
zr!8PZJ#i#|l`lZ*wLYBP-vMG;!$!7lf1v(q1D@Z|<A+#Hdz#fB)GRYa8_iCg7)q<n
z>$Z9cW40fS*)?v)b$TU{O?tt>{8~Qhp*aP%6!$9|j5d`tP-Rk^E1uX@Z<owXx<6**
z1c1}^9l}MP($_VV7n}?bi~6ePFw>bpef1!o29d0th#AV)W&<f=aTXhSn*~;*%N172
zolUF{{%|Cv+F?k}Ls6@)Z!U3V<^yY>DDBciCjs?%Z^BSoGAVN<Qsl&BrqCKU_P8|5
zXk+6TF~dMVZs+3S%2zUb?u_wO=l6wP;nJvDin)t&z1yj!JHQXMF~o>=KV#q^U_J@0
z?coKW#4Nso3RCfb=?*!{^~4;f&ptemXiG5D?8k6u7&h9(p_H9F6wmT=OpI9;0wSF9
zPQFsjXomaQ9$kLiR(Gbxs6g349{{x5BtcwkUz<2IsfB?kk%h`U<|iUW+)aJL_NU;1
zTyDxJ2scv7Vqd<JoiXuT>Wa|ue8xIEE9+h@Aw#ik#%SIh$+VNGs3PM$9!eP_`=Dy`
z3e2N$5c0DOh^pe%!>UgQll8M>Og%u?pr<eLt}qLhPH&58!tIUavOB5L2v|JQ;1A_c
zWnQEmEV^rYX%2M(w*?J&k{b5nOFBoxf3K>XtK?hc4%{4(`V(W~J{fL$ut9n0BGq}Y
zrAqWWKRT)Pr3BOx#8FQOwH6ju_ro1<ulHgx9pO^YW-o}t#_!~ZA3IEQ878XKf2_d(
z#x!WJ+~z_xSA?QyZ}5n3xxA(#EzweqXmV;w#WJwcCE&rsNDO`oeHrKH{Noi<?ub=T
zfp+b$l8>2<M=k{I6%7aVR8qj_q*TH~WRpg3I0<Y-q@*Rkbz=^mj3H15Pgc*tGXYN}
z6|^{{P$KOBEW_9|ayPonJ{^HDJW2qord3nk86(oOtE#_s7h^-yM|w8px9Okz+E>L9
zSj{Tz6dTUw&G;WhbA^`E!Gs!W-_>YQ!^y6nXGB?%E39}v6xavInRYe&ALTRg(!ag(
z1{3@0K~+u7cQcj5l`OXB<n<VeT$L?G=LgICNUB~EPJy1`@6plNFM@tOZ{eyW4SYq*
z>6af;f@CPUb?WAOw7@`CG79@b_PAxa(h(3!QxyU#PfwssO4nvCJw|DTf?BHMRc0y&
z9uGxqv(o-hnQR*DnnUraafs3J^5ng1n#u4#dc|Xd=l$|y%p(<oLnB3R$)`U!gTsDB
z=Xcpd7)I7ZI>M+Z64rTQ*5ZZ!M63_W7$dyiiB3kWTgb?`+jqq#C=n$r?5qJ4V8pt&
ztc<<1@f}x1ijHclcb?Cq_>JQZyS8HRyZb>;WvJw4WD7}<P#=*QddBb|1N1tU_q_Cs
z<N_a5--*sftP7LAkEjd(rCRP+h;J2<utD@NB0|nehB0yzV8|8PNmF3%=e)d`3=#PL
zLjd|t567x!HDlL1CROGmCro%1mc?nz^^9FnR{0mM-O<(rU#B#rt(Jt4ft=yblk;Dq
zab1CxU1z_QXS4?t%)T_hRUs^R-jZV=7A@_Iu7M5&fTv&l#a%q=<A)K8GyrQ(eWOXt
z%6*p>L_+poQ9gR1e~O?)ZInb=7`i`L|NfL<zBAYD+>i<td<+8tnB3cPpC-_LW%k~C
zemGx*iw7>_LZl18yCuK*f*VB)<I57>8X77_xSn@ES3?{MRAQ!=bQ8}T<t}*y_;>~h
z;-0&o_`$fEh;HKH?d|Cfwi)mr4#-99e~acs?ELd50}J_L+LylI)!4#1K9at8lFs~H
zzTe1nW#>VCMod66D}+i)I51<8g_iu5i!pd|N@TY4&!-LWz5Yx@5Q4&jPe5KoLB(Tz
sfwJ(5fT;UF|G&NT?@RyxTLTDO!^EaI=|4GvDB#bFXG+o~Pv83d50(S0k^lez

literal 0
HcmV?d00001

diff --git a/docs/figures/arch_8.png b/docs/figures/arch_8.png
new file mode 100644
index 0000000000000000000000000000000000000000..16a0bfdd39e1e5df7cc1b10a5aaf72d714cddc19
GIT binary patch
literal 33393
zcmeFYg;QM7vp+Ztg9jPhg1bAxZE$yY2^JtoaCZ;x?hq_MLU0YP!8H&_&_Hm{Uh;jv
z_g?K*ZS5bhQ#Ez(+}p>x`}Fy=OthMcEIKMNDhLEZmzRTSfIx5|AP^WA84-9wz{;!u
z0-=7ilaf-Cmy)7Tb9J_{bF>D5<f79ukhD^kv4c+jRz!w@P>Q7Xl;5j>7>gpQI%5>%
zz+u>u6qQR!gLr!Cuu{EX1SrBHc-WxPOahnW3PB$?^_N9&#OQsn)ROCl?=R&^k?XMs
zx7R;6+<u9j!;6={g%$JYVFbCGe|j}Hlm~r@GuDF+&r%4Y;(-53>XaUZM@kC9%F6r8
zME)Dj*1qcv-|k_-)Zd!n^Ba95P@!2Ej+g$Ldsrgn4_;lGNkou#<eo={b!_<!S04{&
zGDTlv#Vfmrq4HPe*)w}~Ij3rvVQFwZDX&Q1VuNTzU{k|ebd&*d`N)RI-eAhU;lS_7
z`T4z4gwbemSR~Ls&8IAsz_XMKf&vtc0#cRa)Yn&BtX`@T=?w8r+kb@<{<7}dB`*^q
z4pY#^`b#BcCFCwY&b<g^M8m;HocI{T|3++L1bZ_||EG=jz$91Vk9~IIoKGBP={N`*
zj#UMY?`~Mmn>zYz^mCz1@o5ZKW)obp!UVd>#FX>&=gS&@zl?0DH|0`C8NC|QuQGpf
z<0I4P)Wc?)C}6*MTM&E5y6)?<P;kX;rnqQBA9=5EJ()FDz-~2hJ06q%^Xm^^pNVCd
z+_CiX?^(U#X0s|0u?gAgsZo@ax(OM!owXISeWG%ghmhAqUSM{tpa((<upXSR8A>Jv
zB*^jUtV#$~m<z@{5K_A1h=^9jB^t`QlRagrr?0mR;?Qp5*4;MVb7z&j_~zF3qBJ6t
z{Jr3^V1*(Bd|@pJv>WLr*trf;JA<=5-ccA3;BKpO1rzQli^K60f_Z}3I3zbvb$T#f
z!ST~rieW+eBwpx&TPcW2;W_mXb%HMoLAb%JW<r%<UvbtYOpIXaLiAk7uO3<#^ficQ
zFZDELL=Q6uqG%8ba)=8SQL!Y`d;C(&EJ+<K9M|BELJ$LPOz?IQjU2Lkq4qRlB{EZZ
zb)ml+^#z(cSP;oL*ry0-8pH$c4Dae;$b@_|Ltg^rV{)70A7MuJ$8OQQ!kG*|-<ohi
zFoGu!Gu(1J<-K8B!|V<jQU}q77MC=u+c40L!sCU-NYfS%RPt6zY6(;$&c_{ydlWg9
z+^LsTPT8_#$Ji83Osm+Mx<j&KCt>vx1SK`o_cOJ7XnU)Ba2JGkLJ@K5Lzxz&oQ_I7
zbSO<PG=mt2<}4E|*(`orL~xKd=&`(Xh-w+Sv#@MnTuNKAIAU`|yJGo7AQbg-Xv>A^
zm1ZtC8}d-E^$%2+wARp8?pDKA$~F3q7ba-V;n!b>w!?3P{REysPdrcfPY3}a6tdIw
zudqXqh5Jw?QHm0#C0Zp2sg$sGqa$FiEnabkx900fIMSKWh|+bXBqhxxy?u|81Reg6
zQuSVd#ZBKv{3o}3u$AN)#b6Rm=|m~wto|(gtPoe0qxd`Oq2#8KoRRpQ^PP$v)sI^G
zZu*~CRNvJ)wm8-tu`Vy-f~5=kidl+wG)5U^8TlDXTp<GR0+<2`Ys6pdzu>s~yZ$)S
z@>aa1xOKm^xP?E{-HF6=K}AG0K@|@73pWTqM-A1&)y_DO*wx@yJFFzAlHy^Rc>lG$
zY&ym^(pIFIQyM{<WKeE!H)cMj0{4EOp$#riS+yi%I&u1BrklZzp)vJFAtyCE&56-p
zOGopa=1*n<rq2wU+E2xOYJW83wU?`B%g<nVwf4ot#cfI+pAV$x)p@hCQ;2nI)sCZ9
zq&)Q2VIo@oWj?h&pYK{D$|DjZ<VrMtbbPI(GybN`E#0Ja-uyEPUwceiu~56*L*|fH
zDyi3^l#_{&fY5T~V5MsX``CX)WCcesLJ%tGEyyaU>K@&uv<_S6T31=;6k+WY@6`5D
z_QAYFxl?+ydVG0jd`FJ{0=@VJA8|9$3lb)hkLYgbIWNdblH!fxx#I2PeiK^8XML^u
z+8*be-9z_@ZiH^0j$g@KQ%Ey-pY7X*ccwSSdDyxA&&#Z&aXlLW8)X~#pYG?NbK!Ho
z^TqSapF@+Ig53^Jo@j=#Jh6hYvgxSl0qNf9FVmfCl|HoU%<Fi57+Kg@aHw6a-CW9C
zIeqhPb?tN63gU`W^KtW|&1+}+PY$0*8W)=qKX2M?4wZ6-W(|&yVGtsDMz&zkzZ4Z(
z5wH;=d;8EoGR`=^AN1`M+>h{p;DLsi=aDNgd7pc4Ovv_!&DZ$OxW!nYj<;@V&3)a=
z>C%Z^@9PKL1&IX*Pc+ZQ`?uGLWC>(7B9lJF!v4Z&BEB8C9aJ6F9r-?oUi9Al-ifzM
zcONbU4rBjZUaudl4lS6uEyo{miurcW2M|2SKbb#iKTUy^zyokEgIx+uzkli(Lc1hD
zCHRC8O!02|Qz3nCL~mAacZ8W#h*Y&yexyM}9qKV^Ndzmp5Y`ptnn(xDgL;^H7)>#E
zGBy#xGT|-fD0ekawU|%9#lfBfXEkR?=9Qk2qm`ro(G69-^eO@R;PO|zfptU2R<GM_
zW<{-xQdWVM)Q_o67q-H5qD)WX54tK?6%pL|Jt|9C9A7WL|Nh>!{pHW77q#E(0LH)4
z#cmdM+b+p&v!6<ntQL39cGQC-3l&c32KlX3M~CJL=OX4tnk$>r-Ti$0?of(CIJCVe
zrMNA<W$K5uv?vay+{f4Q1?a8_7GF!~$2P4r*~z)hv2rFqP!J8j9jThUZDw&@p4mMc
z3m<FO&HS*`ye(bl`Fo0ahWQJ#kgi?5LASjbuft^*?{sz;Z=Qq4;?Me*2EUq|{`zd+
z&0Q{@vz#fMZ#YM-OefVRb+ZMYA{CHlqgtU2Ao!Ey@%_GqK9+nR%2!^<VdD2^-@!9s
z{m8(?erGVw+I5$*tBNP4?4+``rfH%3(HHGqtjkzuZJptD#%RXRbPOl7wFQH(49qOP
z-TwPm%)5NMzVEO+<=$QBLbRH7544@C9t_fqJU*{<wpmx)O)3kC{oX!qYgv8cc}^BD
zBJJCCgm={0J@wkuAznh+oAnc4#g(_QmGR=#cmc_w_F?QnH!m{D-F4*e?rGc2B`@(u
zae16I8arYELeJBmlv7g26E+j~279d1k-g?4TN&L0e_c<a{w7V9jF(IdT_he0tB9B^
zp|;f7zI0G@I&@C7I}zFPEN<$r`f*eLvzsq>PE_At;g8mc-S+;&x0KheQ*i}&c`rpW
z+?^WS?0%J;&-^+uLjE%CHuHCTS8*d_(zCH`(&$~6-&xF$G#uS5-PO8fV~68U$D#r2
zS9gEN+VdAkuJ~7+1=o)DjHi9?(hqChK8rY;w)J<p9sllY@4Tk^y?zpVc-TGFF7`li
z+?x|AfJskQ>l^27bOZY>f2(;}!P`yrcVX6fG0?upt?*udsYy0;tyj1E_2ZKV!(V}0
z?_0E5g7|)&etOvsS*I?e3)j80z4-y#-tL{hE$1yWhn9h@k250pB77Gz4>o6S{s^Uc
zqMy9k@Jv8T6nlLCWWH^=MW3d6FUBqA=^uS}W?`|cyS5nP{B={X>pn10I7?K3aXU)j
zTP+Bc$i;#R<V*sh7y^lRqJZAcD5;9DPLF%`@C5%I;P$_6UV>9wMH0VJ0D<e^)%3%g
zo1+31NI|{0uoH-xgTsF2abSh-C1a8ha?ghPn0R{-($wGEWzKIye;H4@r_4rc&~NS^
zob!uN*hDACCIJu|W34A|qpS>K0`8GP5IABG0&oWhd`00%{`+1AjsXP!?>QI*inIek
z{)<KhxITa01K;O1|GB~^M}UxkU)aFcs}THusNq5i;s4J)7#Da4lF*crmj|wzmaf*;
zPHwi&?q899b^;GjT;%lKKp=eD=P#VR2F)qZ|AL*Cp1Yp1lAxvY8#Z$*XA5gKuQx8w
z{eXnM1cBQ(*6!vMUT++o+yuQusQyt10{734*{LZ0k+?gEQ0XbFQAjzvT2t_`aj<bv
ziK0?aPzbwP*$8UDWd5r;@Jocs*4^Djke%Jr)054Uo6XtvH9Mz(fB-uO7dsahD?q{O
z=I!Ke?#1flM*W{o{!c$JYd1?*I~R95XD5p1e$6eMJ={g8sGbM<@6UhcY3*hAKO;H0
z{nxU91+qWCVdrGyVE^yFfu_RGPX*QNysRDdVRml-o&jTs@^W(t|D*qZy!oFI|5r=B
z|7pqnitm57{9kYW?=7|6tX-v?-vC3pi~f(e{;To-e)(Swh1s81{=Yc!A2$Dc3h-GJ
zRha$1WhRQ6|JY~=h>_F|rm6*80VR9>z&-n24F9=4-%EAp-O9{@K;j^In1q%W+>tRt
zht?p0Xgm8JBFaAgdn}kZE=QD@90MgK14U73&@5|IX;?^96e=<ULs2P39(M9<A=Ub&
z#Yte7_nODyU{`0m@5ws%1DUUb&7_~(i68f`+^j|hDl8<KAgCFHF!;T6Pah2HqsJ7&
zYX~8nj5t*ftcM*;-Gj^lrc40gNJMpmjQsx5W+LCb1X{q!Pz*rLsF42<1jSAPRB<32
zxNi735YQy-6<7&E`2=?PLj?ESgzCHJe+I#l4T`21Fa@!R{_9u?@__N&kQ!?nnFAVj
zgVr<nZ}6b#pv`{`A(EtUbRk%QaIH80I_7|$3O&y#PCSab2ZpuH4k9Rco;@5LH~@(d
zPK?6kVo%*?JDR>j;fu>wpZmkj+xqRy<1L59x|zv5q5b;qr{b&Q)tp+dHdaiWV5gc0
z1a2-cy=s0bfr(ddE%L9)yp%<aawYjhk*myak4)Ot(+{`jd3E6{BWqvWs<dl#tXd2h
z`S|$eV#u4%xS$YppUGFaPzH@s<h)mJLwqg|q<?&k=ca*NpKMkP#u8{|^Vn6I_eW%>
z2S<x*zDmg@#F0oY+D0OR+k~E`BRg$&DU<L!l{USenk`ezV33M<p|N}Ma6YAZFjw^h
z&-VQ1#Fa0jMrmP1L4ZNNX{S$`&Idipd3M4U9*2dv4Zm~c`=$h>)^5uo3_-6`iuKmF
zp-$^<wA#A&SH~rll?*;-BlP<{!SD&{M?-|S4|j+4?oSWiO!^;7NncuzXU+s{{`o$d
z#thLW?mL*Tu^Whjy24IZox24@WoQ*r3&*p#g^bU4ckV6^-S?&`6$(T}(%6i%9r;Ke
z{|u2gTaV$ld7qb$p9KEBCzpo7Zj$3Uw=HTEFrEcU2nl-$@(p9QfW5bk@V~#(hEdZL
z$dV3q1AJs=)~&n9jIf=~%~wuiO7nAC{%9Rzdy)H_QhAYr*ttw8i_2-M>b62TM`t9B
zIYXmF27UMUdVAeW+SB8G`^Zat{PHLnZD<L~=U)UYdRWdT_W3t?0&eAoMy6~JXJhPK
zrV_TJNz}5q=HK3?lo)++jiEp={m2P3y8hgQWzqCLE{GF)8i8yxkzE!R8OhxDuF!pd
zR(gM-Hpj7$6+hOoVPH=jD&m|d-t4q)&|7eKRA=0t3QTImivrpb?Emm*C~vz!&_kh@
zcWo+%zn<&uHf*<PN>mfzAw7v_5r#(C%YMd$A0rq0-;V9X&d0emY&Z|z?}}%$8Z^83
z8Id`!ep0qu{#f=PrLVm_=WtL@nAJLXxf`MY=_4x&TBD+-F7pFaLA_eL#wG*)mkp*8
z#HQYyt)LDIm^GQG-(Dj>(;V*=+4WMRB@PR@&$g{zxPgpBD8Jn-4FYuprs$2h$;nDf
zjiz5wdeWi<MIFug2z|j4$<Vh7RiVNDXtk=PU1DLchH+Vcxk~RNtJB>K)9Q@%fbPdD
z!z?7LOHZ|Eaaxh0vyZPCwM-R)!3d=+>|Hm-k8S#xU=;{qNL5_HoT;C15m}g%Zf|VO
z<>@O<qm%Xaii^X=si)w@YHj9520t-4NC#yIBw0Od(lkI;vb$i~GDZP%vw(<BY!#?O
z3trr)TcIZ4L%wf1fYXvWK(+)_{=oX)8Q6klN@WUZ8Wt87M~m~JpErCIH-_YByRcZO
zmYQBC=Exndw4|Kjvzx3P=P$ECa*@M^!bn{$miBKdl0#f-+}aXen*=_3G;RI%@CM%o
zSN5Tl6nE;DE&%&IM&&HPpAx)R{pa!F_E$lSi*j1L*0*wH99$S}$kB2W^WJW?#LWS7
z3cadMz~4vD%j5?<ME$I~Q$*Lc1RQgYPdI2Y7<EM!gE3rejwJaZcNl2>A1YSQvhS1%
zWg!eX4tw7!5?NFmEe4I8vwjPIhhdS2boo*IDI?lD+!OgBZSv77`lmk989ZQ|LS|dv
zTHZo~Kp(Bg%6Ep7r1H6;H##jF9l!Rp``@Gb^SYli5eYrfr2*znPGN7ZDrFq$C6_Ms
zKL4hlWrl<omjEqNvIOZc;wngnsV)#OO=;KWpvaXmP2?Tl-<8B?Fo;wnMjBwGys8<+
zjL(0<2P5~TDeHInec^hu!h_Jiih@HYHy=&UjwqYN|7NAMGH?&CN}{!a(e3BB`iB3t
zaoM}$l~I5z^M+2{dub?vudBFvT?oIjTtt{T_C<o&!)`)c;8X1|$wiHBw<5`LxBBom
zOGbCcGLAo5=p{DKXlm^01{U$nayf2x8CQee>kcKn#N%AmBqmD|!wGhl^F7b+Z$i%e
z!LFJwj7t0n>~QA2s-NRonkK4D(86G4g`h3`2+bY+&rZe&K5iXN4bnSf8LB3czt>tQ
zBUj|ocp1C%iA{!wVDD26W}k?p5|Ec_CzsQv$=|_qF3Z2AQP}hN6AUo*3JNfB+EDxQ
z&q?QXKK9F($Ekmq>pQ3pr5vcl>GqHZ7Bi(hF0MBNg+@Yf2n)3r=lI^NFj!zngaleF
zG#C=M_Q1YF@w-iVXrK*-#<?|l-D?NJogPY%e6>GM1{%@gydR1^e~+9Vs8R(fAi>m{
zY?OD8KX0Z7Qp+Z2Equ@~0ZbnKhNM(@xR)9;9X|(@hCpC*n<)8tx}OzPRUGL;b_#!z
z(O1Dh$+9I(Hkix}A_9mP<$#rBIRW<cz@PNhMtc^OYJ22rg>hr?jk<TB&jZG67e~vQ
zU&oSZV~5m3$9|Bh7O_ch+~A;{di%-L#5z*nV9HBj_XcjalQ$aX3DFrCnNv-JrKRPW
zpmD^!qZ<^0D>Y`RgQYAjjVz{Jejy>=ijaoJB1+l21_Z#`ORV&LmwX`vrzKh^W2Fu9
z;*O@Jurg{jilJqiGjyd1Rv%Pk@7%F5Z4jfuNO&n^!%jjsUBxF>na}6A^u3(QE$;A`
z6Q?&YDk}t7=I?IQ%S9+oFyAmBiKoYFQ{R#p8KN<AvuH$Z%=&V8AuzGtP>@WMT9M?L
zD(#EL*{a9vA5HlQLvciwpH(aWybPRK>Tg-n>>1#kw*$XD`!!YHQ}?}9OU8n)7O*Ev
zBcOaaJSHu*?@uuo<w{xGjH_10mQ**Gd{WqO?emYi!NGzSRMUud9wIq6e#bA}f@gGC
zmTa7HXRv^P+8S{YS>JFE4wM32yYt6VP|Z++_kd?y8J+0e9`$|^gq_ywTQ95Do1G-~
zopaoDHsb?2{`{dkX|bo9k_rZu#qVW^LI~THTt5g*6uE82U3?ZYQd%4qSH4~E43@Sf
zT7={-hf+odtsj8@8UToa%xxzM@o%%<r1^TAhbtG6&wT<M`lp58{R#W+WR}Y)act8Z
z{b{^Kpf5#}y~%_qL|H!+;jd5!jJpFm-OW8%ef|F2v|S?;{?qoq4h>#D@tMfNMHY1q
zne0Jn3Y^~P=kD8R2cxC&#Jn*l&@Lwq$O~D>k<JjztigGlW4m|=eICc>o@1^}g*bEz
z{|Z^#m^<LXt3Nd!L75P)9||z+cC&W!8*@K3ShX_qgjk1Ko5Uv$V?ajGbwOA-Y5<Y{
z%9msBD?j77Uu9-DEP@*Rt|s=d_Vydl$YBch9R<(Lr2ix9!HNbGV`aA<2gaCs%}Id(
z6uDD_=y}p?P9NjN)(=jJ%mgfUik_;_;R5CzZtx7LA~tk!sA!*l6$IisA~yM6JM97h
z#HqR%E0~fdriz5h?6crw>-^S4y;fDSG5U8oaNW?U&DJv<{3_Gakh6%^k8d98xFssv
z*PV%q&mz}%qSvhXR-(0TB)+R<?Z`lR^&30iK0c1=4&sl#+JpfWwS2OaMc9dglMI`C
zgR7<URvGl9Dc=dX;5ByBD|r?XiZNE5a(AZoidTN||1k0m76%J-K$jxW2x%*d@th{z
znyyht3_xy^mVDM@>1B`i*C{SFB>?dA^^0KVdPcT|{E$rKWHoAwX<0)3LVC8f1Gx`h
zBrPJ!MCg2eTBJA=zj7(;paY(=ymaiILi&et_lN+{OS6E1EdIfbsvtsv-t+fbDx)SH
zcFxdyj_Rb-FX`7|hIypuW*0rd|IjZr1z@72S^e+!D`>9lX}7-Y;48MyCoLUiuY7h^
zwb(cNeDC)k%1!{*zmLlf&J$;elg1<yF8rY1l)S_@Ew7y#M<amx50y(u0U+IW+U@Iq
zCIb(EYyUZT#om9odm0OX{KOZzoc~Dg#Dk*6$uD~}{)6IqfVGAoio70wCJhn=NT+2*
z8gT#h13?ECRVXn<#Ql#gm;mX8IwlLk|ND@HIB>LdDo%>6oV%~9XaTR{f$ACkhafXt
zC!`$T(P{i9qubpm*gkosK8L4aae}o>HZa&yPNS%2P(1v>O5PaufJdvCjcqk?kt?jq
z0pCUB{yaw$1QtLv+z=T1<~wWJWi~dN)fQ8E@0-u6U;_<dm4$Kk_q@KEA8l%t2TG6%
zPA_Dw=)i%6qjp?iM40#&&k&0#Rl3#g%_MA(Wp>&Im<_c<!XhJ!DnF+N!%>FtEA<n=
z=Q$<gPUuC>M1|X(l!y@!QAhxVb<uNJH-(KrCB1^5tmF-uIJ{li{s0oeV(0M-g>ndh
zXBHWL!o*@89m*Lhu}k|57+9`XaZS8<)L}m<MtYXH{<%M46yQdJuJR=@-+N_*9oAAc
zqHw&IvgQJYz9eY{iy%N;=;Avxz+MJ?X#ZkNPuzDw-fg(c*(C_lshJWGOd1S0K*Bw`
zm2WWRzXWQuF1$=}@=&<ID<<61A@CJq+ih@%ZP|@r{~4}jQ|A0w$^^4g-g1~EEr6b)
z!nFp<oCGR&)W~+m5bQ;=+Qn$XvdByC?F!E^<{vVVJ8Henm70t<8J@xn*IUr7+P_Fu
z>XvjW4q{6zyqJ1V>*lfbnsx1fON8C%pszzQw7>?^A>Q-Ol3DtsmRXRIzat87K(?P-
z1NH^aYC?}TN-|7kK?x%N!D08xB&PEeeVJ{Xz(@Of;xHvMZfJ-lNnkj;>C1Ftmqfbz
zuCQo70a5S{m05e6K^%lQTK%ANhYp(R^{cM5So_ziM#gmoBJE$b(bQ;)q?J#`hKh|9
zqPWyS=+VMe6K@>K4{&u{f--~7llf3wnJD9fXoCIgqlkLcBl@=1q>D3ran|wIJZu-i
zqv{{3igx(38<Ze_<}#0e^2G=&gr?>Q{qXMV&R`{C((r1Omkifh?k0MBP%bV0#Vx4#
zoi^S<q3jh?XzE-NB!Emx>ywvYjGs?arnA}#1uaK@rV@nBqa0>hw)DfF@#;i>TcyFT
zM!P61@$$zp!-rwnzNK90&f)Gj@nU2K%%|eLyfi-(GOC$4o`9Zs2${0zS1vp#-scor
zYo~a(ot+>MPH?|(kh%MKuTI;aS{>=oL=Uej=aKBPa-ZlxqTm}S76$rEdjUORR6OoB
zHC)RJ4pLdIrQrlTCJN#wE+|N8P=>Q2Lj7nz=}b?`uSIdxqO1W7&)D%D>kp+~i(-Z*
z*Gq6?MTF+K{GWV61{xW#y?P%SIu2iR?*Zdq`!Hbm)6yu>X}u4hZenHL-|-VUEC}MN
z-}!QG(X|9}Z_lAe2T`qMmAn&S7UY35;Fg0=O;!MNF*mwaH4`9TJ7BC(Vy(ci+%)fW
zR-GJYFlIodRwJh@k2xNQyR3R#pf7LnBP7J*982D5QM3w@(0WOO6HyTe4u@aAJ0I5T
zSHRGx3Q~XNGo`p)vRpx#`=qJ>y*ix4iB5OMtO!NI)VLaoy%p1(?8QRv>81SX;wv?b
zN|_=Bw%_Vg9LG+;eKW85bO{ZfrU5tBV9WK`Qp2S|x(wubPSJ~djSI@6&Kw3^@fiL6
z&|kG}L9rZ=rQfg>-&)BAZf&EXh|?+c#eHCUr|^R}ny0E5iv}#c1j&wWStPInn-yZe
zrG>+S3*q&}G3VB!NMJe3{fnjyv6xcEJ~#0pE<Wu_4l)=%wZLIyamKt%4fGeM*ogvd
z;GR}<XhbDK=D2!d!5g@wlDKf@gb6DedMp86GS?tmX)r@2t#&|+-&NSF?;dS_wKwmn
zz^m~Secf$sMZ!$j)Nr@pC``)YKEy`xj7;fK${nnpjXuKN0})$!u)RJf@mG^!Qa>pQ
z_b72MU@^>6qoGY*l=wYM@>IdHXL=exONK}92a^|xzzmeH%a9<&*9%EG6NeJ%H&&-e
zC=~1t3o90B8kVgALy_@hb&)Ti1El*jpLTFaObBVb0JbLexq6nK(wEm%Jdl8qd+u2e
zK9gFcl9a5@-o-+!f;sC8SGqeOv576@*f0u$&6THAb1lgc%HY;SsyHg<$H`c?+Z%O4
zMxUciQ{;@9&mPPlP36C<SM@87<JOgaeVa3HX&nxM!cM%C92W!Tsh^y<XmcQBWmXdp
zAuf!@ztY)EIq1PN_r&jy&oqqj*|2DWLJXVcE22J%?PJ52AY1!T^#p)td`<XZ{x=F>
zdrIoj`mM(oS{zD`HEMGajuxREHEqn+g>oQ-q0x?{dG;T2p05a{DZb$ZLtof=={Hls
zl@22E6q-E+iA0rrI&jWP5(nfsmZXjGhir(6aO8%U7s}v?fudTZF%m@Epx#Uar+qd&
ziWFWe9vC0SME>_KK^#~c>b<zoCeAEJxY>Kqz%N>QzZ#e(j>LG9XIRDb>n_TjUrSFE
za8ab`zh0uUi%v|<x}EjHC=f{)@}P!Z`!tYPh}uKLgFj=(ux%cNxiIIrA|&(87tRbl
z-`!*<he1=CAh017dCBmvzcYRdyD@mowhuRdngNhiG^tVPcY8M1^jg(^x-i)E?GxY`
zTX(g;J5Ih8y`0mWE0M*tpObSO{XNv+x~o9U??igM_9exUFNekXv(x%s%c9A-WgV<W
zr#AWa_fn(O&)74n&9Ct!tLO2e(=e7as&tw~31m8y!kdMo_(2WkuLgWFNb@!Xz0I&f
zPwGZ7dGT7~+W4=bi0H*h)Pp}o$RtlTI_-{@8p(d&1lSF~r{wW#{ben&;d>$?kJa^S
zJ0E-o+sK!^{=IHDTPmOLbnAys2=Aa;1cdl_yw9w>nH5@Lhvp;uW+|Ya_llZmi*^_z
zQa#OpJt=KcL)&+A)uxaG)l2F9R<Q`l;;ZJo7(N67Ki>_n`Z2|(YwEkQzjJSFz!E!D
z@&r+244Z9}G*tz9BU8NGCbD_8nr!+ELj$hYJ?B1TeNq(qVE8$~#Am+@QkKi_OvdZH
zkz35Si^IJ;MSJt1+xZ|Eq9&s_+JSqU`&I}2)U(qor^_yy!rlP|KrMgpF}uac8L-7A
zSnTo(hnXUI8F7f|c{EVMfvC@HxpEG+&%mK?i~W4@e2p$WVZH0_XtPJnr~C5|KXFNv
za6o|ATKonPK*SL`=Zq4LipPYdWu;E?5pmu_i=77aVcSTqo(4j)sR)A8#16*xm%h`b
zSNmPKhDw7nfHzI`P{SUbNK-SlJCVsrz24!?-Ru4w_5{L~s;5NG&Nh>|S#_q}`DV?-
z!*V{S-{JIGR&JACkEO@g8OgVrqhMlIL}HRB&)X}ji8?#|98L;lGwH}QOKoW2$Z{T!
z;DLRLRGBJQ^1}LY@j-`vRg9L&P_Je#IV*t|q0H-#1Q)d%4=}6aGgaKq`l32}{!<z(
z-3MSt#$;u5k+^0dubEa9;`I77QYMWNOZ8HOAfaF`tp%SC;WhqLi^(W{OmdYgwNs#q
z5)p~9P<b(}jDQ#m0f~EUVKROY4=iXjn}079HYI%i;>~8)`eOv9ZihO$h*3c_!_NrJ
zFIf!sUNQ>m)LTg<(F!%*>Pqqu&d3#wBe5o4BxV5=#1sgHvtHe2hK`+#5$HN221#t*
z_xJP?{P@n}QltIdYY1q)%;~sa!rL!yu~@$FN%PSH#-~o^(;-QV#Biimh%y*!J$v7~
z(Qd+7G<CbX@<{ZK9KX$l9Ixpor*D2J2Y#&nMW*r;r8jbBGIBAW<!V>Th!m_u&xZmI
zcQCp7{nC!1<F5+!Alb1n6r9q<7{Q_!IPJuI7xl^!^r_yNYM)N<8qfAz<LwugM)aEI
zahTt;alfMsVdxZy7I&O0g3%s%Z>f%7N`e_mXt~<gTKfxjcbh{)#m5`TaD9KLe7CA=
zGU`8jS8;0bl=Z!$srXKsR1>?m-}D>y;D!g7g>VBGev_ilR82~cSm94!USUj6);&`2
z(zzz%Pg~u;G)@g#{jsXhLkADe%(*-0VlJ$KPpNf&Iadt>5xUo3JIS<mV3F7G9mkEr
zto=(~v8N87thpbHG;HvV)r8u}kd`EWn?Q6=e-wpPO{Og5XRQxyHd);ig-rK~$~HHt
znR!*%VP)o#Z8S&jo)2^>6&5tn5o$+0t+*?R@d>(D#n+rc>*I;OMfq9cJ`6pzm{-$d
zi|F%)IQF5cmK%03Mn}Yj#=L<q`hJ-ai!3RCQ)FXDQo2+LdARbIB0=|17dcUIISsgT
zoLVlR#g;&G%p0;IKgeho4q7YSVtY9X_j_jV^uLH4HzVIXyo$;iO#Ga-dc%qTQ<}nF
z7D2=5n9~^!harpB-#f<7#uKc1T3bqCra(2lY9J+Fg&ZhSGH)RDl$`kO@+I6jZBK59
z+KWt~AOQ$)%J!tf%-TD&K8PON$K-TL)FduJtpl-Z5*HRDIatC2v7){xWAg7p(WYrT
zlv)G1FfFRLh!_UlEc}a3-6e4QNcD4)kyH0-60fM$#?=)^*buI$TUq#z<=$bF&R`)A
zJS<0AOeJuKeK0^FO0VY+C%Bxtsx%m}O?3yf9AsKAl!|aRIPDtwQG9g5+dDDn>^pnR
ziU!P43x_JFNl#N_r)g*qKkZ8=QmkbwrI`U$l)zcSQqv9hXK>_fc*9!WVE&-RJp%@1
zFzD)gkUtA9_-ZNWF*cMBIWryEadAQYB0r2DZV^%Pu^d}HfK{KnyV4X+pX&`Ys*_p<
zL3)r3fuLDg_jQ%N#tv>tuvoZauAXLGDBW{wx#+UW_%8!5ekf*xteashNr}<%f=^J*
z*ugeMQb|O()$lL4{tanWV<hSah97@A+8uBqwz>SI1qWM=&v-Ik`Po*>^qqA`Z4Fi+
z-1%=mxCe<}1lw4(x2ho}+p+_gOEZR{Q~W+mN?2MG2kxD7%*s`1<+j>68TN*xHbye7
zax_+8r&RNe1^JK4H)ZnP6h_R-xGaq4QO3g<$euHCbWCZYF&6ezpn)0R!}rId3jT#J
zDHuQKJ7~hbn1WMDUYE#19Qi*XlVmkuE|VRo3!hkjEH8{YI~>mbqkr<lH4X;l5P^mj
z&t%3AM0Z_N{s}tXM8JKNbyL?VNsMB}{j=lu@uw|5_9cFtWOhC>h^Zq|9JvVVJ1vCj
zdk-=wc!`@_bNmnH3v?N;q{P94V*eM$h@EHvWGY;{4K4nZf%B3&<z$XlIE;$<m{!qX
zN%|EB_$~54KCwI}JrWUi!l@)R!-wzUQlm~ZMN%QiYLuL^g0Uzz-{Q6SO~#1R+fy>d
z@mEszIuKc5R0H`1SGIwcC6&=EeTV4xSluL4ytr!?5ig)><E)%a3{s<y+`ZCks$UE>
zb_q&?ANxwEItyICTg4{;i#lPrbMY-=Z>u>ejyl$+R`8>A1lJ!Ls^G{I2*N0slzlXg
z>TE>ulGIS7wQk}?n3OKe?`jxE*ZXlhyEz#>6K|N&3T6lI^q|&VuNPkzs9lJ<%0c12
zM+*p~TBxF*lQF&=LS}vy1~?q^jbSX=by!%`;3HG4`)d};bQ3LU{KjYvp}r?DjSPeo
z94kHyg)@2BgRjrA`DwkTG<GszXo+%^)BlBt3~n{PcJ^KQ8Z@OjgpkEfQE@e$*D{Sl
z0lK0eNdb*QB}j2BRosuCh^X^-_*(t8uGr!uI+6^9J~9KRs}+{7O<Y4ko{A3kNW7K2
zws>oJ0EH~r3Psh4i&PaJBAJ2=GP>Z6!9GVzUG?P+;yjhtn@uZ`8z+bwCcPOKkUgga
zN#1yNEv?xUZ+4?}w42}hXy{>ZF2|#Y6P#%5VZ(hVFuVs`V*0q@uf#;dN#b92`=l6v
zLzW7^&j{HpzD`kAv_c6p!;Viu{Dj<oydl>)%A1XL0wEJc0cQmkgxb7LV1lbW-~(fR
zp`N$!!6RIEkc!HINosgZk{B}>IJ-59B*QB`4i_%A3NftOyYf>HbHXZS%s;N<+=F1I
zs;z3!tl{jo%8Q3aE1!f9S@u|KIRbkGKx|<y-l8N^vw!fn3iVDSTg}+BZy3tZ@!gCw
z_tSv}41bgqSt*+*+UM#>{&2Cr0?3KA<DvS7XwKnrl5c&={17O+@!2dTlcO=(&q`;5
zdKI1X(bSB}rILDeo)?#z?{a$8hHuG=jx^`3mG`oQfv6RvS@t`S_jBb@D~6#Mw5KnA
zm?bm!FCT_|H(>9JB6aJiKP|nHTtoSk$m>_Ls;Rd%<9=>JXgxlB*wC&+P~Z^T)B5&D
zrD}m_v)wnkYJ-;QPtk7nZPGxDE5EvNeNClt$Ok(h6#9XPNnNioI&71e*Iw=JVt*gV
zxuQle<i3ugaJl|%%%s;qk;bf>pi}pvRzz5`H0B2@9galD^uT)0*?eP#819%}z3Ptq
z8z)2VMXXOja4I>IiG-XX5if`&@2^iP=nHnzq+2AiHQ!2`UF^-wbq5Bne0oEoU9B~8
z$ybmAB(Othbn0Av#Rg4ovK{*t4ZOW5qZ^a2GHIn%8hdoywhR`tS@uU;rQ6>AK-`_k
z$v9Z3RXqPS<=H+(KnQ0FZS}wB!(&hjV$!Ws8cSzA=M_kB<^UpH9km>d6>vXaF4a9M
z#1PUhIUQrMRjO~4S;W&g+`S&A^7#wIx}pZR!`O6KNHf&-7VF95UkarskP1jj$KcBE
zsdo(_d&Prf@&L!Ve7)UM^Y(1FGhqrE7Y}&(^_TQ=;$H%jakors*YE;eYH^%AaB+U=
z$ZoxL@<@+j>8m%m(dq9N)TZRbq?oPAWjAhv-Jbud_&JgD?SsA^OB_5<ybI=c-bgJ8
zwx6$_Nv2h7mQU5>#0BDfh}CsKLg?G;0)Lt3^t`b58KImjzV;Nd84ZW)j@(MSCwm&>
zkGkx+rX!XdKA?KQHq=sF<Myz=yEM@=662-qXeuMzND4P6KX|d;G@~Rqo7Z8sFC1;c
zSdY!P1E3#3ix+k7c)VKW@%(Kh=zqs!J(X`<Ao)=%LIPwWY56rSiOdd2LvqEfu~dF~
z?mxC-Q!~_{Qm@fsu11&r7Vcg|YO<#p*{IV;dnM1rdlxU8-#OoIYy?5^xn^O2b9Nnv
z)$mg)Tfpn7{O6SRzKSURFd?!T;vWFl?M)X+X?1wLwM3rKxI+N8Rd>(9ChX6ag#{5^
z)>a}s^><eAsbqggFyKjj%N20zh@YIwc>&Oc>e!aZ#M=T@24U(FE6sh<X#gz(SkD+2
z14%ZKtPEJ9FEoXmm-?B9wc1J#-at^eEbsf{^(Mb9qu!Aj1yC1(u<!rXO(P_@OZ41|
zYWj62jePY=i^I1r|NA{4<eti9JxVjU6-gM(X52OngyU6!#9hh(rA!zQ)#Nx83MvaJ
z2z>Gu7#JFw2!_hNfE$+%X4;$J-z?q#RzZLDY7-L*Xx<BxnJJNL(mWy01eM%Z5>O}<
zksUqu#2F&!aR37f9?E|VC60x^c+4ZOaDD;M$EmCSQTSp%&lD1ae2CfdW|eQ_*9{^y
zXegvuIu@Ib7VyW?;LIUMfGC!VIDSh2F3;XVt>G;Efc@@cT)bh*Q(A{{`#aZdLYc%w
zv8UER?jLjL;eeJFRgX0_=y9IS>zbzd;&nXT9p;m7*HFgyRB2Z1#JMje)2ZYV&g!{r
ztpm$*85(fW#g(S~fP;gNPq!G?sK*k)43BnX3R!8klZwS>rNWW~HpfE_QH9g1=T+)j
z=OcWTbgwmNNxmlrG;Q--%8!=@dUHM{Rt1a34wJW9T8jci6a&nJnF>)PF+M&`>Mx7_
z$LWNZ7K=O>aXr)!gJxR^#SFGI<^>zT!+L8DSpy>HE};Xh1b`nVzf@xexSkiYitI$6
zDOkSPAe%X8R-%jDe1``tRjSRttK?5fAe;-kB*Md!MnJ<)^(C%1?cRXV{wSs)2nIaK
zS)jyY*A`SFjS&|_56l@z7JYXp@H<1f+?xsg<hU#eSRqaqc|CVi=*^Erv7Ln!cqEL{
zt^UaSY|<fqPC$|9YSw(Oe~(PaG*GMqo~M$IAX@@IvRSx6+>1th?XPh}ozbIH5rof_
zA5LdVWN|)GJOBz;1(aUUu}~5)azYxQ{($AMTWT=pv7Ig)Ap3RyTvQZ{VhS4Pn#^D?
z;34ey%$3@s3_7L<f(lG;-~m;_6alpkm{0<#4LqZ--h`dvUeTv~VPC7bYiZV790;^O
z=&3IRb_YB*E2Mwp{5Q>7BS)nKb-48W8=%xh*+79UogUJFyp=EFyJH*VS$`geW$FeP
z{vJFiP96|I$oVcEmQ}V?kREyqUHbGU=K{a~g~gm0aLO|RT1JCF8w&id0Yw7Tff6mj
z%t2&fNww7*inN<8322Ur3@Rxt?(>H5uI)ab6uJOvC`KbFb?c1i*x%R^sIE6+)Wlk_
zp#Ys#J$Hu37%GZHenur7qqbOQY}Xftg2;*xn}-Xj)~#oo57IMFy>3uJwRnhoR+N~q
z=)fZ4yQlk;-GO2tLzCTFT4dM^oYwN5=10O;zuQkD*szSwtN`A;XIYBF+uxfh(fVj5
zAHuvfoJ1W@E3BA91F6=jRalC_Zci+eOQMp0dqXKB4y;{Vx^Jgj&(HDlVE$tx5M2o!
zQ6w=K)V!|E7xfqPI9lRtMx82~hX9sF*=#P53MeE>8B_MV*@|?%MR^3`qjY$|3TaF;
zK#Hr(pMr{ts%qv&K&IzemM>Ay$pOol`0n=_;~Rt{1KWf<lP*62bAf36CL8tDPj4pF
z8Q7$I0G1)Y+UXrr%X1v~9o}05U+Y$<Rs?>oT{1&L#l&0`^j2YqJ!hGcVsv6}`_R~z
zj9*YwQ_qh*IEX_69{$MoX?z1p6pDb&PtpKk`Dg;rPAe)y<S^zZTmW5kH5>z3k1gBA
z1r;;Q{&f5M<S*!V!x0`HZkG#BPoMw@BTVCs7!z|`YzDh&IyJnLc-$@6%=@iZ$V3i*
zB4EUw#w8o(+9Uv8@oPh0Nu;wH%3K~URycRvMi{i%<F&>K&?{p#EIVqox!d(sf>zZH
zYIT%ncVUtMJPN_8y$8&vn!9`RT78d2V;BSKSC%WFhOA0D7QfkLOVWO}G@28wSK|;B
zV6m~b?DsGU>e&za++QvR9O^sqpF1zWiUN#8(*UP0p_^3ZfPY8|wHUC&DPX1ul?3@-
z+bDfsX_p4VLOHQ|=%n7PG#Ak-rmOWwpvP14C|3Cb8{FMg`U&S0h-DQ6#WS%1Pq(kD
zH7gSSa&4#bN>I?z$t3e%QP`X<Kr~c#punD`d|>L438Jlb`!%VY!|#k?P7|l@e#AUe
zXDk%bNZvFczyUS@G_4RX8BzJyH&wB;F?{ExhT!Mwq-l*zQ^)u3W{Rcv8;8hBUJAX-
z?Cvq@3wq9JskL5aECaA%=Uuw0=xwP{o>n0CN0>MW2P6cX06=j=DZoumhPa?|KqR)P
z6`VLgDIDqL{x?}Rx$BE>sw7QVbm@XjOfjegY)al|J4L{0n%P=v)NBS!28(6KBlQG0
zTo{1bg^M+}Dd<A|Rv|kCFn2P>ZSF-t0)94@y{nRaOOIF+$c~!D*k5#F^+#e>6iG$S
z2E!v&YE@};(GN4)hJds@fwLr;I%G58UO9zF04mdi2gpVjCVBAyRKB@tft+zanD$GM
z|2M969AA|6&(ubx12-)=a*fV^b~X*^1Ualq(N9peBCoZ3CIaba23cD*w6|uEhvze>
z`#rB&TLznPF`x-!9*$i%+i39VJw{YDGJb?nHCj~`b$xH`+@C4ys;O`K+z%Jx8&K{>
zQs~Pv6i#VTl~I0gbTY5|jAgK^c0Zgc+J^L$VLY2#xX=|sqCBxc6Jq+UCQ%^#X^H$s
zk`cfG%0MbQBV~w5=Ywq@wrkSGb5bVXUW)!>v-742aU6h#m6w}dFGgz>QGY#N{ahA<
zaX`UB-x~})<p;Wbl35DcF8~S}V+hz)I(;sSIW4~uHbB1OdjlncJYJ_}UmDglRYk0i
z`Wh}GfD~{K);z8krQs(>M&FYELOTd1L0A)i0Q54;rv}vT830%(s<>wmg9qq7SZ0ml
z_NB0wnp5N}t6{7@^2i>FprjtabnTXNFC`?yQ)3|mY77qkC~<B(R|$Qy)|$k@)Pu-J
z`ViX#NOs2-6gz5;B!CeoUifS%aAy(6UjtSZl9&dZ{CaE(uY0Hn6|B5adg(&Z_Z3qa
z%o&#T^(=vkQM04F^Y)9qJc~z4FDw9*&7Rzb+ao^Pfu<<HTTunJ0J=6|>zc&#ev|7V
zp*5_GXfU6ctJ2&><daESwRN>1I*rDmp9Qq&841eZ6zuXo7rP?X_lQR&?p96Z3CX%J
z$45mW!_q0!h5$3UYg_pC$5)=$lezDiga~xV0cGDSA#-O%csJPh^J<BkAZOie8XxWh
zTMU!{|Gq!@8!2J1_;Ycq?Erc<`NPVVbXZH&E8AFcpfGO4rfKAup+z}M(-H#Io#RN%
zx@=bfg}6ab4py)Pb%<(WEf;I05y!tW-G|^Fp+WmwYJCf;Q?w7c4F3}CD?rbMBIUVM
z|9(Y00|&MF!M^{LDu@6JS+N{t{P)X@;Thg&8?&(d{|$ixhQj}!Ise}@CrJGJ*KS?g
zJ0YpPmadmoa)YzweLItvhCIi|!CIdkn3=R|i?_zU!pv@%cEB9qJE`w0Vi|iWeB{$<
z=LD5jEfxEDMAR4brI^@n?wk<(`Q1^&Wy3Vv<>4zYV$=NPSZ~y`kV8z7P!u>kr)gr8
zl0umHypqmD^m%$;-<Au3IcGR8&gY!>4onI5cdwKgu5rTG$Y*axM?@m`E}p2Js>;W5
zlk)9ul2<*NjKzOoVuOf4CuDA4)v7P-CUlexUY%*)9j}(*;*+U3$Nvo)o%fk+um+qY
z@A*srD~~q+OGMt^o6+{oPL*zUm-HK<Lf4hL;kspJduhd<2PX|jDFV%BO}9?VJB;d4
zcWn7T->oYV{5EQrFERDktq%W4Fsyts@lw@}`}L{bKu$*eOiMTWsP7gd6=NcreCWUw
zxu8?xy;ma`H;bw4(WsHp{b!H5cYcj+fI^aB+M|gD5`juCZ3jsLC6_hoXgYg_CRxI@
zaUz~{k@LN3;j!Yvd){${JW&!E+Jfu9_q$Ay_`j4O28DY=JmYeZn;zB2fT%jtRwdee
zX4U<VR%aw3KyKwcQgyoV&Ha(}O1piEZ9{@O16FSz^xNL&FJlZ+pT;kqibqm8Wp#uw
z>X2&0OvS~NGg;XqWo4#s)?-n+?O$~rVrE~+kYA?y1PCRd>M~f1B?e$gi;OO0{Hm4d
zlua|Owd>AU>V+l!MoH?b0A<y5=)~GL+rg@Eh(?wy-duQRY+Vf99Vf#OYj2Dau!PHM
zemN*XeCl3O<`6}LszWK+yJ?e7noe;>P1`az?I{U?xKAee&_pq>DGQef)JX80KfZD`
z*s-y3&V9OFt9F;UPVH45d)})Yoe4d&?uZ$wqBGj#Uo{YQ7JnFS>yr0Z7yAAW_^_ZE
zg-J<o>wM3;%~EE`aKgv$MZ&5Dyg2@+{aT}$BC*1WCt@8_{q~<TcAdBixwQjJB>WN&
zt^}X|H9)7rK?mMsii+;xf9Qyg&;Rq(Vz|p-wsf$p(&?+Z&q62~&o^mk)2;ot^K+$6
zq|Md&lpuOhD9O8}sZG*rI1%Ieh({wdDzECv^EoGrK;LZjYx3!OQ&!5#Yb?}M{}7)*
z{+R%=u25$rrP4QIABuU1zI|)pZDQ1J<M|^}J$9%B5v)pQm_ZY04L~v$y~gaV%&gFY
zR&gscL31B}s0w<Xs<s8**X?Pi6+c+V(Aq`C6dgCd@kJ2?!s(KOOc~UBhCk36bd7``
zGyJyP@Fx{9Pu2<S^ZPHZ8^3*o8~$s>IhxI1Vd!2NI(d}%?bk+!ds>&DbF*Lh?Wt30
ziJ4L23j(g)aT$HGso(hI1gA-TSiYA9vF`Vy=;lF6Qu^mRrNw=`GwPm4gv)8j`IQFw
zg>3?j_l47p6{Q{EUu%PCs4tx}EqT2CP8MF?{hH#|_#2A|ck?QuMzS0IQb%y6X554$
z@ds<gkDWN=VsaUIqxz>y_VqT`JgTs#yT*qvl?|1vVK3@`VUW>xDB3ezXvu%nV=c}g
zp`9nln0#XOspAt6^!_MAd&yZ4_)^w;0@1VRm*Jcu&xF89ppsC=lY$*rX-PZ3bGtV|
zr)d@Adgt5NX>KIMddy>msZNu{{5r8Hab;2!Vnc%3)CzK^8$Jm<0y0-pJi?j$Az0yp
zA6DRuxJsaBrtMmtko=zuX7*t+eBz!P6E>a!?hPWDp-+D+Qe>zYzm*nezclqcFc+X@
zQ+X2cR=M%UEE{|M_iY=(qf=~>NN&W|0sM{2AGm&YQj=Gv?tj(zrVsodx@Jqz=YP+w
z{YmB_!>HUC-OsK_QHVDw=a;#5ZAX|NJsUCARMKi*6&YT2KW*2#+I_wLTL4>PEJ;Kf
z)4hr!Bt>NkuSP3fQ0b)-<G1p6Z2?~2oc5U$Vt>2~I9UMxScERZAD8(UcsI^i_api<
zEC|tdE3#l&8#lqOXvVL_840YC-`8Mpj2z!dJRN0>Ga%5kDz}_jy+}TG{R&;5S5f5h
zmO$`qKQD6BUFJ<T{ToO$ZOkudz2n4dyUhbu?9Iki0{QKbY&*T<cgbev5E=RkiTY1k
z=f~w&R*&)!q=h#nMB6#AJz;;swCblgk<A;7uN@0%lPEvl&6oeBc6W)x1uFE64EK5y
zoi6HJoF8<r?na{%sr(nlFMcIFo|S0~p2Hxg20bpM?5rL5qaAwTT*B$$lhI`CagE4t
z=r2mn{??S!S^kxFzWzbCPJ_}zfhe|zy!8c6vR4+FYQNJAdzU45iVO*f30f&)OL(n-
zqDng4;XGl}_V!P;A6fhtOUrPw&FkW~Qax6rJRgy1^J~QIxXUtzZt9Q5w5!I)w2yur
z^ah)LM(ZPC^0o0+#4l3nEA;f)R2K}~p<(398K^tK)MOZ<_OB<NN<wx)@4N%=Dh#WG
zL=8Mr!CZS+^Ro6lw<Ii7uUf7-;+>!Jk5DWJl4(6L1nFh`UQ1g>Z3rLSb$a7eif3)B
zoNYr&LuY26JPddD8YVJDo5cxTg5fW=s=k$_!Qk3o9B~_jdo%?W4qDob`ee3V%YL3o
zsqiyHj*z(Fm2jlcelLuSH-M^|>wFhq@j7Rr2yJKzo1=<WknhsB%T9H)0On$KPL{nA
z5H!x*<vh$bTmtVN^PNy_XP>@mh~LuK`S30`nqqkVT>FHx+2Pjb7S1D&cXUMW>e~&w
z-Q{?=hNLR2!D5vE;=K9GN4OI{UmiSMdk}Bs+nH8O<IA$u>30gN0q*XHCj0x-k2Rbc
zw4tqP%zjV5T?9W`C#p$Dnfy@rWm2L5@xt#M&379bK}7p?O7geW&Qsu{d}YyFEoz#1
zy%+F1ora!ya`4_Et#sYX1d&>rdl@9iB2AVqG+?Et{hNI?c~LLK_yYnFnb{{^L+eyH
z#R{RS!4PQ(#f{LSQ>(Y8#3Ns|#A`-}G`g0m-6*oNrEh^8D;5Jtv*H4#euuovQN>R`
zF>>W-!(xus*+nmKdG!~6qCbUwVtJ6h`i}Hhm|Nbi5dI>A;tapP{ac-Nw}OUR{J6aj
z#m8F7>j1xBaCUxQ{dsRvkoo-Xaaf6o7T@mk-BTBFvDVm-oSY;#noh6iZ;qXqU;MoN
zBU&*;q6Crnfag~zGRFH}q-eCyxOP5LWWDsZP=R}fVeO)lIHVfk+YP7D*M?=zLQmeC
zK|YA-@m0YKm+T{O=T#4}^YG9NM|wc7zHS_%l~rd#7^2<W*DtQAMdh{Fa6Svv0go#x
z6bJs==ppCO!|w8N0dh^}<-imv$ki|zM#SHfH*H=A!V<sM_nV4TR9|5S)35V7QZq1v
zwT#hyKQ|3w;zO4ZPF*;Ptl918<;SC{u;G$jc%qyh1JY<hP(5m5<<;Io4k$_L*{gnM
zZj4h!{36^4Um(AvTQZz(WMtp5XWo&_D|Cve&ym_h&r*t%#(bH=B=N%jlmOa;!PmC+
zf3^3OZE*$9o-hm^+?@e}1PJc#?jGENC1?ol9yB<?CAfwV++7j~8r)q5w?X#s-@D&n
zU)<g2zM7utYO7OSr>CmxS3>8DL2n$BZKwZ6$+LX;)|u5o#;YTWq^GZ^V`*^~;*}dk
z9l2L{CP^DUA#?f1Z0Hll8A<fnj}1yt8|eglQ@DYM++S$thva+jz2h0Uam>=>Q&%<Y
zUeB?ZPZ)28n^b!=qu@k-Ge!7+j-`LxW#}ZB>uPF2D93V;0M};1A!Mqf2l?h%%GO}C
z^9`=agR8AIauLi6Al~KIZm!L6S1l%ya2P996>~D<qPqX+2q$NudXt+yUCYmdYC0Or
z{yH>)Xii(<PDg@12@iG0u&@y}HUhum7{j(LlP-BgTEacpYK}0Y!f@$;8OvWdAuM@J
z(FcV7p~`VCm94X(tJNV70w{|h%U)Bd1UgONw>p!?*`mk*&)m1<Vzj5muV?IsN`}SP
zF&Ez0o#d4ZqMvOXg4c5>5%46i9ky>$GHrF^o7V5TwsS}0r*yJzBiAMdj4$mXny{*k
zewS0)=vZVRc;0qRAcmxolG}9X+TZ^0p5G^u72dltHonKD1r1X2iaK*wS3?AVGZ3l~
zD&Lo3#gZ@=ieChC2}U%;H{NRIPW&`F4(&U!jSOVUG=U2zjm=8iMGXJkjHCe8UZ-Me
zA+^fRJ@~$F#7T@XZw&m%#FTGX=8aE=M=K;D17^A^yZz=Y?8(?3QX7Y5B(u(mW(ZGe
zz|VMjdvED8`3^fsgd+{X^Uci363yRc?6|+Wzi;DGRUZ^veyc7tYLm88Ko9TI0U>Cp
zcf7VRH-i}|u9XLC^nZy*A&<yEs6Tsb@fR;&H1Qo5GV$P<cr6RwGboIRh0`I@HhEdq
z8b3OZacayYb(+Z~jXA=6t9`vT2VELU@y|mwxyFaLM+k_vU6apk0f2%yCH=;{M1*5D
z6af)>w9G(Nq9$Le)hbs`R0O-P*g<^Ss@?Ohngs<|vTeo16odVj>TsYu0U!c=q=&L&
zB>6@Bm`&##^<|k(DxU1wfXBkcP~k!DM4@_O4F8<Ou-D{vT-JGG&6G)>F+KCvm(S_P
zC>V`Y`)yhEzCLi9&3|$#KzcNw52n_Ie`RHQ6?M4YMEXXjZi5y?JWk)>Zz_s}m3`|A
zJrlRj-wkC5Cp{7&$|pUbc{Pn1Hr7+J%5mHm3Fm)gN$QcJ(BlT8#INo)qzSYcza?xh
zvM*BHT$omF`XcyNChwX!NeNv3cT%Sc=M_!h!4B<uc;tLlLdowzs1it>!|1$@UEcrB
zFHuJ-hz2CrFx2NKCNNw%a&qfqSlV(gO15iJ*#V2Qinl?2n+s{5)h>zrVBtTa&Y_`#
z0X)sZM`;zl=1%8#ffpdzy<-L}Wk*hls6lCL{C}IA?nX-AH-ADX(|*LpqvVn9Yt_|p
z?-kUQ>OY6krS;VX^|Ht-k0{#9$Do)c1%3pr7qVU~3!6GG8`7w^bFk=E=*RCM^YNs!
zNyD|nkW(^I<BNp07HkDK@JqeG>1^<s$e_#3bhBOJOGU7Hl|+9y+)M()obI4&=>^ys
z1IhdoZ!P7N{<?d3AW4hmFV+Od(t{xq#{tV<{a%09NknkqR|?^0-VHaCHVE?KQNH`L
z;*g*>9`qu62%GpOlzR#~PN;G&Pomz6EtWk9LrJi`iEmC(8AC)xLUgb4(<Qzoui4r@
znWpJ#>pi^YDU}YCC5bJpumRJro0%ZI@7w_X@vt#^N<q5N!7J3Lh_@xZY7}1nKevVW
zh$hv9FbkwQj}@o&k6NVR&zBnCqyIh<lzb08V2cae)b4F`*bL9;oC?hq7!s~SwJu)(
zGaL4#@a3=lW2S7mR~kv>OPks>y`hj3@;9nz#erXnL+-pR{zRer#kj|1>-OwA?8kZp
zk@2%V>;C$(j|IW_m&rwpu)#?T{T}!~7`YF1OK}l7k!Fh<Gp?^SQHWOM|GjO(#Ckb<
zG1dF{Nw?~Kz<ydWpUhU{skZom_gP%BFlsFdWf}{7Q+IA*S+_<&9}LNxi-qub!`xdt
zAT<I{B?nwCRr`4!rS|pD^X(C}F$f#8wo~dYR)PasQE+p$-`*p0Bx)E-*BKzs#&?5j
z`n+X&4}YeGe<+Px$>Ege1?z`zRPGU2Jb{kj>;rS3ca0-Pr}V`3sC_p~QwDAM38NJ3
z54?FMSg}lDDWob-g5DVBk3^o?r3Y_3KT5Fq9rwHz$Sqh+v{Nm_82pN^Ti~m*h4G`m
zs2Mo@Bwes4C|Cf>L9wTZXYT=LZ2|p8T8@V(wIlssd1D*YHaDn_oU3AHyGkRqF4)#M
ztvvy!KcIXr;gC_T9U8y-8{NP7u4e|1^U7G5r@NhsT!ApwNAe`YN6)F4t#8aWQ~{Q@
z;C>S)4M$|()aQ*xxW10m-fpSBY}}xZVa^>WvwPanB>0|xr)<{}*zt3iMklB1QZpe~
zjB@$H_+%8!HUT3mzii<sm)fCUQK(xM7W+7i1nYPHC~*A!JHJQK1_y@_@PY;4ib2jz
z>5L(>xz|F@E(<vo4E8{iv?|B%QBO6lJ`Tn~2(C~ioFk8d+eOj$pu~7f$_sAVx-wz^
ztIBd-FbX)#8q)o58ULuq<%q_7(pN)jahWlcj&(IH{RmvalOe@+!4Nt85WJSB2m~`)
zMZ{ynV|?E9cewyLpU8vx<+?7Iz~)ar8(HMB`8Wr!jhrEkmT>-5d?)X7*JE`Ur%*y@
zWbIL$yW6NIfjuq0BjwQ0;3H^GV=^6roT$Dg1#L3GkPXl;U!z(dh=fc0Z9tcsr4#c>
z{Ih;9)ei{@l!iRZz#LK}9|EkHF>zj$<$ffNu6a}VKH+i+hA7nc<x<Ners~7X4QbGQ
zueQQ;xTrv!`~?NzAgFdmUK6`o-?yUB<IWl-o}C#4t$9|P67hZ(7%3REqz6$<qlw-d
zh{z6S<dyU5_r1b%NL9U8EEf@uMLA_54b3g_XZBDGZY?3*n#7u6)_kON8fc{^9YPj4
z5D|Oc`5BIdg0dp2&o<<8hg<}lvrO&@E#5N=3r_qihp*2x4458O`JTVx@z8(t+f+2$
zhU#jK(z`|f8))18)m~Tt#0Y;p@;f2>hIrRriDHtV1+Km!WboBOn^~CiO$8L6=mKNK
zr~GSQ*<o7YcC{ukYAL*ut63C++Nkpy39*w+^!#c%d<7Esz*Qe3v)?b}f}n$3MjUEN
zlz?L58S#`7q-hO`^#c7?H#Z<=xnI)3lPV4>NNt0-NfX%;>|F9eaE;=ffTd_vMzg#s
zzK>fa7fwv&oCI&5DypbpH&{FX7yd_3G@CAlOc#sO*!49&dnRT5fjN4hJ!5_#B0(KF
zTCgRF_&1Re-r7#Av+wXU51xZGC5!5J=Gq&Fk#N;xS}O+(M5cYEEqAs20@<8gU8s0L
zac@u^PR%6IwP|hh+E8y9C(zrx_P-g4iDd4%7u2aLc13P}(Z9l%4qo|HesG3A_Z5Xb
z|5dBBv9O=X_0C^$L2qKOFVKBbBgC3WG52i^HN2&1PwMQ>xSV8J+qJDy((dwNV&ksY
z?D>ITk#~gF1lE!h-huY>ViIsFa(1x6#~+vZM?jJYI6-E0iVl*k_CbBM;wA^L?D|WY
z<F))OKEVlvRdCF6)R<97g&_vZopNUi>ATuxvkg%kzBbqXg5f5JE|gvVZAOqSpk$Yg
z=2i?q=y-mHf&752$no}ND;{T~E@@9a4|X2U7Rbv;qvbP7CMMJUh*pOvYUblAMqg9d
zY1yq}8)!!HzQe~&Vf~?JD^p<gwCi@C4eR7`vpEsISjuR2%N_S>&#^gK+Gc^eLGQ9I
zL{{AxOyv=W#fk5>4N2sApmfdmS2CPsQ{uy>)Sbeu=8h{&{7T<wK^L;BX#&0UN*%s;
zMF!7tnvX3l0xc_0hUfd`an7F#a(9-Cs%4%M1kC}LNt(A4QV%)D>30r(?;g_!UEQdu
z!l?OfAADHDx;ouA8?#u6VFhixR&Cmyzv7kzYLixfq*Xlla(k31?@gGhR=8CEB>gLe
zMxn{CcV>NWfhJMxKs`EHf^B6GX3gJvm{ys{Wv`nALPUSh&#=PKX}*uFdDinrvsC_b
zG~AFnN?WHGh+-4ji6t>A{?#<rb?ucK$XJ&!4t$OVk0xPKW*0xw421?*&9N;OdG04_
z);dGtEv&ie#6cN!;*cRazhV^-@I{nb9C9l0njIY|(<LZ?@Pm~nZ5Cd>tRaB#K~&nH
zdZ7G2js9CRA_xye#coX~PyIz4klf1-bw+&m7$JG;Qt){hS&oOVx<lIb2nQe*abM4Z
z;jr_t!Fjj_U?>V8MV7<qKO*45sC^E;2&f<cB&5__sH-@6*tgY+>G{gIdDvEoqEmnH
zftunWE@M;Rhua_T(#!Di^014D*$c<t0wQx%&bBj428fFjKWA#-OaDbQ1~KG7FIy-?
zYM6pR;|YiKZ8sWmh;u=YpI0g{;JB_|&RhO4YBfDEbSi~FNo<AC)e8_nX;NyP%3%)l
z8me*D2M6>4F6A^xUIND3%7Iu8iN1_i!L`F524(~^g^rf`DFsTdEtZ=Pa7qM9CZHZD
zb4>v?8P)K`P-b9CraH)6N%TM??x6Ra^DiV6SA9m_FO68$22!^HS_6}yF@H^<pkTf!
zBk@`sf@1&shtV0po{hijb;5E$o89s(L|uTQ4MQGq?Ms@4bpIBX0)DqC|Awd%3M>al
zQJBFNAdslgNM+Ci3e7!((e=qdjS_R*yebB;F#9G>-2h-2x2^u^t=upCBjHcFK!isN
zVk-~II7J5vruM;i@(cj2pdA<M>I-ehm-KFYfTKllZwkjuUe;bEv?J^V6t?EiM&N+Q
z0j||Fpwk0ZswXWBVSC~E_l5Tcz<@!JZZ%x>|4jOv=7q79?x_OwKtY*X7~2f!0Z#92
zbgCOr$l}^0m{$Xq^XNW=>nRVg2BIvR!UHrl(9#*R6$CgjDUo4!8w(fK<;2tTVD+-D
z!rKhbaG-`0oHKl%1k~hS^%#9m6(0yS8yT1;jHyc!=LCP#87pDSe9&Rc81=$HElL)j
zLMw~PkxU@1j!6N_2ZDte0QE!8L`w&cOC59s$j4Ciy@jJfAXvT&m>6jME2ktbH9YRn
z|B<rP|A!ilF@sI~TPhyv+%mPRe5$moXQ*rNg~n6e9?u)MBiKZv-aK<yt0o>`kjfm7
z7gOK|<4?~ziTN>SiW;*D1>6~6KEP^{nF7MIJ`D(Hwtbq%FB)LvlF7UIm^P$G#hTRO
z@iSGZHD2C;x5EV1@!{#7>&|b`$#rO5ZU~j&Ph!%WY~~0`4Q^{Y&J9}-Uh67b^!u1N
zLFd{2tD>n;L=9NI9dcM#@4~|Mg$HNDP00D#h5+t8w6iw-Y;&8)bXA$ueUDi#jRo&B
zOQY#8|3(e}l_nE3D%bQZKZVC7*}+3Rx&(WOe#J>4QH1bYF~0`UH*yga2Fle=Qz{2M
zADI!+2)_vCL{Y%5pI(#ARVN>31|(cmH)$jcQisJlq>c{oFjvyF^Z6utpYWoDbxGnj
zeBJf0C#DU|zcHrmbb8k5W%S)M*Q`6}FGuz$dMF~@uUhs0ev_*Ov$qp_uZmy0bVU5L
z#`?p1MM<-21><}zQqL8SpyoGxkTzQP1FDquK%iVDrc9lsl0v^dk2RF4bsIJ=I!n)F
zb(JH!xiu+!vqe2uCUmuUMWuK{Ha>^?%G;-q++(t=Olb_6BmU4gzD>wMIE-4=c#8@0
z54H<Ezn!rej$;)YF_(!?Bz$C$Pf@B#3=TsxjGSfhRl#7+O@bk*q|(NV4lF?X!&gJ$
zESfg1%3Qs#7Bo-B%1!Y9rnwX;qKF0r)LnMgSBFYM1irhvH#b5rTqXPKS&(2)E6f);
zR4lEH&osYVFdDCIyh9`t_DSVM{l0}K+SnKiR6%>sXZuPJcXAotw=ob*%TqJJOP+5>
zK<ywBwBENG{o74=YAK3DCiIBtL@qJ*qm8ET3gh*{-HEs<OR`Os9Q~Ub-Tj{)?zX4h
z9QJ$-glKu6o(|RSKJ5$NT}3z**+wO{cq9!O4Fn-P+b8uGtLYB2tp>3A+&tv_REYUE
zsPYC}G@lI5wm<qe*AEUw;wL?~_$;!WUkx%_MSSy_uXl(b*ZJ}Dz+G28*v)Q?^JKnV
zD}KnVn9=8*^>UL%=&!TF?eoSFw|y;D0F0710fdM)Fh)FrCM(7)B=X^sA={-oxyCi<
z@}uVBg#aiWpQ^c_pgP`Cjo;wDqgnL}xwq$7TfgAxZuRjG$9a&5S{N*^s3rFebG>;b
z%x<?P)BRh29MkLXBRbg0cQ6bG7A^VtDdq}TovOC$^U*TxYS{(0vsxT<Qqrl>-#(n&
zZ|TgN>{&f^<GX5YhfjOmVI%83$-iHBK1>D5Mb@fSMM4lcavd5+O?FGa4*R;J`;Qfz
zRg`ZAD-PBF_<qSSl>aOpXmc{iCaQBYh14yA!TVjcnvo&L<aOo6UTR0$AQQN`C<?jQ
zICj#z)L)}_aqB66$_=0PL)YH2xuyUox!Z%nWZRAYGQS)p*>YXffxBAt62;ZyJb~f&
ze;k7{$AfWLL~S(V9?g0a`U|~h1b>pR$J_Ir)M5Vh7()9~RW)g1(_7!Q#0z~q>o&YQ
zHmVuq1hNx1{l<T_8|)IbZw&=TO{F3*D9nGck)uphTg_$*fqu-BUi;v+wohcGpPBhn
zeO`^lLzib?*t(8)bN<GX3<J@77hA}OUn=VIPn8(zj!jz@%ag^GCM&!AVUUeU0y^oE
zg&J@EaX(jXlHKD1`iJZ5K-gM0PQA|V8V$^`JvQ!@Aor<%nU9YH$(^(1vi!VSrT3lt
zz4s?i<=eASweO=mI%9L`xftI(5*8`R$S*t{F_odK5-S;;Mo-I~Y%HWGdo!$p!#X#g
z@YC&Sd(lHE3DVbSaz#+gOQM|6+k!U2tU}=~L@lYDPHvCH8Efoj4D44IXdDOWOmY%M
zWFhjYOV8D?qeq#^P6OjdjBBrd`);^qmVFrRV`tl2Rx{N3dYEeX%Gd>eL(0maS?(I`
zWnoit!?H6%vic>YLQNb64Dl;<sO&1;m23DD(T9G9DnS|fTT$EFW6=i(CTrh3Oz<x0
zAxO)nMkF)*)jm*GRuJs@&XA9>!Lh{<eHi(|X1Lg$N<NB{VML17bjE~Lb)UY~X{Sq-
z)9C?w#;~f?$9u0G5T~e<ej3%wQt!@tmuUp2P&g6cOZ=P1dOfb5(qm>GG%0cvzMseW
zPc4TZFQA<W$VKB6Y2_wot|A-a%LQL=Wn+WK-1+B}AnB78*=-`*229#p*dz_!KH6T&
zD;9o^`pxP{JT+`XLTxk7Vh^oQDuen{cjKCLhdlvUA!7t0E!bB01`0D&6Yo8Pqg2dg
z@zUye_N`Jq4=S-4LDXBee;xk^j?`kQ0$a8hu^{M<1JC}A9IA4;H^qA~SkajOuZTFp
zam)jP6>ZZB1&6Uw0!Dpx@3D6j?qZTG44)jn*YxgT((@-H(i;imot_`w_n?U=MyBj$
zp~2@kq~qb7W|?8)j8n>T5a&PLZ_*pHArSg9MUa1%uC&tnkt>;?RB4;QaAjL%Npkj7
z<7#zg32Ym0j^~F4xvRW*%vQ&lHX`f35M(O;%c6<GF9fcFa)IGbQ4Zr@DGs_tc7KFV
z%3dX_#G{I=iF!;YeiRE}YjAX=d`-7!T<FqoYdzGXVlHKSxgh3wX#Ek_bEiGm!kVw_
zkyw>hT9mp56UxLM3o*l`Ogg(Z3x$KRD|%^PT>^?HDBxz)664MO%2Ni<+quD`xdk$k
zm`nBzF7x4NOLr-dR$Q87pe#9kHkyuu{9MMUiYuyLPutXYbL-``5UE8Q8#{^ae)dO+
z>9xt2fggus*5)pxkS<l_a{v!Yejpja&3bTDZKn9Ax5|S2l;e1~u|r3ZPn_8{*_5?*
zgwUsRB9mqP_<`THh*#%NOGH+k_R}s@iwNiQhqKXFD4=b-N%wT?cYU@VUCUROwrEMG
z=`ryHu5Z9v-oH{BOEBjod941@*JV~wFEQ^b>+SsEY|$cQ{;hpng1wc<LJHxQ-E&cW
z_z<Aqo1R%}?&@8tYcOB+A#Klnb<xRpY!B#&d3P*9<Jf5~wr|a&1G9dc=IeJhks1}*
zTH}py9(a-OvolJ2lv=$o$|OU<ORk8^U6|7!ZThf~SZnCqoPDOvKUAjMA3lq^zfpT(
zhEybZD)84<N0JjNvCgU@yOJ<-5z^k$SHIT%-S|Q&LgdXl5LN`!VNtKzN#~{u5!u4v
z-V>%9mK*!G3-h~vwn}vAWDEHCMfgUZ%kNL%UGCu>77WoX1=STm7ndY%_BPa)0u%*p
za8S9NeIs=9!6cS#_5#rv{UhIl_0bo!eL_$#)rGK#O+fmo(MV;!@z&K}zpF^=f9Jn7
zveDC(&R%tuS@ylLxnMxc)q)T|%4->%Eoh6l4!YTR&_@Ml>;Q)~BT(+_F2u`WlI#9G
zY5jG)(ExqRTv(XY-rpGLf~H!aIb$5et1>|glK!`|khi|ievMuGqiJiFs^g_xD@(ws
zNUH}UgkLrjo>WXwe$nLE>jz&;cF9{*F0J|T3k$Lf(jhEmT!fPrqj)p3g`lWG$|En*
z$92_@_VrYJEEDN@XWZ~R$%MP)`NxAQI`KG9_g^6dCQ`?2%q63mn=SN{_7llNbesNs
z)<Fg}tC@#|f5P_jEtk)|#k?PEjMNte2T|a6aM)|`Ny2}9V01`{V$2c$u7xrY;%LxN
zdeSW9u1Is?^z1m|>i?K|Svu_hYB853n_21|Ik|yot3~uTOJ)MEFHbpIkS<2o2RzoR
zhCNTSr;R1SxxKIDmLR(&1<26rqdD1Y!Zv$2smZh4<IyuZEJs`Ag|}IvnjOZ^UkQVU
zSiD%lkKas+KQ`Bv3AU52cRpk8<)EMcwZ3n4UGPFa{&u_4ygu*!1j4HnF}=fXtz^$X
z^!sre#^GHQBun(!u?8edyLU$8U-_f|W+)*BI~~Wk2zyNj#GVVuvx!4u=PZ&$X<7En
z@hv$)NIzmX&rM=qtM>k3-Y?|(`8V^{*E1&>V>A~B|JiOqyYoISSDE_HvjUEi>>K%p
z)-y{)ezf3gg#+D?62({ttZ$K(p#J$Td9wE!$yDStA?+tZOmQ|$IjWEn$-rTs7+!W7
zK6p1`=lYJVf!0eJ;W0mJ8g&#`1;1gXO_sFgl2qRsh9Fnm@B97hr2nkUJ4F;#_PT#N
z-BoBM8!i%x2<&c0<~>?7RG92~pjy{&HZ_}DnCIfz{X5rat~~Q=(F@l}MK6iPZ^G*_
zLBEjzH0ELVCI~+ESB}13yIzxD0Zy+`G@yWfCeTY-K@FhV&zcZ$aU+^qsYwmxr%~?x
zW<Sw)&!3KaP{(t5$J_Qiy)U?;ZFNgq4F#_o{|xg&#CK6C%}~;ny0a8%;AxLr-w;gF
zo*Yu{z2Ou2$92HjMpG=6t##>Ni7H8Zj1Q;-`nY(acF3qncBLHU=kd47<R0XFYC?4B
zZc|q;kKXQbWg=QCkb>&^K^kF<Q0kr0_upjBE)LUmrvD;)xcyDyQIJ-BQofyLGwjW*
zUfcSniF&GaxeZh9sCg<DGv7#;1j?d_aI+ofq9^ZvEa&WoEG@hJB&70Lh@Y;=WAJUq
z2YHHLHI=ofA|+L`(@KYHqrl6i5nEgQLH&b#joLqSN!uwlbMQ`rR%^dN@cds<*w=_F
z+xqQhnX9KEAD+cfI5)oZUWrH^iz#G^CwI+5?v3Whg~N`_b8=VLMwy?(XhFlcvbMP$
ze5SFVO3>O)u-8I4KAm}Z3uQkuf+;6>MLw|X9dLfM>*0vgdg_CA2uqB!2{*p?n3i@c
z4>g<BQX4ufyf6JKXV-0IpbBzaIpNE$q3534&%sr;sGaJ?Lw59J$aOc-7I4MwoN5+v
zs>&x!teUp#y5e||T6SCq4;PYb?M+pL?Q$P1)gKE}s$tmwe&aTZiE3B3!OVRUT^d!|
zdkDPtHk=CoGz?<I4fvojywAkWvk5Sru>d)7ULO2yRpTax@-_{Mo<BU4j_NWw72;s=
z%<io0iav?hbIuzj7(&?3)~2Q@er40&*NLSBNv^>!H*X@LJ}lvEw8IdqGfp>o+qXI5
zS6+`MD<I~Oo^I4GPP-KLY9Tn~d?Le1Cz~9;=r8lUHs@JlS{KZ)WlyRJOWsFI+T8bC
zpHGT?JKX_O-R_-e>D}IL@JLCc1)VzS)yJ4<haquk2CkLQABWou8nd;Bt78q2A)#L+
zyj4b0g^yl3f?m|MlFR)$Uh}pL4>k5aS`h2iCGmlP^$lKK;0Gr$*NCpIcZxR7O){Tz
z%ppQ~h2^5ao35`g)mR@TJR*<gNG{0#ZDcFdf$04?;9se6dloe#jybH8@V%|RzTm)#
z|Agauue`9S@Hta5fkj}^-TBJ*d<Xqf(?{FYZzj5Ti^ehGVI;iW*lV3d=j|7_LCk5<
z-#ZQHpV>f_G5rN(;D4=yZ@#i8DVAjUbzFLSHL&)d{=PBEV6>zXvQ6wkh`b?(P+?Ol
zf00>GGanq33YD?ezq~jto!k1fEm7ceQw}fPf=-X6^?8TrQj*WdK0T(dhz@if)`b4(
z6kH))_G4iQ4Eu#61Zx;}wCtXIIFq{C7I|Ma869xCU9;+8CnQREKyBpfr`7DVX4U@j
ziDbRUgP|s0KdGOe3O%JZ6%PLT)4v%U_D;^d9HbxCQ$+WzbCQ9c92)JWu^kgb`HeoQ
zxDI2nYb&=3u><e10yKTjJ`rCOf9q}5+}&iml5C;m)0;ytt4)8^Zp>mN*%i!LNhoXi
z$@=2j6=o@Q>3=bf4(S@I{TQoxKf`buE@9kN8QCA99~z;@DE*B@<$l<==hgKX0^eT0
zTe@Luz#g{)5BtI1m4;t~0E=q}+KSe4iU3iLjXor9W_CxVtO>e~8u^18Dk5vYI!W_>
zmYsoje;E1FZR)Tgp4G!n*p2w(E}ME+3n2wzTO#;u<?lArapDKef{wo#y`)H$?j#+q
z4(|l@>HT~)QI5D>>tZQsKKxAQVuu&Oyi7l)d}~YB`bWQ*$l2TlYlo;-7Y<&;goUwN
zD&%G@nZ@Ue$;U{fd)Ze3hGOOgn6N&WqFHsf5)}Ki)_k47QJ#C~G`{hM+yzsaN+j7c
z!(olfFZ5n2CxKV;H`z&YDgyn%#|tbB9Ksa8U#?&039e9YWNyEuiQfGzwH!VQ{7s03
zLrGuD=<%M1$@Sm*c`PYwejRs~`Yhp_&g(GKW*m^SOfQ;*jJTi-zMf}gp5xhsMO_N;
ztbWc`$XGk0%WgV?ypx{GR3jc!P@E<D?osi~gyh-_FkPVQ*6yBZ1cb>Dv!}eNTSW6%
z{A*x?L=KL*n32VNsrb|V_Uc1k&IYS&@7*vMhGuRQkJD)XvDYMioqHauBV^`*yzRO+
z9hbSZj+1np1C&M0PDQ{L$ketsc7bsgro?rUIfR3gg=LlqLIzj*4_pLA*|iaLfyW4X
zc!ZV#MFDZ))^<|yez&3_g2-J;pLZsx%R}St=<xaq>dp8;k~m4Xcadi@7Tp^i#dhd$
z^)Ao;wzV8lQUwvcVFMMuS|UW$sQGr?{a;hFinto82=?U)EyBw8I0gO7^JDlGr6$N>
zY1)9m)e#4%Cj^|yW%3jrI5R?)LJ?5GiIqpr*jsCzx%u{QW6A!n*tB!=I(%<$`mWt}
z4Oqv5l3mxb-d?})$B1^iT~n`m0h!a0FnAA2Za;xolo*&(5v(){4x{(<bK_pxa>jkP
zMTO6O%nWnj-@sx?lmt;_F}H!sIcTzE%>oz&WRm9*?fTZ)8;5)(#-!@e;}!c|460L%
zZ1##j?9Wde&bwjW`*v;BLzJ=Bs*NgI_B#YEe*8QQZuq1XPt<i4tSin|VOU>$e;}T2
zZzdd0Pka1Vqb(XqH%vm0_DgLdrM)_;4|TYAaeveVR$x2tE}~3?!HZ9AQu>aE-14hL
zJHoyebip@d>nZH-zYAX%`gK)ko%YDX+M_M^(Ex*CLbxAEy9@WbCZ;$x(_!y0smd*O
zWtrZ0lI?6L1wom;V3t!s?F{D$1b1|H*hAh5ddLd(htVj{>^!~bO#B)n@tT3Md}pOO
z^2Sadrqsq*0lo~i7^`2t7h(R5>RShTC}gstbt|<Q%HBJqm*pB_w;K8SygqK#bGg{?
zm*fz|CNafSR;rmo4@JDWJeN=>!6rIaHvaykBT<z3YgRE^Hx<-v3_K`C&2X`KbuU-M
zx#M8P+RPG~>|E3m^4|1A@rj?2hv(nIhQs1V3I%%qMm6C=MmA-y!Loafnh!25m=tI+
zqRyf#+(wG+5_~InsxF@G@sevif8hor^f;7H=4Dm2-+l)$s?pdf8cKr+nqjX?OxRS<
zT&PeJ#1Bx_Q8dT-*_qO3veLm00%n@i_NxD6O-UCFp6&5uxlfA%m6#Av69PK{!Utoq
zjuDFRRp$)u8oB{txG^s0#SjtVo!40*W;Z|1bRg*@fQ6b7FqSjYE#bt&9nEkdeM+{j
zQAW!=?cqsJsQlDzt=Z}_TI8?(+N4ET7I3Bs0>smG7@x*Yl^5!+<mL-~@7AmJw7pL8
znO%$|lhT8bdV9W#6TY}u%)fJt{>SI`8Zg_Zagz|CCcvRuK>%Alp)P5?@{0q`1fcU&
zNOJiK2Q2OXdZVB&R0+aa`2Rgr5M6@W<84oO?joAinLgu-0iGJ34=~5`NeZjJ%mp5>
z+M{?P)_eYE&VWB;O*V4y#b{5x2KWr%Ys?7aU+nZtfOlXC<&6JjG2kq~Ww1stw(&Cm
zKr&#NWDrV*7ps3D;N}3g1se#zjN?oTxQJ-`HeWi%b_N(2sK*{H_cEp<(6eKxXVy!{
zAXUI$bc*)=3@FQYy?7;JvE!3o7}EtjN{~r63+2n0O@K3`31`FQrDGI+fSFy)>HW0-
zvzY&RGXMR=taNQ$6sCOAyh|ytUzD`<M+lk>0C9u!wC6QK2dM&#fZ%KfIdYfKu`B^D
z+6*87|5xR^h7pcqJT+mBca4t6o-slSU4WmE7}wyT*`gn<`F11P9ku22@f#7J>#>bV
zE4^7Dl+$y*&8t4k|9<}*z$CPxy8wXqa$`ui1~`tR-?qp*2e2oI3j>?Z^V5$leG{T!
z0JdNVh}H(jx&m(H$#)L^*uhL%#X|s4Ilk^mv)e9OlR8RI?Wxi$pbfFv4?7PN!eE;m
zFImCQM~u%;uo@QQ7Ht5BBxSbzA_SYC;I_6GGpXI@MiV7#&DoSY2wQ3vPrt+0OMa&E
zNvk-&l1)t&ZK6bts^SR1g%}zBo|9!*tW1*K!HMC8o2&Pg+<Pxy(9~Xf4S-f==mOJq
zS&wqb6k?O(OQ3Z>ub$%vKihP(dVshwWUvx_TWy9DUoQ~0xopY`oDb1{2c|JuYc2)E
z=&HrhN8^mY;MK4CUc??PHmU(|4h%p*dK#HylLQ(*V}kl!=lub52D^Finj3)B;o5`B
zFoZ~CMD4Nco_3uVMKFv^<x8#DwDr$2K1JxHNy@a<tQ#be(*ZjX5;$qvERu~Sbp<%9
zwB8d&QAi@{+r*2wzBbC%$pQe$Ozn0H6-$Ppna`QwJ_3nJuoi7PN(ZH0pwT7nO#>bq
za%^q^%$1)vIsjJ6)~`j!sl^@Nl5zPdY|pVwKDEp;>^Y{AEE*tZK|K&eJGk%K4$byh
z`3g+`@E8W|H2s3lgzJ6+l-O>i0wahI#7yL8xkJ6e0MNrxAQ1^`Chx~dWdM<qO5koQ
zX?u)oJegkQn|zt(_l?)AH68cm)k<!-ZS&4fkPpi(I<j%(Ok}(tCT^&&*NIQ$!e8M!
z&!Oe4BZGjv8@0;R2cPSHp%c8px1TQkdL@!$C(m=AUs2sMA~;8h^-Cv&M6+Py-4c*L
zB@DoqhZ2u-43gt%#+r}u17YglYqU1-G#zX5fn*JW1BIm72|rwffKP!z2b$3kC;(;R
z5rZH!E(C3BIwN--g5*P_!u#4Nk&lY=0U(QrWKV&CGSb`hGsgVdYK?*e&m3IY!z^g$
z^U4`H?2nZkxRi(??tLz9-UI+ubfh5jvG8Q<?pAm=%k5I2ybi0sytP!miM9%CORC6x
zvW2T<0+|g7LuvvNnB@CF+xb2L`!GB?Eyn#7ma#B|J(m5C(EVPa)SQXWNBOIP9@0%i
z$L80Us{zmbV?Z7Wt*T3ENdQX%W7+$#Dj&#Ovbccda|fh2A>XXs&G$UpxF_8Xp=#Or
z<{&v!u2a7uPTBfte}fn~6be9)%mkE|*c)qoS0r%g6zKST{BDrHwjN0Da=%a*C-~;m
zc8Z>Tacvopm1W`!p07^4$a`{Glmt^Pq7e_VPq5?0h-usIFUW(t<GE1c_}tJu02xAX
z9QzW0DiAD^rFzGH+=C@b<$s%RX;6zdx7Gm!_qX+XY>%WscqZ#SOL6iWs$>H>HaCJU
zU#=KLxJ{NZCdr_*vvbpVCM0K?=^LRFy?sUDf4e>_MWBQN$hCEL;TS%IN4jxCgFZ$b
z0L`h|L_m5~8#m9FJWL&G!g(Hy<OzVQT=(WngdiEX<e7V*eTIk&s|lY;qwHq}4&cJh
zkn=U@@!W>axcHL##U3xU<TDH<M>ZK*P5^$2OgFpPFjK7&F#ijKyUJVQWrc+0y@{u1
zri}6b3|*lhNXZQq&p?+NF?yfUZS@(5;D*H5D=+8ut`Xi}A3J2%C|h}xQ6O1UPN#i?
zy<;75i~cgGr3_qbu;^-z?fAI(YqobfUkZ6}j~a6a-W@ks?D>JqN*S^L3jz8gfN-L5
z)<suzbRkH|bA#<pE;M1tyDuJ?1@fNdM;4JUA|pAH45Vz<s^yDem+D7v#tHU3E5P9s
zo)C&7h`|P0?vNI!BIva?%lx*J&bDN8Dprxw$f!RKwEKH`fOnSo10DV(3W)OOxdart
zJ_jj;7=Ws8h$qW{0M{N~-GpjY8MpqLxh+?)7(85R3qO*?QBuY}D~uN{pX%xD2zY*~
z{rLhtF?yL#AJ!W;-`RFHK&I<IbB_drw0|TD)^|g-RgYcc26B<S!Gy(Wqry)HdJS)F
z0=wcDgTqHMNaP%QdCH2euxhNyFBEvRk4Mk!yUk(pkgmN^O6uCRL*88=x^uA%n2kJ!
z-na!17W#+#y5LM~;p=~vx>7*DKlT6aMd?hPzQE#}@BYr<8vH`L(#JsRffsR(i5iVy
zii;Br<hAMff^Y#j3nr2b{2Ki(hVUDQrL67gS~ny{^3<l$a#t3GUM=TCid;J;O*y|m
zK4|QW#iWftQ~(m0W^OT-AV2_W^o|Mcp|Ark;>a_fWW6pn$u#4k9?9m|>35XOGm*CG
z4_OhV=g5Ad<}JpF$B(JA9DFzE95;3K&t{B?e^7xP7YCHCr7W6kXg}t<Def2wSC|z+
z>CG5AY>{MAIlhF_-3sk+=#N*J9$I67sJ|_ISE&+06)mS4X1386^|gP;g9<n}-j;IG
z)C1|Ba`iD#gzYU5`5?7?+;x;kLyz{c;`**?Vvj#l(uUkD{UUDUt>mRqoMX?Hm;m4*
zD}3|SBmz*A*_-Fj70S*I;F1U`fBtCzK?s|EY~~yI(DP+N%)KaYTfTOih-GBa1`0U=
z@2aY{V38~oGgWS7>T5<N)iCPWEKNFe|M6GAx%GV$L-Lz8J!J$CmrUgvJ@~UzJdjEz
z%xusDCgn6w$u7z6)ePimGOm$(IJg?SX1x4NBMDa@s=%BBN7G{!2Iq!Fk1^K!aJ}fH
z;nlwJg}Ktrl@{zQ#$#88XrQrwD}yq9I9lav0=(bi-gY_Di5oD2HD=Ohgc!l1H61`A
zEkmr5BqGcoP<k-%_Wrfe73$0>QT5QUp<dyh6eVez6bYCe?MYP|?)e?x0hZHca(suN
zR6Sq{kxW?LvtPThB=dB!45Psuq|ptSp_14JKHayfkW(*_1ra@qcBR1+ALy1SniIwp
zwz21Wm_iD%h{Yuc2Wy7}*C6O9S)E2QaA0z)QLn~C^{+ncD2GYE`J24Bf%1bhEXGLA
zY~xfBrM9@I=myxqnR``gu#wZDujvBAZwc!{N3){UHs5CLDeVFjTR0cOb)Yb6!i=L8
zHDLJ1jF}owc>p+te0%l|Kr0Tddi5C>blaEvdBJWrWCg%!9@Mn#!x>LqZ1Zws#%wDG
zwGR#Dt?1xsLKjfir<neQ<*8)yYM`f0;5F0`#x<4`o%K!)a+^6;7hga8lY6K7^z77r
zrTgy?`<K(IW9QNW?@1vxzNfAX(%411PP2Z@?(~G%X%~VnN!D$z!(cK_ZWKz;FcH0<
z<!_;X7ib*Z(THDVw=5*GU>nhF2QvQrap$rv#EYA~zl@8rM+m$`J8j~Ce^qWrc#0&<
zg1^BOl;T%3Ke5}?so@TcA7(lI$$N8?u&E(URtxRHYXgx3m^lguHkemJ?h|;@g#->;
z@cHZ_l0`Z-CL+uH)NJWkfthrDXoF^Pu7J|+^5R?Zw0uD!dzCNu1!>+W-qP_sq%S_D
z_CUr<P=(t&ee@Q9X!_KQO(k4RV{GoSd^Kn4#8aJS<Qnysi&SdM62PX?fWFcclZqNG
z2)1+nYFZh3vfA1C9%Eg+*UqNeB$dcr<g=Mr^&r))g+C&C4f~`P+gW{Fh8wVd>ua&*
z0^xPTfXrU3MJ5wO7mo_gXWVvX9dNq^cxON|paa=uly3qIsq#`s&SGmkj5|h^1YM#J
z{hUGqJ?npdzP!#5-A-`pGX2Lwte=0Xo%iE{w&48NZ7YxF1V8q=(d%|eVWz2;93H^{
zz^D^qnIi0eHFZw37?EZ)-7KN?rcSG$y=B0JyCfG;0&^&69h>bE*fjN6xwd^YA%K1v
z9cv9&_Prk0uUQiJ^*jz;KA!cb_ys_t<ReZUsY#<uL}s>S9v<a-#!6dtiEdU|14jBF
zOEZhO{4pTUjinX0xrO7OhJXRwnbOE6+K<4WTRW-j)==4aN{XF@2xN4;8~_|}j@R)5
zZV(TcctNli2o@6c5e3|4GZEN?sJv6a;N(PyjZ7+8>Cr_)cS)BWYz(*!0)?!tOcKOO
z!bg}yk3ynAPN+OJEVYBkTzKC5s_^YrZB66;qQB1GfGu}|43OyYE|0R6&1F$eKmP+x
z4q>d4QbuN>M9ukplx);&?`Nuqu_pV45xut#kPoVS>1n|7=x+3$@HtWAz8l$c?5hHZ
zp9qiMOB-1p5JaC~pE^=6v3a_OYelj|D9i!I*sN0P&r|{N;>+3dxYKkcjCk>-;o{R$
zj0cD&1>uJZ)}~im(md?<H*ng$3<v8(SI)bPZ>^suI_(0fob)gX)+nCrB?TPGM^bc>
zOZVMQPy8SJ;Wq97NVO}cBO11YsuY*aNOx=~5OjsM=0TXuxVH)dB14u|IdtQ58K+Wr
z%KNmR2SV&GdlS_~KKqe(HYUh-LEn`A4Pnfm6|;|DHIb)VYlKVicvuM~Mciv+H1wrO
z8vuL>MZS02X@q#EaHca&gKe@=O-~~5MiHw#>K07)DiZ(hO1YA?ns367chinuu2TQ;
z9$hdJLY0^$3@2d-`mgJ+_1}r{gFlop?Qs3qmCS1i117L&DfoR5m<F(m&;*`3*uNO4
zM$`UtQIZintiH*kfES1@{QPiHAgm|tRf<B-MgmwkIDxyW2?kGIZ{4r#(KMc53O+*H
z0%k1is6|1*?#D>T0KXVJZZiNg$aq6<+kXs1q%T)C;W89}iiE)27yFix&=}bZ-TyTD
e|9}w>R25{o;{|n0-+&MbI0YG1>1s){;Qs|A2C2>f

literal 0
HcmV?d00001

diff --git a/docs/figures/architecture_1_math.png b/docs/figures/architecture_1_math.png
new file mode 100644
index 0000000000000000000000000000000000000000..bf81cffd799e1c9d2230e3f5966694bfc01e6478
GIT binary patch
literal 47788
zcmeFYbyOVRw=dXOun-8EKnMf~ZVB%0?hYZiyL*Bp5G*(ZcWWGi6B3-p8;1_=G;WQ&
z;`_Vr-rt*fZ)X0TwQjG~R8dvu)Y<!N{p{UeRh4CMut>2$AP|n6tfV>!gbtj7A~Bu-
ze=9SL*+3wyRT~KjRXGU>N>x`UOB;I&5J>iGQVO~z!5nEYT+vq+BMuLVA(uytAE_X8
zOg!FL>Mc!3DE>!r)yT{qB10lc$?8w#R9Ikea0Ob;2dNrUlQ=0cJrZngP0wJN*(I-K
zzpXpB!$tV*HQdDV5OR$Ua%I<{8VJ?`^%GhtP~Wz{bUDU+YA6yM%$kD&vTqhl3nd~Z
z{>Gk~{=f={Q%!$atR=lXhj%{csI6g}3xaYaLldsw8`E|>^n3Rt$kri=l1rb3N7cPu
z^CvS+CzQim32{-*TxVPRA^6KW2{WSvgS%TA^jFRh%t-|Lj+1A#ex&|fq<}rt;g5g(
zixnaazoSq?;ci(&b|wxEyB`sF+!Fh5L2))190?Dr9Oq8=2}JkJi<)RE(_C-7DdH8s
z|AO3!P(U7%ho4?+J|!@Z3!MnYFqd8?b7vD`lKr*1-kht!ys9#*Qmy(LPpC)mrGRPV
zwP7}8&-}OMUm8ECPS)se&E4{fbSY1xe|)sj4yDqLOoXI=1iSNt1%>o;Nw9da9(=k>
zW*J7F_Y7x#)blkTtH&|OO&O#nyvF`>zdY%3N7!IBc&}ddC0{p#<r|jj)JW!t38keH
zBC&_Tbq(9!zt|y<=;FJ*lN7U@bZWP8+)CBhkSi6DU0pncubo@*`03=&5L1n({=sY*
z(yULuJn`&$Ig3n-flg^qP2<=0!ZVphhf$u$l0&!uD*-$2$E<Va@XY8;7SP(3m#acs
zbSiHO-rZC^s~v7_5{%la$X!3h@R_LFa}X(Rk%zPZD2EVvY-%Jy!Hw2q`<Uzr9unGE
zFzw012aMrhqVL%R&M3yRy&;r@k|EyJprc?sb7ThHr&wL`s;Kf^($z0DLo#!S_t7J|
zo{v3M4(9la{1e$RSm5uQZKS)uZ(lrt|Ds0~<Nfkf?TwNw?)z}&Hv(TUeMCD{*$I()
zB;-^XX~Nf4k?KhqLk4pgt$8%j?@-AjMnqF`Sht0zuyMqlaz@#)Xn%3)qH~8>{Z>px
zL;t0A<j+fFjJowJ?8x^gJ~#xoTj}WK6_!8xVTegLHGQzU__*2)Jrf~8k91fbzA6Jf
zgYC<*7g`}@Vq7`;YUCvbC9kLOb_qzMit=7sGu4yvgUF*db4hYja%XbO#}MO6+iu$i
z+s0??H)P@8#(Q-?8oqm__==XO?uABZY1e+wtKUER7Jr?YEt}Qo@=p`)eZ%SMHJhxy
zp7oeLn3c9CYs6lNApf=EoX)H8Lx7dAug797(%HZT-KE%t$K~#4toPft*UQh~5h`6#
z$K^*vS14B+R~c7ZS8AWIex?4E$9Q{$*&K=!Dio^S?IA8=nph&w`L=~1|BKC^q(AZ>
zQH}+VpC4C#|NcGXdzAXm@3J;w>V*@gg;DbD^4YSsG}!nViL&1lrV}pW=VdSDoCeHm
zG3TOwh>p-!(apq<e$S2*Vbn;fOyV2#-`E(04k``uB$=pT5Sn2{)dqi;Db0CRXjuHT
z6i-u8bC%IC$$}Be*r(N^C9PRn)>xKXmR7c>iCbc<POkd9G_B;i>==p#wS|t9ovW#7
zN-$;&=7Wn4bF+#sCaXC$NF$P^nR3bVJSM`S&?)jMv#FF4o$_o=cy9hSt@|^f&mAFw
zp@Av)KH(!zr=R}%*1$n(x%lH)%&aIpC0sOLH(zZnrq@a-yr{~u#j@10ZqtmVJ~>SL
z=k&))a{aA_aLWnH7RWk;Im^d<6U9MyL0mvwz@$|hQFieq1{~8FBWYDWqL=x`s0Ezj
znC{qo$gm_PRF)NyC0k_(uV2Ov=l|T(|3UNp$h+|KhI5p&jhohUfpZxgMVvew?U$US
zg4{Hu?{PGq4&f}|L`0u`6aR+wZ8!S$3xaQHz2a*uz2v<Vii%1fm8MfUGoGf}rn)Mg
zD()#!q{Tx7A;f9W)Ftk|bPmWQq$vYd9nA8IC6pLSJi}tDK&KGD;W-4>OVwko`%`CY
z>0uCGp{vcOeXiX;?Y{S6Piq=7U1doIQBF1DZ#BE%A(P2?njyrE>j1S!anPA-obRZq
zt6rPaT3BkNTR_aQH96G;9emiKbe>&t9*y69vHG*OX6d**p)O&n^q0-pTtDA{Plg*$
zLhn;Eu$i&@ub<wn%wBveCj-y{u7&e2dlZh^jysO%i%&LFN1hMQaTJd(b!F~#EVTE1
zUWs3?8Ora|fF<ndtzAzIjYYv5X7>1oekAqs_xo-*tg)O6Z{CF*3>ptQo{C*xAqWxf
zH)Yq-2<>a(V~!(*^YTlw3$L@$ODF6uR8+LYC+=?Uyn@Q)R=i_;&NhFWDb-cYRkg+@
zKSX`7akj&Fh5w?KVSU_it$DQ>l1rIL(86%$l4>)XS8dW=vtqgeT^Xgar^ce<rz(xd
zrpA@*h&_p!$2(`AXV!nc^jhMp+_%!GYrH)I1L6e&9sx(2#U$@aZ#91fWY#ydrcT}c
zf-s+qt#dklTH?3MnNIvc1Gfs;^~C1Pyv=d7aji$z__Pe%JE9XLj`U680^f(_mXigC
z%}yJSj8EFHHg3g2T5}YS1bc)EnJp-xLLM5sIb#~SZ(iZoNNMHG*dW;mObQo76u}iJ
zhQ2_ohyPg7TD&GH|2o;TRqI}tZ0;e){8s9FmJ~ROn~gO2iQMHN<($8s0ymCLB6tw8
z4jb@u6Lc+e6K0Lk^LPKci+}vRyqUP!&MkZEdgCrNc4TX#iDa9kf_d9$P`}0RWj(Wx
z9>VqaM%WNS=c;P9^ZU>D0UHfk+~tGX?FQtnEe)#0&Qhi@uX`Ajgn>jZs%7S9<=nN<
zjo<iXrrp7eqY<Jq)x^K~Pf<bM`Vjwe%aF+Iz3TPv*_*`kOtK%``?Ft`=$E~lrs~{-
z(ropN^<YJDN=h5wPF7F6GDsXdG*?t>+^}7u{1sFzoGpg5^DD`k0!#6=mr5_dj*CU@
z4hN063)c%yhez|{)f!e-10$oYlC6ga7qBHwGj$EMs7gp_KqH-{N?+_S#omSO0m~f1
zBiq|#^#04;*6xqJ-DlPmv#Pi=Lwo4Bw6EJ-`nOaSK5P9xl=d|nFm=`qyy{Vr6Il|%
zcQ3f3Kehfn+AHEDAT6Z5h|0CWjl+YT;itc9&=Q0G+R4eoPUs@)oDHG8on0ba8tD3)
zVYRed!>ssJIksMIK76DgYsb!H_WsCzac*I&d(<Nfmp|zE6D+Kbydq|jWQi^7=eqCC
zmBr@9CX=#GI+)*B$P;orU!z<+QnWWRW8nC6(fS~L(qp%13&PRl?`ZTB(j2fcvM6BW
zl@DLuQ{F#p?(5`n7FoOBL+8VlrT}|~`e@$jZn*Ri9MExfra#Qju1&es&J<tnPWBh$
zBD@3Gf)eg73OkQpUhDO^wQqT#c%slfxqIT&mU@}@P`JLatvU`b>vZz7xY*qiX}kZ)
z|B7evLI1{gAJWp4?sVDi<~<{f&wa8Uu!$JQ?V;cjIehqYJ9W%=ae6x5W8!SW*qI*W
za7TQ0Fs{2fn;Ocrp<ItlZ{Eeg0BW0yIO{tGsXBo~fAGwis9<njE3u3H6*)G?6@H@(
zn({=!OTraB;sP~lzhPln&Hm(15~_yRP8NJ{K(uB;rBKt9aVB*lsh`kiX66I|btNGm
zHMrZ`*L&LU+p&D#mjKPX(jjhK)c7#Ez;71tivJ=;H*r4)^_!1Aq>w)FeLjBrs!L=`
z1Suhed@)Zrl;|<^wa}HbR8j&l0_PYYR3uUm8gPaLe1(z7{_|WKi5`UV&vj%FD8dGW
z`XA3I1INct9PoV{^RFYy_waw>YJA{|cMkIZd>S1D1hoHiWF&A8B&H!DCkGrg%v>!j
z9Nny)+^sD1+<^<2&a!%LAQ17}$1jqcI?WL<|BQ{MuDh<1BEOlF1FNaI(?<(dZwKec
zc|d~R{J^P$g}W)Gw}ZW-8^5;@)jv=01Lu#I*{CT0dBojLh)P#Ul~TgV)q;|Xm6Mg7
zN*Ifhl2Xvs+>&2iQu;rJ1OEw8S-ZPC^Ruyed3mvVaj-hMTCu(3<Ktsvf6w;*Jqz#z
zi<^(5yQw#eqZ{?VCi$Q9NLsj=x!O3p+c-H=KF(|U(aFPIh>Gg*MgRHv*E%h{ZT{;`
zj&A=!7C<1|;~lnltn6(6IX5s=@bN0Ys*SgWy{@E<1Hd!j9m4N;_}&ZtGvNPr>%U(4
zuc2CQ7OoOb4!}rv;s4_Le+>TjoB!8=|GZQ0-`>f=`|q#(w>$qaQjqPD^nYW;zxe#m
zRe;gLSb}W-xoN^!nX^}Fz&^gRkyOzHj)0W?^N|An(gWYeBk+BnLbQDcNX~mRIY}{1
zZ>0T9^q=HX*AeK1s5`mg#s%nc#)Y!jN|7Jvk*S2-F}}P}+j&xeCP7J)D2u~&_SzRy
z_8lbz52M{yT*mp~G()6rbU#bT>hgyhTxjbsV9V!l$+f6MpjAKs#slL~GXMNW6oiZy
z3__&@Az}O<PYWs8age*z4P{7?{vV$Dk7;B;NR;GA|L>E&r3waBOOhlM{U2ZZU+Y4m
z4EX=B%>SP8zc}{4=l_56@&7F3|1CZLEj|A`L-_wkoe>J|A~EqdrB(i6ofUNNb2_D=
zl2)7@IREtJ`#%*my`(}p=bX>~F%gk?-Vb+|5T>fOk68g1dY9nT_ZE@O$Gv1xG;g2%
z;}z=PAqkC|1e|M~F1n32Zy`c435Fp&F#F9^vlwnLm)2(xYWiEil<R#ei1;@@BAP+K
zd%d0j`TOg$5~0-B^p5js1IrLJV%Z;S|1#yiv>|Vkb-UXx=Iy@rULGvWECwPDzPi2o
z#~p}XQjJxah;UdAr%D!XW%)T$7slEYrO~=%p<syS7^32VlJ|TkNdH}%HSw+vTj7fZ
zhjMh8=F+jR4+2emcM2g2Z0(xQQwaW%?J*8a|C^m+$E2%^1t&OdnAFPIqzVtplc(Eh
zqSjmhwISc}u5R|KAVX}O_Rj>5L{GQ#V_UT4=)Iv@I!Og#oG8e6CDI_I4_xnm{4cZ)
zU_2$|?6>C15r4S9gH)UTmY)_+NBPHBLh!ejTAO^}?U#n!?@8KE2AOy=-tN44bhkM;
z820Vw)0@)<P99t@-+c_7^MVB|<Y;RG-g8?K83QDc!0IFMZ;me~8l$59Vt(_ukHU+q
z<*;RM_?f`|m~=xv<G;wWMP_R|`m{4y99!P|!)?Sy`1VL*_;<rio~rO8anC4^4w_);
z#_hf@D=)O;oQ`@(OxjRfX|aK&WMWc+?6=A#mi|2u7!yM+QqHvHOe>Fluqn=*R@$}2
z5B`rp;RrSS8M!U+es-^)YcT@pcHrjThd;#s2vy5Fq_lfOuO$y7Y+@sKH4dAJ0>?$t
zvS&ybgpZO!&#)@??^KZy{M<-380KEEU(>JNEh|qi&hjlIYYH#>hjvjbyjjQAgW=g4
zGo$I>5x6|)ubf4#2qm)qMRO18I=27O-(TRV_RHmkZ?E`jqwGXo@Pb8|FaV0sJO)Gm
zT_{Nicvrtr56@X;AIJ;cZ0AEB$?D<C;r&OFtgVn?0hgHu48j(JggTHElfXIVhAH&}
zLyO%f|7<22rhWam^yWPR?p4FqdFxuY2W}-+l;Z>#C_o?zn$4_w?eV_|h8Qvv_-9@k
zyXmK>fIf1sR6udsKFH_&L%ygAC41nlcim>XT}u&b{g}Ar$@+j$?p*z&W{U1$`XBd@
zj6A?*Oo9&PZ<hVwX+B$-O=ky1A$Y;T?=V0+XuwmSf@x7Hf^LuhG)q5wRLEH`N$MTB
zN6E|K!03nnGV)r2x=(YA@aFTK^ixCJCzWR0jm<$JV9Fzah>YcSzYO}wVL%I<?(g71
z859p!eMV}W7Jv@2&`@f0zJFM-A9Is{x4*PZ&7G@|h4rE;3!Q6nIR6#Z`7Lz0IBDoQ
z@Ycc5pzKM=^mFf12oI!0EuX&t8FA2*ofdR|EkMKeK17rY8IbFyIr}=u;wYe5Z9ezc
zTjf$4f~b_L?*S=S@D#Io2G|hFh^}->CvN7Bt5wU@KVKo1626bSLN1?U(isRCf_F+=
ze5b=487jips7N0sX$Er2F$_JRT9zqBo-RF=o%c7JA?UBS%kmh|k@2kGAYETIZ|6m&
zgF{e*ec=$!NkyeCsV4^t9}<+f-(4>IRh9qU9?ypWHe<OXsn8M_5#lz&*Y_&$YK2@K
zg~7I;hXV;YQ||AqSuB@C_){0DSH9c1(gCS}{GUAqmhIQ;QIv#7FZp42s&r)QcM~jd
zuW`YtvwaR=GUWXbvzcXffTtTkDw$2#jq{Y6Oqy8?U$6i5MvC$*kNqsW+uiw0YxX59
zD2N2sXMz2=Wl+@n%g%>1pUt#B|J&m}Y2>zp#)T0Zv5TF_zBGwuz=A@=(8&pPG=!gz
z@b58Gb1*oy9fvnxtwycMO;t9|mq}=A{t2r=2~no4hIb&+<Edp@)MG^My-y=VZgUe9
zX#9YekYg~n9d%1HY1OZKEjToJz=hIK4x!~KK1aXMqqTjmb6_&xUYK@VZ<1Nv9M=x9
zx~=(CzFPk98E5zohQK*Fo=cAE(lH6JFz=Bv2m`3j(30C|`BPtng8697q+!n{l?5v2
zT!&be>a;f9w$klbPqQCx>E4VP0I~x4jVofmZg!gKwd?}JJNeC=@C{&k@%7)&xQ4>y
zx<a*|MA1T^Iwni9vlZ-~`?{x5CbxN1&*%W_<%%x~(jD28FQ**CL&8DkeABUCJCqJ^
ze2KZXpX#eWr|8e?&5SiST%of#J*W2bRkMS$N3Q~C4WMD*cLIk$%~pAyJ=(YW?_IEm
zd#}2A+wwFtV%;vGHEmtfFiDS_OSn@wF2(EYR=`X{Mg-umDD3*G3gSLg=hp`kpgDs=
zl&Nx(t`lgLT$}Ci<(cj=0Tpb<Y`Z-fQjS*{WJ@oZ7GQs>zAkWgHo2xjV(gs@7@7cJ
zA+J1VO)a;_ayS1FsBnIMf+K(1E7pv$2&jeSS~QPm^mpX#D7Fq=@DsqLx#NXA0Ais0
zRXlQ1J@IOTEhw#DZ>t+x7xBvVi(gE5yt%OE=t8FFe5pi5La0w2ZU`q}ZBv}}v1&xF
ze&H@g-s@Rqa*;=y0efBW=qS&+Czvrf#s304_GB_N_Sqof5qrIvD+aEg6uvvt@bYb(
zv*e%%IE%kt!3}yL%@K6BXR;*uD}o|$`$^AN3ckmnXiOYA=>9Yt;?hTMl@)knQ#T>c
zq%|cu-~4bh)4Awlb1F}!GuqC+=)D##U7o+eA=W{Ffw%n(mF6k|q$*Z!!i5<Z7~~b<
zsVW1kN*I|m#cow^^KQo2?;!FI5PQ`6T+CUmohHz!aAUL7tNK@D`T>}i9%_87bEV^(
z#FXG(+2S9rk#;%ER3#61fH24->kCzCjguHUhr~a0xQ`9mrj4D81?pM|(LB8wN%dOx
zSt_&Bv!MjcAm*a;Zyd`Aa!+TeZ6mN3|D$u|1Nd_B!%j)zO`L*gde(%o01&D$t=cwZ
z`#-tP`Ssgl#%K$7tx|imUvc^Bcx}yX|1m1DMn^ue%ZCP~82j2td59x@4Ky^1<{BnY
z5mbAwQl4Y6s=Mh2P|b?kmfXC{#G$2+DGBgyP(?IZs9-0X%W`J@)c+J>WxlLo+Psm>
z0r+Zhv_zM+-ex%ohI<A!aUV=s*?qdOjcG9~6;yyv--CxshF0SVk?XRePDe#LeJ+>0
zjJhfAwq+a}l7`tM#8@%wsT5$8QLHW2R|E*WZD(ZlCnxgcDCp0If1K_ei4WibizWkX
z)xxH_>T`_xtS%H>!4KMc7DOCF%ypWY(w_~}?vHcR8#A=0+aB%_mQgH?M{0B8kF;VV
zt1If-*YpenQJ;|-#kJt~52F#kKBH$4DD(cJ=v@Fz08|h#$D2BxH(LHN<QVk=NY{0V
zat2|5xB30bXFa}ID>(>8m+e21hBn$_<;p>c49Jly^80|wuiJp*(V5zaWCF9d9yA*G
z-+!TwVYkZivGm{fG7h@)IPIs7rCGg!%_c{5B{FFIfTCQ}ClH}x`fuQ}v-Nit+!yfn
z44w$;-x?Dm*S?@S%~sD*M#n^x;6Zs`UY<0yb9B9xEdU#SU#A!>YDEzDJ(g$BABe!x
z0YkLaEH(JCU*!H6*GkyGD9v&(`X62gJjOpTJB<>{_jOkQ$uF<H#@3bN#e8$WeVt-Q
z;&Ywu8wK1X%=z^}s&i$-w0uRl51_ik-Pml4F<OPn<cfd6t%s55QG@#BGUkjm)Sq7!
zjtV38$<r*PjE&T!rE-64bEB@;aT>nK9T@SL)+=6^%8nGzAw{Y@9AEEDbR#rBWE(;S
ztFXZ+BVPNKjPK&6;RNj0^bMQbpO8?mfB><j#|?qqZt{F3IzCxxpR>$JDh{bHU`4KV
zrVN5(oe(jg#p$m!%os=AWy$H%n}*Xy0_yQgf8YuSxsZ-hDl$KYpV6`Sw&K|fc1mGO
zbYvqX(PTSwG#S(ImzK_8P<<uw`i0EtP%TzY4^p~Q$4*hk3u;Z2-fQ*eaR{b!pYsPG
z055XvfqQ0pNrGmhTXr^5OdJbB7+_Bc=#)7D&Wf1TA=~z9s0dkT8+kw-r}9p;&KnmT
z-C}+`nv{%3zrcwzg`&90Q1HC08nE6i_d3Z>1_{%JKc03^D6lP2qYIt56>UZ1PN)ka
z_UrCUH5RHmZU%1wbLh26Ppg<3S1L89i3Fodt_=leYyXS^qP;g9OcRONe46DcsA~(3
ztqu*@e6VoTGy6#_IvYMl*%#5Jb=+;Lnx$`$b7@<aH{K(PW7BNK0<-I>w6&3d=yq58
zOo!vE0SoR35N#f)h<-{3eA<GRd+$ptciLTBC5*<Ps1j2S)Mp~M&VZ}h`pF!4*$R7k
ztpp}6P_4b$Ef*lDbtTA6Dp1dxXhkuxgqFB;6ywFcEdcxtwIjI01#z}RKv9Ph0;YAI
zvq(u>@`5c988}FlRV8#p@yuIOp8}NgmGcB47GR71(f&q*STB%}EvP70u7JJVGiUi;
z*FT@2?X7@OqAK}@aee&xX^d$gd5crSC(NbLu8O)5Ms%FCAZ;e<TJrjg`T~DwJ?Z=S
zyIGf+-ud9zeQfUWen1YH^{sQms03EDw?{ji*UqNtOmaDeodonqiq8e6gCrbZYcoM)
zs%GbLTO>Q{D83N6wCrKptq#|edOC$|UC!H89nKGj?>P`X!=Eu0tO$4heI15bY3X7f
zHozLxn#W$t20Pk%w$x-9O(g)7^3nlSrYdJw^Sb946?8+3st&peC>JvETDFj_UX5hh
z6t=o`lb=~YXHsSTp}bRxH%%AaG+iv__*wwo=JraY-7MhlEx6JMCpD(~9cl{;QhD|B
zODDUQMoWPT9|JEoJ&{h<EG{RaOxHn%crI#P4ssFk96J2!>36O$q5F0#F1VSb_Hn5*
zf8i);oi37+<7-bY9YagsK3lH7QM722n~Q^tm~Izb>JMmyF4@)7BDRgjYpmyF<+r=5
zdPp*y=KXQ9us^66oC>0+jQ4YT=5|RJhVj!CE=D`7KyVsaX8W41N^!5MiYimYG|7N^
zZ;!1brQ51A_BS!0R9)b!SVTeZc|&($it+BMzRnBQbRuiXX(Cl~mSCuSuM^&OGO8os
z0IrsgPJLFUTOcLnM&$3{x;h~qlF5@!;yiCY=+H`PQ<3Z8e3KO~s@Bj?@Lwt*+9)qV
zrJ25HZGAoc%gtQLQj1d^<yWpmgtXs&jST0K`{Y}&kvDnJ&1AMT5Me}$6eW%WDHC;T
zI=eoQiIo>6fV=?{7*nAEQxvP-bqC_hzVoL;Q}^LyjxY2MAd$WZf@wDGJp1zT&7j>2
z1G`$Taqi=|oW69yFZ=Q}{rs0P5Bp_b9bwD9WRy=#gKCN#s4?p^aPnnhV=77wQ<`qt
zp)`)%LyKqAoN5IF>_O3|v^*0pcZIPKzr#q3Uu==Q-z`P%0{ocBL><-bHcqp3P-$Uu
zWDMVC>h^AVRm`z|_KWy)xlLM%$>X~kqj)^MgA>_LPjSj7#AehfI6R5`^Ez+N_)Qd?
zD`QD1bPUe@kWmN4KV&#Genm7v7d;@g_vv}exjz$Q+;Oe<#Q@K1OYnlrR&%N|D2T`L
zDeTQk2Uk9s+`x#w>sO9ez}r#VQL)bZ?o}>Hy1jtG?b@IK;6MUTP!aw*d6S7aY-Ouu
zs?Xz^G34mz4A6I|OoPc{`9r&b;3`SB_>w7i3;&UytM@r@y#NXZ>xvBG@ATl+VIG)&
zyb;5MTkiOYGIh>7#JCIvx!&o9OXt&Ur}?I2fjqzkv-88-PV=fd@8_XC={eC`hEp<=
zHtx(T%bOWauQPpjOXH0;)W)CK(Fz3$TNO4X8!ukUN_E3kbfeyVb$v7x+EIf(8165T
zSfr2c^dhUUaD}fvxC72YduoMF<oat=R96);4xPNL=pY#1)gU1|-oxuTjP5?Llq*6L
zN2cyH272OfA&A^%iP=cs&ZUyPojMKnj<TJoi-O7%KD<x*FnTl6k46RId|=0H5;&;W
zaMa%`X`-s6#W<^G#6Nr2^ao#o<|MAs(s*n=tg4)L^-jjrxaIK3FBzmtfgk!nmjk7`
zpSL3eD?22{)3%=-7v5ILjr6CdI&hY^Ozxnut$+uog~*iqp_y}+n8bSK@u>M(jG(jI
zvv<!gUUojzW#2Jti2W*_)RxB=4*VH#wG#2$TI<py;4xy9e64Sq$|Kfrf3rt7A<HWs
zEcy~-I}3wFR=1AUF;DSwjkR_EyLDdZUOIb*=e$kpS@Vw)RCK)Ua8&bgN`I^C06VI8
zS*hUZ3Y2HOlpGFfZ{P0KamWWz=yDjvAKvMU4d!=b%>#NK)sS~;Fn}2fD<-qA4za7e
zY2d;kTMD>b`Z|qE#yxzV*+#VSRq~Z+4id6Dp3<n_jt>{Gt(%OoyMDs{z1IPIlRur6
zDn!;LU*9|%G1y=XiDXk)P~E9<{v{bIijMESk)*RK%(q})*L)mu$bwln=@zog7_<2R
zgpf9<;FjI8E93D~1MI5W#L+0{oo_yUR2$QgpF!2cncQ~=l}*d??(!ariX0wyJ>NZN
z0k~%e92WZ0rY;+5OQTWgHvCSehihtZq>Ykg*jcpnPM?;(_T<zOFKIgCYA|wZRE~p{
zS%cKg0!t5B^sxA$v)k9^d>6{(a#!fEta!CX@$h1+Y4&53)CASxE+vba9by?OaS1;6
zn$RTT94B6;7BT|Srv_@d_e-TW`m6wWno6~%ZzW+R-FEZlP!c=46TT+J$;zPp-f3;t
z>KE*gMq0eUGdp~xZf6Fuewri`Et*LO{Kahd3Aw{%H9_({72>BDKsF_e#K3v&6$^Xz
zT9B6~_q;(bUU_HF<={g=Y@bg>+sR<F4cD`DAP`yQR5zzOvq!ES`HQI3t{B{n3uVR+
zrY{-2y8NUFV+5bmZH(AJiGU1Vijvc48&PuJkY(_rmlu#G$oo=577jhr6Pt6T3N0$1
zH~u8nC$q5)Clkj4sQgh~sqf5I1DS_RYB$aA(xud&oXOY5iroM7%R(TOuoQhaMJ{H|
zGPPJv!bBII$2F@w49{q5bEjjH3&Cd<jS`8GKGdNa#4RtFUyy5b(>(lj6!Y*kk0-qi
zD-V3~?3FGFx=rl$-c}Qd)ohN6)h?H-KdFNSxYqjWYAu$!@~;NJ&9zE8Cj-yU(aLWE
z59n^?`!HstoZCFSJXw@q)m1ho_qn7vxt=vQCdq6~(x+gaOirI$Tir@ehL&#T@*jZt
z;`@b(^kJPgY$8C$lKZq9$dEHX?3~77?o+K~ICZ!XXv@>BfO%ZhnJS0*uUzXZXGZ)g
z+pcktD`|6xf=nN>03>fQvdbA+(NbjNw+53Cp-!N>M}-Kjr$GQ3LuV_VY{t+`JB>+@
zj~Kk^==C|mmx35m-8G)LQ-$s{&&PD}r;Ce!VFDFFE)aRE$iX7cD(R7Gdh62L)W=#y
z;ize@m&faU23wKzi5K=Q^&Kyf1A9v0Uh|F(Q0=hVv*h&ddUbAQ(ndh|es*R9d0_JZ
zr_6fgVu>Q=5GI~A7g94jCLD?7V_e!OkU!68DZxh7Ur(GaUYbBWc6Gt}s!#0Ci3)DP
zR5=LBHl6#U2oX*rcbO3ZZiAPo!4Q0VG2d%3EP)Nr1&5hW`e%z&=egwR^W#G`#tEtD
z<nF4gq`SWIr^0O3rGefDcY#?9SlLeWCG`9Q?ROUobc6PVW4%rJ)J+(m^2tP)EUd3S
ziMCo%i3`8}Blj7Oj=6e8yz8zU^#fmET)CKYeh1EPg!~WvH=rqbc_WGk9m!d3+;ZPF
z+Xj<auX;8PD<82trXedCwMp~bPl{@jnm~>(n+KMHfHxY#U~k4)QN>}YOcmAQoPoDj
z!(~2hFe$?C?>YX`pti&X3zysps4Vy<9rL%8)zZcV!Urlm;wcjhcj~1o)j|pUP8f7)
zte+Amr)+e#$qN5rzzvb12;kI4j{mp_Bqjs{5lZ~l`xPD3Vb7IF2?pgv%Rk?qYTZv7
zyqv(o-VMT+C&biWAr#)LH*x=#?}Yo1NnbclRbE>KD50d!W0(=CnZS#;t%<D8d6GN?
z_+0Qu0KX)%7-WfppF-#Zbj*h_G0gFP0_gqf<u`w<nG=J|__p6F0RYbA@^A@{wj{<3
zCE73X>0=+2zIYDe7OxDm;pAh4*u&m5kn_7+W7|!gMm|$|5%~ZF!UjC3b6L1mxffB2
zG3At<pV$=H2uys$#KrlBwD~JZr%+hgWM^N_FQtUL%poIp?drH^dQ8imX{6oQ&^%&p
z$F>6JB57H6nzPfg*9{Z>?^-)kXi}YM>OX_n+g+761@vD|Tuk=--2PMGC3sjB)kw;@
zfhJVdwxgD|$iT>XW}_KHTR-MHObvlAMF;oBoNN@-h;v0f2XGU$TW026ou92PIUO5R
zamIW)nmhS*^osxtcr3IIGf{uRj9*{@pS(I?X<GcAp?TRjqdi?<G>$2tbcg^N2wK%1
zXq$;>#G9yc{Nz7%>03VAz!D9R`9uWZ0inq;^}BURE9WyNvDrHJ&2R0VNQqm&o+}pG
zNYsd=QPSmLSS^!E3x_2!;+}pDQza<H#<1pU%7W{C5;18*eZSk3&NSu4^$(5-R2h^6
z7anuc5oT5%_?Uj$AqvA5k=kuP<O+eYg2)~B>C3awii!YLlcsSCJpkD*og==-a9+X>
zhxr0Wtx6!W;h2sgMC4Ii>`2R7mz=Gb#19;AWd{kz_ZFu-#I@!80>=g|y7X;Gd3D;2
z?M_;^0w5%@{7fd*8_;lvemZI#-rKf&DUrYlzMBE$>=a}00si`$diqFN2JI;4Q6&#0
zsDdWhLb&luBUt;rvdG=p*T7;sH(>X4h`;l5_?~U^E@k;0rrmHMR4K$dOl+AnL{Kr#
zB$1aNia0|tAMvJnB@jKq*A;{$LW*`|gM#s_Z|gJ;25-X)RN1eeRQ4HPg5zHxTw({s
z<1{eH;ydUl6UKLW-oKQmXAAMf2g&2TkwSa0KQmxRoyJm+i%Zj|fH_%QQ?vE2#s?|0
zHZ8hpB^6IxhtkSya8##HXz`QxJ)%O(pGJGhoED!!bDzDX8*@Y!g+EMmGV#w(e3Tg$
z;9hu(2jW(!j+#V7)WQ=j4CA3avU`lmOgVyn4rJ%i)Gz^4;d!?Ad>yAF0y+g6`jw5P
zMD;&RI;e<r43yFdF>MIqDvgtk3P#R*#XzY_iZlHC`S>vds1EI7OiEUOnDbUylJ86#
z1jb(lzLz+Fax@qxNMnGQ7!=}2E+`I2#;3{IKZr{47edQ+^u^}&mW$T_*(3o9Np=l+
zeVx&yT#Ra2ia(!IjuBv9T?Rb2D9GxM;BKM_@gx8#XYz9$3y-%d6pfyKx{WG;rSJ={
zgc8`&LjTm#77;aNX~6*?$RkfM+7ttiUcL*P*Rk!mH&5=%3=U>J0R3G7^`G%SZVID9
zN=4IHxt}DjzAXV5qTO4!P8+Fa6^*I+gv65oQU}EOUW&mpU<t~oZF|{bpPw{nG?XaF
z0SN|3ReP}8>I(3N0zdqL$Q7cE@#tY&dGCP4C3QQoc2s4X6nO&ml)L8q!<e~SzL<FY
z(MsN3+rY6Z0ZZlukRX@oB{{q1X5roe06H7Em=jJT%WNPDN)3>@1f1j5e0k@<A5H2v
zEMxKb60KoFtgX!B)4pg+01K;1>K*I+ub~suP`7t>M4>k@H5qa(Jy~bVKVP07a|s@G
zp?o&6g%>3;8UwY)dCV&!<hO4re`Oyw<Q3J(M}}B>P$M7Ef(Tp2>^msR?ZQkNa=6W}
zA2jCMy!3bhkG*SjKGu|9mC>j!Pa~Rx5dt_go<o6XdNg8&&%6g>IZLR)dsUr*i<h^j
zdqBOW3QGa@Mz1I-$#cnLrZQbwq~EDM`!pWcVnd6jpCuNUhXg3M#hd>L5naI3b^e8B
zM9~3M5ax@NH4EBM=jA|G09-|r1@jG8`N3QBdnLlmD@hIBW>!a(F8y-$Q!D2zrk7d6
zX*6{H9e}%(E!YMn>sh|j&KQRF*m99#|JHT0KlDNzn^*Lv)vyJbak+jYPh6kY+tAJZ
zT4PB70V>0bu<r`2HsLsA#KjHegx|QxEGx3nyhd)kI6Md3ZvB>HxCU%0M$u=o<PvCK
z3N+ggoA|p-HaVG?IkauQt-4wQe{_EO+GqW_868zLxnTwX<H2T=ta_D4Uf87%%t9Gl
zTHcxUzOXxaD{NWJ9Rh@9v$L@z)FVE>!2KMaM<g<J%0Hr^aj~%n==C+5KC5fKUKQHa
zdu*osdwzqna;Ha{ZB7oeWOh&3gr9j;yd*Y*Fozef1~mB8^bm7QqvzDzQyZByB<vr<
zp`I1hqq^B4z3EHNY8C8bf@fnA&^)h&(-tcamhKNGpk7<8h+@~vN~gDOTbW*@i1EK!
zdM(KacxMoFD>%z(#?UQVtpz~HmS>5Sd-)AUS&BKRN}LRX?#_E{SzD}X&l~<`iTH-!
ze1N<sWkThgMlY;U_KiNrwQgK=RWQ42xTYsEXgA)E<Ua)rcTcC@WLz&8`CDw%zHX)O
zC2O^^x6xf942!Aa!-&YkZKPfC?HO|;_a6v=R!865<VL)<tw2GPNtL#@$oP-dT>y#Y
zMMSfKfT?4J-V$$?N5#R7xPKnD=d#ZhxmiQ<>h}sv6gQyW*w<G%TT?W;c#btR-MY-E
z5?Z`^XBI0o4KxnW04xK7CLS|%#g$TT*bOJUZ<7H5r1$2??J|2g!$)lG>A8j7Gl*2f
zd=D7DUbyCk;f{uMt`g05z6|wNz5KcSoYZf>`u8*GHOXisD7+F^_43C8FgYnW{iIRc
zo5OLD*DMJ_Iua3zx6O!5TC87A?Ba4GTwlJhx~9Fqy|B`}m8C%FP#N~~ox<w+CAw!)
z_IQ((c6IudqkZyuM(^fyeWo95P7B_9bD;Rmtaf7iN3X?A__UjZ7!2)(_*tjdBJS-T
zH)=(v@MS<r^RsrQvpdr;;?1MqxL&aLRzF<AGqA_Gnvkt{vYx>+d`Zh{YI-M0`)mYd
z=-H5>zrwuNaAriKx6Z%WSd3luw&clbKF%xLkCm=5#lgw6ehzbJTAGgTOq|9N|K@=k
zm!#Cw3>4)CdoD$+7pGgjoPzp^kv#~;1d|%hEGqJ1GK$AJ4DT)<%Q9vj0qBc<@b<$@
zu-7$Eu(F!gx2tR!QbUHs?;`ZpkRTM?H(Ayt&iLoD$i5xB%{!kr;g6;FpxHtLJG8;*
z4FBAn%h2VB6hqgqWL7q}`Fv^+5r;YmcW*iti(rhS=koYW)k-C5uo2Nd3gbkre3hCR
z6U&O(tyP#Mn9-~+CVJo8-e34zet?ZUqq252!JqhK)&;qpSx2wbM{NE}!kk=Ezp^*Y
zU_2&~fN0E``9c5sAR37w-W=w(<b3t31Cn=ae@|01v@5^Ku?gUuDk8?e8@VE!FZ8OF
zp2syN`_JyAv$da1#J9`Y8CS(MTuk1N+??$s$g=GOaEN)xr`S-?rs>U#d4-8IvR9H5
zS`|$W0Oe9zRywD`52*vd&XW~u=-nYMaf4VPfgDVrV#PH4Oz+jNYAsGzcKLo&So!!B
z`w>3fr9Wd^f#NRRsz*;q0FspFCrZ=2$r-wKBb&k`Dgm3sZ3188LAG@uYRS@Ds;A&B
zUsp?^HcBI@CBR8z8qgG}v{yn;v%yq~Vax)s0H)Ezh&$1!g(4_;!j+X&7b#Uq{vKuG
zt|Yw??2kXN;?3V70^V@qn)(SbV0bZo2Dl{I#NrO=V9|0+>u)whP{!V0q*XUA!;ip@
z-kT4358P%kN^s;QJKqZ}zpSQo><tM@r5th{(EQinKLMqPvUJ+^-njQeaG2XpW#%6R
z9zNmMe6Hp{SOGTt=s#Qp&<7X(B%Qq_(jv3YI}BY5UY;kHa5N(oJ>%*~B}cH-Da(5k
zqLJ4A`ESebqMxLNY4Evz6{F_~Im)Q|!ZtZlWJ<(BFv@x^0g2Yat~Jc()eCA+<PbUj
zL*SC$mh{PVu7yHEB+Mr{v%7vDSlQpYQQ@d3xUXLgur<Mdr+bb2hrz9YETTT+^rF+w
zU<ysT1xVcm#C`gmXpxYO#BwsVLGxia$L&RkuGmV8Ws{Ro<{|C-<pii;4-;v6nisRa
zbc45%E%I5KRn}J^9ey#Fq?kq_sJ%*NL&f!~WcPQ!z~A%WhK6!AM%rnpY^yk<zD1JO
zqRF&W?fg_s814N!9sTemyckX7jbMYAaD~62&=)8z%gZDmNhQh~SbYQ2tKaKs2yG2c
z(>&3Gq%rAD^!>qCbT6Kso+0ciYNRa#&$AM9hHY{9gFH`cW{84rtl`5sB&kXLbl|Wp
zZFjQtKk*>e%n{o~9%!f%&r&_rDEez}N)8pwE6t9ZGoV{UJ6_qx0_}Qbf>Ot(p+H0h
zBTsN$VGpQem9Y2FQH1$)A6M|#BbqHygO`~r5|Q>JVe|@rpWir-w>JXKBHD%`Mik`^
zvxOyR6;IJveGf~r=ZL<>Ly48A)R4K(vC+=eFi*M>Vg$1I_C=qc6qyQ^i>1Y=pU#5c
zf7@r&ES`FkptV91aiW9}Z3nt;R>fR&!OINq+4Pt~FMcLn05N+Nk<c+2au;gM)rYF~
z3wgsJ7p=}AEZ^80yP}$4*yVyP1$9dh*Ic3ZZi+U6Fcabr-%<prbD7iiZ9@{WGS}O0
zl1~6gD*<Dhy!F=;vOjkdU~7*xv4`6UUJkqcoa2h#sOTrHq)SQ7K=GU_9c_nlYy)*(
z4dgv0hTd~o71XI~_a@C=nSZB%7koQ<a}`*Qc~wDKs3yW!<)Ri3f8KscbF09VZtMDd
zaz-z40r8_@{`n*BbjX1}CObYK3oQ&Li8&6)p5T$?5F4Z$9Mb=5O-nZ6iGqCpGJkLB
z8L}t?3=z2B^q45q%0Vu#d2fm^wutY17(SSz!TN}t0Qf;qjd8>#P<4$iy9vhFp4E<u
zF(uKZpXwGx;xF=@{d$;V^c&GxL48U?T+SR~3VV)g3&0R{@T^IYa(baQJe=R@wuRnU
z2nMlD&rfrV&J6<r(oCTOTCk}_-H($&HE@pnPb&zE^hAMLT(pl8YH^5Nce{4A0w;kD
zO~vH;*HDig%zbvaZ8`Sa7f-{&Pfl|%x7e+TOj2in=9unmri+G1#e;Jj8k`YQd+Se2
zea8zit|Mh)ZKHMNjZoSUCJ@x;F_Tu~b+)S)!uQra<#*XR`AV=^NK^)S(r)-$upzrg
zO5FV1=q|7e30?9L`<3Fa6dl6y#PoR8)ZdqWSjzm%M^}X}h%G@00SF@L)Lz=^=a|WI
zq{n*GgWEc~wtf#Y%xt<XvQ^j@Y>nK=^y$W9zBBtta;)%;wW0sbYk6$Ch6(<Cg85O`
z?{<(kbgOcz+A4SpN5AEmbSF1_7Z%kEjz&G~K20j+`E5MEZXCq6$t~s@N>8D=CrA45
zjjAU9()-3&k0ARiFaMalXV8IrWkGwYa`B~uc3jSMAIpjIl+bn(m4C~bHhjhNOe+l<
zIFb|*%(au2I@VBhO=Z{nGcN=vcbhD31;COvO*wus*~u=Xf<g|(bI8$x_2hl0|5%}^
z+hF-2#dI7ORByqpI-%ZzOx9u5v2N?}GL}+uKPGt;liB;Xax1l8xINHSHynsr&q>WF
zmn+w4J1VU3xAo3DT_7NrdUjal4%ezRF>f9-H%N0KqOKy)0*&okLicAvO|C5YVu-pw
z#+EMkoOwb;1;N(Drz`iD{L`0F9y1BDeMqQf&&T}TczG(WF(6Jw63Mmpb<-_vpG<WS
zGzv}8YgrOGMySExJ3~jEDr=zuTO@%tf{sQ|$>~MM0-v!n!!ng}qID^8j?);alPaQO
z3*@fn)Y3GR*y;6d^?AQ<_RCA^XuX7VJ0S$!)~kQ|pq}SY9j}D*iI}k()*pjdB9S!L
zNureGoC7axacz**Jme$l<%;Tq*}t)!vIKt8KOeX4vmI*Wa>iK53&2t5rz!|D*m31w
z<5d`g+8ekFO{7;g?1G^b(EAFCcu9{6hgH-dzC1^6YYW|$K)r;uK%(eS$%(6h^j9fQ
zocOKZhLCLPk@d!(lLJP{g*mrXJ9y)ct8OAEZ;UwGS{jt9b$+%UtL-dHeF;><T}SaS
zQ?mU}26#*BpX|Nkc}ZvQ+;>R_yQCSz5&86ox_v@9L20LY`27&gjgvk`CHM9F+vytv
zB`b}YnqkTCQN;}=DO~8>j0^#_ur3Tc{)!lLx=Kzez#=eT7D(Te<Or1R=MfIcjbK{x
zJ2)w?95$0b>;^W$l*$#txY7_fgWw?SwVI)KO{nl0<+)p)E1xQd#-0i#1P>=g6h%R^
zp}2LQhMTVax}7S6Rx3)vMjejs%6{iju!eEaQlaw4iwa<YDyy<05gbE*{7TpvCuh|?
zvFPq@+cQ1SbB<%q$JYkOF3U~G4YBK134P#&LuE;Hoekk#!AL0hcM8aHs-|2ON!s78
z0@s@$x~}o!Whe1a0`cpAWwl=ZA{PdbgF*@3TxJp^tlDRrlUEMO-R%_E49fAxaoQ1t
zK8?G6iJ%ZcWc?TTBW}!(=tHLrSLX_%7B<*zlrLt6hgr_`th6uDG=C%h88Q`~a1}K6
z?X05|28&whYxN2!GRDE4@M9YxQ(0k6d*|h?_8|N}Wo_#L_{GV?gV#>T8xTko^!Rg=
zgG|gtsU5?<Sb3UW?f41v;u?}XPSe_icn{{!HHiWAPi^Sma?lQ0%hQ_CCD}(Wv8#P~
zVJGqO$ya<Ejk1MPq3g`+rH6o#=B?$+WyJDjz{A7nRshg5EVyu;xg|JyYLabPtKc5%
zw`exO37LYuW%C&rpc6j+x=d_zGm(-d|8M}iKS+4Ese0H|dhqMK^wZd(I8E(SbOdT%
zO*4*OtqV%`mq8Cr5JYlQn}ut+!IG_O#=@fINjOiNX;Isj5Zp&&W9$%UG;p9@(&;!!
z8jxc19cMj2V|5N0=)7C#gcPiG107|n1J@&79jq){Ku@$U7Yhm8XVlNrru2b|q`7}R
z19ofabR*?>+v{}Ot2uYvUa@wmU~)4dmCnm^U~=1IqWFw&J#N>{aD!3D=J>uZ=(cYF
zDDo{8`mA7^@HQE~{W;<Rp4>o*6@Xojv^1Q>-%vN=`py0>8bmArYZ&nI@AInPc`SQg
zEC=0h1u37C59T~LFYSoo`l{;OM;V_{btY|vcXd7>I=y&z0{r*QDC!8KGVh@#cThzi
zr+_^JI*}t3kul?kHIe(Z9xOvs+g?#2>OO4aqfZ3$duP2o9WIPGJNKJT_q_@ATW2zC
zQGqwGhdY>23$FiP6pc$cL=jJWX=~^HfJc?zQmHG&?O$}2D@Xc=>86hJ+8*WQ;}|c+
zBn6+xP9vG5%e~I~Jq;hHz|)M_!KOh;ldEo$qIu_R#G+MF51ZEX!tzC9DR;<Dyv%Dp
zt!bbe2<o`p*2CgBzSKHZ$jX&)IhlR$?v<Tjf@m<=n_!Q9`xe>nc}T4PI`tAy7te_U
z?81SdcSX3Y5^4GF(ChMTwqGV+-1)8ro8O$Hksyx-tt@II2x$|y<K^Ab2CLWHx%RKv
z5+VMZMQN0WbCU;;6Cv2CkV6X@?5fu&gUzdX2(li%7nh*SU#f6A)OkB(v@AIHfFZae
zr?R^&6tHKj(>lLya=&5XOL6NJbm>(*@f#N&N~6Pff3@bNuDH2Gjq%v%Y4p&1l2p~U
zS82I8m0Gq2NZsXV{T|Tl7MSs@c?H<>p!&<83#%YcTKJIhMQv$YKZ+LXNq2$dhXjna
zO|Ggm$^~r-$s&KCukDh11?a5}N=X)6znh=wn9c}X#u=!Ji=capoFV>K<>5@F_~w`+
z`+8VOOGs-p_HH&-sbTFTrPCjgu?D+af+-b3W9vrv?HF-kZ(73N)e|(YrJh7ga7ysZ
z_`uPFj?k?@`6y4-lqjzrSJ|$22sEguK;MM*Td=yN4T#)Likxqrl0C{eE$liI#-}yX
ze(1rbGLe<9Xx=rI137MNYCllqXQkY49t&gTO!8d*NV+*bWcj9!QCt=g)p->~b|-MT
z=%xd`#S92W%UR2JOGE*<&C}ANnr?iWK4U*gW&SzsrG(n`Eq3P-wplGs5#QQ34$Jvh
zVQWA?>)iq;y^iXWNL$z^v&fuVpid|$<5DJYU557IROBJQ!sjVQg+3C-bim*?y5L`=
z+zZH^|I(f@&m@f9@ylaXTe+~cn+S}TP~Z7na7SSB*ex4jm~wGm6?8WokoZZUP)2a|
zImbpGXe;9iU|XB}vCA6>2P7jwQ;!i8_UG(aBl~7*xV2^Z9L$r{kERd)b2UzJH=fq^
zsC~;DQb&F@QAQO?rvb3bfJ)JZ)-<}WU9ZK0uOs;M&FK3~ZVF9E?eF-uesCmZU0bSs
zW}gnozFntV@HIxv&opk_jlge5KD7IAJfG!8ZB#>g5$$qSge$OL&9JS^yApaxCcVup
z{&L$4Y3oJYSJexD_+_9|#^x$X=T%H-03%1N9}3b9l7IOPdbja9ro`DFI^h}xm5vDE
zvs{S*tHV2hL#OMp_uH}jC>n`W!#pnp=?;QW8<mhSR(Y|v$^Q29tn5+neC*K`KZ}bH
zG-ssn3EFE4RA^dQi+#9?ZL;~x@O+>4A3+0Z>ieO5O9<`}Dv8kUvB}9Gk~bY>gK_)o
zIzi_;3n%O!<>gVi!@9WXK`!_w=Sdyq@cTn}7Uj?`^{e(o?LL8}9p!+5VlZ6tkXLIA
zUt~lkYvOOTdX1$u*cV!^4E34JSgvP4{#t5xEHmE4Ep;?*#7f|<!JM0sx4jMC3GHK)
z&nS9o+cHJ?R_(NhEwF~+H7|r5?%j9j3zMrMwO!C7WHS1``fieNL*78!tL6tGlCuWt
z>Qdpyo;kt|Qw`LFB`#%o8JkFS1r1cTrSp&RGlbZqMf5%`cE_QOBvLwF*G)l)ri{GB
z-9U<|ggkEtN7sZ5PM{TO^C@Z%bphYu&oxUhiONtBy<O{`3Q)nykP*273Ig8hg#1cs
z$UIYT8RfqhF~2%@E2X|Gq1P!oAztx5?YL|Q@~E^?{~`urFQv4c3_loRS?T!2{sb{o
zSP{KO3+zYRWjov<5^CUg06gq;`iZ11M~H3M;M7gz((S<KK0_)P-bTM(>_~{)RfJ(t
z3}!jW_YUTgD&a^9JN{Br#XhPr!*iUk8_FdGf)ZjUgInxO5c3dppquxwNm=&tI5x;o
zBqML%QJHTe5x8>b)rsKhA<!3Dpbki{!DSrm*$lz;|KJA>66i^CHNGCy;r43ZGtx5m
zVov^{^zLRmkj+;;Bc(T{?i-hicQ-)`qt8RnLJgvQ7}E_(WkHy`HudT(LF76?>3G7h
z#CEes>q2F73w{HJcaN3Wp}VEsm_BFa61n{?vN$HPM*8=e*G~zJ7YVEsbi!UFzqC&i
ziv<hYO)7fQ^Il}hXNcg3v<Hj6Q?!eGqdud#i8&qdc2L5LC5wGJiJo+6gDNreTDv79
zDFNRmKNF_QR;_G-({odckx;(3Bmdlo;khn0^1~7C!x*7y*LH0ba<$J?{%dHV6;3<7
z)1OPXuL_2FOlHM&M_5OZ6-!b*-#at6u8b1z5BM?f>oNAzsq8gsH%szkY>n?vQoK6(
z7{t4ZBtvYfgw9J<99{#zZfM#H5LEW#csO7}y<!D>RN6JXe=4M_w#fxFs@4W-$I85N
z`IBhB0j&!3(0KZ#+{TV%C3?jpEnio|_-<FnNq#*Hj5~d@-f$q9BROn){}=Z@N37^w
za@^Z5x*6>E%;INhOvcBNQRZnG#4q*NjiOEpu5&4Fa=GErj_&!MiY(#yIIe`pmz~!c
zoqW^;jJ?`Q?|Q9I-@yrg-(B23MmWQ~f=#Arl+ShXkvax|ZuvF_>1rcPp0*X71z3Lz
zV%sIw&+xRqSg5i!`c?X|4|C1*2S`Xo5&aO^`gn!h--ZPI5R2!c)Bl5}s|;(p4flY=
zC}HHN(cK-=DN4sn=V(Mgq`PaRG&m%rTe@>}h?I19gEV;Nxz72v&(C%5dG25SZ_@+s
z(+hs2@5fetP$x^=p;e-IKL9FYoiKoW(fjphm}GAlIhI#PJKtL+^?pc6vd$aM594*O
zyn!%(8Z`9AHjzVaw)dHEgM|vX9O5~bc*`%mpVT9v?=W!K*c)nl8`Vr_oKu$Y^wTFk
z(bH?vyRBw@AKN;nIJbD`Nw?ZZ_sq@Z)ns*s?OO^7{RPRk`{eX?D|CvQ-uER501ArA
z4-vh=a~0myj|+o0#8MAp7VY^B$S^3KcSUj$4ngZO*fEa-3o^g@_;2~(3Pai;xKL9)
zR5G6h2hCiwcU`jcKl4S9A2|O!Je%R5HoJ(?8`B9RwWoA^*;aqCVxl75Rz5mdS-Kfk
z54cn@C<iq;IB}pHJ5Z3e+}%$G1{8IKsZhxsog;Bz&1K$*%pHj&Y>j2u=B7t@FD@O<
zwSs&3tB@W1(;8msgjsySMyxzeE%Jot0m9W1-crB?64WT3jWzVWe=C|)leYqkaO#=C
zq|e;4I1DLPVU9eEhU4Ae;nb?n-!Zq$wCHM<0Ns!QXUmnYSRU`4gvu$IRgZ9oYkU~*
z2-0d7EYhtVjaJq-+EcmuHnpp+l1t<1+XMQuxhVs^EH%NdnRgl{-fd*yU8K`X?tNx%
z=2S5?!qwm`sP?-#;9nePri!~nz2Z@SiL1CPeCw2Gt2=W~;~F1t!yqM98*NjSI5+*o
zoo1V1@i6?%imH7|2Xd)IXAYQY%wn5BO3M!*r{9s6Y=s@Qm&+s5_UiK<XFT<*KlN|h
z2)7xW&FSv#jQo%{2L(6ls#ALEJ-30f@PzI9O%#@|8i~I@8Iy@6m5Sr-LF6({5f8&=
z68e|pcYrAoZ~E@+;=*o0F%-X0isf&&8w+?WMYdYhZB3-7Q<@uZSv2rm2Lt}o1gLpI
zH_^16^e*-h#YKlFY-#Ui>jQwniF+16n1KClU<L2*Yv#><JfN?nSW(#eA-!ycm*Iq7
z9#WxX?@yoK1n~vi8v6#azZJ_cm5Dmb^{d>i6*pu#m;=?Bjx>;b00bRoM4fFi{gpO>
z%nhOR783qlk`3&9Q5-m!$VAcSQqW>k8-byc@I&Kpl*Gn<{OJK%W+B&&HNgCnw+MHd
zYr%*?Bub6f^F%#!qw{Rne9yH5_iJbNtNZz`mp{rEt#-LTZ>0&=2Mj%T?_<-geV5`s
z!XB6R*xrK)gC)YHFD5b$w<<6qZW?wsKury~6`iGcfp>k_Qo;}sMVez7p6ZEk>`9#H
z?qtBz)8yZ$8R0z1&WSBCqM(0^o5U3*D57pv2M~Be5q*7ZE9J{XZSk8{n1g+8fY<S5
zwHRQITER(mO$FVM+4!3RgH(3K1>=B0^uHHP*CCg-Strsorx*wR5R*N`VdRI!pK!wB
zIEMzdYC15SkrOs_mYgxeG2}ChQlo=)5rfK#%42T**XfT`{9(GBlXTIs8Y;+_1A-IV
ze(?`Up$=wx*WKmaY<I{@1LRktZcp5Es%4S&#fOSJWfn!5h`9Jl?o_~YW*IZ_Z<5~3
zjTmh&!ij?b+zT@^J+c|X38n~`r?x{YReSOXsH~jtxu<({&g+P~k3JZ&Ic6dKjr04g
zAj)#GoR*C%PuqJ){dmWlNcg+)z>C|};WGAZY}+Xg%)BgoeTu*TnO5_&7moMjC^+<`
zO4Qc5M>lbE-1lAuo-!%8IW?a$C%7Tfi;Nn)P0Kweu6za5k3>n7Ss}G)Paw#54}i=i
z=G$nl77MgoeLWMOowq1*(8)f}udCRV8AtzB{;h3!hfDj!*y_Vsc~@i0lfEJkagcwB
z8q-*|WT$KVKnle{`~A2xjP={@N}L`8mJ^9@N+?~M^D>N;hnoS#D6IP*_{%&C;dKLV
z9GtBYn)(gbPs(wgoe`tnARaZF?mqCx2_rU#$Sw-H-n-jzQAPh_b-IF?Yah@4o4@z|
zr*65WlE__HxgrTXZ*MvUoSlDr@9aEx#B~Q~BBePQ6+H<0=X5-+zT7l<&i@|eAi(<Y
z7c4S(*A`~>;nr?PIAWghgWXtG7h!M%twyiN<n{ZT9hlN$wYt9#aZ=*$Z00RtFfZU(
z5RC?Y_7MNMU(*gp0V0*}HbcqbV0|Kgs?YA6<VdmhD)Ct{%JEn}NB>5EB7ZSJyC2Dx
zy8JELl!p|?T41oiG4)V@9HfLrJ&Uf8Qf0(cQ3>pA)z)RGDIzpLUnDGX@ilhV<LW!N
zf9ZQ|S+DxXs+uliY9L+~=unN3!L0A)EymX<BdXgGbMEXqKn}L*eNFb}cfr*70Bv!B
z;;I7--tFJ)YW)F}$D_5{bQ9vd_g|yrR5&0vlWe_LMPcR_ODEo%QJ%gb`%Jv;zqP|8
zz~2b4zOla2_J#d@0O}8dgZ%dV&N}B&9_?y8x0y;+(4F2asH^EzOj6vuXE6<KodMgk
z`!dxO$4BX3b~cq(U412XeE0kz;aa&<S+g=BIkNUsaes4pfSSSa{M)gMuNs<*LeOEg
z<A%PMs;j6Dum3tQXncJhU`<zx-M&{H2@BZ6w~ATmxsS(jV)eHR8GM#mmg~PxIoIc7
z^?bE^%Z2*Y5`1Y`O*9k7pk|C=p4w2SECQ^0)m>kDRx`wf@o#f7w2K*ZpVyg7fc}E^
z$mz(|3MwQ`F<Jzg-Q26bL%n&0au)daF`KT0>Y3%Z+*J1V^Fi#og-vepsm<gtr?dCy
zt|e)k{v-k_lZLiuc6hx@i7x`2ES}{&FRxo{y-1$-Esh!q{)pepx+!Br*XMs^XLkbM
zlA@e5k6~WB0e$c8AN1y*hgNNmdCj15t)s`TaB{W-b2<%iWzxlT%oOlVO{Cz!-8VF-
zze+5DEA9Ea!m5vEcj0`m8EiVVp}NId-A6=tMd+Qu9=%(iKC`WAp1`&P;yTP?=iQmr
zS&i^MKZO^MUdfxA0M6r5Tyjk=1}JJQp12gZR~itTI8LQ_+Xu~=6}F`+fr>ZD1i>qc
zNcUoV4YmD-`bS2WG>;KYRPWCxUDZI@>7R#EGLCYlx@vNd@o5VnU&AYO1#FyczOO_t
zYg%*oE;pmp=!*ysxK%rHzY{>O8A5f(98q@Na(-u5Sc-Oz*ivST!n7}}R=jI-GJbEj
zdO^on3|nzF!p0kc$&c99G`oyuTjcY0<~*^!9!tCt=yu#B--*0)@R4e3Z}2aGcQzFR
z2+lSO{Aq|*?l(!RUOP8A&k_T1EWG92THa0fOpOcs24-<`p1BR*S0(GoY*yO7f0Wn9
zB%@K^hH{fiR^KD{COD6|$!0KN*g7-JYf}g70gT1aiHWX&ik9Ti>)Of4>a%dy(M&V=
z;l3@T8}TZzX#Mj=l`d*57PPN1<jKqSZ1xwOP9nm2&sx5w#w)f=Z!VQAbS}Cb^uM*5
z2Rj2J%8y6qQjPdcy%3ms-l=Yt(fGVje`t~e=k7m^fWbH~Qr0ZjF8xy#>tJ4JY)C2U
z+C%MYH1kSWo?|;{$^;#JfgnIh%JT#e!AN<m^Gv?gyuh3ytM59%ZtBWLz4iIGQYkAf
zc6Q(EHx;bl$xxMPo!zG-zz0kekWotFu(soC=mY|Id*U|Y?;ssfm}U1i)6^q?=4(2X
zq+rvqc_U_1mfMYs9KY4R8oU=S0!Obq6K2%kp!zgBVoqu-ZWQi}`oBQwpOHP;gy=dT
z1w0_!oFzx$I=#+zeFV2%CjflQi~<_(TasbC)YTSe^mvxhOp_N)NT?hF@z$3BM?SL(
zxKjr-emMKZhOzM`@jC+o3`++E=*i-#>FVaU3nFrbBQp(mugE5N2Pyg8|EWVCMm)&1
z=MhU%4Jp2~Y0R>Y4xRPbP~2x{DkZCjDvXv9-6}rDtF!VeMDX45oqXr1&=JCABv$VJ
zZlCLQ&M{&Q(l~B5Qb?@PjL8%I{n-^vj64W++||fX3RmX<e7*;W3Z*$vh^=mcZ$a4|
zS*Je}0zS`^G?ah7b1*dF$lA&DZ*=fwWPPqv!W$@RtbBa?cK$2A#kwWMXugvUO9h?u
zX|Gio)Y9V3LXIV^xb9k+@rk3qkrUf1E$8e7LfHd{!F<5Ucw4%24K-DsK<uWU7W!(c
z_(+*t&`pQ7Y=cu)jKa8z)WOo14TR@7_nlD+r;}aE!meTFcAU=AvPbcnKQBq<RFz>a
z_sHx*Sa`q>DGQD!z=2|FZ9M+B0=7nZ^Q{wZEa|p3-(rOz!C5Y6dWwu&pf8!%qSpkB
z>gXFCLpG7ZSeQR!CdHRDxu7W$Yg%#d-=n5Ntfu{hH^4<sODJ=iRGtSY*E2iDwO?f|
z;7~hPY=NWKk0ftiMY4Aq!Hi)L2XEd>{!zsT`%y&I%(+fXpINp~kCqc{1!X?3rX<k&
z{;3G<2Ck-e^ygj|GFKMFaJXTj#w2={Q4>R&lr;(P(ysIH{OuGyx=qKFqr3C}kJW=5
zWjlRaY~&!P$NVz(0{8#?^~ku`7rgV+b<9KFf9JIc_iEc0%P|S>9&A${i!(JV4drMg
z0MA<4--DsmN;w@Uj46yv$0z<ntY&@$uJx#IboU$`{w+I`&P!30lNFJv+1gCTiAaov
zN)uJwCGlBTcvc63u#+X$C4eZm^UBi5`#B*G05{h0c{S_Sg7;+XqIe1u7D$JFk#dK0
z7S0~AkV;7Hu~2%*dbV#-*^WG27fVU#Th$)IhWU%MUy~TNw2%@94G;vNXV*Tu&u*@h
zI%Z+<?U}?pCgWgMO^yowg^H}zRv&m`hR+yvC%Qa&G_{#HcdoQ*a-<oBujq%r`(W#e
zmYagFs;ysZal~eqiq)ME__T$HlF6E#X(^pLCp*CL2i#g0!<wnBRTCo;0SZO1NC;;3
z?^AU`Y=uPPL0^)#Q36N+X5jps$?_%h;<1SnE3J7NXY05g;mF_LxzE2#__UE1@CY9z
z52*&W+V8vckN&%wFnV>}RP^_?ZxLL9=9$Bu7ZqQF)bTj;7M|0g9HMN~&O&mm{a9O<
zJNNO|Kfp!i_ipI4$A-8%AyXVAhf^7jfjyQT-7P6?ZbTeZ;l1ToyarOhNy1)9jV!n+
zCm(6smw*ouXxfxj9<L?WQvu`)4n|E*KZW8#B%>T54->5^6rpj$*}h<8>exmF<hRt^
zM*1po2IS1tx6OmNmhpb__Rm9$x&Hs@DKq+v;dJ>S2Eaut|8-?umI?O}B3_Y6R6UA$
zv{*QRUm_Zb!&<}IAwQFO(07#sB5~9$QbQGr?f%Gylyr4(+%5VzFj3Hkxc0Q~-!at7
zY<)5G{{MAYkvPcNtB<0nn{9)i8DJj~rCR<xK%)VFRg&P4{lw)kuD_J{2k}ZMo23?1
zrAxdnh7~kNzy3&YkWKb@g(vY?5}`wI5`p4Uyeejmm%}RF1E75p@0CR#NP-ylo~YDJ
zk>?jnoY-r7*K``7S(F|){q+7_Z1M2|u?^4VrL?H_TXuvk?8B}EtG&m2$Jz5<n6}+|
zE)n`ghl^+fPPrD+bd#0IP(M;?B6P*=>f`7tC%-DF&EQxC4kUf2a$NdgFNaww{)Mo*
zm_W{#jT2?>Ez9p2C3i}4iuWZerPtQ#9s1k*YacGvzOqo(wnu|wttZ<u>uQe4-b0mr
znBvC-y>jC7bdYWeP_<8*58K0pVCC&ow}s+j6IZY^XVt${L^`0q=E|i$Bc1WjWS45h
z$Q3`{M;&5n5{A*kDL|+nvR1E)gjGjiAd@$ClUK0Yc2!|WQD<G#p-#5le4-2pE^QsS
znJF%(iDJVxj*S=?`F{h`;Lf-vDtqk>p@p-Mzj<zcUG0hUc|?d%`<*l@RCy^F2}|ni
z)fH-owNZ5G^++4eS_2ipirs@S*hvQ|1S8`4a7nCAk<Ca3r&qU^ePWwl>Zo5=8#7%j
zSdZ0c=(Bqjo6k(yQtOw?D*qEl&q=*66vu_f&oJQ;yAdI_(1fQLy>?5c5VB`dv=0cB
zr3YM4MWb1Z-^GuI<h&mNM{0Xd*>cjSFGerB7N7YPV0GIa@Jz;H1urAGiWk;I%uzx4
z<y+EWQLePusZm58IFk>~_1Y8>y>I&Bf4v&>jyY_>KMaY}N8*wa@UxI3sHK18?4O!s
z##>NL|MYj8S*qn{mjdcl14d6QYv&$sNJmzigsIn6$$!)tCn@>ovXk=7b>2FJ@$WAL
zxX+l*3MziOxys+QaCo!K=^A9uq6+>aYDhlPku-`^gNL*zR}e}~uFs9{>J?0TNAk#E
z(gBUTRuWa7g<c~g^={h9uBa4BA1FVmOVlsuv%_vCI;#dN=r78BHzGnV@o@eDOlOgQ
z`&V1)kG9;f?-}r&21)KEx$n}_s?e+rl<l04^k4qpsab<49Kynd^3gTc2s%7hq>Q%C
zqa(O80CHZ>YT&wB6+1Z8>++ee-iIiU(yqg9n!27((%G%hT1Enrv`2o>DlB1@QRt$k
zZIBbXTdnBP!+CqNzi^R1n`bc4=}^Mn8~|D5FTL>bqa61Qw?kHXX#=YL`XF<aKkG95
z<`Ejn{DYG(YLpJi$$tr%mBLkDBcO88I`U>cC8-5}Q#))Hdo$I~W26oWzu2&kvPVn*
z@69CK_IteOLD-*)Y)d?*o&5fM8Glle`!C-u^<6iHhto>*6C|^B{wv0oH?{4wmlbb+
zLF8?s8}h2GSK=jU!R)&EE85BYF+^0`p*F(mP~M#M->iJA_9q+7&k^eFB{3c&lRT{z
zoaXx{1M&UwhQ+ZV)!un_N60qT4jNs<WV(41@CCf7Sd>W^Z`7~&M3}EfH2kcI+>YHs
zgT4ln1vhA4BNeJ}4oOhS(imXVPThD6EoQfh?5&E2m!x;(PGIIC&Bae5G`s5_rT#ZF
z5<^36LzavYx}tw(v@wn-eEHS3|8b-x6}8n{JqinXP7ENmj&k0S@kRcrGtHcGgr!S&
zij62p>sua;+fEmDkNqtI{~bi9tR3?!K9rmODLcs@&;9-(Vo9PH*7U*QR9W%sytt}^
zKf@`1uh(9f23b$&VsDc2k6qRM`~1zEaP|#HQd;pLuE6`zCU>c)4ZO3u%hh~Ytq~bG
zhU($*0Dr`$Y>uaJ8lATHRrZ?Shq=`6OLG!P-qPdthf>$oyfHC`fTe$rDSICWNf?g-
ztcE4$)~Ez_*M9VL0uGYbUEs9L(e276m2}c}VqgnjEQC1aI)rJ>H^T)N>p^bc*0p`&
z|8)~AS$W$)RC7`Qlnkhi2FOuK<$=|pNc(5u>-hW)B5MwaiMUSW8ibSaRXJ~xx41K`
znl#u>9H&czzI`Q$>^#yU@Ai#T=q2Jb>_4TJ8-K@*j;sBoMaP2|?4*lSw-bBoO6NJY
zMDfLzQNCkui;{lHJ(+?yzyG)M_SN!{>22%tjhCZ8tXYES(|DFK+Wq%NGg|WGUB?G*
zZ+#4lV;0fz0BwQlKa5rzZSmb7S&>Y33%o7t{>n6dzq-f~kKG5-O(7m3cpb@e2QL~8
z_GC$o_rZHJLnm%~e_3R6j-owG9I~f;KI=>we1%+&s##hQx}Z{4WZ+$S2Zt;mkhs+5
zCz)`p->A{*G-j*j`3mA1{%TmVKHSS+R^e3lIc~BodhI95hoQ$f&`wfP>)P|FaFhDI
zTMi<EseQJEDAKxFdY?;;C_>TRc~>G_-OWPTHR*rK&pleTvX`xBKqaw4VxP#qJ21tS
z+{JxDhb3+K<7(BSm+ab}kZYN=bt-((^A2h&4k;tQdAi~abAxvz1Y8$XdXkSlAiM^h
zrQ-koVK$c~I2q{*QHebx<q%a@Dv5Y%1iP)KOW4s=t7yAPDaV^LJJsnE_Eup%ZjV$X
zIG5;bhHTL6#_pG!tCbW*JJN6d<TXEf2;e)zh;r(HaX4^9{~hAm(el#9GK#;tk=C;3
z75}5Z1=umWej6p&;AUD<sW7@)bPW*Kn5vA`FK?+GXKUX{cRKol5D8Det|F>j%o;ON
zcgr#Jc=k2JFxVi+OvW_N7YLF5BJ3?LU~g7Rl#WFs!w(1SkzWJ<hR$vY$|>9dO54XK
z5#|wlW_uO$%$?&trXnWJ3t?<h^RI4b&^zg%2|7Q^5u6bl?jy6PXf-?dJsEHl=2=%6
z)=qcnS|QiY?Q)uD9!(O$+zj1V|6y53G1mu*Qg%UmJxbu{(wGSvE#@*(-(7l*+Ng^R
zY6^1Mnxfs_yb%NKOe~?;l(c)_F7psNuRe>A@?-Gt(zLxb#0k!-okT=8gmnS5t>2&I
zJ}Q3}&n3K5Oo9yaZX?eBR_8?_fG4-R+QLOZeYf*%OQsMIBx~M86y#_0!s!|>rxcf{
z<bbOnwjBlzFHW=FUsT(Pm$Oc2X+lgRN*92SP>~_S#>@pD<#UDmYZ!8E{6C98ZH#8u
z7R;HAv`1=g3E*Nr9ks_fu=@{1W4>!ApM~Y}0rd;s(g?r~n_vD^uVMLGMk%Ev&!~%?
zR`OfPoRBr1kjMOI0S9jA>l97d$Og1vkP{yKGfIS0M=EU6Mp91CM(a(py(M`g{%Q^h
zG913e6UZWZ%iludNu1{#K1+wy&608P;#vG=MOi_Os(|^&Ug?QCIYid?p>5^lqqcRz
zeemYksJWElt%f4&(XFVS2xf{sVEp+7@qSdd_%`{pA?~mMBl?tb0!uIdoUM-uOa6t;
zUFmy$)7~5q(!pefPPxSttnNFTqCj6j4ljZo^3+MGfZ!R=4qPDPc}dUDr}Kg@>l*)1
z(xY_};$+P!bT)CioqbHMt^}NQ;)cmU&jd?dW8JCF(1N?fH4Pwqulc*!zs3WqMQl0w
zoX@iFmX8@j25F8|@ocrO@DseRZoL6K&b0N#x%b9@5t{LKyF7&$M9LIu%9BEqLgfF7
zo*pl*Bd_Q~MV8ZlNS1HEI}>+$RGiP_Y7PY=N--*kClQ|gM#nt}Vkc;rYcL9D7vF6^
zS8JP<9Hz{tD}cLoA6cG{7#g>8eu~mo6aDTKfWsahOy~RlXL?%gf3|D&Mk|N^IWaIe
zjn<uN9Jl=e*R9&48iNNH{f+kjX*0q$8@PY|{m}WP@zG9YH>&AaZ1~=+Gp|s-8+Z<2
zT;+Ck38MZp<o%?`sucB6ebkQzjrK`aAm!%4VGlu~SeA=haVv%A?20&7ZyFjb`NjPq
zt*dPA7#C`~|5opW7|8wfC!<YDrLechPHake)A*}_V?si;lz%(4TME(}#_r)nLHeBX
zi5Ek%yb=GUCJ<PJ8Huc|qYGzn<ahTu^d2oJH#4JmHj9?0Q%z#t3vKhtdTZiSkFE=+
zt$z9&zsmV8A)IM$UHJ>9h1B;TG-^o=A{?HvFgj@fy`J~<PdrT4W{y0sY2Wo`EDJIe
zsnnJsm^K$B_0d87v757;<X>>jqCPQaK9b+^7I?mcy@NlVBj+JS72&5e{JPa+^rNQN
za_X^u+N~d8u6a2HWW!dsCn`JU*I!DAxyZPfLCXSUr)Qq$Z?>zuNWYntl%nfCe&jO#
zx$doxqrIvhLwg_5PUL7^gi;eR;yn+$Oey|+<}n6-J904Bn!TVj5l(gVpU@$hu(uVk
zymsdwOSV`RkvDT2b?Mm@RjZ_kCN3P)<doOhBH!#<ILR+16Bc>vE2hNKgd_LfZkt(r
z1CtO(R)+-QH*E<pkHR2~wv-Y1WNcGqT(SFmhNW2(N2I`~k7CtDXhX8KZX8Qm*1c4H
z2FLW#G*qcgziIxB6AeSt;_s%pL&qc2=J?uBE9`UCCB;r%X`Jz4N+!l!L|WWZrFWUj
z2SFMvcBrdkb8Bshrc|E|y$JopPi&#L|4YB_xap*Y)cL2<`Q$+a>#%nFNs@0!p{WpL
zlrU{kU+&DK2++zB2Y*0a#+3FSNI2&TyPsc(D~Mj37uDQK<Gt~@h&2=MA^bTXCx7zW
zIZO??A`#PBE-d~z;!F)@i<qE)0-P<>)xbH=NE-OI#GY>AfMUx?J|SphPtCsg)8N;+
zE?zWqoGuogEuh62)u~J5f8IYz{vgK?^{O4<A&;s79FfI>jg)+SYs2URuJrZnL9_pS
zR3=K7b@=sCd!mz>hEVxDz6BmktEi%brjbswD#}mBqFwt6+>2NWB*cg6^Ot|S04prP
zbk>lhAKC;z;QSo2ETZ<W#?}%qy-QR`h*)?HRPcbSXkJ>1bW$5ty8{hM2QfT>D$sz#
zyMH#Mw9!ryp6;hKB~+z=N%}>_Y@pq@slz4yDKeBlUpqCvhnXgZr%dP9Q)l7SsHC{3
zUE=3kuozQ}h~g=i`auatr);Gje-Hm;{9Ib$sk_=v|Ai{euDL&<!L+*Jj5oo1Y^Hg>
z*f@Ga_TB?iT~J<M=FyGEfSVu&)ln2LxD}eu8b8~Dv3^Yv+OtlzQ7T}A$FTD<Q!F{V
zTR`19Uv_%7J(SU((|spRZBD5pWjR6p<AuhGo@9VU`%X^z2jHuAdn<^PEcZ?^GOG5=
zjo4PNWUqcw$4x`^SB?Luc_qDv_G=+5I~IxEPu*gXwIEAl#!ZxvLvrA6X?>q&zFpwr
zdjr9QRecf&KS`bcBpfv9d+4hNUW}jWg+xCLVHc<d&iWQrck^5WA!29D@y;MNm#;T5
zB9RAGal`UHQ6c7+3iNz)Um$xrvlI!(oaLVGh*ltp)rmEFV7H692%qWf%^lcaCjQHP
z&d}=iV<zJHE;*O2SRw%m_%1Ye@_PLCjJaSra)%4S*`XLS&OoI23$S_nKz9tEe!;I*
zh5dhqFw~cD?{9EDF9VR%l<C|m-7f#GtT=51PRBaodgeX&4;13f2!BRoG}`<nce`lu
z96SBte((=u=Vg_Y!=pdo8VeON;$kRnGI3pzo;~M!li%nk6<E%OmU!mx`1^jCr~J_V
zBsvkGLv%)z`<xMoz{#7qcQ(6~U$=Z|4N2Fg{=sW6N2M;!&KgGiH8BX@R^io)|EF+$
z);0bktFTS+JvvmpUax56Cvgm{T{XDw-!yM@IN!FzxpAk>i6h3&lqvj=q@_f^2)Q+5
zeY)!pZf-Yhuep5RKeGewaOa%&-7-h6cbGiEYvfzJBgR8M)Gsl7Y3}XjDu358q!t>(
zJ38$n3H^vIvi!&9K<ZYQ4=IFKjpD7!k*?U6XSA8gduMNMp8=i(i#63xEA11)j4GmZ
zo~nyrRFC(_0F;VRjq3r<?e?y9Kbg-^U6E&vJ+|%mlplnPQ7AmE%dg6qNX3z&l@fN?
zUU3i(Hv*v)gtSiLh$gtDyGIqp(MIA3Y;J%o-g<CoO>2T^f6N`iiQ`A{i6~D;OA6(h
z|A9EI@yT^!(}p=F*3gM1T#$!@z$gDdl6igTx2G!weKW(ZiTrdwh}<?CcPwg#8b_Z$
z@aLXFw`>d#svvjYKZjyphzpeNfpayPMRqy%1B@U|fq>+gr7fHnOYJr-qnO2-Hyi~Q
z&pfpUm?wP3HH}z`FE6PGWYb*U0L}(E5`ojn7_rH<BdjDNfOueH4KsMM<`+I4$7w0`
z88t`5infye#&CC2Xx$qw9eNK)Ab;luW2qR=WeM;A|B#27M;NMmT{mR=AtEU;Isd<a
zlPn{%g5^ki{S+-^5h%z%{><YiKJ;4ZlD6N7xYl#r>7!dVt4-|GMCasA`?LGK&~IPP
zQ)1{e*m@72_hl349nNl?_Xxh@GT+Q$4ZdKm;8}}H_!Z_D$|`wSQcwg}9x?Bt(F5=b
z)z(yoXZMf9N!{R-0adscj<Ub|1JwM+N>1p9u7<sWdDGnq#z^hyL*I+=nLW}ROCY?N
z%=)VQdv@UNP<jHGo)6dJe?R=(`$C(m?K$u&Km`rV+)g`qc%pbxs$PP?$rxhlv*^S=
zMF0LU4<Ik%CURHIpNU{nWnU344mVl#&6WM~V4ku-?XHXE`p~+1QHExDVQ_opJm=h-
z<m@4gsgB!#rZ5`71SgeXAM}RYGS!;n4uD$s(_Z=gm4Byi4Kg5#u4n_Xhm&5#+~Dt!
z!vJ#q8Q4;X=-S(m6&>vCZEM<d>5?Il(g#sKcx!T2OUVA4FgcH490*&CK<|lS0j5{x
zP}e#!QQV&r^()g{S*TbBgjiP3={v{mp*2SYM$q8JO%2``!e%>3tFOV`6rltu*R54z
z?fzjLabe5Bm-;{GT&QDgLW8m!09SWx;s~{Q#&g}*`=#(Xp}Ua{$v~K|@nM1Y#;JSD
zQat<vgg6X{0<JSmxTNA|niBF~f@aFe8=d3NJZRsG@PWtatzwQ}o!pBk)fRSq7ri_#
zbQ@rRX|6{HEj6{!N_|Qx36iBPuIc_r)P2K9m*PJZmS}S4N82X_=Yl%TBDU#0(w;&3
zVgfvAF*9Fw6Y*Y*ZZlT#PB@3`if=vV_D6rzu94=dR>4j;s}~v8p$su+3a0IZ3MHa&
zl}VrP>?ZbbP*`Kl4Rbx`1PzD=E;_yx`yuQ;O3LE!HeK}Ay?|nwLwl}q$KrY82>Gmh
zX>Klb0k)!pgyn37R3Vjg)QkONV)M;*+YG3z{*6SH>}m5ap%3!NLv#kOsFl#StvOje
z9=bCTz8zK+9D=6y9nLn%mEy4ldx@iw1X-eBwpa$w`!j6J;tp2*w~C)gZBQ5KIC3vO
z(x}yS`e)pCq6wqmg;f~3&?0p|q`8J<?6QT@yWapuC^*0Ut{^@Gf^^##hGM%z!-Ib1
zV@V@mup7^>UL1vAY^-Xn<^pr?Tz`6)i*8}pWJQkVTb~L@^nz@|NHBlxxI2fU;77>!
za>n}wD(}&Di*&z;X~|kl1p(2p_}lyZp#LbB$x+b6xfjW=*hG>7v<%(2H*$G{WWVZ*
z`(~>=8ZbxRf}F>h0r-WVZI9)6QTq;@<dmZ{(I8|fOLqG~pR`f=RhqG0j75Ui<>w1-
zh33jGX=H1B{WOPue#|ftPw(JtaB=XGWy1|IAerPtxi~YkUp~SaU;^83|MHI)ti~nT
zf9Hm<5~?SNyMNxHP(<<X7eVS%_a<#y7+?Z5(n#-Q)1l@bZVctnp|FSX1K4J&--gm`
z<yf*1gb#%+E-_iIrV5ImfG>-F7kO|mlB{y~Qu`al?oxkQZK7Ei_n*NhFYUS<Zwv7s
zb31MjfWDVJ1Mc!1h@<MCK>)NeEa<cV9L>+XAIqv;th!nxd&^bw5{DxBV$PJU@x3ut
zOBA8u?=W(M>T&`w)nz&=9<4QEF{sdAySAa-y;|iL$G&2e)p35@h@@WC9nUhj%Eqp@
z53QQV;nFNK+2u>OA7#uT4>5&%ksa|2#&+XbQ~RPlwgV=##w@J%oVIiRyy`V-=nL<w
zVE*inK$;G>bxPhpE^$EQCwj@e8;z3Gid!t0LJIQMKkr6_lLX?BE>$ZXlXT7NM_ZTV
z0x!qUO>TjXs-6DR7A_G}nHQQ{SdOEM<QbQs_uDN4i^TNr;npe6DTBGy7Y9<%@13R!
znS)sruY@{J_MOHu_8pm1Y{yu^k}%$7To2wF8nNUQ+xpP^SF&nb?_?UZEtoGz{`t&*
zH<;ZTsN(_MQmh!|e@e5CJRtoJogd6Nn%uIs{J`~rN45uozai!5(;%k=<#~)cU-R*(
z37IVPA2H%bKW29%oZkqa6{6#=G~B3-w$0GPaFcwE&SG}6bv5q$;?$Xtm%mFEpPjQM
z6gOJ79skDZR*@(uD@UN2rv%|D_9a6`4XxwJH~Uet7`v?^6f*x7fW;sZ0gUS3eD@e>
zZb6J$ZsLFIpGRyl=U;n_0GNAe1StxDaB^DcoGhgx8r-D6SL+1dt;&}J)m?qiowY}2
zk51{PzR-k4+i@eDgq6KWfi~ZEcPJ#_>oy9IWcxaIg#m9^C2Gdno`NW7wDFl{JCrsZ
zh%{!RW8CtcYYq87b3s{NhN5`h(KyUPBzb8A!mwWHk%HhFuGQO}jrS|3JxUb$k)sIz
z@df5qW^Ywqf6_PUwNI7LAo=}HY}6-F`!6@j(uvCZX3RusY@rZUmJKi?i;2byZH4aF
z`lVYvMQI9LIllyKaYYA)V9XQgdjd>uRm={cn#d}QBs?0mz5Pvtw!H@WQD!7m-#B`#
zJ(?5*#gVn#p(X&8bgF`Zo8?Z3g3o=X@>P8kmT4f<#Ya6V++OP5)N#R`kYbZP(N!Ct
z;;>Csv1Wkzb%;#VgPed1oln%M>v3#AP@+)`+uX(J^Y{A@-o~O2@M&<5z)l~%7eE9m
zr6UzVr<U^FmooDfUyWUd(v2g;QC-`y%Xa|3s$MZ=K+Fid3|hI%VW6jgGCIfyjMLUf
z+D51&6XL{aF!au^J5kT06X<_&&hFq6*I|7(`TQ2dSl0<zsf!Ey6e&2a*3|SHZo3rg
zM8_8mr^Z2)(ZzxsKPNyqW6<n1i-rNI?pVDRNu%mK0?>Z1o<^*cNihmJ3E)=Tp^`WC
zutY7{(*4O69whbhwqY#}#=}+t#3D@%ReCcy*qY%%_-CBh@mv9<U6&?{iDu)q#VWOh
zXoA#bhEpW})$rI>)#xtEkyH@6Dwyu(R)ERCX$yPqlD&Bej#Mk}Hw1Y%iSC66K?WZQ
zH(aZ16HJenW+pzor>mI<>Zc+!6_E-P<fyc-;ce(7@nMSy_rZ-AikH}s7J!yXW!03P
zqwv~^fSfM5o$f9DQDp|Yh2EMcr!(p4ZFrU&UixDG5Lyj@z#;7o;Q9|GOPcZUpe+6d
z^{v5-LK!Pk0lo+V)}H<AHa&tI0Va1<knSiE<!g?C0=9sf?W&H+2H||2b&_O2^aBUV
zVEusHObGp&>a+(9Qs2Js^lXa4=`eK1F0L?Y?>YbY;zhPZxavwNHy)L~{4or7AV|Ky
zg(wv8ik_B6!!+Kjuw^spdP4q7#Q7UFo)xwN_2}?(6m;VQ2h+sUmAzGb)AWRUfDk@Z
z@jFwm>1^N6jzgBwKB#3O!7!Qq<vbDBX>Iqd2|ZMPTREhOK5af}83|3nDX#l<Fq%`|
zU!G!G=(;G7p}-uDrT=cpkgUx$js?jFFRW(KIc(ExweSsYu0$9v$&r@OkGT?7*@(;P
zOBlr~4svin>d8=@2Q_m3m;J|pg+8#*FPgQ9Aa}PofNBLFh<ycsD~G{4&NCDhCY*Yi
zFMgG119-RN4>cDgSVCjrR>HyepJ0^O;FQ3nP5FwyUgj@nO8pTj>^63?`XfUyuNyV(
z@_$Amk!F@-2ZZ|CuOq}p_&wG(Iv?i!sq#7&Tb)*;%e_2q1k!39RW0gb#5}YjQogi@
zF8bIqTidP<09{^y;ab}e|H`q&Zfk<FI1M*0xk~^}g(|V=E3z_n(&0&R{bypiS7vg*
zNXQs8C09q(z!T{&{R`CdMIgg{-#vYP{B%vm+Y7deBpms-p)$0l3!pe`yk1xt7$=#x
zUtuD26VDRQb>(bnkb+yn=}Wh<){@@byu#0+9G`9@_}7`i{}_^t)2O^TrbFH+aydBb
z%)H4p*s8Yl&wd&gLU$0ReT#5y%bWiJ+_l$gw`{2yHF==Xkbuc`J}*a1Iln>Ux?)D5
z>E2Wbp>8kEpZ(cb5T={iIl8|Qxs#k>J53`}1@zSlLY8%5MqM&oPL?vfOHQkm{zGpZ
zg(GPXur3i0M9&E?=YT2?_8RqvK&PZf%(U}0i%*Xhnlf}-jwGi)_C{e!F~#Zr5@qE0
z0@G7k+p?PdQ!3sYhEFc<ej3qG8TmbaHQxboHS-$z6zL#wg{=DgYDK|d*X^u)dm4;G
zNREk4hz2|pj(2Av+eRDjXzrA`XOdyQf0k6~h|H+qHD!sPsn%d?39nf^0J{~&ZE#_h
z7V$l#=+9INAS{E0F)*=K%lPL<x}QRE1}SJ@!M+TUUByY9HHyMUX9f%5+5c`dKP3<~
zU`}Fx_wFTxeH+zQYFa<#S8?Meym6{Z!j2|xEwQ089&I~t<m$e474h+X-4Gi<QCSc<
z{V0LD31%BPo1N(BXiY#Wzo9ykR6?}^kHjRZK%(~w;#u0yd~Nj3=ZWHC#krlVR1DG>
z1PagE6#lkfYUm7lTs{dtK@7l7GPlOj`HtSplqeNzf-gY#;@#(%X)}8V+C%*xqh9f!
zp>A|3%l~A<wkOvo>7G5`?N$t&=aE6LMzW}|ZUZMdZ{|rUb<!Pd*DC3#az4PM9{o*|
zY>=85YWGC(c?bJ3tG$D)S*r3%Su8!4gz&Ydm=p}5otVHDNw8razxJ+4$IHD)4t_g~
zMCCQ|&T~8B%XHTI`tS!Ryd>G~Q~c-{muE^sL58K*1_CYHtNhJsads;f^-DiD)TTM3
zF}B~tE<3#<Z1_V%jk#088`|KnQxhCVp;q%cT3*WKyN?c_#Zv$sORYlPYN6kAxU)j7
z!t+nZ+|2=8L-tm*bxeL|N>4&nAZOc1JC}hfFOI$zU4tC{-`q6IzPN0((yI+h<BMIw
z4TDmTnVXQkM;DHVl-rLP`bMM{PsKhCKY6^&S|@?8Qr~`Lz@lC^-S!H5-!nu)pcMq?
z6Z(@aS?JDmwrFklFIc4{9O_x|NzwM<KiR{|IPZo}UX!lJ+hqE-9I|7qFQyLOnQkTA
z1V>wo?%%fz;A5}eohgmPpKN-2!)uA=&mdO=pRZv2MU!h=`1%dYuEv$mrE^tC*9zjH
zOelG}?$a-WBX6R+ck#_HP=6@>g+Hv33THbGd=k9ibkGc4f7`zqmiR@#?qQRS52E7X
z<uxOIYrJo~M2Zc^8Ref{YjU|d_hO--_l}}(l2s&&<x$(Y#q=uR$-3r{)!d{x_eU^?
z%g-&TCLadb0^fu1encjFQ*FlpVcukL&)*)kQ$2JhXS+!VX=A<D;kS>&y;t#=+>B~T
ztU?$eAg5glUubLJvbJPzikCP@a9cPKH{sRVtCGo2RLG^PRF9ToKlMy{V(T<e3$|(D
z4r^>90;zF=6#(nZ1Sdf}-09XtVr%ob<yTp)_!=VKciCA4v9($vdEIXi{xor4&-nCa
z7}{CvTdvPcCb_f<*XvtNfQZ4&pZgMkzT!qL46GUQ7Ek`Pxg?%|q+sl=54klY-)mu_
z<SivCw}tdhW>K`?*S3L7*)7D5P8$)oOtLhXwIIAb-s!M;$m;<Or4XDjGu{gGliv$w
z+X!cw^X+hoG$53&N;GtIA?6#V4E`E}y?&&0ST5sna<8~6Ug$Tw&*9R4xjb9m&ntGk
z>BX=_1Bcr?a9<%ks-4G;)WI5X6?tby1<#W;jPFG@MXP=46Ua4D_p|H2_yRJ3<_ZI#
zv{P~66U&Z}`HHixgFiu}5r$6#t}&lqOfGOm8%^UUfcp4zpiv((k)TI1+nk@B>{SXD
zpfw~5#RKdxI>m2;xAZ-L82pE*)r7;5tzCR+n#!6_*Bz1xw8BVHOQ0#wCrOJ~`8}d7
zUg3)J@UR)lx*7Ft+&_3;*rX42%ocIW6jYFS@>=JX&NuN;pc;6yvoTO;Ek6?O!N{`p
zb@H(yJ(9&|Z0x<dfn;txnNsAT*?`2Zw3X0#E!3UG)S#dlp=N)K+7})ZX7Osk;JU(w
z@iyRB)^b2I0jaoBY!JHLPkS8;{BaE=PplC9ErF8(yJXiK!PmYVjscN>`zZ|lHQebE
z-3E~Sqik(p>j{lBQ30<4->>kU=5yg`miw_#XbgSBHy7W@RFk~7TKSmZmy3lqDy;M-
zvx(Pm!^Df))>!wkkx^@WrtkQQ`utUx=lq+}%KPAMi4aG0yN~*cL+>#EYDzvX3>z}t
z82tqs{IWk$Tw_M<>ZF*%Z1@wyK5J~VS^e5s9uYsb%h{(@GjN<Tqi-P<_Rn>vhc&8o
zroMLUvnyfTdbcPokn(mYY(}kfCyMwWN!#B4mu)vvb^a%ZBkqK{UPGP{Tz%5g*CRb{
zn*HQjTc}C0JS;1?vH^qbVWdp@{9Ju&H7_$i=B?aoE8F#UopQho?`h2hwxRcTxXzf>
z?{i82_K3>}@B#Tmcje|XhV1rV)suqJdlvvL3=qMS2&Hx3m`i0Bo16%#Q0HPwxV<y8
z8+gR5UG{!XzwJ_Gll_h>f&1tqETQ%8#y^(kf#NDE<zD>#T6^B!uOuIXtI;1sqI{05
zI4{JS@YIcwVtNSLNT3(V1zE*FlCrsrp!0L}Cw*?dqMB7;paQkoZRbJRUm8kJI!|L#
zNk=^-_y94rI>Pz0h95*bm!GJ7L702j4?}d;2&llbcmNjgknr&D*fkJ4v<x;T8(Cz9
z%PCj`&qoz5!_Up;%cB;T?(CK>9<p|Md?Izy^0jZ9HC4+1CHdAh>6+R|KA8cg*k2B<
zD-S8rNiF32eRX%V<Gb`=Nocc0m;>Qql{^xbA#*=plQ;kXUjLB<1B^*4)z@NfqZdU(
zf(uuax*?Uf1`80>9#DrV4;K=Wn9jd00Jv{haO@luF!W5-r-Ofq7ZE%lbZbuTutEPD
zd+2I2DQJp{Nd<=%r{;`sqlA`&I8;G&!plo618G)nEuQ39Kj$!@5(h-Ra@L2*l+_aV
zc}A0e`;)Z@j&kmw4t6XcxOukl>^^v%JK0G1SDK*+HC$anJ<h-ucBLT!P<23kA<s2G
zei%aAnR!ImI7_$>6sxz!>0W{1T;fjV0TK*cCnwo~_8ePG_xjW{`XS!MIJr`r?Z<eS
z5bmPnd%}U#$!C^qd6vJ%tZGa92|NQcaY)K*IS2y6Ot0)G#>L`KVO6n-7Py(}M)#wv
zxr!mkALOza{ten$)#^1VIPEDT7Pfq?yc4aI@uBhu3-<fK*~)rBYqTRGRu^SGLc04-
ziLgLQy7o+c3QZC$X;dVZ5fjnAjwsS2G1RvNor#L??(EKoJF@oXG}<m#Cx9*qH=PZD
zUcO4(42Ykzq~Usq?kYVPeEw=Il0b;|x-9%%EcgJhQG$o|_?^qWv~e=;TmMxVQ;qd&
z4-2*kJ;6^2*o!QAU#DBooBu=K!n(hM_*t~~^F#(zT!ivDpfi;9B|N}boS~%QS3;-W
z`_>;1b6gFdk@I-O39srYw$6<kSlArs;bLBf$??3v9u^>;6%9Lb)%0VMd(s1^JItjQ
z^F*9}AOpL|psz;CkW%=-Eb9!!{vD7f@kBdB3=~y#3psVOg3bTQ>P3dBz;^0!CpqW5
z538YjbnO8w2{q>+trBnzBG*63TbOr7Bd75O;nDS1oYzKXArkhy=8g0`bi{KGr@)h^
zm3;1fWJ173F(O{R7{!1dzf(ko*<i*y2;nm_j6{+oRQyD4!k@@cQ~CQ%i&o2-Z;_c#
zm)m#D9WSdQNm$RI-bhT0#@sto46M(&c%WGK+&_vs=il0saw0$bMG9J07JPcRX>w!K
zAOH4Mux`YKUSqb{H4;HCiDQg##n~I%j<*{f(Ly1G{hM!d3nu${>zT@b@lPu?a5Emc
zta87`=lf=zQ_UJOtS)8qWL??grSzAU8$!R}Le)ub`4Xvg?!T)1Dwlmn<j*L6lJAqk
z0Ov{Q<d^&;0qCz&<)~T@%D{U`nulD9pfWJo(WSq4hWkpw&zZ&EV`?m6D3<c1abz%|
z{x^op#!IOG7Pg{SmY#tp58g%mN=Z};3OI|mJmIcN^V>8@N)kx-UJNE^IDHA=d3~&N
z+U$4ICpuq-BtP^WPgV(ClX^WeJKYlz8VN;0A;<0fsN8y@j>MmL_63c2jH}B!=`hU!
zyo=0D$fS*mB8XUJf41Q%{zO~i`6zffW!UgsTkoaucpulDHXsPW_074nGAj>7!;W3&
z4P2sIsFWC_n>Pm;X8;2Fi`Q`TsfT^A;C3cqLE(G-mmdt^_w@vI_6QI8m4MOw59^j6
zfT?N$E*Gm{6Z$ob(%qIFFiMIXIS4NU9rPc3xKp7uQk0&1>kAPG@Dupy59_cUds@6t
zMCUX2+r4*$i`#_(>Y_ZTNj1j0RFseHfBnAP8NQ_-aYwe}Oes5zIMDj)dL|l3%@eHl
zlP#0aVZ+bQ*@%_7opLQ3%OV=2uHM&jb`qUjuh=eSyXepxY}n~;CKvp@#^#zznCT+%
zyZPO}Q2in$(YMQ*;<eVAwaO{S5YjohOL6Ne&jb)h{+|~|lIw}j^cCh`gQa8nfUlc-
zvp+5S=;MVj!w#d~aeUv1`u2W=1Bb-KdqVu6R|vHA_S8MnT(7yN7;B9n2BX)dl#{Z6
zFT|%%XCuHV;1l*|D7r(KQsH5=DH9CblaHFn^IR3=nMYx*=^ST_6xFK**XPo(%=s+t
zxU=0bOBZx0?H@#s@|9dPFg}qZ*|$;qwc;y3X1`IEUQE})TYY-*=62jMU-DRdl4-N;
z2CdOR#77K0?+vQ|Bt;I#x)F>>Hz3m*sq#c$1Wmj2mnYTcNbTaz`Mfw?H^|MKm`6){
zM`>7=ek8(dQy7AX8w;6|#;2lIIk00zk<(hGe<EFtb(Lpu_JhJ4P|X+>sGnG9l@y@r
zWpAod{f-%N&%QiedhK+kxGxvu&57sx#pzM_&XORKDf0K=Wx%^$eRvl+?R9E$V&gd!
z<Z4bC$^)8Yo=X!%ZcmZjE+=!DJsfo`R|<0POxbRam4uN=raeCKRuA%S{|l{q<_c6q
z4kT=`ju=M1XwgHfbcK}Q_L%1ff8{YR8caDtOISuGpO%GRw*d(!@l&I*+|><OzBp`~
zE<<r^aTLU~eDi;XjRrcjlc7UK-upfbE#(1lc{%I2E2Ebef<AY8Qo~^{D{fNpa(;V!
zl6hS;q?B0Hs$@qDUc$pm`jftonqru8@tXe?d+?eL=S5yb0odrw*5;tdNw*g-EV%)v
z;7{k^GPAA55o?o*i!^}>G>tpD$PMt>ag}F>V|uQ@DpzXgVEv(ae1d$}=WvN7qi$Os
z&#7BpE7t_ml%W~CK)kB+=!qsa2jz;$wQJX5#Z3Nu+FXh+bsx)sPvxu3ju?YE_v81P
z&_FVEI^My8L_N`*_rhC>%~~Gtr+Z&Yl;o~|fs(4C0qsf!<S1DhKD>uJ5zFtr`E?1k
z#!tL_h?I&10N;>3f^!KrH{RIar5jet;c{-1vP1SIveOKP+e?707#W^A?L9oPOvjQS
zOJjCNt8<PPoxb&-o-3W~-8zYMJ%(c1088w0VF0SQK;-JlN4Clu#135YVAS<QMZ|VR
z)H|9E6K%po)?4lNYt?vvvQ3U{=l{x;n~r3zbuh^#hX+G{?GNuh8%2>>4B90fYF`FQ
zk7F*rs<-=R85y&ES(1#-^+?fAvNEN%LBP1}E#L92_-palEuE_jHw8MVz)2YpCo$tS
zXlDqiW!`347z+<Y0$VAgXYpwZ7A1%$)0GfZtH^@JSYO0N*56<MbMy;-N}H;F&ZrwO
z&+*vV>-bH4!?TJ8;)8<;?}YR=;rBv(GfBvQ0!+%6Bp`Oh?BJd>=gFmnDY0bsfeYy}
zU<km!pu!jN@xIH(85|wIYU*%MjPDKKL^J8f{?N{%JsBEGw21oQM@g)-w|*D|vfBTd
zHA`^#Fz(A*08>TA5?Uu~BQV{4LA<Cc^%O@gQMbr2Kzg!JY)~Y<;vFcwsvcb?f!(11
z8EGy30m&b8gYC%qVFI+O*!$s<E=934c-s3d&k4)@YDHLNHPCeR!K0L!t$n4SU>EmD
z9P3^)M2l3D74%5LD9n~84sWLY)N}Na9D4cDjyv$Qqo}bhvBTNF`>srsYNMz+F2h$R
zV4Nbg<((qQSQA(RLsHv6`htM~?8BB#?=R+4S66f;pc%Z2jtB4s9?uP}k%H+alz8^a
z<FjrPN967*KA)HR>4Dm=$r6($Z_O&mTuP6{tEiF1i>_ZpTJskjJ-HvU1~aQ!^&z~@
zd;tn*8R?~N?XffHM7R~!nvyd_)FWF;UZgwKtg~+^<{z`>bziSRC-JI+G_veKgU#fa
zA7)5k^vTtPjz?Z_)Sbblaz&QZln6*}Fuk{bmwP9q*Q<3CV{62)o}J8JQ?i+7tp>e6
z(RzQWs=;%1UO)T5gm<<74;V{oL;K6@PcYP;j6xY>a#u0^nwYhIKS7J`Ks2LRV$4y1
zB;_gTo}R5>ODwmA=9Cpt+`X-B7s(#<*uK%NC}K_o_$%EeXx6;P{ft`8wVjY0&4<Az
z1@92-ebm<Hou9S(==H0Nvh(Qekk(gjcd&tdYA$3Av;!Z$SaPEeC5M$UCv4#_{MWK&
zNE8J426)ALR&8_t=JkY1Du4l{TvnRoMTNvtLtO0*G=_=D;DSELQf6v!>dg;%yRf%l
zlDz+E@4Lg{{<eO#FnVXSVGKglAqWOhM|44g=prH$oiG^D>ma%$h#q8=Nz@pfAtX_v
zB#0U<qE7Tk?l`}5?|aTY=Y5~|{qsK0{p<UDzu(!<?7jA0>$}(5YkfY$c5~J175tPl
zVm5Zj=c;xE!-C;;pa?-)o8NU;tEk7^{8qEH9lZn=CXcPCCkIr209^!Gw~AbvQ>mt{
zoF9%izrzlb`&7U%cnjTRYk}H3irwqllyoLSS)>I18`)1wHwe>yUd&^C*j!nx<84(a
zThi19nhG7u>hk1{RcjsBkR5IWrVcWZ_f10Lz6!ou&OJP|rf8K3_jjfshPOVxUcJHk
zsrF1U`_=DIs6XpLYaG2{fYJJfpT7m~SwTj_$xPM`Ml95`OU?`&c;5}9@^RDbsmQN9
z>Qx@^VngZ_WZ-skv43<PtDJF2g<?9bfN1SK?S(|$@7>&Qb=5#R5gp=bW*A+~r&3^s
zfN?>QW+rZvS#EGe29@j}LDU<ZTH9K8ybVEq*afX01<my`*`bhx(^Do#NjU#vtrzd%
z&I^At>S5aw*=tfPC*2)f`-eyY2Jk|6v)VBAkxkD{^|0sdmo#QO65p1{sK+w$J#fJF
zA8S^?QFma9gB9R?<ijR(5fn2LQr`BSFKzbKHEySL4$|7;FCXLfjM-Q+gP+hVnZ4*9
z^HEPh{AMUgf9AWyYbU3ih=6(VyEZRmV{y7bbG<KUY!+?NUh~KyKa9M4S<TmrbF5oy
zn3j6oDL*bV2^Fw<FacgT|IpJMOpi0E_6q99sb6N3v7K@-g05~Xw}_jc!kAn@f2u_-
z$N7(D%#|H><Z5)H0Dta1NJI|$^s#uP=o<aXzM0T?tXTG_IZ~Q?vL#OO1hE-9xEH{Q
zC>Y?cl;TU^&_DvOVjG?8dTestwjpbzjqcCI@mgGDkpan%1Nv?m-HIdow6<K4>FdIm
zS(e83oyG9!oBUbk!h7*t{hsr;%~qO8sRDWrohlYn$guoo44b!5Z_tGP9Mdw)ksWAZ
zGjvshExWzpiTEoshRK1cd#FVDQ=Z{22P5mPO%C*{yryE~pnEH8R%bP^XN>c?wKLF8
zX0Mnd*B7LYTPOR|bDEBlG@WgiG#V9{uYCqMo-eXeqAN4x60B;5mQVvOLr--@%mGO<
z*8Ir$T5j{GtfcODBMKtk(3doeWRSHjkc;Y+g3upfvV_idc&DmoopWmRR61so3-0LO
z$NK=i*Y;hnWD|>-_Tntf;4C{6{AO>1D)6*AO7Q7QR7;7Fp_)cKQ4qjH0rtT7F0Pby
z9SxgY-9v<+`eRYS(rex@m_-hq&cMz$dls412T@}|pKopTbgBZO1&prwaSy;(X?qC+
z`}2~kPL38-A;WtN$6IvZDahV7%$5^~Z!dyLb>8)y_0UXs6*$qPm?VxsaD11UzCw<%
z)k|=MM|!qHZ;$Q&h;FiYQA$_GwzyS<<SX(qs{Br7a+)7vQ^6a}Vh{hRci_dSW0}1B
zZ9-0$)~f8N?fuf*_^SYHuY@^A^ZJY_mvV5&#527koTNOhD+R*?@&5JE8%o<7EbNU{
z{mE^o602WbfQ!b-#k<K+CsBX<YBa79M8Ot}4!ddoSzu)J$bg$7>Si|r)?fAY?0Ja)
z2ZTJ?<>48(=CRu;Avpp#4AL}@SZ4F}`2Dd3YbLnxwo2)&8))>+TPAcsIR2>p5xpIJ
z;GJz$Cwmw6`@zT<kT>2G{qA~%7hF@}MNZZuJ9u$T(7^U|Sm5~9dF8$1WrEtZ_=PFk
z2>8z^y_()KP#gIiGx%LJWJULqwChPOcg#>#l1$B&xRMZHx~@}I%WXNzVtYDX3(8?3
zXUvgm;Zk#rzp`%M84wb*R)(Ooy`qx33GYlUmU)=iYqf%kQG=A#<0tO}Kf5h34XrB9
z#}OpZPD$zFOb8hD@Pdomhn|5fzMH4JZD+u>dG^icdYDTGM$GI|>2X(F`6{p)c+pY4
zG7g<VcEfSuC}2?dwOVhZg6BA`w%X4%(_Z}Kp>&wCujBH&#i=yQ?FR)sPbTrwg2NSb
z#QbHRAZ$#`cGqy<eR81xx*8GY+|QB3OA?f{JG?Z{%YS5incFHk_md@6KQMUa==IOA
zZ>}vx(td*a*KaFCmJvqtSjpz!ovcqQRx_Q{GtY7S5=8!_&w&RiGqw!}vAxSG-xaah
z5#__>iO(S!1&jAYZ521Gfa#VS`_1Z1dY2Y{2c@2{C3KDfns=!;b)iF7m?8>9tR%Xh
z0#7Nl27A7h!2q#VR~Nq5>cnIEZ?N_dI=zuW&@m=Xo=Bt%3w-3ml4XE5+Npb-BQwEj
zXB=o+_RdA{3E+n`Md%Jm_k@zuUU~A6e5~j%_GNY?m*N}|-OV@|3Rr*eGe$bpDe^K~
z2u}jA#6kgnsQQXImiZyW<M+^h!>^BqH=pg3y3n}o7Ax|_Ot3u&;%PZbJ!C9OhQ1dj
zlLC?*KKb+}-Wu)*FqS!&36rv@iQ%VUY-0sJu@U9t$}!D3{1fG_J*s^y6HQlhtf6$n
zF7q;}2r<9G(vOxv8EHM%d3Qste;o{O?d)9>MN$fQgYRYx%tN<qv`uSdZy~k!gh`qG
zl(t@_(cUZ5wp0fCyy%#aW`5s#aBUI;B_pAMGD^nhC`Eg(5?P$LrenkVsu7lo@b*h>
zFP@P_UB&s8Edo_TLs`#k02=Ks2N3TnYh823P4)?MR)!Zu8s@L4v4a>KdUD%S$xzkP
zX^X~uyXaxY5p}>%>sbP-qI&dGtgmgQL4_&jq>sZlN+;Bu1V|UHTtZboNDL=4d+)g2
zzBR&vcUNp*{_6g4`MG1LZqXm2=kRLNpj5-^4g{~#?xR?jFt}*rP%44<6-?0H2|bdk
zjH!sq@isWMx&M$qr;FS<556H1bqU6L(B_R!EK)E(iF=*@#2z)sjlZ;y*bzF6HE-Du
zyNlBVzErji`q0D(7Km<2IGb%yoZ3xo3!E@c{)FY8yumW&`z>E-&(o3;zQvrnz_dd_
zOnwlS@UU2L>fEsZuwUo&L9h?xa`l%xCP!f*tF$F0_;i((GdEo=yDzlHtwKzDwl4D5
zPf3dI-RCz8o!`sikK+z9)&1CVRT}dkLiei658s80mk-T9*C1I*nY%PYs-1q6J9fW%
zAUCQAIDd&yv+alB`f4UcBRXk+)231i4_I+CvV@${r?J-Gy3I}TYw!;3l%8@9)x<Hw
z*fDqNPWY+SW4+P-Be0hA_oNVzd2Ad<%~tgfe9kin5V0uN`DD*_xEmTa1w4d?9V-{e
z6xY+Q+_flZ?iciMW8gJV%VyZJHh6A*XX>~F8<1?YW}g?G7a%Py-P4w{Je>i}I>370
zawt2WewDI^X2R|c8Dq$2<!n*xeKkDtoh5ye6c)r1&F}qBN_5*zgf^@)RFIETc5F9_
z-5$}+Cy!YJy|o`LB+B2uB{$#aldtr(=l3D<mz-KfLA&aj9XJW@E7Zn0$oyy1Jror=
zmX74ICfg_9;}X5=ZZP&+SAaeI?CgB?lQ5YdbI`A4{&^#1d2#*uo`@Tlir$!nVab4U
zM$U$@hEfs>fFt46NzO5VS3{7aQyg3Xoh^4_bqvYXVZsFclz1X+TCSFvF-k*68^$pg
zeVW)k_<8$RK5<ieMn%e5)sy+|3|J(5Yf~m2R6w(++I=~CdAfua|COr{l5))%FUyvs
z0@RiG{B?=HsN)WOHyXZEcu_~`5zmG3EqSdwS%LNdjN}2&>Yv1sMt!NZW)rL*w?dam
zg(wQd`%_YA#EuC(d{UUQ$I*2eokaV~jHyts2V#=QS)k=Q0xI}Q-Sw>dB-~Pxj&>tI
zO_wy-nA#dpgI*LZzu)oQ$q+7fDPuj-v*~`{r{t^Orf?dnYF@{AaZJ4B@qsJ$zCWpe
zrf7#*TC!NHEpX<(Co4E~ynvekKg&gYH4why{FOZQb!Re)>x3bwN%XDo=ak{35uF1+
z?eN~&Vde070M7+F*2iZmt1ov>S`)~q%L1L~SMXl_?91}P*^iF^TMwc;dt<ERr6F#3
zGpV0q6R_UPRzb>>1`C6JyY@>>%lfy+#_67#CN`6S7{x9}+e@uK(z;}_R$vJjuD2&#
z&2y2^xbu^9Giifxw94dtiJy>2QSx3DFlYMh{`SK8DCcf(S?y3pHdOQMby}>K{mS|c
zreCs@a=)EPAjP#}^-{>EAt-wxK3xSs+Ee~zP@R<a<;*(P`Pdzv7~hxpq~<A30=s<z
zT6`PYGrL41gujMVtT4t=--j$`2eBP$^vaO9f`BqogS4@so>#-E6euMewN<ohEHq^j
zm(V*?Z48-@`1<d|ta%`nQ*9Akau6NJ6}*-h6_62m?+A4Ct9^?3JlZ=Jqj=2?U0OTY
zol{A7<iC5L^lfcPi0X3LQgk54!Ct@eWfs(zwxVMZrnj#P1Q=0Kd%5y>E8n6=FEpMG
z?jL*{k-LHdO%a{n;JY6urxSdluo-HL=gAF>F8r^lgBVU(qr>gaQ?M+q39LK$uuv_Z
z>L0$$leA{;MlV+HBqJ;b1H%eeD|Od38<KWv75>Q0C0-^YpIljp)HIuq;7LA;(O)^S
zxE-oHJs4PahPkn54Xkw)WlHXxCD6JU9P_fNdH#qcV_0F(AHV;zues*b|C>s`cuI?N
zyzLj=x%(2`HtD}L&JdeTMXz&O;>ZT4Mo~r1s8DcQkov^CX$LEBZL{(xRT#+nru>T3
ziCJT?J)4hcg7xi4a)dvu_-sJcwNiZl<6!@pnW`x7GH=aE{8eag{h)Vyf<N?oJRx!x
zZoHs<&7>j14gGEINBv9utBn5S<6>ydppb3rcV`K2-zeTmKQrv$GaaOob9{oVZK6$(
z|D*`BRxM=sh>@(4WLAAg_Jt6$6%8o`i6Ny#?yQbO3^sS9w9C06Ti0Dl*<430^O21I
zYX;%u5cemsF;d-{B9a_bGT*JXU)giyWOjjWgPwIMz7m!+<egKKuyxFBjwMkFmLH2P
z)fQm>+yQ8IB3~wP0l3g{Ojx}I;|S+_b*r)NtIwoIxJ<4x_-GeW%SCw`L<#`o2YRC(
z-sJ_(GjViB%SxGnmM!RF^v-Kw{u-hOdVuiPzVQj&xtbnuY#)%nny2Ida%}<G6*LrN
zqGvQupkIB_ae=96x292963j$1Fhr)Ch%I>CoSOb{fT<_uvP`K=7%*P+M{1lOF|ZNh
z?H@z4W96aYoz$7ETAWwu@<O3(QIC6-qkz^6p4lt&L0{|fV#`vP6d*aPd=)%NS|7nm
zQT>1xPEjhaOysO+^@jxt&9kQauI~W8njO9^R9{_gibDxXpYwHb8oIJIkMFm-(e2zK
zdy_laMuF@%-frkV`mvMm|I=s^*j+_T)os(xjP=4!a&)tZ8GZO<y0sq`nLhJQHErql
z<7k%s#+$c|ZY(U;KLACF+KwBipbCVf7PNs55WL8drru5mw|;Bjhohg1c5-kTk+C?g
z`CYYuB2j%!yfsC4jUfRA6N?LgEykSS*|%v-s-HztEz{!-4wS_7nZjh-eN%nPRvT7m
zIFDxC3?~B_J4W21N%o~+_}wQh6ka`gJ~&O8;lTGeu^6RD{qM2GOXxKhI`5>%{5{RX
zZ)88dmEr3PuUib!IawR#o@x0$ueL_;EFL92fX*ev6;}iN^KGxca&szhMRmbcg|vHt
z1_-j1K1bTwy=e5{z1rJr<0ZK5h^jMQYw;FAmEet*g?0`FrFMhw%cwW8J<a1CoV86m
zc|Ow}NouJ+D}lcsWt?jrtod+9H2*I3f;J&AolL`1ztAdZOJL@ST5wR)2`+`C;J0Ci
zCRCT-iMRBRPPgCW*;R!3sv};qSOSXgS+2WnCFJ1nW{w?u&so*vkBB;=S51?-WG4jz
zV?we;M1^`1<1NfrG7jPf*YDg(=m+z;chb710$j6J!C8IJbabhMO!rW4v9!%aSF4dn
z$mbRXDLBT}ew$qV>7F3=naJDn`sy-k=Qk-S#Tx=j3*0T?L3bMRAu!wIJjTSCOC!b(
zYi0~M4!nctof0fc;zdj642ha(7MI`JJ0>w&uEKo+145hM+cnrW>8%^cbz-vZr(0%v
z)vHza^?7=~Kp_W{K2^Ytp<lK8%do1~GLI~{QxZ7}&y>erPV{dKi{dwOMC%p<2More
zp6v<6_S#eJlbnjpPdvdUfB(4rTl3<ZzlmD?Bt%9F5pcDYjLxIciYLiP>>GhOFNT|H
zu+k?jh?d4G`5V}m=^RURm1%CSgJ{_}s?w~`%?GVxZ|+A!{bQWoKPSSvX*)g^B#9D4
z=p<9~I%%_&INWNrjnmK~v1k3>Io|UvSShbWK&ypW@-m;9W@g<;3X;nlD^i>6ZWS96
zOi?e>D^nzKYQ5fLm@n5aD{tdnT#xq9Ap+&1zxPEMc7K)oc&!A%c&*4bK(jev$wHI9
zfL@~(C{EGmS+t2fpUpC_kJ?rJh!Y$k9ECdy%LZJ|_rE!HYfwNak=CIi)wlEJ@L*aa
ztM3iN_B#|&kLw5S7HBn>$i}ig|Bc>M4vc;3R(T*kry3dB@THA^Lh41U-5%yl3r=Br
zrB!<NqiL9%X-To+NnLvx<4wl?icUgZ+vswX?CkTbh8tTUR4R;7iA!LXNb-Zglzqzh
zjag?-ylsKD{#uB_>F>v-T7dX5+uWNbA6U4%O2%Yjb$laiEpo3&rh(SiuEb%nx}NgJ
z(@1-X2HE}TQ>N>{gqLsUh)H1G-!@k8^s{dE6=Dy81e{5Q1Rl`cIKc)FmOW(7bZy!9
z`NrM-Kylg>G~ba`k3InWVS4=r@1_R6>;CiDGK_DpilEF)IW;{9{HXX6IY^K(mfdNE
zG7*Yi_C0&i(wiWw`hyo{^iVn|#10)`pPJK@;y3>iQ+{0hvDv}yXK?)B6DTDpID5T~
zu&UT$-K5KJCP3TstS%!mbjM<n2AkSSYUS}*b-pL@2-Lqlnk-3eoq9TJFjM(B{+G&X
zi2gK+BGf_lKz*`RmccIyv@Q1-&Un8dGJr;wCaVZS>4h1JJ>~YRK{IBhLI$tX@fEKz
z9zcKVs8RIje7gdKr}LZaG#3bZ3AGDPBxy><UOfsBNZi-kB_pN^Ry+aC>ABx;Dozfv
z#^~TAiD*l4y!VW1eNFFUK}p&cjC3DAZ<EB(AY{=~nI>@j!GJ-9#9R&!{}o2~ZpC6D
z?v>iuV~fWx)@1OZH-TxBZOiVAb9uzvY7gmwqDQTUAEz3$kiS5Fjo(vadGyx`0z(y{
zEHNRyML0}a9FyyZ9Xzabtx4S=kLm`5oMf7Z-TnP}aLP1lU#q5~e21Rdk2+CBI>&F7
zlkH@5-OLrLAj*gcFO5pReuaeb@ND>{35AWgEk}dAM(#pHk$v<Ht=wobM#`$;X)0YV
z0|^Bdk-KC`GA}~0EIPSKg<$-_tyceNsM}&}06hl`a143#vL0zj!4eS_LE^%V=ET#>
zFh_^`TmsU%=jz~-k&zE^@q5t_$z^Wz>m2!LQ8cxoEYhhSXn?{bAyu`3APc~9orJbP
zf<T3rVe-muC_n`0a29=cFv=CW73H32NY{M>@0RP0To!x5@5GH%K<fulVoo@}?2s8&
z#JLG$?$bjgAk2o4aLfm5TXM)?Zoj{a7_FbaRgq|CuXpsJ3zmCl<LRD=)@PQiCvQ96
zmF!fGaXxx?gpXqWc9PG;a-#*0nirV+bdZ1c+a>fiF<#PI|MHeZPmF}TYmWE<`{_W3
zImWJ!0+L&qmZM=a2~G<&QN?LK&~~thxAU$(fHTId%!_7E0#0UWDPBE@_x)N&-IZT(
zBP6*PAjnPrEqhKrLLGqxrb{0tHx}*>XVs%L*~>mGZwp+1+<!l+Jt~Kp5GUTr#N?A2
zXibpayCruBWsp&HWYcJA@R=g4^MWBRqR2fO?e%KYS^Z`B>Xic*m_@F^@!_#n9VpAA
zU}_ktwhGw)u*^5m&l^LgN8s07Qj6hDGt8aYt*hj+HU%9-vHQi*={HMhCJfsS;$}HD
zjohbvGCTP(?Ax=O8cInrwmW29qAt-HA;V6Y^?*HlOND;yj%7ge5qQqHA+RqGtr@)!
z&5J+JgVltO@^1UZ4CerD2Wq0%0O?;<-1J6<rs4MKqWsoF8guRW^_K}!sU$cu7g~fN
zUifHx<y>ZR(RI__r(xPkTFya^L(U#*oVpiQCUHuRqApR=9F+qdh`rR?iPaPB2KFx`
z0EbK?wIP@Ge=^UD2M8i6#dzGQrcPd=o@4C%=dpy-+{?Ktop8|SFo(4PWc>ARzg;P3
zn=ro@du{h%W!|GeG378ETyt7n0*e<H3%2aw=YlMj0B4ZW3)5@wR!(lA7=|kjAEY|e
zMm=&~sJzPS`lHf6+V$WJI*u8eNLl!SM9ft*guc~1n>{n#wEDU>z}?#>wWtJXDKc`1
zW|)^P&Ze|3F1U2bTZrzDKYx#C8s2^WRk{zZ@zK5{nW%WMOO+wtB0?sm8}S>p6XGD=
z8h>t2M9+Gr?;(7uG1hhA+)|yLuYI~qmT;XIWfCHWaXG*Xli{N0L_kL*pt;OXlCG3U
zT}bAn%iP}i(1j#=iQOqV<d=ZO2868+nun45rWKMefObtafW8}#kg*FWk$7eOK}9b4
zHC6<c-kt2$+X?8SqIvx((V1soCa2+C*>u~y<IWJIG|uO;HlIg0jDm0SyS$)99nceJ
zCJ64$-~=DsxEAi6b`+IHT9A1~t7yC8PBuZ5=k7bBR^i^ruKpo*Uz2njJt<f)^HOV|
z<Vqnx;8z;$<J}6&kZkOa6yUcVBId1V@8xsa6@#(w%#LWcyb(p-{-EuU0q3!I&w2WI
zz{SjOTQAGIKS(sTF|na|wcQna^VWgik0gnn*<KEIQFz|$Oun9M+CW^lQNTDwYmo1H
zJMJ!1xnNMXGv~5Jna0zKJ*CNhpkaNC+Kw0!Rumt;OJmt`6X$p}37p{l6P4v!u}|b`
zs6Mc$60>xEz8@&K0O0kCH6g%EROiFg;Mm^NqyRSu3PwDOV$BTg=c4!H$KA>*>2S0F
z-{vF0ikyo&z6LtmFxiHc2y?mnjR07F`F#!8?<E}-?|{k34}B;yXq;@?{)&O^4(_Gi
zwy=~dY`Gpr!v(h+wUWbwAHMqT^mtwmB&LfG07^fgC7)L#m+sR8vM%ThsZNxr3U!V9
z2-k|!BDZyT`93>e4;T;PZ%9^30BAi+@>fkgqm8g2*{(f$YLaoWB>q|%A#9`=7uS;y
z&TWk*V<1%h!$WT7_z-(RQ#LFW?;KS$hLv2@OBQ#wr#;f19PMezO+n5QTP1P&H~5KL
z@x1KUt$4(xov?M@ftrhPnmiE(7R2HjW1NW~*w#`Ml)Je;s&QIjs&x|Q%w}K>9hCZY
zy}5AnTbvL4X)Kk%Q4j}3hhum*eJTt&9}K9m3Az(4*iDu+_e@cIa!EhRkx!E}VV{wd
z>RZ7zKsfd~71b63eY<FIK?#f@Fkcz)o6!2hbU2i7q!E@fKBPo*H}i3{sg9^*tt4Ab
zjZo(9^w0&;DHjEh2k`DyyDjZRqhZTaHx5!puODaPU?v^l(6vrrVPHfPKY@G-XabHd
zdpn%wBhNA?9W)|v_W+ghl=JTKb$e=KRrR?N8f2L6o^rtcuD98BePuN7xWa1bwYW6h
zWzfcF2gXz(eCYfix)NYk50)dAiBzg!Bna@}qw9zAlY}$$l&Fx7=XgXqbC-%{2&4QK
zfeoC;h3LXI!m|cHxCoBKg~O?V`_7v~9sIRJXSJ(e*k9as5yAV;7`L~Py2SF(IRg5v
zR&gZY3hNiIKTs^oBb@PuNYf0ER-GPjqsZ~D5fGvqWMw#628syx1rpeP7aUY&@Ys*D
z<pOUG#K*jr-wx)&ILJTxr1stAVgHCRD5hOeVXHbVE?1S9oP00+_>c?p73vxBc#TV7
z4RJrU_Gv;rxpb5uW6bmEq(CX%@XVf;j5xw&RnYG4P1q33H%j|B54aKcOFP67L3?T%
z>P8qGJ`4HV601JSsx~5iv5CH~9`S-oB+(3mlA#u>4>wBIT((FeL^0Y{26`&6E~fSF
z&q*gQA1`Xe>4+M((^{$0@ozABc7HDB?cwI=<9y%LjR?-5ep=c-lxA9*8Fxg5ewCTj
zpjtaSrm$^7YA%@y%XJkKZ}a?U$EBOA5?%A{jEJQ_CfvvJc_zMJz-jjzN_)oCSHDx>
zhpDk>o@6AWbk9+z#tjitX-!WYxgE(T`O>`R0~s=r7|--Tes?=Sl+1HkYN*J*Y>jt}
zc>PX{!dbyBfEBJylm~n7)_^5jOS?4Z<N!d;cJT(v6ZQU_NH}8&Xx60e$`N0^Z3)jN
z`~e*Q-bDMMQ2_XgQsf>fh>Tm#ltnh%&jC-8`E718(~rns(ZmPY9qCyo(PHc_o;Wm0
z1jljB0FL_GjnPCrL9V!OEk%KGFXKWy3iH^YAD6b^tz6_>Qx6KjrEcg60(kb`k8esS
z41PJaO)e`%){H~&hc4HEoOiO%O-)xEAge%$WB37#j#)2hHY!aOS{u3|-lTgCg#WMy
zzlQepwZDHY>qS2i%%yOrWids3S@o-iiW_}zQzCYn2^!m(l-79fhkRzB;`T<%RR*V@
zeYsM@o|Ao^G6U@+=oKJYf!{|ZyNqSf{tp$nm3gC~KsqHjlWQI@!A(7<b>N4)U|;eq
z>H-b}KZJJMaOm6?I<jz)@{Wv4+XF1Bpw#yLfiEh;R>Am?Rnh&V**C1NcZIr(tLPYa
zHa0@l?RK5!e4{w%C{gsOMSzf3QT$VT+jwv0D8q?9`2uE)pbn9iY;gc4S2}xHnvk_W
z^3u$Lr8vubSMlv=@^r{9vF`iE8PA2G+c|n}lJgISj)D~Sy8gOb#~b35(UAef>J}{L
z_lR{}W`;<B(RK-Mk0w#?5j5f(E8Zz}ZyIy*2d@JKT<zW`r6;uoBchqQKe5WP_OWZ4
z<T~ZLX1b%CK8vynm*qrwN1m4po)iS>r#>MO7QR*->Md<c!58Fv?ZiIXQ8UCTkD0_&
zWX=y&v}M6yFIeRO*L?YQzekniMu|+}m8eHz((q_rvx-RhBO@YTN~M2kSvAdY^g{oT
zh5$(Nm1^jNhjG&x{TD{;vz)=X$z0t~W(IN<628(wG2NX9alzoAySUvDKpK(wsn(c1
z1sqL_P*q)KTHJq7lqVJqeRt%rkIR#@$9CKlVq}CX!xVSPge)rX`q3ERP5Lynq=hu4
zY)z@TJ&c@r;7UF}b9wR4pQhuZfLNuSA8&?l`D`&W>aB`o#l$(%sZ-4ni%49O*#?cf
za@BPO!v!@^)GNrbKalJ3L5q=KOMa&xy2^9{59WGNCyzFjX@t=<KOZW~iV!0j0})X3
zDW&2kNoVY@8TuBwKAk9yf=@HwVM?H#-Z2?#u9zC+$=Te?(zqN)kMMp4{A1gm``z^F
z53ZRrn91Ni0=vyA6H39~e1#_QgFE`FCTpQ2?p;$&3IdW~Rm3kgLdhyGV>RPFc^x?d
zfIx<i-=*H}t9(Q}bBTDI9IqT%JoRalH;g3jY=3fa2ob^=K^XnHzkgBp^_P|U+vO1{
zX|{6=*dtLm)SN=!gEo;p$eitJ>&P>Vue6{kaJW2ZQ(k9df^D@;Y>w6ld+M<cXPQbZ
zLzv^*o;mwWv(>hi2XR>NHvoj5zgCBvYffI(x&DPs17XgVu;pMjU`bearZ)M)##kTg
z;&g8YCrn{!p1V>m)2%<rm-M7r%utLnVXHy#y+`7cw_+jU@;Wl3nKJ4|*aeS7!I~j+
z6|u=qBkV)ZL`7Zm?wrP98#e2PQll?Dbz<Pc=|eWJ#&SRjkI5aF+4NPNmMONfP;<?L
z_nGEwK}OU@)0w8vji{}s)W{DwQj^PN*5o#pFBLTXfRC{`1!$@2h<;@g-nMvMynskF
zZ5N|?S01$ee)c7LDS3=-#WTs0dUUFNrc$&0AgX<@>ULX|7&<J;eUHH-G5oE84JP9G
zQrFm=_^oM=x7+uW7D2-_>`sQ}FxzRuuN;6x63KZ^SRbzc8VUOf(i|7|T(Hiz@$-8{
zs)*NXmMXuq>h2ls)g$^}EqRQhKA~0m7K-`KlHzR5aT?QD7{{IH7!RfQoI2Qvl#ISJ
z9A7Qhr&Tj3kVr!m?6=*rrqG9I<)ca(g_}penZTgUged|-Qk5{G_-BGZkN&;aYv$&e
z{ac}mUMa<Ged6W$e(PS0VY^Y!ohfcszX9f7CBjeZn`fpPec@@ymrEjP6)U)E^kvxV
z3Zr|Hq7Hdj-)5N;>>enk7iUKCeyiDD(^#?ZQ`nUo%;SyAY6Lg4SOwwtci8__8lFIy
z0f1eP5fcNe=|hH1iFVYaKUIFBAV#cUa7#JIt0Vg8;Z#Ilc%hS|HLUz?!$bp-{H8nz
za(gObCf>!2OhNdjwz;NO<1!^;rnvV(Vv*g<EQ?NwSYmPZ)LqGb?i^&HZl4y$&Vd7h
zeMJ8;2yc_Q7YKr;zFEEdRqw=ch&#upRiT4Z$JQLSG3}g`rcbDVt5$8bqF;^Qfo*fI
zQcd;{Dzq{869ucqXnGF6Sk~-!k+WU0J$n<K<(Zg+@oa#%^WlBsvAD+am#KAX(u4R%
zUx#gPLdLqL*e?0@%#wx`WS<lU#VYD7eqj@KzVjl7+$1ngWwxmAPqh~cafkBgX~u%(
zz@5wjJC3p5wF#p+RX@g10T^VbRJ7^2^UsXs$m_}^{T3vk@SMEV-2X7lGfABn^nIE1
zJ#O3pR*S=W_r!yx_iZ@h={OYy(aXN2*Y_s#-0B(2Yd>|^Ej2BoL-SrEyoG*6ztnkU
zj%}x^+^V$y&dC`G#1zKxd?{j1@cR69@BB>SKfD1Hl4w3<qF$N5z#zyAJKfluj5tds
z)!!3A49>Y#gK21(0pw(KdHPm;*?+pPmzCEkdxsP2Z=eg}i&`@_rXRDtOAs|aXGzip
zyVYhO$~^wlkvJQWSR*$%H(Cg)A1yN{W?ulc{%uU5OmUO+vqswG|Lt*IY_A`JB3R}s
z3BQsWdg_)NGBN=_;1s(sF>vlQT)gP&o!aK}<Gd3?VhfpM63uEB8{gBVrZA_K)Pt~_
zf+S==+3AX{#5IEC-<6cLJ^4elRE1h3jEYFhK+cYU$R=}YDsE~1^?6b9r%AhGS3=q?
zODuWshb!#v1|4UOoOl>%uZJsMTemnoK(xsl<c=uFYVw1EUb0qF2rDqd^w&q*OOdL~
z_ivKs=Kq5Q{JT9=NFgag4Tkx7v_;b@?+-u}=VCE3f7{JzJBBdt?t14m_X^vr;Blre
zX3nmDdHt6WB?X7?vzJ|UDg$@N23v)dsoaNmo3p;okjcN`ObFrUp9Q!kExp*{0c<?Y
z1pEX)q3Kk+yC3|Qi~4s@_2PoL6k!s&#QOC&Jld+;88v7B7~!tR@$9cPQ`Z2fyA3)!
z)c8^K^8$YsG4qVx;IG{#FqP!2i<mweh>)vOJXur!bF%mJ$Vb1pWPv)tH&r!++Taf4
z@OZM|O;A6puBsUGx)#w>Mh$#B_{VR+-y}d}!D^@atRv-{wkqy$q91E-Lpx@D|8E@)
zVI{?B5y{`{R*6iO&SxY+I!xS>Vt<#77bJXY`h(}AH>>$Op>Aga(PTsieZIAc|A9{R
zpRt_waoe5XDTLZl%dq@y+!Fh9H`Pt+4@CXhtno*S5D}-`6fWm^Yx)tHESr%;K&!7@
zmX8wlg@~r5luU*sU7Z+r{^z-JB<3D}{2T3A&+37(%<o_MM7q~7rQyH-kU@Z~9~APC
zFuSJKG;_uXkP7OeRcLxUQT>dFa9h>#DSz9a#Oxj1_(FV+JCIKM$NhYPwuoPACVoUL
zSDeyPJcJw8!KmzXykAJAT7g{Vng1M0eDm;vf|Cg0(PBs-(g3!}Cw9&3T%f%Z;^8ns
zNCofDkFxtG=A_qHDh#v#L(~7R8Y)^Oc=mgZx}43Om!_G#8bXkge@Sm)LE!r5Hy3bP
zr=8JStweMU>$V)uzYnF6hZIdRPejDB;6YXz*i|Y0+U<FD#IV#?ON4y>aB=FjN`z+i
zB_w-=mbCnEm7b}KQwE#+sn14zhGaipB^FJm+O1y15&tg<{<A#);jWQ^xQq1sD*ld=
zm<IO`i}U}u&j0q|5QMm>SFl5Q|Mu;_f7m&Bp$XL?!e0viyPdn3FC@|8G{YV}mHSJ2
z|IqdSe2||n%v5tgtq{+Jef;;``PZ>Zq!MGfIsH<8BLD3sL=v7jVxy+*th-VFd3P?{
z!+##~3Nta=YX7tA$A5nOU*0)xUZiMEwxvHb{TEa6pAWJQMm+ho1+~6^`ah7wL%xgS
z;<vZc>Gl7m>;HKlv*AQLGvtz^^e_7RUrbu{g>kxbr2ksuzdKFDv3BkgErmDohpgNG
zL=piPjW+FXD2V<KmXZ~Ss2~4dsfe@w|A0~*t5a8LU%uH9@Qz42O+tjy)vnQWjQn5c
C6i#sf

literal 0
HcmV?d00001

diff --git a/docs/figures/architecture_2_math.png b/docs/figures/architecture_2_math.png
new file mode 100644
index 0000000000000000000000000000000000000000..e0c4a9f394e885a90ade43c2d671c6732986239f
GIT binary patch
literal 16848
zcmeIZV{~0n*C?E%r$M8Wq>XLcww=bdZQHgQHf-#qVPo62ZRegodY^ZEzrHc<&%4Li
zIBTuxx#nDR?X$z>WyRpXV15Au1A~_k7ghuVg9rmX%R@tgt{JUCQ($0l66Qie@)ANq
z`0@_6CgxVgU|{0m$*NFlN<*mGnu-!;{?Otu?J$Z3QZRWUz&Of9XbJqh-Z-G}DeVYl
zvkMZqvxusQmJ^L%`8Q8>fTu5c<nZuswd%ZU($DkGoe%2|;}2u)zT-b`auQk|#=&xx
zMiM1&<h8-BNrG#G2Ic2>>zz<bvGn-S<o&7aHm~*+UA(;D{20AHTH9K{fqQSiA`?z>
zZa$Q60fHQOV8poF(*8ek5U;?&E)_ezt3ZQIp&eJ4ev?dgqwIxbOvCS0BpFxP0bRcD
zP$tVLLO^Y=fyJ3~XH#Q=ouQVPe!Hq8h`px?TO)?30f6B!_6m3*$${B_?KQl<F{Nav
z|AdgPduj9$&Ey!AI52|y=UC&rAOgxrp?llXhbh;4)9wL9h~eTN0hFP9<Rq4!j<w+|
zjEY|7uovR=8hhZ0(5o!9{E|?VqDVxEQ3H>~aH^x;1Q?0|8#o;`rh!rz5wUv>1E#l0
z3VQ%7gBF2VO><mk!Id^)QYe`PyQMJ`%65o-N>tDMBkQ%%uV%>?rHECd#*mK`@j;w@
zUu{B3N#%nmg~(ot8la`nXwm_xXqh#ACNdJ4%c5wJxct8Dr<zWAWQ<??14X7ogviYI
zwO&tl?Tg6qX;@`{=7RGJ$l)h!c;%8ik$PzaVYuAW#K?27nY1GL(NQZaaZZ4e2`qO8
z+SA>WfanV;bDg$++~O}cgvlOMVKRFn;h4JIB|utn7Cu}mnkYHNgK*RROVZx5n?I2P
zaGQSgT@YW`8UIMfed;~)9`n9T^&o|~^n)|{L|`{%7Q&;b%!KSF10dN%3=87t-A4PB
z@hyO`<bye1Sy!7s#iCJsFCG?G=m%oH0Pbsryt{;Gexct+uIOv*p;wwEF7|MX@A0BM
z%FikC5O;OXo|IlY19XN6bOEPcnM-_buvq^EAUn66?y4>OO6Ms|EKF9ASkOrjZTA_*
zc3*APvA%zJF05$S<m`_p<FRHOqq}aQEB-;m7JE_~2(3$ddSoobkec?~l%GRd>Te6H
zzGYjHm6oUEHeZt%Baj8{AD5L@BvvegM3!dgxC+Ku91E&`#HUy7ICdP?gRTGUZ0t?Z
zd#R~gggNDQLAQjYK1H7o(C&^QzZ#@`mT8Cx`AJf8JrvBn?c2L!Za~RJ;c97j;`~yQ
zP4yL(Y9rQ<Nr_XFbR6|t?c(Jaosm6>*8&4IBAY88Skea4o#pX2r8{r_G_?2WZ`e>>
zRoIkuAJbRm;Ow{PVEvh$PV;nIJ2l9DL^xi-<>D1R$!HL-oJU<`!{}+BEH}YUuE5su
z!Giqk=)rsaiQ^!PdT1jzJyQJD^=Q=q!~CDsVCMZTsxhoSbN!Li!!v^n{R3kMsRh8<
z<ZOmjl|yuceEx$)ABGPNlSc&Ci(bQTg=E_M=}IIZ&W;#Tf`2*=ju<N*|BDgrHN2vK
zx#+KW8gZU$CI@H^{~uzU@hgKu1!(T!Ui=J6A(*))B{-a@cOpw>%o#zlLUmIR^#B@?
z)St>`IKQ#11?6(lrcTVzp5bu1z>0zy7^Fr&-}JGoVW-3x>oZ#Ms>CQ8II9uOfB85l
zQ$w2%Z663{|75yFYxh}fYlt0>uIpfD<kG^6Y&|HayJ{Qp%IB5IUzVPP9px29tJ_GB
zRUb7kN?u}7mR+13KBow`8;dZ0ka!uzdvhjDbd+G6fR@OH_?!Tj%mukM{1Y+qP};8a
z7vjEnw*o;$4D(OsY)9xvUrIx7W6DJu@jc^+29Jzj+!<OD+9JJ@yn<hYxW%Um7ZqPA
z?7xB=vl@Rk#wtv9NS_wMC^%L)EA^ixwm@)3YKT}C=aFd_^DZnbG%nmLW|5F8Z;&sO
zw5*!vs0h`#Fn?wy!wh9jF3P1TKFh@^UMp(PpE+{51$uC{$94SBk>iu_<@YTfn{=7m
zw}7ojGg~lYGuNH+sT7*3nbSX!UZ9vNon5MOR4<lG&9^Kbnc6!VU#OVkDezQs<&Vuf
zs6;DHt!k3CO<i|yFMcNW=6bX9_V+1$S9wc(UW1F$AKRpVjIu>vK`+MOffWAa`KeMM
zMj-f)9ON0~TgY?B4xAEPB)lbDEV>H=LlI8m$f#kZ-nrqro)5Y)YXpxZyTwsSzk;f0
zI}4`uR|8^ZO15P~r?taCh7tBv_64_O`?@P{l-1rLql6*qL>baznP*X~C<OE|JZ)Z&
zjYF=YlF~%;fFtagilS6$sL|0e!m$DqtWnvFyL6^8p;5=tg$z8Fiv~S4fO;6syasmN
zW0vutLEW{Qa=B8ukCf5~xm55T>WIsT0F`H%QCXjgXZg9NQiXR#UHO7WllsGo+M;xQ
zy5)?X<3NhRi6vF7wyo#<eMMHg_OCTpmoXQ)v*^>7dy;$j`v>epgg*$M5u6bgu)||I
zVp?LbaJYWRlhKjID@D&3o6at+sGqh?wK&4*M{S@FrTu1L(>9~EN+nE%ObxA+UO->y
zs~oGG7+8*y!E3cRU8>1hYg@fuZERU{UU1FX>{x7{dn*i0gxdJozmq|>#?<ZTZyI?J
zX+ORbw8I+Ry4QD^ad?-dnz=w`i)Wke`qQ;4Hd-!3u3|z)_tS>khH=NiE3{9Z59}+#
zOZ&saJ>Bib!H21z6u(VZrC;&R*#PW-E!GB(*|$O1RpiBQy<di*p9xB+78%pfw9sD3
zdpU)HwLH5pEAd%9>tjB+$EcLBJm}9UI}_;x9JnFDA|c744WTEYWTC2|d|}lQgE3}d
zhq1R2KTzDcBY@v<8YquR7otWE<{e+i9;MURw!Smda5DVnd{&53ilQ!ZCgim5?qxx_
z=v(fWG|1fH$7ZvUS$%VRc?@{eB}F3Zllzr^%l?N)-==#>Sadt8S3BG`K`>E;cumGd
zT!px)@UTEyrc};76Eo#EXM^SS;?NDLt~8SbN^wI$;<VCqQ!%16Oirzh<iz)?Qe!1s
z>AD;h?z2~?#3T2a*wM-KKbbXrfpPH}HB>u9bTO&}ScAJJ^CtH<9Itn;rUAe|8Uj`H
z^358L#8mW_&hs;;U8gKd?fH5Ng(Y&MNUD*#yTW@`<L;SQsjWn=<a8{ya=aXTL?3a&
zx(QSqEHhpvSEI6}b@kHA%A2~?mASg8G%K`XyED5q!H{-6J`!8Gk1AKy<Rv9Vhim0Y
z>MxOZ5hQ)7CR}4WoG%g*!b7o+mmIGg){Zm>v>7jUxSs1NPlK<cPu8&IXi2n(>gX+Y
z2ZYD@t0JAWE}Cr~S0}cLo#r$wRN89GZ`Qj_@W#u=wY8ZWpS_ygVLF7{3cO1Xrs1Xd
zQelVbzBem>SMk#7*LB(Qe7Gp<z|+WXX!8z!h>)QQry`-UE&rm^;MlN!oG`a$Nm^;H
zF`}W_aewz4<78Ylrfjt>`?pR_M;T?&FXO<mkYp@gmB?RJ3qkXzr7RWN8ZRzC^>&hF
z2r@C*GOa5YU$>yH5Sto>ZOnhwm6feZu4>fZT{R|IoBH0q)%dgy-AA0|xQM$rok=gc
z|5`8Z@JT(Y{046pn4jnRFuR7s7yLFmeK_50jOZxWr&Fz3^YD<v<kL|@=b^jev+))U
zWraA4YlKa0E0|%zcE=W((W)`6!HFotK0A*&>2Zm+pzDI&$gSnrdpQ{KbM|MUSZGW#
zHWoWA&$=t&i?HiTRAVa{Sx%da#dPWY%vAA2Rye!kHTBDBjidhoLe2x9?_t12@P6bv
zS|+W#Hs@p6LKW0X`${J5i<ZsN<%!uD_J;ed<Yh97YENZ$OMkn|mB;DbIMxuG&Dsx-
zx|5X6`I!~>Cf!wv-%M_<TRh$E*Dt>xsPNKwOI}iUI`5s_U2)ve?=Po+_KLZ-KCMmo
zB!1l26yGwu+3%r5lHJMiWaFMkAD%sW4`1+16ih(A48Kpl{Jd}Jx%uQ9^6Fmd>)<;9
zyA+w1I--NoxqRiZdeZ3s=nrLky#d!j<7@hAbKtr0V9HnE<HpE-LUd#}M>Zf=k@J&p
z18W!$GvV5Zdrk!`@YC2Z#O59X*rB=I#UwbGbHwwHVq{@&R%L?Aoa$%l5@cI2mmsjX
zHlQDRv)B?Kl%|*2P1R;DQ4s9-h=Y-taN_z_-f?O(JoM7W>@XPvtUv{fcPJzTk_`dg
zdl@pZ9lq-d;_gY2LfDP<QNJA7`IZ|>ccc@%mD)F8$|n*ER9nE<iK{z;fuWN8J;5au
zi7&yxz<-#3S94O6k>)hCwWifKvNbTKb+fhujRphbcH;y+S{pm*<GWc~**J2#@euxF
z1SjbEZ!;Ys{y&B|S@IC7$;jgi**X~Gv(Pfr(i8H2!N<qvb}%yGR1_Bd7dYq_522Zp
zlN~1=ovW)Wtt%6)t%E5Y0|y5O9X%r*BO?uH1dXG+jg!6`jg2GGKau<!kFc?$p@X@d
zlew)8{$ITM2DZ*lJcNXQ3H|5zC!fY{=Kn*<#_?Zbfdr)c+e61dOHcP7+#pcyzpb3|
z=5EGTYQpB$#x{;18oaE`?A-qt|NrUvABz74ss2BZ?DYQ)_+K6W0_3Lq%m07L^G{{{
zqZOnr-Y?vA|Is|}myOnjXpk|m&4uN@gYFQ2Edx5p|M&UzuvMK22nP1GUP4&lyBqjv
z*5@Q_5tIQgJWS~G^FM1O0C?c3fSrUL^tnI)B+U6aW&i*d)lDq`1^OJ|>UvS-1J6?B
zT4}oHklET^=5b*oW?|y-&a<YiJ)hMU$HWML0~hK7(*OJRE|i-H6LwF74iVx<pa=c0
zTQ78)ANo)ImH%!-hZqM4zd-+Qa7iFNcnttyh2h`${@D#a)$zZm!1*Hsr1`1+dtClE
zxHNyg?SC-?>dXazqX4Bgt8xCDNjOm38P<Pz{s#_nfi6HVGP~8l|H$&U9mFE!|EBSO
zlSMNq=-1mBxk{;2+OtiW({IZV5Ty8x!5Y<!|Iyz!pxfKs!rGea#@YRb??;s5<>sIB
zek@z3pQ6zYN-`|Tp3g_cb&gNbR8?)$V=N0=m6}a9PfCrp(9C}&B8^-8kLGf%3Fyrx
z@m+7m*ji62TgPSCw}ME5u^5fQPUgx|PZp|ju-NVNhFaVPJTLB-Oz{>PELEEhvOW8Y
ztyX|$0e&!HGWa#0{%u446@krht_vqIJnwoe27{oJ)fUIZ036p<`|~iO7`vVjct^|1
z7L&xAEAXJ+F}96W>*_y{HksCq;c`oj1oD5;k=QN9{M$(;U4U*U6-}#*Bh)(~2sp<o
zTK0WLe^j)cl^=Rh<eF_e-&iU5_vxN5dl1)<rKphr<L(DpY`)9)@cw=vRQCk`(U|pD
zf0lW*9vS8-;qw@-bMchR1zlffC259WO`oT|H6cWvq+JP|Ro4ygwZV9zwacv@rQ)+R
zf=!U~#Qw*5%mSf-NsJ$_H?21ld>eOXD{SiBFBhE~cZX9w@aXhicNZIcLy4Sj6Fd*i
z**u;hn<*k_+RS|h1Dxz26S`a?=Em#bePp!XFP9Yz?nJX?zku-c)P$2|FTPz3;4Sgi
z9h2pF)dp?+_25lifGqnKz;#Yp)!5GIcy=h7X`KDMs`K69<7ML`pcMIRmha;g#qH9X
z>tWTA$!Dd658~ylDTnK3lxYoQ_ia=)J^$TLh&;EGz_jbfomeOug9Hlu%hrnyM<rQ~
z!WHKQ?X}NnT5+4zLO8hK_G&DQ089vQB!u?Ikw;CNmOkIRIo0=z&X0T39Ph?d(NoiG
z_x($R&bRwa-%Wq0S8T`8Ew**Hoe(!Vsse&ZOz5!r!K&8t-|v5*@NW6uAAh0=Oxo5C
zk>>GOx9p|liEnt_M0dVFpR_~A=m3)E$xT9i_P<b-1CIlrDe?<bb(ZBAhlq7ZhefD^
z2a3~8GA30~85hG5qW#Q)T(4jA?7_N_GMN!~pA|KH-|hl$?m<>Oq;0=d<T)K@--*fv
zU$Smmt2J!Y2Zfgp!po<FoK{}%=VlN?RA%rTlqj@h5JJ#S{B?`1t`~5X(X7jZIKHn}
z!5ZaJQ>+?a!P6gq%kjN2n>kNi?!(aFzin}zl%qWVd3}<6*%f}>3`E4v*tQ?QX+mzz
zy3hj!uEjr;Z3?;%AnPN_F0a_MTHk+3OJRtw_5cevU|VyVImI?7pdXq9$HagL;irIz
z<-Qg-<ASAYvvUamjEvB>mJ0!&b`#}RK@&YEt7lX~K5%ML2u$e%fFcc45zYIMngWTl
z4_pT^<rK>CTc!KqvJXwy>7wJM5~WP4A{EP*OAIuVU3C7OJ>B7Y-zqG$dp_iXBPG^l
zdR-5#+?*`8Vn8^H@H-ySG%gyPgVa%VQg^yPN8qQ+lA&#%C86Ud$Lf<u*LE48X%sJ#
zaan!*W!a!UE)>T_E%8R{W?G7lV|qQef|(Iu0E((nQ*mdhx;`Aj&g~!XFEP-JnCswB
z*uYT+j(LOzst$1hsf85jM&b}-c@#dJhegBKhVbXEypMIy3z-z-X$dlCs)~BWYb{OA
zFjDiqWK{$aK@?4jE1Qnf&X0Fb^m;QkLqSztZ|A`mm1tBo$F9O`u_qW4M#@;QL>f}<
zP;7@VuB%>@5vG-;ngB4BxCI^0v8kedJl|K+iCykQ!=SD<nCE#-TW3VR*J>hk7y)2j
z4lg-reF$iTJ&+$pn$dy>kOmD@k70eta$UEt>UjR;dC~r`ie=k|6wn_bDY_9_25tjM
zBa5^AiRtQ^(9A78{>i*Ji<KP;c2kFv^3Rezf&_6jT19~GdCP&Mj>i0_J4QECcK>82
zMjhnp=bLdZx=YD}eb*&pG`<;WrsPWr9XdMFEz$?>qk_aX;y{|MWM;(o5xP!lcUG<h
zZq`d2$595E@rHTu0HZE{zN5*v5G<QRlHtjb)3qGOagGY*@{DZH)0%;h>qz&JkLwXS
z7Huk=OkJCny|!JevZLQ$Zw{w^zHLX+ofpnXXVG%FYz3o1M*UIel%#wAuFeBAo0>n0
zr0clcMmZ$8=A~lWh@&puS5-Go2D<feg-zscc$`#tr_yRmKK5dtHf{KL(uF6~sqSg6
z8piUxa^H+9byJZJMTxON2LK5;FC}Psc&s`rqWm7!LOGvK>n1Xigjd13In=+A{21{)
zsK=40UEBTr==Vq$!pqHTvQiP~XS8`)8<o%ZxEWY?Pnoz95uBG*)wJ%R;}s;7tKJ7|
zp=ZZ(=|Vn#*0@6Z9T~?~tf+3BlL~rdg7?J=8S*On%3yhn*X*qPdK4OglKskaN(60B
zz1BOQ!q^1+(YSpQSn(t4)i!V(sGsD?b2qE>G*i)t2OCx!rCgS5D2-bV;lYxn^ol*D
z_E=h+y?nExNX%r$2a1!PQvtJ1QZO3B9Zj!Y>NC;(jCBQ+L&IZD+RUZEGYgm;HQp~$
zl$iPS>RUb_H?vNVWA5AXfzF2eE^7S=90v8vU3V|O+$@PRC51&BAl<E|$cQ>$`MPe5
zRkJZ?8CYT1>;rNv*{nvxilD9uz7Oo<;NRb5e3T3IRtbo})47I4+EktE!iE67&{1wm
z4N4j31iT9hjdUH4+w|4nk-Lm!CQbu8PHRWLY^6KTt21?gqHa-KEQ(p4NsjmZ<W(WM
z?eC)^J35oDf>v{?udrd!X{f@|-_q~TW|ibBFjW)_<GvVA<_ak5D$)EDR)5AYNh2`3
zFKZ7GNJL2K6h#KQ^TD(Y57X`!?otJYgNn?4hoDP8mul52onWhixNV<$<DZRzV3=N{
zn+Er`isb5<9J5PZ{-4?bB>*be6x8}dDVm1w;xPR#Q@c7Z^7e%m4z^sA9?)T#V)Knu
z(TVmHeei!qesR%ctd9YD(YHPt8D{7gr}T9khDcyy?tP)_7)kqm-kz1c)ZLt_^Xn)H
zrHzMOL3s<U<_YA6TxL*JE&CaQxTEu{mXS1#<h$n4DABa|35JR%;`vx*0YLp=FN?I9
z6A@|dN8B451d$0tho2Wy(-4FinBj+a&2qDW-vgzd#G=sSy#N|6TIjbpf0GU|9@G@8
zfrBq-l>_U(pK04+N2gx%Gj2e(9swd0&&O?0ISdS#`1lpl6?^D(lkfTP=i_%fDts|3
zj_&0^n3WO_0C6pMz4x;3>A1oh4y7Zm`)+(5jX`v&@{<5oam5bl0<sZ=0ffj<L#lJz
zHhspX*+wWmPn?@aKZa?!q&AMFB|Qw<lu4#_%2vdL8q!()jO?FYtxEei=*&M?5mc4r
zzh%m>ZJodsRFt8c6(8n+7gG5;L2Rqg$0A;^ur&4Z=#M^+TcHuLrNw?>Lz%W!m>Yu;
zpJnJb4NC$x;MrE=HXk2=&IMi~q$YMf+Mg0Vk@}VgJ?8bNZqD4?0rXFXQ=A~0!%Xk4
zDKPPX@~;*M`i9&7Fr0fU+H*pj<S5R0*0K=lCjkNFr8wDBT6<7F0}aweBke&jVK7cE
z>+T2g0&oDFcw$A4Hs6M20HmZfGX<0zeF0HI5<JF+f*4M65Nff72|x6a2q2ahQS7Wy
z7-Thrc_>XUyE1~&!^>EBKGOPeZdgYR5(^`6r095`4G}^A^3Kewt`VPzOC`f<a<;U*
ztUX*n9eRTdK>qtE$H5OB#p&<Vo8Av=2QU!S8tTEyC26zY(zB^}b3ZZ7YV!B?iB+V6
zvXXmU9O))}6N-u?rjmsMczkLk3=?!PO)*Tw)6%#lRI^w^e9|SWsc6tLJ1rLz73U~u
z-OQug=F%xQ87?XwlR`diAwCZ?21&)4{S|4bED*QT?3bdj9+bl5tT7Tnvjbv&<_ac@
zDt|9S%U}!o46={W9dR~;*W+b*476<i*)(fJ7^lIJ&~ZMhHRc(|#`qnEQ<uoau=C^e
zR^n%!y^{FP!Vt~%6<=X`)_yyWxNXX`7~<p85A}6cW9OA_;)+<;t~R(-x&RIhpT-Tl
zMRK(?1JTaTmo2N#sGi`qA*Mi?{qcT@b%$t)=ob=mU8EI}Q0_X!b{v6FtxJ|G)@Ezz
z?1i~NgD<&*f<lL6=NKxGf?T#1?hx=5NZsWT2NpI2LG!~RiqrM{6y|>LWk|1QKv?{D
z5-yJiK+H*9mWHaXJZ8%@-8jV#&+DgR&}De+m_=Cy>MH{|>_XhHpV$n|@VMNIpNQYt
z3i`YLpmb85{UDL@Zqa1!<v0!y7iZ>XFGV>@`Y?P`Ch<aZ^JSKpODH!=&=;i_$S!g1
zi0<%yG4Y}>bpyx{CX%#RzN0SjW|334k{fD9Ay<m<6BkbVL*X8_x}0H33nu&=MQJGO
zZI!GX$OYflk}B2|EOVL^fC^?ttLN%V=Il}i%Mr|$GWtQhJZE6sS?(^wlY2#jY?GW`
zB00^5FK_T}fQ($cnTs=HlKymEA^xt=UcajvI57NK$X<X5HY`Dm$&w6i-<=RY%p+W8
z3bfjG$Q~P+P83K?`o(3O0vJ-~gEtYq69B8g*&!K*TE<90?H7OM?w6<+oqMcjaR2oc
zT6AMLh+9M`6oKZ)kHW_@H42D3RN$OP(`Eo{2-FQX=?Uhh#6}ddsKZ<eDs_{XJc>7y
zcW;_}-DhC{H8Y&Bp!shkUJpnfui(Se^f|(21^IW6`%`sA6)%#oMUK5BB}UwDr$#hu
zr|1j)f>G;1F|q#9-<jd~C861p%$@up;rv5h7=SgvcH|DH-|U=$I`Jt&PdA-r3YQob
z;ho>t%U*AW&kQ|8fJXExuaQ15xC#k{O%T30yxzC+%AJ_*Fz4<wq95;X?Zi3=&IQKe
z=9n08_LFi*8(13}rvqVlglcGWj>o)rpbbR7K>)Nh7*5ALN?;f=rj<jw4C|6no~#)a
zjfiwJJ&)oBsM`F}WSk)Uh95*>&Y65vlv!C2ZJ+%dpqpZ<bl<Ie9FrT<+=<_<2vcke
zDJzp6beT^0=w@>K68?mx&TmARdj{|=;GWX?LwnA4lOf974NW8>U;<49*i>+RE}v7?
z<=IsqmD|0UXnI=)@@o!&A<o<Dp=^-95kCH0G@~#`h)G{6*+VL6@+)x<HlL7&l>c#M
zJsGppIGiRw-X6hS@1=TsB`upF1)36f{%wHnufYOXe@;nsbJ-;vC=(E@3tSY_T0-El
zP=h31Y0<mTxo_NW(D~8p!)}KxwV9W!B-_8xeM0Km6B;x$wPucNKn|$s7)X0Zefp}b
zMCxFIr2Na6pXz7Kon=LXN|+Xrhs?+1w%}~x7fNX724FxGFWtSX0kU6X;Ddl>9yj1m
zudp!gM9S#L)s__&IMSKTF<n)}1g%AIv<9HJ*=_kzS#2OHh91DM*i--<6^0hT%^{tm
zRd{%$QQ$M9F`V|xZL7|Ik0>3|^z!!69K)0D$7j|)9v6VpqNR`8Bz+4>%SBZHIeCpJ
zuBEtiZHU7pyFR6nL7v(!&a%XJxmGIV`^m?mW#oZhQ$~HpwPywZ^pO&%{tz$iNn7)`
zy%nu1oRJ)TI3X^%9R1mA=vgKHD96u+B1BKlya;~5#J8CW#u~>o@pVa6TDh9X%a4ob
zuwihA8bIQ;#Z)=GUa)-b(r>81S@sFALczSEBK}CU%=~7ujIWUFfQUAZ1n=`;zoZB6
zf~%6v-rOXz>o;=_C^+;uK5hG)dT=S)7Nf{l<zXJR@91OEeV%%a{3Bo?uTv6t=WDHp
zigM#D-XBRrzb*wi{5ce>L?X+D=_9_Nrb*9%qv<jR@sb27OTR|Tj(r2piJ>^6n2;rW
zSG|_7Ku~!kQ;?Ic6elbA3dKV&5h=kET;q~fd(>+ODs;k@sBCUz<RMAT%|<4*_8P`T
z*z^m*-^RuJz6(cF<6p}*pI&C`VLuq^1#3cleTnMlHGQ-|-om#0+4K3{ww%x}68SDP
zn5ythz&~$;F~C;}NGrH6wTvuYbfF*zu0Ym#8%Yi4XU-vayYxJ^OBT#PhvK_e<{a<;
z$ZJTAZ(%B@AK`URuWF$`w3tJd9HCI0FjM-Yhih3>xo01x(;!8pJqDO?J=4`I!!em(
zC_ZL>$*jsRwdQJt^Rq_*9^WVD=~-UDtE{3?5@9fnq&|DNFi)aAmUMZ~!;6c66gCac
zkN|fqk5}kZ1o|O`E0IC=Kz<8Eex(sOt8iiw^VyT>+;2XI-NDnB!6w8YgzV@>BD!({
zFSD#80Fb5zS4qaSVK2OsX_*nizM&v~l$+}V;fg1N3O`@Mn4a#TI72CD&cVxFN>Zys
za-~KzJ6Rrone9wtO|Ck`3}<ttRCt#O!lZ(h(WCt>h6AD=1t4@>OpZsOp)<}zA$(~@
zn|GDWvd-g;Xd|L#5tR`NmT~7`&3XQSE;n+8{9O@t>tyVq3tinPIq%Fe)6{&7aGNIy
zQFmc~`U4yc?Bhc_$Ib=}3^(iV{^i-)XQ$KtZ;+cZ>FGzw>FMd7!%-AWj+u=V#KtDF
z>FH0PHYF1il?Nry2d&sbZSw`6tGa`1!wU_}Xp{H6kM1b4u?UAeSnZN@Y0@zYS^J3f
zALFhaHx1?mFRKT<O|)x;w9cjX?<|$}(fb|?t?R#Ep1QNnor=|Uaz0)UOq+OLw-DKi
zdGAUydvMO}#;A0khL`RhueZ@Uo4wEFLpi_S1k-sD&{jRQeY{<46miVg#LT^dHgR+u
zZOcBcnmXSqIM$A2*?RUt!?u1Cew^rjSj0FL5H*;egUD63ew>=}dEUQD={`?8KOAVg
zof_hCdEVQ{dTM3f<{^5UIi5S_c#at!@O?a)9(6qS(KSBMI;M{>tF{a)&ljv{eoP`f
zx8#uVUWEsiU6tv+S=_H`pF5=fnTVEfi;vVQ@E$0Yb^ax)JLs^_l+)7g+EAuVqifTd
zvk4$GyU2RoCCgcR*@WU_r@Hur_t>>`N1*HT(7M6WPY`9}+PVVoZj`d$e&Ky_+75;5
zwt4FDMDnn=@w(5pcedWC`|x{V>^jy(>;48gwORPW6LoD8;c8vZYa7M-dPL{GR^}Y*
z<8gp5!~1@zSjM&fCr*XT%>dtf4ad2C><ahuRIJWzgoiH*QuEuH?MCBBI0&-;$0SKp
z$6dt=b86@7UaL*)@B6q=?gIDM9lQ;%Jt|#{J1v<uRR2#1?0T%{2Cp<P?x9?(`qul7
z?~HhOJLm&u8{UUPc)vM7wJDd<4u=A+@mV6sj(l}3;GG&Oq$9kCEqox0j_D&frZskf
zBz^O!;{<WCUIgwZK{N8#=KGhSd|sX(#AF;2H(3{6WE(m+I5m+l45YfmSB*{W){aR(
zu?q!oeII-9D&70>IzcwcFvNRVJmaeLm@BJn#X}1?)j8l?2$%94JLti^uWt0=jP*Fg
z*G6^@5~FZe5GffLEYrFBwe)fUQZ-vA$Zs+Yac|Pbur?n1S(da;G!`-nJrB4xmb^AU
z;hC+}$JST5yCJ>_a6C@M`plAVnDV_WDcEYK;Cl?2%Xu!NW^%1x#r$07ApUFrV~q0&
zuYINk?)*xIBkmoH2{p%yOpwJ+*Bh$5&z9Pyfe$OTolAFlx&d(-!~=t&ytjGk?&MD1
z2`|^{;=l{91FnhV@OmGRA6gy^;eqUD`3rkj`z2ip@BK2Ajf{H>CKF^CI?HMJC!&WJ
zIp60QA>1V*^HJlCkM{>G_E(UyH;68<pA%?DfzR@O+4uFu&L+k<tDwE*mHt?-T_UcT
z*+@@Zf1wC`%nb%8-J6Vs(+c)Jx@qH2>I<Jjc~13L>Z2G(h9v=4GPWKw<<&V)qo$lA
zmld|&mMQzBY%N2-nQgsKG)*5sTjE$1tvPg{dHj%G$aA7Tv8)AheM=jcx=a?-Kz9_e
zMjK~!wDujPoab5Uivyku_ooKWz6Y!H6HC}{Er%j>%kwuSVm!y3RMV~Nca<2B-YA-Q
z=znE4ogiE+=Q8D5S(-;uP8m7eq;vZNyL?Sui}}#*IO=?XV=yITb>X@3(zvU_q5JW?
z5bG{uNfY)A@2+d~KH>Xb#{ts(qJiVMV>am3)*@N3WJ`=)5K(!Z8TZH2L{5W5X>JXK
zn0NN`Z1MQ;O(5Q@o=cVitEkz?bBGsGAH(&1Q>XX+x+zGe_E=geU^4tv#)IRqQg$s(
z57zo+NXyYPwOD(wEZg;uyInHAmX_{sm6O$y$=!l@XNIdv!wy`@M=$Rt?z0y{@kR1h
zpB0M#x_~tf22s>R1)+CZ`iDXJk(hS=g=lMVa5I(m2FqLh1oY&?Xm+y>rr&)pS7?VJ
zw8{RKyJ5})(efE6W<9Ug8&(hAQ17+M8AP)WaEwo-7PP-VbbffNjGyQ{{C1ofZ*4&q
zfTMX3CgW7N;Igrg)U5x!xBLo{q@7lK7yMR4<D_NWxssJ`IsLOm3BaB4&#n19TiX`A
zsh^8FL1^1agle0?$^QPa&fNhK-qT>5YNNT6Ff1ROI^w>Bnqw7c_E$-zREQWhEa$x6
z7BjuZxx$WF>fB^FUu^osfV&Jy>D21We)O|-)X=Vp>OAhz`7U)R;bCKP>IuwG^sD70
zxvRqxQeiL9S?P6ib}ASc80n|78K^|S^zwK&CNcfkf;;0WAV<Mv4+-&@$y~4ZvO3>b
z)mKo(^=Cjgm12Rl&7qb8zFdziYys$uoa=}+z36<dBAfD<v>n&ys(kJmsfP`@Yb4dN
zQ@-5<urio5MjX2KZ+`}PPK9Ct%idtx+g+7!Ry2#Fc&ZuRS;5%vuXmNET874q^W<@2
zyw~xW8}Ii`-VzUo&Rf!K&41uCn6i$fqv=yBh~dzp#G3=3d`Gorlo2f}TyA=PwJSJm
z!Rj`TIcD2;c^$Oxuepoj8xIpm!CC&DISh-`$crT9NLv9#NUjpSV@Fz%yC9><!p8Mk
zL<z5EiIhqK7t~H?&DCL=)SYsbc{kq9T@h&^4R9`gPbXK-zG%5jIICkwYi)Js9CVy`
zfLpT)eohkcbTMzHZdOpqfxO5D7P+I25Cpv9H{U>7>hg{Nh0h$aZ+;X<N5flz!jbE3
zyTPrqeP{dyNa#0~2{G+8I-2j@pSD|Qr7YsOYkGwGg5Xta6yw?Cdgb-kVh_wNy=$9L
zgjEVliYGMJj#aXh#~7BTE4JV0bz)uHc38f)RS<W}&su-HJ#0w!S&S$ekXCXoSJ0hC
zxRO#gY*!70*-<<ZacfG<nwK>y#1)FDG!eMjKnjfJc_4M_rUV<$WPCB4?-y~nWDa3?
z-kJgVkjRkd1)U7AuUi=Wd3&xosgUEseWG<dt5moJ*}xPveADtNyfvL{iSQ0I0X$U3
zY0pAN1TS6Vuu#Y1YV7SH#MQsCZi^M;Qs#itsHwA8*C*V`7d0!${EFN3&YtCroiPt4
zYMUTq5v=x8V>+ty_Y1j{sY5o_uil`@aU$(1t`lV!=(%>H^D~km1^?I=)hG{r7nCo3
zBIROE3^^zbC+hLg7Jx`%7-^BpV+rBnF=)vj=CY0%_bnYFQ7l3orW_;c4k_&$$*G@j
zEOlQrGDSyg4w%Nb30kR*<By1NufpyzQc|s^qF%0AnoxRk-cC)edA5Xx-p^e-v6MA$
zo<PCRB1IOC%JQlfIih#%+@;C5ZZYWq_cj-*!I`<!hJ}K`b7f_n)iRch=Sbot!m=&A
zr=#mL?PQUm3>mkk`<lkX<_BHx%XQk3R4Z9kX4UOn#iHoEy>aUR&v^|aai##Uj&S`z
zB$`35=OT_F``}J_*~mQLF(0knj1@TCOj4mImMr?mqxD=WafdzxE+*_}f~~3_=_c;$
zCRCNWofhl~KAWaXXLWXUGtU9XMEcwwD2r9(v#7b^4Vj}@<>{iXLq~M%(bz7jGu{Pc
zYCEIm@WQ~35hAbOaigWWR)6VfluFwxykas*RaMo=bf$qw9#FIYNo@#Cc=8EN|J<}D
zd_b6=8ap7RuOrB=uC}P|zP9ULALl3<Jm|~*-uYdaH-l)cre>J`ZNFFtuKP1wE#Iun
zXmVCX^Qz)HQU=fItZz0}7}o?-VFOd@$U}P;{xMcw{kY>6mo*r!kIwsbYASuyGuN^g
zllH;SFSj4Ptn*8SV-dA;1-;aCqa@)%*L_ineg-nztg{!kUOQM5NxkmQQ8HgNE27a+
z8!fOzxU1d}%7#sV!kNg1aIO^X^EQsupC;NGxE9V}<lF=^i-CE{AeExe3-kia)U&UX
zitB^xz+R8!e21Iny$&)UMkghAWf;n_v=B+*bJdyQJ7{SLSWVng`}JCG%TEToKx@@^
z^-nd`lgFpMYngy0MLyd61o}w}^^;jwB%}o$0n%RLnD}aPXPg?8Co?+l{V*|ZgPX*h
zS7O?#mwJAUU7SFJ>my26_t7EE_3IQ?NYWUO$LZ9LS!8b&eS0nM6pF{{w)LCr2dg9;
z+2I8iwRn5tfdWINZ<bR1D@+WFt*oVX-#C#awZimVxGw7e0;!<~NX%lbM*8uwu8#%f
zG|0qC8U$ar3j_^RTRB6aKLMfC=V{IlCZL&)98ietWBP04GhnS)2a(0(D9{!ne#Y##
z%x_ttgxu<awC|{L(s6MHPzYArZRlyUM{gZc&x=lH2rv@vY*Gjh2u*V7GtWeef`7wJ
zw_lY_B@x27{VXhsX-HhYv9E3Dvf=$;EvLa>q}DXnkOUBWNz(tc<r6tW)|NPz1i@nK
z_*IpT=<~#Eo6p``Iny&abrj1^*~D=zG!wfBp0n1n02Iz_|2k91575lBmC#eGzcnm|
zB+sEwByc#A(*14yvtx`-=z8&xxefo>ynagbpwq&KJ7~VsUCZc@i!~4*v-KI`lM1S#
z#D}s=NLAbTu9Nl#5i16orbr5D7;8~U65f>sY0X-zYZELum9o2*sJgf?0ju;P58HQX
zday{>P||)y^SldWlP5d50Y`Cpzto+)xK!l;&eru#vdVPvzSPp3a9n-V!*PqiI5n^k
zTKuM#G3#Zo(`;;_7dgYA-a=;@Yz{ncC=5+v$RKuHXT~X$Ja@}7E9FhhBJ;i{dHQhe
zYcj5-q3fM9*F<VBa!P%YI`Km#ZPP?`Cvv8m9xkmZP)zkSai=2;L+w5c3|N%flT6>{
zrglEU6cabz-Exd)k=aU$Y*6rsq(TnUN?X*P+3%0#HR|>N<#WcenKrt-QkUdEp7}pe
z8?D+?$?`F4)Q*?;5=Jgi(`fjNeB-U40tOBAZdSSVlLNwW=7c_j3op_f)=;5KFjiR<
zE5_O1SdK!qv8}5oxa%{p9v_)LH#~*p8{5NP@vg(zZ$TGd6={19*TI?Q-nP)1EJM)U
zig1uKSEgnYUl`4y+yiag(Ou3`ij(X{224JEDQg*er0Zep*Avx`T=X=)-RX^nE*>KD
ztp5GCNP=~n_wF9ilC`h5hxTXde1KsbC(8R+VP7P>4z6H8Bfy)^@EvuG`5M9Qyb$N(
zmh9$9s#Orb&a#f?J5lGB-bw-URsZM(I<8|k2>#^mj}l2wTkz{c_;2+W2qrA{&`}KU
zr@a$UOpOlQ6dX@tsl3F#Au}#y2JT)vc{jahs3OU#l@m%K^-7ogD7^<Qr}7(qV>F|V
zhyVoq!N<CH$XX{tzK4utd<!j{;0FdaiNnznhfB!>CNcO_rVsXKdo~@mdp`3rmS>A6
zM1S^HsdL|BtO*OQBU;|2|1M4FJpjGO4g7)p;l<~RXwB7XDLr4)Qd%!a=(&^(1V7&s
zrL>XGBvCW3?GwhM&r3xC3R={RWPX<Ro8AXThyWBcP%4Hv5?8cLVWDB8v?H|*2;UCH
z_?FwA#%yh90+0q~lN;Mi62lnEeAPw`!G#v%1>a1g@ZGn+&}flfP``)#^wDo~h2aKO
zN&~kVR@Y4R1{XCY)Cp(&KpJFF-J^P-^6f}Rn9vb&Un3h;(y+1Q5b-&{C$-NjW^`}N
z=sC82=~t{#IEBb)zAjWeb5qD6)U~0o9t@oQ+M{5(Lzn<|r12O-$W9Q9umT353){6m
zMe(ihYjcf3CApmSrZWHcz<PHb-FqvHGG0_BclCuRM@xeRcF`_iHZR0yL>+<%N(dPY
z5ym+TaN5uBXswHVn1D~BF$go`g=UV~xDmPcO!uc@F)WK-7-X^dC-W})c-$P~a9+dD
zy%NONLgKqPTYBZL+5v6pzISvghQl4M$@x_T5|;tV?h&;KJh)d;J_Ent$kM_O0`W#e
zPVK|I_Ps68o}OU(40N9J9?Aul+1bDNpk!C)xeIf?Nj{NZlnpBa;^&d@`~0*U@b8`~
zg2AGrl}+KE7NaonwTAT*z9<y2qtp@K3gXq04*x2xn(nSsyWF)D97R@HJngR6BN*9!
zxAGvQ-lz69=KmlJNZV*Qc1`@kmNI0A5gyF1)-8YHu;3|uiHfmkozjbuwH+KfXo4&p
z+2Ny{kVvq{Sui7S7Zo~9`N#UyqY+2|FP_7TI<U@u^=G5d6D^%SyPG943?EEfN`-Ol
z0e3!rIu1C$+g61C1N6GULL&Zi(qS35VA3MBfYwdg?waB{w_H$P8MhddY#_O7c+4N_
zRMVx4>X1xB?`thc*u`W??8rn;MRLI?5$n`G_C4^Sj|R&5>nM)teI9P?eg*M@F)WTX
zp~d?67CE@K7KIiH2;;{z=4VVcuehDX%M6%kR+F#5e1jd0Fm_F2Dz+5{cKpKbc!$o2
z8<o^7qMi1uQl0*e9Q-+ZTPm+(uIjat7B`9jbpP#gI^{+>h}G87S?vSW^C69y+p}jP
zRQlyyCRCZP9BP?;^-c}$F)o<BvbX}QzgchXL~3k|!WShgjWG)(lr|<%GzumFH5X>Z
z;;qNU_v%i~uWhcqwA;$pVXx}pz@W7m`ow6|b6!}P#&uec0Tbk2k&E?T4=s!h3JXPO
zSN8qtDqo`z4dECjFLFRTT8(en#?+A~(|!59+F{Jqe#Q=m>D7~|SQgZm*Mp|Vgur3T
z_z40;{q$8=%n|f}O=v8^q@c*^^J;d-0f+Pu{K{?_AfofIfnh%5u3J}4+TLr*+xXZ7
zjYhwrgIMY?HdQ;v@?+R@b#`sP$=-TfflORubp|1OOw^9VATpQ9NFsGTl~>#g5Epyl
z_FFIa>Kt^-$)FzUmQm%wB*uq~M5}QECYF?>C5gfQ(>3SkVAs6*<~k3>Q^(Hny(OEC
zXcz*)y=KkE?4K}%OqDU1Sq4jJ3kQw75X%~&OnLnU4&q4}`%!UI>%DLepMyyVUHUlj
zs5KFiW&ndE-6HXQfbx$N4dYY((R1cy@{_dNRV;|(7O`4Q&VwwN?KUQg8z#d}sziz8
zm;RTOcx=*XSLwM$FB);uT>{$8Ja^4goRJ`}B!;M)Yv`s_ml4_#_X(>9&n+9Ol>v4?
zC7sdehO~n_cfRr_ZCJS|GQ%14t;W+x-<gy_ITza=9t805Y=|1C;l7L%l0hK^Q<#&y
zmVmiQISImDq@&t~F=xXFDop?3H!C=kYjx#Onq#fs@jb&SJuq>&0JY~Z!-CiB+X6D{
zb5MCuMIi~SvuhyC8P^d(7-2PT5gujIPgEy|P_GePZO=(ae*o*XIpJ<Pblo?sw0#@#
zw8ktfF&SZTupOrfFJoLBEYLNf+7g&~bTSV6Eg`m@!xprC-C4Hs&~z8e*br}dX9mie
zKN((k7Zc84Ju5I&;ow;^pC5=RwV=YF7Ia=+QB`ET<*^foLR#yIA*ds!q5TjUYO0wh
zd!-oS#d8exn;o_4s^R0Mfikt!X{>hDMeIGcM8cQt>?n6SD!q`uQW?iz*pdxzn#8Lm
zlXg9=c&8&xk+re|H(ss<F!fGDEvapds9r>LaaW!=83~93u$q$W=uD}y&^g}%M{KeW
z1N#%BagaP0Q*oEj3>WY4MB}90jVyJ8_>(gA<5-_|KZph6mDkhoo`-=)l4zdTD<+2W
zgKDGOEyjFfILm`ts+2y|Hpi)Q)M36}dWN}+EGhbFdyDeM0=;^$OkWxpx;YPTV-)kF
zhuNi%egN~~I7W%us9efT^V+Ug<yVEQBCgxK)F|Gft5cmt0tzGTg5B1+gS$-ou+k3{
zMKCcCYI_+x^o0i>8dqGuF!YG#Y#A&)#}I~L5(UVy=wU20SP>jMYT|)k@j>Jw!h{oP
z0PsSq`40o@Nx|>6KQbl@NTRiESLw7=YK{u)DVG<!$NcHTcj9mb^b6y}Fm@}k2a8}>
zB1J^hfBV#?{K(-T?#S$#qZ-Fw9b~lggt5?7ZoZVVRMpK66G7Xrq#>mBcDE@$(PuzK
zOZ)s3vB<acN#)u5N9MA~eq8tAgZ@&2SZa5I#19$eXRV|Bn4ccQPsR((l`=o{#}}G~
zk-LAP+bpuX{U-J~&KR+Kp2_K;5mLdNB^V?}BV~Y!>MQrQ3)bEGMj^yIN-=D06Vc35
z6)rTL%wwBG6$!+;Q-d7y#h75M0DS;a0v9W=t@a#OQ!Zbt;&5gXSyQjIy7cXgC&>X-
z*!sIhoFhkXYAwDARo;t$yn2a{)x45U>vdfhb<8Fu8{@vy$`;oG-^h*0pk9DkzKO`z
zMhqT~?17d^@u|u8qWu;gBxssBgjH$-TeAuKQ|XR_R*@cIV}&gT#v@I1vzdCQg>bKy
z{$9TH@6x%b3^lg;5)3f=Sqr0&E^7PXHDgwEkH*_9?pOnRg4JkWJ5lXn0CBwDi)k=u
zl!dVYBtgCQbg0U2xKAijCR@hgV=DOPd~dv{+7tNj(<V}Q5PGA=0eD|BMR<Z(u&)+f
zEx@B3TF`I<PRE^awjOA=;8+v-77+tfiw^dNKP;%V`t`FE*$y0QyM|Kp2e*x6(IeBZ
zD+sqzXx2uy`uBK8ks};m9Yek7e8oy;X>tJZ7Y7$e8S!jKEqGjpbu2N3iXFr8!L2KI
zi>E(hvALBowd*{6vV~hyTa%DB@x3^l@Vnt99~^?npgBrvxgi2!wy&#7<WvAELi|4@
zO9_S*wDPq_ythA+HNL6~7zo6%A|QRMA4Ap;sAI2@p0qo=j{GeD%#N1uGGMUnuJO)I
zv*@Ty384w&&yLwIIO0r~9oZ1-DcfmEGNDhb6_4sX{PiK3s35B6I*Ok`FI$W>8s2E4
z$bb5maobQVNj#M_bXaXtYq&mYO~82F2=WJqXj?8j*KQy_{*^7Iy2KcUxM+-05-ozM
z=@|0G;F$R*EX#&k_mpomM${%(tBg4{9KEK{$au7>M%mX9qsT^T=_9@f0x;tP-$YJx
zg|6zVQOjIayc(b^?KUaf*PbsOs(n)yXO(Q&4PaDghy95TH-{olGJFzV(q73?L4O0a
zsXPC2q=1JtyQ#-pyh!xEOvB2vM^D%~H8!y>Mxl^8R$JW=yx!kQDX`if-Kza-f3)jV
zBitiBw|0)8a-eu9JlKDaJ`$!!0YPhK^o9TxZ`lI_Wv64)*8gl9<w3Mef40WwTnljP
zdCv2u<BL$!MI>tsm-^Bl>O&`^lCiz?xcl3vRPcTno!y+<@(-%zp)^--&`F)`>vA;Z
zdi(e8I3=!GDcDq3e7JM_c17ry>-}#)#HeNfk|M84En}B2#%Pi;M))O~jiy`OBLU|=
z_M(Cj&c!!WZRcGV(M|P9HJ&lor2SJBwhs|>@RO>8QO9w~DD8n^tSyKo#i4xDfu`3_
zWhGd_dE5lu?d(+wKOEO3XxcrXl^mCYq9wNL>XfH%4APJ!S`Y4|YPObD*x@W$6w&98
zFE6{I=HjOP#J~97(0+j(t<_xR`UNMHi`=WvKKK2be+0isE@3q>gnkRv4r1!4%?fWn
zWB>I~!qD8PMt{IS?Es|)?s$~&%Frn<TdNGUcCQ*YNdRd|yz*>+m7@K<O~IExLgV+K
zHxMaNHG|IKb@U@^)6)c{U%^vGq$%S)=VkL~s28_+MwoDT=m7rCD^pVCrFq0|saa!t
z`<TxAFaa3bh)lh#^i&d^uHP|2x_A?0xI~iw!IeBx3AEK0h#B_np(H=)qJtfoqLf@2
zcd;=i7C=$Tz);elT+#f}5Hq%3!|3tM3WUETut|hzUh+GD65^YGMR?b&Gf%orP`b1b
z8|h3mUSg@HgC(j{XsX3CA_VD7bMMe>aC=XD$s8hpQUH$l*c7CQe~=`Hm_ctEnq^^6
z??-CtwL4b%A<fn%UBqCB^_-otWC1O`gGZ*6dT#XF-js0-y<WrQ?#McjpCoLhh{96j
zFX?}9Pmkb#1Q|1QyE;AOr21rpnZurLbDJ(LXN8_BA=6HSR06QqWVNjL{x7QE0jNau
zJX87s#39nLR2Ja|uiAXBv=GVo=c|oXELG6|6zU(CC&NL+vZ`kz|3&F9)87Fu|8MYr
gyypMEHa<QBJ6=B+aiWX<et{z)A}d@es2A{m0K&Xt`v3p{

literal 0
HcmV?d00001

diff --git a/docs/figures/widedeep_arch.png b/docs/figures/widedeep_arch.png
new file mode 100644
index 0000000000000000000000000000000000000000..882ac5f33fc7b3e8695e87967dfa8fdb48246323
GIT binary patch
literal 157281
zcmeEuWmuHq)-E6*h=d@5<cKuV(p@5r)X*s)F*FR_N-9W5gCN~Vw-O>PEgb?wcjtL$
z@2!mcJA0q==lnZd7ck5_Pp!3{do>}-iqhD3NbVpZAz{lrmsCMQLZ?PTx;2A=4!jet
z=D>%9bf?ThLPA+aLIR}hU~6V!ZHk2SJR~;mwwkIIfiJA_AQBIR7aH>;36z4QB91HN
zP8b`Bib2z+thUj$aHkmU-o3(={9u18<d;R7r5Qg<S-)p|ykL1?t$B5utmVW0_4O$%
z#bfJuxi=-jqd&!Q8Tp6$sfxveeAGzh&S)ug*YaNk_~V(vUjq1*ebMeVfxunZxVXsj
z-(byM=k>nfJ=KNJJTLdpeF#m{>A^_8fhNgRJ%le!JW)wYP*t=jkZg#ud6J{)$mhd|
z3%W{C*}b2(SeOX%wlI7O?}(U?ls+>oL>A57J^lO?De+#mIT6Mno5)jxa0s<{8LG(!
zx!~t=$3S8e`>%J)-x~`qq($%1iCFmTCQu!B!g4y@tXV&a9}&W2%TM;N`hT6wMy14x
zP_&|wRyb)HQgfG=a@NNs5IixL>8dETe}E;uDa|J&!e*9?Rx2Fx>&fCddiR)_0@g#A
zTA7H_7F`69P+x{_bFTe|CCM(6tS--*Up;XOZN2^B@7IT<1Z%Hb4pwUHS@1d(Nnukx
zxpSDLl(qUFC>i<<s4;g^gV$MSJzTcZj+mA^$Ck6CZ+9K{rF}P`o>x5Sc^e0#y?=V9
z@U!jvW|rH`p5|boU2P*hlW1{2b8u8iZ!61U+l1ZeL$jTLmX3#>Uvb4tgRt2RZ}%Bu
zkp&}tC+H=w{WKT-IE9WNCks|qs~H>eQ!wO<JI7?-w?1UUOtX&Sw#n}U+LD@dbWVJl
z*!D9f&Yx|><5X~dsNMazYbb7BLFoGe^~X=f?{S3TtIxVHF+|bObNy*o{2#^p5q?M`
zsPQBEV2>9l_LTaIJf0~3%YeJfUr;pfv@74SGP<SqmHQ{A$2=05KP@kg`5cO_AF17K
z3UQ)?7CGX;W43S*24?zWc}ao+iST=JZ8-BFieO57e@QCL=NwUQo=1AVME*?1Ak`Jg
z`0}<&*o8ltL?0+F5@%6(@a@54+sJ-;oUHWxTpUY(MKL~lBJ5|M1|Gg5;U{I54Er(i
zibgwNOPX|mbA|8#XX15}WRRV&>X$ORGhQaL+pqwvR+<GBdyXk8UCd$>a6myv<=li_
zwVhrK@)Vumz5cfHT@6<rPmnM=>zDp-tSu!i;Q8YP(p}h2_0IH(*aZ(t%14D#>Kc64
zKv!`RBb@YjWwD<Myt2GXH4-Z{4RrV+Pcj$=5{RG6gUs+?1mJtsc%;EOA>?fo3wR-t
zgVb74Q!+->!VD2=@)MF1O8ioFGHWzm)J&0w-(6n%I8bSc>pv?7Z9h(<o~0I{!&Qx;
z|8^fNODy#~L0U^<T*@L{H|Id*O<tFpYQ6whVxpLyY*FT@YQHMwi&;i`dSdz-dP(|V
z!VmOnjAUvxxj%|_LbAonKc4=GdUaYcQXy0^RB>KW?ZiJFxFw~O2^#<pS*@C{Dy>iw
zCWMZL3QDCf7`I!tKWk44Eg`=DhCQG=zLWl?8XFF0F(-Nqy)3P)u&ig6l<bIXM+|@L
zo0x>IsxG6h@vefHV+I42E!9VP@wq2?80w*FF8Rsot40g)Z{#VneQT3z!`xEG3uM)}
z)R*$iazEr#C_}*HYN{_BmBZAN3N(sb#EZgSn<SLF7Cspz8GSUWH)`%o?p$2=+$+ny
z@l4?mmn8aW!&6BT8WL<0=u=^<&s>*JIl0CxxGjW7TdcHRNxkYFn8<h5w=#@cqpp_m
zm!XNqQE>nG)>7w{v}OO;OrDpz$8%{gov3tvxeTnZ&zD7ROpZsQTK729rkO&uUqY!6
zTnv@nM$>uQYieG`G!FY2a5iz~cXoOva4>loge{M4eD}k{r^JHXnIsN(Rk6FVb+B*0
zJqQ<n`+#`lp%7a(v19nqTON|?hdylVTp^rVMuRU0&GU`AIGLXoaeBRc#<Tx438G=*
zGMW6UkG=ottg&}1N~K-J^XlBH8k-PpeyvdEYE2h;T6vd*CkeWowL_t1?UjtB`Q>X>
zW?Dg-tmQVJy}8<7$u7h=t4?HlmfKf!ejYd7|Dn;&{oN%261GVyfF<CkS2+#wsB3J#
zG(I;wAH(^8Llsnnb4-@-F8v)7nGSg~Kc7Hn@><GS%G>1V6dnPngBss_vU+kD3o4Oq
zTsupLo^UOH?Q$)J!)0r47xNG#O=nhZC@&+gKKrPKv}S)AS_5?}6^at7@^Ern^Vk!z
z5=?V{=#JUw;vob}bmXi#uPteSG?>mvH;SCpoIN?<+YH;s-eKN_9INhz?H25LEZEMa
zEgQf@i06peMCP18kJxza^bF~gg6QT(_Rs^810JBKU}d98qk+*xu*)#szpe<(NEb_A
zm!MSOOr=d2y^n6xP<m(-(W=mD9&}A27LE~-`_A=&1!*nlGjr%eE+$KZ;qzD9D_jG(
zCOyV`CfX(s%Vf$b%P1E-e|0U?E#$PZhBrrjj0GvaQlzH&M!Tg*mbk^^)p{^P-I2MF
zS<fkN-jiZ)kv-Z|m_TR7ZnkSqu1HJWL9;D))q?wLY$r&?l_5zL9ZlH(t$&<2t5I#K
z!6FMIfsm5360_W<oTHLX+GyHVg8qA9(9u0`h=Dn(nRWL`<iiiObo(9}x*8JPd;)yu
z@sd(e4195)k~n#CBrF-2Ke+U|q%;~GeSD%>B|y}f@_sToCzMMXyx+U$^L?9Thd!6(
zgqO~xZ@#w6k;bv4uROlmVP;?ax+M+v3x>5^&GN&b)Jg1Ee5-{#EK-UgOq+<5g++U+
zp4zZ;CTln=L9T?kTuY;BV|uS<iCKzYN-osh8{;IsShGx1W2&|V(q2WmD-!BsZD7&t
zFw|vB(-PI4#hpc%WuSqnp?B4Jl+%^gRHCRi-So|(ceJvtGO7|f=2kqZldY?vIa)64
z<YmHF-;jAR{YHzFMV(j8dh+>nd~;a=N9Jd@=z;9Yq$ag2@O0i$vCYrIMz^J9T%AB2
z8=J!k?PdjYoz<%T$@a<ZO^lxdnkFi$WzjX|hOlzIvib?{<qO?<9V_VSR*46-N8Qo0
zEg|r_u1>v@$7~{h1xs_WkI&#0>1|Kkr(-%J+pi3{VkiOwPH!7rL|t@GOMDH|#&yIQ
zqhuHUd`iFnO6Kt(Z;5>=Lt|b6UwZ8?%^zi>52`<WYSzB4gVkK^t`S($jV5QNKH)#T
zRGZIB9~vCGWj@$<P|;`U<;s1vvFUgZHqn1Ot=aYKb;~%n&17EK7Ey$OL!;9M2h3rJ
zFo)?an7o9daoKhARlTamFK5_#FORNGg_HayuiMc6a)M4%HOtiUbd!VmM9RB*cIdV<
zt+UHC!OVJnn)CHxfdB1Q+&DohK}NU2gPMhqqv*uh9v`a}hc*6z%}K9Ai{YUlA^+64
zX4utLR`trmORaXtMp&hni&w|_<+*J`{8h$r*1^(_=x%e&b>(U7c)?>I*Gqe6Ht*8o
z%8kQgo`M<6rs@5@dA)VzlONH?<J&<ZF`mO0HOB-n)k#=Qe}53n^O0BHh2>S{lFm-G
z^lW*vPu!mA#efD<5cX3p!ns}wBzF{~eRL$}GfgES#_vCUzR-X5Stx7rP)kN~aY2#n
zzl~9gUYd8u%7XnmoZ8n8JsWy!ct&6N+xhH(zL#Xo(JgDrQf%itwbvVI&EzIt7ioJ}
z=XW3Llh1AMHzS*(VWLn>oT~vgY^bTGjG2N05+m>$0|^zG1PKjzg$#TPBa{Al{S28N
z>DE8qM?pdgvOq%p*Jl)g-|(+U;2S>XpTBQKy+OJS{DlvEyM9FZ{b_XSkGFomzBL0p
zLlRS!kdXm?tC~2Nn%X$NvURHU`wAb0Y4=>q5eex54g4EfMumC@n19SdP18wJL7v~l
z7Q$lm($?6N#T8-)p9e|Ml^=KsF?BKmxk9XM9Qj>^sQ&Q@e&99yZB{DKKR)7QB}ApE
zpbV0*bua~Sv#_zSQ3>Axfk1)|FU|N>B%l3jIPjMc)hj0_JAPJHC=|*9<z%sSFlS}w
z<Ktsx<6!0BU<N+H?C56WWaP?h<M{ZWll-1X($vw!!NShT!qx@^pV!FP*4asjiVD8a
zpRa%J)6~`CW+fZPe;o@rAS?U{D?1At>z}!Sp@Q&t`IRkPO|7*gEg%5RfHj2q_}B#h
z@%jJp<Yq;Lk(xIn*|~T)5Qic>`uCygj;0P0wh&-VC*d1@{cAAd!+#AFWQ89cffWCw
z^FQ7N2rYa^ko6DKgzr=iQ9J?oNM<3a1eZ!c%HUtf+rSU{fBuHQUN_5gU#3Dr5=D}c
z6jO6W-k81}e_y@s+io>oYviZr&m<ktY3M)t8;ia`l~n5~wjZ<8b9`4&%-4a|R<ppE
zk%5ux3;Oty8kTIdCqK(CU7zxh_9<t|xj{eY<g_~7Reo(mb(1j6u{K453<Knggz_JL
zh$5lsqEe&e$_@R855PwnqG3GWkrG8hM#cNz|M2t0!i&V<WhehHD<L6cHr@I!i{RNI
zqu?XcW#9TwivX*GKwqK%c^Am2C@i9)RC8@k|M|Q~D5%sYIR9mvqF;c+q9(`V{D-5$
zY2fRNI{P1K0X)E{K|&7kB^W~fPgDnpiWfO2@ZYwHOaubmvoq%Z-xK_wtE1q1-u^G!
z1kC}5MNyQ${a>gqD(b)eAB6*WfU4{3EBPg4^}kUa5DxPHvd#Z59Dm8m|2yI6qTAUh
z?_Vn@X&OA){S}*>tlRW0G_TR?+y$GASNV7vn%2krE8JMtp5$gjexHGxOJ7gUbq|q)
z0LuhV7o+PAM|B8^h;p`@&K14RkFcMol{6kD7T2vY($Lbb1>g$*nEM)VYY_J>)gMW2
zD+WaWXX9}X+07crs4cp4@1-dQQlK*y0rza1FRNJURx*6wofOw^4YuNn6b1&Oz9kpX
zkO;!Te&zG@-5-I((*Y#dc-@;*=Qk*TR!~H)FUlW0(fnAXQ=M=B>)YByWr22;^}_@E
zAM=zb-5;2TiuLL@g3z%^a*bNwpuL*1sW$6Nw&<^fOlrh3fm!TQaK93`o-V~X_i~N>
z#F_Ovp7y={Sdg>+@@Vq#r%42Zu&_P>oub<H)um9JtJIC)f7Asw)736*`g<=(ehaAH
z*OyQ|`*y-aZiu!Y6-!+A)!Kxhb&-CfyPwt7S*ho#11V~rW?6=!(4m?S7ip=ByszPC
zq1HsHQ2<Nx#h&k>2?@p1SCPWk7mkzC+cS-a06FfjuyS&$GB7Y4O+|=Y<pOgN*7OC<
zVQl;?laQwLsyNyhHwnOJZ8|Zw<9Bgq7t4!Q64r{QQ>ywZL7@DUiIx`u6X;6-e(c>*
zq5Q*+IXu6^^|8s@<I5wcpdQvuor+iK2HvMqR%6ALOCAR+=>UH;XkX23oBS*_I@@W*
zbv_z5UmsEuIfU+e+xA<(<#bqlUeBy#V^cgk**|M|<r~P?Y&S1nX)*Lohnk70JLzeH
zZ(tnFifvC920;a<ScrkoB^T&gQ8sb7&Ut;f$;)GNx*pswbevq=e03HT>WKYkZ%D{#
zs2I;r+9V&{(75#;o1bsL@Ocky$K^s8>r=xoNVbOn8PuRt_Cc2ya{;(r8(Q1j0lt#E
z8Lw~QkLf{EQhl!6Y|&8g^+O{Y*~K#6kqdO8<_KS%m;lko+D8w&C2)l@>$jV9xGaYw
zwVYq-^C1!}0#2}%7es$HHHU}1&o*-LdBXa9uf*q4u6oL$`;lqSt<meN<7Suc?6;&m
z`Qoqd%y5{kQ|&8H73Suu=Da&@aEpHb-UzG}f4uAm$&!uj(ea39&ZvS+n$6C7Y&W{p
zC;W;K(Whl7g$0L&MQbG(X+Rc5s%&RjqIJ^EdJ`>2@?&u*1nJB94FPuNs~3Fe8GU9J
zEB&IQE0(F!ZQo3%ICqNJ5!R=!t({21W%=@25E9Sq_JKvuQp9_|mxtr)L(Zl=)g1TG
zpJSs%I_HOCrF0;pt$~Zvef3I9b;v|T=Q)<A!7Y59D(itvndptsxaHoYNoMbGb^D-G
z{rJrw`Mx{Ra+y230O!ZId)Pb3K220w5!DEq_mO4g<aqQSZj29UKLnRw#%PHdYRo3P
zE`)HL;)e2vACDRMKsu6QVyHQn{chcM2M63QHj&pT(#`_dKcRl-Tl-Y^>QK()wo<*z
z=0-Bs*nS5Lz%1>OrnCG;9@=AB2Zuc)E~_hr1dn>xUE`&trOtTwE1aD4kpggmR=HVK
z7iYiVCsNynUAuhM2Y=Xv<R^iad@cR{*3CP}&6vC%)AcFWYfb{Vm(zU`(^Qkr{`acF
z8DN5o%*xro%{rO@&Zjrma5{%d-f)<F`v+RFf!$ZM8LRB+g{$P-4QOi#xSymWs+P_V
z*I`$C-K^7jd3k7SNwrgMu#})-PC7Um>HvIelG2BwkR6=~9Z#vwo!zGNRQ?#T@a_C;
zy@V(iJcDbV>XyAZcDhtQ*-2<L+kCp5G-H-RhCgG^C-!jr^ujh-G`hOjeCpduAc3Nt
zfLPFlRO9yBA_kA8Sg?EllFu({<*Koc@Q2aY6<*t@xD6&85MCX>@$I*;`h|x9Zp>@0
z^RJ1f6n_+ml4chY>^KPUJq^uY0Oa!;(wzpKaob{0&8|uL()Q@^ZPxym=I>`qd=_NA
zCoGKzgRPd|Ff(gYKKe~!T9b2V45bo;JNjH-9tVwN=b5Y}Zu8wnuws@74POWD8;l{?
zlvf07%^BPtebCjx_WHzhyY=O`&E>dLz`Y>hu}OY|OS#4Du=rkjg7CJYsQC3she!jg
zm$V!buEl4BvmfR<6a;f!&5GM78S6EV&K=+Iv#p9}Z}vYXTaMecZ9Mz{`*b|whEa^g
z+VCT(W{i4AXtQEyqYIl(G<~`rI%%-jAI~lUygDA`v=wYNgQDO&P#is|F%ym_Br>B3
zB#8WRc|5z_DD1c_&xhN!X}&M#wBPOF$m>UVzIIBvA9j7gM<EF7K3mgzwEN{2?)O4Q
zcE|0}q9lcU$ix%K>%ejGS1Ui#UAO|YvWzt%;$v}_8??R}M+GNA>XMNp{2nEaV14Vr
zeE!qmny6;QlWnQD<h0lI4(Y}0^<`R*EOI$^e5ZxsR_E;X6;!~xoUyggdxxy=I$+Vw
z%0M9%pOd!y@_j9l*V(~)nJI{dM!dTjTA<dcVH5Nex18#W=XLU24706%%3L6`?eg)Y
zZS(lT>*nUuN&0mUVCY20VkFxvt3ia0ZKJx9@M$Kq&9Ze<!E$Q+`39=`h8IWGx1VFJ
z``hC+3EY3_gg8;R$0WnjH>wTP{DmPW+1TfWkn7IeMGmgp_G$Ympcuh(i7@bXP`x;n
z&*Q#Zjsoa(aZ*=9?M=spk#<+D6PwnMUoKUT_de*Ao4ti{iRLJ#G{A&xWpy6STmO35
zq2l^FFsI4QnF4Q;0}u*7VL>M9tJ8j;x*UZhvvaO}mnx~xdT|xvE#oeo_3mw*Lh~*t
zvNQ9Eu`H>vG)jWIfyWIn1q&u>!R9rd2kqpkBeWy=>gB(hR<o0)`ffq1wdReM4Vydw
zQi*qrzv<Q&xI7q|?yFLo>F(3eu`xfV_vjr*>GGJ_|5h&EQn8NOG^Oa<?bzupBIww7
zG-1^UyWIVT={?S#;N2^m(aM)xRNCOktlM9)z8GjH;P<X2PWIvg(0`NM5xfh*_}%vR
z5Rk<HHB^6SLr$R!!Ie-B^o6Cb5Q24Iw2d??@-?4T<MQ_d)=IM?|F-zUNVZ9wC7+#*
zhd&|J)5~7Z8*FEV`z_8%@{mK4Y@5d4&F)9emdC2dAY+#IM!PB7P%OfHYPON>z0++=
z1PbH?a~evn?mmxa=MCv@gm|FfziyTT?6TtHCcXl-h9x1}RQEN8ZD+cOU|5CMAu6@}
zTzW2~$^CH6bp4FvfboYbCS`Ig=L|bvftf75){cscOI5vbduY{`X`G(MjxnFGoAp~a
z`$_xQPj2}L=3G3W4T<aL<^QNdj0#a>;|wo)OYcVg!GT&wa~+#6zQ|~yuQ@>T)+1_$
zPf~HyywCEEAC;A>M*v%SJM0@+zVx2F8TPe48Z-s*XQn};mjC!wFuv(xuN#ZlRoL@z
z1bkEnTm)Fm@kkmnreoS3jE=olDjCu;1z0slN9Hr}AHyD&L`OD!aqGWLXHj|F^q_d%
zp25b{*F39J%RFVtqK~x%u-OjYb;{&>YeM)Z?cLTBMMG*S`yS{9jXaxgxdz0p0g^Z(
zjsRR3G17o<<93i6_Q&3|3-+Z$wzt4^>m7Z*Y_v-gr>pYutsOvh)Cse~<^E0+Wy6N|
z6p=q`9dn>^jC8m`3WFO7R`Z#osgSDVse4;5K0PDfGxwKEt$Ph7JK+Sqm~b&nB^_?|
z@exRE(cyC|PTEWr{KbX*_CneM8Ix{1q-cHHw=I;Aslnt7&=mW)dBewHM650z8We8J
zrd8D($eDl+|DfHRi=xPaEkF^CG~L<yLhWAPqdx{WXs#Uatkd%7ga4djF6_%g^JF7R
z!$gC~I49SmBcF}3jyw0UI(!48ohHsBf69dF{o@>L(GU3?GE)qKaL6~NdhVd^iwgLN
zw#Z0saqS3hu&j;<YWDlUPoX0G?!wD`4;r6L)lS~o+3BJWm>(C?xhVLwtU>1TX5D$M
zLOovYyX{2Nic!whbgV%z;RKNE3mr)8a8M_g@fuNeS7v6NuEY)Ed@Kb7tA)z**`K%Q
z<9yAN$jVVNYD_xbk=sO<d=*?F6Y=)+GoB}o{N5~X+!e!s_Kg%9IfT;t<SVIlJiEyo
z9zJKl{7i6)rd6=!D<_W}U&zfADL(^U-wX8(G{^cxpd`qCt=zDEygj>}0>{ryA={Zi
zMv&3jQK5tDoZs!94h~9MzIsf>c+qG&diz#v9x@FN;81x2FTY2&=1Uv>u|ttjL-9eN
z7cqBWH^ySulbs2x5;sb37TucC?=E|EU3#Bi%J^}l`UcAI>YlLGq6p$@C`5j&X}zGD
ziP`6x-&q;>1UX%Bc<y;JA2j2QMaO%DN_}>)s=8<=*gVL(3cfV?7GAcQWKePYY-dq;
zXEAc;048JO1yHtL0SDt3m=|QfRSTQgUmiOS@^D4LZ>JGBzDS(aybpE#rWs{=RR`$$
z9CF*CY=!-_qb8nHQNh_9KF4L({v@9CyU&vy`}kaRQ}~@3*|I--vlUPQ2ASEU^+3k!
z{@(bHDge4dgiJd9{s@w<s2dOri2+BR-9!-3P&JBmSC6buO(;f*gyo~Af4vjvJGZOY
zX*zgI!u5G9R5m$T<mU?&(H2299^6EoY8$<pb1D%4u$I$LW*R+k8B9<}vm4h&3Md^m
z>(-0Q$K>~ZH8eD!jKwkQEPH*_8I(8xbXi-K(V1qGr<))5#btVpZy*fW^YtCVsZ`y2
zfQ^qUa#2d2!{Au+>tHc%WxD?Okt)Y2wxA!f4?4B*$=og5dNac3^S?gm*p!X+9#`&!
z;zUT>%M{&;m1qecD74&hjc>qfqPUGiMqibSiJHKzu(#JTJG8*KbglnD@k~#byzlYN
z{ZftsBsI#Z=FK0nDAOWG=T7X49?p1dJ8w@`_g9qz26?slp?|Fn3i4~*^1&~<1qRJc
z9{s68N1sn$AMf{z7#J#DM4{kcpN%y)T>4y{!e(xlnuVnTb}-S_;Ja2%{r!T2N~<yV
z?uNZCrWx<;x^<Rqw|>&{s8-dn{U4)(J6yP5dRwjgD}hO9?dBQGtA_bpcRoU9Dom<^
zyZZW$oWNUS3n=)f(K;UARCqyU_P?G%1JJQIG)cLwSM*B_{h)%&eobz{wkSB?YIo+f
zWbi2|xAg(N-Dtf>a{(BN&fr4L0RbO?gz|Qs(Sp;3#ov7lu7L-&3Wn`HKqSDHS#T<{
zIFe0FpuZ~-Fxbx`%Lttp0At@7oTt(b@wZ<eE!uz3+0*0X10cFh^wdPpp+6SOjy}`s
z+taOtShnoG)toeal^4bT9lZ*C-SDQe<cPK<%=YmnP>BU`>w2y1akULx=h1BWqNl3O
z`%`C<(Kjb5_w|_=KxL3gZYUtr4W_-UR|3#2Yf4W#8XEGwb?^Q=As4?maX$-9N+;}R
zH@PVTSCR=$J}I@1_Tzkl)lygN^zzrnS`9!}E8Ir4%-trR%gES6Hz!8wRlcH{TsYq+
zie%gLh|=*`1}JLO)sw{2f7Lak`VEy{zQsg|%FOX<K$?<E3X5#k`@9J1ZAQ~o)<Lme
z{NmFaA@i!>J!@<=HPMzuU*5IcPYs8?vV{BN7ey}k@8X=|3DK8@<Lwj9Uv8P~FZVV=
z5{feA4KsJ{brIs5Jt3f#FEx2us}Zhe5oDhA4%S3G1h}wuz0gj_z8VjIv#d(kDKtJl
zenr2|$-<(&_9kB9>#GFBC_Ifb@c!8BU*_*@OjLGz{d&vuz*QMS5k6Y1H+ebb*k6lf
zJF;joD@fJ^w}oU%6v)ea!%vs0$D_XOm(Ffn8w?sbpu`8Ar3RIyjsc!ht$A;v`p100
zi$tM=8|KHzRXK_g6hc#Ggc9~0fJ+OzM61g)nP7MBY29hQ+_ZJMiz|F$Xg{*)_6u;d
zb5ZuWAzU_7KQDm*)d$UEI~Mmng8O}sj3&OPO-&qJ0(XRHyuvLvMPzQ#o}+n<QoyAU
zDkVGL1%yCc{k8)zjgws_ENsdhmNTOjKP#FgWwiv+K?8HrE=p*oiIfBY6|hGedNj<1
z3scB2BWPe;+YyXpg>+|&(s~Fl#i$mBU|J>+KFK0C8wu(pd@q7wv}{$C$pJ}Fzu!bg
zTM}kI5_-zhgMfX$&8WKBk6<Dyh|vLzVBhlTD2+nRN3gFY$wBLugP9@8w==VjmIQrc
z_q(n6Cf9O-D3$}oUYvH09rAu!T@p9s^JS7tvcYfx;MTtm>pWF&VrkeB7ho~>pyrz#
z`R?|E875Yn9QgfcJ5u(wV!Ru_O!ZPB&ij2PPd>32aT?F_6>C+~Nf|jg)pE1mMZxe7
z`%G@LjJIot&kom@team>PgGbuHt+tZB0Xn4YYIvE67B5ZkDoQVD~-IbXQ!e1eY{Gh
zuhoUGfI9So<>V8qPi2`=D>@$&@5b?%XHIe~zJo?k%tQmhM!B=R7ad*L<<;4qZQoPt
ztD;0<1wHd~W`$(lvYy!`mD{p62C!c|0T7>dtWL(ffy(<2=*c4KJyZHI0%gCT4kt4g
zSmSkQ#ZsA91}TY!d=4LzdCp2_8;JcZehdg+i35(bsYNLNjcUQT&mbzA@%(`zg8c~x
zU(LG7_dH|FI#u7t2W5epkHO(Q)saSzV_*3Nz_}X>zW3PLpzlb@#aTs^KjXz8jqMjt
z2yC&uIyK~Gi>^R;$(7m_8;o4Olra8m<K$Na?h0PY%L#7K{9-!BD-p!tV-h_0xZlKS
zbu*<WN~*6UVE=yp{CQ7rp83x$`-g4-BnAjx$wj%`SO`=aU#KIwJr0vl2fxaGCM(mt
z`&V2PkNy*IK8}_bh#{~)tNn}Go<JsGzMhcguL$i8C*ZF>hQmXX7}8w-KK5oOcsqK&
zzJc2sF_M2QF2soMKj!n}xB~=(ACrwB4Ey&FeSrdLnkJnW_<!%<uSvV;0X~M?sUg^V
ze?5m!KMPnECpJ<t|7FMjpE&;yhy!Ak*s36OFyD@GIFhUFOXFK$lEaK$k}YHaw=b*c
zc2UAk4gOkoP6=SRCn2Ipe0+rreYbbi`@v)E{?cXw?Xe$?gL$E2S6vv(w$OSMQ`URk
zSZ#*l2~3BPm4X_^X=<>4HL5M*44@<6s<h1rnSVpP{;YF-sxw=iU6X>(YA3JGI~ao(
z<j%d@wS?{&z{uK1Nb_pe==i^e@XoHQ{3U)F;(#NkHua1a0lr1~&>V2-p_koVddG!R
z38dZQnw=whori?8E6>__E|SVeRI{S690?J{IO7>G&+wEit^Xfe&sX#zssk<~bTe8J
zw7^Hl+E&c*RQZUpxQ#^7Z~?uYr>PxNKJ!E4Bg-4i0{Mzc0_nZr;K?=+!kyh|7M-b5
ziY?!G2F&7q&ErkXT&F_9I3I6L2sYm_8qT0cL=(wG02rRvn}q*yyhV}x9PZSVS23CI
zW_3+oN%86I*&fzRnA4{p>NwDh9ND&3+?c3<`D*kC$b~+!xfOtLgF(C~ZXVuheHZh{
z{RA&Y$Kx;XF*j$yx|K7g5Px?BR)Fxb0Yqsz_{R1GK^QKW4fiA$FTBea`*xv2SCIbL
zkAV-A#xL-%An~Jru{Q=7EuN<|i!dds49I+Gx=Zg+InsS!iD$QWOU1MN_M$`W#So|n
zA&TNF`W%3Uz?if?1ma;-`0DV<m4LU-h;B{4pPjsZm4C2YeR$Xom5WW(?%duS0wGcl
z6$~)au5lFsR{cKS0d`*CFedzPSom@v0rOe7yV}~g#@U4kPrhZ4`NhB!$X~m_fbRk?
ztr&3^nZ8<t@5>MBL|yJhR_)(@{7s-<J}dh<CbNOEXRFn%K}v*77$63AfRLkhB<LfM
z1|wf|<{fM7<vNJY&)G)#oY~e~3hl9VQn4$I%SGCPWs1fOOK!xC_#vS@0^H7fRIYUh
znJADl%2rQrSL%TN)&{xX8Wm{s0crS2IS<&FndT^-73|RHd!Y0Qf%YIgP2dvy9cDQr
zaMS<EoW{|VYHam>66jc`X-gLCnX7h00iS3{O0$D-onQ`P@(ali;~ub&M?9fM2$DV*
zCQ7~hc8802K0)M^S`fbh;TfJ5AOlqE-t5{4YlaeipULV;OOzY04`U@T1LN=z8wI6l
z>UB11JP+Y&vF5&e8}UNQ!d>ntoG}Cf27~;Pv&NX9mss~W7dX%!UnT2#wywwg*gFkQ
zV7i4sBqU_>U+UG4NC=pKgz}zwPEjYlaJ7gUswgR+DfksbKn(03<<9cHAfucr5HWXv
zN(Dz1$tha&kQ;W0cB1gf6~EYe6DP-VpBCIr9)99Tv5S3`9jL+jy0efN9Z?EnXaP0&
z^ilDR`2oUPM%mg(={ie)ACvT8`El$8VR$LQ+mhVOUt^uInw%T3UL*v06ZK_I`m0z0
zZx^MufKt6LRG`J1!W;Y91>a=gas++T>ZXwh6#XUw1YG}rD{}xwYH?0<c^SUY?0<~+
z6@$|VoW(7>R>oEc>zhL$ujm{ua+UQ$(vAP?D~gI$gUjL)rB#i&{t%TB;I)%Vzm!8(
z)sg4oU+~$Y1`sTMhWO3v8-n?=qVO?ot8V3=dohqiarogpi<fEVsRv_9PZLF*4lp3n
z>jbFYXn``xA_|cU%6Pd4gKhsoDQVn8Wj6nN*=aSFLa{$L>GnVT#jO@iU~%GEvNs4I
zC>ny|_OR=sDL*Do-MCCy_pSz8OCK|fOy}hh|ND>_W%*cPL~4JL1}=%P_=~$YBIsvF
z@O@H9VZ8YOv)Tv^Vk&tRLg$i_59pyqk#-LR!N$Yj1O^+qND$*79I5HfVzI|FzV54e
ztKE+=SZiIuo}9A0%MaUm>ey!=&LAArSF{EbD50qqXmLg0#?A|n`RZYp-nD;3)-8i4
zY5?ELPu`3%q%ZCJrBMt0g;xpiP))+SP=sqAr~&er5nb9mVXO!^DL+<5f82uyKums{
zB0+15zt~^H2LOo)b(w#HuzMrcNUp}%soQaHJd@LLo)IH~E#+tqb@3gr&=JG~38fug
zLNN1&33260(b0-C2XmV{tHiC3h>ehRI}c->THZuz|KLMd0VC!Ka7A8H79&ssuL>n$
z798zx^v2qA9zVPk2~$0G%2w(7lNI6{z96H@0#@;hx_dMT)WVSS>#-%EhT8a*W!<IU
zVW+2bxl)o<z?22ILl;89d!2mk2CzGb%Ll@%TfW;O-$KxlAFYA))g$*f?x^8NiBkPa
zLXp7e4XU|MlNLjf*tGhW#CrnxG*3YA8gUaMeuWH`OMNr<yl-KB4b}~*j2VV{V7O<?
zAxg`RI$(^KsKg*x$e<q7&)(LI(6dA~QuSAFe&zp=lxYM9Wjnh61#15zK@L<`D1p{t
zHV)h8G#BU({O{G~JE;$rl}r5K)I%VvuWt#!Vb9nMG6Vt`(MC2J*6dYz;+cKA(`&cz
zjySu`W&8hfP4jlWF1<^;6YM9bk3((UbMO;I?)#^%{$-hY0xsBm!Tlvf0&)6H^?r&i
z-^uvLIir6ZiG-Fwjb8ERfd8V93_u|z`TOEGo&)Ftb;^P%=5T0Wo~INqoIY>Np_Y*w
zSFGO&0ilslngAzeM?KU5Q5@4nM~B@V%psVq@6l|Z7iT_>88&y^;m@Q)T#`N-?u?ba
zz`T(&za6~qsXnQDRo-M20hj~y3u~!4ZAX-Ea-oR&7zx<`?#(bt@gYptq71U#CpKjb
z-<-k>fk-nI&wOWV5V(slIWp=VaC5$wd_@EjDg}^v8b{9P`#p}wkj1Nqwz{9A&aI>I
z^G|Yww8d_Sf#l~42Ml6nY((`!=C?76&SX1zGUxQbtg&r`PbY+27`yxhI~l^x@aSa$
zv46psj3|BnsCzE*%0~^HRIl2uNG*+XSnq?^ovyi_%OTJagvSP;9P+0uBZTSvY34LW
z4@Mm*bx8_wS~DpBi4q&(?Sv108Q~$0R{jhSILfOg^hQ4brt+eVrygHeNqBJCnN=ia
zk%6k`CDX+S+2&vH<O>iOl{kJG0k1JaeWQo@t=#2mhsIE|>Xo4j`9IpU$ma_&?4z_D
z{~G-gZjXnbx6vaEr&kBru6J~$`sn8^Wu%K5pGVvWL!UpDf6a$56ps_QG{(ysoCsls
z=Xmjaj@dvZM|NL|K!I2QdXUiK#yB0lZ9i_19%~e1o4SC0$aJ>UVB|w4gu8~p3)jgc
zh?74<0ZL;k;iW+W89;GGfm|F5H}AX~WH3`E<dZX{W@cepy{y|qnh#+=!|WMk`HzTd
z87o76GO*bpklz@963N0Dx8=8gsEYRiWp#BmABaP8aD?K}$6J{l2GWTd%}_cIzi_D4
zojF}B_R;5)v)%E2m(7!8C6wtk!U`S<Wyn`yAHQnvxa2y1mB{3XAZ(xsz><}mBsUel
zVdp(oyDo;D6(D!HYe7+IITEF%ZiGT`8nNM)!(iLdc9BDM<)vHXcg8+X(<lN-hqBP(
zsIv2QVk%$UOWtWnG<;3>Y9P~wOZ~a*&geWgBB27|ZrH0SM5y&MMuX=!ryaVP+x3ri
z*(P}s3?;)JFbn2fx%bXw6^;un3D<0tINrPHi**l=s#Ay)NUx)~T#+uCZW_xOAI{Yk
zJ5HGecZTc=$L8DwM*)hVDgukmSzA23x%9|sXgv=URI}v=+Cm7pyf0jVO0(j&I&yxe
zQqy&y2*sKYdd6tlzxD_^T4mfV*@6aEA*XRkdjx)M=2Bx-wK-ryc}-nCskgRMBGi~P
z!IP(B$#(~V@4ljQaA}^2r$B&3dKJUNwUINR!USSyw71yFar)Sk4~X!Y2Ru;ES7WHG
z@z{Q1+AE~hHEv)zUiu_bSxY9RIkv#^7^Rhz$2mRwiLUCL?mTWvB6)hjZGOxq#Hghx
z=n25Tz;=is!gVYIMaITvSqx>n1>gaKcc?f?oyF}2LdPM?9a0offLHp>xWUM`o%@s0
zC4wvuRt98cW%rU94@Vkpt3JbUVgx+Q^GsXwe3{C!@dcD(3S_G#-o$=LDDs$1%J8yC
z2-4-Giy|O6_=4u8&dindeLWd5^5i!MuNZFP6S`4RZIQm??w!`HFEzwHfnr5|AiV)S
z0&-GAzR0(%krswkmDH!}d?3tEZKinN60^7Hj5gHHzac%?_PN%vx3@>@JKy{2WmonY
zb`U4v7|q%)p+Rd~$;m_a25)u|gT^I+8f>|4+_AU6O?o(*=_aMG6dW-$-cdO$vzf%N
zuefZ5gK(+vFlqs)ikwA3uypVgP-siOvNm6207?C+t4knZi<WKfAQDfAw_G9}K@l3k
z%0IpbMC#RY6v=H<13M@@<g2G$bo{DtjspZt0vsOPRh>y)4btIywY0C%!32vt3l&)t
zq&ebM$JBf0?2jKCwt2m}%UNM<@->z+k0FFV{yk*<X|B4|<MS%;P~Au<BmnE?{$YUd
z$N(_@YM)K?XXAeK51z}(4FUCB2kk@#KbwJ=sjIn8t%K>|_UtS=sK-TKn!+vfSlhxD
z$o$a~{BQ-b0!~13F-~KudVR}!>B)5t;2>}6oPy#)3!GCv3v+koU+bdbr+p0M-i-ax
zSiO+ME%NfIjtk|F07`jPMVr*d2QX`|!~T8uby4>Z-xTydZk`XmtZH2w$}a2M8c-pO
z!pE@)AoxG+n&4-R{U>bU_*gtMrtI+WyW4k3d29-SGOwYl!_CR#(mf$pQu`Ufz3-3L
zhV!C(nciPFh|It?FSG$?ELOlPOC*sAN?LqJnjw88EVRHabTG|J(rxnU&YCMpM1kHI
zru>15c+QrCdhQBe(Jp74<w7#y(Y_3MqUEb??jtW3#bVlv&$_ccI$2W<g6awiRbzGB
z$3parn}IxtJ6E;==)FXnd4m*wn)l*4&EG+pvTGe-?l}+#ey8o3Y0|(~W5p@Q+M3Os
z-O7Dy(y{f+_6#K}i$9PleRMxZvo3JcJZ+rTy1y%pvR23{{NgA)uN|5apBw0s60=g6
z7;N=Zl6J>#wkFRdw(U}pYW+T4y7QIEQ1K-Rr>UQY&5Ws1wS}6_<Fg9YB1RuHuDv*z
z5F&h{!r{|U*W$)f1>He)aCGE?Ogw!{DKc}FfI5=;n2T$KHj|s_k)p_@Q_}WVN9I9&
zg^cy7N+kwpYk|s^flT!+{)iK`cAgs9-M6+Mm}=)jUr*6s#jr0sr`0J@aLZzD3Rr!e
zfN+Bb^2YB;jdCW5URJoK2q!d2S|p5HYS!m(k+^%lH|6`wHuHlA5s1OsHzsYcvfsgh
zlh@_p7&nqpmB~cqdW6A9(&})Dfq{97^9Mc5ff54|8ag`Zl_y~oRRSJCd)y7#fB^~M
zeZ`9SlHj^5B=5wXh_U!<tOLy`<7Y0Y*_%O%vz&6vUl9#>QV$%W{OE9nXKiNH77vXT
zx0L#arA3%Ewl&@kXL$GiT?-BL55Qw)c=;7Kd&YZ#Lc3y!uaXptM@wuJrRMSO1GA3h
z&fHi2^Nn)TDETDR&N922<K7orqqs5ER-dQNVKQ6z{<moSY8Dprr)2E86xBSZjU3ch
zRa7jtD-Xb_8Z&2HGySKtHBqWDo?lG?2#CMPar-OeV+3zw=&{9Xy)n=*!oR%1CIw&+
zC|9hU<XBo<{P|A)%h$yKHe?KfVBY2U70;DM6NAK2>&eGy_=2zv^_kSPWOV|$<YrTc
z9OgJJh4iUgDt%WXHp=3r3{6pbhjaO^6}ovHygo(u!9{T~%7>-4#-r5)F_)KPLBo#f
zdD$^xB^p!r(5l_|GeL9+no1P30oNwe%toR&x5E!#v)-jh_;iuL>teSf5ipRZY@)S9
zWu2m^k2fdB0POhyV9&#8-1;0=trA>s%}fG;rz2V&Z_K~YKo1>E8+JYYO2l=i-DN4f
zLmZZ`3NxdXpeW>M(O@F_p4)!_4!C+40&qtgDo1>y#(hQ5;6(}8;ytNClao&Ep@dv^
zbE2eyY%+NT>IKV<4~lGYUcR*GIo{{89eE%iMG($gv_2q~w&jz|o;;YBu%=O<<~iFJ
z8XJH+O>1gC$+6q3{S%VNA>s<IZD!RFukVwka4Aa`stC0WK=g7!cp89B*RtlqMtrkF
zAB>Bu^KBQxKrKvm+tldmx&BWoS+X2ogZBFb`q3ZDD^qE_du>&{I^Ef9VW6E2?SdUz
z7DWqi?|b8nCUpp3y&!ej8mkr3$+NAeWn{^W-0hR=(rZ$~B(GyUox)0HPG5+vc8=!I
z_t4l1ccP*2$%s-E5}0e%Z#GFI8O_Jv@m@i65?;aENH(GIr+;$p>{O*dRjn<a@mGJe
z87&}nF9okneRKC=hN_Z~I#6{!V^y|8%R~pJyspqxB`OwrSqF!jw5qqbe3UB$N)c)4
zrlimb8*Q$_q8D_|%9X<s+j2)0wvf3;dDjooG{hH+&-5N{H*74Hepeq%kce@e!bB$B
z+gKt!T+Vb`8~>b+{V_9}8m*9(<fiyX6f^;tM3U#^X5PPLO7uQfp=O!*cJp=9B~YXO
z9-0l9USl7{0<7_jfr>L(+0fpg@3a<&I(@ANMIJcrkiF*GlQx{S@qD8zRWY~gmOw4-
zQnRG$lF-f&l7XC}I&!^PkG>}MkZ)m~1n>9Ptxdh3PM|ZYJk!lS8QsJhJsisE+y}2m
zugb+lC*{a0vKa@L=VwZHqkCJXm?4ONFf$;3WzNkBf3kl-Yz-)V7z(2FeieY{xZJbb
zmjk4F;Q#@_TYzV)L-HGmOg0o!qYFI%&2wgawB6Nx{@5P+0yPeIWdjx2KkdtTWeMz6
zA8@3*c?Fyw?$K4KTFNdw8VlurFSJBzs&Llwk_P;`yBMt7N-IUkbX*Hj6gA%%M$@h{
z3W5|!T_%!@SL=21v2{%|VIx?YcwT^fW^AW&JpL`8pf%Jg>+y^T;j{1PkIZ`FA64&v
znI<z6pd|<-7WS+;JKo;Dh!=jew(MO)FyY81QR2-dT~M{bQn@yZ?xNmdCLN>QH;XoN
zbp-C!txErH);p>|#K*)l@ibp=@vw!Cv!l5rd3<OoUsHwwd@W(m<y8Q#AvX}dZm|9&
z>`>J_k(Qblc*2`9QCo}dVqr6Tu9C*_o8f>l-c|=#U7FYEY(9VXE05-~HI)ai2tt39
zt(dZQF(}gi?kN|Sx_Pqw8*K5rKxYe3k!Hjn@Z)Ybs+Ac(Zr?SJRf5Z=601EPPqJHK
zKkk*B^$Htn;t2Xs=y!|v%?VAAq?IZCqL>)GwCjWP?UZe)Z}DTX_15ooxh(TqMtC}7
zwO4;Qc=JbpNPtw^omq79)3MQ-$4cgxPpb)3S}S)e3H`W>pl}D60JL`BX6F2Z)>Mjf
zKx51q(2VoctcO@3@u>_SZIG{H{Ko)HxQ#<A<_tj8V;h^V6n0h7`*R+2*M8%5or03A
zYc+0mxRt!0&0fTzxc3!uoE(j*XB`bR=wnYix!|^U)XK6X#bzDwq^iz*b9R}`z7wYh
zv;KAeXTu{V?(M;!3|JKwW8MN)zusAV+;^|8a-)@{cmhW7d4u)fvg@&(@Fxg|_W-4~
zwwC*Z&1FN=>ufcLLk!a~elxCzTfX%T_7HH2%_Dgu_BjmFK+I?H9|M6xIgr%VucJqe
zBbhdp7difWSLJDOtK9}4h$iE-2<;s@%=okRU&zWFKuB4R{VrvFmMj>6UKTL#DfQYg
zQ<ayG-goCcM^8@MELvX25NNH_HJX=cRq_Ht01nA71Hgj&n=M+ug#d`*W5J=lIM5X+
zOa~fyxF?gk-Et?(&6V_kVto>Rr(F3A%XfR24S=JmEpSQ<03j6t2HPZ`n}gky&;=@2
zu2z;Oc}#j0W@LC;?LlPX(N(o&uGq9r%<5O~8623$qGu3Fg7iY9m;jy2hFm!l4cu!R
zHbtzn^!<_sci<Ust`{m$7!^wBzTJ3i4KhC6YxaILo3G9Us2#sxT7^VZh~U7?z#z83
z!FIn{zPL&TMeT~Dy1e4>o0Ri|)$lCs8kZ!BhL34$ynJ~6UY^I3mR={nGEk=Z?NQCG
zvUvrvNn)Aq*L?d+_XXns{2Q>ROhEfBe?YWUz-6N}B(L{_Jg0fz!Mk#xREL|c?fd)3
zBS3NPw%g2!^q0=+LTTX(vGuC$5k4X9@y&?6>IUmyiuIN)s0{vI7p1RKzR4%p)<bmb
z^P?N;D!p|Z6wAG5>7&EuqoM`SJ)R!~{Ru4qfnPD84f7uA%ep@-Ggbfz2<QMkL|Jf$
zQ6Zj9Y`L6|5nNmdlpA&wzu{fYjJ^-Mph|trhEg79J4`y;qk1UI;4&hX5gqqP_gz{F
z<WtYl3v)_ncwJ>KYxHM#*wxc%PM1)J{XHj`TSq}ULGIk2#Q-GsTY;y<^p2y)|M21w
zVLXp*Dz<n$Bmw9;a*SUb&PjF%h1c0&lRSM+$XH6Q^wC!3U3A0Q%yIV5DRiQ=pJGBO
z4R`0AHsrc1CZ%cGi0(f3x)?H6ICG8#G+S0PE<a21n;uOB;1~UZ3kbl4@AE(IjOY_A
z>*+dm06oh|;ObOA)MayG6=+iWFdmg>qZf7QZsfo7E&9{^cam7=i0GOL>acf#kC{1j
zpzl(j?7O!nbmYH{4i_Mau)rEy%XQ~cYc^YYl}zD?5cB*9Pu#T++9CgOIzICH_4LSl
z0VRao7`umK226Bx9S@3t2BBx+#OXi*GEgwA*sc!qL_*{E?64tmdF!E8-O+)K<t+7Q
ze=)Pyfm#ek6r;03PaOOinq~m;JX$rPs}35^$@c;7?k62An%4s^+D&}{i1E`HDZr)B
z8qN*)EeCD6UrL&<@_A-lCCucqf&L4f*KKW*Dc)zR+!*!~7CD`s%NQwNP7>s}?ECJJ
zd%eV-=BP1WL09>f+Uuf98yDEgaPQruLae>^j3uq%%dLU?=COg3n;N}cgj^X)I+YOJ
z4xKlMzL#VHpcq~?miJ)&ro&G3msBAScr1W4I9>o+Ts;Cr<0hFCdO|VEIs6|Vpm^Yz
zt!%N+G69y%HQAt!!ny|5<;N~If30TQ;iTy0JZ;NR3X*Z9EBWu6Gu2=|M>EE<Sq4+$
zgzNmm@hWMP>?c3YHa#Gp^b*I0e#}5Nk}t=wnEx12fb*>nH~rSV*+_n)6cA+TVXX#w
zAo6);Jyj`vF3LDKI5ui5M++?+#dTfAO*#0ecw6KQ503W_e=Rt#d8!HNIa|i<kW3kc
zf7^`l8Xg~gRkUAvwTGNCvbpjY_mc~z)c#(~)yCtl$ub2QI&8ItkXv%}WG8ZNea+=?
zTg6A@y0~Y9EQ^}v)s8*$NDkyzaOe3wR~q-5+Y(F*M+_&P1gzhPp{(eqki@+@4V@hU
z`bM${nKXWm%8M_;dkASU?15fCCO*E}<VIy7Je1oh$@i{3j|G-^ai_OMnoHKy<gg`q
z#H<_-M}=InI@k}b&MNz-vch86kDNR0WrfWH@$1N7V@KVCKeukJlJG#@d+Ri|YuKvf
zBtAQ`DtS|xACtwm_UF<9M67}n@o1Ed?GF)oQD`3>24WIySOx>378O7_Rs8}ypS37C
zmVvA|{rqolNpG989jLJQrJN5{dJZX)L*)Y;DhW1Y9nCItd-{cA78YV021Y_f>P$7|
z$ALx&!xmKfg=EpWNc|#bm7r`Rc5ZZpoUpG5;LMyzyyyOX)BP~+YgXH&==)qclLr56
za4Sy{Q&EWm+DTu6CGHZleR0z#CGz7m`Hm685_#AQ>q_O9VKHx4>LT<3>o%Tcf$xZ8
zmODp}w^;1?kJK$~Q1IvLzXcu05+Rs5sPM)EyW~~0-@=TsbUokl`+osoA(x+hE%bK<
zifFm)7bMNc5(Nc{fgZj|P@5Eyo}~X;X^rY#syP(Jt2aFn&zNA<28;PB>V(C0GR2l7
zw`Ph;$Tx%|)7N5EJed^{6B7S3pB<z0W$_I>2DC8?L>eI`v_x%@k0hK%)WNGjVH~T$
z?crx2e6(aFy>9}n4A2pW_L01tn1U-7u<YyS$(SaH<~tQUK->l~j{JShFY{ROBe=7k
z`)~EIcTZty9*z<fuLi`y`fgdz-jZY`a$P2|o$N@<V55pN&GkJ8f3xbRw_P`D=MmCo
zh|_)C-Vx-Ye1K5dDjEZ%b%F;;yDR?1D3A*pyqj`gJpKZ&09;aq7ZMUeBIsJGhYgPy
z00ot`K(EJ6kgvzKO;+<>#nkx`W4ad+4Q}J}J`sGxB&+C`emJh*m392RchOu_pk+9Z
zNZ(bA((B+oQ0%wCmdIt5%~V_?9;fe~2Lz!A^3w-cQGg6-wc95SeV=E=!(FnqXOu@=
z8@K;T;RnMLCih%giCKQ%zeoen<Hr)ezP&B?3kIS_6LPzKE}U=|thnjS5}TOKcLeYB
zt80WPCQ7Xpn4wr>GZ^bxk7k{iBiLqihhQS#%+$d!#xATj@HZBN@Qi^q9yK95{Rtw(
zNPV$xKLBFMlSP0jJYkxYt5;veQoq3gl;!%UetPz{Yw(FPo+zjV73hxMdMF?@c|@V+
zVBFBRGAd37eU)9qTa_GIs?%xr+jjJehbw)8>I9j8#cM^MV#ME$pp~f{|FGK4+8iUJ
z4SBaUUB3?WLyVLfwZ<eTlVPix;$YlonsX_>`0V)!C)2#Z;GDMoLs%Bp`5?#Pd_>q&
zk5gKKP4vqAS1)IeEVZk(pRlvt<N*CJ;7cO&ggpAaq&3zC&^QclAJ=zZrC~N`nljGJ
z%>0QIh$-o`KFVC>XKZ~md7N#r{26;~HZMU1p8$Ac5J2(6A3-AY43G42-)-br<}t9F
zD^CRy6-^e~s}j9BZ+cp<K8Xn-v9OEK$dA15F&n)Q&ijXwAUth=xao&E(BBC4QPU5=
z%#~VLYWdgTTeoq3Isgs8RXQ9AMF~LDuDZ1zi)6`{b%Xv9RwnN85#-hgThc}XW+OAe
zlj>NOUm2-;&$2ew6c|+oWe}hP6L@PC+b$QW6qkqQ%eKG77HE1~$1TEf#1_x~TVQ-e
zf5J0;^*SxSzeRr(jaDhO5D=PYK&|+WO}Epa4BaXakFM^_|Fy7SoI-B~Yc`vt5IWk8
zkhQ~)6S?17a7(@^@O&mIUocw4=meAh^RuP+OU3&<y9)J3RvR&B+GVE^WSkI^k<nn`
zO|=5q_Jw>qir?@Am|XwkAADqCy?ps{rSW)XsJvge;B+50dB;pA8;hkz6fg?lt-@gU
z$1%a~emURze{7RN*M|FgQ;J-*pIv&&CC;6TJQ+UTAIpXO;;*|3-=ZOWJ8nfg`%Niq
zY!&W*!yT_`_mdWa6yFD8z44bSZ1%sY4{8JMs<`K^mEW5))-eQn{<(SDm`WNlHYclf
z!hxQ*)kLe}ff2AlZ<T!6<sm)WIXvKU%4){IMI*DE?obFr<BfcuB_OS&@l500mHnu_
z<+v|%&mB*rJ+Gf>saX-;+L_CTmMekWZ}<PM^8E@gDb}@h|KsWvewnIu+^mwP^xW}-
z<3ug+&j%((@BrsY)!3Ly1;Z6>s&7;h*PBg&6uLF$)~K_-h|}9JW_;S)#>e#3ds&4m
zUr!x+S4h7YWVi^KENrMOi2YGyj_3Xf>-Uj?a88CCpfH@KmCSw{U`9_2DH!2x7VdkU
zbf$Zf@sDRZ(b!yf3`v85ei$fmL~8nL6~nW^L-*tQS+#NpDIq&CgQ}_1eY1GfK4`?H
zem(BXmBFhNC+n~?<VJ<9gQr;{hrUw}S7j-LWicTFXcZzPi2b4ep8vyhK`}#X`&njk
z2&82!1~Q~vJb^6L0MPqrMDV}p`tEov+c)mqMppI;*|M?<QO2FUR|wgX>=D^3LPms;
zy~&onvS*Z$m25Jz_k7RGsOR^-pZBll!;|~IuH!uB_xK*ise0eCi-=j@p}+A3e5BH;
z$|cBYA%P;oZ&jbmKGApcNnyro{M<$2bv?P=knj)OHZ>8UNf9#ns;=RwuA~hA<x+sU
znyR7vFPw!M3Z;C{Y(@C@e!ct&BJ-bg(0zTy-TdK+4ikRFBs_`Bt6)C=hllYUB@yGN
zDG$9%Sd~l+(KDBy&-IE>9w*&+xVQK#A&)ER@gyRWMxsaV%E$iMY(*ueCnq8{rOUqf
zXoml7dYXeW)aOs0zTRbA!1)A&mLwcdk2}uXxW#K-FP{py{#A34PetARC2Mn%Z#rl`
zFO&Oau5%{1pk8XK{*Vh1ZA%C8H1mRI8Urk<J+$(TYZVpOjKi&8Q{7yv{q(D4Yc5+y
z<UDwH_7dv-Dq@!_%~rwOdiZ_)Dj`8YqLBl~pEkzXdNrx2&!UH?8heZbMJHQZ+ZEpH
zm?V0LxF<(@*rPW03fBpv*T34Htj!zo;ON8f$XhU=^3wj{p`_}kXf0Q251o#0YKO7k
zS*%V-TL!GM(4!_8>FE!O{X$DiUybTOZsS3|5#kLSwfM9Q(Y6|46zR6B*mw=51013b
z`uN{8&Iq3aEt}G1g%bcB9Xz4Y(OFYXexq>FquoDW2N_FotdVTvj&b{m@&<Y(2349g
zdv`F7UsN=#mUwqM&Y8^p`IWzKsL5h9`@uHSSoHWu2Ed2+GT1^pbd@{M#4#5KGzH=&
zk^XlG9Z-LC9xYuGAK<d+(QeB^M=SRP4i{2EoQ~w?;G>qc^am>flx{zP{#3Ni6HWYC
z&7oB#Hie>ad4jZ{i=&FH^Js!M=`s?1q~);we1D3&T=2@MynT*pK)?Yf+F$GQ>H*{d
z)ZmA;S7FEpJvuKMjD86SobJzP);Jc$i-GYYbDU`q`&2;$ptXCBNrr&b$sAhlo>%PF
z|1zvN_;FuHCJR(wSTL_Foj_pNp-IthrZF?MpnT>l%X}~Gn9)7``|2qFiR!yhZhT6_
z3BP<4)ay{Eq!z3W;F0iw;w?ypM=d@=VO=##k(%7Mg9L<hQCOu65u@P>na6B+#U`b)
ztE<2G^qHlc@{U3Vd#{4bV21{I#)AQ6`$vhdOh=L;&mN|V{d?CeBH-T!%A-0iD;+QZ
z2-Z)e66?AuffXFNV9$YNtl3yqsb%#a9?*joMnglJ`|}!%2jb9!?<HpXGSaIQ{;O);
zLZi|ts#R7a351tXnYLRqbhWQGWmNLveanq%{~h|&1e)E;7I)bX@**JyKE)4e^@Gg>
zGC#|$5(jv9hYjtI{*;^bBs~J;1x~i+ZCp4N5wh*pufyl#(sKJom2y??qm{KkN-Rs&
z)AYXm(^MmnK`5bcWnZc)d>b8<>jRcwq0DF*U|X+JfW?%1`MaMSimf$&ya^(%H3{tD
zZo0|F;hzV|ax+a?V*eq2s6$x#^2D{@SJ{D@9vHNAo_@=)4UWEJHjtz3qjAC~XtT&z
zm3?vI0p(x8UPV&IYEZ|19#z7>{6z{}5ByVS&@!=TTmof%H6et#Sx}{b8CWF_k&OT0
z*?|{>J3`7Ba#YWFl?t@OK=M178X}ktMn5G+Mn(bx1N*^lip=WODjAE>vKyN(7O+5P
zi{6@p_Ni{ScJk@1*iX_evv7Nn>VJ|#rJD2xW5kr9j91k6UfM!~7IXnH08e`Fc@S_i
z1eBBo&>$l;VG8Zr5x_UPjZ41{+Ucrw&y4`WMxWAp3(ebYCe|sY);0C8#3C2h?Wb5{
z=uO(cnankq5(Ji-TCw6^Mo_O~*mJRFuu3SaH&p^S4PoKe;_t{!5OKS~u$^wi9x9Tm
zo|R_CDye`5U!{|?^)sOXz*5ONk#I4Fy3i*x8PaTvU$&02EK8+(1?LLmCXTtEs)B$=
z=7{kX;V$Ff6YxrcB!_-4{z@6LkOm8M3;ZaYC83ORET#jA1}aP*u2zVE`uN5`;Q}#2
zL4EYv{IXxm@{fZo=c=V2Y~x!s8;SJ*Xyvl#)wDN%<#WnYm~%$TCL&X5rY5cNF~F*~
z3bT~9xbt*d7J{(uetx+ECJXJ%lGA+l+v6}(|J6*4Q9u=JH<Yf3CylD-+5}c*Y=caW
z1uW9B?0%*u3MTag0ED1XffZ}h5)E(AUE8yFuT3O1gf)Cz6)Z+K>e}nHqwt?_mRH~|
zctXp2i+G9NYWa40-FBLm{I{5huU3j$C*p6UMdu~({Yivge#*F`&Qd;V@kBakp<m5{
zZ0PMbw0kW7#AE<ruQE-@m%s5B^#bgdB{0|64<%TjHy@kpyuUtGwKwiEkRlc+8HZ>1
z&cG{*;WS0}i3N4tBb?O@;$n}X`C@m7vatP~y4JhLuvhz;`&~7@mmZNh%0{Xchqo)U
zhwfDA5hQ3=TNH>Zd@R@*-TOK6QT0*NJF8TXp)2t|&G_dD6(fM8QdD@aP)J;cLc=yJ
zfsJN-Xyp7uWaRCruj$kxz6)QDRv#}978P?#H;+}2P^@=$S8q3aWU}ya>R<=+E2%Qy
zZYva@%uM(gznmKueWzG+y7O^Wmduv3*IG#y#pY(jM&gr*YS&NGrUOa=XwoYF@io*=
z-_YM)Vl9?Wa?|MO7glC8vkSuQejQI({G*I`EkK@gY%gl$!gJ#6z+CoL2LP(_2};w!
zta0JiEl{1V)}6V7IbXJ=`R_=yPbS|gxc7CkRvYd13l%@iad>UR;Gte3Zy2Y$FqE&U
zv=_3FxnI%4f6c-8IrGep@rQCJ?`g98k2}+cq|x3sP@%>OzM^dJ0oim3i%(Zl%#TkQ
z@7+c}>Ho>rzbB4zWy!#{JwV7ly`O&N=uuyLj=WoZL#)#S`3>#rS0fbPd1o*|?g)}u
z9C0`?YJ_&ZdH=IAvurJD4Q$=v?;7;)6?a>+7K=R(>`xJAGEBEt5ntsb;g9;U71g>-
z=%m*jmi%&g#gc3xL62Q0`V9{uaUPEq+2d)g*h~g%A3%cwUjFuQE>E=xWz+oK#{Vz3
zL5D`p78<UIn5ZwkK<QWu#*Ll9czy|(Vt)7&xaSe1!M0ltTG1g1Q`A%+{igRGVz<Z%
zFK@*+e{mb-@uK&KZ4H?Z1mZu>QI-y)s9G0QwJyU%A9!DGQG8d&O&sc!?k;7f^`rQi
zGE)B;q9eQil=ov^9E3{g<dOQ?@!S1_8gO`lw7iPbul9Vcr+SV$`5#UBQ7)}kuMKX_
zeYN<iU1e9~<&S`fvR}jyBTQ8YOR(?;=-7bU{N9a`vPWL=MvmpDKQixaA7#Xo_T`iw
zI+zpnHT=%k$qy>mjkJ9kVRy(Uy)tq?Z3fL~{`p)d?f?tZ?%-P=<@hguOYTV^8Pxpy
zwO66|CeoL~C6+Opwp%d(>&|lnJDC<5u2Yss<w(8OL{ZH4k)>j|uiy}LD(`~r%UfjI
zj?|bsV?VlQdh*T<v|S&m(<y&D_(Ke}J*VjsJwD<6v_r*a6#p+4mI#c0|7OJR%j2o{
zfc0l84E-se$Ok;J5pgoE7fDB})d~PW0C`2t*@F~%q}Pmkcp9L+=PkZXz37ZH()1-~
z#bkncdHHNaPju7&YomJ&CeBs(1BK*g;D#9kNdfwBp4U?)K`Yw;6rvz>@?-aLjvDLH
z0Vv*-0M`tDlmV4}R4TCN>7t+qudJn&<qNO-PtZbw1`gm(CRa0+SQ2`pWfoOur$;+b
zK&C6mSNRlXS+#h3%yp|fTRFXRic9QYQ|aoV{%_`f5o2g%WJYG9yC$@QW_NAEs@$*&
z>?*OaPO%)TDw1rBJ3k_69Q8SKHy(g*-w;a9GgZr16qaK92N3%}WMb<ovY-o)&x-`V
zy}Nth;aW{9*g+-%O$8TIg2itX9>+GN<|B7oxuVYzBEZsCaDSlv_5L5_e|XIA6}!33
z(g*)xQ47<7Fe<ukcTo_)b1_`OLc7Ddvm+r^&60Lyz(Uf7Rt?xSqA@f-xUqF!$}x)R
z0ltCYvCaGE>}RymV@b~mwR;NUV+Qhcq~+NDv50+?fm9iH7C{aR4#|sz0HQcq*6yge
zf#%z*-R>Cfe%cWJ6tG;cB?;y6c|c<fsZX&V<+facSxiJh-WdNUSKlN>o(eTnGJm)7
zzZo_vI7n(?NHWEjr>Lk{Z{A@yAMdaP@aPXv2Ox&q9;|SK3V0=DW#xuO*NbLIy@c0F
zoEDVqefd}}UFqZmvH%SzTI6?!Y!@L1{1ZG9R4W&zEnW-~d65uGDkuyAYC}52$GeUV
za4y@=mNgsVkRfGjmYN1-b6A`QI}Z%=@lss~_z&o=Ht+w3g@}1Lh`IbzJ0k*+Rf_Xw
zU)kj%4B{w288L{I--ra~<6fF4pcc#FLp2M|J6KNH>R{C$mw|GNaBK1eTcXqKil6p`
zPs(-F-?9Jjwta}WZ?f^=#Gw`MR#*G0Fdc@4-avU3Q*@RMVK=LVzkB;QfZ+BiE|V9}
z31q-9NUu9PS&IQ{P%NrJ&&T=qM`KC#KMp3Om`}V`e(`r}S89xf2==X-QRdG)(?UiD
zJ@KLVREANPS7-iTvbDKu+%*!b?cU#Qt-1Z=gK4()j@&nvoG$e7hC0DN3g7zxO-h-b
z0f)21A-0Zf9i)&~7MlhkFr5nX7ZMSTf{00WP=XHj6Y?lD<4TYx11f|%<RrATwCv;}
zN2nBE*Wce#TLRe5k3S2CRj%+%JZW2|Ur6Ke$FZQ^v)aPevXP!&C)JUEcX!<R(K+y`
z<Cd;vld9FhG{xP@K^CmRIfY)#`H#m=6$yM*^X8cG>cw;HOV!W51v}^%R&>EeJfJ#u
z{c`t5CPl&Oz2hGk-BdU?VStym`(*d`PV1w#Z2P94?>Vi-O^|oVyLz+@zeGtz9D`hC
z0MrnLVK$kWOfv%>ELKNqE%&WHMFi%DP0uu$24SN7l>)T;5hO|sO-`3`S(lmw!M?Zq
z{zNW_?>{#IGEnOE7L;g5Dv83VqOqPDH}%ydOH9>eJSgq7O61sY|Go#{AGx?<VF^kN
zk#fw{o3CEJS{qN_ev%mJawbS`x0*luQo?tP<mNx3(>0hFfC3q%-&{e}irMvH^fGpU
zJ+}<Xj$$LQ{~~uj!9~8-d55`_4H?H@&1>v>pMCk(@?l&J|Cr7AhN()dz$xv9j`6cO
z-W<ByUuUT+ewXAg^=$hW#)^-(?sPiNXjB$|ju>83p^taf91ebu@XxlW6rm|AeO$q-
z()%q8RQdg23g5~bo9~<ZZ*H@}5+|DcifQys(Fn%27IhrQ`D$Hm!$lNZ3DJh`Q}+f^
zhz$NNOGj!CYi;IuqI~_UXlS*0k(@0n!?j5MeOG$BKe))7$+3+fgI3KCC9gv0xp!4R
z;2;9kGnl-6?=q1&rk;FN&F#GXV(itZ#m~=1fyG8nHTgB0(gNE(A`tyus4D3O5{;(b
z%(oNgb#IdZ2MVnTp_Bqs)lVe7TW@z7=6_e(PgF4dESpbaTj-Z(D7o?7V!`&g!js?o
zdvl_)5w=sRAfsm<w9@|gyY4>g<GtGZVz**%B~uGL1yJHA**}W^#yOInfwY1C$oh(Q
zN$v&E{UxyBra$QBz1;KNp2u6?b3kFYPX%!azI$-WU<_IIU(!%3j)h$I5On(w&Jd2n
z1?Z^*sr&^}O85Mg3DSS)q~|G#*&qL4WO|?!ahkpK#B}J+0CVXcbE&Z?9w@<aKi<E8
zkT}0T!Sz;ju)B8e*A`#FlUSGUjtOg9?(!<%_xQgXqQHz_uTJ|3o%sf)-g<rm-7`M8
zRH}^GXm71&WfTqL>BHeV0jx;DIk~&?kFl0k@?7>O^6&M$=%2hpPwQNo_4YUS!i!F~
z)?r3C0=l9nj6gBt8a`g<%6)OUi=R3C;dXXh0G19)mx+?T_c|?E_T$}X0T&)XD1Q-p
zsDXwYL1WFzPdahzjJaDjxNmO#@W3lW*}ypNJMq4AKQ7Xki+_G_cJk91)9*qf{u1H$
z^F^0N9v7PI;F&+Pzi&X-nfHL1hr$d?tNugbbz9o(5B6<IWvA<}UaDE9lJFs4Jr@Tj
z!fiU)Trh`s@vC|Qq~>i|p_A=!et~dF>EYqwNdQ_mm6-HTu5?|$m(e2;)L@6ym#+9_
z*qVe|DWoUW<c6K4VvF5tLpm`coDS6Y5rt9W60Y32wa3NnZ-^uiG+C9upWIkJ3=T}S
zQ~lsH?aa$V{^bs<X#%Dhi;P{iuGncUQDUVMkWtdL^JgHC^F(`_a<E}~F1RGm5j3p|
zYXsmQ8$r!GS7yJ^^C%(V6M(Is52^9^!wzwH-R{AJR{0U{Gq`j`Wb71`mi0*#-D&bc
zdUfoEe@a-R_^qkMC&hrVg();Nx8Zl{6YG6x#x!z)nqP6VrJ|-0Ta4dKHF*As#07|?
z>r3DW{Ne%Nr$`C>POt0h;dj&vuYoljo7=Kr3ysLc7q_h^*S@ECN+T!GrB8<R-&Y)O
zBJ4Jec>D<#D~GT}fLcZ7t$M^xMRQx;QZIbTsJB|oBEotyW$5?Fj5*=e_qDLr`-s|L
z6uVx&E)0psYCLFTrfmjv=nFr8ZUmdVIKV!}ii&j$J!Csdku*q5UIqj`8R>>YBUHz1
z^c38mG%?o{vE{Nqu(pzFa9w>g0f#R1D&<xqcQ0Qh0ak7~)(AkOjW$Y;_IV!5UnF3G
znmF)(n>CRPkfMS^_(8WmX#;zwg(k9hJFZbt?&MpU^5yX)K_9ri+|hTbKLI#k2T%i8
zY7`7e{^X#g$f)r-?bxZ=AGfe%HYGk_K@>&mLg6nD_uLkufi0jfbNI?!{4d7rdN49U
zg@^s5K<7gsXi9SY?)r$x&R&p0)V<7fX+Zz^`y_ZiOjdGMx(V@8Ww!AAgM(YZ^hwdP
z4!pF?%v2EV_W`T4H$#qW%ark{$p7;QNuE;Z9Mt+@V~t6dZ*oXDTw)Slqt*=i0s3ZU
zfl}eOp}CSx<CDSWi?^+hKvq_u^j4MgQ0L`~wT%u6{;RTz^|Hv(p&b41jEXNmPMC_H
zpB!u_YXF^SqfkJC?xdw_UO=@xf8r!Igzh3l$&oK;k#j7KQ{4^-WKZo8NGeLHm5RWL
z!}%PD5r?{#u52{Pa}j#jDz~Qg%U`{L<`%*;4e@BHr>CbUmzDxR^B<2^;+@Z%H)6?O
z=iXd=jh5FogN>xt@r^!8@vo_LE7!juFAAY|T7;rWBxK7q<?3S5+~uLTrL!$EBtD>L
zW?|HbdynfvQ-F@?k$|(toaxfM{8AMB3cE$TarYw&@`tZv^NYb=DfJ3#YFAfROd>><
z6z~^GC<qps53qDFAB{<mTeD9k`QOeI@D~n~l%~5~pJBo>{AxMRzv2VYGwE!i*bwc8
zQw-|OD!Em9^4>bRTl-Jel{Fueu>V!wo*bYi&U4T7FHd;}4YV&OLHn!;;HwD1;@+Q)
zz8ZjD3Rn{7xAEGbn}CDXtMI7E`em5}+VzL`ObKI^3A}Fju^aAHK7FB_cgNQu>q$fk
zrgI+8EK!*IA(rGoec4P-p~={I(oW8&yPOvN*(dY3%qIubWrrWWCRyeDw0`;d2|nVe
z^k4P4ixeH&o-}#=(o}7c<r4*i0enBx_y%J$LKbYkjl{=iV~Z=-tgP*9my@Dm6}Xj{
zFW~8?5Vr0fl99DA_gs34lZX~gw2j{2=aO5GbpNc}>KOW$gnaRUw8?=Co#X2r%aI+#
zKX-1TFIVkg=%5y~M|tjV8BQFG4m(eihH-x?EH4!g$>8oOlSh_Mjb?w8AbR<Do_uIP
zOXO=i%FgZBa|tOdJRFyV#DApNSR`j41+-iD!A3DEqJrPz2<Qt6lRa~axj!8!bn@j<
z$$N;@BI6u%J*qm{O!|9ShV6M!xjUWog!eK6OoM*63#J_xAs3eUo{+88XDbtmwG@=-
zi3z_POAg$h9~9uTdp}J6V~}!=Frr;KhHobQ!jUW1Zv%O8)C%g0UL2v9D;ii)s6|0&
zGuKI|uBmCz=z|7^18<X)M{tHXeIe}#XVini4A}NR<?Pb%2;QW9zytSYHj<@c_SIvC
zp>>cRocb&D>6s6;W~CN#!<5}8FuQwC3JV=PPukH6x0x-NOP-UpRZdFg3OFi}+I&Ej
zF;UvU+Bxni){rZ<U?&5wed*%vVsKEu*YHlf^e*T)lCrYRU{4aK$Ej<YRKL%UTn&6s
zXp4X~Sp+x&B59tRNT<ARA_=BqHQoH5?n_WI6<iM#njSm*)#ytbHT5_fDPzYhFW1B|
zZP6!eh)rZE-9v9m4G$|-&{GUee(%XIhv4WclhKalVHY-%Kf2o@1;1q_3Cj7tW?qSk
z3ldeZyB@(mQ9ZCjwfb^!+wxZhb$6kDFd!kL0(`+Q{uc2_H!+n$3LF6nJbIm-wUW|O
zE!PENk~KB<V3GCx15}~FytlcJHh>@ZEpb6$HI00tR-E3F7k*lf`m;xG*nSOTxs^47
zu<mv04VQJ2aZCBa(lTXh7Lo^l&9A)<nBPJ9^v6r{OGRsLYcmA1mg*%haQo8BH2{g5
zrGT(E=@DXTOLG8I5{j2DmUiR#R>LM2zrh0vOM-T}$sfVicgF0AEai8j`556g*wOv?
z)5*0?!GTZb2OEAV{5=oGBw{GAG`0(0I-3>->)Cc(%v%o#9zA+gSX9(7+Y!!b)Jbsg
za$}$~3>@1=S+(P`c5LK3g~Kv&jXgZkWDpe=``)wjkc&(HptZA7zE!}L8?#)=|33kw
z6nN6)!35*#^OLWiMFuv)2_}8IKHihiVVMSFV9g+s-3fWeaWHw43b_lex0rvA*Pa&E
zZ5Gki6Q#r9O;m)|!fGwRPx{hB0L6vOtc&y)7CpInV6}AQf8Ga<AO@^4IcwFR@LYaL
zL|(vQ8nlN}S2%9a<mTqCg9!Z9Ow1n_?-z*X)5NHilq|J>+))Q3BR97n(#PKls8b%)
zBZ5twxTDWoNlec<f2dR}(%5Dsd-NZvDL_*D)baUJYGMRErltFUQORN0iur<zRHZ=Y
z7K>Uge5N(7`ip-_O&kmvFS>k(H-Fs8^W6?hs&R1)a#bWl&eKUtS#o)4TC?U_T5)OX
zRL?I+6oOKaYNv@lVXcVL3scd}zvFim@0y@+q;55MX;1B{lYqdmndJ}G&o(wT!a&a6
z5~b^ocKy1bT{~g$bw)*57(>3F;hs=AN3zRf-k&zllsP$z)zKA`uEB)<2nOO0HO?XT
zS99n4?y~PU^!24&qhD9KGF3)BSMZWS6W4<-au5}^(1e6{pb0?)j8hn}2AEZ{JdM6a
zQd3dE=kA&dUMcVcuQs#}b&vh9nzG>GC*Q`95fP84=Czd_&^@lQ88|{Xv7T__Y_&ha
z`L8_rFhBs+2OF%1K`ZT77x3(c<26p!Ha48|&x<WC4FR1%dD3>+Etk(4Vb2hM7WYG*
z+0it&i7DUpr}DUJlFB+)ZU*f61vX8^lLN&@poPzf_UEE>IZS^r@;h&%j*pM;10CoE
zC`ec4`n`+CuD3%<!*%<V`L4d|(Y^Z4PP=JkeYzMGR(Byc#OiP3uX$K+u)TY=Fq+Ln
zs<xq&j1nt*<^02d|4>x6;#51A8wZA0MFnpSKp!^^22vh@oTjZWU1n-(N}uSDA?>-g
z)bk@Nms$Dz9)6@gWO1!pP=u;B0dRs<6?2+tmU4D=>$|HtZGV)y32-I_3-N1_6aGN=
zy^4gMG0?X?wkpm$mx-R87YKa@pyP;&h2@)?n#wQG57x{DQ4k|urfswIp|9?fO9_3G
zaJIc`u37G9(e`Ak%BSrHRVdql&9*Hg&#mg$`*b(VbMw}vCe%Qg!UO-4`}#`Z&<SKg
zP4L=j!~8-k;K@kJj~{b_tk@MWFoDj$=t)8VT_0}{;|R0)buL39hp*@EZ*ICgb<9Pf
z=Y?*i*yb@MT;e3T_pW<YG!vNf+ul?NQ|2f$n<4);$@c%#w=OLzMn+V?TmtzegPw`W
zTks!ftn5v2aPY_SxbD>R!+l90Dew}nBUbIvVF>BU^%}ICnA6|GvMps&v5{CTGV+<}
z!&uKIFE!{Mq;*{nqc8t<2K~;{GE6{mYXSx|xx5?*?%hbSUmk3|5m5U0Lg@n-lScL!
zlwX}83=cO><rNj)d6Mi;ziIpTq2UPJsKDR&>O3dA@NAQ?qM{-kGm%`~I-UG#kF!%K
zA)Z=VGF?|!r#@G<XkWOj7EQj-m_(rFmkeA*+*HZ&&qSXxLmC=8ZtkXz`x!BX&_uR>
zy^N92vv+BGqM^N9(=Q7-KN(Cm>&;oI?d2f_or<3=Ey&<<kocO=(gs9*kbuGyU=Asi
zsd3dZ3G*9B=me@%O!S2KQETG1*=m~B72KUwQH-YVTk8Z_Z%DCtc<vq&s#m#mN}*v$
zFwC^Ipk8`3=v!sgz_*6~Aw{_`ENOfIQMSeLz2pG#05~}=^I;qii$uVJ(hq|KtuhFz
zs9M+|t-o|bl5I*qI~I!wsFr*T=E|=>Y>FzJ*WxfeayHN^zukI6T%j~Gla6+=&FVc7
zE{e>Av^B|TK36dQkYpdeRS|J`uGE|OVEdw>CiK4#<0%gwB_8R@g9K3F5E8xty)?b~
zZ}EkNh4p}myAEvJc4~<xdQ%ylAQN7tvL8i;HS_8R?e+^z#CS3lrf*B?Jjl<y_R|Pt
zH7c>+GLgzIj1zEoerzL<ShNXwU$IY?CghHEM4hUqT8iSLg#h~m!jrngwGT2;jmxa2
zjUD4^2uk-vuZ})~t=`9{&n0x@Tva@KAqE1iLI=RK1sok6C+8DAT0tw)V7r0@C~9bR
z{bDTczRmp23ceA;-$~boHJ{G8-wjcOhrk|wN7COaVqsic3}AB8Y!D7HA}h`+IeIS{
z^43>%N3#{v+nyLc=`70iW$YD<l!<c+f;yz60MyY*oP?Bc<r}Ucqk|^`=#?Y#g-?SM
zNy^E!Y%liDy~i=><Nt|(fB@J&grII<1a=x-=*h!+%gbz+7QhwWu%(m_AMbFOH_l?R
zhI=~hA9ZBAIquWpHjNI6fC~EFtvLyF5;gcSU&ZfwWRs$u>wskI`irwLPe?k-j>nuT
z_is%$0GUHf{&SIY^@S(Gl9Q7cChCOVQ3#-eZZRJx_29e0l9C_B+;;+D1dJ$NZp4;`
zV-10fv;x+#gLOCVq?-&@DAk9;+6u-RK98Lxsi0l&A}6_K#&TNT6ndJwtl@<})KoOJ
zZq!VF`|F-z=Ar8BrxiylL9-TtOEX9RUnD~@xgH?jYz4%)`1szSEoKP7`xhgh)6waN
zpP@4<P)TXl+h=q6)j2Ngeq*_@A)MaVX@*3rRs(l-e(*d`$mbz8Me38S7{^K-qn7fw
zHx$3Xg+HUjhUhd*u_T9DDmhm2>Fbes6l*Mt9tboOT_(C+B0yoc79`3pd?_YX(ts(a
z{gMi_MZWJWD>D^roCb@d`J_KE8+rcJ07sUD?KL5<)3*C~NU2hm={VE)r5i!VT}DMl
zFBbhhPTiQgiQaE0#VDoWL<D6_Dtt8nfg$=2(!r)F%Q9Fk#u{N=*zcfv=fY6{`yqr9
zuSR*}@bmLXnuHlNEP+ntRJ|8sVOd$LaiW_ql;9G@Eu~LJ*Sv;_<2lBK20?{@Mrw<P
zb18hTk3N_HEhoMHu@|zk@bu@1=oe?NDV@t_4SH(WWhLObrCUQnkCb71C|sX3?aZ`p
zXp^bli@$ag?LrGZVF84C!oo$`ZS@xpQW3&p!-oR>gMuQ6b9KrsZ-MTHQPz?MHjn{>
z*Faxm-gS20qy$})prv1m3hAFo&<S+lvLkdEmY+YDgi1(7*p;!n>eqz9TBsnr1Ol5*
zG>5!y3_V~%(I>iuJrol`7QM9mvMl=CV{S50Wf&a-U~>*Rr^QZ=y>>gSM#m>A>WQGt
zqzOTi=O<KA!2pTLpC`7w{YRybOiUh`SrTv{BX9;s7i8v?B)lzW@@XRZ>yCg$E9L?$
zgn)$|<D!yCM;DclFa!hF0OPrShm{q|nUPRXIuk$qyxl5?z>VQw?m9&I1?h=|3#R#U
z!E^MocvvfbXxP!Gv$)sx-WjcdUfh>42l^HgRiHkpo5dG_hC|G2rY$5@AyE(&1H;?T
z?>e}p<_*~35Mz)P^cEOV5<G@$vm?x`jzBRu^D$fqCMCq~yL)IfT($;_y>{xLDwKWa
z&EJ^LC<8v*XoooUyj=7$AX2Y!<ZKP042g&!;IbMg2lqW;fZKbXbO9%R;p{vQFkTdB
z!TXldJtIg;T-<PSm$`Y)#EUQBj*C2B17s%cjScMQHkhR5m6k?V14fC}%gjH@t^*p{
z)N(<cF9_Y5Qp1*)pE{1r)rL(ySAN6u1L2C_yVt67OU?z}$fOEXx7l#fm`EdDI&kXX
z<bJ$*@^B&$zSFAx8qPsWPiiD#5s_XnTz3qBeQP_rW{_$1f;$XERJtEro>Vrz<VfX>
z81HI*ePiq++_i?!B_T_gU8N0y?m};6lJ|N~k%b_YEgMw({Zpk3IHdEu$x9d*jglnq
zWk&g%y3qJL1qip;*cAM55blbMz7p*py=-{8wV-`z;Ihi_n9od3C*LInJWz<(EUig)
z1n+2#$=+2#`QYJIQeu}YWQ%sh?ju=jo8DCGgLqiJ;pc%|a7^5I?<&r7{4f65E(e$G
z;Bwl0_M%i1+P1D<>&*S@*FK!?KKtrr^|~NO#Z4&n^$f#eUCznJ=VWw7Uw<vBd|D%P
z14%f=pvd&h$!9#|gLjs0CFWsS_6-2k&LFqM{Qbh4Ai<Z-U?D+0H&qhp$&PRa5*~9B
z08au%!*B(pT~tiW2q7|Nl)+0ZPq7;AtV|Q;9_?3><x;qrgK~6V{HvKv)>sSq&&1oo
zp~DWRJY@+LzR%>CS1j^+Q>AW{>=e}3S$!0_UVP(Hx-=j}C?WCdTtu!;{0jGz2Rbf)
zxV65x-ptR>SFg0C2YWv<YJ52iE;Z#QYMPL<e{;*r_Y`|1gX}!7wbvS+_b~H3Vpw07
z!W+C*<?mrR7Ve<BZQBIEb`VLkAHET~dQHWTz!fkrTk}ML<MoU|26RYlFd=gd<Z0jv
zEG!811s7LkfAq1q5Gy_rug62Yam#OgLh`rCje1370xP`JD2dybv3P?HP$Lt@#n>b0
zqa*4s01sU3M7l<0()geGmh{`EN{>&?mA&qgTxAIXAai?b!0;UT{N44?VW1jq1sHHp
zNeS2X>KGXSJG{fgorESiksVs$2qGBG4C(dXT1f}hB7O5?c9h@+u<EnQH#a2MX5L7T
zJI;R)cV-<Wz~~>Zxm`f5ukme&pvh3y8Ao#PMbV#G?iB%I%(DQU?23Y&`c|4tBY4gL
z5|t5qCbEm{{3CK$SlIgRsFfigfo{gN+<V?t4-lApiH+ZZcy6LWxIy`9%Vam!jv@#|
z_e^ibrtm;DU38UcBLCT`Ujes-D&_ch)9UM8rM3Fl6ip-JDD?4_{*{%xoTFkFb&+EN
zkoouiDb8~_D3-uQMUkAh7ra0NxM5^OLtYwc{r6fJtb=@Y6_&hbxN+bsQdm~f-1P{-
zb@;<W!VpH$$-Vyjg>sq%6)ELpw`<3hsg>@@UWpZYwOJ_LS>@puKL1K|B3_$mpBl#<
zP;*Vq&-;P$u5($Ou|f(n9!AhEqIX(0Q-Qa8jeRI5NT=KEw+#>W<N9r6W#!U7#<#Wp
z!3wRTx0X}&2`T%rV{re~spyK6e8A81U-$DnmnQrp4Ak}YpvVf1iwpAfyk<UJh_Jf4
z+G&oNnsE(zlir$dMUcq+A%et1GFP68_~U!J#G7>J(G~SmUz61<*ZBFh$#Hp;ScfDy
zH%<*4FAU=R`M-nOWcomm|H3HH@j!7E&1*yR`&&GVga8=N?n)G*92^`(H$@f!ejA-R
zludhmg#9}8Z+c={+GbQV5Uz;5O1AB~v>b47Ic*mDKlA7~Cx=yX1^+qsX&2|Bm<-v%
ziX9K+R6Y67n_F6v!KGkO&@wkSUkB`ljzEoA!>1#+(G{t3>ff48?VIdrL@-Zz-b6&y
zGc@|RgXEs-*@(>MBdXuK8*cr%{DZ_bzTvyQ`Ri;(`v(L)lKqh<whMR<?&qRAEPi*q
zz}>-NG3tV#Gq*?m6p9w`9T+t=^}^3m7B-z4R0RbE^YNOStWRw>6b2?pkQCp*vw3dx
ztF4LHY0>zWdf>|CL_HR(9+BKZ>pIyX$@h~;H9TQn7|HcLyf$p0itaBq)-t^cT=>FZ
z1?kVa$?GyQ7^<3uy_dp$i3_5JmUGy-7?P9`iX83tR>yE}-h2!44t<b|WU0LE*T~v_
zrB|%VSa_89bl7_KZf9k+p6P^IVd11VGvPQ&_6u7{l{;UC9U+(qAZpj>;|KQ29Dye+
zc{HT2eT1yK%}0jDFE7~&Y8{&;@ro|$(_BiDNo=9Du-W7b6haZ63^a+cWQOFT<#la&
zsGzMUS>%00gb&uO2kn7Rs;jEPd_l|dEtFtBe)31mH4MeDnnyeyU5D9(FZPp~S9<lg
zPVDEgLw7=w|ETOL3UU*^o%sGpb5?tJhw@vRPp3N?o8MSl3?i|$5YHoNe#AV}C-r7>
zX7#<_#EtOG*)HpX1L!&Gut#8!HDwb-bDI%m$i@2FnG}5ogSq-sjc>GSG(R}N$nu_h
zb+_dk%zuo{(iyz9w)Y|f<Sp9tuwEOq;aAfSbS#wRViYEn=<`m?KY}3p{_{&RJqtv?
z`x9@{*(F3za|}-tx%vp)>JqEoX&YNTij71emU!rtyeF>e)>XG*eo+o9)(1flZ0fgX
zk0BGwdPGG}j|^^ZAOgEmn7XOW--7=1H0Xss7U>@m2U;0hpI`iF^Vjs6gOxA34*9JQ
z!G5|-dXX@u(Ls@y(;Rom<Mo7WVRr<0?{?mKc*qxAMTYM;(D&jk87}9h#$Q<FM^u^%
z7C!c6`}6;*bVraW_#HimbX;!uVN?1R7vBSyQLwN-k3Zpe2O;7IZRmN0TIx*N6{!sW
z1M7?3kjjY?cG;o1BjZ%>{sT#!_k&_}z|zk|w$lBDIYsjauf!6{zKd&aAJ#sJO_Kho
zoL#KveHosl@j-TMBGBpx{hXyT2__sJ!EJgt1Oz^yR+%ZEaBsq3&oVzX|FA0nx|`<?
zu$?AJ>d|!H8_8;X*jL*x){bW6dWNxc1nx?Ha4iFL9*WAKQKb+7=nNK?eO8fZ8j-&6
zo1U)$Fs5id&uIt!Y>LZkJBwQ7u=Z|Y;RRI815%pE$Gb*9`drJxuQDNX*y09gW$|M#
za$@VLreZ<8EEBdibK{L1O(b4Tc~*h$SC<$bi0b-Swe37m1;k+pP4DUhWJ*$6x)I#M
z!mQ9gyU@<}y1U}*$YrYw{spNoWsTNk<G!SVzn}d~Wa08SgYM7<-NnP6Tg?pATH$vY
zynx%BP$4e}W|$EA(hgF<F0<e_7{ra8oy;LQju&}8(4bP(v^OZU8<}FB;)tVB)eRYP
z*GswL|D1Pk&&3x&03`l?tThRecyzg^BpkZt2n^;^+_`gS9U#$q)6H*xkG&{+cO?X2
zv^1RuezT=QQh`q&Hcizs6<SY9b_ze*r1|@(de=ZMh3VPFd>$sK&R_<n84cdo4FS4T
zSX$ZwjuND%XUD|#f73n1c=*ytcwz43^{>hl*=7&Pv^bzHUhW-M^jEmwK`^-pI99+|
z7lOl(&ovdUglu47Fl$fKOfzU{X*vAa2;*m&#nT@KP6Aie7UcDwc_EL=*LQbN)N20l
zDXwKzJ^v`^SPFsrM?o>RKlhEH#c*qDjt35_L112}VQ7fh<9Oq`DcA|N)$!}HMVtgD
z@&c*cw~2Scso8=WyIUB@jOzvUW6`{>Wv4h?Ke>3NI{b&y(?qvO=gtJY60g39@f8q9
zKzuYIqzp-L9T>oAM8M3sH9$vd08qXUQue8trt8d3l^eXsKWL~ZN`8*0=cHGfGa#ei
z3B4`%Xsct%&2gO<0bkb&0eDelHg94@I4ex4XhZ$wBrM-YAT(C1yX_X@=wYgzlyNCe
zVf>}PPlZ5o6?ZlH^TP&=W;o>T$;bkMlZB}3j7lj;`p|Q>!$R=GINiG4x$<<d+%yNn
zh^rWGfY-~dAH*M8Z(AhGQqCX{TpBqZ!LI-id^?Tx1&1Q2qWPKnQ;ihE+RlYvh0Nw@
z-$5tg!Y~;Hq1XN%Nq=6-!6{(i=GdGXy;Ak|DiCGWiwxe>gxXU?ZE%26mj+>jXxS@}
z+OfBQOWeUSD<!h{69Hp1Jd7*s#Zsr+W|QhhHI2J*1vvQc#FCPOdH^Hhb>$;iT=a~J
z#YZXIkO;QbB0o8>y-XPh7;C`czA2yia4rZ%Brm{lG5C?C1TqPdd(Sbyoa{bT*W3&)
zp$P>DS#&&nHq{X+e+*j^`)$UaRnsC@SH9l}l^e!Nog<kUww9>Led7a%*1kUB3DrV=
z^yqYWPpzMN=^8&x8u#ty!8B)$ZA&E#dh(U7&$Z?R7ur`J4V0DSaiTr+`ih_G0S5LC
z-00b!m&j>|0xjM8AYC{@kW{~g<i5XU6^B+8s8;I6nPo9HAKz&R@)0_!!7v;t{`XyE
zRJC(s8S=<gKz%?`Azdfraf0vjC({xho-I~PI=9{`!~T2AG~o*Ihr0vE)?PsH{Tvkb
zqQOX$F`R<#uMO(O0pvpuPTSdbny9ZZa86tJ6@&*kx)%Jfr)Dfr4=MkhtIWW$^G8-@
zEU)77RsD^3{hP64#VFV76EGT$IWmRbiD}vq<T9&v54T|xw6jqW`AyhuqhHihM=O2@
zAiun*8fl=aQ53Fop?L@RJukRO9@sI0;DPb*;lo^=T2jFG=$vTb2X&@me0?0NgbghX
zN`WLs&?_rtZTmmQ-b{)FUIq6l#d^j_?B2V3=*+E!$bJMR4~X&zc=-BfyS(a$5$x!k
z1y`M3AZNxj=-7Q`j4o{2!wcL}y}dGm4>)!{fq@v!27>_jJ7D)T25veV4=Qb-*4DS(
zvVB?7Tl!XJdD*+gMFfBg!fGOTT)-o6OP}kd;<bs*Q;(k6;G9tBUAjBc74`Z*jtZ?=
zR8>rMu9|JC#ud_6y8s|O1P)Xm9ksRpJgz0_K`{72UpmG+az2=ql@%aeB7()CEb7$!
zvEc240b)@)+GT%z?u}TB-OWEsYRYn>BZ}w?kKyGgGG8^nkdW7Nt3P|1M<gZWfo;)a
zukP&J<UR``EALWxywPRYhRkl38dL!kaQ=%3@rD@O42tmJ)rvpZGE)y+TD8NP_&|1C
zdF6TvK#YE~ThH1us;xZ4Rv?x@!g_%Ls8Cy0Tu&vv3~>Czh{~|ET&@|MWSE?2rpkW8
zYf%i`43Jfj7w!u?kgO&~$O-tMFJz9X*R7=9xhy6Aw*{mOMOD-9oaea4Z8jWST(8|f
zqcZ?QvBeXnqZL(tO$O}4>lYFL$Yixku|uZA*9X7u^k!u%SZ3M^0dyGK%Yq@lK`Q1j
zS1ul1cGYMTFYqY?TSDOWutec*0e1Bl5d@4tS>ttD<W2U_+}`Qa8Akw)bexItUk2(V
z10a2sUkBz;AK7af5A=f~!*K!U=yhG4(COhayHi!hbau!b(1G97ID}*&l&_9&u*hJ@
zF7>0k^1q-s=~Pp!CqV9_p-p>_hY^|(ff}p!M{o3#ACqrKGyzrNM!V4pAN$GJ-e0@~
z-9Lu<vQ7$u3e1fajq_ewF#!%O%>%my&-Ha{u=a!mTq4IH9eKwj(!wYeoYx74|Dw?}
z-s8#X7nx^`8Z^-3%EEp=9eQ7K>;l((+E_NaTxjz1LBEoF8X$o2`1>?h>Nz=c?crf9
zOtex0R@tTl=l9FTUX~&#DhOXaIPb<KRUmy!5C{Yf(3=K-{K3wJX^=a47Z<;@QEjum
zOtwHMdXJ=h#`k9S2Fn~%?0LUPQ|nrdWD;{N@2vYBHK}FFH_}2aYIU|Zw=Go3K>wVX
zP`~){uPY()LiLT1i<pLh1XTMqn|2iy50BbuB^Uu29wl4#XoMjmZUsfo^C$WXZg<Fe
z?ulF8RoLtI6RbpAA8(gyEpc~X_PL&FJ1ocLoX$et>VEudhi*8tbK$Xegzrj$Nq;89
zIl(b2r!!I3w=xW-kTXll^FTIv8<rW4Lwc8DFufYX_Eah{PTS69F*UsXfWgi7Rqvnl
z5P6#U#Dk5F1L>`iuRcBp+w@PN!npngG~go!G$AxJ)Kk#yP!Cl7ps&yQg8$$k<nTv>
zlOLCiv((*!;5mNXizlwrt?b|*Q=mMYy<IQ#<Yl;1l`f^=7hGx*{(Q_k<mQP_Z+a*s
zenmXICdKrrf`If*HFjdVx<Q2KSitzEiqH1cNqDTM;3+x2QE}WapTyNnY&Tncj@HEQ
zgC7uIVzjO7-J{mu)$T%pl!sBr3yk$7gG>(Mfvit8ZGhXx8~ge`JF;{qZ~sWDC*QRt
zzKAVQ9&rsBlO5-Pk%BnAS=8XOI0dy*$JZNxE+gpyh7mLy*KvzLfdKv?2R*O`*-!UL
zK(+#01f#efxT<quq?iF99~|IXcvmnXa8so3#uzHd*;v0FLjUOeP_H2;FeJ&zw3V)3
zM}6mtWbE>^B%d|I%6XiS|02|wf?`tmU(P#@)ckq?Y#4d~h}`R7>Sh|;-N~(aSC}gR
z4XxIm72{G*q0jC~t)KCEmL@Y-`QYbbkqXfTP$CT=P2rB(w&8OXtDl5{d(IyLf@5;;
z3l9B8Brp$hA0>p^<&CW5GvX<StB3s0^U5j#6L`Er*m?o(fQ#C}rD+hl1^ANt3#lRN
zVD(b0Xq>EMwCH^f)bkHe^oNRs<}S`}PIsnrLw|GpLsgO(f#3)Rv@s*kr5!|H`09@)
zCIvr<W*!mfkwV>q=eddrpQ6y-8cSAq!)~)sW#ml~{JcFXfGLX%b1+Pp?8>XQc6bHv
zC!JI2G2nTB85=>O9SOoz#sFmysBpnS$|~vsAQga((Q!f1f^H9h)yOeRgHy|XNP>x+
zz=A`5fP>qI*!$p%j<x-?>(B2k(~r;zygkKZeck50--l(3g_{~pEiS2GmKI9>lZ#gz
z((^pms-YhVIkN<2HY>%FQDxJN(MIQ<%JTpfTGc2oo|c}j54zUTcZ~u%xZi;u=Q19O
zs24f1a{BdTmjfy;39{8RPio}Xw3JiIF9@W0hm-B&(aB4nNxBY-dGRCVzmaGf>PN;Z
z<8g}XXgUwONuoe-APP*oeT{~Wl0;SybeT4}<Y;bgfWD*ws3=f!9@Y^qjx$F;;H`VI
z&JRpe9rk36A8Pjl%AWu8IObOMv3h4FPp-w<5=3H4T0sqOVrJ*mnoJfbf5*o`4RH?w
z2}(m}AKt*%s?;zCDRt`v5&VUlzL5IDJwWaST0M2uZ|R}%$Z^kWXL;x=hI$}3ZL>43
z!5He}U!crx(b1PBa8#p`3TKwXfBDoIC&o9D+bw-DW+hSJtqb@bZ?s>*Z<{6_Ie^q4
z_}ZCtAEKh*4i||aOA4F~)<Me3<~J_k*kyQtNo+#!7@`}_c-(u5J98fUIzcV_8o)eb
zr9n_R=81sBWAW1mbepKTxe38_%S-@fc<!l*@dO!UEH7tN2{=mX4<;pj1pVJAe8K}L
zbx?ASu^VvGSTH|!ziA~`Jky9BIANkPUtEJW_&iP|3V!+eH>2-28E<n6HxBhf#$Vss
z*+()}D1WX5J*hy;K*6-}%0LM^m@W}L(4q#rChl{Q-T-?U13um}j<IV3f>>PDw<ZWh
z2BFA$UD=;s!R_f;fOW}C&2SAjBY<f+ZQk-ab7KzniF|uy&(bjXvy-6F0P{RY%}}ex
z#r38#$rjld0Z1)?6y}-*z0PeCUU_i{zpA6&Lpd;LZ-QMPseoS#(RCr%7u+bALSou$
z^&ULrP<Xmh5-*bcm_NwH8M0>Qe18(w1f-%zf*uxuGJNkyL+S7&w`t<LFf+L3gCMaU
zpXrkaH632BGEB)lflYqBV4-E!edB%cGw9u0nCpFz@bOxly^E}&pQZ8k!5lQ8Ze!Ci
zFw8Vz3pKBd6cgeK;^PG@+(!9ifbhjDZ~B{YKOyxx9TW0(HsAK5vb@I#^1EdL{;`jg
z;TACcJwuzhZ-{AWrgLHiE!%#8t!8Sd=K|?BqYiL8z*TtQ7b8G&-lkp$WaKHZ%S23B
z84qeEfmJKZsp{Sl4L&}Pq5-WQoz;3)ngp<$MD0!99cIU+e(8Bv6AEi#qWYbU{hs1g
zB-3ILimGcsF>@q9IS3xX>1dl%hTa!#ym*?K0>}ZG8Z0mdb|~-FBRID;j?UmVY3Ny^
z!|Fk$-TUb=1KsW023^sdfGGE#=}naJ9Y9u=YQ|^O<MMLBPMqBCMs*rD;}oj>kX~|^
z;XW|HS)?#-&EuELY}rz8o5H^B-X^9*1TOnCx_>j$kZXr&qZRaS5p;Ume2P+2OWbN0
zOqB2QhwwB=F5-&xBmmAG7t^C~h>87hX~p$6rdtXN6$cSTfN1Q$a;kqogBTGvOgQB|
zblLZvE3<QVu249j9pJM;_G1jPN5?@Q$o7C0Bg2T<bwt739S&$#gTYP6OM-T}%CoPJ
zy<vnm?|Wx>k|H%5#2LstznOb5(8Fy````G;sD}iE6Xf}g`68fH!0ucAf&&w<0fjff
z+;4$}<vyjQr8eHo!hb;j5+Nk&GKo9?`pbvX;{YFmDc&7${!1{L0N7<cpN(XEe0@{g
zj&_H?xIJ81S&^kK5pDy?ck-g+C6agbPUq1S1;8wgp2N8T=Pw$>f&&m&(6@XP6#Ak!
z1=^O1#sKsZCEf4m$Iu=`n!s~uT*P6HhhCsL_dZ)rq<fP>3w3nGIqC*Dmk6NYKFTid
zY%Z#6FQjHoT(CNz1%N<`b?$Cp>k@H({_mppJNS6WRfLXr{|Uwu9SI=)!!CmA@{)jg
zk*wyV5V2*g^yurzIh`W|i~{(W>|v)~aEb&_0W99pe=nZj2@gyPH3RJe46bCqFu7Ea
zFvzA-l3X$@p!fAOL8GHHvz~fIuqyN}3h^cT6e{SzSeKP^=fQ6RI01CvAwpuxx95+c
zBgs;t;^Vs+|InfJi`&s8=u?0JtcJF>wp>QwlX6y{8UE)Sw(sEJ-tv>t-+K@6#`JXL
z><G3u%!289Ci=<qG3w;!uQy8%PC!SP|Dxs$AO!q$k+t#KB5>0)ADYeny1q)bid4~`
z4p!^ibhhDB?*Bq10y;lokEEHD)`u{>WNd@?30S-%Bb}-AfN~~vkxZcAMVD@xh<SN=
zp=9@#WEV+B@${p=?LE(wJOz=#0))5pGLiM&-EtH+mB3{JZbVjhjX0_G?`k)|&tD@c
z8mZ)`9k~F0fTn8g<mzul+0t39m6Ks4$Er;)qi+uY$~1?@?;7T_RZfc=6@BNrE}#pR
zx(ZY-lx|jq;eumE2TBAY3i5avCe*P26#)bzbj5IEU}0hHgO+j+816sT&MGXnvm%te
zm0MVU^AhC-6i2rS?T~H{9k2L2?)){%>y!m}$Q4!VnH1Hez6%wi5bD#iL1f<^FEPA5
zKSz#8em|EfH}B3%{?_;<BK6O|xlX^3ctze|(X;+e{3BSYeK2t7u@xBqS;p<{kxtDP
zNGqj}8DxCY-sgDp;wYY+`@~I`0w}rz-UA*)(}xxTnwgoIk9<Z3sjjNZF)~^tpwzwC
z&_X8&sV~ZLP^?P3$Lw6`^;n5Ggi~gqpnh=+x5AMX?|ttE<JwGs_^Du)W`}m&hlR=#
zQv?hg@?A+b`?k%77ZZI7-??YCw0g$;Ds1NkwQl|C$$s@FPJy{`T-)z;kI`^8xr~1;
z$IW+GJbV=QcE5ohbR0i%LIDTq04oMdPTc&WLhlQkby3vU!Ui}jwWX`7ooM#iE~%Ds
zR>MEl=W>o~<fT7kEaL9f3};L2Iej7*ANFtV89P!?Z0sN^&2gGjP3sK_i8&@aEkDJC
z#i%yse-H_x2nj>P3DnU=4;JR8bf|bR!cxf;1OX(TGyvoWxi0Q~W^V|JRmAZEOgsI0
zPlQc+&;xA(jSvjm%IR-L!ooJ%t)AVoli$4C_8Imm$p4u44A#*Qw{aDpa+m-9RddT*
z>9Kp07k+s$mmIj*zTE8Ao#PS=Etz2>zKjC)l-0nY)Y<DR?B&^7lLq&XZO{@u9B_8b
zq8h^3tL6_qtLu^HZ63vJlkR-MD1Xl=P?9_Ugw!bgllFF{n3PoVhuH5%1P6O>Q2omS
zx+C!M(B6_(9C{?+?rjNboi2}&<Lh8V-fD}Zm#|ilMtU}J+!@vUBwR;4B+BfWkjmuR
zo*~|3zhjx}*x}mYWa&HZM5dXh?+ykH3T;ZI225Jx0=pk)GRnL`#=eZ(deKoo0PL98
zXm5|lS@<E&4v^EUKR@Y1d;xd_jim`5uZxH^Ye!Ih3}p<UC`gC*A~lmW9<d?Oa?%s-
zuta$&Fg3|iKQ75~D%C`8Ton|L6+H9aXTrk)M=6D^J7yyuvnzIgGf7ND*T45V*6yIQ
z@siLhM4r`(J3@Nn>l0s`+3R>QFfUWfhCvhr`@s7f6>e+xH9Q7+zrfN?I_3`!B2q(f
zLpz0`zQMPCuHQD#J&Sji4pJvM?JZER{v#}2uDbOyDqL>*l;35MO|-E|v+DP30qvx2
z)5DpXMpr%I6b{pxi?p9nk^ekHQsQ?Yk~t6NNM4|@NCBqd9}*IUk4po%l)-3d+}n3Y
zUu45>KCYB}%wLi{XO^vhmkl7c1tF)%PtU5D6}*d$q&%Kcq_5BI`K@>Ss@-(DC-lNE
zM54T}IwlMs4>r@2j?;XbF7-K|f(qMX<u$=7*oWv)^$j_NZI1!nAs>E^=U*m#C6_6?
z_9y8hc24gnvxv*e9zAD|sZqzW+RaNec5;gUuWKdXh&xX$NmeZl<)F|Vpak`5UX0i9
zrKG0LF~u~i$V~_HbDPUbW+g|Bh(9LiNz2eL;Cdh&p3Pd;VkuWN79{l%MqD74>^{_j
zb9S^f&~|J6^l5jZiDV+@a*@$Ip_BVbDD7I)`09ibUS?V;EbMW~3#vSBYH<av8_u30
z3?j2fXK!NCmFm9rpNwGCy0efPNYfO#Rq0XgQq~2l1vBfl9{ss1PngObGr*oWbI3xo
zDiIQX_uGJ5mc{`7Dqk*Q(!+#<Ovh0jS#N?9Mx`$q$t4L5fu+*Mg7jaFZQNa7qp(ZP
z`|ltbq%FToKx=yY(qWyB8+qsWYm&#&gFdn|z5*+p11gHPv@Kd1%CAF_?};YzY2*-p
zRN1tUk4C54E(TUI&^2q`DAhfcuS!(ztrfXpe*9%$;_JdaYU;rhoV{@MwG=T#U9QNm
z+aD!7o~<ct2=L#<YWwaxmB0P?*LNeC>ZGc~>oNBk6Z;pxuZ%j}A(_wr6~0%eo`V@P
z$l|vrIM%>Tm+*cX>pYApzCJ$@#wkIgAm~Iu)d*OjU<Pdp41aCf$r<396-Ae7^4zBT
z(bsRD`3X%6t2*TA9z|n+3z}SZ6nE)52{!U))Z@ZKuJQQO>bTrGw`VZdG{4!Aa$N^)
zu{8(x_YP+#ql`{U`eS@wy62}Yf8aZ9{3;-#Y)zmoxAU7#EKNgdpzKpo_&8xpeBIZ*
z@TOVW=|fh9$iTa9l`oG)99?*ger|0W`aKf*DiySHee+%bEfXJMt$3cdI@g1v!B298
zi;o<IhDQWrRkDWWTud6)7|l;?u3;r&y&vS|WoEFoGxC2EX)JH<w>RO)^QnJN`~8o~
zm^s%mdm6OQ=DTK_DjHR8d?#FcK0Xy58Q%c^l<bw^nd826U^ekyMu<;yxDUZLc_kJN
zU#Jty)HA=gsZR~2vm!q_7`wVVtp^zA_ue9S`)2N?j>sH(x?T0><DjUFyd{3S^tj1R
zSNQRl%F5xHg$aGF9}n&YAqWXcNqZI7<{_9pQ-#FcC2=0_y5=pP%YbmNicUvImjv=L
zUi&2iQf^Zpu(YbFw^vD(2v|gR^@ut?E{>VFe<^LVF8$Mi>}$kr(;7&_qA_1Ny5N&1
z8Xg8yoA`Z_zIS=4yE_v5Bcme%!q{CA^0?~GmXHfl6{3FL%v{97!`kJMl<jNp2#AU~
znhGnCEM4{T=w?pfJ)3`cbI2c?2A4)0!{1`QA=Yu717YrKMSCjpY<ypxd3suX7`x7Z
zTleNxKl0Js1G|S}Hw`Sjj_K%p-71iC+y&Sf^sA6m-Xw1K?8R_>7KxxP)305gQ!ikn
zp3-cjD6)~VttpVU+6=VE9+d9*(54VJMZ<dRrtKWPnqV>J6H<4Gv32Sot5dIW=L@TN
zIC-e}8>Baa5zD!19k(*ef+L7`BOPYf*<UJ1axE5#YL?ID^WW;93by)9m2{>pCMPNB
z@ndGEce&7FkK2UPUdH5^guVZJb_1?pNpydUfn%M`#`Td@N<Jqt_i;NcZf2KuvuFOe
zk7z#OlH#%?YARGIyfb%ZX;UJyI-P%#yMKoKeU!TZiL+{cx>RyS&=rG;G|$xyUMy{!
z!r@YTtmArK+X>R4yN}Z=<>fxiO%t5E0JxCx3-ig<3;z#YZyi-t)OC$3f|N*u(s4-%
z>5`Bx>F(~9?i6Vdkd*Ee=|&`^1>_>#-O~Nr7oX?--rqOI_XiFfUhg?)pS{;!d#<_W
zL>=6;LEZAB$Zywj_?Hw@p#UBlaMyY=Q*R%dnMn%hhfkTAnW;GQC0W?W6(It8vH`-#
zH<XNaUJ_cXb?MEhdM5)u+eyN>!x6L!&){@Flz+lT^mrGyGVadV^<!phWyWax59iCc
zO)M;<-W>U6gVl}?FMFv6Z?GeL1<3sts@KuzrWD9mG}h6tG(>W=I^aAW@3&xKMW|<X
z{@L>TIZ>HDiF(^K(TB8lty=h+>2Ry+-e-bX#IrToQP%qx6{KrpG>E|P`-wVhfmXlw
zUid+t3pTHrF4x-@F=kHl-ZFc+5iNJX+Oul+mW^z27n6=%txNF|&M}!|CQI+CF*Le*
zEn<^`jb$V%oa8!@QGAANITYSK)~cwQs<-oZze+PYq7B=Jwt!`((Ma-h@n`Ch`KT#D
zqPQ5{DK@;dysczb`zpu2zfFu&_~G}9)n6`LryMU;sy6FiFvfXk-(OQbBWhQir}D+`
zKFG7fMh??c3KWHGB**?LezCuCdT24*KuFo1uxMc(1VFGuCok9Fq$LWYhrK5OhPeN|
ziiD-eFihe#5Eo$3Yk2{fNZg($v?Z!uXMRi*aESysMV|}4eZhI%)M0F?I3toIvRYs9
z<mT1)?`r(YT`!3<ALD3cS<N6cFb>V#n`K^U+<4UM3>+O?d}$E~*2vOeP$`>nMh;^U
zAuztH8E%^UD_<oZY=3OO8i%|5_^v*n(uAkt+URGnX!G5G<E9Fd7W+x}oy!kra+8}f
z%39^f&p+wj=E_L6K&z;sev>pS1V?Yn!W6I9#QwGw-&+QqKjjQ5tts-+OVh|}R4Vn5
zOoV1!E(`7*Eq%mm)XV7#4wn4-yJ$DXIE|HtxC=+r?k;m+!-uEpiGcg{lx3w8L*7R|
zHX6uZdiug((frTH4jc2v-t6BH1n!G`K2vl})2q%5Cl#`5*B5blx|AfYx!dX6!As<-
zg%+NVp^qJ`I&8VI9a9&`V~-al9y#@ti(nftMh*UK35&xQEEgOI7B$0p;){nmtKUEz
zCUGP`*+8fK4c<!qFt`=%Ue2Jy&H%*0|BF+BdjUlISS-gNe`aQkM>6>T212cBG4{-a
zg@;;4>Ik(KyJ{U~{fN-p$EXxby@$~kpAlp4<bndZ9EW+Ia!VxXJkEP+x53jB@ya`Y
zKc)qd(9ena9OO?;(+g!J2QzM1=LzQ-jNA7`R`z99x2&6_?KA8($Moy@2IK+g@Mg!4
zikak%o3unEA(UN)XgNaThftZR!c;w^VxKr^t$_*b!|=(cOEQ|#VXHDz)bCe)PzAgY
z>`zP3XLRuD@o4xX-+j81N?6F-=QcqQ?leB?Yv4@gtSHwJm%CRo5~e70-8{3jlbiGr
z6;&akm+X60WlCxp?>P2efI;)U)_(m)It8hxSQ9$FnRKM7yyVB?-f#A)D&^^~`!iyH
z&TjBTnP99vdr^-`c-Igtbx_m)&A{u7x67@n;fiuA14BEWM6iE6aOgS@d!Be&biVLf
zz5EQ-amDqSxRg}epzSCzDaR!jlU4bT*JJW5b3J=MB^vPVgfhn7L84$#p_n}vrC68M
z140Gcoz_lpDgTv9rC_#&FMzVyjq6_^1oNwUqut)21otB{a_!lh8C#106U*WKpd{f~
z>Y=MCIZzli{&E_nD$@JoiZ)jN@!XH0?6DJ3nS9G|ry1uBQ+<8%%HXb2`||J6w_LwE
z77%;uTfP_{eUO$6mPWsQRiDD#`_<LZzGeP$iVNzZ%&e)QvAA?8<=}L2{`mA53oB*e
z5I+n*&cClE%!OO+-t)O&#WkV2oj~m5%ujC}Z_eJ};1`ONczeT^*vf2MoK^b)lF?!B
zAYM+|NLp^bXPNIm6MfICbTePL9X#Kq)Mo`*R}ntT71j0G(gt7EV&Te&?9|CdK-%Wn
zy4B@L#uqCAHsU01hnq~kznd>vaDP`$VL3S-(Or0~Xx`K;m3hm3t9hz;#^@z1JS$=6
zA}FDy#Vc^VNp)jxGO(Kj2(0Fkk#&?(+v76XiVfF?VMIY`>{caqV)hlxrZpsn-Cz}9
zf#aQmFf+@)0!%CdD1e5_ec>esTrN~RhK>eD%>^r!SJ>ECR(l?GGcVw#YuG<prb+Yi
zy~y>1N`ajN95)!d($0DKUk~YIsRvcA)?Zc2ze-RS7EZD$THlD4X`Ua_G3ha%;{hVX
z0ciJoX;tlHq<^bFbj6KlTg=#}=#;%!BNz8FSx}RS&}}DY^kl3n9c#Hy%txqn(N@I5
z;)iT3P?=Y$CgHKzkug&2ukAmxrt_eKqGF)04zZAv7o1~Bh27XRdF^_Pd#$O-tp>SU
zTIDs&J~!WWOT;6tRB1EmInRMWx;l3c<nH-%%Qk#0#`7p#D@jrv!`0HF&w@U$92P5h
zWcs1mw~*bgZ0$psqZzRnj0Q4*pkvtEs3g4p-nO(+1j?s3&1MlT%2)W<SiH#NmoHC2
zMJw;2qUGyY@+pb1Bndl0?-+DuZc&MY8eVcZta37j9T@~-JuJ(0*czh(BK5EeJIrW?
z#ch9fBcrk)2;^DC^-~XxhzR(}7i;<#W?PDv<d@xFjGY*SWT})3Mr}MivxfvvMCy}D
z%+r(9g>!ynWmer^*gD&3<Ia43{<DGU7*T`@!YW@C@AKAL_YG>^YDef;PN(Y_!Rr_%
z*?v|*KSOLR)Fj{pRehIGdT}sV;oj``MAzl-Nb1;=^|Y}dvsxO69!U$_bu@zDx!xdL
zVJNehlq3<iBjFz69fJfp-rGtArt>X#=|)~52(fdT9g7VMmO039Hplbp_G53xo0ld(
z#`zw6gU#u1C1!8(OtzI<*S{(XnoJ38pa84e#olI}ugEx=PnT{drOY!Wsmxt_T6r;R
zIVVeLB@XM-?Kp(-Q0CJ)9<WWJ#|Sw&Ilmelje~=Ov)m5mVcnpsTrnsbky9DNfEw+}
z_|Ty)#0&?beo)+4`mqsT{R;8#lB?Y-nt?nzQ*qg8>P$*;o{yu?n;0vDgY8Cbs@j4z
z%qa?o6p)b>-5Lwtlg$_0I}1Oej<LLz<z5k-3%IX%LnqJXY&G+fp%*nR)?(t?kK6a`
z`St`;P`tK;37xXHq3|-D)H)`j1ZQhV4|9S~kMsgx_@~&9Q8FJi;vu@JB?lXgm4-s+
zO_$H*8T^^>g#C&`Nbhbu4%sHxKF4CvG__w)kqAnOpKXmhhX9B%{`F9%5hOpkzi(iu
zkNt^>3T(DSGJs4Wj?aJMAyeo$hLOK9FfjaK{Uksv4`ZA&th=o7_Ekr4S3BO#KiqEw
z1wNEtwETI*s+d>z39}JF@?cuBbdm1;8pzE<nDI_qX{jKf$AwjHlnPSNGD8q>fMBwV
zPa6FNS!72*u<shr4~hmmL$5XLM@YJU%#-*}hTb%XaFvotmnItUWkDlsF4VqHDsfj?
zGOSL`<_N7zWIG-Do7P4+N31fR+k@#%?->=uN}+YGzQOHYDg(%PXb~B`dtNoib4T97
zFMAwNU=d$>zu}aQkn&9`va6Vp@ehM<`0CJnt^H~PQ&95$=J4E~jcCc(S0lacU|cb}
zCu6UeXivy_>nbQ>9`)ZeAGK_|SbwXr*o?T{`C!VU3gM-fJU8?f?q>rR3p1K#VY$_y
z<3bE3xfLQ9GEQ`C)fmXLUC3NT*nIL2?qZETdh!os1~v$QQ=6Ui;sK3MMEO`h3aYUF
z0OHCv>JA6YboiE*7L}4Xuj-W8pK~819^lc%Vt*!bvbJ}`Gf$1GmaydN{)+rP%Df8z
zm}Q*&;1;8UuCQ=f2w4Fo!Pv2w<#TV;V2Pe3+^wbiHNtbn>G7B5_b<?5K1IvF8az9|
zO}qP^hF#z9Re#4Vlrb-r+V?aYvkIyG{Ke3A(uiw!wf=Eh?1k+J2SC0eZq%+D4BxSs
zJ4slYP-RKI`!S>Rth7EOG^iEr`3<YAFKtF+K_@jt;L00+{oYxYX}joT-7ojv@A!Un
z1@f_$s_Tbo7238U{=j+*wcqfTKIRw!GH$9LC26WaTluW|a)i4-BOVXamK6cBDGnz&
z`Sdrl8H8s(e$^MWF-j?iW^C~6$09G8&ELe4{35;Odq(PaicgVuy}SyIJ7nCU2NY~S
zTn_Ro9qGK*+xy(mdG}i}?_()IN0aEH{N$54ybvSZMn!)0>iF!lbLB0=+__PIflPBx
zmQ{XXZgkh?7p5O-OS6eyf47VmECTHdT>HK0e5ZUdgXU)-2iUm~(Qon);VvBc+a9~v
z!d5Iq@Jc#+{?oYhC_wX7=F^uiUKIIToRa2*oMgSyz3=|^uj}(yQU_?KWC1<?4Gj&8
zK>wsjmv3jb`Km->YYi;G6B3gf*qRI17m~fFSU<SG77o@)Nn~&Lu-w}7ic|SzONq$Y
zl_Osroq^HojEp?zQ9-q<INysf8JLqFdZXoWewO*P-#&I{xbIH+O|b3RG*<AO4+E*w
z`>Ldc$aYG8=epXePZIA;^mMe+f1lKnViMfZ=F%P+Xxhj{*E!VG5C8G=9{6ZZqBU}J
z_Q=e;Szn-6*K?pnTFeaqNM!tf<UgG=CVhf?lD{f+R{j+^Y<z?AaI9ku1AV*u$>!Bq
zAXM%-=U7p^14Kr-{z58(Vc`>rUPRs9!4AphWVzf~30oUilDKyICvokM@80AwFWH0=
zerI{XL?Y`q&$#l&*q2sXSh$P-z^`{TFG%v+Cg*5nj?UbD!OV>2fZEig-mfp23z^`o
z`_U6X6+r+-`CN+A&gZ)}w<^{zLP;`kG_9e%bam%Qt3FLd$xx*xKDku7b}Y)8o!yRZ
zlOOk-X86NEJ6B>JYv2!(?}|<GP~RECN_dr_1~te?7rQ~zf_mKHEA<`(bLP1qwX_f4
z=BZW}bg3b%KQ96vc1|1$5DWM1c~Q1re}J_K3cZ7gQuhF_#yqaq<zIcSU7fcRPKQ`H
zG~ZN~H`SvQ2=H+n9W$-`r->!I<HsM!pqBF^8ECq=l9Sa3|71MLvf9^H2DuE%99weW
z$k~H#A6|`ru)NrZF-QJ!RdmtI_GteX^~-CgYqC$6PkL-S=(d(x-HS?Q0LB&5sLY#f
zs7pG>bZKsDnOeN_eN^*o?PLv_jZDkTUpCYdran?IJQsAJo*}H`?bJigd|qfv%$Dca
z(>6#-;HFQ(f6^KemaAn14P`bJfTV?4hIfgaFv?uzS~pv1Mw1HQh8vRFe&6G}r&bK!
zD?`^4&a7}*Ezawjx!=J%6|a=5{ic(48R3rFcF?@zxF{;w^TQ)<b(4bsE8b~p=Vu=B
zs)rPo=lpCH?;u;5Jo)wq=8eaqnK)l}<8p@8)K7JiDJ8K~;j*H%hJ#iDZz}wF<@T^A
z@on=_JTlR%O@HbdGDQPr%J?a+B<jOLM#U%$*Ae#==0-&7`Bn;(1FcUI7QR95M8}MV
ze@9!;M4@1!c1FZu8BJ5g>3}oXD8&!KD_o;4`I3YzjI`uV%bsA@zXw}<nKm0pVS@%Y
z|LAw&7!L&^vDu<$p8&5g)Bgq9XoW5xD0v$Li6X#zjl9D6J7q)-`HS2={t$J^M95g^
zaNm-%jParz6)8a9-ATxPPnao?k^BgHLpvvy19rQ26dmH`XDg}SiBQfTGX11p?+|pm
zB(ce(%etXY2z>EfK_|`KfoYlmml&ONtK5kNMI=&*c6blqgF4NpLK!&{Eqxgt*yGw^
zFIX#)V=vv;M~s>ZkJTxc$f9tHXrsO!G9(^5Yot^vE}OFuPwerbuU$*aS|@lU1QgfO
z@QA&VAIBk&VxZS<f-g~VNGfC=cqPYzwYk3c0-T1)<?TOTddI~cR_=c>W>rY1`4qZL
zWNryBRbHa^_ry4*a>INLHCVj6b9@N%c^$%awaO)<IVyN`4nR)@)p&vfuR2gmd+ivo
zxM;v`IR<}zu+R%rCIkApsqS4pXfephD_7O7Fo@L(<7Y;!f=C+bOb?DFkot_NZnZUx
zSabmie>gk|zf@;jY4+EeucDyVbCJl@mlq#?mH95;p8%A!L9Hk1oPP$qrR1+dHZ}}E
z3^;lqi8=TE#_$EW(kfk=EVoH+XisqPXtV!gSWWSv|MBl+H|VEZ?mBhgc{%#+v;7^@
zvL?7%2?D6p>pX9bIZQ6x*=Ycb9B}$*;7RL&J1f~I;>-Y43%mzb0PSbOh!9xXM;qy{
zRw|anr2F;#h8!r<OXzhqiH4Pp6X6u-yl?;TPflUv1>AD>vaRJ+@HVIcS?qHA1hQQD
zyQIRnN|!t6<u!F(ZK&TRKjBqXfEG$`>7xUd%lH*=LmffA$PEsBSVw&Xpz*S@kPxWZ
z`A5}!<OQ?>K(T~8P%+k>1*;js&37WesyKsR`0z6_GGOQ^P)^A@#Cu8Y4XW%w=j%5W
zj2a=TBn*L+NRQJv<oZt|(6iq^#Z+`0>l`O$vC~klegCHZ7RJYr^DLtMlwy1PN_zb3
zY8z+od(`E(Ad<PjS&nUQaZwmyI5R)BR0<sbM@&hXRhj$J@l?5^<>8HVwj4OXx*8NB
zI{-PA#pEXj&>1uUD22sDNurSvmunVCo0}0e+8dE)y)NJj?ZtnhKc1c_M|vgGmSSP&
zGkAuhL&m*JE-i_w;g!HXB9@5C^ywORYb-kmUjzdba0MmHR&`m=<2`~wzy&NGAoH<0
z3Y;Lr{wm7j(_E;ZEEf$Cl{LWl&#(qTkTM-qn0A37nnF$o#dh228en3E<$JM&82N&w
zb9Y)J9nDBMhucN7ZW=xhAKP(p+4dviV?Y8`{63&+3vUBN%d!vyuB(pzYGHu}Fr*^w
zFy)HP?+IcGLQ!y_HE~7yj}M@)<l{paR%@yHo0^N_0&PKS#h`3QlT%hoE$#hKRqS8P
zWdyu`jcNvv06-XqX$F7%_^|_Q@J9>xcaxHFwDnrlkc1hx$k`Kk;d~d$;ZV_j)Umo^
zWyOpiQztnlKxYe_fTQ)BpRmw)uK|_D>XuL}VKY#b@JB}ZXwo$<R=M)4<TJAl(VpyU
z=?}0p8!)5+|6(x0{2zpu0TT^di^HUmOON&2pKm1uy?&nby1LzYPZ3H=zuPb(gufMH
z5N|&Behsw9EHt_i^JRGPMHm@iQ7-~UEhK6pqNpSvg|=f6*<892)t>OrNChloNXFtd
z<5`bCptuIl!8Mabn7=)A;^NRA_7K{`get^=Kj@lR8ULG$GXulsIOMR-{v%$Ln^z8$
z6a<r<xr$>`FeR(|^GAX(W*Kc>hhY8wlH7@LLVSUS$7rQ9xy;f(_PG%3@YMoq=c>Fj
zv4(|PT44jA<O4gVF0HNx1GO5MICc2P8I>Rt;Z7B=4~}$VLb9P`;u3jQ<G)(OJwU5r
zR>H5BG=S#Q2^U6J{Kt*_g;BwxS}i?5Yq0lUefTfs-+u{qnRE{8e;iAxML@$823;m`
zpFab9hcgRsuAJ#|z$6ocUtKG~Z@|F{SuBzO0ypBIk)`CE4Ah{r1c9^yI&8&|FhyHv
zwWi;pm`TABxQBh>|KF}hg)-abLkL_tg=A%)!Mf#u5V%U%(CfzJl-SpeOol=zSQzhF
z5853Fx0D8Lt<?RigUSPQ%!|MbC`Pr&i&HlWum)``v^#p8zkir||Ibyj{(zN9>uRJ1
z8#;eDVKO>8WT1`R0eEV^fH{>7aK^yH!|NIi0#D*8WP(|O`>ULIX=f|E3Aa^sr{lxJ
z>;ZRdKmrFu4Bc@uKc=qn8v;o|k|?8B;5n+X01KzADLx9;g=o+O2OOV%fMz7cd}&+{
zOlvMKu5mOq!#Ec5?61#_8>ndjHJo{5&(7Z1)~Skx?4imA8;@B10CvlvRQ`=XKt)N6
z$E?lE6Xiiqy3rdxhK<z=QGm4ty{^WB{RhCE0D1!AF(hab5)wdmj_B_0Zu{xz`yD{@
zVWW-sS54?fAp+^i5!+oyrOeYT@0S9>$~F>s<cH;cy4|~^(FQwXj~(3J(+W$qB*69K
zV2ms3H4ukf$E>V9ZAfzPZ)ZpOJ(V%8L|-lOk^^B-#sJ!1xQG1g8GGf2wQ3H$t+CG|
z?>z_}UL)YqLCB_SEQ5f!L78UtQ&5y*Hj!#!-~zkF|L|cH;ydnE>PL;cw7O{WU`5o|
zTVK?ytORDR?%{?!?>>S&85FY#`P82UnA1s2D+ndRVU9XB;JE6OFdAA~2K494WS7X(
zX*f;msLrF?up`KdJ(R{|*Yr$?zzj%15fuYv-`guBboAbfqZOFi9h+_OQEI+dslm3v
zN_WS9d-8kmE_NI+Jfp4<!lVeEZSKdIR_h13Kk6N7fiD=D2}MN5{<W9Cz4^O9Iy$PE
z@D%>>t*@}crpWKI<uKh>p;<MB@1r-IXn<ajQ0}`Qz$G2?iVSZ#QHO?4+p;DY|BX&^
z5!-p2`8DmK-|ifuH?Hb*?~x!K1YX#l+&S+9#aLW)1Fb8<<F~(RDX+ejR+i>agg5P%
z6n$*tv_ZmC(ZL}khPO0&giMCvSs4&3gmjSp-9&_}Z)m8sua&L+(wno<&1^X3*n7mC
zYw-ODPXN7;aA2@#)_htG@IxgcgXz@++h0o{l(RkCSPk<o{Q7#`{M=>oQ|w_a?dgMn
zGkjV{=bH9}^)Eze{<wltFOIQQM;}}~0;l=c&xv>|T++_WNl<G);o<+T$Z|5JU_0ko
z6C10-)@rqjA(o5iQB|e*JKO#Pi*t88o06_XCp&K&%7A~E?Y1NM7WK2;sBX~o=7eI>
z#-Dk{8T8Vfr~|8+3Q8-5`anGSR%I%g;0<>HL=1<fj_<j2;qtTPc`L=PMH8<Q)Z$Ya
zJoip~X$0Wm6#awYk;8;qm@Tp4khn#G{{zV{ROXhf<m4B?>`zCvRLw=wBmnm*a#+VU
zy4HggnnDBII$`@>rH}G}`kqQ5C&%gBHP@>gcZ=w#i|tVO)7Fjn_tEO5MCH_I^u_Ov
zHE#FnqUp7HFR*dU_@nlpQ2(jw@pFse*CO*;RKIh{X-hi}t=JqD(r<ftF>d9V8vEL@
zVx~c#xYAEMG+ycC!e{Q}kB@G3=!h4*4wZobGuJ+Ekqh71j*4efYk_FJS*@<A&{rff
zg%dakL{bvxR#y@gL#QJTF4N~M1#lV)LF<NBuqfcn6TdVkgBKH~!0-&<e9&~qN$lv=
zulya`?qjI$d-R*{H!BnMWP!;$$z+bFgOcH0wuml=4$@|g^tdwH$CKBkJ_Fk+*MUbN
zGXTLH5pQSJmTzU1pvnz@W!U3TV29E}n8JCj)4|}`YIRJ~t(6NgdL??@;!7_IOc$K6
z$;jUdQ`BDv83XBbe|0#}FRfmx1`N>?^3?TxRDrWgrTVGDPqxQcSQ$Kycrf*C|6{=O
z8~=SW<~VKhZsh$06&lA{)SbuE6o4k&CL!(?hs%<)b&vR&l}+T{XBaYv+d_#hx}_bG
z<FLOs#l^<*6RY0ZG0(KyA1p|(WVTUgG{?LC(-2=o0+)e6h#V=g;o*?rqlKiU!-4Np
z<$~|^5DEEFAI*kb7zTH_i@1!WBzG=p7#vK39khXx%12Gcw)<J)>-;_a()3koeq3Zd
zA@*m(RZYr{^B!^hZhbxaR}uq3mpjBipg$5CnaO1}MCxVwd3b-csv?!MEM4(kMSBc&
z=GQ{0>4r6b>BR6~iTng5;<8&r12%uWh0L<od7%X8RjN6)UJEQk+TS$O>WP(dJ#i<_
zI=4$y?q(6Jd=h~~Zp)xvO`l%OvWZqxQ*EWKW^a)OiS>v3ICmi#_5EKJtx{^AHu9{x
zKgYdHG=f(Dw3eyHTjo%fD%WgR&zHulzpk^5*xm-nI7vBqT&*vgaA#x!Qmy!pU9G=a
zlm#pO1f-S!96^w%9|kIL%QwFq_zJ+oxy?F;&U@}KPE_ja1E!a<v-6>E>1(8hJ?c-f
zKuja&%Y!g}8{X^Oml=M>ZC2{crKi;g=S5eb)?4stAk;UHbi@8DsgAw&Ne7R$QgTSv
zyRl{TSBkMN5yd*v_K}U!Np6*yF6Ww+?Ki$r``)WN)de3}Y1oVPrIZ&4EP9>P9`6kC
z$GjWYj{m$BXQ+$w60@LJKivaLrwBC~%8cA8oY!UMwr*#}!qRpn`mAZb?DmQ!)`bl@
z7N4|P_O{X{)usOS%3V5@Y5Vw!N8lxm-c)t={N2U`y?&i&PerwFn)RLOA>u{?S6J*G
zlwxA1TvS3YLd(3|!L-v(*SVYUtO7yVPI(5eN)!E<;6KXzDF1?paLZf+4hSN;u$E&2
z;IlzNNm*+#N&$X4x5yDiy&({|96}3G45YG|HhoiR@gnIdrMJJdQa?U|%Ro9IJS!f2
z;(MHR)f@A{T}d<p0-4Wfk#&nnH=8^k?;_uxu5dM*Gb;~^o@_N0WMEdO9!sS!iK8fk
zeBTX8tFasR71PdSI~Kez{bKNw;bbF1U;RL3l8|Q8y<cjgTAZ(@3R?8X>mvCx1Bj7Q
z#jYurnG0qUiZ^9BFOkEr;+9-B*}f!r6#234B?zR{P0!<#;FYW`%$mj||D@IV72H*|
zLNigj(0pY(#FY3AYS6GgCVuuY+9QH%CYOpLJk+wMSLF9FL3W(aj^|3VR-IGfa#=_x
z^Ec=zw-dW+HA8MI;-@`j`-ZYG>egv4FP4-z8vdtpzeWb?v?yPPPffODxq2|_Rq4L>
zl^T2_D2N=EXH28g)?Mv7g?!&69#pYr|65j4?;&iFDZs-So{HmsSADYbsk_WOnByV=
z4fZOsvfzL)y%n&E1AGRvnAnEBx)~HLsx^KbGV&#IoWe$}MM<ZU_1m`Y481&wW8Uy*
z=<jP0zf&&a1mGn_)wehL$t-;KJf13jgE9~w-1S$LZ(=`{Ywt?Hd+VxN{P79N-)u96
zKSiikiIeDRSP<iqe3PA}TQoO6qnI)ag~C9!d-ns)y(z!V1}*J{u>~`qAMNrnPp>Z4
z2u-k_y;q~Misv{Ti#ImYTDX%RyEN*PluSmc{Mzr!dnX<e8uI5ftxm@*YV-Ad&26SO
z-==<yPO8UT8E9yMSZdT8tye|jKiR*Q6&zZyn7m#~d&25E(~dZQ-Y9VGTH6}k6B!fQ
z4|zK1y1Ys^XTt~)FugiBqZmzFkNJv^SdV^p`J+pFIyIU;_lsHm<>a|){kxh%Bi4g+
z1}{nK;&_*VWWB!K0lWKQnt{Dhimk1}V-DfY->S8z$E>zjy>nkqe=m3?>)IfZf$qix
zY|HSPPqE-H{{GDHl9Cs~al9~}4t!Jq)RY3wkXEZ_Ex+(G=u&X55+4oDs54X<)k!X;
zz<|I-OxjM8>}daVDNRt0PfZNWn_vgFbfBdSJzs{)N4yv+@@>>_Cp+|DD}81wrT2}L
zWHLg<jtjAM(&c%s-{@_9O})PToQJ3jU%N^B-SYj_OfC-Vt2({p;`G#W5_=j%&ikAC
zYP_zDn@m-LH%8(7Yf7Di*Soc3vQu-2F;;z~eC%HP)U7vH6cfdYYCC3M_-|Fjg@Lvw
zik86M{`-R1ZKEh7jRx=1FGN$^_M&F--vH$3wtm<N;RU}976pbgX2v`E_NMA>t?w|u
zDb15AW8b|C|B|k8v^NKpL>hsP{A?2M-R-O2T!wOtl+;h@>s1pE`M!U1Z6>alJc@n!
zlH73H!f#I-`Z0>Qslv_J%d>R%^e=Fhb;9G0b=HZ1+9?z7927mR!o?x@)C<STquz}P
z=5_}j8Tt392WvhBvH)$nMzMp|^9%wmyE4GqE;8x+gisRGd5KR(9_Pm<-5_CbA7{c?
zbB9SxLQq07WUajbMRSoq6b<(eLKt)wHcjI<mbke2k!01S&*zR)P6O2KzP8F0vV|cY
z+j<_Cdse@@W%*8mp!nvEN~oHE6SV%GjmfS*<*$A_ze1AB^??@F{q-)o=hl~yvnLn3
z=+l0B)vDIm=UdTB`ZE)|*QccP*u-&)rt1V)y3^2a1(F1*je;v;Jw<^SJLHCcQ+vz0
z(?;Gf3}(c>reytQPn~W2>G>2kmf7eo$&>kNdBFwAr|&z;1`MF)?)M?q9EaQMXN?=z
zhT`J8Q+~1%G5Op69Mah<r}i_2KG*YPr`syb)sov`%JefgFY`4~&0~CSSkamrZDC7h
z?n?gWc@DI|=KL|X$s14SvfKl~2A*7AUrK*}L^wh>;@_#Z*SjXxVILhhN;NeT2maa%
zpl@;m10^R6FqluCH9zcxfANxE8wSD*iNwUjEJ0@oa6BAn#xTFo@ed9jkt+W67Vfyb
zIQ2QPjN-~Rt}65Tpflxjlo$mJhgzYi=gbz9^SK+cstJVQG{|9P%AEYy?Ma&fd`xwv
z-pl!#p6j)ZXFF+?){gT7$(WDUv#-a|PkxKIx%9u;_>?wZKPgnClfB9lA)omb8u-U!
zv1olzzGftoV++s4>UbgU$xBLzlQdW;l|wvcGk=d4Oz|c;wV}bHH*V)eM+h6y7;LC&
zeq0+d@3F9$<u&pUd%W!@<CV(<2WJef??G~UwIS#^wPAMir*$N?=lz({yBm7Fv~o-u
zv(|`R?KAK_sdH8?Ybo7@pmASZcdk_NjN&gR`Djt^tLd+6S$3qqKzh}*obEXVY`q#$
ziRud4by6B;?$Y|xUn*B5EL*9X5e&{wafd&G<3Wmz{rULA+(VYoh?-273QlvmpnN$)
z3Hk8s$cCkF)_Sdo3uLk_in)=5@UBFXV}H^3_IP5v*h?vo_Y67VF&v_WMgk5jD{4Ew
zY1&+}S?eOmbIN03y`&_U_0c4aM8CaGO6BI5j~_^{R(lH)z*nPMuR7!R1P^AN)y60Q
z)1w!Z6EtkuW)7g!xTobD41;R1!CfB<+r}Ht{i=5Qc7P6r_D=1n534+%4dbu5^%AZc
z%bgT=J{{mA+ny8HpA@C^i9@;*qA>WGpZq8)aj<bcTI#H7HzI6?ubgxq0vQyZ8g(?@
z&69-EHLz~4a={_FQyZYUDlPAeXf^v}j>+ZESogArmlsW|_Is*5HisRR{7_<J7u7%%
zg~#|=56eftL;`)!#KDAj2W{7OZ97^12e42({3$h^GS+SglhxcbCptevC4yH~HP^^;
z16~4xL}q|Gd%Nj(Qdj%S@wi8|W>?V7BX3zsczH{GW3w;GeCE_v_rlF<$?SN&adKm;
z3g4_>cFKgLV54p#nz2-C;{H08$r#cpHbv#~=+Ud^Qlf8F#NHX36eRbz<!8u_hetJ=
zkX+3DZf-15zdJBj(?bqZFB^U4S1-;6dWkl_awMvV9<eEp<3h-f)_Y|*c8~0L&3w=A
z3_@-jKf0c|rkzieepchRjys5CG-lcTq}XlRUuUn$zY=>!BB@qecmHj{TwanPY9aeM
zNO4H7O>_jkT0;i|{n`$x$>Ut!#qKIwnpS3>gH5PjDE!o;*7KNTuLjz|qCNE~&PuO2
zTw3DRt)#KVdg&RMCDo6Rk-5$Vs$dzv9?CO9PFXO`)B$1wb3TJc9gdBvcY^_y{^Xes
zF=+c<Z#6(7mh@2}si3y&4-(BOQx%zmct=^E0o3d@dYjwLU;iBG^k#8k*HVzlN<<g*
zN1DAJXPPKe6DM51lQ+X%7^yAbY;wuD^_?rbE<-39FeMq->0V(T{$0JYF*Qz?VwpHx
zUq1g$J<XL=WsQ|;qVBEDI-Lg<<jx%U>z;i#(lww$fDXRM(5xUs4x?(f9nsk_98Ib(
zRjsy4IbA>N^tNtNiD%(8y=^mh&~ClyGBy8_p)G8*k&#<6b%bzzvq7?*-@Y(^RQ7E>
ziVFl8R7)BS0nK=t4IO`C;Ko?hks;DsGj^rD({Q>L1IpONa%<nLH0BsQiI*LT(?JTl
z56hB<eQB&<yNz`~W!sfB5KStlpr8Qs6bfDUr4YSIv9}9g1s_IH)^ba9yunC&*)C<K
zcUE_5tm!_=qbmbJN~bcwQ?c`zxSxt(P2+ersP)vtwvh@lXx{5!t>XXE#jp-v;tYD2
zy~kS0j>^_scU^ybS9RUAeRSRQFkndwm5S|#nbU<Sick2NE4<_|k5f2a2`JLYe^Cnk
zFrk|5oZurTF1*}Hty=58{RJA-evijtTj6sQ!D`KUvIS1jtvqDM25+&j=!z}rCzmz7
z0xYW=YBWuz<A)ezbt^8b&rT%$_T1v;-Pil8!*APsE46eu=jW4rf`c`6W*`t*2F4Ym
zGK~Z7(%oo&DJ!?rw6i8$H#DnUbqQTx=u!H-34~>PlBrTvaCYFthv#h->0@SiInDdI
zis}M~X$v3BE*srtD^2~Ubo*lOaDcNn7jkT@gR(vLD{!P$5DEb4QL$JA9<Rfqe@`@V
z>GeT-yFZ80EItT%-r17Ga#$M8KJCzhKb0t>RKK$o(P%=%8Nh^pw+KaS9}CsLqpsFC
zX_`7OH6h7`3@63@X*&L>cGIS8Rzh8!RXfGBrJ}wd_re?c!>{^&SfUNuoOx%ZH@$Gz
z?jvL!a?kT+G%#O#w>V?JIlr}4vc_jDQ!tuXuz3HLW^;0a4_B8}SpP=f&&5`KrefjE
z$ILgjiLGWo-cvFwhADMV<cU)yM!sQNrlce{EFo}rVQV#G?APfhz%_!3H|&g7p3ZN)
z+AWp!vCd*T(7PXbg=d3?tlTGfcbTi_zGL3p1`<0%8gJHuAXYi~7ItND7%NDtu*~py
zcbqehcCeX*JEM|qgG@EpB5x`qG2J_PQ>C7ScBwGqMl*FV)P!wz8&)>6scyDYtiJxI
zIm6=+i{E`C1)-Zud&mBy3U5)b*)Y>*a6Gsx5`l1Hrm0|RkeAY|6%*_jhpio$)}uQj
zy-F(8N-r1G<)XlXpbaP-xmoxI?046S8s4B4{IRq?J~0`FSp?6*#PKWk=bEkeXB)T2
z3u6F8yzctRKP)p?w8I$z4f)aIV}paj1<vVQN?wj)%Moj$M?*Qt;7e%&Y@4Ht`lwc%
z->l_j+%CwRbP$`97;HE<mLZ2lhU^b*i1(O54I7MwUh(tfg!U)$9Fy^{BwzZlGU}S~
zBOPbW*qEi#vem|?EcZobz1D6`UE4)GI!j{^983_!*T4TP3)f|XJ0Mmpd`Y|aC((rY
z%;s#)TNkzU_=aQMbp102xuI+7DG`Cbg48;BYl>07&4!ndek>+df!uNB+T9Pq3H*g}
z7W?vAx=xj5aZO?=>A0|N8&&$xzk56=cU?GkiJ>uWWth}%nmb;}LuK`bc(t4nnw1wm
zN&JZsbjDZZpK!0wNoUK<qDua3q>`FBi%M7%hcWU!pSj-F8aGO0(9|(wj|^+}PQj`>
zzFZzEgA7D4-Of+IK)@<hw4uvYD1(H1*rR{uYGULMkfP$On+`Yx1gaPSMmo&d8V4IY
z_u|4`wM>IWlG0JFnypRswbR$?WeNg#6bgqM(bO{{-V!aSP7gSOEP4mT-DhO`wXrPt
zUf7BwCy%1>*e(p}6k6{mJ42VO5_2cM>WOiDyK}4Oz>W3zlR4wcO{Yp(=0YfNYgVgN
zK9Uw$?p%z7wl|}MzA!!|$7!-Vay28Xc~W24J|JshLd1qDl15j!-%@<4iGDtG6d4@s
z@pa*>Q^TwEBl{~BBhGl0Ek9G`R+e@8#E;La&rNS$=i&@nAXJxMS~f6!Enw(x({<Z0
zxi#<60+~~sfZuJ;ZfLovd}^)1nOo|GYRji`MUt0kn)SwuszdB!sZISCrVI1Q4WhYv
zsrk2RIn>5%J@))wi5yN8(n$+{vlE@UbKV_G(AM=&tquK_ee%K2j0&qxQ!0+FkK+9>
z;fPK?09ax*yJo-vHfZD~tLlKMhPmCK-_~koxqZjUq$X_PA^8Hqr~$+r*1IM;9k+@P
z;Elk4=?FwX{SE3A*Q6nWH-|x<XS*+*S-5MzLm=tX)m<U5w+2{Fn)O)+{FxS4yQgO#
zL$(gk>e?|6EIsT;WO!YL3G0czN=PJ!d>WEOmgiziH5!Ko@p`c8Mlls78^ithofH?l
zIac@C%=^u6>A>`{lG^6Ih0`6CuM!0Ap1SrUY|3)Bvj7EC+-`y*#cEZVj-(84d%M7J
zExvQ=V$C1)Dau?a$FdpZ0)|Oi>=mAEMrQ(SSCqa(cB<8%FxOKrrv_`}&8}#wTTe9X
z=lJrz+@_NL_i=0f;iG%RKPP~fJ1sm=5HqPHgsK0+498(kmVr5Qc9sxi<oXkKM+(?c
zYn9po9H+0fPxbF>)J$_IPXP|Y8X?4I2w0#F!m@XfOQzQF-A|0$>anQ1cS9<Nim*Yr
zc}U`ro;@IEa4T|oxBxl(1$TROIt~;!3P3}k<=xFC^rqEZ9uebLl-t`CNy&z+cw|f%
zgwa9yxVr7fBBnAsIKLB|n~-{4QO2M=y^w-H{!YI2vAdKM)U@_)xjVsas$CGfd^I=)
zUL-ACGdi)drS*0_R`(CoHBTLQ;Jq&K0Bm<O5?UgIcbnINNa~#hH1=tEd9~Va&(gMk
z7h}inIls>Z7jHZdBZgg^tXXKf*x75l;FTBWpf9b(LVWB`{mGQAd|$mfu~y!R0sXhb
z;*SD^F(s>@^C_+LqZv)*Z);IXv}@BPlsj;xR~nGukS=s#7e;PhFwp^_G*I0DxV%}-
zw=@EMog8FrY75{C@|`d;NLx<0^ZOSWi+^uN1MUaOy|}$^i^|;eBs7K;?K^k`3?!=n
zY|FwL&jCH#1;4K7T<0((j!N5dHf4I&qLrK@FdI`qk_Q0eIl6Q(J1bz82I@c`j>BzV
zN7rr6B^5}5>^uvkGQh(6KBim)Vhrrv9Ka94{V^lo(Xe*hh>bOxR~4p%J$ncI8L<?!
zzl)#uvu7s|qp3F1<S0>8T*@YY3{^@6z<(RakKPJ*CY@uuko)@w!Dsm0dB8j$!^6Tx
zK>=PhTNnXRY@<v8K*ptlVgJju^Km^=UZz0#Dc^I89~Doe<ps+>9=`6LC<96ixpTro
z=|~MqWM%WT>OHoBOlE78a?PBca4_>+CydwDVl}oiYM_k#3sBW2KsB3a*u;$tq+m$F
zdRb`y#_mGD<4ptG`$`G1uSjM0wT*CaNH_%@qY2nhi?F%rOy}7)oBT1Vi85DHfdiCk
z>;^oz5LzMO9E_?)09KM=Y{Jpe(ZMh?HQ-7OYUP%*4UPfxCe!A~dA2V?p8Si{NZcBW
z$){iUlU!zS39xdQ-dTr*r4h|e1%L35K&ad2%i+*I?=NG+w0fsF3DSg#5*=taMR2q~
z8v+=OYW*-2F!YDg2EfsdfX}rI^poYS4J4I{hGFSVh-7!-l94Mm%<C||h94AE!FD56
znk!ku>CyQSFiKYgp!XYUgPyYyy~e&&>)X863+o|i(;HVtdYB1txKhVW)OmC5<|!xG
z;MGozaDJS;6PqH|A$;5pOH@%`CBhc+cN&<L59k??F*G#PYH-K~VzId1ZyRd5ArPJE
zBPMjO)6e|VCRCtek)Srvq^eu-_5#eVrvYCOKh@a$kQ2j0S3Wm$*|kS+Cbu&J^$a%@
zA}Kl5ts*41r}Dp%xM3i`55D9>*z}Tz1(rO}w`d6zI6ben3QvJ>kEc@PdpJ@5U{QZ(
z_#s;M0FxcU&T4h)euk$|s*2A!GC0gN*6+!|G|Zw!z`JP^LyXBUGPFUMs8q|QP3x@v
zuRZtUN^k~r?-qxBvO^ONs0~bj1hNub5Ol73prE0-sQEvS`S9;%hA}nld_>$sPv0%2
z;tmvfe4*bUGKbj%FRBDQfd*Cz(%94%mayeQC^|3GhKhaO>HM2d+^XPdl)XMTz@7#j
z0pvaf!1`iLj~N^qaIJo21%y)ZFe-o>ftQy572S!!?aiPHjL_92t1ztlKTV(grV4S?
ztpHE}5`1{a*ePGl&M!r;>`Aj)dPM1J3)l_7LMz!~99X+C$=onAQ-=bW=m#!7ej(6O
z*ao7V37WrC882Yh(9aw-IzYgqvKep7GttFR){81pKWU?ohP~Fe^bYK6!RuwyHuY`R
z<688%eKjs`rxV%#nMglY2S;iL(|T-na5V>U10Y|T%I%oL>$084^M0Kc0)bB@14|k+
z_=|XaD;5kI_tmV|?fxM=(dt1Q68bk~4Hpsu0>dVPn%#Wvkk|8Kt0^Tb-FB^Cy&{jm
zLkPWvKO>cb*4;|{ZV;!sKIjVk4Kw-%f|}!`8O44;tMRT?!5^x(TT_c*F=GKMX)U@1
z6e|~Q!+fSyR1%g~R#YqX_}lg4Q7=HW4))V2<~W33*IGalaC)Qa2o>bqQDMY)N6Qm~
zU5y+$FXK(V<|{6`J(VV2&XVlEqf{tc{W`EfgY+gKvK@0MNi9{wk4CLhgZGd?D&ikz
z_*OXZ9tZrI4|jcjP+w>@-Rf1Z=dp&1Mk+9ykM*6@wWl6M{Q2`cjgOCjrd{10Uk+}X
zD;+(B&Rkh7rEoazz@Bq744;U`*Jw9xhuSSvI#E%&IL4<&$0R)h^Acz@eNvXUGF>Lw
z<98Yq$PY}+vsG(GQJ!9d@9~8XS9sWkC|YhCz+1q{2MPgCU@G{4O8pLK`l+yx?Bbfb
z;cQMrAShiDOJtam^-pVuwNi>!z|(6(qZDmIHpwh`hF~Sc8yW}t+nV+Ac^e<<&vla%
zOCvn@n##bZ7e2Hj`c_J-)umENzCd#v6JNQtWKFeQ%cJcl7SWu<D_sg$ni(u(=h7+q
zD4+~Zxj=RljcYbbDPMX@nPUP-&FmQJA%d+WjBMw1>Q1763eDVLk{&g^@Rt=U4^>1k
z2_8MBK}qKM_>y_OD}#y}ykxb-JdQ-&PN>iHw}aS0(LE~4aP3?w`ck7UTKm#GBCriU
zJUKGyyqpTyn!b~LwO<h6upAQw`fU?{G~jYR=Y}X!ZE4H|vJ$87OR>fv_EB>k@aJt}
zVX}%GPQkAEuw4N>=-uM4QJij?{klm~wvCig6Gsls6J9R1XmW)Y|8klhpB@e{Q~lv#
zzVkjeN4?uq)rGz{M-l~AC|PYsb#wqvOUwm#^>69H)*rWKhj+2*N<|Q;Z8g59=hwwt
zGTa_*i54#3O8*1mkOf$Lv9YwNd@-dI{z7p`-oR`db`%257d2KB^en2%an3;_%+kW0
z8+j@KJ@`Lg{Tz$icX=>p5)`;fKnz0^6Q@Oj&wSXZzq?J28Nqe}K#Y-cbab3<aw*mG
zJ`#%p{IF6_S{!${m&nND-{OLf?eu%@FE?DNPge_q-tY5XEXG$q$`s_8Nu~t9#y`4m
zPmdo=xqbLOvy`VVed<6wGt$NFV)p0ld<SA?I=uH$W1OM{)chrn?nu~mgytpEv1S)6
z&E?iJ`1959EEOg;GXWeYaj~$LFOui5450VeBUikI7d$jHDQwqA-)U$b??<JZT-b+t
z1$j}@Jg%Q5MBEXZfnVc<m?||;oela|w~(EIrgo385f6oLyPY(sJsx_??@1CEnh*N|
z9W#<H(h;c!DmqX4oHQp}A>>O<Of&}Yvvgk#zQeif&uYT<B4DVI%3-}Ohm-@`&i=jh
z!^VzFEG&U`B;1E26!y-Z11x&UUk(BfW`iyn2Zt)Yo_jVH7S_oh;}}bDYHGw^r2Sv7
z;1UL1qcg~f{pUN?Q9&ehImdnjw~z6H$0@HT@?|cNGO-2$onEh{fe(&G`ajnO3tAWx
zY{#^Nup|;ilZ@vyFNdNN^|C|<xUr`*2?=*@@ymK6HRzBD2<QP>ekcfNz-Q!*%kH1i
zHHzSv|IP$Zg78!N)}KTVYniy@u)xCdUCVcZ$DoA^xRvVV@cc{U5>}f3YMNa^)ZGHJ
zCQydL-ya*6EmWz8fi1!2aeT1FLR;%+CfyQXAh(mKuJ`u;q*NVLaN|k0+4l^3h2Q7W
zN=0(FeWOx?gY|)~3CIbA48Et9Us<TNjLB!JU2b9#{K_v*ovcuTXI%OOVlX>M%;k0z
zm)|}LkPWCZ?zIA0Nr`540WALkA;FYn1py2tMiJamSo5C!F)W-aq)9e|cN36>abddW
z+Il@+NpBt%>u#ms783g5qI?blxe=G?2AOy;oc0+>T|8D=`GT3XW_`((c39XqD$0nj
zsyd)p90*cl(XZ!ZdlHWVj!|qD+UE4wD8NbpI>ySG0zSiDBz&-lDugA;lTr{zI{bSy
zA>K33o@Sy<g*rA>Li^zsu()k&&r;+!`rars*NIS`wr#?-*JcI>FIW`601vORSi>P~
z1nn1nr(2`8Za?w!@U`w-TXqviX*q_Ve5iU?_IuzSe;5zequ_TXMi7+CJAkTSC6F=`
zUOxx)LA{!&rERd9{X^lD{WMGMlPV?=Sk_J#UCKOG$O#K!djSs$#Ac>2senl-Y<bDk
z)4?q<oFQYS5rGG!m1Ld?Vvhn0Fu0u=#Nsz9#wrQjriRK*eH_hVIjBhVvyeZ`=SCsH
ze+JR^KZF_qjEnU&d*^Okb=cX&8d|1kV^fB1m5%~Ta@yfS<CRbUNU$Y({!JihS~8j_
zR~l}|&vS+SBNbewoOYRN+IbNu<~CKmPuI_(CSSr=qX&2J2<P>9gxohQY{W9L$|0n%
z{UPg}5?8dnIPAxNbW(#K!94~^FvTOj5NeRjpTqU_^?{0xvXN2FAOY6-f?xZ2q9PVp
zkhG!!F=oBi^V^JPvn_7A&?bz^!F<><HAmg$j?Ej5-<U=p&LH~pF0o(q@(nm>2+-sU
zgALi>@i9WGr3`cnphWciw>A5XEm2y(4ZrU^+GkDuAS47pL8PMnEEX_l6aqp?JSBl&
zW($Z0#h{u;%g$btrK$u<f*6O*9Th<mr>m<xXD;Gsn+5cyGFP{$u!9wbLwpA!o8NdK
zhxWvF;ETDVBsBr_FAudS3^3D#3})IWwXS-f>yEc6V$^)520=Uvc5X<>$O3S%f<XXa
zHcf?nJUMX&1Vu@})unUzq5|6(a6WYGj3-%zaFE&qEZkXn(k#SL>LHzLknR}tx%(qg
z?M$(kaOWsOGf}JrEj)^kJ*1;ae4u-QSrY_X!wv6|(E#w7F5sgD5;`b=6@ViB>nK95
z;de_gAQZl;<AW~Z=}iJcI1>8Ac#vn2vx+E&gvdB-`gh<aUkzyaG&{F4eFO%vl{=f`
z9{{E7|8b3$+q0P_@PmtuZOm+<TOj+AD@48wtlutRc7IgOx%(yy1esrnwuJDG5O&xx
z7s)0dA>NYUa!HnoSkQ3DZGJlSprvGlKvuZZJV6D_jEAl4IVhzr9X_Xi13&w3@3#4)
z0OZGi0S8r=l_;(zavRt=01@e?&N8XTuUTc3isIyCaUVu05K;+-4J3R;$8KU+YQ1N~
zm78VB)kObwSvCN9#id~B9PBEK4j>XeKQQPN=|F=sGYNi4?g-T9wn0YkCi1fj?5WQ&
zxNW-+S9_~#$F+WW`kxw<X;)7_{}_$|3j`f_lJy0b3}zfv8Z5`-ZiL9l3z|wW4x2ie
z<2kvQ1g?6>Chf*kDHDJm7lPIHCXD+rfCV%Sw#M>dvhY9!4u_P~J|+Qa^=u4=S^$TT
zo;gGgJ^@7$PragF7z+YK1D?cs4~(e`by$AtQw+v0P=C>6nl37pT|(lM@?NpDm#&67
z`OPii4Y3S7k~={3I#JofLB48$lpIn9<RvP*Bk(~%pJeR)2tWP{v|NCOJUl)4qXG+z
zamhZl2AJr=u$_i_NH@QGr*0<S_?4)&Lj5`nS-`JV1UpfnCJ3Fc&q-`0I@aJ0T8^Xb
zn)f)ndwS@}$=~KnCptAEE&Zc*!M+<O1Uzq!!bU2X29a>qb^GJJ@71hEmW-h%9!eY3
zwa&Jo;^AQC^*C67^znb)^C&72ud^iJ1^=3F^#a)%<n&;neUnfO5bFNBJL}MaCrq4^
z`vu-#en+43)|Km)=k4Ln(|pZJ840l>{X%LhWkKiw0Uby-zC%EaN%oAW1FNr~11REh
z*v)dg>`kiz-=N{y5!C<Lbir1u4Ia=i;ypiD*?}FyK@uE}t}W05ZiCQqaJrcLU7JFm
zXgF$3bHdC*77L<xSh~p{u-QI-Wq?=NwwV)Aw)Ca@1W2frfIiU2Gin5|ROdg_%o|Lz
z;R9m|FkkPH((6ie8_Y@t?meQS^V1dFEk0IQO)4bSm2G6ADuJbm(+Iws;sA~S4CEy)
z(EN^AA4)3(b5b7|(hk`>4#SX%{|rY*AxJ;H`=w05SO^E?s<AE~BeYu^c5>=Y)df^o
zjCX{f1V{SwflRU?H!}?DaW9f=I=5obQ#7SSdes^rMw>{d+{03Z2~eZ|%n2a}aArdq
z_RXCDug)FL53Jtl!~yz20=n<ufreofszF~WdUzr_huuFcUgSH(e6AM2|C)f;xrlNF
zLi^u4dlZm^3EP28WyII7mjQP6PW-=j?NAK1z9iQ-hLm7##0cIsWO#1V)dG|y0*~?A
zWGqq-H*8Y;_kIkJ!9jn5a6D=ISCvXsTd{Uqdy0%O#n-EN=zl-52)pIkwrns8v0^ZR
zQma2-v!h7`D?qP&5g0Vh=)bU?1W6|V_)c3-efZK;q!$TJR;aTd##q4(cXY$8S6>y&
zuUlQ;K~7s6!k8Mr+O@GuO0Gsk#Q*2y4?Y8sirLHHXrb50&xp7cfNQVP{oU=}!uQ{<
z)1a`vkEc=Nnnrz7BmB-<6?*t;x`{Ao6BbP$;c3)*#OQsg)1adKae9(P0w(@>LoEdI
zreIn0Kf~n!-Z^s8!lU3Aq1Q;sTPZ;BI}3|Snii}rpqTjVgW0$^_@t0!;n?Y)E=^v*
zQmNN*%AV#bi$;Nsl+Yv87Iv37th&ANe8!!vtLP|xJ34T-FXo0pnujSJ^FR7h+iuWx
zC$r<E#qiEGTVYZ54xm0u`ubKN7lK1cUOJ|Jc{IS>Zn63^?u-Bnq3qK#?3=TZZ@|)e
zcj*C6{S4)Cx4hM7@3Q+Ikdh9*fO0Q%>p}Bl90Dw(a;i~X+#DHq70mwu6%LC#tQQy*
zy_fB*O|%{P!e*-+(OBM=$cGg_`(NOYe<eJ%Tzb*lpr>@y)Z#E!YTqX+A61T1*z5Jh
zm6YQzmJm#*JHBpvHA7cnM%Att)Bo^NyeSX&<=eNDYG)=Y-j><zO^oGAOG&zSbxDC6
z_m@W)L<)=9DEob&eLj5(tyI(Fa<Q&RcFOnmB&G*7rUfu*4$En3LGpnPbI;xjDXX`g
zDF8HBc@aQ_Fx-Qk8j&v4J0c@rQ$h$-D~mAI>(39miyna54s$rORDQO(`Qj62{;~RM
zD>eFfbC8Apg)D;;f>4J&oD9@{vhdh%GHz8RB;>=N@qOL-rnb)VNl-BX>k1GNE9EUP
z;gD|V4p-yltpHX5Jbx2EGiY|I(9FIBpJ9hXjL|}`7D&x!-!X*#fc_}cB6%~2lDcRC
zJ7+Ay1>8_4N=Y^>7Lb8Sgk<rUjs9{V1UdEm+Dwy8u+%B#6TzR_VuDG71ScUkqNw-z
zV1lW;^1Y?RxPSwDf;xx+CgM_qX<YJOcwJC##K%F-B(x|WaBhB6kcnMa^6pk+&b8Ql
z+{Zr@7RNBS;nu%Hf5w<?YxGN@wda(lLQB%E0dBt9|KlzApIN`)IRPsgmZ^a?be(Ej
zzIA@3H!GT-OMz03rh^2qR2Va=8C?QXDC;9UjK=dW`Sn|%emYT@>8mALzW5C`+wKqs
z1${HFOe$*0tXTMYzNXQCj_&uPN_YtVv`zt;zB5H5WpqL%i80WX+vZhXt)yVlI5&JW
z*auh<#>VA$dT|LkZ2NfsKgQlVEX(Lw7nTwb>6Vs8Qo6f41SF(HQd+t}y1S9?25Ata
zyFsM88v&`aUVgRrIeUNC_lMWTBfQVM)~vN=&D?X(Ou7_rUSW_8==xkTkPtS+7l;dN
zE(aj@kdcfx<lZ2e3UHrQfX2ak^{N=0u|mM4=T7Ul0&^iffrXsb5{_f#QJgPR6{<Qb
z>`^#R=9M!=lp~$4qC!dzlHtI952G!xHU_H*5rYZMxSsC4<R1?dH9MDd*r-U;>8;w>
z?|B=Tp%Ky;ghbXXQJ2WTNk!1{yxB|<@Mr?{%>`QT&IX~zUZ+4l>r7Q?b?e!BGA#nY
z&dH<>el@!41`{}L!1V`XB85}XS}sMdC;qru7ai3n8?+9Q*EnKYEyfVNjjqFk4!+L;
zm|k`02{2nj)bv3??!EDxK$%3wGLXMh2z)^Cd1A&RYM(O6J2jT+*Gjd}wA2%|riYGi
z(9z3hE=#aK8#VAiw8_8kEpP8;So1iPYiDUGSa;lUA=m|So4*%kb!L+ifO$Nb#A0;O
zM;_Tg?OZPoP>9d`3N8Qb(~$S9Sc9wY0Vm%RT&giYa;0P0Y*vJT)nE>SF?7n={;{3p
z38j!>o1grP%ln_39*w2}T!p>_DgXm-5M2R{tHC*m1~ssAot9iA>z7<hc%hvv;wUH<
z=Fhvr|1<1={`3GbZkXQh0Ux{oF^q^;om?VJNk=CQpned@p%yKex&*wx^i|7$JpX@N
z883jNoxKt;1*9>8%jJGuxi?)>0J2>y09^`#!kqg#y7_}|_Jl_h{O>{o-+rwHjI|2!
zB2?h8)0~|AMBQ)P&QJwS1K2EP=&1vw<bDAg9aNy_|9?J7s>BQg26vtVz&b8H{n>Ke
zKRZDi8y^`yv}ghs!Paw3j3zp2h4J6x{ExKU-vGO)K4S`UO=C6yJ$5;7QL)>!Up^T}
ztD1&Oik`0L?ca5yI~V%@ND9Fj@>y-}0iX;CU@<T;rz;Ia^YZe{K(%D^iQHg<PAUm<
z0O8&K$MX31PbXzSvn;!+5P$*8t|Vw%je|Zh1prLm_d^x1QMVC`Z3aSK@PLrd{_pLv
z0gPHP&u%Uf5b=Kv8Y+APP&XX~h1k}Y#ilQd7(BMGfG`%||L>0h?QRVOcs&z44^%BT
zG(?L4zq!2mxJ?^SjGPqo#nvEksnIm9r^EB_u|7%dL=7xXfl3<CYgQPbXK<rdy;WIX
zpVU3*VqdIYmXg{@t^WUKvZ5=XG}SDK3SItyLEX`N{F}(?`#R(f_?3N`xF-Lxh5vn^
zTp+)CI=CwwKA@g=3CP{0(`~8+?LiA)VAC~(dpn1NQ6Is8^Zwt5EcBxuoDuhZ>CZqD
z-$8@wp3@7B_Uz_UH2kMsm>nc&=+4w&0eLxP{^<n%S5b3AzE(Qm9vnsqACL!EY%-eh
zr!WZg6QkwfnGb*2WeLnf8<Kwy^FPvVgBS?fisquvd{%J&Wr&(&xJaN1-r3l|i|!*)
zfkUG28TUV&pK1Vj^)Yt3fpcH-EQcJNQ0^{yk=O8e|4ab+FIon|4g+^FMuZ_4A{Uf|
zBv-jC5)K+T`8Ue{gHq@NC-ZeP!ZEtgk8tR)2pgk{ZFE-PU3qhb{nu&#V;UQ(V225G
zaDhvkkT!~i2U5@6bt77m5T9Il^FP1)zXjeW3O+_GL<lC<i^R;+d8&J$>kS}W`y5Eb
z?!S#4*qr}y2CUHa9sIRjc`|KdqWY-u3jU{E05E_)n4q6vuBJX=<LDp^rvX3@Q1q0D
zh^b=}^2Gv`T-Xq$|KB&ki4fSg@KR~MAQXTNzQ(Be`-)F=6iove`+pe`nh<o(2Um{K
znXp%<HcQ(|isceq+XLk8C;if-c!FMbA(XTC0Po&hB?aeIQy)UFnXfCsBU)WiV3aZy
zP>>2!{Gb^%ql5znV3YoXUzrG+;}bE(#u7UO>J?%ZFkST=DdHSGxQbUi(l}jIj-0=B
zF5zS1GQX&98O_Sj371&)Q$&HjyNZVy%4|kjq~4uaAlMiN6{D64l>va5MmpB)WbFYm
zSV0gnF2i{Zk=kGEB_y=FDl5My0_J?ifOdh1NL;%yzs(mk5HI=k#aV(u14U$BM^_E}
z56<7>C4Y~&as3TEKl(%sSrBlYKV07djv@>PU<>~khX$$fkmM#%t(*4qQp+!5%GDij
zM8-#Oiq5Ph5P@ue1Va0yMWLbAy3b8QEWL8HlU^7G>y_)fKN8_@!Qy;^@^bf#jj}l>
z5<fs_&{zo|uo?r3^kT&ib?;_7(}@W-dYc~XL=%Dti1@#ff}?;I)d;egNR>39!5BB?
ze*<eYM*<%Rvc_9bpdDCiHxaKL0sah8bqx(%*i0XEKxujv&_ZSrQFy8ma%2_*bHIGO
zGa>p7KWPu*Ky+d~4Q*agq?3Ss0Q@o)?(XzPUZiu_39E{HMZpJ<3U+|99Ruw~%TCvO
zn=yR~&jDFIx*pvGGBi(!Y$bJyA*b|(_74)&+LmC@)r!+R4nmGYx?Vp|_4iLbrZJTK
z;Q<{S99*o|-bQo{64%}}JAS?*L#r|YV&k)4qlbu15&V1yD_aK<x-!D`x=#Y9EjYaY
zlgt#KVE)3ORO`0)pksX7?X#dcd!W!Wv~8(?lvd!u&z?eg5a9b;A?)3$(3{MvFxTDp
z&0U4P+M6{LPp5|w=cISr35|l{f#lh<$44%*Earc8$vQ}fbH@BN+k<*^q_VJx6WDzp
z$s5W`ucb*d!Ep4odvS5`vw^Ksi0U_`1>H98`sG|q3sz|=`PeG~)ZwA@lJ?}KvGWmr
z>P{wFbPNRVX9#mrkcQ210v@h^&o+^C8|}VUn~Yi+g%%!7z!JCA?O)`3@qL>8GlmEG
z;|3ldMwU?SO)L+p2#oc!Q^uXR^`GB_p1}(Jphavc!P@%S`Il&`{u#_OC^Q5USoeSo
zbn-vDo5su0Ze4ZOZm()_pnUzGtBq42G?4j#wsxT)8XyURa(UoiBN>$3du%8KjAENC
z=BJURK7&Pc!xVQTeFja6#sSO5YFd<r0*e^`Lx~W4;US{wcTdqMn_Cm4XF@+15L#T$
z1Niv(K${LH2Vk$!X`)%PH+6^C8^@qgSY`z_LEfUz3f|8_E(Qa6Si$<oD~bNsR|4}X
z0qh7#psfm8-L9It(^#Yl5QU;D(MgN_o*{U9e}*NMn(INW6NX6OJ%Z_{F%@oM1PnT#
zEH3cd-<V^T_W`7<e@1bhd3z6HUjI2v4I(<-87H4_w5NVouJdbiI}CB&2TYS!NDT97
zfc#*9Gc}^}Xh4ECRM08-wQ33&zWy@$G$>%ie_+|P+sULF?rY$lk*=J<;xGUwLaQzt
z+>4U{@6z>bOZt|M00xLD#RdZnUO0p9O=>{IIRy;`UT8x8Em>H0Q;bZi5H?^9{MJfQ
zsOFNu*Z;Sm_OoZujEdf0gMbRbM8X8vtj*1hp|9iXXTqHdq~k9k>!B(6%-8?Rf)Wxe
zSn#xP?@!Me;xWIHW?|rk0AV!nLk-VZBVbrPT0(&*HXi_yDAk!y)4VHIbGSOv4Z)xc
zksZa2tsw_9bo39-hwK_5$aMFTvz|lV&qWS3{8EdK8cA4$k{Jsg%-;<<rED3EV&UNz
z^fNRYGiYF#<-owBfGTY3g`kf;Bfo~+$OvQ!@M%(DAT#^|9tzjLPC!JtQVLn#_pGa2
z&1FSjd7l0&;h*x`JolIO^VUY=a{EJ%O|KcX))OuZ+Onf%TCv#5O$b0<8Az@L1$pHg
zM6ME2vV0i7gm@?HX)x%8XvoYh^bPO72fW&Y912z+r}zUwHw277CLuUrE2Q}e0i47X
zjMmTu_Bcui8tNTnT%BJ#A%oDjhDM_}kjFao0Xs+t4`ZzJoAIy%sg$ND@)yYKlf7~F
zIbji<JhPdd&}?*}0kJ4YF8Qr82p8>+r1Qq{d@-L*r~j0X3`93?BVO|KPBQR-<14}f
z*)|BMu(ix!;$SgmlthWZe>qWo>;8)@+?k$9GeiWdlPFa2q1ALES9&U297zlGIoVgh
zVx5ct+wgEM+ZghSN^kIhM4E*FSr>m?Xf3?j(qvv(M2o^K6tJ;*F~<xc8+$|%YYPRs
zTzUnZIYY@V;9>9H-JHj9T7Q^LqxqD_02$$#_&#KWXfMD+*0av1*CQzYV}xG++X!QS
z0KB^hbo5dKz&eDpY@vWrGm!xqA$|`0(<H&E2}WC4dD3D2oKGV(0Gsudc_G;ND<iPA
znvjEXj`_9G&wC&J`sSS4csL~<)Qu1);IRBQUuV(0M8QJdi{?$biZ`+Y2iem@;KAVe
z%WBAJCgpot7-abWZHSRS5*V}#0pA9qI>}^glq@LFDnq~sODePep;$w~gIly1@6*nM
z{l|<M{@2$DeSLuqB99&$`}?wlbFUC#OR2zg%pD35Am{iGDR>~p8vn;ZL4;xe2c=+(
z)i$nhbzN7<SRQg*z_s?zbq*(?kpt@-_5&y`T2R;#i`UW8_4<SisEE<-PeHp}abReH
zYZ4|;dq?WWdpJbA+Ly;zBaoT@fQAT@en<U1?0$XB^N1&DI6w=Cjs}Scw6=&amFC03
zBC^*i;efwQMy$17&H(+zWr6QPj$#=O3(WyhIhX=!&mkkyhG@(9Z~HTkH|MzNT=ubV
zO{9c_F}rOSgAG5CuDpKIz#A}-6CWOG#4wA;oB|KUi}s{94Fa3}sKByxru_)TW$OF+
z(@((bLC(|DGY;@*m`O07qm6*KBW!b`{RFc|I0k3T|BAP8@tWZKpFDPgNab2n<(-im
zv$(e|(1$$s`h%!2AI3uJB*R?8uxMXj9_XaMmI%{#K*lc@21JJvHj{CIfhb0N<WuvV
zSFtKXBCLK1*{HH8WKx<U{kF!H3A_>Pu68e>G;{{pAxfBQD@%+C2yh+)gtjS=3Av(*
zH9%aj-fH?gYwZy+7)?X3M-M#^Us?hXcke<9W`XrYUdaR2^#O9;rIg&)Gm~7)WL55b
z`!{a=Nso$6_Do7sORb*F@90x|2NKds=0`lXi_GuQso*b9Pv3^1^B4D9{26F*-W2e-
zzNVbosy2}pKV9#Bt9^96OW?3MFY5YdjX^y0Rr1-E40T3HgRNmwS<au%8`-O!$?Xb}
zFq!@6uEM{A8pr*|>fU{i@~-*hk`W)(dXnGblyOOwHOb`^VpK5syh$@Nb;J*asP`Ou
zV_|5E#(_ZLpKF?f41u9_3wG)T?^GbShM4A}$^{b8XH3w`?Op{AyAz;IRHP{=1k~kI
z7A^GS6Hs%O5<$<w3W6*`$U_#c>mortvFa@`Yw+@xYCZz`Yo(t&-KIVMC~>51E;%*T
z5%cS*^(08>6rYAn)$-g0>ZZ*u19I3b7kKEBm|%v}7y5U@hZ7V|G9FpAns<}RujBVo
zO;^wS))vrx#U3U+g2P3#@@`a{mG(awhlhOr?UC^JmOf{2_Ab1<3Xi2w(PMACD22I+
z09z?JI#0=IZM)EE_7$6fvuw`11t(F#^D?TSyY2dI)olNf!+9iM|8a&Xil5kUh^3n;
zd~_V_Az=FMKn^C@>Xc+s{@5;%`LA5S0@O(qz)iV?+;(H}Z#2?8Tk!O`$eszmPFtXP
zi3z6v4s6u1qY(k;3tt&EVyU}+XtDAG;y6lG86J-#jzzB%IR~ACdiq+YSkCS+^x-2I
zv#C$nXxJc)U5+5(Po3Ls`NYZ=Q9+UVB#R+W1Qvz!m1zFU{$ay3_9__czEt7kzy3wq
zI~DDTXt(~`T~@U+ID-n-%O1HkxJMJJX7DHljj_C{zqdBKsu%?IyQ;rttnZscY!z$`
zOp3~c68C4Fg<Las=POkSsV3|68o<b?6776%#nJ->)|bJj+IVmRU>jxt1)9o}#kEKk
z=0_W&ekLrE2|x0*yd7Y-o5Hs^)>PG+-O_@q9^<p_u6X>w=JXd1^R?%%m>n9qNH{91
zF4W!l7uns2fKmwisEJ?2Q&&WB7-eSti!@Kg4fR8=%$LB)3cdD1n+~~JeJzcFX!g2t
zfoAnrg2t@;ZVw5dFlx?V{pK?XdhH$VwsOc0RkdY*+M)2IVIf8goT$VWjV>G`cH&QV
zP0fN~7z_~qD`)^IVe*n2_D0BqKDJoY5%$)--#<kZVoE@i2D-BYSjT73!jM^<Q|Pq)
z@ji<?!RlGJ6-`ymv|<nEDV{atD_gb`X|Qy4`279Dfp4sd)%dLSvbdc0j*8N*1t;-%
z$7A`}?|Kqi`yMQ{96RAyMwhv=zR7K2>;6Z6bL>Mwhh!OP>#@iStK$7kKaTT}mrHd9
zNq1JygOXqaZtkeO&T>zYYveP|QKBZdtyNbhIkQoE2leX}w<a1xoy6t7wlq$tq|LGH
z+=MzgY;=8Tvbhn+lZlS<WQr#5N{6-d7IZ&&Wm~z2%e1UIL1FtftIgq*u*;`dRCcu^
z=yJVRaV?3srH0Dr?}o!^pR>H<G+uReBKrQHMWMe}eb}xS*Lgu-l)tMf*j6Cls%Esi
zJxbesag-lQcJk~BC5Ww-g_hf0`^AvLD=2tJ*y?)kW4A3ON*VGoudI25G{7IDTB5g{
zLQ0>ZSIg>XR?M-zKtiWSE23^;;T%@DlMsIcVj!~nfZKN*R8cDe5D=TkUx)fnKK;Lw
zfo|l55x0k2Mc*Jc$d?v+27Zqv#=Uuya^}B+O&p;r6teniUw+QSs@30cMSJ~Ou8I>>
zbk$9c`^7W1{zIeI>wKAQ!pYB>_k)u%k5nQREzu(XLgAtyk6ga71<ISy6h4#|7LB(H
zy{t{@Z8K<OhIk+F1PT$6FG?+v%A6ifCvp2P$2D`R?8uggs?A&k4Q;!bt89Kkwbz#(
zC`v}?6|g*7Vu(cOFs7&luG6xW>8XrNc-Y<feo0l_iBDwO8psF$l{Oghe@qU%rxN_>
zM_P5fEUT9FM~UU(OX#iA;v+T-H?Et>ylB*Q%`smJi&x^vFji!=6LQs?s!e;>q%iq4
z_K*1TN#DPMc3uW-c~i8Q!AVr{qCue(RkQRdtu_@ObR(u!9<CNt?xNNa%*!Mta$>of
z+0Eywh7B2Jx5MlBk^<R%VQ0?tJv9gY-k+euB6<*WJz2!Aj}QDHP%Yz1e$1fT;v8+a
zJ%UU}sxJt!L3=S6AU3EsHf-9vC^S{X-sfe+_qbyl?|*lXcu&Inu3_RW5p%5c%1*|#
zc|8ih*}BZ_`69a$(&sA{hNrwcSx~VS?zKc~6XHZ7^~l9us2irvv{P2{6W)@K*R@k|
z<Fp*=F>sXDz;{lqPQ+8Uu~3;D38tM<)OLl{wGIt_qg3~HBy<re0Rf{LualRlSc8t#
zkzJ?#Cdl!KpDD#+x;D&)m7<=(Yrq$_9qLhH9x8@YgQr*>G~aW-&rmNDl&M&D|JD_}
zHg+$|8nzU+?fgRZ)A6?$@fy{PG_#%iz)^>l6KwRL<lozi6x*ngti{w<!<nnXQ9Eng
zl5h8pan7A>oqVJXB)O^Gh<?*a(dfpWM#SH_Oav#<5!9z+L5t5izBAox_%87?z1>0Z
z%K>}p3*0`6?r|wa(55fWQBx-`RO1NgG&nO}R??%N{N3xECVQO2JQRMDV0WB6y2Mdg
zMQFXrh}P|$&62CvnO^X1u;t9f1WT+CzXXPguwr>&*AN_;a(Cjm-Wdc8&B3aKH<@HP
z>rU}u)s`dPgH);L*}#hOVSddFAaqFT1PEY<0NOQ{M2=8Bqb%^p2A8ztUIKH~9oVi}
z!yd&dUa(ax?F!+YwVuH@-V^;b-!aR>-T!jtw%@XkAxq#;a2d^XMNDPAgIlaq*UcZ#
z#)j{{IlA;KGW^BYWHA8a))l4CDeQCvr(){se}}_zWld9FNtPbYoW^wog@idkfrOF9
zIaWgqtfWHwx-LV}E6;QBQ&#w1JUEw6P_IE>T#4s@<M$(#V!y|V@h`b_FqRMHosdeY
z74wGuA#9*QWcsyp5@E2FZ+PYY9+H$(?fm>D?+PAmS+fG>uTksK0x?9t#ipaUjK@7>
z$rq~zYklfdHTGY0FBy0hRfnmBMHYLEy_Cz$+jq-PJ7z1Ech3{8u{af8wz@kSt5@%l
zR@ryPy{QFOgle0LM{u`5+qMO+3&wHfs>A3b{wWHnXNE4=?3mM?-zZ<H-~1GZ1($n=
z2=|QL{W6pV<U#mlRl>fYY!fR3#GU17;ra)bake%5#!&*ae)R>#4+uvjB3F4J&Z~k~
zD6<eSbZ3FJE|Pki%BYCDdfVnP_=3$%MY1M+gKj3ZT*pgDelI?E)81`opx;;QDq?yk
zbG+=!jr))Cv<453c{^rHl#tJK+$vEty2ILCwkyuqf#2a9aa1B`O~yYsOdn@4^VL9U
zC4UsoR$g4Iwfn%dNx7DU_ugxt@r<KlNAj^mMe5j9iu!b-_FMh2!1zvks&acq-7pUF
z8)083Uo|=tExEmg{I_|EscM{`l9f%N6U%+mQ9cTH#%#TW3-0+$9YVwilzGmh<Eu~_
zyKnk9beffwY}ky);sqm17@assIs<IBf7SQ+ZKC^{JkLl5Qn>TZ-ZJQTDk0FIQCw7-
z+ONF^>c{TB7skSc8TEbC4=sTN&9ojZ;yX)C4{!tg!;M)ngmk+W0aUYGP&gvGy}kW{
zLs<<sl`J1~3vv(NF#LE1%!vgE?(_M`YbrOqTYI6kdFC79PwkY$T>WHEhd-S}=tc&!
zO~Orf)ugMu*_4-iDx!%@Bz*yEpO9sOxe)Ez&L*RM#e5yg{4MgNbF@?=6{h2UqV>WA
zmPHvFloq_)jTt|*SG_DB?D+%Lx<3)rA%9VB>%C8rJRigtA*@_u-scP3yy0?smQJHr
zHNs5x!sT&V=%8uLLemyQ-Phj`K_ajP$rmuCXg?o~bl$6<OXQiI4q4fK7tg$NpS~fd
zQJ0p&c6Ng4eXC4$!@aIcqnsnA{sE)61yfidz*lU`cX4MXotn*2%6{+nB3EH^%Wa-q
zf0~1SXyigzzamZom-EeNreKlk@vqk-(!e@4n|AkrDjCk4H0!8_={&@3-g_(8QWNBa
zCmh90k<-x_3eg*vnaE%l^L-S4&DDF2V9_XmS-Sa`FKmQV3Xe<}_>A7E7^X9&_@J^_
z+H({_sh@$UPO2}GzUM;>o)Fw{2$m&e3Rwb)zNY<ZmhZzPo2E-~BcBNR)f?b%7GHjw
z86Mc{{(>MM73HUx>UNvP>kvoUVM{lsKVmj4qFAg~QplzL5T(mw>mKZcny6HoH13cn
zia{aZ@gT#Z%7$qykBDK3rdNH2Hk`2bT<6kFC{xgDvhEEDz*dxj;j31W&+>-YY2B5Q
z=3R*Y8I}}IxpN@GF_v;{M0%5O_Z#GwbuZw#xAnIn`e)0Z9R+RJr<<D=4l&|3n%Tnw
z{n*Dy@}m@P%WqMu$AQa0!vYxiR8JSDtTYAzue)nnxHp1~hS@elai}DIUr`&s(F%(!
z@prJueJ?+Y3EdMQliE=-^;-GI7M6tj*A|Yb<GU`ZJVo&J4}Ydi1k6ZE1U`?yaS)8L
zH<AQ`Y1tlNVE&{?gE;!zO(b~8K+wN|_nS4$@Tj*wUi)Phf7OtN0>*AF>~j~veSp<h
z5lo6XgrS}l%l9hdb(q3c7%xrMXoT1Lg7?*`2lU5V8<TKkHAu37sUP%~b5sr;s$%n0
z>y6ai-%HspGX0UXY7=?+8~cKskJ9V1wv?}Bgu9)_aA{F|YF=QZ&#PtQW6lraiQJVg
z*<gKMw&&;~-Q9YnKl|YThrYH8YO2OqAx~pSOW4;>ahu_nWCepgHMqyVkua(6>mIQ>
zv&bhkiEr4h>=}NoQ3?5Y+C37GSAv36Z=4T{cykp1myT`kwpZ6rjYa{QuYx8*PX}HS
z_FvK>?gBra)EHa><D+Ce>ot%C5;`8cITkV=vo~OQh=ZoF4q3?tbbuZ1>#w?q^71(l
z={*1|$F?#CpTGNKFxqw?PwB`IN1>{HiqmMIulK9ce8f1m1+;DIhOYfwcbdAgkeLIE
z=$IGFX0Od69^uz<%KUON7yozRtoPT~KTh1t0%4n<#l+V6b_uf9t14E&Whdje9TutX
z%h<)PsdD5u`;x8#ODjC1RvGrz)774sQVvbqx?C&0@8$GS)Ji&oEx*>-)v@5t=%06m
zM~@*%M25Li=)q3PeZhVP8WzkdcOPupjocr`+&*yOalm(TP)cFNI{FeOI{GJt8bb?<
z<R5q|;-UkWYG3dTu*6(6$$?Fb`f$_>G$H+N1S!~PD-d^B9`S_01VknC0c17_ibfU6
z#M4>HKu^x3fvq%*o&y7cGoY;&a?)$mL(C}TQ)A2>tkIVWHq_Pw!S*^#B5y?78?#R^
zy*p#k8%Q4hS(pR7eAVAptYQy!v9T9R3aIi?eNh!}7F!?V_#W>_!bv{;b?Q|1g;gYV
znQ_FrzCcDMneu<`I>Vk~0dDYMqvso=u}d@xK6OC*kr_IP-Q?<`Z=>M%B1MJomXk`Q
zH9IH1xFg1TndX;fUuiE%k(>oKQZLo=EJ_*OSZVMUOf$iVXao#8ueyV)hgUS3L7cJ3
zTf@2ozIdF9yF`@&IWg4oAvJZ(-v43yJYBmTca;h7HDg4=QDa#@pq^CLT)j2PyFw+O
ziF|okfMR6@p_4b!BFbso1LKuVG7LqCv<0)lWe-E%d`Ux&Tsj47qlR}UeCI@f$pl{z
za)orIyZOXj))TKGpGK6unUGsXaU}v_b3Yl4D;#Tklm0lB23<oZc)J+5;0}-d$uX$#
z4a?J_Fhu6<Oogd(ahKNpGer1fV;D=6{`m3ub<9o6cbWL-kJ5a&_9A`@nt4q>O-Fq~
zsdS~P21z4z(sy=iIH!uu@U@uBN_6Di2Ldcqh08=k$dn5$1$pcrB5bA|ru98_n2g#4
zzgm!}zKn-KE3c!#)lSQ}9*)-$=vMvk{kDo-f}Ys_?vGbX1IzAEM^zk)$J<z=b)kvm
zw-lT%@}Kh=fLV4|HJEoE*sI-AvL1;NYxXPUCl;#&o2#dZKz9)FJI({3T&0gXo+FYK
z3m+QRx(Zt_vJ0&I*d=sGj5T0>KVq?L1Q@R^sKRCV0RjC$1C2LDIte4Y*~(3VP0MY>
zfcmLJ5?m@Sco_x^oD_x~-2VcDy=5^$M==%<atH}{i7&FWv;@K68N(nr+&Y2xk?@Zj
z#D!*?^d9yXXK)#>RkdE#vhS4HZ<DxO<7ELQRjxl4vwX2dy?X44++`KORb@05TToFX
z`3P6*`cwHdiavQd{J~UH>)0n64$<**m*I1QvtM<?IlW9pGOg$iH!1BKs)A6bM@aTZ
zZas>><UCpvgB&4V8sSc&pG^DXNiw7EB{fPHzE-tblzxst`(a<RrQ4SxVa$MfG>@v2
ziSNH1w5s>eY{PtWnk9mWkOw@=uqig`R~yTW+$5C7MTE~+y$24i?%UUA#W{yx#bxK!
zG{0${;%#$WN8hBusO@=qYEIFeR>)NDm9VDhp5PyTeQLNHUw_%9MUE30yKkox1{{(o
z#m4}()hoU(s}ASV#5}iw=Qr}d6@$G;-o4qV|5Erdl+^ZNf}if)mrCcQ7h8iugmF43
zQ^QxLcX138_Rg6c^!Ku^H?*2z1njmj$aqaUVd>p(JpXW?)=U)-s%f?T5wsoooFTw0
zK{zOAwTv8RsNoz!I@j{X>~X6-*zXH)-h(A>tVpd=RRM8mS5`mDhHUmrvLXaCUTx_7
zTbA~A<_ux&3e}?Vtf=w>FxDdKbRMN&@SNkn!a^M^NHpV2%iq(C+eV+wHZ*uo<9m_W
zt7pLd>;f5pU4*b1f?f(mDj<ii^mwsZ5~SfcM)(+Dl}iJyZn2~>^aA!kYGwSAm#x|n
zcib=~ExBm9>D!M6$AiwYRA17p#Y3;#Bh2`cUGk;1?GKttQjBmqpOl3awtNi+eEoGI
zmO~U{d02GYZ=~=7xV&ytps-@blZ|*c=$5n(<W|n^w!Dh5f_Iue&OJP=!|QT$jPJ<*
zeU06+0}w$*MvUG@kmFW>h8shH@;4_Ty!YUB@OHMGK#s;_6pMuz4#5zsu@u=~!*$jI
zan^@ioNzDP-T{)C7Z?YBvu#|Ms*t|^ext*TS(TcZLwyetr&5y$j&bzYS+5owtPqXs
zn~X9@>g|CE8l4CrWf_LnX|K(B7;}|&ic>}`H!2G_MB<-E_qc@y7J9Dde{!s(f-!o~
zw+SnR6+D`za{-t%SfK7T|60nXfL?9doF~B91crL4v7;THp0ICSx4t3rLl~mteSD6o
zqU+6GYvIq1fyn}CIz+kIAg_W3<hBa>a%6(Jxjyl^u$$(h&}~BH#M^;%a^xzgsIP=L
zV%Jxr@8A$Y1kE>pQ&30#X;~MhCL`+zEgFlqJLmbH!gDiA7*hZkuwjGJ)6+Y<xX1+z
zuSwA96hJ0_?Ky#BT{w&|Y9rN8_z^g3y<6}RFDzDSx$ov;iTssHURhycmXN}kjzC9Y
zICcF!txFG^#7!SUItay#;r~)uo8Cu_iS}e4)~S@l6Wjbg*DzY{-}JpC`^vGwT^EPK
zxgM&SlE>kq*gbfyCzxz0V%s8%lB3!QIMxc8&1D+3V$qgtwjX8xmI+a{y1PjFf5;Dx
z{k@&965#Nc(eL`6gK2m8kD0w=Nd4NTIlaQmi&9B`UX8?>3jw5rUFt6HF&&O%41H%#
zp;-a@OR<cN{<TS(<4;o!9}tU`ENF<hZPRBM&vISaL-#IA_zddlfy+6&J~Zp1(LeL<
zkB7E0hvAofrFaO)NEjgAYZ*quZ{{~gd$K#Jp-bdw8^@`kb!w?bm#C3wJt#bq-MUk}
zEV?&a!x8?_oUE)m)DF<k*O}}5l{TM})XN1${3J`UUi-R-i<gPquufD$h4)WsDyFs3
zu9fICJ_wb+b}T+&?P}*7=2ghY-j7JaqPXVX5b9FR+m6y5vI!yP52>BZrvGC4Ubt&5
z2%Gg)t5>bs?DrBvNQ@b=ZyFXcLtw@ZgSQ6UnvZYw8K_Smae)gk>dzqg6f?DKWGIAy
z(^Amfb{pR;X3NJxsJ%$NOe^Bxl}95a27BZ@%VL7YEUh1V3_Z_ljcokpgSY3c9`}HD
z7M^%0zTXD{PZ__#?`U7Du}2jbx8QH`JzXEs)0gbL6{H`p=S;5LWn!rE`oBk;LE>`f
zmIp3hjb2rfC9l)uUP_7VPEe3J_I_mS_w_#xO6Y{cI9Ai=l%<#WIGh$T_s!hQ5a#ob
zSdDD2KfR+>@YiLoWfQ%MdZ0N4^~J#rfehIpM_-%(Hz-&nUUJlx`sc2Pzx`(9DD-2!
zkX>M6UOGy0Ui}^e{?B{((XQXOwuX`g7iO`DFlh@BH6CZI-OcZzb6w7llEW}e^7hG1
zUNId<HEch|-H`}TJ8CD8Yn!*uq2>1sd}-hd6~E(+CDT=@dLzO(D7^Uo8OwHcnA6rZ
zaCtH(&h>;|<XiRMC;mpM$b>}Vp>4&5MI^i-C+d!VZXeSEzy_WPPa*laeGTkG5G*y7
zUTJ@P$N>p`${|>^eap*we?_^g6Be^RqmlZO`V;``Z{-p3Wzm8<0@B;4!inE)E(6%H
zupxJ~4l82O=~NmxbUHO@6e0%&w~1DL;@u_;qa12Q9-^&$Y5oe&e8qOcI<x(=NYQbO
zgfanSx*GZvhR4%+AEgxR3!r?;fHTsSjDLo?w_(1%_Y-m^AFDQ&)dKa@A*Q%Q7#{jk
zK;&tFBslDy>$*Ragnt~tF15SL{w%I9xMErc4CvFp?+SmNeEU(W&bbzV4MJVp@`31s
z+{Z`~aL^-iLLe|UAT1h&@H_wjk8lydWgq$n)4_n4ryLT<hxg&M&i#BjF%kq^*-4PC
zNdw5`BP`5eopr!cusOU9{pI8R0=j)Jua{D=$^FFt;+TTJCI9$*sgia{IJ(Z%bKqS*
z^XA`H=5ED|xk`i)21xi5TFO8PR-%;IJAzS|G!1iX=v!V45^S)(J_PezK>%ho-#;h_
zTUqNbfB|WbmL%Z81JYjNL8}ZQw?7BA5yIf2QUn*({<6^!8f+=W3C{@}kzRZ1_q!UY
zim!OKMymU#_cXH6Em)LVYONMr%8);H1%U!Gf-0&AOJfeaTJjzr0WjlE)F-k6Ixu|x
zKg0ks&?bJ>n3w^s(^MsyyA5xzuC1Rk5P@H67_G1O7g*1SYiZSa=kah%+^|=sEKR7`
zMY<g{B=~zkPD-Og=<HlkOz;QOZ?9{~x1PS2P!b}3cGaMo-zi#*8t@j4gNrJ12(WBq
z-+qA*zew!uR|ybuJP=!%$diquRwR~5VkU!wg;fH@)DEWJra<{90hbE_4vo>z`xax4
z&lc)1rB3Eq5FvXA)<ToF&2|^Qa<*hnTM3nNap~?NZdCfGqf%f0B;uD^j!lE65TA_*
z!gHy9PCR4%Z#sapM*l-iAgO^@z9~BR3r|=+e*XU1S&atE6rL}|_pc;mNzq<G2mt7I
zA^U^R6|)Otl-@U}yU3*ShN_0q@*%I?)*sh;>|wZ`GDp#Uw-73-F`06|t{>IOYQ`<c
zzr>_AxU;FMS()%zXDb;*1Du@1B+>NtdFAA-Iy1|AtBbzl@;=!V4(v;9&ac_M5oT<P
zA2q>o@!u>}!g!*qWO(BM(gqbELHv&l0ej5rhkM>Uh4d6?7?@oUqfjeUBHJcrme|&U
zz#^fyzuLu&T(8#Hr%A?w2fh~)G4>qD`Q6<;kh|a%zX_N!o?#xixwTF~ja+9uNUQ2B
zB#9)%FM9lICUG+ocY4=h^h&paz{B<S!^<~D9cfb*{mvn1EWT*unsNHkfvQU(-9<mK
ztZl<c{u^JXfa2R%;lY$(k*LalF-@SJK*^&!F@q!06;L<9@|Osp@&7yU=vIlBz7G%x
zbjG8iH!m<O$J*@wyl8d33$u%(ld{F296R{n|CRJ~Wsl*E*YN{qfqgT|WNWA=7BOrv
z?PvJ0OLCL$xt#7WVHghif-YiO2m4?AZrGpCbieUhukVY)mE4!}_WJMu#W8g%Vg{jM
z*qHx=8^Zj*a6?Aqzkw8kTF-k|P=JO0^6>X;BYfz1BQe+ut7{oe*`yuTtB<->bt}tG
zmx6JOit58h;+hB;0kOZhJc$pJo2hgVFnpXyChnGUc2?_tPkl6XqWe{uvgE)uS7*(t
z*yQ%`D&FE<qPffWtUl$}`|%yYn_#^(M_y1qQ5kj-O%2AXKTV)flx77$C_qPYLZJ}G
z6R)6h5-$Q5lMgP_2c>~H+FgL?T&#4!$V9)sMpoM<Vwc!bd0P12h{B!Uf5Qrk@V6i9
zwT~wsKBj~_b@)!4!z<<r&E8CNM!a$bf4H;y0}%njm9c)32O0GLvd_*jlHMA>_;S3I
zUi_{^0#s_TU-B{hSPnt*RTqC(WVVlO!cgxddp)x~F^bel?iYRW!?K9f_HhFFtliyD
zp!HS=A89Mw!KFqew=b)`am2dimWWp}(t5D~5=@^lyrDHtaQO7gh7ZXzR6zZ%A?~&6
ztG{Qbu^XPd#X(GHf-Z(KiM($p6^!UBt5fscaLYp-m-#%%mO8gFJ&^=m0@#`!Fmk0t
zf>kId)8C-BdH$6nWd18IMB3{1kYd;sPPu?+Oq&*%-4j$!U(CyaMNyqdFLj`@JF`28
z#qh~NhTK*modHHX3`<%=QsA1VTN$CjR<Q{udSZK!@w#vJ`n)?wXw{KUq0>2z-Y@Vh
zW~qvum!i$%hOVK_`PzR;z)KvWd~$bHRH$~BdV&bFD?B!XA2(CGMq|?6;`Gt33A*3j
z#dB$1PToca9dkEF4(NU_)Ond{AO@~RJRiyWrzj#EgdQ^!hUx3iKBWW?^ievZX%&Pj
zxJ~`S9v*6&QmIm3m<Le5%09HFfP(2OEim1EI*GuFDQK*>EX@4%CFtWvuRPBD<p+%j
z#shvC%OFnNsT4S$_qqX5%cm<za0^lwnk<DXy*+A;qU5a{@zjb|x06GR0v89K-fvUa
zD5a(31Fv#*+Hq$Fw_{^duO_qGdsIG2!xqk!Lk%CYh=eJ=Kk?$%R*6@Tk9&Xdx8MF1
zXi(nDr%Elpua>)&!xzq2HJDQti;LObYMtcVg527y(iTGbk||mx%9Nu8mq@GaJqCr<
ze*B;o8_AcXJ^68?-l@TM*K0O;3qh%kaKT1U5_(1R1)(b0yO|4>(y#cgU*f6F;evB4
z94GeBS$9%JG^2ipVl$ykA1?bO#TK#U+G&&V^vy~T>c}urLYXLBt%}S=oyX?Oi+dfX
zP-m{OhCyx(@{=7v`gsOhOZudkZ2vdKlm^SF+a%_4wvh#kh%p9|6KDa;Y=m1WY~bns
z54j!WeI+ryD|1K1*AAE2u7Vhn%0;%wW$SZSu`m@r9!Gk*Kg*PIzpd2NaGc~EDrkMP
zhQ(y|`Yh4$na!9f@D)+ce?J-#IN%zJ$?b;U@@<AH$C)aBtvm&83YqD}dv@u0xM+3G
z=K@i1h%S=C8->agt81JrTjy>MIkP@YVlFt?aT4dK)=T4B{e2_5_a*gRJ)R2OGT7O3
zh-)E6578Bd(gWI64M#cx4GoVC6fM_i$DLBKtgMt#0>>Hc3aGi7@a>)tSO<SAQbqb%
zpC5l;X_6849d*H#`g_OSRgSRr=6GITIs3MATt_UiQzojx{)kTAbVA!OMd)BfDvpQy
zMwVCq2>b(qg5$ZJ)`i50?csca689f#@W;fBmB)p143l<pdRpw3DJy8&^Zn?o3YKJ=
zHVY^b%RX4&A)1N7>jE%BpFulPg0+y{d<J3Uvb(n+6_*CDK~BL+&p#<}r!ZfO{Jdi^
zPA)DefX}x#k%t7bAcmzSgAE$_AQ3FvP~1QGq6OcH(VkygolD3ZCD`rt)R}d<&-0tq
zadA>gS*BL|e^=C|0(}pWB$S``^GBv_G||7kK1gADd2`18kT%g1T28_*DLs*&ON`Wu
z)-1Eub>#WUiAZKPA}v}6R@!>}c*-mE0}e$4eY^3p#V=<AuI=+gC`B{NpTvd*l2XsD
zEqt{e<JJz{-EV~zTUG>IO!_v|?{A{$q??7!FHv43Fuctr0TT9Oa!NZQ7ei|s>A7#@
z51jnUFDZWNlWMZpSn>kIy1wRLG3lwMi6M41Y>-6W9ZmFUrFWa1J_pupWyTq0&ZxmO
zFWRxU!Kv@|`?fFq#lK48(c%)2Q>3^lX5Z;<M=%!$zU;-YQ5S}O&d@6WpLOZDv-586
z&#e1=(?BSn3gwG3uJUG5A7pLCi6k$MiJABhaa_uTxM^9Ta?(hC2{LS&?eCy#jBHD%
z9sPF$<S$98!=Q=VbKxeNR48^IkEJ5WS6DB?c7_TP6Fqppk{ak`m?8u9!~sp%1<FHB
zU2ODY^4PAyVl!GEV03>cRw96<rvQeph|)Z^k*ICOU7bHcw&5k+Bni1I{+F2wXYzio
z`<~oEDo3BL@+i7GYpFWE-#gRBvl2_2+7qcJOht==FtlEqDoy(p#DY1(#;Kw99An$G
z1thZG$iIGCI8bneMe^0k9W+I~8T5rcX2bjLRiyx6<H4igYxg;y?l@*~m7AEO$ydT6
z8o#<a#>@|CPex$SF|#ILlC)yT2xmuMWvVrqA-C=m^7?HL1nVkIA5E$|@t49PE<TeZ
zyyiH^vSs7S7tf5xl#fFgh=D^Cc%xLuR+r8tq{G$Zh+9RfT<D5#&QB)Qhgj^BX&?-=
zxTX@Sp2rcO%RJ0Fvw7TIe3fwJq^$VxrE?aRS*sb1M$7hDI+sVRow1_mEr$!>x||+N
zFunriZ~^i`7l;6167pjAYJH}TvYH>N*|w(Lw_e-@z{FIgf~^)uGp~4rCE@aj{HdwH
z=O;iUWtrCjLOt>Qk-_g)2>iA<N?Dw<^K(^YOmX2K|6msv{@t1j8tHLR@RG4EvHxNG
z^CM#+JAC4nMcN$m@N@H@Y!$OoJk9G-LEArUi8i`yY3HUfkMU0U>&HKySiC>jd24~U
z8K?8MUmAzvdia)B<NM&RCdGE6iHNT=!U0E1gH$_**HK8nM59(3UJb1jY?KF!N1ZoE
zN<lSJ*2^MYEHZ*f*U!ejb%WIa7&Y8Nm5_ObS2QL_^(qGFq9UB$kAf@SQ`UMdF^)WU
zQHJWJ`8gI&Ds;4HSKi^1$f0{9#=^e37RZ7PVqI{nq4XR#MD3)|F($Mz8XCm#2BJXb
zb2g|<tw-}t8GCFblAt0?-pFUV@!Hcv|KE)#Z#}~%-$E~uYmC?6S{Hqq7J+a%V&gN~
zoh3KT|BeAZV;sMQIhjnZRDOHt$Vc&yJYi2RDJzJBVI&cS7?pj85yQ34mTs1glf#Jl
zT~|noI7m!o;<j~1RIKDi)9QKdvo}Q}SF&}LPoqpKljLu_E%&rCj%})@DByTdK|?6+
z01Ft8A`|WV{rh+RcO`sSO9(Zc%KcBFyW_S&$}DT&=aFw^HD9CWDs}CByW&L)OkD#z
zvYj7fC~38Z@B-fL9(}dIa=s$lq*L|h870haZrjWhueI-aVf&%>`|e6#A_^gO#I2I5
zolow|FrpOFtOeU2Jsuoq1XLhAu~cSzvx3M|YZ=N`vHx1Y*Pt1IhU)}ZF>A^?n2Z&C
ziHc+F1PQzpbLlhfgB+7(s)-aD?~cD1#h}t-Q*aSW2GU-aM>lf+riMi{X(k_G%}_P5
zRxUDMy4W1#+?#89J)AVQl#6$I@r$6zgGp)nhiZWwmX)qxD?oA?_1s9eQkk51ApFnq
z1weVBSb8guC&*zuoHnu3RAU@h%#-xM>zUqETShNNZNF}$oGlmLzB_OUAIBg|$Kzy5
zC?1K^tz)~voAk6W=T8e0-p!y}vT4Pr)5hf3(!aeW5Q;S<2f~5OPQUWbVKQ2>Z&5Rp
zKkjZ8EWh~pmU|Y$cSGIH%dI7rrZKkL2AO`8<}|Hhm!SzVyd){?%n!e%GAKeZ=mpw6
zu7UxApOP@NN2S+U5bzHq!qj<oj19QtbkJaTWrN`VIH)PP11dbk(Z=pFX}$~3W=7D2
z7{PTL2kO?#zL&M0Lzd$iSr!zHm|!(mv9--Mql)aR&;(g=M@0E?Bd6ovU(9Nv%L*l;
zcM5Mtgo__QHA<O6bl3_JeA*lwc60VyIY6mtHmS&#Z$lDOwZ-Tz7;4|lZ;-|axyIji
zyq8?a2XVr<@1SRKbdAbg&;^@~z0%6(Bbrj!Co`$KBeqM2dxZ;rAtIHHOg!E<Lr=gt
zM)RUdEvj%nk9K-zNu>5G2`Ee9>wmt8X@ZB2RN4IoHG93lN~yyge`oF}`qOd@2%C}A
z9kZxTS8B5}kh`mu6z3|h0)pdYH0CVj`lNjoFX{V<C|&fb)ePk8G)9v1ytH3@VG`<P
zt5Wk-+i7#Qm4J?xvp<qxP(<3X4hceFU$-)OJlr8Owxk_MJaIS`A^9f1lyru*7@T5j
zOqrVYu9O7WZ(VK|)Bv41`PY8AyTL_n_y%RSY7hw%PHR*B7ziV}){^2B_M(_6cr?$l
zy$2Lv25buFAz5{zY@GD`pw-9b&j|6189$Tr*J6yupR!nDMv<uCpjTsl0K~XRIY&Yb
zghqKh@0=UbV4fd}KunUGVZJ(JL52eY3M{<mY6<Wr3Q-L%XRqbWPZMco+~cSzWsDyp
z6OvBXlmN7_No6)$8ei+OCbC++t4JH=C`Fq!RbG8K28;MO$Y~~b|E=D|l@XuXALi$u
z=@%GRxkTugDXuRP#k==4vLu#tY3-&~>O~Tr<-e^H{Ly5P3M-zBC;reZe?u4l?N6-V
zOI^OLN;HbOyonvx#hp|vowVLby8}IupeTGRJl@1VBYGgs<T=KOAmH(y6BLf0Bv#x<
zrqn$KW=lqjn4m=D5ypfP+nb$7Osf3YcX3S=l(t%$BVjLawlxaz8i3~Kc?UW^U9Rua
zRPdf%AKe$kk?CoSZ=Uj#7fEFekvSe8`NIB8W-}tVd{r4`*c+n2IjWGgA-x{h_Y#*#
z^a(**6^&9t912)7pD}Mt1rdL=3b(X2*>Wy!j+X^$v=}N)Y#p|<>P!bjj7xX8e)Z?5
zPPi0!Cie-Lri)waYNXcBdFs4se;*}Oo#z;T=YG<dpOV+1!71}YoGN}7%3-}E7fY>}
zZ?#x?KhkkMo({<o2@oUn^i;L$Wi4{i8@KK?^HR+?>kjMDSBSd4ID)xk(9wg<_>}Wy
zHDxnwFJ(~a)qLQ=d!b-{m83W3aU?0E9V~CS9N`MaCDoPE#Cm=gf`uXIk(Ri2v7i;{
z3SwKm;32<K`;LG?n3$zkCDF+cvuXuRS#U}{tMbyU-uLZk1Tu(s7>;_&cksT6GrrbD
zxj2j_C%@Da96qUDy2)s?5OWzJ^wbYl9<xEfaBb+kM}!Tw@j^NjfWpD7#@SOqC2swe
zkzO*{6<wxwg$|<Fl%w+sQ@<-~nWoD>XnrAxD1IjF+ZBN-tHaTM<-a^IqEpGc*mmnb
zf2SO=CBI9r*J%P0ZUAs-r+`MG23=F#z&~S-qMyd>@rK3spCO@oJ2nIDa9wytZTlp;
zxPH+_``juQW;BW*;^!^jY8!cxI2F$e9J2r;W_}18If?oSckASH&;jW}HiSNjOnNG3
z8~x(XX1u+h$)vDdBO~;NL-YH1`vU(GF07J!<J%C7*Q1}O`=bI}Nn#cUrmwH?V2+E!
zJxciJIlng0>f{pMA-kHvd@!-AKrwITV(5`&ljdCzm%cqLlE*L|jG@NuIchQEeGOZ`
zFKj4`Wo{%fZ~lkr_qu^&-5$OErx_KgGRJytS)Sbl!Lkg!7kQm^1F3ZRgW7_hP!6>#
z6%a6TB1?0Dr?UqbbQckRy94b7B8z0>=I(-j%K{#}c1Dl6MeLi9F%RobFSC88eIN1E
zp6?vhm}BIPrpnJA*E8vvX*K(FWdn*}*H$o6t}bl}jr4rnMvX;VIkGCe%&eL~^R{XX
z+uw&#+$$1AYL26B6hzgNP6Gfl**H(K5uZ&=2L>&F+jRK*18=oj`g0<<O$9ygau?wl
z$JpB?Ra#!Wm~L?RyYTrheRo3;26s$<I&vg&`&299(8Ji~O4>pGZY~zN-=tB$hV##v
zwl!x!X^bP+KCwcJR15a^vavmV^qR8>geADVx_6WA$^qCOeEggA(6eo6>$eJl)sNav
z8Dqh$oZSt}t&jBKOf0ta054&$y*T`hI^VWkYSY}+Oz>huUUBH_x=0~9MMGu>x%$M(
z>3)}X4bw3r+dLfRyr#|K!&CMOAsr-tu$0~^vTj}l@a<Y|7GIz>({O$0Y!gikGE&B3
zN{a_9Z9nJFH^<lPJXDQ0g9+U62tkBCxVC;>jLXi8o*36D0`7NmyOUJ$ZANcXkgH2e
zNA@Xtk5`F^>M2+*4x1xYPBtSQc8djd6X!SC>+Dly*mOgjKGLNFS_aqA&-29N!oph2
z@{i+|Si^EJ3EdDe=wDUcwn12s0QN6Z`PGuu)Un2eYQnUDe(rj9!G7ApY9dEAW4?v0
zwdOtb+c7hbSH+S#Vpgp|pK4W0DSVKsF|zvcJj?G<H;a#Za^mY_&)~1?DR1vjklZxn
zBb*diC^RK`NWBdJmQok1<h?kYs}lX<q4o@_851M`Ipxl<HT<w=OJ8z~dMyzmRz@i-
zf&zW$wMhjail8~?(f&A6Tgyd8EgqZfh7YAyl*NLTtm2W_qG{)h{-0G7f)-Jyo7r()
z21o{<%Vz_FHl#GbuUV;IVwy%}qi^l*Px6~=t6vFqv`)XW2h3mpf$jaa(q(<;6y@kh
zA%}iJa`E#`{djTK<W3^i$8x(~36+E383+g>TJjwes-7E1?3UJ#764=y_rIN*yv8GA
z@?|LxOA#D}Z4E}1Qx-oL7JjJLgQr_=D(8Bj#RrMyNy32%*67>8D%1s8Znz;vH@(|e
zB>go?N;i16hRVIW0z8~5K2F4AQ{Q2BH`fb9=uhUE<*l#2+EGjXO88QkF~u1wPywSi
zhEZ|a<x{}@l}d`nDcnJiORLL&#nn2aHCHo+mm%-ANV|Y(vP<yqiE*%j%^k{vsP+Cv
zca#1GU!rZyiqRgbLOE}#E4<;AVnDON^{be3Nd6k%Rp)z&w>EPxUhOt#RDYC__AGd)
zGS>v=4<SvX`efBwnVWF|Z4BkNUylyCui}yDYPxL!IqG6`=x{B`h<&P5OG+aE`6Ha?
zqwHJ#8`YGl;N1yjy%5mD?>2b(?xOFPw8qQ=R6F{BbpTx~KWwnu!)Rd>mPzMWq+3jv
zt=UB3%v7zXv~l8E60)|7>>u<1hMZ6piMkLt)6*eATIEizd`8Tx_ka&i-exSj8RPhq
zj~a(Z1j)S2G6}{BY)RFR>2tg?c&%}>wg=9Tl-JT^nY6{3DG?%U*n=u{Z91vGHObZ#
z@!bFlInZ&m0Ql1wdL};a{ux&HdpsWH#q)eKn+XDZiJ?0l9EG}aUDzSmtSKMb+&lNJ
z(gk}a&T)@7=vx&({B}7kQDbBpzBv5mbQnsZ#c8@JtL>OEGZ=|a{_2aF#(V2DJ1CIK
z*?gt<ZA*plc}DYFW4X+<8KzzGu)6_wfvsno(*)rYjeKe!BIyy2y}nn7fP#l9_w85Z
zAX&@yw{(osx$ygUrH7N-hZrtdaCPeHaYkZQHs>t*_zN7gt8+J+PpK*C$5@b*)yfbd
zP#13(lxVB{*pKmd9d&S7;(IZx8Z?wF0Ng75W44TmZx*qin>u!M^=xk7^X~+H>~nE_
zR31Bm=|{W2`AHq?3yZ4Yd7BYQE)gOXtXdHD@^*P=dfjC0GbIlGoD>Owd>ON4TizG}
zcZes)Rc#m5&!0AS97Oknos?`q2p8_=0ObV*gGQW_RNwYpGQEen;ZH$=Iqtmv<mA7l
zDTSC^V)-4B_eisoP2<8)tuFQAopI8;hvyov9ifj6R~BE|{k{0ZA6|S*-RgWT_VHA~
zk0K_u$SubCRJo~#u|Sh=rK?leqatE0$%_UKkvn!!GS)wpy8f>FC=NjOu>dwOT(HA`
z_=v7=d4JTkhE{OhkD*)&dm7|aZX**-8;+$vFQ_K$D^8g?@Hyz1r6AW#b$oU-^K0d}
zp#uq>O0#d@Hc1NsXp=7bdS#10y4O%KWXNpyVUs{4&2W4(4~r|Ux5uh!PB-zbyw!9j
zS7pPH-=bXAPDfF1Hj8gfHDAQf1aCl1@_q4A%izpr%7P_h#+q9gaDQHN=xD}Hy})47
zYc1&9Z1t~7^9!0OQclX2eIDfz$cVJA0m;Xi%RrZa%KarZnhn!VG}w$^o{>bc_Ua_*
zy_$g^kypjx@*QV9EodKseN^Gr{%z6lSkOHmLI$!RY>53GjY+WVOh53kg!FWE9hdHd
z8L4iIUFO_%kH^W~1Haw7AZp8N?uKbeS3l0sYHeE`KoW(e*){oBEdVp8pQ0?do<@Hj
z{7gWugR$XK+TJXkj0KKsh|wnMl8Z`H3MUc&F17PXT7y8l+0gx&>xgtXX~*Vzu9d<N
z6JN9Is!M~+#LzcmWYW^*870+sy~GxEfY=A$*Ic5-K`Sb$pi^+3cofWX&NH}-&-9@c
zlNZ`q-tAVus)6dU??W#2>HMID?eksv$J7{v0qf=VC`WpmJ&HrCBm+aIpCI#rg2j-q
zs`3Z#TTPlR#~EvWvKi+8!`@p)Ro#8<!mw$iyV-PuC@I|?(nyDdgoJdLfJh_KjdUZT
zq%<NR4bq`VOG)Fqe)xZ$_k1|d7-x+0^?bW;H+${1=A3KJwbq>1HRJp~qXsTn*s#Cf
zeh>?Qhpf}$Bux*CRM<VA48Xpm)Hy}un;FjNNp&6AAP|?W$UR>QBx)fl*Zsm%)gTK3
zcYc3JP0xr(+)yWd<AcL9nfk9z#n;|i@U30?m;Dz*XTLhEW_q6tJvD6H2VlSY456Qk
zL_W&in7m;q6t~W+2NP{*JlhzG&%mu7CAmiDn~L&2)y4vPlOFj2`&`G<|3-07;{QH7
zgJ6x1Uz72&-oFoy@o`nvh$*OuTu{+!Xt}Oa{h^QPXyRI|=f@Cha?$X6CBw-im|n8)
zJQZCG1*{!&Kc(E7d`cH@#kedCJ-(>AGJcAwJ)Bb&HUHqvtmKFY)`;U@eX_0}4|ZRc
z8MJ-TDb^-rI8&^Le_m6Vun@D~8%Xc^#WORk31@jvwr9WbOtHQCTD18E;a*z1@rQfi
zHHVphLB4<p_PU<)KsjwbfNZhlh0Jc}ibE@9k#|_RPKySmJypZ64&C`*XEyw&+krPH
z>As|;3_>2660h`dbJbfPN~<3xF&FsWi2VBaEUD&@<=}zAH`~|fzFeCPteP_4JqQGq
zU>V~!lBJ=@Gghniag9S^CXR#A2al&l)XJ1MncY&)tsh6_MrM|9D~GfOy%HsPjj1H9
z)*WJEw^>QtXgfl|4n6o$l<+8G)r?6+G5TWP{vE8niJ?<WgAu;;0VN1{?a7|m6$a1O
z7BQ(__;&qH)r-}t0(rsx2G^)%s;7s+UVJW=pF^-8Ur(2Oq^7)<w?F(67H)LU=j(M-
z&^|%i*74@^jf>5{cGtBeHk%eISkVkEFSY-0mc@RzA=Pw5UZbXLb(cv$|DF-sALSKD
zweUzbLm$&M-RlM4&hPl<R8d2JUl$+dL&~}@J@P^V_vQFp;s@5gq|8aR<eDuQN@5WH
z`YRFiYxe!gS*dP8*_&hE$AzDIJ%*1cYpMN)2?D#oB`6*91tI--@REmpkNM33z6qL;
z3+G)D>MM92;Gfi)s_TDJrI!#z!T-R?Of2}z-<(VX*3xM>t4*dOqki=bsEsA+^Y+o=
z6#f0ZXu)tONalLX;(@D+v-8zWck!bcU$-{)Q-!F5U<kf2{80O3TEa){>5S|`u0@8Z
zgUIQx4#~;6)&dFL8XZY}>+6)^$r-<B1KUwrni3Z8p-@?<m~=m4>(iqzVSFDSVI-RG
zCj6ZI;HS38L_&<2dJ$ug!m7(D;$%9eQ8{lMY+_bN$#@&be|iId%S*&=@S2TbN`(7k
z&zBs2eiJd8MiG;lvKMf7WjqNI>e&b30By{{-EgmTB&WgtPgmQTG4L?-3a=`qxNDng
z$R$C|qrPXfmES+k0E>f`@IN4hG)X`v=F8{f<MSAzO1L=s@vNdy6x|NIMurO*JK3Fr
zoXm*<0n6-De9A9VjDZa~%dditPl~;hC8AGA#A8rX8J9v17bGOnDO#^2JwIEQS8FAt
zxjk)R#JNr~<`Khdp`6vsWz~J4Jth5hJsWqlI0!?F5&!h*%;R_LhjEc0{Kc5heO0=M
zoXRZWk<E&3jwVbiFCSXZCIiV-7ViEm??_q2=*<Bue|$W}K_*v)y|%|q!(Ev7=&8^c
z>Kc)lo_vLqGJkn+#2dqgTfOuy?a)%CMN{--pXK1wiV(?HEVgR`kcSUE$R=Uf6x^~Z
zzLaT=er9L=cp`V4mH3IKgXik!kcYzLpG?+7J+V^?6?y2kQH^uCur9J{RH4__T2}|Q
zq?7FK=h^j%(I|1oa}$cL_vJHB2}hh}=@nV~&9`U2t(^Q>WO<d_RGfCNMc5wT<xKhG
zd=kM+#2Ojt+xUxZ@_`dfimY0*(}iBYm<5^nNKA&bdPPvW7VgI?5O?U&-6O(t?qvHO
z6kf(J_n5Q{Ow<}?MgB{leno;Tm$TZc=KYf@c%saMmu441RzfkKTN`30-gXj8Yp!^s
zJx<~y@nJcOEdZ~C1pRb3D<%YvMVmb>3jmtV6nB9ZGfwp)v-hcX*CqgSl|9acuU}tu
zuY1UQe+=<2zErN`-hT{G$#(l2c?aYd6qMWUN;^OKJ0iP=wfm=uf2aVk$EVkLI9MGV
zbGq6;e7hsi-7aQj+b&!lW_o3%E)d4yyLy=}n-NJ5UB0#9o>j~C8^vFC&!yp+A#|RV
z(*__Q*@!?fhY0|P+R{%Qsf?!kJZcDFMG@v=uGSpeb00rFZW*X$_*Ycn*(zm+37(tT
zVLrz{ivi=nI7ILO`n2i24^f7jdx5Hl-MC3=3O*vn1&fh126Q_J$e?H4ZQHUN7`gb{
zrM~w(Pd;I`_LOnTNql5?zD|@R?yS?5_G?@IV9UK%dQ@;&NPR_u*MUUzAN74oC+obm
zKPiJGQuMwjtVFz_b-y7b>R3cR{PTXIP%vM~TP?U<VDi*`*DQ3;O``8cm5!=@m)F>B
z$+9?0!k>>f8{PLL%Z<fzs|`s64XiC?^4Vq>jcawLK9`3*aTgO_Ve8;(^C}dyrHkPI
z=Zi4@Tch7eq@=e-6hKKP44+VdsK7xn6q)i9Qt>P%ZP7v2Jh*L^k8`7|m*Bom3^lgJ
z>%R{3(q$Z+)NAY)S@iDDd~mogyhPr))k#7tHshaLY=Uwf(C0vVpmlEcM*2(nTC+p$
z$DTT3tVKBpqQh^&KDg7G59j@d1Q&jq9R9{3snase8%@s9UhU_L&1*jV+4<Vn{v*py
zyKB*2io1!Pi-+FAP|0KSb@C9K2}X{v&`Zn>Y#Y|Mb@B--pE&RH${ci1e8a1G>E?bu
z8TyYhg=?SaJGhqCrUd8a(mowZja}=H*8n^*ZzfMzQ}yJ8@xDPs3{DrH>~`=NjZgY4
zpIAtP>_?ip*S5O0wSxwP)?8^nwf_|?uNSKu3#UV~d0MCeFj%U8KDgcBAHLjW-KrYB
z9iPvAN6^x_;!WTfyxGGYejfY%TM{VC|7e0WRTMlW@+a$|+aRySHp%uw?>g`9_dbQ=
z%#nRC>?>+pQn=$UEoRSLUiVvPP=}s?jx!`r-)+sRSlj%W9*xDlMYoce(fTlanNt9Z
z(LsniOC;*9Zs{csR8LaAJ!@6vSp@&wvO0U-A?-HW)AHFM<-Y36P-L@FlLhUbnbk!S
zPB&KVlBEXw=}<9CQmO-im020LH8SCP%^^$Kg7C}g-R6a{WI@@i0?Ur4`GqZE;n*W@
zL9Kn_+4hq3$OC17fI&Z;*Gx#T@~bs!|MKIQVru<$#1QUL<RE{#SF2ijn@Nv=@8xb4
zo|UXcFRsiTl1dKt{A+QM(i)pG8ub$fu$a-~jYexd!OOz@t%IsOn%@&PuqTy($;Yvs
z%MQZ7pu#{yREFozpMwTa{KL5t%P&(k!Cg-kY^dyzPYUj!40&DLDm4ypD0J~=b!+on
zqe^rPF1J5L$q*|m)RoZ(Kl1lC0CyWr;>O@}DyJ#$)`-*)`6n+k{+O&#!9JqK_r=0e
zz$Ex$fcx%e-q)RP<*?xlvah*vuda))FiEwIt3j=?mihJuNZ=Y<5Lp#cvB*eSUgb3C
zv{@wJ$9wVqc1G>C27ZrIMZ}lA?Z?-wbipJ-ygRO*Mu@9NO_$oQaw?91@tiBjGq-04
zrRC90n=B4n^NR=g+G~f&`kxLEv_pHal)`^xBT_wItD$*cDl_z|>kT1VxlKr_SB#4>
zRa|_bs<M~*7k}-3Wqlq0MG~n3&AGrAATg%BNcMv(Ye^Rw{Vfj!y?Lce!P?$0^bWOV
zhNg@F<4nDkA!1R=^#KthDz=#t19K(ejJTL#HOtp1)h&Po?-`ztXH&`FFNjX?Y#j(#
znUMdF_*>aZLD7IYVF*MR1;}CS`{Sr}ij|2oze_*GtK|R>#Gbud+_Bl;>vKu|P^(aF
zS{(hT+JE5ml@Jsd0)Q)F<?5%$7qJt{KaR$$J;Ew8Vko36d%{oo?9MS)aYY($$4k`Y
zx5ml(t1Lc{fOJUF#HRB-L6Tj9uvk+ncnbhJ(v5sh1iohallKVz`dnLJEs^Tf>|(O0
z0nT`auodQo&bg8;66FEoT!ZU<VxBBruf2<Vo(|*kS(KOrt?uDr%WhBj)-|4uoqM@f
zUR!{Z#_M%oKGOEyVL*@YC&UU3|4v2U0BOtruc9wA6w+6?CM3?=w~GQPfVrSBlcOKw
zi)nKSRNe3!W!c_n&^=+x`<};N{cydndu$6QTNH0_0F<OelquA5{x#)y<5A+2dMet6
ziVjfBy<P8w&zRwrB@JQqUT-B&WCexFJvCW>6$Y|5FIvcFQHWv$q>%>zyeLmOT~vs9
z5kPFqMrhY%oq&;n@H?5m$>|!46lD7)!FWvNTRza3>7bfU@1t&6r03V9+6zKI*GI@b
zqOeZ?)o_${;Fdy69JCORMIm4r(FEM%TY$FBWyYGe(yrGOWM5`uij3YLpB|I?QlA?m
zx5rWOdpk54Np)IFb*e)jV-(%vfA}8|@BMwz!%T+<BT{w~x(tfDDgMDjx8R_W%}(x`
z6fxa8cKV;^&Tul<{cl3$%Wk1j-3ikS7$%|*=wumQF`z_)eBk3mOjFD2XG;Q4XH<bl
zA?8+BMajWYN?4(0;MEo&><R%Om=muKRQc;ld`7*>tN4tjD}_c6kd``ZYIxi|;3E1j
zD#8ByguDl`_aDX6U&-KsOhX+!kYnDg+36WL)--~h^dy(xU720$eghr>z1#y5uQ?_i
zDkZ*~ldXzv6JwBaNVah-G({r8JWN|ul#3=9EVwFvja29FjQdQhrpUA{0U6_|KRrnc
zN9u5){#y+&r42I5-9Yx0OUFWiL<`%08)Vl%`5P6;p!6d9`}^AsB+yM(n}}pQWcd}#
z8N*QsN4HZ09%?##O#H6aPZ0!<DDQ=%HGsX7<wW(2|ID2^>yiAuNIt=wlxFubassbL
zUTfv+W%(x&-|={5{bQ28%l}}Kqp<(>vSZ`PdkdIzrA<#cW>USSlbimtcDa_mIAH9E
z_7SmoOwkDo<7t8}4NLq?QqQU7O-JdsCLiGpk&g|g{6Dy(sI@eA*k9#M8{d~{ukM;6
z?0Dl3s4>PzUYjG4k9J6Ib;z#hXj?{_D!fXjh?O5}TC|P8`}}+Z6o5;)5=UCdOM;S>
zFN5c*SZ??~|B*=P%XEVKKLc_!!XT4&`7U+O<fW4HnLK6lqBg~Z#vX0e2kj6JHo2YW
zqk7v_X8DI}WCT<yp;ofsPSs;a8u9TG=pi!Bw0ODVFIR>sf5%%Q%ChE<&L>AcXcz~K
zt3Atx*NerfyQ%SBy)M3IE*QOW#3o*b7h=;t<K6h4zq<884$??1rD|Mv-P+xstdwFi
zF>y!Z40q2WAW0GOMD9<J98dcf>SLc%0HP?o`zpa+3ctCd0sojbXeu<E+KqyTP6!pY
zBa1vjfe5J5p|_p;c7tB5j^aih>GHd|B0)JP^iieb!A^rwljsBsG%XqCZwa#i>_aB|
ziT-$-?iX0_*?_Lr>(K3<5T!TK0g>+JKSa)8Wu?|GkLul<NA538I3V6UbZ=pG=1yx^
z&;#h+U%f+KT{RaO5jiQ0uEws>NYX4S1i48}*f@C4V&I=+|A0!MKGpB6-vJ2@@9CbN
zHnQ~(EKgy6?+xW=PdZW(?<ajU7yRV08BLVNk5h+`01xCG>w8ZbUz)U`wQkXQ&sE&X
zU&PWvD&6}G{eXV8=Q%6xzL5smsnEVW$bVQ0Vwg4PB&_&;R5@YxeDmJfmDchto1(h(
zorcRSZuFo$V^j5193G>$FUI)$f|=*y=!nN%LF*oUCr~NE6e>gS<<JjR^53JT%`zxM
zC+kDfm8_2$D`o>aH|<3Uv_nM6S<UueF!#O$coh}rI8Zk6$@8=;DZk^MYu+>URvR4Z
z7QG&$l>_Rh^>PX3GJ()(J*M1FE5Rv;Fvx4LVDDuWOEDQSWq57n3zK3ZVkomVujjM%
zv!{iD;zM>(Okxk#@rG&z)dO7zp(M{;VU`7A=O?#se<ho73b~xWYIJZBJ#71FT?Vi7
z{?Rd6f2;!_GME?H8LIYLEYmx<EPt4w+A6zmCHmjEm0R9}1CulB7#`Tmm@q2A&~*hy
z^{g}+i9pVqReVD^Cn6L?H0bFZVc-|+j`|JCS>(iun5_yvX;4IUi6b;0u0{c1SW1^!
z*52E4Vb`p10NPIc+iknHr#fO4wD4g>eO1Qf*-QZHuu;7knbWK;AsX`=!Wd#IkyY><
zCvD#BKdHZ2VZ_Z46nA4wczSHeu0Cg%BBJmFpEV-_$20Tu{=*IM@b?+D0JX~B9^`Zj
zlmrY8n`y-Y*<kH%%=h58Z*8_;Kh6^LTQ%LD<rq*9YC%|?C37^lw4!d|uy{*xi<=ws
zaew~gLr%QsPosh_fcheQUO5fkG^ybK>F1dyI-&oFh%P5S<1992|IUgDtLRH4BmZV4
zMi%%T599*iBt*86adnt}W=dM4+;Qc$#ve`CznDl^!*<544ke*qamGb=*5Rjo&J`YT
z1AH1ir+^;p#g1-IIJO+Zvtpx9iS0Y~E49Q0oRrN&sbAy}<IOO5`Q%SZ>Fpk(wf~$I
zj%>m$2GE=@GWfagGh>Uv!{^oyHVq*Xq?p3pzQOy0)m1{dGiSSoS}_L#x6Ugck6Y*7
zn*nO9PvyXsETG5mP-R_yVkC37#1G$m0f2+PSNq8W6{ckfPH$w<o<1A1$K*2fP_*ui
zq12Ar@UJm=t)9V11L`AmCvADxz|-s_i<ca~8xmT)pWi|1M!E0#>}?*9kj(k0a?%?C
zD4x;BSEmHayhvg@W51<h<ucFnssDjUzAy4MVu6^O6a3+WARza84Z2fEHrSxtAX>Ha
zRRiKiU_SV$-^Q4rP=E}~2mdQf;Szb{!}i?yun2A-M}ks^B)V<ue6Sa_i)Gk7N>gbS
z@RV7!fAe$V%c;VzotcAAYgDog9u4(;fS61<i#3khjNEY|%dS3P)UbT%{y>20#3hsc
zT$Z_o)J(#QPWeBN2?g;ehi8|Yvu5Ev&JR<n-^|{iwIBj_5Nk7-BF$ZWRXOzn>oD`<
z=`)hz_c5WN<=foKXt|7q1XpC6f|_q`dS8*pW++$l2~CGRNMoCM+D(3s<yxKxP2!EV
zr_xvRGb$y*{=x_Rx2dnScn1F-sb<qX=I%%367ZxJ@sJrIpzmuf%eLQKI_3iy^3{E1
zVSN*+-p>G8IbeJ8%wkr=-KEm|p`j~zq=n*Lg@dpZRN>IK_H5=EPL}`bQJbcpZr@A8
z_qU}dFK=bPWO7hvONgpZ^0O$)s!YpPu5uD=CVO(bSavs${^I9z{=&R?IU<c<W(E#?
zu5HRU_(^p)3@A%!wh#7kWAco=vhzpHLZ^QL#;cgq*Ri*);`L*>>r}O!7HQ0?i}aT^
zb;jz~i+a^BoGE@R>g=MX9A6|4osqX1kR9bNWtFrRiKwUKzrTO0K{fZv*R#5M=q;|0
z_~ZE7dyF%lwmmr5!nqL<`_vWDCPAF-9AIC!p>Y>osNhBkAM!8d53C(@ZJ9)Sr8NHH
z(~Tm02@`t@p!T6IJ3RqI(Fb!G5p8AmS3y18qJWETqaKiAZt~R4TfQY)j*i%;v}NA$
zY-Al;oo)YD9W{F<pU)3s)eyQhxU^nhU)volwdu6``wX$?6QK(wflI5xxV#xu)oTt4
z-NW$-tgj^yab6nN^m^m71TP|kB*TZq7<4Z4I;oe-^glj`0+=iJf-LLu!c>9AM>P`6
zS(nik6i~oQ^1Sg0MS_mA%rMJ=<mzpXSP+A8`ftj{{NXqK1Ki05or9SiRU<d(XG1R;
z1^DVKxZI<$b<eJIUtRE;@!Nl2$Ud=MhOaM36dK(t%*eQ8t>F4sb_sw-O!p~cBB!1>
z4cGc?Qq-8W#vNV~vv^{eJmh==@Q#`rI}f;oX3q|oK+Rt%>mc_TNK`zlCpo}bF?qgQ
z9_<zNbT~z8lj55-ipbM?$)GFeJiH{IO*6N$gKq=}f$QeWiS+VCq68(s#?>DQ*TDn+
zes;GnQD7HVIY<^>N@>@J6=mb^61+meQy~nBtj%Oz!tdr~Z>suJFy8BK@Pzt@Bm7^T
z#><w_av(@O{AQO`7`Oq&Co!uj15UVcz5SIH4?X@UWq2M6xCcGal*_ymI4}4gl0vtk
z&{pjbE3DcItflPE>R_xJy%r`i!K{-LpO#Ya+97>E{ZVz*<19=;*|iti(@=%UySoY#
z8CQLi3LjhfifZlNk=N^f@)?Z1YmQwfRpD?Ez&p6T`2OXf+W=NNLJU42U?k=)^tN4#
zdyiD+y;@19-=*7v|BnjugJ1Yd@M_A*c;EZCIfb~t5NPC>{!Sy;*@BA*2@wsrpH=gf
zU!YnN0r>>t8r%$dXa&30vrqO+mi>yPjhGzk0PiQpa7maJL*pK=jHWIH6h+lSy)F_g
zP~GexK$~HFRqr)%t!jTUl?$=oaj%i6?kICn{H<1qv7+|Ic_Pa(Lyd2_HMZDYW|z<D
z`_#pf9C2wdvEzjqexW5`<k2b8v>9SANHIab<7KWLx{+c45@bi;_?n(vZQ7;`5P12R
z#J?57Fki54D}qBBl@evZ9RV3Wkg>dCf<-7oOYm&d73Wm>pUmir0XH<*uIV;N=ajl#
z`0w9ebP-m7CTo8;*|=U4r>dVH{@i<epkX_*jGrP@joUJ@-Z>l3#Je^)F+-1vXGV47
z^$;L*o2czSnQ8AndA0bgPK1lGpWq)Y^DJfbl4R#1253ATR&O(`j6uMdSE7-t9Fjoa
zB0}(W*$@cQWm+W670j_215DA`t&kTYW`4^8#ZM;TJECXrC}bh|mz#OLjo@5zvVX(U
zR-Z^OS5#l@_8856MlbAp?@9G4GIAH;zp%2p#oK8h1-~Zzujg;jjKyxOKyEciIxS=e
zJQ=TkIwN@qGQ;SkP)S|Pe7IC7#LiO+)tW?nZ-|DudS(vDlAs%C1beB7dY@#O0<3&T
z;7t-tyT7m~dR?jR0mGAJ)@Px?$^U3nR~fy?x&b#VohDSU=X1p9w9uRg2w8qGH}vs!
zGl6xj9yM8f1FEW)ftX;W$WXave7@Z$1S=iJfS`Uh8wMR!|An7gF=4tIr3nqcNiUEC
ze=&hfBkr$gKVFm%8l#MZ{$i})e(~-eJt=Zl2=H6B&G*QVz#)thvd}(3T5WJ>cM-e8
zNaX`wsQ)b%&k92|`_Y?E9y{`&b9Ui-hnYuB?kA|QXFwgADp)`6fL+zlm6n)=0^+y}
zsf8dALmA{!{{v3W;J=BZ0VcA{u)!v7kE$&s_z8q#Bara{tN5?CkLW(Ifh6IM|6pVC
z{}XI14kDywHLSNL;<c1rt!6WA^^V)20ySRzU|Rp-ahB|KhUSRPL;-Pcht#s%&C&3G
znxhcx9tme2=<=coXrt{nN3!E~$iRT_p#fEJPN8Xaazg{6Fi_m_Qg8ml;H3Z0146(+
zE~BPzm4*$apk?>}?SN!3T5*VOk8on9#*7UPmuz-+Qym4&$&L^>7uCBI6M~~OK?i9O
znMM$+5Mb&u&<%t?ISg)r%|8j%$NlG?y>JoTwIJ2WiwwGeC<5dI)U0(3(8p3wu=yhX
zhcp<zeMhmb1}!=q_syN!KcWLXpO&ypZgin6Htj(uyF&pWS_j*c_um`7^R=YKHhcvR
z15I{|LC~ETNMsls8XA&nE1<-yO#}u>iCFLm`o^sMKh|T%mYPzC*Ww*A;LI+QjXpK{
zZ|-rxc#MuJ-nQ#F6!u>n?(dKTZ!YE7w2JaNJ0(B8*dhSRQkl5@CWJm43kGk=mz4p_
zhMPm&-2dU;#&Z$>$Kp=>zh`mJV+#RQnp(hTu$p{&0kKM{T}iLQ&Gn&t!z;qrI|uo{
znyQQXzwt=_$MFDF>jbDMDFa-@P}A%G>+$?Aj>aPHk=g+!a-BL$x&O=Y$Pt5rf|N4_
zxO$@rBOx*oP)j^j$-{>Q^8}Q#UHgh-a7aO=(0d~`SIXTjev|3$#**?sFxO|`{}<AG
zd!|MMv~nG*qy^(;K%9sedVF_=nF@TX1shum<S`I8neWDH{2#|Fee#}tZ{cGyM6eIY
z@W1^Zjt7dUHx6G;w#IvXPT-Dluw${7T|pB6iN10+M3tU*UAwYAF~qSURn~?nCC$Z~
z0-RxOcqeF(&gQz-1Bwp?F|n|S`co%m$njh#V4ft5SvIC*=;*BIvWkQnSvN;Pg#*2G
zAQ&hjn}2&mNI>Jt#d)iVW3O2)o`a4-5w-Q#_h~BwIci5j4i^1yS5Dd%%0_9L_9^`z
z9^yr)oay(zV?^hjTnW1B44@QMtWZ6m|BXoNj=Xzw^>nbo#+j4(7-Z#pLREo)C@~yo
zYeTlDt5g7IcnP41o}HyA5Bu*+PjMj>8~I#MHr2t`Bh3|eIWHLS*D<2a&jnsbG^Vb%
zx9kVt+S`oyE^`@HNN13OMy_}J6dh~~ZZ`&AY&@3-FzQDy@&J|hw||>-#Hc09v6?I{
z1`2`&f+Q1K1n7fN#hj5=kb?#~V+wx1;B$a2j|#b#RJ_hd&KOK7u54hR7;k%8J^&q-
zkpZW@4B+w40|c{=+x;)HY>a{U44||^Z>i?}Yl2Q%iPf_uiw$U#!U;Q{a3^=NK=TUM
zfEo1vjc?ltZmGLmXR+5PlzzY=%XY(2m^7{vu1WXMFMhY}?8|FwIcY<=7<#^xdJZBv
zI<_&qo6)n@?45;8aj&+6w$Xrn_m0hDpXI9s+pI&eQL**2o7jzA!RbdxQ2QGMZ~qi@
zJVYbpCg(-=SHv95ZW{x1e-4&;Mjq{O&R=;ap*KUFpQ`t1^tc7kU5*JUsnYM^!(p=U
zM06KU4aQ5(Oz?I31XpCCI9n>2p9$7rRK<E6E01VN0{DIZbC<C?D}~1&oK1KGE>9Rs
zc`Fo=9YIHK3nWYpA8bVDz}D$PM_e#qSrkJ5k<PA1Q#u%rhpy9gL8Wc5Nj4mGtco&U
zQ<Duwmkp5wDfwQ|T8aaJG+00bGW_1;);H+Q=wO1}+gN%`rqBd=tkZd*33du23#E!g
z@~hCpz?XZF5aZSlCExgb`mg20p)*p$0K%FS#9rbO^=tw(TpFEzr5SX+G;{<oqCc({
z0+^pxUh<qQy(F?|JVL+Qs24A#B63u4KmLl){vnRdF%0wv$B)7og+vgD3>tK^v+Gr|
z0^&3#KwtCr_Vqa<z9Jv!(?YO_#JP-!+<f7RU`-;09EE@%^$<oiw2pPZf#q&`<Y0QP
zsA+R8xZtPvcL=AW&>0`XhBF0837J*#favhW{Qju^d{?_)76(M46NIEyaI<h-BrDjE
z2$ttfg&6mP=D=Ja9!PJL0Z8xI<SXhEFgpvm@a1R(`~h?r3MLUAIPT)p23^-R?={Uo
zNFvT<f<?;(hs#<X+5#fD&f$0l@RSV}8zpIszhXe6@0~6Ti4wk#?m`tW(!<&51hng7
z6kjVY3-`f4->#^U&6kdl{j)pQt08g2_~7Z6l-5{_;b_o1s~w3g7(q~i?c1OuJs2ZL
zA=v8UW=k42Bhbt`81Bp)t-&V*42RwmZVTk0NDH&-dyNk|tKow)0uea0shctTzd~<D
z0RnN}!kA`=gS4yQlFompT{=j+$b2z$cjCi<#0N{A`$P(fZ;x{NH98|DEMD{|%KLa&
zuos**pq3;_RhN}K_`el`!^0(Lu74^4eH>bUsDBIuSVRWYp<UTMXAJ2&5~1I9)Ps$Z
z2nP(einrzg{*hj41@wQQS4rhu1AI<vogql{3UO44qJ4f*!G1u{t4r+`!<H|>nt8*L
z$ng=uR|sALn+}&64Fw8-!+52F*S%HEt(gZQqqg4Wng+H@8iFf+E}6n*q({V|r|JkA
zB()Jg9gdPL0+NX$Vq1FK16G~fligoV2#t6v6Ko}Hh64U8O41FeS@h#o{Cf>FU}58k
z6tqAVaD)&58GHp!1Z89dh4QhO%_D@e@GY?85?`3~VJgIf`Hfliu>gs~Va{N*kMq9a
zr*}e>1*<;%^?PlR=B~ZfqeL}0j9W4f6q{Ls7J-&?^-l|=BQUv3@hc>eA4GvAV8I6Z
zlM%rlNy>j~pl~-QJD`=wzDWJM1v@JQuQ$*8MV-b7g9%#Vusvjiu|;-1@3>(G<n7Fm
zhjHB)z;`|DB`u*sb*x2$r1)9=X>ui4v(-Plg~JBHw2n4*9!{r0j>eoG*tTp~KU)Zx
zK*}#0fh6=E;t;^z8U;38w8%lLHqbS5^;2GV`V}(pt5jhbC`>qDjs%fnD$IAbf(f=t
zopjO#7!9Z>3@o~5Wkib*S`l*SXhC$a0-hac07@q4oCW&!eE2n0@mAqUuT7CC7*6Y!
zSP=!G`9dke)P@o)^7kWPHf>AYj<i6WrqFt4{L=RB0RhS)Fp7eWE}4S9;L?q4dG%+{
za_{^50;p+<`0#vmunM%;@i>e3meD{Zb+XAm|4atV=Lnk5odoNofyq=RSc@x#f!P&p
zvFoLvHxq==2A)f8jAR=byA3k22g?5Znr>pKq4x@17BsDkE~Q&X0x}bn7@V=K<OS1`
z?R*7X7YF^dI^=g0xq+u<(GRL?;)Q03C236}3y;806Lx=Bs$CN2eKMxd3vMLZWEcXA
z?!X9t8^5iW3W2uXKQ||doP-SoI5=$uwh}ZrE!cXu-oK47LE9~Qlt<}mj4)T(m2}%2
z{05t00%%W1F@2Ur76mK77=6J+Nk9d(5o*=V0d^f;h_zI(l~4T%(T)-{xmrkqV%(xT
z7N8NHceDE(AN>jfY{u9<_WbthpcmA>$bxGrHEW_A9%3Z0=~cXMoXj8)9VaWE7z#ks
zfrB8$8IK+JHR3KTu|kW*^hwRU4+dn!@+DD4PNE3`UHhUPYTERyM{`W5M7=e$Jf;nm
zmYD^D(a)hl!$Q%P<bpG45B_i&$paN`!#nFlJuWl2n80!~A>U=wQLjq}{uCcZWeUOz
zoDnwhPK^*e$P$50^2FfMgzm$`Ll9B~wU8V4Am^YGgDqUk#*^oQ83v;_LjrW(!Jm$V
z=7>iPqv|j0j`8w?q7Kb3S%DCAM)VNS3?>G!^u)NX_eFyPA7d#AXDWIR2KbXZKZgN?
zxG+gs`7ag+K(UzC8MsR@;0g&?bW1Ud0d7CE8nQ{wD0FX}5K*6_D8Mrr4Pr_RxY_tr
z2RP#0DR2kfPT%kE?N!Z8=9fQrMkpEs#X3(eN@(@3;K0quJ<1qXWk()K%Cz23N-iHV
znA4Fo2S@J$^UtBLX%jKChXn}0FB?K0)p4fY0`jPMkVh?TnA4#L9yJ<xR1HrK$~POp
zrWB8GtWVpbJ7aj-Spn4ryj-V@o`l=@*RXuOE@Tym@eCjq^^2qpGXUW-;jI|t5G;O`
zhkn5Vtj!M?Mq?2ax5hoGIrg6-=lJ|2=LKZsyq>!TV}MqQNAQgk_$CO8BtT4}^Y0zW
zz)IP|!PX+$b`*F6enA8BBgu<j<e4Dfl+=o381S4v0W=A!^Exm+{P7kb4W74-WyV@p
z+<8hG1z->HUiUK<fJBrbdtmyM9t*{?Zpa>7HtX$`VxTR$_egJ{0)2O`L4k->QxZI#
zszF`Ue+^Dl>^*I{8_HBpGJY9r=EoF`4fGF+D8OZwNb@wrLi2@=0jB#VO<ab+JtS6&
z_51pfgaA_e7{D$R^W_ak3_u+FG-cndN49eZSpNDcjm_2n**Jnr-ZHOgB}`Eo<;@+K
zfmWcesRNZMjRaa%-8md)pffc-Z`Hex#G%b&mRr}i1VzlVt28dKU`RdSs{~MR$3V!I
z14@xZwtNjlONR7i_ilwa(D}on(~t^33NmSeA`B6Ic?9C;cTolpF;=oc8h2YJ5PTC4
z8S(`P-4n8%sEF15PLiu^(6l7UkQbVmHw|dsb$bibq4V)vC$M{6#}Q+&<wg(yuBc`w
zDqP(e15*Y%y0MKS@*(|Kkn>EyL@(MdO2-Ts66k~Cc3pRpun#R5wr#oJkq#I@okFLR
zgBx)E2Ub7RT!BNd1_xF=3FHNVxc-{rTjAt{1aFt|lOVS9SZI-WI3YUOeBG;EJF=a!
zKmgIG@zH>gx|s$y7PsbZ*}Hfy^BGxIA5C};slfQMa3Pp8Wq;TVUz5I~qN1p<eczvA
zOJ_*0w9rs(%MFeeoItVoDtzg0XYf~i7k0$HLR$_G_dg4cH?_%u{+=B1L+rY`g_^W9
z{jkzR*-Y#YFrgJ<WG#>J6Xm~k)FZ&zc9CZpz7oU(qfp)q=s}-z!kmqg-8sFQ{u)w%
z`3%Mf&-b~O;4#tBCH^ehn%wDg1N!-m2`hVK?_2zL7RMIy4_G6dy_D!KV*9ay)*Rq-
z7ZSt|RfEE*BJ%)+20ZizklK2)c2pABx9;OFwmqzbp&6pwt!O;RG4dhU(JrXAscYCk
zHfQn@^~B(Fp_t(L=?$u(?7g5=$cr2?tTeeUu*oVZDi#UbKRW$XM+G$98Y)??o?*Hr
z(bP>2^dS`XP~{L6W*qNVNIQoKKF^Z-r?HVg5-N}@xWIRr7uqK(U<Bu#EWT{tMFSkL
zPE#qkE{j^t3GMtU7TC;LpVN!Iks!T~#XF*a&qwH2n)rqL-E6ONiy=F?h78CH6ZF9(
zuvgDeZ*%5^9B~aip*;DW{$>203o5Xlk(IAntsw|A>r1~BnnJ<nw&1rRWjEZ6jzb$K
z5V9MQ2v!V?Xg(IT5>vrc8E`Yu$5WhAn&7zU_Y{Hxo~?@w+mdAPj_dOi1|x7B2yOts
ztu|7(Fn4Qf55&YM0a;A8bV_)*Y)0}s?HbA<uuD2QAWPnQb897%B?T<+p-=<Sb9m=z
zv*2Z76|l2T!+++o@A$WY4~Qv=_Rd`Bl>eW(+->mxXD<KGT>hVT`5(MXS#0NW&^258
zja$h;kUPHXh|eZ7KN$!K%pN6mM{%Ivc9-jbp*aCV=czY=7fl7YHAr^-3v@nRrV*w0
z#D5Q)o~n*G9LwG~`qBzr%q&}uzT2Z;R*UHQNSc+q<akbn|C{K}ufg8_uYRp8rgKZX
zL^l3vJE~BEhQ#2+!%Tpw>@#qf56eVi-*{00w;Lxj0!t<X&U<~{$7t!Vfn(tFux36#
zlE?W^_s&Uv9xc3$OQ(zbdz9IHxRg}zYw^R!)+X|bC6Nh|sIcW5_^T8)I%l8X)vura
zS)QP6yNwg_+^~~0jZj{suV&M_zapy9L;5egMIIdN(FQU%)hw3wtsi0Mp#PPm+V4E@
zw%J@|-~1l-JtpbRLA(rGh#`)5W%TUd(+_J$74^~W2P1g4wj)&DgrhxJAyyK!#K%{$
zq=xP$O#2mPx^3>{!`J;LekjvK(=Yza*~Ykeup4O{`yoRCCqC~B@5hMn@NnM8$v=S{
zdNut1M7J5WlXTEr8yY7bJWDK;q;Y-PZcLSCb>v&Zlf3)`^+$qnbEL|qnSVYJ=O(rn
zWep)>MCVv;Aj#rE8Pn~}nOUNe`gFIrukH%9@^nghvQ7OT(w%HF=&pn6m70--&S)uy
zBgKV0VH4`IvF7+Tx>C4?PveT3_U~z$Q1hmr2hQ0hn`id2hyR$5S<<OnqL_yI@+rd=
zZn=kqy0P~h(^~oU$AFoW&+SgjQ9-xjZ1rZz+QvzHZUK*Ea^3x6i5<**WQ@J$V+)2W
zEy%xc%+b;qHnh#Ycdcl?+Ul2$%^6N^6Q<jdD3&uj>=oIOjgCAZ!It;H_IX}J+OlLa
zafXvQs&g==no?yrCfaL-2AP@Y_qywi+@P2PuiaOeW!B4)CAOmCt!KgXu)6rKe(R@n
zhvV)<)A>HdOrfUn8Ms_zLQ0hx7oQ&KHtESdZu5J1>}DMI_t9ab_kP^7wcO;trPo(A
z8BFyXt@PYac_;XgYe}|_&5=%vC%RneHO4wml^YEL>QBTDwfgfB<FnmweJoGj^&)Qi
z(2GJQ+&v>C;02mQ3(1Sv_rWf?DkKOV=1?-LU;puw^dz-yQ`Om@)m%9A_L(1KSV7E*
zVuJzjq*?2%v*xU5UC8dm&&k>|VGktarK_d@F7;avtrTxo0SZm^DKXtIhT_ht=Z?J;
zkITOx(Lervzpm?%F3YgNYQMi05@H)7GX9_}*n4)!1a&n1FDWP;T3#eF-g8d)evB&^
zplqzWLeXK=?(LVIHeO$IU|d5yQ6xP!)qWuK&Btn_hQBuZ;JbrN$ty2Y<Ggk=PdUA@
zpi|D`(v=n|niEbl{p;(S_yU^_fr1zBBfqL_iL1vWa4trCyp|#$Am7R3cYhn*p?<wJ
zwDI1;{)&!(KwObZ>_56;QQH3#&}^ZTW|AoQ_E+0cZ)uL}#PAk*{Gmu#FQJx@lz3&Q
zo@y3{34J!p%gS7ykvE~14zT@W-+1j3sjoj-f4!c6=T*W(utQRbC%9}TFFO4$?){T%
z8Y|>ADaGsSZjCi}wux_BI~wgoMtVV<ii&o!1%F@63(*AJ8$Uw^r!Aw#WSh%8@#D$u
zxtQKi#N?EBCl{n9@FS(5qqoX?EUK7S9k=RP*T*~_$5rtu#HR|L!pHD<DydG;Y+a>c
z9lXHIr5dfIQp&#HS97XiTH6>Ob6e%Lude_8tKHF!zP$5+Hac{;)h>;hn<Rr?HBjLX
zVp60!w@!tEr-~3k*N=*HiTGSRg+H_e{qgS%)!{$2=jXa___V6Rdjd|&qu8@4E>7Q!
zh!D|*pi#u_aXb60sCU>ZDV~4EKjr~1VfXAyw5CuvzbYyAkjyB$yWbf-m9hsz&a(<@
z2Aj+vVHd@Kh}Mlz6&?W|jNR^JL-iyY>9v+&v*`S(Cu~&ku;vmyCJEKAJm0wpQiv#D
z@7tKJxxS*q!<FvBT)mWg9zAEGkd#mhBkc78_lU<jVlp4V@%^`)h`kN6tDTZ(+m8L=
z{Hobx-@CEqoV6pWxd1~;-VfeAF>A+f9AtOMWqSApjdxo5kN_7qMs#9s8}mrZZ<sCS
zh&dZ0BWTG^8;=P|s1Oz{Sn{7|F*w3w_{xT_vA#+&>O~AG3AHJCl?KOWvNf)iN5-n1
zGaemh=7S0EiY%EJxj$WN@vhv=!8Rh)A04=a%dU=ph)U(TH2Rs8@Qt-^GdWz=(Upk~
z-$=A@c-YC|$RPGd8*M2Ij*_6<j~xj*b_vpK6uCySM*%^;=opiaN3nr>=X|0@>u9tm
z<98ObggdD}LlL({(lE>06meLz1HH6ea%&L-Bibi=2HhR%GWZy_8lQK}(!HFrD|AL7
ze|&vA-V}dx)1N!uvwt1zupIbN`yyr7XxDd8GKT|C^Jk{UhbZm8*9>l?+@=0qtNyjf
z*x_QCO-dcP3Mnn6Lp5Kx*Y!3?P?U+V|3sZ1pAnANtUE7zoYzm?I`ei0oQ{#QYaXDy
zHD+|?)PK-vO^sU+SbKYZn0ZiyajCJUWnU0yK(0^94*zZ3e9}E4W;sea|0Z|Y-)+J`
zb6W8D<7T)Yr^$?!lp@^wTbxHOMp}bOBFkIJgby(d#uWA9EPpSJc5fXsAB>?{pd<^L
zzu2n+&1s}x$SdGb5h(-;m9}@C;D0Ww|1m<BL%txjTp)gWWtr21>bKlvG7j6|L{wdk
zA6NC($2kyz-%y%yx|zHkZ7pg<#r1EP@xr0>I2V?ePAE4y$`)l`FEHj5{MB15Y%^{A
zConDX?@X>ENynG^YVIsEV=q5fM6NWKt$3^cUR)8cdy}i-C+n~4SJ_v$r7w`a<8#+&
z&ICyuRh+w?iF@3;mYGj)SzEiUX}>u*dx46UeXu{;^lGByX7RnGVtS2ter=uQeVoqz
z!_liCf_@y@HwT~P%ByCU1C#g8^M+omJp`9NmfEM}OVAa`>oSQeS^}nC-S6SRXK&8L
zcyjuoq@CTAR$MJ0;<PM|`W*&FbF&a<gv9Si6I(?ESHI$`lS1*G81r!CN|%o1c9oB|
zd+*&cp73l(zW>IS!<|_>eSNvk(`&Isfw@!SFGxb*LP^Q_^o3gJd*10|v<*{)L*5lP
zlq-x_`&u_w8ix%hTL1DjH}ho#0)-k888NTRNw&JvFpWuHd`$V71x?M3ma<y18`b`_
zzJzZM+cDf4F&Lq3<bV8qnBL7S1{MmbJTbDu{jhoL^lTV=Z{HjvRT$md@1&&rXmOy(
z<<shjQTM*z?Hh;TJz8g-9hV;d5G|wI%zH8RPN?#LWuVQ$t&yQE9NCi95<6CGt>bSF
ziT=;FCvAj!r1qa{l@EM+>@J*#^xY)fRr5x_X?&Od!mY6@*w{-{y~*ZV^(2ykW}}JU
z{(v-)cJ_Gj&um+L$7uZ!v-2i+hl!su$f1#jb@*Hd;;o~tA3fE#(O&N@o@+qvrR~tq
zmSo>K$BOE%eLAJ^Oak}8r6%H)1_EfArX(w+^ZoB4>)%&fMo8kkO{-pkC;i>*F86<;
z?b50hJdv<wV!TbIdC=ZXM%N#E%=1YzPCLTUiqJy5t2I2}D&e#y>K<1r!vL*w_R11Q
zJ>!DazS&_W(x8=EXaBb6H{bZ%xH{=Uk0L5er*3z${;<8XL#jR*YC-AEFbxs%2%kmK
z+n*D|Za5cu*GyGpJc*}V-VTrXZPL!t`h9UtS5oOQc^av&w!@Zh;Xj!42hR#*zgqI;
zFXR?|T}ex!-rzQtyk_bTt2zjxSI-t@dzIQ|Vi9&`AL%c+T2IXzutM^{pv&)f8&!$A
zVtPN5AvfX0Q%d@Djxl-upaCyDcTQ{3gl;(k8QA*V$Z2kcFvS(=K#iM8E{_56%}B|N
zG=1R%=?-PtLzIpWP7dO_Zi1|@8Ej^;y>58y{55($e$L8IW}^)gb?*N@+QTD3qt)@T
zV0_B9sCn!9_ZE0asWcLe2Tv(4toZHa#~+g!zx=D9I7vBXcwN&amMHqO1#OzRgXR(a
zlQuqsR{kHvY$vx1Uf5`4rB75TOZh%ta}V%X?Rb)i2s+m1=q(?se0c2f((_~<Ym1WC
zMJ8FXr0c12RBAnzfm<gUH!4py`}+zO8ao}CAl+QMhpNqzIR=CRXlIkvtT$Ww4cVH!
zu4)bsCH*h<eLECZaq&Zb;W;o&vlWK-`0;QNs1#i>?)lz3?anjR<TlSx{pF<6S?T8y
zktShYu>A0oNm^z33ui`Uy>_$Qqkwkaz&J_cP?xgBEc|EV0tYhpPw6xobxM3YPAw&u
zJ46EV4fr-t-X~C=f2KNq{Ju^qK{Wd}%?$Ol($}4b>i$B1)g+SV%>l0Vsh@<mPucQS
z7&ejOC*N+7If8p>_wf=AJw6Ux9-q^G@?PE~eqguJJ^bv|JjW7g@-Tvc5)!+><BPBj
zDPvNHuC5R6=_Ql$y(3R$DlFs6BZ5yqHZ??+MO2-(-p2L*tgm}G_0A-BBKNnjiOJ<<
z_`$GjF^GqR5rRW%>d@)iwxe!3TB>vKwAtB0NE!x<z=-uLozJD7=@vYACJifsg_lsi
zt<n1;wr56@-+-f{xNvG6(TH8AV=86L=B`HhTzaj~PX)85{{H&Bxl?RjfA9Xs?itj8
zghYOf>7(cBX$(t&tGe^2nlxqBr*=c%wd~w8xknD%H!syJef#OOc-Kigjt!2*INr2K
zzMEx2#{j3$lf}S}p_%2!#u!-nI>Z%u)8F4<MpfM_4yHVaRi7ozP6$1Cf4uyygt6~K
z%)8`~%ED8WN>@^x9M0QYhT(u3aokm-^-VwJ(EKRzA7>c%uIfBQ*t+^#5%QUTqa6~n
zmwNb4Z`cTr5fI>^ey*oP^!O_lU_)DW9G-VrH|@cg?w%}8oUDOoq@8hwKR@k^HGnH!
zBu`)^67fMv%`axSLf||nL{8hY4B=%RBKKR5t}F@9A-9i_gGCy`M_Wz&K9zk(b{H6V
zImsD+lDN;tQ%?EL(?<t6Y2F-6++bI1X$R&G-;>z5igR_D+>AllK;$_i+xxJL^{1sb
zhv*0sU8o8JF_WXIiY2<-EwD~XO}fB1`wM#$Yn0l<Im4G}S7+cxlkB(TH>@yu_?u;3
z8D(;>YvksvoV+F)B@AXFBIB|+R!e|2ppz3^u*7lG7c;ta-ML@SYTDlD-8l8Ismp+?
zyI$OGq+;6P@~SR32!B*?x4Mrv04Mjn?M>OATJQ4b8I&By6mh3CR9}#zN)F4iN`{fi
zRg$CIWY~XZoJES{eD@7Latq}RI<4$=qtb|JKzcg#(#kK2Jp1LvPhQp>OI05FL7RtC
zTW-?pIa*2{6+Ta>t*~kYTiw4M$_*x8YG@+22a@#=TPtcB%VCmCQn-Cm8umr|`ZhsW
zbg?6rm09VHCYxBeth{x2{x-or#ZTLl2&D(><jwox3z$Migahx|<3)Ewuz$Q9iRp{^
zV4B90Dr3s9X_&oVp|AVtx2SuKIPLy~)4F%_W#9HAa1ghyWFA(0gz}vuTX{xP0>SRh
z%$u+;9v>FT1`D@o#L}$r!aA>>4R(^G?*2e5ds9uSFI+Rs@QEjrAYn6Wzp(e^%u*5w
zzE2OoRs&Mlxjtn(o<S{xRPwv^>oLS9^X7^A>KQC?H*PvywVy++e$OC&)ud_M;tdv`
zDtXh~(8thT&nxA{eOy$|L3SOgHBb9`)%Byx5{Kb(TatZGLDfX4D=hPo11vEF5yzt!
z6~pllVwqfF^c(MU_F;6ox~)Cj-buZiDO6k}V!k}}YfocGUx%s;s?JtRBukUh{ZtBV
za2hXZWXn@R<cMq=*I-ODz!PuoSR7(N5qM<N_0oG{-eIb2G55JX3ATMI`)%eR%O`$p
z^VMM#wZ?Ezm!LOlT$yHmg7V!kuLoP10qKc3ybtGKzD7XDvprEQR;KlDZOX-u5}73L
z?|A~uPaVXWZ@vi#W|qgjM?f&E#UJJwLMe>xj9+$KpkHwyI$$B8feTtDqUt>>7x11f
z<;xuBfe-DuX&?J{^DSp(n^Dr{S@2iwNzO%<OuEmaZ>c>Kl3r8UVz;a}ChIIV$|;zM
z<~(vAeMfDXyP#uv{E|2CxXU#T1>PVa3XV}6I3PJJO&9#}-XPIs#HZYuC`on(=rq4h
z6_+5{>97zsY4cMvH&~}eA+Oj>+j+Wwa;s>6tLa73XgO$1@KqWX%9uco_qFSlXXll>
zf7dr}0)p~_Biir<K_$zz6N!?ZKEY*u2Z2h#`BCcPM)%Wz^RELL*2Ow`Pae2^`hv3|
zb-N~)@{J~y2~L^EPvIDgGH!17etHeJ>@ntNn&~kzjSpW|(lx(@%5(q0(;LF6%vPA%
zFAaZ07MZ^b+w()0G$$_5uEfCDxQB>$Nzzq1`Z_Nc&t@$kBoA}2vH7!?`JsPz^|M9X
z;b)cc7D}*PI-w9$_f5G`4mrAo=kKD`d<TOIXv?%&S!PZzdY7uqh60wDbJsNPOGD?X
zhHKz=9@dJWv)L8N*rDk^{08s4#ZN``h!nFWNnW)=&lfknu6}NoZnDn=w;K=L(9)$P
zdXL(O&59<yaPZ+k&lIutd0>HxmE5~d0vc(~T{&~QxOMzoZcK0SC6<2&EE;UO5Lau(
zsnlm0=u!_`1+x5BFSgHchEQH3;RVXrIjc21D#0J5<qA}Y<#BO8v26ML?C;9}17`cO
z@WB293dZ2{+WzhXjSd+yTXU57W$spl&IedNC;saw5`y0{5+qGaDVM16dPxUHd6o=I
ztT}|@ms@q^$9HahpN+=P%FNnkVx&G#AhpP9oDZ|g3U-uZNf3fNG0-ud2!-+DOzYQj
zN?+QKiq~ng4@$G%qd&}lr?IsgkeeWreZH4=iQ_}|J&GtUAsAKcr#|aI7X=cT1t&5F
z);LO;*45=&h;2oM)p0H}Bb+iM3-g%kr>ObKR{hi~c;xjxvVMAdX_$``jq6aDS2Q+(
z!&hC!w37V<y_DbFFdky~YNnbvgOOxz$h9mj;(F#j2iw_SeO6)Q9I1MSG8dX7L{_o&
zHl%2miGbpw$d?n(M0<sbld0(tW3YReKl>s0Q0JgbCmc%RMAeaVbIBOm$*L$?>&TbR
z6YjB~TQ?HZn4e_8d>qXh9^drQnb}KtSqF=v1vFdtyg2MXT!-yQyykUb5Nw%e)kV49
zKMa%+MhN*%`;6e65?x;(+^ko(DJyosg)#}cQ0@^K5S!Dz1bM=Y;}1h}q@g&5NCJk(
zli|oC5_1(1%{yfZ0Usn=<apfXzfzD$zA%T6_>TyGnMyvO^Fc_Zy2n+7b1aWC)y`Wb
zk|&06EP=-^SJaCfzJ*sV)9}Tb__M6Wa|weVPAtK_?+|RligUd+7p-n(ejn&xX{2QA
zr^`-62VC<3k42BVQys$+P6ZzpWe`^9p+j^x#d4Vqh7U{<nq;0gp3rsD8a9M|Ld6!c
z>q^HU{C1!ar`#YF<gUd@0LN2cH`$}>{hl$J-apotFn~^%Khzr@-HSnRxmM57@al5>
z2#ve*Qgl*M&9e3<o6eEVO;j*KkbfbKaw*R1FJi~d2N_Gh*JRfZ+a|+*wXouvrW_D7
z;_;6C3H1Avi@{DJ1z&JeaYKqJH(<J{4p|{PSzSP|M7zgnN8MRPjEtX8@Rb!j3~Dc0
zp84I+?S+wAr%p|glAwJ%mkC8htxO_F1Q}3O<Y~@_KKPwf@!`BV(I7?n4PB^>M!^Z(
z`!?$_j(<m$%yZ}y*8ZZd^-X=AU5$ZJser1;G2V_VsN+DRoiDvf{W>A|e%X_Fn}RE`
z_it7zsvB|I_Qu>Wi@XU5G>`00{Bd)SP?2p7jPiy*hdXbXBx98?7~3~}$7qG81i$Cx
zbgJ4Hug#$`E{+(N?9jv9RO7HhzknBo=HbKM^^L>kRG*7V7;(N0Qq+riVQ@6EC#GH{
zvo57LNi%6`UnFf<VKmpszby8iBl?+^FgnJoC#69Xg8>0gN(^XmN@sQOap6SZ`C{Mw
z*v()+S$=9i4>`raEK^Tt;NV7BrPsgg{cCIN7cpt!6Z5!_Lj^^o=@}f{bXS7@7BDgT
zE?m7Hw?z9qc^z?UqxUNB$??&H;uRaFjBDrDM?+{2M)fb>VbD|^gh%J^OSJ5kH11Y}
zKZfs1`bw=BG?+P@(pXk@iYy$orF+Kroo0ayZXLx?Asu6%%8&c3wq0`CnT9d})&8K$
zKaY%r8?VyS<bjfP%O(5#t)1Zs{#4B`GtT*PWn7f=y)F*qD3RTRL&}E}t>U9G;6brW
zdwQ9+?|!S1=0ez1e7dSM%sDUb<lVYn2Fa!7x9(O;=1H`Sxn^4Kbk(<-we`|VE;D>D
zZmtnN#;4=Pkz!>P4e+-d&b5-d6^&?>kz6?$ya-NtNQj<!dT%)?be1LH_!YSF$<tIl
zS53BvU|@W418031i4)5(J#~++J4sJ3fmq6-b6Z`7kbsqPo##@k<YGJtJVL2V&OFW}
zxf&!o5gQ8xNAn)EKKq{3@gw1Z^Y6pt)f*@-bz%+2E^Hk~`t&b}`^R}7rqDl2#<;Do
z5oBqt6lKX%*CWRuqM_q~`5*|YIoaSG`>2HQf48<LMlB#6j7uQX?}$126l!bK!6%Ec
z?aGsrJ>FX5hJqeiJoCz4$SD&j?EyVQW0fQ_OUOiCe`Xm!3Q?rR)=P0|*cL98B#{Nx
z0Ud_y#8mpo((6%}j~(;H{#25?__VtwV+3b~{OJG3-dhI66?NUZp=qRX8V~O78iF?N
z7ThH`1W173);Pf(f+R>1+=6Rx2?U4W1oz;6H+kQ4>Z^OJ&i`9={zKJ<wf9<cjycA&
z2*+>!#>D7|lm357Gh1OK@vaw(a?DP2_hbx7SsOz}MK?bXtb*I*@Sq-eG=%lWNB#yX
z1)E{=;-vd{Aw=*#`ObyCl}rVBiT>b7Jah#*75SBL%=U6&^Wa|T50i3e@w6wG-*Tr&
z)5q21C0i!Nc=6KD{o@2Q4cOH#oSDroC`zoL(DVk0H9ak?7eQtrb_@-w!RYjrWyF;u
zJqU*r^Aer49m6fdKz0k;dY&pKm|hSEn(oU_k_U<87`N}3+#f3lUMxa$nh1Fjfrdk9
z9sK>bpxXon=F9W}^G@O;IOgk1x4Nm`shiutX<4EMheU%HowdBfH!@Q4>7)8_<~*n$
zIBXv~Ci!rg0V!SDY=kH2$q_p6+q1Os^l2JWu7GI!uv5oSi&V8^^zsOk9+1~{qK#DK
z&3IqF`{BFMg}*0aYtc1A50?}^GK0f&e$>U|btMDMAM47xf;^S8)JzYa-!ySk@v1+v
zPPU6auf@89+dXMUp<M&N!szWT6Xm|gd`pvKvs$+Wscw4-RKPtAjk5AN7v4f6Yz#EA
zwwlZKO3~!TyB+OAa)};l#me`|Oe&Eur5HV6^D!Z5rdJVyT(TZ0zS5D&n#O6REUn<v
zPywMC+>R~D1vBu24Xk-m=8oI!sL>8dv`a#w7*Tz_*|2NlQDi#?W8?e#GZ#ut`QTu1
zB>B;oqEeGCX2XovR%d!g)WNNIH=<$4qDF#q^!J86{CXbm(gigkrY9V$AIF_V_H#aL
zy#Smami(!5it-)7-+mmZF-08UKjsl&juU^K?(jhHdG9XyFaZa*NexlXKgbI^|Da{J
zk3{>c#u!h2@TRO3enzD&`A#O24>*4xIV#kS6L4>?DK_Zr7(?%{1^hLee7}{bm8dZZ
z2bap&3xAEv%@S1^USQcnr0;Z5K)?He3k|2|o>xL9kqdtLyrEFxO;lL?w%eB0;HS=?
z-Viu;dB|+o%ksXL1p_tjTKT9?%+{X|I;D!msfh-xG>jT*%F^Fbe^$VQj>U$jia-}Q
zLzKD7sLyT?mNd;szD@-8WilzTzDy(^1gV<@7PS$eYsuPN`e#*(u~H)(LF3<21<lIM
zSqHNgKg0>>L}}~&4m_?=0{U5!1l;))%7GWWshocp^+GZRUd3P*6*xhm?+JB$D>tRa
zc}j0+o0&ZAcCGLq0aJ*a#yQ3!;Kk@7DpBB_&v`b1VPUZtoHFR1a4o_{X)iCk@Ejl5
zhO~wG6Y(SoPI2911|UhvOMk!%X4%j1B2<W=%48aptUitlhjQ`~HE4Kws<}#O((U9o
zG6r4?JH;E=BlH*+@?&RhrkNAwQlK>Hd?>?vwl(VGI#RCVeOA9d)-slch~Sbhdh|*0
zsHw$^(%e-FYIYWvuIft22lFWN?+|BN$k-g*DT3bmCp|#9uUq(Ew~cp%`;?cvbwb(b
zGvfmTV73H1U_q3zqL3nTfo);xFHq&<1HzaFIm6zLSLCT;);uE*Rk$5*o{3>rk$kXu
zZm8C4nox8m>>tIUbji%ed!zJ$Dg95FA_+BJ<u+&s^;C&1MzHlfA49;s#+F}n1jPKR
z@uVRY7+Q){TIB$SOF+cEO=XW31x4z`IgC^3Ijl^7ft_ah6Am=!qT^S#1#yvC-z*kr
z;(*!tZ|k*F+krtGi<rsZ9H_sfd|r~aq)Y|93ps~wKL1x^H$4LCN?AWYJr{NvV_E+l
z*$I}IaS&JYIYACD64-C~Bkrqc;!D{Yh9CweK3GCtKe&(?JXFha_=PkkAR?t+ugN>u
zg#za703)AYO0<6Mz#?jn9dxQEWyqiEPGK;|m;(pbD*ax{E=q6!1dkY)ggq1*F&8%L
zMAhq3!A~8*wgjP@a3^l_Ob*uD-=au3vO^+?{*j4uig~E9j~W?MIxyn)iJI;>+v7fX
zJ&Tfw_OSFJqM^;|q%w1=LA(_LazHK5pTop_<6yvJ-=z2AnqUL^y&hkE9$3+5USIgD
zS3#S=tR|)KJ<VEkqmv3nNtK8u4AdDbH?{kvX_yRsSPU)#a*OoY)6q!9-sJZyg+wY<
z8*?D_)c76I8X+Dm-h6OVb3_)Osj@t7u{$X^ML#ET7MbP<3DN)I;Ha!QT5Ex7<CV8Z
z$V?q{%byYD_l9Jr5uu7;HF%VzV%!VmHYPc~qm790wY-ay6_E{Uc{O!-wbs=n9Bxb|
zfBJC;(9%agG3b4yeep(4zC#n6(1(FKA8DC^+We=X$GOb)&BvhV<}DNQy%MN}+;GB)
zYP_ufg~78%q(-I6yy<jOE;$JV^FLb^w@Y58n}73ui`KnbC^Yj6^!ZFrH_l4Sw?}Zm
zy(Tnr@It3d6!=Z=KnvE=rTHaIg~nEx5p-B-8aRn`h76eAk)=9!n{N;gqYY)|UM~Vx
zHvRhd6oA!lyVrhvSYED^AA6xqLP|CXOhvzvI-tCY3MpFzn>!UF*&wt0LAW6H-&e;X
z>>|D@W!8js4fYzIa<@xGv?yohuF93;@olb3VaL!g_&M+}UXfZ=<|@_L)Gg6m9L&=D
z&sG`sXa|}$KYfJHnw^Li_M=1^LB8tt$dqD#%rfm%+Lb`^x1QwHhbhkj>QU7n9<u2N
zK@#nV))dZ+bYm#f#6;Hjt}&rd0oAeCu9ulg&Ml=o$Jjg+@Y>)&yY=HZqATNG*){sc
zPLz|+cns<|NL13!E%mRupZnch%ZmxNTzpTTdJWHPKrYMJ+k1J1Y1HoWy1QEfI2^dT
zZvO`Hori8BR@N{8|0n6IkEjjkz(IMqi#^@D`0t<uv^Ml>XBM_~zx%@4ks$m#CPCuG
z5f9^h2xP*hWm;gy!G#=!Jv|iep^I3uY;rlHtplE~iz75J1PbmKwWDk5t#-vo<)I{<
zs1WvaiMHc>$MV;_)eMq>b3_IgTHKiLKCaq>2Sw>i7C0BZmn>Mv#=FSRR8DR|_ieNl
zacONO7epVdTpYS1CAyZ(^4<*vf>k_U=AJmy($x;d(7bjgU@yW{cNcs#ADzA_;;U!h
zG#Pn^{@PFX?6ed3z{&^QAFg6j<2%Ig&x0?@RKMjGy2HMH!~T*U;{7l=!}~YV6UX^e
z{Bc%TmX*V}bCx;>+`H1Pnn6V#Tq9exbTsK%52?1`Pj$%m;cCBrZdx`Rc4XORWZ&!=
z&?YIk;mt6)1e`Pxa?ge?T?XDEBI_FK)W6No!4{+lr=^pa4z3L)L-Nsz2hC%~x^2+C
z#@xydaQ?#b#o-qY6lYD%iFb@BorHuc2eFh}allW=iXfpt9s+Tq4;RT#wo7aGh_phx
zbSH=5LL;gWLyhQwGAROqTdzq_I=Gs8N$cr>S6FQS$ToZvR$BbTEpZoaPB;f~MwOB8
zkij>Sp-%C=^FOz{NAa{dM%K`1iu6}+ivIp|mi5z$^1zy_X+?`uG9(ElcDgw1h6>D<
ztd(Ij$v)!EEk8cK1xD@VZB<RGfSHg4IKT)$f@+w-{XHN|^qPE9lXJ_XE?nruJcc^L
zq*?6Z=}4^*!55<bKyLS(Q>W^czsMms^!l8-*FQLE5*z!q8Dj#9Ug!ypFt2mFM@Ulc
zvlH(x$KCBV$3XCcKG*Is`XNZJjHSwJ(zV^v4#+KALZKdPHEpAJaJ43Z9@fla7=>Y-
zpt3b-62yYTh-IC-!;>6-WR|d~DlZ+JvV+G<C-IlYLNn%flgNsIW0ZK>5lSsqeL%yn
zIK3Y}i;ju`xy#*w;S*k67%c_H@QlOEU^^bY=;*NcvCN=;#Ap>%9FCM}{xK#vAz|$X
zjQv~L>%JQwz>-?kn{e$#iomYk+0gU9O&Bw9_}3kQhq*%*31)$N7bh@1zC$%F9nL#V
zl=?73a5qb!I55oy|25%OxQR{eEX$AK-VRCY8QU%Ed8oJmH)w{spW-&^qNe>!>NLuy
zvMh1|aWVBuaK(b#A2#PjK0825lW@5%CJIZsIe?ocUug!fX&{pDB0MaP@coI@K_L9I
z>3-VqGd_0B<Lfrus#mba0B_r7K~~-tgcj_EI(z{2EtiV7K~ow5=89Soc4jKThoDzF
z;I0oW{Cg~pvb^KdNMFV$=a<`uZ>Lf(cQFo#r=w(-DG%`wTz_rJvlmc$cZ=6DOgQNr
z)tl7bFFL?zUWx0Uy010=rxm27r!yfH(5Bj?Oh%QIpQTrE38P8kqYmnrsyxO{BgD#~
zS@N+<z{jxB^wDYEM86~}4=0TS%DSRwq#Q}BX!=7zpIj*k0IkcLGKqq0Wf4&_AGd!|
z*(D21!$^je0EIFjP%2*(Anm%Wi+G&6<;GK-D>A@fhAMeW9|>cUqOxXI$2h4?kZ;iK
zw&f~r)(m!CAr$gAZKsJ%&JXA^^0rE<rw?X*9)~~9T6cic)v=ZbW^CduhRa3vN0BKb
z(<2KB357{mFG)cjyvMl5o8a#WyzO?ayxx?6o^1x`*)bkr{!91~KgiIn^@V{pvJ$r|
zte+44M~!eB0t4!mV9R+zwXo9pb@tf?WRuqf4(GstWr*@zo%afg;c#UI_hN%b{gyH~
zsyACSmx~yLFi_=>wbF}iSMo~Vh^aZ7!Hj1vWp3=&C?n&Iw@U8sEuOLJ-n#nDU+}iq
z=8d&^Ns3{nV%#g(&12TQW(yvs5}vOWpInKAK@X~#p{147p{yL~zi6@0=1JMiU5i!n
zLlzO=Ua^;mX_97^QTl#z{?YtQQld0z+PCHKA}hJ^?mK@j8Y(+gC2_5XNr&uw`5my!
zTnWKB4OZa9Ed1UAmlGF3b;ts)baeabn=fcqD8Oe$F>tug4V>SV^QjX1jSCfjJWg^G
zL`##<3#XDOn3kU0p2LM+eUz%i2pDnsj8K@R&1(`J5g%x3OTO^@@Z<DP!*Y0SXYG=^
z=%;H8#%B}rb^Q}(U&;6QAs3__M?Z9zUm^U^pkOOnjh<7E`1*{R8{`BESBI3xzI$dh
zG1#;^dpM|FdahrO3jj|*p#?geNKI>S3k=^W+6=L-fYd|k%14RdTZ4wGx*{{UG~3J{
zV5x__yZDC~zSe%1g}$$Jk*tnRvlYV|soJ^ec|K;ah1kgh9>%ecODtEa6t81ozoG?l
zk-d*Tz)T7Fe`r-<``Oqm^4w%>b<a=+d}S*N?m>-AjeoXt=Qhgy?L^AznO#G1EMMrq
zcbQ49Sv?pD(jqLPTirx+qa0o3at{13*XfU|nv)Ui<YsX_YHAtim>^It&_T<pu-}Pp
zo>MSR_g|0ytTW{Sc67{0Apv>zA}kykI*m6fnZBJqVV5e5AA;<vzE-UA;!{_-{S~RQ
znp9HcK&lpaHORt_9^6Qy$O%Ys(SUS$q&8aY@d1_SH=@qy_Ys(n2xODx102roNFuE8
z-=0HC)Lg{T{=!THxWZW#C}_xqu-eXXZD5RLULk=Ks3J}(s<=?=#^V}fA}x79sn$mD
zK6AHKtxH0GgSr!PeKUF4d&AobhhLb#zbEgJ0sKHT<%FeLIC#&#K7P|@VfubunT86f
zb&%*OQrzR5t`a}9!a!Kbt9$EY9F)-@>r=X{i$jmPTPvM8Xz}++SsTQ5p}Uq*l($u(
zOy*yk2t{b4QUEq^fg@^AB#<qUUb_wCm#26#y4BOt1*Q|Ku3bfzt4KH6+!3hkqe2!+
zLa?Q%!rZa`Ms~wMqxR;E!O^hF%Jm<kAT~M|Q*SLHgme!S*Vx*h<n;Hf4p0UY2Pl)V
zL=|Ex2~H(ODu?V@dZa+`us)Y<wdNi1$qtPnLKQh7m1m_X42`g2*a;5TVL{te?0P3P
z<@M{7=l$QhZ+<xYbqVF!RKr#1yHQ!exX=c|5`HgP2xJloLN~BIPfkFBf==0>8L7R?
z7P3r{R7HXFg%}~a%+CK2vIeM1Fyx}?b`7AkI=gG{vuBG-jt>kS;j+^X1D6Qs4I3tT
z^jKgjJ1r5hQ=1zwn;l`3NE9$HyeETD=db6i)N9r`SaMgnsufa&SMeFeP#BNK;6R13
zyVYTSRihv1V(kjx;Dtf=hO%!gMV(*DS+HU`v;ypPjnu!(aGCSc<5A~ZnbK3}TAFcb
zZWUrlbu$j>ATiO|qbt?=ZHHa2eHyxttWpR5>dNjK9c&6ABD+B*YV*|3uQ`NlQgAab
zWqe}vbRz8BJvN7VE@pzCtwFm3`hJ(Np#L-r`U)&uwD;95;68}4;I|74e>oEVfWpa1
zaEyyS!rWZo_Y)?C=)a=u<fS4nvaoM=Vq3yznl2o-IR4kHTkBwQrug<d;p=bWrJP?S
z1c+nyeFrTnkGwrq2=X|l38BPBtMk3in$jZAz{FB8909N;??mNur{qoi;qBl39p>|&
zI$9Vh`gZ8)_<qEEFaop~kn%>z)D&MxJumKb=d&OB@-63GzEyyg%?>>Q`^W7EK0(3h
zbgJ!oB7<{0)3#<4WZR)4@Uh3a;6RN$8(n-~=}*mOz}-u5GQLrFf{;{X{NjeJmf)b)
z#Kl;RM@Y6ULJK+s|L~Tmu`_gtJKm3#p{;bSRd4Y*oP}2t2IIY=qC*Z4;*?mI?Tg9}
zm1e5-`hZGDfxvF`ITU!SgZW+ld6M{_14VFYNF~-{^9CzDPJNRznyh?!XIVvmk`~##
z50wS13&XiA_d|FnV7$XmPAwR{g*<Z0o_l=>5(38d^!s6BhQ%@E3b3-pTFQ5}pQR+s
z4nGX%eeQ%KHpOl-TDZMSK<K6R4O>}qL_hpt;OLS<2nvc=gihY@etj)c=9KC2jf73V
zUg;9SB^bGZWEsn#Zmb+B<Ck31-q<NtPAESbxNcjwId7UoKWVRNRSW%A>Y7;a2>V=>
zYMZHayK0HD@r0j}+gdC)XWbREi6wD#cbeEsQewW$QKha|PFeU0xnAk%2!V(Y{ihKn
zJgw|_ygairCwkLGrUcW+1_mz@y+LlPCkW+RnMxxU@>*=Dkw>o=f+Qb%$OBuz91EW`
zlBJ9Ru^D#nET=$&%T2}I{?3F?j1B{+3;MxTre=N``#CSce+aC*^p;wUoaiOKllmk^
z#!EEgOn^9Il!+3XS_2;ZVFT1-C^|is4b;IY<eKLcBS>qM=@5?Zo9`$KRrT3bl=rIC
ztv+VtE>@-fkeC&>Sf??Z%e^)NrsCFOzsR!M(=zXWD{|sv--wonn5<jHZD@fPn)UG@
z5LOzJN1!OSAO?zJBhQs^l;u*Fui#7RltNaTtI!$AkJQ^nBKA#Db%8DWpe$gjg|3;t
zY4Ho3>xF86(PCsvJ^d^g-Q7@lPA%dq|F<f68OKQ0)57bNP|A)iQV*i5AR0+Tj7&3+
zfitb=!*RLiFVeZ4{T?k(-iITPt4xZ1D{nx->;JoD!}{2Cwd6EFm#}x^D|AK=p7F`o
z4ybDgBrTXnoM&EcYCt&fdwWM#M^!Q1`t>M7kE|dqUgBez_M5`pRQfG~!AB0|{)9?(
zeO}Lu*|Upqo?c1up)iMNVn9r68a*2cA6W(LEq*~^I9gzEO5-ne?EA}6x&jHd#rw0*
zidh(Vn0MU<A++We4v+0DyE`u_I5|nSJ-!9FJhJ-b=$?qVa8Co;h=Q!5r*yjds>#<A
z_)=RsHdWsWBEA@NmD%LH*B2C!A9g@>*-h)KgN6RBB6OjAKOB~hWjC(J$bQ<-J5vvM
zNTA|;Y;_KQ2gIe^b)SkK8<fm|dO+Tf{Rf4RLY`O(rva^iK~v=SH#F1}PVIZLut0cP
zgv2wDcT4a4&i@qb*#>c0b*QrPD-@+4aQmL2LDaYHUOF_qG5)k=_mT)Q-}9M+Q4<|P
zuy7OP4RYps7llF{)S|)I-5qR3x&dPfJ;zMs=SX#CEthF#=-l*cpz|niZf#gVpCp>A
z7>IPlFVd@5O+wI=#Mpkeyt@|gNVOkw!Fr+ch0%g_fhkne+bO~il<XRq;Eo+6+DrNu
z8zMG>>xlq4Y%ohfKoCnb1U7_#a2<!j1wTpDabNp@Tvo#?!hotps@MF^1k(AKN2B^Q
zo0nOEQW|ZYc!Mn9z7BVbB$GBFG7q@4_spOEW##qPcR!pQ1X=2tjSIM-o$sow?^O&+
zIz+=V@^JVy4}F|DOl2E0e$7x9+!<W(&qwhG)COzUU^H8<2o-u!*7<Yl7FTzefZp;g
z@iHOO2#_ZauEd!d=INbM1JzOY!+)!zNe~Z8gF1{-fOtyAZg*V-J{K_>xh#@=D?CiP
zQM@1N^DYYLvfs`tBJ{aTp^Rc>>V@wA`QTb3ZB9~Za<t(>smX>4`7^QAi9AS;9h<Nc
z`>g>3i_$I%I1$Hu41P<Frx?f(%o3l(##iRbIsO0!7A4D2eCvE6QWikfZ+YpLpvPww
zbyFtxt1QxOBdmqnS~a!ZreU>7`mpb%QX+X!*+{Y@98S3V8I!<Z5=w;{uljIZPez1-
z7GoZd(plJ?y0I$UYjCH-)h=0%>OkI~S9y8$%HWkmBf%w=J_M32wOnRc2?|dfvn?up
zDal6F=|pJMo>l^kG6f4c^d9aj-^NuZcG^vJAr@TCX*x&NC1wB<8@|)XO=9AhvYENd
zpL}UNFGQ!07XQ`}FMnyKDSXpZUtr7EmR2_W0WQrKA?LvfbQfAcci|Fm+4*=;tJ{->
zx7pu$%ej0&EX1RrVw+C92llwuXO6GCv73Oy+pt442^E~p5_^Kt22?G5w!%~_f{g+#
zL4ni!YnmNvW`K`l_XVmz6+*DMFWXggN~<vuHBq%GYuQIUH;GOVE(GL>@v6Vio}!pi
z+JB5AB#;!61-ZHmp;}mQB#1x3pC2XvN_l!anS`VIY3VsOM#ETe2dsK8F%lG1W|=as
zB%kN&Xd*YgpLZv?_L8ALnzLORnd(6!V`bH-(Bm9@H@*X|2E{aBVIWnrY`J^&cxtj@
z(-GWkqRz825`br6F_32XLpb4No0HuiC?OiE{d>UrxqG2AQFvRSX|oWpS};x6#l7@{
z?L|4xZ1%I?*)y97;29G`m__qD+Gj)x?T>#;cGX1U+9WBd)yV`J^Pz<NNm-ZL8!KYz
znbw28-^W+Rk4|dVy(6-Y#PKXa{&1TCOhhoEJn92xd)L^__!0fjEK)V!xYs=QP}=1u
z5tKC3?6n!MP~I9z@DcF#VZD{0I~$#=9+6M89%WHzjNB!DG!x~Mw?EohhRl0~E_DB(
zBXQ~pqmV31sQlWsHnSrQ>$2<CH{qcH0s$mGAKX>(L8@haBNw4GbS{)e_FZ)hi94`Y
z=sO3f5LEe<3Zr76vskDob2_g%p`nWxpT5H}ppq9Tg~abqX8_0SHn9iG+iUD7vRNsK
zAjPB5N)KNh<hroG3H;_Gh_CTgsLT%D%@OEN<v`A1EFPl=ns2{XEPqySczz@8F!!CW
zeN#`+Tm>T^lHA0R=<s%5Z2=~q_cww$++Acka@KOANvy)a^2Z=Dgu{1roixhU8s=h)
zWAmJL#m6;usQWUa=Q#H-#zndZAXg}iW-hQ%_Mt9eA+RPc9g!!dUfX=6XZM>KImLVC
zFeATfeT@z4R;uV{z><GW*Wd_)QP%mon!Vz1nG7+12r2;Shl1-FbO+DR<J_<@R~ZsX
z$c#T;9Z|w<Ue3JvTh?mc8stf`v*TZOzrgx<8S$AC2q{06#9{G)F^Gq47moAgmEkEQ
zy<7LDe#KVgM8+PG&|V8uCzaO2ielBBR)V!(*#W;o|K6v#K3~doZe2Wjd9j6aWsw^N
zU*6i#eXV=T-!BsXGwp=qR-t})ws{wy6$W~E<bFeypY3v(xk7=Jxgsn%9SC1u2u=I=
zmC{0ULH{1tFH7@VsE;UJsZ)^U^S0%ii%lwv%G^4Om{!pi*KKgd?8)c$uND1>mdH>m
zE%mb+hJ$aYTlq{GGKtyoS!P(BdJqJWWaLujIUs^S85Le)Q^<@WA%Mt+?R)<*ksaZZ
z0ndlk0oTD$^2?ei90Mi&4~J%lV7-(q;v7jLId#xO#@Lr|x7-MyN`d~&J(ejsgV^w8
ztuM<Uf0X$_sFxW9EWqNSy4NxyGzRB&0b^&ByZjVk)R*pg+VEtR!AQ-Yh|8?|W|Amf
zC`W}ESVJBX&KFxJc#2}-4(ZK2xjyCn994EC3=d%$x|NoIQKfU2WbYI(s(kuqR3XDa
z<<eNXpVK*sUI1^yDUoJ>(iAE&MT2b9^pXIZ4(cMKN>f{Ek*jb}_A^#YPjwH-!>q~S
z3j@33y}dETPUI5xuZCcdg;bvN2g4vU4=Ug?1;Rd$&Y81Ty_HVJ1G#Y-Y~*LW=JBl+
zCffAsm>aP>i#zBk(<1O@Ve`CD7xGQ7neJKsI0m{g%Hw>88SEwskKD08umzO#g$&4Z
zwPXF8cC^bckqa$h3M2*=rU8Eo<2rk-j>SoG^StY=EqeYmt{o7WXlFqj#>9Ck*?cP-
zmmXGSU#H|(1XX^e<DXFe1ewX>=P|ya?QZ^pw*;|GBw*5bg^_BV=RpWHESN&Sdo^EZ
zMoO{Q+Tk^qp5bpm-a$q;V_BAq8@h4Sktq@1BIf&Tws94`kh!R0z0oMKJBS{uc7H8{
zYmUZ@UejjhIm_g2xQt2hxJ`et**OR@^=3-BeW<b_gJsd$ulbsl(lsR1ddD7{mHh`2
zBI6L|ikn@kKKB>xB-1eNheTlJ6=QDY0YD|-@F&VeG&4xGqEF)*5D)qAZpEkdAT;@2
z=#)b^5e{i7v$ZsBLHrpGNT}{PyT1oLqo=>r)u444)Emwjh#|yH8FZ#82xHY!5l@fd
zP?TmgFBA{M-SO|%#M#R*E2k@qVoDX-uHQwwF{2e?mAwnTqkhJ!zJ^>lix53$2I-2{
zMNx+8`M_pB;U)VI4v1po^D19{zY#9gEFP>or?86*x}cE^3>1J>A#<}>0Aun4vE5TX
zb0~i7{DP6fRO4;+K-8Ti`8%WbrXMevS)XN3Kwxid*&OKUv*nf-{z{N&eAZPG-}Ai3
z5X%o0Ze28M6nj{xkLwUa*EzzehM|(!`{m^DH|mSUnwME<Hz6*K6rEuo?_ghA;S>b(
z{fd(9mpV97BqY)<I1m2L(|4^$--5z)iD;+&i2xoiSlLB^8B}`!U6^(D-jixpl0+Pv
zUm`0``p*tCbJ)MXz%?fcS#!#v>lLWD>A%eC^HV=7U24sk|2;1{Q*DdXeyP5Uc-@^o
zGITbRmrFcjLizAXuid3EZlP<07&}TyDKF|^_l7O!9fP+>SbdB@P1w#vo7zEJAItFh
z;_-6OPVn{jzC(iiKj8P$eMPmKmiNx?_aVzz@<iRNfjoH>S|?<ivP`%Zl(EY#{HuaX
z2E9$c3~r|PZkk{C@4mh27P(VwKUqtnKVsQ1@WYfoT_-2)KRHqL8$7BxU{ArfGZ&1@
ze10FHjkwVtQ}neAMcyHS-+vlYxJ~L;b$z6D((P9ovmLHR{mgJwRs_USLaa=hX?CMk
z+Jh4u7r$4%?Lom(mGJ5<1x_Lx`6A`0$uhxD(cA6;#xgE|u-A{x^D>?~9i7W_2hOhw
z3`e5EGerpAJ^Z?~_$f0?!=%0ozb>H@am8>fR{J`t*N=M76SIac;_zg)*AGaKqn>vD
zV8ZUWjD!PiEg*<8h7CV&I3Kok`z@67>q7cQ1gLsmJ$ZyB1&>0Wkv!*?Z~QalSf3|w
z1SshO8NDEUaTW|hv$HPIxRyNbxc5+;c!vWlV`S9G5_E$2Xw$DyoOLj4Z6(KVjja&@
z;RK`-Dv!tDtw%z3zf<{*H>dB@)Cj~15BxB8H_BY~ou|yhJ$#OqjVv$UMf<@aIW@Em
zI|-dHyhA8lX$;y4+L#<jN~{|p7#rO@)l<*QnLReQ7|q;M1OlUR8UlkzIvtF1K_a-g
z5=}p6t<3RULmfXGXD2r3EaeY#%lfukra9UWo;Uvjsp7{)QsT9qd+tk50dBN+Aj8yE
za`4&F)Ix(-vAZJ#H-a4$X%yG@oqiyJ6pn@CD>WM;3a$<QQ3!mBRJNT46b@pguWL5i
zSiaSXBzqo<-1TfR#T%AIcTEw^OzgH88gcMD<PW4{KpZSfN2<Purj+u_+f@pfQ3kax
z$c)24e$Q1DkAO29(J#i28Lp!4($xkV4cYX&A-;WyDi|R8UfQblM$yg*xTo#t^HKCU
z;}Uf;`hh8v3BtPiplcO}`1Ca}G0xepA_FRWouJpke8V5-e_Q0na-)?Ga`l=AJjc#Z
zEM3mEJ^hjGdi?u{05^)8fPxTq|MT;R`0PEg?pVbYxNYw3);p^fRGr-SIpG_my<gVl
z;=Wel6QWU_&?>0=_ue5C4#@vev(BNi$5x7=w(M>|EgZ1L+qB0KV<K&Dcj0M|=|}xs
zmgOam&(1Z|1!c*~F3EFD?EmwG2?0`{@&$OJcCr(5Ixw0{8?S$;DJq>OECvbdUAme*
zRi2<NfaChyJ~tRc9k0B15>uC~_79O}5&j?Yh~-TE@RK=kQsN)K<O?wJm1mqNP=IxP
z%|(8qY*67pfl$J_;4=|{1i<X`-xL%qN0l+X-f7tR4^Q=ff_}W?QON{gBR6)HPb3o<
zH9!xsW>S9t0aXvk8~&s2Nw|O$?L;YB2wGrIJe~g&SOs{nx%0!!!8alx4cZ7&Jb+pv
zO@;f9XM<X*Z~gbF==8dc<@clrvNDK!8Hc4r_k`k^1*j@dWRqL}HGSYAA|$w;pd=_I
zHl?G$pBuq2Q4mSh2E@lTW|=Ol{58^_C}jZ5<_VXAhe&}J<N}~uP{VO5@Pc?z=_b7I
zl*_z;<{5yS{Ffx7wtw3(4uFB=#r^|3p#r|>{|h_$zigcUf3tBK5P}ew_qC_`y__fn
zUxmBNYd&$6l*bP(>hPY>j9`ROizFz(ph&9zEuyj0{&=P3d%K~U@VH5{mVNs3Tsmtu
zVp8%yWTK__m$76*OtF-r3;<waj1<#?H6k$jKbWFdPf_E4n4$%ZpOk&E(J4t=kpNsF
zD#~d$YTwJP>5~OIvMRLr;RU^-h;Mzv;dfHn@`>eu2YPFF3J!o+00X!P8dgIeX}4>h
zSQhEbM>5|;zqB&xUq9qs{srNphCbnw^0`*|Y0RGdQv|sFjG#_{D<vW%&;k_)tL6&R
zdoSq9k=>tE23D(HzSdgPC4S2Qas{@;ClHqFwpDn|!f6i{_u5yFwdIN89>h@{qLld=
z&j#fD=-*X`yeIduiwJ_RN8D4GPR5J@Dnc-71NlGL(5lS;f(@}(W(TM}vhpW-fK4j1
z8;`4%y0x$08vZOg+8Y(Wqw^m8fT9_$D;s^+gI>(}nOJ=4GQp-tt@$!iE^`K-z+gI7
z=n}8}o<n%)0p8T!Gl}r2|6@I}A7Me?w~UICz1I%f_*2b-=-!;1H~{+V=>R~(Goc0P
z05Ll~+*MB+95P^~LM-u)SaAje>aSU~hf~~I{Su5k4e_Q&+sTD#el~A|684&Wr-u}7
z@3+;&qYp19p-;TKet^+oF)LRQb;{Ys#c^=Gb(CbW{QXVW=7&bwT%`*}bJihJXzYJZ
z6TAmSs|Jw6COgp+$lw~E!!O$QdMyuQi)08xU2yVOCR!RZwSg+Bt?0qv5#!^Xq3`(#
z(b{nv;#$ReK*Uh_FoPSxAZK}-SbKIf&-D}up)XZ<yc>RltdYx3kN-@*%k6qOA186;
z*#vMU7*K0Gu>4)pW1@N!f6{*UzW?pp=89hmZvPNkvrlK^KZF)i>gi0RxO6i#1(8$l
zCy+0W-xdmp*>JpRE8?L(naDZvi;jLSniWlkwDX<W?+tFQ;*>&?s2cba5SW(cvprdQ
zB^S7N(%p*CPI;~Uj>T9?du7g5w0q<-4f18Ut5fS=e)B0sd}i>1eZ`w6SMPrSD?a&#
z@X_bKuSMZd_fGa<_*5FMXwe-@9JOt}l&tw5$d$;oY5RIR1>W1H4~fCMKRyk<#q4pf
zU;gt=_Ta&>WcL2r!}Oy7;8GGg9`U3hL5-X!R*fb|+Kig2Ayi7!Yrme)!*(;t(tUL>
zu7{uSCrXvP;O`ElCWK%oNS0q3(4o%g&l}(VVBr4lY~wEC--ou$ACG^}!uQwj4Np~l
zLzR)jvL{(-x{xj#Ij>YB+dkKu<Yj(;SFC@pZ(a8PBGVX}y<nhDIM^-emPs>x{ys@0
zWS~d*cz%hvbKn&N9Zg_(Jx)->6V;2C#^pU8?$r~Z*Owqd;t#+~qj;fH)y6voiPT#s
zVG{>zUR8L*(DfGdQ6eX6O!QRDQ9J|yFZYD9JDd@{TYCnoq4%oxUsasJgI4OL&lk1N
zR^M1svQ!EUyZebDkPxtKn<<3r$%^VK6X?G+`FhCvAJg7VMGiYpZqi(uTmfAAzjgn0
z&JP3#L|h1Z(es37lK>bc|G~4Nv-Q!@J;nS1D#>+6mW6K^Tsj6p#m2*h|9oIvdku)<
zch@~ei&)DFN3UCQ%L2u~2=R~peGz~w40%Ev5@`V(w>ai(jn|vjFwmhfwPVol$iL^G
zYfJ)<ZGhBU&s+v?39*w#5Hvgsh*5^VW3{mF=*<_h&Y=14y33e|5b^Wsqe7hQP`3}i
z8Y_rQ{!wsM_|7Iqhr9mnB!Z|1n3Lnyi@awb3{#L_*#z9P5r&k^dp;bRu(eX<4C!T0
z6R7^%LR|8GU{}NLg;V?kwqg8lU|Y~eL3sQVd1!F-0BOA4fn&$=&1jL++7qRr1f2*c
zRB?$7P(flX(mb6OSVAT?2mnOHzHVhoe81M2LCP`!8h~_v>o?pNPC3%g`1#l8=M9qk
z@=a&)+VRr88%?7TK8G<3DfRo3%{|t%&(JL;dV^Xp^92CKV7<RK=A?<F4#HdqtQFq^
z7xW8dAV&Je+2_^q+;sk;S2aSlmw|Q9*T(ntyHl>qv4kqr0=Z@%r|Q>&l{gX$B>guG
zFFR(IBaO$Uq5fDEEb)f;t+xUjZ*;{i@4FfPc9b7=*PHGqA1oH<+yWd4i*~-R(k93H
zF&$c%vYs_!EI6e;k3U5R@BYq&$l-rKcF1m4M{xaly@k9miuaS|31L^mR8wp9G*z#@
zq}n^818{mTBfkgelhD(xEI;2f?DqMQIjiiw7<+=#R#w(lb>7<N{~)uUuqR_1kt5`-
zeCKrs|Lf4|xhFWNjkNX>!$fd&`ok~t#-EF)Jsx$|ck8o|^Zm{@0XX9QdBseWx1r8T
z47;Bu8IAn2Q}<r+=Eq(~5ov<Myp}{{?|p7tw*D6j;6eDLxZk)SPsp@)BG3Wv&v}SZ
zO&7}5`FNee_k|b3a=v}Q6Jq&ahRil^I0U($0FbJ$a7De(pE(j3z$N$QTh6Zg#~SA%
z3-z1K-XBrxk0i63jom8Fey`HHRlIA}WLWbS(VSfO$^FM2T>rhC@F(_!`1XARcX0eI
zfH>*Vc3Z~w8t@`AG0O^Fx;5m4Z3b`cLpNh7H)i8yHUsP~N)tsMjP}!fd>h`wvVQ`!
zkI#SjmftM1J2&j<&}VU<zPLF3#g`FrmY8rlFXHBSFBMz-d-8ph@KzExcJ<bPotTAB
zHZBZgDYzN8-hNkPaf#2S@oFH+5q+>jVOTEYTdjH(m7=bX{N(-&!`cF1o2q)aJ!r;n
z141L`qLe^<jtCkv6F~&n+8JefzPtO4CdMC2CX68h`kiW!<o`Hs&v2mP0gs=z_1ntv
zBTPRNj^(oXji$L~>|#d)KoAxkRsSAOtgPINw{mFg#5|BlZ(8AK!JOPO2G}y<-<6_U
z#mF1lCNP2T8}D1G4P9Kk`V5IczK@fP+p>Q;1buq0Y?>oYvB-ERwKmK{T=tJSB|{7X
z+T1<>ZqkCxFXumOCqD91oFKVZquaOI_c~Z~<_+B7E;9yixF<`thS1|suIC<<3%CNH
zCtS|j-6QnWg4|BzqPlF1D(V^r<wLxzGYs33Hf-x%qQOAr>o<*(79TfH)lF~zwqjbC
zR8+0$l8F4uto+s5ZqDy|IT6SFR(8+b-xK+XIP+05t?)e(g6I*I9?Mgcl>b!h@QIzY
zWNN@Z?HQ|;@EfM6et3fLTxk=)&cv_<)VGA$$%>~BH!*RtM|OE&zD>S$Uh|&5he%t|
zZ#jk3tkPf)j6U_RqC{?JlASB37wVnTfP8!8`HmNn@t26iqNLlXx1-{to~_zdaw(O=
zo@?zPKpG^pc7a^I>MQ=)0dD`Bv{w<{KD#A~c5+*Yx}<d|b(a$+Q!~IArDNPuqL5*2
z%=6Y=$3Ita(NJ?T@pV|#i=gUazM~RU%J%RLA{+s2VNnN%Duj(s`o?k*aYXHVGpbC1
zoz?eW#manr*c{sK4l(`<_AFWsNdc{%0qJQZD~4&fS^wo!8O>pt;fuiZF`vgkn*z7B
zV>LnpwUVRwA1#xvUm7-oUiMIvWRrBcedH4GrGCu-B=0ASts1qZr+(U4Jojz+Bg(zr
zn6KCodw)MUjfg+D>mxz&L}aR^dp{^b;F1kshX96F1V9;y;9NF7N~7E)?I&UJblmn7
z`)o>xnL)Gku}yx&#-$P8zU6yrAtX}qV2>3Y&FJWQ^A&&nst_4<kk<|n0FB<zJVYv-
zd?>NGj_`y+CAdxo{!vB$$Fapo3kSP}RHAI!XN<m(b9{gAWQtIk+)v9+zLfD@f<0$7
z{JiP+_yQ;c4yFh!A~Zsll(N0t`t6Hn$2H<-EBoPnQ>zd&|5ZhNj9-2tK<m+*piCkG
z;RPjcQz6~!P>TxSNtM(k8XzuzgFGvG&y^P9-j}0d6U-wz-cs~stonNeyO0oF85gpy
zGY5a$gSxB0u_J?>TlfIVb2DMU@CLxB=%zWnm6kv8VO8cw0^>_NN&y!Dy`yCDK8XpC
zRJFG%#2+*_^z&udPa^Jk@%mG?NZ%1uh6T#@7^~^1zYppUnno-Y2BLi&uAJ6lMX{3e
zX90Xp`EJ0NN2b#-8ck0&@RwZM*jSgvx3p|{nZbL6j1%dFf2D_WAdjUQAOdtfA&Yw;
zPkcPEBY2J>Dn9UHf7aW$TQJu#6_$D0)9TK`>ZIq}mx;S4kE#J&Hb#`!iCwmw!2g@I
zhv;gKLwC#dtNgp1vyuCRT81Siek0y!1H}0`C}N<M!SA3!O`^WaPqQ}<j6(Q;>!OnC
zVSU#1b*JH7+qgaF@lLh&E?M&Dn~F^GyxC+E80frf$?_<*0(|$ul(wun2=fAA`(=UI
zEuoQ=sGP`@s-2RhN!Pqc{oPCIvX>stS1{(6NWY0%1RTH3t5N3y4sLD2@!2%nmP(Wk
zlR#$I{}vCfN1){Ere!UqSf}DCf709SY@}1gp(4|6Pq(|kR5AF(Vwwu!T*XPf$SmUk
zoB+@CLi^#kjWWduY-tRjHn2PYezMy5>y|yo+IF*zo`U@cwO-UaF1ZhN=i9X?JtE1&
z4vg{zkQLYKpbpw*V^}Bam<JtR#6FzW2xBMKTW#D<RV}syJ8VsJ>?<B<Oe8?wkYY(T
zDHIVhGm`nX^f`mcu;!Do>y9vhhP6qqLmLbRIq(5i@G=_=4XX)rg@^P%s=#x+KKJrz
z+CEiBJa^_Jn;6-GMgcXqC~mvY;{TgAoPC!HY+irgVy1}e+v?V}?Qm<K|E6fxog|sp
zNFcy*wAAEV*Gh+j1;p8lG2>wupLX^C@aDT>SG~Pwka>0HC3iD>%bAYj{A`u$_{3UK
zD0ZNrbGy6^LXZ`s!T<70WYMcBlnsV2WrJFeS<3;z?*lHNDQ_ke{0isR9?$4crqNIR
z9P3l!1Fgpar}7j7xM&3^UI|d);qVU&XBSf7QpR||gXi9PRd2RJS7pDshl>Y&kB{L$
zv=icMV8>K%8Y-)(RpPQ^C_O}kLPC#PxD$wk%tnP=_HobVLP*y@tkiD<Q)U)p?)aOj
z1cMbQnxOI$b#l^SBidgKEB|#2=I)%^FAytXQTIO$Z4Jj}VBiVHZ?(@q{Xzq%vU8aJ
zghK5-pBj|BC--x&n<w~{^}sgCT1Z;jH-6iSl!?1?fT^?Qpdb=czNoRG?fIUOxq)SC
zh^Ra=OyrlQpWH|1ig&q3&#`orGg(syh0U`MZVxfq&p+9bV+71~8Zqr)nA`mV1h`!X
zz**<a0fam>%6xcpLjjsWYSi_8Z;7vPyMT$2rE6g_aKiiG=n>IUgsJKgsW~oaHN~sY
zYK8f=NR+jnyZf%?xSltEX~l$xj1-^Y$sD#!d7`6!xU^=20*P%V2?<C^%FY2wiO(LE
zkE&T*Azt!Fsxia|CmU>CH0hh?;vfGv@wi<-lOp7}dH?SXc*A3^CAJDgK`~r4-QrA9
zkx_3nyeF#K_y$W(K2YBF0T$p2_%sPfxQkR7Q?vog)07ZIv-XFR_SB1EN`q7&v0>EF
z`2B4dE2h6M$_@qzs)&4tihJL+rOUj4jFNn5PODzae84BVCf>**yjxVT&ol(BjLJeX
z@tv^<b;n3@8)v4PmJSjt3eLeVAQSq`QZJv4;)U^m4YDtn9IkQ#4wP2RxS&*DpkhAH
zD6>8@N8o!>d_)oBnrFrD-jl1kNpTyXbjMcBeBcaCT2_YrLur=kqTCp<(;zJsA-8$V
z!M!cade+Dwa4@pZ#B+=H9|ROj+6?N8SX^*`E(qcvL1h%))AhxG+=ZzWFU%xeswVH!
zZ~|%9vB(`=M{(LadH%`_(Tc(Gta6(ZK(g4L`Kp~J3LoCONNorYD#}Zsf4xLJpAzLF
zXIt(N&!hC65%ZMecNaB*gW1GZP{x(L*TUQT4u^`aILy~48i$j<m3chidY0ZAImH}5
zV}Lot#5Ic2n{%5&39C&}%DnXLhOc?Hf*k(Qp83f;qUeyGF!G<W^c2znQ1kmW;jY?v
zR9#@@Eo9^+zWkj|)}1tX)9-z4*YFA^>b;~ZSN8i&nTuMWkD8twl?fSN;O`NVRb8hB
z*#i@fk$FAz?mOp7Uj=Mx1cwcqzKi1_$jNGDj$E)(vBm2*m(9*lD5_qjN^eL_(OXUt
zah7OA)HDYEgR*J_td1Ybm29%v!gP9Ombtig0RP1OWpd#2xFdpN;!+MX;|D#59nHcW
zErc1bE|UNDrpoQVQ$9r+piWf9h3N*L71^Rf-bVyZAwgRdFjA|a@x9LaMe?uuQ4H|C
z^giy@N;DlN<*>yA1?;(7`=vE<_qwm&72gIS_T_~k|Ad+b{?@y3w@82?CI5ON=;qzn
zsANhOYo}PmZ07yF<qHXt+T2(6J+lzOpyRXnUtl*RLMUKD{*G7pO$0|fH8b8;B)>j+
z03W(IIIR=))nG>e3bW|Hx#lU~^4Qs|OruHkDNp|<PX&c)xXUa!djJOh`Y;0?25M@A
z)dD*&R<b)Fiq{#16oOsG{1GtjIysjY{3{xUvv6%Wh^fsPZTsMJCl`##Uc76(9gq2{
z(P1sX-JR^Ui+3v&Z>xhC5R}Oha4I1%Z@TWXtK0=WeJvo7X4+Dts%-a5cQH2dxpU4i
zm>p%3h}i_*?ef(XNl?LhT#qVMZdB0Z*f{uH`qr?7>D06Gk;;A8%D#o4?6>iCHeW@+
z$#=&$L$w#M&j&%WQ!=iNxJ`71gr@=&jKoi+Egr_pXf>WEc$9v4RTD(!Nt)?=12k<n
z{h6u?0l$k2qsR52Py+}OhY(6x2I)#FL}$Q-8&qOp8<weHDES*Q^ce53zNM(GTh2^#
zA`URy);fFnlUbQ-R5!ib2fCNW#|sL)^wZ(+Wu$fV1yM;Cd=~SHzqbjW2UGT3NuK$p
zPtwTRQzN;`6iED3j=1OgJi+DuE$&rO6rDt=KLwhax-S}ogdv~-y|)DHg{gg+In3Pu
zJ+!%x5ES_dSXcECZnv|B(ljK?K9^~GLm;-tQY&!<5C~aFe8=o-AP5+G(NOkgMW7vA
zL14L8*ppdHYji#WWTMo~oOC}`L$`l_nCL%EX<JW`$<rn*7>d5YtKvripf?l_TuoVS
zRYWQjnek)Qc-`}X;V~c{LPm?iE}7KtiX`opp`3e^o(I|{>@FY^R*s);88JNua*e`l
zrjHPWH?*1Eu!SjruW_BT5%k1hC-eMv{=el}&=5E9s1Jrjzuto$FjXkSPZ;Rbi@xi0
z$b#@vR6pl>sZx1>1XN{?ge~WbD{{sB6uIxOuufl-(^4l6@DWp=Y{Jue0MjHv(gy|O
zTfNEGKwUPLg~JNuT<o&*u950>4Vc0mlw@1!LSjd?w*DE0-y*_$OAr|%0W!j>M~a!M
zzItzY!}`gs48X|6iWtp5S}%V4j{;BD#WsMBx?P1$%Jp<-1MG>*Z0oz`H3g_V<TFwW
zz5yA5YyXYjXedJ<<TwPOo%f6jfxBSJxhD%vn~uG%8rrn|w$d(Mxil45NC{3UV;vNS
zBNKL|4rBXhUWyv|q;c1_Zn&nDeasu8qZ@n`pl6hEWN(bLy!B%1W3f^8mFk_R%l?+B
zmCthgZH#A60wvZw@}KXtbStUgLj{~K?tgk`5alTyUfeV{E_eAe_#9^6Bp#jiE&BA-
zn;zop{u!b7RuU{x2B2vFq~}O4Y;~5}+q4*eg%<Wj$-O$p=}%s7T1*Z;KQgYr=qPlf
z8^DBj2fb-as-h<?*$spKk&}}51>{|V8N7@os>K;<lTiMH-&INptH(G9NYgP4Z)3fh
z-Z%uEh-U&|L66c~W$*0mZ`Ir1tr9g0LZ_%SxnLG~wds<+ss|LLe=Y+rx0rO7Dy6BZ
z^H-E+#h+Sig#XcElO@HEG1vroUgvTX^%o5l>FLYo)lzF4MztEY>jYJryXGeekk2ej
z`3WS68ALCAlr?aCf8fMjqLlf<tG+Plm4xtK4G_Lc^NGc!{28DHS(S729l1&Wi<p6m
z9;proa7asOVrbz^BYf#EmJBPRjHo-=PK^oV5b2_f_aQZ|T(~ICS<8kMd>v=}wmy7G
z*2j(P{$A(q{=y+olK;=VA<dRxK(DJfqx-!Iev){Cs{KeJY3TePHw3v`WF1^>j}SJB
zn+-0D+&*OMIzAr1Z8R||g)ib1h+pmsd5OlEZe03SNyQ`~R8^$~f^gZWIU3GrPOhW)
zEzXB~2A?CKsfaH7c5)sDn=(B5afiwz^u|}##7Bh1SJGnd?}oiSZpZpyxgQjtN8qFV
zumD5~I8z+{;IRmh{`|f1B3}cWz>-F;Q{+?MUtSI!*MKip_6uQ6*fSg>IWl(3Va;(f
z>mmOU*4HOSfBiwj#@!=qqi>6^BC0x?hPYC%Ui9tGtC$LkFG|R#mw(62#d8%t(By-6
z+qlK1Pg@5X3m|L(lu|VbkECDVy)K;|3T--EqXgaffM0Y#n!g&t(JHyHe1B${=*V(Z
zlZ-AW|1<;Hc3tLGR1KMxNPi^4$OIyfnOKoOS^>)Q`k3@@HXzZUCuBF@MjD&_j+Oe?
z=LKGSTFVbXc|_-4Q6`Mv-YK5;J;(U65Q=`aKt>&ipCiL#te7#1OiPyvAjwyLW7`dH
zzDAJDTePJPYRWWtbLCtFv^XiQeBRRdlpsR9O0H|GIhK=1jN}gx;=|8jrCCHCA$F+)
z>$6X)owcKem9xo{A?WD>sc3p&N&BE9+kX}Dse=qYIW|;ZosT61Dq6Op(Bq@79E{f|
zCKEHP_sr3wCK|K|soTNljRi*=KgOp|d*og1k6=$WHBOyLD!LTxy}9^EPF-c&kH6#W
zkwC*4=8SrPrnnV3x@<)?$KhGj6Dr{w(N*|UuOj_&Rj4MW%;p2XO>6k_`hUmh>oYUI
zHhpg<?RI+YYGxbm{1ntn)nT@;wHX#K^%l7ePdyN+&6~Wj7^Eay0rUPvz<n9<u^!%(
zQU6=28AvKHd<l-92@$pNq&Ou5$P+!H)y>NyHuQ8Zm?vpC;Sg6r?h9HE7EPdIx}-Y9
zS;NlfsMl>#8$RGqg@XeX%m6RRAwK^9*r{D<sSyqs^vLdL{WipY6;?3;CgzGYB{?NG
zvvH%k#-KUF0=q|Dx3!XVffJ<DAJ1dr^==1A?Gq*RMVGU`@qHS0Ta=d%<Rb<UNKwnK
zd$g~Sf&<qVrb5xaNGN-~VM*rTF`tT=!4HYMkh2JWv)288ULKy%dDSLI82b#T(O0;_
z*cB(uq74-TEHzs<!q=@#uTuKgvzZ+0A!X`U7bR~a+8^(<I-pblDPQKnFbWJF5Xj|^
zTT})Z58uyK;+Q2Z<_rjA&JRb1zp=uKwRZSCbv)qR#Y4K$PaT6z?GRQw2F!QIduAev
z$@|NI)*y(!yR0Qy;y+bfPDdKG^e^E+K4$=6q^8m0yMyD2z5&H9KUu-TuHrsO)REu|
zG3et714roLRLh5mXOr?36}%vVquXO97A=;gXzy#3l=hW>s1hz-bfGno1;&LSE8+aX
zDqxfERYR#qwksvjqot<%Uiu)9R4<!xqu=C+HidZ7zWLo%<bhq4H3$yu*iV<303(%1
z209!n?4ZBHwTDv#7OHW4ra|n5*4nG%Ho-rf60PGnYxj$t>u;L)Pb(=x{%jj<GQZZr
z+6yF{xElduA1qcZNO#iRCT#M{gr?T{>F>|g8e=>q2Cz=W(yudihCFAY2S#qcXsIIC
zVQo|oOj=i>M>tI{d&fv<rGF`_;&I9S3egR)B0{JR0cw<|Yk`Ps_3f*lY!v!FvFEB!
zTP7puyq1R-2&D~KT=j$nIP#W#<RozLj@2#(6hopsr*DY5x&YjrD#xgRncc-5llhBy
zA;=Y$p1AbK8p!K(*w-z{`lwLu0c-Rmhx<|3=jAb94zI68mfN9eCYbR9%#Eto=Y505
zIzX${4G7OT1C}j)=Kn3akI~KWQAR<UC|n(%LC!DOjMvY-sn|KxfcDL*G&8C<DJWdk
zK%UBk&I)<#LJ-f2EZ)5%6LBVl{$Ndb{QqI^EyJSxy0&o;1SJ$vq(f;?kd`iy?vhsN
z?ir~8L_oT`yFogJ5RvX~M!IL{0fu-lyzk%rJRkmF{-56W!~H>zISyv_wfEZRI@em~
z-uv2#kAIZ<>7yg-qZ~!KGGh2YOmPTCkVsrpS<=xEH{FD<+;d|3Q1anv*A;L_f#&b#
zRx@dtKgo~myrj}o`%YgBNlD1~#E!O?omSJ)`53N5-`x`wN%PM=LC-%ZWj_oGsW!}K
zz_2Z@x1Z<xd{upGju85jSJ)_$I=mG3^0W3{3+|cWK`jjnz$!R}GI1{-Jdk^$YRU2)
z2PE~b@&!n$)9}ZxH8GI~!supsPymcr^4PTNO<VLjLbbz(fLjn_8X1$Eqn(Ew{d_(@
zJE;*5&BpM@A}G7kZq3_!E^_g4;bZq}10S>dQSN8ufdd`t=81JVD6;!5;rS`U<mOb3
zaHRxoc5C+{mE>D^pqUK{vXL^}>e4ojPbl!V)YM#W16uUj4Wd_5H)r5Bb+HivgMg<Q
zI%%F5kXAOi8svTaazp>VJ$7o&#;bZGpFFWMLHF0>{B413tiIlM=fQW8g2uN`4YLYl
zCluqCts{g}IDE3&uroeAh~3R34PeWQ#X)CptC0-FM1P>6>~rzu)Dy>zSKIG*C*v8n
zz36eUPLZRe^H5p>l&Cg2g#|0wQ~mZB2`j%$!Xp&1OYuyxj#6ovBx`J*Ghjs);B*zA
zf7;##PM`9&B-c8%rxCGAu=&U9Sh^)BB6Xd|PX~2V>{VXOxLyihA?V2IG^86>R0i-m
zLrJ);o%$yG%n>u-?;x*%`V8;&WT%0jT%n)qrDD1=WDNomiOq$VpI=c-Tws#ZG{?;s
zm4b-}OAu7vY5WH@#`>?qw`%Q&y=Lz#O<w8a)PT;e-y@Xd+|A(CY?6?xOTBUTcNJTv
zo{tr1ES;T9-2Pzy@$THVH*iPaX4X9u=(K4ZRw|-7QwxBJ9LeRpjYv*;<YrMEGYuTc
z%N3V6{Qz`bA<%I%`LcwK0z>0Q(XYGlkXC&l-C~it1(H7~I8|FVeB${25}UqL{zLl?
zRm}`XvSG4tw}O!KcPLT|wQr+DG54BYn_d|~3Qim6V=7k_8JASI<Eu|^d&4006pG=X
z4BfTOG1C)lEcP!7Ud(}9t|DQI2uox2qdFG9ciauN73LIv-;OT-Ox+0J5vIRe^kh&B
z56t!v{}pXEf_K`W$;N0E=@3jVD5ez`AnFz~iu+!~PZw21R@TyV>@9G+_;?|Hi$314
z@1)h%rmM{5$WgFLGa(lJQ=>a;qUmm#MJvivoaNl9`Z+4;X`m~z9-M(AKxf^|a3B{p
zrL)+I_sU9}*UN(FSU#D4FJxL7cCBI=`0c3mvPa)pYO@`lCy!CAPi>i!j-`M<mu$jh
z$|=(`7^_uqvDQER=$omQ{tz`tuM%wQs<KmWsA7_;+E>$`)kH~{OrR)qEJQA2t+WH#
zHaa<)xfydFNRIk5fe=C!v#JMB)01-4_#3eYtHOQ_KC1U*FYMvFzPVz1<?BSrjVm7q
zO&~+ZBg~u1sVRuWOpG9owmU@*P0}3pxPlV_4@2OLN>Gtvtg-4HH%{{EPkUZYc`r69
zU<DDI3EP#!ITkE5E%I~S5LpKE4U10=^DjM*GG2?$Y0|gw$>_AsQVDAGWS>7+iYr#S
zvJ7KvyEzHz&xvbD&R{JY`O&%!?#4te-twJ{gqI5^Ju|VgpSIO)C@oMV`5@a~anS^d
zVKSW?{G()Yt{gyFhNd+px8!Q85C*gH5_P{esQzxC)9l$1m%5_Z^_8oF%nbH)s?33y
zG_b2~*V?PMEx9Q$R0uY&+2DP3nIc$sG*^=3i&QLkNP`G@p=`MFkpA$7^ZvB>%MS=1
zUDFZ`bsC3+pj({x|AKgXQ_~!|{6=_P*Qb3d^=L?zV+_wg1k5vIOmFLa!-JQHl3j1h
zK)lUs40dTt%W#=7<dq)UdkHm~+7a}OPG3DR++Q5_Y_vH-P8BR&n>w5!Zgk(yt>%wT
zHhCC+BxZY539oDsirE#})`a}1YRw6gtht~Pd>`TfBhr)R0L9iwcCa&O(%8b_ty(v%
zRI07fE_7V%Y0*v<xv1cZ%f4Mv7X^XHvxF0Qx1+*7nk*E)K&Gt~_*;3Hb>w%f`YBfu
zx_4P{wXwA?OoSwKuPm_!RU-7(oo>!g!Q-y;&x_*J<D6d!y%ZN7vm1|jEctL4;*tL6
z8&YamS^55Zc&nE@-xnzisxX0aXo#_cMq9Yq@YBc6d@+P=Fsg3o?r^feF9m*-M2>Ga
zBmVm`ESI6T4i`dmRIhi>$ej%j0v8Ij0`}+6WUA*YvCS7Q3)-E=bR-w-u)RO|@sOoV
zb$BFq+c)+1)Ennh685p;o==G6sFh^?euojL#cFw!MeGM4Y@3!LX*Q;0X=Q77wAkAm
z>UvCYJAOa9wMNk%DHtOz0l^`1@Q7ouPDTgc3c4Ngji6m@#IQHixzqLlJB}>srCYb_
z609?)clgRi4ob>g%0eHzw@>ulSA3r>Tl%}eGQ`W^R%-ZA_^&5)sO8suW~AXSqjQt_
ziiTOt_|vo5UBq9AJWM)XHmbbJzU@HH`8HRNa|o52*lN0kx#_%s2pCaXDMP|0DQq_<
z{F3Xz$4CXNhl{9?$08gEq2uC<G02Xg_dNd!qkZe<MyPIcuFE^bBQr#dUK3*Uw#NC7
zUjj^lc>z+7bN1NCtA9|mw#vw<p*t*RP+*EDP+1itR5X5DJp?l3VIk+Q-l3Zg*P=1!
zl#w}P_xna-vO>Zm<I-r>Fz>3eyfJw-g7cBN3UYad(qJ2GS0<U8+)s^c7l)Pw3VD|4
zSjP1U%1w0F<B{5@w;?NWx~N<9_FaCX=erfG4scG9#4z{LWV^K)q2&p2K4b3c&N5Ku
z)!mja7Q82$5<}a+3)hJ}6n9!_gi`TdrbICG`d<rD*n(9l^)0)7js_b~MVuG;b8-?)
zuBgIHSMdptUS*{{lvCw%N@x|}GL?Dj6|-R{;(1m!<(jeBlE1d9E0azEUeZx&BlNmW
zu;K4zJJNS6o7BJAx$!{wPwzd7%3?MMI%2CliQYI!FGyUB5PZ9u_gJ52aG>=_i<Y)l
zIfSh)an_=sY%ptaPUFMd?cvQ;bzD)NbSvq1+`Zq+tMt+rv%dB-bUqww$m>j<pK;_#
z_H?kO9RE7baSWGBi`P5wMb*2uadi?LwU@<xx~8mqV^9#TV9=lN12dYTTe6nQ?;HJK
zCh3l&yd6KMljqG%H`|9=5SvM`QDo8C5u*Oj>cs{-IhtPlm;LzfG2G%KECE>7rO|Wo
zB%1>3ss=m_7ObB&*&jI~G_@~k;#_{K8l=Kt#@K0=AmK|`8tJHjrRg_M*iIhE6dEFx
zk!wQ0G%e0WdQv1+N$`_K^_8%caEus`O%S@js27mYxw-Hb1(gVE*9+3-n)K02X`fZp
zUIc9H>9L1-2{La6Qo98>cQk45c7%ORh?j^y-0%<-eQvbBaUdIjP+s~nEW`1^U`#2p
zV(%pbGg~cd_RX!ni6%H~s8{7=lJPigR8o$MR5IJn*19SE*q=7ChM&Kx*}=;oMNnRO
zd9Bg_jp;fdG(uKh)$ezr8(7a6E?PH4&5z;x{N<$qZfy~AIs@`4Y(8TSZH6@|fVb`!
zAqXk%DjTqSpL(!90B?7x=u^wBTP2gH!ox{fffuh?Cd2FBgUr9{>&5iNhyesuX!w)n
zlPs4-fcpYd0mTERgNZeGhk`IvZ*9S&qO_{_2vtHnjiDsmRaRciNy|tmo2idI_zr{>
zC-({nT|xU~Eb(IZ2D+clFyAj%gDpmYagxHHVl`@JB?TV~a^>oio&64D>)%9pnXvs3
zBw8!q`r2Kg+{rB82AQZ_4AmO6Yq@3Hh?PebWNh$Z%6%0Z+gR&4ivCjimHXu)&68Xe
z9AB|}59xqxLE_s#*@CJPKf!k_EFt%w+f!bX-YgB;(W26}D9FLiJVm%tLWXv4y*<AJ
z%NcL{Zl=!O_514%!k;ux;`#E3_|P|2I~ToYCm^1IH)cyY!--+w*{M_;W8t2yKQUTs
zZD;4zW^xr#L&OdDB0wZuKhvgI=F$hlom$6`mg8ehQx<ydn#G=u{evSl%TLo#bd7B)
zB7~T7QUB_;nr9!*I0GJ`|8k!}NW`6WnZEu*|K#hjxW4kb8EHpBOnmaJJchHcY8N;+
z%&kMeW3{U{<5NE?(VvNx6YO^#c@=k)L;2~!$H)gANosO_Xvc?RTitUt5`9ef6|@HK
zU+l+2gNd}t%^~$18?8yVrYvuF`=<8a52vs@RuW!a+f#5)34uBkkm~5-yJce|ud2t6
z*luRDr_zcSgqdv{!Cd2?_8NIDwtppxz-ey9Grv8cvsDUz#hw*d@>8*$M)l**g;t9P
zcQ(Hl_&+vpBsfP^wAuXJWoBzwX17GI&CS#Urzj4xNmE)RQ<?dxAAkK28zI&BC|PP&
zN26&7=zk&C$us?LcZtoXg7vIEoE_%XuM2PT7)hCusai8;SI;H6u&f|!O@f2MPs{m4
z-`$J8^eC-s7cc{kEiU9~9+P=}M_HeL6XPJ1nM<Y6F$`5!U7VQ3?u-}LMgJj7z@&Xo
zHfA;J68#^($Vqjg%|!FC^PdNcCbyMFefGyXUFfTC5bx*@)8?qH2ax-DhjvF7iTOrm
z!i!qPZ{-CLX}(+*6Y#{zE0aElpf0q@OWkzp^*;BS#)oUhF8J?Uih83uUTJ$v5hup_
z^jD5V6=D9{T)u%UDo-J|r+mZ;O*h41c$|U+F2S;NzPY`JDv@twK8)cdXHoI>`J!RU
z`HKN&$s6;op~VB7=F?ksIZJ|%B8FtwGOWOD@c`}dGD^n>?t251jTxNr7dYAf103K`
zeV)Xf+o^4f&OHf0!wPzn%ACjj;0-l<7$ld7z(?%Ezd}XFnP{Fk+PzL?8cQdkXR3IF
z@!Wo9h23FJz(dqtsFeGe6`fG;KafQ8dq*-9t40RQOu~E7F`NftG_QPf2YaK|KcO<l
zfG@8wMso901v6nilus7tnpl0$4?Lvkjwh=%E+atSUigT=#50#;#s@rKK8M=!J|+!8
z{J(tQ$F;I0@a4m&GRYK=rJns~YA7CvWoq0NZaOU2(t^JLrhO8?l^2Tzcx2vaM9sZ>
z1P|o@mHf@z6mSDyKL6|WFN6Psz(<%gDlC|N|KI*0t&5KS;2Zs4`V_LcRVM?!eDyb+
zJn8%ozhU1aU?RG=E?^NacRL#X20+2LWd-Iuae(YCPiV`0&@g}f3u#n!x4_s4m>_=v
z_Wg6@k6co2NnyKK4D<)FICsgreBP~vG|4>@x?c|Yv12p<jJPkOw1AoV^zg4UJ)XY>
z#`fTu=wCRz<M=Ne{tJiy-*EWQSGLuB=v(f#y3|k~n<6_O9QrROd+Fg*$Oa&M^Dn~D
z+i9KzbFYihV4=rqbI?THzXv#A6oFViv8=SJ;sXGb91m#5jm2K#VgkO6Nhk(k{Gnc`
zHv1IFcFrQu(4yb@_;Sj&h#SzLp+8uAkCv29E?ORdhDj4icdzG!?i9-v;4m73474Lb
z5mqM&nr8rqWyz>N!u(9|F*6iB=oJCrIl<r1I9p@{f~_Rb?h$+!M@yo@Ak?{QkS-GV
zoo|{%TwglCTvgtfXh5@gv9de(JNM85=HUaLv2gClbJsVzCt~@GMSO*SJ9>^82|Lt@
zdU`MGE)+^7JJAsMp<$Tf$(Y=AM}-bEa$9OlRpaU1Xz_qXNFsawHs}ld?&03EVvRGv
z!DFv!(R}Ehe9Tl(vsWR$vqdBDjLoiyk{5S<3qs=*{%7RB8GWw8hl5iI9!LQM$ut&=
z;Y)%^!t{^bUcT^AV2=*ZPQDxMXTZAMKJT92HTaOu=St~gfl6K#qmOv@9yK}|&@6^&
zw7Xvu@K=JKFyI+{5Pkf+!O8-i{?Euo(AU78T5I3Lm;fG;h{~sU@SEnR80o+5_Us|L
ziF=gGv&nCF#`gmn!J@0WYfy#;_}x80f0j65U?NO!AD{uvk}$<gkWc)bc{|`4jfbmN
zcYTw6>7$_h&&UrS(zLBszhuOecmlBRzY&Ag_iwvp$^ak<$;~NbxSI(Qppk!|bPxC)
zkl$$(?LGV$$%K+dLz8{^Q!FWWDU<Ea7EZK23d3R4+LCvDBLq7A&&a<kqg_yD5b?ht
z2nJ?eC6^f+0OA{!u7BH2N&+p(kd9J{=WZr`)A%Sv{2NMyGH4fRKb5lNOqGF&m=(Q;
z1~iM+_YV*w$$@9sT{V#1O?))a>Hmz}^Ii|)ytL<`ugV>@zf=U$L~#Jx|F#<wApnw>
z+f2m|?`8rEXyo5eiWCEWhY7;bWbptdqVMJj8qn+;6^4I+xBxuEd9{1;Zm`ks0QdN3
z<SS3bAl>FtN)Nuh1OnnS%RXNKh<LI8*zLbA_+J<NuM7Uy1^@3y!5Qg~JM*Tu6lgRr
zFqP&GZuJ%On@^P`tlsg^L?-<VA~z8+)RvA~b(KnYDgoHMBK*<jPBNNPbGe7l(H%Rl
zDWU143rfOOE-@2h5+KyX&bTpvQd9XGNUd{va84Uk|CO9a^IE*@mS6A9rUa;&>0Zy_
zU(yFYFF&B^$L)?6yo(VJ#P7<JcPW}!SPya0rL_K2phVV)Kjk-MQGUJ;5Us=$vGBk<
z3I{&pNTcZ(zO7pzy<_|bcRBfsxl&G`#CpI~CGnrEJQqK?=TOb9$^sjRm%(U21q2ZC
zU8eB%UeD35%JreUrX&dN>XK2jIA8H!5@JzAJB~U3s!Zr?N8W`e^u<UC0?|(eGjjEB
zh68;5^+YV5IQ%vEE+HZS^og}E-+4>$@CcvZ3occOAYTWfY@P&-r5}S+vqt)>xw|st
zr4La00Qm}Fxo6>MfShB~$aNP~3I3*QH9DprBYzO6nD_&d%fCtzS#s{E)x0B<(MLdJ
zdWH_@20-=!M5#;%%*b_r<Nma}N%%%{SCnNBvS9dq^Y?kl`k%5)W-c>`f<2Bg1O)UY
z>j6!-<XtWSeD<gHdBskl2gkb$c$jyU2!X8@DNtBqKGXT1(sT@K(6jH*-8)W>ov4ly
z_DXjK<qQU%{R2}@C!9E*0vrg8Jdy%1;@Zi5o;!<1;@`DpGZ#6?12k6a$x>CdjamNJ
z);2&bYq6moZXEd?$^AL<dv}mNfeeeUTKuZ4sZz_GMb=W#s)S(g8maFr^1l?nG-Ut^
z;dirvcF$E7FsPZx`dh4^FUrq>|5AJI`{oidnewM=Y)s(Z!JzdaP5-fCEDET1|5lDp
zLK?ewRn~=uiT}>qe@%SC|LTm%r~h{5|1Hay&N~`;U1nVz&Q}b~rP#&nzixPRH?wc<
z0q$_j+|42L+avUi%<t8AY!B$-#F+aJb2$qP@0|EZ5uhXc?1jg7Uk9`gKm^i_0Ca+(
zf|sl*;U1fv1p{u~d!KJEQ1;Fafz^s7R2@}~VNl>uaYSOEeYyo%S#7$d+eT(7LVbr=
z#BK9*W0Dchr&u@BzUS;&pk{A3TT`Kt?%n$pRY7TebG3?;^Qt#K&Qa)?oGNz_I`IXV
z<|1~Lgl<JxG0!S85#;E7+`)%CP=}(hs#|Dr6Czn-A@Oayz3gv3KTtS^X=iX{v>*d?
zTt+(;TDq(g2f?$$He)Wz)mxuKh)eg>+8lwgsUZaCumL~)`BW_4gV6J?ieggwJ2a*{
z`Qf;zr!Rk%s(G2y`LvO8F)wAmxq=_HgQ~t8*WvbbBI}UI?5nzSLbIc0SXHZXk<pjq
zLJ_l#^^SpLa$C!Qr3<wpzIvNEZI<EN3eoG20yfnWU62^h$kf8!*cx%J45wxmzu%kG
zx6{q1efr0fn;D61H<#n3vnEf+O=8LvJG!i=T)`8v7r=l?oc8QC$~LLMT{}~o;jAQj
zuiu}aq)iVXjl3=6bwT{qU05klgJ^m7gbVw&k3_NuVcwT;Yp%NfQM0AgSyF5MFPF|{
zX@hlxU&8%(o&rv7=)Btt@TOQMHxMQOJt?G-$jI}&{NkpB-OA<{8M*DJcwMtbvg0W9
zd+WdoA;9e3fVSl#LZmb6l4{t%cxq|hntof!zp)o(*J;Fy9OcQ&jvTDJcC6^2d6J;v
zxXv24X(UK%X5cJ~b2BXJX%0_(GjZKT6ZK1NSzcuQOOyCYi1HGjb}FoMpthu@)`LFz
z261~cyyrC|yOCSInQ|GX6_Va}fhwSGwiWq;(yU%n>rb==3)5_()=^j_bXByME0SDc
zU6~Ou_u4W1cpZq;_{s^oS=-ZLnSs7rnD@nO^Yu#1`*QkvSX-VdrDUGG0Hs}eiWgMJ
zY1=dm6lmMLO0}_xXyZY&oi&dR1%>C7<7&o{^^fYq7ZWy;?MvI8B68*?6Gb1AMxPzU
zDD$Udwobgj^KFV%ovk})(N5=3`%^Y&Vq9HkFeCO=j134IjU%gbo&<@WO4yruIr$#0
zB#_$K(}O7<(^VnNzVN@&|D!wUKVb}4rHIJ{_Zl0hUPr#|ZrtQuUa__+6i|{26q3+!
zis}Ja2UZ{S*oS%i1SvnaqF}`t^a+MtHcf+UX4|F-;I6OAOWysO7J<Sy##s8iT+ZuN
zLITa!tjFgq^@(X`!V4#q&U*1+<^@v{I;-Kcr4lwyK0HJcr%hRdlX3Mj2%2nEI(q0g
zZRk**?J^CLWTbyzbPae7&t~01I}d2>+CD{d%CPMWe|RNW6oQ&`S*Y}ls^ZWF$Ll;Z
z+LhlJXqeKUI(}E(lO2*=Vsu+QH9g&VU)JNIw}7GoH$7H|4|7NE{rcD(7g%Qyc#DMj
zMuSMsfUj-Sx)W=;zmItQF~J0nFO(tt4ZF}6^I|f@u*cBV%`Q6#`{2l|EUaFRQnbcc
zbobHe&k6fhAUNUv6()Cx?;yzBf)-aS!3*;yAfY=_<R(S3PHs~By&D|X7FloLleq%8
zBmekCO3M9XQO^GIAJP1~S0w{yQ0g`Pl?yWA>z{be!-EMab{+YNL_4p!$u685Aq;~w
zFLDMX!G_j#GmgsDOZoF;c!agP%hMU|J$TkF5f_^o-U>0V!QA`9ZDXaC%Ns>0Tb=vq
ztn4*i6~lwF7+v|{)29hdYG=J-<87T1U<FC|m&#wbwgz0i8X{Oh3K~}6SAe&J=wqOX
z4=3V6`}|3<g7mi>Z3TRDnX4k#w}mwzZ618E3C~OTuGz5GzT*6gry+zs8|%YEO@ptR
z&R6m+kF}&`u~-a*{^m1L9RP}^K_gdd`)(W3P#*6_qTQ59zM0alR(BZI=d1nc48>Jf
z-Z!kgHhV|6CyRU=#=F6^!o6UZiQrU-$R#SL-jKsy7}aob5on)=*v7A0ob4fcmr6p|
zc5{{Eb`j;V09qy@<92F)v0I`0R&=ZF`Y5Jk_p7q#8yNff!YkuA57vs=-;D*)tN{wL
zaFQj+r|>BGXJs^+W2aWf_jV<ue4+IV`(oe<g=v(rz3A<?3k2(M6I#3VtnKKoeRYhp
zxB3<d;Sq-@B4~EQc^9-!Try+RJQTod*7@Adqpm{lM4erO?A&wCVcuZi_I1^F!LwhI
z;N)SL_k3gF2aj>v0bUR4m1@oN*p<t;Zgc*GCm}P|<Yl$3rY=4@HbL~PYx1`*?k6$e
zyVq^b6T$lD%%O)(<V~AaodG4e<`oAhpd~K%=c1LZBW;&!7{oR%njZVfAf9&=)KPPF
z<u{`^%lP%M$<lhw@ek#4PHs1>+?nyBzjw;asjI!)v>wg;talIBHu6@_F_#~<(v&TS
z7<%l8cgtQkM;nl~$d0!t_)Sg~??g`=>f;d_d7h*@<LBHMeD2NDZ!0(3O|kyC2L!a*
zxK-u`($+72zjJY$l1bIsK-U6SbKL_BXZBURvRGkm!@_3IJ<p;p;JfS#iIv4S47P3T
zhMllA##^s(PwQhA1JMor<D>D%NrUyU#Z-r>NR82uE7(i3Ra;Q`nOjV#_t~h&z*WQ&
z59^e@NnItn&+6}{$c5;X!g@pZInT>w!3|>a;dg8uOZDEX)Gf#+y;<*(N=n6U^wndt
z$n&A_hlme4x3#XW$K&-zDRE?FR@%$<e-@54PWjfmRT>|wy`s4`uu||sH8O^N{?UWq
z74N|fMvqkgGuB#IMutrfeb(`5U$M^dX!Gjn*?<S$!|QA0g102!MtO5r)5aCl(9!Pu
zSh2?H@Lb=yee3x|Tv21-22Hv5rm+1I1QO0_Js4Kcox(d0Rcj=xhSguhic}A;tw9PJ
z5NYpWA9u>Cno#Ap6Y-b4KV4o#_fb2XgWyxNW-Z%YBMs{&T-{GnkbBOzv)(XH*y}_@
zI%O|-uuMzWZ8B7f^}NxfWUK_Vm&O74PF{~PMEwxlxYg?zBH}66idzF;<nS4Rj1Q<m
z3X^QHn$wF+Qxw(39$ox~eNIA3s9yDE%p>L;E3U_JLMM>1=|xYE>fdVe+CGJcn|{+q
zF5Oys_D`!lO|riBjSfl8O~U?p7XUkbV@`|^4Eb#+e_LD{YJ52yVOfOXd8c6Y908Ib
zVXPRiWos>KE&IgC+OEOht-A@^>o}=<Pw-UB&W`2Uf8%TLE<@V9nc#j*CB9SK#SpGC
z*`V9{l=|_+<kbAk5NxHTzSk>#rAhYj(QyQ<5&8WB#CE<+&d#6o`g(9g&}isGoqqZ5
zwV|D2-s6!}UH9_j=#3)kmQR~rh=FS7q2WbO-6rB~bRyh%4ai@jIYk53qnhf1OM_bG
z6T9u8bKJ9^vAG=P7RXR?<yzd^HiBz7*7SzwYPUx@$Yk7kz6zHzhU^4kq;1~nHg-Ab
zOfM5jUF;i?ZBy`g+8OWgf`@gjc$w*TOO=rat-lht><AcorNl^()9rj+{@SzNfb-`T
zy`b<m{cWyuw?Ny(_Ga#HQP+9@)?#+Znk4Z#sxuH#9H}^sY~B&!`>y%+llRr>Cbij#
zWguB)I0LiFN!9JnQzY@L!dk3+R`*(vqj<LIE=tRT!X$j(D~pneugIY|K~rn6{ylaE
z8{~vmu)*sioA9d4@nXX9^``<2G7jskl*9>f)Or><JKway$Y5jrcd7*Otp0}WA%dsE
z@HX$8c0n^W7zlLlpingSQs1SrbI|U!UP_0+>2Tp&ZNZAY`6W+Fzw3NP{I0=$-V=;D
zmqClf5iw84?v4R#WswidYPS1AYc(Ppk?JUow^Y|r{m0ZT%C>Fthc4UO(z(x6Mt5sh
zF3wwtO;fLWm@?bIigwwu<-(7wraNkOI9|Coj3I6C2t{Ya8r88f?9y9N8*<2T-~BV&
zPrB(mZ&zVTUASY97;HyM%P5qsmHab+4K)02V|FFUh_YXf$Xg0)cf&7U{z5$04I9}a
zW|wQYi(X^Lyt=lw&RuR#2d4xP36Q?f9oMgJ=R?WYE6QG;tvW^GX2#2@D0?pBAxnG6
zRcm&rk@fZ9fX_p4vSumXK&)Z?d*7^r`X5~}>|ANQ{d0%10be?+w_M=gnn~!oK3wE;
zFbl+A{4(|2f3uPtI$TICc^W-(v68}hV^Cgw0m?^A1TFv>z4C1lG4{Sz<W)$%V)DLe
zYb85zbSH`e*1N`9cuMX}&B5uy<l4HHU#g{RO1p}HUVe67yf84h^7jhgHR%EiceR{1
z1YaTAY>~nPVNX<7BVt#tA6<Y$ytutjJdW!Y7r#6gj_(Jz$n=Tj(sjaeMQj(r7ZmYP
zeT-qtx(ZRurd-{lo*!)zwBS;_*Xe3+sho6;1RtfxK>CEdh6l}3@tX|PgNJUj-H6yN
zx*CoL23P~2{PHKMt@6vWRsA%4_A;Ug5wN^h-N)^L9&`y}A|@KV$0O$O6f<r+X3LxW
z8Sl~Di<6vsB@nJP*3xCo=-UnT`j)9m++vXfQ7%ZIL#kzX1_u}Riv!VnJx)~qxlg_u
zM@HUcqLvgFUS_-Qr+N)h-nQP#m^1jn(-Mp#t&*R|owXeX;ow$rCn!(TpX0{fPCQXh
zUFxO|f%N$}ss{R{^-pX$-WmuzY}~S4Frgj4K1wosxLe1UoUd~o|GYtNb1i0Rzy1e8
z);v7T?0G#dIv4$GnskfJdUtJQW!<$L{>^mh47L62;*;HTga5U-JFOMviqOm%Jk>gc
z?ggI?d^2xVI-NZ2Xi5>?>ob2?k>l|Q+<oc<-0*X{fJEmLwVZ-AC85}~F?D4(#}l+Z
z6%p|`;t@WEaWdEHs=}%0PHK`7dD3ttQc@G0G@*(rbTO@s7v1n3twrlV-&ccb&BCG{
z0bI#FOpfJc1#^)VCs!EYx1H6PrO)17#uDk`)mUk;qx=d)wPnSrd1Pc7U&mLG$!Aqi
z5-Kmc=BR(Rt=zrI-&{knXd(kX2@eRzNH?xlMcSKa!@SRyQb5@4UPWmSbaezzh7wo|
zQu@*Fvk)1VvBXOlmE6>lU6%}ta03P059?!JEm?2<j1ZP*&f`7RY1vzD<UiG?(*>M@
zlQ_3r>Y<-6E?ET=IMkwL56co#Es~LI7-lB(7z>YZNClpqs|ynUHte&xU?yI5C;bXP
zTd>C{h#jXXF(BQwsVlqK-^^IBGG-plWfEE)(;Ik%&E|=Iyxxc0*MU}S_i0?$gh;Vs
zoV>E?_R}J=plSUBcgv45FeOuL+N4i=3?5FyvSaSnu5ukeE~0!bryDfCC))jv-5KT#
z<SvV)&#`OVKaHpkE>st!v8p|@oGj7m3u@DHvuWA%R;Xt+(`qQuI>fiDt=79PrleK7
zc@32iu187lYZmt!dGcZfg(wOC-cQs8$ZoF~VRqK2vxQ;kjd8p&01^xcf4kcpDp!Yt
zU$`O-2wQV~hb+nKkNRm2t9a^8t*agA!&ms2!X<6TuE!Zt<cBTIwRF5FtC!5?byme`
zkCP3Sa~84PZ`SWw?S*RZmHP*m^dnk7eRW&y^e>st@YpjnCm0<Hc)bU!QI1BKKWZGi
zEj_p>$3L6jJ@@B?XtJ)>Th*y_iJBV7Uw<_94J9OAI?qk{D9!G<)F(ws3#lJI?9xM+
zoY~HHk#eUuAheF_3|eZAimj8JyB6(|XACe!);mnc!)V?4`O7FxqJ_yG(=pcS%F34b
z2Q1u3<RzflM~=X<zd~+r=Gm3tvQyUBG4L#xb5e}`Hs?j~Z3YO2FARCi2ppk@*|l|~
zWg!iq0!u~(%@^D<3c?5l=i3PmB?hHLlxE)9d6VX8QduYx>*4-8piD7ZOsw%=<f#X0
z_mqTEUs%Q3zUN^Wk=FJOy{t2ta0%zg7q)1l(Tn!-*Gf);uS$x+wh?VM;8XpXn@lox
z!Qy!p<gfU2j(Ps@HIIZvEpW-nR3*HpCPblRs%Eu|JzoWFKkX5JxkMDOue}XaDwrSV
znmoOgG`Bsn%DN_a9$U@YuzAFIn(ftJ*a|P$piQP_gQ|HUuZB~OebUCgY`tFd@d-05
z?z8vqngeo4^Vb!#Alk^1&^}Wtcb$l*sVfS{tb=8z^(A!tZtX!Hjff<Z*wBtS%{)3{
zdpR;BoxNwYjn)7KA*PiiASRRRuHlPk@?t%lly+upL9$!H9M`_iGQO^x6htn@yl>(!
z0%gh#FULHewaWT&LY>^U-SQLhL4~S!#@XJjI2w81ZSk)PVhs&3Si*T^;T7+J9aSmT
z^m0<NT$5}2r3<W3M8cVG^3Vg?VG#29=JHsX&uF5VL)fxDuuoI1ajZp#vEb5Ts7NYw
zY#Og#&~NoivP_$3TS&s+%|FqAthznNQbo5GkXECj%CnqQvB2xeY`*?@f{~QP<i`O-
zk=}&e`v!``ek4*MQaiuA<19F35g93%=KN^BtL$sS=hps)c{TVK6H}@scO%9gPExwu
z38BrAvc=--GNiyvs-1S6yL^RK$RlA0xZcSeNjooqPElsW1~WZ~<ap6!tAh-25;~t)
z%GX4Czy2ct5SfJozw0@V*=C2*2y+4u;igFGb1jn2$0BZipeC%OnYT`VRl*(?q&&Se
zO0LS6_syK#nR^j!@AoZse3PouZ@#YJ>{0TK&Hd@Ox}R=)UTY`QLLU{4S;4U#n0NBt
zj^k*w%m{rlGUVQpOPV<D^9FnMG{F#mUZ*XK;~a8zEhIP9$GWRETe9XnbG^3rY`#}e
z7y;+?W9Raqbhh$8*7arRgn27%Yc4lH23qY>xe~W#<Gt-~B{sO^D(SkmaVXN-_Q#wD
z_l>ck?q{uLAb*4Vd`2_#WzXX{B+)f;;9@{w<<(Mk5$h-&gKf(?3Dbh@Qpu7MyNVvh
ztyDA`yiS0x^tCm}QJth8WxC>4*Oq*}1bxon-3Xd55I_oJ*n2Rn$~l;pmx)VYm5rzH
zIK2<VP!j2fFOcl_o!o9Cu2oCkXn*7&9juZC^i6Bl?h@;2JVH)91(JncQ|MU`d+B^m
zTnRLHjK9qDW|Ax?zPhWx!hNB<kZQ?xIe4e8g^fF~ew#|zVA;dgSf=NX!^X1Sd?GP*
z3gwy5OseoDLqX}ZVp|Q?=n~_eUF(mhOV{BRdcBz(KO~)LF}{QB6`F2^`Ld94jhX#X
zW<#Zx`M!(yyiPl)Q5|lxFS3Y=<+fw&da-*S(vx1$&1d(L)43=>F^KjI*eT_7i|F`^
z7a!kgj)I#?KvJ`9jFklZYcir_Z(7=OIjAfee+&~E)#v%cZT@zsKwM%qYGJ$gIzkIP
z$*}%%wMATF-BMee8fyHzUn5Lg_{;!it2q7EZN-O!v}ApgLqKOMbBZkHPL-RK+@E^x
zNgw7hYHk9kXN3n6#X^lkjwT97J04e2dQ!}$4;|2Z*7h|W@cfLpYt1~H&oU?V$NZ%J
zYsD(;1EX})?9~<I_Q{l<3jS{IxRuLc@;c$0eF3{6fnvOzEc0%Kg<EO)h<3dOBr|h1
z-8c{2LOTfrk{v=q;i|-cT3LdBM-~R-byvUec{LnBHvKHxXWCqYSn?MlQbo9CUI99E
zGPDRf_7}tj)rANuQSwi2I{>#;CC!VMP*y{!mIO3iTaI@Yg%-Wi@qn{ptoRL<O7zN4
zlC75nE-AnW)yLQMzl2x?z*MJyJmQ)IEr3cYc&+uhu;pQlvhD!?)tXnDYWDU0m_fL7
z&0)d9=0v-Bga0ch^*j&udNscZw{5KSRJRtrT%9<NVhyv_Sc<><%Y5!uK58{hEM4V+
zyL{r*-F+uQa$)!X2$mybu~Q%Om#rkMG-nymdqfd|H1is;f;=jkdf-DJ3V)J85W8_J
zPkJO?C8z<%qLhc4+ZDDzsp|KP-lEcQAeox%%zb$W@$$3}dkD7oF!(B=&zBrHo-`SP
zr2Sv!U<IvShH_%-B1K2nc1w(%obwjxMHEE}{GvX~vhuTgt&HOY<E<acAw0?HY83V6
zYvXchVmM3ddLRF3D&H0_OBE#aLhi>MDadhWq$hGr>MtGBA-Tr69P9C;-B%CN;aBI4
zxGZ>{PQH0%!E#Bl5coTy+Y4WP*57PiSL!(igpM&Ft)9(nE#EvS>)5;8dBa^Y(u*;$
z+1k-ncbQ(k=jbG!)ACCx(^NWR{lKyWFqF|`;!v>Qs#Z6S^?C9L#klo1{=(~DkfY6O
zb6b2u&Udn-g>=Y|oL1|C^na_Cits=@!s*#_e7<n%y=TW<h8H<szqU46TMHDJXAh=Z
z1Jdk~I+UOZFHL`)zLrNu$bA~$%p_t}zci8+?$J$3kj>ZTJr@U>e($kaB?}aK&&Jng
z(#RUpjHT1Z=4;?uqi?Ua7WtB+7KvH*+0$;ab*AAO>V4gh9vTInC^H8V-`HMl-`I_X
zWvlwVb3tacMSxSX4D{ymh!hRGthA^3`GlSeueWA^2g>b`??mtk)iip4mMnfW=u)oG
zFmuIriaNZ>lMC?Z{PexcTP10evN|M$vGV%n_2Su)!fDU@*@SJ8TH*Y~N3}sOR15jT
zz2i^D3OQ<uW5dKHjMG4#aEG5y>^Q_Vk!%x(z#^Xi=Ps`pX&Q3UxP74sK(;e}Une;7
z+S5wdp?RM2m1GlMfo^UNQN<)Z9^vHZvc4^_NM|h{dgagXXLHUGpHR1fmA*wy&^mGi
zKCxx@y}b<NM8&K0c9XFBR@Qv#gV*cNd$qUk*^K71o(`r<*H$=$Jty0_78!^7fD}_}
zyLH*x^g|FHo|`z+hFpc!Za>*bq6(t3z|9)ll%r5EEUz<Y$W-HGrQbb$3)I65Dmif+
zn~kEO`MZqbejnJ2nilr5q`$lb`jfxF;7eK5DQK?qIt`bhu`8<0+kQRH{^Lc0I>CyI
zzei2XDjD6H^2Mm7wq|3Mk={Zvj@7;ai78=oaY$XT>%23icA3llHo?S#+qGZsS-DK_
z$EUoswT&S<w}GmOBxhp<!Ji_E*CchOdo$XiVNoj;)Sf3pcBZBpS%~57;|*1>)Zw-}
zujrTZVuno_XIu&Cn(QGUXbIWgHDPJXewJ2Xe^6X$>r(!E$dK33H49jec;U2IH1%>~
zs*MWu%9CSkXGSp@o+MDvx>X~I!M9~?+V0B;=26_YQn(*q%-mwRn2jfF2ToAb!>#R?
zDp$VRnJ+HahGn#lv%p>QiEsrt;H5bRdl+SZrtJtbfarHETy=3`H|Mf%It+Qr`XTP9
zkGM(8g{gNie!(`6@gi>8Dp(=rc}9I5)<c6$14m16)RjYDZLrqSHdsUrNVaX(zGzaI
zL?(V|*VzbXPI$Zi>-F%pissZzHxY$yCF#w@rac!(44b&ry?#SF#gKUM!@97Qb!4JY
zc3=`@lBv2#vxfBn*nJHC9fT@feFS@p)FXZ>Aqx3X@-qTfl%-x;v(G<9u~Zk>zj`}9
z^l(2@rRif|huJaZbGtaUZ{iZr;u;nM=zPsDnVY(CZR#Q_z-98=fO{8+7IXu2GfWPI
zOn288`tqLc5Am1PC@bI4(`R5kL<s5ws!w%Ei&c!LURBiL5=~Z`p7F*&g+?jf7^i}^
zIaBZZS?!|CSC1y9XqNfcjl**&m6H3nkGG-rP9<C1@0HvyHd*wd0f(u~r)u*GhHl0~
zskaQ>Ya_E*+B!6!r>_SAo}=?}x#}gIiXLfPzYuqDH*4Mr^0MB*hK7$)voj>fumC|q
zHx_gtNDQcNcVhgD2VcD21GUbXp00K&s8VBsW|<gw(j;<3usR~f-L8K>jeJjG`ka#3
zNy>fde4dqNgsnkIkzBwa?u1}W`dy6v+y2$}43DY%3O<k=MMg4{)IA@tS}u)#u4vgF
zck*p+Uy;TBUv2^HCb)A8d6Dq-=q^-2JT47l?!9>GH4EsrstBz`+Euiz4+5NPdt1N$
zXb4|}!RWNf%@yn61y7u}I&=Se%v)hVNi|~5$iFW4p6Js5umxiiHnR1WucY(#bUHjP
z52MT~bKPbp_xsHnvAC(lxNWHF_pML;jZLFqA<G}LEIdMty`RXNlHtZbD=ignG%ZP$
z5EQz8pI75rF*ClXn<(+q%;FDZqh%3ESSDpGX$y#Fw5#8gb!TfsG*d0*b3p?pa>2Kb
z{acU?Hu6yQfw;9+O9S#fFFp?gE@+ITUGSLtgmi<JKbxg?Odg{!<J=kB^k<wJxH{W#
zTbnd*#+(kF5@#LL%$J2V?LC%T`Uo$5OV_G!<t<3F+tJXjf#<Wb;{Mnw4jCqy#;-M2
z!~x2jiG9QycCVE|>ldrDo`AYF@IG97(vFnQF8R%GeGf}-q=K^dI#L(=c}%-zdmd0g
zHoKWGIlspWI*GOSdk35HNYdUAvC*pGgrF8V`_BX64O5!NGqn-PgLNqFt_;Z5bGFvb
zOWvi<Exn8~l}X3VbSrqv1y23(N<cxIV@6-(W4~{PZ)-6AWTlk9$<W0}hZGb5GTaiS
zbQ1tV+qEVYSwW)Q@5;*lZ)DCZUl`=)C8TYG**IGn{J!zs#D5ikIn5v51=ryn3S4o#
zwgCUSbR0y=zQ7~YeUR->#4^uV_G8H`LozLOYJth<y@~hj)?LYEA$VaphY2`rk0})y
zJ+ro**W+1EqGOr&om4UKAhV()L98H_QHPhf0EAy<I?9Sa1R$J+PdJ9Os4N$f9jI@@
zphmLJH7Hy@Zu{<XNFUE_y!Udfupn$}$YeivN`g2ed+D)>!v~;#kx_=}CgeC#v(@)F
zu~IGG&vKNclFg+(kPT7cTXIz9LyU*7yCBs?s59~u|E>qy-Yv%Q2EVEpIV+(Q+@Be_
zI=SXE>j@=JeLM<i<~?|Ah_G0}Z!V&f+msa3G>%<EAnX9mH&)sKAoJXU9l&-xVwXHg
z5LTbh3^`Ky%{-JTZWnLrX~i#0@KkADmR(K<XPd)YCOeyZfb3SwmRhE{IJ2eo_NYa2
zrDosez}DJfcC`@1rsAk*!<mM}IRBJ}N}+AS8;Z^QRayObW`-F7wt>X*Uq`2~rXGvV
z+12IhZ720gMIZG{U()h^xKsx(t}RHP?>_66U<)2ImOnqf)F9ZAE-fkEglDAdMrE#6
zoq$_x-usK|cT+b&zBZbD6iSeu8_Y7G6j?vILV1`R^B;VtG`M;|x@}S#l46oN!A6tE
zYIv0uz59j|`SfLDGZ}J%SU`i>)HrfPk&czB<y4MuTlL4O#aJ+2V}JA$=9IgA0Ob6J
zno!foMAuz9uKZhOw*ED2Q3vs~lD>@X23T39Gmd^sFE2FFeTpVG2qCG34y?$?#si>6
zc}??VcppLXcQgce500RNpeG$J+-6--`@_80R9?jwz$)M%M!k_SycN<M5?F~6?@mn5
zxk(e-?4Vipt0?)(NmUBH0GZu~CVP)c5j(dU)k+8h>`dCNnXh&k&H)~|(SG3f4Hop{
zEPp=n*v@;2m}r-anAAuKSiIbLt6R!ZdMd&O`fi9>D=&^^(dZRh@p1pbD<CR7{z=Un
z*$k5HdGi{H&S`d;J45PZK!Q&PA6{Rrj$QvEa_$hZLZhPR_m1s&gy{+C$S}VPE7$FH
z{o<#_15KfagK5@sz`FIz-xqTPV)4@f&)p;m4lY9_$mqC@#y&qA9F5JL*0GK$8^Adw
z=jSg)I^+SzXTf_b-A_QO1)pav1HXbKUR7VonZbe8a2<6v*F!P#vf1C;$JvL^iTJAe
z?W`M6T^ER8;@07}fga24f{P2vAs<b00kBJ$&e$~%Vcig9m;yV@fNc+;f5W`(NQX_u
z#Ad4%{Io!U6f|r-WDZni*DGH?^lOIKVP%LO6K$H87l#A1eE=ye(e+nP6aR(S`OwYc
z87_?aRb0Fh&UygnWG&UOO@GT+n`YZn32m|+P#cqPY)RDYxmfesTOX&%(hS?L|1hW1
z!*%~ebJm|z=F4pWPvh3n{_x2*_5n~uj22K@aTnS)jW02bm*bUDZ2cIN$OujP%9=MV
z4jIa5MVXnujof`kPyLi$*#4xR;l6HMMFVrh2l@sGR1<qjQRYM^dIhMq^V9<j)>lg2
zA)a2hF36p-V)hTMUqmd`*TJrp80{@WJ+7Ur?fDX11Qtx&ww95sm;*;h66X!op*Pxb
zDNok>fw<g1QgAP@$cVb8J$W5Vx`_Ai&BFbtTVOAm8ErT1V9|jlAahx5xv!nU8&-sv
zmLwPGh^76MgnP<<#G`0CO(MT87PM17<`@<d;zD2r`PL0=?hOg=JE{EK+fj=NtLN>{
z?>#4-4&{vr&<veiMO5zkN+-c-%gcWBF#<1~D79wen0_SOPc&+FU>b*)v9$jINX2m!
z*fDa(NR91WTTzP<J>CEtf2j}G-Dm4C=KgMwyA^8W9Cf&E64?U^r@ffFlRC(U7(jtW
z5v+)!5mAVT7Z4jqm8KI>d!^fKSn(Uma+<Hjf(#S(kw88%%emaNLi-PF)0q8~`Kl`@
z&Er3~lO6~1C~aNS#&|?*@!SYvJyiZW><K`3)9)p+hY`JEm?;+kL%`HZlTwcTk4nCX
z@{w=hQ?F|`G=T!Juh1=if!$Ot`0?m&u@3X}tVqOK^YhHU+6E!no?S6lSzN~wy(>oL
z=j`*yE=Du`-4paC?T*Rmx=M3c1-vemRs3&OL0}B*S*lhwaf2Q7Pg`3kk-Z%X-F0&5
zQyl+Eo05z)Ot*f9{5ZjU>o{M_&~g=nDlIk!@@*GU9X0X_Xcnm>EG5oPJ579LYIKw2
z{>v#i=6&}^QFaug{R_gDxz7BVAElh62owQ>ZryNVGJ}5AP3^bqJT-e4;sV13*h}CX
z&-u32IPBF9t!-xZ4pydI+$9WM)Tf(A<{VPS;^00Z)%$0AILVc_q`CHux<SoK3HVqs
z^^<seJdq2(-BD{zoao}~U{IvXLTkE*V+_0NK4)+WkRq?fM=`^$r^31?j)z>J?zsw~
zP+cCfwj7q6@UL}!+Lx`2F1(ERBq>Gbz+%%?nD>uznW0K_P+R4^tTRU{P*MV^f(OGk
zunxv_L5ZDW-v({`xI>Hkxa<5j8e%)RH(%~_1m7@kEm?EQQ^5)<N=P-fYqcpXvNyfr
ziI`IAvJP>s&jGzbw4DaeZXU#>?TzJnctMHWI$ozF<MYB=)dZb@A$|~zmW@{2E+W7R
zvfm$i#{4z57>Ca2f!{lgN7U1sFP%T8ly47~9o=OFsle+v)NME4s&-U<*0J<`$L-H8
zbMCZG_~#dBFir6NAXoRCBV)?Nw>Bfq3}7*-#EPq1K0*vAl^X{ng<C8rXa}8*AZt{c
z?kcR7L|h8c<zjgfy5gLU#yLWOEaOQoa20><a^YA;9teJ~EjisQK~zlSCgTytT-H^m
zV=|=N?S*SBoEkcqoSy&9T?t*}opEP0w<&i~_xRcET{<KvmA`8xu(4U$S8xuV<5yy#
z{)?}kf8$H)x|{i9so{qfCrQZiTDOvxI_U$GPm_;=cGUF!zG>&~8akmOMV3#$YPRk$
z45P{<qDE;(2<Hfw{FOv{a<%+fda4aeb9z}5j~gW=kkiF3%%q%UBMG>XQMQjA4!Mu^
ztF|lW%XvAg?$-Jk#%tVIkuAKp7wAIe;w#%8b|5T{xG3hcKm^7GW9oVKsWHiHR<`vs
zL11r1!sN7^Y<od%M|bzTAhUDe9+p(q7dynG8MiAPV8sZpX|Sj=#hb1@@UQKgSkwgn
z`lwO{Tzf%Nc2wtu^O`i<u=g5o?ak%cv2!Yl$KXR8`40uYI0>W%R8VSOy!NYvphWg|
zO#Nn($nER`WicDU=L!6zm0+e;8O5R|2&MZKl^vfvFWlI(5fB@zqgGpAZ)YsgIL^n>
z4r&sgITZ|3aQ5zj-$74G;mCvxFN+->o76dB2gi(<w33{>J=mnHGMh}NzKU6=;kMfX
z+s0r*da+~R@4U?#E~Ijb%~X}|UiV;XLgn6;cxP$Dp8?v1tYZe4eccjsYbesUgr3X3
zPQ018S}rbGtsLti^)n^$MYaa$XH<<>BdT?`@#n9Lg%>Yw%;8PR&gSvG1TQ7iLcify
zqz-3Jecr}6>QyeodovfG#@y?_#A3}Jn<0=95;`0xE^+92GFC9LADr}_qf{by4y$$u
z6#le@O&wJeDajthvmg28#+P(sB^=i{THk`0PGho76*SpvDY|62mwvFv7xx@^2}I2Q
z;?Kyi&^{0qi-7J-mVqXx1H$1?)KFRShGlZw?#vFkjoDB&r-)Z8CH?%aPgrT&as7ay
zH296}5LC!L7(r0TgKpwh7Ud}Jdzb0Hz$a`ugN-x}^skwU1#GfG%i$*!Ngx>+Gq7D<
z)$t0)pL~9^Yg<sT;@Tx{Q<6so**<9eQOj{&D96i^rX>CAUR$?(DF${C&xEKD?L3vH
zrsAF6!-Ky&AAs{(pJdHTdX89AYIlrqq}ks#ljF2B%WavcZRCR;0I1)iu{v#ASnjFy
zsf+HV46pN8bW#;{9$r(0PXVm%en{!Q0Jr0*)A|Cg6~yMP6S<@($jgl)XSo;<Tx1!m
zofDy!b-d5Ey)d@81MH65D<1KOi)q{zNe*#8*5_MFQ~$H3`nXj)OUBk;-NA}}pLf<O
z!jC1jf~ByQb1;Q#o{69^`YVQO>-qb7NhxG(yBN-cJV4!mT*Aq7G~4cWzv6~Z*lp2O
zx8N_l>e`{bZ=hzB0lXMvPAZD|uQy3xz<<($UD$a`bV?w@tOy=_ExrE5%L{Dyn^h>)
zYI+C8Jcz6N)bQN%+zHXL^U@hL)1KeCjWHn7?dpV)J7npRvr}X)CtJx}gY%P-d5-AK
z1qQV0c4moqE{!QLFBIo)tMTYiVgaJHx7Oo?mRi7Q;YQcXisx&nxg3;ww97W{1p26^
zL`?5ZXTf@spFY*C(Z35s&el*HaHiXSFLv74MPjvdvR4FR?@E0~XX=>yIQUaJ<>*gs
zmfd(&ufengy#!&cc8|XMG3tU6CR>dMv3~k*)N5TwY=_P8?!kBGZ7W<;OX%-+PFe{y
zqK$E#=IpbUJqTxLJW%^FcdaDULE&VT5!?T&*XM4bz8u)Sq4e{{fkW;j)imZVmtpPZ
zs$C?fGx-8vDlT$E)^09Vdw$q0&ribt+B~W?NO8+vCKTWPaW0Owv@*MF`|ZapAt)j~
z$ZXH&5q!!Z(7^aiWRDJM&^0nT_2F>K0C)9`%}3L7DZb%Bjr-s|-)-%b5EL7?!=nb9
z{8``{5Pe}2G@UoM#^ec~0m<#g3RRDM2!&p<*_vMQubL&k23=`)iGJ>HIi8aroF!f~
zA<OGMC(cN<1%wSIE=~4Dk0AO45YotPl4sC;3V*^q#U_TcY3p4%R?u#sD<4v+wI?o%
zC{M9&Vf6pB_m*){wOzb0AR&#Upo9ui5-K5$A|Tx$jUrvr(jkb7gmg$r$ABP13^5pV
z=SYjh2tyArz!2{>-p@JDdGGT)=hOS;yuX_d{Pu=D*Iaw;b**cy|61$6gAi)rzoffz
zT1Ny0G)0PI(|)yL&p(#y34QLFDlxscyffV}`qBh7vv(V0IwNZo-Sz2Bl^w!j;I~IA
zRZ~w4d+P5$z!=E8Y$)kSbDeIuxH1u!#QKZL2BR772CZ}CGtn|T+I%wTQS1Qm#x=jH
zLzx6)kQ;2Jach?y^1(pG=E`V8jCQV`mqYGfpOzT#C10675J<|K+MIVE^X6sWW+}gk
z*@}OQ61E+%vsH8o>`N*rkZv))VOKJLUU1j^9b5ke(XAJVW4^iXFoSn6*Te-CS2j^<
zDHw8l?1Q`qlGTZj-N~kb!rGwmcb|UL6dOM%dp2cU60#te>?(0a=6x)X-^;)VG(DBM
zyPl1Js6TH1GJ4|J$E*z7>Yw)UMO$NTd<zNQ$@m4Kgl#ZmFjYQ*CkN%W?-dzE?~EmK
z(4>N{3PIl8$m>7%nZX|ZLhH)NXC5CXP<ZLpuSd~Nz7c>Wse=CwW*>q@OSI0nGh|dY
zrax_43wpe@U{cal;mrW4{gB-2G(9_wnwni0wnCf{qvu0<Vg6Iyt7;-qso6a!ZxN<s
zLG4O>Xcrgkxn4A+w&_95*<o-#|M$r;sc^3wCWNK#Pu3IB<@=StP+#p<|Mc1@GU5Ox
z$aN>(oI1@f>rzf9GeiTr$5VRL8Be%Ro0iW@Hjz?=xr4=88y^~;ycit2b|jplBTwMn
z*d-V~yDmu1E0^Dmy)}ND*B$YR-(9xmEd^<te|d!~y)!o;B07S4C<_+0o~_?F^^#)L
z&#u&|z~|an|1Esth><~@7d&2@e`<<^TVO`Yv>+@=YW%!gT+M}{hN^Lx_-LRsV2D?w
zTpNAbjk?e6f9`zPb&1zD*3HO4INBj;mEq&c{tibImCdJM1NZJ)pZdi)`sBOlX4APA
zz9>`RN$}e!dRZ2lWPhz7Ve1q8kPTp8HjfR0c-rfImS5-iT;t(weOT4~Syz>j)$_|C
z)$|neojp^K2s93Og2zSqmgQFRcSlg*+%%rI8G5SaezUg6;<TKzFq-%o@3WK7g^Rk`
zxSF&eNK*4h3vPNm?ZNOQk304y&QJv<J|f1LBD3uC-$9O8b7`BQL=*!T#5MluniN)?
zizq+gX%9uikqV&rDu$zUe4yWYC+gt|Cl3TI-E7d+-ZA;8S}B`00j&&C@tLod2|H*Q
zx(UVsmjqT+NqRr=tZ2k2RXHc9n||5czmZ`kx-x{=<`JtT+QwJVI!yGc`c!T@<m%v<
zI;co!tlF60@XMm|zReYA!OOs89_Wq{Uy2j0?m51MkaJBjb7V#*Cl|k!+lc`$Idh0o
zI8`c77q<+aI(d4obY<Eus5LoYEpu`3vUBX_i&(fu<e(e;cxKLy#3}GGPZb0DiNmzp
z9i>MdarI3%4b4&W$$PNJUutX~?jS0xY;w*@HTg^(au^}~#!O}#z;5i-`?^N0tk_5e
z8zfJdgxnp)j5yftdy9J*1Lv40`{m7*jNIsU$%)_9OLuxDmmz0jCx^Lp*-tw;nK3~<
z?HqYb49xCaPD%O$2`S}E8B4hqgav_vz{1$H93S4Y>q~zBg}XZe4xybocUfx)A5c@-
z&=b33fmrzln7|G%Up^L8+P_k@+99%^=+oxFZPuy>{AE5lP#QrkSs7s8%CVS6|I%M+
zZ4z$YKRjmGeR$@>m6eT9@9rcVk%)=r7kS(s=yD!x(;&M#Q#}yBQH55zdmf;8Wv?}C
z`FKYPt&2|uZQ4`6g5n&gycd!)6_`?UwThBtmr*Xv8CT_{EpTfDo0H(vQiP?3H|UY0
zPu7`T-)rXZQnMUU4<^w^ct6ZM&3a;l<g(+m=5#t~7A))P@uH5s|F7PY35Kw8;twm*
zuMT(j&&rop)ZR-ZE*0%@sJ?Y8xmTg;u>tV~^&_T&i*wb-7TD*iua3<-hG(fp8oOHx
z=bqqD5%X>hqKcGO<t;d$f_rdn!=0e0s`)o-Wc}MYb0nHg&m9XDfo^onD1V?ZnpEXe
zTMl$Jx4>GvL}`iyu#L}UAwq;?MMjw+T&UzRi}qrD)<T7y)m$GLzSZ*+30bwjiX=vG
zMH0>o#%GF_SqAf!94$X835|{yP(mAwpOiM6&QSi7tgZ%6)<RFOr)rw8>go0DeH60U
za7m?jZrc7VLVF~-ih>jr50tHj{P=1BLhQ^yV-LD~`HvRHTXAYT{P+zRcUJz#o~K97
zayIKjZ}WU1m%dVK>X!686^x8t+Z~EYl`2#k^M}-V|M2%K{xDFhaNY4ReY+2#xyla7
zCw|$D^b*#Z%wi9=iciMM`G`7Om%m8;l}iJvP8JsHEkt!W;0H?=c-vcJr8>mezR9Lk
z({B!sA#?lRJxzS3YC!Zbb|uTM7dB9%KPl_*eD2HUkUhw=>D3%Ap6QuAYljb>>B%W3
z5_iiuvqOJNb1bgmQGW2JV)$;-QD~oVY4N9CKxR<v#^Wy+Y5Vh$kNK0gawI1p!rRED
zJGT5N+NBa#g}cX^efvk4<!5MKUqJ-b>QhTGx2jxe{ZNzr(rn1yJzhTLS64K~vKzN1
z##BvN5wR(Y*jO>sb)V2+!_0!Agxg9r3W>LJo?T>lK1!$gW-ac`Fx}lESlaE3%d7QT
z2@F%B1#p>ty*ggd%~$6kt%uDgCe2FK8fHeAId1}3TyGugB+{MYo_91{oUUM+WAJ1j
zjq9;_mF1n}H_z876T(bSB~u+OQDhx*cWs@n-`aQnF(|ND?)oI$j2d2n7_At-@DZEz
zVe(Yn8dhpz9vO`?%#+wT`TWjD=AoVf|01>9=(nrS&IRxCO4i=NMsnMxpPS@QQ~Mxe
zMyL%Xb~~cAGPEkD*`Nw8X}ev;u6^C(U=6GBdio6}&TJB%8jd)Q=EJhwGER|~@6PZ!
z&jp{6B54&WZwDqcUjc<RAX!cIhIujS9vAFPb}33!X4TDaJsr{~CDf;~A9uz!HJKV^
zR52{S7y!`qvM9q2C*|=EQk81F`CewPNMlqW{$^HUE_L-(29GAY4kr;YclC&~Hq&C$
zKM%<>wyfI7Xg=TnF|c_uJW#%tkmeW-^BFo?vdJ-<T$T!Y7E|XD{ZZuUL}qESn`m-k
z|5fFt{*z6W^N%I)OWi96K6aV(?-EW;pwiDx#+x$s#6@hZ?9fYDh^DO1t(8;I7xtaO
z12WNaMOrfFxJnqokrzebW~sw=K}~9_FRr*f%j&O>wn#v1;fhGf-exiog-!MF#5h3H
zqt;F<n`R3u+vRp8D$UncRXiOa6cg7Oc}tdBH9U9JRmcP&ulJ*)l9acI--h_^=91ci
zVnjRGJ8PPfhoJh3l$3;kuEACtnS_j9dpq&bXtg4#wT`@EU@Az%uD0P<JG-u6!SXeQ
zJGCHTv6=UyYy|Ug|ME!v?^A!hD|F63R05i^Xbsvul6D@D+nk20wxCZj$#(rDRV_bP
zYyda&({KCEW^eX5;|T9K+Rv>f?m8j27wo9K*f>wz&r2g4#ABqVn|FV=Fs!`U%o^F;
zzE^{yL6Qw2k)H$q&|Y2tAI*gNouyZKdF1>sqUh;CxkmZ+bb9siab<paF8acmq^BY>
ztqR@<TTjgky-_}>88XAk0s(K@W?2=9+Sgg1jF!p1AbVHzRDB_V<Bax{GzKk~pLuR&
zR1Wt>SIkYta40IhP9;yS|Cvbr;j4CeMS<|!#4)+uZ~Wp{V-S<MhCz9DDhSoQqSpS}
zmg+LeN0$na+Jo{z<uqf0lpAR!yvxTY2(R<C0Vi*CYTbZb(0PpMB#lwE(6@dhd#iOY
zc2#+_Z8K@8*Kuzv7k1}VVn5Eu;rCYEmolfq8P6m1YL5RwEp)Hg4erwv@k|=xlD>)=
z+xn}uag~^KKA2gbDe$cN=p+#{tN9uVN*;Y4>Zb<R4!)*8q1@=PV;1_j4nN$xa08**
zd+x@q?~QpcRdjB0#B0-D=2MT@y?s^*{~VHvmJ1#{amYy)&2;|dqpEMD<c+NL{AigO
zWR@bEI^*awcmDc=xtL<_XwSjm9CR9jnEjNHr%o5tU9)!LuDoTCx#enBbDBS1>K=H8
zSeX-D9-psSf<Cjnfc2fU@85sg-Rf);sImVhz=Alrs$%-nt8&Q3(&I5ootc1$u2irl
z&_w-ujMYuKwWGq4RS8a-8(w>Y_+?Hn*>gC3xsXl?O?RHzs~`1jT8w|rbsAw3&^NNE
zLxRp{yWR*C;No(z(XT@f1->64U~gPEt+coxPG!J=5U^Z!&l$nBSGUL(Dnf5?EWTn~
zfYhDcO!m6$F<YBrzO8qKHJ*b0f{E*vk9u*y&ZDYyvsOptjbA>)IeRkh&l)Ix&6f=L
zXbKqVnA|F7Wh~g`!gY6-apQ=?{!<ULkxhJw+Fax<Q+Hv(D;_Y1nw<&TB}&gbj9iKv
zI@5y4cULw7ET7S9-C&pEyu;XWA9Q?Q#I;@D$J4!t>$D~W2rzdpUCD!Cn)qJpMta24
zaWxm`PwI1&V`=Ev$@X@yc)PH>otpd^2Rg&QIREf}T?hRZk~yHwomKfyAO2c`{9n!D
z9(NygLB^N`>O9KPO%~5%CeiVW{L&_GZEo9mSu(YmN=Yk$!q-Xr-lEH#eA>TA2;aux
z`geb!C>{UwqDz4k;d+C+fgkNqEGAY;?73hvwexg;?OO2tNY-kPD`(ppbp4=!TgrE_
zbD!CU%6Swb{bYE}>JJzq3i?vG{!_R3i$-Qnk=t^pu8qC`AGOa3zrY)IYhmWMYb#=|
z?La-TON-~Vo3edZC(oQy<5+2_E&%R3d(xzut3p$ndB1Rt#f~b#I;mMb6AB<eP^>>=
z1Btf9+{3qTrSdo?JeK(oB_8Ev1!=P89U4{1P29fMzyK+@IXM}hR(uAK<nLY3rVHb4
zpusgJU%CVu^V78?EL@36Wqjb%Z!k-jWcol3Oz&D(#y~f6Eky%!hlFkX%lcGvGF2m>
zH%$CVN6@a43zAaq2N$6@!U637AgQUli)0u3Kuu%%?M-R)Z1^Dm7wxOnGI5S>2hQGP
z)vSEO3-Z@+hKuQAO#0@V`(JgtK_fKXvshejJdfzV+NpboITmuXh!)L-&u^d?spna&
zFH}OttQastdbxf81!l>?H5>z5AN^Um<PRIu6lg_8WHTFqp5()oXsv!NALT=K__<?B
z6}-l{^ojN7AGB)6qk#{Aq5(Ppj+nYg9IpWo0ol0wU-SinfDCP@ulTBv-OGnA*}R~S
zF;CIrBIh4vSQ;);!pWQrSC4zSa)tpXGoC-_?o`abnz{8mal{gMVKC0WxFJ%tF4zDH
ziS|c$5*F`Tl>X2n=yQhNs_J@ai5u%}%%AevVBi@#{J$v<G?reJN*mW+0AvA*c!1gt
z&}HF-a}^+X3qL&UaHIo&L>OiNxd(u4*K+@1wfMh{)#49>7VfD6MZghF{QXd#8Ao{5
zk)<_?W7GKifB)xa{wLem|LGzdb-+x;>?^B*<Hq=}7yeILsQ=SNNw@I<i`UGN2<v~H
zIr#X$DYX9E?SYH5J8+C=?#NNE|9*Ra7TLe(#{R1ez(xNE>yLE)&CT|Yu>R4kf0p52
zmh2y!^_Lv}G4Ovj(tmctzg&R-n_Pz<6Itqbcmh%?5AW;$&0+fgha1s$elc*Toym5p
z)}bi)^tebdn$fnKD>Z`he|6a2#<n?Cwb)KUo$Mwo0$fI6-cF;<e&~-*7Z`Bi7hc#1
z*Y=2-P`dGF+jMi^+41$-QO&|>Z4=_K>o)*hPM74nHGsMoX=G+JdLm)3_O|BdDcGd%
z3B^e6kLmSCu?O2vScU}Mwe3r5o=MCEF2u4j-m>oOz8`plWgmZ4^V1YlR|(I>81V@e
z#Ud@xK_7g)`p%{bI$=_zSFFiA=nf)D9xVdTmA>0e>ow6%fwD;GvPz;Mjz_OcD9j@V
z5whFOJz#Z)undv-G$@UJHyMoGK$E98f!NO&lnuOR5EBfS_aO{EKlT0mm5e`aJ{a3S
zD_`EV*^1p<cTMq0zICQ0y%RC*zJscQ7iBGT<L>$ep|)WAqW){M@~T$Y(cY+5f5#O!
zXN!>2{Ri|qi!wGdt-s##-&q+gyCcKSiSsqH<s0Lebv=R(Che4VvFCt2Njl}@>B(}c
zWuNbSkR6M-t68l}?k}3qHaKCAxsu#&*A3f28H_r9y6}^5iH+J(UHB0o?;T?WzA$(v
zd_r%Me5#=N4ArZ`REsI~EXna&=gB)loWhTmh;E37$xqVpV3<;2`hr|vIP%<Afe=f#
zeL220HUQ4~&`5Tg&`>Vz0qYK~RI{i4d%vHYouIXD6e2d8wk*a5zXJi;*$!sAtc{o3
zwVoZLKDv@812>+LmdB1+fWYJ;tY2)pAyU?<qWYEJW+PHPAH3II`VWEv#2KoYOSY^m
zUzu!Z;=omJdfgJz$?;w&VTOcYStIv@rP-QEV-g`~J17oGw@JBeA{u@FIvs25{oH2K
zBy^fPXo1KCy4Xna^9u>PNWccvNJoxBYO^8#y%gQX%7h>(EpXyb2z!i`uya_z^V4_5
zt@(-Al>Lp*UDu-zzu{QHym65St1n)=0*58=_LrhLF=6t4bUa8P4@1Ex@jP2_858QT
zr<`qRVfQ~?&M{r12=ln6J#YtRc6@J2li5=BOOREqMxc;6;JQ_<cXKdK2Wp~Y%c|*l
zO`C&Y`Q%sl{p)CjcB(ghX7>Qq`StDXZ`4x0u5R$tI^*D@<^H;9S0^gjLo4*CmfSd~
z|Jd}LcBxMXPZ_~eHzkRxu%^2A6VLN-Au3am$y|hSq<+r7*UWp|Kq<#VfK&XcMy8l3
zcTQl9v1{XUS#y-(Ise>L4}X<&tlZfP-;E06&+pdu&ahjV28PD%>&@r^u(rf(-*BaL
zy~wGy?v!NGkd$zl5<wp>#x(_INqXP8;{54;q!aymKoqM_$@5#I)Q#;;sLH0kfUOq0
zvoow)=hBb@wI~Uzmv-`7u6AbFRx5U9gGI#NN>h60yL!FYXt9<!d*F7MRB@dHQ-s!3
zHKcXhVJP?h%CsBYd*}Bn$}20>#j;wGtDJi9;M2btpt`xW2hhh$7|meE!*(XiG_dw1
z{RGp`zLE>2c_Yll<;0PKG&~$Ymx&^JA<>C#)^Rc`f#hWe(DWCU;Hlew2`AcAED6FM
zly<<;15&mted(q1!$N9;jRmVes;&35q4V&gl`Hx3>O!rDACK0`tNLs+T~;i-Vr32<
zGga`qYCD-rIgD_kcYB30wPcUJAL)#>03(L9SSy1Na$my>OW@!jdiRtl*Q_VbLxn$K
zDb=t?pgV|2;uO$F^-nofxrohr>D`ckY>gN0vC`_+&3X>X42U;wju)uVc`m(I&S=5R
zHT6t)onwFr02p)N@V&(BZ2vXh{1DXytgfJ8Y7NMxZbn$dZ^F-mi|y&?j&{e@5;{K6
zY2M50Xe+Is4{p+<KR$kwN1v*SMh=QEcE*XLSNhd>$Grs0tk8%Zz{31YD_0?k%J$dy
zH_5J_4u3^wG76X!{2KU|6>ZR!Wd)B}&dh>${w^2qrl_FGoPO$n494v4vo4zA2CYHF
zrzZ+bqt|DJf1*`xTKG)bXf9OwchT$)E3+SsmTGku+vNmJ+=7njmRg_n*7ORZo;97D
z&mxz)6SLg&%cK)f*ey8xZo;#k&p<0+x2>w%8Oyy)h?4~BXGTtd+HZb%M+Y+!QQ_sP
zOlbG`R^S)nbv40R6MoY_=-8Jzgx)e7ym^i@4i9}Cf_-?aK&U%Ogu6;}^T<gwwu^Ee
zQa|Si4F406IaZIZYXfNl3MU%kvr8-WEYn`i-l&Q6x1Jpspr(j!%YIWXWt$U2nP*<E
z6BWiCs1fR($F~+ozzPFaAww_Z{1h|LV&UJHv+yk{)1t+{aQ^G*M1$x4xKUF;7f%Z?
z#*)@BisLdmQiA)WKbY)+8KJp0)DnORS!Z83ulL=m_R+Ut??f8VhCO^JNb17F()dV+
zK!r5v;jLU^Li+d5b?8Lp?<?9n#7AF~|M*bemFTQ3LFs<M0}A~ZonO(fbw~wLX(%|l
zxC+F$<S#yokZbeHnfIT!v)Pz*vfT=r@*I#v2XpfVY&BM-A5qx#_xVq7oqP_@dZvOg
z3Lqt$e%zlQ#;|ddaTM)!I?A?2GO=;|1PXJ!;(I+aWxTCygXqSs&tHoxpE>f+ly2DY
zWya3hf7=}(eEb@Zz|+ENb)*pD*G9_yfhN^>K^N6%(myf9R<H=mt~x4a2p>CWn}f9o
zH30o_LJf*$#_k&MCt(btWZ^q8ZV?+5uUt;Q(`k*7Gv4YdvW`@P#)Jviw%9&iF!H$~
z7q>S`mRjRrzk*M4btpZS!Fk-sGRPZsU?LoHb`sw)7qDdx_R9_E_)ve5L!={Q^>N#E
zS8E}j$^D80H?vO10b7KMo$Kd*u_r-S6s~8<>^4=Ztszca!{VpH7qeD>PZdsYz|YT~
z1d&VDS~Lw18iO06BmE(d*>ThMgF?F?kY5@uFC!)<1Rk0d!AC0tULg@CT7&Qjg4{a1
zU-GR%zus_9OgzZlKdji*n#svFmfJ7Y8t2aP(rp16+i}z*2l;t3Q^}vga8%COT7b`y
zU%iY;z=VEu==RhPnn@9_IJ25(Du;Go#vA&<eJ~-h9{e=DMS1CxgL}bF5HUyc&9P3l
z0Oyuo!KUwCO}hchIoWbJ&)@<^;);F1&kw*zr{q~z%Z`)88vE6Zd`{uJfcBQ1V6n9i
zu;l<C5rH&<+JdB9p9QTnc;j@GfaE2=cyJbQ1D;GPP)QWjk;yBpo~+x+kssw3BXi0P
zA&v$5h~88o_Nh4pM$2i|&{W&ne%h<!Mx3<nV5WH0U4hwN=TDm^Tb2lOUvrq>Nu^hF
zUk_$02bN<A_7HdTRt{l0f^qfrh#(H(R6^I5{fV?mQ-QKDh5D7=1tLWuUPC%D!Otn=
z?pMlPSdge*V8*VvQotk*<%ICAh;uxF<~T-=bOt>Zm^$vYJo9+r?m)~7BcL_BTQ<?(
z<0dVcov0<-I6;XZCxn`zZeI(Pmm}gaV`;85Eqm_J{+8jIn?ZIUFpsYYq!@mh_D`fC
z*Pg)nhc@s@mxnEe>Ze_sB2O%iT9LEIwNr*Kxpzb0s$`E2$@46NV`Yzjn_!fQTu*}Y
z(eZKP4+pZNZ5^*LNS1xlK=oq6NZ3let30V0mX3T;2?e48j2xkD&j==_$`2kmp7tA;
z!@>yn-enqkIFLVjjG8+XFC$<p=VBSHp2}D-9y@;pv9&Wx39g<ngc3BFXocZ1R1}O-
zha7%upeDNPp~n)hEy-l=r2{#|76gvRDd1B(xvWIj<fNMW7@N(5D%#3XF?7H@)?&sm
z>|%@BVL_-W%eG~UKU9e2SU<NdPJ;5InP<PK``Jz>yIifg>#I5Pq!egTmw+uB!9{wW
zHqX;-#m2o6HS5OcT~VTUGC5bpITlb(rbNA?c18lG4bPvrKyns6Tlp45xDB$0VX%<0
zW<`eZz1aAaPWwjhRK$$P*+jt0sw%7M>}UfesK;{8c3{PZfjbYASL~Y~si_wJ4!H!O
zv6e_1h~lpn1Ejr7KtbjXYnTHA2ghwlGp8&Qk$$EMbJJN?z-HqP{oco)L=R1TfV<Uc
z<Jj_Zx>m7ZvFAV$+EaVhfKTf5fN2z=vEDB}|E_ygQ579{?t=d&)iPM3xy?a_=wXIX
z@`S*nz|}Fz@eI6<plu?2!sBc53nP0Bf-XNk<<+^yJC3Vra4#3mpKkhEvq8Q#)bUb2
zDv2M2(Z7L48e|$`I0sSVEKB<cOLo<=g3yTlOYGs8ewIO4PT53NPPD~qB@MAwWwUu$
zCE2_IzX`@t!@}NwA(AOp_3iRS4&kCUW4kPd^UbmFd*Z8pBj+ie_@pwQ{rUM(EFbGy
zE29#0oFq|SuNPQb>rn_GD2QGJ$&p1Y-3(zf%u_wCSVtA%lM=o6)XRt}BeFz@jE6>e
zLobonCvBDg((xSeY+`y)+E(JCYa;gQ;U^uMcG#!#@!kFlijFQ%%zNrwWQB7=zvGd}
z(ePM=B9Awb{_xNVQOo8Kb=vsDUrg}JIVgr7`vF6Cvgq_F7zbr{jz<r#`x)?+Hkv4M
zj$O<bDG6Jl@{m67E&b5@4r^?afg#%w71JeYt#jlVdQ*}euoOo@D!6yWnvMci*DuAy
zxp;78qQ#QJkcR_;^u0-F`ye#*yE@P3I0>#?3RCL6o=@Y>Q{1!aCpQZQ_m-{tGeq5u
zr_~aQ^9NN3Xjduqg_s%%cvwGYl%N`>2xuLQ^d~T`{`Yp8_OVw6^UE3CW<rI4Op6J5
z%rdgS7V4bS{mz>wpB^*Gb=eJ#fH~Gw^Sm<D%QK&%OrR{0tkpD06Pq>{qhvL^qab7r
zvs`U0#3D+-<F(I}Irw3nOEY+Z*2uzdxuoE6EFOV1*;W7Fc{`0m9anl>Z5C78y*=>X
zh*%9Xj%j}_`@J|=-M`82-2C}5Zx?aZ09%yp*)akh+xaxeg&LAd%sB<Spg&i>I3(v?
zI4r6m%JIJL^3fOS;2{|Kc*MAS$(7j3)d8ukcbyd`4GyTcX84AtCWM?*&(h-L(}CIb
zl|x_+)Oz(gfa?8}r4$ZCna;mGa%OpKq<9%_SHkd>Gm(I;6#^S_(j+qsXiPF8N8Y?b
za=j=|wW6`RBlIom$DvE3!P=gz2Z%mx5l?Osn3@2|r!aE0SX}(U;eKt@aDH5NHS&ww
z<zsd7<PlrM;>_!tq>p|pe#Mkud@;SGvsD)wf$=O4udMa+&&a0qVZ|u2hnG^!U-}(|
z7pXR?*B~-*&Z<^H?K;>y%Jjj@l=_hu!%%V9qS*r042;tp6nJr*fK62N0<B|Ag616e
zX31^d5tl<C*I9xy-|_K}zWVaN!pQ|+`{f+Zxn_ze#UbDKS1OE0MhKx;^zPp$wj3mG
zW}h0rhfm5t=EuS!l<$<`DF49v`Jt?{@Y;kqPaI}n%ys4mESs@}-^dan*Hq#0=2CoD
zsW5ez2hoelt>8)wzj$(JgvnH4B1wNTyhui+V6fP+TCBUk^*|avU_d-BZQ1HPSu;@}
zW1qL1J+n7C7%WzlZ|K#il3YxGGI8b=_w<3)O~Q+h`XcUN(>w{vE*17Y?ngFDWGa7R
z$U3<4^7=>yy#u0IM>Or~(}d86$9F#d_L&}Ih}RY{mb2Y|`9t)m*UogW*~v5oY&DYv
zXBdqI-7`rI=yP5TjoFA9;P^pJ26#V}3(D;Yu_CQJfy&=gw)^0&1Ve4Nd=HH{R)7OX
zz>)G%c|b<yi`46h#GFbDv{ZS*7w`#hGHQQ$D<`%2c~BNI9S63dyEP-ah0HNgiP&On
z!d4Y?!y_kO$E=xy45Ywtt!AZg@CP7^k4Nt7FjOT32_e#Jv9niaosTl{g=WRU8g}<#
z1Wb~QGAOM4?3B_Lr(~+iO#<3P`)Wag&q8gULB2yjs)n3NIOl>ev&SI5tr)L4I!0Ah
z2{kSyY~QL7B$J67B;w2tIXh}NDM-}Z9c4aL@AVsDlO9Qy9?ErX^1kJgm|f*9&9W6k
z8kSB23_D%Vy7d5Q!G*<D;v$t(Z~m7SP)7rZA@Omm&&Jdz^_TW*zNUJxPrAD~z!tDw
z)j2kQN#RYzNl%J6IIG%dS4Ic573qXUFL@ol1&*YA=-0xs?8>iP=E2^)HGxOF!oO>f
zP?wxcy7<G<oHKg#?u<-vLFji72^p2+4PBsiVFTg}-c8Ku3xnUIx)jzZu>He2uD#iI
z_@g3(`z{~H`xO-zKY7oyxR8^@uv3S?x)sU%;PR&Y6%kGO2h+~bal<De`NY22j`G(b
z<I-!=fsL^IwG5H2dPKswn143n^l8GI(C?fk713?$WZPzP$BlD9f<&Bi{wT=tTmH(+
zzg@@Cdn<N=9C?f9Ls7Eyo62Q8SP*3%^0p5;RW{z7V;6#-?B1J)mykacnje;J>IrLp
z=Y)dPAM_tRv@#*O_(+{!=*^%(Snc@_xswIzeGpAQRjRsQ)uP6&STE`~W*n)rsGdm3
z8GG;7>xQk$9h@(Ont((*h`0s2z?lTi?hTZFq9VB1cL|>?wagzzmMRij0vG1dGN9yS
z_?qQ8AOVG8M|-=n12&n_3;m>zO8Dsp>kVFveHi%)pS-F7{MYQ)7$w%Kbr7X4nFMt;
zy|H*(>pT+r5Yh5!^@|%JO<G1MtYEuYNuYVENC*Fo7akVqtJFJm?$`aw5VGx}L=*MH
z>cHdCoXweu(V&x^&QZ><7<U3%$Khq{8rgTAFQvqsn)enW8mMEM7CLA4OYc`s-(>JF
zWr$|44=KaO);hhf+X#}yCr$lH7q$zQJ^QKImXW&{0WHho-3?;}Yh8Qc^}B=8pACKI
z{l&GTi3h8u325bxhvS~g-(T!1O^)1rPo`=A+>wYc>6OXY>FHJoMlUvJwPgk)x~A9J
z6VaxAktRNk)aKwel5hks!sOv^qWjGCA3IoH>&-H+Vvw?!#_im#^y7vgBQ4taN0v(&
zq8++FDnlPm&1ezHn+r*|I**tbtX6Nc;uH4JJz{^vYu00#Q_3h<C@`sy7BzaF)avD|
zDSv%KOAG<E;KhGa3!egbLY?6?OV8b+HA2peJ10uH6k&eomG-Z+-pZ+&N<^Hk=cikp
zB4Z>N{UW)uow(bSGFzJ7EZiJ<R%LRCkY#?$)tPbqE#|VeC9HP&dVkhfeM50^CBf)z
zJ`VxKsRL#Cj~cIzE+%n<Wo-~g#)37-j0ue(Ak7v+S(F8+d8&3sI+24t5kOZrik2OU
za>}k^f9J=ECC0(W%|BLc*~pmZuzXAUUzV{Z**XZouPw@MHKQNMh4g%9cr;Y-x-Pi<
zxur&d%uA#B&_8-u*>W(I)>_}ywUN)yk(Z&j6|@zgAn6!h{91;N;Nr<SUG8%Sf3nm%
z5S68Al|vu8PU+&4dK)cau0loZHMs~mz2bIO7S@hAncfY#)5Y6c$OPZePH9WvQhOSl
z&-krz)fbL6H&ce5W+oEnVx*mRezo_vTb~-z7=@`7=7%S%bSiCUy`ziQZe8mvH>oh!
zlT2I1;l9wdwi>VaP3u?*TdT<exv?@+sIhk+84m!cr1k-VY={kbw{pH_1ZUx&yg)52
zT#!%p*x%Rc0bW?mzjm5(@hc!m*a#MKh>c$A6m^yV`m)c_<zy!=MEadtG2?R<qX*_<
zrMnJ@z4vi}oNQ{942p=lTMAZ^xJ6tn=rHJjFC;x;HKEMvt>5)sOJX_cx#n9IEal3p
zE3Oyk_wNiD22_yVR+0ZD4nNYELA+OHn^fmJJ8&)7`<Rj`d(^Gf;bngsPxq9K;FJjn
zv6o4(G`DYygzvk~6K&VsfVUH~9~fyJ(~;O-hC7AkVIoa3JBz9>8C~${Esb*KuAcmS
zNv&yDA3P5r7ilHRJJVv9;_VZk$P(CGtaw~~{Y(HQp%<i6JQ^+3db#tlUTYHhqZ6du
zc`0rob$AK3?2U-LIW!*}GM}wpWu8IH%A_Ls5cZ!2=Fbl5AuoN3@ZY$FV5gz60$KFZ
z2G`=Yipx97>P$b|YM2=bEO%siQ9mN5b!*<~ST7>G<t6hOe!Nzm=aCd^w5VL7CYQ4b
zp2G|0o+wWNOrZI>{z6Jl6O8D|-=$AUv2nC&d46mpmlQS#Oo#)*OpjGz1HjvHoge4q
z(r9=TLqi|(8ig<NX#SL%Kho@Kz|8v2(?m{CDt;l+Sk>q!KL$7_rh4#6LTJRn@=Mg}
zWseJ{<qT2t+ral+)9gV^f|dDT{$=BZ;Gbb69sUm8F1^x(G(4tY>p9<Sme&(7Qw4O!
zrcenS-uA?dbnvCY>!i$;!5sEo2O=>KzZMJC=H)Pxrey-Ewaf`jBL7U&ZV&&W2Si*`
z{%f4tOTGFSwfEU^;ddlu<*1;X&YFC;(D;wNTl>QYR*oGsmS+4!5lC@@i_Wc~WZ#1-
zm`-IvR@E%Yh?dA6R+u%`Onk07&jPdw@k2V&AVINDB6s+uVUyYJgATvtSZ;-*c}4Ii
z1N+Jo2!!hu0VCV@vDPuB`rS*c(H+qZ`xB;%m)+KG@`)GXlNRQwDwiGRbCm76R?Xzw
z<inhFj|XM3&dy;KCpu6oOaIlv=N774PL|jy`FIYQqJ0#i6C2Jax_VaOnfivT%8Tlb
z-dS3&Lh=OIATDco9ys7(8x~{Ld~;5-sn}Z2&#+O8L+2MHwHP9T_RB3<(~a!tY;O`=
zRFB3R+@BrXkIhb*<+0q%5Se(ti4S8hOoUw>3p`cpTv5r<N)Ev$8PWir`sg5^lR$vr
zuhlMp6$&}(ZMb!skg~w@DHl6lkVr5+m*9%Sj74ucy6V>Jo*#C~c|@m0wbkOxIin%%
z#>+OB`dLOd$zb#mI_LbVmOxPTh{yi>tW5oB)#<|X1AwB&A7JOt&%%yOxDh=Mpxq~z
zeP+E8$19m`R6@7>(?`h@g+}d6<sWn&>{o@zRF+RU!oHMFX1hwmNd48~^W$VCefjzn
z@WLbn<rj1@L?0X%MwnW+Bh%{H0AQG>trEHG@6Z4;6bfOw8CvIHJq@zGW;$^)XPB@A
zNPHfL7Cr?-b0H?Hjs=>rh1FfiI(~hMfS(zZn<nqE{Y2sH7)sjGQpf{r>E)Jw%gN8r
zb?-lQJ^s!ud#Qf-1uYC-Dz=NBs4x?9V|%>gHSKtv|M$nQN^X^v``%k4(x*-+Dn4Iz
z+zh*kMg-GOwnR_WJEOZ9-eTFdnzmXmN2u4k1<N5ncm^isX)TlZQ@Og1o<J(2JoN5z
z49^UNII+VlWA7G5>bC|R`+<x={Y_7>ps1H4SL==|9~78{E*Hz4I(R(>_&{CzYqEvC
z$_0lxk6r;t=SLz(?*w9-z@|MtjL1!}f)qeBdg86^R9(BS=m$YspQ*3)GF;Ux7GQ`U
zdLi37NJP#UfqdaDUV=1)%B$rZL9<OZmW!g;{CSENS4+timN$;;fr5@X=J}3lh(qf}
z3}ODo+Z(=94#nHSAKN0rkNNaQ&@I{hatGCM!#0+i4}{%JfMc&t&j04%zC00YwgXB}
zh>&k=M<r5obvIfN*2P2_Sd<<D3!cV+Jl_kefK%=M;o3<{Pt)-MgOKoyc`&ok8K7Z5
z+T6uQN8DyPI(Z9NB(-t4*$x4{ptt46QhGrdbl=@-(;I_&K{ITp6Q>GataFOBX00<4
z;U9R|F&XOF7RRL~RW73-ImhP6Qj98|_nmR?)aL(zj7dbG3j5<`>(%5uF#`Q+rWl7)
z5lR8I3u}^g4B4W^I+4~cU$Sf^Pn7V&O36!d6<*pFg<x2ul&v#)57$Ca=J=!=<{0*v
z8}Ezqx5(PqKqQ-o=P;&IMqF$nR&exYL_51QoY{cK(WiaM0bmNyciZ<rj4lrEEboeF
z<sX&Wc@f!;hKFcr&C0FH<T@R<3kB<J0oY&RW)B?$#Q-!`TlPGRI*`LTpP-;_Q@+-6
z%fzs=;H(qsf~a*EEz)jCZXziSxXfUNL)j#7K%ynN_wBW#?Gi00fjaKNY9j*47{xI;
zb4~PDbx`t_%E_9ZiQyyJNGz;TD;dp0?Ut!2vp?2Q1wxo&?03tv-PAhHw751#018zr
zN~lz55%4t|-E|nOz|43KhK;H<wGT<+#MkC={+=op>EIo7w&50n)*z8J@Wi~n5&OC`
zUu2KgW9UA~xZ|$x7b2zz8z3dFl1(wMb=iAX{<p}S%|MnxkL}j;pP@%#pb(WzyJ0NY
zp~$46!0^Hxt7S%2S+w+4u*xh~DgO<qSJ!^XCjOg3wbTi|f+;^_7Rhd4%lDFA3aIAq
z46j>d3rTEI2j%X!#vWXQ@QQ(u0Xq^3Y1*ipmd{YNgtm^^1-R<Q{I1=40ojUtg_#ct
z`LP6ie~QTW>}T6io+=bs%Kn<iz%%UaP$#Tw2ql&X-Wyg%L7fn<HrYfhzUgz1cj>d`
zHb44eEaRyq$;+<qtEH-t<4~AaS345=9V8c64fQ19z*AKC^vn}hroE7uy)gdkvAIyo
z_E&O9*!!6sUN^&R-g%I&^O$H-8o`DWa~jPF&Ff&YW+ZM?;}hB{fmM0mj;JAXFxmw^
z?xHK{S3m3Ts>$u$R3Nf<>uf9JTuM+^eri;+$$5m#kr#po6T)Z_UDH=8_}K`<=tUC=
zjLOuxjK~_yd9^bbMab8O&jN6m+UCAPdUE(xm&8qI*}b7a_)zP<YYj*M!!}*NS)bKj
zJW17sp5v#}$Xmr;{Gn!M19j&we{X0k{IQ0`QBTst>CWRx&_TafVQ6F{aEq^rS(8^l
z3^)Ug+*Xm?$@?gC<1jCq$S=t;sd~ooo~qnWMW^}s)k`bz@@ohVd&u(m%XcJoOeO?$
zY{C$Ziybk^41Gq1qtKc*2RoMelNE9P08(-kpw71mTU1^BaUsYi(q`dtMI3rSWQx(G
zjbrJ(<jv*x2VZrK!vbxH)?6}65|q32mK{Z3@uTjP){vZp?L&MVpBtUY3_Oc8NoM#F
z*&Mvpf~j2RditUN>k;aQH~f>~rG>}qkv0`B(}p!x8aA^hktj|&zpnb;?-q4@Sug9R
zeWqRb$*1%Uv5r<uFk2Hl5RzU%<5cpZvr5>EjI~;kmdQl`Av=?TxY8874*?5Og+`3_
zmA9T-7)Lksz=*E?cr=KZU}>yjq`H9aJk7%hqxeLdU;aMrHBr$QGkl0zVNAQYb<dxq
z7EjG4IgQVWcLXL&PLn2xPe41$r3M3|gKy&yh5%cp0iw(s;IxJIXZCP;Z+ADaaf5_i
z`6NOm(AJU)uID9Pp5PX#1f5T6T(!#qTqx5Ay<fqPvS_X*mb=8G1^g~QM8u5;c=(`C
z0Wa(};R|=>3vUd-F+O299){vf8F$pfz!AsREQSD;8qlZhZ-qx;L*y;i6qP%0_}KLa
z0<#dO7GVWA#I$6ltWP1A2TWAaoQjJ0&zSgx9SlF)ZOnKl6mVbiGyBW=1#k=UjRZ@>
zFA=t<$NS>|gTHKx%jP7<?9jk!4KS)}E2}x@Cw4a0Oa>r#Q8va}fAdOGlS!k8&EY@@
z4UajP_NvkA#Xlp0X;<J02^urnUdIXM8KKwU;}(B!sMVM79^!ba#h=d%tr>y9LfdX#
z1kdDkrTY(%^FDOkn1a3}Hwg|pL+4a&0^~(l@L?}spVEJrK5*Or$Mm%+b;s`n4F=3z
zAjS<T|KyMTj0fInJq#F-GEt`qecYyR0kX{vBS2Y<`ayQZr1^!rH`wta!yt<fKqiF*
zhg^k4f4}x;#$aZj0C4y1X5an49s^EpR+TsfI)2M5zXM7@^tV8gpbWT0K6}I!Ve(%H
z2sz1G?_FiV%>kzj|NF0?OB@`-sr=o3|KWfCx?cMe9{!tfA*!fX|9J@9{aJ#U?k%f&
zu>Gg+`0H=<x8&tN{*3;XckO?l#lJo^!~|3#s4?LK$$y;dpR0(-Y2tYfe?7PV@87hC
z3ll6WIgz;K_Mh+TAJP0Hn!jnrKU(uQf&BaQAJP0<7JobXM>KyE$iGkj5zW74@wcOY
lMDsU+{QLC(GtnHf!+t+~u<sk=jR$^I6g3}KKClY^e*oq~gl_-<

literal 0
HcmV?d00001

diff --git a/docs/figures/widedeep_arch_new.png b/docs/figures/widedeep_arch_new.png
new file mode 100644
index 0000000000000000000000000000000000000000..62307687d0dc3f19aa6e382c3d9194ebe0d89db8
GIT binary patch
literal 96559
zcmeFZbySqy_cuH<gp`6vNQeW1l!DUDARr+n-JsG)NH+r*l!73gA}QS+2BJuJhje$>
zd*;3+zR&afKJQxZpYM8D3m9gu>pJJ`v(Mh2z4vG1rywVB8HXGP0)bqXk`z;fKrqoE
z5SS_!CU_?zZgCa@!6`5m6;+TD6@@F<SQ(i<H-tbW{lX$HDJW@>B3E2l<6c6ruLcvk
zTuZ)&ke>axbDN18n?TyvEAgoWiJJahUti>9ofL*L5=4(b2D3lLomuZYIE^b$zr1cd
zK~g`DIyspUwpt!qiE~`$of#=No)&;G*yX6lUSWm=Q#Xdc+$&>$_H-h0R|JW+j`S76
ziin^ydh-$;qMSNY+xUhBBHQHHeB144<G_<xyapa{2}0y;Ln9mz;kEw4u4>wkt{(zF
zcwD_4@_Y?lPb7*(<eFHyv3F>ilY#d&dNQA=B0sDRga!r#7WF%f77fuOy46If%QpXJ
z(XZf@FDuc<z#6oOz(5(HMiXuQmjk|!<5ydy4?`0sYj<Tn>`mH)8VVFgNKrfC6n>u@
zGCY}iacK2RGD5E59^w+O_#cYbEqI$vRLO?>ztulnClY9WaGZQZn<nXgD=jKw>MD1$
zTTWW-a|;)RIu3g5YF;+s{#ajA5(DQ~@`beQG9=7Wf4;;U<rv(~;kc`qDu315;Mi7)
zCMk_q;yN)$vxe#TF*_F(TeTSZ?caLv^%n_#3wH-yANC|{Gf)2PF-jBLZQO2&`=tke
z+Pd51|9<KRsgp}{pIGB!egATS`kzgLsA~AIL~3gzc0_jbL)OWr0fK(ARGXs*sbo$t
zRsy7u3;}i)QZNtw4gn6LM1Su44GgNNKZb-)Y^1qD->OC-e_dT`nz@&KC+j&RGQB^8
zd5z_H0SWbwC^DVfnxa|wCUMWB+Ude!ZQ2lI<0rDX5f(`L9)zQKEjc8XaafEOdIt_=
zN<>ph#J2OgdQWtt5KXS`a&sR(x(^lsE(wfE5=w@Eg(Hde?=nEd_1XI{nd&U_@Vk)T
z#u!X6)9Re9$+Rz(esN#DzUd8J#R){>>*IOD1;o5ef|-&r=0sD2iNAYm!&w<1I6hB5
zqBD>YiYS_36kTERmQKVmVO7RGg2laT5moyrH%r}h>9UB`N7j3=&?kIZP>MR+CnQa;
zkH)c_ps{$YuWiTOoNj4fQTz60?DnpJ+l?Chov*oLkI~^aSZM7Jn+Qn$c%k2VAxTIh
zqDuWrl1%X}2RvJn--dKf{Pp|R>y)owKY7}8Pwu*&4DIVT8i5+G7=I}1o86O*42Z0Z
z6l%hn<`4H)X^4J$BcJj)9v4|9x)$>8H>$e$I^C}$4OjGq$E<lUTL_GUHtOsDD8Bgp
zqP{Gy?BNivHhs=bZvTb}m9<nSW4C+p0j)ns#~jD-$6Nobj9pqyz_LNV>rMP6hCqVg
z>N=L;OTMpUUvGTH{d)beSGv3%E#|eo?}p!HzaM>9{d4yZ<V9ezBpWSnD2-%Ba&fYa
zBHA2b#+yw!@FmeA!lEq4Q2H=Od)ar0xLl~b)qt)5yWfXWK0Jgw1V5zp{mqck#|%Vh
z{uFfYLS*|TGg7n@1ybBX(tgGKww>Ht&zxeOikV#bS({3jv9HW!Pt6&j`B3m-wfGHH
zIMa0oCkAU`RYql|SmjyO>nh2NnoO9?Mhqm(J&d|4l*(V3$rzj&*TUArwi$I4=F_M-
zbfPc4mC%wHl&=({NSv5`ZFfa5Ojk@;W+gF*fthiTL4uK0B}Z8(d+0afIwszgTXFe*
zFFv%_@3K95M&cT--}74Ia^Z5*ema|0y{vemVtPlqjowrMlT!C1PFec^v%>jnoPp7P
zn&|_C#X4pYU9XKhjrp6Wn^|Jr?Y3{(s}G9_%D9*B3vE%%e<F`27b1W9_69jA$7@cq
zXv^q~=#*$<jyFXnnk<?RvK6xnHCdw?Ww_YmG+L^DQnsu1Zg+>9t#mK9QFY$$^6UE5
zY1-wS^r*vd?)F^xe97FqS<^X;`RumLUe{##F5|hhS*u>F-V+mj69S98k;H!eHii1&
zXOZUWR(;RJ?%Qx`K3BG|YrSsV#3si+V(K=C&Xvz8bw7in+z=;bAPPs;G1{T!OUsMM
zThU5_X@h~!e{lW2pUJ(zZX9_jawdv9rZzs4o7-|ZynM;FR;y1lS!=P>>3N31oN<(K
zzu8_-<9B~!6;u1x=%-5-)dLmv`)}W7ysa~eXhzeDwn$uZ8RQzYa@cktti-IOb`?B8
zKVUkrB#9&b%4DSTqK>bOKg)q<YPG+;<^JGW`ov8+BRNw!dpVyCycz82rG(y<tcfQ3
zk%gg!uXa+_{SzGvpO+f#vmM;0Qx|?X@{etMTQtcT-h5i?P--1fmSBA{J3gD=F%e7u
zXKq4cwtqZjl6X#Ke5*^Vfxo|U#Pxeak8yb`uU=Me?Fz}L<vYwb4H;8`yZr~*o0YTj
z+xR=VtDYqKgm7$=%P-N6V5OMb=yR9+yrX?Y?$JA$uZ9wQ3$CZGef{Tw=mTcagf|+m
zv$4$a(_XRS<l>$Xj8o+^{&|u7IGLm1u0J{PO%;l*`vdG1v=!V&mQ9V33;AaGU;Dy&
z+_aU4yvo}5bT>+yZ~B(LW%XUr)GyPY^UlQWCP`%#sP?V!t(;YdaZ&SqWEp4Xj?`+t
zAv;N{O80J6z-iDqch~Yz@i@3nLxINW{ba0i2uI{%Y!#<7Lp^889Ygh5>MVj9sV&*z
zxc&Qgd6x7CoLuZ&o?%koqMk^Wl=^VHA~GYYoTE_bITH)P4vT|!mBpIxcRd?8Zlh*!
zcZjS+%qI%PTXXmwHIxytjj{5NlENvqK9?yMZT?^jHKYiy{<+n1<jiZtYvnwMvqWsJ
zwwY~^J5-6I8CLe$qIh}!up7<Xt*0xCE?ZX1mA#cFmv-f-V%9!;k9^O3{lIf|?_&+3
z2BFwiJz{#aeDl6Aj^|>Ze)W@v*DJ3MznY7ah#QLQ1vuCDdK4UH9Hy<km7?J1>bDpu
zCM(q_DmM>U2#+SL4;czM&i2xZk&a4vm9kQLcfg^zDfRh|Lx;oM_Q*PNJ^Y3XKiA5x
z=6l5#wH?(3`O?*9qTLN~hjES$<x6IL+4+?wI}w6k1@X7{7cD+>rLzpa8H^}g**i`@
zS`sew$Pr9*NSv)J^-1!n=6u3QYQAJ{zR<P6)%|U&WB;S*+8fK8+X6!!`aOB2@7vYw
z&?(<jIqyCC%BmxA6jXBK*$s1^9M7tG^d+$;*Sd<bOp|$2%1h}=QZ|xDf0d}F^<1ls
zK5<$7u$|G^*_g&1M;-S|nB9YGXK|Kyt$cjr*UZ5nNwHuJf$jK6eOthdh(lJL@<sQ(
z2mF;si^Ma`X*qnlg3fP&Kr4o~QYlKf<%Wu?os+)he3}bC$_^jAQmVy2V;^5W;WBfm
z(^?s^@?u%1_NHBJA#=fkh@+e13fi&Hmp60xqXLc29NSK^_$6tpeqTci(&pteJ=u@5
z^Lul4>=KMe-x`m%QtfQ)>m9{!9rSt~&U~0LWG&L|x}teGo_w9E;>hOsx0l)<wP)Wj
zP81K-y3IfJHk?==O#F~+`dLTuD`u^<enolohMUz!jRl9n=G>b={AV|Dg)=-p?=|fY
z9n<aq4#=m|ThsepOXg{Qw0u%+Ebz=tyD7|?UJHtT!&;FLVucSw_+cO&C?Fk&8SyTP
zi5VwIDcFf$+WxXsKAOHBH{|MXC{`JE-8-D36!9aa2`JXbBo}&!me%hdb|?J(nex70
zsUh2VT9^YE0&d5xqwg|=j^&+C+%d1|TGWlsc%uDygY{=)qG1JqSbsxxDI-}~2ordX
z1wn_BLomQADEJnFQk=bh2xWkv{qsHy0(oN!0U-FFYvjOh)K@V0M&0x8Z?urtkW1h%
zLh$XB2s^zR6Fm{_^fgQsd<GFw5|xqyzm=ZZ7#dpInpoLox$8`UH?XZGHEbae5<1j3
zR7#O{6+FM!R9W3lT~>zgnUw|069cQKhAd7N)~M$|1f2N5OAAB0CvYc==a#m7PJ%T5
zT)_ukquyqvf&X)fow*>5x~u|R)XK&X&c$+%<sOX?4jc{_urV;=Qxtpn_jd4?AdQKg
zoi!gTtD~bMiz7RWm5nj$eO_K()_ZKMY;1SI6?bi&E$yB--L<s6_3uMYpCe{y`^?7F
z+RoI<5{`QAlc!epc7ilCsDaMD{+*|xlj-?LmbQPF1s2GP`h@j9%RSb!XM<Y>Q19|7
zm^v9g*AO$c05St(2yycA2>f&X|MtoGh!^fuKfm)nC&#^uw_f<@f4x=3*3d@O$^wjO
zCv?7Fe{a6{;olntSW!#AAc}wG{Li~U&_Xx@tY@1hgyU%5Bm(x4(o{@d8T<xXhWdgs
zgC7k4{zknnk9y+wX$t~DK%~S(l%1f9!<S--ySM7rHcAQ~X}GLZJ1SRTF(lI5Cx5mV
zbDgt8?E5<mbULOanD_l-&JwXB`lqux6`L$Kl6^L>XpMTdbVn_+X!W}vaVNA^<Tf=`
z@9<l_aBQibDI4m!{TvI9guwpi9|T=678p##-JOi`zgz)seTYOtZla?TARy5H@y7!M
z5{balHhAs(--ba#pim*S|9J|ixxEAzcoCHS#Jl=GkANB)26MywuTvnvf`SXEt>2T;
z{P%UBqwnJVA4CCzp`$+-qhS~O9~b%0&{$ZjWdBVH1PC}fIu_Q;I&mZ4|GrWJf-(00
zx*I@>SOf%EUI9%6SO4oUaCn{Zf7u8yM=t^bIGg}R8bwX>Uxz^;kgflHr(o1@1Xv0S
zo2b}-9R?^w!hhc>^#7|6|L)%ZuN7hit7>P&mCbfqW^cK2g_e(R;MQ9zVGnJW%~|Si
zuP=vcSygx6vanV>J563=L_jrTd&3@6Ur<eC1jfT4s=M;SM@27Rz4}$<ype1>(?T`(
z<;_hG2G8S-1Ufc0#hT-N^SREnAc=wT<ny!jg&rvCZVuzE2NxBU6}Reme}T<zR^b}n
z?qsO^r04OBr@(*@sh-<oUDqW}KVRQq*4jhG<(dP_ppu4P=O@iu6xcCqZHb3or>A35
zOTXTqxWKgY_TJu1LW)`U=Z?=R%o%FAx;3r8v2$eg54M+V=DQeety84Kl5M6M=j0|G
zDh>D6r>2V^z9pA>Pb1z@-Fj^WTPBw0@&49=G5O(Ur@SB)UY^AE;YRCkz2kLKY<&C(
zGIk@$Qp@4R(%pBbL&K4XRA7+U49CS!B?z}ecUbJB4iZ@Ry`x#gFr28v5)oye#myf;
z${N31?d?)vJ0o}PrSIFfVOo~uDT#6m`5UvB_rsXg8o&5lni_v|b6cI?<<Fp1txl>;
z)YAcz(VFTzx>Y<nuGrYvgx9V;)72_4@GCO!Wf?vYSbJCJca1*TVQIi-x{2%>k<-c1
zPIYHqw?F2q3Wp`8jhR;NFZ`Eh=lr2ur}G25P(TQW2Z!P1-a6f3YPj`S9n|)gMy)vn
zVI19?p(LT2t9uv7jUC24WbM-w$FH`T>Mmkcdoa8*8DmxRo`1Pa^tNoYa$6ef>V~OV
z_`yo;X!0v5DJjO)jWoe014v{|jq`?)X<y#Q!beY^CMgw#Baz?!Y+b5gL--hx6RFpF
zR7-fs6;N~?A04Dfh2FN=UeqAtcgVt{<hfTl?2LbAxZvM0fW6+mgdG1nU4#kl?AzUw
z!x`aJp1H0Jr4-kl<tsnaAAg#BMXGzRRU+^@b#!4>A~y`C5nzOeg?IfP;>TNx8A=4A
z?&E4d+T{)o^Bx%vp3Pj(qeshC8*&MohLb<vuXj9RmGUE`6H^<Wb-XAnwrfCGQ8>A0
z$AKE2$jxqVl%IOTc_F)WG9xX%!mP{1X9MgQC!^1m8+^2{aENWT+q-yOlr+_dJ}@f%
zaMro0U2apK#n#r~@~y~v{Of+(^XR16zWis?Ys|S-Pdno5X4^!H7&5D0y?W)BBh#jW
zYg#(d5EL-o!>Un`+?A`R$D&)sz{l^={aHovvod3Q7hgpqb)m)1^03Y1&#Q%cM#Wcq
za&(v@Sad>|R6a+t9#BRT8V#0MOwTR$7mEAb8mN7ROFB>)xpJADgQ1Ng-vNQhZ3kjW
zQO|#Bv%C7l7}LcP4JHvp6~=2j9Z7t%*sPn4(Wr|lQ&yNX`uL$aiOF5Rz`>4`N6p5m
zmctc_6?SuWR=FC54_WkT7OQ$nEVQ1b<U}7!8n-RXhqCBY_)gchK37`NC^U*Xekd;P
zw_OgN-D2h&*pU7!UeGo9d4GXXcumD@v3AMxsJ%lD%b~tN-ia3&Ylry_RH3!UTe;H%
zCQEN;XJ-ZCR6SES2Q2#w#1)m4#ATOqt72!<5<Ei4In5$PN8ERG$T&<CbgvleooTq2
z0%6`U#6a}S;9!YBnS09z_p5s?hV2}+xK-GR^Sq%(H-}p*ennd*tWW)p+4&Jj7P@Tj
z{&aZL@m--MB+d1t=3v=&q;tb)%Wzm<)-hIS4;X{yW9|{BPp<o$bIWCAWp)h<>aVZd
zaOr-r|25&{(74cQrX`NueH7<j<c^-l-jv;n=h0U0+`!MzwVnrqqa8OQM(In39b!y-
zqxY6d%rGJwP7Z2M7GiroQ;%)_#_{LLO>@`BaEX1k<dONf;oXhl9@&$kO1_|O)gAZE
zc8R#&4xv1pzUZCW-9G_EtT7Ee&8}(kE?1a|!}f;y5{N7D_XP_r4tLiqnP)#NQ&!{>
zTia-H60z}rKWgD$i9A}VSYX-l#=sp?)gSvj(cMuo9zVCX=lu9au_wOZrb4{jGmL_v
z9$D8?V)cs8xh_J?*Qbw<4^}qp<OClEb+__TTtICa*m)1?_OqidpB_{!acG=)7H@-A
z$@96U>DJjq_DXvr7aN;`!;OTK53bAQ4gDz(Uf~%1s3s-|29}OLcL7aSD~b7%T_@T2
z!0T$+$a`O!MtWv68PseO&Hb;%VOU!6u^&zZ36~`Bjb_KRt=FO9^Sn|jMw+$83pO`m
zFeBh;@(En*9iF@6mrOfT-`d6s<r&**L81AUC8@6R3A6raI@tOc=mxv1<1GijBUnin
z9u_<2u;<%Pms_gpe{(B6`NTfbIlWt|SKG6)jpL=J=dNO>^6Fi+{X$P_dVV~68Zj*T
z>ojnjT!nk~VXt%I+oDePD{{u#Xj-CgH&%Q}$O+(c)pO56BBPGtmt`0HmU9dHi-pN9
z$2(7xwMVT-zm~QgSm1R2<3}%ZVz1)ea$L&2fKPPraWBW7sYpKBLn2o^O)gFhmwurU
zCMc%{nQihs8cG?qdsyXhoHZ8QuUD(}1zK49&I<Tyqn*Hn(W05Mo`wN>ouwpB^v#X1
z!KJyhgz)xH5;4Q}P5T<%=1IN%IA)Ox$t7m7gU!dzs@#H?F78?sjsY>_nEm-l86zp%
zn~rG?{3GPPZ@{nM`eQ5a$+wOr<JSF_1sty;`g}|E7$NmyypEdKsb{XQkXr%SgVNc5
zN;<}Q4wi>2d!sIoRyo_w*dK_<F{5X3JV<3FQ^V?OSa4jx%<h*kc5rnmRKpV38M0j#
z4<Lzhupd&@r0TSD0M`}?x6ET_ZX*YAW_gCY!>G0!{kccoI-Q3-f;(}buBw%q2F3lE
zcmGUG$kryftxJ{Ke;dqZrnJM`?|)P|heOPK`Ga#V@bQ++OTil@Ve`K%QdrqmG#Gtj
zdzUV4<3I&-iJfe;ptle1ZtUeX4Agph&TM%*>0>0wBoyiBNQFi;OV>{Za!$N((apRK
zX2D_L#ymKaTfH64Yg}$KRWKaHeGWPvXu&s%D?VC$ojrZeE0`e9N?K_B=dJ#Cz0q5i
zmK?{&JGJtq39M+a=<VA}Z}lplwF(_%*@kWpmD_Iro_0;dM1PG(_H^HB)Sy&p&w!Mm
zjYHJslcrU2T6MXlk2-r-^Fhx%&H>$ws|Y4~5x(lU$;^-g0zP3p*C{V-pA%Co{jGo0
zV|9Dbiho7Ro^~;!*^wW(qGR?)?A<v!{I<_6edOlk;+wCTPx*Gzn|Dicxu$wDGcx|V
zK|x{u?x-{2_``9tn?RvOw?L+y!XA&{oAclU-4BFz@bX|hwA17KGZSH8kX+7~QCM0y
zs99w4WP4$x+O=ZXjf@~q5hG<G;K-mOS*k@=lUXJ3s9=vO9SUWhdN)_=J~7DDw%DUD
z;Q)nJx94)dzUi?$Zil%Y8pUoLy_3Dkzx}9X#~Tl8)?QjG;eq)hb<@v70H!z&bY3Fg
zY&+J(_}E!(DV*<*$=}O4kzevim3?9<jHKFir$k;2K5E<)IllRe$t*6-R&y>f_8F(b
z$KG$!(6BUHgQ2~`Rw3J#j7-D63jyd)8Gsgzn5tf%BDR-Eqz;5D%{x6hTEeR5crD0N
zChboU6na-b%e|YGImo@@7}EtYHu)=5PMS9JiVtImc@XEb$4~g1Lsj_ej@}3#j(=vo
zyxiAd6#GIi80r<<@wlHfU8~eGRhUc(yMM*Jg1WMnTtTvOiijX@UN`a?E+0G(fVDLF
zLZfEs!4%4yR{aI_7P(dV4JmVB9LPd|1}KACpI^!h_nr$PNi3PaXHbHBjyAC1W4GaT
zZ_ZuWV=okDA3}7Ib@f{~SnkvwPO;88Y&0N|Dx1CKQs%^Ta=oJne5_gh<=PVu_L@}L
znCSdDJh$qPv0QL{Vs8h$#hB)P_g;!Y(^9Yx25Ldn7BLP-nAwc)yGd@`u=tT;`mw{Z
z1_#ST2eVQ>dYvIRMmKF|)N|B=xZGjTo@{X;StPJYQP8mWbx(BLM4_hP`{DemH>dK2
z6Y!xT^chX|vzP)Nhm1=VSA!T7$ML-Qn!UHb^>D&*)IGwI+Y=8fS%*4L|BJWX(dF03
zx<B;+aWO~2hpRqHlspy)#3h|<P0qrE)KBe1{Qd58iw8w9tT31ZP8>iO$xHqlyN7!l
zEoH)clh<bK-N!PK$P$azb<eGCwXHc$D@{r*A2jmWd>>el@Uc^TOPW2|X5Z-XcBwPp
zySUS13`KwN2xy3jj@83}(<Si0fX;rj-FUGv-Q2QZdNcX1as}`RMONiYNF<3{bD3$5
z)4`9_k>pl0;JPaGEOBc*j`wDaPO{9f+%}rI<toW=n+Bj>a&dfy12<g5T^tGrfXBwQ
z^@wX<nrm)xx9TOx8!QaK72X~5vCM`hdXS;P97mkfoOO&7X6y^mU>d|RCB}uB;;J#j
z4U2a=srW&h)Hyoeotfs|@0_&-Uu^A+<}{D(-SPs~rgpE{N~SiN+bTXszIR_t4Wr0%
zcp$bAj9xDE7SrCJnAzx^(xZ!z_J70wA}o2XJC>z!HV>`BjfaYU&TaMTtqjFBMMhLf
zhcQi8H{tZAUgkoXte63pPo*M0m}D`no8>pa-yO!S@zrKGFW;Hn^CxBX=ypZJ#Jjg*
zy4K&-)n&J0HCi)V{xN!3cG2_XV04;t;73awe`nmM!~_FkU_X(hOv*nkdU1YUCvaLS
z?v)X?PpI1N%wcV-aX-j>oxHcSM6rQEMzFiTkQ=u#&*U38s3b3FIMUl{{)6~(iG-^7
zD?t@e#6f!Dz6E|ZZt&pJ*H^e2%b&fj3#O3-we0dX&nFfo{0<b4DCB$Hdtu9!P+P9+
z`0(xNmS2b7C3-&E<~9|kHiUw3xi@)jzFV}8^zLjubo(J|Jh0$O%&cmcz-#-XDw{pF
z;Vnf33%!d2)T?yF<+rJ4;|Io;%L=kr3jwSRl#TZ2JL=Zx2ho%5_THEeeyoF@+nPA=
zkzP3lc|pZi(eszps-fB3p-VdjxL77*btCdh71~u!?XkM8pBVi^LP8d&*N+4<k%)mm
zUmf(=FwsF!F`1*B5i(~!)fhG$!)0N!|B1V6>vqM+s@hCoOmzs2fWT0-<BIM^$@K3N
z;30<d^YdGFaoOOBUv5-773Z)>y6Vu&$1nEOy?(a^AVW)^PC^brTVhmo!6X3Om7lGm
zs`EUKTpZhz#D~f825UWQ3m-jr;O#oy6lHmrCF@!BO3gv+PJwG;f}jp|Qp4taH*=-V
zg(wXI^~c#b&$>O(Kv;b*dEVbq{wtH)YYVg1{gXzmvHd4o08U#xG_{K@2214R4nP3H
zzhX#Az~itOSt%36)>6YaQ4N5?^bcEQ<t~-N!6jmt=|LKI$kL|xeg#!7w;8)7pOS8?
z>CH?ihHI9%KhcYYx3=!Al}KcKq2tO(3-8mXPlvKKi*1bN&F>{AmSn+TE*_O-d4l1A
zQ0TDnaD}~5czJaYCRWi{mJXJh-c3%k56jV(G5n4>>_R$zFNW=V%O8DUc%s`$B<^4q
z-2{Vaq9qzNN4Kn1D2gU5^vbJ#nOWqDdy?BLa*)z1mHP~g6>DqCbnCK=!H`?|1H=KF
z#Q;aAM0W8<2VRS9Y2t&5bW>pXhxRoJ^pqU=O~N|&78Abx{3*FpVlikl5VvzdE47it
zFzREd{_bh=q%K1#ZO8}nvb5tTbt{>07M%e>hedVAAIc&nkt4B7qqVh4cIrXx9Q1q}
z8cIoh+=HAat`0HNFMDI=x0@9XTq_-xhM3iIWE|7MNv*4Ixaqk#geVCrAAe0T#1|l(
zxpebn_YFImR#5xK?0pyZ3UFFe@ozm0{5fyEQQ>klj_)bSZ7HY*V$NaDqXoS}K`vIS
zfxS8Q9)dR8WS|umx$5~eok|O`Q$^65gHp!Do_kXfE&hRlmulC3P|OM8lZ(yv7p7^c
zy&JOYl>H&qd1x@SH~@9vH?1{x)LGnUiehhI>@vTZ$=y(_kb@b-Hn>Rfhc^Xp-5`G$
zBpLc6O%$ZmtS5`+K3~cv4(7s2oap`_>B6ET9vmaP;L?}LwP_rA7Jq=OYAm{Th0oS+
z`g6pK@WoZ1^DyYm(Rn)TM7%IvqMWHJn?><~_p>@TqCPt6j;bc9=|EA2I(NUV!0m}E
z`^<CW-`>a`xKNotEdOvvgKfg{X|r_1B`T-7&EL|kw^S>L@+~<6>_2H{c^6GDeD-0h
zN}MS2rwS<Mdyu6u`xN6o+v5AmG}UNY+xSNjEK+aDIlots-RZfrHgscgs`?5g4_tNs
z=0s246VZIdp2<EmX7>;I#W3L?zi@CSon%B76B97ZIla_PeZE)p-|b=Yk`czmXjc`9
zn;84z^E6xkUV`3{j0Q8tcg-8`C+4#CV|X71m73W^wpv+wOmBYQFZ^sVpRc-4A9erP
z4^Gc;+eHTc^ISUmTLAJB%Rk_upMpFr53fL|*VxVbxA2IdmrXPnLuCl+HN&WT%-(2Y
zVB{FOsft+V&tpdFBRvxM>_306uh;pV!Gn2vI|BY4l}qo#FfKb)Sp*yx<i;m6Ne|tH
zT{hFz8N&=fv{qm}{s0xBNnjpztLgo!-Wf3;DCoeC>gJ^~^$ooML*r00tW3tkIvoN_
z7gP+sxLWua8bsSX>QuL%ZTs>`w7{P%2H>4Hb)S~3Z7u{eAg@KB-5c6|wq}vL`j^mY
zZr!50fB$}ZL*v*5WC--dn*<J5XiJJcPd3#(gK5^O*%q5a^{j}?xBB#X?q7&`0Twd^
z)6vCD<2@VnbfRaK05@uoXd<02@;mh{XOlUV5lE(DVL6e-PXEVj{^LQbuRtwEFTL~j
ze~f;buKVXfx?u8ShSt&NS&p+Q{C#d6DL`UwALU|B2maq@<fMT_A|t+-UipvF|KmY=
zpxCkciqGPFm;ToU{a>N{?}GjRs42)a48>gEgASG#^l?N|LaZTA-RDSj#b}d!%V-|?
zGna%*syDIbZk(sOz`kL*0G$-^^dp0xDGQ<vw=9%yYl!JZf+jq}n8fektLh8Aeyhua
zao^f>{`hCm<b2(tyHoS2Oxi9yFDYAG`zkMG!{9|dA@Bvt16%vtZhS%2hcZH3Txyc3
z_8us{b3gdag!ER(SH*uBvG?|6Sgqbyf5&`pv?mxZR9nt+^C(^M0Hvxrd={9_1-*jz
z!Wc<ur|QI{`cLA^u3eeh&ol3DC)RoVH4`snwb4CWS@c!z6cMG*MbUcDf(He$VCsmR
zg+mDVBd-quAq7@N#8}tOyD@merYz%~z7cDD#laI`8COXtfT6vJH}ST;sJ=*2V78o~
zQMWG~z@I^g`FyF6D)(R&(lLaGT<FR(o%>;QNz1N;s;Tldt@YR!E=JTH0O?d&$pu^x
z*`#z`oRfA)sQ_(8L)x^Nr_~aX>b|O<$!dOClC*24)WC&VL%d+10;mA-jfI_`JYbI`
z1q;POxJM?|_@KQ5R@tls?UT&d0al*bX=6MW_JLrv0N4jspU#W*U|wvKIOOCFQL&+4
zyi4<gJ33bhAJ14Q>D86q_;{Z20mJ=7`GFyZeEc(gfxvVLB}a}zk{8M5QhIPwqgKgT
zrJdfY)lb<FH(~SQUbqm;Neq|_33Xn^3vW6TTR&Pa@d~jSJF23Z-8ee2`uOvtC7VHM
z#v-(=?5ofU{*4QJ5AkXO2GC&nYvu*%IjO*%8#+-NsWy1W_Xo%I)i;l6EGYbB_MQh5
zneSVQ#D?2mG(5;ls6B*)DP7puU{a_Kad4FdLifsAfO{t<T1{aAWqrM`SQiEBc-X;#
z_{l8#g;`@IlfvP_BR<CeIgPFFc2b(2l`06H`pl*2()=cG=yYt3wcm3tjjz&MO;*<^
zgDxs1#vKGA5h=oZ<HD26iLL1=-zVRU_T`@wv&3HuI4Z!yJRIOxGhp<qdh)3(>Gy%S
z=iGnH<&XvxeC1&m9wKzr`ccP#7VXyL^MoRg?6=cIczLZ9ss)u|C6T6r_Wi7DH_mkj
zitYyHQP)6<oW6i4mEZ&mn3G)B_4OXlWA_odNt~aXI$<Ec-3I$a((~1Sqz$%{CJgT(
zl;XQ_?^BEUy*j)*1!ZETgLH%%H2OcJB+egAfMtYo%x?B%=U@w|heK;c!N7Rw$ARG}
z+d>y72AYT9W2Y%7;bSXpqqHf`|FZ^zx=tDEQpk71E%BjWQ0V&veI!{T^Mj-TOJcRG
zVjAxYNDr)*39L5lkmI}(M_>l~rF9_H_{9%;f6?s{n#T=8r<5AP$DR)mfl+c7w1l7u
zoL%p|dA187aV?=WecXrd^(tbDEPE=oYzs0X7-&u9-=5KZ{^s#QJiIB3QTIO_Eh{$g
zii8}^7gREMy^}#|WXpS0l*r`XSa18oE*G7znZheO&(eI(WsHSh2u`1VO-11Odg^k~
z2|Z0{mvM#nRT9`7eq8$kvG}G^_EBs&kN26az=d6dKv{9Xz%*R4XXPlYmpRCTKO~uv
z)}pnTtIGH4sO*#nZ`Kp{)_F8K{=Z2ahjxYPHlyq7gQYRS8+dJAZ+e}?Y0R!{Xrb|)
zWz0a0GY<vowwarD&u!r01GuRQzm#BM6YG9R{<RS-Op}-=YZi0`TOEPT8<ZC{vkGOl
zAGL9u+j4@hunaZY@q?!{8Gasb#<IaqJE?yV8+GdO$NRn;hj!yatr$*(!kAxGi~`Sv
zC`2qNSS=Z%>bP!tlg)f6^9xdspP=9D<4fX|NDW7)bIAgI7eJkhZ*{R$=Tq|faI_-5
zzWv1mV?00~2@~sbR|9Tmof((!e2Pd!I*R2H#@x(vuJvxWfptktro6sVxXsSBn5mE4
zzE!mj@k{4sc^2Vxu8Y9w=>q#chG7tWZuhV(yprt&X}7-1)FM_5?;)VtbXM=R;-!Nr
ztg0S^dKyi0G$*i@JhgcrubnRy3%wGhFSTC-FQC*JnBVXDfP+6=2|M3<zd$f};uN)9
zF5jhz;kF9@{#^_k7dJFVr&9WN_}!^@{L8_hP%9~eC;rC$_{ql${2mcTP?=|7W%fCj
z_A0&CcX8tSIP1PNo%J_BHf2f7(#j1!x3AxG-?!TX#UR7u!@UCg1yxY$eedgwRX!E2
z)ie*)5pM$_jbC8kX2y-TXJut9IKds%S(}n(yK=6HwZxLULb|kjS*`mstY08#1dnBs
ztORVs>Zg837=ld7ZA7Ao_YHmr-@&Eu*;$jh?#zhLr|G)YE=DR}ROrVr&YuKa*<+B&
zc_+4b0U0BX(Fvy?`j#nm#L=d@R;1}VgBp}%Y;NIN*;l#>bDMV)f}Y1!GVucaSyXM1
zFM!fu@UGnj4ON!A22a;0OfA#Sp=@oqa$yVL`q_QOM#E)Z=bMR&9<K5@F2}O}b_cbY
z=@(zC9?)c{b#RUaAnU^V=PJfx=%3stbnl!JD0Sc8G)1v5Vj*8XioR*$1sS>9rK28|
zU{!#t;4H*_>4Fs@U;+|*FeS|{cfQUbEL8P)XfuJsG^8npTYh`7|Jgvh_O4Ea-F;JQ
zgD4eI_}oj00sNxS3Yte!$}<yt%iH<S8pN8H$k7{uPHb92=pIG?`EHiWiPadvny^yB
z1HIs9z5veySI?oHIA7hWKe9zxuhJnqDkFzGx%hgHB}mGIk=)gcjA;X_d5_B597$;_
z>I_Bf@U@EWd1Wea`k9?P+E4JdYa>P*^*G2b<eO-U(wisxz^o>>>0)Ay@IfcPr+>kQ
z!j(~V{rNECp>s)>6Siq@FZ5*F8YQu67X73orJ!I2r^42fW8v~lmQ6o>j>+9++f3VT
z-Mp1arUB`s9IdIfCuYsi%Y97=!lT-iAm)zZ$^@A*7KB2w<kg^^R~Ol6tE9kZ9ox{&
zUOHPoQV&bJ#!X{!=){FxqZ4A<pAA%wePL8iyfa&i8?ZiFRT4tEhHqs^)V6fje5p}V
zFCUdMn8>XXY62-NUQVo@ER9>BgvaKdu969|1ETlu*W^?z+LM!?^Rlq0!LZbJ`D9FI
z!`5}f{QUe107Js~_8zFv)A90Z6k#HOCN<+g;*Lf?N#KLSRJD&@?nLL@8xXSIX<pvP
zzUzpcE2K<g_I7g*5phT3kG8EWG^IY#-n=c*>l)mozaZs%L&HH;uHCtt2Wkp}Vg-ee
z*h^>j4-Q9HR8*81@#GI{+MMeY_klfm@<g%JQimQxAy>CLIngK}jdD*rg!@DuvFlV3
zdiAYP=;}JYQ$>_OoSqu8uikI#0MKOW<?eUoKa)^$Mfg0>yp;s%7|wj${1*=Ym^@JB
z!#T4y(lb@|f|0hxRP#G5T3hbQ#c(y3_iGgz$<jlt2Q5mS7mad3<Rhi6Sy|UW;!7WB
zR;r|(Uwl}xK+X}(Zv1W|s=vrIl$Mq@GVy(CU_s|YPKyC$9Ac))-e(_e&vghZnHUZa
zR`qE(e@lcq5yNnaY&4W!p5!iGmKI2&#>JF_n$orzU*z{!Q}uC991L!`_5?ZUHiX@C
zKZ~iB3n%O_eu4Z{Tx(bOU9H$6#o<avM{jA6&k0FSXGquzeSPi~2!2WdD<t=z>Kqr2
z?ZDn#xnZ;Z1+>$hyukSp&aBpw0c}f@zsacb`9tfEkfJpfsiQqTA@k9j<X4zoBgDA&
ze%!1G$rUmgH<jfyOp6<+aj#5HO^yEAscHV86L*fu8;E?SV%4sO@B5eYrD07fg0WdX
z_MQ8@I#2X>H-(pn0^@+WpyYRGvda<kBW!8zLvePPl+(w!<KzU_U(s4@=ML?S{P26e
zSq`GN(ZObyDnT7DHv4(y8TN8e4(K#`{yfX%E)IXaLL8s{_bD6aDq)U`hE^49Z$ZfT
zh1jC@Dv#}SU}(-%<w{Mc&n5im8dEPdSFVyANV;b6=h4l*&+ljxLSMcz6jQvG<HAg<
zixm()E2lbkJISOCe+TGdQpV@SB9%*5DDORE(JV?g$;RPFC{(*_C6?+V&li162V8n>
z&ZX+}h$;^ql!PXSWV#`BfL;<9Fl}6Fm1+h_bXclm>WV4)7?ezPz(Sk*LOqI~wqPzf
zn6fg*q`fu&27DJ##@>ql3Za_<X>&(!O_1itle&p{RvCbg34+r*QkY72uEclHL2-pH
zQ!ST^TW0IW9LVH0#RCRfDA1H>iqvSBWIze#dbrEeXxi*b&fquJ&F(?Zg;YTBP%%B>
zGl$4_{#eYiNRGb0*P9Ir21yCFuHBF3I#PU0`vNh}PH7!J3ZQ=xPsBZ4I3gW8mfukh
z@O&1UB6R@?jM_IKAauGphCdU+XFpG5z}q_<QkLaa#(GS+iWRo7?4L>h*rHVZ8=a?z
zQLo#|NVO4$*9~}?^|*I`HlV?ER61JC%uovn2^D^GKGz?Jmo|WVKRJo{&q1?xCNdSG
z2{>Eqw=|W~%7!h0^?DCr1rN6XNHQ}4`_VffgP@GCnY9Y3nrcVRnv2-8C4G5AsJr&w
zn@^`L!5CCS!05-ZS(?Sta`6JDKeRzw+ztx6tJ!oBY3hPSPl^Y)pPw!py5QhoII89W
zgV;_tz5nom2$)6=Q_sZ@4B$|5=T6Nhe<=u6QRaD}A^1XXgyPPvMhD8A2i;lS&>nxR
z%=CMM#n^YWN}BdGGz*R1#tORj#HM>3?E{!0ET64a8V-t%dRl0dug-!x0ydNjo>pX6
zINkC(TiCB(zt#a4g^v*l$^mp>`GizdEL-#4wl+ySv+duqa8|YDVG~Um!Y7YHxJHPt
z$2A+X-ikR)co}wNA=LVHyabt(rBP=6aig%ra+t5&X)VdQ(*0n&#f&1m`Y~`-2lr|&
zm`;5lhE<Vshy1e;VP^K(MDv=gX_9j*puMm}u<CK`9h-N5o*Md4wbi2~z5vV9xf@*c
z`BQSdhT9f}$fmTq+a<r})eOE5Cp|^m!?-~;)7FE8`*|O{;(>&cmHqNiKlTq(i+6yO
zvo{Kvy?W-HkZP#>WLV6PQ?&0@N8DHKS{hnnFyED7bA0H`9{E`%lMjkSkrV-%Au03O
zeA+rp>ha+_akiLZu2a4WD;F`zt;fCDBx>WYb)SDowKIH|I$MBCWHnNy0Sa8SG&B#2
z9=?god1US=CN2&=R{?^jDB&-9vumFUA6^Or2Ep2r38;;RT!N6rm28EN-nL4Q(I9}&
z*|@~m&+3C{^m-9ewX}wpi`mfW>++1QYQ1jkx(p=ff!cCYEN|zY#z=*I<}6ME@b2X(
z!b@ubCN??Oa}#e&sDN_CIdTui1*g3n@$}4Pf$-xUmEe5xjdn;Kz*J3nTd!*cGs{4+
zNwLOlPho6z;Cg7!)`zZZ>{UJS+~hdS-J&!x@;eWeir!0j;gYiaEHdqkP1w?@bnxq;
z1Ef#l?6vlkN2D7Yy1?OvIxqyK8DEIfgh57PY=+c#`7FGPC*-l8|6r@MDtxjRM9F9O
zlS8G*<nc^uA|`}`jv(d|Mu?o$pU)+KYRZOv-a4&x<&;$rcDqDL8ep$*N<ZqoV-0wr
zQEZ0(8&+RHQMIXjbJPn!Y2Gh&pn$AvOF?<w4KOB*a@RAOZP!jEU-t>d$KKb%oC8HB
zc<lcwBE5Dklm((t>0lAG?GvhZcdVvLB1ClrzWiNJ)lj=yLci*dKhv%E79Ip@AIZJi
zpSU=O)vZl5$Rg{GOaH8ne=|xTq?7(>Q93-0j>V*w<JO&Db?#5bPEf<2khhDT4nG!+
z4z`>zQ#G57-IEiN=YGJ--Zn8Ykphs`ZLCC22!|t2%nHVkv6}RsfgGA&wM6FI?m_E)
zKE5=B-A{AM{EL50cr7ls!pI5Qh@Lti$G-brORY5g9Zr0-BiiXVrv_*~htAhG&rRQ$
z8rXd|qBFgRU*&P#&dSm#{89d*^m$ZRN&Ns0Da)O^nnl*CP!wfTn?)@+kgx3ZxKOEd
zNY6HbW5+Kfc_ec|GRV<T=OksLeIB*rT7dzEDks!?rS@dxP|J3Ch}Sj*l+~jkUOGMq
zHKM$;y~f&20BOCG%G;Y~^^c!nb@c(HhM;gM2hkt@`YM&j<l}=^$$(cWXNDhFO+i8u
zo@rYU3Zy_<D%lZj`A2S-P9<Q)^@BUUb~`a`!4&0-dd{C75+j$Ty+BOG1N8<(WB+RJ
zL_mCQ!GT>SG|>+^Uw_>W*aJ7u%d=<kK`^t|%a`aH!17B$lM8KUnq%l085s$QiRs=_
za67;tK(E7o{`{#O7aDh{ROddGNNcqpmmI%m81}|<TaoroO|}@vv8ML@u*2ej&ELOa
zfQ9FXLj+JJpcLg>ZY}gWnh}5#@=#r5YzkNO493;vf{E-hoW(^ROu-*M(1P&R!7`^g
zid_Qmpy>nzw8=S4#*!fw;6RJxq7`C9A^D|39EIo^4}R=U)oqtkg-A#&hX7`U8em5(
zN4O^|)eu`(4i`ABJ)IfND>m=-C*khR1~|T@_K9X^vPpYV&C;FhGc!z$3XH^9IKG`i
z`4dt}S=lh)Gf-ZjC6@7D8@$k!F)<{t01BO;KDKv)oPFYjl^h>S2eMHC2kXl)UOf-_
zKTOI>H1oBrxLjh5?mNTu4M8^}fDiiRd-YZ_VBqcv3N~EIN9LGxbjxl>qzY8pO!-Th
z_5gHk92VLm;P>|J<)_%M&y3`*97w^gESi%21q%<f5yq{{1I53wAu<U<IxPvpR>~YF
zCDX^fLxTW3{ds8a>#m9-CX2qtdk>hH63b;1SKfCqha7^8RsX^3EuJsj<WJfOb(b{!
z(vPdcm}eu188v^p%(i`8|8nW3B)}$9OQoaXt}E5O+()0fk=NQ(6afF<{`m4x*^5uJ
z=2x#?m4hgs+~|sCXm4+aatrYVo_f>WPe7$TtC%*i|4!ChRUaUcmc>3$+-?Nz0?q2o
zDxW9OA(oX(T#On825sc`^0F{OLPHG~`|{TtnR6re37-83eBVIj+Qf^%y_~k_s-<}4
zScP^fX`@2o=F0{trxf8wPTkZ=7%$le?NS{_YupV18tW*xHIBOX#QPUnLVr!wYlJGS
zLQ@|^YjXY{?`oCs*ee1~&onxAk&!><<qM3lbr!JcD@K%O6^u#EF<`n{nzI>|QX@B+
zT&#gLjg8Sl1@IuYW$B$UXc!|>2WU*Hx)8J=6fzkAj)NlL1;|`lkmIAyxFr1H!>t66
ztSOP%`Lz;|Q|3abdkn?S+)M{0ohc_iowaqq`ESH!t<qPmn8kSmIw0O}s`57Gi>ksj
zt@Hscxuy0ah?oeDVomv|v*VJO)9o<4NbX$0#r-|cQFXy}fetVPIUdy+YTF~`dGY6E
z=<(C_$zMr;{s5?yaWQJ-kaWcqX^)scyYQ=Rl!qve;^EPXCwBs_4s~L}`ytC^TXWi}
znOwT9F1Hs6dh~zfnwLJ#?byEZ%apXmW^Nb%`ukW;TU;yEQ#hh^(|9b(IR5@bhIV-Y
zs?nr2S+H?xZ4o7H<CfU>pytb)nT1MMINViC_BF8}=5N+SiDoG*tL59m8L(wQ<rk+i
zTl@aPKKLjMuy7ucLfSqcDooq;4yHe(LOLjvpM&78jiv*|rJe-1Jp!~;G=|CxF|t-~
zDjxf1+)fDVVt<saf~J+7yY#1~NXd@VD_NUB>NrK{l^D@gY}V&SN0m>_1DJDiwhWu|
z-dWKTKDHM3inyy~q$=>dFYn7x)(vjUSQ868E_57egpYi*iv>t`-Ilao9TTyg2hJ-}
zg=^m`&%>~|BQ<gVi}RVIK%gqml4=1pe}#{1#n{B`Y6RKCW_lfHBI6x9X5FeX6fr9c
z)&@ZUPKVow8sAtg%)BS-KJSiNwk93-uw7`?$?*Xy5I<R3<=nnQM_1n^cAe{aB<=0n
z(RL3iT#&H*UaZC^{Wz4oHewu{gMgK1<&6$_;_L_J{v{|{phk+y#|P8J_X-4gt<Lm7
z1ZBr0S!lnV;>lJlfPz&wQ?mup>_HbyDz)t6cF-}yexPieSYqD$j)*wE99_g-Rn|Gl
zd5c~ymI3(8FhDZ?4GQ-Z%Fmjf-*frousZf-9ni#Ly4f^uirCf_QjbY_0YICAOEBA-
zV5#|C&;psLoDpko;8c(hOK}?3tfm5UU}e@x@)vFHRwGj_HL6{TLqEB!{i#p&HAJ~L
zkZqD9PXs~M<L@0Sxv8G^s{yPv1b2&E*C(as0PQ94`uzv~BzNca;x?_VKmcU?17I76
z@`apAgQm#))6-dRVx~5+u6xDtJMuxjK}g3VJ_5oumWqPkavLOO!Vzu)f42@^gvu`W
zY#9~(RmNbEF|cg~0Hf2_e+OK7vCdckP^CIkW#fphfrQ(8;O+1vHi*1pF+2{}Btjo~
z0?Jee=zl=jn!V>!ZU)xNA2WPLTo&#+1Ma`9f@~)?=(&)jp`r1%tqW|l!dTq2s({I;
zdB+L44{R2r@|qZ4m{>WL%Y4B6C6B)IdUpm0oKdL^wfozDD*#?d5=m+45a4<Fppdxy
zEN+G^BLk2#ii03PBgijXsy4NOow!;M-&O6;XRZUzzoeU6RUkR1^zmkz@O#bq-bbJ2
zv@x$xnixG_XpP5|=5tt_J;0T~f1p^m<nSAGcf@r+A&0jmJQ>cZJ$5nPLiYGGs%DM(
zTfL#>@_G0-tCWDMKrjlQrg=fh+OFgTl_VjD6dJdN0}me)^@|Tz*%&$Bd$)9m=EaK_
z@yl0c@UMDV)gJXUxd_|M24lmj07FPINz9Lc%zflFFGxCEyWUsnm@5D%k?YejR{kVj
zhN5*5u>F%OI^EBDgET-ga^vLP_R=6f)Ef-8esV=mmto-VoV)~35i2{{X(AOVj5~Wr
zzFGeGXlFDg7xZoM6bk$kDt({pFEsu+WwkT2Kyt@@u-H5T)gFY3=$d1=x$Av6r?9lP
zVdTjayf(><1(bj|t*RujKl`a`?q~d4$7+Ve%-<Y5`j@b{p7)Jqv2;eWMXSpB{NA^{
zdF|RneSAC7X$%$24S=Nc##5`mSawy^OVD+jhY^_yLN{O;?^kCjruf^2rUC-{q}TR0
zbJ9CH9hyWHIEYTzTr)zJIQf;&w{PEk41IOU!1?A690VkCabsP1(4od5j!OYMzMUw%
z0m9)I$D@THi<oxYB+ETKd03kUTB*7c)=C2;Jpa{jtQTM=7g%NJNp+z4A};10(CARm
zVifZPc(ImpY+KOF<S%3_TGt69f7zkEOCDJP>^aKqPZvhWZ|2tQO9S>g4XQ&3L?W&0
z1LnC>;J|J;t6q@+;V~(z?gtdZ`-FQA2Sx6a0^uqVld@#d?YB8pZ7S&C2*4F)%YWBJ
zotk(QImO27JImj5H>TzFtl=H6DLKdJE{cNnMmqeCsb#_J+weilN?8ysEDpZwaFN7b
z9Ie&E#HYHW$(jfNR4Rida1^4Ab)i0}tbWzu`foOmL#Jza*9BdQ1Xk<Nhzz%B`DR$|
zc>&e*Pt7V+!ZYQ&KkQG$_|&YY59$rlCaX~__w40f{>3ZhSHX#+pxZvpEal?Ff@2zG
z$ZOb0)L&|qV5(=8+{gXc4g&%5($xd3<e~@+j)+~5?_&P;2A><wTC<xJ)l5Z3h8=tf
zoy&5F;b?y=H04o*KZ$J|Fc7FdC$zgSq|pWFt^a(-ZtoUzKqAomE9M#_s03YjG6&7N
z(#Ol?tc^}%pgQ1K>Kab-xo#|pjGn4l>Q5{`nBc&hZ{=vAI@WLt%m@(X5R=asT<C6$
zfK3gvwRHhmbX0x^)lvttXk4v0Ci$!KLVIHIuFJgb$nk6P0@S%x>;6P6+I@wWZ5_}B
zXstmbS^LSTsS7oT3S90CgHp^b&CL-OMePhlI{YnU=U<0{NCyT0tDCvC?BA_i{fHzv
z(kZt|VOLVTxu6Q7Qzb-~E`~L*excG|-t_vVo-ikJFQ8gtP#sj_#;flrxLbyb4pyiH
z)?Xtr(Kr}Z+a+#F0Z-H{{T4;g9x;jLB3MmF=3H<4-~{YZE=b%)Z!@W8nZW*RpOA9u
z*PTJ(NeKXYJT`eq{yG>NI*^CP&?q(wW4~t8UtpNT?(yu|vlQUKfI|TgSW;bGJ*VZD
z9F~(PoQ_UhXm<>ajL$9|RbN5J&~+Ne0GXY&hZ{xjUm-ECf}9nzd2db_Xn6Z|uwqm0
zDGvGjzW{1kLG;k_)Jpdud=<TD8kqxc>qbL$7-gq?(gInbP7Ex7w$QAsH?RMl*nbwR
z5!N`M?9e}yUZ_M3x<n;6f0)_X6@kXYCaG76>{Pdav$*E<7^4cAn%@!A0`@W##fKy+
z%?6=wT@bT6W^i=0**tdDX2+wXlWBp2#X_Sk1n`{RhQ5JlPzil71vv$s6N`>+wu<g-
ztzN17$(i#Y=A5GZhIXo2aCTrBqB2&={~E1HDfs>Sk5QH_8PY-aR?|tvYMa}%<9imQ
zE#?Hod7L6%0&%f%S62&Ayi_qYsN3%No1{5DfKYEa4%<gWxQf`86QlB}vX5s#?`}s^
ze2<-+rtQniUi!ps#10i^bu0>5%tom2<n>^xEQN7*oRATQNS+7Ib<%(5&<nu#*uWAG
z;Xgw23MOvrKmI4^dx)ZnCM&hu32W~5w%89<yE=xFFc^Hi?u5Xel`W|~Ia;}WH@K(V
z>2R&V5VU&{<y;QV1c}ZED70|6I}y|YQt!?)agNEL;s)(-t?;WW#+^M0MO9G_g`L4!
zwwddo3-w-kO<`%RI#p~r9PQbf6Bl*sUlqq92ci<OWhBG;cb@Ur+PWw~ccpCyaZ|jY
zRuK#G_Y|9R<r1V0IJ~DgXs$W*v1HIztXUjXl;_t|+NMcz?Y2xX2)dn9@3+j~hw4Nn
zdm%x;S$7wsGa$`Q{i09!09e66#lQTnu5v?<c-8aty*5r3L4=IrVZVIy<`M7%aS*Sb
z)sIyEwK{_P|CXweGpJnMNF>Q$YcC1CYhuz1iR86SoeeY{25GwKAsG}2_XERY8hC?5
z+EoQF5V`#VU0wztzEUhOka7ks3sWG>L#MC9KoL{{I1K?_L<&TXGVS9at&YB%J}U(4
zy`6~B4&wirLt%H|J3?Y(?>v6|SWa5_XwL*NXq}e^G{+hK>J}Ceke8QD$}IiA@}eT?
z1B~)eOMTEcX#?zDXVSDnFEM04#km~Rs29fLtzFQoZmj}1`5W+D#ckV!fzM?#>ta~9
zaD6jko^5i^whjp`07Z#%z(Y6KoNZUk(dNx9mKVj^{e()8h$|iSb7BF)IZXwj0<MjD
z_D$FBgq)BL6^zsWt3jX*{Ehd53mf^rbF{f@y*c}M#;6WtA3*!>t?51hfB3o4I3Nlc
ziYeL&p*OB#p?fV2md0pr@I6ztH8XcfRL$1Jc+fjH!HG4yXmWXbb{;?Os-VrJWGIzC
zMO-iP7qtkZHE3gw;xrH6;Qs>Lb`uZC0N&BWl~ltw2}b#=+ba4BH240B>47A$U06EL
zU;Cq+55WKoF!b#@FlKR(07sJrKdtvq!G(0CKfYD;@w$5(j>-GxUOi786teUS2w(@D
zRj!Nom7Lg3E;ZLtLUrg8GcoMC5g@BB1DC|jDWFCoT%bqpPf!8m3LYv0#Sf~30X;Yc
zXy~B3SK1TadWwPJ00X0cH%tos*B4LfjDgPN0=GRI_6d132*2Yp2YtK192mSB8)kAD
z61Udy8c;$2Utb(xNrq_h1+Pm~_cET@apGysVjmuio^WL(U7-?Sz!yHu(-2HNVL?r1
z8di~{;2D&+fOA90t#2{?<n=bln@1@tfFy2{eBCJ*Y)lR8)8V{L8`;16l%q2S>O!|6
z`X^s%jGD=9*>s?7z;?{wZv(WyJU}rni4K!N7?%hg?@lB^y+CC;Mq?ao8-Cv7wqb9Q
zHp+QTnSBtqgeHjxGy<jtHRyCDm{Y(ZkKVvYL6R)>pt5jdyK56hvBXTu1aT(%z3@a`
z2?bpb%zr8Z1n4Ne8KYtQ=U=OMwGr_EoROtn-Vv3bmI)oQER}kdwAZ#oceFH6e2Y%n
z#6#?XOBZk=M#RYJfL`r2l#>-$`+kiO<VZSL@Pw+7BuY!Yxe0f)%i6MHqj_E=Nzs4s
zV2uY7a)3p@`+bGr@c&`&y&tLm-~aJ*j)P+#TPlvo$P8H-#|jxqWhE=JLZNKVks=DA
z$Vw%ptfY(xsjTePFiZB{nV;*S=j;1VeBZw{&N+|!<Nmnr*LAyY*X_C<^R-Ob#_wls
zklv@T>Yd>5`wHJh7J2?1r2%M1A5w9%{HNI<*kDq2t(>-h$}Bz*vp9rMbF8fVfMtC#
zC@9+f%69sYrKr$GIogu$7DeS|_Z11Ri=ag~2@DBEu=(=f4a}q9)rZPUUuBSkdB$l7
z1lq?$sg=J8JU>BjrtHZ(Kem<s1RZwfxzWSdJvnFUXz<S4j2-VNXLLy9y+mcb3{pC^
zDx%MxK!xB#>2u5bhuTX{F>?ZlL|FFi6ERGcyY^sYs$&b<x!Z2qkKneBWcuet0GzZ5
zUb)|uf8qR>SYAme=G0M>t!R+k6p6%0>r56Hne?TvTz`R~c=YPxcOa3ZIos|>m#4I6
zd@Z#9OsfbIjPDj^GLv&s8_(X}g%+fYzj}V`{{>yzvv9)5US87wvn2x5*#VV}LxfYy
zD6lGn&U@L*-`Z$UJ~Kbg*<ZX^P{|jgMX1LvJL!1tQ~{cN68i`MlmF^>Cz?dc@7Gp)
zt{`ppa6XU8A6+@|C!B^g3o^I?*Np3UoGZDj>#Rk&PtMZ&7I3^;X2fzU30AhkPCA#F
zr-!toux}}E{1ZS_BSs!{_|*Ho1IH(s8Or9i<N6*qpQh-oMR#G6_cQdHT{A&V&^w6-
zGyvE!gOLY^o6!X|#c;KT(yPC-0#+u?=ITXpBkIK6;2kg-M_}uotXO6QOp5p~eCDM2
zGFlUNfw}%JDL=@n*!kYl`#7o*78aJT4mfdCfx03C%eZS8lz#q<CQkngB`Ay{D4y!k
zktP3vp0F;u*~LRL^%1x;=P25K1eyX(SN4ZP<n$9Qb-AW8JWdJSypsP4r3xK*2nCeG
z3MKAGnfrw(7V}6UEg2yjE`MQ^M)TNZh&2ct9lRdZ!;G^>Gf;hFJ-obmi($V(`6no-
zw{z<O5Ox!qaQmOXj`M{O;xGRj_dj=C2<=TmC@Gj^a;DIPsb_!kD(_pBfMuz_FTI0J
zu4@s(DSbv)b$EjXTGe(d9+qvxP?40g>RIiy!%R0hZ?&9clO%LvF%NO$jSc=wql#7f
z?of(a-lZtFtTvL&sycPdr9%snbZJEj$^Y_i;t5#7KJEW>T}0KrHmt9gQ;%M}bwDMc
zD-9{$PGC^?kDP1N5j_w^bf%5_sUlnOzUq$U)R}QwvR&=1hx?jRjKKRFclF!k<F|2?
zXmMTTz}3rRzIouIqB$fatV#==?;bv`+m@+L0lF<ZN|rDGN;1R@i0mT|?@;{n((46C
zDJ5O)BosnIcSDvgLGlHE3tXMnGg{h>U%){Q?CdJuJeUtX8i3rN>cl?SoP96ZIyQ{L
z(rO?arnKL@El}m2$heT~%(p!Oc3_QjbZN@|CbE=I(=Rzgr$#pG!otE6wWL+ouaF!0
zRe2ZxTMcP1>+ht~EZJMm)|sKPN1+b@RzFlzh801BYiB#qiVY0%eYQLEU2H@^pi8aj
zHc{}-LqOh0QC>Kf^qzx>N6||_`yItJH<g<&WNxC{n`@kT4Bl(~zD>zTFJ?i^t5LH0
za+65JjnBEHJZaDG^?}JGou1n&hHkRo(P04JC_nj)3;c(}5Ro0txt6_N3%o1_ggEig
zyIJ$lA1SiRyn~->Kw24dG0spyX}j}~Oc-Lqz=VRMXcElBNO|-Yrl*{Qkh%5PZ5fT$
zK>+YT(2^m-+#vOZTTHwX&^)S8_Vpki4`Jv$ngN%^fOoDYmcYn}XrrMv6gh<X>(b1%
zC;vnca|kt(FQ50HUnGrYUwCbQQ|8TOP@EWfSXNGWj%9auqeik<mZJ66FeXGg-4FjU
zHOco;Pfy&9hj}{YP*ftuL>NWfB5}m=DqP7lNLM<?E$?x;<NRCCK~-6*R$*owfeh|4
z6-CbG>aZWap7$L5O$s}9b<Q7hO~%aA?dSd-n?*sO{yeRx_v8$C2MjVF1X2h^G|G@V
zF+Tnfw8bE^a-3Ys2EM__9b8en%mM6{zAdzNOFOp_6#eEim6zKz7r*TH)CG;i_N#<t
zD)MEl;Hiwus<n_FB~bGQwdL{*eX*Cbt++{wrG)e$Zq}EWr<a-9L;d;;J(4DM6)ape
zbNQ#gBx4|c8QUhN?Nzu4cr3Ui2Y}y#lyNPL=fO`296qcEjpwqS1AX9?XZtLkJb8k#
z(_LmJlQ`S<r6S#>Z{&`N$7NG@BwAg@0iA)!20ukIFM@tU-^waYPiT4kX>8F;F}$Q;
zFrx^%<fk}Lf7d&-U{Z-y@rz22jpI)$12Tc5Fsblch%2%y+;hS;peZ4g<_kUD%lzvJ
zae!~9dJBAaO3wolNAkmwfSzK;RbF3&s0C_6xVBwc8-+eaJKp*vAF~cqg6uI@QjX|S
zP^3hsGmwd#-$215qNMCOpxmD54d`D=v=tL<waiO4W`;77p_$IfPF+{a<A!#<2%hoP
zWw7NGF%dXx&b0@ciFibQ8WvrT-#c(SqM4x;;IY12?BvPYPpb_~_7==-EkJmd-|^d&
z+#iddFDG&Mcct!@D-Eqx?N;8|CvyUb%(0b_HUcJtV;yuzB2fNe7QeLd?P6?$TS28D
zob(Kj2TS}UBARYoAOT?X@mB`dj*(Rww9FWsuiB?OKS5W2`}Pgvk>2^wB^qCWxJBC%
zFZ0j9iNqw|DY37Am&Q`n+MrYPx7b|IR>3N-6WFGce+g4Qli{Jb(?iu{iapl@%Mu5-
zAoC`BWh@brLJkB<At@|E*_`3L2(TWR@qt~615vrUWTuNSr-aq`7GpU=#xgiU{?j~{
zxv^UU^xsJzAD^~BDI#L!<Z9q-@Cv<A8G8f;U<;Y3W4l!$U{(6lm@8oJ*Dxp;Eg?fW
zwEl@Oq=T`1^&0_%;GS<&Em}isuK#4tbAxS0wMoGX>QM8xtS+{x3VA{~4Q<!97<<rs
zEjp*Bfuy;D{|aP@LJFCz=j7af2>ge(EK?he&g8uHg5XinDPsALIU=w^?0A?t&w@BA
zQYg*%9sA0^1))|hsG;%pr`b2aXu-6~K|PE3yq)f=0T6^w5S@%C4cnkRr3)o8r2Yji
zmE8xA(?c^v219DQ*yr8a05sdkZJ~@fGi!fss74=%=b`6@3>y)n!knhwzXz`aUm!@x
zSDWHKP)LF0W}}C&VoLR856%unAn<zrNgE<(B6Ly8wicyhbeHb95s4+BrPBp9&(|X;
z2GrRCYn84XE;USuc&1ttGWRP8THHs+u0g@C<CU3Y)vQdqX7zc*<4UWOV^ufmvBiTh
zi>GX<<#=Vc@(kQ`66LZ}2-JRBj`RW#pTff?YHCpY?tc0xkn82yS6(N**ZEiP<pX^<
zAzVUE6OI#h>m1PuMzNN6ZIQFUOVMPVhw7ttbn$nY7vKY@3p6uraD=!M#OT1#`5NlQ
zk93mBVbR=dZFMko>Es(uCT1d$5*RW*fN1Q^YWWkh#VtgrI902o$OO)|%rYw6w4M}r
zE*X9aP~thyh%(^E&A?P{ScMJmS+(DJD6W=#cG*L-)gxQITMgH_3C%gV=YdP_yHd+w
zgd#>4z6QK*OM~PAeSKEIO_x1>EHD4IS{}L~9K<vIzX%y5+tM=GAVZ%Hv?Xe!1g1fi
zT|;vS(p$k}+1r~7_xN?bRv**OhV{@$LrtEM0e@mIj13(H%i($Y=hL2BdRGmEH8QHK
zh5kkNE3@N@XA2N!o~r9w?*$#kbNlbk;jqc1Yz;0TvG({FdC5IKW-!>7bT@4F5X7wX
zqsk8T_`i!ylFxvDc;fOdV0C)pr%}5S2Dh$d3o+k_WDCCCo*Gw76`L;z=^=#m6}??&
zHUvczt^f?d&c1y*h8Xf7Ez+l+MQqz=w6Rjlq!yWtZ};JYiuiS}ELWvWh#*x5sSj#G
zD=uX*aT2C9xw=w1le?hv7>T7u<gT=dZxm!PN)AX4AoR=MEZtn5;J36PY4llmEl^?j
z_zIzh>g07d`-d{rF87sWb9F+H<8*}~Xm|qdb}YpWq!_=5IBT1r1AA5GOiaoE5mF%i
zw3XPztSo`sQBm<ARaTHmhVq89wI*zQn5OOTFLyf%Y}C$BXnATlH?q10W9!~JvVGue
z+l`$(naa$6BDrOcJqC<~$k>C&5DmooS>la_l$6$;{_{eEFl4F(hG{5O1(vw=8l6lO
z9`5|mp|E$pcad(8DmRT-#8+2=#Ffo*t8Ith0R$WIadEbnHW41}FTa-4k<(gTbCN;_
zwFs@4vDQaP;DZ_<*M8==@C&JWG~QrR6_!X6Z5{iC0&%-F;yN$=X@+4LLO=w`lS^QF
zf>S7d?huKnOEZ4bO58ZVDs?I;d$Iy*K7V(;Kq`~gU1s^}6-XKQ9BXa&fP6kQie7n3
zQ?MtdaKN|ON4O-3v_4X^0cyu#aT^3-Z8_*GordzrAUx><&1aycP>}5T;qRBPCaTcG
zs|+)45H5z54VQgXnghFa8YxOk+g796D}F%<vI>!(A$&~e^pRjlOL^%8goK)k9GlYE
zG@<)kRx|)A8K3h9pB=*$hnRt;G&WFwY0tO^kQP?=t3QyW2m`@{4;hW}&{5WR)(Xk%
zCj0@krCXc;?(xqFTa?wQ@<^gO4!mORh-_~tlh3=y00L47kTHNp)3E)ydMV82(u4^B
z4nXL1lz7^-rv@SE1XB8s_XWIcL?U)F9EM^ln?s6eJ&=*RHnA}=E&r>c;{}jZe?&u?
z9<lo>3=khaCMZO1f=u!-f-2>;vDk2!dW48ImwvL<LF7cyAo6kR(Cupp9J)@-A69?3
zEkb;K$gY8wr7w>n8q-XJE+dyC8ImZ#G**H}Y0j56sJaRJ=rK4y1ESf6>w?$u?J>xN
zw9K$>nWx=CzN0bQYd6ae^Sq}|L)hc_hmc=CYVVW+Q-dlAI5x#lEJ8cN&=IH3$DqGc
z3dQ!p;1QF@8kB?A6MLA6)lL}5*YA@ls2gEu&mW*?;-Gx156Xq09$xMfDDpC9ccZt;
zfVMz0f>BYVLYR;;5D9H?ac@XHRLCTeH%jYpX4&%e@tmAvS+HDnI6oHm5@mz}5i<&r
z1Oe`xB=bS!Fw&kF0{*N;+AjUHA5tf`XFwHu4i0P;6`)A~*Vzqjq=BjxV@jmB1z5Ti
zIxj3>bels51@v<B+16>+F+(qz_<@(&9|i;GRBp${%1?yvtk<cTj2usvRJy2HU5kOF
zlEaYs#kE==_8KSw8b$+q!J4=4`tQ5)DKuJ^<9#4bE|ve+;**&P4vVc=cnq<sEiEDi
zO8<e_(W*}{*)SSEnf)fPbCkCGNi|{^BwA<HVGwfs968JcEi*|rA>Y5M#zaP<0Gw`t
zre=8+CO&{G5kO`j>hX6KyPQ#*JJt{Gjkq_m!ujOs<HUKG&h*d&h-8!E?BsO&7m!?$
zr4DQ$X;XLwwC=?1;!v<<HHVOb&mKDvy#a3IYa4JC9vGBTf=-nzr!dk*7k8R?hH`ms
zaPTXp0>bE@e&50Sz}Y+Y-38w}H1?0E*<SASf+~2zq8rfMCQxko--=k;WF1qyy7JXC
zTd%>fOY@oPX{N!S1rjOlmbMLe0&*OJlcbYU$lyK-_h=SCQlnTD!f1GuoJWdcm#BW|
z`=5X7s=d%^`+ofnH5tR!)~$uehL9njCa+Az_o;<cIB!Tn?;OsGEaTKt55g4!G9BU>
zWWxwmQ#O_ni18&Z176y)B%0Afl?{+(6H?O)g>{JmHuN>2zb|PsP*A5nhTu12WL-zh
zQm(?fs_C3#h5#HLm$_=1Z{-pd45XYcQX3!RbT?$h^YHM@|8_*dc36->2<*LN(}{6;
znF*I{3DVZl34CYQ0)iN8R&!ALF3<cl1IE^IF{J80S;Zb<Ou(nWP`9tvDw!e%vAk*?
z1lOM22`sKy?q0s|zy6f@4{+e-Qah0$#h`zKMMn-SmxMm-=|+JH7}35dO%1km=TMSY
zjv+mSAW+?yyFJQ1SZH@+dHVYq;2?#Lxpvjh3{S&YB(wlht1+deKd5vT8RB?KWhHU^
zH^6EXH{}IwFeRvseBH}5vfJ67mq+Q9Al)?yDa)SRd3yf}T|@o=2{Pofk=7@I83tIj
zCo+Gh?Jtz~$?@^27bia--7axM`Al}{`J6-TS&|t#@7NPjG?1bf+a0&&DwbllM&+v-
zrd+=9!cjCPI*q%!qw|ge_sWh`2>U`dXV3~P<q%KFVwviV2+{~$8W*6vKr`c2wkO9R
z?PW~4=C6r7M}iSO!j>uEq(N(Lq*1U&B6C8?=X(A@6jHXZD)(s^**tap{0Ry)WZ8_@
z8Na|j@UXRg_vZ~>zDGlqrWR_A)NoUd2unz`+{p8C1uv(AHXM8`8jb*K7^?Hif)6p>
ze<20NCAYD+rEqlbq&maN_xii&n1Fc;6j{qlXT#{mIdQ&uvMr2r9%6@mvbM`+hJ7L$
z)C>X-dc4W;7<*fp_t@9V$v~MYzOnNtpAZBm6G0EIRTxi(UwL@o*9P@<U?UtqhSQi-
zg=}LK<%Iw*R<pJ3zcN?3mO;N?&cG)uEF0g^O_4>Q|2m~tz>E__=ecE(^^d#1%ol9y
zg&7mjM|1s07pz~`z`#I?A^kPtDXm!{#2s_x)&MhSLo&jprfHa-I|6e{Z>iI{ey{op
zXx$J_IeV1y9nkYB3nLW-|A}Ww2B=J*iH^md^PUZ$VwkJpP&|ggVN<G+7P6?v$lS;T
zU}z#a4ubH{EtBEK4xAo5-MA|mwp=OnjEf3}!S#fWPBe#LPj}ab?>Q*67`722MYxc9
zpRp~H$lZx&MIi4Xwe97EH+|Py(iFX>za6c8QWh}7je>KU1T04_k~a7m3^hqs$==jF
z=H!RS>ma_446h{12|@Xja@qF*%y4Z2X!6ImYBL>4JfX`3zfN%x3snstOv;AwnY<q&
zw`8{;_8e~W(Xe!DX2i?7|LDp(NUMaR3WwikBY~sD5J%YH3*{K4H!VAkXLplTfqkfX
z0nJ<wUq5+s4odDYh(?q1_C0<QsTT_bkE_>>Dc;833%=<vxVN3{@R}vHGk9){DFIFe
zLgVa%={{OoTJ^wvIRca#y;{u}6$3%LjSea5Z>e0`HH6*`xrCQ*0WTRLEozYKN#|JJ
z;0Z?~xnvp8OL24p*O8R&G8yXoP0%aQESb)YYGX&2--p4jG^laOw2`rQpOKNk1Nc&q
z^@BF)<JU<<?pF4At)4%2KB7Q=-i3lpU~L<8m&HrLd?1&OtlN6W!R@{$>_1!!mIn3F
z?t6%!(u96ypz<gETWHvM^7hAcaUPf_gDGY&cN-A~(5rLkI^r=o+H8d1zPPXQdj!I?
z?12qBM@VVXErd3xWsvhfL`Q}SUPGq(jjy!N)L_URLf{xDed6Q{lViq7lc9)al;QV_
z<~vm7wqVSU)b2ZH6_!0;7KioGNHQ8Jie>}qsoJLsyn6PfD2QutddRY38%V8c1R0t3
zVO;^xNtodflxC6g%*Rr@S}Lao7buISqevWiUS(D_azS4ptwjqII<@l1MEgSZ?}@(`
zq(+2|IW}aS#>NOECH*RPAY{{B1gOwA8^U5@xtS?L%1C%Nqg<WscQeJ0hZle%>~=P(
z+wjZ>*Oz9{6ZG0%yI?hXuS3BTA>Fd&-1)#A(O8vUQAeWMS1v?UCUTM;ItY=%)tjud
z-d7_cA~rAVPT(OjXOKq!_MAhdj2%C6q_JVCx$5-eH4EEMI6r-&x0hSHT_w2(SQ%DQ
zR|E4Ngv_LK-aZ^Cot%9OMldDAOQpDveWk@1?sdS4*n9}eWV*kD2!~@JbVP(vgB{ai
zKpy-6(VmHr9!T0qAfe;nL2)Ek!k|(_<)x*i*>t)1;Y`O>gTmYaDR2UfiiA&tX4DRm
zArs1m%(yG$(5>IRi(O|<H2|?K2I%iTq>d+pf`Uk~QM1S;ciF#z2zROE2~4#DO*O-Q
zIUiFd5T}!u@9ZA_4Q(CROgLSr3TGeiFJ6*y$;rYHzw*(UE2!nSkg>HtbYG1(-hL(_
zC@8olbg6aMY3^jT`^z8my$%M_+8SBH-bYT~?yx$`eKvB`;GzDge@2}!gYIF5EkoVJ
zhX(g_AIdn;i_q)_^t!u)4xv?LY}$Laa<gwUq_tMXY3^;0^=AKO?r>lg8HYz7ii<)3
zyr$*5%OZeK*#&C$!x=TYlN5Df=vgF+nSkL&&Y3>Y3d;(K@}AcFq-;qr{`k>2e3yvr
zIJigg!ls%BHc--W0PfWB8UF^v#dxK_YQi^tAfaUX`_()nad`4_G>V42ym{H*NsFL)
z2<5g5S+=FeqewcM0@SB+?K|$8C}bK`-N)uQ!b0|K6tgZDRw|nT!gK`X$LwN2#y^xe
zABPERk>JOQwQRxrbd+AbXaOXp?FGswEF8sbu%t+0fHrEX2xwQ!KtsP&{$ASElwRw=
zyg%+SX-4($HTW>_{dIOU3nnC3dYTjNkeLrf`5n41a<#vMs7w6ST_AZZqNP8DWObj=
ziYvmyM^x2O;ev_oYCG#syI!8tM`-9I>wuh70d*dG`=AAFFybKIN&>J+qf8B2dOk#&
z2#utyJbP;@*Fj~Btp@pk(`OwC2$NS@Y~OcLHYi&@pgi=T-wiFoV?2Q`&NwzY{V+xg
z;M76_4LZC^F#k8ZnUx(g3ebzKmCgqVSh|1!M)y+hfBHr?am%wOV7@lc^)?y<bjJ<h
z_P-EqcXzf^*iedB8V)DbgRlUpFYNn0>Yuc?bGvwwlafYPO64R(lf0m7^{1V17(dpw
zp9uvFaH}r2;VBhsJ?a(2Xb}!!L$+7U+)f{Z5(&A89;5xx;w%|uYu_9><iH0v78He2
z&iE2@P!x(My|ZHkO8g;VP|&K*9DT2i1@pmap@wDRE^`fs&%&tVeIVjvleAVY#8)vR
z)+3F>Ao4Xth4`Z>A6gI!2Vj3vk<=+bGJ5+mK#M<V-YpG;7xQvp4)YWk0iep*zEQ(>
zG5Bxq+*8@%Fjd3%vs;0gG?z@n?~J|s>5+ic_PTy^?(O`OLnuyf;dWcSTMG=YOJSDW
zy%#05A*^8eWM&2b#VnrMI6tn(GI%50E|&y9MHO4cHTa$^(k|Mu37sLM&hfU8_9`v^
zpyag5^nPvoh%PJ>oUoE4+KwNwr7SjBmI<xJJAK&vPq^v6vd*G3WI`gaHsa+TpjTe`
zSnr26R^&LEC8i6X2_ul=oaS*{ploInHYOs#>l#4CxPdpRdk?4$H$6~!-m=6)9As8K
z))-vqb$GrJ`PDLe|4o0Z8gfYSIdu5GAK+(B5PWC*9;~5s!(Qc=AGyI-`x7jP@QfME
zsRQ>W815wuU;U9fKcVnWwb}@l1cOv{UYh2)-b5cclWkypfrqen6V%waNLED*o&%SK
zNEWhSfNDIyPR#O^zzSr$BS|+=JpqO(vh7nUBkTydRXvrUfjnXYhKz%hT7QVo(+kg~
zSr#o(u8Tj^%hh<4?L5I_>jd2JGA<4u$s^MPS0bFmW=<msEjyJnkbDo^MnER%a2SJB
zUH;_0X^ASj#vz0(NPq*wMX_JrBR8Riuci+u+Gjxd-@e8=Ee==IBG6)u(iHd*Wemvl
z#@|{))Iw0P*MR?+b~|k}%bt00VBjv^;xJrYGvMMVj9(b@MH5hZ)yC!frYeW}S;&3_
zq6UFSxQGGn5QA6n8i^D$FRU)Q3>_dhjWQBd0<VMgBuX1)1#&8b3^vk%Dmd+N8O%ic
zGQbLraI^%>@~sdW3+C$c-ELdnYV-Gm#bXI1<<kImlYpl^3s~|QO*>L2OB~)aNfh%H
z9a~TdKS)x+g@FonJPZuI6N~;xXJ;U8UYHq3p?nhA*>xsxQbFwSr^tQes+|pe$b21`
zQP5I<KXiB7Oepp}ys<&N_X|`NI$6Xn6@Nz_#_h;z<P{J=S+kXfsll9QL}+D5EAXqb
zuyxOT%$e6g^mpw=F|1D?Bt;GY#KQVc@3~I6(g8A5h%5u=5kvywqfAl$F_bd(JQG$O
zy6X;kCLV@^SdWJXT91!?D6fb86<w{iyTuPF{p(y-4hbt^1*(!cC}#8=Lg8tVe_p6m
zIKOvJ1n?ML#)xm<!MfssBWXm+i{NQ#e5j|7>^)OQ{5rg~;2CuCb--}wO6zGDG$HHS
zrQ4#qIh|NWI1hFWr|sguUSHF^qL0g<T(e>1Rn1j74M)w*e_{|khu82;&g_c<es1t#
ztuXLqy2keV4#m|lliu!v!=vEfoTpV7OY_cxT01(LE}ZfSJX7#RN*FHwVM>&?8MLH4
zgBhx4eH@pQ!q!fK>5F0r&>Qc51hQ*^tCLZ9rakj|qJs=2F1q|(+Kq{c{{x6+T{_rL
z9KlIqnJ^>E$9K1uo>rmU5QAxh?b02ut7~B;C#s`*D53oIu8)cN5&m_-z4;PQRP_9M
z3}`MMem`*n`|mylgfa9t?`1N@^do-Qkon3j(;I8VS#U1Jrz#Zo_N$VBirs-!{=D_h
zdeGMTRYiE(Z`Y30qvNpkf2l8W(88xC_Ja*gZ2n!GjU;N}0A*W3h02L}uW=Uw>O?sd
zEuMiFjSg4#U)1Unr2FTXxsj8D%$FE<cnFd7!Re<bCQ9@6F;EXkBA2cf#jK7@;at#?
zqJlPSQG!!@-_qbG6}r=Lh!y&v219o%(-f|1NISLU>Y=B%7nl^$GN9Nk=gWZjOW;Gi
zQ(WxbbV1B({`sX=wevk_cy@QE``??%2Yh!-vcRImu(WK#_|4DXFnc$9ke}4t3HUh+
zBF2w|mv9JDxQN#5$UPztIu^y^2s#pT=sggs*E<%l>?{H&bD7~a)qfudN8v2)lu0p$
zhuA{xH>@5FTGY<pg)~QfnveB57lcJ}JMw`U1OY}t52|uszk~b4NrqEoo^V`n9m*w>
zbZjCfTKEm>=}>^0Ss-d%sFeC=*kF=U8HZfvVIIIUh_uK%RAQb^totwZz|qxJ!4B0Z
z*?q(U4$nKF>}L!>LlELg0<?@|Squ3chKoiLskDcHii?A^*is3Lf6JA0l#2i8Yd&N;
zNUOvAzU70%?m}`9W>_}=zFyNJ2w<wVS5yC>r64S;3)A7mb{NnkJy6F;GJU5yg-7f;
zl5`yN-goW+JnkkiQ-si#24x)5lF>)EB>Znz{7#@8h>)l_LJ=w8;BlbZK}Z@iwrYM4
zfrdov6CGQ~E+2%Tt+7(K$Rprrh*ie$wu_mg`?uN6;XSnEE-g|x(e+8Q8#9?rYY50^
zCfj*t7F<~JE}*kFTtNnH|2dZVTh$E-Lf1S<K;6>Bzo~>qvJM!M4t0*`5-N-6f7d&t
zfm+A|04lH^2=gZ<UYA1J#oWF3U=S}TiGgZZQ3W*-fB8qTp3TscXo-$?Y6C`0Oy|Pq
z^gL#=A0PO0Yu%!QhvFJ>2NbRl;f)c;-B@nX@d|*(-{h_9xo8xLZqULh{eENR??3RH
za6*{K5I821dw{F&7(mU_l6+A&|1HOa8vW@F5705IT*yFNMxn09n$!*O04IG&W_vgi
zk2b?X#(L?nFqJyA!(P8I8?P_%+KVkrKp3VaxCQeFfVE#kti9ZC$yKXb2BQ;mpj|Hn
zU%<^O#%A)3&9OD(0z!a9-_(hJ<lUlJw{-1)n~z~eG@7dzE+F_2<^C8A@dN5Vd(UFP
z%-_-WK#y>dyj&385u9rA1bx9?+#q1oafmITu{AJy!Dru{@1SY)!n28)OTCa_f(IM7
zh&UwwJN}f1hG@{{DiCz4qdH5#lc~R;4SxphXi=Yw%wC?=gN7C5Q@yr(HVR<wE{f$j
z8B5g7L^6VC4)HAH6h<;!h<4iBNiD)#d@_Xg)xNOjm!ZVx@U74;g`k@^wWI_`6t0Wk
zhk;7y8(bJ5HdEUx6zaNsnvB&q2#~JbJ63N3ANm9@%O)oeU35HzS3`zoNI*l`I}Mb$
zMUhtB^IJu{CqrfQ2B72@54ddoLbCXt--#sRG4mfaa$WB<)z?bH(YIrKKuGr6yJXxb
zZVQ6S5ARpt(0=%3gO~1rJhA~^0+X_K_aFgr9(b}X*4F1uz>2H>8risB>Vx`=-%u@~
zy~C$?i~9E$el0=<0$_wvE=S|E$S66<<Z=8CwH*nGcJMlpB*wKfko`uZh^i#Oh1%`X
z|KLX?$qZY#PyOQ%I8VKS3V%mPD1pohE@*zu19k@&&SJqpA7LD%$BoEqa~vAt0}d3x
zr;l7nPx{Zi3UCCLj}W%rdSJ_Ng^bUUr&okLJ&U04O3a0LV#U8K%InHiq%?G?=H5Ez
z?A3K@{VqC*((0FWO#gO1P6>z5{0OFuG-Q|~foN?)YbMA0n)QFC{)=1<JW&?(2ZpV8
zCkPtGLo?==rxx$j|3SRHUGljB565Ro*q=W{6XkIQaDKa(d<6<GG}rc6vpf}d9Q{Y_
zBkEj%^#!l7mYI^-H)?xdqrXuNp79#J>z!^~yBrYG<mVj!U6q9u9scXUQTuQ~UuXx(
zNS3d$24GJ7@0x_0cwJo$+vQvPM6lGUhcU1D3Y|Ky|MhQh{a)zz_OeyEWeksU?*6a7
zF6tuw!IUDW%PJB}KctP+Cy2hIdo9Nt$wK~ec<xToiAIhwg^Z&0qLq5)Z!1lv6H7$a
zhIo}%U9KyMee3o8UOnHPv<Fc7EabBqq~Cr=(9KP-%|xsT9v$;X1SgRS5f~m81{1Qs
zu5v({23=4OPq2`0evDTqt2bD(o;Q$b{ArowhjFz(q5dKkWnot4gn~R-H5PK0PPJzO
z2MxI^l-loe8}4h!)WxzL^3RbiEBamz?!y^+Zdiqw=PRs3D*&qq5DcqdD_^Jo{x9T-
zGtd82m}RG9!C2qBsB&nCXnV4?-Q>lHV+yYR{JMReMaW@{OEhUvP@DyZa;wfy!YT!L
zFbC>b=N`VUL-rmmggMZZ9yxy;u{XkN{NC@W{P%mX%|r~}P>Ei2I1y8+ya&QQ5J|sF
zC3z^0?>+ZO8rXoNHFb^1Z;@r;UVfLF5p{8X+A%A8C!Qw!-_>cB7Ykp(ht@IZnBO1#
z=%=xJ7Zk)QJgCF+itatrjsxhbyT7f`e0)Rfiaq~BgdwTLx;TNhrrLbKR6}&I8=Jq5
z7P9VLFd08&&chm%Yk=5;t~Yi|E+&vNJe*`UVjhdQlQ<%7-M!-wUmQ`A={<Npk|cge
zVK%z1+6exWh_%-sUGz7Ye=WVYmgaw(iT}TwS3wTg|2_HtGyDHL^90QQj~rfy3TAg}
zrR4HoyBln;uG~*cQaqzESg;p8XttGJFM-d^LPPZEMEO|9<Fbxogn(p_%QF|cnSVsW
z!rS50?}7Qn1tK_AW#g~O?$aBNsUH@iB_-osQE3W0v6rn^64=DeOr#x~<6V3I+>9hW
z`sZ>*|38=8aSYhvj6|bj-1U$9H?h(AF^}%*k8iI0%+l5%Jhl<Ko`HlJ2tj__q!tLr
zvL|evx8D_A{myMjwVZZbugIY_<%`O%KNq!O5xKbRknKktxp9w1C6nkDN8^8H_$#a&
z&&)3!=`3*6cu(H=v9ekGppj`H<H+{-xh3uIWmluElGS2;9&N{@dC!*Rc=g{2xW~WA
z)R4&YRKT}p%jEY&#PV@YQI7Ya{eo^)7r48=DvqV`m&p1_jVmyPBjL|)o#%ZLfo~`#
zdaC`5(5Z%u=AVu~f81THo7+5W)2^RY?WjHS<O!vsRQaI?O4PAD%3^^fI~=R_7A>vW
zZ10n9iPs_Q?tvd~Y?Mq3gg`~{;7zxKqHRM&n6L2Z&wEfoD>&hsC%EViT8O2?_jWJj
zQSY9e(s)nwzZ#N}7Qgu`dOCtab@y)_^WWel#$ftC(TQb;hLk@fzw;jXV7$mt_Dw0&
zc5E)dhD7n*Ycy`_Gp)tg*NB;^kb9xKihTZbHe!m4R?dt|p5eIa^GnM)XX5o$G=-m~
zc{%K=e=i9xu`ak^`7V6P=VM46cd`4kh(87Vp_75mYYoC8xyqC#C*S2-7xY#R|B$qr
z^iq`X^dF8|2*P1mg2mJp*3^nFZWr*)wrpQ{GFLk|C2chtI)BsZ>IDmzqgP3rhS~G&
zIh8gEima;LW;vz>50rnrzDjFzdSv}|)ySVI-g}{^wq6x<d$(vEV;s|8n7TAVXPehP
z)8ZtHEeMj;h)QI%j@628E!qv{_bi#=Jv&n&_Rw+srHgu<KS3D*XSHkJDzP+ZU#qZ~
z{Pg!)1OEP`1IL48$NT=4c{kp0e{_F}x74sjX(~5!Nf}VI*zT|M{%_7bD&(x8%3DhE
z3>Qzbq|CfB@WJD0nnH-ik%4o^-byr0X_2eP_yrrI3P$zr|M1OvcubeA-|*dwy@S)B
zDv<#x9x#4<)mN2q@tG9g=Iwt^34Xs{^r5S{x1s+oPtsNrJEL$>K##`0-rV=sa+}P4
z->+#s+~XT0<$lr7xWHF*s!wQpFs@)Q?Pb;2*QE0SE$J`VJ5A9mkC)sT9Cu^-O-gj=
z%7*W~pMJish{66KisY}_u9XCwob~%)WPI0uE>vR9_u%8$Sj*jWZGW{?3+HPKMpAuM
zAF$jP%__wdYTw-a^J%cI_G9-(WsTSO?q?q#TtxdKPi*jMednyN+Ej=)U(D$H;!L0P
zg-eldcGRY+HBhCgSM!a=1WhviCc_z0wtMP|u1uPl46p8gb}5=_C)Au~pf-_!UEPJ3
z{v1iab?<IdNZecRBw8Bu4c+C?ZMhhoP-?wnx;QOTU!Pv5gKI%=@n>wGZ1hXNU!1~I
zJoJsn<vreV81?p@G0foEa>a@l=_)f9vSK0&kk+2}a<Mn3_z&(c%g~R$vAcgGCTQhH
z;19RQznw02KB23)M5XZ$R?{ypE|>yxj-&&>Ra@(vxLSDP*n_UnU4H|{=|0g=o|B6Q
zh(?(Y5~!m3<NNe*U4Ad$b%p%-OQK78m%xuT9{xUj@XzFN584;{8U$3#iz){P4T5W-
z{R>+h+8kBODwrsHa8SR<#!vl4r!o%}S&O&Hu1juw;eq;q0JZN-w=ZL`UJqRh6A`^R
zuho;VPR8R<i8dYL7!h@xLXG9}g2BVC5Gfk8kksj*wVw(++PxG}@pv4Gf1S?i*iW-y
z4aT|SFXS0pW5>lfIqKcmwMtW5ZVV*eml@St^4*}Z5*EpcCZ*%KDt@sxq=_H3cZ<jC
z_OA&%wQ1gS6?b+{Ebbt;&w;oKW|@<l_J0Zz<#_4e9<f!a8cqHDqmh>nkH4N>Cto;n
zO8U@M7L4)r)s5@%BxNc4AB;LUEC1T=j_GD%>p2z-m9}T-wgHZi#ypVcEbhb{b-lmB
zF(tSB@9oVBH0zk<ExccrCV!6g?v6r9%P2i&;+oi8K;4-l-=7j?GZPViYLcEV5v0DY
zuSn9Mt5hE@_e!#0ZZ8;sDFkZY4JUGq;PtN((>co7>Pd?wgF}|TA64yKZ!V9~K4gE(
z+A=%ben^^zC17iD+fwoGT$Raq<1Zyau9yfGsrgj1yfd%$n8vSYbB?yYTI67hZ#wdT
z@tpRKYJkD#m7fmjI=LpSt!xxuTlGz+v~nO+B^8yNW%RmzLVKBMfa!hc_m@^_YxR_k
zl-y-35+eQ&ja(g%@%*aw+%MNO+O0OBkgwgHom)ETdGd409cn((i1-v|KECwKA5jzI
zlcr^sEv@$}{xseGy<4BnK2=Pofa5Q<LO*RP5Mgy-x3g6$obpwi|2elIYBd*kK={;S
z%%a1fTz2G++W_aL`eoaag3WXO#I(cJY*%My!h9VES1%k1nfYrnAN6GQ%#R};y^=q3
zFMh-E$B1L~!a~l?jqu&){2lZbEm6>i_U817eG|QU@7pxfwLJZNF+sDlM|e+r$$V1X
zcCQsw=Z(oxs4SJUT96d65F1y`H40e2vRutu6xezF0_#GkOpenp!Q-I=i-U6$^o-ek
z+3aeK!0$DiZF4@Zj(Q-uPtFYYg=~|wE4=AD3xSX^XN~awLodty#<CbDG^gIM<X(1`
zHBaHR2M&%yb`}wub;4y%0Bg`};Ta_Si19!{)yLzVBM+p*Q5sUyKi{3H)AwjA$U9qJ
zbt{Kk&H-O>)^}_96(jEt$1i@B8M62J`LH2_i>cEef1Lj#dgJDdwl-bPO5d~ir6)tl
zf3y3|cQZ_<(ix)7Cbk>Yj+~{};JI)$xXfWm{IY4_(}dm62~>~mHR^14UtemmKiH(^
z((m((bl`TxCyn`@)B8~<MMFI;^|QF-W4t}@Jf_a){v>j-OPo{AI?eo%-TlJmo68rL
z-q^OzuRRJCx%ir*Xr|7lD_}13!)LSXm*%Y!WkQ9jgNwBR+a84F>4m4EIon@6-e&J=
zm)vmb_iRrj1+%mqR9p4M``pa05c?iv8ra3X9%)H8CVBmK<GFkKb-M!<u9dULH7>EA
zv7R*#jEFbKp(O)Noz$-91PiUUL|mWiye+#Bh|Ruk-(o>Mc$#PKs8|=Bm9^GMYa7)O
z>#unsuDtpM7t1&`hPoOO!on--hr<+BxaOoh2>zbJWQotX0fyf*H?5@H#>Cn;{qGni
z*_=1s7QGlnwS9Lq*qa5zC0X*)m;JZhgX!;C&SmlWRl75;Mhf{~=;FR|zVhuf{K~^S
zR#>ODR^gpzUw@t^{<BDR-M*d{i!VWSI~S&|CGybvJ?0rzEm>&mA~Sv+VUVhu?={r1
zQ}f*2YFcya+6j#=-@O>Gxl*!z?Af0u`V^WA-<wiZ)*a1ko~=-n`rc_x-oM1wSTGPB
zb+x(X;urlpuj;yacIrHna>{1H6Z1{=F5XyksSJ=m>S0!~d8vkr`{X>&wDIrNqtBM=
z?}<KVJF{ln!})j;l6dAhwJ%cP-5-}CcqHNvU<mi+#Vm7{MbEF4Q*68nI~*nGt2mWr
z#zZ#17H&%YWQ#QTv)q5st#bdU+erA4b1OgMmiQLaY+K~)dt=pkOG0*N6-xsPdRh}r
z9}Dn*ZZ3|f0<<s_96c&SLzZ$Lu2Y#;(-_K8pBecP_$nfY*^hgGGIUynAX(XOb}43)
zD3WeQ#a2yh_X+as#ah(vf7cnEN}}Bjt_jOm-+hNpWjMn~F*a15*KAg{_$8{t-~vSk
zedN`sn%c<oVn%&3R*^(&1<Iba?x|jrg)ZKEml@57iJw0m{h%%R*KYL*>E%Os^>XT?
z7dhVI*tE#nBy$?F=fIA6_N#f7)#QiY>e<ZbA~|(@uN@}hTfI*f97SLInrB<|qJP#u
zXKQF?G;U=r`t%&_ua1ZW(>cbDgzU-#@fK@=r4LoRu8pRuw}z~q=WVFlw(h0N85pc|
zD>(A1%JlWs)axZN!zvyML+t)lKXDdPf0|H9t;P%PYG>Cr`$u*uE7Wu=sN<(|$OWZG
zR*%~9qdT8d478>4yrS3*u74iC*xz58e-`9lEhbtzxnYJj1(uSNmxx7#f`~JP^Ot>p
z|4tFON=Gwq@^EumYdW{rs{>V8=))sF_}F3n@;cTKm2z=ZQtn&qOY_*s%4E}Ddn}Y?
zB~66{ur;@%P<0FQrz8&Wkr;NzIh7`EW$bTnNJ(PgLp@Vnq!Zf>%9s1#qT(NS?R4jL
zVJ7m+*#6XFyK`R-m#91yb2t}rl)r%)V~jPJl{OINH}GFt6Jw`ZvSg26&bYJ^e=aL|
zv3=`iymOcL!M8=9t(l@^&DRG8H(WODrexy#et5rRxS(=j|NhFSi}FOWJvQ*Z+#J(U
zn~NL!>G-5_YedUK<y8Ypl-n!)jnz^fl<NqI!6rJE<&Wl}$6m*ztB$xoT-it@y)6>K
zUL<fmB7ABImhA4mWjcBwF{u9PErn)}FqL!Jmn{Ser0Kn8Cjz=%1;9riI6Jd2E7@Qe
zJHYDGcc7YwzN~;JWQEdD%3wxD&h|poTF#{<p+7C0JTm7nI{}BW6|)x2pVbR9=Zx1C
zD8Kc%{!I*!D=8hFto;!<S}KBe^R1{g>-V{<EMIZPT)M`;vTHrVx^B^#2F->4(DNdC
zF<wvlfy_5ez7U%wDzD8qvB{d~&{QjbK_)sMdQDR!)^Q3!UvBQgL9sKH@v;FkUw3+C
zfBgx5!-|1y$aw6*C(k|!j|}FyiawhMM!P3i4^&pAPL~gUw%#}Md+odapUl?@)!qYt
zwdRjsr)8^icelRMvb4^qS;iD_C!XCVWbC?S<KXoh+$9-4=WcNyeDLN;@PJQ-%;3F|
z;E^{6kI}fs3`;JH%By;n*dFj^eM3!cvD=-zHWGM@=_jSq1I{`h!yc&~Yt_Cn`INPe
z)2eYvmu5e?U}{xyx<8z{zb&+gQiy%zytqxre|+ld=K+a3JUUfv7W;1RIIHEV-O%!L
zwLSifU-9_(0Y>Lwlh*blk4Dsb{JDZOBCxfmbzfwCZ0oG-DSb8FU*7pLnyF1ux&6(}
z#M4<PynfxpM{4UNTiJX}uPtY2{TLoB`-hpEB|vm&nk`PHf6l)}iT-0%f8A4u?35<8
zh!3M`PpQ~mEOcoe(HePJWco~s7bwh#36J_t0j!2bf)D0_OhKQqP8F%}rN@hqu6haH
zoa7p|8#uO>Bw?z>lb;7>OiaH|jY{@5lf-A&zn5kT4$df?<p`BN<Bw&#_wkuz?~U=+
zXfrx<aok7Nx9QTd^{)HGChznPB=J)-O*B)Q+tPn{B>YSa+k1nZnyRqwea=nCbQwB*
zgSZ)wAcHk_+TCA`KMvmc{^*Eohur4?k241Qsg9Y~*mjPv;c2*RrG}DnLMktL+s$BK
zG1Naz=hlB){x+@KeuIaOuh6vz<Ny0v#sf!uZd%O2oinD*X$90yQnBW`3e<~+qP;oa
zZs#qJFgCH9|Mfg$s%d*jy)-lERl^NR4%h27x4gS^|8BHp-&b>q-Vr!;cWUPIh~1x;
zQx1<W@?mXVWPLx%Hi!SR7*dsdWq$GlkJ>p&RieSn&ktt|!|N_w9Q+|ZIlohTW@IR%
zGHTr-SN?SOWY5{wcY4(-L($)Df6FB9e(t(fGk)mxfMipzt3FS~qvUmUiBFmVgM+#j
z{FgZjEAQ5=;9M7lR^IRbdzm`8qF?A~E>~^#(E5YQZW-mAD*Lc60$3Uy+*W4a=en1-
z_wjv>O8lVVl~YyN80J_%aprk3wch*K6IqrR;eMB&p?fgt8xNz}o-W`JLSXpC-RV$G
zj6kt4KBdq49KO@o`8sXqa*yb|ZSE_%PUAzGN?UWsnx;C64@Ti%oSiZ?6eI`;R2T#X
zZ{|C$q#xiz)$(619@e8ftRyC7ce7-(cgA(Q!lXcycM8UOD}2(=T<2Oh*RQk=voE*0
zd*nwgTSkGojK<{ZWf6JhAg8v&YU8RWcTV?AHC1hy7Tjm%4Epuz#<G)no_*_$Yt|F4
zt_S7h{0?$-*<z_XqVUPzJmSQ5d>&Z#KH@*eTV+--_fE8c^v5RnIlH!0Vb0H2j&V9i
z*luc0zU?+Y^dR8wt+#LEnR?RHg5K!bUDI&Q8A<b?iAUGz1lNfip5UR|ms|2$?rtKU
z^aRVZzwl&FtEqpkeO352@9eWCGf(r3k3HI5)CJ+BUKP>&yPMog_MFl5B+8=|Uc+`w
z1rB}=w(k9ZqkC^CD`Xl6nu(D4)|)v+&TAyu9P=i7td(zioDN*e8Z43+N!aa}GUtJy
zuvj>#*}QB{R*`(dK_dJtU8ZjOwXe}hI4?bH@UTNeals1D&Ud?mY=3*q)VwQ1ogQ6#
z*OO#x;L*1GhcszDD6fgjJUXs8zh5-a&9UL-&c}JiD)9rJYYX+LgSUeRT^@N)r%Q*Y
zoG&ed4A;3oN+ecABC+}&9<zSov1-WD=Ob1*$|5HxFRcno7Be*JvY%{EiSjFZ)&4u%
ze@Zf>vyM@SxuPv%n1^o|hyHpYS<Nn!))Q+R?vEZgmuj}1^%M0ysLnffvUPQro)6WJ
zTRuw|Ys%S({4~Gq<h@dX+vb#tiQv*exvTSDT}eOZ_`2g_O){rf$ZVQd<syrg4}+G=
zl@=Gm9UHf<?vJHbcN<-IADwD_^@8Fm|LrRWHqc+UxyCX-)hPs@oS+-JM~Y-W(T79h
z7q@x+^{S}bdLK&^-io0PAmX`k50ld(G>XHCxk?(|YVAAPEv9{<qcU-|ZO!T-DJ53B
zUtSLo;{y|98jE9{6oT8%FZMnPbq!|w!WDirwTrJo7rjJ3;29pOQ(E=jntq2N`JKl0
z3S_-69}UMO;BgUgN=&adPHhN@bRKQ1^{R+3BINj)br;S?-ojTq@=SG{6&Gy%6dC^c
z*f(9>=-U)wkyP~IV;Q~~jK{L*w%2&g?F&ES%?E6kh6JzAQ5B9A+M*v{y}!sslHPxE
z#_3iulWlkPrw`rb1s*Mi=qNpI4>ynfSOJ@z!ZEHqTMyanR32Hws!TsBJ~B<tc5!<H
zqj9!TR%j(O1l!y|Q>8wcT$xRZ_*wO0Bg5fYOmo{%x%;xBxnE63lSDEqhDRZ*|LRQL
zQ9VsQ284@|b#nn&e88QN3?Se8f*5U~0%^!%q(ZnvIZxRJANKO+h1PLB?10!|Q8TG?
zSh|O&TYWoDy4(6RB`}$f-C;REZOzrKeM+}SZ)MnUMWUvWk@Z5ag(v0Z<U$;J_?+%N
zI`ghi`d{Bug*|Q!TMLV17rRnj&Y#O7x6we;xBg?$%8c{srpmV|doqo5!#-Eq1R0J1
z{N=5o`w1NXVW|qr3PA)*-PyxZt->??V11o8GMvS6H&KTb#Eu&N($~b<R(9z7TU@W&
zw5Sv;+OM4aPLmm<E3g$Q9W-HH`N7mcotfsQfB+jGivN*OiE>U;sr$*-Wf9lL(j`^1
z?;Doe9@vpt!#HBRGWYTGa=kUUtfu?GC^e6UmJ4;$>sdq%o0zd3_C|oQm|gmfVAhIw
zJ9nm6n|_ge*jX17GmmpUWkzH#ijzL|FIMy(+|Rya#)RJFp3lf$8L?hz#U(zR@$fTH
za}w^nGI!k~hd$c&3YILSuxfq6mXfKJ9!GCxTg_r*X|t61E+{4uPXv<E-?80Q<DG|j
zR?pK@hx2qa{VM|xW9`4c`7r37^)+x#hM@*nr3saXH57r+61V+G&OxJ;`8}}=*Fj0E
zJGO+rM|vwCva}LV!m@Q~TL37I*jsq6XLsJ%9-whs=V0AD&tq&Hr*S%Z{-W{acaL^b
zM#==LrhdDbZt$qA+R~ex_x9KIALn1A{KCui#x6A0w^0Jz%W*!El-Bx;JrDVtN|zlW
z!oL>x<qKo?r0X*|-3xK=ucL>%cSePlgTog+N|GM^*gW=7X1KF392Jr<)5UdnTTYpK
z=$HfNSCwehlpMClu{V!3YVL1t5}YXf$c)JjHSRj3ar1A#hVLS?kNL~F`EZotFZwYD
z9D2F)VVy%%QP%GD4xeB;O;ba<Hj!_0qA9Jn@TdipClUYRgj&Q~^;f>suEA&x<x|e`
z#{0W}-1Qs&JYQuu2mbJLJGs0-Qs-39J&HEo=P_8Sh&E2U5S`>q4pu7jb`3nY)8zyY
zP+?t=*RcK6Supv6|HaR(nGnm<F-0$q=j-{hhdGxCDwGG^XX9ZWOJfn|!$-%o`OoN9
zQ9M^=lI4iQ6wd2-2rBdADkw=5k=)@QCbMuCON`qC&*AFxeU<a3h`a&s0@fy1T*S=_
zYT{VSbCuFlU+M}v`4pF5!S4QbKM<SQl&`IeEB(k4gjWmmJ<iz9(KRZTTuM9X7)IwR
zT!7ZZl^W`<(o}yKmYRv!h_0b%Jg@s=UP8f12upOZ{Ux27!iSx&Evj4l6h_Z_I4jj;
zC`DmugP6R#q5Mtj{jL7_LX&XOsUz=67PD6emoRJTe&-co=8+hN_(XwnK_ER^J$zOj
z$6I)pOKL%R2E&3&O)-8jx>~f_itELiDyO|F=Eb26LQd@S#F5SJcUOGp`t{+C32zp;
z|Jp#`n|%iAY$U0|rnn9FKb&ucbt+6sT@+QsryYN<?Bn1*=$TVMLq9f0k$TIFSwh!+
zM8iVp+os(h{Vjg%^h(;J;i6l<ygiavt{KoR>{U?22QbzNP(jf;0RUwExf5G|;T;*b
z)8?IaXB2j~C5!)vSY8dth0dqc=AG#0=3j_~XSKM#7+AasZ|1WdUBB8~?iMX8eB9EA
z`^9m9C558#hfY)7)FA}X2<x3c7i4^XnO~re!aaVKEnD!Tiq8vE8&S*tJN5>Onz*6k
zAH#Z1KW65;(h#B1;y(A3&TqXHkGXgDE+?NYn_#J+mxjVcn^?c>O4A>u7P%i?5B5w&
zruLurkNVbIWgZ=7P#tKIXD#kueDQZ-3A;ES>6-G#+g#Ep&T8vii-&<L8(hQ8c)=Qx
zTX>QkezwVQ_T1xpeQc5*j<quukGDQost@Bl>G16LXqC_58`-E-XP(y(#HE&nD@*|`
z>yM$;Rq<+i0SBA-4`bsJD5_i{e{xY!5_Bzey~1Pou*NM$U-?eKJC21;vo&pA!lC^t
zsx~H^qv@;P@)52-cF<ACw-nz$d2v)N?mq4`ee{y*(f}P7(`e;xS;gSB@QjNaM>OR$
zrIX8Bs!j~^v}oEsk{+6V60fYOu|1IP9<c!!Wq#5y)@|`=qo(gkl|L%*hV>u?CO7Yr
zvN}A*_k*!c1kIaXZ3S21M!x<_M-JW(V0;|~K=`0X{6~K5zQn)RJyCB|iN$!ooqZEf
z$x4rAwx*Z7GwaGYRh8^ycL1e-O^D<A!StV-?!5Bp;i$zdVk^4l*j4xUghb=kP3yJ{
z8T0qwYEOM{dia;&Q?s#lVR(YMPx$Z;#%lu^P3#-2xv!_~<qa4d1>A+O#=Sh;ZR3SQ
zfBf#!Hgn~z!zp{Wo2?AQz7{sLrZ|ktT;bX+#`KE?n5JE&DCX-DC+PW%H}8*ie|+_>
zGHR!<<ocNzBUd?Qb;`@s)!`45)33bN3{9KP$@4?;k>*wITI+Lbou+yq8nMQ6)|yeC
z(!7jcgocV4CzkGMHfv?uhG!8{tieV%Q*xAfsF`;iqmvI8)_C#6?{+%OIL-b^R2lu;
zqLl9>tRM3G7q#2#fhEBRS4&=<U-7lB6gO*luZr1`W9E%#y2JPqvySnR)_qpkHC?eD
z%Y|1W0uReopNQ+CYIxDlaas7HvL-RArt`Ce%-MlE6E_AsBBF#j{kEv^os1q2L_`Wn
zVTThN9@*tPW{8Nf2876{zbB-6dxS!UQaFyDpk7lGGZ_A0RdkZle<DwS=|IWUvyr1!
zhL+F^u;}u}9jV~)BUK5tbDFsDBQ%sVm5q&kv#+CzF4e}av{pOvy2vjwHoCG#;aQdH
zK7}#<;j^XJ{c1@u^DzbgSm}M=P9h`ggzQ<;im=8t`@2l7Cxj!1&ka3D<CyuZImGw#
z%)SZY52JnA33f_955A;VvUR+h)%Z9u2TllvZpy)YNn^+{x-hwv+G*Ti?AoDi4<T%3
z^_9k)&{LBtxR|<Vyi{44?b&YMVQO`rOhIe4JI`&k9Fux&CLi|(|BYskZ?O^dfB3W2
z_l5N-p?f}bF7}d6syg~OLRIe)zf$UCk4?fZUsuWi9%D+G#6<Xd=YDpROR#W<is;A4
z>ht?MoG-5xUM;v*v%9lD^P@Y_j;(RCC!AW;{7JG;vg#M>W6f&MdtWP?XrLM{4fmsI
z97(uf>@rKq%PZ##w|rSOaU=EKk4?Iaip2VdRl>ptaj|YphgtT{X34h1#wCrO^2%5)
z*2ha^g}X-$OEY!r!gOiSR<EYgPfZm{e_I=Pafp*pV*Ps2jB?_Yv7F-VM11uJ&ZT|W
z;a{0&eQA}SXGTabJ=?C}!}ir*-8}yLMB9&_x0m1S=bQgqueW$}1LDJ-U*C;z#I^3l
zN$TR+y1c6K8~#pl%Nu`~BF^_0MJVmB_?19l!3884>)kZ8)*boIRk3(xD!qi;jQ8sn
z%jCYy%f+BnI{3KKradG>P5V#JxU$v$8Aj^`7u3C5?+Qa){a3b?rcH$B6p!-_Eg#S}
z9}hlK!RK{1d{N_Mek;(E2~%!fW1x+SFp?g_b5otPDedAutRlyWqjN~Q5{_a?9d-;a
zTE3s-L`7SSUS+%b^~yo1KU{ukEFyd;X;c<b0CRZOsd1Y|O`5+@R-OJ>>8^dpwW#1Q
z@n`W07vkhg-b7xu*?qP3E+k>#03qL^;*$}M)c^K;ZL-^lsEL#?)nmItyM-Ic8%ByR
z<(zP@DJCNBlI&Mz!m#awAC6x*U!D5@*m}#Lx|*g76bN$ggF|rFU`deR?oP1a1PktN
z!QCaeOK>N+ySqbhch|eg^VYppx4wT!A+z^Pcdyl}*EqF%uAcfqqzg2JzZa)ACtBYL
zbgId$YOot@T~p?Ts5SL7TWQZzdMDcjwPg`6J&-PsjDd@JO+qoasv+WHpNcO}O{AlX
z20cWg_PqnWi<_%E{M*$;SP<6b1Q@CvbUNfTSc^BzS7C<$@AEMz3e4dP!}m|`NhZ*4
zm4eC_u?g&T{Wg<7^zm3vHuu|)wePjN7d(ti;N~0xf#_Mmy!ROC;Ic_-I=e?Dr$QqL
zq1_;(t?E(~N9mZjgnz)%ex})@K{#8cHpB(&9h2jFsoGn}S!vPr76g3=GfKaYK?U<#
zuX@KI%0Q)6%PF}84>oG`O>E#eOU@pee@r+H0Q1j_&@89wmzdjdYx{Y7SZyNFddtI>
zD;JUXW39UrkHh)(cyFL=Ts7C$AKvipNk+VGh=6cK6gUJS+ORMVx+buSZiGuSt?RUJ
zOV(swr18QiUGi_`1HOQs!NKz_nwe)z8yh3mCa0dq9gx|`A=Pp7R>G62^{j8DX&jij
z_j|k1JnF06N>ikJzAuVL%<}307%E7Nm=3;?(5w6uf021mi^&h-z}F83&x~ETMl94i
z09%NVJs5a)=+EKUEv4NJCaC96<qE4DU#FL$v_vR?4OHRz7Xhm`uf-BwG}`hwC5qc|
zyK!I0)~~&enEfCXgXR!u5OYF2{NBR_<39f7aR`CO_mltZn@Fy#7=shr0YUl*%VAG#
zLe41&K^g-74N|=KS-RHQ^#axVn}BMk-CYWX+=DMB(4h)9MZ|(!wR><egw)qOZn(-u
zaK}>RCsdC7AdvJb^FtDW^mFT~Ljm1PcB{kFa7-dVyKGl+zt?ux1#v@1K&o3Jm5NOv
z--~K?R#h7;(2s@mo6}aOa48U<fC30EeAy;ROh}EoOdacFyoG3Lkfb(|@I2om%I|)p
zuWOS<i|f~mO-9ffP(v@PsVI%U;2_>(Z<{TkQa&!2w&)r=vLtoY-auhR5;NAwd-{r8
zbCH7KRUB1szC$WfN5|@RM`plb`DVN4x$?cSpiD<c-Po&#y_>F$mZHAzLpHp5&copq
z3{i0~Lk66*$=(a<=95k4Kkp(#P9h=pBE~tv3o4YRvgmw+rK&G<(B@iQduB7eSBS>6
z8eaG(5e9_E)&!`L`J~WC3A#aZ5e>)lapX0iFyYh<Q4k4jrUWBW{WFYGYur7tJ6bLT
z8z~hssdrtQt4t5J%7DKf!^CdWz~t-14&CULk*XlXc4dY)BslIGgrf^RyAR^~n-9%y
znEf2e1I>*?nl<|qqz6gKg+!-|xk(iO7Pn;DP#GJltRaD}z!AcgSHmYig6fpUQpN5|
zQMDRpIX#!{xTveFGIu%&MLBxXqPFP$f<BuCDuS2$S!<VIGNyr>@ONdea3hBt{s)JI
zg&;|?@%y3l89znHh<TxPk4RxK5vI)odAz_7UnRF1S!ry{_8Xr=OX}VHv;Q8JoEHpb
zsb5l$h{;5j#w$07y1`%!f}}tR@b}<2e`w8GQG<uv{2aNJ-){q#J?N(=@abo|9W~5e
zl;DcgCqIZqeh`OnJ-^~=8h$<Wr_c%>62;Ev6EaFb>1`ZS7cj|#=g$^)6Hu<u=JKtu
zT$l_xYz_iKq>PZgoR^Db*=rzBMOz3%1&qeg==JQ_{;?x*3k11YgsVKT?fvbJPH7mC
z)Vp!w05o#l{3Oz86oo&qKP^3<FRyLmC`dAGcQMI6eGZ6suX*3pvZE?vWl;fKU~ErO
z0J2d{#IJmScaU*+LmCe%*b6hezPOGw*<tzvlfuKRUMR3|^>JE<#*V#st81w3PIkTS
zogpJ;+e)jal&y|?Z^%0gB+n>tcM-kfLx{YLq|vrw>hgUQmQZTNCh-r0O_iB$yB-Kz
zsb{$55VgJ#T2v?lb*PJia)bqYK}yRa>%Q=xB0)R}`3tyI?9N$#Yf$ho;Z;av!3YQm
z96>OlkI0m2!(6fNIETm<klBPP)?d$?3?DwN!kE`=e(0?_4Uyco-yn<aMn>@Ay<Cel
zF#rSUCee_B4qCQ9%ZH=UQ4R~EFV3(1SAn$PIuiDi@SAow2pY1@g7=9dq6rP^(Q7HE
zOOGs$c$QXt-5A)EZtQ{(X1cMOxGHJPA-^t1snO7&g7D2_5XnW)-19VnhHOBTpZD{9
zS1d%Z9~$xVS*!2E^hcpZsh%dQI-JX=1wR7*cUdJ>;jAbs8>Op+Tdt*7vY_>RG<3{d
zh&U42j@HZ$Z&l^Hnct&I4_q6;23A+M69xjidU>Jaq*>@}T=aLpz(Iy%2r(A6yLu0f
zj~|lJF%t%YeeWaIj8YDXCg(dP>uis=N_m-7MZf%wOV2-^zDog`DL}!|q_+azZt)@b
zAdB~~HB@D@kGj}JJW#npp&{6Q_!wZ;CkfK!-g7s5ckEyl6O7LTzHs69LJt;}0?jWi
zmv`ncsM;Y0fua(?1-G#+&G!Vr-X{pbWg;D&zs>(3#MX6LS}h4}mI&<5D10y{6sgSG
z9;Ip#2v3$}3$D}J!2P2>bA=m4vfW(@0V%V=(Icxz|B?n1OnJ-}MSWEXsv-C&9?U!!
zsw`$Q=yv~T(>4c_^~j%mtLXPDB`Jv5GOJNO-On5Uhfkk42zul1K7OZPm`i17+2iEW
zprN>WC_X@e+m}u7H(`?3eYrrwWhv_szYB%QQBn*ASLonmg@ZRhKPgzmqx7L?NlTR#
zX=nzbdx#c}l%$%`p{N41f)9tJ03D0SvR0Gf*D+3v(&3YQpDSU|dOFbc$}L^oNPFy%
z4p7Uy{q>q|MnmUYl-Rc80i)oP`=#H|djOU34jB%(FFrOJ9jbQ{X72sfvzUgdZW}P5
z#r0ky<GD~A;e;~spVa%wm*4d(Sr3LDu$PX>v&c(?NGoPrw+Hxzz`$1w+dS=l>uSOI
zW6cVf^x={9mOLI2K^2;nKtw;G;lExZst5~%G8wnVPdp|!S9k3}3cH_UC7mC=)*R!_
z6}LZmbKHK&7m3|uh{NX{G=v7~2(p|wYG>?i80{hz<TQK)ZHzO1fd&OXM+C|W@7{+B
z>!J-s<(}i0Z~igEN)7yqD<OF>M7_}=8Sp-8HzhX}`17H9a@a*l+&CoP!3Ye|6fF_k
zz#>V+;AqNeLCtmREGL*42;IMM=d~F%1W6KZKd1;oJhPoLbS7|*@0hyO{?JO>;!<`d
zd~8V`s{}*khv&ed9;U+0?~2Csu>yJTudTqefME1I6((kwR`l=;W3VvDWJR?HPFbzf
zOE89x>MDJTWZW6NvASYgMIi>leJ3{UAX?F0DNd;(*zQS~W7ok_5HYUJxHu-o`>a7#
zgJ0NU3d@-CUYLN4On6BWb)R`R3;i8H>PCJ6I_S81y1xD>`rZM950?5Non*K+?MJg4
z0pNzFa}3)xDLGh8i`c4W;wxu*kE7s|5f~PZ@Fd|f%LL9nrN@**1ctPYV^Z%9<gBmY
zoo?D73xq_i{+5x0As#x~6fJ36YKE9^((g^wr<aCuyN!Xqfu34)Y6-yL{?vqu4&wPF
zJ@?BB_D{OOdM5-nfW3DPmA)@#ve)LhuE7vP_iWnVz}(}w@{OwzJMQ?k+X+Ec6B{v2
zT8_wjHwC9Phja@;SV)_*i(`9zk+O0H!+q8yV|jEqk7!i)$PB~VI$yxk!Rph;QafzZ
zxP~XZK={6-YCv`9P8@W4Xn&02PzpqWE?r;nUeo$B2_khYefJ*{ki$wGN4hpUF|7~Q
z(m!?J9om~fD?<{fK@TZPl<G^L)k~#*3c>l4L1%KX4ESGS{ZUg-ISxqzS5V`yIjrdS
zQrD}n+7v{)6{+KzwV&-##Csd+9Gd+&yt|c`CM_$|IvRMF_K4y3N0<~k3<_bD^ux?c
zH=}qe5+z~oU++YRvQLjL7>hGRw@aS9f5t$jzT<3XV8|?t(Enmx2oE)P^k1I{0(+Ub
zJd0fY&Nf{gsJyCgjA7zSlE=3H6_?(99O;ihuWPHWfH(n8$CGUxZ62eajFYYT{w=`0
zZA&6Q@AEMJ24;%-j78&;3-l1ieTPqe{I+&O;Lj(da3r81N8(6Z-?=Z?Kx9Kmv-D7O
zBPMa8yuxzqXl>m5a9s;E_o3B;3=OKG+>=fpy1DO&6k+7K^COZgi2Gf^U#8A=5(1R1
z30WEF?j|kp#{No#&=qMxfj^Ipo$dEuEj;yOqNu*(Uwsq2j($H@&xMMJnOIRLk8vrd
z2k^dbn-(rr=Y@1y_u}FmC*^kCg;nT-n07GHp=5=BwgG#$xBJUa94~;mL?{sg6YUCY
z5BfK(Qx!4lAIjlhBt+CkN~*3nhkqg@Zir&>gXlcsUOlEv>xW{Qs^BNMmv$pGBeKjP
zd|_wf4>n;4(c@XUA7Tf@zY>E#x_yAQqF4tiG!!DW+Lu4UiBSU9OHT@v>~5JOAL*nO
z9Ox4WVXePw1r`(L`i8jd+uQjgUL4ZdaZ@=-FJHUWPM)(U5Zyj*7dkoM#*2Mn>&$8h
z1~R3``?Ec%MyrZ5pX|Gh*O(oBJp`x5uhrOvxVJn|{Uj6x_=@G>Cv7}z_-tw!$Qinn
zt^%^~VBsuH<zqi^P+CexgIu6NCx7Jpvpmrf42IO9nc1Px9G`WHzV5gEUEd30d;FqH
zo^^_i)ZSRI_vap6EEG75_nnW;%#Fphd$61^h%|m-oj*rTentupy(2a&@?t`*bz4Qi
zfJwFXny(>Gw=BGKxpBLFxbI3gurE>Jey54k_L*Y5%ZeEt0N~6a&^7QEkQjW>1|W1e
zA7#JXt(F5sic1~W6?Y1|A?|;M5#p+E74Itt(1s`suC;3m)S3MItZO}GyTNa^qQ7Iq
zj^l~(b6j)9w3K*Ih@Tp#b(2;g36@+IAT=*;XWpY!rdB#K_ll`qe)z@qjrWLCqg>~$
zv-F$SGsBc>r)_HH#jtlCwPu?TbC&9bZ8k0^<1{y(Ajk#PP?$#Q=Iu0*$pvz!<q4+$
z?ygnKC{1sXf0Gg4Si)vn`n7*6eLQGJVFnDhE^8%vg?`AmjQ-gL*=CbkezLa3$iW@T
znoXB?H1I$H)S97~jlUZ-0byfpfqk(R-;S{$$t+K?3yw`2iT1jwr6=6{kSB|Nwr0`I
zype^4wLn?8Pso<gg|r{B03+NbLxr)x(@DnH{$HZy4sxH#KTp+Ldrkc-k0IPM?Yy-i
z+e(cCnty+ou_4GZkIFzJRoB&=+j%C1>`kCC0FEcbb#tqg5y5$~GVWY}jbT#?(HwNr
zLP~(-S@?zJ8%L0Ld6(d!+`p#rrIJA$1oFi1x45<J%UT3s!WJ`Y%E)GuLL0SRSGq4-
zrOtYBFaH%Gw8ga$GG?xmU?+q{Fwh73-x?3lcE`fx&|!*%&u#UlJU%~(e#+Zf)g653
z{|Mm$77W4HA{sS#>9B(Ni<G76=Y(l5KmOJIb%DSFE0PsVP3mpn?AXr<Entc7%p_Qe
zDlkvwXVa9|@ov^iGQ;{QkejXLRBm!$p-WVcn@4Q_LnU^V6vG!mH?9eHSlF(NAI2>o
zW#VMZ&b;8a>PuH>a?(!=k@<#JG}P0HXL`QwiVBi&Z%{zTbbt`=)i?R#n8ezOVaGOA
zB%Z9tR1(^QP=7!l9oR{d9)wL9f*_Spd<7Xr7<FUbNg>-*tF2Zz$A_s%j59-Y9pJja
zgNto1e_uBeVTR7@whzKv#u)zo%9<!YD~CS=J^n4Ci1<@n6_}=(MLv_Tz%8LD=th^v
ze*^-W2vtp2Pf34mxLu^~+Ray{*1OFJy<OmHY261BRTz@0?PjWPPsE<T2e;Lk653kA
z90hL9BBwO>j1Xh`9%n}$Uf0PChnb*B$Izsn+U}h?gy=lQ4y3yMIacDJilGB+3}|cz
z;zSksU|yHB&pT!PzDvvYv5~?S4&yT$yfs>2gJB7AmN-R2s?ZtH4qcR7W(8awj*hl4
zn)H$;yibNQq2C^->lrG20&O~S`o!141EgT5SPTZbUbg+Ra$noCe2#Ax0XLQ_rY(JA
z;d>NVmTeP1jEWZfWzG52teS1W_=E0=*wYTfZo!-4M)p7xd&6Ma@Gv<e@0FZM{dje;
zqRu-NL>V-*R6L9K^H^sa7Uc8^l#a^g9R&<*sN~M$KI6UW`OFnKDjhogd>k%d3$hOG
zY0>wE!tnrgn(9@@F2(?9v?!dsT<qTrApM&HRg4#)6>6iUak7X>Z@`Iw)GNp!+<8_M
zz;^Ph&N<1Aj)D-6VV<b$N{8(_SEeNc>k`(M`O=HJATuryL#7^qA4m5&x~AK3Xj-Ye
ztb9yH#-pG`p^pP&GuM(Q^QZnhxVh7BO~HyM1l6;j_lR%U!Orequ;3qr!(Mx0l%1bD
z;}_czwZ`4(^1AyTutHF2_Vr4`98&Q~A^fj!MWQmzX^QeOEtz3i{`5+7s=2hz878T`
zBpKYa%^P=+fCdQ)NOGsvyojn%47?zHvY(n6jGG&xqb6y_Y*w8=`AJsHXSwpUjDhm}
z%oYB6%0yA@`a~SgQzr1xFLrpsug=V~^Xt~w>Af_&K8NwpP=pTh%?bINI>acQo#7UR
z-B#!X=%T{PJ-2-Gs*YIjvTKY&eE~$JE&wD>$7{BWs=@zRiE7X{no)nyLK$-^h!<jf
zfQfN2d{heVrI)%fmmDLRblCYex&gV?(rR^10zHlvn*o!-LPYvxZM*uRpA#fyLT<zW
z(edkpaZC*H?vFYX5_TR}BL*J@`d(hp6;nG4{Mv`vRRROJn`+sxtQ`JN2;@mjFWmNO
zULyFoO(eLNx+c4r7<6!bEWJC5EbCe9KD@{eG&Jo3gItsoK--cyz{`$WcB&p~1*u}W
z)~0y&1LQ3r`NDFR;xOF5vbt&rbRkN@Y&>A)tS~DlP-q10nPhVLHlRT{^OBs5T>A4e
z1G792cz8TSo^k`v*k@dOj&mPf8M_@a>ANreaB@Eqd2!qOH3Z=JBpS5^CjZC=w<SVJ
zvKJX4FSM8j&Pr`Fsfc4Rjym-5IuMVi=n;Kx!+RR7XdNuB2oCJaTkEx9y30m^_W?z9
zq1q;g@U@!uMB2(f%ZIP$K`%MYAY;{vw0QU!4FZadkpAPrJP;xdxf-``ya)#wy8w6D
zKw97rL`OR!iO&^2wCdYG1jydg)TEvBMXzIqc(4n2zKcVp)U|3O;8FLt`;`~hC-JV0
zHdFvb!YXszp3zkX?@8nWRV_S4*n}g%2L=1o8kN4JM1KQljJ03lR&7jq``c2?`M`ln
zno{Ar)*bCtbNaRcQ7arvL=t9S{a$JMF6yixvF~{;9q)4v;rk&~gSA-oa<4*+c0FG-
z$%^rwo$CUaq1sw+TDanz$FJrBTob-_70J%UGJMaB$yLtOneuJYJ><v5%fFYop=asJ
zBUza2*#J!<nFQg+lRw7#(?SLOmx*v%#9;x}8S|H1%_2`YSxABN{dV^6T^lnAFxZ2c
z+5#fk?HH7kXFQ{I<j)ZUC@R8Hjz4memmdS+-(g$*g{F{?<ivmq>Db&t#;P+7B_kcC
zBEnC?C;IcPFF1A4tz`Ih-EJFOjXqaZj^Eej<xTOx3BB>;onwBoA}d6xv@@)@|MLEU
zoxt!KNrwP`fk{+WnPQ#DVgb=?EDl@k*<3rV#->h(K2UL^!z?W31#O*!=mjs399s9!
zT=fp3#xLX{<?i*7UrhB$Q6clnC=V~2nTQK>=<<!+tIuJTK9w8}J79itHAJUEl`+M3
z?uB}p_ERiuYs9!JI0~3q=lNWGpDI|F$RBm)xT>t}HK>$#Zi8yyx5Hd!HL!OpUV5~<
zK1h&_);FEA3^{|=!uju6<n;n;<(hk($w|(sZpdU=NU<wg3I3dfT@1&)Ydzb5S%CBr
zM08mM2dt+hKAQD};FEe+oniNUal9>G+RA<ZcpesYhZiE9Ls9%N-%{558Pw)ii60_G
zGn3ihv5G7z+&f<)&{pTl*jpQxq-nF$w4}4n<D4hi|9jPdToer_1ia_|O~|4dJpvpw
zOWt#sK68cN;Qr-IM*V66$wI<B!oce5OCA2qOJ&y0EPLOvmtM^$i}7#C!XNv5kV}U2
zYQL3<V%g2Jztt!a6vfQ0_K5GcV0pvn*G^eAXNT3%nQuDAFD4?wuUBVFwW}v@H4J_~
zb6;U}&AJ=$Z9OauJ+g_on0auSHb!7i=x=yuI?CMP7O>8!ga3Ny(W1h~(@$#mIcF|J
z{_$`ONblf)i9tR;5b$2y#UA!x-ss|84iDp>U)kCI0XvLnIoEw^*FQ*M*znA~v+yQ4
zHJmd3)=e<l#ed>cH`YJnwEUgP(!&zyz_@DKkH*sI)|;HN8n}07rI+?<@#5O{TCP}a
zJSLVord@nF$TI7g^fKSf1;NWgL92O88>ccn?a`Y_i&3t-0^-?$fY7dGL{x#C3Rt`l
zZaO(+*RR|6riI}gfZV1AHI1+zv}Zir9I&a@MKZRl#l%bGxZ?fCLUZb5(HqTyG)ah_
z9NqtdphgM^NjEpy_hJ3$r%w1;yNYf+Po4)frxNuQYmb<u<03A44e+)JB*U5|8}&BJ
zhtGf^1v)2<btQ=(rZQ8k2ineT94WLkE}B}>dR}iNR+vh|g9!%qra0qIm@k6q^Dv=<
zQj*&Fp3NcXk=5SdJC938PAQgg65jS{F}frZJwq>(G2Rv9X*k+-W5@;H5nP5fCKWZV
zU!wJ`UB@WcFS>h|IP?VI5LjO>^bv|g>){&sZFF1cRBju=$^;nS^lI9`wxsb1xr*8(
zHNBa5jh|+ljqM0`m<=W3!q8#+AK8RS5-y6|?1z-1HEyG0<~ndIfl<K&Uc3D@WHw4s
z=5zgQhnI+v`?#mWe^PKs-s3Y1eE*1Vwj*b?>QHTV7@f%Yu;v7c8ZYceg!B|?&l=I)
zUbTt4yWJvy+F^sYN!&H!GbQH8nRNG;0T~4h39g9*A8K+;(Z0y%yt%jv>zZ<{Y>Tn-
zZ6Er$_<}eQsxYCj_~N-2eG4;&rN5LDP91JPh@-Y7-j~T-8CtK*x*ng3Qur62&RtfX
zs8?EbzMNMmVj8^qtqJ71Y2uy{E?+k->U^Kop31UKx`*V7*!Gcw`~v59<892=n=!q(
z)O*niv+C2<CLr+om_WEV1wT6x#2oeBfIF_=x$G`<&2Ci?2X0UDDO|(8rR%APXAFf9
z4cDwwkZ<xH!T`sfD=Cp|f^Wes;LrDrPTVh|!Zh9!`ZZAc_<o3G;cGe^X*vvFI!zvp
zm{2#R_OdcH26F6mnRmpc-bm;F93w(2(qq9v74Jj;^~^Rx{&Q-u@n}|HPauMa1Q++l
zmMhPgeWJzr%#5ZYjwlGBC&I>{VR#Kr5CKVv3by~GSn*otNIxNFu^JZx=EC5cdC`8L
z6G2$&b7NRru2)6c?@_3kb>{rtYNO5zN2q*1=xeSRNV-TyM9RnI$u?T*LL^lHvSyGg
zKuxFzz#V@8(6<8&B-SMWN}RFNl>(X!I;7{#+Cc%Zb})?wi%;|s)}3F8gWBH>bjMq8
z872Yq#H<V`+K9O@r0Yk;!Tz#>?ZMIy&!aZn&%EDOf4zd42e{C*AzZ3-OD%D>aQK6W
zhIzz63+L69v!+|+f<*Lwy#NQ`KPqG>+ZpOQGYkU-kawj2)9?S`9BIRY030sx)=4k1
zfiXJqIlxbX;1+@H&EZ2tEz*zQkG~QZ(9qHT@8QutG<31a3X(7g4`2h9J`CL4zXSoe
zDo3ijz<_2gu%382H`s3s=*rcomdP*dNsT@U@+WVFi`yfjzyGTcPBK8T!6gKqBe=FL
ztSEmac>LxcFrU*U_?Ms=PD73;H<n|Qi_(AuumCaOHeRI!{_sA|KhKP4D!?=259p9o
zhzmPheANGY^FLE1ehrfU+&d1MVP2m+G-rQ*w+Q+hyjAT#q?u))EGh|V@3cvg^bESo
z=qaZq7a2+nqaj8@H?&l%QKY^XN}Y`PT9yC|(*!h|(x`yB@NJUNcp)n}*h|$43^d5F
zjVG!ozk0@p`W}!=k7FC=!FOmr8Wr<Cd_?yo>kCHfW*o7T@VxxI&cyI3`pTwFagt!c
zNnG95XzpHJQkf1%QDTz<I?5e`)FYFB@C^<taNsDyBd=@(l4-Ns!jrnFRU)Xu^w`LR
z_%^HZl&8kLR&_#p)Oww7ka|1+X~$yWh(5v$-e#oJkY`jKV=Gqt;U8mR#*VN`N#64n
z-L2a}R8xb(UO1hc+s>Ijt|hOEwztjZP5G1xX=o=*f&>Ljm_`GVu$kGHg0Cl#O}fL)
zH>(GVE!rQ_c1Ju>GNOYna(E4D;e0L$^mA2<JOY=Wgk?B&!ms1P{2}@-!mgWL>fc7<
z`TqHih$OxFj*ykWu}?DW8l$Fcb9L^hg}u3u01it3R3F?ty@hlF3NIS5mXZS=%$chW
zpRM9&beN{I5Ckc)0t7wHH!Z95&VMCcn@~vIl-TwmMh>Kn&B2>Yy%@gb@Y-H22ip&i
zHWXg2g8s5JN~JN+6)(;959}ACL(kg?d46rr3u0U0J1zFFM=)kU)$P=B*YF%g;-eRM
za19ml$K!1nNU=@sFjJ{|^-i7_pH~2Uxj?&jnv<3y?!RPZpN`H&Dqn9bb#HlLpQgL2
zUGaSz;o_-(Q~<tncH~i8lxI!>hp@r^M--Xo;(Gn}od#5BJ-S6XsVe5$db2m}2gB1D
z{jY2W6}frL$NUcz;NsGa@9(JD_Z@dXYV2E$UMF)&eEZY-q9`ZBkhF=T<r-}Ni)Gb)
zdyRt}n2jFwuHqyElR}9vgC_4A9#Uw{jM9dgInifUk0#!E^`l**Zhh=sL}VQN@Cy@9
zYM=oK9Agb*=}xjh;CSHw7dVvP0>}T)d4~4!qCLB9%vYv{lB1-(ymeFb4~doKKaZfB
zP0_;*fVA14Dli1{?M2s-$YMx~dpN#*gRjLP6p@4*sT`{3*Rnqa$ukl65Qk3Pva3xF
z1tejDf8KxHY99Xesq*YGVeM1a_>OPZm({8w-?jiSGxqha(HA*1xVe4a^5oAtD8~!s
zp+ZGgsv2(-`M(}?)B*r59x?7XO7UwsGRX%0vS(~u+cT`k=nF9eM<3k<lr~62Qit@<
z^x8=A?9feyC*Y$7*qRZCgQ{L?FrZ9OwZT1QjCn0k-sR^S5VU8LOCHCG(;m`@g1S?K
z4y=3UDhN}Wi@v15oN1Mei?{h#B9e-ez&=4>v?5RKp8nlKC=&_sI!~qRjo|Tz$xNMZ
z_nCb(GGXnFpT9+1e%AADMh2w`NZ`HM4k4A9$N%%uNq&pcGBqwA{yD=u{j+VjMtMbr
zfOvh~hCnNY_}?YVM`>_Lm?9Vfp(I9>AcP<qSc;$!vY^7VSgdz60`k!efLnsyD4qbt
zE=kbOLhql2G&CWQ@{m~aVd593<zw2$w~nWd#+H`GR&Aa{^Aamlm!q23(qA3EHhL*X
z<I>0SuHhQ+7&%FJLyMC%LMLyR<F24Ur>5C*Lx>}kkM1)oJC?4H;b_@b)}(TZTecQN
zogx*Bj>gR8GtC~#Ik_nea=+$U5o6DWXHXwfk0ypPj#88wp(ePCV9fQj^5Ppy>>QSz
z7jw0GsxGuW3k{^RC)T8YRIb_@F1yX(e6E)^Tb1Kos{3ZUob=7rDrsNUYWHB|La{*4
z^;<;qpS<HIG~1ca>Q@f~<a~n_?@PDqH~EIbMJ}ZJdJLWqlH)hmh~ihb5BI+k_!`90
z<BVn`@_9WAmG#@kuVrPlJw}JQWreP)WZB5LWNQgF?PN~1z0w|^mgcK|)Q(d~mOdbd
zA#PSJ*WvIIFsi{LabvTXJ6utfi`J}PDNH<HE3j%~!oa^CKO}c#jF&2J?5sycEEXL}
zn9KhVvwUwAV_8h~cwAjpLcpTLHljx6g0RY<l^|Dhg3uT?*Rm$}LZ)gYwf+MDGM850
z$1!BCe+(SN=gPja8L4Kky+pDl80ZitT;WY3mQ|a#6l|T>F@a9jo``btbc@__@56k8
z4{U@6!I=bdcoF9Q(Bj_E*%IDW+{srimgppq=+?6BX12>GSa~@D?TlY$$sy56eCA2l
zYXw4KA#c+OX?`rNKKrF|*{Pg({m$m*s<6n8bzoFiq79UjVD{y|og5ov>Wh%_7yY~y
zd~^MLkUgSpcsN*;AuHKcxw3r{g8Fdt>V8GNudu_}f&$|b{ThFE+y#X>@b32~G@5+4
zmd`q6qtPSvYq-$BjltB<>|@t>=);hr_u<zs-*zNtQ#pqj*UAZfy~}N*{5>N-#Spi?
zsGQI!32LV<_nm@@1u~7wu`lG0={z4JJzd0g=-Cpi)U?u%#@p-pG+-fh;C?cM{hrcp
zU2KuFY%KP+SL687j|QI-CIDe3LeEEmEb27HRmZeahzxbhtD$iaF3p@vF}?muE8%U+
zO=(*7;q3_pp+F8^>{;)s|J&SoAZM9~r548b&_^)_JpaB=3Z;!-V*<85{p*8{KWH9@
zJ@~E3`(hD=L5rx6vMue7GFkv)sR9)yM2{xu-OzBI@0PFXT8tzUFL`8b(cx0d?)5Qi
zsR*A>-%}x~s$@VIp2y?xPZqXMDT_J?JetGivqU>dq!445=Z~|Gbt(R1PBf9vuJ9o{
zj+bH&BhJiyxqbrc!#V|@6M_4YL1AFxmO7b>HLt{ET9)oq(N_&*D4#if6u-Y)-M4(=
z6OyQszAP&vJB1_QGl%sx|LTIV{&H$hmGqb0nJ~YeDvy2G0S`Cc{W@F4^{iG01F(1o
zsV|bVUAkr(Pw^_8!PxHUNpx!cGD$sF+msVGj$#7Dw)5b{cQXopC<-t#RNVGN=BQl^
z1bh_#Alg|nLA{&dC(t})pX<Nx3qd!tnM{zZuXbH_1k@WErTA5R;F*=3l3Z-B_Gbkh
zxtS$O|5#Qc_H^=Bv^_UWgjI>Rx^buCbH7Z(WYp43mfb&GYBM^Ls?xiFYSG>pxM)3|
z^_Qz_ruRyHkRehfb*B3*&z+xy2bF~cr4s2zGne%pmBA4SDyg*McI0-e^xkaR;W)9)
z^O3wleF@eFt#)yco!BE^h_TkoFt>cgWE9NLQBZoRI+WPNtB@7rd3!y;;AaMIsCJhy
z<!((Z>RnSN<L6A5%yfDG@9;!J82$;hUibtb1p%K61zA2|)08Of>#b#5A_fI4m_@kV
z%oiIO>V${eYFL|bC{T8a8iX`rF#3Lw<m~2WZB@l7w4GW^Tm7o4@Zzvc&|~DgzkgAC
zJUR08V>Q0qG;^|X`a*CY|Fwzy2^I(WW2znDLW7HmSsoS2h7;5+KR5<z<~6TIlmG1k
zt}a`w;S$ny_qD9@T7skVYL}Vxxc2%3k!=W`Z~g8&e99@Rb7)9sCGdjoH1L(PN|Gwx
zSpPJI3q~~28W}1+WhQ~HG=0wEylt8sSfFc(3)Q~`h5`%x_I-iz-vUwoEl^eF|6E|x
zta$<_AqM8P54^>Qx)2^TAS_br<zAB@zFA(siTQ9_n=hHYHdNNCNZtCGi%nLGrdVuj
zg49=|P`FnjzT~%-+0p>;L#I3>LK&7tDdr!o8dR3y>WGU3)-#jQn5!J!2w4lZrs8vr
z%6DQ|rbJqeb{z<5Ps`@NFOe`e{II(}sV?6tLqHYJv4O8<9C!W%+zGY+ajPi7R)x|@
z)awA{Fc%qeM_&~wg|vK3L(<n9a$`bg&?a&f*D>@hDw2`#s}n&o5J|qhI3kMm&zzPx
zXvk2G1C>N?i|+<QmLqF?0yW&85y=15?d6_R$jBV%URBIJdJdMw9t3>BISEhQG^k(v
zC+}MOUK0x~3z1bsoYrKjx0AU!g$R>P+-zYXw;1?OpyEiEuxYGb7=jmrrb#OF8x((z
zY0~Xa79>WWGD%?)4Fn57*xYli=_mzm!(Zjs+rF=~E7JVkd|e9j5u4eL)afN9cj@=5
z5w==nN0mfPYQg%*q~U#c`-<mbiV?#^pTVa0hf8!yi+q80)I?)dnm!&kvw4=c*9l62
zFm#zObS8OQk#mgq+*X5@`2Tr4eSu#`?5Q%E!9{(`4MfIBY4KjO(YxoDaznN7S=*E0
z1z`z)8|e&$>#c$=vAlKPTLko-e5s64Wtoe)<fyx7?6JX|1D%E*T<FZyz?i2PddAD*
zFSRZ@_jloh8ZHsf9fPOTauT~S@GkHND91FnJPK^{daV|~5lLh>Vw!^W0iMwymU`L6
zaaU(96_4TCtk%FN*jZ25m{uh(OlKvM%_4UH!{edFwU|S^8<ER={aP}~pRQ~c+o`nr
z$D6e0$1rL0c`e;(iBXaY*DLb{4u|OPV<}P(=Be9e<nup2e^ShtGm^N$x4kE1s~-Zn
zSZyDslCF#OxY5ULt~OC0p}jpLVM!oWD(Z8mzwMA_k+)RINcew*{@cI5Ry>26+fHO8
zfN(jj1PdAKcYD_Qj<r$&)#rj>d*6~t^4D(0)!}f2A^!IENzQfMWAWFhf+qHODMmSQ
zMkJ;uhs|Bn(^s)?yt}HX^p|d7(Z^Ekd&i$$*<uwx6GtQAeLT1>2V5u|hNe9f-!h=p
z$5%XGh}pmAt7;?W{mCB#SD$Y>9jXe~gztAdVT$H+^9On4$CRb6mdb5~(#~-IK)eq>
zrA;($l^jjup$_DI(R?eGcHx=hZ_F1Nr!;P+if6vCCsbYk-roj%G5TYCVz*0GXZNnn
z=X9u_wKH-ON4yHWUX>QtQmmiyRJ9ppsIJfbSswl#S{VxZp4eXOMQW(8IrIp(Ex^tG
zLNYv1i7%M`qi27=9f-kNeh3oS(23hZf|^ywDBVGt;DnSvotC5Cw#6D9c>l6DbbC9v
zXUwHruYafW0!#5#o6E9{3GY*xVFPPJt`yEPX!>=c_ve|APc@N!*VC8-RNQ76`3^kr
zQpDSVPqzQxPrd?UG@Fi039yh&fYT(jVsQHe6YAl|?dI(O<uBxNfS;$c#dx+r`%1yl
z<5=4Sg03ILPuttZl*|76>fP-=p*3pfHF}Y=eZCijA7K8muv^ZADPwV_In4|u+S#$e
zF)^>nzkZopfic{d4s+{ZosJSeiEy`vxW!+LEoxOejVdp9+o~N%P;p1>TXDSQloS`A
zu}WbftB)5`%5Z2_v(`obWQ~_j>P?tr_;e5$Kd*2b|6#z7iIT^wMy}c!ByA&lRqq=O
z#Nlhcm$^TjTQjt-#kSpOGH}J<KCUii7K4jFUze*zC5_B%7W3J7*#{Ue)|cw{(8Hnx
zUX`q=1WK&<1gt<&y2wOO`e+f+!4!Aam2OGNJev8vl`TaxefvkpFfwMpYyUIwE|?Fl
ziqv$hBNc47Mm`wJX3Pfo?*`gOZOA1|Eu9k^`67MggpMHKD6H?8kGY&}wlz7$S>%Au
z>63_W!1EAFqVWr@O0Z$FN273L;|A}5MB(CQnUH5vOZ)5=2!S_rVr^Lbmi42Op=pMc
zVjZ7S5hG~4o-R_hXszv@G(VNcL1zOA5;#(@|Bh6PGC71HT+G*W?u#7m)g72WqO`Hi
z@(C!(*>SwPc_!O|?&(>7uMWc`4>;ufjr(IhHgRk2+W3%BDGPw${A~FJevYAk9SZys
zw<JL#@n20Qe&LwcFH7TPT!)v-(<tLRw=<L@1S%!^C>GcS&n5)XC&<$5**R2oy*+c$
zSF|SWaO|MnaOI(aol~j{!-`LI(==x_6JlbA)CsZjsw`4;wec~y^Agt=A`G?mu^z2o
zm1ad|K(!`{n=jWRTb><gy5Ei@sty##YM+n%Z0B`u74qD0Errp(|EO`D=$ql5WlW+=
zY$Oy{qOG2OB(HOA@VmL%BrRcPw$b?I%ZK9-m6=qH(f16YaMI|*NX**@1Pfb5jN#9M
zFuy$R{1cy#P=p*N^{NRC1h76IZu~w{|L*>5Iq)7@9yK6i|3|_*LGE8ae?i8l%qB?6
zWU!=MF@>=@6QLYEs8K4mX4ZG;+Lg1$P4BHrvq@`_q~tSA7%U=oJtIQ_#rUi!LL2O>
z)_$7{BQjJ+dqYLI?J`WEIqMEQN%vx1Px%i6yNX|Cj*Ql?H>g`pUg0&f56spncuup&
znp8gGAo&C_!O*?{q4gA=N5id;$5&0}|APcwMF~Ip<n#02|465a|F3p7e|y`ZQ?9R?
zcVaGbCvrCjOiJm#XD8CZQ03hKd^pI5+Wb%;{OGu}cPma_3gMdTLKwXD4m!O+d@*=Z
zzn7m@{vbU{%UjWWXgS{XwQG!*@w4kEL>C69D1OEN3e2eK70CsUw%4a(g0H=he#1W5
zUt*xfY@Pix9?o-bzK@kPk7w1O7U;ruo_C4W2tcmhQbEgCG^+Tv-+3I)z>V<HU5(W*
zHW7zhnExq9R{1bsarVSl_R?_|@<3&;P$9|8>t6p#pH_mKT0&|k`|K>Zi`Q_F+#MWN
zYN1Ef34|Q@h<DV(VDaDU?<e{&#5|)8<7_A=-3%=71CVhkqTnD!FGHE6K<D3idKtQs
z&W{Up1^_3XtloWY&s&UYl&C(4VYOy4Z`)<JTR)x<UdT=EcfYu9Vryw9v-~}J5j$sK
z5|)T*=sp-zQ8b_~_c@@05`oO#kn61308LU(QZH}G<Wk?R-Xlf%p?~_V#K+;f-8;>0
zn_)*;U9#J5ldxTLj9qK`)L>A@BAAxFPDuRGtQ2Nc$r?rQ&wd$Hs}=`btC;kFo7ju6
z#jEX6!;||FH4i=KTq%39(0$4vbTd`ou{Q5pb<|_p0z76ZHxVkQdv@#rCPJJ@nUji&
zo4#s+^+2oEOXQQ!Zl6{4_;A^1N&*AmD9QdoOCRBSnlg6i_s|>fVVTmQ1p#=93K@{H
zne2@Z>r>7*Pp#v1_bc+SiJr~WmkAk)mqOE;-@9rVl3N8?5~A8=xA;kikKWLxRu*}C
z_AvyShfxFHKs{ey<v4&Aacjh-*d8>^AHZofKrdHRkqr~qF&~FAc#n}Kp+NVkj8?AZ
za?I=IzKIyV(<WPbxlQ18gAW%zs(dI8hx%3`&dH){3#JMH8H-Wl0{&En!T*)Lm?q!~
zDMr~tb-=#8LAbM57lvur=O@#=&0q6SV_$v_cV{GcKcWBX{lCk#cboyRK6iiSniU_R
zY5I<<4Mu9D+L3+vNlFvrvn~3n!Uchk@yeD+`PD50-yh}z7TIJ<e=GTNMj#d*cVvk@
zows)zk44kI9WKeWP5)J3YUGH$T4e5G$6x#bZeaapgm<<x<oPkM>oW+`wIK-ri)&7n
z4C5s0f#g5v-*TY9IEikeX6h`E9H3&c8~y!YA?y1^XBh<vk2R`AdTLu0EdSg_^wVAL
z6=Iw%W{UXoPZPt-Ie5cD%B`R{_T~WpHIJ*od}$$irU1$BI~QAgAc|Y?8^nJSAyGks
z8VZn(9kEL=yUELN6WkUk&g%+IPVpf~d}DQ*=(tIoRNcqTv5N&D@DC#o4iZ%vG=2FS
zulDoS4(*<79RLW}MKA;NhL@QC_goiPKqN%xr12H!it+NSi6NCYBVrkYiKnOW%k{gl
zPn0K<5_Ec6&p&Q?JK~C;Zr8TO*vCn%g|fccyXknH6@~b?=WTs(FScaYda(DrhEg5B
z^{{FwH_+%(baNZX*6WcFMLKTz=JeZnc5Pt$z#U*&$XrevWI6``%6d4X^|@6xDD~?|
zAwNwXeU*0FcNEC(;yS9JPjfyE(?J%5zxnmNwnBZsxENVtjiE>gL=tgO=oFcb1=%@S
zv>xV-3-u@yG$LgskJ~gwdOC=soZM^nor>~w&+=w(=;xFo?tVqz{ttR%)cSlM5L5ms
z)jclSY}1`Nnsp(rnZcK#X)lp4+-E0rfd1dxI9T;6&#;(F9$_umWG3A8T3(F{hCIGD
zGkO&#Q3SGUH2UT_tpo&1^tr72C6B-TD0=gncG0TLxuwgmkjSvx>*pn2Fhx{b_0v1g
z)u>s|b*+n!4$F}TEf32Jc3)6F>T6Rc7Qk4n^V}A%xNXZ}c>$-}g>r2LrgnE{nUVbJ
zK4y?q_m&G{x3;*slQa5>@N^vDGDNDhyh>dY+qAvjQ6?J0?ixy@0NY|}$HT$)YgB2J
z)m=nD3@1<;_17LW5kJJpyJQX5d#$@<*$K4h!ZZdO(CsP%H&1?@`yM~G_ADPkbMFTh
zVqbA-O67BsJv427JTjxbTj$YWH?V-Sl|B?!jthF@8lr5*2lz*`#@K%+-S!*CYgW=L
zO)!pLkED-np~jORi_Yw(gkn!V<Zg}P?Z2FU)xYnU(b&_3pe$C6`SQHvx8lbF($rZu
zdV`3c*4XR<?+>Vr!TgvgdvZ}snE=$ja9+Z3cQ6H?Wd$Ald;>+eZ=eXL;F{!W39IyX
zIqxzalz8%d@iL+$X_}ZNjeYS2>)KJyEr0mn7x<0dqEPpxD7C!jp9-zH5h}j?Ezw%v
zpZyZA`L~KQu|wiobPghq-YcFzP2^jADI<cFHS>YY-j8cNfO-CMXtZ_#q>G_$(UtAp
z9KQQHJ;>chqcP8W8Oms+c;&mUaI2x_vYab^Q#x`5!=mKKJM?%MDZ1wEJgh4FGi9Zo
z+fc~z&1{y|>vkCTBpLH*A$+m-VJCD`-;mEJi&^@9dYOI3)v|Tj@QcOOq;8w1pQuf4
zyld}*Ta!81EN*Q?oxM2UaWd39wXr=Yh@BmA#p_MHKrS8<swKvdE--kDeQ&5&c$f3+
zKwVG%X*+P!^|wW@G>%Jc$`KB{a6G{ND4+K|4>a*8i*dTAM)hgT&76lKgK5&~TJFMz
z0p^!Jwt_I=C*u&+-uJ5=j;9SRJISe4jlYH~DBI3hGmNV^Lh=3{)css^HdF-K(mpw(
z7K494r^^zQSapwtx}yeSF0}x6^If}M0SZP_*Jq<7M)6{ui-uM#t=&~q?ky@!^?FDb
z6?@w{dq%QArAg=tur5|HDaXFH$bXL3Yj=O7{%yCt&8k0F)js07!m+-rl@<4Nb*xZY
z!Uo4XRUdz(@FiAmZNY-A=u#!Npz#T3YIS(e9}@DyRrJuucBg!*b+htJEv3;_`fLfe
zLSHPu)0?|`g#Qi6dX^eCUg?(_7cN$2tvMFZ1MoE#fUh_;kx_iOR-GSuNzQ~HA6#J}
z53XGu0yK+LDHUQqg7}RUvlNik3$C=86Hoza%`sU}>pe6-=|j(3F^u-V@<<j5iXYbh
zjwh14M0=4b#!1!kLZMsQ2S2Xh_3&fj0nRhqWx!Br4|?aM8ZuO26t9pgVHGUAW82G_
z(i%VZpdDc~s_3jGlW}|}ZBv!nE|X1ts_Lo)uJjv1vA93U#60$qwCxVVS80i_N+zC$
z!d2=35J*<(&YJs`PIjd(tWO3CaWv`Ilp)h?Jyb-dgM3w`R+1+#?z@)LMqfPWGssGV
zd$U%9v-7K|Eb>Jtq6h&9r6E>;<4=8lwCbq{+M?HxzrAlCjdVzftaDZ&Qg|(a=?#ZR
zZrophxT|ohn9WJX;my>&?kfohFmR(e@h_dZVhy8u_nV@b=8bK<A?O;0P^+0ZK^;Vd
zWct`LLju}_kg^B;LvSQt#vsQ4t_Iu$Br1d3)ja=nd|!w%m{Z>*+Ox$g%0)Nyr88|$
z<3BM~76Zi^@I-}}f|Em8a<$WvOf>%}WL7Q5;Pq!r?aDh*lTWP$mhX<$hi)kGzk7~B
z7vOeVG0Gnh-GAlWK2rMV99}I)<LkY03UPQ({#9FV_OkHHu>*#>-Ol*w##yreF4Jmp
zB)!P;3}=;u6_(wXxZCpUCr#OJZ(DQr+1~cEb^f8kgZgL0*Pkz|0h%K&cG@4-P>!Dk
z*Yi-ets`rd0EXx#{dSFR>=HRv=RED@uD-8Edux&z391l!K`HwOC#4ta5YFlG#4&%Q
ze2JuKUL@La_b*ZdQM~gP*3xh7ki8_jNWZK%XiGMh4eC0?QdSj4d8qBV#*i(`7a7YF
zeV=iPq($(V=-<)XQ{d7gBr=v6YwF!#UD4CL5e$Kc++~SM8*S;YZrVfUsfZ}K(U=Bc
zljli!izA<5U@RKgv+J53>>~-S@$dJHbN<esv_=U7QNfmanQIo<vnTPB{}mf{Kfr~*
zt5y?>Uu)8BmmB*HlJzV5UZ@=c9i}@8Lky85XBXgr0GThv6rJ!9?l;b%U?XGxoXn3&
zCEm63Xg16%k+9F#8x{zdFTUyK`SDZW9pqm7OJo!3$QPQNL;LZcFE0gjgVpbrl7Bon
zoqJ2X0k_7Le5HpXQ@h)Tn>IF&%CSX!6Y$+`Cm(Bxs2OJotWEP#{V#TD+btQ;wXdmj
z^%Ck#y0i&Hl0dstj28}oF3G;_lp!sVc>lN?|H0{2fnw$7kcxwMsHfImbCggGYfxR@
zJZ+jpw9fKQ_lyhiDSzdIeERDtgGM<rmC%}2j0pmVk$JmoIBMl8zVUregD(zF7y_Ph
z64~%?-C&~sh0jy<^<O~yKtC1;m*%VWHIrrrVDDI|E*2i>tS^DQE0S%eWXgt1iDG*d
z@rbTPx4k`wl+4|o-2p4LyQb?+EUVjp4Ic&)oUWUU>SvTFQV^<55eQ6SRW0(H<zKNW
z)XV{L`jiKYsCM(fVFuww7Z#Fw4RYt{E?n8gu$7d#3Hv<bFTr;NOs~`OmFzY&sIJ^(
zgS(VEc=?2a<462~8H=<wQS94{!fO&ORawP950)3jt*510i+k!Nl6Y;$9V)9T9}BT>
z@XEKXZ9hV1_FM6KcrU9w$yy-ceygvxtP&mDkt}*iddlmpa-g}1Tb*^=l+Ix6!iBDQ
z_YhupDR$S{(y<Ot2qL2)k+ubzZVu17uOf6xYwj-b40z1y;avN9`&+H949v%!S@u0=
zc&>F{>%YdL`*X7PiCl!*4D9`0-$u2177t+od9w(@mcIY|?a&yu^|KDXb0-TZ%-!n`
zN<l~^nt@31f{;Cq2{>p=w!x_c$mA>Zc|c$54@ar-XV4o1@_i(YV_Ft6&$`YvGbKuZ
zAGRYP(Et;q4*GUz?v3_&$)x<^U?l|sy*?8Nsy^`Q6~y--QNAU}P;{D#HJ#gh#dg$*
zpY>yA)fh5btF87qRf`d`q`qgJmLJljW~Yf{&m4bnXC*o$YM-e7$wJWrKtubHH!s$~
zp#if)sitqi^FgQ3Z>4iT+6hZMXy#2w8;H^#q~@U8IfLU*Ym79<P2BQQI%eyA_$Z4&
zpA%1~t(ceSox*xw=>1uj2jSBV{SS>5%GSC+>xBOi9A~4|9qDiMG%V!c`Fw+%X>DRJ
z$+-CsuMAcr%@)Q*+P*LNx;+(m7Y`+Qo}2|QilkgwBm(d^mSIWL&gG&mj`Cc8+B9HV
z!d}5ukXiWhuB91eLQx06C53KIJyclPcHhUN1Y}#^mFzN~j@R?aD|1pIR{@<#a{`Mf
zIm9($W|fL4?>7u1e&*B_2f(IbDUDy@rzXRL_20;Vir$>nh%i$Zdj^a}PYmPI%uEmQ
z;DZfAcW9B87%>-3P@#_mbvAk0=3<%c9!4)V94GTJVbArse)!sMdKw=vzES-3MNLH8
zTfb#%iZ0>Rn{77V9;N$bun1ED{>Qax5jo)<p{@Z;FY+*$AD$q#({vOU8tBL7*t6c}
zJ$8+m!pUr2y^j#k-HYg<E8jGx%eAwzU_Qt*it7Lg0hrCH{n_9jlx6u3%JLKokYGZo
zRp{VG>l*g508J!|6Ia2&yvp~`;Kg0Dcxd)G0g|L_{C6s1*U^wtY#W#4FKM(u(pM!$
zAG)e}>&PXKwCWUB1X2<z9mz*A!%5m=ZhdIDotSi8c+dgtDc!W!0Iv0FDo{UDIEmD(
zW7_Bnr{{xjdr+E`LlV)U3o_!CZ97~%Z4%G;OZpGG?r}Ve=KzHs4WKWQ$=$ATAcNd@
z4elnWN#IGDzOIvQ(B9H79B&9%E;4$j`RhaIH8Z2wFYh_n5AUI~pZL$N-g-~~jljKQ
zsh)gb&FtImjAe-)>{klw^OfAVNM11X83{_K^)=Xbb-&Ps1W8bL5Ag_BJ}(%B@H=v`
z_c4_;G`De6X>yp1=XOhcxfHN~ofRBC4zhccGcF&EU}cwDi}T_O$#UpZHI9uqD>Wg6
zL|P&Rq=R-|Yo23pCW9Q;?c{#E8X-Q58^YRzQ@k5L^<H?5MH4I}&0?0d7z->ULl+4I
zj})^ngdj)cT0K$VJ(jSAcNQ)TD^3-=TCi|_rw=A4Lsqr3jV&>uC1b^i$TQX8xAGE+
zTtozJ#$^XTbD}F@B@btyr+Mv+ZK*|Ij07d>^i$os-yv3RTVJ?L+Rucwnw9EgZQbl)
zUNz7`8P1%4I)J`Pv((mRc7~G(&G6j>TNoO=3p*DB!Ey1B|1{-W>*w~s^#6nV?o657
zhERyw7o|)OfG?f3=Voc7b1p88G~T5<HDL;7<sl-E+OYY}vvz~E_P54LNToEHuqSKS
zYI#^6Rq7Tpv%Zm;1WlyhRK4k(rZq@VhrMI5*BHBIV{`yMer8Q&7Qxp`)V*dmyx!xi
zvN;fJvFNY!59Y>U=3v6(5CPOav&s`w_m(Pu5JULX{~z|=DyXh4iW0>=cyM<M?(S~E
z-7OH@-60U%J-BOdch>}hySuygCiiw%b@%&zRj=Ot$VVz+AND$P%{Av3V~(X4uI5Aa
z*#~OPUq_PdUzD?6dlD@cU?5R|zVOSo_`tp?ClgGH9a7VE4Pn(q@N-RW%2al%SA%t<
zz6HcFhhZ38a)&~{J#b5V&8*A_k-@4h=?h|YL-dYhf;#7G#E3R~=aWS@0;KXepPlvv
z33uhK?vBi`Sh5Lo>piYHP}Xc6hfvx#!?3k^jNvWhM-(e|<gN4Yp5Zqhob7w6^7;l{
ze<*V6Ci3%QWIW9!E0By9EhKo{hR>WsvBpq&FpjArs`hFH05n@r6x4OW!@7*eiy(tB
zk&>0FdU+5+sdRrR|Cc2JMHEcndcBsiYBxI*Hg_{&YzU|lY|^1giyjvJ*c}?|Cg%=B
z^2k)6Src`TP<qvPhZEE#-`Poftw~E{M&+zHkysokP%>(swlhC@(Fknl!fP6B^-w(K
zBdHhm^uj_xfdm6q#_4E#uRAtn2-q<&87hEq4OqtxBI_r0t%4~p+yW6d4iQv>@?SY>
zEX8sL@K+o%^jH0MOOnG(lf6R@Cjv+;rhpxSYCSw*c>g8Jbk}{85i`Oz{4}`}$~nxY
zg|C~o=>L|ia7dsEQb5T%{a&&@-u8R#WdeNe8n%5FN+S-Gzd^s7A_<rMr1M*F!?DsK
zHhw$~*PtHOD)r|-`3PVxo#0>!s9~2QT9`FI*0di&0J6Mj!r$Q5R2?=Xy=4A3#bC<9
z#M=EkjY)XN6;~!26m&AG7tcp?mnD@D^*99ot(ql=_oQO)JV2-m=Meh=JGO}FeT(^2
zZ&;@;m>E=?p;`w{K(*i&?esic{VuipXq_)abu96xQ&Kux2xNdDC_&s|Bh*r$FoJK#
zq-mtlFD?TRP8tsum@n+rXF(aS1C-wI?{4KU!1)`}WcT;{mC66-{M}Ox6r5zHfp3#W
z#lr&kRocorB=<^QR@_%BnQi_Y=X&zEboJ14oQ-&xt097;i3?+jtif_ij=%I_FzUs`
zI=3lDFN4^x<5a6wld`hg!GQX_i;pH+c(VEu++YC2OygJycG{(4wH*4lOrm-3)Cs(^
zY)@B1a~Ai%fH3!6ra3Vvsw5Av_R|yX5&g9GM*lM7sq6JzkG+XPYm|{3z6PiCLDquz
zApuCMW$^}3Ki{{<9S_ktUMy7=*pU%FF$2H%lQ$nNwjrq^vJ!fZnC50Q0KGo1T-jA&
zNe!YHS>Lr(j0jc>(fS)o;^+1Zu6uHqY(fV5vCY^~;louDqo#UncL`zN&wqdXoipjL
zN2$`R!DT{GFQC$Hurxl8MzgI`&26R)EclXsdV?e-rub?<1`Xy12lHGqbL1{2<+NKA
ztdw8LPXr9K@E*~5LLdgY8t;c#ZuY+`Ef(07+9`jO^1L^Z6<^fK9?R@p_IQdIcTF^b
zQ>bmYsM@qMglW0%&TF_1n<z<<Ahez-c2plT(CpF9EmGNj;L$nl@rwwg?53C--|003
zAXhag4F?*QAFd8WIo-c+Hx`VoUH*o%?uPQq--LMt-3+JwSJp9NZI%ZdIbTwMG8d<#
znaS=8(8NLuye{v32;&7Af=1Sea8$_V*v+RHX*NHiAGSBPIcPUDGho~CC*%$V8Z5uA
zc+uQ8xN_Vko5!G&?l>}Ng2|gY34nw>SyASw93ndxTZ*K-M#Yker_p1C@j5@$X0Ejs
z-n;?W;-E2VtT5T&*!&efRp--TL0ttU-i%Wg!|YgwoWaE}VNTpq^@L+Y=SyKfyoZBn
z*}Mn#1mA0ev-Gor?H<melF=9;sTI(>JNI{ikI}zf_%C)CR`qGC4Pui|YE^kaKTtBo
z{<gz*FDciWzWTG1zip?yoBKGj-lo_T3%iB)JYwBr1Aon^@Qt+|XrEDnsD+;!CDP_%
z--%tT!}MtWctCV==q$_78yZ>f^*n%>y4Y5=nvf<H91n+WQojo=Bl-2jcb&N{i%}73
zSGvm{`tCS+<4vm*fKh1}*{elE|2^KX@XpJDwXTr?j(3+cU9J6Qv_KG}LsIH(0ZjUD
zQV|xWMsnQCZf#0liEj!Q^_DYVPc*}}TFX<Y+`)tuar%`8zB3=DHqW}8_&{qcb^U~e
z#<B4_1h@jpf$t2Lqr;|)%YdR%*5<cNmH4H#(cu-xEzm9uzVgYI&~}Y^<Rkh$!_f^Q
z5hkvFJUkfxw~Zul`?ls`i35na6d0zkyOvh8vbX#y!E3Rr1ysUR3Wa}ww1b)0;U?ZO
zVO1&a{cOemMfoW3TO-($r{c16xx~EKutD6wpMJX$ty3~7W5K&M02`uuI%q&-zUjxz
zix})65lG%QDf+L6zlpLh#7+d>Gd|J30J7K;k)7CVSv@1Wa(BnewM^=KaVSR5x6f;z
z$zyBaEMe^2I<`3bt|bsUw9jaB^`~O{_FkN|-`Rrv*L|CSpqAO6O>d$L+`f_g7{eyW
zQ817TYm}_Vm9_20d{abz)*_{B>U+=9NO38sA>Hth%I#(i*ktB!{CRWjM#%R7v}j`Y
zXSYS6S#_U`Ougg&r_2qvnL3c*Fou=SXGLMB&QSvo<|W&%ROT@2yZgz=24u!a`jM6M
zZ&V}_WA1;`GR#?DcFs`#!;j4PLb-{*OYtZKxd@i-7-l`s`iT4h33c{&`eu!HgwHjB
zIwv**ph|tJUI7js@+=!4BPvPu5KFnb)Aa$<X&-@X@oULyS<cr0R5PHkxEwe(eVqDV
zouT(pVm%-hfOwIcQpF~pQb6x>>5y)x>}482#}ar~x6nSUk6T8*L|O%bZXF>B{Hf`N
zrbSvrz(%75lhb~dOaBM<jl3>p#3l3q-KQ>fwj>i9S!DopEYstH*zFGYvO-w`UfM|W
z%nHo<u5xijl5&9{+;Q(}6*nW>VL-8PiaF5<=oPH*D*&|u?<#wD3QGBO?2yXL+SQ{=
z>Z+xm4KCEy9#q3su;L@2BDg&4d(c`_Y?qieD5`C@2FSmkPKv_v0bXM8QU~UQ$pA-y
zx}U#}_1N#}dF{>Zxf|i6$u49&I1CgH<-@^pML?^#t`V;@KLcnTW4=vPy(7JF|02ER
zP@om}s^BWc%ua(OFF(`cq=LtrV}fY6`co4eUSnPbhnx<`0hvuuwsN!li<6Rq^6SIC
zLJK!@ojyRSbKUgX>t8ORDkLTEh17A3AL)~M`?KMMX)7|Sa)}HeI9!kRN;CP?m?-VN
zZOzyU0UXk6>TKH&LBptMh@jPORwe`uELNtWJO~yT1DXq-?{h~l_mdff1oz3hFZ#z_
zgNrkXN^Ard_|W}45u{YzZO`#(#D17@2NjQHqT+ze7?NHmp0bD)Z?1Qe<p@wy96D|W
z!b6Hf1W5+?gDlHYe^$M~Kc1XB-m*PJ{+C}^cFh9hphQ5Kjt7+K%otUDB|;!Md^uEu
zvRdeFTs^JVLzto0Pt0-VE_&>G0Pe3#!aE=Z&9lB9f0fjB*$#IWb~iaTXEuE{2M8WG
zS5I9*4$~ZTxSA0Zb3`X;Oa<Uj3Ab?90jaIbbjeHpdGB4S2xJa^c2ht%806eHS7R@v
zm)9#m_8l;uaDK8aoJJw2nN?_OJC7N<OqG`?`wYk*obmTDD@R@1lhDL%{qcikFp~kS
zhr$wb<98BL`svc^AVcSuU@j-tWLob~>M2K+lNrUTv<&~496(g~XQ@2Of9tmL2Af(N
zy|~;F)J2b5&xR@kxp56=V5ko1!R7Hzz<cf0Zuz*{qOJ<xG2?~<HP!O4oyoJAjxa68
zj(`O{f`t+l+xQl{vUTZjIH0eGS~Z(g^Ie<LhP2r4ZdVoY3ci#L{)Mxll_5?HC><A%
zv}-zST|f>)tdOfIFva=aVO@{EurBPs2TBUqQp0|{);>irpgd_xqkxLGgQ+?#`0=O5
zaR!>!E39vEO<N9$U^a+_)T(jL{<zH5mgb#f`vE3>eTp>q>t<;Tm%@=J8X3k%nGtJd
z|IhN7>GCm4r$i@jensn3qpt<~fxgD#h4AY-i)w&8U#|fhz$u=b*wSo@6r4^)T~6dW
zb8~4_V}o&GV640AN*qR`mj}W^xPr6cf(8m8AOc?|ZOC}coOy7TQD}pr<@MLfLPY+g
z>fBcek~RCAmi-$58vYQE34=<YpUw~+rFmF(e+C9sJn8CHT=fTw`1Se3m5uMr(G-F!
z|Lj3*1O^NVQlnqa|B`UJ*|2$V)UV#}<2}0X&x{VABYyfK`hOs@{L`1c1whYZ^aH3J
zj@NoMI>3yPt3&o!FJ3nCl@9vNuXieNxMcLwJ2b-8JQQ&v4o+Q%6(<-}hLHspF0<5F
zQ!~$5D}Lprc7d~KFiEG5JF49$Fe(luD4r!3Sn}tZH-|98Wb?JIfzEbxmHC>b`*+-V
z9FIC3^efdHle!nXd_2l|#P{9#rs<JHmr)|y-3Wpg(RBnUnA66eBUu2Ak-bZAe3K3q
zY>wS(`>C8zmXMWyO|9@&+ND7SjZbaN7}HX+O~M18c>151P9)kYmmzfkL=`lF9CH!&
z_re33YCXg`B9PF*#g`?Em++e>Lgr`p9h20T5S^Cx{MYZIg=sE~q{la@GCS&~5p0mP
z<?kY1-qp&iA6vN#KTtdE+Ll?3!?I8etdT;#X0&SU4fqfTawnIUWO8rFx&wmQS(=XR
zo_UziNkAz4a7#+(45;&NxG(idlmhEhfPZn3so}{#Y1^1x(UBMXM<<o}S0}|#3SnS!
zzp?JBCArzpt?~S5fG_IRxHTfO*LJBC;EH4hAEeAE*c<-i<k8A|db6h_)OboA##NS~
zLsrf{sz0D4qPMhBt(IwGl*L>7AiBz6&L63r5Xq;iz42$6Cf@m`(FN2$-?H0QvjM~V
zw;cNZEicpFvVH+m4A@w?&w$H;-HR=6MirJP)0Md{HwZd?)*eR~Qa(@0p5T?LH@xHN
zTBH!hdU4wSCY_Y3S^2pZuY4o2e9#a(1mOxFr2n(a&3N3*h5t=+7W+G_IsSBbT}t%B
z=YE&RADrMCR}Z1y$_%E9h|y=wV~6oR`tYhzO9)BRrtOy05G71jto2)$-Sd+12xpma
zgKqrJ8Wmy*n#TujVovARmqGnFk?{x^m3aw=6<+oR*mnhZfz-b$*e`&e=i8X2fauTS
z1=MzqF$8zjOzD{z0iOp|8upR`Bd`wSOU~1Cihmd2ip#L-T}ud*ZOg59A$)%Y_jbT<
z94Zq&=pHEu6!BIs#=eq7_5X-vY~IrbAT*aGg+RW9p6>c=d25-|L&<lDxeCpsgkocH
zm9g`fVy65glN0*=!1NW9=bu^}nP(Ew6W~YI0ZI1OY7^2i<h!1c6M*VYg9whq<3T=v
z{mP$o^bGqe0%iYK1R4_p6H&3WfQg1==)<g^tV=2Mq>K&D`02Fg0shf4737nPZOdPD
z$>>hgXc9oQb47silY7}+?Ok3fDFtj3YN>eZ0t!$Z2&H)T$sN<bai;kTpa<}dVg+$X
z{FXti1|||4F)QOc!BBAyOom)B6QE1g<2DtFw}E+|f*jiDA2Jtz1qsi*o^47%5tKj-
zNRpZH7U2F$GUI`Ei}m1YSso@OzfW@eJ47EH8FJNMNy;kp|6Hv=h?wW&J!64k{UV>_
zs}V^%)do5T)mu74N^AK*$=_q-?(Z@{qH$iWHe}!v{vTD{b!89}8L!%uSK~mfVlw6w
zTefGkMYjIGztX$V9RcN^BNc%UxQQD!#*Y$4G0Jn3y&COZ{kHxMkWiBXH(|f)ia%5S
z)fL11-@4+ysovg7KsN6d0J76WX#?Ei{r>->0A&?V&)~@WgOHv7uBHFGt^-)h|33J?
zrT#Ct{@>Y7OrqHjCy7qBl&5RRBJF1C)7-#FP@PYxyEm{gzhwSRp{q;)V(eK141Q3{
z5$U!#wG8{dK2R2<n|Z!(UO})DSwCjJME@(N_rI=_fQ~p}Zv6c6?-a>S<KHQgXBj|#
z#Scfe?gs)*1AVTM_%1u=758IaG3E$b0pKelP-qCMY5Xo@)%PF#20Z1As%NicD+EaA
z_Ks!lIX1}af;&*E4FJq22=<FuFqHuRl$Z8BZ#D@5kNHsgHU9mX%ano-C;Y}){ty<(
z>nws`h%k294j|F0?;}oK#=uALxrof;|NZ3zfgb~OcjWJbVN|Hu&ru{^S==@-Q$)xd
zjcJ&NP@sq^Hj1X{?{9?+sD2NAZA{d@kN^1L0*_hA9%29cOmDD+wNsW|Y{~Z*1pn^^
z|N9323(^1f;D1TxzbyLy(_?~&;se+f;`u<V9|sJtuQvBeY_tCJRwQB#CK+--mt$kd
zJW9cM!q#W7$!Q8HpmLDnUB)&ATDi|1puBwV=XpK|P=lcR4N?5&1?`A0deFw3#@v4&
zS_B^ae-14I&BYPUmCyh50{E}d0IHS$S}R~k;J;?_eemJ`6MZ}Ta3-gNhR@qh(Do;{
z#&Had$oE2CDf%BeQ0Z*-N*0LfFM$BCw%8Qi1_zYu_W+_!uy*%DT>+@gpZ+2wsDK;}
zs+mE+L;Z2iL-4iPPN<ail#Y!Xq2`ygaN-M3<4F4VBAF;$n<UGM$3+{;3iaih-H!{c
z*74h|c_|4M({vU3Hp@EtTJEpJ@a*lpLclO-{Ql~SdUK0TJgsf+NlAP6-;wu{7l^*I
zhor~@>dqH0o$w@Yop4?iE+3)3;lwT7jq_K&uu2REtV_-$_#HR8HNM--_qRa>;EpRP
zptE@=zTR8AWFVQb4_WImitS$8lv-iXXv%X>E|}yc2GN|AyGa!~`v?Kkj#6ePPfIR5
zU)nb}V|7hWW{Be`eNnKRC@w;Mv^rnwJiNc`JP5oD+Wx*jEelws7=j)Ov%my&?Er)P
ze7(YPc{9kfvq#vGe6=?rRhbSixeDegHJjm{>)q=1icZG3o(jG5edWx;*kNTAO}X0i
zt#F;rZhKT=XD(h~m^@jZcy3KgW0IupI(+?ScPIOtQaH%!<ari`2AqJR9{0G<Cp$;k
z3gxK5nG+-P_hyKQ3dr2q8cj?eVV~XZX&-L-qKC6C9-m!@$dh$rkL`K=AT}fa_LQ#x
zEJt)VSQ5{iX9dpXe#}$y$Gr)iN^en|7FP9lxbO_SA0zE=B4mnUEFQWF-nSI&^F<kB
z@9VeclZ5gtUd5_uV9sW<5m_#<>*5S)WE5O$G*o-`Fja3hsz4@Lp1|$Wy=2uV#$-5l
zv}N?-vL+CE`w85We40FY7#~T^X_){QCfX11Nr;|t3}QNLWw@m{oHA9{`=rpYPrmcR
ziW(bOd<$Pz_Kb%k1+Md7h(i0lp0p}x*G!)h!3n>{1Ap#xf%^z9HlM(sfcYcBICSlA
zg?I`z-wi}c<cTm$r{;bfqf$*d?6&IiAm($kW--bEo)IKrKY8W&^bB>!$_XN!6T-I<
zZFrrc5=yl=ntPjmKx{_6ugwas;5z_It-?@2C8`V~F=S?e3=>9C=$ij$kMKb(ZN)XR
zgM$?Y!dEo!=E5HvG%t;J_bPnY>A=)M`xkVA#9iWLrqV3o(=PnVMys>{%}HU!hgJ7<
zZOXbzVjg{6lDmW|nhHw##s#=CG3hQ~0Yet@OuDC{_<Mi<S*VI%I;AG}rOsQ=Q4pBP
z37=Yn0yiJ({evsI=&T;~W_I@UyH$&6<@Q_-dV8_0vB`m;ij_N08{FOMfDQ*jr-}Gi
z+!nnKT)M0nSPfO67GI2v&%;6S%qh1L%^$1e@H)5BI>|1pa54a7xKAMWmOCm$ibvw8
zy1;j`TeIPEd}MgxbLu^DeQXd|t!ssA)EsV{fCTmo#9x5$4F0_UC{S##D@B$g=IsC$
z<~&DI8UD-7Ly-{C!yqdTA))WfYnhXGU3CWUV}*{3s@WG-6F}c_PRebPesBS8TduSr
zUML;E;kYwG2P=+WJ9$tw2bgHkw0Yl*Iy`w7><kEtW$I;_AGMky-yg4hV$QeGeAxTp
zOQs+Dl)sUk>hNM6?2UJBcyl<0;&9l9bm>(E+i~(s@0*xu2AUZw3};TV(~8B=MZRZW
zdyOCv2!L@&&ACaxJYjpGWG-EFrukq+mP?6_VwBfSYC`2N#Y3c&)7dXN)|h82XPpf?
z%sNi7=v6wse;jM<QU98ZNNn~H`IT&TmN5#b>tw1rtZ7kcszKkKCUq*jQO~WomFako
zJ3rnF$iI}gOMugovRD5J$7xz?c|F1WF@$;6N$UH0`#tKDm({Ph07dYPZDh`uy|c@U
zS@j#IKV?66rm}2o2AW@=ssx<$qY1glfH7?$+R2}L!$rNMx?Tr6v!j=v0Q(<3M6V`Y
zWCw79II7}kEAelLWb*Wv2IMPARZ20CvUhL0@psMvv$!g}XWFM7`l0&vJfU3=S9>OF
z4Rbr8BG3@wy+<}}ayxWQA__=|zva52KR^*EubXDjdwaB>^nSA1hW<zZA$y6@rhWRl
zX1vSTsZSLp;J(L(-^<%xn!XjHjh+_{cw_YY3_?~4f(8rWPTtl8>JRN4E_9q0E?ql1
z->}U=5u7&qoQ|ypm|u8)JbLWA3nd$Mo|BPR<%qt7f}s>ADwNxL?E=4T@RS)Icf~-W
zMGuMJTRQ_I(E%-ESoPI@d7<ITM*Y6YHQ_BI)Lk2F&ZZ)iyRNXdQqHx(Z?2Zr2^pRo
z`?c8yMzz0jB}{02(6)0nN5amRMb|5=qpDfOH<N<mb}{>7WOVdH;Nr*tm+`gtS<Sk2
zv+B#ZxDpX2zxpI90-R!@eim1mPT8v&d=eGfrS0uL$B!p7QaZvp?QGEr{bOSHmJ^V*
zl8Z*)(G~J|FM^XO&=@Ni)2o}TyM(V#O0P-}^yiJ=J0*CH-6lq7ZD^dIWPvn)&_m5;
z=2bEf5KqI=OW2u_FvjIT!*H#gxUsPjGiNcN5mY5qiS-QJ;V|I8vIN1!ja-;B)SE~y
z5)wh&8fw+g3A+%2_3944T(L8qZ7x&fh6rkRq_M%yM`7OiN`YN>iU!NZ9&%j0h;o2+
zeibco&2?cD;EJwumE%o#FTB0o-2Tmj;4OJS0c#qmbW8DEcH(eVl7R!aUGS$)x;Dd>
zfR2nB8%2}qy5aXu&^BD1a$$3!hHflpvE8pyS*RP4{GPVoFQ@nlg>7_8O*t#kjBOnH
zD}_LO9oJ6ZH24&rASPMo4Kn(92rv1?@)V;a0=XwsQ6ZF+pu{c-Eb#+vX}qeE2xpJF
z?;ZAQqpvBpQGRlRt@8pJmWsPbYc<u;BZo=$<jv@rYc|6Hx-}^J7(pyR=F#idC!&-n
z$O#2HBr+tn-+8(066#gbqu|p4xNpdL7x~-;y@H*k4J|=tYUezSLbXF}+dN(N;s>}c
z0ybJ=ouRue54Y8wqsSc}+*`AuAFl3z%X~mt+K|rH@=V#UIl$+<@UE9(9wN^?+th^#
z{amFu!^qX!?&VW-el)M?{!MCC?DPm=)2%b{jmeX-Uasiz{G%=uqi{L==dpUeU96_$
za`zUm_Z1(Qa}9Syoi9i@P45`O8lY|5N)%W!w-Kcgm9zUkxxKxa%!L$4QG5(3a5TFM
z0SgYR7o2KHZxQ#+*J|X^9YFuNW9&JsS-^=%ms-1*u;-~TWgiNDC!{%>Zy`*#$)t^R
z)Hh1l-aK)}ICIA7nMWY{>{6Tvlno>Mn1q<s4H#ZQr1{;e*6BJjG!g!WT173~3_&-`
z8X?dZEV|<RL3y*@nF-7=Ox<$WSF??lc3fmzTKu1i5`OofQ(1?!Cn1OQeeFF<3RX6K
zU3Ur4D4hih_Bjfptt0MhS)mgtfrcF1Yh7hd_v-xi`Ng2V<x;dZOX%ZV(?seD4qRLb
z6b3>_;vd%i@S8;_HUtAAyy<J$2d6=^&Qh5A%lYrRJ2e!#FAdu28N6?^E+o1kJ&rox
zi|RI6s_A(>j+}5mUhjZQyFa(5pEDKOJR)Fk$3o%fi-!`MDNpJ1K=}jCN57k2w1zy7
zFrlrlvK$no>)BrdUQZw@kgI=L1*%!%LS%DVt1=fMZaDZRn&%%BtAoLiw})#=pWljK
zZge@wowhBeLotnjk0u~a(l9(f_@|>)M6dbCVrK{9UL8k)e#_%8T!;@ar`71b54MMS
zK+xOg;hHB#vBj<0CVaHQKawS7jAAFaRhlxNkEn4Av*kDYJk_~`3%^FXvTOZoI#n)x
z7n9-kd^XgpE(vTz<Eqx<A+(%cyC^kZMQH=5ZQ&3;P>m1L|J(^PF-Pzdf)A;9qES~5
z$uMIsvHnz1K8ycw*O|MMHD&zm&-%m39-++L;A=XV4c}^uPP30E->PO~!OSJC$l~bH
z6@`UyXT2Pj5`p!M)Tmm}IUL>h5Lb*KGq~ra-ucBZ6NM&`1%U6A-E7Kgz0Xu9K9($j
zXH9xS<pPxyxazudexFC%=tY9T5PPl&5xGfB=G~MoZZhluIeJ?s@m+d7)`Q>O9>2`m
zv^=kwZ!0qgO=zKr8QZ>XvFmd-v?D1jKotDUhz^*82?FMbUbA1`F27J_rqA#+-#kHa
zvtX&r$>JZkvT{1#>NJf{V#Ayr`nNzRgAQ_FvvZhG$zUOivsLDbIHYmU>hF`$tG-o^
zChIRM=+gS1C5$Hcz#|q;ais>>KJiGaCa9q$4U3JSFlbyH@)%ZkIA|aK<Lb3zZ-_k4
z=vmj6i(ZwipD9Lh#(iKM96en-a)sHGXEvDQ!Bd{Vu$nH$aHrKKV$)m}q$f_dt{VS(
zwf`(djj-_<HA-lGoSemL)eVLcB**dao@t@hxm{rx$+lMW36*x;&=mh%miojQ5y?|S
zD`)<3{YG`s0yIz1_K$*&h;$;-sz$}f&inmfFsRPzo&eA`l8wwc+x}BgI0ryfKVn)P
zM2U}nm_mGh+7hfK<s?x8_uGDUKy*cyF+Z@4+bVw*adPE(<mq{T?guT>V<f+Vms=E~
zPA%nUT}s?_KKKDii0?>XZtcCePko==$T~`@n&Xt>=1#ds&R7+dx~$Fv=hP2dV3?R2
zZQ=P^*GI}1b{zUaq^YQ~E@BNoJ-M_I_a3Rw0Aq8p5R;m2r-6J4!{0<c40Ok(xzPER
zsd=Q%@10jZKTQhMaePWpQS7V7YZMJvmA(gI-U~K2OAo!=cg*!o=?5L<%o1~Z?k$b-
zw>y9IjuUbJr47G`x71eZbINEZ%$2E0uG4=NhK)~k&id)X;*%9y`|<XJ!i)w4uAS|p
z8bi(xKDXE&mSc~bJFrQc^U!3GYR-#uqa*da{PnGmEEL%~t<R-&9=G2ws^#bjn(zZb
zHW#;oNks-CM#`FlJZw;nytZr&)^6wOQBO+PUlMDIBp}dVd1n6E?*<6CmtFxyaGQ6e
zQNCo5OhY-GO%!*Lhng)CjLPr?j4_1wPkv*#b+_@NtAA|Um^B4>IOixw<{#423IY~%
z@6$OTF9|oy0zS4HcTw8f&J2L??zwgQ?3Z|WwE2VLtn#HC{5Jb7EwCd|eI?Oiy-_W%
z6=fXq(FAP(e(1B?Rd!deN7o1vT6;b)`T450!R51i*2zj50hzK9$gWu0aGdvgGdsE~
zFxH2ov1j<{dp5*idvM$?EgNmmOm7YK7vz+=&cv^29D1W;!#oOPEd<kzik&WlAi$PF
zeK}5@_81Eg+I!T7A0;sL<H)KUAEdj!G7A0-bH?kYW=O=OL$p?YhE$*;d}0t%p*V+f
zdi!zfX(ar@`+DPksoOg^6uWKIz%6+BsMg6mcMnB8exY4&=7tgX!xnWVWKKJyBW+dM
z!5_z8x1xHTvp(3t`b|>77p{?v%2f9}-NyoLzJ_YxX60Q55jF1nRzINGva}~Ne~(_!
z4=SfoF3IqL$(zQ+-zgfWgl2TNvUQLv5<x=?sEaeG+X8pNKVFY?z5#9}Ot_{`rO7u8
z18ieonIV5RN^eb=C3cb_V*nj!HRDCL3K=qBHn!69#NP45Qf!MA1i}tEQF*_sCt%#<
zvv+)ZneMa3Yv&N0M}JMofk9Vl?tv5ST?dN2<T>2>_%d4;rr#F0v4T)rcVVMeV}tv0
z8ft64A3lZTB?CDvmffLB2>L?scz-lZSdWH=Cx7Z&-oT2VI`2O^dKT0~pS1rsmB^3)
zN4uKX)V3~Vodj7z-Z$tka0rl$SSd|O$PUT^oI7_pU!(fZb|IDPpuGSKwIIiXi@udb
zc)W0Xs$5x4EDg=j*-u3R^uS1Rw#xCvz3!sbOC_^ve(+@)Ym&U12x9qF%&m_5{=p-0
zqSP#ZXe&Y>c*4W%m)c^h9gPAh%#vTP0`QEt+8@9a4_UDv;}X0!HIILrDlM7UeFh1S
zIPmo0NWWodd&?mjg!g?g*B+|n8+<4qRy+jmN@gKbS|kG5u{$dYKlozYwAw+tdUwSh
z6#6hr&f@GAt*B<q;wStWu^*k6?!vG}&YRa@D)49F*4UdFUsuaYyEID0eW-BD&V2#@
zch&?m#jl1m_~;aTz|Ba5u;Qs%3KV^hm2<Y?2_=8}Yy}eL70bJqn5*0PeRJ70))xq>
zA4%7sW4TVZ8uk7}FcCpv1aWI}<@`7`PfpgmyDHV^&HY}U$*yuH5+H5W1WP&TYd}b%
z8EOy7@IldQ)3$8sxq^R-U=dWGYL<r9wnnh(>(vH~`;_UaS%sZ2{_@g+?$K~V;?*_@
z8~M-4OI;zpO#83-P@q_OS_wlriLW2^E%MChBg9!c0y^&+lg3fWTmpWL$OFoq#6@7d
zYnMqG(sZ*`04~BPHo?(+(xMkP3*O58>`y;)jk8=kEKAt=9bu_xnn>3$7Dy_rrr!cQ
z>Dv7wLmgZj`375<B9rmjE4_-VwRI>orL@64(-6`S3Obop$Z;)(F>)9Cn3f>3jTd@$
zS3I->#_C(Jbkg+Gb(HU0EloPp6*!Bath?idQ-$JePOOq$2*?0!kf+GLfSZ7rQbq(Y
zEbVt?xT~YiX52HR6Jd@&vXWaQZ;a<%Id`AE!hFUxpNb<D>{{+FM$uT6_Y{D{woDo!
zLe<1@R;iz#$#PC_{Au6(Ee*+{8S7rpZ-GJNC~@vgO>!22)Vdh8M94s|GwJ8S4(y@<
zlf%`V&qf3z;<3h4IQzHIG&sUiLh2g6)g3o!Mibm$lm-$|3*Jcme>T$#wG(-+@6geO
zVivH}8Li-mgNX)PCw*nk2?$FWLj|drmu*4;84<o^`4$Ce-lQZ=s9!3vpN~PqM>E}x
z^j4n@h{?ye=(*2a^ScMGZI#jJzlnKqkupgON@s7J@z>h!*+Y9{>bpR`yXqw9jr;<i
zke&VMaVe<D-4bf!xt(PDe9boNYTVXH+!9nz#EWuG8}npG6W|z2ZC8l*D9lxVmcfe0
zE5oA**ihYE^5oS2(2@84TbIN31q*?!;*Wo)r1CYe4@yqG2;f+68S@qyL=ME{L7zJZ
z!!HPuD@Fynl<8%C?2K$8oR*DR(TjaUVll})KuC&jg*mkCQeU6z+;^n^*#98bnOz*x
zgDs5I!@wrJCk>e}_v(X>)_Icf$ET<c2g1@7pBk{mz_wy~>1Pkp3}EU{(B#8`N9Q<K
z!D~E`Br%Qq%w1F`)aCY)OQ<L)w1EDF37qQiqi=o)qN~5DW4r%`R6|7?r7{nHv<TTe
zOO0MIxh7{_-m1WtJ$nJ*en>{cQaowKg0xkXO#zme9udoB9tjeqZ$@UIMyK8IG>kfk
zdJv0=%~Jg?{Am_u0r7hReWpS63zp*&-;mv)LX`~=ti0wDJ#IlFXv}tFO1^7C8~9^}
zYh{KElYT%ujbE_!A!UQ<LWU3{kamzZb&QhqZD+SAXb|e6LNh4OxH`}jfOOBNs~Fr$
z?Pl?3G(7Ja95s25s;*NkkjVqHJJV@`neK>2vkVadLp%fyB>CbaICCB|67uJhjA>tW
ze{q>Ip)PKo$vW$VmXtq1nzFM-CX8l$#2lgq2l{0RS$r{Z!2Gsk04PRnftG?6irWa;
zTxJC5ZXR^Vjp2yM3eFoL<%#|gXr{vYD%_vz>pcVfn!E`3S^1_V1mA7{{i2A|eNW&n
zpW@S>9wXT?XOM1h-JI~ouYGqua0~hFE<KsVj0Ibci?~=U?&qh$cPv#2ZpmAE%}B?g
zEMP7sF)Me9)6?xEemreL<R@-H8C+QL2E`#4I9o(!pya9+$dciRcVhpVF0NXG?do#t
z1cKn1Gfkrv_%p*0&D1%w3wGk%9l@YY-eS>y%(6}Vdbc4uUn1<|t@GkPUEp*R;^Vx&
zAA0s=aQ2wOHnqEfdwl_8ty?MY*xABxV@+>6_Ji!(43phWmgt%vMcX~N0)LSRh9<4&
zL*~??0(}wxUyoA=08RqotdQB64U^HsyZG7V(fn0b0Urz|M>BhfNo_(E?FoMKJ2C9n
z?4Bh!E}g(GgpYx@6{fW!9u3Cfs=$~mMUXsWcW8&fp-Ss-jzb$rn)#an^hYFOm34QR
zNzR~QAukKJ0cW+W%~>Nhn7SVw@}Y1)PbK*J93_}on8#jy*klg3nbm?@7DDy=&Rj@K
z&NEi!ZoVP~F{V^+K_5}~>al}Up`YZbo<eL(33eZYjg=00I|>QmSt*Zsi!~K>AKktW
zA_tM~E3$9&LahD(K$+W<-gXN$x^x03L(QsdVQp%?0yRkmXMM4HKa*l`D}j6t;$=|u
zV3-QSHkpb+tz`Q%?PcGJRrE2VXneG#nVFiijcyycq{Hop##`dwrf3uhH-&9)jvB)O
zc64#EJ$ayzhFQp2B-KPVoNcfH?W`%rK&WSD!9qd?vu!exI6?C+RA-^~|5G~dB7G&r
z)yF9lcrEh2PB+e%6umEl(*;A?)8xZF5-pKr{igUSM;bV8NNGv%!CyI@m9QZye}yBl
zG-y7VsCH=UuT~7^&oG<0<Orm^KqBGw86#d#bT^lK!`*UNX$Y1eh?G(@O0u^c!JRBz
zWu$4Uy*4MA0e3ujiRJ?XiG4`}bWNCv94ch}I^Q#yy09P^Yhm3f!Qc@F81X1gi>rZ<
z=T&Gra<jG2xu!W8CLH5TRN>jx3Rm@#p3H~m#W2K_VE0h5CFp!9PCIedT_^V7Qwjvh
zKwHoE0|Q5oP<Ny38%wlE215VW?&MVwg1_Y7V$CW#QVE8|hk?qNpI5lzVY7S?ihi01
zSu*HI{$1#qGuUE-%=m}^blV|fD#Z#r5hgE%j1xqFFYfvH*GrSzN|!3cUcB<CRC_<g
zWEgkLPbrYI<@an_>X3X8*C?|A$UWaUQ4aagn~{-;i8Ppv7&_6ks$xe=6#lv_`Zobq
z2^RO?lsMZ;i(rJX4ST59MUbSNA~88oM>}dAD&=Fe+F!C`bO2AW>q<4*P+~ZYGQ3Di
zQ)EFMnZCxLpDjZoiD)-FJ;7HUDM>5(?lS$$3g6t*_q$*?1nmRezQQnFC63_xmFW8S
z170vp6&sm>Ky<HrNs1#tY8MBE0G0<HCH{Nst1H~ij>*ZZFl}Siv*RFeJKVy;<mw?E
zih6@VNQpAt+&A>|elS%Vnk%@l5#WVgwFjUDDLn~oU0)n_v8d>iBTng$RNM#|C4}Zf
z(p7$*K&!T+(u|24aH8V{cH%z2WD7Dm)?60C(YeVpp5WwE#OyDkQ%-h%>78IOoApDB
zz?Zzmvtc4w;3UXV0c*Fd!Ovkp1_Rgob4~-V1~@Fokm0}?j|nqELdgWpyHzj3N3}Qo
zlchnp3=CifSOkuySx_j!gN@L{$2s&;^GzOYK2?o*Gj~#3Bik`U5%Gdu;u0arcJ7>|
z(;u!a@TCtWiA`T0vrw&x1dCjiq3^=+!>ofw-$qk`<IaMA2kgyLH&85)1y9j}W<~sq
z_|Z4~`#%KwVe^2Q>a8~iR`6hh0}d=$TjlD2f2wHi05K_$HJ2hJ7<oq-F`zJx+RySY
zQ9r6Zgasr@egHFneN>+8=yNlGLwIdmQ`Z_b8tZ_`IJAj!eBxN$X<G7H@Hs>VjEC2^
z`DQ%C?w3w)HBj{B(@Bt2;}BW&>IqOCQlb1~ESh=&O9=<uDUE(rF!3K74MHG~1e~Kr
zA$v>(JY8b}U8u5eh8dhF4X$ktn|0!8)e~*~vWA>WsYaPENWs2(iH$_n&$&HZq+V=2
zmn7x&&Su5QYp~!|fk!1W@6mYrXD|{t^%5dE2-_&%?r+Sos}Du+MVR;G9!&QKUOEnL
z{q4VjgN9lmTWBhc3o63#k4bk8+8IazPRopRk^{K%2aLwvig`ku&fwAhl>BZvU0D9x
z$>$!SzhPjhH@dUppg=X8<w*7vrIg2CWDa747*_M*2x$W)9GJd=V{6WhgFK2}WnHQq
z`ONAHCG)7a>C#S~mFeheMd;#Pr#uGJ&)UZa2JZ_&_WfQhjZ^JxwTWG03AitUiA77%
z?(#q-16?ge%|BhO>oJcV^l=kt$ox5Ci`?&Gb?jVDsmhoRT8krVfBBrwe*Apve#oCG
z03yzeHQMuu5zz<(0j_Nbd@I|y)ty)f!J{u1={@YG5Ka2EWy~#&p%R*B!M4bI*iz@0
zX1zD)Jgujst$_jJ)R}z_q=O)p4Lg=WtSX!G-l-rw!qmiWC@#Yobui*upcPp+L8yLR
z5snh1STDG>W~2QuZDyz#R0BKQrTjBxd)@+V#52=R)4kSwDCYAZlo)+T0xL|HTcl$y
z#6<CL+&{iHSP&6+9;*SaGH;25Daye&M<xj0*aH2W&4@N#NYz|9A)$^j;q>y&*Mo4h
zD{8*ten_oIyy;$ynjtb+ytli;IUx#2g}Gm1f`oFTLWynSu9pFDu9@BOt7Z`*CEz){
zGL;hkXF0Q5hS{eV_n8jI>!9t}#3sbaaU=K!htos`lII-cKrSqT5Nn8ygNWeo1%?JQ
z%GQsAiXo|{j+PbI<(p}&5I==p;76eX!@LC!{CI^KG_hp}wd_o2>py8(MkhYVY}Yq$
zIq?tSxnF+q&SG05xP)r(Irmtx;e3;OI-kqt5^cYi$C|h~$GZDx$l^DU8L=kygVz0i
zcD`e2ru{MM0)Lo#C!b%E<w+tGKc%E`TbxjGPiRn8Z{R@4W^5?xQ2`fmjFm&Vx$#fC
zixyYzp8&MLM;DERe4HMqQqakS-_sU3WT6fVWJ2P6XvUw&fssEoTerc2MPWR;9GZ!v
zB;Xiol%W?ze?%@#0cB12O;Wx3jv^z_K%Tw*fJjdS8Fk-kMKijDl(9jZC*WmV)9%!u
zJw-_a@89l&;hk-X62jIy_(s{GN}W?vS#e5(&~K`4dW#>fNrTn)IonCdCsJ`Id`$Bt
z?jw);t4Jnk#)(;BG}CpkGW6iFk9R=?0QeK~AC3q5z<!s--O^T_3#m~w)UnIZ4fNxJ
z_Q^T!@m^^&_=#8>U>&<H(sePrSuaFe{{buo7NW`~!S1!P{1|z{%p_0ZdJKnbs&v(`
zdJO?Q4?^Kfpx-o87Y_HR2cJfW5URLk8K&$yQ*32tp)hlMxiCu(gp7wxk5NIs5J*_B
zA&96@fXkQ>932pOvhzYg*8zs-;bRM63b+m{!&t~xK)XNDixq}y2_-?Lm`Sr-_|Y{-
zm{0S}yV%afVyRN7>l(iHmfy!SFzw;;XVS}jl*vk710HK={-P}qcq-uIXwA~^fd?QZ
zN5Nw!bHaMQ?ztu{*UXticx!~+Ju(_cDW*6O2(Yh(5CA5UkUhk%6zBKLyy*)vW3Bhb
zGE^v;7l)>d@<$BpmIz>Y77v>5W6v7|?GVz`ms_ijsqtN`A$Lwck~%gLYJ*vCCCI31
zDXum5gG|PfqjmnfzoSC!n?QJ_%&$xG-+BEAdvAaBQ?mILFq5kUR;j;vM!VHj(@fmo
zBfa3leQkmQt%E-Md1TWf7h?U)``HnQvKT4+FoTWEXwlY^&lreDWJl%*bM~XLbQDt}
ziyr{bGx!j!F<vUITl+68?y9*m&Z9&I%!9qTygKAavX&r{GbO$pK!E84`ZZVOpsH*4
z5?yQ8Xxq>wxLImfcXVx)$n1b=)kK>#N_ybhEL5W7OQr>)7T0wngcshK{Tfm18yTQW
zxBCvXI?Ij_JT}v97tB?@w0vcdBiv+Vh(#J#96`*b*QFlOc*}SG37!qjylYm%Q)bFN
zL<TtGO4kH5{s)tm+<9odiE~gDGwd*aMHfwz0E2hH?^`C6aa=VV4C)TkjFmFi{VC8d
z_M^)HNL%%-Picy@*Hf6!L-5yL8K!O6HWGlmvduk`#1LhMPSIiHgf>w(u<s@EOT|{4
z$qtgfJ_!-(qA9X+#V1cZtpCuq9CNhWv4{<BG)@Gf;ERu{=<Z|)g+DNCH+r9>Fj^To
zx*Y{gD?ONsi>m5r`$Ce~S(rr7Sk#k8E6K^D;$Vqfoi)GA^Hqc%ek?$0T$n2>t<Igk
z5k}M!*$!aRD!bE)pw5~JZM{`P8+thi9mEYA)Rlsv{PXwi^z#A6jFfy}YlXt#8XK72
z8beqRCLfjB<4Q->RRZ9vqbEFJvN!KXg~YNNzis6;w#HTGPWD?0OBHLzhqt)!3FLA;
zho#&Xfc<)F`d{pq4RG?`8%oZHK@f|(GuC>F;143GiBRnu8ulUh=Su@&lQ$%;4MU53
z-}40mw~f{O<SIJHEYlJx2X^DYg~^a9`&>9q7fZ3cpUO?yAp~SGRGq!8HqgD8w6E{{
z@uM?;Z~`Ls_3b5>$1M2cl-Dr{wO%t61}Tjh$cH(i^VPEN^+Irzut`td%iw)<S+B9-
z#zn$$8t}s2I`8baT!$&yTR#G3JX&v~$S1(dQ;ilP&>i9pP`4Z?o?>|4JGKwV0MMC6
z*`YTl6pQYnf;MIY$*<w7vpIhlbwsaf-OxATdPT$H05;g*FKBE$;L|7@LVnmNvzIUa
z-oI~t^>`>Nh3%IcRs#p!19=>+)XJQ{Gr?>v*KrUf)7}k0Cj%QHB9qZ0mlcasBovOt
ztGtsk3J+u3meH41@;Ryo)YmoMh2ZXcGg7emn-&sG;ZF(<k**<6G8+6gpC2~~LdifE
zEdBtR8h)-mdr&K+kOYP+_q)<yE;NSrWORkD$cH;AmpHJiQ?*2gds&T91Onj;<K{+N
z4)6GhUW-&p2xlRqA5nsAbpW#M+^$?q)%=+Wf+TDp$YuL_9zwxNU3~zagbfB+?clXt
z5D{MU!%@tZx&fh@{66J5MXv^@$I*tUo<7A_w0Xp}1Y|I>>JR0lLX4{qj3RY&oA}*`
z7fT`4Brvg9V?rRX>GpaFGad}JE>RB4(t)Ej(p^yqJv^nEcZ!Sw#$tCgVonyFMu}NT
zNn1%UtJCMNh<VJYZFlO-yyxCaBaO=a&orwym`a6a#wByUrOueiQbb0;+#U+4F=R3<
zmUl=x6e!CT%RHSYmXCJwNQFomR5~$8>Pnla3w}RIiDphWV>@$Grok+$+0{E?ViDD4
za8o@FAi6n^V<#nm12RzDsRG3vh+l*`L)%dHISZj9#TaN@-$pcl3NstBT@#f5mIUIL
zGuI&F;i9-kBO0`xnl4Yr;o{TF!}8dc10c;(!CVl3ur+bxY*fv<YAnM#S)5j;7D9%C
zWt!Ha5_3^9)B$_Sn&I9YiMH00BUEZVlH9qOmuJX(yMn6wT~ky8s;22RuMJ;l)QyG=
zS^MeK>mKFE1~yRz3;JZ~wT0*x2^{Jz8)2;?ZS@p|8Mz=+NZfBIP?P#l^`l=^cSqZ-
zim0n)!t#Dhrzml&>w1k!(z;_LFlu8UH~@|XuJ@zpY?=_GIYAXSD|*3UB|H6fJ@U^x
zhxmf0R=Or{=_=*MKjzLcMnJ9@$Gec5oKUecR+Q3r&3%nFi4Nz0)&*w+hrS}I;VH={
zGwFn#V`jv=p?oOv0838Y_Q`%lv5iQ)rM|7e0Z}kD>~c!^=30e+%NeQ4gUY(ANUN-4
z)^c<E;m(Sx=OdOtOxNQyetNx9quSnv5&&}G|3IE(&MK;(4%t7drXc9~k`!@FVO3Zf
z!#HA9{%c*c0~>|0E<(}J7}*1AZc&}w@<US4f>XV2Dz0-eAsK!!2aZwkR_def3ay)$
zHoUMTF}B6Sp4MT|E`<r22?(!=xr<ft2kuypd9BF}pdJ9sm9RpdO0f-NO5A2?X{v0<
z^n-I5+gk)E6WYXmB5!Cyc0?5Bl3Tnfz62>!2$y{O?KUr?bU3_s!yJ(~omkUFMi?A%
zR{OPR74ul23=}n7V@UEE+&7Ri+B=!5qa~iHi$?huL!V8jWTZXSLgi>){D<BylL&(D
zN8E!3^^m&c54GY4mgh}l>SV+TNP3%0-!}oJ-o7^y??VPFHv3$ayKz^SI2kv)FZeDI
z%Qz=v3FB=Q)|$kNObp*pt$zkA)lfG}n~>?JwVEs-0wQoFEh2&}6kUuFsLB!U_DQoM
z&t{GzKB7IsB@8j<_hg4}{<EtGz_(rfWqZp>m*LO(pYFc!;p4(igFhTaEj=S$T7SJN
z^|-}xuw}((Tf>a{aA$F|EyOM5YQPU4@u^)kg&|@ZQaT}a7ES#=ZfIl0yaP@l*h*@I
z^E*AM`GAaf`cY^7D~&CBB@(2sV@&_-{RX2>cRa$$>;eBGLlNRB$AA!4My#WGPn2<q
zk;Qc8%bmc8J?#hjK^jmV=AlzhDy&GWiv*vuUY;eOij6N*)x$+^(>J6*mEL(gxp}J5
zgT4G=Ptd#Ov$oTE?MYO*)e1=dydM$3L^$DLg8*WgS4WALz8>bERDkvcY>j_t#eF5R
z@nBR0u^r{G^<~PO@ZcugS-Z@&n#m^_F%#LrZxferZNo01T_>1Eyjgg7r_IuZ*-St?
z{zE!c6#lbtFiwkpC}|H5_qB?6RiN#seK3?X#5J?oGm@%Kw6vowTfo)zeW)5)9SRif
zw!15!h>|Y&M@Oq>2(LbHuh8KM^p}DR+dwTp*ZqJTeD8E*REVrFWD9kTwtYgCT7+z`
zIt;X0)TM0Ec0Ko8q6lyZ#Uv|7__$2KPGjB8t&%AJ*KppNk*eqMmRE70#uZh{!j#)S
zUi<O20F{jgzoyC{gM{{ZTi_h2FwI6DUJn}K0hRy7E8xyGMSuXNXq01M1~viTj}Otg
zkpP5A2E6pqE1UFT44uXxUylSh^TZDUSH$!>FywGSD{Q64a=f?G>}Y6oDy$gAma(N{
zWw-Y#vrl!vyPq#;6T22Lkz7@vdb2&_UpJ*bdID8og$>9C%Qc`9?+(C`WAv*bwmc5U
zsU|l{utXK2LxFbzcS-2O$fLT2q8QMVL<TDsWwKDI7Dc8_nn!hhyTX4vGn-gl;M+Uq
za&?`|+<WmlU0GQunLKtk-+gH(--73TCFHNJ%O(lTS=qyoBO<=apWUv%yT8c}p)MW@
zlu{~J(q;y&jANb<v`pQ5rr2rKK~}aZeQsdp3!WQn-z2<piC2Lexe|j@ywsM=QHNux
zJ;9R*5Y6ijs(*gvSO`;<{bmkkd9X-I&BkBlGwu;{_Gn83=_aMiCVu7m63_y*Z~nT|
zKF)hU>(P4RTH5g%44=NEB=a29n5lf?5}3}m7X5VgVba}+<(p;4q%DmQaHr9;T;wdG
z?b~g9k;!UBW4qQ>wx&-YtO@_q5B_N{4x9A&LWV^#D*{me`}_g7M>_?BqWtGiJ9p`R
zc2d-Iz9Z=qmT5mII>+z9OH8v8px>-?j=1GEzPyIWbfh@Cb~=5(9HLDAL+3vJ!M5ds
z+ShTI#X(P*G1W@Y2lFItThndPO7NBahwuDKTK0TgriHg;#gIhiWfYCEb-9G}k5g*d
z@efXPm7ek5r4<d$md^Go9wfew{nUdVZr<b4=k59R5D=SbL{^u%`CrY;7TF&3CamG?
zl<wPCLH)lT)tY-}!!{Jgbmnifb(RJy?AUTlvryfG6Syj+7NqUs#01Zg62;skFd<*^
zi}TC~<?38HWc?PV`_w<n;(a|{RjlXO^SM{G5K`WI!$Qj?Axt+=c#y+?9>QVz?5@&t
zrLHCS)pK~wUI}V+!tG<cQe~M!+unGH?;30$<+rT0aSQk`1f@){9*0Ot{*2D(h=aHz
zQz>j2lWV;yuSPEYovn1`xfXM3MmYl3ac>Kn+wBL2607qIOo7YnLuEqlyb?0U=Occl
zg17z31U31|mn#7qu%ZW_r$+*!2hl9(QE%N=q7p0oLKG^n%S)0CXFafHN}1rIH~<Yv
zc)EM+WNsejkP$0hhBw8V$al&~^UTlh^|4LgB{&Kz^8T6<nxs~DviM5v=;@>TnR1fq
zAd=)`Qk-pJqVtF+@CfT|ABIF+3GM0zPXY^e-b?+{yJ(NsBaV`Uv;}_O)l@lxENe<r
z?S%=BBl_ggx=?{9)jy!ngtJ~h-Y{OW?3(gc$HWYNU0!@a@%V^Lh4j07(J*<X6myO(
zC*J3}-F8kcrt(kQq;Is(Lu<O0V=wGYI?LMaAx*tVl8i3WZ)!U4y<tjLxkOY5NooJD
zaF<t%PYF-YElIl;$wH>u2ZdzdY)L(r%zThl?o@o|-4YC8W=Us7-f|o21m74c0eSvr
z4A{76gmQMQ_i(RwR6TzCcn!Y&_mi}qjY<r#ldjRxiMLgX`m%WJB8=0{Kc5Jt2QsM6
z_zoAdc;+D2u*aMofzZ0XVeo2@20azq^K#Ru&N_&+Um<+Mx7tF-YM8!o?UYKap>Fxz
z1o~y^>Q$n>V#aQ)L1v_%DbT3U?9#$jE|ro#%&4>THTh~rzL{#RgI@yrr5`r)*9E%S
zwrh$|5ZdyWA_$aI_ieON9{&A(a-naxybt(A6sOSozS7DmN*yxEWE@qk8Jzb`=B~Fz
zZL@2>3lpqW)U4bmnu9Y)o<$7Q88SqSd#s%ml|DsioWqg7|3`b@8P(*rwJR|QC<uy*
zs6Z@$bOiyWCFr(5q)2Z8+)D4g28f9027yiQpi%@RAf3=fMTiguLg)c05dr}cS_pyr
z#(nO&-*?XWcgMKn8+UB}B*2^XuC?Y|b3XH#PZIh{c!(jNab3=nf@rn1+d6qecB2i1
z6SktK6Pqt8-w+k$^Z8(@F?qru)#>WawqS+FXYTHoS{iA|%@@k;sr6~JYi<@bF+<HD
z&>9@I!O%l`ZM?$Fdootua(y0*>gLe+`BrQy!$e5bE;i0^QSEZn*SXyS7_`=mLi<u|
zRPnZJ>I-qGAObd&V%sj^=aK+6r=69MP%JjQctg4CjT3O(Eq*J2DQdnK3Ew9ij4NLK
z5hIXo$Z=}F`bONjPIw>2RqKG4FQbf}eKW~ajq*xodAG{S`H^twxKAl_WvzX&00Wzg
z$=eAWu)BqEzbrwY){34ip%3G4PQ|u`m>53KY0F!m(8#3LhxNQUTGHy<G15s(YD>F2
zsJtaGbf|I9BQ65Ac@Y6B{PcZfC9Jcu^6~)x=`V`pgkZkKmM!R&q|^c@G@D3pC()!L
zWXDEH5qJ*<?*VA0_1-YlZS$gxO~1U8!5>bU`e_|M6+F7n4&X<Fi)((583>SNE_&j*
z?#$Nbr3!I<^Dm6`qSXngJDpzbR(r1_l{?!dCGiwqzE_jzvfo0|T-A&Zx282YdMKv|
zPg0xP(6aA1`1M@+6BD5YN|uMtyY2g9x2gkE;&C@el<Y+-N7XJ9Ch_CZ^K{D9jnj5q
z<HYr$0Cx+gcwtNG)4^CB(Zmyc>d=`N_^lrZ33hEKQ6tHf$GK`}AMaL2t26EJ{;ErZ
zwdqb^rDF+c9yfpg5X;{0EiZF94PN4#DPLA#WUI32-=wG+>3i1#b48c;Sof{8Bb<+g
z1T7Wj)R!6Wlj~>0+*d3o-lGdbYzGW%$IZgHgl7dyjnF;Uflszw<;R=KVe*3}`&uk8
z(O(0d-bl!p@Ed*cG1kg}%G_OHye(E#znr`<=lACd7gi%UC%*ml+{}RFx+LU^On<OU
z+J`*2T<hB%x?b2g_fkV*^zde#VdvEhgTn%bS4NC&$8*EWSCZa`m%C&9YB5*Dc{{qV
zz_q8bDT`<42OF$cl5eh@z!Y-1%m4IJo+r0Vyp!YQeR03I$kUk(_hVjT49=!?ILr`x
znO$B=>jrcoT5&>cDRiR6#7~xL*;6H9)ZMu3Y_04r(!Z<|7ER|_(idnbOu5;Fjq<{l
zZ)`!`Tck2_N1KS~EqWKU`E|*1WB-#4ux=~8R?Zl@zZPz)IwZqRtsLG{HS4bP2{V9X
zwT%e{j39^A2OI2YWAK5`C#H&0?mtN@o3JNf1eRYb;v!&cTI~wT^$T^Eq9WUTm@QcZ
zDw5z5#(yJEb%<nFtP=Q5L)po##&6)mPl8MoCRJ|WI80|R^TX3@Qp)Z`1xYbqD4$uD
z*4o1LTPo5ro>1eu{rQuh&lItxLk#t9{JD5V)}RfohVjj+evy_mn&p*a*Fy}Z(q>=M
zkybAC<ciB4*_&r7t-DeqJ_od|)fO3BQXBmtt}S&BhLYME&#eTI+s)9bjeY3ZfP^U%
z>Z)`AzRZniMKgHtuvQ}c&5cRlHQ(i(-Px5eEA8n=78f7yU4!d=da6Z>x5NeOro~qj
z7x~Hj(e6ykcUrkcAO{+Dh#@|>H8JCQ{fbWfQBK$?cfbk&&PXQ>AJaYcg=Q+*fpEZ`
z^)Pt$Cpyx<LTd0`5%*=bCpGhZ)%}igr{07ll3Hsdbxlhfc@bK|t<SffzUi8yxw4;<
z5y{*9$qp_`sXoUB0xKo7zed!YIuEHt=@Lu*6_GN<emxZB*4q<j=9c>G_yg5<=f{*#
zr5qyrAA+0H?*{ZaTwnvmT!e_&WcxhS<K%-}DUWClwYY)ZcmP3Air{3UQ^@WS;!oEJ
zku(R5)uG<&dQRqfvJXT-S%$yzTf8yM7P$Nv1ZI1XOM3J+9A-;n2KIkg)Y`%X8LBd$
zC+U{&{hZ`AuX?|pvsWje@m_lhB6IGuVmYEJffvmA6#>)G*fO-`0PbCB&lW__<SNg<
z3vxmPueYiyJvu+RG2LYxtv2tG>MM|8LJ$eTmX#{!Zl@PDk%2`#!Mq;uQa|qfmd0mR
z6B=r9kPYQDCw({K0E*{QrK5lh8>)9}ru!w0Yipu-AyUDz`#Rf#n3T@%12&<U-5xcq
zg(hb1hS{0pz#ahKifyy&^O%<lm`66S2RF_W`OPFf7YBsRk=rljT*Qj9=3-S7G@lL_
zJ7Qknug}pE9GFXQLlR0cXPr;2i=d1TOIv8P_DqQM$wICatUf!y2C_u6%mD@~t$4^N
zfA&yWc#k(<QS<9=e>>v5GdmvArfS5Yw))(;tx2dVDB$BL7oV9j8+{w|z~UHS%y}`0
zT;ACh9uHgKldh&c$`tlCGmI@%;)@U-o&$>!k-TL`ug|VF#&2E2cxZTJfMQbP^~$6H
z3ue!;HWLLtwftHXawSVd?+|Toa&F(4_broITFRl-3t2NIu8m5aS@Ks3ag+jczB7Vp
zv>HgzSXT66pPh>dabFtwal~q^99s(Hsu2AC9u8N!>_1$B`K%a#4x`oATkHzeM_VTU
ziex_ybKaM7xB&ulx<XL5n;f|~UGqia5uz~L^WfQ;BVK2gz`$~bE<xmM`17KG4Zae9
zx{d8BW<SQ4Zqnxs=HwfOfnxl>9|s<x!cE!{?Ke5UbP4>pHp~T#`A!RF-72PWgddn3
z<<}yQT=>^TR-(ex{8J8~qWBWx<vu!9@&jX<B>`5{2|XnSi)bJJbx=<I_HVv=kQuln
z$p$)d9<cuN0b6m_#)+}Ka9jF5;6;r`0Gn#aJdbFv@lL%640PzYSxrnT@VK)!YzzXQ
zx(RF?UvY;nA22x`0EjBCnWqDW_YwHzj(q>-jy&r#@5{iP(oZ8oL(l6*=0JckGr_ZS
zE<dWTuuLvn=}6zMfq*~*J?3%1KxgmSewVcXfzJNydSnW?2PfbjH?t3^Y66o}<obdo
zd~|pS?9o@mufwPb{_lYR@VsyFUFhSGN<I*EE$TXt{}W(Lm*WX(6)$iffIUiY`enc;
zod1svSatp&Xh8TK%THkuz?6Wg{MRQi8+I^wCwN{+`QQHa>*9577%bnkc&y|2e|^<I
z$NBr^{{yy8YUdvQz=5lDt=}|_{?)#J8}V-k{`>I$GhY4k$=`wNe?Z{+J6QcoM*jaJ
z+v;9iE%!i{1$U`VK3J~oJc`%k5HfPB#hVPg`z-%|@W;HQ=l9zJ#_@q4FP5rr4!==Z
z|8}daX=2&kXF}wx_%E>HhSCe)8NhRBtfCf`>Kk?Z$}A^JnJHTURDBZ>(XR5l1wK`x
zk{HG?CglWJ`*`XF^%#~qsLcqHz4Px12R+`Xtw5HK`ry=o0(QNST%{{@9+n>tIvg+`
z&=(Kg+g|)|MDAVPg#XxLuSXco%6}HI5^U;6X(ai(hipv*R0z|*-1QG*6>iKE*BMLI
zWtPO9q27)P64DK-D5MBLrXkb(m9335>tHAT%Dp+c@R9ah;Bx};G9E$LjoWofAn+MB
zHZ~|2#1Gyhh3^$(M)7D9<}L~9F|RFC8V>v|xAJ38oAyV>jL3fAz>g^hC}X5Y;F>rv
zv;{Cvi_;2W4AnN8D&8&)VKVXaj&iBSRwJ%AklZ$D^#ofHIwz%CpJQQ6N8STA_A&_B
zW?zu7aF)1zCuNtR95$nT82b##T+j=*U`}yB-}+Amk$6J5{j(3C_W6H?2qWh+985UQ
zdA9S-+R25O`wc$0%L5I$716%K#mV_DuTatL6{zc+&wazlaB_J)+IoM7V7k>rWDd>U
zV+RjKeb?q(x6WVTJxY@vH-8!dL&H5n#>P^2g)Vst39z9=r8W2<`bXC3nO@XxanpF@
z(xWn)-5&zRhjZKmn#SCI5<ljwgpw9-iR|dtGB?LeiHylDrS<p2K_GCL%r}yn3aI2c
zozb?>(7Bz32$-ndnH7hcBghs6Dh((C6gW9KT@M+RjZ?w>wKzqz*UO^X_F$o$i1uJD
z3tm(haN~}LlToTMe0)E5Hax0(m5iNV9_I92NWYU$1R-1P)krsRCAsSb;)>9S_|Vyl
zcp-{sG!OI9Z~zfm=3D&d&{ro>$T3Le8v9Q6-Lz`u)7+)F@I4xy$0M+FQQH(_kg)_5
zLVl<C_{<#i%Pv+Lu(6?1v@_xlptL~9FwgXW>9~mY0-K^0phEhMjlj)@y1k1S75~TL
zF!60DFtqL|g}f6;seAh96yNaizSob26F@)>LaiBb0v!?mM&jULLs9bHoB~bA79IbY
z)ZV4B`MyZ?y<FH@4Bmgt&6L?8va9Kb3z@9mS&I*EF4c^x&JAd#<={UF*jU-QmXocw
zFn9#TJ8iLbtHr}_PRb_ec7e_I2Z2&T^Sq+5^+2r-1~u%_#w?9iUwYV^zcgc#IJzR}
z;ZmIAUn)#Zh`xpmn>r}+wo1>2-eX$oV#654i!i^77_)rWnz`w*?@w~py)l<u6q6rH
zj|*siqe6aHok^FU+OZB<u6>-rUAG$@z8wuO@<ZOMcU=j>g_Oc&8w+;u6=C@Falqhc
zO4wFYyv%U-b%mAieMT03xnU*_pT``@^DD>nz}9S-i>6sC-mXsU`UV!#B&+20^<rqt
z&fZoVG4`(ij~C*p-JOJJjrCXfqpC|+c9(cq48u1PW9Ji8m0TSS9N79&TAyk~H!i8z
z#<PPRzp_VJLF0PLquM3GV6X|wZs^mAZ9+JcP~_3LG~w(Xs;9bKv#3g>>(U_J!w;i*
zcqwq39fJ+M{dK)g+_JKzTdlF*`uDr_;yR(>pH7+@338H%`%Gfl3a}8oj=Oi#yylE1
zZ?L|CS)Vq?$-b)a{jG2t<h(*&iEPVyf00PbJMyvkz~5;4ieBQOS`qE1&L@Bp)=yS=
zIv@`R7+e%*CKh>0(8CU8>~`yg87#l=62?!4tRO`5`WRYHHBqp`O_QvHf;4nM5Sk`8
zN}=>5XvtBv)%N#gTKl*>=8{9!iVw2e&;A;7tc_`z60X&~W!RuBp>vk+OT)n+7##=&
z!zX+{T|zd_pBE)$Lzx3mg#T2}fF86(;;j@zp!38zF+>=B4o_rG5RLcOME0*r5B3Bs
zgV0skrXX;U!1W)`FP56NEE;FMo2m~3zDX(asEXrt$``|2YVx8WF7UG$&F;hg-palb
zt!TALs&Gn;P&lic)Nh!#Gq`Au3pO$I)KDQRRs{9h-;ezMgf4GG{g~FaJEwyR*}RB)
zzQ0qkFKNSTlLSTn;Lm%D_D(6|E>2PYb;IVC(A@PM-u+cx9nXj3YTbnW?fCr$E(70Q
zpJLUClEy>3^B#NiVm9ItYawl0A^7;P^*8fPV2&!ZzxFGAt0aNqv-#s&^%b|ev+kxj
z$w7BCI?`h0SjRjNL?Q35WPiLcCyJ2RnoXj%RJo!rE*zpPdxVGheHL^uYMh(^&Xe<#
zZmy?ob21)I6b~Oa?D{QU^%@r+omB>MCB!Caf4)_b?)U^Z5)Vdh5ySWU=I)4#nPpA&
zHIj12#w|&?JJHbHhXXBRKhx#Jb9@apw!5HVz9)N0Rsq(GQGycrp8octX<IP=S6!Go
z<I*v+Bp4GWeQfv}C_d=xqeB}anzRa8GxN3ggS_W^<^6iIM6rVce@^f81Y=#0hAodL
zwL}&zbRvVtQGI@R#e_o}JYh5hVq3vUF%sixrw{$=i!N^crbyJYLdkLQ4X=WtMZ&g+
z)uyH*d4<DF1t_uVLl}Efe(Rq?eU{W(s)fK!Zh-uLk4=-)y*~X^b0W`y6r5Nw6!pVy
zNXI%}W$~?jBj(w{)aw}8GfQ=2?m6U=BLRo}M7Fb}^SJz`4N3BI*GDV-rgcdw@@uK}
z&2t|&2HD|}0?kwGkJ4}ggvLI<Ouum;ubpXLd+Bj*P|--0U@qYN5}_w_<+Q9J6yn=w
zN(d9`GbEf_&MsE>0@9HCd_@<Zqn`C9vJXDyv5pjaetNKD;RU(7(M2vG#&Z01MjP<<
z))0yq`db<R6twe&9YeR3)0;8wf2zEdYUN(kJj3Bv5AIx{2B<7#KA@7nrZI?^L%aD$
zr)0Y8d2h@nXd-TfE$!h`Jf4kV)yYCDfn6O3g;ygwdfol2r`Kom{5rHqDx(a2g0DyF
z2oQxi{2un%<{^RamAJ?-i|6^)7-x0SBb+g&yuUa#@0pQB+@0@Sm}6jIr9MI#=`*f2
z9%Pdvktz-H4LBr4VK|7C6HU%ZCW`M%(`Pm+!kMCk#^C%J(JtS`l4AENIlsJ7_~wlu
z?;+FrK{I*vF2njmBaeglfvm#KwzUDr-9nW3q${VYRW%Q>Z^0de>FaOrs+XE?*44Kx
z=c6O4yMrfJpDACXKQgxoLIN4!MpaSA)bOMDiN|#L@St+b0G`GC4`SFl@z6!XyfaHx
zSXq8Tii^3r85M%FVP%@J7+#`-dg0en*S6;Q;5`7ogkP^v1{WQBTV(!49(+n`%C911
z;!1V7%Q2aeS`6Kl7LgGB6eIR?wFxu7=b3@2oAV5w4BhS&ahIQe+&5<<5n<s3%SSU6
z-o6>^bQ`R_$H5!68kxlr{T<Q)boCGljtMchySJ;MyrEMEIU1)@K#ie!^h+Rc-F>zw
zRXC0ue3Ei5VBAY*GE<F?4rp;Pv`q!A1Aco2*W0^U0Q2GEyc93X?$)%@8j?g5C&~o{
zSd&NZz<Fo)!!3Xt1VON!UFsCCcujOXS9UU4C+?z#pU!ySqwbH=NjKY0_}lGm=lb;B
zh_oXAarbb44+8^_eNU#cvw_O^-&(5jXrK4mn_o1sgPSCTjQfiwd9k4|e&kh;YwXf~
zGTqa`i=%(^wm8zyqFt1Zwz;&GSf1rMZq^$r@m#Otj;cD41kbu(WZy-U2=RRh&fT3A
z=dEHUCRU^}@3p^jcQbBX3`K5M=c#A62^%%|<a+Vv(Z4zzP1AuuaK}YO^S>W$Xamxx
ztc@Er^T|#XZW;3V2$0dI*BQk9y)0>h;&~jqxhbG@AiE3zJ^}-LBr7+$Sbf>4(>ra}
zg=;u9)Vq)kB_E!uQ;9*YWUjb}&Yb049BS)i>>520)G29B4Dw;#4}we1&I-Ol2q&$O
zDWi7;gu?oKZe>6_M#lw;V;5(W^xP@&)uECUUv^})R0%PZmPNEW9)a5<H7pJt?{smA
zdruj8-IBfqHW7tDDkWtV<X*#^X_p@Lwy`iwC*teNV2S+h03h#%tMAIzZeitTE>v}9
z0F8g+6uRjdzPDMPJcPWMFRbjzYnsN9abIdoM1XI5v%YOY4QAI0zH~=qanW7k885Oi
zAFuQ6*%;6WK*`*KP4~@9jnj7sZ5nTzrUeU3>{UhOSb607*Efczm1JuVf+QZ7HPxIR
zO4V`ew@T8s&Ku%`Zg=o1nd2|8gBLYEWKGc>UlOhK49T_U;KKJ(;TzjWJLzO}Qs!34
z$!xNRfkchk-u6tIWy%V`6TfP=Vd8Cg_9^{O>ZInvs&-`p*uma73kyL!f}eY_HBQ-F
z$07-jxO|OV(j_)L%ea3pqSjR$^&D+Pi8g%uz138fR(BA@^~1l=UpVuWYf^s=;7w)G
z52p>NOru>vokkD2D40a0U|JPAm>cz3Ofwd9q*l`+ECr@xadr@q8D|;V?F*poa~gbR
zfq=0AdttfWH9b3P4K?BCYalWy%FPvewzXD~lu(i?@1pFLEIf2~npZ@3e*_n1qBPPx
z?=zt|me}sodQXaMrE1uo=Qn@m#?G$3iBW*%HLLL4BX<m&{O(QEe*a16B`vuGCx59k
z$W1wgT@ImPLK+`dLfEFq+u%Sv;Nwfb(<^^OG=W<|d9eG+fNOoh<N3J@353mFde-E6
z(R9F!ut85S6_XhOV}Me#7-&%bl27J6hqlk`;8DbRJhGxLz+t<UvgH?eF3I`*Q2~}s
zc|`L)9cU*g(pwk5=mbbCX}i(AO2FWLV}12g9b(mm-`-8|=&9$=iP36~RL@V?eG^k9
zWF|zvva0Z&=-F^-96ZYTI75AF+(+xY1sA6pP(LBuja$WGwoxEf?X%iiDeO_?Qxd#e
zI<1CaL;1JcTIrvc(!HJKEuuP#ys18kuQbqA5A-C3J^lgRn}HtPDx|msCDBLOP+k+?
zMOS*mW^FLu>B&VroHdsBcYBGuy~8>MVvd^^Ct&xrJ&<I}Xhych+vOp41pB?2A)ryD
z``H&1BQez)MnxiTxGYnXQx1U*JSSzRDmdaq=Bt<6{Uu)~yWDY}t`HNQ(ofY9aDOAw
z@xwqiWvB{KBFtgi5NC8@S$pF&tTSkFtqEf<SW|JaINMvBq$YOUOcw^T1zKtzoT%Ug
zwAjFX;wRC$XDyG1gWLobYqFT=*_)NX_Pftw{rTC1)3gTztYG=;KABI7{AoWocNROI
zMUVAITUmY%0a-qa_>j|U*ifBl)3r?2NQYZbhP6A&S%N@Ghrf)W#?T*DULNQB;+3|>
zW^ly6@!A=_doHp5lE~Mx%3_)!g23rWm)74yi`6<JWFf0h#nm3BZxyc0c}YB~Nmlc&
zA@}$`)NjoiEDk$N$A)f1%w)Mbi?X3k^d$Q|ZhoreG#UZ>Y<qd^EyqFOrWdnocCA?1
zCfB7<`US*`)F3SRQW-jhi_^Cx{SMGc8z<<sR>(QH11q!0tTp7a5D$0TD)`n4)rl#`
z7x>alsAP0cdWK0WbUyvjXEX-PV=y_8Lh-L};vpoNJVr>Qq#Y0K+w^{!$a()F1ajqN
zrrjxBh~kq(!6LeeldkcFh8KOQu{Vy56#SN{;MRCXZ4PG2&*uo`zJyVg)j7;rkm@)>
zT8~tgNQrLq`aYwD-HAJZ`gH1YIPq`*1~*U>rMg^OnDvJOPR%DF^%2**BJ7J@mo1;R
z)CgEecGksswW^kddFF*f;C_+KyU?BQnuox>isk9avN9%T`OMG>?))cM3BMM0SgEIF
z1}F#A{poWc(46oMvUXugY;^yI9HFqP<$JqklmDan1q+P<ZTP#d1IDLY6M-H1J%u^!
zM?iE52ehVdX8w8AxJRiU<21fj5K|~mENIu<6y4gf%b%4_a!5Rfbxw?Ecdq#gx7`B(
zu{ehtRxmRFHpPPHQ{z53p#r-VxHUDggN5bit*;|yQ`B0eAgciP5wy)@!6fhNaO<%-
zwPj)HQH7Kzd%kCLkb-I6;W^CiK%3(zC!ppkdQkoK16K`)7?%B#1XEEw@+&8`#4c~E
zI4>-ihB$QfE6GczF*Czrzg!HW|AgOszd+Mkj2(QOtB*QIY~YB$FL!WWTHv~)P+UFG
z2t5Em0=%uyNy_YcF+Ao`yn3DBdfja8j5Xmn-*#F!J?)|Oq+hRjK}5S~nw9)LHY<s3
z5z_n7h;~*n%d;xrAi(A7`ip4lY_?~ixR5JYAQ}brH>lbA?zmXfGDXL*+g5LA(pxts
zmSc-jz~HL;g4|(^9e%~YiFetM4LCQ)@QuDEVV3~NAg^I(!!!sn_)BakykkaMdnn+W
za^W^Mecu4J6BOWLH~`q+pGeVw=W6VXp5*(ow*pjny&G~TjoYXx70I{GwAOJXK-WI<
zM-O1Ld^=2g0;+no=3s@fWJ%LTZS!#{lrf8)`2+hkM)Dg4x_(iF{$+}al@rl;{`lK;
zf1&>CCN0s$%n$h~1$ZzRDE*tb>(!k?pJ=h6fI8TtA?|w&t^p?)WPJf<i#?{_V929n
zI<0%iQ(Aec+tBc!c$6)xX5v_Im*qVOl<<TrBGzyi1mTn%sv4#IrUbFAgu4os#r%9d
zU3-;rmcb(DBG=Db->2#)t^m%O?kjU{V>ROdY609Ydb8W5|AUj2g)HQoFl_fhe6Y~~
zF8px4I1?UGo74Y${t`Ba;#>Rzo4R2gI32ZYd~L<_IN!aPqN;K&RKhz?z(^e&B5LDz
zJ|iwO)D{#Yx#QsyzS>f8Rm3^PiMusTb*Jk?Gh`w9mzs~8g#x(`OC+RKUyla@N1*GL
z^q*I`<WH~J?0&N0?w|0@<uQ2EHcOZRfhqz&Nj+oV5xgs1&h7p3z9%H|ptyd5U%~AW
z7ORnZOSrYJV$RMI+hT7c3K{@tru%Q~Gott%IS%w+fGGd$zRs`M3wyqxE1tIk0AEGr
zQBgyL6Y*0S7iWz~!5x?TwbOvNj`sE%98g$5y$P)!wYa^ET+0Ol0>2}XPjY-g13GYu
zQe){|IPYNEp~EN&2jmm};ja;hCK+v5{)Ei}lH!x6d3fZGp_8Nx<cj2s0_qdrFhBu5
zi#oY*v|LI?fG=>UI#<!^?v8D)7{IHEoJC*UY8qhq=JBu6TN}JVUv7!HJCC~Dk>pEp
zD=?;781AIXlHsO)%U2I$EVN_CawFQW3l4Be=SDDgoGY!q)n(TPfx!j|Hq<#QxR0Bl
z&azhrkd&*wT24|^r!hS$V-+RY!5!=ZvFNbv8xtETbhtULfPm+|108>^40dMGoRKnj
zYyl$HUB0giy0@kS_2~Z1Fd;Abxw}TOr@D?W{7_cwv`HcWN&>&fSOdVq>CbPqF9Ek4
zJcj{1fBc=ePW$oqshoU2^Zv>^J}+IY9;NFQ*TPIKDk5MQtOOx@utr1&VD!Ey7G2ZR
zC7eZcL@d=uwErBTWwieegp^x)F(c@87T1+7URcL)>^U$ciW?2zXWnhI{!Riv;J^Mg
z*r4+LXzLpGjtSsutrnZgwb#W9H*FIEDpFOv7c<+m3F+QyCaI?ZhU0a@XOCu&kWT#E
z{~^$J+RE$G$w=zCqy-k$V_Gwc+?uF2*OP?Jjc5Xg>HYuCLLPvQd3JfuLS98ycccQy
z?b->-<!&Cq16fKliG}QDpHJRP(=6G&Apy|MgtHX5snioz!f1D9@lkK=CZkSWj#3u9
zzfB2f3rTvR0bn(c%^ME}5<A+~3+3D<N%o;u%O&3rB6MI=tYX5`GN6TwxNEAoYlXSm
z6KP<I0Z4RZtN77<#Lc2(!&LSYly~9Nx`#`9HXu=#C1Mw}=Q0z62>mx9G6OxEn8QJ%
zhzGI$O%FE9gsz<yXunLGuf2A+6<_X-&sE!G6594QrZwyna9&z4+w9l{@Tpr<07G|n
zPllwSfIL<|Kxo~pojz^ZorVE{ScS<U<>qOgXmZf#K<<$Br&T27mAD^AYRGEh)?90I
znK%3pJai?5sOtaiqd#m)Zl5a0F;efZy(%9iZKs;#>}uc0Eo?Kd1Am`VWOZ<qJ?Iy%
zc`Y%V=Zo8$SP82y$xcRo*=AcXKo``vZj^T-;Tq$3@A9vf)`ks6apDsO023}<2Y7Y&
z*2dgYFQB|e1;N9DH%A=O5av`2kPe$%-a2)CQvo=vXN<&2xl5odH~^V^x-C||vI}oD
zbSVmwSe3rm;Vf?TsC}*F7;OnhU=u3Zeju%MV30A%$SgY%%%M}dQqR0Q1O@o+1>CXA
z1@L9{LI~O=8h3+~eNz3JuZ+tXfQgjJ_axoo*Zal-08erThtBEzB00U-wS}{9W}|&`
z$X3PO#r$SJd&o3>edo#IT2oV0_>P#&*n*^02oMEScQt6`mmvC^QjrJa&!*)g)wuq6
zyx$X6AUffw{m@wOfI^q5#kcD!;Z0)h1h1EUD+++@MBpCESCD)$9LF2{Eix0d2nOkD
z&}TI%NGvhPx(NgtiD41hk<0bOu<I*--W9%auQgrs>HtQG2`T=jLLX~L<>We@4Een-
zr8PY%{;`d`bvuhlo<41}Q1n&`sG!X6N-_(6=sZd9Zi3HchSpmJXqxW>l(U*EDr@&@
zEv${+>!H+@vLO`w^A-H?!WhGMk51yPI8tJiO!kRW7n4xyx%yF>{>W4~(@)6z%lTUa
zvJC$Nbk?~Iq2;WMMJf}2oT>*9bQ|iN!{OJ$JKv&ZvrGff!@mK_yO_U>%ie<UPYMk(
z=rL1nJTx0TsJlRw9iUaHE5R9i*XRLS%ZF9OvpsKSsD-x7Cc+ObXEGeJy*{rtZD#ok
z>K@+oX|ZtAwr#ArH1;rB=#U_PwN#<y8#&|x9AT4j$M!bB!b|-wKX8Zd`f|;{vy8Cr
zT^zt{PowKaT&c*Ndlc;wx8%9eeRt2uv)9Oz?oG?7_i<{7a&CjH(xpyx6GbHuUuKP0
zyE6Mwl2>}JsW}dixe|ppE2DWxI&S4ww*9Wsg5k0^3q?P&cnI{Zt1p2<M;0h_s9f1~
zo}~Q9qi5!UH%ctKW!=m<@dv4hDJmDFWtczER}8R8$^n#n5`nm8327Ei$a+!(<}+Bu
zw^Cxu9%L@qSxTtEMP30QGXBuQ&>JMrW!k$-kc1&8yFImgcKJK3IM=%&#U6xM0D5Y@
zo*A*?31my>nXpFpq3r15<HMK9LZ<x6n(-$#Q|9|Kpak@NY@64_F~8*`CxB10&Yuv~
zr-!(;)pZ?NkW7}f-2HhMs=^JbOA|rnbqseE&T=&J3M1^-eK4&H+HcrU-{*Qrt``Z|
zlmh?r56oMc!BZ%XI7BbNT%&cUA2zFk@_>VIpo?s?zw@f|rAO#Y-zk0xFt+-RIjN*L
zSH<6kdfNg~6M4a|anl9u#1asmpK5Jos1?*2H7x;O{MX&pqwYl4=os;5C5g{I5@X|T
z^%I$lZQp=2#!yYWGm+-dT4O=h&fESd{){X^h6@CDkJ01~_7q2rVr}-Q@5_^`C)Es-
z=Kwz1MF}U)#eL~*u6*G*ku<q0rrh{Kb_S#S)Jf_%(C0apl%ezlcKD2TQ3hc=p;*7L
z;KWW%TR1f-&#KjeI@%XHMvD*G2WWfmZYn(?Gill0;SILP-+3nqyAC8e?&qP=s^bs4
z!{X6Wc|Ir$t@Gc-RGmS9Do&1Nd5O#-Pf`rfBCE8_Ni0tFvZUT1#DhJsHO=#96f%1{
zHK}U`NyYCfn3)!NGyN>etf<H>WN(up6fpk2<fLU$`K1cnEyIzmDl0}U8I$6)Jmi+q
z))E&=IO<4BHw;wFOqI$$BVZ+nh{u6B@5?1TdB$xqyw`*qmJl41?7&g$H6$Pj<)ob?
zFYqY2n*oL?KtiaQa|X^=itNKxq1`Hp4n?+?P6pp;0!flxIl%q$`P{F^Ci9kTOmj_!
zuI^-ptb^~2v9H)67s4*%*pm9kh<lqItlp~5{X}uG(_?rQ2z?i5{cPB0oS2Fw$6Xkc
zL1fwI_3!(p<|NCu?NW<8OUP%EDOv`emjFusI#7Wqs|zgrsG9A>wiKPTQY|+HplGM4
znJR|4Su54?q9#s?R?zQYs;%EJOShXu6Ow6{ctz6Wzl0LJmz^4QLw9FHMCuO@aNR!v
zkP}~KurDtqe4t$$kP)2y)t}lLn~;B;8@o4|G8&nb;-?b~bWfmTwV6eY6js-4dhfAN
z5GSiYcF4(l6R2Qc(cKMT5GmaN-jOZ&>y{xl$ISv+T-}DPTaKzb&?r744p?$mBBRbz
zymlTQ;JF;Ch^R@Sg!d!#F#TFO`{=z2IBaUQlUJl5!4YV(StOE3=Pw|X3jFs9l95_G
zd7EL6n*_TcitA&Fg#beNQYUXaPXUO`$2sQV&gz9Y3Oe~r$8u|@LbybYD($O6*6lc;
zlU;OW6+>uoCrB#Yh}*n6*Af8}&w2qKLO>$FS_4h5NLFuW!5zZf(91Hocr^${eGR_r
zgKHU04!ttLV<!~kW;!ZZj|LJlKqcN(upFXw@P<`#nY=43dwODTR#-#EeP9D;+Dc7%
zk9vV-!Y8xU6k78b<D~gXMZH`Bd5AA{Z(BBCVnIa=;2hIm0uj2fkLybOq+A>$+dX8o
z1gHWT^ZwfPQ1iQ#PptF8-KGs|c-J&xr*D4`J`JwKZp|rbMkF5-4a}j;89ag#=gz3g
z+GR-qQr?23OqkD;-Y($~SHn|-venuN53PD?l~9cDq!?FKoEXrdjzz!en$BpwfFgde
z+Thg&z?cHpAnyZX#M#<Xm&e9_SMp0;9Cd07G2Ir%&}}1g8$?-2spV;-jZcoHe&){)
zpk9jW3s@WZwe=YzAXfOO_r~hh+QuA$#VQFH`v7ABdm}%#o5lhWrhex=jS1UUc+8@^
zS1AUYy9lu5Ws6VNO3@4Ecog;p`TEY!H)T8L-)0ocj+ZR9yQORlDi0bX%<xO_iLQ;0
zezA-y*-8)UmnB2$mmSor(jy<sFtCvE3neGM&z5+VyL>hAOH(&SMz%cO!}#0J<nD%u
zmdF~FE`qAXD%!rkbT`5)uSuD|yRo+vf)?;Ai?Fv@<al;cM%BGY#moRd@l}x|A-48>
z<x<kl!{VN+j~m_UMv6)!#f%XQ$8It2K6#u5pksK90rcQ}W_l>J%jaM;5d&yKugC%A
z6D`putZ5^kVl4Ete6iinxSq3Y(S0FRI+;PnL|GXVR&C^wg(B2*CIR>nwa8g|nY*Yy
z{t<gDqWLKIHQe=}L+=irGe*34TWYBtn?n=;n@9rG?_X+XmeR%LxPC+RU;5&FQxICP
zMP=e>Dv<WXAo1409~Or=j3<uQmE5CE^M(g5q+QHgbiZDv+`9I%cQBZ5x|A{*{P?H{
zmunc%X7BZIJ@S{51ST`p(SiG=qcM@3%6$J1ZS^LI`m=CeF8F&)=}ep|I>RhrdT9OV
zd*3+VEl)8KHS6dme;2V?p9z@<gm(Z&eg~xctQ0~IpqK`PC#VZk9k1>Min5;0fvpr$
z`KjuFAY)z}n5Q!!?;x2nC`}#;tY#E?QDZMl)_9;r4<m?K?}wKKFJ&&KRD{~4Q1X)&
zHUBJ2($i=RT7iP90hvI0ZNoAk{02tNJsjxybKyn6FtCX9n-D^*DS(`f)lUZ-^G=P$
zM#_J3mcGeaQ<i2sl4&p)+v&H@AhwlqWC5S4n69@4;H0u($hN&z#MSEb`?l4G0VRZq
zwbWmt8So!K^>kr6g{5^%<-f<%Gg#H!pMtI#e+O`$z`X^JG*l2z>;i<2o!_Wau`#PI
z`PCuE{P;uanF3<j*|f}m#4gQwpSe{5^wcZuZKtQxCi&7M0m;p|Z*GadM0D#9VAHxH
zILsqJ^!@7(e8c@<7SnNuAF>S?(2OPc!7GiqR_}u5Ay;kzMicyf>;y2U|1LoO>$T}=
zz}U^RZfSr0`fpm}ulfK&e>SXi`ycoE*9+!>9>W#6pfQ%o{?jcKSkeD{ozAb_|Mv?}
zK>(xj6+Y?zm;L|kf_82|C2jxQ&G0`?9?+s#1O7uxtVjIk)&0Hvzpekb^Z$MJ|6}y{
kJM{e>`TuVc-|GH>Si-{xKXJ;22Y?@~+q%D@Z`wZjU(8sPF#rGn

literal 0
HcmV?d00001

diff --git a/docs/figures/widedeep_logo.png b/docs/figures/widedeep_logo.png
new file mode 100644
index 0000000000000000000000000000000000000000..2e7fbe6eb765aaf238395a8c79c0bc07d81ae517
GIT binary patch
literal 85253
zcmeFZcUV)~*De~6Do7Cpr3FDjq$<6`Mnt43AVm;RdJPbINhm5C5uzZygd!jyz1M&U
zNSEHb^b&eax$(D6+~@ngd!PI7dCq}{hsDa8bIm#CJKph*F{g$-)>6H6k@+G30Jx;~
z@V*WJKr#jZTzE-AhPzXy=sphsTwJhIR(`CetjzJ)4P<TSXaxW~42y{)*VFA_4qTZz
zc>n(WD~gAd4U_|v9JdZ0e=>MNsm781DxU7jX6M4i!i$`ohRsjuNev7XM}iHDGscRA
zzGtL;2`6Q_;}@hlHtjXzw}L+IPC=rM;fU@8sTBm^y|TBC9k&K4hk6uOIQ5QoE6>eR
z7X;De$E13S7usYSO*$!QX^H;U=!M1|dl89>&z3UpA0lV5`9Dbhk^%y5Q_f!rN`}97
z;k#$Ln^npRcpOUKF#nZx?b_^v_#4WX?^i?L4RW7!UCVhVk|^!Wz_u1rOb_tB4C7J1
z5NH{)YE**~{YJ~n`hq1)kzdK1^eekRy>_7RWv1@$ot^B9O0Rn8{CEMDWM4NwB2F~U
zldxB^pzK1L9Sj~3R5J$ran^f#n7)G3*{GP@_;)XYsiPh;UJL8iu*7a#S(5V^MctEe
zjA{*bJ$Gf~CAQ#B5IR;%wj}D4Fsu;=dw#!}^A=t6NI`09RgJL|oxyDs%a;2t=Fa}S
z#yxHEs|FwFGl#-onPg%8pSVxmTzP9@qg^V?b;rB2O*J=;mow&+>K4(At}8mG!lnbO
zQxu<XlEn3vMsK7ysnhKCyvFIrc0MozMF2Y6@#V*(0alO|znyjUfNA{ow4p9==0^wc
z`xdDRag&<RwfeLjHL}Jg+E*O3RNnwHbJUJREcXHLxlBHa=)cph6xHU|VpuAc{2swj
zT79|kkUBOcg3WEOZ5#RRgExy5MCitHF7NXr_Mf*As@obJ9|g0h1GE1Eh_Y*}z4HVp
z@&eLIB8uKGX|4XS5TfQFzGx26OrxR>x?cR(9+=p<iK;`A;#H9Cd%!Z0JY5h!#|5h9
z%f*+k1ivh%cO;83=lM)@-h9J_L^OzDj=CC97|1e5<4sYk1oWnS)8r>itVBn{sqFrc
zVIbt)4fXFdFO}Kf1FsOP{6+cr-bXSu4$^XxXP2%jXFa}E&XV^U5*(>)lcxTd#~rvH
zM3yG;!{!=kv^m+A3rS5PFT}fpLgsC~1(V5y0{7>Mz2!`Z3|h$NADmG8lQv%PY7v@$
zaYZpVE&g`E$E$B6_LN?yUw)E%`?~%0Jc_50c@Ks&pFQ6CZ2Z}Ih-00BIXpYl$d<31
zMVgK+WI2;5EiQdDov;7AKVlWUYP`yRKzGLS=3RfA;Y*XdCK|Hbm&)jL|K@GqY-Mdp
zZl7vCcrjy9V#qymW#iqYZ`|gCCE+C-#giqM#`GO|Dw$;7wakmU#!7mbT#F3tYQDNq
zxj?g^wdlNX<^tTk<1JUuHUotJ9XDq%-?dA=ld~hS+qcU_a;G_d?hyqS1%=P6ORwZ!
zF@E*<3;5#I=Le#<Y8f)a*1pAjyZBOWS9X_vx8UvDw?S{iAH8~OX!ZKh>&_Q3;dkWj
zz#oc6k#>ZBysz_qB>L#XGqqziP={qH<#>4V!`>T3Jfl&4QSkQ)c>_-ip4Q|M4=?3C
z&5OvBcxv{9;)=z^(0i~bZQdN#ER!5+9ZEfhJlYu5=MM@p*mFFegbuITSK7a@H?rRt
zqWSz>XCr6cmRnRIF;HJIKRqSuaNvo%axHC&wwdOvvVu0!!u)zRwIEl@lWxgxtLD_X
zgF|dX7Q;-r2DxxOq;*C-w}zldbWM=YE1$SipTu6#UeWpTiox}q>Et6Ni`qAFZyskp
z%v4&8Y_rjf&4ydQu@1A2UbYY@kA1EGYUHm1wr499WH}<)b($;MLWvN|vkOj!lYhzR
z_?37@96jB7!S*}_z?R#qpHe0rQmgS?Xxe@H-n4G*1z&eLTDcO_gc-XACVmgb(3p-q
z^Aw9qYYrRa>wE`Z8(yohYw^tAu6?;@E9~v~P}Gton>qG+<#n3tYu8^h3*5~U@io^F
zwq^-`S9Pt51uA@3EKJ16V!(0$l=q@jtVg0iB*5~4_`ayIxxSO<B&$QOc%SgBGo*#M
z(xsfvNYBWm>XkqWe{1ZdfQ*f~b#?dq?$4?HRw|_nhD*<ejSdY(3Y5PX7)cl*6757q
zT&+90j}m4bS)J}V9gWGBXN{AM7nVAe1X$R-Xs^mIt#i_@Jgzcxo^`}dG!H1P7Pri&
za!ubIa*eP^S~TFX;YrYK7z`}?0507ZUl9}!Ve2(*Z(I#_-SsrucKf;%ogeKGxj{dw
zF!g2X-V{ysVc)%!TgeqR`bBj?GJOG6epTdMT;mpTs#Q6SJbh&o`?+!HaZL_Iqu@v>
zo)qx={m#iY<K=+v^AK2mpAKB#>?0@;^Z^tK>=9}!8bF32f}n}e6zt<bi>8IZ5k(Di
z7jrdpCWFwfl@Yhl;N*~4vJ@(AQdLp|GKEVe6dzufzxtZ4l%Dh#utz*qAYu9n+1Kh9
zNnhDRlimnXU~dt#(6c|ey30Qx?xtqxcSX(~G8FXrxPPXfB?H(N-+KGy#7o{1wUUYw
z;DZ0+qs7`qn>L{cL{wS~hnB6@w_9!8+nTJYixM{b$&EMLKYjgF2i9B>O!Tu$9jWj3
z;aSmL-d_f4w%u&M1$>D4O11cO|BY@n@1)L~6M2*XDvnr4DZyNERgnLxytaq7faaDq
zSbOrLILF?Vm~gUI`GrTj8wX7XDQm!cU+*1Ry|Zq!61SW$iz<s$w~eCisBxU$R2fgv
zQMXGk_Y2wx?xA{P*<9M-_E6qnCb>nCIV=C7NUPV|R1?@#-V}_=f{Fe4r^<(=tM;iX
zGZXbIr<hX(!3RRiLe9n+2+O6Y)U4#Jx;UjSo!ANi2@})edXP%fTQpx7pNPJxu~k`~
z`*7W`YrgSC_lx>h4j@cB&!4B;r++DSh?q=FnaxBnL1#-wAX_69vu{aM;=-NYc7-Fp
zEEd(FLbhnHD!ePyem+~$Y}XPQF%}UO;iWFCeiWA%vmAHid~tfnu<Z`t9lt6c%(D6t
z&5}$NUDfSMcY{}Yq9Cz}TGx%ZM^Bbg?jZBtI898BIG-0kFGerZ6tNx3cz7}*QT6Ke
z8#CL1(-Fhm!%JdgiBpL-elL&9vEMNzo<$=w3k_z+YDXSO*c7yIq^9yH@F?h#hf7L^
z^PKrWGfSvHt=OpXFw)sv{0fjP^lGRX>LY4!nyXoLBC-@Vf^|ggS6Uocs6;%rFkO5c
z{W-TzD!pdk`^Ro2V^ypL!Vp`Ee1!?ii(coENajqvBPr%9jXiM5FZbBdm`IqZJawY&
zYlAlg2rihkpU&+w37wcpPWIX<1oWd9!qf}nBF(JLoNE`{vv%_qhb=}~i`X_?)@|#z
z{Toi-bVbDW8l~ho`_9OMyN*X*Pfkr5Y<D*f`baEqHxzy<-1ZChLm!JE5s>%i=0E4(
zs>Z9Tz;a<-a9f33%rI*7jdRX+v|OW{0sMhnz*rYD;4}$UB@{r~_r0$Zz9Cn%RkfjT
zcTT}&u{D_~KqyN{m5d5%a2_zpS7xAqoj+_JPCrR+94#Hq1dTKd`_~~;TIwq2cjkGJ
zWiv4U+h?CoU9mhzIY)i2a5#(~6@0F<JB)JZs+t4A9+W*$&62$is=umu<ARnZ>0FQe
zCj~7)AmjpIAvp}=YmycQ4R$5QhVl6BxVjRjWm5&NwgO(6TrhXIH88-5;r$L(k|gb~
zBO04AmBQTG)AOsNG`p$&Nz6x9`fP+6U_uEnQ-7eUYX3kLF~ZRyPm(4E$|wRdtSbRL
zFHO9Wemw;h_o6^4bNgZtLnnRL-&xH8Fj81bnTu@00T03raH2QX%23T(Qxk9pcTE8x
zCAti_fV(2XeFKP?0i^%A1^^xru@J6xh;ILWh8O?{wF8j+e&!kO_ph(_xF1~W-@h+J
zy#bKp{<@6&d6h<tf0|@0?E?P#!b{v|03}^zH8tFCT?;oWD`&7R$UW-&RypnlrOQJj
zFaW@K>(>vFn$FD~0D#!tPS4QYP*X$N0^}t4!V>h-O3>TM<ySudS#N3FrIVHW3l485
zM`y6Kw;bnxo{+{}|GF*2$?=~@+#TdN4K*KgD1+RrIK%`+1%)}~FLH2j$huitOY7W!
z@VhzgFF8(IcXt<QAt5g>FF`L6L6Dn`&|N7hDIwu|Lig?o;GPfw`#8J5@D^|ebN$l^
zzMuP6U<)@p7k4|5GsmxfU%Ui)xXW>J{(2GN>z{F2dE5PYlQa1Dv~Uv?`t^y>T|r@?
z|EZa~o%R1qvtOV5)9gR<`sa1BzdDnC3bt}n201xdIlIgMvAFDi-t@nG_-CB|G<<C5
zZRKcq-wxLjjGL4E-Fu?KznlHnkN(G|Mt_=0ii`id<-b1pSIb{>kk)py!wvMsFC)s|
zl@<DL_x|hiI#ytiqsOnupE}#Q%isIm<iFhh-AY#I*Sh>`o&EvAf9~Q?Ab(L-h=3^h
zi|f`_)&PJaK<&Pgo;T5E168r(xY6wB^ooZWSWG1GdW7lf>-Uy7Qi+K4sjY6uu|1rh
zdz5A(SxF`H^!=+(FE>ASmFQ}{)QQiF|8_Gys(SP0$LH0zT`p@x5qA>JQ&21jkPxdq
z<T^-BkP$ha0V8WxlDGY5pKTzHN6$Cq!M$5Eb?3?GNs+42XS2^{A*ds$+?h;Wohp|&
zg(85ME)YP%0U-Lnm;Z;@VfF8(>P&R0@xK?mn?AnBdeXZ31(scHEj^1|b@ajFB44Io
zyG-Vqt%JL}EDIZ(rs`lpOTVozR5Wp_?7+OdDnOVSIL#7Kb*x%c`{Jno!nv0vHnM$j
zw_E2Au8PHpNybLrq+U?Q11+day(!qv*waJXkYbZcHIx?fbG3KgEWY7jF05Z5JZ{U+
z#L*g?Whq*Db*=kis{Ow7)fj_BhhVJb0m{~9W7Gy_dOF;}cO)<AYq@g#K*k{T`c{Oz
zO<j!xQFZQ-`b7eeYdO+9)S1q$;r{~og&XEXVrB;N;?nE^!>c10uin(UW6jK!BUKSU
z4gr<E<Pc7~mkaE?_WH4(cspa6dsU~VQoBdAP6swcp;TWY9>C1lB20a@(2MP!Q19}p
z{lVp<8)fH%AV<PW%&kZ|?X9iOEmR4&0K^w=ybrcN-fnPqFF&-gfj90;Si>96zuQXp
z+$}ml8KJw4!aiNSR~pweZZe6oobc{lcHRp+-9LxHtHn!U@tE$D<rpN;2{z-<T#0tm
zxk>1HE}fFj+|Sls@-KpYUtBJ3n$XQOCdZB(L|0*tR!`CG;CMc=CeFA@j`-1)qsD8`
z<FF@fGq$KRRfs2Q^$_u*xzeoMJPGO<bBRC)$w>gBN-$Ro5#bIsOz}*s!cP^M3R!5y
zTAwA<nWYk2yv-eN7e_C~pAIIq8ZzIkTf7Ixyt3_eOvL^y$sg=kkvoIKoVPJ)Df?wD
zbuA`$@m$kI$I(<tE$4p&Pd!byc+7(;95Pq71KM|0Q?Z!w=G~cFh^c<>d+eGsCu7<X
z@ao;^%;|pQv*I4EH-9WXr-#FsR;%(YeAjf>xQVE~Oef3l516GuP_5W+dj)g@y0+;L
ztg=k6?x)*zif`!Y*E(Z&bw{D#b4>{>4p}Rs6y;(_b}PJFa1xmzik0wI#i5FWzSw+%
zYY^73)6m{me}H`2e^;e`q4UXIiSd^;35Yq&Q40ld46@$?V%yrrhzE&FU&If{iKP6d
zeSQlW0h=7cbi|B?lPU7x^E#M3WaAtim2~$mETJRYSK_3_sg&<&gMa;(`kY3e$?#>w
z;kSxo=Z38?P#MQ05Bgx7z}A5jpDH3Qej=a&K-|KKvtd|3`-B7#vN{fZd0K-s%nyDU
zy9>N6iS#r=Z?)55Hnv8mQ5%f5bzY|2<)OdHf6nc)_up9qOC-%t36l;=ryrZ0g`$th
zozd_?a#bYjM1_!FLv+FJe6LLE*>ZnR12!BMA*jv({l{uetvrf>TMM(wg@5dp93Ci_
zTM0gxU%@W#+H9kG&Mu_7GBw^G3Y8MrfE9^BSJO(xCbFkR9iL9Ryz3mX^=n4)NE9`K
zR;b{pQU32oe^pDD;%7+xF0jaUj4*!y3E5xfM+0af@116wSqV95i0t#y1>@MrEkpwO
z$bgMZ&-+I60@3#Z1|vigD2AexD%qnVP{aBMXz!rog|&w!-2bdlno?8T{Xc*L5XU74
zLPsb1fm3IxncCBsM*m!=uBIz0W%gP0Zyi(5-awZhBedA3&r7DgWDmFeC5(GsvHsJQ
z`E8O3{Rhf5o`np5949!Cbd<gBm=BI($_!Z)>SdD6hwG~cm9on2zkcBxTtL$2HZFI?
zq-(~sFx9gKu@KET65T&tP=(lY@B%hqApM-L@yoQwz|rJ+io9z;@f%D{ft1~*{ySHX
zA?Je#a|-Ur8v!xTjEL?8ipRwoJ}Z4)PqzTN+1ms<K7^rSwnpX8j!(XS{;2$$C#$DO
zI``(>%x)Tx6TtcAK~w5879ggXYBu0xZBoHyVWNWg0PqZ~{-NUKyVEfmtj40%iIalw
zlB(A%<|n*@WAeo}RrcT04eC-=?8@x&vNEB^!=LWjZ6c~dCm^bK1@iNx^?Y`L;tF|U
zcw9n1^t*`jj8?Yw%;daTnfqs(d5M|n)16_VfHOY+@1A_6+_d;vE^(%b4K3B)DJ1)2
z){i)|7H(rFa%LoCB=5{ti9e=eaUYK^=$IcC^4(K!_wxger426V;qhX4Bk15cE8i5b
zF8Gh+F=W7vO|BM;(MCZo7RD%Ko<-YE<kb(i*01a4DIgZ$E_%sd^p^TMK)$lmHRt*8
z(}EV6W5kO{F;YBL2wox*NEhrOb-jVdsiOJcB>dG!6LknwC_*Aht0?vO3>`|92=tQe
zliyL0LF9v=dwoubg>_nZsE`w$1IPSvOeT(HcYQ<X2uE&ZX!PjxX%iTjplKt+tCRh=
z!C6P&!aBVul%&f58Y6ervip*5-DTJO2_FWOK`XM;m>t_yS5%Ktu$rsK#|4+`6pm(Z
z!snRr#t3YBOnDOR-~~CQRk)YZ<GC4rAV78{*n*LPWfID6pspka%gLfh;xT2AKZ>{C
zL59pW%tE)JvXFU9??|Qr#;>Ip&u2}yINrw^#mz}qli*FlAuLIJ^mD=ii8|T`9?m_Z
zNsGx!N!0^$4i7rU!uSxZ9_m4rBwAZh+O~XtQ#lj2<KsF<n&57_DUq+C%L8ALXOpR=
zu<-3&*ZwceroZjOlo%K*&veEq>GS6`qQ72aQ3XAWhT58?L5G>#-_Clk8<+dk6g1k!
z!6svlha^xA+`X9*;qkeq-Txug4(u|}=TIWD2!yq)+`}t|CY4)hHRy~~g_TPLS0Gbu
z3U)9sJv-EfdV6{wFeAK4LF@iBhn9Y)V>wUTXkVuuxPrE*pgjHL3RpvAk5|J{c$?qi
zY`0gS0e*~voJ7QqOuL1yz4%3?j>Q4D66wDKM!1(_=hQcVe8Q}l-R=WdjV?dS?^Uck
zd@%Lg#t+M}Cq*A-E&c8_oZc$(sUV${&!1Ex47ZhYu1oNb(gFlgDdGmZh{gq|Oxo0j
za{(T0i3h{=`hA~yV|oTFZ*gF7j>hHusG+dqyVFFX$DZ!`R-)5S8D^*F<Csk@r2p`W
z(fh5m%BT=L*JRMm8Qd4@Oj(e{1N53~P{_)*|ESVclkn!mYu{Wo?(jv3zxbvQy<=|i
z^NNv*Uw3{)PX4Y!E%aar)9#F}CVhhF4|c4Tf&d6Z6$AuzkV>j*(f{$})TYhm5TP0D
zSh#{=&_l$ST{9aD={=)6tCQ!Zw&GZ0glW|t&X>b%{Pfz{#K5J!F*8uTZ8}Ok!!Z4=
z_{{lv+WwLrex!j-a};B`ni6Iw5!w(>U(w0I)(yTEy`v!}&)xo<N^$#H{ZOp+PTlTn
z?M4d_2<hcs^bp?`=Z60G-S?67Hvb}7Ll-BvbFMz@!NFLt!`nckL}EKr8spMa^^$WC
z`)xBZzRDp+?8R`K%h64a&rwGT?i&G~v+(8fv1Mo4-%Qc;fMJd+gC)REQVI`_H(GOZ
zBN#Kt&O$?x3k~~u!s|4{J6)zaw4XcV;|yd_5%*`l@n(a&In}{xTfTE%{@w?x)7ter
zwC<^du<s}zRD6F76}?1_pFm*Kdy4L(W3zJ<a^~!tsNAFBJ^IHX^GZO0Rjr>5U*9<&
z7hD0~__*8mP&{ImZw51DfHa*%zg~ayA?Wwon_E!Y%2kx`d=LAh25wy5fxT%w@#}-E
z?rb+3Yc)VrUVrqpSc*xlyioDvEw*hZ$x(A#9#d$5h=RRM<u+gc%{56%fv~@w+XCkJ
z|Bw%BP|M^oHiLXrb~~#;42jU6Gd{G@?Ya%J%ZG^-ZJzM>Ulr+dq(H3g9-K|%La@+2
z<sVPwX85u9BD(Jb7!OqdaR}!eg86I{1X*t#-yJG6?h>O4>p>snOv-7DXuQ5NG5rRs
zvUL12u6}uclef~w-9`;>tT9<)+2%^0<5wd1s9-)va-`sK4(!clrxT&WXgA;O)7MUp
z2^_EfWWRX0r<*5@61s^n2A)k}t(8VUksw|Zw1Xo%5%+pe9ZxbL_+8+3adFvsErjYV
zLr6VRa1^GYeQv}dpgPGf;A7Si<4Kv&IUMbLVsQH#e^9MAiXFOdfgG>#*)q!>Tp`8B
z)CexcK&M4#N*)q?Jj9W=?xQOCGZZ=dR3kR>J4b8J+Z)Tn+>>rUY+VDK8>-o%wB~}#
zWu}nDf({$F|H@!dCG73l${T`u925S9@)l=`oD29)T_{=E*&!+-)ytx?2U!BkD^Bdp
zi69v>xcW)>qZDiIq@F&=&Ri~<=?IZ4@8E{#Gsnv~hD&mnkYvKcnoPYZ)ih}evYZ7?
z43?7`iN4?v>y@-Ab#3USLpk_;KhN@037ZyZWZAV-E~j3*$fvx0<IG#NJBahJNo?7g
zW!G^JA-Gu6;}Qj<<B&1}Hq%HVw}A~S%_$)gdPHngW4Gb*Rffcp9s27p11=s(%8>f*
z6d9T@LfXp~RcTG7Pu>K1(hIWuW6R=x5{hD{gRS|Wcr0EcCc<u4#L1s7U@t>?bcjT<
z7Jl%n%3v~{q)L{(j5a9ar>!z0fmEf|9JXM}GkcS+;8&%Gqv>b;_2Zy2eC5FWI|+@d
znZHXF#8-3nC46A-e&Whiz~0b>zdG~>UI~pgyhA8aXb4_FjJuzG!-SukMQuw65v)i}
zWbF3ujN?mm6BXQho@slg_I&#kykj+yu~M*XTp726_=2kRs)Fu51^EpdRLpvP!T0ec
z1fX495F*|FiXd=a2L?<bJv>|lFk6Em4kc6fNA#JTPR#Tns<e?a0V};JIZ1ERsPPCm
zqo6Z42Kq(fZvaRhoL3!TCv0W4`tp1p@?*=~N_)N}@7F|3_}F8!#2NKk5H2h<i1WT#
zthzf8zy9K(iYSql<>ae2#D8L`A`TOi=LezkVC>0{Mgb@5bTO)r!^Zx^nm$?fOIEYp
zwAi3s7GUEMZo`rd*4}P(fuLTBwSM`i)*0*XRRjbnF-XcwVi;7}P4=rNKFOvm`@rH(
z)nod!t$P%5t(z;>DaBEt?Vwt4HS{}-joO!Fe&Ih3Zyrr?QX26m%~edZAwD{oIBD&h
z!mO(IP5XoheLrw~=FMGDdYYRnzdSinIi8{2T~*;!rUH)#IZRrI%io<HsqA44c-dIs
zlq7KW&EI%Z-1s2uw6fBva+P7<$!XFN2k@<S8pzLKP{8`+R2u=yzLYy8lfI{5cf6tA
zsLzGT^vwo@C}d}+J<{5rRTyM&Y-@8WF?F(RfmWkvfUE)er1oB+ldr4Y{%O>-{3~;o
zIt~^gID{ez4J|EVS26(fj4~|-X{N4VI>*Z28yjrvtPLdfFs9`8p%(m^?`hpOODPF|
zu~T<EASchgew_5%4#n2Qjj|Sh$dqpNgW{R@D%eZmY{3SENSFT-`kGOoC()y)6q(hN
z-5{}%XoV8GgGg!E0{ZqCr=ql`?R4-{8c`1985?Oy$;3-fVMpSk9i(ZJsxZ)mukS?+
zH|A#|dF!73iskMV>_e_SH}MkxgcCM)R<sObv6UteB>!9AH8E2-7R|@AUj8E#5}W$_
z`|GmXZ*^x@$7jf!9@WbeC+<0yaeSEYZMt3Qm00o)`~F08dzz(T<jT&wu0Z@s2Oa*h
z8v8cuRlN5SF-XcvVkBe@t?!mc2i;rIAszTje&P`eYJMeXq)w>epeXSsrlc$QUp?3g
z4v&Bt30)9TP`?U<VkcJQeKuO_s%Y7z_HX6z4D?Q&St_GeLSlI+Q=G?LAkn)gHZwbw
zIf(g#@H9sJ>jiO8FphH{m%wWnI!U#;ykH0Rs{Uo@;XA8wUHh0kP_{i!T_>0q6PE}_
zGjEL=&v`n?vIG03y)NQi{@_8N(EY7TV&wSf1t1P<ie`V9kI)v-HM(Z9Z%_3;xABo&
z2BY|RNN(=}bnSGy!S6Wi>`gR&!P7vvwBwAscb7nkqCw}!u$d@gYexG0w0QCf9hdG4
z)|f2&bbdekl_8y^<TEeOEk9cr*5#CKCaIz2*4IA}|HUA*UXcWysz^Ci=e1LG+a;FR
z8lwZb>u!BRWVk1)fn@gGQ<`rPs=9qLGNh9`q42ep?dfyqlvVTpcVFjo`Ge6mjMuvC
zX!3g|W&d8r=4!r)X#th;wkj~VSiig{3pu8Z#2hHrU5Wdx<i{w8QQw;%Op$33<ISH=
zS6zjj%O6mc8YDfpu#M>9a>6`$GPxPPAs{nrJtJ-wm$G>k`|hP7YQ}1!7<h&4Tse{u
z<qtG)5cehPohKMfkccYq-BxC*_IZx<;k3iypuy>>DAk9tXv0&=FHd5Q-7s;%HYY88
zVvpT3hTe{D0$K3O?!eK+;OG$8djqdWuJH#ITrnA3$|3(T5N`DGj&Wx*zXa{)PXfca
zK|1+iN9Ka2yuu;SkilDXP%H-BzLmGVDj$t^wjV%TEJl|sKg8oh(Y&2rkEoQ47WTSg
z7^lxArgZ<Nz;eg1g094Mbt9WY#`Cw<6(CUnM!W;payO~oujB5<;71gwsA<j<pcJd(
zBRdxzEPn&o#V(-Bf_D2Hw<|#rq}ffIGTT7j4$8}VK!Z20{4aFF%cU6c(+uHuIX|*B
zAiAL)p!@i9r(ZV7`=K^XSDAT+<E2Zx@I~fvA57_99TYjYpEUaDI=&wW1+E{3WGO-^
zjqVy7>n2i~jCSV}hqn3UuDeA?>AaX7hvB~Y-2=@hJU{V+UWMU8;bD6s7hb>$yAXRt
zvTwG{C19g$-+lb%Htf6!%Um+M=ClyL)gTzVD^yPQ4e2c%V64mak<d#ht_li~1Yf}$
zheJ4#oo)M_?3`|fJv)iI%DKi$cd)G~!$w2=1R4C`R2!v21~C#<Qy^Osprj@Q_XDj!
zfHCsB6bIfO%D0N4NLxR@e1B!t$S+Gx4=+ZYFxbYHC>l?E6!PHBUOqL1za3%6zH(ee
z@lTxS5jRuen>OG|KqE44G=${jF{Ps>w=plpN5H3*(RH4YM@>B;Ru<<%ug7)f_*I5=
z=mf^qg$3lsW0~_C^$dft=BJvQYTrwoZatD2N^SG_U7A1RX@apH^d)oQp(Om6m{FPU
zN%3V9KH#B7f7!R2lVJDzPW4}>Wo<t?<jp;MDp;qvblK$2X}K|q{F^p&msctw>pi=H
zgEiV^`8ENEP4dkW?Aj^HVdbY)g+8|YuX_f<W3C35nep$`LpepWw&NhJ0O*OD#G_lB
zq|cu7|M508oVT5Q>M0?3n`jUhlv<!w@!+EoWues7D&w0yi;lNO^apC4oTFKC#O;$b
zalRJQL?9C8U!-)worV&xb6hBY0umAHc9~a<)R$27k4c9<TI)-tWdm3-JI0^j^G5=O
ztp8vTCp~a7Ipsh~9B=;+PV*Ocv1DiKamxF&!OK4qLUw9%uP5cMZ&GdU>)P{Hq}(z<
z-OAvWhzb4HS6IRLs_8x6q(7u^-kHL+3?NvAgJEoDJb{f3xXh7|#Ge)`uFcOM#mN=h
zm#a`<Ed}B-_FWg493q2q0TkA3?3G$?Mtq_&#T#NZ0*B6-$6yXTI^SqDrAqF%4=xLT
z4DHu>EMO4hOB*dZv8)}l03`Za^j4@=HZ{WWJAO0fzmWxXa?|04%penTOq=anPX_0y
zU3pZVvwS4?M#mNCNzEM%@*}DWgqIG7Az=t100p-n<W`YH1fln?sn=1wM>{=*L6y3)
z6=meb5b%;6TSotP_e+f9%qhJ{#num~pTl2)n#uSiOpalWvNAOYEJ>J%y1Ac?=a-z}
zJ)6HaV8>x^+sy`Yl+_b~eWk&7>eKgP?`&Al>gm6QF6#u@DdtIvB|bfJZy_GLbTm6T
ze)h12`ZB&JUv3-_<u0ta;7v{)*ThvPvBgBwxlGSGc?HptRckV?UZ*!3RWuR9_W6q_
zX>2(drNXJoS)k`yOM*J`<OUWBz5iX`1rP^Z#<?GlQzR)KOKFtdh3|9%dqSiyZi?!X
ze{^J|cF5NaDATeU_A}QM&oW&}inDUa&TGfs<g!_u%-*Z3tY~3yJZ0Sd9V1s$06_XJ
zHYswvYB&E(5{aCWN<P!IWK7=$QdGdNR6wP^#h4X1Dul0j=t_7`9^9j{Ld}mB!X4@i
z6qeq;AjEPgPAQKlfFlX)fQ!G?u8SNhecbGZ`+%f|XQEEa^kOmU&)??zq+nMLXKrT2
zHqUn465q)`1h4e2`i#;Of)n}|)%?3NT!!fVP9vz|3hSeE&axQ5Gn&uW(=MCfs)c;n
zXWs-WJ_kG5ndJ$|yPBQIjJ??LTL|l)b}y;NZ!`apA(H2u_&0Y2QnJpBO|mt(W0ffL
zrNU66#v0S>+P1^uLc?42+@pSlm0dZ725guQx;-m<J1nqvU^!G5!AU44;aCgDGa@U0
z6P6cO6;_M{)|7&UUbpXsJQbsq8`n2lr%NOcMPbXchSv4;{rEQQDjmO?*}iE)PW1M|
zMn}~NfvfRX;F|p=#ZK^cZWjxi6`oC*VT>jWHVi!He>k8j`#L8wEvwQwwqs(R7V#Nw
zU^0Dvoi~(_8Df6*g6)>##P@Q6ySS<jtqVED7|BE1_%UEV{*CM9*2bXG@>~JTKJ}zw
z#kim}f<>YbRcXdVu>ZylT;4A?C3T0;&#huHW;ALHLRvl{Yy6h?d*+>SJx}uSJ1Z~4
zJuBigak5+tv60zd_Jr|IB#iq7uQ&-+JVpS-z<|xp?bRAT@-@4Pi=m`nWf>Vm9FNp~
z)Rb1m6xipp%_$f{j2Aonw$9O!H%ST7{=XYLK9KaUZ3WM*&Y~P--`;Jb6N7*p{gS86
zm1Aw=M%S@Eo>!y4^28{cCbu5k+x6R<sK7Q05jGiv4u7Tg@<*(Ma36HJx<u?$2C~M!
zsYV2UpRGEichr=)_b6JxV2)N{w248Y2$j6?>W?I+#}(8zEc!m+<Mp-MK?MZ`@R>x7
z2daatoJm>j`KVAT*%fixv>u%fdLKdK2K`$gu~{0d(q5A3^$7WSX@cvR2atFNI|Xb?
z-TPy0b09;*;rhlX#3211w)*A8^l_#yyg$X&ead{Pdl23|lb@ieeb^oCp?gd`5{eKB
zPcKaRS_f1wmM}yy99>jCIALYROPcx1-l($}M4_Tl#+;JK?4zA8ojLD~^NN{h5q4cf
zSwl0LCpN+t7LW7tN3lUy{?vQVD0b-k4@*|Q6ObO?v~7E!YaH!*c=rjm+J**x>OfhU
zmAvWf)aN%>P}l-}h+;?7qWLatKQNL0XHMo8I6*bpHg!>p1V5e|^u@5lX`i0Me1+J;
z@zmN!&0?_pR-XPZ@$#JO5nlwMCPP$9M&hmWJHVR=7QZ?LYia54Z|4Ykl*g|L{;#RI
z_tMb_G_ld3=l0JzBQ{(0*&=;n=cg<ng9-gDxt%vyj?)GV&IArssRT1!;z&dIIXC6H
z5E2_jb3+pAvhkl1snGe?=*2azSZ(dxC7ACr%wPHvZMm3Jsh?#}s`;ZIJ3v;%nt{S;
zr(7LYRSiHOn_~T6MFV{l>+eVo2)e;gJoo*;W3&)^6-wNrK)fz=>UFnE*N$CDUA!pY
zsU|||6YLEmEh^%N%ZXgT;ri^2>eyP+-y$w3f&=G_T+SDG@SzU3gCYyDT7loGj=lb~
zDPaXYE`~+We6AVhvcnWPp7mCCF3Z=jLv388P8-(SDu7WWgc7r%0!|2QAjp%62w3Ab
z*S^tkGLdfptv*YwxI)*doxe2qzIm5t*b<bjpR)dJ^CWz6X~8SN<{UlRzNk%8J?XOc
z4;F&{dwuW+g{&)j-AsMdQK{5E{It{XUY>5=nPH^QFyGUZf$Zt79_flOj~?+7*8P4a
zY@$>26aIGpkXe=@Gj#8pFamEmaO39OBN5nwmka<RKjt;vqN(X~H96Pi+4>`4><(1Y
z`DFn8$(EB-gA3SZ1%1Gb55D5zO-I%#G&cAOBY@<M+T10HB5WPpZ+kA&*R309zSpEZ
z71a|Fk&`qeULoXBI&a4CP(x4>Az@ax2ResY3>l~p*je*evXtb)BuS2c1DW>Rx;Kg(
zhTO)e2mRe#DwU)Uy7hk9Be%dNvGQ@@#60r0R;hiN(>26a$H}&Z-_a*70=u%PDyn~p
zkh!Zx@!J=034KJ9G0unUA;oED)fwbI0}6g6Sh@IYpzOmXofYxfojz*L{I0Lag3%%@
zsh{tb<g6WjcZzQ8=C9qmHD?z=f`=wH#qb%(qYh?<p$l|*IRlR%*0B?@8TL<&#A@VK
zC)O){%ud3i1{=Ilg^rr3RXnWFslGox(etb7yT46oL#X-&Qq~VcZ@9_!HjAjoXcLva
z>}$SP=G_2FPwBLWL9Kb&Q%&zfKnMuBZ|C5d4!*V+6mjzxazRu)1kLi8(vWUR<(%^z
zx!W`*`a1aCed@Frf12c;2fE=UK0m%sa!-yq-hx4W?6-pV{T#iHLnJDzpq`%tf;Zm$
zQ(813AztcBWt!KMQpK+YhcGp9*|_6MfY4O!p}BZ%YKzUYQlHxe0#N2S75V;1>`USg
zgBhro$fr|?qJYg^3FEoL%pA7Y()bQW)PI$Zy5N}vv;{Gc@W;EI6tcklb@Lu-o5gUa
zj%L><$ZqPkPUoFPH6yIKtiudm)XbygvFjB;&T8%__*7UtPSG&SX<N<B8owNh=C&lb
z;{PuB@iFv3gqEh!%67E((3*I(vXZ88??V#oS>Tqu242~YIsT%L@#D}keB3loBjNvi
z&#%400qX4F;3oW-Bm}W3RO`E?O=rSVPYw9zBn#84rXx}*ZLNRp|3?Or8XSSx*fAYu
z4Swfl-#}P5x;2aR*!4w<lBsS#7L<cNQ_!q&ktw!>Z*mfO2(}`;av!H8CYNWqdg<_d
zO~l|m1d-eGFepTQ=8BI&QmGS(!V<^5-uldJ+3_8MeX3%7QQ$P!!PDt@$LRp{_IA~|
z?z^SlD7*+Op8r(>Ke@DUfS>9$aU$B%dJp7!mD%`&uee;uQgi8@_tv{i0j^LrU15Rc
z0#~+IBC05fMEGo<kk8NM(Xjx8gyRzU@HhX2s{{fjTWigd1qgI@)4Vx0L}+^Av=l}1
zj3(aa<89L3O|hKQwJ(l+;;J!Vsl4S=qumy$$Z5cM1PjywB#Vh4>P%M0!^@VqapEQ3
z47lSY)c_J_HnxEMb7bQ|UtZ40f>fVuw{=pg?SAP>%<@F^l5IP*05wzZb!anF?$rt%
zGtZm+i%^7W#c}u6%PF4TKefc0#K1-bb{6upx!KF-h5~H4ONzFz)hea)k%jSbHwy+M
zJ8Qk;=kIqsXy$o%#)7s}nFbP+kRVM^_2oKAo1O%GY6l1kx{NzSXN>RxVVM*H4(7VF
zmD7s*?VPmoTNFYCv@P79P5YcYpG|z%xIEF5BT#*mFnFZKhPPLedLYt#a>}nI0bj2G
z{K7s37*&`*Is*p#tyu8zg|}wNU^XYFl%3djcMB4%9UzxHPRRAOc#74E^|W5^;q;Ep
zQkC3wKbI{&%0iX*cnQb^3ydXVLUv18;605)*qr$2<|LN_0(!o9KDelQ_|$=3wS3KC
zTy{O0SRpHY;&v;7n=Zk<71ZU4O5G?rY2#bm)gDph#-9^uQo)hR7UzsuFJ8S7QCuUU
zDs5O_IRYtwp`P{8oHn#&rTc#;o%6~G8J+UBVl_xmb+e7jXAuZ*i+VJy_opHken8@h
zIFUNNI@!|kXa21uD2vy~#-r~tRqd6Z$~Y3*tINgq@A&0GXFRjhSTPNI=Xv=s)YD)G
z9X{TF3^}*J@FQw{!Hm)Xf2<65M(9yfs-<67-zMT9J#*YWLorckPZrMTcSjQb{{MGD
zzkMHi(Ak58SUPMV&$rC+z?^fVd+1Qu<;rV9ieE{(Yl1=Oam)sEZKfYNouqlZ{v+|V
zpX@^BRWk?q`%{<YW7H?sEiUoZO@udep71=p%7eIcl;SXYV)*$Z9<bFKB>qePJ}>4%
zNii1fsEyu4$H}{aq#H~X^0JpwIprRqgTWB%Tyki=$!ILVsC3k6X_0Yh7wQASH1L~w
zZC!<0;~cBxRU~0+PVg#GLABtpitF7!wD_3>cg|`|Xc}=o+1C@9s?}dgl0lPXC1`pi
zAV_M#(v<`6?=HCVP?tAuSAh?~7u|6nXTa`ryzPzZ5>qB@1DbFXyFT?uC=7SNZ%p8}
zWp9})RW{JF&R|!73eqQM7r%ZX>6xP2e;N|?B)MRmJJy07>qC4A0@JR92OQ5F3LOAH
z#e*JF;>oV-K8{$dYLiR|)i1i+xWS5_e(MiGq9a=Tu&%z>3P>RSyDQ`CYvLj|jqk;h
zk6yWXXKSbLt{S-qI|MW~fyU0%fDau6sy+Ljl;Yv#;zN1D9E8`OxLkfN>aw?;=RwY_
z7j_#4)TgiTPd}9zmF?B%bg(WAR^?SFT+etOzth*Xs)A~Kd3-htu`fl#ja;1WSQED5
zO-ykTKoHrm)QP9oKt)a$kobAMnamXSbR`mI3I{$9zag>|QyU)px9>+yCCQ2>d6iCE
z^@TkU{mIq?#Jw5m6AM2H%;$jFiF5Qt!eO^-WJFk{f1lV6QrGw;eRhi`YL1qLCK}F;
zQSDR%x(2*G+Qp|J^D`{(JH}1tk4`gvU((qy39~Cs$cE#MV`o_<m#xcQ+=9QSUK|*R
zJ^k=6JD78juy<RVVfJaa>yNh`eLXNyhnC~@6_ASscA(hM4TKo8Ho_M<Q6(!qaW=cf
zmCnmO{z(~60zqAxxD{J!M3@k$jZRXj2?Cg3SB24HF(9_-a~d;hab=4;e1~k`>OF5K
z4GeDfh`Yjjq+?>*Dy1Kkw|a)-ZiG(=lnU)Z7e}_W8y(I&tBlOID$fb4C#-=p^c$bq
z8hbo}a0u5bBn4oKSf|lpHaYDeb;&C#_Ua8m8LoGM2GLFsC;7N0?1bgsropz!>9XpB
zN#tU#7$W_5l`-fA!{6!|FQVtx@tI^`6M%AT&@p-i216{N2hqTiK$h9Q^LxzkV_tm$
zYd<^9e0xTE6-t{XcDqD#2-#f_ZktK`Jxh<(pSUncVVJhn$&;fxi_UL9pEXd4IJloE
zrB9ndnsj6^TytY2)PI<JOfV5US$TxX7mD9R3*b(#4%gs?3=T@TW0jZK@bMeRY<eW9
z^KtpRa%aa-^vTRI+tMsu!TQI3;sFb*o`Q>?#Q6R?%{aw=C}=?XZCE1x1d5Mzs;LR5
zC>7H<(+)MQIR2<|+$(6y1AKiCo+HE?uwJy*gUtc%?p^X1&$RrWZD~eN1B;uO70eam
z@H^fA!<k!gBC2vHgs`-4ABauvq<Q6mD)N9<u^Y#_pPu==>UsGjGhr!4`tFikg#2Bk
ze6D8*swkogewxug;?-d9fW%fdO}vm-CaCGarZtj*sFefmA%Z1w@;oJwbfWF}LFTBx
z@4=h`YHl|K=_8!Pc(XD29gy*<ksV8oA=CbhzE#%r`T3#`C}k7l1xnpSH_g}{oHH7R
zGYxqX6Fe%giIvE#=HEN+igVp0{9~0KHln`gB7-5w6ZH5z)Ws5bgDtKpo>SMfSG#s;
zDSPh`+PssmC)S_&`T2RY2k4p;_FGa_A%;mpHPdX~u0muM!i*pDkH2trf2u1Q1g-@T
z+i=Y_SBCVj3cy*}0yb20$Y&219M9Z+(*z3d9iBefx)&fm#QaLU5_oQCch~!);LuaJ
z!t(*_y0;rrGjj{*KWvG_Q7Lo4k4T#w!Vx?GNh}XtoM0C+LhfHMp-v1W?JR5%W)5gB
z8zhIW)3&$Io=$D99)jT<n(mKCdJHmrfA(yFmq?v@=ds_tH?|#AC$SrY(FM>-i3v;v
zu8?GNdSXNToxr|)zw*IW#qmGRjoTb*B6hCR_Z@(6%x_iYWC}qcZYYmFO&IoMtG^^*
z+Gs3MyWn+RT$d@E<^ve45^@~Y@e}KfI|+~7j);V=wAmhC->*|w+*(?FLOAYu=U2g=
zROJYtYU6Sa7eQi1?@95{{3GNt&U{zBR8@<TBio8(+1bqC5tW5(k=di!gS!<4VOWl}
zpGTIWb!ApWWexo!Y+zI}TDZB);6Wx+4|EnISFD%_@o0Wa*kJgV#s2T*KiS>?Zg%*R
z+~mFFXBql0Kch3`3R=VNlb88>Yik*Ax1Gh0c2te)t=N5m`Bd1I<<elgnm~IVAZbnW
zU2CBj)xoc-@O58}4fVm=%)2Tp89}{uL#IDCX2!;=8w!laRB%ZZ5NtIzU}3G!f_Rht
zSWiE%a{H;72xdoF;Q-hM^G!aWFI|2zxo^1U@eZ!lEU`?B`-b(TC{$v`Sk{~Gc#YGT
zZqZ4<rHvbYR{k!<B45KZ|Ly6z#!jdieG4aA@7Wq(2zUR_E?c+py9GkLa9>vU$X=PD
z)7%N;!qP#&WVy72$NODdaS5p<G7IA$dPjH6q>Rb1Q_aLVdF`jZ!ejZeD0|{na&$HG
zUkd41z3Gr$uGQe(vhLOxSsla+lnc8O&pTj@Vl10N9-Bu@$(n*w<<Y@Xj)yB$Gy9`H
z=&zpg+sK05P<V%VI>dx4z$r>7@qTo6&Fe(>6n>HH!>kpRcBPdBYIOZ1r12qCeM06^
zanBd<XXx#*5mt(!V{psB#O3kbKCf$#gbQb`H0xK7-UwOna-V5`Z9aLj_Sd5qsPAmA
zcUz<$UVR6A2d^>v5`ccHWihN~5^^S1yXVmtb9^_wG^T#k$D%67pVzO^-gxhXocn`O
zH|;C9ne`}3Vz%~?6t_pgipsg_%4$7riqOF9oy`=ov2w*Qo<v5TXB8jId${3xIl)=K
z4&k^po85GqXv^#27CH$dK_x3D=}$Ojyn)>3%yIfM;kXwJf7sMVS#For!)?E+`sdO6
zUZ`bQ@tKYXZ9s+T#naq6lO2N?v}Q^bZ_hx7o)3yocws4UF77b;+rS`IB7fRibjndj
z>d*Qe;Z@cY9%8nm{nrb|N-8--;$~ky!647>$fxQDRlW3z&KQ0+=biL3+8Tsn2!BTd
zA>+DKufnw&ifK7{()L~W@hAR88)AlHVF)vITc2z~t)RaJ$`E<-0r{6|(Hec(C-1t4
zOb95S#kgMY>=RB0=aqu`8<;SoU@lY6tb3^YpWs(CGt>^H4wZPdt%g<4?^z@s4cffz
z?XPj?f36O_lP^EfmkzG+_@GrgmV1=IcsIKKm8Q>yg<R;88~?_ldQC60Wc5(5=}wVR
z6D|4^L<#!>C5$^I8n`QR>4TNycay0o(XXfvbKfvuQC%n34~w^VxhTu?<H}>p^939_
zMelTv*Z0zFSC((;B~n~fK+VT}d-2x?#+T0y24mhNou>>I8MJ|S_YX>M*(S=TSUL#t
ziakH^WJg=V4(n4_WaApNP|vT?7Db157b$yqp-7A~Oubg^Id}N>R0_Gt;Fs(7{myqr
zTV?Oe6$Nqc1Ov~~vBHqm2--!See!*tjnME~$llc#Oi8eCpR3Pc?+;!5yLSXO9ysJp
znLhgpgcN=#TWz28K&0E9SwaI!{p5tCiKSIn(5wC*h~uifn9T>x9_;J1#tFriVbNjo
zu<JdW;OkO{rKB1InUL{i__$;Zsu}mMYJM$O^GOG)8P#z%fA0%C)~u~@irsPAHVLsT
z9ENc1s5CjM-6O9v?{m6R-42B0Z19AAsbUh{`rQ9>EwOviSZ*Z6OJhT`mx}gPZ@*6!
z+CTk3|8c)vpp(9QVB!TUkEU+bR!f-=_8i50UE}UyCo*IwR&16|X>f6x10a%j;LS>a
z1<K_j&WC`77bdm3>()iy0V6|qxg5S3os68GB+w3yJIF&1A$C(;Qb=<Bw1VEz#aoz!
zw}`h!{)^lCCu(L|ndg@@r7iB}XxlLNW+vV3IYvZYN|Et!Ppuf$R6y#C;+!hGO?kRu
z`^-7FPP44hInG#YW1oH3>q&WueOXV{YlX8+Z1-_ktx!L7qHQygc2Q`N2lu^_*RDNl
z#$@cPFW@m=AN8gEm{l<!ZdqosHnC?1$@tnw3+<uZ>3a6Q&~4&14iq2d2ZhSg)XqNH
zapZ#z14?rn<hxc)z)@}S+uQr^R~iMyrq^JsCt-b?y-Q2BrpYH}wf4q^>W8_{>2(gm
zd}e`|oqW+CNImigl$v-kbS-dq@FQJY`vdvItINP;vjVkv1@RY{9r?*|Yvl}RRxtL_
zmm;=+zcm~lvI$Q#q)bR&OVZWxL(MaJL_LkVdSx?8ZzUJVpqbH;)==11-do{6LG(ik
zT@Q2J7QG~BG`hy%D1s%GfKz@v3vTUdjA;!oveA*xTD$dh^w7FR0UYuRl%j3qZM1ED
z<pbqDuTHsCNa{oN_W6DRv~+u7=unxxzs|3sOFPL70eU2ml&u@1Tfv+Yd#aM1mqLBg
z0y5QB5}UTZr6g)}_BrqekHG{_-P7GJ<nyYlDVbm%4mQIh&ei%!fH6r_lOgCy=&tlE
z3IF)UBdEIqR;*-f$IEah|5nt`5Xg{%j!maVl+LOGP8plzwi%34E|TwSi{1aJkwq&s
z62iAHv}jYbXrmn4aMhus59szGpz-PEy?wEhzI{WzXW#5TG}ORVwiw6H_dx79eR~-W
zW|pAT$qZa_cGAaSi2FQX#t+>7aoMM^uKw>8PQQJBrV3-rc4%r7r&jpp{!GXZA8n$_
zJxisFQp{EeqTv>@toNO<$KsW%+Tk+eL)WnUPfpT-XWG}(uH%OK!?5GGPc@HfFf)il
z&_gCZMU5t!akMoKs4jOZMCi?8UkTAL?tsJd;>pr8pRvW%QPBAQH#nqJLqV#@)Z*Ov
zf@4qqfLRxsdU^dzn%i)j(~Q_x{ow@7EnP9pRoGj4)DK!T&pUQ;>sLx#mV@RQZo_Xd
z%Gh-54=8bvy`ykaTe#kqg^Ha)#S4_=`3e2CBFTLA&8qhOM~g2KE3L;TJF~A~6BVP?
z%49_$VhPS}w$k93-j5F^G>ywPovd`<LN(JO)Sgdj)1=t+)-pZfLT48k9aZ8ErXjfJ
zG-{`$X^Phl!0Qg$a<8((_n19yR`pahD}aM%)trVbe`s5~M$w~l^wq$Qxeh;Q&*+Jk
z;gYH%kfBVH#diPM&mH_SkUXSERPC!R5|7A7gC)!!(wKkBiwu$x{dOL4ui=H6W##FU
zH-zQ3n6>i<<s=Jw>^x4wo0xUnd#9{YbkVwYIxm%3bXI;fi^k>+YF%*Tb+@<KWYYKk
zsZo8gNVg#g_kU6_xw+~#<V@tO<FvrRe4^qnc|#i!DfJX<juK@T0EeGQ3g8e<+Xj<$
zUVVzr%->aRU%RQv^Ze*bqfnAgQ6b~!2>-H(su<1O$(^E_`;3|o;u7Fy#=yz<@$1h*
z^Kw1Rpg><!ck1VLqvz;j+z|=q@QJ@qt#L;$c{Fc(Xu4N>Fq<bm+Kkm7&3w0xTBw&F
zvoP(KcxoW?74hbx!a*j@zz<?{&O0II`xkN~W53<yS)M0Hh$^JRH3BiwQpFg<=`-bk
z{QMzg4CqPSc$C@XX2f|Cb7jT;TX7i-PffXNtp+OUp|7vf;PnHm(U03@W?Ry2AO91=
zYa!d<2RGS?N7}5|q{CJ`=wU*sd|EQJ#Sw$>kU?$6Pn2G=lwMy;O&1Y{v1G7vosQE4
zMAsod#<Ocau8ls9<SaK7a7MdG{yQp>CsDcGUtB1FuTvI;m@McV>8F)A<Nq%Jmc7Q*
zotKZo1A3PQ(;Jv9d<q3HlzptxWJN<9JIM}yYA0C^n~S$x?njI5bbZdPaoAIjIBtBB
z>QI*<|8(%VvY@HMcQ4lz@VTRlra|v6A5nM7vk$$?M>S-a;no7lL0n9+ZZ4q~%#oM$
z_JQzubQtE3U2vFTgaUQv<J?0as|x!ep)uDK*6+sb>h76i%rTZ|m%Ag{>t)YB9gf+G
zye<Ob{!zj7&!y5a4ZDLuAKVFOTtHJ(xcsQyZpeD*MR%r}Xo6_MtM~1dpIOvI?YWov
z(4Xg-2fQ11r(QNn_kA>4klwAL5z&A=f8;l?)Eh8cXpA*wnU#fC?#UM$X-SMRiw5lU
zjV0D6TAd6hXIQ^vM_(s8Gm9cZ^Bgm+De*2MQOT_*A3O#&g?jFt<w~jaeDk+&K1mYR
zJL?#$^^2%imS>yoIs!e@YKykj=DF&h(Pklo+TUW4Mrfpm8i*!v7zF%yq`B44Q$~|y
z5_QOW*5PL~xFOXG9^d!|jR%{>b#9+fR-O$#uW#gypo-CZA|15}b_xMv!bZBt)aIDV
z-$3#psyK?cN$fhmK-M<PPwiT)Gi>lzV`;O{uKnI;_bc%}$`!hKo;{_TxKrOrTS-=^
z6#bby;`(}~xsqaCrRVH<wf_%k?;X|D)_w~g6+}fvMXEy15e1RnC6p)#NS7`h=^g3P
z5)}od7?CCfh|(bxX+dg~DmC<8lolZL5D6h6cgOR-_x}F4;~V3SZ{!a$c6Qi{wf9=j
zeCC|b!~1ei)EEZzC03U@X_y%nda~u6NnU&-=d&=ML|NdW7rZJWO*ATQ&>wJg)<JGG
z>IaFf7`=NAAUD?4uSjI}w>%V><x8GgOy<f=&S9E4noDt8V@+uM<U^4B0z(w6d0gNl
z0NDm$oRF4hY3ebL#z^Zy9dL?zgMAAL*jGt)W4luEW;{(Ka#dZ|m`;=RaK7J6v}Ye+
z>Qc)LNKV~aI0hU_u2*zrWj^nj6IrK;X^`!Z;s~jD+)Rnl<n--%o!#=Y(vyYZT%=L6
zx_YYu(B*hg{KrED?jsz;?WbkEgrGt{=%1R#se-7_8(x4fw1WYvDZSdMV4!lJrn#uj
zDaHvoum1Btz=={<gl9r_k5%QJ4S1`%h+$Cl=K*7uS8KvpG6l!y%VMi6QkR3r7KNw1
zfgK)Jryw2xb;+TJ-{Y@sU1tg`1N@CGYkN}a<1I+b(s;*RYLRtSJRp##u)e_H?>_+c
zd5uMe#b~k`FJGJ5*o+G&B!g)&dgMFlpDS^obq*Xo<gD+uWBay~wx&W@;vtljL2L*y
zDCIXW38Fb&PDFq9__FN2O59rr)iDj~r+fiGh=}x*TP_e0EL@${a%>6rg;Ha_otjl@
ze!4G|@+%rxP9auy6xVm~IAkxV7da>N7_wCHxBpY#wC<Hs5M>5DJA_-vjFRm!LyNyO
zmG!{EuA{eaY4&C-ZJvaY(j<ot#u%s%Z6b1AD|a|N$&LaMhR?>6v;70gwwbiKHwux#
zx!Zb>8|yel!o?40ikUadyWIBJ>5_h}5beX6u?R<CajN~T$qGPJqxfr2GJeQ*uxtzc
zJD89$JL=Q>V=RY3rN^VVGq#^R%5E1;8-gShv&b5SPDaXV<2%vwT*Ru-UC{|XAn$V-
zS*moHI?4FkK^@4om@<O}SM-o}J+ae`U%PrZZ8!3qbS-vrnYHnleS6t*qvqVXIWYIJ
z{V7bzwY4_w?$`vxfd0Y*Ace4_RF{3KcmX_s%-o=1@h(zY+Cx6mE1(W6ZwK}PCNXtn
zT6gY{#ovi&K5Hpdz}N_y;?DDaB>iYCo8%(BBX#R0k39V5RgS(!=!d)`4r#Zz;ikK-
zN}u%`w4}a7a}C$05?c5^5yGcA!y2}5%(uP<0I3(+EmT=A(Lidnu4ilcqt6&h&#j6V
zMv|$yEb|$hw<^H;M$x{xOCwMVLA3E6Q$wsLEg;W6;FXwP$^qx~g^J1HvvmWtcCidM
z;2(YV3^IqDn^F;hgZbb*fDTL0=#@Jx1b*XOPhY*OQKlh9>ctPf6-2mkLACkzva@w{
zk)9g@<B8JgLG-w%u6bFS#C-p-9$z^3K7Rz!;opIY!nvk<HVUuUk9xBHep*Cn4Gc_Y
zv*me^p?zp5F<R3xQtp2{J!ZVLc@#(F&@H_+ScCs*bMYFOMZE1!zKt%V=%37dWc}zJ
ze*mX*rw&8se0yDeeCye0jal-ULkkw9@7p>&fiGSGFycOh8QjU$yAL8`2ad2@^<8_q
z43i30{seF>N;DrRW~=|o0vHMNV+#{R_gp27yd3nAR=UUym0H}a0FWQ)eL_K@HcDAd
z5w$1KZLPK=_ZQwWhP8`K6z6-%mY)xzFqeK~fEI}F7JoZO&1kvv^iHYYm5rT9+Z&gp
z4S?KwCHeb1KsxxpoP%GH3${UtE^RyP127&wwS*c&Y&VLQ;z#(8NcA$~byLPTQ;Gu(
zHTh;x{IJxbF7_bxaBODqW<(`I!mZ-;QQ>{uVVS#C9x`|k+hFVmjIT076P}PNhN==1
zxjk-V2<w%a?0#8(C#)*UtL!3;q0)ZHE?H=E^lnl7kTDa4#Pvdpb=9S;+|hSJh*~i8
zFpn*|m6UMHo*H%7^GF=p8_#Bby$)G8e_J<j9~t)}aQK(0W6|r7{4BUU`skOn85&H?
zh!!j1IX-)dQpult+04@j!c+VVIu2|i>6eaH6A3N&I(m{P2Hw^ipFKUX&*Rw~llIJ~
zqK_j@K(^{uRZ|<f$htL(b~f}ZFC|V}RcIA=la<Uu#W9?~%f*2Nj<gm=IE_{z0wb*y
zRhqds#Wi6hMu+#(F3jXh?*W2+Qwwk$uTf%WPYIYHX3oQy9BBm?pqfVJeD@)Y`r~i$
zleEun(vu;`4U67_4N&GVHjE3WH~6#C1Q<;~5ZZ3HZt_Wumy=Q=YHIEq;IT7eBD>+0
z&;ynICKaD)8L!%aTo+((;FsfhTYxup{f9aKbSAS?w`-D{d(=7vhs^TlI%-`HPDo7x
zXmQL;1pJoVb!YvUO?SB9h<mu+jIhA>$v*%(7&zzkyf4SyN!vdW!!T`?d5+nTa^XB3
zk>lFVBy>cvj)j~8P!rQ)f!p(42dLfc*@agXPwXhSKY<7IPZ8Z?gPKPvkNT~h^43}J
z2zgB*mg&jKF1FS*JSi)=$9!63%4hr-#kM6FraHXu)|Im;p&M<kO{YIz1>dJ%U8H5#
zy!_sW!;z6`_k3esno&ZZNle<c<mmb9*+$Ivry=`%A_FkJO-cV`dvze>KV;rV)`|ML
z-|vl@g6X{`yifMhsSz3#OjgRT5M61C@7Bse?g8(1G1n;p$RnMcclNxx))#fxBC6(?
z<!acrLwd{CkLBRCWqyt5kG7+MWeBOx@*@vK$AS|4&}4>&UM&4F2F%L+BrnhT@3KPJ
z1)LaDsmA%_<Zgk}OFPjZa#uwDq`tAK7H#m8Q39s~RQ%72^k(3?fQhL^Ja>t$`&FK@
z`Qpfc<hRUw9}gDSc*lDN!`TOU0n0T#wmUBG6SxC83G}Eo<XElI*G6$8uW3vYG~VLa
z2?d!2wS>OTeV}U11~eUFZdt8?{CI$_$P7pP29EUwk;WNrjQ<c`G3zQ<-s|>PVwjLj
zu7+yb`X#`QZpKO_>ah2$$m;r$6u3xfq66^z@7dR(Dzu6@`ZHr5_*SvJjg6ho3lKD{
zv`sQ^VQ+g}hfsCE)6!R0$s70FxPCDSqepTR4%|Uf>0YyQ)xWkMtq;nUZ$@`wYX*B#
z7+f^&^in^%R=)RH6({~;xII2xpznU`mz(PyHYXw1F#UjV?O}4C-d~xi4UazFd=TGQ
zdk-g2vtVUGTx8?%;$fN?!oHa*njU7&i;dGrOe#0RNKQ6p*7k2U>COSPE3duB>_e%t
zOUr{S)ac#F-NeII9P^6)7z>93H3?f8JKpGiw+@#NJacRDv2{f=GpfU7mB4N<vCDZ5
zvl19&z{_o9Ef(Rf6j9840qKOS-|j=}HgszZc7+D`863(HYr6e_ShO~gpuGj1lK;z3
zvTxJxQrFbi*rvjbtdLen?ANXnP;;Z#r~YtR;?Vrs&R$5ml74?fy2eMyN60$Sl!h<2
z_#Cmc$6c&!ShXSJB^x45ufF>wP`aNiD>ESsK=8@@vujo6<HD%f`r(xjnI3?KtaVeV
z_ZpMwajz||2B_l_MR@A`f~y2lU#b0Gx5<)#Xyea1;Y-0p%y$<LXuSA{=(y`VDVCYO
z9O)A=?83GJ2Y*emGbVWQw^Gu^*-JoU70@3sOaHZfo;YK5@g(`Ynt)j%dnwu;F*QfE
zwHURW+&nfLdJH1WRhz>hlmY#*zo@t+(l=5<O+trehyDt@gVIgu0^lE^U^dNL4$mwg
zJDC(3sjl27sM`>fi`GJH_P&F?VB@lOhSsB%hdztPk82{=VLEH7Jq0{`bFi93AgR0-
z(lK{Z$nS+M|KI#t?Rk;a!vYgl`=N_?9Iu*K!%(q}viGCr=ilZ(Pq2UeCPsKFY(JQv
zSUe}SgK%LxN}Z6t`yIZEDzz=bnvm~7N(ol&1%u^3UG8zhtk?(+mDfg6Zy=+nhO3?B
zMW{Y5?%e$cu!p0XJf%{`O~|M{p#2L?krj6Wb2DB#Koc@K)G7oJ(@_;VizXZql}s{X
z$_zl*R9X$M_$#%1P5N6(mWuZC4Zhp5LT0D#n|}oyrrlfPH$J9_=*E1bukp{cB{>D|
z>i~5B`r{7;k~Yp+0+N1U^M;tnVr}g;c5BGq&<*^KJ=OHRb<ue3a!%>XYf~L$bU3hF
zk_h~wTC-CraCH2#Sy>VOfpAj(;^(z<4PS2)V}LWfO7goS;&H`|LwegVwHg}GNz|=i
zyw1D-I%Ib4S$VSA)F^e^a1vN4Ix@dinbtbb2_u?Wi_dh4x%Y2Hjf`D;rt9{GrYJzF
zGaQP7dag<+^&YVONpErC$TQSeyGE@zO5=j*%DiOC5RAM5N&Z{m2Wdeu;E-Z+tKD?m
zt-sgHX}oiDx7<g^@rGV@e&x`FY1kXtng-$H`i`>UbW7zSC+A`h6Yg@8j2zuf(b)U|
zC9TDp-@a6E2iVZM>K#vcQ<=ktXhM=S_N}%fBSz<M$;$<bdT$2ShXK9^ZKV`)qB2?f
z01h;O6innD^-Y{gSx@>A1+!<y2q)%V*iV$;WZLH42RS^FPWbkKpU#@*Ur5zh7b=U-
z1~@`GQ#T|L$_Z)02i70LpBZN=@(99@D?95I|9%h)X9l>g7f6&!{YRJKq9jx%ej@oF
z!A5GvK()RrA^0p*_eShdKxxIX|4c#Eya4&oj~Y%~-;(|&TO)VpFa^hY5KiRhA6*J;
z#31D>()wsIF)-()@SkQzm*S-bKOtFXJKI(MU}Q(i$TeoVV$_Gwl6!KjC?8TfCP_P;
z$9vf$Y}hoFs<R$sQ;c_~zuyJ1Loo9Zo;ANIt~GOi4$bCxM?)y?UL*B|US;nCy@-b-
zt`VH^Gy+|}!5gSR`DNyj1N1<xp9Wv}k|%d4FTACtkJ(sRn{ggbZI(6}FB`cgvrbW%
z-9hFAzck>C)EoIuPbXI){tVm>!iyT5eB7R#kW<&*7JffNWl+B$yDISIEZG2^7_jAL
z3{iRfKEFPN(M?8k>OwWm-(Yj6{L{Dc#*noKb+N%Q`D=-8ch^01EQH(jJO<>!n4jQn
z>2E85N#$gF&g(2)%kEcdF>+tFp$68o=(_*}*;fT-q_E?MA|Cu+@|`5q)bk@mdR!8=
zBEyJ>*Xwg8=vu*+fsR2QP0cz?ny|r|jJyzr8Z+F7jree$Cqf&Q$lsBeC%!Q|+8<W8
z+ml@^*Rd(tYp)`en-(NQjI0-h!1_CDI1`#)no0jGPjSQWUzX%puclXmsRrQQ8MU?9
zs`0$Fsi6!ri?GqUP!5s{J6+*JDyl3se<NM)WB}cg%naSwzrN_Qa@qVdvLyCLn;`Hb
znWRPQlFl~mABNrN8C@Zim#1#;3f_NH(PXZ9U*p;Vr9V#=I?-*jH#c`&)bx_`PJ<jk
zUhgk*uRj3*i)Q@&Ot_}fYRKR}z#MI>;$rt}&xO^VkcFti8~aPUdhDfdeeUn!P0&uT
zu${;~O=_jzpq9r4zW-bht&!z~a6;06fME_Hsoa4xy1l7>v3sKzXj5+MHn#s>Z2oRi
z5g%mnA7N9()EUFG9U+Ukw`y8JQ1i3`yi4A?*E|=ng$X$A%kbCqeDN3hvwt;e^8{bB
zBQVB`87dMnr$d`5Y!%)5FLgtF%F+K`olYDLx-i7Uxus_F6&GFlq7Ng1mqQhf1=IkZ
zRea`r<Nd|rMo{&Hl1SgW7;_NQ{7(7ZJYv|BmB&9*1DCId&JH-{DM_0jw3-}r*~rWW
zi35tK%=`n9uck2(Yjq8}!KjMB>vjZ9jYnE@^E<9aHbv7$%JEnJ+5UT1Us(Ukm8z2_
z87q%l{(V+u%Os&mFC^m-0w2X`Hy@5#wE!B;;4g~7!4Gf0vc+}XXFranj3aD`Yv;3P
z0jgb}{Kn$$XQcPibp7}<UJ2MjM&mp=Dj(nMefLJyN95uX{`qHu|7MAA>;eBuoqn%%
z8PmM9EKcR8eN<uEM>ts%I@=fkj4tsvqpqHz&A7p2@6#LQ_xr9b+7ixM>4^*3B`Q@%
z+76igJc+4S$g;b9_cLhUXpL_*yHWx=zF75`K@Z5v^heEJirzs;OZ_}-unHA;`+j<%
zUZw1H2%=9vd!*nu2{%0iNSLQFy{H~Dfc@5n;JtM4hS=EO@jzYy0^8<d*d_oW_VCax
zVPS11zkjjg!@&dFCVr{okR~RG*@KAz{R@%DiI3^fQ4-e6!rY7OMe>SGtrAs5CX;dL
z<C)}0p1eL@ma=6v%LZv|TZhO|E94aS9ZZy>sm1j>M&yyuqY*8&@Tknsgph~*k##Oa
z{{zM4(G*}W(d~J2cibTT+}Y(~>x!ZhY3oqed11579sN63S@Lvk`us*OLU*Jj9<pd5
zLfukOL4Y{@3HXz_|6cdamA8@Zi32fC40Hh9&t~}<i+OT~(weLXImYDR*0lx~j&GP%
zy_mm2eJC?aXQo!>CNz*6rac9lJefI!*dhYJT@!~H0+}>tf5b|#BY3;Yqu<Q%1ML`m
z+vo3Hb>2al-`xGG4seQd_(p>-eKkbz^or+aui{~5N@klP%F%7&rx|gyO<QbLpvbAL
z{4+99XBAlHUzGjit@0010f(Yj3R1%KS5gPk+E4Kvs^%OPW-EvWdYhED&WC|0xo6Ef
zZcrU%0*_+yN96{W%%$r#K8}^9b8OScr@S)Pe$J}dxsoEvzWrU{NAH~Cp<_~%3XCtg
z+w_U~q<vGkF#wRVl?U`gEbjsP-OlG)iDpUYihT4TAGVx+j}Xc*viR8A3ckNGZ+Fn;
zojZ@kiV+HwS|KfC(!{Z&|B%86?f+yAVarvPbgj&e<LB_88wXxCmYoNe{{W__fB+CS
z2tCCh??);EzBSJZkSlSRe8#xiM3H4{dV#FtbLhI<nW#v*{v4lN#soX{F96{pp9Iv@
zDyUZEV+}D`*?x*yhp;Y2HeY(OMZ%~iwgW9aTdyzrlLb??D&STC=aAM#Zn^2;Rr{IY
zFaCWfZ+}}B0!O^;(-#vVi->5aCUvhcb573YhieTloQ74zcsfSfmU8VIRehpO*tL+J
zT;hbtm`XzV<;7`H%HJvGST1}PqzbGMs_XwE{x2uJhu^QTRNso+Fb%SFPqfi$x>7^%
zbBo1%zm(eI_=6Is?qj(i{5!~4I*n^UK==Gm`#G|s&r6%9e+Tex6*#Toc=id5Q<g94
zrUfPpafKYdYYLMk+ee(MFS@EjnyU%E|DZLuPnRU5cYx_MUff@QFt8L$)E^d)BmDcP
z>Z{g&mw?WlM@m%}?lo5JdFfIGK#d49+Fe^{v)Pe8Xa}`)89!oLbgIkn%HQ+Ro@d8r
znwJZzy<;(}<2s@s44J#3Z<IXbJK%VIZ*7?PjIZ=|>5=VhK00$F(QpDhM9v5d0QNq7
z-t3*Dx~u4Q0JI~-X0E>kYpwIHkKZzBvi->9lK5OW2VHEIZmM47t8M_~+jQ`l>-(Gz
z*qyIjW08j3b~^q+$iHiB^$~~$<I*s8wV?Fe-2oZVUj^YdK4cH{wdd8#Bc*}NwrLA2
z5VxDUVs1?)cmd5^K%WLUAE?gye`V=Zu<h+eHsgS6c0I=3w%XJS`bzR^GpOxaan+Or
z!-(LhWFtnp^JQI5YYjsD3Lt!w1bK1jO7nMM(_N!x^9o1@@uOM=@mD(FI{62?d~@%U
zaa6cS^Rk_J|B)rys{i*M6Ia!;od4eLSoh*~#2IK=o*H?#URhRJ6B(3Al;9_R1`H?n
z0P-;`Q_K)BdH{^YP@-#4`mNCq{8`VAlM|;T$4}E5KfSPB5HE<o!L^^GdHYnF!vi37
z|0+xmh*qb6i-}EGT@7)dI8x%$KMp_CCq!3IXj)jmx6R`S7b>#0o&Cq&+6_-^zIBge
zHUq?@!D6J<=o+f$iZ1(^iuv>+V%C|DdHZX^eg10#4g1H%(<yEJ5&aZqrhw*_x&{R&
zx@|zQB=0ROp{)L`Xcyji8?ELw<72>cEHkba_cI#G?iLA1jDuerAM(+mXKXb+HLmC-
z{lYhI$tkF2<}#zFVnUnrf|^3!wgkCQq-1=xZjo+1d3Y_kjQj&o#vKRE%_1sUxG)<M
zbYSoZCx*gA`wX}}FD36;DL{y@JZQI*yPnH@wSdDjgC@iB7EDAR2iMupo?3q>>3vX>
znCm-cZn>ml8^J#W3)5m}F`(y3q4Zh>o})lP1FFJnk4*~$JUOIN4_;D@jwlBB`Z&zx
z=eDJ3H{Wd84g}0HttFcKaOW7hB?HC@c`-%0abhSi6|gS+Wk$Cyh)~<7IYgQt2_jC#
z^0-*6yElkUZYW&j4g#3bbR*H$@-D_d>I5eQ*ZJD2o0)wD8TzY(B2mst`vI6<Xp048
zT(^;}&y*guY+dmAm+Pe$IlQ{z0xKSjVvAyD(li*$9NT(hyIGQL6Z^V*0RM<v_PW20
zMO_rOQWaP?VFQQexuh>%o@WPtrGVcwk+MgR$88%Fw{0cyC+?04LTb|Y8LQg`R(8ea
zIZpkHFzqOg$F~VL2`)C;U!Y%~;{`yS;5oyA&hbxE=^@F<77Ema7V90L!A0NO#`UC&
z$;jBef(yFbtDkCFLq-K_R`%B{GF-;=H!D+S(y)m~R<oc0o6pY<>l(c4_pEC5zv!-u
zUHXkY`2Eh1c0JVQ%!=|aUuCf)=NGjz;x&Ef4FInXzz;ZS8noS?3X9S3w10W}5>UAC
z*Ee_p$o4z5;(gN2`&z}oF(!{OlzLDyK(MjiML}~BqK0m)y3X}3O#%z~VJ9(!UDn!W
z!&$Md;lr)OuWQQRw*#OHBlo8Oxn!+5A8th=cWl)!&#ut+IS#D)eJ(cY?p1utz&Z-T
zVfCB;ajz-bNCNPHbhYUC8zt?C5!nB~LbaGP%4`=97nhUC6-womL(lXE^BWBTvqa6M
zz+oq4UZapaATZan=Q8CP!08{Y37~Dwnm04jWdSR<f4uB+@;cZADAL#cThhR0&FQC^
z?OL~grZPZveV2goJ@PVvDnB<J^?2}5w7q<<!qkM9#`*je^2|di8`I|8u3SZcycWH@
z_PzZVLYkHES$C8dKo6<f15w(isW;DYMUoBejCZpE^}exz!OL;EWM4+&V%W&w%+?`#
z1ko8}GseD2Y86}8;9di(4HI6iC8lYN*!z3#u_qYa<lOXvR(K&p0+jWSs*iGyR(LC{
zXH=_E5mG$aFvsfkaDYu-ne>a6N)T07Azp>3EM4EePye_b1T_#?E#SXYCw&y(O$Trz
znx$DGqC?tJY7j!kPczmx_3rEH%dv$>@;QZ`v-$*G1n7}$-e5j!3&cMru1#GwPjy*<
zH-bKQp~zUO-V~kEaG2|H8HCqb9t5Qj;>Ott&@Ixiz7?yk^;g*`PjA~%fF~>s|I6g0
zzQCrL<L_=~E*hV6{UbT@3L`ePQh@Euzxoq5tLIZYKyh(AF2rfCU|U`iBVYXJDb%%r
z<RxybwwF&ky;nB0*@NERebm%#;laLUY<(71U@Fq}E7jMS{2m%pH~b)7Es@KA1Lb`g
zG|Ng3=)1WmkAg}rAKM0ku%THqxaBRvqr<?W4Iuem?v)wHv*)^yD6+C>!rq`mihY#_
zxb>p>-T7E44^BddPE*T2zyABO#D}-Xd{-_w$3f2aW+>0JP?Eo{I=#~$(dk816zs%G
z7u%^zsyj=7ltei(&(<@SGBqxLuKsv@dyx?}ELLVB3wdj&AKErwbjIO0hn=z#_~8t3
z|7E_A#85%iO7X5xoON;^g5k)#vpQ>}kwiD)9f%)@vj6mx0U}1De_NJSCMq<3$hko+
zX!9OXYZr-Ye8WtUJh5vK10BGcSKk8yMq9;#aQa)XKY9t#YIsa;M8h;`>L0_ubffRM
zISoH)w3gcT@<L6eJpUk9;#sFzf0M#|z5*a^G(QtY4o{1$i<mkMpIudcW`6hrm^R1D
zt*Ub{O*fGuq!bt;VQs1tU=-lfRMME&Y~ppQvpWSbqHM#Pjkz1}K8V|n0P*kJ@P3y(
zH9NeWd6-#}gIBw-b~4#a>3By7tO8)tec9Vs_BfE!NuqhhRFVs&mrU2{?T#Pfzr<%k
zP~TAq9i^KqR8&b$yI-#y-sJztP=d#4_m(abbd`&Zrb-U{szj#%jfxqzS%o(qAJ7T3
znrymG=nEYc^cq_P@PIu}QNW~5plD@|Q2=+%z)19Zk}TfD#&-%Z5~PcEM7pCx-uLw!
z8;VM57~=g*%(6?nu-w2G|77fTl#-P*8VM_B0S=>p?b-^qy2FlYm$`JK1WA!rn9?%)
zIDMpv5PMi<vBJ&nVP0lxLNe1PwJ<<q_l@2<zfMae_h=x;1rHntm0s9Z(X#X*LP>i?
z4GYR5!N{M%rqF`!r$yZW=!Zt4+A(@k{U=k9h||UebO{?!&+P(jG6UH0K&!S+*;{n#
z5gvScdc-VA-O~NiR1z4FQ~*&F98HcN8q(16(&GjM9eFSh8wFl%p4Z>d%$U|ZzNS3k
zk16t+?a)69BQ-Tb!B@6l&Sq;$AMT~^v-iZgF}IE9Z~@rfJ8&-VBam_ddVCWp@pL!x
zD`z>OTk*<Bk(@A~=0`(IAUDs<Nwlc{A$Ceu&h>E>MT^67B_a<K5B(EySaC$)htX?E
zjRf)qOJ3<2i)p{<`i&aW(Y<1k%7iBLg-qn_z|3q-H>~D3IY&QMyU%e_d9o%&e;NWF
zr#`@06A8onwVtz%lROzKNcO;dxMtT<+xapsTjDfL{%Z>D<>OR;rQ`k+(ACbcH1fkO
zxj**g#w1zkEJ<i6?cZmo@h?DupR1o5Iw=*;uy!W2D(zfRS%{sFhkvY(4IC5s@Gg*F
zl*PB?m_XQyw}1?znJGmzQLLcLsk3@3H+(&7P#!$~cv>3*F^Oj}ncQUcY*yYK-jh=(
zf`_FGJ6DfHdZzHlHsu=o#A!8jucTEY9sj3$@=4RTe~kT|?%+ZyD?_3V6N#Ot%P>}D
zv)<@_9iD#<-ZpOS^O4EJa^<!4(#WXBR8}$lCZ1!x6VUwV>etolW_iT~2Hhb6Yeycq
zW+tArd=YR<gmi*o@@GX2<^fHGLo_|I?Y4;h9#`1tT@ia>z6LmljlrZyxyb0KLK|S?
z><qoX!9h1R0jxC33RY6oeb>(Ix<Kg<@)OlDmS60pV3cizSfYxJX@}h#50zMB#^D*q
zkb}TzipDSL!UZ4Bb;t(=f#_vV&p&1pf*Bn^d+DrO^~JIPV>whC4*^UP#E@~^jQhol
zPPI7rv3W#Am>B`>mi1f36}htWsDnNXg|k4mk;5y+1@U0-1*I_($}fCC2fYckqkfSG
z?ocauy82`F$Xr?O&-bugVOUA;tzRua=FOy$awswJ9c=AsXnxW#aX2jsp`f1VI&^k_
zKmaj>uZioI+T#7+7D`r6!O+xkF~K(}qHTa}QJVM<b{exD|C}vnR*0e-FvacW8I8Go
zC;SEn-I7OlFS#fwqvI2MK{B8fcuNb;Xbd!4hB@!+f|zZf+)STu`=Gz@yAd?{(!&{>
z>`VxUR*-+P5=aW-BpG62;r`AIo*Yebb@jmL$C1bk4qlb<C_sm+9(dc&ap&n;46uqj
z+GG*0wV1l3D$Sd2=<3}vF)s?Au|71$>7^7+Ew2|_P4-C4^1w(rm?0Osg)t5}8g7su
zUEE7CSaDjxUI~0c9q1%?9X-*oOyBi1$Wr!R;!#>QvNubCyj=7eni^T(yYvaZvb6OR
zWb_@Pgki)C?++(LAsW<%MODd;Vh)Bz;YumMGrSGR$AxQvt-@W^j{8hR+6PESa;n&I
zx@YrTMg8?>JzkYcVgW#RI_1cC=$;*d%mmynzWCDrqBTnC4@Mxm9+ot46eOPXNLgwe
zk`7%uHGS>3J3(&S9VI<g;x^;5*NSP8>kNHEaleweoP9h{{Ul#MNP#~ve`z>4uHSn4
ziLwd~HGa5u#i|&+_!q*hHLDdnpX_AaPJq_`Lv)ItR^+j~dMj^jk_v`nzWbUpdM}*$
z@2=|z1cH+B^UAjFBHzhRs>WHML5Rvcxd82q^T*oXq{d_Wed<u#9Y@(x%aqjN_DV{M
zLMUvm$420|XBPG6L!W~!5?^*m0>WN3DBN~uW&D}YOnMK{1N;Ra&Epf>OnSJm(Q;zD
z_7otQjr}WCP5eizFfZqN3>guY^`qw2T>noEF1=Wr#4vO;SSH%4YD+c)6;hn!)%qzv
zZk=(F_et#iy>bt#wHs{O{9VlL#@W1;U-Cfw(5$56mg?Pm{?P@!AY~PD2PGjO;dD%T
zH0~p0ObT}Qs0>Xin7Cs*8@7Fc2w=@}x4uQ&`yPwK&IpGP%|nw%;Qs2prd*xCQY~D7
z<LmBzqFFKOBZIORT0VuDWTIfLnorE&$S9b)li46EWE@06DVR%~4WIJ#7Zwx5oa>H}
z0)VM0rl`v$b1NQ0vaW{xvZ+~dXxctOI|=Nh(Z|DJXXC|J9n5k`@w=_}?lSywKmsnv
z3vo`59~t09^Zb9m>tNx@NgU~)M0$@G71R`X-XboV+UBe*I(!j=!?M6XQ)@m*vjq;l
zea5j-eVqev>x0Vb*AtmX#*ti^Qo%&=;NV-Uz~WFQAt#1hpSFayJQ}+zu(8(nOA~*V
z`8eeM<hhksySXQzJ2|wh4URa7h>nF0J*|ubF@x1(>)g0QX>NPZR{cUE^Z!oE_T;^;
zDZ4o_9iluH`y7R?7fdmgrAVD1X`Y=eaIN2h?RPZ}th(x~+w>M(KLG@`A&ksCnNred
zmghBDr*X9Qx0kXbdPsF5a5^9Ue5}Z`qB*Txx$FhZf;A?spj$3YO@Xq1x^^t{nVMeg
z-cBTPe@KVJjely8h3p0Z;KD?;ko`z#ZmM(rL#aV%!hT}PTkB@W=U0sNoCu4e-S`Y+
zkkRsPwC%`C>EXa)hYsTqfZybByI*e{p_A;huAz*77^o*<(j1aCSmvKBlHa|k4|!|x
z-r4kH5KtT-_4V`7npyH9e3!i5IKC`9nNR(R3~`h#i?sS})eErWj`5BW1F#W&zx2u8
z11>V0bGbrZs#|uSPgcIqb3lJXN&-u1r6e9b+!AxS25E#;{I@druNSd;pBDeJWYxb|
zEcn%QynBm%IeDy<eY_prEd*G30nO2bwWnLi<eI({V4rzxYEv`?$QEuYMb(%89H#%(
zXY+>Cg@wwE>5wvkKS}FaMM<M!&3(x1nt-H@*W7k?%P;wf;AZL~3&v^RU%4i;R0c3z
z0kuEo)C}Fco?46Fin|3!;bHw0vmMtFx95HOPt%xlBLSD+Qmst?uf}Tto}pWpU9a0w
z?t)JE8iguvTLhDEzNAKq>FC_yQU1{<KlGS?TXyBtBJ4x5)bRCFkjtyeYKdtQ5qpj_
ze@WC<;;c|T^~|cG|9zZ*%%-F1XF||&X$JuI`Y=pwa?v~D2ue~`Flf&xhIV-74PuB8
zGJe2ABYf!DY}Z{2Ik*AIBO&3>zX~cL<;Wazc{>XKj>BhPz+3BAu=2@;{QdB`9{hG=
zr-#?JfX}wx%G{@`2vb}rYbvGG+dV3vZ;cT~%2b7v-mt2uSmn^gLDR`TPgIa)ohYrq
z4(B3?(kTNh*B|e4L#=PwsSto}Asi>j{bN{{2GnzzHOL~?72}B#<+5I~qx42x@Q<nL
zQ=Y}DkX3<ozLmTL;B#R9-;&>ixaIccgMk~9t!!_@4OD-5v+lEEorobAGRbUiU&t}O
zHO{LdePfLD0PszW`Z!nW0c9)Vfff&jqYlm=yi)ecNZ}j?v+Z&Py#s*8IF)i<GJKUk
z^YjG2;7*>uwzb;p6zQ?f#&q=mtS|$hX+&G2^pLX=50J_lfF#7?t0#@4p#%CO8ok<l
zz@Py|fR%WeF!T_i8;JOXd?t$ODL{$(v>DCWwp{aH#!Z_?R+#V6OpZo6Y<J3i9ywYr
zGj*5e54nl1eX;k7R!Q9N%8tI-SQvlDCk^nzflc<-6f|%h1@*9$0;HMZ2frJLJ1}S_
zvaDs<9e}LK9yVL&cmLC3<4=v>H`U8KqlHbX^26EEO*K{Mb^#t1;GUR3SZ)nOKUGO|
z-787A3epKX6H?a8_%HDufA&kr!GZVwmuqwakYze-Xh)Y8_AFfB_-nwNwFH#@%B&q9
z*<6EgG|)zZD^VF&7m9qAbuC3-`BkGz<yv4_=1WWA#-?|iv&gax&_k`GFrJgcD>9+x
zmQ|we*y+NO5igR$>&DF2$f+_ZrJfMk>8D7%!k8oLQDm*5dC_!K8@9X_Q5yXh!n@4&
zUN6@6{rpsD>?Zp%ek`jMviG=6c*-G+^ch(g4eUaP698`jyj4qo`};p{|8vwTfZ99X
zH=0}-+JWB7nB<m7^|L`42;kVud%z#yss5G}DJD{#iezjM^{aFJT(17s*@$|$M5%|8
z*c1dD9O2RUWFw-kLJqq$cc(n69~}IrAPo@HD{t=^FBMqw4eT3_%ix>hV0|TB1zq{(
zK&mu6a)#<-!@BS*qFpXoiFg4j!-~6nC_O2SZ<_Ip1e^uCPfeSPHi~?9P-%DJKj_~A
zEQQx7xU@IXReQQl`Vn}9t@8RETf()gm`aMe+W1wD`MNQWjZo&37{A(|<+VFI+T`02
z*}4$a$fny7a0?~jy|V{h!vPb4JOFPOu}W9=0Nu(6tDPKvxm(W2+8$wz{wg+^v@qW<
zCm9>!Yjh4ms#)WkvbU)>NM9R`P@0<F`_o?c5=&k}NH5Apkn(dpAMP3(vZn5quR&u2
z4x9fu3}n@hxYL0Dost|gzdn5ZYS-t=#^BgbeT;*>?#pSknNJZo3$3On?m2Q_NMVH%
zJ}LaNMAs9TY@|vdo<SGqdrJ5BfItH~L6BbIH?uq#2!W`*)HVY|z~Pg{y+1~8ZvJ+S
zb;9spxK6?;TsN<~wHFhZkUAB@J^h2%%NOQe9&`r<_<(EYnRc%TB>$3|F9)KL0rE`|
znVBkpS5<6#Xa$HoAh#R&L$LwC*7&1Ylf3J{?y3zj{uFT}Gj00PO&)3E<$fnn3nkm0
zmHrEvmX>lyZ>MMI4Omd8UaL(RbQhF2gkoz#9pwiB=j>Vy2H2ZaR9vB4`Je@<8=;{B
z5bxsjSWgF7z&%jw1w3~efKWo{Rl7MgyOJp|*2NBlW3)l$R0nu+MBUAk)$2MeO9XHa
z=3UMMiV)Y0E#7)MqAZuKJafP`@qv|qQ*nZSY8A6TUwC1^m1M%c#LaKWEt7rD?bcXV
zzO0r5!qtHS_yRBYh#Jmn&nh&Su^!=kjKgQwYDK+1&yA;zA{vIuEW#F8007LV6quIX
zHRG#_%+n9Pn;%%TJWQ7n>+}b#4+z83U#+b5PspB#@hkF@`9?!F$vuiQF+m4mXK*u2
zb=$Oe3XmK-R~duhw<?HzoJAFLjLNS6bCr#o$ljpCn6QTM+14O=qHY+1g`#7JpgT`Z
zxTeWpLUoE(#vkuHND8rrnFeWjr2@he`mp4DZ4XeiQ5@$L(Wnl=8<gv{`xr0}ro#7N
zjemVI5@U8HI{hV2gdN61VuYqUHFU?zV`X8x1|X;gxoDk>R_DqFq<>6sx5d#$7(+mR
zlfYJ2WM^FjW0qK2^|#*>1HZ2mRw?T5$n(!@E}z3(xIO>$`}VajvMRr7Pb^;(B}JSX
zQ@>4-{S@BHi1GP0b>{W_=_G@)lMSM|T`9@c=r8Q}gH4sTZE}LG*VDKmul9H{8zbc%
zk*`Z<a-h0FFk)&k`Lr0d;h7F18`l&GI2>)FXDX=S`Y;YswCESA05>OoAaA?jpvPSZ
zRRuIk>cMg5H5u6En(FPKy8l$vzyqxwmD^FQvcw~fZtHMD>R=_*S4TMN7E;@VC-Jsi
zH!Vep=E3Z|`*ZT$O$_@`fLMHAtoyA+f%9yJAgKOYL6_^-`-y;Yz(A^_`j@@(-M@gv
z9T^Dy)I6-F$}us8Wzgo6>FPrG5$_)iZgc8irHboNHT3@Qz47dD>dW1nD*$KH{qy*V
zlHzaHEzk^PE1b4-7djVub3aS*#$B-n(QoB*Q>w~R*~i^K1WAWXocxkw*=wCRW%80s
zQ~YWo5-vFw34}waH_s?mqxjj*Nz3^FzRJLmeg!W)r}>embqYam8XQI5sp9u6jn4?4
z8j}Dd;K!`!G4(&E00Ft=tfdDZE)yMi9(Z8JUX|7)(7&?MPww(+RJMjGTFyTSpJ_-P
zloUD|d$<mBHe8!v!I+5n8@din;X|h3{AQBQm7&cA$eOv0k%Os_--<=$E+sSmbFnny
zI&bqg(_2RqPY<h?nTsek8>ZKZziRVhT_S){D58=}?*aor2ke%x00F|KrjqS|gP7yb
zUG?@8Qq(K(X5waq{Px`=&&|~Tz9~4KUns!KJE#t$yjP9zk7ez35T9{P4#552I`$GO
zwJIVgZDt;@*xDJfzIO7M-h<$WkF(-uV{()!|MNNk1i+?<W)Xq#Pbe`(FKiWqYVbpS
zn4E{0^NMKuWy=+>->5Gn#-55Y)QuzK1oF=G2>g}glPb|c$4Fh|bvb_B0m9rr10lCm
zA;<yj!GmZ05T_SjP1m+%#)5^Can*~p;i7oFXS?@dA?C`DaKEX`mddGJD|;D1y(w6y
z5g?D#(6i(NjsYV`Bqj`x{QwRFlj0b$R4r4v?9WA(FGu$7k*kl%%X>ZFdbSqJpRilP
z^b_9U_V$P${fgMG*(SLW3w@OB88H!5V`VAj!JFwBQu%$42*-4x3RFsnAZChjHGa!y
z8ir~&FW1RHJtt23*O#Ldi5z!!P!QiTNRapA0jw&>ylsJj+Ie#<@K;G%hRqG5lThaK
z9iR?SAV<f1%eTdoi=Ex}T!Mxz8Qhfv`gfY|axpvEhz~bA^lCB1*WO&;&K!I>O{4f9
zxu=s~Gf`qZ(?u`t`&R@62S)?t^d-u5LM>+*0ah40aQIM=#m25jSIoW$!fTb;<Nt2U
zB5;}(rU*<3&(o6RkK<~msUOc$T|%zPesJajBt%Zg{<lg`?GL-dzzVgMfTkax@ko%n
zmcQQAc<s@}!ho7l55^xH-%AZdc%sNJPeR^48P#X)W^l01?s~3^^aM9H>+4-2d<X8-
zYXB-;Y$YGQV9;m3U}Hz<E2ycYR)1M}(6ijGUP)dMczb!tZ=LA`Nv7}3X+YO~7EVuy
zN8W#<<|y08M#>(*)1{E<4iu|HacE-@#g%;B*$?u=jb|XDeXRL<ZEBDwT5o%_3)<<7
z4|q4NtsrYzaIo%(ai|IecW&nKAWDbB0xRb$Cc&ADzlYxSjjc6-C$}d&Gy8qR+&Z06
z`*E8MUfqjLQySEoZ5MX?#foqL-@n#`Y+~tcsfSW(R!09`{@PRxTq(3%?>GIz{xddH
zz?GWJ1diwzmiHFh1g<oPN~F>+{C^p&Y0!-9vlpe(igOko0$(yv;8&7U=~6a*KF!hK
zcHHr>3M!}F5F;-9{O5S}%|w~RVKz2kiX@K3p(%r(JQu!^x~QsP*NB%#8ck~I2e);i
zAh5N2d~(R=?<(?%Wm&L~h~b)B9#&<E!*zOkMzjQf=ak_zb3K|Vv}U$Sxt5qFzq0S+
zw0^94q-J-N?gDFGkq4Uf*31b(YO(T2OFi1Y;^3q#uB0@hzt^8Ew+AZP`poP}Gw2DN
zox1L<%Ml$B6Y}hnSK!jPIyHH;<s@{qfOvs;nT!Sh#`&swrYI*4)0Lv(NNE*%&XzII
zL8@@Sujm|y1oy9&W<=N5p@~WRfdlE|3_?R9-tlkmTz)|9ws!)mX^{CzA^C15OiSs{
zR@D>D6J0v1Lx9cx3K%zc>z`Dww!3BriYxxBClPjSXv3mcOEY_uVt9iVWn7cddU;%L
zTglJbN(t6|qoa%(6`Hw>J^$D_s*95YKQJCR`R&oU#m*^>-nNL>yAHrXjm2pD(_Uat
zPrW1$wywS1L1|ARudvH~*n%=sp=2c>`d%4HfvUJJLIxZiJTL8c`9}CPW(323_Vi3l
zU;0dt0MDEJJbYQf><P6nhXy~6xGZ(tTNA$Q5x<+L+Aat(#fmQ%T)NeuxcSIdyc<F?
zF_&%v3_<XYb95e-9)d!k1AHn?0+eiA0Rdid6gJ?W`)H}Jmichq$VBsQ;UulwWJtU2
z<+M7rTm3G(@=ldvl%;f2kXdWK{>6>ND8)&jTxx4NCQmLG8HPKxF~Nw|5g1wyvO0zn
zB{OwqGJ9?yI0mlU-j2eC08i9OpkUh<^NE(^`vNdg81bl_%JIJHw0QiTYJ|Ny7Abvs
zE&K8ovbF)Xz85zWfasmZ>@{fbf8w<qe4Qw-?snv5trtO3$^qe>Ua2L*jn@+W$xD31
zUHO3HADo#q(M-6tozeC{Wa+lLmWGXNV5Zd;IJD6hoN9#eePL^0Fk7iy!Ud3D73`JD
z^u0!fBM!-{4BEnONyQ=KkYg>YyZ)CKKD9}%q*7D;V}{qleH{<0MR3+tcqtE&9O$pV
zFj%mgUz$Dm7!VULt|c0^g+IL%a4g<GfMp^QPMysX<Zk%p%sv~b-53hE-CNLOo!Xwa
zniQvR8=zNhN6N*yuanp^>6R=mjw$hq#^k~JAB*WH0RA`nj5xI8Lh=#Q0lZ-rg8xhA
zrxfqsJ<u?e>S6Qy;pP{fhgSTE9{(u`Achz~cBq=i#NH0*Z;K#h57C0s1`%Y>fMb($
z+}UK+vF`OWL}gjnbZy^wuSNgzZ>Rr$@BhRjmhf8cBahgq@ORdB9z1;PSn!KF{kB-o
zt$WK?d9IljSglXiD&7>-b8^aQ{yPa)x|q&J$$eJ|aMkdmXs-muMUDN79oXaM5$P6^
zIT5Hm=r{+zhK>V`zeDP}xAsa-)bickKD83Acd#s4wb_%r-BXEmLXVU)NWwF=4!`!I
z#mt(;dX!Tq-xY30yhAHT&ym0W7D5h3)b)5C`K`BN)1SSr5e>tmC*XR6ZeuFtEUA;f
zm-}r!J4L_LRrG|#w<qX}Q_kJdJ50Tc4K+4Jx7!Ylu-x1|f$2C27$0J;rue?xU3~J%
z>g!eYIHXta7n>v}V0Zt*J93{B!h1UA7f`DB^J8hlhhzCAKXI1mnBkOi=<m^?;`0SO
zVhD$b-VJ}AqYNA=`Bfby=(xFzuavm#@G+C%TKqvgb-t{~taiRW-D~i$)%saoCkZ@&
zEnmZ}$hQEqDa7JTUMpA7qSRo*+4DRPg|gd06$S-Vr+Y_^J&^UI6aza~U=av1%!sW*
z0^<h6s4zRjhVF&fX1WU(P?CX{RY$woEF@+3*`w?*_t3jqwkwULFa61%_teuIyozWx
zl+cK{ZkGsR=W0(zty<!Co~CW^^tKJn##V|@*GKg|lEiso!m`R-w5bQ{0!__jKY(>1
zBr?+SIOJt4di{jAxfip(NEO7s)s}t5LY^qZ_e;Eh@m+@3Wh1LTo%fkhwoW&VcUYNU
zT#843MyhcLv6JEE;?N{$3svYebBDg$XNo`W5?cp$c(FkIf47MH>jg%|ZuR6eEv3Mq
zPxsh}_aRaJZI(7M+yZh(MX6edzdL2ZG=HF7BjSH+9!-$dMBs_n&jLY#d}s>6l)FEW
zE8?9YQ>XZb{G!{mJ)+*^bMyV1>dsE6_sW~@+b#1rg`t=Vbnp}MR(dG$(5$A9_5hbG
zPYoZ<I#Tgrdx9J;BeFp{FMI(I`GUZldw~4+RiXB=cJv3ai%OhgPPgiklJ^a3b5l8i
z%^I)^twQh<-$)hjF2o^cAPZ3_gv(_2-S5;|zsFe;Utx1A$6VYzho6re$3Yta%oZL4
zkG)2FJ9JGuX;aVPW)ID8v~J#L47v$esDup%25KJW#5=egEj<+h@z}|yvA^n1#W#Ga
zTLmLWyUI&pxtK&JU|hKj5R~xQBhSE-ylj=_xz_vdpT?C&UPVT9r^YXuU1UBZRVj0G
zw_AC+=QO7K$%!ZG=%|}ElLg_EKh5NY|L*-U_P$N%C`s%Z<xVu?E3QfI56_&CdrpzI
z%U_<0^I581Dc>+^_S`{TXTRj^=<#qN8Th>ypvkQ#TjD$~Xf{^rA67OoVnT~mm0wFh
zKjNEWeP5ExDO}{J`pS2b5m#ZcInakyS6<DRL4_*@*@3b$mw3&bn(%5E;QN6!^aI(}
z+TqQ=iV9Y34+lUpZ%Wf16XUXswPH;g?ePx!Z$(26i^SajWIaR%7QlqX$hhdT`r_Z)
zydd$*ObOEdOnFDO)uHK0Qc{+{bj4U+%{poO%JLhsxK_k&k?_yja%6F~JQCPq`&SeF
zM;c*Q>6fnRr&Wn<5^gQ~9JbFswC#CRFZ!;q#~5SU?|v$&9OF=0mcNG2j>&Jn^q<mf
zPRR64_>o5DvM+7v@E_h`=+VOXhv5JA0w4?@y7c8JT?IR}4Y(*8pa;@ZYQ4EFt4pC`
z-+;J_#TOL2?h61MG1g;m<D;xa-w~Z632*E_AO@oyN!L6-L?4&K#nwe{S+isJHCWb7
z`TODJMQp)l(c~TGicg>?pMZ8b;!r=@n|~^1gz5l-F!ov2W0~EZ-tUCgQY5JOU1zw~
zWuit+s7`sEdxWB{^71$Yc4t{&j2xM1fgmrO#1yj?IW8we)Hsf~m^^8^NVIsWJK#8m
zBtD7&I*4>s8-S19F{>G2*o?3PGc8KU;SQs0d8XR9H}*SiY9YOQ-)gbTz6Y&=Ab(Mf
z*mZ-4?VyAQ^!K0r+b-dNWeXX2?mIy>%-J0qDRF;SDl%Mee&cvDUEzmoWs$#+0I04c
z>C?zN>mku|`XWF^l{KuNIQTRYC@`oZFFln2UH=OpDCbKFpKULX_7jg@q(tThkW`dv
ztgteN$WR{QA3t0ii2FiM2dQpG`zy>?=^?u>QaI%Z*KPOf1Nl7Fr73l$h_-4GKfV9n
z{J6_?SOo!&K{HvNOi}x4x`MQ$--uO#2QrJQ@q)B0H`a8~4DQ0O_t#HB_3<uI01)~f
z7)k~YV)ge4)+e$=ImxFjy{)Zd%k-sLureZr<aMEh9)I1b6GWMbZ~7nZL24X6wsyZd
z&M}MOg2ddIl5#SMA)jB2*0^|Z&5{i7qTjDZXUhKFz`A(Eivx?W#yL`4xe3RC<Q;IC
zdBn1gMWGz)y4lGc)@W7g<2!t%HQb#))6ca2lKHG3Z3tD&SKH(W|3qNOgo<*il0&k?
z1O0#v5jNJDI^=z}4I}aCZ99vEuW3)tyW3;Y*Vg5{PcqO|#Uh~NNVNK^<rkW1ihwKE
zkLG&7ESBj@`*|F)edKc9J{3lKXC6Gm{=UaHekd|@|81zy+d-r7>cSNKQgsgvm?Q+b
z&Z-%q@F3BxY=SHt;WzRTE%;qf4_G$ixN+Zutpt1C`fsN5>5mAvu6ofH@WLRP5$$iX
zYRK8am&B<e<4&SyQF(a~uy6{221AOXS1pfSfMB$A{8`z6Bk?ONfA{mF%N$$C;Be@h
zSgF;Vea7HuzsNU%39C1Dzi`RL@4Qx=;vwdYg*<-c5m;z@@4Hxl!^K$WX+aXfdr&$e
zbTeADGZ(vnSiO&1r3B-{>BI#G{*(`ZlD75YZ6r1W2Zr+2^9>xjg!>;pvb9?fESR(h
za*xaTKkoy6oc!1A;>Twg>6MN2r<QmE_3kau_V97XYAkDnS#a~kSDSXBrjPDV3Nz=0
zA$zzhiN{)0{gSz4WPjMc^MwP`PA|!e{~VCx_w;bC<R_f3knM;L=?3R%oB96ece5Wy
z^gQId2SYANCVKulxT|NOb!uaP5lwRQHD4XL3&QgN#S|dp!_<Qd7PL2VEEe^1*ViY<
zwr31szpnByLRU4Yk;hg0%zbg3kSRrtzd0c}(N|wQrTsdti({RE^h`W0$3=q;FHk%k
zK6kdYli%CCc{EocN{2rHRl|P*liW6uBr{<NV$)4$NUv5my&=R3e8i9MY&Lsbi>me3
z_AYR+;`gxi;Zco9d~9BtHF|K7vKL~T^bW!wIM^&3>_F{a=k5|^@?y0=T=v(|Nt&K3
zw7viF0=>Mj;KIc?P|ru8D+RUn;0!Q%=BhaiSe64A)HUbuQVwyXQP2lj+LMU|5=953
zm#EPc)V#nyE&^Sr<96hf*)Q>Xaz!$4GZjc{CtMIcNod>uw-ZZyoF|XJc2B$D(RYzR
zZrY(dZIxe5t<=MD9(nqw4gKx`XF6<D^*L!xivgK*1y(jy&<j)Y|8$~dYSYkfFM~N{
z%UbLqkYfM;9rOQn7V15dc!2%o@ctS(7bSH&L?nOBp-Sg<%-c_yoa=w8gU}u9|0At3
z8v3oYP=?hXT<`_eTLU>2d~W;W946VP_L^D97t3Awd&?s<#j}z*^Zi(y#E`dyiV@7r
z$Z}i(@3H^!16)R`DR7SNt*igSA4W4heh8VvvIT9Nv~+ut@i1JsAUA0@Wq42C$I|L#
z&c<GWxzXem+inN{G%sI(t^Ct8?Q^+iO4j?1j_B&sk5VW9>?@U@0QY7T9K;=Y+6(&4
zCX1=ZSGso;l5EVfUMT`1g1C4pLS3NALmB#jG_3Y5VlW6r^E%8L#a@@$1}#qpZN=k8
z$~{Y2rbTo;59eOYH<SBQh392~|F4b>BXN`$MNqIX>AK+?2$lVwN7BMSyh$mX$p(<e
zKy{lZF<AseEr^$(drM{@Utviu7q*ac7#Mr<#pwXmk+s@WKROm&E02Gvi28%O3t2D%
z`@<?7*8rPOmNb7n2jTl=faantgF)oe2-EF_@BI7?>kloYQ9Ru&^1o1JmyCqD?*fel
zTZx|I(KpvQ_;*kY4EBRsiHTb5bl<*A(bdwniCl@IdQPCu&aK3bat83v^XopPar)AB
zb`)C1aM((m(|dRiS9I|rtbwI^Z?Eu6o9Nn-QTtBM9V7IYk#gZHsNf}zOQ&J4=B~K#
zT+}sP{jRQ<l&60>DGwC$zgT<ku%@!MeN<6IMMXtHq^SsqH0ja;7C^d)(jf{W9aNeK
zNyLt{p+hJ^s?s987ZpTmsG$>;5<-y@0wg4GR-Esh$vE$KUFZCM=dYP7hMl$dUh66M
zeLv3=>}|(=q?zJ$x>w)ovqy=iVmW_`Fp0II4pXyhJFpcP;4GI?QZ@Dm^8JUiKOScV
zpJcngzsrQ$hvs?u3bLPj<G3|Q4Cz{+>@Qb;@~0hU!|-u_{z!|$zj|c0SFSHy8KRak
zeNek>`Oc7UC%%N~;kc9K;wii*62QHhnj;Qgc=<`QY`1XRu-11TO8xiv6!UxN5U_s>
zIjQantbF<S8W)GVX_)u$+PUd>A(Sj*x9N!9^BsQKZnbeq7=m)2FSld~2vBWz=BtOq
ze8)N(?r1b|im~sEaEF6%IiTaa={4Do|5+rk#5i6gM+E}waA-`pJ?-?5Ex-Cre2&eH
zzj}gxTW3E})v#7ivd*RS@ymRtmW=c)?5j{=m8=&POnIaG0H!0cFPlFl_C3^8Sd;~%
z1=|IuLA8>M?4#&yPw4y4J_`ydfxX~$t)lvbM}O&(u%F)H(MqS-Hv0w(jJslcEy;VG
zALul{R{$`lX{@EnWUr9H@OTVVb@Ktr{FP$9z?xtAk1_N<JTzvh7vjX*QDWYvm@mKT
zR~|fPrA=K`s#jK<o7u1I6(}xo-yfC_Nd)tKqdEGw-X}&=>tE-_nIKB_xJhlNY&n=U
zsdD8z*c0R%{wKhcy(bNKeq{BfJ?S2Bl}0}^A7_0Ho>f7Nej{HRxZ9O=#Au)<EW~9P
zGP1R_W{j{G(wy$G+#v1#GSxkqbH(Uq=}Z8EdHxlybzYhJI3=BEm5s6%_Oe-ASL?^b
zRPj+vuX-c5MNF0XY}<y;i$f<Z%~i+&Gi;6fUzM3;H_?xI_Ugeu)nlXl6^I!d^TntK
zBa>TPDy>?BkM4aV-?2X(F-qP#GT_#w)gAa)xBXLa(CSO?%KVAwYzXzSwHbezgk;7S
z564J|NKBKe`=@w$o9+=UPy-V1va`J3H^_H}+jXn8Lk&@yIvN@+rluolQj1>}Zf)c8
zhBxcD_5G@dd8GH+;Xt?~pE{^dERJh7Bi;c|<0CB&A6~7B@Ehg36L<WNGGfJ@=n`r-
zr+nV!xo7u6jmNo3qWd;r*l;=N;WUEQCpxsX60thUqlB0Y?QEW*MyzC37J7V$VNrA=
z%~_J*cSFS>iYljl0+uzBh-JfZK4$W>3T5;>1J%3?lKlvvp{sS6sXHP<hqWpZIih{u
z8EelB!r6<PF<9YOC%iXiRw3^r2QoeCa+sz*8_^<MFX@yI+ApPU0a2_5|8O1($-M~_
zF|=;s#6Hb1ZM*)OdS4JiIgI+RAAeMJmu`E<KM5ZN=+Q<R$^Q0Itq8@K{2E`w!Jidy
zQPY0SsQ&O1T-rgxWu;4MOh;<%2%@W~)@SrS2b;}Gko(*3)(?ap8dW<VJhYBck`5>L
z)fkUxQL}GWwoWNG_AkC_xoP7bU0^&%kclzohbvRw>u_y5!P1<LbZMga<g!9diCagy
z{Oqr&@sD7@it6reFeM`2vB;~HG%;PfXgSd~cw(xx%u9-mn;v>QZv&jIF17okFO=?;
z#-JJde)7L%{)syJxoVkue-ZzRq;pV*oUL2<2%9AuA<-P=#;aNH(@-80&cQ8y-f6_5
z`l_Odk4V;*GBNi|!fKEZA?nQGiLMGMw@BnV+p|v5#E%2yu|?P>lZvS{JTPA+V1|5U
z+cqKtlGyl6Z#CIRtJBFa`bXcpzXZ#DwE!`O>6+Tvj^5Nw)AihT#hl#v5+9CRq8>lP
zw``=;f*^e}ccprN|8P=sGM6P!t)};f85!LqvFjk|`+5dk^Q*5XJuig3Zz;AL&f2=C
z2o!~=x5U85d=#xiTkG4K+am()4KlX}{xU*99d~Vai4Z?)XJIOqPphp_|NS}EdeLKx
z^=6M*n~xeF{Vq;&-bvz=U$W$I&QAjwFDKZH+IBsTa*K=nAi?pY1wdap=)9EpknD8A
z`>%HOpl|b%3==Y|^hIIh%8EIy_>haa-gJpsiC8UpE@iDh-58fq(`Yg@yg5$I(foo)
z)26KENx9V^*Td;@)kn#TbBh5j8|c?59s>UVY4Z*i`=o(wZ=YbQ(w*fi)%m{ontWPO
zA6mBYOMt6OOC5A-u>r$#Z^-wybLh~$vLr<2Y58e87lY1`Vad${^*q}-t1=6#<e}-Z
z8}&Vhxrv2x!g#lZI2a3@9Dvp>Zt&sNUHi{)BYKN(R;H8-vU9E_9qcUj4~GxNVUEA!
zzZoJR4Y2ypnB8B@pu1(c0zw6(fhAoKJ&h7=SlpzxKX^H%bgOk{byJCEpvIqV&oTI=
z&>q9^z|p5tE49h^_hoR0hwY$xvu@AHsfT^ih=a0QZ>}|hbmbNwbT<2CDD7(0D&a(N
zO5feaGQK>V0NI)ra*Mb%UO&=VjMt_;M5(-ug0^}<pWt;jcwO;aMSo49I<+H^`^am;
zQ;4j2aJcjfN6*)@xxo%d--5y|n#;n~KZA!p>Crr;{<nP>KnUSp#F}iQkX2e+*O*G2
z84js645`gEsY5LCiMWs7=GKmb{WSt<=iZu^4bp^=<|X<&*u0UI6%#_M%+x(V+7<jL
zzW0-+pr9RMcUidLG@95Zta_0quZOcXMnkqgf^>X+MrCilM{JDig+hs8wwL4ZdAV?*
z6LUHC;Twux&jDq)`_|fjhN9A6z%Hh-XTBxx#5ct9J;&t-TAWPpTcOI;23iP^I*f7s
z(U$gyRrr29v<b5iZ;#l;4rLt~olcUMX(st?dL}3RoY+($iAJ=GW&7K7;yyxp4WQ5w
zE8%PyBy5hjhF4M?YhL%8H9Z5~=Xwo=n!8cSJ7B>vSTNNx{@1_^Fy}`-6;d=OgMeiq
zW_;_0OT%!7Q$e=hjp6x-rUYc^J6f39+sF1U<yu|+YM0tGObH-$QmxQj1BAJ`t7>ub
zrX7>35%4b}NanH!u0t|^4Q6@Lz`A$l&L-5-tj$R-6Y_e1#RK*tV>%b=U6XnhCSzg#
zX|WjIc=t=JcT%sCJd%TXE(QYg(PQs0bu%AP%wJ3OG*ZX+d>ryGzZN!ZcAHxxmiC`M
z|6nFV%}aIi9J3TxFI38muW3qj{G73qNbt&0BPi2rC6|!P!0Z@Kt&wCjCf`~Dd@e~w
zNWXg(d#THB^i>!6cCv@l;u(HcSI)q@3JhC!{nV1tR!wT0M(m5Sl^Q-mPvfkp3F75-
z_Nv#C%#X96D}{Y!20mba2`#eFWO0rF4C(FtQqpY;8B_yu02wd%sgJ!H+rE$4=ILh{
zyy8FS;`pAq&e2jrAb$3^>ge~5e9MGWdnM6*rNz?O2>SY)>4|_$)DF3cUHW_MDXRm>
z)j^@#YFxedzWS}7m{I9UC4N-G9sfpV8o#ZK`Gaxbz~~!wJ*l}fNQj)J6MMN5xjocv
z%El`A%q;8Y%RAvLlopGV2{r|qwbE8m&s;8}!lV|P?S`-MB1%2HBrdZuQ^*bQtr!l3
zNhi|?roxEz?-hS5+ufRXR!Ar_(84UYIA+#3amu)|tZa!RY)PadM${0t!AsCOi7a}n
zP4PN193tiPCT0E5Q()H?v5Ks&{xEK}|J*q+Xp3jBPlJ)o-DvaP=Yn~in(Tc={kZyR
zufmSSCWTqGqCn`>o#eZ;lE}h60B%hDbT7oot|{QdFW);uRe$ObY<62&_YVrg^DIAm
zIQVYcr=j7-S>H>`<r$p)JleFFP_CeZ)J8#q)^#B_$D$|lk4Ju7j`|V#q~z{>?O}jJ
zzwZ={G%P31``u8JSPb`G86h7$tbmxjz2l}2!=YSKVoHp9(UYWFZ!s%9GWz&GvI<Qs
z#4>KGrnfBY?bhPAIP}14FElIO4c7ikXHm2^R3-LHS3$x(gVv1~_tavO%Dt88QF`;&
zVTZY*UtjCnuBNWY)hTswyPC4wBN%Rp;eS0eo-mh{SB)l#{q+<dU13^q5qsqTKjwLJ
zI+?GDC&sA$=tWzvn?9Uy_hzw8?s%Tk<#+7|entP#eLPZ1=Vj`Z2OWm}=zl;|exrCk
z9WHpkapv8;1a&klSnOY0U*UA>E6R#w-Aa1N(T+F-qW4CD8A9z`llT;1>w7fbSs{ZO
zGs1lWWQQoyu!mNtq_c#M&#9h2t@$i7UtFRE82fW7l6n7qpbiHghF>s^qBf+!9{Peh
zIpeQ|)=5|1;W!dbf^5n`Jf!R+hdx$A@MD%CmU%pH-yK2)HN!9ryVAYY?e{hc?c}T4
z@$=1GlKJ*k;rhxt#jp=zA3{@GAq<|_MmJSYuS8Xk2Z`}06t`a+I8TbAe7tifBT13u
zI**>5b>Jt)&EbiB32@=8ML^%6w*Nc?e9uq#M_nJ~f9y7hx(q`p{fPUFHXt7F&J8M3
z`)kn6-;=F?+bKWOz%*iM?QX#F_hDf;>EE20#6DQ-N7MciHv>Zz{iOb+ZSI$g$@x8s
zi<)W^o?{NSidznyh!-e1nPEtJas8*&1kX&v&^co|(7x?ZpsIu(*y1u-jaq;|%=lPr
zFZLl>@EYvz6$SbqIaePiD8%0}y}HuY-EX_S6?;$hfu-#`exC29r7I1m=d=jjmc41E
z<Svf5_XqXc${Z;V#GXJck_z2ji{YAcU}~DXZt+ab8iC~yS?RdJ@C+U|k3T;0hatz{
z%iH$Z47(A1X36Ltx72BogG-H9McLLgs$~PC(g(Mz)Dg&h^1^=0fWB8Kx#d*U*N|;>
z&<bwOV<TSHrnCBg@(5q%N0)Yj-KKt*li{KYXHuC9%g$xV$*6un%=EI_cdVn2NksK8
ziI^UJ(_6oC>6e8$pFp=TlNo>8m)*&WEsOirSY1u8-{ohxUb#T3MrqX(la#^I8zyDb
zA|m1i8cxxr(W<&)q!;_)uPFOU+sR;c1QEL)9>qmIy0$g~Sk-Z6X7IvxL|8LS;1~^2
z7uz2r-z4hz_@?Cc@EJ?%3vZg&`?IID{8n7_!zj$WR3DPSSo?5>25>AqszMz-#PJ89
zZ+XC%i{;t%QCr)}=BQbQ#T~BT+^OI7!@9(_^kT_mOo4v>PmXMoK=bokdKcdVbrpBv
zwUN2(=~oB~)X@tAlff=GC=#&8_Nxn0ZWSQZe$vUx{Zmz=<_gf;?M#vUez1&qU})E0
zrr}9L$KHTwmeq~O4(ZB1TML>g&Ha8*RJAJ-`{);HAAVeBoL*O#B!mfbnkWtM+WjY=
z;`eOfov-nKhPKlK+PS@rpUFk>S=0UmJy9pp!j9}D-{h0#b|t6Er%U+<&LgO|hg?E4
z>l9!2Dlhkv!*F?$^e3Pu6gog0Khi~34o$9e7}PoL*@-Yjm~Ma{*#BmJ*WV}Qo96Bk
zYNkHh+;DU`+9GasYcJH6SA&)UCY)`pPncpJQL{-dD~?42DfAC|GfHhmnG6zpKyZ=k
zx4jqVlx_DPVsnwuIO>Yvu7@2@5xEn%a6omIAv73`SNC7SV7A%vNq37c#>Gz-dy4qK
zvyv9^3DSnln%-tjRBK18mEIxmA@asc;IE-9rev2XXcpKd=O2D)#Xd0HdsJwYEc6I(
zbF3<3VE9x$@q#K<V#EIEt&<sk&sQi@F0ee_^AZopzV4i+V<6<1Yet@M^G~*T1}Pwd
zviUKWscJi<9BKbZ`B;^YWfFak#z3pDm%HjgV8#wUX_feg8B-^InDVPud95`bRg9Gt
zPZ_Guu#}tqyvRk9715WXCqC))#OD~`UUq{G+^wYc2aa%tW6(h0E(^nc^F4Ezy#$+@
z)XBDR-lnyttIWtD$jE#%sB+;3(976Q4`+6CtYMa-<o9^6<~~eHlGh!{JfIJ`Z&A^(
z7zl8zM8$)#tJ6P;%bvpv#n!D|<j*9lic}(t1h&Igq2D(&3ir|Hpw=4GUnA&G-M3>x
zK6<ng;by=uhoUUamHwbcxNF9~)s}TScrUS6i!#!SF-kU#UMQt|-+CK;&%x?4L<ATV
zXY(C@uuSw^5gUVb8#z3fcwvTFHm*ep1=AA#{;26XKfLF2UdPFW@K-{FUhJ1!b~n(4
z34SPDgdW(-`Jtj-<8yPZDjpxKwWDARws-Vd!<X4t{9R)W5D$detlGEhSs<L|KVPF<
z&vJs(emq$OzpZ@@#y?MFJKnkCKRDFTKIC&_ySItp-A(`6&iYqG(EXNa<exK`8iMyZ
zK27DinsV-<Tdt6jn<z?t>V5r+RFtbHWsDruG;<J$WgWt_(@8bubl%iq=9f~{{C4$@
zGu3}bg@-HH0@+kk%f%yUOn<cYTy+nMLsKnR1a)a&a^hbdgzNLL-F#`^HTfnMVs7?s
zn>2C@R)kw@R8fN7Nhf}?jqWsUkIM$BQJjE1<0KwQ!9Vy~w`08YZt6UX#;$eewQ(Rs
zg40Y2%$hHX>%xr8hH7N$b!<NgBdt#L3O9#2vCf#D?IK_1O%CrRGx@1MWiH!!eMpeO
zz_{#tWF&53B;<xHpDGuaybs}Mw_whGk1F7&9wJ3uI|v3iJm=KsbMBdlf2~O4rJe2R
zjRL0?vyhF~hV9F+u`PbYvL&F=VR}`o#~#P9Q_-hWkIQX~%Djx1J%{e_5nUL1G1m%^
zEpt(GwH3^8*IC%`$rFly8rXoaA_0u?<~la)xOvz%AFX|0%eIZ%m2;Q9FT)ijh5isW
zu(;6XmtCxuZT-{utNlu)%BJV=6kUCiSl@~nc%0Eu^_V4#%tNdUejs5yJ~f2ul`zdK
zv&yPzqyJ5TnOmivUz`KNpxUZ%_+i3|1ZFGAr_#1`N_xsVg{HXhu*td2zO}k$J=b+@
zx@@Kb=5G~C|F~Q+t_8m`k8uKeqoZG4w@#`-Cw9gv4DM04k7HP&!s)|NLYSnS{+iQu
zhbDx|Y{>)WW}+EzTPC&lKce8sn^|14=GV&&M&`3!dfpIvZ)BRwy?~^nI4C!WddiiZ
z4}!WXFN|Kn*L+;-F?#ll!9H2(t4AGFbvE5Z4=T~#r93V{2-=bmYarvs^&8&mjyD9E
zFuW;VR<mQ%U*i|ZM~9J~D-3qZ++|~2p}^7+R!57oD?gAY-)Z^MitDrZVMbQYPCGjv
z^aK5ds&f;i5c0)>o>nOch4!y>j{_WrA59csP7$>WT;Nf2W?0o|$P@m~IZNlv2iw?v
z!5pJEZD4Y|C|y06r!FwY?n@&B7%@URp=uBaZE?a5PLJm<($%9C>+VjA=lP*WYTumQ
z{CmRX;R=?(BD1W8iFCbzlKa^Co{GqD4SB_?LBXuERu~@<;PE)5^mM$GG@?p9Bv%Ph
z1Edz_S=OU7Dyh|o5eZYKWSk9nbY<^ib~9{n-044rzDYTWlBE^R%Wp~`5(f#iz$!t0
z5^LnAUGUt1M!Rmkhiv*;cEFD+78xGuUp57GKW&4T)%bfc^^xC#9)jF#XLt_7L-aj+
zUC434H$R{C-c-K)cB`17#ptRbt6(4WPS6YajfL54rjyhlq3#Tp5Bp#bKOonA!~e{m
zJU%R}yX!FI=WxNDYN@%)Is-M?nIzV<wOi=6hXJbIKqh7J{>@j)`xGuz=|RUEyJw;V
zHYX1Nre;?%+O>=9(Fano+^5<M=8(FR$wI96%xlh$B&=5^Wms~5xF>*|!@xt!W7cjY
zxY3@!eu+qJAt?pG^k;fGGD<g9X%%Y$o3%a<lVLC5KFPRb6v5gB7_wMl|NjhS1@Uji
zuE4cYL;XSOrt6vHVW3w+Sn!?hV>__4X`!Zl!!8QAE)FIc%B0z8L?Jz|W0|9}GN+us
z7r1Z?a-MOoggrrA@W|zQ-&^Kf((87)e0qedGKms&%WhYFjP_+|F(>iiw3e9qgw&SC
zj7w69<M2$>oy%Lc%{Vt<J74BuPtUE~Icfp5?vL*Ywj6_<UFi}GZ<)5&@kT^>yz8Sc
zM*E}t8b3vaZ-}PlTn{*(UUxe$F<vY;Tznr;_?ZV;qb^5<>vi9qdwnqYkyXDizc!}h
zEm3QH#_E$zf>kEbJ3KDi>~=Mei1PElQvAb@1OvmT{A$h)8_tlFoo79(5ZlNV@e3hT
z<k)dtnh*J#LnzOz!_&d;jMM2Y*|L5<dNMy-IFh+9D)Hxd+VS1q_Go}GTHPJHZFExW
z_8O^?QVYn_uf}h>ptE6(BYZA?A|Y#EcB9W)7NqPVefP~k<wSl*Io^0J%1l}P_Meck
zo4f?CldSI*1i6Fe*Fy~3$z3D0w$_NhM>k6>rRkG-O<@+nQmnjR=oPxOE~9j>sdd;}
z2=&majP6F{M4BZa0@>(mP0S?ZPo#QM$axRkbwSkyxc%RJ8G9hb+pu=xEqaaC3oc8(
zfeY>8k+3lf4w|88>WO_VZE@-Y8?CMnBQ}-}hC*?*Wrb)L%hr0+=dKCQ1n55IB!jE&
zJzM>c)4o$Oy<bC4f3Tz}_ejM4?a0unvZc+>%YB^tHq6;_A#Xq%ms-30ktaZVz5!7*
zY18u8!lxN?ay3KozV++8BFlJtYQ!^Cak>!0P7vSM=QLckKdlB<gfx5fqS4PX`42n5
zdWW?E9@>z$4yq1R?TWXGe$E%KJ>)`W7+P@lWS2(D*`g+_<gZ}-o9`!%a!f7EjSy<`
z*9c~_rk{S?5t@!3I_FmJd{Mr65MN=nmN+*TfX0D4N)hpA+$JJ{Dm$-Vi7v-r4|<Jb
zmy1%&-nCDaogbL4aa%J|J4|!hA{}T(SU??T7hPHhhD{viS3~C1Y&tfz;{8$}u?w!9
zkOJ}MZB`|RK(~5dV}>YfDR*B9S|OGkoRvvNP3My4eB8LioAR>tXz`uG^7r=E-%`u9
zxtGy(7AdC0Ju2*Qj=XUKm|EcV3)ebhR+I-jm(%4Lp3>Ofk`{-&cOv--I)%e`V>Qwu
zgX<;ELxY{}9JxZ<-|nh&2;QT!S^iq*wx&l<VOY0UtDK0~Oz+#O+~$%&LM{ZPL{`Jx
zkj_x{iyj|B?b=cpgI)Dg9A~^n<#$3|MP5s@RH!uNd~MQIZH{?*&hrVa4{F95x?FQy
z&FS{HwOBpN?Z=k-wG=ALte?lgOijdcZByBkee!$m)_;QLN8uh(Rd?Eaa|m2udOgp?
zkZo`tUdm`(tJe2XwVXwSKdBsoqt~P(R-dm__s2a(D{Yl?Q?D|kf*06t@m;pUti4?d
z3Yi;%7)_mSB)RhMM}4hG`G56Knq1`=@GG7SzhW6WCiVel%G1bT+ZZxT^;1V~2a10@
z#fUm7?43!J-I5Q@?-biiF7L31Sx5~R;rvFdu-V7c#s7c!sQ&RZY}!tAL40${Gl2$!
zG+$gefp@D7Px7@eu?%}KE+0p7YI1CY<J~;XQ6?c}^S=MzKH8e~TX1j>?-?(G0%;2l
z?-|kyu8Q6K>X&U_It6~}|Ln4VuLqqXg#y^m_f{LmeRNfwt!I1K)I1t%x{IH$pgN#2
zqYOllxNgJ9&JSmSfDhL_IV9y4|403x)Gvs3>%{@X<A4j<&mY5Z9e1r%@l<hE7bzE)
z{F@0xL0-!uDXC1j*tJh+@pcQzqiwP9oHt+K!2dhZj#r1=Su;;}5@WsOQ{`7>s5QkI
z;I+UDPYL3*LsX70enO;{Y<qmG8<P9`Fzwi##r`NhnlR&bloTOk{7~|N+ryJh$lH&K
za!$F?JtgabTs~XP@qlLskz!BA5KTx3*BkC0VLJ?(td7quo7kU9Y`9<Asg;wcFsS*@
zD4Mv>Rljj;zoA*Quhp78LPyd?L@}0frsV!S7jKNlJoG!Ku+qloJg@k?kC4TR<aoy^
zgCSIvkTU)&S?xV)2k9x>oaw9a%9k@<_QlQT3V5hYR}fH(6{E*A7FO2*8T)X?u;xr^
z&`V)=c$JUM!l>>h&%1i*r_fs(*br~&Ui!m17&a3n126_LVAK%_o#Zz&Mz69CFdWMD
zBTKvda4X~W5d)^tRNdIkSvJ-O=QFkh(mqY(lT~{v<Of_j{1<85<nlV9#e$Y%yhn0J
zQ7s%c3jRb3KU$qq=WA)YOU&cn>2rgm_uX8t&fFKlhV0#u>$@m6qbbl}w?2d#yr(PA
z&CuT;4nkIHOT4TuFyTYJ#1uNgr%Xs<E~`xm<!4x=>CqWs+qv9(q|#T-D_SU9{<asX
zUj`wu#?a^Ole7)Oe6z>2i;MX1FjO4bcU&rJy?1g5*l3Ku){<OtX`R~wdJW{c+t!;c
zDO&|gV6*);TUUn3d*!C^KzMJ_@b$QAJsa@?#l@9*PxH|HO}|wmoXYqM=&g@Q7~eb2
z2m9fgDCuzX0Yn&}4yET*I*upT;%hyifetzV7QYf+#qe?zgj2Qql<rve)*=>VDHj)W
z*NqX0>KKCL$_Rw`U>wkf$*x9|)PV<jN$olKq9%hnh<omq%~-;g!!V)DqcoV3E3ZkI
zEMLL(w8=pLQS|+*%|9atgM!ufWQV~=YWZ2KZCI>=-5W_bd3bjDY&3-)>lHtjlq*AD
z0PfxEtAQQ%wUk?IR(aU7P6FPpxl+8}dTIjw+D0Ri#$XFapQ&%^GV?EBTAi3Ek{o@u
z`QkzuTa}wy%c2Wro8D*W>5D%lLFkbz?>j=2{j$iHB2Wc`%ql-bpY(D1LORL=AcBC(
z`5h5FeI18?$KE~sj^tJ<EiTTs*pwS1YdF2S=bFKQV0lya?Rvq>uniI=;dFp3Z`-%k
z7I}kEzNmYSkLT!4v3X=7-Mdb(E;t}SWA)4*^LrR}2jA{l+jx$q)ps`Ax$Hd@xCM;+
zy}?wUtvJ%pGx;PSM_~E<E)E&DMCtkKL~qV3DZsjj^DlrIm|w<~Rw{7em7R<5Mdh9!
zhHa0YEo4u=V_|?&jG4U8*l?Q)3n8-{bFonz{@6#bJ2x}K2GU-po9MzM*KXceY?qvr
z`fg_*0yz0Ov&q8QRw<{aNxp1>ot5imJQI}4@C%<-_KfV!nP+EMcGGUG)0^y6muIuj
z{5$YoFhm*!fEKM{lO633DZJ$T=r7qT-d~*x_95G87qSsO!p)-OjlF=1qqjcB`WWD}
z@M0f$L2kleHsJz+Q@$0^U+^WJf@*Pi&o)rrHv8GbP|aLdpvUoHCepl1xS7-vvr%)i
zu1vARo6k2{24A}hokqN^0$Idiq3Wl9N6-~JcS=4PoE6-g9c+!zhPmXn>Bx`A-5-wy
z3MY;?v<wtiq+xUTXMidt$S!L~No*40<*)QwHs7yy1Nmn;$D3L`yybB9*XG54hmnC(
zr$;+P*rp$AZeL@YJ%1w!4o1D6Gw?GtwFgwnk$u;fSjWW9a3y}wj7z7t0X_2hwUkr$
zAV?f*)IBC#zGb7>2zY@^z}pBAiaZ#=D)R2CeQ!>ta-Lq{ARl$eEfRc55zYBxd#6{Y
zy<^RqoS(WUyK8d)3H?36MzDv>w5P|Epl{QO50s!H&=lLig@+mQhyQ*@j8sDM-VY6H
z2BwLgFaCZ)jM!1q4%Xk1AGFXnw4wc6$#7YmPqlSQelMVIuulQWK#3nx3I!2z3P+x9
zoWq<Knt#CWg?D0+ca?|3T6O-K{hTa`AZ*TxZI6j%E)#Vsl4Q8UIQAvUEgQa!k;S4Q
zHOK2cZ1C+A?oImzVnPXEpSd^W?#No32?>ZJ^St;=Os!U1cETQJ0C%eZ-0hhQH7LZ}
z&oWKa<J~f=?^qf3xiS+)*9-Z`-*}vDzETf@=S&O$7!6SW^Sj(FDTTmz3R<@0q;i`$
z91T9Bk6!Ev=c@hmFuB6Fqrx}5PoDL^ya$D@m0i`n7FNL*Bpauc6!R}OXULMi$K92(
znQ)Xo;r})XQmNA^CczzrwZ<5}1vTV*Lw@1~Ui=%Dl01EwZ6+meC^q)%qO@A$x}V%&
z-8EzA2@aooyXqUH7e0?pSfsXmlMvd<KJF=NKINDnujQ!8j}3Ku;I?c$lPh0t-Hd^9
zjX7uAeJ<NG0w9wGLbkU3YXIG!HXHi8Z_|^8w!Mv_y!Ob<1e3&T67ol#l%`jB=Hll)
zliC6vuHns^?}CIwlcv*PB`qq`3TdwV#oqlAsc0Oozr>DpcF*-rgoSw~$lXPRdF!ny
zcG{cvufuKb0Pf62VkKnGl{RT-t~-Rk7-!D;p9}iLQQVuG05T=9r}Za;eYeq9H(a+5
zX4#w~)KOIYf>v(EMtb{Pfo;6FhIDs*Q--YJt5cr;afd^n#e(AXap{6t3^h!M{}VhF
zb;klSyv+8i92_O_HF3$%A;3G5x$K`rT43DKcgD|^Tqx(FF!J_}Zs)gFVy}`2y~UzK
zv)OgmKN=pZ!-l5;J=%?s?u>H*`2g!ihOHnECI>4kalR;28aVRBH9*7bfa%))zxUD7
zWhQTe9(#<EeJ;wqODo}%qm;f)+)bDIhdR}xM&4|F3c|#~(rwtrEsge#0RF6>ZvAzr
zEF0LC45%zQ06G~stk(KMVR&6&iB+zwF|d92m6^84<+$nVyEd6+;)a|P9Fs?}s}`ED
z4ZqXfx@w;G;h+@4o9xs-_6}0XcL%=z7bMT0?fh>=%$dXWV%5B(PUsjir;}u`lVSmL
zW!RgfEI2cB{ijRp4OQ(XZ}m1x4tbx;?!L<$F2wrkp#GjXb~}qq<@MfStWmNio^&#m
z`04xpWYZ)8i1}^qOEEA8#-k>zL0Y6YFHa=|5sg?ORI0ePNj`ASak!r^W(90O_h(Z|
zgo&m{unDLd54ap9m6Uv<_ZJB`Qxm!G5PJd9Jei86_*1GIFia!NhliN|4qO{xfL&sa
zkd_M)cTFsTQFF)H@W*fU99gV<M+BEXy_5+@8femWx&`S2?8wy<YrRr#CGl))bU5ok
zGXJ$DuOo=bZ?czt7$2=DliN=5;f$7hXIBYa*+Nnjxz_x=rp=FVvPI-9@c1m0pfoOi
z$H$p^D4t<q(=N$tZ;8j1l=!oV$5%)q<Nr#kPw$kBrj?g`bP_KRvEJ@~QD$KyfUPJ!
zL^9=A8o^@35-jPuF$>mo%A}aS2X8Z;-XO2^!ezR@Xl({$e8gQfhCr(P>^Bx~@mvwN
z+%FOyg(AlC%~&g135Mi)O9xhedgE)(G#_mGUTzPrQxb)5mqq54<O^E*A*!=%#<mia
zwr0&{YK|7wxlO@J4d9#0^*!T;ZGa5>&Cy#jrNmF|83sPXp=bORDD7iYbFq8bQAJ#y
zFV+B^fVjkY2+j8BqDq?kGg#S*T2vNX@U}_WQn7SFIUp@D(W;C)Ao2;k9i@P`Pd4%E
z96`axn{CX)#)6k0xbhC*7ciTX8!-)hiS5pkUy|JZ{3K5cD%TrV88)Fgyk*wuQ7RHg
z!V~1DPh@YJ0pUOWxA;^OlW@UO4wiJe5fL)iJuufDH$^p^+dP5(;9eZQ5CD;knWEZ+
zO#IJ}+zctJE#j%>yh7Vt2@Y1CeujP>_FKB9p{|y-o}uM7gK)*giJ;~tfra+X;3wF9
zc?p;uT7~_Xo6|&l{4}lZlY4UTNZ~D?X_Z<vOJ3pZWKfUUz>$b+#|@DF3-FH?T<R7|
zL$b}_up@`QoB9QJus)>6j!=Tu3bOERi%tSWvCgm)^l^{^LUPVPF3>&XnFH-#)cVK~
zGY-wZqN3N+EIyW-H|Ga4#mF~*n#JKC9mfY`L>U6r{s?vAq~XSg&L7tS#b^Eel>P>)
znTzn*!n`%oa4O8(FD!h86J8<Sz1yqK3JMj4Dlbdrf~$Np-8(-cJa~ZeO*HLULy;we
zi{(6Wj;+7Y3ER#0!Cs+!dfTy(O{(t|&XnSJxV<5S6bxLqDN;o_>tOV9T(u_t_z_w_
zfZFzr5H;3s2g^_o(m5E;>)5_WMc8uVC{DLZm24M+T_udJAe1gnlFt>#L8tJZd~UcP
z0=Ckfyp@wx*6H1*FiX{PAv<o%U_9r{EbEMt%EL;hCN>7f{KHFpi$a*oqISGL^tZvi
z@4;llW@%BW;3u*ZWm_K52fe-)<cl95o`~6Q<kJ6{w$+$(6~6M(BMv9ZT4BHo^BUf1
zUW_Wa@diRZw2|+x6bHhd-zfrh_&XB|OkdHOLt?Me!RGV%`aru?*t!mB&MA<e-FAbM
z&WCT8baowsIj_b`)4?!xyl7(gz)+qWjxb(&qYlypg1+C7HmRM4C;JX$6L){Av#OhF
zo9cs_yB7|x)b3JCo(ORZmIUAc_C<clOo`jt)M^U+7%|5!x5IzID=&YzCMIjE)jk*#
zD_Km<rD{4Rfu#GoJ&M6ZND3T|dV$!+2AL%AeqHa*v(XN&3CNErHR$M*C3W2U&Rm}p
zmw^&Oecz+c(YQ*muQ@p+{3@9IZ3ms)n8C9gRk3bl8gY8gC=~l<vfFlNhF*Wo;WYEa
zp5z$GGU*2cS+=xXlFl1SjCOMe&v)TMz2VEjK(Rga<$X8oVPu{f@1Ep1sW><U1X@SO
zy$^w`&8{Vd!K?pNSC7KfG*QgSdoA}Tnd)gH9GkvE%x8n!tL)`hlyE&B9Uhso@-Kw=
z$rjJ<=tdCVoF|lHCx?iLv#JWOI{E!92kzvSB@T9z^}T^mEd6;4BctHMloIWjH8yTw
zY1*M;+^tuT8MiSFILu45tUZ_j?qahSu7no@i7ScSP>wHxfkNGP51$wmcKDTL)m%Ta
zn?fE0fO5dV#)bgP!2P?D3{P0$^z}GD74pq=TdaxeVzA^y^o<y_eII9G3=4IQB;egK
z2$0OAeqo0hzeRe=hUq4!Fkv=<q5YR1hwH`VvpRXt78!hZ&XGttR~xJBrc2X;#E4Tf
z!V|cWol5OalDetd5f>G*I&QA66?V}Ro^+ziPt}pC>Mww1{B3RJKrOYT)L+)m?c8=%
z(8kQO@?>&vO46{YIO)5vVB>BDg0Ph_q^o4JNF4T150!KVKL)xM&5dv*hj)<+b{KMg
zXz)3CTA4APKOShdBPhHl&o;)@!~3f81m?ETfL3sjn$O{n_67nY90TuAxA*B<J6rrw
zr(FN^?gqz0pr55HmG56#1f`gsiwO@F$0BgY-N6n>{R&~+0atFDp9_vDR1zMrO@J#>
zZ(xygHTk2uNXjJ<Z11=7_b8!suuZXBbGNKz-v>Ik$IDx_v#@|V0F|xRZO{D|0}o)1
zEdzM?pLEK?-hh(&q8Z89#-ve%UpF@2#0Im{Vn|b}420(WP?;4VG#*9X0TD-ZXVz*9
z*VA#uJs=~fw+30{f|vU!!<(*;dlIX;71Q*G*%=W}jv=aHu(PgBQ1pQLt6<8<)5TK7
zEI4jrl*v6eu<upv02hI;1zdy|!~*X>T>S}SdKLZq4WAm9rfi_LV-t{($PZ4J-nr|D
zEe49go2O4~zJtDNK<fyopQXtzf@)S)Tgqy*lv71K8@;D7IS&p$^;t#(xP`Y`)(jS!
z`=}%9slszNO<!UKBg`a%<p)ra1Jt6!pzzjrem3nWiJ27+;{R@Ku|3bDg#uCurwmaN
z7Tn50`oH?ofb_JnnLt&4l1&#ghW^3Jh=2sCTQhZkt`u^adwR{y4(14Lb50V@pSy9C
zWU5J?{e<or>@k`4ayzLXEDU>SfT}U6#tm`Pu}litSC%|x24@-Z4LZX1w~1cT-TO%D
zE4BkG+gXP!V0|!zHftWd0Z8ej@nn>oRGUR`8AM{>?gIKxd`-W$#{yU8&I&GxZB+H{
z<aYg;YsPTzYM0g7zCGT+PJ6-6p#@zMh(#$4M)fxWaXI_u(j(1<t2)C=>JBNr>~*X<
z>I>~~b#Ae6zo)ndF3L}oX2&~N7O<~NtUI`TM3z92Ugp^B)?yqXpHewq>I4G0e-eq0
z6a!aMt>J3em5Y2%<l#O+I2cQ9N<AY_U3i-WV1O`GSNN4?a}aH&C})NCu5{D;V*klJ
zi%%RZE8hK{;W6BoWO92Tw)Og>R>Fs-q<lz;WZhAAT_<md>G0w!O8CYUsmU6gSKm~W
z(<iN7(%6j$sfLFZqIv-*b&AeOjrx~o_{M}Q3>KVefis*Z_e-aQzGVtpoDI&`AC5i*
zZ5b#rF?eIh?&Fct5J&xyxdl4XXq+%t!*<+u2~BI+gM=s+7#;$O#-H7O3*T3Xg*585
zXAW)uQnjw1>EcjNGtzyWP;{)k*ZXY;t5BP^nD-k8?dq^JkStuVJfR^WB}&e0Z$CD%
z%Kv@9!2g~3#n^N)C(pi!<ShZcI8WD`o^b8WB*pDEG~zR4Ol+Wx$4}ccd0_4~$13r5
zkz>Y@R<fYmy*Dic6TI!ebuGO6O7>~Xif5zTTtM@lK17&0=-h+p2TeSV#rL+r;@iC+
zQa;=`5B{U{LeFs8XL7apV2k%dBn5NiD*%xyhaD-_7wz39W%1iQfk2jy%7xXTRQ;1q
zHZzT^KRR;o?=5=YlZl3T2k6?8R?iRvIefhi=o2(hz0~U$Hd^%4D=BIFe5eq=-^ay>
z<`lRl;C*nF;#S%Cf~bUU48I>Q!GSI?0|?U7Gn@Z3qVZnQ{5~EA5Fct-f6(=M^}+8Y
zc!U;QMk5>qQ8u^{;&z5%XW!~cBNd1L^a8*Zf`||Y;NWcoB|Q4IJIOiVpT?<9<3>tF
zT)lN_z%YbN@*1CgeoKt>9dABjO3@xZ8GE&NwI#u>=K>(N?ZCcqn4z+ArHBA>`!`uy
z_!ADzDvgLog8BO6e62h#xy3C`sAO1~))qu)u~-sR05nrh48={XQDN->x<*kjb6N)+
zT&<$;=pgf<S;}MyI_`h{_O>#qG~Cqfmq@OEI%!RD(7_+fdR~Fh+ktrIg-5^?-7&c4
z`KAT(`)%sQ(=T_)*PT(?alV?7n>@W!JcGpQ+Bs6?7uEsa<Pn&AXc?y{zq#OffV3@9
z9}udQ)!mnK+S3PB+_9BniZacc8@hqpZR1dBkShT@o@Y2RzvxHuYpO~SC+XjQ4GJFi
zn)om#>*tv$r{MtyOZ3h~V|NP#tdt|x-d_a6A2)fbqu)oa>(k2d!ipi|m;EUpes-<)
z#EnUE$}=1Pte3z*nkDS3p4`)gbw6t17`Vb;#{pbEr=r!Eq1Ky3kY{AH)iL0bS?n8l
zZF2=9<10GdD<2A|uXu{YO$Y#+V`Cp<k#CJ&7$e`>KcUv`HCMjc@*Q+l`m-*lvZ`XS
zS*Q76Wx}~Aw5!9Ss<-3-0wAd*y&(aH5Sw+M7~6NZWj%dRu8hPt$=Tm{9P}1`yXRHH
zpynmpwN5I>wO#D2zOt3q<=HJ|&G%aM33GmpVs9(Gif8qcDGYrYTu{80AXm&YfhIy{
zY)~MGpGzpxVlcU!N4w5mB0h^(aV-&u%k*nJ3Q%SH(AA?#=jZIutzZbPUq&^kqgDtH
z#C3<y)qR3zCp%!GD09cRf(+ow^`v>Ueqn=;xZxUuyc1NLq+D3lC(t@al<F{xWL}4A
z&T_o3>PT&cYW>{MPp~trF+P&+n8n9s{tx`&1I1&(BgE>I1+W)!B-iyHbt}~Y84Dvb
zb|gT`t^Zh~7W|)Fp=@DfVPgIN;tJIBZNTII(<C!k=kZ7YjC%MDDLVc2JeQjH&6L~w
zdsbEa$R2xt?s8I`b&tW<fu5%8YEt_jiAd^wxp(i)0}6VjSFv-j-KhB?1ubpumEq}j
zAJ&<)y)lRTqrUNevg!Q2OKSX{i`LrESdhX<N6+v58@c_)vD@~tP3))E$*Zik{FJD>
zZco$5Z?-Qdz|K>toGqYa1Wl6BUv#ukO@OH0(M!xC<nICyB+zyJZ<Q#AgPk))_zP=H
zjuuvM$du1I`88>n=T(&IMK|3{p)5t63nLi|yhNk{DMi0}h($B9?}X1iJ??^O+Aex*
zbOpb;Uj_P!X~gg4{7!~-`s^-AHQ6GI$`_jT1$3LJei8LZFOP|oq?cT0MQCut>U11C
zv%>&dd>8!@g7_FOuhpfjbqT*p?4B46Ta~y^G`%YM<f{b36aRTlo%P<Zyin@*wkd;w
zlcZkAELx@gvOMttpcSS(`I6-wpe$3#)5z48Jhtp&p@rgmYpYRnHD_q9%Ei;3%I~30
z0p7{D7$~;&zk~!82Y2f8t?d5fWbP48m~zQ9%99?5_)KV4I&zX9&f3X0i*9e6lw2W}
zrq~19lwP0nzIoYx3vhgGTFs!}#OQjk5%9MNHAxU3{3Zk8-8aJ-7>B#GiYXJD&x`6q
zEFUbUD8EsOSQ!6t&K(GE{b+hKyRk*$U|eUm!nI(r;YI6eXT8K!LjDyqTp2JxY~AG8
zje=!x)_xPjvNMs|E?L|)e)c9*F<Uq#<+Y}fTOrfUy#2_*7jfY29Zu;ooUxy0LAnk+
zh0uOUv+bs8s_Eh=yLR3&j4x2;fhv&Sl-{9xHx^_RL{j=kek5>ouilGPXqGTxhUpPY
zO>sbd+~$A;z`44t^|h|-2^J>Qmt1+cxeT`TbA|Bfg{9&zHh0GE@sGGU%!{w*ZPj_L
zC?%3?nQPTFi8PHZ>|j5D{(l2(2aa}GIxcHkIKSk4u{EL4Ymq=L95<mJ0_DRg{h&v|
zFra+Nd30U*UoC>b$a(=5E#4&QzfRwdK%BOEu6^PC*u2ihTt|s0(dP4-mTQ?nHlxC7
z`L;FBVp(~aD8g2rAcM4e5FA|AI5{Tgw?-B1;)JCUZ?$Rnj`}0{*rSwZK^AT!>wcdx
zHodf)x}#3PsO{C*8*fEYx$xs@fz6*(>EH;0c;_0axm0a#K#Goj4-KR2tYhg&c|J`M
zYyW@s(b&f-eN2g`C>F0y^LCXNE-oxpD}HEB2j9y8#vW%6K>K?x0FVFYNAiQfliY5Y
z=VLgH8oR4lTvR<>ib+4NPgW1^oe!Y5?Ml7r=aV-c1kJoaD4uWyk~eKx5vI4V3`8$w
z5zQ6Ku!fl%pD@BB^yaJ3f(OkMWhHVcb&n2E2JHD{Kfn;X5ckczg?hexwBhjz<M|rM
zRQG!)^9Ue-%xMUyoF=keAfUJEDnaGvHBZ0!7W*M|#bTIK4l(B-fU@)K2^Z!R<Lv8y
z{7m!Y81#}GWH|C$$9Af@Ze}Ev!?4?~qvP=R(0%dY3&mC`p(7)UGnDb|1J%A^I8t?f
zfx~9r_E1GF4Wu6z_(vQnRk+K`ap>7<>MbAi82AS5fB%Lv-OdZOZZZ7sPwmw02w&88
zPCqOFmAxeu2!ar`0@}v;XSN7S8oF~)cuMsdWV;`-xdAbx*)gf2AqcC@;+KE^W6ny7
zE^2%BBIDn})RGbX<Lpx)<9niYi(%6y9{tMxB+0}xc|E)?Ij+MAg@nOUC^4_eHKJ;B
zwk2{70F(6O)5qrChAid)nF3U#Zo1bEQ{|&;^|Dj+2=R{&0$3cv7d|c3P}Cs*d!!Wz
zPkY<)JfPOw%00(!(z6m)3-+%m)4kOKs>#uTtUNhO&!72i#BN?<!N7AOv{`!Yv)<#m
zl{{2}6F;uZ14Wd;xOs9ZLDA(2t7~Iq`!TDeY2Odl2cpLq;F+5ECqnFgBS^h>Duzlt
zXtUWw&C(_R$?sNZ_D!w9Ei<bqx7Jkx>D4}=)iJ3K5}=L90^Ss|02{t@&2!P=2?*eR
zD~_{e^0WP+ZMjHn?1ih9<Pz3I^kGID?YrSIZFz>Wm62J7CLFaw1v62iQv(?k&l{qT
zamVoq&6`O-!<&}3`9LR(Y0<P@--}!ro-_z{GTu`+O@;I*<zmq>cRQN{a`Zo}jGq4i
zs$(QqWo~^3I4f|P$Q(lcGrH5ffwx2gniq(_|5d*E;aXORkc(eL2J`b4rx&wtA*h$q
zbOz*O(YX;Pp*H)QC-sGjR$B_~Zd_JX4(eoo;04jc2dzwLa|}vowH{&!A#HR{>Fd>8
z<d9d$5l4z|nFw<j9gAzLRhtDua(+Kt3{K~+YwO8*H^}*cyWMwg`<d$NunfZ=Z<VgO
zK>T6KD9dM^aW33u@W;uR*-3=~%;Tzy9sepM(BJ!rDRtWM=#Tt$e2v}RViZL*Aw{<n
zZM67N615oObuU!N^oMF(5;T4MN|f72LDX7gr`6hO@$UKy%X-Qf&|!ps^Ge<5VuKoL
z@r4t(hvzK?49CS?{Zt)3;a<3ud6!nAQOrQ!8T)!0OWmdja~pfnejY8_{#|fwwW!sw
zEZy}?LrG_DB_VOG1Qfjk1duk1xN`h1HIh7iowI8;$6N)CIA++6nL9g1tO);sPQkWE
z1{FopuBS!cfL_=izvGtE$HfPHqkS@cuT_A2GK??k<(rM5<a8h)oq6zcAJ^8!=k;eO
zc3w9bvX2UZKr?Ak;u6B%ejc}?iuutsY9)GQ3TiDYY9f3%SL3~&4RFc(XxAVz@&Eux
zk`wCqA2c78Mx-kMK_v<6sUlW}owa%RbyIN<5AB#$uxnZAi`Id-!jf?lJIfFbZ0q*8
z6{d_rufI~mwmJr9iQZY)u%u+{wNe(km4zmQzJk>xH_O@q8@zAbdgx7se=*Jjhr51Q
zC>L=I+1?T_&!~NSZ~N%PErnm@I(PL$4M31i4iPG4vMJ*acUm|hDFea<g=O)RowhaK
z#*=-0y^%l*^INH4>=U8*x!l7GqN#3``5O~&t;O3Pw1PVB<OvX+<sBs4`w}bn&Z`q#
zIzf(5;zjVnwwY2A=q<4WvXdd@D6k&}a&L101)3_c%@~kGVbQ#dBvFdG#ogXvq|l*4
z?_hSRw~kYO4~-B<^zwla1~Am;@Ae2IZ8$<^q9!PlXMvD>4|rkQ>Ur<(dd#c(FOGA5
zdw&!Z8+}igX2^xa_c^NOt2}9Dn~ssNq*ZC_$9+P^;Uvt}+CdWw!&%`r2gj43K15g!
zh={p!CeZYO9-#gjA^1^-Zgc;MvW*<+ntVt*IYSYDFYOXk=G?si)&3&+he1ADJv~01
z5;bn7Xs>Vq;J)fm+V!#?-{cHKs9M&ouk`md6QHTa{~o}f|B}Zws?^lhW~KXIPRSdc
zJ6oG!X_fcUNG(-7;Ar_$zoIJjvhdN)=LhvUzOE7yU=N?>sPQua2e^+$EnX@hE&(U>
zJEEQ{zQawv$sYL?Fod_Wc?F>LmF<OPZ$N)mnDvzPw3%g$;=&0srNcg~^j1=D7-8=G
zqIC0cPP<d{*8vX+&%FE}Nj`c|Q@=PxtcksWvSuCJ6ZG~xjxVvJRJB%vOPvRoS|KI2
z@r=ZUw{BKgZu%yS+|4!1=DkA0>~8;(=L+nP!)2mM9fzfsR{}bGq1r~cbDQ|cJrtCZ
z=dT0@)O_?ga?NKK1K=XSWWRF6!{6~}nm*h3T4?~B+r|9#{TjPu5X|KT<?&}axIrMu
zyhcKl9Uw_n7kOC#rrdKv;eOIc)cA2Yfa?B*wQTUZ<ZEnM#B9R@M^_XVS3eU25^7ww
z3PLF{&-@aCh>M-*9<J>mddI*o8OzS&YV<+2KvspWBhq#04qy2gZPnFN9IdTjnq{FJ
zH><s1t=aLiTXF;4Imq5V^hQlj#eMeFMyHR1WZYBe71`m`mdh3Dk687anrxcijt*oc
zZ97Ww&(rmMTr^NKo*lgLX683?#b7rd$5WlpXV7~dzs;AE7xG;3D|}}@t00szqrh5!
zo@e3%kYqXoP9fxhvnGlsXL15;e|jR-d|t)0mD?4+VZ_Z4y1DPJVwRDUoX(_tWpx(h
zhm)h-<@uIwwQRm`l8dK^a&Bk~ddj<Wm+{Dz<gx)7eSR=2Ws(PA^-cIY!9Wog+1d{h
zDr&)f;+~P?$A(xK&Q$H*#@Q=Qk^JrbO5#@^72_c0o5!xubk562i4u**>wKq7MLNh=
zqhe%`0rS2lsLG^;sBv7b7<jm6ZmWvA={wZJ)GF5<)t<?74DJcEFK37d6;6G7bze?{
zZ9b)j{B{&R#FJZ7HV<U^Wvj+9(u?f+1$n>Tzs`oYh*w>a7JgthK+xas4RcvrM9~!n
z%NoaiEvonx?jx4EWv$lT`NPt7^!{vOW2>g#H%=?Q)#5)+CaFiUe#@*+YmtIIJWYGL
zTM3$)PJH=e7G8I_4$RE`7h89Fr=bo60%>J+nbO*x`W|3@xI1wN`HuC%&DVCf#>?o9
zV?pfO)75xT-*j-bW!Y}{syRaG82zS!2oP};r*R1E(cj7k_63$b;RtrIIJ}(p+5~HR
zC!-+jk+f;b^xWknByP6#M*Wj#xB(ym+|1GiuP7VP6v%O<JxbyJhiL(>YPiVpTXY%Z
zKd}b)%co45oh@5I3f46{{mLi*WD7JTlJZArNzxQFBj#pOz%WntWpb_yNmR3QC7klq
z==$rZc*o6W)Y%~YjjP71?}duHVa`uMX)l=HP6j2-tV>?|phr}HqXmN1&A)ad4f39e
zi&~ck@qhQa6KZ^0j45V)PpgX?XtP6a(INX?H$680WS0J+1V=}0S#I~Hf&Cy@qUB48
zUnS+R5wJI6Q%c_w-@GMxQ2hEE-#?IeeTWR;rr%^pxqJJZ%>5`5#jO;%F3R!i;<C~2
zePnf|<RmTQgk&Rf>o+w{13r3yvQJ8X1C&v>nA`R224&QAGs#77PVwK$sDl99`J;R=
z+JS@&d$oqW8bdw}vGVT7?|)C9h@iKI>c$)+0IBF?BScZ7*OCP!>j6(Y0puPL&RiC(
zUc}9?-Kr0t(-oi?+!V`*kr&AqcNJyT5_`KrDg{N2z~ORJ_KAmXsRPXfH`4WrB`*<v
zAuBYN_;c&e_CWOBptl1Tev_a`G9j<WcCfnW_!oB6W12k*rz#*XDQU*#DF><h<FaIA
zPC9pceakc~IL|#KJdmzu@JlgvU=crY7sPQF%jUTljDg1PRi;<dD0swYwt3Rr+sqFt
z4U$CTrkvy!ux(*lYLOUF6Pnht==iIdEXMy=Gue{P^%CC~C?L4|XZ`Euk#mw)W?*{_
z2h#!<geSaL%oFoUCirZ0(zZU?n}<Ayz80dlrv=XXYLtd^dUJ_3Fs%o`K@b3@wdTK+
z{QSIn$`E3A`f8{syS&=Qa4QYP@c}F1QFgqZGy0vRj{P%S7W{nDG78XZiY|I5(bF^m
z03mISc&DHw*J2Fz{cW0d?1t~#sow*MMf+o!W%3Wl$fZ$kb(zVzz!i11J&#*%Kjk8^
zMz!yQk5D24gk0%yyPF;|6<>ShPfQ9(j6MO0(Qo!He{Z970>@B^7sqy7^gWN^Ptd6R
z3j0=P%iy>0ojdE#ygF<+s85rfYQF$PJO+d)8PIqA09{}XngTDXj~rvjc+|gf9$l(-
zwogmT7dv*#Qs44{C_GN2p#OnuUWrltL(nGqFls7&=2K*$ONARHWLMlX^X!Ell@gKG
z(FR-I;H<gQ-co7yjUde-o416=lg)-+HY}VZT+!R!FK>}7$thg;n(F^j)F~-+n8>b=
zRk>sAKRGWX;nVg7+xi~r*sv%9mVH10EW2|y>T^@T{T=jW7j~N8R-XCCx~Vsqrm{zt
zZ>9H8r*dI!SE}3Ywq*_OU3q$gSlV=PmNinI7zpi;eWW+q1sbY;ikCI&9(i>coS2Hv
z@=V_saUhnw6aiyo6<?~0#l~nOIiANdms~rJGHH8g9@Jy2O|_l6`0Pf3j^e$9B_NMM
zk=Tp4sDZ#m%QR<vd^kR)3igqVSDk1B>wch!`>mozbGPk-yix~0T3)SNiE)-(X;loU
zCpd=_Ne5lRh?Q(5%2ijQUNjuyreu&dqNF;!g-4QIxbg=@CX_Z$(k7M4`7hsUsD=?R
z%L+w10o?Rkow5GjNo{jnhE<tZqxU7nEYabVLElS#i%&9K?-57H*}W%I+D9qRriyT#
z(QE7f4{6^S*5tOWt1MKIWko<l!6i*aK)N6h8z3NHK{`Z1dXwH0QBjem^d6<Elu$z_
zHb83VAw&pD2|Y?e3naNSuCve1!hP>O&vSkeBPo1k&N1Hc7UQZZvcjUUC(aZ`YqTel
zYbY&UwoDLQZ6p;Fx$(?mMPPlI=yf-92a4-%nFPyXva#QU;Ij5L&%oC8;|rSd2<h7`
z{^dVzOvU!WGokQ@kCxn<;?<Xj)fKLO!p!q#kZw^wKY@7`Bj&T|H#pIc7PN-mu)1g&
zv!y<$kbX>6LWWt&7Jd8q$ZwZh|Bd8Tp133SZ-m&#u#eAvxHqF5CC26d$pO1fk9R)^
zmEa5<J%6P!9dHaTwVyq5&fDrjdiOe0Ebx0ZP{gq*Zf9Gm4I86j1-~R^Q63auZP5ti
zG^6z+Rr%z96kHGs$(rG**f<^H>4LBb$LPK$Cn%`6QyZzH4JR5^n*@I235)FKx-c1P
zVT75a5;gwXNGCYi$AgOj(vL~Z>!N}vfy0Yv%1n*TuqjR5928YvBPGxQ3}RsB#!)U8
zE$@>o($BR4cI%IW86Og*FX@C5o_Qn`4FM}ru24Useb1jLb1Hrb@U?+R=GXkQO1@$8
zGVlP_S6MZGsx{0i4B5s8M?A2KP%gj0__N_D;Vn7)SVJl!=7?H#Z$ZlaO)tK+fz7T;
znAL)@*Y`DI@p8d>hBNK5EANot{TD2P!L<nX?@5;s=VWFP4~*Xdri2#-<%~Xms|#+<
zhY7?i1+AR3AMZlNQHjOx>&P2YYP>^@x{>}YR|lT^hjF+Iq|M+<25Lb2<N~l@!`*}<
z;%abqZY#9gFcqw6n&~k8cAg3ZkzI87yleY9+|F7bn61N6L7RujZJn;D5$F82V>Si8
zOqL1eT1RW7%K~PFIbMh!0(yPnH)RGaADDFD$GwY21XT~KJ;OT0<VMvba4H(qbHe%4
zVk9EOs(jwe-O*2Z86t(TU1@dlasei}lQkXcrmoSQGAlKzNjUL>&Qf5kn?#h|xL2vn
zlpiix*dYp0N6po|AR)mx`{5KD3uDPV_02NHLQY(d08rzJb<kO{4$b2RwVuLd5Mce=
z8)OR<@%?a-k`}QIFKX!uX^c=6;0=p0ULkrDG8@W(!hIHUaL6W(rc5&t&|6hqSql0S
zDCak5^9i<aX5I^68qQ;8T82pl05}X3eev|)4ED82c1rmitQ<2q8Dd2jhC&C4%t(X6
z?4FgY_93PSEs$r>T8@QRB*v`3`&SwZF`x#A<%AcN2}{xo0U2!z+bxQ-A5lo$H4Fc#
zXhtjCgOtIQXYOBaVuP6;=)Gd3JMiU*j7pf8#$ml;6#<33A}xQ7-5C)oD$q*5hAwOf
z6afY~fqC->Shn&o_Ii$+3tAwTCZi<y^XFV@{NTN=6~axUeE)u=_tIlNSZ~j}G*elv
zK8m)ZYcm)$00{!Kb^)H56@V@P@x0HYy~5ggak2TBUe)WlbhR_!`wAj<wQjr5>2iIu
zge3z+!%<Kl6sb@VSuq$;h6c(P6<k1ARZf?>n#T!N!Bp}(yD5WA!XhY^e!V+@=&zOu
zv0Oa@2i~~z5A1ZjAtGH+Q>afg{Lbm9Ur^x&%C$^b;Y<F(+JREg(|ql_CrT#ExIRgY
ziT))M5z1~>CNTo%gnBK}_A4C!ZWOnLChhNL@4R(L?~?OqW=oj@(N?hqn|F(#I%dpM
zdb*kGsN9w4JfWJt{3aW~Z2FbQMP7dAN#ge-;ra*xz?^4`S0A&h|J-Mw&=v8QPj6Y`
zvGuoTt3hG?se;3D?T$(Ksj_TZAhg&%D4*)oS15_7iVpbPH3^b~>b@!_9a~scvfv&@
z7MLyfK(Y44&>~tb=BpjzsjJku3f5U<uUe9QF}aRhKq;o*3te!^n%~EAF2&eYL+^W0
zwY9p=ivf4N1fqY!B+x7{sR_4J1HjEZ_~GP6^fj_Fky7Vy8sVjnQDNKwRCba?mz<DX
zzP(3RONV1LoYpXYHHg>-2pcUi!>rAe$QTmmpxx50LUolIPG1$c&;@BWB4x(_F;pR9
z|G?vY0H^Y16o}+B$^5%u{;2|*_Zc?&mk+-+sI!zS=yv;&^u;3KdE>^sw~Ul<Tp=$^
z?CfBc_Rm~{uK73{o`!~B&S<M-S5f*`z)XYKYw7*_dSE$^+c`G(ak)HW(Ukn-p*YZ&
z__7J!W?2Rn4K6^>w?T_$&4Uf|n1!`O6eaaYh88HSuf}QFVj>;0mpA5_RW+n{=>`sH
z?MQcmP=4Gj@}(aV9P?`4r1g-Sr?JRC31EN>%rN}@z=7td%WqIa<9{pyI2)prXR1{`
zpP9M3>Bfg$|LOw+ztO`3BnexnYi?S1+;5^Lt1w35aeeC(sLd7JO-93-ZAj!b`;f_T
zm-PiemHzJpltLtK=Id1Oyd*jVMNl)qmgmxgmg5SM61@Np*&@LcU`stYqgT;T)iZ&e
z;!GHKxt-_T)lSrdxdKP#j#eY9T(3sWL4);58V_vQ1t>pfR(s<&qKo=-y}Rt2T9FIy
z)NijUGPpA^4PAqAMa9*<%81*?(WnCNsqrBvf)24hcyt2myRzmZZNnt&e@bXzMdDsO
z5PDyg+MT5AUI1(wN!ft?kw_&qA+arQf(MP^k&PVz=JM-SsT7|o+Tt>4SwRkg|0+}k
zer=_WI1`BKha+LVb92vCfkDPUaTyjmjK4XsBWqa5C4Oc($t;jd@p0g0wxt$#<(QsR
zUzUK-qa+p3RLe%{oO_5VxgR_E7@^_1U}#xL^;mb98~4)d&Z)H1qoh_4%sd*KLpm`V
z@Gu|q5#P!Wjl=Xx5{_g-bpDDZ&iKz78%_O`+SHs$+kv`q4=4TaX5zgr=cdFhIY82V
z5o7zoA>(@gPS}-u8775#V5`dP!@NuBIR(0M1Q1sONkMGPDpOK$HAAR?WN`{CGXI2Q
zeAx%9_-65(R@V!39eieEZQh|XpKR8+bYcSoA6@#X{DfM-Ze*o~LASdDbLh8)RS7={
zwmJy2$_gt|K!}qX^(#<j1t75*FsqhFbEF1|g@T(vw6tZlP22w^&MRNd<Cc}9xdP77
zl-8a`?nfHR7D$Rs**@uV2MK^j8ELb1MZ&IA-zmIkNyPYwI<X4Sq@(khjC%7zRB$vy
zZ}aOQPO;oGFk&VV^V6-IY+h=v-U4ovvoVrJ;1Odd%x!#`c{kX%39_6zEBy=H+YH$5
zEtk9NZJRF;RPf5+rT84x1?cS@uwyJ=rhawjPF>-dN$JU#NEw&uq*&Q(I6vvC@v{sf
z3BGmy4H}g`ejUzuv3C=B=8dbK{sa^*ZtiNG(LAtg-3-K-1X)3B!WMmB0a>r4e`$9+
z0UP6JqtH<4hhVY~2R|;W5N-m1r13=HZyTP~_3{C8=jEu(mR-I{kj8~u;suq_a3>Jo
zr9Es;O%YzdUEomG7^Z_biWMgUGVn=YMd$7{HmZ=v<U^bm0!~g_WTKBR(88-mw{_>8
z(#<I8uR8G#!_BXE_bBls(e<snti-LgfiS_hj?H$zrloJy{b@iO*tyYC(#X^X3cV(S
zs8UIwzuLkOlwe!-s8@8<+YtYjTI{eiRWEjbbWl4(t)?&!Oa`=QCKAF6?>;fGqc~e7
zq=Ysqc&^_(>NR`{^2SQ-DXidjd(^i5R}P+V#EBdq9<E=##TL%a>ALge3B8%Z<>z7=
z8Oj06=||sAW`|^xHS<e<*z>5Mb%;?*%c7ZE@vmyh(AX;*>p!L$UcR&Ii<@0N_(j^{
zwEzab)_*W4kx<^EEa2yxPJ#IC9L*qjQLdJ#wS=GizbyPN2akV1h3^n&x!2r9$B=`R
zK~f708#K2vs?znp|7mX`R9>Zyq7|<WE;RV0|L=dI+X9a^Ec?&<HB`BgipKx<KSLf*
z{!ZK%9TS)tb-GT?il7BO@(yyJ*@w2Ll;;=7(XAqi|DM_uC5@^VAmoN=`UF^TG32~{
z%W5LMBhG%R)vuzKr=8oy-0%kr=&)(ZX5~+#?CE9XPw1fih<?$dz9c{Uwhh8yQ~7B5
zW~N#fuDX0YhfyQ<V{tR{wbsK0ex1)O%{uGDJ5RT9qK=)dX1VX_J41`mwX2Pdf`oM=
zbJE-##ho2BL}_ds4RnWRRK<|y$~RiZJ5Rbd5Js`s@oLm?AQYMp7cObaTU}lr6wuS+
z)tZ!i(tt#^&|E8*8yoc*ns*}fz#A2<2F<1u?nLZ<_2<7UhaYpCYfm8b(dQwwv~ozZ
ziWv1MQPF(ErVyWn@o+1zgf!Pm{%@W_;RajacGt{H1*K{DEJAum^C108n`;m8TorCu
z$Yz8W&@|GDXflKUd<uW8u!YibuF9GwI33efw9nkxrecy4)$<s~>Ncy(G2Kj^o>Cd*
zaBu1nf|=KrBxl3s1zefRH={Un4?+*d0cf!q=f_ny)Mps}VssToi(haCkq5zrjmlJG
zIjtA&JbJ*V7Z`gnZI1Twn*@Z$G~2mk-&Iig9B@friWS8*IFEq&TlN$!+kX=p)#m{T
zK~UVuU(RF|gLefhj`KH(EaF-Z>)61Kz22l``G+;F&URtEF2q`u#*l){Ip<U+x`Y&(
zFx>d{=$63eFD#qv9@2>ttSX8Tnj=UGHDK&lG#Kdf2B4aeH2BTu?(qW#z4+VbrJf(w
zU#(K;8|*8|8Vnd|IpBGS^?DaArrB4+d}4pQ)<(2PkS&$OJ!Hd6+i2*a97t<GeNjwY
z{XQAP`X6uzBYSnFLGhpqVO&F=WffWAbhz=#p1c0sGusV{n4z!bAV#iwUXhAa4$n~h
zDI+Xb4v~$pZ8_hyOc;ieY?|%Vr<RjasA^QT2xPn4I1Ne6R5+<_ch8P6NepTD7}x>Z
zmrh(>rzjr~l9QFIteIC_D^gA&oYQ^idenS{^_pa36H>TgRLFY-(V89h<fen`ho0bE
z(OoXhz39@12tF6Po8EFO11+a{_2lwftm2yA1;9zu^^Ei*L75>Ho|-72X(AKfFhp*z
zl&Oi6aByfS(R}SUE6pg5;3*MPf$QcK3FyVD$qO~>?mwN@uQp7&3(&asymZu+kjx*$
z8p@&`i>1Vs7i-%j1lRv6$EP<<!iZ+tB^l#dtm^Qz1NT*~emZR{`lO=Cd^PsyX}FJ{
z`L*Ls4c(CON6-&1phbwWBPNkfSNhmG)6aMcc<2e7oRId7>e_Mg<^`gLEOwU3C3X_~
z0PIRKSXP8Z?e|>VQC&*J28>T--Mk<797WnNP^=ilSvcEOD7YWnrN=w0#*HOd#@Q1X
z%nP1Cye>0c|6(?!8n&W}c4i3EGMvYO_1n*EC6@EMhjYz2!`8tpavH(syp*^@Cmb&!
zp)WxDR#gOARAr<?Edq@vK2{!y6^l7jI6!kw6}GYk8?5^<9y^nb;wdm#C$3f6SeT#q
zu!Vj4!<x-D00(Q&8*$@9xWA7eC`}%zz~9b7*WeV@oBlutT;g+<Z%wCIR2-8NC(hJD
z4yxE^Vye5AYwOCozz8;{wUMQAb-VbU>$+(Y{{8SJ%f_MNvYT!zt+naTeX`;AdXorI
zh}$y5spW<33&{?$vNP?Zyw-9Odd3OXYLf)Az1T9e#6tKplsZ+j5qSl3I;tOjH^#R;
zFeTt3siHGnXtGgW7j(0;H2P(wbaT7V-02n*Dxcs>?TKZ-wFEx}IuE$u>^k56YrKgz
zC#rrxLFyb?L8&nIO`hwJ2%}hK{h6V}w6$ets08MZZs>`#O6EbR>2p)Z);d#;J5V#^
zmuHm~wED~e;c~iu+Irr;C2^&A=_fn>wg_$?%X#YLa45Z*2{Fe@BA!U0@rL>E@w-wl
zZ&1RGU4q16fyi=2P&mg-joe6+rxwEOW&%j(Fwt_2&?GcjeMD=}mI^^~%Wb%Bu6X;6
zVz{?9?B)l%_0YUd!-3}YlLHaTE!9!<n6H7F*2{~Vg-QIpdRf|-52Kp2jEp!BDDor#
zXh!Go%WqmXX42WsRUz>#*D0N2yJYw9K@Em&S=ss%3s<4mxoRV+hI#3Rqg8IsA>SJZ
zWhi{1=r&~fh$+(HW==x~y&!dIBXB^$HW5&=LE8EHP(UNHfh;Mq)upiN#4DjTsql(w
z6IbYVPwT=a$2HfB_)W^Hb(N@?#=HBD>7xk*Rh-fw0N(=}qpI^d`zV7iyOj}^a_V&e
zONokM$15Dgaj_g#J&t&BiwX|uQ{}RSWhYl3_D3AXx?*8eiG-by114S$DsTOF3dl>+
zsV(V8A>z9t*6*w`b!REWT_;p=pMlC*<U%cr<I~rK2XQ9^p{9)qW~PO8xws+mM4pQb
zh5Rp{Bww1IL32U84%-r5ItF7(H>Faalav$f8~zJ@7`0cmJA?UtG={(ZD1(X_6<)}2
z7XIKHE!4nrY~8NDQoU=qbFlY<ZBH+@>WKkE^KHVIco9VvMUngXR7_yq=u#ZUX{ivV
z=@96Quy4_3D4sV94=BqNH*c$J21Uq`==mN0Kug-PE}<uHDN0IOB>JdP57iX+5mY<c
zBQ5+kP$5FF@3Mgsk%Xr+b<VQ`BzQ&UD$saqJPB~4ZIo*oPI!$h;8&bs!-O|cgx}2i
z(;v9MSJ<EA8U&>Tjr9=ABx%@d!byJLh^xwdnq5+Oz;JYw{8D>#nLk7d-&}2+G@fjM
z8W;j!LpfTRYJtjy+lYz1jNsQqw4>mm2m)~+ksUv91t-LE_S!i=hdrRr7Iik7tU^Bz
zES41o4_gOAV@jp)Q(Y8B-)3;My~&($%cZ!m>Xa*j9`qWQIr5?LpiG%TTfE;iX?5-N
zoYycSsZkIU-i{Kin;ovlm@r?RXw^~vMfaMuTA3$qWpPur@c{VnE#@k=?Dfe@+oESn
zJ8jLWd~8~()@ki1kk&!+s0b0y``J+I;@qXRJTwLVa;{J$2$FEw7^t@1FjA(<uVe?h
zc!5B<jW>dL=3n~P{m;Uv8kh5r5HTK=pnfg>Ck2va6&!sIO2sObk$Thr*aw6baK5-x
zmwIx81+K5=*$!X$JoxC=NaLuArY{oQ*)AQkmTOI`%H=LyD0M|JQyjeFUUUH0ptxZ(
zFVQX-Q`C}G3)qQ~^Hu^hAzbpn`5zIapR=;)555XFR#>4QPd1OzF(JE^S5(D;4I%qQ
zo3F+w_iP5xg4Z6lT=e0K!B)L)T^-0l+M))vkq0(xZOkd?tfMU-D}zFI1}eS-vEOxw
zqL_Z6PcU5Lf|3xV5h>9wGF1G}rQb^bxD=awvPsP35HGFts14P#utoiU^RsNW97T$3
zEkzx%IkURlfSQi6r6L0sOT+0Vjprdzq}zD+?-QlN&rP@fFa0@USc~aM@y}Eb7N3J3
zitJZQ8am%jMg2GS6S{i5oBn(R{=tF9_fsK?R~H`Vcx~R%Wxmw4R1&z-4h;W)9;Mje
zJ<xJ_;Dw0%=FgeU&4PGdymcz(Mp5Vg`%xwqr<h2JEN|*15Q}9>^Gi!xP)(@hDKA6k
z^8fcE`$*9bicchhSe$75WkO>pVaG-cs|1`0-h2WyQBSoy)K4U3TA`0z*%MG7d!=p`
z8lx+ZAh4KP!u&A#SG^6j>!<$7@S)H71-XJ{;&=65wS%W@k)&?>!%Hw}Nu75^_09eb
zFHQW?&(OXZQ;sJ@61pqwCA>W66&&jUk_{A_HOK;CA6U)@FITXQZ&@I<UY)sNrqaPF
zBRN_Ta3Eh`ZQ85t8T~su#8JZx7|K}NxiAN_sc~|YdOzko*`DdsOr(J~%KR&c1xKqs
z5E{GpRJfcZRz2x)>du+RX4)D4L--J#KWP~+9y4+F>>Ix3h$W}XePTCy<-*z&<)$d*
z*w2?d7Ux!!$v6$7SZJ4CmdJRUy>~jkbbcrg@$%giC+ej)j<xvW;-GN;HF75jnHK+{
z+A8x7d0Tr{Zbjve$W;xv*HUcWSyzkEIB2#k>fW*0lTu16>D*J^X)$_ykhHj8b*QsI
zh8(6SERQ%>>;*VWeL4?;0*(6LSKfAxuP+|(;md}DA4$zAu8$O#DOR=|uIht~o0h%=
zVYCvP9d{pBx^*7a<8AY3Y_m~U-{lk|;|4rF4&lDosOS373!+IfGt=q$o_jk!t5V4#
z@JNrkahtT2>6Nn8UZv)08ggZIqj5BLE4JZ(Dzy6K$v?ska$0ZA&Q+`DNiH5mX6Ejr
zRc?EPI0OtP$_HhF8o(Y9Fp{30J=>BEtWG{3O(xnDd&RWXwSD>>4SM*1u~M8H&X-P)
z;$1gPL^mbetvZa}L;Zw2<DI6svTQRDS)x|}&aOJLHP5_YDTwNUH08}mqpWuI;mDhA
zG`)bwpm#iAh-1Y__Hca8-=>=0@;TLf_<NTiHSG%Uoo)W;ZD%I$IJ}_xsuetb9zuZ|
zK;{}c3q#z~?Fr(v4X+KW%?&?zp8nRu&C=D^t8G8%;x4^?8_RPn57Ym8@6O~xLjS#v
zhk~gIHe{I3qD{KA>0)gsl<Ve5NERk!-YbJ1_PC}V<knf|0JX=T<$lNXcn?l3EvBk&
z(2ELiycuNNq;pCIaoy!uZ6_Tu`*lO$uJ&)48Jh2vMm)Cl5YxP>>f8iuj&U=a?m3jq
zpbWDQulZj;h5u6^hBU4Lg@|EYZO(vA@5^V<==<Og&m~WLrQ<ECqDrH~fum99?@uk<
zR*Y`7dDaL^FUB{6Vw>${xN=I=|LD8nIe=zBYpYYktF&o<;f278Bk4A%!jR~QkP+W|
zBbtnl>(b;tWx<9NM9g?#h0`rdd}1YLp%jUaF2qmA<mHAQNK=>zsS_+3E^Lx?MvPlS
ztB#pg^)N`Y7DV5E&4;dsOjlSOt$rM2vpU||T(&bCZRZ>ZCrRT=4bBt@(KNQh1`S@z
z&=qFSCi>iyH(lFErv&#O2`Md9TnoRH)|b1IiuHH1HqHK~n&5Rp%DRVq!<o`lHasT2
zJL<+;O9r@&w~Q$B*z2dee^}!_jV#p^k;-}MCbQwf``16038RAD;-VHx=aVH7csoi=
zOX}@tP*;X*f`8t#2|9D$uAQhzWrV<fTA<60`W$W->`TPBmX)d{9xI>p5}+}7(f(FQ
zskp6G1It?IFTo}<G&<`|w)^i=^nK9&p`ard>7cqPnxQOD$GCzT?p#v#ZHq`B++b;l
z8r23ALo!{H4m^A|mM9+=XbkgrT_e4e&6=7_l3b7BvDpw%@(7GuL>qa4a?HGCWqF$5
z?A39y&d*|<`D#If|JjTFC!gJ**wRrEWb=~wk#jQOll~_!`X7E_o<zp;>EA{wYhuGj
ze~?L->O#f!O>;$*)2b#s?g7|1j=U}vb?!0J!VAPzx(H}#GBp2yIrH{v-s?3ZCKMW&
zA5)HQO-+oNJd9K?)rTKAWqyyh{bkY#8=CY|!MP`gGw%pIOvqqa;p-FM2b#Xx@rZ9v
zPxI3F%`VcI+^S_uBCRN7&Zf3%rJ&m93{5CyQysQ&6mzm@U3Ka5t(a?r9mA&9`s!s)
z*f-5HJh{utc!B)Q;pdjCrz8U>RX<^lIHG38sRuc?id2)hyURXC6q}$7{xHP(;K~ed
z>7J^KqC4w{?cXfiM?UPDz1K!L`m!z7w#WOt1Za8El1_ZX`Zv5ggwZ?Wl{w6R5eT}y
zq9z7J4{l9EQu=GGDTP<~v~xLTa=s9jFn*$|l)^N`DUH@Nm&e84fo5=5$>pb=MrR`i
zonbTQ$0t;i$d9_x^wGTr5%slK9&Q~4H?dcA|A6e7%+ABMVZ)wlA>UOA{4BfXyqxHg
z4Zr&b`s0V0=IU1onVN2c<uOcOc8V_zK=<E~t5IF+^JP185^rE@kO^yj9CYDb5K%3b
zfS^$GgD1J%^DoJNj*b6O6v*$bTU_bCap)rBqc-vfcbF^fjqT>3{H%?K{uS$u^m8*J
zP8~DN(>Ui;1w}%ZmkfMG^))#!iPTAc*(UZuF;;5h$-ns${-E@DaB|n`!zOLc01)nc
zCP}cUv$TxjE`Orj#OSeN`EApMVftzG(qc2Zk-N_i+h0In_yg|#_6#LOIBj_SbcWJL
z<0Qa_-qJ1Y;qbqk_1fV>9=yT8_2*8sbb%!k=E+B7y--Frz*d|4=S)0?r=jPj6O<bu
z`2xAv_-@DXX#G{6z!eqjNvJ76ctrJ9cCf(4o&e#QLSjCDi@$Q2-WKn**UX{Cb9X*E
z`Gd3+IpR5>7@KLLv$PLr#$g<n<`rBkQdfVC^y)!}|4B{~7;SDTS7IOVI_W%BmAX1<
zAo=#LV?o$qY2(5cC2jax5rw-+rX&)8aDBzvfB0+;6f>thMP#JBe_n<-)2W1Y&Pd}3
zo?THD={9_;E@@fu0AI^#6-?nPO?+Uu1D5pywH|%p<+g3=e%e<q8ivr~S1uwQWFHND
zwN#Why)p01{HAx+3LbRHIeH7SE8Wd}nw=rF#QwW=k9r$iNgCpq*n1_QG&a~)!j~jH
z;Y7=U+wDYW#geA&hg1PW^(HotYO&xaAFz_Dc&*R;yym}1uL<tjPSDEBti+LRp~eTl
zL}28%<qoPZJv7WdWl+gu4sSoTX<Hwu)FdJV5Xm*Q0>nq>2?@}(ojB{RA8kmcH}(YF
zl_RM>#WeX*=kZBnGrR{~@(+~#!BSe7AN;~ufx0JQW7t6bnp10&BtCL8Iix20k*zaM
zimGJf-YU&?uZt_yg^VGeHJ`f2J7$Z`QT7~F+YRP^1?!TzSHJRgdM&thHydBs^-q2|
zQsf0)7JWuhVJ{<BUAfY~-8JwnJ^k=*nWa8R`uq1gVHY9Wdw!fqy{E7FGAcFh=iy)Y
zk)V&#C87rdn;7tL!7Our4+|&(_(BQ>_AR}g0XiHtQY5XAHXW#t<@$g)p<M~?+YRsJ
zgqeO@m}`+^j}cQdPKmQYI{xkK7v0qY_hqJRavA(CHF)(kGw5|9Z#M3AEh_zD0cSQP
z?wd~LfNy2uM4I(9lsE9l`Q8|LIh%4{W$_cCVlltY7t{Ft0x@`{V|{+))>r`KxA$Gt
zyf7D^dae9z3QFbXjvQ|d(;gneWyHLR?`4J{@8-MYspMTDvnA<qI}yquO$tRngKuzV
zi2n#)xwm}#Pb`$!?3l*Bm05h7JzkmIxgPuI>fQ}`DOzH(5>L{(gc)=RNb7OGdt&4Y
z?TvaM(1y{oTHm2Uy*&ShWqN8Lsal<6wFS#D@ejV+ulR2N_kDU_h;1_SE0i|hT=}&m
z>1e81xdhdVkLsMUQL+PpCu9)*$lf0P@R**Nm&9iW&kuLi?THg*pIUc#OO17S&H0O>
zxrpmdXg*_ro=1e1ooBhOum`{F<6XB?rHOu3nxke{j~sbD_uTK=O5N+7HIlV6`yexS
z2cP6|I~t6hg{H*qiKuW8v@dr2I|Cr|%`5VWs8DjSZjkZ1|HFmTvGcoFE(2cn`2BrH
zL>Ung3A1GEX0Y%)d+JV-I>r6WXN?XZKTP?8HihIWuJGt7(Im+{dV4`n^j`tKG-Wx@
zOp?Tp;Pwto7&+UPHR@#l+VcyR{^ETHFV{NnOM6#3!kKgQ5~KcpOmJe8zAXp(9pcs7
z*BSszjeDQh(fS%hO%MU%XVAUQcUpCz!}yB9w`|tH%Mp%q{PA8+f0a&sH>wL!?lJmf
z2GLdcIxNdnLw%*;7+U+g65)%*)0G*VQx$2UOFQi22J*XMZ`-vjhZV}Lx`vOz&gn3N
zOm+g8BVqD~ItVi=GB@$6zikS8LO0)Nsd&4q9oe`YGg)HN{@Se^07v@+J};eLDi_ma
zYDZ(7(X*wS(Zt8A?zbz>6s7?1bES@cdd`C~I9>`}Hqj)8y-1<(!kpEoJW54B_{v=O
zWdU=5S%j6#veD(fkDp)zmMafMs9)_06sbyfF9+(DxF7MSDGZq(697o?+@-+#>-k<c
zH~a^tN>`1>FA^8NK7Yzjqo|^T;4=080Ji++JA*-hW9c`a%P6y8|LrHz(%q5>G6l%C
z{;DxWs`2q3=5zVh{BDJP-pA!rj$eIB&EKO}tl(Jmg?h;z`f)iICfm=~5Nw|wO`F-s
zVPu5~0lR9lfQSd|k?VdFU)E~;2%FU{DK1je%VzGj8k0GzJCCPO_hY)QcQu*i!<Dhy
zF+o4lM1F=joAv8_8Z=km>@}IRoQvk{BzfrjGc<Uo_A5{$Vb+@yOIZW<%`>a^0xdf5
zVs34|#+`PKXTCqbRMXhOz11BZDKfSrsgT!fI521;8?KqiC$6T1{f3oKaNGmC80R2+
zAX!m-S9^fpFW8uA1cp(1@mr|p%lGzeI|p{)A4QGdj<l{+wz8!v2~Jr{sE-2oCgqM2
z(A5U+T~VAR&8J6h1ZwaD4Tcq=SW9j6I#FckJ7*T_)J%!vBs5WQwVSr;30)L1t8B+=
zEm^yTpC%#bKZ1-P<#vC9odaORV2}ITshg?Ib8lTdEd=1bEv$}xY>po5H_Gi1y0Vg@
z-R}$(H|F#<?JJLLo^C|;q0^yjlGIZd#Okvgr@R~Jg~d1iDAwi%9;+OG11r$beDkYY
zvq%=Nyl<`Cyq3!Mih2M=z4y40nqp(Uq&InJ14%KsJ3KAT0V~@F?7v&B`8OZtaow4k
zNLN-i>8h&(L24H4PcUT59hvUOe}`+fkEG9?ioqCmCtbPS#Z?uIb%XCVi`6;}S$PfF
z%pAnrp=Hm=#-zk?6DRKIxi1KULwE`?V*VY7iUHQZ9y`L)n}2<LLGMk-gjUa>LBY(a
zoTYY`z!L@ZzLH4;&Z@7Vi98{5AUQq#P{GYGL|N&<$dp5}Hc&8Z?q7CnPCr^OG2Zas
zBC-L{$sJNxVVw{}Ux3L<7G1HOdnix5qQhGxs@Y#`5sq?k?tM(><k-yYHQ>cO<^$`G
zeP=5t*nrvp#m*=1`E4NLcjA?JM7oK2Gf%85E<bSZX+^#`gK$Lv%AKd0$}5&IBT|4T
z&Gh6`Y(?lz^{>mai-E3PA8L&X?jU9>>pwGog!H1VL*!sjPpD$AJ_H_(fvjIO>cQad
zfQA>1Hxpb&doQ&$&N{d%SVf?YTFtF<WmBFaIF8Qb??1BfTr6q5J-ztm`QCd2?J-El
zf33o~+iB9c`aRw2joUC)zhCs>4}`q0DoxEWlpPb@2g}j&Q$c<_&Oy=UZqsvV;H14%
z_g$9LUDZG-_dLx@>8mwdV7X|yc+`WMVJj}oOkd8e(xIUJ_cO1hRH8WaYzp?^jGxXf
z_2x_Id&$whiQo0_LBB?v&i-cC!_?A1fjURQ^=~IQiKA2NKLUUk+$}v3rsS1%m0#fa
zyyH!*aPj#YS=;f1B%Zmop*(+T#8uo_m558!y*C<QH#QCGXl=goT`s9{GtPPVI?yH_
z&8YI4C5Lwj#986(L>PX|tl)C+=3NJKSosSy4*%DQh0g<j#vPNnk%(8NdY3o$-+rS+
z!X;TvLobHwj__K&L;nTwO1AD1@$&qr;P^Z-=2!Nv^x!%KaOzo#r)_NL=m!VY%yI_i
zV_@Z{-E_0gD3eL1xz!Fly6}tp!$0Q5w@GZCh|;!|?@Ukk)=SHoUiidR{<Qn(KXeUp
z8fO7~*K*mNtlp^ZcrkkU8b{IaP1~9z<R0Zg&Kjo~!sBh|wo4$QSzbL`r~P&JJ<X1E
z7aem);oF9o#^E0$&SrS>=FMZWShuMFUP4KL`t@M|njg3DW$^<$i+I2n->8g<P?R+J
zh&~*Lu<ASO9OM$0Mvv;`&GwrT59t`{zL%9gCGJ)G#&tr$@k!<?ATsx?#|JcuzvtQn
znC+G-N1Cl5%*-UY<p$L;{I<&;ll5t|h~gNQ!vxYP2gdi@P>cqCZw;pP`Rypv^=bII
z{PTsk3exi*<Ea%3UE5ToFBqA<jWlXJ20PZJTOcwd)zsR>GI{|Xa3dzt&~JTCSA1Lv
zr=$y#`(x<*LYpd*hfaH0^~ie#HY+$XBO92h$PO8#0+Kh^?-IY~5BMC)=tKeEJ(L1j
zn)!)?FpbMWsVSnt;EvC|h|!w+0Fo2)Y0>aJ^y>QVjSLTJgJZf>A0ps{WIc1WxAOQl
zmdrv(zRWo-cpiRRgzu(?(pSg!xe~(_?9~Zqn-gZcXf&NCIr+Wx#gnzu3ULBr_4iH@
zlT`M1Y3F6;yQH_2o-tH@piHZPi8-uK?>C5vBX`&*ef~Z-<~Ozea+?9_<(>1KFBVxo
zY8527F6YAnpj>D7NG421aJkk91utHPubFZ|r?onPM87jV(?>7u7eD-$jFzeWGI+Q(
z5>%aiDQ=HWQ3Avj9)H`MA8}CJtxY@rVD`{aW4vbyJ%xYMv)RWS(sUjS+LP8>nS<9B
zW*R6c!{*3=6jg%pf9!A$bkk}S#Rola>S@Q0-MZHJ()bl>xRtYd{wAV7oSu+-_O{rM
zIg0RyKfk_H<)ecwb*d&=Udcs&Z$Ejo801Qfd$p+w$zfJ0exV3}=xp)Cj^c2*aj>ew
zq^EYF{N9jRdzt^KpU<h3k9iuC0>RWz?zbQ}#D&`(0`-!V)fQ7xVro@Sy0G;<>xVOt
zr=Vt_Bs!(<%j(Rvi)~UO(?`Uy$4JmhB1UV%^IEa3pH8tTg4#4xzl(|l=WWuW2oY|r
z)n`kW73U6piOmTIN>!%%H9|^rCRq9X@{WUBs}X6DlANA+&fdgWJE~>;=6LH{nJRH8
zeb%1y%VaMSC6RlDacvhcNY1}=WZkSH95mCnZ9gF$g+bj60>TwL$lu7p(1-a<noJ)D
zZad4Ub%Y~cI&A8znRZSt`Y};lxOZ5irgD@0=8=lv$&|{QzP(X&+qV)%DV&>%%W}6?
z!Qs_$&n8Y>1J>JIV6P<<dS%|hrw**K+)yyd?a~(_mLJMD91{&+s=KM9$658g?~xM@
z5qDMj)#oW+YMSt-BIVYk?666JIv*-MF3Kx*O2&)&82|~%OuKRJqwu{0ke#+NWSq~8
z{c0$?ffBfCw}@(E9~j|Ab?l1z6PtA2i7}oUkIPuSEY)Xxl=AYpDSUf01I$<QABzdt
zPu;Wem{cfQSoRV19v#XiH-G1w2K0vWmhmiKn$8yL&7CP^(04*FBCg&fg2y&&exFQ4
z4E=|$^|M&h+bT2>&OL-~c`vgVKeYmFXuHe6&-1jWT!F-G)0;uWRw5R#xp6cXLN4Q%
z5*!$j|1;NE>BJcHr*?#n^wnFMdUNfY=UsecUZSJpHH&5g0c^#_WG)VNy*UDQuiimW
z8|gGbNqpc4|5Ra^J%z*^|FGiaOJVic_$##jg7#c@`ZZmy-s(F&aBV|I=%dZEPhF0{
zS8^|{?-T0xr+$*Qzf&OSWStF?8P~HN^$e#dk<P<R-}GJJt@;0M(Ey^fg_EMjF>%}M
zh>rfN+_ti_*gmJ+;?SuQ%FW<YS99kLodJ}D6OrZlb5n(OC^r&bDFtezYwS;~`hI2m
zPS=;*&D@tk++b5BTza_Gz*mXQF*Np;!`%-?Pflt%wRf7mdG`Xfbk&?_ut1Rx?l|>F
zC?ShU%#%nRyke^#-6+<1gG|2xR5L<qnAHKl+A!p^%&j4K7q}{@%Z=$5%x4D{TKcn~
z?qeU2S8mdb@nr$e3Y_V?AT)+5+Vmr|;&~puJ}<Z^U_im~?-F@^zKtproPqBh)ldgC
z-YtsrU7<@4v}!M6qXMl;T5|$Ny~m65V!rFr4;sOp+e38r)4nB@=hsnFL;~cC*3-nL
zygVuC={Yk+%zangCi55j+2A+!^V17f`3@H|pBQ$cJ(mir<Na^N7M+=XXONBku7Eeo
zgMh%kTc2fTZub_p^2l3Go$*q!eDzKpS(~SKlT5$ctxOL7;o}AkCNfL#Ew&4qch+yf
zB-Im4+S82O-r3*cm3fDKx9R4E*hl?@$+|9fIk5CjdSnrm-?jVRc1(KwPQC+nxJsu1
z2cB%^Z`rR!F|*pya{9N{KuZQ!=5}u(U40;q2N#I;18)}TS2bT~{i|!OG1h%UmvD{s
zCB3?)F^M8+?KlH?`9o;m!;~}NsL(vWD(>ylw)uHqUp2<Z&B9I}S^VfJHRbOk<cM-S
zpV$+Sf@!bBdYNDHT*n0zt3Of%FgSbS2rCT!>_P)>MQ_#VkXyUb_exRfgy)Fj;(K)}
z#IsETp)|p_Ec4D~1lK%1^b{bym3KD!l=+tx%I7`s%{KU_%xHi+8<rVbmX;QPdH2uu
z^*%RS?aZQkz0HX29%I{)!c}!_mD|LKCVWl%4>?80E~Wt1f8tx2nW3G&QbTe$e@L!S
z6WXcoFyfdvkv{|Om$|QB0Ypo4Bw4al*Yz(hSYDVy4(8mQKy*BhXzR|zQ(r&vRp^24
zKP@8dbnA97Aq6Tb+w!WlBSqPALUA@#UaMmllO?8l;IJZQUydItoE>lakivSrc|teG
zE??cId@A03u=0L{YvsJUSxN0Y3`H@RAFCR~h)ia;x{Wro|Cws>3+0Rjp-N1~)hsz+
zLX~iEYzLFtzon`YX7dTH32ypwxj<>I*iIniZN1spq|USGQeQ108X*SZb9OjuCh==j
z;rM1bgqj>jK*UY0?T!G4bcz&-8NEma2sg)>I+nuH|G;+X64#J&Hv!s;#+)yHD6i@e
zEhKH$t!lw03k~=qr0+_)-v=XIk!jN;e0M*ox4!^G_?6<If?6=W0w*xD9G36moj=nh
z7c3{h?)sba$<O9tr2&*<aywRSAuftbo|+WLpOVb8nF`yT3rLLz^`5_<R#w&mw(=K=
zVV%?D`MLG;u$2X7(Z>1v-sc~cm)~k=MbCZM*6v>PR&Ouv2)K&ltfmP=;RiMaDK>8-
zQMdCPVwIJH^`n_LM^8ZRqpqhjz=hBr4{kuwt8f<3%>2L?u8NX#t$s6tcbA;aW1Zfe
z^fevGEem;mOSxiw4iOEtzTENM3)`&;-|kw^XB67={g%bvLERBIU%F~x&x(?aX>%%9
zSjTtW0gJt^9pP<nWKB(WXhyiRp=DFC^<BY;_EM|QDdXZ%4L{2L;y;2>@QHTVe-H0K
z%;=@}2BtHMYEssQRtIhL`-k$ihMiLA3Y=_c>E$>Lxo<mwb3yE`BXGU6<HQlBkRLb$
zfd0TQu#c7MBEfm4wcNX)m@yZvw$U5m>S>D}y`xhcGQk))M~%?=iw!tn#{I%U_)Df7
z=@744%XUqRzMcOChlsOf6W5FBQVyV<-*U}~vpFW@?Wk5%OEE&|OGuC@vyo}MUdqL1
zhP{Hj-UE`i#vbTw{*x%C%wGL1P>)vpv{paI^qS%JhP`@UuE$6p*-m5mi;n|mXveUb
z37ox^Yu>lkzp|;lCL_<0>Y@Zgi%ZkeE<A>^Ke)c1bjYhWvb5WDC_9c7(Gy5L^q7L|
z>WtI~eA`H}>ibNwO>55Bn@M<XRWYd)f6X^9u#!m!;A%(AP8gHpikn>kZnuRrb1Kw(
z6~VjLHasuk0<||BVNo!C+Gky@yEngonyEstS@;F-Fk{}a75rxEz=}kRXandijpeSt
zb&8!FIca23=czJYSKDN#jjIWm&7_S{XlC>)bb0g-3A1S*Yg$KaBj!;wacML;5R&TF
zF@`mvsH7Cl6xG<cdSx~Oi@bH4OeMP(Dvl}S0aLgwF(}7hQA`8aUWtV-#ku(Wyn&a$
z-Zc4)66>m0-2>?{Nfy1Xt1JoAIPC=@05OL;(>6S>b!qox#YQ<S^(rfCbf4e#v*rqM
zojcr_^lA=x-AH+W$uhg}c@}Cch(M<#(duz|I+n;B9$nYC>_q2}Z$gawWl)r-(1a+3
zxKQ?x>VV|sQWsD#R16h{b0Hl^m%Zm!t(l98hJ&8PYcCQxQ1`bX`Cj%!WO&|(wYLRL
zNp6ElIp)U-xm^V}4(h_!5}GVGXMa3`6{fx9)l;nPnCbX_J<cBK7`5{Cpo%-F)0b7;
zy``mJ{uc485So6T{840&ohfn$LP*myQ%^B5Dd!;VoBdsvY5T-M^8SxMd%Q)3lF#OI
zakt-AOhh!~b@oyQJ-P$34$60`B2RN()5nk{zx}6WY9CubXrk3UFVj!E_Z8xj@Xn%X
z#p!rj;xxLXFnKp2I_18tap23MPIw}EQf9c3`<D2j&CDj?WoeNZr(9EVHm)PzsBEUK
z#&W)5;fF*NGGI9w!7r|Iiuk3&(2F0|f(?H;``dgmS^v1StRioW;1{R6iQf~8R$jwT
zu%Vw$g!z>9#$-H_1chba)WjV?AZLPP{6P3QhC{)-79Y@AdcpnG0-^QSbsx1UHUztf
z^vc_{NRB6E8%Tj$6F^Via<A_c*)cOrqJd;WBi1u`JQYO=y76Hgqu6kNVXt@=B<A%g
zc=GbX5Bz|^6+hb)@vWS{MXlpfSJYIH-z};}o{uc`5Izt3Mhqm9kW1!$hmC{8^UO&;
z?W@KLmCIID!P%pXb3E_{VhZUWc_#Q;9LR9Tk5@Hd-Gj7CLIn7au~hOfwNU?YTAY9z
zRf)6yJX#YJ4f41(be$Nck)gQ+13f@jD<BRn(ga%DbpRT0xiy;Q)r(i}IPWQ4%H4r~
zH`)+hgi45^lRA-o?UA!O@Z|>*LaD*-dYNhrxhtEWWsFiai7D@&|NU-=?lHg1puZv=
z6lS#r9b{jM>CO+kOl6dPvEmgy{pPjXO3TpKaL`vADOGnV+B0XLsL>ZL@8#K+p8j_-
zv$wi)ruz(Ow-6{Ew>W`3y`cAMGUrPy|JzRmuJ4-iOu7n8s*p!w<LH-?s-B4@cAnfY
z+Vd7KG->=YQqG~)aZyjcGB8d09Ao@*a0!vG9%Jk^wRUk|To?S@_eqU|Tiqn8{<`iy
zgg%OKO5o0#5xR5q5EvnCPi;@@p76T*xBkE@zD46TnK+cGh9QkIl-mrnUfu~h!V}7J
z3BeQJmq*m7ZKDu`$?1$La?qk>R}^*O8b3q1A$`(sahe>607DUoUoo(^_9D?=D{-QB
ziJW6O2QazNrCsZuSW=k_Jp@5*_Ed}4^%&7or$&6VpV|f~J!#$_6CzyX^Twt+z2*Ap
z_3j$0T|HkNRDfP(Z2`(%5Sf>jg|D<zHrODB2e^e7E070N{gD5G>YK~1e)GupZ2DM|
zPP=pt_TCNt^n+TBGR8Gm-WA0&3qeuId&=?mvX^horyB#~)(-8rMHX*S3b@0x`BcTf
zi;U@dNB@3%QX{<!1+Y-v7p!*^Ahw$?Eg?tKi8-llj1_lAo217}tHgJ?PfzHuh0*UG
z*oRK-9%7CsVG@B8>1V=4<pW;*+Vy~^2%8GA+_~AF#IzCti|s$9!PR?0F%@f4F#-wV
zs>7ar;R^YEI<gDoTZ7>W31I(u7kV!m<bG3_m5Z}}*ROR8I1gl#C6_J7H;AbtC4B#`
zTL7Ez(E~c$;vuv3k+YI!6$yzq6AhN*P1+Z4M$a5->VX~t{3Sf~lw>wE!gJ19F+J{M
z{n=BtQxpYkMUT@c0iwbuFcpw@mZDOH{7^3W+wQi9^W65=uLBN@Zde%CwZkMpkA5lF
z{yVjfTyCdnP6kWrz1Jgh9yF#^FfE2zWrghJdDO-AHZy$A9z(8Mf%Fn;fW>{_7Eb8Z
z_s}_s%YD{Ypxc+N^+wAsoN8?FoUyb9DnKCj{7Y_L7%sp5<;tm)M0x)+8`nJahU&yp
zy9?&Wl{JVl*Sgn%XB%SNh(uTQ`}`j`=(?Qola;)-j=HN@{fOz+5W_B|sR#(qrJ9O%
z4fvW`wotgsW}a6lfLgq!Cj5cP-gZFQw6`FxP=FEC?;bwSFIYXxJCH!Q?yQZH{Zet=
z@4~Zi_xGLAfCG498)WW;lbtv22&L3gx@9swIl}*Q873kCFzVLA-<v_~R(8*kKL(tp
zbY}Mt5?dp`2YA-wE%MPEh3gt_#tC53zGcKJ$leR*q1-LUto(KS9LjpWe9yOiO+|+{
zT!d71v=~K(p#Iu?UCm>ISh$=&HuM}=-#>;Nssg_jGJWW%xy%ag$&aVsnZLF#<Drd2
zvpRmidTzCk#fD4F&NA>SN!lu>i%c2D7wu<;Wh^=O_5{_HEmV0~Z^k&C<;J%>o5F$j
zL}@@0nI9c8@ALL4J3jXuj-Lf~YUf-tSe-!*wA&#TdP{DYSWlz7Ww`OVce6`A*PlY~
z5y`hCb;YkjMJZXzK4$L?79Wr|hryz~iHM)MahMY&tWv=u@7pwEqaT=9;^CLRX4{9o
zLMcu}J78-MdPX4WMrk%l_m0uNVdv~e4_)j_t-0H1N9kw{YxSh<j^riy6+4naNQelS
zZ1L&Etj&)c9MMpRJzDrJCsyZ@<Va?I*rXOMBG|OVG1x5%-AC!MHuoY=TDg_vGL}j%
zTi2Gj`KOXF%!<*ix%R&P)60E%*6r!_Zu-#!<k8U(u^u2<5r)(OC^8HIcE*23-MDGK
zlDyvb{$roxdiY+o<*Z4I5UMEhnlA2V;|D)A+eoJ#4^T(F{0vs2&UEkHI?r{num1vM
z4&WHZBy;>SWBd<yUnj@O?SrvU+XJbyjrLGM^8NIu2)2nfX^QmG`e#oGC|F@e&!Abt
zWcu0s0>LL1eVWVwvFUV7F+ZX~>xe2=^SAL)?1jMHANgO_a}0dcGuPHX;|gOF5g+&-
z$4!AA{(zje(}m!a=9ryE>J0&9ar@SQ)pG#iWmsjz-N9!J$+HaPHZ|E?7v6@M4d2w)
zGm1!bDUie7!qL-hbfqhm{Xo8q$K>|vG7du=j4FQBz1e`|-)whGv(cffQ*A28(7*Fv
zqXV*IA+anzg^0zF)o7OA08L_xLxcnyqxX58dPQP;fOlY^p3<C8gaJ~Nfll0+02Yg^
zIOMX;#i&Z=VhM;h14})lObSvlS$KP<%Q*T(;KAb~EG@^P#;b|e-oB$g8A~aPgAvkE
zx70;(6K@u3B+>D!H=cuY)Y>CBW1qOxlXF#0+`gmL6u`TC!5O+W1svw3m=Z9J<G4L6
zH-jEW4tr`<%-Ux&pQ(w|_A(jf6joHcv(P}ZQ3n0|UVV5ZsF>#u`2iFYzx?&I3e=yB
z<H=M+8p0lRX_rVI%wz4|#5o?bcb6U9$5oYly`7}g+<stAst>I#Ts9rByzDoI{SqmW
zVlyFtxqPPOaLhBhSkIInpd$WoF0@PPb`*Bwy^&TLsgJi%RFI2s0n9XCM1&-m#Ql>V
z!pHqgsYhzA89#YWb;D%<^vI0V2>#TBg}R?|PkVfS1+v}X8+--KJ2K)%F%t8G$>5O2
z)RssslsANf@4U`_m*tQGRdmJizSL{@Sqaer+(sEb#JhY^7U%>NxGN7W3p3*P=*i|6
zbZ5~mP`978nW9jH`aHcJ^5{j?DXRL>sa1Ok25wcG;ia;^uvu7C0KV@Q_xmKkzG|;{
z6W__W_(xP8J>h)SKyAD3wI%6@ozzbTWp7o0hG0TI@N!@HSufAw^rJ<@1{!nNYm-R=
zg&2f$tFK?sfj2lg(T)n-)N|h10ko4Y-FA-Jv{t|E75uU-uX~&OcA44-VsdxEU~2xJ
zJ~U8IKyvaE^Bpre{9$;zlYAKR<zw`cm+bg`A9`5Ed}Dd+)>&GY4M)^X#UFpQb4J@t
zVnG>V%H05aeyJHvT~j{@%J{)6$W~yCf|+iSL6#8s^KEQve3Qy;#nk=Y;rW!`hP1mA
z&Qn%{>W2V*|M1B=7R`{qT4?>~lJ&)e+HK}E1&dGD=wA)-YD*Tm@DG0-PT03u_+p(`
zPi(5A^+f$kFVDsbV=bl`Z<{Lg?PBe2z#26v6aRPJ5Gt_${=t*@isP#f^^DMObJJEw
zlje@p4LgO=@r7epif}7)(D%9{{n}B*VyJl|7zEn=SHN4u`yYP~`#}A@C4u7dE6Fnu
zY-^ePXLxS2?E&Mb7H(6wT;3I2W9$~Q73MSI*}W;ae9Li<0)PnY*y5%p{+`JG>S$h!
z&4~R*k;5POVvrRQpOdmPX?F5wrCriJGph<I-SZk)aon`@>6AagWFxLp7OYtLkEMTM
zxT3_7aVCcApGk5eJ$$mL|NWsFs~Ponj`|nx85>9jUa+B!z^V7>XSRu8(k5VEh6Dq#
z{$JbBg$Twb5q{0o;Ps5id5rPYy_82&9zU6|v)a5vPe9f;^7F0;`HoMQYR1Zm5XLxs
zZ=D-s+{2pXo;dzPoa3{AXVSQJlhQPB5)pVj_sd(P5550i-XdZ!!MUpfHP!V^?(IbX
z7|b#-Y(tFCKsl0IcniQII5#B52NhLlF=|~q=~dq{Pc@ncE+rIc_6{4+b76j>LlG_K
z-pAe%*p^_)in%5~;N7<F-ajLn^hv1DGLnmJU64Ah)-41Z>5B%QAAhax3JE-}kr%+>
zP@BOyMX7N%6IQNz49K-@Nhpf!Y<bYDto1#p(EB(AmQ{`mfE=DQqg(WmO7FFsQj7zC
zjFconJ#@6WN9u`zyW5lxRmP_U0*=EefOROP+Ua9z|EuE$mFsIV6A&E#50ro}t)=nD
zJgoJyXG7L6A~s`Xoog(SI@M~;i7WB<cLNyv<93L)P21p{@vems2ieQo(j7Bd`x@vI
zsd+FYE>O?;P{Y0q`;#*}GWbHKm(x^?lP(Gvl<Z+^cVjsjR#%QgQu1t)c03c*X*Vx6
zf`;5IT5InNFl(D+B#GnuFEKiH!T2w~{PYB-GdE-`Z<#O5Z0L>_Owk&ccYXfa7+}V}
z?*qlLzVN+WEZq`l5guRH-#zd+WoLd^T1Y@iG1jE=lxA96bf@CL3?cd1%Q8JX8!J+q
zH2LtRb@M~FMdo!cdFG+R@C#iAOyzG9lAw}`J25r@MpFe2uIoDwZj}L_Kkz1?tiIa2
z>x{Ib!d0cgEb<jk3C*<26(w&cS6smp)rOrT6M5)q$&S&7@PL+asN{LTFNsi|sj>6u
zr{bJ@nMC9d?mGZ-@Gk5xFiZp-p#L;XoUky~^LH#t6nb{OAyHw}!uw|7<3ho;`yV6e
zJG#ocL2E@whh94C+Ww|f5U8!=cr`m(9YLPpeief|$wbp*+3G=lIey`QJSln@$KqD{
z-jbt`H{;ZS(td>tk~(ocQ5&&GcTY}BgaWz>yYSB(S6xW1?ev`I-ZKB5iH|lUP|YOG
z;bV#&FM|;%d9pr>9NcN@|BAfeOxj0WZ)E#V#Fb|kEYm7?JZ~1WA6bGn-NH#t>U$#Y
zmrEgHPq%;X|KLf~qzZ)Jua|>p+fowOXODw=VJ6&O!qg&I@5rwWgRI(=LVSeu`oE3G
zzjxMX-sVFHj>XCsxWaVa8!f6ghg1!J#rR4*p+{Z741RTi^PpEy&52;nC@a#>9I2gN
z_a`x{Cr?m4ozMJU!E&2!8nNjYHVOEm`6@Ng?PZJhbBe18xsXki_W=djtVl7g0m+q)
zR%hY_2Ti@s093ddcluS~Mo%Hngg>nO;o!}R-N|epi4Q7W2um$`I?_^I?3?5|nT<YK
z(diOc@Qj%?i+vU;*}&abv|2m4QII+uhy>}fBzPfka#AOUxE*tDS9(q!EJXz$g%_n{
zzMOmRi8E@Hge~nGzWUp(bos*SU~NwqTG!CMVtM)Uh&^`J+RNeD?V!<z2FE}-rbaW7
z6?PpI1;pZA2cql6qWYJ5024avld6z9CPv;L74_SAL?|@%Q2QT!@ASBCxxSt!Y_>TM
zj`>}4BT=g*>zswCN$mG9Ym#f)96(Dy0z)|2G|tMO&F6lc6L$-nV+vYrW_6ZI7~a`Q
zE+fkz8Ar;^S#!y^h4LNvQqb=0V#?4}nr&CB<2qvXLwY;Kol9VCz@;e~mh+ll0Bp;(
zZUO2>YHK%9BIUVFOw<>=^RJvt3&8JK5mSP(Y@GzXJEkFdRj<R+s$d6*0)drv24`A)
z*JsdH?3(JNW`n@wniThY#pA%l#!>Hz<W+&eYiG_Isk??CzK4tyv;@m{!8#VRZ`56b
zt8e6RyAMXr{h#)(H6H4;4R2yIuO)2~D^WSD!<Z1>acD|}VT?o0#|<&0)+m-3wu6$a
z7$!3gW1TCSa!7>1E9*QMhf-~lnTQ-_SZ2b!&t!XlvueM-pLRYypZ>r9{GR`PKlgLp
z*L_`EH6fctSj8X{=1z6^KYFTgH=vx=CG8Z=kp@Y;V;GobqATFsT%6HEj(2~7GW;UB
zX80hiHJ%mqp+m1H)(5zUym>3hUS&lcb{JLqT!z2UxYeSk7}G%L;!uCe`!$A{wX~g~
z&KlPpVv90(?V-quegx&5uH1^UG@$&A66Y^|6xqP+oT$IRXA!DS#<;`*!BYkJqp$Wq
z7MMu_AaYDxZo<x;$BOpEAdA&v3q`#4Mn^gAFH#yYovEOV6#xPm_SE+Zs1Y}HNxf$s
z+0sc{MhZV!lBTsy(q`p3UgMzI2RR-gBtVsx6Y8QL4={tL60Vb{;?zs@Ft*8)5dD|!
zZ&jaFpp&wk(HZsqR88*Ka;ZXy|08p{Mr(IRSU6TBIzl9ea;<=^ZRm5f@eycgpUj()
z5a2Uc3$viL3Rn8<s)E?~y|+ovL$Kw>4~p|9oIKx^P*OUzHij|GmHfnl3>N}y^;47U
zRjLbYV0`}*zuXil{m;fnlSB(|T>Ca)wEiFsQZ?Rrc!=$c52<$k22{i%3q?5R7Fk!o
zL1nq2avMGwNl(Vy@1+io(YCIn>{-p0HxW3z&R8oV#8T5jF!3BrN>z|!z7p|C$%D5O
ziP_bA@Mk`f(PWzD>=6U)RpO|?!S1OCBZV07?Mi~ENNkz}g&Gx)`}VZzmH~NzzwIrI
z?6sj%2yIU6;Hb-bQOsA>7=!uqtHJ2lN!XeB3rK{MVSzXZ3jjKQ67*LyMD&e4Qvpy8
zK~b=abH+juvDKax5OumX*C!qD^@m1KA|_3Q(;bvu7H3L$Eo$s`*t`dD3M2XU`xEav
zC75Rg>1doSveJf!qq6cC1;_H(*!8|Bdn=~h53672u!W!Z&j|A8#+G-KqVpL)Pq%p{
zp&Hr^Yfr^_UAxnRMi1I#88TYs)<lGGS==2Y5Xf5L_GKoK{S%oXqu&5nXDc}JNH%hY
z8w4WZ=bQLLIif2P4Juc#dJXOBNh;H=qs;re@{-q+9T{I!RTmux!{2L=G>wUP3K2g7
zglx`&T%ObfiUHUci-7|-cLuqaai;T@61>osD{2t8&<-s$g)wwDX38gw8RHqzZoz9&
zEV$z1&{_{JMXG^bc;;>Hf+cut$}p$kofdl@XaxW2Zy``(L@M5#>!Ba%oU1Ohv2(v4
z+dg$At8A2xU?iqndYr60X3K@yjMdyF=cg|KRhWav@!U(zPeVpGMvn@@cgO(h1@JJt
zBD~pv&Q=&JRIX{3<I;uB#oQow<`u;&V-{VtFb3R^ecP#;8H;J=8K+CgZgsS)KzS0|
z@+ZR0Ugccw{q0?hb+ilTbMgte4V#(aq{el5%AOsm4z%retFenEsYb`uc5!c+rR6Hc
zlb7a8$x#=VW%>@0_86(Y#Y9hA0>mPBfouZgFgi>6D+REjBV#nll{p?40XmO0c<*$@
z>mId{Fk)IGDvXm_K6rKg;ufm+fgIfW8e%=t8JYg{nhvNvd+SvA-5%o&HCWynnWr~X
z*8|kTbmatvD&|!3>Pcw_7WAVzjRntx{cxk@jb5rJ_#MC{-H>9Sx5xKX1NR>AihK|R
z>=9vBiv`+Io46*RbAGVeH5|?h6hA?Eb4~}!eaSowYMW}w^ZOxC?(OLBLX9N=m9YGZ
z7YeC_p)bu?j8wh&pa;Y8zbFqem+6?i3ak6#m%Hu>L_Tw8`|^Yur+>DGjzlHd5QFv9
zYc8x2=U(5E;bzfTN@Fd>*^V_-BdV$2Oh`YJRK@I@06XuhH#aArqpBYh0VVeoF#oPx
zYGV1?yI%@+6bi}^b})fUsZ!rCl9pC1!Z~batje~vk^0|L$pPA<?gc>=H-}^VZJmd_
z_@xq~V6A{Xn_aAKIG5=;|Lj6x^A}p8??orft)e^7W`+H5*@tD$>7I8rcFSh4$F)|Z
zre(OQ(uQ3bG}a;C^9{O)ABYc#2$E(M5&wocu@SSXIuIdt%nRdJbYOS_x}ksNV$~eT
z6M2L`1>aJ*W4OemD!$`AYY6N8^tQJFxPE8=slZB2khpv>IzHpJo?!DhlOH#4kdyjH
zFP}g8SCoJGz)k5)WagAlqdf&(hFKtzzbQ?8=o*m^rOJZmLDu}syQBH}8p6-lJ!$OO
zFhFSHJuy^merQyQ5jg!;l7(6HU^t$T7BVG15nX|-QhQxot`9>$2#5<Ui9;b(zMnT5
zM%&X@T)?-mW<~>42X8=i;JZ8}^!{CtRtcdzY)aLBE`E~rFlH;o^v?4V>};_NH%VeO
zP2!G!MYVH2PN!5B7Np~N$~LatYI?B*6fc0H&M#gP`8RcqCbl>)`;~~Gl0tk_tY}a&
zVK=eX?#wWI^4&O86&1X{JRx0<I}y&hyq+ktiq-!Q>Bn)scY%a=?AxWJo<#f|M!EK{
zQ)!nwlg=?0YbbvX4D|+%q|<(fE_|t#3KN2FOZCQ@njH(@$Ejb5_;%`8v!1hg=rd=Z
zh(3g$qzq7t{BfVo`&}rN!R`)*!$YZi*}(6(K92AEY|+b(W#pXQ_l2<nMl)8(()kTi
zV#wP3m$W@@CmpiZgLC}`3G2{Mc;-4V<y1<jFmHHjFpzJ>oG1$!u#}IhJ`0`+Jb^Xj
zb1?v9OxPPM5Nn{ZSw1js3gsyq8x7cn8l)))cfOZ!97i>h;=3TyAa(6rExZxsG2lhm
z3rbt$`AZZl_T`uI5YRSIp0gUMwT-=X!s@=)#nc=Dg!Dzikvce<2NMemP#z$Vn~=^v
zB2HQQykYA`B03DNqQ^8#7!AMa4KhytrEP0c%PS*yJPr}<0!6z!eh9~InBj6}SW*iA
zrY(zrBn}v`n`}gn3#T9H@_B9ToMgA=__8B0+m-%Pc;3GPD*1yY?g(j`FTdeSCzT~a
ztXLrHGy_#-zO{hkIh;}pp0Cxh86Q=o!01RQOso2&wFj@tqA}4WYdrYz%)r(4WRcZB
zmhevjE_5!xJ3AY|qoUDoWk3d_Wxh3HJ#GaV_Li@|s^H<Z|7Lh#w3Wug;Lh>EOJzw*
z^?BB?3Tv_}V9H&;p}*t@@426J7{H^(eTDY|>he0YNQn&CV!<rs1wlJb+qQwKJBMGN
z>pEN52<S4Kt%du9poqlc10_hDd-*Rdq28?>>TS5+2B=$f@z;Z26REfl%vJTG*KOSH
zZOC1{R>VptoY?k5ZnfEuip6)co40IeP+i!r^iv%mZG9#J%@U8cG+i-xO7PDb9f-dr
z$2rk~)(x_E{{VfUHrs3ww_?6#BZ&Z1N6m13S*CwnP92wY=qTOeDnFpxj9(3WnaUwa
zKG`3SWof!WySuEzn5{(5+xu#U%$_^YYyq0t@@yE5k>y=i8CmW3?|*Uae<wM+bjqH8
zbN?cBQ1Mk}$!<hZF&~2{y)*JSQZ{4Po735T@_ZgRfFH53ElDXUDq^RZ6OZ8VqrH#8
zJ3>$wn~W6<Xpv7Mk8Xv0=GeBQPw0hJW%d!eWQG}?AbqgbB%R%&WkA^dG06(o+Nkk8
zM>AB|%J;lq;jpNz{S&Z{;}4cOar+P?f!Gr04ga(Ji}bfZr2MlK=z{#;mofMz-Zo#?
W@A}3t`E6f;pOv|-**#O=i~j{8Q%T4G

literal 0
HcmV?d00001

diff --git a/docs/index.rst b/docs/index.rst
new file mode 100644
index 00000000..32c3a33a
--- /dev/null
+++ b/docs/index.rst
@@ -0,0 +1,160 @@
+pytorch-widedeep
+================
+
+*A flexible package to combine tabular data with text and images using wide and deep models*
+
+Below there is an introduction to the architectures one can build using
+``pytorch-widedeep``. If you prefer to learn about the utilities and
+components go straight to corresponding sections in the Documentation.
+
+Documentation
+-------------
+
+.. toctree::
+    :maxdepth: 1
+
+    Installation <installation>
+    Quick Start <quick_start>
+    Utilities <utils/index>
+    Preprocessing <preprocessing>
+    Model Components <model_components>
+    Bayesian Models <bayesian_models>
+    Metrics <metrics>
+    Losses <losses>
+    Dataloaders <dataloaders>
+    Callbacks <callbacks>
+    The Trainer <trainer>
+    Tab2Vec <tab2vec>
+    Examples <examples>
+
+
+Introduction
+------------
+``pytorch-widedeep`` is based on Google's `Wide and Deep Algorithm
+<https://arxiv.org/abs/1606.07792>`_, adjusted for multi-modal datasets
+
+
+In general terms, ``pytorch-widedeep`` is a package to use deep learning with
+tabular and multimodal data. In particular, is intended to facilitate the
+combination of text and images with corresponding tabular data using wide and
+deep models. With that in mind there are a number of architectures that can
+be implemented with just a few lines of code. The main components of those
+architectures are shown in the Figure below:
+
+.. image:: figures/widedeep_arch.png
+   :width: 700px
+   :align: center
+
+The dashed boxes in the figure represent optional, overall components, and the
+dashed lines indicate the corresponding connections, depending on whether or
+not certain components are present. For example, the dashed, blue-arrows
+indicate that the ``deeptabular``, ``deeptext`` and ``deepimage`` components
+are connected directly to the output neuron or neurons (depending on whether
+we are performing a binary classification or regression, or a multi-class
+classification) if the optional ``deephead`` is not present. The components
+within the faded-pink rectangle are concatenated.
+
+Note that it is not possible to illustrate the number of possible
+architectures and components available in ``pytorch-widedeep`` in one Figure.
+Therefore, for more details on possible architectures (and more) please, read
+this documentation, or see the `Examples
+<https://github.com/jrzaurin/pytorch-widedeep/tree/master/examples>`_ folders
+in the repo.
+
+In math terms, and following the notation in the `paper
+<https://arxiv.org/abs/1606.07792>`_, the expression for the architecture
+without a ``deephead`` component can be formulated as:
+
+
+.. image:: figures/architecture_1_math.png
+   :width: 600px
+   :align: center
+
+
+Where *'W'* are the weight matrices applied to the wide model and to the final
+activations of the deep models, :math:`a` are these final activations, and
+:math:`{\phi}` (x) are the cross product transformations of the original
+features *'x'*. In case you are wondering what are *"cross product
+transformations"*, here is a quote taken directly from the paper: *"For binary
+features, a cross-product transformation (e.g., “AND(gender=female,
+language=en)”) is 1 if and only if the constituent features (“gender=female”
+and “language=en”) are all 1, and 0 otherwise".* Finally, :math:`{\sigma}` (.)
+is the activation function.
+
+While if there is a ``deephead`` component, the previous expression turns
+into:
+
+.. image:: figures/architecture_2_math.png
+   :width: 350px
+   :align: center
+
+
+It is important to emphasize that **each individual component, wide,
+deeptabular, deeptext and deepimage, can be used independently and in
+isolation**. For example, one could use only ``wide``, which is in simply a
+linear model. In fact, one of the most interesting functionalities in
+``pytorch-widedeep`` would be the use of the ``deeptabular`` component on its
+own, i.e. what one might normally refer as Deep Learning for Tabular Data.
+Currently, ``pytorch-widedeep`` offers the following different models for
+that component:
+
+0. **Wide**: a simple linear model where the nonlinearities are captured via
+cross-product transformations, as explained before.
+
+1. **TabMlp**: a simple MLP that receives embeddings representing the
+categorical features, concatenated with the continuous features, which can
+also be embedded.
+
+2. **TabResnet**: similar to the previous model but the embeddings are
+passed through a series of ResNet blocks built with dense layers.
+
+3. **TabNet**: details on TabNet can be found in `TabNet: Attentive
+Interpretable Tabular Learning <https://arxiv.org/abs/1908.07442>`_
+
+The ``Tabformer`` family, i.e. Transformers for Tabular data:
+
+4. **TabTransformer**: details on the TabTransformer can be found in
+`TabTransformer: Tabular Data Modeling Using Contextual Embeddings
+<https://arxiv.org/pdf/2012.06678.pdf>`_.
+
+5. **SAINT**: Details on SAINT can be found in `SAINT: Improved Neural
+Networks for Tabular Data via Row Attention and Contrastive Pre-Training
+<https://arxiv.org/abs/2106.01342>`_.
+
+6. **FT-Transformer**: details on the FT-Transformer can be found in
+`Revisiting Deep Learning Models for Tabular Data
+<https://arxiv.org/abs/2106.11959>`_.
+
+7. **TabFastFormer**: adaptation of the FastFormer for tabular data. Details
+on the Fasformer can be found in `FastFormers: Highly Efficient Transformer
+Models for Natural Language Understanding
+<https://arxiv.org/abs/2010.13382>`_
+
+8. **TabPerceiver**: adaptation of the Perceiver for tabular data. Details on
+the Perceiver can be found in `Perceiver: General Perception with Iterative
+Attention <https://arxiv.org/abs/2103.03206>`_
+
+And probabilistic DL models for tabular data based on
+`Weight Uncertainty in Neural Networks <https://arxiv.org/abs/1505.05424>`_:
+
+9. **BayesianWide**: Probabilistic adaptation of the `Wide` model.
+
+10. **BayesianTabMlp**: Probabilistic adaptation of the `TabMlp` model
+
+Note that while there are scientific publications for the TabTransformer,
+SAINT and FT-Transformer, the TabFasfFormer and TabPerceiver are our own
+adaptation of those algorithms for tabular data. For details on these models
+and their options please see the examples in the Examples folder and the
+documentation.
+
+Finally, it is perfectly possible to use custom models as long as the the
+custom models have an attribute called ``output_dim`` with the size of the
+last layer of activations, so that ``WideDeep`` can be constructed. Again,
+examples on how to use custom components can be found in the Examples
+folder.
+
+Indices and tables
+==================
+
+* :ref:`genindex`
+* :ref:`search`
diff --git a/docs/installation.rst b/docs/installation.rst
new file mode 100644
index 00000000..c63af2fd
--- /dev/null
+++ b/docs/installation.rst
@@ -0,0 +1,44 @@
+Installation
+************
+This section explains how to install ``pytorch-widedeep``.
+
+For the latest stable release, execute:
+
+.. code::
+
+    pip install pytorch-widedeep
+
+For the bleeding-edge version, execute:
+
+.. code::
+
+	pip install git+https://github.com/jrzaurin/pytorch-widedeep.git
+
+
+For developer install
+
+.. code::
+
+	# Clone the repository
+	git clone https://github.com/jrzaurin/pytorch-widedeep
+	cd pytorch-widedeep
+
+	# Install in dev mode
+	pip install -e .
+
+Dependencies
+------------
+* pandas
+* numpy
+* scipy
+* scikit-learn
+* gensim
+* spacy
+* opencv-contrib-python
+* imutils
+* tqdm
+* torch
+* torchvision
+* einops
+* wrapt
+* torchmetrics
diff --git a/docs/losses.rst b/docs/losses.rst
new file mode 100644
index 00000000..6ce8e769
--- /dev/null
+++ b/docs/losses.rst
@@ -0,0 +1,65 @@
+Losses
+======
+
+``pytorch-widedeep`` accepts a number of losses and objectives that can be
+passed to the ``Trainer`` class via the parameter ``objective``
+(see ``pytorch-widedeep.training.Trainer``). For most cases the loss function
+that ``pytorch-widedeep`` will use internally is already implemented in
+Pytorch.
+
+In addition, ``pytorch-widedeep`` implements a series of  "custom" loss
+functions. These are described below for completion since, as mentioned
+before, they are used internally by the ``Trainer``. Of course, onen could
+always use them on their own and can be imported as:
+
+.. code-block:: python
+
+	from pytorch_widedeep.losses import FocalLoss
+
+.. note:: Losses in this module expect the predictions and ground truth to have the
+	same dimensions for regression and binary classification problems
+	:math:`(N_{samples}, 1)`. In the case of multiclass classification problems
+	the ground truth is expected to be a 1D tensor with the corresponding
+	classes. See Examples below
+
+.. autoclass:: pytorch_widedeep.losses.MSELoss
+	:members:
+
+.. autoclass:: pytorch_widedeep.losses.MSLELoss
+	:members:
+
+.. autoclass:: pytorch_widedeep.losses.RMSELoss
+	:members:
+
+.. autoclass:: pytorch_widedeep.losses.RMSLELoss
+	:members:
+
+.. autoclass:: pytorch_widedeep.losses.QuantileLoss
+	:members:
+
+.. autoclass:: pytorch_widedeep.losses.FocalLoss
+	:members:
+
+.. autoclass:: pytorch_widedeep.losses.BayesianSELoss
+	:members:
+
+.. autoclass:: pytorch_widedeep.losses.TweedieLoss
+	:members:
+
+.. autoclass:: pytorch_widedeep.losses.ZILNLoss
+	:members:
+
+.. autoclass:: pytorch_widedeep.losses.L1Loss
+	:members:
+
+.. autoclass:: pytorch_widedeep.losses.FocalR_L1Loss
+	:members:
+
+.. autoclass:: pytorch_widedeep.losses.FocalR_MSELoss
+	:members:
+
+.. autoclass:: pytorch_widedeep.losses.FocalR_RMSELoss
+	:members:
+
+.. autoclass:: pytorch_widedeep.losses.HuberLoss
+	:members:
diff --git a/docs/make.bat b/docs/make.bat
new file mode 100644
index 00000000..922152e9
--- /dev/null
+++ b/docs/make.bat
@@ -0,0 +1,35 @@
+@ECHO OFF
+
+pushd %~dp0
+
+REM Command file for Sphinx documentation
+
+if "%SPHINXBUILD%" == "" (
+	set SPHINXBUILD=sphinx-build
+)
+set SOURCEDIR=.
+set BUILDDIR=_build
+
+if "%1" == "" goto help
+
+%SPHINXBUILD% >NUL 2>NUL
+if errorlevel 9009 (
+	echo.
+	echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
+	echo.installed, then set the SPHINXBUILD environment variable to point
+	echo.to the full path of the 'sphinx-build' executable. Alternatively you
+	echo.may add the Sphinx directory to PATH.
+	echo.
+	echo.If you don't have Sphinx installed, grab it from
+	echo.http://sphinx-doc.org/
+	exit /b 1
+)
+
+%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
+goto end
+
+:help
+%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
+
+:end
+popd
diff --git a/docs/metrics.rst b/docs/metrics.rst
new file mode 100644
index 00000000..0d0e84c4
--- /dev/null
+++ b/docs/metrics.rst
@@ -0,0 +1,51 @@
+Metrics
+=======
+
+.. note:: Metrics in this module expect the predictions and ground truth to have the
+	same dimensions for regression and binary classification problems: :math:`(N_{samples}, 1)`.
+	In the case of multiclass classification problems the ground truth is expected to be
+	a 1D tensor with the corresponding classes. See Examples below
+
+We have added the possibility of using the metrics available at the
+`torchmetrics <https://torchmetrics.readthedocs.io/en/latest/>`_ library.
+Note that this library is still in its early versions and therefore this
+option should be used with caution. To use ``torchmetrics`` simply import
+them and use them as any of the ``pytorch-widedeep`` metrics described
+below.
+
+.. code-block:: python
+
+	from torchmetrics import Accuracy, Precision
+
+	accuracy = Accuracy(average=None, num_classes=2)
+	precision = Precision(average='micro', num_classes=2)
+
+	trainer = Trainer(model, objective="binary", metrics=[accuracy, precision])
+
+A functioning example for ``pytorch-widedeep`` using ``torchmetrics`` can be
+found in the `Examples folder <https://github.com/jrzaurin/pytorch-widedeep/blob/master/examples>`_.
+
+
+.. autoclass:: pytorch_widedeep.metrics.Accuracy
+	:members:
+	:undoc-members:
+
+.. autoclass:: pytorch_widedeep.metrics.Precision
+	:members:
+	:undoc-members:
+
+.. autoclass:: pytorch_widedeep.metrics.Recall
+	:members:
+	:undoc-members:
+
+.. autoclass:: pytorch_widedeep.metrics.FBetaScore
+	:members:
+	:undoc-members:
+
+.. autoclass:: pytorch_widedeep.metrics.F1Score
+	:members:
+	:undoc-members:
+
+.. autoclass:: pytorch_widedeep.metrics.R2Score
+	:members:
+	:undoc-members:
diff --git a/docs/model_components.rst b/docs/model_components.rst
new file mode 100644
index 00000000..00a3a1ba
--- /dev/null
+++ b/docs/model_components.rst
@@ -0,0 +1,71 @@
+The ``models`` module
+======================
+
+This module contains the models that can be used as the four main components
+that will comprise a Wide and Deep model (``wide``, ``deeptabular``,
+``deeptext``, ``deepimage``), as well as the ``WideDeep`` "constructor"
+class. Note that each of the four components can be used independently.
+
+.. autoclass:: pytorch_widedeep.models.tabular.linear.wide.Wide
+    :exclude-members: forward
+    :members:
+
+.. autoclass:: pytorch_widedeep.models.tabular.mlp.tab_mlp.TabMlp
+    :exclude-members: forward
+    :members:
+
+.. autoclass:: pytorch_widedeep.models.tabular.mlp.context_attention_mlp.ContextAttentionMLP
+    :exclude-members: forward
+    :members:
+
+.. autoclass:: pytorch_widedeep.models.tabular.mlp.self_attention_mlp.SelfAttentionMLP
+    :exclude-members: forward
+    :members:
+
+.. autoclass:: pytorch_widedeep.models.tabular.resnet.tab_resnet.TabResnet
+    :exclude-members: forward
+    :members:
+
+.. autoclass:: pytorch_widedeep.models.tabular.tabnet.tab_net.TabNet
+    :exclude-members: forward
+    :members:
+
+.. autoclass:: pytorch_widedeep.models.tabular.transformers.tab_transformer.TabTransformer
+    :exclude-members: forward
+    :members:
+
+.. autoclass:: pytorch_widedeep.models.tabular.transformers.saint.SAINT
+    :exclude-members: forward
+    :members:
+
+.. autoclass:: pytorch_widedeep.models.tabular.transformers.ft_transformer.FTTransformer
+    :exclude-members: forward
+    :members:
+
+.. autoclass:: pytorch_widedeep.models.tabular.transformers.tab_perceiver.TabPerceiver
+    :exclude-members: forward
+    :members:
+
+.. autoclass:: pytorch_widedeep.models.tabular.transformers.tab_fastformer.TabFastFormer
+    :exclude-members: forward
+    :members:
+
+.. autoclass:: pytorch_widedeep.models.text.attentive_rnn.BasicRNN
+    :exclude-members: forward
+    :members:
+
+.. autoclass:: pytorch_widedeep.models.text.attentive_rnn.AttentiveRNN
+    :exclude-members: forward
+    :members:
+
+.. autoclass:: pytorch_widedeep.models.text.stacked_attentive_rnn.StackedAttentiveRNN
+    :exclude-members: forward
+    :members:
+
+.. autoclass:: pytorch_widedeep.models.image.vision.Vision
+    :exclude-members: forward
+    :members:
+
+.. autoclass:: pytorch_widedeep.models.wide_deep.WideDeep
+    :exclude-members: forward
+    :members:
diff --git a/docs/preprocessing.rst b/docs/preprocessing.rst
new file mode 100644
index 00000000..a068db8c
--- /dev/null
+++ b/docs/preprocessing.rst
@@ -0,0 +1,22 @@
+The ``preprocessing`` module
+============================
+
+This module contains the classes that are used to prepare the data before
+being passed to the models. There is one Preprocessor per data mode or
+model component: ``wide``, ``deeptabular``, ``deepimage`` and ``deeptext``.
+
+.. autoclass:: pytorch_widedeep.preprocessing.WidePreprocessor
+	:members:
+	:undoc-members:
+
+.. autoclass:: pytorch_widedeep.preprocessing.TabPreprocessor
+	:members:
+	:undoc-members:
+
+.. autoclass:: pytorch_widedeep.preprocessing.TextPreprocessor
+	:members:
+	:undoc-members:
+
+.. autoclass:: pytorch_widedeep.preprocessing.ImagePreprocessor
+	:members:
+	:undoc-members:
diff --git a/docs/quick_start.rst b/docs/quick_start.rst
new file mode 100644
index 00000000..e21d618e
--- /dev/null
+++ b/docs/quick_start.rst
@@ -0,0 +1,134 @@
+Quick Start
+***********
+
+This is an example of a binary classification with the `adult census
+<https://www.kaggle.com/wenruliu/adult-income-dataset?select=adult.csv>`__
+dataset using a combination of a wide and deep model (in this case a so called
+``deeptabular`` model) with defaults settings.
+
+
+Read and split the dataset
+--------------------------
+
+.. code-block:: python
+
+    import pandas as pd
+    import numpy as np
+    from sklearn.model_selection import train_test_split
+    from pytorch_widedeep.datasets import load_adult
+
+    df = load_adult(as_frame=True)
+    df["income_label"] = (df["income"].apply(lambda x: ">50K" in x)).astype(int)
+    df.drop("income", axis=1, inplace=True)
+    df_train, df_test = train_test_split(df, test_size=0.2, stratify=df.income_label)
+
+
+
+Prepare the wide and deep columns
+---------------------------------
+
+.. code-block:: python
+
+    from pytorch_widedeep import Trainer
+    from pytorch_widedeep.preprocessing import WidePreprocessor, TabPreprocessor
+    from pytorch_widedeep.models import Wide, TabMlp, WideDeep
+    from pytorch_widedeep.metrics import Accuracy
+
+    # Define the 'column set up'
+    wide_cols = [
+        "education",
+        "relationship",
+        "workclass",
+        "occupation",
+        "native-country",
+        "gender",
+    ]
+    crossed_cols = [("education", "occupation"), ("native-country", "occupation")]
+
+    cat_embed_cols = [
+        "workclass",
+        "education",
+        "marital-status",
+        "occupation",
+        "relationship",
+        "race",
+        "gender",
+        "capital-gain",
+        "capital-loss",
+        "native-country",
+    ]
+    continuous_cols = ["age", "hours-per-week"]
+    target = "income_label"
+    target = df_train[target].values
+
+Preprocessing and model components definition
+---------------------------------------------
+
+.. code-block:: python
+
+    wide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols)
+    X_wide = wide_preprocessor.fit_transform(df_train)
+
+    tab_preprocessor = TabPreprocessor(
+        cat_embed_cols=cat_embed_cols, continuous_cols=continuous_cols  # type: ignore[arg-type]
+    )
+    X_tab = tab_preprocessor.fit_transform(df_train)
+
+    # build the model
+    wide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1)
+    tab_mlp = TabMlp(
+        column_idx=tab_preprocessor.column_idx,
+        cat_embed_input=tab_preprocessor.cat_embed_input,
+        continuous_cols=continuous_cols,
+    )
+    model = WideDeep(wide=wide, deeptabular=tab_mlp)
+
+
+Fit and predict
+-------------------------------
+
+.. code-block:: python
+
+    # train and validate
+    trainer = Trainer(model, objective="binary", metrics=[Accuracy])
+    trainer.fit(
+        X_wide=X_wide,
+        X_tab=X_tab,
+        target=target,
+        n_epochs=5,
+        batch_size=256,
+    )
+
+    # predict on test
+    X_wide_te = wide_preprocessor.transform(df_test)
+    X_tab_te = tab_preprocessor.transform(df_test)
+    preds = trainer.predict(X_wide=X_wide_te, X_tab=X_tab_te)
+
+
+Save and load
+-------------------------------
+
+.. code-block:: python
+
+    # Option 1: this will also save training history and lr history if the
+    # LRHistory callback is used
+    trainer.save(path="model_weights", save_state_dict=True)
+
+    # Option 2: save as any other torch model
+    torch.save(model.state_dict(), "model_weights/wd_model.pt")
+
+    # From here in advance, Option 1 or 2 are the same. I assume the user has
+    # prepared the data and defined the new model components:
+    # 1. Build the model
+    model_new = WideDeep(wide=wide, deeptabular=tab_mlp)
+    model_new.load_state_dict(torch.load("model_weights/wd_model.pt"))
+
+    # 2. Instantiate the trainer
+    trainer_new = Trainer(model_new, objective="binary")
+
+    # 3. Either start the fit or directly predict
+    preds = trainer_new.predict(X_wide=X_wide, X_tab=X_tab)
+
+Of course, one can do **much more**. See the Examples folder in the repo, this
+documentation or the companion posts for a better understanding of the content
+of the package and its functionalities.
diff --git a/docs/requirements.txt b/docs/requirements.txt
new file mode 100644
index 00000000..03877a58
--- /dev/null
+++ b/docs/requirements.txt
@@ -0,0 +1,20 @@
+sphinx
+sphinx_rtd_theme
+recommonmark
+sphinx-markdown-tables
+sphinx-copybutton
+sphinx-autodoc-typehints
+pandas
+numpy
+scipy
+scikit-learn
+gensim
+spacy
+opencv-contrib-python
+imutils
+tqdm
+torch
+torchvision
+einops
+wrapt
+torchmetrics
diff --git a/docs/tab2vec.rst b/docs/tab2vec.rst
new file mode 100644
index 00000000..8daf940c
--- /dev/null
+++ b/docs/tab2vec.rst
@@ -0,0 +1,7 @@
+Tab2Vec
+=======
+
+.. autoclass:: pytorch_widedeep.tab2vec.Tab2Vec
+	:members:
+	:undoc-members:
+
diff --git a/docs/trainer.rst b/docs/trainer.rst
new file mode 100644
index 00000000..808e2882
--- /dev/null
+++ b/docs/trainer.rst
@@ -0,0 +1,23 @@
+Training multimodal Deep Learning Models
+========================================
+
+Here is the documentation for the ``Trainer`` class, that will do all the heavy lifting.
+
+Trainer is also available from ``pytorch-widedeep`` directly, for example, one could do:
+
+.. code-block:: python
+
+	from pytorch-widedeep.training import Trainer
+
+
+or also:
+
+.. code-block:: python
+
+	from pytorch-widedeep import Trainer
+
+
+.. autoclass:: pytorch_widedeep.training.Trainer
+	:exclude-members: forward
+	:members:
+	:undoc-members:
diff --git a/docs/utils/deeptabular_utils.rst b/docs/utils/deeptabular_utils.rst
new file mode 100644
index 00000000..25f8ba4e
--- /dev/null
+++ b/docs/utils/deeptabular_utils.rst
@@ -0,0 +1,6 @@
+deeptabular utils
+=================
+
+.. autoclass:: pytorch_widedeep.utils.deeptabular_utils.LabelEncoder
+	:members:
+	:undoc-members:
diff --git a/docs/utils/fastai_transforms.rst b/docs/utils/fastai_transforms.rst
new file mode 100644
index 00000000..f18d60b0
--- /dev/null
+++ b/docs/utils/fastai_transforms.rst
@@ -0,0 +1,19 @@
+Fastai transforms
+=================
+
+I have directly copied and pasted part of the ``transforms.py`` module from
+the ``fastai`` library. The reason to do such a thing is because
+``pytorch_widedeep`` only needs the ``Tokenizer`` and the ``Vocab`` classes
+there. This way I avoid extra dependencies. Credit for all the code in the
+``fastai_transforms`` module in this ``pytorch-widedeep`` package goes to
+Jeremy Howard and the `fastai` team. I only include the documentation here for
+completion, but I strongly advise the user to read the ``fastai``
+`documentation <https://docs.fast.ai/>`_.
+
+.. autoclass:: pytorch_widedeep.utils.fastai_transforms.Tokenizer
+	:members:
+	:undoc-members:
+
+.. autoclass:: pytorch_widedeep.utils.fastai_transforms.Vocab
+	:members:
+	:undoc-members:
diff --git a/docs/utils/image_utils.rst b/docs/utils/image_utils.rst
new file mode 100644
index 00000000..aedb1f61
--- /dev/null
+++ b/docs/utils/image_utils.rst
@@ -0,0 +1,20 @@
+Image utils
+===========
+:class:`SimplePreprocessor
+<pytorch_widedeep.utils.image_utils.SimplePreprocessor>` and
+:class:`AspectAwarePreprocessor
+<pytorch_widedeep.utils.image_utils.AspectAwarePreprocessor>` are directly
+taked from the great series of Books `Deep Learning for Computer Vision
+<https://www.pyimagesearch.com/deep-learning-computer-vision-python-book/>`_
+by `Adrian <https://www.pyimagesearch.com/>`_. Therefore, all credit for the
+code in the ``image_utils`` module goes to `Adrian Rosebrock
+<https://www.pyimagesearch.com/>`_.
+
+.. autoclass:: pytorch_widedeep.utils.image_utils.AspectAwarePreprocessor
+	:members:
+	:undoc-members:
+
+.. autoclass:: pytorch_widedeep.utils.image_utils.SimplePreprocessor
+	:members:
+	:undoc-members:
+
diff --git a/docs/utils/index.rst b/docs/utils/index.rst
new file mode 100644
index 00000000..72df9e03
--- /dev/null
+++ b/docs/utils/index.rst
@@ -0,0 +1,23 @@
+The ``utils`` module
+====================
+
+These are a series of utilities that might be useful for a number of
+preprocessing tasks, even not directly related to ``pytorch-widedeep``. All
+the classes and functions discussed here are available directly from the
+``utils`` module. For example, the ``LabelEncoder`` within the
+``deeptabular_utils`` submodule can be imported as:
+
+.. code-block:: python
+
+	from pytorch_widedeep.utils import LabelEncoder
+
+
+Objects
+-------
+
+.. toctree::
+
+    deeptabular_utils
+    fastai_transforms
+    image_utils
+    text_utils
\ No newline at end of file
diff --git a/docs/utils/text_utils.rst b/docs/utils/text_utils.rst
new file mode 100644
index 00000000..1ad57478
--- /dev/null
+++ b/docs/utils/text_utils.rst
@@ -0,0 +1,15 @@
+Text utils
+=================
+Collection of helper function that facilitate processing text.
+
+.. autofunction:: pytorch_widedeep.utils.text_utils.simple_preprocess
+	:noindex:
+
+.. autofunction:: pytorch_widedeep.utils.text_utils.get_texts
+	:noindex:
+
+.. autofunction:: pytorch_widedeep.utils.text_utils.pad_sequences
+	:noindex:
+
+.. autofunction:: pytorch_widedeep.utils.text_utils.build_embeddings_matrix
+	:noindex:
\ No newline at end of file

From 90710fd0266bb3d2be0a9686ffbcc474711bc5ab Mon Sep 17 00:00:00 2001
From: Javier <jrzaurin@gmail.com>
Date: Mon, 21 Oct 2024 10:49:28 +0100
Subject: [PATCH 15/24] update readthedocs yaml file to 3.10

---
 .readthedocs.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.readthedocs.yaml b/.readthedocs.yaml
index baf59320..81148f8c 100644
--- a/.readthedocs.yaml
+++ b/.readthedocs.yaml
@@ -9,7 +9,7 @@ version: 2
 build:
   os: ubuntu-20.04
   tools:
-    python: "3.9"
+    python: "3.10"
 
 mkdocs:
   configuration: mkdocs/mkdocs.yml

From 112335533661558dfcedc049df99d21b6b60ebb0 Mon Sep 17 00:00:00 2001
From: Javier <jrzaurin@gmail.com>
Date: Mon, 21 Oct 2024 10:52:46 +0100
Subject: [PATCH 16/24] removed docs again. Not sure why docs are not building.
 Needs fix

---
 docs/Makefile                           |  21 --
 docs/_static/custom.css                 |  45 ----
 docs/_static/img/widedeep_logo_docs.ico | Bin 102115 -> 0 bytes
 docs/bayesian_models.rst                |  15 --
 docs/callbacks.rst                      |  29 ---
 docs/conf.py                            | 297 ------------------------
 docs/dataloaders.rst                    |  10 -
 docs/examples.rst                       |  20 --
 docs/figures/arch_1.png                 | Bin 58023 -> 0 bytes
 docs/figures/arch_2.png                 | Bin 51944 -> 0 bytes
 docs/figures/arch_3.png                 | Bin 61499 -> 0 bytes
 docs/figures/arch_4.png                 | Bin 74000 -> 0 bytes
 docs/figures/arch_5.png                 | Bin 76885 -> 0 bytes
 docs/figures/arch_6.png                 | Bin 123034 -> 0 bytes
 docs/figures/arch_7.png                 | Bin 57736 -> 0 bytes
 docs/figures/arch_8.png                 | Bin 33393 -> 0 bytes
 docs/figures/architecture_1_math.png    | Bin 47788 -> 0 bytes
 docs/figures/architecture_2_math.png    | Bin 16848 -> 0 bytes
 docs/figures/widedeep_arch.png          | Bin 157281 -> 0 bytes
 docs/figures/widedeep_arch_new.png      | Bin 96559 -> 0 bytes
 docs/figures/widedeep_logo.png          | Bin 85253 -> 0 bytes
 docs/index.rst                          | 160 -------------
 docs/installation.rst                   |  44 ----
 docs/losses.rst                         |  65 ------
 docs/make.bat                           |  35 ---
 docs/metrics.rst                        |  51 ----
 docs/model_components.rst               |  71 ------
 docs/preprocessing.rst                  |  22 --
 docs/quick_start.rst                    | 134 -----------
 docs/requirements.txt                   |  20 --
 docs/tab2vec.rst                        |   7 -
 docs/trainer.rst                        |  23 --
 docs/utils/deeptabular_utils.rst        |   6 -
 docs/utils/fastai_transforms.rst        |  19 --
 docs/utils/image_utils.rst              |  20 --
 docs/utils/index.rst                    |  23 --
 docs/utils/text_utils.rst               |  15 --
 37 files changed, 1152 deletions(-)
 delete mode 100644 docs/Makefile
 delete mode 100644 docs/_static/custom.css
 delete mode 100644 docs/_static/img/widedeep_logo_docs.ico
 delete mode 100644 docs/bayesian_models.rst
 delete mode 100644 docs/callbacks.rst
 delete mode 100644 docs/conf.py
 delete mode 100644 docs/dataloaders.rst
 delete mode 100644 docs/examples.rst
 delete mode 100644 docs/figures/arch_1.png
 delete mode 100644 docs/figures/arch_2.png
 delete mode 100644 docs/figures/arch_3.png
 delete mode 100644 docs/figures/arch_4.png
 delete mode 100644 docs/figures/arch_5.png
 delete mode 100644 docs/figures/arch_6.png
 delete mode 100644 docs/figures/arch_7.png
 delete mode 100644 docs/figures/arch_8.png
 delete mode 100644 docs/figures/architecture_1_math.png
 delete mode 100644 docs/figures/architecture_2_math.png
 delete mode 100644 docs/figures/widedeep_arch.png
 delete mode 100644 docs/figures/widedeep_arch_new.png
 delete mode 100644 docs/figures/widedeep_logo.png
 delete mode 100644 docs/index.rst
 delete mode 100644 docs/installation.rst
 delete mode 100644 docs/losses.rst
 delete mode 100644 docs/make.bat
 delete mode 100644 docs/metrics.rst
 delete mode 100644 docs/model_components.rst
 delete mode 100644 docs/preprocessing.rst
 delete mode 100644 docs/quick_start.rst
 delete mode 100644 docs/requirements.txt
 delete mode 100644 docs/tab2vec.rst
 delete mode 100644 docs/trainer.rst
 delete mode 100644 docs/utils/deeptabular_utils.rst
 delete mode 100644 docs/utils/fastai_transforms.rst
 delete mode 100644 docs/utils/image_utils.rst
 delete mode 100644 docs/utils/index.rst
 delete mode 100644 docs/utils/text_utils.rst

diff --git a/docs/Makefile b/docs/Makefile
deleted file mode 100644
index c1d05ae8..00000000
--- a/docs/Makefile
+++ /dev/null
@@ -1,21 +0,0 @@
-# Minimal makefile for Sphinx documentation
-#
-
-# You can set these variables from the command line, and also
-# from the environment for the first two.
-SPHINXOPTS    ?=
-SPHINXPROJ    = pytorch_widedeep
-SPHINXBUILD   ?= sphinx-build
-SOURCEDIR     = .
-BUILDDIR      = _build
-
-# Put it first so that "make" without argument is like "make help".
-help:
-	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
-
-.PHONY: help Makefile
-
-# Catch-all target: route all unknown targets to Sphinx using the new
-# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
-%: Makefile
-	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
diff --git a/docs/_static/custom.css b/docs/_static/custom.css
deleted file mode 100644
index cbe8279e..00000000
--- a/docs/_static/custom.css
+++ /dev/null
@@ -1,45 +0,0 @@
-body a {
-  font-weight: bold;
-}
-
-.math {
-    text-align: left;
-}
-
-.eqno {
-    float: right;
-}
-
-div.ethical-rtd {
-  /* hide ads */
-  display: none;
-}
-
-.rst-content dl:not(.docutils) dl dt strong {
-  padding-left: 6pt;
-}
-
-.rst-content dl:not(.docutils) dl dt span {
-  color: #777;
-}
-
-.rst-content dl:not(.docutils) dl dt span.classifier-delimiter {
-  padding-left: 6pt;
-  padding-right: 6pt;
-}
-
-.rst-content dl:not(.docutils) dl dt span.classifier {
-  padding-right: 6pt;
-}
-
-.rst-content dl:not(.docutils) dl dt a.reference.internal span.xref.std.std-term {
-  color: #2980B9;
-}
-
-.wy-nav-content {
-    max-width: none; !important;
-}
-
-div.container a.header-logo {
-  background-image: url("../figures/widedeep_logo.png");
-}
diff --git a/docs/_static/img/widedeep_logo_docs.ico b/docs/_static/img/widedeep_logo_docs.ico
deleted file mode 100644
index 5e699b7ea222f5e43e05c969cde6c92a4b110b4d..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 102115
zcmeHQ30zHS8$VZ>Qqon{LRnIZ2$ikP7G^9%#Au<0GE@jjQTXgF-+WoJkNJi%W7Ib?
zBcpCn2~iAVNh(YeSwhB@(*2&3t~uTJw%u;&dY|+AJ)Lu&cYXf<=UvWw-_Bte9`gyq
z9y~^$S=xeO=E8Mw@Vk2>RUUI2bhfq?_rn=RNsGty>-X;dumQuEI6zAMihDl8T=nNM
zd^ofDc`7qZz-At^Y0?CH4K*D#$Xa9Un30p&P*zmgqMa5WX3OOHES})N%lOuB7sIsZ
zJ9ea<!}820cjkvUc<|0X7g~AD^;ovUIVx?bI<MK7UOUzd*KV=3-IdWT4~2SpxtToI
z(YgHNc=O1fp$|8>jrn=n*q{E>p_hx6`_+L{b&b~j{`>kbN`u?*4rHyLqm*{$Y5K*T
zY5s46%^YU=HDB)+nJ+3C6L|{?C>!b_yclH$m61HW)vgN}hiJ`dK_*v<U%i^$^Xom^
zhL%P@-1OJ#J{GE@JGB2k<Y(taT`z3)z4dDS^N3~bhGulqITe_4a*>JPeqiBfQEMhJ
z*~T$?pmTsthdVJrg-LlvZ}yn9v3`1Qv9s%rnA!J^Ypgx7#^H_M_$fXC^Af+CSu|yz
zf4cCJot`QaVs>^={>9zCJty?>^RseEpQC4yzRoeg*+MmQLF95%v*qXBG%wUEc)9ZS
z!a2&!+>$*lE*&h&7wF{|9SQLbz42u8Q7bp?^M4!_W+%+Mkm<V6bJ9B1;Mk)PGv}UN
z=egEoo44gQ|6yCb_+{3g9h{c!pSxs3;q5Kq-I#B)XAH~uYk>WZt-qy<t`D+|y{Fet
zb(vv-?o-#;=(peGeeP)f+Ceq*<ihRePt190;t{ZFrL*e`VM^u*yCLz9qm(>ey%uOL
zw95@(yyFhXd!5$^dZFtc;;Ze{CO*z<#iQjpLY`@E+u%s3$)Rn&w=6?^lj1^E*CcGX
zbji;&Gh%RlN!g@S6Q-+A`tED|_TRRTvn~!b>lx;ea_jLfKkK3U``Yen&2&u};4$sw
z<cF<th5}A))m%fZhaFEIylTh;Z65#U`w=HXH><DHi!2!YSbdz8)%<Liele$o{5i_D
zwk6kXBG3GAQTSOW%aZKHW{$?`MtciO*A#ZXpc)*kdE+PD$D32~1b$8vPJSPdC-@`$
zU|`{-QH=7RVV#TPOm>(A9m|-y@xPwaQp1kMtYkyJ%K9N8&MmFDG#(Ot?K}JV#fNhQ
z9*4Uz=1<RM+-PNCD!3yI%T7Lc?|XHR+eh+tM@&|=4NdE8m8E5Wr9;lp725g7`s^H^
z@PjaZL~GvRMYD~qvV`lc2j5ukcJ-44L&Hswk?B`yAur+&^fe9Bjj?Z~&1ganzIng%
zb9CQyop&^kpQzMqQi+y8<)umv@8w$Nn!r0p>CJ)1drV$;Ik1tr7W?Pr*@i299z^`&
zY_~G0SI&Sot%G%Tcz)$w6o0_y*Hd{<Gme+Ilt+x6^x%J+zDyT}xy<nWGBNH{k-)<U
z%8&Z{K=koj`Zpg1*!!#*>6ZFBZ)&m2!Cm%X`)ePY&s#ovBr0Homb^V8umsDS{U^zO
zLI1N3dC}QF+b!&8w(xR^z>n1zq=wvS|J~K-;_O9(?Tq8iif&G^ObR!iua)n*@ur3Q
z<8R$kAND!4**f1sHT}@~@IBU|__c<CKA)$UFs<E#UR=|)*cq524BOKDhnGeFO(?w-
z0<8V`*AbqvPszoZJ=+Z#vh2>kZJJ+K(Os{ydv3;FE8)bMzd`cROmdv{yazk(IeD*r
zUeaZ?n{(R6iG6OaT)w-oWJKo;jN6%+Jr^9G?Y8#>@A~$L5$mn3@7vqtop)`n8XUPU
zx^VGBGxt@V5$9dApJ{q~b+TwR{l=hq7cxHS<6KY@6?~U(dp*ow?^F9HhWnSM+_StG
zy1di$nd1hUM~@HpUil*I(y8V7uG_%j4;L3cO$$F&xFtMu<cqQHUUM>%G}k!Y$vqI>
zYL{h@`NXn<GbT*r(J5a%-CR`sca+n<!1(B)hmYLe`&DP*FLsYC-+E;ppC|A!E>qzJ
zFG}CJcgtMOjr~&-bnUPF_~`lk<OTixdoNh9V(#l>DlV~qKl~-|kLbzDjA-eDV!hp0
zgI4*EwDw&v>g1_M2NqlJ@^s27Ox>uF<TlAs=;CHu6kRYn?9Kfw$DD%W+c!)%zaO(U
zEwdzf^P$J9-}=5VoS5alr)1={?HxVt4H|g%3->*X(o@c!yxB9tKFMLl#LJgsZ3cUu
z+kQpU`dCob4ae<)h4vyo^YZ*)kI#f-Z@<d?^~)P&LCPsn6RutxcI?`Kr7Qdkj?I}`
zY&zm)hJnXgFK;i^5VNUE(+9<UG3Th2r)k{Y*c<M}myT*IJsY{hI5saOB{in6%ae7M
zCoEc;on7!#u0W-|UdFzJ^((_WxOX|6D>%3%+~fWh(cN+1ug#uvx^?WymZdgjKAy%U
zH~ob=efLK_6Z~oJvTEEqq4Sy|hMBF*@JGw#wCtd(`l>`>KZnqBFi4~u@Lp%=wTgWj
zWYEs`@4I|?l_H<IVlHwby&-_y{>+LoH}DDR=yLi|hRglD+s1K)kIh}m=DA;tX`9vF
z@!4<ev`O-?9aXCP&F$txH3K8ZvteULO&A$9eAXHxxDI9&kcgd(y*qe}K|7{gDinA^
z1P}p401-e05CKF05kLeG0Ym^1Km-th3Ix<T#We%jTnF)}4m`+5r4D(q<&LW%zaDkW
z4;C=+s4W73tulzH4w6t^-jI(*9r9w!-Bd$<BkGvn>gw{tu*wJkw#a{FNT|9){;8sy
zFc$eIAfZZu{8L3YVJz}bKth!Q`KO9*!dT>=fP^Xq@=q1rgt5p!0SQ$K<ew_K31g9e
z0urhe$Ujwd6UHL{1SC``kbkP^CX7Y?2}r0?Apca+O&E*(6Od4)K>n$sn=lsnCm^9p
zf&5cNH(@ODPe4MI0{N$kZo*jPpMZoa1@cc7-Gs5oKLH6<3gn+Ex(Q>Ee*zM!6v#hS
zbQ8uR{{$pdDUg4v=q8Ls{s~B^QXv0S(M=eO{1cE+r9l3vqMI-l`6nQuN`d@SMK@tA
z@=riQl>+&vif+PK<ez|qDh2XS72Slf$UgxIRSM*vD!K_{k$(absuajSRdf@^BL4&=
zR4I^us^}(+Mg9p$s8S&RRMAZsi~JLiP^CcrsiK=O7WpS2p-O@LQ$;snEb>o4LX`sf
zr;2XESmd98genE{PZiySvB*CG2~`T@pDMZuW08LX5~>u)KUH)S#v=a&BvdJof2!yv
zj79zlNT^aE|5VXU7>oQ9kWi&S{;8syFc$eIAfZZu{8L3YVJz}bKth!Q`KO9*!dT>=
zfP^Xq@=q1rgt5p!0SQ$K<ew_K31g9e0urhe$Ujwd6UHL{1SC``Xy6}gv;^@&iBp$!
zZc6xH4R(r9;&hOIs{H`@r@J3O{*nJmLjaEdRE-DN{;~aI`={3ju>I4^J^TEN{3HL^
z|I_OO*#7C|p634F8^*p&l(=@4I)AvSc>b>oHs49+!M%t4)a!`zQf_+5YshPG9rN>+
z%meQk=F{ov+D3ztC-b`r7V*2qFKO8&exY^`!Bj(w<em|`?(z_$j}HNnNW^sLmC#+Q
zyWjxu{R-L<@f(<W=eRP>uJISz_Dpc}4Y=Oy<3oW?i#(TPF&cXDUcj$Z-lh41rhdFY
zzuy^Rjz}khifeVRAt}v(`!H!7*QA4VxgCuY4HVaUtT}M#UlY&Og+uyV-TEi-5lO{E
z0LFR?;I^bXerpp3W5KSFt=WnP0L3+)=#;^0r57)#4Zk(TQ3oGtZ<J_AF`No#I?UaA
z*TidG!eJcPq;McmTw$#qf_jbnlD+}Y4!`f;pF=UCie)-H$M2Qous-*||I6+J&ZsFC
z3iQ?%xQ?k0-%`?+fe&<|7g5Es9Oi7%QaB{m{OS;g<zH8^P@uQAt@Ps~>%eV|e!=$t
zBPU<P{$D{{s*ORye0lN@zZLyaQ20hctjB8Vo1N(lZMNQPKVl!!Hb_{5s45NuKPD6~
zu>NyM%wK)ZY~PQ+1t|{q;+n!3pby+Vs}J{VT3G+}V}C!!L-7y*?i_)`QdxbVWtX@Z
zo!$w`if1|2m_?wUO)3L(w=aSJI?n}Kc8)u)Z6F~2PaKg|JOsc8MnGH6u4O+Uggrt*
zZ445dE1soT<Ejwo*8hwe{4U@HYXccDFDR4r1>pQI2>Tj-H||zd9;jD*1bhN+@XSZ1
zcF^s8+P3?Eq{%S1x9?(l+OYSq6ddOjpI)>85kLeG0Ym^1Km-s0L;w+>9syXt{xC<d
z(rd;hSm}M?c#n9XKVW+hcL*P8e~?%);;2;Er|9=$x^nfzSMSb@N~QZU<{!^y14<cQ
zrSMXQsdWFEVS>1dXEop3NW_=-eUL={mF^|$D`}6Uy&?%bM8TTe>_wtno67eMb!P<r
zXbTby!ejK|FC4ZEV*qbKU<D6E01;?<2*7VtJdjaRWEi)$5Z+8m95$V~M7$PN#2d(s
z*Ho@6?2@)Wx6(i3K)g>tq{U0-+5&O9RuM0e8?UKcLtR;3rP+n{b4UV=kbmiZg8pCG
zPq6(<_me~pOiktz`#<deas0#aUy2Xl{D;#^<NOEbe^S~A``m!@zsl_d=YKf=llDBF
z$Z?Y<a|!JipZ}%%D~|shFU9d6$A29Er9LO&{HJn1f%6~f<4Gb1rY3WV<3G-Ss_55o
z{=;ddIRC-<PZjM1$N$Rh1jm0I|8f0;^npZ9GifrHIRC@>UlrpT&i^>ARG#xc$g3BK
z<p&le);y>~qp$x$nTa6M#)!t$X`ubcKl(qm|0?>T1{?pNykj6wKCrap^ncRolEH?H
z52!1<|5aMNU&V3QGsMxWBHl4>ybpB^-{@=6y1QULyoc$3@Q$XF@ZF7la6bmV$M>PC
z)M(|x`$~I&%{PG`Vw?pWqTpL(EgG#dja(4Cb4vs4&XQ)kSf|k?{+o~WJ50d+3bCEl
zoZ0VF_HJZq8l^aR_gYFdcE4{A&_0$lN`)G^5U^cpyO)AVXd_pxQHlfpYxT?l?{ls8
z8_A7Q*#;;C`gb2GT)+4Gim=kQh6x|#cgWyf&8pD0uT+NtyuWh`{C{i%px-pIgm;v7
z1PoH8VDJ|7Vel^6@-=~`QQL;81-zqE32>MQ^S^B{)*XT||6kDN`!q~xm@6WH2yhXo
z?)?MLI;oVu%>mDK#~~O%<^z6pJ+pjNyh5X_;?)_Y6|c;w>sg#$RQVMhl^JK>6VVT3
z7RdYeMD)P-L;xJr9xxwapUnqKh^iP6&>KmFb>W0*NrbWaH0nGC%CiHJoJ+H=UVcDW
zvP5_UCv1j97@JR{eh(bV!}?d|5Ahc}Xf4j@KSbL{|EY!Fh@+tYp#O+{L6$T65BiTR
zej`qU{)7G__61qa=s)N`viOZS4f+rIkJuMvIivre|H$Gu;xy<#=s#j#kmcME{sVa`
zfw24J8&b;OOx-~B=R>}dYr^ur-^B8T^(Sy<u}A!C_P>Vo9~IzWaRZ7k?Bno;{TrXc
z{*BL|oFx)vX^HzX(6is@=GJ96Cyf1Vi5tdVtBAuJ;tYE++rWOC-=G}vH)6|R?@<Em
zL)K}C^5mH(>=9Oix}S!+6MY%z&%-xQo5{00dFRFN8gIqHov;D?$wuB~$}>N}c(yd`
zIXVw{mM8DLpskMK)Rn*=_GC?zcbW3cuVNpe)V;L8O}36n)89i6dqGX0t~nAs<iH-;
zzVgJXA@hQLs%o%@ZwB~F0N68w-&LU2kmX_Cjf=p$T|?rp`IMh?XC=~e2DJ5AduwrK
z-r-UP+pbuAN?|)!ZO-EOBKX2mIj#WtKWBFHiav-f$oqX-@T>O6`aK`$3tNM9uMkZz
zn|-d*;^@TxFWg2_$HuMi0eP+l5s!xp+5F&sEl1~8q5C&SH;dKD<A6M|e8eC9@0|}|
z`-4se`ycFoIb#iuKkwQEj=%qmle6S8_h9pob^ZbA)IfCWDZF}=KMT@|$4yz!?7J0Y
zntwsM4_&?!)ZtY<g|$DBlgEMzpIF7wxm4)3aCA<rPSyj~VOzkuYXq!?WP+T8^#oH{
z^O7YUl(`dZ;rdKItY@gpl9w!LpzM5!@<mW5TUqjwB@L97CQ*JFtm!w}{~rVQ!=U`<
z;yOTmHk3aE%VLa-X<By^w1G9ziLi#Hkndf1w@!10FA~X57^WGlH2fo?GWe)*87xz9
zMWk0=G`mg#k;~UA-mg<A%eGDde!%`8QcDnJ5OzI){SCSmNcp>RYN8n8C4!s)u>)b-
zwHe4ckm#Dyvf&U9(|@G+0k`aTaamugO4#2^tK=WTqCvQ0h2%B6R#aX3sSqAeg{*|W
zZx_gR2%iq3R}rrRoM(Ydt<HZT9UF(OPcd9;R~J`W7^E2vX-h!vLjNz*IzEh>9ib1c
Ya_kfZ!`fyogj>HdJ%(XLWWzB32bp(QjQ{`u

diff --git a/docs/bayesian_models.rst b/docs/bayesian_models.rst
deleted file mode 100644
index 2eb95c60..00000000
--- a/docs/bayesian_models.rst
+++ /dev/null
@@ -1,15 +0,0 @@
-The ``bayesian models`` module
-==============================
-
-This module contains the two Bayesian Models available in this library, namely
-the bayesian version of the ``Wide`` and ``TabMlp`` models, referred as
-``BayesianWide`` and ``BayesianTabMlp``
-
-
-.. autoclass:: pytorch_widedeep.bayesian_models.tabular.bayesian_linear.bayesian_wide.BayesianWide
-    :exclude-members: forward
-    :members:
-
-.. autoclass:: pytorch_widedeep.bayesian_models.tabular.bayesian_mlp.bayesian_tab_mlp.BayesianTabMlp
-    :exclude-members: forward
-    :members:
diff --git a/docs/callbacks.rst b/docs/callbacks.rst
deleted file mode 100644
index e8e65c45..00000000
--- a/docs/callbacks.rst
+++ /dev/null
@@ -1,29 +0,0 @@
-Callbacks
-=========
-
-Here are the 5 callbacks available in ``pytorch-widedepp``: ``History``,
-``LRHistory``, ``ModelCheckpoint``, ``EarlyStopping`` and ``RayTuneReporter``.
-
-.. note:: ``History`` runs by default, so it should not be passed
-    to the ``Trainer``
-
-.. autoclass:: pytorch_widedeep.callbacks.History
-	:members:
-
-.. autoclass:: pytorch_widedeep.callbacks.LRShedulerCallback
-	:members:
-
-.. autoclass:: pytorch_widedeep.callbacks.MetricCallback
-	:members:
-
-.. autoclass:: pytorch_widedeep.callbacks.LRHistory
-	:members:
-
-.. autoclass:: pytorch_widedeep.callbacks.ModelCheckpoint
-	:members:
-
-.. autoclass:: pytorch_widedeep.callbacks.EarlyStopping
-	:members:
-
-.. autoclass:: pytorch_widedeep.callbacks.RayTuneReporter
-	:members:
diff --git a/docs/conf.py b/docs/conf.py
deleted file mode 100644
index bcfaa096..00000000
--- a/docs/conf.py
+++ /dev/null
@@ -1,297 +0,0 @@
-# -*- coding: utf-8 -*-
-#
-# Configuration file for the Sphinx documentation builder.
-#
-# This file only contains a selection of the most common options. For a full
-# list see the documentation:
-# https://www.sphinx-doc.org/en/master/usage/configuration.html
-
-# -- Path setup --------------------------------------------------------------
-# If extensions (or modules to document with autodoc) are in another directory,
-# add these directories to sys.path here. If the directory is relative to the
-# documentation root, use os.path.abspath to make it absolute, like shown here.
-#
-
-import os
-import re
-import sys
-
-from sphinx.ext.napoleon.docstring import GoogleDocstring
-
-# this adds the equivalent of "../../" to the python path
-PACKAGEDIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
-sys.path.insert(0, PACKAGEDIR)
-
-
-# -- Project information -----------------------------------------------------
-
-project = "pytorch-widedeep"
-copyright = "2021, Javier Rodriguez Zaurin"
-author = "Javier Rodriguez Zaurin"
-
-# # The full version, including alpha/beta/rc tags
-# def get_version():
-#     r"""
-
-#     Get the current version number for the library
-#     Returns
-#     -------
-#     String
-#         Of the form "<major>.<minor>.<micro>", in which "major", "minor" and "micro" are numbers
-
-#   """
-#     with open("../pytorch_widedeep/VERSION") as f:
-#         return f.read().strip()
-# release = get_version()
-
-with open(os.path.join(PACKAGEDIR, "pytorch_widedeep", "version.py")) as f:
-    version = re.search(r"__version__ \= \"(\d+\.\d+\.\d+)\"", f.read())
-    assert version is not None, "can't parse __version__ from __init__.py"
-    version = version.groups()[0]  # type: ignore[assignment]
-    assert len(version.split(".")) == 3, "bad version spec"  # type: ignore[attr-defined]
-    release = version
-
-# -- General configuration ---------------------------------------------------
-
-# If your documentation needs a minimal Sphinx version, state it here.
-#
-# needs_sphinx = '1.0'
-
-# Add any Sphinx extension module names here, as strings. They can be
-# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
-# ones.
-
-extensions = [
-    "sphinx.ext.autosummary",
-    "sphinx.ext.autodoc",
-    "sphinx_autodoc_typehints",
-    "sphinx.ext.coverage",
-    "sphinx.ext.napoleon",
-    "sphinx.ext.mathjax",
-    "recommonmark",
-    "sphinx.ext.doctest",
-    "sphinx.ext.todo",
-    "sphinx.ext.viewcode",
-    "sphinx_markdown_tables",
-    "sphinx_copybutton",
-    "sphinx.ext.githubpages",
-]
-
-autosummary_generate = True
-
-napoleon_use_ivar = True
-
-# Add any paths that contain templates here, relative to this directory.
-templates_path = ["_templates"]
-
-# The suffix(es) of source filenames.
-# You can specify multiple suffix as a list of string:
-#
-source_suffix = [".rst", ".md"]
-# source_suffix = '.rst'
-
-# The master toctree document.
-master_doc = "index"
-
-# The language for content autogenerated by Sphinx. Refer to documentation
-# for a list of supported languages.
-#
-# This is also used if you do content translation via gettext catalogs.
-# Usually you set "language" from the command line for these cases.
-language = None
-
-# List of patterns, relative to source directory, that match files and
-# directories to ignore when looking for source files.
-# This pattern also affects html_static_path and html_extra_path.
-exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"]
-
-# The name of the Pygments (syntax highlighting) style to use.
-pygments_style = "sphinx"
-
-# Remove the prompt when copying examples
-copybutton_prompt_text = ">>> "
-
-# autoclass_content = "init"  # 'both'
-autodoc_member_order = "bysource"
-# autodoc_default_flags = ["show-inheritance"]
-
-# -- Options for HTML output -------------------------------------------------
-
-# The theme to use for HTML and HTML Help pages.  See the documentation for
-# a list of builtin themes.
-#
-html_theme = "sphinx_rtd_theme"
-
-# Theme options are theme-specific and customize the look and feel of a theme
-# further.  For a list of options available for each theme, see the
-# documentation.
-#
-# html_theme_options = {"analytics_id": "UA-83738774-2"}
-
-# html_theme_options = {
-#     "canonical_url": "",
-#     # 'analytics_id': 'UA-XXXXXXX-1',  #  Provided by Google in your dashboard
-#     "logo_only": False,
-#     "display_version": True,
-#     "prev_next_buttons_location": "bottom",
-#     "style_external_links": False,
-#     # Toc options
-#     "collapse_navigation": True,
-#     "sticky_navigation": True,
-#     "navigation_depth": 4,
-#     "includehidden": True,
-#     "titles_only": False,
-# }
-
-# Add any paths that contain custom static files (such as style sheets) here,
-# relative to this directory. They are copied after the builtin static files,
-# so a file named "default.css" will overwrite the builtin "default.css".
-html_static_path = ["_static"]
-
-# Custom sidebar templates, must be a dictionary that maps document names
-# to template names.
-#
-# The default sidebars (for documents that don't match any pattern) are
-# defined by theme itself.  Builtin themes are using these templates by
-# default: ``['localtoc.html', 'relations.html', 'sourcelink.html',
-# 'searchbox.html']``.
-#
-# html_sidebars = {}
-
-# This must be the name of an image file (path relative to the configuration
-# directory) that is the favicon of the docs. Modern browsers use this as
-# the icon for tabs, windows and bookmarks. It should be a Windows-style
-# icon file (.ico).
-html_favicon = "_static/img/widedeep_logo_docs.ico"
-html_logo = "figures/widedeep_logo.png"
-html_theme_options = {
-    "canonical_url": "https://pytorch-widedeep.readthedocs.io/en/latest/",
-    "collapse_navigation": False,
-    "logo_only": False,
-    "display_version": True,
-}
-# html_favicon = "_static/img/widedeep_logo_docs.ico"
-
-
-# -- Options for HTMLHelp output ---------------------------------------------
-
-# Output file base name for HTML help builder.
-htmlhelp_basename = "pytorch_widedeepdoc"
-
-
-# -- Options for LaTeX output ------------------------------------------------
-
-latex_elements = {  # type: ignore[var-annotated]
-    # The paper size ('letterpaper' or 'a4paper').
-    #
-    # 'papersize': 'letterpaper',
-    # The font size ('10pt', '11pt' or '12pt').
-    #
-    # 'pointsize': '10pt',
-    # Additional stuff for the LaTeX preamble.
-    #
-    # 'preamble': '',
-    # Latex figure (float) alignment
-    #
-    # 'figure_align': 'htbp',
-}
-
-# Grouping the document tree into LaTeX files. List of tuples
-# (source start file, target name, title,
-#  author, documentclass [howto, manual, or own class]).
-latex_documents = [
-    (
-        master_doc,
-        "pytorch_widedeep.tex",
-        "pytorch_widedeep Documentation",
-        "Javier Rodriguez Zaurin",
-        "manual",
-    ),
-]
-
-
-# -- Options for manual page output ------------------------------------------
-
-# One entry per manual page. List of tuples
-# (source start file, name, description, authors, manual section).
-man_pages = [
-    (master_doc, "pytorch_widedeep", "pytorch_widedeep Documentation", [author], 1)
-]
-
-
-# -- Options for Texinfo output ----------------------------------------------
-
-# Grouping the document tree into Texinfo files. List of tuples
-# (source start file, target name, title, author,
-#  dir menu entry, description, category)
-texinfo_documents = [
-    (
-        master_doc,
-        "pytorch_widedeep",
-        "pytorch_widedeep Documentation",
-        author,
-        "pytorch_widedeep",
-        "One line description of project.",
-        "Miscellaneous",
-    ),
-]
-
-
-# -- Options for Epub output -------------------------------------------------
-
-# Bibliographic Dublin Core info.
-epub_title = project
-
-# The unique identifier of the text. This can be a ISBN number
-# or the project homepage.
-#
-# epub_identifier = ''
-
-# A unique identification for the text.
-#
-# epub_uid = ''
-
-# A list of files that should not be packed into the epub file.
-epub_exclude_files = ["search.html"]
-
-# If true, `todo` and `todoList` produce output, else they produce nothing.
-todo_include_todos = True
-
-
-def setup(app):
-    app.add_css_file("custom.css")
-
-
-# -- Extensions to the  Napoleon GoogleDocstring class ---------------------
-# first, we define new methods for any new sections and add them to the class
-def parse_keys_section(self, section):
-    return self._format_fields("Keys", self._consume_fields())
-
-
-GoogleDocstring._parse_keys_section = parse_keys_section  # type: ignore[attr-defined]
-
-
-def parse_attributes_section(self, section):
-    return self._format_fields("Attributes", self._consume_fields())
-
-
-GoogleDocstring._parse_attributes_section = parse_attributes_section  # type: ignore[assignment]
-
-
-def parse_class_attributes_section(self, section):
-    return self._format_fields("Class Attributes", self._consume_fields())
-
-
-GoogleDocstring._parse_class_attributes_section = parse_class_attributes_section  # type: ignore[attr-defined]
-
-
-# we now patch the parse method to guarantee that the the above methods are
-# assigned to the _section dict
-def patched_parse(self):
-    self._sections["keys"] = self._parse_keys_section
-    self._sections["class attributes"] = self._parse_class_attributes_section
-    self._unpatched_parse()
-
-
-GoogleDocstring._unpatched_parse = GoogleDocstring._parse  # type: ignore[attr-defined]
-GoogleDocstring._parse = patched_parse  # type: ignore[assignment]
diff --git a/docs/dataloaders.rst b/docs/dataloaders.rst
deleted file mode 100644
index 02f9515e..00000000
--- a/docs/dataloaders.rst
+++ /dev/null
@@ -1,10 +0,0 @@
-Dataloaders
-===========
-
-.. note:: This module should contain custom dataloaders that the user might want to
-	implement. At the moment ``pytorch-widedeep`` offers one custom dataloader,
-	``DataLoaderImbalanced``.
-
-
-.. autoclass:: pytorch_widedeep.dataloaders.DataLoaderImbalanced
-	:members:
diff --git a/docs/examples.rst b/docs/examples.rst
deleted file mode 100644
index 489560fa..00000000
--- a/docs/examples.rst
+++ /dev/null
@@ -1,20 +0,0 @@
-pytorch-widedeep Examples
-*****************************
-
-This section provides links to example notebooks that may be helpful to better
-understand the functionalities withing ``pytorch-widedeep`` and how to use
-them to address different problems
-
-* `Preprocessors and Utils <https://github.com/jrzaurin/pytorch-widedeep/blob/master/examples/notebooks/01_Preprocessors_and_utils.ipynb>`__
-* `Model Components <https://github.com/jrzaurin/pytorch-widedeep/blob/master/examples/notebooks/02_model_components.ipynb>`__
-* `Binary Classification with default parameters <https://github.com/jrzaurin/pytorch-widedeep/blob/master/examples/notebooks/03_Binary_Classification_with_Defaults.ipynb>`__
-* `Regression with Images and Text <https://github.com/jrzaurin/pytorch-widedeep/blob/master/examples/notebooks/04_regression_with_images_and_text.ipynb>`__
-* `Save and Load Model and Artifacts <https://github.com/jrzaurin/pytorch-widedeep/blob/master/examples/notebooks/05_save_and_load_model_and_artifacts.ipynb>`__
-* `FineTune routines <https://github.com/jrzaurin/pytorch-widedeep/blob/master/examples/notebooks/06_fineTune_and_warmup.ipynb>`__
-* `Custom Components <https://github.com/jrzaurin/pytorch-widedeep/blob/master/examples/notebooks/07_Custom_Components.ipynb>`__
-* `Using Custom DataLoaders and Torchmetrics <https://github.com/jrzaurin/pytorch-widedeep/blob/master/examples/notebooks/08_custom_dataLoader_imbalanced_dataset.ipynb>`__
-* `Extracting Embeddings <https://github.com/jrzaurin/pytorch-widedeep/blob/master/examples/notebooks/09_extracting_embeddings.ipynb>`__
-* `HyperParameter Tuning With RayTune <https://github.com/jrzaurin/pytorch-widedeep/blob/master/examples/notebooks/10_hyperParameter_tuning_w_raytune_n_wnb.ipynb>`__
-* `Model Uncertainty Prediction <https://github.com/jrzaurin/pytorch-widedeep/blob/master/examples/notebooks/13_Model_Uncertainty_prediction.ipynb>`__
-* `Bayesian Models <https://github.com/jrzaurin/pytorch-widedeep/blob/master/examples/notebooks/14_bayesian_models.ipynb>`__
-
diff --git a/docs/figures/arch_1.png b/docs/figures/arch_1.png
deleted file mode 100644
index 7d829a0c3cb72139afbfaae1470cf0997e9179fd..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 58023
zcmdSAcQ}{r8$bT`MxjE99?1+Dp%B?*%cks+y~*Aqg-T`bRW>0znU#{gw~)Q}p1<?<
zd_K?TIlkZD-@oJdISzTpecktUU)OoQ&e!=m&+Db4yyPXk>v$*>>XNjSm@*252_Mm%
zIOpI;4!OJ^3WZl>E-I=hEh<W-Xm4v`Ze@%@Nxh7WKCh^vam{ztgDvqn3iooz6?dXE
zB4L@i_q+5=)VPE)L4GL)k|b*SEI~oOmvqt@K9dOdy~1XGg?)SO*=@Y0RfDeqO{c!p
zR<Wn2vqH8jBddunD?GEK6{a)%C<e!T^#puoR0wrb)bst%9EJvyF?%Aum>a%9f;iF9
zbS8n%u~4cRv$aiutSGq_msWbu<IO`KV(}WP*XL1JpV`w0y^i+Vc<NX+^O9~5MRllD
zy%J`%j-@9O%PK-7R$=-qJk!<a8PP3L|Jah3IGe&6*eG;tb0jUMaG&UQ%MD%jg}|kk
zMK6NbuD%bh!Hf<LmKAI=)7F1J6r_~2)-H1to;+2%CmXRpWgl+LUm7h<?TS}iKR;r8
zI{Wm<woxitzTuAWc^>fzvVgXWTP+l6#s|&n5r19fZ+&o*c1)Wo<#jVNHhTIpcdKW9
zX04UAyJ8*ZE!=7zcA>$9AafEUw-*Y<v>dV|%+eEIlT7lB=<_*Qlrt1A+ZmlWsL*`K
z<dOV~n6p*G{Lcvo7X^E@*me45J*thT$uAdKhCL+ul6RP=e)XATitRP+v?Vs`Q5m%F
zwY&<Q{&~&Sy>&pWY3aw{N|E}+7GZ2P)o;m+_9oouyjBUesg@zaLDCHS;|Cd}u4pzw
zUqM+y+>gEjpRjHd;t7}O&)1V<Q^ZafUwL9L!xjGKajb9S<@J`?J9)Q1TA^aH2D6#h
zS*?mlsDH+i>d<S7ek3qUw2JMdi$Zs3qkNlENfSp|QCa&a&eHX?umr~6VmuhPsW6yQ
zFdwDhI{IC{Bf43PDPMP~b$|fNABXS)37SF*gH#wD<x8y3!hjOj=NLH0RA=*vpvU+7
zID^@_%sMwaQtk8QU%l4;+Iohuh8OHhpnvfh6~CCDSqM`a_Pl6D2yy*0Z7Mbf6rR7q
zdn^XhD<aBf*d_Q(&ty{Y%-B>f9HSGTw~4B~m!G5VIe$sS_C4DjbodkAj~HZi4o^s0
z0*?RSxMCz+WD9Wk<LP=+8(-~v;5hvrzbAPO!S1($aV0D&dmPM8i55bV2|uixPo=KV
zh&-l#Aw{bEhLb8!iqHPqym&xp`(HN#0-hMO+>!rFPnI?yP$O951>;XueTzGCF|T7P
zV+31pX859>J!*(EATPXOb&-p-5=+aM<vT@PQl0L%(FS~dp>ezW=h_HO-);W*F`@jl
z`RR|(iJv7#c(iZjQ*ys*n0&OJ;cDu6C+T(j&uil@;{<=&CsxPLuO;KyW3fCV{u)mx
zNqBh!$M`w#H_~t9-!6RnOUW-w!IAbH(SE&gy<Giq{o@Ii3DnczG%0pko^Tqe?6lG}
zdu7b|E7^ft^6(|uCc>&J&rmLrue}m9LR=wO(QZUnggfYeLm?`RJB%Q#J(MzROo`zt
z<|{I~w_$RFQrYR+Dg5c4VVRBb%??xh8@bcW)A3WQziKnCWFM$<IZ@w>)|3#Cs1_%G
zOvUsUgDZm_@nc3+rUccw$A3LaW7K3i$85qt!raHG`{;)1H)c`>H^%kIjmRBF9mR!A
zYEGTF^KT@zB!?9$#mG`7=K>t@1tN9Dgk)D!-Z3yU4l_tHvOUUI70erH7XItpMf{tI
zg)g5*bpF_5e`rYJ5vAW3pmC{qrR5-tUF(OOc#3jXSC+lr^lK)S-iP<(oL*ZLFA&`e
zj(e$@HFTv^$0E8Xz_i<xuXU!CHNneq=cbeTZ!rN`uZjb~ZL)>b>v7iwuN%A}zkZD~
z;2vq5O<Z<ddYma|V2PO~tLB3|<-B4|w%8_FE{;TvwyIw@Iv@A%^hQ~%_O5hLbl>fH
z+0)o<-sAS+VVChd{e09y+5Fo%^LgxrypG&{k2HlI)A`If+kV^rQ!{-tLhDbXDTDeQ
zia$aOV=UEe2MopT+TYW(Qnhw$|I4(6U7lmq+;bR<tMH!m-E7VZW4!pGSUfqGIOn#n
zZBJuv#;FKo4hLKP<Z8Z~%e}~98go8oHkLcSHYt~z+h!%IV%ecqYd|wiYpLATD%)t@
zG}d&`V!yAc{*~z?bEoz=gJtXLp)WrU-n_|vQ)d$0im4T6owDpc%r$K5yyJYO@?0gg
zhrl7$A=9A^Nh0w#CKH{fb-bVXK04o@UK{LeyF0v|HAyLNB5yA5B=5g@aTa%GIk|uJ
z<7A7|=;FxYH%Do^!O5=0oaH8`JZG<&jK!ZWeB(RMtXt%bDGh3!%k82+C)=IQ{h2H5
znoPJgF+Zs>H~1%gig^CfpY0y42EM_{QIGnDKGTYJ9=(qRwW}m!HgC@bHe^o+?+qU2
zZB@=G>=5i0toe}WU!lS^yYw9M7+rpD2W$TP%V%-^B6n`NTCRl?eh>LUT^lg*K=c8#
zXmX%NKpu`ILFNlKyn+j-gnuXs87H2mDW!21vAnuYO!<gx`|c3O7uqk}CN?ciF^h#3
zh2I9E?t5yhj`)4<+}GVKZ>0<>f5R5Es;U23fBsqSg<g^jHva0MFF}=a>S!)%-uJA3
zn7L!LTFK?6XdlzPUE_Bhb}QJkIZ{3esnbxTaSfeHPz~dZSxTt7=f?2kUixig^*QQ~
zgf-IJa=#M~?y}rp)*o_pcXT&AM}3oeGEGW4g8oZPc5DS_v5FNFE8#Ayvv!sBdQiQd
zJ=KLK&5+(OIm!4`GUc1|1YI>Zq7#}D6drzvx}lZxS+!*ACwsUtSyc6}?Y3h#9upp0
zw_&_xVoSBHJfnh<N<7WT&pFnmD+@=xn3kS>Js;`v<g`3E+Gz@CSC7BUIpytN-?!X2
z^jX_~Un5*2tlUvOYJR*zc~=O}XK6sc`bk5;>Wiaqmf|Gh#^QRf-G20Y7ae6EWv;)G
zCgbB8v>qxYE!QZiuzbB36?f%F*htt(o}X5{Ol<m#^wn~fA?MPT469w|F6a53(GA~?
zC~|i`uGKxwQ01q!UDZW}GSwEMy$y*+i7w6+%N7H9g_UNz(E{HD2zCyZtaG@sSce0L
zql;JfPqL1eg^IoN1yY<-=IYA*KloSQdvfoZ<+7#aV$UL1@AvJlgZHBAfi{#o{3BiZ
zeV@uhJJlVrZiG^}?LYj+rXzX$u8iD}-12_DPt^j}ve*-%o|2E|X*}r_<#c7~n`vW>
zWsfuah-%|b-Pa;^vYWb_GPx6}6B~s%yt#Il=6KdC{%khR9uAX~3e*rf{2Bex@tQpP
zh)t(r$!q@sU*+)<@ho#@KCiBT+Z$NWFTb}l$jZ3oM@p*QK77wlosT-siyFo+*W#OX
zN~)N2pFPrPuZ&)Ox}sA{=~!FLT(l_S;_0%AdE);yaGqd{zsZer$5oD?EOYJUxA3CO
z1%+H6j+4B?{`_6XWYe)XrhnQgb~g|7j+3?z`~8k)BW8`+N_2bhH7_My-=O$%Y=6@1
zr#7Kx_#OLH`ADtTGPS?q)b4Qdr&P<YI<m(2_3|I9s$=AywwpE9oJL#ofx!fZloy1u
zy>s?k4n|Js4w_#V(&?@1HP@2*SRSvOR-5u0dTO^s+TGH^z#_L(zJjtPKnuUb_H`yh
zbsc3Vxhtn+pZZFpPhVyptVkDP>g#c%E;nP~e8#PNi&v5^eylQyf%E=@`z@4~R`XBC
zQ@+7mg`jWLsGW;i=Z3KPJx|)l-ewD)D7c+^og>n<t{a>6!TcGBGqE|@unHvhm9e_C
ziJTmY2|nYXurRKpu;CL1d<$Zb{qtD@gW-(%VxZ6{RG>Ku>p$1X!%yTZ1iq0mfB(b`
z3qYNR|6PG^*A(=Buf{~BVE*?rniKwp5>XMAmWH1yhW5tBHV$UCj)vZ1$h~pxq%<5*
zC=xp48$()|b`9>|XRfO5s4geVYiMiD`ozfAz?jw5+77u7ir<wNK3W?)KB028wz6^H
zbrqmFyMh-!Bfn;&p*p+7(Ncg$T~3io)YjgZii`CQ>m3?FJSr+GetRPmUS%<f{|tx!
z3DB51I@<BFvAMXou)1)t+S;44-R0roVY|c5#?H<HSFkv^**HFNWwCL%`S(r!dmk}l
z2Sa;vJ4bU{8!F_!PYi6G90h1-kO%$q_4hoDUCsaZBpZkSycWD58}b*nyR3KE{<${{
z<wt(Yt7z_OY^5P)ZVhV&&k(%F#mRqm{r~ls|2^?vBh~*mlAG)PzeoP-FaPJrM-ImJ
zqPEuXNJqi{ZP$MW|ND>s8OYCuy!pRY;_qcX`z@@qARa&4KYJ#KmtnHC2ithVTuea~
zeu9x9Uto+V6vN*?k<ZEx3;Za-UW8H7Vj`-p7)!r#t5he+>egp%%+o)nUroO%^O~kH
zOu$-IM(ouG9O9QG&uNL37{d}DD=9lOQL~hN2x1UFq*-uYJsO*xZt*GObBV7ojT+nc
zjEOC-S=~*XscGj~b(4S9Ap7k(3XMbM`=5WLsY4#1Lar{$(p?rtVPFxS{qqTph4Q@o
z_vf=;NYP-ReQ7PMa$fywECy=qzXOnq!lZmrR4=c`=g|E3-Z)gks5+N_?kbG(qrF0f
zlKSv7Bj<k;z#&Afk^k?}!c;G@a8O}ch1pF1nk7u<{Qu|ENx{LQ@Hu)K%Wa#~5_j(*
zKJQ#g){Uo|Gp+WGbkV=6cSo67G)sb-?hY`baq4SP7{W*8)c4NbB<QK1B&C<?%I8UZ
zX*s#b-7z1%HlEpptO(bM`j=g)k_3&veuXfqWc)IY)**?U{G@_K7-S18yk{dM^A1^I
z!bvoNz*fkJL%TrPrjXl)QN>K0<p<3PKg{z@{se*y^Yil?8_jIZ#A@1?dnXGWV7w4A
z-jYKmSUB{8=gI!;Ml1K&FG`;y>PI=Mp(+Xjk1YB=YD`U!`<~(*t$imbvL5C-e6O4?
zW8ZSOVXxDbpJIQhxP7TATtQHyDS~mbJb^&`)*bI7*ZINXOo8#ub|K2`wpx{949s)C
zz7q)DHa9ol*crBM#=fRKetUu#41o}hs=>mc3Q4V7=*^uDAhQWk5Zte~O^k@R>hV~K
zk9)c)a#M>Wo4~T~W1)y-V;J4^#A~*<JvOOjl*=JL>p!j)c^|u1oE$8sI^X#HH1~0y
z(azG~&R?cUZW*`LKh>?e9_}oMPyBd2>~`TAYXrSq+#buUXeW-<V3Bx^S^+Dc^U^1g
zx`9H&*E4l94A-%ed`|W^zMiKX(AU9qt){6%tUe2nd<+wd5QlMX*tT}?ZEB50pk6y|
zf0mMH&C$j~jdGjB-x*IQe{PI=tiE*ax+&l^*CCQ2pU7lAR1(^+=OfTn`EKvTG3!UK
znx1u_n~|}xxaVmN-_|9R<4O6LH}kDT$1s=3<HKDKTXs6sTG7>gGJ(wkk)m@L%#S|?
zHPwdOMILPT>!r&l@)kMF%9C=K1drG^v;28%N{Ysz4fo}Fcs5Ne+!CALLx$cL1ZGSg
zv>)wn9!JIBZFP=irQ~(Y8Y`0Wx-%KId{-7ln5`q>B*a#|6~<Cf(a4e0(3x0nHQ2RS
zU~RYA%6&-Qt&Oq9%8lVW#x8jEjW5sS%H8j_R(!I_g4JgD*C!YbR#j?m22!|ZnJ)I_
zQTOSSz#LLhYN!^rn?z`wy$f@s#sUsz-dBz5gD!U!=n0(?w`7)C4>z^Max4?3Tq5I=
zdU=JeYH?2SFM$hrY8T(~Z%D0rdV!^!4&>q-)Y$y~qWs0t+{^M7x&78`vd_@2Xu!q7
z@7LnEteM=2(mBjYh^seJ)aX2K$@yd%P7?4f-N}BG8+8N=^ue??`nCp5Af@-`IYrXy
zSC=gg<};IxCV#$hu2$UTx~)|vyR-6}Ig(ke?3CxQgPLN@=d}8GuZ?dtXx?g&83krv
zSp3RMV7;E`hhyE*4DpP=1Gez+++sm(lqf!rnD^<i%PQ$il`h6WiFtU{;Y#JstG4w9
zO54ETtVi<U-fWKdUynMZ>^{Sie@((tG!{!Fe_!P7{Ry1f9}lBhBWq7jhWS>5Ioduw
zUdZ_#yLGWbR|mW+c5ZK5G2A^~zbC3(z0lx;abH9<PkN7Pc&TN7laeI4ar?L2*Jab0
z$=(*VLh!pXCw!-m-`XnIf07x0dyZRXy)_z}&=k}CbZac4Y`H0tnU0T7tEXq5Ez|lz
z>*@Y%a=Jxtj&rBsI|2IgQFqI%-k3lXx{*F4Sc$+%0_C@cHu+d2U}?rXNYVWgucB!F
zXxOR6Hlx8Y=cq4R@%=L$E{h(9+WmvCjvLj(vG~bj8@c&knkr>?<GXz_M5oIWy<Ncx
zmk(x>Jn~IfMk<m39QLgXobJY|KP&4drMLc|)1PK?v0?=4s`LK%SAQ@{&ez@zzp|?o
zgpNzboL?M|Yh(PyX%^BDyj^Y_!)YGoQtnoP)<^TLmdRK2&V4Ly+lY~`anY->?Rnh4
zJX&43YPISy8(gk?2{%F3^mlvHMV8_*vf0Xs2oXI|IRKBe2CP``!wx4o-}$?{!q125
zIo!9*D!l;*1Sq|>LL0EqBEz}q^ZJ1!HFr3Am74ouQ9YM`V&zkGOYhD_(&`b-R^xgL
zP<rn@soBXMp4&MPFns{wNR?`fjfD%Kss~-|v0FwSaPHFesksz#Vh5G?I;-H@+gk1O
zJ=5H%jyh3))Ny-)b@^f6`wMOSUezmK7LVRed)lE<Ea4}a<de;8wIeo`HmYTZUGkBQ
zW>Z!ZUac&sJ8IZVJAN2-fL7JU9Hu49yhEKEECn^(rft;?uj<Kep+29U916_TV0w%M
zTq1LKNz_8!u^!@l=e$$;afT_F{6X5`(`UF2-KDd2ewd@jJht{1PzmyuS*W_Hy4{$q
zXbOkYZxm|&oDMI3)o`nkFXeVOI=!8(ip#w^`iJv+aBT77j>ouFCb@^h&k>E^hYMJ!
z>lYVX9=x-*=_vCXXUt5P5HK0v?XUEIP}Nx{wrIG5@vC3ArI*XLqJ(^l&t*2z#oduX
zd$7oOnf+LP+SL9;AlYkYaDOn^>9R1DA2tr^%IBxq*U{L-pXWZjwvznLFUGLMn!u4W
zlb!0lNX&^^`z`IgiUF&%>=^t03I#{ikIG6t@BJUc6?tj}NiH>;87UaP#nm-mo)XmQ
zxcr{JOBz>2F2T|AlY~(lmGU*n!AWeDd~JDdG4h7#;LUgZ6Iud?HFsbL$E?Mx?%3t-
z%y$hJFKo^cU$o{sUfGv-$8Nf~Sk%l~y`}t)RANfUD!|6A#g4X87V}eGBQDCZ_IgWP
zyg_yD-0t^m%L3i%s?D40CSXUST%F9u`>`z@Otw0h<=w}EYjv0u9I~u-N(1dPu_)){
z>E>vHN{S2c)@}$b$;``UM7xpL@lw-cn+;gY&X5ile%@$eKJIhiKy`2o*N_Wj>I44o
zIyX5l4^6j}6|7%JZ7mP?M~%&^@`*Zp!$$F1N%7SWJ>6NB<)d=X(2eG{9UY0C_IhK4
zN@=(gn^bk^jP|Wt4QotXwdQuHec|FY+xjc?W_5F`7gd`TvXeud1oy03(j-fVB@QGU
zHZPvDjcqwT=WUF)OU=R2ZOCphXkF>FNMh6L$>Lzu%b)p2XMc#A`KGV%$OS@Fy%&)(
z1{MulN~}*waf+$wce(Aw{_;f~*FP`3b1O(ubBga2!=>pKmLt5jd)2&i>n%}s!`9`4
z?)S-c97Ibc{p0O0e4Up{t(_QTv&=`U+?@A2B`L{E?uTLcmW8h#)4AIQNk5vu>$Z6O
zHdo|rRR2ibGWXcNzRkvVndNxd+lTU|HHq(4<JOtN!rj)Q%4KKhw>+1JM#h(aKgJ}o
zCUDraui3f$dE)|G)%fpB-Ajguzv7_w@Tq)-Lwp^+o7iyQSx_WfPQZTK{O$)z*!F&3
z(Sna{)p~$l;-mnoW9x0(-Z4b*!FbKYFXu8aoC2)p2ToaazF-|^eoZL09w*(FI)|#b
z#O84Q7ysA3s9SbsA;sE(1U%EdsBbP!EU3AG!qnefT^M!mgihvIRP^z5#x`EK+ZLU;
zCNNvdS37==IA=v)shTBcpJ-ZiI<PV4QqP}C)4#2Ofq`{J4;b+JaLjd@5L-VI7R>>C
z|0L>5+&qTG2n5yk8r((QnJ27HEHX{@>MN*Pr$Ct*C&c@0&pb}tx>)H<<%Nj~x9_$b
zcVzuuZ6Vp`^2!KB=qg&e`qXobk6V9HfSN~+{zpb$H2&sB;s_pUs|GJqmiIYhZFpJu
zC}Dv{b^=*$qH|XYT~9}zT|<Y5xZ`0${)gwoQN{puF+k~G4(9jZ`8~kB|DAQ~qK|xd
zW8+FIEr(^_?X#(^RijfFsl77Aa8R7Y8Qs!7OJZ%8@@p`B`}PM6-<2-TuU#Q$Ct48_
zs9fQzxQx0lYS6NY;rV*PtJY)J#<u#FLef%Vfh{txfi?}#)*jUuY1_wr`gPovp(fjn
z?P<G@=3G*BDxLpdP$cI5(+}k;kt%o($H*?I<!Mg4Fx*0Qka1#W+O=b>>6E8v_}YIt
zufNcfJ<5wcN(fHClP>Mp_8^eYa#@0eYL)`k&{{#dH)L6`J@=RQ@)q_v)C`7=v$uRZ
zzugY;LN6}L{tM3tj}|%8rL1D1%-}z_6P>mnCI3#@z7HJT)6`>Z@%V=P_@t4DJ;@hR
zlo_5Ti(1x475V**%^cI?d%d|UD&3hgPF+-O($#U@4tF-bu=R<e{OT`5Hj_}6Mvo&V
zE8NY@D=JmuaO3{G+)D`YU&=YI+|M6q{6o1Qp)xj3vnHZn)6gmZ(b>60#nH=jc5hS~
zGbgblk<aB^UZ!c4lh1*yZ=H1alLhu_4bHXqiZk=yM)6fY%P$6#ax4it3IS`z<rfnZ
z;^QtjcMc^Vs74tx9lT9&5tR@T@<raed47aaKPl&iDkXVxzEl4x4Jm3*(2Au8vo@h>
z|G?`~mBAm^DMuo^k`)Za8?#T`7v4`EZ1OG-7T;=kbZ@Qec)oSiZ>&3kYvWk=+$e2g
zH&=@!yx%o=zay@U0Gz8*!nR9Jyp9Ni8#H#5B;&T(x^Ft`RN-8Oc}pNXznSUr$2mE^
zEYmy5t`oSmcU%9UQC`Y<4o8E<rXGC!yi+$-EyQ6<t+zfdesNl?`r*EiomtgDe%~$y
zg27Mj$h2<hF8nCx`=y(65jP<O_&qC5!td-jpFW><*hwuJH*r^4MIOU<wY5%O1C5FD
z(tn&T6KSP!k9dxI%yW_@$~mR9`y;uh6MjMhGu=VG-mUoHw=VtKyYdGz0yeams7o^m
z@vYknezui0`!n&QnV6ObU3HDEC}Dr2*``pz&$rpSUlB9&pQrTvB<Zg=rdk%)D_xAi
z(=PvO(Z6o1GE>dHL~e@|PZsi4-<!gyaj7-PguT&2>s^7jc*v!fx6A`a_dgc!I9Vlm
zZb+}XyK|%orZtj=>sC5juNDDj3NHO}i_4*S*luPyq_k{C!u<I5+)}A6YCJ-T?`@kM
zZDQtdpiFj_={Kk3Hs|ZY_nlZ&GJ<!XsK7%0=(b{S6Xcw1%Faw$5Fje}s*skEfJXhU
zb>R4<^0p2o+^<pNSt0Ox>0>H}-ze3>$mg8Rz)NHqxd(Eq)w@aM-tOtsCEfCybK!AL
zUgY*u7w_Q`qvrlP0XFGY-p_pZK;!gqa3qD?ri){42;cpN^(|}3TPl%l-S4Gq2sj;f
zrcF>O#;%Kb<yNuVgI6u0ho`h(cJcCL#N{<MuBC1E{Ki#w7*5_?RqQ?pa1VH3GhBQF
zuuu)K@ab?=9JQjEpY?E=n@#oh<J!0HogW}PI=)!IZtk6f@N)2c_oLFuNk<dMfoDmB
zJ=FN}v%GgGYm2Kla}2+f5-l%%ck4ZW-wTLoo5c?JF{fhZZ6W!<&OsziqY!wQ+`ZMr
zT;SUNX@Tnt<!=eTWbWUN;Vj=f-ita6jkKw;$>q&!+5T1|-gNB8C8B@+n@i|I;`+3o
zsIMbs4L9auq0s5kW=Lh9e44HA?EQI-#b`b6L60AFc9Va;2OfG+a+-CRUYLt~zuCsu
z&O84pv96KyaP>yR?mg|w`-g73zg?=i`*?>fOx10zTIU--%b&krPA%M7#nb1OQ>JU(
z$;??H=l)2v;o#8uptOd!UV1lO4R?cm-ixR05$Q2^3zC!W;ib);-<&*CWu_SP16G5i
zaTzmG=`uK&>BEC#@TK1_@?DS(lx&HWygS+Y!|3ki?rdk0>HSAOA6hU|m)4zxW`YZ2
zwo*lg<BQw6DjZC;SUE-{TGr0rC4YFAT;(oz$k6FCF1(oc+nn5tuPY9yXKokAq|oWV
zsq*4f&?-H=JAVVoD-3vj44yr!vevG*Jchj9AYfCSJG=r)oDtd&^(nYatmRTn6J-fQ
z7P&&zMQSh1Sop;}5jNm0H^@gnKVlV|P!AnyIM`nNq5b~TnBK8`EHj2dzW3A@4<hYh
zYex?Ug*V41Gt*-UE9$3t3wNEgI1(PV_&4Gm>9m!md<4DdDWfooFuGOZJuodrsyiqg
z?KV@akt-djzKecEBH<MaG&2?wNzZ(rMF68<Ga0E}K3ZH%;BlzGSj+q?f8ZP#oH`hs
zA6aw`b%-p)y2hTNbv3o{cD)utMvY46yqI9=t}Lb0EY*q9e%)|Lzy}Ru|A_4@XsP1Y
z%fmzSfz(cX*=OKK<J^bgZ?OsrnNmdO9Mv_uGvrb=#T+&ZFQHnzu#^Pw`2ZUCh^bIP
zV`ZsRI8@^|$FWfKlP_arg#;hEzJ@JD#I+i@NzL^txb!n|2bqo2$I?+dEmiLHpDZNU
zl87WGWdVXsnvFNpnc#CKnT1#(6Yrbox%Q8)MwJco3j4C34fFbsW+z;PONgS?daa`O
zAGHp2wKNhy>V=uijsISRB!<GM5GxjWum9<KVVbZ++40FuH~&%SXyZY*Dk{CdE&h*8
z_y5<4g{e_FR2rR@%J6%r#h>;C8QENJ_xnww;c;?Lnwf=h-dKzsKRP0;x7Ft!ZK*2D
zW`*`foDwK!!c-VYLi0%}`wFbFK6cS*=n~5&t;gZ{u<B*e^`-r`mYpf0TjI(x$+X#l
zNzB>PancKauDwnpX`dBTCr7r5DhfENLH0d{3y5}y;kUF*DZTUa8r|Z%d7AUi;`;9d
zE2V_gyWU#d$|mNG_Z`2Hj9qoM;6c~G2wcq>n_hz1JSSQ+dbpF!F<sYA@X~a5!_HE|
zN<${rJ<NOLcbNOsG!0|+<zY5Hgb@=4<-@)?dJXa+4|MWd%G;c-BPf5}5WMPQD06og
zH!b;&rZPe0uQ%TvWDzs*^TZ=WO%Vn6f;IT5UtU`p*5^GbmoCHKe(A^9d-0Nt(4V`~
zJXr=8BLZm7>}-k_%qRTfCUX5>BpvcTGfg(3l#_Y`yzw4b?D7e;*Dv0oM$D)_17>1X
zb|D>^*MkeL?9+7%-eKE|F&)i@MmCz$qhY#}N)EQd62TEc24}bakAH!w3V*zCV}LQX
zSxaW;Eq0-<vQ|w`ksMv>s8#5e=){Mrt5*;|A%P__GB;m<C3?y-K3J>49<8HlMCwiR
zBJxpD{zpF7k3BhF5^2tj-e(Fi9w3RLlrR_KWFO%AawYcYB+H_sBf@1-amUm1MPgt0
zZ*Q!0sqR1N#pB6QUJhDGPCR>)tFpeR6nz`37w}eB-w5|n4AxFrIn4NIblzZlAo-N{
zv|EV8ZAv0kt00f<$MrY@^YojJSkC@%HI<?wO!8S%Avv-g)))#=M)_uRgIY5Zhf<e+
z`{>Wsmb=Nv$Y%#~M+bkgjU)5#K1uv~dghqgz#;8kB>E$wdBuer>c7>B`rb5o{rF4s
zW+v#Z^(9PRvd$@?ZIZfwUrq>@V<d{6U5;yH>(xKH#XQky-5aBkq&0gw_AEn6;&^{1
z`|GuO*$PG+n9jdJ#Xz?%yh5D)TL0zheQw>1DyNgvJh>(jV%pU8{+Y`dfejD08EzqC
z6ak2EdlOU<TooCwEAN--`+035gK2;Mh%Ztm*X~VP@=?`hsM?un(OjiMF+8xsN7ikM
zetfW0Y{$ntPJ=}-;N)uB=4pY*tE+s=4$?fIpV-jeKEuACPhd08MFx@OQvA!Wg-)&|
z+*^pHh4q!y@H3Z_Qx*e@K@Qr~KObnA+0LKY`<|LF%GVZ+EbmS9pw?(u%QTBY8t0$a
zN&{`R#k;}o4zlhY4-?;2ofRLh`oaA#Q#s%uzR8A;eHbsjQ%0~^l#{zCma|z9DJ8I_
z-DtMvm%IIE95KUxA!?8`M*GTi(P{C)T@9|W2jWGWt4rkr^UbvBvI4U=3(oLW3D7o(
z{Z1wFE*ms<`qP)fO8>-GeOwsg?VLNZH11;3Wt+bxxU5-y|IYup0}6HFjE6$3Y)LKF
zg`i%_T+M5%T@j?4RAhACq4y#x-p<rVW<-Nvyt1Y8HE{KZeoHL2@$o+K4OWAxgZnN_
ziAuKlknUXhXFjk2K{TwD5@+)vZ2s=^>ceJ%#6`@eX6_eGi1+`3DvQ|9&}zj0ut^4L
zvjSXq@)_55I)^@G@t$g)04%1fhxc~o;2oaf(i&Lh+ShRwWPf_lNyk0|>!;#gBO`Cs
zA`*QvZ@W_p%M_+2{Aabrq#>-xd|Dy_Z~gUJ7!ZtZGq=o5qx4tb(^OdP=KID0X%ud5
zp}palh@&+!0knK%QK*KnOOI95CrEYP`}$QP<IV@4d-wYdGu6nL%paX!(4s%HE(QQD
zvCGjqNX)9hy|I`7+UWSi(<Ya=(Vm;{RP$>|YC+^ycoK8i7kffvyJL}nS+SiB$oQEx
zq#`ZbXjOgAoo8g_h*=n{iZ1mJx*yugkbRj0D}Ooqt%m>u%Rp2A^qa-RzDHiJaqx?_
zT69`O7SoTmqsB)U@ZZA4wEkdG>?Q0@Vko~D;(^n3rHR4QOv~xlk;t`I`MUTo?-BIh
z{(HY~BVoL&tA_~gtE@kEK3%-%WP@32{lc*zNn@a-&DHs3XLM4Dvb|!lfVi?A{xjqO
z#mEE71)adC`muMaqv<CacbEgoFzSR>_lUnH5-^(EckG`Z;o*`*cIaQlHsK}QrNflq
z3Ur-@?J&D}dMmN-@4|km2pj}F?*J}gSt{lLO0Cm<I-SIuq41Ske>BGs$(63|=0H&Q
zlVLgW_nFiD50-r|pbxJ-y4F5QsdL|ciJU!sYF+tbW847qUv!8wUICn>`g%QL0qoQh
z*T~x~-^liqp$3mUy(=S%Pm*2!_>PQ+Y5AF}yhUO7-u%i8t3Cd#&M%T%sxQD)`|PH9
z6Wd|l6WxVscVrM6OdA95|Lh(^uLzK<E58nmKi^PyQ1c|_l}VQ?IQf)DJ}mVAdQ;(@
zsB&5dC<(p2;St!9#Vx5!P|LZ4C-D+7dn_U<l+>ql>AV0Nrs&vV<1xv=*asdHAIC#k
zZC>4QY<nCUY#%y$_fQmGix3B}6$ka_=HEtt2q@W}Wc%vi(_N>kh_2Ff^~nOm9NDV?
ziG)(Bu&iR%8Vdk<cUeA}wo@?bd?{g>^rLit*jJ!X@938L=(B*&>j+lru*dr((n3#U
zOYpyYpCPy>PzBFBXFO3%iwYF}bNDwHY-{#2ZtGU5L*>n|%h#JGRxgQsS+rr~r1n^2
zYJyLkqEch3sfO~QMMbeee(=?^trbJ&VD)GLR_`w3C)+W>ik%@#W4HrSU?k<XNl44c
zi2HLoSmg8d?!@4(hinv|=8K^k$+X{up$fIcr1$J2Mw}0qJ9)4EUp`OBf~0KH@m$tM
zf4;q7H>k&%A1*H#dyju)S4nzqD`u?9Xzvr7%2Yt%J)?z@-mY9VHmGH_)QTvvt0wUx
z*2D_<^%<-E$(eBKeX?%~T4BZ9;%l}VhUMXMf}AST)$4<&=5ikQsLY<wv`a}x2LH@W
z8%xPF+M4U^%6Y`>2J&ONQDfLkZOerKu#e9G8skht=Ydd*UE71&XS(=nlCaoVMnwBd
zaGQzcdCcP=PS!YjB=NSH=&8uU<vhC7mD^hP$glKPy4ad`r1-TY31~r)>X+C@b|=J+
zwL#vntHwpJfHtQNl8H3KhPP!m50SDa?yUsEvYbIa5}BE>j3@olB`-)XU6ZmHJUuTC
zc}4NYbo1-D-Ylh?ojI~z2<_+x)X}y@<H4)HWI)QM71L2#Y2L>NAN+)M+f7XK&CK)B
zEz)9PDZJ?vo!=6UlgT-Kpa!~=BmMYsRJBD{nsj)IQ;Z+Xg-X#5Arr%ebnuywc6{B{
zs3S#WW89xWra6jb8q$~MeplyqJ)vJGKT2DxQ8Oz8VoA#EE&jDNp%KdO3tr6~Uy0VX
zfB8|Xyls@bGhe4ttS`j|2`ZrgrB@lb&x3R&M$*BVtn<aOF!3Zwz+)(w3+lQ1`5S({
zUbL3K=R9q%DUhW-s?h1O493$kYE9_*;$*2&ViwB3Ghn!L;`@p06Ct>PFqHxzotuFH
z9rC%VZfAMeI<6i20?9z}DdxG0IJF|5(=|MxH+oyV;2iBln(6WD#IhFt6YgDEq3fzT
z4z2gtofcI2(qu3(4ws&bJ%2Bc%<?jr&ehADfdB;c#Cy}xdRB4mNCgcFi)`e)jtq{*
z0+mrsCv4YmjVe-+A9R})rrj8!YM0>sn4^#^xWnWt`0!7pnr`S7I+-aqkH=z&Yp=mp
zukDUQS(7S+%`g37^bpzOZQ0U{eXAi(dkxQ`J^`llkH<m_HtG+W=I=>JX-0ZuGtEFT
zdHGHX6ljJx-%H+jD(1KIT>_b55eVtDhuppgAgcRekGe0ic$De!eTXY6GLhS)D!@@V
z|5%qzgN$C>wP0n5HP3)sT4+7?0r?SB<Rh31s+5^jv-DeY)e3a8jtlQ1Qwl}`#=t@k
zDFzlE%bh3Ru;zPnqkD4I+B5d~7qU|Es@p^O)^3CV=b`mjJPLU|X>79a<eIdEK=>#p
zU3G`}2~^Up0_fz3axlJ<nkFdCc@;sG8)6|~O;~nmHsJ47P2q-A0t$Y03fFNAzAeda
z&nTuxIUR?WD*Z7l<p?{pw?v+--=FudVP9w^mS!U=oLD?2iKE|ws&{0xuDkS5nKfCV
zSMg2cww*}6RQ)2G5)Hzq{^`lF{corK!FWY*MG3c!##`xB=7JYgslrJaHW$mns*QF1
zxLhn#d%m+p1@m%jM6BPcH6Homlq*N{#dY1l!~c_@>l@rNb8-R0A9<sW3I7Qp=W`z5
zOiuRPczKbWS9PUgHtB=fU4rw?M?2RWN4GRPDJxNuvMggt?DF1UFCI2B)b+UyPbK@D
zgtjH}HE+##FJJnWD*29H&TMo}{Q?9`L>_oamYf57Hj1@nUeX%R9i8lbAf4o~!ZQy$
zJ<1i_rFjtY36#MGn^%h6e-sJ?M~n_D1Z2=BmAy|i%6~h@d#w#Y#|AxH?Gd~AgX5Qb
zXZj78>)z8fkpIET)XS`5wcY2_KbQ-ew#Lz!COSo(;<0))jaI1Rd;V79KRsGd#(hA^
zmF{XRU9cIKMq!ZjM6te`NP+(erk}){>w}!{q3G>;IokI-!j#sjK^37~7Gu0DT!)Pz
zx9s+VSS`qzcD5~%xzwW9{{7V___5p3H`G0n=ZG=x4nEUPly#;Qm`v7n3L;&loR0tS
zT~*b;1jG*MAFqh5<2oka2?@3g-A2GKOa<11XE;$4)`Jy|RB%TeyXBP2ZFJ&iiaqx?
zDy;sD8bJJO#r%Fy5yhmc;+fI!stP&q62-IjP>{_W;^|cb&zZ+{{CxBJ#NxTa)3cCP
zp8#RJ<sL3JJfj>t;36pvG{4l6A8<fmn#<e|R(v5)SrdUNB*ZZ2Q}bkJ?q!Fr!;svo
zG>SpZu6LD^%J_o!G9$I)3wABd;yR!-HSB@j8b$rh1bx8l_UI7`dx~oo*ABoogm3tc
zLcnNgpin$|hT~b@IPf<kTThK5t{U2;>fD>3O~X4fjV?*&+Lfloohr-RxYp?<>24ZF
zU-BsT@y4KO@*vss?XT8|s*6I;l#giByxB3QNq{)*7jq2&64by7&#g|$ouyN#!y4AK
zxG|=f?&39$?)U-a={Pc_pmo-l)a5!n<()UCD;<fQsdb>+?l1{c&R98({_&Mhj>sM2
zSTmyGU;1`a@-3~L*^Lr1aUhX%2(9gpu7>5mfiu@_bgy+b?w+O5##G}Deb;>kXTMi-
zgS(a0Oyh%<5=N&a>qg6$8;6cpYr2}GWHik~4Fcmij(e8}Pa^!^vUWT*Fi%}tvKrp+
zzb$f5ukU4Vw?6U1Fgqv$8?2H+8N7;SdIs_eLQ8aWG+VjhF;t&z-Hx~WrxP67UY61*
zC*k1opX^O`WhzkeESGd!#dUng=ezyD!-N~bJ5Mn2dNmIvO-M#k>Z;r})4UFMK9GyZ
zZMsdoyd`0`y2M0+&H1ZxAtyHtng+P!%)mMg3S%M=uMWxqi%@kN<N>K5r;ekY->}=W
zFI%M{WB;S#`v)IMBJIW1g>CU{28+_u($WT)G2F#{k#SP=Fl{3X>je<nACE&rP9(Hx
zv}e3b;C0H0V72~P%|RE56zR-{eZB`=f*Ndr6!4X>oD)^ZL&CJcx+{j0hi-<bqGMxY
zyYjTS-9SV>YfD<6Y-li!YfrN%VR-BeA$3Ql?wXz_wY$Gvdy+t^E2r}th{E;2T!oiP
zu<0NSG2ktYRCH(TA3+RW8a(~#e_nFkY>8JFYikDTUc#%FI>??=diC&k4qUusaK?}|
z;PIha(O;;2eTny$%f}#u_2#NI7q<&q@_BYpjpKlIk5+pOpRa|;)}~8MBbSx>9Z;I~
z13Z<cj0xc<1$AnAUdDW<KF8b90?i(PfbP-j=P>mA8Ynzu4|bNB-;i-fagVxwHwoO(
zY4bpUSqbp|N;+Nkc{C0dZcDPzSjPTswfqFwr!MJR*m>VCVtHb3z5IGGH4fb+!<_HI
z3$}Gy&W2JWLqCvaW5H0oImKiO0ry{xx00>+PSf@;UY&4|M*~=9mss>RjZz7E>{yxi
zWGTt#vQr{E{1A5PvlSN}tlJ-~mD*EpQg)*tZO`=|?9kKD<r(nZKtsy!o@L(v=X)p|
zm6$)!Dz_oG7cfDH;yMFB+`Bx9SYV22z8U*f&;U05;+pQKg18wD0Re%jFD+tJ+<t)8
zFyDt!w?pq5{*;0WzWwm?X~Z_QA+xO)=TMS@+*3}!KVOHdb@p(zmhB*gtr}hTT=)q#
zr<_@)e4-I_zjf96_>gj0{bl~(!t16TNJ+jxcCsiZj6o=z@(I1zUr-+FU2}Tk5evO5
zj?e+~snW;;s|pQKFiOiN6bcTF&~Nvo;10QChsfx3l?cG-gdFVjsAlj6jjwNV9Ry2j
zzEh8#rO;RoQ<-9u)}yFR3_mH{;OqBfKpej%7spv>{s21rI;584juFri2U@F`Hge-4
z6y+J$Rw`FzhRdwep{I?j`<=<rlLu6B=<ZB~a<!No)og?A=M*j@PdoW<93q+i2O3y8
z;>XV5ADtwwU>;M@b4S>^EabAH&2uo9n(8IB^Ujx89cY``<CKyTPMFhMPOAYzQp8m9
zVZSGGDj04<S6m&GON`gXzuNzKaqU%7C{&&<-od$=jr8$B(2$GgYP2k<HYvs?HpjUC
z7eY{RV8b7b7ee3T)jZg<CFtdu8$1O-ZhDSOB(<qgB3IYxiFI<csW7~*_r)<=j%cXZ
zR3SP2O!PDWDTm5=I3Nt1;~k8)>d#M{`|Kld=riWEyMKF{1-*i^^t)0mziLCq4f@!O
z>wGZ~_8UgulbA7{1$2o39`C7N2o0j2Z+?36{apOKX>ZP>jje8lW?;vQ@*?+{(9~G4
z@2jKL#t<C3-iw}hgO<n_obRbqVvw8-N_d1Eav~zI5D|4yK{Vt$b;s8K8BA&qmj!CR
zaRcq(j_0182l03dM>|+-b0qVud+eEPKqv$AkHFi6vGiDgO9JsTn=rkySZ|B|d~%}c
z4}>^Vw0D5mB0Za8-TpODPqAMrY?vuwLex}@epq?!)Y&?KV|n2~C5U92r51y2J`ke6
z=e4n88A#|u(gh$YArS!a0}7=-Wd|7$DK7<Rq9*}7cLr-qq6z5*h3-rJZqEf`h>#P~
z<3JrgnLoIv5|V*y-T6VBpE$XQ##i_m!d6`|RRoYrAdDgDq1(=&DX?Hx_NNnKoyA$Q
zIQ6N%&L^dlwN``d2!oscNoE_|V4-LNI4lgSjb){p^IVu84|aMuW8V|{gOHrW{`7da
z!SEgf2UJ2Krb70_4#-deI6dumo`Om)E18gzP`K_A1{%fP!$S|XF5IJ&!&R>Cc7UH{
z4cPMb=fWRwpp6oE4>b1`w_ofyKS&IvQKcM70eDY*i1-tqIvLo}FKO2RMu*x?f!!^;
z`jGF!NE*D0WL&af_!K;gAB(4;Z}3{zb9w`2m0MU7qOeDTLobzSLPF4#o@;*E-uu%?
zH)8xeU;!_#h62+I-<x-go30*GRpva(1%eN4vQJ6Yk^bhZu>iuMN)A-O=!lV?xOBiP
z<zf@L;r#;D?66gX(VHQk&=*K<!(wlne|4B?355nSS}UDwaE|(a<Z!C@*c9$_)O=ST
zBroJD?9#QGKH%n#yg<EnygLS17SC-<;^nFi?Zl!Vf80h~k{7nO^QkdtjD!z_yFD$K
z>ZQZ!2cHlEIpZ`J6up0!nFkAkG(kMQ<Nn6<2W}jf)lmT+CV3WMhfc7@{qeqNs*oFK
z1b-kEYlc9*H0UE2`0F`jEJ$1BnIAwMcj@~o=drbnj7(9nq7UGA5xj4N-=F{*C&+5#
zb0_}Oj$k1odf5<g=x?A+A#AH7-nK>~$$f!wz9&0eHi<u$^Svh@%sB_<tTb&ri$k^c
z+huI9?ey;)5ZZSLuY#gNowED$;*I-?ZGwkZ@$@BtYt(%P&}T}e=n7W#jjZTAz_!RZ
zG!#cPvsTK=L^2t394FW}G4gzDw8A1(^n(!iXiLPkfzqc+j!3h@-dG>eb%Ma8#S$XO
z0d(=Iwz?j2wVJ-Ue$Q(5G{*2P1_(48RBO=9Ib2uax<0Wp>g&F}F#Ux2F@o0{te`!*
zF!uoi<HzHNMrd~ZI+Qu|Znh-wsNnNVh49-nS|FYV{l`V-56W$RTXa9yg@*4s1g$TF
zo)q*K9a4gk+l1Bzh0d)c=-?=u7vx}P_XVba#<3bIkw)6k_79*9Hx4R)pWmZAo)97X
zi)0`J&w1eGk>LoJA9&jAGUipzzc=oWJ#Zl&dJ77CGjMx-)Es8pB1Qbqz0ANhM)(9C
zGeik>kuuN*<7ZUbRlNZgxgwvy>=falIIus8)kR$3%M}L;xyzBI20vbjo2@RvN`xuF
zLrJv~d2bSm_(8wy5N9&)W4ECab9=CBNnmbx7+5zjyj(}2p%Wp45S}W^e}?{nhVBh`
zXo%k3K6q$B9kjC>6-P!eD1F0M%F-&&cOkWfH`+iKRBgP{cKPk3T<z|3X#1A)eJHod
zyML*rRCs7du@AmYte1cJ`x3cAS8rDaF5Oxi=Ji~7+hmBpl;;5>K&1a3NKkGrNct+|
z8cUxD+?jVvUd7*LxU5z1$>E3Btgntl20)pk{Rp=Djf;g?6fSY+FaRcC1iiB(!GF2w
z?gR<fU~$=um~}t5-}I^(P=|Cn7=?k-gIxGmxUVhmx5||-4<Vh8%3M|liuvKzc)YjE
zuMlc`Xp@jw)n9_^uSmdpJ&6l@MMWrp;YFnc?Y~pG1vL|XVx5L2?a&9~dwz{L=*^pd
zm2ZVt&Wy$I4N&-=Y?bB1k#d7eK)Wsc-irnTumD&lwZz;Nl_JvYVX0du459?n-!o^H
zd4+W91K4(D$dNJXR(*!PYRQHE0=+5YabJjXE`CVfy(Qv#&!PvA<{i&WOqFd*9Q=h7
zoKCvCNA;@T5pEUiIM<jf5N<2U58yxh!x)w~eSMkH&~o<POHeXfIen@2w~9Q^k4Sqv
zMR0Z<c=~`fnM{DMCHkgBaLCE=VKZbx*2(uZ%d9l|o4)x06FWsiEKVfPFE?z!kEh@I
zc7d(sRDJ*r>+={{pZKU()FD2I5rZK%!{GwDL|&(`#_(GY3k`m3)ZQ#RQ^b%iYZwfj
z_NCb#mCnlxe+SU|0N6sS63lc2wZM%E4JG;d^x97><J#e<K!3A1LdV9@l`gAM2#P`b
zyY(wYIpQWH+5XR2k$p&Hfl@<rW;0~&>5}g#vVaOcLs&JLZIu$74WRAU^~mEt-s%nu
z4GpC#O}!3YM+k5JEUWMm8khDAgA2a_21mg$i~(v)TvdUbGpHK@humSRgKleEpj)16
zUkBjMVUVoRntxZZ#!}V)`V6}Z)G(g86#u1;WB`5?XzRGQ*I<o)0miZc&n+Q+2HHwQ
zU1>T%Nk}-nt~`5-B-K~!Evs7m^I4juIm<XVJeCS^bq#Z%g_~&&g5pDj_c_jY?TmTD
zS|V6N3t%HvbT0D?5VX{Hf=n)>Zke3cL#ieqEWlX`OvJmTggG$p*^X*EuZ)-jPT76M
zmR9vZ4(Xg7n1NF}u)d&vB5H8!-X<I;8T=z8EhhZvvhcW&A9U~!cz8d~*JdQTB_9Dj
zlKSu_(!eT1XfMg#yF5_oQn0)QDUlX(j)hdv^S-&@z9v=36cX#9k8cCo?SDZmWAi_E
zNyh$i&O2dy*dJO+K9y$LYkQ#9ffi`o9MuRXO3YQz^gOt|pxvG#v}||f_p<+Ke&q@;
z$iD&t<dncN^kH+bY_xL%e9w-jaJ6Qh!zifUiMxaqg5Wk9d!>3;w^Q}KyHFiu(`mA?
z?0&gxh{KA3L5f@tv!EiBIpa;OnSxuUZ3#T~K%4>)CI>h}><YB*EcZjG)<qGZ7-Fz|
zR}^6^23aH%bqZP~Q$Er%ze3_I-^zWr!{zc9oXMVSXTXH?P=T^WfEUl_lHZfB!^ak1
zh!2526!!JTJrZ$ossePX#G5b>dmCd-lMq4roVnm8UV}T*O#tm?WDu|+F=2O@k>y#f
z+JyJ<!@2@{oANV_3CGCPv^lSQ)WUXax#PY6+_a}x6Wf_gsfPg%ZO2nDwTR^3Y~k2>
zW#GO0^#Hg*JZx^^AZrAkpXUW2k4P&E$BFySX{aOxb*pI&@gOW0q<MG{NxYy#$#|~g
zJ-gkw@YoNe8Rb6Gx6&PuIskn~R-;veCc7HI&tXL++;^7dD`R$%oXQWIorA+VPSMK_
zNvM#s2@m2r&<;h^=(~s@Mbs)xO~~;1@ARM@(t|<HWEQQm+k0&TNWuP>h(6>J;6viq
zbZDV@=BAt(zI=-W6$O%UypLqKiRQt%ty%IbV_){+!(wwmGA^r_?{O_iN`uFC2hKWq
z+#8<?SBQc$4iDc79?X4PjSzrIg+k1~%ENPBjEWseiK2lBK8$2~WEH0%x7&zdgP4}^
z>Up!_vb+cutv-mN`iP7$@esDSX6SVnjXt)-I|>#ME+PRQ2I9RT3sW>g$OH}1-a>A{
zRM$hcNCA%>$WdG4?lBh`BJ_`;3ikNVr9n)zIDLw_pl#(c7ovC|^2Y{9u3<C9zqWV?
zkshv(dbOh&xX0kg#vtYIyh>ZO9|dV=34o*CW>}FO>j>)vG?(}p2P6*#nZ*rcCF#xi
zTlK(z7%X6~-S?-Xn~}_wKyWW*1a?kb2kR}M12$Bp7*N;akwR?Tj7st0ZH*!^<Xl<>
zG%~*af^!3^mvEpd#r#1*?TNb^q;N}%d+U?ZMAwyH#r{jiph~L)feBC(WCIXOstk%x
zo@<B3BJS}T4K5sZp2Ag_wPw!%cpBgkYgB+Jyuf{MOFQ1%qIJf8KVDyZ2&`yHF`36<
zMhYxmoAq)aQUbRpaYfPzKw)L*c7)*%Uf}T*v|Pa=w1e!p?w&h-u&}Ei6o`Jo(Fzvi
z(2ltv+~~SDW)z|T+@-qXbK)7FmZDu@&m}h}eJk-QX8H#lNY8<QjOTv!z%mdMZrERZ
zQv)-Hp>R;>(_f6FKai7l_SfK`%?l8cT~>7=)BwP>dfn_h+f-OJb_@LH5S?Mw_)e~)
zN<e7=Jw&(4b%m_x9B@!VLv-!&PCSfu-?r#{FQTU7^e8woWP&QB3S6vlc&CT|Cl+UG
zCjNdbk3+_Y_u(?T{ghOf=TCJqr5j-9$#r0Z!0`x#jyf`M$?xb3pE%bI!F`Eg(Xu`j
zsbFo~`efWp;!(T`$DddaBE~@SnYBIt@l0RJ0{>j@%Zso6Fj(#3a#pWPPyb<N&Pal8
z1TPD@u`J7Cs6GzNx5QkK+iC#vyfX7{`hbTII}xjWhKn2y+~^P`7#yq2R>{NzQDf%U
zV;r#uVDML7En)hh@jPK$b};l985uWb;%%GY2$gP~Kp`A8qar*+xASiXpGm%%^5n2*
zAo&*RnaKnRqmfvrpQ@a`N9YO);j-LQ@UkM<u}8Til-QXO2tZI{&9ugwc0adpF0OEx
zZF+etNj<3gOb!4vPNA}}@)xGYhHTo<c1r-{#HH9%VE9!b9!6*FR^C`qY<e~@dc$9%
z*rXB22f5IT%rqQYU+^|GC(R>ZNI)$b$X$Lx$DWl*FvzePQk$eg%aYQ-P3+ONCkIIJ
zz@+thEWcZk<3bOkJ)fxpycI%DZWCO%9F$N83N;9g3q;<w(059P1+`F#71Yw1Aj#ne
zwFTtpB|UOZ49Ou6c2_A)@1=aksn3Kdc$Sf+zvb}D(<%qH?A~3Fbli_RlO%<OjiA`k
zRcfi$$^1dUeJiBUupx{-g6$mopN^1mX#FlSU+pW<V?)m6kstWu!E*c7onrG-yhCU@
zp51(UG!$u7(p0mRPxbs<VA@JxFG2Q299T!#TVHhnO0OPIW&kj9+bq?hlVP_pK!rkL
zG?eOOet&T?hVb?vPLb`BEnyHGX%rUY86-uYUh8NfaaYkgQ~KM1Af23Ea0C2+`zs5p
z_&dW%MozxCngY=Ler?A<GZi+}#27x4Ytg`oO(;N6BLxmliykSH_9W}mkm=9t7tWr1
zgcZf2WfF*nvTb_?t^j1(h^PfveG77u`$A<Bbqd0P2M~xtZ%PiG%>YgX9`*gfMBW>`
z56%`A?<(Z!R6hQ1rDcVz5ZrGH=^9ax{rI<sZpK;Whnjb1NF$OBlz<<^wU7KhD+K+L
z7VdrymM<Z=KSkuW_a|@~ct?x*qZvRaK?o@uem`<RZ~=IRES=@`HMXeBHzlN!1-<Mw
zBllqnDD*yI2j_dJb%r9zq|KLk`cCR7q}(}xnG5nr3M^m>PV(hnnZCM7HB(^^#gZ+u
z%>{vu#kUK5;yrtd`xxZ9Jdh{?3Hn@O*7-o!WBFd!^8idkQgj7yo+)&_0oFHM5InCz
zCJOaju->1frq8t!;p7EO;00mKohQ$>7J46oq6Th5;nbt_Vt_23<{zM`3PH#9p8_mL
z;1M%H*$c#Cwke81=%WS-!tDp4uo>2V<JV1q8$j`yGTdhH1YuY~)bn>ZQ{uvuj|#XC
zxU2>`pkg=?baSJKbP;(nj+n6id{;V%NhT&4pu&}ycjGKY>QII7BM8^s*TacM^WvBK
zoSy7R=vB|F*~;8(HO0~&#D>=nQSd{KY>oPC1D9)tjGjtO`>D9#ZruV7Z2>IWMw!69
z_Do;F5Q%P2AR&)zYW7Xsj@BEbH)9n<A0vT*hVhfvp);@_;LI@hcT%<9$21ilJ1>O4
zAs}Pbtz0ODX9X!<#=bVFlmdri<+fu|D@k5R?F;9sqOd|6xJ>KCG;W;5M$)e5B!uCu
zVMu~z9vAnDL{*rL%`#5Giz!{##Ad9<OT^`e^VxDM5qA8Eg`70H8m|L73HbPnp^rf6
z{*_8Gs=Ht?S5nEd#UrqyZfh6-K)f-D+s*`?MTz%H<yq?!!^b)ZpHRKG2?fa(gqk?U
zAb^c$AVocsi}Cj*juG%Eb6grwYKmsFg;GYA5+)+3LYL&V7!X+Hh?;C&g8v`(zOpNi
zt=qD2_XO8Kkl^mF!QI{6-5ru3A-KD{yC=94+=IKj^+QhI9{t{N`WN*5N<E{fT6@V{
zbFRIM{oOAo6;SO{@kJ&q1s~JM1N8}@Wla<PI+Kv2{T3KiU7)OHk%>Jrv!su=^Iz27
z8Ja$RF$(neVhIb-850O^X93u~sZku5>o4UTwme@6K}3&#5r&h$I~sAUNL$^z?Kge<
zf;<an8ML^hpq<fVL0YQ=>eQbh4aouQ+ukQKlERc~$||=x!sLlVzWe=QyL_2WTSwnU
z*J^d?p<sdU0^=BT@F}}Hh`z%szjx=eA(4;9<8~jTznj?keK6Bokz8E!iOtI^2-b`T
z(V`#L`%XE){HbL|1&w6ZZ)Z9OHgz7P!UWs%UOt-r(mba`0q2BoIu9f5?@5A!1U8)2
zcM|-OPt#Zpe|X;xqVKbBGg9f61Z#<IG=VgVrgLst0QEQGh=gjuv)?P}FUfauvj<A2
z9PjNnQN%$s?i=`U((+&PXw>rG3`4iybV~z>m8BToKu<UZ82gKTa4rb|TiDC=7Afjo
zk!ud%{_C##Ab?Jc$_{S(v;i9?44}$}y+xuxl|NqRCM3{(+y$!N0>rkE%XhtK$YB6+
z8uIYl66%Ao`}V^DflC1(mIkz84)*Ou@R7j;I0zEne6kyK!zc^@vJt}R?dMHsf1eWn
zABY56hd`+1I|2#-))r>xE&3ct*ZYW~>^c6L;cbWo65*d0UPd1ID;|h-Me^fs*uw%n
z`&A$X02LPVpBHQ*NCLHDus<XH@%%V?Hc)}!X@kFIo93_e1?;&QVBQRX_tyZu|6_pv
zPdxx#TmN<Kb%0*)mAHSV{38y``)`MfEpI-H{qH^t+Phm*TVp)oF*g(?qm|cl9&fUq
z_QgR*>n)HVNdKD|fobFgjfIQ;@y@}@wThT@KihcHsB+5fQq=^~XkN{!sp%lCubiSE
zs>~kg7TRouEBZg_w&d*(s0_OBstO1|xjt2^Kxa7BgASMYEF%?*mMMqvjQs0x>n$i$
z|1&5uuE4q)iG1kB*Eof4P8KnKf5=DS6`^b0^yDlbz5ns!2YOu^&g6S_u3ICIhVd$G
z6vy8V@y2IM<6AAd3guY@BWtP^F(W?7im!hPL|xRQVo?GbGu#ytMzx3|={x1ePG{+e
zwE9*{-V#C~sE7T(6T)W#!24<nkLkI)@F=<KPrl?pwJFb8cUU~D8pQ1IsnzV~9sWma
zYF2aK24@0R;-+6o7%E_EK$c>;<&TsU+zWS=!*t4HnALZV$#~3W{5NeQUI!{(Du(N6
zaJKIPjL2jDM)5iNe?+nS6g1#@|9Vo8ZakN8teEeLwEyx`pGxb+-mY}-ES6o2k9v5Q
z{<~Ht5nX?pdmD&v{VLYQ4q(?Gxo{R4f63vrCy5sNJLo>1`1k(rzyD_%i)91uwo6Y<
zM8cM=sm*s^;}FJRZPA#;YeU{qEAW^no1$`7@hg?2X$v!b#fp{TfBwYpdtYu$z(E{u
zsTw;(q0)F6#P(ztii~mZ@k`Qayqk^tspv={EcFrj)90B7itV~DPqk*1Wv{;FZXP&J
z^+I8tdsl@x)SVBPl}C}0ABW<8EbeC5o){GE>CAcLZcmrurTS`9XwAKnzz{hUwa;b$
zfm0D}9JX0J>m9$iR4d)0#~>!~l9VYn9ZJ=uA#EImuj4@n>pCgt-g=Zv9Gpe+8$gSX
zPQbvOi=Em=M%Kz0@p&{ku8Hq6i$y|Z{z_yV8D(c*6UWiyJYdsk^Q6>Zb2m-*=={KQ
z?>+0~sGfh}INJOCH0#zu-@$x){JeyS^DwBrAyHkxMYrNMbvl%=#wIx^EIB+m6Md~i
z=65{*5zp^whPx8<&Vw8M%2k<udY9jxj5}hvM|^1WhY77aHf6RZa-***4j;vCJ52}K
zYo_r_4M#WgJad&qD!*>CJ-0v7FP!yiYK69QyDb?V-d6m9>t8s(Znnvs+xBWWj+3$S
zt1VUIYi@bRY=dt3;B!s?`t0@48C1KvXtMU%wW0ssT+hXPt-3Jn`(ZoZ1NYhV)T!~|
zso%f}E%(T6R$RGNq?n7)3*Q|9*$~lNEvu0HZ(&sj)-_HJxL<Vw>n$;EgT;v%7|S0F
zcJ^k1(o-8hpY9SSxRJ{;rjKm<S{=$1v&(r()Cps9MD9B#KAkOm?OM30E%?}W6T@w7
z#%d3B>>trmQt9S8N`AeK>4ey>c)nUYeLOlc)-&r<5sT33X3w{%R(>hv7G2Z*bq)Md
zaJ(XOx;Y*PWkpvqbZ?7rfQLBN)=R#_*ROi5;FlymnVE~F#cBMR7HR-W2ok{OZd=f1
z5xU7C+*~KIxKZkQFzC41<RuKvlPQ}@p+tm#+8C)6uNW2=GkkhLSE%P2z&!lO<#sfj
zvpM9H`q0LsSi*XSa&fhDxp8b;ox1usRx4=p`m%+?<a9UfI1m#yD4f}L<Q87J`lw94
z>fPy<;Tzo=u}5FIavMyO^3XuAx<k`uxo6`j_V_J5dUejCpd+)5Y3cMam%Y<U-g_nY
zS=e>!8h){d%E@Fl+-xhe^X2Ab*uzO=8{YBrcxj$O=_T1ONVbslI-^(pQoLhZmZGUg
zM{tVK6DPG;Y8!dyRhPu!?dR6V@~OuyOg4erR13c4mnm4zm*0y4we6W}KBg%nIG2G)
zckRrtUQXd~S(Vt!QU@vPL*;{SIN?C@zvV4Fz<ngS>&48uDjn)L&&}hCu6UJUS5`pZ
zi4)8}gJ2ff^jC^XegS7|+hm%HPP2Wn4sYSRAbJ#UX+E!Kf>evWq}Zs&^r@LlpO2oH
z8WaS!&i%jp1IHv^?uVh1Uw>QGcs(NDpKN}-LG+8gNgOyVz$l0%lpDu9)L5WCJ)|{r
zHMiY8e65RhRf~;kC|m5VG#xV~(%Dq^B7dEW=TAYDD3uYXS9y^06LN0zS<egT{gZq;
zUz6cIX0da0qfk(B*cLGIKFD)uL9HHsxlBG20Rt#=cZ=bfegs<r<J<XZnR5$wv~rzT
z=5ChV%<0ydHh-nuykGoilkS7i?0PSW<6%O?d3WZRW0rDF^HRB1mf+GSUMpIxkf|m%
z=&*ATc6v3_nmP2<4Qo-uDzUQ0?C7-}-6`eNa_S@_RRjWVt(XC<%dX&<&lAM)by(Ee
zAe5VdGTV?>Y(Z+2TS+^L!LVItDz(Pr)&nC%{%VNX1EFS}6ZhfdXV(nh*K5g%8-BJ_
zfm7ylV>8g`5tSO^!FFS$@5F;_+xA+QF!y7B@+P6?hJ@G5R)7B5&`o6O@vYKnK)5n_
zd6?#!aRu8G4MqBlyr(Qau5y9O&e}@s+?vtu+Vz7vC;7S;&;CuAG@j|g_5;C$pD@=p
zCgA<tZN`+n+wK~krMp0F77G8;_Q&hqW=3vS!Vlsk<vcj9Qgz{^vgmd9t-!jF{YN!F
z&qD&@^u|*<Uj{nU&2~rZa}!3bcM{S<Jvl~qAc2SG&F`uC`|-4CqB&;D{GR7XkM|8&
z;*e07i-L0F1F*OsSbXLo@6Yb%j$NO1@ob0`w^oYB@zzSQJfU^Ngv|k^VQsXDu~JvV
z*2oU613kwCw^I7G2MO|9pLt*8nGKE0UvoA0bCHf>t>4;~IP9)W_bpH&8VQC{`_?EZ
z(pM%ljJZ>A{r9TkTi39$CY{ruKxZmRjF29CEp2G+YOiQ6uK48M($u*oiR;Xa(@i7F
z1>opkjof6|aOaH}Y`3}mW?wx2ja($_iK#PQu*<T2OX%MN3CQBGq|V&oh0RAVVus;+
z6gji1H8@Kw_+YtVAQ4_$&cGGUxv-%fGejTr4Y(T4>@8a@x+ij`>!wc{9ibLG=^C9f
z`COLW)6uf7QSn0fp0>t5hi>i7d_c0NPSzI+`yy&Nl_l0`g)5J!b!90}oF7n++&N{S
zcst_z>BxX}<#feN$7Bxf`n2242nm<P9TA_aK<)g>GDFt0UZ%38nXws!)wFrx{saH>
z6hi<u;K$gVJoa_G!Urs0_;BhfIF%N+zDfmhosMzI0wB24g~-Bn@^n%Q;{N+M(*<2_
zHXlpUS&qy0<EHJAbawCn^Yt7v;p>P7Sy{k-xVATM2j}{T=XotqZ8yn)L~e2XHW@jU
zS$4TqD2ct7?(pN`=)xHDVsaGdC`kgn_8&}jqHg!S5#uNBr?gXboB0Meah;+aU|pQ)
zT1#0}_9)w<Dkw7Bd))i-_yj+cDtgeEj92rrDB3M23cK}W&3^dYZa?aV>FqPn5ESHt
zY04_mKmtB`IQz%m++baRlj`#ALDck@Iby*%l(eg|Mrd$cY;*s0?#L7GuqX2qoQdie
zw^4jN=a0YKRwUZzO0^)ykF6mRy8MVd?S=t@7o!tl^OeR@ZTH^y1}LeS0P%9viJfoI
z^T{DEJ9|_o`*Mkb*3MfH#Q$e&be@L-)Ztu3v_XSs%4XVy_1v*VW-OfQhYCPO8M%T3
z1ZHKMkr0Jp^qmXK^Y^*!IP8s40jmT1JYo9oOIp?*i>cWk7Wa`@@1X%M{M~}Wlgo;R
zWkiRCdXLuTuMP!k8b^^k1#2*xS5=4m$s-8TFrUMM_D~b;V^{Z<3tvoJkLqTHxw!(S
z;~x7L8MnB$B*F+qEiaG&$3?Sr(?Tf}=8QErT)$2I`<q;PM;@h9#P^f6b11a#zucy?
zZj)b+5_Pm1m$FGurPD0O84m3$Tg@U7H0P2xT!E}xN@v2^IjSg{<mUk1pp_yO=FUgL
zlhy%k90vC2uom<jJZ?2#XY=p6gBR&LsgJs=_^b<KCDyjVbAAvMnJbhBD8F%?(wqHE
z&*2?T{2Ye1>WY!QJlBJ;o|rs25AQlK$w3El1hNi{vkB)p-~ImThMAl?d?<N3_^NAg
z$t$bS<_2omn>^9}1LSgm2?%iS$)ga6o&CZ)_2S;fh1_ExTBB+N^tGb}T_dVpqaNgl
zHd~7cqF<A7<SP3o&5+G0xb26mfUI;sB=8F+i$a^R&G98_jIjq#vAeq2-Os|zSD$O)
zp=1w7?%W7ubEm(Uu`w8Jq9AFD`aZr;Ea=x@>^I%{DiI=R>Q3NvfC&sQRUjqXNH9O9
zId=F#7lN7TTGZa+Yvkt;JQBG-@sh7kkuY;1y*hv0gIM#XXwQD!OkAroYfA4Laq}pH
z0T!VHZYl2)^wz8<yu@=|ZwTR@qX-H9d>+zbv#x>W!atd$FL*ruV|TgUwtM3J6IydD
z@G8Q2iE$3dbFJ;$`6|;=OKDh{+5S?|#-X3r{)fT3yL7d|Yd>SM#=Y-}$L;4O2C!Gf
z@Z8~jabfLsEprS`0q+0$4HbA=3j}e){sOv?%do%;3;?{oe%v`P`L81)kJ6#q-v7et
zkA6@?L{w%_KtyC<lxOII#6bP7nAggk&z&zU&;ltc00;3|uo{4XkcWf%N%#Hr33`{w
z&F!?~adoMG)$1OKXmzEPk^3>do!MzD&1oko=+kHLyp2p+Nq<Frx}Iom-|yKoOM_$d
zjf0;HGd}ptj{Go+=4NvgTH>Z2kmEm!2lYTh?JAO@H>r`kcy5W;m9&XYbj|Wy9bB_V
z34SO~)IrpZarCPzS+_d&+aDdfRT77a%`&nt^_W}}R`m*W2&gW8Z^j%RMON}ZDL?Qo
zsGlBOJRmsELha6n`7}1#EYp;lwv(UI>ymVyJX1N^HpI&PI0#z$+`(-Zb<_);0XB{<
z$Hi|)XmL9<;4T|^n0AXv7waJ7b>IY3mPt)%q`415X$gIE{0XxbF`Ma$5s9@<nOOJ8
zY)V{+Q)_a3w5{46OGu8I70lbg7`20;jk7~h^bH3OmXA|s1FU0_YS8V(m62)WyE?wr
zb(#D_9`n~eOQEAvt<Dhbozs6Deq82U=8k#zJbhy-yv63^ozs}wJf>3UA4x>AYbPbO
z<S*^)>$QB`)yjJ`svq6%$m}O{OG8{}>T*F&iMpM6tae?+yBHk0H3s$ZB4gcNI&RMF
zoSSL2A07ke?=Bnp^ISF@FUB+dJ~zuEZH+mi3<o?v=F4ncTrA0T4otVFNOhy%=6jT8
zcC3-woR$=gTQN0PYWrT+GgfNwc!sv|=(q9t9%z_JtZG%nPdjE%^_y`EuIkkZx%$-#
zSF}AtTX1x!jquc52&_;OjSL-4r@Q6UZ}0m8x&q*20^s6F0@%lQjPC%$xf));hVziK
z`EHT=14ZFS0I_0OlFx5fc+l^bR?$s)Zoma0RQg9qZ@~a#KR&6-Lt{cJVGFk-A|5S5
z!X!j;89FdB%HaO^@GNt&+S6$AYb3<;EHprOH-B?`<kB&D+W66wRo?q*LZDi?6xUJV
zr7gy2EYi9#ZOd!f!&h_7=H(pGQpT6y)JyEfyH%;8v-A5qpOWB(<9m<6$onn7{#QAo
zpsKmUFVLR$9=#AD`Hq<hp6hmWX-j5hGz9G2A0Yu)Ng0GVW$=A4D#d+=?#n*OetfSo
zTw*<p!>ZNSM%?dq?w14_%x+(-y}i;`!`$*`z6pC*4i9uYjCu}wA*}vrYt*HPq_~NZ
z8D5JwU|j@l5i<d8J1rRga=<H1{fio~R_Jkkg&$=p6w3cL7)x+C_$tf6_v1N_?fIHn
z2?!@fg9pMi?7u-u_(T375db45loUV1*D77K7osmmVr!I>0`}bPOcpLz=y<vPHNazA
z-c{1IcD$%od~(iZ!FH2qwTarelz;<nc-iT+D0;12CR#ta*nMc^cTmp_$N5&k9{)S&
z&y<bj=gFau$5OS<+_~Ug^PwwOi2d)ODig)%ucBHbl6*Xh*DfEPZ0%%!;O}lnw6MiP
z0zB%s{Raa#<DCiI8Qu~@ZnoIhWy$wU?bE0ld_Dn~j$Rc><P#|{$s>>O-H^;5F5_H%
zuhV`x4wH&*C;OKFX;End?^^DV-1G8nSl4}C^O<Bt5C!XsOa=E(g}eO@z<u{80)q#9
zox%BT`sq6dmz4A-P1&lYrN|ozvS2VL?yhyXxh3(eA^rcjP!EtIWk!8kQIo7)?etb^
zuM|oYQ*5$6A2=UFQH?!q^rc8#iPKAw-6vXzMv8+oh4XtoFNEZU@DJdT{;1!uF75a9
zRLas>U$^>fA5^%N?HMLbI^y9I-;BnRov7^>&yulBpTo%BUwgm|vQCb|w8?50oFk~o
zvv|C)A9Oi~a(F+<c74Y5D|x;pv+Z(fip31Tj83Maza=VUoorVfli#*yZbZ0RDb)KM
zjqY+?!9~hF%qj{mhMc?F$8z+EsJ(+r9)oyUU(qVR-Phs^rt{(m0eNAXgXUI;h0Qtv
z??xU5uN(wvm5@*emab3ot_RM_7maUhLy#!s)~FSFc#JB(YNz(?j4RsY*3~Cyl>jhJ
zoun@a?A1HA7>}ck!Ui%!qK^Ui<T7CE3<$O{AnOY7KvDufg{l0V7>aNe>m%7F1Dd5i
zn=FF2>gE>z?E14EuvuTuW9#O!k(&2%Iq7;P2KX-^))h9!x9KgNoi5djhpJkY+WeB^
z<_@d*LiC>1oB>xj3Fu}gR?83$n%28{D)L5N9B$0?F3zmloBo*RtYwFH9B~IoA9JW9
zDbGrN*_nx3P>2cVhkzDB2$fBUXV#m!wPCl*OY;*YY`1ywZgP&t4R(zUdVMJ!wahqJ
zRhCt}IhSx>Rg3?O5uC?#e3&E!em_zK_%mJ8F3Be%X?i0f8bn7!a$j#x?+n98seZz{
zu28_$xidj5`>maMt6%PTp&n~?#Ws?^!_TgO#8S9_dIlFD5YBZa*#N)_o>ag`Q?B+8
z(tF)dMLuHks@GLMUe2xMFN}sgN$7X#em)QBgd5A8^eD<0?Yb9y0Ej#7ez&p-Bsc0a
zpg5&#_Hv6_-;hKIT_Ryrr3iDCfSqla{nBsMYbJlVy%0G}L93;=@41vQpA;XbNpsm~
z27SZ<a{1F4OX(7<ht<V!3$nf5(`*ZUD!NDIbc+EfI>Mw>2lt?LoLw6_Q5L^ztcaU`
z1V=XAt}VExF>uEBHOT?i#l!j~)(24M8=V>>${x#jCR-BN5_y>1c$wVpulqCW7j}D*
zJ`J+Et$7;Tru=e)%O&9d<O3kTc*!U5@Js0u@QEV%zI;`thV(azFF+Y<C*K}(Oa3hn
zL0p(yjs}c)V87du@A+9#*)f^odUQJ_;oGnSfm3l5fBRaiXZ3zV4Zypq<iAta3k!fe
znll*JwvlfwO2iw>8E4-o>7+IjN9_5|(8YV=_d5Lorvg(q`IzSSov7{(TmFMVmG4-7
zGZcgzKJ&h!3Cf!FB5=uiaCv?16i0es-j1>;-wR<Xi;BpAZOd&J+=a$b6sG_N+=dae
zW*Iu&AKvmiJIJa){5&#CE5T=TgFjk}A+*yJB`mtnTkRu)g=vpR0ifF@P?5)ny{fHO
z&YEZ0a&2$^c+B`;&i0>7@_|2y^b?KLUC}N{bSyYaBn4H)9dBVkAu*w+iDol4Orw5e
z@ojNzUk$ls=x^q{1_-RjTc19Bl()Tn2k<G(b#3j;5&2DBe?gRIFUl=Nz?z7vx$#L`
zBI;}w6Pr2AIYjMe&~iL_jfy2|u#Il(XAO1!j(5U;L|S#TJ!kSGg&>HW5N>RWlb%mc
z&uCTZ$gfR9#Uo2h!x}oH!S|{8ifWqS$0_f4oFc)Bfl}*0z$=^Y>a$VY^ri1La#uDs
z+_h7hr)l2w7@yK&@$Ed>NYpir3jYN+XsTRUc2kS*LHZSXLjajVpDd}rTlz$zkbw)i
z(=6{Fe<3~u(IYSI@N83(jWV+QKg&A3X4k;T3-)dqTzfe7TfFSXA5u-v33|-?qb0$*
zev?91hM%0zJVXW-@sk^zL5@Dngut6eHYdV`p#cnMcVeQm3vDMxPR&uckhNLr3f}rm
z4E3AD<=Jdl=?*6go@e&k$Lt40bkJ3@oe)VI`oyR4sO8YR{gM-V9bDceOa>Y`(BJsQ
z(?h{_+f_0qAh0)4@mzZR^z`Au${C=Q$OZ4Z^lunhlg0?Dib4NTo(N*DGp>Ltk}CE-
z7=!KT2JL=8c30Om=3S|huB~rVxYW1TrH)LM-?@Y(eEd^rsrCfH)v5759t*B6+w}_5
z#)C;3idlC$7t(79t_=!Hs1;doZeslb{#WjoB?^HAbRt2#9UgYibSV*Mvig9JMZnWk
zsWGn{n0`>o430u+ZZp{dhDjM)mCbLyO8RvkzcUe=6w|06&fsZrJl^sr75Jo;7&%xn
z`fEJu%<_Br$FL-brW5avECvOGI0{2)7{ASS4TsWfhuyeZXNP_doIdxK>6QwGg|w@i
z_)pATkA6`IW6WhyO*XJm5Q2eZBICft`lO`IYo?OCViZ@FR4s5)ai>0_(OQHE@q#Ya
zzi|s&Oa>5tn1@@a87z-p!+Q_eb9(P_!~jOwf{C1-2f3%?=y%N&kP1QAF}+bvzqUlF
z=XW=_LGWrrGNsFd)}xI@zJN?g#&9Q|34S@;H-s`aGEcFPdxz;}+PG0;eZf->luvcW
zFW;@FckEI;bTcfT=V}5~da#J|s8G=R-tT)*4Tzq9paBsYfDmfW)5gyuio=Oa3!Uh;
zY_w2*06LGspb7)hAEBa+W$3DhFC*Co7FhFpj>OL$Q6%0=b9^XvFoQW}tJ`~YJ)Sk`
zyG8{^=M-!bHi8|!@%YMmKv@o1i*0@;$H&`RGQO-(@DW?u5VL!q>?odsOJM$v=x&#Y
z^D*aA=h|8?C>bgaqn2rcCe!0(!(95tL!&y<YqLWiif>ofQ$ZU%k&2m%0QLv0!NIoK
zkM|~gQyw{ycQ@1YGfq1bop?)-h~%QEzG<DCqzZj+5wk1#5ey(PV|0Rdl1?&V%8v$o
znfuz%CeN}W#}2NB^xucx^E=5r>L!ia@tP&IQ*Sp+vV#IqE}9lK5Z^a+VVq~>lG9+{
zAIB!|m1<3(5FE+}D=w9sq^|b(7E0<XEs;+NO0P9X0)2k->Wc`6O&gwTPZAt+*$wZ}
z=9tYD@7c7-7H87PEa9Za#ajIJX<{wDpbtteK_T$Hd(jCVz!?cm{9DTlV1p+e?%pSX
z`xB*~MH}dwnYLf9c*J6w*On+7fQ)VZAjGHjEVvZ|0`c0ZM+8XVT_Y8}#tU(3Th8^e
za6*Yjn4GFLLAqTuO|Pv^IYRioB?c2pv>y`L6+qh=?I&KJif5ObHyC}|6r&3qy^3Kz
z>=x8B%5Oi0n6MgTMl)e4#=sgV!d)tMLh6-GUbn(ZYmo3bt21zIy#G<1K@s7a1};#x
zC}G48<<A0~Gr)cCLL)(ab6x#+z|&r^A7ZV`_^O^LR^G3X?jO4?)U{>upvN8y6Ersq
zc9(Sl19raa5LFcW)DF286-uKMx#WaD@X0}OX|Py|855Zzpr%jciCTh*1go<Zj}_4=
z6qE|W0!iszWjyFl^RCoNj5HVF!3)1t>5(HL3nsXEU)Ft2w98t2Z9DhA5GIoUu3%Vn
z0GZPpiV`qJ#iu6)Lx+tMz$OS=gOCT#axq>Q;em}gLDe`Q#a`qn2)h09&$+^)cje>g
z9up1vFU0<h{@bkg?;xm$jcCbYtNP$)H5sX3!MgSfp{JMF$w%HNIgsc$_RdUsKL|Jt
zCuPPiA;rzm(@kZUans{eJ~J3)q-DW?(k&HWd}boh<~#pPxc{q~NnKl444mwH4rI?B
z$CEsHE}*Yk^J1{%8c=I~Vs&Aey65f#pvGIKe#H#cM@U?8j@)W~8ETS7)9*Q3IJkA0
zH@}M`h7D#$tR2ODfmN44w{j)s!p=$asrQZbzFnB}Tu<Cxq>itg^tk!xiD$|W`yn9M
z*GhXj!?V>pF?4ZF+)u%8ea`qpTe8m%Mi4W{{K-Oxl<z{j9A`pCj!%4rJhfwg&64XF
zpzYHyGt_v1AUI$mBGxM5&~a8G6_0eAO{u-3dnY2ocsJACfd?VDv<uyj_7j3ns_+vn
zQ_kWqNt|Rb|7fhH9(s5|`l<ry3ZC8J?GoE(d`q-ql+z!d6N>?4*zAaIrV-`!%P_8f
z>!0<acFct)57U?j8As}RVHvX2(^#4}G4+?L6MX(>2F_v#mL6Ifg7S#wf4Z}@3!xp&
zn!`vJOhW*Drm)NB5{=6J%9XX0Bdej{*dq;p<LfyP9x4R%SDT!D|AcTeK2j+#zDy@W
z$x(oz69-6Rz~*kAQ9IYb0od5lWpf%xmMG%vy_YK&d%T*tUjyvQ1fihUa}QiqC##T`
z(l#|kzIY0he>AjGu3nRc<zlR{r6Ay&TSa+1hF`%IPu5D>#I7<}Pf$Z@K~pia&OPxY
zKykv)jApB#`DK~d=w?wk;VSw`^!V~2@Y*`#U}3}smYhW$0V?}PI@~txQnfDV^arOX
zT0r?|R>UH0SG0#T)d{t`VrL?%VjVFxp|jMeeY4Hi%B&T7HGCb0+m)lR!ROw7PdB#m
zUb+(S57?Ei?+arTWxQX<0ArRcw(niviXRcm+`)#f^os8ac5y!1^MGx9I?CA{wMc#m
zwr8x7rAU{H-d3td5ztKj<E`xm>W{aBeo?coby%5N1VHVGgx5fK!;iJ2+NK+%*D3;C
zJ&C!$?a&9CIZVrTg<$POo?3IuG#7N?8U_>oIyc#pR?o$cx6V$KI3lXIrQM1-84@r*
z@+v`XowyPj_R7UYG@ykIMs4VY@qz=wg}`eO5LL7CJ6=|WxRg)X@;5?bY0+m*oA<L@
zbt1A>`nWLF?I5C}!i}Q#G*CeWED)IV?#mhg4<!c;C16K#rm<Hl#JI@spd^%q2PgCq
zuM*}7^>S+=S;nA3AqQEJ1s~L_HFj*ga9<flKV)6+=Cc3xB#~4P@PNZa8p-CXc16O(
z5HZ)TvU>i5N-#OrA}#L8sNi2kr1>QE+mm_XA*eDr-XC=x(TE@c(8qKP-oayjr~Yb^
znzGUD$uO~+TahYDK>XG~J{Ni7^+217^LhKkXCRd<w}y<!M>V4c>!12w6fa!U08e!5
zerU=<;kIR(QJ&a$FH#i?=#`IM-B?i_>ikeW?}E8u6;X>&HDJCaVA5%6yIQY)(l?}!
z`tnqx2;3R}^0GAe?b<S5?JGQH7RZ1YW=C_d=&q=@yAk|KD8?-dM!cbifcgE6Loj^v
z>Tn(07-MvRyr`aicGOP^B@LHnjdTd^V*6#&_&h23L7^SeA4-^fP^ccB3;G19|MvHx
z3pU-3CjMcadD?I=81<xI%YhF;bKYd|YRvZ@1|pnt%$?o`64>geBdbYGLsDR!Z{h=c
z@|0Yv`8u}{?5zR<sv&u=oxerux1WcQ5u&fYulJ3V24J_rT^GyD<<-SlDnI{xb*(W2
zs`?OB@z@W8^ci>?9N4=pmn6ni_IlAlTi~8s+%l=T&_~KVd341S>*bRT6{|~cQ+PPQ
z#+s5g0;>FXe{ylFYY<+u=RXS!7D{IyL>^aM!v7in9TjA<J)&0M*Rk2d=wJfZjkVE7
z?QB7IsOuq&0MX|$zXbzk8edbs)^<aBtQ@gPZjul9tE^Tpa{rj?NiIOV*<J21y5<r^
zUG}U<G9{Aydi=Lqd_K3V`VL~d$kt`vyt$Pu?1uasD9|)w@rdS9#+Ov)yCqjrn+OrU
z^^-KTBmzJzZrOI=uLpfX0}68`?o};~%1?Se_N_&tU-cVTJNA?woDst-TPYSX@ge$c
zqkm6PN8P%6ea%o|@W2BDQm!iOq$6OFcbXX8y4`5Z!$OYJs7S&6nNc3S62mM9-dZ%a
z<9c!D8Whn{z<r^Bl#nOmJsi1=y1+zU6(r!~S2+&$KGlLhye>^C67c7-7N1k8#j*ec
zzR9cZ`$^bew8(khCRe4%a{G3Aqr$<T_JDXoD?fPHI`(2m6_stF^eo$!wwFL@zZ=el
zJeCV}L=^CV&l(?ruJL@cFD(y^5V56%mJFwj0>xD07Z#T$p1au()IuqJwX4z1Y-oJF
zZ{>|lXtWmb&1T;P!QWa~Q5j4yAV?KaH|Buah6Z0bohku?0h)><G??-uHY(TjrHtpm
zgWB5c&0W<nJ6w#WK3A~dC}2{tQU5&$jV|q*84k!Y+a!2cDKw;+;eM~$@xl;NX6oWj
zim2eg(}%8gJ(CxRYzVe3bu_|7P3{F|V9bnhS5Q7s*e<}(O|o7vY=qCAMgbB)boN`l
z%tdBycE6I=V0~n57D#5O_pxa5fSfBt1N2K~j>|uzrFz4<sq+#bFRQ(4TXoS9bJI5k
zXMR_;TY0V>)>OkN+o6xlj||JsaBj(OmsdxrE@tQE<@Cc>?R-o~KBUX??U_af2`%pG
z@ReWB_>Sk>M9+2dMhopTL0pDJjAV4G)`Kr@?Z>otQg_#*#9KiIb(`-1p$g~KjJL>T
z5`J^D%C^A|K$)LSGtb)HcO0?D``qiSk$dfGNdHK?%GeE76r?&^?{(yH75UDWGYo?2
z-INbd^`yCjOS;834d|<5dG=$$D;>-%Mw;OM<qG{>$U&B7L*W2Wiya*7twqapUGJC@
zdK%1Zq73Y*ReZUixD!d0XH8w?|6!8^di>f1ndCyO(!kD8W)OF&?dL~Q%$b*adC}Dw
zZ85KcU~neoe=5OKyulhh0HU1h&r;RkNNH_-ggE0~^K+l2Pgz95o<#QK?!fi!k@_$D
zFdju}kcEJ7<DGeb9Jy+)_fy)#T^-hwMf*&4>yQL5&J!?~b<DLsJd=(8_|Jn@_r)9H
z^+yQ+0rbp2WRWOchnu^U{ezPa3TXvwJNJfgKtpo?C;M!n)4FjFeIM7_>Zx;^v=rb^
zPJrt|mc-IsH<--y0hllC%kZ4V0exYEC7(4ky7%#$s=3mS2zWraq&{9wtwL=>j$XkR
z2+O?>{s<7mPrQ2{iVlaoXfMx4JD2Pj=vcLKHmhlai%|*4x7k9KZ@A71POOdQ=oXd)
zI8|~StsQrJ)mlJoDQ+k6)@gDddx004ar{cH%B=N@h-0Vd*wU039V5-91~baa<bK`8
z6~DnR98TWdo$^IwU)EFa)Y{)*$!~`zC^0n~8&4HqY-eKPO(g%>{MRSYa}G3M(TTy(
z2e9AF(D{RW7!klHqNwA@n{?*NO4<De3QH=-iD}wvi|{54YF9p44vhsy(weWI(PZ+O
zz)C9dqo1devC=T&-88|0KNGifD*|R;Bq+nf|48q@2Yn%^&@aSv5wbvj-|!+2{{G9J
zFgXsAgBTRB8F9d(2PX{pf03nsFN;nhppY@-qj?a}=`q{v6PLT+wBV2m5Kw`2vH9WL
zbq_0;u$eUS`+>GrmLm|szqeZT(HiJ&KHiO<QYeb3^^&dc5TU5=O8N{!>kWRvgY3DP
z_F`d5T&7zk{BwM{3&E~B*vA}D{u3ps?Bs8lY~&wI=28O-zy^fNhfCCl@|FraaAi^_
z002f3W@Ex^R0}rQ>cc$mn3R!kBTR#O{ngR(Nl2@qP8#SZnGkn`#MhdfqwfcyAE&8-
zq`gmXbpDm!t!QGik|`u0f@SIQ80$GnzpKBn<D54O*5wIvW+4#x%#A5ltCRx|Fk}Z(
zy$2m-*d`RWIBxLg$%qKwhosB)evf&-G)IEOAiH_B)NIc_$Fu-(!_cde@7;TZU9um5
z|83FAO8k{30?MqXqNw)QD6x<Wa!>fbt=yg+)Uf!QZo>Bl^u52vvXBedbN$Q$bDbc6
z`0}j1=JODbT>2%PPglX8i&tLMcfLV5tUfr1TGcy}VgdOl%1k90O6lu4IVqse5G7d6
zJOzg-*V6*2UXIz{j-G={(t?Jb(?Z<xO;5<%<7aX|uUC262|N^6n<~sgleJEYY*!UU
z=XOkvLgRz}pd`+8R_W?)#H}qi3pY%?da+y5R}2&qy7G{&2oLG8Z{u?$8oBW6cZOEo
z+_Z!gob+7$2N7S&9|yVVX=R#lK>G@g-Pf48Agy_PVQ!=qGFh3g{fQK+7gu!@ULKC!
z2L0yB1NT%UN>#8Y29KdF3$MG0#-q<9$iVbL;s8_gF`(0+&a4*`NU<mOcj+O0aMWqu
zTs=JOts&EE{$T<Qj5Bm-(0jObYkksvbl{@Nbx8U3L8aHSZjMC2AtMpchZf`T1vRO8
z&aGUYVD#Q$2?PZg6DR!3>r_WyCHr)7xQAVfsNHz(<;id-5>IV@&taBp3x&<r(X)U9
zH-8yWWo9h$-@6z42*zTG2A&2Lb#+-fU)AQ2hZxXG81sS%fs<FWPHPoS{W5-l+b<Ti
z(9b!D?!l^9l~qOhQPC3*qEExpp}MiKBd4?1wQYC`&U5LhIu}THKA__I8o?+f^p1Q!
z{dDELT^A*LblE_m7U4!(6r{jx!@1bOYJhkUg{b1^D=zpZIZ&SFr4Ibnr#~HiHI3qR
zJ=D>_IgNE{`#m5&<#4m%6)G*O8vVkG6!}+Tf9;hDIA%|^=SK{v0nXA57hCy9)e&{d
zY2^XgxZ>-yY{!k6#9t$em=z2~w+OZOFMr-Mfxed(!LrxufwttW1qe`T2eVyE_7{b#
zR1G)A!fD|K!=cu_rhikJ>*Or#ZTA@$SwsFmKATtr>FQlA!;dG_!x$qV2KPom#Iu4B
zeX#6Le-v_FY>sp<4O`B$aj<sH(%FjAfVIrd=M5?lM4F^C!pr6y?>OUGcpc{tRDZ>0
z`S6nh`WI}aImHMU@mJKGQQ~xaMYe6gBkHN))QkWb?J<pQZr%(5xQlWg`-pYm{)j#C
zfuzpx5(k15JY;qM?y@hK^TM!F7<ppisQ3QDtll5ndM!IYtYI~eU-?~HM*e|Szx+`U
zId37lwLViIEGMcXgSvQ+K)Z9QrYD}CHCWft&-;bnlKtE&k1)vC-fT+Q$_lpJ0eYEJ
zNTk$EnGRzbj$a+${Te|;UzDlKwtxeY&iAhLXC$^h&}7}*X8KtQf-{~WBs9@GYK`1n
zsqz6!Djbqzuxh*8?dp}_5#GdGM=DXXlz1Z@`Y$FF^Vz*<OJ`luh=vQKte)A8JJy**
zTWpXugwb=e(p5DqBa4O~Nq+~lm(SdC5uH+ERtWbXen&&L0qZ*C^3r>~-4P2mIzmwd
z7_6v4A7r}Oj>(v8JE|c&aQz!JB0hLgfsWbJA;S2oRo-Ri%#IC?Op?LDMKCZoL=y;2
z*fuIW+dV8tWUQiyt&^?Pd<ER!)aMLvai#|)>_4=X=ecoosn0p_!(qRnw4GJm%t10B
z>k0dacdaQ*w|RzVoHKD}7U9Dv9*baVqx&eZu8jzZg`7?^Og=tsz&(0B@kvi<OLe7!
zguXKP1_k#DB(G0@i%E7Ob|?0ft6tpJ*<lj{y7UVJHNRlmyoX*O*U=>I)SNF)9(MkP
z-^gFRcu?9~C}zrOkC<XpC+e=tEqKVM8j`YnalY#r3f3>H9CRmS^0o9=Ye|V?#E<vP
zn0v0{9hiquEVS~9PL>Q6g(F97+Vr@l7knUT{Y`suoECOV)ltZ_A0nt!wd(s?Pi}zv
z5}u;MNo$78%3zaP0Qesg??o!qVlyl-!MP}FhosP`7Pc+q^OFQ`_{_JRkhXuNcNG$7
z)qoQ-S(=HT6O)PkXk@zovXR^|2`<nmZg~Lat{nR6)7@i(T35qFqI?*>7zX{K9)BN@
z5+>cAcKh>SP8|xb;e`S0LP|>|{g!H4KGTpNA~?T~orlna$Z#7BtZ1COHV4_L9DYQ|
z*cBQeu?SPPAoY8G3Cj?rUb~m)%(YD*)BC5B9vqID`Nea0Dwscc&oa71oCC!oD@ev|
zoIc&h9R9?d5(XN;&-Q%JTX<@;jbMX~4-sQXOJiKYnmu#&d?FeBdEv%ol*IRkK+8YG
zA;SVBa7okoa#yPx4ol8}om4<!zyMi(Dz|p}-n_)=6D>3J{yEtQ!_jJk4g%^#1zs#+
zrIigBfDn+qvU+{ZuIUvWP_4)Eq4KE;g@nkh^i^Ja-Lo=eaHJlUt>gJh>R205wgGMg
zbh}AkaM<I$M22bqYj}G?j{D?HVj>>Wp(fB?08TYHrTuceRvJ`eeNV_RRm!xQbnhQN
z9F}-J!|bUY$_i;UrF`^m%07l{fG?~p#YPdL&;z*AM_5-0-BS86<+3h2gluHlr#(0N
z&U@VA+$VeDIC86hYHXRERM&PWB$AAbCe^r$n&h|SplL4JicRrapz83@4(eMuF-h5K
zbK)c-Wkq5rMf00kn&1tNB~(EOAUJr%86JSslE<V`21f;mVFd(&l9e>0R<kkLAlSv<
z<$?`4pJIKKHWb|k3A?2lDHzP(mHud>|LwdT4)}lnm<I>y1d?#CB-P&25hrOPejpeH
z7eIYVf`RjYt7=3ZZ$dYdTLiek&K-e_8a*MilI{ur*t8nS0%w;sGV+reW{ZL)${|^Q
zrn^uLzar%uD$4V-3G@K80DCSmC?^;<x_7{`bkBSc)o+)|wqk3G%c5fC+a5uKxG%R6
znE;gkoyNz<<afZ2cIas9`wa~HYXbKB6c>U8bSO9W;EeV^4Tq!(Ujm0Y)6)LBSg_I@
zoZDOkEKn5?95nHH$O37&$?v(FTo9X;CN@gm%D(2obQ&wL%Jpr=B?`lazex#blAwi)
zdg{>Z{9Kvgan9=D(qZvyPHcg_<GG?CS2Q@V6#jkqFdK$zD_%nR%*NSzQ5^T$w&Qud
zngkCYPzCl&h}B-d&oZ4Ma>#dQzXSBAnELbyM1=XCo(wkC9>51Q5K|XK_(8Yp+`fOO
z{9^kd@{{Q$k>&msg39uamNR9qx<^-(Yk#7a$#7VvMXG@|jMwV%5GIxfNgS>au_~T3
z<xOomN?K|owU^ZVaZewPYh`mB(LjirZVGw|)}uzO12?F}ipDSlk<!K|NgD0gt`Flj
z9Qd)kPR$pVApBVcDsoXu0Bq|l#?}iC8nB)e@=F|b8=Pc$f-_mU$=c!Ayx#mbVyzTi
z^M<7bn~lIvhY?Jm<jSr**P@GURTkDvwcal^2||X3j9HXcF;K83o!GvM4mHX_7j9!S
zbjG}}_j;k<fnNmE-^PH3V*M72R?VRZW^VvuM{{2DLjt>@1E~imSm%9@cJhi-Tr=54
z^6o=vUWjHpqbd`%q0m4HTUBI)0%I~7L9guknz<p790etZRAkZ0<aODOrw9S3Mo4Mv
zR-ZT5#g~QX@#s}`(rFcI3k<^vK4^o(mjjb6kOp25f!XKVVZ4A0qCbs<L7y~}-H8&!
zUTd^!bbZ&bIcG+vrYi$ZhRPQ3mj*GI^9)?(X9fod!65X59Us@u`-f8!6GSrLnZdBA
zWL~Cr&8$Z2+a)t;fpu_^QGYmie8WypKP9p&Et*jlQ>+Iy^8zC^Qq!CG#uZAKIC4B{
zgAZ|d!5}2=C<ZO?WEjjUe8+D-W$T_*g*+a*fIw4h`y&xJ3;H@c2~y<<@bG=nn{L$K
zi$B(~62aofN?)J@3<TfaBgpi0J%yDoebfa5&RgU-7*oIUohNR!Z*w)*3Gnf5)_5F?
zb4is^evKDBhBPoWuW!&?icU`E9_B6c5VY@+Mlhu<L9wmOrJSELz!O66StLo%xRh=_
zDtOYo?6ljpr6nUtf&Qb-|9CJjVQuci^op<Wm(f6x02vH9`frb#3L1mrfCAij@#DRi
zsDU_Z4V{xxP0jvVTj!PBlnvTw!|XUrO%8_h7@JpcEx;efFKD;5RM5uPhjzrN>L-sO
zlrY0yo3oIzI}hTSS6mS_c7)+|r)ALMK{q2^-_Q>TYPPu@&hSE=VlRyJ!uufg-{DJ<
z-iHxKAMQ`u7VyzGIYJ0z!2nYhpCPs%Hh6q<*?YKh2c+uC&~ONg5q7g$4BY-(AAvoH
zSt1O07LcJ5dz6d2?=lv(;np1V%DCVJ|9sj;4N+fm^~^RQYIjFvdE!-*o;KCk;ZTkI
zc@Ts8-Lo=z8@@m6ZU^DFU+}({>ts_A_AG;NAgV)}rr`Tpa_J4UHxk`T^DV!*N4(J(
z%aDK5OVi;-(XtT&5N_nN17$m%Tsi|x&oba)5Cey=C-{@pW+fnz&@<-yXP@|jhwn-j
zofLRh<qpZNE_nk=(N=v8fV_(U-kc%BP0dj>d+8^2+vPkrthW4I06+5>a=*^KEvg+f
z_%vOH42wa$dvR3`&s&Vu+5FNWr08TW{>aq6lJD+vjJ5s_$a+4#5>WAx9mJGeFLtHe
z=^R{d%=C>(fEwPb1hHH0<%tX$$pYoR4l%l5)z44DrCvFx*m;d~+Oqss*-XFkax^&x
zWN+Z3k@`!+;TzZVi4rUb;{6$v;9!SDigquGKNFeK9&9M?NxkKHzW=8kssWC~@aT_#
zzKZ}2m0jv%m+~(4b{(!1aE3c?2z(2vuQDV*Iy_c+*|{Plt&H|lNX=(SpITEf=Gb*@
z+O=r!Tx-FqhoJ3v0(~Wr@SPE!FAO`sHe{*-h)_EVlTsVpd(gRRzWExyV|YjMvlSJ)
zVsHRaptHWVej&1%f)5Gd&d^z#gWM6e7Gl))Q$AK%z!@aTkn9qhb;4>s-k#1(rWHLq
zpM98*`iSHCJe=S`K#Iw>XZ|t&bs4_G?zvRcUyUl`O@JgI1j%Q_VC?^-hZ9eZPq(1*
z$FihW>i#<B?0Rwu`6@GH2^!^KT)eaTwn`4-ESVY;1mzmv&3E*ovA(7^ih#yK{a$&q
zWc1G32e*9D@Z6H#8Q?gQ!6H{#xfQ5``?oj2f3U|d9nm{_mjG1h50GhaSbufxYk&gc
zM(ukRkr{Gp)K1da?E`6WS#?eiK|@eCFjbi_;6KYwNKNkRdMCx!!|anUcq!HY__jy1
zPIvZnWGsn?{#E>96+nt#uhCy(mmKx2gm>wnh;I{W0aeFW&-(aqU|F{qNXAQjql<B{
z%<$wcZH+Zu!IT-wZq8~1<QTgZ$*m%yXPhiX6DRR8j=rlr6*M49N{Ji!?|>8A<?~wV
zVGA7^6hM>2lr3uae&<r}81R;hR~%xKh_k<Hxd{?TZuf>c3JWSmZSr~QvpjyXj!Mrs
zq$%bu`KIlDZDNYWFHHPbJWYBNPtUO7K*MrP{}d+sxn6~43I!;CWYNr7*>+jHn+;a(
zqWRZD=etSJ%<bLmLdd5Fq?{Sz4P5PS#;KJ11h3CuJ_2n_cTgTSKUsWc*Q0k^(@ZfG
z009G;qqA2+`h(4b9(F4JaAb#tTO%=mneuR_lj?A%J=+JdtNylRpQ?F6`dty^-$8`1
zpnqR}(DAfqw?uxHsSnSfVf6n!Ye?#HIYi8gO~fe^R(O<+ue>RvE?)|ocD>$@_hgA*
zlHy%&VMze|!#Ts#f<WgUbnuANhME+LyaUa!Fbar@1inJ`)b&{4KGMm5bJuZvj(4oq
z>_W_zpH4kuMKVlQ%x9O&$YlJRl0WAEGxy>TDbWR@>Ooqxe=A|s5U&9OaNXzp4Zi=2
zq&n*%Yx2~_1h@uL-}MCbQ_7Ga><}YT%Rwcq4;*uXJ9_Zf*^uEw(YCDTib|pAMYtGW
zeOJhSBeQqJWoQ$X%h}&^bQy1}U%HAs(_LIO-a`!x!_AMwbh%z#x6Jyw?y`O0BvCGz
zo&(#pQ8+}nQi)0b-u8iGAL7YL<Szi{#fAd-xp+WouK>6$_+O51`PyFj1WpU+3RFe~
zugalC$Gpc?&25ibgurtui?gcq{u02$vc!QBRa779_BLf&NT7*U`MLsSH>4G-n@|F=
z1dp$O%u;*Epux8N-p4kCE)sG_R6<?H&HTEz6oMNHNSiKI=5-mTcQGG%7{8U*yeh5I
z;`4Qi<N-B#QB6UoUp@fdkAsO*O0?Rc!699AAW%K%dhz*z8}qh>@^#x%CZMe%KEdDw
zLaF#|kXWv+xEzXRQ=rXj2{}0%+c3oNHynPr9M|=`bc$DekH1SH_xazeqcxZS%GK|o
zj%vlz8Rn}%WkD{mR4)UA@>eS6_;fwnnW$2A;xZ6{LFa^*_6lM_@+&oy4*LoC_l9eC
zIb>u99a^61tlWNTT(i9|-znA|mA-j!x@G6CyLp&v=ifhVzR=DZ(T|#@UL&kjUE~#9
zA!2+6NnA+7vqO8HI#=M@6H&c!JBhsfLmfI^S3)N0!H9z(!gvAw&q1p32@2BVU?I_k
zLRYmoo>~I;5TCK^G=vSTu!OFrcP9>z^cVr6zQe8}jQ8dRj46V_Mr}AR4UwRiF#<8s
zgQ1gn1c?E+SX=7iyPJp|tEltK0o1=Wk?G%S!6j&HTPha(<JNlT>jNKua8O6MXeCRB
zdPt*AjHG5PpZ@c5IH^T#s?;E8M4fQk_^=)sfF3<QsVJ)KFKNNhy8Z?_#b?xKA?kW7
zrPVG{qkurqdglBJt+-FtEX}JyH~DVtBDyKf#bMPp{}<s>Q|jza>LKc3hvO-xrOPrD
zS8!1x;P1q*Q?DzIE_5H$we<t9BEizK!C+m977{!(5nyN>SJ&XOZRqGbm0Li*-a7dj
zEhbwnoahM$*h_~7)i@CVv0%ae<EovNnKQ5iov#<AOq=)8aJ*VB>Amb(OR2FrLIH+V
zuNVI3f!8SVnXcf}XH@Ec_@19`7;T9({_&H4Ls9l};gH12M~fHbt|uX;tHUqLrS9tM
zR=17Z+y0K&e77~<AZx+S$v94YLBkD}*`-c((EJ%zBb=1f-q8A*P8WU(p(EAC_rY)~
znM{$tt%t$dCvY*fFC`}){5rR}ZEWL}MyD8FI<fNYZ$dy+7BAJm#AFvQ*>`lv^*Eji
z4f?@sQ{fDs<NKuX-xm!v7oeH5?QYHOt!nb|pFGhp=&^r0JGg&Y<p3`U;*YRh!G*^<
z@2_il7OFdzHC{F-oe1tYyS(=z;he5GUBru_rcYceOVZF86z@ph20-i;|D`Z33xiV$
z>)UUuUfgpN9qa}yjN$C={6>rLRSWXF`=lSDpZ8djknn@H^w8$zGu>a?22%oelN*e0
zoT@#DP6rJ@tXIICkOah$081}w&~kCx(^vyULu=i#iyrrV-iqoWg0y!{jyt?;u3Hq3
zT(%ULZ<+%?_`eh%h}-W1LudkISwwCx#C}g#rTFqf-4<rs-=`gS!j}fQw*B)aAl_`d
zk^3(xxhd=oAO84ThKcb1-EN>9PHgcvWp-!(&n^DP7X27vfc@q=Av$by7ib4buAMLF
z5WzF}Zthr+NC?0|3i;c35@RQShDXZ+z6-?<io-<#+~3yKnJC`;A2t<Ox~j7Bzm;PF
zxifmXn0iJqao5-fl9GTB&}HHP0oW=T6i8C$&m^GhibYKKl-+3hW@z|QJ0fqT+5kg8
zmCzOG?B$D?Ke#~74D4_$D&)KdnFwnY>KlMWg#~Na570LzrxF2OhgTsd`p5x(L=$K<
zm<IB631Vw->*?MKuvPv^Sb0FA4%ff~JrMMG!9~!Rfp4(@PSVB3iyARP7V>sIduS^c
zQrtgX)Sf08<L&K&Wk7E?&)W4!feYFx@kN!CMotJ--z)}orc3B;1BC!F<a_~4U`%oz
zD(E_ua<X^aI&4)fY}7}m4{vW*1qXV&R%rKE5mL~`l1a<p?a#2F#{)@5>VMTm*kF)(
z{#Sc%71c-6><bU@$K8SicY+5exCfU63-0dj8WMsAceg;0;KAM90t5-}?s|qid%t_{
z@2v0cT%5Je%bE*dX1c4ZyQ-_I>sKSp`Z^Ye0=yscDmDs>2+H+Ed!@0$-_db_qcfNV
zjxx<bC;^pLL<K*h2uPmjI~Zh<XZ8eOfb2n#H2o?H-p^AqIaLm%1i8zTF!cF(bR9I{
z==KR$9<U?8@RWE7W&$laNS^d3OMUXc?7;xp1Jm9or}7EBkMm7VP{tnQE~NJ>@86`y
zU*PCMbzqwOa-JvGoU`MU3IdXcsEo=8`S;}id-DG;k$f@(i2vJE)laWTsdo_heY)9h
z321zMS$8r+(4*%e83U;rqrJ8KmAgUtyyE^ge9OV&|G1rGG0O-E5zm4$@z>o`r~3Dn
z`2T`=pm6~fM&i10Ste=VS&o>|wSCT*6kf%8b{cX;{=fc{|I_ZDFX%JUvyLGfpvN#k
z$@$^!+-CD{PMoa_#aZUqI@8D)Z9E;Cd_vdAvv0Xa)zjs2@u&=mefhIX6jy4cUguEK
z<FpF#SEM%WU;D*Oo`n1wEq7;DP1g6%SX>X#jikW1)w0w7W_zx*db%6UOD7*&6F)Hd
zW#8pZ2U8yshB=LfHL&z;tBv+OMGA@Z*xWOWlti{nnqJybo=IJN{jIJ`yQp__ybTkb
zqnP&U=m|U#rr@;WAFfIm6U<7N4#s&-3jh7rwA6@(=n_qd_})0w;nZKTH5`_$ikJ>J
zVMPNt9~T5&qHwdq15{{|RsWuNJ}o6J`_3~MExe0ue$yK7`2BWZ8up9vwT?_Yl+s72
zE)Q~8e;yI)Ig8@cN3pZk>yUQLThKF!9eO#g<5vh^w6D%fL!eed?OGGc1+^H24sR9E
z_Lo_<^jvGSy)RH5H$573HZ~&HsmH$Gr=Q+<F`T0~ZkUCd%muZH>fhm|u5UF9^?fch
zPDDP_&6N*P960<=*7qZjXoEoPEwagXb{*AgUkhH}N+&lJA256W-861vMe%DO>>;?F
zC>y{)=@lQOe!uK@4L{v{VGUv+h4<^2<)HMh+&&mP)=8c|rk>UC7U~QbMe<4_CAw=3
zaO{!uI;Pu^4xC?(Fr{uZ4!>Lv##T(QI`U?nykQHvMlh}cjsrZS#uR<-Zs61d6a+7v
z>6Ke^e?Ps0;XU;0ZHx@9Tsso-O&lgIeR0jcbT3RVLZC`-quZ*&Rj!BYk+(50zY4Y*
z0xb-A`;&_YCYo9MBt7jLOxAUV;A0ht92|X>JR&!Jl?g5EZD*hX2KXLfECpT(9o^{S
zod?G%BrqW1s1YuxZhYY6S-!d0@_a&-AW105-Riw!b;#hXR@T(K+huIndHgzI{ws~A
zeAObEIyxS@&1wz4uX>4zB5-|&cQl^C_NUmR5_`Z0?&fm&D5Xcs#omEMlW!)M%Fqpu
zd#FbPRr$8#9v3=4G)Ee)PlZqAr{sESzth-_l1M?31>we3Lj!p&dhv4#my@G$$fui%
zXD4;m?uw7MXKuXQin_cH`%V`(tq<q9hxloDMp{QXC1q}hm-mmuTcm?qM`h8=eshI|
zuxBsmsKyAp)?kjej#>w#26f|)>U`)6)8l1ZyEJ(v9%e~fcYYi$wY>LTYL-<HJV5DB
zzEiO|j629v<<C4?hJiOeyDNdrvvc@L*OQE**t{f4FK?_$8X05{dU6$Ye7aq8s}b9f
zuzdA+`qFN86G!s^#(t&Ecdp*Ccz^z<=HvB++50DTy38P}jV9UhQ$}XOvDT-1N=FnJ
zF!NbJ?a1qCSlu{r;yeJ6LSoT<G&kmK9~A{hNVifWm6WKYY}jRdBWyhAv=#fuoG;S7
z*MHUmC~T+&zwAf1(0VU!#o=muqphHPblH%ooxNN#IV#xe($Z}7wCsnt)O|F?aglLU
zD--F&5_dLL(=hGjz1lx_pej(;dh4EEM(S*<{@XiL+tsJNzFI5U<4FJ+YfGWx_I`0=
z_o`$b$^<6->N;@n3~`C6siYE%$cy{ChnSP7KxQ`IMG5kY_^Odsx*6)2hh$8k<!O`Q
zyM*{dM)ZNWPWe{JDcm1vqMj)SNLa5VJ~ZIFU_;U(d3ks(Str5i8Iibm<STt=kW5kL
z_h4co$SPxbkxo~*$-@zA6lrqFJ`RGnMM^g16H08EC_GN_`u5kT_b<ShzW0w!`JMMS
zx*~6!c3kRYE=WcPL)5jg|M=IgO>f=xD|)&1PZEbpUWPV{K#}be_FX@K$%nCO0RH69
zFvd)F5vk~%^VNBdVxwIWV)tp=+<fUd!rS|(C13g5_ik^{4CbxZz3wK4&O#JiY6J)g
z5o1}+2i@)jejL`!wJ5Y5F=tSY)|D`0i0(rS3U-RrskTBAe>Ji<H|xuhCbQS)-si%F
zjW|n=oUQMeh3?XeZLv9*)*un3ND@_Br@F>Hu6&18F8p)*Vh!)@AwyZjd=v}(<r5{0
zl@N`-4J|)uHMae3|0`O3lyk|t($WXMeJ`VzK|!bA?<^Md?=%MC%P{1&QaQE{?53P|
z)jY~XEN*r}(@pDMP~9;K<|&8ozk6w?KuWFZ$%QM|J&y=;E!nwg+1|TkSL<f;AeuFN
zZN0c)s=kgrIp~(~jWKHWV1-wy+evV__GjDSTr$}@Zu!JJlG2GTRSbU&zM_$}9~Kuc
zl@mp+Zp{z4PR+k_Dc>EifNSKyd!jKy<lN~ZbFAa*nU?bm?gCnLlZ?F+4NAv}7k7%j
z)BUft$k@<toQmwH>XHzDy}04yF6N7k^bq#pgB(G-pXI@EmcP5t>abmMMic)ggG%E=
zx5bgj8;=KzXkA+g*)*6v+{F_d{5F%x3I+n&&*;t4=(p(F1(t0jjF1lBv^+AtUSQp7
zFoO;1VX$kEmEqob(SVBsBra?8snZP?Q6LPUCKY&J{COkOeiCw@oi;<>CPvq52kAI$
z#@&_PQ0S!G=-=;AlyNu@Q9(F(--6qEu)tup&Zk(j<`JLe;5KB&5yo|1V!k)7XDDfU
z=EVRTd8yQ+?dImddszR}%}A1gJMfy0M?c8#WSZNOS|UnD0{RLHsC)5^)ZY;8;awCg
zsVpO`<`8rV%+7nr!}Wu!k0Kpic2O6*-D16<;px}AZ+laB?ooRQO8Yj-TjYPziDnPS
zt_u#^EWWO#AyaIvX^y-L%Py6<^Zk|W2cv{xs#;*LWG{R@<lCd!QWxAXL3`9hfA{Xm
zv*R<Zg#=m#c354tiliBEoO-u888vlRt$eHygc@g~bWh<fi%%c!%5MSdKih*IS8SQ(
zGO)vI-1W0L<~C)%B>6r!&k038!+QL~(bL10kILO9k!R!D(6r+XYTb^F+*&jLy}i*x
z5FwfM!#a!&cVYr?JX2a3LrTa2GrM$><r(8T5IXom67Hfh2l&%F|NWj=s{{qr^Ks#5
zTIljZR&OQ(F4*7fJ?{yMNd86n=p>8)2jEfG@Rl25Zw(E@;eP$cCHXGHO4u41z?Tc8
z_NBrXHiFY14&3=E(s~&QpVwV4QMRhEH?6HpJgaxqo3$^Kud{SAZ6+huLYHGPiUb7^
zQ{s0zV8fW3*JRr)R>4FJf4EKlQY_*S!UdBvz@KZ!ut-*_cl)*F#eB!-^u4{gAl1<!
z`a(K53=Gz>eU7{IA}j7PU!7YuAEi-@wjGRg3XU`Vs1ZQ5Pk-OU@Av9s(@$(}Q)F|V
zr`reaMP-Uo8opt;(c!Nj!r~);(h@HAxNXc_iQllw!pQS1NetDK8Frk=to?TSBiWx{
zz3wWS5wsVN)8vK^F=F-n_#>j{?lZwlDrbs10qv0%IXkn>2RD<1NMiHSH;E1MtzO$?
z%nh5SoMK<~(7W0|BkK`|1smP@tIoiSuXciW(#WE)hhEy#CTIeUCVZpS1*hy3KpPb-
zdtmRwk)LIoW2&ben*8V*cZI{WiU8%M_*$7vy;UX}(s<#=aR>24vfG827XYi0rN%bD
z%~yvz%8mRH|Kf32nYEX4nAyW2RTb>%KJDEy795hJBYaU2*1B&k<_dY-8e7mcYgC8%
zyhc|{-eE%CY>T+aB0%KIgWI{{l#iv{3(gSARVSM5KB)v|HPv<u8V3@Lr}*d|;}}Qe
zpt6@=zzrYcERECFq@Nd<WrjjgRG(lt?mO3DjrhJ3c{4^r8Sbp4&}ME&u0b~al3xtz
z1xrcY<QXkx7#TtZhB4;w<`R;NiF>T$PnI7?UtE#HsJs%eZL}JtCTlh5bZe5;Jm+p;
zYa$$GC*B@L_hY1yl#Qz}XpGhIV&8s9(&F{=!tmEp(D=HEVj&3us(X#*-(B1v=zE`=
zAJnR>_Y<~27oYJ4UW0I(4AAPR&S=hPQFkJ(w2iXeXT`VFxUO)*gTNplm};R2r6}+D
zomv}5#~{L{%%A{6?DImtxihDRWGJbMMoUmk)<*+Zh}99g&;T6f0coR%RlXhF?WZYq
z)SMfGtn5AA6K2l(xmRxSW#0;kgWM~l;rZU8ETyY1bM;DD)+GEG_LN+l0Ox1r+w>hQ
z{keO7A8%)B+^i_zyKN+R&-sJ&<=CN-a5&o9qs&FVO6V2^>TIHn6|4<owU`JpFt#Fa
z*TpoCi8Q+4aoDJgidbzlXev<7>4Hz>l|R#cX2C=V*TZ@vvZh&J;ih?WYAoOs6j{@K
z77A9vZ??GVbMefIf-(;xuexY^=^++7)x5emPlX_=6uQHZV~a>$D)w1y^8EeJ@VJ}}
zO9^5rm?#At)M9p~^w)C8bnbPH!*{|})uT7t2kM+_3#8xmk7gFev|2c-zc8Ea;U!2~
z$eUeSF`Y?W&GhQaT3SWEK6`9Liisq~C}7<{ccQ+pombF&fw>ujO3{*q%~V!>GXj-J
zP?{}ZE^vZMj0rcJW!qa_Ou?X+Le<NF{awLn#skgJ-qsTqK*+qX8_b8lK^L6V7?wj3
z=ZXnlzFT04jP2d6{xbOcm1DLMC72te99&c`1%jU!SgX4$Iu#WIQQ6?I%=DtA5aYNl
zoaDq40<fTL@Nh^4HvNb$UJ5^!D-vhJN?ZGt`eq_XAz~{aJktCT2)X&BK2Suly|(e?
ze1)`wS&sh5GbijZ!f1w|6*Z}fljKol1zAhsXi|m2Nna8X7B)^&sUXWoED)|{LH0=M
zmi-L;?1}Ib)bKHgqRuABW(p=CJr#a6mlb>aRJz3FcYG(lfEWUq=b2)H3*8nPa^c~Y
zhSFD={rJOp0%P$qLZh+wg1^AJQjq-{!fR;YcEa!RQMC?n+YG~DTa0rLZG%dsS!6PQ
zMNCoB7M&ae#=fZp;}Z%V0&*?Z3EBpLZZ?-S`I|i5<S<%+??ck-TwMdnhY&}}9&3g;
z(sa{um*yypuRU(1-^+vg>EkTjUnK3ecOfY^@(pR^FIQ=pD1dOpWzJg+s(S_fRb*;L
zQzll*5QmpI&dqp!o4iIOB}5|oAjnx!3kry^X%6J&yfQQUK0X;BIcW1L_o8d|B95SX
z3cn~Pfb_UvL9Qc>6fAf0*~N0y^!CS6xzFkk+E5EDxws~X|MXd5|EB_6=d}sD9Xw@K
z=ROP=)g(9rnQu=o^^$W;cFal))fv9u7tR+n_xE`yU;TjfN3yw<WiwGQq*c}3?<_?O
zrj6~XDKEwo7o$A?J#bJ)8b@FZ#{?vy5pVRFSHGYH$f5$#<EK*?tW&pKED6<H_}@y_
zxDytBntfBLWPxAE5c)drkCmI0{FjbAOW|Rub2c7YQU6ce<cu5p0S7LWcfZ390yX{J
zu}#r1E@!O%yg{i=ebFo~KX_<A%x}E;5LIWbyEPZVUF~ziOalL>ZWFa|cMb|c+0=CG
zH%t|hK9p09Bhy`<*fjcdEU&E~h-lE)pSo`uyx3sn4#G&rr3aVn5pQHQm~+iDG#>r&
z`;DMZE0lJSeR+~ikiB`Kn{GONCm}JRdH#F9oeQYGa0-MlY?N<oZ^oZ4%5&tXv-0)o
zwj~zJQFI@_Kky_keVl`Qoh(F<fzV+`H_fwvI6od!>M@#^t&sc8u^G|EqngJ^f%>i?
zw0WJ_ieEHv(eqREZ3F<__mxgqzZSGf^x@zXuY!wY0*Po$z#cuRw^o^*Lh=MU|HV93
zlAK8L<-1<;3<<UY1tNYdg*J^LC-JJVpNvWLlEle#h*N_pA?(!>a5yQhqP&fQni@}+
zX#*ldwijiNt!fn=4TS7fl`5XF{%>^%x4VhSv-G`9QfWngX$$y$%YY4i-xH$%9<0A*
z0J0nj7aamt;ME(Kjp@%dB-(SxSa$-7VfSk=M@eg&S<v@hvAnu-<FVm`uXm;HWdOq8
zPfFRf@(31e&a)tev3(9GhO>U3j`-DllEUNMcFTX>!`8H=lL|TlN!L@Z)4&ps#fKpu
zu{M?o;CNXjocj2y*MWGyGG-|J3-8iKzvzIjv&$4PPt<9qCIR)jGB7&=|EFi7&tQFB
z1OOa#3vb*3Lhw{WB*JU%?It%6)iIi6&yX9nQP`J5hkdjh$iqj?a(+^sw&m;rKa0Eu
zHRpw3v}z1TIbs5%;=UArjx_$prC;~ExkJYjj{`5MJ0M}mndq|Nl54l+@nCCPhd4d~
zHbvZwaOV20O(Pm=jm>OY^3}3O)VVjesb9V6dZSRmj=SyY$3^?p(9^wg)`h@={$$pT
zsN$y%>Wr^iJ~Vvjdq*Hh$UcVcAn+2qMQW_09fG+WB=ff>jnD+Ne{w(cBx33#VmK2-
z3gP3nY?ev+l9TpOFDUrY$`TkHxh{4BvR2Z~rq&TZkYT&S^_$um4Fxrb;Pq|YrTv%6
zL#(#s@|l2NH1#xA?4N>mDCBStjOYJoJebTnx^2$|keLDc3PGCcN#b0Dg0svDDGSNU
z5D*E-4*0~u&DSpXMl>ccls-UVELHs(!xI>E%!36wNqvcQrsUoXkSL}IWTg+ioD&;W
za;JyShJ^e;WxwuRx*<1B*j)U~jk4YMLb3l9=a;BtK05vEJoz{ra7&+p7H-`|r<d<5
zi76J#B@lsu6sqUuhfr>ZCDK@iN_P22#;=07DN|%24wer_>j$*Jx%aoCjGWGuDXWc+
z=VT^5rS(bM-Gu?u$1!Kz7Bd<HPmEeSR)QOqxyMfa_Gd(zKHpEf8bdw?Fz4f8g&!qC
zf^ne}EP%$B@KY1x0&(QmcRlyTn<NJxZeqnIg9lF`r`Fn?$PHwu2+wINwB(oXErm2l
z+<zLp<7FlDKYI}j70b%0sge9)QTd0XT%o<WrYWj5Epl7~S`wuQj8lJ2U0?d{&xN?2
zq2jqN-s{Dct03pWAVq1eC}|OQ#g%t(s+>Fpsd3=aSdWQO97+qo4t~jF)|)z{kGv(u
zu)D8ONvg$YUd5{4(bmskA`bYga>IotnXn=vA(;;zGG54$!Bm4@*%_m5Cb(fHO*B_B
zsvy^junXJ<={1GLmHC)gRjgq9eput1nu?5IZSdcbD2>yFYNJCjM71`e53JCqhSplC
zv&GSNP2O*JUFf%yNT!_If~`b7laGTp*?Ac9nRmve;b+~bjMbrGX<w{pBg3jHq{$vG
zbr5s4A$^Q{O)AmkV>S>*CoKPEb9@w9b#~r|me+3QZjoPB5=TT5@V}wBmrO&f>Plj|
z#FmT91Aq;KVdz2&`Q}8{UB=djMSkR4lB7XCY;XFzSso5-LF4I2>2=914{-IfT0sUO
zR0wJK45Dn|$;B057}8w@888Jbe2dKS{qAn0b}mov=IJ+^WXs}~z6eH0kBrcZn}vYt
zgo7^66#rdlq$d23AWv*2h`mJO8SRNpiPG7or|NS0enu3$0AZm*AgxdhQG3aCTtOW2
zkq=CWaBYg*SU2BgF~`oD=cYF^p<c=3^w~Bx_Tz?bTf5-v!;bj*E0n+PgMfQH31eqK
z$)o<!CzkhTW|OKJx-{H<IFl=baZWcWu<SWYnAhZO>)ksq_3<EFKL+EN&b(wdExq)e
zP*6X_#;{0<)~7tw&IQlm?O|8pCe^UgvmdqgQ&MK8XVkx6;1Q$yXisDp_!_9vmc}RJ
zlgF=E9wKZHMlz++1rf=>smO<u%l)mI<5g`X1Y#vH7)>#`se2`Fr3Xfm^^%`ABTc=b
zN6};Ax}78EH%A6!LCHh2g=p$w=~vYg9XH;f5emw4jnBH)K_<{U$LzEe@oM)Pt!90g
zAlIcavfY)Dw6N&Q>V4oGym@_$kXm=Q+cSh!-_$-08i2-c(?df{aYyk^r)Piwxx$XG
z1k<iAZ!-ualt3s|a*R9<$VoviHGEyUrb~h|Rl1HW3@4h#!w=t9C}xYuD0S(Zp@Ez;
z16L>+@>tfvDEkJbK7Ci%s$+vZu<%XZ3n1JSU3C}#H+SQficQq=-n9JuCf<mo{cXq&
ztQ^TopGohji<)B-*cC<7VVU8C_kPKUwYM2CR0>OYbo%|ap{&NO4G|8_JXzj8NmQK3
zecnhZygkoBOLB=O&oUZx9Ym+g1>UCCxbq6Hwt&$pXL_$jM`5klf_}UK42FDlK)7h~
zYT2E7vlON}42+%2FQXMeSm}>t&9yQ4uwY!w=c2<Ezbw1p4+-y~sQQ~&j(t9|w+)f(
z_f1@(`zV5dWmwQDM58uPCQ2bH&HkTLYf3YUaCx$Mk`cDeQE&U!=hnZjR-89DmH#6L
zV=n9;4MNSdL}jS9-!Pd7SpJ?qM)L^YO}bjnui00;&5&}}>=W7_eL*$*hGM#{uGaJ8
zK+WLRZ!WD5`@Q>EY50|bka=mI&`xw3b0{+QFtQKsp|7FBpirzUUhVFwe%#X(A7UjB
zRFCc#uNKX~FSQD_$p0>Rt$?EYM;0>=@F%ePjooj-RqXanNRu4EKj5Hv>EdJ2-0)sI
z6I>|Nsepd!Q-?aVAT|h5DHVu4au?`)A>01Z6>E)`>Rb^&%`5*L37|ngpRPb^mkT2P
z@v`a=f3ki29xRLX&y3oz?001xa>%{0CaB6($VWy}`xv~)SMU`B$#Gw*b!Suq_a$hv
z-XE`$?TTbdWH$UbONH|9q#@$Y_6a>$h>`xZRAFzeXY;ZomyCBv;D;#C<|K05F(ew`
zMH<^fsp~3WsfGyjjKdqS+`W1uv=+Di%6~Kxq^UsHh&N?{5H%Q@Q^8(PVk{J6wan$D
z<U*Kaq}X01g1dbN6I90vsD!4{yT8-K6qk)s79#iK)E>bA+T_5(5f6nEnJ9d9%GksL
zCi8t+)dPzmqB61UQeJ1~l6tZ3P>4?;1|Y|a*_GA&gYzT0s#eR}%@mDHaHFmq2z&t5
zvVQ|<b*=OX!t8C=(_<TdzF$P>`4RAFXp(Xjo+82;w3G$wKlCcl+7JKEqas36+T1v(
z+D^Aru9SvBm4T{U+N5WeS{e$#j9}dhg|K5Lec@46LjZabF$Ag2&8(%Aw;s}$*>e$O
zxS)VX`)%5ow1+RTU(Ome?PwE@KI&XmdQU~0fv93tlB8sx>U&P&Mf3y=Y5OwL<INY%
zK2%R2*n+kO(3iUM4<g=gl*2(^6NkRDK<UfA=|%C^3sVPJVbs@QQK2|sU0<{0tK*P;
zupBWDCcM)<v3`k)N65j`%_kO+0;Lnp4$$FMmyNY}T4ZH2iB`j!=LB&<y;e?A`WW8h
z0GcKGQYa^<i>xmGJ3VZq_k^4<L5>}lLJ2UqX)bI*Ccz)l?67jo^=rFJs_Jsb$Sff_
zOurD7sqN8Bf22g{V$_`(n=bj?J?<<@IwSCOkaxdkI2*wBwk)2*6K4ycK_sMpWANUg
zl;Vpq3yyhakj+9V(~^Cx??u@0$E5F|hceFm0aT4Xep{DdSjFBiSGiYI`{14XFZWmn
z^H5FvI7)@Beb5jH?;rzX87Nv9o7btMf-^_#I7IHf+3$pTBuXRxct^T>bn%TsoZAGl
z+UyHTF@XJwr2;*-C2mj)e$WhZF{}U;UjLL8xbyYX0K&255BKjHk=QlO&(`E=wBp&*
zY8xJjaRtmZ)*Q7wxOX2i1%|mC$_T8w_aN4%SGOE4p+>qf_f2Jpj(Y!Wb^dl7Zsu*{
z?WrO);~H%l$hBd*L9}KYU|E1R5#)KBBz5g?X-pp?^-6~s##l#HmJUq+Ft9)YFwo*Q
zA}HqOtf_Rv*Y5{<w76Gy;bE%^y0J4=UDw&YhhL|eT-~8H9=Bh}?5OT5Wzaq4*`i)l
zv!Qv0kjOJ2DKXOOuZM<2>SKR~Eb#G+8_>Xc7|uX-V5p2igDI_J2ISTB9bYNJe+|oc
zL3)ZP#qC9PxZkmLrhhTofi4yTx9%+Eqj$Gz#8>!V{0$<2E&@qWcP+-<z^e+4=>nB(
z7r*A^dl3!!;0Xrsyi6D90f9p2@EcukLLke~=2vo}h3H#ir7u3ShQJ>B@K2mj_VAnf
zjzzCFVD|SB5cHuXBE?euG9>)YSG`#=44bKf`X=?m?&wg{lQKd68FDZ&PnyxR;eG82
z#6z#!(-V`uY0y1$GwLuu6Q+RQHD$W>d(&$f3%BVzb+KgqWCC;qwTC!3d@W;sD>D<-
zgAERwQ%kM$=peHFpzMpl8<?eHz4toF(#o6Ka_pV9cq3mll;@K4vr&bMBukgV&1CGh
zEKa$f1^yYt36|7K`qOn6xn7s`(ihLZgA4i$JgGH@#SFF!-r{T(SPz0igD50n7jVMp
z!%^tQSVt$an@D0Br&(re1Slde?UX8Lc%v8O>5jr<hnZgK6y#FF^1=X*s7E|{cVq5|
zBuN5dPf>L8l^p#98-_UxSx^3LFq0VNkXEMPeGYo|uq-kN0I~5W!1}5#t)@DXCCHW_
zLdAEDOHpddNS$!zOXFP1K9IpXnq&~b)REjHLo>xQ)P`yB>U*#teyhXz=Mm%3HLf-3
zEndwV0WcW70}JG~v*8*UmM(nhT*1$I%EGS2HiavAz)K~*W9yERglCa*HBmFh$|=^b
zFev=2vpYD^-+9iAl2SoP&+&TuH>L^X3zi2WP!FV_Ceazpc~F_JI|ymRd~l!|(w^EE
z5V146MCVA<Fh_EAk-k>W776-{12>3&t=|^9UOhV+4(<uzsyMYLY4t3@q$u7fg;}t`
zZ{(n<GRqBhXDsO=N|gP};v>^(kznJl-FVlIu-L}dMXWyoA|EcFC?M4t=N}6n>KjN@
zbBffb$oWi$_yS0)_oY9{Oga8G6wv3K%@VPEx<=KqZxiJBw59Fc_lN<F0(T>=AM$a`
zRVgqNsO~G$qaq5jVIGnlY7BYs9kDjm4?kP64tIm9V#lZZv*eD(v9Iuloc&1XFn%18
zvF)6dFD7c_dpn14*C=+Es-LND=D+OXhbsXy9u>_x>}465HjsR3<U`C?7FnfMNZ^Uf
z;F6Oo+3^N8o8TgNIgULz`{6KGzP;g-(8IwdYK$93*q6gIp)i7tx65=9LeCzc1d<rg
zW^HeFu^_epJ%j%0J5w6<2{TbiaJ9fU0sq*0(G85qr1khWsIyV}Qm`T){JEeITLai+
zcvwdQmCl_#9li^0Ui4<TbhRNPi{}ts6B~R-E%b3V#Sy7BgxwHb>V4X(cb{o!xR5zY
zdjDE9Qr%;+5gl2+nakF{We6?*OD%jAl}0X1KSOvKriT6Re-uVq6l_7o3jix^<IkCY
zurQ%d3GQsGfH%4p&}K)7*7rpBha~8Od0jcZbEd7{iLH0Z?QMhDo4q3bB!O<mS1MrV
zV1)|vD$G2nFw~?M6nSj7tc-;4P-w~{;S{2HJh6UR>ur{&N^Na!1tb3SF@jhw9CqU9
zi==!H%B)N7YRLx%_W}0v8jHV5$sw?G6kdNc?@f)Bov)jk#u_pnzq`C|*OUA8;eInK
zS|rmH3-UhLOR@!b0W};cJmB^tMwF&0WCV;6Y++)S%Fi7Wfd1$?wExXGKxWv8rjnvS
zRZ6F>_`SmdYxQXj-hj5MD7JajYQgFcYLmFr)I4-Wx42PcFcd!Aq5Z&amKstdHHg<v
zB5o_difBk@9SdaI6~Ntv%=?Sgw^c7BT8>4vV_?CtN(`vYR;DVyC>;tl`~K^241nKS
znXCAOl`(5Y415^-(doacMa7X2JuzF9jA~8!C-kwxvTyV~j1G?;Sa8k%j%*bP&N@}C
zwQMlTM>?hQM`5cvo2%t01`A0GwJv=EOtBBMT7dYPiIi8nGylFi+qQj*-XAUd_HbNL
zf}D)~V5qglZ;>)nd-xV)2W$BIYXZTOEQG3WM9hWVYrO_%KU*RkI4PBH8z&Z-{Enz%
zixg{gZ`O)NDIparvM(5Fc^bK&I3pHaT(^xdZhp<e-a%9r`ki?f=O{rvj}%3%!hY<h
zM3qaZ$su@-QabTvXS*7Uj}Jk}{0&)G3Pck8qC+?mExhEX6zI%Moq<G{WqHvL0h(-#
zG9Nt$<Uo7(n86M_j~)oRg68^_G|Yww&FQ;_Q7zxGQr}sP>lvbczqBF*Fulg&_-$5n
z!NIB#NM;vDi9eJWm@Kw<Nt0TnDDD}k8Hw4%*4OdoQvkv(6hdfi0Y0ADJIvrqX!3G4
z>=z8=DAb1Cod$xRAgR_U8LXe6zwqDKsbmK!j+5uJNpqzgmvY9fBC#LRm`DPE`cSE8
zv}lJ9#t-}fbTI5|g})ERDOYq~P8sSa8FXhYQPB}SzV9elJS6*?*j%p<S^<5_;U<P5
zP!Y697^iH4D8Jxx+2fTEy76}Vtr&yOUdQg^`oy4{{(5JGhkN^L@bgP=w!1K{3ox}~
zhJnP0+9<cWlLzUM#ussbQ@s0it6lZRcJJ{LkD2hwyGS++9<pq^?jx8m<?~z`gZqtl
z*C|GtH~%bLVKp^revh?{hr>a6PwKaFss-wMfj!6*tZ>-w9?fu{PU0w+oU%j~3leaI
z<R$Fdz8;zn$#<Y1X>e+UNOEomD<DT)R`!w<=AUQTyn7fxHoy>a<iOfqnXJ+((`)hV
zwQpTqwwV?s=X-P~?pG2@>IvzAi$R!B#dA-zXX2xu`aWKC9Y0AFM1Z^`)&yhr4T|2B
z-1^`%@90hP+|_cjo@Yz0(W6K_LAKv`bnlz!k6G1L3wWoqcC7=NmNY<$$DJF7A_3TL
zNZZk6=^1lal2`Z77e`AYOqqjrcbB`GtaJ<$UZWZwq2mr+uaGZ8L1|mU&tmp8h}JqX
zc8_C4o#CBcImtM!xnQJ~tCI`NeNf8(Bz$Ka%z?5(TKZcW_4-W_ZcF9_bZsW!-^6-N
z`c1BdwJK~m<n36v=o*4eGgzAmGnfD!CF>Y%HC?m=<<b=NwH2n}R(Ow5G>Cs^a5^su
z2`n4z2277+&v7CKO~01++IC^`RQC{O8JJ#a&~%iooY=$(P|0)KxI1?%CjhW}a7C+|
z(fs=Xn~AZHCet}zODHaamOX0@jn2cG@p;Ui1-i2f-ZTVErx>^SL61GJL02~Ls_?|S
zZqdk9N{~GVp@UEW`^~JnP0JM@K$)sOv2lzHV+p!AN(%Eb#JOYvPw&PM3%;!c3yy-3
z4fiWHr=Vn>@?c?HjouyvcOH0aHd*+5IsP?z^8@mtxdC<roQ)5c(?=mbdgl%Zh!2&H
zkK7C&)_CUO&yzKdCw$!wtg1ZU{H-ghHv8)AZ^1@Js=uf>9s{sf?WB$1OX8sSC~y^y
z+=35S5HXaZQI!sYVAv1BL2v)OoeyU{fDK}`LZU`CCDA{{k2)sJqv*?skH8h^jS|!9
zWp%1t8&q_`;V&FR4S+*Je>3E)b<^#o%~1aCLBIP<gX`vGGk)oe>{zTA#sG38RrWiv
zJD;q-h=Q(p<BN+RBuU^+twj|P-5cw6?io&k#i(VX47Vt(;B4=}A50lnKnEiZZ~bB1
zq@#H^UMcoO>ha6&vkv1uqW40+t!@Z?Nj7aag+;8h6_UqqgqltI`)>YX!d|PWHhp}~
z2*nEii&e~}4Fp!yTZ{$adVP|zbn<3CiMWfG2hYtA&Kkik`cXA2TQ{Aq34*EKrL=#c
zjAVOz@os-J+ZieUGK=*&)z0`3e<-d3gLLT3^3qqNT+(g}VPQ)XX8v1zVebwBinyeR
zxe^W7NY{TPrdBQ^CPY4y;r+Mf3+|`qva>_PVDVj1J=^y*lF}7tp#*3-8tqFjxkhOE
zf`~b3o!mC2!HjUtfADw0|J$7XCF2o)f4uO*mYHF*31g%0QyQi=xn6sYN}jeny+)Ju
zrd<ya*k>fdKfqF{JWCJk=Oq3<!haw`qR&Wm`(6wS^P>^FcQ}kSXrCM{#f|GGNBL-(
zx29HP+Mko|1HoL{=YoizQjAFcCb47vt)L~9|8F$O{~wx!-v9z|qaBiikO3$xX)X{6
z@%4L;hVEYgff85?0nXj;`Od#CX$R;uDCrZV&9&z={eNBl^8V2N@<9vFT>sY<z4{Cd
zaECoR0q5qwt~MHBsJ4Q3oBwL`xod}K5Jpqrjr)JS4>qKaodxJmu5kUgt^mY5o@c~^
zZo4=C|7fBDQ3<H9uuuJOUHL;)(w^Z3<@$mo|D%atAZ5Qe>cLV6|DskHnqI*lS~yId
zl{D&qG+_-=W=;6}xklN)t^gcb5H-=wkQDW|xYxg0;Q}e+Lbd$|pz<H=5e@?f5xt5#
z)cY4Z{jXM10azdvuI}xB=M7mmIEa6<_uuUOM_m5BdjHW&|5gsq>*4>LayZBhYr9(Z
zV_|8z&PnswOjgMeN9<2vT2Q{Y_tyAN1r?AR3+m3?s->H$FyZz<Sd)YKniPppG~ztj
zMCJhd;`b#9)(f@r%&qs11h0*y$vj?zTXsNhR9k_3YMxe&rQ=CAp6tdze7^h1suXE(
zbVj{=D%a^rFRAS1{+!9}`A&A6;o<Lw(~&nv?n`$$UzorCblLMHSyKSZ0fHU7)19Qn
zQ67V24q6nZ)A&7HB>HkWmx7<OM_K1Rs$WCXY;eraP`*AJ6@uxHO$L=Mkq<=eyB2i_
zGTsb2N#57hZOY_F+Lo0-WaGQb1IOw2>ix8t-d`V4pzEOr^2T$eyf&Th4(gDX&}p#x
z=s?9yK;66dHA6U8GNSt9V_vMn*SE1WZ;Ms(p*?7jfM=v1mHiYp&pfO@3_Mz2B=H+a
z&<U8czxk;J{WBSmrI*5Kr8eaxrVI-fYs#rOmW2axeSH9j?pHpD{x_!{0<-No|E!EK
zXkjI)WOPKLMM$J6ls<@PoM`fJnrroIW&BVpcldL!jPmw16=?chw1L$AxCi-#12nDJ
z>-~I&3F0wl$U$2??<ZW886XZXnkyhz6WAWl4Xw7CRS@%Bi{Q>PAJ4IXK|V%70-!pK
z)ezr;o0_N^GIIRTGmy{epmLuong{$<39xU8NQ1J+`g0YI1vaSc*u*@D$f(z%z2tQy
z;FDRH>8lmG7rs>xORJc$*yy&mjXGVT9;JQSji=Xle{)8;e1IAZd{-@uYz7mcdrAG`
zFexSb;7Ps7ye{`+zy}7I>V8vf3??R0NQ8AAK*FthBZ(PEea>UNuNK|DGN?1B<5Bo&
zyZ;GkF<B7-SI!jK@yrK1iw{S`soaCbg&D74k#M4=NIb-S!C7+Vg`&*lb=vGp)bpiB
zaE_)`B=I<IhtD_e|LXYLp|=}$4_pU{Cu_aPf>(!&B@5)@a$cLsX3nXE46!bIGdr#<
zW+Q1Tm4*^N2dmv*U`^(Kn0!fd8sSwbQX&VZI4sS+12JO`g_!SM>ir6tC`Y+QrD$fT
zIk(5zX2egpk#TdU=85(oc$ejXyWGW4()Od)r?OmnaCC}wlb=f06Vup%m6P3)a!V(w
zb(fv|M46q5{EZB+!_UdX6Zx_xogYzed2aE_l_srbN-^)HL>r@SuVe{uffa85A~)Ft
z#+dS!TN!o{jBv2Jl=u~I{Hyn0okj&_k7iCMWLb88a?l2lFEQn%zi}Hl{?lo)))R3{
z+$BiVdDj=01(`w*`3%AODqLkcjP@-v;<21?qJGf*a?Whe%C`AprpU^4G=sQ*blpnO
zX}u5ArED2*dYt&3FHD9<vBFN4CC&xg^shS;Cne4Tctzd~oKf#cp+|S0-`p@HRG?wz
z3C(Fa^=S?+j$W0<QBDFX?55PR!tXdclq4+8YM%3~7;XSQo)z=V(<#fSHxfQd;8J7M
zd#vo6NCofog=V9xjZZojAKJI$e0Zm=k%QzYl?0-9T>E8vm13y20_AL@27Zw*xkuR1
z_nU4uQ+>{OPr9(tFOQamHZOSWA8)qy_}}4V1*sM)2owds80DXC6Eqc+5&oo|8%#fX
z7AK6uYqmG@J!9*>wV54x&i3~1+INr*J~B-@E;~FXjo&!jnCCUF%s>En9Ge7r6#ZH|
zHa$IoifBhbTwI(qk-d1#Yoer)uE!@qaE+C@9C;mC)n`84{hqB_9{pBgrRP=N{BiW>
zQO%4SYU}%Dlw92Or(Y25WnFjqhN=78S#th){P2{QXeR2dZu^zJu8+B~LRzj)Fz0`2
zvCL~ZT`3dWh@e@Tjzx>DT|w>HzBQ8mW<i5=-p4Q@i~mc^&8b3oLEN2o)6?S}6RN>O
zKb)B+)bUoPUo9Kf`_)g%IhQ{MnXT5yD&=$ZGk{N+Ph9)5iT2Wy)QQ>?hrcRwIokMp
zGZmn*l4t@dPE~dRzNJcU@FZ{WvVJYns&iM~L$c5+1Lzy!ek>Xs5qi4klGNjN?qK=3
zopZVD_e2a!z+Ol#2uG$M^l*N;`h~?dif^~bF>~bDuJwTnUo!n8k9MV3DBph5*@#t?
z(Bm~0oJHOti6cUWsseBRU00v8$5xV2B>mNy$50Cz?D%3MIDw6B1Vy}e`&DlWoRhBR
z%rdcPythyp^8vDE%@+n<ttCgrR}Gn&Zp%JytW0}UfZ>Z~Sr*a5-%g`87NnRmH7h}*
zpx)t2!m~X^^TP`za+Jn<;SF}ZmQc&A*@dFi5J`|ZUg2Hw0Kowk^xz8gg8^J`wO0nH
zIoCO{nYgb<?HUhiPs7J=ythEb$#T`pY;JziS_*u!IK~o09n|xCcpn&P@u#e^|GF2_
z#?pE}VY>DaKxDV0MqxHgX)xjVXpdiWvlb=9#eWqFp{lsT{*+I@@s##6M-QD<UwITG
z6475HOk+1sBXXigB^)}G?{h#%X)KBNar<d$%cp#Drrg!*T-)3Is?puvgHCiI?T#!l
zp><0rVIH?d7j@bDooy;JJ`DUy4jNKaUvLs0y8W!7NzU0_&q7Ik&5bnWU%m+Da?fUN
zJ{x%>TYpUGR0CuYb=;d`Dk0F%=MAA(fi*WC?+UVJ-U3}eDiYd%v>@aOn7p7o3sllx
zHw<z;A(hSkX%$Ryz{0#x&Ld}|tR%QB3611$6KpqihS+u6mGChzY$p(znJ&W!)u`Cp
z-E~IOEZ&Tz`CP?A66R;$cM1@v^LU5Fh1#JMqL@NJO{rr9ouxbPyzFy@O2ei=&FnDz
zGM~_UE)^@fZ63S+@pf<UiN~cQS`5dy*IF821!81HFBj}uFTrKJgo7gMXZYZQr?AiT
z>pbmcc%&J>(FF|*$+baOJ;`++U<K8Na35Xrri+m~Wy@6x7(z}&vp?16@E*zs^mi2Z
zsei4bmR<Zv`BhcXub94kpbK41?=X#;5WPVscrZ($jto4~tWMn}%$+#X10<f?*-aZ)
zgjh;IsX=y99sy{J2>!Hk*6>O?41g!0i@=Rn)cL&Zb=15urzrFgS03(LP66OU4M=`<
zX*7!T0j^(S1H8A2ohnh$(|@`*SFmO&qxe;XPPIS@0|HoIC_VNXogL{%u8N<mnn?pw
z&<1ki7_3!!nzRoOSIaL~k=9^LBssb6Y~9tK-1olM$!K-(DL)i7mch3D(uLxi&<xe}
zz1c|ZD6+fPLj+d!36u*c6|Bm0nP%yMO_xu50V&xdhf5SdpL5xf+1w1pEFLgaI~B=A
z(DLOHXX?nkc?*#axa4;-(hdM3Ah?LR+Pp3Vw3n_|8efv)-HEoRLp(R))m^VTQbtrN
zODTY&74{ahOdnT{dk<)T<t&sSm+xoVYB>h!1{vI$Z-t?~$OG=5GV1Q_V}SaH%LQ4V
zQ&0&+s_1uG&%Al?kuV_m&YxPB9k1Z?5pzbDtGgx~xW3&hm&M<AOWJyT2`0gW@rak2
zm?j0_m=a%4?M=B!0aJHVWeuAt@^hL}P;DfU=z`Z-CZq5+TCe*zOiPgbB{u32!?<TY
z`a|p6gmna?U>410{D8nG&QNE7R;Xbz)E7P;F5+=%a<l}85(Ix7%3#G@-px%!f7Tc^
zLg5Yr_PJlQ+-6KzNJcxyEueU2Ksp#ris3sLH_R#U5dDdcvzQHf97>r*j8{Sx_2^iW
zVn(k?kSP;~SIng87%2e_u2_eV<h60{GkrG*P({eS<i1AJi9+D^(EJ8^1r%4t8v;uw
zP_}}1MiX8SMEVYAwdlSF_@D02oCe23_hf~kDq2j81_FfzOok*`x#~?!x@rDsQ#gYb
zjaw7`%@%L`(`#E?D2&T$+22Nc#-E)s0R1YR0>8N<%19o=<kD>^Xslnd2;QJ+oKMmw
zI+FB(3fM*4{BiO2*Yz?6aSv945BI|bQz_;On7i#~@!1R%yN#`)g>OJxoN0<7O*6ih
z|D-(k;?xiCxQMYD>bgbnLw*HR*o+MqIy!<24e&r!=^WQr-CtF_X(mSfkUzau?_t}T
zYAZ%Nm-i$<>UB2^EIlE+XhM{a4QTNfXV%(=9w_lI66h<AqvKD0SfQ(90yX);zLZ+D
zUKzkOD9N&f#rFN9Iq11^e2CKee30FrVH<Qt;AbX__VQlZ)*Be;i0X81yB{QB_IaL#
zvyW)J3J3M-544;{T}hzgo@y<H)*HdT1Uf6D*d!BHZT1$e6Eisyq#NJ7&;DbAW!V>i
zXDiJ~hL5d?&3GUlb#+|=YV64>`Oprw&3VxEHSd>6#Ky24&@{9m*eU#)vleN@5xVSL
zz^0QJfAVZ1zk38}el|N0h0KUPQY6y?JOHi2`hG|IK8ae8GpMJYgrMPx?1hUJKbGza
z8^-``VHz_YHN5NZ&3oQ9Q={ybegA0!0Vv0lmw1@mZQp&s1un=VjNq*v>L9?N-fN-o
zysJe%1YkZ%TVY^-P-!dEnE|-X=YCj5^5023(ar2EEuFhhE!5F<nSQU+VmZjKC~)yk
zoxn-5O(SSNv@#5E2dxWPlc={%Y>9(kdrE<H@6&#2BB`rhI!`_5NgAaOIRsTsdJ7|@
zKcIsz;`jYCoG0FXfbK!y$!!T1ORpLF+%AU6L<v|T+iHNx=$5kVX&jB=lq~&0nyZhz
z0D)vJ{q<rx0brWJ-om}?%^t+EeDQYVAvy8<-Irjt_OHG-3RUHU8IYl5gk;G}hj3ol
zHvHtBC3Qfq&YCv1>Vs`Ven}gTIn{UWvAN#q+M6n)Vfv84pYI`n!^Z9VS5w&$Z9^)?
zCU0l}jf^<oicJ0)R{M_CvGa=`9+R1%7`8x8tF?Ba;ZSckc`L0G;LwlRsep=b*Ix#q
z@we-s6-+S13PW)=W67TN)deevQH(tnlqf|vY4T;UvKPRtj+;*9e1b25hXQtASIxF}
z?5$j)1dSLwTFYK?Jo(+jQS+5<9Shmnm;Md%L{ni*EHM{>J03^l^ts>Jp7--s<xJY4
z*Ru&5(xb1T$tJr^mnwZ=U3@V_Jo;WboTGbm9+i~0XpQ_PZ+&R(=HoB3a^Q+9Z}Iah
zr-g!T^Gx_s{0(7BYRWACVO)w7m$W7XIb$~TBd(%@2uzpP#nzY@`oSx6H0I(XCjpKx
z!Z4{8ZDFu4I4ZV9ay#N&1=qaffaro-aRWko@ph2;la74bFYTI?P5F3!=+HtT%0&UR
z6`MV>yI*8knyXDDqa&ZN75T?sM=Q_Lia`Ufl4eTxgol(>NR?SCaZW)B4$xY1s`$7j
zg>@1s+qyutXy%$OI?>_jH>9&UwE62#NDV-SVrretX#z=!bZ;50k{I;s!=s(ubbQO#
zxc`2|8t+uXT|;;`m1CnBnWd@1excGxD$$IuJ#P*4e;E_jz8(y+v<_LNYP(jS^ajMe
zaXZ}Cpv2#1ogK>bz1@*NatgX5K=+;tY7fvH+Mlu!zSWcj)&s1YU|>*W@4*c~=T1Ts
zombAqlxQCw1g`M=zVw>d9x9A}D`d2a64SZdIjydz(~*0yN|IT*At7dQx^#)H1X%kK
zBn84kiZp&0uBK88UK9ttJH2p<%p!Gsw^Gw8F-w%X<Q5+O?1tt3VZC);kCAd6-9iMo
zSTj9$i*x@N&zlD)Sm505oY#R1?h&#}EqEiO=Qo?m-3tL4y2`vVQgmCFTYIoZl5(AT
zvm~F=ACEvjJFtFg0k&HcTR+2`RD}=144p1MUV-6+T6%o6_k$*{cZ5P9trkKUWFAp=
z^GPkTP{INuZwS<$QcZ-7A}3d^&Wa!0zzD>X0^P=g$*w2gQSzmC`6G#CUA_9K?piBa
zkl6+B!b72I!DY!3Q@uBN*2wT1TIb#cc^06N`Gb|360*(6&XgRFLi5{FU9oC)D_&i<
zm^5+E^TFaH=g}RwE?*tZ;DyUg(}*AjVd5x^cg;AGjIb*L7$lQ*c>=S`GZrG!6^OPa
zdk%Z&o=i}Pr7QtDpijj*E-la|sfL2e>X=Sk2X61b$!UJUE6q}BQ=(LS6zg?}F^XlT
zymHx!X59vybfKy$J;;oNO0@vi(<sd$UoGinCHNXrwMWyo5M>)v^E-pev`-@G<h@1i
z%tximM*SwtTLw3%vPSs*q*|wO(Eh5+dR_-LKkhj?gT`^ywZ|h`(DTWUViNdGxR<GQ
zFUeV<pTL!o*aRvmghT;McoKJg50*}o{$Fp7nqrmht|h5v2D{y92je>@?A?AlNL^hy
zj)}nAxO4lRpT(SP)<KiKJs@6w5__F{{H(CUAcmezpT`Sm2zO}sp5!^^89pAjF6G89
zcTHq|JkH6sGK9<itSVZElfd$9$B+j6`ZV9Al0n1gFYnxv^6O}-E2|^nH5O%^yfb4e
zzP@=*=Yn`)IU;6-ibmy7{^x+mO)y?zokRIse5>_yKv;u8@Y@NRF1amQ1yozZtOD}1
zQK6@%&pt3ZT85gFzgc@oB;lJ&DBohcthd9H{uYO?|BSZ(ivh*f)#a78m^f!8r10~|
zBis|4=(0oMqi&B6Xb#8^y>NDU_!COH)Rj?fqPushIB&sXIMpgEi(<)iahOzyAtlfF
z@?Zdw=Ggy;pN4#i@U+d#K~NN%qWCjA6qmS2@*movYTNpws2xltK(1VFI0iyR1_f@x
zQCRp`fENdpBn`M6#KAosi&;Qc7#b_uA3`-on+0rRpVUF#Qk1U)0Cd}1agld3si3{p
zJlH!0^6L4inGEpngD-s$8GRT~1HutKKEL|6?f0Oo@c#3T9`wSqsndKPq4Q6>6|>mF
z0RPo*CwYw=i;h*fI(nD#(Kv$WwqVdCrO~c8mH&_DXc6>>8TXIu<~+wUq;nmlg`wHO
zCJB(s|F-`hRI+-2|Le!AB#8TuexLu}77kL?)4_lUzVml1AVG4^l%7gh!%;s|_n&W5
zW&uf+$1?wV3cL{ujyuU4GgahYPk}ECA<zM)j7?wvdnfD9^y+n)D*gAj|4rgQ?ESxv
bi^rtgcrs#eZS%JP`14jmLA*><Kk$D6r07h~

diff --git a/docs/figures/arch_2.png b/docs/figures/arch_2.png
deleted file mode 100644
index 197e163a54eb0cc9fca5dbbbeac0cdace29acde3..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 51944
zcmeFZWn5HW7dAXI3<wA^q=ewmA<`03BaL)Px0E!}IfyhOBHbMd(t_lO7=$3G2!fQf
zC@Ips``{h_`+YyWU!IQ-KagSOoU`{{d&RY`wYIU^n#!ca^u#b2j8s)cK^F$Y11}ML
zgaqIhp|*Q43`Sh<BrmV6Dld=L_Hws(a<PNKRAN)p&uQx!U&O8l@#o%$5s@cR1X7kz
zVs0)M{^8(ZCPLke2`{iwrZKd-8WV#hH816Ap}~yC;a`cv=Uu+XOWe0^^D?UM6wB<A
zd3w4e>;7wUJ=gD-#M0CYheatEm-jQHoC{ZANz8p|_xD?bZEe11?A^fPZDC_%2-DNq
z?W6A_VftlDU47AfFpUAf_Z-2;+lOJ)ik;~Ab1<rVUM#Zl>ET<R-tCLA?Bg)>p-#uI
zWS32(<&8|f8<Yw!9PT|X_rHCQl9MhXvoV%%8)J+QLuB@+vf*Jy<$n)cvJhI0UW=`d
zjNzv$Oz6Z*Pe@Rg>2ox*y8kssCvRiu=F#K)`K~?nl>K?H$97Up>8i~B#0@<wlXj;|
zo=5JzD(RYU1u*9%6z3SC21#}Xm`dyp`i)X(sifY^os=B2m8*oXmS?6fkc+(!epcS)
z;u@%p6X7K4kPwm`&xvuOxg8Lx)xajKPIE<d?q#0+v)dfcM6T+VX_0%}KDn#MQdBOX
zOiL~D-q>mOL|Bwbs6&CCqu&y}<(VJ*<LX3^(rEtfmHBt0_T>tDeY=CXy_RU3p}m2)
zM+<K*`Uk!rQ|McJJ^riSXl@6U*@6D1TsG84lwSQ_iGP0JD{7ps%<EXLjLsjykHX5R
zqll`o(zQrl6fx$x)k+T|K2zqL9fh^mP0_~>Ze?P7$u|d<1gd$fTwoa$<CRx7`CRI0
znBQd5nRA%PSCKj9x?~Qsry)L=!LWVBbh%S}u!?<{NYiFXat`-51qnDW8qQOIXHY=o
z9ZoJFzukbRi6ebKMuv<aM4hKWFsZ=lFo-lPwbfNFn4*>N7y%E?t(NQ)_SFoRBSASX
zz=O{0T=UD24ceW1@D1VwESVL_J+zcUxMLDe3I2+FSrT>6Ju@^v7mPT<rVz<RM{z^f
z5x?;Q&%K)k#E$&>=Z_J&_Xp(-3pJOSKb<4J;a<oufOu?uy$a5NyK7A|5Oq9D=nu~!
z;g7mI8|=?&cERvf^bE(IR4`*F*`My38670rix6*EX#hnt7mj50RH0zGaf>-pg--W@
z2)bHD((B@iV$`D{+DlPU);0qInzWYcY*Eq13C5A!Z}hG12xw%)XS8L=3=l3#rrk4m
zn`OgTcgclBl&%eFioN=Z373bn=$?9e!Af?<L!4j`<?wL(_3JrZ&wkI>Ex9d9lM-f}
z&n}C_z5Q;mS?2E$ERYvJ^ycD>-wfI8(A@gWxs7~6FXYvG)GxD9$|&+JLc9CdyXm?a
zyU%yi>V#Kld9x8v?)TXBX!IQS+?u;O2lGrQQ4wO3c+8?wS<+PErHi*hQ5n6X3BHsE
zZ}91Bay2VGGy4@YN&Q0R#n5f`dZO`&OIm5kV##F5Lys;ePwQ||;l(kqCnjr*t5lYn
z6-bo^CztnT_url0-+H=mWg&Zh{asfXMdg9Ms1NhCbQ2|Mr4B{LTWB6yE`Kf$>Ra6U
zJURNyw`dJYxJ`HnuGn+YTp8uIFu0`OeT9xIfO|7_D|MIKTzj>gS;Rc++yiA(<q54e
z1%`s}%TeAJq*E;vWYyOT9&%mbp5RjE<~MkzFH=3)kD(<XxxkuR7wefa{CZDF&6Xx8
z&1y8tn6%;7z(IwO>1z$e0^N#{3NOorcpkmaYS%P;;_o!9QeI2QiZ!YDO3`F~C;d~D
z!$$|n_lxiOazea!S$&MYDM+h_yf~2g&9GWbpG7Z2Z}WhW{-Q|KHM%Uftjes?EC-S3
zMn@Aq6S->L>IM`3%szEd;auav_IH<tZ++SQoOWmZ^REv~9|b?fe(L?`^eLc7ZNzSc
zV<m0%`AXul(+d7-^@pclf=aYLIjoc~yMJ;2a_VU1h;pr+Dj2u=p#3_@HpAJ-eau!t
z(CeCsi@vM(5Us<2kf!jIQ}6^*wC<X!V5P_lJL2rGnZz1?S-yiW2R$=bv-G6PClXxV
zi1rIU75gFVkZ~?!DN`)FEAOe8nA@+k7eDWInU0y1n65SZyHws@amaKSzq3Eu*AwSp
z;N&xuW%JXu<7?~dg9i^PAK>iM-{YBPxfc8koDiLG_uciSXd`H24w61Z9`YQz(d1Hh
z^Vpkv;;y$yR{4rAY>W>N3QlZRe7~$|uj!=eqZzSHvP87_Gylta)%O9PsUMR+y1i9B
z#=npJc=EH)r`k7UvFyhiKgpThd#(eTc9(6se49PeTk<_lmuHvjM!x58&aHemULKz<
zou^(gnEn09^sVH0+f-1`+fj!XLlTx%HC^j8({70b(Qhjk686Rqt9RO#wRXw=)NF*&
zSW%#f97*rv9V40vc9ARRV((=|+z{aOciwo6dX@B=xhrZ;PG0Vce15cXR5hV9S$QNs
zan1Qt)GSjS_nc>mPKikU)i`?U%LWX;1-}ZnvbBoYyAAYZ{HVKA*FBad9&DyR8QwCy
zZ?WC{{&Gz71OAwG6RQ@hm3vRmf2Jwpm+FXVjcHppLWnY7FXWrOB9>wLo>60-?G}6D
zhLrzAK+T@pk?u(n&RCnp|IvJoezHi$T2A}50It{9N_p*!mYJ(iovObzzU3YWUKRgo
z_0>PnJJ6PZnU(o_iHd3pM{7o9<_nPqJr^E6)E_=yvv$|bm>x?n^!YxMq|eD3%Gt#X
zx~wZ?Bb}GhbNX_$)QZwBnLcUJZ`^q!^w^Fet>fMA!Q%i4dkOb|3F4pB&W1bHw`(Tb
zh)q&kp13ysT0Q!V=NvrxsfxW?!!$^Eh^2;Y{kV16r+S}$-+AjWY-7K$6Vr*&{m?Px
zbo}eGpe%9N+L%>`_1mcR$fIs&MH)prMa%er*Iz>Gk1CJKHy@}nNQ#cTer=*_Hg0_3
z9RDLNi{f?iWb#RMxM}vy%+kox_2#QzeVYc#T>kiu_^#|uZDF_47y~6m*Y`{w>3VjJ
zbkx_~?6@QU`EBk|uAlFVpLfQp>)IUuq)T^8lkFa?xjqrC;G2k^NN-r*KdCtWDccbG
zOuE3gU>VmOQ54Z}&HCC!=bz5bKR*2s{ru|p$U&j}X0+SoU8%_ttI^u#N5e+m$V-oy
z0`}Fq`OTG&A3kTaWpoyQ7S_Is{Hb6~`KhtWsYIgmMKk;J((RJz-siW<M=86qP6Ibm
zb}Rcn_LYm}GUxWn3WtjRSzDIad@;M-yL31~(<I%Ax;s1d`a?Wp`VqhRi?xt_Imx!;
zHR`1+<<G8LNC!Ls0&V^FyNuzvnC4_-M?lf5XT>XN$JJ>Q7n)5amwfVGd=FeYG9PM7
zU-$fF-gVi#tKmxhj~jl$e(QKA5ig@x$fl+G0z`KGHOQWqZ^U*#t}kEJdKxBtQeF4u
z*&pwGhv^3nvqMaOwht_i^L`(G2|rp&S+e7AwD@$vgfx$Si>dY4>!d&2aL&;775=I2
zk>O|O;xBJcJr2LWQ5krLW9ZG^Y<|72Kg}5IzTN36a(ibbI)Tjg@_E_H&?ox?2a_l4
z2mSGN?3SCB{atin&d0w_I~=5JgUtp~JvdF_NJbA`3Ya??0uzgm^<{vK998B8>K0U<
zVpS2RvE>K9RO|4pEX82t{cyq-B3vSIW2xe?-gh`*VNoC_%+$31jrXbK_*1QzZf4jn
zi7CNXe5v4*q3Og*nG>ym(+~nm3s>CqQW)NwXu`Se?{C)uh>f!|QnlC6fbl?p7KVh=
z!|=f?IQWr)GyLbZ5}fM{`NClc7%bWehWytxn&3C|l>~mEd;b2$OOAq_1AkF~AO8Zx
zzpuuF72y5*HG&U(2D_mruc``u>)CqQ*}2_yboYM9sv8a7Ao5T#z6*oVutPs^Rb93X
z@cdCHeIsup4fX4`?yh{+x7}^*`21Zxpy$A({I7$Tu6EwmXn$81x4YN<rCH9dxDH-J
zZ}YRD&o1$HmS!>1&_>I<d)c8y`2_d`SY(LNXtb2qZTsuG3QGUF9sDKD;^^(|ah;#v
z&(Dv~Pngf$%Yk1|LPCOHK!{&R=qkA4>fHc0Z)^XnZg*M#KIGreQLwvf>*eI(?d0x;
zhMsF}<L=`v&B6i=^q;T4^R)AK`tL|?cmK64ut0w36MjKH0sj9y8{8@dy?b5T$=}Y!
zSi#8^$PA1jbM2bA)Y<j_pHKcf;y>;*`tO}$qW^Q}KR)??-)V5y&P(3i6%6Su^WW|I
z*RcQm@LxAd@k1;Bha~=r`RrXFXBlEC{{QTm3~?Xf>p-xLmz)%|^uceSWY8D<BKU{v
z?{Da}Aqg{oS3eAffvGCo(D#S0<(|u-d_D0t-+9MEKd=VDg1Cn*K-}clE&0P@9(Yay
z^JIq)-QibqMMVb%Cr6g}B~AVKwvg+v=RU=j-loPkuHpMfEL5QPSL2doQ14Nh6r~k;
zI0^$p{_|V?1{#LIQT-%gz46ZvFcerAE&_%CAO7c?g9Qe+DtZ%D`JZ>rUcj*q|GX21
zR965`RCqwD_dibrs~&<8V*l}-5d%YFW9b>n?)~FUEHI%wI12WUZvk>RA&l~xs5<^X
z-h_eor11V{CRQv66pZ7JO9t_O=L{}D!TbsSxnc}X0f~mGe;N~@{vS!8VH+g>oE{b%
zONarfO`SE*|4bVLo1y#X^k8sGax6?h$M%)}|4bVQQ1~Cx12g&m7od%*Rrh_OM!S9*
zlQ!S0uOBdorM*zdU)%jTp<gJ6PcP*!VDQXL;^1A&d1#4zC@jnaPEqu)Sz}ql1#RDG
zKV*{T5)eoVJ=u5Q(JM=^rK$<eleo*#<n%F0PEO8oyxvZ~Sc$Y|K3*M2Wk3z=Rl6%$
z|KGg=<L_<EeyXvQ9pBHF^y9ncHYuJa@gl7%?DRy$WlVth!i6%$cp9DD>po^`F-Bkx
z=$ZRq7b&?~kO+IU#|#dB-FJ=l_|J4$3b!5yk6DYr!eH(MN!elPgVU2k{%>u5TBBD(
z4t7$Q<q6sXO?YX(c7=7dr}10L+{_WL`xX-XAraT;^s&Oc&9@#Gu)SD0wQtD#<6}A5
z16sj*;LCA)@x49}!2*p1G-KRVu)S`xa<@?*WWzs}Yw10`Rqgoa*EiocNF?%|j^eLL
zHm$tLUwO?kITCl3;u4WW6dx{H2G3Tz81zR=&W9aso-^+VOf`A#QfVTQ<N5iCfzzuf
z()xsCcD0c_>7cc~p#3fO*YOv%EGF9g_*1!bx_`;YILglq=1QipDqjfPm<gwX>y@bw
zms#l$$K&el`bYS?j#yq_KkuJMjycm3KOzjQMU7<h2IejDnb`{#$;6{R#!x2J!AxP!
z<Nc+4@0AgDC*#ez*G_K}8GCZQ-J@H5evE7_^j8M{UTw60Pw(!q{`<(kQV~pSLFnPe
zhDXhkdv;+pZ05(?hnLI!rbE8aFMNJ#II2;Gg5BcH>Y9BNM$^lSI63&iZ*ZbnB)kfq
zrC)%72kvb!YvhPOf;Pz9dlC;arX^%g3<5!j#i|nOKfU#yrIU!)M!3-K$7r?ht@vy2
zx0t5MNJ*R8ma55YzSZ-}6T;l>D(}dgzsQ&Jn>t&{;A*sRjc#dNs%-O(=nIT%eN@7*
z(L*`d*tb{bTnS^y{o;*6StqO)`cl7At(H)o9L_tiDaKKywfU`Ij}2FYv<D>(AArTU
z>mX4AcyK12#Ygg+shDErhh@LMwJ8f7_Vag@fVK6rW4=5yFW%eyZu)&gmzF7VkuIWn
zDzHB=ce>qwQ*MxD<M1B-dEFv~*u83!_W9+}YSXRbf*TR@Tamh@H_6?L^btWivzh1e
zQfo=qPLjU7DOb---kR@~*UFb+XF8Vg-$0k%OpWueBpI?1Kfg4XTg)Kl_RfEBSM-uh
za0XBWB|2gIl+zR`|Jb*cdgb$PT_>JDT6XEB!&-(N8^p3$0<Bal!n5~#|Cp?g1lBw}
z9d@dr`k15i$CnyY<qY#&cuT-GhwF5w`QY;Hp*(XwI_q0H_@R9M66e<?W;)P`*3+FK
zTbaG7S1y)n=6?OPTPPpByuZCP>U=q5=h5xgW&skE50kdnM#}R;Z?=biGq#$NOXxdz
z6H6K0GPt%S#FL&iFn_eay|X*v-ml}o*zLMM;ojA|#2I#SG-B^y7Z6wtW-Ma+hGbl{
z-H+qRbtQufry8CUdE%b;SXfwMGI>smZhl`oh&+Fh%XzeFYZOSj^6t{$V9_i+<ogsf
z(19V?Hn3qz%;-^`ejWMY-sdrzDS8bGOO|iQr)g=1E9>JQb7;^9tRJp-HMb33R^Xh@
z()8Y2`z7Z10T=ji`@w4dvi5lCce3*|yh)p~`}36jWg6MMM3fwjYD3QnwRj(mY~s&~
z>b#+T2DqY&_vEH7i+dD&3z@YaewI+ND(O7R+vc4Ys%;M=AfdG)cRsdbv>(cIeLtE?
z2V-8Va!L&pE`BI`GRAU!f3)3YTEH}MqSeQFqJQ*HUyRsLlsLh#akDLZT3YXtQoya0
z>l;rS1Om2h$D=ZR13b)_1+LnV-EUw^^Zfp%cdyC$%k#CbP5mtQF(o?#y56<9Wq;;b
z?#>@vbDx^5cDe7-6r<T1>5%M_$+aH2*R(eI;$yXx1Znxw(j_<)Q6|}Ej!ehDnV~KN
z2enS*z4}SjmsVzPCW7lnA0u^LrS==s`rEPQQl)xJ&tO)wFIljsvBSQZV*KXX&o-SK
z?BB2T>B?{+us?cETWrl5>@_lPx#*__c$VMv7tu{Nvb3du+1$dE`BA~TPEXpJ+-wv~
z-^?baxrXd_&`LC?Ej$p5gV74zj!|5bOj@Af*6qpiQAb$4j-#gewRYG1HE)8MT+cq2
zjhsvC^ZSvJ#@l*RJ#4TP&8Na}bnuMn{X@0*jOwI?dKF4xg{_VZpQM_boQB+50n`mt
zqYEOuZ+ezngD<3bPu5*zUA*c2<0%Wwh-6FnrCmJfS6#x+KT|=YLX7#A&IC;D*4Ev-
zG14%%1s3~>s^4?IGY-!jhVp1Eh|_tEg#sStbNk)f`eWAW?c|rt%m?Y23KXnS7#tf?
z=Je?9_%JG(P|>+gsE9|Cu_^wdDcmYMwk-%o1;^#`)#-P$q%|8gwE1rqHQhZtIb1ib
zx#Dhf1(vdNv>GoB>j?VuoRCao^g^+bJ?m4dT$q($(LmzH6nr94W@w)4kJ1BrJLkOM
z^;np;ke<C>%AgDIFjsAiGa?-XT7K@0H;_u<v8Va;2Bq>C)W_H<j853Bq$%<$C;u_R
zN)RZ*JUV&pUzZs<Q-#V<nNBpjyLbG0Aogc{y7@WLP|6F+o)Mm>)rl+O)4-BVdW0n2
znhuB3I3V6hO$RLHI)0z!fR-!H3WLui5I+mY!il(YQL_jUlOOk&=*H_R3~Tbtr7VeI
zOykdgE^|}=`1~}zNpE*~n028(`14a3aB$gGT_csZ(iRmsVgBPNznB-_T_!SbVK-Ey
zr)!MrioALcRyE)_J3=k3(ROP1%pNb%^!}(X<>A$;%~Ublw~0+t7Y{Kh{aXg)u*qS+
z=Pv3k%BfBr9m8fhFRiK{kGWeJmM_b+haP`rk-F2{c-5pahVO~#AFAgAQco7l-RxrC
z0cTJK0m3JXQFZnlxV*5I23&E&aHL!$aj*YEe|mHLq8c2hCY^&nd-Ky*Zppx%3UjIs
zVl1C|tBPbw;<GVrS8GqoT-Z#abn{jINKcyQF5@ig<r>+dBd+emFgMPLWA=72g_b9c
zgk8UV7wX^59>kGO`hH0zvw2+8WX|}J$E=8c|5loA-P<ef1fCmdO`0~8t)|BDq*J4f
zyy`8|J&ShDTOJ8gVJE@Pc~T5fXIShP1eu~~=I&t=iMSX<ZNK8)skye?U2Nf-zKjoZ
z7hi1M>hk$9I^L8xg%4}<(;r>e%DryjcE`ZQ#~ben@5EfwDVX7dorkY;l1DuizGWlO
zWeavU@nR`m>_RT36L9|t*iX{6q(MDr=74W}Hni+*oT7mNt#2Cb9gckFdYsd7(ISku
zZy?L&B7eHgOOAFm2{nrE*Jh8EN2Bw}J(Uy_Hp$Pthzr0CV<WgAlz>2ZU+R_q3wRHc
zlrE+;jk78x)W$D9L||Iv8-4p02QqC9Ej!M`M3u%#2C_{hHV7sC(60Dcj~7mM1>Stc
zK?{jht{Y`Ak4bG;)5M=ac6g%eM6o=C7qdtBEEeq;g4tpImXW%g7Txo1Grn`An`@~z
z$Bn8RDdTAz)S8Q~MEr^KIH<7KDN>6Mf+9BT4DYMUVT^rWvL5N+&p!UPs%ro9Ym@Kr
zkq&HZ5LT<EGGT>;<DLnmYO(}z>Wa2}C5AnhxVtzY#Kg}(%Pu^8{C(JGV&*)&fH#(W
z32*Gy>w}Zc?{9ClfxzqBG3X9v)Z$`Cl`nlQfWp5)C?0NFqqqim9^4_%-gt$<(=^Fm
z)J4eAvOzfjA=-)pA?z^N==-F|%i5`}ZezapvF~l*Bp5^(<-M0+V{aZ*Uo5EDh4ET;
zPTqO^JxW^<rm(=FLl>Fo_UY?I3v7xsj9AQ0*aO5b-u9?`$&f?@=F$vDjk3>TcyfA!
zA-KFr!i=ukcacE9SF6(2n>V%PsLw&HmeIKWc3p={$k3q#)oii<4PCbbU0CgTlV|C|
zHF`IXCj%CA!hC=C1=Jtv8{)-_zy5tkf61yP>*8EEj4<OdfXxyUjJL9rU~Pu_X-OVS
zV}cuHJvOi7j3rqUf5cH=J{c6v`WkosZ3NDxe^!&%&$oxW^?7=6a7_z*L38<4G#Tsb
z@8s@-4~LK*Bx@HQuW#Dxyp=EPS?M0VpzobKYVh<{uatN`5REv^?B}7rMFVAcfw%uY
zj@Pcbs~DzjCg*j@??>et8LFH|xjy<!dBe}6p=<<hxAvODxlr!V?fn3soH}iPg(f1S
zYH1jWvV{7>fQ{Br!&i?vwTcq@y5qOMzn@ARa?+1=$ya-;F``rY(g(_bYT1EXP{Uil
zftx}4+t!Q}GKe@P?eM+gm~QcWRg^He9rDXxl7LBdDQIP+w4s%8o_qjcrSWQ~L`~R+
zYSYbq22qz<Uw{AJXyGq$aj+Q?sntLLlAW8%wDR4qHiJ}T^hY0qsbTU>&R9vK$=UhA
zV%>+!NZ5INb|(JenTQc_51ID)7;~w>D+=r|7csmmKAu#tV5?gPJ1chP<J8M_wr{IG
z&s`Al%X$$pl*1Syq}R=P!ZUT-zeDn{)o{|#>%|lmjCV2R{i0Nm)x<LZ;+pd=4E$1K
zjNr$I#oxADj3MK@%YO9xi)9*{YO;~h(bv{`yBf_kQWXiwm!BhWd*k0TncOI4<l$$Q
zY$FfMM)Xq>0n+;>R?;HaKY8TQyyj)U<<y}gdV{z20EmwIx6>t)Iq8m;EDhyP2l8Ks
z<Dwb4%PYN%rHbCWDZePD<J8PiDgN>tfvIHmyOg@168zw!<lV}5tg|`g$%**)ZJg(N
zVymMy6ggonnd8|B$>)O1xXoH>ec19|KWvuMM_{)NMe(lu_Mn1YT6*`0)r-nF_SHSC
zSPTz?;r;<gBHrr!>`r8qtlHgF?~lG5>Nxv2OG1WULN_f-oD$X^C2RWJWpqc1@>Tx+
zG=AL;$yOZe^5g}x3(b_x#dDQ$HJc-B6L{}a+Jm`aEf-{ucOF*>=WD!Nvmx$Nr<y1c
zuyfali{|WnGfDl3UN5w7nz`+_k8hNm`idvONHAmul=S1F1n9zp{`@k&?)S^(i{QMb
z%Ic@e(jnisrLx3KzY;cAo=k4)To-P430D>kh>aL|;k)y;ro_-9Kts&0C{-%OPzJXG
z5TJHZFR<8JZEJqno6$O|dhPz6=O2zJQ`7KUxG-8Th~$u!&kinqu?)NV14U0yZ{QV>
z8&K!0VcI59qdw>T1ps}8?9trpz5}nf?3kSBpY~RKhq!P&&4xwIl4_iu_-4iLBE|SB
zVZOvz<-|))f23+1XBzPR?Ux&dWFF`Ek0;t{kDgpv80%E}!go^!AcJPNDuWR(hYYdj
zN<Ko*M&OA~L`hlMwyAne0nZ~H9BqiDp1yQGc9lGK;pg|9YxhsXrQ!~~QjF<N7?hw4
zGJK4JaOVD8JRwjf+-+pf79#~8v#NKainqP0e=_qhbL^vg%Nt3w0c>?sIrlG8Qsl1E
zeiT=lsbTcAO5x4lCTD%aHu><n606{<`UCoF$-gQ3T~&C)(8G0X!-DK};rb`>Deq<x
z6WpTQk28+Zf_>_8jp98XKa0g(4Hj7^C3Hutj8(l)RALZcD4Z|-0I`TIZCk~y`z~oL
zJD1=n=8`DKIB_AT1Q9vPmfl@q#^J^QO_C9TLeX6{8yeh7!_0&F_SCl^i&GuYez%mK
z&n-xavDsyPvliKs<|N)RrpKCZX?U_IZhK<7AS6+3G^jXdRc|EQFL;Xzq|itbEx=H0
zXprFDz;q!m;bTLu-gy5&35#E4gkV>EoZ-VPm_3_J7(;rlE<C@dHEeu;=?kDul8r(y
zW>*1|`)`<lAwnUt!#$gTni6?mHjF~8WEtd4Y*HLL!$pb)v-RF9x=$>-I(gN2Ig=aQ
zpHSUb2NZ)kFkYO92C4iFOc<bzcD~r#KNaLJ{BGw2GDh9c-gWy%#IU-Dn9p8G5+IM3
zfGDHlTx)j1<zy@i?EYIyio8&7Cp;d~e{oVLO5l0ZCdl7#;3z3(NSoV7v&1IvG;x16
z5uf@uk#hsYe%eNBYnuNdp8f+D|4El&|L?VDLi#@}2do95f)r<egYfNuz${+=8j471
z4z)eoqL_4r8${e_E^-|YUE-MhNCd@vO3hr6MU}ktUj|Jd0Hgwh{%s;?zweOOws*>>
z1$AlnbmUTbhNUm>onDEnZ?Q1L5aM&}V-dQ+a^b!EmfS@+|7YdO>VNseZ<JWrhY-qc
z82IxQC93#{ym?e_?*TrQr{B}%_2_`1+k%p)9p9_O>K|2N!XGMj!l(KfBmkxW>a)iV
zFj1EajXp7?`N;6vb5x-L$$Mhqyfy1j64}t!OvHU7(f6h*IlVEps|$tD7~p#vz~T*3
zzmzlPRELr<Ov(CU*JH*oi;p!vGslRR<54lj;ZY_RA8diAf^QLMm2d#EpjAGjh-v8v
z?>_mdkhXNuT%*XO9XDUfEk2}Atde2aB$OohcY}6;`7}BtQb2M-3&90PP7iKhiEDYm
zyCrRDL7p8I(Gabxy_i=-Gj@5VzYH@Dfz26+-<x8I4TgqAry+T@PRhP0JFEKT)eXq~
zz8zl4qodNfufWW$&wn2Ouk};_BgdF`zGH;uS;UU=bf)f*nTp{$3TWIsnzhFq4Nz^J
z(A2SUrOWD{^Eq2RDh5amYjjUu2$IG|3TCfKg37nP$>RszFQ;2srDQRWvzY8GG%JNt
zhpW`7kY}bmV-8JiA$Ag)8jc+?oBlTbeW$sinb?I*FK!1PRz~p~`c0NFjcoCl)oYha
z+^qjj34@^qH$JgaEQ1z^x)vV8aDJNiF!p+!<<#t8SZb$&d%P=b;thMDI`wY+0w&<7
z{#gapnFukTfC!81BP7p6<AWdvrO_X(mMF#ct-z@}_KsinFKeW5lx`4F@l!@pQL~_b
zN#3D((Ee93+tq>N>Bl@`gdU}!h}$~4)bZ8QC~t$o;8Ixoud3~uPr_(b@;E*3d~H1D
zKKAy%to)t?P$*e<v_2VFB^t?Vy>not8t3niF(gp@vTsN-D~z~zH$h)!?k|~+8v?Ft
zF95GF5X|un8s+PTj?5?foGSRt`=s;+Je*3`@f?2Ho`T_SVLh}8t6M-A_O1bAz&{qy
z!TZ0n?yO?S&0idPzt}#Nje~Cr>E=+B#~{8tzoEwetKvP0K;UaF%+%0ATGVHL=i713
zoz(EX{o|wHCwY1^B%&ci?S}C=p39qm3js6bKx{{i*4kIW_S&M7IFFgCOBrl!Cic?_
z2-r~#u<}U50`Jzki_~WY24PV85U_pv${t+pN@&Eg{Mr3Q@77v^=t6)|oO15zX)0q%
z=QaGL%vo!Vq)U*NVK5qC6?aAyr@)fRIDpDs?T|5%zev(Pyq_LL@TOS2lur+n^QpC#
zAr%Rk+dqfUAZ(Y}6I7HMD^@kmqnc|PJU!(hPw_$FJ20%NrY6z+9Y7Hbz|*Zb3I<qY
zw+aZ>zjR>mVIA(@wmn2}t|uUVE;!h{ooFlQQ0E4u!vd^e*My&6cOihIv~w|InNw7u
z!(C)=b&I!au%z6L4-k53djHw8hahO}>DU(VKzqi#amQV>UP^YonV@of-=R2Sq^V@4
zz2RsyRG$Qz%K)TD1Jz^`K&|wdtpdD_E*<?2=Lra6)oYSGHG7LqqM|C_aoyzjz+j-n
z1nn^lVZjE2&nRZRJ9A1In7z_ED&N*N%kB?F+cmUI<zwW;Dr={Q=#+!MS~MUBhTy$Q
zDhY-VVZ}X@9mlWe!oMnc>JsGfOO-06Frv#E6LMyEJ78<oPYGeb`C*%h+P|h|0Y~RA
zY5H*MW9#-%Drs=rg-9x+MV*dEv}c0E9fBL$ho~fgc4uT@7Zx-En<O7tl5Z$3<-?I|
z1M;3S*JllRwdo)!!r+_7F63}DG!hKiE{~pjX%QGwo8l+5Eyvfnder|UUN8K6=&pJ@
z>Bc?wcZk0um_j2cQtU(S2#ZFA9xD?cmS<YlZa6KSh9T!tTGfG8@8Of_|Gj>O6v|?W
zAVJM2B4(GOb;n-7Ed~UAjrZl3@|~G<cf^R*O{9F4&}Ryd#qxmX>%WYHz{$W(++|JJ
zZu+3CEFN>@^Ek@z{_NkL*l%YKO9$5-)f;PbgLQOJ5K4Xy&&$b`!&{>G?W&R@MXn=t
zc_-{{3FlYQEG~_k)5$qu?TmJVhPeRWEltD~c>fi^6)?Fg#O(mAz~Dnflar?TPoA5}
zMGvB`t}5%xubtbCgtgT6{K-A7^(X=@86Soz4IRHWI(@3QQyYS_p8NZdhrlQJC!pvc
ziQpIzfDI`n&ZjPOK9w%n4~cmaj1=Opzgp1xlV$`E_uL{PDF=Vn$+-TijtmD-o$myg
zP4MR%^afy;^+1(|8$Cx?&2*FyH!m7xo@YX?>v&#~?KlC9EDv~wLclDK3}gy(v$6Gj
zta`+Bwl5%hpkX#d7kZ)H#4y(e{=RePLjm0GT~fH{V3kUyg_gEk=E=2^cTB|Uw{*Q|
z0sEYT&t>|Yx7wsBE}2c$CQt@;rp%on_RXAM<)Q=*Q2<${o+$^aVMQQsA3pf~gL^#w
zGnH>qll9v6rzCOqAsx(7)u_wjP(J@31mGp2MP0utKw0n9D~7+aaRs*L_=>I+$c8on
zqo9BU?9+oc4=-z3hVHXpHET(ldmXP#{wK8XOSFt|MpK;#C)y|nd1Q78dC4XH<&f;D
z`Zed#6k!<1xhSc*v~i$51&J9(_>VY<!y+x({h8B`Hb(M6j(3wBMN7+dwW+;Dy=M2e
zP{HS4*;Gkr$=BS>lgtuCpbAwSM}1`k6k+DQ&=xfJ&lZg?fYj7gyqF41I|9{mcPVjl
zEeMqR=*8S}%uZ|_ve@rg{JN<`wLA8!o=z57_UtAz{uf2313;#i5Q+zEvAxf_7M~^J
zL}6z(d1hLu4JPo!^*RMju(Jf59=|3fo@f^$pmH3|RTG74cRqRC+5GV;U$5tztLA#<
zij7C&BdU}a4vieLp+`o1g<m^EBuN=W6ApHNGPnj$p7~YOMh;+AmPj7q0n;NQT$}E)
zOgu6LMaEj|h{+v>)bW3rm(DlQ_rH^hx^%2Y<^;bo(9kGVn>LrdxVxka*!Pm`4>MMW
zA$WrJ_Xz*$ARho^9(2?!kk3fPeQk2C3_U&=u?1a$+1x?9Yr{n`EnV3br^eU6ibr+P
zr6kRzy-=p2eQT$8j)WE?AAO+|RNy`WyW;ko>)|U`RQS8|Uw~D#3=#_hJ62b*0=ceN
z@jA%7xxNLL-Av`NCr<MKu8Gmu;`riSvxql@|MFzdePmDeigQTcWsC9GMWb7Yd>EKP
zz#_zA5+MCnxDU<Qo(-oKGWMR+ea9|6b!|~N^k|Q-7<hoZ?izI;iOyk`sP^VK#Pk!o
zD-Hg#7@*)K+1h!Yz3KdxRa-Y${!VWRJTB+%O9Sv1C8d>EK9oB4YkzHa*P`UqWPAO-
z{Hc+82ML$B;Ona)YP_ehOFYk1Hcw6t-B*~GdmnR-Xe>MA&$>jaBzgQLK~#YrrOhOP
z7=p_{{+jBzB6he{HVvp4-E=;48`tHh)7y_1dN+;Azc_anB;<05e?dNcIy9BGSB2Fp
z*BmC3J;H#zZdY9E_FsiW4?0kNoTS12RVMWHu#<N0?STC)yOqVq=_mda-8Dm{wP9I7
z$&n``1jyc3U5Gh{&70r$3j}1G0)Uc>1n+sQek^}zxTVMeZL^g<ghbpz#x8)@?6`A5
z29>&z{_3}n4}R;n`w#rOQXN|yNV%z-L!4vmLeNQPK=R6*EVwOg?5M5#sc%Q<ah6Hb
z9Ys(GEC%ZPVQ1wdk(*pc9Bn4V^(6qU?4`f72oy+xqt@-)^_4|ojVzI#Y|Xp#@5m)|
zd28-l<|<ipPP`Us8cg0?4y!AsN?JBI;F$uIc!~2HpiKM`*yId{K6T$aDj-1;usH)C
z`fAAmD!z%3${Jt@iKn^BqkIBH#b`Ha^vx>0;P8Bw4L?RZYL_Y}$g4`=MOF;i`S0?t
z#)MIAM1aGteyU0iFdcA)Gf-l?!3C5kBN@j3du6yddUDycBhbgB-M`M%P3BuTji;Yv
zhm6@&F55voL-Nu4uct{VPrigqJa-+@aIC-GJD>2rZT*Sx_kvbtF7QAYGFSmbaOD{t
zUSNmogtggPyIimORe<Fv1CrHi;(d8r)6#ofJNH-xGuyNrNMta=+lfrlJkm(%kb@LJ
zILPM*Z^hc%6{8_#{2&QDMMpDwl?=p48i*s(X}a^-5!6gY><4I9zcxKJU9n}9IEpnQ
z-jOlR%YEQObCM;nk$XUu9QJ7ObDP&f92LA=E5B^K!9fp{(XZ9u5uf=BX<#`YY@Zem
z0tKjJ!MVwd*Bq6qr8~a7hwnwFS|t=`$I|}_Z)eqto2CDU(c(;4FHj;HhOko>V$IdZ
zebT{u+O0%T5#OmNhWyi0L;n0vc##lp#^wRHzFgU)Z3y@qBT8{lU<QpFncc5mE!a;#
zlsVuU4G2gUIC>LIEIHNlnD>a&)k7#w?{>`{i!ifmDc|nb2WivzTs3`O;`FiXrAEcm
z4HN{wQ@}o{kFN>PBL@_*#|K}Y8a|u~SYmwPB;qibBZH3$4JC}sWtjhX-QUdL>Jo84
z*m?Z2jr^uRA&0xA>z#+1OpDKh_q}eWa6Ni!&w>X@R)7Uami0N03j$&zGOqjZ_^f`4
z1+_}NuPvUe0pH%2H?jSh+8BogD`ENMF#?hErkL>c;03?WOC2y!QZ5Aqiketq@N+!S
zL;$T@8aywEX%D1GRw%^U;$yPJIdbc%oqjhNN;Z`l!_mmx6^+@8$_ECyHdX3YW}YMK
zUc~C=*WOfg#?|`#xb?)mO_W_L&unmS{(bfrLxUSHAk&=rCzR15Y!BW~29ZRrKkaJq
z1;*<i?h{e!`)$)r97I`Ox?$~b5I3JTW?yLDI@3e7HFdqqWS25^Gwk^HMS~~$v7naE
z@$G&Z=uLT2(E=<T;B`=hvufpteu=hnIh*nBSJ3M5fJr(F^kFRiAyA&Q(5g9gy18@E
zIX!v}ts^72)s>8(SlpVpe*IR$xBIs$w8IB;B+hUL0coRTIj)Xxj|6)131kHMCU`yo
zJ#EC`Kyke!TPxJ=p+Gx{0{_dcBbJVnHc?}y;}t)H3pXcH?rp2H6DO?u`NUVst#LGh
z0%fD~7rqsaMu3+D*<Xg7?Uxjg?<+Au5+wXCYHxc;wlrJIm*@rXC_KjXB_|}!_jUG4
zJ*j+8Op7+kWol#(_s9m>HB~ju$CrFIOAs{{9m#HT7x~OoK)mCaqGh6@npaIzIcp|r
z1wgCd$D;fKh({T)pg;SoPivN|o2QbqW$rGF3<*&?Ty+fajO^C*D~>t66*nO>#(+w}
z-V+Lye`$QU;`M2oPOo#c&Ci`)!mA{Q-A1esl=c&Nc%DT-YEcMc9c^oX^(8{ueAwlK
zPj$A+Jo*($R0v)~0tK&DqTX2K`hAoUhc~9>z&`e4aD-6f#oSXqDimka!NPvQed7ju
zoi^W}6_{|uhz3_aY4}+Tu!pwpIZbrOUuA26nv{T7f1+DTeAT@5)B0p^;Q6M7ef&5S
zCt>uokxH6_Lr3b*(=DO`+--I)0S$R#o30Z%B(1c|RC^`}!1COBWhuf+0bBEet-ZBp
z&T$Rc@z^X50<zL55CwtbOwA2tdY>!qS>!b8ZOX({5YuMLJvXHp=iidgH5WOdL4!)Z
zhe%FO`z3{^XHbUSGoBojUnfeuB*F6SekbTwX(a-aYn;-MI|h=*;Os!y;VYJdKtUMV
z<MbqCW%2G%o;1{L1%)SB{eCf@7=P8Y^75W4T-V%h5n~GrcC|6vy70YT25VGG65<|3
zyv{LO{o~6?=yLDxbV=y83U{pg>Y8$?MQ5;aE0J{2F7NT)_lNp57Sgg$7@r^@<y-|l
zYuXR5dVvISLnu7{B7Yxg4yZkpgPtp$Z>>HZW{7%WhVXM_Oe$i9T_*=uK=_Yq^;>rn
zMb?7a8}2^jipG`&=#V=qGam}&N8WStpIVFM15$LPn69GXGKG~`A%J|vpm|CU#0e)*
z<R~-NU5tJo&I2N%BL<K(c<;DC3WO6zK=p!4KS80jX|gd>($8&3NZ}631C7h>8`t0(
zLg)j#O}lDJAXa!GVFN@woH!Uf;bU4Y5=o+J^(KMtQ)fs};OX(7((Iw;>ClNJ-)^o`
zw7}g(^%V}Pq3p26b9VqwRd4~Aj!z2<@W^m7qK%j}K<Ya!e<<`?A#2<3d8xr9r|6YV
z=<>BuwUy{~!eg}-6cNZ1qQOw!V~!O<gtWP^(c26ik;l`Wp!C*XifLcE4XU0yn{O{4
zhL!5bL+0u62n7200u(8@lL4`oS_-vr1^sc8IGApBw{#^|;zrM4W=2ZYfD*TBN?>0q
zgS-Aq_X%8p|MD=XGI|8s%Q%ng!+xiVXVSot!7A9H+zOrID5?^uGs^8->wc{$7d;eb
z+aY#Z^3tR8z{aAHwe#KbSe0k=aEs(OoLc<G^$Kn%(T}|!pWIM<1GZcS)a7RMj<^74
zfh7VeZ|v5pp2|bLNiZN4umvL_+@pq1rA#$AXweJXlboR+y})}&qN68?6(na|%S|wt
zfo{w>AWY(LV3Nb2ilTvko+iaj2DCLW8uZ;LL(h;RKp$Q?@XOkTa*^}f2cxF$0j#F;
zXG~!h6mO}-h(YgJEjtLe^78VY8_8Fx+W!P0K*tr^RDO4ESY8K!%IVWz=%~+Pv?;KU
z=OP;*iNjDL+RIvjf;Ku1v)uVuMyio16bX6)MYtIvu~;#}BM^wUg5$o_SbhYqyW)0l
z^3*$;fU~f~0k%=(Wr7?K85MBy0Oe;v&Xr#1HsSB}Y0DRphNutJkQLk%D+KPb0Q6q<
zhw!!v!BPPwBybgI&PP#cl5_B%L}rjl?MvNVBzd1JnZEj^rY3}4n}yK-5v+}<`c^gl
z&u?w%ZxR{Rrh~U1T)eA%3&{O{RPP)DRE-`vALq-4X@XX(LLs|8@gAHq>H{ZS=4uP5
zgEqR13*DLSl&;JEfWu)z=p#&k2hF$#6pK5HPU#-joycXpKJw0&XBv>Va+3jw^Tz~n
zp*JvNxG&|`Vr4Rk^V&^!3<B(7llKGZgzAB?H|W6%Y6?OBHw-aIc(n0~Y~;N|I0C<9
zqDRDOL_9vq8Er*~0N4?9pIsk5cR3gXO0Y?J_eiX16Cjj81#XwW#+d+zpRx$}Z&)3G
zKC=a-UeM*C2YT*~p5Ch<C9J?;B~R8kQid9V<%4R!ej(Lg61<KRJR-<f1u)_$e7Ns}
zu_DFuCbd>@0O8_5qD~-|uT$G?XG;M<Qh_%mJrffj2+*L@G9#*+2bhOBq%(F6g(K*h
zE+msoFvyQiw)*);-Y;R{ct_3k6?B9ZD%fukP#gxmC2X%fKt7|-OKN-;cz%G5H*<*0
z01FpHwE!Rpa)g5&rz~NI3)dV!prfc-ND*&1Jl3bAS2(-@pxgnit9l?INpKaY0)qR1
z4_0n2LdipfIMJF}9xkrmKHXW-TUp$GtR=H~db2nRT>yu~-r{LH7b3tY>01Ii8D42r
z5KQVIK~DA$!j_L>tkvhK_jaaKfET#`o8PpLAbU|bs2J5|e*iW;+~WDY0`eFBn>KNm
zWjiGjqEo@FLzRKit!E*r5%3W5nNk_ouZ@Ml`&+$(vXjB9Pmi@tU5V5)g%m+h(JhJb
zrp&tTbI0`xb_Bxp7(nXdY=QovoDpktkIh3uH(y_Dw0TY3vY&8Gl}&^){P&M9dL4nk
zV_plj5xsxJ8l5>1T@8(i0b`QU!wmp&L=J);1VkZ=F!bG~yL72aOjz3zwLupS648h!
z`Vd%2!GEcIIb^<}wZIDO*f!J-ZJGVb20}f+W0hnN<%gXZC!Ubei@Ym7N}>h4)eHcR
zAVmZk<TTqIxwDXNnMIAi27=KCPz%Ps!RcHgT+YOSqG4%yK9v5gjW<;46e|Uap7S{e
z&!=n&`t!`2@&K?w{iNN1e%T7h)rKgl?;ukb2xWLBhuR<x2i<X|z9;!o0dh|bYs$1_
zk3U`6XQ4z1NQPhJw@?x6@LIUJI{BhOiXSZ?@L|o!N)|ZLIG`{3_v5Q>;JA>_-KY7|
zIoZ_G)B_K!AId_B^%N`4O%4DNo*_cukO@Fr6d}m+qAx<#7jdYy2zKuwxNH|0dbFv0
z=UeNwh4<Npz9%2cG@LKP^C5?=WHnNvy45%XYRJIXRO*&)nM}PSgnT&_aCF2(5bdtO
z0xHut@y=Z+0mK0%zm`glxMzuY3)o3eba4LitP~JUdad3ofT;j-32KJT69Bv&2<|Tn
z14mboEKt5KwK`OM`aWB%98|P%?;UQdw;&Kt$i+Qou-=q?Y7wF?V;Im9s8`K<8-P_S
z8F2HW^Iann#4w25`M@?=EdC7f$?O;KBY|9h66HwxX|@vCt(&%UV*e1J&%CM_07w22
zEL&%cknhi|ot?W|^Qu|boN2B(jr5??UXnm=&6g4+#s*NI_au4}0KX9Wdyq>5(FRni
z?p)9(Kn5$%jMdrJZx0E>K>v0T=%PM&Cd5StBK=M5xBFnG+9&3|D<hGRU-h88^EyHd
z$hZ#BaH#W+DhPxUYpw0u3m~OdO=cVUsC9C*|G=JQ4GuS_jERXUaPQoY;?T_DI=!_&
zKjXhSxAK)k9|RMeAK2;Ow@mC2SW8rfm^%~XjsQqul{{SwKtiQ<St9iefg5=jaoP7k
zNYMh`c4sKA2y*D58o)kkvd^XB!F`BG7_X;*mQ)Pra(?V46F>li?(HI~GH!^X7jeX_
zd@N6QlWVZ+U!0~^`U$rA2$V0d)7OZ2kcd@E!t>{g0VVhWbf;H<R8It87GfeIoi3{s
zXtSvi?c6PZ3!*}y(BJNd?#)J9^j^$_^7IJs42iFFx9(w6ao^uQ1i%L*W4*(t%<@r2
z>$9XlkwXb{q=RljJ7x~TTG{I7r$5TKD!9QXS^$ghQOOdYgS!%$b%mM-9!Pl3-M{pE
zX*dkp>SGFs+{+;XfKe6@4qfkKvLdjHQ2bO*MBKoMArmjl@=2bOU6x7taUAeX7U~{9
zAp%`?X1|DLI^dwgzZ@89Z|RMLW)i^85LPI{5SXj=?=B*bVsM~2UJme;B?_?=s#>!6
z#8ex_UkiDFNb?}P7_5XVQGGm5(FHICPNSk+8&J0-P`Ls3;QU8`Sv-^rz~O@wfDl1p
zVu%?s$Tw^P;D`c<c%aS0q}h#e@;-e-rZqL8KXjYz{dYj#>9~#ASCy7sp<DWvGT$z`
z$s9LVYYU?`Xy7NNOOGRj?FYEN-3Nf+v#BxbWgvhEV8;R~^4%nVF$w^$Y7GuUWgHq=
zAHb&0EPkXxO+{c|5*>jcs1G!on3xzORe*lE;}LJo1f(Gegf&?h*@7;>Lulnm6$0Ah
zBfzf}AU&a?r+*S*QXB_nKE>g&I6$|;vGe8uL*wmU3U0!7bAl}>=5PYT2zkzMhZLbn
zv_TdEtTa~oWFSK@B1_Eumk1LdL_<d4eoi!(W`Bq{M>f(EOIZn0{6&FFw;<;M>Yr#6
zsTLx{0rLjnhu;mJ9v?!`82FNZtH2%Pn1&-DwwwTf{R9C%K^x@c{U=HcbP-q?h`A`v
z)mB5rq~QP%tptf^ih#A;%F0s5pQ(6J88>MBcx-j{hwDD8AAld{cAIPg)Rr?yXR5%`
ztGG4@kUOeYz^?<80a9bTC0Y3>k^>QB(^)kNmmz37&x?trU<dw-id(nBf`b^C@`fyM
zK__20D#)RtUfX(Sac9WkS;ITzy@6h|=FpIXAqG^ffT?||&|&RM;eHGv9=jUN-Fw*2
zXe%4=01mg{jAxb*!~-WtMmji8lN#EFl9Euz^!KxX({a3M1%kYOwaGOQ?!UizpkJaw
zuF<b-4fdlGLdT{jT#U#`4ggI9NY??qyRdv(YyE0+5D(Z4Aj$x%2Z9!Gs!o8zA2CH$
z{L>cc@59>mtMn@=018L|t!jK<P8fPbG63+M!9z5=eM@FM1zfPISfNxy4A>&8U@+Y`
z?zTpbC<JKkN^+A~pXyM%a;vI<zjLqq%z47#mK6TGYjhwCJ{$!u2!v{uDO2DLX8L&w
zU4W{lf@xKd(2hp{^b0!H)mrxx0CL#b(W|w(;eI<_3Ys(qvMi`V0UVe%$n2b^+5<c&
z;~en*Yg!36tibt`PgTakeE<Zk9XSkS4AVxvBnBgNu?ht4I0E~szpc8$Ap%n4P(x*X
z5U+4|!SwTl6)vC)=))heD(mLUgj4_|0-acC+1Gy{A_Q$VA?Ufx0VM+UHKvqY(%XYM
z#t_v0hmF~i6OxgYgY!)v6X=91tY1X|l>yN%!H%s8VogjDfb2_klN;|4Xd?pEklN`{
zYHDC39{Iol9u6cSkoxQ!XMpT3h_PNfVc_)AleFl8v&@<<JnZz?5V%0C8-P#oTIl<G
zoB?|QNNu@xSljM9g;m((5o^%SvJSvaJRkz}wY=$ungF^uak*h9n%9H(1T^xbGQU1|
z{qe^~2_HPx2GArZVb&fPT!S!2DnK{!d0n=aMT99ho~6;J$P5$-jba2b?*R1v4o8tP
z%vZ5pmLU`?943DyjoJu<bbMN=q7W1(raH2p7}jV4$v^(#dY+o+HsYx@3veG6gd3eD
zpiDCY&Oyy*!U4)+wYE&QFd_Sm41Wjgi$(Y)HR?kO4xCW}$O<F_s$%(>LUtpK!BhYS
zK<yaXQjbH9Hs6j2(@oxQy?J31#3NwTdxk*;q8zLOcRq2gerpp;<uf-q$>2~+8@P?M
z0^Rr6DMDV;=Lrz822wj9b3w<#W*UW%8(2aFObbr9hlOdC`z?P62db?JfMYo*m)KnY
zcJ3#Z1rm!g#`rm{n48(bN_jqEKaK5M@=6TO5KK2%k!c|-4VcC}sn42en_-Q`4#2+F
z^0`j=1hf|r2kaiJpSV}5W`GWic-{cc?zc0^oeKyKh})?rh<?$@z5!0tsHO*kyr~e<
z;cQXYU{mIlGxClJReP*y`{gyDvv$#9Y#>?t5C=5lR*6pH3ei|%+9v&15I-~Xh+x1{
zm-AsmcR+>FkPm-#qM4tR@j6vQ)`9yiBP$QEEd=gX8B3RuyKCd}puos#!QZbI1`(mB
z6|@k20E*aXY>0*hHij32kq6h)cPr^EvRVBQv;~@%B5~LL%n88|QJexcJ%n?{Fn>|u
zV7*tETmVTw1mG1NoM>19Cl0w=9)l-T)#hz!Gr|*H6X%<t1T6=a?+P|tuP;x!4fr%c
zTcsgk#S74=kXtgyc}IerLd~pffc%WyJ<j%_Z15cp{?i*k7eO}xY7excLVkM7`!3Kt
za26u_rOOy0qLmp2FUT&FQ~WJA1E}x%k6qTU2qX@M^yc7jcwnW>>RFSL1+xc0Z6FYV
zBUgaVk0hpUPv#ebtnWX?G$W$PRv+M9r2cCU&K=PZ3PW3oBE;OMQkaII(vJk|*Cz&7
z8kn&i|KMz;DfYl|zy-&P@mRpE-()UOmeh6t(?)G@K-drzENnOlL5M2<@Il+%{$V?C
zS~fSplW9IAA(5Ae*1okFw(gS<5j}E)4xX*i;^&`PcO%l@QXfEZN6-QPsn%LPOFBsO
zbu8uEMky<C!o4h@1vRFyg&%o1P&fftT`2{`iIq)+n~cwNi2&pm*%5%pNpFDJ^`#U7
zOri~Nxd9K+I21xb&V{~yz2zLp8EGYbd4DetN8C)~RRzc5BC>~Q{bxlXI)oRTi2xB<
zQ^IRLDT0Ge=r#<hjXnU*#gXLFYr2TDV(za=W49fkoqQl@kh#e`7;dZuoEHWU0cH+R
z10cmQz&BaFyhooB#{yRV?<4>lj~sjN07CRaOc*%N=mg@O$My<~qykVD7(5aeOUYUC
z{r=&fbxQuyqa)DiVCoN)Upo^lyAS)QqmFR0GsHCTkA(L!3Y_`Val0d&7FkOGr5TN^
z9^c=Pg6#&7TO{Pb`_6Dtq;F&~8%jVB;0{VL8FjyhIDlFg9H|eR?8Wi{YwpR9*8{vc
z*s>UfnZLC?5F4FvSh&dTcS$OqP=ycI25h~(pYjD=3JwHr;F$^v&?}1#DvwwQnyBod
z)*b9Fvb(!``A1Dx+2zo^a1a)FEyt^cAOLwC-27=a&Wuz63hp2VfU<hYTVmfov0x{_
z_e=-^)NLNS$;lqB34?Tk8c-XT1-+1PP*sGHAqW-ZttMMMKUJHG@#qxYpWMl2QH*o6
z`Tn<m5?hX>;WOg`1^9tXy*wrovBEqG#XwXHfQV=zN;5dzMl9dF)w>XoFFF7aIf3er
z$9C+}S$l>EOYPkyO%VC4yYK5F+rjB3`qTs)4DJ$`@WAk3?FcDiC|@?+O$MAEx}tol
z{E&%MEZ=QIUjIzN3Btop_LZ+&_;1X>H+2U90t@;}g`t5zo)Lt>0ZF*{xa07Wlz$S3
zdIpB^x_9l1_-1IuFoX$3OXzW{H)Y__o@W*~zzcL#y?;)JhLAQ5?6P$>P1Fmu8PMh6
z)fjuM%Wu(c6nN|@M%HugY@;^#!olg#{?ZK7X1CHC5d_0CFC#ikVP1G}O2DTGSQo#V
ze}@b`J@$KI()7e^wZZed5|2rv<Gc#U63+xqi2!;D9U2VUb-6QALIjoXXE;7_YJ|&?
zLhU(dQX*j+Evw(FpFb7c!0XEpEC$pDnBtg2RLbA^KOrY1B`podJEsed90RBdTIDNl
z(EwP$3MJ`YKf0gO5&^hk;kI+tqP+#&4X7~Mk7%F{kglu~4&=+`gM+!MAkL&Q-5V*_
zT5szs00(`!ftu!uUV8GA4(_`>-sEifVz~KMwFyufa28ampQ;ja51?^~c>~l(KBilS
zssgB)%u)bq2U#icau|@=1NwL*x%QPEgcAYg=ur#n++Sp%2c|w5+^S9n!Ay|<5!N~s
z5L5GfIHNE=0Q|78{95}w>H{21fKi0v$+9%U@7Ls`>=3d`;+Wt*_j1kL_H+p^P)Y-q
z0s=JD`jGtrRLcQ{4(bM<&jNHH@aF|z(&Rh7ad_;m8B3V-$kKvY0OH4ODI81;lKzPz
zNP8EBY7fDG4>17!UjSRWgJ3IEFBY!xPyt&*L#h7o_Dk~!l+{HqfXND@$u5roQlQeP
z&W50l#2$-1CC@ncf7pBLpsd=jZFp1CAR!?wQi61MC`e0pmxOfZg(xB2-Q7rc2uOo;
zhtl00*So=IzUTd(_nmL%`~Np{d(SxAYwz=1d)0BQb*_D=3N6O9c<NU4mdVg58{sG=
zfrsSPD<m)>gS<h&Se~pj_-al~I6iVB5u|}i^(UF&8R-99AKgg2?<>B?jet#Xk1lP1
zC>ONht#n8xJm3v~1{MAZt^vIC>7Fvf-|6Z=&F{?#$k#Hn!j!h%s_4LeC4-?9Dyn#7
z!qH(hp-2S=aILqK8Q=Lq*@nmn?;d)Cg-0RTr+?rAU;Tjf5$-uW<fU>X4fygS!P_ok
z!sxH%BiBAYH4HK(1Zx3&0Y3ikPoV%p?$|px!DTFB{l1Y9Mi&N~@IJmz10^U<opJ6W
z1P_T!r=9_mq71&LCBPuT3ZobGJ~r<3f{#210GLn4;pahRag-3=C4=DrM1TU37jB>k
z2ofPv%lFR#aqlF3B5*|qNRJzRMveq<6ugoLA4!Sx`=*Huw!K##$LbNZk-(Jtz)ye^
zMlznMtbuV=-xyZY=TZzHxHkXt8C+2#WyX0p<Rt|3bN>;-e}wSgn$Ru)3)QKY%J=3I
zReRtOV@eau|JsMG)Tpgd*lyNMH#<+aIDqy;h320^@5p|K1@%?9-RV)I>vh?tjc!DN
zg+|BFv?U*7wZ4oMeW@5$y4(+^af?p#nDS(Td0JW7UB0VkgYlggwVvf`sod{brD9n^
z&mUTL>$d)sHk~+p$qCxtXV@;ijNwv~OUbQJE2mTOXOH1FVO?j{)gib2Qy$IObpxdp
zp~$*l3SIDql6CSQVw5b31S85tjHG2xk|{ID4z(Swx)P>){g-W%adCft3PoCv3!RDE
zFTHGi4hUTkyhr4^`#bUVYnDUW<?1WtKfCT{&Qoa4c45}gP*Z1O!xqj@16J<z4!pZ&
z-q~w9&8BV^+|9-3OGyeszr%Rw*Ee$tm45%$L8WD;?R8IIMq`+dInZs9L>F!PEm6}}
z(G-BGfK#E)FhL>zt$>od0fzwh1J1+J^Fs8LcN}L_cydOqH{x>%*DVV%&}htDq}Gc=
zghu-l29<+}gv!?R>ECE)pHiMo-`!K{DNpUhesV4mt+>E3HdO|iTcu=`F29%fW!Z9G
zKIYOO+Sjg7v1JBgvBZ<j>ZE;IK+sY(+Tz0EbIXX0HWol!zTWk4=6xM?H}&$799eMh
zgw=1)52eg)@L8>G{`EPL_Pt`$t)wDVei}XXoDSP+fYb6w?q{}mEa+Rb<(9y|)PXkQ
zXpL1GPd;-|;8g+hf4NA|AxZ0@Ko|_Ws%|TfRjcYL-S*<|D{-JFvknaxJZJc%g_tpR
z#0bJajU_+Ll%odmB}mz-0Ba9hUOU{Id{l{^-++qFKgR)urHH?UnlKf8<%1QH3;hI!
zM*<0DES4n7?wJ&?aV&WA;<04eb7cQHv=f#u3ZO%Cr+RW|DR+_p?Zs!P0YWJWo0j)p
zo87bAzBI{JHTZVES=O=CfoZxuCjt(_S-<b9w=$A1SJ8ysSypY&*LYBnkTOKH%#?OQ
z%uCI6uz)&7-E1N=#?c{EHCo-DSBA)n?o!E=I8<KJEul$qNT~NsP^Lz=1J)Mz<0w~T
zr7%p?cTIYKe~v2QGG5v0!{jnhx*e|(lFT=xA80qw5gt|I;E7qlyTAj7i>{8htu-&V
z?M?HpB(brAMVE*@?#~e1AFk+fWuO>#W<1<-fgFr&f9@$ZG491Mt(P)sE=hX@hzr?r
zvs1oAuPQQ8Q3Mat(+n%5T-lZ0Vm<Kryx%xodk}M%-oxorYy6Z`2_8G|KW}n7Kw0GW
z2(s2`mTLYgD~Q9x7R39yE_CPYzRR>_dwCg}6>i}rg6{8Z_p)cUJMhSsx8bm20jJXk
z@$Sx!?_^L)`OjW@PidVxJAIYq;K9`}oa5!3&xuB#F1L^gQ;PhrmbVYXq)Ux756{?8
zw1OV4&oO5)+=LiqY>->;_)~B00~hp^wt^K$=LSD_j<AfNwNRc6od5Bt3@wedcXe3Z
zGuaIxU+%xMb{<tdrhUc#MQ)FWHX*nzV3TQW**5Y`x}vZ?L=0ocYvM7u3Afa~du)v&
zU*UX=ZVP1@sL0?M-@DUYk)%0ONQqamBwT@h1NKSFE1(`{<Q5L&zB;%P&pp2JEvaP<
zdmbS~QBMUsH9*M3EZ-_vQ#o5g+Hwv&sOh4MhnYmqD~wnQo<)c8^;K54nkt$3k-3LU
zH{EjVzS)nSG*;BMrL9{v5XN_Il02Pdv6wWkqb7Jps~Mao;GX4~b1X4Ez6~c~o;Kst
zK_W<HOqUSQ$V8IwdA3u`2di(ee7Jo%IbX7V`fGac{mD4{F|jSpB4@}<-Gp`YMU*d&
z`EvhT<DkHbQ0nDV(?juPC(7D!7HvUqfAOguS_BoYj+Fp8vF*yEpGHLLR?dk+W9B_^
zd_AbQq1jc4;j`3{hi=g~0UZ5`-<;1WkvZ@(7}mH;2sl$2$%JID`#nFT*A>#2A45g|
zcGob5SLxm@6D^T3-|vbT_AR_^(&{iYa#|mjEzlmAdpyp*?*N6i;^7IbhLvQCC41*~
z!~K#T5tC~6H2FR@J-)nGqrvNny{~bDb8O$VtoFG|P)QqaWQXD2xpIN4)BIY~h@#hR
zwbE#Y+pbl8r#j_l=Te|z>&S?0S`^vG;vl#^)8m-x@itP1?xK@Kvpq#*nBHadJq}4^
zh7B50sN_bzw$iQ2lWVEb!o7Yfni4(cW475Aj1uP(;4$j24U{9919u8B*`Hvpb+LfG
z+yDt_8g{oT<E&->Z9`nast}xLfL8NdiH?*rBwZpX>J&Tc{S?;H7JPd`qfVg2q}y5|
zUv%QhdQ2)I>HWP66pQE0_2f!e7i_1NuLPHyAFa!k>u2t!uQCzrSgAsuqK@B3`VC*A
z-HV5i`j8g(nuz87E;eJ@`n{pIeLkQz7Crby&}iL*lxhQm+Q!wp&JoLki;`W@-A3D1
z0=TQSr%%P9oY)LcAx}eJ87dE`+l!=x^6w<`Drq)4?=+W8b*&QR;?w4a@f%fJ?-6gG
z2MM5d%cn#P2g>n;Z#b-s+Ifq+Ixca7BRI;B1@L4vy8N}$I95CuaCu^^)nj+gY-pC}
zA8d{`hqf<rckQ3~e*5&Rd3HPZQ`K%hlaxYx72I2zv+PSDCX?h4e-(4-=@ISfrj0A#
z&jb#(v8yI@OXA+M3biEDJW!f$TDdQ-jvL`^>kRNN{u=CUY8Tlb%<<2>DPu58<|)h=
z>#ZFc!=8^=b5nWHyg+ycKh{E27YG$k3fZ9>r-y|I<$Sosdx2NQ6u@<e$M%71&8uuP
zU~-^;Gtx|-6~FhrK%9KUWeoJb))4hA_A5U)rrQD9i*x?gTgby)t$ZBG<Vk+&=~Uos
z<Edi3eT&7pxftJOdF<cROPPuu=8fLNo`(iaI=N^yFsG*g*v(aOpU`31W4hg5quGEz
zunbzW_^+u$as2mY$$TxHU+YM9QGSYG0nPBH2{X(*Iik`$th$XNFN)YQS{~J8CA6&u
zu{!&~<YnJ5VtLaB22wx)cn`s6o+qZ!ma&YoTfw#G#-8gw{JbNnStIk_RPmFv(*j;9
zR+8j<hx|6Lj|UMEbzy&G9ID?KEsiKgoN4j|jeVzL8x}1O-}z3(g1Np$n_v7&JaC|{
zj9%cC9XM-Qj-BK_uWFN_g=TsI3y|X3!*EzW8JfD;ctP0}D&dTudYWI^qJb8|SE(Ar
zsHfNFN)_kV5=Y`QPt=5OlVc)AJs=$_;TTV%GJjhO9;E)v#TH{9sDX&F;6|p>R7IR>
zcHlK7^F#Ix|5-{l7RM65!+GoeJE~pP>ZXybP=v2lqgN`FrepWe(53Vlk3d&C#ePQ{
zt#}GM@xJ$Fw*yNTL<`q9F>&{S;}r$lmHIJGzk(L0AAY+f5VLINntM&l$Gh3T?D>7X
z`2B?NFP5wBROfn(xw7ncT0A9_m#W!w_ib@~xG)j?vKBk7#woZc$@z@bq@!k7v)4UX
ziqFaLrTy}12x-ksUH!$8_trmYcakbHsO|lZcoXC!r60-p!`$ZDrI)m6#v#6{J+1A^
zCVh8cgc-HyS!QbP%AZ3NJnGE0L6<n?8q25nUoaKZ`*fCO0Zd85A-5bMJm0`$@#Xt!
zQv#=3^h|{_O^(rb2AfIKSS06T=tY{LAK?L;2a+0}hP|#z4|&pW#+Yvwf|PiN6dGXG
zvBqo1pGmu9rI3|7nc9dEXJG^US$riHT7*3s5{gSlCs1`Hr?<*tVKWXimF|~#&Q$!Z
z9_cr+*01?gn?r))*?3UF2>~8t)8(R0h^^bz*2y*6&R14^9>ZfE_Cq)@?dz4nY6q?k
zO(#*9c%BPbB#jQf(vyF~oYJGtO%Uhp8t~2Z>4Da?hf-~Sqhyvtu|F6r>Bt82h3ni>
z0rAT|0!D!g^zDa>=m&c+u8a@jjlsD(#3?lmKLZaXrZZLHfu38@`gs$DUXdTuwb^Az
z03J7kBln(Wg&c!Uk}22R$SvwGjy*;Mx+WJD*pYYpy3HcPzt45J-j4BlrNz#*nKmu-
zbl$q9AiM{TT0#srN^i#w*U^4ocS3tRzM#2PDI88-zz%5$E;Z*p0a`9COt&X4E#T2d
zA5}EUMc*m{+@%rq3F-_Ag%hnXu|;aLjNe}YG3!|JT?U5tf@MtL1O#b=jdSAnJ|)oi
zVbr{Qh$}jFzku+wG~Dch=<ZK%0v?FW`enoX`Z=PdiHJvFRup=cs9K|lu-!g$F(Loa
zfHOmkoK>0e!))%b`?StQ+1{M<C0>==|2WjWH<z&)nYU;-`V~lcn_Q)J+pn^2c1i~h
zlkw1fUNIrGfR{%Q-j#&gZn3wCtF+y`3`gf%$EG#;1x}$lH8l|YH2mseRl!cz!G=#d
zC#UKK;8!>FWB&QD-HZTD)Am}fA*GLA+3V=?X8z)yfSUC`I{erV=Fu;e>lG7e%?$Uj
z=RHB6pK^biKrFktwrrhFk6BXHC)G-#*$|5RJDsz1+wqw%B^@l}8@k8nU$Z2*@F~5A
z&5{|l=;=midm?b<H0rKN$~y^8o)|0rao#A74P<wZ@uUr{YUEq!B=^}{LGu|drIXoE
z&$WbxF5=ExN&)>JWmX@(9XAC@W*gN8{c|b~a4M-4uS1_u!y7Kz$|v08^2ivh$SrDz
z?t*i0NUjGF?1L2!;&WVS|2Q4I{dE~lghKN9BRph3X_;qHymzRbpLND^{hcCbv^WP`
zM4hsMg^*>z46{Y1$a+_Y-p^BHCZB@ELrb2YgV_aP;Bcr8%2xiSb`k}uqfhKE;=z-&
z_}jl(spPEmpm*l@LoaIWR=iO9mgETOipiS7bNq;06dM;b3J>QBBM(kNZHp;T^4-DQ
z-eV8b#23H<6iB8vO3L;S0=T}OnyTo!ccwb@JST#wu*@XbCgmL}1Xk`ELh;VYY}9lu
z>~0(*i_#TXelVs9;C16Zx>PT2c~TRXv#(v#G<91v&2S`H+TEy%`3*K{WUbajxG*1E
zRZ?lWBI%F8N7YuTFLZ1QV2Azb*l%)_a%D5bI%!d#>m~{!@dK4e>;jT*%DZ&YXZ93y
zJjVuU!4nSvd<saoNVL1X<!{=NhSB%H06ZH1aE$MzevL<&Y1~w1aHIxiszNIX>!*t(
z?eDLaOp>!wU_jTi*<f;m{TDoJ(6iFC7;LVbI&42)$OhAXJnQwx=I1<`%biur$<eQY
zqGk5j>=fb=qK>H-JC^~|+ydlM+bPEsT9iIo8`bLINQriX%xsgRcfcAh<14piKYCxT
zx&yr?9r1Pazx-NJ@AhzzQ82Pv4258VIXU0~{N8~n*Jyf=aRZh@c-?Yc7yW<_9&j<Y
z{f`Oi&?md3^S#(e@!8nst5`GsFsnteQKz^UV7BGu)@kD?;`Ylb9%$f&J6dULGECI;
z+p4T<;=1_#au+RjA01pvvH~gbQNTY=K&YB`=ZQCj{^OWd?!oa?n?dnRM_X^@k6jXl
zc*oog%wQp0mT!`Mg^KRK?a@|UMkJU53G1EnmnF%*ZVkV6S-yB4ANr&Y1KgFHiLgso
z1G~v?bg%(<t2c_7C}e!%p_7~0eBRlKBun*%Yv^IkqoXXesV!Dy&C)vfEyqWH33|3*
zg*8NfI;*{qmg%o6F}hr!T}qz725Ot#E_~C21f)4NbxNJFlZ_o-(0?KtI%wg8ZJ$cU
zu=%>tf)!@qxtcac`P>7>LclF!_4Bm`Jxse#@?x+j-3SV;Y76m;VvS*QubAMyi`edJ
zBM3nL1+evea|E^sfoDxVfSY$diVbb7Wdl;inS{!yFCu<ssxH@_Nv`gkw44Zmdq3Py
zsETrBV|EF2&51g-0|lg+>(GH{cs%w>H|+T&8qni{<|!!rDOYtYywNBz%x>{|&>GK8
z9yXwWH<HrAIl}Tf)yX=xGK3Mp)jQt9W_2_(mS@&P0lx7xv7ndENV%8T)n2Oy^^_lg
zF(5y?o8bjw8kKZBeXys^t<Z&Pw{Ec&*U8+{=A|Ywz$iVN|F>_e<%DZTl2jYt=Gl{`
zH_y8FqCRej|Mg>F4VZLFTF^5E>g}6n<G9dIIz9}?!M=Z*XLI^;Z$93pp5bY>jd?0E
zkd|o6`M3XkW2mv&oXaIhbi(e!wsp;JWpvz{Y$Ar#$~7E@%+!SFYE2`J<xjiity^eY
zmA)ekbLqY?Z|g3m8|;DYkC1^2c6C9=?_LL|R-tT>xZ~9NCPOV%vh9Psjb>lCLaWL>
zmIxNCtZd9=Da#uufvY3t3$$*;v||ce4)@yXfobRHiv4F6O&|XH#lb=l1W@fS7P_;#
zXLa#<Zg{-?0rDFm005xSuEL+oz<<rS-A_ecQ0avBw7tvvit;V+tKR#5?7k>tbQ3!1
z>cSYQH`Sly=PskhL=~~$PD!D>Frh0A<g6~l#r}LLl<xN6km7OJFye6Nb-**?Fnm9_
zt8%#np8ia9OFtbPG~3`?=6RFzr5;z@TWCHvCXoR-0zih37Oq0IIri$X&pJpya@4y%
zNey3Lu*a92=Xw^va%ZwHKepK#XTNr2K<B1G=w@N)!?UxM?U-V-$rv~Kwo$qjmZld{
zzJ$?at)0Ada_)gzVx+e0P&KhVId>z_ayW1@T5qm+GdHjv#3E3-nn_8qZSUYUdp^Lq
zJJTtkDVcj~Q{&B)cpNFS&8b}^;M}HM>Jv`v^7Hs+LCEL+t+eTIG?zyhF8c_kqdDWW
zp!#AXW0RgK`b(xNJ$JDaVuujMfmoNtg6KPkX;Q-3`e-ZN7P;@ZOw=Q-95QmW>g~tQ
zBdSGjefF2=FI}wOaPiyM84=fEyb3QCx>sauIDTW!T4||W=hkxiT!Z}_t|7ya$L%Cm
zP(zo^<BW*aa>bRTd{$Z@XYgEY>k#qbT20rxGQZ?uP5QoR&`Iq>)u_PFvH50;RQGAk
zEhC-HkZ7e5LCU?mFVm()X17_m+_@QN*Pa$L4Q{K;ahT$AQS(Ux4ATP2I$;8g<$`TC
zhv}8J{T{5Azv1tj{;Y&4{+Y7uEUSSMXtt$#La*%$9hXZ_Y)4ctl){$L(~V9^XwVUB
z;)9E|b65A5D#iSTrW9(+q2PxUw)oBYOd{4)>{3L7S?h}1$|)7=4%rNs=<;2;IdgZJ
zuVrHM?f2)`JFMS2Rm)QkMHLE&O9e?Y#^mQ?jP^`1X;H$FTD|RA6HxUpag`wJ^x@-t
zcyn)Ahi%Jq-~4xMFZbXkwYvfC78-e7J{&<7R~C_%-8jI~L+aP50ONCdHIFE9G>G1@
z+X$safh-)&BqCyy!(jK4uQ*YzJiA<=20s2Gyzl8>h+m%QZYlS|DO9Y>dq_(J)bn|<
z{K&k-dpdoNQGl-0`!U?&PXA?R7{C`I_jkn7yNixWJWQ3O+ps7*TCMh>UQ$#dY-{ye
zo{=_H$W_Ay@7x#nF2JMh%GGQ!NZkJ0Ft;oUF0V%IU4;k+2{)}<_y~<WJ&e;Y1gflq
zsoOLfV`#e1z;JrqjzjPAG0n#G^1M7*=6mId_jH00f#7UnJx-6zy`@SW7rD_)!R_jw
zyb>H~*0k$3WqS!t`$XQ6-bH>5(D1gi>#7Ge#o)|P`c@^9K_4eG9JH`uhiYt*AxE>t
z*D)mk_tgl@wn}#4!NRN3?9;W#ajBHT==X>VA(LsI?grbu9?itbOk%_dw{t=`FMt3H
z(x{Gb{<lPN<&%9YI$z0`PCBW1Jhvj&gtp(kMM6zineZmMZ;`IKxfyT9K|o1;eSMNn
zE<xwD=p^SK_`R972^QEcy|vFpv-zzX;R3&g{&%fTwoS8V&&Po`>ITyg@o(=M{-H<5
ztNg8T%yU;o#^xPGoXJ!9@?CEfV1$gt<^$|&Vb~wYu8T-r!;&Xv_St|iV)16GtYc?*
z<gm3v&wyk;UX%)ITmgn1eh(q=e9PHVtXN64A@1dmy)+iXAO}-+0sCjb=EKZ&lVfvh
zSejG&p@c_W>~{|9pK2t`nvI`!ZZ%`{b0!^+Y_Y3g)RfC#xd^H-<)%VI_<|2+<W#dO
z226eO_<n7;jz*cGc8x;c=$T@C^Qvvyfa#DCG3J&!zql!$jrP1|Gi7--l7Dh3J=EC$
zJ-jW@@|nK<W@hC_7H5?68v#nSg+!LyKaLM(9G2{@=cuqO`mpdLrQt#uW&+|I`$g|3
z;B>TlYcLlKvU<yxOVvoOO>gSP(E#z*Q5feW7-2JK-<K_E&>hFwDzErh>edV~@6aTX
zpL-B29Rn;YA#rwhxhc<!*kSw=eEFH%dF#Y6lY)|zw%2<)47!au6Ygb+DQcM^Myzl9
z7YtffFOUe_$SO~CZYg$~uz>052I>4z9Mol3J?i4y9yU0AY)u|0Y9fz*Gv3e!!P5Id
z)rHaKFm1(i3x}B-;_=D=hV-i5o6pd(yY)?@*4qnLYp=`3gM7etSCUod(Y`)>flP-A
zV`~haM5R0jG)&*%OF(8}d6S@f^X0PQ%>BFXue3sKp|$40cej_`0yq!_WycJH2TDLG
z|Lrd4j)Ko%+)O71$DDS>2MV>Xm@*wla>RF+kThiG?1_uprsMnaWoX1Qr%j~TPOB@~
ztAV{=0#(TyWqNf3hgvSQ(S@ryGuLLj8YO#qi8#|9I+seoT9?v>fFuO|1Nk$Uaw1=X
z+1@Dw6o}QI)oxI~%s;g6Hm1Xpw=>P=$oWx~D|V?Z>8K1hTzvuJCsMNvcrWh%;7p3P
z?z?A_?g=kq2zj+=a(X5k1emu?2lb0jTMOz7^&ED*dU7fPC;(Rht1LFTFFDT9qg0TN
z=8g9_4tJ00mEEevA<{c?qs$2N8z&x)f1Tx@A1u73oOJPTenr;G&9j47(P?#g5m>Ow
z|C}FImkZYVc@ALf_h$Sx3^o!g%%fF6fHqkXT!0`}4?1LFa(J0+-{O*(qRwq}-bxhC
zLIV#pe`y!<MOM$Lv+KSTh0za$hmP-xx1(p_0@fc*lEB@HP_H;G%ELTgsiO;o1<IO2
z=2@;#{Cj{pQ8^??ZzaL>H(Vgn_{;75<Ikf0FFy-{CHKIL2mg2H>SPb!Yz-eZV6vg6
z02mJkm_3O&3Nq~^gdn_zbM(h%Pp)3-`rz9xBLm)B-P#yjHB{Br7eKP=19`IAeFmrN
zY`$|J13gv0PJv6%APT~rx)1c4b>9!aY?ZOOv3Yfag_`6l++U0BO3}ZVu|7=;I`s1T
z{1)Bbe)N^%-@mB2fpd`8tnGVdt98rUogOZGQFi@YY_NBB_yUJk_ax9%#XV&@&yDRV
zn_K+)*-qGb9SgApA(w;5Rn(1X>qW-;lc)s7&Q}P4r%~s-&7N{qI-$kuHLLDxhIJbP
zqS|-Q>Zj*VI>lQQ1{>lZnEGpU{pTx}Cr)`K4~+@jTCP(E(SU_tzVyx%-<a&L_=QTl
z2<=<lYkPWTW^pb%pK-dz%g`3y2^!TYU&#lwnbbFTvh`SfouwP=S~uH40=-%e<0drs
z9n29Ksy8JJs57oeu8->4b*gM{N9-6PAuuZIWZ{E3K*h;@eUaVmuC=Eg3>k&-Db(P)
zeh7wI4M!qzIy5x&!P!f%&Ka1``B>eWSA{4NRkjCI3+NlA224z?nv}PVd8_;A@$MSA
zttYlBN;x$agXzqi;=Nm{kVhAv)}#tiDgP&-*B-U|F=1R2$b1`Tt3^qZYmxYK?^1sr
zq#t5<sT|r9x?x|JPF^E7RL^ll=65jyqzBhAo;NoPRDd^1aDWaR5#}Lr4IDM6L5C>=
z(tW&K@Q&+<^8Db4!u(LW+n<IpgN@+F6~2B0;I&<6%6L7zpEQS-Q`(GmT8CX$TQaPy
z;WjR2aq=ssdg7$%9VIP^ILHj7X?{pp%};g6b4(PNx?3~*{um2#xNmuR{->QC9nXKo
zy2ZB;%#?6v>DPMLusq9qEyQrG%2}>!7CR^j=uh{J4mAyYcT3+@*s)ik-+nvBRg7iL
zQ2G87b481`^X<7R?*^)P-R;t{qRWH)N6P-0&zM>zVjuQqu8Q%Vf1Xr5=^hs9(asVI
z1(+!}9+MK}Z}g67@gq8)_B{jTnp>h9!NiFbd}CM@lE}Q%NHbnIO>{}N<~99_Wtvo{
z<2PMpkuw6=RcL1MbX55TBj^KdID2OVy!P+HBkDXX6ZG+AJStED;vR;chIQ#(1)n31
zD(vSlAVXcNS9F?y1FL^u$-%nPG39RQF!)iQGJeRy-z+-mPsf%(ZJEvejWAE$!h3`3
zZp=XR!7xugn;&WDMf02HZrMFZZ@*Tw+#=K9{f6ymRWee3#UJtc(ZS3PSBSyl0vI1%
z1K~eF;8`i2d_zQEXYYPKb@e`_UMruTi@sLKe(s>l!+ep)2E>llY*1MTo(AS<sUqm)
zAyBe$b;<q?)nMK;*>be-kk=#SfO}p8&GNTyt5&m&oWLfUo4qL6XLr9-W-`M2cP<)D
zTb3;j2Jx7k5<#JWuy)8RA&$bfgMz=82XFauY)Wa)WyDQM295Dx$ap-}A0CYK64cuT
zDAfI@ECb5ztCIHXo1DDv7w^pQftji1zTPrT#}$6wai+Q*y5(V=l5Ce0jyHpMbor(L
z`QF1^3r#;C_F#$o;-d0qP1Ky+!ynSpGnag*`@*O?b0^#nh`nm5Px->RvOvda^4S78
zU8ziF+DENCwtBI4(Eo)hegx`8V$GvKG?Mt_)gbcA)Q*E`Z+z-FEgNo@v~q?|<=@Ay
z%}Uj%nAw(dBLfK&?bRC_;x%groajHx1RPf~9LqG-E*@&NNQuT2`j_oXok$P3(u+?5
zA3`yXskn4aQakY-i9J+pUTp<l+&D(1sW4-O0PP>#!^8DzzWVXl4C)*;_1vlDe*AIt
zdD@Pj5C*bq`0b#MD{6n&XNq$_ylsYqAxQ_AI9Sjw^C)nixh@s59y93{NnoG+H2<47
zT?w|mAJQ4`i&;0iOI|^pxgR5f!NQ6G-be3+b(7pnjGYX5hBQtsRLH<#y~qw`?{hgq
zwIir9_E96&2a_dPO<ceqUg*iQ3&QG7--D%ffp~3v)KpbRU96<yQFVrop=dwwV2oH8
zhbb~B12GJY32v2{t~dPTYXk)MPGs8~VG}S6-g&N65{3nT3#|`a;2%H&LKrZ$?pr0t
zZoWcqGtb$0<4S4Jk&;K5BNZut3UUIpFxpeHz_4`(yh4P*{tz?IdctrW;9Xqz8losN
zsyqFHdnr5d!YJ`D3l2Fw92AOx>I$7XZ+h7$R5C>@6Wcv57{fOCa~*?CTwJ`yn+OI_
zDosPBMwyZ%7hdp!cGtPXTq3*}&^W%Ps(FlIipjSj6STlWMn5~a(=Rel89{r#%{UjC
zH?@(LNrD0Y<-w;#x?lz=2AGPQaV8VvRXJYw>tuTRaLyhH-8V$7I>iDPF)f){>?D&z
zyf({)7ey~8OI9zKjtJnI?wevHL%k^MRAJi%v%-eic_$+&z!xwZ2<EQ6`O!-X>o;=q
zbrN~WCBPQ6UCBbAL-Vuf<i<ISY9$X39B^9lh>ghFv<qd@A1Y6bTf69iinOfbis21b
zZ})dks%oh4f2wNUBfP7-4Vh{PXRl#-|D6gDOuj1v0qw%OfioOZ56zaj>F;n%f?K^k
zIMxbjU-kK{MO9$|%54G#35X<)O!6x+R#;%kdBq9wPuEWx=|XzOxGxsUM3%&HL%Ib+
z-=kgtoiCju4)MIQ_n;%OVO0;O8_`OzfG&jE>1D~lcZ50h4DF+8(I~aI*m^bkh0e1t
z%YpjRSuw?y^=#g^51S-lHU`5bB@VOlOeIee_Vsh)e;Ov}mE7vNX_R>6DBi<w4^u>^
z2uj_08-?(0i;eds|3MeX1t8JTPDk3RnE%|1&+9D_!Kp_6=umUSUG?$9^;%tiYAgUD
zyAiys`7K$SG$ZdaK{*b7&!=5!1QIa!2`547a`8<~?0~#08LNTeEzC*xS%YLU(1AM6
z)Svu2w;xU^JCm^L%?Ca(qmprSU!E6j0^WP?TzHAQ+{tR>UwUZ>zT|pqjYEzChz-xn
zKW#$d|FVfGy3;TaL6tfz8^6Qf|ISuP`vV?W2oUsI_j1o<<PB0=axz-WW$~Hkt6kg(
zn@%?G4k5OCMGphvKvegD{Pq}Kqm}36PAdQsmgJq>d*qaeG)dmVwYt;yFVOM90?y|n
zx;k3Pv+tba_EVbY5P@3~eOWlsk!zf6JPjq_6MMsh(Bs-5|2H?rV7Tw<Me+#ewDX!P
z*#nRIPj+2)YqVrt&+JZMC(bu+EH+n4=Sr#Y>J59QUB7t29Pbb9_LA)AHFqEZj=x6f
zhSgTSR!Hk95+xac$9PnOPvu~ALsu4&V7zzAo12Q1*mq~<y?*sJAKRG2fP0CdRm-4E
zc8r=#_e@+z6K;7J>3=a<tiJts;G`kgSSFbCMtv;}`2t~R2B+h|g^ce!%(Wz2xXYvl
zYo<hG)#O6j=2LvJZC{c`MKZ5bDw~{~z->syR(b6%leDR47a@fCO~YZpbSQ)f{+}V^
zP(gT|M<+0qb<|}JJ{^C^is$<UwLa!L3AWIq6w73jbJ4EDBMfhoB**)gTdT|eMC|FQ
z+kUIpQYU<EIg-7|n)JSECI=<?V`f#z%4<N-$FjD0ily_#+T`T|!=;l|5!aEZXGo<{
zI^<P%D*ce;2>R5L@$)*>d{f1Si~TsRjPzFB((+QpI)CTVAZ}^`7|3$BbbQP&i;bW4
zH+NS39w*|LSnft;T1gYCmWVsa16hJi%NGf{8E9G$;+?#b57DUonSKGr<KGX@AK)rZ
zxnR6+B<i-xe3nkFjMm2=NbgCkm6w&1gS~EU_-?p32X`Ju;UA;pC9LCJ(Dk_>P)`Hf
z%#3a`qu^|_n8~-fpC68ttWz<ftX!z2bf9wS-#Z5)N*SwVy~AvXV<k3lp@C(^`7GHR
zzW!79aI3RJ9$A|vmETVF<mUb^Qw7pXJ_?_!zs6n|Cfi`A_QDTdgjEDud(}=l#_-&<
zN7T0?O)mOm2#K^XXs-GzaJswxC7xWP10<{R+q^Qrb>I63mlcxFga`P-M^XJtH}<0Y
z5vC0sXdJKf$NYx1N29vek2*Tt>r@3A%Xqy()Obh>M0l|^h8)@sX{?nP7EyYAT-hZ{
z{l(uyoXyugBQj0OybCai9&!3bTwblhXOrDt++{9W{v{CCZXE7YcMV=fa)ybpd$)Gx
zFfztBlqD7nNp+uZW$DGnrB1s)T}8GT&}w1B6E@<^{7snON5<zfU3*B7+w-Xf`8F0m
z1^>Xj%tiVai<Pc(5jH1LtU4rsIFQ(J3hTE*8%P>~IAaz8!V`Gavo54tdta|H(wbpl
zo~>Q@YWV{2_BymQ-|zK2+5ScarK3SMHGTd))u&c7qKQWu2Edv=k9!uk`N}H8c_rPa
zI>NQuMXOMKm3p@}VShV&3b4|sQy}xJf1|oX+=9wH=G!}4il<3w!+iVYI$l6uiJhT~
zlb;y}F4FgCjoyph%)`^$B>;R25Te|SC*UX(10e#lMy_I$`^p+s)e3(!(mh&$;q_hA
z>whZ$noQZc0WG!VN2$9x**>*=e!7}=GW^g>ev#xl?jeAjk@w3XrjM+k+vhmvdGoTK
z0{dB_vX~~#Nt~Q@TykFKFAwLp>03j+GQ>!ju7?fh$B5a3pF;r1jO&N}cYLMOw2OV5
z(`g|0z(3a{8?*hL#|El>G$(VdzqUKYfH-Vmd<!Ct>Xjco-37M;Z#{Z{#R_c^K1&(M
z=RJ-+Xp{ZCG%38kN~Ll2%ftS@)8(5^@I9=iT0tRC4Z-CXnSCC`@Vn${dEU~Bh551{
zk171d#~B*D^)R(24x`8l<gKr2YD%LB@&fPuIkqBlfp(|o?nV;~&tz(4+SwXQx5ex&
z#}S7p?vb(lFAWys>CYHjxsiOG8?eP|csZjn^^zDIdYTKlu8O9aw1KmdCH3H_3V0|K
zk<;4mx1nwNlccG#{#(-Ite6RsJt4dZ!mkM*<YNBe;4FFC_+QZ7OCjT8Y`FYbZZ@mS
zMc2yUNTZOOgT)_O%!*z`FHOX01kxac{;A1zpJ04Pos7m-9ZPipeaed8krAxx*BaqV
z*+y2E1Z$2h<3DuTtfEtQmg&0XCZ~<2Ua7VRJ}@oc=iuY}MK8je9){_-rwW6VT34es
zS&s~yxW{wM6lS;x4PnB_9v!V(X$7i0@|RKQ&smuT-q=cs&}Esbr(D=Dn~OEyHE+<Y
zcNapEOz222_*sY`PDuK{B5ognYP|>qwjI4lhD`y43qjaMcPX#`*>1Ev-_QXG9WYjM
z#lVQ%^&ZSc$jfA?2tbxVL|-r*BU%dWb!>A<2x$%kzZz$fTrzRD?D?Jq?6~M&{1?K%
z_LF5nJhreS7M3c_Bt+h$yN%5H_6oTt?(J0c@3g#UXhE_?>Or+?+>Ca-=d?e5$|Jq2
z>e<S2+vUKHeA}3xhPs%uLN1COX2fegIEaU#v&(Ly?3`omkw$(oOmkIrw13I7xJtv0
z(}#cST^YITif_mRj@FHg@&rICgmOjv_%U20d5rJbE_PZxbBkZNHrDB|KBm?m7fJAw
z`<NG@&gTyp{5~lC<6b-4)k(wKnVsYuxv1qlgK7EWqfV~Y{*T+vuz4SSqk(T028@H6
zvh6pys4?STO{#<j9I=?28)Owt`6HY1@cE#`=DbD8x_l;MjyE#mfWFyH5Xm$D#EPWq
z9~%d%H9;F3HP+)8f`fM6&p_>ZWKoWm3$>L*Q6So<v<Z>gU|tHcGPsYd%utK@OBe`o
znva`?01mT6DF+G(PZpa__1diBL7r6Q))1y+{^hW$Qw=jS)m+`B{C+#I5w%sXr@j8B
z&qGwOBQKV@;cEjme=9|3um1A=U5Fypnolg#xM^;&kcNfRj*%P8joVPTdv>oM^sbmh
zP1{w)ji<zvVkPY5ZXr4B$}d98Uxwe_AprdA=$`eVav0QAn<})Wvb+?YQ$AJV$*YDN
z*#iMgW)>TS^g{dxdjAr;2QM|xc!w$vmc6rpGO18$&SepOe@SxGM=+{kOj9_H3rHHA
z?!O0RgLu;OOekN6^wJl<T1DuS1fqP)lpddX!h%}?L&{C7eY<Tje+jWj0>n#Uis4SP
zJko;WM|t}^A?)CHUGEb&4UofJL_TD;RG!iNK|<sci*Lah4OR+bS*P_VLIa#?wJF79
z&`OteT?$g+0p6<?3-)M%dFMCz_%Cw*tYAzHZ`&U7Z~dut=Y*AeyTcES>9d$H%Yp@l
zU$3~rZG5y9K<ApGZWlOLI~<-SW^Cr|T(9Ek8Cm%+((i#U)>6t<IJ9zawpED<xAwMt
zgoDsn`~x}ua}ZY9X7~R0E0m=g#gcfetmuoqyZzKQ8%pdk1wtEz9>3y^S_?DxuSxPk
z)La~cS7n$5(Awg2VCEJfQ?7jKi<hNL0@uT5w<dXi>8;_z@Sqb-MEum5{Y8-Lz$Dqr
zb=b3Ws~NE1Zm$UF9ycAx*?eY>lhG?+xOa^G0x0JnS8M9>$9Y!YpV!isYU=Lu#NmE8
zybR5y0ZSw%kEm#YRuBO~r0igmBnPXn-ihJ<r(6dZBxaV|2uUn*8RURn0V%Vhd#kjm
zZ@Zi>%}dEtl{o^%T%&oyTvJUFO}d<$dP>e`>1_fud3@`G%|N2g-ygOO=Q%LclDjz6
zJgL^6%FqMox|aL1PG0ldr~;sUxU1ZSG)t-7n|YG7n98JBx%xu;2h~uyx%fYhc1V{Q
zFCL_ET|hZw+4?>EujS2R0W;(4LupJqKhZUg!%Th}F}-!x(F3^#Uk#ao<CapTUxP@u
zQ|u(MKUSNZ6utUn05tD2!j@lvu1~*qGK-}_+eL3=r+W@*5BO2X8gw}d1sF)8Ed%c_
zw6Dk#n|+k{%FXW%jT}HK$KI;7c>`ByJn)}=zHHHEc!L}vd_eQ)OP*mp(o!2s>{XON
z-lYeK68L0Y?kIp1*eRIyWK2997k8%4HN@jN74MG^${tnPcAKHt4L>M`&8PuhwbJ6n
z=vTU`nX{iv=5n-?ctPueNW$&Dj2leIUkGQMv?LfjR+=Pb8v1osJtdN`dWR<XFN&|U
zDKStWM?k`#T(z(NqpNP<8rnkwn()&_222Z52TtF=m<XBbME$t>C$;pn<-odVF(9t>
z!ro2PDTz|d6jmR^vqh)(xhXn%F$<?w*}c-8-s!muiKOKhY_=$UC_pqG9<R|dylbCd
zuSk2@?u=I|Maw_7GH3d>dkmyV*6g1+>y^jU!{R`e4Fo93f#=2NIgHu?iA)wA54};&
zMJEOAY$N}0L^?8Ma)I0(jb0IY^{}!vTTfEwl$j~)N@_=WZ$LKnQ*B2#y~SE5dMkbp
z1-QMJNwXa^SEv$TY(otHFYmZS5eG6??LAN0D*PVIJSP{DO`YKZtG`-co^Xj)U)bTL
zc$$b7E|gb&@&2dnsYcnHj|a>kF}LnUh_{%;XD+X=xKKSwhr|c=R0kR+;TgaRgd)7U
z!FEhrBmtd_VW)_AXNCrc4eN|!9?y)OW;=W^dRcl%iZ3|>Lz<-ab&d?9LkPq6*63Ju
z9xf1{UI^gRj6X;PYDLnMwZqZ>x0=$%ZoeZ-KIAs%E@)XoLhJbxWL7^%segNuj?jcs
zm&)3EGHEmHQm87wA_PqdW{SWw@-SB-Nthgb$u6=H>VrB;ya*wkfrigT6qo9=F<YHY
z8XK$8IjvJQ8j<p1D+W7m4txn>%k9X(Fz~?Kg8vK!kwFuPGXC5&UPyf<Pe+>3%g9L5
z;pQC~RkaFCeFUe@40sJ-8rx!!#1q6ST2Z1`EA9dBZZK-xk;a3z4H99G#SyQiw{bC`
zq!Ir~`Z33Uk~Bg)h)*1QTsu!{8Y9$^f<r68BBvw<8E+0>*JG*sb}c-Ixuyr{+F0?|
zUjxUrA0CJ0LRQMCmQbv#i)9ddfF(U%51;wcp>Xfzn`F7#b~=Q|M7dA5MsG;y_F~sa
zZL)wm{LDJLjqkqB;Z$|oZF=>~;#aJSkA!?gMt1h&@r}~6yXDo4S~dDXA?CA6Hn#qL
zw5!{jX;d@Sdtjxg;xEt8S9O|estWk#vRgU;_V&#WeXtPX4Q2$O#Zvy4#fAc(SjysA
za|<)rEcQ;T<`COsP*RWSbo`5SeBukR>^a^&Kr>ypqZD@=Ozm~Qu|8&+e_Q#^RIXir
zIxcSaNDjW%R#~Mk)V_7DQ98n5K>+?*-`hpY>~ctfOemUAsV!Ku_!4$igQ+@VU&R<Q
z7ykLY8IOqCx4DNbpOe$_HCTRMBpyrW^I3Ysb`w613Jyfz6Ej}vQjj3Yoo0ooU2B!0
zH~9;V`TbLU_hvN)P{knf|7`x-qs?zeQj7oenQwlNaql4<e$`1S#}8vFJnucn04<7y
z{>>d3sT?Ow+(>V_=#}HZ_E8@a*T*HRf(d$-z;F*%<MeHOUs>Q$mSeyA0s9qo@axB^
zJDOD5{GxAziQVC`7~zN@ws_1*Y?nylzJg8|9FYXZ2;u*Oy%F(FR_6o@fy-ErP8#m{
zqxL6KgXR!{{8Zk6zy54E3)0P=5>M%j@c&U{TB{2(0qY@0|L7s$e==Lb5zEwZp`bFl
zJS_lt_dl7J|197?68Qfo0T)Pu0rrQe_D5qYNBbr!nb$t>ft(v;lD?cWXkde-IpBE)
zfYdH#jAc&+1E%BG>IsF!uY_SD#8%w29*f?#;9pkWee3x^ehdDpawLKdJ^)e-OU|93
z9z~9&Px(A2zpoN(Yl;x|gjc`p=?VHV<_91MyYrjHjfq1-A?VNAz!>@%GV~YIsWFqf
zLxRD}+6e@7AK+(@;J3{y89=A?`Z*cQ)cvuYVaXXe_;AvZ#N2A|=K6je<&Re*sKI;Z
z_xYK!tb)sVj!!L)f(^q7D*}EO2lcV1s!Iam<l~t_Krevw5y<IrFs1tp-h4*?i2m_v
z4l;Oe^OFpX!Fup=*EjHERqr67!VziI1EP<O7{^%pi)3Upzoalh5V(Duq9H{D-kg)*
zPU`WhxhQz=B>qo$Y>eP#I|IX<|7`m|Z~H$={{Lm$L623}|L4CiAjv{bFcO&yx)X9=
zZ8817joQ&KOu<kK#!8WuNW~@7XxMhGrDme1{>U`YKdg|kpn^?O5r(k7*nkzh;+-=y
z5BS^mCwK%$yX*bGH>0>XJ$p1~J(Z{6iT;$W$M`z6{U<vAM*>93eyJP2_dWxS>*)h!
zZ<E(gk`E(s<oM~P@m=rxnM`^%(sO3q@cVe6P3+T~@q~km|FUWO%iH#aZpoF>j?}f0
zWf!S$MJ|0*(~EeUDj^$cu*G;ou|vyVM6vg;%u*=TmYjVi?1V_3pga0wRtP@RUqw$c
z-lOdIw>>VG)D-XG>xS0$x8*!y7k9NPq!vQL)}_n5|NTHXCcyjF#IHo}hqCwbLou<S
z<A-6|YoXE!)ED$=P0R2bj1;f}WX-DHSMHh0UiFC+`>d}rEpdYGR!^9W2f{7a!WY{l
zTkjCoN)%1jQ^wZUuV4{yB>nBGB=tq(^G=S}6ch?$TJn<k*}Ucc=TfN=ge%gSq~ke2
zjQ7xv0!eMNj+u5Bq#M|1qtB$2_}NS5G@)QG8!EB>z}d1EGd_&O7r#h0&si3&H(eUl
zd@UCdT;4*})OUQtja>g0h%3JQv;(;~8Ne*gvj1AR^Kyt!x<*3tLz+=oC~rCr;vIKt
z8dAQ~GD8&jr<U>B>V|&j(7NU$p#H)B3>85*?C<5!5OlFkp0n&heA8vu|IkFcNBqX4
zN+fzcW_+Z&-)=`^Qz|pdOrtHewVl|kv%ef{Gxb>iGC(igXaB-zs;7RMtjT%Hc=)YH
zgGG5bru0d=mrs9Q&A>Se+RHRoDXd-o0|#)R*V!S(kVU_qhjX8-Q(DCLy6M`#8M3tA
zf2gKxyeYtzB`{SGqYLIHI3}A}liS7QKRl-CMR-xBKv_%E&s5~l`neO6_nWj$y_jaW
zH&^fkHlM&<pJwHmAJ=-N1;=_HmJN+FixG`%yDZGnpDSP1hLQ2O2>$JfEc*_VSSPV6
z^C8yH?q5T=JReLN9v=K9<0)eCZ|vCL@~!(ch>2Z0&T*qa_Jn2r)ZhYiUkO)oydS*3
zIW|{m$5AzxiV(R~==k)wLhtV5(X8F;O0Mr#D^5Qh9(p#nEX?M3&Dh;t*<Wa}mUMgF
zT<xRJ_rr}`pavW5;Gk_erq;e2IY~k)Eslu*-!!WbiBNDHEtS`%@fjO2<q<hDH<%FG
zLZW&z?|tqm_77?4o9Z4IXmI&8UY?UOt(Udwgz<2e#}0Cxn!Y*pjn9#6Q|pXrc|k#>
zXP*qT)Asz;DDQi(wnR?VuiPu%XMdS%jSQ*L8;oCaDwd+fjXGs)P$b>uP#tc}C_z|l
z7rI^d`*Ck_Shj!iY4gw2ams@zOvJc6`Sq>GBGT*h@fkHmpv$zA4Mf#SiHfaDO`&Ca
zKFb%uB|4SD=_R(~i|!z`R#IZR$I^sP_>l?IEi2FSbX10iSDv(CFYl#&Wzu9%39KnH
z7IU)gBnjp_-8b2DQwpC~Uvo&?hB&4_)NmMv^EV%TctHYw(RYL%Ym{PdPO>Gf@|hRj
zh^K72*u49%>)cXTX7tjfD~(;Qwg_SVNB;n42K!ZLcoydw>Y!&@@?Bz#A|*5k9x3p!
z?M4O_$uaFTUHgNa(5)w(m&yws>O;FQ$=Q6IWzxC#zs;MEzVEKJ=DTYigWurz#M9<U
zmCCfM--RDyTynJrx37|+|7YCs8~%vqH*hqV`FPCvGGpX^vUkkA#SQj&h7hzuvG%T7
zHC$WGtDtaVpO^J-SRZtu$)V}4-s;DmswMr!{%q-+MMH~GRXQs6vT%)GULHQvY|N(a
zxUpQO##3G`ADGus<Wb_aeUjJu9ApMVGE4N&iYq$<&>wi9^yT^P9+M3e>z05%@5R{=
z{??{1-InfQY0PTa2pOBr4>|qsuIoQ}+(}TD{KhA49fMdggU<!;TaAKSHznVHM-h+u
zxFOppGQjLc=RHcb<JHWv<6M{`7cRx`3#X17(B_+}#o1@qkBfm$`LY+527j3Sg)3CO
z$+%ThxlUHTFTJo#j3-=iuoA_zi%|iG{BG#}ZV}~NuoSO(+;#Z<NQ>}rYk~GmhKTB=
z{JI<1@v6MBnDZV+|HBcRE^Hr~W=2aYOLG&khqj>kWxs^M{{6(m{@UWG*BKAPn=p2Y
zo7-@KvCNz9N-gB@Q>GRvURjNtDP+<KHtbL1(kcb7Noc%^zP&GDwZ*b!VC7Fkdvu7x
z$be&vrs|u#$?M5UvY0$LJ^3xd_FqKV4eAWUgx^!#!q~TXe%ayv40-)d7w6hz=I>{F
z#mI3=%-v<ZAvdjG&Ym)0t408ubZq1lQ%g1<^~LH2-(ChHS1|>Fq#eqZ)uvEkgl_o9
z^7&vn1#Ch8LZvl)lB-?_!jI6~)>qw3_1q^hC0A!++t8WM7CGyNZiYuhtku^oni(^c
zakL(ZjtSgCR%bma5{OvY$Ip2#j)bU&nccsh;}>ef-z|>G!~cZ+6GUP2p?lt9`!MK^
z%cozOb^F^CY_g|EyT{R5;I<U5?q{KrHnDcm-k-^`lL1M6z4@fSQ<Xg4g+%;uN7BJj
zWJ$File)mkb|aG|W(1q(k)&aNZik7FAmz;)MBx)IZC#eMPrNy?!GzsPM0P+}@?u5p
zLjp>63C3G}SC=t(Q3<Y_GM%$!5X?>8q%KmxYI^1M;2tjuWN_;!1;^JIUbuMo`z{8*
z_`GG!K1tkOCLC01_symSlfYa5RywtXB!d&9@M<aa!81$OV(S5B;aZnX&bPEDsUL;i
zhc(NdSxSK;=e_cQK+&IYv)~zsMKW5LijCm#+{<hg4)2-eP7I+)fXLAC_4gDF(P0%5
zg^-_}RcLtLSyiafS-OX1AulBpE^mS><wE@_OO#W!>$AIh1;jgLA;{Q+5-G5G^d5rh
z8v1fyQ1S;SRn7~T@;F%B(Xf7->})Ee{*6r^no9Ly_#i5A(&({k33I$2Fb<Czi;`|v
zTm6!Ua?pEp`yq%ldF+z#T7LoH;Pvt)o$ZEwBa{4-j1^Y=qsOp1-sC77*qqczKqz>0
z=sko~#V%B!fzNiYRrOx2a5hU?y*7+kkYPl~vn6fneLnsOUTSChL`gsc27ZnrAhS2B
zxFJWJClR5m-AG@=Z$RjH_2cH$ZLGZG?U!aF3c@8scB$pT<lJ=nDQx{69(#&LEMSYB
zjQ=QHx7QzGJwf%sxx-`XZ#B&`d1NiuxW#S9fLE^_KiyPVvwEqkd7(czA8G#~*_K$J
zC)%}Uo?LR88!NcPdS&jp*>mosUXQ3mAA^wRew7UI*_X3n2-7Su8_@>KNj}reL}KmB
z$WusvY%r_v5+di0Pu=geZw9Y*e-j?DK9*h;I3Fp>XM96ovCVWVSud3(V@hKZQ_hc$
zQgbqn>QFwmDajcvIkGO=Gj|?`)n@F2&Ip-;Um(HPPrn_i7e3M(iBt+$ueqQ=KD@e1
zC3!5wfnZ@Pux+d+aYN{DFp0fkrV@@j9zo6B<)N2F*Ii7-Zk#8xia5W97Fnn0YlaWs
zUtsINK`<U1Ty%;gSQtwcb&`rB?bt-WV;ydiwaPJZHy*S;Q-Fod-riC+ayD#lJ~zn2
z2Ky&OX<QWvPG*^YR3=Z@CCHM$a}HsC5lp86&rtU3m8AzWEYQPDe_Zl%mSY#2`F!r6
zcFf(eO(r+_jOVuIVwlR*nYq*G=a)hwM?`FDzCv7JtGUw=QK=(=RPJ?qNWRgzw4($W
zu_jt_9uYwfOeQ=F+;sKUYnP--W(IRg!1j4-#e6X#7MoY7c=^8Ure$=Yg;UZ1Wop1T
z9ZT=ma|ubiQTKOusnhG-oS3@<1~$QF#-=n!+O4iobe;m<Q`hi)nCm@-+m`}HkdujR
z_K8M}DlM-awecz%b?wmiQ$IwrVD$|z@mMcw{%B^PUAcNCxM9Q3ZyvUmeGa1d)q?TZ
zz#kL-$jf%fv5oRtcIT575U$U2#L!zesD(K^-w~p#8)DuRng5>Xm=Bqq#c<IzMAz4`
zUo0KFSjSAs5+}+2f0?TO9t_J*euJTo3Jg2Wv>?*Z9_3YCaBhXdMUO54K3l^(g!aut
zxj1|+p&Y(|d!ZYa!T;6XSw>aWb$$OFBt#k{rMpx@KpK%!5kwFHi9;Paq`SKj5ROU-
z5)uN3&O?W!ga@U&K~O;QS#Vv?b$j2>_xH<tkFf`XF<`H~_S~!If6m`8YTMoJx5gC~
z#MTKzYWz@^tiEF2LPezl5etrrZ}l*h)q&enZ{I5`3W#;5^aMRdeuoa{&IFMpep5Xd
zVrSU*{DviCCX@xOd*sQNdKc}}v~y;8%ypBnBn>*{T2jd(@Y5}$ER9a-1H{kuTFNNQ
za6Ew}S8+;UcsdfUl9-(4qtUs#H>=ao*zD7M+%sYwvCA98=djveLYPMaNrA2uC}eDI
z?GQJ0UioTHnS#EFT->R$=i~<8Mu9DE2HMK7ez#WjoO`Q+bQs$uQ9H|Hb-XAuvQjSm
zVM_a~9mMuff%m1b<9*G(I<jpVKQ&cpW-U}GLhNL6JQb5w>t~|k0g$k|Lp1^}PM8T?
z-h_{7??amkyG}mM5qL1RbJ2hMvb@Ee`&j8EJu;4Jr}_O{AqJL=W{{f{$=|&sA<d2r
zX+Zs6oFpsx9C6jPvsx=9f*_qQK(tuhxS@ok(Z|q(ga>Ha*$-?B`Yeo`tqord9)*w|
zxnwm?8Qa7*+HKpiK_bkDS*!U1Z7L+u;@|1?Ul7A%Ok$HGwzaDKPEg7pRU6*^jKWfO
zgY_5_?Mp--t-0DfHI=s0${-6@LE`egwQ#b4-sGiMV`<x|VxXSb!7!O<cvf2%BQa%X
zv~W}K)TRexuZVK$1vxn-JG)K6>yb|WmRRf_OcqFq3S?b@xfRthYFZ<J)uWtpVcIE{
z_-QJBJKf9-`g~W?EW`)Xo&XYw-dA!VcGB(!*mrtT9?(>}73kbozatQFmI}PGg+tZL
zH0xGOcT1%*PoiDd5L}%%+}s8)6w>c64(El?7VJ7^;|;%1-x-u$je{|I5Q@A_C!bvG
zZZR|n%MR2Vzb4(JqIE-Aa~6P8OLC?()ga(X@Xf$P^~Ossg1uuoZgaH|4yk^9Xbe6@
zRllcqwzbx;3SnfJms>ss#9kyM0$+GIS=aT&!sf!Iqn4CItnn;g*Udd-QoyG*5IV*I
zo3%cuYHDHPiP|2KTWaFh8Q{%865PhkzedI1d{yL2XL0o*9rY^`2ppT^Da870>xB(_
zuqTX_o^HBpCIqtLHLzHw)_{X}%%Hf`da|Yd=}n>jNxZJ^Z@9b1gf$3#3d*y1B0P^6
zakmYdI5B^;t_#$>=!zu@D9pg?#O?fR_7^?s92rf2f!{_HKQd2dh^aRQ&Zyf<3+Gm^
z1CM*ke&+@Td`1~XgH`>xVL#0yU_7a-&tF-hO1j~~)lT%+h8X@HW5#ZAwRm+gcZE<K
z+nn!7f_Vwi!%Q3hP|<QLm9z8IFF6qV_btZVK%l)vlnBw<S^7Mt5)epX{U!34)SVbU
zLh%)~M6*B6Q4svb{%f^)hur34#!;Zky;frNQX+Hn^LA)B=J9pTdPWWRjBxtk_ToIP
z5zMN%lFkr@1r2QLGHfF5t1E@;Si_d(G+W|B?(I4e8rL8bcn5M}v^T%vjQZLFV6hm%
zfPy64WJA+pD|vJoS*4q8qdME2`#UZ_X6Qzus0uRTCu~*(3K_zWRk@lX*7+vf4Ff*5
zmL#P)P%?Qc$5C_8y``3Xy8jv5!g1dH=&r)=K~dLh;lR<{Q5HKLA1tU(-g^fvDP(w$
zZ^N&2aAN&ZtMmIpt3$JK$FLyQgRI$t0HMBGN`KAjBdlr~{?ii|aoVDdGiK3j!!Lo^
ztWK;OXmpb%#&Ewf)Zq*9XFb8)z22LYmN+=iEJa=)N$NZFFvrHrOk8eM*dTm5k!mk6
z(W|Zc(o=r%e$Z&!uUWW!xiw9jc!q*kbB5nqLjB~GwR(fpPH@sU(=A+P39!<_HfCFY
z1THu)!oP53?1htzYkU(KP-EBp#HG#?bTp}GF=W_uFg*HjHMkzkUpwX_zovM8*}&K_
z9OEw<9m<*%o>Y>98QFu}HMScV&gk4Gb-MM)z&mSKRdT@Rp>L?W_WUl^uY)evQ9BHX
zmEfQ`bK9HDAr87TPlZin9O78>7#X{`dZnA42crC%j|Dl0BpU4<v%<Kob5%=M+Wt)8
zbn^50rIh2wU+s%s*-l4;o7>(A;#kk_Z7{k?S3lAG{l-|Se5G%PBSTk4Lnkk@3q-*W
z!3IxK<?%_I7%qD&1wSVWq@b$z8;+e~D6G(R`tFL`Z;fM=Z92x;oG)Kh)iiP-D_YXj
zH>9{sl0qpECsY5oRUzY1dB!ZoF$q%P$#wOGC;q-iV{Q#TGau~s6{1<I*&sOt9xCvL
z!}>QCoa|bd2ql@^51$*J=LLV{`PIc)U!1Y)Z$`&!B38ArsJ5uY>YZCXY4!Z+4L|SV
zZ7AjI_+|pA-w>5@1+&DCvO&#hX;eFM-?NShva!TLRKdVmHRjvn^I5s+?eev{5&u0A
zx%P@&f|!mQq|8T;Au!ga2f+!LQ<;1bu|p?rfj=~{L~>PQPx;o$*m^t%!!(J^k&Vf(
zPf042(7JJYP}rB|Tz8klt#ZE`os;;Nv-_pjwOMr-zcwSAubW4sym~{8ew6TI<*nCW
zks>%wajyD3_5uFEbt^l@6g2uj2kBeh;Sk=5;t<M?uB5k-3K;HX-XHV7VT*rO?)z&#
z#cHL*!hHHj1W)c7hOQXPr=2YvB3pZSd=K=lH`y7b*5#~32H9?!P@T!8huU75l^W#A
zaPVVr+3OTN8uuuBg6K!>U8^rc+bj(=`!6RkBt0rEghP}S+qx5?)ulF;$m}C=DN}x6
z8#lQX3aGWbH`8#k4YfTm_@HR~t_O!KLRcheWa`yZ5Oi>MlP5h>q*Yc?^r;(a5&n@$
zxDEngSH8$`yZk13^*HAX9<dh9wlMk$!9jEdKfW->{b0L*NoVVYZv<`pOK|gr4&HMf
z>o=n07>tNjr-TC&sdV2Jv4%k5v9gTP_GvYV=IR)%(0l3DW3p(URQ^_s7aKq;2D^8g
zV!s7>T|(KEyu=pEcfR9UzN8d|)_mjD@UzF?h9B(L{oBS!7mmrd9)YmA`<f4m{Dkg1
zo6?l*Wm+do_1cgYcy1$O-h{nrB)S;z8=PB&x*N9Av)CkF!R~G8*rmay>`NJ)IRDDv
zL8jt%Es)Co8k&!EvvC{;=j9;9^E{7a*j}I9WW4xfVCSXR-FF5P;&Nrn#X!D!NymJ`
zPA#iai#I%`d8kLhJ9+oxUii12^;3TD3RQ=VvryQ<wSZJ>p08c@vtjW;{10hU+5?vL
zyn>y)tt@)0ejuwjx>8)Ia}-OAd0kI8#TbZb(GC({q-b1@RwS63Qfj@{+7S-%yH~Rm
z_nvi99;EKAJuy{0Q%Fp{!Yj~jq>j0OXTeY)yeP|AsGtJV#5{IparhZ-eHNOkFHI}*
zB=?K+H_`?RG9Q!+DUuZp!<~IY6T8Btdpdu9E*O9%omvlR*TIZB%$q5tgkZRM1x-p|
zjH!Zs<6CRJ^VG9cFxEMhs$NlW9;|6!anWY`QnC--?znBaZ+f(2`;CXl`iWs=gddir
z-ge#7@1aIyd{MT>tz4Mam?4wiL%0nU_pIFWCL!JvC&|~KehU0Ut4@yRCQ^o5cNW6m
zcnk{`rdo*EzpC)Ooi<cR>A8w6;vjh~XSI$KDe3mAC8i%eNo9lw-IkQ_%;eR!=?xUR
zjUL4D6_JGBLQYK0*%2bGT#|i=KcZ);SEVLi#<|?I>WNy8?x6J~hUxeoE`OIG8CyC#
z{8>v<GA;gWnhWw&Uch^}Iishz=&iHlQ!T{2VYD_JDma&?%EzdowCU@3B>JJ{Y)i0C
zn0&b;2eNt3U;Wy;|9dx_drx25<&fb2-R;~6o<LF9H>@y}QCC?sY9@^d#5z;iZ;xy>
zWgM3JKpM|T_*%Tn-t-4<GAUf+D+=quqKoo`Ee&pnCODy&r&7OHX!mawNw3eBfHZa5
zD`6QdF<keKmH3&P+ddSK-dv(Nl(?O7#-v&;BD?s;`NPX}+@reRM08KAu`8GMV*4`l
zRC!GTJ{=k@>r*-3LR=~7R#?VO<t*V;TN0>%KJsY};aOq5p~7j!l4Nikq6ai_GZBPB
zLDw<10;3IP{IS7?SO^`HqXU`IN8h5~(i_Sbfo65WzbZN67qgb0`{R~GcRIEUw$-_g
zVVV94dqvIFa^z>$(HCx+4Pvt5LQQgpA5SIxF5g~>T<34i@fi15>d0O$54+)YT<}X|
zrzQCDPRaa_?X_(z-_EOA6EVNo5m<b|&w2L_2!?`(Wa>rY@vfETfi1Y30ZGXV{xhkP
z8LH@NTrE6I$~t|Vi#U~@|A?XSzYcM`H+V1gv<LQEDH{Jhiw%ZYQSQh7`Dh}kTU8@g
z<ia2Q+QV93OnnkhqN0<N!8v<ftNM}9m!qO|m?eOqQh*FnjZOD^))H^`7tiZ86@nNh
zn8|gP7^<RJ{HEd;MU-ESG^XicRbeq*(UwQU7OV_{^eZyWIbPFgQC#Y1hGKP>qwk5&
zu@bMCQq|f&>Y1Qv*&=zXYiRNn^RTd~$@%uCb6#v&xb8t8^X4<?6uGk2>2HZub&=M#
zZVPNw^~-3dE0)EIq4sW{bu0;5jP!2NcCu>|q%&l>Zpe3zgQ@Enc4uvJsKN9{;e*!=
z95MH=qMDp;R+!iSd?qsYL`w$2A3@++#17-8#sldCFiR<^rQY0;)!!V`z5USf8xBkZ
zaBUe7HBcc7cCl33;z#>?^W9l@a=~$R*(*Pp*1A~4+pfN8WmapDZzdz=HLCSxNqc`Y
zkt@~0;nrKLJ`ROVrjZTbZQ|=hQp2iEl<f3UOiR<?l&Vl8^1QEq>~73arg=DCIfMOi
zU8JKv*5xO<(AfPEtL0A)Sz_})y{j-iq%ChY)BEbfuUygXpSX^gPI-6*da_IugIs*j
z8;*2<X3jVF?wOzaz!LB<tlYNlod3ZbsAlb?0>omRRP9b{jBsZIN1r}QhDY0q1#9W0
z^~)*M-CjmZM(e864~86XhkYxe2*4JkL)1KZw3Jqj?SgQV(y}N2%q~4%j;tk&Y~Xu}
z&mV9wUp~i{rLdFw1tqj{y|qfBx^3^)-46*HrE<B$2@OQWt<$tUb&qfjQ}$rOH`5Lv
zM*AYaw&rV3uvCX`-Hld^mD>$$QcI*{f00|T9N2WM1;wC75`?82*idL_AP7t-q&3Cz
zCAOSd;(F@emUBYzaeXdz8(x<Sp4*VQ)~zNjd*2W``~l74d$aUITqwCbHPMX>&nKk6
zjOG*RW8)#X)v6cDr)bjuR5o3Nmm-h3DMW;~_8yLG6_j}UIKpuJpP=xuC9+ULoL7LL
zVCzz~j!p1?0-#sFV+!I`H%??LM-ju8i9Frh=uvZuDI`pU0h3lSQmAv_Ms~P)8F44`
ze*(f{r`ge2s$0XF6pg13(XtBHaq%8MdS1vZA~C4NJ*D}%^MeV9odk-{i&%i<%UFN}
zAOY*Tcx#}k+YxT|p4+mK{w%c^$zV%~#~mxdK`td(+@25Rl!}zW@c%12g2aDT`s(iY
zwlZLl!<frp3cvph!DQ^RBm1m?!8#*;>8*zR6>RZGJ_N?5$GpHD4j2R%HG_uzXUzgm
zgiEb~I;D~2Qtbx<_WtyUgY*plZz~`!)5~|^ypNNq#C-xWxxqws>FXaM$$%))J<5^0
z5Yn*DD1%@N1YYcAyNUsBrY@3=dag@_4mtEWmW)okiQz>&?mrh$WpYBicH&axFO*3C
zTuS6XsiYBaB6d;eKNlceU57}yDfKd6Rs_`L^2&|?a)OUa0(Aen9~86$g<gX&NU*}=
z|9G4q=2Ll~RCvaUApEx?0Cot=kBFWWd6~B5pC{&D$fUSmDTn@B5neLjC2@$twZX^y
zr^bx_8W)xze~@$kj~dHhNCC0aOkijm?}bVJxnw?GsF)&HhnfB?0?Gg^w$psyh~nRD
z#P|%@=&Agx)qmX&3i2ZYMI6pB@kjns-HTu1xPguGnY*3-Wn*CM|C;%aFZEwD|JgSF
z+nWFAg8v;f{ut^0&!AzSfCAXnY{0T{=oej90+_$W%I~(0doIdv!GszLriF!_D>Iva
z4_=J9xV(TDR9v$}lNt?(Vh;de%2Di5y)+oRBEg8J8}vE*?|(%RVSdqr&`Nn}RoYH+
zo29wu&;mh<kDCc%>E-#^X_@P`J{RBy<@|6H@CHxG-bR{r|9z_rnUWcl&!mgk;(KKg
zt(cRN)b9ndnyuP(uTxEp0?o%#M=LC%i;3zc;OULG@C3K%!L)FI&bedy%`E4-nQVB4
z-L$@Du`U<DgvJ5z8>^GeM`qP@DZSUc+H)xr<7VkM0hGv5qBHPfbR{P8ccOp*LgFic
z&a+fz@I5Kj$bZU4Z&D-NpY&Xx%fRRNhwHprPX;g+zC{Tbi#`OCjpC0^-#w?|ck{Um
zD$2^ZcH?F8w`y0Iwd-7p#6`N9M*;G)3oxJQjP<;KjRqV9`d@s4S8Q?pHuzXFWp)Ig
z_5?wd(<K@}89%}tzDNm*T=fHFuAaAcjnw;<jpt{_Z-A(VisB4RMqNRu04zKwy)xTL
zG0`7wWFKUMuRU$>a2^L_c-ct5`2O#9O*f@ae<sl0wU>FA$jMYS;SdR!jOqYNoA7|}
zO#D|Xk#yJLk9n+gzGtHAg~)89_K2sS^d%+dj|_Tu?5HO7%gy88d93B{0>1aU)p9R_
zrE3_#MHhb#t0n^q-UC1rW3!v8pQSuM|1{egYL&on;<a8>WdcB*NBD5{M@ihj&PxTP
zO70#-@DyJK8GEic?|N=GaIAm*l+2<SW6th6>6|>_bG*s_l^piO0+8~=YFdKvKC{G1
z%q5n?DBJ)D?})Z1M9qgx*lwyYJ^1$|0CEBF97u6E@$yTZs~tG-_WC)b_yM5dfzIcF
zItCNA0r8V0J*9^CN9ntRGm#Rx+z9|_N=WC}%U`?EBPIB(yCb!q+U#BCb`cyh*Vqvm
z*2&;o-=9nh(VvJpxL8bx`UCLYs#t3|+tMD|IF?`%#5j!A^nRln9Vf~sEF%A!Z7B6j
zQ8c?v?sc}9>lUrp(MlHGFxS&;C8h9y6bp&7!}%jIHZj>A31JNYk?nK>SW&X4k%n}`
zqO(C{w4eAFOMGK)i1hOlI1(>DV`hSTh)g=EuK=DFy9at15FTY8hG~E0<|lTcmj)E6
zw{fJ;#9{>2ak=8jC=|mP);&BMjlG{6@oadUDXC(-N^9RY$RE~*NSvSSX)fk$2uk|8
zU5BKxEE9lk^Hf=;k6Gszr`S{y&vQb_eY9~VV1|Vze!a0yD;ABwg$(~zkf`XoldVvF
z-*QB{kW3%&lH+KgvDyJ&^*#G)8QK&9yYM~$PZ8>Q{b(P1iC>+kX5D#b!~F@%_z_Ko
z0*}*js%yxtx(xvyf(5nleeoP<v*YQnuj{_cX`aQ-k82zj3+I<kz(eVVQTGyuZ>5bC
z%H;tAX96Irc(%lj)+)di-+|KrkZ&3JCzI}1hVSKXDiiVy^L(l&P3m`@_NIJo#=^-+
zqfFhK`(`t|HVx-X0$9PRrio+8bZp>Yp?q)innBKto9#faEyX<nnBg%_qyQc+mt#vw
z7l7I0Ee@nluLX6GLI4N#J^6CS4Lf`Qzv_Fqyxnw8D7k~WGWJ`_`$%hf!{GHTJ(`~6
z+vzc(F(L|8o0)*|t4XsmD2`PJk>NP6Sx&>7J!AI**knx_Vk%%Mfm~W?q+r)bmLk!*
z{>mZfp4(%RjIa$Pn{j7MMZdhyy-r%w@as1LdAmGj8Z`a*WYMz@Ui>f3*3c`sZOkYk
z$l51X5iB%c%h=m}M1JXj#_dKgU_OlFC!<#_r3XG{4%X>`TItYx?c6(+M7lCM5I{Db
zNdt`J<}E^tRM#nYhiUJFa7$ui_AANOA4$F}k1EAg_C-aVv{M>fJ!66?s-TzQ6eHIw
zQC}V3TY-cuvO+Bau3Ne`1=ps(f(n1i>(ClWtQpBy=Xf+d1lYxcm6hziXvyo9O;<mv
zGlFiK2xze{T=MVYcp74ZrPVfWx%K!{&`fjeCUPF%Se<V2)nLzg{FdF4^##Lo3`R>9
zzCseeKFPIr2`XX_s?-i`h;pO=Jg`pxg<q|R0LteSP}IJLChgc4ij?A&uC+zJ$&Uj}
z())>baUWqjuK={-&r$%@{Okw4($YVV(-R=T#OcW5ujf@5x4{e>aW-dwZGVDCECN=0
ztWMnpO;4}t4Dc!)_WDF#?4QNE-_tSu{5lEXdc#R?zK7E}>6quD=+91uLSU)D@ulxq
z#@x2|MuyiEH9tj`;l1SBt8SX^{(CX`MHa_eaO_b`Jm;v}iK}s_oeNpUR|8+kY>FiL
z4}s-^Z|9z2G}p4fKK-Cm0&iD~Y@l@@_2C?+hM@bzO8GkZyrZ~Qdoir79(#G|d#rvC
zt6f_NHIl6B{W1^K&d`Ia6ZWkHW77pRx(@Yfn#kT1AwJ)|hjMpw`~wH1jtgT#5g|%-
z2L`^UqXHsNBfX7te)NlPYaKi#<a<9iZU!+bA}{&gUJzWNW`QOGapbUc68G6)%8dsh
z-p{Xy)k@^1y6B*PbTI(9eMOJ=X2sW#KKeYYj`$(tvn6~P<vS9FEHY$_H=^F1JUs5T
zKgur~(8#)9D=YbXA>n@2lt)3Od;5EBWF8d(&4$6~Rc|+J>!v6O-^(BQUfZGc9xIt~
zG^eH9*0h}?NcFDA7&7jDgYF%&b>dsqEI}Q(vfYi0L|%<YYdpE{SB$}_M_tKt`I_eE
z(QCnjEAt{}%@ku^&$E2^hv#sCDO_=9?C5+WOU7f#K{5|ONJO5caXfx=od-co(fg+9
zn4Cv5sNM5m(zVDDL6DN6ZujL0GThK&Mo|J}llj3;Q0Usl%L<YUg|tkxTKp)kF#`zZ
zBDbCo<)Ryh;V$jj7E*%7i-}jU{K&}Z!IlY1{D2k5XIN@j^0*hw+HBdqma<w(rbL!H
zq9d92aSGlCOIj7eC?O>Y?MhMFGa;%Kf!Rrf^=f&WU$b8IfOoCg#LLfjFTiI<mgvdf
zH>RQEUHHtMm+#2aC=mQ*x-?F;W_39EO}n|JU`64pId|}elvg>)q}S~<5D9Jd_cbvx
zY$t0q#fXonPVA3x;tkj0lrs6?y;SS8b9eP(jeDYV$p`ZHZp1?T%;*z{7WMDIYhE&I
zL;axF9f?MqOM9}Ea|+!MxL6BVf^Bd^!7m+!(4JRHU<(?gdDQU%(uFn92r%ZFi^Nuz
zqf)19<uZ@PDMO+jT;+ZX$kB31_(@^;T1nf>d3$&LhF+>?8XBgkqm;zvU~N<f(fIp4
z)GRwH%~MPv;?=#fWl4^3i#b}c?GPM=E+Z@!2<#}(VzCQS?1H3OR2LI?*rh-9P8b@|
z^KKKOdxB6qahpi{h6R}l2c25_=W@EQ4i^acC*k{Rr0_c4tkO&+BoZwJ8Jkf0p$K8-
z0@cNN=?xeA)pw&p@yq<*si-$>lzGV*b&cU3NT8CjzF{m<6w&KiIL&9*!wXZpLvicv
z`cS_QzyKrkNS$La7_Z?H{iSN1`^DtRCVSj#e$=hmKg1=^;BR$Dedr9-WQb&hgasn7
zPVBlpHTkeJA8BPL;fMs;qK&7CsywuVl|WW23QxWq9PJ{m5`)%lFmvRy_>HTmcQ*6h
z*6ex+L7U->zMmr&E7Lv=>}!&QDWDdx8h;>b>6f1u>sqMF`o=0kP*-AWp2J&kaM{0i
zpeKEeV7JnCFA(l(i~KkxW>C0=a$_GyJ=?5k#|=)%99z;9Na5hnyeE>%`pryCI?9B{
z%+oR@AH{Bzz@bXWMMTBaD>jopsUj(kF#}M~72KO(wTQMWYFbu+`e8eqo&07xTnI7e
zSB?@#$pQU`OQqTqw~PHg83te+BiA@`wO^QktW1M|2h2mj`|~k7=WSO_8uVSBapG7A
z3fnN}N|6g*|4+^CO8z39in1-}!H01IRjJ?EN%Y^ij%lvsBI2|TfS5-<q;nIj**ESu
z%!qFavMs#!W~b*+$awO=IR2vrzL7Uv5dw=C%Z_X>jb{?Ug;YFl(3dk4J7S-XPY=K5
zhfBsTNAb1^tUVl{&|_t@!Um0v#=X>DW&S#|wXlNXTJHR(nKAMW0ZuT>lsucd8Vk^=
z_1%?yW(%Z|u=MUA4y0S4x0<7i2s*$$e>{$W&aLo8Yc20F+z1Kpi{P>9A;`NW-m7W6
z8ASVBHtM=LI0F$qakEHA{DQM}$MmC;a&}eigfQ$?!($HBRVB$~=l!BB!D`hsbsz@f
zSA{o>&e)B?uaSqqH1H#Pl=Tg7MD}@>?6>hX%L>0!)qu3=2$FUOT)aR%8>~KacKhhn
zqg&w)12yh@OC_mZAPvKqvYy*)u;%0tK!tY7!a|skwHbhj3T(~#X!9OvQp~pk*6e*9
z+}U9+G?ollfn?Frz+8qK>+InAQ*l!8kjJShNuQs1`V7L+X3s;27;L3&ycHFt1jT(8
zj}2}!D$v8u?GDmRTnQqCtPlud#0{h8rt0*$-~gRodiX6k0V9<5{bm+1wIqS;j4Oo5
z$lG$?D@59VygS;#tN9KD)ZgJ-(zj$O%9~rDfMy7zOLxPI)`p%aVDl0yYI8!EyLUuz
z4-+6_84BomyB9WodE(hCheXr@Z4S#;YkV9~zgMjlIg_7>dg*3Iu?5E;z2+VA{Fspt
z5nNCTr(q;krVBiNR-kz^NafZphNo|ex@`6gZEwuG06hrfl;pt#na6g+f$YvvNW7Dy
zWWTBDS>VrE>YJ)*5=Ut%)a^Q06$r#F$}fR9te&|OX&i16f}ZIxOa>!7Qyj7lLq7pO
zYK3_gS4rVj)I7~@gs;j*DNC8?C*fyzlG{(pZ&!BE-|WJff+ms)R@h_F)hOQYZF6HT
z0S(rnIjbU;aZpO|$RA7&8*u4T;jxEF5Sb5_+~&Cih7N%sU*X#yzlgS}R8oBhMWO|<
zWPa9lv9QiO$8aHIP=7uyy%~5#CH0BAZQ+eF&-~I{!}A!#m=-h#!FO8T>3T${PLWH@
z1$kH8&OdR5ZrUwi9L1}tgQ$r}lk%>8R@G5UKZklSh(3TEO`o4F-(}ea15s#{qEPmE
z56>qP9oCUwE+h+yxtaGv;cu{2vD3kKyyezWKA_y-&91Qb_(N!Xa+f!x*H|?PQW86O
zciiGzgS?za2cLUsWRO-sds}<%%j0Wbi+*56#Za?TeTXFIgc!xoz9iid@dq^OU<Bm7
z6iVhn@-00lI*SQ7oE@aZH^NwlD0>@8G2!!lueG0T;l0;POTuyhUzp7Hj}efVMTz>4
z+p!7o*FjKSP1-*#zo0mFLr+!a2&hOpi+A>8DjyIn$u)Q$^xw(17|sdqN#dbw$F!oG
z5&LfOlfkniMG}MW?tJ-rY0YYhVQXQzVeM-9bX0ic09alJ^WoW!k*Ac+ka@=x?z5ES
zwr@9ssW=WVjb~F(y5}bst5v#1y7tT6>DX(7%9HP(2GQlH^V!NUv6igx2*Bt<skH~d
zN?hNKmU=myBdvtnj^kMCSS7yekpu>60Z)CbR5$Y59+V{L1QCMDzt^HsPd$0x9r=d6
z@^+ddck=1ki3&I&-NP*5bOK1)Olm=R(Cga<^uzs9Y42LBai&K3FbMbPHdtPnqiY;<
zg&AaN_*lxeSx9+q)<O7=A&jYj5^VW0pid})fJy=YqMu!0!M`P%&38oe2Zs$y3R|Oz
zTX=B~i}mV+KZ}iUMQhOC|L%u1p*4ZuIMXJBaS`~6H+AKk8T^d+RmDJ|T2{#Dc*FiT
zD`(Rf6+VR){?l;6;#K^4N3c7Lq2{NvX}h;f#d5DCI2a}qI&MpL!-|9|=B=k;5Z&4P
zp+Q5|BK-!BF;OxOdMHfTq*bUy4FIO|izoO&FXPRHJWpMsPuW$}jVj9=k~N#UN=?Qw
zHYeI^l+j#Ommm1am$tP@Ye`<keU2Y2PDqu4{1PBa3mzS8P|^%Ym(Vt3Q&7<NdN<n4
zNL|R7-B_%Ae-*AAZRDRK8lvi?&k_i9kUh?CRD>*A@i{e7PDSI+=(Z|fz`)j04Sj3;
z10>p;dOX$Vysw(OYDX(Le8OHVN$~CpnIoZT$f<=%S5M=e*C`Rop%;Uy664>=nY>8D
z*iqDe=Io*_LHmf(Ar7nbaJ3&`qxgbOXSh@n>}GE|(x*(L?qIY0R=Kqr1u+N6@~bXs
z`+co(^kRcMALB-WOx?(3T<AB-yOa>hvXg=#(m788$q2TXL=H%OukF`00Q>Jvg@JI+
zoZQW=6Z`1Wx^^ci+2j4qU)G9%;r#AZ7<XvRr{5rdbz8^9p%2%^SHlQXX5Pf5#lrYF
z1n*+QOgzox+#^+^lTk~(U2`}WDUUE^O%fM!;Q46G)r0*+BlNjOu4*FgBUuYBK}t3D
zyJe=IDb(l;)gqB{?xEdx2r5kLIr+2ZIggdq8uzhQIqTHbd{UkD0sb9fK~t*ok+7EY
zSS6LCLhBJ&?n|iA!PMx`TFEjC5oOOtbw{sOagx!hrJj}?S_bK-e*!pW^balRM7_Mc
zaSS!Qm4)|_w@BHp8Oalf$Lt2?zDXq(TKly3UOB#Hj`iq%ILmLaBfZH!18qd`tHGrM
z1ZKo42xSwMNGnQsH^$uHVUFXb$-#8XL>H~1>j8#}jk3y8=m9%a8nla@dG*hW?(oHT
zpBadeiWi;^DRw_$_y)T4JzPw_&wTA3BLiiqHithg6R;W!I?!AFnjcKs;C0N!sg}d5
z3u?Pu(e^_R7YUBNe$NCM+l&8uktXN2dk;#S5!(I!A14Q9%te$ay+uD|7zX2VI@IwW
zKM>f`1K&p*bL;#AuvdeBS#bj?Wd7H;Qmq2KGtO<RuArOcObeQYU$n2w>*PN8i}XN*
zC}`o})&9Qxt;s-h7nWuJqj>OIXP{m)>1OTk{@ri$r|;{ZBB19NewzWLk;*^Mx_Bbv
zcQ65!MaTWCZWopG^S$ufENk=Uf~WlJ0$Gi(L&g=l{y2XHMWCcX5hr_vDeC`Q-OGn|
z2?O_-HH*m~_k*%#7(fwc?wY-q4!3^{Vsf!Ks3Z~iS9LGS+93jmisuqMOCUAMrFH)L
zS@sy5Sbo2X_+M-ORmx!~I9%~fBX_(Em;a~I<Luzu*#s&7aX%>K{{u6h<5cfa?okn~
S{RZ0y$OCy5xnfyZ!2bZ3Yzip=

diff --git a/docs/figures/arch_3.png b/docs/figures/arch_3.png
deleted file mode 100644
index ba211fbfe5c8fe3f3215d8c2b763ee744c2961d5..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 61499
zcmeFZg;!NwyEjaWfC5TNmvl&XC@tOH2uODb5>kR7-7SrPl$3->r+{>KclSFNKF@RR
zbIv=y@%;hs9%H*VbFO*i?}|Cs+J01!lX!&k3<U-T=8=@7s1gheJQ55H+$u8o7v}ur
z!y*g}O1Z_WR|-<EUXdy|*uJr_GKGPW{1_YmKtWj(*Jtk?bE-egL(GUL9#6BM3QMnN
zo;{}{dx#+&?3-aEfunA~7#!^LNH>S321j@_1d%=jkzw7B0i}D-=x1Q}oe!B+;@#bv
zknQgDUaIRZ&)RIQ*(yH_jZ=w6%42$%2(s=t|BD(nW22>na}gi-L!V$l<oI~1H$nag
zFe<rgP2E9EFtWX_ea}5_j;_41#TrRNAHZPwIgks5#`_-HJN;PwNHqaNdZpN~8)bEX
zpf8fhB=S_W*32(D&&|Z|DK-9w#LADzN5YzjFmQ=Iv6S$_qpwbS3H4YvgSJ1G2Lv-?
zWqxXekN@;ZMzGsl$H0FwSTSvXK>9j5eYxpeCgx(<A=;F`Dqf1r4W*)WW7_m?&Hmc9
zOEO-rgGKlOkJutXU_aV%FLAc%WsgP-9u|M!%iHW5$~;LgioC@56-=%^&yu_*D{Bvh
zW_Id_4Lqzu6Dh$KI414^@)eY9GC1^7i$Bxel$bm(VP{mzmB+L*xph<~&&uPGz{6(m
z)3jK)W#c4fZ4iC-yhoq(&_4a+7UR@A@zL~C`sL2iH+iDx-KYJjUHYU(1LwUVkt^-E
zZXSK(qTSnV6T9Uai^mv=4Wu&?xdYt~<BR*mnU{MfF(&YH9d2Ic;=92yWB3TlU_AWl
zBk&D@0Ru(&yTL{)5h8KoqUjSu2WigeFttRVF3f}8HI`zAuU0S#`4a{72TWGwIAraK
z_`1)vUVTM3Pqj)Mq>6+4tpnrJ{RKaDmI)^R0*1ZnAUi6Bc1Dy3mVp$OE(2aQ<Dru;
zCd;d%3V6BZM}6by2p^C!P;uahC1LS};o^L-4H#))#0=QRk?5LjzM=o|`87{tj+EEz
zZilb)p!%oRKHjk(>^{mTA9MpWKT>{CU-JmMY{ZRMxe?f{embPgG%zS1j4}~u@Slh%
znIl#{rt_1|Krv@lLA`-X_3wYBo+-CZ_UFMP5!*~=7Px3b-mkC(&5njRy@59i$ZoJH
zXv~3*3!ZKiI*-+V1<gM{=l3LPL_cdOomWI4bwGw66z|2rS@cDqu$O#7E}}*jAc?OO
z#!gx+$>)H(Ar=@pfJYb@XlT^SB8R6hLm3#P`AIW?wq3>GEsJbIXhMC0U@!72U!0$6
zN0Jdy8KD&#Cw@JGwh!Yk;^wqwy_VUI#|A?4cHBt)7-pZ3+S(SC?0f9nYEo;&r+IX!
zOGvmvI+j!qa^1{4S<*rW+HvPy=g}7i7Wd{K?586;ATau2|4hb^z`#62HudLi!EYgI
zL2bcP^v#!dqC|Rn(Q4W%+j`Tgw#c{$WB(~zl9iGtnq0CVyDHm334Y^ALC~=rcuDk&
zFsaDVREwAB>;_L`*9z7Sm{65JocKT}9~Z?Hg&s8!NfI@uNP`6*LO>NBB|9NmkfW2q
zpW_*o*OlDkxO{O~v_ij<yu8=hl>4OMQiao*j3Zu4TtK`*j7W`?4v)r-#tvJJR)sD_
zWnB$VHJet84vGE^4G#S%t)42ON(((cjXUi@>|yLFt**jm9vQoC(t|JwZHX!QdQpOm
zrS(9k#{#i>qCzrz8J}tBX{Tr;Xqi<@R0NBsdxY_j&>mBymVLC38EiXeeQk{MF3w;y
zQ1em6ZtrD2t9F~LScX#mP`-oyN+_N3$ZHN+=g_wmn@>4DC4JP&pL|lK`!@bhpxLk)
zU*BpUQ;L_<DTTAfjHrN&SM8<X3Bl%<XGzZlpBaS_J;P-W<iJm|Nh(OnNit&(sx;SP
z(t25}R9vCOoY*bH$(E|w|D%&|P;KmVB<}6r$nJ0A;TL~C{^=UF_~V}SddPI+`9|F4
z_l@v%iw(rh;@?GM@3Q6pm~G^(+m6|e-I*JhV_1Ki&6qIwt<V->oM5S8J8mrc!hu7}
zO2yh~0MD$KRgP`e!gC6Nvy4OPMFD%QDN6EWB8seQl1u;3e)|N9BxQlTsZUnzoINj!
zxVG5L5*{S1C2}P<r4?~;+3d#E?l?ASk85RXZ&$ln6_{+8C7MmVy%_Co4KY)-a2`l9
z+Ock!tZTar3o8g~eiPpZubpI_vEwntIc4i|>hh!>sh;eez!kz3-IWbaDs~Iq8(sTm
z-WtBIF5D~o6NCLPrVjF#NaWtgS;#rdeK<l}d$_ujKDPIDsn>aSYkI52Ny=_wX=tl(
zr`x&M#cMTptKF4v{?yO9SI(5gsL7?;F1{w+?rwcyy=-VHg?e#gNppQ-A!iwTLv`Wg
zk9G&&ME&f$){arL+5sN@ucb|UICD1PNI@M1E1%9Mu8NQA*X2*q&r0{baSWc2J~V&i
z4}Sw!jdY5z@!+Fh(gzV1YB$UMXpCPGZDdV>i!Wcjq<@tjq#0O@Y>A#1z>HFgdWW$<
zTt>TSpRJh9Ud|Zu44Xui;N-<5TODN`*BhJO?u4zfw`DEkaonCdD$~9-gBN;7)qNzv
z)nUxRds+rH1{;1ws3SPJ%=`_(b;0%P8gQIsyqQc3^jrzreMGX$lxkGr`}}TG?xp88
z*Gjh$&6*12Zjs9=DpBkS+bKUd+-cf4au`fC*2%tNG)kSw&ZJ(xVC3F0m~``S@-Rjs
zqaa(#mXwNlUYAgiSj%3aY(>X}amM7L^TYZexK-bQ6t!C`VkAmdBKZq}62%7kP$OY{
zN_UF<>#R6J?ZO(B%Hwv{Xj6i?hR&1z8+V>JJhtvrC_C7e>c_<<rPK8&TCp{S)>XTk
z*CX(jo}+)hQWeW;zhfI9FQweOsatn0zIb+Fd3fc$f05ZJ+$gN{yJ6PiW|!oJ5Q_Kq
zxIu$qN8nz-b&I7Kj+m*KeyDrf*!%M9g6q74Few5)&I#+uD*S5A%390Nt+=EoZBf%v
zx5d8N$<m2A0XcisjFT=^y}4FrE<-LGr?ZDXhjBz6e4KmdT9HcjO+yXkWzr3AUyXF6
zUZ=Xc)b6|;FD|P$KZ_S=5kNn^+_o;{%x9Vknu@R3ySUB2*%7LEUm}p<lCj=g{UPf^
z1BW38uH}xU<<_4q&XHdyLzkJa4uWh*PWh*Y3`W0IM-FN@ArM9qyI;I+Vb+zn`TU*8
zn8=d5#QVo4!j7on(?6A8EwXuXYOATf=Nx6vb$wUM8-3c8bmy@jb6U_n+?~gjN|xFs
z#P*)^Y<r#OpmyP?Ywc<Zr%Ip^!*OA@?RO|q{57*~?Y7s&OTPM>ZR|Dryb@kL0rxOq
z(7KtET!Qaha?_O!?peP|zHG$Z6vs_HuGZ#Tb55&W@>sjp9jK4rv)|QiB5`V}pfBGN
zarJcFgTMXoGiU>Sj=$TT{nSks{d?a2$Cl{wyiNHcZ?@awvaymgr*yNqFtddL;<KYm
z{hPFtt1;i}wU{+i=1RRkkF_49Jv$_>yK%Vf@l{_`H~xiqr*y48V)<pP<Ie7Csa>+S
zvzeeP`JlROPi2nC)Ap#*n%(4hBj^*lF$t<r!TZ9C-plD*s>`0xGAjK8{hlU#Z_AtA
zy9P6UV^5vlSUYNMSOg+Fr6(}9=y1Xx5q(?;V1}*>(ma$h3hsQQ;O;)=UG7Si!5ipv
z!C>~lBG)`@4o9iX5xY@df<?~E@}P#%*6wL{y5pNDk`HbngE>XhMw&$A_q-jL3oj77
zm3P1MLVBua-8{GE4c{Jwym++Ku?L#i5K|4QH?p!Ybl^KO3<B&k7)0<57Q6*v3I6k4
z9F_(K{(c+|1}4Y?2H{`V$bnDj6#?GRn!lg$QGqZIz+X?m+bsj`-&eyUWx)UYJKQRm
zgAq}FB_#zum5m)tO>G>_ZJlJF3j2Y9hjx;hjxaDdRL~n%N{MnG&>yu>(Qwj`mEkqE
zwPrFju{APfa<jIB=)my1@q(|`rcQ>WZq`;dj=XLH<o8$bg745UGdbz~B~F$C<QlRH
zq_1opOi4MJSeRJI1yM*zN%<X2-ta1kivMdl_)CD?+{wv~mzmks)s@MWjmg%*jQIr*
z4-Yd7D>ExABe;Um(cQ+$(2dc?k>W3rf71~)bu@OcuyeAowIPM*8XDO;I|-1JLyZ3O
z`YWfYo5g=|vT^*ET0lW&XomR(6ASZyXoIEv&?v8hg`26Brl^HAuo>VX$i>dee}Db|
zn)xq||5&N<kCj~P|6KWxng6*`)zQ@9m8~^k=_L4H?)ul_e@^~uAwM&u`9Cc2*Ua~$
zz|Mjw{LKIHnIOuRME)6YBcX+;ybAaPAcJ1800J;He?Ot`I+RI`O~f!T!Z1>zA}VgM
z+o=y?u*Aj!_m~v@qC_!7VMTgFV9{ud#6{Z(Sy1T|g<;`I^E3Q>GM=C{k>@9fy)b)W
z0MqL;A4DS=kwH_=z~?<TL~It6IFmY)mbkf_*Z(tdW4EyGthOP|gML*sg_06k7zPf*
z2L^!@1{PVE^bdjq22y6YQ2(ocj5cEk<L#6G^EwzfKbCo_f3Ny`DF)IH`2P@shI~lT
z+#cSO{SPgF*u8&o`q$7rhCvu?DePbF{~GgQnJ4=1r14(-&k;g68LCB&|9;i~5&(wc
ze~I+Ur$F=v(ti^Zrp3qgKLXxc!=Uwlj@Tiy#0HQ3_p5+S9{+cneEP2uelix9Z4LL8
zSoP)FwX|sISUPv5H=blgYQ<4c<@pr@(eTNMc%6cq`fMSNrsT~kS*!~bY=7+_Y_i-L
z>9A8ZCT(L=^5bGPdFZ?KRHZ>8yG4}K&ZH`xN<OJP|ABvs%cO-xLC{Mw7+<?5!g&5{
zyKaww4`JYlF>7|dPb~)$*sx^HX9f{Rp)&vY{Uz!tuH#Y|S9fvsEZs^^e0+g&Ubm)M
zYV5g4$xEP_WF$-rsh9!(V_(uI=gZOZ%SEM)QP}iqo&9`!3=fFiGAhjn$%9KSTk<ZB
z)_RgSEI$=LBH)Zkb(t)Pn2D4W6}&w+cC;>SIE~sIDZ;HA_}q(Gv)6dFyjfJri7j}&
z|H}dwvBV7i!Q&WA3USF=hn3}(c>Orz&jO<jZPY+-U!|wQcrnk2@X6-A;ZSiHBIrwM
zUe3PTYk1keyu6&CCux2B=c{JLZ-LVH3_3NxgCprw3p*cDr_8ICS<p)mY*v^J#HD-R
zYBDulA6AZhL3G_7SA5f(V7SQ*c(2@?%`vOY<;o^!JFIlCPCg*<Bn={R(a5~I?nLt)
zDN=u3@utsYrp{TMaN2P|AhBS)K&)-yMF$~;ZNo_%<^kQdGlDMyFm46|0tS+4ri32y
zc|!A0&TH8ZN<ym0<^zsLt9>gK1A@_1ujB9r9`h_?xC2{@671>afl=*T`E;hdG>=05
ztL+Ls+&y=9UehH%R3?^TvF}zBi|!X?i0vrw4Ywy_`aNmycJaL^nVFgEYOoD4+M*7a
zd6hC2HQRnp*E$X#x+=ve3*8(?Zv(33?{7S6uXgKmBt8?13G@p<u5Wv4=RqWi;UkRv
zFvX#p7IW<m1JexCczHx!pHZ|qk<EH{<Hqn;Aa>o_2)dbGWx+2@Bt7iJQ3qnLV@2cA
ziW>If6Y~Y?bWewvruK|?XS)HLPU2`;XGDey>%>B=ubMY9KXVVe$q=S_Y!>vYBxT^-
zT>Wv)Rx8$uoO^%8`17fbABb-S2e?mo+;<XGBBWLbl@GMuJxje)?M!u@rM<a4nL_!w
zk(bUiz<=~9QTn)txv7(L%Ic|?c-|lL(yDNlKgHTy9=c7}z+rFxh&1h0&%IYF*5bH%
z8zrZ!RqK#*A?UVm!1gFhHkmUklxrrtzHVJIAl#_+`An|j+U!_~uC~jZm+QsRfRNa`
z9$+bD0Q2zm7y_ab$rv0)Ey|vFX08D$Exv;`T>Vac?;EPHXB@0|)2vQ~oo932w(lEH
zCoLD*TY7_k*6%fL2^tBWjT@ueKYJ5U61x(_XO!{c^00^5w0`G%v}sSA*t_i4yhi(5
zrA^)7Z6ogEGUKeyb|R`t_4-n8y%JA%IKxtQaTvh1m#<>VroQWR%2t2<bS>SRiGa)I
zlOEsk?^loLR|<!-DkXF!zD!nGQu7K4Dk{x}OLCm5ebrF>^JXTAZDKUN+vH%et^C(x
zz68B@D_La#(dd0w6=$>W2-2U9sE%5y>$&I1q;L<mijS>QR=m&@xa7(yy8+yCGBSi?
z3a*$n(?rL-uMZ?FhO$4_a=huq+W?@?>Zn12Fl~Y$T|(ye1P%ik1=ftIa>LZ=*&Jq`
zee7>W39JOOl)Grrly<dDv(2$m+ua%GG2MWYE%dwtlD;)|1k@R!yX&JbXO+_?cPW8t
znVy%%-7&PQ7hl48=~x|CdnHE-RZ&;e^;1#}zjx>1rtA>)RrKRTJ>27a`D>bl-CQB_
zs^29dU{mtV-f~wo>I(f9x@+rvZn8}Xx8=i6kI?z|g)7IS0Gj-mlg6sF@rmqPF);ye
zM*7px@)5Ior;R7?crIsl$i@bv&zIR3QOg{1^Ape7z0`FGj@cBS3BC7h#Y_+A{%uCI
zfNwvlLD~HTU%;c%{kE5F;M0$zem+bIQma^fP2NX){uezCiadp0@p>K>(rw&+oQv$U
z6ugI6J*_vp!Hb<!woR!@iAfSiTkquVL5>6W@eW-#?nxIaEV71thS@+`E}4BZ>;uXw
zRRWfNBk`kYklfVPd8*6e3t1FIvKEE!cp8p!ty19@^o9psGu37*=<cGc<K@d<$`kaa
z@;bA+ry>sJGFV&j@G;bCW~H}1gg3+bx%?Ryi3CLFAI`i7)BIl^ebab%Qa?OQT-+1N
zT*89K>tx=lIcBgau%7WDOAMD9EzR?=D>wwEY6QRPg>qyip}GloXamK?#l_&^G&3Ms
z{UTWskNe%ZM2a;pMgMwWfqGjqTb4;9-xsXaVe9L<liKwDd~qkO@DTPquGt?CYy_?M
z`2@Cimj>m5tdf6RUv9bJ1QPf{$z6QwlehO5#F&Lp6r7U|P2&h9+WvK&KTKgC8Xd=U
zU3EqC{*2@+{+b?krHfs@6(Dh6jw~=)(u5L~Hl{G1Re(P#2C|^<(r2MrcHxv?ks(f_
zJHF0xnvBOf76BqwWCbNJ>;il_+f!(9G0l=>Dr@d;_Phth_inHDNE&wE(-yPX9c`I$
zj^d%=v5+oq<6&D%@(w$Hk`j9ek4?L49#SG3j19wkxHMb86wYTf!#3ALjyBygJ&0Q!
zIh7}$9?aIjkVQfKjL+-LC^}q&XZTldL$n+(?V^lC$gE4Qk64O@>}ncH#rZM{Jfpf;
zNd+F)WpE=&Nhv54p?YVFJs@DeycmqEQi);{qu;dj6l6S$)3*7@J-oAMi+wT0Gu#}b
zS*q7Sp1$-Ps&I?Qq-Y@&tzO6o5k6Dg_TT9khxp8>35Vyo&!!zw1>$85GXxgO9tn?q
zyS=%RAh2w@yRCmW;d!Zwcf2z-HonLCeTG&L=$86E`;n=&LI-ELF6m@N>Dg4Z?Veu9
zdk<Q)%ExAF1?CeaEWzIKDGhgD>>WoPMbllIK0V{0^O(~#Th!xBFJV!DjWEAVtOzUW
z+B$)AT|@hrFWc`C_>vR*#2Az@OtT>7=(F-3Ci>06nB7{dSod-ZBU<dvU!ba(KO{wa
zN#H?<&IUX4QIY_2GFe;Y?L>Kw`)<urIQ_8(XMnF9zEn&Kqx_e!D)*yRY>(E`mJHXu
z*>?15*lmX8@ceaJX*}}}`o=}dZ=d9VwAD5}XWv0oUAL3N@|(M@M1Qb~x<4)n(`@eM
z<w@bW;=9y<rGS!pp_I0^!J76p=UU1wTGu?2jmJ8A&)0;@fZ&T~F(EEEaS}&`27_+g
zp}ER*6t@c!#O{<@x#4M%Q2QAZ2h);Kx!m~xmgK1&ZU|1bol@Sm`4`6bvrS<w!nnLe
z#g}^*Yw65bL`|shGl4_8BzAiZXGv|9C8Y;bv30{uPS!s`wfHe2gzauKdrs`e93Wyp
z{AU`5hx+u{yTv@b+B4^8QI@4Bq2nqkEZl1;jxu_s0YA}&t}&`;-xtkkTs`q}bs-C>
zp|CH}=iQMGNU4}o&0=WNb9A9#JzeKpOes^`KH|Uczwu%HMOayNf`ok@I<5cy$xc}*
zQI)^ke?$DjNHnv<3Ja<Kd)Q$6<Og&?jZQrvu5-Jt|JTsB92TE{a55MN_*QV3$M{aI
zGi)jo)Rc9-HpFO(;QO97zPH<AtLqrWz@zQ^tjVcP!>T)u<+n3Ii7YQR-AK0mP{4WJ
z>$v)isj%pfL{a7Dkiu7ES?0GUUzo&JR)`_!YQQ0X(0Jw<BrJ(sGtA{AN~-B_8uNMA
z>Ik$;*dD#&h8UnWYOA;Bk|M*;ln-0;Z`w%hwoGamW(e*9D6R6O=4L6|hFs<~9F)-#
zv^1w`Ev49ESV|vXk$7KYo?L-0HAHWWdEyC@R`t|YDW60v)dsl*_QSGznhmB`1(dqg
znS1T{W+{=<oM{1|#rTE|>;F?tvIfR?{%Hccg;Hf%tW{CpIAYkG*O@LEO$p2IfXst}
zd`C2+bjF<$(XLTEnRG#cP37yCD(ZNYQ~iTGRZNBPd<6ARYAw-pY<9_)Lz(JmvMdoQ
zU*IvLrF_`vJuxxktTCm!bP5nD7u&GebIIC~o;lPZ2sPReEK2t{^}igfv0obR8QiPW
zYsD7bEs@#nvqFKNp;O<Hu0)Nq)Tc}5mCn!}dYf3+XDJmO!si+~ZnEYUQ^}C_0Vr_4
z$pKy3I%mYL3YIl{2emHRM4j`NIR(}G+46TtB3XzfWmy!rJ{990mG(tpq-%fX$9trj
z>ZfAgE;)fdN?RaLU|B0I%>pCcX-cjkFH~g-z1~4iIh^xr$lG4Qo8#eJG)A+9mVg6Q
zx63JAb^>O!>A-Kq2&*Mj!MI4fqp_DVg)Vhl=qbwC)rS&mO+|Z`0{}1Yf_1V!^iF%-
zop^<jEZDhIYI~-Xa3G;{2v91xuh;oT<$|uC(n|#~F)lB(cWG|eB++QD=e%vA>(Gt}
zZ2E**0a8p%Y99OgJpGN>X7kE+fBxs2Y655Xi^CPnYA*df%Mbj576bLOmcKJZoZgA<
z91wEZ=w$K?gTy9~I9BZ4{t16DFhY(<suD4q9*zC;=@64<Ipy0?|A$YrPG;Sf!Unf?
z<Hs49XWFAVe^inchDncwXZic0AfNH1Ze~7cwit47+KjRh+fbE$j%I{QR{y;f6<z@O
zD`<P$)n>|%!M-LWDd&-;wT$I2eVz_#R}(RLZWq_;!HeTM^``p1+|jKZ7KO+)CWo=X
z!10>K{T9^0;RxsCk@Fakpcf~IEQa7Gq@z~*Qwb(qrmQLGLX4}YM|5+YT*=S2gQAG-
za`)20(JW++(U6}&p;Hd_W-+m(7i9CwaFe9NuQI2RaygNzGWTvr2TUHN9V~T7qz{WF
zc$~-Zt$BgO6Fy5XjkiuFx4TiM6@Tipb(%j3Q>;<8sMau3*3w+1ND#AI`Me?b;;Hpb
zzQCGRZnAtpz?S97=E#>Ib8IR!_TIYsFzvuy7{__v2jcH`Rk7S-qaMOD&guGZPhv8o
z9VhZQt|sW2o#A7n)Uk}{vJ{O4p_7z+#*w!DaUlLjh$ztl^lYN?xua{&n^AnGRFN#d
zIp9FM9kx0aNpMqq_udI75I456Lw70T#5{#Pe79fILpG1Qjrh-MTr98<=a1u|*X?fv
zv|mkN$4^JnpLCfP)=Sj#$!&0~?1qdjc01lZcD`B=H50~)Prrh;;)|fFEPmW4fFmRS
z_NFgM5|m674!kP+a?rsh3#={WVbq#eM&{;qz5UQ>y}W+>VCUgT!4sP=e7F3tTuoO6
zj_p<UJ8=u|H-D;tU7!mI+kK7uQvbO2krdYeSrFlqLA@Iji^)ohL1N9mFDvH6)-ig{
zBdYCz025R_HYN1TGjQyqwv3i$YwS}WpHkx_a>XX^UZ0#7h-!&%lbaM@Mav6HX;gAD
z83&~INWH7glS_R@l=*SM^J=FW*MmewW_<#gH_GXFP=Y{?_l8=q))IBv!$>GJBviW}
z><ox?e{Zdhh?g_v%ep-IHpRkZL#N1#yHb}IytV!*d>t#^2}^YBNoxq~iF%&2g(h#I
zLR9wF`)NhO8-nK|U~j(fCmfEmcrE`CY(f1KeApgIrWDh+S@?V$?!-1TBd~u)*Y?Ez
z*~`Ual@PXc-4tEbRtqcKWa?Yk8~q4JjJN{*9(#|3wu$E)w=e8H3UVCR`rDbzCd0Xh
zX<9ApLWT{n%@ztTs#SAqP&e}`xV2ISpV)ikv|+N@%g1czIo5rp?U-;;N@BMlm;0hX
zSf)_ki~s)Z#!$|p3@Yn%N~%+9Vz<&)#D223fyERskEkUGv64GN9~|r2iQ~2=%2CRF
zQBP6);J@dG>+G)DJ|J|*&>tM=?tPN#ApW=&pON;vn}ky`f6|<0h4JiM)ODZ5&&%pU
z=eK%hS_#|_N@6c{nMM_e0Zd+s!5Cr`+AT34T)!sO!#f{)%7Fe17W;d@3LQmmisCbS
zaUp5S$I`!kPH+pSI3tQ<t*?bLH`4U8Hx6Nm5&yTQ)5j+a?BhAb5tDV``rQ#0WvZgu
zLUBVYK136G?*H6nlotYJ$)0EL$B~R~e<dZG)|v$(7T7EKCxJtfKMNi1$mc;YEzLq@
zC`CEE*z$SRRTjg~i;+{r2E_5Wx3Gq^&;!5$+5&|S3$lLi0Ulf<<va?!*#jJV7vjSl
zbiBXEG#K9!0gcsl0|Eq645|!NFK4D|rsKXpM&sryp1z<)fR4c52QWksX%~aU|M#&K
zmIREU0wLM%KhNk$5r9Hu$gE7zS<b)D#WJ`-GG0WsC;6`-7&tOedyqvU>@oj2*aq5h
zEQ#RzaQyd?9WpFvMv(nJh5m;gAWahvnknv=(?NgbyO&BB88o&pF-&y)|2b$61Zu-v
z6uj5tU(&x61}^j?T_XSIU<fc1O*qCG8Poy&OL`1AAUOsCKJGsU6$pVq_Wm{K|8*UV
zF9tZA^e64CLH_69e>&}d)BFEp%6&{<$yw=-*Hz6I)e4lhRifXo$;wTC2@%H>J?B{|
z3|09`!K0eQbC>j5uB}e+@Uil}Y^13`&yDQY6OfgLWJt>wENL~4;;_DYX_H&d#dogG
zDHXoRduiQOFd^<>NlaB+H7O-m)u7Zp02*pwFAPGU>3r)Gc~B}MFoLgdqOlm#>9phI
zBRkXQ+&7)&MrLw!Zu+CN?>xH-raOnrY0>T-xBc9Q<r|-%#si=?sWj<f2VvLU0Vl1-
z#V5&ZQV#(}eFq1sY8z_WxpR-Mf<|wx!VHCb>wXIbpeu1Rcm&m^V&wIF;ak7=O5Z$I
z)y`P$$*Sev(aw2hubaa?yyxQg8aQlvd=Vdkblw$C`!?#j?UzYUlwv>f&(&>@Xl0S^
zooV%Mu3D*Vo!ITkmpk_=Mc@FHDh+b(8JbdHXbNd_dOL@Ua^7lu_v7zRt&lY?_;$7w
zRWb9EI9|s7?;2~c#!ABH5;AHl=6JbNN+K^MqPx8-!$qR@pbR5jwKDo^v=+2`CHDP*
z5~v|Dut51{RJeYp%xhaFXBPsGKcPcIdg8U0xRMtFBv|&j`1d;i;pRKwq}$JA!~hQ>
zqzc~@IS2TLH4Uz#N`G_}J4Rj4+GYKjZS*bOZr8Y{KMvyD!1B2c6onC*aQ$7a$!jUg
zgi5D+Z?;Y{z7y=?Cf@2%U2KQ$W!v1e6URtm{#AkzQX(=>4<aH>S}w*pATWVX>o(!a
zuY&a{iOIoKBC<Xxg$DaHc|nGrZjDcf1@GR>w`8Cnl!f3BAqICb=N!u@B;TEdm{Y{7
zhV^OVRvv%lnM>qhy*z`i?=p1w0OR`=oR*phAmBm3j%|*ka^<N<KRHRg`Y7tVC3>;y
z%CknKvvj-s^q#z@D2(C!7Xw4Od$>gktxC>rj~ea7WlJgGQlZG;n9&c1Jn&}V899?z
zf{VY0b+Z~AGQ2j$2VKz8**)au!&vc}RwDO)zei6Jij8;&TwfKSVs#TBxy#vjh;9t6
z@lOT0YTFj(7UU&->WHR{bSq6(x}}r2oR+8%xax2RzB7)XjM}D)ziHwMRQ>JGm^dI|
z!IEQKSY|GJ)#OzW48ECji!G#K3Fiq)Q{_J<=qecM<$7;K11bFtDD8jqsM#Ap{tE)<
zzzz*!QHr&ISCljr%IyiG>s5F^PVT@}t8(vO7XP~4;ERSyqrvr>)7!%>@Gbp24pU4-
zn;q=ONmR8QkDa{_AKqsU96(!xkWdCR=YC=aM?d70z8z}nbFT!@C`dSNr9H<!^=iiz
zG^YHYa!??U6W~hCeqHUK^z-}lM?FsZ#}mZ%z!aSVw{f_xiCha!8FFT@2o~7{%H{A8
z1#rM8Am{}9<wU;}KUgTBIUmy&*q1>Uq=0I@kIxLis!PX%EugS4MzG}UJ3c9sVP>^=
z(T^1~9<QmqCB^=%Q8@SYlw|3?Lj1$(7SsygcD>974=^oC^BAzRUt`s9s&cOjm|;2)
zoKdIWW7L2Q*55Zk<R`fCL#lyrxONuBC^TbTth|APxan3aE|7C=V_a<NV|ah@{HHDP
z6a?h^n2;p#{qgxUwlk1<SX~5Py5FBh;mDzIhXNZtQ61@Z7s|aAH$YaB<TC6A<)KBE
z)H~J6Bky}`J*KHNRswGGL~6d$)SPJd81mVb&l$vk0ZL-r+zFtxo~H}{+)ONY@0S|o
ziBeSDFFnIP^_Ftjs;mx2M7lRjvk)-MkB88@AxVRtJDd3H0Qez|SDRvHudwoCSCYdb
z9O{qdXAB(fv%au6s7kDUiP8{oo^;-wqD&n2+>F+wGb(1lbaT|`bnm^Gc!q-qcQ1GV
zKqJc0JTBxZ`G;1wKT@&2x=#(WzSh>@6!V=-(cYpduxr-N;Yjzqw@kzv5bEskxo*IW
z(+G!eIHEiCg-nQEei!qF>ntcQQsK8U9Mv>{Vigvd60%BfKoPW(pDg0c#_Qwq&8C!Q
z4r42lV#fD1m9Ny;pLO)KxWE|&WNQoPdNzZQdn+0YYmb221T+YlhxP&Q-z;~;5{b1t
z4M?hX(bKBS(p6niTzKvN)mRNEQ67@s4ekhKJRLt4uPyFp=?r8!zv4iZT#Jd8?pW3~
zKz{!BT5=%2sLK5Ta5Dv5@@(PeC%o$wMGHaZTpet))_M7KKHm#S;D3|=16bc5uC8{Y
zcwZ}<qqdVHl*>vg+6}}`AN_>;3lT{e!w3x2`=agkigfQo?vsg{@#GdyJ?VZ<+3dxI
zP|sb&Rj6%2W&)+$&N$Qz;2jYGL}m@v*JM*)stGQ#>UKk=g$(p^TV$NSlD-6z`fJ=5
z_+|o-9d#JA;{>*44F#EVro(ZiE(God*7OXZKErn&?6nZMzPkZDX$TwiEE+~NnMWYT
znaueg$_7c*KIeKfhrs}J0+;#&Fg+}@gSsFeE(8=cLB`~@)P;MhEsgS0HZ#Ls|5sVb
zli$bwINQ`VkUEvkcWZU%Do|m*3N-k83zL!(I4(HgzGM195y4fID0+MOvmVY%LrE@H
zH)qih)ytgc`Mm+qfdSl%!ty{ah(liXephRCvEs`xVk1$t@=WLX%300aW8-g7s(?Xw
z0<@xRZMp9OXF)0xQ05%q>AI&HaHAhJ&Aan|qxv0Ll(Xmg8iL8h4^C|ylj>O%j-048
zw-jZvdKU%0dt72kfku*&*aqqu_XC9!@}9m3>r8eHR)RpAoea{|j6W^ZJo>yMJ-k|r
zI_vVyG*K)YF^j6$fz3Q7%7fDW`A1NT<p*2TKq(#rt{hcE8ExFMBvIlJtX_>&T4;7v
z>T5rO^dOBu1g>Nd&XWc~lLBns7t!a)7ubD)a@9auJC#mImu`;(anlDlzJA8D9SDQq
zDM?D6Y(_$dxFvL&xl@pwQm<=SV7DJ6*Pil+EgJsbKCE&8cJ|9Epat|#6gIh`j^et)
zcpySKkLQrR_>)Ac-=!FCmwFdUBFK*++!=kO23+KU*=IDrGkE`@9*yhobvR*;ID98d
ztNX^`KHUhbf&5YKmnsdSiREeVa&c52$qq&24}JKLWO+sc1$LeBQOfpL!k0nH_hBLp
zI;B6Iiy(s-aQOPo=|%SGuS+$HBrC;GU~BcqOixm4^!-iQ!UmSW4#kOfh(LUhrj`fx
z7`5L&OOC8#Rh6n0ZvZLy3!5s@$vymif&fs7Te4}O2&2UeUhY}tA<CYwR#)zRv-PXC
z>T^8GRnxlFOO;u*r)8-3_6PwTJ`LRC>U%Gy3g41MolRgqxiGewERr@u1XVsd<Uu2~
zQlkfNQKr@3z@-3YdA6X$4n;;3MzEFEa}PR)<<bBu_vtz_heCF?Jc2cq$wA&J7UX*t
z(!k^j4r=^R?Ig8EAa|!vY3uCZqj+#Mq$a_O(qGV6S}}gEYW5(DXh@OY^nOz!j0D`u
zikw^s$%Q2aszb%<JSDA0ULN99x`QR*=2u#`u0%nVU+;~}F#ak+1r))-M28H~N$Tsz
z-=%LCW~J$2i$pKmQ0Ty?%ea!vSC77;CL_Sc15p)jmIiGL{VzX7VA%18Sl5>|R<|Xg
z!4=#Sh06>~CEu=0x{6)DtG+QryAOaEP&_lX(P&{IM`$67-s;g^9KDnJ*=;tg^u91)
zbyT2_=ZnGDWtCalk+1xJ_Z@=3y89v#EFg0rbNTv>F!R7=ey*?_^!w`5Suiw>iQp<_
zlNw#FBozBs>o-st`jw~!$rlqQoGpx^CKG>667<^SSG*~Gk$<ftD55I_=rPko{SJ>A
zdl(Iwh@lfg7}zEd_vaiA<6yh!A0sS^GWvrSu_w1PE_NW;YGR%-$YCT~XGSin-q@_G
z*tt^KCA@k?`Psc*8lY`Aur_CbwhF>85WZhiAr_hOzFc6W)wudDdVAfNfcDXLO~sn5
zmgBR7zc8O!ox;0F2h}G+ovQbEKxPMc$oLiN2$m-xiyo@+EY9;);QUO<`r%$BdYFnC
zz<9T7rBYJ298>ySDHA-j+E(X6nm0)twRC-dVgaK4e?5-zpXWgkQiSFDg|-mLBPE*!
z+V1aLW6+=ij+*O{xIkhbbYL=13`!ZOi!%VbP+vrmW&}vE4Jf}->vn_0@Xw7CsIdP?
zyrdXle~j_K|GwYx69?ZbgBQ_(U3`zhUNJj}J_mBlw?L4#k`xS(A%hhGWPW{&-VOym
z4IhwX0$&LZTEPPCF3Gyjq15u9y+06y3RGyC7j(psg!A$MbKdQbilA(00$nxST7V8&
zJ`e&tTFNHm`tumTFVFYaeZWycz^0|WKMBUL0m;CwgJcinnSX9(o8bXjbW$Y*NWf%3
z$twKucSyi*(9uJhhQUKfz)DCZMEU?oC1hxSJpVG}6_k%qKtGGM`jh!@e9M3gnSiPe
zNr(UxCuPZO#e`-ofFP3edMKdNAdT<>f^2udpx%q?3z(oQwunPlnL-H#p1=H_2_|3?
z>=y=_8DVZ9RSUlB56B%6bYQ%&)r}Z(-~i-69F|__s6K)UO8UNeS`ei~avzpXEQ?KO
zV>NFJ*{uV83z7;8AXb5a96IL#?16?p_mA5MHv<-M#0Jw)M9c%$z7nLgNDzx+0I=`k
zNcE7g<KSe-K%M{F{lprWNTbLRhS>Q6SRk8d*rR|I&E8<*u52Ehwtyv+)Ijn6Kt9M+
za3TOB!7PiArF|=b-Li>ZV1v~doluwJa-n=*Et`O)&#9Egp_G>q18C(SNN_?-$e?2$
z&1QQ5U1Sol;M^#@10pN^3aIq#?ee{@6hK!JMAZ8;0~6>`o6!A&XAnl7L+L>zk@w!{
z>A>iBFIwRtvSJ|mu|5VRL!?OenPL55OYb4+0|Z3iZ?1t_MbNn+!+#c5y3c0X5Jtk#
zT<$UQk6k|O0GQyHkoLXT6_k98&TorYkgj)t_pR|yE{Gk5BveIH9=Sq-Ab|ZfDTZO|
zLukeVG8LZ@9VADDE`*W8JQrx<J=nGGzs~*!Z8C*v0Tbzm;Z_inmr&4^`z$emC=~<?
z_S*<QLr#tX(92-|(+LlWE&)s^h=P*~fvS@n*bQevnFEqa1{m7JkJT9xYF`Mr{LVP|
z0Yo+gc$4M{Iu}GrcwQf<u7?Q80Sg2~cJr}^V$KI5OYEDC2TgnjCLDi;zYT4f4R8S^
z-Gk<5&_o)rp<8$i+<iPm0bL^u+i0PQJ8%wm-Qj0@Z}f;1FvCM|-&`Xj00~K3f}cV&
zLy)OlT2&x9Nb3Q^i52<#qmS#CV51!`Y54bnCcr)B3E>1}Q2$SWlnHEyE?D3R7A%DF
z5JPPs%_Bh8B0us0B>HD4YOok&?xRKwikg<Zw~$ncApSZLOgAA9fTyF7;qN*y&>^z_
zY~XyFn*iY-D3E|~KtPOrm?h}m9V|qaA5Q88G+_ne{Tm1CI3nZ$CP1lv(U$|7;0HF`
zC!|HZ4{=IJ*Iyy`ea$UE_xWeUIf$JwDG!)&;GBjaje!g#6!yoYhGw#%UFc450Z_?D
z*abkLyeEAUvhH~RfHZStiyWj!4&*>8?s5pB$k46MPPp)Tu;3g>*n;cG59JYsXMpUG
z0tJ+?VM%~H&S^9fpv@&|C{Rz+Uw#A<mlcF;#KZ1C_Xj}#INZUY0p84z<e$Dj0Pyt&
zlDJ_*0wO@ZoQG|{uQtShtQ$g<5i~Il`ZfbglW$P#<f98%xU|7rLKAMlhVwj1C=ewK
zNl4c~?7Xj3-`ta`at$@?9xki!&vD~+B|p=lK42I}pi9BEy4!yY7YER*@dUXS`619x
z5-tqHuTLHzo3@zEf=mgFDJdSfg(3L>%<|?df#(7+oB-_%3A{T75KiAHF%mGJfPn~v
zXAsWiq0nuUGQ^MsAr<inZHz49F_?8y$dLgSSqF4QKC|3v0@xWyk+M)wf&sFJ>`+6W
z2GHjg5)GgYK@T6l<P|9qnBB;bP6QVJ4e0t28(vrV1NTG;8;DB+7rcC84%v+#U}PVQ
zIzWW<KX;>f!R(MoEbINo5~Lr2mUA<ppD&ef^QQ=)us`MR8ju&=fGy`~<xoES*Ihs-
zFq{2>4(b3!pfL6DKNtY|!M}#{#U}#_BTHr2gZ!}%RY&C%EF@nOAY@l3xR<yAKri_~
zwHrv}2f_Imc`2O=OvWL=vAqNeV?+r<k5$am0y*-y!Kr8$5Hc!qffr#20>|UR7$6Y>
zgv)izcA?i`lHno38-JiMDWz|_EWp1qkVD>H2@fF(;DM6jmoswq2QW*dLJK59<^>-5
zP4UV5|2g#kIrRTU=>J!QGLZQqqQ>{&n|+SGdp9p_QG3Tk)!Sw)LcTw-7P?Z-6uq*s
z(O^23%Xi+v;qU8h6p+&h+UI@ebCy|G5YUaNnhI#QyI1<zGxpV8rA;N#sLbXR=t6F$
z3Oc_&V<GF<4K}czlxdbPg>$SFkJ&`bB#p_Fx#EzB;hA6xss@4c2U3~A_OWeH)MIFo
zCTmhzPv9?hsvof)%=UDv#63!EN;ne2cOQEVo<wn8UKKHu3_w{KG8$FaXO1<NtC3YJ
zxz?Q|Y)t-fIRrEDCOIUha4Y4gL#n}pV>zTJJWAlqT%(ZHs_Xdr)4<BiStFakn!3LG
zSH?H7li?4eXR(Md%^lrPYx5nG{a3C})YAyZ-q-YA+WBsO`{i=DS~PY=5e^Qp4wlIg
zen4s`(X5DM(7@rKe6Vf05*sbOE21gt6G(OW2JUu0_mU&1orKXXGip#N(fP^F=~O4(
zm!ulKDs|HSWmqtxA9s#=rF%vCww<=%P4(^tKC>4YkvxIv_T;xykpiu%s5Q=8+b%ku
z^@wDXlTD%JX85Fp!1?b}Hp-hG-KvekC@AEI>%Y`{F4l<EHXW=w-9~F&d!psN$@5Yj
zGWd^{IXDwko)eVR9KNr;`&pecqxdd~o0?wcZgz^uwP=0r>#eN!=>g-AdER6farF{!
zOLI84f!I8#C2%B0gq0ya0WI@2rxeR4T#a-%!#mZf^kXMm-$oqweq5#M@EIg+6ntvZ
z%#o(r@V0(X`&~Xr=}e&`?juWZj$--V<gdHdXp&^TgU>>EbXAt>^qTFI@?>aHK4tR`
zQUSh6g)hrS3_qA0e5POd;rKYV;ImUj^9OPn&?CbJX=Chz+mg+EaWM$Kiw`vz&??QE
zq?YQWyc7)WoazNoXS~-JTywTY&^uTAw0lo@oeN(l|8j|L;<Aq8ag0y6*!D0!XEf{a
z+2Y0N)=nZp{_rQi@75`@Qqc38C^@FP=Ua51tKQK9?1*^_e-SvEI2~y*y!@_V;yZ=@
zrhVeBuC7RVp=Zw@R9abmq<+{7YKcz^6|$Q=D24*c(*D8@a5$(R*yL@ghDo*(r%!A6
z99M7U9}7A~EU($V7!JJM^DW6W%nU=5rNe2TIV$wL7?D4mUoMGCz8W~GF;cqx$RhbL
zo^a46|Hs`=&!_3kO?V4ZXc$NZZKToid30z_(&K4Yden0R`p>ODPwURUziF3AO4iZ^
zYPcP+BS=GseA0F%AA}2#5hU@}e>m%|=R3wUT^;tb$}7@79B?!IX8PK<_v(<^jBLfT
zt4G;)NH|=`hGZCCOT}I2u3UW9<BH&7BX7<Gom#&B`#b5dXR0CH<leWnQi2H_`f=Bl
zT54Z)m^r=|k;^9iiTy6It(a8>dxXotg{rGwe5rdnU6DyRRG8R%Yo$0jinng=D~COJ
z+*(QOFp6IghO^Fp9Iux~ABtNV6ZpuoOXLCP3%D=YEy9|8snw!RWm}OFl`LBXPS%q4
zQiv->rQWnhYs_4hs^L5yrR1*tn%_N(8MG>gG%*@d*{7<~G?|Zlj|9PU8O%fD_^^EE
z8M#tx10AI=#=kAtUYtjhEDsOhye`&KU5Oa&Gf8hCz&0y)*So`snONu#)-3wq%IHFN
z!Q3<2EkVK@W7@w~cj7Jmfyh!SbjBs-*JEovE3ZqR#-?ud;oGYnBdeXktA!`UFIW<9
zZw8FzJ!#gQ8#}Its*py*pOK;&{c3kb&JF{~$=aM25AGo{f+(Kn(au2m*!!EX4`<U}
zWhQSW;=vPg0`qltu7iu%0m3Bit-FVly|7AUMi(!~W=KBRyq6ZAK6vSL8Lc;lRD=?e
zm%eWO?8|P&?ivw=K2~3KS0XufI(ROdY^g(!AyrO*=2q?a?1W2zta^^;7sI48vs#gw
zCa;URMXuvIM=37J7JKRI(gYLHq4qhFDDb0{8dcn@0!z7mm(k~6b+61a%{cnc<lJU-
zUVlH=SSv|)>Q2pSq2%09snQNgqf&Pkcz)`TxtAm{ps8_rWE)Vj_vhPpndG59_BzxV
zyZ4`Judr)#=dgIb4F4S3sQj?J*z=s1{DshELb!^B=aUfvdNeu5#j%ybtChW`yU?;#
zBF<%*+V{=0gNH^<`4QYJ$MrH40;e7)o3W<z&Uu~9w;%Hch`g??wO8zqityZ*GmE0q
zUSz<jjv;!T55jSUUi=}-QkmYgdDX@!7~WT7Z=9x5Y+sOQlY8PKc(*@3b?PaBH~tir
z;o-kevqmwIgIVxAw%AzDSMy}QI9|w~C0L`g{hY)`(u(J7<h1U_-BFzF+3@wgh;@IX
zPT7$6=%<akb;C{e?s(QPBYTxiRnbP<C?kKm`>rw@72O1RFRtG8OSuhp>1UkzR1XM$
zk9)oyD3{9{+Q6$jSzAk=Ca<qHH|P$(`5fnUscGDsv)Z@$`s(SK<Fjc3lU(D{aN?RS
zOGB3ZcIr<sZ|Aqw?~d-;>qoGz3f2g>-^~kGR2A&%W&25pePUw0yG+g*y#DC#dwRO{
zJ%6v^t^t$0Zt8Oh-(cFoK-2jk^0)Z3rqSSU<|23R2h;B0wdZJh;!C6JoU3D_<=<N5
z3O1Zxs=tX5m&kP@zU~j(#OmHefyWl4L;kb+c;_sBCk2ye#=g^(f2sEUs)yAZb-eXs
zF5RF^)C&0>w~Vr*T3kUkwC;Fj>Z2z7+L22FuiHvCiJl{7Pn)zcPY=#!T*vq9VUERX
zWfJv0JZ3~sv~w=ryN`7pdVl_5Hj6t``x-GA|EJdFXVM|dItBj0(<)qjP7&g?!$w+(
zv1>vn1LdRHq69}oMY<MY)WW&^WKDFZU(L!ZamfShrrKvD{=Uv+KQj!#o(6%f#Lbl6
z*9RG1_gg+HZdGPcSguSC2&1rXuCq0RG9-hS&c(kTKPrvmu;ff=PvGn}isqF2%<qwa
zPmAZNOpWg_yfHXc7k~4D8jZ`8pSw0x$6+vX@ZF_Yk6!gI^KfTaLuMGy&do@6XpwWI
zv(-?0N%&Xuf%-HOmZN|z84B(6L8O||f-lTTKX8yzQ^dC_2E5;ROw4&-rPGS#W*OZ&
zUQ1Dk5-d&rDt%zEQ@z*!JP0L&zFvSdn#67DT8<K}?&MmCEh|G{Z10DC^mhG#(6F7+
z(c2GRUFgQLzh*eZE{Hu^G1#aiwuT*XR0z$7Hh8F=YGz~(c#b1hS+{0>+Hf9ipupc`
zD!={n##JjJ?OkSQsrQ}N7fcg6X`^U*wBsa{?`A2(J~NTnUeuGsj-l>467>B7SHH)<
zvbwDgvmVEp^&$@wl<Mf4qx|+EAm*YFt+U>1+1q7($;w%AnN6W}U?!zAyr*b}uxxs?
z9<=x9#qbBCqm5mqx}KO^5iS>p{>Jk?A~i$1P5dvZ1=`6<&tCj4k7>B|TF>u^{(Ls{
z`jpDw*Xza4EMdr9tY5iJ8DT@gWBiAFY16D^b)n4sExpXHwsp#6PJ+&KVRBsF7DBop
z4gR2$bdqzoazO0CwASjZx1Gg$_j+1<OYhw-<&hvs<H*bD*G%c=9`??30dfU27L#6k
zmVJwT(Ww?9%>2XE3Y<IGqYfNYZ`W%nFg@lh4KqTePd<gH$nmHhC|B%azYU*AQgh7;
zkq6HLn=g4!O~xly;kkx$%U_=G6ey@MzFX6Mmn2@t<S^1!VlxyOANOF+;QiZQ`iR0-
zLmMxGsv9Pbd+*9teP2F$3t}U`{WEI271b=$7ONETb(5>?Y!y9qa^&-BEAS(UjPvyy
zGDXFVGv^!W9S_|w7JT=si=nfmQ{cK<<t5@QOVvyLcjcdIPEuZ;Idy9;tv2+n2XR(b
z{nU-BG+1z$upW57I@tA;DZYnUpiqHReS71$DjjErkv9o5*@<$|H)vK$6R*ab>)l>e
z^g5OR{nM7_?r#Pv`(TgjhpWtAp6Juol0joNATT2#$8+JcPQknKDlag_?z-{t@Y7b-
z*zipoHd5DPZbyQ<j|icsgx;)ILH+_)Z_oa8yy-&3ouQ|)Al%^Ji|uD8wdOx1IZsFo
z$sXpdT;t|E*%DF}4@<M~_!NJ8E-!Dped9!IH);^S`^L~RzGo`-JjPs?WqLI~v%S>>
zRWTa;4ugzNoSpI9<&RpHY-zop!*OT-Q2p9Nk1k5rka#&httu-&@B^NLVHYRjAN@gh
zCZh@!8RHEA-~w%DV>xGa@n6!*8n)-Wobxp2w`QS-vElot+wX{e4a8br-MR(_$q7wm
zE(k5Yh@2lj91yfkk@4O%rtLZ=6zF=lQ|CSExIPybPV=c@>*dD8joX;Y(S+yc#`66j
z{k|k(-KnSPN;9#YeS0V<x;+QOi}f=-E?(ql+aV7|5x|cINWW#ay`YEsNvE?|6I670
z=hcv{t`3`_0;1B!)11V-yTL22`uO2nO)bL=W!1v5x?To`k-(<@g>d<vl-eC$qE~Cq
zmU~{?i>B5Ijogmm(~fgnZ=^#xrx%sU4d>;01`%^6eLl%=U?K5ZORrtJa$9&O9&8IW
z{60A8xNEdsBye#49MrplEi{Ll+W+)Qht-h9bn}730@Ycf#EF1(rNB=8o)@#nlZtR*
z0VnS<^D=d3Z-wVF9>d4u>5giPGtp_D9^Vt6vu!l?tc3L3Zb>oA;kC*=9I_CXsG6bB
zvb6~CEitp0J9w<zzmZ3@ErULF+4@%ZLl&PR)#6<TiFYW|<FQ#UujhkVr*4KBS@~Lh
zulthXOxFBj)bR3|H>1lH#nqns`^Fmn%%2BGC<x5CZYFFX)<x*n@M=&=*=Bxe0Ly5<
z;&k%({W|IW-Hz`|WjQBY3f<l!skKIui8ro;8-%<;*R***3-dajKS6r8?U=gGfA&;<
zeC#W8aH&vgi?HkX;0a8fcHfwz?dMY71=|?`i}M}kgm=lD{5q0^wfIpFKNwaE(0>Tz
zUlCMWHoMk2eJbDhO3&R{FVcNF*>%R<S7&%j^vujjJ<EgwsV*$vrnqRs-K;Qv7hxrL
z2<3wY>`GkhHruTy43R1C=D2q6#giln#M+X!-)GViElv*@WDi>AgT&~wCw9Gh8?M^*
z>C|0#COtpj$b+9Kg*IH``(~NGo%r8o02S&(*^*x5tXCf1P*-kPHU(M^HqzvhzVQ`(
z@AYXYZ?(P0*Ut<?{4=g|0{OCeBt)|=`I<hS6I9k>d3H6ZDpeS+)-TBoGad=g^hulz
zjnT3k(5AK+EbB6NRl(;<#e$zy4LGB>tEcX}TIs~|)XR>Psv8%r0ej@7B{G_&v5oSV
zDi3_0;=HKE;M@LYok^5(sdJV8iTJ9V=c(~!WfuB`<>=S6iZ$=PeBHJ%y!?5Eg-j{L
zC=@ELST#Iz8IJEsPRqvesnxEb)0@fzuYv_of9h6#-HT_6_$(BrHO8?HcL`AO1+6ad
zRrD@lwsOp(vW36zt+mR;?i02o9HBP)#_O-Q;&>+5La{;Mjt(h*S-Ua;)CR8$9`SMQ
zn0|NrBQWdieJPVYgaThB>zXiVK~?bSlX>G-3I~E~xW}MOm}b-E?Rv>e4|!$9mW-pe
zvyYQs8-DWM&0rJUEn`XLu>><}yS9H!MOhssxw|;7sOaN?^SJroiiA69(c#6J$nsk;
zI=5cen0VNky6!eRyDO|nPUv!HOlnX!Ksu_4+c0B|X?^nPPS~C0&8q-$5ovlfCpyi(
zI^EtYp_RLmDAa!Xi`hGyy&pCeSC>8T)ebKJ$p_U%j<uN8@p_)w{451MYVNNoz^drU
z?2R1UlGsz~?P}FpH63sYjEVFKSrD8e?i+s!;)GgHg=~*+!>DY6Tn%fJ1iLn}_@A}&
zgk!Q5pQSSo+E&AACbf0glveJl3hf<X26&`C)fWG83<E>34E<jKBP)82xEPjSRM%#<
z;3meigWUU^<dEgqjWYaR1_V6UP53&{`g~__EX_++Z%;b=K^VtGakaU8MSs^=?L`}v
z?sy@p4u^BoB{x3TYSv3lBi#SP-dn#_9kuI&0$aKf=`QIKlrHHOkdiJ1X^`$zQd&|P
zlt#Kk=|(yQq`Q08hxdGE&NXxXgE_x>A*{XP>F0j#&A>2IqVDp1bUq{|4N0}2EQz8V
zEwM_exFM$pAF*C?vx4sY$xOLS-|&36lO~N9k_P23pf3olCX-LS`j_3Ae4R+fUs*93
zZ#%R!poQYGMr*cZhW&Q7_qYl5`!^Z1OdOz7^_{OdlKr_uuG>lT-a%Y1AL3q#VtvHU
z-^*;p>5}3%z}6O_BK?LX0^z7F|HVy$0wt(#BI?m1_HargKBtt3C`Jl=v~`?H8q5Y2
zA~W;mD=5!Wqu4Yj4l~1$)eiKtdF&>WLS>#T1yZ!<P1%aXlRWfagOj+0w#Y1AElaLb
z7EJ|pxj*%_!7F?}PPQ(ev9LSfa$3I56ZJ|N-lnndtYT`KjhlDVhFgGV)9B;aw8=T5
zxl#UL&}2aC3GHf1upp7w9Ec_!FY@2;)!84!nu}E_T{2nrhFOKCfBz9y=6u_EtbESH
z`NxBQW^{yaRdJ=55zI_n9^RJTOCvKf2=Ua)#ee?iW_`1`I+dl{bGCK382QRwEx&Su
z%C1*OGDB%d)jo5PXR{8~y&&BN?K{TS6c{*c$}<zW)GFP`dauWxxlRB4*YCl%c6Vw|
zfj#|ZnXMBG;yw&FAt&Y7$n*_3UyyjSI!*X4I7%luoorl}?mxHi+c?E%o=7<+{x_(b
zC4!O7na!WFzvn}qarY(Fqs{eLO&Wt<xwah{H_hGYU!9AzvKm_<hl+r+hq_*B=?CX+
z4J!~<%nTc9l|f+zzH(!Eny7D}U)F@g7bT$;Hb?QIF#YvKD~TbYAdOOnl&htGjGGqV
zK_)UIbgeJytKXaGuP<3C*Lkpd-@ER@*}5H^9<AKJA>yuDuPZ#ZJp<!IZDvetKmBqG
zU54<?PP)<hUCu?Ils9<(s_3%NdAIUh>7OT@wc++#H$B2)Ml#?GDn`gH@$Kt{|K5GN
zPgON!QaGa6{qjaCS<|EKbPlOX!v$}2ETl?PSDOw@lI1jPXt!)*D)&kUCC+@4B^i<@
zh+=kX{i=VzT`Fi%Kgr?P{q_272Vzy4wb5i4DvM$<=E5I7oZ166t@3tJ5a*xov3`ux
z7!3X7u~i;9zVS2dqVTO54x*0cj<Gd9=hlyk%O1Wqt$W0AAN(+P<LZ<X6+yk@Z$pd*
zLUpyqzZ}eKnDN7}l(ri3_Ippd+`DBN_V&$r{9j|yj6MF7yv1iF`q5IWkI(wj$9((k
zxffF*husUDcR$sv*WX5~N})?JSMH*<w^oN5DvLrTHe#*0iZ~G2QU2MaU^CLQ{noUI
zrR-+F`{81?>4W=xRpQORHj5`SDD+pNjLo$kqYNo>IcQnVaMoGxiEW(u&%Kv0lwT>6
zPHfK@ju)y(q{f<>3|ZM7zgxKtuk0e^{D$QAeiX5UTr@Yt_k9x{L(BWT*3R>ztxA49
z8JAsqk6Gtee~Y+v`i7}2WboE0Srj+15p`^KsH_fky|!*&$=1vWmL6Xe5RO`$y!yKt
zb^6>wsNfkG!TNb#cYhicmae#T_2cq1Zc!<poVFu1sB-2)G4AJNqx62rK(}AQ3Vj-X
zw!pu)>#20#ven9g3F61)UYdz?==ip2uB&$b)g)?K7pjz1nB3k!WobB{3uuZ-Gx=6N
zBISO=hiKz%eS<}G79QSm&cltnrmt!58RHmpRNOJh<YeRT;q#B#F}E-NTv{?yU-OV-
zhKTCrRe?}0s5gI(ByGIhIyKjcPEi&)wcW{*X9y*aT<3Z%Tu-ST^gDi@oMl9vdY=4E
z&8I(cTl^@ojAQp=gr><aPlG_qiZaKqYnKJ5Ooh-@QvBT5;AqV9@^YOgkD;=z(nH~<
zr%3<Lnk3&0!rhp1%U4jqN&2o)vk)fSOm4kl8K6j#xn=MH>!f1Mif$5za3(C}S>d;&
z7Q?S&zBg)^@1|dxG*!;H?k;Q_aj(geMDiDJr&eOhn*MzQ3P{drj^B?REEqBn&3_vH
zK>A$!jRrI9Y3Xw{Q(~U)tcoc+hQuY3rk@r@Rg)#dGzpDH8r^QGCj&ZB5%iXdBh1o%
z=VhZe?A1$TE?m8POztz{Vjvxnz^coDK|KCEq@ix?+vo3c(yC9{ZvKSZb+){?O5e1R
zBCdS3x!>|4#YI)wbK;`2IqXw<R8wBa<>ptsE2XoIkB!_I-yP*HP5OGLTmr>@tEj2g
z0<mo;^JyG<?)L~SKp{J)c^tVmn~`ZE-D`_@bNH@>k*P4=baL#ly_Icyr}CehMTal<
zAh{JRt^{GEilA8r8NnbBh2|Ejq}~($^~(U3=;ldr>-3gQQ}R33K!Y?%!knjkdK@Bq
z>6&J(&Sa9xDrbt~gctPcgj@dyU2+RLr%N@Vj|cU4IfXBC5@hoWE(VMO#Vm2gHYMr@
z=|^;u$K`qt#*hQh5%TNHK+YP-b99kL26{d_cKH*nxOU1`B?c|Qd|i}fk-_tvfnrmO
zk8XyrfnyIARl|Ux17rp}Jm{K!Lu=Q(KMSd}wy>@gD##B~OA-Rzjk8X8d3Z(8uG~2I
zo9YaHFcAa`0R3eO&=|d-lu06lfe)g5g+p-z;37*IT1Jcjs{p{EenRD-F$-Zm_#0nZ
zkbxh73J4|WS|WipgpqOS5dlJn5?Qqq=^k2%;f)q(l23s(43$~RXotZs`C+mTY6B>g
zp`Hwc+B<-NR9OCjpTt086<PerVH<>}zWaoq0D7}$V2%G*sSE@#1i0t`I^hY1c9sGt
zjf}A&M}RHqc>rw>YU6;`_WvI7e_!!`1@-^dMF{00Ovhn8^L%rvJX`(8n}K|CLtY5g
zv;;H^#*p9&jT^LT!k)5dQ38gUhUf2wMfaMoZTL>d)x69^em~%Eh#}|!^=A>FUsbT9
z<KwjvX^MD2;-uo>=vZ9E9UJU*bJ*no$fbH4-HulQSyO`j%I_XP)RZcVSMVPU@Erhg
zENK9w3}rMNJcai~=36lEdITt!dc%phb0k7Ai3<KuXcQO{12z<NGJbXHFQIdu#{;83
zG}##qnj1w(TLDNLK&v2#h76@W09u#DwgSG3HNYwrje<un(-DlJ7<IvzpK6?6u3Zg{
zgN_2~9-7p%1rOVfonZ;AfeH<E2r)}32+Yj#C!Tb0G$Q(gDU&GK{e>p8fn=W1CJ&do
zwFKQ9%Rk?Vs@r}3Y=(Y5dYw6(D^E}d2&^Dlp*u%d*aM6RfLK~B;5y!d%kze>FL;xW
z(R4bYJ6i8-_4TEF7DR`1Mq=|N$mi}dAD#GlH;Y_}c4gjIOQ*ZbbzM`cJa80jWO67M
zGyW_M@Gl9-A7*~a+O*t$9P7}_mBX(Ciw(5Tq^72_Xs_N|wp?v7cc@Ih`)NHpp};<4
zT{C~PPY4Z<VV6aNjkJSGG(et+z=M(LU?RGU<q(QhK!gy_Y4TeJaD&{ghXy^}_BSPS
zuDiS1mXyXt#3X%&fQTO58_%r!j`aZ`3jzB4d`v>tz#Bh?0j?h^Fw@UH8_jFG@OH0Z
zU-<oGN!sIt$3uY{Ki>54OunU2%U6mA{8hb8=L^hTWG-(HNPp2lFQg42Dsf%E1JKxw
z;x|C|L$93v!M)^#&D^uw>+=^vGx1CsM>QtGWp}$Zi@e#~$Lm9hfcHjODuS2}42>WA
zttCODldS-SiqNnU;0*Q?1OvnE%z6zU<EJYv`Tz@-iF-*3kCl05ZW|9HJG*i)76luP
zR5&)i$(5c-dz*b*1YbIUbZr8BkBo?kB@jAzaYG;<T&CUFmEPZ!()hU?R%Pmz{L$e5
zrisjJ{e1W3t=H+FqZzAG(C#BMeacYtVR|%U(C`8N#{ztwDc>VVb}j;10g;pK*@Qv$
zhp%tFFW1b^Gl_XDFFG!;wfG<%o2@C`;15g}@<<+w{h?V(4nOoT*6Itv3Kq(9pcyA$
zSOSZ-F{}+51qRdF8Jk~hz<m27q0<)m?a9&?R>Jg($+V~2lP~fWG6134oeTN{vd%OV
zcvp#Ug#c1|K;U<C<H>FI4q`G`sQ2EAe7V*ef1F4|_48~@S%haB<6r+*n%&u&L@>)Q
zzw$dmLgqh1bU6@ai<mtfz}mz`4JV4U|EUz0w%q9`@vS`>EBi22W`rz=6&a>m{ozZw
zNmnut71NQEbtq~BDn1jeOIGj!H_8dbq)U_$6KsRskH=~vsiV?~WE4Cek=Ssi1kyc!
z;2I#LYLST|7h0XR$cPs@pBqc%9?4TA>4wDBR}>ngg{jm+ljBjXag1s?${E5HHuL<O
z^9?oHjXJq<@dtj$1zp@0gK;+V^`A<J>UJvn`g!xKtR^uFYHA^%;5R35ow+MgY~VU6
zVa@k<9^BR&pllTn2&wv6*cPlxo8bSF*>vI#2W&XWjckl$Lj+meNZHPNbJ`9L4#pHK
z5Q5fk7>ccUPhMjB+2LJW@3*W0U;_7SB<N8NkVZTI!q*&@L2t7<9C)#q6dSt;dK!Fx
zLuvX?vV_j2Xkh6NL&JVqAf){KuLHtS`oi?RPBk2#P5k{8HUMYlGVh%Dq2=}r?&L{g
z3{L1P>Z4W!tfXB<pZi;vy1n{shz~E?x_X*k`)SF;1dbpu2TKy+0YKvA4FNlaIawQE
zB4T6J`N<*l%X+rj+4h)793F@jFH*M9Q8$gb{FujHowSnRVJCJ>s~Mzx)W`@6VDyRd
z0EUW++CSsxhkzufub)2c_O8_z>f-)ty^ai^qd5RPc3(Fo0YzGIs`G@lIazz8z@An`
zzaVtG;7YL(;ZSI8z<QrhJb}S7w?DMV@S+C=*v5BJc4K*pcq09ORPd%6-72*mv#gXs
z8_ss(8)VQ)EeW=2;Mb|J0hBYq?iYzI6*e(E;Q0yxHc;kw1PO!l_5_c>a4Oh$<{^c&
zyz&xQ^to<I^UYD;XoZ<PWIIV+m;7?sqf;SRr6+`6jTJQlxH>@M2643Ujc-jF`0!*u
z?g+t31a`1`_3;5Mj1(RsAk>b{+E+c_n>PcJ+$h~h1f=a7l&F&Frv`}DA`g#|l(<^7
z>yo+E12Z@v?hO6`=<3$Oxd*#n{ds_HQm%lr6G<xtfj5|hIoVtksAijP3}>3#)Ow$7
zj0Dsz>jz-kyA^4m*dN%F!BYWShpbv=j}I?Ki`|f-0B9Kcli@$cq8R5??#J%g@Oa<2
z)@|mc^zXf?56-n+Y~Hbnza55Vv;G@&yqg7X-ly5;{_bcg0ITBO(|jbPnO6ua8xVLM
zjjJhtJ?5ZSNTLJ`LZ%yakAMpOKM-VOd^n-<Ghpl!yl<~QrE38WDGDM%R}O}W;4Rj&
zj_E19pX|*S#&v*f>ZRCh<gnpmaE)LfCE&F`4M3eoyTLDd9!Ky6ZUJ(;M4HH#?JUUi
zfFq2{7MVvJz?p%d)@aXWS-t(P@q^xF!7cx-Ll0d{Y%Icqoo@!v(F471FfhDKQ)a&+
z1vyTIY%7R~4iGAK=O*gQtuT9*`d5*Ib<WML=3ExNG%*EsI2=dM^gVXXikk?*RGcU!
z1mCA-dJFLMMN}M){}ddj#$)K@SF^&{V!qHw%k9oo6_;a4P-4qJFh_2eG$>6J>ssVB
zY>KY|#>GzIb{9zSCx9wzkr1PUcSZrS#O^S_GuaJDG8jRzfWWmHup!K;94rvPfYGCw
z)b}VZka;&=@MXHE07M?mWn?q}NNNY{e4xOcD;C@~@`vdP$-Sm?{kr8~5?T=v0~<g^
zGS>ZFCDY%z+87)5nVQ3Df`z+j18`|10F$c+IAUl34-E?w+BpQ(HH`<x`o~2fDLi&4
z-2tE%lmkS7W`n7Gqs?9(V`DQAQIGTmCY%9|9o8ZV?}wz-fB%xJxaJ#N)(?aXBymir
zu*tzQ9dr^aQyH@Q0dJ7<1&)ldz@Vi$oLop}Q1Dp6YWEiha*mwnWOo6jD<BUv1D1wc
zZAGWbGTxH&VKRsknn-y6OMV%sfjtajd~9+*je*=ZM7~(F;Qm7$5S!F~m<#b`M-c@4
zJfjURw%Wk4O}lHl%zjtR4rcdidk0>~6|B=1#z-#KN&o|tg{Fb6z`)WYkvT<gidh4)
zE%pyxGRSd3t-%;1{m9c7hby_j!<z55@vWz-B~U|jbfT?z5VV!!H8K#u*YIv`ZvO<~
zcNvS-3u89swt<=ddO1nQ8n8}@4TA?{AUsx<=0oYMU8IiT+D?mrt&p-d7z=151U~j8
ze%T-FOM%h|&}rsZoi~@qYekiVTtd0^dA=|(SNrVQGr*p{&Gy#)bA|U7A2L(2>;pa<
zWW3Su|BfU1VdD;;BGi{&cVLOI02Ub@p=>C6L6%+Og5lz6l^X|XdmLP>M9Bl@u?Veq
z;NAX))IGqx6`|{dmoEn!k)g%?0w_C&L;V5U;F#xD9W0(%6A>W9qkUURY#WWF{&-(3
zBW=#rRq|~%fa9U0uR_#2aQol@jTUmBz!DNmfcJ+hM8FFKYk*EwcMM@BOA>7gi1IJ&
zsh{s-r_;89O^$|0MM0dzmT2Hz2WUs1#Kz7-$LR&0T!q(r6?pQ$$Q#|hY|qGfEaLzl
z=@`rN=8dedzv+Jg<X?pp;2@tSjRb(lme;AVVZ~!mit{>~&W9cyMB*xzkS)ljp#h+8
zWl=2KQL^9!Ha#XlI9T{D%A#1O>MzC03veiSKSs4YnHKL`QxGISXx36&KrRT98RJ^B
zq3y^)+u>4z+{Y6!(IC;GbfGnfW){STv#GLn8<`<bzXa$4Ib?!Ol;|;oI4}{oTLWU=
z56|2Y#k(TO)ENCOps@PCP2I%cbPo+BLV3hDv4DQ@l@yzL*{ic%W<V9IoDj4P79&FT
z1OzwcyMy3zphnLvkL7p)I$aDt2s8fMa1Gdgl*rxGbCrJsDGY#0v;qfNV8783mG0mf
zxB_B$rf$Gj%KBo_v$Cfo1E`32bRr_X8wd}?!Gj0c9e`BhS!64q!dkuBE`2c;<RsUT
z#A%WP;@@9pD`3iZi4!iZeG;S&Zp%Rg{aLFZ8R-TbKY#66pMa5=!2d#nY9hydX$8jB
z{HaKZe$z}5BVbA_0Qp#Pcy0{fsRH9>_=rO8CmdmSfJ2fRIJXx9{_(jAM<7XAhK7bL
z<_?L$xTRLDUFI||6q3Y&Y~-so-Z_YS;(C{ifS!86oKg(ui{I@`m76n^fohukAP~<J
z4VfNrS(K0?BzlwT4~C0RzR?N2-2}l-EC@e-dwv8Qz95lnD)fcsY?V<qvR0W(8(%wp
zIZ~A-GsAp32i&Y+aX>qiG(rdpXr9@R3uk!5??*|!oYT)GkZSORk-1D6g>c2onuDF(
zf>F9qh2-b5P@=&f^DX1Ql8yWp2Phi%1j@LyePARAYh)-h{)Ot8G@2bH%mjpsTPRYR
z!t3WbzA9j3iGBmmtXUE#>~+;WUZ`dU2pr=9{jO#t2)}+#$O5zG+X4=CLncxb;%fl~
zJf=shJ$V*DWFr1?(9L@lyv~(N|05;_PX7<35hPR&r++m2Ad3NrHe*$@r2wM#^ugHy
z{{f7@76mnWpy(CKhBmDJ;}UQlIAmlHt7QK>u`8=QL~t%V*wN8Zzl|hFHaLt8T-c18
zROq|akYG1p-fOAU)rb{T-O7wQ5Q(@<L;1q~d!q55`-Ag9z5!IJ<|j*lSvL`o=1rwa
zSKQxS&zevs0L|A1IdV6XH&ck{8L}XtF*V;mh8TpM_8oU+i@<)*(6gZ1HGzS=i{9({
zEFKv1uiUaHC#!(s*rLdr2^7b6fUuNtvuQo{`-!7fVfqON#%<n@EfglIEN=}`!I`aW
zLtu`T;GtYY;HhHjOH}~ZV+U6jJEkNspkz`GM$@{H{q6yo{QdnvW-8UA|Kk4MNkeH2
z|3lJ6@d^z`^;u4kI8(fw98>1c0v7yf@iVW*$(`$Dp#~XQ@i0I+tTIs%h^jqizV73_
zez&-pR>yt5Q0Fozz`cK?l+oYckh*r=e4ctNur4?x>n!=$h!PIhPRK8efW0@GN+L%-
zku8Sb*>ZEH>c?~I8THW|*_Eq`N?%%0ct4aPgBJY-z^OM4g3m@kg~nyvi3}y@@Qbb7
z;1^2hqCW#tvY~{>G;ml@ezFnlvprFK1VVzogXIo7&62n2$2a2ewAiySHobaUkQK%b
zrU~=`t8)5C9FQ*4TF?9}#|X}4@FS!VHNrN*O8phV3eTXN9tYH<2RLq8m%TY1?8b|~
zVdanga_Qi3Ww`tRZ?6XN3mX<Fm-T;))(hDO*TcWE%^3tSk5FE{fI|(~@b>N(LV?k8
zIB%)HAIS<z<#(22Cu$$fJlPyWYD{O4@rB8>0)V<jhAi0ATB{$z>mP4ep@h=3C*~Lb
zfHE#6Tq5o_4!g|&vvT9a))Kvj9Q`Ku3eQUi#y3St<cAG_eb<BlSzZaAHkt(<2vw=m
z_JkrL7CFmTF0&Xo1VouC%Re#r1h&+0xS^QvPY_2AQ<LOzW#U>vQk@GZG2KnExDmXr
zPQHMuiu`x=LfocAg6Kym(V~>_fY5W2R}LZz$EB0&2Km$)V1-6;;){v}r{e^td*<!h
z&yS)7gWLy*RuY>$&gDQ>Yqr$t7sIN{yZu~57Mzb#61LQ8*B*#63ev$#-~n=h0FQ#K
zu?~<Nt36@2r3YJM0bBcWDwsrqi~PBDxg#VO&~|cxXsahfq=mePS-Zki#s0t;+y<@;
zlix~Llzd+Ts~O<<bpXk9U-uW9T!pV(vQ24|U%=L0f~_I@i1w4CT+lpn@jrM9#b10b
zd#QJwGWzvSn>TucCO~Ltv0uY8zbP87RuFL6i~EevY$*_!%I6sT979S9o_0wP4i#bj
zE4L)B42~a&7&cFvLC6-pp6;Op)ht}*?4>{dT^)E>hs6(wfc*X4ZLVgizIzLTr?D&z
zuZk(yY^V}+<(G3w+k~_SAcNT)2r1(u+Cf@kgc!g%W&9<f!_&fFpY6sXVi0$O9CED<
zi6Rap;#)&9y)?%_W}}~jHDp&hf5=c4!60v@nEV`P>PaI=VrH(f=a9G_o+}4CiQ6J_
zfqM<`M^AcCo@|UrZ~=m4#%U8_AY?J3zU`DK(XJ>cfnlwH`>cp$N;2%J>=$aupL%T|
zCL%4_;m~`y-RM?ed<vWJjXs&2?M$0>M^nAd6c2*!FU#Um`gKzQ<=`9g=c58@hDN~l
zE-@QKuZfUDIG8R_rIC%Lk60)6rh)^%D?@hxoRloEK94gFbYT0E2gaSyrBB12p;KR9
za=38HETz~skMsq7+?k>RjP1!#0ni2{6oj;G)R2gA!?`Mxa2f|XZH>!vOpTPANbhdN
zEsSMd7IN*|Fo+s0oiX$uNn-ec_(le3yBV+;F$~J&LhWZa$3xPJcVAVZ6CFq|ffB`x
zl;z%BoqLz*PHH$=91*w@>ZnISVyi-U|DOFuHk4NV@H9{{piH<1Tno5WmnRYB1b)Bh
z;1cnCC#hZ8{Ry&xv9;37*RC{ob}Q5t#0_PGUiP#xNjDLv)gcqWg?m6<ZS507`mg{2
z0gKkKS+1!)q~4T~fbiTR2jq1+Pg+e>K|GZM1Sp0{gNYi_$wy!roCEBnpedTx-<)UZ
zeDVtLv<DP$J4xu1eS}XK^&4x?YJZD;xxY9d0gngvT-(h(pG0>6xMH_APe4#54>bH`
z8hNEHJQ}b99BxOZfI2ZgRoF|1Jv%V-2ry+!a{w_lq6TkAf1tp7hTQt7^!-@g5lA8B
z{NNv-RAhqqi0e$WtP_%ELz<;+SGe?Web?p3v-mf1Koy<<Rmhj&PD`S*+Z?6aH9{`A
zV$*Ld&^t2<WC4bgvoG*519);c19)I8M*G65Z5Nd^2b%3JQedRuQ<t#c(c`7Scx~ni
zUR+LtC^wE(_YGPgC>sB!M^8wGfm_0*GPK^C%9h4z%A|SWq5vaQjuj0>Q6UT@e3nfE
zhwVE%f=R~T4bpO2kFy=9pjW7p3BL>p{v#AxzegX+prA6t>+GxGK`X+{%er?XS+cdU
ziQs>80>PEZZ5-<39!MC1Tx=#`30Sh3Zo-nwl9oGn5zQlPXpv}9aY(oFUq}mhUX<yZ
zkAtkJ2h^#=YYs@kf1)C*5av80x21y<@B>!PlB&$K2iFZmxTYZNdYvPKb4_;xE+3L_
zTECXNEBU2jWT~F%3bWX@03>-(rwa%VN(3@Z8w7Uq`q?2du8eOh_y&WjKGlA7;Q5q*
z{Ln!r6ofnsb_B}6jDbzTBn&}5GGlH9e8cPS>Woe5pk%s_PLHZD56l8GFmtaS9X>~)
zeFfEx+}HntTB+=>&-hiWVv(eLxb;Enko*DVmt8tq>zi5l27(8Ny*a)%ixgh#lWz)K
z*{jxI%e+7Wc?rQo%hK8HOmQ|b4uNy80@n6P8qHj=wXNO8NN5=u%EvIUE;R_)Vu2dS
z-rZcDHgf>CuOJrQ7!Q+q1Qh#%JX#bRR`Rxv8joIKm_{8cCOk!&%2-OaDnX%sst=?I
zx+7G0;OzfY>w(Ao0?G`%zHkVufNdRBay&c$W*h_VNJ04HQ+_@e_{NF6CgURgCLOuB
zXPNnxATHZmef7)ZKL-<kwX`!`32!S2MMc1)%4Z0B<p75bl<PZoZtFW4%8%gPItRgK
zV`Y~>DK8#mWl&;43pn9thHF9E+!G#A^PkI3qC5a~H_Mb#z^LB?LiRi)TT}wJ57dmm
zWkGxaq<BWU&p*h&wLJ*!lwJS{i#!f|`-B`&XIl9Lmdwy(dq|2Np%7C3cVEBHI0lA@
zgklXT8KuB7AhuYlm+plJ%YX_jQv}J21Rw<Rz`&k#kAY7o|6D+X^dOGDM)25=58gZn
zrKFyC=0tP~;kWwkn{*w8T<*A`-jM0JN7wep#;tYQj6o;iRp58oW3daH1U-xQ3ypP9
zQI1L>NcBG_dnslJ>M8E~0mZsCz4;m5Oh6tTskQUO;8o>;Td<@c<0iL+2Nrbf_b*Td
z9*IEuINcx1pj;4Z<Ow~OC_E-^t?OZyLJF_qg7>wxjo>Dw3q5`sc<-#^I?hQd`ZAAm
z8^4fD;61m90)VWAynHMr4c<AlwSXcK#{X;OKg)O{j}O_J@6)0ph?KcK<3#~v$a?zT
zmu5X6{QCfJSSU-W0M=}WrT0Mb6DyGK0xzfjm;XmUpP}PsbvQl<P==@iG6FB=A%$Sn
ze<`m54<<@3(UxP;wS;2aAvkz#nr~sfNLB)FxxL-BRsRwAQDCxf;cu$(OND2`6SpWM
z%Y2Z32vu%XpI0`*&fxjaQsdYhe_p;s@ojIFolyt_;TZ-@;m_?8L>pHDEkxkglIRS-
z+uPwj3-XKo`SmGEv@Z#A()Vjd^tpqNW$0jYobs*eD@(L0s0a-2xmm#{yujwrI9DUA
zUEVc>{El$~*hHJ`w<m~7qvkr`4{Rt6zwhO9*mcadL9V=$1x_s@AXbzSJvC?GC-{Ro
z(T5E#1=j?aLHpH)&m%@+@Ow@8Qa0s1bWlHV(2Vw03|pi=NVzIu;4YNlb2;o{c*lT?
zu|t4Ua~28mBU!*q@B><Gj*?_anhKLMTB5bilLY1+5k7Cr8x`S2BN+<<uyy^o4;K*k
zzZdZT6A8FOxw$(ot*{y}zcd3a;OikGS)Yzm2l>0%+ym~Y9*h@@8~60&7$gs3BsN9P
zhzEjJO20TfASXG37P3)Pr0q)^ev7LaYCi5}Tmu@KvcV+WvI(Rw+{kL~FV;^C9Mqk^
zVcU7_iMvcIZ|}dEo3yACtN1s#%;$47tjgzg5_spOm$$w4ud&8`P}{(Jx?Gwqy@gWS
zaV>h=wy}-Be~1?Q!mV$R_U-8+-TTq+>zW*Q$JhIxr!K4aR@lv?)-$i>k6BLxh7zNb
z17<GC|Lt?&J?LEEWgC|pq~tOvK&d3;j-Ko^En_3b>Ym18WO(ns__3*x-_#n5B|q(M
z+dWXTepLVOnth`CI-=eq(pg(j%%*<#>5#2w`CspyKkGSi`~T7%kFO)Z_|f2>7H+dX
zYN15s`0@9nu7fM`>lQ!yJ{d%H>k1{ym7|(94jxJy!h^i?OYnIdhNoW!ci~u0LZ*l(
z@jlT--T0pn$^t}no4h{PSudBqCOV3dzZW$WJ_!aN_r5qpEGEoSJX^9v9MO6SX|CAw
zW{iGr$O_(MJlKxb<tK=frnO2A^Nt~Gd(%F5@v$M1Jt97x1GG6!{AtfqctwxqQicLL
zGzkUORy5qI6v>9%JY_#~v^wvjZ$HBl)Fd?!@@of!s|D=Xh-vBG0t-U<m=t4O2Q1vC
zJ*&>bCKLUmR~<(BUB-u93<2~#qvsZAcTe3*Q`s~PT&!7)L+iOxAe)!9r(qZj<#?b8
zoktvU|C-~THMO0xl$gqz&m%NHdc)A>_DXlYPUL)_sHwWXYm#tUxw!NYA92&R`&rW)
z1w1Fg;%IdvJC|~#oRQA+qS+IMAU}f0C-;OJ`LQ1i>cmh*ZVSfA{_y-dtmE`w=9zsz
z>mSk=AGd`tQ(GQd!V>#n_~J{^(10DoW_~byM`5%^hVj>7hZMbC2=p5nWCW4%q%QwP
z7Fms=ZzJa%ZkDL(T$i)dw`b)(H{XOiTV-)0)_f=WY~ZpNuU`0CHjxgGR(CHo-6NJ7
zJ#z0}5p!RdQri1YvP9OWSf|(N$7&;UlxclWs^3O@pXO?pZB6%=CZx5SX-EHD6&In`
zFU35<Gg=nu5}#1OkeRLcdZ%0N<BR*6<5yZpN6^bWA5-_dLY5+I<y{>z$rqfirCYRi
zu>Oq>UyhtTCn0jnVsUpRZPUop@D0yPqyA^Q)#2ka29K6mHXl+xHlK_eiP=!oKF6?C
zwN%f<aP1f3o_7@{r8x~}mQhU|``6whhc{kZKj(6GR8Zre5wo*ydwWv*NT1zNhn$l}
zXM?Y%W(gYGf7L9=g!Oed@#<E%NJ{t8e)lQknLCl&4-2)H$qND3%Tc=NI``P>y0%5Q
z1rb5%{!SRG2d~X>s((_o`AVzJY=yJK-<rkK28VTt^WHQp{Lx8rH6M%osC0FofI1Cr
z$?8(A(hLbd28x)JCkBnm4HGsKX9AMFf?o{39)36=H=$>2_{@G!bG~V@(-Sw8*x`Zc
zZjNow<kBYar?Os(&7ht|pn3XR(vOx9Zi~~!gAR6K*235MN1xi}lph>foyG4YNa$K$
zLZ0$4d=@wKl~I}3FGM1Zn%F_J-Q~2^yVp_M`w1MP>*Qxhi(b`lZP;I{==2kAFsd4J
zBPy$Vi#ap#ZYd2{R_P}m6)zmlr9^Xx=9^0;W_~yM+2+`EN~f_keiY4x>(#7^Kd)U8
z8S?k|f<45{bT{(**ky{Q)5ea7?y<95oup#v%0$!~NrOb6`<vM-Mu}zvN|EAg{k&(_
zm`<?b=<_>|oVB91iJ@q3L>lzGDhgUF5uGhr3d9;0Vb9X=H~Vn>`)9)zr!4F%2Vc`d
zdbaAqz+ipfS>K5Z;a5$s(x7jt&C5&G^P}gdQabOdRM3CF*keSB*QE6DMYg^YGiV{x
zcS_E8CS>v8{ZO@-)zb0u?#t1T-1L?0Z#+BfJZx8FiZfywq?%jWpmg_Fa+Ae5QOg-W
zC*SUBo3Ha-+_RU75UQ1i>A1U$Z;xyAertFeYx#OB4OPFZm;z((Zpa4F_}#%j<<$mB
zW>cwrsu{c(XT8k)&vH$i_SJ%w^_Ypa=3ew#)ioa%UJnTb{p3J<{$m!|Ix)b*e#e?_
z+=yeB<SFL0DC^f`nnbqWpEOdt1U!`wk#f(<l=LgTCz?#Qoc~ZyTtyHy7s(N3+4vGL
zFpI8Y&7OUY9O5^>8iT?z!>b-nZ^(jk$bhiQkfZM~EAo}mqYhK21Ds^<VdXt09W9@l
zK)6OkBPZu~RkgVUWWr?GQoFWeleF~cpPNs9KE=<#{7Oy4Wx_b>G!$wl-fk$Go9D0U
z8m^ul`!!!hc)q9Y-C@+C%|v!md;>2oa+J#B@Zfe1d@`{jp^)v~Z7#EDuau7kgGc0P
z{klRag(Lq)4`Wj2y>e3T&x<_IFH!64_8QMVa97QX-jZeaZn(A&*BgEc$-SE*KitFm
z<!2b!mRu)qT1q^!P~#S2c=kxDCgrJ5wgoDO9QcA7rAzBc;k06-*{7DJ>Tk$7J5G99
zKh2=w3zF%1c)3p(^KTR0O1Hk_U%(0fHyc2QZil{3nRCP>VbIUDL<*x7|HQ;D`z!MF
z)a6=4Pku7a1PODK2nl(7?(trn<JMQQtM@HW(UKHhE#4uyeLKNBo6VUF+waYszix`W
zyb~ot{E)Qnp%qr&T*@;#gz45?vzi^VS8ru{enoLo^q9W~d+c0Dmt~<zZT2_HhI58z
zp#^_*&8LCu0+;icxoDvuIZE+5O60;!7bJxfrB%&~xQG{D=++gFqLYiKPcG$nR-Cjl
zq=hNm*0gVhjeq~EZoka^lbU&zdODPhMbPI&CyZU@$x%~5a<$z_QDL@enfCFUnR;Jd
zzh<S!_0MmT&HA>Qnh9P9<xfw`dVG<%?K;o>NJJ`P99JJ?W2t+^gc?c*41*!Js9<y3
zMf|?~QCggzZFThRcf80!JN7jpWO+Gk79-!>=S^{$=I0gr8(&UvZJVCq3-Am>B4TLw
zHdD=O^Fm!?p0Tao#z_Gi9tFWkjz2z}yTq&fJrCpY9?6yaHbwZq>`L{k`CfOoNN2&^
zj5pdPYHPM!xuu4AFgSU}ylRSP+ai~8Xz{#`A@WL9C%+BesTIZO=s4;`<^63Y_!D~M
z=*;vUOL0TN1V!rf&OS}l=RAK^i=(|&4#`G9bLes|b`jq4)#Yw8t497U#cd-ME~4$-
zRQ+2Y{g)p4tjrDH9{b&P_7b6)eLTmx8lLsi*}^QkZPX*AT)5glk8T)ec?{2gddu{D
zB>Qq{!RAipy13V5SbtMZcrVQ(Ti1k9f77k>7~2v1>7CU6`l*|N(yh-=pRVuwF!^Mg
z#x?UH2l`iW+Rp6;R@>HEQ7bfyo0PRk#`&A$-swb4wbNvaIf>*(UAQ%8XumgJ+DF;G
zQ$KLK#Wd<9?+b_?RMRk+_=`%tUC6{P9~60dVx7Rzol2APGnG#ZEp$kF0K5N@=v&^e
zbJA#Q)|Q_}HtN$(6)?@6iXN592mJUsN_}-1tg-s|v<07XAy~uq$35n<7<zi=&0|;U
zjz!_4Fc(7c?P7AppV;DhaBZV2+gLaAmODugu78Tg<Ud8LbVvd`2k+&nQrjO@BTUp=
zhEEv3|NUJ%%qbv>^o=^%1$gw;o+3JQL#Mg@S-yHlv==#qRe{n3PJ2azPLwXWghpO2
zPMW)CPY#V(c^WCGV7Q2caC@ht4OhfIWC~u?o%f=YJnXOaqYl8tzFeP<KI4t^-#Djd
zsv0h}Wu&NmSvQa6376(#ZJVFP^LII$Cq|}Z3;3TYeQUW^b6}xsf3(?-&&t2*TB5}d
zPZf%3Ib@~=3SmyQv-m!0E{$G^5mfLE90f1(QuGH+q6j~-Z9FqkM)R;BnA9RO@+^H(
z-B2yrdTuDe!HZ}yz9=JK^$NXhk-LU)*;Aa%8hmg)S1&6^C$+zZz6a6*E+V!twkUF6
zzIRrFXvA4EneT1;5)*5cN)U+-v*AC>wQ%LraQ$m#sjf<dR+fpH<Cfu%;$;L!1VB+7
z>;5R{^=Sv3gpV{U&Yn_=m%zKo%tkc^rL?$;yY)Zh%DwrWwNMGB)#xD?4Z~3~!Y)Xi
z{E^GX{@hhQ-8DOPk}iRxEJ6FGm{)fDU^Qh1d0OW+cdoODcFU5L(A9=5MS#C$6f7Kl
zUp>2oW_*KLyenwmYNeMg*K&gnxA~gTFs@>ECBBM^b|TW`J&HNejdj_-_~j=zzD8z4
z4)!#^wUYhlh?|XM{=9MTK@TrQaQ+UqKApI**guRo7K$YQ<;;4au+Mjmj9y$WNGxqR
zZLRsWy}a=-qyGISS5sVX@-j-C1tFseLKE|rNO+csVvE<Cp86p+^m?|m4YSZlt0k!?
zd#}n@i}q*uCakaU`WkhFUW?hv1%Elt{}G=UqVLMI)E_Z9P)*T>93Vtevo+!}x3C)~
z>rNxAtA1rQ<ltOY=3RgF!`p(??h73{rR&lkwEgGsQs-DUi`hFLQC)XPZcqa7#|XMp
z$+H^+;Szi?JhR;djfM?a+d6hA%nS8x^d<65?z?n&lei7^H_Hrv-o~vr=n;x`>IRSa
zkc-fzBo@7{WY8A;D3hLo?!yU{7zn-X!qcHrodT4?C-qm_&Vfu}V`=ftg$)mrw-QSf
zqn5ORl?;+~_eP4QJmJ*j6Xg%!%VMZZXv+uYjEsrO5tBKe9dt^J8opIFD{z;rk9$p$
z%&5$wC8e4vAtXc{p{i+M<m+h|NLY!p@l6LVdvYU3Sf^RNabT**D(1TGuyTz{HXZ}+
zBle3qPm{(1`kEbPlBaC3x_ixfGd_Y$3vQ%kfo&`JeZ493DHlz@$J^onp<T-GWgm0n
zQ7bl?RE1W1fKr34OeBYvuwq2`Iq9w@A*Huq_9j_~Yi^?AZuRYf{7`qc<-CyKGrFh#
zIEZh&u^FV+yo9)yFG6+5;%c(?Y41(W<yU0w_pu@&hnfCEkMrAMy|b4bU;~&yBlDRu
zya*PFOpDg*ZNTm9p#*w_*x%fSi^u`nKbgL6&hI#yeUR~luAWy9?W^ZdT9MhKH)@QM
z&r!B-@@PmvD;C3+QOm*5g~l?zGlRo7pR5nLZlH15-RtW6bQt^~p;r4Ndp2@zHN!x3
zlHl_<gVEmjDep`)ecB&Qb`1O_Ub{sk6K_bQB=l?^FRCZU|K*&C{4udOfOp`$;V}Ns
z`;LD`UX%T2DZpG&dQ$qPqBG(kUdA!YPifH7wHBPsMzstlwsVkGG84s<$tC|7?PfA7
zeZ*-YPDm-n;uB!c+ITH$ssp#Z%<ODKICR#^Z|eo3&wuBd4KB(_B(~}cpV2assHPiN
zNn{J~tWgjM2|I>gW*Rn`oaMffP2;>+ImffCo7F{;+;_HFayb^_);YsV;P4etM<;$I
z{*YL|Y9BKrhb#c)SLk+n`jskd!k!rUD&Z%H8U7@Ye+*(X5zt!%2bRi8z;OT>u@e&U
zf&y*|=cA&rE+zNy=<=O+@x=BE_Z?B*7U-VK#^!K{U0Na$bQs80h&74SW*H|<&$U}x
z?j0f;7dZyaXwjLC3^z%Nl$OY_p2qzfTM-<*&|mpzWXLKhR%h^9UL)NjKWoy%#4VPK
z4t=MQQ`gVsZ^~CRAvPlt=9Rgds0G`W{%sH7Z};Yr5U0wT4lpDZY>I`Rv@xru;iP#f
zZ!apLAlz72dtUG~O}1s1#>Ge*u&%ujnGmbZ59&U{c!Jpciu8&s@^@Qi7AMg|L@Ex7
zr*2xT2u9c8aK=~eN7C?7tjMJ`KALjPx1aB9Zda6nKS;z|M-7i2QEl(0{|jZ{O>goM
zNW6?b;U@6Fk~v#Xo8hiWPFypq&k*)g^7gCRZh{45rUWTrTW`<Ysxqtb4<<LOMjaYB
z$~@;goXw)l4u!@5%OJ}?yPBDm%bA&gNRe!dZGKPizLYxGKeSn4{om(fL`v^$?vWNM
zmn8daL_df`?Sr%Bm`%6knEqV6B2knj?UsKQ030YcVy35DI4E-Z!bs_%vStP0GT{?c
zT6DuzG!w}W7pu0is+2~CFT{Tn@0%cDF=!J`MMM{S+@c`#n_iG){;Kr<!Vw;{@7);s
zJb74vHVP}dn3dd9!(-eJ1A%w50%_-G7|~PM6lQC1tASku2jQ)a#dNi2)52j*jSi;y
z;_cIv_eMjH`u8sIls^$s=vrOf5>W&=i|@n2|1AHYK<wc`tUf(|cT#Q`2*)cKkk9mC
zHF#jRx5m4RS--))V%~0coBFgv*62|tRwkyhF*f(z2Q$y(_3M=O*2GYWmX48^FV&yZ
zOF$KDDbNU@#j>I_@|{`|%iG_MsIqlr`SPCE**CwEK!$KhxDT0V5yw+{pEKL{>*Iz9
zxRNoc?x8;iI}4vpn%fQMy9~C-^p|;`2(-3$V6{Uzy|X2|AC^btDEKZCm-mE2u`$gY
zU+-B61!WV$(~2>;%g1|kup%)TCiYDd^@|EatRJ889}vv3p(i{W5<`nyr-XeN55}0q
z9{xnWn3HdjZ`nr@FMN6L^O_pHIr-#^r?6%H%=8888e(;-#k&3uQv}_6G(_czLPJGC
zsd|JUa3g=q&fI(%a&!V_H!%k7NE;|S`kwMS2XOL#tW?S`IV1oymSw-%iZE&oYmTmZ
ztC$zGK|$h!)>-UKDuw{-HMihz=O>8Yu!BKc5OG5x2I%s4&~c;PKcu5Y$If0;CR;3@
zeJ{I)ic3Xh<eO$t-aLaUi;r0A-7}?~9!csw8w3|9u{cTM_K{dAp1+DKd@vJTbnJai
zG+~ouYv?;bcV&gednJl|CpJTfv)Jqq5Q*^-k*H$s8%9ofCcZg`&H(w}NlEhzO_J`f
zEqA9oDq%NmXiwnJ5L+u4$EXNbOavWk(*@l&T$*4EyMH!M==Hl-n2Rk@c=L`4I7oc8
z>HBn_@X`dx%-nad{W%oO7s%8~cx<wSO18DF)QGR_5Cq4ozI`hv6b0wL_5A%wI$9_J
znPdMMrJ+;fc8shgO_|eGO^c5|KH@=Z;2NJD>t|mVoS~y(Wu%kzr&Ab2;v*T-v{vtv
ztE@P~CrV{~(sU--N1flN)*cO*$c0B0*Oq;m4o%+uscS?ku|TO4SxBe(49ox4R*6F8
zUKDm)`!Q^t?Dut~UIWg8I!P*>X+tJE*F_Z2Ir&fM!q24?A)(BJD)9>Iv*bQ65rc5?
z5I0;;J<Ch>unkS02+uAzJIo$Af6tzibO}VXx#`wNzira}s*^sbbNJa(B5%*0gljp6
z=W0*D#MhC7d`~6ob9!#8?V@6TS?BxHR7v+r=J0+UGLg4o(+IUJUaD?%?J=<`NKUR6
zAIVCD1}G`s_)Zdq=6Q?{w%++y#r--b4HrOq_Hl}A(uq*F<;TszY0c}4P3hddf)W(^
z7nR2xYhv%dELxRubANn?`J@c(Fz4U?CY{6t54O13xYGDRCrkhq`lDBmqz5nFpp5Oy
z;>TSzvYRV#$6n}5cTdPneB?h3XX=$mH6(g*7%X{pkq@i-yd`Qk;cde;IXgbvPk6z$
zigqb4=R0y!#MUv+*0tTZAo<}$quj)+3QJ6^W#l>2h5tHPje+l%t4v*j87#ME-S$Tw
zE|fMf*W2X6(U5b7opL|ycN7R-uWu`zr)w-Fnq<=l(CkU~sm))Mk#NmDxem6E@aRs+
z{o#eRy@=6uj(V|o-9*)iqzLtQsJvCeANriBv=5dL9TbX{64=Mz8(ktu6%O4wN;Rk}
z4kmA|uFCGCdo4v*^R@QFN$K6X*~u!++0?D?$L6mM@7=m5v`&WMq5E^24;a6~Hj*V;
z_=L7TxcwEuN`3aZ=F~*1-j-xM+XB)eSBjDd-(kn~6r&kw+S%5R(>#B_osThiHL>Rp
z{iIaS&rtix7bdO%Dj(Q?KhP7B1&R5)x&lr0h5f~O@WY)FWDQ7pF2vNEasIz2&fN9}
z2pTA3v>g<-*@!tch_LV2IIuhU)y9nWU1rzT_g5y|X2$CEi{ZRq?Yj3Yc|Sz&J*xlT
zPLgzJuVFFL>3V~}E7@9pVuiJ8*9{FlsRj$w*q}S7jVIcI?&giOWrLNw(lje%>u~2d
zP|rvffUy2QO*3(I#22Ut;eYpMf9e%82<E+o2|t@$BwSadeAV5Ug%ZMXVb(>y%h}e8
zT%Fm4`1}dtQ=D_3WYD3amHr<(R8&j>Btd>&>q@R{zDNjap!3jL!;%mtiHcyrwRdX`
zTv!a@|InEWEI@yP$cP|o6wT$`DB$)Gv-Bw*?PYPukiS@fipLHA>GE-2C>g!@ZQXm$
z@ix3ohDS5Afi{sVPVqa(%d%4jzS>=eD*FKuhpvF2)#~e`Z9Psr36%Ruk&^i$FXWD2
zdUsvq2Mvso_*fq&0<U>4{0HqVY<8(_7xT!DW}`&<Fs3Vet<V1@7Fn9$FZ!-YT(>Od
zVr2-si_<eLjv^m0q~D&+T?yRvfZTb`wQnGO@r>OL3Atkd&oVouCGwtX(p;f*erSzZ
zN6U7@q-El2j~RVwmPFp`?898%O}GI~9sas*e2eLaS1Tk-hzG;U*Qbat-S%4;($QL9
zo?CW}ANS_C*qk&iUZtfHA=K6&OyaeWk2H@jSKj8$I`pP(P3w~wGN|Q*M7SNWX)j~0
z)$nU<sl(sz9}r+ndb<8gYN`bVq6xDbbWW(75?cAy(CG;x%3lZ?1p1QF|LLP*RW)FJ
z>ktGKU4N#9)-+T;%pKh(TAiFN`GrRf-Fis<p4g`s@oxCFt^eWm@!R>{*Y!$tr7LF?
z(r4sxoR-;NO?r{4g$z5sz3||T7pMC?gSEZCP2kC@rgI+Kpl(5`@BRD5X6~^?%fYX_
zKjVm8wo_O=6w;CMA^G23$&SxgMviiGwmew>rd=7|gjCo~A^GbGc_AF>mOCQUwT!Pc
z=XbK9FKkUKycyK`7TBg=x?Lu33<AKF-HW=Fln8%Cb8N(-43bibyPj{d6(_gTpbgeI
zdKWvQZ$IM$V@99wtkwE=LR`#$eD8cvd{O4H_QQG``Q;_ryxfn7RQ2^m)TRXGF@fMn
zt2s3^^z92pH=8eP^7L9Fmqf-;iv-((yKMtBvO7Csp~@9@)I;x<lHBr~ycPN0V~u3v
zi%~eL$idu-du4pagcmzf&HmYwxeSh0wv_!BJ8UAJRd1p!7ta+qn>233c&IT~^aDZ9
zs|WPH4mwQ$a^JNDHKOWSEAQcy9=7g)gdyqteaga4?%EEO<r~T<=din<fqp#g-k`T2
zCn{KU+_)B|k)B3AQdi@mf6sTD*sqbM2m8?Muk-@`BT3ahs=H6&Uk8Ii<EzwLik+BG
z^IV+FUmBM5Vzj;P8qE?7E&7-f*?$zTl_peMHn{9uy~p}!<hzoD-*YVrCnW!<d4dJ8
zQ*5QjYo}&QFj#_6OF+_?o3#I(VL)<#rPPLRGEVzDpRMPGxtHN$G3U(<O-ErA*i=c1
zmv=vS+|LSh7T<6+bjh$rmkr!HPg)GOw>5K{HOjbB{(=;kaZn2zM}-gu7nTU%0@1$I
z4{h|&%t<SUzUB&}Wv=;h>*tv@I*2EW32PbXsn6uQ21cHPta~U}IbM<`e;d<lMRwoK
z+Qnu*Y|wj>4$nDcCeqn;;3L!t?MYzBfBtn&lYP0(jYYz(Ma*To?n<CZ%_Q;HLN@r?
z>eebkXB%EGU);y``aw$M{7kn@yd&HcD9(bhCZ3+4g>rd5<kR>HcJzpfyU1PFGk$%1
zr@L>!YGZ#w1YFVX{C4El_g~0^*|9>+@0Bm_!g#KZFwuUyvSy=BhQq-Y7a8tD*Naj5
zs@z-NDz8^*1#um{iORcbXz_7xRFUSOf#LW$2&=8|04$nw+i}W*ec2Ng`R~L9#VbVp
z5P%`6r}$or5&gz-`2hrn{OUi-8Cu&HncXq5<Cl1Pl+>lq%&Ote6Q<eD`-`bi>F@hP
z2K2J$Nb*h~XI9m>Q61aXXQ|t~QQ+b8+R16;{uv;sa)mPFR*SN$Rq_2UBGO?wziBc#
zTVlHMt)3WC)p_rx`_uM%@a5TrZu^I)NaSKhX*CH0OD<7n6f&S0ter2I(Sw7So^tEC
zdFN?;>9xr2K5)~J-QccgM1UC2|00fIk!9Wa?jRkZzUU=j<1PJJw8jTGMwWVfGuQ(b
zEHigm43CE$CJaS1JIN{n;k@w~uXDzU=kznQxLrOyMZ{`4h#My)WGMK%x73med2wVj
za=fsxc1lihL1if5TKJEVUr5!vMoZInvcw_&KB7zb`zl!UcNYizZ$6k4_K4~U243=M
zf?`JL`1Wo2-Xdz5ZYwTr_&2PXn{ophGkK>A8&q=?gz#5bi_P7iM~^R8ba?y?Xi7S!
zJ+DMdiLke`RFDw}sRP=~s=|?tP5Vaw^xEmLA|<ajk#@78d-fTxA;F^tcY=NxRaLN4
z8D@wz;}-^m`+N9~WfA4n*EIKuaf%(J+T1SXb6Mzlvp#~o=dXus<p~XYG>6vI9)Bc6
z#`Y5NzHZ_r9Xjv!O5eVu`}t6y7%*!uS0$wF53zXYjV1mE6q5K;-p5wwCPLvIaZPIW
zO8dF9#XwDgUtrr=Jc+4#n$pz~QaW>q;Lh7^j~_QTw@SNW1;c7lGEJQGg6q2i$HG@~
zj6?R<1g>R%dad%7?<41Z`8bUXDg2}vNYbnqqe(&0EzG)avZr@5<GaJ|==QqTied=|
zp-SD?3eBm)MB}O?&S5<3mlo({{Z_wD9#>d&yr~M@vQ<tmVh71Y*<=BCW2G~R2pLdb
zd%I`X{6zl)Y(_?$=piK<Klk#=-H##LJ`-(U0=d1}n;M)6GZgRk0F^PUE4LCqO)95@
zSX?>gAt|jCo1o4>IH?6r_$c38RW0f@k{BBFrCTI@U+j{WL&Uo6We>V+;>RHvzy8tv
zfZ5Y)5~7#>(f?Sykh6n!Bp34d-i-h?k;7LrYcnl)qoSRIG<bx_wikq1ey7!I+f`Q6
zIj8{@r;Bl*SwCTaNU@F#VQQHDkpG9+gEkYLMEF*RzUK3G*)?&qP7M++V#DM;B~*W4
z4?y}R+I(-{=G^;>?yaSc5lF5o3=d43;tDdkT$P@gs(PSKttcrFwnlk_Tx;`LiqdYW
zz(MmN5^()|@0L<nPhKY*mnMFZ>*Zl=q|!FBZ16kUy<v}GMPI_-(p;P0@Lt3DepXhi
zP1s3>3A!M{Uiwx!dhygEf3B(~{9Bo}Z@gF5`aU_vHpYdvw^Fa~<byhT&uf+cAtc1@
zEz@PiBYIsG-(0cR%{F6Juy5@b2sf8n&2&}Ii!HTYz9FxCy+C|W)6_8wl|p07d<m{o
zWx(e1;Hoi@z1|FkuWKW91>hqdU;h|92&}WrE*a;tTWNId%L8d|nN7=pN&JZe*`m_V
zv{*Gz18zZ_QlLC}%gd8)|02jw3h(e9x@DG^|MMktr0;?x=xx2LD!ID8vE`NPC#bi!
zdxu9JeB(y9b@djbO6|yK*BdXmYHFF6vCXFT#<PRz$p1>D5S0Hma<VHu6@#}MZODG`
zT6Q#}PNfm7-#&KL_OGFXT}-oI9dX~W9g|)c?7zIU92I0mzbZ<{A2Qo4s82J!{*3Wy
zkWigsSc;rYa}<xg`NO4gRk+o-1jX_Eahr`ygb>-zHKL)#-!J=4*r5%byf1Y*Tw_>>
z1aupMgSMvv{k?kBCv)-W0PH|N(P#LhZ_)LI{2-yL>Ze8u2<s*PAFLLB=%cNvKaRqh
z&PJGSnW}$86CXQ!P9`fJ{jfCfl1NB>aa|R5+!Wa&ONsp-s1D^{HbLP+Ns8%<X@@mi
z;fRkE?}@gJk6AyI1SviD7cnaPxPbe0f@;ug&7@SAq12@y{{?-=DPeq<Fp{?8WvN2G
z3u#u<?9$BVv@Q8Vxgn26yxm)yhUOa6D$Sp0DsN4ohKCE(9w6fxGpeJ#HJ`BHYPgnZ
zTGraEkB_q!$JnlR&6w|N&c?_#Vpj2RWsvoekb=M$q!-K5O+N5D$VI+hhzqvkdiS0H
zqp}uzuW?JZr!S(Ol2@F1Sa=r^(OU1rr|na7SMEVqe|B}^PJxi~=~1Z)jiqyRY)I6W
zl1IZOO_Mpz4mSxMsLD_NU+ukRP#r<Hzl*yC5?n&i1PGhp4k1B<C%8i(xVyUqhv1f=
z!Civ826uONcm7RsUOA`EmwW5ht-7~fYJY&)!%TNi_w?%3zqOu^(^#7Bw{CB<Dv8oA
zJOy|HkzsG>QP^a?e?!5jb?eaR-jwD5XMH!=3psPFbwvy=Dmgh>9%4S@jLvMt*03Pj
zVN}uxbTJUTRqxgPPV+c}xJtoxifUuUp$~9~K*5sZV|n7$gXO0m<&v_xKk78yg2lzk
zd$=29rf>l(<Ub3y$eVXR4*5(}d_$kdAbgLCy$e;d+G?Sy=Pn=(Q6y<8dq_H>Lz7W6
zW+7Zyddsp?!1)pJ#!IUR#)f)C&Gy?qy>6$La}zy$9bxkg`rBW|MYzE+w<n#c)9AFJ
z+CEu3!yMk4x5CA!-qjQ@*f;7w9prxveGOwrZN}+bEb>KZVv4^p<u;;k2gkr6WXhkO
z(>#(tEOpp;hbcE~$dxWhR>N`Lx#Igg31MgPOI0>1hjY3o&|$n-{F`kQX|NAYwZFg$
zqy!aq=Xqfx*Ik8`-hd#lsy9NmuhmtM%M9%0nR9)K0pd21S%d7bK|f_&w_q1o<a|bQ
zfyin@Sx!{gNQTi+nV{L<?<zAu=J!YBNWt!}bZH#PcAg0u#@>8Un4ryCHC@>J8UB&L
zZPG`C`7WZEFl~Z(Uac~Ad=FI~h&Aky=b)ar=I<#t-o`|7@DLzTst+Y{_?H_k>HXLc
za<p^P>DM=G2d7Wr*o+Qyz%o&=g|QpV_drD{x(*vSJ~N1%G+17B&Sd^=X?@0eEAu_7
z--q#Dz1!dTiz+r)cp-7rbXg38`|k^Gm{8O=fBra@8`<`%zJ(8X5?t+G^~HBy@AoeR
zFSvQ1p1K?1y3gFe`Hkc#zgNdyb%8$zL}&H&qJoaK(qA1UwDM2R2i8XTBEUB1VPDW9
zv2Y6+gQhqt`Ar3WCfP!@|L($dR7ylsjq%5g1|{E8Bl*&VS7f@M$94xY+Ef`G=f2)y
zB;dU);$%&@OzGR=6d!o1WNGkgOW3i%l*IAy3VWQqNE#njTCP^Um17EJGbvVHewv`{
za|{qe(Y!*lAcI@vVBPV|)Kq9<ex-UK`BMG>eJA|hhokkT*tr$cX+nqC$HG|@IV*@{
zyLY*SdF`tu*>*^8?!^Tz0&nlwkWM>Y%fmyHpAD84+WMnAs&C`A@hR;ikIQ_CLJK6Z
zia}sLi1W;b98$F|G-LnqI>6mfs#)DFWZy@(ic`+o%rI!wqSItL;2U<(9&M%5z*Edj
zjpnd$9f%>vMR2$shU4Pmhg8KNT-8pzMk{Wf0=lWu$TZE<FZ^76c9_EV`xo}RZABuh
z)z|#ip(fXEeO7QNanP5x6B?$YXj12yEI?W`Xoo~lrjKD$%NH7t9E`EVWziBW_!j!}
zJ5p)lYxPpP2zaK5Stw6$st*Kj45IUmggX}QaTI2;SQ~dI^XkMu%apyT0r6J)m|7c9
z`e+egt-7+PC-zrk3-crmg7nU>lOp2pVhlT#?IZ*F5(QQV1hPEJgIBQlsjNk4(fUe4
z+Tu!Ba9}^!^C-z+YBbT!oEaTvEJk6Xp3665zu&#^B3Qc-=OwiefKQpX%iRgRzs;*8
zY8B|2YbPCqWeWUu0~Y0<fiGrNlQl6}p5c4lohOXbK}^OZe$vfFFi^b7Vp=r|@9pcn
zkL0n4oFxFSyQ#(2TFgzVYW8`jGQ!E)Pu%{ZCQUDyaCyj*|5~#{SA1>n3SXk>EHjmb
zzkD=%MYSuvekHd&r&W`6x4fZQkYnzrS#75tTOIi>&$HI7AEczfrOfct$CMBTPH4uF
z;&ius<Lw!I@+#BWudZ$(kv;#40NU(jhBGPn$CdeHvGun%NFB#YbWMD(Qmw{|9lP0)
z3CUV(XKgBoyX<2xfxIrlN%xoG3_YEc`=gCN6H^rRa}Uvx!M9@BPg0Yhf^!pW`Wbb;
z7`Jr$1vHG0a^<@NsnMA5zw-0_=Z{J*@Ar%Sol-5K9_3&!H-Gj@hMd>B_~X<Y^%@uD
zjn_9MHERt9U%R(4`6rT>@Dde9yqbHi`LiE}Lvf0Cu}Q9-CHP)#71jzD6VSw%=}gh^
z43c9kXf5pVZm3sv^R|<X_R3ckDIIL>A|8vM_J;C0!S<mV3>f+oT#>m7y#A?-TK(Iv
z9=8p8R<f0vK?X?yzsY(btx9Ecv-tG<{002!unf`<pak0dTp#yxtJJ~8tM@(Zntx~d
zUrYPB(*p9u#_V(ffgMz9rEiCA($4o152imIjZ(B9*i+oMhRQ8a+=Y$I8R>SvXYn+q
zD(^B#=pWUZ-2QP>j&b=3WulyJv=rtq5uYkFHN56laLqO?PLSH8Ic)IrSQA@GJdFfO
z!m;ENpu%vN`auGg_H~L44N3ggQYbMFO511$yOv965ls2Q+;c<cG-!iI2Y2GGS1VnX
zYbGmCezIoN8)Q3Fe<!_Fkl=NOHXsCxzp_%o17;;{C>mUgOV7`rEF#eDXB0e$_Cio&
z4Kc3BzFJF~Ca+}Q{GHimANJpwA`j%r#`C-J7otb+-Cv|v3IKsVK9I0mLc+&9WC6zN
z-AWJGEo{<{-%O)G%NtTC9!ht$YnpZ?ao!&#92V3>j2&k9+ECaSYkn5Wqe{15%a?Y4
zr9MTZbwKFzsy-3*e6FNLF^VrT?7E|-aKG-08V{qB1y#`89@uu;h4!iU7}iLZsvY-R
zL3e&!_kB3vP-3Jyo?Gc~Q!j4O65nx~wq;M^Z^~rfL*8Z4y{UcdZO<zg=wiUu`177W
z;`g?<z_qw{n7t<b$LuE}p7Cv_fu-pgwzNLQXpVtQmRaUahh=|z{jQ3U8bE_wBu2p+
zw?1QYYrDt@Z)p6=&?xzSIxGgk<|Zjz@kiKoXjBpP?~&e=1ZlD_@3*enhEt@Z8w=bq
zCn3@U4h~Y&4Zw9&jB^s)FJy~0IgmBV{VY?aTZG5CWAexP1U|)I&qWDdAc#1_w!nKF
z3{lavAOL{mzWmWLrups9E3)$z?QThaK<>Ho&*I+O{;>t*m<g7Y(N60gl|mm^`|Xgx
znWcGaIae`Jd^E?}ib`z{ydPfDZNIH159(d6B1xw@ndp7J_Iw(#@*z!qoWaRVy4t=#
zy)4=wg0kc4u+R8OZCE3!pR0WU{~xZD*nszs1tR-Xpp1VNX#IkWQBx`|4<4iiYJ}p3
zb-?p_Vx2v(9D}j+tFnC|ym55?4QAtI`K8%pzzH(4cmKf05wnk$>N?-=Z;S$?t?IWK
zL4)LTXt9^<uM|Zmj-E{mw{WFM?#ar#t4W6?``RX@?gR7dF)ZStwiO%rcdFz5NT2l{
z!zM7?RltV0UC~=_cK3Wcqef~C&-@`*2dZ4=A7vXZ2eCVb?Ah=s4x)rP{pIGOqWHnN
zBVx8a?4tW#_T71=MmO`@XEzFKc4sOTWZsS8YT8=u@UX_aH&8{cSo}znpGj+fGq>UX
zHK9%FMd%rTbE)^w0<Z1e-+uEATifIJ(6yq;57Z&x8=Tf?s(P0!-oSA9^z`qNVgV(6
zJGcKX2;E4xST)*t0C`+{z~F7m;t09C-cek9<V#iDt-If71H8>{Vm~3p*#>LKowo$?
zGev9mz?J?dT;xDqI}kYfbprm>+vi{PR<d5G6LqZLXf9*b1nSil9xUa|QX-mGk}cQF
z`<dRwD8VSbt(&s5Ow*09Xtj%3?YS)e0;yX5qIZ*%CTi)V*S~~u7Dl+~o#&F;$RLz{
zf|m0)`{T}}ly)<|WgBa`c_Z8P1TIB$S?3z~>g;C1lvIw#A+e3=-hXFDt`vT2u}5c$
z+v8BWrPkkk?(0cj{Dja{+y^>H&$e(x!@2@^Wu(k4{8kRpSl;eg9^N!)!qw7NC9zJH
zO?cQgNc`M}d3XSu6*M8#r{fZC#XJVhi`~;ZScNAvX#J_2d2EEnFDF;gh~mab#y&kt
zJnbio)Ea&Sd&hPk|4bGA-Mm+SE+yb{UToFzHh#|4r=chro2H8f=3JyJN1sA2TWqg}
z_{A3>4l?rBkC>Pl8zYU#kD{gU>1+7SUGD$mLSf+P?o}%m_umP$<OBmG=@Ni>{b%C`
z``P32vNcsf+_$+M70=fHYJ-8>R$I&CD|!>gosGb2^|~DH9@BO731^hc7(1!3h*BM=
zq@g!lmy-pTRjKE6-t%Q%^~m>^cRoNX;~+)G{WF#+nakyaNFuqluvjW*RFcBYUeeu0
zFyHwmU1o++D(<6OFO-ajo>$XJvDo<y3|dA>6m#&8oshlPzT!{MW-CpvxCkbyv*)YP
z5oJ3PPv&1u2Jan!V2i>NcI#!%ZpK0olk_-N_5gDlL<F_L{28#j|21IgwFkfGm{(9{
z2lR(m-1C>XqUFz~NAczLLMnQXxLAJ_`i(x2Sv{*3dS#c-ScUiU3(4_yTLr4W>vK)%
z?pH-m{*enHrKlZBN@sm(NHOKx^0yR4ZVGlJS~QEjC0;~5vje6_x9RPv4^gSXGn>Iw
zhjg_ciQ(GO?r%plRD6`4JZjWmF6377T<LQH1C{5#SNqDL>7A*C%l-n#ktfbixy<28
z?8P&IEt<OhrLf7rK~VuA7yg-);1%@pPmQV{YqVEb4zY6EeU$O<*QJnq_@`vR(bI61
zUZf?-GsD-Q%*KB)m}l>Al)5GF9P7HHVs-YEb?+mEBPu@Kn)DR%s((_Ns~lz{o(6}4
z#iC^9Q<^nkQ?DnFFp8{BMoUU+kNIchdpgfQiSYq52Ev~i<DJ<5vKjTCaH;>qi~WBk
zV9q}dzQ-%7=>a)I{F}IYt<@SzJP7r}N`nN%i~RO_oM0oMK#XAOtKO%`APDot(n|=Y
z6b{Hdk-#xR@fhxf1=3I;k$(_=|BrxSLT0!Z;`nW7wzw?0NX3=`xt;BOj~g)pqDaO2
zn8?Dy*q<LGM0Y)zYWs1tRu*}zvqT@~jV0=)U;MMu9$Wt2Akdd1hxvETi>40pTv9w{
z5Y{XJLrxC9r^F`^iw5hmfKtScT`p;l5bu+|_!o~cP(F!JJNFj+&{<@U=`5Dg`dQM0
zposi)oYea0|9tNQz7N;Zqm&f}`m7#O{IVS$Xf}*~W`iJbaZ12B&%bHCgNt+H0Fodd
zZ?2KxS1=~TKejEI%Km(h4Zauhiimhe@R;GkS*x$`1a6XpV<73}pUPDugUVgtXk9No
z2f`VQ?GMcVsr-MQ{C~Rr|HUyc`6wCq=O+O2m<8ipAdcjZl7JOP-o8^MP+8;gFHr)|
zT>H%3djWfSe7F~_{}oCe3f^26aLhQsLu2v8GY$*Egjnsgqm&j5;m|(;+>uugG%ddt
z{#}mpgjKhxqZz&E)nl0SBIO^8;<MJ=!GFS_{}TsppJ997*I4S@tHNoynZ|QEn`bq5
z7XP9_>!#8yJRR{uJapzDhUa3I?C9z<0!8BArBG=qx3xh)QyqEoYLR?rW*0M~*b+@I
zp}6mmFp8&XUSqg-Z9`aWaz5vY=uqjiWs<Ng*vIW0g2&yp84|<0I@5F;$1`p9xO>gr
zn8cG=f+Kk2TL-DDl}T;KvN?g5^68Fk8?gkzCVvC;Q#agpCsIEzVD7Aq9yE4me#N8O
zAi4PRIh9J}bt>Hza{}_I%L?+<R_&+bKGm~KmDq@mpi+b0DyuI87*{8&Of}T52sLMg
zZzHptV69C3f;)!w+K!Mv?8`|vq;OKt2fhu9jU11P>EI@nM0`vk@CFRA$V%?v9}tU9
zYrU6|e-kAfJ{VZtR^7cRFu<9rGx@Go<1)1)2J-icle({&{v7;KrPuH>d!i=#o}<*g
z>bBB3F{5^!tL7qb_+?1w1x<v9p?2~6dJ$`c$|z0uX<SB;xMR0WZG)GeDk3s!UYXXN
zEOB6kj2t&xOxGJ$?9P{WSogA@Wal*2?g5{TAoe)pQv~gHGS<LU?vt?Hg$A*Wi^Bo_
zo;Hk%vx{qQ1M&4l^P^~rnF2>DiE~xn#yi*M(v;LHvuI}b<QNysXR+M-&XiiN!R7Zn
zmLEl;ss@E}>l?=DddePdsSxeVhB_f}4zBC@DJP;EEj4A=7lYSpYX|&NU1gsLyC?<}
zkJY@{$QAEfQWiimF)@$~Dg3E;cO3#VocDIa=2fe-{hn=PQP+x*&xO;~C)??;+uJA*
zXP2!JzJ9>?LI2ggInhwYkNk&QA;;p6j)rfJDIV~SzH}kSJw^CRgB}_0s_F=*pJg&d
zwcy=u3u~j*sbBAd44*q@To;`qL$~@_Lds`|<2okEO64NhTvI}!KUq{;3mW-^##b<o
zOdy)_p*M_Q)F1Caf~D%Sab-CBG{&Lx+?4x64EdQx42f~$Iak>EeY^ck_gC8)N0QDk
zbcyxuL*b&Gm4OMPm)~YiFMf68N<#)mZ1=f%=}goI>sC#{qlO)qZcLprua&rF2-z{X
ztoJ|^Yr+H>f%s+KT5k3&xQo}~p>%h*Hg?$(n1@fhQg^#jrNrh9XPvxj+{%+4edALi
zSM`9XK>?kDDtAaV40h$;kJHCd+vB;4o6#vz!c%?SRh2~cPF{E&#W$7`0h0f9?fHJ~
z;jX9YD!4*%)Jo!a7UoNZ2uP(~s?YE9v{>`ZYJw~wcW{xZ4IrJz6(`C)K*KM_w=a{p
z|D=j%zAhg;+xbb+*wb&@%j)-Hpdlaen4Y?Q42@qO))jVp?K^r;Zx$ne1aN(Ic@H$a
zV9)sk73bTO{{lvzjMVje=|RDptk2KJ*rXK8P!z>3t`;1HFC+Xfa$`m(6u-<%1b#i&
zxR5%S>tn_*Uym}pCY~Novd2AC^QX#M+b%!4UB|-~MFC4l_dZ6d7P7s_ZXvi{U#|XJ
z;e9z~_+~hyuVn0Q>Avt>V;}||Y~VG>qwuK66^h$eEadEtM3SNJyx9D(?tUreSg$NS
zaQ0I^JBFK#HtS@KbMpkAf{Z`^Qq7sO%V9<G*NYE93~eCc%Ncg8>q1e~0_JW6vk~Fb
zk)MsX<)1M2=6;I&N_8wt`MAT?L({(cz~g#rKUL-kL~4~p54V<bpE@|BOD5IDJ))76
zI^3mFhnd;NS?FHb_ATz-n^kCS<ED_H#M06k4y!$b=+e^R-(`(o>4vzcIFk;_chu+c
zJhLK#r57|lQ{%hifqa|0DJ!r#tJs2FXBb5Oz-q2u@o}bRk+)i`UfOr;O_{*K$j@|t
zn}qmMeY{|v?|q0iLU>g+)Dv^QV>g#Jb*pbV8ORuqpJO!PW4gn3Vv>aCjYV>?#C_pl
zp0&{RDrq9>1`jiuY2gWPh5OfEm}?(0<EU`+pEPL2VlSER$Ex$^To~3aCU;o(r?984
zHr((PHT$#;W3D*@og(c?oRZp8guo}uj+!vt1bqU|H%9^ck>}gS8GF%;_0u<;u+u#)
z7dPA15dCxYpwoj?chMWGiykp5(fzsy*)101ic+DvGy}+B*21TZIGeZzIaG{=I+aF4
z7n6uZ1KP**FS;uDX|Q9}N*D#}WONsPqeLteIR?J0tN3@m5vAr`P7<U)r>v6}WKoS*
zn=eQg7Fs8WBR(jbpY!Iso2y+%eAqk;O*=8cteX8KlMr_I(Ozk{v*19<fH&iw&MN}X
z9Xk2&G`{qUv;X#-@J%h{rf!_qd$*yI3v@_Z!H9X#YL9)<!OJV{H64OODP`6-)<rD%
zZ&dwQT_jQQZhWhrImt{*tFN=F=p~DEK1sj8;y<?t!#2e?^>P)V4GFU^&?LtqMnd=N
zj}hrn7+vtbGMLIQd2?64?#23W?s1XU-E}2S&!^9y-zYTscCn?lpV@poR`&O+9jUy#
zx4}6}Zb({mJ~}G%L3A~#_;)u}+el!KzfSQf)<?!sGlsaz&SAa<NUb|jb0ot;BQH|=
zcvK^n1Ccl_r3ikLnVwoLKr-451~!Q<$$vAVq5X){BX2bjZ$uoKn7TSEq8(0bRN^Xa
z<s&#kW4#s1E4)Ehxw`^?xO`<jT&Jh8$C@)#jJdY-#!|w67td54mhH~?+mI;%LcbIl
z{}0~|)``#IDX5h>f=S%Fv7I*{P7xCsGs6SS-~3s=&7tOvF?{Cy>13$m7rM#r5NPA*
zq2?~9UeicLTNx%JOlfuq$9o!li6|$=Swi#Gr96W4&sBuHB9pmXjTVW45pKaLs?ZYg
z#c1uCP2~my>_Mn8nQ1RxT~0DPMdOd8<D2?%Y`!rtt?NG?H;?%u3nfHr14(r$@WaI=
z!X0>qFFF^;T*X3iRx)ud>b`~OY@*OOetgyDX~e>vN{Z~agqx+gG(Z4NQOL_bPh151
z1*p!TbSbZHYnT(B=-ax<m9dPoSX~LLXS$@n)|mR`vK4vil;@K|Dr|s1fScS=Zxqw^
z3zOYP4*$d6kWxb&ms%J{h=%(M4ad4Sn;V_y6`EoX37PPY{?J78h%5?;4Jv7=bSBRr
zzI(<{<{$_<i!q5m4}DXsD|Bn?Al7}IDaaV27Zcm8DxSS|_mn(G=)~VX>Z-MN+xf7{
zdjrK>(mC4PYVu60r(tLI>}XyjAX_YXVeFz=3rnI>2ABS-bph_y+diS5I>j@M*SZ-6
z_Euf@iU8(bO3T})k+;uE3t?xI@#fTetM*hxkfymC>?v840^PRMu$_nsIK6L^-G$yP
zt)MKI>=wN=*hEEaoFRdNacpOi)!UqgEmIZt%IKO_eLz#e`zwA@&eVykW3EmTdHJ2n
z!og$s)+IO9qr;bp0c}t)b-1Fz?KMRs@2d%eZroAk7TS#(_Xk;u-T3-_)r%`)f~ky1
zJ_J}={I~EZ&C~d4D+@=NH)lDWcT~M%LadQyAvroebrY)6Mnc&%nneWCN?NBR$)<h!
zXtzeG6OpLF+pSF~DI|VwC&p>J)H@t8n$Iuo;vkt{f2zab+qPEEkb*5v-MIEI>B3$N
zwTK^<Dj8G-1`wXopXc)#G)~NKwSB<S_sK7I>F1a$X&X)j*`#c?OW=^?d*bvlsI@p{
z{H=T|prHJPTQ|1-5G@4+1Xr7UUvmv4>lYjcO=BZ)AZgC;jiLpyqK9@Aua((vXDVjq
zvP9=5`#aAW-yu~-RP0OaZ%j)YbZz3AZ_SjX;0<&o{BE+ovKum~N#nY52-}0c+Lhh1
z<Zbmh{<y``XHBNxdOc2?X`re#ioUWhK(3?>sc@#a(wRnLP(yI5A25!`@tK|WWj<B;
z9AJ~}{iq_d`Y$^dUe(~P_}H2i_O9;rZEL1xRNfuZWqnUQOAP#X)AqjFMvtl4b4J{A
zG?Qm;rNQOI4>gSvD<tWV0Dm7uxyR&JWFsw2D&_%Cwz|1rJJ?3cALgHK@@COX=T!{g
ze{<ic4}?xfRlv_z=a3g>2f@^Yh%Pmqz3+*zm5-L->f4ZvQ-?fDV1(s%@akp5a$@pk
z6kb(23~*)gH{O}VsCZ#dICezSj(<qRi1UM|39kNVG^WT%Qj%?mhmbsmhBR(F?LAi0
zI(>YBYg&~$`~&(N`w#(c=E=ffx|JnwB9^!|lsSP?3aob}b^7`Zb^PMNSn7vwB?a^2
z6Q}BR@-xzt=1Lhc`M>2-La~Gk`1i+{Dtw^z(XM|!F|M8AQ(H(sUYY(xV~1$UqeF_(
zj@rytdh`P6*rT1y?#0}O`G-=5j#)txz2>-iCpMGGZK0z!>X?ZobEth7k;0QkH4Np7
z`Q(gg{9waRpR_mv;(2f9rjl{sluyGo&47BzQ5Yux?6E_0@T|oO6x+tQJ|8sqsr0_!
zIgTdPDj_*`d9OMRI`*mDN8+}8Dg-aBz9gkIh-fg@0HV;}hSzc`$|}|Ck(sM;aPQS@
z=D}>7XnBrHq^#X$t&`0)orc{IRN_PNxLw}2J0i!}i#)p2KJ2$HJ(m?lbTUVlV!Hgm
z=_-fpo$nNgqgJ53SIIlcy|kCjaG^4;*ZBJBf;PaKXw`gL*dEv>z@Ue3iG1!{ykXea
zA@5`ub63|6lPqDARp7D4UU4vx1~2l7^+)g>jEgsN(YqMcUpkD65%xFIXOeH47Zj9>
z(RS+xD%)#uB|fR}S^M*a5<s)4DHQua?_z1iE)yY)y`)hcjr`#QCs7=dZ%){J+kxbq
z{DR=DtARfvK}JnM8M5rJ{A}UyF6jN6V!PSCkwaGhZy{243lgUXTE5Ww@zi$t=?A`8
zzIUy#(v3L=rMJ|Ji-Lc23s{e;)mn-ItsQ^_eC|NjhP^02?rrq~`-`S0<}dbW#_W($
z=Cio5ciae_<VtSQ|D%l;qhxtM&xcDny=-YC+3;o~e20iAT%q|JpNil7Mv8Ib&^6~f
zE`r5ZKNJk(QdEBBadys{KRdc@ff&pR;`jc`<L_74d0sW$LnGAA@lOW`r9ghlt1GkJ
zxbot7N^#V&`MlEdfu~pwAghI1sd@~rF4lK(@ojEaU)Whw5QU|<*u=U)zyWOAJj`yl
zGg3+x^u0vP9s7mtOud@~#<lt70kOP5iTu_S>44+}-{lmsZ=phKHC05;{Z!Ow`L<;d
z09P$MaDWj<-L4U&D!2Ml21b*9MKGF-*v0#9s!D+v^<~Qa+u|0eUPK6yCjmEI69SBx
zYpO%VFD<qX`w@yAu8~x*G#<IC>;ySqvYQXY$iXi4rK00}-i3_*TtsIRI)a54`g!xK
za)v_(f4koc_we`2S&%~!-_A&oaPRRIp>^fI!plvAq2VS#1SF;&`LY_b-mG&<53b&C
zuzXd!$H=6|=F#5*Wh^UnEej1*IdNl`1kYr)+|sx2dTYO&MsaJKTH;Xl=}P8_+>(R{
ziXDl0emzgaDMGIp+hm&!{vnU58UkL#8NMtaP;}x<@GJ~jTPsArbwn;KY{hP)d{|<)
zOmeICY+Atq*NDR=bfMQ6w@#((_$=p%X$zDNA!!yC1RgUnI0LS)?Gt0T!a!I3Y+6*^
zc!qP`Oz7QpS-Hctohxy=*Jk+*Dwyu*LVs(zzslpg7l1HpUl79!>E$!~dDf0JKT4N4
z|LlZ?mTRGUIN>y>h9oq~walZisTQMtIl4CxWFYg4!kYAKu7AUW+id+rEv{u8y%RDs
zBgE;Cw5mXOWvI^8XqPWjDo65jNH@Qc{um#bGXAeo1TAdO)bkaJ8)_1kl2vO}OKWfA
zY!fzS?c<lLl!;*PF#?PVh{w^>;2g;IgK#1=vOeCt@*+VuDXugjJ##eo2Vktq1uyzp
z-JQx+d^Ao_DfN{}zrkk^-ZiXxWgE-(93vCr;h$Mp^lYqKs|GnH`4aRys50p|U$5UN
z!BaRVt>yzGi}sq>He9GY=jF#mci--!??XQu*^+kUG~P7i4S}33H=Z3&$p?c&`L;Gv
z$Bj}0d}jNmQ(d1APz12m00=#{zJ=SRen|pyBK6_6FI4)yVDI_Pfz-Y3zZ(=vEh9Nf
zcAnnw^gDSo2e5PV%{!!Z1I<$XRd;1P$X-y>&V5`uow{)J6K(XTk~L<ribVD!ox2ZO
z5Lj<Q3%22nlVnoG<#Y82oM}q}OJz10i>#r&baXC;XxN*R45?({;tvs?aFODFJ15j(
z!4n4)$hh`(PzE0+LD6k|Q4+h@3{CKy4cB~^_e*Vvy@iM}<X3D$d>1*S0gXD1d1S4^
z|C{Ho0&gS=BP1t!aSIXQqa!Ns(<0*Lwu6J|{ck}(Lw=8Mvn(2oumcAuetmN^!`rp-
zyVWcE<zTx#!$d%R{7i3NNksYCh^34se$jrXMrUp(?K6L8?Kmkrhsu1sg?2fxDQ_nj
zK#JG`l=OT2J87fhI)i2s<2>ZAD|3}nj?9Ep<)s>=9CjzEN5N4AL~^gt@*NHhdAVIK
z3^+nps2?YKy%g6KFchg;7R1Q4&^s1Y==XXfd1G(C^VF%!n5sW#G$F$<VN4ATv2K9&
z)<%NvQ)$g3n--!+^;*H`EhAbMRt;k#YhL(3d5+*!Yk`JRakIGN!R9oHR-qZP0k;ef
zU6>uA=s@Hn!pSj&wAPd)<?~Ptl4$<oU-xbPadZdocK&2>9?J~l4*nzIA%*$fP{Tsj
zuY5TPi$!Dv4V_)%<Dnx@nmFE`Gi8wcO98Px)quqPs+h6W<3_4_`-4&OxJcF~iDWI4
z5R^fiO@$>{UZ<5J7p0(~<tgs+i!7Obd=K}PWzI8d%}s$D6zpn*J+*tmWHZ`&Y|qR!
zQ$(V6@;L#bd94vv7C43C+Um0k^k&xaQdSnXmCD;aR&^2-?eHW+N4kUVtKGBpD5p+~
z$)2<)9SgE@UTe#^#0W(kIq;=-e8!x3lR6i}0?>YK5L&-%I)^%O`li<i?kh18pKi_|
zyTZSan(rm2SH$-1!J5|UBr55K7A;aJ<iZ0=-;fCh!yi|n$z%_#%%Ua|&WkY|`G&4K
z?_X->ggo23NK>zqt_dm}tc#6R<k*cEe#`0BlyCZ+QQv|pM0t?e{oyA&p6|w+062GO
zBJCe@Rp&>|tY5x<<vEGek{{BV%es;9?$mjde}EQ;w&UMFzozh#Y#rQ;RY<Rc48QXy
zuTX(?f9rUq{GIKMDobJN`OnZfj3>Jq;>FhHM~cg8k~*u}N!aKy5zBiQeDvxY1(=sH
zZYM)}1QTp#$T;I-^NP!k$(Tq_OzQV*?+NXoHb0U##=)4fUE|cKI_3<eKjaDp+r+v-
zJ`~8Ok__y?1gmPV26?!U@0S~uG5HXUiK52PXuH-?xt#_+z4uNw#git^(B0y)Po4*%
zbj55qWv?bVP?nAqbGLtL-K{)4D7{7CzjLHQ(Q^sjkv-C7goWF+2(-OIDjwL>%*!X9
zuabBGFFAp_NduU4fz4ALmb!3k^WuW0+M~RLf_VNPiRaiptc6NDK~SyLg++K{JJKaJ
z1j4N)5x6jH9Ia*@n@KT0r_|<6xMj$y?vZ^V0=fjd<#pfOtoTYyjJ|juJ2A6Qt^*&z
zT$w6U%8=Jky{*u$@qyRY{cEEuG9FcJMC09cv?qQZ7o;g2GBIsz#9or<Y_D1tB*UcH
zoM%s#%`wU>?IygjiYJE%j~d2+j^ESx8^Hik2+z4p)3L`B<1mgF)di<VhPjr-+1l6K
zzMh6H`(b3>3^W*Iqi0<qM#^L+lh`P6f+DV7o<Hb~9#VkwdF{+Tb+G<B2Vx?8IU>QE
zg&f52NG=ZSpmG9#?msRL4Sb-^X!l^9FXk+U5vKgUyhmy?Xt>W}{mo9^?+MH#4bZT|
z1qUpoWx<zJKA}L3OL>>+9om_v^jfwt{aMAt#@0&X&KfTb=EILLT}etd7$HP>5}OYU
z(;_jJPhxs(KIwG)Lgbra&UuKS$q13Db7B2CVqXgKXn2RdWaFGEIXB*ssguT+X`XNm
zt-D%DmgtAhzY@+yNv12VE?D$7(v3g$3eg*MtmOEfU-8Lpk0>JZ9li%0*EE(Ak|-k4
zbQ1+e3Ims=QX-*|wnus8Ps#p8Kip(jI<4}lX{r$}gxJ{0#%2^H=dg_+g8=)3aTniH
zq-fSsxEt?roY)TIy0_$L;r^>1dQ}~z`?FmUb!Wo}ILB_rnRyjl?t?p<?LA5lJQX1(
z`~D;;(mxyVZb>Mba!AkVk>Me}v9EUyh;-1>{T<9*&<V8rLa9DDXg(?CX1?ds+6?ZP
zLn&tUqSN}fRnxyG_Fa51N8~D$$U=)?920%PHK`zAz?QiEk?*dfR_>RRe32{?zF_DV
zH9>neri1{GP}eOZ$!M(247U7S(Cz6#?gr<<X0q><Itv%APIFVuOmM*&IzHIxHw{Zw
zb2AmPjQY^K&^s@N1RYf*)31rHI?<516qL&D2goeti<B0sRi~hdWi?1;Fru|5bA`yQ
z8v|^=P1d1OG%I)v+@bt-Hf9uc9ZTq)OH?ps#5_FL0^Wnh$cC}y81bt_rsCP(*T3j0
z_C-W2->n4obZCdX#tYRwDN?EL*dhrU`pyR06hpWOw_!m9_=NYBrGr+mkPgGtWeM`|
z98L|1k5Oouh3l6T#!=!T{h(R(^Pa=Fw#JT#^dQa=8tHPl-++aW^7{=6a{Ek$q-LpJ
z)eEYNlbv{7mh6>AlEZh<OMdob%)AYW)5>_-WhI!1_J>IP#E7c9(YFdW&29^>I~qGU
z>9<%G=Z*GAA?zv_69>MwBz)w@1{dZOs^3PSjcLq@Mef4yDqW_-YVPLRIkUaMg!l87
zg5-b3sa@8QD5ZuDIoaq8iB+$K!d#DRyeUdm8uti24>3~w7=UDho9}_%S@=o94e~Zw
zsp-vkZ|<tiA)S0l=^#a-V5jsEtF2}D=A7)tP+grZl?{8Bn~GD%8-hFLaCWkt=~Wk<
ztd+xg^+;ax28Y^{x0_n}l;n!$-C{TwT5~ihm}vDd1y|B$vw?Gn6wJZ&LK%y9)h{PV
zj)ZUT+C7~ImHo_JEW~<h7a3DlNm1=o_-eKm`EGxIucL_;=qP>O*Fd$#>(RxVHSanb
z%crZq_iFSQg8jkmkUi*FXKUj3_SND6pNxk^$1Yr<bP=0iPhkuhrHyMENC-9y)bTw$
z)VO%Qf;v79f3i>2{Z$Lsan_#W&6aC-cFgwpzMSQ_MpZKUc*D1g?;_}Rmb=2pM+$T$
zkVkpF7T?B0rhxV#8LHbKsY%A)jM<dF4XA}Bc8R{SpOYKpm-pDdR*aEh7%^gD#T|KM
z<5oldq{9ZAV~-(HlK+c7ON<sa#<=PI26V+oWN14$C38$^B*?EeD@rwQt&8P!%tJDu
zYYF(1&%3ngf62!k(e_5G{-&>i;q#b?d>tCCB{!NHDvz98O0Jp+=}T3nK&In2;0hLx
ziI6#zU;oc80Wy9gm7-akI2?B|9*0d!K<$$=zrqxZH>=|m6D|>?Ztn$=nrM&zjNf0W
zc={AUgX7ZsF}VZwUpb%XQ(nf$EppGUzeQ~OUyAtuqm<!TWdh0;fA;^iff|76_E6ad
z)O{0xKCtcv!m0kWgy<uH<fXLN)t|rgfCxYofn^>*QXY5oV+GzJ!!~lR3jRx2@OT>Z
z3XmZMQ|SLW37q%$gLU<&a(W<O^8c>`e-CRV0Tq;G+Xs-^>fb&uQh`!wm7iig{J$Lz
z2Kp#SXxyLt^na5719?)oz$!QzaBI*;{HPcD=f}emRI1|%^S`McK;Z&hz|n|@;5ecG
zR%VD4NI%b^L;cUv{`n<B8VH(>Q=eG>kx~7#*c~8i557p>e=C{j^J8%I-957o#$z<_
zKR+O4E(+{QK>R<8_2-v&d_d~d<u2Su{%-{aC<D;+TEt8^e;V;0<@4-%d>~f?0h~aV
z`ETF+r;-2D$p7ide@EK?Gp~yW;{zU#%$z$8(sr+UEk&%C6u9r+wfm9Odex)iB9v4S
zw7+Jf=XeqP=G7|&VcBA>Rz9UBKd%?K0w3SCzqSK#*&Tj~eY1=G`J?%ndi&JAx~Uk+
zt5prQh1FO%?ZL%>z(+7M;6VZaBcRYt(Re_*+z2AxKK}8K<18Ei)9Ot@4dBB6I;fWl
z)$;u2^E;GBbmKpc4=w_1B6?^X(tlm<&lSR50cmnK_Q^}1f8GXA`Md#ikA*|$#lP+i
z4xR(1U#X^W2G&3C4X%tX1@(WgpaLT=4)1Rc5&_0#Jiq>YIty^Q^Z_%`T!ny=9V<r`
z6pN6-p%ndZ`E#P10Qpd-a^7j@@C@~JbrTvsByBj&0TVFsus{R&p?GKfS3Pq*1qx8T
z0p7jS<!UWfAK+$LP#MZ)Sf45nvD^b>&iW(RIIo~mMb@3i<Qg=IWMgnQ`QJ89(?A|A
zh2Zlz6m#z6mjU6#gQ<qg(fcxhu^XqG>A=W$zSv)nH%MWk%P;o20`zr4Zl~p5d{BT6
zl#GT*?>ht6h<s;`N1Qr<v#?1p)|V4-TItQf=J>M<hmo2BUN9CBz{eHzA=4yb-2^8-
zK7$@|nckMgAi6@xMzXy|z!u<hGIQ@13vqV6*)cfz&CU)uG{2PXrargIdiPOB`>>*b
z`=x7{-HIC344b)3*Dx7<aReZ+EI`cF-yZ#0gGscNYrfF14um$LA*Q_yRV>%8i;oZe
zqj*|OBNLLhxI1f|B08x0n2h+djFK2S(<SkdduG&@#lqVCH<&z!0{x5|N1bGtHP6wk
ziY=f4{*xRK{TO|0y8#4JFGg9PpsbrZF%CSo-o8ZkBB`jbXZ=E$yn1|Bt2iecsqP7Y
z1e~aDKR32pd(e;tNJQ@ec3{Y#%<YR=<DFf4`Uj116=Q}+?#@W9{egv#UBkdE^O;{G
z00ir=eN$uuXF6Q&1Aeb*bQU0UC45p;A2X>KQ&`_j4`chvP^q-v`_M?pW%K0=S4Qt6
z?=_f=pU>sT(8T%P%)`QjJECOIyh+(_tNnKM94S(6yZm@Q*I!ovyCy_^vs)H61v(w}
z218ES*3aw|6mMfd{o@4K?iDXKLe~4!5$RTgo<w2zBtWuPP{Q56j+A)K#D9C}t?Y8t
z{u}WB=JqxnMrB3fpDC&_{aVim#6hx<jPxjggS{C+f7lIrBKamvEdG&X?fo8CnX<38
z;^D*bxYosll5PmZj9X_=%km@kA+`klldN78Z*h$c5&%x`Kmu?intbNjsx}flb(^6@
zfFuu2d3MvG^aN|t&un$VcPX?InnWo=HgeY&PXnJGAQnrO^2>+CJ)fZlY7+yL_mZ<h
zFx+?_+ZVvJGT|UHDHhbCS?cgBTaA?HXjnO3A~OdF=t;<|<l01#f7_wmyyM0q=vr3u
z)}eqziIgw!b-v;S`9{raFvtp;f(i!1STRxZFT0O)-^-phr0!=})3sR5Z~QRCfk<*Y
zS0vV|;35FPtK-Kl<(mQFhbPa`2uV1f04(a+ho(?2;j<jV)ftmANsC#_PWWVQdyL&z
zJVVT^)L)TdCun2&xocR9d6zv&sJaZ3$ZQ4$J@=TT`<elcj-;JNwiF;Q;lXpY7zC8P
z%?2(VlOlT)d^bXq0sG6)WLVr?z^@gIeXafj5-rSa|C|jC`8yb*uWwZil5Kz3bJ=b`
zowFYR5Wc5GerB&y8*)n1@dW|;)|-^q3C`|)8ttsx-8Ge;<xWAB@cne9X(EOGuFqx?
zPY6EKkLO94e3#E@eoK83{ZP$QolZjX$|03VG7+jJEii_$WwCJx+o$=KjDrH<MA-sx
zvMF&c0hBmh!_Zg*TdAy#cwm6<#-w;uPU3yY2&J~gh$Q=aH#^V-k6<%d4!|8Vwg3>?
zcLPPgwsSY*-QQR)x1|A);&*=5t}&whTI0eRY7`;#JB|%hSk`-h0lXDB;=Wjok`~q$
zbpQaxcHSV~*P<IEfZ|QqP9=B<0O2B*qLY{Dbbw47ma>+b(_C1$`ZXTlxL^Nj1hC1C
zMqK9tSt8%`A$VfAI9;s=qTH!&VY$lHN%!%vyX*q&K0x?0)N?oD4_)FKJBP*I->(5(
zHxUeH79Pg)#O_qx=gyNfDGo2x#-+3?3`aVMEt;S^+-PyUhlVZBSDx}!@rr!kGicZV
z^zPrA(o)>vJ@-{{WkO5B#-Ao^Lp|4xC&S7J*v-_wpypN7K3uL~>hceFVALm4gmU~?
zou1I`2u3z_RKG4)%{BSDW}@3X>#!2C4j%B=U5&F3=Up6NbQ~ckS@pcs2c$#oHiHz$
ztVb**DmIhRD6Ot@L+gzK{!<f+EKa1-Eyd)rS*u>(1zPo+{xN?|*RxL?_JX$N4N<PQ
zfa***U+G0zIX_z1TWM*&+vRr!0N}yT7&x1AFi6-;lpGF%nfoE$2<SbIg{W)q9V5&1
zC4}uR#0rPIJ9BqVT?buYxJyj{;#C2{t^-ho+H=>vjE!;Xc2iUd(&4GNz7+sUo?za1
zH?G8_yG#2XKc+!mr|1sp6n%)tM=tbJp!k#Al8rCv+f*KR8@Hu-j5h<4>~k#JrX`;Q
zn`@j3cSSzCwJ{kqyc`0xCz|XwfnTEC)e&k};COdHIb}Z`Xca-m-AzwTRa5nfgehFz
zj2P8kx^<9n`t`bA?au8^^-%|r|Geo6zv!!4ouwl*7_2WGFf-<juBQ)1+{ro=uUKCd
z$Rb?uo43C=!T;rWK4sv4&+;b1T=AhDPxGC=gk7Yi4t^Nl^_Qoy>JI5msr)+OEO^aT
zwN*Gys4a))AGvfE!_w3o*S@)hb8b|o)AS9z%FB<m=6_fQ6M5M29AK9v4Iyxsa4tA6
zvr-@gela?R&U*JrWp~nj9aqhD%6)o1qxgrq+tr%)oZSpV`Oh&ub@q5=GOnqKY0|q{
zI0r>3e2zx0>3UeNRt#?X;`;Zp?+b4TTSBdsu@UUv2SI)(ELH)!C&Kj?F@Y-_==O~e
z_W6KbaR=#xuGdQE#XVJ*Ak0%B1X_JZy0UVA_8gnquN@Y2A8q7KclC#_2{_@a;>h0C
z{hTz{?hW8VkzU9wvJ-a;jW_$XocE;8yCs{-C81AQ&(-nmPMK)X)T1fT^z08>23>-5
z3gxuZI0;{eR*SX1?gxE|>)0T+!LCX-?7FZTGFO71S1o{rVEGXwiEBwi=_S1x@B@NX
z0J(%UPQm#C6ML|GXk)QovmyMfc=vNu5+j<Pe_jd^dx?}tt*-vlP=(IonL#|bZ&cb^
zv3O{^Snkn?u}V`N+pw#A!U+OOJyYbDU5%_iQ3hB&2Vc|RH*8ZkPf0F|!?#<5>OmtZ
z$v1Dhk0xYNcF2h1pKg!Qp@@(`Z9fgXiMq6~6>adSUW-xvVQ`Fd(Xc<YCQRXO8M18U
z<EemRwN>1kM!l|z`RW<Mwvwe~PN76k7;Amd?K2!K@vp+sb3R7Cpv$afzQ)j&!Bdf3
zV0^6HC3qc$z6!}8S8>N@H`9r$&_qiz41D$d(N6L3d4;FA1o*(HDLZ~nBkJ$7kqs@S
zcn^-#N%h+k4{pU(dCx8T&1sFNe(4kri`Oc}Ib{jguY4!1pxp5B{1CvxlYG!N#y(JD
zth3>ae0!T^D*iE?<G?TGg-!ka8vVdFD(oFK;NaptwykV{6n;)88i0AT@Oj$LJJRHi
zROm=Y(ovz^a`m}Vh?DG*!gCH8Zf(gOtnyE6>NT7+GKpoqbLtF1i%sxR-D2-xAL@~|
zAFk-j2;!9N822%}XlN*7tUr}2QA9~0M?(o8Jw^8se1_YoS)=u!tmN@cVcM7c+58Z;
zSs~0lc=e(JrGymzQ)608e8_VYn!+c$?jOtO(cqeXlMNobXR0+o1~-Ebl8}Xda-!aI
zCkcVMQA8j;y)ZP&QS(nRhu(3=bIQP0ji#<Yy_BLrdd;9R(>QwL#0L<e`~in`#M)`k
zJk`|N;UZx2Pl;9-<#RD=t&V|FxbSqfb8avNm2k{*%T5I3#ImBFAV=$=`b2b(h44wK
zzwpTC&cJ2qlZxTZcSAj)#B?yTx_YWdPz=)g-n(#L!aqR(4QB8${$g)tB8-b?;0I*T
z{{#R_G0U^6+k$nMV;J)DAhe%5RiS)|fbpM>rQ~`xmVW)xELt&;mJ024?QSV8nk1yK
zZR*p-4@fz=Qf472Y{-CG>-2u88cVKtBU5&=oyt@H7-e-C`|Y3E!b@A+YXjN=M4Vzl
znA+j*ve*tN&~o!<OCDGRg{IuMv(rfFKFuc(@9<%Zb%Np%Zrd&y$(c3uMndBFT%9-^
z2aus+_iuI8eVa+_XCqQg3(288(Xb9hgrm;AjwbHIm=(vqq~%dmPU?IOvdZLAI16ft
z=+;p;-B8b1LxL^9ULDS>M+o9{UzMlJ`?ho2J&O9vG7i8JgsqJxjWv?iU9S=|c!3%C
znd<jzMZ$IU^&&BD{@7<&DG6}E;QscE;yg8zUHfM_yNHC_^x0W@n0L(3N7y@HFJhh!
z4I=RD%~XY@Fa|&dw~6*CGQ@coKEKD$!IO-dQd9ZtoF^H`Qq|5h1jof&lm`(_h>9HW
zG6rMU61jS8>eRh{NSNpuJq4HXcz#my*A*Ynb-e<>iI<(TJZi`5)PGBX-M;Om8_Ef6
zgGBc!-j66PCC&wKc13XloxoD?j*{Z~sYMPS;&)w3fTwCJwPZ{=H5Ql|!*({3ZtllV
z$ZWGI^I&OHh~QS@2xE8PWwBg3oOT6&ZzOe(ot1#2*!Q&ieU5sDy4&)49j807u{W@G
zdU^<Mq;bqHH1a3G+aY&ReJ#h4*z#HKoCWo9+4B|wGE%(Qx##)Mxs^ej|4JHdGOyDq
z(kb+B`-Hf{y60%e-4^Jc+mva3KXGoa7b=Ok{m?%ldjZs{NH0b$7rhid?S=}>sHUx`
zew4~*5sC!>BSpIKkzM(OqaP(qO-Zaql79Ki{(v;+D6jkrpJ(42Dvi?*>k^jzcnHT;
z3vM^?x}Blu<tWKc^K&;&3O&K1OU9X%`o9x=EsZEYS(EQln`$>aO;ON0CfadJD2GX$
zseR9TkFDKP8}|hhVNI}a)H$`Ww+vN;uIXjL^;g{@lC(MNTICF9hNDgC<8e}Eaep`1
zo@`;e3O(C<Qk+1~qqJ?X{>?2$x+6v0<wr#=meXEL#H2FCSI}$`_*Rfty~6gZ+Y2Hy
zkTjDgkZ0Q#2s30)X6hrjWFEndm7@UD@H0Fw?>2qG5Nm?N_G-M2(tV7WAMHVnqD)MR
zI^_-d*<ELHFC}&V8ps-DZ!2w?Bd8W?O;kqZ`^iLXXDD4n#MXHD1{>Y-ZRbMkm-@5u
zuOxz3yJZ6~RV|4vWK25mhUkA5UbM@qC74w&`Tfv7EZ)Kr!2hVNtn;A!aGtt-XaDWH
zlhN@7lLdd&H8z4<m{bx;PNUQNdvTK23=IyV*l+raLd_gi7?@N;<;@dxe`wTG`a?ut
zw7MdFWcSo=^;?U>&gGLq?0Ju;{&YOT%yB@cMF5WS4a-2&7|lLybK20<(Mo45995a{
z*=cQx6P_R8ph~>R*B8}yI>)dlAT=DytRBe86)K*3Zv1lMC!*_Mnjj~-1zt*DBmxGp
z5(@wa3&SUK*}l^HQS($~ncJ*fz<Iu!7h}{Ar>eK?s|~RIWGCN1hs-c`UX57-W7<>r
z_go!(EFQ_&I&DR&7a8uxm?V_8jPIKN-3Ie}i2}JBX?IF$vj^x;!DdKG-!gZ3R+3Z;
zKf(w_{XR}!53EkP1TbNLoRpKhB!}fd*-TOc_nm{_pXFfASNp|snp_T-n~D)gkvZV{
ze3|bf;jn$ZXOH>MC4Ki{4Yh{bOHO|%xM7;K?i^t6*+KOC;<NzUI?(QRE#u=lzsjB@
z@A@0qVLn=0TkDN}^xxr6{rVzqU1cI$j(t66It7-3iP&F1Wte6um&=yQd?}v=hsl3y
z@dcX&P;cLPV&ioN6EMSs<OUo7*s#R-cT4Tj4PI9w7E86L+;+tzi;DMuY8rB9aP&5B
z^u7#`t7R#wRjbU78~UlBpb(tJ-U)uxrYh<vaq4Fc&h;xYrrhDMTJ4(jsYa;2%^Av1
zp_rN$7tJBxo0+AS<Qs9ti*;J9n6KdT*mm3jxbV}ytCu49)gkMrp(a{1Qc3s<*V_g1
z1Byu_kw0Ez#A9?7Ijx;C$x|#K6K~KM4yOq7j$$3gj*Mu)|H^KpWX`N+u>_tO4*_+g
zHe~N#bzz!Fqu7t{&=|J!)OOTD>>FuQ+>Jg{HK7Tz%5khb>Z}@Pu3Xu~gCxjcxW#O7
zKC0^aJ29!&2D8}D8w>oEUxr=~E7)K3JV7uTtu3#;xYeD7{MzTVhTfRK6dGu2<CRbO
zypV7pK2<~=n15?g?FW2WGK#>1^b5PyodNi=K3Z<+KE$gze4BR9#j6KTL{?~m)nNvh
z5%KWd!6MXb+gD+#Tm#77e6H7Ur4(oE@8ungl#IXm%L$c>f1UX>nJIK+l;Z8Q%No7E
zA4`GuPNKE7K&2vR-uj@*Wa^FDK4WN(U(MZT#Iw+{oo~t>Y5+9fIurrCAf52C;h1<u
z+%M2{N-S|;uyPPVv79_au~>z)$k#^Qk}N-oH%+q>8fWR4v<^-R$8SxRq?fFdJX`BH
zKP8-5WEt-mob)R$2@%wl)ZTs?!y3yp9^KiO!aFmGP+Ahz60Ba3^#%X#`ixm&)yFOW
zF1y4T9^Y4YPbEY>mg|GWOr_~pzlhU8w%L#hGV~EWMMKCV4ZKp@#H5vN>sjDuT~lqL
z!b*`@C3#Za_X45rGwEb}2f)7Szso>1wl~_{^eVx*_~4I068sEROv8Gg?qS(bggKam
z3TZ?~GQsM0^sFU_ni*U(AkZPeoQZnerHdBRX`sIhqQFDI{hfiv=v(})4M37_@Zmi=
zyM`&TE<(hZdE3xE>A{21tFbYnH2pZuDmSPHTE{~G1@1J2fX}76gd)8o)Wa`X=q05$
zExF7y;H>&2`$3XB0?9i|y<rvO@!W(*9Js;mZAp;G$jAq7OQEz9@QNl(<T!c;-hEG>
zFCqNHbBguF1~j{{h?LJ2xX{(RKRx8HYcxa}I502!0-3^~&Ql!)s{iE&eF-j-0hLT^
zjjZ($-|J2HPe*Dv0_e+xqk)eF{-@WJfC4yYl{~|V|9Kj?euf~Fb~u9V^1m($irs<q
z9PN$5b4tX2|98X3n+Ya!{JX<8oCy><_!&~S=RfZP{+$riz)*%p>fi697YzKpy6|tU
q{+ny}KgIk{YyNSc|G(Hb?-4Jtpr;&WTMMDUkJx*ucZF}Yy#5cHc+5=z

diff --git a/docs/figures/arch_4.png b/docs/figures/arch_4.png
deleted file mode 100644
index 0c59e6ce34413d78dbbd0c918dbcb73133311edd..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 74000
zcmeEug<DkJ`ZlS6l%OCY%@9%o(k(5efRvPUcPcFs64ISYh_rw(v`R<~Dcy~9H+*Xd
z-}n5^`4c{_3mJyld+oK>6ZdmJ_cHLAyd)OJeGCKy1T1MOF+~Iflr;ndBm=Y?;GL|u
zR$K@O7++0AMW0EFijqIGvo<odFhoF*3XG1u`AkWj&};n-OR_Hl`t2~h*Z3Lu!cP~|
z_h}wdpyNIb@J`c{Bv9353JCDR(#oW(AQ0|<kIL{Km2uIB5u<fo@4H{?xfg{+{Q3EU
zkoDT=db0Bx&%!v=c%C1D&Y?gp37Y{SjG{Hh_ppLpUvDPvK*S4q%PT+-EjAWn<nN1u
zpq#Z(*XqxVAlL5PLF0D1ee8Kxyp}xZCc+&bJ4&ITSnn-chwAx2$Pfbg@$;Is2#ZY=
z9g%ou5qvSIu}@^Si-8Y5Ez#Tf(m=FrVRcjlr1-XIYGmPl(cSh3+H8OPR|3Dj3t+jE
z9$bqY8yqYv*lMDw>pL9qJY}Qv=}BbjY~6uu)ZwgMq#=J<tTcrSMoGicsNwm7?TK}Z
zRIGgSBjKAo;?u-_FwC8H(hS3+HnpgGclbM=oMoI+XG^(LWyi<P-RA0WE6A?1uzdZj
z{xL0j4G){pP*Q*?fr0Bgg%WCZSpo*>>F+5<1qL()kC_y+6mHuXoY^Z;e$3{Pym$9;
zhq~$189OH_TaDO#nl>HsE!)(<Wu}og68))r470!bjk3iKTK8bdEjr|Sod@mj!{?d^
zU0!z#inXpZ4y}Dvo8G~VuOT0k%<61KkInCpV3}<n#vLNcvO9f}MdX6Sg6kzHi;JG;
zB~XOIh>IcoO?Rn*1eG*?+7R!R-BZrU5S4hZmfM@{3y<;{^DGeJJ`d$EY%*JXC7@`E
zC(@$P5Y59eNw$dZhQuIsX(D*Fej-X9XGZvZi14^<Gb192eoTx9k&ztnVH)y_G;{~=
z+mA%IOOWO3u{s8EP~M{9-XcIEl|m#EMvC#etII@(Ag;?kc;jKcRS`~~*Uw2hlN;If
zt~NxPH_N}fZ`|AQLEOLy_QKJ{^daXL^EL^4n1Q+^niY1p!AFyvg$@DZtzJ3`9TA?0
zq6um#_CueiX&5Ff%C}CDl6_&Ks_F8J6n!_bM6A<U9w9}(;>|-OuD5?h(C&9Sh30~o
zgvsJ(Kjr2^rHQTj(|?lYfZvUz7H9uQ;pB4^ayvBSZi#kWf@yCQDqAT$N)Z)`cTz-(
zA&<%PrTFX!m&E<TJMTU4^LwS&{z(3wjx4pGzk0CxJNhPNU9(4WaY1oaaf0n=^L#Nr
zFPandNQxg=U~&>wp}h2B`bk=!Qm_4Eyct_pXwruJ1`OBu!**liw4!aBZDU1pg~TY2
zCT#&3*Zby~7n@lw#%_;Nf;yWBC!Hs8raGtBCvR?~qS>J^`P}`Uh%1SEdkf9bm-h$J
z50W3Ze%yQR{aL|*`Ud`CgJFYQ!)b%cG}APKZE%JZ8#PZPrBqHvS%#e=@)BN-|BgKP
zl7xvcE6dZBOB86X1&rQ>3PL*#AYajk-ab%>iQtOBiRcU`i<o#$cL(`BF(foXZb&L8
zQ!|Y}(=8&qC9%zZ_HZkAj$tlwcKugf7GBPgGN&U2N34c~fJBWri3<6{dvq>zHg{F%
zl^-T4FRI*okwLHV@CJhs9RWi>z4nU-%0C#0=v?VHqqm~>=(V2x$)<R$m2fje@}=a6
zLX{YC+RUP#1GYf4wwRFYdfEp%2Ko^?NqUwS1<Hc?qiw?XZeU_lB^L+UMs+tHu*v8X
zyou56_fy9zS!+M~%=WTTPCQNVbI)fxow=ZgO21_|<Q#*{O8($;1SbS)d>+Ot(=v<g
z^E2)>=Ifa6U`}#(*rRe(8xs?db%!1a?h^m`bU)#~;C;OilKX^@{WypctP*k(G82p+
z`<I$%Fl#)?SIjTbV2N*)<z!D*hgJW2(5*7C_dCXH{r6fIX)kMEU|&nGX`kyynI6L>
znx&XO-<CobO_xyr<agx`yvb1LGhWJGv>vb?I5*KX!L=+JPaD$hde#`GA7`#+J*Y3n
zYR92rp={~UdC$0=O`d(+)NKTXvzSAgHRmzZ5F>Fo9z)JK!3p*qW*bM9pd^qz5^T}L
z*~Xg7wajiDcQbAwo-45~C6|lKYApu3YG3zqP$T2zO1X<gj=_>~yz!9PVSj7Gd*c_T
zj-3g5tClsxm5oOsAvq!SMzI~pFB2@&R$q^Bj#xYGIpI~^sG@iyaEx;N@YsqV`R<R0
zMq0M@ycK+TPTX@FL)|dek<HIDWb#Jxrt*&RZ?`cQ(C1fE2iEgu+8xK2N0)y%NZSm}
z^elf_ZFS6da-YvyZgS?E-1D()mp3HSt8*&1iLFSrIbWPwEbf^}qMcrvQC}RI%ACEs
z^kQna?`1RJP}TUGhUR`_XeW<OUSZuj!Gu-l4gcnxx!{AL<NTefMTI?_{lX1T0$n_E
zbQ3IJ<Wr>b8+#~AHv@eV-ikb;bur(F#Qho8NKxlE{Y3N$gJ`P1x?et;IZpOF7L3AM
z=eSd(#q`s*8P78we`R`q|1Q}J;$7BZ_DbqXE+eb<*0|+jv*I6vG2CvN%A?*D-G|!S
z<sD=J<smEq>l(Thx=TK}w|*03vGCUfR0dQnsv&Vw@TN0QF>uAb>>!bwrB;E2Zt%N|
zxE3B*ohY7#)vG_FbP1nLQjT~Ww~|!N;Y!!Yk;!PNwn&kOTPwXQH<o<F%EY~@JM8k>
z;kEt^3Mz`33@Pa-n##DGc<AF2C5wm5xckgbn$?z@0S!8K<hNQi!hT1{NhW?GR-{_O
z>8X7Xo79@5AoDTi!OJfd%B4F^Y>|e<F*Uz-VW+M<Mm*N8BN(f9%~f~u4GKrAFf^ho
zzF3y6{W<xKZ0^?Imj}t0d-;aFld_O{{j_q?G5_%Xq50Oa=f+`rt#GZdVpq+$>FF97
zs}P3g%Ajt|t7gCTcPBs0#R<d>#dU&Q8wWhTp5&ZlZ-z({^KlMY4wn&?tCvE}gO+0w
z@ERjVBhK=@UnV|{&wQ7;Ud}Y^RMwtlvG3I5w6r(A<+T+<@|urx{XipJ(YCIq=4<iO
z8Z*(~&B-Up&Q8!(v%&o0DwF+Kfgb`mdq*plUpPNAkNA(omaHG1eLh_kD)A@~NOMYC
ztS^82@of#qD-J^QRde&@zGcqeKX-eM(nUA@t;qKHM|*Vpi^{{h)f`YBgp;}+%KTu_
zl05zJjYOZsoV&oY`VY#g*em?L(mc})o=j*t<Xh%;#zf0EmF#}}x`gxB8&P{Xt-Y<;
zT*(y4Ekf)bock+_Je$y|?Use(5rQ&-T3q|7@y4zolGqa#E$E8-;S;{9)0Mjm4A}*|
z+5)a2U_mR#cC(1TamkOC*0_HBS@3Bo<}^QM1iSnt--2TbbmsNKiB@M-?7HomRvno`
zT?xb2Wf5mL=XK<>x8MDja3=U$T_5kc$l-j;-U$2=`8E5GLarzKS$^?A!M;PP@kEI6
zR43{F_L0tM%I@)i_sK%kf+0((b|1C|R?7V?(#li2vo>$lX;uB7sOO3&s=v)Y4K$zI
z9M3dKwg0LoZb{rMZ(LWNAaS$auC;t@u(RYJjH6F>ODM<V%VGP`=o#dwEvOixv#HZo
zN91XKx^`Y;%&+gJ*&b~}`w|g_#6}Sh!5RliI1ts#i5Q{hBq!yyVp`6*mo(COVD{0P
zbTP874j01hHbk@v^!iYY(oFGFr5QxD^pCG;5njG*YjQZ}8_HD(_(6fNhxzixFe<;>
zS?5G(j^LSs>$&?4d~M76i3LyOCV#Z)?V09v0Ak-8s!1Ek$ss%huh9@t5bq<Pf>(&(
zTM&`>?`sJ}It1j4_mL0~{7n&1u0A6Ve!{=Pz&HGw%b&;*eh4?gZ+PI_B@OBN(<p0c
z$k(rt48Y$IM3h9OrNK`neLF)#D|-`b2j!M1_?753QtI{y2m}!LH=?v6^#-{AfT^;Y
zgPNQyufDY<^D6^uJws*}OB?uo5cpkq!Ana+hgak-mKIj_ye<Ni7oXq-ui<aAP?BGK
z#KByEQcdm|xu~_BAvq`WBj!hxf*9oF<otF9M!bq*5?7al-vlU492{(TSy-H%otd54
znXT=NSy*{^cvv2>v9Pf*fln~myIMKCa$&Nvr@Fk!^?k$)?e*<UZ5&Lkt;pf`eWhpZ
z=paBz2_NY1*X45>x|seml9m0{w7>*e;D2FZWq!o+_uk-Ae)zk*&rDqmE!4$KEy0?B
zF$6hT+4wI$|6hOkXT*P9srJv6tZbbBap}MQ^nWjXVQ*+BYHbO|bP)V!zplpp_aCn=
z<Y$3T{a;IQxy~2wf`t~u;Ai=}X@VF{4Wdb4A0L>CDJX-Vz{=oXh-}~=y33#N*FGcT
z$u?*R2*L=`Vj{{eh$~|^qm;$_>o@trlS*b8=>pIgeF7?qd?<bJP^pc>aBx}IDjz=d
zNt+FxrSm})d5#%C`Bd2s#?vf)x|=_iV(8b>JLOn+&Swm3cc@c!T;6}JJMZ*6pWJ^N
z7aIjv7y<FW{wazQ#tr0?8N=}r`5%83u15>2n0e=gfP_Z=fB*49MU@&=-^jwn`yYQq
zK%!oA!TPUj;P=rLLBw+=sHXSz{a<%Q2;|d7{_nxSN7bHrH6){2V^C21eG(Ui$xj~y
zBcd5pTfZ8%QH%Ovk}e&z$Fui*eoo~&6T+2YJJUqE^j6~b<;=-9P^3mt#{~?<{uzM&
z&OKH=UrbVdHJ`U{=SHmSq9sXPso$w)t(CS!(#m`w6`*EhlqF!+r0YoFkkLX;3b}gf
zZ+FOTjoU_GX;(J~t2getK;pXlTUD;dswtRgYt+8e_0C+`fOcfMsQ*(Flh5tRe5xAX
zES9>@OdeeA#EA}=)cJ&{#KoL+^}TnO2im?_j?DJuC@EIE*ygC@Cycu9k8u)gu68-E
z4nr0?Q@KR$&?}0)Ct$iyOYZ-ZqVDV{*L7w5&3d%O(AP%S6zgi*N3VXMER9u_FWJnu
z$CD}yLnZdM+F07&tk=*|P*80BR@ISydjm^TVUy_EASy(?8pcSd`7f8&YZujNeVl2m
z(lL_E^MqBtNJK=$h~NVa#BQ$j?gEdqdx>!;X(VHN5>Lrvi@`3fj@Nl4av7(4Bi431
zi_e#a%W|8Ky8RL#n^LBFo<Wyp()~#zQaz67<vq)-#v+1=*b|!j3$z%@%m+fZ`6pZ?
zCs&3_6u()hLtQpMN{@#SbHVsX_}+BSTAm*5&Q-0~vP(P3RWD=m|FvOq3SdL%IUDyb
zO^<+p+io^M`e4ymu5YP5A3SBtdfhp*KQ;+lxX{7m+pn*8&AMk3sy?XXYqWe656Up@
z&DvVZPOYqREiccO!5b+vH;xgNkd&;dIUOOc7O>6!aDKRuYS$^S-;jcL`}XasL&U%;
zHG)FVboSN9(EOIFjzq3EvkTNYA|!%tY7bQl7-N~WBDz0G%G7zBM(`#tT=`mk6Yv8I
zw{0R`_=9?RVwn7}INX}c`gKaE<3@3Q&@7~Pb8D??y_KP`TCdh~Ilo-Y+zfh5ZW~)+
za?+qRQR{Bl9>-ct7*-LKxiLhTAe*n4`Q8N2xg`R!)z02|Kpj&);qg-2^VG2=nXi12
zQRTGV``W7-2+3^;b$r(^?X#%Ji-yD@E0npMSkSGadgI3}XR+MCfJ~X#gv_tap(K&A
z@obGLpJXIBTr=2rwiY_cjw?KDx98>9%@ykU8zlPk)OTg<7kfr~vlS8x&vy$w;|)C=
z|McBvT3hFJP?b+!$}T~$7J?=5P-z@9DWvc#rawW2I;|!wT9jZBb4ra@yNbs~{mJ|8
z8|r?%*Aj?N+wO|~du^n`_LF9vht~VM3`T;QmWztNlKGqytbt9NcT;CI?f=9Ps*q<*
zWl+gm?Dq|1u+82#S?<q|ppwMq@i=i=2`-4DS(8uUr=M`!4v800&ypo7DOJqJ<J)me
zZxbxbO6lhf_B=V+?wsdPs&#X1iDl7A%054s!GYS&NUZ3Wnf+!{`SYdC!%-puHt)96
zEh)ZS0&)1f|K&xlZ1vRC)JIk>vpzZ}r&lNDeFo=er;2I9i0sR9T07bWFUw)o9uHr9
zF=%>l&g<+pQDUqR|JXDF%B4Zf1_DuznTluD&bZY?t(Kmio|v2#S&r#qy*IMr7iD_;
z7Jo7iy3^x*g^szTpsW3Ix@~hd-Fdxx9+Z=LjdM0<q0=@r%@1I+l0MJ*BAw<R2z16{
z2HMM~KY7<~PbC=|yj|fjBMWQPUXbl;aILi*`BvO%S{3NdjZa})&}(hfnPLemR4q)=
zZ+w3@lu`X_15~?tdUR+zba}^{RN!@-J(=rlI7+HB4N}MU`N@LN41b-?)DLcVYJ&AB
zdZlU2*m3I=w=L=3LVKF%pXzNeHv!Ax(m|tSjy1g*I~o>!vbxh-#!oo+G*m~;b~b+E
z@FlfCM-)O%H7s{W*T$-v1)YuwVIjEf+xGMA(9>7jW$Y~k#F5{oG<__!xe5l}Gu${$
z>SA?TQS)c#d611IFc5#wiQD`H;t;6c_xbt!2?3J^ckTJv8-I3h=jL6rCD~wzi~DO*
zWyz32`vq8zSyQ}?6L+LJwlUH{s9KqTanEqEQAdJuS<b*>*%9x;--L&UCw0J8<O`c+
zy-I)^(E9C8A?vd)&q?o_cp>8Wm@vKl`Rc>%g=U`4=wNBfX?c9Vgz8?Lg%!@;Z}z4-
zEBZ$++49NLJ99@((G|UMJEHg+3PPu)(0-`5pQ%hF1&qW_E4INyp4oOG>78-Ght!JW
zxmX=NeR~)=`+|v21ILX4;|1BW!P?QycMP`s^NH5HNsha+b-!~}KUnfY8^)#^gX9!d
z^c`WZ9k^qM6Pz@qg%^y-LqC81Jh-Q+LmgXT4@ty(I*LWgKe!1}3r|uBvRrk7rX%dv
zV;w%@!^UA#*-e~<JYjzZ`p?qJ3l^N)yf#|d=sXLUWe2g1G;5Coa|_W|N1(O=(v#!P
z6Yj<CO`fOY-OuimcKLn`ZB(COexb@|T;9}_xi6w)&r*Bv@h8+_ES%j~emkkQK+(R!
zBL%o#VnQ1?;Z@eKptf1C*5$m<FDib49qK1gNZ+imMv|L=Xc3WKW%Q_Kvx(RiCSyLm
zwkH8HpQ&D#ulQ4J)oJ-i2n_NLdx5QMunn7({~lk}%GvSy2lvrkaSeRug5}*3n%q)r
zIc?V?8_1gW>m`W4eX~w);ohods9B}_L)&a=u>lJ)xxulNwBGp0f`xI<^HV;*@RWw+
zT-JdVaaBCC$$D?>g)LKm*u6g1Mf3GNQDZmWSfRv)6qjjqe|xCqN-qoo!(RBRwVgyG
zKSNeP<UCwJlb)Q&(U9`75NAN`cxMU!xPq^?!e;8pia>XKP7Y6!OqhQ2{c~Q+0re8t
zl%_oCc@A|Q8Lz`<(-Ph9KGG@8kyo%nR|jOLy_)^ti!5J{gR0r`)@rO0LR4c<s}zZA
z(wilhwCxsO(L0w?PHUp+jYY&p%^7cZxMeu-bH1dGX6**neS>Ld$WKifwaBNVOIsRp
z2Btrj_o(A;i5WL7ZGR9tt5{AS>|B!_H6?>#N$HI^LDr`0VvJx(L~kaNc2V>@o!vLT
z-y~}gT+g##d_Cv}{{C1bwrlIpaG80cxl+P|&|Vk3iIbz{!eUmQ<t2!%(P%a$6%`4L
zB%z~BtI{Fx*nqdze&~y<URzpkH-9YUB`y#~0C4b@SK=?+i~P1ig-bwS6mgT6_+ic`
zv)^Cl`4>V<Y~{BNr{!bCXdK**ma>amqL^NOYv5_r!FN92_dGXlntJ+-hwI43M3F70
zA(>|Ou{X#$_`E6wnx-jr%LP?yF2l)zd^HN?0}6e^{+}d2IJxbQIolrnp3P`l9#x};
z%@kNCB`P$G4;&0sYT_lD&L=<*`jix)3HEIZg|$ZyIcXN=>2l0^p_@`GWeLyvj;BVI
zsa{w%2)P2IEVlf!qVx}ns6_~@I-qp>a1nyN-l1v|qf@J@>BMH3DMCEPco=Bb9H=lu
zUI@jo=#WZIOyD~&z><TsF!(|0DH-xH&)ek6wqdgFA9f<)bE=*ZjVvlKpXt=KH=bsO
z8V(i^_oay-<13s%N(&Yo<MSD`<xa*tfFGNWRhB{f#{#Qr-t0Fu&5ok(?}bI24nu?T
zesV<q=3v%R$qQ6juxf14PVGX6jTy-CYy5n~%gdXt<brwadCbDouz3~)zl%qo)IkE6
ziPLj&IXEK&TXnT~fBX##%v42ol!V)coHL%y{b+}K^XSdA{LPu+j3-=LU82UNrqc_1
z4PyXq>SJk2r)LP7BmLR?nGDv0*nM=ob5*B`^G0`qe;^4Fo57=*p1H1avf8JkrengY
zvXy;YW&Y!|W1q#Hd7LZ{rYaQ@EM!dr?G@ckn8a!Xi`9dqwc6~jeE|pV3l*Kgc<@Ij
z7e>)+!*`U>(rX?nG1hmc(E_M1sW&gKqRPE#Y=6R&jCNdDYV>rcSD{#%XKb~<Z1cxz
zARmcCAReUMwGvY)Hil+5y=2Hr$Su6Wo{7XAq-<*($lfx8YC%Kj8-TitKP(f%A9SKX
z-M(_$`Ze~Lm}sO+`MOVTRYTT%SEYACg*m_?X3R61SE-;V9Ml}dV-|kz;bt&-juc6&
z5Gv=l$ee*rQchSsQLHbSq@)?hH^xnVmR{DbS>pdVK4vhyU&|0b*S6fG`{SU;jJ5LQ
z)6w7gFS)+kXqMDQBtYhav<$U4)-*+`ICB=e(%*`j_vP51G>ox#^6Qfh3~A;FXa-g_
za5C9;U@HYRV5<g8tB-rE<`&kPO*5%~on6Ig_Q>DG&?=-ID5xx;b&s!j1Dr`t6Ma>0
z<vj~Ws$-1S?w{XsiF2*dGMkwRZXMxN{!n=)p6e3N+ZR+ikc=Ptm%E^*tP7xIu?!PO
z&fW7ay+aQ3{$M%EhQh=bd1|cNnoo$yM^o;|g0R{`EB8woyyvlx?jiJM&%(Zz<^wZ`
z2hBXZKV?VSq8SEAI(Jt`2KciR?=^rgSC4@Lt>VBB84+@+Zd-w-qQ`>(2PJWio`#q!
zy*mixQ~mf*DN7bxm8t3=unLqx3EaC$eN9VWJomdqamHAw^XulHra{?l%K_{dbaGUA
zbu#mgQ2SA*`47%BOp#bgy+kFKFHOl!pG#&nNj1uuGn+xa#20vhpEaCi^#t$SF=0@=
zypDOsdOUoPgFo~NOYPx<+N|ykEZ_qBo@felCD}ko&)}eS?5U#^t$dPAh0)KY?Sw}`
zo{Q}q?ge1<&>%vVglxyAt~gfx>Asbx4{f>J4mJkQgw9X42gTkohiH~tX`L|3Jt%N-
zjOhha(d&%ow$-z!E3DbmSeYqcaWG0-)=;$*#ivT3>&f`A49&oEPMEY$&Xyr6)T%N{
z7;thhwK6cs$gOhllxMVcKBl=_pf*jr7D~c5Pp6m>5LgCmDHK4s#->k!EOJh!dh*F2
zF~tXhk_~%cEkmQza;YaX6qL8kp%WlR%KfP16zLv%T|P0;DR|tTjxI&AJo)|YjQ?_P
z=GTN#r;a{QRV|PdLw4%tI}%BNzd4)q9BoYfEY4nJ{$5#?XQh~~4~DZ`v6psFFBC{t
z&l%v~Y`~XJZvE@L_P}{LJrN(dUHHm5ewS|xG7dj(VsN)X2+g!6I>l*{+7rE%!Q$ct
z)9^EmGPA~h&0OUiMQ}^VntZC@>aSEG&lM=|+dPmSXhiOil6o{-unq)Qjm-4=elQ;@
z5G)ule*!s6XYVYP8z@USun*yQ`ON}b&SnXIhf=_O_x<);n^e!&C>xBoPAGL42v?hy
zA91!+oO(~?F)iOYRB9R-UpB`piSJrz>uoNI>V7ijJn>s;=0UD%VJJx|%;IEZ;%pw2
z73Hq7vGbKN1~gF>z4skK;S*Y;Z7ld_|H!r;YAdDa4^!(sPP1Kd?>Sx^K2#&?g;Xa9
z2YVja0qffWNG>7nt&h(Gv)Iw8gu|su7PBu^=YPywcH=6mBHCKY7LIr@Z{4zA8HKV#
zdtFJd!!#oZ(@k8d=~VFic~Dpn@FfYr{U0K`x>26gx*a}%C%6V3%H{hxG|!I!<fw9e
zr~Rqz*nuXT8Px&OSSqNl=-<!SI~&VjI(4L3<z(4xU3W^bpa>uoesHCE9z*v`<vhsr
z4vTpe4WX{v^Nlc*9`jO?8i`1Wn%BE*tkt{yF9Wr#Na2gY*F;qQ?FC>Jnlq#$BdR$$
z6e|B@H|jjhwH&RLez#ZSae9Q3*gxC38mJx{3Bya|bGBN+Y(5~wA?9R+EsI3D!+}S9
z;NgDV`55=oR{w#2?ziOJ)Da<CM;*;ox9twDVq(FHg5T6JRuyT|<GujsJ%na{O&Kkn
zDsCoar%5>ZO%&cj!W;C=d}u5suXpp0qOPTk@m%I?S7l^@de{;BKh-NW1UEy0*K=v|
z8p8|Sv0=841}XDBy9gzWI*Qww=B_a%#ZWjm{@v-MB`Vll@bu_irKFZ3gTg>hpEC^G
zlLOn{x^FW{=@8eg*+9<(Zev&J<GocmO_*WMv?dK5%r}fXcJIBtKJvD0v-l=ymn|<>
zC+>oUs4eUdp7B@KaU%NTeklK7sr~7ZBmIjnQQVQ7(@BoeUxRAg`opOt8=xz{D|nor
z++9%YFrcb%JN#PgNMjNb$?h7v!m-9JqH_y2<T|a%%mB-X;x-<G9B9}`>a3Z~EOvjI
zPP%QPU3ZjV2dl<ku=r_^=uFet!STAL#3;k{G;7vap142t#i~-FBlDXWIH>6;oHIb4
zL%GUWXlTGK5p9WyP%i0@{<{IG6(v>~Vn$qua{PR;7>Nei_A#Hhjd8N*Tw@7n7n<0n
z^URGSoQ@8ISZc#S9hi|xUvSO$5u=Aylyr9(eZI^U2TuSa#nQLLo{2=OkaA~1QE#Zu
zvyR(&GWtG;S;kI}a&o!ucO=)nA!9imm|?lmH}fy|Ck?K1X(q^}FUOs3T(V?d`ydUu
zb|$3s(4@(9I37AB^E&qQ3u6@8S-pRS=M3X@rLXqsz2-{R09UedhM{|M`5-Q8_0bjf
zYiabbWPO2A;*JX+x?~h_xxq(~7PQ&$F2`l0Pa_sFVPYLa`|&anqK4;_cAW@;BT9j1
zs#5l`y`W&O62uia0bV(dOK!V>Wq5RLW>*ur{FOW!jH8I&wDiyC|MkiRaAwI|QA@~I
z-u|yKg%P}!F~TZjAosHW_1Kpn`T}KOfgK!wmWA^l!(9`wiuA(M%+b0@*XitEV*(P1
z3%;hJ@q+&`-1Q^;|H=O4od0WQ6)7hJ2%0-YDQ_D$-gXzKKKkQXfYI_$gknhS$M(}_
zP1DmSG?08->`SKti|&<rE39IBn^@}LY6m6@k+5wwhaR#L_Hzq);5C=6PFK__G~ABg
z@$&fzp;MwqZ&iHmWB=RMqc1YeB06~h1>#Q9c~v~YGe9zMkGd*Dl;A5nJY&SQw@q$H
zieqBUj^Va<Fdk!zN-L_y(@cY+5Sp=k=P`$7%H6;AR21Be8gPS&31rGEbM+P`41m&i
zH)lQFNylll{mGLHdn@@Ys|Qc<<eWOgMm2mgm~<sD>mQ1mp9y57-XEoxB@Y79-rX+W
zv8isD@0gUWSkl%P(Pu7*#z@v_zIWcFprokIEnQa}xHhR^=zaB`zidA)1FpSQrr7G@
z@+Of}F$vAqJ6Qd#{6{K2+mB^uF-aw^tr@rI3%D|OoM`&V)m!Ix0;<Ww`CfCI4>EmI
zKKYKSA*J?Cn(`Ht@h;)_zx#(q4T>CQTjG?z(^rH{R#b~nmVa}AedUY$vRv(G_bdw!
zPsKGH#jtQKpLyx#wHpYNlK{`{@ovt%_8l)7@}t2!fx|<p)HFYK!#IR)`pYbi=_e<u
zI+Pf>iM{(97;qWDs4+e)*Y4$AiY`XxBKdKzHsM>4yTc2U_;ftQt-EBq^5xg}MnIy!
z4JcbY2<-2p;@u7y78rz@uLOvtl&$?t&Bu>F(~?%N++t~oqK|Y$hOk|G{(n-$;4v~3
z$jv5}@oLUjcQ~fs=5m-0^b_qN&xxtDdp@^&K|24IZt<d)x>!td9uP=c&^P(7jW*0n
zc&FGDudK!#sZ;GZeI(K;wT>a8gJ+1jbXwWCql&p7J8AKHZivJnNR>><8UMPPN|>N`
z+;%R+PM_+wr1H`It*zhakX2bUk#szjT?TZR)<oOwZj$Svyo43OB1iO0@m$Y^JjIJ@
zI#}#XQToB{>}di0Rcm~?;Zv@av<)P@y-7}Vj85ZKZ`{8((4_(o&sjNydG~7HdRTCw
zhth#O+u}#WTY9%zAgh|`C=2F{uU-f`ySEu%4}*Y626xO&th6^*Z*7oyFX}5LIj&Y7
zE7ZfJw4|%Pz(VM153J>?GH&2%*nTY;*DBAD|K~{&5LrOl+s1o5c4cqC*TuA=0?G^<
zZ@cze--sB_NXBNQV=VL)Ds6`9i_6y$(R%-NWbcQ-MxL!R&Rkn`IfmFODP6a}1hh;}
zfios1Bbgy3Y#N!mbX01f-0dG%*nroXWZ!$4uGaA2gV#(GV&DO^+MaMost%1;K$?%e
zKJ5{DVU3IrTl7xQCLZ<ma^rr6C%s)58}rqqZ4!n13Kk$o8~4h~7Fzszl1q=RZ73R#
zq)iUHQ!t~y{p0l!aP!2(Rk+GUULB~EEWh$o-@g~rDdG36if}SYqfn<7O{RkTppX@k
z*tPE>z3^(#N*h;iy7mvN+W~^ynjiP77h}Hh4+Xr8bUMd+T)J|Mr>Iv%`!mb(?-qGe
z!z)nlpGwzGBHX~TA2{<zMMl(x(li?xyH!K|2id6JQZfF6zcc&=pW)#e8|T$+q+-2h
zhM5D^oSiF&H+C9Ro(@B&u?x$Te7<sX&HqJ)AS_mbWi5>CrMO<!AmN=eE{NGg9V2l;
zcBB*UMpuvZQ>mN*CUarkzYjzXjOf;ERomc|JCJK3g%kggYHuuCGW)4S_=d_fyZGjU
zo79gl=d(Glx|pvY(@Pi+Zuj+5^jD8rR4m-i1TCZS57;jGjcpBCZLfICF6H++Hi@Vj
zQ$?Sg^m<juCqc^;^@PdD>)PK4yyJp|WM=*Rd+c%~3pdSr*6OIxb8Gu~whO*VHjB?(
zKP&>0GjP?aUSG+-z8fgWukq1ow>-+1CBg{<SL2Kdoqn%6lM(I4brdXm3zmg%)UYYw
zD!Szm1Pt$1v%7N9{2p-qgQNplb>IgE!xQQJNr2<=@Fj~2-+L)Bu<w+Q_^<QS9znq9
zPDMk~X)7LDt?V$pu+6vxp3^yoj2iA01_gaxTHwu2l_SB|-mJ)oTU_vwyI@sn^Du?C
zdqA3xMD<w=2A;NgRkoj0)jv}U1gxSAil6_L!Pbu>-4-Ia-JGv4C|>lWM;_6K?M}>w
z|A(VKAJ|`;=7UGB87<@X>y^x_&*$I~|Huq^laei8^t1;S%<|aZ;InhLl4z#Z^#&k#
zi^0tj>F3X@%XF>1U*`T4d+^*Am+g!SR<3<G-)<Lf5ntW&VmcY+-BZ7#@IawX_J5w0
z6u8pDZI%1iYvoDEUCsS=MTmd`XMDRZ{D9Kdd3*VMcfF+0hyAHv-Nc6azQgq@q74Ff
zO4eHEpSg+%-YBS?mc!Ia+49l!k<BvUZj^jRG%gAGq@(t1*|Mjf-a0Jz%dJ5dzR70g
z-?_(lZGe;RaP-KPEqQH!ab7>C!T_oBE77`^Ug%`wtwBe^i-du@t~tM7&_D0e`+h{8
z^v-<y>4*KoR?xYe>y{*o2JM-)UIihhSFd1m4G;b`LBO3Xuxi}D4qCalK-norz@p6v
zD$j2#BjpP330ar{kdHt_z9H2_^5Ol=7MTi}kLvWT)-x|nrsFc%6#k)h3FTZQR1B#?
z?V1eGsZlI7d0yBcLVG=o*BID_q18}!?W-w-fo9e(&=@wD{S|Jv)C0*;&WVujcU<fe
z0d4t^XS4k(c=l$uHRQb~k(eNPTTk@UOYl|m)F@_}Kg5Fe|I=sB!X8@=E5F`fV*<n&
z`@~znzX^Sk6s*>oZN}ZJp*E1cS4Ya*K0p5?+av$ea<C}<WIOdd3RE|vx83{3RUt?9
ze9rt<%{QMst))yi^c!d0`RR|-R$)6cJM%#(2J}B=;@J!{bm}~MYTe(oz&Wvn?Z2aW
z3I738%Y+4$Tw8M|3TOb%E*I9tc%C0mY=IX~Lm%+W4jHFLt7^N7zrRBtv1sz~Lub_I
zI$oNi>f{u(ngxAyXs+l-(a?Z^uy?lzTYb^-XJd6d!a<K#8W1)ejMc~Mbwb_CE=bp|
zE=+C;W+{EVacv|AIrVdGv4-D$P}v+7pLNH-;w8?LiDfpDP>W$yk06PjIkEz2$L;IF
z(3J1Sa1}M3J@gLNZl}<B6zF2Nog8k@{rR1%n6JU%dUm*w35W%n^w^IQ+~|J?{7+>F
zBE)909x<NNG6CL2rLo<qN5=UD+MStliJUnFbp)HDhyY(=)1~%d)6ntP`3?v~54X*>
zMl%>mZ-GvuSfp#E!#DGRf*!!3(gOzQ%g7AUwbRyzf^fUyGC;(B=>UjyMW($pN_AFi
zqfk6NJW4qCy}SGcSj>F4gq5p=o;-`(_O7?hE1R=y$V#n34@VU6jkpd*K;tt%fBrn{
zTbh+oH;{<f7xw(-Cj48VAp*9B*~^+)RP+*Bb)h~+-vP`|E8MGyxvZl2UALsFoK|Nu
zW3?lEP;X_uIp~ApOBd8?nT07N^Fr`x<sY+CIvVTbvwlXemQUt|&Us|g5WucO1<HvC
zaDSmuh~8Cx_2xz*WY!Ghv7i41mu577AdAJg`!LQhE&-(#u#idiy}NRK5a_O`M|fO-
z1IJ7`Apos@m9Weo={%#nzKe?{-9yfB4_L1~mVY8DnkgPcKoh76SfcVh@hj>VR53zu
zu(VJmxpL`!gR;8&Yh!GVf0T*%obpC1Y()V#2SL7p`G}az<vpdHJw5<n(CmbAhpO*~
zDN#iaQ(Yf70p2Cym<KSzJf3IOfFbU-ApgjB4*JVupWJ)7uLF~=BQQdhI?)FID}VUv
zarzo;)YOl65fq{~O#AcVtIv*>X<BAXQ{vd$Aa?dj3eNg1dq2N0MozjZxI(@qa#_Fc
zr|naBsa)YP@6YQ=7rnVOUQ?x9pgF|A+J*jiAP)FIRa6acuLhDJr=<MBnC%7>6b8UU
z;C=eYUqt*aMYA1=18<&&-UA?_oV#FsQ#+Mlk)F<89UZ_3YP!MY?vu@CGPc({ChFBL
z#qsLufaEO+{s#0Rc%-CjyDp2k*9N~44U(x&OE1gSGf>k3aSc&Chj~o(`Ps4E(T*vc
zBEo#Y{jAb)dA=k=Zl@a^uk4ev&*qIkZ`1EK$w-B33zDStvn2s10gf#05bppOzzoR`
zq**DiHm4ii?-M8gwZ;Eaym}j;o}FwfM5e(d28$Ma$%ea~o@YwHMPpaWNQEfo{lrc*
zwI6VOaH(csaKMc!{_Hz24#7)m!I%mFso_PgO1xny5K%mu;4<xLX#bI`Q&)?EimFa2
z@AH@V7=@RHV;@Ve_bvzT04$B%X{BLXj5N>@bx7a?syhMn^Eh9ux*W~MMQ&5uy?hBa
zFmUhZ!;a`lpH2<Rm;;dq0<V>6WaG#|po|5rlmw!9(0*cV`@B8df`9H%D|wAAPc(ox
z=u8OuT)tB;gz6wUs33SKHtun3IaFpIdxu$5jo0xHHIOPrB=Oj5I6T7+Jjst~&^-G6
zaT7|LH~9ePsalBRnrZvnd@%`$)(D79J7YBv6(z{YIM5D{*SwJd0uTN{cDa9+6`tG#
z%wJ#P_WC^}&<Y3xtfrbo@I4A3t%CdCFE2!7*&gP{kNdU+Lm3hw)8%!64Pt77{f|-a
zJz$>V@BpR#e&&*$j#jDru@e!yQE+=Amjbg+EhFe()$J?~C;%SZ(8%`aA@k8+*YAN>
zl{G$^*L8Ob#$prFCy5X?Vk|L9Nf|K3+1&x1SkRuOeUtxNK0t8CTWr<iyV)x!B&Exa
ziaO(fyu5TSNTNWiiiX`776r6vZ81zQsU<@n^km8qmK3)D)_XN5Y!Uz3UZr9I?d7m(
z%X)3EG-<>bZpW+TTm3Js1N)1?<%y+MV;b77b9d>Ve-zV;J^VfcgOqqa4vX8KI1<UD
z$6@8(pMb1*TPG2sG)dqbB@hdFRKuIPMkSG+Co^O{xvDItqZP#mKUZK;6Fji%s3uGR
zgh#)eV~a0^dXvcBfQoMgvMU473H$1K?h%>MtF8cOaIO|UGT+%fW|G_4$-8T$cbY>^
zt*l1~v2GYXoXa!c4?=mjurIz(y1ET|9!59Dfh2AYG&t-Aw4CEnv_UJjV}Nuusp%RL
zdON@wH(K)8tKF{mM8PH#v~W5)0m76yuu66l?)&jajv)ERPEJmG>Dy^eoTkod&AQf+
zzI>*(c22#@7Q4M{OHJq|D;w+v(w-5)IPkCz^R0N}6jteK;MX+HJe_}yNe~VN@tt@t
zH<{c91?cvS{tUo%f<8Vz8P7jSWO=~bbjZeGfx*=C(;9)J2XRt6yJE1SgP#=fki0(+
z(P4*SjmVDHE;q3sgaNT){A(BE&Xl|Po&)99I;H~!$vHs_<Kre+e>b%XY-;I<E#z`j
zzuoP^q5-XQdN?$Lck?5FTEleYTYmLUkBnl5)a{qh2F(+7N-hBkJja&0CXxivA8Y45
z@uk~!SWiY#-Ix2B;)OhG<KLCP`gx-SdDXsCz}RT&1qilUV_koFurMPKY)!Lfo?Is6
z71}fzU5@TeAX|IS9KxiO6+Bd|WXxa)OB(VZKPM@kX*FWsmN%a`{Fz?1;*h2F?AyaL
z+yYg-(Q4Ncfbb;XTFOd3OR!9t07HFjGi5jUH5q+x5#@T40W82*YSAXH_mKptvc;ER
zf1tt7N)Rk6mQq~-2xvyGD<p)28;H#z$av7Qcihb9ua>%R8hQ%<c*qiyCJSLA$u9Ki
zo!!}4>h16&;B#KPNZ%myWdfK1S7GVA6R*3%qq<69hMYB<n8X*);2nhq&k9){W7V$P
zaQSBfP%^?}-;fI>WME`1Defg+nYou3S+Z_Ry8oReNcx;)APW0iMhMp;zzX5$_%v~H
zOb)IG_zL_KAalkvlfGdj=hBo8L|?e38=OHU=sJ)%ep${HRSSd8pxb66AsDnNgcxEy
zUj5OrPTOr$oc>vw*W66`6E9Zd7@0np$8?2j1ICopv!;)DulMhv!}g;f)0^ew@=4sU
z@LYKD*9VLB;lk0S-s}$!D}&Fs=Gqple^OlI><u{O;^9z&c2%S)U?d57YJ)t^^wSRu
zUrrgG+WQKZhf1;y+T*$-wQd@XbPrIQo!3E*y8J0ar6z(|>13b0KTq|IIf|8%BA95v
z6|RvG7jFUnC5N{TU}@~ehE4a}H1911^J&|^WZoKx2@6)whcBU3eJ6_3>agJ-*QJPh
z0IYtA;s_Z084v(%e-BPPbHC*Z+h-qfup<{|*u|1$Ol$18V}JOK{+5376w|i<E5lSY
z^J;(K*=R5cp9;uJRD>+rcgEhSaFv(<VFjpsep@{VzLqHQQ-h=?TJ-o*W#{!9wLkx*
z@t*<EV{d5x+$)ycUzqzW-RHOHe%rx=a(Qm^iICVhe0u;#Q@9^X6KZ}=_Pw*jBtnGY
z`tjCqFs%(}1hFeQo*Zm`bzD|yNfD@0e=_n(5{KQKV|>Z&TFHl&@f1|gUn;n-)V1dX
z6^_d>j4!{n_Z1!k)mbJ`3(>SZGU*~$YJ=5DF0gd-T|@^SsFWVela}^_RV)&;hxE;}
znD=-|$IsE32Niwn1!pbj=vG5)4J%Rl3w5lWw9jTE6=p$WcWd#}hXljY%A6ppwIA2=
zL9`4N@Cas&++>$dC3J_|{iqb^b(Q>Z$e4{`N#Q^*oiJAfTKlYyl_xe*i1>Cn5sjJJ
zt6$j;J1POIJ#ttq`QWy(_{YNcR7dZLjGswy8i!Qo;oV75^<j&3^oWW~%exDqGbUNk
zKSXY#e0IZbZIlPpMeh#>LAyvg%sgM})Rh6mpft?LYcud71f`KHVm$Mu(Qic8gNP}6
z$}2}HCMFg~Bn&!TU;1)YJ5?#V*DaBVIW59KQ5Ki+Wo@+Wwi*()zo-hR_$q^)fa(G7
z&K+t{T5yzX>=-|*`$y;(fd`nbDD2Lw)Z>qoz<4m_PbPa>33}wRUkzF)fgC9cUU#)d
z(a-o(VT^|PV-it<>N5&d;nJX@Kz2?Gmc=IFHNbNL+UMd$Ots|QVR$s2sI377OK4fo
z;I$e`7)~H9j<ii)%x8r<jqeAucC{K@2Hs7=;wXKQqap?Yb{P)*Vcnzw6sYuCm8J2~
zb6g_$CY2}-^UEMB%Hpm6cWtL-X@8x#QBpw6udm7uMuYEVy%7?;P5|zmNaO@ZEN|AJ
z&D6`h4vXaJ>FJ!k8>v%P!==(+RPrAJnHf8}%6VXt)EFd9W~e!sh-!PGGZrM{rywiA
zgKtl@t9@jby{b^@HI6<5I0fdpP7;0*d<Qjkfi###olCz7GcYhP9K<nM?Ki6&IWKZ!
zIt)+r4>Zhe28&+VO*fzeX>F<}id+=w?FzfO_lupIvn`Qxg>`3aGRB}}dN@|)Z0uSk
z*<gC-(jd>_xl?HWz0_s)euQ2LbRBI#e$OOyx)W*879*G9zNhZGlo^{L7Kn#-7%mJH
ztQ~gq;Eux|KGG;F+{#lgX=41ORqF;76Bm~T(W`@PSR&-Uw3Jjpc2&0fRq<F92T*&^
z819ctW*+FqwInY3mq1iz2n4+}(gUF4EXk9PbbfFKR*4!2i5Mek<k(8SoG`rpxa;R-
zrN(AEEmjhB^edbSPGc+q%<b@}Pa1=OaWMpU@w)lPNDdYbiOc&^8HXz+I|-B)@FMFF
z_~$H$!dW!K09dv&RsrnMz&v9cbn9eU=bWvC<^u_z1YXnhe(S)LxTItdto8zCf#=Jz
zG7KVM0+1G<!ed_kqif`h1Fz>EJ-TX$h7ouJla4!b1QOfuv@~)!A~TUI1vOn}CQm_e
zoR<~(MKu&qtY+e?|H>QWqgIe34EI)sriTNvtS4%XR5gsoH4d6?-@il^OQ0nhi_On`
ziEHYUk$6jt!0N+!1ob+P8v17w1zLPS78U>i`e3qFcMQ(OX#qet#KIwrx{@H~yMj~Y
zNs!0btjGD;Z&}YYg+y?et8{B+yxCc#h<^BDtM~bB&TC)_LNqhQUDB)65QGGz^mxhY
z*?hrfOu6tdRcQ>*p#THEVa2Q5@~elJ<JQ%ie$WAGgBl9bLH84&9c43WzXvbjOr8`1
zsbv;{c8O63H9tRp(ap}Qc1MvnFa_dK1HTJ4R9DU$u0sb_+R>g15P>wnBjU>grl$|J
zNZW45Ayv|g11<YpGnpr5J0jb|R5Z;Bo9`9|>E~HMRU--X@z&!Ai{PNc9Tx3TTGnp*
ztI1@5^Bh`7Sf+@VKtdVaVz5X=BJ=_MEkfp~qM}Ep!=*VnI@#zXq4fk^B;J6LvE(F%
zQ!L1Dih^OZ7eOvx!!V}%PtTjvTIom{BXuWm2*OpVp4R!D{H3!<{U->dCkO}*j)F^o
z`U*#Y8U|Fns=Q7sbS%{yK0_JgjaarHb#7H}!}w^V-z;`f(5vLd>3n!Mna^Sa&T7Oz
zdgUXVDyRk5&}Tl4ju3P|ihJz>^b*Bq2m#l%uWthh%M}~LPiF&<^PuX*BoovGpo5|^
z@iRf#IuHiK0mEB?-qO3M@ZbYdK?FGbG4y`wwvrddBJoX`7)I8T4RGwk{A?AJ*CFxm
zn&p2ZzV0O=fG^#}L8Du7FI$8!Vrd?Ak^O=^yx2Xoxf?ui<ftI98x{S7nt%mkk+BL7
z@-)t{=SX8xxSkMcq})2TSiiY-6ly;Q-0Wk^Y_0q8(_EGOLCj`CE?>c`HIo4Q!Y0gr
zsY?(xLvi4QGk^v_QDvkevpwH22gtZ^Q2k|O5y7y8y+J9b;|CIh#zKKXO9UeK6Tevn
zB-KJ~juP)(EWiAh<q23yyv}PDoTYR+wbepc@6;~GK{EmP3eo)y(M3}O^y=ZdU$`{R
z<n${z)F1<*2Hnxk4Zw`O<5e85KtAi147tzI-h{9`P`LlWE3k(Xsq*LM!RE{y;AW!(
z0?<sm(!A9rz%dDZbS$j05M<;luY3<Uu{;;mfD1ydo&^QaO%QgY_D3>go!|%tuZ(!C
zuvw41bI_b{JtQp=Wzpyk2$7NB-`=buoo7di!RzvDrcp>?1me8gTEkiuHm{Zd<~!tC
z1q$4+%6TZ4`M^sHPSeiGv&&zE({l91uV2B*jn-1rUa_FsfkK@LMt}F&aLLjBn^>qZ
z^&mnTcBP4Y1&4#^!HspXLc39tg#V+Mf^c)IrMS)$!J}v&h`f_cZC+Op_$zQvh6xhA
zi#Sjed_`%4z^3E2`3`J?xJJ1Z6HCnwUf{j9QONQQ#atDpZpd%XlTClex72JXc(`2&
z!2lfqNNBgWq7FS?tLiW!SRF2le86KzZG(z)i3z;C_yF{VDn)*~7#B%?@@RMYt4TMd
zD?BBD`rsnS1JC9M!oRGzH}zO?h)}J`hCqCymZ`3E(K)~fI=wl58~`@=@g~3LnOkJT
zNYSgeuG=3$K<NQ!FbAB`;d+vGrR}f)Vh&hLd9uhwMx~~;Ss7$4xw8ijwnTxLAO*0D
z+$ry=C<iq)HMGr@)LYFPevAmDCPR>VjlL<a;JbNyrD9lsq}~)t!f=a#5!c=R6M)Y~
z?6<V!69KIzL~{4lWv!$u2duBZn&q;cJxPT)cGn4<qcBpBWBvdqx~q@=<ZeyA#W|v*
ztlV|>2B%QefVz_!@U5Hs>9|5jwkSJ}0`5l+CFzZ@WIlKi0_S75p99>)y|9IK9pmJH
zFAa}YV8CM>xfe<)?lCwoGn?!@9uCNtry#Lp0$CxO!7p5T?P?c<kf?N*my@6jQ~dml
z0)8qF&UEZM?eF+|4WQH>&J)99&`{r#b2)cVbpdW0pp144tEn>K#S6rP;Ee!S8$w;K
zsHux`B13Z$@W$}WLd0ni_ki0**uhT<Ue_t43P!-u9alp<8Z~8_`04I$spSYK6sV!`
zd4l|Jqdo8jWNGtSWxh@<-SZ#R8}KLzPGj9l^1c=lRl~~#nJ$M5^VcQu7WK!D&(zZR
zv;-mxoD5g0lXnip!p8n4pC&fFj|5;8ezIe>Oc4)mQKYc4aulOhWncS`V~~#^{QR15
z9w@>EaL=XqsQOndffn%Fp2p_Ji^}tiIt}%AnMX40>n#k72pYMBU*H@VB@X|Fmk`!l
zwNtT45Y*twTZw9O;{2>URZK1ptwKi^Xl{+zgMgqj@%4wwAq8;cECXO+BWI2MeW{?!
zk`^Tt4jQ4c!539EIU-{6?j~Hsc;r}jx|axWtPC76!czoD;@sX0E@+7RXrRuOe!aIs
z|FYaF`f2n-X!dgrk@^p)4mUOA#+N$%YQsP7iaDZNf-{jBdX4Wr_}QLZQFr!WQvAj;
zZ5Lw;-l+h`_zsb64>qRkW`EtLS1m}A{<ugM5YKMB_sYP*lF%ERZ}<g5kzr?wfD;kh
zj2L+%5;$6dpN|MNXu*SmVu}A~@sklzKFk;U)-4%ZzNE_rrf?b@aPRQ7AHZEV5q)K>
zYM_0-YYYqMYiZzTxptS~v89B+-{=A6fFzt;f7E6G!iTsk;Jw`MppH@Mx+7{S3L3WF
z*)9e|6unweTD<o_nYk)G5a90K6!yJbJr^}VP5OJAUg$XM`H<j8EOT_qT{iXDU(SGY
zgE0WGtCz;$-9#EEhy!%4$+So;vPzC(<H%Sq8IlJr_#a#h0Ad3C5wO$_Vy4SV#UYX~
zU#9i4MDrp6L1<{RfZYi0L&7MOl#~WTUkxwL0)nCrqEqYElcPjS^w=b9wlh^oSfb_8
z$@4V8etZRak;i7zOUO^59qm9_*!isAuoPaX3`Il1`OluKN+n0>Pw(X7)hSaJIACPs
z9=iyD89KtTDjjjGqJjsL$ULrF8Ka>0kOgeerZtR0^oTDF6{Uy52DIQJxk(@CdQ}7O
zSuzQfxm{H5Rm)QX;$&n4{9qF`2=G1MEMKzUS_jr%&V~Ky!WN1`KolPtHM;>9(;w8;
z1JS9$bVDGU>y`o3bvtWui(W32|8O?)Oo&?o_Yo4HG9U99S^<m%A_&2w89j2e14UtH
zV^YwWXipW=DbTKY2nvH(piGqpSeRPBDYzChr1mn`ZhQt~ec0-`Xg_%Ip?I8ZN&*Ls
zZTu4)1h~;w8I~GqaL%cmFCiNXW%OZM;4Fv^0nGcKm}6g>nqneCfnw9d&aEdydTT((
z6JGlPQ805)%^PQsy)d6#IZ_gEB(JJWOn59p!$eGeUJh)ur0>}|H-qx$F#5fTI!{l)
z#wzGyMm$GJst3);cD5k^qkaY4rb4oE1J^O2|BqTlBcGKB4^J=iSs>~qRD29JNrs(-
z@S?bMc`^yu&X0U{q4znS!cqhn?mkq0)b@(cX~o1TXjd$?G|;OGWgeEy0MZ+Ne)EeG
zBROXd89p@fFki2J-rqznN1n|<p_%~{<#3jEX%t#=hwd49AcN~g&_+O92BVdn2~fg0
zy?O$q%2JyjzUW51=*c|Y;LwWe;aqGcKrY|z3BAR{^ae-jS^&dH16mb^@fNzHiV{<j
zNZsAtA4}F9B!{n>PjXn*aK;@x(5VTv@5NaPuljR<A5J@c3NKP^H6z?FK+H698dKGA
zhp3krMT7SDFJN{V4of}Sp?+()QmOSIhfo9a&+6|7`6;fpsp38&N)Mz6fC?fmtE6_o
zt}<&@=sJ-NKvpUlEWcR{_59|%YT`}8^WS)17Rv=(p@P#jCG3+$uYQW1SM(UlX{c#B
zqqT=q-F%96kXirZ-K|W=<^C-Iv|0cGH5;o>Zi0d&_n&3UJl03R1xUd|o?_-xZ%)mo
zi`0PY1tK2OMiv`2eEq&4fm9s{jOG`R_I6+ffK#CWw(uhRJegLhF>i=g^ake`<1)rJ
zXPN>2;m(Y;B}GDWk)8w~GnQ4q=@&TpCyxJG<9&?>zV-G01t4NRwS>11xHJ}V5w1&9
z%#a6p_fIwV1xZPcg#`Ls>}{_AU(8g=zZXcG@@RLNZanpdEOT*&7Y(9Ae4f=uHgL33
zrb+4<8u<npq84v1vHC@G{s@*TR2RQH29i4{2kyyU-d_#SR>_%GH!mc3-c-2w_*5F@
zYoM<X4F~d!-j;v?0SD#dgo2S4IN2l4G2jPE?|j!OF$&?|`bAX%T8(6Ai^S3Zd$YBf
zfbRUOaW%7^jKKJJ*~HYSsHlhE>CkRpV1_Ob{7{cbjNHJ$Ua_bJnovr3^*Z&_zs(!@
zW(GJwkQEcQ_9r`0!G;_+07+wH4Sp~mIN8?~K3wZZ`WP<WaJ>M)WQIUAKfj>HCDn6Y
zUP`K+!taXYae{eKxYSpIJ<5Ay9RQawZZLSiy@}Hbbnor$@7iKnWB}?0q!0)bitBBK
zYt-MfR3-s6XcV8<Z<h4v)e|$81#sq}C=5=^E@BRYoofSCI0VEWqjGz>X}~{sJPF6c
zfFq*+0Lk^DNE$0|0OtD0XJ#gHmYwptwXZN&qwI_S>kM_2JV=qn5S!Ad2XKNdxQbrm
z`)z=#GSnvluMu%kJkRnbYM$#XkCYdJ68<S0pz7wGpqGW1cOP6Q6GE_X5hCt+fBsB&
zxKh$NTeCkN2|r>D0+`(MGm*D9TC|+U-r*sA!!v<*5Efki^wGnMhu?!lj7Uzn6|Z-{
z05585e>hR;m^<4Z|F1%Z1nnNx#hh@!4jtmA7QiE|Cq6u(!^uN<mkb1fyQgn6K`k`@
zAr|y84@YbQ2)!L{0SZsb8woW7NCnsoTcaGxC6R$9$kH3+rf$YSol1wC#jnEvYA0&A
zfQCJdt@Y|lNeB4yO@Ow|_Q*@gN%$}OC;&d{zjrdns)85gKxP7VDN8Q#>vGDz6|#W(
zvq`vuoi^1?5#ILrAg~vQN<|HvaKpDAP-1dlwek1Bss9Mz+cE%Lz~ea_bt*VWCxW{p
zAaULW2ZMTvJBz@)0Lz~T4sioc7BH!RDj02lcps1&%Ahp>iM7W@LS*RzN58u7J0D3_
zp9R~1hpV|zasURGD^VSO+QCTm|FQR;(R8(4+b|+Qh#t~(B1MUkkRT+8P7uBKkmzOW
zEkT43ov4Wx(W3^@Mejt3E;?K9?VZbY-S_*9Z+zdM@Aor?kufrCd#!bz^PKaT$2{g-
zfCb!#p`5BwlJoQ4l`~bJrx-$VzKu8=I&pfrE7hY6TYPE2eCWF<!Wx+E6I{7h?YdIM
zz+i(tayO}n+CJtr5Ke~ByM{K8`wi}76?<oO-0~m#F^&%<d*{VhS}38CWWR}lzW1u@
zM3`r2Uloy`-$$R5M;`1kC9lg)7G(0HZFzkE26f|Ko)i-uAL-t7PBUO&XWdL{HMoA?
zjIm4;aTa;%j!5zRF=;7;Wfs-EYQ}ZDbLnuR#i2za6*d{%-DMh4(SJm)Cm?w2&{@4<
zja=D+bg<H`e1BShxX{!FN=Qzi_<$USl*9rnEynXQ!iun{7m5RYB5cE_yUKY^nAQy|
z_JMgQr0JhT#-Qae71EbknS7H3CtnYq9<kPbWb4h5Tys{L>p6mma;iVwf7Q19D7!tG
zueKP<l!fl`X!2G7PZIc0Fpb69Dx+%vCMN80j0uSl6mrY%ma}CH$RxlwGl2<4K=g_U
zhKlg;7!C#!4ZP~XqM0n}5Gj{XHQxLNRXiQcuDI77<A55qkJ?Y3gn<uHfVXi*9Ip~%
zC6KEAbS~n4*;V~odL7zJoxWWk7KKcNyvsxIS1^s`gQBi>R%(wUYc71baa>$NLc(ei
zE+iOWcx@r4G?;n-Ii>FON%xo1WaID9xNU{JWaZ1&NspCz*&Yu}kl@wxdi0QDBnY^)
z07KlysTQAq1`bn{Cm{bvNXKr4%Ln$*`6!Jghsx}L*#%~?(`TJnUKn7<lu101dI+Su
zljpQ)yqR+X44EjZ8VIM)YJ`VS^E?LwR!qseu5v!oDANAv%;|A*#9dr#1PcJA(XCUH
z(RMco`Wwf)<FUgifJWRB6|m^FXJc7(|3~K+E<YxW8PV}LE&>c})_<q@)BQ|Htd!5H
zQ%c`nOk9d4p_?$m%qGf+Ed>h_DXiwl0WUJa<^{tXT0l4@z0w9Ju^Hx=Fhh=v1pVky
z)6ssSm9vCyFBov#=?@hV1T-+-wptm!AH!prfDlaJ3>W}Nhrz4@@x;e&NV!`=yEP1O
zjt%B<Ob`8LS?p`jGby;!nIOx3o4kwarTz5v(YB|kw00=Z%#*5XrRl)51oGlq(=!j?
z1w1VqF`C{09GeC{XchX7^aUYGc55vGZDqDNuu#59M@Ls&tha_+sv%@jNCC4}5P=M|
z2RfiO&(skkba@t|j}&*Fn(U92*|)ast-!1mZquH(kBU$)*AYbuu_J;HRueesA&B^U
z{?3Ji7an7j-uxC^S5IGVGIfxQHU>d2GZsR3zx>byDzEpylcwONuq%MCky?1B2;naB
z91fv4+<l76F1iP@lucjnlBpyb#LO^P5x1Gc^AoXv1=4pxL6Mj5Dxc|y4hSpQyjGy;
zl;9e76Uzq~m1#{-AU0pzS{)xh>UIUCN;Aa0?{l_T$R=4}j?&IS<0)ss8sbu6(o0XF
zsbWElkHx3taoe*E`k%=pmg)pTX8&#nc>5Qs@d{;jrU;pnp?4OoJw4~n+Zn$CIZG;J
zzs5M?ox9>h$dT*8I^gp{KU?}%u9l<uFs!<lXWIstnym)@v$EL)B`ny{)B_TbI}a)e
zMb%2j@pd06xWl{|r?na!a4)0HoV>h)IV^$KF%xl*V4O`_@hHOifsSrlg23*~b{L6C
zNa!F>fM?kn{|3P+hH9pWVhk)iv5SfSOcFT3j9lpNGm@KdA*l%%pTf1hv=wDFs`~}-
zDC9vF{59ZZDoMJS7RvAf*}{y{jmJmMIm6-qdmbE-*-7>%D4A>ibXGF!`$~tHGk`mB
z@ZMpTo?K@*mxT$Or9#OYUGU`{SRgq24nZH`DiEGJlv$2?p)Vs0ghE~9NyP=148*E0
zVj<Kkq+u$L50|;_z7-}wRxA`t-Piz@$awZ-XXOtQ3O#}-Jrs?Fr7R#AA?cMLa&AiH
zMEo(ZFj=65^}J%NF`}OYh6!Q6WZbTeGRa1jA)#ya{Daqny2jb_G&L~PCT{f{JwoWx
zUd95E6-Vn!@GWcy&-9Qg%IY<MMx0!SirwVt|KuWS5ND!l76<`GS>Y{@{rq!8DgYVu
z5BK9TpzsFza$ohN;PKV|__!D7DT)oOseWW`;v;)W7{g-}@)Z}Ukch`uLypwmF?dkk
z2=rEa(3XJ9PB?H(QUDng2CkT#MKniIxROfr0>WABB~}CAlEIolcu;`BrbFLTEB2-W
zjPb@!JErc!u?iJ}gjrM&YJz&HdGC-cN<2)+&aQBNxg3FOkohs|>FFUQE-w6B2n(iW
zVF73R!_NTw!<AErm`puM*XS5{5PFywR0;Vr5~JCXxko3*hf*NVnu==$^sUS41u4A0
z1p$}&FgMrT2K8<Pw$Vbj1UyC)I5<O?E%0TAuQ1XCyi{9GA>5gi&a~zk+x*c2l9Gj+
zDR6^AwANmb@<t2pfJj%ko5=x6A)I{Wh|Cq}vX6T{{1B2ONgQE@RC^jnSv?CdaFPB2
zXPh|dA|llTZU7V-<|C!@+@|UG1?+EU-`wOqdn#Atsd&&^@Kl&=Ymm^!pmI&xqUn%{
zO9<>5)hoKf4Q_ca%G2VCBJ_CggI?tLuGJTWnCiKebbSM<r-Lms@>q^>+dH~!FET+*
zgv;o0&mc?GKhy!}7qzZ+l_L8iE?zxp75#_<G)$rFRYwYdR9Rp)ObF0AQoyN^mA!2Q
z^PfVJlrSJcS^;?@UW_6*at*oN-80=mnp~pA$)%BURPIpqQz^H3zsk?Z;RCPa1Z$SS
zlZFF1w-u(A`gdqKyb5!Rpwn0wUADxw(4Fk3hBT<6F=zzpglY56pX_&fv>@~>Bn4>?
zjC6w|yODdw!=H~0`zImMirqq(hc5+d*SKHOf}g|FX-Yne2@4=jh_ZNk4eqP_;yld;
z;(3w0?1$}KTUH#%oTK$S%J{A$i&y3O+rFpW&~8BVpD7ZMWS`|BGM99+$2Gtg7F;)n
zWYC9)#mNUbq-YqzH0Q3UE&|wrau5_S8Hb#pqV#|NJ`_qznn(!EfWot9aNH!FF8i4<
zSzvd>U{PrYJ^DxcjY%Qm2{^GrLPCIpC52KN<kz7b2G11fMNB4@a<v6wGMykl2FpqX
zzou}f!3nJOl~;81l>!lwr2#htrKT5w|7cBm@&-w)&H@|uxBnK=ZtyusL)nA;Yn`Dc
za;>uSnz1(RF<fFuQid>*K-`KU%sUixBvFUlh~pVVJVN~HV*@b^5GEp$v;(PmhzBA8
zoS?-R>9g%dg~5cdgyrox`1r})q*w)nadh9C#5twHJ}WJ2lm90TVM*<8cY{!H8?eHI
zA`h4`FxMbHkI)pjH^2zIY}v1Z?6Xw9TB*tigcmmJQ=-UFG(>0#6J>7G(a~8zha1@1
zd#q1i*)@u@0;vVm!Qf3Ai2ae`NM2zz2>fo5-IV?a>A-uo%xus;X85GXs;`xYr&?0g
z(0O_`0Q6K__9X_%)1I;}R4c!<>e)xB3V`lMe{tY=#nNNbTz3oFF|{u{&c&ntA~W4U
z6D}1(FIq7SgFoRZy%?<^up$D8p)RRY8CK2GyshOS#WK5vr&qm&%DMx=LGj;&BWh5{
z4bk|7o|uY?6`78Pz#K%B5@|o*@$yk!4p25HoCH%KrsKKzB3r$`&*%V<>FD4c8RRn1
zujgEIa2VN9dkDvuiJSWcKsICJNtit`P<iic;EeeHEp>U_wSu8C`+L*DMo3&hs(&EY
z+MmgXlL**2vYLvQK^Vh}@jT%TUfl=r-5%F35M25A_Xj^j6mW0ONQpECJK==d0g8SZ
zAe0$ud&+b2#n)<4;0H+0u{6LR=#J|=1IZ{J?(%rm_|bP>(;mixO*JsncAB|^97`)J
zD=d{YVlWzs35df{bi%#}_1o^t?Qo6nj1;f9Z3Sb-6ca>VU9=p`tEsRahVpU&$snM`
zx%TJt_{bh+L8NOPyM_g|Arw9$bOV_bBKO<~gdF181tRJ*?PYoS7Xe5G4ZJ`$A`Tq^
zSvpl?!D3J5!LB_$@LtAGA{q8s0wOxDieR?+)mKo*Udyc^MpCzlXJBs@^(nXEf(f2z
z{T;~e-ghMJgX*6V2AUxNY+<}g@O(%%s{9=;W+MK}$t|F6Vm-LH^Zyy#HKFANFtG(x
zZXznrmWPX^04kZdqk0u?GD(K7S=`$~Ac(3EoSyR*Safh#hv$KbVMmm*M1n`pfh;vP
zOW#`gqXM#5!;2l|Rbq(a5F+j>b3jC}PzINTg5^0q@j#@$ye5U<@^SaW?0HW0(Y~iV
zC650spagE*bazt0L#Q))0KBYbnn+~1JowY41ab3_QQs_U`vO@3NEE@|+k1FgFUabG
z@4)i{+|~>Q&Nc!qDQpqqGY2^<qZ;qU17e1U)>Ci8S=4&^YI=DLp=ufc8%s;vg@E0I
z9Zp{CWd_JQ1cCfYblZu^c+>Mu>8@fGI>7~<PlUw)5Xug3et85mFK3<_SY%lh4FsFQ
z+Pfokh9Z`z$)wxq@$M|D5+Ld)HKcfA=iyg~@c;i~-Gqj*?UdUPMQQyz5=4rHh~!=#
zhXG9-<9y5XRYWHEgB&avuJB-+y(mrLht14#OLZwiW>gV__`y%@86SAzA9f3&;1WfF
zLt=o~#ZlgS&_IieG*^m)R-haPJOaU^Dm4L#6;K-F-Gbi{nLjYR51Py75?o|!>*}&E
zvvk8gj^hGdD~4!*ky=`8G#(;z$ty>=Ohd?nkz{TNoIe~wVWjQ?sDb-_&<eV44|3O2
zglahzFvP|kd^Q0M7~I=SFv!W&?76Al6vUn;!IO2OKUeNSTD%sBpr}+pdK<s4TQ)%_
z2|7mh<(ZPNIvJ^;a22cpmeF@tL7Ff?4+Hd*B=iLc4D)<I09FEsqvmd_K_ZJ;f;%6i
z4@w|toTxllyxR<u91C9!bSxh<DLg14?#0Ez<vkDj4d$}VElB*BAhoDJt0X}=2<Yfn
z557n9JAFrLZxA~STQZ<esl7sCx>1<H4xalnoYw1OcD{k0g1G+yEsFyyBR_hNLV?X^
z-^?+E=yIfoTMzr{4zj7|OPsFpLr8WH)^r7ngM;IX<<X)s1&DFNCnj{E%K=QE;us}(
z>(;G%7g}yXxJyDP#H|%0D_lT|P?MJaKpKCHXwZ=00168q-@4k`BTFF>Ft13;uD9_W
zzZ_SJ2HF(VT5+U3&#XFwg4b~^Njm!86XUjMy|J&ip^`rzTdPjQ4y;>-MKC@BR4kD;
z(May&Ge8a`24}1cGRE^ua&jto6L3Qe!Th0r?aw@Bii$226Vl%8gRJc@4CIutwEai$
zt*hq+HMM4N7Es>Qao=wU*E<`aK$;7gB}GtN*O|FOlZ*4*RURw~;;tEdM63^cbBN}H
zWG&Wdz!nCYDSS5s_Cp*Ojw#Mr4GjoWBPa$sTwyaf6JPJ;dZOubGWiSn1JJS>P!tHO
zeg(K0lo=$<fDx*ZzC{-6>$q5$E_g@=gXlhKT-Usit}e;&TY6?6w&uHqp?+_{PVfx!
zvd5Qsq1^*22SSJz0udR@8GN>dZ^FPF$JMNKRYMH5rRX&bIo$Z$%m5Lb;&`+Wxco}I
z_u2@>b_z-AbSjvhHd)tIR&@=e7KB9beVEB&99j3ka!m8-)29^_I8;b*WAg%Ghy*;4
zXw;GlRMWQaB_c-_`}LhX<MwL>fervHNiSb!0<x3a@}A(y-n6kTe(d*qui<DQHb*nn
zdmqbzWDnv@yUidp!^F_>y!tJz?KpWpLq6G83-<~$oaYy3Wi*#v3$DX~FLb15@)J)Y
zY`eNb2#(Rg;4~!j^&6xG`k*u}esZ+$Mogkb4=gVj+|YIhmjyOIOVV}edA<mSXMx15
zN15gJ3+#_T%^n5*$hj2t38{mHQg+2u9W+M$?sr}Nxf@#bZ)W(KdEy}60Z`1B`!Qgi
zA}Rq}=ur)`fgE!_q6=WCV+dMwvvn2$^F|8uAn5pr1O_NUFaoC<0?L+8sV~*O8{*C9
zbMeAcvtwqP^Z`pb1OkK?z)4qmGpnC%PcHsEfjR5pms1bWEj>ZTRFXPEOb&xel(PW!
zBd``1ps5NP-HgL=9W`}9Zh({u3gU!#62B^@2W9B&0_i1BR6l)S2ODEb;<(YwYy?=|
zW^JOTP_yIy*#UZR8B!nLA%EpCM?{5jrrL8sKP(8}0pl;qxL5~0gmTFF31?ZM*Cq2p
z$_PNW!f-DHxTqJt)LR^o|2ujU=9u||#9^M$yG&Zy1bZpe{ofI7J@k3dKm=UGZ<|V>
zDfvOk4>eWD2+bq=(^6&5MwnQ@nxV=s=rDK+y~l<bb~9mek2yS+7s~_xr2(6Puw_4w
zPa!RTJr3cv_7rr1DgPoBTX?|)5*)Xy7v0XXPi#+QJP%S(-~<da1W5@Pe)W<22owYt
z`EaU_jd0;_A$GHGFI>kIB<s=<{|~cwL(pCGj&jy5P-=1A{{m99d6Q&1m@k%B;EA@#
z=F_HuA+gXBuq<6OcxKQew~$I$|2HLg8kl@4-KzfSGZ5-MP<lJ*v`)S+L#Im#Sj6E+
zZy=eM9WH_-fM76q5MPSq+#tJwoV)KUNtJ6#U4#^b5FRXbKgFo#MHbvqald~RiE_B}
zvBHneSt2AY6aWfLV{fk68KeMZ;_^jm7|BN<zEOJeM9|RzQHM0h{j0QT`uGIVUS8$3
zN`f(y0_a6=2jsx#;sZ-81mDJhP*(g~UBcR{iG*j5v&@uaWQ+tJ<H>BTa%H>J_a}lW
z5*5%p@dy7+H@&COH#j?~p91lTabF{j(EmjK5M~xdk5UvFYe8vE8WWXm)QEWhy>cNy
z6-JPNBZGt;e(vnXN+9E4JQMMb00rOl;xXyC0nz+D|629Ao^&}HQ$+IJ65ibTOt=(K
z#DB-C90?(c6XQFl5bZ^)DxpT%W=y7DApkT#(KDfU@RdRvSY;!ENS*zMt%)KQ?fzZf
ze^+i7|7DK_Fpr!tMY-Eh8$wkIq-e93Hi6LiQ6&tJ46_RO*SV6CplTAUU_``h2!3;x
z(hu}n-DUPjE1muFNbk99Bm?|S(v>R7s$TdJ7zXXd-`TI?tKa<-N{g|;g(XW&uxgdP
zuv5$a9R@AACG%+%bU;7a>p?jLYIdZk3Q`EDo@j?{P|dW0!m54lUy>d=vp^j={QNB)
z15$&syB+#_wJ$z@-=K*N6>p7DHz#yS*HxwCA1Q&->$QUbJ7^sKQ9E>R+=q@|T{-yr
zor=}K;WR8BixJ}N^v`(Z;KSdaa3-zJZZ8k@N_Upr#z#(ZEG~3!^s<EYlVX%{`~SO6
zcP|1HA7DY4eyXw%yYl(-=bkJzu1y!df`)9zMi8cTA`E2~dp3hE9%j7Sm(%rN8bNKq
zyOs{Qi{n*QP~U1fqSy7O#Y4JS*Yc|2Y1(KW11O2CIz3*cMt_(+vo2!ss%mh2ZL8~u
zzOAYe3X9#0Yj#D_utmM&n5<wOGhkn8Cln;KSf-SNFi{wRKR|c8BAR7T)qrKgq+$t|
zWKATm*~{?kDAf32lN!hG6w1STv_(ydNKtiFFyrr60?z>YhLw_W5vkGn(^tp{t=N{K
zejjTFT^!^@mbM0VD`S01>dn|jy0;nzw&P0ywtsr71w8$0qZ(c0mVsfZOfGMR(zHUF
zbj*}<bmy77gz+!cG^d?oPSq0fEP549YBn9>q>N1rc<<l+rS&%a7{??*%;fL=_(`+l
z4SCMcFmFRVQV}6+CUMfYVR0mwlQNere~<ngeG^+Azp`*RoH(*vkM`)??swPHDl!Sq
zO3|YWE=Y}H)ljZjgQF(~Bp6E5<+&a)hSxhhq0)e8q9AWi1Kg1^`vBlwWFgMobjUCX
z$pL1*4>GROZoV{1d3pG>5Xl7IrH+n9Y7~(Bczp&|I>Z>5$WPt}1kk>V>hksV?ExCk
z$om`6Vxe7FWIM(7=cuu<FqttX*(TntV-VxActKXv3xy1(!#=3^EHqH+kgt|O3l+VA
zzHaw4nid0#kiI&h5pWJ|#~{!K{5btLl+K@B2f1N>=dE`@S<-4d9+d<Mf^*cWSabY-
z$6<MpWE?~62?lofO^iv|1@sdTI0|A0nb!>IQL?IwK;-_yGwvGZ4D6LZ^O24Z16+uf
zcoTP|$|DgRh8#QqMd#aSF$~rVXV-dn8yf(v6!h+<K=^;Is(NJxDxy;pASZecV2(`j
zUQ#@5(q&9sl)63hwPZd_*-KS+3kugb^sxcZhQS6)fY=QYOUQ!<w6lCXjDrQ2jKp2r
z7zA)9anuRPsA?Gw8`eLMhRICgxHjSCYKe=77g9QaN6Pq-Hx~|F8j!nu8MnXQ#jSV5
zB1Y>U-NnF=rLC6Ig`vZDuIf9PzJ}ViNY-==SH~?MEaVcj6NwmdRxFyOAED{sUDD_Y
z$0RPUCkC#kXBsvF7+PNfjFo_codm$72MXviP<Q=>!jn0?(cTk)@s{uB0a<o;AO>^H
z70^|@bBJ^O;<~><S`~CIYjZ=D{A|x*Bk~%b0Qjq)wg8HgBrz}<xib7Dad?69wihZ_
zBK3c&JOr8K65=4D(@B{^k3p_=Yd1Hs)dN_y4zysOwjvgI{ZvpdbwfgY+hU}&FHcX{
zUq=Nx)MPXQjlJNeVJ-($V}oFnx-@hqhFw8<IbB3wJi8PIRtJ3^v|5s^@oMjkdz{LE
z80MoP#SNm!Dvu<1;IG0MwegSKygt=iVcp<&gO>MDEB)26pQQZ7px^5Q(tMr5f6}qb
z^~{DvJR7@#*lXb&g06b?|FJPzc$#H)5~ONYPXoZ+%fre2s#Wf=;Q<u9z4?RT^H@Rj
z7?U0HqdqN*;z$hy3O<b21l{>z^MWpt-x7irhjgH4$tn#O2d7_>!9HkV^L{4sz|>>+
z1}HMLt;&ysErfym+z^`p=w1j@>DOzgQ)HnoN+D-2Yh2?YCJ(>$7Q?93S{s~h=7_6?
zk1-9^1EDYv!2xnrCRjBv%`37O+Ht_3yyofhpcft{&w)1W8!YnH2ssM#UrTGpLp&Vh
ze;_(Ht%o<;Y|Mo5I^G~+hnWMPN~qh<VQ%9Jl`7`SJ!fb?)2UAlz?jANysR#uofpY7
zkmLK;2<928uWo|RU~4smb8^Ft=U4nc&mly6fp}sN6&E4E$oLkH=98n6LeQ+!PeH!O
za)DTg?1|7NFy_8_oW7^FSVv;P1}q<LJPd`2&$l?123IOzok?3|H)oTb5R=m7!_?jg
zW0In9hPDqz_pd9LtIxqfM=mAAa8@Y8bv%mqE$EG8(^9W&h2kv6q#O653Ad;Wa*<`q
zK1nhxXo;cZu}w%Vvzo-Z%&H#lyfyy>>^==V(W@Rw?sLf3^)XZr{Q#m-3`~{zM8WHy
z;{<m=Y>x$^=5!D$_JDCfx2kaXBYQKm(X#vY%Yz{h{x5IiOc27oPQt?B`(g75(Jo>G
zcP0eRG!5GLiA@5swSShe_=)Ncd@X38LT-J+6xS*=5$V@WkpPF1I7=FL?q9Q}CC0<L
zAr(nQ4qV5xIO6vP_M7j&Al#KrfcVJ8bZ-n?5g{-IY%42VY<l4d@VKG=rl2h;1W+s)
z%7|x;p|X?L3T(BZg^%q&EA>UlvvC^g81fK~Ucy@f<V(re{&lLD#Ezt1P<W1EffqB7
z#tXPaLZT;2^~f#>>aov(s7$bX81x1Mm){HHZ8<YWtse4uVA$W4^STUDJ+yiI-MC%y
z266nUD&0~~*!(0f;%r0&F$t5jVC??7^DOP~Eo2+de!Q3=Q8=fbukYiCR4YOt^%Vor
z3q#NfO#B_HR>6?Nz?6Zg)LUrKomEXXF_Z!W$GvaL#)+%sXnqE2?>$Nj&@|#JymB!T
z{`uO8Fd>*HV(2<M_@$G7GqBS(kL=7kV3nc!Hi%l_j`bRE`60OYug^a}rJn#$8%SJB
zZ#80VB7j`0K&Sf~1}RLa4X)3f_MD3=TKm8=_HoZXTJ$2V-N-fMyXc?LLXG(a=;33w
zgX@iWFaP%fpYYZe5pjz@<rwJPZbK5*2c;!{AC)}anwRd^EO4-#Nj<I7O{%^qbxV7m
zf8=mDyJrt<5D5`s&`=jBt)|%D{kuM9#0U@PDAWlm%*rcY<?r@k{pT`>ws*>>k&qa*
zIu^0KS`i2($FlEP6mQ|)T**AG=3?=8ygi9Qv~kl7M0(uN_43Gi3LA-8^^Yi49E)+!
zpM8}OD;83kl%UYQ8Ni!twtdO$KSn2w_^u7AyWHI2K{SH0pqfdC;+Q<BKA5z5Z-Faz
zgvXd`K9?vYgo_j`xh+O`KXDpe)PlD(cxO~XAIXb!S*nx=jeKXv<|3IRjDiH}#1-(|
z9iLbJ-w<zp_G3k{2RqQjslK`D0Ere8g*KEqZ$Z7z6B`aYv8R>a00#C5uGP(_+Jxm$
z-5{Cg#a95;PKM+9l!b}jKYJ@AiUqfp40_{kpx81%uy;q9#Qs0Qlz8n4g7ZVSf~lAX
zUp<BzDJf1a%qs}LqjZNxMVAt{{st0EWp$EM-EpmkHyQ#c19Tg$DIsj%wipKYo(kOF
z;g5^u|D2u|3|+Z8f(Ia~nuiG}_coN4ZuI@fXD4`90}1uFTuodY7b5-#XhIQzN+~Hx
zE;OjgS>wcEVI7cw0q7)t5tWnh><tCVSS_+X;i0YX#%#M3L<i=Ceu+5Vb;Yss>Hn<J
z*QcIQO~Ap#Lsv|uWW;l#e{mKDW&q+<Xq#E!mohh8;#8hkbK+BMsCEfsdPP44mms}h
zTG%T(0m%(Fr0sa`oI!^3?fGXc>!UL8AGAT<0i+KQ=TohQ?S8UvatDw<OTpyC@Be#1
zor!h3k&sS5MaA#<Tv;5!JD`Ubht{9&<dB!=|Lc0h*|A_KN90T%LdMxmgD<71LFOep
z-zU#TB8}G-ab0No*lK1@M9>p5@Z|!jP}qMQw+9mXBOqEYz7wW~ante@YXFNxt*7d{
zX)x5UFd)b@WR%+WS~~~nY-b|7Hx&-+9J?(tU#KR&Hz@fZQ<{I0B;<t^4u!8_U^=KT
zJ-vJObePyrgR-oJSmi~j@J1?JBPw<FSlt9Y?rY(d3*F{S`5Z*|=JqSEkw*_G&3oPn
z&ABSkk6yz?g*@}cj}!6K`kD?f7>EJDrND<PfEFgc54-Q56F-56jSw)chByRQXv4VV
zc&YsUb+G>69baPk|53fUwwCB;w^puIXuI&pjH7~xNjz{66{$D%R0E<z!Fev~DMqZr
z-PNdNZ=j4F6pG|kh^+`PuPg^G+<8F`F^vREnZQBxTacNj5HD8Vxmvm!Lc{>xee}G{
zhe3@Zp>KHdF9^e})(w6~7<7h{9O18a)=)Ln#LwSgEMwe1z$}74vLeL1>j0Q;Zj0UQ
z3^lcoc-G`-MkPH3fBc>-Pe1SO7~>1}3uK;D#Bca$WB@$miB!L*A~vt3x?nEl<{JPD
zs(%mj?wew>(tA6ZzZt#~0fJe?w!3nuWD<BVB!=PO>6W$52zMlatD1j~c4VP@7tF-d
z^BkV))k39LHwhg2ow2vUB%a565*k!32FeQ_;#L#p1>aAA^PYtB&kYJ3P-i3eOM7;y
z_O~@T>cL5m;U>S4b?Ggu#B(f~&|nT??&SXpgFD)y)ombzLnStzd>KAN$f#@7rJNme
zZ%}E81UZ>P_E?VskH2v}J6>kM;H!T8+mRIRh#EU<5R(|aHSbagj<wngJBwiW+kLVN
z<z@oj2MA!474pm1vH$fo(G6P{XRz^_9Qc#2H^{Qr##6fmE5m1aKYB(y(I_&m)m6zB
zdrkw7-H$g93)69ByC4|W{ZMsU0`AC}I8Pr3LQC>$_eJ;&4rw0e%go=>SC+qR<1T{B
zO?re$^~=5gYO?zdi6@mV%`1L9xN9OT<z)<9l=GH=Iws(FAuQz!=KY`W-e?t3KXHS{
ze!-knH<<g#ps({W7{;=}2WHs%`6sYnkn;priXqm2L%g~rDO;UOFYkXh>wh;3j{g6i
ztp9%}%To|*PVVH1U&(4RSJlxol;x=?yq__-ZQn+IazHzUd_nQBoZ#>c;w^#Q5Rdn1
z1Yquoc!&FD|F4dSn1FAhcGYBIr~f+jhd?qOHYS<)I0Yl=x%~0tx|JoK1#dbltI5p*
zfyBn+yw|b2vXOVrr#44++aI<3z_&ZDHQJBkb>fsv<O*1d{$V{kV6ZhY<|Y|-pMgm#
zzINn8qTG{-zjA&FQ&}Ez{$!o=Ro;__tK(ey9lvvomQ2w}92;n-k^{khozK4+#&^d)
zbUwLx=0qv~%GOOo5YuNnbG?P$;5mL_Y-4)ql2uE)<hjt=bPnh(2Wtjc7k6NL>)}YR
za>Y61j?9z2A{Bmd_jwMs(o$b^&s@u*b;pVQA5(>aV9mqFdk%L}TSJDnV|dzJs#Hrj
z?;ob5FAergD!x1}6y12}H2i{lt6YtoPxi)u_1Kjc`Ak|T({g5yQg9+Nbo!^SJl<no
zo9|{=+|ay#%Dl%*$|w8!+U919mD|Mhi)>W>#J*E9D&@n&_kYu$=0=jPA1sH2+0Axw
zd>hJsIOF`;^EW}<An#e+W8UFjBmj4N{_NQ<5&70wboiDRR7;ohU79mDNE>xd)Jq$E
zRIaO)Q8!4#;^3I$j67rXlzMH7|LrB)&WqvYC97Q#wv-DyMU4rL>!kJRC}R)VA^aKp
zCz}*#?N>@si#z74k2<c;{$wCU?^P5V;B?_f=uF9|cv;OoCCRu%fxe*pk{bOfQ;*V!
zX4k0aMYBMVVAxHjpXF9rjg`@Rdizc6rYkA?^;P(ZuVaMqNSWXJU%4@0e0Kkh_FQ+6
z`^P`Ke{-U=8g#&U<6$>>c^4Yv;p|-^_@@AWHUgqu&0jy0Oi|is2O-(Iy05!O-E6c+
zyML>H#nV?FyUnrtuiQ6eaQW2`X<Ok_&UicFb1+ZmO$y((jlx>36HfN$1E#0<*~1cv
z(fn2-`@-UfUR$k^qc2LInl2J^^0w^uu3wl|uVT+%k?eDgP<J0wX}oDRbo1;US#{2H
zUxavycNR~J?nYnWGagoQxt`UJ!?p6ebX_Uhw4{SAT4nLx4o(<NLN<k$?l9(9ZM=l>
zXt+<|@@B?{b@c$*Z3Fc3ck_#=s|oGTqGS`3adv5*2k5zdHC!2JlS-Voj%rw2JDl#a
zhPnuaTb}Wq)b_|``;}f?6s2r@YTmeex}aRb=eH81SEcL=o~Q8--F7OgW+HanS|gzv
z+9x}9I^rHJKjc|g80SyDD0YQt?e9t5R&b3UK7Qh?O>21xy30$3Z=okf&<B&f(~yLV
zn2EEND)(2Z_4>EWYcZR%Ng|z$OvN0$rgcA(nHO?nOK#go64cYHjO?vz&TU+16y&d*
zqb%E*V=A+s`BZ;PI#z&d=02|zyZ-}rL+ib^+{WAPU$Ao=UFPKCLJmjDoa1}0S`#OF
zq#Bl*#0+f8<%}mbc0M1>A77}=5PZj+G!x$xJNi{va#zvKBG`PJnrdjBEFwcC;uGP@
z<mL?>KAMwFBB}%T+<Mo)<LBH@OKFC}x^*7zOg|Z*qbQfLR5nIg^|F<@?o?FV{SiR<
zWuzf7%9j0FmB|3RCWkvt!Bumn1EWo%9-{Ltm(%So=(r~mt`0J<GXyBPJ*<6TJHF=9
zmJoe4@Nnj?lE}fJ((T+2c?RiP)TOSsG-^hk=`f}Eo${yXl%O9(U#}wa7>yX8BrUr0
zJZmt&0!>^Rd<J5?mWW2KA~`2L0?vZge4p@s4Xp`^*`ck#z;rb<B-p`F7GBUWNYh$C
zuWLL6G*tDW&g+Vp5EEbZ+tb$BQSu(WB-htG7PObMUnm#LOa?XZ1bLLq#jf5`$@lJi
zg~r|NVZMlB=lQ1Rx^OBl^8UT~qI~>gbFG1Aj3fE;_jccK_*&)vv9J6=a@XPY9ScU1
zuVY0q@2<|C%<nlZNMCX5w(z987{6Ug6<~c?V<36>r@G5BkKopAwP5r}sgZk1_n}Hx
zx7MMlFNw}apKI{e)q@+`CuZUM9O@p=ei25GS~9Mve*eAgD%<q2T!ikEZUqiY&W|X$
zs?!z*r>2T01`CIx<2}iKM5A(H=CV&WetWo(l=POzJ>I*1DRV&SZI||iH@8Q9drTQJ
zGhOsTN*mks9-qGrm9JeI(MAUvt16M=huG2?k1l7Z&)uBFR;CD2^PKs<HvRmO+uAJw
zol0`0g|6p2>w@c~H?3G1P%B5L7aFVZSGK(J3y-`TKbJnM4gNH$@jSJ7TVN-aW~}4X
za^xAsU9lAnSr=vVv>)fhq;6_#R?M|VrO18`Hjr???|8%fc=7s_(E=YuuKiENBbJ>n
zM~ijqq?Dn#Gj8pd=;F!D9HVj@JGdjU*^@k|`6<Hod8_I=e{Rv3@i-*;4P-f)zf7;8
z=ZsAp5`OT3Ze(MyI^V0kmi}n)&3Myso152(DT3K4=e28&k({ZT2=Qr&8t2VR-Xk7E
zKnJI-5Ipkc2U>&Tt6VFxv8KCdVb=?oO1NKN;#f^x)ZS=GZETwiPdC&Xdf>y?anLd*
zVW7^htv={s^mTGOQn<sZM`0*5SB1FSa%TQ&I5!h}QT+7A*Z1P3hZTB6`s@#Ow)9DE
z9d0^_zSXtNWZ>KNr+l7Vqvh{YGk<4UrrsZ4_3PxIiI0v;2r09iB=gRR**mqyzZ^1t
zl?INll<;<&_G#Rn?4n0f1u}VhkE>QCjC<5pgFl+@wCL--BeN!VO~tlP;AI@y?#^f6
ztR3AEeKlV?N#dYCo?AbLiVN3UEzxM`4f>hYW85iXLX`N??1R8wk>TYBOHB^b8zjAq
z0ryP4$%M1u9UH$hT^4Bc`PsckrDw^ztA7zy5geS~71Jw@U8C*n8hBotEtlIR{u;qN
z%OCV9K~B#nwi9<*{vB1J4rN-NWAm+{AIa$=ZPR!Q#Rdnx_)6LiRe_Il`a?R79a;<$
zcX2H0=E^BuRw=se2V=ETS4%s5v{_}=$(U2d-S^ZgPE_ml^UK8>I45)Bx5o^;Tn09_
zr*1dd$Nt__%)hN*C@!3D(2#alkMf;|m^y)nN)C??-^s^2ijVGCIP#&jem1NP5`;}_
zb}CrD2;U76ptdFN5Obpl3^AA${6pz3WlLcF5#MFVm}^RTtitt!yZYfz{y*cKlQLY9
z*T(XhMXY9>`(3KenSc5rErQZ6s=N9%hQTs;&)R>VSHoR>pAdJlBQmE(7(9y*EkBr$
z!(KTyD)`KVPTiRA^@5A>824?|h&DydN|&Pqr?_yxJ{7vp=gEBY8TW!{IzdQa!?)lM
z%efVXp#ir?))l&gKM0-BD80Iy@G_S>TlCU}O4d1kiPGt}<l8t7X_zzPq}`s=@AIvy
z8xpM=ztmK#&@k(K?N2_hv@)pL!E*{bM8WmV?lohpT>G*7v1u_=e)kEQ%a_FEn$2XY
zcfrAjB$`M|V!Hg*FV^ABs_KiGTdciZs*jwtDnj<k)P2R2XLc?z)v*O=Rn+Irm;{Sn
z(-{ym(lLC?xB1W(=fme!$}ETe63@wyepgjX;TPqs$L;iEU)@^UdS~X2%Xb5fdUr+h
zYz6a-&;#73$?M-U_tSrh>e>jR*+$mg_~foVbs}IFWMe72lsm3fF&TY$C{<-u)z`G=
z9QQbTb1(kYksgn8hMs-!^ijztA3>cl%K*W65tY+b*4?FIfqE$iMHAUo^_*57Nz-?R
zmX&icjy}5{-Nl0I_gS9pMBg8oncN2$`l;vGVEy?t)5x*%+ZKWSYpXp)y7o?fgVP_P
zsy1v}s+2V9<tU!C@?^?v<Jg6^N>8#G{VMEp8M~$LWoh4}sZaEX!lPX7^u^odH@`@S
z@u)8NGc&q~eDX2a4>J2LCW2nn`Xqm8+M^(^^ilDQ%Z5wS)ue}?19NL*Qua}aORcN-
z@ITA=*-?HDZM;Z9?)N%wVX4P+rC@`KxK^vW>>LkL(}S875E2ZP$;V<!T!S$f#=|sK
zIMJq!(&fAFs#|04^}1^t55zy{MUO`95nz2I#!aKD&R=RP)m~nymm(^0E0V1I$d!1(
zw|YmicR=f_^kh(&0>StXq5AaHy7YItbypv%xpFkHd8io;31>{dtucI$d80L;V8CXb
zc_bgFzcCjTZtIFCWJCMuR1dSjB>&MAUE@^e^`Czn`6p**GW$*=N<@e!XDTa}2UIwG
z_#P|gNU)mm&gTynI&5H(n2r2v7cbuOxm%Roh!MRv?LLzl!qWHMliRL0EBzY<zW!=e
zpU7K|Rikd#bEwH2#d*K)n*GPWZkUAlWEo~Cek{MzFc<ywvL))%UPyiI^h#4?Q+?@S
z>mJpDIg8iqRA#5lGgp3kx;AovvESma1w^<0F?Ja}|3I<SKri&XB-?nqx&_O6{KTYI
z<ND^a`%A1(H=+|&TAYrET-+^U$`$za4_On5*$AmzywI9uvQsZ?DkOCpj<jYBh6v;a
z#&kZA2e*>Bc#wn$zVb3z_{o$|`z`lSWbX0ZaZI`qQ4^vGZD3B2;)n8u74ea=NX0j%
zd50GzKfr-5A3fUeig9QwjI{*#8_(Bku|wzX?P-SXZHlvJ5YndFqBgyfwq-N@i8?*H
zHvUK8+VB4BeN~!d5(;{3SL4^wRtI_w&(_EE)@xZ<*0K|e-(8I4iuApfpyWr8o4W41
zOB#i#BNsYxNb|ysNeUyDaN&v3;^}pwz@M&TcmIfE&Dbt;YCme$b{<OTG#Gn4>|w*}
z`T0JLi{^Tdu(tQS*0ZX9vB<YJHiRBu{BDor?@lpm%t=1IR7u2MB#^)QvCwNJUnu<R
zMbt{;r7!dm)a?6)-HUz^v4$CW3v>&)9dY$j#p@RQoF@kC+(EbJ&=IEEb8I7*wE5Z@
z<na;<9C}<O=o_TI`K_qxYTp!evMG|wIH*ssSawqWEJ}4pJGnxqQ!F@O^s&gZQofyO
z%ysSQRzb~Oru9(kRn9_V9Foerg;`6z2Em=+swo&B-sP(5)g0tQZ?QK9mWG_yT;F!Z
zL&eTzC7l<3%Kw&6dy4;yb*EMBsL3x<+nod>_Re7ejb_`CQx4`Ou9<d;V-;yD!N0=}
zlxS5$9x{`WQoD$q>>*=Enlzh+8aplWiA!fYdcIWA$^~I6*TjI5T3OYu8K#kbL&ddn
z;sXCfo9?>v-Rng!$2hE>)SD1L_Yu0cf<ZB1L*8NO7;!9P{7Wim`nR#22cF$(dlNZ&
z#6VqUX*+{4Lsu@YGT>Ke%-`z7MlG+XfZji0e%Jg7sJn3@)Fmrev~Q|bc`&cf>*q3A
zP6jR2P%HC){$-G_Hh35yn$lx`M`DEE*2C`TXgzjyUMksKlb2BE{Z>ZZsDtyWUg6=A
zO!z|&w#5<_#u4i=y^AQL>o1aT(%A1h$tF2G6D^&d=@R~6)N{gFTQ(3=@Xh86sqB|(
zsTXQ5HrbTlmaiILPcRZ@7H1KGE3bCCahp`X44#8TB}B5~<XXgosY>e$Bg_#^xF}<{
zy@cid*?6_6-C)b!@ummmej7hk!^_<E_l_rz(GBG2J0-6scz*ahxx;x!X+-)w48ok(
z%F(JAhfW0y46I-YktZ*t{XQ(0vQP(cM+(_`<F5ith`VJt!4p<NgfmkRvFb^T_U=(~
zyoHoqh}D3Lou-g}GfIQsRv5;i@g6x$E+@XaY4L^+V--z?1zgCWv_pkeQ+RGfVD+Kx
zokr~G*HINIZ05z%leGJ!XkNu{F)3eGr>se7%>NuIxeK7d1zMh%&+|_)UcMo`TBR}D
zvvy&W>8dyTWKd_-HHVPm>qAm%dDfj9EqIvP9_gu$m)D)T<wwZuKa7;Wm^yYEA0Sr>
zO_P+ZEAHHV%#sKY-$#Pt)(VR}33?;)4!!kpLRw{<o!;Dcdz(8hpYu^uT)u0R%oaC>
zjq6(t^6wMZQVD*HLfLMd@91#eUv^Tle@LxS_e86<q0%;f?eqhzvy>oL#j|v_i*x<6
z?s7Z#MoNtDiZa^7)KW1gO^A?_T;8Oz)wy$6NR1lk(0LGP%kyD=hr03EvfAPXuDzh{
zMO4GLN|rB`hxxkZ8VK-UQ~l{P2P_XrF%V`p>3pv)gpbn_%>OTQMok8Z5^2vijw!Ho
z?MCmbl{%_czU7mp*S(oBF_T@v7+xU%xOqBgU#H5kAtJ>)s^5Z(zgYIzwLZNu9_5j5
zLd~2qv8PpG7%UYlfBaRo$Yv~E{h-3**x^&(48xDebE~!(?yLImxjRqNX}9Axc@4hq
zJ}kpW%_>M)>@NLMXlpUQ?pVQN{9r-|4~4E6`V&0vpS@Wbxt!~o6&q4Pr?c<{o$mBn
z>{+R8XGaU(Bi@}v)YZ`=t>$~JYDRy8Y5uyYKK{crvc77(N^03Ns(~UPs+W9GX2F!W
z&4HSIK>8*R7p-MSf<D2)z&JgPznbs#pD`87@#ibZm9GENDZ9={ei{EV_5$&_2n7*I
zVo~DfU3axedq&0ayi>T}dDGH;##O*lpic@9FBGlg>HIOiJQ`h&LOnn)FWb5c4*GN1
zcI+n(*^ax-3am=lh)I#Y<wg%&eo~5W`2>qm2&-F|_NGVbJImMlfyat`1d-WNzr`zS
zgC!4tFkG(7Uv+0Z{B0Q)ZmJ*=mHRm4#_sI8?&0RQ4;!WdW1)}S)UOt=;M<>6&nC`H
z`_g<{A)*Y5ytG*vLNm)2#(Dp)<nf6))vhYHW2h^ilNbMqo%O!1zK*@?{@&1c!@t}T
zVH`gR_S_>0a(X3Ysd-r}cT8E0bTXGqmB&jR7qeb#=4lqQYB76IKm1G+$tV#jUi83H
zsv*susu62c^}cDP>%%QB``F1ploH<xP5CVzrq8`p6Ew+4>HH*d!8~rsni}UZi-yX{
z4=Xk8rGrZT-QStMPEYNvG<z++?;d6=%<uQ~yw;RUd>_+)Y|!r)9Dau-JMGfO!l$e8
zzRhXuDj5|+o3oLJ_%l&~55G%(|J(c*XT_;>V?c0YIW|{3-h!rdWOXp%4d+(}eEY1q
zmS*bCk+ROJ(0w<-kS#UNmL!_FADWq~aRLqs4?1+%1X&MdI}&-5X(CgaLnTMi?Wvlh
zQ8yX`A3VFi$XIn!s@U>72Uft`JiNjBbvN<K)sxp=jB+tgACDE)TaP#yC}g^SH)wEf
z-k$i`W4@5yk$&S3>%|Jj63fv@-#2`c&1Ri^LD$XaJFg7O=c-6W%#_Cj7-=V3zqj>s
zqRBL<0(PQqvB*B-Pb$}3>QaH_wb9>egL#P*5{q^ToewKF_t2EtM}^#1hx*rst!N5?
z%d;?M=`6#}(#P@a0^NU8tyjK*$_m@6#QueTt?H{om`$Elt9yKjoHAzj(Nm8r2Qf>P
zX$;T<uSo|f4|!e(4-g8u@7gVFJzUt_D7V=jze{aYHn(kZI<l_Vll+vyUZOSQ9FxZa
zvj7{7YQc{i<EW-J%eaIgv_tjYu1gELD3<XRx;&+Gc)_`VDp7c;II>P~mB)~rfUjEq
zWACp-F*)(Xs4*pvS30j1G|cQA8yB*c(gYo<L$s<|h})G#y|Z0WC$2d>y*p(Qmb8?z
z$G6$<=d!cEIt^4PUB(C(*vKZdv06fVp7<_}cZ-*t3w>V7<$S{_5L?_6`>9+tq2O4S
zexC5wyhc%sW)-|0?2h(xbvYV(iO`$Y3nC0}s8(~FRh-1|Oo{){T4l!<H`O^lAVsq!
z{-mW5*o`z+9aW$oC;EhYA^Fzh$&MeKCWNPSf2#JiD-(-EP9CA;<IOm~1dwNM1lm66
z{Uhj+=^x(ilDTx46~m^sIWl9-u9f1Hr;2IHXB*w<bR)04h7j}0(|Za<#UE0p`0O<k
zVy&(%GrJ39CHQX_z}rvQPqP=v$Aekz6%IRT-cVspwprvq-F-);^Nr@~f!kJR$fZXu
zHLT?+YXL^73(@EUy{JyBtIK;9#q9D@IVYQm;W^q*BnPr46o_`?{UrH6i&*VWJ8%q+
z{%LR{&AW)1Ifyyo6?IXV`}53T!Z}~|jlg9L&E3w1s+1)z%cl10JVjCWe#|WAD?T2=
zEZvEsaS!EKt*<2S%Dw;noNl7f6>dql%m)#QenHL2a+f1061RC;rgk|3Kd-#+4}W#4
z71DG6PLsyD;U6Q@eV>zCENW|boJ&H}7hhOE%v~KECEWY_6aC#}l%hFWAYf}iyY|ta
zpc0F*aQEB_=DeY&>sXzCxVjfI24+?_qf+)-J0+6!;<wOids~f!rS99G3R%^K(Ib_t
z><@lp*97HwPkc5nHxfA}{1%v&TrM?v(MX`B&dX{D^A<h;w=KIEt~>c0wjHl|msP$8
zPzg<T<XV^*lhR_6;!<N?M9r#Ra~heHb?v2Y46__|+bo@q-6=J>pyAq!Zee)g;&A0q
zxWRY#13#8BDOzYXSjWA$;<LuLC^JWne&b)y{jQKDTKO&AEV+J5kX6f1Cc<^QwA}VE
zXzS~sc*m)=dR5=_`LS8{s)&a_e~J5`w8JQ`_1ihtOgGfe<|@A$8=A&``e#YtQ)}<`
zw?}BQ`Cn5ciiQnue|*Amvf1{Q-wpMp#m?O^a3Q+L-_9{joBhD0`qq$EsmA((rW(KN
zdo$;@w~rMjhDgn>(+{L5yBV#bH2U4WOeDqI(%m$P&reyGIJZ!4w3o&xFwM$1&3^b6
zO0OhoZ&^`V7ckMlkoL{Z@qL{lOXc5SD?gWcE;5A_Stglhc9G02QOYHiRvU6mLW%pw
zr6b4l=X%^ba3wNkI3y>@(f5SUv&+{o&7E^%GJa~g4Xts+j|XH=6$oFl-x#ME*9c7)
zw|JrDr@|tDJ_%;#xL{tFF!=;o@OQ*m4twj7R}4H{YpKQy8wK`Ym1C_Mc~-1D3E=<v
zxZ9CbzfPWWO&v4%c}6T_#9w<VN^6L%u+q_<;#zmj>`$x}j-`Pk)l##{B;me_bo4Z_
zdfw`?HI5#U-sO8tmv4h=tR<7(sU`eV)TsQ-@!RROTZ<>{jF+`B4{OA4UH6zHcEG|L
z+g&IyR@M`+(60K-kHPnnPHcnUrs~n^(RlSInx=@VQ?d$Nk*CK!hT`ePeC*dU)6Qqc
z6Q9h<H4$v>2VE9y@(D8upJRHPCyi=wHc?vb5_$iE#Y9!nvA0HwOpoIu+G+U-lRouO
zkq((}&g~Ff0^-`Kq8tuM!(K~6<<fHN?Xs=hlS+oyl0Amae{&93l}miAE?g(X8#5A6
zkq#lxFc1~6?^+S`@Zoce&_7t1jIa92S0hEPSRqHvduQ#QbkyxPq&5YkVXfJW*3v(J
zC*r+TmxyP}o?T(<u{FxtTX2`BrQ_Oi(fhU>{VP5~j3wx^1@YxhGsXAa)QoehagUOY
zaY9HZMYaxVr!OZ&h+A%%8u7Ym#uL<e7%yWukNA8xP{qdkh=F&7<Q~_{>!ZvZ!D{Wg
zb8srsy)1`u8fxB|1XvF5rwJs8F{Jpd+}rUZC$^-P$Iddw!JvK{Qkl!GC^$H&*7vHP
zyg8+TZM49m;HKfWmB0NGLAlDzO+t=TO)Az0CaZ&kaZSmmGWJ3CnK}U<DYX)_b1ROl
zvTCY#6uun<9GEgOnn#yN`}?oG?(w;5DD*CfRH?P~u6mY6y~rg8Z`%7B0j2?asg5-n
z%=Mf;U-wmi=pWEU$+>+?YkgvlW9{T;X6^k!|9#oqB|DF&(sDyZ#$j^#7A7c~)P__o
zW7WTZgda)zt4swqJ8N$C#Pv9Mej?2r$9QTqv#YJL?lu`uw7$8W!1{M#K<9TslmBtm
zD=f{xVV?K)0lc-uIf444Pp#C`odUAsE5(U>`yW0Lcag*&$Fk(tVzGa`T9Mu8`0VF%
zK^@YkKid{KBm-7ThGx{-4)@+D%q3rON_vS|_owjI+V|<Js16Ldl3<j_ifdDTHa@EJ
zcWxT`1KB$!0gbyf96moA%Irr({s`Tu&GnU$*DrCu(Mex%jwB$r3*Ts~E~wL=^{}ZU
z;`)ulj_d8r+X1^ns1C|+=3EXKKIPB1FRV%-SP+jGc9GetH7~LFOHBvQN`5frc;Ti@
z`Jrbu&q~N)+UQUAA97gAbRVxccq$EBq)|tY;jpu_JoEA?Y^z`9R787x@<i>-OI+S0
zOhB!vd#}}yi(v#<$MZXiI1T-1yztAL<Z6M*C4XK2+r5rAEi}t>_*d<6^GZMY7~|(t
z<%|E8$=2j2LYouHWO$8U=E!C@3ouC$kC3zxw6m<RW;l13X-g$`|C0D5+tzU{DMV`V
z>W`i=PyVbu^LM<*L9nxz*0OkIFHMV?&TJ6}xpS8hGP+ux75b39w9y|#Z+`p)g*h@X
z<n^Co87FP>yUNl?-%W1iwPB?c5csFHK!Z%?of2faUzz=@40#3zNrQWT=3R4P26ro{
zRzU&BYNLVQ6|{r&Ab{R#CXRLf4@m8w&sdCAs6_<qd>Mi18aO{$=C0?c7fB-BeYbWa
zi$5wrM|%o1&MAPzZ|*gwIkPVnB<xNc$k}1!#N?{NygkBXlb-KLO6xVxq-)$&e2w+m
z^2s*tc;R8=-MAn7X`{9?lU5tMsaY@Gd(rqb9-GK3><)HTUOB3PjxS=uz>%~1jezsk
z$FZLVDnA{nl%T0BB5BT@|H|PlGNbFM%Gv78O(FT2Y}ez7{$9EArbyoV-2=0}Tgi%;
zG}mDsB{UZ{tUZPfQKVn3&h~^lg=&JGV;0^$_|5^#*!`SBX~0gp?+W~1oEyvd!iq@d
zYko-If^i6l!Z7eY3h`rS-P@^~UYUuS*1vyM$sS%!r2vhR$%<*gHPhKJ^;A0-URm$n
zCD1+Lq@1rWnmf}0!w{fvXX$=$6*Kf9u(DTPp<HU4KY(#D63`zKC>!&**kMH-hIsE8
zI8XjUdiIcc8*IW=A00epy%WJ;87Hn|A|}i?KB)e9pv{7TDfPiX`QJ?TCO0{L{rv;=
zZ0~o61+Uu(%x~{<nPoU+-6CJy`XlEuv(}k^_kcGFv<55950kPtA6ongKrG1&`(>U+
zwwNjngrH2ayxKt`4!kd+CGeW8hscQb5*GKwOn4t!mgbM1C3~cE5M+DK?bE9g?|g_a
z{Y(XY?YG;hV;g%eIba;#3r%ARFtO*nqVJm+n8SG!Ms=?{OG(F$!kDiC{>nI5Vtl4q
zI!xGfUMz5^7g5$n^v+T``{@o*yD6@37D+}U>%a1Qdl%oOK2&~a{FLVQ+H3!hDhADs
zy@vbD1p+NK?;pok;&yDE502gROvR19TFBauk<H0HnEjg}iwN(dXFr~*-ZD(CE`YAm
zp}XHe>IwR6m5SN*U1YS%49JJI+2OT&snE50|Et%g?cXLkcmdF+x&`!|&VVV0a+@mT
z?jJpjxvRnkLhQf+DR{=TmIjkN1?Zdj3*V=0>F4m}=Svi=zk?9%mLn_J0ypVB1Dxlj
z1<hglVv~g0R#lecnb|jNm8m_-q79oU%CEtTYFIelmQNoNHs=bW?YMDT3u%V7GoP2*
z(&iugBKz?C|FI8plO3D4KZAa@K8m66oa^U$cmW9(Be7U@AfPn1ARQ6QA^B21)KU1>
z*DmpGDM)%+k9{`$vVuv4OKmONqJ#y*l)`2eOKousrro&S6!%+o>6WMm=RJn6QD)Zl
z$xl=?5=ab%;cU&IS>%_)wtA~*bk4I;N&@jRY1nOpnB?f!8~2`<;%7139y$LjSS6ML
z&glach54K>R+Nv5E4W~}1(4;Qn<7u>q@YY+Gn{HA8OE%9u1|f2Xc%;V5hhttHnP(f
z=ji;r>GInmvY=7+A_Yg2#+qJpCTq$>rWML=7xQ89oTujgp$_B#PkvNcxl7CnzYh3Q
zV)3f?5vnOeSWKxc<jCl~c!C%@IBIBoj=edi&O^Mc?JCo*@NP@dY0k@D?cHX79U`e&
z&7qkS<_)PG7h4z2;DvlK_p#g{os-Q9X$w9f^MR#N1segYTn3NGk%x?bMjui^Y#7MZ
zyv%ZBb@KP>{^1A{Wa#mRov|9!x7EZI(j->m`tw)VMg){|qIK^qeKvPtR2uHH$^EsT
zAV%{s<-)qy$>9>sL#K&%4s02JT{bv^qgkL_FW>9NW!TR6kfv$rR<B!KVB_eAR|A&^
zHa+4Zt2(s8$ukt2wRY!1{0b{&fDmvY2|6}#A-tHtBp{$#=*cMYNyI|)CtLY|MW|qK
z0`b<vw3Wy5m4yi(5<z}r+(l?}8LN`&(vBy|#s%0)`DfL43_M^oPYS$$!FLNfY1oB*
z2{`yE=WCErXVAQI$+8!w>mos5@`n)77wK^Ga2U+M1mkA7A6;JKxOd@Ia^DwG{Hrim
ztUJRq_|lboLLlMScT9xQFIV{<@UDlO;#{Gh{Rg8)d0yr7tktUE%|>B)|K1~VHq{S*
z^?aN(m!DnGqg<X=wV{Of{M!ONj-P)AWUM**yK4Iudc+%ybEQ(#?5EAw7RG~`_o5SQ
zAZAAK$JiJ~@EjCJdUr-b!$3T1z(7-B(3=@O8J5hoj#r!+N`gv)bAh>q7PlhCGvW2Q
zZ1InowGyQqO%MK)L1_k1N#8$q?z8*hbnM&su&+dRm*{WvMLDN9TMG2p@}Fmrf2xw9
z-EEtrs<y&<?4(f=@sG1x*WLaPdv6_8W%RX+3erd`AdPfOcPTAl&`3*nZn_bqLFo<w
z0qK-(0qO1rrMowtwZY%H-*@g9_ng1)IAiR;_IP8xYtA*-oX>pbn%gx!QxMU&-5;$r
zmM9ZExjjOPZXAo}Qbp4pFSqdGEm9#jPxfFa#_^SB)+Z}F_!O^Z$3za9enkbul(4rY
zatdGg^3u0!_8jUmP*d@=&Q#ztyl#rG_u|NyL`ET(Pd~u63<@>MrP*OUEJ8{f*xEHh
zDB?OZ>>LH9IE5+-`*U@7C(?l<TV9=RWorL^MCOhCyEKWh7u0y>GuD6>B$S>+rR6*S
z*@myrti~5WfrlKZt$Crj2WTzIb%0Q<x}McYUjvx;lq?WX2rTC#SqIbsw}=|OtW+^o
zK%6PwS#0R_krYf>Uff%c7i5?&KQZzTeOoHfbW|i6MQDJR68kXOI}Sus0-E~NbFCCt
zm993^RaesMPgE}g%VYi&TiVMX)3>i+$%F?%;tbe?w`(Jt;B3lUORAK!v^D$v3xVH7
zEm`nI?!nFRPgI(ls&v7(#|8CD%@UqlZK->iaY9CD3QJ6xTMZ85SoVK@xyE9xBv;g)
z?5{T)`O->JLLCm-eCa=)T#R<oc*cd96aN*~$V_=${Cc9S#AZjcLKctSow20J9bJf7
zgKNL3UGTi`P}l3TV3IGAb1HsB+<UAp*ZpNplV!|n*%<0*4@ojL9p|M&ceob~7W_^W
z<$P}qP$B<dq45OcH)Oukdp$JWT#>!mNzyb8>q~G`&J(shY8P*$wNEjE0}iBaTu-Ie
zJ*RD7BPY(96s<(*96ypzrfkGL!&bvSL`;0~vgKC>X;M-`GZTjM@s0}(*a#ePO97IU
zD%`C~jFUmCClS~UbJ{4p?l-%`<I1tbZ@b7n6D5<3%ilUppjBqYh$N@i=2IUu`0dhW
zcyagQBGM@Px(1Z}wE7qxuEAMp=A@liP<EZZXV*a0x#IVTu%6R6vzeph-|M6Lo{Nbq
z|3<Qnt4~SvH%qhSg=})tH&-JT$Z79Lpx|2Iq-t{F%h;9xC_3HWFRf0R41?k}ZBdDx
z4AvGlcz>_AbqS)ejM_$R$7WMY8J=+Ox2er`vAShqn*QZShxELojyGO)V=Tkor1VoT
zpI7tJsd{6y|HGxBk88ODcaP%dt;2Fmw|RX+#N6NUa~18rCRL4vol#9IV(8@NdM<pV
zT(F1fCVIHJEPyeYB6||a!=t*Z7~09@w@e%vA`yLo{~?IdiFAt3HRT`jnHMw^3x<aW
zm_l|&@{|NQ>Ja7u&f|(`;A^$~MNKGsNIxgpvieZ`t3tHe_{ktg6{9TF%Sy-`sz3cB
zDcE7!wtzV|%>d236SDoYaF?k5eJI1%d0kWb_(7d0h{ztJB%|;ypZDRI#%2DEmO1$g
z)yz$8!&LEuKLGJcS{eF#o!A?y)T~ahw^TP_((f3wl$So47iiE75V;ucbL}gJ?!2eV
z19oMtQv2}>v7F81YV5NMfTZ{e(`32W`h4T)7qN==097^lOh~}D=J8oxyA&m6Q052c
zAHHl;{m$+d+o1%Sf`i0wfc2-1^ejZxRVpbgI6Sc2t`*JKR_$heGiSI|y_Itz`5?dy
zSP!ewz`)-xh;}I9`(|Svpiag40LIg-GbZr{FWd~A7u6~vY(*>HuV{EOqF@E=)Tktj
z1toR9R$-VK#{bm9PmRKdZ9g|^Vs+JbX49m0xPc8(R0>WGVyHe4BW_xK)h@~G_~tpx
z&&p5;9RJKZ*ZSGwWWnf4zk9Pp)SS#V+yF7Nnlq@*1$!&ek8+N3kDQPq0=1(pm5Ep8
zmSe_olA(B1J2g=7y!!X4Z_B4Z+7MMQ&7`oM0&>++S3Ih&)NJ^_i!E)q-XKnQ9he_6
z*y{frjvm)HKjTNF-9)mPHeObO4oMi9gUmk?ClR9#E^oHJ(qLgelKIvaaWjX)3dia1
zY-@piF>hLmJP-we;UuhgI#Sp@rD=m;itLiz?f@o&dP(+fCI&uduSZzwHtC31kP2!a
zW|xSn7=*aftm5sRc{Qtaaqs4q6MR!w)S~1Lzy4}}v8g+TR*IO;M6(~czs}dZwrW;>
z^Lg65+0kiL60;PZVP!;RG&fCv%{Ds9c5|l{3FnbPf;$J*oc`7CG^gbC>%>=;&bw-d
zvvd0vs<p&dZp17)3vA{EcJ&%|OldT^-K4qf-n<TFBr`v>lAql~y1(<p-@|4r{quW|
zZ1Pyl6czZdFSV%t?koDwdmA_S0aNh<bqa82_sBkMr-Bt=o7-vybERYw?l<1H_jtnH
zTMW8C3H(J=PHod;3e;FdmaU;XB}R%R>~1_I>FRLRrzfouLh4WO;ZxYY#Z+^!7-aup
zpj^gGvpCRu**2yt59h3893WV<5Tf!vn&U$sGgiFI3l4Lg55dbdhMib`?nzJfq<H^4
z1EBWg%Ix~*;?PyqAO+@S%);J*9iF#UPtb$(mczmzMzLRJ;7QyktZ;&}o80D9KDoXu
zsJ6`5>-^g$4YQ0*GncBk7|x4)>;$l=?>EBE^RsQMWjgR#OPoMrV(_;(cW(1>q+Qn9
z!jBK(^1Ikf`E_hz8EODU-dl>(<|r4Jj>TZAHLe%T^ZL|nG9&+VZSUriq#K}HRH4%H
zD({SvEzk{IUTIWa3oHs!F7N#(KhZbA`H|kYA9{O>22sCsnj+pEQywm_VtO^dM~<Re
zF+LhL(EedGOC>i@uxRT#s9Iq6^Grz7eniNrK}RFr%i4D33EwKspBPrr*kgIiOA&IV
zqq9Fu52!BzwE_(yfF0_U^tAH(#ABiz)oq-KAmB1@!M;x`m3^B)1itcNTXzyor_xde
z&$VfLgAH|eW8L~Xei{4B>fr8dB9yqMkNn#E{AUWcl4AR`0ohYmAA}Xkdv6&_o{6+e
zj0@oP1#<meXHhTMz8vQ3c(ly}>@60P#-b3Bnqu!u(OE-x)gh@ZevaGsq-P*+6Tw#4
zeHaiR*fI9fQaHz#*x0=&EMygYedtG1gK-mdYJmQ6SkAtPG;8%mfdmtu>nHv&7bvKQ
zByr|tYT?Te^<Ve%w?4#9^TC#^_jt1TZ8HUV_R5dcmE0Q;2-zo~m3_Abvd1c3Bs=xa
zAG=q`AMd@l)o||qm0?xY_1{8^pt8lSFpcH|umNI=OfWNe!6!H0sOCb53P==17NAZy
zPoaFA|58`oPa;KBlr^cB##Zcbu%dkM?xR~eAJzGe6*K^QvB98kgBiy3@Ojlsq0B~9
z!uNTxnoiaK;zd@4KN@AxAmt<nLakc8<v?oL&$TI$Hr=lJq47zR@$psOO|;sz{=rD>
zOiovotnrE$8Sy)s&}`f~+XZ_4*!M+op<`f!XSm!#mzpN&ZX)D*qpGBlA5ia0ydiVN
z{FHRFl)YGkclvsM0zr6hf$S>tOTOm9`b=se&(98g&#|(ucEML=@o7#e5FOKO=v;J6
z$w79618=c*eA&L^li(6mR0vSi=qGp^auL@@GJ{V(lb%1Hu}#CBb6EedyZe?b-twY7
zP*g94aoexcUJ%&*LT*2#`f)dEf$etQ-Qnr|=oxdtFlcmd+9{?8u>7qiy@FRt6p!2H
z_h-+0_XWM|l#w`VutS2|QXiu7%HKz#oT!3x@d{?VuVMK_b!bI2?cASGev(=D*QHqu
z60Foz%X{5{Ol!x*RAZO0yyb53{;Ms=vQXugLaRT&pY|pX{QT~+Ax$;Dl~Y@*SVPF5
zc23j1Ro)D(RMXaK(sy#r@-TDqhKg#}wu=b;OfpaN({Nb)s<_(UIk#ZA9#K7F@zvO=
zYRV=2q)$l19VCueQhe2&_9OOa|8SUmN#?cg9R_(T?eoR&D^q_6pbc_bh&7TnRa(dG
zxkoRulgzKrzRz+rFVY=Bi)hS6XNBu>&THO{SKp=|7nGZE<a`E>i9qhYfyyGg!jIm-
zcixO|z8`0mu`YNFr1`EBPef^IPGd{6rSh|J3Sd29rttj&Gv9$z_FH04DljLtMcl<l
z%tjMZ^Ppri)zVU9Hj$%5?X?$v>M5(+Z$?Q&Lo!%ia_Mu);u@Bz>Z(l4hrXUq_#2o4
zb6D15k18Gz?Rdc(o<(7!1j0~5%0G+?%2awmW$C1{@B%<Y0;pcJw7##W;P!S~_swpK
zb8R;jKVIM7i{?1q{@J?d;kG`_nBEqXM8bQljaV&S>mHEd>u{?u^Xnh?E+IsN<d^UD
z2kydBr)Tc=7VuLTbs$5WB$dqUYENTqG!RRwL4vB|r5>wFkPZS>nuN)}`(K+8$XsQD
zX@(q(cKoof0tebsy_ij!+!5fP2GHlg9OvE52wN0iH<8Uj(10i^@S!2WhR?aqgb<`h
zcaybX%K9M0ACi(*Y+6`=nLqs=mZKr{+tkFga@)sISZfa{OxQw?#z2MF0x5?vo?8%O
zo%sJum&4;zA{WSQApM*t!@9c>_WE?<cnQy7dTE9Q@fqXWjg)C>?Kp7qxJ=F*)_h@O
zDt}ytE%ITN$=9xkSlj&{`_^?{rv<nVh#y(9eJyCt-wXN+au1aLm4pyFEV4J<6d@a4
z7b5&WHhpmIC>3>v@E?fJKNlt%%}E{2uMtGWM;Uh2b7Sie&ruykVi*{>tRp3Y#a=^W
z9XHx$rO77UO#adokC6lyE0IgQUFg(c-h;MQN24jNm>@-x`=WpqRv_upf&dz1PomZ)
z?rxt=)tLEYODd5xsCiDnET55=g0-gd&V(Q`r6AWAWW*@h5K@c`MdJ5GbeaF^+-fzy
zceQ)sQJkY}0j*xrkwP_~6q@kTJWIt5I}u8_4H?&CYE~OG)GT0~6Q}mbR<h6-@@~NG
zKCvLD*AydqWVy-S`A&5Irf%=JWG&gQf<z~f4_T=k#JKdCBm8U)bF@`ERMO7mugN_5
zkoVilOH`t_c8M)c6DUM~X)_KsY4oqpY^xQ^&1P{w&cpwV{Ij8H6(P;pt=CbF=iB(V
zZ+(w7pd=#0bnj-c-iE6E?xsvr8TZRJ{uM1L#k&`i0Wx$HLyBV!NbFS_N(YsNN4Lap
zFQIN%l&7eNj|baZzt>kH@>>@gVylK)_!*}bvzL3cS!!_GHNn!k4Z<BNwcA=%-*EA?
ztuOwjidf{ntL22p^D_h@z*eTp&>GRhLLq0v#Dyc;5r^-Y4G08o>}{!!0VR&OkF%JB
z<GUe7)F9xc3aJ<QTzwBvUwm3_5)D82C8{!s>H*2aGn=Vml2>SIE^?hIoQx!g?cT#j
zP}d7A?qsy)Dl!%=2@dmfIr>ykg5q^;7I~8gmFmytXBB$AW;b179je5>R}6(n0@qJr
zfK>3Cv9+*fb-r}b!9_1G(_E(fO_!k4LB735kF%1Z)y9-CV-60<%Y>}RdtVDdgzoSA
zn^)H><Zs?dvj4pN{CudbZ|f+X;ERbmW`802N)j6?nJ3{I#Z7CnLi&}t<M#?%UblS?
z1&R7>1y|wXhII;4{t__@$HrlHmyy)bTce)Tm)clttrxgLzi+4hCTpI+zcWwG3()}p
zRfvE55IK^|8D#|Ak?*?~51D3y1=2)arR>d~<u`k-&f%j(VPoRtvDI%3Coo#=j2UkV
zUGYeCuU>n7u1p+xHQ?c@eTApv5nO?6EI$Fm>8drE?hwVS3X#WqSrv;^RpUVMsbYn<
zNj-pn;x{<f3)wmPl{~Zm^Y}=v@MAgfeIp8<74xL<=bA@6N+=6N5HA^RW8dP<qsp#^
z!?C!z7G7)X5lzJ^T+~#lx>j}B<c?n*5Oi=0k0!4yePt^*jAx7?{Ps$Ay3C+)vXcS3
z!XuYw{t0JU%DcUwz0c>}=eRdgr*XefAi!med#e4ho-gCKkzTHr5zE@}JEwwtTlFhD
zubK4w1qe+Kl`zY!w~eQd)yt&Tvx5ahh1{R+Hpa@+<z(l-hO-9>?xyKIG7a@Wqh!_w
z1cE+3*!3B*t=iIdTE;we&vy}DcddDwyzD=<E>lO=poZ40uFCW4uhb_MfGOo82m|^6
z7A6JC`9l+A6zky6X&hGO{Iy2$_6a@}h$~5}WP4_-H|rb~vocP6gYD;ihU*~=c~Omv
zR(8R5ww2E{Rn9J&UUKY&J2TGVwh67XX-im}>3FL%<5VP2Ea*pz3u898Pfwkz)xH$E
zfJJ~ce(ik^CCSw=pJFq6FE%OhkG2eFcOS{>q;iFAwHZb$+=p`I*K_4ndA26g#rmDh
z62OtR%G@d}xbv_AszrXlHXNuYd_V85h<?Zqd04nyJUX@H0=(*Af^Zh4-_9+cpQOh?
zq`Y$Q*NDM-(oC^p?wH&-&ScCP-r37u3v%tNw3UoqS75P>N$YAFRXKwmd`?OnbL-L*
zm$#w*g_b;4kiMf+FZ;<v8$S|~n0dMxjmME86$WS_3TQ%FFOnkm_S9>!o-O4|rN&|#
zQ_qPfZ0ib3!O%HaB&8tb9Q_3xuwY+yu@uZZ%d{^V;8U<z5-wnUlHS}#`hty8{z=;_
zeSHb$@_f1g{+Y8=%}z@1M7q=Vn=ssaoaD`sZ9skE<GnV8ly%8xAOx%)pON*SHy6g8
zx%!PHC<ny9Uj#Blg6XCr7C~jAN!O895E?PTme`>Ab4Ni65YN(NWHx3KImPID&oK-v
zIE5l9#n{lXMP<dNPXv?~qoL#DTI!Zp&lC?Yle(74?o@-sFGMO7GFjg%XkO#>xB6|a
zu_W|MQQA?U2RprUYujEQty_ojVx?*9uirwyYzua(zixk!tM`R*2C6ci;o~|g5)%|f
z0gU7IJ0r@{&Uk<y<~oruUnqn{Nvi*3&n$g)7~8jcb`Oc0G;g-xvdr9tJkQm?6AGFS
zPR8ZqF^R*g-mhKayS4j5i+moITn-T4yT}!;95kR_oX0furam^A_kRVfr>(0d)cObM
z67yWR^@t>k%rf(wC%0vJQ3~InWUXMORZUp_qBwWiWC25ru#I)(#@Y87d4)vJ+ubB{
zeKFzoP0f^A@@Dm?`v%?%2J>F*rdhN%EXt#9vjZf#izA1hz7@`=*<_prS!?GDA(7Ih
z^1v|BU)NH`PJh+sDFOCwUUt1D(|YuE?6H2{tUV{%Ka^(KA2)(lx7kgBHI+XPzqF+w
zUmgT|X$Zd_&7?tO=cu@*@Z!B|U-5EG0U6d3fK+7ydSnO)drd@Rd=z~^uIrhANcheJ
z-N1UMbZ?sDYp*OZ&jyQ6h6RP3b->{>FYLdiP7s-~?-lE2@D#9a-IOy&mpySNndEIi
zg-{VYDzl%!;#$Ns)=b@%iw4i@xTCOkIQi4gF!=1X65F7~cJ)UO$-Ld<l9&^P-WEiC
zDSjiKR`gmCP&~e+ub4c(4@*O{kOLHG;BZUvVH#MaN%7*J3L+Sk9xDKUBIxW-SW5jY
z2-itwip=uC3Pj=7x0n~j7StQYwz}FeLI0wMygWcpIY6o!)cK5HKCVkKT(U`VeXD12
zlNBV`)8cD_98YS;l5HoUfgkN}JDx^bgkg4g6ez%Xb-kN=sRL*Z(A(hIn#;a%z~EzH
zluGv^0Y??f?`#Jj=aFybLGNdL`fr^cI>_5F%fZY=RqlU`g)iZb$0#ev2WNVUP_6I5
z?0+El_ER@t|6dGF$<ZJfAoG==2^w_A#{gDaWbU5@J9U+6z?h0ShU$9iD&DxM!@??$
zj8EEoJ;}Db#b&$;T(gYl_%7<^UKhpJUs<oug|2*ZdJ-1a6PgTFYe!4E;Yk30^4)?b
zHeGc2M%}BcqQqj8Ws@)1caeT*Opt#+HJDPeyG5-j{(n59OFxTsjbu_)3&gdXt|U4O
z9c;8~jxZ3I8k@!%ni`6rqocLPNf&4Hs&(79^<TS>1_1>?$);^SCK@CTFQ#J{q*ebk
zd&R91dGo?Q*-Pmg-3~C63bo?fT(5?r+<}YJ`}}&L0Ihu%aF-WV$ZoZ+B15(Egl7k8
znrbui^4d{qU>;C9q%o}~xW)kV6p&O}wMQOI>XTq&6*>_mSYy){8gB6_Jfe*0=N5}2
z=C@%-M+n3F%pxzjCL$3@RWAXxU5F-^>XFD<#wrPq;~Na78n1>~C;g6p(WLS%%zeu?
z3ax#=y^V>V)Z&MCi6+VjymtA6a$>cTS4YYWfEG$k=6dc?BPn8rzx{?#CRND#*Fzlf
z`_1FokGNRa{8HV`FW-Mt1hPq97?JB4Q4Ihpqz#+P`n$kK5IU4?4~y+Qcs|@&0y0yw
zpK(&xKOKF2!dSE;u#sA9zj3~5U&Eta1)$5Aw|&`kDf@4ht?E3ttLM%;s76*QZq4PG
za?@GK*j2t0g*HgF&7U%<8C@jru7k}nMnId$jHftH=zVz<?igUDWwVf#sZR7?$Pq>U
zem1=~@QvWl;U%=F#K?M|ee1_Sl1s4LGi?D+MGo@?582N&PVXN^+plSnt__ez&!i@Q
zdt!C^*_0}vYZ0>Zj_DWX)N`#E8l~d&4U@<J)XNXqUjg>{wD;=u-xjR<`$ga|kc67f
zl>>-A9NNW0j0Yh4U!BF$56d35tZjrR%MA%M91{ehpmXIYrmF&mE+ar6lwV8)p-aE2
zcDHw>NSQP#^5ErkeuIu;TSxvwNs)vvi{-*rg=6y<ZTvtA@8dn&>^>8=e6t?}N~tR6
z3h6Z2Iejs*tm*u8YJVHEB4PpYaDiw_5kTaSy(xx2TD!t^nV;-s)Wud8`@9P!ITzB0
zw9rJfM(m~myl7<5*95PRgKG6Fai5I;Z@1ay=8A5Fhg4cn#OJwEmV2mVRs_?M@GkES
zu*B~Zv@k#_dT4^~wLj2VNmsl(C+S@wY~yGie_G+SOa;OokD~9dW2})Y6XtZ2?KPEE
zWFtk-<+eY*y=-*bno?`0?d$z?oMXD4_qFF|F{?^YFi&1L{dA*q#YS{)T@F-q-SbBL
z+wq$)q1Tl@UO(6IT${<bwI}4Yb2<c40ycI2Y_ZAJ{b1|yAtLW!nxotY8C)arzlY`C
ztC%|iZ$XZBd~7~%TSaA3P?6BG<<(xr1oVqce!5~JZS2_ZgJPj$DHUkcl@Q2nd+<A!
zYeucic$ckEDC%u78=U&8MawDGzi&_dO>qyGkq#nh$--0XiQjvO{{KA0VT`i`RL`82
z3L0ho>}1DiC~-B@LoQ0#qyTnSrIP7~IoMYEA<CGg9Ttf2(a%;qa!E@C<Yj9eK-1YN
zZ7X0y9W_$=>nvYvmgWp#FnaN_erTF^dNcJV*I<oS(m8p<#**izP21Rfg|w2~N6SQ-
z`VH>iLf<E$I@l!asrXh3fQ_~`Bq3;XIbX8b^uMt)j-WPTW>8QJ(2pJ>@_JAMJYlgr
zmqY6@Ho(ykQ-9HdBmhF}eQ5H!-UpU0p@3V2A;_N1?k@Hq&cZSgj048rc}JzXhX5Al
zdYaK4OCSS4Vt9Da5|*AixHol(k)rz&FaXvI#$vlx5hLUn)x&!D0Ur?L^_XCHT8&d$
z%8LQ>fDjN#0O&3fly}-zPw(-1>H~PM1&lbQmAzqCmC;Jp?Wc8zz@x(foT{nvvul<=
zaKwNAHmQz5kfCt7rxN&er$e5>hTIGzZ9o|*9wnNU2ElQJ0(86d3GB}5Eo>fI3a~f9
zw{?et-7G8W{$n4D&|7i<4d;g4`2Edst>%zkq+u%y3>QCug_B;oGd)7XFey5oaKe-U
zTj<`8Y+W4`gyoNXbYZ@S*AwiIzqqE_A0$cwIC?4scBhsT3Naowm?603)lTpcZ)EQ6
zC>r@=o<soZ62Weq$aj6uVp`f=wuBD@vM*k=fVIT7`yuT&1z1Z0SBc+&%>jJbJ8s$1
z;b!U`QHu?xkj-c)FO7+I^nHFk1Akgd!tVSo5D<<U3}(pZ;LrdtVd$yvH8~IqEm<B2
z00VZT#h6er$cwG2+lImz+8&fJb%=N7P;!K+11a>EDoh>zMXCoE$BK6nAb{CDUBm!N
zPK>kXV#YY&&+2EeJ5@>Bx)3D542fO101lSe<9jSBv<*2>fffh=33em5Ihh(d(T*K+
ztt3o2ny^7GbN`iKBAoyV2reSLjG2H`{QfaeTy9+H?j&$WBPxLhc||05LW&9$@TV3I
zxYMQ(Q{NE&8JHmokA?)G2>*|J*hULfO5Go}&k}J*79wJt@!`+iDTgi!a0Ac3rvAbs
z1<EHOEm{!^M$-C^NFsFGfOjVWN#N(~s=?|1l=l~Y6r}+enLgiPch;x`m9oRZfM;wG
z9Wa;Jf%nRnbs(<)84SKi9OfW}Cv}heA-R*FTi+=M8w|L=9#P!*gb9={D?5E!4oGF`
z-U&n}^IrG_lK?ni*m&ry(C7Jw+*UHApUfav)1nHybGG2+RDutfArydffVm8Q@h^zE
z$SCargP)*)MdMQatLEbV!V!oycgpd9mnYzU{huw*mtaW<!UM18jnxl_&4}HZE+k1h
zz)c6pxFt6bJR-`}ayN0HAi9uD5j;?B37GPcziH9X08-4S0%WPJA9H6Ixwa<Pti}&v
zN+9P0iXCP+i_~)Pfc5$#{PPM%=!4k_SD*mDDpq4Br~wl`3EHL!6hs0JyzI?TkEKBt
zc1CL&{gej%-NitI{T}Q-c;)zz2}c6{@tz;b=bAem_C|SKdmMy+9OJVO$aU8PA^hN_
zcT%?oIz9DWdO9rw6(S)EuS(q_J{!{5CIC}IkPNil!D7)}NWHrj3{xINLb2{aPMj}&
z+ESmIH_#fO*bYS`6dOpwo(EbC^PM4M(AeA=a-e58(qMUaI6Y-bc>o5C`e5bHPudo7
z!Sd_KD!^f+fa3($Cueq9z(xXoEo894WRp*a@PWavsN42^MBp=>Eqyml4O`YSJ`cKY
z5gr_J{rlFb1n{lI5bG~I;cePqoINVJlKq`4Pq=1*xfO%A1JC}Cc?1(FK@8h-CRWdE
z*a8Kw&+U|JqWqneA)>@eU`c)j*GijpS4@NZV80BoTwA(ZtP$wvror5h5x|pFYTMd^
zlp5Fq{J<#ke;f0FjpT0Fk3f+1yraz@M;@?{5FliQY0u#>XHYo}Fkyo(C5DTDI&Z(J
zzPlOR;XwyL0f_3ar%%DtC8b2m;%HHAhj>sU;_s#$0dHxFyfwAE;Gg`(d|*2sN>sd7
z1fW><IiWH=mItHpm;HPWldSJgik31_j;l{qz}|z`QTV=w8Dq491``9^4BPmG6AkB|
z;Q5&R2Kaa17Dv@mwrH>2h6Z~ZOvY0%GeK;Q|Ckocc^g7fCp9J*{-^fTufUo|i5cK7
zO{IgU@cf%}YF03hpZe0jBBuk^@)GAdDT{=Ue|4aMzsenl@;MF|KkX3n$A7cKECW;q
zQ7}*iEf`!6+g=LsmNLL$Fa}WA^if>ER3@EaU*>M&#o(h~0F}u(dA$9H38+j{j6qW{
zP)WgO_2IF=kWu*hpa7M;w`zBv8w`WYv+5&bc`3m9cz}jA9rOU%>iUF6>&4xM0fxWN
z#ISManScqphEZUeMZ(YU-vBNxUCYMqW`@-<rtvXsFfBmAydKgE{YQP@z{c{$(eW|&
z=Qi!fPR?9BvHs3nX9BY@GrkMcnQOe3vwJ~FaKiL!?)uD9Fc7rgVbVJF24cFTznkL|
z*ra%R4&eW1QQsj7e(BDXV9cz-J<f!<U`~PqkzuQ-Wdb~T_J{j4-<?2#Lm>yIP)$ha
zsV_`p^zzDjo4^$A3Nmfn$-Ew%Qfw2S_CfGo9c5a)fyKBn>0&BU0E@KhI;aXg>H+fZ
z1Gpo5*aL&bKsZcQI;w8;@A04~&<q?52Uq_H(=bOwCeZqsr1GVB?D^gEEr3Sa?B43{
z{(C;f7od>=F|7NkgkU^0NKSE2dBFAUACnMaI^|;oBvd1>R{HGUVgeN$Aom=Rg7`53
zXaM%Zja|HWFrJR}!I3+o!49rhdbGv-`JcuV0k6r2`^vZKuH2{9ut!h>jNb~~8LjY6
zFqE`F<2!s@-T#fh^f?$m{vL^H|6`yrxuu6$GL%4L-nxaH!p0AFO)&XFarfIa%n}fR
z>!T+Oi)>!N#xD`PNtFb~6TBy)ng$z!Jh<KljhOAuaDsPefd!=(YW;yK1sG8WzgC57
z93ZS_5hGY+E<-^HrW5a3$ML_f)zbxE3tG+$&v1bKVCMMEh&ATSn~`BlTG#^O1JB+^
z@K65F)EI%Ol+Xy07I$EnlN6_ucEIB|m3G*#*aa61%lR03SDp#YD*Rg-&nh2YJe~>%
zLaINdpIv<cmK{3g-kpLe105iLGXF~owq{{MpakFeQQhUaAL{Gc(4_&xaXgse)tCBp
zS8wVJGXmDay=Sm%?|)(^z+4bCdXhh3*^I3DpZ4uN5DuiZl@vDDC{KN#BENLDSOA_A
za2#cqz6U{V@B;<^CLW^wlLc%^-u;5zfHldw#i4ir{tyL&IL0HaxYvFJDIX+vy&D&>
zQHq@akET?boTi%ZEzF<$l4c+bbfV~v6BsF2ctW;)H(neWa0`MnQ6X4(QTYhJ2zNed
z>f<ZI>wudM;DhpDf@tvpt~rHi8!57tKiM)RY#awKfNQn8?)=F<#LzoS&m<GfDvmtx
zcfAo-O~(DfW+oQpd*lrMt){#4cTKLl{V;&ilrr80>oa%3dfOG4BK1_5YhK7{3~ht$
zX&34hur0~`KE|(*UaWWmd$tUCpH-Fd$bV%u|1Z=7;JATp?vBa4Gkk7|2qpmm7Z$jD
zPyDZhahT&MU^v0%0b6U4+^>K#mLlIr*X}36T3}YNePMBcss$~EbKxn<Hq0e)f`v)n
z2NF)v$$|Kn4f|uj#O&O|Fau<sgR0;|B*3l<u<N8d?E3$H+W(KFmh+RscCQ0xy4=H8
z--xMkUK11vSi3|ZX9wNKtS3U`mJGl&blz1cOdw>!z(jpdpdv+zuo&>aATb(mKY&xb
zy#gPZy`3&C|Lk?E2J-n9{~ymngQ#v}PtXmc0%#`-;=9fS3-FGjG+Ik>fK}lcJH&T{
z_IlgfZbsVQuCX)dl;!61M=>lksPZ3P?|nboEL)^Le;C>887XWQYmWGp%sch(J1=#v
ze!G%(_v)SvqvJ7=KtbfV{DphSc1!wWIIIXp9)zX49YrN?rn#oV^AFUy@JdIL&j3pw
z)$93Oou}5GXXW$7BdxVoP)lut^QGt`A}|;-H;OHMi`mAgD$$gMm~sQxWO6rhG_U>R
zl-8qPnJD!Trv}NcOIhLlBqgaNbNtU)0MmI8aGo^PLSZhE_%dtxHHUf?FRZ=-9p7`|
z&b7Oth6Q64!BHl8tz`BcsQ92<3-uYA1yW&ySvOW4!~EiBDtE89yZb_(;X<lEJk&Kx
zpBuuf5?#l$YBPe~#GkEZxqQzDNEuX%5-QC`rF1+`RaSeE|NI7>W0dQ_InyfZpnt70
ztXnC!`54Mu)UFzhCXc#DqyoyI4(eGAw>a^x73haHR-n>%;YUHL4UVU0gn))a1@9{f
z*1;j%f-*FOi^hPC4Z87Si`Ug~Q8vp&KN|r&d(fmc;t-KGVg&O!9Lvvm@KGE?6pImU
z+EKnUOU`#SF+9I7OfA~lqUYE4LpO7NqmSndajIl}zkD;fAWW><Qw9}vuM~7oyUzI#
zJ1G<)J#ue9rTv8u-2;%^SI&KFhyrHynvVu`%k?^>k9L2R^^Pi*fDkEFV#Qn2XlH-o
zdot(^If*UvzZO!t6EA!PN0Q48aSA<M694k4Jp3(adHVr$8<fg}7eVUMv-(1j1<oq(
z{jJ>64LW>MX#a3zZ%~q#4+Fg*Ck|4Yk-_0QubBBROSCW5B6jABt(~_HN;vAa>&*p?
zJ1l!*j;*BubGe63!(o1Pig%`^a=g91rYE+Ut&gg+ug5>|m2CMMLs&UlZyP#ejt64s
ztKKoQ_ea{$a~uAvK1C<yLH|hEkId7gn{NifE!4(uyV<$d(qC<^cV3ape2d=_IMPe9
zS-1%#xo%ykb%iGF-0FW#%5{g%Jn2bgYd(@`-d;d%#1D+msh?nruka>OWg{+MuTw)4
z*47xw8T@@Jkn$*x)b^&u`f|VImHl8lBDHpI(NX$DUXb$*z4W<N{G;i;n48XYRV&5P
zkzd$vVUfNTD&*oBw8<j$y^zBT>-h!-LgwctA{KiMl!$}LLGfaXZc2T=>0VI;3t`b8
zh5Za239FWd1{Ca%hD0v6+cZFRY9r4B{4dvJY%8M=tQ<wjIHmQ>BhQWccMi!22s{e>
z98=0iR$Zt`?=~(^kUVRa2CkGBxl5rqAI8~tCj8z_PtD#=QKbULW$7nrSk!0`G=G}g
z3Q)@xSrf18HO&+Wn!?M0gYz3f6%@75LV@z(?T8a-NGE<S;lHWT8S<o26#|+|zOAuN
z@mb=oi2PTbc9AtX2O4<9fEGBepyaCuv_t3xEztH)%b#)@%Ydc|;CbuRyAO2f5k&_N
zbs@5oYG`yWtBm2ZvW<!_F^;vN`{YkAZwt%!ySZ66-{tjDlr&6g=$K7Ruk$${K#Uzj
z)s=?hlPcpUMM+j-Oy0)R9zp$j<Sw@B^eV1)b4*ONZoT*+EBQ?p7m3@qK}H9?4?k|g
z3tx=Z==OfG?k8!>K8DIQDobxwlG9@g%#RK(vV2A*C8Ez9z9f;T?}F0GePdh@R&O^$
zeq?&ka8qvfor{=5n>FS3<<fy)=~FkkIH9_+zouUr4!W+dD7|Rm!JhBK5+I1+ciUjY
zG+lOP5nt2y+aF|He!d$g{%Ns$^(wCZ2%$<#k({5N4s&mKfh#ALF0J_SnESEuAs$MV
zZOuFPhz*QwW^a<I)i<aOhy@D`^~x@Vk_|RlLab<qzA`Dan^-aav%EiBcZo^kizfXV
z)A(xD=;!#OsJD<X&N^BLd3QHzXQ2v$Uh+VgLU3CUQU<Ii3`{J&8*fYL=MvqGU)aSy
z^%tHx^+`Z@)1w92+@!#|(`w;24}vC3vZ}==w<F&{!}@YN!?{JzW_0%22L_Z9FKxbk
z=!r3x`&J`#n^ve<RiGzOUZdRLV%Pd*SuYoq?tu;;uNoF`M~!Q2B2TA|lGuGeS3bmj
z@HlJK+adz70;&4lf-V8K??!z8wz90Gu7`>P-_Irbs+2Q9LY@luwy}#y(R^^?15_@C
z_7!cXP7!Nj)nf0#ivMmn!hYijTl1dRw=f%BT^`o7Z%}+W)<y#8+CkJyQly3_#4*cN
z)lja(lF+^?&g+^t6}h&yt7_?m4(9~)jILyEbJ19!x;feF_Y~FyQo{g+dFv{hWWm^x
z=_eEFcM)^;XBYduln6p$TaU2X)2igq8m5;YA+QPN%!?Mi4)twWEuBrlV1@tco(fpK
z{`tWICcSCzLd-prg(T&sD;*=2>SSYSQy=L<Iy#9*8VBaEV0jg>4s^&tDe=P2lOX$u
z^;mL-t1m>Qw?h(Dqd|sGuIbX<HD<i7oL;-@$rlQguP0x=EqZ}r#(fr=v5?Fgnl45k
zSik=6W8aQADG9u7E=k>5w88q9?TcYgK@xAvq?2Uo!KMl7Te**i^3-m#08#b+EWON%
z9nxAKhva^se&s;dWOg2;YPn$QG8*v&v03XNvEjyA+9*L+1oE*h^>3uy&l2FuBiFwL
z{bl0%;@R9At+vL#t%L69I>#6892OHru-2F*@n7W=-xljM7U(p&_y6whA>p@&bs}%j
ziGGhq32U|uI_#V#4W@qveehyHH+h#v>O|V4jQ1W$OzS<Lr9tt~`l+)iU9V*QLs@{z
zXA-1X^xxoL7${?1+RNL}7&s-JeHSW%dQE^+A5z!0m5{$VsPD9S7O><RIDfkeIQs4O
zyJtkM_J`B)$D;=UL#MXG<s(iT=W^I%`+shq4s<dst@(EfVw<C~Rl9kYDR6w~U(Uzv
z=w(^_^?AnXs&4G*vqkCgH04o$_U*0qZ-vyJ8<txF^2ekn&WKiMTbMmw=bjw4(@ib}
zK5~ziF=LtERi$mX^0p7>_V;b{PEC}(U^ZrXiuwbSeh!5JCR7P@$m?jq=J&5zQv_Je
zIE~{V$coWzgH1V)jZ#m}#&yj`cWCJ$Z#MbQt}ewyzSGW8F0^!YruSvjq~kR-mRg|2
z+9KDi_+se3wkal~SAM(j=WpWNv5Yu!*S1>8Pu_Hl*uKp{8UH;)l5RZr_Fv@N9mNh5
z>+fpO19T2bzP6Hi<TlMkgJU!@AIH;aE>w0`@i~XKr3&I$>?E^K`D;6%LDD1`=3F@r
z*HK_bHIOGVaS{G|dQV{huS<=7NWOy~#BKKIa;%_)MXEm@H`YQhU3J#IlF#mClCo-Z
zGSL|cUHo9G>I=QC%0FNCS5jkM|Eje`gS?`{jH`Poq71JZd6}$iDH4hFl;mP}vwvi}
zYGg}3`7reB&*PHXwV5WjvLy2`|M^Ypv4*+6+LTEMC?$0-8gob*y7>OG@})KfPuG`U
z#RcPzL&^+l*_k?$ULO--abQ-UVC`&ig23Cx=xft6w|HK}rqZATa2VvN+=c(|SXKws
z9@qVsN43>knnz#0J+fIQiNdpCq&Qofd~an@^9g+~lvh0=&Cl$|Ax+2QsndC0#^=UT
zEU&2_+vpk`f&%q-3d=Xz#4-aU!qjS&4>C}wL&Uz&d0$JqhUgCnAPwo!5Z(qkTYf>;
z+-dP<(;k}3cd+OS6D>jw80?zu@@CV0>>{)UC$;Pq2K5)U!bV`}hqh$D{XNY%GuwHw
zc$E2F&+$blXAq&oxv;7P;;UWVzjYbiwe{KGnI|$=s0u`}XEXke%=c=w>V71|#yNOw
zhkr-{PIOfYtTY=t6jlm;c>-K&F5ZpzehHYS$8XlN$OI*+_p<{lH-|Z;WS{5tT+T{u
z-l%txF8VsqManT+_42)+rzG08$xb;IuxV#ni|@DK_Ft+*Gk33XCY?#_2&$Pg4ixm*
zjM!fUuW{#IT?q@(k7in=Vx(Z1>&Bx>d|HW@owm@M)4CU~=rWK9BPal-&dhu|AB@CW
zRA%$9x;OCuUES+RyHrBzv)I_*x2Aj^sbotu`=i)WhexTPadCVdQta?G_w-6kb&w=a
z=-Aq#Ss&Z5$!<|;y-uJE-H30~X%REemiNcHMY+=@eCvKVT;JIbHJSxzm4m1X`{DB{
z=am8P_Y%+F2Ta-R<15nfa%EP$Nk@I#Wc{w)o#vx$kcgNrAKh5Z@G8TXsjC5>o+J71
zQUiH)q*B0-@(F&jM`2sZoUB#X(1vjWU9Px!5l(1)kk<BKHEn5crgmTA<@p;=#pY}7
zz3Xh=G*yS*`A9?W*<QA_isF5g4wg;Fn(C`pv#75g`5QbVU-1YJ`&PV>E2x`i^OCxK
zi#@pJ+_^;Qq#e%lnGWU_`f|T;t5NNrC9|0GXu3J2p5lFQ-T|b*lpQNA3QKAr_D#P9
zgZ?o8DikqPHvXxF_R90$o0K~(*BC1~4sP;zDmdM6hlu`le4V!}i>L_VIanfT$$C+5
zVuyE|s~l*5{(`eYWZz7|GeEB-s{eVw@+iOrWk{X$G?3Te!~5jP*!(pQVFQ&VHxo)e
z!qK&qzULL;wM2^pgjBpX?;&+0c6iSf(BUlDo#us?!wGV!>%Gq@-aC)IA1c!<s`(&Q
z`*N=BQz|jB7i8%+l8|I@+;g%pYUjJ6@gPu*{<tRv+w^mRAw~P=r;~c!sUY92-vm1l
zqEQ7Liug(0a1qT=6clTF;Mej6(&6bw09+^Vt?BWF1@g|-E%7d7=2gL48piP$<)ru&
z_bUah94t&GrMQv;uk;`3-6CGcj;>#$>|eBaWtuR)x|Utk<7|DbxU8uA_#yi1CCIlD
zSpwP$;CVdKb)PyY=NX*OGnm(Rn!7OmpdnfN@?nxtv{dJ1wynujs=lCU7iO^fvh`vv
zH%DnJIvVRrALT%bC~D!n*n<`V_H<f`)@=-?SOanRVC4k2*}vPMz0Z}};FScX`GSc|
zJ+~biI3t1qd7o3b)g5`1IUILm=={5y;`3cx)(*tphKz<HY{JJ2n%MhpzpjU0{+a<%
zHtz28Niyr&*FPz5mUrReCb~SJ6RS-pM1-UHBB-RvoVD2=oUREQ7-zvpIWHb4B9H~f
z6XFWRU3t~Dzswj0r_Q`?#Y6A2^2b@BBN=KSIek#Tt$Ff=h{df|`Jv6jtH#SBv@o<X
zG*9$5$f8*%+qUA*1M8p8FnSakhpF0xiA>usWDB!C;hLJhUNwI@QXAuT>^BY?fz~Fr
z2NFR;)Ll^)wwlC308xDBhaI#)^8mh$3BKaC!TOsa|0&Q*I}JwRAv1d<g@f&(tl`EF
z!@twEh&&)eB`AJwrkjzB#ngV|XYWfpS6u^Y`M+MfZwWASM>6!7qUTnw3i_&K)SDG=
z?7iv}R2$G;#$2?JFOh0HM__+`IPV`%Atd&9%x@%17wP+r%F+V)zz0xlZUXgS2t}%A
z`|={WTNg&2@Z64+0p}7e0KoOm(!`cEgK&AI#%3-I$3$RvwkDqP<$JYIM-lR$&Y&T0
z4(LAivBq?^`uihdo?i#=GmziV_VU0gm}8=;q={0|F78g|Fdxne4Ex`Pl^2)Ls)q;e
z&YFXS>5P61@u}R>=-CLf&}{R-*q4X{?bY2cFw}RRWj<zI#w<H0B9Qs&wMeXhex^Ez
zv#){|Y<(`Gsjl#syEjzbdkCwcvjLE7C0Kt%CYC*XrdsPjX4_?bHA>Tx!xjnG5Ks&~
zFWJb9>!i6hW_i9orRp@!&YT)fQ@Q%;wyW0P$NxdRdeT!Hwj_&?=DlbHtO_q1k?gpi
zUb)VOSj$)o7TE1e7<`te`>gNKPF2>**BZ1$bpA?vjB`7YjG{-+Az~nKb}BPbGhg|3
zil%_F%Volb2ROgpMb-zwRlbYuj^)+%EoUom7?>0YeQ}TzApJFMqCYu5fLz`RBr=hQ
zG2Xq$k$>WCdn22$H&bO!Ni*d8GISXI%Y$Hdv84c%Pv0uAYku@!wysb7unqF(-*=aR
z>W@8aS?PAVCLn+NE|h7^hSkQ2eAoVa)9OCC1yrCSU6eJECcV=AopwZQ?vV5kUWzND
z&-fQ7AJ2kF2Fn~?2w#tR%DMi|6h5sFw||dpGa??&n9mDQKY1AH-2o%J8lPBG7K)H(
z2XyLNctH}o(4#N9OH970R8nLG2?Dd!uH7Bbo86j*ojH_#4tcbc?7VnIjk{P`UlVrk
z18JG<bcf`%2VtodCarWjf&QyGphw$Va4<?#n_o9Mf<RGjh<zwNOF{4VH|@;?hqtP@
zOnsm|=|4m!8S#H2GK*T5vECr6Etu<1<}I`z*7>4UUv0AUyxBTMY(u3`l>rWaePP6s
z>^~*)sPF$=n}K61D$rCEy>}mMj_cx3kwq{cem&8o|JHDc{xmIZp}x-J`cnPW9lu(%
zsSkxb@J|)$S6fs_zH|^8AjGHIdidrgmYAKjob~!5Hup<;dCwh`<14Ml^=bZ2FX$JL
zO?7_w=olQi*MU2Z+&=|}!!-Bj&~<5;+i4MlfRZUH@4-7;_COTfmVl-?P_rlMC9EZ4
zcKs38>7$ps3`He>-u{MnZtl>tE+PaZ;bki+`5wz&svgtVEh7V~GaVsXC`chS@R^Dz
zVrYs9d+Igq6x`e+b@HGvxqd@D_(>2@u_nS!BA@&q7f<#6CuQjK@6spx7;GBAiKW`Q
z49p2$Azz2I!nM|Pawee|y8TfgjLsMGkNcaoa?Y<?DxVx9EUono-)t_<fwe6fR0*$I
zmx9>;R{q2l%D^r8!=*E6#QxWd%~^qJDAaJbc4+P60J$BSJKM~L19J;vdwuW9AvBiv
zPd)~naEG2ec{k)i=+Cfu51xgYKZ6k^g}o8`2XUJAWNBp=3pA=ig4swwW5V}7Cv=;Z
z($PYv%AuqWQ*=FOl---mB8zx;_n@-N1F6B>XMm2W<xg#)j|dV$xXlDNRS$1RSF)RR
zg+g&>7DiP`S+31Jn~9fiw6K~}vExb7wt8anhK%Pe&k~$gb;S4r1#c9)_Sa!RQwdx3
zoi9V)l){?sY?(j9b(<GT*6fO10~HRZB7rlI`i0NOMM#4Nky6gOP5^GNL129%RI@L{
zjJ%NiZ~ESi;8I#Q!9JT+FPYFsT8Kpf{9e4<w_+YFFU5xQGqu`LmFo)x2Kjl5>p5sa
z!=b45TXaFa&BZ=~F|_9{p@&yNFO_$m$Nu^Imme!dTSRx0!)lUW_jxtDFu|#~Hc}%r
z>j=9;Sxw&6NpZl5sChRl$!*6&#{}*W*&sgcdE?1E-6)gz0ZTtsVVf4q$-9Wb*F}tl
z-6AdItT%ASbbqW`xi;Zmox8jb`IfM&>++g3sF2?aWm}eD4;vZF4rhM!AXmjGkH=^t
zIpa652<IF()7|8`xssfWgM3st)3c2y)f90^4Y)9r#mr`LG5mCnr19{JW@)=H)*1B4
z-qKh8%j_w=GVBl>Z>+)>t$18zpFy=;+4hc(2!=D{+4ACl7QGDlEY%-{diOs1xjMYE
zt|pM`v55M>QX~daJuBz%2E(4}f8MCzWGB9s;_fIS`}OF>`ybBftfzgbNSRsBXHi1O
z^gA_*gsvBL<Hy1nN!o3)L^OjlF7P*BdJ3ramb7i_i6b-bVE3-=NDP}@nYRc24eJ#J
z;NhP$a~vnM3g72UKl%Sqa@$%i@m?b=$Araf+oW8qE)i>Q@0Y=0q^Svf9A0@`OJF#6
z`HG9{@_wy)au=5{lT0fpR3$g;|5y}Xrxep04jixO>RA>R{q`q#Zywij*}6q*T_udC
z1A*y8{>k|Y*W{&EFDDA!kAW+lBZA2AzCcgUTOoYM-_TAT68jv6`J`{WB<hDq&z_3-
z$1d$6M{+!pr>h#*F56E7H8MZCbLwp&mVXD$1=xFAW{FW$_ZS|xX`d%`t_PoyxS+Re
z>0o(!56wRLu-|2~{>lEy*Y>q1plqM6<7XG5;dRcfH2JGeBo@+5T-?2rC#glx=J?q{
z4i|<7zgkvJF;qk#xlRxHFK0uSmeNwLo3gvgGP*|TF9BQAmbKw>>9ByVW68a8&Ed$g
zzXow3Y{6jb&t$;LD5&ZVaYL|IBHHmCLLaCCmJ0X+#OMFbBLK+=B5DnECXmUEx@de5
z6oPN$fZDdRwC-5ZIvP8QV318E#b<0t=~eGAvf8*xXq?!}qm;_(-wU7j@J6+sY5(Ed
z3M017zA+wr(vE%8fE1#Oezpk2r!~du^eD>#0k-c@1fS@Rn9M9m3N$Re#-BOtZWRWn
zHgQzJTY5}hxs&LJyEUq6f*8MeWyT-NAcRW0{-}fk>Ab6x^Xq(vQAG6;$q77P!BQYy
z3bU;a78GF?9o>UM6*5dvVAibF>=^K%t!e;uy@ruMa1>LhkaxdWv7O9+G|K%q>!t;C
z9u#x3m38#k+`F#xH?kac7CUc8sU}EO#5bX>#m%za+%Cxx7lw&dhzJM9dhPL?W5fY^
z=n$*E%~7GThzsTZv(IsL!R`a^lCjk*ZnpcsOpabjj*fB65jrlVq4Un`<u3Q@;1vD*
zgb=O3vX+VQ9)NQM!HU682<4YCjWXq_A^TiTudmr2gJV@NKE~Q#d(>&O_2wxxEVm`I
z7CoB1S#|w5NkPx3UWY6h&b5r`opabtgj)6h2X81oKvd4g@X~JgcpU48?03~P@o$5-
zcIq?2e|>(Cu2_jFbIJjsUI{PdMv$+#Y_B&QvEdiApsA}rpqc9d(%%!HRVKqY3qT3H
z6<VxxHz82~VW%ue)zcY-djbyU1VxXgx-Xy#F`f9)2jTNoQ!FDoO}uwqkD(=;A0k0n
z35^x@RwvNlpyXjC#{c5u@^O(VxDgWvV^jyHNj9=M+}*p`Hbj1FjxgEaLrG-VH&%5=
zoaG_`C0)FI`BL4gPBdiRf_zoJ&c@Q$BwHZ~2f^3EdeaLTA8YtC`7`u<Wqy&p^ov5x
z#I^f*c#ZJQezp2YDo@{B>o3+qpfDG5qyt?<0S`loKzM)8;(6h={bnDVFoE@(vdUA7
zFcJ3&R+{GhnJh)4foQFDQO`A>qZNxeNY*e-Zp52f&OnwOf_q=lAM&^%C+bFffN+#A
zsPQ_6C`!kCL`TO$w58KfiJ;JrWllU+`n9b~|M>OuTV8BaJk?1@Z_aX$UX(=c3>HOq
z>R_n@A2j2e;@BDGwZSTe9+_~rNHv2@hgP?YL9ivOQ`VS#k0~IO&+Dj^JAaI}jJc3}
zVHhOJp}qqH!S*`4>Pt%aPFxZ<zm8UplU+W?s5gra+p4{;ATnzsHNbj?METZ(EP5-a
z)ti(_Buh0lgpDWDVG%N^8rQkiBh^!6o41Hg76p<ZlR~nhYK0uDeE6PEo6dsMXe`q~
zmakN0pxX~n*H9P(ra{~&IA`iMx~J%O@27!!ml`D5i8eHlEmzMZFxd84<hW=IE|qd?
zvq>UYaM3ld+RG-l_h<t-#UFV^N$g$gNsp7LUg=x4Hcrl*IeePO*e`e1ydb}+{=_!H
z74ZB<wHBnR{AzcD*}@mqq|E-t?JJ{@$WM>sb-ZvXZJgi6YbHn^+s9d?YrtA=a9E<7
z?1ZOOZ#vQ;Sa&s5o?JzC?yDqYC!5rd?))g+eR*<3?#Ur{EHzi+y6-E9i+|p3S;DhQ
z(5+_F9J!6o+QqAa-z+=qz1J)YWep}kkN|{V?6NjenLlKfm#>7Vv`xG=IfWW%T4v#k
zz6j`F8%4HKE4R)2;gI*NC|#MQdDbDeUP5KDiQUE+Gg&f9wOpSqheogo&pDNy@c3B~
zVQ}4kSx&2g%$M7nc1IS3Eoi2|--*x;2fOI;pHpza)mMlCfs-w(AXvZ0T>ZQPSn<8#
zGkgSpoV51Uihgo=?CM8044flL&vz0%rBbMEC;nzE$%(2;{Be?`Vbeq5YOC8;VgF#-
zKC+L;qHs9E3i{)Uibua&mkd<*5+E?n{W)*VTYyaQXsRQ0zf1(gh0LPJME6lW-j^&F
zlfT)>ZA}2ayhhAoGvhs7srr#STx*@w89rC;k1Q6tj|P(ZZYqNvj7_UVV$VDt0uytI
z8Vr2fR3iPn<GT0Ra;DGh*O#NX13|n|!^145!fF=-klDp|SqfxoE!@T0<+Z+Io<;x-
zMNM_-@$fZ-uDfwqa<4GzHkbWGlvD!gKRtbML7Mgw@A@ElVKG46=fWzuxf`1Y9GbSg
z&yui+1fy)=u*|U}b4hGG+ROhjK(CjxkY{_|A7EC>@V4M(y4OYZO@;z{XNHf8NUC9*
zctVR@T9C7KOn}gyBV4Pgk#eKurqZU_$L9%|BiBhMon!|{2%&aMRucJm25?LL0dKCK
zkC)DeVik--{QuY?-?H?r<G7%TE!h)1Iz;~J9&S)DcOg{s%YInxl|nM*XGQ{VzA)Td
z!nQyyrR3&BIzz2-mHNFhRJ+&dapB~nc#zvw+<b1rw$wy3d$;+1k2Ckxr@roj2Hb6)
z3zMS7x)FcAuNe8X2k%F?-jt2zG5POCsUWX@C%UWfQU6BBgirScJ054z?k3tvN&VvC
z`m@XHRLAD{MA*T7iS{pCF)+?vkDTr=fZaA#jjO@zuhaJV$bP>`qyNwVQD>jR!82nN
zA{d8W+`pArQV0-Ia25kC0oh0_0ahXQOtZG$X*>61WFwS(x?USfUy|c0OBpQcHAFYQ
zz?>rLs@$aYTq90_gn4kxy6C;eFx^0*JP)amLa&NV#t$@yD?0P6G_sCcUpl#b{_Vrl
z)ho8#-VVj^vH#KDcLznaE&U225(OkFC?Fz81_4nrN-zK;8Oef@qvY&}NDfMrEXW`k
z$;^->Dp45886?9Hhdl7s;5qkr&pqe9SM}<v`u;f7E^5#0wR@%RUfsQZy~aFuLUngA
zR!y9VO8*wdU;dD{@xAXnbl*tHD*;mTR=$*4E_x@CRqOpoG*aH&zGv1#bqD3bP7};%
zNP)-lo%mCeirE8km7CnJaYF1GO_5FaotW#6O-QZS+Stf6T|O6$kx>OD)dvZmS$pt(
zG$bYaD2C|eqHZFO;8a|9cF{Vw67<k+P+w5wCb>n@_Gxw9TbJZL>#itfe_`i_ol+_E
z6CK$}w}52t5|L+2C~7>S)T9_D>%t$y<EBGC%-aa+jV&aiJdYc^ovWvk7xJp&>>6uZ
z%$&ZH&B0>Rjr@)!3uUtO3c}Cis0rmsTPbbz+mXjtG)7t&zdyp!nJisDT6?z`%3$L!
zp?!GYXXfP!gDr*a2KiOoPuCbY_4Ktv^?qpGzSt$M5hSNWw}Vle7`tqsE$)!0u!M&0
zk8SN)_!CtGSM7FAcEoPd-yvwnEJ<df4ovzfUQ#EaUmd;P^|m{ZE3IFSV}$t2lJ~c@
zWCFnao;#-%(y;svZ$%$Caahl-Vs%{+AxeSrPrxSD+X)!E^<gt%;a&~#03oOq`CEYA
z@w8YWxiJa<>K0x&gX@Re1uuw{bB320S)Rq0a1dOZng(ZLXzSsI7qx3kDO82FB-*w+
z_<uPioyQ70-R${cd}xMJ|4Cd;5Vb{wTEs)^eQrEF0yBc_h2?jCVsS)46H)s@7mJ?u
zkGw3-5qWR*;kgjq1Ee(e#16(+>F-z7-grsocayn!8ZE5E7}F5Fe2Rbd%IpOhvH(Uy
zdmpILTMlcHJ;hBqnYBkrltcUZ8sj(Amu?4<%2b@zBzQtD_2!WhM<}O;pzOF2`bus*
zl@+jqR`#A`gqW+#2igzFzbx;wR%EGh+B8|lSvig!U%BoiO7eX9>Nfp8wO4@YYO*|0
z_P{y!cKpWLrCV+p!5?1*`(7EDlrVvPofIi1%;8mBI7*&=?pk887`GAios5o~2e;;K
z>F!&t48n2QicEv=HN3kFSD$oKH>z&M)KTSdEsfg-WCYMd=kRg&BefOIi9h*7snFfD
z85G6pC5OhmNDv*_K)=$8E%3P|Gfp}TW=_;9dh-(#-Lk|>qnlX3+^!^6V3!rne*>82
zZ}-<uAeo-nOoss1$#Z5*TzPRfRcd<0?D=+dA>;{s$A^3FtM@a?br9{Vz&1$@G2e?n
z?>jN$i#M64(x4Ils9%xQ(e>d??xbn;iUuL_iRP009F0Keh4*Z(%%2<1j@hFY8b4QC
zbf0zYzRi|9a2jXQjr+yNH*=T7NUnGq>L1p|8u^A9%_RA1UxFitvSDUthxiu#BQ3_N
zHmf^qj34}Xb&gb5qoL{fKtbtBfySrHAM)@W?GEtG`7LUqKI#rDw~sMuwxz!QqzVlc
zL-A!08&OsblIygKOMkhDEDEDBwoSL7?KJH+;t^aBR(7@|VRH4mQ*rn9n#SHf5^C|-
zZf-z|d~IWvm&K{1Nr8bEBUS`wDY>X`jNMJY6;|P^Wq2C;@JbIn)7CUBdZ9W#=UTJ8
zZ3y+-q)qJBa_{Nn4_YMkIvXzWiRB)9&usvJB8XDsaS!Dyb+LoBs8@42%0nNyiTHhF
zH2wL4#SDoH280nIe4g8*tEYKy6XbS|yAVLpBIlI5{2aUk)*SpjuOmuj&B+rWkPD2@
zM~?%(+aUXA279~wr?HMunERe8KB=>h89y%;*E!HI9FMv8`|&SWXy^-Dy!J!<5<k1l
zX=zVESHu%IgLmUNW@36#Rnm4V(A<N(zG$X1M@Hf>t{+($DSs{WW(GBbKH)Wx)Uw^C
zG-K^IU{;g+fT(~!fQTT3u;9woCv5MZqc;z3=DmTE>OVrP=QU@~32hQWKC9NQ#Om%b
zDC5R&iZjX77xPup`0x%8a-M(hQz+x=$v^i}59Tcy;JnuUG9X}XI=_m{-L-weElJXz
zw2=$%@)=r#<TG<ir$@J4;&qv3CPn?Xll$?k8u4H4Wb=06$jlcH6^AeNJZn$WFs|RS
z_o1vwH{*rfocq3icKl%X{3~nMohv@@_Q%&^X<Yktj&@O$?bE=FG>l;k@c_qP;>(Wt
z0`P7;)}{!D9U19VgT%{;B(koPQ4+<XF7m){7BM*w9_t4qg{xg0DtA7yWgLuzzV4`3
z8*(Ae)MQSr%}Jvhi@<`m?<fJ}=jDwDdyh|%I!i+NMZq%nof4b$flj)23Y)`o>3YQp
zlvUzq4$mf`^_0mStMdE;DvdFH#j}c$MZIH=S>#3@9gJhLOlzNw$vLehv?`(=GGGzD
zTHz-Nz<H_v1+#Y^Ot1LQWI9=?abppq9lE9;A8%6A65215uX3x$J91o?gbg#3z1`%<
zT|9LRAn}D<Cpay-yV#uP|2r5oIHE?rs&vqIsEy1(!GW+ae9;)^Rm;QZ*g1V$j*RT@
zfe|#H;sK%(doG6ZNv`tWzzctEs@Ayi%n)&Acw4QH-g)av^{*fE*5cW9lu~FE0}5e4
z+zdTV2ulAqyy7ne2h&#v0UQ}hfW_mLX$zSDdX_YR0XjA38th_o9vCfS0G`~C>}P7!
z_Pj?K_N+TxZ1!SapTd7TL=7xr1HLDq$|Ujq<UF7_0jU1uFpx-Ws^=>-$b={q!KzXX
z)AoJ8aFAEG_Q~Wv7@6&#8KS1)HKv>Zc*%Tna=JywhyS-}{h#Za{BIC*XJ(L)pj7>}
z42u+|!Vx`mO}C>}4!^_+j;1%n!NQBVU%XKV9mZbJk05=4#{wQtian8l4(G_V__xEH
zK~Mj<47mXmJhzuImlOyE6&4RFo74gO4XFMTe2&#jgrJmy)bus%i{vEW!%OKmJHOTR
zvp($qgYyLOcBTe0Ru_K<uK$a|?>Dcoi79IWrh$Jn;n(JX`v@kGC_u<;gn!bFKZyVF
zkw6lZm^AG?kIhi{%Zo`GH1WJlBgx;^LhcPp)ORIb`*T-+Z)k+nOKbu{K)}Vnt^%7~
za0p8HrQCjal8Nz0`~DagC>{XUKP0CR05kU2m((sOK`iMI^LvWTf3}zqn<~V_Z1M-$
zKMe|WjS|e8_bk_&zZs4fc;`WW8jrM0`=7@8myeTdpu}POW9GkUC^&`x9!Q~j84w5k
z+gj=&pv3OGVwt}hi<3s*ivZJ)N5OykZ)^GgXAtOdsSEG7z9@&&b^f{>>@06>;McVm
z<nGhTWJU4t$ZBTG-uYPSI!et>lqDPdmZ+3j`MgT&11k0hJ3W3x)CDZj?ZU@<eF7${
z0v@XlogSTu4zn0>%jy+@a?{y8aYXgb;anxTnSs<qzL$e3i`0u#IJkHO<V^p%;Zu>C
zuwi*~`Pc7$dsxEDWY=F)DRBOe(!alUft|@t`0bZ-|3>QP!y#Jo3d`}4i-f;a^4n`6
zROA)(_n)5q_ZFNyTp=bHDYZp~`~^WdB7%`nmZH;tYB+cex_t%@H6-jJ_ZI}Of=6=a
z2cduNuha{v16*8}l?2hNe?jmKcyu*z`M37|+JjUt94KPDBk9e560Exi9=(xq{?B1z
z`d0IzPVZp06XYBIxi-TOFX~3`F^m3M$#13pnXX?Z;GgOGm2>`)uHWQEfYYu+sQ>A~
zaxtW==K<$4kCjr*vW}a>?aOCwi{jws5deKszs&OQ`owSpH&*xg{j#sD2~ih4cN-Wx
z&)xnJ@zr$p9xmQ&2$09bEDqWSCsIhGTZh~~BOLWp?Zgp6{lx8d=F`JXTt=NVx4~IX
zq8{zW3<8wud>M0|YoE^+?`_VRw(%gAqh>jaQG^5|P6bNQ0Vh&VLZvm}vqkpm(@$m8
z{v7JJCwd5Ib)J*homiGU-h()I%hEneu&vqSD#)9REuV?eF_GN+3MtBzKxYkl?TiY0
zn(ENH12yA!#C_^#`&a=Ld7tmanP_4pLhmt9x8Jx=2p2Y-Ym?ahOcq1!{`m~;Q(m9F
zwmHWcn~KTTg9YB`9ILO_V!3oiI~M9IcFlpVPH>(IOpy|X1tuu=Qh*$fwtV<vwpHM@
zm@i7AD<xU2&Uf|_-DcknX2j|bxwZ#!=EqfIR%~rYOs)aGY`$8^cRasS;w`pHet?~s
zIxA|exY~v@D3c{bk5lOtu!6ka!0Y<N=febvuxND&Uq6qk+U&FfbJw0<y*<R<<s){n
z2AyZyQFA!MD~y7#H`AN0A4P!Wi0?7BJI&!oUu!Hc$zwZhO{D8O={Hyo$}cz5+qNek
z{$Mj*Dya1?8+9IjMkcma_lh+$z_VsEY4xe_xTnMRS_7FWXF=8T?QG3zj$!l6q#{|G
z$4_--;ghfFizp#t{K5V6J(Y(WZKm|rCCX(xV;-@gm+_yS&->~Pen-!=jjLFpY}f`7
z-Ey?sBH1|=Z&Qu#TYlH&X|nw`wu;4LGhsYYe3B4i);>F69L|ksZf+Lnymb|^I~Cl9
zNxRapm{Tz3prA0co<>{!(NqsvPv*VmyPW9I`PHU!CT6+z7-MU`5#L|MI_x}beKstC
zcE?Y0zdO8jIY!@o!75yQ+pJY282>`~sIyKbcPmZaq6!?dJ=_Mr*T#6f+foePUmc`O
z*=<|~{e<M?^o#F|2zU%3lQp;@GmPsJB1gN^#bRB>!qp3kg96FxA<OWk{5J1>WJ-(^
z*f;3%LODE)EPP`El~R7(!$o;rVq#+|Px*+`ggE}0rZ6T}HAE+3+`i?q+q$#}hHXi-
z;dvXkCY%<EsNr7WPWIgRy0<-SJHA=4Ys6iU1H{ecN~g>VDYi2-v#W6s?~Tab9n0<x
zlg@rydta59%y|=mxwf=ZSb<zS#ywamH$dw}qX?(b>)}40NBjM?HA6R-B|Z`K%*Zlj
z50xiF4lj>0%C4EP?8!@PygB?7?qy{x0rA;KZZgLS^M#jJuUOC6%NFwSx<}1$<`&*D
zmaxn#)zDYy`4-nT70f&4s^H^hw3lOmu`p&Ze->3vfFf;QjV|zfcJ$?H_=iys{o^%%
z$se+(N=3>DrqEX5e&dI;!ftTW;ME!CTZ6?Aijv$gTlmqDlWauE+-{a`#d1+wwk3P>
zU>lkE);B2Ha{_u+X(nq%=spfND(LvGiH>%h&fRuLVc6iesr&_umZ$kM{-p>S7e$di
z3Z`!?<d72aysjINrOY{VFQiAO2-axP;)53WV3yP^^O+4b%I*axOM#3QH|LpX%^2<v
zJr{%O^qjw3@&C}2+>hBAmlQ=+Ot(7m?|n8&u+}ImJU|!tBvF&Hw4!ZY8|Yi#L-9L0
z_d{(xuMPu^(dHGL`sDUS8H(ICBdyE}+SgTdhq0=_Dk~#~BX;RH%iwOwrQkky{!7Xs
zinQGDGzIpi0VlvrY&}M3y8hkR+QSUpFd9qQQ+bCTNltxVlQ*I=)ef2%?HMmFC8$tv
zesvyv7Q5l1Y!$BYqWcbSUkkac%ea?|o0je-ZS8IojH`%J>DHBD@LijAPCnei8QrUC
z8q5PXtGqNlC&$^S28&b@VGpKpKs#d4{ifZ;92v~tVn=<BG0&>4;0rKwr#-qnt}cm#
z<zy$-O7grbs+M&hksheZOe^#SCra)WBaiQvA6$KO?VG4iTW4;-e+bDKzQvLBS4@A=
zM}9m$WXGy~xBWF3f7`q!8l7BEj<hDd$S@~7f-PcK?PEy_$kMUtBrkRr;;M0lM;%pr
zvvcMp-qDhcyYp-xZgzF0Y<*x;=gz<{tvKk3tx;Pk24sH7KFV&LGdSMYtXaxOXbgwn
znhLPc!GxQ=QOC<@#UyVg&PC;YT^yF&eAlJsCIfj7i_#vUejo6BoXl(fu3MsH2IJA1
zzdA#Ad?CTB48_m_;vNeL?Wl~U0U&6bwLr7>-VQ=#i|0`WpVBMMI}T4nJdcJ@SmoaI
z*)C4W>#N-^Ey(L5J1}O|>FC!P@q1X}ez;=mGwe_~Y*nnN4!asIR%Dvsw|$1Wm8;_h
z4pbD1I4b{8r&k;B3>OcdF!6aSdoNku{FsVZQ%5jc<DUCcUMwv{F~@rSuJyO@qc4af
zUxcf63axSCkEXa=;(B^PY?F8Wg|2I#Q=1EBoGEZ^;@p@Dbg9`MveaBMHha0-JrcFp
z|41y4H=I4WFZM2>RpDoRbq3{_eX@CX{qDT^3|=quJzg7$9<rUOEYDI2YhG=E?_?``
zqOWZ~x+rrb_sy+O`}kjW{t~O_JjgLVW$x8wkM=VXduLG#?jilD7g*EuVc-_?ZDZBw
z@LjT{(x`Kqr4q+o_s|65K<olL8&)SezSp@ZI+f~s&Z26KU^Zj*6Kr+O2k3A&xb1cA
z&oi=?rTlv`vE4?Vl{Zq1maF)Ojb#OC?#%_hc%nK4#-Pn>s$4FyT(Y;eVf<ihXWN!1
z)?f$TuRa%OQ`yk-^t20h)$b)0q%)_=8);6Q1ww1dP|KUvu47%=8@FOE`$Ij%UK2%I
zYw3M_Fh185=d3Oz<V{AE*WZJ>Yhe0t!X{c>!`w3^(W&pT`sgLQI-I+0M$=os=~1j*
zuM?p1m7Hy=i{FdUv3a)&mPfa4v0LE@uLJQ>%qhF6;r1kHw%#MDP5$m9cp<{niY}Ww
zK=;o$RV_c`Z3{N(-s5k%gxvkC(It?EPB|Ev+o-r5?6_LJGn<+0ZlN!Dx-%FG*-Q(x
z(-%Z`GP=*d-+v<R)8fV)zDp|{xA2hJs%L+TZjL@`3YKlwm(b$x&i}=n!GIN*BX0U6
z4DmjTsws2=ol>e27;aSk%x225r&}@IOdx%cw1GZ*?PjNF0UYH6x^3>%wZ!Ya9&C}~
zb2!Z?J{(WeL%SxUGt%UdI`5uD>zgu*HPZHEsf3-H42#^v*n@dfl(Q~8Ers`e^f!7j
zyclsy!dtNMK)6?DWZ)qlL3_xHOmuQzSG#Bz*1VB2ls+@Xz=cWXVa+gWxB2d6Bwi?d
zeV6uP?W)%*XBRd1!9D<hB2$jk#OFETJ~#E{bGhKpHTvmy=rLIg74(8M`d%aD6{hqt
ztn?K*<x(LY_0r}kUfb+#stVuv%wnHo7BW-xqY?8jN08bDal1zIpU|Pj*X(HdU%=EH
zqw`uX-t1`yzGz28t&PI5Z%Vd}0<Y&JaSmDRR)(fe=<JyMWvB1QN+WBUz*6k6^%meY
zxn<GrpcNdnADJ%g8zg7QucsrHaWNWHAN<)6=@ze_UMg2|>S9dVf+J?Ysb8;rL>$-E
zchN@6yNrT~&kHxX&qG_%YnA6VCDyWXBx-z0ww@rWT}k#J9oL28)&1JxVE#q?XR1m+
zBDN1kSH1g-$FYpT5&mRHj3VTr8`X3@`UeH^G~tNs!;L6;!HEmlt_RC;FI-ZLeDJ{m
zYYOCVb2d*r%%L|3ey5e=r8?G-8-L^73KZdvAsP|tHAE}{4;#(X1*2Ex-B)um4q_cx
z|NNLlO}nftm~6r9{dPr*c>ca&sErZ$ncdVVT$eYHMU}FtAkNwKNnHh9<r&IhW8K#<
z!312F?kWC!A#Z}J{oaDk>AS+XiRi)i^_dqwmStU^f!|_<m&=ytgKc~KX+k}&mfHCn
zF-JTw3C6fsQal2Mf)h@Ftcoh2um%L*W!&!IYnT|Ul_t4H1xBO0fpvzmws-m{W5EFL
zEYcf;Q&ZqM!`WLlkh!p{6$>A3mQa+=S;s@A4coxOX22?ISZUk<1lyi0s*@s=20xk&
zPLqjweP(w`JI!8>uKsFRpoaVOyA0tlMf0+=C?nR<A6M*REv#qt&!Du+`a{k4;rtk2
zy0oX?xAmc&oDbTEl6j`?PXtP9pUrTe;K2tR(b;`QSfz>qxCiPI{26<Pdz&p!g}_K)
zZBzYr8@1>wdR8P~=nM)M8g4ab(M6vVT`sgZzCe5>EYv<b&Exq8!M>g<w@tZc&wxjR
zWSKCp&O{@(`rdM5Jwk<aDFu=U@7NaCOy8c5b$S>HmKotibR#8X`L-SjrQl-R(TXB8
z33Pd?G3!m!+lM@Z4wo^+k)$KNj27DlJE=4C_+nht0`;@}&3Ymdbh3xQsiAlN5@(I;
zVtg<60yBv4;F*ftTxb>)AB~giRmU6cQGH+W>2j3H1P}a6a~v^-J^90TL`t%K^F<8e
z2b#D29g0JAcFKa2!FzWyyH}UwoFqr@X_xhDr9&1u=n!Zen5K4JW+@O#fh-k|AosV&
zU4BMJh<V6KNma<NyBCpQSsQ(JbJZQ3Rct@cOdzyu5Mr(g>Bi;Ndl-@;u{=<+lAV*E
zZK`}Eo?NG=pcLIH#ASH#qS8$}%@i$;=2@kFv1K9FoC{IdVHeYug)M<!j=|}3ZRUaO
z5sh3UESRPML8$N^DwJyYF1@28*YZ=zh?ZGFXGMk92nnsTiga&qT^ev7`(Ju~FB+m6
zVUdrAx^q3$fVTL0=mQKCVK%A-$;#;Z$iKPRvYT)Unnk%*BGD$Emjd63OvDD!QuUyw
z;!BTv#pRG(dG<n}JN$!B;NyXW*I6TZs@Cc`!V6x!<~?<aJnp`Xw3_10!KXJKN412o
z*|`9pNNsM&x_tC?vSrau;HviyBFCZehbM9TIpErG)MD56#0xjx373jpa+H7~q;K>I
z14mOROJN#LHNrP<Ca?kevQ*hbJ1R@7r`Bmh&j{l$bv?`pZzyAeA|fFYmHUNZgANBs
zE0U2WWNw_-EtBI=L*pBPsJeAof$qA|kDrYX6uBsw_%J~Cm{tOAJC<<3KQWOQpzc?F
ziFqng*ID?Ror%uNE`2&uOCj89;`O#ap0($0@lqL~Z*((CtO^VpjZwM*CZ~NPzSUv=
zy+7<Oy}?o_fzjKlb=dw>`eD#RS2D{HUiYUN_TR3q(cBemB9{TKvz2;Sai>Vm`oK{%
zy-mdvH-)XOAPrAz7<iNmACoR_rJK(0pu6bB14VSpsh0WMC?NYo**Vzb-C8K4;}V&@
za7f(sBI@))3?eo{@6+!it%ycOfzL7%qQov0n~yklN`oQ_yjCnRO=Su%>bS_)O2&};
zkJoNG5}qg(La1Q3{-Ki3XJY*b$&D{p#Vbd=4_4elJ|%g#2%kY2xuy#BX{8U%88aSe
zK^V-h(1|4a9POFgw=e08ZNzYdgMQug0sU=W-UQD5>|C;k4MTw<Sb98KM926|+E>i9
z$VVTly1iCpy0DkxzL?cp<FWin9?4GM-{fV0+3Eh&!Z2K6fH(0vRMiD6-!YUrjp*iG
zXY`;eL=v;LIAr7sd!lEJ{OSTM@5MOSWNeQ&a-G9Y_;NNfA+v6DYk&JkK0R{F$r!#;
zHoOt7p`EqArsKKZ)b?q1Y%_W1$2!Ygz}2uXng(8Ii-VUyG29jL`rH^9b0nw(L~~2b
z<ko|JI=wH>qfY&Yz?8~XFw)2vkN|v9$h+@N)`mw=!givAR;q^+Pb@E(3KOpULb=<a
zgwHhnxEB6fW<!^)ZuvQ^y^lsnlz~y3a?eMg;8hZqR$Jsjk5$qszJfqPJfqWh<(YwT
z{`d8%F<uk{IzAY5&ZaeRirw%sr?id45c5!~&rKC!^U1?ei%`UX?Oc*eXV>kyL@`#X
zkQmE_-GQ6g306NCBT?K%UzlfZrz_tHY1)~SJg(%@{6Q<Q*yn4Gwha2ZHFd<-NzFaW
ze2?xKS>!{W$`4$3<7Ai0<6P6&o1S{xdmdUAje|MPSplIARaF}CrNzh>xuCr>PK_Q+
ztrtUCwnrSg_&*OPlh7DSPuSqBnan?eB4{R^jN@x;6xP3A^NMJ`^->XXy9}r}i<C2j
zK(109^GAW#(=KXOhKDK}G)Y(#aT$2xFP3b9&fN0jvg_wHepfd_wN`S(c9*jgx&K}a
z04BitPD1=0Q4c!^-M7T-c&V;k*YhTMqnd&)NtVZ~n*HFrE^7;m+sdl(L5_FN6tknx
zqT<6so$SL1v_jdMD!hC*0?|5FpPK5dI>k2I#a>1?goUT-t1@jwsVZfKd4T!vAHZD?
z6GeSNKJ~H>1HPB}L}}w|vnDMv{hNCIv^8sf@7)xf)9Xf`&X}NXK@l-4OC64+A|f<C
zMaCn&h!(je1WZsJig3k!E1{JxWZ)A|3PtQP-!o9)Z*o`t#*?ozkzc#bMHx2?bO%|1
zBxK3|VBjQ{CctYqV&8(^iVr*ptLWdGmzUHI%c7}WF&-_f(~Dw$`GUVw4{m$EfZ|*5
z>vOaQp6g%g<;arF*kiTf8Dh+J)B-upyMx{0<?LWF>F(iCY>wCn=&z`0<E^b)SZj9b
z_PV7_(s%kPd+ofl$L^%RFhi_{_rdUZXa8f<Bz{)@nDIUx+tGm`8*n7Vp2|aRbq2~E
z&F(w4k3?qUjI*oK%T+Ila*YC)*-cRD{`71{QKRKnT0qQO<Qu?wt_DKajDd=69}&S0
z$+S=1mgJ9A*jF?gw<6;g{L3qOC1&EW3h8bRB<F{sU>tUuw)mrhyJlXk>qiT{NT*(3
z4`19+^Kn-_FgL9?Y^v6q(Y>dS!&v9PvNS)w5QT@r|EdKsL=k<jz1uS<h3Q4QLzd=H
zh~a2p^BGQ(2Ye;Uub2utM2ur>yg^vt+$DJ%F}m(*Ay!l{jcIoWdeWbfq3enG0pL^d
zXTrl@uEK}MfDbJ`oInGb5A^8!k!u^bfm6|%I;AK)`rUD$qOS=v-m7Vl<g@BA&WWfR
z1OCtH(*cSX<sNZpUwge;+OIEat*4UEIa-3MIY6~dGf1+I5{%ouCHM~ZR2J#D5A%SM
zaWu{@e;UGnZeLX_@7^?BBdHY$&rlR_U@C7+P~+6aK|e}1-SW%2Hk=mBqpcLsUE76o
z_R^y>9r4XCgCFZH33mglG-D}f{nC`Lms-%oGhG7_2vUobUrE;D4&%%U<EirFOXzor
z@vXhJwZL^XQ3B!E&)a~r%0Bs>L{$}{rBo+uvNvMLpni&AduxUc7-hG1pCb5IDyGBK
zBP@zLZgQqn$*gf`CpQh8qKJDSo$D&TQi7!8WjXW3D22d6)?w+wgb=P5#e6o|mWY{L
znpO%>7DJsN<gk&YR`PPtW3@WR+B^koH7+<0N#g5=&UUP`u*O%&JFLS>7GMb)Ei%Lj
z1H$n|EtD{V`ixUgTV1{>D0N%#C3mhMR%_I~TNr&jD|!0J7GDoLDq_Px$-t0p$P>?F
zL>Y6Q#PzkvK~(olw*wbcAKd7NYs5Gj({+lyla{vyypk<yP9)RYViaJ1+Ndn8gh&7C
z{Pz@Ke>LAWqqAh1xjg#pY{rE)CU_>y>%bruL=B(k27q8J6hQ#bu!OYq8YDEo4sbFj
z2`RA^4<DdmHP?x{jjz4}Nyk^bOog}7)p7$gf|X-J{#d4I8FCU>Ht~M}(6`ED`DyfE
zGfY4&9L`VrI>kyDX69E0SVbNBkt#wPc)i&+1>K55@CdoK%DB4sk{NZQK#|6_Ok`lA
zrVww6-Jq;W`A|{v{-Qo@OY|lBxlsG<v?iSpvN!p*pn(;g7d8!>s5NuL9<~$|))Vbj
z&Ls~UL_NRVY@=lyD&!t^-}h^?7VVqajs(pz9sseAZkste-ip1NBwS52f)0MnDrK0B
zcQ~KIaewb*O91M8#?!xiX~k-k##Mo;paLh>L@IS27D+wISIle|Ni7{|wNl#O8H5Us
zguhM8P}9JBRb+HnoJ*kgOtz4f6Yj2by&z50Ov6#V0~?{vB`|!}q#!+b{vj8?U^q7V
z#`!X0MxrS-eGr6olVv5E46_H*!zHx!+kibPA{Z`BA6%!cL87g#ugdT6sH~ME=c()j
z2PZDX_gtWqqXtQxHk^jLwJ0seLIZEXeVbRiF?v{d$Sge>?MoB%S*bY9NnfLVHY%@j
zHvTb1JAH{|*Zy@MSDHMW;<RWzH*VS3(PS@H_#-@_Vi57Jxj`7dcG*P5F}x#<#9o)b
z(?A1HxY#HrJy}&gsY!y~s!+PiofsS1x(|h|#=Wazm}C99r`F6yK0e8oXm>|L@Wz**
zX1p1%k=?KoS{z&qm~9Mm3>(V?%;Ju&GqyOCkYG3L-KiM8lfB$JuJf)S=2--yU{*b(
zswT?Hx<t^6*d2s*j<{i3@S073N)8)#4}fN^#QSD{nB~>q_wmf`0tqFZIhS&GBEKOz
z#|+!DC%_9Zt-wu8J`rcEf8PxBi%AvH?DiSpWv}czps{Az?<BBG<T5p=t=dU|Enh(x
zc3WAI=k4$Bp0F+;>I-_|P}pOFytdRVm|)RR`fmh4X}$C06-z4C9K<Km_bYWNQ#?GB
zh-x&^o!^N4@$lO<Ae%hBu%bUx!y|B%13eQ^&tm#hTmPx**SY&=0I^QgPlo!31OFw`
h|FFgXKDIa_d+;N1L7DoJ@hR|kUsg#b|Bm6y{{!c{5E1|Y

diff --git a/docs/figures/arch_5.png b/docs/figures/arch_5.png
deleted file mode 100644
index 035af8a176b9f84993aafc9af7f8dc06ed4ad816..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 76885
zcmeGEcRZHw`v;D5yA=|5gd`!O?7e4^k&KM&QL^{m+%!;>>`mD_B%3>xvLkyJviIKK
z^TO@*dVk*U-#@><zJGi@9&&fNuJbz2<2=Umcpk?oNLf+lBHmRz3=E8ma<Wn?7#P?v
z3=CKn&N*;}Wyk+J1_oZSxum4BoTMbRvV*OOxs@>nhHOw|^m%1fEi&YiCwt;E4BShh
zL>|Ox#1i+W-)%84(cmKP1^T5tmLb(JWDN{NUers!RZc3=^Ael+B{s{nKMP*d(&NuB
znvRe(R<TD%Q{uLZLraOSi-J=l6{dehFm5>&XeJOaV}#N)MLpXo=Qesg9<zNLiM5Ih
z6vK&*rZ)+A2FFm#n5t_EV8c*oacyPrI#}EDCAnKm{pvi%Wq*eo;;*9pR_&dt{|3?b
zV^Hrss96lNT7etfj%B+|ELCCZAD-#<#Gm*k`P0~sK{#s?TG$w{*ycz&EQub;jh1Wr
zT(bf5LB-Dl*)P8fuEmND4!$qeWTtEQY#{JK(sKK~{qW?8y6yWBI};A!#v-NBax`vu
zC0}QTjE|=5_iY<xqZPk%NSqhE`-kF18~%FB^)%z%X3dBzmql9d9Ht%6Wy*TfX2wQO
zUgB@{D#)y}vi49$@!Z6%5#$o@PY5(8ed7LHsf3REJ}I-@pU+7q1y2|Xcvw|3lrGsl
zIdoLL@jg>f<_ZZ<tCsoiLvFt7Ts2Zx8JZ2KSM8I7=2!<krF)V$nJ0eqm}E+AH*K~h
zHX2YrZr^Ts`F65_%*~^<PpWDDTmNFQ=AU&$Yz_6WOh$VXZghUDH2Xx$0HU8f!{Ojg
z2DuxI9f1_PkHF1Cihh8zAn+tU8P0sA#J(Q;$C&7m!#%$6kcY9z#!D+LQylp$c~%%P
zS^e3}D{NN9q%;k&<a!L+l6iz?iB_>4^ii;%x){i&RPw|THjJzt44%@Jw6FxmVJSgO
z7HUkU6fE@=TqnOv9Fl7#Sc<5Nt$l>>r#OfUq_FF<nB)?$C?ts?>n)7ChTMJUm{2w!
z2)mG9#%`IN%S5@`k?Wo>`|Q1ZW!)ch87~+~Xo&AmEh6P-7Rr={JtLVBO7hiTmzw<+
z2Hw-h@8Gw{iEgWyVSglG^1qjYXU48};Q*HStW8qmo#Hf2*ZGULZQrqTz``F1<zZ5w
z93PRkyg2xc<A#}l&;G*kx0f5OE`i3EfH8(`5iiPG!mawku?KK!2OO*p=@ta(A3r#)
zy)4m<+Yf1;%aW^v@KEQ=3OkU^+<o!3{mQi$FCIN^;ZVF{aG&l)fL5^9bH)ZWLkkXt
zm{&1XF=8z^e}$v`)xXC*rYyQ<h0jM`1=m5cez}fHLh09!d?zpzAF~rU*M=~Cv-a)V
z9~Jv%`)}om<<difx;G1`_+Ne>S6|6+Gxg#~dez=QHs(4;_`Chj(%AXsWE=-Lt3S!-
zc!Uh%(khPeGogC&ddm6>^;aJFWhpt)og?1)YW!8<>%rHDe^~!u*axS{a?uHf-;m8t
zD@}7y!I~k;4p>(NUovgC+0+zol}Q)qE(Q*fRESlyKcO$i?SFbrDJqOVj4-VIEmhd)
zgIkxeUQ*D%4pZot%}&=%5lQz7%WRBqcAVH*&7EYPjGtKgQI|oKy{pFOOv4+kEiEct
zbC>cVHPe+_Znx}69x|#iC8$k5yrQ1QsLgba+2j@}a}T4w`ZcwBX7XF^j4P3=k(-Qq
z%Cng?JbH2GLu7Ph29>I$C{o6!UpNtnM(Rt6-(O03bBmdA@RkfCyLy3|SpHD6#FcaS
z1hk1oLG}?H-?q8rjYvJC40~Q^T`XB_+0Ek8`KE9;MJ1~<%fVps6_aYWJg<WDD~pm@
zV&34mAnmLHqEbDJ=&l#0zf6T&|F*Ivcsp&<I%^I~iQf0F*cIELm`%MJcUA1_;}FWL
zWIQi;$>VI|vg6X@OnCx6nrX9X-^o|WFVSX?ZMx6Lov77T{o`84!`{vAD2t`;#h=%I
zadriDHU2X1a(^%1X*|O)6E*v3=JmAs4EAjP&)i<mG^H-nnapY1UfbRyGea|k^@ov^
ze#4*2-$IRIEH!QWjHEanc(tw6tex7gn6_{!a*vpM4Z`_~c;z^=c`A(Y;s;{!6kOw6
z+CI10$I!;9ie?T5TQ%@CbLR5Taht}RkC}?)kFQJ0<>$9qjH+00tkdb!PScq$bF<2R
zGGiKR+HbMb)AaSFsk*syd)(s%>zaYeZ@VEO*&!&C=vFM9IO~)Jk3qgcTbE52qN;OM
zG@hb+@I9tI8`4CQdL|P+dz4VQaGs06<Z^#U8|UCk);N`-iK4lpv*Ob={3+bO3(37p
zdE+h4BXdJ@^-gkj{o|c;ISWnB`7YjnGv*pxg~vAity>h0sUFw4l-Wg>C)*uO|DG=D
z98b9UXJ%Y$y8n0j1j&s0?~N{<@5236Bc5Ns_n20+3mW7V)-92a+Ppp&@I8Anc)Ncu
zf4yp2X_IiPaM_pCkcb-B?BX-516bL)P58|DApf|hw>fUQSuTeozJz|GseAF~j^rI?
z$>ad77x_4rgqhFT@d_^-A%0&kV*F#D_8^U?nDymV5-N3y4bB1XO1esZ6PuQ%n7Ja0
zqWZol0WV#(A;0pD9sRYkR;s|V5ca?&ZNqZI8UNf1-J}`pA~k`PfmPF*Fg_ZgcWl3z
z`D1iiDHSH@9@4*F7I7PNFWk1-S2+wtX(``udpnV!7RD1ZpHR)~e(M`=I*YO9G)*3&
zR&GOKIB}PgRbatzz|F(S!{{6hEzNkEtXu>`WlVN#1y6~p6%!j`i_Jy1+Il7MtAPXc
zg(mIL?l1+J_*4oN+8M&m+H27XO$kc!@1w5i<dmy@TyNkCH>QZH`LWS<;4Ww)XzM<R
zw?Ja4v7Y~=aHtATJF+~-x^!`NzZ=WatEVfEK3_q{le_&!A>GnJ<+OAD&ea{u)ji+k
zop-eowGt{nYeviu7O6PJ@qFj|3~L^Jf3fs@zuxjL>0RTy2Cv+|_4*X=XYXgOgve0{
z^YvQ~l#-WeeXOv2H5U~}^et>C>@eR?C;nb+`t$UqGS&f?(v}RXEtgK0naz<^<Z2Y9
zhcMsLw)R^U`?}7W;-Y&s7LwiH6ZaEcT`CqV`tpma%(kLM>qQAScjv8h__Ejr0|ui@
zmUa%a4i>~qd<sNUTvDb{Wl!Hft>JydOJ=!XX*t(5$JhO3qjUG2<Vt`I)uzZ$r(w^B
zvbP<YPVj4Que<Na*R$)%9K88NX+&u$P~clV3ty0WMBMc;&pb^qy`qf%Q~FxkXyd1c
znLWgHaYr7@5u4dfznU`n6KN6~#kqa>w&te=S1Nw5HBRjfl9r0rA{>8@eEa!|GJ2m~
zuVUVN=Z<jI!92+nb7p~%zNmW$5NPG_Mh3+ve#N1WHSX`f6r|2X9ppz15|rr(PdO)5
zjC)M&>$O)!FWE2Z)loUsl`t32-FEeIUBWti`Z-{Ra8#tpooCZcf$&r2a!`GEaptU2
zt}pjteo=41mQ%9nXo%_W_Ul_~y9NhI8+*Ne`%@89#_S*Uy9l%|CS6^<UU}ef*zBkA
zN5kj~_L0iIMz>{Z@Ao6Sz3~RwmLDjJ#`u-8Z%b;UlwP)LwbndO)@K5O35}>Oh-drc
z?6m9-9n$YMzbc|PSTSg>BlopDSUjpR6*2PCZHctIse=ipv{NC%uqA{^1YsjxC@?zr
zvy(hjQnHVba<HSI%-uz~A}m7#evC`am^kIQsMmNO)9)Urj$`7yd+%`*Lr15%!Rbi2
zKUXQRo(5wRU+3HawuskZ`{?Uzu|p;IBkyy>`qrq?DPOFH0GvN-<KLG6h<$0SDQBXf
zfWZVl<6yusuVP?>Pnh6a43pyQvoz){46Nhp0R9G;W57@EQ3OAsuTby}J#+FCE9?cv
zdGJ3X@a>iYL*I=JOTj{ahGl`@Vcb@gl#>HLRgD~sjcpvwY@H^STRwmbxOTEyju;rE
z^w2k^oC@7C7{AwCP18wJ;l7ZOtu@=DC$^7`+1#w{pm8un+=ReKYh$NJ)Na;RHjYAW
zqBoB35CWf}%j`F(k8g3Z6uqITpiC`k>tIaH$Hu|NaYGD`nwna~;faZmij?%}<KTay
zH_V)z?1b3aU0q$-T)Ek79ZcCd1qB7!Ik?!lxLCm*td8zBPLJGJZ5(M&20@P_W$b9=
zU~cDRZfipgjr-`at+SKpjT_L5&c06OY3yeH&zo!<PnQK2$PWF6os*4&{cLRTs0egb
zNZH)Y*h)*v+#1LXyhDsnfJ5Z?{{Q;TKX3fklbZiL$;l<a_3uaj^{fB;sJf%EgQTrB
zc&C%tKl^q1@V|d~`k)9qwDf<8;zZ8JSAn3#@I=_pHcbpKc+J!b?Bg|aDJ3=V6KEOq
zg$aGWb@CJX?EKQHXTKW*Ljpri>b9C2=KL^rf|~l#_mxpnMoOYiYMfgshKMk{mv;k+
zRuM06l}~hb4zt|AxpxmQ5cbla`bJ>Nvn;2l)bx}(YjsoGb>oZ0RrB+TL$NzUy{}CF
zjClU>mZm!JXcNWc8%Gep5fT`f|MkzZxgUnVMDle5p#TgR4mI-s{DUN=#+eBCzgLfM
zLQ>%%;Jcrxf8G7BL7^Anz;-VF*Kp8-W9MOTKOc@CPM7|BHsC>j2_LNgo(Fhv+YjS*
z3Kc$=p49(Xh6Iv2boBqs{q(;+)Dnp4^H1-SUHV_^2g~FC|CaY(JMsVK-P>0(>`ap@
zHfg=8RrL6InnHXJJ;qdfa`aS^r{Xz+Yq!{|m;L2uQ<P8D(k=<;%^C|M?!Oi(j}uHO
z{II^ho{ob<xpr?+IQv2B3;6+F?_DSNy@fJ*Iyxy*c72wMS9$E@L@&QQ)ugsd)HqJB
zY199u)`qx#zrO^?kJfrS;|N&xu$cE3r^^p^7aE9@^I65f&!2DJ-&sqS3uj=<e^Y;|
zfYg(~02s^0710xXi)t4?V)5LZe_MBSsDnqsL`TB>FhV8$UPTt)%bIA^;lbWyXS)1u
z@%ig+5_p$yA*M*<TN!l9Y`RAyS#=(A88=;?5}_Kp$>BV!=Ds!T>ipdCZ;QP<j@f3O
z>(U69!-Om@9$r{s-Jy1p`(N_4%Y~=*Wk>_8?ZU-#3b&6f3N?yNa7}>Pwq_%0fB6fc
z{Vf;_w$4IJ;_#$7inUG9)LX!9Ifa0V`^gXLjoEI;hed46GT4rTpRB`GGZh=-ZEI(W
zJ=8w4*Q|%-=~k%pJ}9+N@Be77ny|Y#R6#q_nf`0^V0-)xSXQLK)Xvy-zYAn+vOz@j
z6RVTWLih<>#xFgY_$(egvP%?lesZw8A@ot>c5NKZ55H6C_YhIzeC%=idG$nW`wzz^
z8pDMoR0^54Rk@q5doADInT$4=*j^bQn9R;h3Ui+83BsF8@UGq*uq=MkNaQ{lsWGwE
zF7E2GqR6hq`q`@E{d4_8a!SgD^;ILXE5bNxqeq8(ODY-n$@l0lUAnZ{dJ6}(z+&T@
zY68pop!?jf%4NZ!(cq?>^X%6k;?*CtZ+=8FYtV^^=#F?U)ugdaT|4!kZonD+#F|zM
zI-R)dg)EH4SM0Sv;|cP#OYbFlZ))w`rIij274zPmZkK$4e`(`Bmnjw8FX&YRAxzk1
z{#N)+rB9<02(!%PJ_?eyD0TyVwOiFrGb-j?S#M<qxSXby-H#5o=z+$(H|fvOK)d!2
zuYsmJH~2@LDCk%jjOnW~cTm@AJ*QEF?AGG&QnB*q1|aT@&aq#aO7gqwGYh?0^u<iU
zD%?1ZQ*G8q`ZX2ad!Lfjvqo}LCV*Y2+HSmyx{)mGs#r(GVTjc4(ahK8eGtzY6ENVc
z-YTV5a@mLqm>${2eDyU!haZxJqICQ=LkM09o4w72Fm6-D2l8g!c@58Ae+eLLj^#F8
zcg7Xnv-|#<(qZkd0;A@KltZJacTqlzgJnv-`<sKdcb~t{@uuLn;bHG6OqY*#vA{4o
z=VuY@v$w$C7)JkUf1}r6lialNBCI`0l-aVkpqpFd6&ZU$bQn|aSAvZy<!0S(ZrtvY
z8{<^*{I)%Hv0?P`_X>0?{v;fNCntOIbvBonGz#+aBXYI>=9#o)_h*z^bQ5iORXrn^
zJZR+~o!-ol3zt@lPxf3C+?)Tty>%;`R_67OgRRj*(@91$-YA_<RvmT80aQMkTY`t)
zZQDb&-kv+tsc+W%vP>;AfGc|UxZ6P0!s=*$O(`fS=#sI;p?z9Ps?3{v9q;dMuoB-?
zj8IGz5_Aymy>SwCkh$2wzGHvSLX!xSp-6L!$v*d+R-8dA>6kD>onVBrL23baeWXC|
zL~Fc8GKIL0SHe>FdD{7sQB%0MeH*{+$dHd>WPBeLt!9v8@-Bz%h={j$YaFk&f%irv
zt5HL+{i239r;AO^oY=;<LEF0g!u$uR(*9E+h7XdLM(cbN)_@dWtI9lHNaQq<PS{h~
z`~8i~pljnQca2p;3C6~rlGx5y{k)Z63D2rvj;=%Rd46e>WnHgT8JEZIOs2JT9~pSY
z(>S-ly0;+lpsLR{ID~P1M@jsEXR~*==F&j_U|JJLRKnb7$c_%@V2VDmkl6MZR{SNt
z?~Iaqmhe`-aq=MC;m7eDkNj&7yEP0-y@7u7(ITqne2mTE*0>|daqN+)76$Qr*4LOC
za6VW|YnpPyCaVT=nz--S4ZgTYag9lJICW9I&hg9`q9p2%CXDj=I!_%2V%gsB=TBwI
zYa*L{MnRiu*rW}Av34t;Um$C!Xv25Cb`hqIkzFsb@jb>nvzaf7YoJbbw9IB$NKx<a
zd2SpvYZYHLiJlLSB)toZ-UYGAKWA<r`0<x@v)2>buYu*Zx9>jna%Qqmaan83?~K9+
zey{q^uXV@BnHDya+qO{KEsuTX8?Sg)IdU(OssB~224cJ4G<haeqqI9F(wsYrbk3ng
zxuY-P8z~8iF?+k0V<GiV{6Zo;*q8cxD^{IPAIVF|oITYwEnBX1FNJ6Yx(Ih?YzJ^!
zc9^^fW{Ti78JrpQb5F+G4kmY6u`N08$8`B&9j<y=(c4!9&zd8dO$)gXm$nN0RhFOQ
z-BLFQHiZj0m2vL%jIi+%%<}i9$`HmEsf|YQT0ZnvJSey(*aDa$jl#tr*1E4wotO=x
zd@tE+NSRUnEcz8;&-Uc!Vg;PLc=hYyZo$@H{9He*^9_R0ozxsIGzxJ%9oEX2vJM15
zc)wp2J?k`K@hly6neXG=n=b=FuCI=NG>r#`XGx=wX<x-Tm!qXSfE>O#+udw3Kb<$H
z?o82a>)b7e?Z*_sV=AA6!QnENY2p$$CH>~Qw+6ooF+A>hYzcF#axU-#d`1=J3aw^p
z-pX|~eF1TAEm*dCre=E!ZMmb!29+q}E3F&2jJ*}Hi&h97rBj$Ro)0wqCEt|qTKY0?
z89+ceKPo)lDt@$YL`BT&*z@*_ri1m8<^G;pMFaPdRm1Aec4pYVh*cklo=K93dsy}B
z@Y4V~<^uxgR=g_hsptPn%<(0#Y{Y|kB(|I?B+<ed_4A|oFM*pP<nWxy{30x>l1HMR
zn;!$XH4624>v{@DuEJgBdKAh=@q1h7tH4q%1dH$*)R3gYYTUQGT11@>cczm0>x%1y
zfDE0lP}KJ|^F2jv*QE<O{_XqT>UKT#z+<ho+<tt1Dr8D3&R|Xh+~;R-r@1j#E&KhF
zf=%DG%w;+;tKrJ-5r%R1XL7paTWU|9q_y_>>5PKFJy?EIb*1s6`rjbdJ;l2#LujCr
zjiAQ4kA1xi?V}vwckbMAy7JD%lHOTlcRF=f$Vle5K(sp-j+(p#u;rtDM^Awg&#gs)
zXg5nlUxlOTFd_Fc7oUDnb|`l?wKBFB6GBD)39(R5o>pr6RCDxF!LK%aShv78pTjf{
zmnUv+<<=nL+s~>$%Csmf>X1)ZY9(wiq29C)@hzMQ>$@-DXfoN%US~x*710ZD%)khP
zQOVVSCLTWjqS0K~J>KgL3^xfPFCG1<aGV-wRx2?QJWX~`LI7*^kYCXDKh|U9C|(SO
z@Q`IHRw-QLInPS+py@YiW$XLz2XYR*R=$zhH}niXMFD3x5H04r{X4*8?LtZ63jYPp
zXvbVrtWOr*vjyTzUA#*n;$PZgZ;#bPCB|c|w{W*lZ+<o4vP#lue%ZR2L(REqUr4<S
zB2Mp?kT8yj({yN!LId0PqIjqFtp>Kwe2y9U{p_W|+Sp|cHeFlQ;_og55BocNc5zdu
zrlt~CK3hUhUJ^|Hx__qjDNx*28hcKr|MRn79A{+LBe5#$i63w6*-9QG+}5WbcnWX>
z)F-g8u}Q7$+%Uu=W$pitfOlLGTdwFS3Pi(!i!Gsy*%~Bf-^3Eg;SHm1^e&=Bqwy4_
zS*FqqE&xS8+>&QKdhPATgrnB3$1|7~#V72MWv@apm8F;0dQr2*Ugfv(9r?=e97$1m
zmEq*Ryr#6pNi^mue+V#7X>G*oQ+Lsb;D|QLA@>cUBxBS4uq0CT_}fc?Xe~~hU{TuT
zy`{Q0zw`}oVEontPY8FqY#RPbUnzPj(LGcD{KDK^SFvho1Yfqe#V5V_FT}S~D%3tb
z%-1f8f9&hKFu?0sXWANP43Nr-8X{|!&vYX@F;UeqOF=>H))miaqtXi>XahVJ-{Zjm
zjF|rq2L`}978lxmNp!*h1S$()wutV3_PUP(F+W_z-$rkO$l`Md^vyE;99t5vCJ^2C
zwo~sZv8VJHKoavC4Wk0}_%PrZlMEeOw9MsNAFq8a`23d})+!wz|6+g1_oyV>bf0l*
z`?g`KbjbRa#i!U*;L{jB1qBOwDrnKU#8}^N;@Yh7-rMY6U{y(zoiAKi>R!9vIvQSK
zS?t~!e%tWNi;K=xL<8vqR>`!57Nmv%Ejz^$iftPf+p0gZiDJ=O_ihTqI!$Hr=mAK~
zUrFIT@kiU~D3FXS$jhtCk{&6?lZot1gl{U}ICT7uN6M1bPXCeS?tCaGu*c_GOk<3C
zl}`J!#QP!;Sg65E1u)5>m)GK40uws`Ij)IweMPBanS7Cf0LD1_9338PQqyNITl5zj
zSK_OG|NQj)c*O5{!oLh;D^Df3x9=s3)wFt%8tzlu^?i6$nK*k@X3$(r{DVbzj;phr
zXk8or1kgG*{pvpbg1C(<N7ExURbqv(v2E@v3U9vhU9j)whN)9@vxxnY6y0L4_`WgM
zOI)Ld{UPUmDdYY9QS-<A-wyQ}M(@)Xd2U}nUH#Bakl2e^THHC|1N`XVUFWDxLOBgl
z7&R36mHRER+<cCol7tFj<=X*qTA6K=?a8AqJFCXt+O1}BBJK_erb+Z9@Ym$r4f;n-
zk<1I91GMofjhmyk)n{E|7Su~VyrvSjsWTb)Xuj#Ow=rjWG~$(N!WOp`_b3QYTWvY*
z4&9or%Iy?9{MNay&Q_pr(|^<jEf^huht<-{F(AGGGqSz<_!GSk!fkimd=QIJBxO)L
zMFMj$Nz`+G;lb@VymqypnF-@me_I+;QmErA_VPTgj{tmbP0sb?!`5gl7~8p3!Mw|=
zzHfXNBsT+UDcQSMNl8iTDz58Y&8v5CWH1Lnd&xrQj=}suX>*jIqe(p8XQc#zM|DyQ
zuzVN7hw(46YWX01$d=7NTb60!lj8~EVsck{g5)WULWqlid*CElptI8gB%a}Q&v{PI
z^3-}NlKEK7a1RBRvF)*^g!8@G3d8_ETlty{e<2jN5|36xxGfF|>VO1qp*M%z=8f2H
z7C{0?&K-NL_#CAT=Qp!48g(=BYBjcUkqn3LOd7JeT0PZ6%{F;eGw$<|#AG{ypa=GN
zFHh^mz`Ii5BsOE+nc&VnZ#5BFP5k^9IR`I^<EZt{4^^nPGjSI__?$8zTR${*$6NgI
zR~%>a1f$_f=gq_H*k~n@m1nFVA=(u#z1*+sAE%7SN?Os#)8yPcurSHk%{#ot%jY0B
ztIR#yC6MKkZLczJwrG`Kt12I*PECHaJP|C>9o3zyF~@fv4)2Q?t|4$V$q{60Qd?Ne
z^O67S+BA%xyQH49)B8wPHqaWR`&*R>YiJaJ2~YrGiYomn3ZP@Lo9U$2$y7}I1D9e7
z1zBP=tBA-Dsju7zt(SPlLQd0$qxYqTQ=_<ZVihfBQpt<6r>f|kvrKEVIa={=xAZcp
z=k$T#Ogi-t3g_tqBz^d=TRf9IW*Ot-N~8OV3_rv6f|;G)Fv$$3-pO(uahEB3^dpoe
zXt<zGLqA`;^yM#G^@s27Jh!6qUVjj_CFIcQb*Q_a{y8VO52!<>FF+OWR}UYl%%}7^
zsg(5nZBMpM_9H*~3vz;QJEB?3%K&xsa^D5GDnGHyVPrdC$4a6ksjpBj<ds(2K&j;d
zg<Z^6j`DqL<hwiL&j7xw>g_btuzCNW%tp`pef<+&qc#12xXG*}>?^|ist4-w$?94b
zJ%$bh8$*hYQ5n0IF=q~p5s?ySyq_$3{#16<hUAz=T7}nyoMeMhcY~GXgA+}^;V>-r
zi_vZ)j2ObMH1mJ!+fWO(TS(gaS|@Rk+F{^(kUpm|d4>B2oy_ZNE9}HZq;b_N4HT7n
z4P3gMQFmWn;s`2=&<YM=SWm99dAap`HnS);=t>FAw-{#zX*cqvmlNJRQIc~GHY3%H
zcVCf`(87s~9%`=IxU8w)R7$#R6kF)R?NUi%WIijZW4AG@Zfyy#9caYCT5^BG&8A$3
zZM2zkrTOVMoQkh8Br#*QpUIsCbAtgR62xir;K--jwkd}ER)f_KPg2`w^Sjrb>sDG6
z4)zq<3><R|H+pk&REI5D)}jEY5&Bj)>B$#y;WYSDtApU5{b$_dH25!T_-)@#sT2BX
zJBOP^J*as#|BI>-X~vZjig1#18EH2gq{SbLkSPZsXPvuq)Wbl=A94AcC|X6&0Ttz^
zR>eLQ^ESkZErn#WZ%ohkmpo=ki~?zHV9}Rik;CnAwwaOlcVDGxm6%kr8O0VQh0KM$
zymk51=_cF&PWQ4j-U(%df$2+3jk8{MX{+`yCw;@B!UN>fx&v-T;Y5`1tL@}Y=AxI;
zkRY23fXW&g`qTUYNre}4JEcf6=ZY}a-a8CE=`&V$nGa0XmV$EtoqI$flqnpuOpCHZ
z1cyr9pVJE|(*DC`S3}6dGT`+K+W$r4hI(Q8qb79h(YAS(yc`QgpBXHsR8r>4|D=Xx
zYcXycuI85${fjs}-3*Yf7m|R554=nLeJbMrd^?>1lA2@Zp)Q&O`j4BCy?DUoF8u7U
z`)9a+eF|fjK*V@47yM728)b=&1FLbFHWNTw%zwPN5fXAOiN+~%KSg6`Ehq&lr7!Hn
zbM1ev$Zz5T0?sh{lKFoMV_6$)N+G4tRrGfL$BHCY!K}h<zeWDXRrK54@`1MX5>Q^0
zIosniz5^jay#r>|SoY`MzeMo=|MntjVGnx0x~w5rOzT#rQu=PN1xgu^#jn%|!Th;{
z;KMgD%^mJZydY{w4r@q0f46Jv+CNiP1j2I?V#GcZo(~g(n1(aDhupVEzD0cW)xv<{
zC7ocSE0}&nq1pa$lUy3VeoJ!S*+6)PeI9<nKPoJ%47UF^Y3)B-C|e-WGRaEhXp;1G
zzedXHZFHEYPI18noJKakE&jKtYFk;uo~FjifLH%4oe0!k?)N>n{bxVIE`N|~*-<C*
zc8d9`yCc}hoaoHO(M(LFmR1P=>+$&(7fS1!3ISR%#`u9?eYm@v(6)=Pdo9tDevzoO
zuB5Nv=x+jyl7)V(R)(&8LdvM$PT~C0gadjHP<3EK4?Op0MA4&#T4CyZ9VK#3i7&!;
z^C@1kpM37;9xuQAbUe6KdePOGHzEYR-`l}pzuWQYDbY>>u6q;VcqkvVpBgDUAzqM6
zM^7K{Bs`R>BxJIlJ$=Gv@$AO^d%&6tU&Oba?EwK^-+Q`QL)#!Rv7XNq!=CQS?-q%4
zA3s}}Mj38Byl;)(J|r20V>%2w)XvtB4^)B9VT+l_(Y!_{SHavE?3SEt1<8%iu`1f|
zEgH1th_M1U@av2G6k7Iv5x9Lf*zZlep8D$I?Ae63{8Nu}OFcrhKjU7I3W;bJEaxs-
zlk<QJF&i#=XM8pf8MxJ@DZ15O?S=xBuUpz-#KWtUw5xy3-trDUeX=w%I&rw4uBv_c
zU$18buMhunFZ1m6m+_=VBLWR9UoJF#x=cD2JJ^D-dsp<*-Aw202bVYUao_m@;l|7N
zpJtJu4f#U~5IOew=L~0yx_;@yw=3;%D{6}XLtCLg0mxy4y|N9z3VpsL^75fR3A&&0
zm6AGVx)lwiR~a1Te|BMsC{U<R{{5Sf)g0-Yq_DU|9HyaPeF|=AJ8w14BwPq-iO2Sg
z2wF>)3y}ll0;|134d>%~Lx-5*U6h<AST95>6=5a4>v29E=&Fna3W{$9Vd=VOgXzF;
zu*-c_n*8cA6;bw4GmHdw=xYn>6r(mz8ek;~8@Ak)kvyB81}HvO-ihQoyTDEqxSJXx
zb;R3jsXZ@?W9Gti%fJ$sjr-UB4`%f2#-bsM5S(|qik9292Fk@bK*}ojNgmGY6}+&B
zXUIyU2BP0QZ)~HHEpI&Tiks7&J&gxkphlJo!`TIT*xSM6sF>#X>i4CL1eiy4u6|m%
za<b`E)$a>=&ccS>bFeqeYK+9_moCHnVqRI&Ied52|EXd>9gCE6zBSQNNY5by$_T&F
zDK+(237D;z@JA<n^n<e6)T_OwbbmW5)7Xnq{atPF;Nj<}d1tzbqSB(eKWDc3-<H3v
zgmSSild@tx8YLt3o@zn-oPQj5(^XC)!{M)t{+ST#K!s~P{#`~YTANY6um_%_QPZlw
zj`r&Pmvk?|a%|3t{#5j*+Wp}#njwe24+(`Oy*c@rvkMz=s|Q3yoNs_3X%|`_jo#3B
zj?CFchF%NH*wq?W_+Y%J^aZW(nE$n#7cP9&-EZil`&vn1QqBeE380_o;BCmDac+H6
zedZc`Xn^66eYt-VJ%7I=cwRM2A0<snn4$Z!Z{7KkQ*%HbLAC03&0nf$ZIielWyMJU
ziDHj6^LGJ4*;$0zNKQ`GJ$Vw}3~{!<_n-}*RpCQlFysV2P;h()xuuh*eq#|Iv$ZWV
z+eZ=BIl(2qiYCu+wyPocN61Ne=KT+e0|mD#q!w+qf%M&`g24NP(}o;yTZU0E7!M|T
zd+pwU2R#$AvC$VkJd;@Octqr!l}tbI{_5{+gmO2fu2&YEWF~jS;S9?Bw-|h}-yfQ>
zcKEh_f(P7`l)y$r6h^|MV~ueyTt!#YaLOftr=@J_I=i4LMeX3p8&t8M`dw#y2BC}z
zH_2zBiR5!1@;`U>u0>FUbBgBKIJ;m&D&V#K;o0-&awR6MY4+m{m79T7uYwkOF7cPq
z**HJM!Bl?*>m&>3Q78OneJ%NH$EiSYTmStz-SSjn#*$&Zg$B`&zPxCBhD%Hb*kdmH
zKT_tuGC~>x`(k9QF8O-mH!#tpz;I+8>(a7)dW=Zbg3$q|TnDUW_)pha3q!dq%A}+b
zLa~69bokpsZVrll!PiB#S0@^c3Rl{8_kt;fZ2H7=`Dc<s);d_JUVI{%IgHcExmO*Q
z5*+raXX9(T<2m#;EOP=qprTN&e0uuAL&jymM#`F*W$my3Qd<2=tRxF4Ob9!>J{r;@
zqkH@+Xsza^KWcEa!Q9_e3=WqY5PFc59j}d6R7^skj~*C_OlKENaT7q5ZFghtCYA4g
z(U9-afrp&kTu(kd=#u%&w3yfaqy^^*X+i^TrBXqJC=008McNEk(pyAP(;1u@g#ImH
z5YL}Vhogt}`|(3|2h?)YeUA<u_BO3&0I7q2O+cB;n9IMG?sZ^@dx$A~RafNh<xqw<
z_Yc#n^zR+am$WzeV_$%>DY<ETyfY!#03j@07uSAsDg>WfA6yo+Yw+DbHmp*pFIt0N
zE)_)77#7&4d&i2upHbrx_LEy|<~i3Q$hl2^fD&n@sT!MJr5ea?BUt3w?w})RwF20@
zLFwSl(`5jT7E1prT^5*_H3}lmQ+X*bo!cLBY!{rkv?G5pXvZ{Fa%+3`n_qv9yzI9n
z=PgiDpb4Sij{<yNGbjnC>s7hb4p{F#N6(W52<7`zrQTDapoHNC26b#u0k+8K`@Kr1
znKY$jF(Cv#F0Oq!)&t*K#*R#J`)x6UYMNWVI=E~HMKzY7oc9JwDvLcfEX+r$E4<b_
zWI&1~z2`o1W(gx~;5`epvU}+Fm||P;#k1&^D~o?w1X-3)+Er1HkNWj-or*G46pRnf
zduFb?Bv-PRf5pLN8e&#Vj@SvMk$eIQMJhRJj9Q<pVk20zxY~HV&aiT|I*^o(%xjm^
zR0pYa!D_TtPy9TjD3BITPv*3_x#&R<v10CS@+WzlsXI-D5hz`DI=3C5Tj{`_9Oimh
z0WYXD0m^I&%R>J^)M_~}dc5?jPw0JUzpxJqpPitWfl_R;F<b^z*%$!35QaozOs$Kq
zh7IkAWj=^%6`XXpX2cBp)PVgqbUtWPLX`aj=yxbKZYG`S$&c4hN(`Iq3wnscMSEUU
zD+p^g=Q4%R7pQ3<HH`%g0QgrpBy2`&^IeP7m$;3aB7RP0@3Q^LEPh;dD3jB)ycv(R
z$V_XVw0n>=tUUhyjzp=S>||4<F(?g(iTfUUj(o>H3qK?v9_0%#eMEPC?B5#$h0pG0
z<kr$Cy9L=8n4%nLKF~EmloqB+B`5qTUafgImI9-sEAKkrXftb??9E_3_^CU-G;4~l
zr$A4*f6Ia9EDn$WTgVNDFQB)kj#vP&{7y5ScL7tK&Yu<o3Yi2jy^ELD+P8>y<P|hD
zjAm9PnQTH)R=n3NQt>P(crECFYl?)LN8Vf)O(=UZ_%6zh6a9ivE|9iHRM?Z9U3lyV
z=$M(QR-ovL?HqH1WrYOr;mv(PonWc`Py?$=rm92dZEHEH6|T#zt((6^N(5BdgBrUz
zsK>S>iyHvWkA3ZLYq(W^v7jyi|C!PzflAN%Ga+a6o*UrxS@CIoF!Tc+y|KT<G#${P
zTozroDnM(*@ZzOXakckK_gy?|9%gZ58D_zw6C!dlyp}(o-|hF?S)B|AHTx!`H=<EF
ztL5Zq$6*%$qMC3-Hk$L<*Ea;5r3Y4MvhSf*lGmynE+JtwsEWF`d~-3Xld^LEku^H%
zLP!wG6S;Lyq9H8%lZMeixxLW?VbR<W{Vr}ey0(LX$qX`9ZFeU#ic>MU)gbYLx;jTe
zr@=i?sar3ZJlv`~()!cy@U}`nB4#bOdZh0y0Z5e|+-dTi`N1lu@452lE)*N`v`VDA
z%0X#fy)#oOc|tVdEOv_kj(wWZeHM-QlSu=EgD;nQIgO$GAeUOhyo|)ef|?~t?$;R|
z_M2H<Rr$iTU@s$ihsnp*ib!QRTRoe8)FJ`Y8JU1IyXBwjRm_#BQYbw_hdvDmmGTZb
zk)Dcdh-Mk;MHpK``C+Z@ueCR)$%Uu6t^8Tc8u%@A!&U4iCs~hxS2F(yBBmtincDV%
zvBD8(o=A6J`%84`5)BB%VZc@^^VsmtGEx;l+r$v@{*!1CA)s!akpoxI%A->JL@rjq
z{(V=j2D=|~{#xWhgMQHV%%$-Meh+%gBbc>`5G=szTHW|kJlNjPy$0<mO}T}&OtV^{
zq8{s^AZE+i*?pK}N_&DfX%s|>p8rsOmU+s`jsfCux8HaC`<rNBCvtT@2WcRP5fKrE
z{hW;2Yol^O5HM{g6{*^l>&$ltOYk1fy88hlEBxClGWn5e_hA+O$TO{#1IY?Q`WHd;
z>!@)tH>}Ec)~4h&HDg8uA6k=g0-bG46m}(Y)mm0tcMj^)zHP-%sP(}<+g3&5#h<Xh
zjfppIFM5lO1B@^Mgx%HtTy4n`wAbvmDZ|c;4gh|X+L%2$f7mWW0a|w55u=;vXJ(Se
zz4|X-(g9_yf8lXevBHf{8Y?yQ^%in=>8(VellE_dTSsa<(*dXYK8Z{a^k6~WklrPD
z6X#qc+S-7>f{BX3dCo$i+!zQHon^KLHRR8<7X`hx?VyG-rDwt&GBUDV@5DxUu5DRa
z+O7NsDgTyeq2nYr08PEzSlvXwqYZqt{<g%@Y85}RH;()#8K(o7Acp$FKOn&#yUTGD
zYQG>CuuB;s;p5>^+ud3!jP`D*>8ujJdvGWoB*xuaCA@N!PuMi5;@{j>bl;%sFQ*08
zTL251KqFeZ*UoB`ImJbE)_w#zkvH}k(WeR>is#2`*&_=G@b%R;5O$9BSF`AG_N|Wk
zvVj})4l<%a6(2|HL#kbWfwV9zRHio&u4(xtDqp8;R_{5xZaMX@3<cs1T{(0!hXhIw
z*vMj7#`V)JUlzjL&@BUS;)jNTH$5nl5%H$x0zjnSOBg_L&G{P+*4(|8$YjdeZajXt
ziF-W`X5A&+Pk&c+NML-HJ@<h_XSN>r_?A&MV|q9v-%y6)6qusQz?OJ>dd*OtDnXhA
zq_|w5_)jkGQ~l&e=xwl|OjG!aXPTBST6mRF`#Qmi&uCq19=G7{v+ZnYjy0j?OX<`0
z@yeDE^Jcw9j~}mhI&1_tI!rb-f}ogY*75!|>6eYOKn2x!3GD6Bu7A52*=iCFI%m|x
zzb_0`IC}JF49xJlE)LZWEqvW0b(FcW#S<Rm*i2m4aK{wp$R80JKnCciQsvw*o2Fg^
zUshJ_j|dX{90e(TkD2r|+cB2^pW6KTxb4saD-^^zIS@zbK`R?n_!*rseChc(eA*~A
zWcb<M%f9!jx-aEz@o~b&{(Pa!vLj55!-#%3`Lg~El{g!Oy8r{P?y<GV`t&?u6ewe9
zsNIc4d$uXavmIik?44ZLu0laE1;8yX&rPewc>coRE1Y+<KUvKzvAAxC|4pW%Yl@*X
zt8&~ea7^02L2d1W%V&al&ct&dyj9kD7)YVx_WJtFuXO2<tHov=H=wAbs>g<X<`{~=
z?ipDwqbr2_($qI@++k2kVtIP*BGHEa0^U%auWw@vXX@w`fuWA9Tq5Uf?;tV3Z{RRu
zh!*lTwven{5luq|bu8uLOLbdi`Cp?xd)83r8H;|kW{!HUNq0sk8(IbL03XMGjVBJB
zhGY{fw;QW|9tE}0NvKX=g-OQzkj44=ZpZK^OJRBoUyY!hDbF|SLL535aoGg*VO;9V
zvIhkP>szC~(O^Z5Z)s(otWS4%Oj&63%I%<KNCjkg$l11z<`eKilMR9FR9oU*Y(GNL
z^-uzgWYAG)rE&}5=>AHIijskotoGg`+}FdTF>CI)Or09Al7}wy((KjR3%bJ@3KUwE
zPT4$Geaa@SarCj=du=T;?AM4+7lBlQ^5m}^ooKWsdv9#EI~P*_SO94t^&jy)@)o7#
z5D*Z^9u$bak2x4}K;#Wr&4fhuPdeYj+%;C&F31{a!q#}AQvj<80(fIpV?e=Gew&2#
z*>3rlml<z>cG7UrLf04IA)|q|yJ5(_Ni)Sxg=-fNnE?oBasVAofF9b;9Cc=LL5H-7
zrpRV%>*d0iR>0{FYjRcVKW4KI1r*HP_qGDkh=`7u6^*;VsqpY&WYc46F_CY1jw+&F
zI}6R^f;Kr8kXOEKj%IJ}%2w@fVK8Sy^NbQi;QexAcIb4cTEh@vY7erWg~2lUNG5ft
z&GR~_rY$Xm@_!m4rTsK`^O$6wN|BwwF1oA7UH^IRXp>=D3>T-3jwnnO#udSFC~^yP
z2JdQxK??t7Pr9|3>tX_r^?+JWp#eJ>wYPZyIJvn_Ax1PC@1+C)_*O&2b(&*ha;z6-
z+LrJZG>QI55Okz7sP*CkHBa~P5dQsvXKi~gB%GW+t}hH~*iyg0W4*DD0=*M^(o@su
z5>NSEFdA!%+DYRJ2CRTIFfj0E$sy?0qI>)Ht%oXNg9Q~#&Rtn)(IkV)hvh!Z@BHl|
zA%WE2)vxpMF6&W3Q<`HnKsF=7E+5eoJv0~t;lDH97s}K0y;knlSOIH%zYQ@~GH<R&
zHiS?#w<QXv8`Sx5*^Noy;y;lqZ6^?*JgGRN<iK!>{%KdxTU`e=IB&GnK`{fg2|HtK
zgN~Z=I-0IP<aszq2UsbEoI)W;w=hs@{JZ|S)w6Lw`;^mi!GC2fq~#tPIrx-9YUh0|
zW(u`o*REb=`uzDb4$YV=ti-fE%5~IRqu8{aO+Jd{Ehzf$C1O&Zq9nfw2v)QQ<)U>G
z2IGrY_}iAq44Sqg)~4F!fl6$cP**ri7E8>T!C*<meKnqC`b=K?+xFx@zpJ-KJd-T(
zdC}>aG4MvL!aO`^{_gPw(21oUKvB29`bsTZ<*qIMdl$t?2@F49Y|wpi1In#H<F{-$
zgTlo48}X?7v0R_gi~~*|7;E8jH9F)n;{lzJ23|Mc6!y0{T1oo#HAdk1KRi}7Cx%5K
zuJBq!j7X$`m|JYs&jq4fF#W(?bkPj81~wpx3QHT!f8*jvNlE<xAQMVHiO9*J){u_h
zFFY8iL0OiL74Qe*KP(Gt7<voz!e@SE%0mFN)~^KZpvb^*KAbI1rw(d~NVCSHbmZdp
z&v%lmo$~B63b%pL|ItfKDaCQqgaFR!?p$FrP?B_`@2^4PVBVRwVwanRcG@Bk9WLiR
zi{2Fh7)Td>VUF>SdeMUXqK6wc*Og5?bybWS3Cd3IQ&ZDHHyxMDyr%p9%J<2VcCl~(
zQyA(n&gy5$fU{c9{+-5sGGVt&<iLdC^v`!f+p6-udjR_IpTi{}*lZ){uz<5Lr0^jg
zPy}VwWq`6cJLn-V)x2PiE~)%q&I}jt>({T-3KAZh^NgU+DU{E8;78ykS~_62xccn?
z4uv}SfzD@n&<ykhbS!fjqA(6xT2Uxe1lf;7wD_EX`1JXo@LV}T|D%ZQnapH5fXaM!
zCdsn3OLLwD1js+fBk@XgT!&%4!W*h|mLGASq68N4T4Zm&q!_dVfY1?eS4X}Ct-1YR
z%r??=ky8_sRT%>X<}iRqo2%W|=zI_Np+f<H$Z!wavIE<ez`VTjooa4!3#fKy`$exm
zK{pe0Fw;-`5E4lb>9HUS&#7}9B`$ZIdhK&=4A2#HOiU1i@;;iu<VWb|Eg|_g2*7Rw
z;C*krA!HJGS*}O*aOg}zO-eTp`oND0Er6s~MHf#~D!(VVP+$DgxxAzYo2$+2b3fn7
zzP-^{84YS-C}&tSyFpYGt4<RzR*1jgGHSqoK(T3e8uO91pq9G(x4e`2WUVYTA8bd(
zX1cP&y}&^ZsF`1s7VuO9QDzh&$Cd^-nUOS^g&sYS^c<5cOzL=NsPZ-l=~UO$DxV#5
zN65@$yR^XlFn|Fic*G<ePV;>a{GUD*9p;&p!SIuU^I7##1JPCvRlBQ#)YQOpp;W41
z!tzw^%MZbLwv>VcC#5kYHuhrx2W`@7JT}6h(+{8p(|CLNH`FM9;Zqc--AH(0md3up
z!|>f-kr9n|ZHeIo2@=r}ml)cmL_?M18B~YW$s6(*J^-Et_u|D!2!sKEl3|Jkjth)f
zw5Ea5za1eMO$nq#_Y(vjK<y(SHwY40MqzE0zxst%zl%UsMmX+$Jl&iN7qZBuEY6Jq
zJ#o_H09llIPhRYEo%_E)p;iz!tK|nUdK5_D8iA3efqc2KBE%A{b@gB<((BjaUYtlT
z5*zddrJCYrn_^1K$jAflk#0L$8%85}j+sCH6AcoH1v^i0EfhjNfC^xQ;3EijvLFOu
z(c3Ye!d6X)mAaRDTj%5@%S3+Q(8WD~ekOmsrCn`eF8l%L;xxP8_4=J2aiH4%8Yin&
za-ht%(DVSbrLOJrf}Wt)mp_-DF4yo9*pXNG@YWON21NniTpZ5jM`%{L<VS$BGz<!H
zbfAS{^+m=nX*fbQ9R*YvI32dIn@ZOq;G_mRiX!P9gimMppTskg3=cHX-DSUk4;T&L
zy&C~BkOq7l(dEk%&uiW_0)L?5XW+AQ7l0ua)Apop3RtOSFI)Oud#cm%{`2SvvJ40K
zmmk1mf~Z5rt{(whpy~@kDmIW#E=RHbRD>h!icw&C6CwN~p@p@(_v#em`9e&e3iZeQ
zIgyuLBp_dMpD>c0I2U%ZzH0Z9XD?nfy`~hN0Dh*Y#247r)MU+jfH%kRF~PYa$pC^#
z05R`@P<(fRV;}T=38k7fp_wi+pn#Q#==Uci=d!dPU|r|Pk;y*0*_UrAOMuO!SBU4^
zXejsz8dvO!P?gSeP$$WImrKclju|75`o5{FpsR&xcR-b##O<{07|D*C<@;8n>L^D`
ztfQl&Ghc@v1RFy6Lof5tnF}yBgn;vG1dn;Al+W(W&4aBFfR^t8+Tnu5ds(!b1APXD
z7T6V3CyS**Yy)qyVUq%w?6&~+`~XLT(t)Ok{p79$p>GiZh@37O=oC>pXj3e9TY20V
z&7LfrlY!1U?Vy7^m@I53ehSqKKLn>bW`-(C^7N`6Hbt@KCSz%qm?#*`F&u!n%#WLe
zp=`|*0cc^Za=UlSTg3dIRWF`wY~y86%4fJ7ed1Ba<d2;i$P-@0)r2#uG8~|e2q=YQ
zm6b!;ox^LXkpTB>E(~TSla%`&`5a{Vfqubzz%0&d-$6H6AfWz|TAbw5ZFuAt6LTe&
z*D`8-aaadr#c8IC_!NBi9VUMK{!)_xYAj<mexN_`hmLJcx+!+;_McZ=u8X>jalE-z
z@wdHCZ36-DC&zUhs?#ym{Xiw+!5v9SV_<D6H6G5fJmz5$%o=SPRq}PfGs9q1Udw(y
zE@Pmtks6ptB<ShaOi@RdvyPztn9-ulspHXoZlt)Z&v)lWLu4fAM!f|>Vi-tQhm|~(
z_%BJIe!)QdeM$B1Y_7xMzWamEhQKw74$}cA3+*M!7DS)}m+6_~rWa)41#s*}hv9w<
zHH7ephE6o)4WRKydaD$V!2OS5P&oeE8sA(t>cb4VEP#QQw{E-&xzu_Lp_U1OA2_;V
z>T%qlGiC}i@LG8e@wcz9i!xoL;EPxp|1J+&==SgXE#5#PBjH`Xp;vg!LPsPfeg{xR
z0Ul4T=<&C_E~$aj#`1PRMegn+e%(ISMPH(AaMCOtbb%Fv6C@9g-sdMBtBUW96ma;?
z(EgYoc}2<!g-?_AB-X;})tBFN?ysFXIZ80OQF3{Y<FUuSL)y~AZQZq2#%0>Z0<ymd
z2*mpP`|oQ$FQ7#t)v*;51l}c6lv@w-&2*#&;8Teu)EQ_we=s}dkC93M*Y0-Nb&{XV
zjaXTJ!K5vLvBq;N3iJZXjCjnk9^6L(H?v29eE^i5ofxDFp#Z@{Y0`$`Uog=y5%*O$
zEp{`u6MY_mNKw7<yOt-q^5KHjz(+ZDtdR=G%&x2llI&PM6K`ear`#d<>h=@`#R^cB
z2?MIGRqa+Ja<Kku<fqj@;7Od&fMVL@hYquo>5)QqzCPct+Eph)#-gbV&U>&H)@;U!
zhyQeiBbFWf#1FS6uLAx&fRHNc60J0Ju&t(AhUe7kEx~Boj=E+fCo^@$g>+@<P#C1J
zMg3O01`*w1V}L>!b!IhyAjAlR9cU^yAN-W>wb^glnQ|KoGNUJrVc!NTZ+o6D?;~)c
z=>Y)^C+53ch<ZuFoCZ|h0UTx#wKUn^Uh&|g76XcoEX0Lktu3`mmNE^5E|z~_8Bjdh
zJ%;gPa$sAPW>xu5X10Y;Zk`l!o~0Z0-by@-{tYS;GgFu&<9v!xu-9jWB0GS_LAsSr
za$@_-lIGnxks4b84JS~*@IOls)KSO*Ot+sYx1qzJjR}G!K1XfBOTeQj=jFOVeRSIe
zD8P?G0BTqKSr%o-moH!F0nD!*eB^N&t|`Eg?P277k`r%0r$NoYAT72tK@6Pmn=B6H
zg)~8M?ywQKB8OLFQ0)d~3hx>v2KbpBK-_3x{68RS5Fj_PU*bZiR#64)c9>Q1)Wgu-
z_+b8v0YWQ48D=KyEH;}#YLW1x+Y7=++iy^SYM2C}>?dWCuxml}a{YzFU+x|!b#){$
z7%DMP?!n3Q(xf22-C1I)1o_&mfN?+*Xp2)Y(;M3p0`ZRJ!YoSCpga`BIjMyhF}G#I
z>(OJmPZfaY0xm_D0q8H6_9;J3ZSFT%X4?$T(X5B@g9!NO(W6d4h07(2d3W+VK&<?-
zs-Ljy;&kjU5bdJ1OD$%OkYU2EHsakWgD3M5fQo<R<gO>EG}OWmoaohsN|=A~$w+WW
zSu{DiCsiPIA0vcWxZ8n@h8}d;L&s9HRnlLIJH?CLKu+YjMPgv^-h=)Y0Ca1p1w=m+
zA3p{(qLFMHl;e(liqC<EJ20?ywiwW|tmK1v(q@=~y*b}c8*-I544gM?0?uy|fO;8p
z6?+jhOSlnIA1l{!s4vA@2K5C{Wn&Bj=f22+b4zN4kV^0$MyOP?Ebj*nIy8avOY5m5
z;3V*VCldW^ATS47?bD-S$fqz6tyddwCd&eOL5F!)K`o*gB!3$YeCJ>NIcq69_T>Mk
ztCffh9I~NWms|knlvzRk-_?u+<WT(f#e~}^GWZR$?K|w#Sdi4j{6b9@b6>>)wOfK>
z!Qo8|NJL0v3ASRA2qRR60KL0%z+*oFpm}vRw@|3+YLg@?4nWeLYh>D>7~BH_(#OTV
zpImS+(4F`5{P{?>V|#X!9s|J^N+8KOA3w8^0C*W9s(><2#h!b>4Ev`<hU5oM-eYaL
z!v$Wg2!@z@5q$308Z{png6daisilUX&whE*0XPFIz^I&pB#FNlkFbNfA1%EZDKX*y
zFg%8jfG@Z}NT6`ciF~L<0lP|r!o{O6=NL5#5_|~8ekqAtAH4W=2d)b_TPCe0;F%4e
zpxXU5-2g{Z1xz69%bnC?ea>TpTFxOt-wO!zct1tlK76zt)rVnW<&QWG1pKQ(XUr1_
zrItTq9S8FQR6ujHGrM{$%MCclrY9llq1Jn^X|g%md7J;q4@4u_k&UbJuMo@6z)Eg{
z<Qo%Stp#2mFZ1RT?bv_huP9aM+)V^HXSuUbHkt;ip<L#jbc{Nm@-pKjYj8AG!DDgb
zcL2qLs)V+Nl0y3hk`TD;-LaHT`r`mFg_j^;f5&5eng*Q1iUO=wa}1{u)LGJ&|Ev?S
z4Ea3+a=Bg5xAIRj3KA&ykM@%?3dqW!&Io|JBcbRZA%P^o0!aWPz_aob1hs%xpuw4t
z0S1L50C|BJG7Va!L=JF%s^7l19$RxQ(+9wsK=qny;y$`@Jm&AQKEtr_wk=#|$Y5=t
z;n^vPo*v{cz7SdZUEr5IoCTEE%-;H?kRLzagXk+UeFL@-5WDO^jM2->cG(Z}<9M&i
zFe8?sR&3iV^icGB2pwtfH2X;k_tFP~nF@#@I4%v*$&7)?LUci_kaNxmrW@d8Aghu$
zG&FRunF0q|Ie-x;;6jaC|6;9Bj6kG{kJ1pwa<nB0eEm8K^j^q<28~JJT06m6<p(r|
zpRD?;OXWXOV75bXw<mc3z^^^9{2JR9GftR3)GhJtsm!nA(5G@UCF;m!wo48=ft{^a
z_33^bj~wtA;gxqPC@_6NzwG*!0mcD9X)<6!lqBOGAZY^aZiY<D<;T5=C}9Yj&;XAL
z`8r0Gv_L>PN`vS<k(l_d8%GoB{ji|D;|cU(7EJyj6%A0;|G+DLY0!42OV&YnA8aOs
z3?SuzwsV%R4QvA~c<{BEJuA>*>N|)Zg+L94phg&JtDHNKN-QU!z`g0<?<Kr9!Kwv+
z4d9q88uL3Z(o1b2jnV)!_jY~)wic%x>ONI3jsO7@7wjzqvR>mvA!t*Qj>rI!2<bM6
zbWpdCTE2F-tB|Y`;zJy;if-v=o4^}&gR%0(?tCP~{7=#;0|m;M#wa8vm-PTo4c0PX
zP9Q);bmfYfemy8JmV->_CrM5OK%=$L1~jM2k0L%KfPp7pJUe&nrqZOTg@m*-l#<!N
zNy%`?H486Q(?Urzs9k4dXaY9HZvyNngT~7_;1d9+2>yl#J(v>(_%Ix(pnklP0updM
z>!}miQ7Be8&vw0q8X<rz^>i51fgR|AkW+r$3zFL+a64}7G;IK2{zvRhx+DOc9Mm5G
z^+?=(dAWna57ZNWa)-Qo2;bKO3mNsrTnA|E8gN!lYQb}tB&bQi7G<NxA08`a`!R(D
z!8(uxbhaPP^_OIVvWH|Rpu_uLJP?K*LLz8SRT>3P(t{@%AB+QrM)n=hE4<(37r~ZF
zph{pCn+5>+O+tT*0q}`)T1bFzwyaXKfS;8DODBOs@oRv5h5ml)2EmjG3Z^Ue+{d1-
z3fpCABtIDogna32<#+xN#RiJa!^8Nqk{Etpfi)0^|Mmp$p$2{$^-OOYfB;?S<O9(|
zyyI-p?h~}5P+Qe%hYS@YOk;2YXSGMic830QTfwVU@aR8_UX}+oUw6$#C<K`a#ulYl
zsyW_`Mj~LMJ--1@elP*5jH`=POHC=apVC?O<TX|%L-f}_y)w%lV2nblJ3>e#KUkH=
zFS8CvoTzp<C@2(0V?uwsVigeQKY&2eK_=sqA>n#y8T64zuv2iv0gb*6S?f09CkA{U
z;#88Q<7**jqkqHh|6%Vf!>Vk%bzzuJaRLfc5EMaDq(kWrMG&N0T0}ymTT(?71PmG^
zq`N~J6%Zt(BqXJzyK9fz=UMAp>-hFQ_OJbKy|2$7VBYh(M#nkE4N)vUAt3_Z_`9Xx
zHZANuS$+4j=PvqCAO5n_2geCvCm{BmqFzI%^uI4KVIo9|U;nc69W(&vAk8B}Bh8Je
z7-pjX7HrWV{;SI`Rqwxg0vy~rQXQyFIjxKuAkG}>2jR%VisQq*qYghJhVd<s0Yi!B
zPKaV7qVdo!>*jw0sE^zGAra*)&N0-g*I^xlJffhPYK@aiA8~>A(5b7x@Yx<<BD;t)
z9wQRFB3n-iPmNbehaw6@grD5jK2`$>J!Hp`MHUdfOam$)tiE2-31}P(DpcdXX#r&)
zH$+oO=X+lPObx96(vKc{gbO?<RaI5tLv^}wsOY3X3FU^x4_c&gXny%tF2paOSCX?H
zYT##gVX7o?CG@)3t$JxV0r};#F-WkyTUifDT{`d)g0^)8g9m(^P70u2-?HhEKp!f=
z*VZ+~j?()|&ckWcZRPOcV%Bi62kQ2Jfk~3`ry<x)6F?z(#|1X6yG<ci*dVc#mk*kR
zwpu|Ed~djH08)7S|G-fR2tx*wQcXb~eP>T2Gf+ro-jyt?0M!VjnsDYR<P2M$lNPvO
zrf$P2`0~OaB(FtAXR1sUI<fx^dR|Lg0Ckir`>?}BK8EknHP}7z7SIBF()~yrcNI)Z
zR2UJ0v-d+%gBAV>N~jV=!^yr|upkBSRb_;ecK}(5WRyrZ8bJpkeHtdbn%ng>@S!97
z-Z{|X6gUevpT{srHW6?_7}GTUSEuj^^(*Y8dX?&tP#u~it-#Av=x~FrjaohD34dOT
zG#Fwh<@14WNbr(ZoC?5|3ZU>RCrRz!LYLhwTFA^{pGVrwda(I&fZcfjcpn%7CYKGI
zxCiI_zl9;RjVL6qLo6-8LWwi=%JY!S5O7Z6zt=0E*y@QCm(Jk5<pjd(!L>$O70oca
zRX~cm6Vx9GFjH?(eh|pd2TMXlm$NaD>^;OlEr2_Xd-|>dAi|zV2s-;Bj0hMS1P?Gc
zdv^^(iv=hwcNyNwU-$j)f{KEVZ1Ck&R~#ZDq6nz}E5JsJtAKpCGunIyZovRzDRJ4l
z5*Vt4EH@%x@JG5TV58p8enmWn?-(Kby%IUtlr#lNb9as|4+Vf2r@lvat{A}DGCwz<
z-<squlF1JPpm>#9G7bVPbA%!4?~4%4qPZ{tRRN^r7%Fkhj4U>SXvlp&Gu^Ww-3icn
z{|=NHu{@Y0eqz?dMESZ|ZkX%YzJP;#`Omyx3{^fN_S(As1k+*Klw$V%JDz^N({Ni3
ziCer_TqC5lliHQSg&dXiG$$ZO5buZtJsojEX5j0(%O30DV=#92IW9mBAdZX-zoip%
z+%BRLaWU-wz-jPOgV{(>NrayM<5d3@*`+bDbvvn+sIIM0=O`ZA_a#;$irp#r5hDGw
z0}IOSv97`$VU|60RoO4i=SA7ay!#gNR!H}yb(!Bkxk4PtLAx^&xoj9|Pwm$WHae{%
zv<iYNIv{dDMa5t7_VUZfU`B$v&6P2uULIUz+YeIEfm!7P@7Xy}<P?Ymxwl@hFq=&M
z;YxL_YuxWBvAj9e>QB+Mp?7nUF*DkoZ~knl&#{S@Po`FB%29FlXazlHxO&>xdT-9R
zrX?#x;#ib11nqY^=tXOb`o$%)tJ^a{(<4)<7pWB&dB+&WsSsl%s73$^2|~}UdsiRn
zckBlIE!IL3vOb&OBM*bMX1~9_La?YOe?)5l4>t=Tk#KWYgW7&|ozCso``zCCY3Gho
zt-?>9iaRSAm{Mog{^8t8$Ts$N8q8*UTE83o_?Jy(TU^a`87bq@YF$R7QR}s!5+B3r
zaWNLHP45HqQQC15I7?)K8Z(>)OnX!T;TYnSyczALzMY?gA#qO_R#mHZ`=`IU*{$Gh
zCTR{d{ilcQloLgiR6PB~^3jsOIhysD&E>tZv1TJ{ZY{0cQA-I4ADc5Ux@~q~u3b~;
zf!*~U!@J=oX{zm<;Ba0JCLoR1<0_!1^<PICtJNhrj;;XC00~p2rKJ&QFj9%0)K#*q
zF1}d;gCn9ap7-bNb+_a`jXkG^(05`7UJi5J&hE#b){FLoxS!}b%y{X+q&uhk`h)P|
zrVVtu_mA5{<=KxVY-?Wy21b-Fct;+c>G)&*2T}vg`CiuS6`uS%i_V=ZOw-Nn2h;1*
zv1(<Dr_}aZ&Y1OSi@e`@UBA$8`@Mf`G2%<WNd%OiFBKoO%KM#hxT>WYx1!F&@Z&8l
zx3F-N&O1+)euE^>q^kHU8N(;L8zqHTMRch1HQ1FXO??C%HscPDn^^{{c1kj4g~WB}
z^TRkl7!i=QQ5+PZ$6&{k=1=c(SqWk&go~<GJ*1cvs*NV8*79;+{h@1Goj=f-ZB7lC
z5h-Y(QJeEU)Xp0XqxW~5GZx%-a5zw`wyy!xwGf;M(+rDX6kG^KkpS$NvA}LZ8i_9I
zj`tN}1Z+9Mq(wk;Y5yXs1|q3hNI-?hcqtfbd+p+K^h<AEu2s7ekw1|4B|gYJ9r7Jt
zDnYl~@)2{Ksq!;D6MbBD)NkK@F^r5i;c8;pD;Hy$O2wO;+I>Z6U#m^&b(Uf`YxB>s
zbli%mkK0a-uE!~2GVbUA0>(#u%p3}lwh9i#*uGJLv3NSzSn{P>vo3<=$Tvs7%OF6*
zUg_(>cpmNQe4pMs#@X1)?LMOyEgM^{&6X{#eIg|Z`|JA>JS#)5rw{kCy9;}$bc>uG
z#T~Q}=T~c};t^Z3v*sxLiQ=AnH22Hr)o>-5i9{~{s#>SVVYgd^Nd!B^;#jQhP&J>y
z!}dW7?=b%am#}$diK|;iml;G{uN3>_j~;qiWu5=LE;=sAyu1$r#%VbZg=#RVZxX=q
z(n5tAOvM|>s|TThqmiX?2f&BZP;}YkIb8`pjT6aADITkx&JLFBeL*|fT=FCK#qcLm
zmMWhsbXKn8gN{CLe%g9pwJpufG|$Nl=HK7+(0y^v=02BN`@`(r2NFV7i=sKG>Wd57
z+)s0yD({w_4<Lw(@cF5$p+k1E>o$Yuo_*B3zffdGAU9%8^RxIz?Y-jf)XR)zk5)rI
zG^bbvO9>CFb^gva$bOxn`^qTiP_xrm!(?1J7Vl2l-B&Ba3mmY>LsUdiP%hr{V}(YR
zjz(<eVB|vWfk9`zhTo$)@%*n$KPPM^`>)MKds@l<X;lvY!NV2lRjX(-782&+{Jn*h
zf%hB%!E47ArsCR2MTzAe9*Vw2NitsVvQ2YU9^AgXuEwAdE^rq~E3Yc9_v>82UYPY3
zuRvm5#UC0_fraGm&DdXJGNF)tf!;^^EzLAO?9lXnhhSJ?xzuLe@E$(y2pBt8Fm{zU
z_ws~UHRc(rTxcTa)W^d12n7~!R-WC-P5C0yDOcqx9zl^{{pM5g=ym?SUBlY33T>bK
zj#$~6i?>?@#*VMSs)_e2{af!znu^q&GbGpg(k490k_aZfe_dB2xY%konr2R^mC>sW
zQCD&#1D8?vV69#u0og+D6;3M_YiX&(ds+Im_icro_uo|;qeFvc1LR+vJrR%j{k0`%
z-Ko97!{vUgR^Rmdxur$tCZEZ3Qsd!l0*tQ5u1*8}LYqR-++{LxHW|lRg>%0a!mB=4
zNCY!}4b}EGvKg*mSaIg&U%$NbB7QEDKt*c4rxb6&F<F*3yub#(bBNdH6f3iaLfnG*
z!OOP2lp1~hA<Zmn*DC~By059I5ub|{?W2d09YCfDEpLKRJd{HMCZtSF0ej-sxCjKM
zYIa6@R(Q-G?uF7Eo&Cd#T1G@P!8s=;Fnsajsq(?w6^@5r-6^;&qN_)zSZXemSntI*
z-K|%5*c`uQIcAbZbU12ZsCnjD7G~>io%N@&?o`GL*D@&{v9hZ*w^1ymY@K8+?9BT#
zDIdyLZbjuwtc>v7=8`B&;2n)BK2J&j9iUPA_R*-13GZDk#?URc&)*ah^TjH1P-u1L
zi#QnHUsSgrO}~cmoNpmfOI?3v-^wGg$t_)~|LOSPq_vm*oVe$dEsJ4RHmAp8#!Pqa
z_l<{qJDyj#-J;J@SL#dftkow>mS``=e)iyaQ#h&p-Eph6McZimrC00mD6BTRI3N97
zr0iVJM1oM+y!&gWkmtYhXM>SMV&trnGiJ9Otffz=L=BWqXf7h%XNVMA+-Ua<Ghw^T
z-$*fneQvTQ?Fs0<D)`9I)!0}+cHK*??!o7zvM7q@sJ|r`Y%fwV4zAdyk7XyOOHJYz
z{r+wQUY_PXSHuzJ-J=4_1dV*LTjd9<$M|X`a?4fw#xGtO2#cjh(!^ej>P%Euc+;mg
zwJWPuh3C2=WYna%E1#vs!V=5B8S0ZAe@>^ujp^uUdV`0=6E9vkI)-*NPu|q4aAl`Z
zZN1?n{zGgGtqPgm@1#73Zv$s##LCxl%Suzsg0f(Gl4N_(t1v5MW7}W5snyD~CtToH
zZx}_-vDa;rMZ(^@N_@#Ywxc=g{c9|R%7bIgM5sugpE8%L4_8QR$xzu8DX?V5={sll
zHs0E`JaE(c#NrU+Dj+d#s{%4_?`w_@p%VJ&dSU<zKre6^>NkbbaB<aM+DTfCx1=bA
zh8@z5^LX&G=YciL+qYds4>grDEv<jN<!}|L(brxgI#g<o7M_ZhD1YXyl^Bz%oxFdu
z$mFra$j=t_+P6PA9k&jC^*{U-x!trZH?q$#{j2oJCePu|+&NPT&AH4SnaAR4Cwm0@
z6wceECnwi3X$xj}Zf@N$E0t<=Y~oq|P1DuIUqf27e;lxNw3T*gEnhW3ofQ2u(?Dxp
zd_zvUG4S5IocKr!>P%QUH$If2TLE)6K2^I}2626D?OGfO5e}gjc(VgHV#<xr)bteJ
z%gc>Fcj=_ULUhc&Z~nvnf@6AmQ*@(+;UhdhR(4T)LH%Aw0wMORP=ZWqNM!jh872;{
z1PPJ>yuL&S<xsx(=CIqa^N|5EtiF|_mwtIbC?-TA^YUHZw!cX&&2!yo!o97tnHbgZ
z;ht7@d|x`MVSC9+Vb*5jLH88jNB%gK?>GCpl6Ryjcx=Lna5S+|Ki1|K-fu3~5r?;O
z`5f~PdEbdgt;`!|yZ5lJ&&9bef!}*afC>ux!`>W{Ai$-#uF2u9sbEAZ3=7<9$IAWA
zQvVQ4y1%l|St^yts^af!)w1!qQKDtVC+@X+z)-mE(o7Ej!|TMQ=jHa>a@#_8JQb~P
zr^7n6H*QDeUMspp7b}oTbo;nQfUghme1B)R2kJJfbbhA^$>k92C<1&U=eYMRL)9w!
zhmS&xXGXRYWKTIrAz`~qb>bbtRBsahhFgU-E``v(Ld}zc!Kg9UV3N{P`Fqtur8ifS
zye7Bi2C9y;&N1}b6u|<y?f4<S-HlX(#q{rDKlQwSIZChGJJ@QiN>GM<IJ<`m$<qsb
zRnvvCK`%XQ4Ao@!9Q>i6a~9w5x1yBsB2k8~ON+Ul*}Uw+<_!yH8+kW2_wpm2Laz58
ziCuo0<lXC_94a)7b6k_ipm+v`V~rRBX?pU5yI6}t+$15)ZW$O3+fc3=XdFmjcVWpG
zBS6KVrb`bf)psrLnZA=Mi%y&oOAmrRGtzJ@C;smD%sk~mw2#1s3gy!b1F!PO670$v
zdpn<=2ssQh&7Q=hna4!OKACEZzT=aAneyP?R-D9Eb=|IP(*2~5E@f9%y>C+NdNUl%
z?Q?~`&95Cz6-XR7Y1TQ|nTbt#TJL>I{JjZ-*3~_0k>a=%!x}ye?}5|AXFeA5#rVU1
zqmyqzT%xQ-oGTCGyYqvMVy}J|A4`k$On2g^kus)ITkwJk_-ND*dbFnw@G;;_WFDxx
zEn%F5kKz9I5?g}j`l~h3MEx7cC<LS=af~vx9Fnjgt^OT29~&C-t=HmPIpWUxf*Wr$
z#<_%rZ?9imX70}$iI@#$EKT6pckKy|E*6+KpKly(+?i>|w1m|JfBia;@}z(N#={52
zf?h9cqU9EYul35*9B}+BChmJu+N;AM6yg;{XRx^6w((utI`X#S@yI#1rT!Wjv3xGZ
z015}Oy<7HsmOY6N$W#j~GU7y?RjG6@Q$4}ouIOq{eKb2e+1tmrs2&aB;E{ne51Z|9
z#rtt<1?Oi&IWj`chDDFb!M9xOE<YiG7Gv<}8yPXSA6;K~?^_lmJJ?jl|Nr9^Go-qN
zbawwbSK8fJ4}N3V$i)Z?-wc0}BE>y%q^7ZLa%6Kvqn18uXRz{N!I4D&vK7C4JG(kh
zTd)1Xy!D+(&RcZ?&*;hcISg0ht94FPkB-(zFQynCsTSFu7<upnaY&qfcb202kK?>g
z@7{gJsz2j}YK<S)m$YlPNJd3m>7CtDT~2GsLS#bF6rabZ<FY7vE&4_(kIVJtG=b}?
zo!``xram|7T|Ine>-40Tz4>bhae8hzq-a|{m|vf~D!h7v)zi7=u7o{%idw1UV+57|
z>Ur&cX%?fiQFwxf%XmHA_eHwIN=|QVg~Jw+d+y-WiRp?`a&lIWXvm3!9M##c;v<Vc
zUO)AH$rk8KC|p7NvWh2}nU>WeNNq68n*GkVzyS8sRLVQdw7=Sd;wc6nbj3$sYz^s%
zo5PjHh{c~LLD4I8oTYnncX^5QtF=yemY(b0&mbjL=am!j&e_a^&qiZe*}5yQ)rS_%
zIepp(xw#+SILfW#vE6?6VXr?X_BgKa>vVHh6WeLo5drO{vE#bsw027KSfdSoe(pGl
z!zG$c<%H#kHIYEDv9;A>;mR>H3Rt()z~FdN^pM4+u&2HUh{2UG6^0n|BEI#nGDUoR
zd||tBUvOdh-Q8;EC~ka}xzY`}xggRKT0eihQCZ`0Y#XR|?pMskjYgW|jOJ@ov0OAK
z%Qq=p7x}0G=+bK>7L$BM<a@ghWgVT{h;x68KneHqZR=q%*Od+`3IQ6+V>bbB{|N8h
zPkTk@|4`J)>?Zge(ooQ8*VWeXp33?8RMIq{HLE9>u3vOxUUYJ-Njsv-HR#dsY?wP=
z^4d_gdXq(CbKUj$vi%G}2mSJM%Hj5n@oU^R77p_Pmp7WIDEn*DUy;5jAK)6GEhshI
z@j7DS`C7FVaU5xj(=CeOZ9Dg5(ggkC#bkn4T~A|BWLtkQwsSYQ`RD7CBZ=C+sKUXl
zOd(sFU)lX!y}9<+WREpCCSTAuF-XkHq;_}@d5&&Mu*~`7kDW}~@D%Xnk?=kJUhCy;
zSzA+J{LawL=0Q)|VQ>lr3Rky;_yLs-@#*k4OQ^TRF{`MmCQVH}00fW{v`Vz>?F+9d
z#r^rJl>y_4!_c$0*n8f8{Kci_hI8g#xbb9m!;;wcqS(i`!P=1&p(o<{Sqmpc2h;7r
zD$4>Z>^(1njk1%HCe|04`ZX^`R%acoJUTjYyHWNba_h-PpdiB`4|P+x|83u$pRZ+k
z592poZnmBboqTW?m`qE|nX9i`-eKFwZndr3vZ&G)y8O#_%wEn%q{a3{h~hzW)3MBC
ze;iY3#omiGZ*}az5Sg3eLHYL1=G;l&0*^NvgXd$9OB2(Z{muM_b2}y8U0#)Y4jZ&T
zxbKzDd~X`NrJuh3MN?vpFDKq-o0@D!`Np%JH0Ripqkf-X`8(?Utr+Tj&r1!nahv9R
zbB#~*4n@D|@ksn!5#Ntw6R0~JuJ~STUv5s#52IWq4OnJe+!aWGJj{POL1~caLPln$
zs<(G7_|e{78ma!MI{c`pC<}2{W05eJeldg(IUR-t$7)mM3ECK1@*7KrCOr3F;xpH`
zFFUDM-H@-ZdNF3#x0<(iudmO%oo)Cv8`hS3sJ~jcV2@w>pnOep`b$7)thVrFZ9c<2
zzM1SL%5MkFtUfC4?%}W5HeQVO3cN9BXS+zuxBO&YBiN?8P<yi9%cqaN{*kTJWcj(e
zqwSQB$s(2m?MH2Fgui~p?@%F2|9$CH=~2r_U9*#Y4h@ZntH%05$k2LwyqI2icz6tq
z91{={_Eou#OJJAj$?rkN&4d)WOyql^qUnNkXM1=vPUH4NWKrq;l^1@fL^@Dzo8W%x
z4UBL+g8iH82S?D&4C@(6G=#7t3!JkylkynUVRJwp3mg_<C4m$?63?%R69QLBL3CDj
zc6<IYCv?Qu*c8RWBO}wHzJ7YsImxcKF$Vb)NpNhq_Jznm80}L@aWwdG=zJym4^Sca
zYYWB1ygVhq9%(*&_;7Ls?<m2Y4*~05xOlODXr>y$@(du`5JoqW0Hjlp0G~X1C@q7j
zK-_ENmsCFZVO>3J!VZ7)CJ|J^>hp7aRu-17H*~pU;a$fgDo}!?{Q>RMZ_P;W6mu5}
zgxp+Zw{iT4prYyWGV}w|a~?;tva*JSg{8or#@GfUP^);i->=5WO}UU}h!Rr&iNPyH
zYO+vD{=m&>b*r8aewatW82tB#R#sMRW5;Z)tlfP5`pK}(HcA`Tg}EaYqdY{I8ZTL*
zaMA35nk>P_-lX%zk)ln2(*Wqu@5KweTo~46VPVP2$>{}vnx=|MB2>2Z?3{>@T^2fU
zb_+WG0BM|mwu;<;&Fkl@jvV6#%RCs}W&+ss*<X$O4VopqfN$PA@(!jF>MlG|LBNGS
zkbI*`v>Jg0mcU7@r9Kk^GsSVA)~^m0-GKg&eqv?iy}7wLm|BbW*qTp0JoKyq*kWK{
z;CjW`C1{prBYBFLVGJ}9xq<U9aKjnTOD1x_6UrXAL30v|QTxln2zJ$n57RmlB=UE6
z-P%8hXxrP{OXYpDLGqhtaNaAt?D9|{(}QOW-9LN5&sQ6MSaZX2A1<MQhKTR}=wWhB
zPFpph-JLtH@QA6cTMC6Ra0g>T2>bM2`_Tivq=4!~m&sG<$6*ng<ZR~~BKRR53ZQdZ
zYHIHQy`D&S^Tjs|4%6Kz<7cg)_8=aJq};b70?7?Q7GfP@EI}=R`NEd>dchi$ks{W>
z#wHK0+zYFI+oO4!tLeF~U2AeFRYP{C7{3CcV{D`Pt8t))A5FlWZNsBVPF`nVanaVy
zKYsj31mqGMBcqI_rsj;IBmn_ITkO+;0)2U48#)Ipcu7YL14M!%aOaWhvXxLpM{YuK
z!3=U1FErA;PUuXBIRh)`9D@#4!pk2P)oH^K33>}+^EySm85$8xNF&1Ds`s_8DRAYA
zh%f=<+u5gKuV>!orW1!ztv-MtwZO8aDSgO-%6zuMgd0MCFZk;v5lU7FvY)^sh8#IH
z{QpTng>{Ccar}PZ_xAN!uKjLoZEfv_zH|E+4y^CU1Hjki%4Y>6K!kB}B=)}hP8{eB
zt!K&X6d{-*;f05-Uq@O@L+24BHVG-Irj}OPD^jKy*HhHI7Ahi7R>_f1+rvjfeQ+$G
z?}~!H>)EK+SD^3e20}ExGywP;eRdT?!@QJ~tB;2(R0T$y-c#VfrA8aC^dL|kEIs@o
zR;uVM$Scya<4I0aMK6L$v<P5Pfj->8?Qs|_ZEbJY1kDw8U1pzTt6y7N3w-seP%-!d
z(vBRs2Os-6nIs7EfLNb%0n&Fc725E_n&MO*kYUNhq$DYr8B?>e%7XvFU>onoLs}SY
zATTE5|Jx$RO3}jTBAl8V4pUz}097Jk$WaIA#9e?h0+_|o8a?p@Wq7MX_n)rhcFJck
z3;Ow7ZWn_0GDeEYTQgM9CyPUtb1m+vilK;*0@6S|Gho@Jne1TZDm69L68d8EBUR-f
zK)eti%gD(1FVp5A?F=^rTJnfA-3X2SlL1OCdwLKdCg}88oFGMLAzVQ@J3E8uw03m-
zL|9mb>Wt=*g}BXG_L&o6Np1neiu>I@6(tf#z#+b7s`8*h5KzP7A7+?^d-@LxNl2Xj
z{Gbzqof<QWQUEi*(5ANd7w`E4fFZ_@SpVAO61c@c*|`<4@83YSZYs4tg<Hz{t*<+5
zwAMk#+Y$zn5IYR$E1PDuOk{0^9g^x5&&qE7S9%a5g$=jek`UjEXh7(u$Hh_c>{bD&
z1!jcxH=1a#U%zf>kwA^K{}$kjas_5zf;d5+Skk;RbpE_u6g=|_-tA8q+SLv8<;#~#
z9H!+{)6%lQs6oeG9aI5^nf5lwjsMQmfT72ro;f0BeHAQYqN};`Eanz;F1b;Es9<pD
zGh4xv9UFMlfI`z4Y<TIfzK+a4eSV^%1$u_Y?iM448~uM20ug)-O-&XK4n-u*gUUqm
z)2ArKcu`%*c3fEB-$UAmu88Y>QtJfPC<#o+*%}r$Xo1IqF?ig)iU(1$%s4>AW;A^U
z2M2G&GhMsJ%D`|FVWK=?FJi<iWI+tXic?5IkP%h5Y#svgod)NkfpgiF)B}hm?fG*w
zh3jc=Xi946>*r9~R(ExYsUlMf;}$?T+0fA6fJ8yTcb=D=eS>jB)S#+3-4RX2VF#?=
zXq8~Vz$j#fyn+HBApc}{Zb2i~*Ov`wXK?*?4BUbcSi$jiMg;>p_{|(hL&a#fy{J1q
zHI<Q)a;vbg5IQ6A?xt@5#V@-hRfv505q!iz<lbe(jezlefgfQ8ZiEkX>8FMpU|_77
zz=|4=qObDtX)-Cr{eS_zwCZZnt;NBff9>$NB^+KkG+_n~kAct8{uqG5$5RmCvsxiX
zV3%$nz4n&b4NJh$RIbz>v{61A_SzYVxRP*+4(uof4X3??LmrEC<bvV$o@e1h$ka1o
zq0j~R^)(e;-OQ2_-duRyi`V|9s)<Pw%vw_o&|CkVcQQt>7Nw(9aM(DcXLd-@jf*0J
zpvvh-0W%qHJ_Z`~4b^+ifDV5ksw^+BcKh}Vm?is7hC*B%45h(mh6h!JKogN(|BYoT
zGTcxwpdL=ECBeQ1r(xRe!RWb)>S}!$moEtn3gUxd)2avs10Wv`svYPT8?<T*!k6d;
zx8L@1LktHiTuDud9H?r18jN=vjvcH;soa~11^o}D6yO|~9v}wEg&2qb^v91M+jd@4
zs;&VCckt~(s%+Doz_cWR9Xk7gkqySRuqYVg311@{hs>?9smTf;D}Z{402t*nonQk*
zegRlei9^@W7d(K%Wg@f-5pnnEg%z3;h*2-+UI55dHmXVU5}Q^Iw=JH-@2~hZ$NLMd
z-@mIPfaa0-e_vXK3CE$6%DAd~2HcTd$S=el;h?s`mX$P9WuS1C@sS0|U`zq$*aaYA
zuDY@Os^!Y>>Ug|R8vuVGge(2@+o0nEbPjf$s$4qS6a-8i2n4SpnZI4{hk{^OUQrPc
zP}#k(e1eRed_G*;(B7B{iS9uQ9^gu&n3O|@&cj`1cmLi6U5&73WUxmF6@mW`BLmj-
ze8Hs3D=BGtcvJ!>@9UnU!u&(bYjA*T_{a3)O1K^Nt{vfd;1_s7q>}6(kU>y;(-_7D
zXGVO9X~c8P<K1tA9ltDSXH2QL0U;N{&54V&S21<hWhH6fq2ear_RbD~QFH}StWtit
zkG0E?)YXlKNx2fh%)|6XTQ&V^VKcR(g}^_1Fvj8QQ38tZ4Y3w1^+KcF&ylfnziS{B
z$eDa4<<K4RV%dv})*E=VP1-)t9Lh%~ECAO`r<)-24;#UW&`JGuB2R#gzn+L|fJu4_
z%Je!%LRJ>S+jHm6-K+Kd)ZN|fv_5sG2DU{Z4i?y`-@^b=FJg;s`o5{Z;+R5+9wHg1
zvQ>ETcO}&;+BeG3m2%QChWFyE?|BYYx#m;5jpJ@uA=}(t-3-Y8ijgEO73w*9Ld8rG
zy~t26WRsw)f)9QgnHl%yNuoznF(5cDFy&UT(FIxY4Vldr%s+{wdH(}hTTR2E^Ang=
z>K{EnPPNmKGY89AP>4w#el3Pwt(JWJIM;n66#R)nwVP^7OUsP*)8H}m6yga%T1HM|
z@E67D=5HpN0t2pWn1W*2Ax(^`Sp`?jbA4oSz5Ly~8sLDD7aWn1ky+OJ;?hYF5hB(D
zM356Cz^<i;6T}uo@UPgS7olop(fLV*41!8z9Sy=k=LId`;9c<T9<XHVu{P7`zqW|h
zjt?G?jlU+2>L4Gda@{^e28KU_5MDAg(vq!TLx$y5U0lkL_jXNY*F{H1d(3`ty#TLw
z+8wn(^blZNK)}&B5rZePzK+8p#c8k%5CZ@=ZabcBNq~cGZEgK!U{8-TFi<f;+~Cuv
zPxF95VA@Xyunb}Z2(Tm-u4PUyeCg_iI%bilT0pW@@R)K`3J{4fL=f3I+XLWf-iK|M
zHXhZ(V8l+&nc%+|aljY7sW;e2p+v`k{0Ll}fF#kkJ`{97;@;|6+)hFOq}+w9=3eo`
zU}VJzyqCrbHmS$E;{J1Z133J04AThh4}D+9>Q9&x2%^y55b7RwJvGO;X(g7GX*D)B
z&S?8^8CGY&B)YPtrDb4n@Kz4~zX#BU1N@EN&Vmk#pMkHJ`~_eC_}3~CfZTQJyldXQ
zd#3>f6XZo|F!R$5<BOtQB8q<*=%x;zj&)3bw1Tk<dGb$Pzz=*go=p=qz*hkrbxK}N
z?g#A0ar(|t9#wa07haH6@lgQreuxPc1MY1*xe-IhAB>>S3nxUW5ks25!Ny>$;KyU$
z!OX^{0FZAjef_u_JUo1kGk0ZVWq%bJ21@^Z1Z^)+Ac@>e486z*@}KXn0>gC!2rkZW
zBk?l&^EDMCqkI_5Vi6NF81p%)dDs?}1k&uhGAd@>{J`Wd1JR4jl5)n)nJT1U#mA|L
z|5|Zh1pPcBpe`((=wA<`0Mi?LfSeZ(1_X$Aoe`{8HxF8J`*%qr7Y0k>X41O(#|&u5
z1p{a2CC9CRh}xvTz0g$bcJLyO)RYtph=}IL>gvcj^)n(3Jz5&Bs0#mM$F6`K`%^*3
zMr&*i4-iof7y?7s4G(x5kJBtmdlpq%S_-MP8l+dq7#{%0jWRPcfsN<Bbm{y%U~i6(
zB@ujNs1YQY2&(88SO|MR-H<l7LQk5uGA0LVU3c!>5q6k*8SiuK4qE}_1cvRNF?swe
z#Axpja7dl1_bI`})(b8F<6@s91BHi8RJf2^!du`HAt&_$=uPF;Pd&|h_p*?69q&vX
z|3Q|c<E~1|ji*?`aD*Rl^9&*awByLo+DcoDk^!q7yzRiu%*+z-kKmPg5<26A)BzCa
zVA)&(ZVQpSW;6z0c&2$2GhhI!xJ>#H0E#dFvx0X8n~|U253Tu}*RRhaNJ5^6Z7NDi
zAuvn&(@%5o-=()ZgrJN*O-_Fv10`)c6P6)7`cFr%N8q{Drw!J0AF#n&FvHrqva=8(
zvRq)6d*Pom_>RCC7WvQ5OVO498I!giKxxD-60vAAcLX>u3VLj2z-V4B;5oAZ=DQEi
z*!&&DRayOL=<m#iBzz3y5&78SHIT>hS1K@hG=T6ysV9*fJ%r83&E;D;j)e*jlxw0*
zjOFD6@0*(b8h93oMCXW7aDY<$Eeg1TQdOCTAZf=0)T_>hvSrg_Z}=f=fWQ(XtTc3+
zIL(8thT!04nK<D;CUG20A}U`o&btN<uw448hYNx-Kq+5(N&bu8I0;#+K}JK9&sDoC
zT|VLxi-e~z<wD3d4sqyTe}w4t2gg)4bPQUsIri!<5-gN}NwK#4%!aw8fy~LZwzc&F
zdfpkT$}FItwsl8kFtc`{t5Wda1>`p8X|LLSzdCoy@8@a3W<A#lO}JS-geFstzP^g+
z`GtioKuEJdR4#G2`fSv5Ly<$LKoVLx#drL}{&g;p<>L<{Sw2vtW9{EE2|R(v*Ovob
zd$_K!=PB;B(JtWeCn;cR=fD{z`+t4p5B!IU^^buMHnGe{w5#gy=>(5e$W|oB<^4|1
z;NYB`?9ELD{l^<t7eAb?zMx6jWF%dga<f(PYO0i~WXjbGS8ri2ys3~kdV@DN<1mNS
zxXK9sM@z{|w&Kj$XrxTU?hM;T1P|6!xo)?0bchDfxH5Ib?IsLHJH~r;x=)H}p@1Kl
z=g;e5B|`?R>Ene|to1F!ncz#;pMoc*c~?ZMj>W-6{uL1V!6LU;Ssy8_BgtjOmTu#W
z*;kbc&J&_&u^2x$Eb3H)e+4a;3I)54l3;fj)a7nMb}TD>_7E_^%rFnw(1Mc*-}cWx
zH%6Sd^=y~ppdy5=cO52ae<=^YkfX<;Qz6m0N{AZ8u<I5*hYbI2b0|$Z=ubCH&9i`~
z9Q`;c`0q1(CEwsnUd7x{xq1e5jlp-NqUAEW5f>NBO@@o{OS2-n(A0{C>Ev5b@Jxqt
zpL)a7(=5WmF|CnYw$rT1Hncb><e#xT3X6Ytke&Gg1|5u3J1KSZ)pJQ4G`k#*bmD1B
zGR+$okpW~C7)7xJRACsaCMy_7iBg-97?P`U#(7Ua^8UwzNZ~=&-9IwY9^;RT*!)V9
zZIlAYIxP;}4`R8OD3m>2{i72|n1iACCIj2g(qYD^3p^Dwpjz$k5cU5jZP2f9Zat;-
zyLE%sSPius5?f-T!VWA@$NFL|0v}4!<=i%gS+ibP?Fj1v5~7X)U!}vUk@2Q4>(&0h
z50TU%@s-0yw7-yk<~xR7y|P0|M*kECGw=rIxg;gOa29*$UzUS<^TJznfaL*pXk-<t
zI|aSjzYNFm^F^q8iGmZyuYlfn#Z8CrA^~vQm>Z~zcS_df=T*Dejw^Kg(D->yE#Z9^
zmN`*4FAasiyQXN>PLQniFIlinXj+oz_LE6};03z4p3O(T&za#@WsE?VGR4rqGePyg
z8-p@orJfTc5%(96aT$U~^~R~8PQZqL{ykN|p_j6Ar{}|RQ^F(Kf6_A(V|;z1F_IWw
z^+$>9CNkpymhOcyp?eSv=$t@1H3p#97qy%vO@If;L+V4|7z2fUs^%i+LEw~fgDZg`
zXn7oI<HWDnv2FCgDLaJ65V?d2J6j=6a1|3rD~0-1jEnOxNib)C!=IDFRc>QnioiK0
zJBYO5Avy^_)rIowR+p}#AZq@Ysh$qBtoZ{|oV}IKkLXYU$L5z4qNVr`%HlF$B(HKm
zy|E7UI5>=`)8bnmxXvCvT*rqp>G?I<0{4xF*UupfYBiFiPhS>sVJBnP`GICc)`<U0
z6BK9=t;+qE#Ha}woUyf4E3PndxTX#Yiwd3&9{&8g9_C*&5f1wtd$vaNqyvWmL(QFi
zIuN+{kL-`%#i94(IcNRM*Aqp0qQf9TI`#J#<n-9J`(&1-epu#n93({Mf0Y_$(tZGy
z=U~9R7l5U77+5*M8<*{0^z1*Bni)-t{>_hrYl>VXLcrk^E*9eh=lN21PW2f#eE}>!
z3w-@L1>Pm80$^u<+9-H;2X_{3AOUKI!57zeZwSqZ0`(QZjXs6r*iV&OOpm*HDrX=o
zgA}N97j@$Rw*V71=pZMOgP{_2Q^`410YitZ`V|E_X9{3{g^A{w3*_iFd7M)?#bsw(
zPlK##p>CYPd6>)7YG#1?b4I89u__NRK?wa|`xGs@)siRW@<TKcjrd*IQu7fuH2eT{
z%z#KNj8^LDKZKfLgoC;N!efNHa5Mcq*w0I;MK_Rx4OeDR!okiW2YUhL5&v6-%$tYY
z80+Y&r-7PW$}MWCnA;9=r_fJtOU@|JTKK<aq!T?b#eLp`kbnGkJwc~;Ijc#36Z!{n
zf&FCiJp9<6p0=0<hmKoG@oCUsu@yw!Fhbq#L!H7&mSdoS!)Id)J{OO%KKAqY!{qDj
z>-|Cw7uat2&y|K245c4~#D9L{+ZO4oD#(q)9mu;5HnD&Jmre8_x0^g8z#Ay>$Nc35
zc>JP_<U<Ltb2ZD1y&ycaXYs#z;$QMV^*>+sKV<wL0r?+2`+sCXHc2Yn4$mIdKIZx$
z?(}?lxKgpDCnd$2OI*R9H6n>Ox_eHMy8leHZ<{yH^LknVa(HbZ7RsKaZ}vzqF1j_T
z0l2;bHw1|gFE)hESzYYUqX-##E6Kc%8>tkZs@H$w((r-SL|XnWCtqs&yRPYqjgH*d
zlOK#S+!OU{ju+e-_DTk{dX%F1&!u+0ZL~ig`lR5r(XCmS!0s&H8g@X@EaW`7Igz|M
zKI$c(AV!#aH}baFYh?)x)!0rfO=0}4JF8*c-g{!dR_>fVuwL3-J`>Ypo>Q?i{K#N2
z*P++i$#v(;0P%I)=p!7AADxZ<17hT)h3Db;FF96>KM*(K(CV1U3ey^sp$gEL{D_jz
z#S+yolS^*kKmSOxY8E+B@@NmQ-HK8BR;6ctWTMsM%W7%n#Z_PToJaU$hRW(Db;kOP
zRGRb$sSc`>(hSm+%&bH!-cJs{o9LMfpdf7Yo<D18Ii99_K|%`E>^}NuV<y09tL?NE
zZ{n#MwfwB_LW^!ptGjEa`<tm&qYM^tuM%j*gGcIylh5&fytj`f%wQN>V4jc4wliea
zNeaV$={&ZNQ#x@8vnqKc-AUeJZizRcBlP!n6K=AJZE(3VdfE*};|t$Z&f@eMEqqwO
zDBzf1o+v-3;9F5n!H)a8gHuEM-ozjctIM~3PqcIwu}Wj*RhD<E+mZ34D$B$Bp?M#w
z?v~qp)gcfM7d-EL`0(m9dsH)Sy8$L=SM^HPhOPqKL#1b%5$#Y--c^$!H(gXchryz*
zFr$Wy=?Zoz6-Dztc^?-EHa(7}e{2=Gk=ZSwXiyjIBJ{BNLu#^l_fY1>t2)AG0-b4J
zTiy+wzd6vSi+Dccfe|8RZgAow7{v26-UvH|lPmn0L^3GwQ)mMT?efwoxI!`Ebhwt=
zcECHDb^eQfD%+<V*vM7hZX{Dl?s_>Ge{rFk897=h)1iD^k5@Jv#Pz#^&<_Oqqm+dA
zO?KYQ1|vG|6*ifSrPp-xX1__L6A6h%L`e0v({9GPswN4@t7pGDa!NOEL^ts*aQ?(&
zS4~EF39j?7X?4AGem6)AL4S)LUN{6s1thr9T)4@C{th@e3OLF4bZx{7&@V+`W$D)s
z2F&@06onnWDgT6dUPLp_iOzcPQ}HCCmU7R3V5#&zftkB_`hb>yeOu>rB3pV#*RuoS
zjJX0%jrFC#`P9ODXz^qn+W5UCuE2|2)LRA{r_H{-?X%C7kRDGZHJ>|t``&3{*HZqS
zXyy&RJ&M+SX(jm&s_{1innS63XnyT<`PV6R)rh#hO6RXr$$Hbdbs_X*Uqnn}*Up$u
z^Mj$s?AXbl`lcnP>CMv(zUj$D%&ZJ(di1*W+d3{I*9#1K@%mxBngqYa@kF`=cV_J?
zk2*QzckVvC-^Fh7F`dfZ{aJtL;X>q1r^(M``#VGD3Y$z{#0@fses1w1sPLc*YELIV
zk$Ls*noEwh+uDHT+m5<^TeaI}S6Wch@*Y#EhMfuewS+z&(l%t3yWu)7eSVPd+=`Er
zr-*k6$@+BVNL65ea@K28ek7~c!uU+P;Jq_6VxDq!k4G4OwoR?zj`<9SSX<6<P>p!~
zun%l>>>G+0yfR$zEHA!>c{zMG=wb@|hqdQo8!L{D%wrjApprLuMRZi5UUIXxKS=$y
zKi8OY-AT1}PMvPa{3G#&w#Q#~uImLp)argxsA1C86e9iUah!xls|j6Oy2F%@p=I{f
z%|8<i!qsMxpY`{|hie&Xzc5<WO0E5Fc2tlW%|q;>r2Eo~FeJC3Wkx@!ML$)Olpwj@
zgM$-+<ISscNe2G`4)%5*ft9KWK2{^{1>-}1^E#7)H*e?gqcUj{KMQY8a>fi~l@A=b
z>DzjI>dw9o`=xc;Cbv%c=ze+?&k>U#+)0sYcsU!pJM^TF?7T@iR&3PTFw)A^a-!mA
zB=#G9Q(Ubn8Qp_OhT6*WT;86nd4kSv*^f=>LNc14t+`Kcy2q^AbICjqUU)|p^u(4w
zTfx@VX2wB(WKOM!^Jr(MZk|)-sPgsk@EV&}YEI|3&!Lwc>JCX{Iw+Hu@wRy0c-ISv
z-R|iO>K9%&KSR+O$t?Fi?7+8Cc$V8~LG+?Ir{lq`o}d2Nb8E@69bI4fMf~@S<;;J3
zR0gP@3H?z}`Pjc;NY{kBW<*jr)osa{h+Wuq*CPs+8PE?3h&BJ{GB^{*b^cfNlw#(P
zty<~J9Wk^X`A6L!)2mB~El%^}2^xu>ELq)Ieu7Ot@$&Bmu0I~NUp=9$_8y<&_X^M4
zjMP_R)SxQpwHOOyh1V>=wf3&Aym#i(J)|mjWjQLEmyPKWk#}R&nj~X3dN*`8L^sIM
zu+F~3Y*hZY<*o$Ry@iT&L9Vj(xSc(>!n|un-<xBYKf{cXJu%zII;}KGuEYT1`(G`^
zJ9kzU%!0a_yJ`$_Jo+9nmCrHVdgPqI;nu?@%`HLc_SlTCAcMxsE>%uh?An3)-&lU}
z=gobv$h3*?krbfg<DS<iiZeI`XuHSaI;eV*`mGS!>3%yMlujQ>eNAp5F{%hZs*t8W
z<d$0?R{)U_ar#Aha*|(J*l&hjgC17aJyDrqVv)*PIqMFu1Pc|<aq2Fdqod~x-49d~
zZC4L67h$jU#P?&MbfPUQGF9Lu4|gE{#&GLg8Qy{EfuTw2reR&#S83A%l%7Z-13tO8
z+fs!z*>{zqUY0)t@m~T3T{Ir_SQyJT#^eObMr<4?Cq6kDNiwQEdAmGT!z9|-F)uD}
zUEFtxP;v1Rqt7Rj3)jtjo3|PSu`jU|k0&ngD-`O8l<jaAI@5}@3>13=6`&um6`U0+
z?j9TqeRW@kMxU<wcDC7+_hh#!&+|x&?OSoYW)8w1UpiWkH2dXF?^2sQ5}QpV@bi4#
zS!frdXW!PK_1rV$v?<Z_<^GTRHS}N0HWIuwW4fl_&pwx-BkUGh7gKf(=NH+T#it`$
zCm!Z#+D@U@G<2+ejn`+bI2#gYCp1(0#re{wM{f^I=zbGDDbKYK;dE>uNM%tiy6W^`
z*4rff+s8{mv;JeAjgv#VBlpo8R}_4(5wG>jC_cOh6qku<n`7~=-gR0!Wc<A%*E3<z
zxCOiru`GU=@;i`kCVoYLjXZU>(Vf#g({=R14ujr<8P-jDXI*ZsIQ0|j0~fw_t*^I<
ztWU??)ZN)g$QG~|8_K1(we9&R;9YvFa8&FX+L}{)WI5MjvVRQ!e1bVyVSxS2w_nLF
zTw29EQ|dF9u+oWiiZAaX$tUeHVp@J1X4U8MP9mm7c?cT;qU7cv@YTA3q<$y@XL%{u
z3;RZ~-uejVILWm4;KST=`pUvxKDo8!@9MXNGNfshgRi6rYW*?F^)Cox>(96b101hj
zlEz}c5lPQ)g(MnIOJM2IuvJ$Yea2RrHV9k8_Eqpp#$XbQ=Hn5Sui7UCrPM(Tn)m9M
z(tc$%Z+T}q-%0DOJJSC0l7miYwfCE@*zK;4)CGU;y}{4+VVcqM*<|8RQvGgdZl@JQ
z{9dqPB{vP%<Hu5m>55LWgwuq)JnvPK*YC)VrBi7)S4t=>ac<ZNS)Wm7Z;Ck^q;acK
z?_-Vb1*+m55x&aZYM<Ho8;XauSey^a*73zpXLe|~i1;Wjv!K~#H#($g(<!ez<J2VT
z&^Z{Nw{@%8{(LX=ldI8<hCQ{$<P{^{5|=iy{?!c^jj{$k&e=VKcIl51&Z8fb)aUk3
za`%W%HVM}*OgjzU5K3cmlvZZ@_^WC}|H{Pb8a{^yiP4we4+C_+Tp`aOULHxGTRMYJ
zC>&ojYC8Df?KPuG>Y*7Qh2fjlZ<~9(gc8<isyV)DkZXPy@Q$6|vSQ#=_peVcevtU>
zlj+OTUsjkp=*m`2uYWp<yy?&FtrC4hG*gqtB(sG))@r=;O*8K@{jH61@yB=j!cVM>
z3;FlI*!+I-jk<QQAyms)dE#(nc70u}?Xy)f?vnp%OD6$2^BL>idkboo%@0Cw>Cp0z
z_^+Nt;@bu~2m!HiBu(*wgLpXl_aQ1Np=8H3#veaEH#4VSfUaQnJ}S>*^nDU*N`rnt
ziZPS$?LSuUmoa_B9&feRQ>}U5w{t5mALBB8xz7P!^)2Z_a>MfgMfZ?{Lfil&xo98W
zNTn&dF~b+dk#SFXUN)<u{!vn)>wTip=aOZsmo>6WUnVm$zao`7{_@l!{bQ;BX_}xr
zE4TPvTP6~+`!mfy5kIg0+!N~_Q>+CK7vI@G;LDUsxU^=qRB~E+*Pi^ZT-*)Ce%pK4
z7Ose~8_p9Y3-dQ3Pr<W^o7OJWF<vm-_M$8Kq<m@?9p_)q|6bx^kK+f@8E01AOA67P
z=Y)dvsAS1%-KyHdj_-3wjS}&qi*AoUo)#g=yt_A(*f+z<AikoYqwX&mqFt(Zla6Sk
z?4FT%gAzF2KMxw-e@W9-2xn=3ub*?`cGPe^n8BZ0%;m)4eCSDm+5C*qcmGD4(A4*$
zKi5pz`j!-azH%0KuTjoT^w!?fCu$@Sa2T~PIqzZJNw=0Pwe8udTHcr^ovS6>AbQR5
zEQXFdio?GIQE5pO5<}uQ_TBv(>dJgJ!Cy^@?rk`Wa0*_kQGhO}W%M2Txr@e4{u@W;
zhOmTZX7@>0-pc38nIMrNL-kp{KQTOs^{73rwY&UL>FrV0g5A(3>t_X=_pjeN$A0OU
z+mvqF#~_*@TgUq-O-D0WJ11Lk^WJq$!JB{d<`}L|7k2p)ZOYd$Eo2swsg-Mwk%kbu
zEeXW4wrV{WBB)Nh2BQdu$HvCNmx8$Cty<JMM{f3>G5C5x$0g#o;w@Ew$+A4EYO@eK
zyU%m0y}wPL_>ek}A4k1T)W%o~<q?zRUEpa9mnr!0T_7;CE9-Q6`Hf=rw|cELYdU+)
zS`O9M5{mmWJU$d-2W*$lUTnTjWxmwA_f@IJjbN`J;bKPQE9K+DGKEZ!uMuw66g#(j
zhBT@@-18%%IWLMVLC8uy;w7omzc!taAA3hsW>oBKPM17p&el^$w&&xU$!uv_!nj4}
z*cD46>l)YjAF5^}?Vcf9_@&S5Y0SBjPoKJ0iSZLneWLM2iCTIbV<s=Aw^Kwm|LcP|
z$)4bes?T9(43of?K7Y$znCUNF%!4jN=LhRM;R_|M?aG195AHP!e+&t_YWt1qlr$!6
z*iDWasReJ6Kw|krkKZSscp5R%qk8y=k+!Q7F~&D0R>0YDX%@$PyM}G`#v%0hL@=$H
zGrIh4zALQ1;SqJ8x*)Y(eibWPI3J%sc|&D__c@2ZxZWkjtR8$@a=$%8c)QnK$_&8>
zx4T|!Ud%r!CoSJ@d6b<eH@DWD=0rOP4WM58Bu<(aULFt*mdWfcWqaj(rf;%gceuuZ
zv|VGq$*=$V-S;dXWQ&Y$6D1q4@vD_pcdCW1T=R&VJ(Y#swK}re^gh{+v43!-mjS)L
zx#HJaVrNV3^Wl4&q2r+aS&`ClmQU>6p8_f{ezCQx>|=OJ-4{?oUN*^tH6QiYzrSH1
z4A0PfE6=ttQy4HgAWbWsc#m4?eeUwG7}LhuTKf~$(dRx%7r$5EU3%CwEa5o}9?5t*
zbm56Je{xel16p{;O!v<u--VGc!QZv#TpJIr2fa-WXBk_x`zi*$OT=cb#v$uq?#<$N
zgyd)Myr=)$4_i$@VylzX9~XHbVYFooV8mgmoB2!v)Xy8k>Y(04)tWmi;Ph)XQ)9j{
z0p)!++HkgCe}9x!D4-rM9YrUr725g;-)|9L+xhbP;79xWUY`<X^4G1-XAt<s?asG2
zzWVTD_44zZ{GCY?TfYcrFWi%+ZLae^zVTSSx~B+dYOyi$3UBZHZc>5AZ?^_aM@Y}@
z6<zwD3Qo$}WR_|vii%&#+Fd0$wl(SA_$!f%2A}nKUN4$T5afLRh=og}Vc>W~_<VTo
z)PTp0aQds)<f44N)qYYwSkroe{YJ6w<l^$8VbnE0zUB`zXTEuVVrM$_qcg(vhdvBg
zI=dE3W7bxXKed)?hh@>}sZl<?Z;=*h1b&Oy=U1NX63mOui1G~>1=Ukt%xCTCr`(et
zogU6GiV$Si`k)NN@5;)5MS~yL>2IZn#2UsK_sI63hvMs-fW#Vy>`#|1lBjSXPJOb`
zfY5W<^$a`iq6z-^=<>iB+(ESTAh8`eO83@;Ww3R7<5Qo2d|9^~rZh%$k@}fFON{dD
zG)uwaV_yCs$r+E0b-sWeZ2axR_xDv@?w`=akozq!)ki#Aq<!8mG07506%#rqC6isV
zOwFfsP<LDqoaQ}br2jnxpuSI^sK_MD22Q6d6`P8Fu52!>#AYaQOndaJrNwL6sE$q7
zs@8~oCqaF1FXp2kBh{P9!;X6G>5DW*HwC2{xrEFVU3m%UU$4k3X{$r8mg%zYXirE*
z)!>y=jWr*Y3<$_$rdA$4c;!0RQFp>Z{w_Sj>4SXCv?{T9zMz$f>WG*m#)*k!>PV#;
z{Z#)n^f$r5z98|%UJ@9dA9C$Dg`<P2D0<F_!;Y)Jf}z(z4Sa!Me&la3|NQ>hj6ULV
z51U<w>xHtKM9xo2(g+ks&;OdI;h=ldxi;iejYmt!VEyUq+%BkGY1X$l3c5_8=~NrM
zGNAHQV`9S<cK&!8Cs9zCTY8)RWLR=LKTbUVu0e>ddV6dXZ%Ie`{agu-w+ijw=cOSb
zI(nY2??TIGoSZ(Z!{9F&zL)ym+_GMehLE`Yo|1g8+abkE$QQRTjaK2O^x7Xjxn4ft
za{Og9ckp{@@fYIRjq1l?*l+Hc3p(?KGwOB89^K<bXdW<epXvWMUaOn^@7?){*>Mj^
zQ6Lr`V<0Qx0XbsO-Y_eas+^FmXFtNZ)~qBn@TorYnU9sy`R!|HVUcqad_$ZG!Yg#G
znH|z`>3ELu9ZENQAC*GvpwU}Y|C@Ip^9<MA+nv6Tz1u+zo^r0nkFq)%=P{w>)p!$U
zZVFeWxF7$yV<OdZ#(hb`@RSOhjsYgFx03kEaLIc$7NUoEY&}(zjXw8Pw^tiZUDhZr
zT@}n>jA)u+zg7CR-+}ydj@Jn3qW%m-<j+sN4ZZE$V|Cdcg3#5`4^^MvZ%)dF{%L>c
zoaxd}&N(N{Mfbb9I#A)NS+gAIuNV2uV2?mh<h@4*m64rZG}4J8Z)sNoeludHSZRfj
zo+5XcFB#smB)9D^B(&SVE=1rQ<NJXFsELH&tM%s3c(VATq5;~GRR-fUZ=4=e;`}f3
zW$L#Y0=}vniga9_!@5@E`+0tt=Y3u-wEO5hbKDD!<Cl5ypRkTe>-`3p;k6NB-j$6t
zjZ7J_EU3mcdw7XX+c$S_1>Su1>g(r`Ctea;t4@n@yhN1Wo7z(N)Mp4~2}GTB4LXBQ
zUo-Ss=}ltz?8u%U9CcTB_AlA=lijur(%iPEvDeLTrn$7Gt&1)AYmOw5qC{c7M-aKl
z$w{&-Mqk45x|f7Kzh;odt^7KpXYulJaw#4QGZOa$_lm9?1$w!7eu=Oi1)EQsem%WY
zr;zB{6IL7XG!e2Km3vpcmWCN8P9OX(?D_0|cdGh&nB1#mvey?-?i`9`tq(Y(<J<E}
zRDA-<P07^+ypOINeR7m2jUG1YQN=_8g|*o5)^wN+QBv&U@1XLAgZl+v$AtS*pn07D
z9i%|YHYPf#T_*HrlJOQXs1P!-Kd{xE803mQ-%U(|dN_Y3qZjIKr`Jd*sODX4WyoTF
zlK-lGUe6F1Rj<`$wW^Jo+I^JP#-Ld-;bjgS{%qp>E0Cqalhg2UH;~~yd*GKBw*tw=
z?=&8)GKjOY$M1Ve7`R1|oq4(%uraeUoYa2(K6yTA6#e7N6F;UJY7Rb27k)ovZjE7z
zl(o6J;3-{xvKP7=AVXyRw(${<<Dc3GPowZST&^DDf7Qj5?~U?Fbd@?@^fzNBe3Z$P
zMw<F0UBQu6?NiRfJMOu+%jh+0gd{e)9i=ab6`r^@-e1YXeu%;qS;^?<k$&l7C2dcJ
zS{eH!`)tf|((cli_+Xxy)@$#~=t2(cs0Q|P$6}&4HQrr%t3u^;K$?O5M*Ysca%s)0
z?p4^X;S|f#?6K$_Yusp2XZiakI^RtTh5Phtm<C$zzME<ZbKZNopBsE$eI!uUBhEj=
z^2(sIZ<D}+lD-R(K0i4N7n^!UpJ~UK(3UW{*vYPY!X)*>ApNq#snuiOZSj6Xjh;7K
zEX6dwnGHqv&dnh%Me+?&)J12$EWBn~h6}bbo)>D-FU0&N%m|SPY1QG8oR~Ef#xx}5
zL~}4vxis~wIu;Z<#o<_-f%Ul!XC9W)->0|7)2;+^|J26NK#sgUe(2seLu8+3eN>)d
za^eyqI4{!YWyLLPY)&KgosV4WW%bwti>u)tvx=h^{CqNoD~-dkOoQw4WFEU$MVMHi
zw%eysjkj^3+xQH5#mqJ(lW2p3P;b4ViSAXeOzZ1gZ@#As^Yu+97JccsbenbY-Bsmz
zZ`MBt>h$|v$LD@!{-8|F?EC61-jnc)>E7CEl{`;J07IqPtC-Q6U-4Qwa^^0V>C<qs
z@e5XUOShhUR=?3nZG3&J#{2PH=}uO*>Yv4x<_l6>(u+HKB456!Z;>~;pCvT8r~XX0
zekeuff3f%0QCUV`+aLmx(xr4L9nu|wfOL1afJk=>2+}AmEscnDr!>;t2+~M*e&<Gi
zGw;mzeY0l%o%OEuEdP-Ext|mJ?6c$A*DklY_CPLm)m?9Cn=1-kA*{kz?a%;@?(7!f
zv@SDqy0Cxh+|O;I;;#`kKUVdBrtR?-E;76W!9oM}q^gS|{$2ZZB;<7IS0)%=?Sy_q
zv1W$|>kkzd@Sdw_MZX6ZlIj^BGc1$NdL~M6?))DAa^N2^Yq^=9Br2eRv+l{NvtUth
znHN!@*ul1C)U{sW0tdw|BE8om6!I_*J@)v!K2<V7$#F&XA__f7?;$|}aes0=)(O*}
zJTQLGL}bhV+6)GUAilK-uEhSoS7H_!dA&T_==x^(o1o5t7i_<~3G+AymzA6Da9`{n
z_Ba~%i&Q-_Euv?_D6*HDrJua^{6n3}?;xuApw3gv9FeqOBrlHA?-19%i3W;GF~X@8
zaZc;H+x+YUfpoZM22Esh+l>5oHQ9sK?+i3jk_6&-SV6o!c+#1B<zrRM<h{YpQT8l`
z0D>;H`vdDygg6u@Pezm#W}k~wvls}cD)!Udx)#XfQP_p9bWE*j?3UW%n`!-8hi8Br
zI}{LqGBJt)0(S`#ewS>}9$+@TZ-v+bVBaYhK{><+FMxcN1|6p|W-3i`?K*jXX)^0J
zdT@do!H+Jxb5c>UbV<64prOog{azDMU)-g938+T=U2oEb9>X%3a0DuizcmG#)OrAf
z8wcpuwd_SkZ@q>`ryMv_Wg`RdQ&FqZcM8E-WJ|uAm^hH*@l#UdK3sG$Lj>VAKv_71
zRbYN1HP;G5{=5dHw!#x)N{x`ChhNRv_kts5-?ny=1u7m#0p^x-P*Oqy<$=^u_CSF5
z)oBiZ{8{IY6sXWpU|OY(xkn%UZm1LmY~yr*oo73C*W_oSmi0sYRZyT%DZeSi1=yXp
zK*IoYJ}fLO4$!Hm-_f7+rSM-o48YgJvUT}k!v67b&AbJcQ!WjZol;$8{V!+;c7dyD
z$`X$g8f!hvQoV}TSmOmMlG~~&QKjPgppS}|{vDu!MwnNQWLlxu__o8o2wJ)+fo6PO
zoyf<(0nB-`pF904UY_#}U~0?<Buv_Q%U*9lEx>Un(Zb}*l|YOBWO2}AFw&89D`*2T
z|LHz5WW(KJMMD{#4Fe)g>WQ5NrPx<V3<e+r6I*Ub+BephAG%*BRn$3{Rb;`eoP&Q7
zA}G;=?t9}95_w0sQrg2?5*y0Rvi|UW>Y&lAHe@e{QwBZoQ_7Bk&`@efiNB@?mvO^X
zn?wN*jU~W{HD$qjeFMym6`*uDeKL+t<voBoud&N#&}ey?VvT6fYL#Utif%K=XRKR|
z<-ZWPnbDxtZ{Rt=-86g;$l`|qC%QLv&uHM;&h!d<q%h4e#(psNGK4V$sB?gN!Guk>
ztLli~nl5-7HKCf>!(Wx}_?)39tipwv^r*<SJ@!Ftv+f5&>XM|1!%~CcEh$IAfzqzr
zw_yS7dOxV_i2+g%%4pGt{$D4NPAx}R=YEu{kS+D`X(j-tUxnjr^Z-Kl5g>-%`th_%
zly#2l@`@x+Kwm$9(1C23eGxR%8JVwlrSrKy7g~Afzy=s5oIs1*^n|CcqD6l9rMIGh
zW-d9P7uuC<<aW==s>e1mfWc%2TZ0_R&a1yLnEqW_1{D7{Ij$g#(!oAt)_bPSDxJjK
zIh!7yix!0GcDb(v<%uG<cJ#`F<ah1@$l>{-HGRa^WLjw59>Y8qC%gb_qZ#D)B<%(h
zR&b9R$v;i@n)jWLpKvTDB6MWZFz@&OJg}Up>MJe0KCgG$rTXQo3WJ1)CUhE^1UZ#v
zw1FaLy<XghsM_2&I!OMzhKvm?P;Qm$5llE`1n8*^|5dc*(V&kk+G)dQu(#r1ea-|j
zSCDcMw*7)-pp;Nc)?YM`_HpikIunxrLif2Vv1o>G3MNAhBsK4Jl=li=ARP}NSb*1w
zh=f4{w!>eZW&{89w5L;TAfAgM+e3d_LjIe^R^ZXG+6K0&lidn>o!1s|smh0)hx!>i
zp(d!lK`oyUZ?jbp@3Lp_=Cam9@iKIS46@;UzM>t1ES4GJwmzTlF}Pk<JJ`f&t<!Y%
zr*9ok5cC9m-ha%K64y!_eBpXmUo3khC~I-YBX7x;m#6#CAp4lN%uRQ2)L-HHvYy8W
zr00`|EkmZ>PBoL5`EUhO-HvLpDb~G7BEA|RI(xfF;nHq@GADky-is!kOwXG$m=<&x
z%lH!H1nncYrd$YM4d`{k-L`gz=Mm}b%bv!H{koQ2u%K(|ykxq&xnW7YItp}P7FzYW
zcoxNJ1LgH{x$|T{C$kBDo{!19lF(%YMo%M6jlQ6x|80{)Do6??vaR(>4>H(%28qiv
zM&heHbW)0@gt2|y;cHwLAAvgGbyAY4npEag0hzfq0l8Ykpb;w$F9d*UNDzq*+v4PX
z1k;e!iVWRjLr)FkEqwPD8ZALJLb6#nz{mlx-RrbbPtX#qX$*A!>^BA2<(Clav_Re&
z`FfALbr8V&tEj5RPSDMFC9;-Jpa7=WoQ;jw#VU1C5F!&Wjb|pwzkCTVmx!5il0S?w
z^fEjEXdv>XxO-fMNXgWXGk8?fPr?_vSk<}&TFgpEXqP9!Htn#9+!hp!?Q$Q^Q*Nhj
zP38EgW1hMz7tU=77LJtpYdj~o<rDuzbqVD}beFRSqT?6Hru<Q288#hKTwR!FMGTxt
z<XLE!i9CO$F8D;#raQJLlO!vTPoSk@3`d}9<Xc>HGIv$DNKbxyN%m-S$38S~;#<gV
zGU;6Xo_%DWPV*qmu|tlA-~cbGe*}U{cE4jho#>JQ^MO9&4O=DgV(`c8S8EQ}`|<M}
zzgF~@r@B*71>tM{<@o7l!)gn<Mx>Z0`S#%H;opLM2-9N@DXhITBo8Z(9+kXozIknI
z#}Slt4NK;Bp_91`xy2z)ah}wPw;7nY<10@b&A&8RHz~RvJTZkXvlq;TlfR$C?1v)I
z+@F+Beo1LJJHsxZcDv2`4`>Np06G<pWr`tstdrD6u1`_MJQO~S@4Ene6RN*;>RoK)
z%v<@o(*i&vCn?Z}BO5fC{BT)iJsG(G$`uXGGLr^C)!oY`fZ)ZvaPa0`1VCr2PK=3G
zP%nV|BH_sUe4nB;&k&z%8-TIDG9umhWn{h-1-r;aaSsz5GUmq8*+(n5K`vb-(DBAA
zxTepMPLGGOs;PWV?Mgbp<LB2@5&olaHF?MQbpl1hD@fTwRx%GajN7HNHFJ7nR~0~3
zBAv$2s$7WlOHwf(e4@iQhfp|q*fiMhmO-*SDEcw4uypBVUx7ZQkQ#+zWrj@R2A%kG
zMvC4B+j#}2m1Q-os15;((yfVp-`AOxZ|%>|E*IM=iGsaK@bYjK@~~&G*H@*@3hZ(;
zh_ALfff<X}TAu!$?`O>2Ll5mzA%UhnSH7EBgJaFjeb)N7VVz2A>uH(<hm|11vey+x
z5amG7L-Q3Tsl+O$+t{$4CD;L15-AST?aiJM;rRDYZO4{IrHQ*qKb@pY&jkuJy~;Sr
zbZA|Y*^?az2DWL7>RPj?bYyPpd_I3ix8eJW+E4cUh25)6O=fDDBvGWG24<>~!4UFb
zD4>^u?U@U%A{rb}OajiO0fa3&Qxi@pCjfBIo$-ZyCUDLA{Jr>2s-p^&&)FOv|Isd0
zLejEne*9H-yMK>%5o8))gN%Z)feex^?_2!h<mWm_Zhy?tGcDpQGprO#%<M}Rlb?u+
zs0<#g*O<?5p4oWPi4A-myX`Y-S8?FouNWyvKh9gMy3LXmSNqzjUauWd0D^yJMP-rY
zk|?dAXC<liFMn)T6U0X}1?JZ9pt@&Y&j9w=YTcJ&1D}{Ez6kGhepb~g_3^4#3CF><
z7r9trgs1kRr>IPxQV5CiN)xZ#BtaM{*{=GqA($ZNl7vq}PZ9la!M9|7e?Rx<XbEET
zyhQ9toQGNgh|?SEY!CWvF@0bX(&G5-p?<xgS2%tkW;D!OKl!*fPd4V(^pd<-#-0)B
z{$5cOX#Us&)50Pf0uNa`=&%4a!%rW8Hj5)7+;UnC0|Tx{_`jE}p<jg2+6Lp06XRi3
z;;#1EJpGuPqGZ&v&#5$=IBs15FK1%j%wg<oFWF>XH9xmwZG7I)X~9(NTm2@9|72ZL
zpor*6;lBTrh0)g`qHoq6rG6aWB^X$9O!n8N+j<onHSu7l%igxUFCl@=^3YF3=(~dB
z@{zMFH&s>O6<VQ$=VP9j+PItOassH4(lxzpsT*Bm9#ns{OD|~<a`>?-P!iGYyxO9Y
zy7=MEE)N+!#gGzAM8t3eUzjttdw*NEy9qFriIsyy78tbuI9mx(n=>WZpiW|<`gas1
z<V8tzAm!`hw0_`$plczHnXnC*aDFiU6nsFA{vd1qnzp@?fQLZ@(nU=B{Q<OOKK~3v
z4V*y9r}!nB?aTWG#;F~VjB^1%EALbU4B8vM2nnJ%+ZXR@tysV6H*OTtuU0^7*d<q>
zGe@l*_G}(?qR$Eq$5KI1^fB8iAylP^X6o-G(as@t@wz=IH_kj$3U3*+Jf$T-mE=@@
z9w?&E9rI+L?>VE;u1<ZNKI0eJz0^&<#z`$Cr;;#jQheFz14$6HRXWN~+cIMH=Eb#!
z=R?`QLvuMkI2+#%HCq%jlWTieVe`wXH<yQJ^I-nvS$?Fq8q#mNIXLcqN`1)~xUdG_
zH+O10K>rqgrxN0NCj-HuYo48?5hB1WOfNlxoTArQ0B&heq|aln0Px9E<?~9`0P|Oa
zb8jy_K67E6kMmTw&1B(gmmS=xG%m!nm_yx^LlUTPCy;wHt@Dxv1TdB#a2H(VI9*RJ
z1_m#}uF5EV?n&n#U?IE0D^BCk*h%n_oZW=?Ok8!ov7%G4NhOJF2a!RYsnYz;<z&y0
zFwE{zGmN)0FFsqaXl$L-Yu*hA&ex#$12A0vSMKxAiQ4n-12|DGnH5OCEoX%X^d%%+
zNfdKm-fiovH3y04+m^1@2$4>%_~}A7hl9-RQ|Cu8Y2RD$q3NBAL1=H^tq(k><N&oT
zIP;_u9yqY1f32%l>%8c^3ZFG&ZGkt1w82-nx|Mbdfh|mq8bb|(T}8uQK4rm+0Gr|3
zZdB6DxN(6~NShYyZo|t7CQxQZqBCW`R4^d0IxW~Pj%2#mu#=M@*6U=s5rqAaF3xzh
z)0%u&?+C{}8H8@SIQL#O+s=Raijnx)gV}TWhFevX+!HNro?m(_j->WG)hE>_qso*W
z6uqr-{2w~heVJg3#2XYiXE?Dee?{5f`y7Q25nv=@B!qyts#6$(_=(yK?8%S<%ni)V
zqZn8q+KouvJBT7k6+?{>knyriCR}8FH(#_kmz!e0d*B9lJi92Qlhk7rG?mw1K1hO4
zm#4;WyAD9;a2Sg06||<)qf2`QRY=zjSawmvr}4Q3z8|+?__*`jIuLVbZWKcm$#k&k
z%fcRGZfBEnH9hCX5prp@G3U=Mq0lN8Edmf!Yvf8-YQSjz>h|WAhp)5u<0yI%X19uK
zaMNu3E`z*IjeuORD;{iy_T;l4d7~KY_@YRv{)xvbz?9FYH-Rz4wfqeS>Id*34W^UB
zq=<0?K>9<2m~<O7b}y4iDgkOBmO>K0aDmG_!n3j9w#fz~_2MQ`k_`YgMpF9Mu&@wC
zT2TM)u6{=FijmKEaql5<+{W?1Ce?i7Cs9^zIIy;Nj$w7NUm1u!zH6$MO%RbzdhjnX
z+Ics<r3MCUhmLC0N-45`12sj7T({B2L@jBW?S)&0=+Z~5(DRcH(NF?mp@^BrZWF=^
zQi|xw`Kzz8aI7~I^Bx09qrfVn1BzpvcuF*e2~W%PPwDZ8XPJ(~3+g2Krh{k1&N>3E
zHx+%ib(@Q!=U#S8om!*=n&j|xOKg84p~``{PM8+jk#69$w9rG>V70VP2I(bo23bZr
zh4PSsVVA4$(B!-X3S4|CqWXq{qLk4;d{b1#VUu?3OsDbF6?-gnz@zkc4dgR^cz;UO
zJ7%0rBo%TK>A+?A)<4%!lcwfV@L0dOai3ndTGG03=r>6M{aMj9mY$Gv+F_%5$`qDF
z!})>kfak2e3&-IEhz`{qE-nJV4r6aAPyADHwgj0xuS4EH26yF{{pStphAiDXXh~wD
zMtb?bXYnetcq5rexgsV8b)N>dWa_Ldh32X74jgp0t2b^AMb#TuB3;}4wouKIc$i>)
zRy->`I)jecl^n4v-?2cy&a=03{?x$-ca=G1(th`L-jgF!T)~)?n~TD3Ix(rjyw|HY
znw{Z9hy6#)`-%5QxEns!7PQSiX3ifvSwwv(jIF+-;*b7;V5MqWe|<MEEWc-dEPOl`
zT##6Oj-czs;RDJpFjCOLM^xlJFlh!U_^QshC8-6<!xQx1VQz@Li{Qcjs{*?uShvmC
z`7v_l8Mv`LU2k4t!K$qj=An~NoND%EJwarm*k2d7-uRqm(tg3_Os_JZW<x7=_&AT^
zjdQib9HY^nJTnQwdX3n9p+17iJ(3q@L=CI^q}P)y!R?iJ$xUJp8DjA_i+sU#aW>sn
zoMYUR$R#6TpX{rZwCD>fs4j4!ZaFTZrL#|U!WK1nJFbl+N!>89*tO!Ja>!Az(RzdE
z<Zwq8%c{Xuik{&yT(XVJ(q(fMHqVz{nH@?tjx`%aDwJi5D3=B7`LG+OLDSq@VK2I>
zaiB*7KIO{F7gL`)%4(RG^sjtA%eG!;L5}%$^%0qQ;{8B;7>J+ey2dqVy=k;Rpzq%j
z`Z$$|7(5jyV!_bkVt|fkeS6@{&0_H7cW;Hwy5)eRD_J&xPh3>A?e|=U?=YH)A5Wsw
zu;?aWg$@z2q*a|4d7!yeJK!R(L^(m4dIa_E?$O^unFDc=PAG-m6ta>+3hmK$f$+CV
ze5lxGJnfe@(UTh(SA2oAHG9rL5m&Y;jM%>rbrGE(zYhRNS{F%JU_3!Q_2-~jUho;&
z2uOsPeEoy_U}K<oHd&NhL#~o*<<B|tvdX1J8ST3n?>c~3PUK1fP<4vPoSu}lFFde1
z6lIe}YCol8Iam+skgDHa%e`Uk_MbG~sOIW?5$us!j+sx|2O#Rktr0;?=)8x`A~8o@
zJb~hzX-DhtSqMh6GFc;)r|O>DOoSf;;DE7>K-G-Nwjk%6dczue!4aLFpyLZ-<G#8_
zy-sU5p|871o!zb2!(50pzZcCm#!amW4~S-XM6b}O!;hj$Dj-|`3{-fQJ#f?+;!tFN
z{;8w-m-8%ESQ+t%skCHIy1d+*f*y*#Bi>kpDESV0__D;=5^46}yp?d?atI9^m@)M)
z7#>4<=We$rgr+iqB;fixkf)Q<I-bB`q$j`^MJcgkz&cb6pP5JjaozIQ00jUbwWI5(
zk_x0Ibx5H9X(puR;qU6IBEG*>rFW9(5n$3}?b_X7*$_Yw|9KP1pO0TahRem0VT|1c
z9H;gHB-Y<J(Zd*aEYLPlPDL^N)j{?)csK@KS%U;52sr$qiedm$A6C%Tm(_$js$a<n
z%gc-^TGZ()ChSul{&xOavQbwv34(80IiE-KFcr5fQ|XLB&`t`G0(-YQ7SF5VnYzp_
z4qY{X(>S<!K=S(6Qjh2xl@jl&*FIKk73E#A>L(3H>$$e^BWF14JYPvKh;{1XzUfIJ
zr4QIoqu6@XKX6)ZNpD+@9xOdsVK^xq=8x%*b&$;jNPujI2&-CM&%INUE!B#ur7@RI
zOzr1(l%>xy2qFJ%&&7I^?U|-vSNa?^J5?~U1?84K{)!cC*m!R5K1YQ01eH2Pbcso;
zYqIfpw@U4JOT6FsOoM{jEQcz_2L*hS3m={`>*i(h)fk>mQ5$=4^5RAIcj^jBGTXV{
zG9V!?PZnv12a7oInyy_c0sgL;-SEdGCTwNPD<0~jXXKeR$x_BJ86O@ZqEUnDxYm}K
zROb7`0_zvXBo52~8BU9@aYk^cT(8`|Y5K6LFISGJ<oYHbHtWCpNcaBXo7ekS6M}7u
zIlv?ANcPtL0NP+R(;YCA2veDR9%wQEusm(tNA)+q=F(|4N(|6Pe0v{r78xTQyovF0
zvKy6eo<M52TD0M-aOc+Jb(a6h*motZ6jeFVxJsr$0RqRY3l~&0K@@Uz7Ho9vvuRi|
zJf)e)?{5AXCHptWdne6th&{-xobhDI{_s9_<#<#5GLwO;+6Wz6;T?RwusXztVxNLu
zR#mCwU0UgFGr~vmS4d;6k}{}xDWF;l;diG9KC;#;aSms?`9^z}KJf8{?x1@30<6wC
zFalV91qS9}@+(9%x^EV0N<Ob5K+_>T<ijM6;_W!iSmSeeQj%8xDhW@CbEwMx8!Bot
z<I6+7;7z=+zV;sDTnGLxrE0Bq0q=7b>3F_}X1%HNIfB!@(UWwh=16N)_knDizk%!n
zaZtMVzq(GmX&rVCJgCfw-Q%pPY^D3Ud{${;OJoK*DGcb}V+ni-heHjV$B!T|k(YdF
z79d{QV{<uD>6nm$5mu^>_H167X{(mlbs-6WlP{&T%tIq|$6Csh4`Y3jPQOhrk7^;6
zTnOnTE%dQ>q=?0@J1PS!8z)N1I|yKO5V`+k07Pc}<BELuec6V>UTlSui4yAgJLRaA
z(aV2P$-7%tXSDes5fuXbAUJ<as4nJtp)?dfoguzx!6x6Y^aseLNY6>x9H<0KRzN9!
z2wOy$yy7}yKo1X=cM(F{$NO8yJ$cn|${!3BG_=_^iRv`p=!Y0^@^j$9X7UtV8wFtN
zFN8$UXtu04ui-qC5tVe=B|v!mrlEDa&5>}|Xtuh@(q)g<o@X1ib|a^(BQT~XlgyQ`
zU65;T=#bD0#Ci!&d@M>Ea9ONRj5jHd6-%W}*W(2GLOVoUF7v;A*-PWzL!^g4^tk*&
z<cu#>`k=pD{0JEi179TS4=-c5<CI<Y-qxm2giuBAJCk2>_Yr18w?!f&0}mvBqs72{
zzXy;t{fM@y`@XpIGZmRLsO#`+_2EPU;D$ES`d8orn*BDcXjB64Z{*bVA1;6)jQ!Bg
zk2GUGEq!qYIrvJAfdTDC4eE7e^1S>KJLmncN&`pdgockt->LN;%BmH_l*z^tQdew9
z1^Zr#|4i9>Zq0L$yg}?XF>`zx099p(hHsa?DeHSwCPOnY5<P?Ew0El4Yw!)el6bP~
z*Q*%uj*X}~R3kYVB-2dWeSR#c2!md+370c!>&1tUPw_9x*t=OHD1)3SS=`5g4{xu#
zt&kx~&cfNz=WyVly_5ejRlm?b!Fi_9_{-A1aZ<wgkMoLuVP&q`4?nf#ko?VUp2YUa
z{`D?sAn$@pQfLdtPX<l~X3+)~{NuIbH31wb{%W$VU1^|VG)UDKRjwWYFBj=W<Tla2
z3Xb0(Eo*x2bZOKnO0RvwDeksS_x-y`SNBKf4wpa7OdI`cth390hwE8o`i6H_Ozm%1
zT2u37Q+v1Qv;o+Fi4=cDq-?OzlGJ+E)}SoX+?gzgm(%U6SB|6nNYR7dp7PhyM?l3G
z3_Sdy9etDbF+=>Fh9HrkmXbPj8d97iY`Gmx_A0a7hSji@397#lI*&6M&GFZhhiJ#3
zuB(m8?zk1tNX-S*DTm*eOsSGUmPZBep%R4A51FR~=z_Hxi+)WMfM@CN#xp}f1J;M=
z4?)%1!W$S13On5+(zX{N<(M^2V~@1?ZT!P9-JH%I5RTS_(O#}^n=4=LXbBXuLO~Gm
z^9YP>B*_S?K-*a7Qy=|?iJ~KM1M7z@q+D`X*aROef2Nv8DSvYEI#iusu;swIqH<-z
z#*c{nBv0gz;;%#CrQmoOjz?z-@|b0(M&9&_nX!r3e_|QAZ2qN7&}?bh)h?i<z{)b-
zFU7z{(BlmT$qj-d?*Eh<z<LDp+m6$^P#Z$*yspM)d{9xlL>mLH>erH(ijXVu@q1UE
zx^RmDh21E(%;v2BC;0r*s|5bsQ@&3`iw$|pa_0ogn`Iz&=A5FOM@B$vplrW0ylAgG
zzyAzgf^(C8f?O8hZdZMa13f;h9D#ITn?n@F{+M~%K?WWy9;~~-&jG0^N+q-doTnK8
z{id`z6Sn~(k(&iiKa8oIS77}$3%KfH6xBeiVgKoU4I3DCIHgUb*YP75Lu62A<v7=o
z$r1!cotYPcofrgj`M}%~*`zl!kL<f?Ae-KKSdb`;DTN}#KA->2O+sqqvHlZwyu=k}
zhS})bT8C8WF|Pn2;^FZXWdl*e^ia9Yt?4k*$=4(yiG{N6KJzM!ziYs1TU|En{)*h2
z7|rbG?;s4`vUkzOX0^#MNqu?3BUG&LH$8Q-M>d8DQ29UqR%s(ZRoXyn2tw<N3E=p|
zXZHCrMnM+7`A;Z)K}UAs76BWhmr}>aa4C(qk<(Ws&!m&vzjPO#kb0uM<!y>uxn8QC
z9hw%7RB4(F9i<6%TO$Rf9H25y{bh|_EY%&V>Uw$9Jd5+RFZEtIRG*b#vL&w@By!^6
z65`<2_twa}+n6S}LW_OG56z%RY2Wf+PSC~yCo>rY2Yx|uo)%jAf0}*c34rwh<9e+o
zOw0fhZ!ra|XU|cAc>f&1?u->v5MFA(6@LBl<}@oqFkT*e`=&Ob&NYa-J<V(JvVv7j
z<F?4im_hvKl|6yk&$x)oZ?~d?JLkvG;Gtp^b%ivU0`}~jR-HATZ^%w!C72s2>Dtsn
zxvSUhyv$X_lK+-)mtYJD__W6Nsz&radD<Tnl?I_hpgX6Y=NP(kgpYBRV9TKuWMndu
z<pnF~*54p6Zaw;c;<O%Pv|9C~CSY3LV~&qQJwoFn)1<8P1_-X^<7G2;SOnL3x`+y2
zb)~s?k=i+{*4oaiCr`Neg)9PFuJ=kBE5Z6^-l&Ug2j;2kuLfLRVfu-_U4W}W#t{|=
z^sGlAp8H*?90h9O9beMe+@FTS8NK*>g=o&f(mn-2mehMG=)n80QXpU{rU@%xWYon3
zmiN!p+x&7au)GzunkV^bJ}~S`9Tj;WpXRQa)Tw}ql!x@EbNo%>3UzKf9A+xH>k<P*
zVjp68td@=d)E9K15Xwq<c%(SXf)Ga6ATaMToyZcTVHjET41S^BJ6M7{G<!5SO8Mh&
zDF~2qEZQR~n&El9QM&xogvutlGB!D4^TlSFnR5B@`J8~o!~Q*CeceetEUMZR;<vgr
zq6N6tKY%C1|IKKaeJg<Z_?%S<6r}`q8J|3zj0x&9@x(yIXcMa%$bm=3Da<>tg!>zH
zx3PoE4248X?q0Om{;Oy$Nf7}&7BVXTTL8tf?Q#)It%Q_Je@CI@$uV4F4=8vH6mjud
zw1vgu|M}`8<r9$fdn)L)FM?!}&O|7=nbx!8Z`bOFe7v&rn1+gR0{}Q^Qg4m|m;F}I
zb9!PU-|99L(!o230#_4f^kYeSGvkQdffK#-t&>%q`vJ;)t+Ng>!;Sp2OT*U|rNt-f
zy#+@c-_@<_N<zGdr!sp$1u_y1wN37E9w<uWW9F~1M3l4Uj@Hco-QxTccyx6hYp^$-
zH@1=jOntAqq~2)8|IO3&EpX*O2K%k{w?Y-^-+nhjr#||-3y3K2?yf7wwV@2&Dwc{h
z0VWL}z@+ig{KJ@!$n2AVIJzovarLcBZ(tbZ6c!VOn#f4@XH<^jEB*`WGK(84z13fQ
z&G+3t*qAjh@#l>&6d)Sk!+SA^-(N!NaPhkaRHfyj88Blllck&XwT<jGNP*~qxR{mi
zGKllmtVg9~2L;fK?#-Saexu6nNzs>8^C>n$vu|7p>_pHx@E$gb#v~aZ1<Bal89kxg
z$;FaPQtWgM`SiR!i2+P>(cN1DDy~E1H2R;`By1&6Jyk6N|0((33zYN>(+XfG0F4ZU
zcW8SDK?T5(<%0=HWb->rV4y(Z{N|2v>Ag)loEc#QRmN(<Ca%b%+t7iq6$aCb-*2gb
zQ<zx5*;d5(=85aY7_W!jS=x6PQ;fP>_%u?lUnMF|8l~_6&UB&K!GFeaSL+e*jW*sL
z18@6f)t1S2iwK;8T9elyF(?CnKf<6MF)*4qKWhBoQyzw+-lsfysVBHtLmFOZ+oiUG
ze}%NzwK+oq2r2%(wT?^w??O?9kb@c|XuzC7M0s;A1qFnKi$Y&>K>=Yj!Y67_K$!mT
zo%F(n{wV^K4`_v;E11_KR(T;0{J02+BT|8;GPVeOJn<BYymIOwRE9z-bXgP)A`qJT
zTQo1izo?bZ4-jygn2>Re0`vbeU_03uI0KM#ffV<!3_HXBhd3zoV_|<K)N@`2%8ZFT
zhHn1nD2V_4WB<L3m~{UwApiTs$p1gi8N>=lea{LPD(G`Pm5leY7|MAd`RlhV0Zs)N
zq{JWymyQYB@4%f&4BgtIK<C}s8AUh%B{TI26tR0&@mtBkQ((G2xPLwc1|fjjmK>Mm
z4YDB&W|ds<yMNqLes6&e-;fA_Z;J(#AkS@haEX8wCj|JWX7Y-aR7}9*hK#nq?^P!F
zA#+NgQj@gd%ve8yAEC0OVu4^bA#h-9_I~eWL|QOlAH$qKKvsf49pD21Q-Ff$^^Yv=
zQtaotNa2+)LBk9Ra=!!-qzoz<=(|X)4}ifs2h)9Das|F4`U>X853@~jf&vhkJ<mRm
zsntSSdS)0OFT#8OMjlwVQ=%?HUVi|VgUVZ`kut~(tMFgAhXangA&^?>8iU`*e(rpf
zVFw0GP96l4cBA~AQVIGo==V^e#DF+^W;8Y+Ig5qClX_3~paH(Eq+ih(`umU!%%D#r
zyuIN4+kwhv0hd5q^&dqCZ?Gs^E5SufqA)BE!Z@B5-shDcV78PC!)S(}K$i;m;6r9Y
z2O;kG!Dp~S%}NwmamZl!s}th$(C|e*gW=5R+Tq?W4?xI?0H2$`f^91wiaFuq5toyI
z(Y%@boqxa4m}xgP$k6wowQPdc(sp*XaScLXB4*sagVBer!MgvO4?IXE8GQcI`F9lL
z86d7B^2}eM9t+T1RO3r}bAnMok;Qu$+X(cu(@~bgzo(Zz0q;xbU@1EY5R6sWtz=o<
z;<U)L_^OB8gDcf3{J@dZ#y<JGgfRXs!2ZMJVb?W610-a{!;$$x^002m8}K20_rwX0
zke8;(-bcKr{3m1pFAko68vTlT2dw#{dTKF$Gt3ql5d^jD`}eO=gZJ0YBIZDJG|(pr
z%rshx!18kvSVQ(|^j`DFU{xN8MTFl^mIDl*TTUf|{NL~mdB7e$l8k&LCkhr(iV(H~
zi3vO}+DBFkjX;DJ35*A2tNR{Y3*PYa1z2>)3OH#a0U)@Y?2?hxPyG`7kcl3O-@|Qt
z!21&(J?{C>8TkOFYq;F`VmKRA3en=RU}F$~=?2QkWTrqPz!U%?i5T@v`Tonm;k5%Y
zAW~ELxR;0#tWSVA_GTa_Bx7|Kif+&ZLvVo1uLPnI{6~hLK{EWo{N=M{5kQ?nobd~L
z8w;$zhfOFU^oYnCFrcUi%z^(5$Qc^Ye3f{=7Y@L@e#RJ>2?V0RW%*k5eu^nzzI*U+
zpZ~Kw>_FtFzP2jqA%f|m<4%2N2cy}UyM!zmKRvKnwv+LajnLMCj~~(v%-lm4h(+)l
zQ<O(N*r=9=h%WaFj}OU>IQy56|K6M-1r}0{k>@oWn5WH9cfRosIC~+fSjP89a~zDr
zwh~|58~QRN!M<Q1=YuG$#gMR9&frvjpaLUVMnX@xpXOs6aF)V}FvJo6$w4ZV8%D-V
z2xbBUH=1>#b$brP7*jU+6)N<?wg7OLehtghKKplAyCIA3QQ<3gVo1({)kUA=067-n
zGd8_Ho4Hg#&Nh?bAO72@2S1>7e{5xH$_B<SH$hG-{~G8TOv2r6Jm~$RQZT^wACX-6
zH(g9^NO!~wJ7AE2@no7rrO3Smx&Vh0Aruxm`v_Pq<%{G)-3a%yg@Km^stg8)3nkPP
z2|WMaP=eZ=2FTg84$|s9@zN4B-*ed*%KzvZ$lz?t$x$3I1pE5<P1Y+GMrgkGbd!*-
zV+Mk@-4-17Z>z|^Lg#RzKe%lhkYOq!3g}vq0KwGEV^YF_<ZM3ytk3O7QGvgw3iCTD
zc>mYQ-250{pvx}1kV_fmAzjA(yzssdju33`q~ANS|HusvBsbZaN^%%rc?6@E{74{G
z)}=0)3I?zv4-{qn;)8tQzj`<olF9XnYS(!R_?h|}rK>xz^f1)`wg9fAC@_#yYsaz2
z|5gSpF8ID^#{3?jHNtE$#N_^;o+IagHzz@sYx=ttZE-D#EZq#efS%<|4BNl;f`sh;
zGnEGP#Oxw040IaF!CF6n6?6JKX-BLUm=@hX{v+VYL{MD{*buFwu(czA!c(_a8UuNF
z5iZCz(1h=OE0}#xC`B&v7-oqYQbhhONZrIR@aov@2oL^)Q-G!e+2?-0M9KAGPwU*3
zbigm&Ku*+g!ay@FK(<BU7(?>xCk=TbmCT^z`xwp+0~)G`;ond_!UTQ@8YD9BApq0!
z!h3)FKOkoi5>vNC9TO7<Upr=hi2@!w{mWiBq<@BO1rfu(|Le2){}*^S1bT|A5)nKG
zkT4Npv23suhAe;mra#UsnN{E|Mz8Ljd*c6!sQ&LC3vc`HW&HmjAQO~!t2@XuwH}$~
zIfY(R{(!7E-yI$CkJ#Tpp!Y7-7%*<gW&g%f;i0I?|At<(RU01tjg}Zyvc~w=v10>U
zrL628j=^6U-v1JcgWiG3kkd?e?=(jn-aE}W%<zWDn9NLgbq)~k8J|b@^c!$XAbE$)
z)!&ejOrNpq3A%=cYJ3yxQO-s8&NQZLErqM!7%6C`?JIYuTbE(!!sc&~zw<u9zNz@l
zAek9OHYdlg&V+ddNY6CiG*Uo?P(u^d`1C**V4~ljvqIs&o;NVFW#eH$0PiWOpMcMC
zh{JEDM-1?v=3YXe6ZmJLMzGNR(1dl)yI3uctQ3v49jTa`G8o*XSQ?u#U!=9!xq5W$
zbEc$v@<qP19er=qlbbP*wb=TDG`OwysvfCtnnCob2t{$x=#XZ61!XeG3%W^rjA54i
z+xOu>R-U+a@KL`patqv#Bya7}4*|^~hc0&>aE+>U?*_|OvRUDO9`6|ZAB9co@c)5Q
zmzqd=0FvySHqA<_xNg_UZ+euAS>j>J!RRFUpms_5NbkRvG~xQEY|jE7WRJ6X*POL_
z^M1mZJ<uKPWs|PK0N}c<1CLj~C4tzic@~3vG2p~QaWg>$j51)K4TxvCeDFf-$$B6;
zeCLx?3Ib6i4tPgRGs=7V)vs?K;&NfahGlt-3Qv|O4u@9e4{eXb*5ca#h;C^u-rgA7
zYxam0@SaFfEz=f^TKVO4ud>kZ<B^_1k73*_Oy09gV8Fb+yW(ad9u4zJ6nvoxTlk0^
zzVNB&o#lUl`-@OgQ3d&ByDP7^g8mE2pgSt3%H~+XCrl(xg)M<@cZUV-Oi&<&i-(uv
zEPcjm{_|@kY?uS+@S&aOL`KABigdCuEXJN15`vWc-00m}tq4XI876J%QhT%K6@DsD
zh2&G0+jBCGFY8^#{Kh|ZJ@{0(B&naI^~hPW3$IK~&MaBhbz$t}g3^s_fsVcr#~>M{
zcYz9);?D-ceD{~5>qYFPW1kktH1Jp7W`1h^gVR+ebcUVd#n-qZu<m%bOHwM#mBn@A
zxkBGKeU3waq7ElXoHmhi(;*spV`wW-<C=heU7;>3)8u-_W!7_(^*wPgafLTs>mcge
zRrS~0K8wgOGL{ju{ue>|Mbc6#*@x~V=^SNbc9(;2$BT>H8%YF*Du-j1o5DGAQM)%u
z@}vu5bfae31Yyn8Tg29xSTF7)W;!VT*Bt^?zBGExZt)Pn?0WwfC*p{C4|63&Oid{~
z0+(0B?e!{6;S}w=<Wn?x8i^tV$upZgM@fv)#YtwBkH6_+5v?BdCY6L3V0aZD+cCa{
zZ`zrC&-F8P$KKvwgF$IB$yzWnc=Y~^^W2~Dy$cagLeu%kBAggzN4q?v>B&&!Hv>MJ
z!hryeCSe{CB;3cc2VJ02r>eV1zp+k%6x8QsfijMBP|PF+{!hLCEUISF=gMY?X?2S2
zo5O0aP#8a8<F*=F`au4Ry)2yukc2DPHeKWnPJwEs%=c(SmrxxPs45!)ot}Ojm4S*T
zB~Wta-FdM7^zy;2B8pGSsP9f*8YgO3vR$QlYJpL8O!C3*^%;wJrTb^}yN1?PFEG;r
z^X*;Lay8e4m4+XCN%l4-j9QCUq*8m{Ty4(=j!FA1kJ_GyzS^m_3x3RBfW@cQEv_+;
zrs~Qm%)aj7|Gwl#<tSw#p!s^Aq<?3=m%@}UPh+e2QdQW2w$yHm)aS+?Pi}MMB~rq`
zx+eS^uGfTdg=@2I6Wu;7(bTp>wzJ&IMOX%-m8cdWXs_LpFgc&TN{l8v3(XDPt5(AJ
z#SRZy9&et(AGXacK6<0z$^Fgl;^gxzhkk?G@fN>azW#dau~X;Q7dC6CG?7EoRETi~
zq92u7Ckx>@ZJw^B`n{pBOLrh+dOV&waP2<y1@oF=oIm|EQ<|*e<WeT7OZ+27;7Ub{
zNHp<ud0gbq^jeK+SlRjRG*vK?Xa=0rEAHnoNN7HQyC|Z5oaup^U1tDCmFnv`_Rutz
zLzGJIXXSnmDW1PQ3lA?cMTfa&VhK}WRK7p~l+ig%I#uVP*!#8gYK22Hl}*0w3@W*g
zY-Ue?H<H;=6B2ws@21QyQ7ki+Vhx(mND&zv&sPd?yR0lxY;dz*rW<K}1gZ|RR{IjJ
zWIyI1?@i;FD6O#@GlG)4kcZLPqj$9LQ5@g9;7?di;<c_<lYD+DFTUud!)o|yv%B&u
z8~U11x#v<2Os9|X=hN%&vCA~CL^>9{f=6gg%&ORaH*q6YF*EAj2%gDUI+&WwRK2NU
z!<T+_lW@KKeQAa5(FM6Va;Qft0-dojMuN2k#xAY7^FgbPOf%NH!zzokR{G$2NY{G;
zPHmsHDU1mOosQC1o%kOVp2>^4%QVOru$yM~<wYUG;jCq<nOFJkoV}GYw)W&=Hf9h(
zNA%IAN-6&?Zo22LqX(~W#uhA!WE%6aFNM#Wn!d2YrY#S9_%!Y)Wbbpk)Thx6IU3!s
zob!F*-nI8~Km}j+aA`xGlcZOueq})ASnU8t;7f=TL1fo;*1p5V)|lm1Xhzk>`RR$}
z`q`p#sdQz*O<>0hIie5lG~_Bsx`K`z{Uj3|6tsB*(MgB63=FcX3Zx6$Ztk399I3tI
z*1L_-2x-WY*5rb2V+Mp2@LQRW8?0GjkVyMPl;1rAoidD<XIioS1SM{3_E8IO?LlXy
zX7^Rfz?K$UyiE;Ybf4>-L%iYrpSisy6h1yho^SG=wMY?B0F>|X-wc{}kIG(tz#aze
z9^0cXgs2eg*u>sHa$4=hOV@a+`ZA*4bx-;F{J^^t^|$HT&!kO2gfGM*l@anhg~b<v
z!L&a)4s_YvW9R)wv)*?yfdjluF&3W5PBX^uJasJI+)d4x9d+I(aE%$^>EquGO}VvM
zofCPw1|;Y5@Q=nuNjXta!pheTn3)2<2V?A_qKe}W=+j=JoTM~rA%$Bd@1{%ep&EUm
zaLH|xkvYIHyCK-ijkRRw32k3qAJ`*MoR199?fsc$RHDriwmBrguv5-5sT9hw+3qY*
z>^>D77fV;|N#|Xa!r?S-RDxbBN&)Xc(Kq%wUlU=#Va&3t@PZ>#Ec|<4d0_>-F(U4p
zSI<B~#}JhwT5WfC9CfJjD34C7kmLAZC`q+uPDk`WUFYrKWlnEJ8(o{ggnI<+Nkf`K
zIe<Uw|EMxsR9yJD`oUMP)G8V<y*xP?%Ejwou^z;ymtsQjeKGi(ZmGp>Z@}jAHY~>J
zXiMB`#G+xvEuVY&@DRf}>Et&Cg(61dl0n!5k^Q90Pm};&lX?6!QV%c!jPaPG0zwTe
zN5Jp{Hd72#Y-W?gd$1j>BS9MVBM}X=i9dRY1%W83&f-C==6cJ-Q7Zj!e0}d(3169K
zg%i#~OjQ{4+dC1j_0_^ZC!c<P#a*I{*=GFFl<xIXnK$livZn-!KSLf`-UXEq&O6OT
z+v>1NG(q}o`=t^C*P-<j=`eN@8kR4Vwqs5oDlKl<SSyn;PLrDDO50dg9Q5argD}sP
z3l2l=E&{~hf0uoCd!oeFNUPpwWS~4fBR5XDJZcDePgR(((+zI@2};WZKlB}|Ypy^3
zQqU=jKJVIP5`ATg#pIq`5lh!~6svGhnDAW5xc_Mi!K+mRWlaGOC03?yvj;uKeb=Z&
z3SGY}dlq0zdcNOv?RALe6vPJcYsH*dFoAwhWFd4#U9pIx9iJpOtIEh3Say0*1JbUx
zxs7@{9d9rEiNChMefZ*8J<MtLr$dMP8=d)z7H;)13++1--V)~q=OeyS@!Yy4hR-@j
zw=o7)*esNvSEn0P^L05_Cl&UZMIH2#hPBL6h4jC$tjmbEFINT-C16|<`bXgn+UwD;
zWBqBh7JH=cE~2Rd-y2Q#y#a=)M5X8Kg~X~4eYPyonTzlNpL~M+O-`bX1+_j-z_yoi
z;_oim4?Sts(GJ2M-wv>UTKfA>4D;Sb^sRTeF7?)aiq%Y_9L&986fJ6w=k0P{FfWQ*
zba|u7_n4ra^2c=Sc1_yB^*Nr<H&Jrmb@{s{gW7a-y41KZW3=d>wMQWj-%aZZ;l6+h
z1}^{E1MOx|`q5e&wr24eChYu&M3cpTg}Ma)bD{1HHRfw!zplpg_@1XDXVivr8$*6B
zU$sRlXO5kotn(3U6%7h!o|My|^tDR%K7K#BVasJlA6G)QQq#`+?W!t9bg9PW<n`oM
zQ2R;*MG<-}LC5xO=Y>i~@;iYzx0xniy_cL#A(IPNHFgelB}jQ>Xo;eDgcW1547j4?
z)eowJ9G4D#V)y7M7OEbz1?c;pQLT?oae=l0K7Ok~BKWd1jUUAm&&#H*KMy;?TJh<_
zMHbhe`>&EtHB?WxSSGKPZ6zb^PS)&2&APn!+2blaa8*CaoXg@-MULN>BF5EGuaHOA
z$4Nl9>ee8yt=i6S{I>bXd5bIT?G8!EB*XRok<^xvUV6g<W<o-cz*!6R2ve}H`1BF3
zld$*|cS&vxQz4zT1a7uM)|$$3-X2Qk2(R%s*pB?JuMy1Kn)&Nmw8q;4`249qWI|u{
zZ5!@ihPOK5A6^ZReRwE=o*te`ww+Z%nKhdpag;l>Ly;OXx#Ni5z$JEtEW_dZ%$VZy
zLz`0{or4>s<f#Ke3T0k{L2K{*lYuse=eZC0*0B<7^b>vuYpray-gq)69M$0nFhvkx
zt`%JF`uoPL&`s?w3gU~_(^H@ApiOB&g+tpy^4kltNA*8ON!-vr9ij=H(#c-YtLP5)
zcp<k}%KGagQao2QZjB<_0H8Ji@|&8aAZB_nXZ?_3C^dxlw10mmxEyoP^TB|qK+?G*
z{0)pRZBGQwYhlB?wY@m|8J#S}XN3(iAMhB2&h`_QIbzTZ^}i{Kg)7Vt9tlN(#*B<l
z!wKD*hql$|94^lqo>)IuMp8|U{Vs+{ns7;^n#t~6rOV;5V&C7#7e7chfnvJXwO}5e
zB#!hOg@Ywg_3I}j3dM3>G=5s3pxr6<_fOF{Dx}1RH5-J;N=6A@8QMxP((V)RS5v+!
zRU`GxGDE~-^f*vmX=Oln`}*O#&%jf@EX^)^o{SK^h6xy*!&2uI!l}BDyf~|e`T>T9
zKg!xiWv5N~e3L@kP{IkU%&gh);jWm^r^Sf*znev1K5#FF<>EJgm>%@<);U10^vDB|
zgxY7_F;lttu-)a&!v448n{cJjN^Mf3%?{M8L(%VVIy(KC3K4v&C&l(5Lco1v8CI9U
zfpE8*55jm1vd%ANXKb#N{#1GEg|;@UKeAGlXH@N5Ca7i#dnrJxQnG^jjt+i6xq<Q9
zXHNO+EKbXHwW!M)Tt$=hp}}p-m*w=!2^6R5{my3P9$Zh>T~;d*+*$QpgE;{a>+<xC
zr)wY_Hok*<=9~d{aW>=<)p}!dMoBl0x5Sh$So%i>F4vTr&aGFN>I&O)`nuNWxhu_l
zx$?o^b~s#ay}{o-t|8N6Or6fFtl6z?%L&OdbkuW05%#5->T|X2s7Tvmx1~^|h;E(~
zNKJwBRd7R7y4@|sqteOOSH?$Pb9qvAim|-;NCE+3PT0DS2{o?cY}_;dY4N2aOUl5k
zA0BZ*$oI7=?!NM7*S!S}yQNQ0m<F7Ht7+Meq+y#euzvF9?9Xh%<=I|I(#t&g)YoJA
zO3CZTq`w7tZHtFMBOTBqdxS6N>M^_dgmxyVEeHJwi-G@H3c7A9f+jZ{qaQ*-GLUho
zU#FoSlV3jY$42$J7P)#LkakmO5NPo57hUc=-Ve0m7Vk6tN)%K$G@(InPnVR|0ZC(2
z32>>q7?odt=s4H6CafrNh_uuT(`@gHss5h0`xP%?Qrh&UC1DVd=WGAo@s>3gZaEaq
zID&iAb}GY+>T~I+Q;Ih_fA^LS-$|kV<zglyE?=g1_)^=qiRsn4YRnBne@3K(cVBjT
zKl|!ks9uPRRwf#w*1U+&Ctw~oD4G2!_R4gd4}Wq>+kWiQgjt1Fjm2NVqF2JCI^{rp
zj)!D@GnRPnA-2A5d|o_gB+9rG-T~3rl9@1w>+7oNRqy-$nMa(%?YQYWkQ$p}ARG=s
z)7c4a?06qqYS{J8crW8!W57e)(oAE5ucbz|QvtM(WsC_pr=%FMB3P}uZ`b*+Ey`U7
zcLfLM|B!lCr@J>Er_Ii$&U+8qdtTKq{@j4O2qf*Rf76UNF+10YKhH&Is+^n0CgkmY
zd0M1LE~V!_`UsvrB#*g~a88}zyH#l2Sz1GXFY^*jFcnMV=tEq!VAxy~tHPr1@67TR
zxT><P$dJa`9unyE>b<`4YpByA`+lMPjlub>G4gq%LVbm)&RzCQSo7kyiLUj%<__Iq
zUxn&gb8Ho&i#OtxeW)(vGwFW_;2NX8h<EY6u0GL2$Q-aJQDH%@R;k*K_O7otz~@8`
zqc)MEA4h<-C3eex;y7wYkl(P>gCSkV#ISLF4Od;GHG&&!>(p~dIh61C3n4>kr01&O
zl-*C(&$MrYp+6BBQ_$x|j{LjsQ?kLM<LQzoHo<5_Yv#q{7{34)4Z9?XbL{4HxkavK
z?<>_7h4NLa=Ax3Oy4gK+e$}%dK9z_CNdX`{WJL`+!QWapmv49h+8Aq>8OuDYqC%@1
zNiy%TKd!*ae%s7v@W*vMbvCk5U9IEKR{_xKl3-}uCvRp?Aem0HwM)-+zA}}G|CVhL
zj_+;Uc=EUpM##W!5$kE(<4>w<-^DDPwOm%G5+u<C+5~(?&31j%7jI5%wSJft@Y{Rl
zDEW(!bd}C0xSXX*XDd<bO7;~8Al3vvp8u?Oywg?snWXks)I5}A>-FaVh3ECB!rU$-
zu>PftLHdcSXMXV*@RtV>vGy)MJ_{$ncW$|B6!ewW$!?N4d6MCSl}-0uQg`fN^{gQu
z9_m_TN0}4vV|Ufvu6tx1&i9bfYq*3dxOlwBOt!<j`!)T3UsM`C<LJ`Y%W=x>tOrJ`
z*}JLrB>e5p4<mNj36TW)&3UXO3SJ+yrqq-Y;yxX73r;T9FcFSVc+tpZH~r2M0ajtb
z1@Fsl)%?!!)(tb6UQ=J_V0!(TgyU+dXT9@#fx~#ByXSSSVpXV@HZIi0aeXviTG%0n
z{vu}ZX%_4wy^A;j9>sfgfhDCKk-2;O^;zTZeKkA}Fu1=SVC<FHlOjgmsswi_H#ko9
ztTkb+*SwjONxS+gXQws4dSWZ`=~9>LHmv>2(Ozog0HMJ0)Pnh3BX{jjy_@>a`YR7Q
zQx)S0{A+H^O2di#7*&gk%4Y`sHZUyz{P`g3x7G_!YpVe3ze;)Gwt$j;a}oWU_K>{g
z%x`lM1#IVe0*S%x5a*h+f4$hw*RpM!W{q(gtj5XF#!sEj8q_zB*au8KA4b@YGUR(9
zMm&Pw64E;BllfZnp7&^%pbz8O!NruPL{d@YY**@NNW$&#TAJj_0+}&MoTlAWP&MJR
zSM7MP{`T}8`tg7FbiwOXX>B-$px&@et;9#;j9=<5y17yhH*VNzz5uZn8FLIzMUU;1
zbfv$@U6H7=pNt+})U=y@bv+h~_X_pQ(5JmTuRrGJ!zGN3k;01+jce+ccX#5#AqsVy
zn;+ly4xfyi>&(YE3-HBLcyqZv3E!G2Vs0A77R<%+y$s&ABJrNrp-s$QN@CfP_gpG;
ztbsAH{meSicouV;v(Fl)6>u|!RfoCtPE+n+J?riKy3o1DXSwkcrxtU)d54^d?amnj
zpgefq5o(k*JmP;%bX53<QkPCd>rY3s7umSLWYBKL__{-qr~QjfY3B>aVa=z#$&4(%
zKAq->#5_#p$0PRX^~kEdlfy?=aQ?lNjk~dy0cZ64Uk^|@w-eaZ@?8I_gq8ohO1Ld=
zOk#VG)m74r3?Co%d&Ql>>vUc0<CpFLm_@i;UxwF=%`4Tq10=6>6^d)9R7<E%bVq7@
zZc0497iJTpMdU0fCUYpVqgb2&^7X!o*gnWHtD_+_4Sp$|%&64mI2Yen_bZU2^*!}W
zciGENZ1Rs!UhcdMx4)Xil*@bDoa`yp7&m53r+tk{sm*o%i#y_da41908Bfm<QoD`L
zpEAd@g|QCSWolF<y>Kke;@4c;78imUt8o5j`abcs+gU1llWp4z*9BWIZt;eOo<3HX
zbEv)8-e#%YO<3(uaUtKITlieVy33I}(@S(hyO{ci{`t5W^JdT|TlWdZ`pydb{^PkQ
z;S6LhgoPU#CX~LDJklK>|KV^6l)6pU;;c6>%^EfXcG>woOVSb&v|;^)zLX1Ko)6FZ
zy3FTQV%*ea!)UtqP-42xL|P|Goq8);ip(9Z><<PXZ@yi4d{(1n8Nl?jbW(7OAUF!;
z?QZpQ1=sX;Y*;_O=NX!TH4Em3#ZIpb`Ex$09ttsAwrv~k;xIcmQM}vzPdFRbtBU$r
zHohs&y2BowabQykyEUiZ_jaF%n>=Q&p6E3HL2~ujOh=vY(~#|_4ij)r86&yB$-jkJ
zRwb<eIwL}!1Vdt-ZvGRVR+#;NeMqH;cpMrBBz~Z5cI^fX60LrG&QaCSsi|Sh8sSUE
z<kR5Aq{71E&7E#`$~#XlZ9LRCQ&P`p>9XT#?!<mpP3=GCUGaX3@Ooe87km<nkVnFB
zv>H93zRZs|&+ry;UJ3tXkWsiAara_7a6alY7+5ktmz$}YStab;{HZs=YT$cY`k6pP
zi13r%Y8P1(H{lPjIdAUIw;IB@X<_;*4I&>v&Ol9y$YUPEIcHBl*-qRfwWMj<<+SM}
z&90luhKOm#w;3(>1U}YckV@W>W&0}eV{~sYeb`%_Ka23NMR=cu{v#_2i}n$<yT2_}
zJ{g=J2fxMkvq~Pm8!53lwr=jU{oEzLkzv4`R=+NBhAKJZ-(te0)n+~<w>4yhUbB|@
z(a@-ZApwDJK6|p2nPEHXi)5YU><LA#lWESgJgxU64aLLZMf4~$wI9@W5Pv;l|M?oX
zeMB=1==DmLKPou_o&!u3cYTzY8|C91IENXY`lt0<LtlLSM_KHXFU~^3Y@_BS0>*j^
zy`pE#QQ^0;O2V2$JC(;0LrLiBx=)6kPEPHwekgGApIGz1IwwfK9zR}mlF~@+WKfD`
z{-#QVbnHyaWWarTsm9D#UQUfZ{Kz6XI(PV4Uzo_3fgmD32`QSrU~u;n{g3v(GaAnB
z-J3Cjh>~d0rRZTq841Bi9z;Y45xpdO86vu27#<Nlksu*@P4wQvL?k-V+o%zB^j^oi
z$@6dLoVDJw-Y@T$$A_C`=HB<du6>o?RqnlOG}7+-KBxE}1mRYRpCg6ny|)j$XvQ{N
zmY>!Z5@#hMZ{P0LnQ}9D7Fl`RVm&o%?MFj8<%~@!#526rlr*N&o^FI`kNy~Qsi~a#
zcC?upbWn@#dLk3XQ`fs{$Q$k{T9`R3?YiEyu|4{=-6Xq8+%Yj+%XGnZWYdY$^Ge^O
zmuxA8P;7x_4ek;BMXd?VsR}&d5A*D!@aw1#&oBqLW3{1(Ox&NZVRA>E@o!r9i{Xjh
zt7SO*WbSAQ%MppXFoosvAk;0>+0u40pZ60+v6h&Ck^MKkHTa@)JJR@t3RN3DMXmy#
zvKJ0-f4c_(l!~r_K;HwOT>ES$wu;gNU!3PLu|*8=Y_u8u{)F~0261GQBtIP0SAMY8
zjD0otK^$eX2a}92@JX(m9x>f~9+$%PR<xv>?RfDzZfl<hN2+f=j=5s6dcDe2Zq+6h
zgJ}2lYItB<Ihu8lRrNYL<{AXkBHZ^~lSxQbR;=+>e2ZJ$?#K_m(!;8_SUN@7E(;kK
zX*sLX%8~W;9h+iOh=glu!?Q=qErZUmhQpN#p=ihAVD7M`i^zkX#kRUwcImuc7O>v@
zcQ#-?Uf(RL#(r6h_CHtsDz^Ah86EXP#=_&gRj=|hl1{$)Uq`PfUfHE;2fw9I7nc8x
z_Vg!<<v1hR6gZ%}9IpZ&zjHlT71MQ;^1^XlVMjb}$uVnaK1&|`a5rm(CKV=AA6n_?
z-V7HT`u0w15&E*4&B?WSaJB`FcL<d3u2JpQ89kdbZ!{U`gSHi4u4$O<KhiSrKz$Xn
zZ{l=$xUEp9WInicb0hc@FUgk6RT%+kjb_A3?IS6VR7etz<9TFjXsC35#JG2R=&hpy
zIL^Ae)w`qO^POg>%^MZFvR^oOwfeF^tlh(!u5b=V<L*n^H6v=i?|UV5cG7IH>!|Ke
zZr{$hP?-Jp1(7?<uB~~MuW#<nSW1mt+-JChwXotD5j0Ndo~h^0Moh#rB|KfO{%IHI
zQSqwFjG~X86!u`;T2(svyRES1Jx;cL+J~M+88!nGnU0(WeT%7HqnPSk%Q4inM|lLs
z^+Vu#qym@v)IF<JohYNmc*`<bli{nZUEY@G$wOULMyI|#vB>t02wyJxIbLfhG1Xs?
zNf@>mwHkfC$v>@EI0V7$e_x#Fn}x@!>gjs7EaR=#iuDB<jt6Zyt&4};rE#m~N?XsF
zz6%=fm9?8)M*Dk`U}`yJ_RD(49qDSUJfcsuIl7TOH;&aet6>u;$GhY!Ty#lDX?kwD
ziCm@xQ6@HVWR4d99Llfi#oo_Ad3D&!DTj+wOdEqHn8&+|4kFCet%bmyuMgbtIarn1
z=$qwyzsF?M=3FG_n$4@7o^o*Cg{+^(a$~6Z8k2#Yb?Uh~MF#Ux@24w{v+v5^JXB*>
zO?hYDl@?>{CE6UWd|a$+HqRhEBB3q+V9TW2dlv$C&qD3ngaE@;lbqLh_HD<_aY*0e
zx_u$8S3vu2Lm`}1GhR+OkH*UK+ap8?1eui0naaK;w+!P-1#R~6#?Vqjo2^T!x(2DH
z3;dnQ@Abw}TckZkib*tgzqNbeci6o{U9)?co*e~ucKzfX?}UlRvMFGr%?!n({m?gq
zq~%<47qK4NlGV>+nKA1t)pcL(eVs_#T1Z}1A>zyKNS2SMwH4_Z9@<qDx8E$s<qQAe
zsKBJ1o^v2dn8dZ4AGtcIwKlr;-X5nil$_PZ?*E9yIQ5W2`@4lYIZ;9pmMev?A4*{x
zsvfyv!i{oPm}NV|BrgY2ChldmfA4Yo5xul$%EQAE%R}gda>WMeiGh<A#PiY#VfYk7
zBk}Vg+u0gR*yi;r8;^u(g{0`IkLIlQL^cmwEm%c~1*#b@(E$lfl(1act8Ss)?yFdP
zfg63vph0I9@Pp7Rl9e@2y35hIsoF#V&P4YoEjr{-e~Fmr{dj!ca+t9FX3TwP^p}Hm
zafSFtKL`|Cg7xy+8Va0wFP$$&W9XeQWHozau#h)=Ln>rWel|>;b#nwKQPc28#qb%5
zqcn(0UP%UiBXP5KcWPAfUAEfbKur#F?Bl)B{^c9=?uVZ1yvPQ%o*U@1DRb@;Jat{#
zBev1SVog<xvaCcsXiip_Aa=`TnzHPlKe?;>%;;m^-d1Q#FC4Nc!d}gCE;2>Tsnj%-
zZ`3{5xgCF8ZT`$>qsgV-Kq&SMx;ZM-WvCCv+!DKK-G|><Tso5;w%kD`TzbvEe<H2<
zc1f*9uadD8(q!0>?40Y=f^}ts_AQ@dju|Iu9{+p5TKAz^T(hgz`00OXl;MKmI@yKi
zx5heQTB(7F^A>A;i+Zu<Dhebu356Sa_A`QEtjTue4=7;^MyS1@gSKg<o#$_IzJ*D@
zR^(D==Io=UU8K%yIJVBI385LUqe{xoiM$gyx`=js<YJS0kawdo{X<NI_D5qDiG~*q
zGrLPtaqrMEVbqol{Rc*37`(x@G+rRpc{^?WTYF60YfBEWg~_?99R^9FqW(VY%_C8!
zBT`LbywmpZU|;oYGZRbu;8Mu2e%)aZEW$~QoW=+J{0?LheeBt`ewlVuE!xM4<W;&o
z(&Y_>`cT+?-xk@=u{8()%q{c3al|V4t`N6(Yq0S`$bTttwXP5ckV8LRin^3>eOTNg
z3D$B$qrt%Wg+6KXgBq8dy}q2{N_lyS+KIO?o@&*%Yw!s6q(?)L1+jcb2z`mVuf39s
z&#gszW0&UChOvikhV6*7pJ&)hN5dFisUsfve^3-_ED5iwwJA`?wtv{@aOg>zb!$xX
z&bBM4mz_zvBw2wyI;MZg;)$TynAuR<S3q(|Xraa@-I%HLVimjIA<w4py}KOpU3<>L
zA_3{TQQIEfYmzsnJnNYxGv-o0Br}4P!5}IH3Nm~@1#gQLyYz*Y+UU0nr|TAWQ$uZx
z^CP_5&aISZQ!dX}1{XCGZ2dblk2eynDHo4@d?p+%%1s0R{2}3*Fsy~l+8MjtnlsnX
zuxe^#b@*i}50!KEZh9Ed<4*~CRirl6dLb4ARfLi|U)!S`ZBvoLZ+c*%af=GIcb0ZP
z#0IDwGe(6U(;TcQ9EWR(Q-n99<~%a{q+4=?bs#mCSbi}kdA2Fvrcr!N;mwaMm7=kH
zf#QzqF$X)zC1{sBs|uz8+foFt<B}&4x<*hImkT3A;`d^YfW~H%mwdJK$u1$CeTW4%
zctJAl43w5H8WBBCa+Z#~S6>nHNlD4bjHCSh?S<-<5I^k&EOvf5@SV^Bo0h2T*I&wX
z?W?ub6%&y~uO87i;hra=;j3<+D%S_cW&+3@vM-YN!q3i8$=OIsQoizIt>x3a$9%VC
zOc0j%z^h*MYw|U$RafZ!vv=)m$ehDk7d6kMLHJp;Ck`Aks36(zqj-iL4|#C9wk{k}
zfhrB?nFjWk{f7AQ1&AleKg6`T=FIoF-JD^QR9T~y#`S!QNfc8udLF3jlpS9``g!e;
z&Z0@Sr%JZb9Quhe&QH6gtY^?}S^<+h>Fri(N+0etn?5jGYWjH5XS;7k!ZwLWj}e1d
zQW_GcF_0Kky~4?^uS{Gg+?r?STKcB-8nz=tuQYA$LsZ{(b9H`TtAu?j&#f**(b@gz
zm_~=%wos`Ja!Kx^^r;Nej8uH+#ex`VYV#_a=i&F#hwU*@3kyxQYGYDgZ(1`78PjL=
zp^V>uXlV-*oLaof?q=Zr{Nh}nRFPKyassd6#e*?ezLKuj*+IqQ8o`2!1IO9w9R+7^
z(yilXgtZw7Y2{Z9@J)q88kVmeF+@cC!peWhKZ<8EZdml%)a70=Jodc8QT&-){O4{d
zjXe<ycRQLEk)#FHn=A(VbNOv#83PsFV$~plkkhN2YUo+$etB3!j3zvxidTSdH7>8U
zTItwxB(;iPprGG6Sy<RAYr+?1jNVRu-{(51bw&@RFp8G*#B0S+WfV?JiQ;pLhUwEf
z5$P>9!Ibo+<;KVRaj;v~XI5(7qhiHzdB)|@O6H<y0s)t?aiO)s*Pw)&6dtvABb6O1
zvXsJ{*()2XA#2@MTe35c*Q8Z{Z+iCL!qFOD*bry7x#yA^F|1g3oeClt6aJYg1vhZu
zRyqp74<!=X=e@8+#ExOPS&PHrpK0iO314e>N=Fx++VK`R4Yxg;5JK33S6>uL9w)o+
zmiUOCPkTGLKhZm3@4lrp?ep}lH((oL4-lG)M-*!qju1WSo|J$BR^^@;vvL{1XnHNc
zDFviQ2P-kuhM{R4g}6k5DTV(rR*vcqcLMLnp10^4&$D@OP+u`vuQf19qW!*}lp#BQ
z3to1c@v<*s1_HEARBrcpoLG`5EYd=boU2m+lDj$t4(+b<=!Jo7)IQZmVq$18H$Wmp
z5RA~KQTL!buS0cktD>}3)7|1RS5)+e_y)yl(?MlL;ydQzHavZK**S(q)M~T{E}xU}
zSzOBgH`+`NE$c)nkHXY&$#Hv~t}>6iZuqjm9A<Pa-QHZyLKXHu4C*MtGp@=(X`6Bp
zfGLXc;{VG@ZyR;HyOdKin>38c6q$V`iO=9k(?Kt(wl&mHi`woCmVWQ}&?BGZLRtib
z@?O|U+*iyArL|%>g?~|H6So$aCQyi@7rnkjkna}&pFtlpqc_aj;%7duk6rEYHmhl7
zkA%|@v)Al$iAf+&q1D3+4=6DiU`Z}_^3nfqWE=_!*)sZRcOH>MaXO7)|Nq@Ip4P<N
z&<At9jkyMcg#Da2Nl-145kvJ9k{DzUz<zKyBK83QQpaAB0M@h+3h)ntt3YRkyMvJ!
z8mrEA66)TlAhvodejrbTd3^o^HGy8~j#M;pEm^O_T`F=e^=p`GSTb7jYyGI8Vg(`$
z5B(`w)c=3Uq5+yf1n5)-E#QKsJhOg85F!4~vy%Q$i2izRv+^Zh#ETPAPP{~;bp@;&
zIUqj_0)^QAX)1Q2L=Pb&=isM4=Jd0_F1bR0JEgdo;yXkbMp`b~K}c8=KiBWu|N6>J
z@b7p2iW9Wp`IviPM8Wx0mXqQCH4Z|Fl!b$QaSl_+@t+_3<11;r2rdyB6`R|Df>Sd7
z{lZla26e)Tvt9fjZ=R>1CBOe8GWOqZ`R6cAxj-~W;8bKU{_&{)n3pFQxu-ad=f6J`
zjPP6#OsZV99&_p6Ncrb5uM)t>o7X;4{0}oG!i=1yC2vFaJTv(ZKK$e6zsddQ0wMlh
z-G4~MiDdpQ%qK?dw<`awyZ>f)5PzGhe;N3{E%U#5vHwR$^oVzLCe9)DXlH!<TG`-J
zi$w43>oT63S$U9tMIuZzBk=Xz?XAuK#rM%Gn~YRAS{!W+xMg}Re9Vl+kdQIXZ{E}f
zkV^?54{5~b&{KPcI8WNd!}m1XspHfG=5ncTo@ALc+h-Gf?9m%><L`_Qe=@0#r^y|z
z`o9-751t>Xbm~c@<b6d$48iPKk%dD}?EuDrg8qo-vUXTq-EW$~0kE(0Cz*`B3YkSr
zeD{9%5KI$1W|)h+6j`+{#bC8SW3D}i&!MB(%IA&Pd@MoAhpg$mL^^QR8Fp}LmfwTg
zkp_FwUD7S0DDw7sD1C~BTp<dzFA|+PZqEFAqvl$nCcBq5WcEh2$lo5(Zkt;g?XmWq
zsolV-eYyz3TA<?kkO)jub)?A|7mE&a`x(R=J79!rZ@b>eHUkO^al&Mxo|{%;;7F4}
zZ&=k_%1l8$b%=LZzzOP7f`W(3!{wF6F6H>2Fzb~z=CW};1$1W1u-S3?1i&TerM8&z
zuakL2J^GlRx5guTuqakM^1StZ`Ix8R*1ojHWE~l&U0&6~r$Ma&LpQ_J9WqitS(a8(
z)4)ah%QoHv1?;C(T_H_C?^`=Hvb((tqT`LdIs2g}r*?nAA`V%0ecb2RTkB0gRsuLx
z2>1Dza4?gZh!zcj)m98WZxw2%mR>Zmc=zN5b$mDYf^8m_;F+rA^2E^WwL;slZ9Z~;
zbABq4H&WnX8P4g4j3uLv332gk0;&qhiCVm#Y8oqU9jW`Qj<nqqpKG~2>Snx=ZG0q9
z7CTuVSe^I9b1koWS=6*K+y&2n{poyqjPQ`+o2;D5nON@Dp9!}%JMYW@3dK3SRxx`k
zKF9d`fT_~OTmAaH;nG<LUr#ziv<ASDLp@-*D?Wu?kdi%Iy|WYBp{nas>$TaXlqYn!
zA@<`|kJm=Kq{Ym8OSnH7&2~tAW6jPOu1#{>YjQ*Uxbt8x);90DJncJ~?ZNuP{Q)f8
z0ee*D0mRxMZXWIa-aOiorzlzcwJ-9~y*FnEY1+Qj_Ko<s!n1f}LWXH@;Rm!{hii?P
zBa^(ENS1N~_sM|xYAR^I&Yhz;aO#u5peqikLMUdkaWjC^>Ks#Nfxz*skoqs%wjs+z
zb|YhCc}j*4=O8PGn{1i>!xPy*wf7>b6JJPZ&gE+n64WEa+Ql^sT7|3s%-5<?E0M>3
zIaBu4C3S1g!BEalB+GEU=Yh1)?ljDYS^vl3#t+WH36rPnfo#uT;yTW7lHjI<=DO2W
z>|_LU3T~V5w!h4^EE}>28O=EK+B~en=Y{0f!+Q6IO`bgUj&rE>Sl##`chIN8aSS+d
z2F}H5Z|m>JosVyBg<;utQE<Z;WI?eRJ1dl`JVKhVZVsQ*EAoz21HC)693kwPA{WK2
z)SpW4;0IvshML+95Hskok8-<S@Q2*%HPy)JOIJjey=`HBipBR{pTC${FA4>3sdvJb
z&FC)Pfy={(ppGjyqff0(FYmJ+Rmp%M3pmlN=uEFBkPai!{M!9btrP300t`az*#zRa
z<kwo~9U6%%ilNV1#hJgqV6~qUB|6^Blw*#2HW|GFsI%uwOB-?nFX31e1J$f_LDF`s
zc0BR-QLNXo=Hq5y$5FyQo6IDj?00VMW9P2(6{`;Y$%0lkJ>Z|ZKl>}Bfgp{~DL(93
zeb~6WgFPOV(N>J7YO2I)>zXqF$=di6KTk8NWaD8_Sjbe2v#+jPQSda>4Q}D;eNNRZ
z?y+uG{yuzfy%iRb{nY@qR665JLj5e?WofwYGhrWZ5v^<C_(OtFP;~X^COGD+n=cin
zH|8<Ro|<4>JNPAeZ@GHKpxUmW?hLvR^@wo}SZU*;Ywf%({hLF<Ha#%xVYj*YC*)g-
ztbx8f0Z*UL>n(yoW!4-_F6msa9Ftxzh|zQDo9e-TZ8r5c-BC0d(YNi_wRFJ_zxs>y
zd`w(%6F!&C=njAQVNV_Fy5Q;<Y9&C|vaH(5#HUU0<{)o^`{j4u+e7vD^U^}U2<BB>
zHn0Q>LuL0$FlQE|jI=wmwewfEKwIIT`3lGC4ezL)S8he?Jf(<PV)Q-&;z)y6SIemQ
zDF#7W)#svG_%orySXjiZ^}9NOMeInEb}apYj@o0a=-8p-=HigpZlfIG{BphP525CS
z5@PIX>AIZV&SC)v>*sUE9B@0G3a`v~*RwI&`=*{2TS{V(>6Dx{x#PVb+<~}MrHIuJ
zFR0n|)>+in1(W9B>B^+!&<o_(eDL7HG<g$3ewAXxeJX5{Nz!Gu5GzWy&9rDz+V_}p
zF~4fkcK>ILGhK33s__I8E^pHKl49}QZEi!<B5aEx(6Qh>)g)n)+o!l4PPqtMue&Fh
zOD{a2es2E)hjvP%NioF;93J<DD!vmO!&X2Yud+$$Q6lV;x<9&to+b3Ohhonc=#or6
zKdx#odugw?Uj8EcrE7E15ENt2QwH84OLCg+cv1t8OERmTqm$-Pt4L_);hR63{^WYf
ztXVNdN?a9oX;+53aj}~JqhO>Q*>P4J!`V~YzCq6B@$2=ZM2gQbp^z+ZA_*J*-LM`o
z#ODm5w$5Un6dsqSDzk9ewsF>JYp{?E_c@RBp6u$uLn6^UJ3zZVyZ1mRx>aS91r7Yx
zTkK*2aG4#G=w7NyjgJzvyL+>VfulxIn@@^YhZkpnUm)K+x;^{(JEMMy$0$tQ&@~iM
zo=`jRd98sJ!%io1u5;nQw>@toTidHmh3i=}cTA{v;)vV0_Y)i&dUulHM*!2t6%bL&
zAz!;+brtFOLha1ryfP*;=-9|k5=YHG$CEaUNLuI$xHE;YvLhK?oxhwM7aYzQ_lfPp
zv#%7(H{fILQ(rPLWHCmb){A83B@d2bpG^iZov9kq+*`=b_tST%ktneKfnk3MFW~RO
zsH)JQfq9;@;P$bV`*i1c{zad$v)NS4fzlG};~Y<W#&xo)yMywUB!(a0!&a$ZsApI{
zpq49<4c0HSGjCl{7iKbaeqFuSw>NhNHve=E&njkd-+knmu!BAN8QBP6jI6;d&H*dr
zA+#KXP124#pS9z=WWzC7XXOu+rqOWF`HdCX4%Mbgg0RIBgY+EIJ;#Pjdg56{P}x8X
zZH<q2HNQFn<e28Kfc*i6pgptkjYeRr6=LWDL#NEUV_76GMNn>D&A~UTBoY31S-}x<
z1ov{7IY)Fv$j*Fb;tM&p;pM!-#^iu9yp8+ks&PJ87s8}l$FIwA!}-VKkD$6AYH@cV
zLFD5i_L#8jwfK!$IlE#S96pG0A1iBhQHCeSzfAjC*2T1Of&9@quDrp#x=D|g7u{?p
z*_N>f&bWPJlJq{LZJ0zqxm?bng7|*c<~F5p#e|P^_?{pO`RD0qeTcNbmR)?5sB8+>
zj+<)z6Czy>eiYnm1jHw+&f^IElOsws<y_P~g^xeGklKbWTIE#`KV_u$M_D6~P+EjG
z@o>d)Rl6}soMxoV_>-C?H{<nnca_s?5J`Sg1XMNq6zn65WZ?_r6`0SSO0y@;tyP4M
zQaXx*bjOR`)u1L)YBTK$XkV;4ka|n2aYFT@$k1`6k*l2O#MyW^qlyIf`c7E5cZt~2
z3;ceSpze3A$Q^`9i8lqbq0~e+<~(_uWpzbr3huG>R9YWWg@1ORtBj<v$`0W@C+KhA
zz-)O=E~`)If!Gyp22S!vGPRVWDIvlvbM<2nu3qR*9gxUVFlg2URloOAkG%_bk5+s*
z=k6E~_slICuNRV#HO2?)XCQmy)kX(D=ZmszQKlr|<y>BG_~Zj57v!=AfsmUCp058;
zZG9H|2|DGXB;G1o8&8ihM(=$yEMX#lM8jnqQJak?a4*Y|vL1O(6W4<-S}8A8{5NF2
zc`Zl<2T}UDT&2yjZz0@Z)HQqkZKI#E^bEv>=GRX0)t6}-54CNuM(MuE;R?+h=%x$E
z)N5bE+rH)?y_->7>SEj|8VIJgZ~gc#mq;e^kMi8bDx~*%GcQKtU2F9p+4v=8-gtWS
zE9=gYv<p;F8xf;)x9rx!U=Ff#7is%@(p6$iTHf*_KecFbtpYbiv$&`=d2`plkEpWB
z<bcL4Y@^QzpiZZ+TN2TKmi65*UI@?q+4xCn?He5~J9ywcg6j%#5k(VE(_F4X>1n9I
zeAX)E%yEsu)OVh!Z3w38fY#U;6)|A!<K4V0eoX^~dyC~@;#;`Or7pd#%<9xGq228G
zRt`wq#iQ3HhZS!UvT;u7Pc09r#1)gqjg%RG$R60Gs}6q#4Lnp7lo}`5O2|`2txlw1
z9BA$;jq-CRXbX<=@t!Pij7ga8L!(U<wUX*3&{M2cdylq;NNpvr4oNLsGy&l8+2Vb1
z9Ul}o3wf`C!^iY!0p>g1WRR%GYCw@2C|{|XK#C|>j%!ULPD$CK<a4Yei|Xy+`3TlA
zl+Za|?qDn#p6I6+Md*3w+m&)o-sc*NPi=P@p%h>gA-#ZuY9P$2j59l0{6wG5+EPIk
zdH1^qX9({EWo*`8rLEV~$U$3%Hyf8YB`)bIHK2F4E|70txRt>V46AB8JwBLlxsIy<
z`Na0f*Fj$s#~pjM-bbBR_2(oBCJJIoLPmWKA4KUXp~9tp8+|C(9@%HaPXiS=5Y_YI
z(Lk9}E0#|>Kxyx)`PxC~Ua7tsI|%xnR%H<^=)6rCuyfI|cy+;Qm=bEUuIz(NrI5Gb
zSCcz()@j_w=Nqdq*vExKA>#mZKQjiF)bhJ`pJ$?b??a`4gHE%ct!VVoe!vzLyHu}g
z?|l1|=#@9yN?an!FQ&K1X~}bxVD(7vf<NU=re1@|NAzb<&H|V<WdZ9CEY3(7@ab|~
zT%dr416+N$I<VL@%^d++tcEj&(j}^k{bI)c7)^AU`~;^R95bn%=DDQ=tP2Q0MSx))
z!u}Wpr85wC!698~0lbM`qOQCwsUu|ZZ$}>+rHIw!6?aHY!`$X7m)|%$Bpq3OsIzIT
z-H@IK5>~vszrieS^=e#v`+Dr#BxOozy_7eJb^r>nk6v??*}Awf)g<^T^U_E0UmCL=
z@%0-OE42qt%Y}?9`t@upk-c=Px-Y`6(ed0TkC3$PH)$ZZGYd&8he0s&x*3n@M$3xu
zW$B6<de!K)F0)GnzsOA;HP9L)x9cepr04<6%Bnw&>@zXfJ>NvR<@H9;wJACB?9QI`
z1{=subDf%vo9?X7$xnq%f_yiKW=qC<I!c=wP1u`_)RWzxOGTKzJy=<D#2}~wOBRb-
zU*JDDHZXI$?DF3G;0~hJ!yb49*tEe|r?Fq(f&FlIDgxR6UdqL0b+w*`%rjK53o&tC
ziLJ|`?K!tVmUOOM;ypU4PlOXsYfui}PI0UbHOx4uo!dzTVO?Whrms=DBLT#hf|6Y?
z!?mYK6fN~@kG663rajLdMO85##IJxjZW8WTue!HY<bIX*U4V`30{9i-%6Ic@71DeF
zi3bvXj#1yPaJAkU1S?LaTkPJOT^O`Ai>TS%g3WG~!2AJF=MJpJn#{)}tw+Wnk~Eei
z{G;R^yS&HuWovh0eU3Z-D7Kj20^EwKz3~>c3Yf*hd&;aZRCyuqNp=w<PD=lBeY}0^
z$P4U0;fCQSC4$xN{!FqR$eY~ptl?lH?_K*0CNA&%*aY+>U8}fvk68}w1;SaTioza=
zIt7iHWJmz8!k!h_<<V-gv##?x;v7kyclmWbPN}>7syMNfHSBI9*WFj^$VgOIv)ZJv
zHAc4i`UMLES6_;?iTL9(BiY2GC&vlfl5R$y)Wmp{{GGZ?yMMi?lIyK~+m)G=-C@WB
zwg_;V<q|WWwZ8nSBBhz3w1yB;JPzW@`T@7WCxI0L_o5Ye&2?S+v_>80WRf->b$<sS
zew62;frdQ08fZDf#B8X7#6kgPt}M#1T<A#Pm)tuEYSWQ?5x`=FiU*)u;z)0L=fwJy
zdSp&>(+-|M=^Fqg93Ze&QA|D<URoXDdM9YCrmvk}qad7Z2+H8$e*VN!nMhIoQBBxC
z2%9tAJn%LUY2mDW0S)f3Va>g=UmnR58SJTVeIajd-_a*K#MS`VnFeIG`EVe_gq@#d
zN8}@i(71DtDqS|SjHh$bgk|~?!_`nqJAm=RCS9>dOAe-DSLF!1Q(x2)vb;mtZi=qy
zZ2A>lORejyn5y~}sUpz-5FY}{S5d9Ut^lwMD|VHboxvb+cT2<1UtFTjS^QEF`It)`
z+qp|)W6l~Ka|;S_d9ub-#ccWEI-ERfiicKb%=4B~QYA1GStDpOx<}iSTBoux4-<L%
z?Pa8qOw~$fO$UHl%MEEgZYZH_dh%XfzN{=q$dPx5&m&*;AUcr1K<=3J)!%*ZB}?~X
zAldi5MMvN6)IvC$b_xI`kp1TCUhJN|Yc)V{9pf5>HQlVu#T7~HYh_);IBvN=TXaQF
z#=DI%=M=RFAn%Ja{(P*DWQe1>VczoAWVv$I6*ot?$NZ=AbhiqO3jf&Q8dCq`{e)XL
z)o+Jzx^*UoaP+m_dH+H8treI0N_NH9i{J%@J!8caH=gFxN#JzX5OCd$Nvfc6KZ6lM
z8kp~RXr?>acJ`kzMkIEdUGk!xbFk-8B%^jp-Pg1)=FZi+mhpY@Qy_R|HT<w!uCZJb
z)bR1H02K1ZOeY+%arTe=O9Usb{ktc4wf$gurU~Fx%u{l!^_SX49^4fm`{So=>@DHQ
zRvT`WC7sWs@ksGQ)fQp1M~!_b$1MINY;_TQ%)VfBIO;YY^_gSmDhv6E0*(rNvTS>I
zUHI-Eklyde{$%s2V-(0S`lD8A23*$S#jQ1-B1H3UlUBZ!;2%H9ffWl^I39%SZ*L74
zlT|eiT;*Gjq9euVL4nAp^!p-#w5yiqT4kg+yyS!JOMH)@OlEybwauc<XypEc389I2
zD0V-JDQsg-Y{}xiIisY!@}5*6C))kupCPO&YQ@`%4lUBau{Nf!i8v=)<?~(qG_`RM
z#EQQ?<+UqJA~n8F9MrmVKK`TN^5>R;SrA>`mjfs!a;cf;ZrLQmi4+k-yDz+?gl1?-
zZUN8p8KfWGW9Dn$+UrNfCH33u%b>Q;`5Up?S0o>p->nldZ?Lu_C9U#OzaLRP>Z-qp
z+4YU<fB5SD{4~omZOk3gjues=p7tB*(`e(CifWLm8_l<9>7MV&RNIX;&IFJl{nNpg
z1Z`e-<(A58jr}MNPHp(0iT^WJb0TZQ^kqDg2<es=qGupZTr;MKWeYz^OvXmr@769c
zP+_jeA-QcbvQ}^om`MOd<;@+6C<G2x1x{Qh5#+aTjzFHzQU!TQq&?M8FreYREi=^h
z=eHotiY%T)zpL<?__Cix>U!&D7T05sH4Gw*?!4q<|9t&X|2M%6e(G)pSo=Lw@UG&B
z4lCM234%HMbI)`Kr@%SY(BCO4_u{3oq+X!Wn;;+6*&KQMRuz>0AP=clEB+*I=Tgrl
zl04rRTqyf{LVL6mEC~^=6MVs7lLwL6C9uxF&U`AxMHl@7%2O&*Fim2hB2@Zluw8rf
zd7t^?_ujGUsh@Wd6Bsg%i&i2-q>MUAX|kl<;!!Bn6nswSvCfl#NMxyTm}XY!t+4#b
zh<w0K_RNM*9p^5E`lZ8d!zyPA4iLbuYl4i;_V=PzVf~^o&Jk_XfUFZ2DtzDwGQ(~^
zT938S9B3@zHNvFw;tzmpg^2AhcSoji{FVk5!7mb@3&$3zh$#+l9~Vn5yk(;tv{!C$
zm6m|hn%U7d<dk&ZYj7RNnSY7oi76!JF>Q<xgRh#~tu_wk<!berS7TXl5(8e_LzWF^
zzJ<n9w4CIc8Spnisx*wsrpjfJtH1(e%HW?GsqJku9>gz%rvq=sfWPb6b!H)y{f~b3
zH}T9Z*D8NSHk|nyN@d24Od+Z$B>5?&9k*s-H(+4ukLJeXLS=ULL2}oy=+=A?yI23<
zqP(dkUbI!H{6k?~U7D6kH2!efx|kuX9pc>hw3;s_M{|je^{NC&96YePOtbe`v~Uy?
z)*+7@a=Z{m70n_YqSc(&XQ7v=UvAIc>H&HZ$nUFRDs0;}Z*gpg{Z;s6r|XnB1P_&f
zTh|T#g1T`I6bk85o^<eV(duNg;k3BFsOo7M<eU~3)B}hH-U$@wF#JyBvr%{*AMG?y
zk3MorQ3-z#a7RAL8W;h%s;0ya=J}ick)_72li1RO5d@k)a_bBKRnPeKSE4;2A^>4R
zDJh(J(jsu*66kS;6G;K;_p+T}g^cr{79y_NH$rxgtDq7GUfTF)iSn;vC1!>ReB}^1
zNN<TK&&{ynp}ufBy??*x=>Ywql(bKH5N5CMVM7&ulCb|*vGe4wU_Q`n@iMpL=3o68
zfAQy356S5tfcqz7KhP=u*QbMV_)>s@xm+eS`@8t^A2)KT-vFhG0NdveF8;SCg1(QJ
zAlwfbT=*dNTfKfi?62|?ViNF@F|YaNlm9x^6TVyqJr39YS>gGQ8;KwRbYSr+cxL}R
vO=<j1=)aY6|FJrMujq*+|6fEGb4=nez_mhlBb<>4{3t6t_#^MOiQoSLzo2J-

diff --git a/docs/figures/arch_6.png b/docs/figures/arch_6.png
deleted file mode 100644
index 3969511041e5fb8074316475a6bf74531368431c..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 123034
zcmeGEWmuI<`v(kDDgp{Bk`jt^3xYHV(k0ypNJ&X|OGt=HHwuf85*CeslF|**N-t{B
zz2F`8R#^AG@BO^T@qBr{{5cL>7qjM?Ict7r%#5!U<t46Rkzk>qpj?xZ6jMS$xon7n
zg5HIB8MNG?x$*=B1*_OxR8&z)RFqQD&c?*t(ijCr@^y6V6-8xDLLaz0Yw|0Ut2f@>
zbi0*tOIUjG!yXO8-K%)g!M<sR5(Mhcn1h3TuIWPPDhPy!LM}0eTw+@EW5Q~K8-5FH
zJN3D18Gm}ZAY`*P4o`Mo<5`%jG@a*1p>rtENV?95^7d|9%&UV64kN>vxP1{H^i7{&
zLCn}#YLlQ>7$_>43w3QlEGTjv&Yd(Kh^-^<+fQmKL$9FV``O(S3XS#Mv~~D0|C)LP
zh4ScW&Dwj*4GevecovacVwI+Tky$R!{ch0``^T5O#@rItyo7=l-yVG*U3f@zr{j*^
zgQcLA*Tn(BtoR?oYSClE!ej;8%ygc;8V!D$vfeFy9GN;>w=WxYFl!fS%wHNSb=L(8
z+6WsrK3%XqwrQ1&m2Y7azQXflnl!KrXS;(u!}zdWBZ>&0zw_})2I78}q$gEYeC*r}
z?oN+_tU608H^r}<v{!3*9te#j1)CE*cMVX0-sg}dV3eBvmSR%yoTh-2St(QD#>?j?
z_R9A@X7NZ6-RA7nH2-zN!A1U{MvR1}U7vE(Hud!~^O(E%Q0gw@?Drv)EV2Ez-LB+T
zeM-ab{f>}#bIpV<Zk<2G+E$uI)`~TzxAEd@C?_N`yW6hD=68y-&UTFAjSy$rAs%NE
zyP&b+`3TD5UCr|m_>94Xhb3J04Aw|?i9CMV_@;rKG*@J}TD(u|jg5{4wtS{MOO&|m
zksQVi7RzFSyUp>$x-?p%dAMfDmhrvRF=#zHC_ZiJ#L1H^DA@-noTVEX?~~{!#CTAd
zC{Y>G&{fl}I{4mT6WxNM%YVJr`2!cj9}^Fo0F7J{l~@=p#^?4kW;&E7&p3WuX83CT
z8F#>^;U}Hh<*cu+FNt-olz;PFC))NyUB?RZ!F`6~N69bdYxb5Q;}T3X^X=_MKOIU|
zIutB_!w(pA#5YBh%r2E&XYi9w!!l!4!A77Zzv>cI{~*73ci_r35t|RJY-o`Nym_dk
zU+oPDIsy^DFkMiSa99KFe|fl2>0DQD2>MC0&+kE2i@R4}`12_Sr5z@EuXqO@!L%<1
zm96B>dm?Ig10;!+!Z|7PCHd?KVNU|zbranQ3^Xw8V3Q}(m%Setq#33eK;NwL>;;=#
zTxeW%oL~p$JYS5TYD<D4S<xL!94_K&3~e9g2J){dU-jxITdqG7`uXzF<t{wah^?lk
zX(iis+op=-3h{9s9ohm4?vR!l)s0LSQxCS3(C%i!pUyvVe|1m8e_mNn#k9j<_PhNp
z5l;f|#wMomE8cqIda`=#dZMSk*$NK#FW)+7G;Wk@L^P^RGf$(~hGj@TxX%-LPckQ?
zG{a5_9d<J(Xj>lqNOXy?sL0cmix=pu1&`mZ6s+uiPF;L;#Q%;$%zN(lxbM5)QM{je
zN{5dgLQ4JSz1)ao4n!x7AL8*ot2MFRe)eE9caCu`aTfl)F7sy2p$eDd-G{MS;sW9|
zPsr3L8Hngy=w9AdqgP=_Qdv|ZQq7>(Vz|s`LPx+jM6aiMN2Q*Tn9h}cBYHD>mtI$K
zDeEq$Zo-vt32ling=#U<w3)>~hwB2-dSXJd@U#dzM*1;233^u50u{mh@pfUN%Q)Am
zl8av3M)fxBKaepZaF2O56sUO(y4G=+{Xn}(?n#<bc3-xg{#+=7@}SH^Imgf!(4|`s
z!xCO=Wslx0)qN2=5NO(O%GWvH$&%#huuJ8rF(D=(>sfgyxI?;>PLe<(NMaaHMncFL
z_>efkIw2<kl3>aiRAQ#ZqV+gmDIcoE8s8?%#gVMp_2v7WUbW%f!I&5D!L=Upe)fUa
z1Fik$1Fj!s`ix;Tu$ZMX*qcRj*rlcXp4?&g421zxSk|J=u+8wP*)uadtIv~ZBhPvi
zo8B75S!me&FcM?8d#GipV&%|HWZLmSo@3J7V+@0<=%EyQ4riq?R^n(pmYj2fQ`fgH
z+c>HOWr3`*Fw17HcJ^HEWe(H0D{%|)+=+E5x!l~=YcZ9p_I27nv@*0;%3UmTp2JMz
zO-EiF47D|en5vpPb|)CFTGfnJH64bB=Y)SXiS0z!POwT_bsOUvvvJyWx><d>`mVdc
z5ylb2ku^c`?Ro|iUE8m`6?}P4kLK1#db`-iHnL|Z<W1zw<sIeyw{RA&&ab8p!}Df3
z94D8@m+Kv*UXIN4Eq_{VbIf=0oX=ctcINxJ>u1#=Z%koW=T!bOwj%ZA>Ef@&qQ03V
z+G*H~=HkdN$n0&H>aU#v?H0a~>Ph#;mLb#1ZXW%-!a6v?l=Yj-K`l9RVf!OT`P<ct
z3cI*_h3nn~&u&s)HM{l-9f4MUc^3nA<+WdezX%(xi^X~*Uc=j_yLExnk3}Cdilzo>
z2Iga0;ARD|VijVa;{75oqMx?Sc$&dk%p5{;n?jXzhkcZz>V6fsiFHR?+;Y*2qWT{(
zk34i##(gV#5A?RmJ1K(8!&!sjTF)w;!TfTu2MID+`D=o!f~yxb(75jMeqi~<$Q`HM
zNhUXYUyb_BI={=9YvI22vC_%gubPVYT;9zlsl4ZmTS@xz(3P&~A%w|TWASbtUai!Q
z+(hyrJM*K}XQM7|4sJ%5?^4~J$&i$aqN$3@iLc~@DqAwJ;O((E>3p%;2yWE3qr`60
zdOP@DP9ia#REY|P+gE!hHmNO1LFQx39qmsQDka;^4<e08V`{$dbRk@MOn7Wu$FNp!
zTc~g6KQA1w#?p$e_+(YOwsbs*Zs9RBkVl;_r|r(seXsC798tCCn14WWV6l1Ry?*ea
zR=8GJsi$Vr9I-~hE`;U1^5a>JK}#S!;JDu63BeQNC;FkTO~YQr$2rGY8{tx<d|V?|
zqou^<nkAJMq02D|H=Ev%zdy<M)lQU-hXg?2<;<f_r5%};dro~$u-(Z`pUoIDH$E=-
zzScV>+q%A*;v(sq7ovkL$;ZjgPL-=Ke&iQbo9)F4)C=J59<Eq@;>u<j3mS`s!VgZe
z5vxK_uL6NIr?kbd<^CW2YaSXrB(zwyuvi{g<{E6+={x)&x)EeevCBW+_iX5M`MX{X
z2aG%K$XyR)>RELq5D{f$Mr0O`3cSB8VXTT7+!`p!Gtc0GRF+ehLAElcTFcb3hHlj*
zoVu+??dG)gw`FlB-%V~6;_%|yTUq4UsQk6nx^Of`P%2Q1Xa8%msV9^y_Lx<-a>euD
zF<&)e<@N$&RspY`fNMAqXw}3{CTSVB{CG)?>&J$IbXW``KW6NDxi;T|V@l<W+rqJK
zcXce>c1^dA!l4ezSiCIa?BNVYKk@$-1jC);Z*%3`b&<m@%UXY3A6cBWq>$^)agtv&
zT(IYmYC08e`m39KZ|hJWk+O3%?0dWrwP4IzqBn3|>skuQCV3UY?xfvUeOle9;nJzn
zvHGAz`f$tX%cGfQ$&T+|Nm~;)%A4RSQ)C`CTeVi4&$nSgVYo&V*g`p8pAI?>$4{sa
z+e3?}^*8j}>xjKA5NoG3ru;@8Ivvq3X|+)?$X+VlM6tm|6MlWk$B7iB?>HyLO(`ws
z)JF>K^mW$Znp6?`GktE98||o=6<5E$!770~K`76lVt)AOMvJ1Y-QMhQ$~Tg$5L|y3
zWfw>L^5`XgkCX1HH#veQ3a+P~mv8A=eVtnHMsE(noZgygfrB76#8^YhL{1Kc0X$=(
zV4#wqTmnz1;8zfp^!!;Il@0~{uXZ#Plpu2yjNg6a!ROh}Tkw1K&flNt?*mb;fWK~n
zUzaqrKfNy-rlJ3NM(+aOp@=ApN=boFWg|OdV{3ae8;7@F7=%H?)t8c*_9!R>)Mvk_
zQcCyN!T7`GDjE(Na<aTeHdZVK&ut8iSzN4Mg5D@7{4TuU(aPAtfYQav(%PQaMd03F
zJ$S+MSu^WB%D=ieSP0zHkW-`-wXrj%<YHlCVY?@YMM+7?Z};4US4m9#_v_#<fqP~S
z4lj9GS)HAoS)4goZ0t-~*?D+)SlJ%1K6t<kdNA9&T00oHFk9PG{XNK^am0-6jqJ={
zI+)v7Q=W}$U})p$AaL*A*+l0*f75B~Vtz4`wf%3l0E4V&->|Z?u(6(x4PNCxYvolm
zcQLlq6f?I1G6Qo6@^EwV|JDD$zPXqY=}nD`H`%#3{^M1oum1U}s=cwDsErkv(?Re;
zU%%%?{_^*W{H$lJBZ=Z~IserP1TBcg&w8#jK`giV!z`f3JLX~vD&P}X+1U^30{DmS
z@6WU6a)-N)RE{Vp!YEQ=A}TJZD-)M0o*N^stWVWwe`cD%Wa^{O=M0w4iFzd&>u@y~
z3+pBUrqWF;$sklrNp5dpd@<n+(wMdJimlYWpE&mL$<b>2wSfixZeCOQFB`6=sS;hz
zQ+p-1L$rTj;!z6wprE2*{P)8rmL8hmoT?nOtN+?3-iNXo*Y|(F4hDQ1=|f3FtI>4x
zU+2JJ_d@?4Q~ynjZxRLuJI^<Un17vsGI)dZU&qFry-yk3dR49HUnf8n37h)Yv4tsV
zQAK3sKZj8Nn}9Ge4UaGXx7_};@La_-wDVJC`k%r*>m-af!~L&g<9(MB#*4g_XZY{Z
z_amzN*Rg$s<<b1&lAhg``4{bC;^8&!U;9_hV%XipYn;W%)%Xv~$Y%AOi4YPjsGLuz
zSSfBdYz)3p$+=cKtQYDytjm3!Z{_oTGk5Jf88WPMp_wrODH?^x4*iD_d_}={!!sZE
zX#FdCdQFXh^_Pui4WsafmldU_%9FefVMXmM)s3GG8m6a1S>J?njb_6WbwvJf(lY>9
z%OuxJrb3pUZz^KN)OP*b71O&Nnx@IO$1I_81$04o>?H3v4m@RdSnPRpdc0)>?c{1N
z6%4rb$7}}5eJC?)TO^kMs0{=2IhyzB(bUYhD->bsdal&s5#(g;%;h>vU)Jj#Uff?_
zOpj<zw5g3B6kJHMZ`QHz9L%8(czk!Q_GoS82cVZ+SwX-tYpWI~<tu$w(#>jkt-IQ<
z5!2LUcTDA_5nS*6hW#d1uN$I)IInXHYk6yTOADjr5AOfr#f%*YEc9B%+PT0uW4;<i
z2oUhWzu1pEjaem=Ob!jjAQ+Ff-?|>cvfvH{TKeSP#~Vz?#5{J-^PpY4gA82dc1mn^
z-2!`!*sNdh_*P4lt#>l~hV{J>71JS%#k^{o#?cjs-7&twy;j=PMto)2cVyKwZ}<%7
zqSe<NCS$+da6#-08~I)$2ri%U;=?wQ7d)8#mQ$f2gEdooI9S*$$2}1hkV!8o$MM|q
ze0}B9QU;sXun!ykF&f{?H(X8*7NA%5VH}rO%}tv8uvv&sj~9eyfIK?giU2Lm*SN>=
z=$UeM%MRufTc~4oZjxG880fgcAcHKmOLGwFw6f6`xt0exS8vg%)Eu{=G5W_j9DfKR
z4{0LM&hu#fQDoH4>QC4hrFq$tw)2&gfiafRTHWc1&PmGlP|iY%J4@|)0Ba)<-Ywds
zmycI~vDWCW-TAX{gd?#q4HF!mIfniLg0ziGN+@;*bFrqZh}z9IMx*@FAqMBA1))>d
z<3$Ofwpt3?#vAV@rJk<J;q^n!hAB;en8R)<@E3XbQ)bFOg(l;_VJ;m|diS_e2mw>r
z$hYy#rLp4K{7yB~B6MQg!PbED&Lke9KzdGs1po>yi!VFpIF}}Z(I+~)%?j(^6qlgz
zbhz{KE9$LMegTb>cKwt0%yPTH%9^8;<uz+*n0uq^eK`G1Vl++YDHQl9ypL_Ua%onx
zt_-G&hf7zF#Kb&y-x*YgB1CZoIJMQx2UXR>w~M#>h9)P;&1vKlAj*|ByQ3EGCX*k`
zgmE<A3uU@V;ju1~F+zE|-y&p~8E+C&@=85nuzzBAHd3KEj@_tuxypU7({pcvkiu)<
zphIA9d?Zzg*c_H4LGF70$Zf)L_z4z7>BzcfXQ+>QWZ`2di`L+kpAbr#@LqIWe!PBp
zKC3U=eKK>zgotZ~dR_-}6(@(wP-6M5ZXxfoDAnR6bD6r(k`6UDS~)j$#o^V{6ZmQI
zRL5g%W_*rre*3Kcy-9Z)kL_NGIU~mW5+##po&E<F=FO4`R;BX^7UFJoQqJ0&wSXt+
z?sW=5ex<}=UFV6-mDGa0`;l2GZVV>LjzgAe(soitt_#V|CUvRQF&g)l?T3#(ZFqvN
z;ebG9(!_0R$PFrSS*0QvT!myTh{Ed;FE8)C_?Qz}X%Ph>ghK}~s1fG6gH9X6RL|Y`
zs4?AhpzXmyYdZZ5z3~TO@F*2c(VCn!@k35)p3()91xz=rSZk}}7!+IdIW`52B^Shd
z?v6n1Tgcrlvu0p2?3VOX>K^Npu;UnQYfZt}wpCz7-<-0J`eN<71-PrvjKJ=fjrWnZ
z!Ka+eB?s@Nb^ulUy`HeIEY&OTb*r06EF9cxiB_I3cyC6gI`DdLG!Z&~O3&IoS#z#i
zc9@#=I+*9KiyzcZf+^C~V$+NX0oAuP5lr1swpM2kW1uJ;r@w4D=Pt7<FP@uUruBiS
zOhr#?QlShs3isNpo->N(Ya(ngUFZffXloepV&|kUr;mQBv&nKa5#_}Q;mYV<D~pup
z%~8ZT;*EY+z<k*=V4Baasv7cg+_uGd*rp1W8K7-l2~os(PE#eMN#Os+8lIZu-^yNU
z6GY||mth!!YWRUD20z4iQp`$j?zZUn`cCdCektQj_r4mKv-LUdn+c$D7>1nM{1E@+
z8-&v)G5YEf6yQ-BP9w(cQ1z-Z_Ri;$9PG^z6cmJdE(RV)V^b{p4_fZnKj)AQ5s_U1
z_%=6VPRkq-Wzv?x-036f7*#fPvRgi48L%x-<0DuzbB`GkWg-w&#(8VHgQJ_tli8bq
z;-qCr+eX*xuB#jSf!lI^Z07*|WsVKdX3W91R<&HPKTTMtQOmj-Wx|T@A4exRyXL^+
zDaB#g*AZ)ock2uo+85x!W6)j37}?|8qU<b@)Mi&~6O=iPG;vX>ZGbp<q@_ZL*@5M`
zWN3=zz}s2tJTl@X&6h32WwjVA&fJ(uVTPuZd6{AY?ERZ8VwvSUsC>rUV#qh?ESk&_
zhSn^i#^+E0-ee{(so|^<(i{QZYqh5cxI_Z+`<QAOIFpo7F`&R?RRQ|4&oZ`8?O0)^
z&w4k0QZ#BbD&|1isOWGhSAyNL&7?2ms4py*v!5EV#!$|QejjlX@<GSR^N=a<!N^XI
z_vy)A9kjjr#EY?=XwmD1a>7LP<c&x#eZ<6d{`Ieq=HrZt+m%!MZ61b8kt`U-MTILD
z%QYh=q`gxkwHX@DHP>T^mLDqsWB&t-D87e;8t$v5P;ZgCUUAZ$(I0&{{tegp^7<Vi
zn%5(FsWOY-8Y;dm{jgt<(AFTZ7HiUoe;WBoQKvpfQO7CQV-=u<atfjdU`=d9tkYq_
z4P~7ICJUKiVs^+;zv;#|7bm%d*0+0V2{*R<DUQ=#j#-r(?__!r#-6r)TC)<DwRJ7p
zwEQq#ZH(1KnLj?x3C-b*TvV?+R6TGp&ng6?Xx?f4n3d02Z3gVQeH2jx_bgPzA;B^i
zS6C7~TvfF;7gw{btXvr%t~=RzZ0ic$giXJ<3z&XjK2kZYWI8tkN$v}aQ?X`WXT<lf
z>C0K!d%Cnt?!NL=P%k{$a51)&elXGGja-|W&Sw5NNZKKGp>Pj+eDsupEH{MLz*7lh
z<1Vf-%No6f`+sC6Y>(w2wD5)J=#%=nQ^p_>Pupg50a&+I+S#N@K7u~UT=wH*zhfzm
zVc(`!H14>}HNM3(|2Eu~DS+yIzgD16{o@dOQ$iNr^y75Q#6#M6kue-Hs3KTw45HyT
zC2hsUooXK?W;<jbYj#r7lp1Vxa<j(jVi{JQ#B>QBt!!4@Hl>a(WL-Tz9Ij(YWv!*n
zc6fBebcvn*CCr+qI$B+ir@Fk(KtHSA{Xi*+Ue#LJHb#Y2-AVAF^>{`7_jgoKcV~bT
zU9_|_+O1sidnGI5&+*}E=P?B@AztIa0g5^KIoB3gIHX)m?nFr@jN4_*(2fTqju!c)
zYFX_sh#T=^?=KI;^u@}s)|4E-{8DjW74~v>BwIw**1Rwe0r_e$ko`0O|G^GHV}=j4
zz}qSFtYlu|s9F%#JsdfhexMH#o$G#ancX~zVM-`!l1xSCdg04{Iqv9X89sVT=~Z2i
zjV3-aR!`TZoUFE)(VfZnG720b(j1{&Kc4w3*{U9V0jsFp>JYeob}3_m*ks13M6{2b
z<V}!ynw+o#!c_Y%p1BA8D{TMI;ea%&8hrW5jQQ2i+iyLNR?Fc7P&XXI5O_MhP#8;X
zR)(@WsX(ynOgML&OyTV*Xk?9tV7SboeG8eh!;*zLPvvyA0T!_D2{CV_Y_+QLWt>RB
zlP9yg`it#h5P;?h=*BWJCM&cjM2{x=CAH-jQfBV!rLRoBxWcZ$zadsBH>~GwWgK}p
z?!Esj2pbY_0`Y<vk1M@&TFx)ub@g^8#Gi_eEHw?G&!oejg7!~Pzd3h*oJAnOetkuK
z_&mTSQU6IgwJ*N=ldF?x(TcIAdWNuW|4CN+5_|AjK--Sn(#^lsA-`(1O-bWu0z6RL
zVfpNP852(#4*9Cf>}O%eNZR(>y7F#(Iu+|d978wvDS6{$$lg|l^W~kAa3c15%j#NQ
z1$Mcrc#)IU$}g0)w5*iMIpt|nJ?sp^9}dw)fmV_%d~(O~3>_QC4CkwmBliT=fw7vd
z3$kkg*&8j2xaVXiD5jO%QAyb7>)$<GE+p(b(&$5<1bIqczwr)Hqe@qB<y?$5Z)rKP
zU)-2YUCo{1agY=R^o1@~cLm3rrnnX__Z6#~(-%&jWQ@>6Ycu=B^(pCE6jYiRE4LlN
zYxeXir%aO_)ec-na{%2-ES)cSj@KLdhf!@<>yGUR+A@6d@tap__9i1$^xWX`RnC!e
zT=5x4I<kL!U(X$o7tIq%)SO2iW)FP(6@+Szd-g!Uyj12K&5Cjg_dXqiB`D!V>Sati
z%D~a+1r+6fNh)Y3puQ_uhKCM-Xm)(%FlF+FeJgcr)bi@#^sW*f6j-tISW;hJ*rbVV
zlJlgi=7ITT(MfIC1P#QwF=|Qs`MW2@T$a_VWdYl7VrPdR1|JNS*G7{mK$&SA885M)
z<u2vrF(x+mLsQC>NZp}%U-o|b<93Z69dvuwZoYICs)1*1yH|nOuDzvSp4mSxoO|+-
z#c-bt>qL$mI^w;_H>SGdP0e_0mWpR}p;88;CH?Y=6R}KI=Q#I2k`FwW%~o1$2qa%g
zve!|D6<6ON;C35sKy|IN;acTFSC`}7<Nda`S>=>80aszDM<7%kEj_fZi}j1czx-vZ
zi|>|hBwDh{<Tj9DZzT{52Vbn-(qwOMFYGuyLQu==?S!_=I!Fw77^wy~_GcoT8PZ4f
z-LMR$zcrji=X5pLnHG5VXGpy&J1qNxPX-<V@-9`HB_%g+DSEsd6Fhqmn#H!uLA|Q1
zuiSYl>?DZLtM2+q@0ww|YTdp}#-3A9QlimHRkH`F8#{v6P0c{)QDltYkUV_xamrkQ
zTlIvce}(n}I+o8e37d|cxbtL21fy)q>uA$|7R3K0>uc3;6J`ny6WI@(+SCI0@~D!-
z%$-b93xa!<-M1&bj#gJ34ha#oDK3Ut&U)Jfjdwh^d*==dLC(18zfgidnCv_`QrebB
z2Dvh2PEZf+7SyhtA5d0c9(MnJHz*2t*}jhCz8Xl-ApKh@+>?IM<G#vCAj}_81u`-c
zo**~Z+{j5P!5^)#osk}t+ci}X=^(ZF@>G)bIcq7ErxVBr0G!q**>}b3at!O(-zN81
z|7Pc|^!8nSD1OQPg#vB!`JT6TMX>-r$U5=X;LfpnAIKQYet$RTmP_22p~%eqF81)f
zNx#aQW&P8`;m24}eU%{T_-tWs!AnIG5Dke}sRq%RnAP&ySK#6F<4h9FvFKlU0BdNA
z^#H~a0M=tB+k`{EvFC(cJ9GPj+n7{c4d)j9fd^t)AHJ%zZ~+Gz!g&3S{<#+C78uU_
zmfVN6dZl<obbOnVTfjr%oAf9v0tQ3flA=vksimnY@qk3DG1Nhyzow3%OI>CTb_KZ^
zHgH%n2cw(~M)Afmt>pW-6oMWfNA7^&4^jr{-bSe_k<~g(SyRCMUTtk|BFO-Cu)(sE
z_DptL%E6}pEV<x@ZKxDaYWZ2DG7LYMseIgSE}owL)u*8$WxU9?sTaBVZB~+2Yb1>s
z7AHG}-o{V{X@uN2S+{rYuuEI8{ci7*EeG?g)j(2fy4YkBa!;6yKEqj(I#N`EuVsEU
zZ_O_bENPIO7kYRn#7EU;37zFXS#DKcQXJBKk20Jfw$d|arIpxrX_-ujaQ3oVJ286>
zv%0RM>QM7G3*`Cmuv$XADoO0~ssSY#N37xZ?(|G6xnKIG8Vni&Z<D*uzN=+%PB}MO
zsY@8_a=f3bo*?0xP3?h+0Z6o9+dhH<_fuTw<FU`O`47wQpF{Mrf~MV9iX`p|_^=45
z$hXr&ykV&^dJF+~U<mRWD_AlGeFt6i(?cECxoGUYaeIX<Z5a_+tgtTNWH$Ye`UDsY
zDh%&-YQfB_fBIp!aa`{c$VqZgz(NJJmh!x75_fFH6v|CoB8S1>!#PtqdA;cpgz})K
zq6bfoiiGS%ICx6PGEj9+)9>(jo*>}f@Uq2iwz>yqj1!RnEWJ+-=7=SAZBBr(j==Wb
z)+g=^X&1gVk@lQp)NfsqZ-4DLAq@|tA5=aV8LNnDs|4vVb|`b1Mpi9E@g7J(0-PqS
zAe0{qA+4JpuMwjbg~_En0j}oFO8VLfbCHvm`W;P_wcMDaqfI)uO=aL#44q2S-a$yf
z$k-E8rFSg!lj~0i1p=ofjkJn94gQVBTTeZHk~Jybod+(8PLOB~+SGKV`{{A){!<-6
zh8P{YR=Pk^&&mEBzwJ~U)|zc4HE)KjNB;Lrl5Cp`VxQO&h+Be0U$Z)--RFs|W+t8B
z&j>?dye7gLQ*WB8oOa)VZjjb8g+P5jK3cvP^6P0V<KbRu#T{+6E|A#}0gSc5r>K`;
zk6N1Q^!FG9H2{hx<K_qDhCMf$AVr6kBDV2W9c4rn#_~;UnceQ6Y();4@lCtMY+Z}#
z0&$)u-R@tt`9R{Rni~dkySpJAPQ_C;hjO<~o5gIo6jPfBMY~o~9K1J@Wk%rTLf#8X
z-(c_;pExG8bxc?aY7duCPh*ZOv$4<D^pc)1p7Ne(hVS{B#vl<*v%u>o${%V|!;=&-
zv^6hd`ohrlIAnSy#Vddvy~q`yM*6;99MdbQR8TE6mVG#=f==Hia6W?EJx~;T?>lhq
zLi(iyAcA$n7??`@3evheO^~taoUD`(ZJU~hwV*b*>vZ7Lt?SVHWFr~wL7SUf(V@vj
zYV|ds^4<9wp$C<5IrcE#^O~q|+AeVZqY=IY+bnNmc4gY<@~f6U)#yKpH!TDYOu_WE
zrX~+$?D)=uW)4ukPB6gVxhUTYlVL{r#ep^11d3|QX-^XVlqApM3Exp{Ov4vKm$fb$
z)95h-L4+a+pN<2tY4}->d<y?15=lP7%HW*@Hy6h@e+d7xr2h<(4FOSB%;6HvpN8|$
z<@ZB^7C_m7hFtq|dH&-;m>IxD>3}--AIAPU)&H&*BlmtmPZ^v8sjd3AO1p0+&==><
z`qFvT@Sk%ig6v3^d1p}WpZxt{hqDy$nG>Xb;Xcl7?(YX*El`JikyB`WVI}{`Xj;x0
zEDC(#mH4+g29$vD%6K}Sm$v^|+?Qhpmbmn5kmq0LhygaFwUnFqPyYUB$Ls~*QyI7L
zB25109Hsccw3YNH{*(9%lK)?sL|W-38V{m?tMDhXE~m;tchf&weIK|XDqdC=I(fmZ
z**!qqt!tSDNHi&Cp*4Ml%p~)r&*YD#uYYT35{h{8E4?KYwL@Zp+vR&yFV`GeFV~=7
zlTfJm(RQx*Q3>N!v#MdcsB;kr>jEQQa&zHCGGbrx+jL>>0=#;)k9hSeGl=TP$$HtH
zHmmvszYsO&;jKcg309$2<W>zP312qHk*wlt8d!Cg12n0T<WmwN+@f9H_{uaQ^v!VL
z8@jOetTC&GkDi&1M&B0^AyDI??^)L&=eq%*oOl~Bs!DYGF_Ktwu^qmgl*g@aKkePv
zP4<`W6`YV}pWV7b=pVmxxl}LuJg>o{>=6R5pC;*?`&Xb{U+YU)6oNNqC`LGVCo9Uv
zw)`DGdp4IEE!NXM8T)>4C0BX}Nvb`#U<9GJI!FYd_`b%#x>ZcfHj{dyxq)C!m3641
zxiTuz*wcMVmw5M<mFSB;jA$e#dNjep9%<l4w#99F#r@~9VczRwy&FSTo_gA$*rQqc
z%z-SnPfNn%Y6u2%BhS@>;)?}{xD=@S9|KVLYz#58pZMBH7_2aQ>T30GSv8o3f6%X>
z0|TXJt}61JFMMA$P)8^^WEDO|TK9|P!Y#SpuWfOypX!8ITGqjHU;(QQMn7&6iXSnN
z4wn!!(EeEqls#==wWyJMdR{aMr%j>V(mNCl^LEJ!201s0dG%@Q?tVYIMN6T4EyL*O
zC`k@%RQg<R073tW<37SEXt$b^NM|TU%QT+c?SAn|Zx^8;mglV4I6y_UR{Lu<&T3;K
zS1$;8TXs<w2(vl@ifmOC!Yzs1O%+E&^d@XGB+?%rysA8j3$8RJC1{ALq!~5UfPOgq
zfV4MCayG^yRW<@?kXaf<ms273T?$5tFmKv&-c34}k9a0Sr-Kh1Mu}>VPoIosH*IFC
z#Us7qKt&l0yWDgaSuJ8M670Kcb05L_yB5%Egt=9xS0YzxhNJ&0^~S0)mr5Py`PPSz
za5?}&635vRBoOk|xzsFFY%1$=GOM$k?exntfDLjO*ikBhqr0Ecr=E&5{VW~W^Sz<o
zi`2xKDcrKG-csS6KdcxIw*i}Q1D_$M1gjkR+J{TX2KC?D&Ayj!FA|X&!F4#XC<I2>
z$l?a@XdLsb+71l&rJIb0M}xR!g&rb71=ShtwgV;cNM<^YLBsD3jC$qkcw3><H}tCp
z(pE%OKYQ7qZ=*sA1(A&Jy9n6vMgMdUsmV;%x4vT7XoYO+Ntk!}U0ybxyRt*yKJ+)P
ztEPmN8E`xl^M3gLd@|wxHNOs+-_a$^d$UKUt-$F+Bw77&p@`_hkvUHM9_^rY$|jO0
z+D`==`U)2j0T@{-Z6=b){b>9t(b0<ucNNB0Qe;-7f-*SzJD{j!C_dm>_weoiGXbBl
zWnDJHh)&n#RawhNFR5|7qA#^t{8$%pH+8INdyw{gMWUjq0`H-6L-{GP_6ojMc;^w2
z*-YsTlYOTG(_5!n4-%CfTRBcRhoA2RQ1)bl)hJHJvx*c-q`g23P|yo0D(}=}DpV|e
zkgKam|BCqGgXut{d-Ex!2@gw})(7>1%Wl}KKoyOVAW{*~u}_SQc$;XqQuqX$g{o{O
zcxOrLJw{hQU?XS(J8BdrNX&@NSpbd!gbGXF&5^?q)muc)Q@p(!&84c0g$b8b1KEZN
zP3yx7mSp3)wMGSJM%gXnEd=?VMV!kK#rG5N6QAm{@sO4giCP-lUBOD@!{810pf2HM
z##gPpf(}bWMa|kxj<*5_lKn4VuweEapd(d8O4)^fq+}aG3)nlkZ3NddDQ?ekLiHyB
zrm)eVUcow=saG|s`}Dk`isBmsLY|WC^=1BZ)=&YmJz{P?bsnzr%YK3xi4w?S_W~#0
zKDMg3hv<ULa260`Y~L90x@f|nk$HA(bOm@gZvjnXm92TL0Ja=lQE@nhC?QjAaP#>V
zfUuA7AOJ+wbQL_LX~XH*)=sXa&y=<=e}t9d7~B*FP~7~Pnl&FAJN!?81>71FFaQ%`
zDa1vS@H1gFBlzKC{7LVZ!h334nOZ??IjmOcF~PqW1r<B3>CZQPeT3VAm)AO>MLEZ2
zA7KmZuo}-bI*(%yxf@G&UrC`@DR<m%Q1>jzBtUipmS^*lK1JL;_YA_%-eUko1qR#N
z$n^;feyPxV9G9oikQs5q;)`bI+W+5aJ?t9RgX=5gF_rTMVasVAuG{0mz0m8@ssieL
zTukB~j+WgE$^S7Ohz%oU=Px74JV7c=&o3^bFc)1_M3xN4kLKK(ITJxFIwG#(jkNIN
z?)z%7%5h1BcODtaM9wkidHfm5mjJAVlAmeI50F%ndiA$tj!my<xfEK3R2uGo>Ms54
zQ*dE!*=GgR!%XS(gAJ6zX}%cElcmprI?{Z#lJu3w#69l@7Vg8385A!@|F;Q`_QZQP
zmW)%$%zoAEYI21c$mosMqgBbDD=h60C}Js*;XF$MVa?^6M5jM|*?)}~kz_X=JaqiN
zh;F#(C$XUJ`~CuDOVR+Pm#*l>UNqtD%RZ~e&T85}T{|4{C72|w`kur7yh~S0LwNY2
zmx?}!xILArk;58cu*TLRikw*|gqgbva~jn5)l1}RRJ?A}T{s0{Y~UxfUiLU%G=Xr&
z_pyYBUHkTpnW_9oKm4jivt9#px!pT7>6big{6`(AXa}Iiw+U6f5Ev@jwR>!9wT>TW
z!YxQMN)I@pXOY{F2g9?<6-pPxmV5?W23*lcq91Qx<ypNwZR5Vgp1Jv(57=ZBqVtp0
zkZTU-#TXx9av)TGm1wDRG5ZLMUd0!5rEi`TR2Q?2FOC@xX@7T3f9JrAlwCr`Nk8ji
z%2|+DwJNKRAj5eWfb#-!GMkAZw$*$S(Xq4mj@jAbVW<iLl7XGx0su^ZZnEGZjgIYC
z6Nk}p17BH2Q_qv`VBkGIP#+caUkttpFl!dcbisHad!LD%-h7^(&bAxfY-SU!HNxdF
zG+q3>Tf8J9Tj_NO(ke<j%b#)#Qjt(Nh9GWz;0F6iYidc<x1NLR5qGyvDj<h>NP*U6
zWnnyyxY4D17pp(5;1br%WN?vl2O;}Oa+S5|APM(bywn!i%VpX8>2?m~g*GvG0q*d+
z?;~SiC~%m&!|W$T=vt2r7X4Zp-}WAri|IQeIr@A6lcn#~S&>AxL*K*B1(ufvJ}4gw
zj;TY`O2d<c!1783foPz5j#v|tID3+TG4S1PL4p7?bKf`I1+Q)Q^2Y`){~aRgROcLJ
zBW2!8AV5*ZO;5XaA#==u<gfb@yCGM*Q&pT-KZ*N;u*?*MWsNWJAvj2E9)ja$+o-dg
zNCY!8^laFMA&%^xy3|#8sNb@(N+$^YrGIE@D^*ONkgZ;IkzS;E0f5`#<I+MFQ#eRj
zj=7sBp-PSW<rd@b8B~QFs)$aS_|mODJ^7@PW{AYxq6`RbW8ZEe5nfVqmUiJ!5h{Dc
zO1J!Ib~2+N2W)elB??7HA9gOflE{LWpP^?~UNm9i)C#x6Z5fS=Y7;WD;CRic<`e-t
z_OdSNt~FMbR>nl)69Mv!?WwkCq_;~xe|^Qi(71oPqS#?b!x{5<>@88%k9hXyyH+H*
z6I$m-Nrcm+&Ju(NZ>uf}JwC!GXk>+Y1T*BvPGI>dY!>E`JS>R_HjvT5Uxiv~g!e@u
zbp{9RFxb(B?Xi&v%4Y;wk`>63I>D-#s}eUO71YK-)HgLb*XOAXuMRxA&|Bx3H{ZQ6
zBy`b)@rgxoe|C=O^d|wx(wJ3Qt4S`EDb$JM=uX!I2+)2QqI?-?vdFuX!SOi!<;WOW
z501k$$-2yg<&i@bDiiIgLxH!d+}Pua`}*$IG3Cd7YohZ#4X_$Z&o+F$a7r#92mm_)
zoyWFHnD_JUMHj?U!tP(`@?Y6%)Yec58Ipe%PJ;qlhX&ao$sdpCt#AwJaPzC_mAE(M
zaH1XrdQFaJm8zknErIM&m<4VL=H5jtMfo-zoMV;Hd2-?0D0|${0EQ8Ops?VP=)6r6
zb$jzXdXFIgsHqk!hzj*f6za4D9qFQxz~p~~beIk|i1timUFN12T@Kzax-1kJ(rrn_
zV$*C`90i-DIyIT4T0w~(EYII)Fo|nK)46bVKEljr5TK}z96slR^w-w5wU{q`eRs8Y
z<D}u~{(8}LY6}&OM^8<|+nkS9vuI61*y2aoPtvWbRMS1HqVJ7nEuOeZTwpd{wl;_y
zC$5JFA_<^p8?DJFlkFD*gr1(GgCCS7#@;?fi_N5PAv<^PX4K$4#_+OQBu!sn;+gLJ
z87PB^RjiR~5&zjC9~{^<hx87qBU#&$Qd)jKvxWKx#q9Q7b9^MrLI-AnGv1Yqtk>)i
zplgOOtP5CIj0Fy?>JC9TGC)TI+2wPN9-e37Jug5nHBwu|`MFi#an?XDNACM1V&{ej
z5(>Y#Pfr<wFVu>Ph7NocPDiKi1sx;+C;KteGr}*LF#o#(sHFpXO$(t+AlDE6RB<n`
z`a6KG$k;qPBo$HiC;(XtebBjp!!%TYkk=lWC7kaDgJ9YogbE23{5yf?<O-kFTa_9F
zk;I7cf)bQJQL7j(Cct^`Luv9hO;z-qaS%$S0BH!cKDsbLa8MG1JzOZz4M~Z@{=l7)
zc%VGI@S?)3KxTR`laS=^^Yz(T8BA1@>_R{<r~nn=)mj%+fI=5==uz}57qT$p1$_5k
z%ejthn`U5qaZ>ix3)S^_gKnq!I!Hkaimwi!zw$E!Qns!Oh*&Zt;`BQg4~nnaS>Cg`
zxO`D3hXRfgJYVe~XJ=Jsh=jm1r#zRVFDmGES$07j$#gMQB!IsplYm?>kSzfy6L=HC
zh0K5wU?7IXI{BP||2_f?WZ?~Z9g^N+zL29LGZ?J&e)NB+2^EBvmoXQR_Ij=X{hIrT
z^ob#nmY@x6Z+s@#`CI|{XT{;wjbV4BUQ6JJy7CqOw5#X6{^zmqZFMl1KK<|oPiFjp
zvYGIB0%XK|4j7;zRh&Rto4_~ZeEodk?tqzT0j0e7P#}LnhCda6jXS2L7g7Xk^GuEr
zobnfKW-g$m0r$i8izdpR#WT~O5LvxgVLfNgz?9S!DK?;FqXXSU4Aib(XfxrAiuwYj
z3o0-$J!mL{4?60Rl4KLG@;S$XmvGNt1hM$p8R&|;{m(95MF5-HPHYRfkRm4DcVHTk
z5xGe5`G7y5j9grV6p1u5bHJ1gZe7>8962jNuhuIXdq};q&&(YW@s8jEd5N;Y3Qn_I
zyQq%<I5<N+RX-J+b83BgL2ftjE6wcYc@rvHJ~dz<9~@=;!+`G)xC&4*<g(0+)XNdn
z*>W00dhh?~1&TMIo61!sc4RjHn-0a(GdrfBVm@~qK%1=qN#q2-R-Nx{^z?yVG2~v!
z7n&1JBM0>Fuwji};0+}O;9Rgk`h4Yp++Vm342H<GyL(QpZ{V4p#^0K=oj0MPbpmag
zm)#hoxL_a^z@;Gc^EnY<9^d~q1b?ufM~rV0jbvxSX{LaVZmzD2HZup%k{*$^d#*W9
zp;!k@1yVdFyQom~fXE_Kf`k@Ho0M#iK(}wUY8N5f|5y|lm@2?T8Q4}XbZ+(lOc{AC
zd+mY>${rUmRx>FF^F>vxM;~<ahUokUoBvh%1bcx19G~HFA{7IDht9xpcYj&nj~qc<
z3Bb|aV+VO}<T42LIloex-}}=G#rMBK3E4=_l%@~ptbLV)q;4eNb2bo;BFM9ZAC+H#
z%!L&HHi$!Df7{fve=1g#xW2Z8vrj)g_{*aRBd~Ce<Pl|Z?SDkg1Ax-5QT0zof{ZBt
zycF<{?7^2G7@})7!(X-IC9sd7+?}5Od%s=y4PfMs?d?B%K8sHQHd2M(I^(w|1AIqA
zrZMpw!vBk4pcwK`ZD$L^*A0kzU~AFrcO3lB2-FQM+-=Djj5En_<^UU#x}sM8H)p=J
zfI82sJPN-B_+1nrP0-(BNJ{~0V~3x2^mpL!k7~IuIrv(j@6liRgB*y~%=>%?ng3oO
zX{>-c{m17>v<b94`{9%9*|*te<}VQOrtG(J{!g?RnExkQaR2WgdDH({EhgUo0m6Sq
zZ~uq2Ap7|L7;Ak+@GGW29ZmO2w8*N=JjU)MpE^E#aEq*v|K)CFd5cA1jkYV@*lAlr
zXj7j|Cr7XCl>{;1*6yR{kpAt*zh(euN~7BGr)cK;0z?m=K4`=m^CLPeY`cY0!U**}
zIGQ8i!A}U{sbaM396BB@ggK5(n%E317uFF*2yNJ<P9&Km&>Iy!Sy<VsQmdduc+tu(
zey>V+{Ej}8XvnIZ`?Tw|ZENaU%HsHZVgVjua^0|43nQ+Fb&9|YBzE!>f7EK~exj98
z_r!3*Vg>p?&RsoI0=_MhVczX;cjhDf5V*-f#k|}HDXzPvKNj2<nxmUE(a6XG<gF|$
zEJCfyw5YwDX6M;kR{EWJ)!SK5_ZO5mW|a#Y>#;pa2nLIpT?ajTshz<s#N+|f4Q%E|
z?);lQ?4EkVD}?+z+qvvhkB_3OT~B!28=Y%wln&hIy27~;v};Gh{d1yONlu~}l*K|K
zvQ@i?Jt3B<BLszB+$y$V+x^Z;s$laj+d;<PZuczHPZeYN6YR`_WUdvp)agkkF~-#R
zVw}s)phBz`36Gsv-j_Z<_ML3{=?|x0@FE0GV4HIK`ZlnN&0_Qy!#ykyyV>0$h&bj~
z_jd}*!r-gbda=;XjnW6fQXG+iIOgy&aGS2;S<=gKu89;2h`+??VH;h*y__jhn+lKP
zgZG|CPd@k3v@{ueBTVc3;tDQx@wXJbGhw}ISk=Zk;`|?hCj#t&ZmYLB{*Li{UjXCy
zl%v^SJn6ZYuHlj6KYc(r84SxLB{oHP#cJ3jLk#!VKg>DldNtM9j>4_O?7PO9!O7gE
z%?&Mhi5A}0WTI(fw&5d_WzA;+s@KEornba7INatXp<gnS>|DMG&X*AJ*tv9bBj90_
znF{v^FYaag5&&9C7QZfY{T+-d8Ub;JUrBQtggbwA-AKGu@TTPW$13;f{lu&NQ+@ri
z2?~P56y}<FnkMm?{Z<s-{49Na22F-WO}7aaZE0F$j3z^_P<RNQY^`IDyf?#UV$)XB
z)3VO1W{!?;lOGdLBnRi2H@sGg#)d<nL~2a>g5c_HGhs|z^>Iyg^3lnT?D5uQR`HQ(
zlC26;-T0S@+UEi_7PDQ|<g9`{6a?)oTL;qu3N4-G{CBWf`6WCYJDZ?aM$OjI{`_A6
z$L2#l`Q-?1a9lG}!+90*v8Lt+jKM0J!;pqL*mbvZc&2he@Y-;UhDX6`t*x*A-A#OO
z4}5U0(xO9f%;8qD!WF^emzMndJ@!pSuI%m<ZR7UcMA$5B?o0;(l~AAbu|$UJlN;U<
z&4^PkE=JRxL4r4PG0EjEQzA`P=#!owz^#zVg$X{afX6!CJGe%xg{Sw9<PaMlJq;tn
z>wcJ4`o#&p)<L?KucZQj;v;(D*x$w=91d3DLvo)b6(O(VS8<+e0>#wPWj{P16+g!X
zXmWl|nq*`UcCc0~o@lzBemVNi)R@i0FEA@K+aoi*f2%NlF_{xqrK+Y!_vVphzlPDy
z69$`5*ZC=0rSCmsWnYh>O-8t<Rbw(r!O&kcEzPCF0&Kkpj>DP{SP(N_?1qh|tj^x>
znh2u=V&@LOO#Iy>m(9|-Bd*bBm3hfW-lxZ#MNbG1bjF>#pC-4_rQZ44CF~z3dL3bO
zXJ@P35CBtH(Aw#o*u%oZxdy6B)Yb6A(nklcR%v)ohZ*6ax_ghPv=8%_Tk)$ry-#gS
zP=DYndzh$Ju`u`U>|A^E27f@+8r&pMz2mrVQZ&Y*pU<yTint|TRkoj!ZeKjYrXX~Z
zNVl^wmY7dOm{M+<x`SzWc;C*Kvf1W$ByaW<7;otHFV??v%7<q;f(yzgWrQ5}DQ#`>
zZg+A0i4qQ_(q}gMu~`iED<?nw9?k#AEne!Aea%T_46Y~zgR}kA1$7(WBvHS|Xy@Ho
zNUm|uqJa?e!$iDCyQ+V_S+S3YEWCa~7|$`La0=Zwv~L!<;srgfu`^9vRd{XRT}PFO
zx*cA~O19O>XTiU@C}Z1MdfLS@W>7si6V_j+T?@f)jJj@Fabm}Qw9@~?P1D_6hQn*<
zc++qr*Qg-ac7~=gm1UB)w)E)~2b=WX-mYdS<@~SHTB1<q14N!?PGN3di6T=;&&ozU
zwoUVdxA(m*>GmH2hdqU~30I>JPX`^n2+Ai;BniYJQz}|rYX#L55$<qz-ozXBgL7A!
zM=d-*lw0F2j=!)VCoiAcFk68R^KK;S+ax1u``}E70zG%wfM5f=NQci4?PVGfS&Q)U
z$=@loF_3P2{q^5j>oYuHss6#q`l|G@KRgSe#aMct8&Pqt^Ih$sJE~hiypVY2<-D5b
zI-X5gxCoh@nuGbHjWK1LU|lCY=rGrgns-iKTlZ9oo99Z=9!0UJ`L063C6!Tm!MVd9
z_NfZpkiA*U^3E3{Ic?zj318i#rLmDd_5Q9BWdefy@--#fgVZ`g(V*+E2kY#_2!!u!
zwKE%?B8C`m3uva6Vj6n6&FW8oZa7V<p{46&A`#e=ofFJ3t~;?C5h^~hUhgz+Z;|0^
z<>tvsn~x#&N@6yO>2h?yFDB-y<C~d}a~gfOv2v@$G!=40v$qwkKKXjvU2<pKVdC4y
z+(Ux!7IKPP*vwACYTI+(BSJ%`%@1nSsDfe9Ev=wHzrpw07yawf_BPuvIQrETv<6G2
znGxi7em`Z^r;XP)*7qD{aDoH%;s!TO8h)4uv9`eFC(MP(mhw;5u$V4Un~y-Z1(OQK
z?#bMV>)e%u>aigDy+&Ydl=H0;5uvU^LNZPsU9P(zlyJ-KUx~G`c97L^9ju#fxnhGK
zL{TL*Jbh)m=0Wg?dv_`HWrl@_IZw%Ixo%DWuU5?j%h-wT;C^p!Wm%5SsEs}-ZHtO>
z_@auFg1!>w`xx7!pZ*Hdi~1h4*Y_uf`E(PS-M5e3TdLt+_}u#5-^JRy9IISz)+M<e
zh8K;cm^R~ZgoWTUTqo{JiB=@(2cg2lE<yWYJ(=5$+if$?K^<bB>~|4{`7Dd>^&PhQ
z?arG)34jDw*zwbp^Qk@5xg?C_;gbx)ZwgD`mL(8U<Pv=Vfw}k0wb%&wT8T;7m*LL_
zqJ1B{tZD|QbW!`ei|q+c?l^Z06~QN$WgS{0ygY1=k7K<%V;J41-x)!-1TL||AnV<A
z>*86IG&Sz?n{|_k+ISpK{ZE~rA*MT`I2p6-5|b$B``S;5Np0M8)W#PU;(5H*5j#|!
zUh$4Y=8U;<W_5N_P|ir2n%u1An3#GQndu$Q%esYd_+DZD(N@K;=Ae%jKka%VSeRId
zosBd0`oC<})Ax?SWk3;lm7rE0ufN{EaQQF12M2e-MvK6S+wWq;fHWwUFdlpwa-Z#H
z38kReZCUZ8Hxhcpr)~X0iD8i$6UG?qx?RNnj@;vP?;T;gY9@7VAK#O{l{c#DnsgTG
z2hX|rfFyP{O;g<k`s~cit7q$LjQVXnbC#)}YNh0Xog(baKH0b64%VAvSms3S!t;{@
z({RqvzqqEUS)i`juH6)PWIi;|#>h-GW?iFYS=Nd*;;zoHEOP{#;DeFzn~%A*t5zi&
zI4)a7G;(y;#xNsdVtjI&cbiG!EtdAH<TEV_!9x1QHUultZ8cj3ErxT0-U*&H^KJfa
zp4{P-nHlrY|Ex}To|V?uHU{~A+khSr$`0>?$bo{XR9E|zFUxSKdy41gt2p-pVp`bA
zq~F_Gzc`iNBe?e+??($|6+b`n&A|8g-_=Z|5BSJ&ofsw^1x*-*&<93em)joEM(DEE
z&YY5mtJZnAIPH(-df{1kTD>lab-=nwehM#QKlOkPfeVEu%#aZo#0U*-Fnx5YNnFOp
z!E*GT$#uw#eFwXzj^lxE(ZKt3nV{lsDlaGV<Jr$bj=V$tv7EKuUl#kH<z-nMx*vB4
zjg#<s?QO4YMk@R$i0plJm8__tD>Y@=f|wuF6q~g7c|~Ngacc|ya!~&)7b-FWfX$!H
z)dF`N1<2a-UrMUY$JZULiNT3sxy69%6Abp~xw^xp>Wu{%`0IWzXQSgI--Tjj_the&
zv%oYu)^QY`e<DvvP?xc3nrvUbvVLh{?S{5xt<tc}bj3{J8#tMZ9@p|LqP*+73Fcsb
zSlqYkhSI|3fgR!80#TPt7IfFW-%W0COgK=5!VP&!`<3*)58DNbVC50r*423xD&SUO
z7}IU`-UV=#{#Gc{apmhbxU*k}xgYVKIPD-jCwaFz`WxdG<XP&3GMBG`F!)}r0)WK$
z;KtuxBke4!$Y<|z`@QuvaCX*eQvCVX)q`WSBNLe?`DIJ>w#(r{DH(Qd*pF!*2Ggq^
zZ?w!9?iAj+v3ESvx<W<;)1?wPTkjJB<_Dj^l@i*sSm)8K1gX43tKLxx87*SBO2aM3
zb-LF5O<NbXAuk7F{*{Gc9eP*i^+~TS1;glQLI(Zyf&Ri~5A4-g&$`19cnhxuEBLP@
zhPyyCNCfq;E+^u1@+fl{Wor5iS>ElcrZ9#>q*`pM)_(49l7u?d+&^LohQ%~}lLjh-
z1bY59+yKBFGv|-0-`l1G?*MaeE*oto*liaUhQUwQ>+{}DrIy*RR3Jzu2=>RfY#J?_
zVoSk=wh|AoZ4Y+|-rB7<`fvIbMshs(SCdYT5)A<?e}9A6TB9-=F&3PH9fRQdCX9?}
z!U|UF;LUrwMr!JCRPfGW>TxI;NNNOWYODJbY)5UJR|>M_TZXjYgtmxlxXq8TyT~K>
z9b~iYVIn6<sZ)%4?yD2>a8~mH%K0pb6YA@H$KC99v%xM~(fzzN+Vf}m2W$Ru<XJxS
z!*jUsiB?yx-O+5M?MU9c528UT<XyY{H|-en?99r%f4RChmvd_u^*=m=-!F*Wpp9gk
zLi~kPs6>{%s|OF<kI!&<Rle1WidE=z40x?7Yx?m-*EhOQ$olS&RSVm`jWI_c(Aw7d
z7E@^PkhXJ<C9Kq%b?MhL`q`tA8;LErxLS2Vxp%Dpv4i!GeP!QPu!|v^qE9xFtoJg7
zylrZ)(DYtShF6{EP^aE;ZGG%KD@*4fIgEM_Ms}x;cfRe%Sxi<=YprjMAki0iU|q9X
z$G)&&5)5KkV_Ck{?~=GR%E9kV<Qza;U{MXOxWkGDhxqx=qTxrI>z&0%{Q>kil?O3F
zMxpr%A?1^YY285we5;$)k7n|;4FieAn}pVQ@698QnMR5=aS;mvOp+Q7@49645j$;1
zh7C4kiMmkWrkaZ+Y~pu#m}{aZsos+>De%m#SIoqS8i6>ob#yXYq=RQ+f_!Z4<%^*p
z8J2NAF2qo5F@m#eYIKErgeltr+`fur8qz*h-{_OEX|ipxB-1}ynvQ>3wcDDd_gY|&
zhp(V=cS`f~RvUI*XET2?qZP~<I48ktyS3$Tw~>|fcT_>9SrjMm(2W_<QzG--K|%L?
z<tpQ1K^drqny&GOiq*{2Z4EW82h7aLTp(s`3hho(SyP_7CULS-yuV@K-4O%s3+f&U
zK$sC^L?F<Lj+HN|>*Dqz9@g6vX8O`|XT<W;HlH*+i+F2uY;I~#jGO15+b*BLo>2}~
z`r|))50PN^EMDyMVeK)5-KOb~6v5laXC$qiaE;ya$J_nI^dgAp$a`J-3_1Y~g?1On
z+?PHs!^Y1{UtE5ETe#Qhq-nTOIsGjtrMt`VnxV9ek1=iN=~m@*`;=*_=b_7@P`9aw
zJa5WU-I|TiqAhWb!~c)Hw|=WKdiFpCK~O|c6r@8!q@*OI5u{PNyE`Q|A)<sx2+}1j
z(%ql}BHe5nl<wHTrn&1y@O;lX&wcLw1I{lzz~1kAXU)u-HEZTGvpUyY_=qg?kce&m
zC7NyXCoSNRN{;B}vOHdxYFOs*rJGQUqYvMM4yEPCBi`h0^nG_!&7UshP8h5(skZL*
zpK(~=A{<z+sM!^rl<Z}BF_teTj8yU=RxMeZ?=~8Qq`GKG<pM#;z)$8`N-X<5)5K*z
zc2Y<+eB|)WEVTx1uKm!43%%#s&cVcl7dhmEuBshkX9ztwq1AES>RcTLRm8Wu1)cvq
zqm;`MIozU&j5VoFpIR*NE;CM_&dORl#yVpu9|c`mmUf7B+%nMVs&{mef)Ve+=|4!N
z7~R;`aobG<Evt}!y2fq=-BVT5ONB{c^wk8vnwaTWwLGjT+!QHknUSvz?;LQOd#Bju
zZ7KynvU&l`oA0dzlZyjY1OpI@mN4=%XLk)s^#y2XPa%H$8_saiX^}SY?;g4qaTBMO
zW?=L!2_NQ9ok*?LAbHh=6nLXgo6jbX67jN<yhPphq}_j=<^|WsFrAfd{=qQ9nF1?p
z>68ODJ)Nv|$NZH{T%_V?pYIW8L#^7L9{)QGPiu4Wu8^q{?I)UPPS%eMPiz^9vl2>A
zo=Gcb5Okg`wnR6zW@)M3qVMwr^}@0w+l|<U1^h$I_aZ!LqhI!KgmtF6t!LvI<nx$z
zgk-s-q*Khu6Kq86a*>YPMHtQiN03-hz%`BgxJR~lg%;_}Ts@mVY>*RCjB&Wp;B4~5
zFss2>=b*w{EROTjWKcx;-d?$yMQiT}n~kd0ZUJ{~rH4A4f>t|PB$}Me!(M9-?)|f-
z8JVMGlbmcizu!tp+)}PtFuIl-?qp5W?d#?A^7ynP{^>-rNuX8j?```gd9Glku30dc
zvo1bGrbCj)PUZH^sP#74_KeQ$D(wgF7|oXoOW&%wXdY_Y)Sb|e+;Pr8^vkdtKf$wV
zXHsD%UEV`PKi9qOPws5)vHzQH4nc2pthwdMqpN1HzRmg+n-Mr!)8G4@{_a(J4q&f}
zNL$eP`{(O#Lcr;|8kuu+)aPJeKL3NgyB4%iI|T>bl5X$LWRuT)hr`n95@BTAID~zZ
z8(x_hGxB@~4)*l96^1*DvBelW5f0wjsT`J+o!=x&wC3YohRCJ_6{`i%8M7x<bv?QI
zWa5CXZ=Y~wtPq{Bcj8L1vroFt)Pb_l_7>T;sHh?L)go5b-2p|D5`TrBp(&<LCAaNt
z)e;J`^(xbjpEU1;g9|f^he2ucZj5!gdC8P`++1HW)4Lbu=CyZ^fswq~EYRI&s5h9r
zt)*xRP7%DsxSbbJifaHO#IDx2e2J`30s*&2oU4!;3o(F5tcL!l-3wIoqBw}hV80i=
zjLnS$wp8KeoJSaz0nSm;^lB>4-w(g(fqeKiX$}6%@BW7~V1^D>4#S6Hfs3>A1b9oe
zu>B=2Dy#+o=3ET!L5CorL@>;<l@N#jUzp8(&=9agPcrA?Pl^{%_2od}`3WHpf~vfG
zIrRL(;UETY;d=NF?c)GoH?wlNsL-ch*uiIs*a7nw2Q6O!TOedK|5v8=2uv)JXb(#M
ze!d4eWslnqCH@9d{zr+D&^qAec%brm7TFoV0<ud`dPy$Ss1Q8rOOr=-p+^6SpDKL=
z+@E{m>g0bROFlN>s}rBOFVSMZKs{B&)1n)Y&Hvm5K68FT5qAOAxfuoE8HMSCm&lzj
zAf2mZF$`tDeN;eVwe-isl?Q)a%AR2G7UH}9ms>{_$hbSush~ig4uJlfnO)bOIbLLb
zzk`$K&8SlM%kM^liAiA}LGtUl0e)AxZ|r#iyQsd6+vMc?=d#X%!Ub@1z3CBjwD;i!
zj>f>qYqN_`$YV9|==X4yOO(rh;-oboq}z?O+|a)|BE$v0YV(5R5~DN=;G?aHYUt3X
zpb6UlDeeC$?f)q)Sh@edEUkFsn)gocaEwFSZ)~^?HbC$Q$pWLsIj4G2#QNeXShQ=a
zgBOs`(qdo@^q-$)#-HImUG;E3hNAKZbMw~2US{0jrPqK#VqctnbYTIO0WL(5d<GcD
zb8{h-1I#p$O;bw<DY+o`ek%9+evaYqck=;3EW-@-g-zqwaGY}*I)N$UOjEt+)Aj;F
z3tOKIUcf?2zXBRk#7}{98p>48OAqBqiViXvnnq05i5`gGwEQ9f8p5KYSpdV3eSm9r
zfxLVH8G7sS!T-YJQlNt7s&cEK%MUBmmlCuKC52jg`Ic(2U&l4O=2Ko?o=f~rB!#d#
z4d?F!K+PJkqNTHiE0>TS2vx*Vzi(hwHmG6`I=D7{&B-T<)_$c9wKwscn1_%;UqO5>
z`%%`9OOwe7?Bc*%?Tm{tQBZdo?HaMUnrP##?%>hN1b{xDtmR`iHSRX~0PWJ#1Z)~D
znU?ki&Q1f`UD-tQ(*8Z}dA;^%y{2d-zvkyV0-gI%s}U``@93<cgUya}JXDYHOBU=C
z#CdNnfv>(`2O{LKxc3j}G*I#_uVONeN$iP9N6av20XZ{l7#h5{x94tC4AFoJXu!X3
zq<taRIMBnQe@8Ue_a6;)*FO+*F^4=|ujB@O-VDB@6Lf%fN}x|o-NYZLrNK2o0lV42
zqkmQ#P|zBz^8MccyUd%LjW{j%npU@Q_%?<1<YGYAGCGiZYjd|P-vcCwKqL@ZX<y**
z;#2?$rJ^~8|3;|04)sAP0`2>nocd)6Ko4AU1JEgSZOvf;+8XeH_+^KcFU9`_(xR^k
zH~yg)zqt5B>$uaGkC?<)Zv7-S1YLCAgU%UzAW4{cK|(4-0;%i71<)-HL>l{*8nfB|
zgM@JpL}w9ryalwkn0bpQ7eMMVUR2QzwGkHTgjN<gDmJtM0+wLPl!G0*`1-wzKLL#!
zs8EltRnW81e$s8B8MG{Gt_1yJ4C)W&bC4#M<lre0+?nT>514s?In9KokX`sEXqY=x
zcAy_y6KLv52x;XiP$PQ~`*{w7!B||lK{dY5A*6iWYG#)}RK3q&Qa)`}|Dea9vIz9C
zOn%3VLFTFlwIH?s{f^-0b~}qM+<FKq+i(KuPN#cYi5KhYf1X1D{6uv6<8!h=&;gJN
z>XJKbmKI!0(<&Z9GSZ?g2OP%Fr(k*Sa?xsBx-l_Iz+Fp!U-hpf4A8(_y_h--x<wg+
z-g)*z>ZZ+H^;^N!HU&R{&VJ?s^S0&v*CB!kn3?|jGwJ_T{^E&Qd%WFVv{pX$6EsL8
z&^_#n27Q3@p&n^fp*0DRo$~}v&N9rS^b%Is1K~WqKWSb1OB65Qs1*~hG|SOOK!4o3
zfUR1f9W}G_gbq{@EaVM@c@pVB3JDVOzW!JBP>=sN&`paRbeB{;{*B`u02;S3GXrU=
z>G_}^R<@gi=Qu)u4%+{elD=G*{J=iChubGyuvQ50;k?mjmrFe!=$WSV<m!AAp8N+&
zQNID_cut7&2w+=cm001I$H`@RU~#<-Q^o#ECoyGW9#bkT7N4CSwJi3gm8d^v$SoAP
zP=}vTE%ExW`DM?7&mh!DFS&oQ&VdUI^MRV{#G^X`&HM>^(j~uptSHN=JW-)}6;S;c
zoNhaW=Eg69o?jp)o_u&0^&c9!3H2o`pLjI}66i*tr4M1z_iMyE4rPjfMBx7k%Q${W
z)4BNme<H+>1z|3zwu-w*`{S&e^dBF7WWynRBr0|FBNf#hhXTtsK4Y1W5)auU-Cj-P
zVA9aMRN(kb9quM!LPgESzff`9?b%g@u=A@!9<0Jw?7K#0G&Y>qy-wFQ)GaArUqQuG
z)0qa`=&N;eRIhf*{=}lI6UC@bRpl^O>`_Hefr?2njfL{&zV1VZTWGjKS5UwYib_;8
z*6+Iu%a|0<0ML&>%v|*n@#UdB2Gj1ONYKq!0_<&gBk{7})7QZA8YI9o2QZ<B!3ZMq
z;KZ^snn3j6^6@AFZkP20r=`B;o5CuYGK8^DUGncTsB~=4Wxj?+C_u+F@L*~?xFFz)
z&o?aGCbY=}2CB;uFsZ3FLA!J*SJ%qb$vRDq3LB=GpTQA3OZ{{XXz|c<3pa3^xGSu)
z3I9@z`8MQ=u*_m9C#Tr?q`(hGCcA@eV?sf%TBo&9WuS1(f$!gkgI3E;ea4w~5FHJG
zj$@{dgWG=@aRucJ(6K)i!|;z0nADVojtg?vv9KbXSB50>G|FZ{^RsA1^<rxKyYbL-
z3*Zs%hTv?<3p%)UK_7qvYZ>f+gGysDYE{0VqN1W!%#>j>Zo8YU+fbj;F8v!!!$$~A
z15F|Ruk4`Lp56v|!cYd&f78H~5SiZJn(0dw!%{Cc3RKEbku~W~VyXgU{dxEV;&}><
z5H6m^nymH3E5F48fp%G`TD-duC#IUlH0T)>#_PQNYpFkLZZ29k%I7MEMs@AoTR@1I
zQGj)YvsKI&Q=piB3RGlD-GTN}MXvZzqn(~0b(e<n;=y5E5;VgTfP@MV5d4xP%_fi*
zIRR#hRIFw5Tr%Q0U<8k~r0k`XQKYeQbsKbk{P<zC(3RL%Y%Cr7SpSXDudweI@(?Nn
zL%f+l)gr&pEL@2iK-nlZ&(%xHUJFs7s^)8b48)_8ie}cPmQUhiGwmYDR?1eOdR}-P
zdhQz&xEvsA`rhRjg#}lHKX{IXg#U+n4wsmQE}^G6{s2~3RA7JlYHJ*4&Rn}J3dF%2
zOxz~544Q`+moE)-M|l%7FlJAE`45N+&@Q~_t-pW#?-nPy7(Z-c+G4JAy+r$iM-dPx
z|IYb*AwgPJ2iD8#pw1`!e-zbz1N5gW=csJ=J|~xqe56dx#-^}di;!!Njz@*ofXZEp
zfFgsO7ykr60jii;=@ACD|0R)UXhV6LbY+(Cs0ccFsqQ3x<IU;sfvcV8&-kQ+mxkXl
zczQ{)o+l9LO2uuOXaAN5i{0t58(uANaO9*SAJI52ba56$v2de9-(X}G!ZqBwmg;&r
z3n4aWK%XY&%YR%!y^duyTqsP$Wy3gDZXGe-86VVz5y5Ks^DT}Z7W7)7Vj#|N+2)YT
zM_wC)e=v@5+`aja6Xo9qb5xDDXIuNS73o?d8KPsc#24P`3qwS1CIZ22F!JDC&H<MZ
z$U(2wQTvh!-+rPV?X5*@ZrZ7rm<013%to+zpL(=LF~tX4KI;Xt^g%;o7N;xp8T?BE
zw;F|Lj4#9UL0rsQpjgwLRaBQ_>sk8HNJmMisl_Z7l+ww>=uM7Qi`*pxWJt&W^TtYA
zer{d4m4Q-cGfCIIp$2P?+Lbxza)}MgPS3VR^i?}ssu$|{DCKL7+!T8Snxc>75q%<n
zhKU7lY^xn6hUWf_5L%&T&n@%`fmqr((9zFl44I)^hC+kp8|I^>*(8EqTGcKa%(ivA
zQA@WCC+;Y+L63p}539hNS&s85mEQr1qU`umZ3UPiwT9q_mu83plpkObF#qaF5zPQ?
zrWI13T|Zy2u#Gxl1{?x$@Fr9sV>~+t<^X64dMJPpEDlsLyKunu2#tv9e>PCy?r^1j
z8i&oen%CKhOJG1i7`<YqM5auvhby`qWMYPe08_37Kf8Tye>}jjGmZ*!_eTMtDHEpy
z5EIW0Y`_D%-DTz2C)RP`MhRhu`40LL)9xsX!LQ9Dq4yw#kpL_2?K5^oN$3F|HSm0q
z(0+i>C{Vq=`Ig>)dX$E?w=u;c8^`exG&dCgs*?8zG`3AK>Q0(?>7336jr1NoqR`ab
zb^Ztlf(6)1h$>+Jua%8!<dY#C-3Bgtpcs8%GG1wK!d*;35ZDfxithvz0oT2VU!a~7
zbGbwDYJj=+c1q7(S~Z_^bh$)c8tbw0NH@fMD`;-|$Z@{oqsPHEjVA>RVqyvf;PZ_<
z8Paq17yv;v;s0hwH3y=T#3RS}XO{Nx_m^`D(#`ub9cF*sUK%aS0j-&3`ZA<VH>6&=
zL!yrc9H}3e)jFTmD=@1fY3Y}8@qlZ?=_wCsFLilZ+6OpA@_snv%msCOad%mCv{KX3
zGGt=e>TA&!M1bfMD1nSVyh_JEABi4VEXh~`*n_aQKu7NgSwY%yKEDLoLi2uFX!q-k
z=Zc2|(ptft6Vs&@3gi$ypaT_az5OiX`6zpkDyqU>YP3VDSReS1;vYRveF)i2Bx+u*
z&4MoPk+JNiQoGB8o-PhuZ-Mr3zXTi&%#%tye`7er6{&zV%ybE`?8LNIKS0_ExQIEd
zIQ`A%>YdxSDH&ArKea_Nw7)^eqJADivX^@^<6NKV0iPwT^UrrZy?a2Pb4t<X>;iy3
zixP!f{L|;_*FmRLf<Ax_sq#1|hMLuay*23VD`S4>O+r?{$P80?obvz^9sKda%y{{}
z5cuVafg}B2zJTp63^hwWIcjfhOxnq*R4G^OWol|_x{wd54<jlZV(}^qu%Ks8(y#qx
zF;FJSD#GU!VSpSNDsC(2E9VnleN<>M_=x@P$ah~1po`3G22Ge%z*$+<AZ-f<(?)~5
zE7v_OLg@Pxu7Km;OBUKfzy@J&RHAT652mtD5CKaY=m4G}pDdt|E^)gbD--BoN}F&U
zWCu%uOwuR9*#B}84EfW`T0D>mY?X#f^C^%sbk1g>ZS-=h5vpKft|(pa6J^jVmzv$A
zBdjBqo$d^6H4yp^t}37>|GMR0{Q#4kIewDqhjI@rmWNyfqL<S;e()H!X9W}{*r+|K
z6YVjl<<L&|J8lMO1Pd^Ul%YQZkXaNG0(-(f8O5|BkXN#U17zaP$Dk-g<;9RGG3hjF
zet&NpTpZH|ghXxHmEgmKrV70#ff2lWb|}N<ocQb4ahs^A8{N?qfHM^F=*g{r41UA1
z3H%)?-)q>jKi*-ny;@Z>AN;Dki6wR&dJ|~5(Zro6mW^{Rdk1KfOKTFqq}Bwp98p?5
zw~Y{Q)I_HN7y60}p9g;U@ae|AM-o8ncONPJB6^p07f|rJA3T~clsbTME+^0@khS%t
z0XbSYU~z=iqgVg7-8U?S?z<M|ed&S62Mfux9|$=uK52QM?0#ptPXo<7;SreDKvaef
zC57D%xXW^<Po_1RC2MC<LvbA>e{sKXD(^&pqf0(zBxhD*$dH<k)ZtIsYBk<T7%L}z
z@0f4U>>$0SVLX$Ivv}i8%xecn*8k<93QYr}!sfnXwlq;wahFl8XXj1t<m11q*bCf-
zg<E2}!Kz+k6HUUs8Ld_AB(`2f7dq~(D^SH?drMF?rXP00>U2$>$8lb_TUoN6MU%F{
zsO=MF>&fu<Gt{TA$f8nd++(ov&h*1RF=&3HBj9)2CLgI@(+gi{i$|)QnnWnQuW39L
zl&|!1+KC<sqiM%hs2<SPd7AhXnQV3JtlLp7bkZ42auU0<N=oClZ4`YKHWgImr7_8I
z1ExCfxi6Za$YHS=R!cs}({=bd|F7mb)`NA97>gmgI77VdL@&3Qc%-kdU##DF%?(_o
zV?30r?g09{kJhT(g*?o{K%v*ZF?B~Ft<n<@j00Cx`+#S;D+2t=L=NliFE<GXB(`U}
z*2|b;$cOT#m}*{;fBe&&|4!^XV^BfCiNqY&Y|TbR)?(?D$8I&>o>|dd(wTxp91=cy
zLEGU63*EY(uI-ZVz>OJUI^O84(gI@?Wh)^P#gpBGo<jH?D{-@4tJ6%M_5J<Gs%Jag
zN=Tk^_gea+2Mx6$f`9(Z9ot@cT$Qz-$eMhz`;%O_BU<;w<WzjZIO(zZJ2PB)+!_4e
z_fS9psEIp^Df>F4yg*60Ur;=HB&AViK?6<-AAzSYy*64#swVvvva@Px6t9Ip?7hy`
zwtc&3CsjS}YDnug`#aU4WQE}OD7)cNh-D*u;H$cLl*d3no0;3T^CwY%xz=C$zxyUQ
zk0T<Y+`B`glD#HplrtKi%e6KKMbA~Q^tC3s+3b#HlGh)-dOcgmzJAF4^qO9?<;g<(
zbcH~8IL&t3ysqE1>Gwm`ZyVm+$6jzpx(Q|$!?ba_&YX62`saF|p9;Rs(OM!S+z|&?
zC%8uN`8-+6YWrl)VEF3Pv)>MHr&@K#l}AcExC(z@v6@_^ff7gM;p*gSg{n7j53q8O
z38DA}bOC|bjizWMV5gmVEe<Lw{@Sx?KA%Q^9BIUSr}0R!u|ko7*a0vkvwLH<E;?e_
zw*lAmoPDmKvu4SkF=VW7F!54%GgXW~WX~**jN@F7{y^B@g6VnOD$i!#7R?58haniY
zVz7n1W@ag91x6`ty%{}qlj%TwQA~~`>FyMBl=^erndFpK1FlU5&#CB?MnU@%+p=7T
z+3xE_Tx0WyikynpueM&%@WTOYi&#S(IH#KkgZYU;WDJ|KM{dLM=1f;ice{Y6Sdx3O
zQ-`T{9c92)G~<F81qZM#B58@)G#7FnmY4>WA5TQX5}&%ri3bzW05O-mT%vyjKA(OJ
z&1k~C>m%z1<2_QZ)v?He_@u|JkpoE_1k76D24ar!7~hjxmco>Honlr>6}D2S9KeND
zZZ;uq-kUv=Bc<jI$K`YK?9>_`sa#?ZOLm4^Ozv|070kOtAHwnIx};k6<g2}!wOLJ&
zRwHcfbr~BW2P9MR$y}?EyMk*^S{8nN6eD}6`=V_;S+FZVnnl+h`*yh~IHl>$AE=%6
zlc^u<;?wK!g(~G}v<<=@rY&)%<9tLcbkobLHcaiP1v*lZ=So=Prtv36c~?o6BA@;K
z;qnVlS&H`lmy;7D&%xjzS@j~{1N7}F{H)ng#percmAganGNA;>Ti&aJlt+7{T;y4D
zy7VOr9+n}=b2m2mqPOR6RPwRA=Sv}ua#Rc2Scs9a<vltI1S~oZoz^ZjIoeo~zeTF<
z@HKQ*IIIp6(k~xQ&b39#b|vxKl9jF?WmdALCerJr4@plA4rtjcB!2&>noYsLicgY0
zjd?)(N-UhXX%Tsn?9Cigd{T5)=~2Ez&>FWZuDa-+JpM{kr9Ve}C5*Chcl+4!-XCjn
zey`m)>*9^j8Hd@$DF^9t;^3{BNU2}4T*mLUOSzMGZFgIopUTP?%(Qe@&I|FfSS*}A
z9k{x;djG)zqWPq2Y3jJonAo@_q%qZdN?wJ52_0+y)L%p%PDYztlb@!)pzEFJjNQ5)
z(-G&j=8dCUY4@^t-pqJC^3bIKvD@K&Ml*A-rnq{8EjlYowIQCjLVA~O<6a${y6lU;
zAOCXRRGmKq{>SY82Du>;R*g*KcJIiEn$Wfhmxag3FCY+3lPw*$wi6(UkN1>PPa5pn
znoo$|U3!dkHxl?^D%DO)RFL1WJ=YF-M|bHJngW$G<e+WR6Acx&3G6BA4pS2^cT&t|
z#BrpZj9Ts<P!Bzz$xeyg5k3XRqjlItFqEAx7+=I+-Q7{>s$1<gzm9R6?mi8IV%X9+
z7e`D>zj2T0fni?V-b?buAy2PQj7NXq++)eF_&@1zG{nQ-A`c6>Ekg)od(!p0Yi6RF
zNmMsRwlfoWwLZe=?Yk4V5u@LA7VK=LS1c{w#Pk|Q$LVkYr&+gSj?<~X$_Srj^|$+%
zHGX9LHP(#wS_5~92~X8qBik!>1^%m}jw#!CwsHRLyd(*?uoiT#-O_n@GNm?s8Z(`9
z%`2(VjHc3%jcd^*PhpU2Zu7UIeMpb~xQ^9UCFn!>E?wT+Ghq#zxZweU;P_$M^JkT#
zjCE$9sbB9z5Lv-gqd)=Qiq72bS0BZN{1Ss7!V_QM=J;DNGVxnwBQ7U6zA^!(0{i>P
zv)5QhGvmF-SZsF+SZr&n9?#p#HGU{ci+|5s+3<i!Bv*ZK%2Fa<|3TER6<d!*<C%P^
z_3?^dFhg)km&Th))i-HjW|u9f-Y;Ky5kO=$Tk|yQ1JUHONBk<|mRyPL)U_krJPxzA
z1-tKV%!j{2)X)-sawGlJ<nPab9k$^;tiN2mndVF@F<$5Sfk(>!U4K>lPR-FmvUgxt
z4yUt%CE{e8FV9aIyK$1Wk=8vQzG>g_3)8&+>(_<l7$ni~Td@xj$OTXG61k}_wg(RA
za}hLx@j?sVCXuz}HG&;6Wo?|@a4S5!D)>?kS&pcbMNiDI2yR5otufrf3b4=DjfM7I
zoSR;YqSc?B!mi`sxSYIkPA&z_E`NR?oN<4>dDhs=ogSxj724{Pg|57bDNs43l~*D+
zu)?oK#l*%}C{CvdSlX2;{cwLbyDCB8;Q$#u*OZ6v>bP6e&ScLO>e{UbY^Ew01yoDE
z4%i`1v3PZ(wNXqFx>Dm2@LIUHRAdK39jsZfLcJd!kDAS*>_GX5y{s)jpkq+y_926-
zw#JhJ#(vrg){R|q7u*@0mlbku2t6gb@>eXQmZ@KJqFW1ZC&wh=l6!524-}|=uNyHf
zpvN}Y5iXWv%*Syps|Y<UJ{`z*OvMYaq)<*f^qfbrUY(RWoIYyJnqq6&T~tvNzgdT0
zevF)=scP^bKk!^Z>BpPTBFAt;mkScmVZkEEW8WS0AF4r;a0<%<r?Bqk&ye?z`ES*#
zWd8OQScGq+?om{p)q6Nh)g3&8d781%SPN<&?&i^5JrX2ec(hJIYdO@B+PGsL`p(oc
z+7Y|oCBkFcox*E+S`shU>d5+80@;0HuMviO<ivVEu6wHGx0tI(6%1RE_0_PLWKf8=
zr+Zt<SGk*hAUQFYd7bVfljesbskB2BJ#n3VA>8y=`S3kdzN2voc8r%is-d)rK@u?Q
zFW(~>RAq69P6V2`?CZJ&%Vztr=WF_^W)$aZ()3&WvEXq!bZeu0lp7yxOx$NGJI4!%
zij`d}lMf~*Bf0!np27-NVD#Hdeb}^%J)0&INZU9a{&mB??XQg-78`unQM`2={Rzw8
z>XFp)>#Or#8OS_U82lqT7Cg49goAr|udkmqdf>plO>igfdN9Yn>rW6ByXL6tRi^%G
zo!4NpVE<UTm&e0^#TG}Cv7mm>M_oJL3dK>rSH~*ZyNGojmdkJX2g{%|dQy>d3LhKJ
zd2{Hov@hpL{J=)pWSR=|#>y^|=;#pmE@Jy1UOPgkhW@AlH0aa-ynm~Qc1wLU(CvEZ
zss%dXEV=?9X3ggKW4Q3qUFZa8mkAX4J~ke&Q<DC12w6<nAOBeM>jvZenjAOP4qMxG
ztiOn&1}9gFjF#uenZ@3zsfS-bL+T$l&CN$29KM8|nXynH*=&An$6yrj%F7`#BRgoj
zDa0ESD=-@LP~TheXvX@daVFR63h4Aq@aYXEQB8T&45ASb>Fv8qO5&ExA+OG(l>6Z&
zK2_mrXo63YRxdtQj{JIy<9l;q1(9?CS$7k?MYK(c>;^fux!0k6A;Wm3<~YBO)Be}8
zsXB9!P;QJ<Td~gz0lmNB4)e7mmTKId4?X3Vbmmc9MI66W`rc=HwMzDqNJ2Tsp6Sbn
z6VG0Xm3oJQIR7o)oe{Y1Uc^&voL1-FNuALZi-Y&F^CgjQVl3o8h06=B8_&P~s<UFv
zo*w7-arCOV?^SG64&0teCaig9&N;aEfPyF==bjc{<G89${sUWVi`gpmC)SfJ$SBX|
zT$qat`_jPg?3gv8fqRdJIz5Jxs92oyZ<okUExJ^U`+6FW?TON=ZLWULSkvCk{!Yu0
zJ#*vRj%^P?yEZy+y#6Rh%sSk=p6Bf7Wnw4VUN`4`>o>vO_zjFK4ZQH3$IDY2JcK<3
z^;M<Ul5hmJs|mwj!sha{wlq3iMEq-Wc315jy?auo%dJS}jk6m+nt#da(r4jgEHNz&
zCG2|Xn_|H^$;Mv~w2Fh9K{F+k(4<ImI4z%|%DIC?*8;gXV;i%1IG5D@tA{ce&xk5O
zGRzMfb65u)^7mvV&@ax@-||t%Dr~vx^o9y_SwQ$DVx+`WI*L&;RNWeWs9*!s?^_fq
z3aFdI(W1DD!Nh@6z>*ZS;~tPmQquG$tjUkIA0w(_-olV<wcJ8oEK|lRGNHYBgsy0%
zWn`r62kEE99uZzXl~pnFQOk|{YrDmZjg~k8{Jt^TW8y74?|;QNL@Y7=9yw}0X+L}G
zuG>_vBPEzc9bjLPw@1~ZKWbA+=?kA5=MU%b=)q_&A~@l!+w&T!p9$krODXJ#85M8i
z>w)i__V}+I_V5?c^a#s{7!wLO1iG2!kA;#ZEbeY<!mE7muSp^rm$_=)Z<@)6-sBRG
zAtN7U_{FZ(KS{xBU&Gv=Rr(RX^HDr<-FqNUtu<*r4QS7=*flZop_M=RH4C_ow%FQ?
zo`;?k@=8qf3<4X5d(z9~$SK)1)3&(Z>wHL;Bg=v?-~1#RA`w?+V}>2hIt!;Cu*CgI
z$6g9)n48(_%7f?>d-pL9aAr`bIDH7$;)<OcD5B4dj$hcyb}K;1*`rY3wwH%*zKfT+
zpZ)uzp9JrRnV0Z!Zn{u8R?4uh$n-k>Qq>R4&{^|Ga{;=@mM7h~X#7%r5Wy~Pcanic
z{vMITY0(O4Ab&y79@#*{$o`xAq|7gU`&UP7!zadmR=PX7;c#&~?bD@-4x7tA#D0ON
zfoX6jT%R9x4}_qKyonXAUPM?5Q=mjH3zKHKQ*g6qnoJ_EGD!GYHfp>u0^guOgB&%d
zwcF<NX073K*LV={(z|K2_;0E4n-<@Vr@X6&>rSoIUgxci76_184<Fv0dHQ^BOf!D-
z`NGwu(dHvKvby0&|H~fkaFz2X9x)N&jtTUd7&j$8ASLvztv|_kNUkx|vt<}*OTCFP
zT@DG87aXIwj;_u7TrSwuFHV2x);ckPEvuJSO@BvVN~85sp}ob{uST)C_yQeywo)t;
z+_X9V)j0WQjp@!SF0pQ0rEiY3a4MZAiMyLg#TN@bMzN^&Qhie7UX#@<gmq!}S5GLF
z)SRv@PK7Om4<M%;r%G$(mq=KXggdZ)Z?s_`yf=ybWES2mO_<t*gLAS9kNavV;rqOb
z9G8Gwz683iGwb?kh;Jl<9KAR7=hb!B22&d?lwCiE9aoxsUZCw~e;&%B!x(y8Z(a$1
za`M^uS9o29`UYY6&k@#G1`b#0WS$>8u1Cv3_1nlL$JF0%_pbGC`0Y*hezR$;rrcJG
z^htI6?t07k%e7r%op|~AjTuE*p}R+YF5`pWZ)!eq)e|jn^Ixv?^X%u*+acSZE3QNt
z{?(f-***Dk>l$mPZo<P7s)pSgVf+O-J|AyIXF3inx^V8jc)RWic85E)RMULGQ(q{f
z=e&%IVPTQ*yDMpwno&ZRuqqYkXq$8;EFG3jkm575ia`N35iN=<!X2L}$t4?55S=Wo
zXGf+{ggOi~%Yo>WlKcTfl#V;!le@V7=s5jx^`ywJ!*XJCOH>jlq5(ERz-EfPMa*Bc
z-*|GIx$pC-Z;{Jw<ff6kwx^z3zih0`ei&y<kcn=$f1L`B!4CY0*RAf>$8^WWCLOs-
zKkt6Gr%sYqYSr}x-FVoqoqBaR8>;-2`VO9jYoL-gEgln(dLaalg>O|Y=iv+*vSfoP
ziYhnXGH_DZahcRKai=_XHgZ^%mQ<KJ<HksL5c~Y*{#tW+)4kQ~qP=T}M;ngzQY5;#
zB|C^<g_5qvB^wyMi5b5UIL#zkj{HkVCI6Ye-&+I2j^>*`)YwBmOcE>)(Jxfd&!3RM
z6P^&&W|)NJ#6%o#1n<4=C1b5NW1&4sTIiS<@)+W=a_Wj<PW;+F{8HBN&Fn&Q4h3SW
z;gzmEQbd|@cqnDa$?TgrHrXB0?T)&;5Bml?7zP;su*UM_CBmFLEDeNZme>kn^!qrB
za<^RrV+_Y_yOQek=HK;oYkHC5XPJJ43BI{sfmVRY4Fz!Btfhi4ZF?#zxT!3U58Tk?
zu(ztdG*(ei3vThnXQFRD7k9vIOb{43$T{67Q)u}_<HH3c*y@IxhoJ39I0=5%aF%Gc
zfz|E5aQK?t!Ni|i)WQuP2obiPwXq9dIb?XH?Sx59dV_xN4q=%_TzQP1%ErFur^Rvd
zSF5+v5Ya|mqkOM+?;NiZK5m{|+T{}X#D&4amgDS7zgG65T;_DRrul&67lpX!{lo{N
zV^ZF01$8&J7uuJIQkQ5)ie?HuN?q&dbv>rG(%qY=?>1E~V9|%2JsGJx(O|)pe>>}O
z#*dU=I}$=?O~Pwk@pu|ZUemIZ*=M)jh;Avi01IRZ=+$B1eH-6(FO9#`DS+kVH1}}N
zg&k)Agj%=|u^tFzL3XsHtj4P_3;mueSGXQZJ`+~DcNEx_7@i=upcIJN@MKoQVK3A+
z`NTU37hBDFWN!?UUW2+KHci|Y&w7C$78=_lwjnDU7aaYi%5t{VB4aM7J0ClHsCd^X
zmen#cmSDcSZll@ddiM4##Q|uKKx;OXWV|j0`(ySw9WlG;!BZ67^d=_W*0XoFVXM1E
zfO@i<mm~JpOZ+JjT)gBPE4X%k;7r3l`cdN|4FCEadRO98I&f!HIN_UIkhi%*O3Kt6
zL{MQDNKXXCfP_9kZXhy@)4g+lo+E%>8rAj=mWaY3g$CV}PN7+zOx$*c)$uNlqgbNR
zX=1M|US+j}0X)<*EJ;vxa?iwhH4r{=6hc;`RE>-?oWITkbBJ4_G-{y`Ur{VHQ$<Xx
z!R|XRDK%<0yXuy;$o%}UG-K6ox7yo+5l4nK`c#7aYjov;+`&@I_rPmL?x{dDqw#-I
zo*xByMI$ENa&jH`dyuO|h_z*Fh3w_BD_zmTGDmX##(!p6IyV)0Zz`|SWK5k6x6mOS
zDc;=g-uNEGL>6to!GVmE&JxR!q<s+#Wa@6VL5Qt9K)shbmGcf!-Aw$!;^EidU#_Jp
z(w2vKet|E9)2x4BKl=c`)w;hq7A2T5a8lQqgmHGCu4<#Q`6fmtWdMw4R-*t>;k3%u
ze~+4p>-Wblru3kvxqWuaQ~m6n)x&3%u!1HswoN&Bf+7Y^5(fggxM_bbs~b2-N9^wv
zV`yO~hXX<2ZKl}z1AO)GK-}-%5askuU`tI)AC&#gR9ry`x{<40qXGdNKpMU;Q<m5b
zIpLy>^XZR(QIXjeJ^s1KfWX9F;RGr!75yn*Zug>hkY~h{T=O+s>wI|-$Ys1%t!<*l
zlxQZ90@+WO4=^u3E)0$^5GxEWkp`iF*}5%ViP$4Vd$%P<{2I9LFUgi;2u+h$>0_76
z$38L9Xf~W3R-A6<JA10S`7<|+Cm+{E@zFh$U##`HVNh=K^dYZ$E%QdOH8bg$;F2;m
z2ub@+9sYVN>s~bDPb(QhqF$RHKf2KXmki&c#1-yn&*#ze3)M21NAK&Q-}B&>L}%r7
zd4G#NQ+UL2_t7s__&8gY<CKBO%|u6utD*Z*tQ#K86Yk>G^PC;j-sH*Qe^T&fo|+o0
z)E{}(#NtUJ?YoZ<BMS`@dA#uXtX}=oapVJ}he8qp?z#@sn}UVYxx6Z9*RFL{w`uS1
zLQ@=N|Dx!hWl%1-|7&GF32E1yT5O_x7PGq!Oc(1KuN)W3t0e`O)&jwXH39180)UO6
zow<T$^#J6v;xkh}U&Md)=zQ9vS=vF)n~vXoC#=>D(U!pDINSTY|A~@s;<CItl#y-0
zM0*&}i;;WySvfj8LL=z(4}bk!=n36yh}KB0yWA|pfS8l#3PH}V+3WCm6+7~$Zp*5D
z<&7MOmkXK?Rm)$)BHRWsPOhJ@X@>UCI3F$H_1{UjUk}{NV$T*zDC8kS<ZVqxP!QB~
zxT(}xVGNEf3b|%BBh_wi;X`MqHPmicPET+G<GJ#v*)w(GiGSn^f9P6CMbs>!O%Qv#
zx4+fUxhMC>_vp9HP;RU?Lh2@0BhwNIA@B2mu9URjT{DpzlADa8{@zqXcK}Kg8;UmD
z)0)d~e!FWkynTd1?>5yw|K+x5W5)A}I33d-9Tg&}r9bUttjQjSB?H+mOw=D(uEaf=
zu{0li^}fo+W6HbaJH{CrRY0%gA5P55^=hX~CP#2m%b6`pB4tiyI-!|NX?1G(Dvuo%
zscZXDRZD+T)0fDNG=A9Mwtm7wM18_^Jwa|;r15>=TGG6AGRxRQ5SlcppNj5&9;bS8
z)WGij98~73cJ`R!wUlwd37;4Q?yR$`0t^T<#wpj$kz|b1JEU!L(cnZ%Q3o4*vN3UH
zMxosByG(!8NsTQLqrI-whD+J$i^R#;XpLj;q6f>mn7`G)j8g!A_exbo%|_X%z)C&&
zr`(jQ6D{ub)s16u@>49yULH*2JL6<MNWvee?mWzXulqsbF=6t@0M5nH8<n}x63*O@
zB@SyG$X0B5`<QEL^@O|~h2<{6dGbyoQRp>rwnrtsgYys+i=sknYcD`V$8GWZk9^(F
zx^%p)Cp)n@&&c`8t#s6XHjGha3WTZ>Q}ed?p7wbnSVYC>28YO)4)tDmx2`84X!d-~
zNZn=pgZlJKu3=l5g}@^$?x*5d5kIe`qmw^<0G<C{mOTBb#j;22*^`G;3of_|T0xnO
z2e()zDtRDmz2mAxobEGq8*hatXV{#)Og(a1+Xg)m$ZVE#8<gM&r1Q@Tp7?xUE;wr8
z>nH_xsea?^==IH~4BFYvujMgX3Xje(3as^;`SC7@O}{T1g!9U_NB%_5?AU=EfF-;R
zUunsAp*%17aZ9#XAPg2LVAtj1B8OOkVO#vs<bx3`&d!sLDzozwIR$VhvOM!I+HOv@
zj%9awdazPZqEN3TeDG>+g|#u$^I*_c0eSF}6=9mIb*hoD`LN8fY0-o1U62bGUhYZ)
zJ&{ayLDcgFK8qoT)L4Y*%Kf9GjUe}$?%M$rCcP~yVhc^Z#<PWXOOv}A!x`7Kt2>{F
zIxUsIoo~0t_<X3Hy*1#e@cEnd{06gP=8l8yQjKhl+cg|_X6>DgV)wj?PJT+o%$sbs
zHUw^&Cb7VE@eQo-_xwXto*QXq8&=9p-f<?ZgXs-_>rZrru6z;qJ&}rqn#bPkPK@@1
zBgRtQ<_!8ohC_y>{@jKpjzqAAsH|Nc$_Fo9r!E@gcKc7yOpG|Mp@!gQELgQWs_`es
zh0};GZPPedp-m9j#5U@0b!XKTa!~i<X8gGN&dG&6clv|+RAtE7sB;%t;H{}AX1|{I
z*?#kA4g2kmr&Qy=-+xB_*?R*;8`A)_t+P4dGQWOd&~Sx(4iF0<p~Y+BLL1BHR?G)3
z4@IfvjXrntHme{FWkP-zrRyv>a!$fVrs7=R^%)0X<pmvMztz4l5pvw+)JE4OJDuu9
zUxmcCM{rs1n5t(x-@gdh!6B7I;_8{Ctt3CyZg|waM)cDZVS6MSmbS8c$a}r~R$gRJ
zYNH&#G9oX=aECI(?<SpG0w+T1xZFNQ6^D$!o7fk>?X?e92fZ^lJ*mwiG7qEEPPm$F
zFMSGZSEbB{cF3YfR}J3;Xyud1@@S<*%H`(!#4?YH3d0^%5@1V7MXxa(er+7|toy@>
z=ozomZp<}JCN>vv-y?Cbb!k`@b;90%a$F40t)7b_rfHYuiot!Zn!8^c+Wq6cODHyf
z!L9g=M(wpR7cHk@HgF*ENj-YHHNkbYmP|bBNkFfqxm}h|(6x}m3Wqb#gAU)l2hUHV
zjt^|uDTCJuj*bde6ly%GokV5iN%;L4kgUH$>)~X#*XOFLabg1JCXy!{BNmSz$o|OR
z{WR1g+=zj#!lBpOJkj6iSRbvJ)g?J{!?Pnnv^unFMB&%2H?PO%Yng}NbQ%<^&A#!t
zKf^;zo`mj+z`6$09;95K<GiwtXe-(Nxw0$!dRahDclQ`al}K+?L2w265J@^R*Yi6f
zb*?WXiXlOL`=vZWfay#owp?MfG?~bD`1VolM)#2p5tmzcZvBp}kxW%t2+{jR4|a`$
zRI&BPNb!^0?~#5(hn_tR&BH-8{7!X4ivIll4f-dVgRpeJuUqJ>)n{GW&(r@Hp3LoU
zu^;v(2p%zUwSt6v%u!K(|D?gFC1Z-%naZxM%Gs;3RU#vDp2G^BtkVf4k1rA;{)b-t
z0G21qV<-VIC2yS&if$JHh0VxsTQe;g#_dtyUe9ESGK)bYka5pHlW3pG>31Z9zU8BL
z9qn;I@1tdwf|lBnf#Ygz^8G3TPl_xYjj=MfAG={?e(-|IiH3k=ZZ|QR4Fb~lU+XQ&
zFiG>XU9*$szr#FmQlmY$atL{)VJ&6_BtXAQb5!Wf$!^xb>?$jokze^8bgbCLUabk-
zDN*l!xe)8X_PY*W2WP?=W38wsYl;t!tEn5{#|WHr&e}Z-?WOUeYFMP3ax~+JX57r_
z@JYy$b}I3Wdn+bn0^a-Ir8DcR<zTuVwfl5S@QS3Hs~bn;&no>t_LB+G-IDK<rmZr=
zNz*BK7T>zc?@|x4^(ILPKZ75+qciE>o-ta-n2Ob*;hlPWO}U65abu?Oon`0poV0Ev
z2w1+a3p(&P+-c}bKX5$&NkzP{Dwjvf+stk-i3i!cPK3y>9Ca4)d+Uh$oapZNG5#k+
zQs<GdxXG@!Y@_W`$fr9ku6S{kEcYj!5|+NF)|KFQZOrSCA~b;ijwf^zmQ%E|yd~4&
zjwE(o>~ZrW?306Qd!IPd@+A<DNZz=&L2H3Kg`D#AkkZ}V$o$DJ8o?*y8-tFh{<*qv
z%V08enK77%&uHj5N3!0J^z=9PC)78tcXJUL{vdFX=Q`1ZOD_1|GH29=J&SXba+-`_
zeel2_f5ofBWr`I#02NywQ6iA}3q5v!{K;K@zWvN=8__j&7dBFYjHw;)EcaMm&Yr6;
zUr*B)CxB1o_zAAtaqkt;9~^hV+B=;85LZ9mbBpq-e`zs9XrfZH5MjTOhD_Z{G>Os2
zomkmXUhpPF$~oc<yXCtyz>mx;k{mm5jn}Ve!N$LsgIp%3Xf4g%rbQkIv@rqe{5WV?
zUL|j?vwM1CvBBGysy-L<?UyA+D9+)@doxzb`J>$UQo4w}w>NE%16q>2yrgkwO!ef*
z{~JgNL_<87Hz!=CB<tLFwa!qEDr&*4?tKm3<E{LjUN7a&VM4g~eXd}zu1hWX%8|NT
z6s-Kt5*TN+m!G62cd)SawI0KX)30;-Lg{%waa_#Xv(6E&&*QrJX|_GO|EBGy@R<pC
zukIU59Fny;g)e&fXbZ{q8wDsqI;t?Q#YWkaNN;con}{bxer@;w*P$Bim+)Rn>IpMA
z{Ze_0C(9?k9{-VR&G^S}v#aKH<pRHk{Ev9g!n|<!7byBHZKv$jq#|pd@M&rk+&gl`
zCK{;dI61g#L0m&}dYbz^e6Ozu_ZbXk_F_e+)O5l*znk7%7h8WtKydUeBhmfU(O1@(
z97rC1-Ccn+e%r3Z8a?-*CU{Ph<C$KApvV3v`i%@})%4FRu!1`M@RBcqlkGc|Y{&(~
zjfM!vV@kL)vNA!dD(yhCJLx2ImSnyx)K3zIBy%Y<e&T35Ko<J{w%1M5`FNk5s6q%>
zJUY2gpcG6VLNPuIeCuzsrO3+71f8m03!=Zd+q7RIl>l(HBGN*zKZ~NCYq0|$;<TmW
zwYVw&Y@xcBVh5b}>ipD#kXKZAFX8!D0nQ5Vg$=VG1Zo4|8M)hP4PJEs(cK<G`V_)u
zv)PPu!*PjGP9x(KA7?{ga6Myi?qjI9Wh{Ed#|V?d{E1HLd_HTVgD399NRDd2Uj32#
zAU?atM2cuuN7EiJ-EAyX5!}Y;D=5hT<iGzZBH|oE3}MC4^R=p|W?Dj{Z6>O#XgRu=
zK7RbDP~~8pr5b+-eJO<j5Laz2-+9|E;2i6<OB}+A_iZK2w@SOQZ?4NTh%i&mUufX>
zPp97BC~z-iX6&a)O64pfa{vmJ3?gfx!UgDi+FREdE+Rx%eD=}zR!0oozk8gWAQf^{
znAq>C1M5696<c;beb9eBz+U{R0u~7SO}CUi&9L@2-(JZ)$#;|1!i#ogteA*Wj(f{b
z1toqEqHb1ytRF(=-(f}3MG&Bn_E91gFM(GGG<;^Qhn;bpj<N%>lO5Olzr1{~pd3zg
zKKmvB6b!WdK?HKLdEc@X7iKp<Dgq?18vx*!ERm)aFk&`=DFllCxPjJ31)y{I{2`Z>
z5G_~<-1a{@P!JcOc_T2Pa4uV&7I^%P^LHPEu$W%~>iKE=Az=AigB(RuL5TOaqZKdH
z#dYXWOq#R+&KlitdMNFVTxSAZC;if;qq`;}Q+LjJ@edDsj;Rs_u{Odr2_{D9a2cEV
znd1CAKpXD)0MttRXJKC>0K8ZDszlFDAhXn$k*C1d<^w4et^~l03RyTZo$rsA*kgat
zKfr*bFYG)o4FwQ99uxwA8icV07-GpFd?uqGZ?A{58j4RPB$S*7goZEin&$xGK7g-|
zcqQ8P`n5lZ8ev?sJ^wF=*gKtTRRpbUIJ0)O1kkV!t;ax*v9)n6LLhqVWC43@|3BEn
zj|L#EbP%)}0Q04Ti8zhc$EzGb1?X_v=BGQ*QR!O)z&4)l&aIyFVF;>JFsXU*XbkXq
zxP0j$H^hGMx_yypP{0!v5P&HO0CgY81$BNjHhS7?be;2N`Yu>XS*6sw=OGk{KmdxB
zDjg>9p@vEZs0}bKTnf9?Z><q@8M5(DwtxGROJzt!o>xlLAKB2UMDm_1RJs^YbS8I_
zb0EH+Dj2q=oMf*23*e0yv7f=^k~>3F$yK8SN*Q6;5+YfkQ)krkSvd3ghqL@&vF8l5
z0!z+tEHiEToP2Yz93-!j`BVKjxGR`^3Mv3mg@H=7UjU2@jL{t^GzX$R$)c9hW#@~{
z1E}o`p19l_i069jfS`EJ`^{z5fS^ImfbC0y<k6nkj6a0niN@{t{lq4H>5}2}inPfI
z@e$|z<5mND_dQs8aP=W5W4fn~n0Q66PE|1b>tl0>^5+iBVMs5@)}GN@|Bs+Y7?BSI
za{5HCn68*57tU@f7t3iG1r<kR^W1AWANy@8;P=CFUgz?{ZvbuB_D0N7TkuOOMNLI$
z6kk<|W@YJJY!qU!zK1Kf9#-E2GRYl&^y$3122xn6Lhmy#s15*VSM27TFOERi>c~N;
zbMmyq9UwRgB?zvTks<muNz+S^!u!mVxs{iI`IEFw4U=W~5xRr_8?l<Uq`H<4nuh(=
zdv3)VOF}wHmXx@>5H7fXXn+G;^2UmyKTmbB(s0erYcqgm&jQ%@_9SdWvOiPysy14_
zKg8o{G{9rJQYz;Fn#%x7(a_HmZYK#tx2gqpdoDEl&RQJrFkw-F@M7cb_R?1CSy1tZ
zWnf;(fqDxc*gd-$g)%*0#n40(EibLu6dDBU2f#YKwUg~%4?=%|$Pu;o>9JQ08QnSB
z$*vMew2_#bBL{d)#ik9CH}EU@h0#D@?u!Q7_cbe^*%*L@+?Dp(8jk@--lTMKUM$sb
zET{m(&cUyHi(G&aicCTJ<P6_pkZjO%RQT^KT&KtsKEZ&}D3r=BkARl9f)yu8d$)!%
z45V~oW_gMTE>@hH+B7)nMN~|AX<oZ|p8~+9BffljNUvF*`%YS>_gpvs1;|<n0K~53
z)dX{u-S6<b3~`z^4HK~S7{mfgTlH=YjW+Mc_c-???S2*d`pdOt${x*CFnL$|h1ES0
zdq1w<rCO_&gvL;4j_~4fezczOgl4@;5!5V%PunLEdrcOq4xN>1$|V5!mD^_zVR}bz
zuWs$y$(7TyV@Kij29{(fOtlTyH8t);j?KY!&%(e#wtXg2TEgeJ!i}^M1C0b=xM#l1
zr?dUH8Vq23Bz(VtIwKlJ#ts0jjqgZ!3@YkA5`*M1w(0DPzczypu$u3&>&ji9forV)
z31Y3Ylw|>1JC1bk9sZ8$UBR2-4#rrn)`BkUf^K<de1u2q3$(AdMi&n^7augwcA{Ww
zp|tQ9aY+tlU2AVEsAy6b1WC9haBQ<5g5~$T{<4lpGa4aiaL<3Tk(_Uz%osN%3yS-2
zO~>ktb#G(8j`A2+tk<rS^$S*`0#J_Nkqtj$wqGBzt>@nP-7_58x`qO74r}86%9Aa#
z2^QF!scYbRrN2{9p!|GB$>(|KwA7R0JMO-c$7cBRX7<Z8@kd=vlz-O*HMBQk-ilv+
zgbVsM!x?(n3<k3Nay_cvr1C42X0>)DJ~ujuj-Ty_MckPj=4_bCy=AX<1XBC)+9?6B
zgYkP`6P~&VhvPMIShEUKrK!}oFQ-SCT10*nL3eB&vc{N07!!9cT>zNulX>R{Nt=w?
zU!@%b1SW(6&pxtpt8!W@hH^*h(d-MI+b#Jn)Q<BXSX;o%S|D6<CVh4QClE$%@2yX$
zmouZVoBoEvPS63`{?GvH0hP>D`NjPD_=dd@Dh&V|$EA9YNK)JvmO1^w(poPVA|1Sp
zZ0(cR+56L{-|$mji_%(CHFC@bLAKpt*>dLjP^~m)bp*!PN?`(!E`3GqGLA}4M$gU1
z7k|g}2ges`@zyz6$AfZ&$x?-mC!RC640b#LinXoAFfR7-b3qu{5kLb5-m=d;DxvrX
zX)MnN>)UzEFZ{w{3-O_X@(fPVkJ{Bvl0QR889?UzgDH#3-;bvs03$>$>_EhDz0x-(
zRm?#;adfLYC+qvO);n%>?-NH;T1)8=c~zq^ZaNxexl|yFvCV5MB}mN`kbT?~zuj4(
z*;!~hmG8mdWxP&5(Y_Ost(jHJcvu(@f8~OU(3-$7`d*u_11jwVshg#DIaZt&i#6n<
zw?@0ba)GLO0!-5H3<1}vNKW$wWQ8GRl6W=^DrQaY7Xv$xtyYu@Dj1&Mz-Ns4b^D_J
z<QoRCv;so2cUAE0Fjxa4SJKcY=0-;6YF|kL1TGx}6=ytf^5sePBevE6I3HyxDZiU=
znDzr6a)5a?YN#u_?kye~5SFj~=a&9h))dC}P*1Z-WpSeBb2sf#(x%$cnvk1rB_HAC
zY0u@Q^J~{StDA$|CDt^y{MB_Zwop)Nz#16NYkpYJ4AubSq|n{VXzVl48-N(vV|+1i
zq~Q75#)~~E4xsIUurFltU%x&B-#@nPxQEI-eam2-$+P0wTS20YU1xz$Ki)aGSxZaf
zrqxL-^9yQtO}SIdC-9z0@xz@pIC2<6`860#77JoX@3=-JP>A~r_y6v3ish;Z;`nl=
zmrzpWf4~30u<Mvp)~1?Xu|oi)8z8e)Y}7Es5Klqt_kjzP`Ulu+1Sup_0PBucg;L3P
zVSlL^?Y91i5k9es*;#?h-ymRS?CtGUsP{OyEQq{<S`T)ct+Iy;C9hvU4d}HK5q&?)
z4B%TkM4-$%I<};NHDcnd@>P~rl>_H|U$>yDFqP@6Fk+iuxFyUfT&>p)R0>9DLzM~5
z<ce1NRcEv5q+58jjRq{tM-ASnH_RYk>U_r&mLXwQc^d~d_%CJLySpVoiIaVJ&P$sB
z=hEf$+M}3Em}5YRj})k=HeMdchH4@**NsU2RtpPR0M05fwAkb0W2)fJNTxQznAA#D
zXk`x+o}3=~j|@~9I=h)VDRxi%@k~yj`02P`xcDoiud=t%zKhd+Rbgq;G6YnE2zMnV
z&@T0#ojo>3W3{hTyM1|{hIa#%^SU<w*VV?P2y^g(3c^5f7`0}(6&tX6*}yQg$^W?h
z*V65a06&M|eL|xFstTsTI5kvQ(I#I@ZwX%ur+T*$wx%gw%|){x2V{v;Ka+n8>_b!A
z^RKUMYb00OcfqSx7~O1-2^8i}kIx~$W7iF8x%YA3L$1c~^1w(aL`~D&z~7fqgt4I{
zUd;qnso3MdAzQ1exFcCm*QREz{8QpvsE`UQs?7|*QWj30XfZSe;wC6PbUN%UqS2rx
zMk{YNrySp^*brV$ni=c7;(ecbt_^>4G#~NRwd?zzs+X4uA}vRhOvE_Jf~SM;eHZ?q
zd!0<0T%nu1-QH)~;kD?z?L~%J8E*YjY=#z^1#4)mIwT&LVo)KPmGkoIb6{r#|2SA&
zF7f(r;CcvsKZ>W`r?npl*~LMreB!b12;N_d>kZb|zH}7g+RJaGFVdYtVp?*r!<DCw
zncX&Om7kok4Gh8##Cyq@h*F(a2npF2q*uX`Aj?s5{yYFHuf0h$+(S|@2!Cue2iHuu
zZ-O~&&kEkz`_XuZv&Jx-trgn<pXES(d5D|_>~N9!+DTC_8yjhWeRSCRV{AU+UfdC=
z!vv)JX;2p#4O}mnb~MG$AYYu6hv0boK2I+31pG4dTuTlrbc2k)Bo^?p#6g)lHB=c2
zRaq%1efobW`|o(H-#307=IT-;MJg#|XM{rbxRf2Tvq?r|Wo2_UB!r9*sce-Ik?pE9
zP|3(jTqUBERVeE|&ex?r@9+1%@5kf!d;e9vIbY{Ej^jMW^LU;+M6F~dHuNJLFJur2
z$WJ?xeg<#GS*a4cbtLa;;6(P^-O~Xx-}04JgX@w-{Zicq>WAH8*qf^_Q%UT#UK>o`
zU-$XVO>3#Os$slrt<4wb-=8W@<hG9R4c0xYSJHoQYjSBM;Fa_DXW23Hw`*pe>m2&l
z8@iScplIh;&Y^S9Z(kC(BjYMWHR299EmjOa;z({<Q*!Qkoz-E({bb+aDaAGD)aWLM
zYNDXip~A>z4Q+e-b&2($ldv5JE;)*Y`kTU=fEPgx^=5+SvzC<0e^v(L5dgtz0>N6l
zT62U%GpM5D9elf*J^(!biB9(_oy>af{DrpH(!-p;zL_K}JeNxSlo!+rrLx36l~|6Z
z$r#(bsc&<t!7l(>-yf=t$_YgldAYG{yF~epeZFcw^*evPBH3;-)3|nOLEihs5Sz{W
zhtfK?!sZ+XFNz^o;m~`JCdGT}iP4}}a_!Qi9FxM(&LmtqU&3>A$nA4Y%S8?851mhT
zAF$~@>e?&xt@7qJS{bCGOs_Th?xRrTX4P$8iZ)%0B|%4{Lvs{U=Oa&BLH*ki6Et(=
z3)>=fR*D2LD-N%G+Pig|?KrHDgIYogB}ay=`M1lGy?jfZatHd}SYI9`KQ;m=tRBEZ
zs&=s7nxua=Ff&-Q_R24wLs2BqO3`~WODi9%M1FiF%RVLor#b|Lk>{l$0;O0Eir`Q~
zD_)u3H&XzLr``%*HQyuUCeT%I{2YVuS)=lJXAO!z;r^QyvSR{Wi;=`X^NNO(F|tSu
z!lEZmyA8ybFEw4>R~6IQ<m+be`+e<xpi(4=^^z0*n-%AD09$&bYNDt*92UAf(ux(F
z;Y*~W<mr!35v9ZL<+=7er`I>~S#%&`SC}VbX)P+;F36KSIQ>-r05-2Cl*C9$O}d-~
zc7423_AsU2g7Y@cHtctvq-(b(!i%gdEu;=L1aJEJ$BNQnPm}~X%)JtSffC`u+D>&v
z=Jc7Ov0dBrMJa1`6ZHbnP~b#fs?JoZ_h_pYYS|F6=pL)gUc?TPYd@%~x31nLY#1xB
zm>~>|(?zf+$7N;z@^2K97KX#Ip6rI;1?>X8n+_1FCMtync~N|kvRm-;D+96HF`dY@
zhC})unMqB(4fc53mZ9~q3wpgFdBK~b5q}-%o7JF!WIyVr2J#9S_x+=%DHA@A8hv#_
zOE9bPj}+iC?2*jFlSLTXf(l361yn_sFJ8LT^5@sLys(u&;yz<KEs1=`ZpS?SMVZoc
zF-*x*QFbpTi@?=rKiQ#|+zIc09;$K?+8j;sH2i3MS5tv%;n>)#bVmrQ4NGklzwNrd
z?iMC&2g96v`hgfWh7pXB*_qh=lpDs_<w6^cB|9P_*lFT86otTHOg^r>Vtn^RsYlS`
z!Lu*e{qJvfTtCV%7-!Mi3<5leXrTvrxAP;hN_vPfbx18WoM?0^+#AM42wd|{K8=Kw
z8S6xE=`C>YUa;=Wq4aUn2EY~sxK3jY_kW}CBIgL06b}E|k+Nv7Xp-{UGS7!vKZ&CW
zDS}4ozGmg0p2f)L1$D10iYI*BbDI+8*Ioo<lFsRe5|vxh&`b(z7%4NEXECeuFGktY
zA0O)SLYL-IJ84S-*M${fbq30ZG*JUL7i$l8-ZJK(i!H^+KAcC*@OWQQtd}?<;sajP
zw($U@WKvRX!^~&LUJFAebQVau{?aCZ#ZLx5D|axK2OIWZTlIBl6n|jsqraOX6ZiL1
zs`q0r_Jt?c`h9B!0vp~lJ@<Qr1^6hCM-n_&d1zICG!xrOQ-p>MqBl^=KpyD*#{%o*
zC&V3#V4*%gW3f$!+q5-TD=Amy!bW1f0@l<G$&E!A<+0}m;M;<oU<`m$BW3ugX`o2I
zHn50|vOPu2GGWWrbauM{5(>@65)F0$Z%C?5I=h4N|NX%aeBkrVz#mPX*bT>)YZ^Dg
z7=?PPZFZSbl!37phoG4c-TDntDgrh+1;qH0=*GhuSfmKqyeUjX3IBhRodAD98FZ8k
zcpAx&1TdHrD1+|MlIVXI=SWGbk``b+jnbpB*sk=TT?yN3aKsYr%H0})1C;r${vXM4
z$Wx*=Y6-E}@RH~MOqw7LlAjVLH9^1b8r@Y@qzD0cOt)Zb7BiEXDT}h(M}*YJ%etvX
zVa4G2<iQ%BV-60vh%Q5E=j~i<60ohtu&uXeea=$O7CLo7QFZr3+f&9MpCQ8UYNC7)
zjFU7_c_xFAADqX=#w}m|HlY7az3jnl-AQg7eH)o)L2G6OLimY6b%rvx>B}UrD*_@G
z4--I*+Ah*De+Hhp=@*VN`zfoYm9dK0bYrWh-QCL(8)&LMWy=hTw&SEAHTXb<$3T@b
zxez|DPt9~LA8IQPSUl9cYh1*QTo-^E{rUPc6Ke*C!#bh^19A0`jajHS(RvMg6L{cB
zEjMs}PCZr6c}ZDyuDTsGsUqSgKu|P&es{t)Wp!mKW=Z*op^sABKYozGHOo(5-`%rA
z_MnQ9+~^qNl$}5eZ%cDNKdY;KX?<zCWWXKV!X~3X6!c9*(By8is)oifIG=`dpZ?LQ
zBy6VUIGvBXdK9F(5I6Yfd~5qNCR%gY`ULkNAuMIgxEZ%s$}Is1R{CrG?mcvUrw#8=
zS+_uLQNZ}7++Ukbg%7yRO64F5#!d-swvnD5R8{^4@|5@(WD%V^5E<nKFZ}Rc`u-@R
z#txV}TnmKsP5&M<k!qWy`&-Q&Ow{;#e6?rb*=cH^DM3stoTo)8fgj;EmY`;0Vw(N_
zXjmH(x_r>`>`^m%onW2yjdDLn5NPS92@)6gWZdN@GgAMs-0Grvml*LH?4VnZ-42X)
zfQs0R-z8$f3xxpby3_Hy*{{daP+Vqk#8P1%D|~H%OaziGY|JwDN6tgAGtItzl!8%&
zE?zH7K@e0S1tBv2lbOy)&RUn#hzM(%fZHy;*K-DGFP2a(6u)&3c8wz(1gGG9QG4Zj
zI6)*u?-b4j0vl*OJ%l>)(XkD#o&9*B%V+cY&JZ|Ns2A48s)0G+{8AZg3S(I!)q;p{
z<wi#pPG4Bpk%HnPK_s;iPw}Ew)tta~Aij12NM9h=0)$mlWfq3)e|NbYR#!YIm&`y7
zTt(!@od|(SAPpnyxNB^gB8-bFb`Tvpt)>F5Sl6P;O%SnfL89&PwiBO{*R?c)@C2vu
zR;Mpg1oGUvu+2SgEZUd;Ph1tr04F#3qWKDQ-kIK#L>`T;e{<1scX?9&%JdTLG@(n$
zT}_LLK)wOXYNP`b02s?OyFwpb%B&5!{*!aog7y_YIXMfDhF?9#nO8u5F?&cwJ&XpD
zj?09T#O)`A3VZcfLDG5^N`7N&A^}*kI08tBkX43~0`!?t3=s%Xg`2tH?scsPCb4aj
zN3jjnV*+tbzMjiL+jk5U%;vkA3wCNpbjW}a>ud}S4uo>rD9{3412zQB)#cwub%<Zr
z^&-Lqo&LM)cYv=YApws6qJvpH5PU)YcBVk|%lf^~fTe;P>`*Egipz;`+q3|;UuvX;
zH|8T_{(vQD>YtvZ3NDoRsP_DgQIrv^V2i5x?^CuBR2ZgmYYVHm>c*{X0%|e44*NV#
z7O|8nbo}(`--(65iN#{%R8Y+PVQK2Y-WnO<0|E)`5#5;6NQ%T|NPAu{D?13C;7RG5
zoKQ7N*2igY&qeOP;fw(6v&0UIbUAMF0T|_t>v7i$5!N7vlbwdiSma6Sz@d;35d|Td
zuHgB}^&Dno&NrRE+&QFGn9W11CFO{2#F8mWSg4WqfFSmq-lR>bp*_KE+RAp&=W)od
zcP9*sEe<0RBM4O_E~*vEt)Ew)8W>l97r|EWDLi009&|Pk1th>Q#O^T*%F>&+FaGR0
z&d1GtMAW8U;MDWjYY;Ub>fs3a$IEeFn6yFnTNMKz6n7V4kDpCFu>ezm$iu=zW!HJM
zW;SOkg<8n@j&m9q7|gysx-GpINNAqdi3wAb83gg(W{|<VB1db$s&c_>=vvCaA6o{O
zb>z798%m%Hsl?%sxvLX2KGS5CiJ^u%bZR9~j|`Z1*txzcPCIuvbc?!B0d^d1Gnmz#
zy%!IIYZwmR+1g$o8j4=TMr>7{2R?(g2<i>F`ge}xv%K1{O(A0Izd?}X3}9tBE@99J
zhm5Rfp8w-|CF(Qvl{>~DS5B;HnHyF9w37H+u1XK>7J65ZwJteieHcd$^RBn<0-Yc~
z&vOMv3DAirSC3L^3z=L8-3KHmx{Hz=Uf(eJxgyc7FcKSWK)-HwXaiw{$mJ7M0MiBC
zZ1dMuCo*{f;ZQ8kjceMN;WjfM16@_;eRB!(#*mus66^OD4+2$X#?Bb+^uIbzG)AV~
z#ElIoEmU^EuBe8HUM0|ik8#m!#K;f(*cx33(0w=wl&{xkb1ffvtk&&zrf3}4iu1gr
z4X-&L`<O$Dv=S84xa%x<*tsg;eK7>EQT%>_8&SgfjV3VFA>I(fJ4a|Bkr4;2?7Z_H
z6mlSBb=bH-YBVHyQV5*6aARFE_AxTY|DTm&e-<+!CzEvPR7Ip!)^+|V@JveUuU=cv
z1y_Tx_1!@Rp5rvX=F^Q}+6%!3#z<{>c7*g1gc$QbFe?y_ar{8wm}lD*W_X)G-9<lk
z=Q79a+NBz-s*8Sz(E-NUNXbg(nt@V9x5&UeLl85@230+ZQq|%RV;iXWZ=j&bJ3I0m
zu@cry<YydVxp?ERl53)k&b^V*%1x#PIgUH6E<}v@j_%N2haJ1oT}cwc!4@TNWtU4M
zqeSBDWJSA0-s$AO0XHLU6j;&?PhP3P&t8U$G$g8}2-Kn<R@IVkzaDaz(6K3nn4voc
z((5S&x56xdtCtH1E6@M+o)~Fmk?dF{8J)ztiGl23ZH;9p4+l(dN3aRwkJ<g69fdOo
zu3Qad0Kr4tVro$RkJFIR7i~-fodE1ou0Cwc=Er^PsF(>G^S4?{_9cSzhNYVa9=aHo
zSji&BNc(5evJC_OaKen5#$a!*%$9+dazFrPbN=d)YmBtOpO(gW+Tkt+1xk=2PBpFB
z+>mtP25%<?LfV&bY<!`Ae!9pDYOL=t6>cjdnV7#}&ov=VMtj(vBljN{iQ;ylB);ZR
z{VkYie4~Smjuw#23L^g>++9luLwU9-FPHt=tbOI5R&b)j(ww&1MU6KU0X=``%LWtq
zSH(ubip&W+Ao1MJ!6Evsr#J~13tQm|8y(B=^!K>*k42O7|AAAOkU#*J{o$EjJs|Zs
z0OW($zhniA-mx}^%Z*|#E<QvzXzlMJ1zZ*AsAHQ`&fEX1Ar;)_pW_CBK*Z$;LHR7a
zVbTIto$uU_!A3d7FMmA)JllTIuNQ{}cOD_lzz(quHybljAA@+wtlV4x3#JISX_S6{
zW@dhZGKVPCf%L;xU#4n~e<n~K4t}7dMc^Qg3xr4ba}$n=lM(O-X}8OK46seUM+?<3
zy+8Bch0)Cb%7D_2h}+UskYn_(a{+v!FqdP}+b+Qk5dg~m?7|ZvxFLX|GIsb%4Yqc;
z@_S0Gg^>{jMc0D;c0^eauA>|8?Mn4tFC<ant~0~AkKVY5<BEXP;NYV~Dtie)OaPMM
zx!C>89hh#l;($8kdPj~%+r838p>sol1*+=mt*7D@okqEhr2d)>QX;xS9V+w%P_+l7
znnEqsgz}G9uQOwkW@bPLZaOWN1{`@RUwcc%AnTP1(XqvY!ha=w#<F@iF_B@_9gNg?
zdpfV-XqqRWo}j-fI)t<uTC|rDJMk!b1jh-1Q_>|}5+zT+C2f9|y=DE5h=_1U4iAQ*
zxOgATDi%f%^pU%ZLJPE!I4-Z{#j`-bl@bE{&9lJz`x2(|D#3wRE!F#Pr!x=0PVavt
z>UR{VOa~uj_Lp>TroIO@-mKx0z&<n?!VSFA(xik-e|cV}oS(K0KpP<(T?!8<m}qAE
zM;XV-`D4x64>$)Uk)l{a<W;UFVW0_81xjDdpLYbCxzIG|s&-S@3d&e{10S$9oPM&9
z$!4~{yG%vB58djQ(=s>161YK#IA2*o%f3Bqb2IKE54JrkG+}=x_M`OT_xF#E0WYQF
z&+!f?o83#jv3u6z5-MEJb~yRUfeW3mgidtwQuf<Gh8jSI08)z$L`c9RblBgVi1(K_
zA}FGSyoQ2i-pBHPn>`1cJ%PB_Xl1~8y541x{D;#HWz&KmGUrh`7LM3KKN4chgG9(R
zloUZK27&TAWh<Nq><jTB^5!TAn#T}O?%d*60-n^`CMv46wZ@apZ~nfYJNaQ#z22{V
zq!~}%BL>?=gsvIyQO~>6*?BwfqfM}*XA4KaaIiyNYU0I*lE#L6_81zrYIElqzg)gG
zHMQD4Qq$X@&>q^udTTD|U1-*q%(=Z&!)L?VWrx2k$nR&X+C|~~B#@rr+LhPpQA!gL
zM53?*RzCQw^3M3bl?UVXWIp1J%sfP@3tkHd;=)+S^`i?s5r23Rb`zOtn|U{8Sz*$~
z&9Vr=4|c%;FDS%_mJYxT;td?+YFP&b{~ZeLMP3ra>x8sEU^HgqB?Y#rTvah(oW4dy
z+`aFxlnF|d)bo}(v}`Z3d{lD!U_Q7LW6DePm8oB~tp8eI>izV6!%e%HMn0QnyTLVl
zfD|ZfcA*#kBu0%pG?U1W2otG@g|w3&>P`gBPYOa^^Lm?TosOZAQJ!0Wxq()O>c6q@
zPngM(P60M<;4WT)#icAt-Nw@{8bIIQKFgbONUKG6E`Ev;;}sOt0#<c-3BG=y{zKMN
zw3y-9KR<^uVALZjVp>2<5-3g~EGdrC0RDoWP0m*nwu6UV&Nl&^wJrtveBgnN6O+am
zVE;E6Mz8DeYtEIM@>Br0g12Ap?CeHPzE=_H`PIF7ro|TJKm=7Y)wra?I0ml{__(+W
zNZ}6c=u?)Dypkk|lmo81oi9zg?$G3zfeUL4JEG^#+IlJC5e;o~KwW4NxVW5TS0dEi
zv><ZTVMpclmX?;1l5LkXIP>9~_N~W0t&hXWIPx~~jv3r%upK_AK2LUp0c||2zoO8*
z+Sm@U3+HMv2Li$S=ll$hL{wj#`Si@%VsG?SLMEQJxtZ>U8@4_Gns7chc&&1h(dz=N
zQTpL_M^#cVe6N0;-`cB^^xm^c1Ty^BkXA!1ldBQWfdbhRy;q*_FtWgx&z|IG#w^8U
zI>xB8W*79?@X)t`NT5~en5MH~T(K%4Ka_;D5||ziU(=czYH$QDOT$`UmvqJ8DyzWJ
z6w&*1YU-RS@a66J<5k#7>2`85Ch3QXYmlzOKM(e~CbLp6(8Cwjd-zGeaQ94$6LWKU
z%^QL$CEfb;)#Eut0%t#;4Q(i-!kvX*XFlN}jwQr^1e3V?GE_1MBt<k>mud12YB8AZ
zjNutW7MMKA5qEjpzU$x$YC$CyMG~kITQd?TZ)Hj!9Rnmgw9PWPTml=>X%ej9*qT!^
zUNKIeN2>$q6@df~9oi4}Iwd2|aQY&G>E!u;AD9KEXrPW776C_5^6~ED#&O!53AC_j
zSM>KE$239}mZarNpP-B@VTPlv<<B9}1{{H--M=0d{Ma4<rHNddh9K|pw_J!WvAfu^
z*Lvsehlo0q5X)?NNgTUbn=K*;I`=PW?W1l-E3-;6<icBV!9nbNt;&V$fhR4ZTtx8&
za^(@}qWGDf12ymxLA`;jE0K^5%i^u2d5O_ab26dBxTBXvTMgsJ-ph!8!b7%znU}cB
zZAQtl>SK10P%|vhV@J6$-yehK2uup3NnR)T1VnIpyI?h+d+vZS0P%mY9Zo~oyi<)-
z0mCZ1e`vgg8gj-_XMN}NWzhbwU`)o1p-bo@$^|;>#2Yt+fP~B^{8=G2P7iI(!(De2
zu)$^+VX$qEvjmb68th%B@kZ)iS{O{%$N-_B;4DzN5lKd+Yf2#hE2xBweoi{>Z>uJ7
z2<HL6JrZe}P09Fnz%-fN4LXmK=FnJ}f2FY*#-jKX1kwjgFc9B%hLn4*-Q)A(5aktF
zRNk!<R-0h*ss#Zkc*Ho0iCNSoury|ciTt=MwEBs$Ja1YVQ`o&z8IkAF8U5ISV$Hx^
zuaHs!z%&)moo%ennIGS@>bv3xdEu~hi`6J8<!sPRev8E13lW|me2e)-P&15!Z3JN0
z?Srrn5p^^ufd{ngDNy*r0$@mb{3v0}(;(@>S_Zo<k}(o+P7C;3HlLOp?V>u&#q-9E
zXl}9qoMfEq*3BrZSZ!nn2oscf)zZ?scBZtUXIaD``=Y$?(IO_Vd$7PqfGEX`YLqI7
z7;n&IOoSN7%3^tcktB|e1@v^9|27q6PD6lqEV8bv8c%j$m-kObC;BE+_Jg&cc6`KM
z_@(Lx?*zcv11-vhX>sp#2*-!)hjpJr^)-lxiMur<H(Wxl(BS^tUGcxZ^|s?}e1FEo
zMIr>xlF*SWOw-ZuQ>A~wv$Ps8Cm9Pjs2m|Xy%o&6CJd9NT9}0L*ko^_XVZ#lVq)TF
z2=FCyPQoh~Vqj?f0*eKdaH25zk=c#8ltN!WY;Tf?l>ZSDfHdrv2mf@l6PrPov@#WC
z(WO(eBVfMYIS=p|-tZtgf|z~=Fs7fQqJShW3yEmWU!RH##F38riY`QZW-z_7jir#1
zfd}R6Kk?}*p^a+8*g4VR0qu4gIcDR@Hwtq)slCykc*tCEHXN=xcQFJlV>zrPp{NfW
z(@u~BFJ9O=ZukT5k(&xgsZ;6dXIq6aH3U-kpHsuOfPna3zDM5*1-+clH(k})i8%vE
zc6@fw?c<CfJ%Dpydw&^_?||>e9eQw<&jhIrGY-lnkJ2OE{`{Ok0l${6ZVD6(gR(u0
zasRO?aB~nDVFni?n)%U94`TZNb<=}Ah0fztpyF(S3UMyPZk?sJ%Ca-f%+{DajYlF~
z_@D?eHbxK#utgE)Szxdmu-Lbmec)zh3&XgzS-IGqW@ERux+3mHCQQ7$mx><2f#c-v
zwFxf%_Dzk3ak?JPy<PgyPR#3jiDZ;tAOsR5FVK<?u=hZiaUVX|tbN@VZ3W9_)UFOg
zIDml}n}Wgz0=8mT5QOP?`;fkIV~lQ?0Q=1d^Gkz&e1bpagJ+Ba(Mu~ln1~_*au~75
zs6ZPK>%6v&At?B7MXBahZG#Q?Wu(f-i*>EBt#^Y>uEK|d2&lrJxQ;x4qOqz8Ow(9a
z3nd-BZjq|o-y;)0J8-5{3o2%68^l4G{V;<ShGZ0T??410d&)S{?qASDs|u&fk5lkh
zqawP2C2f&nmq&@$A1r3oUU$%Sh#o8m{4-II2cg)H(jA8hq*5f+1GC+TgMAvjqTS(-
zlq)qj_rgDy6Q7I1{y%`_MbT*?&lo0!3Z+1?dHunu7ut}J;5~fUL=>K%SunW>RcsOy
ztVQEM00x^P!y7h-R$b1xMiY`@r&Z|^9YFNqxu!!W&ptRr*{q$yrkdW~<>(EBfhu?5
z4I8&40MM(Z$-bM4x(QY=v&2MLLHiXB_IJj)&gcxD8(Dg4jymoLLKVoYLq;4s`hjzF
zY%*SijoUad@G#^jS|Er*nC5fX)kEtb0$X!tZx80-1|UD_-23Y{xESGGyhDf1^Fi86
zCLb1hYPJj^`llmFkWa8ve;fjRd#Lu>0#(n?$wqoJ>Z64;kSzfV@KF;!3qP6zo2CrF
zadl2^7>jA<0hlreBREN^k9|;2VDyS1LD>{lGZOb7=U|Qq9KUmU(9jp9W<Mg6pb<lP
zq@tc1<U?3QOOY`u17=cfkOXY3c=!<A6Eb2sCG!!HzXi^q19)!U@Jh`~sL_qyuXr<S
zxU&eJ7r?`Y(e;oIBmIf6McR~?SF|5z09J_ku=e&NR7AjBH(w+^%SS(8iI-f=R!KuK
z`7Nc;T!^ha1m#9msfd%X)h#7#OPB*uZ4Nt?VHsb}lK_wV9iltA-v}RqJh_F-RAAj<
zP9N&dwHX~(<jYaNnkZaaZ-|_Cpy#l;%k8H^MMOI0!1weEe31%ggKof4L{Hs}6jH}b
z3*!1X9s=i>4B<YyOa{59QJW-4#PZ@ueCJ&gVOE8JGC;Vui99QSA~j<XY>XyNKjw}6
z&j%!4gehwYD5vv~En$+4lv14ylT5($?Jewb#t`Q5!~0)xuQFwgJZdNSOO4|Jwn8Wb
z89$O@FAsK(>OY>}CpZUw(N_*&YFW|jsXik737MaoeI+}`4z)}icI{3;w{%fK*4KQj
zRu=`=OxkI!L@dc1kpaI}T~sL`jEh5#MU>NRM%qO^*vP3H@CsE##0k8ip&=r*clz^y
zj+q~j^a`I+?ixVU`!Wv3nJj#OQEx}QM|Wcbji55_Ea*G)Id5^WX2&83TP{<h8Hp3&
zHr0fEcX)iB#xPHNXZoR2k?D!A9N*ophP`r!t?1arr-*;DD{W=)^6Z$U?XgR<EZP<O
zJ+=j}z4A9({>n1@;an42<=0l>=r(r~QTeYvN(9n5*p`sq&&Ionn(%#9p_i*rtAWRl
z4)q(HLJ=Fd<cLDQyVTa!|Hq&pD$wk|6=FCkc8yMGdp~zX1Ry?|W%BsSt>Yl!*ph1N
zzU9msNQk+ph>wTaQozZPTRons?(bK{O=Q163^DxksiMmS%Det;kIq!#mFn_8sU`Bf
zpd&6k2%0f|^u>`(Rx_NRwV!?0N^Q`R-+FUp^k&{LTZ-vi{-d_EYku*66h<~(-pYPG
zUfDc#W<+QB#C`oE5GP-okW*46J%GiqVb5z6Z0_rQ3l6=%E0Z9!2L$NDd2HI7A_VoL
z1w6D(@t`y`de#Rj&V0zK)y3LA4l2z9PGlxf(<@Z%taT+TVfT_7?;kb0R8y$R^!w!N
z>9YANqt9=J%_l7{c1bbs?qJ&cz~ZY^8mEMlh(wF6)z<^RJ$}9QHP;Q9cMr_|-A!_S
z|0p)~$)!_kTPn@)_G51>x%Nu`pzU@Xk2>M7G;h7#mD6eAM`&WHT?1+LMbLx!j5~&R
zKbyZ8xafx%jtH@H0~fb*k-gwFwt8|#Uyjg#BMP7(d;&D=;O<IHPUeMxs^Q`3jw6v<
z*@gDs^i4b`q=~!^=q0hw9!Vp?tIXv_B~3-C<Ft`v$GP^Vj7J5TiGYHB$x^Q^WPQRg
z>zI4`j|$aB*}4Fq{@P#1S*2vO1}bahozw!$p3?@pd~_!Ml-OwTwfh#Qgi$B%aBWBJ
zROGQM^fCCV_;)d|F7fNXo?HHTxv8z!Ntn^+@$XBcKfC6fbvo;NGYt1N4z~r(TfNcg
zo#VpAK}0svmw(5O7G4n{#2Q&fdUKJH;0}4%x<YYMbeD|pc*2%_N|5oL;6??!_VXxb
zVP(Y=4FSg?6wnzBsq%*zO!zw`YhCtVc$q*v%xs=KGR9s?(0M*lJJ$5pl%vo_Nxv~M
z^<k2qvIl*=?eDY7HGl5TZJ0Xs>&XN$tvmg7-q@)Wdy~*kHO@n@N!v&CN@%OWNsEoy
z8=uWXwucFcdMKbT4(&uOk==Jqj$IeC72T$^cpGe;8UV=0KWsLV0B67o4BO|(t-4%w
z#N9XfCo58vuPBL)Mt&396{@Fgn#1mEA$jwMHqbANCL|r#Q=XljYFgNsW$U9gQ85}T
zp+|SxMG4a5(o_2i4_U$6BKssPMmqNN(o_lRlO5&;lS!MH8VkLw&lyRxf0%i(Z!N7u
zT5hujl%(JDIcIfsC(pDipKF76j2tI2K71Z)O=U~Dr}zHXpTts!l|RF(U#^Im-cK0x
zd$BXSeMokweEI2=F6;bvc}_Wo_kO+ClfyQ!ME;E3T^Tm_z0~n(<nZV0m}u=3OHp0t
zp!@YatW3;$oMX4b=UZ89JtW(EYVz&rz<vA4<LHa;J+AD|5~|+rnCi5e$$i**?t>WM
z_mwSqf*;-Jkdgb9?qQp&NjzoJb-&A*x?y2;4`^9|)mgPeHNW%4vw5DDTQpi!YC&|&
zY*sv~G?B}!_WtzDJ=>Q9m6ndfs~fUkO!I8B-P;$(@;~pq*%r2X`E?7_98|$6+kvH2
znA0GPg~C#E?aZ~Kkc!!V_>A0AZ~NAf92we0bn836R~jV54iQRM-%jUnOqWQP-TG>5
z4ea6r2Bu<-Kd&e(k892aI`Me!k<8*XDrj=X$?g-I;yGa*7{+;ZPRvzguQYpLq?Vo9
z+n7YtvX4uimuW(`_qQ234zz9i7M8MUs+4n%B}8avtF6GTkiK5t+S4;%3|AhX^bW6_
zo$Zmor>3~8zEAJ&@Ci}t{L;qNyzUa|A$;8V6Q>!+`so-d>J6I0*Hns}^49+BwWXQj
zm;0W&(PSv(+ZkrxKdz$@zTdt}^QbrSTL-^N0pr@g@5&Hk?M!5hW5@D5RKJEuU(y?*
zPH?JU2-q{-sdnpVZadFro5-sHCUOetLhzjX4Y&S-`~fL}#XN~Ws{_OP&uJ8QMcCc6
zkk(o#*{1nmZZmtOjq>Ra9{I8CYx!&n0Rm+<3&FR8XtZwjPZp9AJd7vuZDu>7_u<}$
z&$xs$Yu=qFmHECX+!(*wCnZ?ME@mYPuE8jS<_uF*6WnBr8A*`81y^Y|a+TP4rXg>s
z3Hi%WgeA-HY%~XcJwYIL%{tYpp#sqc)b{Mzh6w#)HZsKB?8NDaU(3mtVoXuAsB`d$
zXL9)Jv!D(pw%S)(E?n`7-VP6=t;;{ljqv)t@%Fh8Jo@7JCiSxVrJ+4Fqw@u?omT!R
za9hircy#Vi_<MiNYwYr?Ql&;t;dZNECG8wVWyQ6RXb=?oXBvH%)MoR?17AD5uWE9b
zEUfLUf9Gp7rW_t@3Rp{#n{`V2FGSY6Wf|r082dKAdgY><XWGd?Jm81df2RH6_4t*!
zH<jv-2cdO1_*9Ef|HI$sB)Cui4p!`<$(+1-c=5Iht#wJMacitp=e1qHN-e}_B^Vrd
zsO11-jtUP$fQ!7#!DF{(a&r778S`_Zf7H%xHm|>Z!9`xaDK`AW_NWI1m)kO=AD!(y
z^$E;+(z{`|D)pKW=Pfn?C6x-xcF$r>k{=&sX-l#mOKTS%_PrvzE4(Tz<>B;b${4Hm
z>x?1MQ|dj6u1y6?VrBWcOYioKK5S*%m-JjZ*+cQmjSibVA%YgUw+|n2j<E`OoIX1;
zz@+iLeUm6M%bfs=rR4>;Md#CG3oErU{Xy>M?ZwAG00QU`4ro$Q_S~YWb;Ua9=lIDi
z`cvVuzB*ij5CzFtf}T~p-+#XI5_8l<{};81xt}56FQwN13S#luF!*L*tNyqg`$w-I
zvIXz&y|QJXwF-NeG&^@)`SZXT(ZtcIUx_O{l6ohSgS92S3?^y~XIJGv#Iy27^-9?5
zDOHxxS=BH2p9_?9dAR)N+pYXY6ORUCkJ`%k150+rtXw%fCwxDQJAc)TaXOb4<VqT8
zoptf+w0oIRs>pX~bDko}rog1G;Vab{?GgsV`aag#?{~QprKhgB4ER25yScDMTm`S|
z*&Q|DBpJiMJhT5>?$^)@j$ht{ORzJ|eZL{XznJql>iX-cWar3t;yH(XhufR_>{^Sz
zZQK@MC0EsSLivsF*`u85D<vg;?~=ox^!@3~HAwOGGynCAAKu>*0!H2OAlHMFKU^>9
zlGLtRJgOHiD^(oOQhw={oYZh9bj#_3h8o?Hh36__0g4`*P&>Rw#Pr8k`p*?RT^_3i
z?^~_?cP%CCre3nJ4B!12<pi3+E!3&cZ9(<nm#TJ<PVH<6K8G4_AS0m;K6$b1+tQZ~
z$COacb=~FuVa*Qqyq@0T1|d35iM4O)fp5y{RMU#LN|98#-?v+d?=&y+d4`Fc`;f8g
z^ls_etYz1$avdZ+cp~j{Tl*il@Vky5PZ8blO;-VTNbYc-)^FF0`zJ!)TB^r|9UW6x
zy90C^3BG~lwL4^T?$GV}KB%PCtyiE+&(V@Yopi=?+-2VOr(tl-RMx#N6P`t<waZfr
z;_YD%q)!j}mMh3R76t0Hi0yr`hb8#tYUKs%^s7-1E;z{HGzwh0BFv^cAI!P2$}R6K
z3hLcC`!!qd@x9K5I9^7d_75l0zvSc|n_lyb+;Ttt$B1S=fADCf*yo|IM|Fc@So{-$
z@7kn@em~U^s1O*dAicoDth!A8@X&wrkWIk{ONXnm9aL90x{6y)5@Jj0id!AaKfim!
zZ<o*GD#_@x<er+*F8)5X-AVxL+@-7P*O)Z#gPm*slO&G|LrT+oy<iC3WiKKCM+{Lq
z0f1DU4_%B{l|rha-$MT5Xf_I)43S6SUOmDh-%W^BqH26uqpN$Kbpx%xXnYTwh|ibe
zgp%VQ#EvhNoD9$WQL6Xru*Z`Oetj=8iEFMb_~xe~gA10O?y3D50eE}gJrAEaE1xX4
z{q>t5u|2PYIY~|9>CdU9LMxxpZ9a3>8nwxS?^|@#@_58H{t6$20^zKDXKw!vqo#0w
zJw?}x@o~3bo+xWPO?LfMy|J_UV=?jWKs#s0rUqr@I!Ch4(letES*gSeERJ&AV?#@q
zTZMZzjO^`cdZuvU`Ou5idps(VwEjKjG-?+V%tfw2(m>ltS-D>9@Q1IOAQa^V`W^hw
zry<drSXF%Q#>BK})8wL2$w2#Bi>t%i@duuo_o^h+3JwG>6njjxx;4>lZ4Rk(E!!C)
z_u{@q(?{vRUzK97yB+pS+s~kMeY_;^I|64I+!CLhb>Zy<F9;)FG11$gQ1vVmaus9Q
zWDVuc{Wx%mp5;0`_R{A5=)<nm`W7fmDnL@|dBxSeuAF0)y-ngqa)QlAw;4ZFKf}g8
zXgf0_C-J(kVZn+|tf`yJsNzH2f>!@M`L=jvcZqMmYWPYUGJnxtC98g7iu&!c;xBmi
ztYy;oNIxFk+nQH@{yx}HtIR7`o_oCXa$a!irq8u|gQI(XUwLlgu$=bCX`lRC->{h{
z<*S#S&&TWwu&!4Xp1Yq$cD^?LlTU774E#UsrK)pnZ&kF2-@miG*xa0Ha=#{>zxcD7
z+UtswlcAGMO_uHD=F6=tW8ddRMvA9s3kL3Wu3Y71G@5&$ovyT?E;0W0N0#qz)$LIo
z7k!fxH@=h<I3*^REhIY;RAlsxzlOa0Wc~%QDSV~UQcu3(LHeKKkQ(~YPH`9NWYWeS
zdWQV^rb&50vxB9RgR5@ScS&dD8GS|=SS-6He$j@!p^rzom(x!TMez2JmF4%Z%7tpO
zIm#~$L<0OOg3wKa`u5SYi_q_23s8Ph=tWQJ(vuhY`a6LWs`x0@a^9=hO?b6<#Z#-|
zde;%}>Y7=;@}I9?X)sUrSF|gzj9vC?y7922sIWa|BCz}AwO8WN8l3!#zj>P0<P7wb
z6`T5q%ndcA(qAh*EHv0pX7lKBgg;DtmGy3Lhxx6&bF-h?N2AN1-Jj^$=wQn5&0O~8
z>!YLl3T<r0ni6*X>~J2>^mpQ^2pzULoNJl76h9z+5A-fo{ni=gui4uYdpET@SHbpy
zn`0%po4Iwn3#a;=*}{lax`JPkg7UdbZhecEoR*>fz+&qcALRe4_nveHr=8u?Q}r@4
z&6(xl3$Jt@m2Ehrn=!~Q_lzUv1&Ye*iA5Z^4ivbD8W}=fWdwFiwvova`Kbm>X#m=j
zbWQf(;QT7>5Ri|I&E3%8lh_U@4``H%nsLjxyeBGfq2aK1+jp03oVTQEu8c>;-@o2<
zyxlp!z11x=rl-qsa#6bYa`%^A46Wmuj5nX_6iCPqZI36s3Z5#xmf?Bq-j~L3XKg_2
z<au166F=1>Zln;p)*IK7ZDikX^kAaGV3(NYUh3|TEyk=;pG;&IGP@k-{&b|9ws16b
z))*aTo&K5QwAWAOcXyCeZLw7G-PWi(EgIi5OgPo0>)!?`#c#YKS>1K_*|kk?_oYPr
zQpx+K-Kw?!W?H|{StCyVYTUiXAqU-<g$Jie)7#Gcj2HE6>QM7&80{oE?J1KR{khR8
z*{E&oT<e8}6SVi+7LQzzmHWh3afzVgB*d{AONhO^H}x*pe{#_>RbXgOEtLVBhcU_w
z6*Li4B5I?(NBxXCg3!X7ehQy_u{@<Fl2i7}sY4g{MA2pSxb*zYPqdngEMv6^9}zAJ
z8n&w#J~lTHt#>ZJ-MEAN{6*W6gKTliO#`^cV(xbwcvozI-jYph2tO`u-mYT~-Au<1
zmX_94<&H~u`?{=OPlH}7&&u6g26mXtS6S@8IiOu2NpIu(^$`2-?DpvNq|PS_@0)E(
zjj|`YPrv&iyr16I%{iQHG59{=)zkR6WO%4r)uW~+w!R|wjNL6S+v>xNr+SSvEUdi)
zHAdb)SZT;^KTl`lYoWl-+LddtGyR~HyIc+V*PVKPXcJ7lT*bvWa_4!j5N$Imb%bQL
z0KT^q1Waq{G0JCg-o-(MqYBVW8daXdqNuZFZ4}DkcLNg;q}L8IGy(Gj+QW=FRP$AT
ztWr<x@=s=y)uZi)H*}iv+-lyS`+s)&!p_l4Uo2wuUYLwU$Bk7SYs(ZCRt_7$6Uw!J
zemW-qazWYc@V4sSO&?y^Yo)IcyYI@|ypb8%@%ww~LvkBLim9Ecev<u_E`F&$zs*Gj
zewZkd6?f>fm-zGLUW!Xs0`MTJ*W5kAQV}-WtloKh==E6TiZX-3VHc@q^$cNe>%yi<
zPk9_3XSc^bF3ot6{w29@38?G{KkA)tkH5?<Pi0}2R}KzcBhu%ciLY_WjVtMLe&*oZ
z(Ls<CD)gVdDt_uK^9$W0w|*y;+70})`lzEj|L(4?x8v^uoj+ohIahvEs_*QN*;vF8
z^pn-R`gC4q=)6n0<I~;2Cgf73z5J&V+HA}}59arr%dwuwkFDtK&&hZr|0p2@A|F9x
zfu;tb&rOg(%%6Q83B~b_xNaO#1}x<j9w~DuE)d{Yu=+&=;U|7|X-avbzoK9Zhk_Zr
zr6y8TROJ26M8m^QRcyZ@^2k<EO10whd3LLg%SGE*&F0d6Wy-B6y1WfY?9aEk*(&ZG
z;4V9wtT5IP_j$a3Nv&r3iA3JH_{(wHU+!pRUHeV1RGDRc{dmv)J=|*1I}W}d7ud@F
z`pz-aZAH?;#cPM%W`Qwzz9i72tzh4;!|k#if(@qOa}|Nlb<&T;g)P=Lcov5H4M^Tp
z(!I55u4|#4&F4|rhFzMEvcxoJ?44gm`nmYM<$U6P?w2$*bMeT%7`c{rCvJLnc6`Ym
zw04hLo_{YY@zs%dSm<S=dCuPYU-Pat4Xzdhje^%fqLq29B7lqIDC+gfp6U87PT%bI
z8zO(BqPlTg2J!l;>ZD%>|M>X%NPOsryTNiP+`ctQIou*6J%x3(H-F^-tLz`Yu99y(
zu?UaxPG{dEm$|9@ELN%1$&L08^=RIlKAqTgyS>|aZ-!JiRkTu!W<0$$d}K@bn8`%f
z6yHS0FKU<h=I@Yt7p*AP(~>&b`JSIMZ{Mq%UD~b+zE{ni*N)$!&dmvvOfUjB+u`H#
zym#Er$qqG*PzKi3=)kb{<r?>jEs_w39S)sMCBz=zbHkOJQIWdNvv*410J{C^L7AG5
z-3WpBRr>#c9{{{HD{zeU+eXzMY2%O$N7bO%Qw6?@{kIHK_fxv&(}(JO!)-Qhv2f77
z-`?@|a^ttt3=IaQ(rmukq6~K95{8qG1F;LaVL1<};*@OjJ-%m5nv6}YY}+QCv(NY9
zRlR#Azw?iE7C#zBfzROC1*#6`(D7{PiM1uqrsdToWp9Hk+_SGwT&M_{zjE{W%&O~$
zhjHmN^5Q?_od&nN^gBh;iLN{lDGsZ>8n-GK?k?S5?Q*#$JzQ(TC-(KZGvU{cUf$aF
zEbmR?aivM&f~+9#=e)X{>Knvs#|tyJv+TOo-ee}Ww6IT-dIQ5l2NNmwt2ru>;j8^;
z=6ebHce>^>Cf>M7gz{fi%HY4}I8irzgOGUL=>3TcmU7&h8e+vOPKC)rfu^<A6?>yP
zRDV<l->(1N)61{Ou$X2bWO`e5t|z3)x?;2tuIUrwt+UpzUJ87GQlZaA^|iv%(5_<_
zgq82JdZ^Q;Z#loZFe`kkZDkvS^zZQ(azUD1<_jaG&&`#C&w6C!WO3zCA7%J_`mF(r
zdy~ZGoAx&C!HvJp7W7E$omF0dtM;sIhJihPJ1SW|W+2SdE*gWga(r)=D-UQc&Clmg
z;OY20sE+hV?z!Xu5Lj=>SEwuVz>Q9g7l+bHQhM}$rLHe7;|!Df>dmJwRVsgdcdqd?
z<C8Ff+OU##<C=o6DV?QLJSfloi>G3IJbruL`)H-Tqo6gZe5L+XCumk(z3%Ar$7Sd6
z`tr8cPN_S{?olat?fQ1n{P#?OJ8|XkqS5yuZ}Bi4MM0~@Zqk?jz>b^+g^4p)tEw9f
z1%H!2WWCU_H~qTFux=<og8Ni^$;3p}yDtgAL9p$5sqVsWNbM(`&G|-ie$#}hg{Nd;
zUVB#h+_J`4Wr>(RXQ@K}M<qe?^1|)_H;3!Q-5*Rh2(rt!Ur)JJ+WUy8c0uJs?VzV_
zub*|Tfeex1yv?hdkDln<%74rf+;L4uqTsNu^rf^C`mN0lQn+VB$`ws}3bpSlnqCiI
z47ID=IGr^aaBHTi*1k6fcc}Kxo{2*CmE-X@$!ZsP7`_eLtG5+enzs4)C?|3>j9pPO
zdTt}hXm%%rZsPs7lRozcZoLl|J~1IYCARb?EaS(Nvv8Iak6UNx%e|hEQ<}k;**-xa
z#vj+`02N1Sh^eSI+aUL<pQW!pumUg@@TlkL<4yaL0luGZr!#NlfB-g}I`ESx&j*J4
zuwTjM*%0=7^tkMo%q|@}6%GC7GaUx|_e{QZ4jC`Jc0Y?{d+Dr0O8mynJ(hYwM(PFb
z^7l?ZzAvS_P_uEbX=p=BtaeSd+suItMic&<gK~1rzz;sD9yC6l_)2{k?;h5DjG;Po
z?P)3SS?%+NAgP#1L+$nHf*h&X=?$+nHi&=gU2_WYz46Fv7sF67)D~FzOlGsW4JB1=
zUmu;bf%vEY;l#NSlpB^{S7Fu*?CZGz_Cuks(3p<chwhqUu_6%DG5AOEb03Y++8l_`
z#bJOtxi@X&NCsFvqO>?Vl*#}_Qeq*<`;|R>bd=JbCJ1`Wo|tTIAaBRP)7#Kr5-|+M
zY_a%R5%lh=>XHF=m<*_;HgM7NpumJG>?~GZD&3v?xzM-%Q_GEr&_F&#E?3b;+zcu>
zDxdL&f+G*iEY55iYWIw^=41dpr9fw58*~n%XMH<!G`5{_P0Zw)YLBFZ5m0WrnT*Hk
z<5mc$K!YC8{{IcN%jPcg99F`5!*(Ae07%bK{c$q94JG7~gbMMDfK*J2TI|DW_M|+R
zhSh-Aj=lebZjhZ}&0!v)Um>Y~lJ34ugGFLe(kS4H#oI5k%6jL1hAz1Kl6?re6_pNS
zrFA48gmEJxE`V<j1XcojaBooE{gwY6W7+^~*vekZG$1M@H2*`50p2wafrI?M8rJ|-
zBadDxY#JN_Iyj;IS15-75p3)+)W%6>Lx<o(uYEq^7CMBY$|9iuxhPTwTxbYnS^ulY
z(y$@OFA%{^ygoiTS@3i#+O=0OpXSW8EvSfzXG3`jA_Cugfx8U-(%bTmlJ3Fk>j%oa
z>8bJDyUgGvc2v6iS#$7c<d$oI#8A)Vr$b6}2pyY*#dZwU<5dQ~zDatk8iE_+?KAsD
zlQ&bBf`d5|L~o=Fr)i1dgP>a<x!zX6-z$-D@M$O0%Cq~UnS2rg#Vi?gKv)e<=s1Le
z73%$t2!^#)7Zd|T)dm5V@_KSHi~$K$Bo|<l*+z+qK+D!ych0i~5NXX5iRkwkec<;G
z?dPY!G&jKpp0&%-=f!72QU7`W_oEYx(|2LMrBw%hfzXv<Pob~(2B&w3$!*RDJiy7k
z$IsDz9fAGgO}_OCld|oA$>3L+c8G}xhJUVp^j$y}=Ygu!V?8;O@ZFt6c%)g=&@d4_
zkBz$ASrvmG0zIkNUon%0iJi^i=Rqe+y$M|>`ls}F;0F=Ljvbijn5F@~vb(fW;1?9b
z%kca<X`H~HmbCnPt_?wj0H{iTCR_?_IUCf$pC8gyfjl_czQ1-YLK^8+4to&|&I1l;
zq-s}AJvOmoOQ@{CGlw&ggV4pyPwO+5?ySz|jv`3>c2vUVnsmV0b`tDhj&vSW<r@D1
zB#I#mdPLX|SOsa1*+G+DXZ6`r@cdQ$mOYZ1@B$7~&#D{3jLR5;eDY*mI)X`SGbA5%
zVu@ND)Ke%(waGB#!4D!uWhho<1@A@(Lkpqz2D3uv&5QK!oX^e8)hsg1%TAQKf@JK!
zz%ck5RL7I3e9Kh<Q3v3(4`In1)Q`~No@q0H*SBC|(7%F+IZ@l4?_{3=YDd_g6<}}X
zQicIYPQ%LLJUBLEZH)hea?}G(=y{#{vDb`VfRx(3pF&R>MYEMnhI8f5F9~8QboXZ%
z9PG6|XT2_AWEnKnneCg=v%s8;o6xZm`04TW2n}ob9jKqYVqmC84|gDlL>bo70BGFV
ze&<_SaG;KpeGd|q!&l3?PQs27NIQ|i>CoAMu|smU!r!vDXnWHJK*=h00X@h}MyMY>
zx1Fu<Jtq2bA3N|YcmRTSxcy=cyt&DvmxM{wfANs;d1Sts4zzHSt>L$a+$~)rBa)Cg
zs__#5!UnJv?(e0FY?~Sp6R%;Vt4X??-(8@skeu}GLk+TCH(`w4!>lwE;NE#KYXY2!
zc-$(|#M1fvDq0yen4poxd2djZW*j{*6v9)wEeP|2pgCZ-HP($q2^0Lvjla)I7=R^Y
zR?|s5f(Qldmpk5cZ%40)BTguNAi1WsBy12v$Ugu(Xd=)MAD<e8LG1o=lOPlS0h6M?
z{SP{`2BI+I`QFF}zg-aI`P!e)fc<8ozi;;72-s>zI7!rPiYOldAtJmU`v1RMqZw+Y
zOn`c+Hqc*whIj9i0H)KzAU()vx(Ad00BzWif*VP#NP1~6BCS*vkOE{Us&R-swuRfF
z0Jbr}H3={Us%m8Do<Eplhq`aHw6zm~`@2oBAai@lLoM{0p;s~5e<^kvjI1Dv0Lp|9
zn2ZL(Ua<ESABqm+u7K@H8sEhkj!8fCk0A_9%T4)5I+-#C9AS$e%jTU=Ap!dzWETEL
z-VM7RM|=vcH!pQyP$!`fFuM<6Eao#OJ2eo*-fg(i8aV}!mM%vu8wjx~yEIvmv#yv~
z2&i%e_|m{c!i5MsPsbelnhB=3<pnI97tqH#+}$mJaeuPitUllY^tlkATFVEAkOVlj
z=#^)J_b+0%<up<`G1RC6hKTVc?4ORQcnI9G=Cpka%+S)C`D|ARGA*j8e*05b3+lJo
zyk)@)9;i<;mR;Voq<&T#;S8baNLA1;5g9ZEeo(_x4BCE}aXbR@%DkO<FV}woXaDg!
z4&s$oei~`%>0Jjfr50dx9Y_(>-kn<eFI~fBAV8cFvZNIo3u>{|0s?l+Cg>Kth@?#C
zpcJ-YplO?Vz-AF1;<s`r2%y#k4#l7}XA%Mt37sfmEB>s7iY?P6u-vICk;9bbBD{5v
z)O|4aI&i2Ctxqn4H8};^dT1j*kiyeilc5Bfj4+TI`$Bo+73b4*(hjO@-XXvkgmXg#
zAd3iab<6TrEdrOWqLRXJRl(SQv;xnKR?oCg9vr`ic+?k&*ct#e*8)TW@OJpVtEQc~
z2*SH~6h=}WNVmXxuX7<0nKX3a93;)HU^*}O?BOzvB7*GhzMaSmix(|B^<t|_cj2`I
zW@QA#*?@YYpx;aSC%%tj#&ltyG_bD9h<!+6AG|A)z%e146cv9HtbYbbVL9Ck4_I*x
z2_VD#0Mtzdtf2OAV`v@(llI|U(F%Ts=z0r5=jqI#=ZD?Gv9N_(@vu8aq{b<)#p#a+
z7aRasN#I;UBYM7oj@txXjNbyao^}@+7*@HNBV00|cNH><9#F_?z#=^*A}!@4YrqP_
zfEg^7pb8^!e>&w1l8E4aZcoq;I~>Q5GMSkNpaVYzyp9%NEartyk^W<`4VZ*6Y9>&z
zOw*m;ZwU<wq~A}lQV)UDh09D;{7`SamHGv`rrf&p55f)Mg)sPXzJ<QudUT_A2x8AX
zTY5IUvrNgA!UU|5iHsYP*40&GiZY9sQ)|A+i>3)^&pB@j+VcpqpWHh#z<r#jk#g(f
zgogqW0bD|G?Jnd(?8VSg?WC+SkAYbUoR4u<r2W;@FFB}L=)4-e3{DvsgCJg_`r>FS
zDUqT|Itz<t>*d`+OYKQFK4*L5@ELx<;3uF?1}H~0n5ej7#1Z6U)S~3j{z>CKL^l}c
z<*vQ*=$1dkOs;3c{6OL<@DgM<ZoP*cMfmc}Wz@0@0l|KF6SyKh8Z5DW|M3Jx><}5f
z(3Wj=-4cC1;wg=ctxP>p02&Sf_cm9K;ohKS7OvRS8#fap9^tPaFy}@uZXyI56f5(9
zme)x&_K_=Gv`9Xd50Gf;F)d)sqWTguUEj9{bS<iq>dUAZeUcI!AqWt^#7-sXJ9MK%
z{X+;J)UGfWyI$@bjTSC>X=*WvUFQ*SOIAmm^ZuqUh0o-J<23)S;)1^Orj2a_$QGIy
z)_kELFg==Ts}24hfJfDDNPCJRA6X%AHlQruc%AE)Nt7(R88*b2`VGq+?FXn$hjAZ_
zG;@|8x<)i1k;mV#X$pQohX^v?62-*xN;B=rCHgi5)~tH(K`*frEDWrMK(dB&th?6$
zoe@`dkVCF%hR&jkIc^uZKI9?YEwDY4pk~cOX>5$s_elsd4Hg%p`b8Ynz(L>*NZ#P8
zM(CS>(a@DdA7FXD2{c>*=JtZw9x)s;+M>Dr;+KZGfz^9@4Nka!JBE^RA20@U?3P7m
z1ec#%-cu;3A-SwJfN`X&_Bg5o7%Ec?14F@FOhNW55<h}oM&*GfnSD)^!g1{AtMMtI
zBcUf(5M?52rW*zJt_3??V1>bkGpF~WzE#O^1~#MtPVy&5Ju;NrMM#0N9U_EP+C@Oh
z#wx$RX~B~qLPR{h@ZhE}YQ@e{3~oiZQjpyL9ApMMNQ@=ulQ8KwcjM4!6kxC&kC>aC
zxrnGF9w2xyy-BAk=@dr(fRqv%d-);Yyq=uAqXs-9Mbl$<F$IVqjKc|B_IP%kcF{Uw
z?$U5P)mIoo4-{Y$>n!=72~BV(0>obnkf|Xnzj~dt#TlMw0yel`FP@zu{m}NqGHr8O
z?P~-f1TSOT{36{7&V;B>BO`WM=$Stpgf2)~M>nqsUV#^E+({H<q7E57<rUv_Dfb(L
z@WL_2T`m)J-C_D>MRFdFQ)HUXp!!1JglHhBs0--5Z2S>J6TTkqs^=NswTl^}G|BMB
z_;nOl|0mm_b?3+>ZtWy6%V@aiL^L^9-LK~fZlIdYJ*nOBKZssva>CE&VR8&;a{9as
zFu8;0eptD5=3W8#k3Qn$W7sS8Q~!J6X3BYXfGty!i_1M9u@9EHcqgR@5I)sfpyU}A
zG3DH31QZ=|Ip%sEOfPg3rAnUZ(Q#_e5=jv<NkqOC&<X7}xG_oTM~yfE$GPCzW+;Zw
z7K7vblb?%v#Z|dMI<_m$1C0G27@5pPJU1MA0$d`Vrsdff->D(%TL14tKr!-DpP}gQ
z<aWa3P=i23Dl9Xg#;*vhJ;?K1`CiMb;ymOqnA{J~qpk=#jyT=dd8Rd?-5UN=YfTlq
z_5EibI3!4&nrOYKr&6!yXVI#P!rZ)%VmDW|Ak6K?ja*1L-9cWGN%VKd=`<1;L>{=2
zW`Uwe3lc7lK=6ln_~YLte#MjubybLxPmO5br44}jTw<|AT<sAVZ~}ETzXarQKhRY_
zRF(6Y*5Mc;!V`CTwHrA^I|TFz`8w&`@Q&s2s!sd;*wpaLLSP?lEwh4n7#op&eBFl#
z20$^vJ|e}WbwHg$!mLUfqzgFcZpAuX;ZEX%K7u5G;vD<<<dUM9(Orsp1fi&oh)203
zG9OqDOi%!=%?N66$uxH>Z8bukE+L*e&`}nf0F|tOPjtVbL)f9O>?E`mCjHNNaB@DO
zQGn~0_M#B%nXzCM()Le0uxCndJ_Ar?0xVkY0+}6pWBMx3q(1ccV3B|PmT0e)&s7K&
zU#hWKf369FCN*>b_)mf)nh=G`7JBM)^l;CeTQUd&S%wS2aSFZKN@9R}Zt$&kG855l
z^8w2$9oOCx5r|bP5St``lDI%PUOBl=Vkro)Y*nBHSjz=D0PI@a{$?1Bz}X0PZRDFS
zcr6KVr$q7q{odxu;U?RlUtjq)Y}%SkFIN9H_IyXjL6qzV;pAjwhm*`1(G;RxRDqM+
zhd5v0B=K;P80LEjg+@24a`|a@W7z{_Gk|-e^R~(cEV0y<y6rFTP*gTd$)K565U&6t
z_GE-X=NDWHelR0zj8Nh@iqcgj4C$cH2Z0$;TPjK&0C8F#_>gZY69Tsk59dcMq{U4(
zfb-*UH@gesS+Wz(Zy+Do`~TOim%xws9*a(tX4TI;-|OA3xbPA#%v{ZD+|z%@ibZ2#
zL8b~C%gq=GNeSFZlV8VjrfI8D@7kG+xbr*-{0E_f*k!Q#0i*{tBDb(|gL83VBSm@e
z_HcuTV5x#_;9E@VpEBGDXLk|K?l__bfki@uj220hBmzadCA5E>QExIX)K@{+cxd+f
zHz77Op$+2ZoMa|6A;eonh09`s38m-fa?@5rgb{o1M$8lL1paCCB0{l6r30v7XN`|D
zY7NQ^_;lspSea-nX32g|GBq0OwCY_r(yZ+;RuNFE(^jKqxHC7R&vSEnlS$M7Ai=t)
zqFKa5E1X^@LY8rzYZ>4syTH`EAv;`-hyhbm4UYl<_fCKzn40|=DcW4*05COip4^+G
z2scPzoB3pA%~5wg%H+jzC|b;ZY2Su{{6V642@&**_}fW*(7{X&`H11}ILYR4Z)dq%
z2t`KZfX)Dso&X!p1&5w}0WUyns6NmUG;fx)VH<&Rmq9vZ2TX~$=x{Zn4NP-2g}^Vs
z2B!2SBW*J`SrbjkZP(^YFeTJm1hh)fCRX~=`rHRT1lY({i+S>IEVMkV9#X9dLXM@0
zBO=5=$d|Iyi+FKyD1xfyze9jl7)0`hVoPuWDCwgrH#!E&Z9yxe3eE(j%a#U|&Wgfh
z7Q#mg>_72Z*OAE!`Tj2?X=qe9b;$s5JlvHqvBLqV&?|HE^R=Ow^dsdyM9DQU$G4x~
z7C;nqbC;g(jrs%zbPCO=q^0QKe_Aba=@5}ZBDnX;D?gKbzjkXu^BT{8*;Cl~!F)d5
zbMX@=*$(7js|Q;&7>ULmAO~$u1)a3K2*v#*BLc@skpsjk`ThNgH}X>oDBU^e^^}34
zd-@{9qEu7_D)fFX<c5zT?*wBgWkXQ|mWU;gdO@E<KP($_5HCWoq_ta-6?CV}<Aa>U
zW_69H2uqGy&Y{94MCoOP0LBhX4EvxAJUMY0dH7fZHKIiL;yAnyN5%QSUYt%m|M9E?
z?gJ060c~I#kcy3h3Pk=@LMu*KC;gr)+=%lH*168UxojITCw>_sxxZn3*8;9-Wj+IS
zh@t*baKq|!ZYf#6QCe~vuA{=N9%>kZ#T`3lpoia%?&=wL0v+;jU<2D8@2OV@0uU4m
zrb_<ioDL@<rqVr<r#X=eumuwI(10j1pGuX0*>$jS>^T47Q1F7z?MO3(POc{)u)MKZ
zvB)dYfjiOp{|cPlP@B;)=<;w_UYzkBO*x^htsQL@G?6#&x{8Iju=z1)epoED{Q%4l
z6JGFJH^Tf_2?5XwdKL^bGmDe>OdSa*b%Y>&R}P2ZGThjpu=2AD<p+M>yz~df-EaaH
za01rveRr@CtKsg6l9y#g-+2Kd5~9Ol3R8}SylgY=bq6MWA|wZHOuT?rXZI{>37}L3
zl5BCA_1l8%jRqQu*s%}L0Wc7T4xqwu(R$fzr8I+n1xuK%r+pqYX&DH)P`>l`^wVvK
z0T;cu1RTLs|Kx%i22*?+WlT+!0lA1mmJDXawHXFuihd2Z*<CbPtCl?srjsG13zq~R
z2);OV+6#{0Eqf@}D&sVY)Ua++LM(Czcc*<u8-EbFgLKcw;ELM`ro!MPFB9B5{*!%N
zz>I!9)h0*_*yT)7lTl{ykD9+WHR1*A`-0g8Ak(u4q34dQ=0%KN4o4xnm3rF;xD;-b
zh&seY2N&!#G~^wFjv)ucM*!WQF1;l;`)<ms|J^JR>biJ%KdbDKkU9i{oL|9YU|R#{
zs0v!d^M6J1RV0WK(3396TdG-}ZG*<0cgvN%a}+*9;HcOS3a~JcyoZZX7!=?^BA+$7
z^MhfmiZ9P?V5O+G^K!ZG{TCcCAlE=<vIVSl9&i}KOJ?CH^VV)s2=OM`<{=gc<ColZ
z7?_~l>r2dfyBD|vZ`wM}^1}5-3x1LgqNs?#u!ty?j%Y0ak#VT~e24Rsvh_PQLK$tD
zhP=@+d^Q(s*`DZJL_(zsQH=M`16mMh9zn3N+MSl%cKH9M12}>w53x#nWM=vgxQzcD
zEFKMJaLUL8&ka!|6BlP45@7?lvDqB;6CprO07<>r+^h~Vfs$8<R%*Z{o?X62=UC)l
zJH2=u?DQI9QRb|JO)nRI5e91}xPf{7|FHL#K~=rsyRg!o5`vVJg0MllOG-+*_m=MN
zmPYB41_?pBn+?+KmTorP-OX9}JLk;2@67x8{qmnZ<G`L-vet9gbzk>-uBW#rQ4LTb
zoCZ8AYQIwlDdIPk-yO}9o)5rR%K}@DfQ_lZ!736@Y}gZO^M1mn{|4wW*ua|4h#CQX
zF@UUqH8%>A8wGII&o4%Z?5mTlu_oEt{-T1&t+z)`DKj3)cg?4u#(Ecu#?psQNAW1a
z6>X#wacVk$Qxf>gQCI7KU=^<s@ZDJQI0OV=e4tqV!{^jX#DEQKpXBZUY5^FR6blt9
zBp`G9H7lYBjjMe7{)WLN)H%)x^cH{pl?;ym)>ER&G-**?=|w#zW(3Ar_RU`)w}LmA
z^;C@sz~;gg7EwQZI|sM5sg^VS{2xYbUJDe+`GCkHo|C`(83~f1;bMgGlB<6-NejKE
zCREmOUATW=J^PN5Buu(s*`XsyJ(}DKJ(m%nYV8aE`=662D_#J5YzY`d{bJcX-#<0_
zdh~eF4*>vp!9m9W(i5;n6)#$mc6VNz@ar?4$=4%B0K0sY``_gvl>6hrEb}$~kG8N6
z(LPUY-6L98ZW~aAvsS;R0syoZd1b`HJ)O;9Vzo!7W2+yST<S3`O$kRobo@kp*A3+7
zAoRcgC&@n7ME~#NB8h;Gr^Kd>*_Aui$L}@?^j?v*GH2631l_&)@A7DA0T<wM8Kf=H
ze>WxiTLj^F3j|Pro9;RJK(ci3cXuYEWR%xBZ>aqK-_s>wT-CyI+w8ZW%T)s0paH@e
z$$twX`dj_KiyL-gepEa;*(C5gXVsq!u2?jCBbI-C-*b`wE<aiwQ3QHeYJ%rC;s4zf
z-chtDU?^Z`F<<5{K;F|`kD4+gq!*n~L&&y-{$B|J#=XA#r&}OB&)+K60PT_d>|XM}
z1;MjV{O{s2e!71;zA?3;g8Y#sPG$;p2=iLu|5o;Q^naJ%x!0%yydj`(@bLP-n?kyy
z{m){4QH++rZ0&!{{XgdZpXmOd=>GpRnM=HwZnBM^IVlPM7Th4cmbU+BmsRrU7?}>a
zFyz}_6O3h)!9a5trKY?`_9w-C^+xS!5d7&WU(57LeId8{C-l*QKVA1=c74zZGCt)I
z>09)>lxYnp=;I@SyfG=(V?+*rfx-B;E;8_+vxoD((lhq|?EsX6D8#jji8B%$D?C&p
zv*8n%$uKJO`Z$MT{2ZBJdQ6;8%MuGv*MIdlPky;@+()eM{rT$tz)Rfs+92>cYHoA?
zW!z~O;c6Txg?h>OzrvdCjRcVN09?R&x<!K*p$eUNGF_3i`{ea~Z31pTnSc5|he3XQ
zN%x&h*%rq2Jw1f@4Z7}z)Ur1DR1SVbDd#TVuTkQI{zzBeEBSpR;yc6YXY;tw0@Gc3
z@z$}u(ck4Fd>CF9k9d1A_;NFYzTgFux!({R59v(PX89eTllH#b_Q*boA{lr_0~BC(
zNoo-Vjc{Rdb2BLvxhssIIu&mJuR27%0)T+@yAB^oKm4-k*!^u#lvcw!(Y>TOORe9b
z@bz^wmBifDqwactmgb+PU+!ABycVl`Lel-Hur?-oW<6FLQ*2q8k&W{#YB@7jHV;k3
zGKOepE8<hfyLzu^=cfFSSJQsK*UryIwH;@*gN-slFM}BDx0YzMFK4Ezd^(9|Z1lZ0
znca)heU6bGZMQM$H4oy&<$_A9{zDb=b*Uj4Pk3w-sPtp@;8Lh^&WCxev_*#s@0$~W
z6oifH06G)fB5L(CVOB|2LC^v>1bUq<Z#lzwqz#)Y=u6qmB6KeY?bVcf9pT6xz*#E}
zezwQ2mlmE5-68}lB^AIk0B(K*t7M8q$hMfd?eVUd?%akp(Q-b~mcp)zm=}QYH4S-|
zljO!#4AOsIy?2}#l&%$HNb|g*h0^eyRo3N4Z9<fFSliag2qWgJT%p$BYQ+N^bzRn$
zqKE?Qc`d%S3lCt8qcm>9X^EFxa@x?OneXe)NVbLMPof@P*L)w11^X0nW7(`e+Nn2O
z&5Z6w78>lY=0#@Ug(+WddPZ`;i@q>tm6Yo;^Q@{b7*4|7gtBTb`a<WY@}uHYyQfU9
zkAkPxYaOcBqf6(a9slCrOw3IrVNBR25&~*^zs|p2iTdR=I!x$XJIx)cY%dECDhK`A
zc=-g{<kiw5x}MtKuBYXd(c5pyaC&W5vJIN~)0`LdeDU_s_Ifo5z6KW=lN#6p+0N!;
zC9WhTVQ`A}h{`8=;tm8VapM3>mkn&-Y7WV5>Q{H0{{ce8<e5ZxVF3M0Zna=n>N74Z
zQ2$^mJXSa4=jrdf>p~Wy2#WZ4nL-7B_exmdySyj(3@-k7sY>s*7=Pcl)Wm=L1i5$}
ze~m(3RibSI&d95?UH`L9=5YDD+ZjoAZNj1q#f7`>h|`B7L%*b<_(Nm9>+!qx;vro1
zc|pr*bpK@MA1%kFUmlv%=G@X-L~hr=`2WFEp7#uav06!?Qy`b9GFm;$KObs0eWCfe
z6RGvectqfR?%3zuP#-?yGoEsW3c8Vg{-<1%yj4R~_|&4HKVjn7qmQ07&%U-)y83e=
z{OCM|AsL^`ElnMku(iLSs4Hu0*}jd6((5;<>n9Bx$sWC5l#}I==N}i9-~Om6J6MQ_
z;IWHoTk>-?>}t57XJI|<QY5je9?Y!JbN-E22*kkF^q{&i!$x!fmE6Jc84XuIGtRi1
z-5i0u;!A%D%nz$X_ua=tz>juoT`T~MwMqoLjmN{eZ4&2pMBd>#W7u`c8}s%0+5yT(
zJPe_Kk%W_Pi-xviFAy7<ZHwt}g(f~=WT?_rR7VMVxHx}H5oc9~abUb)*a9AuIsy+u
z5QhG*|9%?LT%Et}+}<bnc%ANSG|YOK77vL*mMBODHlitE4Lw?;8ahwkMJco;HwN!w
z?Yto>)~s|BItWX4D#hm&DZd`M%PH6-SIx7ptVX;IaWqoc99+2Rl)84gHywd&W_Ua_
zI=c3+s4NXl3w^pj%I>Uk?M^z`jZQO$FFt-K&|vmZm@yJoGP~D=_qRX7pe6qAygZAc
zk9*(ClgeT!()_qk_$j`l{4}f#axzlL+OOWN&+s=)SKaYG9hAoYvn{dbckK(^yS|0I
zb+%O=`O*EAaUH|D7<7iP+?xpR%W6A_@q4%$6I?B@KX%GXSWj@k;P;ZozOZ??kOs=l
zy#MBe^{qv<>}FPmUw_J^QxI#h-tn)Nz5Da>?}t-VJcE-i<fV-Xe#99%uvdgXzv-Ir
zvbc}QL!yhOv?qPfesZNhu*!NiDdf)?Uo2+_qDoSo6v5<2`RdNE#$3l^rL<Xgg>@;k
ze&#0ZjhI^2dv5oh+<v!V*z!=m9$e2kNlLNttA5D1uf50Jg@0x*E_)4cx%_Go`kp5-
ziGu|ql-YW_g?W1Sv-q&-IN4>gZe&%<Z$W4L<Kc=`8Sl<isXu}asIF}81u0EI7D@Bj
z>*CUrus{0u`&CNAH$Gk$seYYA2i5tqefVt_1pgfDFXX=u(xqCjpMsx*cdn65YzD9u
zx0TjJ!}eadL9F0oRqN@N+k2?#h?$P*g>__vKk0o``PjQ{w=2QxlY#3=^Pr*3(8igj
zsB0Km#t#vhu|u;ImJ2sM{Q~&8QcdQtIp}6c9^2MLucqZ?f%l(=$a~-3jnb2FFs}9#
zgAUJvQr48y&5X9lu2Y~o=(dr~mH(#x%H$`VX6D~ap&S_NOqFY(`s0G*H}yq=byhC8
zbl@1K@BP>&ypLv=f2X6Em3<0P#|;YpyW1$fNnF_~0HGM$4A?o3+_=1q%7~k@9CKn7
zhTlZ_c-Z5QK_}l<x#{#qL0{D7Z`YX-T`k9}n;H)s=s)&ZNMg!0df9SM<y_Rw+!t3X
zZ*f)l7PQG^X>uCb_rVJRy)LDZ?ftW>(badWh7dSWO_^#y)&g0}LOUYrXcYY}{Li6?
zUA$`P?mP!fWUi{|lcF#^<S2jRTx8P8^t@rrWcVcU<mQS`!^262+$vA;$nm?2PuY}-
zHp|W7{+vlxibemqr(%|>?ncjv>Gyz$+pTKTa}l9GPTEsH;Jy{3dbfIx5`0ecyXRrx
z)vfbd#rmWLGs^C#E;_!_#p6k<xk2rzos_e91+iva)Vke(Dw*iTAM})sY#nT1Mz&TW
zvXjY=Z1$FTuj@A+?a2MS-!pi=xf&VPcwFf<+Ul&$lW|TWgzGGLvfE^YY<L!6=?aUT
zjNDQ}^q2vi4MIG2vv=I;<%Ku8zd(I>{m#YBtaEKQ0%+Vtu*<f}5sT}~sqP$>TzbFZ
zhWjelCz-YC(82xg5{D`>VYxhu`|APBpjaGOWCdL6+>X~jIf1ovJ@;A;;d)n%(372Q
zkspUsi=YKd??JCAoZ%jM%F!xwyE7CA+0GP9QW@~J7Nv%y)U3y3Y=_GDZzqew=VGV{
zTjRu7)pi=ys7*e}UmH`L0nXn)#J(kz(HbulKnmuh;cJ4&9~w9O=khG8Q>BrIIPrc#
z9$Y-K;Yc$zH(rf<8&+$77@|{QXMT+*YdG?ABv(g|ptEas!qf{+5~TLsv)5aTlP+r)
zAzT3>I4!o|n?9##&MVH9@ZhSGiA!VGOS2(o#EX!m1wG|dMHw&R_&?;WdySeO3ZCC>
zb<_JBeLNjiR)n2X8nJ$MhTNCb5s|W(-Rt%>yrbkAOAW9o+v@%f-C{2xVT+V<ft{gr
z2jxTM|NdPBOIKtVAg`Rw{~+<X$Qh%H(7FHkc`g0k<K5%UQ<3}ue0cS@Ko>}g<!2<%
zzr6$E*jScu^qV~*`|(}W3%%zVbrv7fd^gBwubg~Jc}$9HOBsm{n&}s+xdIT`>&HEQ
zry5#>nLnd<@3*FlQS8p6ea>k=`glK+vdH<k<Y5&g0aM;UaS_>VOl}t;)#YEMPvQA&
zXwSV-yC`!=%Bk9}Os++BciIa%@rGBPTw5KQ)#*5M1`ETFtank>Una5gOeVDX-qNo2
z5_p%ojSoEf9bFy@n8<h9ZuC00&uzK=-nCLo?7gkQc;G#1nlQOcJ<Po66owi+rnu~r
zwroY-F!o%&pNenKlD^Tg+bezyJzeD8xmkO)n|)bYa#-W~g`T%ODMSm#nGts(uX=wM
zdoWzGx}lLWU6O?K_~YZ0i>~0?LDBRwTo-NOORkg!qr92B!la&WV6)JsS9Tvh7P&-b
z!!jDLW|@CZocT-B7mG<lQECvG+sU__hk2%oO!dpBVp_z;2zDxYe%d@PDRB7cS1VaH
zKEw`X85B*Oa%8@WJ3xh`hA9%2!oES9b`-Wpy*<{jobKkYuAo&r`_b}vQapdXo<){=
z5rITFIx!e**lxGZ6(Y-q*YZ;@Q!khAK>v`2y-v_S1vS(s4$I#DTwkB_DR$4+-G=&z
zgC`*M2_;&Sq;r5CPql_LZbZpx-gV3FAYqB}+s(A8Ud%4F8cMcbVz2Hy5*|4-X5VqK
ztt@N#Cuc|3vA2C1izW3`h4?{#@+7)&cFht&k|En7lf^&He=zEVKQu{Et(B1^-k8)_
zC+^bO`QM!YSv3{%!9rM8sebruIS6+7lKIy4Ffp+l%6R)Y$4EAvd_K;0ssoxolWKd|
z*%=)m%A4tG9bs~8a_&Iy+6o?so(zl3YQ54pn^N_}zr5ZZ9FifWK+eWBUN~7$Vz^f>
zCP@<w%}9y*p~Gukbp2DkQcf-vqICZmS=!k-oYnKVuvexBA~^%><B^)OSlS6F9!OT^
zY*O^22X?}&S)Rz$n{K!qJ$f!^Lwx71!e1)qK5sWPVU2q%;l5qJ#b)5mB{9I3c1<ik
zXkmBf*>-~y`h#}2HISeqkT2NxXWc)PM!}YNHh(2p6y_{aKyB+fd~fY2M<D&%<!t8x
zY9RA-ao*MU#ZEfEPgaBNQep*l>I(N!bb1fO-fEp;^*e~;oZ2hi^Qf5FGgdcKgpKgr
zF!!EJn$tm_jXuj})%jy}&^G)k`X*fD{@k>;(SpRu6mi$j^LajrVbhj#P<P3u@HS|x
zRj+D(En}j_+B0`)lS?G!S&lI0nSoVm&`%DM<X8dE?7TL;v2aQsp}p#BhI{vWSh)}2
zDF6E-=`MtQFX;|`S&zerM|gNB<Enk`HGakDiPFlm6SjO>-k^oEHwU`$QENNhYD=Rm
z7fw6S{^Ck4H8yf+cjL3+XvGhYFKvTB&$B(AHTo`kJS02w^iFBD^-zsf6!!~X@%OVv
zKTxPokvZ7hi;GkjCTq6cIy6y<Y`cW`W?|Fg<(`k<yVP7GYksej(u7!7Etn3!sF);c
zFQWw?RQtYg)*R~%3^xK7xO(;6lx18Pu~F=oyNs{9`;$T!N3DeKGYfO=#%0P)xP9-T
zPo2K3gkQ3&bguN!?$RS*_xsi&+TC-qPX+ReCXt`6t&f{GbPfb}Z=1=aojiOxUOAN(
zTI}Eso~8V{c|`~PT7Lf4!YAMVYeeqFIjRc0^)bb|DziVQ`uP@ac1<Q(&`8rRZr7Oy
zOVb&Id+siIsO9UDpHZoithD=9twuRden_GUoj)ly*tPY6ea7ydR{@cQen8(8!TnK(
z7F_diBf*7jDadE%+_Vaj)_^88^(Cm5`<$VA-5h>E%xE9npFI8yr$T$s^Bdx7kYeUQ
zkO|D=$B;_m8q`SLrG`q`ikP`KQFT4}vtl&#+eA^a-=XV~q~DV3@O`~5hpRPLuJajK
z@{p%sd&7X`A1NI)LHFO*1<j{Y?qqd%5XZOQlL{WfDxLwK$$dwmI@8p<=;YfMjZhq(
zfO%GUWEEi#%ft@IUTU|ITH2DWxces;A2f(rYPGTMxH_>o5o%=(X7dEN^fTe*^7uEy
zRpzXnAmqC%D__q@Ncday?G)kHT}B(>KW5l49;<hAAd0#$(ic|*MXDUbJMZO%wXf>X
zUvHC)5BYlJOS)H$=59<`-AHO&j)mt6kclwaDP))hklZjAH%W}zHc4mqZO5T8;u#(O
zYU>+?n9i)T7)2)k68aVBvV2i)VTU~sab@TaZM<#<XolSmyGP~DDR>rCEAuek)j-|r
zGb$k)+9EqIUhXz`+BnbAkt}u&R@um`9zfT0o{<=~hbv;TANki99UL!~u7yY+;(%=e
z%*gust(^zL5gGdxt=yYgJ(qLae8)YKxx$P=1=?cKq5M2`aFmBx5;-zGOAagh&q&|F
zdrGzoun6jrx2H|-Qe$jwH$@R9!@nmj+C(99)fHKsG&X9uI0>46+TB>&QOVU+tqupm
z9WxQfNh+Bfy*x6Fg}*}*q^6@w5=)k>XQQCa-71UAp&~2&ZYCT|9W8TU&dPsi8~g53
z>v1N0G5s$&@{OHvqn|+ILA$`7`u2mO<xFoxNOd$iwl)2^jSYg5XW8Fp%5XQ+{5i%+
zyZ?*L!%|+yJ{T7Lg#=BG0sm|yOlT|v>LK?O{jAkdIACtGN{?cA)|poVbSt`OQO;*Z
z9=~MF92wLOR0{$P7_;YoZurBm0!S_6kQKCN#EQae<nUCVnK9*az{4|nKQ!VN^QD48
z1+HN?=uqn;id}K}=6Tj<-z~=$(Fm0H{JK*_WY#12f;WgMOOoYD&Sz)(ljku}SR#`-
z7B#JcHeJ9G9_+5yXI^H5Z|{eljZ^D<HT_1&W^zSFD+}_g3MOk$>k!#*TBS6LZAuQC
zFWz5nJDh+&&SKx=Ls%2yOU`mDw-N@(-i*cGyllG#5_9^wk2dhb(wpKGY~sjd20M9g
zVSAxGk*gvz{pyW6U+;<S*_to$e_Uf`65N(2v!xUHH>4G(gzcMCB&pNyX@MHFm1nU(
zCe>e~@X~QS!RCgRBSXA=wA0O0J8yM8%R&0yVdcwhT1xEsX5w3eOp;>NyX_p+n(gXo
z&WhYA;)T-Rg+B7rKpMg~^V;>4DR?vHcY0$73A9jPJmY>&0rg|=x;f7G<=)xto^*J*
zSkwP6W{m}z-2b2Od+)=d)>vcE{-)uUIa0>XcHkzMi${w{G1N#?v51(h23VC~i=%Ii
z49Q2MOZ^)t-hRw%xjbY0OTmld6KTyCIdc|AV$yh9c+R&QWxfV>Y469azpDd^B8G1g
zFv$d9G7mc>XRewepQS3chVO0#IAxO3aYhbx8oxKTmyQ*ioG*P4z{?eYtDOpa)JW?b
zeeSPz8jieNN;}woAJ&luh})z>efuY#h1C^5c*-K$k>R;}ZkNP-EO;JOVEfAgf#PLB
z_81ZX6>pGr3Q#e@XMB>Lx!Vci$n?AoKjx<WQGFJi4XIAwz4W0tAo#;!c%k(u#jmCM
zXFE<iZBDA`LCYT-`1^C)AYv}wf^dEKQn(cGyFUXq@Ak<-N$_NX<?2|Z?+9ckQs&?@
zLLjXs)Vgl$al5bN(yYzeH+Irw`6`!H#yMf~ThbyY%wu#@$fR!iQNu-_-HJ{pbTcFk
z(>vo*7m?kQX2QewMpWB$`@<>6Kf>({&`9D=eaz%!xd~ql9|eH%tH(DEe~3qRK;qb4
zbSnaJmb<tHvq@gRkr)TyPSxY&lYDDZfci;rcm}-^3n>X&$29s@^l#1v2ebIzlEUGg
zS*o<Qo7_x5ZG&e_7h8+FcfO*Y=kHsS^V>9EKgtI0Kg!#)b2EPXqP;*g2$ZC)2p#@1
zbM-pvmzp$>4BeyacpwLastU;|dBh5KDVtQ|Zfg~il2DqQp;2Xhd!GF`U0Y{u{6cz1
zIk~C~+=n*8J5Xbt?=Uv>MaG0Fev=baCkR)rdS;#Pr(}om@Qp%`-K|#?^ocWu02h<-
za;D9^#e*RR1(Op{WGgf)SyJZw&aJ67c;b2LzD)(fb1W(B-xCel$j+5^ej(d1czgm&
z&ZKxV0z3EpvPgmk)__)b$RieHZL5`Bj86hU!SC;ky&6yTU_p4&z!^@UeB(Z_2eDfR
z?ltNw!*h3<6m@F{;<fXkh~5Rpm@g;(Uw!AEx&E7ARb>IKxC!B#qi(NM#;<=m$y$Jr
z@|tDv(Sm610@5KSi<votnLK>OqC-D<v{fCn+juc4C+%RBbEg-crUUdD<aH{M%?R4N
zAei%3jqmjY#C=;$o&Os{HL5v^>$js|;a_*#USq9&+iFVu>#9BLDLNFx38(-W6<cC!
z2teig;0zptsS$#vt%{4RR;^Q!i$4h9q8#kiaw?g(H2Q{^#7u86a&qEw2wtO5V<Y5r
zNj8BN-ck_6<)F!g{Yc;>Abt~uFly+(#sky)<H&sgZFX~mWiFgXh|I)0nLaojv<d(`
za#R^&hh#7<vOvY8`3Dk@3VHD`7Uv2cpQewYrW}IF1k4<!d+nw7-)?ZhueIX4_qa^v
z&4>+~ShDg+e0csWJ6$)-ob)gC^9Sj-VLJ^KAW^h&M?-T2mbS2jicL{+5Hpj_;^v0)
zGel7)D@5P6$i)!sR&=%f)xu1HO&0Eq<)U1YiDMz<gCUA;!#Y8i1MNb$YUz7k`Uh97
zL8L*5$zx76X?TG#j)7pib^5N)8l?O}M;%4u15m_>DpCWq(?y>&XSED@NdLPoQBE#X
zf!8Hn7baHfs9eQX6mKDyp{vQNHkA9_waI%mB-!_aIZ&BQ#(hcgX`R{dH8_!g9zK)T
zaycI1SGdelM1?Y%Q_zd8u2&LaV#!6sV<=mKLyu==;~%`VZf~%I;m1D3t)-w|GEX!y
zQ##srDSYE44u7T!+QhLAxH?Wc5ZBQsG{xO#=O|lxt#TsEH<P2UnY(^Bqx!JX1@(N8
zF@XhfQPzVX%u-Y^??pUAZ-qRLHV*wH1!s?xv_qx-y5qSi!LR-4u%2j<B-~l-@s*h+
z8>-tgOLk25jatQ}5r6wX7NbA$(Nx<RoCK)faZc7@(a|#o(b0ENmyq7D$Q*Hsy<tbA
zm>KtsdB|VOJWRnwdXHvuM2FzzK9@##bH_qemUru><@U??JSYtHX?<hU#y^%qEHETO
z-n?8GWR%a$!!u&o?REVX5keKcLQf}(KnI+QoX-D-2s4rdJ<SY_9Z$`cMNek9;~G}`
zjAl}-P^2aod^HzePSce<6_lK(ms-yXE;Spd&*0GmU4=@*2?-7HQ0r!{+}BxsC7Zq%
z=IU22=i|7!xa<8c{;e7&#<%p`-%$dOji7~t8sm2&d3CCI%Fw~ARxg8GnKgLEh#1Fo
z_47zCjwM4zxiK2MRtcWSTB&?HRte2rdbzf7>p-+3(&F=>XCKn7!ss&J(u)u6Y!95$
zKKNvbW1UEo(=66rdn8X>{rX0J*RYd(GZu?x{CW5dZYFj)VTtBiiqz!cFC#*dcV)!g
z>FG-chaQnMU35eQ?G22iG(CG0!Tua2*{dlJyuK=}ka~r1u|V~{#_=hy&?>ed(ys1<
z7WZ^BV*DAmqTyU9Z&7FJUvSU^Eb8L*wLjx5=4|-i$Ms-P@243|zu;t<J1GPqCsMs2
zSxFgMKUFqytW1A&-Q<Sgext2s>XgnnXlaN0!s!QmAHhOD1bjnku)O(*?t@CaK|~N2
z$G4(>nQnDIG#>)LIPGULOq?^9yY+?k$`sDuw|1`vsKB#FlTr|L9|ot7AH(<>%*w%T
z_*0=D!^9q|5>H~p;neH~tD)EeNbpS=tuz#?ad!rZuy&*-VJngGMTN|%AWJGr{i8P>
zo}B7)4DtwPJcnyRF?m6xf<`H+XLaV&Y7|m1cLs5Bk$i@iuf4e$BIy(3yjnM*AqB$X
z;+I-{6Q?zcJc1)bKGGQM!pgxHpMOP2nag$0%X~qS`Q7^DKqheF<2K{i3Xc#xM5Ty^
z8T4@XXv=9?kY{N30ba)&Y5x{*;t_q<UBfXs^b0Mkt$<$;7-GD?N8gXZ-ms`8gKzcs
z?AkK_to)%2OgfT;7r3iL#aCuZ!!m(*lzePR_YN;C=6P}@aZyl=78+4-!gK_gQOsRW
zNy<<7&5SUW4H|o<L#z+@)k+}XI}7*Ifu8!W{JsTAZnZtQ2e-bcj$0B+kVWS!R4Gu&
zel9$=l5*W!6Km#3$u3nuSzRM2WEdfcr-&OqVx%$NFCiV-=@~LntiO1nBhcg0J`o9I
zH)5<$%jXP#PEYvRH_>r_><><Ebqf*_vS88VyFOly8^*&@?Jj4z7uiosg#CmJE8e^{
zOukJ+vC&^TnSKv9LSvU45Gj<@Ic=1&5t!H*0n1NVeIEhb2^4)10DNAb`Dw-uv8!El
zKxh~U+G2lCq2Z(8#^QSsqt<8`SBPVwRFW{k_Q~IyB?vPiy==LNv{ynWOwIQizPdbD
zM0(w|<QpU%qKQJ>%tOq~b0X=^R^k=Vs;trvp)u{eGkFhP#mgW<`NRI@ASFV~(VYhi
z{&V(s4QuFxI93i<4NGQ4h@7DhhzG<(l^v@rA`vD#fcQ6F;^Sp=QZcC4myb>|%TH$q
z+m`ARcRxB!KRUZRDH8kH&!vg8%R-K_P1;n6Km&^NS?01pO7eao_^Klysg(v+;wY4j
zfP%f{+NQz=zHDL1^az;ZC`d(<Dm956gbtV^X=5!a0w*maC`<e<3{+7b6P10zNx%&Q
z%r9o$z-WYmFOFM#_4&&ju7z4V@MSRBazBCr5=X~TuFb?avM)NjSyWb>;!n#)VoXS}
zAQR5sU3-Ca1f#h7AM4vaX6OU5I33Ld)u`AvHTQwj(HHxEr`Mh@FI{nc><l^fBE;O@
zJ<M44P-nruy3q8X*#r-?<%vTQZ9XmRLtp+9H`>?CuI!-iO3v;iM?FO1Z+}d`mdo;#
zW-)~v9fuAjHQJ!Q?)niQfP+1Xw=NVg4bQaLN;^a+uJ^8}*%6G8-x72O8ztfewhcE(
zL%uutJr)ZF4fL(554sa~+dL`~rFLreW{5|S=Z78q1p}F77r90}EE{TXCp3%cM!7ue
zWJ&e*|N6txpAuC<UsKnY`4B4k+4eOGKUKgfCZo4ctZks?LLIEMoKqFEHpubg6~360
z4#$khOkK&Rg4c70Rp&c)aMjT0ZPnzkvvKl+<)6cspXVSn8wI6a#`ckuH?Hyx%=U2o
zOzrJ+`z4}7t6`$zVA2<5kvcGZt1RL24r@(o@L+L^SL<xalw{}EE<uNq7lsUVd!@Ia
zBVqr`$a44!II+Q#nvbT7il(QJ{66HKY5aJS9<t5lTJb<IIko+8ZQP&QTVJ5<2;({7
z78cpz0c$8`g0vmYlEB3U<e>|~kz8MgXr90AYM{Y@DFVsG{bR45>{D{_MM|ITuYkxQ
zfpcW01tH>CAEP80*{iV%bwcacQh1nmZC0s(Z~yoRTV%f55)s3dH`)+Bk!9sfi7jC7
z3rA+cKfyAHBRCCXKVbKJIKD-%eFKr$AKkx}KXvi9&AL)&2`ii|5&LM(Mpc=OPdl5`
z_gN4$8BA(qO6dFJQt#Yf*=DLBQ9R=K(-70{wYS8+DWu7lt8g^m-(g^v1CIvf_42dz
z+F+#eN^fMl(*c-$f#1V^s*3XcA+t7p*V0T6zf`+>HK|{2``U821Y2*q<*7Nya|``>
z4N3dm`Bm!_22IFCM~2XTut2o^{s<UJdh)*n@TnRy!5`#P%sH0m?Pe+BFJ0gER*MD&
z#_nrIBk#E0;5quxPsVh2VnR_htw{=km|#kUv7ZDoqD|KDIql_vpHX0DGbY{<8CD$7
zGMrP>b1x604ek!+(h`VAj4+^;s+;}x0+Y6;9~bIaIV2p^<7g}{nc`Be8Zrkb+sIAz
zbrD0)auM}6-|*4U;nDQ`5)i)hF6=7m$vDqWh3YUgQ7Qt)-27@|<U{EmjXf_2vr-`!
zuc>`|QFYMSE`WCUf*_bQlBVVHLry&YA{o0I0rhhNG|=+`*Y_QA6RGpG2B!psh83ce
z#JqwHx6<dGeF7h}4aKp1@7%n1u!Wj2tejtoxOdsnY%~M<<i>$fWOWYZAR3fR{#HRe
z`&|6o#+YxN^LpJzIFcPDKPDxM2mBs0C#?RC@5Hl_5B1Hp{Lc2kmtJi-ez_|?bZh4l
zyHL!7zM^`mvAjN%ks?O|!k8aihOWt#U~qG4s~jQ`6HBHaNUII)H3br^h?3hFK45(a
z<MTTZu~5(ZcJMi=tN<zVavquTG$+^oXQ&*s-1sM+QP5rkjT{3D<+aXkpXSTq-Zx#z
z79U*Gp3=FV-47yhEC85J=0Y2KUeD_)?$ZVjfUlUlKCJK%EK=M}cG+<ZEw<GgD_d~W
z7>LegT{688k61<zVjCn%MD1x2HlqJ)ui=lzE_;7Lp;8>VH`gLp>fsowb8D;?hX&X+
zMt0K-{=$839<sx3kt6PrLKu{#2>Y#jUUd-x+QetzCjjz8(g$~=4;EWc(|pZo{WBNb
zH+DWwVYW^g6*fPC#FUXqM1~<);<y;6w6cLUv3wvn7}WY!@CP2stc;X9>z<pga6d~8
z?}y0j+LiIV_AF=TXY%hj8EalX_?9W9^>}O987WNiu4@M&mA`f;x&A1iX&X~HOtAR%
zV31MHl?x|E?M4B;z^4qOfNMqf<5$b|c~*#k*;nsrlLp}sJeXD%!B_E!9=t_AaTCrC
z-H#jpYQY7~(zs%Xj}G-Z{nwyjIr%U#mLtSMNUfW516$C%C7;a?D|;b!8k9=~%LPr)
zbd(L9(%}pDq2Ep$i;PsUrmL%jtSXO&%H3rqyE|0hd&HDzP^!&LU8vrMc|`<c9e0wU
z(<E{GR;jRT3G0L+yO-|HoviK3Hm{JwjyA`x@T0T^xV-8eOE>+W`(o+P<g@x>b*hU5
z?tHoaSjpN&dR(b|nQx@76;ePoLHf5a`N*~<Q#lwBZR$$}_Qs0{ylTf2>64kQmnzv7
z*c86)TOX+DUW(7Nym~JwQuOdQ3PCBVONAPMuA*0%LG2B~`Z`=370^tVu+^;};-3U>
zPRl)M#u~IG=pvhXG^<p?#QH?t>HRUqr8p_(Iw$-k%zP(MsRpB&5_IC(h3L-D3w?+*
z-*jDd5yS@8kkpVYdmfkI8JdGAr-r4y@XP9_-CF;z-+gJFXp4;?=J72zqB4jpuC?3S
zVWt*5h{GSt`%5V4Le<fLnw5=b$o6@mY;9KuO<d+J*7r_Aa>|(p1_0jb7q@=@%5p1H
zm<mEi!VMFVxe}`PnbAoZvG_%)rDPrJ>Er=`+M~4FT;Emaz7k6cxC*R1uR5R}3uPJD
zr<t1?GhzE6bDVxeh)%Qd_fll=_mnOZ>!a-Q<~KR0*#5-f1yZHKao={;hx_O{OSYDH
zLA4EZ4&tf0V?|`9qQ>`V_ML+4>{B+^Kjk}jR4T3{_~}S*)U>8htdb}nj=$6>iz_TP
z+fFF4dF4Q_Y;E4c*IayF1q|nD_&E33y%J3}1>C@|+^cmyPq1-^85XZnb{;`p=1+Jd
zAVWbCrM$9I>N$5xCEsJMxuSl*m{g=L%T!-huALhl)dmfS7Wvhg_VQBfP7FTiZkAfu
zx;%WjSg4c>;Vn{Tgeod5n^7f6<u)~S)F_9sqlFxEs7cXANJeazEVq7Vu+e#|vCY@G
z?%!<GRJQ!<Rm5yu?AZ8^6=fmw<i|{bj$_h#LHuT?M;>@1MW<p<iS}CMz)xt@spuZQ
zy(J?pvCew}ZqQpA6!~Kb-0%t7Y>rd;eI`!<;2V)&c!%>c4*-Dnotf4y^LN@y3noS5
zx-a6IqWNu)6YkLA#Nd+3iBSHc;gQy$$1e%Sfb@b~2cFy|;?8Y)^!huS%47@Qy3{$;
z^Q!m0sS_DDgS&bn+JH(#@m9lz!R+xy6K@3zeH{?bM}aQ*w!fhWQ8)}b&H~Wz-;J%V
z41RAlm}Ou^ET0*-e$#->4koemS>n=uKnify%Io$Gu>}5#Q?G^R0iTdjw0@r^Mky>9
zY80r2HQFDvoR!UgD3=Q|d+qD+NB7h~6_aKQ33#+=n;B!xO*yKwE5=VWDBP4ZxEF^q
z6^-sB_$r4-(QR9QK*ZL;Y)LGBw$Y4*yyKb@3oj45BDFb+wLM>|F(Ot^GkEDoBUU#>
z;&m-Z^|ehkx8;^!JNAeZsTkDE>ook>iJ<S2eAMoCrabdxK6i6~V$EF_>Vmx{_WEMa
zXRV^S(nV*e2`Dsa=08!<ox;<Fe>{kx?gNdL-XYtfo4YU#bmVk_ZfbFaB>0%BU{>&{
z3ADf09s%In(n1atZ3>cnejPFZ2JpDnJ$e6+Z*U)~8uB-3<7@Be?ly+vn47_jNQfOW
z+qN?0;#-PNX}m-jG$<bAR#JxK_X+Ojuav|PHY3iq{}y^;;~2rUs43daIE-7lD|fqz
zYkwM_Z$=Ip2i>q--rwwdQi&t@!szc%gnlsoT}EOkcF_aS#Nj7*VQ2T9R87Pi-p9M+
zf<KrZOP90+4C`k%jv<TOj~w)6S=Qj1k5uB1OANI=y~|M4Gdmx7EeS1N`K43&SUkf|
zDAfgMv%;4d^;@!j$?~CU52aJc-a=z14%H?ZI)o?D@aSE5-%>2S8;*~*Hhq+!GBjY5
zg=P-U=9fZ%D!ug(6#LfK3vzYXQrs5qwl={!?A-QjYRL{KmL}@aQ41c&0%4_?Sm|pG
z)am+7R|5Cr-=gml{U!D|yySn;TYKfz&PTiOa0DH_>1^*QgDIWfMt`+T8Q0+{AQA~<
zoe<y|Q^p_S5pUVPJel((Ab$Wm7H<|6rkvcu+O|E#!#wQ{?CV?K2a|HNv)KSGMo^(U
zdsZp_pU$b9XY7z`JT*KBxRk1}$mu-{2S;r3tEquEy9anFTDNFZXg*pIL6IWi+n-5C
zkPDqP*b(TufT5ao3F_zLz@po4{c(KK8bkq>xdaSNJ5*AEb>3@W{Msp~x6jlPFF#KC
zr3@)=PjJuBYo%z?^xM-ei|s3C1x?14;Z_IR*YAD%N>$4S@Iu+*$4Sk5p<DKXHys=&
zhJ*YxL<TWD1V|yc6Q8dDZtDzpASgFS|0j*gLT(53)yEK)bHUK4G0|?Ei~#$BN`>bB
zKcH6PAtp<~Cf&H`Kyu#sb|3mKO{w(tE`k*&C5`Gb>H$*p4%O>=FyT;CN7FOBYVhu)
zUZ!_xj>Ff_JvxU23g@i_0zx?AU%ue^2WwV7E9T2yt<>>wD!KXWf9G(Pcb(uXlJ$w>
z^)ic57{rz&*;R^bj}BfolQS56avlHYl?Z)x6HbzTF?SO*yrOYJfGcq>+}-U%tljA?
zQ`4fusng$n@;a;7>lmv!Ji|<lYN}YC(b<^+8;B4b0@Ur##OsWK3|7Ykw=zIrx4p8O
z`P}ovjO<ZSX{ORUW77coSch>RJKcPE8iGfK6+Em9h#@^~^s2L-$(gG(AITZ5GLb7(
zduxzy78brYU<*hK+%%@~*iFJ%RPw@}PGu@mlf`S%C%Kva0sR3zI@SqV!`nyR4?cc9
z0w$%)x#dv1$O+YP!d+}_6|cH^ZM<$rcX~?0=`Cm<_nD16lT1qd*E*`4NOp(m#IcUO
zcht$RN5>oS#@!LryzZLkJuGD~Ul7mK;^Bs5r#_<s`Nola@JGRS!;I(b8Rv_fIY{nV
z4k9ajVUZ8GUHLzagvSvNHy-eAtd0)8xCMgnw@!Y_o!BmuEZn?5P?DQQET@?hIpz;;
zUAC?*8`ubD){l3$$7j%^=Q1c>?@X-GTAiuygogS~mrvEUyH1%9x@fUlhcfwaImNzH
zUOe?H^v#d`Yv;WaNs^~h>{%9u+r#JGkxNO4<U`NgBfWfG<$Y(}E)e`s9{cHFj*^K^
zeUAG?CzpV5F281ba}u``sg~)GOKlI<Z%kC%ktB`vpeRf(q*^*xB~BfctSUvHFQ|EG
zBJ`4jJ=$Z{vqiVMx!6qt-34vihOUH4r>x(~)-cBbCsU}pDH>1y4G-hpeM`}6J?al>
zd@olsB-Q)2JvAj1?I5ih3@wD^aIpw=Y8rxPJ+v_>cDhR^5o4%#SBx+2!`CRCGHyDd
zar2U~yi`oN_+P7fJ2)cSX_;HiOICuPr)Aae$Bs!y6<6>|(i_zUz06p4_oWLWeMcOr
zT`bk{g=YT6+b+@%hL2+yx%js)XjLU<TYfjJqSB+>;a_qC+>@{Xx&QhWeBzNw45d2q
z2$3+8JvlhU-#Xt4i7gkmdiC#HL|P~Q{1(uA6Brrsa|~CKLnPCP>{SDIo8&C2C>v*;
zC@R5sBoj`@!tRrNuV7f*u=voZBDi?3n&4JV+~yyIuVu|I_MAF={3^K1oi_gvP*ELL
zCp$Pey#M^=@=F9P98nMuW6<0l$qj`y16xLaB9mn?mQ&_IWxhpS^37t}F}U)HdoA12
z?vJu=IQ|6#eFWS!Z(lOcmv*^+=_|x9Nn|1~CGBMGuMhI+P4LK6T&o<G!P>Mx+<8m8
zWY8i0WVRv)6435-chSvwWeUrZ_M*OKHCc-m{~__XKUAseX{d=ap(OLp*?vvVs}yO=
zZbj(f_Uxr*8&ly$usLG1mwOJi^~<0HQGy3>^!}rISu>f*Gs*{lhJ@#fJZPcmEPkwQ
zi?Z_hfv7l5<1=cH_=F;~DFOSCD^*tv_J>c7OI($2x|Zbn(a;0fD3c}0%3h$;^i;D-
zY0+S?XBr#2!2!zvZ+q%i%eW=H*7Xw3Kmxc!1V+4w;Xvt+-QT7JtG$JM0FlHeO+83~
zinleeF&#J!b(Dr><4*?jL=iouX6D7~?kf*k^q4Oa=txGOoR=^6B{}OU2Z62u4;?{c
z8WV#()_mrTJ40~t64_UAdHXwew*fRQ15Oo<_lt?Ip1UT*(RQ6aao$Uc)TC*Eo{@!i
z<9nK@7O%3Yi;v&my#>yzZ*-+b(gikL#;*@1F{Y>jr$9c5;C%&4c*VXU%LH0j>gQQV
z0hJ`e91(E<;#MC0^o{|zAsgV&fZlXX=Qq--#++_G#4j+l^66n#{<<B$gwyjBOS;|@
z8RE4aEgewR>aI{o9wzBhy?dkei@6agsg{b{ddgv78LU}x)hSHE&wbL4pn}F1;C_mh
z`<w@wRqs=5^wCr~JZRxYRLox>-%Omh3K8yIj+@2js=`osbuOb-O?|nKe8&CYFGo*Y
zdy<GZ`pf}Ssi-^rk^eX}59S=`irvzzwfv#e!ySV?EIaD2!EN2120O->?DKBEwr~AW
z!F7U!0EY~IO`P1kpY7E$FGr}?6<vHWP(m_x(gL4W`YMez@7_*S=C4)$+(F7|F;(+G
z%_a@Z(vq$vF8{GbPO3?wyC?J(G;e}$M7nJLcjM@f$5Kjf<lxmyLj(=dNIa618GZWS
zQ3Md=e>^~hC>u8jIN%64L@<SpDw$QEpU>+;Cw-R=ba84tS)#puwr%Aw;Ir_ZL4l0V
z?I3;XP-InS7SlT~((V6f0R$q6{3D1&5Zb3PA2UD0ip@cLGpz530CV`D|7VW+`%dtC
zv=>IHE^fLF7`WYrEpYEvFlZ-Xnv5M!yoY<ARm9uJMuA{HJUDca+gD+L%af>Cn9vX<
zN;w>rRxy{4hIonmm}TvuAUp+kcV7XsEvE4BJgGV1N;$us`Tg<&xXF71ICVf-mj&al
z={n{P;Fq?wit^{!Ap+5wZ)t09YA*PLC9_GeA4OR++m>8#)=L+ErhxH=PxG|H3i078
zt*MQl(Ah;A^+UR=$RhW0NM3Mf{SIi?;w80M%J%>x^f3l94g%2Y96_H#*aPleJm>mD
zeCVc8&P1{NGSS^p;S>A&58i!6`()_6*B=)K3%eT=rw#J}R;{?ZCVIfV)}9!Ng4Up&
z<y}Y53(6-`93`YTR^@iN0puDGfV4mW=Hd~OT7eawc-RqnYEm!!;d9f+*QvVECXCMs
z!jPUb?;!)@jhYhd%&)<<H)<C7LR3rVNA@#}X9!Iqws%rE2Q#}&*&vz?KqB(}!9Hb`
z(>lt2?eUTmJ)P4ZYsXI>-ghlF(XI81OBX6<*9H35q~u2zTaN<*IzjzE1m9tZXzH;L
zr%7S*_u{-76<3_!%O2QWCtfAKw?JbLL6zO%oMLX@P}bj|V4lPvb{7vwHYCQZkQo=G
zXQehcejD~m=2G}N4E#&(@RuJYld1~kJJpV97oUT>6)SpP7ee=i7Y&E3wbN>@Gi|G0
z*XPZEl;!&1tWn;=9e&(xl+_|v!_2#`&s1QMdLi%fi-6~lPB~P=ApzB!6#~s~33#Wc
z7j`CpcT3L@dXJ{tb2?EId&fWntLTgGj{aJ3IaSPGe5TRVj>HZVJ|ey`UH9ClP;Z8i
zrz|)nY}n_X%=gC}OR<c9)~76ib^d=Dz{tGnx7MlUQ)Gmh0`44cUdD4vGBWS^P5?bd
zl<vRW@*&5>=4j}x!@cooj=GMFxo~Cuyz-f_W})y)PTB9#S-F=!=<MOr^6~V!rsmJr
zGx~kjT!}&kWO|@@noTTedB|9+uY{$?tw8W{I4hrRKl7pDc7a5|GH-`Yq}8NEt8Tq|
zaEA(*P;LRrfJK8c)lr<Y9^tRU_WPl*vp3K0+^yD3Nd@tq(O2lT{@{2wiw@TB+Wwz|
zNv!>Ceh;e*vrClj2-xLgig2)o$D^8MVoZ5#mUft|kB_B=U{e~gpFL$xPZ06($?&(J
zD>%?{J75kH13fIl9K7HmCSu0+eNnau?tKNsQ`0i1Uamz2Y<XtA?7G-8R&7Y@-%I#`
zeEhVr2{b6jvWqrrCsjb>SdK9=X)O*qWtqpJdCnXmg%&<oB9An!{Jwn&NXgI8uf%{P
zUvJz`0GwcM7^jpX;@7HHs>9W9g&@@EysZLA=ZeSG4`&Ey7O5A1-{Ptv`I@tg{$ztA
zSs)0OIXd6)$#~(@V|N?IJwu-k`9gFNARb|pI@c8N`AZ9c%$yraS3pVe02%5j)ME2h
zbK=L{%VS_vJ5D0*n`on@&s1Bs=?W5eckw8zOXu_bNQSlh1~<uT)aBtUc6D?+MaPq>
z<HNGq!W2gPGv7PRNaO&~!*Ww|5c#U9M3}#sEDR)UkIviUL5oFl;Z=4cTD8${k?!b|
z{~^p_Vb3VH_Y`}53)Z&7rL^eTmNrLZO0zM+YB#8A*RtgL(ANra)_1_mo@DBsaLb5J
zz*}Yd5*^vjrL*heSYfI#UqT~XCc_KNTc?>-mfseN<OK>>`Yyg_=@L|_Oc<v~p#Drh
z-|i<N`x~g*11Imxt$=lH>Iu;OS3Gh3Kihx-bqRW5;l<6|m6eqz?N4DCvR4J5_4uSX
zo;6crIq?w%n{4eg-67_pgSuVm-?~PNR$O$n6-*y8&0^lxqRV|ip!;n1U*W#+MeR`G
z4@1Snk6F}MMAwp8<f!#p;kw>kqXLU#MJ>M~Kv`pFJ4^w_4lHxRw;#)=0$g+>f?Uk-
zMzJVg2iE(&115XiijBh9u?5@HnT}Z)LZ)=UG8S})3CTrtmcgjF-{2-Tp(EavnqoPg
z9<j4x1mih9?C}k+S7FGZ9Gmr?xTsV#V1T%1gj$ld!;VYBl_}9Q7;z~QU$18++(-zJ
zg}PkT$dur(GX0*2#N7Ndn8m2X4Eru;sgTN7<-4fTues&AeROONlheNq!3)H5JT5Lk
zpBxbj=(_;k9{d&KguHgpOX;i+i~LJa(P17`*EKXu{W~g7EFkOOs#5uXnr`V1CmCF7
z4>UR581{VqX(UJTCvYIEN~5F6BSvgT@lq^ed$+r7>}$z0r((6jIP0hylK$5#$p5aF
zqJDAF!P5C=CL0<S7h=c`6h~``-xywEG(FtIHu_?9Ec&#Bx^zLIJ2mF%l&>zmJq^W&
zGRwIKx1=)8*Kw0r-db?vUACxS_?>3FT6j$RM=QV2Z{EEt=*%A&y%ZV0jn-sc<+oj&
zuHCU}Z(;=%euAv4^?_*V8M~5&<f$!1=a1yy;u|hyRC}FEU0e1P6oKKp=SqU73Waza
zN44KAp(%&X=Jk^t<-@)adD2)NJwAL!*-yN7u<`9&b&)S7yOv$SNEj>l@>J9vDB%Dn
zjJUb+Co0ZqE|UwWMKUj+KXESM2~bc=I4~P(L^21Z&s~)8J%u}x_`io`h#~$OYo)39
zAS1lPqd3*Wft0M=-4&F~Ma3GYy&z4NM1Sp}S=lkV68KPNmESrADJNRtkLi+X>?z-_
zm5h*!BQ#vnM!+Sc3(}G3<ANm+(3t+8S73@GW~%!1qlKwc5|U6Sf6ug9mr}zLFxY8#
z++q%=Y*cqU=NTwSS`V3Q^MaDkostG3{z$k0r|~o?V+cRQ7!RbVkLJl0cl}a8s2qeu
zCu1nUV@MV~#?%>YwE#oq8Yn2FU%tR8;VU23HX@+*Uqa)3niC^kE*{we;j@YV18NUW
ztL}NSb_~IPVe{4^<xQK}X#u1!fg2s+>!$dz)C4eAxv-smD*^Xbf@l;gS@nMhB{_so
znS~6!q=&T6>GdW>X3%4X%8z9CFJ?~9xy(;VX(X2O0dE8(2R2`RY_lA8(O}8P{u}Vs
ztkoZb9W?=U=m(F^?q5CnqQoT4X3qF=i!wJ8fzo^GYtQy?FEH7Cia6upt+%q2Ot(Rt
zuQHcRf`({mx~QGC;7MtN!A_6cC*uc?nZ%zX23Uasop2Usw0OkxuKX+wQtUJhp`Gw<
z|Jq-6;u=rWK0^e>T8|nM;x=>dS3v0-C;R~oyJ0X!{`a22|9<B0oIjzIRhq#WWIDx>
ztZJ*u`ah4Hx}g}!$#_l|<$iezm6rTb+$g)@(6f>$=vN9hOsoY>`R{u?o3eW<<*UH#
zS}PvKYK;0^|1#Tn$TeP*5_n2XlsCr)D}s#&zyY9Qx*J}>2EaJ2y@}Z-ysfCy&%-l1
znGbuOQ)!g>@P$)AD2|{DwWG_`S}J4YHJP;$b{Y}k#sEMG)9eeAt$WPcP)#j+=3S|e
zt>p{y&HqsibLcdfqhO8*v0<iKM_&D{lfXkAt^_1MzJLCVISAnrjzN;49zg#K{@{UF
zb@gQ8&-nj<7il=zm}2x(9vnU1PIcT~k$kHIq-gSliaw}vj|BYj-F<NzGYJe5V@a_X
zXSV1Jsi@JNX>z<<!f(9nmqx-Ifn2ujM(D=hbve--Q42k~Dir>Dxa4^$j`aaQjBwj_
zXvuYV+H}EWa{ybX=C=oA`%u*^&*m$TS}iZFXuFw&Hh14oRYu=LCMCviTG8YmHfcMq
zDx~ObR<JiYn9ZE7Zx{<1?qFFnN4FkD%s~Y-O5%@)y>^VwQR<U&U?ThckDVbD?^|re
z)4GN!nd*hK`dkh={5DP00;3SvS@>v7mt*>?toN+@9pu2<b=Gmrz#+sNc-9NwmEv9@
zpET)Th&B6H9k0T+@mHm~eFXrgxt-wW{@CSTJ8dpK82~s9|9?(12{rO20G!4e4{(}D
zS7VkZr=gxlDyblOVc4<ojeFS7aksU~<k<2WQlzG$iO$}$a)Elem#B;_fc^oK4kA7j
zay@A5*-LZdeOzSnQ>T|9?jYpMlM2iTz^buDR-&}2(cYoQp)F3eNtkXCuKMA;+Q-(L
zmb{M{I){%nF43)*DYJ>|{$`Xs@cgOo?PMG>M@Qy)f*wHSF1(x?(s@_y6%ogr1d;=i
zO;N4M&dzG<!?tp^|5Lif(Vw*cVAcqX6C2%^bMC56QTsitgx6)`yItmZm?sAtHiXW6
zC+Z^C8k^M5FV^LfOl-?9*NaoKcJ8gOo#din1zR?&S-?vM%%xPa4Z=P><ax(iwq^Df
z(*KLSw~DGd>e@#UX_Qn-Iz>vPq`On3JESE9LAnG1De3N(?rsF>?(XjH|JvaDo$uzH
zn{#u{81EPihp_ivd#<(Sn(@qM&7a$yB49zi&I-aGCKbPiO`EAR?@8qBA1NSZ$Tbq&
zxmw;3j~R1Kc5Za7b(OLf2$+=INT7B(jb*4h>^b~s{T=C*5lU%=Ww!;cJd@12nO4#|
zi1SN~FW^f~Zc$I!M-1FCY6@!mDlf)Bq{bi3B+EeL!gP<w35m%jXch<Ht#o=*p7)!E
zK_R9`yF=J|!aucf=fpr7E(cwPjbd)E6!(yekn4EYC!=BNq_uUeLhvs5Ids!!-?ysw
zt5oSVtH=QhC49j=6>g#b)@CdV?J3-kO4y3~s4cfpZZ|>-V~t+`Alfc6KvO1``P7*}
z5LJayXC=b6wS$2<lrBr7db}12uGM~AVEi7Q#z(kW;asv!_!)<gaqWAxMK=Q)7cnjI
zOOPn|#c{V$>_iu`R5V5(h4k!m%Ivq)Qku=hAFcZhJDR_+&u?=2BW84eqkg5}qAqiH
zR?7p^ABz7be(DLu)J<+5Ng19HCOHD-=oH)7^hF148xH&z7c~l%+lKw*GZ6_$ILvM4
zOJDirIS+~JN#R)#-wVO6P@1?ItB-~rSs3z(oXCICBSZo8+*W3>x`jA}L%mLSxpGr&
zqc-4BY8lU^R9(gs<hF1-BAbtHW555z<r%4RVy#7pw%vWUFuSjG77D?rf#7=%a8oC1
z(RQj++;(ok!^#xR>DOGsl&lS_cK_}H8YBS8TIq>+YUT|8a7^Alo*|EVs=8RnW*9SE
zpa7=|38Vi75lNB0gJy3xc#4HBcbjz-7{_<nw&e_vePjl8E1ezM7Sw!UB}HVS7_u0`
z-TH0}W(w+Q_ZSqVv`jyU{)#Gu(S;(UP>*uXfrVPe^Lj;dXU$H1Kenx<+XQlr)c#l8
z>VNCz^vfo6XPlfuu2*^+^6Ldmao2m3a<kk>lL8YrkFJ{8?H#$aG6Jv?d&XOk4O%$4
zSDOlm$~&I-HD717(Y#Ez*5FnQDlV}*({re4m$W!f<aLr!dzq8U&XP1Dy4DtqW#58B
z4KmUr>sk>DlFk5|3oqr)l`7Yx>2JK%D1y%%_#JQ!y1s-3WRH`mlEASESiLH&b3~Kq
z6FczA8CZD^sVOf1y7R*Majv_+mH=HyM2=t({MzHUV@GDm*Bze{y&DZ`NI1>Zr0MJD
zmJ)*Hb0ST6e{!?*6p4{gs-@h_dk{f+G9i%xek}eDuILV!c@GRL2)7X7M0)WQ{3=7m
z=d^gj=6(vI9b(Xa>H)^M7{_9C8J*9%4=0ld=QpNDfbk%=mv2H|0<hkc4VlkTjl0T{
ztY=5B-*=z#Ur#MD$ws0Joq7d&K<FF(+FIBf=UUOJlrQ-&2m}hJL%HB&qcFGJXr+R$
z1JO(xEIfU>E$8pk1)^rt*<%E45&)`HNESwYE!|*YOYnWYH^ik8q2MW%eH(d%oM`yr
z%3@lmx}TayMIgH+f&`jpQJ=Kk>X&d<L4nuo$1LrvLh0I#x8m<UHGHt1wJTM3YUzgC
z9&yJFyM-WeJIdP=uhHSh^wJ2@>akSZKF`MYml^#oJV>R^nFowV>g$+=L6Ziexgrr2
zh2t+4VeXAE4~&@`7eRZ?_rO1Th0!0QusoUam%}u0h4@LdxJL`GcRcG?t>%El$f(S(
z<O?If2gzML_1+<aCdx%?PmxenWptK1lDgq{;kNvsTkYtOI**NPUy-PE_cFb7O!Z|>
znwUO*doJE&`FIpu8%gWl27rUmf)J(Z6A!g3{5RyCpbut~m#)`Nhu1vFd^a;tj<9Q5
z&06SF*YQ7^F_Y;o*?3&<ACda3qThW8Yh^O*O{IGN3Ok;|SxWkXC+8becG0;?Y!eeG
z-|-L_z;{DMk2{^!1znRbFg!YYH}&u1!r@Tf(Y{OTt!ZBFFGa{C<b*UC*^97jrLn?b
zQ97EY`lrwS_D?t399}s#BkdAMkhFVyy`m;xP>2(C3TeUqQRy8HgcbYpuJkPkpAeXI
z4EJ%MRj}<7)eLpUx1<XOZbeqnS=NdrGFCH`Y&8Zo8dQt5+H>SCMYd|brUt#$-&wYi
zh`TGf9A-$jA1#qD{YEpM1A9yLcRg%v^R|b*MB--4<c*r-n{kKFU#T@kMMH8@A{kyE
zN;p*7%#e751LLA4=PNGWeL&caDCb*@&rXEn4U_j4?}IPJo1Z-(*fY+IX880TZj38M
zdjGu;YmYYN3q7t*D_*&OMyP?hjt3vI^867FKm5UDs{X#yrb2gz?}?_=TrCW9+c4GV
z*3BCYU<pUPx91BYc%L4u-@8GnkI{xLOR@$$WQOUqh{ePWx}KHb_r-*j#S)6yy$rPF
zAKy7ih#$R-=MO!sMJgz7eC+`%Bm4$V=0*7yZBrf(m^=y{@4=!9kv<1Ja7>5C><Xyi
z$985oXyd3JqqTlEkLwU(J|}d<EoVasppxm~`1@oYFwiPY?00|Kt*`!m5l`9_$;seS
z-gBynp04SD&|=C<qyEO-PHmL?eaXr~vk{Dx8rtsOw04<drY{2({C@?cFS9<+kr*xB
zS7Aaw2KXe#uMN>%q1XFwvzBBNf8{w?O8`8~=ib8^8d2y0_IRP^_mk8{hwe=*^(XJP
zfD-?8hQ9z{4ol;`e*<cEq`0N?<mcR{*`MSRURB+?HCn`_*D`+RQdvW+lnJT^PZ+*Q
zs6i=?V$iC}ZiPw?3l(3D&}iu?Ppa69bi2P2J({!ip8@6A5_NCf1Eav``#WwLzyu_v
zLLnSGj7E-lz4Q{3qj)`Qgbc2q>!j6{z2qF58<Ro(PUB4(zbi<J&x9_?mEZ`2pFeJH
z@!{Q-I;fgAalbZUxN)hCw;|Z|wwa1$d5-Gg;L_gpI|mtNu|usFC!1UM#vzoXZKWa2
zeUB}<QC=O91rm+caI=$8-gAHZYdL?Lo-}JaDb)FaxNutwiIM)r48mj39F4|;RZ0R(
zsD~IPaCJg8LB&_jD0zQOyRQnAeX>k9{!nQ<S5`Q4eEz*n>Wj2G@2HvCb{Ym)VOz^+
z##6mS)T-77Hkv-qeK8PF-MwjhsR7ew6*j^$2Cgs~wbs0Ui${K}PmNZ&K;sKRBR`#|
z!}&65C#c<)6)}E}vfLjAP-W+{ms&gXy!Upq{()TI=2BbGzD9vat(M??M%)h#k|M{9
z=@#9%Yy}wEg;IR8qn^KDaY%K9Evy>U!anAcLVs`!zB%LXC%Xo;U?|%Lw8*a|*pHy6
zkVRQJX$VN6qc27qR<@7MH+fe>erBZaCEK~Ke+CTxkJhuUrJptH)coJ)8=+XQhQy;C
zub90&O=po(pwVoMRWKQDvSskb!A7$1@_m4{q<r!GCn4^hFCCJ0lWKOTxbOfgU2fnq
z-i@QuI>|{?t?krKOT;@~zCmFa$gNW2l^1s%gGHnTuy15$5eo=4_NsnHgFy!I2_yoJ
zV#XcnoRj8U1xVCMK=BVUj%}_0r-1%{Y4NEAAnQapNnl0r%Em`>Yg<_R(V*ElBQk;@
zR2Hs&P>udI!zc*<#hN>aA|-#@{E&lzuo<xQmkd!yi&F#F9bDnwwB3N18(h3pGkRJ$
zG3`G?GdWsH!gpJ@tImG7jlPRGV!H_=9!FBYzFbAr`shaVE;7zOuuLJ}d*e(YgNsL~
zCLjWJMi(Pqg>k)b^m$eoM%=)RHz@3|9d*FTB<S1eYC{NMdVwFK7P<So7n?Y%A#S;I
zGBUo@XG5iGZI@p+6bO#;#-o@tK3D@9*6fSA0olic>&gnpKn8K*wGa)QNZ>Dv41Q`h
zIAu*uQ^{C#CO1iy-aH^<J$qje<~lkT^1aqA91uvuB8TB6t^(Np`e^>>o?@3wZ=+ew
z5b4J%@Ch^>_|TwPtB8z7mw6f$U%4{8j<)x?rMCXl`0v7UoFUys97v+S%|n|ynI-X&
z-R+F`<@n_XexojgbBCkKc?0{|Gqnl8z~?jG)U3{dR4S?)NA%_0jN!$@==1xoM?&=9
zrHjO4iT-TVkTE?o-E_=c7Xh_~_=~}#)e~<HDRq!zAm~ZH+C0kWE-@H<)0?7m=o8R-
z^A+LIF8b6hDCdehEjFSFIxXfk2J)FoI-w)bX-Puw!HhXqi7iM6k67M*u4XhdJVy?l
zVDgHF_>#gxL$P3_=P<V+S!Q*wA@q)KlvHh88WrhvL)AH7-zZ8hBz!c|QFl1;Hae+)
zp%?1j|B;&X!G?#WW-dxUq{_UYu<^gsLN_zrXL&3H(;tyZt1Rtj?T$bSs(0q9I^gE%
z40ysXsPEG;csRCp`hWk@`SR!Yrg_y`tMGh$(C<^dflmJ4IaouqTIG4$CtaOHouaqx
zo453m?c0+E1!1Pk+psjB)UNq?1jEcfq`**C?&V^S?|@`NIlJuiII=T9M)0fgI|NF<
z3_kB@NUaFej0VgG7OK(wqQ#&dl`#JFZdmJr0UYS}+m>X%riQd&1pLQC-8nt}kY#p;
zeqb7DE57gao%KJ3d3R2OG0)_MBbB~&e~Ld_$bS!q+2=^3ODTQzH8dXWY5#rY+&<-U
z>^a(sk1=#~*$*KS$}eYJVekIILuRk$gdugE3NnaO)6CFM=fhX=5BYIYiq`-)CNi>5
zJYP7yM%m{4&NvjD?ha|+$x4H$XE~E1yg7jH0<G-U|KeUipCEN-l0$KI)h<vYD=XJl
ziPgDG!)f;G#fr=L7k&MIQPtH8Jpj2uh~0&)+2yfc=n6`|FbH_tx}Hhbb6Ooc&o;m7
zi{X9xQg5vHQVK$3Fa2v{^rZYu+P3;WrUQm#GCE$mDTg)qtVMMn(4p@Ed)B-3jukqG
zZdaJN%$^Rh3KN7&0eM#o`UKGS2Nke;RAG}~Z4ekXh;%@^2HAQNYz7U5lrA+0)3gn3
zkkR2;*x!IpQew6HIieYxRf$IlKjGfW3&Zk3AW$Sf4L)#RDYyO-ogE4tCR&VG<#ExP
zD=_vS!tXdpnlzFAn^E~0@Esaql&UE@aAQic_d;v!RKBlU?OeBz=nP#K{$3HR*4nDW
zF4oM&%#7Q1A-LJO<>(E4l0pTmLq<(v2yO*zvKYTJ>?S?uzRdpOzkPhPDmi5X3;$N~
zdFiM#L{*MWxnpq&d=R*-15O$hasX6})V-|O^;gh9qU&^X2q)`ellakrqBwRcDgxJu
z4QQV6r<Lep<|Yi#DSynwn-ZD;N*gZgBRy&+hcaI;cEC9^4<LR3@el*5Qo3baI@eC+
z@A*^My+-w^zE||^Ld5_WbxLv;iVX}_HZ#%s&>~5|?y!*^4%m-cn-fB!PxzRjKy|+Y
zp-?jD(NS3EH$7C)(I_4wI#1HApPeA0Gb>3(!tZIC@_h0}W3tUb&D{fSnMOAu8r+f*
z+v7wMTmOTY^4ChUHngx?n@XlTje&>87OrmH)=XYNDr((yJ1BTung>N^49~FeQUbtn
zB7`dEhRYo^2K~x$CwOe_hY(-zag?2DtdlY5nCT4h0UB+)t2Hkmfv8GAbKpan;XBl>
zHv~{ed++fdPZP+k>_V)1ymG>c2@sy(SB)XM<<nbSQ+-4v2ZkD)PJSp^JKbdMXnIzy
zLyQtH@HiBQjA}kyIkSAcQg%~DxvxqR2$-j9K@I~)2*49z8Z-x7ki1w35dv_8C;Qgu
zS?kbk#nNU&!!s2b?mc@xc~=~izoUxN>a^NCM3FxDm%d=Qo`xkvkaq{$#;4s@-W=TM
z3$Peb^&^hUXma@`K3eg-x6;9Uo#AWk9<f7$9K8ISHXH{{E;s%L%~D~fdA`pAU9oIg
zkVJAB!<ED90Yvc@)H=u=O|M7L@MWAsn?eTla1%(<&ew&AKU1m<amGdq)pwt)tFJKh
z3N)%yxm6*7a;slP?=LMSY0ZTwG@G>x>P>~;ks1wO#eEpnIn9dkCoZ3QN3Y;{uHRBV
zOn3R4%>j58+!Kr<HZa{E^@u{2o9~OT>dm7<L&0=^^tD5=Tq7fH*R9al$0?(8@E&Kz
zb686yy47YXDBQm?`KnKm>0GU9Jr^rmL!V%bZ|6b@Zh4;J8m1WZg(x}}#%$J54Fu|V
z;~XCcyjxU>qFAN?z(Sm?4QKnmJ-+bY{lE<{e`H3(@G@*eyuRtoM;ttbLdU*+{&M5#
zko6Q)Bhxv!ZL!u*)r*<;B#hF9gZgso_~=h~@C}BZ)jj~vU@qsbbV&IBGUXL_4tyeR
zeS!3N+Ea|F*Z3hOk`*KdgGKx-`Xva!M0OfVm&rqJA;P)lk}4|1uXM}TXtCrRX+5o6
z-<d9v{4w%5EtmyF3Xp2E?Oj-Nm;opgI5y6Dey~2~{#Mne+lnSGubrg!V^?j(&Z6U|
z?YjmCS(m)abRPNawpNSv+u4{&gJ=|ftGF$$KfN*tHor#dvHKFDMobFFzvOTiPT#Il
zIy8Rq?2vtHaMTlZJ;?uH8%a6l`sE@*&z6`{>Jdh#S7w|aOCS!$2*kASTFA&oeJ!a0
zP=6f5Na<<Nm8dun#U~-<dwG45G<2B5wzA~y@A<a11C{Vl-S^UBR|$7AIjwKLDhkl>
zo82HeGpH9)!aV;&`+&HkKn}TGwh?Wl$5XT7iQj(X)X^R+E2h4<<5`k4r#@v-@QDr@
zygMU{`ZfC2C+pf2Nl2(zY-YacAI5PI*NgwI0-$m=5`Hgo4by2GO%Gwd#Om)0Rd$Dl
zh7q8Cp!|lbK_=sRYnPN>OH$8FI4iyPMo9&kT!qV~dr7gxaf(XbQVbpv*Bpu!_7@t`
zhfZ>fY+X;$Fe0o9T`iwuhKf5(-!P2cYn=0ag_Hxb5#Z$WUG%HdHe3p9vdO&Ck}BP3
z*go~Wb{TVdog+{6^-2T#3t8qr)jBcTRRr9tDdfT^Pk3lbkkY+<H^&DJo{Nf<cTcmw
z<=SLBu1N*%F_U^H(U4SfD~R3N$x>|Se~xaHQj&_M`J+uwRcPy?KA#kg_B||2?K~pi
zUwM1`Ko-EqwpEPFP+W!!?8~_FxVJ&zGD279yek`hhG>9444WrNG9g&!XPQIThtmZy
zCO@Qq<JtUKXj1E8F$sN%WT*4Iw1Ze~XZ|q36~*v-Gs`_-K>y9~$1Z>C@rR*wwSQMJ
z2-58|iiu~mgCu*U`WTD-Wci85Kq!SW>m@*Sx&3>G$>7+C>6uzm6L_pO=G_2pzw#sQ
zU3W|x)yfR7(P?o9O;m`Qi~6>?_I<n@)7+A_qa^+75LoJS0@#%#Cbv#ttR=&Wxgi9Y
zYK!T5$Fu-9DlOs{Bq>1vG%e)RiWWB_*G}YbXLoI<f2_?-xMaDE27ntdE#pqj`W~QY
zg%E&UNclt?rVpQ!T0izHW*)9Sx$oPY@N@seTEY&?hI14MjUw?`WH?&sw_%y)nn;l&
zW#8u0?)JVR;J=+V(e-~V7!^IDJf7whbD(kpjw=b|xQZ+47vO;7T1_ckBGBHXb}U}6
z=vzp7rKW!yiLE{od1P5%5bW|8+=%`Rcaz^jm9}bTd-vn@yWDN7$#T=QBb*QPWXhNv
zypJ?ijblQ_zD3M&db~T2pMUE3{2AWE^~Zu`8e@#TRxpXoM*D6SymKwQc1lPFh!jDZ
z%3YzHwI~s1E87fJE$CG;j-Q+S4fOoA)J_;>GQ^XVewUfNmaRYZ{37O-cya0r;g~wt
z8v5UyTY64>$BYj2DYt0aUbL32nQbL7LVL*htlF>5M?xV>XpemnV+}GCYNHNL@g^y#
zxvktx`d@{Hu3KlEgpJ*$^YQ8u4N4yp^EHK?Iv04Pemmah7dXBfO{OD?DIJ=g?H-TP
zhez4m4INF28h5H&u7MO=B&=d~SJA&C-zi{Laqwt<bhy+W8gHCrlOg=*f=h{k5c%lo
zi}Ks99|%^I5F@~vW7_lb`%^Q<@$^VVghUcxK_dO{6UGVO0$}}QdlEtR>{=Fnm3I}<
z=D7YFB~7Be;njF2?}`@^qLsMAdQ&d!9HR%N7vuZ8Zr#pkI{#0~THI@IO%omX`2;TB
zjl4|xYyBFXGH086f87<_dw^h~zV+WGf8z~msR+mR7r%Nkg<M;X(2cnZgGRV>GPrEL
zHSpooi7R2Hp%lt3O4KQ><R<n(@MqerkQ3#L4k0_?@Eg>cg8GA<bPIF{U~$Ahy3Ss0
zDXd^nIkE#F)8}z;w;1BkCqf@SA3ztE=uHa24w@jjFf_x<{R1B;__3RmHZzD(T)=x&
zbhCY^$_FD!1UBeSZixkwiSRu-73Y4)E5srY$Wsp{V)u8m2g@`IA~J;VVDlKpvAQv;
zasff7!3inS3@hS;d|XHJ@$(g*9kG5s5+s3^Bh)F>wGu(&s~r1a6h5FDW{-(PLH*Mi
zh@?;45C*g^1pUS#DLm98_w9fil6`vMK#qwox<W-R9Q@e7sk-+*+P{gLKc5bPpr1wE
z&szSi&@Xqo$InlcY^uJ7EF?la1cf}1KSXgu04qr(QTGEPS;6BbRh^+&fl|CX%nb=Z
zu2Ctp1R;y?SjlNU4}iR)_W)yx$W0dD;j5JXn<^{S174|nj2-p|2wFq{_V?ckfo$dR
z^ZMPHMk!zbF=8voUN%Yo;k+Sf@?dd$eTWB<tV-m;B`!^IHqk+T2ivsk%^Pn29I^<%
z9+hF93G&K+mEph2;3@Fzzsdk1DgVE-44-M*CQ5`=UUy(t?bU2Ef2UC{c%KOS1e`dp
zAYdg0Izrijp?&||{UrefXJ~qTk$!HG6fg;PAiUCDAJd(L9Ox#VM{m9}JVz`30u2O0
zuz_u)fMy37#tX_^OYH4C%!h;P0E{c+d|Mvz!<hze;6p%=^F@3jWVnbAz4PgPR=(;-
z$Q!LVo=kCNH9pDXg}m_sV$pbI>K<(9!xv7%yl#76d#id4g;*4ZPW)Eu;i@S)S~-mI
zacae;iWOv#B2YN-&ZxS43;6}|<MEn;kAza<RnVWHXHV?jk+vqtQarp;ln7;>c-nI*
z1bLgU2oQ2`!jzWjKSH{Xg0EANj_>z31wsnh9v2@#tRf%u6SU-BWGF0#hf&}`dB)0W
zxpeFCHSn)1W-{6`+Z!!0Xwsce2Xv28SRp~CR>r(74jG{t5>W*a?NdHV2xboV1c|ZF
z*l6sX5X?j;0=k&tKNIW(UFJ|>Ns0xSCBVk>gB8K*Z4!W(#)qGYcqnh2I5sqYBL*Pj
zL@u|=gHa{+I6wz75~;6ALB6h7$o-5oQZ39D;&&c?5U|1bEs}OS>xV*d;xbO=<-K?q
z(H#|hA5ZeZgD?H%jtvYt13A~jtRLs61=hr#WD(G~;=u>~N*8e-*LZ@IFAiIMw{r?2
z7he@{WnTWY$B*j_SzQX~7kYX*EUHmOm^xHsJHD4tkCMcJwJa4EEW&{d58P@QHmfBu
z#H|AbgXkC*_`Zo$Toqqqk|r6Eva%Q54+}_+M*C!{P6zSmA@ByPH-&6e#$@*R8W;%;
zAIz`$aCsJmtV!laD_;EQe>SVv=0BT-1_cU`P&$Y5|JH=~Ne2elxrt1pKOY6$69U5J
zmbo7%Jt*eR%7CXHXz;LK+n`#c(jPBN{-4G4<pGmRE$mfULcjo%^E|jTejHK~abQjL
zXB_H--Oi=}sY$tXhbN#=JZ_$fn7crLchP0`1{IjwNhOKd<3cYG0bRL&fA|J63gk>6
z|Ie9#d=NYp3!v1HKmj}<2xyq%e>LpC8U}RH<G&jA|7{zF@IQtIqX_#6O5Wsu%ykhB
zIODYy&$=IA{s&7)1x(7$2p%o+KU)C~{)88B&7X(RXyG7#1D1c+yZ4nZB=CMT07bCC
z=2aNOVF5bbqxE!$*%5YsYIbId8u<|J???duQ9##f4V?R<37YV8H}{S<aoY9rGk9<g
z*O=M1QMcWHF@PTzLIpw!%c>WrLqtLo2gW%hz2E46?3WNS_`<;<d0HySXyk(K0#fhX
z?H`y&|0BTv{tqc1cn3`Kamd4PC}4&y{G+j4o3N(Ojd;94hG6$dmTl|f?1=><st5vc
z@%o&RAj1m*DjMVpi?N%e%7f9bk^tNg{)t79Oq3s385Dp)izOf7HM@C}F_5(ob3y0_
ztr69bn1nA6{ejZpS6=z!6in5M76R7jFaD@r!dF2C{0aa_pxOK@FzzWDdIQN@q=4V-
zagX7K2+)S`C;*Ows>*Z7l7qU!FKc^{PSJ5`#L$3m*m<M@`ObceWdXZ_fC~J}Kgr0^
z%>EHvcJp;zz|i=TK>9_MD&2zj@MA5{q9?(sw&6b}rO{3iswCh%|A+bU0F^NQV@8W;
zVJUrwY%s{@)A0#GLWzv3ye0vJul*<dd4gdtM0l`zq>qw;q46GtS8*~^Ap*j8K%{?<
zWf4u&=TZLlso@z|Jx0;T)mzgt0Mr{X;vUHAJxcbAFW`|7;ujL-J^G+-Gy!nbK9H39
z-vYP&U(k^%NY^|L0D%+XQ8KVGB9EBl*c{%N5Af>65b1-wU#<NFoQKB+=Ii+f7*s3A
zyaf3$0uxxhf0B`-k^Nh}6$4!sFgG#C)_47P>;HeY^;A00$3AwEjWS8UNc-!_y_0^U
z9o*KD)FA9=RSKZE-UlkDl&;qc|5UX9vu)2z08~^6GuZeI4g6SYPPvK_&|A(&aIL6y
zy5k+hC}095F*c=(jDVa7Qqe~(^E>0dn|8f)Sf^xnIr*hgWB=i3?o05Db-!RJOMkEU
z%%6{gBE)y6RI45NW~<eB(iwK+A9QK%8VIzz7hUGNlCpv%YhGD|-+Y^8sGPYjN^fNR
zIFZ;HP>(3JIVIF^VHGg$cb)W_Y+qVy&z36|1;9h8exl3~fQLCaqDGth5=I7}gE%g-
z@Axv?+Ez{Mw?d9f;xhmGKU}bnvoH4QF}0Ct_<7dSncrUnF{$qNk2>yL2JhM<9qguD
zQK&|)Os{QMyHvAX5JvT;k-D&KkB)7%XWNc`B4HkNCqmHan`G=)+!YV1ZrC`2{qr!s
zD(`55E$&#R?E0ItWs5=#gTK+LZyINXyGBgubL6%6ZFw-@`;u;z&-NyWsZfc+MZ1(<
zhBHPE&6#h1+OIe*c$2UCR`GUr7ab6B$Y*?^APxYEgUG}Tx^|2<?8#~OHSMyO<y{E#
zCesBMcF^}I2F=jh+RBZ2FIXgQ<IU364@oMW!#{&U2|zoIiDE^SOfih2uzFu|cplC#
zz<+>*A!dTqYG*<AXuVUvQ|XOcGd#KGk5^&!LD;|Chaahu_H~K}?zl4-!sAic99Q44
z9RwWSehQxt7dyJXBRVBOjrjdKzn$)=inD4qWM3&tI;l4z)!_Hv+B6;{dZxKuHV-Q#
zMoB8vNYv}{FNlsYHY<^}{*mFOZV4gU4YLJ;sVT{)PSU@!v~%{q#k4w*z{S-lxhU}J
zHT|p)-HzmJy{z(G=S~rHxip<4M`A>`L^Q$%PBg^w-uOj|+g!Qoz+#t;6UlDzD~F#c
zG*37}%p&A0$R{!bCD%4IX}oc`zF9|At~BlCUrp}vUI^?t+~B6E+yoFmq2Z7~&(lf`
z&a4-E5okbV%s?x+JN|t~kKyJGEyGU)pvk^E>w;u_I`UZ}C60X;pOR*>=U-5(X4sXC
zA3|eTJ(_TwVRhTHeclY0Ki8`xA=(sznY2Si`=1;RHrFXmPfawG&DAgckj$8s@K7)K
z-pz&2+H$>kWrwMEkY;z=vMHafLS22@K$?=BN_UJ9g?2nUo||sDyS@V0;MPuydPB+Q
zLY;2sxAbu9{Rt8Mlqlg$RKZ<wnj$qpqK@AMq|()iSNkhljMMAB)-h27=Og(uh-E$X
zuy#Lz>ZpsTBlz#DU3l@!n*6*cFH^EU-!FZ{JsOj|+EhYdypVoxDMoea-2Qs}6B1Xv
zfRvXO6jAW=|J@nl1EK8Jxf<lcltFi-e2zQ?k?gbC6bi}6jdhE;I#u3Eu@KUsRAJ;C
z`3#8?)RcNLtPc|mz|C+jB@^nD*CYHcs++`sPAS5Ue8JoXUPoXYc>UMn@Z!#MpO1`S
zqEzwio|oV1!4~U~q%_Cx(-0S@#8=}7VKDK)pFplyof!6R%*sbf<Gw-Q9ouYWv5q?O
zLL^TazH5zSQTyxR`$aG^FEqC)!MZS(J%&SCElHJN^8S1G^;+7ZChYffSRxmR-{k?{
zH18YlZ)NwKtGwu=wv#*BtKIG+6SQ43joJ;nB0u);og2}`Nk-9c{Zi$1EBz33S0dYY
zcN}##c5&LFX?>Bkf3nbOW62ycH7nx8#kg8)B!eH}GCkf+U4GWb>lPnM%YOX2(7!f(
z;rMH+?ag|R$!>_(TU9n}+C0wu1G%YM+l2U!2JN30T4a^Q-WhUjA&K~maL}owtwuT#
zMsgB5OF53yzC2>lLmMiPo~*TXTKVf1ccN7%Ih5H*qGRlK|3%&|U$bDGy2a_&$9h?B
z#fh4>xvb$1lx-u!zC?BFdB>>azQ&!&HKaebHYqJzF3w}Q`1c97qA1zhzYb(%wkG5(
z*$9qOX2NNb=Gu9)CC6qiHLp|Vn%qh6;AHUcgol2kxj5|CDf_##8Foglq@juI)&}N|
zxUcNLDbsfB%-R#=F8tUNDoK6n+dMy0IKQv3KTO!n`JA=dvR3`RiP68ZlbbhuJ^w9U
z3N6ENjfguzh8ZnsIk<o%7IMB{t%Q<iuEnaIdG0K#^D)Gt_8y%e_Sy6cJ-%i_YL&S0
zY8xZJ;KnnjX^q^fs`t2Wb=)uI{#Ml5?Y`l5JUNKb&Q;1U<6Ayn83m02#0QJl@OQmg
z;y+%!pxG6c=>B`1M}d}CUsO?+;)J&=yrCjuCbRoyCb#2MLeY}=bSdmSyjY)`gxp;9
zE+oC?&e0)Hz?Ks$#Na2%V@W_+8|Rry<44k9l$VM(-s2ga3o%dXB~+fm%sr2y*E*?T
zaB5ILi6R_JK9JI1GJR7RhMVY9e<qrDuldfXf7KQr{vd<7SpTVqimuG%UX83;;H?5G
zT3qm}X%#Ac<KeEOC^fD*WzUT-f@^t>^|2hY^XInWO&uc4g_+&{sy>N`$m48VyoweL
z*H8LCmK^71yer~|rD)+I?)}jhxoFv#_x$H&i~2sp8-mRIA=Ado_Z5tCsQWK`rPsU;
zNVDATrHgp_ZCGL@b_>#avOd<`-exkCH+_E=`#dgz^n0=Cm`coGRs`{l2m2zo{kIWl
zioTa@L}Sxv7$mQ;au0rTor`?Lojqcq!-!^}m9GhHAo5IOpN#YSPcMKF*IV7Jug-^4
z&k&cZ){VR!^H)#po^qr5d~!3KA3M4%3$Z?Iaj3i@)=~U?k;tXZ+h?S4H~Dl5NubH$
zZkHc44eCPbOgzV`yy5yLPuiw4n{>FleR(tOA$Pi@PLMv5`#kIg9CtVRPZvP>c-_^>
zRfkxVw$j`7lPu*$3OeQI`vN-0cdfI*;UfZWdS-4hRJuC9h@eZ@bv>II=645tAVEYN
zJbj|gVx!H~Oi`vHjHk<HZ0}0(y0)!>u9TV;0m5PXpX2*YacBg8ySv8H(De#WHLpI^
zg&^`I7^e!GUU_y9mvE<$s?R~2TJt-IyXj|=@3)GB_~t+zR_|Gta_O=hH*Thb-%L#a
zAjz4(>83|$Uu!1I(EIaKJks{DvBoHQA>zr(P=;iNfH#u(9P76|jU1(XwbXGyE&WNE
z2mFfs54N#gSW1Sw>vf_~?bF&p%R6Q7q`69$?@0Xh((itEhQJaz$WQ+bA6(W<Aw>(Q
zM^MBi4ME)%CeNPUj=4i%+)z$-^O#@8@t@8bL%!2J#-q0C|L$l_L-|C1XfIfO9H|{=
zpXM&UUm{TnGym|`$4}})tIcb!?%RtaCr{3+m)CaA`Q!Vy=le{KEi92g-e26Z)$D&S
zZqiEHJGtEXXezZoUD!CV?Ao5LA;d00Kl#Pl+}dF6@&F_yfOK9x_2VLK_u8-p8)?j_
znS;)QJde@i$?VRhg(4G`KT@6ifv`916bZ+IWb%2crq&z`jZp)GQ^YrfA$khd8`~?d
zrZCt!5|Nf2M=#Ju<G>r|-cfwl!uJW9Gqr%i)Y&DKqZOQXxaRuj0}QI7Y!V?H=S@An
z`<$q4Qxc=&A%`nTobRY~B5t&3q5?67r*5)5am->4W;6NR1aB`+X4^-n{GaxFi1Ocb
zOxMHBJSTx8L^%j=YC^?nHro_k_4eZW^_zJ=^KSolWuRNpUyDLKS(?IMwo}V+=KlOw
zN#WSPz7Q2W&p>06?oHQZH22a-Voj)P+)P(?2`HgBt2Sztz?CbEPu{`t64V?FnXIr*
zu~5!q8P;!Bs-`y5s-y)sZI=FcAFsoFcSn7V8Vd<h=kv67@FvVyzeY|c6STcYAhA~_
zk;2s=o@xpl6{DbO%&E^0xRrCE#~IYb3fRajZJkG%ddUlOKnB&$rnogSWIjh(c#f|3
zQtzCsxfYRRK@Qsst*ysTea}IRqhD*E<!_E<)s<|rz_Yso^ga|sH!2lDBsZsnB@dsm
zVr{pv-66|M)JO(*Bk#egpMgXqlubeMz8+UGny-b7Z`_Hd(m}0!5xFvJsrL9!<y5zS
z;B+lP2MXLJ<3un_$YpuO9@Bo{K83T-x46o|-P*!Yrphiq!Nv_I%xieog8L&cmR*cm
z%8ZUm*Slh=y_t#CUbDXS;`Lm~)2ap@DtG7ORSeweHEHwKp7Vv;2Cc%wP8pRVdEe%h
zK!$bckuWG7*~L>alJ*(Xvk8>;wFr7mHKhWLm`}q0dc}wz#q!BU(kRPBGH55Rn1j39
zQK#FOS8xsV>GcP~6gM5l4A4e9FiwN(y+-A1ye*OX#8!6$eKq%n0|G>uhFOcIo*J0T
zXuM>6kfs|ROFwo(%CJcUam#Tp$=Y;06}F8m$3GapjWkJot@Ybckeg&@xY$MG*<05M
z2c?#Y*e_^diQLx$?d%%!Rrm0I1>DaT4Xx6!Pw=|hxz}V*a)xs6ZRWM))PglvSSncE
zRIpW<D+*(yotkZ@@8vT>;Uyx&zKQgW>suFKW<IZ#DPVaS5{LeHJHn=1m&8|UcT8bF
z@aii1Q%@{eGGzzt9eBE+Xs=gw=Wat_Qg6tIeaaqrZ>lF;NEnhH1?b;ZixRXP#qKwp
zHe#?>%vY&HDP7<axZ%(pN9`q0$G8;p6Qh1T-N*2Kbufloc4SnECetn@_j;mr>?Ts>
z%g3+yPnE<r-_2~;-FJ{VAqbR`l;c&35x!aFksJivJi^kzDH+v8wMsbcs<&?s?5Yce
zOk7aLaXY#GqRLX)bKxQ)$LJChrL+l9xA6(_ixt=Y3?C5bvQOkqdMiYr{W=dmw<+b)
z*MWZ7Llqh=;_Wn)p76S=*=z~37Y=@pZFj<umIDHAN*fwIL%~UcqalAxtfgc=J=x42
zi(MW{Wzphr%#rIh-qm5Sva_^P8txp2aa3VcPb9L0LbKCnnb$`B%1Hx+%-OQNeXW`)
zy9>Lcg%-HG(kCZ0rx~t=h->Pz6@gD=?epr{*f*QeXWr+F+Nd(Rq+|Vrb2%)HZq4L>
zKV4*;iBuWIHW)){fr@$<1d8r$!iILrz|*{lQ6^DA$6GorGE_WSfxlT`P^&4qyY3)I
z^t~5DIP~3T?f+!oruPf3g`>rSlpkULBaJTOg|gb|5TV_g)8eVG_$#q(-k1X3;-FUo
z?0ASv*T{iWc(o+gZM1C9ICL@KDe+#&rzkM3|0U~)fO;N+$K@P>OUO5?MNf&a__LGp
zz>|Nf+%6`u`iD5~#sCG&Wh=OJgo)#gBTITVf!e-i&>1S+7tbE{Nf`JRDx>~HX~@Yi
z?2T2FN#aRZF*g}X4*@-~s$GZewWORD#oJ<Czy*-^oPV@%1fEg8Y-%U5=0`tOhr*<8
z)Z&tqMf;x2Yk&HLZGWuE_-IQ9((B37#g2SG3nYUsPF$Rz=`uz8`)5~lIV0aDreHs6
zMpRW^lyWHShax^b*}p{@R<4yM6!l7EA(9>p^R=w6^d;10=2G~Dc=bke-8G*IsrH@X
ztLUDe;sq*MufjCCM=esQ1_j9Zd7rE{rGM3Sp<XF<+=Ee8x#c8s)QCFl|Jd>M1)-1>
z_1)5|p-;cQ!H{yw7O)&IR-8U&-31jiLQuX>4yvM$c466cNux7;zDf9JSwu3K?jqSI
zFmEtX5AHy})MKT+lAE(U{XzH)_u`VLvYhMS7sI;D2uXL59V}cxAdIX>=AROiU=M7(
z%CN2c!kp<UI9@!_N+~p&CG0&b^^xwsWFtvNGB*Y6i`_?~thXn$3dc-60#ewki{BYk
zZQj8RFaDS?>Lfg-rW?|!j{}M8h}nHrf&wSEHcbwE4>%aaJ=lhQx^jE7gUDP&F)Md9
zvYDHztyS6cL;EK^Fvrg&6%guF!5t;RG`0jQOlaUQcrUJLf}b5<(O0Fco5^mR>Jd__
zEB2$jT{ruzPS=KWQ`{#T46A;lC22Ib5L99%Ww~3-<UFg(@CB6@?ttyrVdl;|)8#K|
zj#^3WwabNQOqtSj+!ub$LN~L*rrbD%KeL|#;=XqOYViU@&AD22Y{WXhB^^#bNfoNi
zNiy@8dEF83vHNMT%5_QvRi${G;^(XN#gh3&Z}^i>NE6Hm=+P=r*H0Fq1zbCDwGm+w
z+lDx0R}{_k2}0PWP(zPw)hX;$#e$OFf8~QD>i&EiuTL+G{ioAk`pSZyoMEPPPSp{(
zsn00`lV#A<-iEZG6)~oyYbU&Ogu|!&aiV#YZzkYpltSHRkFn_bmNghq2$FO_fs$Oh
zU;L~EO^**I>=q9-dacCc)##^^4|EumdO9Uhg>S*N?UhM6bcA*-uiwu?{`%bdi^)}v
zGXd`VLI$d(i;#ZKH#ZK=7z~Edfq8Q@lPO-eBrTij4~^9FB@*V!2I!IK%cO~mk&GVC
z^cBQ371A;6JEDhM&Ye2zND|7Rn2kqWD~pcWC$I`S>?zw>dJ#Q&DXG)$y|Ul?2I-U2
zz8ol&F(GIm)GxyEXQ6^9YgE|u7`TS=5Fr&XIhOx2_~^6QJoAO~4ymRbeg0PVB`G4)
z6_u?;x@IJLBLPVYu;@sPL}!7+x(yhZILY*tZ)|g2XQK1n5cJx~Zf(}WG4x8zD5ft`
zN2v<#^s%_PA}o$=QzqJLK$jK6!6ko;MJ29Ap+kI|nD)=?`GWN^NtQ)TIy`O2WX<l(
z&Kn}Puo{vMQsFtHGw)Zgyr5Bqxxxhk&^a=NGGdP+ulNbY-fejnS-W8EpVhofDO^gd
z6(sWM=;jWNu-4Saq3LOFsS!v-q)*~h@GVJ)Ne%x>&z-cq&%-5#_p&pcTiMyqQ+!o_
zWOdM*!K{#^-wYkjNUjy8|2O(}a5k<f;^8fKA+mz62li;XOyE~mw7VkL-q!wxZ0>Td
z;|fR6s7@E;!AD$n;iU6J)R%j$LdagE9_NSx)-jknvbjbU!vf;Ri#Dr+M{{O8J4%UR
zPq(7RS&Qr{2j9Sq?K{dN)QK>nlUB1RgD&}4!TSs<VaP3LhGn|uuiP+N-0j>bbdQ9(
z8Ie!VGcI|&W`vB{$h8TWm(`FM;SgQ%KyNvRXVQjpD@GCzde-}(X9GlLoU1lb2u}r~
zKfCj=g{~)O@ceb71<kwai}HTy+M``#>$>9^sDe9D0kd}P<#zg@BdCVlE$(r&PVm;B
zQx19-&gWqzmJMOFf%B+0ulFP9vI_-YFUc1&Th(^p_#d-mA~M8a$zl%xBA{)0c)?)L
z?p)4nTxS<<8pZwWxn!aDyuEZL(R@L{*^oo3H5~z7uXAYv)p%@or}_3HVNAn(TvnuG
zJGwy5fy%{+XF<4vF}cp*QnbW+&3LlWDHr8_<+965&6j@z!|QExES`7Lk=6y|B1AzE
z1%Z)k1~xKtmzQdlN+TTdjJ{GXJIUGg@5$v8S}k9~=*?HZVJz|Oor&*TuNy|S*b^PH
zgnStdDV&fPRVI*`zJx5qKtE|=!qvV@d7)RM;39;QeGWyk43gZD;}G#ZeY=Ci)1=(!
zNP~>tbOWvNSR*QXye81^<O7{n1OVSAK<E69X;-s_ug#y4-NG25eiA_~EX4Gf-zP5F
zH!f|`fM+S_6r0#jH<9&<A<!uRB+Q}2l)$4hWL*kvxyH&(4YaD!gwI;*50YO|=Na%|
z@p5vmwWrrSXoG=S<~+}!B#6iotLvA*OF)>I>hAK#N=!Lj>Oj<cM->H>M5ONcrw#$T
zc4C%z1m+oGBaK|+A?KaltMnF^i`$U{{yej@q>){Ar9=l7{*AMqFo6In^gVd}<49Un
z(w8)pQe$EPD>s+N8K$GZ{Sssb4>KgAB)pO48Yg>W9@|?TV_5{688bY?2)PCgCu8fw
zcDza2nhaa*;xKpAXPpYi!|0rQ|FUX#?;c=4Y*yu`#omN7(mLlfl5-ueXgKfDsUFjt
zKG4fxP>QP~g-eysw>{k~y}<PIGeV|@Jw~q2MXZct*{oDTsK!?+cC*TSyhyjyHZ4L7
zQWp}SMb}sj|Lcu7WFF#ro>`Ly#T~BdvHawjVn8eeEwy?^4Ou#{@Jp_+36kFa!eXn#
zu|geNX}Tg%9r4`C8BUL|C6XOx(i5a>)5)aM09VVuU`AH?v8=Itc^h^fGmd9g=*98|
zg<N6vzV9w=`7LRM%j@BVb_S#+Zc={cW<8yl@N*G>qnCG+j!hwgruRmRF=B~xWLL>o
zRQRCZe!j<BT5jGsL$0pE{)$zkO*HyHZ|X=bQR9}66CKYpM$OXf`&xonLF%-6P^mp;
zE2Ki`wRA|;Je-HZ!Mar{U)pzcRLfmw!Gx92>i5Hi$}FU9o79$A4m=3s>32^v-aEbp
zho@iK#iE-D*%q0a^(K|5K=E+Y*1GF>;=@m8NqCN;isRh=PdeY%ZJQTkqQ>-hnc^=@
zI>xWAxu`!7&Jz)6(bB3seU@|ZarN|@@0|D=5kQJthmS!XL$g0O*%`hwWg=+YmAJ2j
zn<s#BAwefOn>zlJTC+5!Zqgtf^w<^%kfjac!O830o~bTv>$t`eTYhZWrMo!&(2}(E
z`Wtw%*>6P}TxMgr(Ef~iF^pgeDiem#_8oGkkZ`o85~oCBh2x58Xkp+I&0c9U=y@H*
zpm8}2h&rE7R?NU7PJH>adz116Tki{6(DgkdjE*SU{!{VLC~TL$pPzGS;{LGWhM#or
zu;m8}i=3UDrKem(f|NDUr&XlKeoE^bpQ<M;T%16hojk$!$5XRetB81OvIzTt*7tdF
zYGVB=@BE-DTc<^6*3O#v8B-xP`Q*FW>p2&@&|7Ktd%s{$E$1?es_r|?LZ^}EEDOgY
zXlO1qJ^OowNM*DJ9U0V8+7rc5+L!&e78<dP46X^|8P1Xg8tV4=&N9lxfk&QvJ^Skl
z`Pd^r`EydV!$-`=CgCksO>47x|7ZqrFck_1qVh0*BNjn*KOq*&SF_`_yVw`SxGfox
z(EM3*T=Q+GdToj+?K)f;jgRg{tj&$TU&VIimkQ+f8E%P=#mIMF=Vu$e95KOj4yCG3
z8HcqKm*5b_?3g#|2_s)-l0!3A2H<=e>5|Y#(L7k`;CH+5o({TlwwT|Yc}n%%*KQ^P
z6^kxapr9WQxoXpU+%z9nkJ&D`0sr-%ausVPvOCVT{q8tbra9kTd3xu)Wz$@m>2^51
z8~PmYA<5u5v&Po}Hsya<H1!GMnM{6Ylb4S(BL$A2Q0l*0@{vE4wHGAH^QK`ApLViU
zP3?mEAkNY&GgDAnn`ClVk}z(&{3jgmRr`9F60wL2LoT#?4LV&O)WSK%-e{Bwf%48y
zjk>>DU*(XdS*g}q5pugIod7YVJjFC-Tv|`<c0p|q`+B^7rc}&Q!G8IeS~tGy#iF4P
zft}b3WjgB@R)^D?HoYvfc21{TigvZ(*am(W+U`fBiSNx;VLhTR%u*<NO@fY6DF|dU
zt_2)E<S5+IM~M)N<lm2|_q!!IjPTp3=-Li{$>=lmVD+zwJlHCbr+$NWoS9!{7rBeR
zc5T^cH;bLSDw%tU%`7|b6c%GY>QIn6mP!!GD7f#`3#*_Za@IIW<f4OeJso<dKRG_w
z@3z{GMXymrp;SOOBg&IS^Gn2mpP2Hij0<wu%dyEOlwX|e%vmSXlJI-$sf}C1nl5j)
z7M<Qxy!0JRR>GSxZ5Ez-^;~K0SGrFJ)qYtJf!v+JxICX`$>x*pq1?PpLg<|R^Oj{}
zvAYl5i8a$#B2P_CUS-MHR~Ux?HNb0ALH!->NMsW^x#d0&SB+}P<;N#grJlzPm#7c_
zfrqrNGj#vfAuyL4zfV2r=p<(hn$dL7dLTAmvpH9Z&U9AZgyVi7p*Q^6f-(2j+{Wg_
z;2bM|4gT*)xr-mtobS|@T278f^udqY>S5+U#dIZYlT4rs%)wNW(-2+gw*$J6V<vaI
zAt#^9Ia|6s<cwTV+`FV49Pcuk;}Z?vuDJl0bKePPpnxUH(ip79p3fIhvD<C@+%87~
zDuQxePbJ@5l<P=q#ni{n(z4;^lhWsLXf;19#*nWuiP59)f{#+KqPSOuH*J1pfI}y>
zX_k`FlTl|(ZRB|?rcNBl7eGYG*RMH}7W5|BXSjL0%ywBo6jrZH!=9`2z|pbrP&){h
zbCCP^S^w~UMuQohal`c$&2ZW{PjP+Hg{{&~P0HZipNvR_YE}!Pn8YAHJct?%RM%Y(
z>8@Rs27UQ>#=9(Ex=4TRvvzlQRvNP(JGN!_xvzSrZLnj{jIdPGv6s8QZmlJYxA)aY
zVboaJ?b+Gq#)<{P0dB+^CimNm770Q1ORl9|S8;d6VFq(KFZ&OkR(zF)5^X$atlO}T
zQGCTceZGie>uic=zmnNq1JBqy$r5vOeXcTt;(qht6O?g<d4aurVfl<*oJ{}@$>klB
zdhh%cEHDU~N;JuTBGu@vT(%Nl^}_nxD%rmb+D-Gm5u^*0OuaiBy-kqqMfbfNV5Lwz
zH<oxRGH+6ILObhIGjFG_-9MrKRM`n!)lIk^_<H_LL0qdch+{N+AgZ-GY@Y5-yvw;&
z34z`Ht726xS3RO}(G6?OnwT&PXHq+MUcvHlmN9+>{=1ge3PbZG^O@A=w{a(zY^yAR
z0%_FO9W{s0Tq^#R;$|`s?I>LSKCbl_R^|$&b}Fy&R_pnkqBfS9a5<8*$q~23vvT%3
zt}I;V<Bt_Fw|ErCQcP$zi|Dm+RQok^cGNW$>;^aCq1P`HbLjC!iAsa76Tcjmmiq4<
znyyZcbNpsaaILeQuK8wvy1?xb?e59ei2SL%C#^2~E(z-YNon@gwEM|UpQXQzKyJHO
zPtLWQ@1ZSO(e6c^cQcDrv7;yr*riDR7<H;)P~O6IdO7cO%3R3JJsG|=#A{k_J^Wj<
zx?<*zkyq7&xeGinG{q*{6;5{2L6d2DbbUr}tk;|d?)hE;e3@H`ezY^|F#*%%T6gbk
zg~==&=f2Syu5*eyC)dEoq|yRrZo7$T>2Z51{m;MJQMc-QOt+((E4c`!^zNobJG6D%
z@y})1KA<*nT2@6&qr$rKB%QhrUf|+&2%{F4H@d}qbG2y`u8etUeop1cmj~rlk-O&X
z6Hu3uoml|)eLVR<Vu~TKmvLQt?Mu>h#r8rC=kQZvqR-Y$u^02@f<esZTCPFe!l==a
zs`a@}h5S_`rW>b5L^esa`v<5t1FFia=5#kjTT4cU*tVr>X$Phr$&m<)%!@b8hAo#_
zRpPaTEN!;Asi=6`MiY1Hv8Y*%YilgP>0KsPX=0D_W_ms|-7Wr27he`!!#Y=)A3_<x
z@n>W4HY9D|dKubS^u)^Ypii?muTq=Me;QgWpL>daZg6)Q^x3zd=Sr=7#cb|>3m45D
z)V(iS#}3W-ThnUeYBvBbRZD-_e74%SyveiRUV2nX?)=@=jD|SsxX4{@loLW$aRds_
zI$r6)l4SLcZIDId#=HpX7E%Yz)kXcL#RA)N=t}v@z|Y@UF+F*NZD+OhYnz9Q!(W}K
z>64l~dwM$?Z#R9scTx7`wl}9-waSSY_L<@W%{;g4d$z(P`2<TBKl|nCj^4Wb>F5u#
zJrWd6xICyPFYp$34;wCF?2E>nBLjPL82LBSST*|scT>M;ekHOLlN`G5y&H)y>fN$<
zR&84`F^TUuCoQs?;`%m+WgR0yi7{(ZDR@!GX|b4~^b)BcZvg5XZ-b}v>$yQ|@2Q}h
znWE#2B5G_zt+~eF+FZ<;CvMrT<&%tT3HmS#9P@S*j=!rcD~UM;GyYvF%U=>)7A#*r
z?Q?UEG%@dplBCIb=hG#e;i1gE(iLA;(QCH^TcWo1L90z0)1)`0?loP)|BYeYf9HBx
z?uZk(iXr1zVXG2MF@rPmfllvadR==Ki?Q+Mrj&X#!TG)Hy;du&ah3Na)@QT6s%7EQ
z0X@kJRRe8x>(!|e&VFb=r1TG3Xuj0%y-i6{?s`}Z17v@HDV)l=(mxgF|H(hjy_Q<K
zl_a;^ovr?yqWnU|YVeOHCMP$aQqm7c<2?9!&fnJ=u2ff_kF-t&wP(Fu?iMXIzisGN
z)_fd#stYUBS9d*H7m2B+ifRToy%3^pbmxu5dOYbiMP;roE`56StiGlG{GJ*u8H@2_
zlKo%VH;$u%$#-cMjJx+W0)ut!QUlQhj=uRmlO-yS5#>7z?Z-9E0W8r`--e0_jnBW;
zo7<tX4u|svXuO<|<C7Gh9!%~6<tB>pSOL`DmtT$dfIB<w(#P!0Znd**e(9nlZ}wB$
zd_dMaP*$DDK45ZpBjqk_aDtknki)0uY=rz{Fa51fA0F>obT`!+EMl3p4UArRFwwHe
zzU^&gaWngKyUgRVNiCmfxA?COhAir_UvDW4cf?)3-;6Bvh;V91Qu2OSrZJB9whG<1
z;^YZ4mG5Vfr78WCrP%xV`aHE!KO@kQXr-MDsqo{4RwiCj)4dDL#c6L;Qmr|51%jjQ
zP=bhN>Sgo#T8!nB^Yg>bYs**M3a8@w!lyiT(f83)W_aErrto3K&kuF-uc+#PB8z@r
zeFm!&F=7zqs*|nf724&Zh#?nKs((NK=4lmf<Ndce5>DLm&k2>M0)~INhdrA-Ycxh&
zL`=GGgP3cH*lx~QernTmE<&-r&g^0=mmjS7iof4@`15i&Y0YY0^yj$g%!^s&(GtaZ
zHic@wmy{zc>7j3aZ1CJ&?2bp1b<7ap`>$=@aebYa4y>{Mcp!A_5*7PfvEXp;?);L<
zuvfaGG~d(0b{@4%OLyq(R8U4dkrUrK)Txlj{<kw}M>>W_bo!^bNK2(|t}g#fO}5@*
z)>UDXKeQ3pFnSW&$aDtutHP4(m^-3V&?Yc?3NacHU8e!E`qgFXj{m2<w~UIiYyXC6
z2`Q0~wvg@+=}Q3xl<pV>X@(AoAw^Ic1woKhdZ?kh1f*f;M!Jz2V2EdrKL7Q8c(42Y
z^sZ;EXRX^0Ea9AU&)!G=j^94cb9;IuoLx%#T=9f+nsY!K`d?KDZW5wqud*$TE^2t~
zQX?*wd^>Wqd+REcEx(^`?IwM<6FawwC^E4iMi$&Enwn$VthN-(+wxgTs5N3Xc+vP=
zSfSk4Z!PgfRiLMG5vV1S|EHEvX%in|S%K+B@{^B$*(bI-`Nq(F=lFI<do4;NY1g|d
zv9%)h_p_jPQ5%h6-c?0>t4hS*)+`XPM!A*P%&Uc(Lw0~BA9*ziZ0ybS9DYu5C+8D>
z{)`7-3l{>{B}QIzsIzIGn7pz9uDlUtO#W7Ec5%LrAqt|v1IiOavq+zVAFKBJV-O#U
z+2X~jp3)i3^@pz4SEWReJxAWQw;JuuAR_)__Z4@y4Khz84_{h*#s=>xin)HVItL94
zysG-roCh|W@>M0R-t8P1j@g1RVPCuFN-G&^LzyXg81z!>%6OYjB&v3CNLaoHC1V!u
zIYH0vAM=D&VM=|>%p;Qd7qi;MVJqs(QY?-jz0k|C%)amf{Q|)2M4}*mKZ<_Y7#1N;
z!JbompndRqfGsb-?pNnf@A2>St-<TRGLN0n>G38T1ld}#xi)m{><b@l80<`ovcHna
z^s3rajD-}|j*G+2nkD2?h;h~@b(Pu5Rl<~_=0y9ZU3ZA&(w=vB>cB^jB`>EtZ5#Pn
z%1zaqxX)3AO?^8Z*ImZs-*`$H)xR1qSuvlgPbG*uc!s(LTOSxFTd*DLS6u|4=WfJe
z=PywZ{RR8H%mN3*zQXFN7h{EHuygRg2HwuwN}F%&QpZP*Y-e=6PGzn(FBm*=R-%Wj
zI7sYl&Z4gr4yT$WQBBaDlp-3sjj@NExhhiK@ktGK`kgvB>S=oB)ivf?_>p_$Q8|+I
zg4;o2*<X+zyRW)~C+tOZU$qgWxbAB4KjFwL&%zmbFh$H&)0jUJjWbKr(No4lRGz}=
zlk=lEaO!(3C;J*YCf0!4#`A(uQ@0fTRXS@pkMxV!_RnW4Thfi?4TB#ZMa}A^UU=Mz
zQ&l(Sx4NaVkE73Xz|I%jSiw1g13c$@!KXi-bM61}oFSaN={#m{pZ&T1qk<L%rSsP)
zbSr8pQ|Y^+eO-7`wm{JZ>Nl>A!!l<+v&_!Id}Zjaugr9f`Nl``3IFc+9aL1q>Sp41
zU%$H3mXEwm89VJ|q*4&^*Gm!d#&dITj%8g|?>`QNartBtTf#yh+g{f>ld}td9vVJv
ztp8NT16{p~_8C>3>X$y)tnS8c&f}?fh8Abm<$m}J=5J){i1CywsH2{qnVFbQ#Ld{g
z*Pd=A!WW&a*qsj)-Fb&X6Q}n1bo7|9=&dPe@NxvA3nLgeu_N1Q266YCpTQp;M(Yi}
zmRbt+QiJf4%Rt1Cx4j*HNi%z0_9~EHB?;x;?+<a9usVf&=Ck=Hrzw`1HEtvFdkH!t
zw6-sbS?5Q@#L#$MH#|G!A*cV%n?pw>Rw3qL24zo{)7=jDl*oOTG=7#-#HpHPb3`9b
zgzVPrv`q4rXqjM@mUy<qvm3>kb}dC&a}CICjCFpxrKSCPa6CEYb--I3O&aP-Z1cSX
z%4Jh?so++Zw_TqgC399ArR>F@Jt}5kQZ72lGm^Ws?{q4>dI+kvm1)TS*<=4}wtjkA
zXyeC012wccBt7UN<~Z7_6lCPw$V6mGdAGli(dSQ>Rl{eCozLUlc$i0MO9|4rRfz3!
z2$H(EFVg5Po03XqQr5498Z4jCeC$%5V;XLGJ*UYlcu=YTjUUYh)1i%{&+qYgupy+0
zWi!>#Q5!F(Zd_K6I_qBP`iQ26XMb_n*w998J3j+31pyB>L6lD|6@J^*Hx`|2XLS0#
zy{uF8=F%@)(zI_}=+Oyv-|KblF~RIeKQ_ttC=TL30RQRlJ6s5kuCO6leRjTg+m+S%
z!(%RExW*ox!FOPzdQEYtYssF~Tr4E32jWcFU16@t6G}(;!^<@_CVZcV%_6<})Q5!n
zTgBGq8#Hatts-D>S5OqsIK(&y(DCH+z7;vPuQJW&OwTpl>e_^wrt;Wt8ML>#ITWNt
za2lSb_zJ)1nc|Z%n@v0e=iKJ*qUrlR^d;<7UTNr}h*#FcGgJ*sj5kmlGUJzQle4O1
zCKxpI@`0x&_1}s?TfBT^CZ|TDdSm#pN=IBxl`mc9vBKaYu0;Q;y_ffO$qu_$9o@HY
z`^$FIJk1=in!qWq^^nxF^;|7Xms?APD&K_V>K4~WK%J@+Nc-M(Bq*H`j%++Ji|VaZ
zGt?byx(ZdYsfGAH>L%#3tlA~opeY?HY(B(&j_q+FtHEVfZf9+!;Ww10l}=1dtR>=6
zoSTkr-5;Usv`JL-RR6jOAJ*aEQ(C8+n&AUJe|v?EN7gf@jQ{lRUXp*{Bb$iQ6KAdo
z69Zoc$lz2?SZQR1VnPnFp>JY~F205E^|zc~^L6jwyZ?&i9LQJ10kS2ok2|Nac^_xH
z7#X#`PWT59K3g@a$bnngeWI$^-eRUWy5;p<->+8c;8+zVKI_`|TLC*sBWATO?!+Gp
zN1GN1A?HkPbe(MPCH#ddcStZ6&U>Q2mar9j#|)1mM-yoIkyHtz&T|qUt6FyEveqy5
zM%@R@R|64IOm4E-M?3Z*d~c=;E>KJ}bffCs{G<UZH=btJu~Hz7Ec4&(tD{sO92s!Y
z2`j_uZ0l~yhmUNB*c_fU+!GP)5QRm;(=*L<DLV^g0YGwh`zYniTyTF=d54fy&F^oB
z+}Jf(38j!_)d%x<4g1Ly&`!9PxY%qwMPKeZM)$Fn+0$ZnJf6OAVS%AN!8U|(Jw<g#
zV9#z!v){kn?t>dej@}<yQRPQ=W&_ReoBLy=QeBdwmtA$(9d4g&IIH$+7!p(J(Ftyq
z)8!3h$(eF#k@!RG!W#)ic_HmV<$3!ECm;p6fP#3>PqZR$N=m!I+_b@D{#-G|FJzqV
zmwdA=Z{JYEI9er0uW7jWI*-dts#0rc{Bwvgf4bbvmul)$@&JOrfZcljS#ISIzl&vB
ziHBp}D+~<56?f;p;M17&l-bwn<y}mUC2@2F$(G}RN#wcN9W{U>OGePQR!%5n)2jcL
z7{aKyC%#}7yEva{k9^;WHyF^ong2r41Eo#2=%7O9tntSi`D!CwKI5wN8<PvW^dj@D
zq^T&Z?EgARFKo1v92+bu79;dHGWd>IcTjna%nwFZ9717)>>o^L(GcB9hK;vj;=iA<
zqbhJs?-~w5CdInp^q0#4GHNdAbc)-Q#;EG+>4A~USaXaMVfnp31YK5qA>lDK7@hrg
zv_iVN=|PW8AQt)P7yphtLe}VKf5n(sH!W_3M`0BR$ONiAJ<P{DqyMIr>&9w~oH&HT
zm6*A~NOX4B-W2Odc~^vE;goYk{~@kKi*%3avGMXA$a<>(y+Sw)mV#lJ3~KI->*4`P
zy~W)AzE>!p2v8=rIL!HX{AJnxO7Hc)^J*HUY#I@e-t#6iZSZl{$2iuQ{43}x=ad5k
zJBI^G&m$ha`pm^<gQKkpK`^1LzjB$ZGnudjvP%I||9gd1P5JNLHtQc=LMUai0&ZY#
zzc1Qfz(xq%is((j2P~Y#n}32p%+KzxK?+y_Y=mki!q`#Tn2%p4`s27i2k?KSgK5G@
zzNuRcSH3)ojJ#$$g0RHcThd=&usQHDF9K8d0#pBc#b1`^@7=lIxdLJGSlD<<APiW2
z8ka<TVQmU3XPYkp(tNGKns3g$EYHU0K?WrEg;myb5KmGE^KrqyYGZp$dou(~^N0td
zwYk6%iAT%@+Wjd!cl~9de^WzF?`B**nED2hnu*q*D?laxKPX*3jI{0bnxqR5JP1j*
zXy=;^+WD}^Sg`TvxdD0eonTjsm`ly%n|}`*P;4O{JKx5Jt200Nw5;i0!QYU8y$1I2
z=l*~Dp-;<7@$rbue^C)Em_qhne+0DQ5w_h^P;URX0si;jqTR4@1oRtm9{T>LQNZv|
zRs3aDOw0Z|p?{b2e<k!^8~U%8{c8yN6Px}UhyEL>|C`JHlYjg-x&Ge@KX@K2=4xht
zxj@a$ahDHkSfs*^1q|)0&ATV6AI*IXz`ge6GsJhIM{Le#<!rO4sB7>=0)MmQ(XeLB
z)XsQG=K5F>WJ!|k4nT<iFMu|n6+jE|8B|GrIC!`=*ji9I>-D@h@`y6$8A!Dvo|E(y
z$S9f1s`c6WT0`rBG`W>wO;xBK0r!km!2Ni=mY!sBWI@%h4;GVU))9T#99+z<19BE%
zq!|62J>-$xKNw@QO2CR^q+j_l1ioFChuE>G=m(-*`YvWVWx@gE(kD&YWLL|dA=8D=
z69s=1)E`gsK*c6l@NQtB=R>C>vFIb&aQj0R-;;E8wWX`e-75{L+dQ`^T#iO`lalOf
zICiUc8_@Q5X>kNRxqUFuiIb+&Yr;U_CJAQEBHT^&4ewN$Ya_uIe!I7Kx|*7l>U+lF
zU<;ir6L9Ghx125G_DcXq9Ia0vEM*-B<!KWeU}%C=&mdVH0Bm2+t5}|1YwL`;+pDg~
zWbS);YMIG!MZjPey>ZKdC?FvR*M1}~?sTVim$2>00zWBOagHo&w|V}XJpG^Pfk@=M
zycV19>GJ0+<63u8lh4!uiTmPLjp-KsV|YWC8g+J0>n9a9-eL0kpdw3e&<Qg2@->a2
zod7cZ@ii2q4LFxD3x+=S?0&u8N-lH$a#YWZbPV%=Ag^@_>{uVMmJw$0-rr2Vi3ps7
z&=efK%tCQ>iE8X6dXPNdxBr-2>b2;kRi;+|!+->>>Iphv0?}V&<GV*~be*ksKGscG
zBqd&Be$&KVHt>&v2>mmlBY)a?6G%NV^pX*rT8cVbiM_`HO46I5#BDcug;3KjNf*ac
z{iM{zdo3iieSK}KM!c3L-rFLG7Gf?s{Jf`xuhjvHC+PEv_miavec#YzO-Z%gqJ^m}
zHL$xzH?*B|4jHv;jK&I$BhGS7UIySB<tHeOnTZ3r`C;!IO>b^qajp`AQY{x}LAHm<
zWBS!fq9rrxt8;t!t(95tT@lorpI`BNUgMgCXCK)T@oA0t6WWu`sMH2Hw$mv$R7<3z
zLS<E$zK(^k_>M>5@3%8JBmkw)6;!W$u}Fee2+a8$L|`mhVn{BCpqkJpw^c`au_H>%
zq5dfT^mhkK?~pPdRCLbA1(~nQC)B~{^2tCX0R>zusl=><zCO=A_X~4j?M{uw91JCL
z&?^QhHd<hiB0TN)1AWB$uPG6$Dc&`14_+i&m8o*J4VSrG9F2)xov+ID8oBgpSJj{;
zZHC#$s1v29ox6EfN;;Tbkm#dL%SUb-OMrP_D(!o;c@pJWRO6Ia{jK6^CCQ=2$)e%n
z*ifu#yo;;I&V3H%fs%Ik9!MU{W4h$jFNLbIANn*V*v)O`KI>_T_}$JxTX5Q;I5GrV
zj?<lf8$>QWp2H^p%4`4UgNRFx?Z&IiopV}Vs>|td+3=bS&$IRH8QS;j$ZQEJ5rK<M
z$d!$bzBx5w%)~oxdvtl!(7|nWXt!TOnP<6>2bLmY8($4H-<x05uu&-Y9O;(58S@t7
z%J;CD`+hqX-ZzfgBEccNceklN#TzE5OH&Q%aw(#>xB^$`xvLK;p1B@0fD%4gjGB4H
z;@E_3(UTx>^rWdo5Bn=biVu0fb>V$^x;ldpbfh38p$H|i9o|Dr!_J<}c8WY4CzIHs
zxISvaNs-z7qpX0yIZBCGkDWCm<mCI%&j+b?FJvxU4>pS$X?U{O63*l_=_r1aa1J>B
z{y|$iA9ef~I}J7K-QQ1Uax`$2Xqh75ciIgJw7%@EJLt2$xs-Fy7<egf4pn(Ur0!Ix
z^5pR?%j8$XW0pq5*GykLTE9mds^OT=zg|V{AF5QlRT@@(x{@@*jbA?Oru0gKN*BN3
z)2!zz6B5`+i~PQ{m?%Z2e;AIb(=+TQmwu@+{Gea)`felI#%VwLbt5RK)u`B190QVb
zT(Z;Xh%hnZoqtz7h-N?4ex`eJiofko?>YrG*3DjAw0k}8q-4f@DiK~W$yq@#QD%+o
zfuUwrORc9W;HMojmy~^iZ;Axvm(yc89dky)R59Vu#Q*A(<_iIc&+p*;HU8I~6pea&
zHDLV=q*Y4)U<I5}&zoFLRYMDjp$lBW-t|t@clqiMH8SUfy|+q27-;2Y4svwrIB$bz
zVvyX8=lg}{n&MBveUmrXX+sAGfgy|Rc!td82O=7f?NYQzLC()M)q!O)?`Tcl$T@_?
za*ADD?E3M2*?F*bQeGG?+~%M19xMT>(EqS<#@$fY$uO?Zp=yCZS6#1PNCXB_+Hd`(
z`n-&4g9r78L&}9X#VPRIjLPH3jZ!skoAH%C)7LI6284REQ@pzHjit}`9v_V8LaJ%h
zf{hu;nH~)@y7UOd=&x6gQR^Bl><>769EBU!nrSb)ZByk&;%|M%)`%c6f*p-`DQ#a)
z_|mw5$DKPT2)+Br-o_!GF9doTfM!F}s0xR<hUU`aSAKoIL_HIugyIradi=04V~^g6
zkozT`;S6HDdZV+fE=hK!&+NfEx`9S)nh(Ku3RpUr#CX0%HA^2lZjRFki+exrg@a~6
zIp#++635D~nMECArW{*IXJi;25{|A<SG(}f^jC@<cS;Hm66j6WRQ$BE596tD%-V!k
zn(Tl`;u4+h0}YECm!ttItTQj46MY(sfHb*?KBekGx*D=SSur#0!Rwx2@Lm$E2v#c&
zhwa_0R(X>F<=l)P`z?HLB1GL@ib1U#T7NVuK6S=cc|2*m>N>Mdcc>~Y>Q9{nA02cn
z*(&X>SzboBQK>H3DW|fMPu<~s<G_;kW97$n47K+x0GC{nzl!$MJK03LA-CW+QYj8D
zdXj>G>;#!#Jh~hqxNtdEd@|8m+<ac=4TnIAY936{d@k`kDIFnr<gA(fqL*u|^>kc`
zOk5(`={5LX7Vql|l83W8&Y?Z6s>8o0X>v<Wp*8DS;%Kujz7*8i@sz`=-GGeV7=iiz
z%4REYoX-du#~Hzj+LC0rY;<Kfbe+BcQnjm|y4W>ar*ewP?_5pwmr<G>^_S#fD?XOC
zPp>UU`bwm@Pg<9lpS=oU8cj}O;{)81+`NMy+Hg<I-mtCBe(?D14Q@-8Aw^zYzD6cj
z$IDaoVU#Tc|9eSanG(ON%RZ%wS7NE3N$sjvy)O}dOj2HD{de9-o$s~cdNdj><in?_
z(dbsoZ*z3pQ<Z#)pHm>bHES8-7wrrJrf(r4yV|E~D8mKmHAIR48H==}gQxWHXMY>>
zXdAsI_e5&R=z$<RLtC+3D`ua)=49yY8|MKXQruvKut0Oyj|k%@_^0Dfp@km1v(4vK
zIFBLxP%g^KWE|F->bJ<^SFM4PH}&2tUA~HGy#4jfulM56xd-@M=aN!{2xIqC{g|SP
zA4<h_P%`>zmFYH+EM+Wf(Onnk2mPL5<5nI!AoP;&$LV6ink%ziL@{ghcmGiUZK#(r
z{l@^3)c>TdOx!kZfG0j>Z0LC1*AhGp_Z0#63N%x1Dl{rxMB#N$@lH;A<8RraT<x{5
z(xCHA0n4SP(g0fwY~g4*cRqj?Dv@!bTjaR$XQ8&!(?F}K;9O;6HQpSR_sSJIPQ}TA
zXAAQ&U0aSnBDjj`O4%6<NyLjx-|;!ztouzZa-P5atK1642Jqqqp<_|M%;)LN1F0{Y
znhz-!SsEtnD_km2A=Bw0%wG^9B|*;5$rqHgY3vGNcK}eak`7S96<83oMD&V=O$Y1=
zl^7JV!h;wj-xQNPWoJ+K%VXS5UDc)!{oQSAEM-Zoi!)jmr(?ijSqmfKRLTFP|EiQH
zY});(M{8AyI#GMVK#nT4(FDqM+M*rQPn>~XK1!0K4u?8OSI68{tohJx0HIFF^!3I|
z92aAO9jcprf4JQk*<C?KoRYmkw+m9so=siRT@0G`6+HUUJNGbBa=)`PQ{AzhkBU0<
z6rhS3(#PH7D`q^FC@d@nr)N*(v>e5!ie9Np=hfRJZCCqjjSI;ih{(aY&fRZfeNd{I
zu)Z~;S2ojflh<|l&V9~cT&ld=Mw}w|DKt4DQcw4p&jgu1`>-lN%CvDS`G0ET2YVpQ
zCU5e_lCU@!!Kcd%+HP(eSq0kIpMYlDBQ4%wT!BZI%khT1w1rZ?Si--$jq3l8G7Eq*
zpBBaj*<hmuRk7*t_hA~g6ONnd-MqCu{hVIf{%xyN_P4$J#8>f<9+TGD8AlmxYa!|T
z8k@In0OnlL-&+cGR#tfI$f~d$VUX5v@+-_UX%2v%p#8AP0Z(Bc*|Z-dTJ)hM#!T1z
zR^1DxGZyp4h=(p^w~KL`e~aCtKrBW|5Kw<Rx|qEpP~-28)ZZ$toH@FN1Q5o%@fj+w
z9|HwV?mP4j2ysjC9r3JFKjkP0-4-gH$;3a+4|ID)Tv=FdoI~B`o?2eipo@h+(=UzQ
zs#EH#=)PoqnmKnrkMMb(%pc{v@IKzUHGSPxGfJsqZsKKPj;zW<H7eR3^@a{+?}Bbc
z@+p$q>!<TU<bn1BqsAUHIWvyPovOvimk_(g5AxkWicT{HZ)<O=A2e%l<boEg+{w}g
zEcUuVG;BX0K_lS3rmB=k=L60)HLd_UCWNn6=Mxe9uJ0|f;RY<hAB%XhzVzS2#N6lD
zoZ|O1e>NYdGY*?-p3;#upbyXh+~Ae>Y6)@ka_~tggmbwJ|8oq|UvIAMi#cLFxUKZX
zA+l=ZAd4#W=i)prP9o*JNq@aGJWPr)R9zKIotnj=@#1)1Y@zY;sHn_6VjDP?5Frq}
zy1dcV^=H(ot+^>&Ptl8ZDy$0FVS5J^z87cs?-smsy)=GT*tiWvf0K=Z=<=+ReDp=E
zq29%-i=owjMjqT)de!0i-r$}Xz;G`ULEXmXoXFr%C!p6JIYm=_oyyh_6avuH#x#hW
zh<>>8GLAq8lZVk>NTDyyV(gg@O6-9|llH8sjp*uX+{TM1&D4#_Ov%!G<glE36qyW}
z{f`$HJ}H52VfBEAp8HGF;2?w5GaD@~#tV`tvm3m0EojYWtSfo>g~XAoX+dPfMLV;V
zZL;AuJX6j0UFfNvHs5vl#o{j_Org0WNt4cx4M5gYHGd_tpIXj@WBU>ZJwj&vX<yFr
zlXb7UzH6L>eDK8zF+@HY(F*GhlT}H+j-<?S_*0J{Yg$?Z=q`>UmGVBV_$}r!-UFN9
z+Zg!`Ql2)tM8FxC{i-nn6q-no4fjG4J%Gp=Mbe8lWjo44w8w3pYHWJ>nPjy4j|$5)
z4nH)J*2_$%&(Nq^&g6`dzi&)+M*3~rddorM0+evM3ody{VU4#6>h`-z6g7A$GT8#8
zS<2l9S@+eiZf^bVrY5j!i81D(DsRX--Ki^6(Q3+y1?}}T&7dN@<vsmn5HtoFoLZ^z
zk+SVZN$iw!1SBXjd*17m;jEdB#SuVet9mpGvN*m!u(D!Z`M_1k?#Z0^9#a`}%o{GL
z|GaHbUt*r`t8(JJ*nr*%>zjU|?LWsS=CaZ=zAS7OC3Q>~f3*1o*2o?QN?(mfJ&*s?
zvJW)VBfMe1=Tz<oBsd1k;|TnCry`ejALHADkKN1n20U~kiWte-hF$Qj{dV%0LP0qU
zaaNT&(t7b2C)s1xv-+c|^n^C&=pM<X#}h5qmF>zovX$M6Df>&ueG`#z|Ngs$+KuGJ
zWi}(6*|7J@c{v4jqoC-ll20&c#lbVRIa$hu=wF2t23Rj@RLiQI$TjvCO>d5um>r0U
znYM+#gml3-R~2E+1;DaM^N;=aImS}G%~)u1y{V2S)=}DGO5#jTZIt38znC7mS``B1
z(tMtUXF6Qd>7Ym+M67vouH_P*09Y7;7*B_}T94i28pHt`afP+XW8nB^#MB0Rw=ni)
z0PKtHv)26(c;)OP_T6Eo$41{tBXn20e`sJc7?W_)PL)|xaTl>X%qt#@e!`k{+kiVv
z76x+Ms!8ffoY6B@w(~T;4G-rp=<W(oFu%fyBQPCCgkD9~g6et+`VC`cxNsfTmD=3~
zm;Sw$MIrQo+6g^qwF9p^38GiD05$Subl}+D!??%qt`lXU#pq|{;NIuQzkIJ**R&^$
zIunn^Y^VaiBu&*6X+7|3+hF731s&%(D>?n(9XFTa5sMl-sRm(hKN@tNNQ%z5B1<>9
zt^g_!PY(8mfcY7{i6fTfXa#Ut_&8jK+08IbdXwa1#<XuDkXK7DW9Hd=J79B;LR~<I
z(%A=U2|mwpa0G${eZc;w0cBe3W9K-=mKm(meZ+Kr1K%elP`?@tSR(d^$QjFpp-=Bn
zASrm_<<q3`s>WQ3R1cKcC_ZM1PW{6Q!{l|Fh=oS9eZ%RBrv$z4&f;edlmR$3_aZ>&
zF2w<$N2;8R>%JS@gH}vfh)4Wz11YH*>-ouqi4p4xa1SCJ4&t>O?TC~xPIi~$3Cq=+
zX@jC#>097^RYfvO^CIWxqr1x0y4zg2mw40q<quM|GI89m-!QXvt*);$OQf4J*BXqG
zygXUV@<#30pQXtKg)=cqJKSO@;D3<vCL_rLZ73}2l_b^}OYVN1+Ki>RO1>P9$sVyI
zAL2{oZo6O#(e(<?iW2yX5Alhj&@Uy%CDrTRJJs1Gi=S@tEIt_by%Vv$I|%bxOAqN$
zV1O=J#`%=b4AL=JId+ea3S^meM9Ozh1tKwg-Sf%-I<s0gE`xRjM8vkv4-5~dfM$v-
ztlL*IyLRA=u<O2Imh^3frvjwwgd~TCiRTCrzldWvKTwQsJ57td^YrDru@WAia5(i#
zQ07ZzZe|*t0sbR`M=u=u$kxvGp4l~7(F%VKgI<F6a2UIV9c%kIcJGIkkOymefEIis
z_Ol{X07QJJH=?_k-S|AhwOMmR!y4$65?MPL&aG;e<t`nG=x-%v8C7o3``64G_Y_yC
zQmld|6qajL0l@%Q7tempB~CZ>pd^(VZR1Cmpsi~x;^2GVd=r?QwViIVz0^#1s^>~#
zon#9Oy<r7-UwdKuvEO#-5Gtn7xb__)!e<U{C}vhm{vkQLR2OYyXt&Z^c_GMvjvg7;
z5%WYYN?XZbp4{rIWqb;n0j_YSHzR8OoQqUf4RmiUc1vE2B6rhTUbC#+ii0Rs$cI;h
z;**4SK?o2UY5}C8<4uxhYnk@DS`BNSPqj+nY~kDLZM~(7xYzm{gws>}B;JGp2r}oo
zHqZ<!<M9vQ&X@(jXk?dp3YJh3R}jA4&IQgf&e=|HhgCF3%%gozPGeJIDs0F4c!azT
zt?t@>Ki-~Vkn%RL{vJ1Nd6i1qNhh&Wg0{jHXeYZhZ@FAQ`I{qR`wO>&!ATG_(ZSBv
z7YQDX-CLTkhc1}fz3hZl#PtTQJ2|W*n30)rJ?ZGR@PZQ$L+m&V70VXhF%VF?F<6C4
zo_O!pi|MsXpRgOSN%E(`=GBE8V~K1iGju49cUiNYEa)eBLgy6j^O$atiaunPFIN7(
z=Wh%sF}`vue$|eQ#Q_-+qI1BZ1$i!rE#y>>zE?U9_|n}$V47i3YdvLO3oWjna%il#
z5@!e{TA{6YvQH9XtD0k2?)+166Uv%3onha=!yg12cqropo}+RSR8a)6;6^cID9Aen
zzC=;IxonV>;(N9($4swPDzLr~!a}xR3`paT!G_tfrP?#9XQllFeoFTFCGh7WJ74D7
zK@p^6pBbA-&y8Fa^Eq)8OZ$E`9c^!HMXC8-v>Vd^HN9CqmhGmEyu-$0T@fVfhR115
znQ8fY!@bK-xBhG+|G>QhsP6<9t`L?0;sqN&ALHia?q_%bGdI=P9fj^QfdSzsf$$UX
z%DN|_{wc^RfsHRJ#wzhj-P{P3;;%V6Lf=QtO)dxrMD!z4yRmTz@ccRfC+79yg6tg<
zMCT*~<the6h7%BKUVUnirIUQ)_%Lj~o>?J|zPq)ei*SG3CLiu>gQJ%q%sL{4nx>NA
z*FJeQcsqnKR_smUzJ`Zh&EP4F$L-_rJ|G~BiwkNP@`Z!_5+-!`<nwd;y&@PEQHzLv
zX)Xf=DggJfze?=ZA*^n>h0UYiYh6XD2hfNl9!|aACnvWlSmq>ZJGm(3$uYV*lb6up
zmuqI<PMsrn!`Do(+xAcMrektgDA%j<dH9sZ%_zA0;7=}72Dz*k@>6d0#c~>-;$dbE
z$1`&T%a&Ha%{WbjWq;qqq(if42B+8)C~{Q8eKTiEgJlRn*v@xYD6gF(YvXl1)1g=k
z2|_K!v{>2jlpm+N&D}heM(V+HUWKgN{YoO>S;4*8+9_tz^TMTiog1M<9@aL7`%7@h
zdI(_r@X)4)8(KB=-6M%u0SWXn>0WCTk$HgH+9`B!#eb;~$(U&9K4Bgt<2vJ$fg^C|
z&35^ib47RlOZ_QVzU-Gl1f^Os@f!zfJhyKOhuV5(%0@U)rbBb*<fW3w8muG}@(XJ?
zvtP!na7?nqwSW%GN>FR8VB$-%!Rxj3FbL$JYh+ye5|F|=eWI%_Q(6OZlzUOqXPr!+
z_mTcq7`AM%aI3hLEFhRzn+d_ZZzW@Q(D`zYdpqRP*ze*<-qR|O{Eby2cy?4MTdwvs
zyeY!2WhGM~;zQ44<c0fE{^dT@Wh~j?(FAXApr>!55{`hOU5Pw6Me1;|$z3cSVcb>i
z!c=}2Fja)3gKq>3bmZzPp~Db4Y8JJ9G$rjl&EjWcK}FBgmZ3;*vFFIl_LTRNELpb!
zCGw0&?dL*3Y*_?+DQz@!;RBh|a!SM0C;C-+^PH=l%u*4f$b7q_<>-dWClSAO-Z<v{
zSQ%6I!_*)DraUdycK3r6%_80n@pivCMhSPj!N^ezkSiV`Y<ugSP%;yBYD{F3t(uxb
zQmIwFT`?61!i1N_a(^lgr<dsXz_&TEhCBL6qEKhPy1jp1(`@9<_lnJhCwH_>f-N3R
z^y+|AG)nAk&e$I7rc(f&fXWwP4y!?mxTsY3HcG{|y!G2?47i<6JAV{hln0}aBgPd7
z-aPG}e8)Vj6+3)C9KyFT68T)?Q!mIxl-Dbk$9`io!A5&E16XCA7ZKmN$;J|@p1$c&
zCEd7}U-KYLxuVgF7LSCF#dod(YDinWj|A?)9{LM4Q$*_v1Hd56T@kr;!tWWsAStZM
zCe6ZxAf9ZwNtShL-df}&2XFB?!E7_%yDxh`WT+G)WFx4|sa%*wT$mR{UKd&3NxFee
zUQW6avCY>TBWD_(<0Te>rEDAC&#SuoP3pa@<6^DbLMi2ZD^x8jS!xJelk<L80BZI#
zbXngSx915~01XG&vpTe1Fo8^F-*zEt6MhnAPMISW;K<q@iM>VOJQn0F!K~63sd&Su
zf7pdwsx4~svSF6<X@2Ra6sc1`KrIt;`z)Kts5Vlz{ERGFkJk>Kb0n6TLzuPUAzGvg
z*F1t%HyXxe95fu{^2)y3|I)(Nqb@!dl4uO~T6wSJu2Rm*(pd1$ve-<c#?PH`GudmQ
z#LT241+tvqHQ@`BAj`^MgZqbb7Uv7raVl*l=z<LcgAK*6p^p+2ShL7pWRS`EZMnb9
znI|{!8+RX=nFa;ymOaiTw%0Ij7=M6)zEv`vpV?kfp3@}4Qq=>Ol#k`eWYDnuHId9W
zU1_iRj3i{0)8dC3#)DI^abQEVh=!uj>AKqxt+b|9^^-i_(%}q6u}(*6uOj<C*Sq0L
zXDxN@^tLzQnHJv~k_6mHvUqC`2iWttMEx{n_L^{Az}_fg^RZwZx!@2kbGZUXTp3>t
zWysiwyU2?OgKuw1%7FJx#<%yZgOb4{r3(`Lu_h`^iFh=)wgkC<S@$AEasn>sI4?^(
zfWTEyG*@?JTreB*#u13IQvvGwgx|85Pd%G`n5aZXTC_B;n%-5=qVp-ZozpPGpg&Ir
z+F0N_t`JwBheyGCa1>OfrNIm$a*hZ$ApQXu_lB*z(EH9W@76Dj<q$)q6?cwI?yP#T
z?Ag1gt;<-von06E3dy>?m>gU>&x1p#^^BD<^tWF}aFJJ*&QH4cM_O_QW;bXbwZU|3
zQI0l+p#EE;C1@=pUs#PhGtQ1b^qm7$0o82We7$c{nf=Toy*@t3NzI@F!!mfk6?K8`
zi2~}R#f@0TmXSW<nbIFV#Le}^NYUv*zf5ZF(Ws#@l~_c_(D*m20_K8Ek}?CY9C7o<
z9|K7?MpZo5v}dg2l)*+F*mv2X(Uk^psL}Qm`Dw+o2D@>;i%VQYL~DEMmXJjo!P|4+
zJkOkXWR$R5`4K=9X#M8=JL?<$h@o$M6C0H#y*Fe4xK`42!4@H1`_4K*r7B5zx|jQ_
z#}}fus=cZ^7T#{HuFT5hlKX8_Nn~jUq=ICESe>x1E#F+S=y5UhNm@L9&Ju;+F}>l>
z=GB}oqOTGjyl0ddqn|dUh4MtgyBMw(6$6s1d{fFJz~W>&4P{<uba6KFd5uKfA2HR$
z%GnEvZjVtT6#@L!a5fg|?Bt-UUV1j`wo40-sg}ARCuQh+jaT$I<+mw7&6UuS*H-(J
zEoo_($)UQda^WugGx1PfkMT+Uqa8%eHInSiDCtRfK5^&5x1ArKLU1_de9=&T<+vi)
z>;jE|f$U6ZZfRBOE`-`gz0%2xAS<0@_#gR$Ln9Gh>m;#h$H(?-1L=htm7MNi?|Y#d
z@E>9hgt+(~zhYBbb7Rl%gmG(>spc)g#$SbK57#>zU8*=t;9O<TAZY1{0%&eK?loUy
zK$~@LgC^007hfNhhhCUI_|w8!%JDMjq}`qBrJe0#GsCmQ5go(caaP$(emiASGnO(s
zhGDVeki-(6>k)9Kv9!9a6ZUq3*4~~(p$Mp(wVkr2S<;WsJ$Li$w55*~-<34GRr0y3
zRL6R@^_Ib(g~RVH9RIBNN8niTQwf|Wg4C)F9xFDZ<IaC(iF?2AF>eL6!>pjSZ?Q4;
zE!b6b5j@dy$+Q;hxeM($4X|xF;0@NUJ4fGP31OHRds{84ktx<OPgqsL{o9|CDXuL7
zPGPYXea$m}vg=yAdz^(`SbW`jobSMCsTmi8Vad!zJA%4CoQZ_Z-HjpIsWJ0V0!a`b
zzK!2;aT#hHaLetsENd3;3$qKxC&R57XX1|{@j&Sg%9lfYc0;6?Wf-PUVG;mHB#osv
z=KnBawUnh)-Koj;ao)oSsJVlks^F;*iS&fi$KLWWlg+G26UT5Uk9IL03xZdcm2Fd3
zmPEqW{)A`iZLtIx0PIol$xav?WJlv-x)ZuU3k08X+uvpkYC<sf-LIeqv;aH(_TZ%s
zP2={R`?Mz)9}SvvP<T_MUz0NWE6|8e^TY&Wz*J61p?LN?s~0Leku5JU$)0ey7|Yz2
zG*!}_3A2r;1_T(a#{CXlU{%xb-@lh?+4YGd3()WOza$TO%(m!w3%#w}%nEjTq)+B?
z%{nqgGZmx2ZudciVId#@ES5TBPH+7Nhk44fzUxktCt7>wH(~xck&NjPX1l%Gcg1j+
ztW@aFvIvJ0uw=4qw%(T1V&rrel<Z<S5!!!j>;oRV>i#p|iz8hG@`XgAp+mihpF&SI
zKnH!{EHbkZ=o<*ak3!r)T&)ip?=#nAbbR0!b7x(Of1|QR^;b3{HvwB9ax@_devOU<
z&Iq2EA;v5VH19w7VR-^dM1jTj2q6HEzhZW=d$%SKLBYcAD(K_w=au#|J=oV6UwR#`
zQT8m7;sgVlh*<tl;>Ku%a5x5ZrEILdc4o&+JsgZqN5}T3DMk3H|Nb}0&p1L5*il>}
z(9Co2R^+atrW=IkK)dghF+}2jZqR)SxRrm3Ut^5-&-eV>4~ozQ`u{ir=E98|p!J1<
zd*WYvb}+NYjEZMkMev^|0_^!MrY##!x#B-h1Pl{ESNQ+69$D7g!03pYV?pBix2633
z8xV^vrq$;^F6!xj#qw{m`L9_1)nxt;isgzte%mBs#Gcv^3;cT~|KdsEKQ9CS7cd3;
AqW}N^

diff --git a/docs/figures/arch_7.png b/docs/figures/arch_7.png
deleted file mode 100644
index c77805365e34413818691fb212e700357a380932..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 57736
zcmeFZbyQW|-Y`lEk_rMMQX(KNBE2aoX`~wk>5}eNDG5oDl5RG=r9ry8QxVv7ckFL&
zpXZ$Oyx+a=|96ag7z6fNYtH%W`KvWIA<Bx<_&5)6P*70tWnN0CprD{rp`f4*Vc!Bz
zsGvEDC@45k3vqE}8F6tMWd}P`3u_Y;l$RlK30P{X>qLHQ3rA5yM0cWyJczT1MP=cr
zS*{ecx9`Y)^8fb8rHw=nm-&&FhLgHPOBG7F2CJ4921%W5l%$wG87>d&E?Usij^J*?
z;c7CX&vk7;gbN{rNaiB*8~82rcsh?+G=fj*0p;&rS$SF0&li4pI^S^IXb#S5h!o$x
zz5n`I4uXBx7WG+?-4tHu#px+ReGKD=o-hhgh<yP~LR#SdD{K5c`Vk1qoNb>igjeZV
zU&2#U*)lXvpQnB6nUJS_`E;K;BhT8TF5eVkistOQjX0rvy=`i?ayVMX_uTi>b{pE~
z`^H=3^$rfExKBF{0x(zUTHN@A1&{0a)_ubg3xd0svsv7ZSm|Dzkgdp;yB{J32lt!k
z8FhK$Tk&)&@hsjw&nc^o64b5`Ct+2@kGyaE`E{G`d#<-F_QrjK$8uE0sDmPEWxRqA
z>rbN+{(6$h5ZkT$xQ^&X9a;i{I-gk-Haf|TQ=K4CRu;uKgLfQA_HmyX5qIzz-lG=o
zsL|$3Ul#VksZ%2@&NF;Ma6maok;mt$xZ}D;Vo&0E_{=Df#B42mlH;CZhw&-{vn4V1
zHF}5luX{Jq5JO3J8JDadhF>VNO4n99pPa0|A#?Du%lbuOdhp^==lvfv*jjgj*0q9f
z5n!N^{h*~$;<G!82w1f4_R4=EL>JlKQY+u}EA+fNMvK6OKoO-?NYI0RVUB!VQX&Ro
zt2>OjbdHLiX;0yb*_<wo!n`=$hg;@SiN-fMm>_c~ZPU7qf$=Q>=cyRQQOkYS=35V&
zF(J)ZXfzm)N<Cs;Gwji#JBZ>qrr(Kw!5RBQU<xCy6y2pom=8NB{jOrO_IGT}boAOK
z;jVPD)K--GKw1p3m)MkVXl8;@jY75P2(T&kU!X-jef&V?g-Vo@#HX&P7jOAhLSt#;
zx)if+on!JxBulc>rF7|MQLIZF(4{Fqloe0Rqq6*{C^h{A3t!TBnA}`I0E<mDY>3+&
zQzx8Clwzm?u3m{7mpM2r_HZfo-B&*GWc-Nm!&b?qXnPAjCfmCu7@84<9pgX0+82MP
z;N#!H+@VViHe1qmqo~IczE2&rw!F4P_EU42W&d&y|LpLL;>`96<%Y#yafn`vc<#2|
zi$}C6PZN~opD=FGKBUv5ak}eqdpB4jgJC!sE{n&&aF;6tH@GllHu$)Wt4+~}&^}7y
zkwr|AOllT3QDm$%GlLB+B~1#0KI3t;a8z8^t+!YXH2jII3><U`F{d#~pP?~n(LqlN
z?x}_f(g!^H)|Xq9y_NG^wd9otleijp0snZ+1G)Yz_G}I1%{;O^!aTd!*w|08bFmq*
z60&l!%&`Pa`YIb57cW(FwTG|FM=hSsJeV<<A)nEop%6&1l{!sl96f|v!MTqdkED+X
zd+<zjO-jl|-OC)}90CuVR}z@-k*JbH^vK7d#W}>$F;A))<f*9jt6i(P7lf%5sznwq
zXi%yd=S>$%=B?xvYs^Ajw2PHevTSpxzOgHFs4QmtC^iK78{g-;y6@HD&|%wwDa8@f
z_1w%MF|ki&Rm@b-QB+*NRff*MI8`dHLk7QwL2*<*rZBO%M&qz_m91Zi_=w>h12$vP
zS7qLx7w=DXh^xFR-|Nup9P8-oq*ZFqpw4K|Y*tZJQEx0c7dd4(t!`LP)_glOdgPeV
zDRI*l9TLR2Fe>R>?3C$D2HzR68KoQ>ox~l9f!hoV=AOPVR9%!7QT5HdVY>D~|LC9K
z-}O4P#Ujv*6`wRHuD{@QiW&a>jR&_VdhlEyR8tU;CsMSNvvC#hdYH>|n^NSG+da4C
zq<YHvG|^JUT*b;^FxIGtr~m1^^^E1kLf;%~i*>ekrgmCQJpW{UbN$4XKj!?8S)&Na
zs&)mr4Eot;y$(qxK_r$d)^TstO`Q85RJoLqYTH%pBujkx%_9TJ_ZYFE_=?}ju$VKE
zGpjMm)8~?Cleyj7%@7mA2u+(4I4?}8(664it-CzgIH5i1-AdXv6x8D1qbzu6D=<Qd
zPDv)n!{6vo?m{GB<667i_bzp#ex#=TJS?m*Y%F^qdvUytyWP2P(}pL8vfH4)@we2w
zLucJBmmk}ibzz>Vmv2s+Al8s_$cg8X0%zKz<Z@5h$lTA2gO0hmb73oSzm8)P=4OYI
z*7Ucpr#h#kmnu5fm^!0ly7`;k*R9rAj)hh)7Y66;(ih2nlYOInLwr5`JVf<GO`0=B
zGn<c^XPW(K*1lninFZW#NpIN<v~xWE?q!SbO4G*u_D$EhvW~gDxe0vvY2i}~twS{@
z%dRneRqFRiAH!NH=x72c%4mPR$#Val++~;<{Oa>7^;b$6+nKnTy2kf8D~wJ$>%)%2
z^(j0>1|BJrP7*<hM1&d{ZoC70UuXwF>>YmnXe*Qvkjco(d8^{!b&&eI#R7GdM`1H%
z2h!knQB1zdwc7t3qH1P((ic;qX1Z@mDMOp}IBQGkyf17?{qUA*O}?%w)|t;$|J52A
zb&Hf((~znbgV)#9M2+}~gq_4j+6IwU!A!{Vhb|>GCXYDs<Qlp|Zw-lwg(AD@+tbJV
zzmsE=myMk5<Ik?Fucr@9U1f{t+lkH}p7JNH;=L5l_H_6>M_!}gE^p0e8Kd2$(N%lc
za8P(4(ifiQ`8IV1vR_|)u5qZ@@2hi1Ywr;43hgE!{zdDL><{y}{J5^{FG#c5Ew!!d
zznk0dMt`Hfj8{o8WxKXXoG&jou}ffNwPPP)(NP(zcw{bVQ|3@sd3GWj87Jt|jG(zl
z_l{dGU@L5l$CfRLor#N2@G<B&^j?v^q-OECs%EfJxYS9gnw`8m88SJqv)kZfn5d&I
zrakyEpW*>eIZwQ`SDnQ~PVb81iXe|FZ!b?T%Ma$S)nBZ&VJnU%QYM;lcIwv}^TmbL
zjf)IZF5Y#6C$*Fwg0ik&7cxhqDJf!t6b1bSy{>m%S9U$h1$|sM2e;C!PT*Ti^M@u-
zH>j84QR5+Z2{-qp^C`4g(#oXj8Y_ny&Ik7`%^%3IHES^SUZ>@IEO@k)5Nl{lNgy=m
z@ZLLXCvSbYxbuFe?1QL}=lJO1<lzT7SvdpCw4J(Ng}z2!TV8H15PxJWP_#_e^9=PU
zfRDjOF>|p7Uk=psCHYk6nrQT1JBF;;rbY@Z`p_Qz$PV)-8?18BQ8GNJcluR+xw5P|
zrTG9_t7l{GXe)S<bZFO!bh0Hi238yIHqCdEN8C4RPme2juk7^dyBplASn3N8HRg&1
zVY!R(i+AVN>yo@r#~ZFZ#ZMC;G>|9mc^9QupLa4h7TJ8E33XjcE|;z_p$K=crl7^c
ztH?PK;ZstfbP*<B%e~5+!Sr1T-`BpK6&8aYX>zHA4Nq=Vw+D+k*bJzyd)J`V{%!sX
znj@Nv22h{OYtvobTo_y6=fKE?m3j&f!vnF)*2c)K$l^32VIj(|mx)I;zrt1$R-x&J
z)Ly;kCkj7e4y2?|A65@lKKGmSM1{+sd?G;Es2oo8kcb*a_}vXav}HP;27gBxJ4cat
z!w2c0d`M-H@S3Ze-0x){)h2pgW~Nbie-mBBMJe1Efiqfyc6~=l(JrA`<gHT#qGZR{
z@_xhy24W0>hGlsx^rl{#5M;2SCR#G43JNI9;2IkR1N8w4Cb&Wczrv^#|Gt(&eS(7i
z=Q$b*N{|H##@};5^84o}>hJr%KGCB;qF{k1MBvx$8`{5Sqf>oD|JOCz5OSTEs<?~{
z_*6A^Ffp-pG`DkN*w;4#4{pDEsqKh@LQ0SPMU_!`ya#BSuu#)-(o&EYFt)Q{fAiMP
z$b{X^<{c6a3dBtST-um8y`gclv9@&-a1(m;XNCZ{Mn2|vMDu5gla<gTEd^y7aXSYS
z8Xoqi>`xyF<IvF1Kpfth3aCg({k<IA2|Y4*a(XAg!Qtxa%I?a=Zs%ae!O73h&++sb
z$FpZ_U<R9`yRFk3H#S>Gy1#(@3yy?|qp^d<J0}Y}TN)(XH%4~OPC}0!Arbxi=Px=<
z+${cs$=302wg7`1$Po@s_NN^Gh7FcNkWU4aE!<43wIwWUfXo06Vcus?A%EuoKSut8
z@gFO-{;`su@1HCGG4g+}RChFS5Vx}dD4m4=qprUf|8wx~g%A!T=l_tzUt<3A6v$Z^
z2g31hoeAUgk{o^mYJ6xRp`->rfsrA9P|d-gCx3k+uXTR+-LL_Li=xO#h^e`uZp~xG
z-fN%f*mv#b$9}?0^!R~j`+IW7?_V67gK5acFeM~kp^{*r-hH~49?b6<9;|(rjwbZm
z6B_1#-+e#4j^k$?&4Q+f1oK;)2KRT~?`#wpa-SY?&8w#nbB2}D_@SU-|ND={y)Ig}
ztXByccSTW9G4A~Jry1uCgzwJZH-8?}Y@=aBRuWgB|7!~JDUI7Zynh1%AVp^|v8gN=
zO|kw19-BsVZ|VL&K%@9q;@*LbDE1Nj6TIJ23+MkNi;TuCE9DF2Kf$B41RDQMBOwlC
zuNnPdLf5Xlm!pYwr8hxI4nKiSmmxRRqhQYGa(kpizpipC@CA}9X}_hPFZam);w};!
zkAg?uwtioVNxdMp{&c}}q}pb-a?JY2``aV=s*Fs!)kToQJ}!roX|r4tk$a&)Sa|^R
z^1m?O#-oC0-NL*7TD-F@gm~qkmqW>F+qPzfwB~ZBE?YkF*}?A^r5;Q&y{L)nSA1yL
z?=}6F?k+Uuu3%>=EQN72Mom|k^^RoAWs3#gau^cBk_jVa8>tq-Vb@#AP4kH+VbPe1
z+ZZoo^xP_wZp44|IaoeTc;2UaxA9VTyg>cXdoZc*HsokX9DpFJYn6#%$QE+qW7}M8
zYQm!wP?JqyXR~WO&q;Rs6*8cCB?+ijVG!MWNluSR>(^ZGe)xOF3K|zFeDQ=@N>MT5
znnYcxz;dD}akoybAyLG)5nnKlSv^`+k)K|@J*W5~J{p&4U(}*6LSMiM9#ws9-eQ()
z|FQmL#>$0Sn#w)P>uSIA`~>EHXqvB_9&o)Cqa^&9E`smT;%R;Nr{;r|-fWA}oN~)?
zW(r{{uf3MOt*I@^*0tM=$uy%m3Kb$ma6&8(%=ef;H7~PKTIk}w#(oeD#w6u1uvXGK
zhTg)zUa8lvFg+O8H0=KI0k^pCO%Nr6YOW#)K32W~d;Q)Ecc7(}qKbj&1OpGI@%G$$
zNJI8Yc`thfY}9(Tnr&sJlk`3X<mhm>31I-HroowYtF1!_3>e%bw4dF)fI6A;P44Z_
zyDnz0EXuqI*QqrBTxmHjs{u7&%ajU_-kGl>s}@+K1*W-!*#~{diZt>OG_zE9)3z%S
z<u*K(@cQeM+4CN*)1ukkqf5cvdagy+d=KLtIoRdX7CLpwoT}vR7{=L)uPv<>^O2-J
z7wfsw*HwhI&YSUeX{({qN1nux1MkkHKI|_ncy}F>Vzz_U(tL%v1Qklju21tjyA%bt
z@1283L`5yqNbKF1-+#Tip^gVKizXg296C-=IrNouSoA*a#^;?Y7586D^(?7B`xSPg
zQAjTr|FS)lBy$taCVf#@XFxmczLt`iwi%|7#4}yUaN&WtK22a&FM!L{!He~M5!Wzp
zp?6E7o=IJ=q$%kI_$IZTG-w(|GBn>9k)=qfbqbFasLKJ}b7(jDG{B+PhCb)nV<q~!
zNAZsvEMv^iR}u_=4~ifvH|ZbL9rW`qc74RB$nv?qI1$OkYlt)5nX5V31|p(ceLQHe
zf40|3n4-S5*l@a_+ZSTkI2i4|thYZT{FU1(UJYuXe{;4R3Fp{oZ}_!8Y3SQsx7%n1
zea&q#(zft5Qv;LDLl15vE|4S9dC|s^=5vuRWcroYj@@}<to!sB%x9zEwf*Gda`rPw
zm&JK}w&N(&&p<(N=lilmC*xzCvDx>pbr&60dO{|&b0gTyEp@oHOJDnM0n0fl2&^?X
z^Z~|la%M4Io^9ip;Z1)z?L4W^xZD{D^UgHR#3?WqyBzjaoq>s5?FCvdHtLU!okPFb
z5P9EhLYsP0J$Edc9`HM49c@i>4O@DB1Rf+-{@d@w4{X32`TJt_HNa0aRU93tIT`J4
zYs<q+H@_fqL%+n-cx`UgZkC+XUG6rCEUsO2ybpuBtV(b#o&xK(I1*%oeM9tx=rwL1
z^%E;W-b{*|Oc~SJ0Z*0%I61GLtp1Yt!DUKO<mqVSjC23J+}lmOVjVudvo^4m%LRGO
zuXfa!<EKA^s0U#B<#x#|RImjPtF})orOs|g!_t}Uq-&u;w_}HGQalQWhoz}0Ie$LV
z(b&SR8g~|S*}8vee^hmd+hlMZVkrNz%gn^~XGLeYV-cV8P*X@er*Vgb{P%v%k0n+C
zHf^8mWh!OfwgumHxjq~e5jkkVqLz?HnSSG0^_=nKHcV+%+<ELm-}`vHB$U&i-{^;!
z1T?w*D<4z=9#N0Dxh&}!vvKki8qfv-!e0&N@s016+Kzwu9Z{M+G$D_pQbuMhD!14E
zHsi#PxwOVhPRCT0R&)pX6P+j^&-cvFh7f(@n7Y*qkA8XO_(E$p3cM~I7Wi7kH?t7y
zx6k?4l8s^9=u0-*9?#57;SM|@H*CyqUkYhlF3*<s!A6tbH1%nLZNhIGSbHB=)u`N=
z*JL)}YcU@BCU%#rq;2PPDUdLNR9|h+ZKr1K(Sv7|C94H74eD~e_Afq>_fdEE_lL)?
z)jwf&<74a%2?%okDA^`YN5QFSw&lb9@GFbaNmWgc5ki1*a^NPpX|+K9sG1(%f7LQT
zh#-0Ux$+kljDd~eCdIv>I|ZHZcV_XfV5m+_*C$~+TsLRoq)~0S_!2WzTpg|IOsgA4
zl?F3}9)0spqY8H$R!XJ9ADjpZ<;<}TG_4s&@OSi^7KH)RyT>HD_ar))<c{dKC;q@c
zPp2+@35q1CEjYw2cb9c&JeGX`F~YSM>}b7B<xy^CD--f%HaMdlxHY@sm<@CSKAMY~
z!;Gw!)@cKTsqrW2CZFp^rX6MQkb;_&ZjJoXrF7b^ng?0N;zsBn4Qd=3>p(~~PBJrO
zB!(paNG_Bfka5~wXrNm?sS7qJ8*Usm=5QHiv0Xjkznu5IIZO+#(Pz1+s*z+jvb`){
zwAc$XOkPS~u?*%3EY2*1@Bgf<vYIRz9#q{oTIq=!79M5^v?X=K)vYYvpS7rQop;u3
zn~S)QP*30O=Mu6`bl#j`a}sKl0AX(q<S=ugWx{CKTQp0_S6}j-T+i4~(79rwh0<$)
ze>4_}c+2Gm=QuB8{G!-?ak5PzM6Oq18h%)Iy*arwoU25w>$~=Lp$4-6un03-5BZ*2
z+L_uWNaJDhqDf@Cax^(I!&j9Y;zVt_T5dkAxECxE-&etrac$Z$DlzG5fJZG1xo99?
zTlTU>2(*Z!OJ&R+_C0TPnN?GI{W>+Bs5pjPDH^Kyp<%k#*{ZGYjv1_B^{W+*?)bQC
zMDbTLCyJCpG;MMa8M}x<p7rEU;1P(4OU;5qhd;0t>%yuxQ=CR*&NX^X_l9&WiP%hK
z7vSAXeTiI$#lHXpg_<0*1Zx<xcw>>~dCaVC=HG@I$8{r!*z6jP<F9>S^DeVLH`Q(t
zZs$(ziwm4|%AY^r98Bqg8RBc|7;<rnHV&rnn(nz(8ZN_}^>8k-7i&xu%-+eZ7*S1;
z6~`pbHLDMs{_=xYKM1ZA>HFrUY0?J00xny+cdXF&Pn;~kz^2NN&U%RPf^`WGgWd9V
z(ZWa;<A?oa_WJt}Cz`qmb%f_ioaVuC$zouTV$8;;!TW+PMU<@DmE1l8`fDeAl(Y6=
zd}lB1<4d#IGZyT8JSxfgjy0z!_g~e;N>^-J1E(HI@v4US>}7JVT9z}y*vn^g1l+E!
z4hM$?+La_Jb(<uh{j3^y*JLwB*&Emt0y7fX*oGM-eN0U&%8!SQ+aSg{UB}v}K*&!@
zP+Bs-s}u#Hpkd-vT1^h5=D|6Bd|(@BI28{f=!bnJ3k*usc<%A+Y|1!tr4R~8=IJ1R
zTN0S#Jj{aQ@#_Hvq;IV175Z9Mb!n3a8f9J9Gq*OvvB-I)Ei1FvO%dlz=A5eIGcN1f
zPGxe{0YMQ-_8VES&>H>9G^Y&A6qA1P(%qw&kJ%EiE|O|OO5ChjGN<vpQk!!EBysY7
zORv5VfILLB3?q3~_b60KEX+OUg_CJkf}s!pqNM>IRhcHl$#Q&(9=k<6H`ZBE(Ycz<
zY{|)bZ-|H2dTOX77q$;Cmicg<t4VB>V2wezHbHmj6**NPVF=Z)<C)nBA!d{knIxOW
zp{li8EBG(~iSNq@T;hJ;Ht-|6Laj#IY3>sh#afDMYEr)zGLwFy0!Vs%mj#&RD>PyT
z`QX$(lIJAWTYlrzzE_-vpMeMSZ`zl8Gq@kD56i01A$i`h4Dzq=Y9^m=WyBDiueM|7
z;|-;T>lK`ZaloJ1JKjn(v~MT5OPQI@n6%&03OO70F&N^KOD-q6b`Fphiy`1^8esgY
zILS1aZ~P{Hp!PyDts!IEd@S$PR_E}ffI;VNy@4WwfrM#1DuK9)36^12OU3KCtMS5z
z0w(40`J;EpT!;7I@^AEp9IZojNjH&6-99L4-0m*W`T1F)rsRy(s7ovLER#`g&EzN~
z!DW=Myq5>K_=xadN8oes6W;kWADIf-l!<rWaNX*}Y;?&P9gkTYFVx&EO}+U6U0LT6
z`bCzaq0uxd*0oO<!=)r1nfjf?rTpp#F2~ff%H;FIwa(o~*}ExX!+<~j;R&p5eq;V$
zfk9kV1trVXsRL((hxg7XXEK)wE4MEUnFFP_#2*j~JsEP}`9KBT*ED|^8E`8h*XmQy
zFFwMbYJ`-qS92!wwa%C&1^!CCqcRx3B1Z#wLWS6uu@=cH@zdx4gY6_VZuc}pkAd@7
zMx}hZll+q4A>YtId=|wOh;?_@EZ_S;8L#%k4ETq3T|f>AKT5d>+lCKk9?n-7XzdL#
zS_xkIZdXlaFzR?%DP%CorYxn;!l8Y*WbB_btmeMk4__}h^s)^PrjV7XmJ2YNc^Ks~
z)_RO=jAa6^iQ2BMWcOhhi*+Mo1tgdP>XrrZ0RhrmG>*CK`z_+sm6}ikAzjTB71xuk
z>0!(Dkh;_&t5W5)v-X;2SxQe_O{GWn@h*yR%pPdX+c5_Q0Y5;5-k?J?3xzgbSRPrd
z3gExHitb-Z|2bEql0yh#9XJ4~ykvoTA?ffCr67IffJ#n<F8`n~(+)bu@zf}b?hNAv
z-aS%K!_tfc)vJZ6P@&P13=s=^!@oMB6*4A0c$|EC0sBT=fXrwnN5!fy-tn6CiS@>_
zWiFDCLXK!H21e7I4KHvT6&DK`zk`r8`*ZSQYU$_LD<!Rz*+x)g3lEQFUGz4^uUqG~
zv|Ct#nk|zspM}*8{wAY>j^6~1_X3O3N%^QJ@MDP@8zv?p+amefoiRFLH#q1EUKU@E
z&uL2s1`|MBZb@veiGRc!RzG9|B~N&@5|owF1D<}@yS%;x3UlFy9X-aO5|GNYU+x&*
zKJJNWoJ9C=qa$#Ed+<LA+>ySNxg1V(R}dN1Z8oBQ508xlf^t6F^+i~177fO~s>>i<
z>q)Ba<{Y2yOeXI=;&OKVVgG7{A3@;ku_ZAm%4F%;C=KFhC=oCr`nFSS8f9yLP@PJ6
zoNOiTzE@N7*%-?=POG`&8y2Tpnu@V8`tA6AFMr6$toTuaUPD|Pe}1HI>c@(PL(Sdk
z$r#sMyfvusnN7zh@&U^eC^6(Y&3cjgKuAGFKPc)*hYN`?*;I4nLvAvPX3hC)!&8r3
zcIKuhehvukvYI<y)_APnt18=`D1Y!J_f;gdNaK*ryx00C>fokv+1_Bvw<VW1zrr|(
zNQ)VvWi2E2BZli~UI*Q9lAN>f-I}#jug<`aB;Gv12Db%4T~QpfKt<BTXDpa}-FCxP
zWSH!*;C(s|_p>q66}ml%I*KXOJ<Gq(zM)a*L{K|n;aAn@;Vc7vNm2z@e@NMTmx9O1
zvms8tcck!9J8+C~jwG@O{0d~JV*O6n@QZ$gOd0ZqiTEi@CL~cuz3+DFml$(SF1ir@
zyUgWYjv4f;2^yTsx7+%9=vLo0cG>*W974!(5q_<>oOI~p_-&N4A8x472i+QJ^!C){
zaMX?YGArMfJQT`LOUm_h`Uci2WlwD`*uUh;sp;8vRlv@`+Ftys9=5tZRQ-^PtzZi<
zY@{gA{{HhXo8+HWpqge~t!COZH8mn~EL8f;{gjZYvYO$w4kNK}=o|wH3%u-gNknkX
zylkIL)t5r5+mRpCwpi3nu5K=OjY_(vmNg@p99P8q##0}K@un}>X3i#Ys-B#&4K(qy
zX(5U?gq?LAXOq8RJ%~TkVlU4q0<5Q#(zv~^y|jR!{Uiw{r1}~6ZN-3iXvx}^yW&m1
zo(YlTagNr{%*WeLiN%A-C*Oz%-FROgPqxpkYDH&E-TI&5<qqg6iEWK9Rdz!n#rulQ
z%U`#kadP=}(*`|^j_(k<J{ooCrcapiPModY8t<sw3jWjcp#yEFr=WgN23g>X>Fq|#
z%7ubF7=qGy?3G0^jC{;CFo>Kndg59cbZGu=QGvO9Ev<b^bf&8X$`6Lj|LSx-1L<>D
z7SH7i6x2a%xTtMIf&rF;nPq8(*#jlXzgk#b0ibn(3*RCFg#X>?@-Mx82l8WLx$^I>
z6LP?x75fgP+;4#WADuSrB(T(4NYV2j&;ASUAF_K#duO^_rP@a8@hmTU{I;&9++)|k
zFN8O2=}z?BOsX=namDP}%saH%EaM5u4&3^&0|kGYk8gn2dY=!oU;*k?e&1YujQiMi
zMYG&_Wiq<XGDkvw>RrX}FLzXOI($^BSN&CTR>f3m4^nM;%=G`L3I+5i(dKHN<0EY`
zqWMHY&ZKt)X3}eD*HeAmL}C_W1XZnd5NFanRbztncB>TpU~gZDAo-)k07lV0Y=g9s
zM*z|{EwtH*ffxl`#_o6%>R^d#GOHvaXsoJi&$tB($pbyw>_Q$Edrs?MWJ`9*5H%pk
z_cG!uGIG-s4HZ71@TFqYMVuE&%re%!n$`)DYiF46x6#6nVGG^<gUjo&=-%EdvsdZ>
z7Y(_8SG7X<IxM8zV)&lp2Rh!wjOaz4+)rwGz9DMnZgIb3OQO8oV*zl0%N^ias|&_?
z1a7-=f0vw#O$^qI_bDDrAAHCzeXl~|8q;$*$7E8lo$^D42?uG~&0%O5gd8V0&R{QK
zs5~h>KV5qpSX0&07nJlvMO8~Jzg(qz&O^O%;hc#zHsjA$s0Y2rJ)KBpY9xK?Xsa#-
z$07?hriC6X>hz#B=HRUKxyKX#&f&YL%AKGee^@F9TemAh&=c9Z9AUmQ-Kae=NcH+w
zl<(Jx>#W+n@~Vg^g}UR<GtMpw&cm@FU&p42GC>WHju-TsM+z>VG}wCf$)@QE4<X7G
z`?zFoL;WlxYc!dN9-W~#m$b`tmCwv+=U?6DXFye5$=seuhDJDEr*@XvPT;S~cCM{2
zYKESEOE(_l@_iY^wb>Ia$R5vJ{4sI1L^j)v<KACb*#Oq+-_smO3U2x1JMMn&_pML<
z7(J@{)*AjDCM{l9aZr~+AE$Gqup7Zdi4-A?F-SW;y{e!?UN@j6kqVhm;qfsy2+D}B
zhr9@?ND5W(l>H#Y?yzw1$8$7y0+Ca@QmG>&@k4(btF`X0P(e4>9=+TQdtY2!N*8sm
zaoZ|%>dzzUN$ob>x&ST;Y7^K%R};o#ND#rqL*zvh1K(eOJ#_yx9JWYGE^ts_SgV4b
z@6q*`Yc8`MMj^5NM_D+?WU;3;0c;?@Q%s((LFl$R&)Y9a!t>wQiw9N6e7Hz>DR~$r
zhRmN(P!YhhUd_6WJp@Riwkb$^H#peuauVVm_5R{<nsP}D!qE_M3XLtO3^M#GBlJhL
zzd%K3Pg9T7?A7B^+llaU=X1u!e20)1(gvG76T&_6Wu8#G8Cl6cvk$-?Jn-B@7C?L9
zqB|Sf<tLkpjrn%Z(pxRwj-xZaOx$2gf&FQ5-`V?&($XrOBB>5m`fK?+S6p^am{524
z{GOi7$EKY5UhW+6nVBufqdG+m52=T5uYvAbc)S9E`M7DkzU-hc_qf0`weEBooEXm6
zDtHmZHVEmi_ee6!R%@S6`E`!$q|hu8gH59(5Ne6sG-YTMYa%pBq#~4;pBW8vpF91o
zQl@^AM@afs4DfenF9A?wcr9sz<c03e=kc^urdeK&!%8WE7ePGV)y@S}u7frE3Lh%L
z(t41!2=;IA%H#Lx66K%W=|88w*gaz>;8~lxjzg)KSy=7!vS(~+(Y@F3kMyZQEb$tW
zQM(IJ1(QrY^?h$`aAmzEo$8UYGn5|A*3^lqlD~eOYRMb-hny#RpkDh<xQL`Yl~{`_
zv%y;Tv|?=|o1{kqK6D*hG9pnxdGNmRZ{kA$@r;y`+(_a{f`6C9s8gqHbx#*PBN5p8
z{Mr;&U_Uk+t(t>_bqi4I=cfb|E4==sAOx_H*F51X9OKM&=|(`8nN9~iRzIO&H?gAJ
z;m2iukCbARmfzCnsSip>*h)CoI%n^#uS|+^H{!#G(osk#TtDOB&v^C(^!$1CbB`5t
zuckiQAuVq3w)Gc52_D3Lf!Rh}F_^sKDk#oe1>YsdWez|>R{jo(_;=KeNCSh=m!8bp
zM%Nu!DUC5DrHiBXj6swOsGfEZld7TpF#&U60>e^8{$Qn>)a&E5wceBPCbI_NX#z?d
z<pLw2e4i;tZP+&?gP@QCO_+OwAyr5^l(36#vQn%U50o@Ee|-RSWcQ3mZ%8+$s^hvu
z{Gr1OfZi|GA_jQ6YMToPBV)R%!_PO$M)yY#8wT5w2+StpJUyTHwUyw^Nh_|0sw{u8
z!1Ng;`-5r$gf6#2kvx1PM|sUXr;$@X$*HEXsaO`QRLzCB4zk_jrTdLV`nyV|ZX>A^
zJpfMSH$D@xkJ0r7iH=(n!ee6XYq0Q2aUEwhwDZ!S<dL&D(kz4T(zvy~-b1D;dw?<}
z#h9-6I0LAu)JB48NzNGGLHMVwu#`Wdf6M{i8!bc;0hawg-7dPhf^Zd%M}OKL>=k5I
zw|50MhyM%M8K|HP*sp)UYUX&b<a;cg{cm6q`-;;Cj4(1Lw;Fw=m>$phubiGPf7wYg
z$c^|#s-*zk&zMWsVV!T61A@3}m7qICzfvi)U$hq1>j-UpKuY`$aOA=0KiR<Ll8HZC
z??S?Hh24n*36}<}hYd@vMxfKw+II^Za4ky150vBJV;v6Gy;T!U(<xEnFuQv>=HTtY
zkBbzip9rw#8#tpS(wZeOHGF*<6Q?&$;g?SvpQx9mCJt}ZJmpE@LB^U!;4#mtH}jCM
z`P5&x&b5@EM^65j@tWNMj-NeF=UuOM9wFL;NH7!F&sj-t-cvAB|MjLwT6b-t_ltgI
zlct|4)e}F{dA{u<Q7;12KAW5yNOvRQU^3phdV96TO5EbJYEHozUynRfvMNLDcJ458
zt|3T_eYW{G03+$7*oWI1bEYI4;%ObAn-K|ygm)kXz#4y&OjWy-6rDdy9xTms*^^+>
zuaa~VOp8a7FN`F=aygF&uw8nR#2#I@F{>;Vn{V%@W+TL-W|LEfv@T^pli0e<4A>~?
z#8b6b#M-=@9m-X8vW1sf*4uRA<E9Ug<YGvI_nfh*S(HE<FZ_dxt_w0jRv~=PhC|iH
zFyCVcV{{@}xiUct>4C(6G%$*#EeL@$WNBRc9})|faN9&+)X(!XI6+uJ(m}lEG6F$r
z>pQURr4u$baTNb3>>|WZg6V@vU4L7fn%dl#D-Bq;YQvu+BLsFWFc!Fw{3TejhxWO+
z@NGX?LiY(90;9Ybxjij|6yuZ2xzVPDA0Q3pJ(Ae?)SMJRths+NLY`NNYu9Gxex+`S
zLspN^X;N-1?6wnPm=8$;9w6cA(K8zfP)=Z5e-TmP;qYNlwgEo&gG#;yRJ1Cj1G!<|
z0BOFjTC?B<vQVNa@7Fv2E>>Ogg(|o@ZazP}W`?s*-bh{G&UX4;q*SMXir$EznTvqg
zvNYv3fkoG~71p}EgOl6L`q_zULVL6^{LJUIlxkE+t6lO1L?E>Hkck-f`{tAWUXyv4
z{@dA)?Tyw->$09N<Ip=~^AD=LwSAe9mi<4xtRb%TV)=^+@5x@h^Dn!FF2LKX6>N#9
z$i~ffYaZv61A0)<Btf_h<uSqs$eI(-KF>M3<P^@0S9rfdJR8@YAun?1QQMdF&VT%8
ziwJ^1D9AB^+?rSPrQU_sUl4xQ1_uGToBBSdj{RHeLXxs^$4)}|50TTsOG|7jdYG{s
z5;iZIZhRvP#VP>_rPofjBvz!R<*;GTwnIQdfuugty<&`jg6t(avn0||4DL-eTIAPU
zaK5N!zZ@bSXVG1|?-;l2!KD6%Old4g_K&B^0<YLV8yLzWlOzaZ>2ccN+svA4%h`{T
z+WETcvd7sOHN^$i#jO?PzAni3`7}|Kzz4X@O(Uy=D1HAfm+6C3&vJ{O%ciS^X<RA^
zBkEsYez=t&SCbOSONhxD_u0Fg7|@P_x(mW?{V3TO22ev+fM^?J{fY$`WYQ{W5n<o`
zsBorYailx1R@D7d8q|%7<!mL##w)$~BjOw2e9v;Bc}z&O8pv!t1NLU6<Z9*yX4CqE
zXdCL3eG8N#{wE?0Da^S}ABf^wrT#2z0U-gSP{aZm5~7~@cTK35+g#hZuy&n<htZdt
zRHcygCX&mUWtAi13jdTt0xp?OD<7j(tEGB2M8*Ds0gz+la#;Kc8Au@j>81jcb*r|@
zyAN)c6#SEueLF2A9^JLmh|>9!lDL5`@6T@AbQLh$&lM<fVVlAd`99N48w{*0q}w6~
zu)qM-*1ZCx*FF;k1A&Sep~!&*;5iI7@Hddb`3!jqBdswb2RXn922$a-n31vfOc4y!
zv1k4O1EMYf)bz30E^^X8{2TUL8K7Z$Hy5O%IRmN8&qIQ&3*>?vupp7t%oln66hLi(
zOY0z^z6CAw&zzQjA_;~p*z9>bS)9m87hu6UuPGt&`mff4`V5lrtXE)xlwK&(q|rdq
zf&Ff6<T=ne4JFukPI)Q}Kty+taeeRd3n|hvnSkuFVq<<F!{ZPzix;Eg&+h@Vco_~n
z5qCr^lIiyf0CP1}ZUYiqGywD%bsCv3W9Nf8S(Hi@$aFOtB+|0Xoi4~Z-vM<$cYJj~
z#K8Uznke5HjQ=`u^LIZ3tbIo>y7#WVy$4K3L30P?0RiS8)6@i)ntDZ_z)G4Z5Du4=
zk$dUC(@{|WE1Cb=%zusG|2v)lG&MfYk$LNR7VK#HtC$Q{sV_DjV>a>ldj#px<o5B?
zl6^bJehcEoXG+(TP*GXn)|uX9-2rYh5xC8_l!}f*z#Z<%YbxKxm;+hMTv8b(aC(0Z
z8qx>?FT3@<LK8#f5y&EL(|rJ5wF^L?-F)|TlN_WiX0-vKG)o2`hAezyL&o#~tWFfP
zX%HRXcBV8uiXs9&UiLl{i105!^9ox#XjE?yxOd1Xi5dp#DR8o<rq;*^15QZZy3-W_
zC=%S+BJqy}4^)1<qy<sp8Gz6wki6<41&}0`e0zlb9hCOpd5v*_qyBT!QPc%cG;MDN
z#m<rdY!XS~h=KGLbZuTt{Cqyi1&|CnVUmb7H-Iv{fm|2q=`jc+P%-}98>68IY{ouS
z!3Hq~M7q1>1i3Y-fCaZ4mON~5Rt=kKr+*4r=KeXN`JZ%$3CziQ|M>-oYi|LBC}+O@
z<4*tym%f<fop+$%duP`{ii|ACqn7?#fT!~10fW0QK|g?w{?k1W(`Z5J{=TTyqQnfa
z0D2S50e+r9s-9fc$Z&&A^$`8Ps{60%{{Ot|{N-`IuDT;fGS}!1HhQm3B84o?JWeg<
z7c~tLk0?vLl~a9p7%WD<+v=Rg_58R^l|PoniV+I--jhZWt4B+qJrT54`yPjcc5hI`
z%4Fe6<abnHlI;3OcBs}Y!mtU^a_Z2bE&lfULcwA3o80f!o3)FkSDb9-zU}A-zhfic
zf|EVbrwa|@UwLQQR&{q9AG^81?DDirfm3MNG1MV@2i(&z<<(DV$KRi&b7hx>TtClZ
z8K@oIYS&tx&*w5(En0NZst~&taJX>~+TG))zG5kLT4$cFurQncZXcHH-a&gXg6oa0
za5zxMwX)?HG;?C$(LMak!ibPbow?IF3320D<;1Hge{{CteNC<cv&Ek@$+8N>r{L-B
z&LxOe5=mmI+fotFX>_dHsyL-upV9B>p?Ayr6dmxN8099{EZ4F;D5BY8hl{8dC1H+@
zK|isJCA8{|*E_#RGR#P;vYb|C)?}ylPAegcNj(!Ssr9IgWzm%4DfVTcuUj8+4)~O`
zcS2y2;`YPm;3NsQz30v>GF9(h=ykXq@F<KGoc@03@pAOhWbbbrYA-(N2*IC*X&%Rx
z-5G|AZyip_=O@k55RW_4_>?P~1_Tc8xSy<3WXUIv&=<5`r6FQhJ)J9Nea_n(9A0mB
z&@(gYQxqFCv{b7{_6j%`bSI4N##|gXc*~Ku(CJH~uRQv9YqEJ1%>ak$bnY_xi2CWY
zM;wQthQ{^bd$^(5FbiQP6{0ETfN-%%E<(V8V{NnYT0wem$k6q;ZA6>O_sTo*u+^wr
zVo$5myym&}T=%qdx^DtU%45N8wd^71XXh&?+jrO9=Ea{GcX+K&o<6Mk{VVfX&E=<U
z7NU?YTAPcRZ^fNwf_C_Qr*<tilF#l!8m~6Yz^UD)EXlBW6U6la!dyA|g59uDYUbgv
z5-F>e<)!n%azki^_QV9O+uQI|${<jC-913}TfNi;`Y~>1Q|gKGH24(YbJU{m@cS+T
zmsGspl{d*bd8JCzHXf1e^6ikv#&`5C<QL0q`m&b0^)F9yLMnx%9p>=@jZkQWw_&y{
zraPiR>~_;~>k$t83l`<&w_Z)IotO!%t;8_`uDcKy=%UzB!Aj|6N$f&{BF8K5T*z^m
zp3C{m?(Xi-<fFU$b8(4NQg<On+-~8bs~`d;R+uVbXXyd@GIdXbOt!wmGHf1tO=h2H
z+N_~dzP`NhZE{!$y?OLC#IQc_A!Na0nTLP9?K&f;zN@i0M?u#F>Rs3S<3q0sPeeOM
zi#XJLd$u6A@7f}yQpnUWk>@yK%k?EHH)sc=w02@=fkODR>FZEwniQm76AaJ&<b4k4
zJgo*pNj861s_OP`LJ5L-niS_*B+Z_W_?P)!SIKq=PrWMHbaOcJT$8lj;c<7}U_G~3
zAR4x(70?~H-C4yv;WsnA$Pud~oWfG5S)AZ~lFt!DJ)vjdyS!->zOMzdjj2DM;62g!
zEFh?vKpbhOzLl?3;oRP;u%}hykm4m`R+b{OQOeW51s#c~xf$Q*4<kbu?+)7yd!0@}
zGoUb=S$2`D?^>&09FD3VI$O@X)wCEd_XnrDxoZu!a4DCcF88Jxi<CpbxfeDY0-NgA
zwGiQ<XA3@Sg2Tk`8p17L`mWJ2N;6H(-j;?wm-A4Se3cqFbjRb_Rd=pPk4W!ir~%VR
zp-mYKjbX-4MmWj3#6>UFwQ6~jBWU~SM1I~g#F_DjdSeC-t%FOEktum4a6o%}qIfox
zyTiEuYxu#$>|)Qa#RT14DSX$@s!IILe2=N*vSFGQ$|EbuhdX5m;W`&#b@uN|!RFkB
zn$WkGycc43IZ{0?h-)gZvFqZ=Iy3pk8Qu7Zt|VzHK%Ud*QNfF$!n~BdWWPEs{<~_j
zDcW(smX4$GxxVFC9;-lysM-GTx~lHhy#CR94Lj748blCYqx{8!n~QLyb_H2zd}b4A
zC%eE?uOUjZUIpG{l1ux>N&UKYzg^d@+X;ssBJB8frs5B-aMQ>HzUysz+T;dFC=D2s
zrKGH>JIZNgeiZEVsc*Fs!5|c(6)K+Buijya7qY!%tfu8n+UFPV=cB~GyXfucyQO(j
z^|7`P>i%>Q19d<axV<>V$<(*dck6vpWns6z3MDy3TtB(Eh-{A}C4a)rs^_-<&XYpf
zN&GOwsNLb&ubaYb;~-*Vk!tAinLt**LiuJ1_sVD_R_{3(-fa4UXH}k=@8?M9F2RSI
zt7dy^^N2*uLu!YrW4cI%s=>>5Km2Tq&_|%(>x&^5Q=ZjD0g9_>KC8s*bI$c}(cLLY
z1U`JcT~$$-=EV52_EpTL%NdNcDf(k&n$LY=guVZ+b4NMzI7b<RfbrUp(870}6SE|#
zp97@2V`htQkY_?_PA%$oG3)O^oR+>5&`=9+$=0|x$GGrN3;B-|Pb0Vj!O1SuPkL60
zL~DK8Z?Q}l=XUjKHV5P~yuWG65ZF3qUR`W1o)mPiZxclK#4%4jb6%=m;JpcOf0;ON
z=Ux;1wyresL#OFUz4al(R>U(^SmXXG@9@)CLq!?_#DlLo@k&)wH#qt_sV7w&9AUjI
z(1Fq>6sxndtr%LBD$Dk>d8tJ#Avd^s-uwxiM!-*t(rIhp=8`*BeYGe0qa>v@w6`)p
zu#b)BQbPaegCgc^`}-GW+SS%oaP(s*CT6!QTc2Z*DGcnCkDBN&>JSf~n8+r?pDv`)
ziy*ENgqVC!xBVUZhR;veF$m{_Qr)0>H46sU0@d4%JV%z;quII%swQLTlv|r>*Msh4
zb7q0C9l12CS96lkp@%+ieJ^*V&_@$<sUOW&+wp25OndB_t_5|I<Kv#V!L98*@T^`>
zkIGYc2h`}~!i26T^|nXtDK6-m$<h{$p^EJjxvps;<yUFkCv*)u`C$>#)1JE?Csj9*
z2K=88{xlaRs2nb{%MU!&9sS%*yHi%D=?(Ni`)-S5<K8WhG2Hh^<hiARbgIW<yj`wb
zzH$4@B{58@!qxt7j`9dHV75*h5Jf7V-}=LbqKt~k4Afp{4cB^2uJNh}$J~Y#DZAnk
zv0)F~NOq5;eoQj_3Qg6cEZ0@osa)$VL>*eXrmEB81q8LBL9R8#5w{<@`hL4D#tsHk
zqQ{}~*qlVgI=(B^Vsn}n_<NnHArqDj-+I;g&^7E@@@%6pB0-wL$-bbrb@3Y(q0srb
z88`gks)Bc>P^S#r@QZeVsTaRtt*a2*$?|)saqbu!IE=@pZbPuk=ph(&vmv}aFIIlJ
zBRKu+SfjwjY>tiE`zo-3+j^h~E6Ziua`?LaR%A~c%NN}n+lY_EY^Ou57c(ds6};Yu
z<E3zye79kTCIgp`6&VqUQ{EP@rx48x&+HbcN3vw}-F-JwhO0O1y-#XRr%*Cx+pq{5
zehhE;ZT8zZ^lr1-dw6CNM;})gFZabZ>g+6dZoffn-#Z%WP{U()opEY=#oN0!>KlDH
zAk?Qww*{KzP2l*&@h#`uF`ycgZ&1)hX9ON?ILn>PGeBg&Y^S_yj+uKeB6lSNH!_wf
z21>T}*1BboP-cyN*{0dtcbfCW-*SfOqW4_UnH_fRI5^b1(jjcNx>=ak7cy#a=>$s?
zx_GHsvk|(SvA8I5z+LmR15vT-d*&0{kiv42?fz5GM&=$Q`EvMJBmLa5Q%dP1d7GM>
zOI_c}w8(PI-0a-Mpas_dmc<PoqHm4uxKTVPP3|`PVL<niH&M4Z8TCSE=xFfcOm4&)
z%UpZXuUnCZq*wF`Nv>7&ou@TEd86di?nnJ6mzo9}93CI~w2cBg3pq|<=_BWgh?7mk
z`#Hfalj-SCug<Nyjmr$_k9DBW)Jj~+@Oy1jRfMm@+)pfnb|9~GQ*=p=k>BN5)m`lI
zRYaVU)<%ld>qhX^e<&f<PhK_@IkO*tqouK?_AI4y>8|5xvYpzzZdI&&M&%*4x2J5T
z9H)#d#K-5K^Oz$;`oPk@S?Y%IZm~%1Hd_W^NYe2(c7D|%$;t5#%;jyGi3Gz$TC-%I
z()ueSky-kSUgz0m+bXDUDzC<gb!9Js{>uA=P0v(YyEO-m@2fLdCch90EbpuX97mF$
zxXrnZJ|-9brS7>IrHG(tV%4jZf56>1;8X8#PMm>mL(B%wV~1>Hq^&~{z6gHK#mR0A
z!W2AFCHZ4Zu`0_r>!*-GNP2^N#r(l`CZAm>1;lACe4#q4x4@Sn>3YYlaJJ;Qqzv|E
z-o|!%)6RC@F@9=^({3klq;2j6SsmYCvdr&$b*ZzHo0E3&1>83cRq#enXq50O({j7E
zksXO%#es|<0kccp57R1#nG9IQ_4#FHS<>{Rz*4<baqpIucX0a0aS1u*z|N|aju}oS
z&#hOwM~n4jWC1~Bo*x80h=FABqd`u%KgpiT-V?)hG{MV(Bt(Py(Tz((Ksqc0zNLPV
zWtSqLiB)4T|K`)xV%F)Q0*wEng*VF#>Z|mXr^HjhqfpJdGsR&h<zS$0F_H6rvBd7C
zD=DQ!5#v_@xzEBBkA0X}-5$FB5cQl*xH%}UK(uiPgvB`SdcGlq^l9u3adLT3s9a^B
zLyt*eQ}l`8yCv|L8Bcx0mlTUWCQZA=nyrz{>}wMn*Y7&Ni--}h?=8zNj_uY%`h(Lk
z?}W&eDjzy^g`Y6(kbTwulIl_J>J*m|5ad{&E#PL3nY@pG!HVb2AZ1VLOv%04q*7r@
za=x}JBWSInwfFsZ`P~fkUT5o%!d#DsRf<=$Ghn&%0!v;s%O_z(#YTZ9<%9e0aBc?I
zU1!OE%|$rcR}Z!s21ql+PGg}7f_DNg&wmnQc+oAz0Xv<BvoX|q9Spw~jGDZS<B=3i
zaNjj&u%YqP_@Eh%n6Z#@u6?C{apI}BLva&||L_5Xh)rNX+$?uwI(W73=Jcm^ZR%m+
zX?DUxHYNFun%0~Tn2|zBE(5&pT!X@mh0k`aV2|+_Yh}R~6Oz`QV(_VE6e)ir;F+o`
z(`zEclp;&40y(RQJ~o#JD4{yQ%#vBO1xk$f?FboJCvHbvIK9LFH~`LJ|CBF(ccL9M
zvoEEXf^+G)bAH+*$Km^GiOu^1CLyI-N#G@}U^jGfNoUq@QqQ{{0ukQV%2qSjpR!7$
zauKjjVb?{>d)m{wxs_?=son>v{2RE*%zZ6XD6o3xm*{@%0%y9r+b*`K_IqZ7519nS
zV-Mx32WYq#=Wb=nZ51<EX)1v458P?uV@wn%91j~j4fkpA-CS@nrB7i$3ZJwK@!6?=
z!s^D?)JHW{TLv%Y{J;imWq=^o{I=X)uuUDoh0jjS;8xt7Yi@lr%FB!1%RXz(a3jsx
zpU(DyL5^4Ovm|^g;-F!TOGCR<h4PmIdX53|_fMOcJS)f{$szYntj3iauRYEPwv~?j
z4epMy$a+xB7Wk%4y@~q~5JWtL^YxZCZlTu7Bf*`=!)HupX89nt38vm)r=zd#Qv1Xm
zZ43`gf5^Qdf<VS8)fgl|=6c7OM*%<8PZOM)U(%!$EkRFpj;BkBdr9W~JvoAZC`Q%n
zsoRzfd)cOd>R!jGMr%ikgxS{zZkw3rw-h(Gqy%hixwsZymykm!n(mQ8Y#VGs=Y)P^
z3Nn~<XGs6zWjncidW#T?v3mC-frTNwLc<BTWx{RMZC!mI!?RNFs4VZ8Q;`ehh7{L^
z5n<Si+_v$2`xN@57V%EJ^(opE1*>)HNJ%pp$BlUS?rTT91@NC~^y9COeFY)+%Sx9x
z9mX~@Mt@m8nea8fn2Ve;>5e8Tmu|J%mj0kmE2q8}oPzzrY-%KNwf;MJtvbar#`LdO
z?RZ6hSPn^b{FSv#cSIUlWyL!aLr6JQ&}(c;<A%jW192&ZMPLqJ4moRlqwq*$I$b#2
z#nV*mY$p(&Tz5F;AvLq7n#$QF7Gj#Yjahc-SRGE;d^MnpzI{lTygPU0?6We#XuruN
zGE!rHHgYFN?hRw~zO$R%jJ04@sZWXXCvVH|Hv+|@<eQFKMAr@_O-7sBP9dp&#n*cI
z8<H+wk&3yO&0W4wulC(Vcf+}*bk&EGc1OE50wGj9<g>2(%XUVQof3h)qx>D>X;vdM
z^20@)<y4&>5V9(GAn$QYVlAJla;NQbUrA0{`=BNF_Vu*G4;k3(_%g!gLaf#s1?52&
z@_zxqg9hU0yo&<|aXnre;c{6eE7r~K&k&o;P5Cma4cW}G4>a+n1cU#7GrE@N_W!zd
zBBx?z6gJb!(lcm|$PAL%*9^~gX|x(XNZDl%xAVJC`VT89trMqEshh<MjN`GUU5*kB
z&uSjYjFmAqUwFru>Psm35enyp)L*aKoJ<;e-*mFU6w4|!EY88#8wXxE^_9V<P<t-6
z&{It0sQq#^O^@34^K(Vwhm9Aaa?>_a-p%N+X(p5{FGSnUQ&-QfJJMFB*223hgz-wg
znK4eq>D?&F1a<}(Cg?EEr!3rLpr;+@A6$f;sZV~2&)@AY=LR{)t+|7&21-ThTe+&_
z32c|U(2a|C%|B0@j5^og9T&|IN-O0l6`%wYK4a>m#~PO%?#fG%j=nH488#=cpdH?m
zgd6KygvF<EnvwCHX=lxflZA|nA-0rb$H|hB^Eqs_z-0*x`&Ucb$&EGVj*eKm*G3?}
zp(36caX1u&JpnbG{Wwh8?+V*2IG$c3sM7qvZnroQ(OPEs_>9_&ff*l%D`(NlEPnKQ
zv)DSjU!3A}Vbhn8M834<`ekGpsi!M1;nEyK$PXj2S(9bQ0BN4FX-2e1prwz7;l+w{
z@aV?Jk2#0Cy&=%lrBtV$CMBl{vnTM7cy`_D-=PG3<5-dF8OEXoiBG>V3VY~=r{3{C
zZ}%R_`-z~UGS_^L7;WgJJ6IC8n}I!`Im)s*ISoU$2VJV^g=*OIYz1PJlISj`u}Upa
zJ#e*b9Id8^xZHH06aR<3w~UIi>)VF~k(3q`X^?Jd7(zli1PMt&QRyCrPC-CGTDnC*
z8l*!)8U&=9p(UmJ-6!7ndfxc(|NgFLEkAf&&Yb7ozs_UU9NL$y%@(t+#pT+(2!NjV
zVo}<r>)de~MA>E2E-Ke-*ihCNL2Bp6MWN%11;escjNSe6od=&xRa$CACzN}YYYmg-
zd!5+t;pxB*%$G3|C$bJuoNM2$y_>ainW#4_=uNxMy;Q!IoBD=CfN3|*vwB>yQ)g1o
z`IMtdPB)y?@<wrQI{0R|SMe)3JYG*EtfukB@Jufnn%Sni7~$A@ViO-z*Iv`&TT#Tj
zd8<YjL3h}_)gxZ(KT9msU9mc0WxhIL&fj|JUKJs8A11z<b#-&~*!^W)X?uXR^AL@H
zX=AU+)v;)e<J07hNT5R6-a>%K45QazZdaPe1|{4o&?^`J@9*6r{E6Q|54|MzeWkqB
ze_8W2tGZp>CihwCC(~TJa=NtQM2oQ6DBs}*t)`ag^NN{cR3j{^vx0Bpe9CH%4z&wf
zNGW5!sfCX&j~V||@IQ0TEU?5}11YnoVofTya1YA4AW=hv#MQ#P{^Q#1=BczeLi~E0
z)<CZ!9o3?D-rt-y5t*K4u1|hQkBt!Pr&#x#ju=Y}%c@zOf7Uk>aS^HMjOG;k(YU*4
za)jN|q#xSwIIC*Psb>4$YF763PxOdI1CtLk8AdGC2W62*So{5}B7erWZgWM|t_TyH
zIsKIZywR>c@4bEeQw7nHiYYp^VT>Cf;%qyY)Zg#I(EUR0t-+A-QmCG)P-E=cmd##h
zpa`W$p_*ERA)dsynvZ0j)-69aAAZ3TDv2-@8!Xfo`J~_Tk@D?5eK@PpAFWK<<N9|I
z<hW@}sU^md?Hk{$4eQiCne~j0JM+t|kv#Imd`rB~-Og_*uxZ!c%JMvoYObN96>ed^
z%G=<On{oUEnLY5E*<w(C>p(jX01B|x<_FSvuJn)dq-<&~W*N6N*eM?(JNh;q;uBu1
z)Yk(?F)jL=qZ9Z>nqy9DoO;Mn<Yye}9HsSjMa(|UWcpdUkX_=rUOazR&@d~NGr<3=
zWZ})A@yA0rVM9xqPUy^5a9+}uvth&GpVhSM$2t!A3P`a(j4(16{Wq**WBQ$5m)3dg
zd}{Gy`VI&2)`C5Xjtp`b1O=5wzF#$S6O;Os>WjqN)X$&VA_p;)Etn7fX6oP^u%_ch
zjp^S$ByINdbMz4P(m$!%O1phD$GhN9j2w&bEdKGlYOFcEBh68k2cD8?^ruOp!St)d
zqjzMoGptA1HF}P+vm*~FS9u-UMvlI>W*$ZN?O^@C3nzXLVIFfld|;qBy(-2K)P7u-
zN63!yj)%YCl?@bQ5ru!>y1)sVkX&2I2%ZoC9nCt7ac^Bv_&3>doutGBX*^AB<Wn?V
z$>LPQNv{CUlV$%`B9l``e!r(v8>mQ97`~;|jM(hX#GigKN5h|UA|b*3406qvl!}fm
zVC)1_$9q`M8+&cb#H}YbJQfq=1ltWqXYmdzWI-qS;2Xxy_hWHCfp18rC}_4odl1HH
z!=_*+2DDov>uDKi)S>RG!f&wS69M&9hX|kFMnUrckKWaCc{XtiBwnb$6ZjZB0C#r}
zg*l$y@?8R_?;_VgMP&f3%NUt3Yl7VmpkYZ*`Gg+~lR={$|NGMaCGY<V(*Lhi2Fyt`
zN27vu&`92XA?f_*d5c|V&sf0!rRfEhf%{n88XEOPeZ*}$3!AVkZg_n>;}ZWY>uJ}E
zw<#p)ql5RL2V4W&aD7b60Y>aTVK1$$bT4&9LvT&;8aayZpVNAm=c=V~jv2b7^d<{y
zs!Cisk%_rg@V)y*bFEl^Gb`U20xTKZ=H|^*pXz-1fOvF&ifGTb7v-RXU=$7Cnq<&<
z>GOS5QoytglZ@{jKxQSZL57QFza2=`_@bO}t1{{fG`evG&3+dTeB}Yd8oDpR&hX=x
z&9}$|ZIXW%=nPzCdaoqB4b;t^|H5TduUiW$j{Yw|-=%N@Ahj6nqTj5jfN4=+H}zau
zpy5IJlx9seF@^@WLkKKNzRG51si<xnl*8F_Z7@(c6pAtP;?X0+DMh>IfB2i_roge^
zS(=lA0hxJ2&{}pm4@`t+JGCV-RQ_0N0T`xe0qe=mlRadPR)O|fHGwX`_L+uHS;K$1
z_9gOf73u=;nY>UUT@vtWNtzen9A$o1(0pRH!pr@YK8FFZ<8k9PHm&?eE_0q$ltL-d
z(a~ST(3{2q&XvA2CH?SsYkl`~E$~tQIXgXQ(G3C3i2ACgV?z^<_07NYG;@^VIUjpp
zc|f=vW=oxsW&8c29mPM-i7?lE92+d0_WJ}wjj+}M#?`@}F*VlbU)BAF#-SNp$#iHw
zd{MEK3YvOE@dXd+1~5o3bGxhos1*d0<uGmEVfJ!hVXD2%tY_)$PjNg`nuo?iK^yj0
z9pSOjsv;~N1Qid!qdFnKCH-sjX8lz8<@wRpUw}|FWjrfrGtKOIepK1i8P8?hSkiaG
zc?T8XNi|YR?+>d<fZbCAy4E24cmV12CH-~C&CulsFdN!tvHK8utY0&r3sZ`Sa}*ix
zn@#|I_0rqbcCx~p)PiW%&}O1cHt;SP9l)3%!F4~=I}gC%4Ltswz6@^T21vc3!UAph
zFvm56$1ed$mMWy6#aIzXYaniP4`<ys-KjDSH#D!)<@=)FS5jPNBmMEk$MM8%0W5$U
z{AnIU;QtOkfzjDn0t5im*equNJJ9&+zcTz&8u9^Z%<NZ+%1<nsU*yxoJ!Lw457q`-
z#V_kEM)Er$>BW3R3kIoBBUGS$ohnqY`Cs9S1Q1K{m&bE+SD9XG89}uifrq>_C7^jO
zy7C6#sDZ9i-DGZCn&;63**I?<L=r!+MvM8TX^0i<oqU$Gu_<cEv6yf6g-`ojjgZ9x
z)UvJ8M2!@LFIUsWQ26l)xao4muxd0|!<Pp@T#GtLF~DFSNkYJFI-e7Wu=3jTfc%yC
zh38;TdAXN&pn|FBaAQR2yYBaa^k#tRw&aUJRr1P#PiAAB0yo^OC$8@$$2bY32ev>D
zn$9C8!KOse9qn(rCe*6A!%HB`n?nx=TJ2i>^LZh&FNB_Tygjp=6K@m+ZqgzCcKK^o
zII9Kk&00JGH!v@F&~PpdE^0KBTHI`ddk@gT9Dr#6w{yr*j2$Xr3h51OCbzApHLQb^
zP?kIP@sEw0BLE~oY*>DY8f2pc!9$=h3v4V&VD_enj?HQx0=@4hl%p`gh}Z$3S6*L5
zoSj(G@!fvw7d`#>fEM6QRN+$r(5Z%{rKM{2;1Otq*bW3!ErSCUV6A6}y~KQ&kqm|?
zNg=ILjAcDSaT+sn=U~b(haXGrIj;1Us)=6%oR;OatcdSb##;b>%joMvXNX5IC=Pq>
zqD}^Ql+`?C+hk2GQU$iH&u=vYL@Hyj?vL#FqP|Lm4eXpNfQ;HYsIwgX7MWyIs{ruu
zXn-+dHwm_a#>3InK;^eyJVzFYF|99Xehr5s5diuuM>dr7D1I9-=gqsF&7YyDJ@H)N
zHV}&3R!kkf_(^)Q+eV(zH`bdK)5n}(J5xwl6rlMufenVnnedr4U;?Zr%G&!4Mv9Fk
zSagb#=aQaYYChTpfK?My#a<^17&Y^LIGaMB0K5%{Y>wt#h~H{}-xB9xdh|Y$acGUF
zE9B?gH2^_EQC0l(^`LPDfC@{_nJm%g!m*mE9SJ`jER`_pjwyZRBjU2%D^1@pZT=hV
z0S~bPHDKd<4<4e9Hd<uJ2Ig#%K2nM4!Dn6Hiuw<>-#4a+Ss4uH!v;OdV)(|4;GBo1
zUX=|8fDMcH#}i==bI*o$%A1n_&ux!zD#6kzKGqFVTZX;&<zajZt4rr6T8%<Ie*BS7
z`6o~26W*qlzVbT|+88ZNng+;i6mni`LL0z>vH*v2GcF@Z^A7bwgR+9E&&&OwdjdeN
zSuxT5VtWf=*8vmCbxx9n4yg?~sHKugz5)AX^IRwmg2uUR4Zu%rHk|sS<Cf{bAoVK_
zRM#KC_5f@6M;m0X-ovJ3p-svES;g@2zGM29USu`daq~0Y8~VSt-flH3BBGO@d0s66
z7{<!-tn~+(;+|*RqKWaMVYL9NMg!<-_Q^P~t$uXr1IBApRmCuXSO)W^Wlq_MhOh0v
zbURq11$c4wo>aZn_Q<8rufq2fw(_7!d-ea>phIO&S4&RiGmo0KYsJk}O9q&HRd*?b
zJ~63cloU72W>?L5>DM}~)8UY^e_|`H*Vt}2Hp<h?4Ie8|xrAo(O`*LOm3FX@K)~lK
z?6@)>RPPDo%>fAT6iSo(PJ_#@L<M(`N08bK^2kD_96;iA0ze2R9Yg0>pk*gtNuERA
zPadoV$Zxjff+xN74I>xqHguUWQ`^YbDdq(BJGP*>F)_OC)^W{TqJYb`f#}IaE51Yw
zKtq%|I4S^-8?)_8Z-lrZ4h*|E-YodruRF5{B<?wP)q9-#-6T2&w2)obtF*Y&Fi|m(
zi%QI-8UtyUcA=h`Eyi!(HGs|d*W41?*9tHM8g@kQb3{)-1X-$1V3E0iD^6b9dVYrU
z7vT4iM2!wBOW=IdEtP`ME(ZxTn!G$kiM8wCTNl%hHo$5aE$~tS6-aXdR1?DS@775l
zDMhs*MXsr-0hlgT<I&Z?Wd^?Y0Z1e_rnbu1695J}0Nk_g&Pe(kqXst=bILwIVs6O%
z{OYM71c8<Zm?wg8G*ACy!pocP30nMS&VI-ziivt=r$mYH0x&{~c^WyJx;Zv8wOII7
zccpfQ(9!(B1RSOp^{^D06cgl1&egwTP<yDu0mK&>k7+=%_tj~JWBQLDB)@nR9|{?H
zUph|!#KNsY{~#*2cXs5;J-`wZ0Fje)q7MVsOl}^)4aKI4xp#w0Z1NIrH=f(%=~d?M
z^LrCw#x&z_)zd%`l*H9Zt150`dF9C0XY}gU(2fWJ1>ld}SyUVifrT%qNVKDexy-uC
z;fbHTJjb#7Gg>HZEro2QEwPyq5rv!ge&*4rgIkLpLHJYHa!7Lqu%QALP8JHbGr?jN
z0auAx59v7!?=q*#<Mi**>@ff)ojjjIjxCF!J|R8XuG`uC{N=xq0h87{5E*<*Ql0{K
z;w`WfY0eKuWN-OiV?n@pM@LZr=}-ZTy`Y{e6^F0Cd_waLb*joX9#EjOfANdjld*~E
z17Vtn-8ayARzUE}W^Qj%qo5h!texO`7X8Rph|UIZF-P$;An@sO7PbPm>`ok~p+$!m
z3;<~ieS5)6{xlrGB&G|*!)gB?d%bj#@-kHJn>^$qz)#v<Z#W8rFzgms;Cn>y8BDx2
zu$};{KP3<~nT?m2K=B)sZgd2se+S?gL$<wCfIW9m%ij)s$5%VwEFZ|Wp89@tf0~p-
zFB^)QXvR^K1Z_!EX0~SCcVdCx>JmGirl%Bf`jniU{8BJ^6r5%iSh3VUM9>y-DXN>9
zG<Dn;e3gP1<zFb}z1m51+KO5%L8uf6p`31xL;&DXg(Ly%*B2*yLjV!T)T)aWQa%gt
zkUaZrkRizXH-prHai-2C9=N40fH7+AT{>f)n*up*85$U>8f-5BGCF0Z?btO}XMfB<
zz@ky(u)KTgmj(#z5<$FZd9VrW8#a+{J)#uEP0~hQr)Cgpq$gts6u$r3@Bn(oc}Kg*
zfCYq*u>dkv2EukdyZZ?wHmznX-=KZrYGBT*_f4(Ak+Pt`<u$;xasa4}6TV9TqOss$
zeORs=^z(l<Xz+n`&JzG{wl$syU{NRLdE5@(<rz15A)dSw1_8MisIq7qVawg}UBre^
zV^eSS{U@~<Bu+=ieR$8saTM%P65V7&9|-A}0IN$F*5-hg5S=Kkb-e+Gw*;(Eza(!2
z^XPXmzv)#f+BBZNaTkr9{mbL9BoELN9Uqsxg@gHbz=6<LQ0@WfR_Wj05k}4wL<!g3
zWUYrbsG`8a+N1W4pTPfrgt8Sgj4CizI32$J0HjIJ^E4O|=q5<P$L{@_&xjUzN3cF#
znst}#aa%ug{OeLHh$xsp{Qbd$9&>Voyg;;Y6&PwJAm?9GMjZhjVh=)WoEFF)=`Ztd
z&fg0FAf%)dV+|Cga5fwawXy9(D|YJ2K#_Sebb<4Zy8jC!VZF5n9iNIF#3>ctqRtzS
zK#Zo?Xr~BdGzMoaeIEp^s!jxFr5#k3)YB6{h+=kvPzfbzD1cG1JK!ypq4|UA=A7lY
z+MlT-%Gr2gD*D+u-Vom9*6<aa_Z}xfu*_=+M7YV@dytLxtP1c&BSY$ai{KK3%0L6w
z+z2sRz+dGHn*nz3R%<t-S}Hq0Vy$fY3}!p`@@BAdfY+S7b9HrYyAGTj>~tv_Opy?i
z0aQE)&ewd7OB9@S9vEA{;rI$P6g4aedz8xS#N@nZ?PnM{c%;)&J^<Zy&rf|0(wJ#1
z<s<<W$Tvz{9z+`en4HHRCw6)IRYi4Dy5wNL0gB1Zs)pa84Y8#UEdDAbH}=?jzcvyE
z*%PBJ`5hw&wBj)XWWhwIMt)^TZ$)5<?2{lXALk5s5$qN>Ujyj$>uWx^*}MN@S8t}=
zsV8Zl>Y&!B&92ile<Fp&z;KPpJi{<EGs7`c7=TLm@|GXAJTnL7(2VDtYoJTK=JBl9
zYHEr*e8={pbK-MC%4z@X+m$NKNWTDjA%a^d1iQC=gYc<`ntgAZFSZ582-%xH6os>c
zICub`haQ{YmapU^yN%y07Q<g~t!HXuu}N5E0BCVKu(o$hH6j_o2X4_2V71WSQbbX>
zg+fjM+1#%nB6nIJVz3y@yc0$t97Xg<{-u91Ee;LXaVFILZbjz2KJ?yZy86l1R8=kj
zYJLWw&ZKby&%wVFe!LYRhsI|2`aar)%KXLYzJ}d5<EGTVOI`F$2oQh<U3uMIyp2MD
zX@?53hb-VF2LUD{i)_d}KC|w7`QX2Oavyeq52!2LYLRBJhT))D$;<-;YX|ueAC%k7
z4ojU5AGA8cDOnU_m<75S%1}{h<~e-D+>g`?v<ue&84iGF8w6Q1i|5(F?9<!V;7|gO
zZY`L!O3B7aHd~=KU0>B%=~9R|eaP1?WNotgq7eOavxCkH5HpkjakxREk_t;WJKj+%
zHg00Pk{kryI)s!XQP#odwcjnw&rfc(wBF7CK#XRkIDfP?<-pv&IZ@7HJz1e)_q@np
z;!BMzSoJ__QS@2$*h>bD*)feGgPJAv-o^INAb!073>1PAW)w{PP*RF0T-1Clc2Lz&
zrD||o#oPPDYSh3A-yF}6_FeV8g{k|g8Axp&a0G3REr|lwiXd#wlnEw!!ObN97}G~m
z7gG`u@Sdv0PZEvB?&8GWVy2cKd^ijombFQ@5qLyGn$=D^Ot7(!1@pSlErRo#w4j9m
za4-X#I+NDGD-X0q)Eh<+P@xz}%2Qzm-1h}VpP}7_*5!DG?TPtid*F0bBk5$Ziytl2
zKy;+BTk0efma|&zPkEjxgLMW%c^0{F3V}|tvRkM$1P@U#BYt5nh-1n~q5>r8$RxiM
zanHn!v0^pk&#<gV3Q;nWM4jO5K4fT!oy8i*RgDNLu@8b&Tcd^KhwH;ZrawQL|3nfn
zf!p~I`0QeS@{@cF!WK+}8i)5F+fZiH{+15%n=Y<MV4Sc6vT1Q|m}|GcqOvhr$;M$&
z{ps<WUU^Wp$pU&l#hu$pgz0mO*k`xd9UPA)zAl;P75d@!OcF5ZjNYe8uYpr~f`JD3
zrOZ^~c@7tj-yAP}^}IlvW6_lcr2KrBHHcf_xO%7rR2HsCa~c9Ww4<Flt`7a3nY#Gy
zI1Z)ZT(w*PYJSH+B$5UQkUKHr>Owe-j5iQqJ2M`i&(A^CBV5I;6vrhzG&(#$pAVX&
zoh+C_goxdF-|1;f;JO^H(7jDA06tst#XK5xnCHYojw^Qo0<g{jGD1Sb7xm7&mEMHE
zI?f`yFz}LVaG_lZ%{EL_6t$Nir~jQHjj<?9S*TYTRiLx?H_i?ag@%9`4aCI^$9F-=
zEOGkwYmhtL0~It$1*#zwpfR_B2vVohVq)sXHiJr(*_|s8jjLZ>oGzzQP<!n9YL)vt
z`QO4EPj0?EKVG)bfuZra{$1j;UzEuQ5k;S4*ru<KCzd2)yviz)m*5vxJz`IaGguU{
z`!eJqKkJ??2&0*Sg4R+~_y7V*rW1`<*-}?pO+=l<>ohEm6eCF3A?syH=(Ag*?Mhoo
zGtch}@cmTqaz)$@m_fAZ&#00p1H$0HB2;L=%H8taeYAUaXqhPD%zF4|EaK#q;#{NW
zvPQkGCl^fV&Hy^tJ(olvI&d1FG(ScIfwL?KTyg=Xq$V)%OZQtsNZ9lkFXfqmIA9{^
z>`woh3CnGP96nYw6|@BFZ;McXQ_cgzVqQF!!hX`shFVY0ZTqub6lJc?B?591I>jyF
z%|<2xZy}4x)LdY+x1m#ytqqiNNuwU%gW~Nzr+1|xC<R23+59WFZT-|INVC#rCJ|`#
z6Kc<c$Z1!YPLY8?4@22)RKOuyOqlE6fIvJ-R#}W<1V0E>ET*e@fE~p807L-w6ELH<
zNU@{PP-lrGm;Z3^oIsX?7-e=ZIUapAg0nsJemvXJLRM&s;se)G9Kg=Hm`8sJO|Yyc
zD-`eqg<Q5N_^rkXT5;(%lYBk`R<i4sG@E~BQIm-vVIyq@WI)3N(s+vn^U-azdWL3V
zb^v27Kj*QJyWE?giC1t2iXaVP=vPl}p-DDtqZ!;EosTUMKi}llUnWdIK`9^tjI-#X
zogs{C1|#N6*jn{5OQ2k_92EtNgaM#<!Z@tQc+fqI&;U{)OOP?8umhS>a9zs}Vaxzx
z@Mb&Ihz>s=E;JB1a@7QnUR|E&NxjF`@BxzgBmOG^I;03zYLZhhXCS%`t<SAeEzarW
z&~GOL!uZ@Lh{8Zso(Et04UmWv#XUtB)KZffRg>lUEl1HPfMkI`x6u%I22<ay&4KZW
z;W8E%KCM^JkV2Pg!MU-Cl8BRNMqmMnZvwgg<!Z0X+GLr2B{^Vq#{!sW6``k;2>Jq7
z+S7*thLP!lOmL~ilNUQ8Y<HCDU1$h^;f0}o<^FV!!0wUV*?~C=pMdSG@*Nz~>;&Go
z8bFyS8|EKC_mFH>LdB<o(HL!oFr_dlL<KM^f0l%ziTKiHQE(V8@F0(jKyI3sbiS(t
zX*~?~HE-!b`p;)I&J8(k*>H-9wp=<3=tLiM(Evy(&1&x!WP)N?=!44M^d}3OK9RI}
zBFO{fpuKrdhyXMArpDI~sw>pFAZ6~Vp<vO-#$>_V2A%~l|3rQssF6dX8~~qQ2L_<6
z31EkRqW1LoNc007Sc>M>0wF5JG%AV~8u0V9f(?4rcC==_pPzZTvVpQveg{(c#>!-1
z88o=Xia2k`{MR;nMuPN&BqM=lpXMbxI1cYuI#mL0c^YQs4=epCY)vI%9w+J{__JyB
zpk9am%17`Yv%Cq%%gjQP^<6-$l5gB39!4n|7saTuQM^Wok12xy2AJ@Q<d)Aao!!Z<
zNlkt?$P>Q;XINGz;eNEa5K95Oe?w?V0>s$?%btY}<Sc<L9|oy-p6mX~Jr?j=GAO{?
z(F4)rMwPK6*o|uj#4!iJ<$PWj2*uCb8OidqB_`6gS-#%?jzo-=EMza$UvmzM2YK?5
zbbpcR0Ha^UgDw*t<s6ub5^pm4AQYLGpeo@&TmUM+>d}lU?|@5*VSbi*OOZATgeJfe
z&&$8uTWtRgevhKoaW#Be(;VbUWlU{@_~AgeJ^};C{S&KEl13Sg)38<s3aCKLFa*W(
zP03KrP}!c28{(Lc&qGvp5kzGHDAK4W@|nK_ynmjfNWzcbp()w?6iATO^Us}%^MwER
z5T@P=34*4tASl1bZ}GcXiCqKj=1c^1z96-i-oCV7`$enQ<Rzw+|C~kUx*7`{R~ZC2
z2^js7&9-Rpommd12x=HYU0bzY$p)Cg3sMO>5HtWeR9AzdvWnMTL>9l&Q3T*x5!7lG
zLV)3+_i3W2KclIp!W8PIw7@CNBIJOVy3Ly*PjYjS8v$gdyS0#k$wx)JhskbO8v{Zy
zMSBpxA`;{vvOqcmsm=Ra0YNwf$sq26DDdAuCuu%~?>mNG>0hYb<$?aqCk*R%-?P!b
zHU3*w9)xta0og;azLs4QP=(--Mu1=vM8S;d`G0w$Q38SzoCpQMS1A8|0Pf-bi3z&T
zZlq6scSpw(88-TY2_`WN(i0z5$=I!+-CvQ&f9H|ZT7Jc74rF?r&IWu(lt|s|oEElu
zazvwc!)baHt=Nc&F5Y{0TUrSi5jTWN@nN09U&x^EK>5c-@Dk-A0;)GdveV!@@JH$0
zN79J#y{#vyH#e3;&1a>ij(*GhARvfVdH050A}CtlsavGw!f*c*Oz@I~Y+6`vc4#%A
z)u#N-Axn@`{{Idgf%aTWT%2)0of{Kh`tv*%L`o;rd@G57pl}Kd=%BZtm+&e`BR(tK
zw};$(<jo;XI;t>SZw^h3^)=taLpe3$B>`0+_YY#|=xb=tC%iE6DTVR=aR<EQO;q46
zcQofBIxv*r?%<HE;kNjHZw@U^q^@;(@);)mg9dij#P5d2zy8r&#Nj3?xP=K`()$Q^
z!GzJU4HuQ>CAd3uNXj{?rJF;0JOVu1<8cl2dxCh-0#Jj%(bI`<XzsKm1b7CCq1NeV
zg@-Mz%C#TJfxv|SJEUl%Sm({5Pvim5kgxvUz2m>j9e{O1V@akPny+=*-;jdlMlksP
zS1|rhg5i6M-tK%u<>B?nt^BDhhGpg$(PoXHB_W8>LgK@79RyTYj3FnI&F-ir=d-*x
z0|XE|<q&a?Q*luVeVY4v#>^4OZKumclJ0YR+6|Fc@@Sov&gEYr#U%I-{R32+b?UB{
z7G8<C9NaVOON-8w343_vKi_KmkF!lA38#d8>k^Tf9UoNN3p^Cb=6WoyZd)G(#B&)9
zC7Oq>|MVXFsxh1x!|BdA?1(VECKNe5=BZk3Q;YEl(YgG+RXf93GX8S#M=ZhqcDWs2
zdkA@l%|wNk7bMS8<<Yx7Gx$?UsCCo=UX1}qBij=0e`6-&^OYjDCGNG*-#;{dJR9FZ
z{}Ct|qz%+K%rL)77Nex?xkjZ!_sTC0hXwEDh}bWcucC(P)twVCt~UCO=lbAtsz&mE
z-mp8mE1yv$n6GV^+l*~FEnIE6|32OqXq(w$d7S;lE3`wiUS@AAMpl1C7BV$3Q7ruN
z%aR6`l5-Q}s`XmQ_@xrlz_WpTj%E5+Z6ONf@ZyZlXLRmKXeY=2RCwVP%6mC(QEkZ!
z{te?nHoHIF&fS;Ko`hW6NDcoAl)KYr_*3o*mcD8}urjxHkU@NpFrcF)WBA>lvK^Ku
zQ8Ijp<$H=Tu+n}M3}2-iYX^Do^*}Kja17qNzaO+%9LIl5ei8)V$}Kz0A7ZMM;Em~X
zuiE}KLC>_lFoR(#)5#Rr(i|=N&l;q7HXD^@?ap&f^e;bDSn=0Z{i3|{mN(2Bfr&rX
z`|W{Xyg;v^nfrqM;WWd>vzj}g<{a|y=rW}Hciv(M>MDvNbcQ!akgX#>sx~=p?dK+V
zg~?TSjm)fU^LsC0pn7gCOOshDlX$}orFuMDBCG|%?=612nB)?+Qflu6KO9|N)NL+5
zI7Xs|kcuS={#X#CfUH|@FC8RutyF>ma^qy^rgs<2j%i^R`vNz9<BEl=>F3{zbzYGY
zYeB<0yBFcXYH4ht7g9FQ5y`?ATzl7QRIto%cV@GCYs$&=C>IY4#+*(fZXe98??bUh
z_o0cAVl=;Wi7WDyQK#Q1QwVGES-!IQ`Z|)ybo}?T+LIkJ#Ta%Tv)+`Hs2qg_($u;M
zwQENMqbA2oQkAX822_G%<4=BO-I0%hVKFGgDvLUwEi1fZI-(?ujw>GY9PE3@W<;Tw
zDDvJQ|1jk<Px6HYQ&9`OOR1mJ_Be~k-WaVLllI;2Pi$;CeW{V4Y>;_p(A%|)JVFB0
za~_-ZCrSSuv<<nPTH<xSfg6&<EuPc+`R88Ta4DA<6<4ZQt~WkK26=qZ&`grcTFQeR
z^)jbCX7?Y*QpsOEwd-#W$m;f)^~HzM$%e~Jo;x32c(<9Qa>_Aw8*fPW>3{a7Bqs?w
ztISbr`ew!4-}9Dp$Y1n$>DRZ{=9;zd7hD48QutmBV<hc#ET%NsEq+7<asA42$7fd9
zSc;+Zv`~!Z@b7SN_V`e>BhjRG!QR(SjDx;wKnm|YTH9_~KdMZ+$EIraql%m1{(6a@
zCb0iqEg>z^=XC)yK{KmOVING7C$>{zh~Iq_cI1zgW|PbM9qL#(ryK0`zek?TZug14
z>xtT7{{9ZbDOI4e%YD*{jO9(TuId<tT7BvHMOD|p@{y`K!Qh<e(XT8Fe3k(@YITQa
zOpYnd)?e!~4K=@B$bG@1bm{eP3fF>6fOqWYw8e#=BPYe^L=i8XoZ4AqzcjVoe{Exj
zAy|`4enl)ZPG5&Ff}ZoB^p!^Jim`eusj-lu3#`Ut1vSi)SMpkVbbIgQ;*G!Pvst2T
z;!>Bw>x9N_9zDLJ{Z6e+Mi&a!;oSGjbBUA>pN+&Bix@>khfT#7{c-9W?&=un(Fip$
zoVM9h&l%Aq9SmLNn%=f|BR{fMqK;u%fe$=G@R1J%d+B{Q!p4`2sKH*%3D-ogCd>DR
zU&&_aA4D3x-7?x5-8B?8p(;#kh#Cq(-`=g{rO0v;InqX0ox_Pa9$r=6!%-y+juzTO
zYNTv)d=PeZFl6J4#C^FnyJQ@vm?4mdB&cOV`?xaxGJJ+lO%!*3=gRmx#vN`aqgllj
zFV!NStto;VO(Oq8E%aGO*X4{~z@vJ98IIkaeCl_n;}Zu5gTz%O>^=pRMQl0&ulANV
zJY26`+skMVMgBZ%(BYFfOy>2|4o_1eU-f2_O@Wuga~Yr4xLR-VrF&~ZCc7jNm~8KJ
zf#z*u*++E)TXAphh;ha7{;f-h5m7p;;tFaVRO=ZV_M_FLEJBEHKHAtvO}lr97}l?o
z@cwo0%yk((|JPUaE#2}LMUCx*a$%%e*r})fxt;g{la+Dwwa+3$7*2l^sX~DuX8M?K
z)96v&;M2zH!FHW3{O)_XXu-rogJRx2!J2v+Vp#Y#2$iJ;j;0G*9p8D0iU5VV*gq!w
zOV)7Xu&w322%n~v<Khw35WFkXW|J)02+qcFs%$pXZ{j%dzCBFOHhR}99(5Y4Al`|g
zJ!`jnXvD(m6%|n7@`tmQzVWE+)!zKLkQZ~K3>RsbIsD>%68Yh#!+kDh_q$8MCqeDE
z1*HsU0PR05eq>^M+nFGHcU6Hd{+pP~Cf}U~FOE10ub@CLQLO`5wlnqyGwdYrA~T9B
zmjH*1!MN~#V7pxLX;yMeVEnF->pU}V{#5W{=rN7|+NmOI<9wR1hsK4slz^3MJHC+v
zw__@W>iRM<sWAIX*G}Rl{lcg4-1pMC-NldVYwvMc_870Y8sQEU@5P^+Tv_M|yoqBE
zsyT0T{6mSoEo?$a<$YNZ%J`j{J?l+mq-@NeI+m!A)UO4Za(}pIh#H<v@ALdUPq1n5
zZfv}UrJmNkU|aLmZTORZ^0$32{sN3ROE|t3(J)h#b19mO*-W}?rIuq!mc&d(L9I0g
z3B*c=V#{qfHGJ;SmX>b9b9*9T_R`w(d})Dx!m?Reiim2ue0Qq0kcd)^{VgHkR?6D*
zubXnJLZ`^Li!ll*MSH<qaARUBic}=~-xn;Cmq@%CITGVg5<vma=anC=TvEop$9J6B
zBwCeop~(I7ix8uqVh!i=XP&LaWj{uxJ%sxhM@vU3yk!j~-p0ZXAHlTM3}q~HxnjK$
z4J+X>mM`l9mnFLGlUC{qIr+-W)lP%@TlL+z9;obu1j0nO&sKSofP0M&%jc2k9m$Mo
zO5B|%zOtCL-4mLSTydPOGLhUmDqA4eXdSiJ4Ik`UhMAo1UCK#iw;|=$!`~0>{uEmh
z_BLE8?hWsMhInVpth-r)+#c;p2_zDHIT5BS*lPay1p*77GzOO4=UiEg!_=>_vr+h3
zPsz*Z$$_GCOh@X~x=~&YUpKpkP#OvO7=hTcu;3nB)g~;BUFS6ZUQPA1qCQIQNUzH`
z(G(5>ZmmgAt0O<O(80bqQ$F#&C{Z~o`Z!lFd4+2}u(a?(bkv6M{KM;PWz&<ig8jqG
zvBd>XJ#??rK#5Qzs*SNXPWWz3x+p9UYX}EzJg8X>u3mA-OJ3&#{~Yj=VF9Ji*6JDI
zxF?ac=+@g~!z~|VR4pwQeurIS@7`|vwq=X|l>F^Dl^E+hXJf7$+jh*m-!B_h!iZX1
z9e(}Ta(`#_$IJAq2Wn!YW6mp$+h@8rO8jZehde+9=Xqpnkm5x#1wZu}95X1cO`zQ}
z`01KhgmfWdjc6xcz2*f+{!C3Csm#^iPuZ$w4?9bPx0sHc&*++s{X}Dmm@b`D1=K&v
zaxLDAYogoTkeR-fT4tTFDLV*z*W&Mv$9Pq*d!K#M(^{7>O>Oaf5vQVHUlvjQa{)2i
zpus!X*~q-gULAhubdM@^g~~w0In_<uoHncAcLk~E;4(+YE)i9@tD_i|MlsHw4vB!1
z6`H>zkc9K;ljWz}b=C#mF)jX0lOFtEa_d(jyaNT+6g%4M$!uL%@I!ZhrFKcr09Bva
zo!l{Tz2?^rZ}Gn-t?i>=ZKs%9on<AZ5^L}9ISyGG_Cq@;h7wXtZ9=M&^F`Y>`~tF;
z?Ye9J7wy}1Uku*w?v4N=XI1E~RCW0rJzh_W{}xRzL?Ka<I_~m0@b?bgeXRs<JAX^k
z+eZffXshEx-eU!u0D@w`XMkOu!(Pp^eIc8?Q;I}q7L+lZiRm}Js9z&mIwhSx{3=Wp
z*xK-L`gU`xDG2lPXNJ7_pCN14QyF0=N@tnUNlw?3C_BbHnO#Zf))G7vouwFZ`$+tS
zANxkzbG1sv?c8jnq9FLZM~vJLF2<*??C3O>TUbcE-$#$bm&$BZo%}9c4KXaa>h4Pi
ztG>TMwW@V+wwp{uiVu5kbuuX^Jyzi%bY($80M?cddWbyx?}T>KdsE#bULJbZ-{MN)
zAF%$}-Snh~0-t1)J1=da8VGsged$zfm1M58*oB5~&r27s+v>AhY#f~ph>7EljYE5$
zA(q%)e6CgY=3NTi*_kQRWXH5i!j3KPmyrf(`_F-o#s!%8<DPTxF*X~;HXFeYlVUF{
z6wDkyf_>+_OeMujTH*VqNY2_(AS{Kbjw^Yv!rAybwC424NjuNuvv_fLGcFfAYjgmv
zH=|&5?;4E>c6DXV6=pu6<942IWxq%%VMr+$j<~}*78r@_pQP4eLq1ear6!1e?`rh4
zFgz@U@oNwxjKZCuns6Y@RcAGHcU~t>=W?;jESZanerv;ngvx)z=^mREMYQw?Vmn}^
zMH8{BJ6fpJ6y1Oc;~W_wBcb}CG6@gk`nDP^ayN1@Vm0Ni0hNc^$3KpMI#i*tvWR4E
z7kkU_c0RR}l~dX2=#JGQPWGK9()g@hkac;#(uNaIZ7AZy=c<9<cJbocGCfXllyx|i
zw?1|#57axg0AaX+Qcsm5CjMOYso3u?cM8nc*Iqm}8lzR2Z1p3ik|-RP#A&m@eo9d+
zIU?7K{6$f$*pAxrp<rlHiB|mKx{)+7)wE-?80IEo?EyZmqjMuB{?*VxfqW^FCq`Jj
zlcQ;IpCV37-z%E90o<29=`gdkX(xVP6p<RV(xQae-C%S)Ycj9jZ@W%54Mql-E*Guw
zn0|PAWMgXowgPogD@EM)_zSGUBVn&c41fCe!qyvA;MJ1xZ9ebz?_L=Ft$);^HGsg>
zt|F6W$<SSy7<|7B?4JESD&p^)*l?_OectKUQi!i?wBw#g;Hu_Qfv+l7k&Or{hJy-9
zWrc@m(C%(n(wy|x<Svfq<u#jVwW`%96>0LW(Dzm9hofJU4t}!9gRH;+XYq(!pZ!O7
z7<Zq!^z#!-yNC(;iXJL&;!rVuisKPX6~wMM(Pf*(i_J533i>;KmTj}x_(rZ4b~F7B
za+^{7-I~&+Y5~X<r}DFp(Zs=?BvhVPV}bqD5?cbw%&-6q{P!E@uKL1VX+a#7R3kIy
zLkE9(%9qOueHmd0I>Gqcb*|@4E{f%rr=qDrS_q+^Ia|Ta<_lC$;PtHIpKQN~xJ9D|
zzmMVq+$RhOdP-;KRX8+-TW70`Xx?thy0)L_+x|m#etnyyrF9IB9}r||ehSB1SRfZ{
z3KIFQ^C2kEwoN)W?{=P~`KrR~Me{s35WmH#3GQioJb7+AtGB6;XE?5f5iWCH5bm{K
zC_CT64@X6W4OX3bQ^P6h_QUw97XNtG8KE8W(meIoi4AYVrb^06F!7D}b5u(891_IF
zW>Mr+^N;rCFBK8}*rKUP8HEC0c<%nj{t%SE_lX}4``taP(WFO3;eE*hqd<<Z;-sGR
z?a&@anfI3(G<aTD3emv$h<x`gNg!8sbFIs%vZNXBq;f$y)&Rgz;{qiLrubLkL6%s7
zUU&<HBJ>R62tPz!l=J!sXlfVZ&N{XZ?%{RS!#MD%_bQTY;vK@)CfnH@_$jG07R{si
zI!-EHbp)?ppXXCu?i8F&7OH#mFZ`f;Ihs~5!}wSAov8XYTg$nu-K&{sq*g7Wk`rD(
zAhX9x8_GCK?ROkJ|5o+dI%g`X_Lcr&v^#6SD01+PlNzgDUW{fGVkc68a#s1SjcAde
zr*1}o*PMN3z2&RKG1}F>J2ixaljwev+>eZE5z`U2jM9Jb?S<_q^qRd4UG`H4E$?n!
zZ0GurTeLUergZbgD6zQn7R%^s#~d4zHJE%S4s(=KQ{W+<J|7@faV67N3m<b!n{fYD
z&u`V|RQk<^aYkQv)|-O}6F*F5{z6X{2t@04-PMQ;U)@fkw{bwwcPV<V^vBK}*i~@f
zKfE55=LA$mk39b=21~{lXL1E^tBcr^h}^#M)U(%CvlMTYd&C`B482O27D^2_GllDR
zlDdAu623p#5pY0C91S#J7vyr1Hjs7YJ=H(vxvG!9eZ5d-IljuqYFL-qSWB|gU;p&m
zGsB`M&NVWH2M5>Jz>BL5uXZY@e8Kw_-+Sr0{IsTk*><f%^vizR!Ms7$Kw@|B4ot^i
z=SfI34b|Q(PA+~5!Ff=&!<p1p^)my*QcJ-dn(pxU90NX&+GM-_RfV>v?!@2Om!xNe
zi!9p7FYN}1B%&#cIeGJ4ZEKyEYpv!TZ;#kVHNF~9cKwlQa~!8Jy~h|l{dE~@ep8B=
zD!f6mp`G!)aQmD0o7=P1DzLl#_T!(wKcb#<Nj7eEBtxtoIrGNtanzU(D$`z^gf=qe
z7FsM%h9bL)n~E;WEa>~lg<gDtN>8d`m-DH6ec5*oW_p+Z3KS2uOLqzt!&cuYt}d7o
z`U(>NF`V;GwHpv)pN@W3C8s2u9YViQa{pRP&iMENgV%tY2h(_3@s3208OFk1pft@T
z5dR;&FON-*>C0wZD#?Mpo$al(Y!~dA+wah{aopW~VRSDj$YVcMv8exk9sa_C;^05?
zo<I~5PeVDG9%_&iOdff$9PciPa9RIsZ`u|3dst2lhK;ZK>4-li!Zmm*J5Tci`UTG5
z=SJGrzZx~u4^}CgC@CfUb=rp>4%8E>F2;1NEzF!d?28=r)IEtHmM*18$=U@!?b!KA
zZ~J2(t~nf5l<JNN;}|Y@KT%<a%R$Mnnd+d1zJuwuvqC9c<?GH+?IPtkeX^;EWUHwY
zHAWiUx=QP~<#;<LD5O;WWd~}8(SY{f&)2|?Ps$z*pdy=w??18>(5`>PlRGtNPao&v
z@|LhOoQb7_1U1+_b9^tV;yPs|c$Tbf=({abPEQivwH?0N7<SlUY&8Pe{x&nn=Z<pE
zf?=1v8P8SiT=5(Inr;laALr@2+k0;#F4$z5Lkp5jQ6+N2g2wi^MdbvgtiB+SfB91|
zj$!Qz1Uhc(V%`-UGX|8cLCc@tzbv*Wd_+to&`HO|*U|70#`K|`wds1@@lD(gr*}#S
zz08Yy&jc=vs<9(d=B|~?+mnf?SUbX*7MPUcql!|R*ypgeYU-PCcN~!$9-h}VQb|K*
zLSjU*N#7?Q;0Ga{-rQ3<b0|(Ap=69N;NTzkEUjnUY`hwh<*OT@zO0_fs>zK=>#j{3
zn23`LekePu7)cv8ovpjSdM5hqnTyExRa@7DKTbt|D!xbj!9+h=3Y`hswaeL_5)KHm
zomR$ax9~30)sAN(wd6GW$=4=b+?Y@;iAV2XjWEBeMyY)>X<B-)ajda!Id*D^y<^^d
z?qX+<N-;pZK;=J^^j)+Nz0EI3Co6xlEL{IMhY_Z`#790IOd4^bC$L?4QXY~rR>QL;
zA|qe7b5bYEe!oZ5$D!50ahh%KS!G#f#;G=JSIj0`ZA0k!(FZ&FSi_IwJb5wWKavD5
z^<k#yd-snYJLikMUtSRE!`6<*b)NlY9jWV{6qdR|#OSHx#p6O@nlpnm=8QHsl<V$V
zopiRjaOBePYr_iGX~R}Q*a_LZQ?`(h#|s-?TzH&6nUAMMd@U+M>J2+y7&pG|G@@+m
zXeoJ@1Aa%mpfZ`FPju3oSpeON1KtC7s^6mzrB$<8R0~xhp}J5u`in3~y863U9}3%H
zq)tRS=nJ?Bh<RSfxS9P(ER^K55YH~ZHwsd)J{3fX9BFt@NB@_sy6aIkj>dl6M7|e8
z!N-{ijd|3@F3yeh*X(c%kHH`Otgfj_2d@u_#PqbCKwzeX3#HC`g`x8BwD{B2XZRVy
zRCGP`gno{&CD*tIhd<n%cI}M&_JE9;@^{dYC|SfG7~5_&GQzq?3#D+|)(GHwJB>(m
zzwr)fxy?P$OQet6Z919n_dLxJt$AS4hHGj|;muQHzm7dq<E0d!qI?n5vDo|pNfm2(
zzdhb`k0kz%f9}lBN(9Aw3|pE92O=MelIxfQac3f1^(b_jhgt%T+>t|CrM`1ogKW_<
zMd>6OYFNiFcd?(>Td_GhP*}9NMzB9j*?Uie*|m4y$tJl2DTKzfy@%OlPgNXxFheAU
zb#dqCRGIz9q&vTt%kRHW59M;Vpa_BspCMn7s;q{m#2Nc&gsh6hF@;v%A4scH5qAlr
zhxIot)0#-Bm+e>R*XQ!t8x?;j+Fw;m?J~kA)q6@Ax|0-N+;f~3WV9#R+rv)oJX?SL
z6j_ldff=#O#xw3Y?%7er1&R?J9za*!8(rUR@eVphJ@<H^Of#t#Zhc$LzID4PBNHkp
zAlz{5`v_2U7EG84V6sA}>@Z!Ql094DO#QDom@-0=C~+oG|8L^^5CuN3@Ml87%|^24
zTlbkZ8xOmmLpfN<Y#kvt%hNmexMc}<u@)9yvULm`3f*UzC#L>E1oc{o*#d%GolK7V
zURhRsHaRZT%Did2RnCU`C~~l-`>=-lsNEen?KG^H{D6at=D|oOi3$}~G6QzGBRDr3
zU9Gkr@NPCbD{9@mu3wbUj%5Tsv+Vi+=X&CS-MX|NXb{#jLA^V1v!BwEhzD<%3)RLw
zZG(SEVCwq+Yc+-))6!^mvB^!75@)-j)*?lgP!e<>VS^SP0n|jzXggWMYH5|TL-+f!
zP40unMf38>zwY22W4uRXP)oCy4r<)7T~cuUZ|m;=*t)Z0v^(vUO&oH40OI(Rp*KAb
z`tyDu8)zXWF{;d-flNQ{!G*u%I9Bs^P>KoS=S_<=;4KSsg65?rYN%XqdSB?6M)HY$
zUPm%v46P1n(#mhI2@R<Giua%HD~#A@=D4?=Rp+KGUQka+NRtA!K3csubIavahlV@U
zm{t7{F#?w8tm?Ix49amrs01Zyq{a;%3dECmnf{_j>ffqM^Ex&cVpv?@=dXYL6n+L?
z5KI8Q;3|w9KGy_m<rNse+|8ywdRqYX_f5Fi+U7A(n+m5C_t8}|?++MPw;!3jC;AOH
zu&w%2O)!p@D&hI`mEKX#EBEURP=Oz6UXEsVUUBkGd6yjywfr)n7Ix9~V&Kd%b&q5)
z7F<GIvE|(9rVBJ1k1KVO(A?$U@lHJ@{IT{GYiiT)paYwbY^S|8G7(T%^_(q2F#~wE
z18Rq2wY)*sIhOZ1HM>8WGaDW$H_F{ZkrMYj2tXhEBXeIhIp;W(<dYaRA<a}=e<oCQ
zk%z_r4%Mbh&aW08d?;<+#xTtbNM(fj==29=KKRjC-;%i_MhA=(c#v(GEK_mMKI-zQ
zc$to8+UhSV)w&}EMz&`CSo!h^zjx9}u1VJ&4Srp*b^C_sB*7u5Zf3Y?Y=a4k1gA@4
zU?#kU&@I?Y&VBVB|HxIksn#^!wUZ<CjZ|?DsiDO9`YuP~H^>cS)ebLt2&iVu$)8sd
zr@-u<f3u7A4J^D$1l^t`&qtUU0nngCZ?7bn9vG7VhU+}&6-@_dn0DqMQw45@08+$?
zm>Ib<71TujuA;EM$b#{c4|Nsf?w2rKU3dA*#Z)%C#|XO!gb?vte6XLcfk&%ed4AkB
zXp)WCksu--o6CtI8&V&=sPV(?7(nc8*3?U6%+R<Yj>Tj1G~Xkxf>%s`26K>)PSzat
zw3bx|J+f&zaX`eu4bPXm4q%)D@8zXI8@00M?=c9`mdYzaK^XMK#q?+TSh3OYT+>$0
zwQ5@KTeVb;vWmQ~x;aBs<OfflJaMubm~+}&5_UUWBw1!p4jNkSl!>H!KYP;TwTEpe
z;;af=rIYn$0+VKGAyD}_Sd(!i>h;?3?#{XS>vE$Z+v69lEH3s3ZVyoJ<E28P^#=Dw
zaD&MqmnvaPWV(0%Ut83eArMhdXLogC?sm15aJT$j+nAF!-{yWSyRufIcSZ*>$5SW8
zH2QL{Z3GxJR5`Jrp`7siwO5wh)@%RVd{2N%4xj$vu^mp<%s`YNM&-i^L%u~#;E3nK
z(?)ZFC;GFRiWiluB>PIR&#{i*_Sfq(v*oAVdrxdO%I!W*U2exF5*B+KHK-Wo4}SL@
zzT7j$O}}iq$}|2W?DB{~Atm|e_YI}`haXpTW!NuPF+wO9%HRrDv$YO}s=JL&8}-A^
zq(V@yn=UQZp|&Vxy2&$w6fZ@w&U-q;(BX_@SK*oPF=LpnbMIG6?UF=;!f!SaanCar
zu0NvtOSD77DR183K$L0hz^%KEqujjGCl$iJdt=_^UdFftD9M5j_j=R`Ds>dnOc%7M
zkSp(-=GE>s;GKJ&N()rf5GHf4*Ewykv7EVF6Ruwuls0^oZ;!GuSv^RS6dkoD6vH<U
zcE_ia3ukxJ$0HK_HL+LugpIN0Fp-^1*q>^;hU;=u@49?DzUj)6(>d+{{D3>h*7IxJ
z(A^XJLA}N9Gtg%=d-&(jis87UdT)+jm?@BHfmbe-n%8|m?=r!oS&Y{I4?J$cbygWP
zwXjV{L&L9~<pYC3d-<MJ!v2ySb_Wx3@*wy26kHR`f_>O|>mgBakj3G7`nO%_tlm>P
ze>6W_e|(<votaVX(c9mzCC}&U7u<x1p<$K;6{zEWMpgCZwp0iTJPi{rku8XJ47}Y<
zyFttEgP;tml3ZVQ()HZ9M1^D81ySlFt;dQxC4&a0K>v%8MK*`OWh>xk8X=5<R|iC5
zx5A~fp$6KY)o)S9Kdlqhf}=!tuJ?5Bc7;6TV3m#73)ie*XCEIgCoi$0v)_@k>z@*)
z0tGF-hoDKL44O4c*c?=Ew7yQu{kF1`qowH&)yfj-$xtKq*8Aof-&F4?bgWy#j7%ln
zjx_1WG;uqQta!U`dr!h%D)Hiv1`GF+${s$ig=T%+fllKSXVX&Tnr^nWad|@i_lqHw
zbm22cu~9c$hkM=9o}?e)4PpDYAg{#3TX$m}p~-`)O{tB&UmicUEg$U``2Cxw%H-So
zNPvY99j)?QuXO@rDHC<-b&i_3)vnZ^pklo+2`JX*NSR4E)KX7(jX1`C1^KAGy?^7<
zDv95|;sOgy=o#5;kK*lkl}}+_+Y2TOKct5Oi&DSjCdBhv8J&3FJ2>O7d@%Z6yvRtq
zsRN_8*s&&1!{FnE4Wqqn*h*2bE2UvKe^jn>ZDo2$*k!o`Xlxh{uQ2>!hQ=tI&mFex
zPjI<!=B&zZCY-L@aO<cq2=v2BKy{k}Zk*+3T#k9;W!}WnEb=QiCt<H$ZPhJWdRUXW
z*ohHO#(Q1S8JA0E5e)x`Zlj*@X1+Wqn9%mxDxRz3k1f#Q7`I#CfSqwr$xXD$xWSKY
zW-~xZlQ`_uW!?XA`vKp+U48~`ZTEhHchQ&ey~VZT3jf&={jN)Xy@(A}P&>;sjB7IN
z7L0R`l_8QIR2=yxNv(DI9Ipnu$Ktu3?Y(1xzNzbm;e9(Ip&N}|dGKFbHYW;dT{~&V
zc%^7+H<M?WH*N&QmX*e+E7+y=&GD0oMXKWqAqMSli#3?A7Y#iJKU8bxQ+>891&5DD
zBorl5;fnaLL9I^nsqN>j7$!BQB+?>Us?p}{EDJN3YX;~}GraG48zzy<Rmbo0x3$Gg
zG1tUtXO!|%1ixp|m3P`bN<jbVW$mCpnV8&g|Bg3f>L+p2nY&p(=GAzh=rLi!4|3|>
zz^Myz!d@O<?)THiefl?iMjL;7n-%<=*PVmm5ZsEFyZ5B?98#2;9q9x4H=cs|3ZT|%
zST>R!?t;FbUsJsY^C=9)7-hT62HUSJ1T)mWo{k0bzu$O<ILU@HSisU#l5|KZr*v$;
z%n4p?d?%31`BM26zRDOFw=_bxLTdu!`?#80k+%C3W^uer{{JB*R02O95Dn6mV*D3y
z8Q@hH`1lBt{3;c@mfvgcTCBlQ&|tt9cwdU_aGunWPKxPQ8u**96Yg$j_oy($Llu8V
zj7AhQBCY*z!+!hY#!Up4kx>e(PzUj@7zgd<+V#(zf`i=8U$6E<4$sP3{xg-axyI3k
zgfsl<Y|#98*B6I}ZkjLl#;Gg+k5e}WCAss}nU7yfZ-kWa5Em!i!(9_)FGPt$;@gw@
z?4sNnELyXIskbW2yZNA*6dh)-(*!4?0%fptl*&2w7ox96?g{e@@;5am??<)31Oo((
zd%=Ln2F&_X#yf=Zr(*GZj-JAi4>r&1H*y18enh+7sEZ?*zhpFo)Mdx}M$%k(Gk@Ns
zMl!zz*?s<QsoZJnS|N!?EayzO<7j{)yV&;V7K=NWO2{Z!z0R^aoy+o4`YG&c@}$*x
z$#p06mm91$c%-=KgI0;+SiZapI@~U#W4J5RWcXdr(9ZHuWqo2aGJFAc->mum)^avn
zjU3Zr&%7Y%!Is0S;izkCBfu<pHsPj~2TF;0&jY;@Ztr~B?%K<CPQATY8x_aYG+9Z4
zT<OlRs&~P(E$cjwXuiXEBZW0V|CPcZz~!U>-3!?ph1t~!rlm^5Lg=IHQ4y7&hsO^f
zgZrnyh7*c@fi-cWb#3H@vY00>+0^+AZ19G|zs=B*U%e9O&sbX5?BRyRn>g)@J7kG9
zsuy`Vmljm*&yoxwudiE^2yBR>hmjvL4E$wnrH1%emf}5TJGsgp48^c&>dEa?nE$ZQ
z6LE{HA9cTa*Zo2$0wzwoBC5R2EtA%~rMB<`UsoKCXPtZ;=0RG#t@TWqZZ%M%4vO2W
zqVIR7^n~iz8x4lID5g!C3B?xbYi-^#PBLxxET<z4jcm||4_QTC>u|NB`)xf`ul&E-
zd&{t>pFi#w`H6(mDcv1P$HJl@D5=s&2q+;)cMC`gNH<7#w=AG^NGsh-cQ?yDi}(M3
zpL0Js5AN$cIOqDwu9sfl-DP%WzB8YhdA~oeQHS^Lbl0toRIs+k0jK;C;#p;Nxv953
zEcwZQ!p3H;I&k&@-6b@{{HqpY*n0n&tmXNSDXc)6+7FBF7oMY7=Sx7}rZW7Stu2*^
zr)_6eC<`NyK!9CwykgMopP!ixxWj8C9!s;tFa-;la+-Tq%+4FT@Cv#VMhDx)`UkLS
z4rR~&HyK|$llPa7BKlnMPua`h*cZ+YoO+3It8!nop>H}0(V83=c#rd@btK|SvwKs6
zcIN7VwqPdOhponReD1reFGNk!4qYKytg!N%VP_*A&^U<AS}~d|zz)Vopb*NBAW_?E
z&OHwjs$|)nnR+*P%>#m)89)B@I3}fas);HVDg@jcHqgjN$qYu~<j=&X-^$=AgD&sc
zn@AJi2pW@IKk}+3`T(!x9Q>?cx?l};zZJ&*qWVwdE&*cqS;n?Sp1&*LX1NFZXA0i!
zC_LNb(;ITB^eZyQK=^1@NY>Iu*dovwx(Q>#>b(|wZLE-~JL-kIGcE;+9~tVoO|M+C
zGE;Bo5g+a^w88Br;W|J=Y?z4{pY<{gTXR^T(@!)ik7_;&_+97a797Xg<fw%}G%aXo
z3)>eonP4cN^v8=O;o+tr)q?A}%bVg@;V&{|-|dd3DA|~?%$Q)4w)SC}=2Cm5fi;(W
zTg7*==F6ZnO*_uF!D`%GsancdEZd?tyi>Y{6H~j!+;LJPXF7JAJrZYCWFAA=Y)lF(
zFb>#C=YE`%Qre`0-*ng4hlfgQ)}It_m56A!aS@9payR7$_9C!d|E~noD~4<llN5D&
z#2P+KEHWWtFx(am(Pe);&1Rctwvf``(u9s1GVF?1CAk`&mQ$t0N1iCw5>?EDaEo1J
z3<OEeU*8qu@qw@SL!nV_l)Otl1R45VT!@Qh1En!;opf}kou1kBw){wuJ^{=f5$!~p
zSr6L7I93+|sRdyP5*M&?jcj~@6n(`SzOIj`z~tttEAhYW>(j{5ROkv0W+C-^WtSbo
zKgTF;n-F~#X?|)ZoCnhQ+AW5F%za4mij*h?fvu-92K`YY^4*($F8E;g?7rq67OE#b
zO2aP8*9D?bq?R0C3GEB8&ZJ-#r8KE39Oz4De^CF*;U!YC^nTTSip~nD?bbrxW<cBR
zZ348osr8!61;}?A#0a6T-@OxpKYn_+YuGQ3-zrtf87n@;tbtUUN+K&52-`7IQMNqy
zEKsk*ZhzaF73wfLQZtVAS2juT?{>@o({>9|nf|YYp8EgTTk(+#m@5ovl}B0dfTlJf
z=-W5iO_O^ff0h4_q)}?-fXkvn1Zb3#NFP;@I^TVHepfGn_8I7`v~LCPjC)pMGI(+T
zCu!MJHoDLndhec2_4c4&w{99b)Us<}ZHVeax)@>J5sg0>yn9&odhW8_Iv^@Bf1D&K
zlI_3w1U68J<uJ{vi0`OM)UYePyNBF-e1Px0Ed=}o7E0hAT|xV@g+>i7f6`!nj*s+8
z#4CcbS+oSsVF6!G(M!4KnZ+324$9SR==pBT7Kmce4BQ>>?0~OHkmv0?9w03|z=ZKW
z2}XJ=fJwRR+L%!0N*&m+3_ibpO8)T~N=$fhFd=Z>fvzQW_w6M%uv8==O-L?x7~DR|
zqQofA$6)?-_ZSm`vL!uQtVIPC!x2p|Y|4R}=Wd~v-1PuiHZppYDJx)bnE+0@fWG@=
z6Ze6M%^kH=;VV7TQ1K`Ulx@NfznK^qDmBxizskJ9ms~a~T$DDxEgK%FR^Te>UO_sJ
z^nP^d&4bH?&hLGQZXGidplp52U(SjpLB(KY`|(c(qp;JwA!gzI2RVlLWBfRw09Fn`
z7VymuQDsh0!%^7NesB<GVchQNGJ~=S8njuVfr?RQWy%^-2W>iDARO`Vwru}jZ*q)k
z9a|I>5mIFC>VKR6|Mlia-fRDTytM&+iiAg!kp|h*{`)BUPe=W~nasS~F~_x?i^TSO
z7TbN7fJF*v0{>I-?A4_tVd}jpD^yi!;FWu8{x>2k)@OK!gZxMs19C&1TW$8-e{V00
z6l5m^Z$!?bpPaC7e<ko96&wuFn6>~T8s<xUCUCj<@4B?cGxVznvXk$HA*cTaRhLzY
zmR2#nJ>ZN2el6gnL7x>sUPrl;0$w2N^$impkD6KXSv0s@^LP0;ej)I5<@PQWLoTIu
zUoWRvVc_mM{{)6N;I=|;m^+XC{pP?B#!hqniJ20_C@93Qz?UCS{ChI9Ifjvdw9ElV
zHLyv*<YSzg=pZmS+8nYfZ&#`Fy0_PX?Hjz{2tCmn+{^z~2xyxB(+atgd^?eK$Nr{Y
zkzMsW@A6F@jK)x!=##?y?wnUfW#^5fyy0;4dY<X|z)Z=Tbsx}86qx^73B$m!OJz2G
zLskRbdjDVn-_f70!+LvIOZ0p|@4Lq~)b)6SDZ4+~m-pR(RFFy*=3`1r?GVY`?@Vjs
zO}eV-O6DJiyn@YwZccY+?Dv5^#p&{1`xoMXVPlVQ-lZ=2^2Ij3?S#PblpkL{=?|?C
z^UYW;v7NACCtqexIdJHLd<4p7KTp6xz_(7+DVjB=rMaV@CjlueNNbJjmxT+jRnwg^
zr7EW>87$Z9$!_JW=Mj1D{#>Y*9=?=|=Fc8!EKAi9@rvxfar2uEdKhv*%tJA{hBr%l
zr;;PBxy`N0S%>Dk^+DO2V8ixM<?5-U?17EiY}Qwq^)tBP_XkoXatP58Sri)FFC%#W
z1dW)oIJw{K+F$+Hb&d7ZZHVVi7JZ6GEgESpjxj)uJC3mCa$Uv^*mjb^ncDr;DrZt4
zAqv9fJ#}l`&azRp_1WT0UPqEU6NjYP@#XMgG&(HqG=SaV(X609PS|hMS9z2JCq23h
zpBoN%U+^lNZSF$AQkn#EhmRv|cUActnvv@os^F8wrRYjYkb!8AIz`;|CC>h0w1<E_
ziN}RY)I4GSUVnb;W!}{4=ViiND-k8OVPo|(`{VG$$y||Zn5%?efa?);8kt_pKQ(hk
zHscrc7R`41Ehl1!Wy25s0%+toAna+KxVtI=JuYi2F(E`^dZ-_S+_{NaUwz7u2`O1M
z9T=D!S&6C4Yt2I=;xRD1{tm@K1)6}fNyPPQ*HY^jJ@I^L&<e7zHt}lbl#{^c&(AIe
zjzFTBPSJlmj#J7dF~#-IpO3IqS<X6tciSYdUK7%4rk+L5b8l(5>+aPAdfq9KKQ%!O
z^a?5pS{jk+zw}QHlu`7P9WG4@jOYH5v-1FF8{VOeefsfz7JS@X@+SL+t#ypY-o1(>
z+?^-e(Jjlf#S}y=PoADv$5ZSKWi-PVFZ<TLOq2s>V<-;ZwMTzyyT>txPV*`SN6P*t
zf<ZRN;yk#^ew|boCso4p`K8m<a}A#4oU>i#Y{)aH06*#H)e>vv{^pyloWfP*ktkW*
zrLnX7EI1?We4R^i7YF(^8L|K5e1{C3s#u!R%-x{RG$CljVQv8g%r8I#nBboygvTKg
zFfZAhsDSINfs-bSU;4r>(ruOUIU;HuTegPIU@zZSk!&^Q;XdSks!_y~!2F{6`>*Ei
zWnW8l`Ub33v!zMJ%2xb-Xu5deOT=Kj^>&10IuzZ>YUro#oAuau6C#n<@8lx5`Id=W
z_k7&aEcKf?Q(a^pDKcLfN7zO6Q;K}U<&sO|5^B6LV5{vE_c#a~rxEjH$vrh^3vc+M
z&+bo@6WMvCT=9E-eP${D*v^Y)y58m)0(-j7J#{t+KV<k_Kie~_>ASoBXROpDj+$57
z;JTV$2-}L$a@I6`T14BNl4<`~Prwau;tNZYw!1lN1lk)%@p`DsrHR*$bOsNm?hXK@
zUDJ@?cEI-O*p<Y%=N#gm!KDX0=ksf%*O`CO69~}Fq89U)2jjveE`3D9-z=&Xg>;^#
z@2-SJVT+G>zi9g)qK}=z*6zL8h&e+(7<SyN!_$y5Fn3JD<Sdjh<<PXZI$F(NtbBj5
zqdrqUNrdz`wOolmkXKl5G(4Q<%vZ+!82X)uS<KCLx+|Ol8ey?kLoVDGMOs=KYN;=u
z6X$kEN4F!{CI7k$Em}$8yQ2fT=ahnPk{9z%qp~gwXXY#VRQcR+5trU*4k)(PQ6(kG
zs=ZXQb*sH(W`+Io6<h9`-TSYc4>!CPFJ3e&aYHMd(2tAtwIvES{gzdWo+{?9l8Y>)
zR-ZF83|F#*r@&aACx3BO(p8)|@LaP`QPvc;V+htTS?oT5i;kr?2$hPQa~!=Oewd0@
zvr>0wCjY})qqZ7_Q;v+djQP*23o$QMp$mg<e%JHGdKDPwc!AKSj9QXotYO=Fj<6Pg
z4e9<@K?H@PJtyp5-%Z5XE)OG8l<gv>;wp4_R@>ULbr>t2uTX4E6p4q!rr0zK3~y(N
z5)2Efr_Mi48se}ln#Bk+HQru^M!Ig|*4%L2*AKt-Lh8$RN!3YtA>^*x%g7ud%|5GI
z&T!{1!a{E&bL5n#QC9TO_1J<~i0l}W0xT6T5Z|W87HQ%J@boipzJ@P$2=Hw76dLc3
zdG+nTEUjrZHqTXRHVLEtrsp8ROkEQN`N+H<Q(I->z5n$QLc4#_XsIW~GIakRvD(4s
zbfT$HH+G7$w5!qxBJO4@J;hv{>t`=sXJx03jJ=d8--v|Hv^*AHZ#aju#hmf}YOT!(
zDIEzowTKh{=ZX?x2=ScJx5QQe+GSD;1Oeh|g0P!a7GY?AR66&Lk`fV8^G}}Rl&5=M
z131pb#D5ta7WAThdYqfHM=;wo`hZ>K%A)VwPOVTxN^H1;Nrmk~?6K+4&Zg!0Q0l|!
zU8vX9Q{O~2S|zd4oQ~br-qiuq0iMnG^#?i;Hs&XFT-M)!hpPzcg^11w#@w8iiWKa1
ztFpS35p~<@<E_x`;K^JhHeU}n+3)U1oBQ)dgJ)ewGi&dPxi6+beJ;~`bm#J-!FB^r
z`?15<2mM}RB1^CWXZywZG+RAlp?O)a%iO8Uu0@LuZ$Vz2QOIZ3-pkKxvndp9yU-iO
zcU=(yGTd`)bCvVME5&>B&F0jponbdh&NyT@+crGd%|>I~sS(S(+nt$2Qb$Gk_Q;K%
z!z^vk4_a@3D71#~YSJ)TTEnYp?t3TrPWF(IB0N7YGRZBfZC%C01{Agh#SOAemnhB+
zjw|W+m|tbbH<O{S!cyQrTQXFtQ7h6l%a%-jJ1p8Ze(KdfyZp_?L?M;-+jNUrr|#67
z;_ynqc}1Be#Xrlsw2kJ?=wL(l>!IZBRbTlM<uMm$EU~eB69r~cW<ciEQL_7Mw|zP>
z_(1bgmXO+=TK1^rL1MF3n$y9?^8F8|6PFc249Y{wrt`2q?MLUE<9W-eoI69q)1zv3
zYnz`wE`8<oQkU>eRR1cKRS`mG%Mg2$Bi=vAgB!P@$&^M!1`B+0`l66E5%1BKGb`0;
z*4`k8WiqaQO)Hxc=r_-(V+GzVD1RScwwWsKHI9-RVIRE=PCQLnB2pK7CEjV&>A%dM
zZtih@wwU%Or{5@UO=`z^yro~UFe`BWbbQE0)Xn2OVvBC!ihq;u6>g`Y=i4aTirPSy
z?Zc((`Q+Lbqmx0L3KRdhC#iG@N6LUt<u8}6_JW=2LtyLsSUdH`IzYgm%5HHpwGdc#
zJ=+W|p?iC&8+6wTXNm?wVYT~LToS8*frwLnC|#I?9f7^~Cm>B!piK&cWP@v`!%CgL
zk!;5{rcNV$dr#!2|Gv8ORgDc==<aJ}PqnnBWj`XUu!x(ZZAjv1?Ptml?bwWc`*HIZ
zG|Vxf^*7aPPmaZKM=ZPNQlLddQuZ4?aLmn<-p89t)C$4|62F361lB~K;uPDsf63e<
ztF|S5-={LZU$gabQI92+@R3U^ROwVRdataA(oldcwE@228bZy0cn!`7hVnmnK1STo
zuwR*}i4=7oC(xx>U0Px)ZVWwlO}J<^SUl$lvz&q1R9V~Ox|j{6jU|&e0n-w7c>ZFD
zSct4_C$@uSw3lX$=yCj2D4gqBvwCmvESKb*veLD7q5hFP%TcD4N!V9$-b2rZHD=>;
zVCTJ~ch_)%M&tOZp4yoM!t7|pN~$}b+FjZ{mjYiTEU_t+&_EuV^ZXH~F1xmpC;U4C
zg5faAcU!2Roj>h12Z1TmLi=@ntJsMDv#9s*_veTPJ-?%^7#6d`y_m5lrR$rX#Y=u#
z)RQezIzD!@Sm9`y-+K+@e+uNetjUH9CoI8ZcN(*IJNNcxKYrLV;1wy#F*tptTKs}E
zmr+FUAb=auZZ;klB@0d7gGC28S*d(Ivy#lTtq90*oI)6Ose0v(Q;k%;8&DLn?;G;%
z@S!MGkv7t`<7wN=;yG-O^U2Yg3p&zj1X*?_JrDv$Y8)&5SMsX)@>H@s_U<})o!WZN
z@zYt@R{k;~_ZHJ3#Xaw0;+&Ow-9FDgxLmjoYS@ML&<U|hra+BruJ#!*c_J0)^E?|=
zypJdvVNWWiGW|H$KRBdSFe0`~=!S$#Olyizz-5h9MmiTqZ`8RM4?eE);S!9PdRktv
z(Tn3D@%@>NzVE~6+3ek^!pi%FupH!pymG;3bkL=T?W-z5Ro6S)-JP=fLSoVUSV_hs
zAGx*!d;Z?dR!!7Oz0AjPQUXmPS8baw(ywIOqF_Q?mx)R=6FdQxbrPd<Et&eh%1fH=
z)P!j_nFl4|nPa8;-8_}<M&pqmqbuLq%qWA%cKM1hHJsP+nEG0j#(kz);G>qno0BYS
z27QVZ_)>Q#;m*g2lWT&>3R^*4qDDOToeg(<>PZcGjRl*XGpM1kw&yuU>eoJTypsM0
z3q#d1bm!E2cy$tdO?Dpd0#_l@YhX>418eUY^sY~uzN&~Xh`H>;((1cVz)Qld8M*cY
z4wrEn+?)IPEHc@9*2%wT&3@5e)Y(#r^Q2Spb0p~D@D(v0%&A&iJA}O-piIbRSp`G3
zVvX}-^c|bulmky(O06;3bsN*;Y1GBrM3}V6_&kv!$}HUwmFhk3?Xj-blV+xcOMFm&
zj%C*!!VizLkpd4gF8qHGHJ*GL`cLggOHG!tUj6gao@Y|yL(`AK$-Of-L`zR^>cQYc
zJ@$T3jms<vK~9qbix+*9v~fv)u0+Gk5&DNasx^*wuFTRuO3g`!!k<F+fy>Ei?)~&@
zu0Vayfa(JFraG%=oAT#ftbVv&DXabzLWwRVU!b4Sp&@DA{nbvAz*!bg2dOIEhrJ%F
z6Yw$%=RKOLdCFa%#Z{{C=i}m3N3qd-;%;YTQQg*t)5DUvip0IQ<q`!mNyN?0+z$qV
z9xpoY$cCO8cGZhixApJp*U+qyY)xiY{@xhjJaBEmx65X=K4jzuH*}To3F7;wn&LMK
zmyQ;;+Ad#M5>L|_pKzNLovR%eB6}Sa+3TPd&Cq~}mJ^Cfod3P9>f(HwRm<QmWDuIx
zJowH!9T$6dx;(@pUoXx;Nt{{RXlky-H?8~fwRP#UOlqU23YPv`F4O)j9?fqvbV~lH
zNLn5pkeYgh>93|+K#D%;7Csrt*V!gf3b$0@Zb+ux0Vksz_6~*T#uHl9VZc|alk%{|
z2&~eN`o>z9bm;Voe%PQ^Y#!<k1k{$u0JFxm;60x44c9j(Ie*x`zSMK_&?p97X$O1Y
zrva%>cr<K%Ax&`E*01hCh0#Q~PWcG@pp~+?v_(pu;8e04d%jQ5MNQZdX4+b$uVk(l
zeT{U&{`&f1;m*{qPGZ@0KLjQY1~!=oHFIPvnN7EmUM7>_VA`nf4~jEv-K)Z%FEbYC
zM-OjY`cOrn?<VwXXP9%6m>{uATgr3&WZZ%w@=qZ3qZFN?*_LTu&sT568Jq7O@>as4
zKXmB8^7RaQuJrEIx;qm#vowo0vvSADF2A?$ES}h0X@9WrQ(RF|LXIKrPubM-g3+qN
zDZ4&AP$}-@r<C$)<PP?3me!<Iw#(TGCWtNAE=B;*Sc{16uv-Dck~paY7Bjux_EztS
zZ=QOR#7_g8hLYkRQOca8(_a0!mZwgLGPxb2mfe#I52tD7H1j(x?2szwGYt+giIb{F
zDEdMp`N*p9{jl)U<<srz<epF1$-^3`v-}<8tg63rJF7NDD`6On_EE{h2JVXKc}B%8
z94$HLPx={>#f(xssy89LaG0U5C&w-2T@q&Txv6svj`4FI2m~5|s(W8^V-g_Qt>uN0
zabK6S--U{PB%s4ZgLfajYV*W3ZlJCX!NPHalt9|yD{nj!)k+Y{U^`)DZVAqBODL}F
z6A)LjebD^$pdPVhX%agbOa*&5Cj2<uKVaV3R)ae^Qf%7%lrrwEV@gu1XO)2m1oH9x
zlbcdQ{H2uhUjH&=IwU1GJ9HTviWte&dr?l>t5koA4S3)70Ztz=jaMGIO=+?Hpo+@I
zcA0<wIoC>J$GLYso+bFu&qj|eR72r@-#&~HLE$v*{px<D)g`XsQ#WFljqSNC5?M{;
z$Yb!r$kd=!3IvM*>g)y$xKp@OyX)9K+o#g)wdS`1aI0lP3s$|JYPUZ6wxk1_%#4aN
zIo-3U0$_bK2<9Xgg2@dH6p{{3iWmi+5}As;E%?%F6O)9)u&VN+yqjDswe&}Zbdq1<
z`Lz%KN)mKgT>0jz!jVpK{g3MG;yrytI7gR#*<d}B<Cr7WP5<Gei{D&qBfUwm<HRQQ
z5^F{`9%<jCv<8EHqVkbb+xqf!IG@V_ud<<W=Xy_SX`Td~V0nCIB%8*aeJ1fz?h?w@
z3)f#1zxQ!Qwq!~#d@Gg^lGM=eUUB}_CG0{vOZS-yM>BCUNxIx7p@}aS?UW0((Lgqi
z>(Y49c=h0$iG6pBRMMYetDO(b^n$okTt--rYxmBS+j=jHtak}`P=4qi9ul2PbBFit
zcAR_R_S#=Fkr!SYm*l$BG_r+v;_d`37(83*aDlnJy%C`}7Lj!W(g&k<%cbeF;^8_%
z0mp}itH7u%t{)nbOFw(+t6{!aF^~O7Kd_Ja=Gp`|dmwh|<9O=b4LB51Wbtl_+_q9W
zmEw=tmZ3+j4L33fif--IS*6(D(9<<1ilxv$z&V1bhCHKg2wiMkHSkD|6ZAFd(rxpV
zr{=+UY;G-fUMuLiE#Yx)I{Q4rjgUb$C<FcdCr+m<(tW{o0X^=aqM4<Gm2h;J&f{Hy
z%C}=)a(5d~+!1R3GjY=b9pvq$QIDbD_UXXX12C&I+|KIme`j^MR(a?ZT5{+3VI44j
zkp5vE$`C33(H3eu3k!7`TIrjvyVK_OstisYBK<<8tMn39WLd%gaM`kh6T?~fMc-&+
zbzX|+?(BEw0O$<N_Lbgh2enmICpPVlZUStdCYJ}k6%;_=P^uPP^$2Ei=Vric(WH_?
zt@utgjfgU}`HgjJoDlw!uA8qNEZO{&t&mmC-{Pms{)T9H{qEGas^q|4PG`#=F~ere
z9c-|p&=GXlv+gjx#>3g-Tkm)S!zlMQ>Kq7St2>aO(5bN@V2lj4@VbxdISMvqg5q8n
zKTa2!?YlyHF@yBCwU3QwmJm#nBH>=kw#&RrGo2&Geb2&aBnF+FSI=o+BLzm8@vRq9
z&eo7g;r9~8GB6Xgn~&BapR3MkFHhS=b|m{3<~Thl7~Kl0&LCVkKipkP2us;dm40a&
z2At@;&a%^<*oV-kDicM*6(5SJb{F1wRF1=$a(8E8E2Hq%5aK;No|PYALih>^bR4?P
z_F+Jbk`MJ?b=`mcryvDvyZv(i9I!Wk_2Z<OOKluc%jque1EvEUFdgI${%x=H7{Bn-
z^Fn5!c~2cdDZ%+_)4K9sp`*6?Ro{BB#e@}Un{5${p*wqMDWsuVh@PIX&r;d<w)53D
zCEAk0_E!`@eg&LXRG&DAxQl72g~Gmo;}uhUbD8a~Utxpw25TS1><jLn8_wYxV+EsG
zd@(l_F`cRR5j{@UtLj52H*dDGZDj8)&kQP4Ua6E6w4KjGeofiD+Qqi{G8bn={kk;?
zE>hCbUa0+NlV7zYLm=;kOMQZV?F){4(|Ym?2E;j>lzgzR2y*%xp3W7t{yn*b<c<(~
z2iBb*`kB=S3w_6v4-(kg@5I$U1he{pRQ~8Maz(LGp3K#ZF-bd4$>H~I`WExTI!cUV
zCFae^CWsGS3rhyuk;zW2giH0cp=ddUQ&sk&edNP>B}6M`G~r-}s>|N53j7)tzGXCw
zI{XrmRK|f_l@uJ`DtXV#8JHn+F-dZr$tycf*NPCQG@%&#8UELYtx?}y_?{?g9yl2H
zHbyGCh8~{nm>X2<d-9xJzx`PvLc)vP5E{Ar*)i0sKKK35xL%dM0_$o$TcXkBSFqhy
zFk(|S(oi^~v40!m@GLMvgq6##%dQdbwFZ}=VEav|gN;%DcJ1;!dp^Jo0-ax}md}WA
z*E_Mu>Q0tI_!9|O1$IXD*7YVn!xEC=@5E-OCTMSc^wbXWWn;R$dHH4Nk{KVlHgQlu
z#~wijk3Q_%?}Q3Bi+KMNoo$IzzauMEnbdo=<k2f*ar!&?_4X@%rg`sergW1&t71K-
z@mWnfn-ceQ){n(KeU2uy+A=rrdl8;4rz25Ys*f*@k<aFms9-5hlBnOQk7EZvKpTn=
zV~aKId$*=*{eWiFs03oA*Tm`%&CX`b9~QyfDeKrKPrX?YZWemRLxY<n$KXoy1%}R<
zp@laMvwd;p!9<kI+uJNKb|xjZ+C0Rus@$7ybLG@2p7>LQq->hP<=Ihu8?o02*r&xA
z^0MAPJNyprg)}qAP!7oFfLM}o&vDOf-Mvm83$HaIgG(tBqDb6Z8%5UZAB}G@uVAiC
zSM2@rTL-+Esg$_wi&V|}P3})-OMWe|N$7yXtVe0Lf}P~MKsU<gweV2L!pnKMv+~Ig
z$ln7#m-nWd@zqpPE}kgL{!TMoiL26_59c%#veJ0mEM!W?nf&Wv6Y#dtx8Mk`O!!v(
z7f%cMm3e>4iuz641Js2Vf0eQWuWR-y>*XuuLvh!N$a>ntR&8KO{KSof9I<2rUI8V+
zbR1g&DRB0e2dD^Swo=VoMdTfDMkh;JGY6>&e0aAd>h_opKQ-H8-{dfU<UF%s1jP9w
zieHy4o&{4TIJ*p2R41yGV+nAjs(`ajkaQZUh?Pd>P?NXSvKaTYd&uC3_fqk;L@#wK
z1l_55q+Aa-p1IoJP@@Ii5CmQ`6Tp&w4Q828EJ}iu>BF}I;HOAQ?aox4!R1`V4}1*G
zzj-YUj)l<HL1tnAXu8e!__*8h)f4ABx}Rvsqe_gwN$KznAI~!*Pavf^Z@*9e)4$@R
ze-lrF#E1V|;>j8kAe(voHzn!qVK1^C%jl6xj#%>H2mhDzY!p-y25|6Njw)sQzcQvG
z)g!@3&57oR=X}U2|L;Q+B<2rj$9m9>hHS0>J~)txoDK8o`u^8JIXLEh`xtpv&hd!x
zZ&uy^9G4e82cnMPY=3^_x%Ph_-t9=GNYRh4I{*FDe+&I@P5-Bd{M$YLy%BE*kbg&y
z|MPMBvs5BLX2=09PBs9vyaK7mky-N>z6J-LcOh9S=~4l#O3A`ihV8-DpUND!wI>_h
zYCBDG6_a8CIt#u}%?*@3nn2>`brwSd{wnW`>O;;V)NkIjUjaanO1WYCyVQ&0O<7-b
z+!{3mst6x|Fl2zt{wgP16M#cxv=I3_mg@~*VO2sT8bDU`wq10<PVl}4!oC3ggqD*4
z8aK!U1WX?*<<)Ax@EZ?Y08&L406gmG1BuuohbyrCdQ3n<^do@d6JEXypz)+#L0wqz
zLt~+R%K>zc$e)^vD4bJZF!gi8*JQYZUw@FmZ${v-JsG3d<o*I6y>bCBUehcc&1Y{f
zdO@??a0n14X(5oGKLzm0Fo8aC+;dFhVa;3&Mt>t6%EAJ6F~*h<y@mvU*X05RU)S)>
za-4o1;4I0_-NnGouG@bE<}38u-;Go13-Q~tX7aJKLh>Y}I8ZO2E&KL#9v{mt!{zR-
z(!Pb>zOKYdz@nK0KiN$R%!m})_&;`$NKH)yD9CF7!(sD6Tp>A5Acud1MpUmzD!>v9
zu8-ud10qGVgoi7;MhT@_sa^_5%xW08)O>@GKU^JPK{R??3T7*%Bw~?rD}<5@L~=mT
zNV@?!(`>9nH=6MUK>>ghJ!k;Dp+)`_F{iLdCV3ga=E?@Z0u_*UUR;LHVMA4<D%h$-
zuQ3rxUAW%gP}Hh;XM@!mwF5ny|0mac^2N)Scp%Aq_}O``QcC2ZG&R(dQv<y>fse%l
zpy>`Y4|mtcsylsMZZQ&<i*-nzN}3l!B!*jeQpMV3J2%ZU1Jq+Le6;w@h}bR+zH$l}
zg^Y+as?xU|%QTUPV3FoR2XgYK7r&_;1{?&&D8T!D1t_kpfPSG`swXa&3UPTJp5w4N
z+FkJ*vj4&S%Cfw{^_X2TO(G?XQiOi)NmIw^nTH<;%l8z#w;95h1Hg74kRx0P`4oE6
z(~`?r)6!z{1Ge#Vn7ZYSYzPTgo^fB|g+qCPT3MT47e;|*dB)C|fW(j_E&)CB_}*kF
z;6$J!845-aB&dTR)T*y=$_n6u-U-t=`n0|N`owHFYo9FKbm?A@2$to-xsR!;+XHwa
zjYOJ|Rcfd3nc+_#G^sBv09fz@uq5saLgt<)@MeFJD*8ZN1$Z>Hbg_^Qw!6>D_Av}H
z0`TekGof^C_E11_I9==9Hv|m4Y>@F{dfzs$Q-?cM_`{fVHYvYJe=;6?p<?9f&}tUu
zlI=yu?Zef8$IzyT7x)8+x#mbDz^?brRP7yEV?0_u`e(_b47BQ8!xXmD^MK>x{Ib%x
z&j&zmNEy{_hIefkco-Mb1uiD^0b<J+5TOYen*_xK5CEo1d<5?1w{%=u71CbRA*sp|
z=ZtxP;`#*1==DsYSa8~%Jthk<A}3&yV{8o+f8>Mgo2uU=?}+0lWyUuwJjNl>W|E8k
zdK1F4?-~!7C2M2Vm*}|EM`Y;wThsNG50Yq~_<RE;k<Xca8BQ)xK+SSUEKcyVW8Z5#
z<xQT_07Yk<4i>3gJ?%t8UHbIA%nMeXuN={e^rScu_XCJNC*Nn2lapi2rhIyxos~t|
zf}P#<IJb&Q?!_i_E<0hasM@%3chWpf@A6t<d$!uR4`RMjIV-;FKK2V@^f{o%Jni#@
z7=vD&wSk3<R5F!rGZ^y*l^8I?Yieq;ei3{HNWHMWH97i0_jEQ1K%BZi`YfU+bN89N
zd<fhPV2sAUED2K^xAjsb;g>vpqvI{fNBC`98R{{K*de5@x?bPau1uwaBp!FW9Sf>>
z8#caI?^L4_;+x*l4TS?TYTG~e9$2T;(J$B>t-XrtP`C{0izYScRKN@dt-o8cJ5kxG
zdx-msC|NTTg*`;{3Q$1ycsc<G&`yw%@##-e>WWH$-qk#<eUf6=zNy^Oq41<4&L>UG
zX~%xOZ}H%5Vn&yc)oh#b#cK<QVRoeZ<p~$)EtQoVde6X{U2!@>iwP|WtUKVbm6<~+
zCnqP#`7#95#uvI4IcfEx<1L=o7XqVsp&o(+dB;9lXS>5mY$o01cMB3M<K9)znOEX#
zh^y!XNRHrt+s@q{i)+f2X)wD?Dha|=CtJX88`g;ca2LmvTEGfH=iqs4DCS4Eun({q
z<4TQyp=*5w{+&#vSkAl2K_F(l{e0&;JwN%|U51~8SmYuv$s6g$JSssk$!##uni)#W
zbgQ45_{nlL5Maj!%pyq9I3WD(Jw?RbOoSR<7m->~k@E1d+S*+0Wvta_rUj`~)q{EJ
zjq~?$Jy^8gyn*k|H8)tOJ8#W2jJ4aK+`+hFlAKbWDg{ZtF~hZQoT{#?*N%$KF~XaU
zAsl~<0e7OL(u#!S-pBa%Np8jG&Sx!I&tYAAfXhT6wxNSST#GrB)>_Sd&zCmFciBkv
zvIV>|4HCFbdvi49WlN?hSD@&g`aGw06{|^W_!{>@x|`|uSguY01-qL1Zo~ok;Lz5m
zbkMJq7j<2VfvYxIGB<cs=Sl~@)AuEGOws+~TI05X*2i}fKcjqver#^q45fvr;!JwJ
zTD!cnFm#$`GebVvpD_B4=(B5!5g1iU;BFU3>-2oxPc5c~JOR&(eFpF~QbM)wXEQMu
zVZ$3)-%GK)_v-}2sTAyDlYHXuPn}KQby!G{rHR-v&jPGo(JPYc^G!Ta28dFM=<-G-
zhyipQ$UzR+(Eh$8A*15;2+azGt*WLltZ>(X$LOCiF!)98OnOdwUf(~e+WWR`wd0Sz
zFf=^?^}IneZkCasmVD;lzzheZAu%!XzQK&u!L+teVr;EXwxD!Qie^vGiA{1@*c6|C
zi^iu;OnujDFnvTa`i2w~fIqkrj4_%az1!Ep5@HMx0AyO;G9Pjf<d58Y#BEP#V~Qq&
z9}Wrv=1b$kK9Re$FD1QepuuLBH=ZJ{a?XP%q*r``TKp}U-PF!C-$BS@OQr7O&W?5L
z7cf`Owej2--bYQ9KSc#`9*kj1s#(*S4XsH3viFjDQw!ioo@f!~9~QJoahvra2uhS}
zM_82rm^8#`dy?4vQ{x0-o9^zHy}`AJ_l$Av6Z6otSsggtX$PRxI=m-#6QLSBg7iu+
zzIIs8+hf%xjDG(Uln<}7>iU(!)Xgb0oy8CRBeXb-2`ro$6OUA8V=MyHMm&R`iZ3j<
zF=$sBD;7DB9V7V3=g~vG$14D2u-MXb!s}ou$ckQs8`RBj^IbK0>+ZK|3+dGiC)A1Z
z*B~;`(K7iJzc6es<J?76+88(C&Eu3NgHO$8(r>?<%x9uB$W%dzo5zh2K5fK^imEFE
zYD;dqa12#$iYCX1s@wh@F3Tk1QLpirXU_a^m4V+bfHPXcM|MDvy>#pcaCu_M*j%)w
z1i$+0`~4@dVZlL+$`2RPYk6uiEggkQT}lg}5{RRM4D9poa?YlS5yjy$I2Gw2Y2!r7
z^hd1xp_~j4O+4y}+)k0Z-vj=3XssFX#V1p9;y2AV1!{hs%@<4G%ch%5vNJr`uNrxt
z>J=|3b~(DUC?=}r+q=R`Yo{wR9q8XX3ggb?freV0j&%aVARQlf|7mhoN_p?yKkbuo
z6w_N0)T1&Qn7G?!hPzJti_gqwJubusOk@)+==GC{Fqx9n)2KW)v&9uH9#$*mveH6=
zyYO&7mHf}Kfas&w7((BmSn!UY>_-Vjn?E}x#Yzi+o@{GrsSiL!lP(ZONEJ)_+EAu$
zr!8PZJ#i#|l`lZ*wLYBP-vMG;!$!7lf1v(q1D@Z|<A+#Hdz#fB)GRYa8_iCg7)q<n
z>$Z9cW40fS*)?v)b$TU{O?tt>{8~Qhp*aP%6!$9|j5d`tP-Rk^E1uX@Z<owXx<6**
z1c1}^9l}MP($_VV7n}?bi~6ePFw>bpef1!o29d0th#AV)W&<f=aTXhSn*~;*%N172
zolUF{{%|Cv+F?k}Ls6@)Z!U3V<^yY>DDBciCjs?%Z^BSoGAVN<Qsl&BrqCKU_P8|5
zXk+6TF~dMVZs+3S%2zUb?u_wO=l6wP;nJvDin)t&z1yj!JHQXMF~o>=KV#q^U_J@0
z?coKW#4Nso3RCfb=?*!{^~4;f&ptemXiG5D?8k6u7&h9(p_H9F6wmT=OpI9;0wSF9
zPQFsjXomaQ9$kLiR(Gbxs6g349{{x5BtcwkUz<2IsfB?kk%h`U<|iUW+)aJL_NU;1
zTyDxJ2scv7Vqd<JoiXuT>Wa|ue8xIEE9+h@Aw#ik#%SIh$+VNGs3PM$9!eP_`=Dy`
z3e2N$5c0DOh^pe%!>UgQll8M>Og%u?pr<eLt}qLhPH&58!tIUavOB5L2v|JQ;1A_c
zWnQEmEV^rYX%2M(w*?J&k{b5nOFBoxf3K>XtK?hc4%{4(`V(W~J{fL$ut9n0BGq}Y
zrAqWWKRT)Pr3BOx#8FQOwH6ju_ro1<ulHgx9pO^YW-o}t#_!~ZA3IEQ878XKf2_d(
z#x!WJ+~z_xSA?QyZ}5n3xxA(#EzweqXmV;w#WJwcCE&rsNDO`oeHrKH{Noi<?ub=T
zfp+b$l8>2<M=k{I6%7aVR8qj_q*TH~WRpg3I0<Y-q@*Rkbz=^mj3H15Pgc*tGXYN}
z6|^{{P$KOBEW_9|ayPonJ{^HDJW2qord3nk86(oOtE#_s7h^-yM|w8px9Okz+E>L9
zSj{Tz6dTUw&G;WhbA^`E!Gs!W-_>YQ!^y6nXGB?%E39}v6xavInRYe&ALTRg(!ag(
z1{3@0K~+u7cQcj5l`OXB<n<VeT$L?G=LgICNUB~EPJy1`@6plNFM@tOZ{eyW4SYq*
z>6af;f@CPUb?WAOw7@`CG79@b_PAxa(h(3!QxyU#PfwssO4nvCJw|DTf?BHMRc0y&
z9uGxqv(o-hnQR*DnnUraafs3J^5ng1n#u4#dc|Xd=l$|y%p(<oLnB3R$)`U!gTsDB
z=Xcpd7)I7ZI>M+Z64rTQ*5ZZ!M63_W7$dyiiB3kWTgb?`+jqq#C=n$r?5qJ4V8pt&
ztc<<1@f}x1ijHclcb?Cq_>JQZyS8HRyZb>;WvJw4WD7}<P#=*QddBb|1N1tU_q_Cs
z<N_a5--*sftP7LAkEjd(rCRP+h;J2<utD@NB0|nehB0yzV8|8PNmF3%=e)d`3=#PL
zLjd|t567x!HDlL1CROGmCro%1mc?nz^^9FnR{0mM-O<(rU#B#rt(Jt4ft=yblk;Dq
zab1CxU1z_QXS4?t%)T_hRUs^R-jZV=7A@_Iu7M5&fTv&l#a%q=<A)K8GyrQ(eWOXt
z%6*p>L_+poQ9gR1e~O?)ZInb=7`i`L|NfL<zBAYD+>i<td<+8tnB3cPpC-_LW%k~C
zemGx*iw7>_LZl18yCuK*f*VB)<I57>8X77_xSn@ES3?{MRAQ!=bQ8}T<t}*y_;>~h
z;-0&o_`$fEh;HKH?d|Cfwi)mr4#-99e~acs?ELd50}J_L+LylI)!4#1K9at8lFs~H
zzTe1nW#>VCMod66D}+i)I51<8g_iu5i!pd|N@TY4&!-LWz5Yx@5Q4&jPe5KoLB(Tz
sfwJ(5fT;UF|G&NT?@RyxTLTDO!^EaI=|4GvDB#bFXG+o~Pv83d50(S0k^lez

diff --git a/docs/figures/arch_8.png b/docs/figures/arch_8.png
deleted file mode 100644
index 16a0bfdd39e1e5df7cc1b10a5aaf72d714cddc19..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 33393
zcmeFYg;QM7vp+Ztg9jPhg1bAxZE$yY2^JtoaCZ;x?hq_MLU0YP!8H&_&_Hm{Uh;jv
z_g?K*ZS5bhQ#Ez(+}p>x`}Fy=OthMcEIKMNDhLEZmzRTSfIx5|AP^WA84-9wz{;!u
z0-=7ilaf-Cmy)7Tb9J_{bF>D5<f79ukhD^kv4c+jRz!w@P>Q7Xl;5j>7>gpQI%5>%
zz+u>u6qQR!gLr!Cuu{EX1SrBHc-WxPOahnW3PB$?^_N9&#OQsn)ROCl?=R&^k?XMs
zx7R;6+<u9j!;6={g%$JYVFbCGe|j}Hlm~r@GuDF+&r%4Y;(-53>XaUZM@kC9%F6r8
zME)Dj*1qcv-|k_-)Zd!n^Ba95P@!2Ej+g$Ldsrgn4_;lGNkou#<eo={b!_<!S04{&
zGDTlv#Vfmrq4HPe*)w}~Ij3rvVQFwZDX&Q1VuNTzU{k|ebd&*d`N)RI-eAhU;lS_7
z`T4z4gwbemSR~Ls&8IAsz_XMKf&vtc0#cRa)Yn&BtX`@T=?w8r+kb@<{<7}dB`*^q
z4pY#^`b#BcCFCwY&b<g^M8m;HocI{T|3++L1bZ_||EG=jz$91Vk9~IIoKGBP={N`*
zj#UMY?`~Mmn>zYz^mCz1@o5ZKW)obp!UVd>#FX>&=gS&@zl?0DH|0`C8NC|QuQGpf
z<0I4P)Wc?)C}6*MTM&E5y6)?<P;kX;rnqQBA9=5EJ()FDz-~2hJ06q%^Xm^^pNVCd
z+_CiX?^(U#X0s|0u?gAgsZo@ax(OM!owXISeWG%ghmhAqUSM{tpa((<upXSR8A>Jv
zB*^jUtV#$~m<z@{5K_A1h=^9jB^t`QlRagrr?0mR;?Qp5*4;MVb7z&j_~zF3qBJ6t
z{Jr3^V1*(Bd|@pJv>WLr*trf;JA<=5-ccA3;BKpO1rzQli^K60f_Z}3I3zbvb$T#f
z!ST~rieW+eBwpx&TPcW2;W_mXb%HMoLAb%JW<r%<UvbtYOpIXaLiAk7uO3<#^ficQ
zFZDELL=Q6uqG%8ba)=8SQL!Y`d;C(&EJ+<K9M|BELJ$LPOz?IQjU2Lkq4qRlB{EZZ
zb)ml+^#z(cSP;oL*ry0-8pH$c4Dae;$b@_|Ltg^rV{)70A7MuJ$8OQQ!kG*|-<ohi
zFoGu!Gu(1J<-K8B!|V<jQU}q77MC=u+c40L!sCU-NYfS%RPt6zY6(;$&c_{ydlWg9
z+^LsTPT8_#$Ji83Osm+Mx<j&KCt>vx1SK`o_cOJ7XnU)Ba2JGkLJ@K5Lzxz&oQ_I7
zbSO<PG=mt2<}4E|*(`orL~xKd=&`(Xh-w+Sv#@MnTuNKAIAU`|yJGo7AQbg-Xv>A^
zm1ZtC8}d-E^$%2+wARp8?pDKA$~F3q7ba-V;n!b>w!?3P{REysPdrcfPY3}a6tdIw
zudqXqh5Jw?QHm0#C0Zp2sg$sGqa$FiEnabkx900fIMSKWh|+bXBqhxxy?u|81Reg6
zQuSVd#ZBKv{3o}3u$AN)#b6Rm=|m~wto|(gtPoe0qxd`Oq2#8KoRRpQ^PP$v)sI^G
zZu*~CRNvJ)wm8-tu`Vy-f~5=kidl+wG)5U^8TlDXTp<GR0+<2`Ys6pdzu>s~yZ$)S
z@>aa1xOKm^xP?E{-HF6=K}AG0K@|@73pWTqM-A1&)y_DO*wx@yJFFzAlHy^Rc>lG$
zY&ym^(pIFIQyM{<WKeE!H)cMj0{4EOp$#riS+yi%I&u1BrklZzp)vJFAtyCE&56-p
zOGopa=1*n<rq2wU+E2xOYJW83wU?`B%g<nVwf4ot#cfI+pAV$x)p@hCQ;2nI)sCZ9
zq&)Q2VIo@oWj?h&pYK{D$|DjZ<VrMtbbPI(GybN`E#0Ja-uyEPUwceiu~56*L*|fH
zDyi3^l#_{&fY5T~V5MsX``CX)WCcesLJ%tGEyyaU>K@&uv<_S6T31=;6k+WY@6`5D
z_QAYFxl?+ydVG0jd`FJ{0=@VJA8|9$3lb)hkLYgbIWNdblH!fxx#I2PeiK^8XML^u
z+8*be-9z_@ZiH^0j$g@KQ%Ey-pY7X*ccwSSdDyxA&&#Z&aXlLW8)X~#pYG?NbK!Ho
z^TqSapF@+Ig53^Jo@j=#Jh6hYvgxSl0qNf9FVmfCl|HoU%<Fi57+Kg@aHw6a-CW9C
zIeqhPb?tN63gU`W^KtW|&1+}+PY$0*8W)=qKX2M?4wZ6-W(|&yVGtsDMz&zkzZ4Z(
z5wH;=d;8EoGR`=^AN1`M+>h{p;DLsi=aDNgd7pc4Ovv_!&DZ$OxW!nYj<;@V&3)a=
z>C%Z^@9PKL1&IX*Pc+ZQ`?uGLWC>(7B9lJF!v4Z&BEB8C9aJ6F9r-?oUi9Al-ifzM
zcONbU4rBjZUaudl4lS6uEyo{miurcW2M|2SKbb#iKTUy^zyokEgIx+uzkli(Lc1hD
zCHRC8O!02|Qz3nCL~mAacZ8W#h*Y&yexyM}9qKV^Ndzmp5Y`ptnn(xDgL;^H7)>#E
zGBy#xGT|-fD0ekawU|%9#lfBfXEkR?=9Qk2qm`ro(G69-^eO@R;PO|zfptU2R<GM_
zW<{-xQdWVM)Q_o67q-H5qD)WX54tK?6%pL|Jt|9C9A7WL|Nh>!{pHW77q#E(0LH)4
z#cmdM+b+p&v!6<ntQL39cGQC-3l&c32KlX3M~CJL=OX4tnk$>r-Ti$0?of(CIJCVe
zrMNA<W$K5uv?vay+{f4Q1?a8_7GF!~$2P4r*~z)hv2rFqP!J8j9jThUZDw&@p4mMc
z3m<FO&HS*`ye(bl`Fo0ahWQJ#kgi?5LASjbuft^*?{sz;Z=Qq4;?Me*2EUq|{`zd+
z&0Q{@vz#fMZ#YM-OefVRb+ZMYA{CHlqgtU2Ao!Ey@%_GqK9+nR%2!^<VdD2^-@!9s
z{m8(?erGVw+I5$*tBNP4?4+``rfH%3(HHGqtjkzuZJptD#%RXRbPOl7wFQH(49qOP
z-TwPm%)5NMzVEO+<=$QBLbRH7544@C9t_fqJU*{<wpmx)O)3kC{oX!qYgv8cc}^BD
zBJJCCgm={0J@wkuAznh+oAnc4#g(_QmGR=#cmc_w_F?QnH!m{D-F4*e?rGc2B`@(u
zae16I8arYELeJBmlv7g26E+j~279d1k-g?4TN&L0e_c<a{w7V9jF(IdT_he0tB9B^
zp|;f7zI0G@I&@C7I}zFPEN<$r`f*eLvzsq>PE_At;g8mc-S+;&x0KheQ*i}&c`rpW
z+?^WS?0%J;&-^+uLjE%CHuHCTS8*d_(zCH`(&$~6-&xF$G#uS5-PO8fV~68U$D#r2
zS9gEN+VdAkuJ~7+1=o)DjHi9?(hqChK8rY;w)J<p9sllY@4Tk^y?zpVc-TGFF7`li
z+?x|AfJskQ>l^27bOZY>f2(;}!P`yrcVX6fG0?upt?*udsYy0;tyj1E_2ZKV!(V}0
z?_0E5g7|)&etOvsS*I?e3)j80z4-y#-tL{hE$1yWhn9h@k250pB77Gz4>o6S{s^Uc
zqMy9k@Jv8T6nlLCWWH^=MW3d6FUBqA=^uS}W?`|cyS5nP{B={X>pn10I7?K3aXU)j
zTP+Bc$i;#R<V*sh7y^lRqJZAcD5;9DPLF%`@C5%I;P$_6UV>9wMH0VJ0D<e^)%3%g
zo1+31NI|{0uoH-xgTsF2abSh-C1a8ha?ghPn0R{-($wGEWzKIye;H4@r_4rc&~NS^
zob!uN*hDACCIJu|W34A|qpS>K0`8GP5IABG0&oWhd`00%{`+1AjsXP!?>QI*inIek
z{)<KhxITa01K;O1|GB~^M}UxkU)aFcs}THusNq5i;s4J)7#Da4lF*crmj|wzmaf*;
zPHwi&?q899b^;GjT;%lKKp=eD=P#VR2F)qZ|AL*Cp1Yp1lAxvY8#Z$*XA5gKuQx8w
z{eXnM1cBQ(*6!vMUT++o+yuQusQyt10{734*{LZ0k+?gEQ0XbFQAjzvT2t_`aj<bv
ziK0?aPzbwP*$8UDWd5r;@Jocs*4^Djke%Jr)054Uo6XtvH9Mz(fB-uO7dsahD?q{O
z=I!Ke?#1flM*W{o{!c$JYd1?*I~R95XD5p1e$6eMJ={g8sGbM<@6UhcY3*hAKO;H0
z{nxU91+qWCVdrGyVE^yFfu_RGPX*QNysRDdVRml-o&jTs@^W(t|D*qZy!oFI|5r=B
z|7pqnitm57{9kYW?=7|6tX-v?-vC3pi~f(e{;To-e)(Swh1s81{=Yc!A2$Dc3h-GJ
zRha$1WhRQ6|JY~=h>_F|rm6*80VR9>z&-n24F9=4-%EAp-O9{@K;j^In1q%W+>tRt
zht?p0Xgm8JBFaAgdn}kZE=QD@90MgK14U73&@5|IX;?^96e=<ULs2P39(M9<A=Ub&
z#Yte7_nODyU{`0m@5ws%1DUUb&7_~(i68f`+^j|hDl8<KAgCFHF!;T6Pah2HqsJ7&
zYX~8nj5t*ftcM*;-Gj^lrc40gNJMpmjQsx5W+LCb1X{q!Pz*rLsF42<1jSAPRB<32
zxNi735YQy-6<7&E`2=?PLj?ESgzCHJe+I#l4T`21Fa@!R{_9u?@__N&kQ!?nnFAVj
zgVr<nZ}6b#pv`{`A(EtUbRk%QaIH80I_7|$3O&y#PCSab2ZpuH4k9Rco;@5LH~@(d
zPK?6kVo%*?JDR>j;fu>wpZmkj+xqRy<1L59x|zv5q5b;qr{b&Q)tp+dHdaiWV5gc0
z1a2-cy=s0bfr(ddE%L9)yp%<aawYjhk*myak4)Ot(+{`jd3E6{BWqvWs<dl#tXd2h
z`S|$eV#u4%xS$YppUGFaPzH@s<h)mJLwqg|q<?&k=ca*NpKMkP#u8{|^Vn6I_eW%>
z2S<x*zDmg@#F0oY+D0OR+k~E`BRg$&DU<L!l{USenk`ezV33M<p|N}Ma6YAZFjw^h
z&-VQ1#Fa0jMrmP1L4ZNNX{S$`&Idipd3M4U9*2dv4Zm~c`=$h>)^5uo3_-6`iuKmF
zp-$^<wA#A&SH~rll?*;-BlP<{!SD&{M?-|S4|j+4?oSWiO!^;7NncuzXU+s{{`o$d
z#thLW?mL*Tu^Whjy24IZox24@WoQ*r3&*p#g^bU4ckV6^-S?&`6$(T}(%6i%9r;Ke
z{|u2gTaV$ld7qb$p9KEBCzpo7Zj$3Uw=HTEFrEcU2nl-$@(p9QfW5bk@V~#(hEdZL
z$dV3q1AJs=)~&n9jIf=~%~wuiO7nAC{%9Rzdy)H_QhAYr*ttw8i_2-M>b62TM`t9B
zIYXmF27UMUdVAeW+SB8G`^Zat{PHLnZD<L~=U)UYdRWdT_W3t?0&eAoMy6~JXJhPK
zrV_TJNz}5q=HK3?lo)++jiEp={m2P3y8hgQWzqCLE{GF)8i8yxkzE!R8OhxDuF!pd
zR(gM-Hpj7$6+hOoVPH=jD&m|d-t4q)&|7eKRA=0t3QTImivrpb?Emm*C~vz!&_kh@
zcWo+%zn<&uHf*<PN>mfzAw7v_5r#(C%YMd$A0rq0-;V9X&d0emY&Z|z?}}%$8Z^83
z8Id`!ep0qu{#f=PrLVm_=WtL@nAJLXxf`MY=_4x&TBD+-F7pFaLA_eL#wG*)mkp*8
z#HQYyt)LDIm^GQG-(Dj>(;V*=+4WMRB@PR@&$g{zxPgpBD8Jn-4FYuprs$2h$;nDf
zjiz5wdeWi<MIFug2z|j4$<Vh7RiVNDXtk=PU1DLchH+Vcxk~RNtJB>K)9Q@%fbPdD
z!z?7LOHZ|Eaaxh0vyZPCwM-R)!3d=+>|Hm-k8S#xU=;{qNL5_HoT;C15m}g%Zf|VO
z<>@O<qm%Xaii^X=si)w@YHj9520t-4NC#yIBw0Od(lkI;vb$i~GDZP%vw(<BY!#?O
z3trr)TcIZ4L%wf1fYXvWK(+)_{=oX)8Q6klN@WUZ8Wt87M~m~JpErCIH-_YByRcZO
zmYQBC=Exndw4|Kjvzx3P=P$ECa*@M^!bn{$miBKdl0#f-+}aXen*=_3G;RI%@CM%o
zSN5Tl6nE;DE&%&IM&&HPpAx)R{pa!F_E$lSi*j1L*0*wH99$S}$kB2W^WJW?#LWS7
z3cadMz~4vD%j5?<ME$I~Q$*Lc1RQgYPdI2Y7<EM!gE3rejwJaZcNl2>A1YSQvhS1%
zWg!eX4tw7!5?NFmEe4I8vwjPIhhdS2boo*IDI?lD+!OgBZSv77`lmk989ZQ|LS|dv
zTHZo~Kp(Bg%6Ep7r1H6;H##jF9l!Rp``@Gb^SYli5eYrfr2*znPGN7ZDrFq$C6_Ms
zKL4hlWrl<omjEqNvIOZc;wngnsV)#OO=;KWpvaXmP2?Tl-<8B?Fo;wnMjBwGys8<+
zjL(0<2P5~TDeHInec^hu!h_Jiih@HYHy=&UjwqYN|7NAMGH?&CN}{!a(e3BB`iB3t
zaoM}$l~I5z^M+2{dub?vudBFvT?oIjTtt{T_C<o&!)`)c;8X1|$wiHBw<5`LxBBom
zOGbCcGLAo5=p{DKXlm^01{U$nayf2x8CQee>kcKn#N%AmBqmD|!wGhl^F7b+Z$i%e
z!LFJwj7t0n>~QA2s-NRonkK4D(86G4g`h3`2+bY+&rZe&K5iXN4bnSf8LB3czt>tQ
zBUj|ocp1C%iA{!wVDD26W}k?p5|Ec_CzsQv$=|_qF3Z2AQP}hN6AUo*3JNfB+EDxQ
z&q?QXKK9F($Ekmq>pQ3pr5vcl>GqHZ7Bi(hF0MBNg+@Yf2n)3r=lI^NFj!zngaleF
zG#C=M_Q1YF@w-iVXrK*-#<?|l-D?NJogPY%e6>GM1{%@gydR1^e~+9Vs8R(fAi>m{
zY?OD8KX0Z7Qp+Z2Equ@~0ZbnKhNM(@xR)9;9X|(@hCpC*n<)8tx}OzPRUGL;b_#!z
z(O1Dh$+9I(Hkix}A_9mP<$#rBIRW<cz@PNhMtc^OYJ22rg>hr?jk<TB&jZG67e~vQ
zU&oSZV~5m3$9|Bh7O_ch+~A;{di%-L#5z*nV9HBj_XcjalQ$aX3DFrCnNv-JrKRPW
zpmD^!qZ<^0D>Y`RgQYAjjVz{Jejy>=ijaoJB1+l21_Z#`ORV&LmwX`vrzKh^W2Fu9
z;*O@Jurg{jilJqiGjyd1Rv%Pk@7%F5Z4jfuNO&n^!%jjsUBxF>na}6A^u3(QE$;A`
z6Q?&YDk}t7=I?IQ%S9+oFyAmBiKoYFQ{R#p8KN<AvuH$Z%=&V8AuzGtP>@WMT9M?L
zD(#EL*{a9vA5HlQLvciwpH(aWybPRK>Tg-n>>1#kw*$XD`!!YHQ}?}9OU8n)7O*Ev
zBcOaaJSHu*?@uuo<w{xGjH_10mQ**Gd{WqO?emYi!NGzSRMUud9wIq6e#bA}f@gGC
zmTa7HXRv^P+8S{YS>JFE4wM32yYt6VP|Z++_kd?y8J+0e9`$|^gq_ywTQ95Do1G-~
zopaoDHsb?2{`{dkX|bo9k_rZu#qVW^LI~THTt5g*6uE82U3?ZYQd%4qSH4~E43@Sf
zT7={-hf+odtsj8@8UToa%xxzM@o%%<r1^TAhbtG6&wT<M`lp58{R#W+WR}Y)act8Z
z{b{^Kpf5#}y~%_qL|H!+;jd5!jJpFm-OW8%ef|F2v|S?;{?qoq4h>#D@tMfNMHY1q
zne0Jn3Y^~P=kD8R2cxC&#Jn*l&@Lwq$O~D>k<JjztigGlW4m|=eICc>o@1^}g*bEz
z{|Z^#m^<LXt3Nd!L75P)9||z+cC&W!8*@K3ShX_qgjk1Ko5Uv$V?ajGbwOA-Y5<Y{
z%9msBD?j77Uu9-DEP@*Rt|s=d_Vydl$YBch9R<(Lr2ix9!HNbGV`aA<2gaCs%}Id(
z6uDD_=y}p?P9NjN)(=jJ%mgfUik_;_;R5CzZtx7LA~tk!sA!*l6$IisA~yM6JM97h
z#HqR%E0~fdriz5h?6crw>-^S4y;fDSG5U8oaNW?U&DJv<{3_Gakh6%^k8d98xFssv
z*PV%q&mz}%qSvhXR-(0TB)+R<?Z`lR^&30iK0c1=4&sl#+JpfWwS2OaMc9dglMI`C
zgR7<URvGl9Dc=dX;5ByBD|r?XiZNE5a(AZoidTN||1k0m76%J-K$jxW2x%*d@th{z
znyyht3_xy^mVDM@>1B`i*C{SFB>?dA^^0KVdPcT|{E$rKWHoAwX<0)3LVC8f1Gx`h
zBrPJ!MCg2eTBJA=zj7(;paY(=ymaiILi&et_lN+{OS6E1EdIfbsvtsv-t+fbDx)SH
zcFxdyj_Rb-FX`7|hIypuW*0rd|IjZr1z@72S^e+!D`>9lX}7-Y;48MyCoLUiuY7h^
zwb(cNeDC)k%1!{*zmLlf&J$;elg1<yF8rY1l)S_@Ew7y#M<amx50y(u0U+IW+U@Iq
zCIb(EYyUZT#om9odm0OX{KOZzoc~Dg#Dk*6$uD~}{)6IqfVGAoio70wCJhn=NT+2*
z8gT#h13?ECRVXn<#Ql#gm;mX8IwlLk|ND@HIB>LdDo%>6oV%~9XaTR{f$ACkhafXt
zC!`$T(P{i9qubpm*gkosK8L4aae}o>HZa&yPNS%2P(1v>O5PaufJdvCjcqk?kt?jq
z0pCUB{yaw$1QtLv+z=T1<~wWJWi~dN)fQ8E@0-u6U;_<dm4$Kk_q@KEA8l%t2TG6%
zPA_Dw=)i%6qjp?iM40#&&k&0#Rl3#g%_MA(Wp>&Im<_c<!XhJ!DnF+N!%>FtEA<n=
z=Q$<gPUuC>M1|X(l!y@!QAhxVb<uNJH-(KrCB1^5tmF-uIJ{li{s0oeV(0M-g>ndh
zXBHWL!o*@89m*Lhu}k|57+9`XaZS8<)L}m<MtYXH{<%M46yQdJuJR=@-+N_*9oAAc
zqHw&IvgQJYz9eY{iy%N;=;Avxz+MJ?X#ZkNPuzDw-fg(c*(C_lshJWGOd1S0K*Bw`
zm2WWRzXWQuF1$=}@=&<ID<<61A@CJq+ih@%ZP|@r{~4}jQ|A0w$^^4g-g1~EEr6b)
z!nFp<oCGR&)W~+m5bQ;=+Qn$XvdByC?F!E^<{vVVJ8Henm70t<8J@xn*IUr7+P_Fu
z>XvjW4q{6zyqJ1V>*lfbnsx1fON8C%pszzQw7>?^A>Q-Ol3DtsmRXRIzat87K(?P-
z1NH^aYC?}TN-|7kK?x%N!D08xB&PEeeVJ{Xz(@Of;xHvMZfJ-lNnkj;>C1Ftmqfbz
zuCQo70a5S{m05e6K^%lQTK%ANhYp(R^{cM5So_ziM#gmoBJE$b(bQ;)q?J#`hKh|9
zqPWyS=+VMe6K@>K4{&u{f--~7llf3wnJD9fXoCIgqlkLcBl@=1q>D3ran|wIJZu-i
zqv{{3igx(38<Ze_<}#0e^2G=&gr?>Q{qXMV&R`{C((r1Omkifh?k0MBP%bV0#Vx4#
zoi^S<q3jh?XzE-NB!Emx>ywvYjGs?arnA}#1uaK@rV@nBqa0>hw)DfF@#;i>TcyFT
zM!P61@$$zp!-rwnzNK90&f)Gj@nU2K%%|eLyfi-(GOC$4o`9Zs2${0zS1vp#-scor
zYo~a(ot+>MPH?|(kh%MKuTI;aS{>=oL=Uej=aKBPa-ZlxqTm}S76$rEdjUORR6OoB
zHC)RJ4pLdIrQrlTCJN#wE+|N8P=>Q2Lj7nz=}b?`uSIdxqO1W7&)D%D>kp+~i(-Z*
z*Gq6?MTF+K{GWV61{xW#y?P%SIu2iR?*Zdq`!Hbm)6yu>X}u4hZenHL-|-VUEC}MN
z-}!QG(X|9}Z_lAe2T`qMmAn&S7UY35;Fg0=O;!MNF*mwaH4`9TJ7BC(Vy(ci+%)fW
zR-GJYFlIodRwJh@k2xNQyR3R#pf7LnBP7J*982D5QM3w@(0WOO6HyTe4u@aAJ0I5T
zSHRGx3Q~XNGo`p)vRpx#`=qJ>y*ix4iB5OMtO!NI)VLaoy%p1(?8QRv>81SX;wv?b
zN|_=Bw%_Vg9LG+;eKW85bO{ZfrU5tBV9WK`Qp2S|x(wubPSJ~djSI@6&Kw3^@fiL6
z&|kG}L9rZ=rQfg>-&)BAZf&EXh|?+c#eHCUr|^R}ny0E5iv}#c1j&wWStPInn-yZe
zrG>+S3*q&}G3VB!NMJe3{fnjyv6xcEJ~#0pE<Wu_4l)=%wZLIyamKt%4fGeM*ogvd
z;GR}<XhbDK=D2!d!5g@wlDKf@gb6DedMp86GS?tmX)r@2t#&|+-&NSF?;dS_wKwmn
zz^m~Secf$sMZ!$j)Nr@pC``)YKEy`xj7;fK${nnpjXuKN0})$!u)RJf@mG^!Qa>pQ
z_b72MU@^>6qoGY*l=wYM@>IdHXL=exONK}92a^|xzzmeH%a9<&*9%EG6NeJ%H&&-e
zC=~1t3o90B8kVgALy_@hb&)Ti1El*jpLTFaObBVb0JbLexq6nK(wEm%Jdl8qd+u2e
zK9gFcl9a5@-o-+!f;sC8SGqeOv576@*f0u$&6THAb1lgc%HY;SsyHg<$H`c?+Z%O4
zMxUciQ{;@9&mPPlP36C<SM@87<JOgaeVa3HX&nxM!cM%C92W!Tsh^y<XmcQBWmXdp
zAuf!@ztY)EIq1PN_r&jy&oqqj*|2DWLJXVcE22J%?PJ52AY1!T^#p)td`<XZ{x=F>
zdrIoj`mM(oS{zD`HEMGajuxREHEqn+g>oQ-q0x?{dG;T2p05a{DZb$ZLtof=={Hls
zl@22E6q-E+iA0rrI&jWP5(nfsmZXjGhir(6aO8%U7s}v?fudTZF%m@Epx#Uar+qd&
ziWFWe9vC0SME>_KK^#~c>b<zoCeAEJxY>Kqz%N>QzZ#e(j>LG9XIRDb>n_TjUrSFE
za8ab`zh0uUi%v|<x}EjHC=f{)@}P!Z`!tYPh}uKLgFj=(ux%cNxiIIrA|&(87tRbl
z-`!*<he1=CAh017dCBmvzcYRdyD@mowhuRdngNhiG^tVPcY8M1^jg(^x-i)E?GxY`
zTX(g;J5Ih8y`0mWE0M*tpObSO{XNv+x~o9U??igM_9exUFNekXv(x%s%c9A-WgV<W
zr#AWa_fn(O&)74n&9Ct!tLO2e(=e7as&tw~31m8y!kdMo_(2WkuLgWFNb@!Xz0I&f
zPwGZ7dGT7~+W4=bi0H*h)Pp}o$RtlTI_-{@8p(d&1lSF~r{wW#{ben&;d>$?kJa^S
zJ0E-o+sK!^{=IHDTPmOLbnAys2=Aa;1cdl_yw9w>nH5@Lhvp;uW+|Ya_llZmi*^_z
zQa#OpJt=KcL)&+A)uxaG)l2F9R<Q`l;;ZJo7(N67Ki>_n`Z2|(YwEkQzjJSFz!E!D
z@&r+244Z9}G*tz9BU8NGCbD_8nr!+ELj$hYJ?B1TeNq(qVE8$~#Am+@QkKi_OvdZH
zkz35Si^IJ;MSJt1+xZ|Eq9&s_+JSqU`&I}2)U(qor^_yy!rlP|KrMgpF}uac8L-7A
zSnTo(hnXUI8F7f|c{EVMfvC@HxpEG+&%mK?i~W4@e2p$WVZH0_XtPJnr~C5|KXFNv
za6o|ATKonPK*SL`=Zq4LipPYdWu;E?5pmu_i=77aVcSTqo(4j)sR)A8#16*xm%h`b
zSNmPKhDw7nfHzI`P{SUbNK-SlJCVsrz24!?-Ru4w_5{L~s;5NG&Nh>|S#_q}`DV?-
z!*V{S-{JIGR&JACkEO@g8OgVrqhMlIL}HRB&)X}ji8?#|98L;lGwH}QOKoW2$Z{T!
z;DLRLRGBJQ^1}LY@j-`vRg9L&P_Je#IV*t|q0H-#1Q)d%4=}6aGgaKq`l32}{!<z(
z-3MSt#$;u5k+^0dubEa9;`I77QYMWNOZ8HOAfaF`tp%SC;WhqLi^(W{OmdYgwNs#q
z5)p~9P<b(}jDQ#m0f~EUVKROY4=iXjn}079HYI%i;>~8)`eOv9ZihO$h*3c_!_NrJ
zFIf!sUNQ>m)LTg<(F!%*>Pqqu&d3#wBe5o4BxV5=#1sgHvtHe2hK`+#5$HN221#t*
z_xJP?{P@n}QltIdYY1q)%;~sa!rL!yu~@$FN%PSH#-~o^(;-QV#Biimh%y*!J$v7~
z(Qd+7G<CbX@<{ZK9KX$l9Ixpor*D2J2Y#&nMW*r;r8jbBGIBAW<!V>Th!m_u&xZmI
zcQCp7{nC!1<F5+!Alb1n6r9q<7{Q_!IPJuI7xl^!^r_yNYM)N<8qfAz<LwugM)aEI
zahTt;alfMsVdxZy7I&O0g3%s%Z>f%7N`e_mXt~<gTKfxjcbh{)#m5`TaD9KLe7CA=
zGU`8jS8;0bl=Z!$srXKsR1>?m-}D>y;D!g7g>VBGev_ilR82~cSm94!USUj6);&`2
z(zzz%Pg~u;G)@g#{jsXhLkADe%(*-0VlJ$KPpNf&Iadt>5xUo3JIS<mV3F7G9mkEr
zto=(~v8N87thpbHG;HvV)r8u}kd`EWn?Q6=e-wpPO{Og5XRQxyHd);ig-rK~$~HHt
znR!*%VP)o#Z8S&jo)2^>6&5tn5o$+0t+*?R@d>(D#n+rc>*I;OMfq9cJ`6pzm{-$d
zi|F%)IQF5cmK%03Mn}Yj#=L<q`hJ-ai!3RCQ)FXDQo2+LdARbIB0=|17dcUIISsgT
zoLVlR#g;&G%p0;IKgeho4q7YSVtY9X_j_jV^uLH4HzVIXyo$;iO#Ga-dc%qTQ<}nF
z7D2=5n9~^!harpB-#f<7#uKc1T3bqCra(2lY9J+Fg&ZhSGH)RDl$`kO@+I6jZBK59
z+KWt~AOQ$)%J!tf%-TD&K8PON$K-TL)FduJtpl-Z5*HRDIatC2v7){xWAg7p(WYrT
zlv)G1FfFRLh!_UlEc}a3-6e4QNcD4)kyH0-60fM$#?=)^*buI$TUq#z<=$bF&R`)A
zJS<0AOeJuKeK0^FO0VY+C%Bxtsx%m}O?3yf9AsKAl!|aRIPDtwQG9g5+dDDn>^pnR
ziU!P43x_JFNl#N_r)g*qKkZ8=QmkbwrI`U$l)zcSQqv9hXK>_fc*9!WVE&-RJp%@1
zFzD)gkUtA9_-ZNWF*cMBIWryEadAQYB0r2DZV^%Pu^d}HfK{KnyV4X+pX&`Ys*_p<
zL3)r3fuLDg_jQ%N#tv>tuvoZauAXLGDBW{wx#+UW_%8!5ekf*xteashNr}<%f=^J*
z*ugeMQb|O()$lL4{tanWV<hSah97@A+8uBqwz>SI1qWM=&v-Ik`Po*>^qqA`Z4Fi+
z-1%=mxCe<}1lw4(x2ho}+p+_gOEZR{Q~W+mN?2MG2kxD7%*s`1<+j>68TN*xHbye7
zax_+8r&RNe1^JK4H)ZnP6h_R-xGaq4QO3g<$euHCbWCZYF&6ezpn)0R!}rId3jT#J
zDHuQKJ7~hbn1WMDUYE#19Qi*XlVmkuE|VRo3!hkjEH8{YI~>mbqkr<lH4X;l5P^mj
z&t%3AM0Z_N{s}tXM8JKNbyL?VNsMB}{j=lu@uw|5_9cFtWOhC>h^Zq|9JvVVJ1vCj
zdk-=wc!`@_bNmnH3v?N;q{P94V*eM$h@EHvWGY;{4K4nZf%B3&<z$XlIE;$<m{!qX
zN%|EB_$~54KCwI}JrWUi!l@)R!-wzUQlm~ZMN%QiYLuL^g0Uzz-{Q6SO~#1R+fy>d
z@mEszIuKc5R0H`1SGIwcC6&=EeTV4xSluL4ytr!?5ig)><E)%a3{s<y+`ZCks$UE>
zb_q&?ANxwEItyICTg4{;i#lPrbMY-=Z>u>ejyl$+R`8>A1lJ!Ls^G{I2*N0slzlXg
z>TE>ulGIS7wQk}?n3OKe?`jxE*ZXlhyEz#>6K|N&3T6lI^q|&VuNPkzs9lJ<%0c12
zM+*p~TBxF*lQF&=LS}vy1~?q^jbSX=by!%`;3HG4`)d};bQ3LU{KjYvp}r?DjSPeo
z94kHyg)@2BgRjrA`DwkTG<GszXo+%^)BlBt3~n{PcJ^KQ8Z@OjgpkEfQE@e$*D{Sl
z0lK0eNdb*QB}j2BRosuCh^X^-_*(t8uGr!uI+6^9J~9KRs}+{7O<Y4ko{A3kNW7K2
zws>oJ0EH~r3Psh4i&PaJBAJ2=GP>Z6!9GVzUG?P+;yjhtn@uZ`8z+bwCcPOKkUgga
zN#1yNEv?xUZ+4?}w42}hXy{>ZF2|#Y6P#%5VZ(hVFuVs`V*0q@uf#;dN#b92`=l6v
zLzW7^&j{HpzD`kAv_c6p!;Viu{Dj<oydl>)%A1XL0wEJc0cQmkgxb7LV1lbW-~(fR
zp`N$!!6RIEkc!HINosgZk{B}>IJ-59B*QB`4i_%A3NftOyYf>HbHXZS%s;N<+=F1I
zs;z3!tl{jo%8Q3aE1!f9S@u|KIRbkGKx|<y-l8N^vw!fn3iVDSTg}+BZy3tZ@!gCw
z_tSv}41bgqSt*+*+UM#>{&2Cr0?3KA<DvS7XwKnrl5c&={17O+@!2dTlcO=(&q`;5
zdKI1X(bSB}rILDeo)?#z?{a$8hHuG=jx^`3mG`oQfv6RvS@t`S_jBb@D~6#Mw5KnA
zm?bm!FCT_|H(>9JB6aJiKP|nHTtoSk$m>_Ls;Rd%<9=>JXgxlB*wC&+P~Z^T)B5&D
zrD}m_v)wnkYJ-;QPtk7nZPGxDE5EvNeNClt$Ok(h6#9XPNnNioI&71e*Iw=JVt*gV
zxuQle<i3ugaJl|%%%s;qk;bf>pi}pvRzz5`H0B2@9galD^uT)0*?eP#819%}z3Ptq
z8z)2VMXXOja4I>IiG-XX5if`&@2^iP=nHnzq+2AiHQ!2`UF^-wbq5Bne0oEoU9B~8
z$ybmAB(Othbn0Av#Rg4ovK{*t4ZOW5qZ^a2GHIn%8hdoywhR`tS@uU;rQ6>AK-`_k
z$v9Z3RXqPS<=H+(KnQ0FZS}wB!(&hjV$!Ws8cSzA=M_kB<^UpH9km>d6>vXaF4a9M
z#1PUhIUQrMRjO~4S;W&g+`S&A^7#wIx}pZR!`O6KNHf&-7VF95UkarskP1jj$KcBE
zsdo(_d&Prf@&L!Ve7)UM^Y(1FGhqrE7Y}&(^_TQ=;$H%jakors*YE;eYH^%AaB+U=
z$ZoxL@<@+j>8m%m(dq9N)TZRbq?oPAWjAhv-Jbud_&JgD?SsA^OB_5<ybI=c-bgJ8
zwx6$_Nv2h7mQU5>#0BDfh}CsKLg?G;0)Lt3^t`b58KImjzV;Nd84ZW)j@(MSCwm&>
zkGkx+rX!XdKA?KQHq=sF<Myz=yEM@=662-qXeuMzND4P6KX|d;G@~Rqo7Z8sFC1;c
zSdY!P1E3#3ix+k7c)VKW@%(Kh=zqs!J(X`<Ao)=%LIPwWY56rSiOdd2LvqEfu~dF~
z?mxC-Q!~_{Qm@fsu11&r7Vcg|YO<#p*{IV;dnM1rdlxU8-#OoIYy?5^xn^O2b9Nnv
z)$mg)Tfpn7{O6SRzKSURFd?!T;vWFl?M)X+X?1wLwM3rKxI+N8Rd>(9ChX6ag#{5^
z)>a}s^><eAsbqggFyKjj%N20zh@YIwc>&Oc>e!aZ#M=T@24U(FE6sh<X#gz(SkD+2
z14%ZKtPEJ9FEoXmm-?B9wc1J#-at^eEbsf{^(Mb9qu!Aj1yC1(u<!rXO(P_@OZ41|
zYWj62jePY=i^I1r|NA{4<eti9JxVjU6-gM(X52OngyU6!#9hh(rA!zQ)#Nx83MvaJ
z2z>Gu7#JFw2!_hNfE$+%X4;$J-z?q#RzZLDY7-L*Xx<BxnJJNL(mWy01eM%Z5>O}<
zksUqu#2F&!aR37f9?E|VC60x^c+4ZOaDD;M$EmCSQTSp%&lD1ae2CfdW|eQ_*9{^y
zXegvuIu@Ib7VyW?;LIUMfGC!VIDSh2F3;XVt>G;Efc@@cT)bh*Q(A{{`#aZdLYc%w
zv8UER?jLjL;eeJFRgX0_=y9IS>zbzd;&nXT9p;m7*HFgyRB2Z1#JMje)2ZYV&g!{r
ztpm$*85(fW#g(S~fP;gNPq!G?sK*k)43BnX3R!8klZwS>rNWW~HpfE_QH9g1=T+)j
z=OcWTbgwmNNxmlrG;Q--%8!=@dUHM{Rt1a34wJW9T8jci6a&nJnF>)PF+M&`>Mx7_
z$LWNZ7K=O>aXr)!gJxR^#SFGI<^>zT!+L8DSpy>HE};Xh1b`nVzf@xexSkiYitI$6
zDOkSPAe%X8R-%jDe1``tRjSRttK?5fAe;-kB*Md!MnJ<)^(C%1?cRXV{wSs)2nIaK
zS)jyY*A`SFjS&|_56l@z7JYXp@H<1f+?xsg<hU#eSRqaqc|CVi=*^Erv7Ln!cqEL{
zt^UaSY|<fqPC$|9YSw(Oe~(PaG*GMqo~M$IAX@@IvRSx6+>1th?XPh}ozbIH5rof_
zA5LdVWN|)GJOBz;1(aUUu}~5)azYxQ{($AMTWT=pv7Ig)Ap3RyTvQZ{VhS4Pn#^D?
z;34ey%$3@s3_7L<f(lG;-~m;_6alpkm{0<#4LqZ--h`dvUeTv~VPC7bYiZV790;^O
z=&3IRb_YB*E2Mwp{5Q>7BS)nKb-48W8=%xh*+79UogUJFyp=EFyJH*VS$`geW$FeP
z{vJFiP96|I$oVcEmQ}V?kREyqUHbGU=K{a~g~gm0aLO|RT1JCF8w&id0Yw7Tff6mj
z%t2&fNww7*inN<8322Ur3@Rxt?(>H5uI)ab6uJOvC`KbFb?c1i*x%R^sIE6+)Wlk_
zp#Ys#J$Hu37%GZHenur7qqbOQY}Xftg2;*xn}-Xj)~#oo57IMFy>3uJwRnhoR+N~q
z=)fZ4yQlk;-GO2tLzCTFT4dM^oYwN5=10O;zuQkD*szSwtN`A;XIYBF+uxfh(fVj5
zAHuvfoJ1W@E3BA91F6=jRalC_Zci+eOQMp0dqXKB4y;{Vx^Jgj&(HDlVE$tx5M2o!
zQ6w=K)V!|E7xfqPI9lRtMx82~hX9sF*=#P53MeE>8B_MV*@|?%MR^3`qjY$|3TaF;
zK#Hr(pMr{ts%qv&K&IzemM>Ay$pOol`0n=_;~Rt{1KWf<lP*62bAf36CL8tDPj4pF
z8Q7$I0G1)Y+UXrr%X1v~9o}05U+Y$<Rs?>oT{1&L#l&0`^j2YqJ!hGcVsv6}`_R~z
zj9*YwQ_qh*IEX_69{$MoX?z1p6pDb&PtpKk`Dg;rPAe)y<S^zZTmW5kH5>z3k1gBA
z1r;;Q{&f5M<S*!V!x0`HZkG#BPoMw@BTVCs7!z|`YzDh&IyJnLc-$@6%=@iZ$V3i*
zB4EUw#w8o(+9Uv8@oPh0Nu;wH%3K~URycRvMi{i%<F&>K&?{p#EIVqox!d(sf>zZH
zYIT%ncVUtMJPN_8y$8&vn!9`RT78d2V;BSKSC%WFhOA0D7QfkLOVWO}G@28wSK|;B
zV6m~b?DsGU>e&za++QvR9O^sqpF1zWiUN#8(*UP0p_^3ZfPY8|wHUC&DPX1ul?3@-
z+bDfsX_p4VLOHQ|=%n7PG#Ak-rmOWwpvP14C|3Cb8{FMg`U&S0h-DQ6#WS%1Pq(kD
zH7gSSa&4#bN>I?z$t3e%QP`X<Kr~c#punD`d|>L438Jlb`!%VY!|#k?P7|l@e#AUe
zXDk%bNZvFczyUS@G_4RX8BzJyH&wB;F?{ExhT!Mwq-l*zQ^)u3W{Rcv8;8hBUJAX-
z?Cvq@3wq9JskL5aECaA%=Uuw0=xwP{o>n0CN0>MW2P6cX06=j=DZoumhPa?|KqR)P
z6`VLgDIDqL{x?}Rx$BE>sw7QVbm@XjOfjegY)al|J4L{0n%P=v)NBS!28(6KBlQG0
zTo{1bg^M+}Dd<A|Rv|kCFn2P>ZSF-t0)94@y{nRaOOIF+$c~!D*k5#F^+#e>6iG$S
z2E!v&YE@};(GN4)hJds@fwLr;I%G58UO9zF04mdi2gpVjCVBAyRKB@tft+zanD$GM
z|2M969AA|6&(ubx12-)=a*fV^b~X*^1Ualq(N9peBCoZ3CIaba23cD*w6|uEhvze>
z`#rB&TLznPF`x-!9*$i%+i39VJw{YDGJb?nHCj~`b$xH`+@C4ys;O`K+z%Jx8&K{>
zQs~Pv6i#VTl~I0gbTY5|jAgK^c0Zgc+J^L$VLY2#xX=|sqCBxc6Jq+UCQ%^#X^H$s
zk`cfG%0MbQBV~w5=Ywq@wrkSGb5bVXUW)!>v-742aU6h#m6w}dFGgz>QGY#N{ahA<
zaX`UB-x~})<p;Wbl35DcF8~S}V+hz)I(;sSIW4~uHbB1OdjlncJYJ_}UmDglRYk0i
z`Wh}GfD~{K);z8krQs(>M&FYELOTd1L0A)i0Q54;rv}vT830%(s<>wmg9qq7SZ0ml
z_NB0wnp5N}t6{7@^2i>FprjtabnTXNFC`?yQ)3|mY77qkC~<B(R|$Qy)|$k@)Pu-J
z`ViX#NOs2-6gz5;B!CeoUifS%aAy(6UjtSZl9&dZ{CaE(uY0Hn6|B5adg(&Z_Z3qa
z%o&#T^(=vkQM04F^Y)9qJc~z4FDw9*&7Rzb+ao^Pfu<<HTTunJ0J=6|>zc&#ev|7V
zp*5_GXfU6ctJ2&><daESwRN>1I*rDmp9Qq&841eZ6zuXo7rP?X_lQR&?p96Z3CX%J
z$45mW!_q0!h5$3UYg_pC$5)=$lezDiga~xV0cGDSA#-O%csJPh^J<BkAZOie8XxWh
zTMU!{|Gq!@8!2J1_;Ycq?Erc<`NPVVbXZH&E8AFcpfGO4rfKAup+z}M(-H#Io#RN%
zx@=bfg}6ab4py)Pb%<(WEf;I05y!tW-G|^Fp+WmwYJCf;Q?w7c4F3}CD?rbMBIUVM
z|9(Y00|&MF!M^{LDu@6JS+N{t{P)X@;Thg&8?&(d{|$ixhQj}!Ise}@CrJGJ*KS?g
zJ0YpPmadmoa)YzweLItvhCIi|!CIdkn3=R|i?_zU!pv@%cEB9qJE`w0Vi|iWeB{$<
z=LD5jEfxEDMAR4brI^@n?wk<(`Q1^&Wy3Vv<>4zYV$=NPSZ~y`kV8z7P!u>kr)gr8
zl0umHypqmD^m%$;-<Au3IcGR8&gY!>4onI5cdwKgu5rTG$Y*axM?@m`E}p2Js>;W5
zlk)9ul2<*NjKzOoVuOf4CuDA4)v7P-CUlexUY%*)9j}(*;*+U3$Nvo)o%fk+um+qY
z@A*srD~~q+OGMt^o6+{oPL*zUm-HK<Lf4hL;kspJduhd<2PX|jDFV%BO}9?VJB;d4
zcWn7T->oYV{5EQrFERDktq%W4Fsyts@lw@}`}L{bKu$*eOiMTWsP7gd6=NcreCWUw
zxu8?xy;ma`H;bw4(WsHp{b!H5cYcj+fI^aB+M|gD5`juCZ3jsLC6_hoXgYg_CRxI@
zaUz~{k@LN3;j!Yvd){${JW&!E+Jfu9_q$Ay_`j4O28DY=JmYeZn;zB2fT%jtRwdee
zX4U<VR%aw3KyKwcQgyoV&Ha(}O1piEZ9{@O16FSz^xNL&FJlZ+pT;kqibqm8Wp#uw
z>X2&0OvS~NGg;XqWo4#s)?-n+?O$~rVrE~+kYA?y1PCRd>M~f1B?e$gi;OO0{Hm4d
zlua|Owd>AU>V+l!MoH?b0A<y5=)~GL+rg@Eh(?wy-duQRY+Vf99Vf#OYj2Dau!PHM
zemN*XeCl3O<`6}LszWK+yJ?e7noe;>P1`az?I{U?xKAee&_pq>DGQef)JX80KfZD`
z*s-y3&V9OFt9F;UPVH45d)})Yoe4d&?uZ$wqBGj#Uo{YQ7JnFS>yr0Z7yAAW_^_ZE
zg-J<o>wM3;%~EE`aKgv$MZ&5Dyg2@+{aT}$BC*1WCt@8_{q~<TcAdBixwQjJB>WN&
zt^}X|H9)7rK?mMsii+;xf9Qyg&;Rq(Vz|p-wsf$p(&?+Z&q62~&o^mk)2;ot^K+$6
zq|Md&lpuOhD9O8}sZG*rI1%Ieh({wdDzECv^EoGrK;LZjYx3!OQ&!5#Yb?}M{}7)*
z{+R%=u25$rrP4QIABuU1zI|)pZDQ1J<M|^}J$9%B5v)pQm_ZY04L~v$y~gaV%&gFY
zR&gscL31B}s0w<Xs<s8**X?Pi6+c+V(Aq`C6dgCd@kJ2?!s(KOOc~UBhCk36bd7``
zGyJyP@Fx{9Pu2<S^ZPHZ8^3*o8~$s>IhxI1Vd!2NI(d}%?bk+!ds>&DbF*Lh?Wt30
ziJ4L23j(g)aT$HGso(hI1gA-TSiYA9vF`Vy=;lF6Qu^mRrNw=`GwPm4gv)8j`IQFw
zg>3?j_l47p6{Q{EUu%PCs4tx}EqT2CP8MF?{hH#|_#2A|ck?QuMzS0IQb%y6X554$
z@ds<gkDWN=VsaUIqxz>y_VqT`JgTs#yT*qvl?|1vVK3@`VUW>xDB3ezXvu%nV=c}g
zp`9nln0#XOspAt6^!_MAd&yZ4_)^w;0@1VRm*Jcu&xF89ppsC=lY$*rX-PZ3bGtV|
zr)d@Adgt5NX>KIMddy>msZNu{{5r8Hab;2!Vnc%3)CzK^8$Jm<0y0-pJi?j$Az0yp
zA6DRuxJsaBrtMmtko=zuX7*t+eBz!P6E>a!?hPWDp-+D+Qe>zYzm*nezclqcFc+X@
zQ+X2cR=M%UEE{|M_iY=(qf=~>NN&W|0sM{2AGm&YQj=Gv?tj(zrVsodx@Jqz=YP+w
z{YmB_!>HUC-OsK_QHVDw=a;#5ZAX|NJsUCARMKi*6&YT2KW*2#+I_wLTL4>PEJ;Kf
z)4hr!Bt>NkuSP3fQ0b)-<G1p6Z2?~2oc5U$Vt>2~I9UMxScERZAD8(UcsI^i_api<
zEC|tdE3#l&8#lqOXvVL_840YC-`8Mpj2z!dJRN0>Ga%5kDz}_jy+}TG{R&;5S5f5h
zmO$`qKQD6BUFJ<T{ToO$ZOkudz2n4dyUhbu?9Iki0{QKbY&*T<cgbev5E=RkiTY1k
z=f~w&R*&)!q=h#nMB6#AJz;;swCblgk<A;7uN@0%lPEvl&6oeBc6W)x1uFE64EK5y
zoi6HJoF8<r?na{%sr(nlFMcIFo|S0~p2Hxg20bpM?5rL5qaAwTT*B$$lhI`CagE4t
z=r2mn{??S!S^kxFzWzbCPJ_}zfhe|zy!8c6vR4+FYQNJAdzU45iVO*f30f&)OL(n-
zqDng4;XGl}_V!P;A6fhtOUrPw&FkW~Qax6rJRgy1^J~QIxXUtzZt9Q5w5!I)w2yur
z^ah)LM(ZPC^0o0+#4l3nEA;f)R2K}~p<(398K^tK)MOZ<_OB<NN<wx)@4N%=Dh#WG
zL=8Mr!CZS+^Ro6lw<Ii7uUf7-;+>!Jk5DWJl4(6L1nFh`UQ1g>Z3rLSb$a7eif3)B
zoNYr&LuY26JPddD8YVJDo5cxTg5fW=s=k$_!Qk3o9B~_jdo%?W4qDob`ee3V%YL3o
zsqiyHj*z(Fm2jlcelLuSH-M^|>wFhq@j7Rr2yJKzo1=<WknhsB%T9H)0On$KPL{nA
z5H!x*<vh$bTmtVN^PNy_XP>@mh~LuK`S30`nqqkVT>FHx+2Pjb7S1D&cXUMW>e~&w
z-Q{?=hNLR2!D5vE;=K9GN4OI{UmiSMdk}Bs+nH8O<IA$u>30gN0q*XHCj0x-k2Rbc
zw4tqP%zjV5T?9W`C#p$Dnfy@rWm2L5@xt#M&379bK}7p?O7geW&Qsu{d}YyFEoz#1
zy%+F1ora!ya`4_Et#sYX1d&>rdl@9iB2AVqG+?Et{hNI?c~LLK_yYnFnb{{^L+eyH
z#R{RS!4PQ(#f{LSQ>(Y8#3Ns|#A`-}G`g0m-6*oNrEh^8D;5Jtv*H4#euuovQN>R`
zF>>W-!(xus*+nmKdG!~6qCbUwVtJ6h`i}Hhm|Nbi5dI>A;tapP{ac-Nw}OUR{J6aj
z#m8F7>j1xBaCUxQ{dsRvkoo-Xaaf6o7T@mk-BTBFvDVm-oSY;#noh6iZ;qXqU;MoN
zBU&*;q6Crnfag~zGRFH}q-eCyxOP5LWWDsZP=R}fVeO)lIHVfk+YP7D*M?=zLQmeC
zK|YA-@m0YKm+T{O=T#4}^YG9NM|wc7zHS_%l~rd#7^2<W*DtQAMdh{Fa6Svv0go#x
z6bJs==ppCO!|w8N0dh^}<-imv$ki|zM#SHfH*H=A!V<sM_nV4TR9|5S)35V7QZq1v
zwT#hyKQ|3w;zO4ZPF*;Ptl918<;SC{u;G$jc%qyh1JY<hP(5m5<<;Io4k$_L*{gnM
zZj4h!{36^4Um(AvTQZz(WMtp5XWo&_D|Cve&ym_h&r*t%#(bH=B=N%jlmOa;!PmC+
zf3^3OZE*$9o-hm^+?@e}1PJc#?jGENC1?ol9yB<?CAfwV++7j~8r)q5w?X#s-@D&n
zU)<g2zM7utYO7OSr>CmxS3>8DL2n$BZKwZ6$+LX;)|u5o#;YTWq^GZ^V`*^~;*}dk
z9l2L{CP^DUA#?f1Z0Hll8A<fnj}1yt8|eglQ@DYM++S$thva+jz2h0Uam>=>Q&%<Y
zUeB?ZPZ)28n^b!=qu@k-Ge!7+j-`LxW#}ZB>uPF2D93V;0M};1A!Mqf2l?h%%GO}C
z^9`=agR8AIauLi6Al~KIZm!L6S1l%ya2P996>~D<qPqX+2q$NudXt+yUCYmdYC0Or
z{yH>)Xii(<PDg@12@iG0u&@y}HUhum7{j(LlP-BgTEacpYK}0Y!f@$;8OvWdAuM@J
z(FcV7p~`VCm94X(tJNV70w{|h%U)Bd1UgONw>p!?*`mk*&)m1<Vzj5muV?IsN`}SP
zF&Ez0o#d4ZqMvOXg4c5>5%46i9ky>$GHrF^o7V5TwsS}0r*yJzBiAMdj4$mXny{*k
zewS0)=vZVRc;0qRAcmxolG}9X+TZ^0p5G^u72dltHonKD1r1X2iaK*wS3?AVGZ3l~
zD&Lo3#gZ@=ieChC2}U%;H{NRIPW&`F4(&U!jSOVUG=U2zjm=8iMGXJkjHCe8UZ-Me
zA+^fRJ@~$F#7T@XZw&m%#FTGX=8aE=M=K;D17^A^yZz=Y?8(?3QX7Y5B(u(mW(ZGe
zz|VMjdvED8`3^fsgd+{X^Uci363yRc?6|+Wzi;DGRUZ^veyc7tYLm88Ko9TI0U>Cp
zcf7VRH-i}|u9XLC^nZy*A&<yEs6Tsb@fR;&H1Qo5GV$P<cr6RwGboIRh0`I@HhEdq
z8b3OZacayYb(+Z~jXA=6t9`vT2VELU@y|mwxyFaLM+k_vU6apk0f2%yCH=;{M1*5D
z6af)>w9G(Nq9$Le)hbs`R0O-P*g<^Ss@?Ohngs<|vTeo16odVj>TsYu0U!c=q=&L&
zB>6@Bm`&##^<|k(DxU1wfXBkcP~k!DM4@_O4F8<Ou-D{vT-JGG&6G)>F+KCvm(S_P
zC>V`Y`)yhEzCLi9&3|$#KzcNw52n_Ie`RHQ6?M4YMEXXjZi5y?JWk)>Zz_s}m3`|A
zJrlRj-wkC5Cp{7&$|pUbc{Pn1Hr7+J%5mHm3Fm)gN$QcJ(BlT8#INo)qzSYcza?xh
zvM*BHT$omF`XcyNChwX!NeNv3cT%Sc=M_!h!4B<uc;tLlLdowzs1it>!|1$@UEcrB
zFHuJ-hz2CrFx2NKCNNw%a&qfqSlV(gO15iJ*#V2Qinl?2n+s{5)h>zrVBtTa&Y_`#
z0X)sZM`;zl=1%8#ffpdzy<-L}Wk*hls6lCL{C}IA?nX-AH-ADX(|*LpqvVn9Yt_|p
z?-kUQ>OY6krS;VX^|Ht-k0{#9$Do)c1%3pr7qVU~3!6GG8`7w^bFk=E=*RCM^YNs!
zNyD|nkW(^I<BNp07HkDK@JqeG>1^<s$e_#3bhBOJOGU7Hl|+9y+)M()obI4&=>^ys
z1IhdoZ!P7N{<?d3AW4hmFV+Od(t{xq#{tV<{a%09NknkqR|?^0-VHaCHVE?KQNH`L
z;*g*>9`qu62%GpOlzR#~PN;G&Pomz6EtWk9LrJi`iEmC(8AC)xLUgb4(<Qzoui4r@
znWpJ#>pi^YDU}YCC5bJpumRJro0%ZI@7w_X@vt#^N<q5N!7J3Lh_@xZY7}1nKevVW
zh$hv9FbkwQj}@o&k6NVR&zBnCqyIh<lzb08V2cae)b4F`*bL9;oC?hq7!s~SwJu)(
zGaL4#@a3=lW2S7mR~kv>OPks>y`hj3@;9nz#erXnL+-pR{zRer#kj|1>-OwA?8kZp
zk@2%V>;C$(j|IW_m&rwpu)#?T{T}!~7`YF1OK}l7k!Fh<Gp?^SQHWOM|GjO(#Ckb<
zG1dF{Nw?~Kz<ydWpUhU{skZom_gP%BFlsFdWf}{7Q+IA*S+_<&9}LNxi-qub!`xdt
zAT<I{B?nwCRr`4!rS|pD^X(C}F$f#8wo~dYR)PasQE+p$-`*p0Bx)E-*BKzs#&?5j
z`n+X&4}YeGe<+Px$>Ege1?z`zRPGU2Jb{kj>;rS3ca0-Pr}V`3sC_p~QwDAM38NJ3
z54?FMSg}lDDWob-g5DVBk3^o?r3Y_3KT5Fq9rwHz$Sqh+v{Nm_82pN^Ti~m*h4G`m
zs2Mo@Bwes4C|Cf>L9wTZXYT=LZ2|p8T8@V(wIlssd1D*YHaDn_oU3AHyGkRqF4)#M
ztvvy!KcIXr;gC_T9U8y-8{NP7u4e|1^U7G5r@NhsT!ApwNAe`YN6)F4t#8aWQ~{Q@
z;C>S)4M$|()aQ*xxW10m-fpSBY}}xZVa^>WvwPanB>0|xr)<{}*zt3iMklB1QZpe~
zjB@$H_+%8!HUT3mzii<sm)fCUQK(xM7W+7i1nYPHC~*A!JHJQK1_y@_@PY;4ib2jz
z>5L(>xz|F@E(<vo4E8{iv?|B%QBO6lJ`Tn~2(C~ioFk8d+eOj$pu~7f$_sAVx-wz^
ztIBd-FbX)#8q)o58ULuq<%q_7(pN)jahWlcj&(IH{RmvalOe@+!4Nt85WJSB2m~`)
zMZ{ynV|?E9cewyLpU8vx<+?7Iz~)ar8(HMB`8Wr!jhrEkmT>-5d?)X7*JE`Ur%*y@
zWbIL$yW6NIfjuq0BjwQ0;3H^GV=^6roT$Dg1#L3GkPXl;U!z(dh=fc0Z9tcsr4#c>
z{Ih;9)ei{@l!iRZz#LK}9|EkHF>zj$<$ffNu6a}VKH+i+hA7nc<x<Ners~7X4QbGQ
zueQQ;xTrv!`~?NzAgFdmUK6`o-?yUB<IWl-o}C#4t$9|P67hZ(7%3REqz6$<qlw-d
zh{z6S<dyU5_r1b%NL9U8EEf@uMLA_54b3g_XZBDGZY?3*n#7u6)_kON8fc{^9YPj4
z5D|Oc`5BIdg0dp2&o<<8hg<}lvrO&@E#5N=3r_qihp*2x4458O`JTVx@z8(t+f+2$
zhU#jK(z`|f8))18)m~Tt#0Y;p@;f2>hIrRriDHtV1+Km!WboBOn^~CiO$8L6=mKNK
zr~GSQ*<o7YcC{ukYAL*ut63C++Nkpy39*w+^!#c%d<7Esz*Qe3v)?b}f}n$3MjUEN
zlz?L58S#`7q-hO`^#c7?H#Z<=xnI)3lPV4>NNt0-NfX%;>|F9eaE;=ffTd_vMzg#s
zzK>fa7fwv&oCI&5DypbpH&{FX7yd_3G@CAlOc#sO*!49&dnRT5fjN4hJ!5_#B0(KF
zTCgRF_&1Re-r7#Av+wXU51xZGC5!5J=Gq&Fk#N;xS}O+(M5cYEEqAs20@<8gU8s0L
zac@u^PR%6IwP|hh+E8y9C(zrx_P-g4iDd4%7u2aLc13P}(Z9l%4qo|HesG3A_Z5Xb
z|5dBBv9O=X_0C^$L2qKOFVKBbBgC3WG52i^HN2&1PwMQ>xSV8J+qJDy((dwNV&ksY
z?D>ITk#~gF1lE!h-huY>ViIsFa(1x6#~+vZM?jJYI6-E0iVl*k_CbBM;wA^L?D|WY
z<F))OKEVlvRdCF6)R<97g&_vZopNUi>ATuxvkg%kzBbqXg5f5JE|gvVZAOqSpk$Yg
z=2i?q=y-mHf&752$no}ND;{T~E@@9a4|X2U7Rbv;qvbP7CMMJUh*pOvYUblAMqg9d
zY1yq}8)!!HzQe~&Vf~?JD^p<gwCi@C4eR7`vpEsISjuR2%N_S>&#^gK+Gc^eLGQ9I
zL{{AxOyv=W#fk5>4N2sApmfdmS2CPsQ{uy>)Sbeu=8h{&{7T<wK^L;BX#&0UN*%s;
zMF!7tnvX3l0xc_0hUfd`an7F#a(9-Cs%4%M1kC}LNt(A4QV%)D>30r(?;g_!UEQdu
z!l?OfAADHDx;ouA8?#u6VFhixR&Cmyzv7kzYLixfq*Xlla(k31?@gGhR=8CEB>gLe
zMxn{CcV>NWfhJMxKs`EHf^B6GX3gJvm{ys{Wv`nALPUSh&#=PKX}*uFdDinrvsC_b
zG~AFnN?WHGh+-4ji6t>A{?#<rb?ucK$XJ&!4t$OVk0xPKW*0xw421?*&9N;OdG04_
z);dGtEv&ie#6cN!;*cRazhV^-@I{nb9C9l0njIY|(<LZ?@Pm~nZ5Cd>tRaB#K~&nH
zdZ7G2js9CRA_xye#coX~PyIz4klf1-bw+&m7$JG;Qt){hS&oOVx<lIb2nQe*abM4Z
z;jr_t!Fjj_U?>V8MV7<qKO*45sC^E;2&f<cB&5__sH-@6*tgY+>G{gIdDvEoqEmnH
zftunWE@M;Rhua_T(#!Di^014D*$c<t0wQx%&bBj428fFjKWA#-OaDbQ1~KG7FIy-?
zYM6pR;|YiKZ8sWmh;u=YpI0g{;JB_|&RhO4YBfDEbSi~FNo<AC)e8_nX;NyP%3%)l
z8me*D2M6>4F6A^xUIND3%7Iu8iN1_i!L`F524(~^g^rf`DFsTdEtZ=Pa7qM9CZHZD
zb4>v?8P)K`P-b9CraH)6N%TM??x6Ra^DiV6SA9m_FO68$22!^HS_6}yF@H^<pkTf!
zBk@`sf@1&shtV0po{hijb;5E$o89s(L|uTQ4MQGq?Ms@4bpIBX0)DqC|Awd%3M>al
zQJBFNAdslgNM+Ci3e7!((e=qdjS_R*yebB;F#9G>-2h-2x2^u^t=upCBjHcFK!isN
zVk-~II7J5vruM;i@(cj2pdA<M>I-ehm-KFYfTKllZwkjuUe;bEv?J^V6t?EiM&N+Q
z0j||Fpwk0ZswXWBVSC~E_l5Tcz<@!JZZ%x>|4jOv=7q79?x_OwKtY*X7~2f!0Z#92
zbgCOr$l}^0m{$Xq^XNW=>nRVg2BIvR!UHrl(9#*R6$CgjDUo4!8w(fK<;2tTVD+-D
z!rKhbaG-`0oHKl%1k~hS^%#9m6(0yS8yT1;jHyc!=LCP#87pDSe9&Rc81=$HElL)j
zLMw~PkxU@1j!6N_2ZDte0QE!8L`w&cOC59s$j4Ciy@jJfAXvT&m>6jME2ktbH9YRn
z|B<rP|A!ilF@sI~TPhyv+%mPRe5$moXQ*rNg~n6e9?u)MBiKZv-aK<yt0o>`kjfm7
z7gOK|<4?~ziTN>SiW;*D1>6~6KEP^{nF7MIJ`D(Hwtbq%FB)LvlF7UIm^P$G#hTRO
z@iSGZHD2C;x5EV1@!{#7>&|b`$#rO5ZU~j&Ph!%WY~~0`4Q^{Y&J9}-Uh67b^!u1N
zLFd{2tD>n;L=9NI9dcM#@4~|Mg$HNDP00D#h5+t8w6iw-Y;&8)bXA$ueUDi#jRo&B
zOQY#8|3(e}l_nE3D%bQZKZVC7*}+3Rx&(WOe#J>4QH1bYF~0`UH*yga2Fle=Qz{2M
zADI!+2)_vCL{Y%5pI(#ARVN>31|(cmH)$jcQisJlq>c{oFjvyF^Z6utpYWoDbxGnj
zeBJf0C#DU|zcHrmbb8k5W%S)M*Q`6}FGuz$dMF~@uUhs0ev_*Ov$qp_uZmy0bVU5L
z#`?p1MM<-21><}zQqL8SpyoGxkTzQP1FDquK%iVDrc9lsl0v^dk2RF4bsIJ=I!n)F
zb(JH!xiu+!vqe2uCUmuUMWuK{Ha>^?%G;-q++(t=Olb_6BmU4gzD>wMIE-4=c#8@0
z54H<Ezn!rej$;)YF_(!?Bz$C$Pf@B#3=TsxjGSfhRl#7+O@bk*q|(NV4lF?X!&gJ$
zESfg1%3Qs#7Bo-B%1!Y9rnwX;qKF0r)LnMgSBFYM1irhvH#b5rTqXPKS&(2)E6f);
zR4lEH&osYVFdDCIyh9`t_DSVM{l0}K+SnKiR6%>sXZuPJcXAotw=ob*%TqJJOP+5>
zK<ywBwBENG{o74=YAK3DCiIBtL@qJ*qm8ET3gh*{-HEs<OR`Os9Q~Ub-Tj{)?zX4h
z9QJ$-glKu6o(|RSKJ5$NT}3z**+wO{cq9!O4Fn-P+b8uGtLYB2tp>3A+&tv_REYUE
zsPYC}G@lI5wm<qe*AEUw;wL?~_$;!WUkx%_MSSy_uXl(b*ZJ}Dz+G28*v)Q?^JKnV
zD}KnVn9=8*^>UL%=&!TF?eoSFw|y;D0F0710fdM)Fh)FrCM(7)B=X^sA={-oxyCi<
z@}uVBg#aiWpQ^c_pgP`Cjo;wDqgnL}xwq$7TfgAxZuRjG$9a&5S{N*^s3rFebG>;b
z%x<?P)BRh29MkLXBRbg0cQ6bG7A^VtDdq}TovOC$^U*TxYS{(0vsxT<Qqrl>-#(n&
zZ|TgN>{&f^<GX5YhfjOmVI%83$-iHBK1>D5Mb@fSMM4lcavd5+O?FGa4*R;J`;Qfz
zRg`ZAD-PBF_<qSSl>aOpXmc{iCaQBYh14yA!TVjcnvo&L<aOo6UTR0$AQQN`C<?jQ
zICj#z)L)}_aqB66$_=0PL)YH2xuyUox!Z%nWZRAYGQS)p*>YXffxBAt62;ZyJb~f&
ze;k7{$AfWLL~S(V9?g0a`U|~h1b>pR$J_Ir)M5Vh7()9~RW)g1(_7!Q#0z~q>o&YQ
zHmVuq1hNx1{l<T_8|)IbZw&=TO{F3*D9nGck)uphTg_$*fqu-BUi;v+wohcGpPBhn
zeO`^lLzib?*t(8)bN<GX3<J@77hA}OUn=VIPn8(zj!jz@%ag^GCM&!AVUUeU0y^oE
zg&J@EaX(jXlHKD1`iJZ5K-gM0PQA|V8V$^`JvQ!@Aor<%nU9YH$(^(1vi!VSrT3lt
zz4s?i<=eASweO=mI%9L`xftI(5*8`R$S*t{F_odK5-S;;Mo-I~Y%HWGdo!$p!#X#g
z@YC&Sd(lHE3DVbSaz#+gOQM|6+k!U2tU}=~L@lYDPHvCH8Efoj4D44IXdDOWOmY%M
zWFhjYOV8D?qeq#^P6OjdjBBrd`);^qmVFrRV`tl2Rx{N3dYEeX%Gd>eL(0maS?(I`
zWnoit!?H6%vic>YLQNb64Dl;<sO&1;m23DD(T9G9DnS|fTT$EFW6=i(CTrh3Oz<x0
zAxO)nMkF)*)jm*GRuJs@&XA9>!Lh{<eHi(|X1Lg$N<NB{VML17bjE~Lb)UY~X{Sq-
z)9C?w#;~f?$9u0G5T~e<ej3%wQt!@tmuUp2P&g6cOZ=P1dOfb5(qm>GG%0cvzMseW
zPc4TZFQA<W$VKB6Y2_wot|A-a%LQL=Wn+WK-1+B}AnB78*=-`*229#p*dz_!KH6T&
zD;9o^`pxP{JT+`XLTxk7Vh^oQDuen{cjKCLhdlvUA!7t0E!bB01`0D&6Yo8Pqg2dg
z@zUye_N`Jq4=S-4LDXBee;xk^j?`kQ0$a8hu^{M<1JC}A9IA4;H^qA~SkajOuZTFp
zam)jP6>ZZB1&6Uw0!Dpx@3D6j?qZTG44)jn*YxgT((@-H(i;imot_`w_n?U=MyBj$
zp~2@kq~qb7W|?8)j8n>T5a&PLZ_*pHArSg9MUa1%uC&tnkt>;?RB4;QaAjL%Npkj7
z<7#zg32Ym0j^~F4xvRW*%vQ&lHX`f35M(O;%c6<GF9fcFa)IGbQ4Zr@DGs_tc7KFV
z%3dX_#G{I=iF!;YeiRE}YjAX=d`-7!T<FqoYdzGXVlHKSxgh3wX#Ek_bEiGm!kVw_
zkyw>hT9mp56UxLM3o*l`Ogg(Z3x$KRD|%^PT>^?HDBxz)664MO%2Ni<+quD`xdk$k
zm`nBzF7x4NOLr-dR$Q87pe#9kHkyuu{9MMUiYuyLPutXYbL-``5UE8Q8#{^ae)dO+
z>9xt2fggus*5)pxkS<l_a{v!Yejpja&3bTDZKn9Ax5|S2l;e1~u|r3ZPn_8{*_5?*
zgwUsRB9mqP_<`THh*#%NOGH+k_R}s@iwNiQhqKXFD4=b-N%wT?cYU@VUCUROwrEMG
z=`ryHu5Z9v-oH{BOEBjod941@*JV~wFEQ^b>+SsEY|$cQ{;hpng1wc<LJHxQ-E&cW
z_z<Aqo1R%}?&@8tYcOB+A#Klnb<xRpY!B#&d3P*9<Jf5~wr|a&1G9dc=IeJhks1}*
zTH}py9(a-OvolJ2lv=$o$|OU<ORk8^U6|7!ZThf~SZnCqoPDOvKUAjMA3lq^zfpT(
zhEybZD)84<N0JjNvCgU@yOJ<-5z^k$SHIT%-S|Q&LgdXl5LN`!VNtKzN#~{u5!u4v
z-V>%9mK*!G3-h~vwn}vAWDEHCMfgUZ%kNL%UGCu>77WoX1=STm7ndY%_BPa)0u%*p
za8S9NeIs=9!6cS#_5#rv{UhIl_0bo!eL_$#)rGK#O+fmo(MV;!@z&K}zpF^=f9Jn7
zveDC(&R%tuS@ylLxnMxc)q)T|%4->%Eoh6l4!YTR&_@Ml>;Q)~BT(+_F2u`WlI#9G
zY5jG)(ExqRTv(XY-rpGLf~H!aIb$5et1>|glK!`|khi|ievMuGqiJiFs^g_xD@(ws
zNUH}UgkLrjo>WXwe$nLE>jz&;cF9{*F0J|T3k$Lf(jhEmT!fPrqj)p3g`lWG$|En*
z$92_@_VrYJEEDN@XWZ~R$%MP)`NxAQI`KG9_g^6dCQ`?2%q63mn=SN{_7llNbesNs
z)<Fg}tC@#|f5P_jEtk)|#k?PEjMNte2T|a6aM)|`Ny2}9V01`{V$2c$u7xrY;%LxN
zdeSW9u1Is?^z1m|>i?K|Svu_hYB853n_21|Ik|yot3~uTOJ)MEFHbpIkS<2o2RzoR
zhCNTSr;R1SxxKIDmLR(&1<26rqdD1Y!Zv$2smZh4<IyuZEJs`Ag|}IvnjOZ^UkQVU
zSiD%lkKas+KQ`Bv3AU52cRpk8<)EMcwZ3n4UGPFa{&u_4ygu*!1j4HnF}=fXtz^$X
z^!sre#^GHQBun(!u?8edyLU$8U-_f|W+)*BI~~Wk2zyNj#GVVuvx!4u=PZ&$X<7En
z@hv$)NIzmX&rM=qtM>k3-Y?|(`8V^{*E1&>V>A~B|JiOqyYoISSDE_HvjUEi>>K%p
z)-y{)ezf3gg#+D?62({ttZ$K(p#J$Td9wE!$yDStA?+tZOmQ|$IjWEn$-rTs7+!W7
zK6p1`=lYJVf!0eJ;W0mJ8g&#`1;1gXO_sFgl2qRsh9Fnm@B97hr2nkUJ4F;#_PT#N
z-BoBM8!i%x2<&c0<~>?7RG92~pjy{&HZ_}DnCIfz{X5rat~~Q=(F@l}MK6iPZ^G*_
zLBEjzH0ELVCI~+ESB}13yIzxD0Zy+`G@yWfCeTY-K@FhV&zcZ$aU+^qsYwmxr%~?x
zW<Sw)&!3KaP{(t5$J_Qiy)U?;ZFNgq4F#_o{|xg&#CK6C%}~;ny0a8%;AxLr-w;gF
zo*Yu{z2Ou2$92HjMpG=6t##>Ni7H8Zj1Q;-`nY(acF3qncBLHU=kd47<R0XFYC?4B
zZc|q;kKXQbWg=QCkb>&^K^kF<Q0kr0_upjBE)LUmrvD;)xcyDyQIJ-BQofyLGwjW*
zUfcSniF&GaxeZh9sCg<DGv7#;1j?d_aI+ofq9^ZvEa&WoEG@hJB&70Lh@Y;=WAJUq
z2YHHLHI=ofA|+L`(@KYHqrl6i5nEgQLH&b#joLqSN!uwlbMQ`rR%^dN@cds<*w=_F
z+xqQhnX9KEAD+cfI5)oZUWrH^iz#G^CwI+5?v3Whg~N`_b8=VLMwy?(XhFlcvbMP$
ze5SFVO3>O)u-8I4KAm}Z3uQkuf+;6>MLw|X9dLfM>*0vgdg_CA2uqB!2{*p?n3i@c
z4>g<BQX4ufyf6JKXV-0IpbBzaIpNE$q3534&%sr;sGaJ?Lw59J$aOc-7I4MwoN5+v
zs>&x!teUp#y5e||T6SCq4;PYb?M+pL?Q$P1)gKE}s$tmwe&aTZiE3B3!OVRUT^d!|
zdkDPtHk=CoGz?<I4fvojywAkWvk5Sru>d)7ULO2yRpTax@-_{Mo<BU4j_NWw72;s=
z%<io0iav?hbIuzj7(&?3)~2Q@er40&*NLSBNv^>!H*X@LJ}lvEw8IdqGfp>o+qXI5
zS6+`MD<I~Oo^I4GPP-KLY9Tn~d?Le1Cz~9;=r8lUHs@JlS{KZ)WlyRJOWsFI+T8bC
zpHGT?JKX_O-R_-e>D}IL@JLCc1)VzS)yJ4<haquk2CkLQABWou8nd;Bt78q2A)#L+
zyj4b0g^yl3f?m|MlFR)$Uh}pL4>k5aS`h2iCGmlP^$lKK;0Gr$*NCpIcZxR7O){Tz
z%ppQ~h2^5ao35`g)mR@TJR*<gNG{0#ZDcFdf$04?;9se6dloe#jybH8@V%|RzTm)#
z|Agauue`9S@Hta5fkj}^-TBJ*d<Xqf(?{FYZzj5Ti^ehGVI;iW*lV3d=j|7_LCk5<
z-#ZQHpV>f_G5rN(;D4=yZ@#i8DVAjUbzFLSHL&)d{=PBEV6>zXvQ6wkh`b?(P+?Ol
zf00>GGanq33YD?ezq~jto!k1fEm7ceQw}fPf=-X6^?8TrQj*WdK0T(dhz@if)`b4(
z6kH))_G4iQ4Eu#61Zx;}wCtXIIFq{C7I|Ma869xCU9;+8CnQREKyBpfr`7DVX4U@j
ziDbRUgP|s0KdGOe3O%JZ6%PLT)4v%U_D;^d9HbxCQ$+WzbCQ9c92)JWu^kgb`HeoQ
zxDI2nYb&=3u><e10yKTjJ`rCOf9q}5+}&iml5C;m)0;ytt4)8^Zp>mN*%i!LNhoXi
z$@=2j6=o@Q>3=bf4(S@I{TQoxKf`buE@9kN8QCA99~z;@DE*B@<$l<==hgKX0^eT0
zTe@Luz#g{)5BtI1m4;t~0E=q}+KSe4iU3iLjXor9W_CxVtO>e~8u^18Dk5vYI!W_>
zmYsoje;E1FZR)Tgp4G!n*p2w(E}ME+3n2wzTO#;u<?lArapDKef{wo#y`)H$?j#+q
z4(|l@>HT~)QI5D>>tZQsKKxAQVuu&Oyi7l)d}~YB`bWQ*$l2TlYlo;-7Y<&;goUwN
zD&%G@nZ@Ue$;U{fd)Ze3hGOOgn6N&WqFHsf5)}Ki)_k47QJ#C~G`{hM+yzsaN+j7c
z!(olfFZ5n2CxKV;H`z&YDgyn%#|tbB9Ksa8U#?&039e9YWNyEuiQfGzwH!VQ{7s03
zLrGuD=<%M1$@Sm*c`PYwejRs~`Yhp_&g(GKW*m^SOfQ;*jJTi-zMf}gp5xhsMO_N;
ztbWc`$XGk0%WgV?ypx{GR3jc!P@E<D?osi~gyh-_FkPVQ*6yBZ1cb>Dv!}eNTSW6%
z{A*x?L=KL*n32VNsrb|V_Uc1k&IYS&@7*vMhGuRQkJD)XvDYMioqHauBV^`*yzRO+
z9hbSZj+1np1C&M0PDQ{L$ketsc7bsgro?rUIfR3gg=LlqLIzj*4_pLA*|iaLfyW4X
zc!ZV#MFDZ))^<|yez&3_g2-J;pLZsx%R}St=<xaq>dp8;k~m4Xcadi@7Tp^i#dhd$
z^)Ao;wzV8lQUwvcVFMMuS|UW$sQGr?{a;hFinto82=?U)EyBw8I0gO7^JDlGr6$N>
zY1)9m)e#4%Cj^|yW%3jrI5R?)LJ?5GiIqpr*jsCzx%u{QW6A!n*tB!=I(%<$`mWt}
z4Oqv5l3mxb-d?})$B1^iT~n`m0h!a0FnAA2Za;xolo*&(5v(){4x{(<bK_pxa>jkP
zMTO6O%nWnj-@sx?lmt;_F}H!sIcTzE%>oz&WRm9*?fTZ)8;5)(#-!@e;}!c|460L%
zZ1##j?9Wde&bwjW`*v;BLzJ=Bs*NgI_B#YEe*8QQZuq1XPt<i4tSin|VOU>$e;}T2
zZzdd0Pka1Vqb(XqH%vm0_DgLdrM)_;4|TYAaeveVR$x2tE}~3?!HZ9AQu>aE-14hL
zJHoyebip@d>nZH-zYAX%`gK)ko%YDX+M_M^(Ex*CLbxAEy9@WbCZ;$x(_!y0smd*O
zWtrZ0lI?6L1wom;V3t!s?F{D$1b1|H*hAh5ddLd(htVj{>^!~bO#B)n@tT3Md}pOO
z^2Sadrqsq*0lo~i7^`2t7h(R5>RShTC}gstbt|<Q%HBJqm*pB_w;K8SygqK#bGg{?
zm*fz|CNafSR;rmo4@JDWJeN=>!6rIaHvaykBT<z3YgRE^Hx<-v3_K`C&2X`KbuU-M
zx#M8P+RPG~>|E3m^4|1A@rj?2hv(nIhQs1V3I%%qMm6C=MmA-y!Loafnh!25m=tI+
zqRyf#+(wG+5_~InsxF@G@sevif8hor^f;7H=4Dm2-+l)$s?pdf8cKr+nqjX?OxRS<
zT&PeJ#1Bx_Q8dT-*_qO3veLm00%n@i_NxD6O-UCFp6&5uxlfA%m6#Av69PK{!Utoq
zjuDFRRp$)u8oB{txG^s0#SjtVo!40*W;Z|1bRg*@fQ6b7FqSjYE#bt&9nEkdeM+{j
zQAW!=?cqsJsQlDzt=Z}_TI8?(+N4ET7I3Bs0>smG7@x*Yl^5!+<mL-~@7AmJw7pL8
znO%$|lhT8bdV9W#6TY}u%)fJt{>SI`8Zg_Zagz|CCcvRuK>%Alp)P5?@{0q`1fcU&
zNOJiK2Q2OXdZVB&R0+aa`2Rgr5M6@W<84oO?joAinLgu-0iGJ34=~5`NeZjJ%mp5>
z+M{?P)_eYE&VWB;O*V4y#b{5x2KWr%Ys?7aU+nZtfOlXC<&6JjG2kq~Ww1stw(&Cm
zKr&#NWDrV*7ps3D;N}3g1se#zjN?oTxQJ-`HeWi%b_N(2sK*{H_cEp<(6eKxXVy!{
zAXUI$bc*)=3@FQYy?7;JvE!3o7}EtjN{~r63+2n0O@K3`31`FQrDGI+fSFy)>HW0-
zvzY&RGXMR=taNQ$6sCOAyh|ytUzD`<M+lk>0C9u!wC6QK2dM&#fZ%KfIdYfKu`B^D
z+6*87|5xR^h7pcqJT+mBca4t6o-slSU4WmE7}wyT*`gn<`F11P9ku22@f#7J>#>bV
zE4^7Dl+$y*&8t4k|9<}*z$CPxy8wXqa$`ui1~`tR-?qp*2e2oI3j>?Z^V5$leG{T!
z0JdNVh}H(jx&m(H$#)L^*uhL%#X|s4Ilk^mv)e9OlR8RI?Wxi$pbfFv4?7PN!eE;m
zFImCQM~u%;uo@QQ7Ht5BBxSbzA_SYC;I_6GGpXI@MiV7#&DoSY2wQ3vPrt+0OMa&E
zNvk-&l1)t&ZK6bts^SR1g%}zBo|9!*tW1*K!HMC8o2&Pg+<Pxy(9~Xf4S-f==mOJq
zS&wqb6k?O(OQ3Z>ub$%vKihP(dVshwWUvx_TWy9DUoQ~0xopY`oDb1{2c|JuYc2)E
z=&HrhN8^mY;MK4CUc??PHmU(|4h%p*dK#HylLQ(*V}kl!=lub52D^Finj3)B;o5`B
zFoZ~CMD4Nco_3uVMKFv^<x8#DwDr$2K1JxHNy@a<tQ#be(*ZjX5;$qvERu~Sbp<%9
zwB8d&QAi@{+r*2wzBbC%$pQe$Ozn0H6-$Ppna`QwJ_3nJuoi7PN(ZH0pwT7nO#>bq
za%^q^%$1)vIsjJ6)~`j!sl^@Nl5zPdY|pVwKDEp;>^Y{AEE*tZK|K&eJGk%K4$byh
z`3g+`@E8W|H2s3lgzJ6+l-O>i0wahI#7yL8xkJ6e0MNrxAQ1^`Chx~dWdM<qO5koQ
zX?u)oJegkQn|zt(_l?)AH68cm)k<!-ZS&4fkPpi(I<j%(Ok}(tCT^&&*NIQ$!e8M!
z&!Oe4BZGjv8@0;R2cPSHp%c8px1TQkdL@!$C(m=AUs2sMA~;8h^-Cv&M6+Py-4c*L
zB@DoqhZ2u-43gt%#+r}u17YglYqU1-G#zX5fn*JW1BIm72|rwffKP!z2b$3kC;(;R
z5rZH!E(C3BIwN--g5*P_!u#4Nk&lY=0U(QrWKV&CGSb`hGsgVdYK?*e&m3IY!z^g$
z^U4`H?2nZkxRi(??tLz9-UI+ubfh5jvG8Q<?pAm=%k5I2ybi0sytP!miM9%CORC6x
zvW2T<0+|g7LuvvNnB@CF+xb2L`!GB?Eyn#7ma#B|J(m5C(EVPa)SQXWNBOIP9@0%i
z$L80Us{zmbV?Z7Wt*T3ENdQX%W7+$#Dj&#Ovbccda|fh2A>XXs&G$UpxF_8Xp=#Or
z<{&v!u2a7uPTBfte}fn~6be9)%mkE|*c)qoS0r%g6zKST{BDrHwjN0Da=%a*C-~;m
zc8Z>Tacvopm1W`!p07^4$a`{Glmt^Pq7e_VPq5?0h-usIFUW(t<GE1c_}tJu02xAX
z9QzW0DiAD^rFzGH+=C@b<$s%RX;6zdx7Gm!_qX+XY>%WscqZ#SOL6iWs$>H>HaCJU
zU#=KLxJ{NZCdr_*vvbpVCM0K?=^LRFy?sUDf4e>_MWBQN$hCEL;TS%IN4jxCgFZ$b
z0L`h|L_m5~8#m9FJWL&G!g(Hy<OzVQT=(WngdiEX<e7V*eTIk&s|lY;qwHq}4&cJh
zkn=U@@!W>axcHL##U3xU<TDH<M>ZK*P5^$2OgFpPFjK7&F#ijKyUJVQWrc+0y@{u1
zri}6b3|*lhNXZQq&p?+NF?yfUZS@(5;D*H5D=+8ut`Xi}A3J2%C|h}xQ6O1UPN#i?
zy<;75i~cgGr3_qbu;^-z?fAI(YqobfUkZ6}j~a6a-W@ks?D>JqN*S^L3jz8gfN-L5
z)<suzbRkH|bA#<pE;M1tyDuJ?1@fNdM;4JUA|pAH45Vz<s^yDem+D7v#tHU3E5P9s
zo)C&7h`|P0?vNI!BIva?%lx*J&bDN8Dprxw$f!RKwEKH`fOnSo10DV(3W)OOxdart
zJ_jj;7=Ws8h$qW{0M{N~-GpjY8MpqLxh+?)7(85R3qO*?QBuY}D~uN{pX%xD2zY*~
z{rLhtF?yL#AJ!W;-`RFHK&I<IbB_drw0|TD)^|g-RgYcc26B<S!Gy(Wqry)HdJS)F
z0=wcDgTqHMNaP%QdCH2euxhNyFBEvRk4Mk!yUk(pkgmN^O6uCRL*88=x^uA%n2kJ!
z-na!17W#+#y5LM~;p=~vx>7*DKlT6aMd?hPzQE#}@BYr<8vH`L(#JsRffsR(i5iVy
zii;Br<hAMff^Y#j3nr2b{2Ki(hVUDQrL67gS~ny{^3<l$a#t3GUM=TCid;J;O*y|m
zK4|QW#iWftQ~(m0W^OT-AV2_W^o|Mcp|Ark;>a_fWW6pn$u#4k9?9m|>35XOGm*CG
z4_OhV=g5Ad<}JpF$B(JA9DFzE95;3K&t{B?e^7xP7YCHCr7W6kXg}t<Def2wSC|z+
z>CG5AY>{MAIlhF_-3sk+=#N*J9$I67sJ|_ISE&+06)mS4X1386^|gP;g9<n}-j;IG
z)C1|Ba`iD#gzYU5`5?7?+;x;kLyz{c;`**?Vvj#l(uUkD{UUDUt>mRqoMX?Hm;m4*
zD}3|SBmz*A*_-Fj70S*I;F1U`fBtCzK?s|EY~~yI(DP+N%)KaYTfTOih-GBa1`0U=
z@2aY{V38~oGgWS7>T5<N)iCPWEKNFe|M6GAx%GV$L-Lz8J!J$CmrUgvJ@~UzJdjEz
z%xusDCgn6w$u7z6)ePimGOm$(IJg?SX1x4NBMDa@s=%BBN7G{!2Iq!Fk1^K!aJ}fH
z;nlwJg}Ktrl@{zQ#$#88XrQrwD}yq9I9lav0=(bi-gY_Di5oD2HD=Ohgc!l1H61`A
zEkmr5BqGcoP<k-%_Wrfe73$0>QT5QUp<dyh6eVez6bYCe?MYP|?)e?x0hZHca(suN
zR6Sq{kxW?LvtPThB=dB!45Psuq|ptSp_14JKHayfkW(*_1ra@qcBR1+ALy1SniIwp
zwz21Wm_iD%h{Yuc2Wy7}*C6O9S)E2QaA0z)QLn~C^{+ncD2GYE`J24Bf%1bhEXGLA
zY~xfBrM9@I=myxqnR``gu#wZDujvBAZwc!{N3){UHs5CLDeVFjTR0cOb)Yb6!i=L8
zHDLJ1jF}owc>p+te0%l|Kr0Tddi5C>blaEvdBJWrWCg%!9@Mn#!x>LqZ1Zws#%wDG
zwGR#Dt?1xsLKjfir<neQ<*8)yYM`f0;5F0`#x<4`o%K!)a+^6;7hga8lY6K7^z77r
zrTgy?`<K(IW9QNW?@1vxzNfAX(%411PP2Z@?(~G%X%~VnN!D$z!(cK_ZWKz;FcH0<
z<!_;X7ib*Z(THDVw=5*GU>nhF2QvQrap$rv#EYA~zl@8rM+m$`J8j~Ce^qWrc#0&<
zg1^BOl;T%3Ke5}?so@TcA7(lI$$N8?u&E(URtxRHYXgx3m^lguHkemJ?h|;@g#->;
z@cHZ_l0`Z-CL+uH)NJWkfthrDXoF^Pu7J|+^5R?Zw0uD!dzCNu1!>+W-qP_sq%S_D
z_CUr<P=(t&ee@Q9X!_KQO(k4RV{GoSd^Kn4#8aJS<Qnysi&SdM62PX?fWFcclZqNG
z2)1+nYFZh3vfA1C9%Eg+*UqNeB$dcr<g=Mr^&r))g+C&C4f~`P+gW{Fh8wVd>ua&*
z0^xPTfXrU3MJ5wO7mo_gXWVvX9dNq^cxON|paa=uly3qIsq#`s&SGmkj5|h^1YM#J
z{hUGqJ?npdzP!#5-A-`pGX2Lwte=0Xo%iE{w&48NZ7YxF1V8q=(d%|eVWz2;93H^{
zz^D^qnIi0eHFZw37?EZ)-7KN?rcSG$y=B0JyCfG;0&^&69h>bE*fjN6xwd^YA%K1v
z9cv9&_Prk0uUQiJ^*jz;KA!cb_ys_t<ReZUsY#<uL}s>S9v<a-#!6dtiEdU|14jBF
zOEZhO{4pTUjinX0xrO7OhJXRwnbOE6+K<4WTRW-j)==4aN{XF@2xN4;8~_|}j@R)5
zZV(TcctNli2o@6c5e3|4GZEN?sJv6a;N(PyjZ7+8>Cr_)cS)BWYz(*!0)?!tOcKOO
z!bg}yk3ynAPN+OJEVYBkTzKC5s_^YrZB66;qQB1GfGu}|43OyYE|0R6&1F$eKmP+x
z4q>d4QbuN>M9ukplx);&?`Nuqu_pV45xut#kPoVS>1n|7=x+3$@HtWAz8l$c?5hHZ
zp9qiMOB-1p5JaC~pE^=6v3a_OYelj|D9i!I*sN0P&r|{N;>+3dxYKkcjCk>-;o{R$
zj0cD&1>uJZ)}~im(md?<H*ng$3<v8(SI)bPZ>^suI_(0fob)gX)+nCrB?TPGM^bc>
zOZVMQPy8SJ;Wq97NVO}cBO11YsuY*aNOx=~5OjsM=0TXuxVH)dB14u|IdtQ58K+Wr
z%KNmR2SV&GdlS_~KKqe(HYUh-LEn`A4Pnfm6|;|DHIb)VYlKVicvuM~Mciv+H1wrO
z8vuL>MZS02X@q#EaHca&gKe@=O-~~5MiHw#>K07)DiZ(hO1YA?ns367chinuu2TQ;
z9$hdJLY0^$3@2d-`mgJ+_1}r{gFlop?Qs3qmCS1i117L&DfoR5m<F(m&;*`3*uNO4
zM$`UtQIZintiH*kfES1@{QPiHAgm|tRf<B-MgmwkIDxyW2?kGIZ{4r#(KMc53O+*H
z0%k1is6|1*?#D>T0KXVJZZiNg$aq6<+kXs1q%T)C;W89}iiE)27yFix&=}bZ-TyTD
e|9}w>R25{o;{|n0-+&MbI0YG1>1s){;Qs|A2C2>f

diff --git a/docs/figures/architecture_1_math.png b/docs/figures/architecture_1_math.png
deleted file mode 100644
index bf81cffd799e1c9d2230e3f5966694bfc01e6478..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 47788
zcmeFYbyOVRw=dXOun-8EKnMf~ZVB%0?hYZiyL*Bp5G*(ZcWWGi6B3-p8;1_=G;WQ&
z;`_Vr-rt*fZ)X0TwQjG~R8dvu)Y<!N{p{UeRh4CMut>2$AP|n6tfV>!gbtj7A~Bu-
ze=9SL*+3wyRT~KjRXGU>N>x`UOB;I&5J>iGQVO~z!5nEYT+vq+BMuLVA(uytAE_X8
zOg!FL>Mc!3DE>!r)yT{qB10lc$?8w#R9Ikea0Ob;2dNrUlQ=0cJrZngP0wJN*(I-K
zzpXpB!$tV*HQdDV5OR$Ua%I<{8VJ?`^%GhtP~Wz{bUDU+YA6yM%$kD&vTqhl3nd~Z
z{>Gk~{=f={Q%!$atR=lXhj%{csI6g}3xaYaLldsw8`E|>^n3Rt$kri=l1rb3N7cPu
z^CvS+CzQim32{-*TxVPRA^6KW2{WSvgS%TA^jFRh%t-|Lj+1A#ex&|fq<}rt;g5g(
zixnaazoSq?;ci(&b|wxEyB`sF+!Fh5L2))190?Dr9Oq8=2}JkJi<)RE(_C-7DdH8s
z|AO3!P(U7%ho4?+J|!@Z3!MnYFqd8?b7vD`lKr*1-kht!ys9#*Qmy(LPpC)mrGRPV
zwP7}8&-}OMUm8ECPS)se&E4{fbSY1xe|)sj4yDqLOoXI=1iSNt1%>o;Nw9da9(=k>
zW*J7F_Y7x#)blkTtH&|OO&O#nyvF`>zdY%3N7!IBc&}ddC0{p#<r|jj)JW!t38keH
zBC&_Tbq(9!zt|y<=;FJ*lN7U@bZWP8+)CBhkSi6DU0pncubo@*`03=&5L1n({=sY*
z(yULuJn`&$Ig3n-flg^qP2<=0!ZVphhf$u$l0&!uD*-$2$E<Va@XY8;7SP(3m#acs
zbSiHO-rZC^s~v7_5{%la$X!3h@R_LFa}X(Rk%zPZD2EVvY-%Jy!Hw2q`<Uzr9unGE
zFzw012aMrhqVL%R&M3yRy&;r@k|EyJprc?sb7ThHr&wL`s;Kf^($z0DLo#!S_t7J|
zo{v3M4(9la{1e$RSm5uQZKS)uZ(lrt|Ds0~<Nfkf?TwNw?)z}&Hv(TUeMCD{*$I()
zB;-^XX~Nf4k?KhqLk4pgt$8%j?@-AjMnqF`Sht0zuyMqlaz@#)Xn%3)qH~8>{Z>px
zL;t0A<j+fFjJowJ?8x^gJ~#xoTj}WK6_!8xVTegLHGQzU__*2)Jrf~8k91fbzA6Jf
zgYC<*7g`}@Vq7`;YUCvbC9kLOb_qzMit=7sGu4yvgUF*db4hYja%XbO#}MO6+iu$i
z+s0??H)P@8#(Q-?8oqm__==XO?uABZY1e+wtKUER7Jr?YEt}Qo@=p`)eZ%SMHJhxy
zp7oeLn3c9CYs6lNApf=EoX)H8Lx7dAug797(%HZT-KE%t$K~#4toPft*UQh~5h`6#
z$K^*vS14B+R~c7ZS8AWIex?4E$9Q{$*&K=!Dio^S?IA8=nph&w`L=~1|BKC^q(AZ>
zQH}+VpC4C#|NcGXdzAXm@3J;w>V*@gg;DbD^4YSsG}!nViL&1lrV}pW=VdSDoCeHm
zG3TOwh>p-!(apq<e$S2*Vbn;fOyV2#-`E(04k``uB$=pT5Sn2{)dqi;Db0CRXjuHT
z6i-u8bC%IC$$}Be*r(N^C9PRn)>xKXmR7c>iCbc<POkd9G_B;i>==p#wS|t9ovW#7
zN-$;&=7Wn4bF+#sCaXC$NF$P^nR3bVJSM`S&?)jMv#FF4o$_o=cy9hSt@|^f&mAFw
zp@Av)KH(!zr=R}%*1$n(x%lH)%&aIpC0sOLH(zZnrq@a-yr{~u#j@10ZqtmVJ~>SL
z=k&))a{aA_aLWnH7RWk;Im^d<6U9MyL0mvwz@$|hQFieq1{~8FBWYDWqL=x`s0Ezj
znC{qo$gm_PRF)NyC0k_(uV2Ov=l|T(|3UNp$h+|KhI5p&jhohUfpZxgMVvew?U$US
zg4{Hu?{PGq4&f}|L`0u`6aR+wZ8!S$3xaQHz2a*uz2v<Vii%1fm8MfUGoGf}rn)Mg
zD()#!q{Tx7A;f9W)Ftk|bPmWQq$vYd9nA8IC6pLSJi}tDK&KGD;W-4>OVwko`%`CY
z>0uCGp{vcOeXiX;?Y{S6Piq=7U1doIQBF1DZ#BE%A(P2?njyrE>j1S!anPA-obRZq
zt6rPaT3BkNTR_aQH96G;9emiKbe>&t9*y69vHG*OX6d**p)O&n^q0-pTtDA{Plg*$
zLhn;Eu$i&@ub<wn%wBveCj-y{u7&e2dlZh^jysO%i%&LFN1hMQaTJd(b!F~#EVTE1
zUWs3?8Ora|fF<ndtzAzIjYYv5X7>1oekAqs_xo-*tg)O6Z{CF*3>ptQo{C*xAqWxf
zH)Yq-2<>a(V~!(*^YTlw3$L@$ODF6uR8+LYC+=?Uyn@Q)R=i_;&NhFWDb-cYRkg+@
zKSX`7akj&Fh5w?KVSU_it$DQ>l1rIL(86%$l4>)XS8dW=vtqgeT^Xgar^ce<rz(xd
zrpA@*h&_p!$2(`AXV!nc^jhMp+_%!GYrH)I1L6e&9sx(2#U$@aZ#91fWY#ydrcT}c
zf-s+qt#dklTH?3MnNIvc1Gfs;^~C1Pyv=d7aji$z__Pe%JE9XLj`U680^f(_mXigC
z%}yJSj8EFHHg3g2T5}YS1bc)EnJp-xLLM5sIb#~SZ(iZoNNMHG*dW;mObQo76u}iJ
zhQ2_ohyPg7TD&GH|2o;TRqI}tZ0;e){8s9FmJ~ROn~gO2iQMHN<($8s0ymCLB6tw8
z4jb@u6Lc+e6K0Lk^LPKci+}vRyqUP!&MkZEdgCrNc4TX#iDa9kf_d9$P`}0RWj(Wx
z9>VqaM%WNS=c;P9^ZU>D0UHfk+~tGX?FQtnEe)#0&Qhi@uX`Ajgn>jZs%7S9<=nN<
zjo<iXrrp7eqY<Jq)x^K~Pf<bM`Vjwe%aF+Iz3TPv*_*`kOtK%``?Ft`=$E~lrs~{-
z(ropN^<YJDN=h5wPF7F6GDsXdG*?t>+^}7u{1sFzoGpg5^DD`k0!#6=mr5_dj*CU@
z4hN063)c%yhez|{)f!e-10$oYlC6ga7qBHwGj$EMs7gp_KqH-{N?+_S#omSO0m~f1
zBiq|#^#04;*6xqJ-DlPmv#Pi=Lwo4Bw6EJ-`nOaSK5P9xl=d|nFm=`qyy{Vr6Il|%
zcQ3f3Kehfn+AHEDAT6Z5h|0CWjl+YT;itc9&=Q0G+R4eoPUs@)oDHG8on0ba8tD3)
zVYRed!>ssJIksMIK76DgYsb!H_WsCzac*I&d(<Nfmp|zE6D+Kbydq|jWQi^7=eqCC
zmBr@9CX=#GI+)*B$P;orU!z<+QnWWRW8nC6(fS~L(qp%13&PRl?`ZTB(j2fcvM6BW
zl@DLuQ{F#p?(5`n7FoOBL+8VlrT}|~`e@$jZn*Ri9MExfra#Qju1&es&J<tnPWBh$
zBD@3Gf)eg73OkQpUhDO^wQqT#c%slfxqIT&mU@}@P`JLatvU`b>vZz7xY*qiX}kZ)
z|B7evLI1{gAJWp4?sVDi<~<{f&wa8Uu!$JQ?V;cjIehqYJ9W%=ae6x5W8!SW*qI*W
za7TQ0Fs{2fn;Ocrp<ItlZ{Eeg0BW0yIO{tGsXBo~fAGwis9<njE3u3H6*)G?6@H@(
zn({=!OTraB;sP~lzhPln&Hm(15~_yRP8NJ{K(uB;rBKt9aVB*lsh`kiX66I|btNGm
zHMrZ`*L&LU+p&D#mjKPX(jjhK)c7#Ez;71tivJ=;H*r4)^_!1Aq>w)FeLjBrs!L=`
z1Suhed@)Zrl;|<^wa}HbR8j&l0_PYYR3uUm8gPaLe1(z7{_|WKi5`UV&vj%FD8dGW
z`XA3I1INct9PoV{^RFYy_waw>YJA{|cMkIZd>S1D1hoHiWF&A8B&H!DCkGrg%v>!j
z9Nny)+^sD1+<^<2&a!%LAQ17}$1jqcI?WL<|BQ{MuDh<1BEOlF1FNaI(?<(dZwKec
zc|d~R{J^P$g}W)Gw}ZW-8^5;@)jv=01Lu#I*{CT0dBojLh)P#Ul~TgV)q;|Xm6Mg7
zN*Ifhl2Xvs+>&2iQu;rJ1OEw8S-ZPC^Ruyed3mvVaj-hMTCu(3<Ktsvf6w;*Jqz#z
zi<^(5yQw#eqZ{?VCi$Q9NLsj=x!O3p+c-H=KF(|U(aFPIh>Gg*MgRHv*E%h{ZT{;`
zj&A=!7C<1|;~lnltn6(6IX5s=@bN0Ys*SgWy{@E<1Hd!j9m4N;_}&ZtGvNPr>%U(4
zuc2CQ7OoOb4!}rv;s4_Le+>TjoB!8=|GZQ0-`>f=`|q#(w>$qaQjqPD^nYW;zxe#m
zRe;gLSb}W-xoN^!nX^}Fz&^gRkyOzHj)0W?^N|An(gWYeBk+BnLbQDcNX~mRIY}{1
zZ>0T9^q=HX*AeK1s5`mg#s%nc#)Y!jN|7Jvk*S2-F}}P}+j&xeCP7J)D2u~&_SzRy
z_8lbz52M{yT*mp~G()6rbU#bT>hgyhTxjbsV9V!l$+f6MpjAKs#slL~GXMNW6oiZy
z3__&@Az}O<PYWs8age*z4P{7?{vV$Dk7;B;NR;GA|L>E&r3waBOOhlM{U2ZZU+Y4m
z4EX=B%>SP8zc}{4=l_56@&7F3|1CZLEj|A`L-_wkoe>J|A~EqdrB(i6ofUNNb2_D=
zl2)7@IREtJ`#%*my`(}p=bX>~F%gk?-Vb+|5T>fOk68g1dY9nT_ZE@O$Gv1xG;g2%
z;}z=PAqkC|1e|M~F1n32Zy`c435Fp&F#F9^vlwnLm)2(xYWiEil<R#ei1;@@BAP+K
zd%d0j`TOg$5~0-B^p5js1IrLJV%Z;S|1#yiv>|Vkb-UXx=Iy@rULGvWECwPDzPi2o
z#~p}XQjJxah;UdAr%D!XW%)T$7slEYrO~=%p<syS7^32VlJ|TkNdH}%HSw+vTj7fZ
zhjMh8=F+jR4+2emcM2g2Z0(xQQwaW%?J*8a|C^m+$E2%^1t&OdnAFPIqzVtplc(Eh
zqSjmhwISc}u5R|KAVX}O_Rj>5L{GQ#V_UT4=)Iv@I!Og#oG8e6CDI_I4_xnm{4cZ)
zU_2$|?6>C15r4S9gH)UTmY)_+NBPHBLh!ejTAO^}?U#n!?@8KE2AOy=-tN44bhkM;
z820Vw)0@)<P99t@-+c_7^MVB|<Y;RG-g8?K83QDc!0IFMZ;me~8l$59Vt(_ukHU+q
z<*;RM_?f`|m~=xv<G;wWMP_R|`m{4y99!P|!)?Sy`1VL*_;<rio~rO8anC4^4w_);
z#_hf@D=)O;oQ`@(OxjRfX|aK&WMWc+?6=A#mi|2u7!yM+QqHvHOe>Fluqn=*R@$}2
z5B`rp;RrSS8M!U+es-^)YcT@pcHrjThd;#s2vy5Fq_lfOuO$y7Y+@sKH4dAJ0>?$t
zvS&ybgpZO!&#)@??^KZy{M<-380KEEU(>JNEh|qi&hjlIYYH#>hjvjbyjjQAgW=g4
zGo$I>5x6|)ubf4#2qm)qMRO18I=27O-(TRV_RHmkZ?E`jqwGXo@Pb8|FaV0sJO)Gm
zT_{Nicvrtr56@X;AIJ;cZ0AEB$?D<C;r&OFtgVn?0hgHu48j(JggTHElfXIVhAH&}
zLyO%f|7<22rhWam^yWPR?p4FqdFxuY2W}-+l;Z>#C_o?zn$4_w?eV_|h8Qvv_-9@k
zyXmK>fIf1sR6udsKFH_&L%ygAC41nlcim>XT}u&b{g}Ar$@+j$?p*z&W{U1$`XBd@
zj6A?*Oo9&PZ<hVwX+B$-O=ky1A$Y;T?=V0+XuwmSf@x7Hf^LuhG)q5wRLEH`N$MTB
zN6E|K!03nnGV)r2x=(YA@aFTK^ixCJCzWR0jm<$JV9Fzah>YcSzYO}wVL%I<?(g71
z859p!eMV}W7Jv@2&`@f0zJFM-A9Is{x4*PZ&7G@|h4rE;3!Q6nIR6#Z`7Lz0IBDoQ
z@Ycc5pzKM=^mFf12oI!0EuX&t8FA2*ofdR|EkMKeK17rY8IbFyIr}=u;wYe5Z9ezc
zTjf$4f~b_L?*S=S@D#Io2G|hFh^}->CvN7Bt5wU@KVKo1626bSLN1?U(isRCf_F+=
ze5b=487jips7N0sX$Er2F$_JRT9zqBo-RF=o%c7JA?UBS%kmh|k@2kGAYETIZ|6m&
zgF{e*ec=$!NkyeCsV4^t9}<+f-(4>IRh9qU9?ypWHe<OXsn8M_5#lz&*Y_&$YK2@K
zg~7I;hXV;YQ||AqSuB@C_){0DSH9c1(gCS}{GUAqmhIQ;QIv#7FZp42s&r)QcM~jd
zuW`YtvwaR=GUWXbvzcXffTtTkDw$2#jq{Y6Oqy8?U$6i5MvC$*kNqsW+uiw0YxX59
zD2N2sXMz2=Wl+@n%g%>1pUt#B|J&m}Y2>zp#)T0Zv5TF_zBGwuz=A@=(8&pPG=!gz
z@b58Gb1*oy9fvnxtwycMO;t9|mq}=A{t2r=2~no4hIb&+<Edp@)MG^My-y=VZgUe9
zX#9YekYg~n9d%1HY1OZKEjToJz=hIK4x!~KK1aXMqqTjmb6_&xUYK@VZ<1Nv9M=x9
zx~=(CzFPk98E5zohQK*Fo=cAE(lH6JFz=Bv2m`3j(30C|`BPtng8697q+!n{l?5v2
zT!&be>a;f9w$klbPqQCx>E4VP0I~x4jVofmZg!gKwd?}JJNeC=@C{&k@%7)&xQ4>y
zx<a*|MA1T^Iwni9vlZ-~`?{x5CbxN1&*%W_<%%x~(jD28FQ**CL&8DkeABUCJCqJ^
ze2KZXpX#eWr|8e?&5SiST%of#J*W2bRkMS$N3Q~C4WMD*cLIk$%~pAyJ=(YW?_IEm
zd#}2A+wwFtV%;vGHEmtfFiDS_OSn@wF2(EYR=`X{Mg-umDD3*G3gSLg=hp`kpgDs=
zl&Nx(t`lgLT$}Ci<(cj=0Tpb<Y`Z-fQjS*{WJ@oZ7GQs>zAkWgHo2xjV(gs@7@7cJ
zA+J1VO)a;_ayS1FsBnIMf+K(1E7pv$2&jeSS~QPm^mpX#D7Fq=@DsqLx#NXA0Ais0
zRXlQ1J@IOTEhw#DZ>t+x7xBvVi(gE5yt%OE=t8FFe5pi5La0w2ZU`q}ZBv}}v1&xF
ze&H@g-s@Rqa*;=y0efBW=qS&+Czvrf#s304_GB_N_Sqof5qrIvD+aEg6uvvt@bYb(
zv*e%%IE%kt!3}yL%@K6BXR;*uD}o|$`$^AN3ckmnXiOYA=>9Yt;?hTMl@)knQ#T>c
zq%|cu-~4bh)4Awlb1F}!GuqC+=)D##U7o+eA=W{Ffw%n(mF6k|q$*Z!!i5<Z7~~b<
zsVW1kN*I|m#cow^^KQo2?;!FI5PQ`6T+CUmohHz!aAUL7tNK@D`T>}i9%_87bEV^(
z#FXG(+2S9rk#;%ER3#61fH24->kCzCjguHUhr~a0xQ`9mrj4D81?pM|(LB8wN%dOx
zSt_&Bv!MjcAm*a;Zyd`Aa!+TeZ6mN3|D$u|1Nd_B!%j)zO`L*gde(%o01&D$t=cwZ
z`#-tP`Ssgl#%K$7tx|imUvc^Bcx}yX|1m1DMn^ue%ZCP~82j2td59x@4Ky^1<{BnY
z5mbAwQl4Y6s=Mh2P|b?kmfXC{#G$2+DGBgyP(?IZs9-0X%W`J@)c+J>WxlLo+Psm>
z0r+Zhv_zM+-ex%ohI<A!aUV=s*?qdOjcG9~6;yyv--CxshF0SVk?XRePDe#LeJ+>0
zjJhfAwq+a}l7`tM#8@%wsT5$8QLHW2R|E*WZD(ZlCnxgcDCp0If1K_ei4WibizWkX
z)xxH_>T`_xtS%H>!4KMc7DOCF%ypWY(w_~}?vHcR8#A=0+aB%_mQgH?M{0B8kF;VV
zt1If-*YpenQJ;|-#kJt~52F#kKBH$4DD(cJ=v@Fz08|h#$D2BxH(LHN<QVk=NY{0V
zat2|5xB30bXFa}ID>(>8m+e21hBn$_<;p>c49Jly^80|wuiJp*(V5zaWCF9d9yA*G
z-+!TwVYkZivGm{fG7h@)IPIs7rCGg!%_c{5B{FFIfTCQ}ClH}x`fuQ}v-Nit+!yfn
z44w$;-x?Dm*S?@S%~sD*M#n^x;6Zs`UY<0yb9B9xEdU#SU#A!>YDEzDJ(g$BABe!x
z0YkLaEH(JCU*!H6*GkyGD9v&(`X62gJjOpTJB<>{_jOkQ$uF<H#@3bN#e8$WeVt-Q
z;&Ywu8wK1X%=z^}s&i$-w0uRl51_ik-Pml4F<OPn<cfd6t%s55QG@#BGUkjm)Sq7!
zjtV38$<r*PjE&T!rE-64bEB@;aT>nK9T@SL)+=6^%8nGzAw{Y@9AEEDbR#rBWE(;S
ztFXZ+BVPNKjPK&6;RNj0^bMQbpO8?mfB><j#|?qqZt{F3IzCxxpR>$JDh{bHU`4KV
zrVN5(oe(jg#p$m!%os=AWy$H%n}*Xy0_yQgf8YuSxsZ-hDl$KYpV6`Sw&K|fc1mGO
zbYvqX(PTSwG#S(ImzK_8P<<uw`i0EtP%TzY4^p~Q$4*hk3u;Z2-fQ*eaR{b!pYsPG
z055XvfqQ0pNrGmhTXr^5OdJbB7+_Bc=#)7D&Wf1TA=~z9s0dkT8+kw-r}9p;&KnmT
z-C}+`nv{%3zrcwzg`&90Q1HC08nE6i_d3Z>1_{%JKc03^D6lP2qYIt56>UZ1PN)ka
z_UrCUH5RHmZU%1wbLh26Ppg<3S1L89i3Fodt_=leYyXS^qP;g9OcRONe46DcsA~(3
ztqu*@e6VoTGy6#_IvYMl*%#5Jb=+;Lnx$`$b7@<aH{K(PW7BNK0<-I>w6&3d=yq58
zOo!vE0SoR35N#f)h<-{3eA<GRd+$ptciLTBC5*<Ps1j2S)Mp~M&VZ}h`pF!4*$R7k
ztpp}6P_4b$Ef*lDbtTA6Dp1dxXhkuxgqFB;6ywFcEdcxtwIjI01#z}RKv9Ph0;YAI
zvq(u>@`5c988}FlRV8#p@yuIOp8}NgmGcB47GR71(f&q*STB%}EvP70u7JJVGiUi;
z*FT@2?X7@OqAK}@aee&xX^d$gd5crSC(NbLu8O)5Ms%FCAZ;e<TJrjg`T~DwJ?Z=S
zyIGf+-ud9zeQfUWen1YH^{sQms03EDw?{ji*UqNtOmaDeodonqiq8e6gCrbZYcoM)
zs%GbLTO>Q{D83N6wCrKptq#|edOC$|UC!H89nKGj?>P`X!=Eu0tO$4heI15bY3X7f
zHozLxn#W$t20Pk%w$x-9O(g)7^3nlSrYdJw^Sb946?8+3st&peC>JvETDFj_UX5hh
z6t=o`lb=~YXHsSTp}bRxH%%AaG+iv__*wwo=JraY-7MhlEx6JMCpD(~9cl{;QhD|B
zODDUQMoWPT9|JEoJ&{h<EG{RaOxHn%crI#P4ssFk96J2!>36O$q5F0#F1VSb_Hn5*
zf8i);oi37+<7-bY9YagsK3lH7QM722n~Q^tm~Izb>JMmyF4@)7BDRgjYpmyF<+r=5
zdPp*y=KXQ9us^66oC>0+jQ4YT=5|RJhVj!CE=D`7KyVsaX8W41N^!5MiYimYG|7N^
zZ;!1brQ51A_BS!0R9)b!SVTeZc|&($it+BMzRnBQbRuiXX(Cl~mSCuSuM^&OGO8os
z0IrsgPJLFUTOcLnM&$3{x;h~qlF5@!;yiCY=+H`PQ<3Z8e3KO~s@Bj?@Lwt*+9)qV
zrJ25HZGAoc%gtQLQj1d^<yWpmgtXs&jST0K`{Y}&kvDnJ&1AMT5Me}$6eW%WDHC;T
zI=eoQiIo>6fV=?{7*nAEQxvP-bqC_hzVoL;Q}^LyjxY2MAd$WZf@wDGJp1zT&7j>2
z1G`$Taqi=|oW69yFZ=Q}{rs0P5Bp_b9bwD9WRy=#gKCN#s4?p^aPnnhV=77wQ<`qt
zp)`)%LyKqAoN5IF>_O3|v^*0pcZIPKzr#q3Uu==Q-z`P%0{ocBL><-bHcqp3P-$Uu
zWDMVC>h^AVRm`z|_KWy)xlLM%$>X~kqj)^MgA>_LPjSj7#AehfI6R5`^Ez+N_)Qd?
zD`QD1bPUe@kWmN4KV&#Genm7v7d;@g_vv}exjz$Q+;Oe<#Q@K1OYnlrR&%N|D2T`L
zDeTQk2Uk9s+`x#w>sO9ez}r#VQL)bZ?o}>Hy1jtG?b@IK;6MUTP!aw*d6S7aY-Ouu
zs?Xz^G34mz4A6I|OoPc{`9r&b;3`SB_>w7i3;&UytM@r@y#NXZ>xvBG@ATl+VIG)&
zyb;5MTkiOYGIh>7#JCIvx!&o9OXt&Ur}?I2fjqzkv-88-PV=fd@8_XC={eC`hEp<=
zHtx(T%bOWauQPpjOXH0;)W)CK(Fz3$TNO4X8!ukUN_E3kbfeyVb$v7x+EIf(8165T
zSfr2c^dhUUaD}fvxC72YduoMF<oat=R96);4xPNL=pY#1)gU1|-oxuTjP5?Llq*6L
zN2cyH272OfA&A^%iP=cs&ZUyPojMKnj<TJoi-O7%KD<x*FnTl6k46RId|=0H5;&;W
zaMa%`X`-s6#W<^G#6Nr2^ao#o<|MAs(s*n=tg4)L^-jjrxaIK3FBzmtfgk!nmjk7`
zpSL3eD?22{)3%=-7v5ILjr6CdI&hY^Ozxnut$+uog~*iqp_y}+n8bSK@u>M(jG(jI
zvv<!gUUojzW#2Jti2W*_)RxB=4*VH#wG#2$TI<py;4xy9e64Sq$|Kfrf3rt7A<HWs
zEcy~-I}3wFR=1AUF;DSwjkR_EyLDdZUOIb*=e$kpS@Vw)RCK)Ua8&bgN`I^C06VI8
zS*hUZ3Y2HOlpGFfZ{P0KamWWz=yDjvAKvMU4d!=b%>#NK)sS~;Fn}2fD<-qA4za7e
zY2d;kTMD>b`Z|qE#yxzV*+#VSRq~Z+4id6Dp3<n_jt>{Gt(%OoyMDs{z1IPIlRur6
zDn!;LU*9|%G1y=XiDXk)P~E9<{v{bIijMESk)*RK%(q})*L)mu$bwln=@zog7_<2R
zgpf9<;FjI8E93D~1MI5W#L+0{oo_yUR2$QgpF!2cncQ~=l}*d??(!ariX0wyJ>NZN
z0k~%e92WZ0rY;+5OQTWgHvCSehihtZq>Ykg*jcpnPM?;(_T<zOFKIgCYA|wZRE~p{
zS%cKg0!t5B^sxA$v)k9^d>6{(a#!fEta!CX@$h1+Y4&53)CASxE+vba9by?OaS1;6
zn$RTT94B6;7BT|Srv_@d_e-TW`m6wWno6~%ZzW+R-FEZlP!c=46TT+J$;zPp-f3;t
z>KE*gMq0eUGdp~xZf6Fuewri`Et*LO{Kahd3Aw{%H9_({72>BDKsF_e#K3v&6$^Xz
zT9B6~_q;(bUU_HF<={g=Y@bg>+sR<F4cD`DAP`yQR5zzOvq!ES`HQI3t{B{n3uVR+
zrY{-2y8NUFV+5bmZH(AJiGU1Vijvc48&PuJkY(_rmlu#G$oo=577jhr6Pt6T3N0$1
zH~u8nC$q5)Clkj4sQgh~sqf5I1DS_RYB$aA(xud&oXOY5iroM7%R(TOuoQhaMJ{H|
zGPPJv!bBII$2F@w49{q5bEjjH3&Cd<jS`8GKGdNa#4RtFUyy5b(>(lj6!Y*kk0-qi
zD-V3~?3FGFx=rl$-c}Qd)ohN6)h?H-KdFNSxYqjWYAu$!@~;NJ&9zE8Cj-yU(aLWE
z59n^?`!HstoZCFSJXw@q)m1ho_qn7vxt=vQCdq6~(x+gaOirI$Tir@ehL&#T@*jZt
z;`@b(^kJPgY$8C$lKZq9$dEHX?3~77?o+K~ICZ!XXv@>BfO%ZhnJS0*uUzXZXGZ)g
z+pcktD`|6xf=nN>03>fQvdbA+(NbjNw+53Cp-!N>M}-Kjr$GQ3LuV_VY{t+`JB>+@
zj~Kk^==C|mmx35m-8G)LQ-$s{&&PD}r;Ce!VFDFFE)aRE$iX7cD(R7Gdh62L)W=#y
z;ize@m&faU23wKzi5K=Q^&Kyf1A9v0Uh|F(Q0=hVv*h&ddUbAQ(ndh|es*R9d0_JZ
zr_6fgVu>Q=5GI~A7g94jCLD?7V_e!OkU!68DZxh7Ur(GaUYbBWc6Gt}s!#0Ci3)DP
zR5=LBHl6#U2oX*rcbO3ZZiAPo!4Q0VG2d%3EP)Nr1&5hW`e%z&=egwR^W#G`#tEtD
z<nF4gq`SWIr^0O3rGefDcY#?9SlLeWCG`9Q?ROUobc6PVW4%rJ)J+(m^2tP)EUd3S
ziMCo%i3`8}Blj7Oj=6e8yz8zU^#fmET)CKYeh1EPg!~WvH=rqbc_WGk9m!d3+;ZPF
z+Xj<auX;8PD<82trXedCwMp~bPl{@jnm~>(n+KMHfHxY#U~k4)QN>}YOcmAQoPoDj
z!(~2hFe$?C?>YX`pti&X3zysps4Vy<9rL%8)zZcV!Urlm;wcjhcj~1o)j|pUP8f7)
zte+Amr)+e#$qN5rzzvb12;kI4j{mp_Bqjs{5lZ~l`xPD3Vb7IF2?pgv%Rk?qYTZv7
zyqv(o-VMT+C&biWAr#)LH*x=#?}Yo1NnbclRbE>KD50d!W0(=CnZS#;t%<D8d6GN?
z_+0Qu0KX)%7-WfppF-#Zbj*h_G0gFP0_gqf<u`w<nG=J|__p6F0RYbA@^A@{wj{<3
zCE73X>0=+2zIYDe7OxDm;pAh4*u&m5kn_7+W7|!gMm|$|5%~ZF!UjC3b6L1mxffB2
zG3At<pV$=H2uys$#KrlBwD~JZr%+hgWM^N_FQtUL%poIp?drH^dQ8imX{6oQ&^%&p
z$F>6JB57H6nzPfg*9{Z>?^-)kXi}YM>OX_n+g+761@vD|Tuk=--2PMGC3sjB)kw;@
zfhJVdwxgD|$iT>XW}_KHTR-MHObvlAMF;oBoNN@-h;v0f2XGU$TW026ou92PIUO5R
zamIW)nmhS*^osxtcr3IIGf{uRj9*{@pS(I?X<GcAp?TRjqdi?<G>$2tbcg^N2wK%1
zXq$;>#G9yc{Nz7%>03VAz!D9R`9uWZ0inq;^}BURE9WyNvDrHJ&2R0VNQqm&o+}pG
zNYsd=QPSmLSS^!E3x_2!;+}pDQza<H#<1pU%7W{C5;18*eZSk3&NSu4^$(5-R2h^6
z7anuc5oT5%_?Uj$AqvA5k=kuP<O+eYg2)~B>C3awii!YLlcsSCJpkD*og==-a9+X>
zhxr0Wtx6!W;h2sgMC4Ii>`2R7mz=Gb#19;AWd{kz_ZFu-#I@!80>=g|y7X;Gd3D;2
z?M_;^0w5%@{7fd*8_;lvemZI#-rKf&DUrYlzMBE$>=a}00si`$diqFN2JI;4Q6&#0
zsDdWhLb&luBUt;rvdG=p*T7;sH(>X4h`;l5_?~U^E@k;0rrmHMR4K$dOl+AnL{Kr#
zB$1aNia0|tAMvJnB@jKq*A;{$LW*`|gM#s_Z|gJ;25-X)RN1eeRQ4HPg5zHxTw({s
z<1{eH;ydUl6UKLW-oKQmXAAMf2g&2TkwSa0KQmxRoyJm+i%Zj|fH_%QQ?vE2#s?|0
zHZ8hpB^6IxhtkSya8##HXz`QxJ)%O(pGJGhoED!!bDzDX8*@Y!g+EMmGV#w(e3Tg$
z;9hu(2jW(!j+#V7)WQ=j4CA3avU`lmOgVyn4rJ%i)Gz^4;d!?Ad>yAF0y+g6`jw5P
zMD;&RI;e<r43yFdF>MIqDvgtk3P#R*#XzY_iZlHC`S>vds1EI7OiEUOnDbUylJ86#
z1jb(lzLz+Fax@qxNMnGQ7!=}2E+`I2#;3{IKZr{47edQ+^u^}&mW$T_*(3o9Np=l+
zeVx&yT#Ra2ia(!IjuBv9T?Rb2D9GxM;BKM_@gx8#XYz9$3y-%d6pfyKx{WG;rSJ={
zgc8`&LjTm#77;aNX~6*?$RkfM+7ttiUcL*P*Rk!mH&5=%3=U>J0R3G7^`G%SZVID9
zN=4IHxt}DjzAXV5qTO4!P8+Fa6^*I+gv65oQU}EOUW&mpU<t~oZF|{bpPw{nG?XaF
z0SN|3ReP}8>I(3N0zdqL$Q7cE@#tY&dGCP4C3QQoc2s4X6nO&ml)L8q!<e~SzL<FY
z(MsN3+rY6Z0ZZlukRX@oB{{q1X5roe06H7Em=jJT%WNPDN)3>@1f1j5e0k@<A5H2v
zEMxKb60KoFtgX!B)4pg+01K;1>K*I+ub~suP`7t>M4>k@H5qa(Jy~bVKVP07a|s@G
zp?o&6g%>3;8UwY)dCV&!<hO4re`Oyw<Q3J(M}}B>P$M7Ef(Tp2>^msR?ZQkNa=6W}
zA2jCMy!3bhkG*SjKGu|9mC>j!Pa~Rx5dt_go<o6XdNg8&&%6g>IZLR)dsUr*i<h^j
zdqBOW3QGa@Mz1I-$#cnLrZQbwq~EDM`!pWcVnd6jpCuNUhXg3M#hd>L5naI3b^e8B
zM9~3M5ax@NH4EBM=jA|G09-|r1@jG8`N3QBdnLlmD@hIBW>!a(F8y-$Q!D2zrk7d6
zX*6{H9e}%(E!YMn>sh|j&KQRF*m99#|JHT0KlDNzn^*Lv)vyJbak+jYPh6kY+tAJZ
zT4PB70V>0bu<r`2HsLsA#KjHegx|QxEGx3nyhd)kI6Md3ZvB>HxCU%0M$u=o<PvCK
z3N+ggoA|p-HaVG?IkauQt-4wQe{_EO+GqW_868zLxnTwX<H2T=ta_D4Uf87%%t9Gl
zTHcxUzOXxaD{NWJ9Rh@9v$L@z)FVE>!2KMaM<g<J%0Hr^aj~%n==C+5KC5fKUKQHa
zdu*osdwzqna;Ha{ZB7oeWOh&3gr9j;yd*Y*Fozef1~mB8^bm7QqvzDzQyZByB<vr<
zp`I1hqq^B4z3EHNY8C8bf@fnA&^)h&(-tcamhKNGpk7<8h+@~vN~gDOTbW*@i1EK!
zdM(KacxMoFD>%z(#?UQVtpz~HmS>5Sd-)AUS&BKRN}LRX?#_E{SzD}X&l~<`iTH-!
ze1N<sWkThgMlY;U_KiNrwQgK=RWQ42xTYsEXgA)E<Ua)rcTcC@WLz&8`CDw%zHX)O
zC2O^^x6xf942!Aa!-&YkZKPfC?HO|;_a6v=R!865<VL)<tw2GPNtL#@$oP-dT>y#Y
zMMSfKfT?4J-V$$?N5#R7xPKnD=d#ZhxmiQ<>h}sv6gQyW*w<G%TT?W;c#btR-MY-E
z5?Z`^XBI0o4KxnW04xK7CLS|%#g$TT*bOJUZ<7H5r1$2??J|2g!$)lG>A8j7Gl*2f
zd=D7DUbyCk;f{uMt`g05z6|wNz5KcSoYZf>`u8*GHOXisD7+F^_43C8FgYnW{iIRc
zo5OLD*DMJ_Iua3zx6O!5TC87A?Ba4GTwlJhx~9Fqy|B`}m8C%FP#N~~ox<w+CAw!)
z_IQ((c6IudqkZyuM(^fyeWo95P7B_9bD;Rmtaf7iN3X?A__UjZ7!2)(_*tjdBJS-T
zH)=(v@MS<r^RsrQvpdr;;?1MqxL&aLRzF<AGqA_Gnvkt{vYx>+d`Zh{YI-M0`)mYd
z=-H5>zrwuNaAriKx6Z%WSd3luw&clbKF%xLkCm=5#lgw6ehzbJTAGgTOq|9N|K@=k
zm!#Cw3>4)CdoD$+7pGgjoPzp^kv#~;1d|%hEGqJ1GK$AJ4DT)<%Q9vj0qBc<@b<$@
zu-7$Eu(F!gx2tR!QbUHs?;`ZpkRTM?H(Ayt&iLoD$i5xB%{!kr;g6;FpxHtLJG8;*
z4FBAn%h2VB6hqgqWL7q}`Fv^+5r;YmcW*iti(rhS=koYW)k-C5uo2Nd3gbkre3hCR
z6U&O(tyP#Mn9-~+CVJo8-e34zet?ZUqq252!JqhK)&;qpSx2wbM{NE}!kk=Ezp^*Y
zU_2&~fN0E``9c5sAR37w-W=w(<b3t31Cn=ae@|01v@5^Ku?gUuDk8?e8@VE!FZ8OF
zp2syN`_JyAv$da1#J9`Y8CS(MTuk1N+??$s$g=GOaEN)xr`S-?rs>U#d4-8IvR9H5
zS`|$W0Oe9zRywD`52*vd&XW~u=-nYMaf4VPfgDVrV#PH4Oz+jNYAsGzcKLo&So!!B
z`w>3fr9Wd^f#NRRsz*;q0FspFCrZ=2$r-wKBb&k`Dgm3sZ3188LAG@uYRS@Ds;A&B
zUsp?^HcBI@CBR8z8qgG}v{yn;v%yq~Vax)s0H)Ezh&$1!g(4_;!j+X&7b#Uq{vKuG
zt|Yw??2kXN;?3V70^V@qn)(SbV0bZo2Dl{I#NrO=V9|0+>u)whP{!V0q*XUA!;ip@
z-kT4358P%kN^s;QJKqZ}zpSQo><tM@r5th{(EQinKLMqPvUJ+^-njQeaG2XpW#%6R
z9zNmMe6Hp{SOGTt=s#Qp&<7X(B%Qq_(jv3YI}BY5UY;kHa5N(oJ>%*~B}cH-Da(5k
zqLJ4A`ESebqMxLNY4Evz6{F_~Im)Q|!ZtZlWJ<(BFv@x^0g2Yat~Jc()eCA+<PbUj
zL*SC$mh{PVu7yHEB+Mr{v%7vDSlQpYQQ@d3xUXLgur<Mdr+bb2hrz9YETTT+^rF+w
zU<ysT1xVcm#C`gmXpxYO#BwsVLGxia$L&RkuGmV8Ws{Ro<{|C-<pii;4-;v6nisRa
zbc45%E%I5KRn}J^9ey#Fq?kq_sJ%*NL&f!~WcPQ!z~A%WhK6!AM%rnpY^yk<zD1JO
zqRF&W?fg_s814N!9sTemyckX7jbMYAaD~62&=)8z%gZDmNhQh~SbYQ2tKaKs2yG2c
z(>&3Gq%rAD^!>qCbT6Kso+0ciYNRa#&$AM9hHY{9gFH`cW{84rtl`5sB&kXLbl|Wp
zZFjQtKk*>e%n{o~9%!f%&r&_rDEez}N)8pwE6t9ZGoV{UJ6_qx0_}Qbf>Ot(p+H0h
zBTsN$VGpQem9Y2FQH1$)A6M|#BbqHygO`~r5|Q>JVe|@rpWir-w>JXKBHD%`Mik`^
zvxOyR6;IJveGf~r=ZL<>Ly48A)R4K(vC+=eFi*M>Vg$1I_C=qc6qyQ^i>1Y=pU#5c
zf7@r&ES`FkptV91aiW9}Z3nt;R>fR&!OINq+4Pt~FMcLn05N+Nk<c+2au;gM)rYF~
z3wgsJ7p=}AEZ^80yP}$4*yVyP1$9dh*Ic3ZZi+U6Fcabr-%<prbD7iiZ9@{WGS}O0
zl1~6gD*<Dhy!F=;vOjkdU~7*xv4`6UUJkqcoa2h#sOTrHq)SQ7K=GU_9c_nlYy)*(
z4dgv0hTd~o71XI~_a@C=nSZB%7koQ<a}`*Qc~wDKs3yW!<)Ri3f8KscbF09VZtMDd
zaz-z40r8_@{`n*BbjX1}CObYK3oQ&Li8&6)p5T$?5F4Z$9Mb=5O-nZ6iGqCpGJkLB
z8L}t?3=z2B^q45q%0Vu#d2fm^wutY17(SSz!TN}t0Qf;qjd8>#P<4$iy9vhFp4E<u
zF(uKZpXwGx;xF=@{d$;V^c&GxL48U?T+SR~3VV)g3&0R{@T^IYa(baQJe=R@wuRnU
z2nMlD&rfrV&J6<r(oCTOTCk}_-H($&HE@pnPb&zE^hAMLT(pl8YH^5Nce{4A0w;kD
zO~vH;*HDig%zbvaZ8`Sa7f-{&Pfl|%x7e+TOj2in=9unmri+G1#e;Jj8k`YQd+Se2
zea8zit|Mh)ZKHMNjZoSUCJ@x;F_Tu~b+)S)!uQra<#*XR`AV=^NK^)S(r)-$upzrg
zO5FV1=q|7e30?9L`<3Fa6dl6y#PoR8)ZdqWSjzm%M^}X}h%G@00SF@L)Lz=^=a|WI
zq{n*GgWEc~wtf#Y%xt<XvQ^j@Y>nK=^y$W9zBBtta;)%;wW0sbYk6$Ch6(<Cg85O`
z?{<(kbgOcz+A4SpN5AEmbSF1_7Z%kEjz&G~K20j+`E5MEZXCq6$t~s@N>8D=CrA45
zjjAU9()-3&k0ARiFaMalXV8IrWkGwYa`B~uc3jSMAIpjIl+bn(m4C~bHhjhNOe+l<
zIFb|*%(au2I@VBhO=Z{nGcN=vcbhD31;COvO*wus*~u=Xf<g|(bI8$x_2hl0|5%}^
z+hF-2#dI7ORByqpI-%ZzOx9u5v2N?}GL}+uKPGt;liB;Xax1l8xINHSHynsr&q>WF
zmn+w4J1VU3xAo3DT_7NrdUjal4%ezRF>f9-H%N0KqOKy)0*&okLicAvO|C5YVu-pw
z#+EMkoOwb;1;N(Drz`iD{L`0F9y1BDeMqQf&&T}TczG(WF(6Jw63Mmpb<-_vpG<WS
zGzv}8YgrOGMySExJ3~jEDr=zuTO@%tf{sQ|$>~MM0-v!n!!ng}qID^8j?);alPaQO
z3*@fn)Y3GR*y;6d^?AQ<_RCA^XuX7VJ0S$!)~kQ|pq}SY9j}D*iI}k()*pjdB9S!L
zNureGoC7axacz**Jme$l<%;Tq*}t)!vIKt8KOeX4vmI*Wa>iK53&2t5rz!|D*m31w
z<5d`g+8ekFO{7;g?1G^b(EAFCcu9{6hgH-dzC1^6YYW|$K)r;uK%(eS$%(6h^j9fQ
zocOKZhLCLPk@d!(lLJP{g*mrXJ9y)ct8OAEZ;UwGS{jt9b$+%UtL-dHeF;><T}SaS
zQ?mU}26#*BpX|Nkc}ZvQ+;>R_yQCSz5&86ox_v@9L20LY`27&gjgvk`CHM9F+vytv
zB`b}YnqkTCQN;}=DO~8>j0^#_ur3Tc{)!lLx=Kzez#=eT7D(Te<Or1R=MfIcjbK{x
zJ2)w?95$0b>;^W$l*$#txY7_fgWw?SwVI)KO{nl0<+)p)E1xQd#-0i#1P>=g6h%R^
zp}2LQhMTVax}7S6Rx3)vMjejs%6{iju!eEaQlaw4iwa<YDyy<05gbE*{7TpvCuh|?
zvFPq@+cQ1SbB<%q$JYkOF3U~G4YBK134P#&LuE;Hoekk#!AL0hcM8aHs-|2ON!s78
z0@s@$x~}o!Whe1a0`cpAWwl=ZA{PdbgF*@3TxJp^tlDRrlUEMO-R%_E49fAxaoQ1t
zK8?G6iJ%ZcWc?TTBW}!(=tHLrSLX_%7B<*zlrLt6hgr_`th6uDG=C%h88Q`~a1}K6
z?X05|28&whYxN2!GRDE4@M9YxQ(0k6d*|h?_8|N}Wo_#L_{GV?gV#>T8xTko^!Rg=
zgG|gtsU5?<Sb3UW?f41v;u?}XPSe_icn{{!HHiWAPi^Sma?lQ0%hQ_CCD}(Wv8#P~
zVJGqO$ya<Ejk1MPq3g`+rH6o#=B?$+WyJDjz{A7nRshg5EVyu;xg|JyYLabPtKc5%
zw`exO37LYuW%C&rpc6j+x=d_zGm(-d|8M}iKS+4Ese0H|dhqMK^wZd(I8E(SbOdT%
zO*4*OtqV%`mq8Cr5JYlQn}ut+!IG_O#=@fINjOiNX;Isj5Zp&&W9$%UG;p9@(&;!!
z8jxc19cMj2V|5N0=)7C#gcPiG107|n1J@&79jq){Ku@$U7Yhm8XVlNrru2b|q`7}R
z19ofabR*?>+v{}Ot2uYvUa@wmU~)4dmCnm^U~=1IqWFw&J#N>{aD!3D=J>uZ=(cYF
zDDo{8`mA7^@HQE~{W;<Rp4>o*6@Xojv^1Q>-%vN=`py0>8bmArYZ&nI@AInPc`SQg
zEC=0h1u37C59T~LFYSoo`l{;OM;V_{btY|vcXd7>I=y&z0{r*QDC!8KGVh@#cThzi
zr+_^JI*}t3kul?kHIe(Z9xOvs+g?#2>OO4aqfZ3$duP2o9WIPGJNKJT_q_@ATW2zC
zQGqwGhdY>23$FiP6pc$cL=jJWX=~^HfJc?zQmHG&?O$}2D@Xc=>86hJ+8*WQ;}|c+
zBn6+xP9vG5%e~I~Jq;hHz|)M_!KOh;ldEo$qIu_R#G+MF51ZEX!tzC9DR;<Dyv%Dp
zt!bbe2<o`p*2CgBzSKHZ$jX&)IhlR$?v<Tjf@m<=n_!Q9`xe>nc}T4PI`tAy7te_U
z?81SdcSX3Y5^4GF(ChMTwqGV+-1)8ro8O$Hksyx-tt@II2x$|y<K^Ab2CLWHx%RKv
z5+VMZMQN0WbCU;;6Cv2CkV6X@?5fu&gUzdX2(li%7nh*SU#f6A)OkB(v@AIHfFZae
zr?R^&6tHKj(>lLya=&5XOL6NJbm>(*@f#N&N~6Pff3@bNuDH2Gjq%v%Y4p&1l2p~U
zS82I8m0Gq2NZsXV{T|Tl7MSs@c?H<>p!&<83#%YcTKJIhMQv$YKZ+LXNq2$dhXjna
zO|Ggm$^~r-$s&KCukDh11?a5}N=X)6znh=wn9c}X#u=!Ji=capoFV>K<>5@F_~w`+
z`+8VOOGs-p_HH&-sbTFTrPCjgu?D+af+-b3W9vrv?HF-kZ(73N)e|(YrJh7ga7ysZ
z_`uPFj?k?@`6y4-lqjzrSJ|$22sEguK;MM*Td=yN4T#)Likxqrl0C{eE$liI#-}yX
ze(1rbGLe<9Xx=rI137MNYCllqXQkY49t&gTO!8d*NV+*bWcj9!QCt=g)p->~b|-MT
z=%xd`#S92W%UR2JOGE*<&C}ANnr?iWK4U*gW&SzsrG(n`Eq3P-wplGs5#QQ34$Jvh
zVQWA?>)iq;y^iXWNL$z^v&fuVpid|$<5DJYU557IROBJQ!sjVQg+3C-bim*?y5L`=
z+zZH^|I(f@&m@f9@ylaXTe+~cn+S}TP~Z7na7SSB*ex4jm~wGm6?8WokoZZUP)2a|
zImbpGXe;9iU|XB}vCA6>2P7jwQ;!i8_UG(aBl~7*xV2^Z9L$r{kERd)b2UzJH=fq^
zsC~;DQb&F@QAQO?rvb3bfJ)JZ)-<}WU9ZK0uOs;M&FK3~ZVF9E?eF-uesCmZU0bSs
zW}gnozFntV@HIxv&opk_jlge5KD7IAJfG!8ZB#>g5$$qSge$OL&9JS^yApaxCcVup
z{&L$4Y3oJYSJexD_+_9|#^x$X=T%H-03%1N9}3b9l7IOPdbja9ro`DFI^h}xm5vDE
zvs{S*tHV2hL#OMp_uH}jC>n`W!#pnp=?;QW8<mhSR(Y|v$^Q29tn5+neC*K`KZ}bH
zG-ssn3EFE4RA^dQi+#9?ZL;~x@O+>4A3+0Z>ieO5O9<`}Dv8kUvB}9Gk~bY>gK_)o
zIzi_;3n%O!<>gVi!@9WXK`!_w=Sdyq@cTn}7Uj?`^{e(o?LL8}9p!+5VlZ6tkXLIA
zUt~lkYvOOTdX1$u*cV!^4E34JSgvP4{#t5xEHmE4Ep;?*#7f|<!JM0sx4jMC3GHK)
z&nS9o+cHJ?R_(NhEwF~+H7|r5?%j9j3zMrMwO!C7WHS1``fieNL*78!tL6tGlCuWt
z>Qdpyo;kt|Qw`LFB`#%o8JkFS1r1cTrSp&RGlbZqMf5%`cE_QOBvLwF*G)l)ri{GB
z-9U<|ggkEtN7sZ5PM{TO^C@Z%bphYu&oxUhiONtBy<O{`3Q)nykP*273Ig8hg#1cs
z$UIYT8RfqhF~2%@E2X|Gq1P!oAztx5?YL|Q@~E^?{~`urFQv4c3_loRS?T!2{sb{o
zSP{KO3+zYRWjov<5^CUg06gq;`iZ11M~H3M;M7gz((S<KK0_)P-bTM(>_~{)RfJ(t
z3}!jW_YUTgD&a^9JN{Br#XhPr!*iUk8_FdGf)ZjUgInxO5c3dppquxwNm=&tI5x;o
zBqML%QJHTe5x8>b)rsKhA<!3Dpbki{!DSrm*$lz;|KJA>66i^CHNGCy;r43ZGtx5m
zVov^{^zLRmkj+;;Bc(T{?i-hicQ-)`qt8RnLJgvQ7}E_(WkHy`HudT(LF76?>3G7h
z#CEes>q2F73w{HJcaN3Wp}VEsm_BFa61n{?vN$HPM*8=e*G~zJ7YVEsbi!UFzqC&i
ziv<hYO)7fQ^Il}hXNcg3v<Hj6Q?!eGqdud#i8&qdc2L5LC5wGJiJo+6gDNreTDv79
zDFNRmKNF_QR;_G-({odckx;(3Bmdlo;khn0^1~7C!x*7y*LH0ba<$J?{%dHV6;3<7
z)1OPXuL_2FOlHM&M_5OZ6-!b*-#at6u8b1z5BM?f>oNAzsq8gsH%szkY>n?vQoK6(
z7{t4ZBtvYfgw9J<99{#zZfM#H5LEW#csO7}y<!D>RN6JXe=4M_w#fxFs@4W-$I85N
z`IBhB0j&!3(0KZ#+{TV%C3?jpEnio|_-<FnNq#*Hj5~d@-f$q9BROn){}=Z@N37^w
za@^Z5x*6>E%;INhOvcBNQRZnG#4q*NjiOEpu5&4Fa=GErj_&!MiY(#yIIe`pmz~!c
zoqW^;jJ?`Q?|Q9I-@yrg-(B23MmWQ~f=#Arl+ShXkvax|ZuvF_>1rcPp0*X71z3Lz
zV%sIw&+xRqSg5i!`c?X|4|C1*2S`Xo5&aO^`gn!h--ZPI5R2!c)Bl5}s|;(p4flY=
zC}HHN(cK-=DN4sn=V(Mgq`PaRG&m%rTe@>}h?I19gEV;Nxz72v&(C%5dG25SZ_@+s
z(+hs2@5fetP$x^=p;e-IKL9FYoiKoW(fjphm}GAlIhI#PJKtL+^?pc6vd$aM594*O
zyn!%(8Z`9AHjzVaw)dHEgM|vX9O5~bc*`%mpVT9v?=W!K*c)nl8`Vr_oKu$Y^wTFk
z(bH?vyRBw@AKN;nIJbD`Nw?ZZ_sq@Z)ns*s?OO^7{RPRk`{eX?D|CvQ-uER501ArA
z4-vh=a~0myj|+o0#8MAp7VY^B$S^3KcSUj$4ngZO*fEa-3o^g@_;2~(3Pai;xKL9)
zR5G6h2hCiwcU`jcKl4S9A2|O!Je%R5HoJ(?8`B9RwWoA^*;aqCVxl75Rz5mdS-Kfk
z54cn@C<iq;IB}pHJ5Z3e+}%$G1{8IKsZhxsog;Bz&1K$*%pHj&Y>j2u=B7t@FD@O<
zwSs&3tB@W1(;8msgjsySMyxzeE%Jot0m9W1-crB?64WT3jWzVWe=C|)leYqkaO#=C
zq|e;4I1DLPVU9eEhU4Ae;nb?n-!Zq$wCHM<0Ns!QXUmnYSRU`4gvu$IRgZ9oYkU~*
z2-0d7EYhtVjaJq-+EcmuHnpp+l1t<1+XMQuxhVs^EH%NdnRgl{-fd*yU8K`X?tNx%
z=2S5?!qwm`sP?-#;9nePri!~nz2Z@SiL1CPeCw2Gt2=W~;~F1t!yqM98*NjSI5+*o
zoo1V1@i6?%imH7|2Xd)IXAYQY%wn5BO3M!*r{9s6Y=s@Qm&+s5_UiK<XFT<*KlN|h
z2)7xW&FSv#jQo%{2L(6ls#ALEJ-30f@PzI9O%#@|8i~I@8Iy@6m5Sr-LF6({5f8&=
z68e|pcYrAoZ~E@+;=*o0F%-X0isf&&8w+?WMYdYhZB3-7Q<@uZSv2rm2Lt}o1gLpI
zH_^16^e*-h#YKlFY-#Ui>jQwniF+16n1KClU<L2*Yv#><JfN?nSW(#eA-!ycm*Iq7
z9#WxX?@yoK1n~vi8v6#azZJ_cm5Dmb^{d>i6*pu#m;=?Bjx>;b00bRoM4fFi{gpO>
z%nhOR783qlk`3&9Q5-m!$VAcSQqW>k8-byc@I&Kpl*Gn<{OJK%W+B&&HNgCnw+MHd
zYr%*?Bub6f^F%#!qw{Rne9yH5_iJbNtNZz`mp{rEt#-LTZ>0&=2Mj%T?_<-geV5`s
z!XB6R*xrK)gC)YHFD5b$w<<6qZW?wsKury~6`iGcfp>k_Qo;}sMVez7p6ZEk>`9#H
z?qtBz)8yZ$8R0z1&WSBCqM(0^o5U3*D57pv2M~Be5q*7ZE9J{XZSk8{n1g+8fY<S5
zwHRQITER(mO$FVM+4!3RgH(3K1>=B0^uHHP*CCg-Strsorx*wR5R*N`VdRI!pK!wB
zIEMzdYC15SkrOs_mYgxeG2}ChQlo=)5rfK#%42T**XfT`{9(GBlXTIs8Y;+_1A-IV
ze(?`Up$=wx*WKmaY<I{@1LRktZcp5Es%4S&#fOSJWfn!5h`9Jl?o_~YW*IZ_Z<5~3
zjTmh&!ij?b+zT@^J+c|X38n~`r?x{YReSOXsH~jtxu<({&g+P~k3JZ&Ic6dKjr04g
zAj)#GoR*C%PuqJ){dmWlNcg+)z>C|};WGAZY}+Xg%)BgoeTu*TnO5_&7moMjC^+<`
zO4Qc5M>lbE-1lAuo-!%8IW?a$C%7Tfi;Nn)P0Kweu6za5k3>n7Ss}G)Paw#54}i=i
z=G$nl77MgoeLWMOowq1*(8)f}udCRV8AtzB{;h3!hfDj!*y_Vsc~@i0lfEJkagcwB
z8q-*|WT$KVKnle{`~A2xjP={@N}L`8mJ^9@N+?~M^D>N;hnoS#D6IP*_{%&C;dKLV
z9GtBYn)(gbPs(wgoe`tnARaZF?mqCx2_rU#$Sw-H-n-jzQAPh_b-IF?Yah@4o4@z|
zr*65WlE__HxgrTXZ*MvUoSlDr@9aEx#B~Q~BBePQ6+H<0=X5-+zT7l<&i@|eAi(<Y
z7c4S(*A`~>;nr?PIAWghgWXtG7h!M%twyiN<n{ZT9hlN$wYt9#aZ=*$Z00RtFfZU(
z5RC?Y_7MNMU(*gp0V0*}HbcqbV0|Kgs?YA6<VdmhD)Ct{%JEn}NB>5EB7ZSJyC2Dx
zy8JELl!p|?T41oiG4)V@9HfLrJ&Uf8Qf0(cQ3>pA)z)RGDIzpLUnDGX@ilhV<LW!N
zf9ZQ|S+DxXs+uliY9L+~=unN3!L0A)EymX<BdXgGbMEXqKn}L*eNFb}cfr*70Bv!B
z;;I7--tFJ)YW)F}$D_5{bQ9vd_g|yrR5&0vlWe_LMPcR_ODEo%QJ%gb`%Jv;zqP|8
zz~2b4zOla2_J#d@0O}8dgZ%dV&N}B&9_?y8x0y;+(4F2asH^EzOj6vuXE6<KodMgk
z`!dxO$4BX3b~cq(U412XeE0kz;aa&<S+g=BIkNUsaes4pfSSSa{M)gMuNs<*LeOEg
z<A%PMs;j6Dum3tQXncJhU`<zx-M&{H2@BZ6w~ATmxsS(jV)eHR8GM#mmg~PxIoIc7
z^?bE^%Z2*Y5`1Y`O*9k7pk|C=p4w2SECQ^0)m>kDRx`wf@o#f7w2K*ZpVyg7fc}E^
z$mz(|3MwQ`F<Jzg-Q26bL%n&0au)daF`KT0>Y3%Z+*J1V^Fi#og-vepsm<gtr?dCy
zt|e)k{v-k_lZLiuc6hx@i7x`2ES}{&FRxo{y-1$-Esh!q{)pepx+!Br*XMs^XLkbM
zlA@e5k6~WB0e$c8AN1y*hgNNmdCj15t)s`TaB{W-b2<%iWzxlT%oOlVO{Cz!-8VF-
zze+5DEA9Ea!m5vEcj0`m8EiVVp}NId-A6=tMd+Qu9=%(iKC`WAp1`&P;yTP?=iQmr
zS&i^MKZO^MUdfxA0M6r5Tyjk=1}JJQp12gZR~itTI8LQ_+Xu~=6}F`+fr>ZD1i>qc
zNcUoV4YmD-`bS2WG>;KYRPWCxUDZI@>7R#EGLCYlx@vNd@o5VnU&AYO1#FyczOO_t
zYg%*oE;pmp=!*ysxK%rHzY{>O8A5f(98q@Na(-u5Sc-Oz*ivST!n7}}R=jI-GJbEj
zdO^on3|nzF!p0kc$&c99G`oyuTjcY0<~*^!9!tCt=yu#B--*0)@R4e3Z}2aGcQzFR
z2+lSO{Aq|*?l(!RUOP8A&k_T1EWG92THa0fOpOcs24-<`p1BR*S0(GoY*yO7f0Wn9
zB%@K^hH{fiR^KD{COD6|$!0KN*g7-JYf}g70gT1aiHWX&ik9Ti>)Of4>a%dy(M&V=
z;l3@T8}TZzX#Mj=l`d*57PPN1<jKqSZ1xwOP9nm2&sx5w#w)f=Z!VQAbS}Cb^uM*5
z2Rj2J%8y6qQjPdcy%3ms-l=Yt(fGVje`t~e=k7m^fWbH~Qr0ZjF8xy#>tJ4JY)C2U
z+C%MYH1kSWo?|;{$^;#JfgnIh%JT#e!AN<m^Gv?gyuh3ytM59%ZtBWLz4iIGQYkAf
zc6Q(EHx;bl$xxMPo!zG-zz0kekWotFu(soC=mY|Id*U|Y?;ssfm}U1i)6^q?=4(2X
zq+rvqc_U_1mfMYs9KY4R8oU=S0!Obq6K2%kp!zgBVoqu-ZWQi}`oBQwpOHP;gy=dT
z1w0_!oFzx$I=#+zeFV2%CjflQi~<_(TasbC)YTSe^mvxhOp_N)NT?hF@z$3BM?SL(
zxKjr-emMKZhOzM`@jC+o3`++E=*i-#>FVaU3nFrbBQp(mugE5N2Pyg8|EWVCMm)&1
z=MhU%4Jp2~Y0R>Y4xRPbP~2x{DkZCjDvXv9-6}rDtF!VeMDX45oqXr1&=JCABv$VJ
zZlCLQ&M{&Q(l~B5Qb?@PjL8%I{n-^vj64W++||fX3RmX<e7*;W3Z*$vh^=mcZ$a4|
zS*Je}0zS`^G?ah7b1*dF$lA&DZ*=fwWPPqv!W$@RtbBa?cK$2A#kwWMXugvUO9h?u
zX|Gio)Y9V3LXIV^xb9k+@rk3qkrUf1E$8e7LfHd{!F<5Ucw4%24K-DsK<uWU7W!(c
z_(+*t&`pQ7Y=cu)jKa8z)WOo14TR@7_nlD+r;}aE!meTFcAU=AvPbcnKQBq<RFz>a
z_sHx*Sa`q>DGQD!z=2|FZ9M+B0=7nZ^Q{wZEa|p3-(rOz!C5Y6dWwu&pf8!%qSpkB
z>gXFCLpG7ZSeQR!CdHRDxu7W$Yg%#d-=n5Ntfu{hH^4<sODJ=iRGtSY*E2iDwO?f|
z;7~hPY=NWKk0ftiMY4Aq!Hi)L2XEd>{!zsT`%y&I%(+fXpINp~kCqc{1!X?3rX<k&
z{;3G<2Ck-e^ygj|GFKMFaJXTj#w2={Q4>R&lr;(P(ysIH{OuGyx=qKFqr3C}kJW=5
zWjlRaY~&!P$NVz(0{8#?^~ku`7rgV+b<9KFf9JIc_iEc0%P|S>9&A${i!(JV4drMg
z0MA<4--DsmN;w@Uj46yv$0z<ntY&@$uJx#IboU$`{w+I`&P!30lNFJv+1gCTiAaov
zN)uJwCGlBTcvc63u#+X$C4eZm^UBi5`#B*G05{h0c{S_Sg7;+XqIe1u7D$JFk#dK0
z7S0~AkV;7Hu~2%*dbV#-*^WG27fVU#Th$)IhWU%MUy~TNw2%@94G;vNXV*Tu&u*@h
zI%Z+<?U}?pCgWgMO^yowg^H}zRv&m`hR+yvC%Qa&G_{#HcdoQ*a-<oBujq%r`(W#e
zmYagFs;ysZal~eqiq)ME__T$HlF6E#X(^pLCp*CL2i#g0!<wnBRTCo;0SZO1NC;;3
z?^AU`Y=uPPL0^)#Q36N+X5jps$?_%h;<1SnE3J7NXY05g;mF_LxzE2#__UE1@CY9z
z52*&W+V8vckN&%wFnV>}RP^_?ZxLL9=9$Bu7ZqQF)bTj;7M|0g9HMN~&O&mm{a9O<
zJNNO|Kfp!i_ipI4$A-8%AyXVAhf^7jfjyQT-7P6?ZbTeZ;l1ToyarOhNy1)9jV!n+
zCm(6smw*ouXxfxj9<L?WQvu`)4n|E*KZW8#B%>T54->5^6rpj$*}h<8>exmF<hRt^
zM*1po2IS1tx6OmNmhpb__Rm9$x&Hs@DKq+v;dJ>S2Eaut|8-?umI?O}B3_Y6R6UA$
zv{*QRUm_Zb!&<}IAwQFO(07#sB5~9$QbQGr?f%Gylyr4(+%5VzFj3Hkxc0Q~-!at7
zY<)5G{{MAYkvPcNtB<0nn{9)i8DJj~rCR<xK%)VFRg&P4{lw)kuD_J{2k}ZMo23?1
zrAxdnh7~kNzy3&YkWKb@g(vY?5}`wI5`p4Uyeejmm%}RF1E75p@0CR#NP-ylo~YDJ
zk>?jnoY-r7*K``7S(F|){q+7_Z1M2|u?^4VrL?H_TXuvk?8B}EtG&m2$Jz5<n6}+|
zE)n`ghl^+fPPrD+bd#0IP(M;?B6P*=>f`7tC%-DF&EQxC4kUf2a$NdgFNaww{)Mo*
zm_W{#jT2?>Ez9p2C3i}4iuWZerPtQ#9s1k*YacGvzOqo(wnu|wttZ<u>uQe4-b0mr
znBvC-y>jC7bdYWeP_<8*58K0pVCC&ow}s+j6IZY^XVt${L^`0q=E|i$Bc1WjWS45h
z$Q3`{M;&5n5{A*kDL|+nvR1E)gjGjiAd@$ClUK0Yc2!|WQD<G#p-#5le4-2pE^QsS
znJF%(iDJVxj*S=?`F{h`;Lf-vDtqk>p@p-Mzj<zcUG0hUc|?d%`<*l@RCy^F2}|ni
z)fH-owNZ5G^++4eS_2ipirs@S*hvQ|1S8`4a7nCAk<Ca3r&qU^ePWwl>Zo5=8#7%j
zSdZ0c=(Bqjo6k(yQtOw?D*qEl&q=*66vu_f&oJQ;yAdI_(1fQLy>?5c5VB`dv=0cB
zr3YM4MWb1Z-^GuI<h&mNM{0Xd*>cjSFGerB7N7YPV0GIa@Jz;H1urAGiWk;I%uzx4
z<y+EWQLePusZm58IFk>~_1Y8>y>I&Bf4v&>jyY_>KMaY}N8*wa@UxI3sHK18?4O!s
z##>NL|MYj8S*qn{mjdcl14d6QYv&$sNJmzigsIn6$$!)tCn@>ovXk=7b>2FJ@$WAL
zxX+l*3MziOxys+QaCo!K=^A9uq6+>aYDhlPku-`^gNL*zR}e}~uFs9{>J?0TNAk#E
z(gBUTRuWa7g<c~g^={h9uBa4BA1FVmOVlsuv%_vCI;#dN=r78BHzGnV@o@eDOlOgQ
z`&V1)kG9;f?-}r&21)KEx$n}_s?e+rl<l04^k4qpsab<49Kynd^3gTc2s%7hq>Q%C
zqa(O80CHZ>YT&wB6+1Z8>++ee-iIiU(yqg9n!27((%G%hT1Enrv`2o>DlB1@QRt$k
zZIBbXTdnBP!+CqNzi^R1n`bc4=}^Mn8~|D5FTL>bqa61Qw?kHXX#=YL`XF<aKkG95
z<`Ejn{DYG(YLpJi$$tr%mBLkDBcO88I`U>cC8-5}Q#))Hdo$I~W26oWzu2&kvPVn*
z@69CK_IteOLD-*)Y)d?*o&5fM8Glle`!C-u^<6iHhto>*6C|^B{wv0oH?{4wmlbb+
zLF8?s8}h2GSK=jU!R)&EE85BYF+^0`p*F(mP~M#M->iJA_9q+7&k^eFB{3c&lRT{z
zoaXx{1M&UwhQ+ZV)!un_N60qT4jNs<WV(41@CCf7Sd>W^Z`7~&M3}EfH2kcI+>YHs
zgT4ln1vhA4BNeJ}4oOhS(imXVPThD6EoQfh?5&E2m!x;(PGIIC&Bae5G`s5_rT#ZF
z5<^36LzavYx}tw(v@wn-eEHS3|8b-x6}8n{JqinXP7ENmj&k0S@kRcrGtHcGgr!S&
zij62p>sua;+fEmDkNqtI{~bi9tR3?!K9rmODLcs@&;9-(Vo9PH*7U*QR9W%sytt}^
zKf@`1uh(9f23b$&VsDc2k6qRM`~1zEaP|#HQd;pLuE6`zCU>c)4ZO3u%hh~Ytq~bG
zhU($*0Dr`$Y>uaJ8lATHRrZ?Shq=`6OLG!P-qPdthf>$oyfHC`fTe$rDSICWNf?g-
ztcE4$)~Ez_*M9VL0uGYbUEs9L(e276m2}c}VqgnjEQC1aI)rJ>H^T)N>p^bc*0p`&
z|8)~AS$W$)RC7`Qlnkhi2FOuK<$=|pNc(5u>-hW)B5MwaiMUSW8ibSaRXJ~xx41K`
znl#u>9H&czzI`Q$>^#yU@Ai#T=q2Jb>_4TJ8-K@*j;sBoMaP2|?4*lSw-bBoO6NJY
zMDfLzQNCkui;{lHJ(+?yzyG)M_SN!{>22%tjhCZ8tXYES(|DFK+Wq%NGg|WGUB?G*
zZ+#4lV;0fz0BwQlKa5rzZSmb7S&>Y33%o7t{>n6dzq-f~kKG5-O(7m3cpb@e2QL~8
z_GC$o_rZHJLnm%~e_3R6j-owG9I~f;KI=>we1%+&s##hQx}Z{4WZ+$S2Zt;mkhs+5
zCz)`p->A{*G-j*j`3mA1{%TmVKHSS+R^e3lIc~BodhI95hoQ$f&`wfP>)P|FaFhDI
zTMi<EseQJEDAKxFdY?;;C_>TRc~>G_-OWPTHR*rK&pleTvX`xBKqaw4VxP#qJ21tS
z+{JxDhb3+K<7(BSm+ab}kZYN=bt-((^A2h&4k;tQdAi~abAxvz1Y8$XdXkSlAiM^h
zrQ-koVK$c~I2q{*QHebx<q%a@Dv5Y%1iP)KOW4s=t7yAPDaV^LJJsnE_Eup%ZjV$X
zIG5;bhHTL6#_pG!tCbW*JJN6d<TXEf2;e)zh;r(HaX4^9{~hAm(el#9GK#;tk=C;3
z75}5Z1=umWej6p&;AUD<sW7@)bPW*Kn5vA`FK?+GXKUX{cRKol5D8Det|F>j%o;ON
zcgr#Jc=k2JFxVi+OvW_N7YLF5BJ3?LU~g7Rl#WFs!w(1SkzWJ<hR$vY$|>9dO54XK
z5#|wlW_uO$%$?&trXnWJ3t?<h^RI4b&^zg%2|7Q^5u6bl?jy6PXf-?dJsEHl=2=%6
z)=qcnS|QiY?Q)uD9!(O$+zj1V|6y53G1mu*Qg%UmJxbu{(wGSvE#@*(-(7l*+Ng^R
zY6^1Mnxfs_yb%NKOe~?;l(c)_F7psNuRe>A@?-Gt(zLxb#0k!-okT=8gmnS5t>2&I
zJ}Q3}&n3K5Oo9yaZX?eBR_8?_fG4-R+QLOZeYf*%OQsMIBx~M86y#_0!s!|>rxcf{
z<bbOnwjBlzFHW=FUsT(Pm$Oc2X+lgRN*92SP>~_S#>@pD<#UDmYZ!8E{6C98ZH#8u
z7R;HAv`1=g3E*Nr9ks_fu=@{1W4>!ApM~Y}0rd;s(g?r~n_vD^uVMLGMk%Ev&!~%?
zR`OfPoRBr1kjMOI0S9jA>l97d$Og1vkP{yKGfIS0M=EU6Mp91CM(a(py(M`g{%Q^h
zG913e6UZWZ%iludNu1{#K1+wy&608P;#vG=MOi_Os(|^&Ug?QCIYid?p>5^lqqcRz
zeemYksJWElt%f4&(XFVS2xf{sVEp+7@qSdd_%`{pA?~mMBl?tb0!uIdoUM-uOa6t;
zUFmy$)7~5q(!pefPPxSttnNFTqCj6j4ljZo^3+MGfZ!R=4qPDPc}dUDr}Kg@>l*)1
z(xY_};$+P!bT)CioqbHMt^}NQ;)cmU&jd?dW8JCF(1N?fH4Pwqulc*!zs3WqMQl0w
zoX@iFmX8@j25F8|@ocrO@DseRZoL6K&b0N#x%b9@5t{LKyF7&$M9LIu%9BEqLgfF7
zo*pl*Bd_Q~MV8ZlNS1HEI}>+$RGiP_Y7PY=N--*kClQ|gM#nt}Vkc;rYcL9D7vF6^
zS8JP<9Hz{tD}cLoA6cG{7#g>8eu~mo6aDTKfWsahOy~RlXL?%gf3|D&Mk|N^IWaIe
zjn<uN9Jl=e*R9&48iNNH{f+kjX*0q$8@PY|{m}WP@zG9YH>&AaZ1~=+Gp|s-8+Z<2
zT;+Ck38MZp<o%?`sucB6ebkQzjrK`aAm!%4VGlu~SeA=haVv%A?20&7ZyFjb`NjPq
zt*dPA7#C`~|5opW7|8wfC!<YDrLechPHake)A*}_V?si;lz%(4TME(}#_r)nLHeBX
zi5Ek%yb=GUCJ<PJ8Huc|qYGzn<ahTu^d2oJH#4JmHj9?0Q%z#t3vKhtdTZiSkFE=+
zt$z9&zsmV8A)IM$UHJ>9h1B;TG-^o=A{?HvFgj@fy`J~<PdrT4W{y0sY2Wo`EDJIe
zsnnJsm^K$B_0d87v757;<X>>jqCPQaK9b+^7I?mcy@NlVBj+JS72&5e{JPa+^rNQN
za_X^u+N~d8u6a2HWW!dsCn`JU*I!DAxyZPfLCXSUr)Qq$Z?>zuNWYntl%nfCe&jO#
zx$doxqrIvhLwg_5PUL7^gi;eR;yn+$Oey|+<}n6-J904Bn!TVj5l(gVpU@$hu(uVk
zymsdwOSV`RkvDT2b?Mm@RjZ_kCN3P)<doOhBH!#<ILR+16Bc>vE2hNKgd_LfZkt(r
z1CtO(R)+-QH*E<pkHR2~wv-Y1WNcGqT(SFmhNW2(N2I`~k7CtDXhX8KZX8Qm*1c4H
z2FLW#G*qcgziIxB6AeSt;_s%pL&qc2=J?uBE9`UCCB;r%X`Jz4N+!l!L|WWZrFWUj
z2SFMvcBrdkb8Bshrc|E|y$JopPi&#L|4YB_xap*Y)cL2<`Q$+a>#%nFNs@0!p{WpL
zlrU{kU+&DK2++zB2Y*0a#+3FSNI2&TyPsc(D~Mj37uDQK<Gt~@h&2=MA^bTXCx7zW
zIZO??A`#PBE-d~z;!F)@i<qE)0-P<>)xbH=NE-OI#GY>AfMUx?J|SphPtCsg)8N;+
zE?zWqoGuogEuh62)u~J5f8IYz{vgK?^{O4<A&;s79FfI>jg)+SYs2URuJrZnL9_pS
zR3=K7b@=sCd!mz>hEVxDz6BmktEi%brjbswD#}mBqFwt6+>2NWB*cg6^Ot|S04prP
zbk>lhAKC;z;QSo2ETZ<W#?}%qy-QR`h*)?HRPcbSXkJ>1bW$5ty8{hM2QfT>D$sz#
zyMH#Mw9!ryp6;hKB~+z=N%}>_Y@pq@slz4yDKeBlUpqCvhnXgZr%dP9Q)l7SsHC{3
zUE=3kuozQ}h~g=i`auatr);Gje-Hm;{9Ib$sk_=v|Ai{euDL&<!L+*Jj5oo1Y^Hg>
z*f@Ga_TB?iT~J<M=FyGEfSVu&)ln2LxD}eu8b8~Dv3^Yv+OtlzQ7T}A$FTD<Q!F{V
zTR`19Uv_%7J(SU((|spRZBD5pWjR6p<AuhGo@9VU`%X^z2jHuAdn<^PEcZ?^GOG5=
zjo4PNWUqcw$4x`^SB?Luc_qDv_G=+5I~IxEPu*gXwIEAl#!ZxvLvrA6X?>q&zFpwr
zdjr9QRecf&KS`bcBpfv9d+4hNUW}jWg+xCLVHc<d&iWQrck^5WA!29D@y;MNm#;T5
zB9RAGal`UHQ6c7+3iNz)Um$xrvlI!(oaLVGh*ltp)rmEFV7H692%qWf%^lcaCjQHP
z&d}=iV<zJHE;*O2SRw%m_%1Ye@_PLCjJaSra)%4S*`XLS&OoI23$S_nKz9tEe!;I*
zh5dhqFw~cD?{9EDF9VR%l<C|m-7f#GtT=51PRBaodgeX&4;13f2!BRoG}`<nce`lu
z96SBte((=u=Vg_Y!=pdo8VeON;$kRnGI3pzo;~M!li%nk6<E%OmU!mx`1^jCr~J_V
zBsvkGLv%)z`<xMoz{#7qcQ(6~U$=Z|4N2Fg{=sW6N2M;!&KgGiH8BX@R^io)|EF+$
z);0bktFTS+JvvmpUax56Cvgm{T{XDw-!yM@IN!FzxpAk>i6h3&lqvj=q@_f^2)Q+5
zeY)!pZf-Yhuep5RKeGewaOa%&-7-h6cbGiEYvfzJBgR8M)Gsl7Y3}XjDu358q!t>(
zJ38$n3H^vIvi!&9K<ZYQ4=IFKjpD7!k*?U6XSA8gduMNMp8=i(i#63xEA11)j4GmZ
zo~nyrRFC(_0F;VRjq3r<?e?y9Kbg-^U6E&vJ+|%mlplnPQ7AmE%dg6qNX3z&l@fN?
zUU3i(Hv*v)gtSiLh$gtDyGIqp(MIA3Y;J%o-g<CoO>2T^f6N`iiQ`A{i6~D;OA6(h
z|A9EI@yT^!(}p=F*3gM1T#$!@z$gDdl6igTx2G!weKW(ZiTrdwh}<?CcPwg#8b_Z$
z@aLXFw`>d#svvjYKZjyphzpeNfpayPMRqy%1B@U|fq>+gr7fHnOYJr-qnO2-Hyi~Q
z&pfpUm?wP3HH}z`FE6PGWYb*U0L}(E5`ojn7_rH<BdjDNfOueH4KsMM<`+I4$7w0`
z88t`5infye#&CC2Xx$qw9eNK)Ab;luW2qR=WeM;A|B#27M;NMmT{mR=AtEU;Isd<a
zlPn{%g5^ki{S+-^5h%z%{><YiKJ;4ZlD6N7xYl#r>7!dVt4-|GMCasA`?LGK&~IPP
zQ)1{e*m@72_hl349nNl?_Xxh@GT+Q$4ZdKm;8}}H_!Z_D$|`wSQcwg}9x?Bt(F5=b
z)z(yoXZMf9N!{R-0adscj<Ub|1JwM+N>1p9u7<sWdDGnq#z^hyL*I+=nLW}ROCY?N
z%=)VQdv@UNP<jHGo)6dJe?R=(`$C(m?K$u&Km`rV+)g`qc%pbxs$PP?$rxhlv*^S=
zMF0LU4<Ik%CURHIpNU{nWnU344mVl#&6WM~V4ku-?XHXE`p~+1QHExDVQ_opJm=h-
z<m@4gsgB!#rZ5`71SgeXAM}RYGS!;n4uD$s(_Z=gm4Byi4Kg5#u4n_Xhm&5#+~Dt!
z!vJ#q8Q4;X=-S(m6&>vCZEM<d>5?Il(g#sKcx!T2OUVA4FgcH490*&CK<|lS0j5{x
zP}e#!QQV&r^()g{S*TbBgjiP3={v{mp*2SYM$q8JO%2``!e%>3tFOV`6rltu*R54z
z?fzjLabe5Bm-;{GT&QDgLW8m!09SWx;s~{Q#&g}*`=#(Xp}Ua{$v~K|@nM1Y#;JSD
zQat<vgg6X{0<JSmxTNA|niBF~f@aFe8=d3NJZRsG@PWtatzwQ}o!pBk)fRSq7ri_#
zbQ@rRX|6{HEj6{!N_|Qx36iBPuIc_r)P2K9m*PJZmS}S4N82X_=Yl%TBDU#0(w;&3
zVgfvAF*9Fw6Y*Y*ZZlT#PB@3`if=vV_D6rzu94=dR>4j;s}~v8p$su+3a0IZ3MHa&
zl}VrP>?ZbbP*`Kl4Rbx`1PzD=E;_yx`yuQ;O3LE!HeK}Ay?|nwLwl}q$KrY82>Gmh
zX>Klb0k)!pgyn37R3Vjg)QkONV)M;*+YG3z{*6SH>}m5ap%3!NLv#kOsFl#StvOje
z9=bCTz8zK+9D=6y9nLn%mEy4ldx@iw1X-eBwpa$w`!j6J;tp2*w~C)gZBQ5KIC3vO
z(x}yS`e)pCq6wqmg;f~3&?0p|q`8J<?6QT@yWapuC^*0Ut{^@Gf^^##hGM%z!-Ib1
zV@V@mup7^>UL1vAY^-Xn<^pr?Tz`6)i*8}pWJQkVTb~L@^nz@|NHBlxxI2fU;77>!
za>n}wD(}&Di*&z;X~|kl1p(2p_}lyZp#LbB$x+b6xfjW=*hG>7v<%(2H*$G{WWVZ*
z`(~>=8ZbxRf}F>h0r-WVZI9)6QTq;@<dmZ{(I8|fOLqG~pR`f=RhqG0j75Ui<>w1-
zh33jGX=H1B{WOPue#|ftPw(JtaB=XGWy1|IAerPtxi~YkUp~SaU;^83|MHI)ti~nT
zf9Hm<5~?SNyMNxHP(<<X7eVS%_a<#y7+?Z5(n#-Q)1l@bZVctnp|FSX1K4J&--gm`
z<yf*1gb#%+E-_iIrV5ImfG>-F7kO|mlB{y~Qu`al?oxkQZK7Ei_n*NhFYUS<Zwv7s
zb31MjfWDVJ1Mc!1h@<MCK>)NeEa<cV9L>+XAIqv;th!nxd&^bw5{DxBV$PJU@x3ut
zOBA8u?=W(M>T&`w)nz&=9<4QEF{sdAySAa-y;|iL$G&2e)p35@h@@WC9nUhj%Eqp@
z53QQV;nFNK+2u>OA7#uT4>5&%ksa|2#&+XbQ~RPlwgV=##w@J%oVIiRyy`V-=nL<w
zVE*inK$;G>bxPhpE^$EQCwj@e8;z3Gid!t0LJIQMKkr6_lLX?BE>$ZXlXT7NM_ZTV
z0x!qUO>TjXs-6DR7A_G}nHQQ{SdOEM<QbQs_uDN4i^TNr;npe6DTBGy7Y9<%@13R!
znS)sruY@{J_MOHu_8pm1Y{yu^k}%$7To2wF8nNUQ+xpP^SF&nb?_?UZEtoGz{`t&*
zH<;ZTsN(_MQmh!|e@e5CJRtoJogd6Nn%uIs{J`~rN45uozai!5(;%k=<#~)cU-R*(
z37IVPA2H%bKW29%oZkqa6{6#=G~B3-w$0GPaFcwE&SG}6bv5q$;?$Xtm%mFEpPjQM
z6gOJ79skDZR*@(uD@UN2rv%|D_9a6`4XxwJH~Uet7`v?^6f*x7fW;sZ0gUS3eD@e>
zZb6J$ZsLFIpGRyl=U;n_0GNAe1StxDaB^DcoGhgx8r-D6SL+1dt;&}J)m?qiowY}2
zk51{PzR-k4+i@eDgq6KWfi~ZEcPJ#_>oy9IWcxaIg#m9^C2Gdno`NW7wDFl{JCrsZ
zh%{!RW8CtcYYq87b3s{NhN5`h(KyUPBzb8A!mwWHk%HhFuGQO}jrS|3JxUb$k)sIz
z@df5qW^Ywqf6_PUwNI7LAo=}HY}6-F`!6@j(uvCZX3RusY@rZUmJKi?i;2byZH4aF
z`lVYvMQI9LIllyKaYYA)V9XQgdjd>uRm={cn#d}QBs?0mz5Pvtw!H@WQD!7m-#B`#
zJ(?5*#gVn#p(X&8bgF`Zo8?Z3g3o=X@>P8kmT4f<#Ya6V++OP5)N#R`kYbZP(N!Ct
z;;>Csv1Wkzb%;#VgPed1oln%M>v3#AP@+)`+uX(J^Y{A@-o~O2@M&<5z)l~%7eE9m
zr6UzVr<U^FmooDfUyWUd(v2g;QC-`y%Xa|3s$MZ=K+Fid3|hI%VW6jgGCIfyjMLUf
z+D51&6XL{aF!au^J5kT06X<_&&hFq6*I|7(`TQ2dSl0<zsf!Ey6e&2a*3|SHZo3rg
zM8_8mr^Z2)(ZzxsKPNyqW6<n1i-rNI?pVDRNu%mK0?>Z1o<^*cNihmJ3E)=Tp^`WC
zutY7{(*4O69whbhwqY#}#=}+t#3D@%ReCcy*qY%%_-CBh@mv9<U6&?{iDu)q#VWOh
zXoA#bhEpW})$rI>)#xtEkyH@6Dwyu(R)ERCX$yPqlD&Bej#Mk}Hw1Y%iSC66K?WZQ
zH(aZ16HJenW+pzor>mI<>Zc+!6_E-P<fyc-;ce(7@nMSy_rZ-AikH}s7J!yXW!03P
zqwv~^fSfM5o$f9DQDp|Yh2EMcr!(p4ZFrU&UixDG5Lyj@z#;7o;Q9|GOPcZUpe+6d
z^{v5-LK!Pk0lo+V)}H<AHa&tI0Va1<knSiE<!g?C0=9sf?W&H+2H||2b&_O2^aBUV
zVEusHObGp&>a+(9Qs2Js^lXa4=`eK1F0L?Y?>YbY;zhPZxavwNHy)L~{4or7AV|Ky
zg(wv8ik_B6!!+Kjuw^spdP4q7#Q7UFo)xwN_2}?(6m;VQ2h+sUmAzGb)AWRUfDk@Z
z@jFwm>1^N6jzgBwKB#3O!7!Qq<vbDBX>Iqd2|ZMPTREhOK5af}83|3nDX#l<Fq%`|
zU!G!G=(;G7p}-uDrT=cpkgUx$js?jFFRW(KIc(ExweSsYu0$9v$&r@OkGT?7*@(;P
zOBlr~4svin>d8=@2Q_m3m;J|pg+8#*FPgQ9Aa}PofNBLFh<ycsD~G{4&NCDhCY*Yi
zFMgG119-RN4>cDgSVCjrR>HyepJ0^O;FQ3nP5FwyUgj@nO8pTj>^63?`XfUyuNyV(
z@_$Amk!F@-2ZZ|CuOq}p_&wG(Iv?i!sq#7&Tb)*;%e_2q1k!39RW0gb#5}YjQogi@
zF8bIqTidP<09{^y;ab}e|H`q&Zfk<FI1M*0xk~^}g(|V=E3z_n(&0&R{bypiS7vg*
zNXQs8C09q(z!T{&{R`CdMIgg{-#vYP{B%vm+Y7deBpms-p)$0l3!pe`yk1xt7$=#x
zUtuD26VDRQb>(bnkb+yn=}Wh<){@@byu#0+9G`9@_}7`i{}_^t)2O^TrbFH+aydBb
z%)H4p*s8Yl&wd&gLU$0ReT#5y%bWiJ+_l$gw`{2yHF==Xkbuc`J}*a1Iln>Ux?)D5
z>E2Wbp>8kEpZ(cb5T={iIl8|Qxs#k>J53`}1@zSlLY8%5MqM&oPL?vfOHQkm{zGpZ
zg(GPXur3i0M9&E?=YT2?_8RqvK&PZf%(U}0i%*Xhnlf}-jwGi)_C{e!F~#Zr5@qE0
z0@G7k+p?PdQ!3sYhEFc<ej3qG8TmbaHQxboHS-$z6zL#wg{=DgYDK|d*X^u)dm4;G
zNREk4hz2|pj(2Av+eRDjXzrA`XOdyQf0k6~h|H+qHD!sPsn%d?39nf^0J{~&ZE#_h
z7V$l#=+9INAS{E0F)*=K%lPL<x}QRE1}SJ@!M+TUUByY9HHyMUX9f%5+5c`dKP3<~
zU`}Fx_wFTxeH+zQYFa<#S8?Meym6{Z!j2|xEwQ089&I~t<m$e474h+X-4Gi<QCSc<
z{V0LD31%BPo1N(BXiY#Wzo9ykR6?}^kHjRZK%(~w;#u0yd~Nj3=ZWHC#krlVR1DG>
z1PagE6#lkfYUm7lTs{dtK@7l7GPlOj`HtSplqeNzf-gY#;@#(%X)}8V+C%*xqh9f!
zp>A|3%l~A<wkOvo>7G5`?N$t&=aE6LMzW}|ZUZMdZ{|rUb<!Pd*DC3#az4PM9{o*|
zY>=85YWGC(c?bJ3tG$D)S*r3%Su8!4gz&Ydm=p}5otVHDNw8razxJ+4$IHD)4t_g~
zMCCQ|&T~8B%XHTI`tS!Ryd>G~Q~c-{muE^sL58K*1_CYHtNhJsads;f^-DiD)TTM3
zF}B~tE<3#<Z1_V%jk#088`|KnQxhCVp;q%cT3*WKyN?c_#Zv$sORYlPYN6kAxU)j7
z!t+nZ+|2=8L-tm*bxeL|N>4&nAZOc1JC}hfFOI$zU4tC{-`q6IzPN0((yI+h<BMIw
z4TDmTnVXQkM;DHVl-rLP`bMM{PsKhCKY6^&S|@?8Qr~`Lz@lC^-S!H5-!nu)pcMq?
z6Z(@aS?JDmwrFklFIc4{9O_x|NzwM<KiR{|IPZo}UX!lJ+hqE-9I|7qFQyLOnQkTA
z1V>wo?%%fz;A5}eohgmPpKN-2!)uA=&mdO=pRZv2MU!h=`1%dYuEv$mrE^tC*9zjH
zOelG}?$a-WBX6R+ck#_HP=6@>g+Hv33THbGd=k9ibkGc4f7`zqmiR@#?qQRS52E7X
z<uxOIYrJo~M2Zc^8Ref{YjU|d_hO--_l}}(l2s&&<x$(Y#q=uR$-3r{)!d{x_eU^?
z%g-&TCLadb0^fu1encjFQ*FlpVcukL&)*)kQ$2JhXS+!VX=A<D;kS>&y;t#=+>B~T
ztU?$eAg5glUubLJvbJPzikCP@a9cPKH{sRVtCGo2RLG^PRF9ToKlMy{V(T<e3$|(D
z4r^>90;zF=6#(nZ1Sdf}-09XtVr%ob<yTp)_!=VKciCA4v9($vdEIXi{xor4&-nCa
z7}{CvTdvPcCb_f<*XvtNfQZ4&pZgMkzT!qL46GUQ7Ek`Pxg?%|q+sl=54klY-)mu_
z<SivCw}tdhW>K`?*S3L7*)7D5P8$)oOtLhXwIIAb-s!M;$m;<Or4XDjGu{gGliv$w
z+X!cw^X+hoG$53&N;GtIA?6#V4E`E}y?&&0ST5sna<8~6Ug$Tw&*9R4xjb9m&ntGk
z>BX=_1Bcr?a9<%ks-4G;)WI5X6?tby1<#W;jPFG@MXP=46Ua4D_p|H2_yRJ3<_ZI#
zv{P~66U&Z}`HHixgFiu}5r$6#t}&lqOfGOm8%^UUfcp4zpiv((k)TI1+nk@B>{SXD
zpfw~5#RKdxI>m2;xAZ-L82pE*)r7;5tzCR+n#!6_*Bz1xw8BVHOQ0#wCrOJ~`8}d7
zUg3)J@UR)lx*7Ft+&_3;*rX42%ocIW6jYFS@>=JX&NuN;pc;6yvoTO;Ek6?O!N{`p
zb@H(yJ(9&|Z0x<dfn;txnNsAT*?`2Zw3X0#E!3UG)S#dlp=N)K+7})ZX7Osk;JU(w
z@iyRB)^b2I0jaoBY!JHLPkS8;{BaE=PplC9ErF8(yJXiK!PmYVjscN>`zZ|lHQebE
z-3E~Sqik(p>j{lBQ30<4->>kU=5yg`miw_#XbgSBHy7W@RFk~7TKSmZmy3lqDy;M-
zvx(Pm!^Df))>!wkkx^@WrtkQQ`utUx=lq+}%KPAMi4aG0yN~*cL+>#EYDzvX3>z}t
z82tqs{IWk$Tw_M<>ZF*%Z1@wyK5J~VS^e5s9uYsb%h{(@GjN<Tqi-P<_Rn>vhc&8o
zroMLUvnyfTdbcPokn(mYY(}kfCyMwWN!#B4mu)vvb^a%ZBkqK{UPGP{Tz%5g*CRb{
zn*HQjTc}C0JS;1?vH^qbVWdp@{9Ju&H7_$i=B?aoE8F#UopQho?`h2hwxRcTxXzf>
z?{i82_K3>}@B#Tmcje|XhV1rV)suqJdlvvL3=qMS2&Hx3m`i0Bo16%#Q0HPwxV<y8
z8+gR5UG{!XzwJ_Gll_h>f&1tqETQ%8#y^(kf#NDE<zD>#T6^B!uOuIXtI;1sqI{05
zI4{JS@YIcwVtNSLNT3(V1zE*FlCrsrp!0L}Cw*?dqMB7;paQkoZRbJRUm8kJI!|L#
zNk=^-_y94rI>Pz0h95*bm!GJ7L702j4?}d;2&llbcmNjgknr&D*fkJ4v<x;T8(Cz9
z%PCj`&qoz5!_Up;%cB;T?(CK>9<p|Md?Izy^0jZ9HC4+1CHdAh>6+R|KA8cg*k2B<
zD-S8rNiF32eRX%V<Gb`=Nocc0m;>Qql{^xbA#*=plQ;kXUjLB<1B^*4)z@NfqZdU(
zf(uuax*?Uf1`80>9#DrV4;K=Wn9jd00Jv{haO@luF!W5-r-Ofq7ZE%lbZbuTutEPD
zd+2I2DQJp{Nd<=%r{;`sqlA`&I8;G&!plo618G)nEuQ39Kj$!@5(h-Ra@L2*l+_aV
zc}A0e`;)Z@j&kmw4t6XcxOukl>^^v%JK0G1SDK*+HC$anJ<h-ucBLT!P<23kA<s2G
zei%aAnR!ImI7_$>6sxz!>0W{1T;fjV0TK*cCnwo~_8ePG_xjW{`XS!MIJr`r?Z<eS
z5bmPnd%}U#$!C^qd6vJ%tZGa92|NQcaY)K*IS2y6Ot0)G#>L`KVO6n-7Py(}M)#wv
zxr!mkALOza{ten$)#^1VIPEDT7Pfq?yc4aI@uBhu3-<fK*~)rBYqTRGRu^SGLc04-
ziLgLQy7o+c3QZC$X;dVZ5fjnAjwsS2G1RvNor#L??(EKoJF@oXG}<m#Cx9*qH=PZD
zUcO4(42Ykzq~Usq?kYVPeEw=Il0b;|x-9%%EcgJhQG$o|_?^qWv~e=;TmMxVQ;qd&
z4-2*kJ;6^2*o!QAU#DBooBu=K!n(hM_*t~~^F#(zT!ivDpfi;9B|N}boS~%QS3;-W
z`_>;1b6gFdk@I-O39srYw$6<kSlArs;bLBf$??3v9u^>;6%9Lb)%0VMd(s1^JItjQ
z^F*9}AOpL|psz;CkW%=-Eb9!!{vD7f@kBdB3=~y#3psVOg3bTQ>P3dBz;^0!CpqW5
z538YjbnO8w2{q>+trBnzBG*63TbOr7Bd75O;nDS1oYzKXArkhy=8g0`bi{KGr@)h^
zm3;1fWJ173F(O{R7{!1dzf(ko*<i*y2;nm_j6{+oRQyD4!k@@cQ~CQ%i&o2-Z;_c#
zm)m#D9WSdQNm$RI-bhT0#@sto46M(&c%WGK+&_vs=il0saw0$bMG9J07JPcRX>w!K
zAOH4Mux`YKUSqb{H4;HCiDQg##n~I%j<*{f(Ly1G{hM!d3nu${>zT@b@lPu?a5Emc
zta87`=lf=zQ_UJOtS)8qWL??grSzAU8$!R}Le)ub`4Xvg?!T)1Dwlmn<j*L6lJAqk
z0Ov{Q<d^&;0qCz&<)~T@%D{U`nulD9pfWJo(WSq4hWkpw&zZ&EV`?m6D3<c1abz%|
z{x^op#!IOG7Pg{SmY#tp58g%mN=Z};3OI|mJmIcN^V>8@N)kx-UJNE^IDHA=d3~&N
z+U$4ICpuq-BtP^WPgV(ClX^WeJKYlz8VN;0A;<0fsN8y@j>MmL_63c2jH}B!=`hU!
zyo=0D$fS*mB8XUJf41Q%{zO~i`6zffW!UgsTkoaucpulDHXsPW_074nGAj>7!;W3&
z4P2sIsFWC_n>Pm;X8;2Fi`Q`TsfT^A;C3cqLE(G-mmdt^_w@vI_6QI8m4MOw59^j6
zfT?N$E*Gm{6Z$ob(%qIFFiMIXIS4NU9rPc3xKp7uQk0&1>kAPG@Dupy59_cUds@6t
zMCUX2+r4*$i`#_(>Y_ZTNj1j0RFseHfBnAP8NQ_-aYwe}Oes5zIMDj)dL|l3%@eHl
zlP#0aVZ+bQ*@%_7opLQ3%OV=2uHM&jb`qUjuh=eSyXepxY}n~;CKvp@#^#zznCT+%
zyZPO}Q2in$(YMQ*;<eVAwaO{S5YjohOL6Ne&jb)h{+|~|lIw}j^cCh`gQa8nfUlc-
zvp+5S=;MVj!w#d~aeUv1`u2W=1Bb-KdqVu6R|vHA_S8MnT(7yN7;B9n2BX)dl#{Z6
zFT|%%XCuHV;1l*|D7r(KQsH5=DH9CblaHFn^IR3=nMYx*=^ST_6xFK**XPo(%=s+t
zxU=0bOBZx0?H@#s@|9dPFg}qZ*|$;qwc;y3X1`IEUQE})TYY-*=62jMU-DRdl4-N;
z2CdOR#77K0?+vQ|Bt;I#x)F>>Hz3m*sq#c$1Wmj2mnYTcNbTaz`Mfw?H^|MKm`6){
zM`>7=ek8(dQy7AX8w;6|#;2lIIk00zk<(hGe<EFtb(Lpu_JhJ4P|X+>sGnG9l@y@r
zWpAod{f-%N&%QiedhK+kxGxvu&57sx#pzM_&XORKDf0K=Wx%^$eRvl+?R9E$V&gd!
z<Z4bC$^)8Yo=X!%ZcmZjE+=!DJsfo`R|<0POxbRam4uN=raeCKRuA%S{|l{q<_c6q
z4kT=`ju=M1XwgHfbcK}Q_L%1ff8{YR8caDtOISuGpO%GRw*d(!@l&I*+|><OzBp`~
zE<<r^aTLU~eDi;XjRrcjlc7UK-upfbE#(1lc{%I2E2Ebef<AY8Qo~^{D{fNpa(;V!
zl6hS;q?B0Hs$@qDUc$pm`jftonqru8@tXe?d+?eL=S5yb0odrw*5;tdNw*g-EV%)v
z;7{k^GPAA55o?o*i!^}>G>tpD$PMt>ag}F>V|uQ@DpzXgVEv(ae1d$}=WvN7qi$Os
z&#7BpE7t_ml%W~CK)kB+=!qsa2jz;$wQJX5#Z3Nu+FXh+bsx)sPvxu3ju?YE_v81P
z&_FVEI^My8L_N`*_rhC>%~~Gtr+Z&Yl;o~|fs(4C0qsf!<S1DhKD>uJ5zFtr`E?1k
z#!tL_h?I&10N;>3f^!KrH{RIar5jet;c{-1vP1SIveOKP+e?707#W^A?L9oPOvjQS
zOJjCNt8<PPoxb&-o-3W~-8zYMJ%(c1088w0VF0SQK;-JlN4Clu#135YVAS<QMZ|VR
z)H|9E6K%po)?4lNYt?vvvQ3U{=l{x;n~r3zbuh^#hX+G{?GNuh8%2>>4B90fYF`FQ
zk7F*rs<-=R85y&ES(1#-^+?fAvNEN%LBP1}E#L92_-palEuE_jHw8MVz)2YpCo$tS
zXlDqiW!`347z+<Y0$VAgXYpwZ7A1%$)0GfZtH^@JSYO0N*56<MbMy;-N}H;F&ZrwO
z&+*vV>-bH4!?TJ8;)8<;?}YR=;rBv(GfBvQ0!+%6Bp`Oh?BJd>=gFmnDY0bsfeYy}
zU<km!pu!jN@xIH(85|wIYU*%MjPDKKL^J8f{?N{%JsBEGw21oQM@g)-w|*D|vfBTd
zHA`^#Fz(A*08>TA5?Uu~BQV{4LA<Cc^%O@gQMbr2Kzg!JY)~Y<;vFcwsvcb?f!(11
z8EGy30m&b8gYC%qVFI+O*!$s<E=934c-s3d&k4)@YDHLNHPCeR!K0L!t$n4SU>EmD
z9P3^)M2l3D74%5LD9n~84sWLY)N}Na9D4cDjyv$Qqo}bhvBTNF`>srsYNMz+F2h$R
zV4Nbg<((qQSQA(RLsHv6`htM~?8BB#?=R+4S66f;pc%Z2jtB4s9?uP}k%H+alz8^a
z<FjrPN967*KA)HR>4Dm=$r6($Z_O&mTuP6{tEiF1i>_ZpTJskjJ-HvU1~aQ!^&z~@
zd;tn*8R?~N?XffHM7R~!nvyd_)FWF;UZgwKtg~+^<{z`>bziSRC-JI+G_veKgU#fa
zA7)5k^vTtPjz?Z_)Sbblaz&QZln6*}Fuk{bmwP9q*Q<3CV{62)o}J8JQ?i+7tp>e6
z(RzQWs=;%1UO)T5gm<<74;V{oL;K6@PcYP;j6xY>a#u0^nwYhIKS7J`Ks2LRV$4y1
zB;_gTo}R5>ODwmA=9Cpt+`X-B7s(#<*uK%NC}K_o_$%EeXx6;P{ft`8wVjY0&4<Az
z1@92-ebm<Hou9S(==H0Nvh(Qekk(gjcd&tdYA$3Av;!Z$SaPEeC5M$UCv4#_{MWK&
zNE8J426)ALR&8_t=JkY1Du4l{TvnRoMTNvtLtO0*G=_=D;DSELQf6v!>dg;%yRf%l
zlDz+E@4Lg{{<eO#FnVXSVGKglAqWOhM|44g=prH$oiG^D>ma%$h#q8=Nz@pfAtX_v
zB#0U<qE7Tk?l`}5?|aTY=Y5~|{qsK0{p<UDzu(!<?7jA0>$}(5YkfY$c5~J175tPl
zVm5Zj=c;xE!-C;;pa?-)o8NU;tEk7^{8qEH9lZn=CXcPCCkIr209^!Gw~AbvQ>mt{
zoF9%izrzlb`&7U%cnjTRYk}H3irwqllyoLSS)>I18`)1wHwe>yUd&^C*j!nx<84(a
zThi19nhG7u>hk1{RcjsBkR5IWrVcWZ_f10Lz6!ou&OJP|rf8K3_jjfshPOVxUcJHk
zsrF1U`_=DIs6XpLYaG2{fYJJfpT7m~SwTj_$xPM`Ml95`OU?`&c;5}9@^RDbsmQN9
z>Qx@^VngZ_WZ-skv43<PtDJF2g<?9bfN1SK?S(|$@7>&Qb=5#R5gp=bW*A+~r&3^s
zfN?>QW+rZvS#EGe29@j}LDU<ZTH9K8ybVEq*afX01<my`*`bhx(^Do#NjU#vtrzd%
z&I^At>S5aw*=tfPC*2)f`-eyY2Jk|6v)VBAkxkD{^|0sdmo#QO65p1{sK+w$J#fJF
zA8S^?QFma9gB9R?<ijR(5fn2LQr`BSFKzbKHEySL4$|7;FCXLfjM-Q+gP+hVnZ4*9
z^HEPh{AMUgf9AWyYbU3ih=6(VyEZRmV{y7bbG<KUY!+?NUh~KyKa9M4S<TmrbF5oy
zn3j6oDL*bV2^Fw<FacgT|IpJMOpi0E_6q99sb6N3v7K@-g05~Xw}_jc!kAn@f2u_-
z$N7(D%#|H><Z5)H0Dta1NJI|$^s#uP=o<aXzM0T?tXTG_IZ~Q?vL#OO1hE-9xEH{Q
zC>Y?cl;TU^&_DvOVjG?8dTestwjpbzjqcCI@mgGDkpan%1Nv?m-HIdow6<K4>FdIm
zS(e83oyG9!oBUbk!h7*t{hsr;%~qO8sRDWrohlYn$guoo44b!5Z_tGP9Mdw)ksWAZ
zGjvshExWzpiTEoshRK1cd#FVDQ=Z{22P5mPO%C*{yryE~pnEH8R%bP^XN>c?wKLF8
zX0Mnd*B7LYTPOR|bDEBlG@WgiG#V9{uYCqMo-eXeqAN4x60B;5mQVvOLr--@%mGO<
z*8Ir$T5j{GtfcODBMKtk(3doeWRSHjkc;Y+g3upfvV_idc&DmoopWmRR61so3-0LO
z$NK=i*Y;hnWD|>-_Tntf;4C{6{AO>1D)6*AO7Q7QR7;7Fp_)cKQ4qjH0rtT7F0Pby
z9SxgY-9v<+`eRYS(rex@m_-hq&cMz$dls412T@}|pKopTbgBZO1&prwaSy;(X?qC+
z`}2~kPL38-A;WtN$6IvZDahV7%$5^~Z!dyLb>8)y_0UXs6*$qPm?VxsaD11UzCw<%
z)k|=MM|!qHZ;$Q&h;FiYQA$_GwzyS<<SX(qs{Br7a+)7vQ^6a}Vh{hRci_dSW0}1B
zZ9-0$)~f8N?fuf*_^SYHuY@^A^ZJY_mvV5&#527koTNOhD+R*?@&5JE8%o<7EbNU{
z{mE^o602WbfQ!b-#k<K+CsBX<YBa79M8Ot}4!ddoSzu)J$bg$7>Si|r)?fAY?0Ja)
z2ZTJ?<>48(=CRu;Avpp#4AL}@SZ4F}`2Dd3YbLnxwo2)&8))>+TPAcsIR2>p5xpIJ
z;GJz$Cwmw6`@zT<kT>2G{qA~%7hF@}MNZZuJ9u$T(7^U|Sm5~9dF8$1WrEtZ_=PFk
z2>8z^y_()KP#gIiGx%LJWJULqwChPOcg#>#l1$B&xRMZHx~@}I%WXNzVtYDX3(8?3
zXUvgm;Zk#rzp`%M84wb*R)(Ooy`qx33GYlUmU)=iYqf%kQG=A#<0tO}Kf5h34XrB9
z#}OpZPD$zFOb8hD@Pdomhn|5fzMH4JZD+u>dG^icdYDTGM$GI|>2X(F`6{p)c+pY4
zG7g<VcEfSuC}2?dwOVhZg6BA`w%X4%(_Z}Kp>&wCujBH&#i=yQ?FR)sPbTrwg2NSb
z#QbHRAZ$#`cGqy<eR81xx*8GY+|QB3OA?f{JG?Z{%YS5incFHk_md@6KQMUa==IOA
zZ>}vx(td*a*KaFCmJvqtSjpz!ovcqQRx_Q{GtY7S5=8!_&w&RiGqw!}vAxSG-xaah
z5#__>iO(S!1&jAYZ521Gfa#VS`_1Z1dY2Y{2c@2{C3KDfns=!;b)iF7m?8>9tR%Xh
z0#7Nl27A7h!2q#VR~Nq5>cnIEZ?N_dI=zuW&@m=Xo=Bt%3w-3ml4XE5+Npb-BQwEj
zXB=o+_RdA{3E+n`Md%Jm_k@zuUU~A6e5~j%_GNY?m*N}|-OV@|3Rr*eGe$bpDe^K~
z2u}jA#6kgnsQQXImiZyW<M+^h!>^BqH=pg3y3n}o7Ax|_Ot3u&;%PZbJ!C9OhQ1dj
zlLC?*KKb+}-Wu)*FqS!&36rv@iQ%VUY-0sJu@U9t$}!D3{1fG_J*s^y6HQlhtf6$n
zF7q;}2r<9G(vOxv8EHM%d3Qste;o{O?d)9>MN$fQgYRYx%tN<qv`uSdZy~k!gh`qG
zl(t@_(cUZ5wp0fCyy%#aW`5s#aBUI;B_pAMGD^nhC`Eg(5?P$LrenkVsu7lo@b*h>
zFP@P_UB&s8Edo_TLs`#k02=Ks2N3TnYh823P4)?MR)!Zu8s@L4v4a>KdUD%S$xzkP
zX^X~uyXaxY5p}>%>sbP-qI&dGtgmgQL4_&jq>sZlN+;Bu1V|UHTtZboNDL=4d+)g2
zzBR&vcUNp*{_6g4`MG1LZqXm2=kRLNpj5-^4g{~#?xR?jFt}*rP%44<6-?0H2|bdk
zjH!sq@isWMx&M$qr;FS<556H1bqU6L(B_R!EK)E(iF=*@#2z)sjlZ;y*bzF6HE-Du
zyNlBVzErji`q0D(7Km<2IGb%yoZ3xo3!E@c{)FY8yumW&`z>E-&(o3;zQvrnz_dd_
zOnwlS@UU2L>fEsZuwUo&L9h?xa`l%xCP!f*tF$F0_;i((GdEo=yDzlHtwKzDwl4D5
zPf3dI-RCz8o!`sikK+z9)&1CVRT}dkLiei658s80mk-T9*C1I*nY%PYs-1q6J9fW%
zAUCQAIDd&yv+alB`f4UcBRXk+)231i4_I+CvV@${r?J-Gy3I}TYw!;3l%8@9)x<Hw
z*fDqNPWY+SW4+P-Be0hA_oNVzd2Ad<%~tgfe9kin5V0uN`DD*_xEmTa1w4d?9V-{e
z6xY+Q+_flZ?iciMW8gJV%VyZJHh6A*XX>~F8<1?YW}g?G7a%Py-P4w{Je>i}I>370
zawt2WewDI^X2R|c8Dq$2<!n*xeKkDtoh5ye6c)r1&F}qBN_5*zgf^@)RFIETc5F9_
z-5$}+Cy!YJy|o`LB+B2uB{$#aldtr(=l3D<mz-KfLA&aj9XJW@E7Zn0$oyy1Jror=
zmX74ICfg_9;}X5=ZZP&+SAaeI?CgB?lQ5YdbI`A4{&^#1d2#*uo`@Tlir$!nVab4U
zM$U$@hEfs>fFt46NzO5VS3{7aQyg3Xoh^4_bqvYXVZsFclz1X+TCSFvF-k*68^$pg
zeVW)k_<8$RK5<ieMn%e5)sy+|3|J(5Yf~m2R6w(++I=~CdAfua|COr{l5))%FUyvs
z0@RiG{B?=HsN)WOHyXZEcu_~`5zmG3EqSdwS%LNdjN}2&>Yv1sMt!NZW)rL*w?dam
zg(wQd`%_YA#EuC(d{UUQ$I*2eokaV~jHyts2V#=QS)k=Q0xI}Q-Sw>dB-~Pxj&>tI
zO_wy-nA#dpgI*LZzu)oQ$q+7fDPuj-v*~`{r{t^Orf?dnYF@{AaZJ4B@qsJ$zCWpe
zrf7#*TC!NHEpX<(Co4E~ynvekKg&gYH4why{FOZQb!Re)>x3bwN%XDo=ak{35uF1+
z?eN~&Vde070M7+F*2iZmt1ov>S`)~q%L1L~SMXl_?91}P*^iF^TMwc;dt<ERr6F#3
zGpV0q6R_UPRzb>>1`C6JyY@>>%lfy+#_67#CN`6S7{x9}+e@uK(z;}_R$vJjuD2&#
z&2y2^xbu^9Giifxw94dtiJy>2QSx3DFlYMh{`SK8DCcf(S?y3pHdOQMby}>K{mS|c
zreCs@a=)EPAjP#}^-{>EAt-wxK3xSs+Ee~zP@R<a<;*(P`Pdzv7~hxpq~<A30=s<z
zT6`PYGrL41gujMVtT4t=--j$`2eBP$^vaO9f`BqogS4@so>#-E6euMewN<ohEHq^j
zm(V*?Z48-@`1<d|ta%`nQ*9Akau6NJ6}*-h6_62m?+A4Ct9^?3JlZ=Jqj=2?U0OTY
zol{A7<iC5L^lfcPi0X3LQgk54!Ct@eWfs(zwxVMZrnj#P1Q=0Kd%5y>E8n6=FEpMG
z?jL*{k-LHdO%a{n;JY6urxSdluo-HL=gAF>F8r^lgBVU(qr>gaQ?M+q39LK$uuv_Z
z>L0$$leA{;MlV+HBqJ;b1H%eeD|Od38<KWv75>Q0C0-^YpIljp)HIuq;7LA;(O)^S
zxE-oHJs4PahPkn54Xkw)WlHXxCD6JU9P_fNdH#qcV_0F(AHV;zues*b|C>s`cuI?N
zyzLj=x%(2`HtD}L&JdeTMXz&O;>ZT4Mo~r1s8DcQkov^CX$LEBZL{(xRT#+nru>T3
ziCJT?J)4hcg7xi4a)dvu_-sJcwNiZl<6!@pnW`x7GH=aE{8eag{h)Vyf<N?oJRx!x
zZoHs<&7>j14gGEINBv9utBn5S<6>ydppb3rcV`K2-zeTmKQrv$GaaOob9{oVZK6$(
z|D*`BRxM=sh>@(4WLAAg_Jt6$6%8o`i6Ny#?yQbO3^sS9w9C06Ti0Dl*<430^O21I
zYX;%u5cemsF;d-{B9a_bGT*JXU)giyWOjjWgPwIMz7m!+<egKKuyxFBjwMkFmLH2P
z)fQm>+yQ8IB3~wP0l3g{Ojx}I;|S+_b*r)NtIwoIxJ<4x_-GeW%SCw`L<#`o2YRC(
z-sJ_(GjViB%SxGnmM!RF^v-Kw{u-hOdVuiPzVQj&xtbnuY#)%nny2Ida%}<G6*LrN
zqGvQupkIB_ae=96x292963j$1Fhr)Ch%I>CoSOb{fT<_uvP`K=7%*P+M{1lOF|ZNh
z?H@z4W96aYoz$7ETAWwu@<O3(QIC6-qkz^6p4lt&L0{|fV#`vP6d*aPd=)%NS|7nm
zQT>1xPEjhaOysO+^@jxt&9kQauI~W8njO9^R9{_gibDxXpYwHb8oIJIkMFm-(e2zK
zdy_laMuF@%-frkV`mvMm|I=s^*j+_T)os(xjP=4!a&)tZ8GZO<y0sq`nLhJQHErql
z<7k%s#+$c|ZY(U;KLACF+KwBipbCVf7PNs55WL8drru5mw|;Bjhohg1c5-kTk+C?g
z`CYYuB2j%!yfsC4jUfRA6N?LgEykSS*|%v-s-HztEz{!-4wS_7nZjh-eN%nPRvT7m
zIFDxC3?~B_J4W21N%o~+_}wQh6ka`gJ~&O8;lTGeu^6RD{qM2GOXxKhI`5>%{5{RX
zZ)88dmEr3PuUib!IawR#o@x0$ueL_;EFL92fX*ev6;}iN^KGxca&szhMRmbcg|vHt
z1_-j1K1bTwy=e5{z1rJr<0ZK5h^jMQYw;FAmEet*g?0`FrFMhw%cwW8J<a1CoV86m
zc|Ow}NouJ+D}lcsWt?jrtod+9H2*I3f;J&AolL`1ztAdZOJL@ST5wR)2`+`C;J0Ci
zCRCT-iMRBRPPgCW*;R!3sv};qSOSXgS+2WnCFJ1nW{w?u&so*vkBB;=S51?-WG4jz
zV?we;M1^`1<1NfrG7jPf*YDg(=m+z;chb710$j6J!C8IJbabhMO!rW4v9!%aSF4dn
z$mbRXDLBT}ew$qV>7F3=naJDn`sy-k=Qk-S#Tx=j3*0T?L3bMRAu!wIJjTSCOC!b(
zYi0~M4!nctof0fc;zdj642ha(7MI`JJ0>w&uEKo+145hM+cnrW>8%^cbz-vZr(0%v
z)vHza^?7=~Kp_W{K2^Ytp<lK8%do1~GLI~{QxZ7}&y>erPV{dKi{dwOMC%p<2More
zp6v<6_S#eJlbnjpPdvdUfB(4rTl3<ZzlmD?Bt%9F5pcDYjLxIciYLiP>>GhOFNT|H
zu+k?jh?d4G`5V}m=^RURm1%CSgJ{_}s?w~`%?GVxZ|+A!{bQWoKPSSvX*)g^B#9D4
z=p<9~I%%_&INWNrjnmK~v1k3>Io|UvSShbWK&ypW@-m;9W@g<;3X;nlD^i>6ZWS96
zOi?e>D^nzKYQ5fLm@n5aD{tdnT#xq9Ap+&1zxPEMc7K)oc&!A%c&*4bK(jev$wHI9
zfL@~(C{EGmS+t2fpUpC_kJ?rJh!Y$k9ECdy%LZJ|_rE!HYfwNak=CIi)wlEJ@L*aa
ztM3iN_B#|&kLw5S7HBn>$i}ig|Bc>M4vc;3R(T*kry3dB@THA^Lh41U-5%yl3r=Br
zrB!<NqiL9%X-To+NnLvx<4wl?icUgZ+vswX?CkTbh8tTUR4R;7iA!LXNb-Zglzqzh
zjag?-ylsKD{#uB_>F>v-T7dX5+uWNbA6U4%O2%Yjb$laiEpo3&rh(SiuEb%nx}NgJ
z(@1-X2HE}TQ>N>{gqLsUh)H1G-!@k8^s{dE6=Dy81e{5Q1Rl`cIKc)FmOW(7bZy!9
z`NrM-Kylg>G~ba`k3InWVS4=r@1_R6>;CiDGK_DpilEF)IW;{9{HXX6IY^K(mfdNE
zG7*Yi_C0&i(wiWw`hyo{^iVn|#10)`pPJK@;y3>iQ+{0hvDv}yXK?)B6DTDpID5T~
zu&UT$-K5KJCP3TstS%!mbjM<n2AkSSYUS}*b-pL@2-Lqlnk-3eoq9TJFjM(B{+G&X
zi2gK+BGf_lKz*`RmccIyv@Q1-&Un8dGJr;wCaVZS>4h1JJ>~YRK{IBhLI$tX@fEKz
z9zcKVs8RIje7gdKr}LZaG#3bZ3AGDPBxy><UOfsBNZi-kB_pN^Ry+aC>ABx;Dozfv
z#^~TAiD*l4y!VW1eNFFUK}p&cjC3DAZ<EB(AY{=~nI>@j!GJ-9#9R&!{}o2~ZpC6D
z?v>iuV~fWx)@1OZH-TxBZOiVAb9uzvY7gmwqDQTUAEz3$kiS5Fjo(vadGyx`0z(y{
zEHNRyML0}a9FyyZ9Xzabtx4S=kLm`5oMf7Z-TnP}aLP1lU#q5~e21Rdk2+CBI>&F7
zlkH@5-OLrLAj*gcFO5pReuaeb@ND>{35AWgEk}dAM(#pHk$v<Ht=wobM#`$;X)0YV
z0|^Bdk-KC`GA}~0EIPSKg<$-_tyceNsM}&}06hl`a143#vL0zj!4eS_LE^%V=ET#>
zFh_^`TmsU%=jz~-k&zE^@q5t_$z^Wz>m2!LQ8cxoEYhhSXn?{bAyu`3APc~9orJbP
zf<T3rVe-muC_n`0a29=cFv=CW73H32NY{M>@0RP0To!x5@5GH%K<fulVoo@}?2s8&
z#JLG$?$bjgAk2o4aLfm5TXM)?Zoj{a7_FbaRgq|CuXpsJ3zmCl<LRD=)@PQiCvQ96
zmF!fGaXxx?gpXqWc9PG;a-#*0nirV+bdZ1c+a>fiF<#PI|MHeZPmF}TYmWE<`{_W3
zImWJ!0+L&qmZM=a2~G<&QN?LK&~~thxAU$(fHTId%!_7E0#0UWDPBE@_x)N&-IZT(
zBP6*PAjnPrEqhKrLLGqxrb{0tHx}*>XVs%L*~>mGZwp+1+<!l+Jt~Kp5GUTr#N?A2
zXibpayCruBWsp&HWYcJA@R=g4^MWBRqR2fO?e%KYS^Z`B>Xic*m_@F^@!_#n9VpAA
zU}_ktwhGw)u*^5m&l^LgN8s07Qj6hDGt8aYt*hj+HU%9-vHQi*={HMhCJfsS;$}HD
zjohbvGCTP(?Ax=O8cInrwmW29qAt-HA;V6Y^?*HlOND;yj%7ge5qQqHA+RqGtr@)!
z&5J+JgVltO@^1UZ4CerD2Wq0%0O?;<-1J6<rs4MKqWsoF8guRW^_K}!sU$cu7g~fN
zUifHx<y>ZR(RI__r(xPkTFya^L(U#*oVpiQCUHuRqApR=9F+qdh`rR?iPaPB2KFx`
z0EbK?wIP@Ge=^UD2M8i6#dzGQrcPd=o@4C%=dpy-+{?Ktop8|SFo(4PWc>ARzg;P3
zn=ro@du{h%W!|GeG378ETyt7n0*e<H3%2aw=YlMj0B4ZW3)5@wR!(lA7=|kjAEY|e
zMm=&~sJzPS`lHf6+V$WJI*u8eNLl!SM9ft*guc~1n>{n#wEDU>z}?#>wWtJXDKc`1
zW|)^P&Ze|3F1U2bTZrzDKYx#C8s2^WRk{zZ@zK5{nW%WMOO+wtB0?sm8}S>p6XGD=
z8h>t2M9+Gr?;(7uG1hhA+)|yLuYI~qmT;XIWfCHWaXG*Xli{N0L_kL*pt;OXlCG3U
zT}bAn%iP}i(1j#=iQOqV<d=ZO2868+nun45rWKMefObtafW8}#kg*FWk$7eOK}9b4
zHC6<c-kt2$+X?8SqIvx((V1soCa2+C*>u~y<IWJIG|uO;HlIg0jDm0SyS$)99nceJ
zCJ64$-~=DsxEAi6b`+IHT9A1~t7yC8PBuZ5=k7bBR^i^ruKpo*Uz2njJt<f)^HOV|
z<Vqnx;8z;$<J}6&kZkOa6yUcVBId1V@8xsa6@#(w%#LWcyb(p-{-EuU0q3!I&w2WI
zz{SjOTQAGIKS(sTF|na|wcQna^VWgik0gnn*<KEIQFz|$Oun9M+CW^lQNTDwYmo1H
zJMJ!1xnNMXGv~5Jna0zKJ*CNhpkaNC+Kw0!Rumt;OJmt`6X$p}37p{l6P4v!u}|b`
zs6Mc$60>xEz8@&K0O0kCH6g%EROiFg;Mm^NqyRSu3PwDOV$BTg=c4!H$KA>*>2S0F
z-{vF0ikyo&z6LtmFxiHc2y?mnjR07F`F#!8?<E}-?|{k34}B;yXq;@?{)&O^4(_Gi
zwy=~dY`Gpr!v(h+wUWbwAHMqT^mtwmB&LfG07^fgC7)L#m+sR8vM%ThsZNxr3U!V9
z2-k|!BDZyT`93>e4;T;PZ%9^30BAi+@>fkgqm8g2*{(f$YLaoWB>q|%A#9`=7uS;y
z&TWk*V<1%h!$WT7_z-(RQ#LFW?;KS$hLv2@OBQ#wr#;f19PMezO+n5QTP1P&H~5KL
z@x1KUt$4(xov?M@ftrhPnmiE(7R2HjW1NW~*w#`Ml)Je;s&QIjs&x|Q%w}K>9hCZY
zy}5AnTbvL4X)Kk%Q4j}3hhum*eJTt&9}K9m3Az(4*iDu+_e@cIa!EhRkx!E}VV{wd
z>RZ7zKsfd~71b63eY<FIK?#f@Fkcz)o6!2hbU2i7q!E@fKBPo*H}i3{sg9^*tt4Ab
zjZo(9^w0&;DHjEh2k`DyyDjZRqhZTaHx5!puODaPU?v^l(6vrrVPHfPKY@G-XabHd
zdpn%wBhNA?9W)|v_W+ghl=JTKb$e=KRrR?N8f2L6o^rtcuD98BePuN7xWa1bwYW6h
zWzfcF2gXz(eCYfix)NYk50)dAiBzg!Bna@}qw9zAlY}$$l&Fx7=XgXqbC-%{2&4QK
zfeoC;h3LXI!m|cHxCoBKg~O?V`_7v~9sIRJXSJ(e*k9as5yAV;7`L~Py2SF(IRg5v
zR&gZY3hNiIKTs^oBb@PuNYf0ER-GPjqsZ~D5fGvqWMw#628syx1rpeP7aUY&@Ys*D
z<pOUG#K*jr-wx)&ILJTxr1stAVgHCRD5hOeVXHbVE?1S9oP00+_>c?p73vxBc#TV7
z4RJrU_Gv;rxpb5uW6bmEq(CX%@XVf;j5xw&RnYG4P1q33H%j|B54aKcOFP67L3?T%
z>P8qGJ`4HV601JSsx~5iv5CH~9`S-oB+(3mlA#u>4>wBIT((FeL^0Y{26`&6E~fSF
z&q*gQA1`Xe>4+M((^{$0@ozABc7HDB?cwI=<9y%LjR?-5ep=c-lxA9*8Fxg5ewCTj
zpjtaSrm$^7YA%@y%XJkKZ}a?U$EBOA5?%A{jEJQ_CfvvJc_zMJz-jjzN_)oCSHDx>
zhpDk>o@6AWbk9+z#tjitX-!WYxgE(T`O>`R0~s=r7|--Tes?=Sl+1HkYN*J*Y>jt}
zc>PX{!dbyBfEBJylm~n7)_^5jOS?4Z<N!d;cJT(v6ZQU_NH}8&Xx60e$`N0^Z3)jN
z`~e*Q-bDMMQ2_XgQsf>fh>Tm#ltnh%&jC-8`E718(~rns(ZmPY9qCyo(PHc_o;Wm0
z1jljB0FL_GjnPCrL9V!OEk%KGFXKWy3iH^YAD6b^tz6_>Qx6KjrEcg60(kb`k8esS
z41PJaO)e`%){H~&hc4HEoOiO%O-)xEAge%$WB37#j#)2hHY!aOS{u3|-lTgCg#WMy
zzlQepwZDHY>qS2i%%yOrWids3S@o-iiW_}zQzCYn2^!m(l-79fhkRzB;`T<%RR*V@
zeYsM@o|Ao^G6U@+=oKJYf!{|ZyNqSf{tp$nm3gC~KsqHjlWQI@!A(7<b>N4)U|;eq
z>H-b}KZJJMaOm6?I<jz)@{Wv4+XF1Bpw#yLfiEh;R>Am?Rnh&V**C1NcZIr(tLPYa
zHa0@l?RK5!e4{w%C{gsOMSzf3QT$VT+jwv0D8q?9`2uE)pbn9iY;gc4S2}xHnvk_W
z^3u$Lr8vubSMlv=@^r{9vF`iE8PA2G+c|n}lJgISj)D~Sy8gOb#~b35(UAef>J}{L
z_lR{}W`;<B(RK-Mk0w#?5j5f(E8Zz}ZyIy*2d@JKT<zW`r6;uoBchqQKe5WP_OWZ4
z<T~ZLX1b%CK8vynm*qrwN1m4po)iS>r#>MO7QR*->Md<c!58Fv?ZiIXQ8UCTkD0_&
zWX=y&v}M6yFIeRO*L?YQzekniMu|+}m8eHz((q_rvx-RhBO@YTN~M2kSvAdY^g{oT
zh5$(Nm1^jNhjG&x{TD{;vz)=X$z0t~W(IN<628(wG2NX9alzoAySUvDKpK(wsn(c1
z1sqL_P*q)KTHJq7lqVJqeRt%rkIR#@$9CKlVq}CX!xVSPge)rX`q3ERP5Lynq=hu4
zY)z@TJ&c@r;7UF}b9wR4pQhuZfLNuSA8&?l`D`&W>aB`o#l$(%sZ-4ni%49O*#?cf
za@BPO!v!@^)GNrbKalJ3L5q=KOMa&xy2^9{59WGNCyzFjX@t=<KOZW~iV!0j0})X3
zDW&2kNoVY@8TuBwKAk9yf=@HwVM?H#-Z2?#u9zC+$=Te?(zqN)kMMp4{A1gm``z^F
z53ZRrn91Ni0=vyA6H39~e1#_QgFE`FCTpQ2?p;$&3IdW~Rm3kgLdhyGV>RPFc^x?d
zfIx<i-=*H}t9(Q}bBTDI9IqT%JoRalH;g3jY=3fa2ob^=K^XnHzkgBp^_P|U+vO1{
zX|{6=*dtLm)SN=!gEo;p$eitJ>&P>Vue6{kaJW2ZQ(k9df^D@;Y>w6ld+M<cXPQbZ
zLzv^*o;mwWv(>hi2XR>NHvoj5zgCBvYffI(x&DPs17XgVu;pMjU`bearZ)M)##kTg
z;&g8YCrn{!p1V>m)2%<rm-M7r%utLnVXHy#y+`7cw_+jU@;Wl3nKJ4|*aeS7!I~j+
z6|u=qBkV)ZL`7Zm?wrP98#e2PQll?Dbz<Pc=|eWJ#&SRjkI5aF+4NPNmMONfP;<?L
z_nGEwK}OU@)0w8vji{}s)W{DwQj^PN*5o#pFBLTXfRC{`1!$@2h<;@g-nMvMynskF
zZ5N|?S01$ee)c7LDS3=-#WTs0dUUFNrc$&0AgX<@>ULX|7&<J;eUHH-G5oE84JP9G
zQrFm=_^oM=x7+uW7D2-_>`sQ}FxzRuuN;6x63KZ^SRbzc8VUOf(i|7|T(Hiz@$-8{
zs)*NXmMXuq>h2ls)g$^}EqRQhKA~0m7K-`KlHzR5aT?QD7{{IH7!RfQoI2Qvl#ISJ
z9A7Qhr&Tj3kVr!m?6=*rrqG9I<)ca(g_}penZTgUged|-Qk5{G_-BGZkN&;aYv$&e
z{ac}mUMa<Ged6W$e(PS0VY^Y!ohfcszX9f7CBjeZn`fpPec@@ymrEjP6)U)E^kvxV
z3Zr|Hq7Hdj-)5N;>>enk7iUKCeyiDD(^#?ZQ`nUo%;SyAY6Lg4SOwwtci8__8lFIy
z0f1eP5fcNe=|hH1iFVYaKUIFBAV#cUa7#JIt0Vg8;Z#Ilc%hS|HLUz?!$bp-{H8nz
za(gObCf>!2OhNdjwz;NO<1!^;rnvV(Vv*g<EQ?NwSYmPZ)LqGb?i^&HZl4y$&Vd7h
zeMJ8;2yc_Q7YKr;zFEEdRqw=ch&#upRiT4Z$JQLSG3}g`rcbDVt5$8bqF;^Qfo*fI
zQcd;{Dzq{869ucqXnGF6Sk~-!k+WU0J$n<K<(Zg+@oa#%^WlBsvAD+am#KAX(u4R%
zUx#gPLdLqL*e?0@%#wx`WS<lU#VYD7eqj@KzVjl7+$1ngWwxmAPqh~cafkBgX~u%(
zz@5wjJC3p5wF#p+RX@g10T^VbRJ7^2^UsXs$m_}^{T3vk@SMEV-2X7lGfABn^nIE1
zJ#O3pR*S=W_r!yx_iZ@h={OYy(aXN2*Y_s#-0B(2Yd>|^Ej2BoL-SrEyoG*6ztnkU
zj%}x^+^V$y&dC`G#1zKxd?{j1@cR69@BB>SKfD1Hl4w3<qF$N5z#zyAJKfluj5tds
z)!!3A49>Y#gK21(0pw(KdHPm;*?+pPmzCEkdxsP2Z=eg}i&`@_rXRDtOAs|aXGzip
zyVYhO$~^wlkvJQWSR*$%H(Cg)A1yN{W?ulc{%uU5OmUO+vqswG|Lt*IY_A`JB3R}s
z3BQsWdg_)NGBN=_;1s(sF>vlQT)gP&o!aK}<Gd3?VhfpM63uEB8{gBVrZA_K)Pt~_
zf+S==+3AX{#5IEC-<6cLJ^4elRE1h3jEYFhK+cYU$R=}YDsE~1^?6b9r%AhGS3=q?
zODuWshb!#v1|4UOoOl>%uZJsMTemnoK(xsl<c=uFYVw1EUb0qF2rDqd^w&q*OOdL~
z_ivKs=Kq5Q{JT9=NFgag4Tkx7v_;b@?+-u}=VCE3f7{JzJBBdt?t14m_X^vr;Blre
zX3nmDdHt6WB?X7?vzJ|UDg$@N23v)dsoaNmo3p;okjcN`ObFrUp9Q!kExp*{0c<?Y
z1pEX)q3Kk+yC3|Qi~4s@_2PoL6k!s&#QOC&Jld+;88v7B7~!tR@$9cPQ`Z2fyA3)!
z)c8^K^8$YsG4qVx;IG{#FqP!2i<mweh>)vOJXur!bF%mJ$Vb1pWPv)tH&r!++Taf4
z@OZM|O;A6puBsUGx)#w>Mh$#B_{VR+-y}d}!D^@atRv-{wkqy$q91E-Lpx@D|8E@)
zVI{?B5y{`{R*6iO&SxY+I!xS>Vt<#77bJXY`h(}AH>>$Op>Aga(PTsieZIAc|A9{R
zpRt_waoe5XDTLZl%dq@y+!Fh9H`Pt+4@CXhtno*S5D}-`6fWm^Yx)tHESr%;K&!7@
zmX8wlg@~r5luU*sU7Z+r{^z-JB<3D}{2T3A&+37(%<o_MM7q~7rQyH-kU@Z~9~APC
zFuSJKG;_uXkP7OeRcLxUQT>dFa9h>#DSz9a#Oxj1_(FV+JCIKM$NhYPwuoPACVoUL
zSDeyPJcJw8!KmzXykAJAT7g{Vng1M0eDm;vf|Cg0(PBs-(g3!}Cw9&3T%f%Z;^8ns
zNCofDkFxtG=A_qHDh#v#L(~7R8Y)^Oc=mgZx}43Om!_G#8bXkge@Sm)LE!r5Hy3bP
zr=8JStweMU>$V)uzYnF6hZIdRPejDB;6YXz*i|Y0+U<FD#IV#?ON4y>aB=FjN`z+i
zB_w-=mbCnEm7b}KQwE#+sn14zhGaipB^FJm+O1y15&tg<{<A#);jWQ^xQq1sD*ld=
zm<IO`i}U}u&j0q|5QMm>SFl5Q|Mu;_f7m&Bp$XL?!e0viyPdn3FC@|8G{YV}mHSJ2
z|IqdSe2||n%v5tgtq{+Jef;;``PZ>Zq!MGfIsH<8BLD3sL=v7jVxy+*th-VFd3P?{
z!+##~3Nta=YX7tA$A5nOU*0)xUZiMEwxvHb{TEa6pAWJQMm+ho1+~6^`ah7wL%xgS
z;<vZc>Gl7m>;HKlv*AQLGvtz^^e_7RUrbu{g>kxbr2ksuzdKFDv3BkgErmDohpgNG
zL=piPjW+FXD2V<KmXZ~Ss2~4dsfe@w|A0~*t5a8LU%uH9@Qz42O+tjy)vnQWjQn5c
C6i#sf

diff --git a/docs/figures/architecture_2_math.png b/docs/figures/architecture_2_math.png
deleted file mode 100644
index e0c4a9f394e885a90ade43c2d671c6732986239f..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 16848
zcmeIZV{~0n*C?E%r$M8Wq>XLcww=bdZQHgQHf-#qVPo62ZRegodY^ZEzrHc<&%4Li
zIBTuxx#nDR?X$z>WyRpXV15Au1A~_k7ghuVg9rmX%R@tgt{JUCQ($0l66Qie@)ANq
z`0@_6CgxVgU|{0m$*NFlN<*mGnu-!;{?Otu?J$Z3QZRWUz&Of9XbJqh-Z-G}DeVYl
zvkMZqvxusQmJ^L%`8Q8>fTu5c<nZuswd%ZU($DkGoe%2|;}2u)zT-b`auQk|#=&xx
zMiM1&<h8-BNrG#G2Ic2>>zz<bvGn-S<o&7aHm~*+UA(;D{20AHTH9K{fqQSiA`?z>
zZa$Q60fHQOV8poF(*8ek5U;?&E)_ezt3ZQIp&eJ4ev?dgqwIxbOvCS0BpFxP0bRcD
zP$tVLLO^Y=fyJ3~XH#Q=ouQVPe!Hq8h`px?TO)?30f6B!_6m3*$${B_?KQl<F{Nav
z|AdgPduj9$&Ey!AI52|y=UC&rAOgxrp?llXhbh;4)9wL9h~eTN0hFP9<Rq4!j<w+|
zjEY|7uovR=8hhZ0(5o!9{E|?VqDVxEQ3H>~aH^x;1Q?0|8#o;`rh!rz5wUv>1E#l0
z3VQ%7gBF2VO><mk!Id^)QYe`PyQMJ`%65o-N>tDMBkQ%%uV%>?rHECd#*mK`@j;w@
zUu{B3N#%nmg~(ot8la`nXwm_xXqh#ACNdJ4%c5wJxct8Dr<zWAWQ<??14X7ogviYI
zwO&tl?Tg6qX;@`{=7RGJ$l)h!c;%8ik$PzaVYuAW#K?27nY1GL(NQZaaZZ4e2`qO8
z+SA>WfanV;bDg$++~O}cgvlOMVKRFn;h4JIB|utn7Cu}mnkYHNgK*RROVZx5n?I2P
zaGQSgT@YW`8UIMfed;~)9`n9T^&o|~^n)|{L|`{%7Q&;b%!KSF10dN%3=87t-A4PB
z@hyO`<bye1Sy!7s#iCJsFCG?G=m%oH0Pbsryt{;Gexct+uIOv*p;wwEF7|MX@A0BM
z%FikC5O;OXo|IlY19XN6bOEPcnM-_buvq^EAUn66?y4>OO6Ms|EKF9ASkOrjZTA_*
zc3*APvA%zJF05$S<m`_p<FRHOqq}aQEB-;m7JE_~2(3$ddSoobkec?~l%GRd>Te6H
zzGYjHm6oUEHeZt%Baj8{AD5L@BvvegM3!dgxC+Ku91E&`#HUy7ICdP?gRTGUZ0t?Z
zd#R~gggNDQLAQjYK1H7o(C&^QzZ#@`mT8Cx`AJf8JrvBn?c2L!Za~RJ;c97j;`~yQ
zP4yL(Y9rQ<Nr_XFbR6|t?c(Jaosm6>*8&4IBAY88Skea4o#pX2r8{r_G_?2WZ`e>>
zRoIkuAJbRm;Ow{PVEvh$PV;nIJ2l9DL^xi-<>D1R$!HL-oJU<`!{}+BEH}YUuE5su
z!Giqk=)rsaiQ^!PdT1jzJyQJD^=Q=q!~CDsVCMZTsxhoSbN!Li!!v^n{R3kMsRh8<
z<ZOmjl|yuceEx$)ABGPNlSc&Ci(bQTg=E_M=}IIZ&W;#Tf`2*=ju<N*|BDgrHN2vK
zx#+KW8gZU$CI@H^{~uzU@hgKu1!(T!Ui=J6A(*))B{-a@cOpw>%o#zlLUmIR^#B@?
z)St>`IKQ#11?6(lrcTVzp5bu1z>0zy7^Fr&-}JGoVW-3x>oZ#Ms>CQ8II9uOfB85l
zQ$w2%Z663{|75yFYxh}fYlt0>uIpfD<kG^6Y&|HayJ{Qp%IB5IUzVPP9px29tJ_GB
zRUb7kN?u}7mR+13KBow`8;dZ0ka!uzdvhjDbd+G6fR@OH_?!Tj%mukM{1Y+qP};8a
z7vjEnw*o;$4D(OsY)9xvUrIx7W6DJu@jc^+29Jzj+!<OD+9JJ@yn<hYxW%Um7ZqPA
z?7xB=vl@Rk#wtv9NS_wMC^%L)EA^ixwm@)3YKT}C=aFd_^DZnbG%nmLW|5F8Z;&sO
zw5*!vs0h`#Fn?wy!wh9jF3P1TKFh@^UMp(PpE+{51$uC{$94SBk>iu_<@YTfn{=7m
zw}7ojGg~lYGuNH+sT7*3nbSX!UZ9vNon5MOR4<lG&9^Kbnc6!VU#OVkDezQs<&Vuf
zs6;DHt!k3CO<i|yFMcNW=6bX9_V+1$S9wc(UW1F$AKRpVjIu>vK`+MOffWAa`KeMM
zMj-f)9ON0~TgY?B4xAEPB)lbDEV>H=LlI8m$f#kZ-nrqro)5Y)YXpxZyTwsSzk;f0
zI}4`uR|8^ZO15P~r?taCh7tBv_64_O`?@P{l-1rLql6*qL>baznP*X~C<OE|JZ)Z&
zjYF=YlF~%;fFtagilS6$sL|0e!m$DqtWnvFyL6^8p;5=tg$z8Fiv~S4fO;6syasmN
zW0vutLEW{Qa=B8ukCf5~xm55T>WIsT0F`H%QCXjgXZg9NQiXR#UHO7WllsGo+M;xQ
zy5)?X<3NhRi6vF7wyo#<eMMHg_OCTpmoXQ)v*^>7dy;$j`v>epgg*$M5u6bgu)||I
zVp?LbaJYWRlhKjID@D&3o6at+sGqh?wK&4*M{S@FrTu1L(>9~EN+nE%ObxA+UO->y
zs~oGG7+8*y!E3cRU8>1hYg@fuZERU{UU1FX>{x7{dn*i0gxdJozmq|>#?<ZTZyI?J
zX+ORbw8I+Ry4QD^ad?-dnz=w`i)Wke`qQ;4Hd-!3u3|z)_tS>khH=NiE3{9Z59}+#
zOZ&saJ>Bib!H21z6u(VZrC;&R*#PW-E!GB(*|$O1RpiBQy<di*p9xB+78%pfw9sD3
zdpU)HwLH5pEAd%9>tjB+$EcLBJm}9UI}_;x9JnFDA|c744WTEYWTC2|d|}lQgE3}d
zhq1R2KTzDcBY@v<8YquR7otWE<{e+i9;MURw!Smda5DVnd{&53ilQ!ZCgim5?qxx_
z=v(fWG|1fH$7ZvUS$%VRc?@{eB}F3Zllzr^%l?N)-==#>Sadt8S3BG`K`>E;cumGd
zT!px)@UTEyrc};76Eo#EXM^SS;?NDLt~8SbN^wI$;<VCqQ!%16Oirzh<iz)?Qe!1s
z>AD;h?z2~?#3T2a*wM-KKbbXrfpPH}HB>u9bTO&}ScAJJ^CtH<9Itn;rUAe|8Uj`H
z^358L#8mW_&hs;;U8gKd?fH5Ng(Y&MNUD*#yTW@`<L;SQsjWn=<a8{ya=aXTL?3a&
zx(QSqEHhpvSEI6}b@kHA%A2~?mASg8G%K`XyED5q!H{-6J`!8Gk1AKy<Rv9Vhim0Y
z>MxOZ5hQ)7CR}4WoG%g*!b7o+mmIGg){Zm>v>7jUxSs1NPlK<cPu8&IXi2n(>gX+Y
z2ZYD@t0JAWE}Cr~S0}cLo#r$wRN89GZ`Qj_@W#u=wY8ZWpS_ygVLF7{3cO1Xrs1Xd
zQelVbzBem>SMk#7*LB(Qe7Gp<z|+WXX!8z!h>)QQry`-UE&rm^;MlN!oG`a$Nm^;H
zF`}W_aewz4<78Ylrfjt>`?pR_M;T?&FXO<mkYp@gmB?RJ3qkXzr7RWN8ZRzC^>&hF
z2r@C*GOa5YU$>yH5Sto>ZOnhwm6feZu4>fZT{R|IoBH0q)%dgy-AA0|xQM$rok=gc
z|5`8Z@JT(Y{046pn4jnRFuR7s7yLFmeK_50jOZxWr&Fz3^YD<v<kL|@=b^jev+))U
zWraA4YlKa0E0|%zcE=W((W)`6!HFotK0A*&>2Zm+pzDI&$gSnrdpQ{KbM|MUSZGW#
zHWoWA&$=t&i?HiTRAVa{Sx%da#dPWY%vAA2Rye!kHTBDBjidhoLe2x9?_t12@P6bv
zS|+W#Hs@p6LKW0X`${J5i<ZsN<%!uD_J;ed<Yh97YENZ$OMkn|mB;DbIMxuG&Dsx-
zx|5X6`I!~>Cf!wv-%M_<TRh$E*Dt>xsPNKwOI}iUI`5s_U2)ve?=Po+_KLZ-KCMmo
zB!1l26yGwu+3%r5lHJMiWaFMkAD%sW4`1+16ih(A48Kpl{Jd}Jx%uQ9^6Fmd>)<;9
zyA+w1I--NoxqRiZdeZ3s=nrLky#d!j<7@hAbKtr0V9HnE<HpE-LUd#}M>Zf=k@J&p
z18W!$GvV5Zdrk!`@YC2Z#O59X*rB=I#UwbGbHwwHVq{@&R%L?Aoa$%l5@cI2mmsjX
zHlQDRv)B?Kl%|*2P1R;DQ4s9-h=Y-taN_z_-f?O(JoM7W>@XPvtUv{fcPJzTk_`dg
zdl@pZ9lq-d;_gY2LfDP<QNJA7`IZ|>ccc@%mD)F8$|n*ER9nE<iK{z;fuWN8J;5au
zi7&yxz<-#3S94O6k>)hCwWifKvNbTKb+fhujRphbcH;y+S{pm*<GWc~**J2#@euxF
z1SjbEZ!;Ys{y&B|S@IC7$;jgi**X~Gv(Pfr(i8H2!N<qvb}%yGR1_Bd7dYq_522Zp
zlN~1=ovW)Wtt%6)t%E5Y0|y5O9X%r*BO?uH1dXG+jg!6`jg2GGKau<!kFc?$p@X@d
zlew)8{$ITM2DZ*lJcNXQ3H|5zC!fY{=Kn*<#_?Zbfdr)c+e61dOHcP7+#pcyzpb3|
z=5EGTYQpB$#x{;18oaE`?A-qt|NrUvABz74ss2BZ?DYQ)_+K6W0_3Lq%m07L^G{{{
zqZOnr-Y?vA|Is|}myOnjXpk|m&4uN@gYFQ2Edx5p|M&UzuvMK22nP1GUP4&lyBqjv
z*5@Q_5tIQgJWS~G^FM1O0C?c3fSrUL^tnI)B+U6aW&i*d)lDq`1^OJ|>UvS-1J6?B
zT4}oHklET^=5b*oW?|y-&a<YiJ)hMU$HWML0~hK7(*OJRE|i-H6LwF74iVx<pa=c0
zTQ78)ANo)ImH%!-hZqM4zd-+Qa7iFNcnttyh2h`${@D#a)$zZm!1*Hsr1`1+dtClE
zxHNyg?SC-?>dXazqX4Bgt8xCDNjOm38P<Pz{s#_nfi6HVGP~8l|H$&U9mFE!|EBSO
zlSMNq=-1mBxk{;2+OtiW({IZV5Ty8x!5Y<!|Iyz!pxfKs!rGea#@YRb??;s5<>sIB
zek@z3pQ6zYN-`|Tp3g_cb&gNbR8?)$V=N0=m6}a9PfCrp(9C}&B8^-8kLGf%3Fyrx
z@m+7m*ji62TgPSCw}ME5u^5fQPUgx|PZp|ju-NVNhFaVPJTLB-Oz{>PELEEhvOW8Y
ztyX|$0e&!HGWa#0{%u446@krht_vqIJnwoe27{oJ)fUIZ036p<`|~iO7`vVjct^|1
z7L&xAEAXJ+F}96W>*_y{HksCq;c`oj1oD5;k=QN9{M$(;U4U*U6-}#*Bh)(~2sp<o
zTK0WLe^j)cl^=Rh<eF_e-&iU5_vxN5dl1)<rKphr<L(DpY`)9)@cw=vRQCk`(U|pD
zf0lW*9vS8-;qw@-bMchR1zlffC259WO`oT|H6cWvq+JP|Ro4ygwZV9zwacv@rQ)+R
zf=!U~#Qw*5%mSf-NsJ$_H?21ld>eOXD{SiBFBhE~cZX9w@aXhicNZIcLy4Sj6Fd*i
z**u;hn<*k_+RS|h1Dxz26S`a?=Em#bePp!XFP9Yz?nJX?zku-c)P$2|FTPz3;4Sgi
z9h2pF)dp?+_25lifGqnKz;#Yp)!5GIcy=h7X`KDMs`K69<7ML`pcMIRmha;g#qH9X
z>tWTA$!Dd658~ylDTnK3lxYoQ_ia=)J^$TLh&;EGz_jbfomeOug9Hlu%hrnyM<rQ~
z!WHKQ?X}NnT5+4zLO8hK_G&DQ089vQB!u?Ikw;CNmOkIRIo0=z&X0T39Ph?d(NoiG
z_x($R&bRwa-%Wq0S8T`8Ew**Hoe(!Vsse&ZOz5!r!K&8t-|v5*@NW6uAAh0=Oxo5C
zk>>GOx9p|liEnt_M0dVFpR_~A=m3)E$xT9i_P<b-1CIlrDe?<bb(ZBAhlq7ZhefD^
z2a3~8GA30~85hG5qW#Q)T(4jA?7_N_GMN!~pA|KH-|hl$?m<>Oq;0=d<T)K@--*fv
zU$Smmt2J!Y2Zfgp!po<FoK{}%=VlN?RA%rTlqj@h5JJ#S{B?`1t`~5X(X7jZIKHn}
z!5ZaJQ>+?a!P6gq%kjN2n>kNi?!(aFzin}zl%qWVd3}<6*%f}>3`E4v*tQ?QX+mzz
zy3hj!uEjr;Z3?;%AnPN_F0a_MTHk+3OJRtw_5cevU|VyVImI?7pdXq9$HagL;irIz
z<-Qg-<ASAYvvUamjEvB>mJ0!&b`#}RK@&YEt7lX~K5%ML2u$e%fFcc45zYIMngWTl
z4_pT^<rK>CTc!KqvJXwy>7wJM5~WP4A{EP*OAIuVU3C7OJ>B7Y-zqG$dp_iXBPG^l
zdR-5#+?*`8Vn8^H@H-ySG%gyPgVa%VQg^yPN8qQ+lA&#%C86Ud$Lf<u*LE48X%sJ#
zaan!*W!a!UE)>T_E%8R{W?G7lV|qQef|(Iu0E((nQ*mdhx;`Aj&g~!XFEP-JnCswB
z*uYT+j(LOzst$1hsf85jM&b}-c@#dJhegBKhVbXEypMIy3z-z-X$dlCs)~BWYb{OA
zFjDiqWK{$aK@?4jE1Qnf&X0Fb^m;QkLqSztZ|A`mm1tBo$F9O`u_qW4M#@;QL>f}<
zP;7@VuB%>@5vG-;ngB4BxCI^0v8kedJl|K+iCykQ!=SD<nCE#-TW3VR*J>hk7y)2j
z4lg-reF$iTJ&+$pn$dy>kOmD@k70eta$UEt>UjR;dC~r`ie=k|6wn_bDY_9_25tjM
zBa5^AiRtQ^(9A78{>i*Ji<KP;c2kFv^3Rezf&_6jT19~GdCP&Mj>i0_J4QECcK>82
zMjhnp=bLdZx=YD}eb*&pG`<;WrsPWr9XdMFEz$?>qk_aX;y{|MWM;(o5xP!lcUG<h
zZq`d2$595E@rHTu0HZE{zN5*v5G<QRlHtjb)3qGOagGY*@{DZH)0%;h>qz&JkLwXS
z7Huk=OkJCny|!JevZLQ$Zw{w^zHLX+ofpnXXVG%FYz3o1M*UIel%#wAuFeBAo0>n0
zr0clcMmZ$8=A~lWh@&puS5-Go2D<feg-zscc$`#tr_yRmKK5dtHf{KL(uF6~sqSg6
z8piUxa^H+9byJZJMTxON2LK5;FC}Psc&s`rqWm7!LOGvK>n1Xigjd13In=+A{21{)
zsK=40UEBTr==Vq$!pqHTvQiP~XS8`)8<o%ZxEWY?Pnoz95uBG*)wJ%R;}s;7tKJ7|
zp=ZZ(=|Vn#*0@6Z9T~?~tf+3BlL~rdg7?J=8S*On%3yhn*X*qPdK4OglKskaN(60B
zz1BOQ!q^1+(YSpQSn(t4)i!V(sGsD?b2qE>G*i)t2OCx!rCgS5D2-bV;lYxn^ol*D
z_E=h+y?nExNX%r$2a1!PQvtJ1QZO3B9Zj!Y>NC;(jCBQ+L&IZD+RUZEGYgm;HQp~$
zl$iPS>RUb_H?vNVWA5AXfzF2eE^7S=90v8vU3V|O+$@PRC51&BAl<E|$cQ>$`MPe5
zRkJZ?8CYT1>;rNv*{nvxilD9uz7Oo<;NRb5e3T3IRtbo})47I4+EktE!iE67&{1wm
z4N4j31iT9hjdUH4+w|4nk-Lm!CQbu8PHRWLY^6KTt21?gqHa-KEQ(p4NsjmZ<W(WM
z?eC)^J35oDf>v{?udrd!X{f@|-_q~TW|ibBFjW)_<GvVA<_ak5D$)EDR)5AYNh2`3
zFKZ7GNJL2K6h#KQ^TD(Y57X`!?otJYgNn?4hoDP8mul52onWhixNV<$<DZRzV3=N{
zn+Er`isb5<9J5PZ{-4?bB>*be6x8}dDVm1w;xPR#Q@c7Z^7e%m4z^sA9?)T#V)Knu
z(TVmHeei!qesR%ctd9YD(YHPt8D{7gr}T9khDcyy?tP)_7)kqm-kz1c)ZLt_^Xn)H
zrHzMOL3s<U<_YA6TxL*JE&CaQxTEu{mXS1#<h$n4DABa|35JR%;`vx*0YLp=FN?I9
z6A@|dN8B451d$0tho2Wy(-4FinBj+a&2qDW-vgzd#G=sSy#N|6TIjbpf0GU|9@G@8
zfrBq-l>_U(pK04+N2gx%Gj2e(9swd0&&O?0ISdS#`1lpl6?^D(lkfTP=i_%fDts|3
zj_&0^n3WO_0C6pMz4x;3>A1oh4y7Zm`)+(5jX`v&@{<5oam5bl0<sZ=0ffj<L#lJz
zHhspX*+wWmPn?@aKZa?!q&AMFB|Qw<lu4#_%2vdL8q!()jO?FYtxEei=*&M?5mc4r
zzh%m>ZJodsRFt8c6(8n+7gG5;L2Rqg$0A;^ur&4Z=#M^+TcHuLrNw?>Lz%W!m>Yu;
zpJnJb4NC$x;MrE=HXk2=&IMi~q$YMf+Mg0Vk@}VgJ?8bNZqD4?0rXFXQ=A~0!%Xk4
zDKPPX@~;*M`i9&7Fr0fU+H*pj<S5R0*0K=lCjkNFr8wDBT6<7F0}aweBke&jVK7cE
z>+T2g0&oDFcw$A4Hs6M20HmZfGX<0zeF0HI5<JF+f*4M65Nff72|x6a2q2ahQS7Wy
z7-Thrc_>XUyE1~&!^>EBKGOPeZdgYR5(^`6r095`4G}^A^3Kewt`VPzOC`f<a<;U*
ztUX*n9eRTdK>qtE$H5OB#p&<Vo8Av=2QU!S8tTEyC26zY(zB^}b3ZZ7YV!B?iB+V6
zvXXmU9O))}6N-u?rjmsMczkLk3=?!PO)*Tw)6%#lRI^w^e9|SWsc6tLJ1rLz73U~u
z-OQug=F%xQ87?XwlR`diAwCZ?21&)4{S|4bED*QT?3bdj9+bl5tT7Tnvjbv&<_ac@
zDt|9S%U}!o46={W9dR~;*W+b*476<i*)(fJ7^lIJ&~ZMhHRc(|#`qnEQ<uoau=C^e
zR^n%!y^{FP!Vt~%6<=X`)_yyWxNXX`7~<p85A}6cW9OA_;)+<;t~R(-x&RIhpT-Tl
zMRK(?1JTaTmo2N#sGi`qA*Mi?{qcT@b%$t)=ob=mU8EI}Q0_X!b{v6FtxJ|G)@Ezz
z?1i~NgD<&*f<lL6=NKxGf?T#1?hx=5NZsWT2NpI2LG!~RiqrM{6y|>LWk|1QKv?{D
z5-yJiK+H*9mWHaXJZ8%@-8jV#&+DgR&}De+m_=Cy>MH{|>_XhHpV$n|@VMNIpNQYt
z3i`YLpmb85{UDL@Zqa1!<v0!y7iZ>XFGV>@`Y?P`Ch<aZ^JSKpODH!=&=;i_$S!g1
zi0<%yG4Y}>bpyx{CX%#RzN0SjW|334k{fD9Ay<m<6BkbVL*X8_x}0H33nu&=MQJGO
zZI!GX$OYflk}B2|EOVL^fC^?ttLN%V=Il}i%Mr|$GWtQhJZE6sS?(^wlY2#jY?GW`
zB00^5FK_T}fQ($cnTs=HlKymEA^xt=UcajvI57NK$X<X5HY`Dm$&w6i-<=RY%p+W8
z3bfjG$Q~P+P83K?`o(3O0vJ-~gEtYq69B8g*&!K*TE<90?H7OM?w6<+oqMcjaR2oc
zT6AMLh+9M`6oKZ)kHW_@H42D3RN$OP(`Eo{2-FQX=?Uhh#6}ddsKZ<eDs_{XJc>7y
zcW;_}-DhC{H8Y&Bp!shkUJpnfui(Se^f|(21^IW6`%`sA6)%#oMUK5BB}UwDr$#hu
zr|1j)f>G;1F|q#9-<jd~C861p%$@up;rv5h7=SgvcH|DH-|U=$I`Jt&PdA-r3YQob
z;ho>t%U*AW&kQ|8fJXExuaQ15xC#k{O%T30yxzC+%AJ_*Fz4<wq95;X?Zi3=&IQKe
z=9n08_LFi*8(13}rvqVlglcGWj>o)rpbbR7K>)Nh7*5ALN?;f=rj<jw4C|6no~#)a
zjfiwJJ&)oBsM`F}WSk)Uh95*>&Y65vlv!C2ZJ+%dpqpZ<bl<Ie9FrT<+=<_<2vcke
zDJzp6beT^0=w@>K68?mx&TmARdj{|=;GWX?LwnA4lOf974NW8>U;<49*i>+RE}v7?
z<=IsqmD|0UXnI=)@@o!&A<o<Dp=^-95kCH0G@~#`h)G{6*+VL6@+)x<HlL7&l>c#M
zJsGppIGiRw-X6hS@1=TsB`upF1)36f{%wHnufYOXe@;nsbJ-;vC=(E@3tSY_T0-El
zP=h31Y0<mTxo_NW(D~8p!)}KxwV9W!B-_8xeM0Km6B;x$wPucNKn|$s7)X0Zefp}b
zMCxFIr2Na6pXz7Kon=LXN|+Xrhs?+1w%}~x7fNX724FxGFWtSX0kU6X;Ddl>9yj1m
zudp!gM9S#L)s__&IMSKTF<n)}1g%AIv<9HJ*=_kzS#2OHh91DM*i--<6^0hT%^{tm
zRd{%$QQ$M9F`V|xZL7|Ik0>3|^z!!69K)0D$7j|)9v6VpqNR`8Bz+4>%SBZHIeCpJ
zuBEtiZHU7pyFR6nL7v(!&a%XJxmGIV`^m?mW#oZhQ$~HpwPywZ^pO&%{tz$iNn7)`
zy%nu1oRJ)TI3X^%9R1mA=vgKHD96u+B1BKlya;~5#J8CW#u~>o@pVa6TDh9X%a4ob
zuwihA8bIQ;#Z)=GUa)-b(r>81S@sFALczSEBK}CU%=~7ujIWUFfQUAZ1n=`;zoZB6
zf~%6v-rOXz>o;=_C^+;uK5hG)dT=S)7Nf{l<zXJR@91OEeV%%a{3Bo?uTv6t=WDHp
zigM#D-XBRrzb*wi{5ce>L?X+D=_9_Nrb*9%qv<jR@sb27OTR|Tj(r2piJ>^6n2;rW
zSG|_7Ku~!kQ;?Ic6elbA3dKV&5h=kET;q~fd(>+ODs;k@sBCUz<RMAT%|<4*_8P`T
z*z^m*-^RuJz6(cF<6p}*pI&C`VLuq^1#3cleTnMlHGQ-|-om#0+4K3{ww%x}68SDP
zn5ythz&~$;F~C;}NGrH6wTvuYbfF*zu0Ym#8%Yi4XU-vayYxJ^OBT#PhvK_e<{a<;
z$ZJTAZ(%B@AK`URuWF$`w3tJd9HCI0FjM-Yhih3>xo01x(;!8pJqDO?J=4`I!!em(
zC_ZL>$*jsRwdQJt^Rq_*9^WVD=~-UDtE{3?5@9fnq&|DNFi)aAmUMZ~!;6c66gCac
zkN|fqk5}kZ1o|O`E0IC=Kz<8Eex(sOt8iiw^VyT>+;2XI-NDnB!6w8YgzV@>BD!({
zFSD#80Fb5zS4qaSVK2OsX_*nizM&v~l$+}V;fg1N3O`@Mn4a#TI72CD&cVxFN>Zys
za-~KzJ6Rrone9wtO|Ck`3}<ttRCt#O!lZ(h(WCt>h6AD=1t4@>OpZsOp)<}zA$(~@
zn|GDWvd-g;Xd|L#5tR`NmT~7`&3XQSE;n+8{9O@t>tyVq3tinPIq%Fe)6{&7aGNIy
zQFmc~`U4yc?Bhc_$Ib=}3^(iV{^i-)XQ$KtZ;+cZ>FGzw>FMd7!%-AWj+u=V#KtDF
z>FH0PHYF1il?Nry2d&sbZSw`6tGa`1!wU_}Xp{H6kM1b4u?UAeSnZN@Y0@zYS^J3f
zALFhaHx1?mFRKT<O|)x;w9cjX?<|$}(fb|?t?R#Ep1QNnor=|Uaz0)UOq+OLw-DKi
zdGAUydvMO}#;A0khL`RhueZ@Uo4wEFLpi_S1k-sD&{jRQeY{<46miVg#LT^dHgR+u
zZOcBcnmXSqIM$A2*?RUt!?u1Cew^rjSj0FL5H*;egUD63ew>=}dEUQD={`?8KOAVg
zof_hCdEVQ{dTM3f<{^5UIi5S_c#at!@O?a)9(6qS(KSBMI;M{>tF{a)&ljv{eoP`f
zx8#uVUWEsiU6tv+S=_H`pF5=fnTVEfi;vVQ@E$0Yb^ax)JLs^_l+)7g+EAuVqifTd
zvk4$GyU2RoCCgcR*@WU_r@Hur_t>>`N1*HT(7M6WPY`9}+PVVoZj`d$e&Ky_+75;5
zwt4FDMDnn=@w(5pcedWC`|x{V>^jy(>;48gwORPW6LoD8;c8vZYa7M-dPL{GR^}Y*
z<8gp5!~1@zSjM&fCr*XT%>dtf4ad2C><ahuRIJWzgoiH*QuEuH?MCBBI0&-;$0SKp
z$6dt=b86@7UaL*)@B6q=?gIDM9lQ;%Jt|#{J1v<uRR2#1?0T%{2Cp<P?x9?(`qul7
z?~HhOJLm&u8{UUPc)vM7wJDd<4u=A+@mV6sj(l}3;GG&Oq$9kCEqox0j_D&frZskf
zBz^O!;{<WCUIgwZK{N8#=KGhSd|sX(#AF;2H(3{6WE(m+I5m+l45YfmSB*{W){aR(
zu?q!oeII-9D&70>IzcwcFvNRVJmaeLm@BJn#X}1?)j8l?2$%94JLti^uWt0=jP*Fg
z*G6^@5~FZe5GffLEYrFBwe)fUQZ-vA$Zs+Yac|Pbur?n1S(da;G!`-nJrB4xmb^AU
z;hC+}$JST5yCJ>_a6C@M`plAVnDV_WDcEYK;Cl?2%Xu!NW^%1x#r$07ApUFrV~q0&
zuYINk?)*xIBkmoH2{p%yOpwJ+*Bh$5&z9Pyfe$OTolAFlx&d(-!~=t&ytjGk?&MD1
z2`|^{;=l{91FnhV@OmGRA6gy^;eqUD`3rkj`z2ip@BK2Ajf{H>CKF^CI?HMJC!&WJ
zIp60QA>1V*^HJlCkM{>G_E(UyH;68<pA%?DfzR@O+4uFu&L+k<tDwE*mHt?-T_UcT
z*+@@Zf1wC`%nb%8-J6Vs(+c)Jx@qH2>I<Jjc~13L>Z2G(h9v=4GPWKw<<&V)qo$lA
zmld|&mMQzBY%N2-nQgsKG)*5sTjE$1tvPg{dHj%G$aA7Tv8)AheM=jcx=a?-Kz9_e
zMjK~!wDujPoab5Uivyku_ooKWz6Y!H6HC}{Er%j>%kwuSVm!y3RMV~Nca<2B-YA-Q
z=znE4ogiE+=Q8D5S(-;uP8m7eq;vZNyL?Sui}}#*IO=?XV=yITb>X@3(zvU_q5JW?
z5bG{uNfY)A@2+d~KH>Xb#{ts(qJiVMV>am3)*@N3WJ`=)5K(!Z8TZH2L{5W5X>JXK
zn0NN`Z1MQ;O(5Q@o=cVitEkz?bBGsGAH(&1Q>XX+x+zGe_E=geU^4tv#)IRqQg$s(
z57zo+NXyYPwOD(wEZg;uyInHAmX_{sm6O$y$=!l@XNIdv!wy`@M=$Rt?z0y{@kR1h
zpB0M#x_~tf22s>R1)+CZ`iDXJk(hS=g=lMVa5I(m2FqLh1oY&?Xm+y>rr&)pS7?VJ
zw8{RKyJ5})(efE6W<9Ug8&(hAQ17+M8AP)WaEwo-7PP-VbbffNjGyQ{{C1ofZ*4&q
zfTMX3CgW7N;Igrg)U5x!xBLo{q@7lK7yMR4<D_NWxssJ`IsLOm3BaB4&#n19TiX`A
zsh^8FL1^1agle0?$^QPa&fNhK-qT>5YNNT6Ff1ROI^w>Bnqw7c_E$-zREQWhEa$x6
z7BjuZxx$WF>fB^FUu^osfV&Jy>D21We)O|-)X=Vp>OAhz`7U)R;bCKP>IuwG^sD70
zxvRqxQeiL9S?P6ib}ASc80n|78K^|S^zwK&CNcfkf;;0WAV<Mv4+-&@$y~4ZvO3>b
z)mKo(^=Cjgm12Rl&7qb8zFdziYys$uoa=}+z36<dBAfD<v>n&ys(kJmsfP`@Yb4dN
zQ@-5<urio5MjX2KZ+`}PPK9Ct%idtx+g+7!Ry2#Fc&ZuRS;5%vuXmNET874q^W<@2
zyw~xW8}Ii`-VzUo&Rf!K&41uCn6i$fqv=yBh~dzp#G3=3d`Gorlo2f}TyA=PwJSJm
z!Rj`TIcD2;c^$Oxuepoj8xIpm!CC&DISh-`$crT9NLv9#NUjpSV@Fz%yC9><!p8Mk
zL<z5EiIhqK7t~H?&DCL=)SYsbc{kq9T@h&^4R9`gPbXK-zG%5jIICkwYi)Js9CVy`
zfLpT)eohkcbTMzHZdOpqfxO5D7P+I25Cpv9H{U>7>hg{Nh0h$aZ+;X<N5flz!jbE3
zyTPrqeP{dyNa#0~2{G+8I-2j@pSD|Qr7YsOYkGwGg5Xta6yw?Cdgb-kVh_wNy=$9L
zgjEVliYGMJj#aXh#~7BTE4JV0bz)uHc38f)RS<W}&su-HJ#0w!S&S$ekXCXoSJ0hC
zxRO#gY*!70*-<<ZacfG<nwK>y#1)FDG!eMjKnjfJc_4M_rUV<$WPCB4?-y~nWDa3?
z-kJgVkjRkd1)U7AuUi=Wd3&xosgUEseWG<dt5moJ*}xPveADtNyfvL{iSQ0I0X$U3
zY0pAN1TS6Vuu#Y1YV7SH#MQsCZi^M;Qs#itsHwA8*C*V`7d0!${EFN3&YtCroiPt4
zYMUTq5v=x8V>+ty_Y1j{sY5o_uil`@aU$(1t`lV!=(%>H^D~km1^?I=)hG{r7nCo3
zBIROE3^^zbC+hLg7Jx`%7-^BpV+rBnF=)vj=CY0%_bnYFQ7l3orW_;c4k_&$$*G@j
zEOlQrGDSyg4w%Nb30kR*<By1NufpyzQc|s^qF%0AnoxRk-cC)edA5Xx-p^e-v6MA$
zo<PCRB1IOC%JQlfIih#%+@;C5ZZYWq_cj-*!I`<!hJ}K`b7f_n)iRch=Sbot!m=&A
zr=#mL?PQUm3>mkk`<lkX<_BHx%XQk3R4Z9kX4UOn#iHoEy>aUR&v^|aai##Uj&S`z
zB$`35=OT_F``}J_*~mQLF(0knj1@TCOj4mImMr?mqxD=WafdzxE+*_}f~~3_=_c;$
zCRCNWofhl~KAWaXXLWXUGtU9XMEcwwD2r9(v#7b^4Vj}@<>{iXLq~M%(bz7jGu{Pc
zYCEIm@WQ~35hAbOaigWWR)6VfluFwxykas*RaMo=bf$qw9#FIYNo@#Cc=8EN|J<}D
zd_b6=8ap7RuOrB=uC}P|zP9ULALl3<Jm|~*-uYdaH-l)cre>J`ZNFFtuKP1wE#Iun
zXmVCX^Qz)HQU=fItZz0}7}o?-VFOd@$U}P;{xMcw{kY>6mo*r!kIwsbYASuyGuN^g
zllH;SFSj4Ptn*8SV-dA;1-;aCqa@)%*L_ineg-nztg{!kUOQM5NxkmQQ8HgNE27a+
z8!fOzxU1d}%7#sV!kNg1aIO^X^EQsupC;NGxE9V}<lF=^i-CE{AeExe3-kia)U&UX
zitB^xz+R8!e21Iny$&)UMkghAWf;n_v=B+*bJdyQJ7{SLSWVng`}JCG%TEToKx@@^
z^-nd`lgFpMYngy0MLyd61o}w}^^;jwB%}o$0n%RLnD}aPXPg?8Co?+l{V*|ZgPX*h
zS7O?#mwJAUU7SFJ>my26_t7EE_3IQ?NYWUO$LZ9LS!8b&eS0nM6pF{{w)LCr2dg9;
z+2I8iwRn5tfdWINZ<bR1D@+WFt*oVX-#C#awZimVxGw7e0;!<~NX%lbM*8uwu8#%f
zG|0qC8U$ar3j_^RTRB6aKLMfC=V{IlCZL&)98ietWBP04GhnS)2a(0(D9{!ne#Y##
z%x_ttgxu<awC|{L(s6MHPzYArZRlyUM{gZc&x=lH2rv@vY*Gjh2u*V7GtWeef`7wJ
zw_lY_B@x27{VXhsX-HhYv9E3Dvf=$;EvLa>q}DXnkOUBWNz(tc<r6tW)|NPz1i@nK
z_*IpT=<~#Eo6p``Iny&abrj1^*~D=zG!wfBp0n1n02Iz_|2k91575lBmC#eGzcnm|
zB+sEwByc#A(*14yvtx`-=z8&xxefo>ynagbpwq&KJ7~VsUCZc@i!~4*v-KI`lM1S#
z#D}s=NLAbTu9Nl#5i16orbr5D7;8~U65f>sY0X-zYZELum9o2*sJgf?0ju;P58HQX
zday{>P||)y^SldWlP5d50Y`Cpzto+)xK!l;&eru#vdVPvzSPp3a9n-V!*PqiI5n^k
zTKuM#G3#Zo(`;;_7dgYA-a=;@Yz{ncC=5+v$RKuHXT~X$Ja@}7E9FhhBJ;i{dHQhe
zYcj5-q3fM9*F<VBa!P%YI`Km#ZPP?`Cvv8m9xkmZP)zkSai=2;L+w5c3|N%flT6>{
zrglEU6cabz-Exd)k=aU$Y*6rsq(TnUN?X*P+3%0#HR|>N<#WcenKrt-QkUdEp7}pe
z8?D+?$?`F4)Q*?;5=Jgi(`fjNeB-U40tOBAZdSSVlLNwW=7c_j3op_f)=;5KFjiR<
zE5_O1SdK!qv8}5oxa%{p9v_)LH#~*p8{5NP@vg(zZ$TGd6={19*TI?Q-nP)1EJM)U
zig1uKSEgnYUl`4y+yiag(Ou3`ij(X{224JEDQg*er0Zep*Avx`T=X=)-RX^nE*>KD
ztp5GCNP=~n_wF9ilC`h5hxTXde1KsbC(8R+VP7P>4z6H8Bfy)^@EvuG`5M9Qyb$N(
zmh9$9s#Orb&a#f?J5lGB-bw-URsZM(I<8|k2>#^mj}l2wTkz{c_;2+W2qrA{&`}KU
zr@a$UOpOlQ6dX@tsl3F#Au}#y2JT)vc{jahs3OU#l@m%K^-7ogD7^<Qr}7(qV>F|V
zhyVoq!N<CH$XX{tzK4utd<!j{;0FdaiNnznhfB!>CNcO_rVsXKdo~@mdp`3rmS>A6
zM1S^HsdL|BtO*OQBU;|2|1M4FJpjGO4g7)p;l<~RXwB7XDLr4)Qd%!a=(&^(1V7&s
zrL>XGBvCW3?GwhM&r3xC3R={RWPX<Ro8AXThyWBcP%4Hv5?8cLVWDB8v?H|*2;UCH
z_?FwA#%yh90+0q~lN;Mi62lnEeAPw`!G#v%1>a1g@ZGn+&}flfP``)#^wDo~h2aKO
zN&~kVR@Y4R1{XCY)Cp(&KpJFF-J^P-^6f}Rn9vb&Un3h;(y+1Q5b-&{C$-NjW^`}N
z=sC82=~t{#IEBb)zAjWeb5qD6)U~0o9t@oQ+M{5(Lzn<|r12O-$W9Q9umT353){6m
zMe(ihYjcf3CApmSrZWHcz<PHb-FqvHGG0_BclCuRM@xeRcF`_iHZR0yL>+<%N(dPY
z5ym+TaN5uBXswHVn1D~BF$go`g=UV~xDmPcO!uc@F)WK-7-X^dC-W})c-$P~a9+dD
zy%NONLgKqPTYBZL+5v6pzISvghQl4M$@x_T5|;tV?h&;KJh)d;J_Ent$kM_O0`W#e
zPVK|I_Ps68o}OU(40N9J9?Aul+1bDNpk!C)xeIf?Nj{NZlnpBa;^&d@`~0*U@b8`~
zg2AGrl}+KE7NaonwTAT*z9<y2qtp@K3gXq04*x2xn(nSsyWF)D97R@HJngR6BN*9!
zxAGvQ-lz69=KmlJNZV*Qc1`@kmNI0A5gyF1)-8YHu;3|uiHfmkozjbuwH+KfXo4&p
z+2Ny{kVvq{Sui7S7Zo~9`N#UyqY+2|FP_7TI<U@u^=G5d6D^%SyPG943?EEfN`-Ol
z0e3!rIu1C$+g61C1N6GULL&Zi(qS35VA3MBfYwdg?waB{w_H$P8MhddY#_O7c+4N_
zRMVx4>X1xB?`thc*u`W??8rn;MRLI?5$n`G_C4^Sj|R&5>nM)teI9P?eg*M@F)WTX
zp~d?67CE@K7KIiH2;;{z=4VVcuehDX%M6%kR+F#5e1jd0Fm_F2Dz+5{cKpKbc!$o2
z8<o^7qMi1uQl0*e9Q-+ZTPm+(uIjat7B`9jbpP#gI^{+>h}G87S?vSW^C69y+p}jP
zRQlyyCRCZP9BP?;^-c}$F)o<BvbX}QzgchXL~3k|!WShgjWG)(lr|<%GzumFH5X>Z
z;;qNU_v%i~uWhcqwA;$pVXx}pz@W7m`ow6|b6!}P#&uec0Tbk2k&E?T4=s!h3JXPO
zSN8qtDqo`z4dECjFLFRTT8(en#?+A~(|!59+F{Jqe#Q=m>D7~|SQgZm*Mp|Vgur3T
z_z40;{q$8=%n|f}O=v8^q@c*^^J;d-0f+Pu{K{?_AfofIfnh%5u3J}4+TLr*+xXZ7
zjYhwrgIMY?HdQ;v@?+R@b#`sP$=-TfflORubp|1OOw^9VATpQ9NFsGTl~>#g5Epyl
z_FFIa>Kt^-$)FzUmQm%wB*uq~M5}QECYF?>C5gfQ(>3SkVAs6*<~k3>Q^(Hny(OEC
zXcz*)y=KkE?4K}%OqDU1Sq4jJ3kQw75X%~&OnLnU4&q4}`%!UI>%DLepMyyVUHUlj
zs5KFiW&ndE-6HXQfbx$N4dYY((R1cy@{_dNRV;|(7O`4Q&VwwN?KUQg8z#d}sziz8
zm;RTOcx=*XSLwM$FB);uT>{$8Ja^4goRJ`}B!;M)Yv`s_ml4_#_X(>9&n+9Ol>v4?
zC7sdehO~n_cfRr_ZCJS|GQ%14t;W+x-<gy_ITza=9t805Y=|1C;l7L%l0hK^Q<#&y
zmVmiQISImDq@&t~F=xXFDop?3H!C=kYjx#Onq#fs@jb&SJuq>&0JY~Z!-CiB+X6D{
zb5MCuMIi~SvuhyC8P^d(7-2PT5gujIPgEy|P_GePZO=(ae*o*XIpJ<Pblo?sw0#@#
zw8ktfF&SZTupOrfFJoLBEYLNf+7g&~bTSV6Eg`m@!xprC-C4Hs&~z8e*br}dX9mie
zKN((k7Zc84Ju5I&;ow;^pC5=RwV=YF7Ia=+QB`ET<*^foLR#yIA*ds!q5TjUYO0wh
zd!-oS#d8exn;o_4s^R0Mfikt!X{>hDMeIGcM8cQt>?n6SD!q`uQW?iz*pdxzn#8Lm
zlXg9=c&8&xk+re|H(ss<F!fGDEvapds9r>LaaW!=83~93u$q$W=uD}y&^g}%M{KeW
z1N#%BagaP0Q*oEj3>WY4MB}90jVyJ8_>(gA<5-_|KZph6mDkhoo`-=)l4zdTD<+2W
zgKDGOEyjFfILm`ts+2y|Hpi)Q)M36}dWN}+EGhbFdyDeM0=;^$OkWxpx;YPTV-)kF
zhuNi%egN~~I7W%us9efT^V+Ug<yVEQBCgxK)F|Gft5cmt0tzGTg5B1+gS$-ou+k3{
zMKCcCYI_+x^o0i>8dqGuF!YG#Y#A&)#}I~L5(UVy=wU20SP>jMYT|)k@j>Jw!h{oP
z0PsSq`40o@Nx|>6KQbl@NTRiESLw7=YK{u)DVG<!$NcHTcj9mb^b6y}Fm@}k2a8}>
zB1J^hfBV#?{K(-T?#S$#qZ-Fw9b~lggt5?7ZoZVVRMpK66G7Xrq#>mBcDE@$(PuzK
zOZ)s3vB<acN#)u5N9MA~eq8tAgZ@&2SZa5I#19$eXRV|Bn4ccQPsR((l`=o{#}}G~
zk-LAP+bpuX{U-J~&KR+Kp2_K;5mLdNB^V?}BV~Y!>MQrQ3)bEGMj^yIN-=D06Vc35
z6)rTL%wwBG6$!+;Q-d7y#h75M0DS;a0v9W=t@a#OQ!Zbt;&5gXSyQjIy7cXgC&>X-
z*!sIhoFhkXYAwDARo;t$yn2a{)x45U>vdfhb<8Fu8{@vy$`;oG-^h*0pk9DkzKO`z
zMhqT~?17d^@u|u8qWu;gBxssBgjH$-TeAuKQ|XR_R*@cIV}&gT#v@I1vzdCQg>bKy
z{$9TH@6x%b3^lg;5)3f=Sqr0&E^7PXHDgwEkH*_9?pOnRg4JkWJ5lXn0CBwDi)k=u
zl!dVYBtgCQbg0U2xKAijCR@hgV=DOPd~dv{+7tNj(<V}Q5PGA=0eD|BMR<Z(u&)+f
zEx@B3TF`I<PRE^awjOA=;8+v-77+tfiw^dNKP;%V`t`FE*$y0QyM|Kp2e*x6(IeBZ
zD+sqzXx2uy`uBK8ks};m9Yek7e8oy;X>tJZ7Y7$e8S!jKEqGjpbu2N3iXFr8!L2KI
zi>E(hvALBowd*{6vV~hyTa%DB@x3^l@Vnt99~^?npgBrvxgi2!wy&#7<WvAELi|4@
zO9_S*wDPq_ythA+HNL6~7zo6%A|QRMA4Ap;sAI2@p0qo=j{GeD%#N1uGGMUnuJO)I
zv*@Ty384w&&yLwIIO0r~9oZ1-DcfmEGNDhb6_4sX{PiK3s35B6I*Ok`FI$W>8s2E4
z$bb5maobQVNj#M_bXaXtYq&mYO~82F2=WJqXj?8j*KQy_{*^7Iy2KcUxM+-05-ozM
z=@|0G;F$R*EX#&k_mpomM${%(tBg4{9KEK{$au7>M%mX9qsT^T=_9@f0x;tP-$YJx
zg|6zVQOjIayc(b^?KUaf*PbsOs(n)yXO(Q&4PaDghy95TH-{olGJFzV(q73?L4O0a
zsXPC2q=1JtyQ#-pyh!xEOvB2vM^D%~H8!y>Mxl^8R$JW=yx!kQDX`if-Kza-f3)jV
zBitiBw|0)8a-eu9JlKDaJ`$!!0YPhK^o9TxZ`lI_Wv64)*8gl9<w3Mef40WwTnljP
zdCv2u<BL$!MI>tsm-^Bl>O&`^lCiz?xcl3vRPcTno!y+<@(-%zp)^--&`F)`>vA;Z
zdi(e8I3=!GDcDq3e7JM_c17ry>-}#)#HeNfk|M84En}B2#%Pi;M))O~jiy`OBLU|=
z_M(Cj&c!!WZRcGV(M|P9HJ&lor2SJBwhs|>@RO>8QO9w~DD8n^tSyKo#i4xDfu`3_
zWhGd_dE5lu?d(+wKOEO3XxcrXl^mCYq9wNL>XfH%4APJ!S`Y4|YPObD*x@W$6w&98
zFE6{I=HjOP#J~97(0+j(t<_xR`UNMHi`=WvKKK2be+0isE@3q>gnkRv4r1!4%?fWn
zWB>I~!qD8PMt{IS?Es|)?s$~&%Frn<TdNGUcCQ*YNdRd|yz*>+m7@K<O~IExLgV+K
zHxMaNHG|IKb@U@^)6)c{U%^vGq$%S)=VkL~s28_+MwoDT=m7rCD^pVCrFq0|saa!t
z`<TxAFaa3bh)lh#^i&d^uHP|2x_A?0xI~iw!IeBx3AEK0h#B_np(H=)qJtfoqLf@2
zcd;=i7C=$Tz);elT+#f}5Hq%3!|3tM3WUETut|hzUh+GD65^YGMR?b&Gf%orP`b1b
z8|h3mUSg@HgC(j{XsX3CA_VD7bMMe>aC=XD$s8hpQUH$l*c7CQe~=`Hm_ctEnq^^6
z??-CtwL4b%A<fn%UBqCB^_-otWC1O`gGZ*6dT#XF-js0-y<WrQ?#McjpCoLhh{96j
zFX?}9Pmkb#1Q|1QyE;AOr21rpnZurLbDJ(LXN8_BA=6HSR06QqWVNjL{x7QE0jNau
zJX87s#39nLR2Ja|uiAXBv=GVo=c|oXELG6|6zU(CC&NL+vZ`kz|3&F9)87Fu|8MYr
gyypMEHa<QBJ6=B+aiWX<et{z)A}d@es2A{m0K&Xt`v3p{

diff --git a/docs/figures/widedeep_arch.png b/docs/figures/widedeep_arch.png
deleted file mode 100644
index 882ac5f33fc7b3e8695e87967dfa8fdb48246323..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 157281
zcmeEuWmuHq)-E6*h=d@5<cKuV(p@5r)X*s)F*FR_N-9W5gCN~Vw-O>PEgb?wcjtL$
z@2!mcJA0q==lnZd7ck5_Pp!3{do>}-iqhD3NbVpZAz{lrmsCMQLZ?PTx;2A=4!jet
z=D>%9bf?ThLPA+aLIR}hU~6V!ZHk2SJR~;mwwkIIfiJA_AQBIR7aH>;36z4QB91HN
zP8b`Bib2z+thUj$aHkmU-o3(={9u18<d;R7r5Qg<S-)p|ykL1?t$B5utmVW0_4O$%
z#bfJuxi=-jqd&!Q8Tp6$sfxveeAGzh&S)ug*YaNk_~V(vUjq1*ebMeVfxunZxVXsj
z-(byM=k>nfJ=KNJJTLdpeF#m{>A^_8fhNgRJ%le!JW)wYP*t=jkZg#ud6J{)$mhd|
z3%W{C*}b2(SeOX%wlI7O?}(U?ls+>oL>A57J^lO?De+#mIT6Mno5)jxa0s<{8LG(!
zx!~t=$3S8e`>%J)-x~`qq($%1iCFmTCQu!B!g4y@tXV&a9}&W2%TM;N`hT6wMy14x
zP_&|wRyb)HQgfG=a@NNs5IixL>8dETe}E;uDa|J&!e*9?Rx2Fx>&fCddiR)_0@g#A
zTA7H_7F`69P+x{_bFTe|CCM(6tS--*Up;XOZN2^B@7IT<1Z%Hb4pwUHS@1d(Nnukx
zxpSDLl(qUFC>i<<s4;g^gV$MSJzTcZj+mA^$Ck6CZ+9K{rF}P`o>x5Sc^e0#y?=V9
z@U!jvW|rH`p5|boU2P*hlW1{2b8u8iZ!61U+l1ZeL$jTLmX3#>Uvb4tgRt2RZ}%Bu
zkp&}tC+H=w{WKT-IE9WNCks|qs~H>eQ!wO<JI7?-w?1UUOtX&Sw#n}U+LD@dbWVJl
z*!D9f&Yx|><5X~dsNMazYbb7BLFoGe^~X=f?{S3TtIxVHF+|bObNy*o{2#^p5q?M`
zsPQBEV2>9l_LTaIJf0~3%YeJfUr;pfv@74SGP<SqmHQ{A$2=05KP@kg`5cO_AF17K
z3UQ)?7CGX;W43S*24?zWc}ao+iST=JZ8-BFieO57e@QCL=NwUQo=1AVME*?1Ak`Jg
z`0}<&*o8ltL?0+F5@%6(@a@54+sJ-;oUHWxTpUY(MKL~lBJ5|M1|Gg5;U{I54Er(i
zibgwNOPX|mbA|8#XX15}WRRV&>X$ORGhQaL+pqwvR+<GBdyXk8UCd$>a6myv<=li_
zwVhrK@)Vumz5cfHT@6<rPmnM=>zDp-tSu!i;Q8YP(p}h2_0IH(*aZ(t%14D#>Kc64
zKv!`RBb@YjWwD<Myt2GXH4-Z{4RrV+Pcj$=5{RG6gUs+?1mJtsc%;EOA>?fo3wR-t
zgVb74Q!+->!VD2=@)MF1O8ioFGHWzm)J&0w-(6n%I8bSc>pv?7Z9h(<o~0I{!&Qx;
z|8^fNODy#~L0U^<T*@L{H|Id*O<tFpYQ6whVxpLyY*FT@YQHMwi&;i`dSdz-dP(|V
z!VmOnjAUvxxj%|_LbAonKc4=GdUaYcQXy0^RB>KW?ZiJFxFw~O2^#<pS*@C{Dy>iw
zCWMZL3QDCf7`I!tKWk44Eg`=DhCQG=zLWl?8XFF0F(-Nqy)3P)u&ig6l<bIXM+|@L
zo0x>IsxG6h@vefHV+I42E!9VP@wq2?80w*FF8Rsot40g)Z{#VneQT3z!`xEG3uM)}
z)R*$iazEr#C_}*HYN{_BmBZAN3N(sb#EZgSn<SLF7Cspz8GSUWH)`%o?p$2=+$+ny
z@l4?mmn8aW!&6BT8WL<0=u=^<&s>*JIl0CxxGjW7TdcHRNxkYFn8<h5w=#@cqpp_m
zm!XNqQE>nG)>7w{v}OO;OrDpz$8%{gov3tvxeTnZ&zD7ROpZsQTK729rkO&uUqY!6
zTnv@nM$>uQYieG`G!FY2a5iz~cXoOva4>loge{M4eD}k{r^JHXnIsN(Rk6FVb+B*0
zJqQ<n`+#`lp%7a(v19nqTON|?hdylVTp^rVMuRU0&GU`AIGLXoaeBRc#<Tx438G=*
zGMW6UkG=ottg&}1N~K-J^XlBH8k-PpeyvdEYE2h;T6vd*CkeWowL_t1?UjtB`Q>X>
zW?Dg-tmQVJy}8<7$u7h=t4?HlmfKf!ejYd7|Dn;&{oN%261GVyfF<CkS2+#wsB3J#
zG(I;wAH(^8Llsnnb4-@-F8v)7nGSg~Kc7Hn@><GS%G>1V6dnPngBss_vU+kD3o4Oq
zTsupLo^UOH?Q$)J!)0r47xNG#O=nhZC@&+gKKrPKv}S)AS_5?}6^at7@^Ern^Vk!z
z5=?V{=#JUw;vob}bmXi#uPteSG?>mvH;SCpoIN?<+YH;s-eKN_9INhz?H25LEZEMa
zEgQf@i06peMCP18kJxza^bF~gg6QT(_Rs^810JBKU}d98qk+*xu*)#szpe<(NEb_A
zm!MSOOr=d2y^n6xP<m(-(W=mD9&}A27LE~-`_A=&1!*nlGjr%eE+$KZ;qzD9D_jG(
zCOyV`CfX(s%Vf$b%P1E-e|0U?E#$PZhBrrjj0GvaQlzH&M!Tg*mbk^^)p{^P-I2MF
zS<fkN-jiZ)kv-Z|m_TR7ZnkSqu1HJWL9;D))q?wLY$r&?l_5zL9ZlH(t$&<2t5I#K
z!6FMIfsm5360_W<oTHLX+GyHVg8qA9(9u0`h=Dn(nRWL`<iiiObo(9}x*8JPd;)yu
z@sd(e4195)k~n#CBrF-2Ke+U|q%;~GeSD%>B|y}f@_sToCzMMXyx+U$^L?9Thd!6(
zgqO~xZ@#w6k;bv4uROlmVP;?ax+M+v3x>5^&GN&b)Jg1Ee5-{#EK-UgOq+<5g++U+
zp4zZ;CTln=L9T?kTuY;BV|uS<iCKzYN-osh8{;IsShGx1W2&|V(q2WmD-!BsZD7&t
zFw|vB(-PI4#hpc%WuSqnp?B4Jl+%^gRHCRi-So|(ceJvtGO7|f=2kqZldY?vIa)64
z<YmHF-;jAR{YHzFMV(j8dh+>nd~;a=N9Jd@=z;9Yq$ag2@O0i$vCYrIMz^J9T%AB2
z8=J!k?PdjYoz<%T$@a<ZO^lxdnkFi$WzjX|hOlzIvib?{<qO?<9V_VSR*46-N8Qo0
zEg|r_u1>v@$7~{h1xs_WkI&#0>1|Kkr(-%J+pi3{VkiOwPH!7rL|t@GOMDH|#&yIQ
zqhuHUd`iFnO6Kt(Z;5>=Lt|b6UwZ8?%^zi>52`<WYSzB4gVkK^t`S($jV5QNKH)#T
zRGZIB9~vCGWj@$<P|;`U<;s1vvFUgZHqn1Ot=aYKb;~%n&17EK7Ey$OL!;9M2h3rJ
zFo)?an7o9daoKhARlTamFK5_#FORNGg_HayuiMc6a)M4%HOtiUbd!VmM9RB*cIdV<
zt+UHC!OVJnn)CHxfdB1Q+&DohK}NU2gPMhqqv*uh9v`a}hc*6z%}K9Ai{YUlA^+64
zX4utLR`trmORaXtMp&hni&w|_<+*J`{8h$r*1^(_=x%e&b>(U7c)?>I*Gqe6Ht*8o
z%8kQgo`M<6rs@5@dA)VzlONH?<J&<ZF`mO0HOB-n)k#=Qe}53n^O0BHh2>S{lFm-G
z^lW*vPu!mA#efD<5cX3p!ns}wBzF{~eRL$}GfgES#_vCUzR-X5Stx7rP)kN~aY2#n
zzl~9gUYd8u%7XnmoZ8n8JsWy!ct&6N+xhH(zL#Xo(JgDrQf%itwbvVI&EzIt7ioJ}
z=XW3Llh1AMHzS*(VWLn>oT~vgY^bTGjG2N05+m>$0|^zG1PKjzg$#TPBa{Al{S28N
z>DE8qM?pdgvOq%p*Jl)g-|(+U;2S>XpTBQKy+OJS{DlvEyM9FZ{b_XSkGFomzBL0p
zLlRS!kdXm?tC~2Nn%X$NvURHU`wAb0Y4=>q5eex54g4EfMumC@n19SdP18wJL7v~l
z7Q$lm($?6N#T8-)p9e|Ml^=KsF?BKmxk9XM9Qj>^sQ&Q@e&99yZB{DKKR)7QB}ApE
zpbV0*bua~Sv#_zSQ3>Axfk1)|FU|N>B%l3jIPjMc)hj0_JAPJHC=|*9<z%sSFlS}w
z<Ktsx<6!0BU<N+H?C56WWaP?h<M{ZWll-1X($vw!!NShT!qx@^pV!FP*4asjiVD8a
zpRa%J)6~`CW+fZPe;o@rAS?U{D?1At>z}!Sp@Q&t`IRkPO|7*gEg%5RfHj2q_}B#h
z@%jJp<Yq;Lk(xIn*|~T)5Qic>`uCygj;0P0wh&-VC*d1@{cAAd!+#AFWQ89cffWCw
z^FQ7N2rYa^ko6DKgzr=iQ9J?oNM<3a1eZ!c%HUtf+rSU{fBuHQUN_5gU#3Dr5=D}c
z6jO6W-k81}e_y@s+io>oYviZr&m<ktY3M)t8;ia`l~n5~wjZ<8b9`4&%-4a|R<ppE
zk%5ux3;Oty8kTIdCqK(CU7zxh_9<t|xj{eY<g_~7Reo(mb(1j6u{K453<Knggz_JL
zh$5lsqEe&e$_@R855PwnqG3GWkrG8hM#cNz|M2t0!i&V<WhehHD<L6cHr@I!i{RNI
zqu?XcW#9TwivX*GKwqK%c^Am2C@i9)RC8@k|M|Q~D5%sYIR9mvqF;c+q9(`V{D-5$
zY2fRNI{P1K0X)E{K|&7kB^W~fPgDnpiWfO2@ZYwHOaubmvoq%Z-xK_wtE1q1-u^G!
z1kC}5MNyQ${a>gqD(b)eAB6*WfU4{3EBPg4^}kUa5DxPHvd#Z59Dm8m|2yI6qTAUh
z?_Vn@X&OA){S}*>tlRW0G_TR?+y$GASNV7vn%2krE8JMtp5$gjexHGxOJ7gUbq|q)
z0LuhV7o+PAM|B8^h;p`@&K14RkFcMol{6kD7T2vY($Lbb1>g$*nEM)VYY_J>)gMW2
zD+WaWXX9}X+07crs4cp4@1-dQQlK*y0rza1FRNJURx*6wofOw^4YuNn6b1&Oz9kpX
zkO;!Te&zG@-5-I((*Y#dc-@;*=Qk*TR!~H)FUlW0(fnAXQ=M=B>)YByWr22;^}_@E
zAM=zb-5;2TiuLL@g3z%^a*bNwpuL*1sW$6Nw&<^fOlrh3fm!TQaK93`o-V~X_i~N>
z#F_Ovp7y={Sdg>+@@Vq#r%42Zu&_P>oub<H)um9JtJIC)f7Asw)736*`g<=(ehaAH
z*OyQ|`*y-aZiu!Y6-!+A)!Kxhb&-CfyPwt7S*ho#11V~rW?6=!(4m?S7ip=ByszPC
zq1HsHQ2<Nx#h&k>2?@p1SCPWk7mkzC+cS-a06FfjuyS&$GB7Y4O+|=Y<pOgN*7OC<
zVQl;?laQwLsyNyhHwnOJZ8|Zw<9Bgq7t4!Q64r{QQ>ywZL7@DUiIx`u6X;6-e(c>*
zq5Q*+IXu6^^|8s@<I5wcpdQvuor+iK2HvMqR%6ALOCAR+=>UH;XkX23oBS*_I@@W*
zbv_z5UmsEuIfU+e+xA<(<#bqlUeBy#V^cgk**|M|<r~P?Y&S1nX)*Lohnk70JLzeH
zZ(tnFifvC920;a<ScrkoB^T&gQ8sb7&Ut;f$;)GNx*pswbevq=e03HT>WKYkZ%D{#
zs2I;r+9V&{(75#;o1bsL@Ocky$K^s8>r=xoNVbOn8PuRt_Cc2ya{;(r8(Q1j0lt#E
z8Lw~QkLf{EQhl!6Y|&8g^+O{Y*~K#6kqdO8<_KS%m;lko+D8w&C2)l@>$jV9xGaYw
zwVYq-^C1!}0#2}%7es$HHHU}1&o*-LdBXa9uf*q4u6oL$`;lqSt<meN<7Suc?6;&m
z`Qoqd%y5{kQ|&8H73Suu=Da&@aEpHb-UzG}f4uAm$&!uj(ea39&ZvS+n$6C7Y&W{p
zC;W;K(Whl7g$0L&MQbG(X+Rc5s%&RjqIJ^EdJ`>2@?&u*1nJB94FPuNs~3Fe8GU9J
zEB&IQE0(F!ZQo3%ICqNJ5!R=!t({21W%=@25E9Sq_JKvuQp9_|mxtr)L(Zl=)g1TG
zpJSs%I_HOCrF0;pt$~Zvef3I9b;v|T=Q)<A!7Y59D(itvndptsxaHoYNoMbGb^D-G
z{rJrw`Mx{Ra+y230O!ZId)Pb3K220w5!DEq_mO4g<aqQSZj29UKLnRw#%PHdYRo3P
zE`)HL;)e2vACDRMKsu6QVyHQn{chcM2M63QHj&pT(#`_dKcRl-Tl-Y^>QK()wo<*z
z=0-Bs*nS5Lz%1>OrnCG;9@=AB2Zuc)E~_hr1dn>xUE`&trOtTwE1aD4kpggmR=HVK
z7iYiVCsNynUAuhM2Y=Xv<R^iad@cR{*3CP}&6vC%)AcFWYfb{Vm(zU`(^Qkr{`acF
z8DN5o%*xro%{rO@&Zjrma5{%d-f)<F`v+RFf!$ZM8LRB+g{$P-4QOi#xSymWs+P_V
z*I`$C-K^7jd3k7SNwrgMu#})-PC7Um>HvIelG2BwkR6=~9Z#vwo!zGNRQ?#T@a_C;
zy@V(iJcDbV>XyAZcDhtQ*-2<L+kCp5G-H-RhCgG^C-!jr^ujh-G`hOjeCpduAc3Nt
zfLPFlRO9yBA_kA8Sg?EllFu({<*Koc@Q2aY6<*t@xD6&85MCX>@$I*;`h|x9Zp>@0
z^RJ1f6n_+ml4chY>^KPUJq^uY0Oa!;(wzpKaob{0&8|uL()Q@^ZPxym=I>`qd=_NA
zCoGKzgRPd|Ff(gYKKe~!T9b2V45bo;JNjH-9tVwN=b5Y}Zu8wnuws@74POWD8;l{?
zlvf07%^BPtebCjx_WHzhyY=O`&E>dLz`Y>hu}OY|OS#4Du=rkjg7CJYsQC3she!jg
zm$V!buEl4BvmfR<6a;f!&5GM78S6EV&K=+Iv#p9}Z}vYXTaMecZ9Mz{`*b|whEa^g
z+VCT(W{i4AXtQEyqYIl(G<~`rI%%-jAI~lUygDA`v=wYNgQDO&P#is|F%ym_Br>B3
zB#8WRc|5z_DD1c_&xhN!X}&M#wBPOF$m>UVzIIBvA9j7gM<EF7K3mgzwEN{2?)O4Q
zcE|0}q9lcU$ix%K>%ejGS1Ui#UAO|YvWzt%;$v}_8??R}M+GNA>XMNp{2nEaV14Vr
zeE!qmny6;QlWnQD<h0lI4(Y}0^<`R*EOI$^e5ZxsR_E;X6;!~xoUyggdxxy=I$+Vw
z%0M9%pOd!y@_j9l*V(~)nJI{dM!dTjTA<dcVH5Nex18#W=XLU24706%%3L6`?eg)Y
zZS(lT>*nUuN&0mUVCY20VkFxvt3ia0ZKJx9@M$Kq&9Ze<!E$Q+`39=`h8IWGx1VFJ
z``hC+3EY3_gg8;R$0WnjH>wTP{DmPW+1TfWkn7IeMGmgp_G$Ympcuh(i7@bXP`x;n
z&*Q#Zjsoa(aZ*=9?M=spk#<+D6PwnMUoKUT_de*Ao4ti{iRLJ#G{A&xWpy6STmO35
zq2l^FFsI4QnF4Q;0}u*7VL>M9tJ8j;x*UZhvvaO}mnx~xdT|xvE#oeo_3mw*Lh~*t
zvNQ9Eu`H>vG)jWIfyWIn1q&u>!R9rd2kqpkBeWy=>gB(hR<o0)`ffq1wdReM4Vydw
zQi*qrzv<Q&xI7q|?yFLo>F(3eu`xfV_vjr*>GGJ_|5h&EQn8NOG^Oa<?bzupBIww7
zG-1^UyWIVT={?S#;N2^m(aM)xRNCOktlM9)z8GjH;P<X2PWIvg(0`NM5xfh*_}%vR
z5Rk<HHB^6SLr$R!!Ie-B^o6Cb5Q24Iw2d??@-?4T<MQ_d)=IM?|F-zUNVZ9wC7+#*
zhd&|J)5~7Z8*FEV`z_8%@{mK4Y@5d4&F)9emdC2dAY+#IM!PB7P%OfHYPON>z0++=
z1PbH?a~evn?mmxa=MCv@gm|FfziyTT?6TtHCcXl-h9x1}RQEN8ZD+cOU|5CMAu6@}
zTzW2~$^CH6bp4FvfboYbCS`Ig=L|bvftf75){cscOI5vbduY{`X`G(MjxnFGoAp~a
z`$_xQPj2}L=3G3W4T<aL<^QNdj0#a>;|wo)OYcVg!GT&wa~+#6zQ|~yuQ@>T)+1_$
zPf~HyywCEEAC;A>M*v%SJM0@+zVx2F8TPe48Z-s*XQn};mjC!wFuv(xuN#ZlRoL@z
z1bkEnTm)Fm@kkmnreoS3jE=olDjCu;1z0slN9Hr}AHyD&L`OD!aqGWLXHj|F^q_d%
zp25b{*F39J%RFVtqK~x%u-OjYb;{&>YeM)Z?cLTBMMG*S`yS{9jXaxgxdz0p0g^Z(
zjsRR3G17o<<93i6_Q&3|3-+Z$wzt4^>m7Z*Y_v-gr>pYutsOvh)Cse~<^E0+Wy6N|
z6p=q`9dn>^jC8m`3WFO7R`Z#osgSDVse4;5K0PDfGxwKEt$Ph7JK+Sqm~b&nB^_?|
z@exRE(cyC|PTEWr{KbX*_CneM8Ix{1q-cHHw=I;Aslnt7&=mW)dBewHM650z8We8J
zrd8D($eDl+|DfHRi=xPaEkF^CG~L<yLhWAPqdx{WXs#Uatkd%7ga4djF6_%g^JF7R
z!$gC~I49SmBcF}3jyw0UI(!48ohHsBf69dF{o@>L(GU3?GE)qKaL6~NdhVd^iwgLN
zw#Z0saqS3hu&j;<YWDlUPoX0G?!wD`4;r6L)lS~o+3BJWm>(C?xhVLwtU>1TX5D$M
zLOovYyX{2Nic!whbgV%z;RKNE3mr)8a8M_g@fuNeS7v6NuEY)Ed@Kb7tA)z**`K%Q
z<9yAN$jVVNYD_xbk=sO<d=*?F6Y=)+GoB}o{N5~X+!e!s_Kg%9IfT;t<SVIlJiEyo
z9zJKl{7i6)rd6=!D<_W}U&zfADL(^U-wX8(G{^cxpd`qCt=zDEygj>}0>{ryA={Zi
zMv&3jQK5tDoZs!94h~9MzIsf>c+qG&diz#v9x@FN;81x2FTY2&=1Uv>u|ttjL-9eN
z7cqBWH^ySulbs2x5;sb37TucC?=E|EU3#Bi%J^}l`UcAI>YlLGq6p$@C`5j&X}zGD
ziP`6x-&q;>1UX%Bc<y;JA2j2QMaO%DN_}>)s=8<=*gVL(3cfV?7GAcQWKePYY-dq;
zXEAc;048JO1yHtL0SDt3m=|QfRSTQgUmiOS@^D4LZ>JGBzDS(aybpE#rWs{=RR`$$
z9CF*CY=!-_qb8nHQNh_9KF4L({v@9CyU&vy`}kaRQ}~@3*|I--vlUPQ2ASEU^+3k!
z{@(bHDge4dgiJd9{s@w<s2dOri2+BR-9!-3P&JBmSC6buO(;f*gyo~Af4vjvJGZOY
zX*zgI!u5G9R5m$T<mU?&(H2299^6EoY8$<pb1D%4u$I$LW*R+k8B9<}vm4h&3Md^m
z>(-0Q$K>~ZH8eD!jKwkQEPH*_8I(8xbXi-K(V1qGr<))5#btVpZy*fW^YtCVsZ`y2
zfQ^qUa#2d2!{Au+>tHc%WxD?Okt)Y2wxA!f4?4B*$=og5dNac3^S?gm*p!X+9#`&!
z;zUT>%M{&;m1qecD74&hjc>qfqPUGiMqibSiJHKzu(#JTJG8*KbglnD@k~#byzlYN
z{ZftsBsI#Z=FK0nDAOWG=T7X49?p1dJ8w@`_g9qz26?slp?|Fn3i4~*^1&~<1qRJc
z9{s68N1sn$AMf{z7#J#DM4{kcpN%y)T>4y{!e(xlnuVnTb}-S_;Ja2%{r!T2N~<yV
z?uNZCrWx<;x^<Rqw|>&{s8-dn{U4)(J6yP5dRwjgD}hO9?dBQGtA_bpcRoU9Dom<^
zyZZW$oWNUS3n=)f(K;UARCqyU_P?G%1JJQIG)cLwSM*B_{h)%&eobz{wkSB?YIo+f
zWbi2|xAg(N-Dtf>a{(BN&fr4L0RbO?gz|Qs(Sp;3#ov7lu7L-&3Wn`HKqSDHS#T<{
zIFe0FpuZ~-Fxbx`%Lttp0At@7oTt(b@wZ<eE!uz3+0*0X10cFh^wdPpp+6SOjy}`s
z+taOtShnoG)toeal^4bT9lZ*C-SDQe<cPK<%=YmnP>BU`>w2y1akULx=h1BWqNl3O
z`%`C<(Kjb5_w|_=KxL3gZYUtr4W_-UR|3#2Yf4W#8XEGwb?^Q=As4?maX$-9N+;}R
zH@PVTSCR=$J}I@1_Tzkl)lygN^zzrnS`9!}E8Ir4%-trR%gES6Hz!8wRlcH{TsYq+
zie%gLh|=*`1}JLO)sw{2f7Lak`VEy{zQsg|%FOX<K$?<E3X5#k`@9J1ZAQ~o)<Lme
z{NmFaA@i!>J!@<=HPMzuU*5IcPYs8?vV{BN7ey}k@8X=|3DK8@<Lwj9Uv8P~FZVV=
z5{feA4KsJ{brIs5Jt3f#FEx2us}Zhe5oDhA4%S3G1h}wuz0gj_z8VjIv#d(kDKtJl
zenr2|$-<(&_9kB9>#GFBC_Ifb@c!8BU*_*@OjLGz{d&vuz*QMS5k6Y1H+ebb*k6lf
zJF;joD@fJ^w}oU%6v)ea!%vs0$D_XOm(Ffn8w?sbpu`8Ar3RIyjsc!ht$A;v`p100
zi$tM=8|KHzRXK_g6hc#Ggc9~0fJ+OzM61g)nP7MBY29hQ+_ZJMiz|F$Xg{*)_6u;d
zb5ZuWAzU_7KQDm*)d$UEI~Mmng8O}sj3&OPO-&qJ0(XRHyuvLvMPzQ#o}+n<QoyAU
zDkVGL1%yCc{k8)zjgws_ENsdhmNTOjKP#FgWwiv+K?8HrE=p*oiIfBY6|hGedNj<1
z3scB2BWPe;+YyXpg>+|&(s~Fl#i$mBU|J>+KFK0C8wu(pd@q7wv}{$C$pJ}Fzu!bg
zTM}kI5_-zhgMfX$&8WKBk6<Dyh|vLzVBhlTD2+nRN3gFY$wBLugP9@8w==VjmIQrc
z_q(n6Cf9O-D3$}oUYvH09rAu!T@p9s^JS7tvcYfx;MTtm>pWF&VrkeB7ho~>pyrz#
z`R?|E875Yn9QgfcJ5u(wV!Ru_O!ZPB&ij2PPd>32aT?F_6>C+~Nf|jg)pE1mMZxe7
z`%G@LjJIot&kom@team>PgGbuHt+tZB0Xn4YYIvE67B5ZkDoQVD~-IbXQ!e1eY{Gh
zuhoUGfI9So<>V8qPi2`=D>@$&@5b?%XHIe~zJo?k%tQmhM!B=R7ad*L<<;4qZQoPt
ztD;0<1wHd~W`$(lvYy!`mD{p62C!c|0T7>dtWL(ffy(<2=*c4KJyZHI0%gCT4kt4g
zSmSkQ#ZsA91}TY!d=4LzdCp2_8;JcZehdg+i35(bsYNLNjcUQT&mbzA@%(`zg8c~x
zU(LG7_dH|FI#u7t2W5epkHO(Q)saSzV_*3Nz_}X>zW3PLpzlb@#aTs^KjXz8jqMjt
z2yC&uIyK~Gi>^R;$(7m_8;o4Olra8m<K$Na?h0PY%L#7K{9-!BD-p!tV-h_0xZlKS
zbu*<WN~*6UVE=yp{CQ7rp83x$`-g4-BnAjx$wj%`SO`=aU#KIwJr0vl2fxaGCM(mt
z`&V2PkNy*IK8}_bh#{~)tNn}Go<JsGzMhcguL$i8C*ZF>hQmXX7}8w-KK5oOcsqK&
zzJc2sF_M2QF2soMKj!n}xB~=(ACrwB4Ey&FeSrdLnkJnW_<!%<uSvV;0X~M?sUg^V
ze?5m!KMPnECpJ<t|7FMjpE&;yhy!Ak*s36OFyD@GIFhUFOXFK$lEaK$k}YHaw=b*c
zc2UAk4gOkoP6=SRCn2Ipe0+rreYbbi`@v)E{?cXw?Xe$?gL$E2S6vv(w$OSMQ`URk
zSZ#*l2~3BPm4X_^X=<>4HL5M*44@<6s<h1rnSVpP{;YF-sxw=iU6X>(YA3JGI~ao(
z<j%d@wS?{&z{uK1Nb_pe==i^e@XoHQ{3U)F;(#NkHua1a0lr1~&>V2-p_koVddG!R
z38dZQnw=whori?8E6>__E|SVeRI{S690?J{IO7>G&+wEit^Xfe&sX#zssk<~bTe8J
zw7^Hl+E&c*RQZUpxQ#^7Z~?uYr>PxNKJ!E4Bg-4i0{Mzc0_nZr;K?=+!kyh|7M-b5
ziY?!G2F&7q&ErkXT&F_9I3I6L2sYm_8qT0cL=(wG02rRvn}q*yyhV}x9PZSVS23CI
zW_3+oN%86I*&fzRnA4{p>NwDh9ND&3+?c3<`D*kC$b~+!xfOtLgF(C~ZXVuheHZh{
z{RA&Y$Kx;XF*j$yx|K7g5Px?BR)Fxb0Yqsz_{R1GK^QKW4fiA$FTBea`*xv2SCIbL
zkAV-A#xL-%An~Jru{Q=7EuN<|i!dds49I+Gx=Zg+InsS!iD$QWOU1MN_M$`W#So|n
zA&TNF`W%3Uz?if?1ma;-`0DV<m4LU-h;B{4pPjsZm4C2YeR$Xom5WW(?%duS0wGcl
z6$~)au5lFsR{cKS0d`*CFedzPSom@v0rOe7yV}~g#@U4kPrhZ4`NhB!$X~m_fbRk?
ztr&3^nZ8<t@5>MBL|yJhR_)(@{7s-<J}dh<CbNOEXRFn%K}v*77$63AfRLkhB<LfM
z1|wf|<{fM7<vNJY&)G)#oY~e~3hl9VQn4$I%SGCPWs1fOOK!xC_#vS@0^H7fRIYUh
znJADl%2rQrSL%TN)&{xX8Wm{s0crS2IS<&FndT^-73|RHd!Y0Qf%YIgP2dvy9cDQr
zaMS<EoW{|VYHam>66jc`X-gLCnX7h00iS3{O0$D-onQ`P@(ali;~ub&M?9fM2$DV*
zCQ7~hc8802K0)M^S`fbh;TfJ5AOlqE-t5{4YlaeipULV;OOzY04`U@T1LN=z8wI6l
z>UB11JP+Y&vF5&e8}UNQ!d>ntoG}Cf27~;Pv&NX9mss~W7dX%!UnT2#wywwg*gFkQ
zV7i4sBqU_>U+UG4NC=pKgz}zwPEjYlaJ7gUswgR+DfksbKn(03<<9cHAfucr5HWXv
zN(Dz1$tha&kQ;W0cB1gf6~EYe6DP-VpBCIr9)99Tv5S3`9jL+jy0efN9Z?EnXaP0&
z^ilDR`2oUPM%mg(={ie)ACvT8`El$8VR$LQ+mhVOUt^uInw%T3UL*v06ZK_I`m0z0
zZx^MufKt6LRG`J1!W;Y91>a=gas++T>ZXwh6#XUw1YG}rD{}xwYH?0<c^SUY?0<~+
z6@$|VoW(7>R>oEc>zhL$ujm{ua+UQ$(vAP?D~gI$gUjL)rB#i&{t%TB;I)%Vzm!8(
z)sg4oU+~$Y1`sTMhWO3v8-n?=qVO?ot8V3=dohqiarogpi<fEVsRv_9PZLF*4lp3n
z>jbFYXn``xA_|cU%6Pd4gKhsoDQVn8Wj6nN*=aSFLa{$L>GnVT#jO@iU~%GEvNs4I
zC>ny|_OR=sDL*Do-MCCy_pSz8OCK|fOy}hh|ND>_W%*cPL~4JL1}=%P_=~$YBIsvF
z@O@H9VZ8YOv)Tv^Vk&tRLg$i_59pyqk#-LR!N$Yj1O^+qND$*79I5HfVzI|FzV54e
ztKE+=SZiIuo}9A0%MaUm>ey!=&LAArSF{EbD50qqXmLg0#?A|n`RZYp-nD;3)-8i4
zY5?ELPu`3%q%ZCJrBMt0g;xpiP))+SP=sqAr~&er5nb9mVXO!^DL+<5f82uyKums{
zB0+15zt~^H2LOo)b(w#HuzMrcNUp}%soQaHJd@LLo)IH~E#+tqb@3gr&=JG~38fug
zLNN1&33260(b0-C2XmV{tHiC3h>ehRI}c->THZuz|KLMd0VC!Ka7A8H79&ssuL>n$
z798zx^v2qA9zVPk2~$0G%2w(7lNI6{z96H@0#@;hx_dMT)WVSS>#-%EhT8a*W!<IU
zVW+2bxl)o<z?22ILl;89d!2mk2CzGb%Ll@%TfW;O-$KxlAFYA))g$*f?x^8NiBkPa
zLXp7e4XU|MlNLjf*tGhW#CrnxG*3YA8gUaMeuWH`OMNr<yl-KB4b}~*j2VV{V7O<?
zAxg`RI$(^KsKg*x$e<q7&)(LI(6dA~QuSAFe&zp=lxYM9Wjnh61#15zK@L<`D1p{t
zHV)h8G#BU({O{G~JE;$rl}r5K)I%VvuWt#!Vb9nMG6Vt`(MC2J*6dYz;+cKA(`&cz
zjySu`W&8hfP4jlWF1<^;6YM9bk3((UbMO;I?)#^%{$-hY0xsBm!Tlvf0&)6H^?r&i
z-^uvLIir6ZiG-Fwjb8ERfd8V93_u|z`TOEGo&)Ftb;^P%=5T0Wo~INqoIY>Np_Y*w
zSFGO&0ilslngAzeM?KU5Q5@4nM~B@V%psVq@6l|Z7iT_>88&y^;m@Q)T#`N-?u?ba
zz`T(&za6~qsXnQDRo-M20hj~y3u~!4ZAX-Ea-oR&7zx<`?#(bt@gYptq71U#CpKjb
z-<-k>fk-nI&wOWV5V(slIWp=VaC5$wd_@EjDg}^v8b{9P`#p}wkj1Nqwz{9A&aI>I
z^G|Yww8d_Sf#l~42Ml6nY((`!=C?76&SX1zGUxQbtg&r`PbY+27`yxhI~l^x@aSa$
zv46psj3|BnsCzE*%0~^HRIl2uNG*+XSnq?^ovyi_%OTJagvSP;9P+0uBZTSvY34LW
z4@Mm*bx8_wS~DpBi4q&(?Sv108Q~$0R{jhSILfOg^hQ4brt+eVrygHeNqBJCnN=ia
zk%6k`CDX+S+2&vH<O>iOl{kJG0k1JaeWQo@t=#2mhsIE|>Xo4j`9IpU$ma_&?4z_D
z{~G-gZjXnbx6vaEr&kBru6J~$`sn8^Wu%K5pGVvWL!UpDf6a$56ps_QG{(ysoCsls
z=Xmjaj@dvZM|NL|K!I2QdXUiK#yB0lZ9i_19%~e1o4SC0$aJ>UVB|w4gu8~p3)jgc
zh?74<0ZL;k;iW+W89;GGfm|F5H}AX~WH3`E<dZX{W@cepy{y|qnh#+=!|WMk`HzTd
z87o76GO*bpklz@963N0Dx8=8gsEYRiWp#BmABaP8aD?K}$6J{l2GWTd%}_cIzi_D4
zojF}B_R;5)v)%E2m(7!8C6wtk!U`S<Wyn`yAHQnvxa2y1mB{3XAZ(xsz><}mBsUel
zVdp(oyDo;D6(D!HYe7+IITEF%ZiGT`8nNM)!(iLdc9BDM<)vHXcg8+X(<lN-hqBP(
zsIv2QVk%$UOWtWnG<;3>Y9P~wOZ~a*&geWgBB27|ZrH0SM5y&MMuX=!ryaVP+x3ri
z*(P}s3?;)JFbn2fx%bXw6^;un3D<0tINrPHi**l=s#Ay)NUx)~T#+uCZW_xOAI{Yk
zJ5HGecZTc=$L8DwM*)hVDgukmSzA23x%9|sXgv=URI}v=+Cm7pyf0jVO0(j&I&yxe
zQqy&y2*sKYdd6tlzxD_^T4mfV*@6aEA*XRkdjx)M=2Bx-wK-ryc}-nCskgRMBGi~P
z!IP(B$#(~V@4ljQaA}^2r$B&3dKJUNwUINR!USSyw71yFar)Sk4~X!Y2Ru;ES7WHG
z@z{Q1+AE~hHEv)zUiu_bSxY9RIkv#^7^Rhz$2mRwiLUCL?mTWvB6)hjZGOxq#Hghx
z=n25Tz;=is!gVYIMaITvSqx>n1>gaKcc?f?oyF}2LdPM?9a0offLHp>xWUM`o%@s0
zC4wvuRt98cW%rU94@Vkpt3JbUVgx+Q^GsXwe3{C!@dcD(3S_G#-o$=LDDs$1%J8yC
z2-4-Giy|O6_=4u8&dindeLWd5^5i!MuNZFP6S`4RZIQm??w!`HFEzwHfnr5|AiV)S
z0&-GAzR0(%krswkmDH!}d?3tEZKinN60^7Hj5gHHzac%?_PN%vx3@>@JKy{2WmonY
zb`U4v7|q%)p+Rd~$;m_a25)u|gT^I+8f>|4+_AU6O?o(*=_aMG6dW-$-cdO$vzf%N
zuefZ5gK(+vFlqs)ikwA3uypVgP-siOvNm6207?C+t4knZi<WKfAQDfAw_G9}K@l3k
z%0IpbMC#RY6v=H<13M@@<g2G$bo{DtjspZt0vsOPRh>y)4btIywY0C%!32vt3l&)t
zq&ebM$JBf0?2jKCwt2m}%UNM<@->z+k0FFV{yk*<X|B4|<MS%;P~Au<BmnE?{$YUd
z$N(_@YM)K?XXAeK51z}(4FUCB2kk@#KbwJ=sjIn8t%K>|_UtS=sK-TKn!+vfSlhxD
z$o$a~{BQ-b0!~13F-~KudVR}!>B)5t;2>}6oPy#)3!GCv3v+koU+bdbr+p0M-i-ax
zSiO+ME%NfIjtk|F07`jPMVr*d2QX`|!~T8uby4>Z-xTydZk`XmtZH2w$}a2M8c-pO
z!pE@)AoxG+n&4-R{U>bU_*gtMrtI+WyW4k3d29-SGOwYl!_CR#(mf$pQu`Ufz3-3L
zhV!C(nciPFh|It?FSG$?ELOlPOC*sAN?LqJnjw88EVRHabTG|J(rxnU&YCMpM1kHI
zru>15c+QrCdhQBe(Jp74<w7#y(Y_3MqUEb??jtW3#bVlv&$_ccI$2W<g6awiRbzGB
z$3parn}IxtJ6E;==)FXnd4m*wn)l*4&EG+pvTGe-?l}+#ey8o3Y0|(~W5p@Q+M3Os
z-O7Dy(y{f+_6#K}i$9PleRMxZvo3JcJZ+rTy1y%pvR23{{NgA)uN|5apBw0s60=g6
z7;N=Zl6J>#wkFRdw(U}pYW+T4y7QIEQ1K-Rr>UQY&5Ws1wS}6_<Fg9YB1RuHuDv*z
z5F&h{!r{|U*W$)f1>He)aCGE?Ogw!{DKc}FfI5=;n2T$KHj|s_k)p_@Q_}WVN9I9&
zg^cy7N+kwpYk|s^flT!+{)iK`cAgs9-M6+Mm}=)jUr*6s#jr0sr`0J@aLZzD3Rr!e
zfN+Bb^2YB;jdCW5URJoK2q!d2S|p5HYS!m(k+^%lH|6`wHuHlA5s1OsHzsYcvfsgh
zlh@_p7&nqpmB~cqdW6A9(&})Dfq{97^9Mc5ff54|8ag`Zl_y~oRRSJCd)y7#fB^~M
zeZ`9SlHj^5B=5wXh_U!<tOLy`<7Y0Y*_%O%vz&6vUl9#>QV$%W{OE9nXKiNH77vXT
zx0L#arA3%Ewl&@kXL$GiT?-BL55Qw)c=;7Kd&YZ#Lc3y!uaXptM@wuJrRMSO1GA3h
z&fHi2^Nn)TDETDR&N922<K7orqqs5ER-dQNVKQ6z{<moSY8Dprr)2E86xBSZjU3ch
zRa7jtD-Xb_8Z&2HGySKtHBqWDo?lG?2#CMPar-OeV+3zw=&{9Xy)n=*!oR%1CIw&+
zC|9hU<XBo<{P|A)%h$yKHe?KfVBY2U70;DM6NAK2>&eGy_=2zv^_kSPWOV|$<YrTc
z9OgJJh4iUgDt%WXHp=3r3{6pbhjaO^6}ovHygo(u!9{T~%7>-4#-r5)F_)KPLBo#f
zdD$^xB^p!r(5l_|GeL9+no1P30oNwe%toR&x5E!#v)-jh_;iuL>teSf5ipRZY@)S9
zWu2m^k2fdB0POhyV9&#8-1;0=trA>s%}fG;rz2V&Z_K~YKo1>E8+JYYO2l=i-DN4f
zLmZZ`3NxdXpeW>M(O@F_p4)!_4!C+40&qtgDo1>y#(hQ5;6(}8;ytNClao&Ep@dv^
zbE2eyY%+NT>IKV<4~lGYUcR*GIo{{89eE%iMG($gv_2q~w&jz|o;;YBu%=O<<~iFJ
z8XJH+O>1gC$+6q3{S%VNA>s<IZD!RFukVwka4Aa`stC0WK=g7!cp89B*RtlqMtrkF
zAB>Bu^KBQxKrKvm+tldmx&BWoS+X2ogZBFb`q3ZDD^qE_du>&{I^Ef9VW6E2?SdUz
z7DWqi?|b8nCUpp3y&!ej8mkr3$+NAeWn{^W-0hR=(rZ$~B(GyUox)0HPG5+vc8=!I
z_t4l1ccP*2$%s-E5}0e%Z#GFI8O_Jv@m@i65?;aENH(GIr+;$p>{O*dRjn<a@mGJe
z87&}nF9okneRKC=hN_Z~I#6{!V^y|8%R~pJyspqxB`OwrSqF!jw5qqbe3UB$N)c)4
zrlimb8*Q$_q8D_|%9X<s+j2)0wvf3;dDjooG{hH+&-5N{H*74Hepeq%kce@e!bB$B
z+gKt!T+Vb`8~>b+{V_9}8m*9(<fiyX6f^;tM3U#^X5PPLO7uQfp=O!*cJp=9B~YXO
z9-0l9USl7{0<7_jfr>L(+0fpg@3a<&I(@ANMIJcrkiF*GlQx{S@qD8zRWY~gmOw4-
zQnRG$lF-f&l7XC}I&!^PkG>}MkZ)m~1n>9Ptxdh3PM|ZYJk!lS8QsJhJsisE+y}2m
zugb+lC*{a0vKa@L=VwZHqkCJXm?4ONFf$;3WzNkBf3kl-Yz-)V7z(2FeieY{xZJbb
zmjk4F;Q#@_TYzV)L-HGmOg0o!qYFI%&2wgawB6Nx{@5P+0yPeIWdjx2KkdtTWeMz6
zA8@3*c?Fyw?$K4KTFNdw8VlurFSJBzs&Llwk_P;`yBMt7N-IUkbX*Hj6gA%%M$@h{
z3W5|!T_%!@SL=21v2{%|VIx?YcwT^fW^AW&JpL`8pf%Jg>+y^T;j{1PkIZ`FA64&v
znI<z6pd|<-7WS+;JKo;Dh!=jew(MO)FyY81QR2-dT~M{bQn@yZ?xNmdCLN>QH;XoN
zbp-C!txErH);p>|#K*)l@ibp=@vw!Cv!l5rd3<OoUsHwwd@W(m<y8Q#AvX}dZm|9&
z>`>J_k(Qblc*2`9QCo}dVqr6Tu9C*_o8f>l-c|=#U7FYEY(9VXE05-~HI)ai2tt39
zt(dZQF(}gi?kN|Sx_Pqw8*K5rKxYe3k!Hjn@Z)Ybs+Ac(Zr?SJRf5Z=601EPPqJHK
zKkk*B^$Htn;t2Xs=y!|v%?VAAq?IZCqL>)GwCjWP?UZe)Z}DTX_15ooxh(TqMtC}7
zwO4;Qc=JbpNPtw^omq79)3MQ-$4cgxPpb)3S}S)e3H`W>pl}D60JL`BX6F2Z)>Mjf
zKx51q(2VoctcO@3@u>_SZIG{H{Ko)HxQ#<A<_tj8V;h^V6n0h7`*R+2*M8%5or03A
zYc+0mxRt!0&0fTzxc3!uoE(j*XB`bR=wnYix!|^U)XK6X#bzDwq^iz*b9R}`z7wYh
zv;KAeXTu{V?(M;!3|JKwW8MN)zusAV+;^|8a-)@{cmhW7d4u)fvg@&(@Fxg|_W-4~
zwwC*Z&1FN=>ufcLLk!a~elxCzTfX%T_7HH2%_Dgu_BjmFK+I?H9|M6xIgr%VucJqe
zBbhdp7difWSLJDOtK9}4h$iE-2<;s@%=okRU&zWFKuB4R{VrvFmMj>6UKTL#DfQYg
zQ<ayG-goCcM^8@MELvX25NNH_HJX=cRq_Ht01nA71Hgj&n=M+ug#d`*W5J=lIM5X+
zOa~fyxF?gk-Et?(&6V_kVto>Rr(F3A%XfR24S=JmEpSQ<03j6t2HPZ`n}gky&;=@2
zu2z;Oc}#j0W@LC;?LlPX(N(o&uGq9r%<5O~8623$qGu3Fg7iY9m;jy2hFm!l4cu!R
zHbtzn^!<_sci<Ust`{m$7!^wBzTJ3i4KhC6YxaILo3G9Us2#sxT7^VZh~U7?z#z83
z!FIn{zPL&TMeT~Dy1e4>o0Ri|)$lCs8kZ!BhL34$ynJ~6UY^I3mR={nGEk=Z?NQCG
zvUvrvNn)Aq*L?d+_XXns{2Q>ROhEfBe?YWUz-6N}B(L{_Jg0fz!Mk#xREL|c?fd)3
zBS3NPw%g2!^q0=+LTTX(vGuC$5k4X9@y&?6>IUmyiuIN)s0{vI7p1RKzR4%p)<bmb
z^P?N;D!p|Z6wAG5>7&EuqoM`SJ)R!~{Ru4qfnPD84f7uA%ep@-Ggbfz2<QMkL|Jf$
zQ6Zj9Y`L6|5nNmdlpA&wzu{fYjJ^-Mph|trhEg79J4`y;qk1UI;4&hX5gqqP_gz{F
z<WtYl3v)_ncwJ>KYxHM#*wxc%PM1)J{XHj`TSq}ULGIk2#Q-GsTY;y<^p2y)|M21w
zVLXp*Dz<n$Bmw9;a*SUb&PjF%h1c0&lRSM+$XH6Q^wC!3U3A0Q%yIV5DRiQ=pJGBO
z4R`0AHsrc1CZ%cGi0(f3x)?H6ICG8#G+S0PE<a21n;uOB;1~UZ3kbl4@AE(IjOY_A
z>*+dm06oh|;ObOA)MayG6=+iWFdmg>qZf7QZsfo7E&9{^cam7=i0GOL>acf#kC{1j
zpzl(j?7O!nbmYH{4i_Mau)rEy%XQ~cYc^YYl}zD?5cB*9Pu#T++9CgOIzICH_4LSl
z0VRao7`umK226Bx9S@3t2BBx+#OXi*GEgwA*sc!qL_*{E?64tmdF!E8-O+)K<t+7Q
ze=)Pyfm#ek6r;03PaOOinq~m;JX$rPs}35^$@c;7?k62An%4s^+D&}{i1E`HDZr)B
z8qN*)EeCD6UrL&<@_A-lCCucqf&L4f*KKW*Dc)zR+!*!~7CD`s%NQwNP7>s}?ECJJ
zd%eV-=BP1WL09>f+Uuf98yDEgaPQruLae>^j3uq%%dLU?=COg3n;N}cgj^X)I+YOJ
z4xKlMzL#VHpcq~?miJ)&ro&G3msBAScr1W4I9>o+Ts;Cr<0hFCdO|VEIs6|Vpm^Yz
zt!%N+G69y%HQAt!!ny|5<;N~If30TQ;iTy0JZ;NR3X*Z9EBWu6Gu2=|M>EE<Sq4+$
zgzNmm@hWMP>?c3YHa#Gp^b*I0e#}5Nk}t=wnEx12fb*>nH~rSV*+_n)6cA+TVXX#w
zAo6);Jyj`vF3LDKI5ui5M++?+#dTfAO*#0ecw6KQ503W_e=Rt#d8!HNIa|i<kW3kc
zf7^`l8Xg~gRkUAvwTGNCvbpjY_mc~z)c#(~)yCtl$ub2QI&8ItkXv%}WG8ZNea+=?
zTg6A@y0~Y9EQ^}v)s8*$NDkyzaOe3wR~q-5+Y(F*M+_&P1gzhPp{(eqki@+@4V@hU
z`bM${nKXWm%8M_;dkASU?15fCCO*E}<VIy7Je1oh$@i{3j|G-^ai_OMnoHKy<gg`q
z#H<_-M}=InI@k}b&MNz-vch86kDNR0WrfWH@$1N7V@KVCKeukJlJG#@d+Ri|YuKvf
zBtAQ`DtS|xACtwm_UF<9M67}n@o1Ed?GF)oQD`3>24WIySOx>378O7_Rs8}ypS37C
zmVvA|{rqolNpG989jLJQrJN5{dJZX)L*)Y;DhW1Y9nCItd-{cA78YV021Y_f>P$7|
z$ALx&!xmKfg=EpWNc|#bm7r`Rc5ZZpoUpG5;LMyzyyyOX)BP~+YgXH&==)qclLr56
za4Sy{Q&EWm+DTu6CGHZleR0z#CGz7m`Hm685_#AQ>q_O9VKHx4>LT<3>o%Tcf$xZ8
zmODp}w^;1?kJK$~Q1IvLzXcu05+Rs5sPM)EyW~~0-@=TsbUokl`+osoA(x+hE%bK<
zifFm)7bMNc5(Nc{fgZj|P@5Eyo}~X;X^rY#syP(Jt2aFn&zNA<28;PB>V(C0GR2l7
zw`Ph;$Tx%|)7N5EJed^{6B7S3pB<z0W$_I>2DC8?L>eI`v_x%@k0hK%)WNGjVH~T$
z?crx2e6(aFy>9}n4A2pW_L01tn1U-7u<YyS$(SaH<~tQUK->l~j{JShFY{ROBe=7k
z`)~EIcTZty9*z<fuLi`y`fgdz-jZY`a$P2|o$N@<V55pN&GkJ8f3xbRw_P`D=MmCo
zh|_)C-Vx-Ye1K5dDjEZ%b%F;;yDR?1D3A*pyqj`gJpKZ&09;aq7ZMUeBIsJGhYgPy
z00ot`K(EJ6kgvzKO;+<>#nkx`W4ad+4Q}J}J`sGxB&+C`emJh*m392RchOu_pk+9Z
zNZ(bA((B+oQ0%wCmdIt5%~V_?9;fe~2Lz!A^3w-cQGg6-wc95SeV=E=!(FnqXOu@=
z8@K;T;RnMLCih%giCKQ%zeoen<Hr)ezP&B?3kIS_6LPzKE}U=|thnjS5}TOKcLeYB
zt80WPCQ7Xpn4wr>GZ^bxk7k{iBiLqihhQS#%+$d!#xATj@HZBN@Qi^q9yK95{Rtw(
zNPV$xKLBFMlSP0jJYkxYt5;veQoq3gl;!%UetPz{Yw(FPo+zjV73hxMdMF?@c|@V+
zVBFBRGAd37eU)9qTa_GIs?%xr+jjJehbw)8>I9j8#cM^MV#ME$pp~f{|FGK4+8iUJ
z4SBaUUB3?WLyVLfwZ<eTlVPix;$YlonsX_>`0V)!C)2#Z;GDMoLs%Bp`5?#Pd_>q&
zk5gKKP4vqAS1)IeEVZk(pRlvt<N*CJ;7cO&ggpAaq&3zC&^QclAJ=zZrC~N`nljGJ
z%>0QIh$-o`KFVC>XKZ~md7N#r{26;~HZMU1p8$Ac5J2(6A3-AY43G42-)-br<}t9F
zD^CRy6-^e~s}j9BZ+cp<K8Xn-v9OEK$dA15F&n)Q&ijXwAUth=xao&E(BBC4QPU5=
z%#~VLYWdgTTeoq3Isgs8RXQ9AMF~LDuDZ1zi)6`{b%Xv9RwnN85#-hgThc}XW+OAe
zlj>NOUm2-;&$2ew6c|+oWe}hP6L@PC+b$QW6qkqQ%eKG77HE1~$1TEf#1_x~TVQ-e
zf5J0;^*SxSzeRr(jaDhO5D=PYK&|+WO}Epa4BaXakFM^_|Fy7SoI-B~Yc`vt5IWk8
zkhQ~)6S?17a7(@^@O&mIUocw4=meAh^RuP+OU3&<y9)J3RvR&B+GVE^WSkI^k<nn`
zO|=5q_Jw>qir?@Am|XwkAADqCy?ps{rSW)XsJvge;B+50dB;pA8;hkz6fg?lt-@gU
z$1%a~emURze{7RN*M|FgQ;J-*pIv&&CC;6TJQ+UTAIpXO;;*|3-=ZOWJ8nfg`%Niq
zY!&W*!yT_`_mdWa6yFD8z44bSZ1%sY4{8JMs<`K^mEW5))-eQn{<(SDm`WNlHYclf
z!hxQ*)kLe}ff2AlZ<T!6<sm)WIXvKU%4){IMI*DE?obFr<BfcuB_OS&@l500mHnu_
z<+v|%&mB*rJ+Gf>saX-;+L_CTmMekWZ}<PM^8E@gDb}@h|KsWvewnIu+^mwP^xW}-
z<3ug+&j%((@BrsY)!3Ly1;Z6>s&7;h*PBg&6uLF$)~K_-h|}9JW_;S)#>e#3ds&4m
zUr!x+S4h7YWVi^KENrMOi2YGyj_3Xf>-Uj?a88CCpfH@KmCSw{U`9_2DH!2x7VdkU
zbf$Zf@sDRZ(b!yf3`v85ei$fmL~8nL6~nW^L-*tQS+#NpDIq&CgQ}_1eY1GfK4`?H
zem(BXmBFhNC+n~?<VJ<9gQr;{hrUw}S7j-LWicTFXcZzPi2b4ep8vyhK`}#X`&njk
z2&82!1~Q~vJb^6L0MPqrMDV}p`tEov+c)mqMppI;*|M?<QO2FUR|wgX>=D^3LPms;
zy~&onvS*Z$m25Jz_k7RGsOR^-pZBll!;|~IuH!uB_xK*ise0eCi-=j@p}+A3e5BH;
z$|cBYA%P;oZ&jbmKGApcNnyro{M<$2bv?P=knj)OHZ>8UNf9#ns;=RwuA~hA<x+sU
znyR7vFPw!M3Z;C{Y(@C@e!ct&BJ-bg(0zTy-TdK+4ikRFBs_`Bt6)C=hllYUB@yGN
zDG$9%Sd~l+(KDBy&-IE>9w*&+xVQK#A&)ER@gyRWMxsaV%E$iMY(*ueCnq8{rOUqf
zXoml7dYXeW)aOs0zTRbA!1)A&mLwcdk2}uXxW#K-FP{py{#A34PetARC2Mn%Z#rl`
zFO&Oau5%{1pk8XK{*Vh1ZA%C8H1mRI8Urk<J+$(TYZVpOjKi&8Q{7yv{q(D4Yc5+y
z<UDwH_7dv-Dq@!_%~rwOdiZ_)Dj`8YqLBl~pEkzXdNrx2&!UH?8heZbMJHQZ+ZEpH
zm?V0LxF<(@*rPW03fBpv*T34Htj!zo;ON8f$XhU=^3wj{p`_}kXf0Q251o#0YKO7k
zS*%V-TL!GM(4!_8>FE!O{X$DiUybTOZsS3|5#kLSwfM9Q(Y6|46zR6B*mw=51013b
z`uN{8&Iq3aEt}G1g%bcB9Xz4Y(OFYXexq>FquoDW2N_FotdVTvj&b{m@&<Y(2349g
zdv`F7UsN=#mUwqM&Y8^p`IWzKsL5h9`@uHSSoHWu2Ed2+GT1^pbd@{M#4#5KGzH=&
zk^XlG9Z-LC9xYuGAK<d+(QeB^M=SRP4i{2EoQ~w?;G>qc^am>flx{zP{#3Ni6HWYC
z&7oB#Hie>ad4jZ{i=&FH^Js!M=`s?1q~);we1D3&T=2@MynT*pK)?Yf+F$GQ>H*{d
z)ZmA;S7FEpJvuKMjD86SobJzP);Jc$i-GYYbDU`q`&2;$ptXCBNrr&b$sAhlo>%PF
z|1zvN_;FuHCJR(wSTL_Foj_pNp-IthrZF?MpnT>l%X}~Gn9)7``|2qFiR!yhZhT6_
z3BP<4)ay{Eq!z3W;F0iw;w?ypM=d@=VO=##k(%7Mg9L<hQCOu65u@P>na6B+#U`b)
ztE<2G^qHlc@{U3Vd#{4bV21{I#)AQ6`$vhdOh=L;&mN|V{d?CeBH-T!%A-0iD;+QZ
z2-Z)e66?AuffXFNV9$YNtl3yqsb%#a9?*joMnglJ`|}!%2jb9!?<HpXGSaIQ{;O);
zLZi|ts#R7a351tXnYLRqbhWQGWmNLveanq%{~h|&1e)E;7I)bX@**JyKE)4e^@Gg>
zGC#|$5(jv9hYjtI{*;^bBs~J;1x~i+ZCp4N5wh*pufyl#(sKJom2y??qm{KkN-Rs&
z)AYXm(^MmnK`5bcWnZc)d>b8<>jRcwq0DF*U|X+JfW?%1`MaMSimf$&ya^(%H3{tD
zZo0|F;hzV|ax+a?V*eq2s6$x#^2D{@SJ{D@9vHNAo_@=)4UWEJHjtz3qjAC~XtT&z
zm3?vI0p(x8UPV&IYEZ|19#z7>{6z{}5ByVS&@!=TTmof%H6et#Sx}{b8CWF_k&OT0
z*?|{>J3`7Ba#YWFl?t@OK=M178X}ktMn5G+Mn(bx1N*^lip=WODjAE>vKyN(7O+5P
zi{6@p_Ni{ScJk@1*iX_evv7Nn>VJ|#rJD2xW5kr9j91k6UfM!~7IXnH08e`Fc@S_i
z1eBBo&>$l;VG8Zr5x_UPjZ41{+Ucrw&y4`WMxWAp3(ebYCe|sY);0C8#3C2h?Wb5{
z=uO(cnankq5(Ji-TCw6^Mo_O~*mJRFuu3SaH&p^S4PoKe;_t{!5OKS~u$^wi9x9Tm
zo|R_CDye`5U!{|?^)sOXz*5ONk#I4Fy3i*x8PaTvU$&02EK8+(1?LLmCXTtEs)B$=
z=7{kX;V$Ff6YxrcB!_-4{z@6LkOm8M3;ZaYC83ORET#jA1}aP*u2zVE`uN5`;Q}#2
zL4EYv{IXxm@{fZo=c=V2Y~x!s8;SJ*Xyvl#)wDN%<#WnYm~%$TCL&X5rY5cNF~F*~
z3bT~9xbt*d7J{(uetx+ECJXJ%lGA+l+v6}(|J6*4Q9u=JH<Yf3CylD-+5}c*Y=caW
z1uW9B?0%*u3MTag0ED1XffZ}h5)E(AUE8yFuT3O1gf)Cz6)Z+K>e}nHqwt?_mRH~|
zctXp2i+G9NYWa40-FBLm{I{5huU3j$C*p6UMdu~({Yivge#*F`&Qd;V@kBakp<m5{
zZ0PMbw0kW7#AE<ruQE-@m%s5B^#bgdB{0|64<%TjHy@kpyuUtGwKwiEkRlc+8HZ>1
z&cG{*;WS0}i3N4tBb?O@;$n}X`C@m7vatP~y4JhLuvhz;`&~7@mmZNh%0{Xchqo)U
zhwfDA5hQ3=TNH>Zd@R@*-TOK6QT0*NJF8TXp)2t|&G_dD6(fM8QdD@aP)J;cLc=yJ
zfsJN-Xyp7uWaRCruj$kxz6)QDRv#}978P?#H;+}2P^@=$S8q3aWU}ya>R<=+E2%Qy
zZYva@%uM(gznmKueWzG+y7O^Wmduv3*IG#y#pY(jM&gr*YS&NGrUOa=XwoYF@io*=
z-_YM)Vl9?Wa?|MO7glC8vkSuQejQI({G*I`EkK@gY%gl$!gJ#6z+CoL2LP(_2};w!
zta0JiEl{1V)}6V7IbXJ=`R_=yPbS|gxc7CkRvYd13l%@iad>UR;Gte3Zy2Y$FqE&U
zv=_3FxnI%4f6c-8IrGep@rQCJ?`g98k2}+cq|x3sP@%>OzM^dJ0oim3i%(Zl%#TkQ
z@7+c}>Ho>rzbB4zWy!#{JwV7ly`O&N=uuyLj=WoZL#)#S`3>#rS0fbPd1o*|?g)}u
z9C0`?YJ_&ZdH=IAvurJD4Q$=v?;7;)6?a>+7K=R(>`xJAGEBEt5ntsb;g9;U71g>-
z=%m*jmi%&g#gc3xL62Q0`V9{uaUPEq+2d)g*h~g%A3%cwUjFuQE>E=xWz+oK#{Vz3
zL5D`p78<UIn5ZwkK<QWu#*Ll9czy|(Vt)7&xaSe1!M0ltTG1g1Q`A%+{igRGVz<Z%
zFK@*+e{mb-@uK&KZ4H?Z1mZu>QI-y)s9G0QwJyU%A9!DGQG8d&O&sc!?k;7f^`rQi
zGE)B;q9eQil=ov^9E3{g<dOQ?@!S1_8gO`lw7iPbul9Vcr+SV$`5#UBQ7)}kuMKX_
zeYN<iU1e9~<&S`fvR}jyBTQ8YOR(?;=-7bU{N9a`vPWL=MvmpDKQixaA7#Xo_T`iw
zI+zpnHT=%k$qy>mjkJ9kVRy(Uy)tq?Z3fL~{`p)d?f?tZ?%-P=<@hguOYTV^8Pxpy
zwO66|CeoL~C6+Opwp%d(>&|lnJDC<5u2Yss<w(8OL{ZH4k)>j|uiy}LD(`~r%UfjI
zj?|bsV?VlQdh*T<v|S&m(<y&D_(Ke}J*VjsJwD<6v_r*a6#p+4mI#c0|7OJR%j2o{
zfc0l84E-se$Ok;J5pgoE7fDB})d~PW0C`2t*@F~%q}Pmkcp9L+=PkZXz37ZH()1-~
z#bkncdHHNaPju7&YomJ&CeBs(1BK*g;D#9kNdfwBp4U?)K`Yw;6rvz>@?-aLjvDLH
z0Vv*-0M`tDlmV4}R4TCN>7t+qudJn&<qNO-PtZbw1`gm(CRa0+SQ2`pWfoOur$;+b
zK&C6mSNRlXS+#h3%yp|fTRFXRic9QYQ|aoV{%_`f5o2g%WJYG9yC$@QW_NAEs@$*&
z>?*OaPO%)TDw1rBJ3k_69Q8SKHy(g*-w;a9GgZr16qaK92N3%}WMb<ovY-o)&x-`V
zy}Nth;aW{9*g+-%O$8TIg2itX9>+GN<|B7oxuVYzBEZsCaDSlv_5L5_e|XIA6}!33
z(g*)xQ47<7Fe<ukcTo_)b1_`OLc7Ddvm+r^&60Lyz(Uf7Rt?xSqA@f-xUqF!$}x)R
z0ltCYvCaGE>}RymV@b~mwR;NUV+Qhcq~+NDv50+?fm9iH7C{aR4#|sz0HQcq*6yge
zf#%z*-R>Cfe%cWJ6tG;cB?;y6c|c<fsZX&V<+facSxiJh-WdNUSKlN>o(eTnGJm)7
zzZo_vI7n(?NHWEjr>Lk{Z{A@yAMdaP@aPXv2Ox&q9;|SK3V0=DW#xuO*NbLIy@c0F
zoEDVqefd}}UFqZmvH%SzTI6?!Y!@L1{1ZG9R4W&zEnW-~d65uGDkuyAYC}52$GeUV
za4y@=mNgsVkRfGjmYN1-b6A`QI}Z%=@lss~_z&o=Ht+w3g@}1Lh`IbzJ0k*+Rf_Xw
zU)kj%4B{w288L{I--ra~<6fF4pcc#FLp2M|J6KNH>R{C$mw|GNaBK1eTcXqKil6p`
zPs(-F-?9Jjwta}WZ?f^=#Gw`MR#*G0Fdc@4-avU3Q*@RMVK=LVzkB;QfZ+BiE|V9}
z31q-9NUu9PS&IQ{P%NrJ&&T=qM`KC#KMp3Om`}V`e(`r}S89xf2==X-QRdG)(?UiD
zJ@KLVREANPS7-iTvbDKu+%*!b?cU#Qt-1Z=gK4()j@&nvoG$e7hC0DN3g7zxO-h-b
z0f)21A-0Zf9i)&~7MlhkFr5nX7ZMSTf{00WP=XHj6Y?lD<4TYx11f|%<RrATwCv;}
zN2nBE*Wce#TLRe5k3S2CRj%+%JZW2|Ur6Ke$FZQ^v)aPevXP!&C)JUEcX!<R(K+y`
z<Cd;vld9FhG{xP@K^CmRIfY)#`H#m=6$yM*^X8cG>cw;HOV!W51v}^%R&>EeJfJ#u
z{c`t5CPl&Oz2hGk-BdU?VStym`(*d`PV1w#Z2P94?>Vi-O^|oVyLz+@zeGtz9D`hC
z0MrnLVK$kWOfv%>ELKNqE%&WHMFi%DP0uu$24SN7l>)T;5hO|sO-`3`S(lmw!M?Zq
z{zNW_?>{#IGEnOE7L;g5Dv83VqOqPDH}%ydOH9>eJSgq7O61sY|Go#{AGx?<VF^kN
zk#fw{o3CEJS{qN_ev%mJawbS`x0*luQo?tP<mNx3(>0hFfC3q%-&{e}irMvH^fGpU
zJ+}<Xj$$LQ{~~uj!9~8-d55`_4H?H@&1>v>pMCk(@?l&J|Cr7AhN()dz$xv9j`6cO
z-W<ByUuUT+ewXAg^=$hW#)^-(?sPiNXjB$|ju>83p^taf91ebu@XxlW6rm|AeO$q-
z()%q8RQdg23g5~bo9~<ZZ*H@}5+|DcifQys(Fn%27IhrQ`D$Hm!$lNZ3DJh`Q}+f^
zhz$NNOGj!CYi;IuqI~_UXlS*0k(@0n!?j5MeOG$BKe))7$+3+fgI3KCC9gv0xp!4R
z;2;9kGnl-6?=q1&rk;FN&F#GXV(itZ#m~=1fyG8nHTgB0(gNE(A`tyus4D3O5{;(b
z%(oNgb#IdZ2MVnTp_Bqs)lVe7TW@z7=6_e(PgF4dESpbaTj-Z(D7o?7V!`&g!js?o
zdvl_)5w=sRAfsm<w9@|gyY4>g<GtGZVz**%B~uGL1yJHA**}W^#yOInfwY1C$oh(Q
zN$v&E{UxyBra$QBz1;KNp2u6?b3kFYPX%!azI$-WU<_IIU(!%3j)h$I5On(w&Jd2n
z1?Z^*sr&^}O85Mg3DSS)q~|G#*&qL4WO|?!ahkpK#B}J+0CVXcbE&Z?9w@<aKi<E8
zkT}0T!Sz;ju)B8e*A`#FlUSGUjtOg9?(!<%_xQgXqQHz_uTJ|3o%sf)-g<rm-7`M8
zRH}^GXm71&WfTqL>BHeV0jx;DIk~&?kFl0k@?7>O^6&M$=%2hpPwQNo_4YUS!i!F~
z)?r3C0=l9nj6gBt8a`g<%6)OUi=R3C;dXXh0G19)mx+?T_c|?E_T$}X0T&)XD1Q-p
zsDXwYL1WFzPdahzjJaDjxNmO#@W3lW*}ypNJMq4AKQ7Xki+_G_cJk91)9*qf{u1H$
z^F^0N9v7PI;F&+Pzi&X-nfHL1hr$d?tNugbbz9o(5B6<IWvA<}UaDE9lJFs4Jr@Tj
z!fiU)Trh`s@vC|Qq~>i|p_A=!et~dF>EYqwNdQ_mm6-HTu5?|$m(e2;)L@6ym#+9_
z*qVe|DWoUW<c6K4VvF5tLpm`coDS6Y5rt9W60Y32wa3NnZ-^uiG+C9upWIkJ3=T}S
zQ~lsH?aa$V{^bs<X#%Dhi;P{iuGncUQDUVMkWtdL^JgHC^F(`_a<E}~F1RGm5j3p|
zYXsmQ8$r!GS7yJ^^C%(V6M(Is52^9^!wzwH-R{AJR{0U{Gq`j`Wb71`mi0*#-D&bc
zdUfoEe@a-R_^qkMC&hrVg();Nx8Zl{6YG6x#x!z)nqP6VrJ|-0Ta4dKHF*As#07|?
z>r3DW{Ne%Nr$`C>POt0h;dj&vuYoljo7=Kr3ysLc7q_h^*S@ECN+T!GrB8<R-&Y)O
zBJ4Jec>D<#D~GT}fLcZ7t$M^xMRQx;QZIbTsJB|oBEotyW$5?Fj5*=e_qDLr`-s|L
z6uVx&E)0psYCLFTrfmjv=nFr8ZUmdVIKV!}ii&j$J!Csdku*q5UIqj`8R>>YBUHz1
z^c38mG%?o{vE{Nqu(pzFa9w>g0f#R1D&<xqcQ0Qh0ak7~)(AkOjW$Y;_IV!5UnF3G
znmF)(n>CRPkfMS^_(8WmX#;zwg(k9hJFZbt?&MpU^5yX)K_9ri+|hTbKLI#k2T%i8
zY7`7e{^X#g$f)r-?bxZ=AGfe%HYGk_K@>&mLg6nD_uLkufi0jfbNI?!{4d7rdN49U
zg@^s5K<7gsXi9SY?)r$x&R&p0)V<7fX+Zz^`y_ZiOjdGMx(V@8Ww!AAgM(YZ^hwdP
z4!pF?%v2EV_W`T4H$#qW%ark{$p7;QNuE;Z9Mt+@V~t6dZ*oXDTw)Slqt*=i0s3ZU
zfl}eOp}CSx<CDSWi?^+hKvq_u^j4MgQ0L`~wT%u6{;RTz^|Hv(p&b41jEXNmPMC_H
zpB!u_YXF^SqfkJC?xdw_UO=@xf8r!Igzh3l$&oK;k#j7KQ{4^-WKZo8NGeLHm5RWL
z!}%PD5r?{#u52{Pa}j#jDz~Qg%U`{L<`%*;4e@BHr>CbUmzDxR^B<2^;+@Z%H)6?O
z=iXd=jh5FogN>xt@r^!8@vo_LE7!juFAAY|T7;rWBxK7q<?3S5+~uLTrL!$EBtD>L
zW?|HbdynfvQ-F@?k$|(toaxfM{8AMB3cE$TarYw&@`tZv^NYb=DfJ3#YFAfROd>><
z6z~^GC<qps53qDFAB{<mTeD9k`QOeI@D~n~l%~5~pJBo>{AxMRzv2VYGwE!i*bwc8
zQw-|OD!Em9^4>bRTl-Jel{Fueu>V!wo*bYi&U4T7FHd;}4YV&OLHn!;;HwD1;@+Q)
zz8ZjD3Rn{7xAEGbn}CDXtMI7E`em5}+VzL`ObKI^3A}Fju^aAHK7FB_cgNQu>q$fk
zrgI+8EK!*IA(rGoec4P-p~={I(oW8&yPOvN*(dY3%qIubWrrWWCRyeDw0`;d2|nVe
z^k4P4ixeH&o-}#=(o}7c<r4*i0enBx_y%J$LKbYkjl{=iV~Z=-tgP*9my@Dm6}Xj{
zFW~8?5Vr0fl99DA_gs34lZX~gw2j{2=aO5GbpNc}>KOW$gnaRUw8?=Co#X2r%aI+#
zKX-1TFIVkg=%5y~M|tjV8BQFG4m(eihH-x?EH4!g$>8oOlSh_Mjb?w8AbR<Do_uIP
zOXO=i%FgZBa|tOdJRFyV#DApNSR`j41+-iD!A3DEqJrPz2<Qt6lRa~axj!8!bn@j<
z$$N;@BI6u%J*qm{O!|9ShV6M!xjUWog!eK6OoM*63#J_xAs3eUo{+88XDbtmwG@=-
zi3z_POAg$h9~9uTdp}J6V~}!=Frr;KhHobQ!jUW1Zv%O8)C%g0UL2v9D;ii)s6|0&
zGuKI|uBmCz=z|7^18<X)M{tHXeIe}#XVini4A}NR<?Pb%2;QW9zytSYHj<@c_SIvC
zp>>cRocb&D>6s6;W~CN#!<5}8FuQwC3JV=PPukH6x0x-NOP-UpRZdFg3OFi}+I&Ej
zF;UvU+Bxni){rZ<U?&5wed*%vVsKEu*YHlf^e*T)lCrYRU{4aK$Ej<YRKL%UTn&6s
zXp4X~Sp+x&B59tRNT<ARA_=BqHQoH5?n_WI6<iM#njSm*)#ytbHT5_fDPzYhFW1B|
zZP6!eh)rZE-9v9m4G$|-&{GUee(%XIhv4WclhKalVHY-%Kf2o@1;1q_3Cj7tW?qSk
z3ldeZyB@(mQ9ZCjwfb^!+wxZhb$6kDFd!kL0(`+Q{uc2_H!+n$3LF6nJbIm-wUW|O
zE!PENk~KB<V3GCx15}~FytlcJHh>@ZEpb6$HI00tR-E3F7k*lf`m;xG*nSOTxs^47
zu<mv04VQJ2aZCBa(lTXh7Lo^l&9A)<nBPJ9^v6r{OGRsLYcmA1mg*%haQo8BH2{g5
zrGT(E=@DXTOLG8I5{j2DmUiR#R>LM2zrh0vOM-T}$sfVicgF0AEai8j`556g*wOv?
z)5*0?!GTZb2OEAV{5=oGBw{GAG`0(0I-3>->)Cc(%v%o#9zA+gSX9(7+Y!!b)Jbsg
za$}$~3>@1=S+(P`c5LK3g~Kv&jXgZkWDpe=``)wjkc&(HptZA7zE!}L8?#)=|33kw
z6nN6)!35*#^OLWiMFuv)2_}8IKHihiVVMSFV9g+s-3fWeaWHw43b_lex0rvA*Pa&E
zZ5Gki6Q#r9O;m)|!fGwRPx{hB0L6vOtc&y)7CpInV6}AQf8Ga<AO@^4IcwFR@LYaL
zL|(vQ8nlN}S2%9a<mTqCg9!Z9Ow1n_?-z*X)5NHilq|J>+))Q3BR97n(#PKls8b%)
zBZ5twxTDWoNlec<f2dR}(%5Dsd-NZvDL_*D)baUJYGMRErltFUQORN0iur<zRHZ=Y
z7K>Uge5N(7`ip-_O&kmvFS>k(H-Fs8^W6?hs&R1)a#bWl&eKUtS#o)4TC?U_T5)OX
zRL?I+6oOKaYNv@lVXcVL3scd}zvFim@0y@+q;55MX;1B{lYqdmndJ}G&o(wT!a&a6
z5~b^ocKy1bT{~g$bw)*57(>3F;hs=AN3zRf-k&zllsP$z)zKA`uEB)<2nOO0HO?XT
zS99n4?y~PU^!24&qhD9KGF3)BSMZWS6W4<-au5}^(1e6{pb0?)j8hn}2AEZ{JdM6a
zQd3dE=kA&dUMcVcuQs#}b&vh9nzG>GC*Q`95fP84=Czd_&^@lQ88|{Xv7T__Y_&ha
z`L8_rFhBs+2OF%1K`ZT77x3(c<26p!Ha48|&x<WC4FR1%dD3>+Etk(4Vb2hM7WYG*
z+0it&i7DUpr}DUJlFB+)ZU*f61vX8^lLN&@poPzf_UEE>IZS^r@;h&%j*pM;10CoE
zC`ec4`n`+CuD3%<!*%<V`L4d|(Y^Z4PP=JkeYzMGR(Byc#OiP3uX$K+u)TY=Fq+Ln
zs<xq&j1nt*<^02d|4>x6;#51A8wZA0MFnpSKp!^^22vh@oTjZWU1n-(N}uSDA?>-g
z)bk@Nms$Dz9)6@gWO1!pP=u;B0dRs<6?2+tmU4D=>$|HtZGV)y32-I_3-N1_6aGN=
zy^4gMG0?X?wkpm$mx-R87YKa@pyP;&h2@)?n#wQG57x{DQ4k|urfswIp|9?fO9_3G
zaJIc`u37G9(e`Ak%BSrHRVdql&9*Hg&#mg$`*b(VbMw}vCe%Qg!UO-4`}#`Z&<SKg
zP4L=j!~8-k;K@kJj~{b_tk@MWFoDj$=t)8VT_0}{;|R0)buL39hp*@EZ*ICgb<9Pf
z=Y?*i*yb@MT;e3T_pW<YG!vNf+ul?NQ|2f$n<4);$@c%#w=OLzMn+V?TmtzegPw`W
zTks!ftn5v2aPY_SxbD>R!+l90Dew}nBUbIvVF>BU^%}ICnA6|GvMps&v5{CTGV+<}
z!&uKIFE!{Mq;*{nqc8t<2K~;{GE6{mYXSx|xx5?*?%hbSUmk3|5m5U0Lg@n-lScL!
zlwX}83=cO><rNj)d6Mi;ziIpTq2UPJsKDR&>O3dA@NAQ?qM{-kGm%`~I-UG#kF!%K
zA)Z=VGF?|!r#@G<XkWOj7EQj-m_(rFmkeA*+*HZ&&qSXxLmC=8ZtkXz`x!BX&_uR>
zy^N92vv+BGqM^N9(=Q7-KN(Cm>&;oI?d2f_or<3=Ey&<<kocO=(gs9*kbuGyU=Asi
zsd3dZ3G*9B=me@%O!S2KQETG1*=m~B72KUwQH-YVTk8Z_Z%DCtc<vq&s#m#mN}*v$
zFwC^Ipk8`3=v!sgz_*6~Aw{_`ENOfIQMSeLz2pG#05~}=^I;qii$uVJ(hq|KtuhFz
zs9M+|t-o|bl5I*qI~I!wsFr*T=E|=>Y>FzJ*WxfeayHN^zukI6T%j~Gla6+=&FVc7
zE{e>Av^B|TK36dQkYpdeRS|J`uGE|OVEdw>CiK4#<0%gwB_8R@g9K3F5E8xty)?b~
zZ}EkNh4p}myAEvJc4~<xdQ%ylAQN7tvL8i;HS_8R?e+^z#CS3lrf*B?Jjl<y_R|Pt
zH7c>+GLgzIj1zEoerzL<ShNXwU$IY?CghHEM4hUqT8iSLg#h~m!jrngwGT2;jmxa2
zjUD4^2uk-vuZ})~t=`9{&n0x@Tva@KAqE1iLI=RK1sok6C+8DAT0tw)V7r0@C~9bR
z{bDTczRmp23ceA;-$~boHJ{G8-wjcOhrk|wN7COaVqsic3}AB8Y!D7HA}h`+IeIS{
z^43>%N3#{v+nyLc=`70iW$YD<l!<c+f;yz60MyY*oP?Bc<r}Ucqk|^`=#?Y#g-?SM
zNy^E!Y%liDy~i=><Nt|(fB@J&grII<1a=x-=*h!+%gbz+7QhwWu%(m_AMbFOH_l?R
zhI=~hA9ZBAIquWpHjNI6fC~EFtvLyF5;gcSU&ZfwWRs$u>wskI`irwLPe?k-j>nuT
z_is%$0GUHf{&SIY^@S(Gl9Q7cChCOVQ3#-eZZRJx_29e0l9C_B+;;+D1dJ$NZp4;`
zV-10fv;x+#gLOCVq?-&@DAk9;+6u-RK98Lxsi0l&A}6_K#&TNT6ndJwtl@<})KoOJ
zZq!VF`|F-z=Ar8BrxiylL9-TtOEX9RUnD~@xgH?jYz4%)`1szSEoKP7`xhgh)6waN
zpP@4<P)TXl+h=q6)j2Ngeq*_@A)MaVX@*3rRs(l-e(*d`$mbz8Me38S7{^K-qn7fw
zHx$3Xg+HUjhUhd*u_T9DDmhm2>Fbes6l*Mt9tboOT_(C+B0yoc79`3pd?_YX(ts(a
z{gMi_MZWJWD>D^roCb@d`J_KE8+rcJ07sUD?KL5<)3*C~NU2hm={VE)r5i!VT}DMl
zFBbhhPTiQgiQaE0#VDoWL<D6_Dtt8nfg$=2(!r)F%Q9Fk#u{N=*zcfv=fY6{`yqr9
zuSR*}@bmLXnuHlNEP+ntRJ|8sVOd$LaiW_ql;9G@Eu~LJ*Sv;_<2lBK20?{@Mrw<P
zb18hTk3N_HEhoMHu@|zk@bu@1=oe?NDV@t_4SH(WWhLObrCUQnkCb71C|sX3?aZ`p
zXp^bli@$ag?LrGZVF84C!oo$`ZS@xpQW3&p!-oR>gMuQ6b9KrsZ-MTHQPz?MHjn{>
z*Faxm-gS20qy$})prv1m3hAFo&<S+lvLkdEmY+YDgi1(7*p;!n>eqz9TBsnr1Ol5*
zG>5!y3_V~%(I>iuJrol`7QM9mvMl=CV{S50Wf&a-U~>*Rr^QZ=y>>gSM#m>A>WQGt
zqzOTi=O<KA!2pTLpC`7w{YRybOiUh`SrTv{BX9;s7i8v?B)lzW@@XRZ>yCg$E9L?$
zgn)$|<D!yCM;DclFa!hF0OPrShm{q|nUPRXIuk$qyxl5?z>VQw?m9&I1?h=|3#R#U
z!E^MocvvfbXxP!Gv$)sx-WjcdUfh>42l^HgRiHkpo5dG_hC|G2rY$5@AyE(&1H;?T
z?>e}p<_*~35Mz)P^cEOV5<G@$vm?x`jzBRu^D$fqCMCq~yL)IfT($;_y>{xLDwKWa
z&EJ^LC<8v*XoooUyj=7$AX2Y!<ZKP042g&!;IbMg2lqW;fZKbXbO9%R;p{vQFkTdB
z!TXldJtIg;T-<PSm$`Y)#EUQBj*C2B17s%cjScMQHkhR5m6k?V14fC}%gjH@t^*p{
z)N(<cF9_Y5Qp1*)pE{1r)rL(ySAN6u1L2C_yVt67OU?z}$fOEXx7l#fm`EdDI&kXX
z<bJ$*@^B&$zSFAx8qPsWPiiD#5s_XnTz3qBeQP_rW{_$1f;$XERJtEro>Vrz<VfX>
z81HI*ePiq++_i?!B_T_gU8N0y?m};6lJ|N~k%b_YEgMw({Zpk3IHdEu$x9d*jglnq
zWk&g%y3qJL1qip;*cAM55blbMz7p*py=-{8wV-`z;Ihi_n9od3C*LInJWz<(EUig)
z1n+2#$=+2#`QYJIQeu}YWQ%sh?ju=jo8DCGgLqiJ;pc%|a7^5I?<&r7{4f65E(e$G
z;Bwl0_M%i1+P1D<>&*S@*FK!?KKtrr^|~NO#Z4&n^$f#eUCznJ=VWw7Uw<vBd|D%P
z14%f=pvd&h$!9#|gLjs0CFWsS_6-2k&LFqM{Qbh4Ai<Z-U?D+0H&qhp$&PRa5*~9B
z08au%!*B(pT~tiW2q7|Nl)+0ZPq7;AtV|Q;9_?3><x;qrgK~6V{HvKv)>sSq&&1oo
zp~DWRJY@+LzR%>CS1j^+Q>AW{>=e}3S$!0_UVP(Hx-=j}C?WCdTtu!;{0jGz2Rbf)
zxV65x-ptR>SFg0C2YWv<YJ52iE;Z#QYMPL<e{;*r_Y`|1gX}!7wbvS+_b~H3Vpw07
z!W+C*<?mrR7Ve<BZQBIEb`VLkAHET~dQHWTz!fkrTk}ML<MoU|26RYlFd=gd<Z0jv
zEG!811s7LkfAq1q5Gy_rug62Yam#OgLh`rCje1370xP`JD2dybv3P?HP$Lt@#n>b0
zqa*4s01sU3M7l<0()geGmh{`EN{>&?mA&qgTxAIXAai?b!0;UT{N44?VW1jq1sHHp
zNeS2X>KGXSJG{fgorESiksVs$2qGBG4C(dXT1f}hB7O5?c9h@+u<EnQH#a2MX5L7T
zJI;R)cV-<Wz~~>Zxm`f5ukme&pvh3y8Ao#PMbV#G?iB%I%(DQU?23Y&`c|4tBY4gL
z5|t5qCbEm{{3CK$SlIgRsFfigfo{gN+<V?t4-lApiH+ZZcy6LWxIy`9%Vam!jv@#|
z_e^ibrtm;DU38UcBLCT`Ujes-D&_ch)9UM8rM3Fl6ip-JDD?4_{*{%xoTFkFb&+EN
zkoouiDb8~_D3-uQMUkAh7ra0NxM5^OLtYwc{r6fJtb=@Y6_&hbxN+bsQdm~f-1P{-
zb@;<W!VpH$$-Vyjg>sq%6)ELpw`<3hsg>@@UWpZYwOJ_LS>@puKL1K|B3_$mpBl#<
zP;*Vq&-;P$u5($Ou|f(n9!AhEqIX(0Q-Qa8jeRI5NT=KEw+#>W<N9r6W#!U7#<#Wp
z!3wRTx0X}&2`T%rV{re~spyK6e8A81U-$DnmnQrp4Ak}YpvVf1iwpAfyk<UJh_Jf4
z+G&oNnsE(zlir$dMUcq+A%et1GFP68_~U!J#G7>J(G~SmUz61<*ZBFh$#Hp;ScfDy
zH%<*4FAU=R`M-nOWcomm|H3HH@j!7E&1*yR`&&GVga8=N?n)G*92^`(H$@f!ejA-R
zludhmg#9}8Z+c={+GbQV5Uz;5O1AB~v>b47Ic*mDKlA7~Cx=yX1^+qsX&2|Bm<-v%
ziX9K+R6Y67n_F6v!KGkO&@wkSUkB`ljzEoA!>1#+(G{t3>ff48?VIdrL@-Zz-b6&y
zGc@|RgXEs-*@(>MBdXuK8*cr%{DZ_bzTvyQ`Ri;(`v(L)lKqh<whMR<?&qRAEPi*q
zz}>-NG3tV#Gq*?m6p9w`9T+t=^}^3m7B-z4R0RbE^YNOStWRw>6b2?pkQCp*vw3dx
ztF4LHY0>zWdf>|CL_HR(9+BKZ>pIyX$@h~;H9TQn7|HcLyf$p0itaBq)-t^cT=>FZ
z1?kVa$?GyQ7^<3uy_dp$i3_5JmUGy-7?P9`iX83tR>yE}-h2!44t<b|WU0LE*T~v_
zrB|%VSa_89bl7_KZf9k+p6P^IVd11VGvPQ&_6u7{l{;UC9U+(qAZpj>;|KQ29Dye+
zc{HT2eT1yK%}0jDFE7~&Y8{&;@ro|$(_BiDNo=9Du-W7b6haZ63^a+cWQOFT<#la&
zsGzMUS>%00gb&uO2kn7Rs;jEPd_l|dEtFtBe)31mH4MeDnnyeyU5D9(FZPp~S9<lg
zPVDEgLw7=w|ETOL3UU*^o%sGpb5?tJhw@vRPp3N?o8MSl3?i|$5YHoNe#AV}C-r7>
zX7#<_#EtOG*)HpX1L!&Gut#8!HDwb-bDI%m$i@2FnG}5ogSq-sjc>GSG(R}N$nu_h
zb+_dk%zuo{(iyz9w)Y|f<Sp9tuwEOq;aAfSbS#wRViYEn=<`m?KY}3p{_{&RJqtv?
z`x9@{*(F3za|}-tx%vp)>JqEoX&YNTij71emU!rtyeF>e)>XG*eo+o9)(1flZ0fgX
zk0BGwdPGG}j|^^ZAOgEmn7XOW--7=1H0Xss7U>@m2U;0hpI`iF^Vjs6gOxA34*9JQ
z!G5|-dXX@u(Ls@y(;Rom<Mo7WVRr<0?{?mKc*qxAMTYM;(D&jk87}9h#$Q<FM^u^%
z7C!c6`}6;*bVraW_#HimbX;!uVN?1R7vBSyQLwN-k3Zpe2O;7IZRmN0TIx*N6{!sW
z1M7?3kjjY?cG;o1BjZ%>{sT#!_k&_}z|zk|w$lBDIYsjauf!6{zKd&aAJ#sJO_Kho
zoL#KveHosl@j-TMBGBpx{hXyT2__sJ!EJgt1Oz^yR+%ZEaBsq3&oVzX|FA0nx|`<?
zu$?AJ>d|!H8_8;X*jL*x){bW6dWNxc1nx?Ha4iFL9*WAKQKb+7=nNK?eO8fZ8j-&6
zo1U)$Fs5id&uIt!Y>LZkJBwQ7u=Z|Y;RRI815%pE$Gb*9`drJxuQDNX*y09gW$|M#
za$@VLreZ<8EEBdibK{L1O(b4Tc~*h$SC<$bi0b-Swe37m1;k+pP4DUhWJ*$6x)I#M
z!mQ9gyU@<}y1U}*$YrYw{spNoWsTNk<G!SVzn}d~Wa08SgYM7<-NnP6Tg?pATH$vY
zynx%BP$4e}W|$EA(hgF<F0<e_7{ra8oy;LQju&}8(4bP(v^OZU8<}FB;)tVB)eRYP
z*GswL|D1Pk&&3x&03`l?tThRecyzg^BpkZt2n^;^+_`gS9U#$q)6H*xkG&{+cO?X2
zv^1RuezT=QQh`q&Hcizs6<SY9b_ze*r1|@(de=ZMh3VPFd>$sK&R_<n84cdo4FS4T
zSX$ZwjuND%XUD|#f73n1c=*ytcwz43^{>hl*=7&Pv^bzHUhW-M^jEmwK`^-pI99+|
z7lOl(&ovdUglu47Fl$fKOfzU{X*vAa2;*m&#nT@KP6Aie7UcDwc_EL=*LQbN)N20l
zDXwKzJ^v`^SPFsrM?o>RKlhEH#c*qDjt35_L112}VQ7fh<9Oq`DcA|N)$!}HMVtgD
z@&c*cw~2Scso8=WyIUB@jOzvUW6`{>Wv4h?Ke>3NI{b&y(?qvO=gtJY60g39@f8q9
zKzuYIqzp-L9T>oAM8M3sH9$vd08qXUQue8trt8d3l^eXsKWL~ZN`8*0=cHGfGa#ei
z3B4`%Xsct%&2gO<0bkb&0eDelHg94@I4ex4XhZ$wBrM-YAT(C1yX_X@=wYgzlyNCe
zVf>}PPlZ5o6?ZlH^TP&=W;o>T$;bkMlZB}3j7lj;`p|Q>!$R=GINiG4x$<<d+%yNn
zh^rWGfY-~dAH*M8Z(AhGQqCX{TpBqZ!LI-id^?Tx1&1Q2qWPKnQ;ihE+RlYvh0Nw@
z-$5tg!Y~;Hq1XN%Nq=6-!6{(i=GdGXy;Ak|DiCGWiwxe>gxXU?ZE%26mj+>jXxS@}
z+OfBQOWeUSD<!h{69Hp1Jd7*s#Zsr+W|QhhHI2J*1vvQc#FCPOdH^Hhb>$;iT=a~J
z#YZXIkO;QbB0o8>y-XPh7;C`czA2yia4rZ%Brm{lG5C?C1TqPdd(Sbyoa{bT*W3&)
zp$P>DS#&&nHq{X+e+*j^`)$UaRnsC@SH9l}l^e!Nog<kUww9>Led7a%*1kUB3DrV=
z^yqYWPpzMN=^8&x8u#ty!8B)$ZA&E#dh(U7&$Z?R7ur`J4V0DSaiTr+`ih_G0S5LC
z-00b!m&j>|0xjM8AYC{@kW{~g<i5XU6^B+8s8;I6nPo9HAKz&R@)0_!!7v;t{`XyE
zRJC(s8S=<gKz%?`Azdfraf0vjC({xho-I~PI=9{`!~T2AG~o*Ihr0vE)?PsH{Tvkb
zqQOX$F`R<#uMO(O0pvpuPTSdbny9ZZa86tJ6@&*kx)%Jfr)Dfr4=MkhtIWW$^G8-@
zEU)77RsD^3{hP64#VFV76EGT$IWmRbiD}vq<T9&v54T|xw6jqW`AyhuqhHihM=O2@
zAiun*8fl=aQ53Fop?L@RJukRO9@sI0;DPb*;lo^=T2jFG=$vTb2X&@me0?0NgbghX
zN`WLs&?_rtZTmmQ-b{)FUIq6l#d^j_?B2V3=*+E!$bJMR4~X&zc=-BfyS(a$5$x!k
z1y`M3AZNxj=-7Q`j4o{2!wcL}y}dGm4>)!{fq@v!27>_jJ7D)T25veV4=Qb-*4DS(
zvVB?7Tl!XJdD*+gMFfBg!fGOTT)-o6OP}kd;<bs*Q;(k6;G9tBUAjBc74`Z*jtZ?=
zR8>rMu9|JC#ud_6y8s|O1P)Xm9ksRpJgz0_K`{72UpmG+az2=ql@%aeB7()CEb7$!
zvEc240b)@)+GT%z?u}TB-OWEsYRYn>BZ}w?kKyGgGG8^nkdW7Nt3P|1M<gZWfo;)a
zukP&J<UR``EALWxywPRYhRkl38dL!kaQ=%3@rD@O42tmJ)rvpZGE)y+TD8NP_&|1C
zdF6TvK#YE~ThH1us;xZ4Rv?x@!g_%Ls8Cy0Tu&vv3~>Czh{~|ET&@|MWSE?2rpkW8
zYf%i`43Jfj7w!u?kgO&~$O-tMFJz9X*R7=9xhy6Aw*{mOMOD-9oaea4Z8jWST(8|f
zqcZ?QvBeXnqZL(tO$O}4>lYFL$Yixku|uZA*9X7u^k!u%SZ3M^0dyGK%Yq@lK`Q1j
zS1ul1cGYMTFYqY?TSDOWutec*0e1Bl5d@4tS>ttD<W2U_+}`Qa8Akw)bexItUk2(V
z10a2sUkBz;AK7af5A=f~!*K!U=yhG4(COhayHi!hbau!b(1G97ID}*&l&_9&u*hJ@
zF7>0k^1q-s=~Pp!CqV9_p-p>_hY^|(ff}p!M{o3#ACqrKGyzrNM!V4pAN$GJ-e0@~
z-9Lu<vQ7$u3e1fajq_ewF#!%O%>%my&-Ha{u=a!mTq4IH9eKwj(!wYeoYx74|Dw?}
z-s8#X7nx^`8Z^-3%EEp=9eQ7K>;l((+E_NaTxjz1LBEoF8X$o2`1>?h>Nz=c?crf9
zOtex0R@tTl=l9FTUX~&#DhOXaIPb<KRUmy!5C{Yf(3=K-{K3wJX^=a47Z<;@QEjum
zOtwHMdXJ=h#`k9S2Fn~%?0LUPQ|nrdWD;{N@2vYBHK}FFH_}2aYIU|Zw=Go3K>wVX
zP`~){uPY()LiLT1i<pLh1XTMqn|2iy50BbuB^Uu29wl4#XoMjmZUsfo^C$WXZg<Fe
z?ulF8RoLtI6RbpAA8(gyEpc~X_PL&FJ1ocLoX$et>VEudhi*8tbK$Xegzrj$Nq;89
zIl(b2r!!I3w=xW-kTXll^FTIv8<rW4Lwc8DFufYX_Eah{PTS69F*UsXfWgi7Rqvnl
z5P6#U#Dk5F1L>`iuRcBp+w@PN!npngG~go!G$AxJ)Kk#yP!Cl7ps&yQg8$$k<nTv>
zlOLCiv((*!;5mNXizlwrt?b|*Q=mMYy<IQ#<Yl;1l`f^=7hGx*{(Q_k<mQP_Z+a*s
zenmXICdKrrf`If*HFjdVx<Q2KSitzEiqH1cNqDTM;3+x2QE}WapTyNnY&Tncj@HEQ
zgC7uIVzjO7-J{mu)$T%pl!sBr3yk$7gG>(Mfvit8ZGhXx8~ge`JF;{qZ~sWDC*QRt
zzKAVQ9&rsBlO5-Pk%BnAS=8XOI0dy*$JZNxE+gpyh7mLy*KvzLfdKv?2R*O`*-!UL
zK(+#01f#efxT<quq?iF99~|IXcvmnXa8so3#uzHd*;v0FLjUOeP_H2;FeJ&zw3V)3
zM}6mtWbE>^B%d|I%6XiS|02|wf?`tmU(P#@)ckq?Y#4d~h}`R7>Sh|;-N~(aSC}gR
z4XxIm72{G*q0jC~t)KCEmL@Y-`QYbbkqXfTP$CT=P2rB(w&8OXtDl5{d(IyLf@5;;
z3l9B8Brp$hA0>p^<&CW5GvX<StB3s0^U5j#6L`Er*m?o(fQ#C}rD+hl1^ANt3#lRN
zVD(b0Xq>EMwCH^f)bkHe^oNRs<}S`}PIsnrLw|GpLsgO(f#3)Rv@s*kr5!|H`09@)
zCIvr<W*!mfkwV>q=eddrpQ6y-8cSAq!)~)sW#ml~{JcFXfGLX%b1+Pp?8>XQc6bHv
zC!JI2G2nTB85=>O9SOoz#sFmysBpnS$|~vsAQga((Q!f1f^H9h)yOeRgHy|XNP>x+
zz=A`5fP>qI*!$p%j<x-?>(B2k(~r;zygkKZeck50--l(3g_{~pEiS2GmKI9>lZ#gz
z((^pms-YhVIkN<2HY>%FQDxJN(MIQ<%JTpfTGc2oo|c}j54zUTcZ~u%xZi;u=Q19O
zs24f1a{BdTmjfy;39{8RPio}Xw3JiIF9@W0hm-B&(aB4nNxBY-dGRCVzmaGf>PN;Z
z<8g}XXgUwONuoe-APP*oeT{~Wl0;SybeT4}<Y;bgfWD*ws3=f!9@Y^qjx$F;;H`VI
z&JRpe9rk36A8Pjl%AWu8IObOMv3h4FPp-w<5=3H4T0sqOVrJ*mnoJfbf5*o`4RH?w
z2}(m}AKt*%s?;zCDRt`v5&VUlzL5IDJwWaST0M2uZ|R}%$Z^kWXL;x=hI$}3ZL>43
z!5He}U!crx(b1PBa8#p`3TKwXfBDoIC&o9D+bw-DW+hSJtqb@bZ?s>*Z<{6_Ie^q4
z_}ZCtAEKh*4i||aOA4F~)<Me3<~J_k*kyQtNo+#!7@`}_c-(u5J98fUIzcV_8o)eb
zr9n_R=81sBWAW1mbepKTxe38_%S-@fc<!l*@dO!UEH7tN2{=mX4<;pj1pVJAe8K}L
zbx?ASu^VvGSTH|!ziA~`Jky9BIANkPUtEJW_&iP|3V!+eH>2-28E<n6HxBhf#$Vss
z*+()}D1WX5J*hy;K*6-}%0LM^m@W}L(4q#rChl{Q-T-?U13um}j<IV3f>>PDw<ZWh
z2BFA$UD=;s!R_f;fOW}C&2SAjBY<f+ZQk-ab7KzniF|uy&(bjXvy-6F0P{RY%}}ex
z#r38#$rjld0Z1)?6y}-*z0PeCUU_i{zpA6&Lpd;LZ-QMPseoS#(RCr%7u+bALSou$
z^&ULrP<Xmh5-*bcm_NwH8M0>Qe18(w1f-%zf*uxuGJNkyL+S7&w`t<LFf+L3gCMaU
zpXrkaH632BGEB)lflYqBV4-E!edB%cGw9u0nCpFz@bOxly^E}&pQZ8k!5lQ8Ze!Ci
zFw8Vz3pKBd6cgeK;^PG@+(!9ifbhjDZ~B{YKOyxx9TW0(HsAK5vb@I#^1EdL{;`jg
z;TACcJwuzhZ-{AWrgLHiE!%#8t!8Sd=K|?BqYiL8z*TtQ7b8G&-lkp$WaKHZ%S23B
z84qeEfmJKZsp{Sl4L&}Pq5-WQoz;3)ngp<$MD0!99cIU+e(8Bv6AEi#qWYbU{hs1g
zB-3ILimGcsF>@q9IS3xX>1dl%hTa!#ym*?K0>}ZG8Z0mdb|~-FBRID;j?UmVY3Ny^
z!|Fk$-TUb=1KsW023^sdfGGE#=}naJ9Y9u=YQ|^O<MMLBPMqBCMs*rD;}oj>kX~|^
z;XW|HS)?#-&EuELY}rz8o5H^B-X^9*1TOnCx_>j$kZXr&qZRaS5p;Ume2P+2OWbN0
zOqB2QhwwB=F5-&xBmmAG7t^C~h>87hX~p$6rdtXN6$cSTfN1Q$a;kqogBTGvOgQB|
zblLZvE3<QVu249j9pJM;_G1jPN5?@Q$o7C0Bg2T<bwt739S&$#gTYP6OM-T}%CoPJ
zy<vnm?|Wx>k|H%5#2LstznOb5(8Fy````G;sD}iE6Xf}g`68fH!0ucAf&&w<0fjff
z+;4$}<vyjQr8eHo!hb;j5+Nk&GKo9?`pbvX;{YFmDc&7${!1{L0N7<cpN(XEe0@{g
zj&_H?xIJ81S&^kK5pDy?ck-g+C6agbPUq1S1;8wgp2N8T=Pw$>f&&m&(6@XP6#Ak!
z1=^O1#sKsZCEf4m$Iu=`n!s~uT*P6HhhCsL_dZ)rq<fP>3w3nGIqC*Dmk6NYKFTid
zY%Z#6FQjHoT(CNz1%N<`b?$Cp>k@H({_mppJNS6WRfLXr{|Uwu9SI=)!!CmA@{)jg
zk*wyV5V2*g^yurzIh`W|i~{(W>|v)~aEb&_0W99pe=nZj2@gyPH3RJe46bCqFu7Ea
zFvzA-l3X$@p!fAOL8GHHvz~fIuqyN}3h^cT6e{SzSeKP^=fQ6RI01CvAwpuxx95+c
zBgs;t;^Vs+|InfJi`&s8=u?0JtcJF>wp>QwlX6y{8UE)Sw(sEJ-tv>t-+K@6#`JXL
z><G3u%!289Ci=<qG3w;!uQy8%PC!SP|Dxs$AO!q$k+t#KB5>0)ADYeny1q)bid4~`
z4p!^ibhhDB?*Bq10y;lokEEHD)`u{>WNd@?30S-%Bb}-AfN~~vkxZcAMVD@xh<SN=
zp=9@#WEV+B@${p=?LE(wJOz=#0))5pGLiM&-EtH+mB3{JZbVjhjX0_G?`k)|&tD@c
z8mZ)`9k~F0fTn8g<mzul+0t39m6Ks4$Er;)qi+uY$~1?@?;7T_RZfc=6@BNrE}#pR
zx(ZY-lx|jq;eumE2TBAY3i5avCe*P26#)bzbj5IEU}0hHgO+j+816sT&MGXnvm%te
zm0MVU^AhC-6i2rS?T~H{9k2L2?)){%>y!m}$Q4!VnH1Hez6%wi5bD#iL1f<^FEPA5
zKSz#8em|EfH}B3%{?_;<BK6O|xlX^3ctze|(X;+e{3BSYeK2t7u@xBqS;p<{kxtDP
zNGqj}8DxCY-sgDp;wYY+`@~I`0w}rz-UA*)(}xxTnwgoIk9<Z3sjjNZF)~^tpwzwC
z&_X8&sV~ZLP^?P3$Lw6`^;n5Ggi~gqpnh=+x5AMX?|ttE<JwGs_^Du)W`}m&hlR=#
zQv?hg@?A+b`?k%77ZZI7-??YCw0g$;Ds1NkwQl|C$$s@FPJy{`T-)z;kI`^8xr~1;
z$IW+GJbV=QcE5ohbR0i%LIDTq04oMdPTc&WLhlQkby3vU!Ui}jwWX`7ooM#iE~%Ds
zR>MEl=W>o~<fT7kEaL9f3};L2Iej7*ANFtV89P!?Z0sN^&2gGjP3sK_i8&@aEkDJC
z#i%yse-H_x2nj>P3DnU=4;JR8bf|bR!cxf;1OX(TGyvoWxi0Q~W^V|JRmAZEOgsI0
zPlQc+&;xA(jSvjm%IR-L!ooJ%t)AVoli$4C_8Imm$p4u44A#*Qw{aDpa+m-9RddT*
z>9Kp07k+s$mmIj*zTE8Ao#PS=Etz2>zKjC)l-0nY)Y<DR?B&^7lLq&XZO{@u9B_8b
zq8h^3tL6_qtLu^HZ63vJlkR-MD1Xl=P?9_Ugw!bgllFF{n3PoVhuH5%1P6O>Q2omS
zx+C!M(B6_(9C{?+?rjNboi2}&<Lh8V-fD}Zm#|ilMtU}J+!@vUBwR;4B+BfWkjmuR
zo*~|3zhjx}*x}mYWa&HZM5dXh?+ykH3T;ZI225Jx0=pk)GRnL`#=eZ(deKoo0PL98
zXm5|lS@<E&4v^EUKR@Y1d;xd_jim`5uZxH^Ye!Ih3}p<UC`gC*A~lmW9<d?Oa?%s-
zuta$&Fg3|iKQ75~D%C`8Ton|L6+H9aXTrk)M=6D^J7yyuvnzIgGf7ND*T45V*6yIQ
z@siLhM4r`(J3@Nn>l0s`+3R>QFfUWfhCvhr`@s7f6>e+xH9Q7+zrfN?I_3`!B2q(f
zLpz0`zQMPCuHQD#J&Sji4pJvM?JZER{v#}2uDbOyDqL>*l;35MO|-E|v+DP30qvx2
z)5DpXMpr%I6b{pxi?p9nk^ekHQsQ?Yk~t6NNM4|@NCBqd9}*IUk4po%l)-3d+}n3Y
zUu45>KCYB}%wLi{XO^vhmkl7c1tF)%PtU5D6}*d$q&%Kcq_5BI`K@>Ss@-(DC-lNE
zM54T}IwlMs4>r@2j?;XbF7-K|f(qMX<u$=7*oWv)^$j_NZI1!nAs>E^=U*m#C6_6?
z_9y8hc24gnvxv*e9zAD|sZqzW+RaNec5;gUuWKdXh&xX$NmeZl<)F|Vpak`5UX0i9
zrKG0LF~u~i$V~_HbDPUbW+g|Bh(9LiNz2eL;Cdh&p3Pd;VkuWN79{l%MqD74>^{_j
zb9S^f&~|J6^l5jZiDV+@a*@$Ip_BVbDD7I)`09ibUS?V;EbMW~3#vSBYH<av8_u30
z3?j2fXK!NCmFm9rpNwGCy0efPNYfO#Rq0XgQq~2l1vBfl9{ss1PngObGr*oWbI3xo
zDiIQX_uGJ5mc{`7Dqk*Q(!+#<Ovh0jS#N?9Mx`$q$t4L5fu+*Mg7jaFZQNa7qp(ZP
z`|ltbq%FToKx=yY(qWyB8+qsWYm&#&gFdn|z5*+p11gHPv@Kd1%CAF_?};YzY2*-p
zRN1tUk4C54E(TUI&^2q`DAhfcuS!(ztrfXpe*9%$;_JdaYU;rhoV{@MwG=T#U9QNm
z+aD!7o~<ct2=L#<YWwaxmB0P?*LNeC>ZGc~>oNBk6Z;pxuZ%j}A(_wr6~0%eo`V@P
z$l|vrIM%>Tm+*cX>pYApzCJ$@#wkIgAm~Iu)d*OjU<Pdp41aCf$r<396-Ae7^4zBT
z(bsRD`3X%6t2*TA9z|n+3z}SZ6nE)52{!U))Z@ZKuJQQO>bTrGw`VZdG{4!Aa$N^)
zu{8(x_YP+#ql`{U`eS@wy62}Yf8aZ9{3;-#Y)zmoxAU7#EKNgdpzKpo_&8xpeBIZ*
z@TOVW=|fh9$iTa9l`oG)99?*ger|0W`aKf*DiySHee+%bEfXJMt$3cdI@g1v!B298
zi;o<IhDQWrRkDWWTud6)7|l;?u3;r&y&vS|WoEFoGxC2EX)JH<w>RO)^QnJN`~8o~
zm^s%mdm6OQ=DTK_DjHR8d?#FcK0Xy58Q%c^l<bw^nd826U^ekyMu<;yxDUZLc_kJN
zU#Jty)HA=gsZR~2vm!q_7`wVVtp^zA_ue9S`)2N?j>sH(x?T0><DjUFyd{3S^tj1R
zSNQRl%F5xHg$aGF9}n&YAqWXcNqZI7<{_9pQ-#FcC2=0_y5=pP%YbmNicUvImjv=L
zUi&2iQf^Zpu(YbFw^vD(2v|gR^@ut?E{>VFe<^LVF8$Mi>}$kr(;7&_qA_1Ny5N&1
z8Xg8yoA`Z_zIS=4yE_v5Bcme%!q{CA^0?~GmXHfl6{3FL%v{97!`kJMl<jNp2#AU~
znhGnCEM4{T=w?pfJ)3`cbI2c?2A4)0!{1`QA=Yu717YrKMSCjpY<ypxd3suX7`x7Z
zTleNxKl0Js1G|S}Hw`Sjj_K%p-71iC+y&Sf^sA6m-Xw1K?8R_>7KxxP)305gQ!ikn
zp3-cjD6)~VttpVU+6=VE9+d9*(54VJMZ<dRrtKWPnqV>J6H<4Gv32Sot5dIW=L@TN
zIC-e}8>Baa5zD!19k(*ef+L7`BOPYf*<UJ1axE5#YL?ID^WW;93by)9m2{>pCMPNB
z@ndGEce&7FkK2UPUdH5^guVZJb_1?pNpydUfn%M`#`Td@N<Jqt_i;NcZf2KuvuFOe
zk7z#OlH#%?YARGIyfb%ZX;UJyI-P%#yMKoKeU!TZiL+{cx>RyS&=rG;G|$xyUMy{!
z!r@YTtmArK+X>R4yN}Z=<>fxiO%t5E0JxCx3-ig<3;z#YZyi-t)OC$3f|N*u(s4-%
z>5`Bx>F(~9?i6Vdkd*Ee=|&`^1>_>#-O~Nr7oX?--rqOI_XiFfUhg?)pS{;!d#<_W
zL>=6;LEZAB$Zywj_?Hw@p#UBlaMyY=Q*R%dnMn%hhfkTAnW;GQC0W?W6(It8vH`-#
zH<XNaUJ_cXb?MEhdM5)u+eyN>!x6L!&){@Flz+lT^mrGyGVadV^<!phWyWax59iCc
zO)M;<-W>U6gVl}?FMFv6Z?GeL1<3sts@KuzrWD9mG}h6tG(>W=I^aAW@3&xKMW|<X
z{@L>TIZ>HDiF(^K(TB8lty=h+>2Ry+-e-bX#IrToQP%qx6{KrpG>E|P`-wVhfmXlw
zUid+t3pTHrF4x-@F=kHl-ZFc+5iNJX+Oul+mW^z27n6=%txNF|&M}!|CQI+CF*Le*
zEn<^`jb$V%oa8!@QGAANITYSK)~cwQs<-oZze+PYq7B=Jwt!`((Ma-h@n`Ch`KT#D
zqPQ5{DK@;dysczb`zpu2zfFu&_~G}9)n6`LryMU;sy6FiFvfXk-(OQbBWhQir}D+`
zKFG7fMh??c3KWHGB**?LezCuCdT24*KuFo1uxMc(1VFGuCok9Fq$LWYhrK5OhPeN|
ziiD-eFihe#5Eo$3Yk2{fNZg($v?Z!uXMRi*aESysMV|}4eZhI%)M0F?I3toIvRYs9
z<mT1)?`r(YT`!3<ALD3cS<N6cFb>V#n`K^U+<4UM3>+O?d}$E~*2vOeP$`>nMh;^U
zAuztH8E%^UD_<oZY=3OO8i%|5_^v*n(uAkt+URGnX!G5G<E9Fd7W+x}oy!kra+8}f
z%39^f&p+wj=E_L6K&z;sev>pS1V?Yn!W6I9#QwGw-&+QqKjjQ5tts-+OVh|}R4Vn5
zOoV1!E(`7*Eq%mm)XV7#4wn4-yJ$DXIE|HtxC=+r?k;m+!-uEpiGcg{lx3w8L*7R|
zHX6uZdiug((frTH4jc2v-t6BH1n!G`K2vl})2q%5Cl#`5*B5blx|AfYx!dX6!As<-
zg%+NVp^qJ`I&8VI9a9&`V~-al9y#@ti(nftMh*UK35&xQEEgOI7B$0p;){nmtKUEz
zCUGP`*+8fK4c<!qFt`=%Ue2Jy&H%*0|BF+BdjUlISS-gNe`aQkM>6>T212cBG4{-a
zg@;;4>Ik(KyJ{U~{fN-p$EXxby@$~kpAlp4<bndZ9EW+Ia!VxXJkEP+x53jB@ya`Y
zKc)qd(9ena9OO?;(+g!J2QzM1=LzQ-jNA7`R`z99x2&6_?KA8($Moy@2IK+g@Mg!4
zikak%o3unEA(UN)XgNaThftZR!c;w^VxKr^t$_*b!|=(cOEQ|#VXHDz)bCe)PzAgY
z>`zP3XLRuD@o4xX-+j81N?6F-=QcqQ?leB?Yv4@gtSHwJm%CRo5~e70-8{3jlbiGr
z6;&akm+X60WlCxp?>P2efI;)U)_(m)It8hxSQ9$FnRKM7yyVB?-f#A)D&^^~`!iyH
z&TjBTnP99vdr^-`c-Igtbx_m)&A{u7x67@n;fiuA14BEWM6iE6aOgS@d!Be&biVLf
zz5EQ-amDqSxRg}epzSCzDaR!jlU4bT*JJW5b3J=MB^vPVgfhn7L84$#p_n}vrC68M
z140Gcoz_lpDgTv9rC_#&FMzVyjq6_^1oNwUqut)21otB{a_!lh8C#106U*WKpd{f~
z>Y=MCIZzli{&E_nD$@JoiZ)jN@!XH0?6DJ3nS9G|ry1uBQ+<8%%HXb2`||J6w_LwE
z77%;uTfP_{eUO$6mPWsQRiDD#`_<LZzGeP$iVNzZ%&e)QvAA?8<=}L2{`mA53oB*e
z5I+n*&cClE%!OO+-t)O&#WkV2oj~m5%ujC}Z_eJ};1`ONczeT^*vf2MoK^b)lF?!B
zAYM+|NLp^bXPNIm6MfICbTePL9X#Kq)Mo`*R}ntT71j0G(gt7EV&Te&?9|CdK-%Wn
zy4B@L#uqCAHsU01hnq~kznd>vaDP`$VL3S-(Or0~Xx`K;m3hm3t9hz;#^@z1JS$=6
zA}FDy#Vc^VNp)jxGO(Kj2(0Fkk#&?(+v76XiVfF?VMIY`>{caqV)hlxrZpsn-Cz}9
zf#aQmFf+@)0!%CdD1e5_ec>esTrN~RhK>eD%>^r!SJ>ECR(l?GGcVw#YuG<prb+Yi
zy~y>1N`ajN95)!d($0DKUk~YIsRvcA)?Zc2ze-RS7EZD$THlD4X`Ua_G3ha%;{hVX
z0ciJoX;tlHq<^bFbj6KlTg=#}=#;%!BNz8FSx}RS&}}DY^kl3n9c#Hy%txqn(N@I5
z;)iT3P?=Y$CgHKzkug&2ukAmxrt_eKqGF)04zZAv7o1~Bh27XRdF^_Pd#$O-tp>SU
zTIDs&J~!WWOT;6tRB1EmInRMWx;l3c<nH-%%Qk#0#`7p#D@jrv!`0HF&w@U$92P5h
zWcs1mw~*bgZ0$psqZzRnj0Q4*pkvtEs3g4p-nO(+1j?s3&1MlT%2)W<SiH#NmoHC2
zMJw;2qUGyY@+pb1Bndl0?-+DuZc&MY8eVcZta37j9T@~-JuJ(0*czh(BK5EeJIrW?
z#ch9fBcrk)2;^DC^-~XxhzR(}7i;<#W?PDv<d@xFjGY*SWT})3Mr}MivxfvvMCy}D
z%+r(9g>!ynWmer^*gD&3<Ia43{<DGU7*T`@!YW@C@AKAL_YG>^YDef;PN(Y_!Rr_%
z*?v|*KSOLR)Fj{pRehIGdT}sV;oj``MAzl-Nb1;=^|Y}dvsxO69!U$_bu@zDx!xdL
zVJNehlq3<iBjFz69fJfp-rGtArt>X#=|)~52(fdT9g7VMmO039Hplbp_G53xo0ld(
z#`zw6gU#u1C1!8(OtzI<*S{(XnoJ38pa84e#olI}ugEx=PnT{drOY!Wsmxt_T6r;R
zIVVeLB@XM-?Kp(-Q0CJ)9<WWJ#|Sw&Ilmelje~=Ov)m5mVcnpsTrnsbky9DNfEw+}
z_|Ty)#0&?beo)+4`mqsT{R;8#lB?Y-nt?nzQ*qg8>P$*;o{yu?n;0vDgY8Cbs@j4z
z%qa?o6p)b>-5Lwtlg$_0I}1Oej<LLz<z5k-3%IX%LnqJXY&G+fp%*nR)?(t?kK6a`
z`St`;P`tK;37xXHq3|-D)H)`j1ZQhV4|9S~kMsgx_@~&9Q8FJi;vu@JB?lXgm4-s+
zO_$H*8T^^>g#C&`Nbhbu4%sHxKF4CvG__w)kqAnOpKXmhhX9B%{`F9%5hOpkzi(iu
zkNt^>3T(DSGJs4Wj?aJMAyeo$hLOK9FfjaK{Uksv4`ZA&th=o7_Ekr4S3BO#KiqEw
z1wNEtwETI*s+d>z39}JF@?cuBbdm1;8pzE<nDI_qX{jKf$AwjHlnPSNGD8q>fMBwV
zPa6FNS!72*u<shr4~hmmL$5XLM@YJU%#-*}hTb%XaFvotmnItUWkDlsF4VqHDsfj?
zGOSL`<_N7zWIG-Do7P4+N31fR+k@#%?->=uN}+YGzQOHYDg(%PXb~B`dtNoib4T97
zFMAwNU=d$>zu}aQkn&9`va6Vp@ehM<`0CJnt^H~PQ&95$=J4E~jcCc(S0lacU|cb}
zCu6UeXivy_>nbQ>9`)ZeAGK_|SbwXr*o?T{`C!VU3gM-fJU8?f?q>rR3p1K#VY$_y
z<3bE3xfLQ9GEQ`C)fmXLUC3NT*nIL2?qZETdh!os1~v$QQ=6Ui;sK3MMEO`h3aYUF
z0OHCv>JA6YboiE*7L}4Xuj-W8pK~819^lc%Vt*!bvbJ}`Gf$1GmaydN{)+rP%Df8z
zm}Q*&;1;8UuCQ=f2w4Fo!Pv2w<#TV;V2Pe3+^wbiHNtbn>G7B5_b<?5K1IvF8az9|
zO}qP^hF#z9Re#4Vlrb-r+V?aYvkIyG{Ke3A(uiw!wf=Eh?1k+J2SC0eZq%+D4BxSs
zJ4slYP-RKI`!S>Rth7EOG^iEr`3<YAFKtF+K_@jt;L00+{oYxYX}joT-7ojv@A!Un
z1@f_$s_Tbo7238U{=j+*wcqfTKIRw!GH$9LC26WaTluW|a)i4-BOVXamK6cBDGnz&
z`Sdrl8H8s(e$^MWF-j?iW^C~6$09G8&ELe4{35;Odq(PaicgVuy}SyIJ7nCU2NY~S
zTn_Ro9qGK*+xy(mdG}i}?_()IN0aEH{N$54ybvSZMn!)0>iF!lbLB0=+__PIflPBx
zmQ{XXZgkh?7p5O-OS6eyf47VmECTHdT>HK0e5ZUdgXU)-2iUm~(Qon);VvBc+a9~v
z!d5Iq@Jc#+{?oYhC_wX7=F^uiUKIIToRa2*oMgSyz3=|^uj}(yQU_?KWC1<?4Gj&8
zK>wsjmv3jb`Km->YYi;G6B3gf*qRI17m~fFSU<SG77o@)Nn~&Lu-w}7ic|SzONq$Y
zl_Osroq^HojEp?zQ9-q<INysf8JLqFdZXoWewO*P-#&I{xbIH+O|b3RG*<AO4+E*w
z`>Ldc$aYG8=epXePZIA;^mMe+f1lKnViMfZ=F%P+Xxhj{*E!VG5C8G=9{6ZZqBU}J
z_Q=e;Szn-6*K?pnTFeaqNM!tf<UgG=CVhf?lD{f+R{j+^Y<z?AaI9ku1AV*u$>!Bq
zAXM%-=U7p^14Kr-{z58(Vc`>rUPRs9!4AphWVzf~30oUilDKyICvokM@80AwFWH0=
zerI{XL?Y`q&$#l&*q2sXSh$P-z^`{TFG%v+Cg*5nj?UbD!OV>2fZEig-mfp23z^`o
z`_U6X6+r+-`CN+A&gZ)}w<^{zLP;`kG_9e%bam%Qt3FLd$xx*xKDku7b}Y)8o!yRZ
zlOOk-X86NEJ6B>JYv2!(?}|<GP~RECN_dr_1~te?7rQ~zf_mKHEA<`(bLP1qwX_f4
z=BZW}bg3b%KQ96vc1|1$5DWM1c~Q1re}J_K3cZ7gQuhF_#yqaq<zIcSU7fcRPKQ`H
zG~ZN~H`SvQ2=H+n9W$-`r->!I<HsM!pqBF^8ECq=l9Sa3|71MLvf9^H2DuE%99weW
z$k~H#A6|`ru)NrZF-QJ!RdmtI_GteX^~-CgYqC$6PkL-S=(d(x-HS?Q0LB&5sLY#f
zs7pG>bZKsDnOeN_eN^*o?PLv_jZDkTUpCYdran?IJQsAJo*}H`?bJigd|qfv%$Dca
z(>6#-;HFQ(f6^KemaAn14P`bJfTV?4hIfgaFv?uzS~pv1Mw1HQh8vRFe&6G}r&bK!
zD?`^4&a7}*Ezawjx!=J%6|a=5{ic(48R3rFcF?@zxF{;w^TQ)<b(4bsE8b~p=Vu=B
zs)rPo=lpCH?;u;5Jo)wq=8eaqnK)l}<8p@8)K7JiDJ8K~;j*H%hJ#iDZz}wF<@T^A
z@on=_JTlR%O@HbdGDQPr%J?a+B<jOLM#U%$*Ae#==0-&7`Bn;(1FcUI7QR95M8}MV
ze@9!;M4@1!c1FZu8BJ5g>3}oXD8&!KD_o;4`I3YzjI`uV%bsA@zXw}<nKm0pVS@%Y
z|LAw&7!L&^vDu<$p8&5g)Bgq9XoW5xD0v$Li6X#zjl9D6J7q)-`HS2={t$J^M95g^
zaNm-%jParz6)8a9-ATxPPnao?k^BgHLpvvy19rQ26dmH`XDg}SiBQfTGX11p?+|pm
zB(ce(%etXY2z>EfK_|`KfoYlmml&ONtK5kNMI=&*c6blqgF4NpLK!&{Eqxgt*yGw^
zFIX#)V=vv;M~s>ZkJTxc$f9tHXrsO!G9(^5Yot^vE}OFuPwerbuU$*aS|@lU1QgfO
z@QA&VAIBk&VxZS<f-g~VNGfC=cqPYzwYk3c0-T1)<?TOTddI~cR_=c>W>rY1`4qZL
zWNryBRbHa^_ry4*a>INLHCVj6b9@N%c^$%awaO)<IVyN`4nR)@)p&vfuR2gmd+ivo
zxM;v`IR<}zu+R%rCIkApsqS4pXfephD_7O7Fo@L(<7Y;!f=C+bOb?DFkot_NZnZUx
zSabmie>gk|zf@;jY4+EeucDyVbCJl@mlq#?mH95;p8%A!L9Hk1oPP$qrR1+dHZ}}E
z3^;lqi8=TE#_$EW(kfk=EVoH+XisqPXtV!gSWWSv|MBl+H|VEZ?mBhgc{%#+v;7^@
zvL?7%2?D6p>pX9bIZQ6x*=Ycb9B}$*;7RL&J1f~I;>-Y43%mzb0PSbOh!9xXM;qy{
zRw|anr2F;#h8!r<OXzhqiH4Pp6X6u-yl?;TPflUv1>AD>vaRJ+@HVIcS?qHA1hQQD
zyQIRnN|!t6<u!F(ZK&TRKjBqXfEG$`>7xUd%lH*=LmffA$PEsBSVw&Xpz*S@kPxWZ
z`A5}!<OQ?>K(T~8P%+k>1*;js&37WesyKsR`0z6_GGOQ^P)^A@#Cu8Y4XW%w=j%5W
zj2a=TBn*L+NRQJv<oZt|(6iq^#Z+`0>l`O$vC~klegCHZ7RJYr^DLtMlwy1PN_zb3
zY8z+od(`E(Ad<PjS&nUQaZwmyI5R)BR0<sbM@&hXRhj$J@l?5^<>8HVwj4OXx*8NB
zI{-PA#pEXj&>1uUD22sDNurSvmunVCo0}0e+8dE)y)NJj?ZtnhKc1c_M|vgGmSSP&
zGkAuhL&m*JE-i_w;g!HXB9@5C^ywORYb-kmUjzdba0MmHR&`m=<2`~wzy&NGAoH<0
z3Y;Lr{wm7j(_E;ZEEf$Cl{LWl&#(qTkTM-qn0A37nnF$o#dh228en3E<$JM&82N&w
zb9Y)J9nDBMhucN7ZW=xhAKP(p+4dviV?Y8`{63&+3vUBN%d!vyuB(pzYGHu}Fr*^w
zFy)HP?+IcGLQ!y_HE~7yj}M@)<l{paR%@yHo0^N_0&PKS#h`3QlT%hoE$#hKRqS8P
zWdyu`jcNvv06-XqX$F7%_^|_Q@J9>xcaxHFwDnrlkc1hx$k`Kk;d~d$;ZV_j)Umo^
zWyOpiQztnlKxYe_fTQ)BpRmw)uK|_D>XuL}VKY#b@JB}ZXwo$<R=M)4<TJAl(VpyU
z=?}0p8!)5+|6(x0{2zpu0TT^di^HUmOON&2pKm1uy?&nby1LzYPZ3H=zuPb(gufMH
z5N|&Behsw9EHt_i^JRGPMHm@iQ7-~UEhK6pqNpSvg|=f6*<892)t>OrNChloNXFtd
z<5`bCptuIl!8Mabn7=)A;^NRA_7K{`get^=Kj@lR8ULG$GXulsIOMR-{v%$Ln^z8$
z6a<r<xr$>`FeR(|^GAX(W*Kc>hhY8wlH7@LLVSUS$7rQ9xy;f(_PG%3@YMoq=c>Fj
zv4(|PT44jA<O4gVF0HNx1GO5MICc2P8I>Rt;Z7B=4~}$VLb9P`;u3jQ<G)(OJwU5r
zR>H5BG=S#Q2^U6J{Kt*_g;BwxS}i?5Yq0lUefTfs-+u{qnRE{8e;iAxML@$823;m`
zpFab9hcgRsuAJ#|z$6ocUtKG~Z@|F{SuBzO0ypBIk)`CE4Ah{r1c9^yI&8&|FhyHv
zwWi;pm`TABxQBh>|KF}hg)-abLkL_tg=A%)!Mf#u5V%U%(CfzJl-SpeOol=zSQzhF
z5853Fx0D8Lt<?RigUSPQ%!|MbC`Pr&i&HlWum)``v^#p8zkir||Ibyj{(zN9>uRJ1
z8#;eDVKO>8WT1`R0eEV^fH{>7aK^yH!|NIi0#D*8WP(|O`>ULIX=f|E3Aa^sr{lxJ
z>;ZRdKmrFu4Bc@uKc=qn8v;o|k|?8B;5n+X01KzADLx9;g=o+O2OOV%fMz7cd}&+{
zOlvMKu5mOq!#Ec5?61#_8>ndjHJo{5&(7Z1)~Skx?4imA8;@B10CvlvRQ`=XKt)N6
z$E?lE6Xiiqy3rdxhK<z=QGm4ty{^WB{RhCE0D1!AF(hab5)wdmj_B_0Zu{xz`yD{@
zVWW-sS54?fAp+^i5!+oyrOeYT@0S9>$~F>s<cH;cy4|~^(FQwXj~(3J(+W$qB*69K
zV2ms3H4ukf$E>V9ZAfzPZ)ZpOJ(V%8L|-lOk^^B-#sJ!1xQG1g8GGf2wQ3H$t+CG|
z?>z_}UL)YqLCB_SEQ5f!L78UtQ&5y*Hj!#!-~zkF|L|cH;ydnE>PL;cw7O{WU`5o|
zTVK?ytORDR?%{?!?>>S&85FY#`P82UnA1s2D+ndRVU9XB;JE6OFdAA~2K494WS7X(
zX*f;msLrF?up`KdJ(R{|*Yr$?zzj%15fuYv-`guBboAbfqZOFi9h+_OQEI+dslm3v
zN_WS9d-8kmE_NI+Jfp4<!lVeEZSKdIR_h13Kk6N7fiD=D2}MN5{<W9Cz4^O9Iy$PE
z@D%>>t*@}crpWKI<uKh>p;<MB@1r-IXn<ajQ0}`Qz$G2?iVSZ#QHO?4+p;DY|BX&^
z5!-p2`8DmK-|ifuH?Hb*?~x!K1YX#l+&S+9#aLW)1Fb8<<F~(RDX+ejR+i>agg5P%
z6n$*tv_ZmC(ZL}khPO0&giMCvSs4&3gmjSp-9&_}Z)m8sua&L+(wno<&1^X3*n7mC
zYw-ODPXN7;aA2@#)_htG@IxgcgXz@++h0o{l(RkCSPk<o{Q7#`{M=>oQ|w_a?dgMn
zGkjV{=bH9}^)Eze{<wltFOIQQM;}}~0;l=c&xv>|T++_WNl<G);o<+T$Z|5JU_0ko
z6C10-)@rqjA(o5iQB|e*JKO#Pi*t88o06_XCp&K&%7A~E?Y1NM7WK2;sBX~o=7eI>
z#-Dk{8T8Vfr~|8+3Q8-5`anGSR%I%g;0<>HL=1<fj_<j2;qtTPc`L=PMH8<Q)Z$Ya
zJoip~X$0Wm6#awYk;8;qm@Tp4khn#G{{zV{ROXhf<m4B?>`zCvRLw=wBmnm*a#+VU
zy4HggnnDBII$`@>rH}G}`kqQ5C&%gBHP@>gcZ=w#i|tVO)7Fjn_tEO5MCH_I^u_Ov
zHE#FnqUp7HFR*dU_@nlpQ2(jw@pFse*CO*;RKIh{X-hi}t=JqD(r<ftF>d9V8vEL@
zVx~c#xYAEMG+ycC!e{Q}kB@G3=!h4*4wZobGuJ+Ekqh71j*4efYk_FJS*@<A&{rff
zg%dakL{bvxR#y@gL#QJTF4N~M1#lV)LF<NBuqfcn6TdVkgBKH~!0-&<e9&~qN$lv=
zulya`?qjI$d-R*{H!BnMWP!;$$z+bFgOcH0wuml=4$@|g^tdwH$CKBkJ_Fk+*MUbN
zGXTLH5pQSJmTzU1pvnz@W!U3TV29E}n8JCj)4|}`YIRJ~t(6NgdL??@;!7_IOc$K6
z$;jUdQ`BDv83XBbe|0#}FRfmx1`N>?^3?TxRDrWgrTVGDPqxQcSQ$Kycrf*C|6{=O
z8~=SW<~VKhZsh$06&lA{)SbuE6o4k&CL!(?hs%<)b&vR&l}+T{XBaYv+d_#hx}_bG
z<FLOs#l^<*6RY0ZG0(KyA1p|(WVTUgG{?LC(-2=o0+)e6h#V=g;o*?rqlKiU!-4Np
z<$~|^5DEEFAI*kb7zTH_i@1!WBzG=p7#vK39khXx%12Gcw)<J)>-;_a()3koeq3Zd
zA@*m(RZYr{^B!^hZhbxaR}uq3mpjBipg$5CnaO1}MCxVwd3b-csv?!MEM4(kMSBc&
z=GQ{0>4r6b>BR6~iTng5;<8&r12%uWh0L<od7%X8RjN6)UJEQk+TS$O>WP(dJ#i<_
zI=4$y?q(6Jd=h~~Zp)xvO`l%OvWZqxQ*EWKW^a)OiS>v3ICmi#_5EKJtx{^AHu9{x
zKgYdHG=f(Dw3eyHTjo%fD%WgR&zHulzpk^5*xm-nI7vBqT&*vgaA#x!Qmy!pU9G=a
zlm#pO1f-S!96^w%9|kIL%QwFq_zJ+oxy?F;&U@}KPE_ja1E!a<v-6>E>1(8hJ?c-f
zKuja&%Y!g}8{X^Oml=M>ZC2{crKi;g=S5eb)?4stAk;UHbi@8DsgAw&Ne7R$QgTSv
zyRl{TSBkMN5yd*v_K}U!Np6*yF6Ww+?Ki$r``)WN)de3}Y1oVPrIZ&4EP9>P9`6kC
z$GjWYj{m$BXQ+$w60@LJKivaLrwBC~%8cA8oY!UMwr*#}!qRpn`mAZb?DmQ!)`bl@
z7N4|P_O{X{)usOS%3V5@Y5Vw!N8lxm-c)t={N2U`y?&i&PerwFn)RLOA>u{?S6J*G
zlwxA1TvS3YLd(3|!L-v(*SVYUtO7yVPI(5eN)!E<;6KXzDF1?paLZf+4hSN;u$E&2
z;IlzNNm*+#N&$X4x5yDiy&({|96}3G45YG|HhoiR@gnIdrMJJdQa?U|%Ro9IJS!f2
z;(MHR)f@A{T}d<p0-4Wfk#&nnH=8^k?;_uxu5dM*Gb;~^o@_N0WMEdO9!sS!iK8fk
zeBTX8tFasR71PdSI~Kez{bKNw;bbF1U;RL3l8|Q8y<cjgTAZ(@3R?8X>mvCx1Bj7Q
z#jYurnG0qUiZ^9BFOkEr;+9-B*}f!r6#234B?zR{P0!<#;FYW`%$mj||D@IV72H*|
zLNigj(0pY(#FY3AYS6GgCVuuY+9QH%CYOpLJk+wMSLF9FL3W(aj^|3VR-IGfa#=_x
z^Ec=zw-dW+HA8MI;-@`j`-ZYG>egv4FP4-z8vdtpzeWb?v?yPPPffODxq2|_Rq4L>
zl^T2_D2N=EXH28g)?Mv7g?!&69#pYr|65j4?;&iFDZs-So{HmsSADYbsk_WOnByV=
z4fZOsvfzL)y%n&E1AGRvnAnEBx)~HLsx^KbGV&#IoWe$}MM<ZU_1m`Y481&wW8Uy*
z=<jP0zf&&a1mGn_)wehL$t-;KJf13jgE9~w-1S$LZ(=`{Ywt?Hd+VxN{P79N-)u96
zKSiikiIeDRSP<iqe3PA}TQoO6qnI)ag~C9!d-ns)y(z!V1}*J{u>~`qAMNrnPp>Z4
z2u-k_y;q~Misv{Ti#ImYTDX%RyEN*PluSmc{Mzr!dnX<e8uI5ftxm@*YV-Ad&26SO
z-==<yPO8UT8E9yMSZdT8tye|jKiR*Q6&zZyn7m#~d&25E(~dZQ-Y9VGTH6}k6B!fQ
z4|zK1y1Ys^XTt~)FugiBqZmzFkNJv^SdV^p`J+pFIyIU;_lsHm<>a|){kxh%Bi4g+
z1}{nK;&_*VWWB!K0lWKQnt{Dhimk1}V-DfY->S8z$E>zjy>nkqe=m3?>)IfZf$qix
zY|HSPPqE-H{{GDHl9Cs~al9~}4t!Jq)RY3wkXEZ_Ex+(G=u&X55+4oDs54X<)k!X;
zz<|I-OxjM8>}daVDNRt0PfZNWn_vgFbfBdSJzs{)N4yv+@@>>_Cp+|DD}81wrT2}L
zWHLg<jtjAM(&c%s-{@_9O})PToQJ3jU%N^B-SYj_OfC-Vt2({p;`G#W5_=j%&ikAC
zYP_zDn@m-LH%8(7Yf7Di*Soc3vQu-2F;;z~eC%HP)U7vH6cfdYYCC3M_-|Fjg@Lvw
zik86M{`-R1ZKEh7jRx=1FGN$^_M&F--vH$3wtm<N;RU}976pbgX2v`E_NMA>t?w|u
zDb15AW8b|C|B|k8v^NKpL>hsP{A?2M-R-O2T!wOtl+;h@>s1pE`M!U1Z6>alJc@n!
zlH73H!f#I-`Z0>Qslv_J%d>R%^e=Fhb;9G0b=HZ1+9?z7927mR!o?x@)C<STquz}P
z=5_}j8Tt392WvhBvH)$nMzMp|^9%wmyE4GqE;8x+gisRGd5KR(9_Pm<-5_CbA7{c?
zbB9SxLQq07WUajbMRSoq6b<(eLKt)wHcjI<mbke2k!01S&*zR)P6O2KzP8F0vV|cY
z+j<_Cdse@@W%*8mp!nvEN~oHE6SV%GjmfS*<*$A_ze1AB^??@F{q-)o=hl~yvnLn3
z=+l0B)vDIm=UdTB`ZE)|*QccP*u-&)rt1V)y3^2a1(F1*je;v;Jw<^SJLHCcQ+vz0
z(?;Gf3}(c>reytQPn~W2>G>2kmf7eo$&>kNdBFwAr|&z;1`MF)?)M?q9EaQMXN?=z
zhT`J8Q+~1%G5Op69Mah<r}i_2KG*YPr`syb)sov`%JefgFY`4~&0~CSSkamrZDC7h
z?n?gWc@DI|=KL|X$s14SvfKl~2A*7AUrK*}L^wh>;@_#Z*SjXxVILhhN;NeT2maa%
zpl@;m10^R6FqluCH9zcxfANxE8wSD*iNwUjEJ0@oa6BAn#xTFo@ed9jkt+W67Vfyb
zIQ2QPjN-~Rt}65Tpflxjlo$mJhgzYi=gbz9^SK+cstJVQG{|9P%AEYy?Ma&fd`xwv
z-pl!#p6j)ZXFF+?){gT7$(WDUv#-a|PkxKIx%9u;_>?wZKPgnClfB9lA)omb8u-U!
zv1olzzGftoV++s4>UbgU$xBLzlQdW;l|wvcGk=d4Oz|c;wV}bHH*V)eM+h6y7;LC&
zeq0+d@3F9$<u&pUd%W!@<CV(<2WJef??G~UwIS#^wPAMir*$N?=lz({yBm7Fv~o-u
zv(|`R?KAK_sdH8?Ybo7@pmASZcdk_NjN&gR`Djt^tLd+6S$3qqKzh}*obEXVY`q#$
ziRud4by6B;?$Y|xUn*B5EL*9X5e&{wafd&G<3Wmz{rULA+(VYoh?-273QlvmpnN$)
z3Hk8s$cCkF)_Sdo3uLk_in)=5@UBFXV}H^3_IP5v*h?vo_Y67VF&v_WMgk5jD{4Ew
zY1&+}S?eOmbIN03y`&_U_0c4aM8CaGO6BI5j~_^{R(lH)z*nPMuR7!R1P^AN)y60Q
z)1w!Z6EtkuW)7g!xTobD41;R1!CfB<+r}Ht{i=5Qc7P6r_D=1n534+%4dbu5^%AZc
z%bgT=J{{mA+ny8HpA@C^i9@;*qA>WGpZq8)aj<bcTI#H7HzI6?ubgxq0vQyZ8g(?@
z&69-EHLz~4a={_FQyZYUDlPAeXf^v}j>+ZESogArmlsW|_Is*5HisRR{7_<J7u7%%
zg~#|=56eftL;`)!#KDAj2W{7OZ97^12e42({3$h^GS+SglhxcbCptevC4yH~HP^^;
z16~4xL}q|Gd%Nj(Qdj%S@wi8|W>?V7BX3zsczH{GW3w;GeCE_v_rlF<$?SN&adKm;
z3g4_>cFKgLV54p#nz2-C;{H08$r#cpHbv#~=+Ud^Qlf8F#NHX36eRbz<!8u_hetJ=
zkX+3DZf-15zdJBj(?bqZFB^U4S1-;6dWkl_awMvV9<eEp<3h-f)_Y|*c8~0L&3w=A
z3_@-jKf0c|rkzieepchRjys5CG-lcTq}XlRUuUn$zY=>!BB@qecmHj{TwanPY9aeM
zNO4H7O>_jkT0;i|{n`$x$>Ut!#qKIwnpS3>gH5PjDE!o;*7KNTuLjz|qCNE~&PuO2
zTw3DRt)#KVdg&RMCDo6Rk-5$Vs$dzv9?CO9PFXO`)B$1wb3TJc9gdBvcY^_y{^Xes
zF=+c<Z#6(7mh@2}si3y&4-(BOQx%zmct=^E0o3d@dYjwLU;iBG^k#8k*HVzlN<<g*
zN1DAJXPPKe6DM51lQ+X%7^yAbY;wuD^_?rbE<-39FeMq->0V(T{$0JYF*Qz?VwpHx
zUq1g$J<XL=WsQ|;qVBEDI-Lg<<jx%U>z;i#(lww$fDXRM(5xUs4x?(f9nsk_98Ib(
zRjsy4IbA>N^tNtNiD%(8y=^mh&~ClyGBy8_p)G8*k&#<6b%bzzvq7?*-@Y(^RQ7E>
ziVFl8R7)BS0nK=t4IO`C;Ko?hks;DsGj^rD({Q>L1IpONa%<nLH0BsQiI*LT(?JTl
z56hB<eQB&<yNz`~W!sfB5KStlpr8Qs6bfDUr4YSIv9}9g1s_IH)^ba9yunC&*)C<K
zcUE_5tm!_=qbmbJN~bcwQ?c`zxSxt(P2+ersP)vtwvh@lXx{5!t>XXE#jp-v;tYD2
zy~kS0j>^_scU^ybS9RUAeRSRQFkndwm5S|#nbU<Sick2NE4<_|k5f2a2`JLYe^Cnk
zFrk|5oZurTF1*}Hty=58{RJA-evijtTj6sQ!D`KUvIS1jtvqDM25+&j=!z}rCzmz7
z0xYW=YBWuz<A)ezbt^8b&rT%$_T1v;-Pil8!*APsE46eu=jW4rf`c`6W*`t*2F4Ym
zGK~Z7(%oo&DJ!?rw6i8$H#DnUbqQTx=u!H-34~>PlBrTvaCYFthv#h->0@SiInDdI
zis}M~X$v3BE*srtD^2~Ubo*lOaDcNn7jkT@gR(vLD{!P$5DEb4QL$JA9<Rfqe@`@V
z>GeT-yFZ80EItT%-r17Ga#$M8KJCzhKb0t>RKK$o(P%=%8Nh^pw+KaS9}CsLqpsFC
zX_`7OH6h7`3@63@X*&L>cGIS8Rzh8!RXfGBrJ}wd_re?c!>{^&SfUNuoOx%ZH@$Gz
z?jvL!a?kT+G%#O#w>V?JIlr}4vc_jDQ!tuXuz3HLW^;0a4_B8}SpP=f&&5`KrefjE
z$ILgjiLGWo-cvFwhADMV<cU)yM!sQNrlce{EFo}rVQV#G?APfhz%_!3H|&g7p3ZN)
z+AWp!vCd*T(7PXbg=d3?tlTGfcbTi_zGL3p1`<0%8gJHuAXYi~7ItND7%NDtu*~py
zcbqehcCeX*JEM|qgG@EpB5x`qG2J_PQ>C7ScBwGqMl*FV)P!wz8&)>6scyDYtiJxI
zIm6=+i{E`C1)-Zud&mBy3U5)b*)Y>*a6Gsx5`l1Hrm0|RkeAY|6%*_jhpio$)}uQj
zy-F(8N-r1G<)XlXpbaP-xmoxI?046S8s4B4{IRq?J~0`FSp?6*#PKWk=bEkeXB)T2
z3u6F8yzctRKP)p?w8I$z4f)aIV}paj1<vVQN?wj)%Moj$M?*Qt;7e%&Y@4Ht`lwc%
z->l_j+%CwRbP$`97;HE<mLZ2lhU^b*i1(O54I7MwUh(tfg!U)$9Fy^{BwzZlGU}S~
zBOPbW*qEi#vem|?EcZobz1D6`UE4)GI!j{^983_!*T4TP3)f|XJ0Mmpd`Y|aC((rY
z%;s#)TNkzU_=aQMbp102xuI+7DG`Cbg48;BYl>07&4!ndek>+df!uNB+T9Pq3H*g}
z7W?vAx=xj5aZO?=>A0|N8&&$xzk56=cU?GkiJ>uWWth}%nmb;}LuK`bc(t4nnw1wm
zN&JZsbjDZZpK!0wNoUK<qDua3q>`FBi%M7%hcWU!pSj-F8aGO0(9|(wj|^+}PQj`>
zzFZzEgA7D4-Of+IK)@<hw4uvYD1(H1*rR{uYGULMkfP$On+`Yx1gaPSMmo&d8V4IY
z_u|4`wM>IWlG0JFnypRswbR$?WeNg#6bgqM(bO{{-V!aSP7gSOEP4mT-DhO`wXrPt
zUf7BwCy%1>*e(p}6k6{mJ42VO5_2cM>WOiDyK}4Oz>W3zlR4wcO{Yp(=0YfNYgVgN
zK9Uw$?p%z7wl|}MzA!!|$7!-Vay28Xc~W24J|JshLd1qDl15j!-%@<4iGDtG6d4@s
z@pa*>Q^TwEBl{~BBhGl0Ek9G`R+e@8#E;La&rNS$=i&@nAXJxMS~f6!Enw(x({<Z0
zxi#<60+~~sfZuJ;ZfLovd}^)1nOo|GYRji`MUt0kn)SwuszdB!sZISCrVI1Q4WhYv
zsrk2RIn>5%J@))wi5yN8(n$+{vlE@UbKV_G(AM=&tquK_ee%K2j0&qxQ!0+FkK+9>
z;fPK?09ax*yJo-vHfZD~tLlKMhPmCK-_~koxqZjUq$X_PA^8Hqr~$+r*1IM;9k+@P
z;Elk4=?FwX{SE3A*Q6nWH-|x<XS*+*S-5MzLm=tX)m<U5w+2{Fn)O)+{FxS4yQgO#
zL$(gk>e?|6EIsT;WO!YL3G0czN=PJ!d>WEOmgiziH5!Ko@p`c8Mlls78^ithofH?l
zIac@C%=^u6>A>`{lG^6Ih0`6CuM!0Ap1SrUY|3)Bvj7EC+-`y*#cEZVj-(84d%M7J
zExvQ=V$C1)Dau?a$FdpZ0)|Oi>=mAEMrQ(SSCqa(cB<8%FxOKrrv_`}&8}#wTTe9X
z=lJrz+@_NL_i=0f;iG%RKPP~fJ1sm=5HqPHgsK0+498(kmVr5Qc9sxi<oXkKM+(?c
zYn9po9H+0fPxbF>)J$_IPXP|Y8X?4I2w0#F!m@XfOQzQF-A|0$>anQ1cS9<Nim*Yr
zc}U`ro;@IEa4T|oxBxl(1$TROIt~;!3P3}k<=xFC^rqEZ9uebLl-t`CNy&z+cw|f%
zgwa9yxVr7fBBnAsIKLB|n~-{4QO2M=y^w-H{!YI2vAdKM)U@_)xjVsas$CGfd^I=)
zUL-ACGdi)drS*0_R`(CoHBTLQ;Jq&K0Bm<O5?UgIcbnINNa~#hH1=tEd9~Va&(gMk
z7h}inIls>Z7jHZdBZgg^tXXKf*x75l;FTBWpf9b(LVWB`{mGQAd|$mfu~y!R0sXhb
z;*SD^F(s>@^C_+LqZv)*Z);IXv}@BPlsj;xR~nGukS=s#7e;PhFwp^_G*I0DxV%}-
zw=@EMog8FrY75{C@|`d;NLx<0^ZOSWi+^uN1MUaOy|}$^i^|;eBs7K;?K^k`3?!=n
zY|FwL&jCH#1;4K7T<0((j!N5dHf4I&qLrK@FdI`qk_Q0eIl6Q(J1bz82I@c`j>BzV
zN7rr6B^5}5>^uvkGQh(6KBim)Vhrrv9Ka94{V^lo(Xe*hh>bOxR~4p%J$ncI8L<?!
zzl)#uvu7s|qp3F1<S0>8T*@YY3{^@6z<(RakKPJ*CY@uuko)@w!Dsm0dB8j$!^6Tx
zK>=PhTNnXRY@<v8K*ptlVgJju^Km^=UZz0#Dc^I89~Doe<ps+>9=`6LC<96ixpTro
z=|~MqWM%WT>OHoBOlE78a?PBca4_>+CydwDVl}oiYM_k#3sBW2KsB3a*u;$tq+m$F
zdRb`y#_mGD<4ptG`$`G1uSjM0wT*CaNH_%@qY2nhi?F%rOy}7)oBT1Vi85DHfdiCk
z>;^oz5LzMO9E_?)09KM=Y{Jpe(ZMh?HQ-7OYUP%*4UPfxCe!A~dA2V?p8Si{NZcBW
z$){iUlU!zS39xdQ-dTr*r4h|e1%L35K&ad2%i+*I?=NG+w0fsF3DSg#5*=taMR2q~
z8v+=OYW*-2F!YDg2EfsdfX}rI^poYS4J4I{hGFSVh-7!-l94Mm%<C||h94AE!FD56
znk!ku>CyQSFiKYgp!XYUgPyYyy~e&&>)X863+o|i(;HVtdYB1txKhVW)OmC5<|!xG
z;MGozaDJS;6PqH|A$;5pOH@%`CBhc+cN&<L59k??F*G#PYH-K~VzId1ZyRd5ArPJE
zBPMjO)6e|VCRCtek)Srvq^eu-_5#eVrvYCOKh@a$kQ2j0S3Wm$*|kS+Cbu&J^$a%@
zA}Kl5ts*41r}Dp%xM3i`55D9>*z}Tz1(rO}w`d6zI6ben3QvJ>kEc@PdpJ@5U{QZ(
z_#s;M0FxcU&T4h)euk$|s*2A!GC0gN*6+!|G|Zw!z`JP^LyXBUGPFUMs8q|QP3x@v
zuRZtUN^k~r?-qxBvO^ONs0~bj1hNub5Ol73prE0-sQEvS`S9;%hA}nld_>$sPv0%2
z;tmvfe4*bUGKbj%FRBDQfd*Cz(%94%mayeQC^|3GhKhaO>HM2d+^XPdl)XMTz@7#j
z0pvaf!1`iLj~N^qaIJo21%y)ZFe-o>ftQy572S!!?aiPHjL_92t1ztlKTV(grV4S?
ztpHE}5`1{a*ePGl&M!r;>`Aj)dPM1J3)l_7LMz!~99X+C$=onAQ-=bW=m#!7ej(6O
z*ao7V37WrC882Yh(9aw-IzYgqvKep7GttFR){81pKWU?ohP~Fe^bYK6!RuwyHuY`R
z<688%eKjs`rxV%#nMglY2S;iL(|T-na5V>U10Y|T%I%oL>$084^M0Kc0)bB@14|k+
z_=|XaD;5kI_tmV|?fxM=(dt1Q68bk~4Hpsu0>dVPn%#Wvkk|8Kt0^Tb-FB^Cy&{jm
zLkPWvKO>cb*4;|{ZV;!sKIjVk4Kw-%f|}!`8O44;tMRT?!5^x(TT_c*F=GKMX)U@1
z6e|~Q!+fSyR1%g~R#YqX_}lg4Q7=HW4))V2<~W33*IGalaC)Qa2o>bqQDMY)N6Qm~
zU5y+$FXK(V<|{6`J(VV2&XVlEqf{tc{W`EfgY+gKvK@0MNi9{wk4CLhgZGd?D&ikz
z_*OXZ9tZrI4|jcjP+w>@-Rf1Z=dp&1Mk+9ykM*6@wWl6M{Q2`cjgOCjrd{10Uk+}X
zD;+(B&Rkh7rEoazz@Bq744;U`*Jw9xhuSSvI#E%&IL4<&$0R)h^Acz@eNvXUGF>Lw
z<98Yq$PY}+vsG(GQJ!9d@9~8XS9sWkC|YhCz+1q{2MPgCU@G{4O8pLK`l+yx?Bbfb
z;cQMrAShiDOJtam^-pVuwNi>!z|(6(qZDmIHpwh`hF~Sc8yW}t+nV+Ac^e<<&vla%
zOCvn@n##bZ7e2Hj`c_J-)umENzCd#v6JNQtWKFeQ%cJcl7SWu<D_sg$ni(u(=h7+q
zD4+~Zxj=RljcYbbDPMX@nPUP-&FmQJA%d+WjBMw1>Q1763eDVLk{&g^@Rt=U4^>1k
z2_8MBK}qKM_>y_OD}#y}ykxb-JdQ-&PN>iHw}aS0(LE~4aP3?w`ck7UTKm#GBCriU
zJUKGyyqpTyn!b~LwO<h6upAQw`fU?{G~jYR=Y}X!ZE4H|vJ$87OR>fv_EB>k@aJt}
zVX}%GPQkAEuw4N>=-uM4QJij?{klm~wvCig6Gsls6J9R1XmW)Y|8klhpB@e{Q~lv#
zzVkjeN4?uq)rGz{M-l~AC|PYsb#wqvOUwm#^>69H)*rWKhj+2*N<|Q;Z8g59=hwwt
zGTa_*i54#3O8*1mkOf$Lv9YwNd@-dI{z7p`-oR`db`%257d2KB^en2%an3;_%+kW0
z8+j@KJ@`Lg{Tz$icX=>p5)`;fKnz0^6Q@Oj&wSXZzq?J28Nqe}K#Y-cbab3<aw*mG
zJ`#%p{IF6_S{!${m&nND-{OLf?eu%@FE?DNPge_q-tY5XEXG$q$`s_8Nu~t9#y`4m
zPmdo=xqbLOvy`VVed<6wGt$NFV)p0ld<SA?I=uH$W1OM{)chrn?nu~mgytpEv1S)6
z&E?iJ`1959EEOg;GXWeYaj~$LFOui5450VeBUikI7d$jHDQwqA-)U$b??<JZT-b+t
z1$j}@Jg%Q5MBEXZfnVc<m?||;oela|w~(EIrgo385f6oLyPY(sJsx_??@1CEnh*N|
z9W#<H(h;c!DmqX4oHQp}A>>O<Of&}Yvvgk#zQeif&uYT<B4DVI%3-}Ohm-@`&i=jh
z!^VzFEG&U`B;1E26!y-Z11x&UUk(BfW`iyn2Zt)Yo_jVH7S_oh;}}bDYHGw^r2Sv7
z;1UL1qcg~f{pUN?Q9&ehImdnjw~z6H$0@HT@?|cNGO-2$onEh{fe(&G`ajnO3tAWx
zY{#^Nup|;ilZ@vyFNdNN^|C|<xUr`*2?=*@@ymK6HRzBD2<QP>ekcfNz-Q!*%kH1i
zHHzSv|IP$Zg78!N)}KTVYniy@u)xCdUCVcZ$DoA^xRvVV@cc{U5>}f3YMNa^)ZGHJ
zCQydL-ya*6EmWz8fi1!2aeT1FLR;%+CfyQXAh(mKuJ`u;q*NVLaN|k0+4l^3h2Q7W
zN=0(FeWOx?gY|)~3CIbA48Et9Us<TNjLB!JU2b9#{K_v*ovcuTXI%OOVlX>M%;k0z
zm)|}LkPWCZ?zIA0Nr`540WALkA;FYn1py2tMiJamSo5C!F)W-aq)9e|cN36>abddW
z+Il@+NpBt%>u#ms783g5qI?blxe=G?2AOy;oc0+>T|8D=`GT3XW_`((c39XqD$0nj
zsyd)p90*cl(XZ!ZdlHWVj!|qD+UE4wD8NbpI>ySG0zSiDBz&-lDugA;lTr{zI{bSy
zA>K33o@Sy<g*rA>Li^zsu()k&&r;+!`rars*NIS`wr#?-*JcI>FIW`601vORSi>P~
z1nn1nr(2`8Za?w!@U`w-TXqviX*q_Ve5iU?_IuzSe;5zequ_TXMi7+CJAkTSC6F=`
zUOxx)LA{!&rERd9{X^lD{WMGMlPV?=Sk_J#UCKOG$O#K!djSs$#Ac>2senl-Y<bDk
z)4?q<oFQYS5rGG!m1Ld?Vvhn0Fu0u=#Nsz9#wrQjriRK*eH_hVIjBhVvyeZ`=SCsH
ze+JR^KZF_qjEnU&d*^Okb=cX&8d|1kV^fB1m5%~Ta@yfS<CRbUNU$Y({!JihS~8j_
zR~l}|&vS+SBNbewoOYRN+IbNu<~CKmPuI_(CSSr=qX&2J2<P>9gxohQY{W9L$|0n%
z{UPg}5?8dnIPAxNbW(#K!94~^FvTOj5NeRjpTqU_^?{0xvXN2FAOY6-f?xZ2q9PVp
zkhG!!F=oBi^V^JPvn_7A&?bz^!F<><HAmg$j?Ej5-<U=p&LH~pF0o(q@(nm>2+-sU
zgALi>@i9WGr3`cnphWciw>A5XEm2y(4ZrU^+GkDuAS47pL8PMnEEX_l6aqp?JSBl&
zW($Z0#h{u;%g$btrK$u<f*6O*9Th<mr>m<xXD;Gsn+5cyGFP{$u!9wbLwpA!o8NdK
zhxWvF;ETDVBsBr_FAudS3^3D#3})IWwXS-f>yEc6V$^)520=Uvc5X<>$O3S%f<XXa
zHcf?nJUMX&1Vu@})unUzq5|6(a6WYGj3-%zaFE&qEZkXn(k#SL>LHzLknR}tx%(qg
z?M$(kaOWsOGf}JrEj)^kJ*1;ae4u-QSrY_X!wv6|(E#w7F5sgD5;`b=6@ViB>nK95
z;de_gAQZl;<AW~Z=}iJcI1>8Ac#vn2vx+E&gvdB-`gh<aUkzyaG&{F4eFO%vl{=f`
z9{{E7|8b3$+q0P_@PmtuZOm+<TOj+AD@48wtlutRc7IgOx%(yy1esrnwuJDG5O&xx
z7s)0dA>NYUa!HnoSkQ3DZGJlSprvGlKvuZZJV6D_jEAl4IVhzr9X_Xi13&w3@3#4)
z0OZGi0S8r=l_;(zavRt=01@e?&N8XTuUTc3isIyCaUVu05K;+-4J3R;$8KU+YQ1N~
zm78VB)kObwSvCN9#id~B9PBEK4j>XeKQQPN=|F=sGYNi4?g-T9wn0YkCi1fj?5WQ&
zxNW-+S9_~#$F+WW`kxw<X;)7_{}_$|3j`f_lJy0b3}zfv8Z5`-ZiL9l3z|wW4x2ie
z<2kvQ1g?6>Chf*kDHDJm7lPIHCXD+rfCV%Sw#M>dvhY9!4u_P~J|+Qa^=u4=S^$TT
zo;gGgJ^@7$PragF7z+YK1D?cs4~(e`by$AtQw+v0P=C>6nl37pT|(lM@?NpDm#&67
z`OPii4Y3S7k~={3I#JofLB48$lpIn9<RvP*Bk(~%pJeR)2tWP{v|NCOJUl)4qXG+z
zamhZl2AJr=u$_i_NH@QGr*0<S_?4)&Lj5`nS-`JV1UpfnCJ3Fc&q-`0I@aJ0T8^Xb
zn)f)ndwS@}$=~KnCptAEE&Zc*!M+<O1Uzq!!bU2X29a>qb^GJJ@71hEmW-h%9!eY3
zwa&Jo;^AQC^*C67^znb)^C&72ud^iJ1^=3F^#a)%<n&;neUnfO5bFNBJL}MaCrq4^
z`vu-#en+43)|Km)=k4Ln(|pZJ840l>{X%LhWkKiw0Uby-zC%EaN%oAW1FNr~11REh
z*v)dg>`kiz-=N{y5!C<Lbir1u4Ia=i;ypiD*?}FyK@uE}t}W05ZiCQqaJrcLU7JFm
zXgF$3bHdC*77L<xSh~p{u-QI-Wq?=NwwV)Aw)Ca@1W2frfIiU2Gin5|ROdg_%o|Lz
z;R9m|FkkPH((6ie8_Y@t?meQS^V1dFEk0IQO)4bSm2G6ADuJbm(+Iws;sA~S4CEy)
z(EN^AA4)3(b5b7|(hk`>4#SX%{|rY*AxJ;H`=w05SO^E?s<AE~BeYu^c5>=Y)df^o
zjCX{f1V{SwflRU?H!}?DaW9f=I=5obQ#7SSdes^rMw>{d+{03Z2~eZ|%n2a}aArdq
z_RXCDug)FL53Jtl!~yz20=n<ufreofszF~WdUzr_huuFcUgSH(e6AM2|C)f;xrlNF
zLi^u4dlZm^3EP28WyII7mjQP6PW-=j?NAK1z9iQ-hLm7##0cIsWO#1V)dG|y0*~?A
zWGqq-H*8Y;_kIkJ!9jn5a6D=ISCvXsTd{Uqdy0%O#n-EN=zl-52)pIkwrns8v0^ZR
zQma2-v!h7`D?qP&5g0Vh=)bU?1W6|V_)c3-efZK;q!$TJR;aTd##q4(cXY$8S6>y&
zuUlQ;K~7s6!k8Mr+O@GuO0Gsk#Q*2y4?Y8sirLHHXrb50&xp7cfNQVP{oU=}!uQ{<
z)1a`vkEc=Nnnrz7BmB-<6?*t;x`{Ao6BbP$;c3)*#OQsg)1adKae9(P0w(@>LoEdI
zreIn0Kf~n!-Z^s8!lU3Aq1Q;sTPZ;BI}3|Snii}rpqTjVgW0$^_@t0!;n?Y)E=^v*
zQmNN*%AV#bi$;Nsl+Yv87Iv37th&ANe8!!vtLP|xJ34T-FXo0pnujSJ^FR7h+iuWx
zC$r<E#qiEGTVYZ54xm0u`ubKN7lK1cUOJ|Jc{IS>Zn63^?u-Bnq3qK#?3=TZZ@|)e
zcj*C6{S4)Cx4hM7@3Q+Ikdh9*fO0Q%>p}Bl90Dw(a;i~X+#DHq70mwu6%LC#tQQy*
zy_fB*O|%{P!e*-+(OBM=$cGg_`(NOYe<eJ%Tzb*lpr>@y)Z#E!YTqX+A61T1*z5Jh
zm6YQzmJm#*JHBpvHA7cnM%Att)Bo^NyeSX&<=eNDYG)=Y-j><zO^oGAOG&zSbxDC6
z_m@W)L<)=9DEob&eLj5(tyI(Fa<Q&RcFOnmB&G*7rUfu*4$En3LGpnPbI;xjDXX`g
zDF8HBc@aQ_Fx-Qk8j&v4J0c@rQ$h$-D~mAI>(39miyna54s$rORDQO(`Qj62{;~RM
zD>eFfbC8Apg)D;;f>4J&oD9@{vhdh%GHz8RB;>=N@qOL-rnb)VNl-BX>k1GNE9EUP
z;gD|V4p-yltpHX5Jbx2EGiY|I(9FIBpJ9hXjL|}`7D&x!-!X*#fc_}cB6%~2lDcRC
zJ7+Ay1>8_4N=Y^>7Lb8Sgk<rUjs9{V1UdEm+Dwy8u+%B#6TzR_VuDG71ScUkqNw-z
zV1lW;^1Y?RxPSwDf;xx+CgM_qX<YJOcwJC##K%F-B(x|WaBhB6kcnMa^6pk+&b8Ql
z+{Zr@7RNBS;nu%Hf5w<?YxGN@wda(lLQB%E0dBt9|KlzApIN`)IRPsgmZ^a?be(Ej
zzIA@3H!GT-OMz03rh^2qR2Va=8C?QXDC;9UjK=dW`Sn|%emYT@>8mALzW5C`+wKqs
z1${HFOe$*0tXTMYzNXQCj_&uPN_YtVv`zt;zB5H5WpqL%i80WX+vZhXt)yVlI5&JW
z*auh<#>VA$dT|LkZ2NfsKgQlVEX(Lw7nTwb>6Vs8Qo6f41SF(HQd+t}y1S9?25Ata
zyFsM88v&`aUVgRrIeUNC_lMWTBfQVM)~vN=&D?X(Ou7_rUSW_8==xkTkPtS+7l;dN
zE(aj@kdcfx<lZ2e3UHrQfX2ak^{N=0u|mM4=T7Ul0&^iffrXsb5{_f#QJgPR6{<Qb
z>`^#R=9M!=lp~$4qC!dzlHtI952G!xHU_H*5rYZMxSsC4<R1?dH9MDd*r-U;>8;w>
z?|B=Tp%Ky;ghbXXQJ2WTNk!1{yxB|<@Mr?{%>`QT&IX~zUZ+4l>r7Q?b?e!BGA#nY
z&dH<>el@!41`{}L!1V`XB85}XS}sMdC;qru7ai3n8?+9Q*EnKYEyfVNjjqFk4!+L;
zm|k`02{2nj)bv3??!EDxK$%3wGLXMh2z)^Cd1A&RYM(O6J2jT+*Gjd}wA2%|riYGi
z(9z3hE=#aK8#VAiw8_8kEpP8;So1iPYiDUGSa;lUA=m|So4*%kb!L+ifO$Nb#A0;O
zM;_Tg?OZPoP>9d`3N8Qb(~$S9Sc9wY0Vm%RT&giYa;0P0Y*vJT)nE>SF?7n={;{3p
z38j!>o1grP%ln_39*w2}T!p>_DgXm-5M2R{tHC*m1~ssAot9iA>z7<hc%hvv;wUH<
z=Fhvr|1<1={`3GbZkXQh0Ux{oF^q^;om?VJNk=CQpned@p%yKex&*wx^i|7$JpX@N
z883jNoxKt;1*9>8%jJGuxi?)>0J2>y09^`#!kqg#y7_}|_Jl_h{O>{o-+rwHjI|2!
zB2?h8)0~|AMBQ)P&QJwS1K2EP=&1vw<bDAg9aNy_|9?J7s>BQg26vtVz&b8H{n>Ke
zKRZDi8y^`yv}ghs!Paw3j3zp2h4J6x{ExKU-vGO)K4S`UO=C6yJ$5;7QL)>!Up^T}
ztD1&Oik`0L?ca5yI~V%@ND9Fj@>y-}0iX;CU@<T;rz;Ia^YZe{K(%D^iQHg<PAUm<
z0O8&K$MX31PbXzSvn;!+5P$*8t|Vw%je|Zh1prLm_d^x1QMVC`Z3aSK@PLrd{_pLv
z0gPHP&u%Uf5b=Kv8Y+APP&XX~h1k}Y#ilQd7(BMGfG`%||L>0h?QRVOcs&z44^%BT
zG(?L4zq!2mxJ?^SjGPqo#nvEksnIm9r^EB_u|7%dL=7xXfl3<CYgQPbXK<rdy;WIX
zpVU3*VqdIYmXg{@t^WUKvZ5=XG}SDK3SItyLEX`N{F}(?`#R(f_?3N`xF-Lxh5vn^
zTp+)CI=CwwKA@g=3CP{0(`~8+?LiA)VAC~(dpn1NQ6Is8^Zwt5EcBxuoDuhZ>CZqD
z-$8@wp3@7B_Uz_UH2kMsm>nc&=+4w&0eLxP{^<n%S5b3AzE(Qm9vnsqACL!EY%-eh
zr!WZg6QkwfnGb*2WeLnf8<Kwy^FPvVgBS?fisquvd{%J&Wr&(&xJaN1-r3l|i|!*)
zfkUG28TUV&pK1Vj^)Yt3fpcH-EQcJNQ0^{yk=O8e|4ab+FIon|4g+^FMuZ_4A{Uf|
zBv-jC5)K+T`8Ue{gHq@NC-ZeP!ZEtgk8tR)2pgk{ZFE-PU3qhb{nu&#V;UQ(V225G
zaDhvkkT!~i2U5@6bt77m5T9Il^FP1)zXjeW3O+_GL<lC<i^R;+d8&J$>kS}W`y5Eb
z?!S#4*qr}y2CUHa9sIRjc`|KdqWY-u3jU{E05E_)n4q6vuBJX=<LDp^rvX3@Q1q0D
zh^b=}^2Gv`T-Xq$|KB&ki4fSg@KR~MAQXTNzQ(Be`-)F=6iove`+pe`nh<o(2Um{K
znXp%<HcQ(|isceq+XLk8C;if-c!FMbA(XTC0Po&hB?aeIQy)UFnXfCsBU)WiV3aZy
zP>>2!{Gb^%ql5znV3YoXUzrG+;}bE(#u7UO>J?%ZFkST=DdHSGxQbUi(l}jIj-0=B
zF5zS1GQX&98O_Sj371&)Q$&HjyNZVy%4|kjq~4uaAlMiN6{D64l>va5MmpB)WbFYm
zSV0gnF2i{Zk=kGEB_y=FDl5My0_J?ifOdh1NL;%yzs(mk5HI=k#aV(u14U$BM^_E}
z56<7>C4Y~&as3TEKl(%sSrBlYKV07djv@>PU<>~khX$$fkmM#%t(*4qQp+!5%GDij
zM8-#Oiq5Ph5P@ue1Va0yMWLbAy3b8QEWL8HlU^7G>y_)fKN8_@!Qy;^@^bf#jj}l>
z5<fs_&{zo|uo?r3^kT&ib?;_7(}@W-dYc~XL=%Dti1@#ff}?;I)d;egNR>39!5BB?
ze*<eYM*<%Rvc_9bpdDCiHxaKL0sah8bqx(%*i0XEKxujv&_ZSrQFy8ma%2_*bHIGO
zGa>p7KWPu*Ky+d~4Q*agq?3Ss0Q@o)?(XzPUZiu_39E{HMZpJ<3U+|99Ruw~%TCvO
zn=yR~&jDFIx*pvGGBi(!Y$bJyA*b|(_74)&+LmC@)r!+R4nmGYx?Vp|_4iLbrZJTK
z;Q<{S99*o|-bQo{64%}}JAS?*L#r|YV&k)4qlbu15&V1yD_aK<x-!D`x=#Y9EjYaY
zlgt#KVE)3ORO`0)pksX7?X#dcd!W!Wv~8(?lvd!u&z?eg5a9b;A?)3$(3{MvFxTDp
z&0U4P+M6{LPp5|w=cISr35|l{f#lh<$44%*Earc8$vQ}fbH@BN+k<*^q_VJx6WDzp
z$s5W`ucb*d!Ep4odvS5`vw^Ksi0U_`1>H98`sG|q3sz|=`PeG~)ZwA@lJ?}KvGWmr
z>P{wFbPNRVX9#mrkcQ210v@h^&o+^C8|}VUn~Yi+g%%!7z!JCA?O)`3@qL>8GlmEG
z;|3ldMwU?SO)L+p2#oc!Q^uXR^`GB_p1}(Jphavc!P@%S`Il&`{u#_OC^Q5USoeSo
zbn-vDo5su0Ze4ZOZm()_pnUzGtBq42G?4j#wsxT)8XyURa(UoiBN>$3du%8KjAENC
z=BJURK7&Pc!xVQTeFja6#sSO5YFd<r0*e^`Lx~W4;US{wcTdqMn_Cm4XF@+15L#T$
z1Niv(K${LH2Vk$!X`)%PH+6^C8^@qgSY`z_LEfUz3f|8_E(Qa6Si$<oD~bNsR|4}X
z0qh7#psfm8-L9It(^#Yl5QU;D(MgN_o*{U9e}*NMn(INW6NX6OJ%Z_{F%@oM1PnT#
zEH3cd-<V^T_W`7<e@1bhd3z6HUjI2v4I(<-87H4_w5NVouJdbiI}CB&2TYS!NDT97
zfc#*9Gc}^}Xh4ECRM08-wQ33&zWy@$G$>%ie_+|P+sULF?rY$lk*=J<;xGUwLaQzt
z+>4U{@6z>bOZt|M00xLD#RdZnUO0p9O=>{IIRy;`UT8x8Em>H0Q;bZi5H?^9{MJfQ
zsOFNu*Z;Sm_OoZujEdf0gMbRbM8X8vtj*1hp|9iXXTqHdq~k9k>!B(6%-8?Rf)Wxe
zSn#xP?@!Me;xWIHW?|rk0AV!nLk-VZBVbrPT0(&*HXi_yDAk!y)4VHIbGSOv4Z)xc
zksZa2tsw_9bo39-hwK_5$aMFTvz|lV&qWS3{8EdK8cA4$k{Jsg%-;<<rED3EV&UNz
z^fNRYGiYF#<-owBfGTY3g`kf;Bfo~+$OvQ!@M%(DAT#^|9tzjLPC!JtQVLn#_pGa2
z&1FSjd7l0&;h*x`JolIO^VUY=a{EJ%O|KcX))OuZ+Onf%TCv#5O$b0<8Az@L1$pHg
zM6ME2vV0i7gm@?HX)x%8XvoYh^bPO72fW&Y912z+r}zUwHw277CLuUrE2Q}e0i47X
zjMmTu_Bcui8tNTnT%BJ#A%oDjhDM_}kjFao0Xs+t4`ZzJoAIy%sg$ND@)yYKlf7~F
zIbji<JhPdd&}?*}0kJ4YF8Qr82p8>+r1Qq{d@-L*r~j0X3`93?BVO|KPBQR-<14}f
z*)|BMu(ix!;$SgmlthWZe>qWo>;8)@+?k$9GeiWdlPFa2q1ALES9&U297zlGIoVgh
zVx5ct+wgEM+ZghSN^kIhM4E*FSr>m?Xf3?j(qvv(M2o^K6tJ;*F~<xc8+$|%YYPRs
zTzUnZIYY@V;9>9H-JHj9T7Q^LqxqD_02$$#_&#KWXfMD+*0av1*CQzYV}xG++X!QS
z0KB^hbo5dKz&eDpY@vWrGm!xqA$|`0(<H&E2}WC4dD3D2oKGV(0Gsudc_G;ND<iPA
znvjEXj`_9G&wC&J`sSS4csL~<)Qu1);IRBQUuV(0M8QJdi{?$biZ`+Y2iem@;KAVe
z%WBAJCgpot7-abWZHSRS5*V}#0pA9qI>}^glq@LFDnq~sODePep;$w~gIly1@6*nM
z{l|<M{@2$DeSLuqB99&$`}?wlbFUC#OR2zg%pD35Am{iGDR>~p8vn;ZL4;xe2c=+(
z)i$nhbzN7<SRQg*z_s?zbq*(?kpt@-_5&y`T2R;#i`UW8_4<SisEE<-PeHp}abReH
zYZ4|;dq?WWdpJbA+Ly;zBaoT@fQAT@en<U1?0$XB^N1&DI6w=Cjs}Scw6=&amFC03
zBC^*i;efwQMy$17&H(+zWr6QPj$#=O3(WyhIhX=!&mkkyhG@(9Z~HTkH|MzNT=ubV
zO{9c_F}rOSgAG5CuDpKIz#A}-6CWOG#4wA;oB|KUi}s{94Fa3}sKByxru_)TW$OF+
z(@((bLC(|DGY;@*m`O07qm6*KBW!b`{RFc|I0k3T|BAP8@tWZKpFDPgNab2n<(-im
zv$(e|(1$$s`h%!2AI3uJB*R?8uxMXj9_XaMmI%{#K*lc@21JJvHj{CIfhb0N<WuvV
zSFtKXBCLK1*{HH8WKx<U{kF!H3A_>Pu68e>G;{{pAxfBQD@%+C2yh+)gtjS=3Av(*
zH9%aj-fH?gYwZy+7)?X3M-M#^Us?hXcke<9W`XrYUdaR2^#O9;rIg&)Gm~7)WL55b
z`!{a=Nso$6_Do7sORb*F@90x|2NKds=0`lXi_GuQso*b9Pv3^1^B4D9{26F*-W2e-
zzNVbosy2}pKV9#Bt9^96OW?3MFY5YdjX^y0Rr1-E40T3HgRNmwS<au%8`-O!$?Xb}
zFq!@6uEM{A8pr*|>fU{i@~-*hk`W)(dXnGblyOOwHOb`^VpK5syh$@Nb;J*asP`Ou
zV_|5E#(_ZLpKF?f41u9_3wG)T?^GbShM4A}$^{b8XH3w`?Op{AyAz;IRHP{=1k~kI
z7A^GS6Hs%O5<$<w3W6*`$U_#c>mortvFa@`Yw+@xYCZz`Yo(t&-KIVMC~>51E;%*T
z5%cS*^(08>6rYAn)$-g0>ZZ*u19I3b7kKEBm|%v}7y5U@hZ7V|G9FpAns<}RujBVo
zO;^wS))vrx#U3U+g2P3#@@`a{mG(awhlhOr?UC^JmOf{2_Ab1<3Xi2w(PMACD22I+
z09z?JI#0=IZM)EE_7$6fvuw`11t(F#^D?TSyY2dI)olNf!+9iM|8a&Xil5kUh^3n;
zd~_V_Az=FMKn^C@>Xc+s{@5;%`LA5S0@O(qz)iV?+;(H}Z#2?8Tk!O`$eszmPFtXP
zi3z6v4s6u1qY(k;3tt&EVyU}+XtDAG;y6lG86J-#jzzB%IR~ACdiq+YSkCS+^x-2I
zv#C$nXxJc)U5+5(Po3Ls`NYZ=Q9+UVB#R+W1Qvz!m1zFU{$ay3_9__czEt7kzy3wq
zI~DDTXt(~`T~@U+ID-n-%O1HkxJMJJX7DHljj_C{zqdBKsu%?IyQ;rttnZscY!z$`
zOp3~c68C4Fg<Las=POkSsV3|68o<b?6776%#nJ->)|bJj+IVmRU>jxt1)9o}#kEKk
z=0_W&ekLrE2|x0*yd7Y-o5Hs^)>PG+-O_@q9^<p_u6X>w=JXd1^R?%%m>n9qNH{91
zF4W!l7uns2fKmwisEJ?2Q&&WB7-eSti!@Kg4fR8=%$LB)3cdD1n+~~JeJzcFX!g2t
zfoAnrg2t@;ZVw5dFlx?V{pK?XdhH$VwsOc0RkdY*+M)2IVIf8goT$VWjV>G`cH&QV
zP0fN~7z_~qD`)^IVe*n2_D0BqKDJoY5%$)--#<kZVoE@i2D-BYSjT73!jM^<Q|Pq)
z@ji<?!RlGJ6-`ymv|<nEDV{atD_gb`X|Qy4`279Dfp4sd)%dLSvbdc0j*8N*1t;-%
z$7A`}?|Kqi`yMQ{96RAyMwhv=zR7K2>;6Z6bL>Mwhh!OP>#@iStK$7kKaTT}mrHd9
zNq1JygOXqaZtkeO&T>zYYveP|QKBZdtyNbhIkQoE2leX}w<a1xoy6t7wlq$tq|LGH
z+=MzgY;=8Tvbhn+lZlS<WQr#5N{6-d7IZ&&Wm~z2%e1UIL1FtftIgq*u*;`dRCcu^
z=yJVRaV?3srH0Dr?}o!^pR>H<G+uReBKrQHMWMe}eb}xS*Lgu-l)tMf*j6Cls%Esi
zJxbesag-lQcJk~BC5Ww-g_hf0`^AvLD=2tJ*y?)kW4A3ON*VGoudI25G{7IDTB5g{
zLQ0>ZSIg>XR?M-zKtiWSE23^;;T%@DlMsIcVj!~nfZKN*R8cDe5D=TkUx)fnKK;Lw
zfo|l55x0k2Mc*Jc$d?v+27Zqv#=Uuya^}B+O&p;r6teniUw+QSs@30cMSJ~Ou8I>>
zbk$9c`^7W1{zIeI>wKAQ!pYB>_k)u%k5nQREzu(XLgAtyk6ga71<ISy6h4#|7LB(H
zy{t{@Z8K<OhIk+F1PT$6FG?+v%A6ifCvp2P$2D`R?8uggs?A&k4Q;!bt89Kkwbz#(
zC`v}?6|g*7Vu(cOFs7&luG6xW>8XrNc-Y<feo0l_iBDwO8psF$l{Oghe@qU%rxN_>
zM_P5fEUT9FM~UU(OX#iA;v+T-H?Et>ylB*Q%`smJi&x^vFji!=6LQs?s!e;>q%iq4
z_K*1TN#DPMc3uW-c~i8Q!AVr{qCue(RkQRdtu_@ObR(u!9<CNt?xNNa%*!Mta$>of
z+0Eywh7B2Jx5MlBk^<R%VQ0?tJv9gY-k+euB6<*WJz2!Aj}QDHP%Yz1e$1fT;v8+a
zJ%UU}sxJt!L3=S6AU3EsHf-9vC^S{X-sfe+_qbyl?|*lXcu&Inu3_RW5p%5c%1*|#
zc|8ih*}BZ_`69a$(&sA{hNrwcSx~VS?zKc~6XHZ7^~l9us2irvv{P2{6W)@K*R@k|
z<Fp*=F>sXDz;{lqPQ+8Uu~3;D38tM<)OLl{wGIt_qg3~HBy<re0Rf{LualRlSc8t#
zkzJ?#Cdl!KpDD#+x;D&)m7<=(Yrq$_9qLhH9x8@YgQr*>G~aW-&rmNDl&M&D|JD_}
zHg+$|8nzU+?fgRZ)A6?$@fy{PG_#%iz)^>l6KwRL<lozi6x*ngti{w<!<nnXQ9Eng
zl5h8pan7A>oqVJXB)O^Gh<?*a(dfpWM#SH_Oav#<5!9z+L5t5izBAox_%87?z1>0Z
z%K>}p3*0`6?r|wa(55fWQBx-`RO1NgG&nO}R??%N{N3xECVQO2JQRMDV0WB6y2Mdg
zMQFXrh}P|$&62CvnO^X1u;t9f1WT+CzXXPguwr>&*AN_;a(Cjm-Wdc8&B3aKH<@HP
z>rU}u)s`dPgH);L*}#hOVSddFAaqFT1PEY<0NOQ{M2=8Bqb%^p2A8ztUIKH~9oVi}
z!yd&dUa(ax?F!+YwVuH@-V^;b-!aR>-T!jtw%@XkAxq#;a2d^XMNDPAgIlaq*UcZ#
z#)j{{IlA;KGW^BYWHA8a))l4CDeQCvr(){se}}_zWld9FNtPbYoW^wog@idkfrOF9
zIaWgqtfWHwx-LV}E6;QBQ&#w1JUEw6P_IE>T#4s@<M$(#V!y|V@h`b_FqRMHosdeY
z74wGuA#9*QWcsyp5@E2FZ+PYY9+H$(?fm>D?+PAmS+fG>uTksK0x?9t#ipaUjK@7>
z$rq~zYklfdHTGY0FBy0hRfnmBMHYLEy_Cz$+jq-PJ7z1Ech3{8u{af8wz@kSt5@%l
zR@ryPy{QFOgle0LM{u`5+qMO+3&wHfs>A3b{wWHnXNE4=?3mM?-zZ<H-~1GZ1($n=
z2=|QL{W6pV<U#mlRl>fYY!fR3#GU17;ra)bake%5#!&*ae)R>#4+uvjB3F4J&Z~k~
zD6<eSbZ3FJE|Pki%BYCDdfVnP_=3$%MY1M+gKj3ZT*pgDelI?E)81`opx;;QDq?yk
zbG+=!jr))Cv<453c{^rHl#tJK+$vEty2ILCwkyuqf#2a9aa1B`O~yYsOdn@4^VL9U
zC4UsoR$g4Iwfn%dNx7DU_ugxt@r<KlNAj^mMe5j9iu!b-_FMh2!1zvks&acq-7pUF
z8)083Uo|=tExEmg{I_|EscM{`l9f%N6U%+mQ9cTH#%#TW3-0+$9YVwilzGmh<Eu~_
zyKnk9beffwY}ky);sqm17@assIs<IBf7SQ+ZKC^{JkLl5Qn>TZ-ZJQTDk0FIQCw7-
z+ONF^>c{TB7skSc8TEbC4=sTN&9ojZ;yX)C4{!tg!;M)ngmk+W0aUYGP&gvGy}kW{
zLs<<sl`J1~3vv(NF#LE1%!vgE?(_M`YbrOqTYI6kdFC79PwkY$T>WHEhd-S}=tc&!
zO~Orf)ugMu*_4-iDx!%@Bz*yEpO9sOxe)Ez&L*RM#e5yg{4MgNbF@?=6{h2UqV>WA
zmPHvFloq_)jTt|*SG_DB?D+%Lx<3)rA%9VB>%C8rJRigtA*@_u-scP3yy0?smQJHr
zHNs5x!sT&V=%8uLLemyQ-Phj`K_ajP$rmuCXg?o~bl$6<OXQiI4q4fK7tg$NpS~fd
zQJ0p&c6Ng4eXC4$!@aIcqnsnA{sE)61yfidz*lU`cX4MXotn*2%6{+nB3EH^%Wa-q
zf0~1SXyigzzamZom-EeNreKlk@vqk-(!e@4n|AkrDjCk4H0!8_={&@3-g_(8QWNBa
zCmh90k<-x_3eg*vnaE%l^L-S4&DDF2V9_XmS-Sa`FKmQV3Xe<}_>A7E7^X9&_@J^_
z+H({_sh@$UPO2}GzUM;>o)Fw{2$m&e3Rwb)zNY<ZmhZzPo2E-~BcBNR)f?b%7GHjw
z86Mc{{(>MM73HUx>UNvP>kvoUVM{lsKVmj4qFAg~QplzL5T(mw>mKZcny6HoH13cn
zia{aZ@gT#Z%7$qykBDK3rdNH2Hk`2bT<6kFC{xgDvhEEDz*dxj;j31W&+>-YY2B5Q
z=3R*Y8I}}IxpN@GF_v;{M0%5O_Z#GwbuZw#xAnIn`e)0Z9R+RJr<<D=4l&|3n%Tnw
z{n*Dy@}m@P%WqMu$AQa0!vYxiR8JSDtTYAzue)nnxHp1~hS@elai}DIUr`&s(F%(!
z@prJueJ?+Y3EdMQliE=-^;-GI7M6tj*A|Yb<GU`ZJVo&J4}Ydi1k6ZE1U`?yaS)8L
zH<AQ`Y1tlNVE&{?gE;!zO(b~8K+wN|_nS4$@Tj*wUi)Phf7OtN0>*AF>~j~veSp<h
z5lo6XgrS}l%l9hdb(q3c7%xrMXoT1Lg7?*`2lU5V8<TKkHAu37sUP%~b5sr;s$%n0
z>y6ai-%HspGX0UXY7=?+8~cKskJ9V1wv?}Bgu9)_aA{F|YF=QZ&#PtQW6lraiQJVg
z*<gKMw&&;~-Q9YnKl|YThrYH8YO2OqAx~pSOW4;>ahu_nWCepgHMqyVkua(6>mIQ>
zv&bhkiEr4h>=}NoQ3?5Y+C37GSAv36Z=4T{cykp1myT`kwpZ6rjYa{QuYx8*PX}HS
z_FvK>?gBra)EHa><D+Ce>ot%C5;`8cITkV=vo~OQh=ZoF4q3?tbbuZ1>#w?q^71(l
z={*1|$F?#CpTGNKFxqw?PwB`IN1>{HiqmMIulK9ce8f1m1+;DIhOYfwcbdAgkeLIE
z=$IGFX0Od69^uz<%KUON7yozRtoPT~KTh1t0%4n<#l+V6b_uf9t14E&Whdje9TutX
z%h<)PsdD5u`;x8#ODjC1RvGrz)774sQVvbqx?C&0@8$GS)Ji&oEx*>-)v@5t=%06m
zM~@*%M25Li=)q3PeZhVP8WzkdcOPupjocr`+&*yOalm(TP)cFNI{FeOI{GJt8bb?<
z<R5q|;-UkWYG3dTu*6(6$$?Fb`f$_>G$H+N1S!~PD-d^B9`S_01VknC0c17_ibfU6
z#M4>HKu^x3fvq%*o&y7cGoY;&a?)$mL(C}TQ)A2>tkIVWHq_Pw!S*^#B5y?78?#R^
zy*p#k8%Q4hS(pR7eAVAptYQy!v9T9R3aIi?eNh!}7F!?V_#W>_!bv{;b?Q|1g;gYV
znQ_FrzCcDMneu<`I>Vk~0dDYMqvso=u}d@xK6OC*kr_IP-Q?<`Z=>M%B1MJomXk`Q
zH9IH1xFg1TndX;fUuiE%k(>oKQZLo=EJ_*OSZVMUOf$iVXao#8ueyV)hgUS3L7cJ3
zTf@2ozIdF9yF`@&IWg4oAvJZ(-v43yJYBmTca;h7HDg4=QDa#@pq^CLT)j2PyFw+O
ziF|okfMR6@p_4b!BFbso1LKuVG7LqCv<0)lWe-E%d`Ux&Tsj47qlR}UeCI@f$pl{z
za)orIyZOXj))TKGpGK6unUGsXaU}v_b3Yl4D;#Tklm0lB23<oZc)J+5;0}-d$uX$#
z4a?J_Fhu6<Oogd(ahKNpGer1fV;D=6{`m3ub<9o6cbWL-kJ5a&_9A`@nt4q>O-Fq~
zsdS~P21z4z(sy=iIH!uu@U@uBN_6Di2Ldcqh08=k$dn5$1$pcrB5bA|ru98_n2g#4
zzgm!}zKn-KE3c!#)lSQ}9*)-$=vMvk{kDo-f}Ys_?vGbX1IzAEM^zk)$J<z=b)kvm
zw-lT%@}Kh=fLV4|HJEoE*sI-AvL1;NYxXPUCl;#&o2#dZKz9)FJI({3T&0gXo+FYK
z3m+QRx(Zt_vJ0&I*d=sGj5T0>KVq?L1Q@R^sKRCV0RjC$1C2LDIte4Y*~(3VP0MY>
zfcmLJ5?m@Sco_x^oD_x~-2VcDy=5^$M==%<atH}{i7&FWv;@K68N(nr+&Y2xk?@Zj
z#D!*?^d9yXXK)#>RkdE#vhS4HZ<DxO<7ELQRjxl4vwX2dy?X44++`KORb@05TToFX
z`3P6*`cwHdiavQd{J~UH>)0n64$<**m*I1QvtM<?IlW9pGOg$iH!1BKs)A6bM@aTZ
zZas>><UCpvgB&4V8sSc&pG^DXNiw7EB{fPHzE-tblzxst`(a<RrQ4SxVa$MfG>@v2
ziSNH1w5s>eY{PtWnk9mWkOw@=uqig`R~yTW+$5C7MTE~+y$24i?%UUA#W{yx#bxK!
zG{0${;%#$WN8hBusO@=qYEIFeR>)NDm9VDhp5PyTeQLNHUw_%9MUE30yKkox1{{(o
z#m4}()hoU(s}ASV#5}iw=Qr}d6@$G;-o4qV|5Erdl+^ZNf}if)mrCcQ7h8iugmF43
zQ^QxLcX138_Rg6c^!Ku^H?*2z1njmj$aqaUVd>p(JpXW?)=U)-s%f?T5wsoooFTw0
zK{zOAwTv8RsNoz!I@j{X>~X6-*zXH)-h(A>tVpd=RRM8mS5`mDhHUmrvLXaCUTx_7
zTbA~A<_ux&3e}?Vtf=w>FxDdKbRMN&@SNkn!a^M^NHpV2%iq(C+eV+wHZ*uo<9m_W
zt7pLd>;f5pU4*b1f?f(mDj<ii^mwsZ5~SfcM)(+Dl}iJyZn2~>^aA!kYGwSAm#x|n
zcib=~ExBm9>D!M6$AiwYRA17p#Y3;#Bh2`cUGk;1?GKttQjBmqpOl3awtNi+eEoGI
zmO~U{d02GYZ=~=7xV&ytps-@blZ|*c=$5n(<W|n^w!Dh5f_Iue&OJP=!|QT$jPJ<*
zeU06+0}w$*MvUG@kmFW>h8shH@;4_Ty!YUB@OHMGK#s;_6pMuz4#5zsu@u=~!*$jI
zan^@ioNzDP-T{)C7Z?YBvu#|Ms*t|^ext*TS(TcZLwyetr&5y$j&bzYS+5owtPqXs
zn~X9@>g|CE8l4CrWf_LnX|K(B7;}|&ic>}`H!2G_MB<-E_qc@y7J9Dde{!s(f-!o~
zw+SnR6+D`za{-t%SfK7T|60nXfL?9doF~B91crL4v7;THp0ICSx4t3rLl~mteSD6o
zqU+6GYvIq1fyn}CIz+kIAg_W3<hBa>a%6(Jxjyl^u$$(h&}~BH#M^;%a^xzgsIP=L
zV%Jxr@8A$Y1kE>pQ&30#X;~MhCL`+zEgFlqJLmbH!gDiA7*hZkuwjGJ)6+Y<xX1+z
zuSwA96hJ0_?Ky#BT{w&|Y9rN8_z^g3y<6}RFDzDSx$ov;iTssHURhycmXN}kjzC9Y
zICcF!txFG^#7!SUItay#;r~)uo8Cu_iS}e4)~S@l6Wjbg*DzY{-}JpC`^vGwT^EPK
zxgM&SlE>kq*gbfyCzxz0V%s8%lB3!QIMxc8&1D+3V$qgtwjX8xmI+a{y1PjFf5;Dx
z{k@&965#Nc(eL`6gK2m8kD0w=Nd4NTIlaQmi&9B`UX8?>3jw5rUFt6HF&&O%41H%#
zp;-a@OR<cN{<TS(<4;o!9}tU`ENF<hZPRBM&vISaL-#IA_zddlfy+6&J~Zp1(LeL<
zkB7E0hvAofrFaO)NEjgAYZ*quZ{{~gd$K#Jp-bdw8^@`kb!w?bm#C3wJt#bq-MUk}
zEV?&a!x8?_oUE)m)DF<k*O}}5l{TM})XN1${3J`UUi-R-i<gPquufD$h4)WsDyFs3
zu9fICJ_wb+b}T+&?P}*7=2ghY-j7JaqPXVX5b9FR+m6y5vI!yP52>BZrvGC4Ubt&5
z2%Gg)t5>bs?DrBvNQ@b=ZyFXcLtw@ZgSQ6UnvZYw8K_Smae)gk>dzqg6f?DKWGIAy
z(^Amfb{pR;X3NJxsJ%$NOe^Bxl}95a27BZ@%VL7YEUh1V3_Z_ljcokpgSY3c9`}HD
z7M^%0zTXD{PZ__#?`U7Du}2jbx8QH`JzXEs)0gbL6{H`p=S;5LWn!rE`oBk;LE>`f
zmIp3hjb2rfC9l)uUP_7VPEe3J_I_mS_w_#xO6Y{cI9Ai=l%<#WIGh$T_s!hQ5a#ob
zSdDD2KfR+>@YiLoWfQ%MdZ0N4^~J#rfehIpM_-%(Hz-&nUUJlx`sc2Pzx`(9DD-2!
zkX>M6UOGy0Ui}^e{?B{((XQXOwuX`g7iO`DFlh@BH6CZI-OcZzb6w7llEW}e^7hG1
zUNId<HEch|-H`}TJ8CD8Yn!*uq2>1sd}-hd6~E(+CDT=@dLzO(D7^Uo8OwHcnA6rZ
zaCtH(&h>;|<XiRMC;mpM$b>}Vp>4&5MI^i-C+d!VZXeSEzy_WPPa*laeGTkG5G*y7
zUTJ@P$N>p`${|>^eap*we?_^g6Be^RqmlZO`V;``Z{-p3Wzm8<0@B;4!inE)E(6%H
zupxJ~4l82O=~NmxbUHO@6e0%&w~1DL;@u_;qa12Q9-^&$Y5oe&e8qOcI<x(=NYQbO
zgfanSx*GZvhR4%+AEgxR3!r?;fHTsSjDLo?w_(1%_Y-m^AFDQ&)dKa@A*Q%Q7#{jk
zK;&tFBslDy>$*Ragnt~tF15SL{w%I9xMErc4CvFp?+SmNeEU(W&bbzV4MJVp@`31s
z+{Z`~aL^-iLLe|UAT1h&@H_wjk8lydWgq$n)4_n4ryLT<hxg&M&i#BjF%kq^*-4PC
zNdw5`BP`5eopr!cusOU9{pI8R0=j)Jua{D=$^FFt;+TTJCI9$*sgia{IJ(Z%bKqS*
z^XA`H=5ED|xk`i)21xi5TFO8PR-%;IJAzS|G!1iX=v!V45^S)(J_PezK>%ho-#;h_
zTUqNbfB|WbmL%Z81JYjNL8}ZQw?7BA5yIf2QUn*({<6^!8f+=W3C{@}kzRZ1_q!UY
zim!OKMymU#_cXH6Em)LVYONMr%8);H1%U!Gf-0&AOJfeaTJjzr0WjlE)F-k6Ixu|x
zKg0ks&?bJ>n3w^s(^MsyyA5xzuC1Rk5P@H67_G1O7g*1SYiZSa=kah%+^|=sEKR7`
zMY<g{B=~zkPD-Og=<HlkOz;QOZ?9{~x1PS2P!b}3cGaMo-zi#*8t@j4gNrJ12(WBq
z-+qA*zew!uR|ybuJP=!%$diquRwR~5VkU!wg;fH@)DEWJra<{90hbE_4vo>z`xax4
z&lc)1rB3Eq5FvXA)<ToF&2|^Qa<*hnTM3nNap~?NZdCfGqf%f0B;uD^j!lE65TA_*
z!gHy9PCR4%Z#sapM*l-iAgO^@z9~BR3r|=+e*XU1S&atE6rL}|_pc;mNzq<G2mt7I
zA^U^R6|)Otl-@U}yU3*ShN_0q@*%I?)*sh;>|wZ`GDp#Uw-73-F`06|t{>IOYQ`<c
zzr>_AxU;FMS()%zXDb;*1Du@1B+>NtdFAA-Iy1|AtBbzl@;=!V4(v;9&ac_M5oT<P
zA2q>o@!u>}!g!*qWO(BM(gqbELHv&l0ej5rhkM>Uh4d6?7?@oUqfjeUBHJcrme|&U
zz#^fyzuLu&T(8#Hr%A?w2fh~)G4>qD`Q6<;kh|a%zX_N!o?#xixwTF~ja+9uNUQ2B
zB#9)%FM9lICUG+ocY4=h^h&paz{B<S!^<~D9cfb*{mvn1EWT*unsNHkfvQU(-9<mK
ztZl<c{u^JXfa2R%;lY$(k*LalF-@SJK*^&!F@q!06;L<9@|Osp@&7yU=vIlBz7G%x
zbjG8iH!m<O$J*@wyl8d33$u%(ld{F296R{n|CRJ~Wsl*E*YN{qfqgT|WNWA=7BOrv
z?PvJ0OLCL$xt#7WVHghif-YiO2m4?AZrGpCbieUhukVY)mE4!}_WJMu#W8g%Vg{jM
z*qHx=8^Zj*a6?Aqzkw8kTF-k|P=JO0^6>X;BYfz1BQe+ut7{oe*`yuTtB<->bt}tG
zmx6JOit58h;+hB;0kOZhJc$pJo2hgVFnpXyChnGUc2?_tPkl6XqWe{uvgE)uS7*(t
z*yQ%`D&FE<qPffWtUl$}`|%yYn_#^(M_y1qQ5kj-O%2AXKTV)flx77$C_qPYLZJ}G
z6R)6h5-$Q5lMgP_2c>~H+FgL?T&#4!$V9)sMpoM<Vwc!bd0P12h{B!Uf5Qrk@V6i9
zwT~wsKBj~_b@)!4!z<<r&E8CNM!a$bf4H;y0}%njm9c)32O0GLvd_*jlHMA>_;S3I
zUi_{^0#s_TU-B{hSPnt*RTqC(WVVlO!cgxddp)x~F^bel?iYRW!?K9f_HhFFtliyD
zp!HS=A89Mw!KFqew=b)`am2dimWWp}(t5D~5=@^lyrDHtaQO7gh7ZXzR6zZ%A?~&6
ztG{Qbu^XPd#X(GHf-Z(KiM($p6^!UBt5fscaLYp-m-#%%mO8gFJ&^=m0@#`!Fmk0t
zf>kId)8C-BdH$6nWd18IMB3{1kYd;sPPu?+Oq&*%-4j$!U(CyaMNyqdFLj`@JF`28
z#qh~NhTK*modHHX3`<%=QsA1VTN$CjR<Q{udSZK!@w#vJ`n)?wXw{KUq0>2z-Y@Vh
zW~qvum!i$%hOVK_`PzR;z)KvWd~$bHRH$~BdV&bFD?B!XA2(CGMq|?6;`Gt33A*3j
z#dB$1PToca9dkEF4(NU_)Ond{AO@~RJRiyWrzj#EgdQ^!hUx3iKBWW?^ievZX%&Pj
zxJ~`S9v*6&QmIm3m<Le5%09HFfP(2OEim1EI*GuFDQK*>EX@4%CFtWvuRPBD<p+%j
z#shvC%OFnNsT4S$_qqX5%cm<za0^lwnk<DXy*+A;qU5a{@zjb|x06GR0v89K-fvUa
zD5a(31Fv#*+Hq$Fw_{^duO_qGdsIG2!xqk!Lk%CYh=eJ=Kk?$%R*6@Tk9&Xdx8MF1
zXi(nDr%Elpua>)&!xzq2HJDQti;LObYMtcVg527y(iTGbk||mx%9Nu8mq@GaJqCr<
ze*B;o8_AcXJ^68?-l@TM*K0O;3qh%kaKT1U5_(1R1)(b0yO|4>(y#cgU*f6F;evB4
z94GeBS$9%JG^2ipVl$ykA1?bO#TK#U+G&&V^vy~T>c}urLYXLBt%}S=oyX?Oi+dfX
zP-m{OhCyx(@{=7v`gsOhOZudkZ2vdKlm^SF+a%_4wvh#kh%p9|6KDa;Y=m1WY~bns
z54j!WeI+ryD|1K1*AAE2u7Vhn%0;%wW$SZSu`m@r9!Gk*Kg*PIzpd2NaGc~EDrkMP
zhQ(y|`Yh4$na!9f@D)+ce?J-#IN%zJ$?b;U@@<AH$C)aBtvm&83YqD}dv@u0xM+3G
z=K@i1h%S=C8->agt81JrTjy>MIkP@YVlFt?aT4dK)=T4B{e2_5_a*gRJ)R2OGT7O3
zh-)E6578Bd(gWI64M#cx4GoVC6fM_i$DLBKtgMt#0>>Hc3aGi7@a>)tSO<SAQbqb%
zpC5l;X_6849d*H#`g_OSRgSRr=6GITIs3MATt_UiQzojx{)kTAbVA!OMd)BfDvpQy
zMwVCq2>b(qg5$ZJ)`i50?csca689f#@W;fBmB)p143l<pdRpw3DJy8&^Zn?o3YKJ=
zHVY^b%RX4&A)1N7>jE%BpFulPg0+y{d<J3Uvb(n+6_*CDK~BL+&p#<}r!ZfO{Jdi^
zPA)DefX}x#k%t7bAcmzSgAE$_AQ3FvP~1QGq6OcH(VkygolD3ZCD`rt)R}d<&-0tq
zadA>gS*BL|e^=C|0(}pWB$S``^GBv_G||7kK1gADd2`18kT%g1T28_*DLs*&ON`Wu
z)-1Eub>#WUiAZKPA}v}6R@!>}c*-mE0}e$4eY^3p#V=<AuI=+gC`B{NpTvd*l2XsD
zEqt{e<JJz{-EV~zTUG>IO!_v|?{A{$q??7!FHv43Fuctr0TT9Oa!NZQ7ei|s>A7#@
z51jnUFDZWNlWMZpSn>kIy1wRLG3lwMi6M41Y>-6W9ZmFUrFWa1J_pupWyTq0&ZxmO
zFWRxU!Kv@|`?fFq#lK48(c%)2Q>3^lX5Z;<M=%!$zU;-YQ5S}O&d@6WpLOZDv-586
z&#e1=(?BSn3gwG3uJUG5A7pLCi6k$MiJABhaa_uTxM^9Ta?(hC2{LS&?eCy#jBHD%
z9sPF$<S$98!=Q=VbKxeNR48^IkEJ5WS6DB?c7_TP6Fqppk{ak`m?8u9!~sp%1<FHB
zU2ODY^4PAyVl!GEV03>cRw96<rvQeph|)Z^k*ICOU7bHcw&5k+Bni1I{+F2wXYzio
z`<~oEDo3BL@+i7GYpFWE-#gRBvl2_2+7qcJOht==FtlEqDoy(p#DY1(#;Kw99An$G
z1thZG$iIGCI8bneMe^0k9W+I~8T5rcX2bjLRiyx6<H4igYxg;y?l@*~m7AEO$ydT6
z8o#<a#>@|CPex$SF|#ILlC)yT2xmuMWvVrqA-C=m^7?HL1nVkIA5E$|@t49PE<TeZ
zyyiH^vSs7S7tf5xl#fFgh=D^Cc%xLuR+r8tq{G$Zh+9RfT<D5#&QB)Qhgj^BX&?-=
zxTX@Sp2rcO%RJ0Fvw7TIe3fwJq^$VxrE?aRS*sb1M$7hDI+sVRow1_mEr$!>x||+N
zFunriZ~^i`7l;6167pjAYJH}TvYH>N*|w(Lw_e-@z{FIgf~^)uGp~4rCE@aj{HdwH
z=O;iUWtrCjLOt>Qk-_g)2>iA<N?Dw<^K(^YOmX2K|6msv{@t1j8tHLR@RG4EvHxNG
z^CM#+JAC4nMcN$m@N@H@Y!$OoJk9G-LEArUi8i`yY3HUfkMU0U>&HKySiC>jd24~U
z8K?8MUmAzvdia)B<NM&RCdGE6iHNT=!U0E1gH$_**HK8nM59(3UJb1jY?KF!N1ZoE
zN<lSJ*2^MYEHZ*f*U!ejb%WIa7&Y8Nm5_ObS2QL_^(qGFq9UB$kAf@SQ`UMdF^)WU
zQHJWJ`8gI&Ds;4HSKi^1$f0{9#=^e37RZ7PVqI{nq4XR#MD3)|F($Mz8XCm#2BJXb
zb2g|<tw-}t8GCFblAt0?-pFUV@!Hcv|KE)#Z#}~%-$E~uYmC?6S{Hqq7J+a%V&gN~
zoh3KT|BeAZV;sMQIhjnZRDOHt$Vc&yJYi2RDJzJBVI&cS7?pj85yQ34mTs1glf#Jl
zT~|noI7m!o;<j~1RIKDi)9QKdvo}Q}SF&}LPoqpKljLu_E%&rCj%})@DByTdK|?6+
z01Ft8A`|WV{rh+RcO`sSO9(Zc%KcBFyW_S&$}DT&=aFw^HD9CWDs}CByW&L)OkD#z
zvYj7fC~38Z@B-fL9(}dIa=s$lq*L|h870haZrjWhueI-aVf&%>`|e6#A_^gO#I2I5
zolow|FrpOFtOeU2Jsuoq1XLhAu~cSzvx3M|YZ=N`vHx1Y*Pt1IhU)}ZF>A^?n2Z&C
ziHc+F1PQzpbLlhfgB+7(s)-aD?~cD1#h}t-Q*aSW2GU-aM>lf+riMi{X(k_G%}_P5
zRxUDMy4W1#+?#89J)AVQl#6$I@r$6zgGp)nhiZWwmX)qxD?oA?_1s9eQkk51ApFnq
z1weVBSb8guC&*zuoHnu3RAU@h%#-xM>zUqETShNNZNF}$oGlmLzB_OUAIBg|$Kzy5
zC?1K^tz)~voAk6W=T8e0-p!y}vT4Pr)5hf3(!aeW5Q;S<2f~5OPQUWbVKQ2>Z&5Rp
zKkjZ8EWh~pmU|Y$cSGIH%dI7rrZKkL2AO`8<}|Hhm!SzVyd){?%n!e%GAKeZ=mpw6
zu7UxApOP@NN2S+U5bzHq!qj<oj19QtbkJaTWrN`VIH)PP11dbk(Z=pFX}$~3W=7D2
z7{PTL2kO?#zL&M0Lzd$iSr!zHm|!(mv9--Mql)aR&;(g=M@0E?Bd6ovU(9Nv%L*l;
zcM5Mtgo__QHA<O6bl3_JeA*lwc60VyIY6mtHmS&#Z$lDOwZ-Tz7;4|lZ;-|axyIji
zyq8?a2XVr<@1SRKbdAbg&;^@~z0%6(Bbrj!Co`$KBeqM2dxZ;rAtIHHOg!E<Lr=gt
zM)RUdEvj%nk9K-zNu>5G2`Ee9>wmt8X@ZB2RN4IoHG93lN~yyge`oF}`qOd@2%C}A
z9kZxTS8B5}kh`mu6z3|h0)pdYH0CVj`lNjoFX{V<C|&fb)ePk8G)9v1ytH3@VG`<P
zt5Wk-+i7#Qm4J?xvp<qxP(<3X4hceFU$-)OJlr8Owxk_MJaIS`A^9f1lyru*7@T5j
zOqrVYu9O7WZ(VK|)Bv41`PY8AyTL_n_y%RSY7hw%PHR*B7ziV}){^2B_M(_6cr?$l
zy$2Lv25buFAz5{zY@GD`pw-9b&j|6189$Tr*J6yupR!nDMv<uCpjTsl0K~XRIY&Yb
zghqKh@0=UbV4fd}KunUGVZJ(JL52eY3M{<mY6<Wr3Q-L%XRqbWPZMco+~cSzWsDyp
z6OvBXlmN7_No6)$8ei+OCbC++t4JH=C`Fq!RbG8K28;MO$Y~~b|E=D|l@XuXALi$u
z=@%GRxkTugDXuRP#k==4vLu#tY3-&~>O~Tr<-e^H{Ly5P3M-zBC;reZe?u4l?N6-V
zOI^OLN;HbOyonvx#hp|vowVLby8}IupeTGRJl@1VBYGgs<T=KOAmH(y6BLf0Bv#x<
zrqn$KW=lqjn4m=D5ypfP+nb$7Osf3YcX3S=l(t%$BVjLawlxaz8i3~Kc?UW^U9Rua
zRPdf%AKe$kk?CoSZ=Uj#7fEFekvSe8`NIB8W-}tVd{r4`*c+n2IjWGgA-x{h_Y#*#
z^a(**6^&9t912)7pD}Mt1rdL=3b(X2*>Wy!j+X^$v=}N)Y#p|<>P!bjj7xX8e)Z?5
zPPi0!Cie-Lri)waYNXcBdFs4se;*}Oo#z;T=YG<dpOV+1!71}YoGN}7%3-}E7fY>}
zZ?#x?KhkkMo({<o2@oUn^i;L$Wi4{i8@KK?^HR+?>kjMDSBSd4ID)xk(9wg<_>}Wy
zHDxnwFJ(~a)qLQ=d!b-{m83W3aU?0E9V~CS9N`MaCDoPE#Cm=gf`uXIk(Ri2v7i;{
z3SwKm;32<K`;LG?n3$zkCDF+cvuXuRS#U}{tMbyU-uLZk1Tu(s7>;_&cksT6GrrbD
zxj2j_C%@Da96qUDy2)s?5OWzJ^wbYl9<xEfaBb+kM}!Tw@j^NjfWpD7#@SOqC2swe
zkzO*{6<wxwg$|<Fl%w+sQ@<-~nWoD>XnrAxD1IjF+ZBN-tHaTM<-a^IqEpGc*mmnb
zf2SO=CBI9r*J%P0ZUAs-r+`MG23=F#z&~S-qMyd>@rK3spCO@oJ2nIDa9wytZTlp;
zxPH+_``juQW;BW*;^!^jY8!cxI2F$e9J2r;W_}18If?oSckASH&;jW}HiSNjOnNG3
z8~x(XX1u+h$)vDdBO~;NL-YH1`vU(GF07J!<J%C7*Q1}O`=bI}Nn#cUrmwH?V2+E!
zJxciJIlng0>f{pMA-kHvd@!-AKrwITV(5`&ljdCzm%cqLlE*L|jG@NuIchQEeGOZ`
zFKj4`Wo{%fZ~lkr_qu^&-5$OErx_KgGRJytS)Sbl!Lkg!7kQm^1F3ZRgW7_hP!6>#
z6%a6TB1?0Dr?UqbbQckRy94b7B8z0>=I(-j%K{#}c1Dl6MeLi9F%RobFSC88eIN1E
zp6?vhm}BIPrpnJA*E8vvX*K(FWdn*}*H$o6t}bl}jr4rnMvX;VIkGCe%&eL~^R{XX
z+uw&#+$$1AYL26B6hzgNP6Gfl**H(K5uZ&=2L>&F+jRK*18=oj`g0<<O$9ygau?wl
z$JpB?Ra#!Wm~L?RyYTrheRo3;26s$<I&vg&`&299(8Ji~O4>pGZY~zN-=tB$hV##v
zwl!x!X^bP+KCwcJR15a^vavmV^qR8>geADVx_6WA$^qCOeEggA(6eo6>$eJl)sNav
z8Dqh$oZSt}t&jBKOf0ta054&$y*T`hI^VWkYSY}+Oz>huUUBH_x=0~9MMGu>x%$M(
z>3)}X4bw3r+dLfRyr#|K!&CMOAsr-tu$0~^vTj}l@a<Y|7GIz>({O$0Y!gikGE&B3
zN{a_9Z9nJFH^<lPJXDQ0g9+U62tkBCxVC;>jLXi8o*36D0`7NmyOUJ$ZANcXkgH2e
zNA@Xtk5`F^>M2+*4x1xYPBtSQc8djd6X!SC>+Dly*mOgjKGLNFS_aqA&-29N!oph2
z@{i+|Si^EJ3EdDe=wDUcwn12s0QN6Z`PGuu)Un2eYQnUDe(rj9!G7ApY9dEAW4?v0
zwdOtb+c7hbSH+S#Vpgp|pK4W0DSVKsF|zvcJj?G<H;a#Za^mY_&)~1?DR1vjklZxn
zBb*diC^RK`NWBdJmQok1<h?kYs}lX<q4o@_851M`Ipxl<HT<w=OJ8z~dMyzmRz@i-
zf&zW$wMhjail8~?(f&A6Tgyd8EgqZfh7YAyl*NLTtm2W_qG{)h{-0G7f)-Jyo7r()
z21o{<%Vz_FHl#GbuUV;IVwy%}qi^l*Px6~=t6vFqv`)XW2h3mpf$jaa(q(<;6y@kh
zA%}iJa`E#`{djTK<W3^i$8x(~36+E383+g>TJjwes-7E1?3UJ#764=y_rIN*yv8GA
z@?|LxOA#D}Z4E}1Qx-oL7JjJLgQr_=D(8Bj#RrMyNy32%*67>8D%1s8Znz;vH@(|e
zB>go?N;i16hRVIW0z8~5K2F4AQ{Q2BH`fb9=uhUE<*l#2+EGjXO88QkF~u1wPywSi
zhEZ|a<x{}@l}d`nDcnJiORLL&#nn2aHCHo+mm%-ANV|Y(vP<yqiE*%j%^k{vsP+Cv
zca#1GU!rZyiqRgbLOE}#E4<;AVnDON^{be3Nd6k%Rp)z&w>EPxUhOt#RDYC__AGd)
zGS>v=4<SvX`efBwnVWF|Z4BkNUylyCui}yDYPxL!IqG6`=x{B`h<&P5OG+aE`6Ha?
zqwHJ#8`YGl;N1yjy%5mD?>2b(?xOFPw8qQ=R6F{BbpTx~KWwnu!)Rd>mPzMWq+3jv
zt=UB3%v7zXv~l8E60)|7>>u<1hMZ6piMkLt)6*eATIEizd`8Tx_ka&i-exSj8RPhq
zj~a(Z1j)S2G6}{BY)RFR>2tg?c&%}>wg=9Tl-JT^nY6{3DG?%U*n=u{Z91vGHObZ#
z@!bFlInZ&m0Ql1wdL};a{ux&HdpsWH#q)eKn+XDZiJ?0l9EG}aUDzSmtSKMb+&lNJ
z(gk}a&T)@7=vx&({B}7kQDbBpzBv5mbQnsZ#c8@JtL>OEGZ=|a{_2aF#(V2DJ1CIK
z*?gt<ZA*plc}DYFW4X+<8KzzGu)6_wfvsno(*)rYjeKe!BIyy2y}nn7fP#l9_w85Z
zAX&@yw{(osx$ygUrH7N-hZrtdaCPeHaYkZQHs>t*_zN7gt8+J+PpK*C$5@b*)yfbd
zP#13(lxVB{*pKmd9d&S7;(IZx8Z?wF0Ng75W44TmZx*qin>u!M^=xk7^X~+H>~nE_
zR31Bm=|{W2`AHq?3yZ4Yd7BYQE)gOXtXdHD@^*P=dfjC0GbIlGoD>Owd>ON4TizG}
zcZes)Rc#m5&!0AS97Oknos?`q2p8_=0ObV*gGQW_RNwYpGQEen;ZH$=Iqtmv<mA7l
zDTSC^V)-4B_eisoP2<8)tuFQAopI8;hvyov9ifj6R~BE|{k{0ZA6|S*-RgWT_VHA~
zk0K_u$SubCRJo~#u|Sh=rK?leqatE0$%_UKkvn!!GS)wpy8f>FC=NjOu>dwOT(HA`
z_=v7=d4JTkhE{OhkD*)&dm7|aZX**-8;+$vFQ_K$D^8g?@Hyz1r6AW#b$oU-^K0d}
zp#uq>O0#d@Hc1NsXp=7bdS#10y4O%KWXNpyVUs{4&2W4(4~r|Ux5uh!PB-zbyw!9j
zS7pPH-=bXAPDfF1Hj8gfHDAQf1aCl1@_q4A%izpr%7P_h#+q9gaDQHN=xD}Hy})47
zYc1&9Z1t~7^9!0OQclX2eIDfz$cVJA0m;Xi%RrZa%KarZnhn!VG}w$^o{>bc_Ua_*
zy_$g^kypjx@*QV9EodKseN^Gr{%z6lSkOHmLI$!RY>53GjY+WVOh53kg!FWE9hdHd
z8L4iIUFO_%kH^W~1Haw7AZp8N?uKbeS3l0sYHeE`KoW(e*){oBEdVp8pQ0?do<@Hj
z{7gWugR$XK+TJXkj0KKsh|wnMl8Z`H3MUc&F17PXT7y8l+0gx&>xgtXX~*Vzu9d<N
z6JN9Is!M~+#LzcmWYW^*870+sy~GxEfY=A$*Ic5-K`Sb$pi^+3cofWX&NH}-&-9@c
zlNZ`q-tAVus)6dU??W#2>HMID?eksv$J7{v0qf=VC`WpmJ&HrCBm+aIpCI#rg2j-q
zs`3Z#TTPlR#~EvWvKi+8!`@p)Ro#8<!mw$iyV-PuC@I|?(nyDdgoJdLfJh_KjdUZT
zq%<NR4bq`VOG)Fqe)xZ$_k1|d7-x+0^?bW;H+${1=A3KJwbq>1HRJp~qXsTn*s#Cf
zeh>?Qhpf}$Bux*CRM<VA48Xpm)Hy}un;FjNNp&6AAP|?W$UR>QBx)fl*Zsm%)gTK3
zcYc3JP0xr(+)yWd<AcL9nfk9z#n;|i@U30?m;Dz*XTLhEW_q6tJvD6H2VlSY456Qk
zL_W&in7m;q6t~W+2NP{*JlhzG&%mu7CAmiDn~L&2)y4vPlOFj2`&`G<|3-07;{QH7
zgJ6x1Uz72&-oFoy@o`nvh$*OuTu{+!Xt}Oa{h^QPXyRI|=f@Cha?$X6CBw-im|n8)
zJQZCG1*{!&Kc(E7d`cH@#kedCJ-(>AGJcAwJ)Bb&HUHqvtmKFY)`;U@eX_0}4|ZRc
z8MJ-TDb^-rI8&^Le_m6Vun@D~8%Xc^#WORk31@jvwr9WbOtHQCTD18E;a*z1@rQfi
zHHVphLB4<p_PU<)KsjwbfNZhlh0Jc}ibE@9k#|_RPKySmJypZ64&C`*XEyw&+krPH
z>As|;3_>2660h`dbJbfPN~<3xF&FsWi2VBaEUD&@<=}zAH`~|fzFeCPteP_4JqQGq
zU>V~!lBJ=@Gghniag9S^CXR#A2al&l)XJ1MncY&)tsh6_MrM|9D~GfOy%HsPjj1H9
z)*WJEw^>QtXgfl|4n6o$l<+8G)r?6+G5TWP{vE8niJ?<WgAu;;0VN1{?a7|m6$a1O
z7BQ(__;&qH)r-}t0(rsx2G^)%s;7s+UVJW=pF^-8Ur(2Oq^7)<w?F(67H)LU=j(M-
z&^|%i*74@^jf>5{cGtBeHk%eISkVkEFSY-0mc@RzA=Pw5UZbXLb(cv$|DF-sALSKD
zweUzbLm$&M-RlM4&hPl<R8d2JUl$+dL&~}@J@P^V_vQFp;s@5gq|8aR<eDuQN@5WH
z`YRFiYxe!gS*dP8*_&hE$AzDIJ%*1cYpMN)2?D#oB`6*91tI--@REmpkNM33z6qL;
z3+G)D>MM92;Gfi)s_TDJrI!#z!T-R?Of2}z-<(VX*3xM>t4*dOqki=bsEsA+^Y+o=
z6#f0ZXu)tONalLX;(@D+v-8zWck!bcU$-{)Q-!F5U<kf2{80O3TEa){>5S|`u0@8Z
zgUIQx4#~;6)&dFL8XZY}>+6)^$r-<B1KUwrni3Z8p-@?<m~=m4>(iqzVSFDSVI-RG
zCj6ZI;HS38L_&<2dJ$ug!m7(D;$%9eQ8{lMY+_bN$#@&be|iId%S*&=@S2TbN`(7k
z&zBs2eiJd8MiG;lvKMf7WjqNI>e&b30By{{-EgmTB&WgtPgmQTG4L?-3a=`qxNDng
z$R$C|qrPXfmES+k0E>f`@IN4hG)X`v=F8{f<MSAzO1L=s@vNdy6x|NIMurO*JK3Fr
zoXm*<0n6-De9A9VjDZa~%dditPl~;hC8AGA#A8rX8J9v17bGOnDO#^2JwIEQS8FAt
zxjk)R#JNr~<`Khdp`6vsWz~J4Jth5hJsWqlI0!?F5&!h*%;R_LhjEc0{Kc5heO0=M
zoXRZWk<E&3jwVbiFCSXZCIiV-7ViEm??_q2=*<Bue|$W}K_*v)y|%|q!(Ev7=&8^c
z>Kc)lo_vLqGJkn+#2dqgTfOuy?a)%CMN{--pXK1wiV(?HEVgR`kcSUE$R=Uf6x^~Z
zzLaT=er9L=cp`V4mH3IKgXik!kcYzLpG?+7J+V^?6?y2kQH^uCur9J{RH4__T2}|Q
zq?7FK=h^j%(I|1oa}$cL_vJHB2}hh}=@nV~&9`U2t(^Q>WO<d_RGfCNMc5wT<xKhG
zd=kM+#2Ojt+xUxZ@_`dfimY0*(}iBYm<5^nNKA&bdPPvW7VgI?5O?U&-6O(t?qvHO
z6kf(J_n5Q{Ow<}?MgB{leno;Tm$TZc=KYf@c%saMmu441RzfkKTN`30-gXj8Yp!^s
zJx<~y@nJcOEdZ~C1pRb3D<%YvMVmb>3jmtV6nB9ZGfwp)v-hcX*CqgSl|9acuU}tu
zuY1UQe+=<2zErN`-hT{G$#(l2c?aYd6qMWUN;^OKJ0iP=wfm=uf2aVk$EVkLI9MGV
zbGq6;e7hsi-7aQj+b&!lW_o3%E)d4yyLy=}n-NJ5UB0#9o>j~C8^vFC&!yp+A#|RV
z(*__Q*@!?fhY0|P+R{%Qsf?!kJZcDFMG@v=uGSpeb00rFZW*X$_*Ycn*(zm+37(tT
zVLrz{ivi=nI7ILO`n2i24^f7jdx5Hl-MC3=3O*vn1&fh126Q_J$e?H4ZQHUN7`gb{
zrM~w(Pd;I`_LOnTNql5?zD|@R?yS?5_G?@IV9UK%dQ@;&NPR_u*MUUzAN74oC+obm
zKPiJGQuMwjtVFz_b-y7b>R3cR{PTXIP%vM~TP?U<VDi*`*DQ3;O``8cm5!=@m)F>B
z$+9?0!k>>f8{PLL%Z<fzs|`s64XiC?^4Vq>jcawLK9`3*aTgO_Ve8;(^C}dyrHkPI
z=Zi4@Tch7eq@=e-6hKKP44+VdsK7xn6q)i9Qt>P%ZP7v2Jh*L^k8`7|m*Bom3^lgJ
z>%R{3(q$Z+)NAY)S@iDDd~mogyhPr))k#7tHshaLY=Uwf(C0vVpmlEcM*2(nTC+p$
z$DTT3tVKBpqQh^&KDg7G59j@d1Q&jq9R9{3snase8%@s9UhU_L&1*jV+4<Vn{v*py
zyKB*2io1!Pi-+FAP|0KSb@C9K2}X{v&`Zn>Y#Y|Mb@B--pE&RH${ci1e8a1G>E?bu
z8TyYhg=?SaJGhqCrUd8a(mowZja}=H*8n^*ZzfMzQ}yJ8@xDPs3{DrH>~`=NjZgY4
zpIAtP>_?ip*S5O0wSxwP)?8^nwf_|?uNSKu3#UV~d0MCeFj%U8KDgcBAHLjW-KrYB
z9iPvAN6^x_;!WTfyxGGYejfY%TM{VC|7e0WRTMlW@+a$|+aRySHp%uw?>g`9_dbQ=
z%#nRC>?>+pQn=$UEoRSLUiVvPP=}s?jx!`r-)+sRSlj%W9*xDlMYoce(fTlanNt9Z
z(LsniOC;*9Zs{csR8LaAJ!@6vSp@&wvO0U-A?-HW)AHFM<-Y36P-L@FlLhUbnbk!S
zPB&KVlBEXw=}<9CQmO-im020LH8SCP%^^$Kg7C}g-R6a{WI@@i0?Ur4`GqZE;n*W@
zL9Kn_+4hq3$OC17fI&Z;*Gx#T@~bs!|MKIQVru<$#1QUL<RE{#SF2ijn@Nv=@8xb4
zo|UXcFRsiTl1dKt{A+QM(i)pG8ub$fu$a-~jYexd!OOz@t%IsOn%@&PuqTy($;Yvs
z%MQZ7pu#{yREFozpMwTa{KL5t%P&(k!Cg-kY^dyzPYUj!40&DLDm4ypD0J~=b!+on
zqe^rPF1J5L$q*|m)RoZ(Kl1lC0CyWr;>O@}DyJ#$)`-*)`6n+k{+O&#!9JqK_r=0e
zz$Ex$fcx%e-q)RP<*?xlvah*vuda))FiEwIt3j=?mihJuNZ=Y<5Lp#cvB*eSUgb3C
zv{@wJ$9wVqc1G>C27ZrIMZ}lA?Z?-wbipJ-ygRO*Mu@9NO_$oQaw?91@tiBjGq-04
zrRC90n=B4n^NR=g+G~f&`kxLEv_pHal)`^xBT_wItD$*cDl_z|>kT1VxlKr_SB#4>
zRa|_bs<M~*7k}-3Wqlq0MG~n3&AGrAATg%BNcMv(Ye^Rw{Vfj!y?Lce!P?$0^bWOV
zhNg@F<4nDkA!1R=^#KthDz=#t19K(ejJTL#HOtp1)h&Po?-`ztXH&`FFNjX?Y#j(#
znUMdF_*>aZLD7IYVF*MR1;}CS`{Sr}ij|2oze_*GtK|R>#Gbud+_Bl;>vKu|P^(aF
zS{(hT+JE5ml@Jsd0)Q)F<?5%$7qJt{KaR$$J;Ew8Vko36d%{oo?9MS)aYY($$4k`Y
zx5ml(t1Lc{fOJUF#HRB-L6Tj9uvk+ncnbhJ(v5sh1iohallKVz`dnLJEs^Tf>|(O0
z0nT`auodQo&bg8;66FEoT!ZU<VxBBruf2<Vo(|*kS(KOrt?uDr%WhBj)-|4uoqM@f
zUR!{Z#_M%oKGOEyVL*@YC&UU3|4v2U0BOtruc9wA6w+6?CM3?=w~GQPfVrSBlcOKw
zi)nKSRNe3!W!c_n&^=+x`<};N{cydndu$6QTNH0_0F<OelquA5{x#)y<5A+2dMet6
ziVjfBy<P8w&zRwrB@JQqUT-B&WCexFJvCW>6$Y|5FIvcFQHWv$q>%>zyeLmOT~vs9
z5kPFqMrhY%oq&;n@H?5m$>|!46lD7)!FWvNTRza3>7bfU@1t&6r03V9+6zKI*GI@b
zqOeZ?)o_${;Fdy69JCORMIm4r(FEM%TY$FBWyYGe(yrGOWM5`uij3YLpB|I?QlA?m
zx5rWOdpk54Np)IFb*e)jV-(%vfA}8|@BMwz!%T+<BT{w~x(tfDDgMDjx8R_W%}(x`
z6fxa8cKV;^&Tul<{cl3$%Wk1j-3ikS7$%|*=wumQF`z_)eBk3mOjFD2XG;Q4XH<bl
zA?8+BMajWYN?4(0;MEo&><R%Om=muKRQc;ld`7*>tN4tjD}_c6kd``ZYIxi|;3E1j
zD#8ByguDl`_aDX6U&-KsOhX+!kYnDg+36WL)--~h^dy(xU720$eghr>z1#y5uQ?_i
zDkZ*~ldXzv6JwBaNVah-G({r8JWN|ul#3=9EVwFvja29FjQdQhrpUA{0U6_|KRrnc
zN9u5){#y+&r42I5-9Yx0OUFWiL<`%08)Vl%`5P6;p!6d9`}^AsB+yM(n}}pQWcd}#
z8N*QsN4HZ09%?##O#H6aPZ0!<DDQ=%HGsX7<wW(2|ID2^>yiAuNIt=wlxFubassbL
zUTfv+W%(x&-|={5{bQ28%l}}Kqp<(>vSZ`PdkdIzrA<#cW>USSlbimtcDa_mIAH9E
z_7SmoOwkDo<7t8}4NLq?QqQU7O-JdsCLiGpk&g|g{6Dy(sI@eA*k9#M8{d~{ukM;6
z?0Dl3s4>PzUYjG4k9J6Ib;z#hXj?{_D!fXjh?O5}TC|P8`}}+Z6o5;)5=UCdOM;S>
zFN5c*SZ??~|B*=P%XEVKKLc_!!XT4&`7U+O<fW4HnLK6lqBg~Z#vX0e2kj6JHo2YW
zqk7v_X8DI}WCT<yp;ofsPSs;a8u9TG=pi!Bw0ODVFIR>sf5%%Q%ChE<&L>AcXcz~K
zt3Atx*NerfyQ%SBy)M3IE*QOW#3o*b7h=;t<K6h4zq<884$??1rD|Mv-P+xstdwFi
zF>y!Z40q2WAW0GOMD9<J98dcf>SLc%0HP?o`zpa+3ctCd0sojbXeu<E+KqyTP6!pY
zBa1vjfe5J5p|_p;c7tB5j^aih>GHd|B0)JP^iieb!A^rwljsBsG%XqCZwa#i>_aB|
ziT-$-?iX0_*?_Lr>(K3<5T!TK0g>+JKSa)8Wu?|GkLul<NA538I3V6UbZ=pG=1yx^
z&;#h+U%f+KT{RaO5jiQ0uEws>NYX4S1i48}*f@C4V&I=+|A0!MKGpB6-vJ2@@9CbN
zHnQ~(EKgy6?+xW=PdZW(?<ajU7yRV08BLVNk5h+`01xCG>w8ZbUz)U`wQkXQ&sE&X
zU&PWvD&6}G{eXV8=Q%6xzL5smsnEVW$bVQ0Vwg4PB&_&;R5@YxeDmJfmDchto1(h(
zorcRSZuFo$V^j5193G>$FUI)$f|=*y=!nN%LF*oUCr~NE6e>gS<<JjR^53JT%`zxM
zC+kDfm8_2$D`o>aH|<3Uv_nM6S<UueF!#O$coh}rI8Zk6$@8=;DZk^MYu+>URvR4Z
z7QG&$l>_Rh^>PX3GJ()(J*M1FE5Rv;Fvx4LVDDuWOEDQSWq57n3zK3ZVkomVujjM%
zv!{iD;zM>(Okxk#@rG&z)dO7zp(M{;VU`7A=O?#se<ho73b~xWYIJZBJ#71FT?Vi7
z{?Rd6f2;!_GME?H8LIYLEYmx<EPt4w+A6zmCHmjEm0R9}1CulB7#`Tmm@q2A&~*hy
z^{g}+i9pVqReVD^Cn6L?H0bFZVc-|+j`|JCS>(iun5_yvX;4IUi6b;0u0{c1SW1^!
z*52E4Vb`p10NPIc+iknHr#fO4wD4g>eO1Qf*-QZHuu;7knbWK;AsX`=!Wd#IkyY><
zCvD#BKdHZ2VZ_Z46nA4wczSHeu0Cg%BBJmFpEV-_$20Tu{=*IM@b?+D0JX~B9^`Zj
zlmrY8n`y-Y*<kH%%=h58Z*8_;Kh6^LTQ%LD<rq*9YC%|?C37^lw4!d|uy{*xi<=ws
zaew~gLr%QsPosh_fcheQUO5fkG^ybK>F1dyI-&oFh%P5S<1992|IUgDtLRH4BmZV4
zMi%%T599*iBt*86adnt}W=dM4+;Qc$#ve`CznDl^!*<544ke*qamGb=*5Rjo&J`YT
z1AH1ir+^;p#g1-IIJO+Zvtpx9iS0Y~E49Q0oRrN&sbAy}<IOO5`Q%SZ>Fpk(wf~$I
zj%>m$2GE=@GWfagGh>Uv!{^oyHVq*Xq?p3pzQOy0)m1{dGiSSoS}_L#x6Ugck6Y*7
zn*nO9PvyXsETG5mP-R_yVkC37#1G$m0f2+PSNq8W6{ckfPH$w<o<1A1$K*2fP_*ui
zq12Ar@UJm=t)9V11L`AmCvADxz|-s_i<ca~8xmT)pWi|1M!E0#>}?*9kj(k0a?%?C
zD4x;BSEmHayhvg@W51<h<ucFnssDjUzAy4MVu6^O6a3+WARza84Z2fEHrSxtAX>Ha
zRRiKiU_SV$-^Q4rP=E}~2mdQf;Szb{!}i?yun2A-M}ks^B)V<ue6Sa_i)Gk7N>gbS
z@RV7!fAe$V%c;VzotcAAYgDog9u4(;fS61<i#3khjNEY|%dS3P)UbT%{y>20#3hsc
zT$Z_o)J(#QPWeBN2?g;ehi8|Yvu5Ev&JR<n-^|{iwIBj_5Nk7-BF$ZWRXOzn>oD`<
z=`)hz_c5WN<=foKXt|7q1XpC6f|_q`dS8*pW++$l2~CGRNMoCM+D(3s<yxKxP2!EV
zr_xvRGb$y*{=x_Rx2dnScn1F-sb<qX=I%%367ZxJ@sJrIpzmuf%eLQKI_3iy^3{E1
zVSN*+-p>G8IbeJ8%wkr=-KEm|p`j~zq=n*Lg@dpZRN>IK_H5=EPL}`bQJbcpZr@A8
z_qU}dFK=bPWO7hvONgpZ^0O$)s!YpPu5uD=CVO(bSavs${^I9z{=&R?IU<c<W(E#?
zu5HRU_(^p)3@A%!wh#7kWAco=vhzpHLZ^QL#;cgq*Ri*);`L*>>r}O!7HQ0?i}aT^
zb;jz~i+a^BoGE@R>g=MX9A6|4osqX1kR9bNWtFrRiKwUKzrTO0K{fZv*R#5M=q;|0
z_~ZE7dyF%lwmmr5!nqL<`_vWDCPAF-9AIC!p>Y>osNhBkAM!8d53C(@ZJ9)Sr8NHH
z(~Tm02@`t@p!T6IJ3RqI(Fb!G5p8AmS3y18qJWETqaKiAZt~R4TfQY)j*i%;v}NA$
zY-Al;oo)YD9W{F<pU)3s)eyQhxU^nhU)volwdu6``wX$?6QK(wflI5xxV#xu)oTt4
z-NW$-tgj^yab6nN^m^m71TP|kB*TZq7<4Z4I;oe-^glj`0+=iJf-LLu!c>9AM>P`6
zS(nik6i~oQ^1Sg0MS_mA%rMJ=<mzpXSP+A8`ftj{{NXqK1Ki05or9SiRU<d(XG1R;
z1^DVKxZI<$b<eJIUtRE;@!Nl2$Ud=MhOaM36dK(t%*eQ8t>F4sb_sw-O!p~cBB!1>
z4cGc?Qq-8W#vNV~vv^{eJmh==@Q#`rI}f;oX3q|oK+Rt%>mc_TNK`zlCpo}bF?qgQ
z9_<zNbT~z8lj55-ipbM?$)GFeJiH{IO*6N$gKq=}f$QeWiS+VCq68(s#?>DQ*TDn+
zes;GnQD7HVIY<^>N@>@J6=mb^61+meQy~nBtj%Oz!tdr~Z>suJFy8BK@Pzt@Bm7^T
z#><w_av(@O{AQO`7`Oq&Co!uj15UVcz5SIH4?X@UWq2M6xCcGal*_ymI4}4gl0vtk
z&{pjbE3DcItflPE>R_xJy%r`i!K{-LpO#Ya+97>E{ZVz*<19=;*|iti(@=%UySoY#
z8CQLi3LjhfifZlNk=N^f@)?Z1YmQwfRpD?Ez&p6T`2OXf+W=NNLJU42U?k=)^tN4#
zdyiD+y;@19-=*7v|BnjugJ1Yd@M_A*c;EZCIfb~t5NPC>{!Sy;*@BA*2@wsrpH=gf
zU!YnN0r>>t8r%$dXa&30vrqO+mi>yPjhGzk0PiQpa7maJL*pK=jHWIH6h+lSy)F_g
zP~GexK$~HFRqr)%t!jTUl?$=oaj%i6?kICn{H<1qv7+|Ic_Pa(Lyd2_HMZDYW|z<D
z`_#pf9C2wdvEzjqexW5`<k2b8v>9SANHIab<7KWLx{+c45@bi;_?n(vZQ7;`5P12R
z#J?57Fki54D}qBBl@evZ9RV3Wkg>dCf<-7oOYm&d73Wm>pUmir0XH<*uIV;N=ajl#
z`0w9ebP-m7CTo8;*|=U4r>dVH{@i<epkX_*jGrP@joUJ@-Z>l3#Je^)F+-1vXGV47
z^$;L*o2czSnQ8AndA0bgPK1lGpWq)Y^DJfbl4R#1253ATR&O(`j6uMdSE7-t9Fjoa
zB0}(W*$@cQWm+W670j_215DA`t&kTYW`4^8#ZM;TJECXrC}bh|mz#OLjo@5zvVX(U
zR-Z^OS5#l@_8856MlbAp?@9G4GIAH;zp%2p#oK8h1-~Zzujg;jjKyxOKyEciIxS=e
zJQ=TkIwN@qGQ;SkP)S|Pe7IC7#LiO+)tW?nZ-|DudS(vDlAs%C1beB7dY@#O0<3&T
z;7t-tyT7m~dR?jR0mGAJ)@Px?$^U3nR~fy?x&b#VohDSU=X1p9w9uRg2w8qGH}vs!
zGl6xj9yM8f1FEW)ftX;W$WXave7@Z$1S=iJfS`Uh8wMR!|An7gF=4tIr3nqcNiUEC
ze=&hfBkr$gKVFm%8l#MZ{$i})e(~-eJt=Zl2=H6B&G*QVz#)thvd}(3T5WJ>cM-e8
zNaX`wsQ)b%&k92|`_Y?E9y{`&b9Ui-hnYuB?kA|QXFwgADp)`6fL+zlm6n)=0^+y}
zsf8dALmA{!{{v3W;J=BZ0VcA{u)!v7kE$&s_z8q#Bara{tN5?CkLW(Ifh6IM|6pVC
z{}XI14kDywHLSNL;<c1rt!6WA^^V)20ySRzU|Rp-ahB|KhUSRPL;-Pcht#s%&C&3G
znxhcx9tme2=<=coXrt{nN3!E~$iRT_p#fEJPN8Xaazg{6Fi_m_Qg8ml;H3Z0146(+
zE~BPzm4*$apk?>}?SN!3T5*VOk8on9#*7UPmuz-+Qym4&$&L^>7uCBI6M~~OK?i9O
znMM$+5Mb&u&<%t?ISg)r%|8j%$NlG?y>JoTwIJ2WiwwGeC<5dI)U0(3(8p3wu=yhX
zhcp<zeMhmb1}!=q_syN!KcWLXpO&ypZgin6Htj(uyF&pWS_j*c_um`7^R=YKHhcvR
z15I{|LC~ETNMsls8XA&nE1<-yO#}u>iCFLm`o^sMKh|T%mYPzC*Ww*A;LI+QjXpK{
zZ|-rxc#MuJ-nQ#F6!u>n?(dKTZ!YE7w2JaNJ0(B8*dhSRQkl5@CWJm43kGk=mz4p_
zhMPm&-2dU;#&Z$>$Kp=>zh`mJV+#RQnp(hTu$p{&0kKM{T}iLQ&Gn&t!z;qrI|uo{
znyQQXzwt=_$MFDF>jbDMDFa-@P}A%G>+$?Aj>aPHk=g+!a-BL$x&O=Y$Pt5rf|N4_
zxO$@rBOx*oP)j^j$-{>Q^8}Q#UHgh-a7aO=(0d~`SIXTjev|3$#**?sFxO|`{}<AG
zd!|MMv~nG*qy^(;K%9sedVF_=nF@TX1shum<S`I8neWDH{2#|Fee#}tZ{cGyM6eIY
z@W1^Zjt7dUHx6G;w#IvXPT-Dluw${7T|pB6iN10+M3tU*UAwYAF~qSURn~?nCC$Z~
z0-RxOcqeF(&gQz-1Bwp?F|n|S`co%m$njh#V4ft5SvIC*=;*BIvWkQnSvN;Pg#*2G
zAQ&hjn}2&mNI>Jt#d)iVW3O2)o`a4-5w-Q#_h~BwIci5j4i^1yS5Dd%%0_9L_9^`z
z9^yr)oay(zV?^hjTnW1B44@QMtWZ6m|BXoNj=Xzw^>nbo#+j4(7-Z#pLREo)C@~yo
zYeTlDt5g7IcnP41o}HyA5Bu*+PjMj>8~I#MHr2t`Bh3|eIWHLS*D<2a&jnsbG^Vb%
zx9kVt+S`oyE^`@HNN13OMy_}J6dh~~ZZ`&AY&@3-FzQDy@&J|hw||>-#Hc09v6?I{
z1`2`&f+Q1K1n7fN#hj5=kb?#~V+wx1;B$a2j|#b#RJ_hd&KOK7u54hR7;k%8J^&q-
zkpZW@4B+w40|c{=+x;)HY>a{U44||^Z>i?}Yl2Q%iPf_uiw$U#!U;Q{a3^=NK=TUM
zfEo1vjc?ltZmGLmXR+5PlzzY=%XY(2m^7{vu1WXMFMhY}?8|FwIcY<=7<#^xdJZBv
zI<_&qo6)n@?45;8aj&+6w$Xrn_m0hDpXI9s+pI&eQL**2o7jzA!RbdxQ2QGMZ~qi@
zJVYbpCg(-=SHv95ZW{x1e-4&;Mjq{O&R=;ap*KUFpQ`t1^tc7kU5*JUsnYM^!(p=U
zM06KU4aQ5(Oz?I31XpCCI9n>2p9$7rRK<E6E01VN0{DIZbC<C?D}~1&oK1KGE>9Rs
zc`Fo=9YIHK3nWYpA8bVDz}D$PM_e#qSrkJ5k<PA1Q#u%rhpy9gL8Wc5Nj4mGtco&U
zQ<Duwmkp5wDfwQ|T8aaJG+00bGW_1;);H+Q=wO1}+gN%`rqBd=tkZd*33du23#E!g
z@~hCpz?XZF5aZSlCExgb`mg20p)*p$0K%FS#9rbO^=tw(TpFEzr5SX+G;{<oqCc({
z0+^pxUh<qQy(F?|JVL+Qs24A#B63u4KmLl){vnRdF%0wv$B)7og+vgD3>tK^v+Gr|
z0^&3#KwtCr_Vqa<z9Jv!(?YO_#JP-!+<f7RU`-;09EE@%^$<oiw2pPZf#q&`<Y0QP
zsA+R8xZtPvcL=AW&>0`XhBF0837J*#favhW{Qju^d{?_)76(M46NIEyaI<h-BrDjE
z2$ttfg&6mP=D=Ja9!PJL0Z8xI<SXhEFgpvm@a1R(`~h?r3MLUAIPT)p23^-R?={Uo
zNFvT<f<?;(hs#<X+5#fD&f$0l@RSV}8zpIszhXe6@0~6Ti4wk#?m`tW(!<&51hng7
z6kjVY3-`f4->#^U&6kdl{j)pQt08g2_~7Z6l-5{_;b_o1s~w3g7(q~i?c1OuJs2ZL
zA=v8UW=k42Bhbt`81Bp)t-&V*42RwmZVTk0NDH&-dyNk|tKow)0uea0shctTzd~<D
z0RnN}!kA`=gS4yQlFompT{=j+$b2z$cjCi<#0N{A`$P(fZ;x{NH98|DEMD{|%KLa&
zuos**pq3;_RhN}K_`el`!^0(Lu74^4eH>bUsDBIuSVRWYp<UTMXAJ2&5~1I9)Ps$Z
z2nP(einrzg{*hj41@wQQS4rhu1AI<vogql{3UO44qJ4f*!G1u{t4r+`!<H|>nt8*L
z$ng=uR|sALn+}&64Fw8-!+52F*S%HEt(gZQqqg4Wng+H@8iFf+E}6n*q({V|r|JkA
zB()Jg9gdPL0+NX$Vq1FK16G~fligoV2#t6v6Ko}Hh64U8O41FeS@h#o{Cf>FU}58k
z6tqAVaD)&58GHp!1Z89dh4QhO%_D@e@GY?85?`3~VJgIf`Hfliu>gs~Va{N*kMq9a
zr*}e>1*<;%^?PlR=B~ZfqeL}0j9W4f6q{Ls7J-&?^-l|=BQUv3@hc>eA4GvAV8I6Z
zlM%rlNy>j~pl~-QJD`=wzDWJM1v@JQuQ$*8MV-b7g9%#Vusvjiu|;-1@3>(G<n7Fm
zhjHB)z;`|DB`u*sb*x2$r1)9=X>ui4v(-Plg~JBHw2n4*9!{r0j>eoG*tTp~KU)Zx
zK*}#0fh6=E;t;^z8U;38w8%lLHqbS5^;2GV`V}(pt5jhbC`>qDjs%fnD$IAbf(f=t
zopjO#7!9Z>3@o~5Wkib*S`l*SXhC$a0-hac07@q4oCW&!eE2n0@mAqUuT7CC7*6Y!
zSP=!G`9dke)P@o)^7kWPHf>AYj<i6WrqFt4{L=RB0RhS)Fp7eWE}4S9;L?q4dG%+{
za_{^50;p+<`0#vmunM%;@i>e3meD{Zb+XAm|4atV=Lnk5odoNofyq=RSc@x#f!P&p
zvFoLvHxq==2A)f8jAR=byA3k22g?5Znr>pKq4x@17BsDkE~Q&X0x}bn7@V=K<OS1`
z?R*7X7YF^dI^=g0xq+u<(GRL?;)Q03C236}3y;806Lx=Bs$CN2eKMxd3vMLZWEcXA
z?!X9t8^5iW3W2uXKQ||doP-SoI5=$uwh}ZrE!cXu-oK47LE9~Qlt<}mj4)T(m2}%2
z{05t00%%W1F@2Ur76mK77=6J+Nk9d(5o*=V0d^f;h_zI(l~4T%(T)-{xmrkqV%(xT
z7N8NHceDE(AN>jfY{u9<_WbthpcmA>$bxGrHEW_A9%3Z0=~cXMoXj8)9VaWE7z#ks
zfrB8$8IK+JHR3KTu|kW*^hwRU4+dn!@+DD4PNE3`UHhUPYTERyM{`W5M7=e$Jf;nm
zmYD^D(a)hl!$Q%P<bpG45B_i&$paN`!#nFlJuWl2n80!~A>U=wQLjq}{uCcZWeUOz
zoDnwhPK^*e$P$50^2FfMgzm$`Ll9B~wU8V4Am^YGgDqUk#*^oQ83v;_LjrW(!Jm$V
z=7>iPqv|j0j`8w?q7Kb3S%DCAM)VNS3?>G!^u)NX_eFyPA7d#AXDWIR2KbXZKZgN?
zxG+gs`7ag+K(UzC8MsR@;0g&?bW1Ud0d7CE8nQ{wD0FX}5K*6_D8Mrr4Pr_RxY_tr
z2RP#0DR2kfPT%kE?N!Z8=9fQrMkpEs#X3(eN@(@3;K0quJ<1qXWk()K%Cz23N-iHV
znA4Fo2S@J$^UtBLX%jKChXn}0FB?K0)p4fY0`jPMkVh?TnA4#L9yJ<xR1HrK$~POp
zrWB8GtWVpbJ7aj-Spn4ryj-V@o`l=@*RXuOE@Tym@eCjq^^2qpGXUW-;jI|t5G;O`
zhkn5Vtj!M?Mq?2ax5hoGIrg6-=lJ|2=LKZsyq>!TV}MqQNAQgk_$CO8BtT4}^Y0zW
zz)IP|!PX+$b`*F6enA8BBgu<j<e4Dfl+=o381S4v0W=A!^Exm+{P7kb4W74-WyV@p
z+<8hG1z->HUiUK<fJBrbdtmyM9t*{?Zpa>7HtX$`VxTR$_egJ{0)2O`L4k->QxZI#
zszF`Ue+^Dl>^*I{8_HBpGJY9r=EoF`4fGF+D8OZwNb@wrLi2@=0jB#VO<ab+JtS6&
z_51pfgaA_e7{D$R^W_ak3_u+FG-cndN49eZSpNDcjm_2n**Jnr-ZHOgB}`Eo<;@+K
zfmWcesRNZMjRaa%-8md)pffc-Z`Hex#G%b&mRr}i1VzlVt28dKU`RdSs{~MR$3V!I
z14@xZwtNjlONR7i_ilwa(D}on(~t^33NmSeA`B6Ic?9C;cTolpF;=oc8h2YJ5PTC4
z8S(`P-4n8%sEF15PLiu^(6l7UkQbVmHw|dsb$bibq4V)vC$M{6#}Q+&<wg(yuBc`w
zDqP(e15*Y%y0MKS@*(|Kkn>EyL@(MdO2-Ts66k~Cc3pRpun#R5wr#oJkq#I@okFLR
zgBx)E2Ub7RT!BNd1_xF=3FHNVxc-{rTjAt{1aFt|lOVS9SZI-WI3YUOeBG;EJF=a!
zKmgIG@zH>gx|s$y7PsbZ*}Hfy^BGxIA5C};slfQMa3Pp8Wq;TVUz5I~qN1p<eczvA
zOJ_*0w9rs(%MFeeoItVoDtzg0XYf~i7k0$HLR$_G_dg4cH?_%u{+=B1L+rY`g_^W9
z{jkzR*-Y#YFrgJ<WG#>J6Xm~k)FZ&zc9CZpz7oU(qfp)q=s}-z!kmqg-8sFQ{u)w%
z`3%Mf&-b~O;4#tBCH^ehn%wDg1N!-m2`hVK?_2zL7RMIy4_G6dy_D!KV*9ay)*Rq-
z7ZSt|RfEE*BJ%)+20ZizklK2)c2pABx9;OFwmqzbp&6pwt!O;RG4dhU(JrXAscYCk
zHfQn@^~B(Fp_t(L=?$u(?7g5=$cr2?tTeeUu*oVZDi#UbKRW$XM+G$98Y)??o?*Hr
z(bP>2^dS`XP~{L6W*qNVNIQoKKF^Z-r?HVg5-N}@xWIRr7uqK(U<Bu#EWT{tMFSkL
zPE#qkE{j^t3GMtU7TC;LpVN!Iks!T~#XF*a&qwH2n)rqL-E6ONiy=F?h78CH6ZF9(
zuvgDeZ*%5^9B~aip*;DW{$>203o5Xlk(IAntsw|A>r1~BnnJ<nw&1rRWjEZ6jzb$K
z5V9MQ2v!V?Xg(IT5>vrc8E`Yu$5WhAn&7zU_Y{Hxo~?@w+mdAPj_dOi1|x7B2yOts
ztu|7(Fn4Qf55&YM0a;A8bV_)*Y)0}s?HbA<uuD2QAWPnQb897%B?T<+p-=<Sb9m=z
zv*2Z76|l2T!+++o@A$WY4~Qv=_Rd`Bl>eW(+->mxXD<KGT>hVT`5(MXS#0NW&^258
zja$h;kUPHXh|eZ7KN$!K%pN6mM{%Ivc9-jbp*aCV=czY=7fl7YHAr^-3v@nRrV*w0
z#D5Q)o~n*G9LwG~`qBzr%q&}uzT2Z;R*UHQNSc+q<akbn|C{K}ufg8_uYRp8rgKZX
zL^l3vJE~BEhQ#2+!%Tpw>@#qf56eVi-*{00w;Lxj0!t<X&U<~{$7t!Vfn(tFux36#
zlE?W^_s&Uv9xc3$OQ(zbdz9IHxRg}zYw^R!)+X|bC6Nh|sIcW5_^T8)I%l8X)vura
zS)QP6yNwg_+^~~0jZj{suV&M_zapy9L;5egMIIdN(FQU%)hw3wtsi0Mp#PPm+V4E@
zw%J@|-~1l-JtpbRLA(rGh#`)5W%TUd(+_J$74^~W2P1g4wj)&DgrhxJAyyK!#K%{$
zq=xP$O#2mPx^3>{!`J;LekjvK(=Yza*~Ykeup4O{`yoRCCqC~B@5hMn@NnM8$v=S{
zdNut1M7J5WlXTEr8yY7bJWDK;q;Y-PZcLSCb>v&Zlf3)`^+$qnbEL|qnSVYJ=O(rn
zWep)>MCVv;Aj#rE8Pn~}nOUNe`gFIrukH%9@^nghvQ7OT(w%HF=&pn6m70--&S)uy
zBgKV0VH4`IvF7+Tx>C4?PveT3_U~z$Q1hmr2hQ0hn`id2hyR$5S<<OnqL_yI@+rd=
zZn=kqy0P~h(^~oU$AFoW&+SgjQ9-xjZ1rZz+QvzHZUK*Ea^3x6i5<**WQ@J$V+)2W
zEy%xc%+b;qHnh#Ycdcl?+Ul2$%^6N^6Q<jdD3&uj>=oIOjgCAZ!It;H_IX}J+OlLa
zafXvQs&g==no?yrCfaL-2AP@Y_qywi+@P2PuiaOeW!B4)CAOmCt!KgXu)6rKe(R@n
zhvV)<)A>HdOrfUn8Ms_zLQ0hx7oQ&KHtESdZu5J1>}DMI_t9ab_kP^7wcO;trPo(A
z8BFyXt@PYac_;XgYe}|_&5=%vC%RneHO4wml^YEL>QBTDwfgfB<FnmweJoGj^&)Qi
z(2GJQ+&v>C;02mQ3(1Sv_rWf?DkKOV=1?-LU;puw^dz-yQ`Om@)m%9A_L(1KSV7E*
zVuJzjq*?2%v*xU5UC8dm&&k>|VGktarK_d@F7;avtrTxo0SZm^DKXtIhT_ht=Z?J;
zkITOx(Lervzpm?%F3YgNYQMi05@H)7GX9_}*n4)!1a&n1FDWP;T3#eF-g8d)evB&^
zplqzWLeXK=?(LVIHeO$IU|d5yQ6xP!)qWuK&Btn_hQBuZ;JbrN$ty2Y<Ggk=PdUA@
zpi|D`(v=n|niEbl{p;(S_yU^_fr1zBBfqL_iL1vWa4trCyp|#$Am7R3cYhn*p?<wJ
zwDI1;{)&!(KwObZ>_56;QQH3#&}^ZTW|AoQ_E+0cZ)uL}#PAk*{Gmu#FQJx@lz3&Q
zo@y3{34J!p%gS7ykvE~14zT@W-+1j3sjoj-f4!c6=T*W(utQRbC%9}TFFO4$?){T%
z8Y|>ADaGsSZjCi}wux_BI~wgoMtVV<ii&o!1%F@63(*AJ8$Uw^r!Aw#WSh%8@#D$u
zxtQKi#N?EBCl{n9@FS(5qqoX?EUK7S9k=RP*T*~_$5rtu#HR|L!pHD<DydG;Y+a>c
z9lXHIr5dfIQp&#HS97XiTH6>Ob6e%Lude_8tKHF!zP$5+Hac{;)h>;hn<Rr?HBjLX
zVp60!w@!tEr-~3k*N=*HiTGSRg+H_e{qgS%)!{$2=jXa___V6Rdjd|&qu8@4E>7Q!
zh!D|*pi#u_aXb60sCU>ZDV~4EKjr~1VfXAyw5CuvzbYyAkjyB$yWbf-m9hsz&a(<@
z2Aj+vVHd@Kh}Mlz6&?W|jNR^JL-iyY>9v+&v*`S(Cu~&ku;vmyCJEKAJm0wpQiv#D
z@7tKJxxS*q!<FvBT)mWg9zAEGkd#mhBkc78_lU<jVlp4V@%^`)h`kN6tDTZ(+m8L=
z{Hobx-@CEqoV6pWxd1~;-VfeAF>A+f9AtOMWqSApjdxo5kN_7qMs#9s8}mrZZ<sCS
zh&dZ0BWTG^8;=P|s1Oz{Sn{7|F*w3w_{xT_vA#+&>O~AG3AHJCl?KOWvNf)iN5-n1
zGaemh=7S0EiY%EJxj$WN@vhv=!8Rh)A04=a%dU=ph)U(TH2Rs8@Qt-^GdWz=(Upk~
z-$=A@c-YC|$RPGd8*M2Ij*_6<j~xj*b_vpK6uCySM*%^;=opiaN3nr>=X|0@>u9tm
z<98ObggdD}LlL({(lE>06meLz1HH6ea%&L-Bibi=2HhR%GWZy_8lQK}(!HFrD|AL7
ze|&vA-V}dx)1N!uvwt1zupIbN`yyr7XxDd8GKT|C^Jk{UhbZm8*9>l?+@=0qtNyjf
z*x_QCO-dcP3Mnn6Lp5Kx*Y!3?P?U+V|3sZ1pAnANtUE7zoYzm?I`ei0oQ{#QYaXDy
zHD+|?)PK-vO^sU+SbKYZn0ZiyajCJUWnU0yK(0^94*zZ3e9}E4W;sea|0Z|Y-)+J`
zb6W8D<7T)Yr^$?!lp@^wTbxHOMp}bOBFkIJgby(d#uWA9EPpSJc5fXsAB>?{pd<^L
zzu2n+&1s}x$SdGb5h(-;m9}@C;D0Ww|1m<BL%txjTp)gWWtr21>bKlvG7j6|L{wdk
zA6NC($2kyz-%y%yx|zHkZ7pg<#r1EP@xr0>I2V?ePAE4y$`)l`FEHj5{MB15Y%^{A
zConDX?@X>ENynG^YVIsEV=q5fM6NWKt$3^cUR)8cdy}i-C+n~4SJ_v$r7w`a<8#+&
z&ICyuRh+w?iF@3;mYGj)SzEiUX}>u*dx46UeXu{;^lGByX7RnGVtS2ter=uQeVoqz
z!_liCf_@y@HwT~P%ByCU1C#g8^M+omJp`9NmfEM}OVAa`>oSQeS^}nC-S6SRXK&8L
zcyjuoq@CTAR$MJ0;<PM|`W*&FbF&a<gv9Si6I(?ESHI$`lS1*G81r!CN|%o1c9oB|
zd+*&cp73l(zW>IS!<|_>eSNvk(`&Isfw@!SFGxb*LP^Q_^o3gJd*10|v<*{)L*5lP
zlq-x_`&u_w8ix%hTL1DjH}ho#0)-k888NTRNw&JvFpWuHd`$V71x?M3ma<y18`b`_
zzJzZM+cDf4F&Lq3<bV8qnBL7S1{MmbJTbDu{jhoL^lTV=Z{HjvRT$md@1&&rXmOy(
z<<shjQTM*z?Hh;TJz8g-9hV;d5G|wI%zH8RPN?#LWuVQ$t&yQE9NCi95<6CGt>bSF
ziT=;FCvAj!r1qa{l@EM+>@J*#^xY)fRr5x_X?&Od!mY6@*w{-{y~*ZV^(2ykW}}JU
z{(v-)cJ_Gj&um+L$7uZ!v-2i+hl!su$f1#jb@*Hd;;o~tA3fE#(O&N@o@+qvrR~tq
zmSo>K$BOE%eLAJ^Oak}8r6%H)1_EfArX(w+^ZoB4>)%&fMo8kkO{-pkC;i>*F86<;
z?b50hJdv<wV!TbIdC=ZXM%N#E%=1YzPCLTUiqJy5t2I2}D&e#y>K<1r!vL*w_R11Q
zJ>!DazS&_W(x8=EXaBb6H{bZ%xH{=Uk0L5er*3z${;<8XL#jR*YC-AEFbxs%2%kmK
z+n*D|Za5cu*GyGpJc*}V-VTrXZPL!t`h9UtS5oOQc^av&w!@Zh;Xj!42hR#*zgqI;
zFXR?|T}ex!-rzQtyk_bTt2zjxSI-t@dzIQ|Vi9&`AL%c+T2IXzutM^{pv&)f8&!$A
zVtPN5AvfX0Q%d@Djxl-upaCyDcTQ{3gl;(k8QA*V$Z2kcFvS(=K#iM8E{_56%}B|N
zG=1R%=?-PtLzIpWP7dO_Zi1|@8Ej^;y>58y{55($e$L8IW}^)gb?*N@+QTD3qt)@T
zV0_B9sCn!9_ZE0asWcLe2Tv(4toZHa#~+g!zx=D9I7vBXcwN&amMHqO1#OzRgXR(a
zlQuqsR{kHvY$vx1Uf5`4rB75TOZh%ta}V%X?Rb)i2s+m1=q(?se0c2f((_~<Ym1WC
zMJ8FXr0c12RBAnzfm<gUH!4py`}+zO8ao}CAl+QMhpNqzIR=CRXlIkvtT$Ww4cVH!
zu4)bsCH*h<eLECZaq&Zb;W;o&vlWK-`0;QNs1#i>?)lz3?anjR<TlSx{pF<6S?T8y
zktShYu>A0oNm^z33ui`Uy>_$Qqkwkaz&J_cP?xgBEc|EV0tYhpPw6xobxM3YPAw&u
zJ46EV4fr-t-X~C=f2KNq{Ju^qK{Wd}%?$Ol($}4b>i$B1)g+SV%>l0Vsh@<mPucQS
z7&ejOC*N+7If8p>_wf=AJw6Ux9-q^G@?PE~eqguJJ^bv|JjW7g@-Tvc5)!+><BPBj
zDPvNHuC5R6=_Ql$y(3R$DlFs6BZ5yqHZ??+MO2-(-p2L*tgm}G_0A-BBKNnjiOJ<<
z_`$GjF^GqR5rRW%>d@)iwxe!3TB>vKwAtB0NE!x<z=-uLozJD7=@vYACJifsg_lsi
zt<n1;wr56@-+-f{xNvG6(TH8AV=86L=B`HhTzaj~PX)85{{H&Bxl?RjfA9Xs?itj8
zghYOf>7(cBX$(t&tGe^2nlxqBr*=c%wd~w8xknD%H!syJef#OOc-Kigjt!2*INr2K
zzMEx2#{j3$lf}S}p_%2!#u!-nI>Z%u)8F4<MpfM_4yHVaRi7ozP6$1Cf4uyygt6~K
z%)8`~%ED8WN>@^x9M0QYhT(u3aokm-^-VwJ(EKRzA7>c%uIfBQ*t+^#5%QUTqa6~n
zmwNb4Z`cTr5fI>^ey*oP^!O_lU_)DW9G-VrH|@cg?w%}8oUDOoq@8hwKR@k^HGnH!
zBu`)^67fMv%`axSLf||nL{8hY4B=%RBKKR5t}F@9A-9i_gGCy`M_Wz&K9zk(b{H6V
zImsD+lDN;tQ%?EL(?<t6Y2F-6++bI1X$R&G-;>z5igR_D+>AllK;$_i+xxJL^{1sb
zhv*0sU8o8JF_WXIiY2<-EwD~XO}fB1`wM#$Yn0l<Im4G}S7+cxlkB(TH>@yu_?u;3
z8D(;>YvksvoV+F)B@AXFBIB|+R!e|2ppz3^u*7lG7c;ta-ML@SYTDlD-8l8Ismp+?
zyI$OGq+;6P@~SR32!B*?x4Mrv04Mjn?M>OATJQ4b8I&By6mh3CR9}#zN)F4iN`{fi
zRg$CIWY~XZoJES{eD@7Latq}RI<4$=qtb|JKzcg#(#kK2Jp1LvPhQp>OI05FL7RtC
zTW-?pIa*2{6+Ta>t*~kYTiw4M$_*x8YG@+22a@#=TPtcB%VCmCQn-Cm8umr|`ZhsW
zbg?6rm09VHCYxBeth{x2{x-or#ZTLl2&D(><jwox3z$Migahx|<3)Ewuz$Q9iRp{^
zV4B90Dr3s9X_&oVp|AVtx2SuKIPLy~)4F%_W#9HAa1ghyWFA(0gz}vuTX{xP0>SRh
z%$u+;9v>FT1`D@o#L}$r!aA>>4R(^G?*2e5ds9uSFI+Rs@QEjrAYn6Wzp(e^%u*5w
zzE2OoRs&Mlxjtn(o<S{xRPwv^>oLS9^X7^A>KQC?H*PvywVy++e$OC&)ud_M;tdv`
zDtXh~(8thT&nxA{eOy$|L3SOgHBb9`)%Byx5{Kb(TatZGLDfX4D=hPo11vEF5yzt!
z6~pllVwqfF^c(MU_F;6ox~)Cj-buZiDO6k}V!k}}YfocGUx%s;s?JtRBukUh{ZtBV
za2hXZWXn@R<cMq=*I-ODz!PuoSR7(N5qM<N_0oG{-eIb2G55JX3ATMI`)%eR%O`$p
z^VMM#wZ?Ezm!LOlT$yHmg7V!kuLoP10qKc3ybtGKzD7XDvprEQR;KlDZOX-u5}73L
z?|A~uPaVXWZ@vi#W|qgjM?f&E#UJJwLMe>xj9+$KpkHwyI$$B8feTtDqUt>>7x11f
z<;xuBfe-DuX&?J{^DSp(n^Dr{S@2iwNzO%<OuEmaZ>c>Kl3r8UVz;a}ChIIV$|;zM
z<~(vAeMfDXyP#uv{E|2CxXU#T1>PVa3XV}6I3PJJO&9#}-XPIs#HZYuC`on(=rq4h
z6_+5{>97zsY4cMvH&~}eA+Oj>+j+Wwa;s>6tLa73XgO$1@KqWX%9uco_qFSlXXll>
zf7dr}0)p~_Biir<K_$zz6N!?ZKEY*u2Z2h#`BCcPM)%Wz^RELL*2Ow`Pae2^`hv3|
zb-N~)@{J~y2~L^EPvIDgGH!17etHeJ>@ntNn&~kzjSpW|(lx(@%5(q0(;LF6%vPA%
zFAaZ07MZ^b+w()0G$$_5uEfCDxQB>$Nzzq1`Z_Nc&t@$kBoA}2vH7!?`JsPz^|M9X
z;b)cc7D}*PI-w9$_f5G`4mrAo=kKD`d<TOIXv?%&S!PZzdY7uqh60wDbJsNPOGD?X
zhHKz=9@dJWv)L8N*rDk^{08s4#ZN``h!nFWNnW)=&lfknu6}NoZnDn=w;K=L(9)$P
zdXL(O&59<yaPZ+k&lIutd0>HxmE5~d0vc(~T{&~QxOMzoZcK0SC6<2&EE;UO5Lau(
zsnlm0=u!_`1+x5BFSgHchEQH3;RVXrIjc21D#0J5<qA}Y<#BO8v26ML?C;9}17`cO
z@WB293dZ2{+WzhXjSd+yTXU57W$spl&IedNC;saw5`y0{5+qGaDVM16dPxUHd6o=I
ztT}|@ms@q^$9HahpN+=P%FNnkVx&G#AhpP9oDZ|g3U-uZNf3fNG0-ud2!-+DOzYQj
zN?+QKiq~ng4@$G%qd&}lr?IsgkeeWreZH4=iQ_}|J&GtUAsAKcr#|aI7X=cT1t&5F
z);LO;*45=&h;2oM)p0H}Bb+iM3-g%kr>ObKR{hi~c;xjxvVMAdX_$``jq6aDS2Q+(
z!&hC!w37V<y_DbFFdky~YNnbvgOOxz$h9mj;(F#j2iw_SeO6)Q9I1MSG8dX7L{_o&
zHl%2miGbpw$d?n(M0<sbld0(tW3YReKl>s0Q0JgbCmc%RMAeaVbIBOm$*L$?>&TbR
z6YjB~TQ?HZn4e_8d>qXh9^drQnb}KtSqF=v1vFdtyg2MXT!-yQyykUb5Nw%e)kV49
zKMa%+MhN*%`;6e65?x;(+^ko(DJyosg)#}cQ0@^K5S!Dz1bM=Y;}1h}q@g&5NCJk(
zli|oC5_1(1%{yfZ0Usn=<apfXzfzD$zA%T6_>TyGnMyvO^Fc_Zy2n+7b1aWC)y`Wb
zk|&06EP=-^SJaCfzJ*sV)9}Tb__M6Wa|weVPAtK_?+|RligUd+7p-n(ejn&xX{2QA
zr^`-62VC<3k42BVQys$+P6ZzpWe`^9p+j^x#d4Vqh7U{<nq;0gp3rsD8a9M|Ld6!c
z>q^HU{C1!ar`#YF<gUd@0LN2cH`$}>{hl$J-apotFn~^%Khzr@-HSnRxmM57@al5>
z2#ve*Qgl*M&9e3<o6eEVO;j*KkbfbKaw*R1FJi~d2N_Gh*JRfZ+a|+*wXouvrW_D7
z;_;6C3H1Avi@{DJ1z&JeaYKqJH(<J{4p|{PSzSP|M7zgnN8MRPjEtX8@Rb!j3~Dc0
zp84I+?S+wAr%p|glAwJ%mkC8htxO_F1Q}3O<Y~@_KKPwf@!`BV(I7?n4PB^>M!^Z(
z`!?$_j(<m$%yZ}y*8ZZd^-X=AU5$ZJser1;G2V_VsN+DRoiDvf{W>A|e%X_Fn}RE`
z_it7zsvB|I_Qu>Wi@XU5G>`00{Bd)SP?2p7jPiy*hdXbXBx98?7~3~}$7qG81i$Cx
zbgJ4Hug#$`E{+(N?9jv9RO7HhzknBo=HbKM^^L>kRG*7V7;(N0Qq+riVQ@6EC#GH{
zvo57LNi%6`UnFf<VKmpszby8iBl?+^FgnJoC#69Xg8>0gN(^XmN@sQOap6SZ`C{Mw
z*v()+S$=9i4>`raEK^Tt;NV7BrPsgg{cCIN7cpt!6Z5!_Lj^^o=@}f{bXS7@7BDgT
zE?m7Hw?z9qc^z?UqxUNB$??&H;uRaFjBDrDM?+{2M)fb>VbD|^gh%J^OSJ5kH11Y}
zKZfs1`bw=BG?+P@(pXk@iYy$orF+Kroo0ayZXLx?Asu6%%8&c3wq0`CnT9d})&8K$
zKaY%r8?VyS<bjfP%O(5#t)1Zs{#4B`GtT*PWn7f=y)F*qD3RTRL&}E}t>U9G;6brW
zdwQ9+?|!S1=0ez1e7dSM%sDUb<lVYn2Fa!7x9(O;=1H`Sxn^4Kbk(<-we`|VE;D>D
zZmtnN#;4=Pkz!>P4e+-d&b5-d6^&?>kz6?$ya-NtNQj<!dT%)?be1LH_!YSF$<tIl
zS53BvU|@W418031i4)5(J#~++J4sJ3fmq6-b6Z`7kbsqPo##@k<YGJtJVL2V&OFW}
zxf&!o5gQ8xNAn)EKKq{3@gw1Z^Y6pt)f*@-bz%+2E^Hk~`t&b}`^R}7rqDl2#<;Do
z5oBqt6lKX%*CWRuqM_q~`5*|YIoaSG`>2HQf48<LMlB#6j7uQX?}$126l!bK!6%Ec
z?aGsrJ>FX5hJqeiJoCz4$SD&j?EyVQW0fQ_OUOiCe`Xm!3Q?rR)=P0|*cL98B#{Nx
z0Ud_y#8mpo((6%}j~(;H{#25?__VtwV+3b~{OJG3-dhI66?NUZp=qRX8V~O78iF?N
z7ThH`1W173);Pf(f+R>1+=6Rx2?U4W1oz;6H+kQ4>Z^OJ&i`9={zKJ<wf9<cjycA&
z2*+>!#>D7|lm357Gh1OK@vaw(a?DP2_hbx7SsOz}MK?bXtb*I*@Sq-eG=%lWNB#yX
z1)E{=;-vd{Aw=*#`ObyCl}rVBiT>b7Jah#*75SBL%=U6&^Wa|T50i3e@w6wG-*Tr&
z)5q21C0i!Nc=6KD{o@2Q4cOH#oSDroC`zoL(DVk0H9ak?7eQtrb_@-w!RYjrWyF;u
zJqU*r^Aer49m6fdKz0k;dY&pKm|hSEn(oU_k_U<87`N}3+#f3lUMxa$nh1Fjfrdk9
z9sK>bpxXon=F9W}^G@O;IOgk1x4Nm`shiutX<4EMheU%HowdBfH!@Q4>7)8_<~*n$
zIBXv~Ci!rg0V!SDY=kH2$q_p6+q1Os^l2JWu7GI!uv5oSi&V8^^zsOk9+1~{qK#DK
z&3IqF`{BFMg}*0aYtc1A50?}^GK0f&e$>U|btMDMAM47xf;^S8)JzYa-!ySk@v1+v
zPPU6auf@89+dXMUp<M&N!szWT6Xm|gd`pvKvs$+Wscw4-RKPtAjk5AN7v4f6Yz#EA
zwwlZKO3~!TyB+OAa)};l#me`|Oe&Eur5HV6^D!Z5rdJVyT(TZ0zS5D&n#O6REUn<v
zPywMC+>R~D1vBu24Xk-m=8oI!sL>8dv`a#w7*Tz_*|2NlQDi#?W8?e#GZ#ut`QTu1
zB>B;oqEeGCX2XovR%d!g)WNNIH=<$4qDF#q^!J86{CXbm(gigkrY9V$AIF_V_H#aL
zy#Smami(!5it-)7-+mmZF-08UKjsl&juU^K?(jhHdG9XyFaZa*NexlXKgbI^|Da{J
zk3{>c#u!h2@TRO3enzD&`A#O24>*4xIV#kS6L4>?DK_Zr7(?%{1^hLee7}{bm8dZZ
z2bap&3xAEv%@S1^USQcnr0;Z5K)?He3k|2|o>xL9kqdtLyrEFxO;lL?w%eB0;HS=?
z-Viu;dB|+o%ksXL1p_tjTKT9?%+{X|I;D!msfh-xG>jT*%F^Fbe^$VQj>U$jia-}Q
zLzKD7sLyT?mNd;szD@-8WilzTzDy(^1gV<@7PS$eYsuPN`e#*(u~H)(LF3<21<lIM
zSqHNgKg0>>L}}~&4m_?=0{U5!1l;))%7GWWshocp^+GZRUd3P*6*xhm?+JB$D>tRa
zc}j0+o0&ZAcCGLq0aJ*a#yQ3!;Kk@7DpBB_&v`b1VPUZtoHFR1a4o_{X)iCk@Ejl5
zhO~wG6Y(SoPI2911|UhvOMk!%X4%j1B2<W=%48aptUitlhjQ`~HE4Kws<}#O((U9o
zG6r4?JH;E=BlH*+@?&RhrkNAwQlK>Hd?>?vwl(VGI#RCVeOA9d)-slch~Sbhdh|*0
zsHw$^(%e-FYIYWvuIft22lFWN?+|BN$k-g*DT3bmCp|#9uUq(Ew~cp%`;?cvbwb(b
zGvfmTV73H1U_q3zqL3nTfo);xFHq&<1HzaFIm6zLSLCT;);uE*Rk$5*o{3>rk$kXu
zZm8C4nox8m>>tIUbji%ed!zJ$Dg95FA_+BJ<u+&s^;C&1MzHlfA49;s#+F}n1jPKR
z@uVRY7+Q){TIB$SOF+cEO=XW31x4z`IgC^3Ijl^7ft_ah6Am=!qT^S#1#yvC-z*kr
z;(*!tZ|k*F+krtGi<rsZ9H_sfd|r~aq)Y|93ps~wKL1x^H$4LCN?AWYJr{NvV_E+l
z*$I}IaS&JYIYACD64-C~Bkrqc;!D{Yh9CweK3GCtKe&(?JXFha_=PkkAR?t+ugN>u
zg#za703)AYO0<6Mz#?jn9dxQEWyqiEPGK;|m;(pbD*ax{E=q6!1dkY)ggq1*F&8%L
zMAhq3!A~8*wgjP@a3^l_Ob*uD-=au3vO^+?{*j4uig~E9j~W?MIxyn)iJI;>+v7fX
zJ&Tfw_OSFJqM^;|q%w1=LA(_LazHK5pTop_<6yvJ-=z2AnqUL^y&hkE9$3+5USIgD
zS3#S=tR|)KJ<VEkqmv3nNtK8u4AdDbH?{kvX_yRsSPU)#a*OoY)6q!9-sJZyg+wY<
z8*?D_)c76I8X+Dm-h6OVb3_)Osj@t7u{$X^ML#ET7MbP<3DN)I;Ha!QT5Ex7<CV8Z
z$V?q{%byYD_l9Jr5uu7;HF%VzV%!VmHYPc~qm790wY-ay6_E{Uc{O!-wbs=n9Bxb|
zfBJC;(9%agG3b4yeep(4zC#n6(1(FKA8DC^+We=X$GOb)&BvhV<}DNQy%MN}+;GB)
zYP_ufg~78%q(-I6yy<jOE;$JV^FLb^w@Y58n}73ui`KnbC^Yj6^!ZFrH_l4Sw?}Zm
zy(Tnr@It3d6!=Z=KnvE=rTHaIg~nEx5p-B-8aRn`h76eAk)=9!n{N;gqYY)|UM~Vx
zHvRhd6oA!lyVrhvSYED^AA6xqLP|CXOhvzvI-tCY3MpFzn>!UF*&wt0LAW6H-&e;X
z>>|D@W!8js4fYzIa<@xGv?yohuF93;@olb3VaL!g_&M+}UXfZ=<|@_L)Gg6m9L&=D
z&sG`sXa|}$KYfJHnw^Li_M=1^LB8tt$dqD#%rfm%+Lb`^x1QwHhbhkj>QU7n9<u2N
zK@#nV))dZ+bYm#f#6;Hjt}&rd0oAeCu9ulg&Ml=o$Jjg+@Y>)&yY=HZqATNG*){sc
zPLz|+cns<|NL13!E%mRupZnch%ZmxNTzpTTdJWHPKrYMJ+k1J1Y1HoWy1QEfI2^dT
zZvO`Hori8BR@N{8|0n6IkEjjkz(IMqi#^@D`0t<uv^Ml>XBM_~zx%@4ks$m#CPCuG
z5f9^h2xP*hWm;gy!G#=!Jv|iep^I3uY;rlHtplE~iz75J1PbmKwWDk5t#-vo<)I{<
zs1WvaiMHc>$MV;_)eMq>b3_IgTHKiLKCaq>2Sw>i7C0BZmn>Mv#=FSRR8DR|_ieNl
zacONO7epVdTpYS1CAyZ(^4<*vf>k_U=AJmy($x;d(7bjgU@yW{cNcs#ADzA_;;U!h
zG#Pn^{@PFX?6ed3z{&^QAFg6j<2%Ig&x0?@RKMjGy2HMH!~T*U;{7l=!}~YV6UX^e
z{Bc%TmX*V}bCx;>+`H1Pnn6V#Tq9exbTsK%52?1`Pj$%m;cCBrZdx`Rc4XORWZ&!=
z&?YIk;mt6)1e`Pxa?ge?T?XDEBI_FK)W6No!4{+lr=^pa4z3L)L-Nsz2hC%~x^2+C
z#@xydaQ?#b#o-qY6lYD%iFb@BorHuc2eFh}allW=iXfpt9s+Tq4;RT#wo7aGh_phx
zbSH=5LL;gWLyhQwGAROqTdzq_I=Gs8N$cr>S6FQS$ToZvR$BbTEpZoaPB;f~MwOB8
zkij>Sp-%C=^FOz{NAa{dM%K`1iu6}+ivIp|mi5z$^1zy_X+?`uG9(ElcDgw1h6>D<
ztd(Ij$v)!EEk8cK1xD@VZB<RGfSHg4IKT)$f@+w-{XHN|^qPE9lXJ_XE?nruJcc^L
zq*?6Z=}4^*!55<bKyLS(Q>W^czsMms^!l8-*FQLE5*z!q8Dj#9Ug!ypFt2mFM@Ulc
zvlH(x$KCBV$3XCcKG*Is`XNZJjHSwJ(zV^v4#+KALZKdPHEpAJaJ43Z9@fla7=>Y-
zpt3b-62yYTh-IC-!;>6-WR|d~DlZ+JvV+G<C-IlYLNn%flgNsIW0ZK>5lSsqeL%yn
zIK3Y}i;ju`xy#*w;S*k67%c_H@QlOEU^^bY=;*NcvCN=;#Ap>%9FCM}{xK#vAz|$X
zjQv~L>%JQwz>-?kn{e$#iomYk+0gU9O&Bw9_}3kQhq*%*31)$N7bh@1zC$%F9nL#V
zl=?73a5qb!I55oy|25%OxQR{eEX$AK-VRCY8QU%Ed8oJmH)w{spW-&^qNe>!>NLuy
zvMh1|aWVBuaK(b#A2#PjK0825lW@5%CJIZsIe?ocUug!fX&{pDB0MaP@coI@K_L9I
z>3-VqGd_0B<Lfrus#mba0B_r7K~~-tgcj_EI(z{2EtiV7K~ow5=89Soc4jKThoDzF
z;I0oW{Cg~pvb^KdNMFV$=a<`uZ>Lf(cQFo#r=w(-DG%`wTz_rJvlmc$cZ=6DOgQNr
z)tl7bFFL?zUWx0Uy010=rxm27r!yfH(5Bj?Oh%QIpQTrE38P8kqYmnrsyxO{BgD#~
zS@N+<z{jxB^wDYEM86~}4=0TS%DSRwq#Q}BX!=7zpIj*k0IkcLGKqq0Wf4&_AGd!|
z*(D21!$^je0EIFjP%2*(Anm%Wi+G&6<;GK-D>A@fhAMeW9|>cUqOxXI$2h4?kZ;iK
zw&f~r)(m!CAr$gAZKsJ%&JXA^^0rE<rw?X*9)~~9T6cic)v=ZbW^CduhRa3vN0BKb
z(<2KB357{mFG)cjyvMl5o8a#WyzO?ayxx?6o^1x`*)bkr{!91~KgiIn^@V{pvJ$r|
zte+44M~!eB0t4!mV9R+zwXo9pb@tf?WRuqf4(GstWr*@zo%afg;c#UI_hN%b{gyH~
zsyACSmx~yLFi_=>wbF}iSMo~Vh^aZ7!Hj1vWp3=&C?n&Iw@U8sEuOLJ-n#nDU+}iq
z=8d&^Ns3{nV%#g(&12TQW(yvs5}vOWpInKAK@X~#p{147p{yL~zi6@0=1JMiU5i!n
zLlzO=Ua^;mX_97^QTl#z{?YtQQld0z+PCHKA}hJ^?mK@j8Y(+gC2_5XNr&uw`5my!
zTnWKB4OZa9Ed1UAmlGF3b;ts)baeabn=fcqD8Oe$F>tug4V>SV^QjX1jSCfjJWg^G
zL`##<3#XDOn3kU0p2LM+eUz%i2pDnsj8K@R&1(`J5g%x3OTO^@@Z<DP!*Y0SXYG=^
z=%;H8#%B}rb^Q}(U&;6QAs3__M?Z9zUm^U^pkOOnjh<7E`1*{R8{`BESBI3xzI$dh
zG1#;^dpM|FdahrO3jj|*p#?geNKI>S3k=^W+6=L-fYd|k%14RdTZ4wGx*{{UG~3J{
zV5x__yZDC~zSe%1g}$$Jk*tnRvlYV|soJ^ec|K;ah1kgh9>%ecODtEa6t81ozoG?l
zk-d*Tz)T7Fe`r-<``Oqm^4w%>b<a=+d}S*N?m>-AjeoXt=Qhgy?L^AznO#G1EMMrq
zcbQ49Sv?pD(jqLPTirx+qa0o3at{13*XfU|nv)Ui<YsX_YHAtim>^It&_T<pu-}Pp
zo>MSR_g|0ytTW{Sc67{0Apv>zA}kykI*m6fnZBJqVV5e5AA;<vzE-UA;!{_-{S~RQ
znp9HcK&lpaHORt_9^6Qy$O%Ys(SUS$q&8aY@d1_SH=@qy_Ys(n2xODx102roNFuE8
z-=0HC)Lg{T{=!THxWZW#C}_xqu-eXXZD5RLULk=Ks3J}(s<=?=#^V}fA}x79sn$mD
zK6AHKtxH0GgSr!PeKUF4d&AobhhLb#zbEgJ0sKHT<%FeLIC#&#K7P|@VfubunT86f
zb&%*OQrzR5t`a}9!a!Kbt9$EY9F)-@>r=X{i$jmPTPvM8Xz}++SsTQ5p}Uq*l($u(
zOy*yk2t{b4QUEq^fg@^AB#<qUUb_wCm#26#y4BOt1*Q|Ku3bfzt4KH6+!3hkqe2!+
zLa?Q%!rZa`Ms~wMqxR;E!O^hF%Jm<kAT~M|Q*SLHgme!S*Vx*h<n;Hf4p0UY2Pl)V
zL=|Ex2~H(ODu?V@dZa+`us)Y<wdNi1$qtPnLKQh7m1m_X42`g2*a;5TVL{te?0P3P
z<@M{7=l$QhZ+<xYbqVF!RKr#1yHQ!exX=c|5`HgP2xJloLN~BIPfkFBf==0>8L7R?
z7P3r{R7HXFg%}~a%+CK2vIeM1Fyx}?b`7AkI=gG{vuBG-jt>kS;j+^X1D6Qs4I3tT
z^jKgjJ1r5hQ=1zwn;l`3NE9$HyeETD=db6i)N9r`SaMgnsufa&SMeFeP#BNK;6R13
zyVYTSRihv1V(kjx;Dtf=hO%!gMV(*DS+HU`v;ypPjnu!(aGCSc<5A~ZnbK3}TAFcb
zZWUrlbu$j>ATiO|qbt?=ZHHa2eHyxttWpR5>dNjK9c&6ABD+B*YV*|3uQ`NlQgAab
zWqe}vbRz8BJvN7VE@pzCtwFm3`hJ(Np#L-r`U)&uwD;95;68}4;I|74e>oEVfWpa1
zaEyyS!rWZo_Y)?C=)a=u<fS4nvaoM=Vq3yznl2o-IR4kHTkBwQrug<d;p=bWrJP?S
z1c+nyeFrTnkGwrq2=X|l38BPBtMk3in$jZAz{FB8909N;??mNur{qoi;qBl39p>|&
zI$9Vh`gZ8)_<qEEFaop~kn%>z)D&MxJumKb=d&OB@-63GzEyyg%?>>Q`^W7EK0(3h
zbgJ!oB7<{0)3#<4WZR)4@Uh3a;6RN$8(n-~=}*mOz}-u5GQLrFf{;{X{NjeJmf)b)
z#Kl;RM@Y6ULJK+s|L~Tmu`_gtJKm3#p{;bSRd4Y*oP}2t2IIY=qC*Z4;*?mI?Tg9}
zm1e5-`hZGDfxvF`ITU!SgZW+ld6M{_14VFYNF~-{^9CzDPJNRznyh?!XIVvmk`~##
z50wS13&XiA_d|FnV7$XmPAwR{g*<Z0o_l=>5(38d^!s6BhQ%@E3b3-pTFQ5}pQR+s
z4nGX%eeQ%KHpOl-TDZMSK<K6R4O>}qL_hpt;OLS<2nvc=gihY@etj)c=9KC2jf73V
zUg;9SB^bGZWEsn#Zmb+B<Ck31-q<NtPAESbxNcjwId7UoKWVRNRSW%A>Y7;a2>V=>
zYMZHayK0HD@r0j}+gdC)XWbREi6wD#cbeEsQewW$QKha|PFeU0xnAk%2!V(Y{ihKn
zJgw|_ygairCwkLGrUcW+1_mz@y+LlPCkW+RnMxxU@>*=Dkw>o=f+Qb%$OBuz91EW`
zlBJ9Ru^D#nET=$&%T2}I{?3F?j1B{+3;MxTre=N``#CSce+aC*^p;wUoaiOKllmk^
z#!EEgOn^9Il!+3XS_2;ZVFT1-C^|is4b;IY<eKLcBS>qM=@5?Zo9`$KRrT3bl=rIC
ztv+VtE>@-fkeC&>Sf??Z%e^)NrsCFOzsR!M(=zXWD{|sv--wonn5<jHZD@fPn)UG@
z5LOzJN1!OSAO?zJBhQs^l;u*Fui#7RltNaTtI!$AkJQ^nBKA#Db%8DWpe$gjg|3;t
zY4Ho3>xF86(PCsvJ^d^g-Q7@lPA%dq|F<f68OKQ0)57bNP|A)iQV*i5AR0+Tj7&3+
zfitb=!*RLiFVeZ4{T?k(-iITPt4xZ1D{nx->;JoD!}{2Cwd6EFm#}x^D|AK=p7F`o
z4ybDgBrTXnoM&EcYCt&fdwWM#M^!Q1`t>M7kE|dqUgBez_M5`pRQfG~!AB0|{)9?(
zeO}Lu*|Upqo?c1up)iMNVn9r68a*2cA6W(LEq*~^I9gzEO5-ne?EA}6x&jHd#rw0*
zidh(Vn0MU<A++We4v+0DyE`u_I5|nSJ-!9FJhJ-b=$?qVa8Co;h=Q!5r*yjds>#<A
z_)=RsHdWsWBEA@NmD%LH*B2C!A9g@>*-h)KgN6RBB6OjAKOB~hWjC(J$bQ<-J5vvM
zNTA|;Y;_KQ2gIe^b)SkK8<fm|dO+Tf{Rf4RLY`O(rva^iK~v=SH#F1}PVIZLut0cP
zgv2wDcT4a4&i@qb*#>c0b*QrPD-@+4aQmL2LDaYHUOF_qG5)k=_mT)Q-}9M+Q4<|P
zuy7OP4RYps7llF{)S|)I-5qR3x&dPfJ;zMs=SX#CEthF#=-l*cpz|niZf#gVpCp>A
z7>IPlFVd@5O+wI=#Mpkeyt@|gNVOkw!Fr+ch0%g_fhkne+bO~il<XRq;Eo+6+DrNu
z8zMG>>xlq4Y%ohfKoCnb1U7_#a2<!j1wTpDabNp@Tvo#?!hotps@MF^1k(AKN2B^Q
zo0nOEQW|ZYc!Mn9z7BVbB$GBFG7q@4_spOEW##qPcR!pQ1X=2tjSIM-o$sow?^O&+
zIz+=V@^JVy4}F|DOl2E0e$7x9+!<W(&qwhG)COzUU^H8<2o-u!*7<Yl7FTzefZp;g
z@iHOO2#_ZauEd!d=INbM1JzOY!+)!zNe~Z8gF1{-fOtyAZg*V-J{K_>xh#@=D?CiP
zQM@1N^DYYLvfs`tBJ{aTp^Rc>>V@wA`QTb3ZB9~Za<t(>smX>4`7^QAi9AS;9h<Nc
z`>g>3i_$I%I1$Hu41P<Frx?f(%o3l(##iRbIsO0!7A4D2eCvE6QWikfZ+YpLpvPww
zbyFtxt1QxOBdmqnS~a!ZreU>7`mpb%QX+X!*+{Y@98S3V8I!<Z5=w;{uljIZPez1-
z7GoZd(plJ?y0I$UYjCH-)h=0%>OkI~S9y8$%HWkmBf%w=J_M32wOnRc2?|dfvn?up
zDal6F=|pJMo>l^kG6f4c^d9aj-^NuZcG^vJAr@TCX*x&NC1wB<8@|)XO=9AhvYENd
zpL}UNFGQ!07XQ`}FMnyKDSXpZUtr7EmR2_W0WQrKA?LvfbQfAcci|Fm+4*=;tJ{->
zx7pu$%ej0&EX1RrVw+C92llwuXO6GCv73Oy+pt442^E~p5_^Kt22?G5w!%~_f{g+#
zL4ni!YnmNvW`K`l_XVmz6+*DMFWXggN~<vuHBq%GYuQIUH;GOVE(GL>@v6Vio}!pi
z+JB5AB#;!61-ZHmp;}mQB#1x3pC2XvN_l!anS`VIY3VsOM#ETe2dsK8F%lG1W|=as
zB%kN&Xd*YgpLZv?_L8ALnzLORnd(6!V`bH-(Bm9@H@*X|2E{aBVIWnrY`J^&cxtj@
z(-GWkqRz825`br6F_32XLpb4No0HuiC?OiE{d>UrxqG2AQFvRSX|oWpS};x6#l7@{
z?L|4xZ1%I?*)y97;29G`m__qD+Gj)x?T>#;cGX1U+9WBd)yV`J^Pz<NNm-ZL8!KYz
znbw28-^W+Rk4|dVy(6-Y#PKXa{&1TCOhhoEJn92xd)L^__!0fjEK)V!xYs=QP}=1u
z5tKC3?6n!MP~I9z@DcF#VZD{0I~$#=9+6M89%WHzjNB!DG!x~Mw?EohhRl0~E_DB(
zBXQ~pqmV31sQlWsHnSrQ>$2<CH{qcH0s$mGAKX>(L8@haBNw4GbS{)e_FZ)hi94`Y
z=sO3f5LEe<3Zr76vskDob2_g%p`nWxpT5H}ppq9Tg~abqX8_0SHn9iG+iUD7vRNsK
zAjPB5N)KNh<hroG3H;_Gh_CTgsLT%D%@OEN<v`A1EFPl=ns2{XEPqySczz@8F!!CW
zeN#`+Tm>T^lHA0R=<s%5Z2=~q_cww$++Acka@KOANvy)a^2Z=Dgu{1roixhU8s=h)
zWAmJL#m6;usQWUa=Q#H-#zndZAXg}iW-hQ%_Mt9eA+RPc9g!!dUfX=6XZM>KImLVC
zFeATfeT@z4R;uV{z><GW*Wd_)QP%mon!Vz1nG7+12r2;Shl1-FbO+DR<J_<@R~ZsX
z$c#T;9Z|w<Ue3JvTh?mc8stf`v*TZOzrgx<8S$AC2q{06#9{G)F^Gq47moAgmEkEQ
zy<7LDe#KVgM8+PG&|V8uCzaO2ielBBR)V!(*#W;o|K6v#K3~doZe2Wjd9j6aWsw^N
zU*6i#eXV=T-!BsXGwp=qR-t})ws{wy6$W~E<bFeypY3v(xk7=Jxgsn%9SC1u2u=I=
zmC{0ULH{1tFH7@VsE;UJsZ)^U^S0%ii%lwv%G^4Om{!pi*KKgd?8)c$uND1>mdH>m
zE%mb+hJ$aYTlq{GGKtyoS!P(BdJqJWWaLujIUs^S85Le)Q^<@WA%Mt+?R)<*ksaZZ
z0ndlk0oTD$^2?ei90Mi&4~J%lV7-(q;v7jLId#xO#@Lr|x7-MyN`d~&J(ejsgV^w8
ztuM<Uf0X$_sFxW9EWqNSy4NxyGzRB&0b^&ByZjVk)R*pg+VEtR!AQ-Yh|8?|W|Amf
zC`W}ESVJBX&KFxJc#2}-4(ZK2xjyCn994EC3=d%$x|NoIQKfU2WbYI(s(kuqR3XDa
z<<eNXpVK*sUI1^yDUoJ>(iAE&MT2b9^pXIZ4(cMKN>f{Ek*jb}_A^#YPjwH-!>q~S
z3j@33y}dETPUI5xuZCcdg;bvN2g4vU4=Ug?1;Rd$&Y81Ty_HVJ1G#Y-Y~*LW=JBl+
zCffAsm>aP>i#zBk(<1O@Ve`CD7xGQ7neJKsI0m{g%Hw>88SEwskKD08umzO#g$&4Z
zwPXF8cC^bckqa$h3M2*=rU8Eo<2rk-j>SoG^StY=EqeYmt{o7WXlFqj#>9Ck*?cP-
zmmXGSU#H|(1XX^e<DXFe1ewX>=P|ya?QZ^pw*;|GBw*5bg^_BV=RpWHESN&Sdo^EZ
zMoO{Q+Tk^qp5bpm-a$q;V_BAq8@h4Sktq@1BIf&Tws94`kh!R0z0oMKJBS{uc7H8{
zYmUZ@UejjhIm_g2xQt2hxJ`et**OR@^=3-BeW<b_gJsd$ulbsl(lsR1ddD7{mHh`2
zBI6L|ikn@kKKB>xB-1eNheTlJ6=QDY0YD|-@F&VeG&4xGqEF)*5D)qAZpEkdAT;@2
z=#)b^5e{i7v$ZsBLHrpGNT}{PyT1oLqo=>r)u444)Emwjh#|yH8FZ#82xHY!5l@fd
zP?TmgFBA{M-SO|%#M#R*E2k@qVoDX-uHQwwF{2e?mAwnTqkhJ!zJ^>lix53$2I-2{
zMNx+8`M_pB;U)VI4v1po^D19{zY#9gEFP>or?86*x}cE^3>1J>A#<}>0Aun4vE5TX
zb0~i7{DP6fRO4;+K-8Ti`8%WbrXMevS)XN3Kwxid*&OKUv*nf-{z{N&eAZPG-}Ai3
z5X%o0Ze28M6nj{xkLwUa*EzzehM|(!`{m^DH|mSUnwME<Hz6*K6rEuo?_ghA;S>b(
z{fd(9mpV97BqY)<I1m2L(|4^$--5z)iD;+&i2xoiSlLB^8B}`!U6^(D-jixpl0+Pv
zUm`0``p*tCbJ)MXz%?fcS#!#v>lLWD>A%eC^HV=7U24sk|2;1{Q*DdXeyP5Uc-@^o
zGITbRmrFcjLizAXuid3EZlP<07&}TyDKF|^_l7O!9fP+>SbdB@P1w#vo7zEJAItFh
z;_-6OPVn{jzC(iiKj8P$eMPmKmiNx?_aVzz@<iRNfjoH>S|?<ivP`%Zl(EY#{HuaX
z2E9$c3~r|PZkk{C@4mh27P(VwKUqtnKVsQ1@WYfoT_-2)KRHqL8$7BxU{ArfGZ&1@
ze10FHjkwVtQ}neAMcyHS-+vlYxJ~L;b$z6D((P9ovmLHR{mgJwRs_USLaa=hX?CMk
z+Jh4u7r$4%?Lom(mGJ5<1x_Lx`6A`0$uhxD(cA6;#xgE|u-A{x^D>?~9i7W_2hOhw
z3`e5EGerpAJ^Z?~_$f0?!=%0ozb>H@am8>fR{J`t*N=M76SIac;_zg)*AGaKqn>vD
zV8ZUWjD!PiEg*<8h7CV&I3Kok`z@67>q7cQ1gLsmJ$ZyB1&>0Wkv!*?Z~QalSf3|w
z1SshO8NDEUaTW|hv$HPIxRyNbxc5+;c!vWlV`S9G5_E$2Xw$DyoOLj4Z6(KVjja&@
z;RK`-Dv!tDtw%z3zf<{*H>dB@)Cj~15BxB8H_BY~ou|yhJ$#OqjVv$UMf<@aIW@Em
zI|-dHyhA8lX$;y4+L#<jN~{|p7#rO@)l<*QnLReQ7|q;M1OlUR8UlkzIvtF1K_a-g
z5=}p6t<3RULmfXGXD2r3EaeY#%lfukra9UWo;Uvjsp7{)QsT9qd+tk50dBN+Aj8yE
za`4&F)Ix(-vAZJ#H-a4$X%yG@oqiyJ6pn@CD>WM;3a$<QQ3!mBRJNT46b@pguWL5i
zSiaSXBzqo<-1TfR#T%AIcTEw^OzgH88gcMD<PW4{KpZSfN2<Purj+u_+f@pfQ3kax
z$c)24e$Q1DkAO29(J#i28Lp!4($xkV4cYX&A-;WyDi|R8UfQblM$yg*xTo#t^HKCU
z;}Uf;`hh8v3BtPiplcO}`1Ca}G0xepA_FRWouJpke8V5-e_Q0na-)?Ga`l=AJjc#Z
zEM3mEJ^hjGdi?u{05^)8fPxTq|MT;R`0PEg?pVbYxNYw3);p^fRGr-SIpG_my<gVl
z;=Wel6QWU_&?>0=_ue5C4#@vev(BNi$5x7=w(M>|EgZ1L+qB0KV<K&Dcj0M|=|}xs
zmgOam&(1Z|1!c*~F3EFD?EmwG2?0`{@&$OJcCr(5Ixw0{8?S$;DJq>OECvbdUAme*
zRi2<NfaChyJ~tRc9k0B15>uC~_79O}5&j?Yh~-TE@RK=kQsN)K<O?wJm1mqNP=IxP
z%|(8qY*67pfl$J_;4=|{1i<X`-xL%qN0l+X-f7tR4^Q=ff_}W?QON{gBR6)HPb3o<
zH9!xsW>S9t0aXvk8~&s2Nw|O$?L;YB2wGrIJe~g&SOs{nx%0!!!8alx4cZ7&Jb+pv
zO@;f9XM<X*Z~gbF==8dc<@clrvNDK!8Hc4r_k`k^1*j@dWRqL}HGSYAA|$w;pd=_I
zHl?G$pBuq2Q4mSh2E@lTW|=Ol{58^_C}jZ5<_VXAhe&}J<N}~uP{VO5@Pc?z=_b7I
zl*_z;<{5yS{Ffx7wtw3(4uFB=#r^|3p#r|>{|h_$zigcUf3tBK5P}ew_qC_`y__fn
zUxmBNYd&$6l*bP(>hPY>j9`ROizFz(ph&9zEuyj0{&=P3d%K~U@VH5{mVNs3Tsmtu
zVp8%yWTK__m$76*OtF-r3;<waj1<#?H6k$jKbWFdPf_E4n4$%ZpOk&E(J4t=kpNsF
zD#~d$YTwJP>5~OIvMRLr;RU^-h;Mzv;dfHn@`>eu2YPFF3J!o+00X!P8dgIeX}4>h
zSQhEbM>5|;zqB&xUq9qs{srNphCbnw^0`*|Y0RGdQv|sFjG#_{D<vW%&;k_)tL6&R
zdoSq9k=>tE23D(HzSdgPC4S2Qas{@;ClHqFwpDn|!f6i{_u5yFwdIN89>h@{qLld=
z&j#fD=-*X`yeIduiwJ_RN8D4GPR5J@Dnc-71NlGL(5lS;f(@}(W(TM}vhpW-fK4j1
z8;`4%y0x$08vZOg+8Y(Wqw^m8fT9_$D;s^+gI>(}nOJ=4GQp-tt@$!iE^`K-z+gI7
z=n}8}o<n%)0p8T!Gl}r2|6@I}A7Me?w~UICz1I%f_*2b-=-!;1H~{+V=>R~(Goc0P
z05Ll~+*MB+95P^~LM-u)SaAje>aSU~hf~~I{Su5k4e_Q&+sTD#el~A|684&Wr-u}7
z@3+;&qYp19p-;TKet^+oF)LRQb;{Ys#c^=Gb(CbW{QXVW=7&bwT%`*}bJihJXzYJZ
z6TAmSs|Jw6COgp+$lw~E!!O$QdMyuQi)08xU2yVOCR!RZwSg+Bt?0qv5#!^Xq3`(#
z(b{nv;#$ReK*Uh_FoPSxAZK}-SbKIf&-D}up)XZ<yc>RltdYx3kN-@*%k6qOA186;
z*#vMU7*K0Gu>4)pW1@N!f6{*UzW?pp=89hmZvPNkvrlK^KZF)i>gi0RxO6i#1(8$l
zCy+0W-xdmp*>JpRE8?L(naDZvi;jLSniWlkwDX<W?+tFQ;*>&?s2cba5SW(cvprdQ
zB^S7N(%p*CPI;~Uj>T9?du7g5w0q<-4f18Ut5fS=e)B0sd}i>1eZ`w6SMPrSD?a&#
z@X_bKuSMZd_fGa<_*5FMXwe-@9JOt}l&tw5$d$;oY5RIR1>W1H4~fCMKRyk<#q4pf
zU;gt=_Ta&>WcL2r!}Oy7;8GGg9`U3hL5-X!R*fb|+Kig2Ayi7!Yrme)!*(;t(tUL>
zu7{uSCrXvP;O`ElCWK%oNS0q3(4o%g&l}(VVBr4lY~wEC--ou$ACG^}!uQwj4Np~l
zLzR)jvL{(-x{xj#Ij>YB+dkKu<Yj(;SFC@pZ(a8PBGVX}y<nhDIM^-emPs>x{ys@0
zWS~d*cz%hvbKn&N9Zg_(Jx)->6V;2C#^pU8?$r~Z*Owqd;t#+~qj;fH)y6voiPT#s
zVG{>zUR8L*(DfGdQ6eX6O!QRDQ9J|yFZYD9JDd@{TYCnoq4%oxUsasJgI4OL&lk1N
zR^M1svQ!EUyZebDkPxtKn<<3r$%^VK6X?G+`FhCvAJg7VMGiYpZqi(uTmfAAzjgn0
z&JP3#L|h1Z(es37lK>bc|G~4Nv-Q!@J;nS1D#>+6mW6K^Tsj6p#m2*h|9oIvdku)<
zch@~ei&)DFN3UCQ%L2u~2=R~peGz~w40%Ev5@`V(w>ai(jn|vjFwmhfwPVol$iL^G
zYfJ)<ZGhBU&s+v?39*w#5Hvgsh*5^VW3{mF=*<_h&Y=14y33e|5b^Wsqe7hQP`3}i
z8Y_rQ{!wsM_|7Iqhr9mnB!Z|1n3Lnyi@awb3{#L_*#z9P5r&k^dp;bRu(eX<4C!T0
z6R7^%LR|8GU{}NLg;V?kwqg8lU|Y~eL3sQVd1!F-0BOA4fn&$=&1jL++7qRr1f2*c
zRB?$7P(flX(mb6OSVAT?2mnOHzHVhoe81M2LCP`!8h~_v>o?pNPC3%g`1#l8=M9qk
z@=a&)+VRr88%?7TK8G<3DfRo3%{|t%&(JL;dV^Xp^92CKV7<RK=A?<F4#HdqtQFq^
z7xW8dAV&Je+2_^q+;sk;S2aSlmw|Q9*T(ntyHl>qv4kqr0=Z@%r|Q>&l{gX$B>guG
zFFR(IBaO$Uq5fDEEb)f;t+xUjZ*;{i@4FfPc9b7=*PHGqA1oH<+yWd4i*~-R(k93H
zF&$c%vYs_!EI6e;k3U5R@BYq&$l-rKcF1m4M{xaly@k9miuaS|31L^mR8wp9G*z#@
zq}n^818{mTBfkgelhD(xEI;2f?DqMQIjiiw7<+=#R#w(lb>7<N{~)uUuqR_1kt5`-
zeCKrs|Lf4|xhFWNjkNX>!$fd&`ok~t#-EF)Jsx$|ck8o|^Zm{@0XX9QdBseWx1r8T
z47;Bu8IAn2Q}<r+=Eq(~5ov<Myp}{{?|p7tw*D6j;6eDLxZk)SPsp@)BG3Wv&v}SZ
zO&7}5`FNee_k|b3a=v}Q6Jq&ahRil^I0U($0FbJ$a7De(pE(j3z$N$QTh6Zg#~SA%
z3-z1K-XBrxk0i63jom8Fey`HHRlIA}WLWbS(VSfO$^FM2T>rhC@F(_!`1XARcX0eI
zfH>*Vc3Z~w8t@`AG0O^Fx;5m4Z3b`cLpNh7H)i8yHUsP~N)tsMjP}!fd>h`wvVQ`!
zkI#SjmftM1J2&j<&}VU<zPLF3#g`FrmY8rlFXHBSFBMz-d-8ph@KzExcJ<bPotTAB
zHZBZgDYzN8-hNkPaf#2S@oFH+5q+>jVOTEYTdjH(m7=bX{N(-&!`cF1o2q)aJ!r;n
z141L`qLe^<jtCkv6F~&n+8JefzPtO4CdMC2CX68h`kiW!<o`Hs&v2mP0gs=z_1ntv
zBTPRNj^(oXji$L~>|#d)KoAxkRsSAOtgPINw{mFg#5|BlZ(8AK!JOPO2G}y<-<6_U
z#mF1lCNP2T8}D1G4P9Kk`V5IczK@fP+p>Q;1buq0Y?>oYvB-ERwKmK{T=tJSB|{7X
z+T1<>ZqkCxFXumOCqD91oFKVZquaOI_c~Z~<_+B7E;9yixF<`thS1|suIC<<3%CNH
zCtS|j-6QnWg4|BzqPlF1D(V^r<wLxzGYs33Hf-x%qQOAr>o<*(79TfH)lF~zwqjbC
zR8+0$l8F4uto+s5ZqDy|IT6SFR(8+b-xK+XIP+05t?)e(g6I*I9?Mgcl>b!h@QIzY
zWNN@Z?HQ|;@EfM6et3fLTxk=)&cv_<)VGA$$%>~BH!*RtM|OE&zD>S$Uh|&5he%t|
zZ#jk3tkPf)j6U_RqC{?JlASB37wVnTfP8!8`HmNn@t26iqNLlXx1-{to~_zdaw(O=
zo@?zPKpG^pc7a^I>MQ=)0dD`Bv{w<{KD#A~c5+*Yx}<d|b(a$+Q!~IArDNPuqL5*2
z%=6Y=$3Ita(NJ?T@pV|#i=gUazM~RU%J%RLA{+s2VNnN%Duj(s`o?k*aYXHVGpbC1
zoz?eW#manr*c{sK4l(`<_AFWsNdc{%0qJQZD~4&fS^wo!8O>pt;fuiZF`vgkn*z7B
zV>LnpwUVRwA1#xvUm7-oUiMIvWRrBcedH4GrGCu-B=0ASts1qZr+(U4Jojz+Bg(zr
zn6KCodw)MUjfg+D>mxz&L}aR^dp{^b;F1kshX96F1V9;y;9NF7N~7E)?I&UJblmn7
z`)o>xnL)Gku}yx&#-$P8zU6yrAtX}qV2>3Y&FJWQ^A&&nst_4<kk<|n0FB<zJVYv-
zd?>NGj_`y+CAdxo{!vB$$Fapo3kSP}RHAI!XN<m(b9{gAWQtIk+)v9+zLfD@f<0$7
z{JiP+_yQ;c4yFh!A~Zsll(N0t`t6Hn$2H<-EBoPnQ>zd&|5ZhNj9-2tK<m+*piCkG
z;RPjcQz6~!P>TxSNtM(k8XzuzgFGvG&y^P9-j}0d6U-wz-cs~stonNeyO0oF85gpy
zGY5a$gSxB0u_J?>TlfIVb2DMU@CLxB=%zWnm6kv8VO8cw0^>_NN&y!Dy`yCDK8XpC
zRJFG%#2+*_^z&udPa^Jk@%mG?NZ%1uh6T#@7^~^1zYppUnno-Y2BLi&uAJ6lMX{3e
zX90Xp`EJ0NN2b#-8ck0&@RwZM*jSgvx3p|{nZbL6j1%dFf2D_WAdjUQAOdtfA&Yw;
zPkcPEBY2J>Dn9UHf7aW$TQJu#6_$D0)9TK`>ZIq}mx;S4kE#J&Hb#`!iCwmw!2g@I
zhv;gKLwC#dtNgp1vyuCRT81Siek0y!1H}0`C}N<M!SA3!O`^WaPqQ}<j6(Q;>!OnC
zVSU#1b*JH7+qgaF@lLh&E?M&Dn~F^GyxC+E80frf$?_<*0(|$ul(wun2=fAA`(=UI
zEuoQ=sGP`@s-2RhN!Pqc{oPCIvX>stS1{(6NWY0%1RTH3t5N3y4sLD2@!2%nmP(Wk
zlR#$I{}vCfN1){Ere!UqSf}DCf709SY@}1gp(4|6Pq(|kR5AF(Vwwu!T*XPf$SmUk
zoB+@CLi^#kjWWduY-tRjHn2PYezMy5>y|yo+IF*zo`U@cwO-UaF1ZhN=i9X?JtE1&
z4vg{zkQLYKpbpw*V^}Bam<JtR#6FzW2xBMKTW#D<RV}syJ8VsJ>?<B<Oe8?wkYY(T
zDHIVhGm`nX^f`mcu;!Do>y9vhhP6qqLmLbRIq(5i@G=_=4XX)rg@^P%s=#x+KKJrz
z+CEiBJa^_Jn;6-GMgcXqC~mvY;{TgAoPC!HY+irgVy1}e+v?V}?Qm<K|E6fxog|sp
zNFcy*wAAEV*Gh+j1;p8lG2>wupLX^C@aDT>SG~Pwka>0HC3iD>%bAYj{A`u$_{3UK
zD0ZNrbGy6^LXZ`s!T<70WYMcBlnsV2WrJFeS<3;z?*lHNDQ_ke{0isR9?$4crqNIR
z9P3l!1Fgpar}7j7xM&3^UI|d);qVU&XBSf7QpR||gXi9PRd2RJS7pDshl>Y&kB{L$
zv=icMV8>K%8Y-)(RpPQ^C_O}kLPC#PxD$wk%tnP=_HobVLP*y@tkiD<Q)U)p?)aOj
z1cMbQnxOI$b#l^SBidgKEB|#2=I)%^FAytXQTIO$Z4Jj}VBiVHZ?(@q{Xzq%vU8aJ
zghK5-pBj|BC--x&n<w~{^}sgCT1Z;jH-6iSl!?1?fT^?Qpdb=czNoRG?fIUOxq)SC
zh^Ra=OyrlQpWH|1ig&q3&#`orGg(syh0U`MZVxfq&p+9bV+71~8Zqr)nA`mV1h`!X
zz**<a0fam>%6xcpLjjsWYSi_8Z;7vPyMT$2rE6g_aKiiG=n>IUgsJKgsW~oaHN~sY
zYK8f=NR+jnyZf%?xSltEX~l$xj1-^Y$sD#!d7`6!xU^=20*P%V2?<C^%FY2wiO(LE
zkE&T*Azt!Fsxia|CmU>CH0hh?;vfGv@wi<-lOp7}dH?SXc*A3^CAJDgK`~r4-QrA9
zkx_3nyeF#K_y$W(K2YBF0T$p2_%sPfxQkR7Q?vog)07ZIv-XFR_SB1EN`q7&v0>EF
z`2B4dE2h6M$_@qzs)&4tihJL+rOUj4jFNn5PODzae84BVCf>**yjxVT&ol(BjLJeX
z@tv^<b;n3@8)v4PmJSjt3eLeVAQSq`QZJv4;)U^m4YDtn9IkQ#4wP2RxS&*DpkhAH
zD6>8@N8o!>d_)oBnrFrD-jl1kNpTyXbjMcBeBcaCT2_YrLur=kqTCp<(;zJsA-8$V
z!M!cade+Dwa4@pZ#B+=H9|ROj+6?N8SX^*`E(qcvL1h%))AhxG+=ZzWFU%xeswVH!
zZ~|%9vB(`=M{(LadH%`_(Tc(Gta6(ZK(g4L`Kp~J3LoCONNorYD#}Zsf4xLJpAzLF
zXIt(N&!hC65%ZMecNaB*gW1GZP{x(L*TUQT4u^`aILy~48i$j<m3chidY0ZAImH}5
zV}Lot#5Ic2n{%5&39C&}%DnXLhOc?Hf*k(Qp83f;qUeyGF!G<W^c2znQ1kmW;jY?v
zR9#@@Eo9^+zWkj|)}1tX)9-z4*YFA^>b;~ZSN8i&nTuMWkD8twl?fSN;O`NVRb8hB
z*#i@fk$FAz?mOp7Uj=Mx1cwcqzKi1_$jNGDj$E)(vBm2*m(9*lD5_qjN^eL_(OXUt
zah7OA)HDYEgR*J_td1Ybm29%v!gP9Ombtig0RP1OWpd#2xFdpN;!+MX;|D#59nHcW
zErc1bE|UNDrpoQVQ$9r+piWf9h3N*L71^Rf-bVyZAwgRdFjA|a@x9LaMe?uuQ4H|C
z^giy@N;DlN<*>yA1?;(7`=vE<_qwm&72gIS_T_~k|Ad+b{?@y3w@82?CI5ON=;qzn
zsANhOYo}PmZ07yF<qHXt+T2(6J+lzOpyRXnUtl*RLMUKD{*G7pO$0|fH8b8;B)>j+
z03W(IIIR=))nG>e3bW|Hx#lU~^4Qs|OruHkDNp|<PX&c)xXUa!djJOh`Y;0?25M@A
z)dD*&R<b)Fiq{#16oOsG{1GtjIysjY{3{xUvv6%Wh^fsPZTsMJCl`##Uc76(9gq2{
z(P1sX-JR^Ui+3v&Z>xhC5R}Oha4I1%Z@TWXtK0=WeJvo7X4+Dts%-a5cQH2dxpU4i
zm>p%3h}i_*?ef(XNl?LhT#qVMZdB0Z*f{uH`qr?7>D06Gk;;A8%D#o4?6>iCHeW@+
z$#=&$L$w#M&j&%WQ!=iNxJ`71gr@=&jKoi+Egr_pXf>WEc$9v4RTD(!Nt)?=12k<n
z{h6u?0l$k2qsR52Py+}OhY(6x2I)#FL}$Q-8&qOp8<weHDES*Q^ce53zNM(GTh2^#
zA`URy);fFnlUbQ-R5!ib2fCNW#|sL)^wZ(+Wu$fV1yM;Cd=~SHzqbjW2UGT3NuK$p
zPtwTRQzN;`6iED3j=1OgJi+DuE$&rO6rDt=KLwhax-S}ogdv~-y|)DHg{gg+In3Pu
zJ+!%x5ES_dSXcECZnv|B(ljK?K9^~GLm;-tQY&!<5C~aFe8=o-AP5+G(NOkgMW7vA
zL14L8*ppdHYji#WWTMo~oOC}`L$`l_nCL%EX<JW`$<rn*7>d5YtKvripf?l_TuoVS
zRYWQjnek)Qc-`}X;V~c{LPm?iE}7KtiX`opp`3e^o(I|{>@FY^R*s);88JNua*e`l
zrjHPWH?*1Eu!SjruW_BT5%k1hC-eMv{=el}&=5E9s1Jrjzuto$FjXkSPZ;Rbi@xi0
z$b#@vR6pl>sZx1>1XN{?ge~WbD{{sB6uIxOuufl-(^4l6@DWp=Y{Jue0MjHv(gy|O
zTfNEGKwUPLg~JNuT<o&*u950>4Vc0mlw@1!LSjd?w*DE0-y*_$OAr|%0W!j>M~a!M
zzItzY!}`gs48X|6iWtp5S}%V4j{;BD#WsMBx?P1$%Jp<-1MG>*Z0oz`H3g_V<TFwW
zz5yA5YyXYjXedJ<<TwPOo%f6jfxBSJxhD%vn~uG%8rrn|w$d(Mxil45NC{3UV;vNS
zBNKL|4rBXhUWyv|q;c1_Zn&nDeasu8qZ@n`pl6hEWN(bLy!B%1W3f^8mFk_R%l?+B
zmCthgZH#A60wvZw@}KXtbStUgLj{~K?tgk`5alTyUfeV{E_eAe_#9^6Bp#jiE&BA-
zn;zop{u!b7RuU{x2B2vFq~}O4Y;~5}+q4*eg%<Wj$-O$p=}%s7T1*Z;KQgYr=qPlf
z8^DBj2fb-as-h<?*$spKk&}}51>{|V8N7@os>K;<lTiMH-&INptH(G9NYgP4Z)3fh
z-Z%uEh-U&|L66c~W$*0mZ`Ir1tr9g0LZ_%SxnLG~wds<+ss|LLe=Y+rx0rO7Dy6BZ
z^H-E+#h+Sig#XcElO@HEG1vroUgvTX^%o5l>FLYo)lzF4MztEY>jYJryXGeekk2ej
z`3WS68ALCAlr?aCf8fMjqLlf<tG+Plm4xtK4G_Lc^NGc!{28DHS(S729l1&Wi<p6m
z9;proa7asOVrbz^BYf#EmJBPRjHo-=PK^oV5b2_f_aQZ|T(~ICS<8kMd>v=}wmy7G
z*2j(P{$A(q{=y+olK;=VA<dRxK(DJfqx-!Iev){Cs{KeJY3TePHw3v`WF1^>j}SJB
zn+-0D+&*OMIzAr1Z8R||g)ib1h+pmsd5OlEZe03SNyQ`~R8^$~f^gZWIU3GrPOhW)
zEzXB~2A?CKsfaH7c5)sDn=(B5afiwz^u|}##7Bh1SJGnd?}oiSZpZpyxgQjtN8qFV
zumD5~I8z+{;IRmh{`|f1B3}cWz>-F;Q{+?MUtSI!*MKip_6uQ6*fSg>IWl(3Va;(f
z>mmOU*4HOSfBiwj#@!=qqi>6^BC0x?hPYC%Ui9tGtC$LkFG|R#mw(62#d8%t(By-6
z+qlK1Pg@5X3m|L(lu|VbkECDVy)K;|3T--EqXgaffM0Y#n!g&t(JHyHe1B${=*V(Z
zlZ-AW|1<;Hc3tLGR1KMxNPi^4$OIyfnOKoOS^>)Q`k3@@HXzZUCuBF@MjD&_j+Oe?
z=LKGSTFVbXc|_-4Q6`Mv-YK5;J;(U65Q=`aKt>&ipCiL#te7#1OiPyvAjwyLW7`dH
zzDAJDTePJPYRWWtbLCtFv^XiQeBRRdlpsR9O0H|GIhK=1jN}gx;=|8jrCCHCA$F+)
z>$6X)owcKem9xo{A?WD>sc3p&N&BE9+kX}Dse=qYIW|;ZosT61Dq6Op(Bq@79E{f|
zCKEHP_sr3wCK|K|soTNljRi*=KgOp|d*og1k6=$WHBOyLD!LTxy}9^EPF-c&kH6#W
zkwC*4=8SrPrnnV3x@<)?$KhGj6Dr{w(N*|UuOj_&Rj4MW%;p2XO>6k_`hUmh>oYUI
zHhpg<?RI+YYGxbm{1ntn)nT@;wHX#K^%l7ePdyN+&6~Wj7^Eay0rUPvz<n9<u^!%(
zQU6=28AvKHd<l-92@$pNq&Ou5$P+!H)y>NyHuQ8Zm?vpC;Sg6r?h9HE7EPdIx}-Y9
zS;NlfsMl>#8$RGqg@XeX%m6RRAwK^9*r{D<sSyqs^vLdL{WipY6;?3;CgzGYB{?NG
zvvH%k#-KUF0=q|Dx3!XVffJ<DAJ1dr^==1A?Gq*RMVGU`@qHS0Ta=d%<Rb<UNKwnK
zd$g~Sf&<qVrb5xaNGN-~VM*rTF`tT=!4HYMkh2JWv)288ULKy%dDSLI82b#T(O0;_
z*cB(uq74-TEHzs<!q=@#uTuKgvzZ+0A!X`U7bR~a+8^(<I-pblDPQKnFbWJF5Xj|^
zTT})Z58uyK;+Q2Z<_rjA&JRb1zp=uKwRZSCbv)qR#Y4K$PaT6z?GRQw2F!QIduAev
z$@|NI)*y(!yR0Qy;y+bfPDdKG^e^E+K4$=6q^8m0yMyD2z5&H9KUu-TuHrsO)REu|
zG3et714roLRLh5mXOr?36}%vVquXO97A=;gXzy#3l=hW>s1hz-bfGno1;&LSE8+aX
zDqxfERYR#qwksvjqot<%Uiu)9R4<!xqu=C+HidZ7zWLo%<bhq4H3$yu*iV<303(%1
z209!n?4ZBHwTDv#7OHW4ra|n5*4nG%Ho-rf60PGnYxj$t>u;L)Pb(=x{%jj<GQZZr
z+6yF{xElduA1qcZNO#iRCT#M{gr?T{>F>|g8e=>q2Cz=W(yudihCFAY2S#qcXsIIC
zVQo|oOj=i>M>tI{d&fv<rGF`_;&I9S3egR)B0{JR0cw<|Yk`Ps_3f*lY!v!FvFEB!
zTP7puyq1R-2&D~KT=j$nIP#W#<RozLj@2#(6hopsr*DY5x&YjrD#xgRncc-5llhBy
zA;=Y$p1AbK8p!K(*w-z{`lwLu0c-Rmhx<|3=jAb94zI68mfN9eCYbR9%#Eto=Y505
zIzX${4G7OT1C}j)=Kn3akI~KWQAR<UC|n(%LC!DOjMvY-sn|KxfcDL*G&8C<DJWdk
zK%UBk&I)<#LJ-f2EZ)5%6LBVl{$Ndb{QqI^EyJSxy0&o;1SJ$vq(f;?kd`iy?vhsN
z?ir~8L_oT`yFogJ5RvX~M!IL{0fu-lyzk%rJRkmF{-56W!~H>zISyv_wfEZRI@em~
z-uv2#kAIZ<>7yg-qZ~!KGGh2YOmPTCkVsrpS<=xEH{FD<+;d|3Q1anv*A;L_f#&b#
zRx@dtKgo~myrj}o`%YgBNlD1~#E!O?omSJ)`53N5-`x`wN%PM=LC-%ZWj_oGsW!}K
zz_2Z@x1Z<xd{upGju85jSJ)_$I=mG3^0W3{3+|cWK`jjnz$!R}GI1{-Jdk^$YRU2)
z2PE~b@&!n$)9}ZxH8GI~!supsPymcr^4PTNO<VLjLbbz(fLjn_8X1$Eqn(Ew{d_(@
zJE;*5&BpM@A}G7kZq3_!E^_g4;bZq}10S>dQSN8ufdd`t=81JVD6;!5;rS`U<mOb3
zaHRxoc5C+{mE>D^pqUK{vXL^}>e4ojPbl!V)YM#W16uUj4Wd_5H)r5Bb+HivgMg<Q
zI%%F5kXAOi8svTaazp>VJ$7o&#;bZGpFFWMLHF0>{B413tiIlM=fQW8g2uN`4YLYl
zCluqCts{g}IDE3&uroeAh~3R34PeWQ#X)CptC0-FM1P>6>~rzu)Dy>zSKIG*C*v8n
zz36eUPLZRe^H5p>l&Cg2g#|0wQ~mZB2`j%$!Xp&1OYuyxj#6ovBx`J*Ghjs);B*zA
zf7;##PM`9&B-c8%rxCGAu=&U9Sh^)BB6Xd|PX~2V>{VXOxLyihA?V2IG^86>R0i-m
zLrJ);o%$yG%n>u-?;x*%`V8;&WT%0jT%n)qrDD1=WDNomiOq$VpI=c-Tws#ZG{?;s
zm4b-}OAu7vY5WH@#`>?qw`%Q&y=Lz#O<w8a)PT;e-y@Xd+|A(CY?6?xOTBUTcNJTv
zo{tr1ES;T9-2Pzy@$THVH*iPaX4X9u=(K4ZRw|-7QwxBJ9LeRpjYv*;<YrMEGYuTc
z%N3V6{Qz`bA<%I%`LcwK0z>0Q(XYGlkXC&l-C~it1(H7~I8|FVeB${25}UqL{zLl?
zRm}`XvSG4tw}O!KcPLT|wQr+DG54BYn_d|~3Qim6V=7k_8JASI<Eu|^d&4006pG=X
z4BfTOG1C)lEcP!7Ud(}9t|DQI2uox2qdFG9ciauN73LIv-;OT-Ox+0J5vIRe^kh&B
z56t!v{}pXEf_K`W$;N0E=@3jVD5ez`AnFz~iu+!~PZw21R@TyV>@9G+_;?|Hi$314
z@1)h%rmM{5$WgFLGa(lJQ=>a;qUmm#MJvivoaNl9`Z+4;X`m~z9-M(AKxf^|a3B{p
zrL)+I_sU9}*UN(FSU#D4FJxL7cCBI=`0c3mvPa)pYO@`lCy!CAPi>i!j-`M<mu$jh
z$|=(`7^_uqvDQER=$omQ{tz`tuM%wQs<KmWsA7_;+E>$`)kH~{OrR)qEJQA2t+WH#
zHaa<)xfydFNRIk5fe=C!v#JMB)01-4_#3eYtHOQ_KC1U*FYMvFzPVz1<?BSrjVm7q
zO&~+ZBg~u1sVRuWOpG9owmU@*P0}3pxPlV_4@2OLN>Gtvtg-4HH%{{EPkUZYc`r69
zU<DDI3EP#!ITkE5E%I~S5LpKE4U10=^DjM*GG2?$Y0|gw$>_AsQVDAGWS>7+iYr#S
zvJ7KvyEzHz&xvbD&R{JY`O&%!?#4te-twJ{gqI5^Ju|VgpSIO)C@oMV`5@a~anS^d
zVKSW?{G()Yt{gyFhNd+px8!Q85C*gH5_P{esQzxC)9l$1m%5_Z^_8oF%nbH)s?33y
zG_b2~*V?PMEx9Q$R0uY&+2DP3nIc$sG*^=3i&QLkNP`G@p=`MFkpA$7^ZvB>%MS=1
zUDFZ`bsC3+pj({x|AKgXQ_~!|{6=_P*Qb3d^=L?zV+_wg1k5vIOmFLa!-JQHl3j1h
zK)lUs40dTt%W#=7<dq)UdkHm~+7a}OPG3DR++Q5_Y_vH-P8BR&n>w5!Zgk(yt>%wT
zHhCC+BxZY539oDsirE#})`a}1YRw6gtht~Pd>`TfBhr)R0L9iwcCa&O(%8b_ty(v%
zRI07fE_7V%Y0*v<xv1cZ%f4Mv7X^XHvxF0Qx1+*7nk*E)K&Gt~_*;3Hb>w%f`YBfu
zx_4P{wXwA?OoSwKuPm_!RU-7(oo>!g!Q-y;&x_*J<D6d!y%ZN7vm1|jEctL4;*tL6
z8&YamS^55Zc&nE@-xnzisxX0aXo#_cMq9Yq@YBc6d@+P=Fsg3o?r^feF9m*-M2>Ga
zBmVm`ESI6T4i`dmRIhi>$ej%j0v8Ij0`}+6WUA*YvCS7Q3)-E=bR-w-u)RO|@sOoV
zb$BFq+c)+1)Ennh685p;o==G6sFh^?euojL#cFw!MeGM4Y@3!LX*Q;0X=Q77wAkAm
z>UvCYJAOa9wMNk%DHtOz0l^`1@Q7ouPDTgc3c4Ngji6m@#IQHixzqLlJB}>srCYb_
z609?)clgRi4ob>g%0eHzw@>ulSA3r>Tl%}eGQ`W^R%-ZA_^&5)sO8suW~AXSqjQt_
ziiTOt_|vo5UBq9AJWM)XHmbbJzU@HH`8HRNa|o52*lN0kx#_%s2pCaXDMP|0DQq_<
z{F3Xz$4CXNhl{9?$08gEq2uC<G02Xg_dNd!qkZe<MyPIcuFE^bBQr#dUK3*Uw#NC7
zUjj^lc>z+7bN1NCtA9|mw#vw<p*t*RP+*EDP+1itR5X5DJp?l3VIk+Q-l3Zg*P=1!
zl#w}P_xna-vO>Zm<I-r>Fz>3eyfJw-g7cBN3UYad(qJ2GS0<U8+)s^c7l)Pw3VD|4
zSjP1U%1w0F<B{5@w;?NWx~N<9_FaCX=erfG4scG9#4z{LWV^K)q2&p2K4b3c&N5Ku
z)!mja7Q82$5<}a+3)hJ}6n9!_gi`TdrbICG`d<rD*n(9l^)0)7js_b~MVuG;b8-?)
zuBgIHSMdptUS*{{lvCw%N@x|}GL?Dj6|-R{;(1m!<(jeBlE1d9E0azEUeZx&BlNmW
zu;K4zJJNS6o7BJAx$!{wPwzd7%3?MMI%2CliQYI!FGyUB5PZ9u_gJ52aG>=_i<Y)l
zIfSh)an_=sY%ptaPUFMd?cvQ;bzD)NbSvq1+`Zq+tMt+rv%dB-bUqww$m>j<pK;_#
z_H?kO9RE7baSWGBi`P5wMb*2uadi?LwU@<xx~8mqV^9#TV9=lN12dYTTe6nQ?;HJK
zCh3l&yd6KMljqG%H`|9=5SvM`QDo8C5u*Oj>cs{-IhtPlm;LzfG2G%KECE>7rO|Wo
zB%1>3ss=m_7ObB&*&jI~G_@~k;#_{K8l=Kt#@K0=AmK|`8tJHjrRg_M*iIhE6dEFx
zk!wQ0G%e0WdQv1+N$`_K^_8%caEus`O%S@js27mYxw-Hb1(gVE*9+3-n)K02X`fZp
zUIc9H>9L1-2{La6Qo98>cQk45c7%ORh?j^y-0%<-eQvbBaUdIjP+s~nEW`1^U`#2p
zV(%pbGg~cd_RX!ni6%H~s8{7=lJPigR8o$MR5IJn*19SE*q=7ChM&Kx*}=;oMNnRO
zd9Bg_jp;fdG(uKh)$ezr8(7a6E?PH4&5z;x{N<$qZfy~AIs@`4Y(8TSZH6@|fVb`!
zAqXk%DjTqSpL(!90B?7x=u^wBTP2gH!ox{fffuh?Cd2FBgUr9{>&5iNhyesuX!w)n
zlPs4-fcpYd0mTERgNZeGhk`IvZ*9S&qO_{_2vtHnjiDsmRaRciNy|tmo2idI_zr{>
zC-({nT|xU~Eb(IZ2D+clFyAj%gDpmYagxHHVl`@JB?TV~a^>oio&64D>)%9pnXvs3
zBw8!q`r2Kg+{rB82AQZ_4AmO6Yq@3Hh?PebWNh$Z%6%0Z+gR&4ivCjimHXu)&68Xe
z9AB|}59xqxLE_s#*@CJPKf!k_EFt%w+f!bX-YgB;(W26}D9FLiJVm%tLWXv4y*<AJ
z%NcL{Zl=!O_514%!k;ux;`#E3_|P|2I~ToYCm^1IH)cyY!--+w*{M_;W8t2yKQUTs
zZD;4zW^xr#L&OdDB0wZuKhvgI=F$hlom$6`mg8ehQx<ydn#G=u{evSl%TLo#bd7B)
zB7~T7QUB_;nr9!*I0GJ`|8k!}NW`6WnZEu*|K#hjxW4kb8EHpBOnmaJJchHcY8N;+
z%&kMeW3{U{<5NE?(VvNx6YO^#c@=k)L;2~!$H)gANosO_Xvc?RTitUt5`9ef6|@HK
zU+l+2gNd}t%^~$18?8yVrYvuF`=<8a52vs@RuW!a+f#5)34uBkkm~5-yJce|ud2t6
z*luRDr_zcSgqdv{!Cd2?_8NIDwtppxz-ey9Grv8cvsDUz#hw*d@>8*$M)l**g;t9P
zcQ(Hl_&+vpBsfP^wAuXJWoBzwX17GI&CS#Urzj4xNmE)RQ<?dxAAkK28zI&BC|PP&
zN26&7=zk&C$us?LcZtoXg7vIEoE_%XuM2PT7)hCusai8;SI;H6u&f|!O@f2MPs{m4
z-`$J8^eC-s7cc{kEiU9~9+P=}M_HeL6XPJ1nM<Y6F$`5!U7VQ3?u-}LMgJj7z@&Xo
zHfA;J68#^($Vqjg%|!FC^PdNcCbyMFefGyXUFfTC5bx*@)8?qH2ax-DhjvF7iTOrm
z!i!qPZ{-CLX}(+*6Y#{zE0aElpf0q@OWkzp^*;BS#)oUhF8J?Uih83uUTJ$v5hup_
z^jD5V6=D9{T)u%UDo-J|r+mZ;O*h41c$|U+F2S;NzPY`JDv@twK8)cdXHoI>`J!RU
z`HKN&$s6;op~VB7=F?ksIZJ|%B8FtwGOWOD@c`}dGD^n>?t251jTxNr7dYAf103K`
zeV)Xf+o^4f&OHf0!wPzn%ACjj;0-l<7$ld7z(?%Ezd}XFnP{Fk+PzL?8cQdkXR3IF
z@!Wo9h23FJz(dqtsFeGe6`fG;KafQ8dq*-9t40RQOu~E7F`NftG_QPf2YaK|KcO<l
zfG@8wMso901v6nilus7tnpl0$4?Lvkjwh=%E+atSUigT=#50#;#s@rKK8M=!J|+!8
z{J(tQ$F;I0@a4m&GRYK=rJns~YA7CvWoq0NZaOU2(t^JLrhO8?l^2Tzcx2vaM9sZ>
z1P|o@mHf@z6mSDyKL6|WFN6Psz(<%gDlC|N|KI*0t&5KS;2Zs4`V_LcRVM?!eDyb+
zJn8%ozhU1aU?RG=E?^NacRL#X20+2LWd-Iuae(YCPiV`0&@g}f3u#n!x4_s4m>_=v
z_Wg6@k6co2NnyKK4D<)FICsgreBP~vG|4>@x?c|Yv12p<jJPkOw1AoV^zg4UJ)XY>
z#`fTu=wCRz<M=Ne{tJiy-*EWQSGLuB=v(f#y3|k~n<6_O9QrROd+Fg*$Oa&M^Dn~D
z+i9KzbFYihV4=rqbI?THzXv#A6oFViv8=SJ;sXGb91m#5jm2K#VgkO6Nhk(k{Gnc`
zHv1IFcFrQu(4yb@_;Sj&h#SzLp+8uAkCv29E?ORdhDj4icdzG!?i9-v;4m73474Lb
z5mqM&nr8rqWyz>N!u(9|F*6iB=oJCrIl<r1I9p@{f~_Rb?h$+!M@yo@Ak?{QkS-GV
zoo|{%TwglCTvgtfXh5@gv9de(JNM85=HUaLv2gClbJsVzCt~@GMSO*SJ9>^82|Lt@
zdU`MGE)+^7JJAsMp<$Tf$(Y=AM}-bEa$9OlRpaU1Xz_qXNFsawHs}ld?&03EVvRGv
z!DFv!(R}Ehe9Tl(vsWR$vqdBDjLoiyk{5S<3qs=*{%7RB8GWw8hl5iI9!LQM$ut&=
z;Y)%^!t{^bUcT^AV2=*ZPQDxMXTZAMKJT92HTaOu=St~gfl6K#qmOv@9yK}|&@6^&
zw7Xvu@K=JKFyI+{5Pkf+!O8-i{?Euo(AU78T5I3Lm;fG;h{~sU@SEnR80o+5_Us|L
ziF=gGv&nCF#`gmn!J@0WYfy#;_}x80f0j65U?NO!AD{uvk}$<gkWc)bc{|`4jfbmN
zcYTw6>7$_h&&UrS(zLBszhuOecmlBRzY&Ag_iwvp$^ak<$;~NbxSI(Qppk!|bPxC)
zkl$$(?LGV$$%K+dLz8{^Q!FWWDU<Ea7EZK23d3R4+LCvDBLq7A&&a<kqg_yD5b?ht
z2nJ?eC6^f+0OA{!u7BH2N&+p(kd9J{=WZr`)A%Sv{2NMyGH4fRKb5lNOqGF&m=(Q;
z1~iM+_YV*w$$@9sT{V#1O?))a>Hmz}^Ii|)ytL<`ugV>@zf=U$L~#Jx|F#<wApnw>
z+f2m|?`8rEXyo5eiWCEWhY7;bWbptdqVMJj8qn+;6^4I+xBxuEd9{1;Zm`ks0QdN3
z<SS3bAl>FtN)Nuh1OnnS%RXNKh<LI8*zLbA_+J<NuM7Uy1^@3y!5Qg~JM*Tu6lgRr
zFqP&GZuJ%On@^P`tlsg^L?-<VA~z8+)RvA~b(KnYDgoHMBK*<jPBNNPbGe7l(H%Rl
zDWU143rfOOE-@2h5+KyX&bTpvQd9XGNUd{va84Uk|CO9a^IE*@mS6A9rUa;&>0Zy_
zU(yFYFF&B^$L)?6yo(VJ#P7<JcPW}!SPya0rL_K2phVV)Kjk-MQGUJ;5Us=$vGBk<
z3I{&pNTcZ(zO7pzy<_|bcRBfsxl&G`#CpI~CGnrEJQqK?=TOb9$^sjRm%(U21q2ZC
zU8eB%UeD35%JreUrX&dN>XK2jIA8H!5@JzAJB~U3s!Zr?N8W`e^u<UC0?|(eGjjEB
zh68;5^+YV5IQ%vEE+HZS^og}E-+4>$@CcvZ3occOAYTWfY@P&-r5}S+vqt)>xw|st
zr4La00Qm}Fxo6>MfShB~$aNP~3I3*QH9DprBYzO6nD_&d%fCtzS#s{E)x0B<(MLdJ
zdWH_@20-=!M5#;%%*b_r<Nma}N%%%{SCnNBvS9dq^Y?kl`k%5)W-c>`f<2Bg1O)UY
z>j6!-<XtWSeD<gHdBskl2gkb$c$jyU2!X8@DNtBqKGXT1(sT@K(6jH*-8)W>ov4ly
z_DXjK<qQU%{R2}@C!9E*0vrg8Jdy%1;@Zi5o;!<1;@`DpGZ#6?12k6a$x>CdjamNJ
z);2&bYq6moZXEd?$^AL<dv}mNfeeeUTKuZ4sZz_GMb=W#s)S(g8maFr^1l?nG-Ut^
z;dirvcF$E7FsPZx`dh4^FUrq>|5AJI`{oidnewM=Y)s(Z!JzdaP5-fCEDET1|5lDp
zLK?ewRn~=uiT}>qe@%SC|LTm%r~h{5|1Hay&N~`;U1nVz&Q}b~rP#&nzixPRH?wc<
z0q$_j+|42L+avUi%<t8AY!B$-#F+aJb2$qP@0|EZ5uhXc?1jg7Uk9`gKm^i_0Ca+(
zf|sl*;U1fv1p{u~d!KJEQ1;Fafz^s7R2@}~VNl>uaYSOEeYyo%S#7$d+eT(7LVbr=
z#BK9*W0Dchr&u@BzUS;&pk{A3TT`Kt?%n$pRY7TebG3?;^Qt#K&Qa)?oGNz_I`IXV
z<|1~Lgl<JxG0!S85#;E7+`)%CP=}(hs#|Dr6Czn-A@Oayz3gv3KTtS^X=iX{v>*d?
zTt+(;TDq(g2f?$$He)Wz)mxuKh)eg>+8lwgsUZaCumL~)`BW_4gV6J?ieggwJ2a*{
z`Qf;zr!Rk%s(G2y`LvO8F)wAmxq=_HgQ~t8*WvbbBI}UI?5nzSLbIc0SXHZXk<pjq
zLJ_l#^^SpLa$C!Qr3<wpzIvNEZI<EN3eoG20yfnWU62^h$kf8!*cx%J45wxmzu%kG
zx6{q1efr0fn;D61H<#n3vnEf+O=8LvJG!i=T)`8v7r=l?oc8QC$~LLMT{}~o;jAQj
zuiu}aq)iVXjl3=6bwT{qU05klgJ^m7gbVw&k3_NuVcwT;Yp%NfQM0AgSyF5MFPF|{
zX@hlxU&8%(o&rv7=)Btt@TOQMHxMQOJt?G-$jI}&{NkpB-OA<{8M*DJcwMtbvg0W9
zd+WdoA;9e3fVSl#LZmb6l4{t%cxq|hntof!zp)o(*J;Fy9OcQ&jvTDJcC6^2d6J;v
zxXv24X(UK%X5cJ~b2BXJX%0_(GjZKT6ZK1NSzcuQOOyCYi1HGjb}FoMpthu@)`LFz
z261~cyyrC|yOCSInQ|GX6_Va}fhwSGwiWq;(yU%n>rb==3)5_()=^j_bXByME0SDc
zU6~Ou_u4W1cpZq;_{s^oS=-ZLnSs7rnD@nO^Yu#1`*QkvSX-VdrDUGG0Hs}eiWgMJ
zY1=dm6lmMLO0}_xXyZY&oi&dR1%>C7<7&o{^^fYq7ZWy;?MvI8B68*?6Gb1AMxPzU
zDD$Udwobgj^KFV%ovk})(N5=3`%^Y&Vq9HkFeCO=j134IjU%gbo&<@WO4yruIr$#0
zB#_$K(}O7<(^VnNzVN@&|D!wUKVb}4rHIJ{_Zl0hUPr#|ZrtQuUa__+6i|{26q3+!
zis}Ja2UZ{S*oS%i1SvnaqF}`t^a+MtHcf+UX4|F-;I6OAOWysO7J<Sy##s8iT+ZuN
zLITa!tjFgq^@(X`!V4#q&U*1+<^@v{I;-Kcr4lwyK0HJcr%hRdlX3Mj2%2nEI(q0g
zZRk**?J^CLWTbyzbPae7&t~01I}d2>+CD{d%CPMWe|RNW6oQ&`S*Y}ls^ZWF$Ll;Z
z+LhlJXqeKUI(}E(lO2*=Vsu+QH9g&VU)JNIw}7GoH$7H|4|7NE{rcD(7g%Qyc#DMj
zMuSMsfUj-Sx)W=;zmItQF~J0nFO(tt4ZF}6^I|f@u*cBV%`Q6#`{2l|EUaFRQnbcc
zbobHe&k6fhAUNUv6()Cx?;yzBf)-aS!3*;yAfY=_<R(S3PHs~By&D|X7FloLleq%8
zBmekCO3M9XQO^GIAJP1~S0w{yQ0g`Pl?yWA>z{be!-EMab{+YNL_4p!$u685Aq;~w
zFLDMX!G_j#GmgsDOZoF;c!agP%hMU|J$TkF5f_^o-U>0V!QA`9ZDXaC%Ns>0Tb=vq
ztn4*i6~lwF7+v|{)29hdYG=J-<87T1U<FC|m&#wbwgz0i8X{Oh3K~}6SAe&J=wqOX
z4=3V6`}|3<g7mi>Z3TRDnX4k#w}mwzZ618E3C~OTuGz5GzT*6gry+zs8|%YEO@ptR
z&R6m+kF}&`u~-a*{^m1L9RP}^K_gdd`)(W3P#*6_qTQ59zM0alR(BZI=d1nc48>Jf
z-Z!kgHhV|6CyRU=#=F6^!o6UZiQrU-$R#SL-jKsy7}aob5on)=*v7A0ob4fcmr6p|
zc5{{Eb`j;V09qy@<92F)v0I`0R&=ZF`Y5Jk_p7q#8yNff!YkuA57vs=-;D*)tN{wL
zaFQj+r|>BGXJs^+W2aWf_jV<ue4+IV`(oe<g=v(rz3A<?3k2(M6I#3VtnKKoeRYhp
zxB3<d;Sq-@B4~EQc^9-!Try+RJQTod*7@Adqpm{lM4erO?A&wCVcuZi_I1^F!LwhI
z;N)SL_k3gF2aj>v0bUR4m1@oN*p<t;Zgc*GCm}P|<Yl$3rY=4@HbL~PYx1`*?k6$e
zyVq^b6T$lD%%O)(<V~AaodG4e<`oAhpd~K%=c1LZBW;&!7{oR%njZVfAf9&=)KPPF
z<u{`^%lP%M$<lhw@ek#4PHs1>+?nyBzjw;asjI!)v>wg;talIBHu6@_F_#~<(v&TS
z7<%l8cgtQkM;nl~$d0!t_)Sg~??g`=>f;d_d7h*@<LBHMeD2NDZ!0(3O|kyC2L!a*
zxK-u`($+72zjJY$l1bIsK-U6SbKL_BXZBURvRGkm!@_3IJ<p;p;JfS#iIv4S47P3T
zhMllA##^s(PwQhA1JMor<D>D%NrUyU#Z-r>NR82uE7(i3Ra;Q`nOjV#_t~h&z*WQ&
z59^e@NnItn&+6}{$c5;X!g@pZInT>w!3|>a;dg8uOZDEX)Gf#+y;<*(N=n6U^wndt
z$n&A_hlme4x3#XW$K&-zDRE?FR@%$<e-@54PWjfmRT>|wy`s4`uu||sH8O^N{?UWq
z74N|fMvqkgGuB#IMutrfeb(`5U$M^dX!Gjn*?<S$!|QA0g102!MtO5r)5aCl(9!Pu
zSh2?H@Lb=yee3x|Tv21-22Hv5rm+1I1QO0_Js4Kcox(d0Rcj=xhSguhic}A;tw9PJ
z5NYpWA9u>Cno#Ap6Y-b4KV4o#_fb2XgWyxNW-Z%YBMs{&T-{GnkbBOzv)(XH*y}_@
zI%O|-uuMzWZ8B7f^}NxfWUK_Vm&O74PF{~PMEwxlxYg?zBH}66idzF;<nS4Rj1Q<m
z3X^QHn$wF+Qxw(39$ox~eNIA3s9yDE%p>L;E3U_JLMM>1=|xYE>fdVe+CGJcn|{+q
zF5Oys_D`!lO|riBjSfl8O~U?p7XUkbV@`|^4Eb#+e_LD{YJ52yVOfOXd8c6Y908Ib
zVXPRiWos>KE&IgC+OEOht-A@^>o}=<Pw-UB&W`2Uf8%TLE<@V9nc#j*CB9SK#SpGC
z*`V9{l=|_+<kbAk5NxHTzSk>#rAhYj(QyQ<5&8WB#CE<+&d#6o`g(9g&}isGoqqZ5
zwV|D2-s6!}UH9_j=#3)kmQR~rh=FS7q2WbO-6rB~bRyh%4ai@jIYk53qnhf1OM_bG
z6T9u8bKJ9^vAG=P7RXR?<yzd^HiBz7*7SzwYPUx@$Yk7kz6zHzhU^4kq;1~nHg-Ab
zOfM5jUF;i?ZBy`g+8OWgf`@gjc$w*TOO=rat-lht><AcorNl^()9rj+{@SzNfb-`T
zy`b<m{cWyuw?Ny(_Ga#HQP+9@)?#+Znk4Z#sxuH#9H}^sY~B&!`>y%+llRr>Cbij#
zWguB)I0LiFN!9JnQzY@L!dk3+R`*(vqj<LIE=tRT!X$j(D~pneugIY|K~rn6{ylaE
z8{~vmu)*sioA9d4@nXX9^``<2G7jskl*9>f)Or><JKway$Y5jrcd7*Otp0}WA%dsE
z@HX$8c0n^W7zlLlpingSQs1SrbI|U!UP_0+>2Tp&ZNZAY`6W+Fzw3NP{I0=$-V=;D
zmqClf5iw84?v4R#WswidYPS1AYc(Ppk?JUow^Y|r{m0ZT%C>Fthc4UO(z(x6Mt5sh
zF3wwtO;fLWm@?bIigwwu<-(7wraNkOI9|Coj3I6C2t{Ya8r88f?9y9N8*<2T-~BV&
zPrB(mZ&zVTUASY97;HyM%P5qsmHab+4K)02V|FFUh_YXf$Xg0)cf&7U{z5$04I9}a
zW|wQYi(X^Lyt=lw&RuR#2d4xP36Q?f9oMgJ=R?WYE6QG;tvW^GX2#2@D0?pBAxnG6
zRcm&rk@fZ9fX_p4vSumXK&)Z?d*7^r`X5~}>|ANQ{d0%10be?+w_M=gnn~!oK3wE;
zFbl+A{4(|2f3uPtI$TICc^W-(v68}hV^Cgw0m?^A1TFv>z4C1lG4{Sz<W)$%V)DLe
zYb85zbSH`e*1N`9cuMX}&B5uy<l4HHU#g{RO1p}HUVe67yf84h^7jhgHR%EiceR{1
z1YaTAY>~nPVNX<7BVt#tA6<Y$ytutjJdW!Y7r#6gj_(Jz$n=Tj(sjaeMQj(r7ZmYP
zeT-qtx(ZRurd-{lo*!)zwBS;_*Xe3+sho6;1RtfxK>CEdh6l}3@tX|PgNJUj-H6yN
zx*CoL23P~2{PHKMt@6vWRsA%4_A;Ug5wN^h-N)^L9&`y}A|@KV$0O$O6f<r+X3LxW
z8Sl~Di<6vsB@nJP*3xCo=-UnT`j)9m++vXfQ7%ZIL#kzX1_u}Riv!VnJx)~qxlg_u
zM@HUcqLvgFUS_-Qr+N)h-nQP#m^1jn(-Mp#t&*R|owXeX;ow$rCn!(TpX0{fPCQXh
zUFxO|f%N$}ss{R{^-pX$-WmuzY}~S4Frgj4K1wosxLe1UoUd~o|GYtNb1i0Rzy1e8
z);v7T?0G#dIv4$GnskfJdUtJQW!<$L{>^mh47L62;*;HTga5U-JFOMviqOm%Jk>gc
z?ggI?d^2xVI-NZ2Xi5>?>ob2?k>l|Q+<oc<-0*X{fJEmLwVZ-AC85}~F?D4(#}l+Z
z6%p|`;t@WEaWdEHs=}%0PHK`7dD3ttQc@G0G@*(rbTO@s7v1n3twrlV-&ccb&BCG{
z0bI#FOpfJc1#^)VCs!EYx1H6PrO)17#uDk`)mUk;qx=d)wPnSrd1Pc7U&mLG$!Aqi
z5-Kmc=BR(Rt=zrI-&{knXd(kX2@eRzNH?xlMcSKa!@SRyQb5@4UPWmSbaezzh7wo|
zQu@*Fvk)1VvBXOlmE6>lU6%}ta03P059?!JEm?2<j1ZP*&f`7RY1vzD<UiG?(*>M@
zlQ_3r>Y<-6E?ET=IMkwL56co#Es~LI7-lB(7z>YZNClpqs|ynUHte&xU?yI5C;bXP
zTd>C{h#jXXF(BQwsVlqK-^^IBGG-plWfEE)(;Ik%&E|=Iyxxc0*MU}S_i0?$gh;Vs
zoV>E?_R}J=plSUBcgv45FeOuL+N4i=3?5FyvSaSnu5ukeE~0!bryDfCC))jv-5KT#
z<SvV)&#`OVKaHpkE>st!v8p|@oGj7m3u@DHvuWA%R;Xt+(`qQuI>fiDt=79PrleK7
zc@32iu187lYZmt!dGcZfg(wOC-cQs8$ZoF~VRqK2vxQ;kjd8p&01^xcf4kcpDp!Yt
zU$`O-2wQV~hb+nKkNRm2t9a^8t*agA!&ms2!X<6TuE!Zt<cBTIwRF5FtC!5?byme`
zkCP3Sa~84PZ`SWw?S*RZmHP*m^dnk7eRW&y^e>st@YpjnCm0<Hc)bU!QI1BKKWZGi
zEj_p>$3L6jJ@@B?XtJ)>Th*y_iJBV7Uw<_94J9OAI?qk{D9!G<)F(ws3#lJI?9xM+
zoY~HHk#eUuAheF_3|eZAimj8JyB6(|XACe!);mnc!)V?4`O7FxqJ_yG(=pcS%F34b
z2Q1u3<RzflM~=X<zd~+r=Gm3tvQyUBG4L#xb5e}`Hs?j~Z3YO2FARCi2ppk@*|l|~
zWg!iq0!u~(%@^D<3c?5l=i3PmB?hHLlxE)9d6VX8QduYx>*4-8piD7ZOsw%=<f#X0
z_mqTEUs%Q3zUN^Wk=FJOy{t2ta0%zg7q)1l(Tn!-*Gf);uS$x+wh?VM;8XpXn@lox
z!Qy!p<gfU2j(Ps@HIIZvEpW-nR3*HpCPblRs%Eu|JzoWFKkX5JxkMDOue}XaDwrSV
znmoOgG`Bsn%DN_a9$U@YuzAFIn(ftJ*a|P$piQP_gQ|HUuZB~OebUCgY`tFd@d-05
z?z8vqngeo4^Vb!#Alk^1&^}Wtcb$l*sVfS{tb=8z^(A!tZtX!Hjff<Z*wBtS%{)3{
zdpR;BoxNwYjn)7KA*PiiASRRRuHlPk@?t%lly+upL9$!H9M`_iGQO^x6htn@yl>(!
z0%gh#FULHewaWT&LY>^U-SQLhL4~S!#@XJjI2w81ZSk)PVhs&3Si*T^;T7+J9aSmT
z^m0<NT$5}2r3<W3M8cVG^3Vg?VG#29=JHsX&uF5VL)fxDuuoI1ajZp#vEb5Ts7NYw
zY#Og#&~NoivP_$3TS&s+%|FqAthznNQbo5GkXECj%CnqQvB2xeY`*?@f{~QP<i`O-
zk=}&e`v!``ek4*MQaiuA<19F35g93%=KN^BtL$sS=hps)c{TVK6H}@scO%9gPExwu
z38BrAvc=--GNiyvs-1S6yL^RK$RlA0xZcSeNjooqPElsW1~WZ~<ap6!tAh-25;~t)
z%GX4Czy2ct5SfJozw0@V*=C2*2y+4u;igFGb1jn2$0BZipeC%OnYT`VRl*(?q&&Se
zO0LS6_syK#nR^j!@AoZse3PouZ@#YJ>{0TK&Hd@Ox}R=)UTY`QLLU{4S;4U#n0NBt
zj^k*w%m{rlGUVQpOPV<D^9FnMG{F#mUZ*XK;~a8zEhIP9$GWRETe9XnbG^3rY`#}e
z7y;+?W9Raqbhh$8*7arRgn27%Yc4lH23qY>xe~W#<Gt-~B{sO^D(SkmaVXN-_Q#wD
z_l>ck?q{uLAb*4Vd`2_#WzXX{B+)f;;9@{w<<(Mk5$h-&gKf(?3Dbh@Qpu7MyNVvh
ztyDA`yiS0x^tCm}QJth8WxC>4*Oq*}1bxon-3Xd55I_oJ*n2Rn$~l;pmx)VYm5rzH
zIK2<VP!j2fFOcl_o!o9Cu2oCkXn*7&9juZC^i6Bl?h@;2JVH)91(JncQ|MU`d+B^m
zTnRLHjK9qDW|Ax?zPhWx!hNB<kZQ?xIe4e8g^fF~ew#|zVA;dgSf=NX!^X1Sd?GP*
z3gwy5OseoDLqX}ZVp|Q?=n~_eUF(mhOV{BRdcBz(KO~)LF}{QB6`F2^`Ld94jhX#X
zW<#Zx`M!(yyiPl)Q5|lxFS3Y=<+fw&da-*S(vx1$&1d(L)43=>F^KjI*eT_7i|F`^
z7a!kgj)I#?KvJ`9jFklZYcir_Z(7=OIjAfee+&~E)#v%cZT@zsKwM%qYGJ$gIzkIP
z$*}%%wMATF-BMee8fyHzUn5Lg_{;!it2q7EZN-O!v}ApgLqKOMbBZkHPL-RK+@E^x
zNgw7hYHk9kXN3n6#X^lkjwT97J04e2dQ!}$4;|2Z*7h|W@cfLpYt1~H&oU?V$NZ%J
zYsD(;1EX})?9~<I_Q{l<3jS{IxRuLc@;c$0eF3{6fnvOzEc0%Kg<EO)h<3dOBr|h1
z-8c{2LOTfrk{v=q;i|-cT3LdBM-~R-byvUec{LnBHvKHxXWCqYSn?MlQbo9CUI99E
zGPDRf_7}tj)rANuQSwi2I{>#;CC!VMP*y{!mIO3iTaI@Yg%-Wi@qn{ptoRL<O7zN4
zlC75nE-AnW)yLQMzl2x?z*MJyJmQ)IEr3cYc&+uhu;pQlvhD!?)tXnDYWDU0m_fL7
z&0)d9=0v-Bga0ch^*j&udNscZw{5KSRJRtrT%9<NVhyv_Sc<><%Y5!uK58{hEM4V+
zyL{r*-F+uQa$)!X2$mybu~Q%Om#rkMG-nymdqfd|H1is;f;=jkdf-DJ3V)J85W8_J
zPkJO?C8z<%qLhc4+ZDDzsp|KP-lEcQAeox%%zb$W@$$3}dkD7oF!(B=&zBrHo-`SP
zr2Sv!U<IvShH_%-B1K2nc1w(%obwjxMHEE}{GvX~vhuTgt&HOY<E<acAw0?HY83V6
zYvXchVmM3ddLRF3D&H0_OBE#aLhi>MDadhWq$hGr>MtGBA-Tr69P9C;-B%CN;aBI4
zxGZ>{PQH0%!E#Bl5coTy+Y4WP*57PiSL!(igpM&Ft)9(nE#EvS>)5;8dBa^Y(u*;$
z+1k-ncbQ(k=jbG!)ACCx(^NWR{lKyWFqF|`;!v>Qs#Z6S^?C9L#klo1{=(~DkfY6O
zb6b2u&Udn-g>=Y|oL1|C^na_Cits=@!s*#_e7<n%y=TW<h8H<szqU46TMHDJXAh=Z
z1Jdk~I+UOZFHL`)zLrNu$bA~$%p_t}zci8+?$J$3kj>ZTJr@U>e($kaB?}aK&&Jng
z(#RUpjHT1Z=4;?uqi?Ua7WtB+7KvH*+0$;ab*AAO>V4gh9vTInC^H8V-`HMl-`I_X
zWvlwVb3tacMSxSX4D{ymh!hRGthA^3`GlSeueWA^2g>b`??mtk)iip4mMnfW=u)oG
zFmuIriaNZ>lMC?Z{PexcTP10evN|M$vGV%n_2Su)!fDU@*@SJ8TH*Y~N3}sOR15jT
zz2i^D3OQ<uW5dKHjMG4#aEG5y>^Q_Vk!%x(z#^Xi=Ps`pX&Q3UxP74sK(;e}Une;7
z+S5wdp?RM2m1GlMfo^UNQN<)Z9^vHZvc4^_NM|h{dgagXXLHUGpHR1fmA*wy&^mGi
zKCxx@y}b<NM8&K0c9XFBR@Qv#gV*cNd$qUk*^K71o(`r<*H$=$Jty0_78!^7fD}_}
zyLH*x^g|FHo|`z+hFpc!Za>*bq6(t3z|9)ll%r5EEUz<Y$W-HGrQbb$3)I65Dmif+
zn~kEO`MZqbejnJ2nilr5q`$lb`jfxF;7eK5DQK?qIt`bhu`8<0+kQRH{^Lc0I>CyI
zzei2XDjD6H^2Mm7wq|3Mk={Zvj@7;ai78=oaY$XT>%23icA3llHo?S#+qGZsS-DK_
z$EUoswT&S<w}GmOBxhp<!Ji_E*CchOdo$XiVNoj;)Sf3pcBZBpS%~57;|*1>)Zw-}
zujrTZVuno_XIu&Cn(QGUXbIWgHDPJXewJ2Xe^6X$>r(!E$dK33H49jec;U2IH1%>~
zs*MWu%9CSkXGSp@o+MDvx>X~I!M9~?+V0B;=26_YQn(*q%-mwRn2jfF2ToAb!>#R?
zDp$VRnJ+HahGn#lv%p>QiEsrt;H5bRdl+SZrtJtbfarHETy=3`H|Mf%It+Qr`XTP9
zkGM(8g{gNie!(`6@gi>8Dp(=rc}9I5)<c6$14m16)RjYDZLrqSHdsUrNVaX(zGzaI
zL?(V|*VzbXPI$Zi>-F%pissZzHxY$yCF#w@rac!(44b&ry?#SF#gKUM!@97Qb!4JY
zc3=`@lBv2#vxfBn*nJHC9fT@feFS@p)FXZ>Aqx3X@-qTfl%-x;v(G<9u~Zk>zj`}9
z^l(2@rRif|huJaZbGtaUZ{iZr;u;nM=zPsDnVY(CZR#Q_z-98=fO{8+7IXu2GfWPI
zOn288`tqLc5Am1PC@bI4(`R5kL<s5ws!w%Ei&c!LURBiL5=~Z`p7F*&g+?jf7^i}^
zIaBZZS?!|CSC1y9XqNfcjl**&m6H3nkGG-rP9<C1@0HvyHd*wd0f(u~r)u*GhHl0~
zskaQ>Ya_E*+B!6!r>_SAo}=?}x#}gIiXLfPzYuqDH*4Mr^0MB*hK7$)voj>fumC|q
zHx_gtNDQcNcVhgD2VcD21GUbXp00K&s8VBsW|<gw(j;<3usR~f-L8K>jeJjG`ka#3
zNy>fde4dqNgsnkIkzBwa?u1}W`dy6v+y2$}43DY%3O<k=MMg4{)IA@tS}u)#u4vgF
zck*p+Uy;TBUv2^HCb)A8d6Dq-=q^-2JT47l?!9>GH4EsrstBz`+Euiz4+5NPdt1N$
zXb4|}!RWNf%@yn61y7u}I&=Se%v)hVNi|~5$iFW4p6Js5umxiiHnR1WucY(#bUHjP
z52MT~bKPbp_xsHnvAC(lxNWHF_pML;jZLFqA<G}LEIdMty`RXNlHtZbD=ignG%ZP$
z5EQz8pI75rF*ClXn<(+q%;FDZqh%3ESSDpGX$y#Fw5#8gb!TfsG*d0*b3p?pa>2Kb
z{acU?Hu6yQfw;9+O9S#fFFp?gE@+ITUGSLtgmi<JKbxg?Odg{!<J=kB^k<wJxH{W#
zTbnd*#+(kF5@#LL%$J2V?LC%T`Uo$5OV_G!<t<3F+tJXjf#<Wb;{Mnw4jCqy#;-M2
z!~x2jiG9QycCVE|>ldrDo`AYF@IG97(vFnQF8R%GeGf}-q=K^dI#L(=c}%-zdmd0g
zHoKWGIlspWI*GOSdk35HNYdUAvC*pGgrF8V`_BX64O5!NGqn-PgLNqFt_;Z5bGFvb
zOWvi<Exn8~l}X3VbSrqv1y23(N<cxIV@6-(W4~{PZ)-6AWTlk9$<W0}hZGb5GTaiS
zbQ1tV+qEVYSwW)Q@5;*lZ)DCZUl`=)C8TYG**IGn{J!zs#D5ikIn5v51=ryn3S4o#
zwgCUSbR0y=zQ7~YeUR->#4^uV_G8H`LozLOYJth<y@~hj)?LYEA$VaphY2`rk0})y
zJ+ro**W+1EqGOr&om4UKAhV()L98H_QHPhf0EAy<I?9Sa1R$J+PdJ9Os4N$f9jI@@
zphmLJH7Hy@Zu{<XNFUE_y!Udfupn$}$YeivN`g2ed+D)>!v~;#kx_=}CgeC#v(@)F
zu~IGG&vKNclFg+(kPT7cTXIz9LyU*7yCBs?s59~u|E>qy-Yv%Q2EVEpIV+(Q+@Be_
zI=SXE>j@=JeLM<i<~?|Ah_G0}Z!V&f+msa3G>%<EAnX9mH&)sKAoJXU9l&-xVwXHg
z5LTbh3^`Ky%{-JTZWnLrX~i#0@KkADmR(K<XPd)YCOeyZfb3SwmRhE{IJ2eo_NYa2
zrDosez}DJfcC`@1rsAk*!<mM}IRBJ}N}+AS8;Z^QRayObW`-F7wt>X*Uq`2~rXGvV
z+12IhZ720gMIZG{U()h^xKsx(t}RHP?>_66U<)2ImOnqf)F9ZAE-fkEglDAdMrE#6
zoq$_x-usK|cT+b&zBZbD6iSeu8_Y7G6j?vILV1`R^B;VtG`M;|x@}S#l46oN!A6tE
zYIv0uz59j|`SfLDGZ}J%SU`i>)HrfPk&czB<y4MuTlL4O#aJ+2V}JA$=9IgA0Ob6J
zno!foMAuz9uKZhOw*ED2Q3vs~lD>@X23T39Gmd^sFE2FFeTpVG2qCG34y?$?#si>6
zc}??VcppLXcQgce500RNpeG$J+-6--`@_80R9?jwz$)M%M!k_SycN<M5?F~6?@mn5
zxk(e-?4Vipt0?)(NmUBH0GZu~CVP)c5j(dU)k+8h>`dCNnXh&k&H)~|(SG3f4Hop{
zEPp=n*v@;2m}r-anAAuKSiIbLt6R!ZdMd&O`fi9>D=&^^(dZRh@p1pbD<CR7{z=Un
z*$k5HdGi{H&S`d;J45PZK!Q&PA6{Rrj$QvEa_$hZLZhPR_m1s&gy{+C$S}VPE7$FH
z{o<#_15KfagK5@sz`FIz-xqTPV)4@f&)p;m4lY9_$mqC@#y&qA9F5JL*0GK$8^Adw
z=jSg)I^+SzXTf_b-A_QO1)pav1HXbKUR7VonZbe8a2<6v*F!P#vf1C;$JvL^iTJAe
z?W`M6T^ER8;@07}fga24f{P2vAs<b00kBJ$&e$~%Vcig9m;yV@fNc+;f5W`(NQX_u
z#Ad4%{Io!U6f|r-WDZni*DGH?^lOIKVP%LO6K$H87l#A1eE=ye(e+nP6aR(S`OwYc
z87_?aRb0Fh&UygnWG&UOO@GT+n`YZn32m|+P#cqPY)RDYxmfesTOX&%(hS?L|1hW1
z!*%~ebJm|z=F4pWPvh3n{_x2*_5n~uj22K@aTnS)jW02bm*bUDZ2cIN$OujP%9=MV
z4jIa5MVXnujof`kPyLi$*#4xR;l6HMMFVrh2l@sGR1<qjQRYM^dIhMq^V9<j)>lg2
zA)a2hF36p-V)hTMUqmd`*TJrp80{@WJ+7Ur?fDX11Qtx&ww95sm;*;h66X!op*Pxb
zDNok>fw<g1QgAP@$cVb8J$W5Vx`_Ai&BFbtTVOAm8ErT1V9|jlAahx5xv!nU8&-sv
zmLwPGh^76MgnP<<#G`0CO(MT87PM17<`@<d;zD2r`PL0=?hOg=JE{EK+fj=NtLN>{
z?>#4-4&{vr&<veiMO5zkN+-c-%gcWBF#<1~D79wen0_SOPc&+FU>b*)v9$jINX2m!
z*fDa(NR91WTTzP<J>CEtf2j}G-Dm4C=KgMwyA^8W9Cf&E64?U^r@ffFlRC(U7(jtW
z5v+)!5mAVT7Z4jqm8KI>d!^fKSn(Uma+<Hjf(#S(kw88%%emaNLi-PF)0q8~`Kl`@
z&Er3~lO6~1C~aNS#&|?*@!SYvJyiZW><K`3)9)p+hY`JEm?;+kL%`HZlTwcTk4nCX
z@{w=hQ?F|`G=T!Juh1=if!$Ot`0?m&u@3X}tVqOK^YhHU+6E!no?S6lSzN~wy(>oL
z=j`*yE=Du`-4paC?T*Rmx=M3c1-vemRs3&OL0}B*S*lhwaf2Q7Pg`3kk-Z%X-F0&5
zQyl+Eo05z)Ot*f9{5ZjU>o{M_&~g=nDlIk!@@*GU9X0X_Xcnm>EG5oPJ579LYIKw2
z{>v#i=6&}^QFaug{R_gDxz7BVAElh62owQ>ZryNVGJ}5AP3^bqJT-e4;sV13*h}CX
z&-u32IPBF9t!-xZ4pydI+$9WM)Tf(A<{VPS;^00Z)%$0AILVc_q`CHux<SoK3HVqs
z^^<seJdq2(-BD{zoao}~U{IvXLTkE*V+_0NK4)+WkRq?fM=`^$r^31?j)z>J?zsw~
zP+cCfwj7q6@UL}!+Lx`2F1(ERBq>Gbz+%%?nD>uznW0K_P+R4^tTRU{P*MV^f(OGk
zunxv_L5ZDW-v({`xI>Hkxa<5j8e%)RH(%~_1m7@kEm?EQQ^5)<N=P-fYqcpXvNyfr
ziI`IAvJP>s&jGzbw4DaeZXU#>?TzJnctMHWI$ozF<MYB=)dZb@A$|~zmW@{2E+W7R
zvfm$i#{4z57>Ca2f!{lgN7U1sFP%T8ly47~9o=OFsle+v)NME4s&-U<*0J<`$L-H8
zbMCZG_~#dBFir6NAXoRCBV)?Nw>Bfq3}7*-#EPq1K0*vAl^X{ng<C8rXa}8*AZt{c
z?kcR7L|h8c<zjgfy5gLU#yLWOEaOQoa20><a^YA;9teJ~EjisQK~zlSCgTytT-H^m
zV=|=N?S*SBoEkcqoSy&9T?t*}opEP0w<&i~_xRcET{<KvmA`8xu(4U$S8xuV<5yy#
z{)?}kf8$H)x|{i9so{qfCrQZiTDOvxI_U$GPm_;=cGUF!zG>&~8akmOMV3#$YPRk$
z45P{<qDE;(2<Hfw{FOv{a<%+fda4aeb9z}5j~gW=kkiF3%%q%UBMG>XQMQjA4!Mu^
ztF|lW%XvAg?$-Jk#%tVIkuAKp7wAIe;w#%8b|5T{xG3hcKm^7GW9oVKsWHiHR<`vs
zL11r1!sN7^Y<od%M|bzTAhUDe9+p(q7dynG8MiAPV8sZpX|Sj=#hb1@@UQKgSkwgn
z`lwO{Tzf%Nc2wtu^O`i<u=g5o?ak%cv2!Yl$KXR8`40uYI0>W%R8VSOy!NYvphWg|
zO#Nn($nER`WicDU=L!6zm0+e;8O5R|2&MZKl^vfvFWlI(5fB@zqgGpAZ)YsgIL^n>
z4r&sgITZ|3aQ5zj-$74G;mCvxFN+->o76dB2gi(<w33{>J=mnHGMh}NzKU6=;kMfX
z+s0r*da+~R@4U?#E~Ijb%~X}|UiV;XLgn6;cxP$Dp8?v1tYZe4eccjsYbesUgr3X3
zPQ018S}rbGtsLti^)n^$MYaa$XH<<>BdT?`@#n9Lg%>Yw%;8PR&gSvG1TQ7iLcify
zqz-3Jecr}6>QyeodovfG#@y?_#A3}Jn<0=95;`0xE^+92GFC9LADr}_qf{by4y$$u
z6#le@O&wJeDajthvmg28#+P(sB^=i{THk`0PGho76*SpvDY|62mwvFv7xx@^2}I2Q
z;?Kyi&^{0qi-7J-mVqXx1H$1?)KFRShGlZw?#vFkjoDB&r-)Z8CH?%aPgrT&as7ay
zH296}5LC!L7(r0TgKpwh7Ud}Jdzb0Hz$a`ugN-x}^skwU1#GfG%i$*!Ngx>+Gq7D<
z)$t0)pL~9^Yg<sT;@Tx{Q<6so**<9eQOj{&D96i^rX>CAUR$?(DF${C&xEKD?L3vH
zrsAF6!-Ky&AAs{(pJdHTdX89AYIlrqq}ks#ljF2B%WavcZRCR;0I1)iu{v#ASnjFy
zsf+HV46pN8bW#;{9$r(0PXVm%en{!Q0Jr0*)A|Cg6~yMP6S<@($jgl)XSo;<Tx1!m
zofDy!b-d5Ey)d@81MH65D<1KOi)q{zNe*#8*5_MFQ~$H3`nXj)OUBk;-NA}}pLf<O
z!jC1jf~ByQb1;Q#o{69^`YVQO>-qb7NhxG(yBN-cJV4!mT*Aq7G~4cWzv6~Z*lp2O
zx8N_l>e`{bZ=hzB0lXMvPAZD|uQy3xz<<($UD$a`bV?w@tOy=_ExrE5%L{Dyn^h>)
zYI+C8Jcz6N)bQN%+zHXL^U@hL)1KeCjWHn7?dpV)J7npRvr}X)CtJx}gY%P-d5-AK
z1qQV0c4moqE{!QLFBIo)tMTYiVgaJHx7Oo?mRi7Q;YQcXisx&nxg3;ww97W{1p26^
zL`?5ZXTf@spFY*C(Z35s&el*HaHiXSFLv74MPjvdvR4FR?@E0~XX=>yIQUaJ<>*gs
zmfd(&ufengy#!&cc8|XMG3tU6CR>dMv3~k*)N5TwY=_P8?!kBGZ7W<;OX%-+PFe{y
zqK$E#=IpbUJqTxLJW%^FcdaDULE&VT5!?T&*XM4bz8u)Sq4e{{fkW;j)imZVmtpPZ
zs$C?fGx-8vDlT$E)^09Vdw$q0&ribt+B~W?NO8+vCKTWPaW0Owv@*MF`|ZapAt)j~
z$ZXH&5q!!Z(7^aiWRDJM&^0nT_2F>K0C)9`%}3L7DZb%Bjr-s|-)-%b5EL7?!=nb9
z{8``{5Pe}2G@UoM#^ec~0m<#g3RRDM2!&p<*_vMQubL&k23=`)iGJ>HIi8aroF!f~
zA<OGMC(cN<1%wSIE=~4Dk0AO45YotPl4sC;3V*^q#U_TcY3p4%R?u#sD<4v+wI?o%
zC{M9&Vf6pB_m*){wOzb0AR&#Upo9ui5-K5$A|Tx$jUrvr(jkb7gmg$r$ABP13^5pV
z=SYjh2tyArz!2{>-p@JDdGGT)=hOS;yuX_d{Pu=D*Iaw;b**cy|61$6gAi)rzoffz
zT1Ny0G)0PI(|)yL&p(#y34QLFDlxscyffV}`qBh7vv(V0IwNZo-Sz2Bl^w!j;I~IA
zRZ~w4d+P5$z!=E8Y$)kSbDeIuxH1u!#QKZL2BR772CZ}CGtn|T+I%wTQS1Qm#x=jH
zLzx6)kQ;2Jach?y^1(pG=E`V8jCQV`mqYGfpOzT#C10675J<|K+MIVE^X6sWW+}gk
z*@}OQ61E+%vsH8o>`N*rkZv))VOKJLUU1j^9b5ke(XAJVW4^iXFoSn6*Te-CS2j^<
zDHw8l?1Q`qlGTZj-N~kb!rGwmcb|UL6dOM%dp2cU60#te>?(0a=6x)X-^;)VG(DBM
zyPl1Js6TH1GJ4|J$E*z7>Yw)UMO$NTd<zNQ$@m4Kgl#ZmFjYQ*CkN%W?-dzE?~EmK
z(4>N{3PIl8$m>7%nZX|ZLhH)NXC5CXP<ZLpuSd~Nz7c>Wse=CwW*>q@OSI0nGh|dY
zrax_43wpe@U{cal;mrW4{gB-2G(9_wnwni0wnCf{qvu0<Vg6Iyt7;-qso6a!ZxN<s
zLG4O>Xcrgkxn4A+w&_95*<o-#|M$r;sc^3wCWNK#Pu3IB<@=StP+#p<|Mc1@GU5Ox
z$aN>(oI1@f>rzf9GeiTr$5VRL8Be%Ro0iW@Hjz?=xr4=88y^~;ycit2b|jplBTwMn
z*d-V~yDmu1E0^Dmy)}ND*B$YR-(9xmEd^<te|d!~y)!o;B07S4C<_+0o~_?F^^#)L
z&#u&|z~|an|1Esth><~@7d&2@e`<<^TVO`Yv>+@=YW%!gT+M}{hN^Lx_-LRsV2D?w
zTpNAbjk?e6f9`zPb&1zD*3HO4INBj;mEq&c{tibImCdJM1NZJ)pZdi)`sBOlX4APA
zz9>`RN$}e!dRZ2lWPhz7Ve1q8kPTp8HjfR0c-rfImS5-iT;t(weOT4~Syz>j)$_|C
z)$|neojp^K2s93Og2zSqmgQFRcSlg*+%%rI8G5SaezUg6;<TKzFq-%o@3WK7g^Rk`
zxSF&eNK*4h3vPNm?ZNOQk304y&QJv<J|f1LBD3uC-$9O8b7`BQL=*!T#5MluniN)?
zizq+gX%9uikqV&rDu$zUe4yWYC+gt|Cl3TI-E7d+-ZA;8S}B`00j&&C@tLod2|H*Q
zx(UVsmjqT+NqRr=tZ2k2RXHc9n||5czmZ`kx-x{=<`JtT+QwJVI!yGc`c!T@<m%v<
zI;co!tlF60@XMm|zReYA!OOs89_Wq{Uy2j0?m51MkaJBjb7V#*Cl|k!+lc`$Idh0o
zI8`c77q<+aI(d4obY<Eus5LoYEpu`3vUBX_i&(fu<e(e;cxKLy#3}GGPZb0DiNmzp
z9i>MdarI3%4b4&W$$PNJUutX~?jS0xY;w*@HTg^(au^}~#!O}#z;5i-`?^N0tk_5e
z8zfJdgxnp)j5yftdy9J*1Lv40`{m7*jNIsU$%)_9OLuxDmmz0jCx^Lp*-tw;nK3~<
z?HqYb49xCaPD%O$2`S}E8B4hqgav_vz{1$H93S4Y>q~zBg}XZe4xybocUfx)A5c@-
z&=b33fmrzln7|G%Up^L8+P_k@+99%^=+oxFZPuy>{AE5lP#QrkSs7s8%CVS6|I%M+
zZ4z$YKRjmGeR$@>m6eT9@9rcVk%)=r7kS(s=yD!x(;&M#Q#}yBQH55zdmf;8Wv?}C
z`FKYPt&2|uZQ4`6g5n&gycd!)6_`?UwThBtmr*Xv8CT_{EpTfDo0H(vQiP?3H|UY0
zPu7`T-)rXZQnMUU4<^w^ct6ZM&3a;l<g(+m=5#t~7A))P@uH5s|F7PY35Kw8;twm*
zuMT(j&&rop)ZR-ZE*0%@sJ?Y8xmTg;u>tV~^&_T&i*wb-7TD*iua3<-hG(fp8oOHx
z=bqqD5%X>hqKcGO<t;d$f_rdn!=0e0s`)o-Wc}MYb0nHg&m9XDfo^onD1V?ZnpEXe
zTMl$Jx4>GvL}`iyu#L}UAwq;?MMjw+T&UzRi}qrD)<T7y)m$GLzSZ*+30bwjiX=vG
zMH0>o#%GF_SqAf!94$X835|{yP(mAwpOiM6&QSi7tgZ%6)<RFOr)rw8>go0DeH60U
za7m?jZrc7VLVF~-ih>jr50tHj{P=1BLhQ^yV-LD~`HvRHTXAYT{P+zRcUJz#o~K97
zayIKjZ}WU1m%dVK>X!686^x8t+Z~EYl`2#k^M}-V|M2%K{xDFhaNY4ReY+2#xyla7
zCw|$D^b*#Z%wi9=iciMM`G`7Om%m8;l}iJvP8JsHEkt!W;0H?=c-vcJr8>mezR9Lk
z({B!sA#?lRJxzS3YC!Zbb|uTM7dB9%KPl_*eD2HUkUhw=>D3%Ap6QuAYljb>>B%W3
z5_iiuvqOJNb1bgmQGW2JV)$;-QD~oVY4N9CKxR<v#^Wy+Y5Vh$kNK0gawI1p!rRED
zJGT5N+NBa#g}cX^efvk4<!5MKUqJ-b>QhTGx2jxe{ZNzr(rn1yJzhTLS64K~vKzN1
z##BvN5wR(Y*jO>sb)V2+!_0!Agxg9r3W>LJo?T>lK1!$gW-ac`Fx}lESlaE3%d7QT
z2@F%B1#p>ty*ggd%~$6kt%uDgCe2FK8fHeAId1}3TyGugB+{MYo_91{oUUM+WAJ1j
zjq9;_mF1n}H_z876T(bSB~u+OQDhx*cWs@n-`aQnF(|ND?)oI$j2d2n7_At-@DZEz
zVe(Yn8dhpz9vO`?%#+wT`TWjD=AoVf|01>9=(nrS&IRxCO4i=NMsnMxpPS@QQ~Mxe
zMyL%Xb~~cAGPEkD*`Nw8X}ev;u6^C(U=6GBdio6}&TJB%8jd)Q=EJhwGER|~@6PZ!
z&jp{6B54&WZwDqcUjc<RAX!cIhIujS9vAFPb}33!X4TDaJsr{~CDf;~A9uz!HJKV^
zR52{S7y!`qvM9q2C*|=EQk81F`CewPNMlqW{$^HUE_L-(29GAY4kr;YclC&~Hq&C$
zKM%<>wyfI7Xg=TnF|c_uJW#%tkmeW-^BFo?vdJ-<T$T!Y7E|XD{ZZuUL}qESn`m-k
z|5fFt{*z6W^N%I)OWi96K6aV(?-EW;pwiDx#+x$s#6@hZ?9fYDh^DO1t(8;I7xtaO
z12WNaMOrfFxJnqokrzebW~sw=K}~9_FRr*f%j&O>wn#v1;fhGf-exiog-!MF#5h3H
zqt;F<n`R3u+vRp8D$UncRXiOa6cg7Oc}tdBH9U9JRmcP&ulJ*)l9acI--h_^=91ci
zVnjRGJ8PPfhoJh3l$3;kuEACtnS_j9dpq&bXtg4#wT`@EU@Az%uD0P<JG-u6!SXeQ
zJGCHTv6=UyYy|Ug|ME!v?^A!hD|F63R05i^Xbsvul6D@D+nk20wxCZj$#(rDRV_bP
zYyda&({KCEW^eX5;|T9K+Rv>f?m8j27wo9K*f>wz&r2g4#ABqVn|FV=Fs!`U%o^F;
zzE^{yL6Qw2k)H$q&|Y2tAI*gNouyZKdF1>sqUh;CxkmZ+bb9siab<paF8acmq^BY>
ztqR@<TTjgky-_}>88XAk0s(K@W?2=9+Sgg1jF!p1AbVHzRDB_V<Bax{GzKk~pLuR&
zR1Wt>SIkYta40IhP9;yS|Cvbr;j4CeMS<|!#4)+uZ~Wp{V-S<MhCz9DDhSoQqSpS}
zmg+LeN0$na+Jo{z<uqf0lpAR!yvxTY2(R<C0Vi*CYTbZb(0PpMB#lwE(6@dhd#iOY
zc2#+_Z8K@8*Kuzv7k1}VVn5Eu;rCYEmolfq8P6m1YL5RwEp)Hg4erwv@k|=xlD>)=
z+xn}uag~^KKA2gbDe$cN=p+#{tN9uVN*;Y4>Zb<R4!)*8q1@=PV;1_j4nN$xa08**
zd+x@q?~QpcRdjB0#B0-D=2MT@y?s^*{~VHvmJ1#{amYy)&2;|dqpEMD<c+NL{AigO
zWR@bEI^*awcmDc=xtL<_XwSjm9CR9jnEjNHr%o5tU9)!LuDoTCx#enBbDBS1>K=H8
zSeX-D9-psSf<Cjnfc2fU@85sg-Rf);sImVhz=Alrs$%-nt8&Q3(&I5ootc1$u2irl
z&_w-ujMYuKwWGq4RS8a-8(w>Y_+?Hn*>gC3xsXl?O?RHzs~`1jT8w|rbsAw3&^NNE
zLxRp{yWR*C;No(z(XT@f1->64U~gPEt+coxPG!J=5U^Z!&l$nBSGUL(Dnf5?EWTn~
zfYhDcO!m6$F<YBrzO8qKHJ*b0f{E*vk9u*y&ZDYyvsOptjbA>)IeRkh&l)Ix&6f=L
zXbKqVnA|F7Wh~g`!gY6-apQ=?{!<ULkxhJw+Fax<Q+Hv(D;_Y1nw<&TB}&gbj9iKv
zI@5y4cULw7ET7S9-C&pEyu;XWA9Q?Q#I;@D$J4!t>$D~W2rzdpUCD!Cn)qJpMta24
zaWxm`PwI1&V`=Ev$@X@yc)PH>otpd^2Rg&QIREf}T?hRZk~yHwomKfyAO2c`{9n!D
z9(NygLB^N`>O9KPO%~5%CeiVW{L&_GZEo9mSu(YmN=Yk$!q-Xr-lEH#eA>TA2;aux
z`geb!C>{UwqDz4k;d+C+fgkNqEGAY;?73hvwexg;?OO2tNY-kPD`(ppbp4=!TgrE_
zbD!CU%6Swb{bYE}>JJzq3i?vG{!_R3i$-Qnk=t^pu8qC`AGOa3zrY)IYhmWMYb#=|
z?La-TON-~Vo3edZC(oQy<5+2_E&%R3d(xzut3p$ndB1Rt#f~b#I;mMb6AB<eP^>>=
z1Btf9+{3qTrSdo?JeK(oB_8Ev1!=P89U4{1P29fMzyK+@IXM}hR(uAK<nLY3rVHb4
zpusgJU%CVu^V78?EL@36Wqjb%Z!k-jWcol3Oz&D(#y~f6Eky%!hlFkX%lcGvGF2m>
zH%$CVN6@a43zAaq2N$6@!U637AgQUli)0u3Kuu%%?M-R)Z1^Dm7wxOnGI5S>2hQGP
z)vSEO3-Z@+hKuQAO#0@V`(JgtK_fKXvshejJdfzV+NpboITmuXh!)L-&u^d?spna&
zFH}OttQastdbxf81!l>?H5>z5AN^Um<PRIu6lg_8WHTFqp5()oXsv!NALT=K__<?B
z6}-l{^ojN7AGB)6qk#{Aq5(Ppj+nYg9IpWo0ol0wU-SinfDCP@ulTBv-OGnA*}R~S
zF;CIrBIh4vSQ;);!pWQrSC4zSa)tpXGoC-_?o`abnz{8mal{gMVKC0WxFJ%tF4zDH
ziS|c$5*F`Tl>X2n=yQhNs_J@ai5u%}%%AevVBi@#{J$v<G?reJN*mW+0AvA*c!1gt
z&}HF-a}^+X3qL&UaHIo&L>OiNxd(u4*K+@1wfMh{)#49>7VfD6MZghF{QXd#8Ao{5
zk)<_?W7GKifB)xa{wLem|LGzdb-+x;>?^B*<Hq=}7yeILsQ=SNNw@I<i`UGN2<v~H
zIr#X$DYX9E?SYH5J8+C=?#NNE|9*Ra7TLe(#{R1ez(xNE>yLE)&CT|Yu>R4kf0p52
zmh2y!^_Lv}G4Ovj(tmctzg&R-n_Pz<6Itqbcmh%?5AW;$&0+fgha1s$elc*Toym5p
z)}bi)^tebdn$fnKD>Z`he|6a2#<n?Cwb)KUo$Mwo0$fI6-cF;<e&~-*7Z`Bi7hc#1
z*Y=2-P`dGF+jMi^+41$-QO&|>Z4=_K>o)*hPM74nHGsMoX=G+JdLm)3_O|BdDcGd%
z3B^e6kLmSCu?O2vScU}Mwe3r5o=MCEF2u4j-m>oOz8`plWgmZ4^V1YlR|(I>81V@e
z#Ud@xK_7g)`p%{bI$=_zSFFiA=nf)D9xVdTmA>0e>ow6%fwD;GvPz;Mjz_OcD9j@V
z5whFOJz#Z)undv-G$@UJHyMoGK$E98f!NO&lnuOR5EBfS_aO{EKlT0mm5e`aJ{a3S
zD_`EV*^1p<cTMq0zICQ0y%RC*zJscQ7iBGT<L>$ep|)WAqW){M@~T$Y(cY+5f5#O!
zXN!>2{Ri|qi!wGdt-s##-&q+gyCcKSiSsqH<s0Lebv=R(Che4VvFCt2Njl}@>B(}c
zWuNbSkR6M-t68l}?k}3qHaKCAxsu#&*A3f28H_r9y6}^5iH+J(UHB0o?;T?WzA$(v
zd_r%Me5#=N4ArZ`REsI~EXna&=gB)loWhTmh;E37$xqVpV3<;2`hr|vIP%<Afe=f#
zeL220HUQ4~&`5Tg&`>Vz0qYK~RI{i4d%vHYouIXD6e2d8wk*a5zXJi;*$!sAtc{o3
zwVoZLKDv@812>+LmdB1+fWYJ;tY2)pAyU?<qWYEJW+PHPAH3II`VWEv#2KoYOSY^m
zUzu!Z;=omJdfgJz$?;w&VTOcYStIv@rP-QEV-g`~J17oGw@JBeA{u@FIvs25{oH2K
zBy^fPXo1KCy4Xna^9u>PNWccvNJoxBYO^8#y%gQX%7h>(EpXyb2z!i`uya_z^V4_5
zt@(-Al>Lp*UDu-zzu{QHym65St1n)=0*58=_LrhLF=6t4bUa8P4@1Ex@jP2_858QT
zr<`qRVfQ~?&M{r12=ln6J#YtRc6@J2li5=BOOREqMxc;6;JQ_<cXKdK2Wp~Y%c|*l
zO`C&Y`Q%sl{p)CjcB(ghX7>Qq`StDXZ`4x0u5R$tI^*D@<^H;9S0^gjLo4*CmfSd~
z|Jd}LcBxMXPZ_~eHzkRxu%^2A6VLN-Au3am$y|hSq<+r7*UWp|Kq<#VfK&XcMy8l3
zcTQl9v1{XUS#y-(Ise>L4}X<&tlZfP-;E06&+pdu&ahjV28PD%>&@r^u(rf(-*BaL
zy~wGy?v!NGkd$zl5<wp>#x(_INqXP8;{54;q!aymKoqM_$@5#I)Q#;;sLH0kfUOq0
zvoow)=hBb@wI~Uzmv-`7u6AbFRx5U9gGI#NN>h60yL!FYXt9<!d*F7MRB@dHQ-s!3
zHKcXhVJP?h%CsBYd*}Bn$}20>#j;wGtDJi9;M2btpt`xW2hhh$7|meE!*(XiG_dw1
z{RGp`zLE>2c_Yll<;0PKG&~$Ymx&^JA<>C#)^Rc`f#hWe(DWCU;Hlew2`AcAED6FM
zly<<;15&mted(q1!$N9;jRmVes;&35q4V&gl`Hx3>O!rDACK0`tNLs+T~;i-Vr32<
zGga`qYCD-rIgD_kcYB30wPcUJAL)#>03(L9SSy1Na$my>OW@!jdiRtl*Q_VbLxn$K
zDb=t?pgV|2;uO$F^-nofxrohr>D`ckY>gN0vC`_+&3X>X42U;wju)uVc`m(I&S=5R
zHT6t)onwFr02p)N@V&(BZ2vXh{1DXytgfJ8Y7NMxZbn$dZ^F-mi|y&?j&{e@5;{K6
zY2M50Xe+Is4{p+<KR$kwN1v*SMh=QEcE*XLSNhd>$Grs0tk8%Zz{31YD_0?k%J$dy
zH_5J_4u3^wG76X!{2KU|6>ZR!Wd)B}&dh>${w^2qrl_FGoPO$n494v4vo4zA2CYHF
zrzZ+bqt|DJf1*`xTKG)bXf9OwchT$)E3+SsmTGku+vNmJ+=7njmRg_n*7ORZo;97D
z&mxz)6SLg&%cK)f*ey8xZo;#k&p<0+x2>w%8Oyy)h?4~BXGTtd+HZb%M+Y+!QQ_sP
zOlbG`R^S)nbv40R6MoY_=-8Jzgx)e7ym^i@4i9}Cf_-?aK&U%Ogu6;}^T<gwwu^Ee
zQa|Si4F406IaZIZYXfNl3MU%kvr8-WEYn`i-l&Q6x1Jpspr(j!%YIWXWt$U2nP*<E
z6BWiCs1fR($F~+ozzPFaAww_Z{1h|LV&UJHv+yk{)1t+{aQ^G*M1$x4xKUF;7f%Z?
z#*)@BisLdmQiA)WKbY)+8KJp0)DnORS!Z83ulL=m_R+Ut??f8VhCO^JNb17F()dV+
zK!r5v;jLU^Li+d5b?8Lp?<?9n#7AF~|M*bemFTQ3LFs<M0}A~ZonO(fbw~wLX(%|l
zxC+F$<S#yokZbeHnfIT!v)Pz*vfT=r@*I#v2XpfVY&BM-A5qx#_xVq7oqP_@dZvOg
z3Lqt$e%zlQ#;|ddaTM)!I?A?2GO=;|1PXJ!;(I+aWxTCygXqSs&tHoxpE>f+ly2DY
zWya3hf7=}(eEb@Zz|+ENb)*pD*G9_yfhN^>K^N6%(myf9R<H=mt~x4a2p>CWn}f9o
zH30o_LJf*$#_k&MCt(btWZ^q8ZV?+5uUt;Q(`k*7Gv4YdvW`@P#)Jviw%9&iF!H$~
z7q>S`mRjRrzk*M4btpZS!Fk-sGRPZsU?LoHb`sw)7qDdx_R9_E_)ve5L!={Q^>N#E
zS8E}j$^D80H?vO10b7KMo$Kd*u_r-S6s~8<>^4=Ztszca!{VpH7qeD>PZdsYz|YT~
z1d&VDS~Lw18iO06BmE(d*>ThMgF?F?kY5@uFC!)<1Rk0d!AC0tULg@CT7&Qjg4{a1
zU-GR%zus_9OgzZlKdji*n#svFmfJ7Y8t2aP(rp16+i}z*2l;t3Q^}vga8%COT7b`y
zU%iY;z=VEu==RhPnn@9_IJ25(Du;Go#vA&<eJ~-h9{e=DMS1CxgL}bF5HUyc&9P3l
z0Oyuo!KUwCO}hchIoWbJ&)@<^;);F1&kw*zr{q~z%Z`)88vE6Zd`{uJfcBQ1V6n9i
zu;l<C5rH&<+JdB9p9QTnc;j@GfaE2=cyJbQ1D;GPP)QWjk;yBpo~+x+kssw3BXi0P
zA&v$5h~88o_Nh4pM$2i|&{W&ne%h<!Mx3<nV5WH0U4hwN=TDm^Tb2lOUvrq>Nu^hF
zUk_$02bN<A_7HdTRt{l0f^qfrh#(H(R6^I5{fV?mQ-QKDh5D7=1tLWuUPC%D!Otn=
z?pMlPSdge*V8*VvQotk*<%ICAh;uxF<~T-=bOt>Zm^$vYJo9+r?m)~7BcL_BTQ<?(
z<0dVcov0<-I6;XZCxn`zZeI(Pmm}gaV`;85Eqm_J{+8jIn?ZIUFpsYYq!@mh_D`fC
z*Pg)nhc@s@mxnEe>Ze_sB2O%iT9LEIwNr*Kxpzb0s$`E2$@46NV`Yzjn_!fQTu*}Y
z(eZKP4+pZNZ5^*LNS1xlK=oq6NZ3let30V0mX3T;2?e48j2xkD&j==_$`2kmp7tA;
z!@>yn-enqkIFLVjjG8+XFC$<p=VBSHp2}D-9y@;pv9&Wx39g<ngc3BFXocZ1R1}O-
zha7%upeDNPp~n)hEy-l=r2{#|76gvRDd1B(xvWIj<fNMW7@N(5D%#3XF?7H@)?&sm
z>|%@BVL_-W%eG~UKU9e2SU<NdPJ;5InP<PK``Jz>yIifg>#I5Pq!egTmw+uB!9{wW
zHqX;-#m2o6HS5OcT~VTUGC5bpITlb(rbNA?c18lG4bPvrKyns6Tlp45xDB$0VX%<0
zW<`eZz1aAaPWwjhRK$$P*+jt0sw%7M>}UfesK;{8c3{PZfjbYASL~Y~si_wJ4!H!O
zv6e_1h~lpn1Ejr7KtbjXYnTHA2ghwlGp8&Qk$$EMbJJN?z-HqP{oco)L=R1TfV<Uc
z<Jj_Zx>m7ZvFAV$+EaVhfKTf5fN2z=vEDB}|E_ygQ579{?t=d&)iPM3xy?a_=wXIX
z@`S*nz|}Fz@eI6<plu?2!sBc53nP0Bf-XNk<<+^yJC3Vra4#3mpKkhEvq8Q#)bUb2
zDv2M2(Z7L48e|$`I0sSVEKB<cOLo<=g3yTlOYGs8ewIO4PT53NPPD~qB@MAwWwUu$
zCE2_IzX`@t!@}NwA(AOp_3iRS4&kCUW4kPd^UbmFd*Z8pBj+ie_@pwQ{rUM(EFbGy
zE29#0oFq|SuNPQb>rn_GD2QGJ$&p1Y-3(zf%u_wCSVtA%lM=o6)XRt}BeFz@jE6>e
zLobonCvBDg((xSeY+`y)+E(JCYa;gQ;U^uMcG#!#@!kFlijFQ%%zNrwWQB7=zvGd}
z(ePM=B9Awb{_xNVQOo8Kb=vsDUrg}JIVgr7`vF6Cvgq_F7zbr{jz<r#`x)?+Hkv4M
zj$O<bDG6Jl@{m67E&b5@4r^?afg#%w71JeYt#jlVdQ*}euoOo@D!6yWnvMci*DuAy
zxp;78qQ#QJkcR_;^u0-F`ye#*yE@P3I0>#?3RCL6o=@Y>Q{1!aCpQZQ_m-{tGeq5u
zr_~aQ^9NN3Xjduqg_s%%cvwGYl%N`>2xuLQ^d~T`{`Yp8_OVw6^UE3CW<rI4Op6J5
z%rdgS7V4bS{mz>wpB^*Gb=eJ#fH~Gw^Sm<D%QK&%OrR{0tkpD06Pq>{qhvL^qab7r
zvs`U0#3D+-<F(I}Irw3nOEY+Z*2uzdxuoE6EFOV1*;W7Fc{`0m9anl>Z5C78y*=>X
zh*%9Xj%j}_`@J|=-M`82-2C}5Zx?aZ09%yp*)akh+xaxeg&LAd%sB<Spg&i>I3(v?
zI4r6m%JIJL^3fOS;2{|Kc*MAS$(7j3)d8ukcbyd`4GyTcX84AtCWM?*&(h-L(}CIb
zl|x_+)Oz(gfa?8}r4$ZCna;mGa%OpKq<9%_SHkd>Gm(I;6#^S_(j+qsXiPF8N8Y?b
za=j=|wW6`RBlIom$DvE3!P=gz2Z%mx5l?Osn3@2|r!aE0SX}(U;eKt@aDH5NHS&ww
z<zsd7<PlrM;>_!tq>p|pe#Mkud@;SGvsD)wf$=O4udMa+&&a0qVZ|u2hnG^!U-}(|
z7pXR?*B~-*&Z<^H?K;>y%Jjj@l=_hu!%%V9qS*r042;tp6nJr*fK62N0<B|Ag616e
zX31^d5tl<C*I9xy-|_K}zWVaN!pQ|+`{f+Zxn_ze#UbDKS1OE0MhKx;^zPp$wj3mG
zW}h0rhfm5t=EuS!l<$<`DF49v`Jt?{@Y;kqPaI}n%ys4mESs@}-^dan*Hq#0=2CoD
zsW5ez2hoelt>8)wzj$(JgvnH4B1wNTyhui+V6fP+TCBUk^*|avU_d-BZQ1HPSu;@}
zW1qL1J+n7C7%WzlZ|K#il3YxGGI8b=_w<3)O~Q+h`XcUN(>w{vE*17Y?ngFDWGa7R
z$U3<4^7=>yy#u0IM>Or~(}d86$9F#d_L&}Ih}RY{mb2Y|`9t)m*UogW*~v5oY&DYv
zXBdqI-7`rI=yP5TjoFA9;P^pJ26#V}3(D;Yu_CQJfy&=gw)^0&1Ve4Nd=HH{R)7OX
zz>)G%c|b<yi`46h#GFbDv{ZS*7w`#hGHQQ$D<`%2c~BNI9S63dyEP-ah0HNgiP&On
z!d4Y?!y_kO$E=xy45Ywtt!AZg@CP7^k4Nt7FjOT32_e#Jv9niaosTl{g=WRU8g}<#
z1Wb~QGAOM4?3B_Lr(~+iO#<3P`)Wag&q8gULB2yjs)n3NIOl>ev&SI5tr)L4I!0Ah
z2{kSyY~QL7B$J67B;w2tIXh}NDM-}Z9c4aL@AVsDlO9Qy9?ErX^1kJgm|f*9&9W6k
z8kSB23_D%Vy7d5Q!G*<D;v$t(Z~m7SP)7rZA@Omm&&Jdz^_TW*zNUJxPrAD~z!tDw
z)j2kQN#RYzNl%J6IIG%dS4Ic573qXUFL@ol1&*YA=-0xs?8>iP=E2^)HGxOF!oO>f
zP?wxcy7<G<oHKg#?u<-vLFji72^p2+4PBsiVFTg}-c8Ku3xnUIx)jzZu>He2uD#iI
z_@g3(`z{~H`xO-zKY7oyxR8^@uv3S?x)sU%;PR&Y6%kGO2h+~bal<De`NY22j`G(b
z<I-!=fsL^IwG5H2dPKswn143n^l8GI(C?fk713?$WZPzP$BlD9f<&Bi{wT=tTmH(+
zzg@@Cdn<N=9C?f9Ls7Eyo62Q8SP*3%^0p5;RW{z7V;6#-?B1J)mykacnje;J>IrLp
z=Y)dPAM_tRv@#*O_(+{!=*^%(Snc@_xswIzeGpAQRjRsQ)uP6&STE`~W*n)rsGdm3
z8GG;7>xQk$9h@(Ont((*h`0s2z?lTi?hTZFq9VB1cL|>?wagzzmMRij0vG1dGN9yS
z_?qQ8AOVG8M|-=n12&n_3;m>zO8Dsp>kVFveHi%)pS-F7{MYQ)7$w%Kbr7X4nFMt;
zy|H*(>pT+r5Yh5!^@|%JO<G1MtYEuYNuYVENC*Fo7akVqtJFJm?$`aw5VGx}L=*MH
z>cHdCoXweu(V&x^&QZ><7<U3%$Khq{8rgTAFQvqsn)enW8mMEM7CLA4OYc`s-(>JF
zWr$|44=KaO);hhf+X#}yCr$lH7q$zQJ^QKImXW&{0WHho-3?;}Yh8Qc^}B=8pACKI
z{l&GTi3h8u325bxhvS~g-(T!1O^)1rPo`=A+>wYc>6OXY>FHJoMlUvJwPgk)x~A9J
z6VaxAktRNk)aKwel5hks!sOv^qWjGCA3IoH>&-H+Vvw?!#_im#^y7vgBQ4taN0v(&
zq8++FDnlPm&1ezHn+r*|I**tbtX6Nc;uH4JJz{^vYu00#Q_3h<C@`sy7BzaF)avD|
zDSv%KOAG<E;KhGa3!egbLY?6?OV8b+HA2peJ10uH6k&eomG-Z+-pZ+&N<^Hk=cikp
zB4Z>N{UW)uow(bSGFzJ7EZiJ<R%LRCkY#?$)tPbqE#|VeC9HP&dVkhfeM50^CBf)z
zJ`VxKsRL#Cj~cIzE+%n<Wo-~g#)37-j0ue(Ak7v+S(F8+d8&3sI+24t5kOZrik2OU
za>}k^f9J=ECC0(W%|BLc*~pmZuzXAUUzV{Z**XZouPw@MHKQNMh4g%9cr;Y-x-Pi<
zxur&d%uA#B&_8-u*>W(I)>_}ywUN)yk(Z&j6|@zgAn6!h{91;N;Nr<SUG8%Sf3nm%
z5S68Al|vu8PU+&4dK)cau0loZHMs~mz2bIO7S@hAncfY#)5Y6c$OPZePH9WvQhOSl
z&-krz)fbL6H&ce5W+oEnVx*mRezo_vTb~-z7=@`7=7%S%bSiCUy`ziQZe8mvH>oh!
zlT2I1;l9wdwi>VaP3u?*TdT<exv?@+sIhk+84m!cr1k-VY={kbw{pH_1ZUx&yg)52
zT#!%p*x%Rc0bW?mzjm5(@hc!m*a#MKh>c$A6m^yV`m)c_<zy!=MEadtG2?R<qX*_<
zrMnJ@z4vi}oNQ{942p=lTMAZ^xJ6tn=rHJjFC;x;HKEMvt>5)sOJX_cx#n9IEal3p
zE3Oyk_wNiD22_yVR+0ZD4nNYELA+OHn^fmJJ8&)7`<Rj`d(^Gf;bngsPxq9K;FJjn
zv6o4(G`DYygzvk~6K&VsfVUH~9~fyJ(~;O-hC7AkVIoa3JBz9>8C~${Esb*KuAcmS
zNv&yDA3P5r7ilHRJJVv9;_VZk$P(CGtaw~~{Y(HQp%<i6JQ^+3db#tlUTYHhqZ6du
zc`0rob$AK3?2U-LIW!*}GM}wpWu8IH%A_Ls5cZ!2=Fbl5AuoN3@ZY$FV5gz60$KFZ
z2G`=Yipx97>P$b|YM2=bEO%siQ9mN5b!*<~ST7>G<t6hOe!Nzm=aCd^w5VL7CYQ4b
zp2G|0o+wWNOrZI>{z6Jl6O8D|-=$AUv2nC&d46mpmlQS#Oo#)*OpjGz1HjvHoge4q
z(r9=TLqi|(8ig<NX#SL%Kho@Kz|8v2(?m{CDt;l+Sk>q!KL$7_rh4#6LTJRn@=Mg}
zWseJ{<qT2t+ral+)9gV^f|dDT{$=BZ;Gbb69sUm8F1^x(G(4tY>p9<Sme&(7Qw4O!
zrcenS-uA?dbnvCY>!i$;!5sEo2O=>KzZMJC=H)Pxrey-Ewaf`jBL7U&ZV&&W2Si*`
z{%f4tOTGFSwfEU^;ddlu<*1;X&YFC;(D;wNTl>QYR*oGsmS+4!5lC@@i_Wc~WZ#1-
zm`-IvR@E%Yh?dA6R+u%`Onk07&jPdw@k2V&AVINDB6s+uVUyYJgATvtSZ;-*c}4Ii
z1N+Jo2!!hu0VCV@vDPuB`rS*c(H+qZ`xB;%m)+KG@`)GXlNRQwDwiGRbCm76R?Xzw
z<inhFj|XM3&dy;KCpu6oOaIlv=N774PL|jy`FIYQqJ0#i6C2Jax_VaOnfivT%8Tlb
z-dS3&Lh=OIATDco9ys7(8x~{Ld~;5-sn}Z2&#+O8L+2MHwHP9T_RB3<(~a!tY;O`=
zRFB3R+@BrXkIhb*<+0q%5Se(ti4S8hOoUw>3p`cpTv5r<N)Ev$8PWir`sg5^lR$vr
zuhlMp6$&}(ZMb!skg~w@DHl6lkVr5+m*9%Sj74ucy6V>Jo*#C~c|@m0wbkOxIin%%
z#>+OB`dLOd$zb#mI_LbVmOxPTh{yi>tW5oB)#<|X1AwB&A7JOt&%%yOxDh=Mpxq~z
zeP+E8$19m`R6@7>(?`h@g+}d6<sWn&>{o@zRF+RU!oHMFX1hwmNd48~^W$VCefjzn
z@WLbn<rj1@L?0X%MwnW+Bh%{H0AQG>trEHG@6Z4;6bfOw8CvIHJq@zGW;$^)XPB@A
zNPHfL7Cr?-b0H?Hjs=>rh1FfiI(~hMfS(zZn<nqE{Y2sH7)sjGQpf{r>E)Jw%gN8r
zb?-lQJ^s!ud#Qf-1uYC-Dz=NBs4x?9V|%>gHSKtv|M$nQN^X^v``%k4(x*-+Dn4Iz
z+zh*kMg-GOwnR_WJEOZ9-eTFdnzmXmN2u4k1<N5ncm^isX)TlZQ@Og1o<J(2JoN5z
z49^UNII+VlWA7G5>bC|R`+<x={Y_7>ps1H4SL==|9~78{E*Hz4I(R(>_&{CzYqEvC
z$_0lxk6r;t=SLz(?*w9-z@|MtjL1!}f)qeBdg86^R9(BS=m$YspQ*3)GF;Ux7GQ`U
zdLi37NJP#UfqdaDUV=1)%B$rZL9<OZmW!g;{CSENS4+timN$;;fr5@X=J}3lh(qf}
z3}ODo+Z(=94#nHSAKN0rkNNaQ&@I{hatGCM!#0+i4}{%JfMc&t&j04%zC00YwgXB}
zh>&k=M<r5obvIfN*2P2_Sd<<D3!cV+Jl_kefK%=M;o3<{Pt)-MgOKoyc`&ok8K7Z5
z+T6uQN8DyPI(Z9NB(-t4*$x4{ptt46QhGrdbl=@-(;I_&K{ITp6Q>GataFOBX00<4
z;U9R|F&XOF7RRL~RW73-ImhP6Qj98|_nmR?)aL(zj7dbG3j5<`>(%5uF#`Q+rWl7)
z5lR8I3u}^g4B4W^I+4~cU$Sf^Pn7V&O36!d6<*pFg<x2ul&v#)57$Ca=J=!=<{0*v
z8}Ezqx5(PqKqQ-o=P;&IMqF$nR&exYL_51QoY{cK(WiaM0bmNyciZ<rj4lrEEboeF
z<sX&Wc@f!;hKFcr&C0FH<T@R<3kB<J0oY&RW)B?$#Q-!`TlPGRI*`LTpP-;_Q@+-6
z%fzs=;H(qsf~a*EEz)jCZXziSxXfUNL)j#7K%ynN_wBW#?Gi00fjaKNY9j*47{xI;
zb4~PDbx`t_%E_9ZiQyyJNGz;TD;dp0?Ut!2vp?2Q1wxo&?03tv-PAhHw751#018zr
zN~lz55%4t|-E|nOz|43KhK;H<wGT<+#MkC={+=op>EIo7w&50n)*z8J@Wi~n5&OC`
zUu2KgW9UA~xZ|$x7b2zz8z3dFl1(wMb=iAX{<p}S%|MnxkL}j;pP@%#pb(WzyJ0NY
zp~$46!0^Hxt7S%2S+w+4u*xh~DgO<qSJ!^XCjOg3wbTi|f+;^_7Rhd4%lDFA3aIAq
z46j>d3rTEI2j%X!#vWXQ@QQ(u0Xq^3Y1*ipmd{YNgtm^^1-R<Q{I1=40ojUtg_#ct
z`LP6ie~QTW>}T6io+=bs%Kn<iz%%UaP$#Tw2ql&X-Wyg%L7fn<HrYfhzUgz1cj>d`
zHb44eEaRyq$;+<qtEH-t<4~AaS345=9V8c64fQ19z*AKC^vn}hroE7uy)gdkvAIyo
z_E&O9*!!6sUN^&R-g%I&^O$H-8o`DWa~jPF&Ff&YW+ZM?;}hB{fmM0mj;JAXFxmw^
z?xHK{S3m3Ts>$u$R3Nf<>uf9JTuM+^eri;+$$5m#kr#po6T)Z_UDH=8_}K`<=tUC=
zjLOuxjK~_yd9^bbMab8O&jN6m+UCAPdUE(xm&8qI*}b7a_)zP<YYj*M!!}*NS)bKj
zJW17sp5v#}$Xmr;{Gn!M19j&we{X0k{IQ0`QBTst>CWRx&_TafVQ6F{aEq^rS(8^l
z3^)Ug+*Xm?$@?gC<1jCq$S=t;sd~ooo~qnWMW^}s)k`bz@@ohVd&u(m%XcJoOeO?$
zY{C$Ziybk^41Gq1qtKc*2RoMelNE9P08(-kpw71mTU1^BaUsYi(q`dtMI3rSWQx(G
zjbrJ(<jv*x2VZrK!vbxH)?6}65|q32mK{Z3@uTjP){vZp?L&MVpBtUY3_Oc8NoM#F
z*&Mvpf~j2RditUN>k;aQH~f>~rG>}qkv0`B(}p!x8aA^hktj|&zpnb;?-q4@Sug9R
zeWqRb$*1%Uv5r<uFk2Hl5RzU%<5cpZvr5>EjI~;kmdQl`Av=?TxY8874*?5Og+`3_
zmA9T-7)Lksz=*E?cr=KZU}>yjq`H9aJk7%hqxeLdU;aMrHBr$QGkl0zVNAQYb<dxq
z7EjG4IgQVWcLXL&PLn2xPe41$r3M3|gKy&yh5%cp0iw(s;IxJIXZCP;Z+ADaaf5_i
z`6NOm(AJU)uID9Pp5PX#1f5T6T(!#qTqx5Ay<fqPvS_X*mb=8G1^g~QM8u5;c=(`C
z0Wa(};R|=>3vUd-F+O299){vf8F$pfz!AsREQSD;8qlZhZ-qx;L*y;i6qP%0_}KLa
z0<#dO7GVWA#I$6ltWP1A2TWAaoQjJ0&zSgx9SlF)ZOnKl6mVbiGyBW=1#k=UjRZ@>
zFA=t<$NS>|gTHKx%jP7<?9jk!4KS)}E2}x@Cw4a0Oa>r#Q8va}fAdOGlS!k8&EY@@
z4UajP_NvkA#Xlp0X;<J02^urnUdIXM8KKwU;}(B!sMVM79^!ba#h=d%tr>y9LfdX#
z1kdDkrTY(%^FDOkn1a3}Hwg|pL+4a&0^~(l@L?}spVEJrK5*Or$Mm%+b;s`n4F=3z
zAjS<T|KyMTj0fInJq#F-GEt`qecYyR0kX{vBS2Y<`ayQZr1^!rH`wta!yt<fKqiF*
zhg^k4f4}x;#$aZj0C4y1X5an49s^EpR+TsfI)2M5zXM7@^tV8gpbWT0K6}I!Ve(%H
z2sz1G?_FiV%>kzj|NF0?OB@`-sr=o3|KWfCx?cMe9{!tfA*!fX|9J@9{aJ#U?k%f&
zu>Gg+`0H=<x8&tN{*3;XckO?l#lJo^!~|3#s4?LK$$y;dpR0(-Y2tYfe?7PV@87hC
z3ll6WIgz;K_Mh+TAJP0Hn!jnrKU(uQf&BaQAJP0<7JobXM>KyE$iGkj5zW74@wcOY
lMDsU+{QLC(GtnHf!+t+~u<sk=jR$^I6g3}KKClY^e*oq~gl_-<

diff --git a/docs/figures/widedeep_arch_new.png b/docs/figures/widedeep_arch_new.png
deleted file mode 100644
index 62307687d0dc3f19aa6e382c3d9194ebe0d89db8..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 96559
zcmeFZbySqy_cuH<gp`6vNQeW1l!DUDARr+n-JsG)NH+r*l!73gA}QS+2BJuJhje$>
zd*;3+zR&afKJQxZpYM8D3m9gu>pJJ`v(Mh2z4vG1rywVB8HXGP0)bqXk`z;fKrqoE
z5SS_!CU_?zZgCa@!6`5m6;+TD6@@F<SQ(i<H-tbW{lX$HDJW@>B3E2l<6c6ruLcvk
zTuZ)&ke>axbDN18n?TyvEAgoWiJJahUti>9ofL*L5=4(b2D3lLomuZYIE^b$zr1cd
zK~g`DIyspUwpt!qiE~`$of#=No)&;G*yX6lUSWm=Q#Xdc+$&>$_H-h0R|JW+j`S76
ziin^ydh-$;qMSNY+xUhBBHQHHeB144<G_<xyapa{2}0y;Ln9mz;kEw4u4>wkt{(zF
zcwD_4@_Y?lPb7*(<eFHyv3F>ilY#d&dNQA=B0sDRga!r#7WF%f77fuOy46If%QpXJ
z(XZf@FDuc<z#6oOz(5(HMiXuQmjk|!<5ydy4?`0sYj<Tn>`mH)8VVFgNKrfC6n>u@
zGCY}iacK2RGD5E59^w+O_#cYbEqI$vRLO?>ztulnClY9WaGZQZn<nXgD=jKw>MD1$
zTTWW-a|;)RIu3g5YF;+s{#ajA5(DQ~@`beQG9=7Wf4;;U<rv(~;kc`qDu315;Mi7)
zCMk_q;yN)$vxe#TF*_F(TeTSZ?caLv^%n_#3wH-yANC|{Gf)2PF-jBLZQO2&`=tke
z+Pd51|9<KRsgp}{pIGB!egATS`kzgLsA~AIL~3gzc0_jbL)OWr0fK(ARGXs*sbo$t
zRsy7u3;}i)QZNtw4gn6LM1Su44GgNNKZb-)Y^1qD->OC-e_dT`nz@&KC+j&RGQB^8
zd5z_H0SWbwC^DVfnxa|wCUMWB+Ude!ZQ2lI<0rDX5f(`L9)zQKEjc8XaafEOdIt_=
zN<>ph#J2OgdQWtt5KXS`a&sR(x(^lsE(wfE5=w@Eg(Hde?=nEd_1XI{nd&U_@Vk)T
z#u!X6)9Re9$+Rz(esN#DzUd8J#R){>>*IOD1;o5ef|-&r=0sD2iNAYm!&w<1I6hB5
zqBD>YiYS_36kTERmQKVmVO7RGg2laT5moyrH%r}h>9UB`N7j3=&?kIZP>MR+CnQa;
zkH)c_ps{$YuWiTOoNj4fQTz60?DnpJ+l?Chov*oLkI~^aSZM7Jn+Qn$c%k2VAxTIh
zqDuWrl1%X}2RvJn--dKf{Pp|R>y)owKY7}8Pwu*&4DIVT8i5+G7=I}1o86O*42Z0Z
z6l%hn<`4H)X^4J$BcJj)9v4|9x)$>8H>$e$I^C}$4OjGq$E<lUTL_GUHtOsDD8Bgp
zqP{Gy?BNivHhs=bZvTb}m9<nSW4C+p0j)ns#~jD-$6Nobj9pqyz_LNV>rMP6hCqVg
z>N=L;OTMpUUvGTH{d)beSGv3%E#|eo?}p!HzaM>9{d4yZ<V9ezBpWSnD2-%Ba&fYa
zBHA2b#+yw!@FmeA!lEq4Q2H=Od)ar0xLl~b)qt)5yWfXWK0Jgw1V5zp{mqck#|%Vh
z{uFfYLS*|TGg7n@1ybBX(tgGKww>Ht&zxeOikV#bS({3jv9HW!Pt6&j`B3m-wfGHH
zIMa0oCkAU`RYql|SmjyO>nh2NnoO9?Mhqm(J&d|4l*(V3$rzj&*TUArwi$I4=F_M-
zbfPc4mC%wHl&=({NSv5`ZFfa5Ojk@;W+gF*fthiTL4uK0B}Z8(d+0afIwszgTXFe*
zFFv%_@3K95M&cT--}74Ia^Z5*ema|0y{vemVtPlqjowrMlT!C1PFec^v%>jnoPp7P
zn&|_C#X4pYU9XKhjrp6Wn^|Jr?Y3{(s}G9_%D9*B3vE%%e<F`27b1W9_69jA$7@cq
zXv^q~=#*$<jyFXnnk<?RvK6xnHCdw?Ww_YmG+L^DQnsu1Zg+>9t#mK9QFY$$^6UE5
zY1-wS^r*vd?)F^xe97FqS<^X;`RumLUe{##F5|hhS*u>F-V+mj69S98k;H!eHii1&
zXOZUWR(;RJ?%Qx`K3BG|YrSsV#3si+V(K=C&Xvz8bw7in+z=;bAPPs;G1{T!OUsMM
zThU5_X@h~!e{lW2pUJ(zZX9_jawdv9rZzs4o7-|ZynM;FR;y1lS!=P>>3N31oN<(K
zzu8_-<9B~!6;u1x=%-5-)dLmv`)}W7ysa~eXhzeDwn$uZ8RQzYa@cktti-IOb`?B8
zKVUkrB#9&b%4DSTqK>bOKg)q<YPG+;<^JGW`ov8+BRNw!dpVyCycz82rG(y<tcfQ3
zk%gg!uXa+_{SzGvpO+f#vmM;0Qx|?X@{etMTQtcT-h5i?P--1fmSBA{J3gD=F%e7u
zXKq4cwtqZjl6X#Ke5*^Vfxo|U#Pxeak8yb`uU=Me?Fz}L<vYwb4H;8`yZr~*o0YTj
z+xR=VtDYqKgm7$=%P-N6V5OMb=yR9+yrX?Y?$JA$uZ9wQ3$CZGef{Tw=mTcagf|+m
zv$4$a(_XRS<l>$Xj8o+^{&|u7IGLm1u0J{PO%;l*`vdG1v=!V&mQ9V33;AaGU;Dy&
z+_aU4yvo}5bT>+yZ~B(LW%XUr)GyPY^UlQWCP`%#sP?V!t(;YdaZ&SqWEp4Xj?`+t
zAv;N{O80J6z-iDqch~Yz@i@3nLxINW{ba0i2uI{%Y!#<7Lp^889Ygh5>MVj9sV&*z
zxc&Qgd6x7CoLuZ&o?%koqMk^Wl=^VHA~GYYoTE_bITH)P4vT|!mBpIxcRd?8Zlh*!
zcZjS+%qI%PTXXmwHIxytjj{5NlENvqK9?yMZT?^jHKYiy{<+n1<jiZtYvnwMvqWsJ
zwwY~^J5-6I8CLe$qIh}!up7<Xt*0xCE?ZX1mA#cFmv-f-V%9!;k9^O3{lIf|?_&+3
z2BFwiJz{#aeDl6Aj^|>Ze)W@v*DJ3MznY7ah#QLQ1vuCDdK4UH9Hy<km7?J1>bDpu
zCM(q_DmM>U2#+SL4;czM&i2xZk&a4vm9kQLcfg^zDfRh|Lx;oM_Q*PNJ^Y3XKiA5x
z=6l5#wH?(3`O?*9qTLN~hjES$<x6IL+4+?wI}w6k1@X7{7cD+>rLzpa8H^}g**i`@
zS`sew$Pr9*NSv)J^-1!n=6u3QYQAJ{zR<P6)%|U&WB;S*+8fK8+X6!!`aOB2@7vYw
z&?(<jIqyCC%BmxA6jXBK*$s1^9M7tG^d+$;*Sd<bOp|$2%1h}=QZ|xDf0d}F^<1ls
zK5<$7u$|G^*_g&1M;-S|nB9YGXK|Kyt$cjr*UZ5nNwHuJf$jK6eOthdh(lJL@<sQ(
z2mF;si^Ma`X*qnlg3fP&Kr4o~QYlKf<%Wu?os+)he3}bC$_^jAQmVy2V;^5W;WBfm
z(^?s^@?u%1_NHBJA#=fkh@+e13fi&Hmp60xqXLc29NSK^_$6tpeqTci(&pteJ=u@5
z^Lul4>=KMe-x`m%QtfQ)>m9{!9rSt~&U~0LWG&L|x}teGo_w9E;>hOsx0l)<wP)Wj
zP81K-y3IfJHk?==O#F~+`dLTuD`u^<enolohMUz!jRl9n=G>b={AV|Dg)=-p?=|fY
z9n<aq4#=m|ThsepOXg{Qw0u%+Ebz=tyD7|?UJHtT!&;FLVucSw_+cO&C?Fk&8SyTP
zi5VwIDcFf$+WxXsKAOHBH{|MXC{`JE-8-D36!9aa2`JXbBo}&!me%hdb|?J(nex70
zsUh2VT9^YE0&d5xqwg|=j^&+C+%d1|TGWlsc%uDygY{=)qG1JqSbsxxDI-}~2ordX
z1wn_BLomQADEJnFQk=bh2xWkv{qsHy0(oN!0U-FFYvjOh)K@V0M&0x8Z?urtkW1h%
zLh$XB2s^zR6Fm{_^fgQsd<GFw5|xqyzm=ZZ7#dpInpoLox$8`UH?XZGHEbae5<1j3
zR7#O{6+FM!R9W3lT~>zgnUw|069cQKhAd7N)~M$|1f2N5OAAB0CvYc==a#m7PJ%T5
zT)_ukquyqvf&X)fow*>5x~u|R)XK&X&c$+%<sOX?4jc{_urV;=Qxtpn_jd4?AdQKg
zoi!gTtD~bMiz7RWm5nj$eO_K()_ZKMY;1SI6?bi&E$yB--L<s6_3uMYpCe{y`^?7F
z+RoI<5{`QAlc!epc7ilCsDaMD{+*|xlj-?LmbQPF1s2GP`h@j9%RSb!XM<Y>Q19|7
zm^v9g*AO$c05St(2yycA2>f&X|MtoGh!^fuKfm)nC&#^uw_f<@f4x=3*3d@O$^wjO
zCv?7Fe{a6{;olntSW!#AAc}wG{Li~U&_Xx@tY@1hgyU%5Bm(x4(o{@d8T<xXhWdgs
zgC7k4{zknnk9y+wX$t~DK%~S(l%1f9!<S--ySM7rHcAQ~X}GLZJ1SRTF(lI5Cx5mV
zbDgt8?E5<mbULOanD_l-&JwXB`lqux6`L$Kl6^L>XpMTdbVn_+X!W}vaVNA^<Tf=`
z@9<l_aBQibDI4m!{TvI9guwpi9|T=678p##-JOi`zgz)seTYOtZla?TARy5H@y7!M
z5{balHhAs(--ba#pim*S|9J|ixxEAzcoCHS#Jl=GkANB)26MywuTvnvf`SXEt>2T;
z{P%UBqwnJVA4CCzp`$+-qhS~O9~b%0&{$ZjWdBVH1PC}fIu_Q;I&mZ4|GrWJf-(00
zx*I@>SOf%EUI9%6SO4oUaCn{Zf7u8yM=t^bIGg}R8bwX>Uxz^;kgflHr(o1@1Xv0S
zo2b}-9R?^w!hhc>^#7|6|L)%ZuN7hit7>P&mCbfqW^cK2g_e(R;MQ9zVGnJW%~|Si
zuP=vcSygx6vanV>J563=L_jrTd&3@6Ur<eC1jfT4s=M;SM@27Rz4}$<ype1>(?T`(
z<;_hG2G8S-1Ufc0#hT-N^SREnAc=wT<ny!jg&rvCZVuzE2NxBU6}Reme}T<zR^b}n
z?qsO^r04OBr@(*@sh-<oUDqW}KVRQq*4jhG<(dP_ppu4P=O@iu6xcCqZHb3or>A35
zOTXTqxWKgY_TJu1LW)`U=Z?=R%o%FAx;3r8v2$eg54M+V=DQeety84Kl5M6M=j0|G
zDh>D6r>2V^z9pA>Pb1z@-Fj^WTPBw0@&49=G5O(Ur@SB)UY^AE;YRCkz2kLKY<&C(
zGIk@$Qp@4R(%pBbL&K4XRA7+U49CS!B?z}ecUbJB4iZ@Ry`x#gFr28v5)oye#myf;
z${N31?d?)vJ0o}PrSIFfVOo~uDT#6m`5UvB_rsXg8o&5lni_v|b6cI?<<Fp1txl>;
z)YAcz(VFTzx>Y<nuGrYvgx9V;)72_4@GCO!Wf?vYSbJCJca1*TVQIi-x{2%>k<-c1
zPIYHqw?F2q3Wp`8jhR;NFZ`Eh=lr2ur}G25P(TQW2Z!P1-a6f3YPj`S9n|)gMy)vn
zVI19?p(LT2t9uv7jUC24WbM-w$FH`T>Mmkcdoa8*8DmxRo`1Pa^tNoYa$6ef>V~OV
z_`yo;X!0v5DJjO)jWoe014v{|jq`?)X<y#Q!beY^CMgw#Baz?!Y+b5gL--hx6RFpF
zR7-fs6;N~?A04Dfh2FN=UeqAtcgVt{<hfTl?2LbAxZvM0fW6+mgdG1nU4#kl?AzUw
z!x`aJp1H0Jr4-kl<tsnaAAg#BMXGzRRU+^@b#!4>A~y`C5nzOeg?IfP;>TNx8A=4A
z?&E4d+T{)o^Bx%vp3Pj(qeshC8*&MohLb<vuXj9RmGUE`6H^<Wb-XAnwrfCGQ8>A0
z$AKE2$jxqVl%IOTc_F)WG9xX%!mP{1X9MgQC!^1m8+^2{aENWT+q-yOlr+_dJ}@f%
zaMro0U2apK#n#r~@~y~v{Of+(^XR16zWis?Ys|S-Pdno5X4^!H7&5D0y?W)BBh#jW
zYg#(d5EL-o!>Un`+?A`R$D&)sz{l^={aHovvod3Q7hgpqb)m)1^03Y1&#Q%cM#Wcq
za&(v@Sad>|R6a+t9#BRT8V#0MOwTR$7mEAb8mN7ROFB>)xpJADgQ1Ng-vNQhZ3kjW
zQO|#Bv%C7l7}LcP4JHvp6~=2j9Z7t%*sPn4(Wr|lQ&yNX`uL$aiOF5Rz`>4`N6p5m
zmctc_6?SuWR=FC54_WkT7OQ$nEVQ1b<U}7!8n-RXhqCBY_)gchK37`NC^U*Xekd;P
zw_OgN-D2h&*pU7!UeGo9d4GXXcumD@v3AMxsJ%lD%b~tN-ia3&Ylry_RH3!UTe;H%
zCQEN;XJ-ZCR6SES2Q2#w#1)m4#ATOqt72!<5<Ei4In5$PN8ERG$T&<CbgvleooTq2
z0%6`U#6a}S;9!YBnS09z_p5s?hV2}+xK-GR^Sq%(H-}p*ennd*tWW)p+4&Jj7P@Tj
z{&aZL@m--MB+d1t=3v=&q;tb)%Wzm<)-hIS4;X{yW9|{BPp<o$bIWCAWp)h<>aVZd
zaOr-r|25&{(74cQrX`NueH7<j<c^-l-jv;n=h0U0+`!MzwVnrqqa8OQM(In39b!y-
zqxY6d%rGJwP7Z2M7GiroQ;%)_#_{LLO>@`BaEX1k<dONf;oXhl9@&$kO1_|O)gAZE
zc8R#&4xv1pzUZCW-9G_EtT7Ee&8}(kE?1a|!}f;y5{N7D_XP_r4tLiqnP)#NQ&!{>
zTia-H60z}rKWgD$i9A}VSYX-l#=sp?)gSvj(cMuo9zVCX=lu9au_wOZrb4{jGmL_v
z9$D8?V)cs8xh_J?*Qbw<4^}qp<OClEb+__TTtICa*m)1?_OqidpB_{!acG=)7H@-A
z$@96U>DJjq_DXvr7aN;`!;OTK53bAQ4gDz(Uf~%1s3s-|29}OLcL7aSD~b7%T_@T2
z!0T$+$a`O!MtWv68PseO&Hb;%VOU!6u^&zZ36~`Bjb_KRt=FO9^Sn|jMw+$83pO`m
zFeBh;@(En*9iF@6mrOfT-`d6s<r&**L81AUC8@6R3A6raI@tOc=mxv1<1GijBUnin
z9u_<2u;<%Pms_gpe{(B6`NTfbIlWt|SKG6)jpL=J=dNO>^6Fi+{X$P_dVV~68Zj*T
z>ojnjT!nk~VXt%I+oDePD{{u#Xj-CgH&%Q}$O+(c)pO56BBPGtmt`0HmU9dHi-pN9
z$2(7xwMVT-zm~QgSm1R2<3}%ZVz1)ea$L&2fKPPraWBW7sYpKBLn2o^O)gFhmwurU
zCMc%{nQihs8cG?qdsyXhoHZ8QuUD(}1zK49&I<Tyqn*Hn(W05Mo`wN>ouwpB^v#X1
z!KJyhgz)xH5;4Q}P5T<%=1IN%IA)Ox$t7m7gU!dzs@#H?F78?sjsY>_nEm-l86zp%
zn~rG?{3GPPZ@{nM`eQ5a$+wOr<JSF_1sty;`g}|E7$NmyypEdKsb{XQkXr%SgVNc5
zN;<}Q4wi>2d!sIoRyo_w*dK_<F{5X3JV<3FQ^V?OSa4jx%<h*kc5rnmRKpV38M0j#
z4<Lzhupd&@r0TSD0M`}?x6ET_ZX*YAW_gCY!>G0!{kccoI-Q3-f;(}buBw%q2F3lE
zcmGUG$kryftxJ{Ke;dqZrnJM`?|)P|heOPK`Ga#V@bQ++OTil@Ve`K%QdrqmG#Gtj
zdzUV4<3I&-iJfe;ptle1ZtUeX4Agph&TM%*>0>0wBoyiBNQFi;OV>{Za!$N((apRK
zX2D_L#ymKaTfH64Yg}$KRWKaHeGWPvXu&s%D?VC$ojrZeE0`e9N?K_B=dJ#Cz0q5i
zmK?{&JGJtq39M+a=<VA}Z}lplwF(_%*@kWpmD_Iro_0;dM1PG(_H^HB)Sy&p&w!Mm
zjYHJslcrU2T6MXlk2-r-^Fhx%&H>$ws|Y4~5x(lU$;^-g0zP3p*C{V-pA%Co{jGo0
zV|9Dbiho7Ro^~;!*^wW(qGR?)?A<v!{I<_6edOlk;+wCTPx*Gzn|Dicxu$wDGcx|V
zK|x{u?x-{2_``9tn?RvOw?L+y!XA&{oAclU-4BFz@bX|hwA17KGZSH8kX+7~QCM0y
zs99w4WP4$x+O=ZXjf@~q5hG<G;K-mOS*k@=lUXJ3s9=vO9SUWhdN)_=J~7DDw%DUD
z;Q)nJx94)dzUi?$Zil%Y8pUoLy_3Dkzx}9X#~Tl8)?QjG;eq)hb<@v70H!z&bY3Fg
zY&+J(_}E!(DV*<*$=}O4kzevim3?9<jHKFir$k;2K5E<)IllRe$t*6-R&y>f_8F(b
z$KG$!(6BUHgQ2~`Rw3J#j7-D63jyd)8Gsgzn5tf%BDR-Eqz;5D%{x6hTEeR5crD0N
zChboU6na-b%e|YGImo@@7}EtYHu)=5PMS9JiVtImc@XEb$4~g1Lsj_ej@}3#j(=vo
zyxiAd6#GIi80r<<@wlHfU8~eGRhUc(yMM*Jg1WMnTtTvOiijX@UN`a?E+0G(fVDLF
zLZfEs!4%4yR{aI_7P(dV4JmVB9LPd|1}KACpI^!h_nr$PNi3PaXHbHBjyAC1W4GaT
zZ_ZuWV=okDA3}7Ib@f{~SnkvwPO;88Y&0N|Dx1CKQs%^Ta=oJne5_gh<=PVu_L@}L
znCSdDJh$qPv0QL{Vs8h$#hB)P_g;!Y(^9Yx25Ldn7BLP-nAwc)yGd@`u=tT;`mw{Z
z1_#ST2eVQ>dYvIRMmKF|)N|B=xZGjTo@{X;StPJYQP8mWbx(BLM4_hP`{DemH>dK2
z6Y!xT^chX|vzP)Nhm1=VSA!T7$ML-Qn!UHb^>D&*)IGwI+Y=8fS%*4L|BJWX(dF03
zx<B;+aWO~2hpRqHlspy)#3h|<P0qrE)KBe1{Qd58iw8w9tT31ZP8>iO$xHqlyN7!l
zEoH)clh<bK-N!PK$P$azb<eGCwXHc$D@{r*A2jmWd>>el@Uc^TOPW2|X5Z-XcBwPp
zySUS13`KwN2xy3jj@83}(<Si0fX;rj-FUGv-Q2QZdNcX1as}`RMONiYNF<3{bD3$5
z)4`9_k>pl0;JPaGEOBc*j`wDaPO{9f+%}rI<toW=n+Bj>a&dfy12<g5T^tGrfXBwQ
z^@wX<nrm)xx9TOx8!QaK72X~5vCM`hdXS;P97mkfoOO&7X6y^mU>d|RCB}uB;;J#j
z4U2a=srW&h)Hyoeotfs|@0_&-Uu^A+<}{D(-SPs~rgpE{N~SiN+bTXszIR_t4Wr0%
zcp$bAj9xDE7SrCJnAzx^(xZ!z_J70wA}o2XJC>z!HV>`BjfaYU&TaMTtqjFBMMhLf
zhcQi8H{tZAUgkoXte63pPo*M0m}D`no8>pa-yO!S@zrKGFW;Hn^CxBX=ypZJ#Jjg*
zy4K&-)n&J0HCi)V{xN!3cG2_XV04;t;73awe`nmM!~_FkU_X(hOv*nkdU1YUCvaLS
z?v)X?PpI1N%wcV-aX-j>oxHcSM6rQEMzFiTkQ=u#&*U38s3b3FIMUl{{)6~(iG-^7
zD?t@e#6f!Dz6E|ZZt&pJ*H^e2%b&fj3#O3-we0dX&nFfo{0<b4DCB$Hdtu9!P+P9+
z`0(xNmS2b7C3-&E<~9|kHiUw3xi@)jzFV}8^zLjubo(J|Jh0$O%&cmcz-#-XDw{pF
z;Vnf33%!d2)T?yF<+rJ4;|Io;%L=kr3jwSRl#TZ2JL=Zx2ho%5_THEeeyoF@+nPA=
zkzP3lc|pZi(eszps-fB3p-VdjxL77*btCdh71~u!?XkM8pBVi^LP8d&*N+4<k%)mm
zUmf(=FwsF!F`1*B5i(~!)fhG$!)0N!|B1V6>vqM+s@hCoOmzs2fWT0-<BIM^$@K3N
z;30<d^YdGFaoOOBUv5-773Z)>y6Vu&$1nEOy?(a^AVW)^PC^brTVhmo!6X3Om7lGm
zs`EUKTpZhz#D~f825UWQ3m-jr;O#oy6lHmrCF@!BO3gv+PJwG;f}jp|Qp4taH*=-V
zg(wXI^~c#b&$>O(Kv;b*dEVbq{wtH)YYVg1{gXzmvHd4o08U#xG_{K@2214R4nP3H
zzhX#Az~itOSt%36)>6YaQ4N5?^bcEQ<t~-N!6jmt=|LKI$kL|xeg#!7w;8)7pOS8?
z>CH?ihHI9%KhcYYx3=!Al}KcKq2tO(3-8mXPlvKKi*1bN&F>{AmSn+TE*_O-d4l1A
zQ0TDnaD}~5czJaYCRWi{mJXJh-c3%k56jV(G5n4>>_R$zFNW=V%O8DUc%s`$B<^4q
z-2{Vaq9qzNN4Kn1D2gU5^vbJ#nOWqDdy?BLa*)z1mHP~g6>DqCbnCK=!H`?|1H=KF
z#Q;aAM0W8<2VRS9Y2t&5bW>pXhxRoJ^pqU=O~N|&78Abx{3*FpVlikl5VvzdE47it
zFzREd{_bh=q%K1#ZO8}nvb5tTbt{>07M%e>hedVAAIc&nkt4B7qqVh4cIrXx9Q1q}
z8cIoh+=HAat`0HNFMDI=x0@9XTq_-xhM3iIWE|7MNv*4Ixaqk#geVCrAAe0T#1|l(
zxpebn_YFImR#5xK?0pyZ3UFFe@ozm0{5fyEQQ>klj_)bSZ7HY*V$NaDqXoS}K`vIS
zfxS8Q9)dR8WS|umx$5~eok|O`Q$^65gHp!Do_kXfE&hRlmulC3P|OM8lZ(yv7p7^c
zy&JOYl>H&qd1x@SH~@9vH?1{x)LGnUiehhI>@vTZ$=y(_kb@b-Hn>Rfhc^Xp-5`G$
zBpLc6O%$ZmtS5`+K3~cv4(7s2oap`_>B6ET9vmaP;L?}LwP_rA7Jq=OYAm{Th0oS+
z`g6pK@WoZ1^DyYm(Rn)TM7%IvqMWHJn?><~_p>@TqCPt6j;bc9=|EA2I(NUV!0m}E
z`^<CW-`>a`xKNotEdOvvgKfg{X|r_1B`T-7&EL|kw^S>L@+~<6>_2H{c^6GDeD-0h
zN}MS2rwS<Mdyu6u`xN6o+v5AmG}UNY+xSNjEK+aDIlots-RZfrHgscgs`?5g4_tNs
z=0s246VZIdp2<EmX7>;I#W3L?zi@CSon%B76B97ZIla_PeZE)p-|b=Yk`czmXjc`9
zn;84z^E6xkUV`3{j0Q8tcg-8`C+4#CV|X71m73W^wpv+wOmBYQFZ^sVpRc-4A9erP
z4^Gc;+eHTc^ISUmTLAJB%Rk_upMpFr53fL|*VxVbxA2IdmrXPnLuCl+HN&WT%-(2Y
zVB{FOsft+V&tpdFBRvxM>_306uh;pV!Gn2vI|BY4l}qo#FfKb)Sp*yx<i;m6Ne|tH
zT{hFz8N&=fv{qm}{s0xBNnjpztLgo!-Wf3;DCoeC>gJ^~^$ooML*r00tW3tkIvoN_
z7gP+sxLWua8bsSX>QuL%ZTs>`w7{P%2H>4Hb)S~3Z7u{eAg@KB-5c6|wq}vL`j^mY
zZr!50fB$}ZL*v*5WC--dn*<J5XiJJcPd3#(gK5^O*%q5a^{j}?xBB#X?q7&`0Twd^
z)6vCD<2@VnbfRaK05@uoXd<02@;mh{XOlUV5lE(DVL6e-PXEVj{^LQbuRtwEFTL~j
ze~f;buKVXfx?u8ShSt&NS&p+Q{C#d6DL`UwALU|B2maq@<fMT_A|t+-UipvF|KmY=
zpxCkciqGPFm;ToU{a>N{?}GjRs42)a48>gEgASG#^l?N|LaZTA-RDSj#b}d!%V-|?
zGna%*syDIbZk(sOz`kL*0G$-^^dp0xDGQ<vw=9%yYl!JZf+jq}n8fektLh8Aeyhua
zao^f>{`hCm<b2(tyHoS2Oxi9yFDYAG`zkMG!{9|dA@Bvt16%vtZhS%2hcZH3Txyc3
z_8us{b3gdag!ER(SH*uBvG?|6Sgqbyf5&`pv?mxZR9nt+^C(^M0Hvxrd={9_1-*jz
z!Wc<ur|QI{`cLA^u3eeh&ol3DC)RoVH4`snwb4CWS@c!z6cMG*MbUcDf(He$VCsmR
zg+mDVBd-quAq7@N#8}tOyD@merYz%~z7cDD#laI`8COXtfT6vJH}ST;sJ=*2V78o~
zQMWG~z@I^g`FyF6D)(R&(lLaGT<FR(o%>;QNz1N;s;Tldt@YR!E=JTH0O?d&$pu^x
z*`#z`oRfA)sQ_(8L)x^Nr_~aX>b|O<$!dOClC*24)WC&VL%d+10;mA-jfI_`JYbI`
z1q;POxJM?|_@KQ5R@tls?UT&d0al*bX=6MW_JLrv0N4jspU#W*U|wvKIOOCFQL&+4
zyi4<gJ33bhAJ14Q>D86q_;{Z20mJ=7`GFyZeEc(gfxvVLB}a}zk{8M5QhIPwqgKgT
zrJdfY)lb<FH(~SQUbqm;Neq|_33Xn^3vW6TTR&Pa@d~jSJF23Z-8ee2`uOvtC7VHM
z#v-(=?5ofU{*4QJ5AkXO2GC&nYvu*%IjO*%8#+-NsWy1W_Xo%I)i;l6EGYbB_MQh5
zneSVQ#D?2mG(5;ls6B*)DP7puU{a_Kad4FdLifsAfO{t<T1{aAWqrM`SQiEBc-X;#
z_{l8#g;`@IlfvP_BR<CeIgPFFc2b(2l`06H`pl*2()=cG=yYt3wcm3tjjz&MO;*<^
zgDxs1#vKGA5h=oZ<HD26iLL1=-zVRU_T`@wv&3HuI4Z!yJRIOxGhp<qdh)3(>Gy%S
z=iGnH<&XvxeC1&m9wKzr`ccP#7VXyL^MoRg?6=cIczLZ9ss)u|C6T6r_Wi7DH_mkj
zitYyHQP)6<oW6i4mEZ&mn3G)B_4OXlWA_odNt~aXI$<Ec-3I$a((~1Sqz$%{CJgT(
zl;XQ_?^BEUy*j)*1!ZETgLH%%H2OcJB+egAfMtYo%x?B%=U@w|heK;c!N7Rw$ARG}
z+d>y72AYT9W2Y%7;bSXpqqHf`|FZ^zx=tDEQpk71E%BjWQ0V&veI!{T^Mj-TOJcRG
zVjAxYNDr)*39L5lkmI}(M_>l~rF9_H_{9%;f6?s{n#T=8r<5AP$DR)mfl+c7w1l7u
zoL%p|dA187aV?=WecXrd^(tbDEPE=oYzs0X7-&u9-=5KZ{^s#QJiIB3QTIO_Eh{$g
zii8}^7gREMy^}#|WXpS0l*r`XSa18oE*G7znZheO&(eI(WsHSh2u`1VO-11Odg^k~
z2|Z0{mvM#nRT9`7eq8$kvG}G^_EBs&kN26az=d6dKv{9Xz%*R4XXPlYmpRCTKO~uv
z)}pnTtIGH4sO*#nZ`Kp{)_F8K{=Z2ahjxYPHlyq7gQYRS8+dJAZ+e}?Y0R!{Xrb|)
zWz0a0GY<vowwarD&u!r01GuRQzm#BM6YG9R{<RS-Op}-=YZi0`TOEPT8<ZC{vkGOl
zAGL9u+j4@hunaZY@q?!{8Gasb#<IaqJE?yV8+GdO$NRn;hj!yatr$*(!kAxGi~`Sv
zC`2qNSS=Z%>bP!tlg)f6^9xdspP=9D<4fX|NDW7)bIAgI7eJkhZ*{R$=Tq|faI_-5
zzWv1mV?00~2@~sbR|9Tmof((!e2Pd!I*R2H#@x(vuJvxWfptktro6sVxXsSBn5mE4
zzE!mj@k{4sc^2Vxu8Y9w=>q#chG7tWZuhV(yprt&X}7-1)FM_5?;)VtbXM=R;-!Nr
ztg0S^dKyi0G$*i@JhgcrubnRy3%wGhFSTC-FQC*JnBVXDfP+6=2|M3<zd$f};uN)9
zF5jhz;kF9@{#^_k7dJFVr&9WN_}!^@{L8_hP%9~eC;rC$_{ql${2mcTP?=|7W%fCj
z_A0&CcX8tSIP1PNo%J_BHf2f7(#j1!x3AxG-?!TX#UR7u!@UCg1yxY$eedgwRX!E2
z)ie*)5pM$_jbC8kX2y-TXJut9IKds%S(}n(yK=6HwZxLULb|kjS*`mstY08#1dnBs
ztORVs>Zg837=ld7ZA7Ao_YHmr-@&Eu*;$jh?#zhLr|G)YE=DR}ROrVr&YuKa*<+B&
zc_+4b0U0BX(Fvy?`j#nm#L=d@R;1}VgBp}%Y;NIN*;l#>bDMV)f}Y1!GVucaSyXM1
zFM!fu@UGnj4ON!A22a;0OfA#Sp=@oqa$yVL`q_QOM#E)Z=bMR&9<K5@F2}O}b_cbY
z=@(zC9?)c{b#RUaAnU^V=PJfx=%3stbnl!JD0Sc8G)1v5Vj*8XioR*$1sS>9rK28|
zU{!#t;4H*_>4Fs@U;+|*FeS|{cfQUbEL8P)XfuJsG^8npTYh`7|Jgvh_O4Ea-F;JQ
zgD4eI_}oj00sNxS3Yte!$}<yt%iH<S8pN8H$k7{uPHb92=pIG?`EHiWiPadvny^yB
z1HIs9z5veySI?oHIA7hWKe9zxuhJnqDkFzGx%hgHB}mGIk=)gcjA;X_d5_B597$;_
z>I_Bf@U@EWd1Wea`k9?P+E4JdYa>P*^*G2b<eO-U(wisxz^o>>>0)Ay@IfcPr+>kQ
z!j(~V{rNECp>s)>6Siq@FZ5*F8YQu67X73orJ!I2r^42fW8v~lmQ6o>j>+9++f3VT
z-Mp1arUB`s9IdIfCuYsi%Y97=!lT-iAm)zZ$^@A*7KB2w<kg^^R~Ol6tE9kZ9ox{&
zUOHPoQV&bJ#!X{!=){FxqZ4A<pAA%wePL8iyfa&i8?ZiFRT4tEhHqs^)V6fje5p}V
zFCUdMn8>XXY62-NUQVo@ER9>BgvaKdu969|1ETlu*W^?z+LM!?^Rlq0!LZbJ`D9FI
z!`5}f{QUe107Js~_8zFv)A90Z6k#HOCN<+g;*Lf?N#KLSRJD&@?nLL@8xXSIX<pvP
zzUzpcE2K<g_I7g*5phT3kG8EWG^IY#-n=c*>l)mozaZs%L&HH;uHCtt2Wkp}Vg-ee
z*h^>j4-Q9HR8*81@#GI{+MMeY_klfm@<g%JQimQxAy>CLIngK}jdD*rg!@DuvFlV3
zdiAYP=;}JYQ$>_OoSqu8uikI#0MKOW<?eUoKa)^$Mfg0>yp;s%7|wj${1*=Ym^@JB
z!#T4y(lb@|f|0hxRP#G5T3hbQ#c(y3_iGgz$<jlt2Q5mS7mad3<Rhi6Sy|UW;!7WB
zR;r|(Uwl}xK+X}(Zv1W|s=vrIl$Mq@GVy(CU_s|YPKyC$9Ac))-e(_e&vghZnHUZa
zR`qE(e@lcq5yNnaY&4W!p5!iGmKI2&#>JF_n$orzU*z{!Q}uC991L!`_5?ZUHiX@C
zKZ~iB3n%O_eu4Z{Tx(bOU9H$6#o<avM{jA6&k0FSXGquzeSPi~2!2WdD<t=z>Kqr2
z?ZDn#xnZ;Z1+>$hyukSp&aBpw0c}f@zsacb`9tfEkfJpfsiQqTA@k9j<X4zoBgDA&
ze%!1G$rUmgH<jfyOp6<+aj#5HO^yEAscHV86L*fu8;E?SV%4sO@B5eYrD07fg0WdX
z_MQ8@I#2X>H-(pn0^@+WpyYRGvda<kBW!8zLvePPl+(w!<KzU_U(s4@=ML?S{P26e
zSq`GN(ZObyDnT7DHv4(y8TN8e4(K#`{yfX%E)IXaLL8s{_bD6aDq)U`hE^49Z$ZfT
zh1jC@Dv#}SU}(-%<w{Mc&n5im8dEPdSFVyANV;b6=h4l*&+ljxLSMcz6jQvG<HAg<
zixm()E2lbkJISOCe+TGdQpV@SB9%*5DDORE(JV?g$;RPFC{(*_C6?+V&li162V8n>
z&ZX+}h$;^ql!PXSWV#`BfL;<9Fl}6Fm1+h_bXclm>WV4)7?ezPz(Sk*LOqI~wqPzf
zn6fg*q`fu&27DJ##@>ql3Za_<X>&(!O_1itle&p{RvCbg34+r*QkY72uEclHL2-pH
zQ!ST^TW0IW9LVH0#RCRfDA1H>iqvSBWIze#dbrEeXxi*b&fquJ&F(?Zg;YTBP%%B>
zGl$4_{#eYiNRGb0*P9Ir21yCFuHBF3I#PU0`vNh}PH7!J3ZQ=xPsBZ4I3gW8mfukh
z@O&1UB6R@?jM_IKAauGphCdU+XFpG5z}q_<QkLaa#(GS+iWRo7?4L>h*rHVZ8=a?z
zQLo#|NVO4$*9~}?^|*I`HlV?ER61JC%uovn2^D^GKGz?Jmo|WVKRJo{&q1?xCNdSG
z2{>Eqw=|W~%7!h0^?DCr1rN6XNHQ}4`_VffgP@GCnY9Y3nrcVRnv2-8C4G5AsJr&w
zn@^`L!5CCS!05-ZS(?Sta`6JDKeRzw+ztx6tJ!oBY3hPSPl^Y)pPw!py5QhoII89W
zgV;_tz5nom2$)6=Q_sZ@4B$|5=T6Nhe<=u6QRaD}A^1XXgyPPvMhD8A2i;lS&>nxR
z%=CMM#n^YWN}BdGGz*R1#tORj#HM>3?E{!0ET64a8V-t%dRl0dug-!x0ydNjo>pX6
zINkC(TiCB(zt#a4g^v*l$^mp>`GizdEL-#4wl+ySv+duqa8|YDVG~Um!Y7YHxJHPt
z$2A+X-ikR)co}wNA=LVHyabt(rBP=6aig%ra+t5&X)VdQ(*0n&#f&1m`Y~`-2lr|&
zm`;5lhE<Vshy1e;VP^K(MDv=gX_9j*puMm}u<CK`9h-N5o*Md4wbi2~z5vV9xf@*c
z`BQSdhT9f}$fmTq+a<r})eOE5Cp|^m!?-~;)7FE8`*|O{;(>&cmHqNiKlTq(i+6yO
zvo{Kvy?W-HkZP#>WLV6PQ?&0@N8DHKS{hnnFyED7bA0H`9{E`%lMjkSkrV-%Au03O
zeA+rp>ha+_akiLZu2a4WD;F`zt;fCDBx>WYb)SDowKIH|I$MBCWHnNy0Sa8SG&B#2
z9=?god1US=CN2&=R{?^jDB&-9vumFUA6^Or2Ep2r38;;RT!N6rm28EN-nL4Q(I9}&
z*|@~m&+3C{^m-9ewX}wpi`mfW>++1QYQ1jkx(p=ff!cCYEN|zY#z=*I<}6ME@b2X(
z!b@ubCN??Oa}#e&sDN_CIdTui1*g3n@$}4Pf$-xUmEe5xjdn;Kz*J3nTd!*cGs{4+
zNwLOlPho6z;Cg7!)`zZZ>{UJS+~hdS-J&!x@;eWeir!0j;gYiaEHdqkP1w?@bnxq;
z1Ef#l?6vlkN2D7Yy1?OvIxqyK8DEIfgh57PY=+c#`7FGPC*-l8|6r@MDtxjRM9F9O
zlS8G*<nc^uA|`}`jv(d|Mu?o$pU)+KYRZOv-a4&x<&;$rcDqDL8ep$*N<ZqoV-0wr
zQEZ0(8&+RHQMIXjbJPn!Y2Gh&pn$AvOF?<w4KOB*a@RAOZP!jEU-t>d$KKb%oC8HB
zc<lcwBE5Dklm((t>0lAG?GvhZcdVvLB1ClrzWiNJ)lj=yLci*dKhv%E79Ip@AIZJi
zpSU=O)vZl5$Rg{GOaH8ne=|xTq?7(>Q93-0j>V*w<JO&Db?#5bPEf<2khhDT4nG!+
z4z`>zQ#G57-IEiN=YGJ--Zn8Ykphs`ZLCC22!|t2%nHVkv6}RsfgGA&wM6FI?m_E)
zKE5=B-A{AM{EL50cr7ls!pI5Qh@Lti$G-brORY5g9Zr0-BiiXVrv_*~htAhG&rRQ$
z8rXd|qBFgRU*&P#&dSm#{89d*^m$ZRN&Ns0Da)O^nnl*CP!wfTn?)@+kgx3ZxKOEd
zNY6HbW5+Kfc_ec|GRV<T=OksLeIB*rT7dzEDks!?rS@dxP|J3Ch}Sj*l+~jkUOGMq
zHKM$;y~f&20BOCG%G;Y~^^c!nb@c(HhM;gM2hkt@`YM&j<l}=^$$(cWXNDhFO+i8u
zo@rYU3Zy_<D%lZj`A2S-P9<Q)^@BUUb~`a`!4&0-dd{C75+j$Ty+BOG1N8<(WB+RJ
zL_mCQ!GT>SG|>+^Uw_>W*aJ7u%d=<kK`^t|%a`aH!17B$lM8KUnq%l085s$QiRs=_
za67;tK(E7o{`{#O7aDh{ROddGNNcqpmmI%m81}|<TaoroO|}@vv8ML@u*2ej&ELOa
zfQ9FXLj+JJpcLg>ZY}gWnh}5#@=#r5YzkNO493;vf{E-hoW(^ROu-*M(1P&R!7`^g
zid_Qmpy>nzw8=S4#*!fw;6RJxq7`C9A^D|39EIo^4}R=U)oqtkg-A#&hX7`U8em5(
zN4O^|)eu`(4i`ABJ)IfND>m=-C*khR1~|T@_K9X^vPpYV&C;FhGc!z$3XH^9IKG`i
z`4dt}S=lh)Gf-ZjC6@7D8@$k!F)<{t01BO;KDKv)oPFYjl^h>S2eMHC2kXl)UOf-_
zKTOI>H1oBrxLjh5?mNTu4M8^}fDiiRd-YZ_VBqcv3N~EIN9LGxbjxl>qzY8pO!-Th
z_5gHk92VLm;P>|J<)_%M&y3`*97w^gESi%21q%<f5yq{{1I53wAu<U<IxPvpR>~YF
zCDX^fLxTW3{ds8a>#m9-CX2qtdk>hH63b;1SKfCqha7^8RsX^3EuJsj<WJfOb(b{!
z(vPdcm}eu188v^p%(i`8|8nW3B)}$9OQoaXt}E5O+()0fk=NQ(6afF<{`m4x*^5uJ
z=2x#?m4hgs+~|sCXm4+aatrYVo_f>WPe7$TtC%*i|4!ChRUaUcmc>3$+-?Nz0?q2o
zDxW9OA(oX(T#On825sc`^0F{OLPHG~`|{TtnR6re37-83eBVIj+Qf^%y_~k_s-<}4
zScP^fX`@2o=F0{trxf8wPTkZ=7%$le?NS{_YupV18tW*xHIBOX#QPUnLVr!wYlJGS
zLQ@|^YjXY{?`oCs*ee1~&onxAk&!><<qM3lbr!JcD@K%O6^u#EF<`n{nzI>|QX@B+
zT&#gLjg8Sl1@IuYW$B$UXc!|>2WU*Hx)8J=6fzkAj)NlL1;|`lkmIAyxFr1H!>t66
ztSOP%`Lz;|Q|3abdkn?S+)M{0ohc_iowaqq`ESH!t<qPmn8kSmIw0O}s`57Gi>ksj
zt@Hscxuy0ah?oeDVomv|v*VJO)9o<4NbX$0#r-|cQFXy}fetVPIUdy+YTF~`dGY6E
z=<(C_$zMr;{s5?yaWQJ-kaWcqX^)scyYQ=Rl!qve;^EPXCwBs_4s~L}`ytC^TXWi}
znOwT9F1Hs6dh~zfnwLJ#?byEZ%apXmW^Nb%`ukW;TU;yEQ#hh^(|9b(IR5@bhIV-Y
zs?nr2S+H?xZ4o7H<CfU>pytb)nT1MMINViC_BF8}=5N+SiDoG*tL59m8L(wQ<rk+i
zTl@aPKKLjMuy7ucLfSqcDooq;4yHe(LOLjvpM&78jiv*|rJe-1Jp!~;G=|CxF|t-~
zDjxf1+)fDVVt<saf~J+7yY#1~NXd@VD_NUB>NrK{l^D@gY}V&SN0m>_1DJDiwhWu|
z-dWKTKDHM3inyy~q$=>dFYn7x)(vjUSQ868E_57egpYi*iv>t`-Ilao9TTyg2hJ-}
zg=^m`&%>~|BQ<gVi}RVIK%gqml4=1pe}#{1#n{B`Y6RKCW_lfHBI6x9X5FeX6fr9c
z)&@ZUPKVow8sAtg%)BS-KJSiNwk93-uw7`?$?*Xy5I<R3<=nnQM_1n^cAe{aB<=0n
z(RL3iT#&H*UaZC^{Wz4oHewu{gMgK1<&6$_;_L_J{v{|{phk+y#|P8J_X-4gt<Lm7
z1ZBr0S!lnV;>lJlfPz&wQ?mup>_HbyDz)t6cF-}yexPieSYqD$j)*wE99_g-Rn|Gl
zd5c~ymI3(8FhDZ?4GQ-Z%Fmjf-*frousZf-9ni#Ly4f^uirCf_QjbY_0YICAOEBA-
zV5#|C&;psLoDpko;8c(hOK}?3tfm5UU}e@x@)vFHRwGj_HL6{TLqEB!{i#p&HAJ~L
zkZqD9PXs~M<L@0Sxv8G^s{yPv1b2&E*C(as0PQ94`uzv~BzNca;x?_VKmcU?17I76
z@`apAgQm#))6-dRVx~5+u6xDtJMuxjK}g3VJ_5oumWqPkavLOO!Vzu)f42@^gvu`W
zY#9~(RmNbEF|cg~0Hf2_e+OK7vCdckP^CIkW#fphfrQ(8;O+1vHi*1pF+2{}Btjo~
z0?Jee=zl=jn!V>!ZU)xNA2WPLTo&#+1Ma`9f@~)?=(&)jp`r1%tqW|l!dTq2s({I;
zdB+L44{R2r@|qZ4m{>WL%Y4B6C6B)IdUpm0oKdL^wfozDD*#?d5=m+45a4<Fppdxy
zEN+G^BLk2#ii03PBgijXsy4NOow!;M-&O6;XRZUzzoeU6RUkR1^zmkz@O#bq-bbJ2
zv@x$xnixG_XpP5|=5tt_J;0T~f1p^m<nSAGcf@r+A&0jmJQ>cZJ$5nPLiYGGs%DM(
zTfL#>@_G0-tCWDMKrjlQrg=fh+OFgTl_VjD6dJdN0}me)^@|Tz*%&$Bd$)9m=EaK_
z@yl0c@UMDV)gJXUxd_|M24lmj07FPINz9Lc%zflFFGxCEyWUsnm@5D%k?YejR{kVj
zhN5*5u>F%OI^EBDgET-ga^vLP_R=6f)Ef-8esV=mmto-VoV)~35i2{{X(AOVj5~Wr
zzFGeGXlFDg7xZoM6bk$kDt({pFEsu+WwkT2Kyt@@u-H5T)gFY3=$d1=x$Av6r?9lP
zVdTjayf(><1(bj|t*RujKl`a`?q~d4$7+Ve%-<Y5`j@b{p7)Jqv2;eWMXSpB{NA^{
zdF|RneSAC7X$%$24S=Nc##5`mSawy^OVD+jhY^_yLN{O;?^kCjruf^2rUC-{q}TR0
zbJ9CH9hyWHIEYTzTr)zJIQf;&w{PEk41IOU!1?A690VkCabsP1(4od5j!OYMzMUw%
z0m9)I$D@THi<oxYB+ETKd03kUTB*7c)=C2;Jpa{jtQTM=7g%NJNp+z4A};10(CARm
zVifZPc(ImpY+KOF<S%3_TGt69f7zkEOCDJP>^aKqPZvhWZ|2tQO9S>g4XQ&3L?W&0
z1LnC>;J|J;t6q@+;V~(z?gtdZ`-FQA2Sx6a0^uqVld@#d?YB8pZ7S&C2*4F)%YWBJ
zotk(QImO27JImj5H>TzFtl=H6DLKdJE{cNnMmqeCsb#_J+weilN?8ysEDpZwaFN7b
z9Ie&E#HYHW$(jfNR4Rida1^4Ab)i0}tbWzu`foOmL#Jza*9BdQ1Xk<Nhzz%B`DR$|
zc>&e*Pt7V+!ZYQ&KkQG$_|&YY59$rlCaX~__w40f{>3ZhSHX#+pxZvpEal?Ff@2zG
z$ZOb0)L&|qV5(=8+{gXc4g&%5($xd3<e~@+j)+~5?_&P;2A><wTC<xJ)l5Z3h8=tf
zoy&5F;b?y=H04o*KZ$J|Fc7FdC$zgSq|pWFt^a(-ZtoUzKqAomE9M#_s03YjG6&7N
z(#Ol?tc^}%pgQ1K>Kab-xo#|pjGn4l>Q5{`nBc&hZ{=vAI@WLt%m@(X5R=asT<C6$
zfK3gvwRHhmbX0x^)lvttXk4v0Ci$!KLVIHIuFJgb$nk6P0@S%x>;6P6+I@wWZ5_}B
zXstmbS^LSTsS7oT3S90CgHp^b&CL-OMePhlI{YnU=U<0{NCyT0tDCvC?BA_i{fHzv
z(kZt|VOLVTxu6Q7Qzb-~E`~L*excG|-t_vVo-ikJFQ8gtP#sj_#;flrxLbyb4pyiH
z)?Xtr(Kr}Z+a+#F0Z-H{{T4;g9x;jLB3MmF=3H<4-~{YZE=b%)Z!@W8nZW*RpOA9u
z*PTJ(NeKXYJT`eq{yG>NI*^CP&?q(wW4~t8UtpNT?(yu|vlQUKfI|TgSW;bGJ*VZD
z9F~(PoQ_UhXm<>ajL$9|RbN5J&~+Ne0GXY&hZ{xjUm-ECf}9nzd2db_Xn6Z|uwqm0
zDGvGjzW{1kLG;k_)Jpdud=<TD8kqxc>qbL$7-gq?(gInbP7Ex7w$QAsH?RMl*nbwR
z5!N`M?9e}yUZ_M3x<n;6f0)_X6@kXYCaG76>{Pdav$*E<7^4cAn%@!A0`@W##fKy+
z%?6=wT@bT6W^i=0**tdDX2+wXlWBp2#X_Sk1n`{RhQ5JlPzil71vv$s6N`>+wu<g-
ztzN17$(i#Y=A5GZhIXo2aCTrBqB2&={~E1HDfs>Sk5QH_8PY-aR?|tvYMa}%<9imQ
zE#?Hod7L6%0&%f%S62&Ayi_qYsN3%No1{5DfKYEa4%<gWxQf`86QlB}vX5s#?`}s^
ze2<-+rtQniUi!ps#10i^bu0>5%tom2<n>^xEQN7*oRATQNS+7Ib<%(5&<nu#*uWAG
z;Xgw23MOvrKmI4^dx)ZnCM&hu32W~5w%89<yE=xFFc^Hi?u5Xel`W|~Ia;}WH@K(V
z>2R&V5VU&{<y;QV1c}ZED70|6I}y|YQt!?)agNEL;s)(-t?;WW#+^M0MO9G_g`L4!
zwwddo3-w-kO<`%RI#p~r9PQbf6Bl*sUlqq92ci<OWhBG;cb@Ur+PWw~ccpCyaZ|jY
zRuK#G_Y|9R<r1V0IJ~DgXs$W*v1HIztXUjXl;_t|+NMcz?Y2xX2)dn9@3+j~hw4Nn
zdm%x;S$7wsGa$`Q{i09!09e66#lQTnu5v?<c-8aty*5r3L4=IrVZVIy<`M7%aS*Sb
z)sIyEwK{_P|CXweGpJnMNF>Q$YcC1CYhuz1iR86SoeeY{25GwKAsG}2_XERY8hC?5
z+EoQF5V`#VU0wztzEUhOka7ks3sWG>L#MC9KoL{{I1K?_L<&TXGVS9at&YB%J}U(4
zy`6~B4&wirLt%H|J3?Y(?>v6|SWa5_XwL*NXq}e^G{+hK>J}Ceke8QD$}IiA@}eT?
z1B~)eOMTEcX#?zDXVSDnFEM04#km~Rs29fLtzFQoZmj}1`5W+D#ckV!fzM?#>ta~9
zaD6jko^5i^whjp`07Z#%z(Y6KoNZUk(dNx9mKVj^{e()8h$|iSb7BF)IZXwj0<MjD
z_D$FBgq)BL6^zsWt3jX*{Ehd53mf^rbF{f@y*c}M#;6WtA3*!>t?51hfB3o4I3Nlc
ziYeL&p*OB#p?fV2md0pr@I6ztH8XcfRL$1Jc+fjH!HG4yXmWXbb{;?Os-VrJWGIzC
zMO-iP7qtkZHE3gw;xrH6;Qs>Lb`uZC0N&BWl~ltw2}b#=+ba4BH240B>47A$U06EL
zU;Cq+55WKoF!b#@FlKR(07sJrKdtvq!G(0CKfYD;@w$5(j>-GxUOi786teUS2w(@D
zRj!Nom7Lg3E;ZLtLUrg8GcoMC5g@BB1DC|jDWFCoT%bqpPf!8m3LYv0#Sf~30X;Yc
zXy~B3SK1TadWwPJ00X0cH%tos*B4LfjDgPN0=GRI_6d132*2Yp2YtK192mSB8)kAD
z61Udy8c;$2Utb(xNrq_h1+Pm~_cET@apGysVjmuio^WL(U7-?Sz!yHu(-2HNVL?r1
z8di~{;2D&+fOA90t#2{?<n=bln@1@tfFy2{eBCJ*Y)lR8)8V{L8`;16l%q2S>O!|6
z`X^s%jGD=9*>s?7z;?{wZv(WyJU}rni4K!N7?%hg?@lB^y+CC;Mq?ao8-Cv7wqb9Q
zHp+QTnSBtqgeHjxGy<jtHRyCDm{Y(ZkKVvYL6R)>pt5jdyK56hvBXTu1aT(%z3@a`
z2?bpb%zr8Z1n4Ne8KYtQ=U=OMwGr_EoROtn-Vv3bmI)oQER}kdwAZ#oceFH6e2Y%n
z#6#?XOBZk=M#RYJfL`r2l#>-$`+kiO<VZSL@Pw+7BuY!Yxe0f)%i6MHqj_E=Nzs4s
zV2uY7a)3p@`+bGr@c&`&y&tLm-~aJ*j)P+#TPlvo$P8H-#|jxqWhE=JLZNKVks=DA
z$Vw%ptfY(xsjTePFiZB{nV;*S=j;1VeBZw{&N+|!<Nmnr*LAyY*X_C<^R-Ob#_wls
zklv@T>Yd>5`wHJh7J2?1r2%M1A5w9%{HNI<*kDq2t(>-h$}Bz*vp9rMbF8fVfMtC#
zC@9+f%69sYrKr$GIogu$7DeS|_Z11Ri=ag~2@DBEu=(=f4a}q9)rZPUUuBSkdB$l7
z1lq?$sg=J8JU>BjrtHZ(Kem<s1RZwfxzWSdJvnFUXz<S4j2-VNXLLy9y+mcb3{pC^
zDx%MxK!xB#>2u5bhuTX{F>?ZlL|FFi6ERGcyY^sYs$&b<x!Z2qkKneBWcuet0GzZ5
zUb)|uf8qR>SYAme=G0M>t!R+k6p6%0>r56Hne?TvTz`R~c=YPxcOa3ZIos|>m#4I6
zd@Z#9OsfbIjPDj^GLv&s8_(X}g%+fYzj}V`{{>yzvv9)5US87wvn2x5*#VV}LxfYy
zD6lGn&U@L*-`Z$UJ~Kbg*<ZX^P{|jgMX1LvJL!1tQ~{cN68i`MlmF^>Cz?dc@7Gp)
zt{`ppa6XU8A6+@|C!B^g3o^I?*Np3UoGZDj>#Rk&PtMZ&7I3^;X2fzU30AhkPCA#F
zr-!toux}}E{1ZS_BSs!{_|*Ho1IH(s8Or9i<N6*qpQh-oMR#G6_cQdHT{A&V&^w6-
zGyvE!gOLY^o6!X|#c;KT(yPC-0#+u?=ITXpBkIK6;2kg-M_}uotXO6QOp5p~eCDM2
zGFlUNfw}%JDL=@n*!kYl`#7o*78aJT4mfdCfx03C%eZS8lz#q<CQkngB`Ay{D4y!k
zktP3vp0F;u*~LRL^%1x;=P25K1eyX(SN4ZP<n$9Qb-AW8JWdJSypsP4r3xK*2nCeG
z3MKAGnfrw(7V}6UEg2yjE`MQ^M)TNZh&2ct9lRdZ!;G^>Gf;hFJ-obmi($V(`6no-
zw{z<O5Ox!qaQmOXj`M{O;xGRj_dj=C2<=TmC@Gj^a;DIPsb_!kD(_pBfMuz_FTI0J
zu4@s(DSbv)b$EjXTGe(d9+qvxP?40g>RIiy!%R0hZ?&9clO%LvF%NO$jSc=wql#7f
z?of(a-lZtFtTvL&sycPdr9%snbZJEj$^Y_i;t5#7KJEW>T}0KrHmt9gQ;%M}bwDMc
zD-9{$PGC^?kDP1N5j_w^bf%5_sUlnOzUq$U)R}QwvR&=1hx?jRjKKRFclF!k<F|2?
zXmMTTz}3rRzIouIqB$fatV#==?;bv`+m@+L0lF<ZN|rDGN;1R@i0mT|?@;{n((46C
zDJ5O)BosnIcSDvgLGlHE3tXMnGg{h>U%){Q?CdJuJeUtX8i3rN>cl?SoP96ZIyQ{L
z(rO?arnKL@El}m2$heT~%(p!Oc3_QjbZN@|CbE=I(=Rzgr$#pG!otE6wWL+ouaF!0
zRe2ZxTMcP1>+ht~EZJMm)|sKPN1+b@RzFlzh801BYiB#qiVY0%eYQLEU2H@^pi8aj
zHc{}-LqOh0QC>Kf^qzx>N6||_`yItJH<g<&WNxC{n`@kT4Bl(~zD>zTFJ?i^t5LH0
za+65JjnBEHJZaDG^?}JGou1n&hHkRo(P04JC_nj)3;c(}5Ro0txt6_N3%o1_ggEig
zyIJ$lA1SiRyn~->Kw24dG0spyX}j}~Oc-Lqz=VRMXcElBNO|-Yrl*{Qkh%5PZ5fT$
zK>+YT(2^m-+#vOZTTHwX&^)S8_Vpki4`Jv$ngN%^fOoDYmcYn}XrrMv6gh<X>(b1%
zC;vnca|kt(FQ50HUnGrYUwCbQQ|8TOP@EWfSXNGWj%9auqeik<mZJ66FeXGg-4FjU
zHOco;Pfy&9hj}{YP*ftuL>NWfB5}m=DqP7lNLM<?E$?x;<NRCCK~-6*R$*owfeh|4
z6-CbG>aZWap7$L5O$s}9b<Q7hO~%aA?dSd-n?*sO{yeRx_v8$C2MjVF1X2h^G|G@V
zF+Tnfw8bE^a-3Ys2EM__9b8en%mM6{zAdzNOFOp_6#eEim6zKz7r*TH)CG;i_N#<t
zD)MEl;Hiwus<n_FB~bGQwdL{*eX*Cbt++{wrG)e$Zq}EWr<a-9L;d;;J(4DM6)ape
zbNQ#gBx4|c8QUhN?Nzu4cr3Ui2Y}y#lyNPL=fO`296qcEjpwqS1AX9?XZtLkJb8k#
z(_LmJlQ`S<r6S#>Z{&`N$7NG@BwAg@0iA)!20ukIFM@tU-^waYPiT4kX>8F;F}$Q;
zFrx^%<fk}Lf7d&-U{Z-y@rz22jpI)$12Tc5Fsblch%2%y+;hS;peZ4g<_kUD%lzvJ
zae!~9dJBAaO3wolNAkmwfSzK;RbF3&s0C_6xVBwc8-+eaJKp*vAF~cqg6uI@QjX|S
zP^3hsGmwd#-$215qNMCOpxmD54d`D=v=tL<waiO4W`;77p_$IfPF+{a<A!#<2%hoP
zWw7NGF%dXx&b0@ciFibQ8WvrT-#c(SqM4x;;IY12?BvPYPpb_~_7==-EkJmd-|^d&
z+#iddFDG&Mcct!@D-Eqx?N;8|CvyUb%(0b_HUcJtV;yuzB2fNe7QeLd?P6?$TS28D
zob(Kj2TS}UBARYoAOT?X@mB`dj*(Rww9FWsuiB?OKS5W2`}Pgvk>2^wB^qCWxJBC%
zFZ0j9iNqw|DY37Am&Q`n+MrYPx7b|IR>3N-6WFGce+g4Qli{Jb(?iu{iapl@%Mu5-
zAoC`BWh@brLJkB<At@|E*_`3L2(TWR@qt~615vrUWTuNSr-aq`7GpU=#xgiU{?j~{
zxv^UU^xsJzAD^~BDI#L!<Z9q-@Cv<A8G8f;U<;Y3W4l!$U{(6lm@8oJ*Dxp;Eg?fW
zwEl@Oq=T`1^&0_%;GS<&Em}isuK#4tbAxS0wMoGX>QM8xtS+{x3VA{~4Q<!97<<rs
zEjp*Bfuy;D{|aP@LJFCz=j7af2>ge(EK?he&g8uHg5XinDPsALIU=w^?0A?t&w@BA
zQYg*%9sA0^1))|hsG;%pr`b2aXu-6~K|PE3yq)f=0T6^w5S@%C4cnkRr3)o8r2Yji
zmE8xA(?c^v219DQ*yr8a05sdkZJ~@fGi!fss74=%=b`6@3>y)n!knhwzXz`aUm!@x
zSDWHKP)LF0W}}C&VoLR856%unAn<zrNgE<(B6Ly8wicyhbeHb95s4+BrPBp9&(|X;
z2GrRCYn84XE;USuc&1ttGWRP8THHs+u0g@C<CU3Y)vQdqX7zc*<4UWOV^ufmvBiTh
zi>GX<<#=Vc@(kQ`66LZ}2-JRBj`RW#pTff?YHCpY?tc0xkn82yS6(N**ZEiP<pX^<
zAzVUE6OI#h>m1PuMzNN6ZIQFUOVMPVhw7ttbn$nY7vKY@3p6uraD=!M#OT1#`5NlQ
zk93mBVbR=dZFMko>Es(uCT1d$5*RW*fN1Q^YWWkh#VtgrI902o$OO)|%rYw6w4M}r
zE*X9aP~thyh%(^E&A?P{ScMJmS+(DJD6W=#cG*L-)gxQITMgH_3C%gV=YdP_yHd+w
zgd#>4z6QK*OM~PAeSKEIO_x1>EHD4IS{}L~9K<vIzX%y5+tM=GAVZ%Hv?Xe!1g1fi
zT|;vS(p$k}+1r~7_xN?bRv**OhV{@$LrtEM0e@mIj13(H%i($Y=hL2BdRGmEH8QHK
zh5kkNE3@N@XA2N!o~r9w?*$#kbNlbk;jqc1Yz;0TvG({FdC5IKW-!>7bT@4F5X7wX
zqsk8T_`i!ylFxvDc;fOdV0C)pr%}5S2Dh$d3o+k_WDCCCo*Gw76`L;z=^=#m6}??&
zHUvczt^f?d&c1y*h8Xf7Ez+l+MQqz=w6Rjlq!yWtZ};JYiuiS}ELWvWh#*x5sSj#G
zD=uX*aT2C9xw=w1le?hv7>T7u<gT=dZxm!PN)AX4AoR=MEZtn5;J36PY4llmEl^?j
z_zIzh>g07d`-d{rF87sWb9F+H<8*}~Xm|qdb}YpWq!_=5IBT1r1AA5GOiaoE5mF%i
zw3XPztSo`sQBm<ARaTHmhVq89wI*zQn5OOTFLyf%Y}C$BXnATlH?q10W9!~JvVGue
z+l`$(naa$6BDrOcJqC<~$k>C&5DmooS>la_l$6$;{_{eEFl4F(hG{5O1(vw=8l6lO
z9`5|mp|E$pcad(8DmRT-#8+2=#Ffo*t8Ith0R$WIadEbnHW41}FTa-4k<(gTbCN;_
zwFs@4vDQaP;DZ_<*M8==@C&JWG~QrR6_!X6Z5{iC0&%-F;yN$=X@+4LLO=w`lS^QF
zf>S7d?huKnOEZ4bO58ZVDs?I;d$Iy*K7V(;Kq`~gU1s^}6-XKQ9BXa&fP6kQie7n3
zQ?MtdaKN|ON4O-3v_4X^0cyu#aT^3-Z8_*GordzrAUx><&1aycP>}5T;qRBPCaTcG
zs|+)45H5z54VQgXnghFa8YxOk+g796D}F%<vI>!(A$&~e^pRjlOL^%8goK)k9GlYE
zG@<)kRx|)A8K3h9pB=*$hnRt;G&WFwY0tO^kQP?=t3QyW2m`@{4;hW}&{5WR)(Xk%
zCj0@krCXc;?(xqFTa?wQ@<^gO4!mORh-_~tlh3=y00L47kTHNp)3E)ydMV82(u4^B
z4nXL1lz7^-rv@SE1XB8s_XWIcL?U)F9EM^ln?s6eJ&=*RHnA}=E&r>c;{}jZe?&u?
z9<lo>3=khaCMZO1f=u!-f-2>;vDk2!dW48ImwvL<LF7cyAo6kR(Cupp9J)@-A69?3
zEkb;K$gY8wr7w>n8q-XJE+dyC8ImZ#G**H}Y0j56sJaRJ=rK4y1ESf6>w?$u?J>xN
zw9K$>nWx=CzN0bQYd6ae^Sq}|L)hc_hmc=CYVVW+Q-dlAI5x#lEJ8cN&=IH3$DqGc
z3dQ!p;1QF@8kB?A6MLA6)lL}5*YA@ls2gEu&mW*?;-Gx156Xq09$xMfDDpC9ccZt;
zfVMz0f>BYVLYR;;5D9H?ac@XHRLCTeH%jYpX4&%e@tmAvS+HDnI6oHm5@mz}5i<&r
z1Oe`xB=bS!Fw&kF0{*N;+AjUHA5tf`XFwHu4i0P;6`)A~*Vzqjq=BjxV@jmB1z5Ti
zIxj3>bels51@v<B+16>+F+(qz_<@(&9|i;GRBp${%1?yvtk<cTj2usvRJy2HU5kOF
zlEaYs#kE==_8KSw8b$+q!J4=4`tQ5)DKuJ^<9#4bE|ve+;**&P4vVc=cnq<sEiEDi
zO8<e_(W*}{*)SSEnf)fPbCkCGNi|{^BwA<HVGwfs968JcEi*|rA>Y5M#zaP<0Gw`t
zre=8+CO&{G5kO`j>hX6KyPQ#*JJt{Gjkq_m!ujOs<HUKG&h*d&h-8!E?BsO&7m!?$
zr4DQ$X;XLwwC=?1;!v<<HHVOb&mKDvy#a3IYa4JC9vGBTf=-nzr!dk*7k8R?hH`ms
zaPTXp0>bE@e&50Sz}Y+Y-38w}H1?0E*<SASf+~2zq8rfMCQxko--=k;WF1qyy7JXC
zTd%>fOY@oPX{N!S1rjOlmbMLe0&*OJlcbYU$lyK-_h=SCQlnTD!f1GuoJWdcm#BW|
z`=5X7s=d%^`+ofnH5tR!)~$uehL9njCa+Az_o;<cIB!Tn?;OsGEaTKt55g4!G9BU>
zWWxwmQ#O_ni18&Z176y)B%0Afl?{+(6H?O)g>{JmHuN>2zb|PsP*A5nhTu12WL-zh
zQm(?fs_C3#h5#HLm$_=1Z{-pd45XYcQX3!RbT?$h^YHM@|8_*dc36->2<*LN(}{6;
znF*I{3DVZl34CYQ0)iN8R&!ALF3<cl1IE^IF{J80S;Zb<Ou(nWP`9tvDw!e%vAk*?
z1lOM22`sKy?q0s|zy6f@4{+e-Qah0$#h`zKMMn-SmxMm-=|+JH7}35dO%1km=TMSY
zjv+mSAW+?yyFJQ1SZH@+dHVYq;2?#Lxpvjh3{S&YB(wlht1+deKd5vT8RB?KWhHU^
zH^6EXH{}IwFeRvseBH}5vfJ67mq+Q9Al)?yDa)SRd3yf}T|@o=2{Pofk=7@I83tIj
zCo+Gh?Jtz~$?@^27bia--7axM`Al}{`J6-TS&|t#@7NPjG?1bf+a0&&DwbllM&+v-
zrd+=9!cjCPI*q%!qw|ge_sWh`2>U`dXV3~P<q%KFVwviV2+{~$8W*6vKr`c2wkO9R
z?PW~4=C6r7M}iSO!j>uEq(N(Lq*1U&B6C8?=X(A@6jHXZD)(s^**tap{0Ry)WZ8_@
z8Na|j@UXRg_vZ~>zDGlqrWR_A)NoUd2unz`+{p8C1uv(AHXM8`8jb*K7^?Hif)6p>
ze<20NCAYD+rEqlbq&maN_xii&n1Fc;6j{qlXT#{mIdQ&uvMr2r9%6@mvbM`+hJ7L$
z)C>X-dc4W;7<*fp_t@9V$v~MYzOnNtpAZBm6G0EIRTxi(UwL@o*9P@<U?UtqhSQi-
zg=}LK<%Iw*R<pJ3zcN?3mO;N?&cG)uEF0g^O_4>Q|2m~tz>E__=ecE(^^d#1%ol9y
zg&7mjM|1s07pz~`z`#I?A^kPtDXm!{#2s_x)&MhSLo&jprfHa-I|6e{Z>iI{ey{op
zXx$J_IeV1y9nkYB3nLW-|A}Ww2B=J*iH^md^PUZ$VwkJpP&|ggVN<G+7P6?v$lS;T
zU}z#a4ubH{EtBEK4xAo5-MA|mwp=OnjEf3}!S#fWPBe#LPj}ab?>Q*67`722MYxc9
zpRp~H$lZx&MIi4Xwe97EH+|Py(iFX>za6c8QWh}7je>KU1T04_k~a7m3^hqs$==jF
z=H!RS>ma_446h{12|@Xja@qF*%y4Z2X!6ImYBL>4JfX`3zfN%x3snstOv;AwnY<q&
zw`8{;_8e~W(Xe!DX2i?7|LDp(NUMaR3WwikBY~sD5J%YH3*{K4H!VAkXLplTfqkfX
z0nJ<wUq5+s4odDYh(?q1_C0<QsTT_bkE_>>Dc;833%=<vxVN3{@R}vHGk9){DFIFe
zLgVa%={{OoTJ^wvIRca#y;{u}6$3%LjSea5Z>e0`HH6*`xrCQ*0WTRLEozYKN#|JJ
z;0Z?~xnvp8OL24p*O8R&G8yXoP0%aQESb)YYGX&2--p4jG^laOw2`rQpOKNk1Nc&q
z^@BF)<JU<<?pF4At)4%2KB7Q=-i3lpU~L<8m&HrLd?1&OtlN6W!R@{$>_1!!mIn3F
z?t6%!(u96ypz<gETWHvM^7hAcaUPf_gDGY&cN-A~(5rLkI^r=o+H8d1zPPXQdj!I?
z?12qBM@VVXErd3xWsvhfL`Q}SUPGq(jjy!N)L_URLf{xDed6Q{lViq7lc9)al;QV_
z<~vm7wqVSU)b2ZH6_!0;7KioGNHQ8Jie>}qsoJLsyn6PfD2QutddRY38%V8c1R0t3
zVO;^xNtodflxC6g%*Rr@S}Lao7buISqevWiUS(D_azS4ptwjqII<@l1MEgSZ?}@(`
zq(+2|IW}aS#>NOECH*RPAY{{B1gOwA8^U5@xtS?L%1C%Nqg<WscQeJ0hZle%>~=P(
z+wjZ>*Oz9{6ZG0%yI?hXuS3BTA>Fd&-1)#A(O8vUQAeWMS1v?UCUTM;ItY=%)tjud
z-d7_cA~rAVPT(OjXOKq!_MAhdj2%C6q_JVCx$5-eH4EEMI6r-&x0hSHT_w2(SQ%DQ
zR|E4Ngv_LK-aZ^Cot%9OMldDAOQpDveWk@1?sdS4*n9}eWV*kD2!~@JbVP(vgB{ai
zKpy-6(VmHr9!T0qAfe;nL2)Ek!k|(_<)x*i*>t)1;Y`O>gTmYaDR2UfiiA&tX4DRm
zArs1m%(yG$(5>IRi(O|<H2|?K2I%iTq>d+pf`Uk~QM1S;ciF#z2zROE2~4#DO*O-Q
zIUiFd5T}!u@9ZA_4Q(CROgLSr3TGeiFJ6*y$;rYHzw*(UE2!nSkg>HtbYG1(-hL(_
zC@8olbg6aMY3^jT`^z8my$%M_+8SBH-bYT~?yx$`eKvB`;GzDge@2}!gYIF5EkoVJ
zhX(g_AIdn;i_q)_^t!u)4xv?LY}$Laa<gwUq_tMXY3^;0^=AKO?r>lg8HYz7ii<)3
zyr$*5%OZeK*#&C$!x=TYlN5Df=vgF+nSkL&&Y3>Y3d;(K@}AcFq-;qr{`k>2e3yvr
zIJigg!ls%BHc--W0PfWB8UF^v#dxK_YQi^tAfaUX`_()nad`4_G>V42ym{H*NsFL)
z2<5g5S+=FeqewcM0@SB+?K|$8C}bK`-N)uQ!b0|K6tgZDRw|nT!gK`X$LwN2#y^xe
zABPERk>JOQwQRxrbd+AbXaOXp?FGswEF8sbu%t+0fHrEX2xwQ!KtsP&{$ASElwRw=
zyg%+SX-4($HTW>_{dIOU3nnC3dYTjNkeLrf`5n41a<#vMs7w6ST_AZZqNP8DWObj=
ziYvmyM^x2O;ev_oYCG#syI!8tM`-9I>wuh70d*dG`=AAFFybKIN&>J+qf8B2dOk#&
z2#utyJbP;@*Fj~Btp@pk(`OwC2$NS@Y~OcLHYi&@pgi=T-wiFoV?2Q`&NwzY{V+xg
z;M76_4LZC^F#k8ZnUx(g3ebzKmCgqVSh|1!M)y+hfBHr?am%wOV7@lc^)?y<bjJ<h
z_P-EqcXzf^*iedB8V)DbgRlUpFYNn0>Yuc?bGvwwlafYPO64R(lf0m7^{1V17(dpw
zp9uvFaH}r2;VBhsJ?a(2Xb}!!L$+7U+)f{Z5(&A89;5xx;w%|uYu_9><iH0v78He2
z&iE2@P!x(My|ZHkO8g;VP|&K*9DT2i1@pmap@wDRE^`fs&%&tVeIVjvleAVY#8)vR
z)+3F>Ao4Xth4`Z>A6gI!2Vj3vk<=+bGJ5+mK#M<V-YpG;7xQvp4)YWk0iep*zEQ(>
zG5Bxq+*8@%Fjd3%vs;0gG?z@n?~J|s>5+ic_PTy^?(O`OLnuyf;dWcSTMG=YOJSDW
zy%#05A*^8eWM&2b#VnrMI6tn(GI%50E|&y9MHO4cHTa$^(k|Mu37sLM&hfU8_9`v^
zpyag5^nPvoh%PJ>oUoE4+KwNwr7SjBmI<xJJAK&vPq^v6vd*G3WI`gaHsa+TpjTe`
zSnr26R^&LEC8i6X2_ul=oaS*{ploInHYOs#>l#4CxPdpRdk?4$H$6~!-m=6)9As8K
z))-vqb$GrJ`PDLe|4o0Z8gfYSIdu5GAK+(B5PWC*9;~5s!(Qc=AGyI-`x7jP@QfME
zsRQ>W815wuU;U9fKcVnWwb}@l1cOv{UYh2)-b5cclWkypfrqen6V%waNLED*o&%SK
zNEWhSfNDIyPR#O^zzSr$BS|+=JpqO(vh7nUBkTydRXvrUfjnXYhKz%hT7QVo(+kg~
zSr#o(u8Tj^%hh<4?L5I_>jd2JGA<4u$s^MPS0bFmW=<msEjyJnkbDo^MnER%a2SJB
zUH;_0X^ASj#vz0(NPq*wMX_JrBR8Riuci+u+Gjxd-@e8=Ee==IBG6)u(iHd*Wemvl
z#@|{))Iw0P*MR?+b~|k}%bt00VBjv^;xJrYGvMMVj9(b@MH5hZ)yC!frYeW}S;&3_
zq6UFSxQGGn5QA6n8i^D$FRU)Q3>_dhjWQBd0<VMgBuX1)1#&8b3^vk%Dmd+N8O%ic
zGQbLraI^%>@~sdW3+C$c-ELdnYV-Gm#bXI1<<kImlYpl^3s~|QO*>L2OB~)aNfh%H
z9a~TdKS)x+g@FonJPZuI6N~;xXJ;U8UYHq3p?nhA*>xsxQbFwSr^tQes+|pe$b21`
zQP5I<KXiB7Oepp}ys<&N_X|`NI$6Xn6@Nz_#_h;z<P{J=S+kXfsll9QL}+D5EAXqb
zuyxOT%$e6g^mpw=F|1D?Bt;GY#KQVc@3~I6(g8A5h%5u=5kvywqfAl$F_bd(JQG$O
zy6X;kCLV@^SdWJXT91!?D6fb86<w{iyTuPF{p(y-4hbt^1*(!cC}#8=Lg8tVe_p6m
zIKOvJ1n?ML#)xm<!MfssBWXm+i{NQ#e5j|7>^)OQ{5rg~;2CuCb--}wO6zGDG$HHS
zrQ4#qIh|NWI1hFWr|sguUSHF^qL0g<T(e>1Rn1j74M)w*e_{|khu82;&g_c<es1t#
ztuXLqy2keV4#m|lliu!v!=vEfoTpV7OY_cxT01(LE}ZfSJX7#RN*FHwVM>&?8MLH4
zgBhx4eH@pQ!q!fK>5F0r&>Qc51hQ*^tCLZ9rakj|qJs=2F1q|(+Kq{c{{x6+T{_rL
z9KlIqnJ^>E$9K1uo>rmU5QAxh?b02ut7~B;C#s`*D53oIu8)cN5&m_-z4;PQRP_9M
z3}`MMem`*n`|mylgfa9t?`1N@^do-Qkon3j(;I8VS#U1Jrz#Zo_N$VBirs-!{=D_h
zdeGMTRYiE(Z`Y30qvNpkf2l8W(88xC_Ja*gZ2n!GjU;N}0A*W3h02L}uW=Uw>O?sd
zEuMiFjSg4#U)1Unr2FTXxsj8D%$FE<cnFd7!Re<bCQ9@6F;EXkBA2cf#jK7@;at#?
zqJlPSQG!!@-_qbG6}r=Lh!y&v219o%(-f|1NISLU>Y=B%7nl^$GN9Nk=gWZjOW;Gi
zQ(WxbbV1B({`sX=wevk_cy@QE``??%2Yh!-vcRImu(WK#_|4DXFnc$9ke}4t3HUh+
zBF2w|mv9JDxQN#5$UPztIu^y^2s#pT=sggs*E<%l>?{H&bD7~a)qfudN8v2)lu0p$
zhuA{xH>@5FTGY<pg)~QfnveB57lcJ}JMw`U1OY}t52|uszk~b4NrqEoo^V`n9m*w>
zbZjCfTKEm>=}>^0Ss-d%sFeC=*kF=U8HZfvVIIIUh_uK%RAQb^totwZz|qxJ!4B0Z
z*?q(U4$nKF>}L!>LlELg0<?@|Squ3chKoiLskDcHii?A^*is3Lf6JA0l#2i8Yd&N;
zNUOvAzU70%?m}`9W>_}=zFyNJ2w<wVS5yC>r64S;3)A7mb{NnkJy6F;GJU5yg-7f;
zl5`yN-goW+JnkkiQ-si#24x)5lF>)EB>Znz{7#@8h>)l_LJ=w8;BlbZK}Z@iwrYM4
zfrdov6CGQ~E+2%Tt+7(K$Rprrh*ie$wu_mg`?uN6;XSnEE-g|x(e+8Q8#9?rYY50^
zCfj*t7F<~JE}*kFTtNnH|2dZVTh$E-Lf1S<K;6>Bzo~>qvJM!M4t0*`5-N-6f7d&t
zfm+A|04lH^2=gZ<UYA1J#oWF3U=S}TiGgZZQ3W*-fB8qTp3TscXo-$?Y6C`0Oy|Pq
z^gL#=A0PO0Yu%!QhvFJ>2NbRl;f)c;-B@nX@d|*(-{h_9xo8xLZqULh{eENR??3RH
za6*{K5I821dw{F&7(mU_l6+A&|1HOa8vW@F5705IT*yFNMxn09n$!*O04IG&W_vgi
zk2b?X#(L?nFqJyA!(P8I8?P_%+KVkrKp3VaxCQeFfVE#kti9ZC$yKXb2BQ;mpj|Hn
zU%<^O#%A)3&9OD(0z!a9-_(hJ<lUlJw{-1)n~z~eG@7dzE+F_2<^C8A@dN5Vd(UFP
z%-_-WK#y>dyj&385u9rA1bx9?+#q1oafmITu{AJy!Dru{@1SY)!n28)OTCa_f(IM7
zh&UwwJN}f1hG@{{DiCz4qdH5#lc~R;4SxphXi=Yw%wC?=gN7C5Q@yr(HVR<wE{f$j
z8B5g7L^6VC4)HAH6h<;!h<4iBNiD)#d@_Xg)xNOjm!ZVx@U74;g`k@^wWI_`6t0Wk
zhk;7y8(bJ5HdEUx6zaNsnvB&q2#~JbJ63N3ANm9@%O)oeU35HzS3`zoNI*l`I}Mb$
zMUhtB^IJu{CqrfQ2B72@54ddoLbCXt--#sRG4mfaa$WB<)z?bH(YIrKKuGr6yJXxb
zZVQ6S5ARpt(0=%3gO~1rJhA~^0+X_K_aFgr9(b}X*4F1uz>2H>8risB>Vx`=-%u@~
zy~C$?i~9E$el0=<0$_wvE=S|E$S66<<Z=8CwH*nGcJMlpB*wKfko`uZh^i#Oh1%`X
z|KLX?$qZY#PyOQ%I8VKS3V%mPD1pohE@*zu19k@&&SJqpA7LD%$BoEqa~vAt0}d3x
zr;l7nPx{Zi3UCCLj}W%rdSJ_Ng^bUUr&okLJ&U04O3a0LV#U8K%InHiq%?G?=H5Ez
z?A3K@{VqC*((0FWO#gO1P6>z5{0OFuG-Q|~foN?)YbMA0n)QFC{)=1<JW&?(2ZpV8
zCkPtGLo?==rxx$j|3SRHUGljB565Ro*q=W{6XkIQaDKa(d<6<GG}rc6vpf}d9Q{Y_
zBkEj%^#!l7mYI^-H)?xdqrXuNp79#J>z!^~yBrYG<mVj!U6q9u9scXUQTuQ~UuXx(
zNS3d$24GJ7@0x_0cwJo$+vQvPM6lGUhcU1D3Y|Ky|MhQh{a)zz_OeyEWeksU?*6a7
zF6tuw!IUDW%PJB}KctP+Cy2hIdo9Nt$wK~ec<xToiAIhwg^Z&0qLq5)Z!1lv6H7$a
zhIo}%U9KyMee3o8UOnHPv<Fc7EabBqq~Cr=(9KP-%|xsT9v$;X1SgRS5f~m81{1Qs
zu5v({23=4OPq2`0evDTqt2bD(o;Q$b{ArowhjFz(q5dKkWnot4gn~R-H5PK0PPJzO
z2MxI^l-loe8}4h!)WxzL^3RbiEBamz?!y^+Zdiqw=PRs3D*&qq5DcqdD_^Jo{x9T-
zGtd82m}RG9!C2qBsB&nCXnV4?-Q>lHV+yYR{JMReMaW@{OEhUvP@DyZa;wfy!YT!L
zFbC>b=N`VUL-rmmggMZZ9yxy;u{XkN{NC@W{P%mX%|r~}P>Ei2I1y8+ya&QQ5J|sF
zC3z^0?>+ZO8rXoNHFb^1Z;@r;UVfLF5p{8X+A%A8C!Qw!-_>cB7Ykp(ht@IZnBO1#
z=%=xJ7Zk)QJgCF+itatrjsxhbyT7f`e0)Rfiaq~BgdwTLx;TNhrrLbKR6}&I8=Jq5
z7P9VLFd08&&chm%Yk=5;t~Yi|E+&vNJe*`UVjhdQlQ<%7-M!-wUmQ`A={<Npk|cge
zVK%z1+6exWh_%-sUGz7Ye=WVYmgaw(iT}TwS3wTg|2_HtGyDHL^90QQj~rfy3TAg}
zrR4HoyBln;uG~*cQaqzESg;p8XttGJFM-d^LPPZEMEO|9<Fbxogn(p_%QF|cnSVsW
z!rS50?}7Qn1tK_AW#g~O?$aBNsUH@iB_-osQE3W0v6rn^64=DeOr#x~<6V3I+>9hW
z`sZ>*|38=8aSYhvj6|bj-1U$9H?h(AF^}%*k8iI0%+l5%Jhl<Ko`HlJ2tj__q!tLr
zvL|evx8D_A{myMjwVZZbugIY_<%`O%KNq!O5xKbRknKktxp9w1C6nkDN8^8H_$#a&
z&&)3!=`3*6cu(H=v9ekGppj`H<H+{-xh3uIWmluElGS2;9&N{@dC!*Rc=g{2xW~WA
z)R4&YRKT}p%jEY&#PV@YQI7Ya{eo^)7r48=DvqV`m&p1_jVmyPBjL|)o#%ZLfo~`#
zdaC`5(5Z%u=AVu~f81THo7+5W)2^RY?WjHS<O!vsRQaI?O4PAD%3^^fI~=R_7A>vW
zZ10n9iPs_Q?tvd~Y?Mq3gg`~{;7zxKqHRM&n6L2Z&wEfoD>&hsC%EViT8O2?_jWJj
zQSY9e(s)nwzZ#N}7Qgu`dOCtab@y)_^WWel#$ftC(TQb;hLk@fzw;jXV7$mt_Dw0&
zc5E)dhD7n*Ycy`_Gp)tg*NB;^kb9xKihTZbHe!m4R?dt|p5eIa^GnM)XX5o$G=-m~
zc{%K=e=i9xu`ak^`7V6P=VM46cd`4kh(87Vp_75mYYoC8xyqC#C*S2-7xY#R|B$qr
z^iq`X^dF8|2*P1mg2mJp*3^nFZWr*)wrpQ{GFLk|C2chtI)BsZ>IDmzqgP3rhS~G&
zIh8gEima;LW;vz>50rnrzDjFzdSv}|)ySVI-g}{^wq6x<d$(vEV;s|8n7TAVXPehP
z)8ZtHEeMj;h)QI%j@628E!qv{_bi#=Jv&n&_Rw+srHgu<KS3D*XSHkJDzP+ZU#qZ~
z{Pg!)1OEP`1IL48$NT=4c{kp0e{_F}x74sjX(~5!Nf}VI*zT|M{%_7bD&(x8%3DhE
z3>Qzbq|CfB@WJD0nnH-ik%4o^-byr0X_2eP_yrrI3P$zr|M1OvcubeA-|*dwy@S)B
zDv<#x9x#4<)mN2q@tG9g=Iwt^34Xs{^r5S{x1s+oPtsNrJEL$>K##`0-rV=sa+}P4
z->+#s+~XT0<$lr7xWHF*s!wQpFs@)Q?Pb;2*QE0SE$J`VJ5A9mkC)sT9Cu^-O-gj=
z%7*W~pMJish{66KisY}_u9XCwob~%)WPI0uE>vR9_u%8$Sj*jWZGW{?3+HPKMpAuM
zAF$jP%__wdYTw-a^J%cI_G9-(WsTSO?q?q#TtxdKPi*jMednyN+Ej=)U(D$H;!L0P
zg-eldcGRY+HBhCgSM!a=1WhviCc_z0wtMP|u1uPl46p8gb}5=_C)Au~pf-_!UEPJ3
z{v1iab?<IdNZecRBw8Bu4c+C?ZMhhoP-?wnx;QOTU!Pv5gKI%=@n>wGZ1hXNU!1~I
zJoJsn<vreV81?p@G0foEa>a@l=_)f9vSK0&kk+2}a<Mn3_z&(c%g~R$vAcgGCTQhH
z;19RQznw02KB23)M5XZ$R?{ypE|>yxj-&&>Ra@(vxLSDP*n_UnU4H|{=|0g=o|B6Q
zh(?(Y5~!m3<NNe*U4Ad$b%p%-OQK78m%xuT9{xUj@XzFN584;{8U$3#iz){P4T5W-
z{R>+h+8kBODwrsHa8SR<#!vl4r!o%}S&O&Hu1juw;eq;q0JZN-w=ZL`UJqRh6A`^R
zuho;VPR8R<i8dYL7!h@xLXG9}g2BVC5Gfk8kksj*wVw(++PxG}@pv4Gf1S?i*iW-y
z4aT|SFXS0pW5>lfIqKcmwMtW5ZVV*eml@St^4*}Z5*EpcCZ*%KDt@sxq=_H3cZ<jC
z_OA&%wQ1gS6?b+{Ebbt;&w;oKW|@<l_J0Zz<#_4e9<f!a8cqHDqmh>nkH4N>Cto;n
zO8U@M7L4)r)s5@%BxNc4AB;LUEC1T=j_GD%>p2z-m9}T-wgHZi#ypVcEbhb{b-lmB
zF(tSB@9oVBH0zk<ExccrCV!6g?v6r9%P2i&;+oi8K;4-l-=7j?GZPViYLcEV5v0DY
zuSn9Mt5hE@_e!#0ZZ8;sDFkZY4JUGq;PtN((>co7>Pd?wgF}|TA64yKZ!V9~K4gE(
z+A=%ben^^zC17iD+fwoGT$Raq<1Zyau9yfGsrgj1yfd%$n8vSYbB?yYTI67hZ#wdT
z@tpRKYJkD#m7fmjI=LpSt!xxuTlGz+v~nO+B^8yNW%RmzLVKBMfa!hc_m@^_YxR_k
zl-y-35+eQ&ja(g%@%*aw+%MNO+O0OBkgwgHom)ETdGd409cn((i1-v|KECwKA5jzI
zlcr^sEv@$}{xseGy<4BnK2=Pofa5Q<LO*RP5Mgy-x3g6$obpwi|2elIYBd*kK={;S
z%%a1fTz2G++W_aL`eoaag3WXO#I(cJY*%My!h9VES1%k1nfYrnAN6GQ%#R};y^=q3
zFMh-E$B1L~!a~l?jqu&){2lZbEm6>i_U817eG|QU@7pxfwLJZNF+sDlM|e+r$$V1X
zcCQsw=Z(oxs4SJUT96d65F1y`H40e2vRutu6xezF0_#GkOpenp!Q-I=i-U6$^o-ek
z+3aeK!0$DiZF4@Zj(Q-uPtFYYg=~|wE4=AD3xSX^XN~awLodty#<CbDG^gIM<X(1`
zHBaHR2M&%yb`}wub;4y%0Bg`};Ta_Si19!{)yLzVBM+p*Q5sUyKi{3H)AwjA$U9qJ
zbt{Kk&H-O>)^}_96(jEt$1i@B8M62J`LH2_i>cEef1Lj#dgJDdwl-bPO5d~ir6)tl
zf3y3|cQZ_<(ix)7Cbk>Yj+~{};JI)$xXfWm{IY4_(}dm62~>~mHR^14UtemmKiH(^
z((m((bl`TxCyn`@)B8~<MMFI;^|QF-W4t}@Jf_a){v>j-OPo{AI?eo%-TlJmo68rL
z-q^OzuRRJCx%ir*Xr|7lD_}13!)LSXm*%Y!WkQ9jgNwBR+a84F>4m4EIon@6-e&J=
zm)vmb_iRrj1+%mqR9p4M``pa05c?iv8ra3X9%)H8CVBmK<GFkKb-M!<u9dULH7>EA
zv7R*#jEFbKp(O)Noz$-91PiUUL|mWiye+#Bh|Ruk-(o>Mc$#PKs8|=Bm9^GMYa7)O
z>#unsuDtpM7t1&`hPoOO!on--hr<+BxaOoh2>zbJWQotX0fyf*H?5@H#>Cn;{qGni
z*_=1s7QGlnwS9Lq*qa5zC0X*)m;JZhgX!;C&SmlWRl75;Mhf{~=;FR|zVhuf{K~^S
zR#>ODR^gpzUw@t^{<BDR-M*d{i!VWSI~S&|CGybvJ?0rzEm>&mA~Sv+VUVhu?={r1
zQ}f*2YFcya+6j#=-@O>Gxl*!z?Af0u`V^WA-<wiZ)*a1ko~=-n`rc_x-oM1wSTGPB
zb+x(X;urlpuj;yacIrHna>{1H6Z1{=F5XyksSJ=m>S0!~d8vkr`{X>&wDIrNqtBM=
z?}<KVJF{ln!})j;l6dAhwJ%cP-5-}CcqHNvU<mi+#Vm7{MbEF4Q*68nI~*nGt2mWr
z#zZ#17H&%YWQ#QTv)q5st#bdU+erA4b1OgMmiQLaY+K~)dt=pkOG0*N6-xsPdRh}r
z9}Dn*ZZ3|f0<<s_96c&SLzZ$Lu2Y#;(-_K8pBecP_$nfY*^hgGGIUynAX(XOb}43)
zD3WeQ#a2yh_X+as#ah(vf7cnEN}}Bjt_jOm-+hNpWjMn~F*a15*KAg{_$8{t-~vSk
zedN`sn%c<oVn%&3R*^(&1<Iba?x|jrg)ZKEml@57iJw0m{h%%R*KYL*>E%Os^>XT?
z7dhVI*tE#nBy$?F=fIA6_N#f7)#QiY>e<ZbA~|(@uN@}hTfI*f97SLInrB<|qJP#u
zXKQF?G;U=r`t%&_ua1ZW(>cbDgzU-#@fK@=r4LoRu8pRuw}z~q=WVFlw(h0N85pc|
zD>(A1%JlWs)axZN!zvyML+t)lKXDdPf0|H9t;P%PYG>Cr`$u*uE7Wu=sN<(|$OWZG
zR*%~9qdT8d478>4yrS3*u74iC*xz58e-`9lEhbtzxnYJj1(uSNmxx7#f`~JP^Ot>p
z|4tFON=Gwq@^EumYdW{rs{>V8=))sF_}F3n@;cTKm2z=ZQtn&qOY_*s%4E}Ddn}Y?
zB~66{ur;@%P<0FQrz8&Wkr;NzIh7`EW$bTnNJ(PgLp@Vnq!Zf>%9s1#qT(NS?R4jL
zVJ7m+*#6XFyK`R-m#91yb2t}rl)r%)V~jPJl{OINH}GFt6Jw`ZvSg26&bYJ^e=aL|
zv3=`iymOcL!M8=9t(l@^&DRG8H(WODrexy#et5rRxS(=j|NhFSi}FOWJvQ*Z+#J(U
zn~NL!>G-5_YedUK<y8Ypl-n!)jnz^fl<NqI!6rJE<&Wl}$6m*ztB$xoT-it@y)6>K
zUL<fmB7ABImhA4mWjcBwF{u9PErn)}FqL!Jmn{Ser0Kn8Cjz=%1;9riI6Jd2E7@Qe
zJHYDGcc7YwzN~;JWQEdD%3wxD&h|poTF#{<p+7C0JTm7nI{}BW6|)x2pVbR9=Zx1C
zD8Kc%{!I*!D=8hFto;!<S}KBe^R1{g>-V{<EMIZPT)M`;vTHrVx^B^#2F->4(DNdC
zF<wvlfy_5ez7U%wDzD8qvB{d~&{QjbK_)sMdQDR!)^Q3!UvBQgL9sKH@v;FkUw3+C
zfBgx5!-|1y$aw6*C(k|!j|}FyiawhMM!P3i4^&pAPL~gUw%#}Md+odapUl?@)!qYt
zwdRjsr)8^icelRMvb4^qS;iD_C!XCVWbC?S<KXoh+$9-4=WcNyeDLN;@PJQ-%;3F|
z;E^{6kI}fs3`;JH%By;n*dFj^eM3!cvD=-zHWGM@=_jSq1I{`h!yc&~Yt_Cn`INPe
z)2eYvmu5e?U}{xyx<8z{zb&+gQiy%zytqxre|+ld=K+a3JUUfv7W;1RIIHEV-O%!L
zwLSifU-9_(0Y>Lwlh*blk4Dsb{JDZOBCxfmbzfwCZ0oG-DSb8FU*7pLnyF1ux&6(}
z#M4<PynfxpM{4UNTiJX}uPtY2{TLoB`-hpEB|vm&nk`PHf6l)}iT-0%f8A4u?35<8
zh!3M`PpQ~mEOcoe(HePJWco~s7bwh#36J_t0j!2bf)D0_OhKQqP8F%}rN@hqu6haH
zoa7p|8#uO>Bw?z>lb;7>OiaH|jY{@5lf-A&zn5kT4$df?<p`BN<Bw&#_wkuz?~U=+
zXfrx<aok7Nx9QTd^{)HGChznPB=J)-O*B)Q+tPn{B>YSa+k1nZnyRqwea=nCbQwB*
zgSZ)wAcHk_+TCA`KMvmc{^*Eohur4?k241Qsg9Y~*mjPv;c2*RrG}DnLMktL+s$BK
zG1Naz=hlB){x+@KeuIaOuh6vz<Ny0v#sf!uZd%O2oinD*X$90yQnBW`3e<~+qP;oa
zZs#qJFgCH9|Mfg$s%d*jy)-lERl^NR4%h27x4gS^|8BHp-&b>q-Vr!;cWUPIh~1x;
zQx1<W@?mXVWPLx%Hi!SR7*dsdWq$GlkJ>p&RieSn&ktt|!|N_w9Q+|ZIlohTW@IR%
zGHTr-SN?SOWY5{wcY4(-L($)Df6FB9e(t(fGk)mxfMipzt3FS~qvUmUiBFmVgM+#j
z{FgZjEAQ5=;9M7lR^IRbdzm`8qF?A~E>~^#(E5YQZW-mAD*Lc60$3Uy+*W4a=en1-
z_wjv>O8lVVl~YyN80J_%aprk3wch*K6IqrR;eMB&p?fgt8xNz}o-W`JLSXpC-RV$G
zj6kt4KBdq49KO@o`8sXqa*yb|ZSE_%PUAzGN?UWsnx;C64@Ti%oSiZ?6eI`;R2T#X
zZ{|C$q#xiz)$(619@e8ftRyC7ce7-(cgA(Q!lXcycM8UOD}2(=T<2Oh*RQk=voE*0
zd*nwgTSkGojK<{ZWf6JhAg8v&YU8RWcTV?AHC1hy7Tjm%4Epuz#<G)no_*_$Yt|F4
zt_S7h{0?$-*<z_XqVUPzJmSQ5d>&Z#KH@*eTV+--_fE8c^v5RnIlH!0Vb0H2j&V9i
z*luc0zU?+Y^dR8wt+#LEnR?RHg5K!bUDI&Q8A<b?iAUGz1lNfip5UR|ms|2$?rtKU
z^aRVZzwl&FtEqpkeO352@9eWCGf(r3k3HI5)CJ+BUKP>&yPMog_MFl5B+8=|Uc+`w
z1rB}=w(k9ZqkC^CD`Xl6nu(D4)|)v+&TAyu9P=i7td(zioDN*e8Z43+N!aa}GUtJy
zuvj>#*}QB{R*`(dK_dJtU8ZjOwXe}hI4?bH@UTNeals1D&Ud?mY=3*q)VwQ1ogQ6#
z*OO#x;L*1GhcszDD6fgjJUXs8zh5-a&9UL-&c}JiD)9rJYYX+LgSUeRT^@N)r%Q*Y
zoG&ed4A;3oN+ecABC+}&9<zSov1-WD=Ob1*$|5HxFRcno7Be*JvY%{EiSjFZ)&4u%
ze@Zf>vyM@SxuPv%n1^o|hyHpYS<Nn!))Q+R?vEZgmuj}1^%M0ysLnffvUPQro)6WJ
zTRuw|Ys%S({4~Gq<h@dX+vb#tiQv*exvTSDT}eOZ_`2g_O){rf$ZVQd<syrg4}+G=
zl@=Gm9UHf<?vJHbcN<-IADwD_^@8Fm|LrRWHqc+UxyCX-)hPs@oS+-JM~Y-W(T79h
z7q@x+^{S}bdLK&^-io0PAmX`k50ld(G>XHCxk?(|YVAAPEv9{<qcU-|ZO!T-DJ53B
zUtSLo;{y|98jE9{6oT8%FZMnPbq!|w!WDirwTrJo7rjJ3;29pOQ(E=jntq2N`JKl0
z3S_-69}UMO;BgUgN=&adPHhN@bRKQ1^{R+3BINj)br;S?-ojTq@=SG{6&Gy%6dC^c
z*f(9>=-U)wkyP~IV;Q~~jK{L*w%2&g?F&ES%?E6kh6JzAQ5B9A+M*v{y}!sslHPxE
z#_3iulWlkPrw`rb1s*Mi=qNpI4>ynfSOJ@z!ZEHqTMyanR32Hws!TsBJ~B<tc5!<H
zqj9!TR%j(O1l!y|Q>8wcT$xRZ_*wO0Bg5fYOmo{%x%;xBxnE63lSDEqhDRZ*|LRQL
zQ9VsQ284@|b#nn&e88QN3?Se8f*5U~0%^!%q(ZnvIZxRJANKO+h1PLB?10!|Q8TG?
zSh|O&TYWoDy4(6RB`}$f-C;REZOzrKeM+}SZ)MnUMWUvWk@Z5ag(v0Z<U$;J_?+%N
zI`ghi`d{Bug*|Q!TMLV17rRnj&Y#O7x6we;xBg?$%8c{srpmV|doqo5!#-Eq1R0J1
z{N=5o`w1NXVW|qr3PA)*-PyxZt->??V11o8GMvS6H&KTb#Eu&N($~b<R(9z7TU@W&
zw5Sv;+OM4aPLmm<E3g$Q9W-HH`N7mcotfsQfB+jGivN*OiE>U;sr$*-Wf9lL(j`^1
z?;Doe9@vpt!#HBRGWYTGa=kUUtfu?GC^e6UmJ4;$>sdq%o0zd3_C|oQm|gmfVAhIw
zJ9nm6n|_ge*jX17GmmpUWkzH#ijzL|FIMy(+|Rya#)RJFp3lf$8L?hz#U(zR@$fTH
za}w^nGI!k~hd$c&3YILSuxfq6mXfKJ9!GCxTg_r*X|t61E+{4uPXv<E-?80Q<DG|j
zR?pK@hx2qa{VM|xW9`4c`7r37^)+x#hM@*nr3saXH57r+61V+G&OxJ;`8}}=*Fj0E
zJGO+rM|vwCva}LV!m@Q~TL37I*jsq6XLsJ%9-whs=V0AD&tq&Hr*S%Z{-W{acaL^b
zM#==LrhdDbZt$qA+R~ex_x9KIALn1A{KCui#x6A0w^0Jz%W*!El-Bx;JrDVtN|zlW
z!oL>x<qKo?r0X*|-3xK=ucL>%cSePlgTog+N|GM^*gW=7X1KF392Jr<)5UdnTTYpK
z=$HfNSCwehlpMClu{V!3YVL1t5}YXf$c)JjHSRj3ar1A#hVLS?kNL~F`EZotFZwYD
z9D2F)VVy%%QP%GD4xeB;O;ba<Hj!_0qA9Jn@TdipClUYRgj&Q~^;f>suEA&x<x|e`
z#{0W}-1Qs&JYQuu2mbJLJGs0-Qs-39J&HEo=P_8Sh&E2U5S`>q4pu7jb`3nY)8zyY
zP+?t=*RcK6Supv6|HaR(nGnm<F-0$q=j-{hhdGxCDwGG^XX9ZWOJfn|!$-%o`OoN9
zQ9M^=lI4iQ6wd2-2rBdADkw=5k=)@QCbMuCON`qC&*AFxeU<a3h`a&s0@fy1T*S=_
zYT{VSbCuFlU+M}v`4pF5!S4QbKM<SQl&`IeEB(k4gjWmmJ<iz9(KRZTTuM9X7)IwR
zT!7ZZl^W`<(o}yKmYRv!h_0b%Jg@s=UP8f12upOZ{Ux27!iSx&Evj4l6h_Z_I4jj;
zC`DmugP6R#q5Mtj{jL7_LX&XOsUz=67PD6emoRJTe&-co=8+hN_(XwnK_ER^J$zOj
z$6I)pOKL%R2E&3&O)-8jx>~f_itELiDyO|F=Eb26LQd@S#F5SJcUOGp`t{+C32zp;
z|Jp#`n|%iAY$U0|rnn9FKb&ucbt+6sT@+QsryYN<?Bn1*=$TVMLq9f0k$TIFSwh!+
zM8iVp+os(h{Vjg%^h(;J;i6l<ygiavt{KoR>{U?22QbzNP(jf;0RUwExf5G|;T;*b
z)8?IaXB2j~C5!)vSY8dth0dqc=AG#0=3j_~XSKM#7+AasZ|1WdUBB8~?iMX8eB9EA
z`^9m9C558#hfY)7)FA}X2<x3c7i4^XnO~re!aaVKEnD!Tiq8vE8&S*tJN5>Onz*6k
zAH#Z1KW65;(h#B1;y(A3&TqXHkGXgDE+?NYn_#J+mxjVcn^?c>O4A>u7P%i?5B5w&
zruLurkNVbIWgZ=7P#tKIXD#kueDQZ-3A;ES>6-G#+g#Ep&T8vii-&<L8(hQ8c)=Qx
zTX>QkezwVQ_T1xpeQc5*j<quukGDQost@Bl>G16LXqC_58`-E-XP(y(#HE&nD@*|`
z>yM$;Rq<+i0SBA-4`bsJD5_i{e{xY!5_Bzey~1Pou*NM$U-?eKJC21;vo&pA!lC^t
zsx~H^qv@;P@)52-cF<ACw-nz$d2v)N?mq4`ee{y*(f}P7(`e;xS;gSB@QjNaM>OR$
zrIX8Bs!j~^v}oEsk{+6V60fYOu|1IP9<c!!Wq#5y)@|`=qo(gkl|L%*hV>u?CO7Yr
zvN}A*_k*!c1kIaXZ3S21M!x<_M-JW(V0;|~K=`0X{6~K5zQn)RJyCB|iN$!ooqZEf
z$x4rAwx*Z7GwaGYRh8^ycL1e-O^D<A!StV-?!5Bp;i$zdVk^4l*j4xUghb=kP3yJ{
z8T0qwYEOM{dia;&Q?s#lVR(YMPx$Z;#%lu^P3#-2xv!_~<qa4d1>A+O#=Sh;ZR3SQ
zfBf#!Hgn~z!zp{Wo2?AQz7{sLrZ|ktT;bX+#`KE?n5JE&DCX-DC+PW%H}8*ie|+_>
zGHR!<<ocNzBUd?Qb;`@s)!`45)33bN3{9KP$@4?;k>*wITI+Lbou+yq8nMQ6)|yeC
z(!7jcgocV4CzkGMHfv?uhG!8{tieV%Q*xAfsF`;iqmvI8)_C#6?{+%OIL-b^R2lu;
zqLl9>tRM3G7q#2#fhEBRS4&=<U-7lB6gO*luZr1`W9E%#y2JPqvySnR)_qpkHC?eD
z%Y|1W0uReopNQ+CYIxDlaas7HvL-RArt`Ce%-MlE6E_AsBBF#j{kEv^os1q2L_`Wn
zVTThN9@*tPW{8Nf2876{zbB-6dxS!UQaFyDpk7lGGZ_A0RdkZle<DwS=|IWUvyr1!
zhL+F^u;}u}9jV~)BUK5tbDFsDBQ%sVm5q&kv#+CzF4e}av{pOvy2vjwHoCG#;aQdH
zK7}#<;j^XJ{c1@u^DzbgSm}M=P9h`ggzQ<;im=8t`@2l7Cxj!1&ka3D<CyuZImGw#
z%)SZY52JnA33f_955A;VvUR+h)%Z9u2TllvZpy)YNn^+{x-hwv+G*Ti?AoDi4<T%3
z^_9k)&{LBtxR|<Vyi{44?b&YMVQO`rOhIe4JI`&k9Fux&CLi|(|BYskZ?O^dfB3W2
z_l5N-p?f}bF7}d6syg~OLRIe)zf$UCk4?fZUsuWi9%D+G#6<Xd=YDpROR#W<is;A4
z>ht?MoG-5xUM;v*v%9lD^P@Y_j;(RCC!AW;{7JG;vg#M>W6f&MdtWP?XrLM{4fmsI
z97(uf>@rKq%PZ##w|rSOaU=EKk4?Iaip2VdRl>ptaj|YphgtT{X34h1#wCrO^2%5)
z*2ha^g}X-$OEY!r!gOiSR<EYgPfZm{e_I=Pafp*pV*Ps2jB?_Yv7F-VM11uJ&ZT|W
z;a{0&eQA}SXGTabJ=?C}!}ir*-8}yLMB9&_x0m1S=bQgqueW$}1LDJ-U*C;z#I^3l
zN$TR+y1c6K8~#pl%Nu`~BF^_0MJVmB_?19l!3884>)kZ8)*boIRk3(xD!qi;jQ8sn
z%jCYy%f+BnI{3KKradG>P5V#JxU$v$8Aj^`7u3C5?+Qa){a3b?rcH$B6p!-_Eg#S}
z9}hlK!RK{1d{N_Mek;(E2~%!fW1x+SFp?g_b5otPDedAutRlyWqjN~Q5{_a?9d-;a
zTE3s-L`7SSUS+%b^~yo1KU{ukEFyd;X;c<b0CRZOsd1Y|O`5+@R-OJ>>8^dpwW#1Q
z@n`W07vkhg-b7xu*?qP3E+k>#03qL^;*$}M)c^K;ZL-^lsEL#?)nmItyM-Ic8%ByR
z<(zP@DJCNBlI&Mz!m#awAC6x*U!D5@*m}#Lx|*g76bN$ggF|rFU`deR?oP1a1PktN
z!QCaeOK>N+ySqbhch|eg^VYppx4wT!A+z^Pcdyl}*EqF%uAcfqqzg2JzZa)ACtBYL
zbgId$YOot@T~p?Ts5SL7TWQZzdMDcjwPg`6J&-PsjDd@JO+qoasv+WHpNcO}O{AlX
z20cWg_PqnWi<_%E{M*$;SP<6b1Q@CvbUNfTSc^BzS7C<$@AEMz3e4dP!}m|`NhZ*4
zm4eC_u?g&T{Wg<7^zm3vHuu|)wePjN7d(ti;N~0xf#_Mmy!ROC;Ic_-I=e?Dr$QqL
zq1_;(t?E(~N9mZjgnz)%ex})@K{#8cHpB(&9h2jFsoGn}S!vPr76g3=GfKaYK?U<#
zuX@KI%0Q)6%PF}84>oG`O>E#eOU@pee@r+H0Q1j_&@89wmzdjdYx{Y7SZyNFddtI>
zD;JUXW39UrkHh)(cyFL=Ts7C$AKvipNk+VGh=6cK6gUJS+ORMVx+buSZiGuSt?RUJ
zOV(swr18QiUGi_`1HOQs!NKz_nwe)z8yh3mCa0dq9gx|`A=Pp7R>G62^{j8DX&jij
z_j|k1JnF06N>ikJzAuVL%<}307%E7Nm=3;?(5w6uf021mi^&h-z}F83&x~ETMl94i
z09%NVJs5a)=+EKUEv4NJCaC96<qE4DU#FL$v_vR?4OHRz7Xhm`uf-BwG}`hwC5qc|
zyK!I0)~~&enEfCXgXR!u5OYF2{NBR_<39f7aR`CO_mltZn@Fy#7=shr0YUl*%VAG#
zLe41&K^g-74N|=KS-RHQ^#axVn}BMk-CYWX+=DMB(4h)9MZ|(!wR><egw)qOZn(-u
zaK}>RCsdC7AdvJb^FtDW^mFT~Ljm1PcB{kFa7-dVyKGl+zt?ux1#v@1K&o3Jm5NOv
z--~K?R#h7;(2s@mo6}aOa48U<fC30EeAy;ROh}EoOdacFyoG3Lkfb(|@I2om%I|)p
zuWOS<i|f~mO-9ffP(v@PsVI%U;2_>(Z<{TkQa&!2w&)r=vLtoY-auhR5;NAwd-{r8
zbCH7KRUB1szC$WfN5|@RM`plb`DVN4x$?cSpiD<c-Po&#y_>F$mZHAzLpHp5&copq
z3{i0~Lk66*$=(a<=95k4Kkp(#P9h=pBE~tv3o4YRvgmw+rK&G<(B@iQduB7eSBS>6
z8eaG(5e9_E)&!`L`J~WC3A#aZ5e>)lapX0iFyYh<Q4k4jrUWBW{WFYGYur7tJ6bLT
z8z~hssdrtQt4t5J%7DKf!^CdWz~t-14&CULk*XlXc4dY)BslIGgrf^RyAR^~n-9%y
znEf2e1I>*?nl<|qqz6gKg+!-|xk(iO7Pn;DP#GJltRaD}z!AcgSHmYig6fpUQpN5|
zQMDRpIX#!{xTveFGIu%&MLBxXqPFP$f<BuCDuS2$S!<VIGNyr>@ONdea3hBt{s)JI
zg&;|?@%y3l89znHh<TxPk4RxK5vI)odAz_7UnRF1S!ry{_8Xr=OX}VHv;Q8JoEHpb
zsb5l$h{;5j#w$07y1`%!f}}tR@b}<2e`w8GQG<uv{2aNJ-){q#J?N(=@abo|9W~5e
zl;DcgCqIZqeh`OnJ-^~=8h$<Wr_c%>62;Ev6EaFb>1`ZS7cj|#=g$^)6Hu<u=JKtu
zT$l_xYz_iKq>PZgoR^Db*=rzBMOz3%1&qeg==JQ_{;?x*3k11YgsVKT?fvbJPH7mC
z)Vp!w05o#l{3Oz86oo&qKP^3<FRyLmC`dAGcQMI6eGZ6suX*3pvZE?vWl;fKU~ErO
z0J2d{#IJmScaU*+LmCe%*b6hezPOGw*<tzvlfuKRUMR3|^>JE<#*V#st81w3PIkTS
zogpJ;+e)jal&y|?Z^%0gB+n>tcM-kfLx{YLq|vrw>hgUQmQZTNCh-r0O_iB$yB-Kz
zsb{$55VgJ#T2v?lb*PJia)bqYK}yRa>%Q=xB0)R}`3tyI?9N$#Yf$ho;Z;av!3YQm
z96>OlkI0m2!(6fNIETm<klBPP)?d$?3?DwN!kE`=e(0?_4Uyco-yn<aMn>@Ay<Cel
zF#rSUCee_B4qCQ9%ZH=UQ4R~EFV3(1SAn$PIuiDi@SAow2pY1@g7=9dq6rP^(Q7HE
zOOGs$c$QXt-5A)EZtQ{(X1cMOxGHJPA-^t1snO7&g7D2_5XnW)-19VnhHOBTpZD{9
zS1d%Z9~$xVS*!2E^hcpZsh%dQI-JX=1wR7*cUdJ>;jAbs8>Op+Tdt*7vY_>RG<3{d
zh&U42j@HZ$Z&l^Hnct&I4_q6;23A+M69xjidU>Jaq*>@}T=aLpz(Iy%2r(A6yLu0f
zj~|lJF%t%YeeWaIj8YDXCg(dP>uis=N_m-7MZf%wOV2-^zDog`DL}!|q_+azZt)@b
zAdB~~HB@D@kGj}JJW#npp&{6Q_!wZ;CkfK!-g7s5ckEyl6O7LTzHs69LJt;}0?jWi
zmv`ncsM;Y0fua(?1-G#+&G!Vr-X{pbWg;D&zs>(3#MX6LS}h4}mI&<5D10y{6sgSG
z9;Ip#2v3$}3$D}J!2P2>bA=m4vfW(@0V%V=(Icxz|B?n1OnJ-}MSWEXsv-C&9?U!!
zsw`$Q=yv~T(>4c_^~j%mtLXPDB`Jv5GOJNO-On5Uhfkk42zul1K7OZPm`i17+2iEW
zprN>WC_X@e+m}u7H(`?3eYrrwWhv_szYB%QQBn*ASLonmg@ZRhKPgzmqx7L?NlTR#
zX=nzbdx#c}l%$%`p{N41f)9tJ03D0SvR0Gf*D+3v(&3YQpDSU|dOFbc$}L^oNPFy%
z4p7Uy{q>q|MnmUYl-Rc80i)oP`=#H|djOU34jB%(FFrOJ9jbQ{X72sfvzUgdZW}P5
z#r0ky<GD~A;e;~spVa%wm*4d(Sr3LDu$PX>v&c(?NGoPrw+Hxzz`$1w+dS=l>uSOI
zW6cVf^x={9mOLI2K^2;nKtw;G;lExZst5~%G8wnVPdp|!S9k3}3cH_UC7mC=)*R!_
z6}LZmbKHK&7m3|uh{NX{G=v7~2(p|wYG>?i80{hz<TQK)ZHzO1fd&OXM+C|W@7{+B
z>!J-s<(}i0Z~igEN)7yqD<OF>M7_}=8Sp-8HzhX}`17H9a@a*l+&CoP!3Ye|6fF_k
zz#>V+;AqNeLCtmREGL*42;IMM=d~F%1W6KZKd1;oJhPoLbS7|*@0hyO{?JO>;!<`d
zd~8V`s{}*khv&ed9;U+0?~2Csu>yJTudTqefME1I6((kwR`l=;W3VvDWJR?HPFbzf
zOE89x>MDJTWZW6NvASYgMIi>leJ3{UAX?F0DNd;(*zQS~W7ok_5HYUJxHu-o`>a7#
zgJ0NU3d@-CUYLN4On6BWb)R`R3;i8H>PCJ6I_S81y1xD>`rZM950?5Non*K+?MJg4
z0pNzFa}3)xDLGh8i`c4W;wxu*kE7s|5f~PZ@Fd|f%LL9nrN@**1ctPYV^Z%9<gBmY
zoo?D73xq_i{+5x0As#x~6fJ36YKE9^((g^wr<aCuyN!Xqfu34)Y6-yL{?vqu4&wPF
zJ@?BB_D{OOdM5-nfW3DPmA)@#ve)LhuE7vP_iWnVz}(}w@{OwzJMQ?k+X+Ec6B{v2
zT8_wjHwC9Phja@;SV)_*i(`9zk+O0H!+q8yV|jEqk7!i)$PB~VI$yxk!Rph;QafzZ
zxP~XZK={6-YCv`9P8@W4Xn&02PzpqWE?r;nUeo$B2_khYefJ*{ki$wGN4hpUF|7~Q
z(m!?J9om~fD?<{fK@TZPl<G^L)k~#*3c>l4L1%KX4ESGS{ZUg-ISxqzS5V`yIjrdS
zQrD}n+7v{)6{+KzwV&-##Csd+9Gd+&yt|c`CM_$|IvRMF_K4y3N0<~k3<_bD^ux?c
zH=}qe5+z~oU++YRvQLjL7>hGRw@aS9f5t$jzT<3XV8|?t(Enmx2oE)P^k1I{0(+Ub
zJd0fY&Nf{gsJyCgjA7zSlE=3H6_?(99O;ihuWPHWfH(n8$CGUxZ62eajFYYT{w=`0
zZA&6Q@AEMJ24;%-j78&;3-l1ieTPqe{I+&O;Lj(da3r81N8(6Z-?=Z?Kx9Kmv-D7O
zBPMa8yuxzqXl>m5a9s;E_o3B;3=OKG+>=fpy1DO&6k+7K^COZgi2Gf^U#8A=5(1R1
z30WEF?j|kp#{No#&=qMxfj^Ipo$dEuEj;yOqNu*(Uwsq2j($H@&xMMJnOIRLk8vrd
z2k^dbn-(rr=Y@1y_u}FmC*^kCg;nT-n07GHp=5=BwgG#$xBJUa94~;mL?{sg6YUCY
z5BfK(Qx!4lAIjlhBt+CkN~*3nhkqg@Zir&>gXlcsUOlEv>xW{Qs^BNMmv$pGBeKjP
zd|_wf4>n;4(c@XUA7Tf@zY>E#x_yAQqF4tiG!!DW+Lu4UiBSU9OHT@v>~5JOAL*nO
z9Ox4WVXePw1r`(L`i8jd+uQjgUL4ZdaZ@=-FJHUWPM)(U5Zyj*7dkoM#*2Mn>&$8h
z1~R3``?Ec%MyrZ5pX|Gh*O(oBJp`x5uhrOvxVJn|{Uj6x_=@G>Cv7}z_-tw!$Qinn
zt^%^~VBsuH<zqi^P+CexgIu6NCx7Jpvpmrf42IO9nc1Px9G`WHzV5gEUEd30d;FqH
zo^^_i)ZSRI_vap6EEG75_nnW;%#Fphd$61^h%|m-oj*rTentupy(2a&@?t`*bz4Qi
zfJwFXny(>Gw=BGKxpBLFxbI3gurE>Jey54k_L*Y5%ZeEt0N~6a&^7QEkQjW>1|W1e
zA7#JXt(F5sic1~W6?Y1|A?|;M5#p+E74Itt(1s`suC;3m)S3MItZO}GyTNa^qQ7Iq
zj^l~(b6j)9w3K*Ih@Tp#b(2;g36@+IAT=*;XWpY!rdB#K_ll`qe)z@qjrWLCqg>~$
zv-F$SGsBc>r)_HH#jtlCwPu?TbC&9bZ8k0^<1{y(Ajk#PP?$#Q=Iu0*$pvz!<q4+$
z?ygnKC{1sXf0Gg4Si)vn`n7*6eLQGJVFnDhE^8%vg?`AmjQ-gL*=CbkezLa3$iW@T
znoXB?H1I$H)S97~jlUZ-0byfpfqk(R-;S{$$t+K?3yw`2iT1jwr6=6{kSB|Nwr0`I
zype^4wLn?8Pso<gg|r{B03+NbLxr)x(@DnH{$HZy4sxH#KTp+Ldrkc-k0IPM?Yy-i
z+e(cCnty+ou_4GZkIFzJRoB&=+j%C1>`kCC0FEcbb#tqg5y5$~GVWY}jbT#?(HwNr
zLP~(-S@?zJ8%L0Ld6(d!+`p#rrIJA$1oFi1x45<J%UT3s!WJ`Y%E)GuLL0SRSGq4-
zrOtYBFaH%Gw8ga$GG?xmU?+q{Fwh73-x?3lcE`fx&|!*%&u#UlJU%~(e#+Zf)g653
z{|Mm$77W4HA{sS#>9B(Ni<G76=Y(l5KmOJIb%DSFE0PsVP3mpn?AXr<Entc7%p_Qe
zDlkvwXVa9|@ov^iGQ;{QkejXLRBm!$p-WVcn@4Q_LnU^V6vG!mH?9eHSlF(NAI2>o
zW#VMZ&b;8a>PuH>a?(!=k@<#JG}P0HXL`QwiVBi&Z%{zTbbt`=)i?R#n8ezOVaGOA
zB%Z9tR1(^QP=7!l9oR{d9)wL9f*_Spd<7Xr7<FUbNg>-*tF2Zz$A_s%j59-Y9pJja
zgNto1e_uBeVTR7@whzKv#u)zo%9<!YD~CS=J^n4Ci1<@n6_}=(MLv_Tz%8LD=th^v
ze*^-W2vtp2Pf34mxLu^~+Ray{*1OFJy<OmHY261BRTz@0?PjWPPsE<T2e;Lk653kA
z90hL9BBwO>j1Xh`9%n}$Uf0PChnb*B$Izsn+U}h?gy=lQ4y3yMIacDJilGB+3}|cz
z;zSksU|yHB&pT!PzDvvYv5~?S4&yT$yfs>2gJB7AmN-R2s?ZtH4qcR7W(8awj*hl4
zn)H$;yibNQq2C^->lrG20&O~S`o!141EgT5SPTZbUbg+Ra$noCe2#Ax0XLQ_rY(JA
z;d>NVmTeP1jEWZfWzG52teS1W_=E0=*wYTfZo!-4M)p7xd&6Ma@Gv<e@0FZM{dje;
zqRu-NL>V-*R6L9K^H^sa7Uc8^l#a^g9R&<*sN~M$KI6UW`OFnKDjhogd>k%d3$hOG
zY0>wE!tnrgn(9@@F2(?9v?!dsT<qTrApM&HRg4#)6>6iUak7X>Z@`Iw)GNp!+<8_M
zz;^Ph&N<1Aj)D-6VV<b$N{8(_SEeNc>k`(M`O=HJATuryL#7^qA4m5&x~AK3Xj-Ye
ztb9yH#-pG`p^pP&GuM(Q^QZnhxVh7BO~HyM1l6;j_lR%U!Orequ;3qr!(Mx0l%1bD
z;}_czwZ`4(^1AyTutHF2_Vr4`98&Q~A^fj!MWQmzX^QeOEtz3i{`5+7s=2hz878T`
zBpKYa%^P=+fCdQ)NOGsvyojn%47?zHvY(n6jGG&xqb6y_Y*w8=`AJsHXSwpUjDhm}
z%oYB6%0yA@`a~SgQzr1xFLrpsug=V~^Xt~w>Af_&K8NwpP=pTh%?bINI>acQo#7UR
z-B#!X=%T{PJ-2-Gs*YIjvTKY&eE~$JE&wD>$7{BWs=@zRiE7X{no)nyLK$-^h!<jf
zfQfN2d{heVrI)%fmmDLRblCYex&gV?(rR^10zHlvn*o!-LPYvxZM*uRpA#fyLT<zW
z(edkpaZC*H?vFYX5_TR}BL*J@`d(hp6;nG4{Mv`vRRROJn`+sxtQ`JN2;@mjFWmNO
zULyFoO(eLNx+c4r7<6!bEWJC5EbCe9KD@{eG&Jo3gItsoK--cyz{`$WcB&p~1*u}W
z)~0y&1LQ3r`NDFR;xOF5vbt&rbRkN@Y&>A)tS~DlP-q10nPhVLHlRT{^OBs5T>A4e
z1G792cz8TSo^k`v*k@dOj&mPf8M_@a>ANreaB@Eqd2!qOH3Z=JBpS5^CjZC=w<SVJ
zvKJX4FSM8j&Pr`Fsfc4Rjym-5IuMVi=n;Kx!+RR7XdNuB2oCJaTkEx9y30m^_W?z9
zq1q;g@U@!uMB2(f%ZIP$K`%MYAY;{vw0QU!4FZadkpAPrJP;xdxf-``ya)#wy8w6D
zKw97rL`OR!iO&^2wCdYG1jydg)TEvBMXzIqc(4n2zKcVp)U|3O;8FLt`;`~hC-JV0
zHdFvb!YXszp3zkX?@8nWRV_S4*n}g%2L=1o8kN4JM1KQljJ03lR&7jq``c2?`M`ln
zno{Ar)*bCtbNaRcQ7arvL=t9S{a$JMF6yixvF~{;9q)4v;rk&~gSA-oa<4*+c0FG-
z$%^rwo$CUaq1sw+TDanz$FJrBTob-_70J%UGJMaB$yLtOneuJYJ><v5%fFYop=asJ
zBUza2*#J!<nFQg+lRw7#(?SLOmx*v%#9;x}8S|H1%_2`YSxABN{dV^6T^lnAFxZ2c
z+5#fk?HH7kXFQ{I<j)ZUC@R8Hjz4memmdS+-(g$*g{F{?<ivmq>Db&t#;P+7B_kcC
zBEnC?C;IcPFF1A4tz`Ih-EJFOjXqaZj^Eej<xTOx3BB>;onwBoA}d6xv@@)@|MLEU
zoxt!KNrwP`fk{+WnPQ#DVgb=?EDl@k*<3rV#->h(K2UL^!z?W31#O*!=mjs399s9!
zT=fp3#xLX{<?i*7UrhB$Q6clnC=V~2nTQK>=<<!+tIuJTK9w8}J79itHAJUEl`+M3
z?uB}p_ERiuYs9!JI0~3q=lNWGpDI|F$RBm)xT>t}HK>$#Zi8yyx5Hd!HL!OpUV5~<
zK1h&_);FEA3^{|=!uju6<n;n;<(hk($w|(sZpdU=NU<wg3I3dfT@1&)Ydzb5S%CBr
zM08mM2dt+hKAQD};FEe+oniNUal9>G+RA<ZcpesYhZiE9Ls9%N-%{558Pw)ii60_G
zGn3ihv5G7z+&f<)&{pTl*jpQxq-nF$w4}4n<D4hi|9jPdToer_1ia_|O~|4dJpvpw
zOWt#sK68cN;Qr-IM*V66$wI<B!oce5OCA2qOJ&y0EPLOvmtM^$i}7#C!XNv5kV}U2
zYQL3<V%g2Jztt!a6vfQ0_K5GcV0pvn*G^eAXNT3%nQuDAFD4?wuUBVFwW}v@H4J_~
zb6;U}&AJ=$Z9OauJ+g_on0auSHb!7i=x=yuI?CMP7O>8!ga3Ny(W1h~(@$#mIcF|J
z{_$`ONblf)i9tR;5b$2y#UA!x-ss|84iDp>U)kCI0XvLnIoEw^*FQ*M*znA~v+yQ4
zHJmd3)=e<l#ed>cH`YJnwEUgP(!&zyz_@DKkH*sI)|;HN8n}07rI+?<@#5O{TCP}a
zJSLVord@nF$TI7g^fKSf1;NWgL92O88>ccn?a`Y_i&3t-0^-?$fY7dGL{x#C3Rt`l
zZaO(+*RR|6riI}gfZV1AHI1+zv}Zir9I&a@MKZRl#l%bGxZ?fCLUZb5(HqTyG)ah_
z9NqtdphgM^NjEpy_hJ3$r%w1;yNYf+Po4)frxNuQYmb<u<03A44e+)JB*U5|8}&BJ
zhtGf^1v)2<btQ=(rZQ8k2ineT94WLkE}B}>dR}iNR+vh|g9!%qra0qIm@k6q^Dv=<
zQj*&Fp3NcXk=5SdJC938PAQgg65jS{F}frZJwq>(G2Rv9X*k+-W5@;H5nP5fCKWZV
zU!wJ`UB@WcFS>h|IP?VI5LjO>^bv|g>){&sZFF1cRBju=$^;nS^lI9`wxsb1xr*8(
zHNBa5jh|+ljqM0`m<=W3!q8#+AK8RS5-y6|?1z-1HEyG0<~ndIfl<K&Uc3D@WHw4s
z=5zgQhnI+v`?#mWe^PKs-s3Y1eE*1Vwj*b?>QHTV7@f%Yu;v7c8ZYceg!B|?&l=I)
zUbTt4yWJvy+F^sYN!&H!GbQH8nRNG;0T~4h39g9*A8K+;(Z0y%yt%jv>zZ<{Y>Tn-
zZ6Er$_<}eQsxYCj_~N-2eG4;&rN5LDP91JPh@-Y7-j~T-8CtK*x*ng3Qur62&RtfX
zs8?EbzMNMmVj8^qtqJ71Y2uy{E?+k->U^Kop31UKx`*V7*!Gcw`~v59<892=n=!q(
z)O*niv+C2<CLr+om_WEV1wT6x#2oeBfIF_=x$G`<&2Ci?2X0UDDO|(8rR%APXAFf9
z4cDwwkZ<xH!T`sfD=Cp|f^Wes;LrDrPTVh|!Zh9!`ZZAc_<o3G;cGe^X*vvFI!zvp
zm{2#R_OdcH26F6mnRmpc-bm;F93w(2(qq9v74Jj;^~^Rx{&Q-u@n}|HPauMa1Q++l
zmMhPgeWJzr%#5ZYjwlGBC&I>{VR#Kr5CKVv3by~GSn*otNIxNFu^JZx=EC5cdC`8L
z6G2$&b7NRru2)6c?@_3kb>{rtYNO5zN2q*1=xeSRNV-TyM9RnI$u?T*LL^lHvSyGg
zKuxFzz#V@8(6<8&B-SMWN}RFNl>(X!I;7{#+Cc%Zb})?wi%;|s)}3F8gWBH>bjMq8
z872Yq#H<V`+K9O@r0Yk;!Tz#>?ZMIy&!aZn&%EDOf4zd42e{C*AzZ3-OD%D>aQK6W
zhIzz63+L69v!+|+f<*Lwy#NQ`KPqG>+ZpOQGYkU-kawj2)9?S`9BIRY030sx)=4k1
zfiXJqIlxbX;1+@H&EZ2tEz*zQkG~QZ(9qHT@8QutG<31a3X(7g4`2h9J`CL4zXSoe
zDo3ijz<_2gu%382H`s3s=*rcomdP*dNsT@U@+WVFi`yfjzyGTcPBK8T!6gKqBe=FL
ztSEmac>LxcFrU*U_?Ms=PD73;H<n|Qi_(AuumCaOHeRI!{_sA|KhKP4D!?=259p9o
zhzmPheANGY^FLE1ehrfU+&d1MVP2m+G-rQ*w+Q+hyjAT#q?u))EGh|V@3cvg^bESo
z=qaZq7a2+nqaj8@H?&l%QKY^XN}Y`PT9yC|(*!h|(x`yB@NJUNcp)n}*h|$43^d5F
zjVG!ozk0@p`W}!=k7FC=!FOmr8Wr<Cd_?yo>kCHfW*o7T@VxxI&cyI3`pTwFagt!c
zNnG95XzpHJQkf1%QDTz<I?5e`)FYFB@C^<taNsDyBd=@(l4-Ns!jrnFRU)Xu^w`LR
z_%^HZl&8kLR&_#p)Oww7ka|1+X~$yWh(5v$-e#oJkY`jKV=Gqt;U8mR#*VN`N#64n
z-L2a}R8xb(UO1hc+s>Ijt|hOEwztjZP5G1xX=o=*f&>Ljm_`GVu$kGHg0Cl#O}fL)
zH>(GVE!rQ_c1Ju>GNOYna(E4D;e0L$^mA2<JOY=Wgk?B&!ms1P{2}@-!mgWL>fc7<
z`TqHih$OxFj*ykWu}?DW8l$Fcb9L^hg}u3u01it3R3F?ty@hlF3NIS5mXZS=%$chW
zpRM9&beN{I5Ckc)0t7wHH!Z95&VMCcn@~vIl-TwmMh>Kn&B2>Yy%@gb@Y-H22ip&i
zHWXg2g8s5JN~JN+6)(;959}ACL(kg?d46rr3u0U0J1zFFM=)kU)$P=B*YF%g;-eRM
za19ml$K!1nNU=@sFjJ{|^-i7_pH~2Uxj?&jnv<3y?!RPZpN`H&Dqn9bb#HlLpQgL2
zUGaSz;o_-(Q~<tncH~i8lxI!>hp@r^M--Xo;(Gn}od#5BJ-S6XsVe5$db2m}2gB1D
z{jY2W6}frL$NUcz;NsGa@9(JD_Z@dXYV2E$UMF)&eEZY-q9`ZBkhF=T<r-}Ni)Gb)
zdyRt}n2jFwuHqyElR}9vgC_4A9#Uw{jM9dgInifUk0#!E^`l**Zhh=sL}VQN@Cy@9
zYM=oK9Agb*=}xjh;CSHw7dVvP0>}T)d4~4!qCLB9%vYv{lB1-(ymeFb4~doKKaZfB
zP0_;*fVA14Dli1{?M2s-$YMx~dpN#*gRjLP6p@4*sT`{3*Rnqa$ukl65Qk3Pva3xF
z1tejDf8KxHY99Xesq*YGVeM1a_>OPZm({8w-?jiSGxqha(HA*1xVe4a^5oAtD8~!s
zp+ZGgsv2(-`M(}?)B*r59x?7XO7UwsGRX%0vS(~u+cT`k=nF9eM<3k<lr~62Qit@<
z^x8=A?9feyC*Y$7*qRZCgQ{L?FrZ9OwZT1QjCn0k-sR^S5VU8LOCHCG(;m`@g1S?K
z4y=3UDhN}Wi@v15oN1Mei?{h#B9e-ez&=4>v?5RKp8nlKC=&_sI!~qRjo|Tz$xNMZ
z_nCb(GGXnFpT9+1e%AADMh2w`NZ`HM4k4A9$N%%uNq&pcGBqwA{yD=u{j+VjMtMbr
zfOvh~hCnNY_}?YVM`>_Lm?9Vfp(I9>AcP<qSc;$!vY^7VSgdz60`k!efLnsyD4qbt
zE=kbOLhql2G&CWQ@{m~aVd593<zw2$w~nWd#+H`GR&Aa{^Aamlm!q23(qA3EHhL*X
z<I>0SuHhQ+7&%FJLyMC%LMLyR<F24Ur>5C*Lx>}kkM1)oJC?4H;b_@b)}(TZTecQN
zogx*Bj>gR8GtC~#Ik_nea=+$U5o6DWXHXwfk0ypPj#88wp(ePCV9fQj^5Ppy>>QSz
z7jw0GsxGuW3k{^RC)T8YRIb_@F1yX(e6E)^Tb1Kos{3ZUob=7rDrsNUYWHB|La{*4
z^;<;qpS<HIG~1ca>Q@f~<a~n_?@PDqH~EIbMJ}ZJdJLWqlH)hmh~ihb5BI+k_!`90
z<BVn`@_9WAmG#@kuVrPlJw}JQWreP)WZB5LWNQgF?PN~1z0w|^mgcK|)Q(d~mOdbd
zA#PSJ*WvIIFsi{LabvTXJ6utfi`J}PDNH<HE3j%~!oa^CKO}c#jF&2J?5sycEEXL}
zn9KhVvwUwAV_8h~cwAjpLcpTLHljx6g0RY<l^|Dhg3uT?*Rm$}LZ)gYwf+MDGM850
z$1!BCe+(SN=gPja8L4Kky+pDl80ZitT;WY3mQ|a#6l|T>F@a9jo``btbc@__@56k8
z4{U@6!I=bdcoF9Q(Bj_E*%IDW+{srimgppq=+?6BX12>GSa~@D?TlY$$sy56eCA2l
zYXw4KA#c+OX?`rNKKrF|*{Pg({m$m*s<6n8bzoFiq79UjVD{y|og5ov>Wh%_7yY~y
zd~^MLkUgSpcsN*;AuHKcxw3r{g8Fdt>V8GNudu_}f&$|b{ThFE+y#X>@b32~G@5+4
zmd`q6qtPSvYq-$BjltB<>|@t>=);hr_u<zs-*zNtQ#pqj*UAZfy~}N*{5>N-#Spi?
zsGQI!32LV<_nm@@1u~7wu`lG0={z4JJzd0g=-Cpi)U?u%#@p-pG+-fh;C?cM{hrcp
zU2KuFY%KP+SL687j|QI-CIDe3LeEEmEb27HRmZeahzxbhtD$iaF3p@vF}?muE8%U+
zO=(*7;q3_pp+F8^>{;)s|J&SoAZM9~r548b&_^)_JpaB=3Z;!-V*<85{p*8{KWH9@
zJ@~E3`(hD=L5rx6vMue7GFkv)sR9)yM2{xu-OzBI@0PFXT8tzUFL`8b(cx0d?)5Qi
zsR*A>-%}x~s$@VIp2y?xPZqXMDT_J?JetGivqU>dq!445=Z~|Gbt(R1PBf9vuJ9o{
zj+bH&BhJiyxqbrc!#V|@6M_4YL1AFxmO7b>HLt{ET9)oq(N_&*D4#if6u-Y)-M4(=
z6OyQszAP&vJB1_QGl%sx|LTIV{&H$hmGqb0nJ~YeDvy2G0S`Cc{W@F4^{iG01F(1o
zsV|bVUAkr(Pw^_8!PxHUNpx!cGD$sF+msVGj$#7Dw)5b{cQXopC<-t#RNVGN=BQl^
z1bh_#Alg|nLA{&dC(t})pX<Nx3qd!tnM{zZuXbH_1k@WErTA5R;F*=3l3Z-B_Gbkh
zxtS$O|5#Qc_H^=Bv^_UWgjI>Rx^buCbH7Z(WYp43mfb&GYBM^Ls?xiFYSG>pxM)3|
z^_Qz_ruRyHkRehfb*B3*&z+xy2bF~cr4s2zGne%pmBA4SDyg*McI0-e^xkaR;W)9)
z^O3wleF@eFt#)yco!BE^h_TkoFt>cgWE9NLQBZoRI+WPNtB@7rd3!y;;AaMIsCJhy
z<!((Z>RnSN<L6A5%yfDG@9;!J82$;hUibtb1p%K61zA2|)08Of>#b#5A_fI4m_@kV
z%oiIO>V${eYFL|bC{T8a8iX`rF#3Lw<m~2WZB@l7w4GW^Tm7o4@Zzvc&|~DgzkgAC
zJUR08V>Q0qG;^|X`a*CY|Fwzy2^I(WW2znDLW7HmSsoS2h7;5+KR5<z<~6TIlmG1k
zt}a`w;S$ny_qD9@T7skVYL}Vxxc2%3k!=W`Z~g8&e99@Rb7)9sCGdjoH1L(PN|Gwx
zSpPJI3q~~28W}1+WhQ~HG=0wEylt8sSfFc(3)Q~`h5`%x_I-iz-vUwoEl^eF|6E|x
zta$<_AqM8P54^>Qx)2^TAS_br<zAB@zFA(siTQ9_n=hHYHdNNCNZtCGi%nLGrdVuj
zg49=|P`FnjzT~%-+0p>;L#I3>LK&7tDdr!o8dR3y>WGU3)-#jQn5!J!2w4lZrs8vr
z%6DQ|rbJqeb{z<5Ps`@NFOe`e{II(}sV?6tLqHYJv4O8<9C!W%+zGY+ajPi7R)x|@
z)awA{Fc%qeM_&~wg|vK3L(<n9a$`bg&?a&f*D>@hDw2`#s}n&o5J|qhI3kMm&zzPx
zXvk2G1C>N?i|+<QmLqF?0yW&85y=15?d6_R$jBV%URBIJdJdMw9t3>BISEhQG^k(v
zC+}MOUK0x~3z1bsoYrKjx0AU!g$R>P+-zYXw;1?OpyEiEuxYGb7=jmrrb#OF8x((z
zY0~Xa79>WWGD%?)4Fn57*xYli=_mzm!(Zjs+rF=~E7JVkd|e9j5u4eL)afN9cj@=5
z5w==nN0mfPYQg%*q~U#c`-<mbiV?#^pTVa0hf8!yi+q80)I?)dnm!&kvw4=c*9l62
zFm#zObS8OQk#mgq+*X5@`2Tr4eSu#`?5Q%E!9{(`4MfIBY4KjO(YxoDaznN7S=*E0
z1z`z)8|e&$>#c$=vAlKPTLko-e5s64Wtoe)<fyx7?6JX|1D%E*T<FZyz?i2PddAD*
zFSRZ@_jloh8ZHsf9fPOTauT~S@GkHND91FnJPK^{daV|~5lLh>Vw!^W0iMwymU`L6
zaaU(96_4TCtk%FN*jZ25m{uh(OlKvM%_4UH!{edFwU|S^8<ER={aP}~pRQ~c+o`nr
z$D6e0$1rL0c`e;(iBXaY*DLb{4u|OPV<}P(=Be9e<nup2e^ShtGm^N$x4kE1s~-Zn
zSZyDslCF#OxY5ULt~OC0p}jpLVM!oWD(Z8mzwMA_k+)RINcew*{@cI5Ry>26+fHO8
zfN(jj1PdAKcYD_Qj<r$&)#rj>d*6~t^4D(0)!}f2A^!IENzQfMWAWFhf+qHODMmSQ
zMkJ;uhs|Bn(^s)?yt}HX^p|d7(Z^Ekd&i$$*<uwx6GtQAeLT1>2V5u|hNe9f-!h=p
z$5%XGh}pmAt7;?W{mCB#SD$Y>9jXe~gztAdVT$H+^9On4$CRb6mdb5~(#~-IK)eq>
zrA;($l^jjup$_DI(R?eGcHx=hZ_F1Nr!;P+if6vCCsbYk-roj%G5TYCVz*0GXZNnn
z=X9u_wKH-ON4yHWUX>QtQmmiyRJ9ppsIJfbSswl#S{VxZp4eXOMQW(8IrIp(Ex^tG
zLNYv1i7%M`qi27=9f-kNeh3oS(23hZf|^ywDBVGt;DnSvotC5Cw#6D9c>l6DbbC9v
zXUwHruYafW0!#5#o6E9{3GY*xVFPPJt`yEPX!>=c_ve|APc@N!*VC8-RNQ76`3^kr
zQpDSVPqzQxPrd?UG@Fi039yh&fYT(jVsQHe6YAl|?dI(O<uBxNfS;$c#dx+r`%1yl
z<5=4Sg03ILPuttZl*|76>fP-=p*3pfHF}Y=eZCijA7K8muv^ZADPwV_In4|u+S#$e
zF)^>nzkZopfic{d4s+{ZosJSeiEy`vxW!+LEoxOejVdp9+o~N%P;p1>TXDSQloS`A
zu}WbftB)5`%5Z2_v(`obWQ~_j>P?tr_;e5$Kd*2b|6#z7iIT^wMy}c!ByA&lRqq=O
z#Nlhcm$^TjTQjt-#kSpOGH}J<KCUii7K4jFUze*zC5_B%7W3J7*#{Ue)|cw{(8Hnx
zUX`q=1WK&<1gt<&y2wOO`e+f+!4!Aam2OGNJev8vl`TaxefvkpFfwMpYyUIwE|?Fl
ziqv$hBNc47Mm`wJX3Pfo?*`gOZOA1|Eu9k^`67MggpMHKD6H?8kGY&}wlz7$S>%Au
z>63_W!1EAFqVWr@O0Z$FN273L;|A}5MB(CQnUH5vOZ)5=2!S_rVr^Lbmi42Op=pMc
zVjZ7S5hG~4o-R_hXszv@G(VNcL1zOA5;#(@|Bh6PGC71HT+G*W?u#7m)g72WqO`Hi
z@(C!(*>SwPc_!O|?&(>7uMWc`4>;ufjr(IhHgRk2+W3%BDGPw${A~FJevYAk9SZys
zw<JL#@n20Qe&LwcFH7TPT!)v-(<tLRw=<L@1S%!^C>GcS&n5)XC&<$5**R2oy*+c$
zSF|SWaO|MnaOI(aol~j{!-`LI(==x_6JlbA)CsZjsw`4;wec~y^Agt=A`G?mu^z2o
zm1ad|K(!`{n=jWRTb><gy5Ei@sty##YM+n%Z0B`u74qD0Errp(|EO`D=$ql5WlW+=
zY$Oy{qOG2OB(HOA@VmL%BrRcPw$b?I%ZK9-m6=qH(f16YaMI|*NX**@1Pfb5jN#9M
zFuy$R{1cy#P=p*N^{NRC1h76IZu~w{|L*>5Iq)7@9yK6i|3|_*LGE8ae?i8l%qB?6
zWU!=MF@>=@6QLYEs8K4mX4ZG;+Lg1$P4BHrvq@`_q~tSA7%U=oJtIQ_#rUi!LL2O>
z)_$7{BQjJ+dqYLI?J`WEIqMEQN%vx1Px%i6yNX|Cj*Ql?H>g`pUg0&f56spncuup&
znp8gGAo&C_!O*?{q4gA=N5id;$5&0}|APcwMF~Ip<n#02|465a|F3p7e|y`ZQ?9R?
zcVaGbCvrCjOiJm#XD8CZQ03hKd^pI5+Wb%;{OGu}cPma_3gMdTLKwXD4m!O+d@*=Z
zzn7m@{vbU{%UjWWXgS{XwQG!*@w4kEL>C69D1OEN3e2eK70CsUw%4a(g0H=he#1W5
zUt*xfY@Pix9?o-bzK@kPk7w1O7U;ruo_C4W2tcmhQbEgCG^+Tv-+3I)z>V<HU5(W*
zHW7zhnExq9R{1bsarVSl_R?_|@<3&;P$9|8>t6p#pH_mKT0&|k`|K>Zi`Q_F+#MWN
zYN1Ef34|Q@h<DV(VDaDU?<e{&#5|)8<7_A=-3%=71CVhkqTnD!FGHE6K<D3idKtQs
z&W{Up1^_3XtloWY&s&UYl&C(4VYOy4Z`)<JTR)x<UdT=EcfYu9Vryw9v-~}J5j$sK
z5|)T*=sp-zQ8b_~_c@@05`oO#kn61308LU(QZH}G<Wk?R-Xlf%p?~_V#K+;f-8;>0
zn_)*;U9#J5ldxTLj9qK`)L>A@BAAxFPDuRGtQ2Nc$r?rQ&wd$Hs}=`btC;kFo7ju6
z#jEX6!;||FH4i=KTq%39(0$4vbTd`ou{Q5pb<|_p0z76ZHxVkQdv@#rCPJJ@nUji&
zo4#s+^+2oEOXQQ!Zl6{4_;A^1N&*AmD9QdoOCRBSnlg6i_s|>fVVTmQ1p#=93K@{H
zne2@Z>r>7*Pp#v1_bc+SiJr~WmkAk)mqOE;-@9rVl3N8?5~A8=xA;kikKWLxRu*}C
z_AvyShfxFHKs{ey<v4&Aacjh-*d8>^AHZofKrdHRkqr~qF&~FAc#n}Kp+NVkj8?AZ
za?I=IzKIyV(<WPbxlQ18gAW%zs(dI8hx%3`&dH){3#JMH8H-Wl0{&En!T*)Lm?q!~
zDMr~tb-=#8LAbM57lvur=O@#=&0q6SV_$v_cV{GcKcWBX{lCk#cboyRK6iiSniU_R
zY5I<<4Mu9D+L3+vNlFvrvn~3n!Uchk@yeD+`PD50-yh}z7TIJ<e=GTNMj#d*cVvk@
zows)zk44kI9WKeWP5)J3YUGH$T4e5G$6x#bZeaapgm<<x<oPkM>oW+`wIK-ri)&7n
z4C5s0f#g5v-*TY9IEikeX6h`E9H3&c8~y!YA?y1^XBh<vk2R`AdTLu0EdSg_^wVAL
z6=Iw%W{UXoPZPt-Ie5cD%B`R{_T~WpHIJ*od}$$irU1$BI~QAgAc|Y?8^nJSAyGks
z8VZn(9kEL=yUELN6WkUk&g%+IPVpf~d}DQ*=(tIoRNcqTv5N&D@DC#o4iZ%vG=2FS
zulDoS4(*<79RLW}MKA;NhL@QC_goiPKqN%xr12H!it+NSi6NCYBVrkYiKnOW%k{gl
zPn0K<5_Ec6&p&Q?JK~C;Zr8TO*vCn%g|fccyXknH6@~b?=WTs(FScaYda(DrhEg5B
z^{{FwH_+%(baNZX*6WcFMLKTz=JeZnc5Pt$z#U*&$XrevWI6``%6d4X^|@6xDD~?|
zAwNwXeU*0FcNEC(;yS9JPjfyE(?J%5zxnmNwnBZsxENVtjiE>gL=tgO=oFcb1=%@S
zv>xV-3-u@yG$LgskJ~gwdOC=soZM^nor>~w&+=w(=;xFo?tVqz{ttR%)cSlM5L5ms
z)jclSY}1`Nnsp(rnZcK#X)lp4+-E0rfd1dxI9T;6&#;(F9$_umWG3A8T3(F{hCIGD
zGkO&#Q3SGUH2UT_tpo&1^tr72C6B-TD0=gncG0TLxuwgmkjSvx>*pn2Fhx{b_0v1g
z)u>s|b*+n!4$F}TEf32Jc3)6F>T6Rc7Qk4n^V}A%xNXZ}c>$-}g>r2LrgnE{nUVbJ
zK4y?q_m&G{x3;*slQa5>@N^vDGDNDhyh>dY+qAvjQ6?J0?ixy@0NY|}$HT$)YgB2J
z)m=nD3@1<;_17LW5kJJpyJQX5d#$@<*$K4h!ZZdO(CsP%H&1?@`yM~G_ADPkbMFTh
zVqbA-O67BsJv427JTjxbTj$YWH?V-Sl|B?!jthF@8lr5*2lz*`#@K%+-S!*CYgW=L
zO)!pLkED-np~jORi_Yw(gkn!V<Zg}P?Z2FU)xYnU(b&_3pe$C6`SQHvx8lbF($rZu
zdV`3c*4XR<?+>Vr!TgvgdvZ}snE=$ja9+Z3cQ6H?Wd$Ald;>+eZ=eXL;F{!W39IyX
zIqxzalz8%d@iL+$X_}ZNjeYS2>)KJyEr0mn7x<0dqEPpxD7C!jp9-zH5h}j?Ezw%v
zpZyZA`L~KQu|wiobPghq-YcFzP2^jADI<cFHS>YY-j8cNfO-CMXtZ_#q>G_$(UtAp
z9KQQHJ;>chqcP8W8Oms+c;&mUaI2x_vYab^Q#x`5!=mKKJM?%MDZ1wEJgh4FGi9Zo
z+fc~z&1{y|>vkCTBpLH*A$+m-VJCD`-;mEJi&^@9dYOI3)v|Tj@QcOOq;8w1pQuf4
zyld}*Ta!81EN*Q?oxM2UaWd39wXr=Yh@BmA#p_MHKrS8<swKvdE--kDeQ&5&c$f3+
zKwVG%X*+P!^|wW@G>%Jc$`KB{a6G{ND4+K|4>a*8i*dTAM)hgT&76lKgK5&~TJFMz
z0p^!Jwt_I=C*u&+-uJ5=j;9SRJISe4jlYH~DBI3hGmNV^Lh=3{)css^HdF-K(mpw(
z7K494r^^zQSapwtx}yeSF0}x6^If}M0SZP_*Jq<7M)6{ui-uM#t=&~q?ky@!^?FDb
z6?@w{dq%QArAg=tur5|HDaXFH$bXL3Yj=O7{%yCt&8k0F)js07!m+-rl@<4Nb*xZY
z!Uo4XRUdz(@FiAmZNY-A=u#!Npz#T3YIS(e9}@DyRrJuucBg!*b+htJEv3;_`fLfe
zLSHPu)0?|`g#Qi6dX^eCUg?(_7cN$2tvMFZ1MoE#fUh_;kx_iOR-GSuNzQ~HA6#J}
z53XGu0yK+LDHUQqg7}RUvlNik3$C=86Hoza%`sU}>pe6-=|j(3F^u-V@<<j5iXYbh
zjwh14M0=4b#!1!kLZMsQ2S2Xh_3&fj0nRhqWx!Br4|?aM8ZuO26t9pgVHGUAW82G_
z(i%VZpdDc~s_3jGlW}|}ZBv!nE|X1ts_Lo)uJjv1vA93U#60$qwCxVVS80i_N+zC$
z!d2=35J*<(&YJs`PIjd(tWO3CaWv`Ilp)h?Jyb-dgM3w`R+1+#?z@)LMqfPWGssGV
zd$U%9v-7K|Eb>Jtq6h&9r6E>;<4=8lwCbq{+M?HxzrAlCjdVzftaDZ&Qg|(a=?#ZR
zZrophxT|ohn9WJX;my>&?kfohFmR(e@h_dZVhy8u_nV@b=8bK<A?O;0P^+0ZK^;Vd
zWct`LLju}_kg^B;LvSQt#vsQ4t_Iu$Br1d3)ja=nd|!w%m{Z>*+Ox$g%0)Nyr88|$
z<3BM~76Zi^@I-}}f|Em8a<$WvOf>%}WL7Q5;Pq!r?aDh*lTWP$mhX<$hi)kGzk7~B
z7vOeVG0Gnh-GAlWK2rMV99}I)<LkY03UPQ({#9FV_OkHHu>*#>-Ol*w##yreF4Jmp
zB)!P;3}=;u6_(wXxZCpUCr#OJZ(DQr+1~cEb^f8kgZgL0*Pkz|0h%K&cG@4-P>!Dk
z*Yi-ets`rd0EXx#{dSFR>=HRv=RED@uD-8Edux&z391l!K`HwOC#4ta5YFlG#4&%Q
ze2JuKUL@La_b*ZdQM~gP*3xh7ki8_jNWZK%XiGMh4eC0?QdSj4d8qBV#*i(`7a7YF
zeV=iPq($(V=-<)XQ{d7gBr=v6YwF!#UD4CL5e$Kc++~SM8*S;YZrVfUsfZ}K(U=Bc
zljli!izA<5U@RKgv+J53>>~-S@$dJHbN<esv_=U7QNfmanQIo<vnTPB{}mf{Kfr~*
zt5y?>Uu)8BmmB*HlJzV5UZ@=c9i}@8Lky85XBXgr0GThv6rJ!9?l;b%U?XGxoXn3&
zCEm63Xg16%k+9F#8x{zdFTUyK`SDZW9pqm7OJo!3$QPQNL;LZcFE0gjgVpbrl7Bon
zoqJ2X0k_7Le5HpXQ@h)Tn>IF&%CSX!6Y$+`Cm(Bxs2OJotWEP#{V#TD+btQ;wXdmj
z^%Ck#y0i&Hl0dstj28}oF3G;_lp!sVc>lN?|H0{2fnw$7kcxwMsHfImbCggGYfxR@
zJZ+jpw9fKQ_lyhiDSzdIeERDtgGM<rmC%}2j0pmVk$JmoIBMl8zVUregD(zF7y_Ph
z64~%?-C&~sh0jy<^<O~yKtC1;m*%VWHIrrrVDDI|E*2i>tS^DQE0S%eWXgt1iDG*d
z@rbTPx4k`wl+4|o-2p4LyQb?+EUVjp4Ic&)oUWUU>SvTFQV^<55eQ6SRW0(H<zKNW
z)XV{L`jiKYsCM(fVFuww7Z#Fw4RYt{E?n8gu$7d#3Hv<bFTr;NOs~`OmFzY&sIJ^(
zgS(VEc=?2a<462~8H=<wQS94{!fO&ORawP950)3jt*510i+k!Nl6Y;$9V)9T9}BT>
z@XEKXZ9hV1_FM6KcrU9w$yy-ceygvxtP&mDkt}*iddlmpa-g}1Tb*^=l+Ix6!iBDQ
z_YhupDR$S{(y<Ot2qL2)k+ubzZVu17uOf6xYwj-b40z1y;avN9`&+H949v%!S@u0=
zc&>F{>%YdL`*X7PiCl!*4D9`0-$u2177t+od9w(@mcIY|?a&yu^|KDXb0-TZ%-!n`
zN<l~^nt@31f{;Cq2{>p=w!x_c$mA>Zc|c$54@ar-XV4o1@_i(YV_Ft6&$`YvGbKuZ
zAGRYP(Et;q4*GUz?v3_&$)x<^U?l|sy*?8Nsy^`Q6~y--QNAU}P;{D#HJ#gh#dg$*
zpY>yA)fh5btF87qRf`d`q`qgJmLJljW~Yf{&m4bnXC*o$YM-e7$wJWrKtubHH!s$~
zp#if)sitqi^FgQ3Z>4iT+6hZMXy#2w8;H^#q~@U8IfLU*Ym79<P2BQQI%eyA_$Z4&
zpA%1~t(ceSox*xw=>1uj2jSBV{SS>5%GSC+>xBOi9A~4|9qDiMG%V!c`Fw+%X>DRJ
z$+-CsuMAcr%@)Q*+P*LNx;+(m7Y`+Qo}2|QilkgwBm(d^mSIWL&gG&mj`Cc8+B9HV
z!d}5ukXiWhuB91eLQx06C53KIJyclPcHhUN1Y}#^mFzN~j@R?aD|1pIR{@<#a{`Mf
zIm9($W|fL4?>7u1e&*B_2f(IbDUDy@rzXRL_20;Vir$>nh%i$Zdj^a}PYmPI%uEmQ
z;DZfAcW9B87%>-3P@#_mbvAk0=3<%c9!4)V94GTJVbArse)!sMdKw=vzES-3MNLH8
zTfb#%iZ0>Rn{77V9;N$bun1ED{>Qax5jo)<p{@Z;FY+*$AD$q#({vOU8tBL7*t6c}
zJ$8+m!pUr2y^j#k-HYg<E8jGx%eAwzU_Qt*it7Lg0hrCH{n_9jlx6u3%JLKokYGZo
zRp{VG>l*g508J!|6Ia2&yvp~`;Kg0Dcxd)G0g|L_{C6s1*U^wtY#W#4FKM(u(pM!$
zAG)e}>&PXKwCWUB1X2<z9mz*A!%5m=ZhdIDotSi8c+dgtDc!W!0Iv0FDo{UDIEmD(
zW7_Bnr{{xjdr+E`LlV)U3o_!CZ97~%Z4%G;OZpGG?r}Ve=KzHs4WKWQ$=$ATAcNd@
z4elnWN#IGDzOIvQ(B9H79B&9%E;4$j`RhaIH8Z2wFYh_n5AUI~pZL$N-g-~~jljKQ
zsh)gb&FtImjAe-)>{klw^OfAVNM11X83{_K^)=Xbb-&Ps1W8bL5Ag_BJ}(%B@H=v`
z_c4_;G`De6X>yp1=XOhcxfHN~ofRBC4zhccGcF&EU}cwDi}T_O$#UpZHI9uqD>Wg6
zL|P&Rq=R-|Yo23pCW9Q;?c{#E8X-Q58^YRzQ@k5L^<H?5MH4I}&0?0d7z->ULl+4I
zj})^ngdj)cT0K$VJ(jSAcNQ)TD^3-=TCi|_rw=A4Lsqr3jV&>uC1b^i$TQX8xAGE+
zTtozJ#$^XTbD}F@B@btyr+Mv+ZK*|Ij07d>^i$os-yv3RTVJ?L+Rucwnw9EgZQbl)
zUNz7`8P1%4I)J`Pv((mRc7~G(&G6j>TNoO=3p*DB!Ey1B|1{-W>*w~s^#6nV?o657
zhERyw7o|)OfG?f3=Voc7b1p88G~T5<HDL;7<sl-E+OYY}vvz~E_P54LNToEHuqSKS
zYI#^6Rq7Tpv%Zm;1WlyhRK4k(rZq@VhrMI5*BHBIV{`yMer8Q&7Qxp`)V*dmyx!xi
zvN;fJvFNY!59Y>U=3v6(5CPOav&s`w_m(Pu5JULX{~z|=DyXh4iW0>=cyM<M?(S~E
z-7OH@-60U%J-BOdch>}hySuygCiiw%b@%&zRj=Ot$VVz+AND$P%{Av3V~(X4uI5Aa
z*#~OPUq_PdUzD?6dlD@cU?5R|zVOSo_`tp?ClgGH9a7VE4Pn(q@N-RW%2al%SA%t<
zz6HcFhhZ38a)&~{J#b5V&8*A_k-@4h=?h|YL-dYhf;#7G#E3R~=aWS@0;KXepPlvv
z33uhK?vBi`Sh5Lo>piYHP}Xc6hfvx#!?3k^jNvWhM-(e|<gN4Yp5Zqhob7w6^7;l{
ze<*V6Ci3%QWIW9!E0By9EhKo{hR>WsvBpq&FpjArs`hFH05n@r6x4OW!@7*eiy(tB
zk&>0FdU+5+sdRrR|Cc2JMHEcndcBsiYBxI*Hg_{&YzU|lY|^1giyjvJ*c}?|Cg%=B
z^2k)6Src`TP<qvPhZEE#-`Poftw~E{M&+zHkysokP%>(swlhC@(Fknl!fP6B^-w(K
zBdHhm^uj_xfdm6q#_4E#uRAtn2-q<&87hEq4OqtxBI_r0t%4~p+yW6d4iQv>@?SY>
zEX8sL@K+o%^jH0MOOnG(lf6R@Cjv+;rhpxSYCSw*c>g8Jbk}{85i`Oz{4}`}$~nxY
zg|C~o=>L|ia7dsEQb5T%{a&&@-u8R#WdeNe8n%5FN+S-Gzd^s7A_<rMr1M*F!?DsK
zHhw$~*PtHOD)r|-`3PVxo#0>!s9~2QT9`FI*0di&0J6Mj!r$Q5R2?=Xy=4A3#bC<9
z#M=EkjY)XN6;~!26m&AG7tcp?mnD@D^*99ot(ql=_oQO)JV2-m=Meh=JGO}FeT(^2
zZ&;@;m>E=?p;`w{K(*i&?esic{VuipXq_)abu96xQ&Kux2xNdDC_&s|Bh*r$FoJK#
zq-mtlFD?TRP8tsum@n+rXF(aS1C-wI?{4KU!1)`}WcT;{mC66-{M}Ox6r5zHfp3#W
z#lr&kRocorB=<^QR@_%BnQi_Y=X&zEboJ14oQ-&xt097;i3?+jtif_ij=%I_FzUs`
zI=3lDFN4^x<5a6wld`hg!GQX_i;pH+c(VEu++YC2OygJycG{(4wH*4lOrm-3)Cs(^
zY)@B1a~Ai%fH3!6ra3Vvsw5Av_R|yX5&g9GM*lM7sq6JzkG+XPYm|{3z6PiCLDquz
zApuCMW$^}3Ki{{<9S_ktUMy7=*pU%FF$2H%lQ$nNwjrq^vJ!fZnC50Q0KGo1T-jA&
zNe!YHS>Lr(j0jc>(fS)o;^+1Zu6uHqY(fV5vCY^~;louDqo#UncL`zN&wqdXoipjL
zN2$`R!DT{GFQC$Hurxl8MzgI`&26R)EclXsdV?e-rub?<1`Xy12lHGqbL1{2<+NKA
ztdw8LPXr9K@E*~5LLdgY8t;c#ZuY+`Ef(07+9`jO^1L^Z6<^fK9?R@p_IQdIcTF^b
zQ>bmYsM@qMglW0%&TF_1n<z<<Ahez-c2plT(CpF9EmGNj;L$nl@rwwg?53C--|003
zAXhag4F?*QAFd8WIo-c+Hx`VoUH*o%?uPQq--LMt-3+JwSJp9NZI%ZdIbTwMG8d<#
znaS=8(8NLuye{v32;&7Af=1Sea8$_V*v+RHX*NHiAGSBPIcPUDGho~CC*%$V8Z5uA
zc+uQ8xN_Vko5!G&?l>}Ng2|gY34nw>SyASw93ndxTZ*K-M#Yker_p1C@j5@$X0Ejs
z-n;?W;-E2VtT5T&*!&efRp--TL0ttU-i%Wg!|YgwoWaE}VNTpq^@L+Y=SyKfyoZBn
z*}Mn#1mA0ev-Gor?H<melF=9;sTI(>JNI{ikI}zf_%C)CR`qGC4Pui|YE^kaKTtBo
z{<gz*FDciWzWTG1zip?yoBKGj-lo_T3%iB)JYwBr1Aon^@Qt+|XrEDnsD+;!CDP_%
z--%tT!}MtWctCV==q$_78yZ>f^*n%>y4Y5=nvf<H91n+WQojo=Bl-2jcb&N{i%}73
zSGvm{`tCS+<4vm*fKh1}*{elE|2^KX@XpJDwXTr?j(3+cU9J6Qv_KG}LsIH(0ZjUD
zQV|xWMsnQCZf#0liEj!Q^_DYVPc*}}TFX<Y+`)tuar%`8zB3=DHqW}8_&{qcb^U~e
z#<B4_1h@jpf$t2Lqr;|)%YdR%*5<cNmH4H#(cu-xEzm9uzVgYI&~}Y^<Rkh$!_f^Q
z5hkvFJUkfxw~Zul`?ls`i35na6d0zkyOvh8vbX#y!E3Rr1ysUR3Wa}ww1b)0;U?ZO
zVO1&a{cOemMfoW3TO-($r{c16xx~EKutD6wpMJX$ty3~7W5K&M02`uuI%q&-zUjxz
zix})65lG%QDf+L6zlpLh#7+d>Gd|J30J7K;k)7CVSv@1Wa(BnewM^=KaVSR5x6f;z
z$zyBaEMe^2I<`3bt|bsUw9jaB^`~O{_FkN|-`Rrv*L|CSpqAO6O>d$L+`f_g7{eyW
zQ817TYm}_Vm9_20d{abz)*_{B>U+=9NO38sA>Hth%I#(i*ktB!{CRWjM#%R7v}j`Y
zXSYS6S#_U`Ougg&r_2qvnL3c*Fou=SXGLMB&QSvo<|W&%ROT@2yZgz=24u!a`jM6M
zZ&V}_WA1;`GR#?DcFs`#!;j4PLb-{*OYtZKxd@i-7-l`s`iT4h33c{&`eu!HgwHjB
zIwv**ph|tJUI7js@+=!4BPvPu5KFnb)Aa$<X&-@X@oULyS<cr0R5PHkxEwe(eVqDV
zouT(pVm%-hfOwIcQpF~pQb6x>>5y)x>}482#}ar~x6nSUk6T8*L|O%bZXF>B{Hf`N
zrbSvrz(%75lhb~dOaBM<jl3>p#3l3q-KQ>fwj>i9S!DopEYstH*zFGYvO-w`UfM|W
z%nHo<u5xijl5&9{+;Q(}6*nW>VL-8PiaF5<=oPH*D*&|u?<#wD3QGBO?2yXL+SQ{=
z>Z+xm4KCEy9#q3su;L@2BDg&4d(c`_Y?qieD5`C@2FSmkPKv_v0bXM8QU~UQ$pA-y
zx}U#}_1N#}dF{>Zxf|i6$u49&I1CgH<-@^pML?^#t`V;@KLcnTW4=vPy(7JF|02ER
zP@om}s^BWc%ua(OFF(`cq=LtrV}fY6`co4eUSnPbhnx<`0hvuuwsN!li<6Rq^6SIC
zLJK!@ojyRSbKUgX>t8ORDkLTEh17A3AL)~M`?KMMX)7|Sa)}HeI9!kRN;CP?m?-VN
zZOzyU0UXk6>TKH&LBptMh@jPORwe`uELNtWJO~yT1DXq-?{h~l_mdff1oz3hFZ#z_
zgNrkXN^Ard_|W}45u{YzZO`#(#D17@2NjQHqT+ze7?NHmp0bD)Z?1Qe<p@wy96D|W
z!b6Hf1W5+?gDlHYe^$M~Kc1XB-m*PJ{+C}^cFh9hphQ5Kjt7+K%otUDB|;!Md^uEu
zvRdeFTs^JVLzto0Pt0-VE_&>G0Pe3#!aE=Z&9lB9f0fjB*$#IWb~iaTXEuE{2M8WG
zS5I9*4$~ZTxSA0Zb3`X;Oa<Uj3Ab?90jaIbbjeHpdGB4S2xJa^c2ht%806eHS7R@v
zm)9#m_8l;uaDK8aoJJw2nN?_OJC7N<OqG`?`wYk*obmTDD@R@1lhDL%{qcikFp~kS
zhr$wb<98BL`svc^AVcSuU@j-tWLob~>M2K+lNrUTv<&~496(g~XQ@2Of9tmL2Af(N
zy|~;F)J2b5&xR@kxp56=V5ko1!R7Hzz<cf0Zuz*{qOJ<xG2?~<HP!O4oyoJAjxa68
zj(`O{f`t+l+xQl{vUTZjIH0eGS~Z(g^Ie<LhP2r4ZdVoY3ci#L{)Mxll_5?HC><A%
zv}-zST|f>)tdOfIFva=aVO@{EurBPs2TBUqQp0|{);>irpgd_xqkxLGgQ+?#`0=O5
zaR!>!E39vEO<N9$U^a+_)T(jL{<zH5mgb#f`vE3>eTp>q>t<;Tm%@=J8X3k%nGtJd
z|IhN7>GCm4r$i@jensn3qpt<~fxgD#h4AY-i)w&8U#|fhz$u=b*wSo@6r4^)T~6dW
zb8~4_V}o&GV640AN*qR`mj}W^xPr6cf(8m8AOc?|ZOC}coOy7TQD}pr<@MLfLPY+g
z>fBcek~RCAmi-$58vYQE34=<YpUw~+rFmF(e+C9sJn8CHT=fTw`1Se3m5uMr(G-F!
z|Lj3*1O^NVQlnqa|B`UJ*|2$V)UV#}<2}0X&x{VABYyfK`hOs@{L`1c1whYZ^aH3J
zj@NoMI>3yPt3&o!FJ3nCl@9vNuXieNxMcLwJ2b-8JQQ&v4o+Q%6(<-}hLHspF0<5F
zQ!~$5D}Lprc7d~KFiEG5JF49$Fe(luD4r!3Sn}tZH-|98Wb?JIfzEbxmHC>b`*+-V
z9FIC3^efdHle!nXd_2l|#P{9#rs<JHmr)|y-3Wpg(RBnUnA66eBUu2Ak-bZAe3K3q
zY>wS(`>C8zmXMWyO|9@&+ND7SjZbaN7}HX+O~M18c>151P9)kYmmzfkL=`lF9CH!&
z_re33YCXg`B9PF*#g`?Em++e>Lgr`p9h20T5S^Cx{MYZIg=sE~q{la@GCS&~5p0mP
z<?kY1-qp&iA6vN#KTtdE+Ll?3!?I8etdT;#X0&SU4fqfTawnIUWO8rFx&wmQS(=XR
zo_UziNkAz4a7#+(45;&NxG(idlmhEhfPZn3so}{#Y1^1x(UBMXM<<o}S0}|#3SnS!
zzp?JBCArzpt?~S5fG_IRxHTfO*LJBC;EH4hAEeAE*c<-i<k8A|db6h_)OboA##NS~
zLsrf{sz0D4qPMhBt(IwGl*L>7AiBz6&L63r5Xq;iz42$6Cf@m`(FN2$-?H0QvjM~V
zw;cNZEicpFvVH+m4A@w?&w$H;-HR=6MirJP)0Md{HwZd?)*eR~Qa(@0p5T?LH@xHN
zTBH!hdU4wSCY_Y3S^2pZuY4o2e9#a(1mOxFr2n(a&3N3*h5t=+7W+G_IsSBbT}t%B
z=YE&RADrMCR}Z1y$_%E9h|y=wV~6oR`tYhzO9)BRrtOy05G71jto2)$-Sd+12xpma
zgKqrJ8Wmy*n#TujVovARmqGnFk?{x^m3aw=6<+oR*mnhZfz-b$*e`&e=i8X2fauTS
z1=MzqF$8zjOzD{z0iOp|8upR`Bd`wSOU~1Cihmd2ip#L-T}ud*ZOg59A$)%Y_jbT<
z94Zq&=pHEu6!BIs#=eq7_5X-vY~IrbAT*aGg+RW9p6>c=d25-|L&<lDxeCpsgkocH
zm9g`fVy65glN0*=!1NW9=bu^}nP(Ew6W~YI0ZI1OY7^2i<h!1c6M*VYg9whq<3T=v
z{mP$o^bGqe0%iYK1R4_p6H&3WfQg1==)<g^tV=2Mq>K&D`02Fg0shf4737nPZOdPD
z$>>hgXc9oQb47silY7}+?Ok3fDFtj3YN>eZ0t!$Z2&H)T$sN<bai;kTpa<}dVg+$X
z{FXti1|||4F)QOc!BBAyOom)B6QE1g<2DtFw}E+|f*jiDA2Jtz1qsi*o^47%5tKj-
zNRpZH7U2F$GUI`Ei}m1YSso@OzfW@eJ47EH8FJNMNy;kp|6Hv=h?wW&J!64k{UV>_
zs}V^%)do5T)mu74N^AK*$=_q-?(Z@{qH$iWHe}!v{vTD{b!89}8L!%uSK~mfVlw6w
zTefGkMYjIGztX$V9RcN^BNc%UxQQD!#*Y$4G0Jn3y&COZ{kHxMkWiBXH(|f)ia%5S
z)fL11-@4+ysovg7KsN6d0J76WX#?Ei{r>->0A&?V&)~@WgOHv7uBHFGt^-)h|33J?
zrT#Ct{@>Y7OrqHjCy7qBl&5RRBJF1C)7-#FP@PYxyEm{gzhwSRp{q;)V(eK141Q3{
z5$U!#wG8{dK2R2<n|Z!(UO})DSwCjJME@(N_rI=_fQ~p}Zv6c6?-a>S<KHQgXBj|#
z#Scfe?gs)*1AVTM_%1u=758IaG3E$b0pKelP-qCMY5Xo@)%PF#20Z1As%NicD+EaA
z_Ks!lIX1}af;&*E4FJq22=<FuFqHuRl$Z8BZ#D@5kNHsgHU9mX%ano-C;Y}){ty<(
z>nws`h%k294j|F0?;}oK#=uALxrof;|NZ3zfgb~OcjWJbVN|Hu&ru{^S==@-Q$)xd
zjcJ&NP@sq^Hj1X{?{9?+sD2NAZA{d@kN^1L0*_hA9%29cOmDD+wNsW|Y{~Z*1pn^^
z|N9323(^1f;D1TxzbyLy(_?~&;se+f;`u<V9|sJtuQvBeY_tCJRwQB#CK+--mt$kd
zJW9cM!q#W7$!Q8HpmLDnUB)&ATDi|1puBwV=XpK|P=lcR4N?5&1?`A0deFw3#@v4&
zS_B^ae-14I&BYPUmCyh50{E}d0IHS$S}R~k;J;?_eemJ`6MZ}Ta3-gNhR@qh(Do;{
z#&Had$oE2CDf%BeQ0Z*-N*0LfFM$BCw%8Qi1_zYu_W+_!uy*%DT>+@gpZ+2wsDK;}
zs+mE+L;Z2iL-4iPPN<ail#Y!Xq2`ygaN-M3<4F4VBAF;$n<UGM$3+{;3iaih-H!{c
z*74h|c_|4M({vU3Hp@EtTJEpJ@a*lpLclO-{Ql~SdUK0TJgsf+NlAP6-;wu{7l^*I
zhor~@>dqH0o$w@Yop4?iE+3)3;lwT7jq_K&uu2REtV_-$_#HR8HNM--_qRa>;EpRP
zptE@=zTR8AWFVQb4_WImitS$8lv-iXXv%X>E|}yc2GN|AyGa!~`v?Kkj#6ePPfIR5
zU)nb}V|7hWW{Be`eNnKRC@w;Mv^rnwJiNc`JP5oD+Wx*jEelws7=j)Ov%my&?Er)P
ze7(YPc{9kfvq#vGe6=?rRhbSixeDegHJjm{>)q=1icZG3o(jG5edWx;*kNTAO}X0i
zt#F;rZhKT=XD(h~m^@jZcy3KgW0IupI(+?ScPIOtQaH%!<ari`2AqJR9{0G<Cp$;k
z3gxK5nG+-P_hyKQ3dr2q8cj?eVV~XZX&-L-qKC6C9-m!@$dh$rkL`K=AT}fa_LQ#x
zEJt)VSQ5{iX9dpXe#}$y$Gr)iN^en|7FP9lxbO_SA0zE=B4mnUEFQWF-nSI&^F<kB
z@9VeclZ5gtUd5_uV9sW<5m_#<>*5S)WE5O$G*o-`Fja3hsz4@Lp1|$Wy=2uV#$-5l
zv}N?-vL+CE`w85We40FY7#~T^X_){QCfX11Nr;|t3}QNLWw@m{oHA9{`=rpYPrmcR
ziW(bOd<$Pz_Kb%k1+Md7h(i0lp0p}x*G!)h!3n>{1Ap#xf%^z9HlM(sfcYcBICSlA
zg?I`z-wi}c<cTm$r{;bfqf$*d?6&IiAm($kW--bEo)IKrKY8W&^bB>!$_XN!6T-I<
zZFrrc5=yl=ntPjmKx{_6ugwas;5z_It-?@2C8`V~F=S?e3=>9C=$ij$kMKb(ZN)XR
zgM$?Y!dEo!=E5HvG%t;J_bPnY>A=)M`xkVA#9iWLrqV3o(=PnVMys>{%}HU!hgJ7<
zZOXbzVjg{6lDmW|nhHw##s#=CG3hQ~0Yet@OuDC{_<Mi<S*VI%I;AG}rOsQ=Q4pBP
z37=Yn0yiJ({evsI=&T;~W_I@UyH$&6<@Q_-dV8_0vB`m;ij_N08{FOMfDQ*jr-}Gi
z+!nnKT)M0nSPfO67GI2v&%;6S%qh1L%^$1e@H)5BI>|1pa54a7xKAMWmOCm$ibvw8
zy1;j`TeIPEd}MgxbLu^DeQXd|t!ssA)EsV{fCTmo#9x5$4F0_UC{S##D@B$g=IsC$
z<~&DI8UD-7Ly-{C!yqdTA))WfYnhXGU3CWUV}*{3s@WG-6F}c_PRebPesBS8TduSr
zUML;E;kYwG2P=+WJ9$tw2bgHkw0Yl*Iy`w7><kEtW$I;_AGMky-yg4hV$QeGeAxTp
zOQs+Dl)sUk>hNM6?2UJBcyl<0;&9l9bm>(E+i~(s@0*xu2AUZw3};TV(~8B=MZRZW
zdyOCv2!L@&&ACaxJYjpGWG-EFrukq+mP?6_VwBfSYC`2N#Y3c&)7dXN)|h82XPpf?
z%sNi7=v6wse;jM<QU98ZNNn~H`IT&TmN5#b>tw1rtZ7kcszKkKCUq*jQO~WomFako
zJ3rnF$iI}gOMugovRD5J$7xz?c|F1WF@$;6N$UH0`#tKDm({Ph07dYPZDh`uy|c@U
zS@j#IKV?66rm}2o2AW@=ssx<$qY1glfH7?$+R2}L!$rNMx?Tr6v!j=v0Q(<3M6V`Y
zWCw79II7}kEAelLWb*Wv2IMPARZ20CvUhL0@psMvv$!g}XWFM7`l0&vJfU3=S9>OF
z4Rbr8BG3@wy+<}}ayxWQA__=|zva52KR^*EubXDjdwaB>^nSA1hW<zZA$y6@rhWRl
zX1vSTsZSLp;J(L(-^<%xn!XjHjh+_{cw_YY3_?~4f(8rWPTtl8>JRN4E_9q0E?ql1
z->}U=5u7&qoQ|ypm|u8)JbLWA3nd$Mo|BPR<%qt7f}s>ADwNxL?E=4T@RS)Icf~-W
zMGuMJTRQ_I(E%-ESoPI@d7<ITM*Y6YHQ_BI)Lk2F&ZZ)iyRNXdQqHx(Z?2Zr2^pRo
z`?c8yMzz0jB}{02(6)0nN5amRMb|5=qpDfOH<N<mb}{>7WOVdH;Nr*tm+`gtS<Sk2
zv+B#ZxDpX2zxpI90-R!@eim1mPT8v&d=eGfrS0uL$B!p7QaZvp?QGEr{bOSHmJ^V*
zl8Z*)(G~J|FM^XO&=@Ni)2o}TyM(V#O0P-}^yiJ=J0*CH-6lq7ZD^dIWPvn)&_m5;
z=2bEf5KqI=OW2u_FvjIT!*H#gxUsPjGiNcN5mY5qiS-QJ;V|I8vIN1!ja-;B)SE~y
z5)wh&8fw+g3A+%2_3944T(L8qZ7x&fh6rkRq_M%yM`7OiN`YN>iU!NZ9&%j0h;o2+
zeibco&2?cD;EJwumE%o#FTB0o-2Tmj;4OJS0c#qmbW8DEcH(eVl7R!aUGS$)x;Dd>
zfR2nB8%2}qy5aXu&^BD1a$$3!hHflpvE8pyS*RP4{GPVoFQ@nlg>7_8O*t#kjBOnH
zD}_LO9oJ6ZH24&rASPMo4Kn(92rv1?@)V;a0=XwsQ6ZF+pu{c-Eb#+vX}qeE2xpJF
z?;ZAQqpvBpQGRlRt@8pJmWsPbYc<u;BZo=$<jv@rYc|6Hx-}^J7(pyR=F#idC!&-n
z$O#2HBr+tn-+8(066#gbqu|p4xNpdL7x~-;y@H*k4J|=tYUezSLbXF}+dN(N;s>}c
z0ybJ=ouRue54Y8wqsSc}+*`AuAFl3z%X~mt+K|rH@=V#UIl$+<@UE9(9wN^?+th^#
z{amFu!^qX!?&VW-el)M?{!MCC?DPm=)2%b{jmeX-Uasiz{G%=uqi{L==dpUeU96_$
za`zUm_Z1(Qa}9Syoi9i@P45`O8lY|5N)%W!w-Kcgm9zUkxxKxa%!L$4QG5(3a5TFM
z0SgYR7o2KHZxQ#+*J|X^9YFuNW9&JsS-^=%ms-1*u;-~TWgiNDC!{%>Zy`*#$)t^R
z)Hh1l-aK)}ICIA7nMWY{>{6Tvlno>Mn1q<s4H#ZQr1{;e*6BJjG!g!WT173~3_&-`
z8X?dZEV|<RL3y*@nF-7=Ox<$WSF??lc3fmzTKu1i5`OofQ(1?!Cn1OQeeFF<3RX6K
zU3Ur4D4hih_Bjfptt0MhS)mgtfrcF1Yh7hd_v-xi`Ng2V<x;dZOX%ZV(?seD4qRLb
z6b3>_;vd%i@S8;_HUtAAyy<J$2d6=^&Qh5A%lYrRJ2e!#FAdu28N6?^E+o1kJ&rox
zi|RI6s_A(>j+}5mUhjZQyFa(5pEDKOJR)Fk$3o%fi-!`MDNpJ1K=}jCN57k2w1zy7
zFrlrlvK$no>)BrdUQZw@kgI=L1*%!%LS%DVt1=fMZaDZRn&%%BtAoLiw})#=pWljK
zZge@wowhBeLotnjk0u~a(l9(f_@|>)M6dbCVrK{9UL8k)e#_%8T!;@ar`71b54MMS
zK+xOg;hHB#vBj<0CVaHQKawS7jAAFaRhlxNkEn4Av*kDYJk_~`3%^FXvTOZoI#n)x
z7n9-kd^XgpE(vTz<Eqx<A+(%cyC^kZMQH=5ZQ&3;P>m1L|J(^PF-Pzdf)A;9qES~5
z$uMIsvHnz1K8ycw*O|MMHD&zm&-%m39-++L;A=XV4c}^uPP30E->PO~!OSJC$l~bH
z6@`UyXT2Pj5`p!M)Tmm}IUL>h5Lb*KGq~ra-ucBZ6NM&`1%U6A-E7Kgz0Xu9K9($j
zXH9xS<pPxyxazudexFC%=tY9T5PPl&5xGfB=G~MoZZhluIeJ?s@m+d7)`Q>O9>2`m
zv^=kwZ!0qgO=zKr8QZ>XvFmd-v?D1jKotDUhz^*82?FMbUbA1`F27J_rqA#+-#kHa
zvtX&r$>JZkvT{1#>NJf{V#Ayr`nNzRgAQ_FvvZhG$zUOivsLDbIHYmU>hF`$tG-o^
zChIRM=+gS1C5$Hcz#|q;ais>>KJiGaCa9q$4U3JSFlbyH@)%ZkIA|aK<Lb3zZ-_k4
z=vmj6i(ZwipD9Lh#(iKM96en-a)sHGXEvDQ!Bd{Vu$nH$aHrKKV$)m}q$f_dt{VS(
zwf`(djj-_<HA-lGoSemL)eVLcB**dao@t@hxm{rx$+lMW36*x;&=mh%miojQ5y?|S
zD`)<3{YG`s0yIz1_K$*&h;$;-sz$}f&inmfFsRPzo&eA`l8wwc+x}BgI0ryfKVn)P
zM2U}nm_mGh+7hfK<s?x8_uGDUKy*cyF+Z@4+bVw*adPE(<mq{T?guT>V<f+Vms=E~
zPA%nUT}s?_KKKDii0?>XZtcCePko==$T~`@n&Xt>=1#ds&R7+dx~$Fv=hP2dV3?R2
zZQ=P^*GI}1b{zUaq^YQ~E@BNoJ-M_I_a3Rw0Aq8p5R;m2r-6J4!{0<c40Ok(xzPER
zsd=Q%@10jZKTQhMaePWpQS7V7YZMJvmA(gI-U~K2OAo!=cg*!o=?5L<%o1~Z?k$b-
zw>y9IjuUbJr47G`x71eZbINEZ%$2E0uG4=NhK)~k&id)X;*%9y`|<XJ!i)w4uAS|p
z8bi(xKDXE&mSc~bJFrQc^U!3GYR-#uqa*da{PnGmEEL%~t<R-&9=G2ws^#bjn(zZb
zHW#;oNks-CM#`FlJZw;nytZr&)^6wOQBO+PUlMDIBp}dVd1n6E?*<6CmtFxyaGQ6e
zQNCo5OhY-GO%!*Lhng)CjLPr?j4_1wPkv*#b+_@NtAA|Um^B4>IOixw<{#423IY~%
z@6$OTF9|oy0zS4HcTw8f&J2L??zwgQ?3Z|WwE2VLtn#HC{5Jb7EwCd|eI?Oiy-_W%
z6=fXq(FAP(e(1B?Rd!deN7o1vT6;b)`T450!R51i*2zj50hzK9$gWu0aGdvgGdsE~
zFxH2ov1j<{dp5*idvM$?EgNmmOm7YK7vz+=&cv^29D1W;!#oOPEd<kzik&WlAi$PF
zeK}5@_81Eg+I!T7A0;sL<H)KUAEdj!G7A0-bH?kYW=O=OL$p?YhE$*;d}0t%p*V+f
zdi!zfX(ar@`+DPksoOg^6uWKIz%6+BsMg6mcMnB8exY4&=7tgX!xnWVWKKJyBW+dM
z!5_z8x1xHTvp(3t`b|>77p{?v%2f9}-NyoLzJ_YxX60Q55jF1nRzINGva}~Ne~(_!
z4=SfoF3IqL$(zQ+-zgfWgl2TNvUQLv5<x=?sEaeG+X8pNKVFY?z5#9}Ot_{`rO7u8
z18ieonIV5RN^eb=C3cb_V*nj!HRDCL3K=qBHn!69#NP45Qf!MA1i}tEQF*_sCt%#<
zvv+)ZneMa3Yv&N0M}JMofk9Vl?tv5ST?dN2<T>2>_%d4;rr#F0v4T)rcVVMeV}tv0
z8ft64A3lZTB?CDvmffLB2>L?scz-lZSdWH=Cx7Z&-oT2VI`2O^dKT0~pS1rsmB^3)
zN4uKX)V3~Vodj7z-Z$tka0rl$SSd|O$PUT^oI7_pU!(fZb|IDPpuGSKwIIiXi@udb
zc)W0Xs$5x4EDg=j*-u3R^uS1Rw#xCvz3!sbOC_^ve(+@)Ym&U12x9qF%&m_5{=p-0
zqSP#ZXe&Y>c*4W%m)c^h9gPAh%#vTP0`QEt+8@9a4_UDv;}X0!HIILrDlM7UeFh1S
zIPmo0NWWodd&?mjg!g?g*B+|n8+<4qRy+jmN@gKbS|kG5u{$dYKlozYwAw+tdUwSh
z6#6hr&f@GAt*B<q;wStWu^*k6?!vG}&YRa@D)49F*4UdFUsuaYyEID0eW-BD&V2#@
zch&?m#jl1m_~;aTz|Ba5u;Qs%3KV^hm2<Y?2_=8}Yy}eL70bJqn5*0PeRJ70))xq>
zA4%7sW4TVZ8uk7}FcCpv1aWI}<@`7`PfpgmyDHV^&HY}U$*yuH5+H5W1WP&TYd}b%
z8EOy7@IldQ)3$8sxq^R-U=dWGYL<r9wnnh(>(vH~`;_UaS%sZ2{_@g+?$K~V;?*_@
z8~M-4OI;zpO#83-P@q_OS_wlriLW2^E%MChBg9!c0y^&+lg3fWTmpWL$OFoq#6@7d
zYnMqG(sZ*`04~BPHo?(+(xMkP3*O58>`y;)jk8=kEKAt=9bu_xnn>3$7Dy_rrr!cQ
z>Dv7wLmgZj`375<B9rmjE4_-VwRI>orL@64(-6`S3Obop$Z;)(F>)9Cn3f>3jTd@$
zS3I->#_C(Jbkg+Gb(HU0EloPp6*!Bath?idQ-$JePOOq$2*?0!kf+GLfSZ7rQbq(Y
zEbVt?xT~YiX52HR6Jd@&vXWaQZ;a<%Id`AE!hFUxpNb<D>{{+FM$uT6_Y{D{woDo!
zLe<1@R;iz#$#PC_{Au6(Ee*+{8S7rpZ-GJNC~@vgO>!22)Vdh8M94s|GwJ8S4(y@<
zlf%`V&qf3z;<3h4IQzHIG&sUiLh2g6)g3o!Mibm$lm-$|3*Jcme>T$#wG(-+@6geO
zVivH}8Li-mgNX)PCw*nk2?$FWLj|drmu*4;84<o^`4$Ce-lQZ=s9!3vpN~PqM>E}x
z^j4n@h{?ye=(*2a^ScMGZI#jJzlnKqkupgON@s7J@z>h!*+Y9{>bpR`yXqw9jr;<i
zke&VMaVe<D-4bf!xt(PDe9boNYTVXH+!9nz#EWuG8}npG6W|z2ZC8l*D9lxVmcfe0
zE5oA**ihYE^5oS2(2@84TbIN31q*?!;*Wo)r1CYe4@yqG2;f+68S@qyL=ME{L7zJZ
z!!HPuD@Fynl<8%C?2K$8oR*DR(TjaUVll})KuC&jg*mkCQeU6z+;^n^*#98bnOz*x
zgDs5I!@wrJCk>e}_v(X>)_Icf$ET<c2g1@7pBk{mz_wy~>1Pkp3}EU{(B#8`N9Q<K
z!D~E`Br%Qq%w1F`)aCY)OQ<L)w1EDF37qQiqi=o)qN~5DW4r%`R6|7?r7{nHv<TTe
zOO0MIxh7{_-m1WtJ$nJ*en>{cQaowKg0xkXO#zme9udoB9tjeqZ$@UIMyK8IG>kfk
zdJv0=%~Jg?{Am_u0r7hReWpS63zp*&-;mv)LX`~=ti0wDJ#IlFXv}tFO1^7C8~9^}
zYh{KElYT%ujbE_!A!UQ<LWU3{kamzZb&QhqZD+SAXb|e6LNh4OxH`}jfOOBNs~Fr$
z?Pl?3G(7Ja95s25s;*NkkjVqHJJV@`neK>2vkVadLp%fyB>CbaICCB|67uJhjA>tW
ze{q>Ip)PKo$vW$VmXtq1nzFM-CX8l$#2lgq2l{0RS$r{Z!2Gsk04PRnftG?6irWa;
zTxJC5ZXR^Vjp2yM3eFoL<%#|gXr{vYD%_vz>pcVfn!E`3S^1_V1mA7{{i2A|eNW&n
zpW@S>9wXT?XOM1h-JI~ouYGqua0~hFE<KsVj0Ibci?~=U?&qh$cPv#2ZpmAE%}B?g
zEMP7sF)Me9)6?xEemreL<R@-H8C+QL2E`#4I9o(!pya9+$dciRcVhpVF0NXG?do#t
z1cKn1Gfkrv_%p*0&D1%w3wGk%9l@YY-eS>y%(6}Vdbc4uUn1<|t@GkPUEp*R;^Vx&
zAA0s=aQ2wOHnqEfdwl_8ty?MY*xABxV@+>6_Ji!(43phWmgt%vMcX~N0)LSRh9<4&
zL*~??0(}wxUyoA=08RqotdQB64U^HsyZG7V(fn0b0Urz|M>BhfNo_(E?FoMKJ2C9n
z?4Bh!E}g(GgpYx@6{fW!9u3Cfs=$~mMUXsWcW8&fp-Ss-jzb$rn)#an^hYFOm34QR
zNzR~QAukKJ0cW+W%~>Nhn7SVw@}Y1)PbK*J93_}on8#jy*klg3nbm?@7DDy=&Rj@K
z&NEi!ZoVP~F{V^+K_5}~>al}Up`YZbo<eL(33eZYjg=00I|>QmSt*Zsi!~K>AKktW
zA_tM~E3$9&LahD(K$+W<-gXN$x^x03L(QsdVQp%?0yRkmXMM4HKa*l`D}j6t;$=|u
zV3-QSHkpb+tz`Q%?PcGJRrE2VXneG#nVFiijcyycq{Hop##`dwrf3uhH-&9)jvB)O
zc64#EJ$ayzhFQp2B-KPVoNcfH?W`%rK&WSD!9qd?vu!exI6?C+RA-^~|5G~dB7G&r
z)yF9lcrEh2PB+e%6umEl(*;A?)8xZF5-pKr{igUSM;bV8NNGv%!CyI@m9QZye}yBl
zG-y7VsCH=UuT~7^&oG<0<Orm^KqBGw86#d#bT^lK!`*UNX$Y1eh?G(@O0u^c!JRBz
zWu$4Uy*4MA0e3ujiRJ?XiG4`}bWNCv94ch}I^Q#yy09P^Yhm3f!Qc@F81X1gi>rZ<
z=T&Gra<jG2xu!W8CLH5TRN>jx3Rm@#p3H~m#W2K_VE0h5CFp!9PCIedT_^V7Qwjvh
zKwHoE0|Q5oP<Ny38%wlE215VW?&MVwg1_Y7V$CW#QVE8|hk?qNpI5lzVY7S?ihi01
zSu*HI{$1#qGuUE-%=m}^blV|fD#Z#r5hgE%j1xqFFYfvH*GrSzN|!3cUcB<CRC_<g
zWEgkLPbrYI<@an_>X3X8*C?|A$UWaUQ4aagn~{-;i8Ppv7&_6ks$xe=6#lv_`Zobq
z2^RO?lsMZ;i(rJX4ST59MUbSNA~88oM>}dAD&=Fe+F!C`bO2AW>q<4*P+~ZYGQ3Di
zQ)EFMnZCxLpDjZoiD)-FJ;7HUDM>5(?lS$$3g6t*_q$*?1nmRezQQnFC63_xmFW8S
z170vp6&sm>Ky<HrNs1#tY8MBE0G0<HCH{Nst1H~ij>*ZZFl}Siv*RFeJKVy;<mw?E
zih6@VNQpAt+&A>|elS%Vnk%@l5#WVgwFjUDDLn~oU0)n_v8d>iBTng$RNM#|C4}Zf
z(p7$*K&!T+(u|24aH8V{cH%z2WD7Dm)?60C(YeVpp5WwE#OyDkQ%-h%>78IOoApDB
zz?Zzmvtc4w;3UXV0c*Fd!Ovkp1_Rgob4~-V1~@Fokm0}?j|nqELdgWpyHzj3N3}Qo
zlchnp3=CifSOkuySx_j!gN@L{$2s&;^GzOYK2?o*Gj~#3Bik`U5%Gdu;u0arcJ7>|
z(;u!a@TCtWiA`T0vrw&x1dCjiq3^=+!>ofw-$qk`<IaMA2kgyLH&85)1y9j}W<~sq
z_|Z4~`#%KwVe^2Q>a8~iR`6hh0}d=$TjlD2f2wHi05K_$HJ2hJ7<oq-F`zJx+RySY
zQ9r6Zgasr@egHFneN>+8=yNlGLwIdmQ`Z_b8tZ_`IJAj!eBxN$X<G7H@Hs>VjEC2^
z`DQ%C?w3w)HBj{B(@Bt2;}BW&>IqOCQlb1~ESh=&O9=<uDUE(rF!3K74MHG~1e~Kr
zA$v>(JY8b}U8u5eh8dhF4X$ktn|0!8)e~*~vWA>WsYaPENWs2(iH$_n&$&HZq+V=2
zmn7x&&Su5QYp~!|fk!1W@6mYrXD|{t^%5dE2-_&%?r+Sos}Du+MVR;G9!&QKUOEnL
z{q4VjgN9lmTWBhc3o63#k4bk8+8IazPRopRk^{K%2aLwvig`ku&fwAhl>BZvU0D9x
z$>$!SzhPjhH@dUppg=X8<w*7vrIg2CWDa747*_M*2x$W)9GJd=V{6WhgFK2}WnHQq
z`ONAHCG)7a>C#S~mFeheMd;#Pr#uGJ&)UZa2JZ_&_WfQhjZ^JxwTWG03AitUiA77%
z?(#q-16?ge%|BhO>oJcV^l=kt$ox5Ci`?&Gb?jVDsmhoRT8krVfBBrwe*Apve#oCG
z03yzeHQMuu5zz<(0j_Nbd@I|y)ty)f!J{u1={@YG5Ka2EWy~#&p%R*B!M4bI*iz@0
zX1zD)Jgujst$_jJ)R}z_q=O)p4Lg=WtSX!G-l-rw!qmiWC@#Yobui*upcPp+L8yLR
z5snh1STDG>W~2QuZDyz#R0BKQrTjBxd)@+V#52=R)4kSwDCYAZlo)+T0xL|HTcl$y
z#6<CL+&{iHSP&6+9;*SaGH;25Daye&M<xj0*aH2W&4@N#NYz|9A)$^j;q>y&*Mo4h
zD{8*ten_oIyy;$ynjtb+ytli;IUx#2g}Gm1f`oFTLWynSu9pFDu9@BOt7Z`*CEz){
zGL;hkXF0Q5hS{eV_n8jI>!9t}#3sbaaU=K!htos`lII-cKrSqT5Nn8ygNWeo1%?JQ
z%GQsAiXo|{j+PbI<(p}&5I==p;76eX!@LC!{CI^KG_hp}wd_o2>py8(MkhYVY}Yq$
zIq?tSxnF+q&SG05xP)r(Irmtx;e3;OI-kqt5^cYi$C|h~$GZDx$l^DU8L=kygVz0i
zcD`e2ru{MM0)Lo#C!b%E<w+tGKc%E`TbxjGPiRn8Z{R@4W^5?xQ2`fmjFm&Vx$#fC
zixyYzp8&MLM;DERe4HMqQqakS-_sU3WT6fVWJ2P6XvUw&fssEoTerc2MPWR;9GZ!v
zB;Xiol%W?ze?%@#0cB12O;Wx3jv^z_K%Tw*fJjdS8Fk-kMKijDl(9jZC*WmV)9%!u
zJw-_a@89l&;hk-X62jIy_(s{GN}W?vS#e5(&~K`4dW#>fNrTn)IonCdCsJ`Id`$Bt
z?jw);t4Jnk#)(;BG}CpkGW6iFk9R=?0QeK~AC3q5z<!s--O^T_3#m~w)UnIZ4fNxJ
z_Q^T!@m^^&_=#8>U>&<H(sePrSuaFe{{buo7NW`~!S1!P{1|z{%p_0ZdJKnbs&v(`
zdJO?Q4?^Kfpx-o87Y_HR2cJfW5URLk8K&$yQ*32tp)hlMxiCu(gp7wxk5NIs5J*_B
zA&96@fXkQ>932pOvhzYg*8zs-;bRM63b+m{!&t~xK)XNDixq}y2_-?Lm`Sr-_|Y{-
zm{0S}yV%afVyRN7>l(iHmfy!SFzw;;XVS}jl*vk710HK={-P}qcq-uIXwA~^fd?QZ
zN5Nw!bHaMQ?ztu{*UXticx!~+Ju(_cDW*6O2(Yh(5CA5UkUhk%6zBKLyy*)vW3Bhb
zGE^v;7l)>d@<$BpmIz>Y77v>5W6v7|?GVz`ms_ijsqtN`A$Lwck~%gLYJ*vCCCI31
zDXum5gG|PfqjmnfzoSC!n?QJ_%&$xG-+BEAdvAaBQ?mILFq5kUR;j;vM!VHj(@fmo
zBfa3leQkmQt%E-Md1TWf7h?U)``HnQvKT4+FoTWEXwlY^&lreDWJl%*bM~XLbQDt}
ziyr{bGx!j!F<vUITl+68?y9*m&Z9&I%!9qTygKAavX&r{GbO$pK!E84`ZZVOpsH*4
z5?yQ8Xxq>wxLImfcXVx)$n1b=)kK>#N_ybhEL5W7OQr>)7T0wngcshK{Tfm18yTQW
zxBCvXI?Ij_JT}v97tB?@w0vcdBiv+Vh(#J#96`*b*QFlOc*}SG37!qjylYm%Q)bFN
zL<TtGO4kH5{s)tm+<9odiE~gDGwd*aMHfwz0E2hH?^`C6aa=VV4C)TkjFmFi{VC8d
z_M^)HNL%%-Picy@*Hf6!L-5yL8K!O6HWGlmvduk`#1LhMPSIiHgf>w(u<s@EOT|{4
z$qtgfJ_!-(qA9X+#V1cZtpCuq9CNhWv4{<BG)@Gf;ERu{=<Z|)g+DNCH+r9>Fj^To
zx*Y{gD?ONsi>m5r`$Ce~S(rr7Sk#k8E6K^D;$Vqfoi)GA^Hqc%ek?$0T$n2>t<Igk
z5k}M!*$!aRD!bE)pw5~JZM{`P8+thi9mEYA)Rlsv{PXwi^z#A6jFfy}YlXt#8XK72
z8beqRCLfjB<4Q->RRZ9vqbEFJvN!KXg~YNNzis6;w#HTGPWD?0OBHLzhqt)!3FLA;
zho#&Xfc<)F`d{pq4RG?`8%oZHK@f|(GuC>F;143GiBRnu8ulUh=Su@&lQ$%;4MU53
z-}40mw~f{O<SIJHEYlJx2X^DYg~^a9`&>9q7fZ3cpUO?yAp~SGRGq!8HqgD8w6E{{
z@uM?;Z~`Ls_3b5>$1M2cl-Dr{wO%t61}Tjh$cH(i^VPEN^+Irzut`td%iw)<S+B9-
z#zn$$8t}s2I`8baT!$&yTR#G3JX&v~$S1(dQ;ilP&>i9pP`4Z?o?>|4JGKwV0MMC6
z*`YTl6pQYnf;MIY$*<w7vpIhlbwsaf-OxATdPT$H05;g*FKBE$;L|7@LVnmNvzIUa
z-oI~t^>`>Nh3%IcRs#p!19=>+)XJQ{Gr?>v*KrUf)7}k0Cj%QHB9qZ0mlcasBovOt
ztGtsk3J+u3meH41@;Ryo)YmoMh2ZXcGg7emn-&sG;ZF(<k**<6G8+6gpC2~~LdifE
zEdBtR8h)-mdr&K+kOYP+_q)<yE;NSrWORkD$cH;AmpHJiQ?*2gds&T91Onj;<K{+N
z4)6GhUW-&p2xlRqA5nsAbpW#M+^$?q)%=+Wf+TDp$YuL_9zwxNU3~zagbfB+?clXt
z5D{MU!%@tZx&fh@{66J5MXv^@$I*tUo<7A_w0Xp}1Y|I>>JR0lLX4{qj3RY&oA}*`
z7fT`4Brvg9V?rRX>GpaFGad}JE>RB4(t)Ej(p^yqJv^nEcZ!Sw#$tCgVonyFMu}NT
zNn1%UtJCMNh<VJYZFlO-yyxCaBaO=a&orwym`a6a#wByUrOueiQbb0;+#U+4F=R3<
zmUl=x6e!CT%RHSYmXCJwNQFomR5~$8>Pnla3w}RIiDphWV>@$Grok+$+0{E?ViDD4
za8o@FAi6n^V<#nm12RzDsRG3vh+l*`L)%dHISZj9#TaN@-$pcl3NstBT@#f5mIUIL
zGuI&F;i9-kBO0`xnl4Yr;o{TF!}8dc10c;(!CVl3ur+bxY*fv<YAnM#S)5j;7D9%C
zWt!Ha5_3^9)B$_Sn&I9YiMH00BUEZVlH9qOmuJX(yMn6wT~ky8s;22RuMJ;l)QyG=
zS^MeK>mKFE1~yRz3;JZ~wT0*x2^{Jz8)2;?ZS@p|8Mz=+NZfBIP?P#l^`l=^cSqZ-
zim0n)!t#Dhrzml&>w1k!(z;_LFlu8UH~@|XuJ@zpY?=_GIYAXSD|*3UB|H6fJ@U^x
zhxmf0R=Or{=_=*MKjzLcMnJ9@$Gec5oKUecR+Q3r&3%nFi4Nz0)&*w+hrS}I;VH={
zGwFn#V`jv=p?oOv0838Y_Q`%lv5iQ)rM|7e0Z}kD>~c!^=30e+%NeQ4gUY(ANUN-4
z)^c<E;m(Sx=OdOtOxNQyetNx9quSnv5&&}G|3IE(&MK;(4%t7drXc9~k`!@FVO3Zf
z!#HA9{%c*c0~>|0E<(}J7}*1AZc&}w@<US4f>XV2Dz0-eAsK!!2aZwkR_def3ay)$
zHoUMTF}B6Sp4MT|E`<r22?(!=xr<ft2kuypd9BF}pdJ9sm9RpdO0f-NO5A2?X{v0<
z^n-I5+gk)E6WYXmB5!Cyc0?5Bl3Tnfz62>!2$y{O?KUr?bU3_s!yJ(~omkUFMi?A%
zR{OPR74ul23=}n7V@UEE+&7Ri+B=!5qa~iHi$?huL!V8jWTZXSLgi>){D<BylL&(D
zN8E!3^^m&c54GY4mgh}l>SV+TNP3%0-!}oJ-o7^y??VPFHv3$ayKz^SI2kv)FZeDI
z%Qz=v3FB=Q)|$kNObp*pt$zkA)lfG}n~>?JwVEs-0wQoFEh2&}6kUuFsLB!U_DQoM
z&t{GzKB7IsB@8j<_hg4}{<EtGz_(rfWqZp>m*LO(pYFc!;p4(igFhTaEj=S$T7SJN
z^|-}xuw}((Tf>a{aA$F|EyOM5YQPU4@u^)kg&|@ZQaT}a7ES#=ZfIl0yaP@l*h*@I
z^E*AM`GAaf`cY^7D~&CBB@(2sV@&_-{RX2>cRa$$>;eBGLlNRB$AA!4My#WGPn2<q
zk;Qc8%bmc8J?#hjK^jmV=AlzhDy&GWiv*vuUY;eOij6N*)x$+^(>J6*mEL(gxp}J5
zgT4G=Ptd#Ov$oTE?MYO*)e1=dydM$3L^$DLg8*WgS4WALz8>bERDkvcY>j_t#eF5R
z@nBR0u^r{G^<~PO@ZcugS-Z@&n#m^_F%#LrZxferZNo01T_>1Eyjgg7r_IuZ*-St?
z{zE!c6#lbtFiwkpC}|H5_qB?6RiN#seK3?X#5J?oGm@%Kw6vowTfo)zeW)5)9SRif
zw!15!h>|Y&M@Oq>2(LbHuh8KM^p}DR+dwTp*ZqJTeD8E*REVrFWD9kTwtYgCT7+z`
zIt;X0)TM0Ec0Ko8q6lyZ#Uv|7__$2KPGjB8t&%AJ*KppNk*eqMmRE70#uZh{!j#)S
zUi<O20F{jgzoyC{gM{{ZTi_h2FwI6DUJn}K0hRy7E8xyGMSuXNXq01M1~viTj}Otg
zkpP5A2E6pqE1UFT44uXxUylSh^TZDUSH$!>FywGSD{Q64a=f?G>}Y6oDy$gAma(N{
zWw-Y#vrl!vyPq#;6T22Lkz7@vdb2&_UpJ*bdID8og$>9C%Qc`9?+(C`WAv*bwmc5U
zsU|l{utXK2LxFbzcS-2O$fLT2q8QMVL<TDsWwKDI7Dc8_nn!hhyTX4vGn-gl;M+Uq
za&?`|+<WmlU0GQunLKtk-+gH(--73TCFHNJ%O(lTS=qyoBO<=apWUv%yT8c}p)MW@
zlu{~J(q;y&jANb<v`pQ5rr2rKK~}aZeQsdp3!WQn-z2<piC2Lexe|j@ywsM=QHNux
zJ;9R*5Y6ijs(*gvSO`;<{bmkkd9X-I&BkBlGwu;{_Gn83=_aMiCVu7m63_y*Z~nT|
zKF)hU>(P4RTH5g%44=NEB=a29n5lf?5}3}m7X5VgVba}+<(p;4q%DmQaHr9;T;wdG
z?b~g9k;!UBW4qQ>wx&-YtO@_q5B_N{4x9A&LWV^#D*{me`}_g7M>_?BqWtGiJ9p`R
zc2d-Iz9Z=qmT5mII>+z9OH8v8px>-?j=1GEzPyIWbfh@Cb~=5(9HLDAL+3vJ!M5ds
z+ShTI#X(P*G1W@Y2lFItThndPO7NBahwuDKTK0TgriHg;#gIhiWfYCEb-9G}k5g*d
z@efXPm7ek5r4<d$md^Go9wfew{nUdVZr<b4=k59R5D=SbL{^u%`CrY;7TF&3CamG?
zl<wPCLH)lT)tY-}!!{Jgbmnifb(RJy?AUTlvryfG6Syj+7NqUs#01Zg62;skFd<*^
zi}TC~<?38HWc?PV`_w<n;(a|{RjlXO^SM{G5K`WI!$Qj?Axt+=c#y+?9>QVz?5@&t
zrLHCS)pK~wUI}V+!tG<cQe~M!+unGH?;30$<+rT0aSQk`1f@){9*0Ot{*2D(h=aHz
zQz>j2lWV;yuSPEYovn1`xfXM3MmYl3ac>Kn+wBL2607qIOo7YnLuEqlyb?0U=Occl
zg17z31U31|mn#7qu%ZW_r$+*!2hl9(QE%N=q7p0oLKG^n%S)0CXFafHN}1rIH~<Yv
zc)EM+WNsejkP$0hhBw8V$al&~^UTlh^|4LgB{&Kz^8T6<nxs~DviM5v=;@>TnR1fq
zAd=)`Qk-pJqVtF+@CfT|ABIF+3GM0zPXY^e-b?+{yJ(NsBaV`Uv;}_O)l@lxENe<r
z?S%=BBl_ggx=?{9)jy!ngtJ~h-Y{OW?3(gc$HWYNU0!@a@%V^Lh4j07(J*<X6myO(
zC*J3}-F8kcrt(kQq;Is(Lu<O0V=wGYI?LMaAx*tVl8i3WZ)!U4y<tjLxkOY5NooJD
zaF<t%PYF-YElIl;$wH>u2ZdzdY)L(r%zThl?o@o|-4YC8W=Us7-f|o21m74c0eSvr
z4A{76gmQMQ_i(RwR6TzCcn!Y&_mi}qjY<r#ldjRxiMLgX`m%WJB8=0{Kc5Jt2QsM6
z_zoAdc;+D2u*aMofzZ0XVeo2@20azq^K#Ru&N_&+Um<+Mx7tF-YM8!o?UYKap>Fxz
z1o~y^>Q$n>V#aQ)L1v_%DbT3U?9#$jE|ro#%&4>THTh~rzL{#RgI@yrr5`r)*9E%S
zwrh$|5ZdyWA_$aI_ieON9{&A(a-naxybt(A6sOSozS7DmN*yxEWE@qk8Jzb`=B~Fz
zZL@2>3lpqW)U4bmnu9Y)o<$7Q88SqSd#s%ml|DsioWqg7|3`b@8P(*rwJR|QC<uy*
zs6Z@$bOiyWCFr(5q)2Z8+)D4g28f9027yiQpi%@RAf3=fMTiguLg)c05dr}cS_pyr
z#(nO&-*?XWcgMKn8+UB}B*2^XuC?Y|b3XH#PZIh{c!(jNab3=nf@rn1+d6qecB2i1
z6SktK6Pqt8-w+k$^Z8(@F?qru)#>WawqS+FXYTHoS{iA|%@@k;sr6~JYi<@bF+<HD
z&>9@I!O%l`ZM?$Fdootua(y0*>gLe+`BrQy!$e5bE;i0^QSEZn*SXyS7_`=mLi<u|
zRPnZJ>I-qGAObd&V%sj^=aK+6r=69MP%JjQctg4CjT3O(Eq*J2DQdnK3Ew9ij4NLK
z5hIXo$Z=}F`bONjPIw>2RqKG4FQbf}eKW~ajq*xodAG{S`H^twxKAl_WvzX&00Wzg
z$=eAWu)BqEzbrwY){34ip%3G4PQ|u`m>53KY0F!m(8#3LhxNQUTGHy<G15s(YD>F2
zsJtaGbf|I9BQ65Ac@Y6B{PcZfC9Jcu^6~)x=`V`pgkZkKmM!R&q|^c@G@D3pC()!L
zWXDEH5qJ*<?*VA0_1-YlZS$gxO~1U8!5>bU`e_|M6+F7n4&X<Fi)((583>SNE_&j*
z?#$Nbr3!I<^Dm6`qSXngJDpzbR(r1_l{?!dCGiwqzE_jzvfo0|T-A&Zx282YdMKv|
zPg0xP(6aA1`1M@+6BD5YN|uMtyY2g9x2gkE;&C@el<Y+-N7XJ9Ch_CZ^K{D9jnj5q
z<HYr$0Cx+gcwtNG)4^CB(Zmyc>d=`N_^lrZ33hEKQ6tHf$GK`}AMaL2t26EJ{;ErZ
zwdqb^rDF+c9yfpg5X;{0EiZF94PN4#DPLA#WUI32-=wG+>3i1#b48c;Sof{8Bb<+g
z1T7Wj)R!6Wlj~>0+*d3o-lGdbYzGW%$IZgHgl7dyjnF;Uflszw<;R=KVe*3}`&uk8
z(O(0d-bl!p@Ed*cG1kg}%G_OHye(E#znr`<=lACd7gi%UC%*ml+{}RFx+LU^On<OU
z+J`*2T<hB%x?b2g_fkV*^zde#VdvEhgTn%bS4NC&$8*EWSCZa`m%C&9YB5*Dc{{qV
zz_q8bDT`<42OF$cl5eh@z!Y-1%m4IJo+r0Vyp!YQeR03I$kUk(_hVjT49=!?ILr`x
znO$B=>jrcoT5&>cDRiR6#7~xL*;6H9)ZMu3Y_04r(!Z<|7ER|_(idnbOu5;Fjq<{l
zZ)`!`Tck2_N1KS~EqWKU`E|*1WB-#4ux=~8R?Zl@zZPz)IwZqRtsLG{HS4bP2{V9X
zwT%e{j39^A2OI2YWAK5`C#H&0?mtN@o3JNf1eRYb;v!&cTI~wT^$T^Eq9WUTm@QcZ
zDw5z5#(yJEb%<nFtP=Q5L)po##&6)mPl8MoCRJ|WI80|R^TX3@Qp)Z`1xYbqD4$uD
z*4o1LTPo5ro>1eu{rQuh&lItxLk#t9{JD5V)}RfohVjj+evy_mn&p*a*Fy}Z(q>=M
zkybAC<ciB4*_&r7t-DeqJ_od|)fO3BQXBmtt}S&BhLYME&#eTI+s)9bjeY3ZfP^U%
z>Z)`AzRZniMKgHtuvQ}c&5cRlHQ(i(-Px5eEA8n=78f7yU4!d=da6Z>x5NeOro~qj
z7x~Hj(e6ykcUrkcAO{+Dh#@|>H8JCQ{fbWfQBK$?cfbk&&PXQ>AJaYcg=Q+*fpEZ`
z^)Pt$Cpyx<LTd0`5%*=bCpGhZ)%}igr{07ll3Hsdbxlhfc@bK|t<SffzUi8yxw4;<
z5y{*9$qp_`sXoUB0xKo7zed!YIuEHt=@Lu*6_GN<emxZB*4q<j=9c>G_yg5<=f{*#
zr5qyrAA+0H?*{ZaTwnvmT!e_&WcxhS<K%-}DUWClwYY)ZcmP3Air{3UQ^@WS;!oEJ
zku(R5)uG<&dQRqfvJXT-S%$yzTf8yM7P$Nv1ZI1XOM3J+9A-;n2KIkg)Y`%X8LBd$
zC+U{&{hZ`AuX?|pvsWje@m_lhB6IGuVmYEJffvmA6#>)G*fO-`0PbCB&lW__<SNg<
z3vxmPueYiyJvu+RG2LYxtv2tG>MM|8LJ$eTmX#{!Zl@PDk%2`#!Mq;uQa|qfmd0mR
z6B=r9kPYQDCw({K0E*{QrK5lh8>)9}ru!w0Yipu-AyUDz`#Rf#n3T@%12&<U-5xcq
zg(hb1hS{0pz#ahKifyy&^O%<lm`66S2RF_W`OPFf7YBsRk=rljT*Qj9=3-S7G@lL_
zJ7Qknug}pE9GFXQLlR0cXPr;2i=d1TOIv8P_DqQM$wICatUf!y2C_u6%mD@~t$4^N
zfA&yWc#k(<QS<9=e>>v5GdmvArfS5Yw))(;tx2dVDB$BL7oV9j8+{w|z~UHS%y}`0
zT;ACh9uHgKldh&c$`tlCGmI@%;)@U-o&$>!k-TL`ug|VF#&2E2cxZTJfMQbP^~$6H
z3ue!;HWLLtwftHXawSVd?+|Toa&F(4_broITFRl-3t2NIu8m5aS@Ks3ag+jczB7Vp
zv>HgzSXT66pPh>dabFtwal~q^99s(Hsu2AC9u8N!>_1$B`K%a#4x`oATkHzeM_VTU
ziex_ybKaM7xB&ulx<XL5n;f|~UGqia5uz~L^WfQ;BVK2gz`$~bE<xmM`17KG4Zae9
zx{d8BW<SQ4Zqnxs=HwfOfnxl>9|s<x!cE!{?Ke5UbP4>pHp~T#`A!RF-72PWgddn3
z<<}yQT=>^TR-(ex{8J8~qWBWx<vu!9@&jX<B>`5{2|XnSi)bJJbx=<I_HVv=kQuln
z$p$)d9<cuN0b6m_#)+}Ka9jF5;6;r`0Gn#aJdbFv@lL%640PzYSxrnT@VK)!YzzXQ
zx(RF?UvY;nA22x`0EjBCnWqDW_YwHzj(q>-jy&r#@5{iP(oZ8oL(l6*=0JckGr_ZS
zE<dWTuuLvn=}6zMfq*~*J?3%1KxgmSewVcXfzJNydSnW?2PfbjH?t3^Y66o}<obdo
zd~|pS?9o@mufwPb{_lYR@VsyFUFhSGN<I*EE$TXt{}W(Lm*WX(6)$iffIUiY`enc;
zod1svSatp&Xh8TK%THkuz?6Wg{MRQi8+I^wCwN{+`QQHa>*9577%bnkc&y|2e|^<I
z$NBr^{{yy8YUdvQz=5lDt=}|_{?)#J8}V-k{`>I$GhY4k$=`wNe?Z{+J6QcoM*jaJ
z+v;9iE%!i{1$U`VK3J~oJc`%k5HfPB#hVPg`z-%|@W;HQ=l9zJ#_@q4FP5rr4!==Z
z|8}daX=2&kXF}wx_%E>HhSCe)8NhRBtfCf`>Kk?Z$}A^JnJHTURDBZ>(XR5l1wK`x
zk{HG?CglWJ`*`XF^%#~qsLcqHz4Px12R+`Xtw5HK`ry=o0(QNST%{{@9+n>tIvg+`
z&=(Kg+g|)|MDAVPg#XxLuSXco%6}HI5^U;6X(ai(hipv*R0z|*-1QG*6>iKE*BMLI
zWtPO9q27)P64DK-D5MBLrXkb(m9335>tHAT%Dp+c@R9ah;Bx};G9E$LjoWofAn+MB
zHZ~|2#1Gyhh3^$(M)7D9<}L~9F|RFC8V>v|xAJ38oAyV>jL3fAz>g^hC}X5Y;F>rv
zv;{Cvi_;2W4AnN8D&8&)VKVXaj&iBSRwJ%AklZ$D^#ofHIwz%CpJQQ6N8STA_A&_B
zW?zu7aF)1zCuNtR95$nT82b##T+j=*U`}yB-}+Amk$6J5{j(3C_W6H?2qWh+985UQ
zdA9S-+R25O`wc$0%L5I$716%K#mV_DuTatL6{zc+&wazlaB_J)+IoM7V7k>rWDd>U
zV+RjKeb?q(x6WVTJxY@vH-8!dL&H5n#>P^2g)Vst39z9=r8W2<`bXC3nO@XxanpF@
z(xWn)-5&zRhjZKmn#SCI5<ljwgpw9-iR|dtGB?LeiHylDrS<p2K_GCL%r}yn3aI2c
zozb?>(7Bz32$-ndnH7hcBghs6Dh((C6gW9KT@M+RjZ?w>wKzqz*UO^X_F$o$i1uJD
z3tm(haN~}LlToTMe0)E5Hax0(m5iNV9_I92NWYU$1R-1P)krsRCAsSb;)>9S_|Vyl
zcp-{sG!OI9Z~zfm=3D&d&{ro>$T3Le8v9Q6-Lz`u)7+)F@I4xy$0M+FQQH(_kg)_5
zLVl<C_{<#i%Pv+Lu(6?1v@_xlptL~9FwgXW>9~mY0-K^0phEhMjlj)@y1k1S75~TL
zF!60DFtqL|g}f6;seAh96yNaizSob26F@)>LaiBb0v!?mM&jULLs9bHoB~bA79IbY
z)ZV4B`MyZ?y<FH@4Bmgt&6L?8va9Kb3z@9mS&I*EF4c^x&JAd#<={UF*jU-QmXocw
zFn9#TJ8iLbtHr}_PRb_ec7e_I2Z2&T^Sq+5^+2r-1~u%_#w?9iUwYV^zcgc#IJzR}
z;ZmIAUn)#Zh`xpmn>r}+wo1>2-eX$oV#654i!i^77_)rWnz`w*?@w~py)l<u6q6rH
zj|*siqe6aHok^FU+OZB<u6>-rUAG$@z8wuO@<ZOMcU=j>g_Oc&8w+;u6=C@Falqhc
zO4wFYyv%U-b%mAieMT03xnU*_pT``@^DD>nz}9S-i>6sC-mXsU`UV!#B&+20^<rqt
z&fZoVG4`(ij~C*p-JOJJjrCXfqpC|+c9(cq48u1PW9Ji8m0TSS9N79&TAyk~H!i8z
z#<PPRzp_VJLF0PLquM3GV6X|wZs^mAZ9+JcP~_3LG~w(Xs;9bKv#3g>>(U_J!w;i*
zcqwq39fJ+M{dK)g+_JKzTdlF*`uDr_;yR(>pH7+@338H%`%Gfl3a}8oj=Oi#yylE1
zZ?L|CS)Vq?$-b)a{jG2t<h(*&iEPVyf00PbJMyvkz~5;4ieBQOS`qE1&L@Bp)=yS=
zIv@`R7+e%*CKh>0(8CU8>~`yg87#l=62?!4tRO`5`WRYHHBqp`O_QvHf;4nM5Sk`8
zN}=>5XvtBv)%N#gTKl*>=8{9!iVw2e&;A;7tc_`z60X&~W!RuBp>vk+OT)n+7##=&
z!zX+{T|zd_pBE)$Lzx3mg#T2}fF86(;;j@zp!38zF+>=B4o_rG5RLcOME0*r5B3Bs
zgV0skrXX;U!1W)`FP56NEE;FMo2m~3zDX(asEXrt$``|2YVx8WF7UG$&F;hg-palb
zt!TALs&Gn;P&lic)Nh!#Gq`Au3pO$I)KDQRRs{9h-;ezMgf4GG{g~FaJEwyR*}RB)
zzQ0qkFKNSTlLSTn;Lm%D_D(6|E>2PYb;IVC(A@PM-u+cx9nXj3YTbnW?fCr$E(70Q
zpJLUClEy>3^B#NiVm9ItYawl0A^7;P^*8fPV2&!ZzxFGAt0aNqv-#s&^%b|ev+kxj
z$w7BCI?`h0SjRjNL?Q35WPiLcCyJ2RnoXj%RJo!rE*zpPdxVGheHL^uYMh(^&Xe<#
zZmy?ob21)I6b~Oa?D{QU^%@r+omB>MCB!Caf4)_b?)U^Z5)Vdh5ySWU=I)4#nPpA&
zHIj12#w|&?JJHbHhXXBRKhx#Jb9@apw!5HVz9)N0Rsq(GQGycrp8octX<IP=S6!Go
z<I*v+Bp4GWeQfv}C_d=xqeB}anzRa8GxN3ggS_W^<^6iIM6rVce@^f81Y=#0hAodL
zwL}&zbRvVtQGI@R#e_o}JYh5hVq3vUF%sixrw{$=i!N^crbyJYLdkLQ4X=WtMZ&g+
z)uyH*d4<DF1t_uVLl}Efe(Rq?eU{W(s)fK!Zh-uLk4=-)y*~X^b0W`y6r5Nw6!pVy
zNXI%}W$~?jBj(w{)aw}8GfQ=2?m6U=BLRo}M7Fb}^SJz`4N3BI*GDV-rgcdw@@uK}
z&2t|&2HD|}0?kwGkJ4}ggvLI<Ouum;ubpXLd+Bj*P|--0U@qYN5}_w_<+Q9J6yn=w
zN(d9`GbEf_&MsE>0@9HCd_@<Zqn`C9vJXDyv5pjaetNKD;RU(7(M2vG#&Z01MjP<<
z))0yq`db<R6twe&9YeR3)0;8wf2zEdYUN(kJj3Bv5AIx{2B<7#KA@7nrZI?^L%aD$
zr)0Y8d2h@nXd-TfE$!h`Jf4kV)yYCDfn6O3g;ygwdfol2r`Kom{5rHqDx(a2g0DyF
z2oQxi{2un%<{^RamAJ?-i|6^)7-x0SBb+g&yuUa#@0pQB+@0@Sm}6jIr9MI#=`*f2
z9%Pdvktz-H4LBr4VK|7C6HU%ZCW`M%(`Pm+!kMCk#^C%J(JtS`l4AENIlsJ7_~wlu
z?;+FrK{I*vF2njmBaeglfvm#KwzUDr-9nW3q${VYRW%Q>Z^0de>FaOrs+XE?*44Kx
z=c6O4yMrfJpDACXKQgxoLIN4!MpaSA)bOMDiN|#L@St+b0G`GC4`SFl@z6!XyfaHx
zSXq8Tii^3r85M%FVP%@J7+#`-dg0en*S6;Q;5`7ogkP^v1{WQBTV(!49(+n`%C911
z;!1V7%Q2aeS`6Kl7LgGB6eIR?wFxu7=b3@2oAV5w4BhS&ahIQe+&5<<5n<s3%SSU6
z-o6>^bQ`R_$H5!68kxlr{T<Q)boCGljtMchySJ;MyrEMEIU1)@K#ie!^h+Rc-F>zw
zRXC0ue3Ei5VBAY*GE<F?4rp;Pv`q!A1Aco2*W0^U0Q2GEyc93X?$)%@8j?g5C&~o{
zSd&NZz<Fo)!!3Xt1VON!UFsCCcujOXS9UU4C+?z#pU!ySqwbH=NjKY0_}lGm=lb;B
zh_oXAarbb44+8^_eNU#cvw_O^-&(5jXrK4mn_o1sgPSCTjQfiwd9k4|e&kh;YwXf~
zGTqa`i=%(^wm8zyqFt1Zwz;&GSf1rMZq^$r@m#Otj;cD41kbu(WZy-U2=RRh&fT3A
z=dEHUCRU^}@3p^jcQbBX3`K5M=c#A62^%%|<a+Vv(Z4zzP1AuuaK}YO^S>W$Xamxx
ztc@Er^T|#XZW;3V2$0dI*BQk9y)0>h;&~jqxhbG@AiE3zJ^}-LBr7+$Sbf>4(>ra}
zg=;u9)Vq)kB_E!uQ;9*YWUjb}&Yb049BS)i>>520)G29B4Dw;#4}we1&I-Ol2q&$O
zDWi7;gu?oKZe>6_M#lw;V;5(W^xP@&)uECUUv^})R0%PZmPNEW9)a5<H7pJt?{smA
zdruj8-IBfqHW7tDDkWtV<X*#^X_p@Lwy`iwC*teNV2S+h03h#%tMAIzZeitTE>v}9
z0F8g+6uRjdzPDMPJcPWMFRbjzYnsN9abIdoM1XI5v%YOY4QAI0zH~=qanW7k885Oi
zAFuQ6*%;6WK*`*KP4~@9jnj7sZ5nTzrUeU3>{UhOSb607*Efczm1JuVf+QZ7HPxIR
zO4V`ew@T8s&Ku%`Zg=o1nd2|8gBLYEWKGc>UlOhK49T_U;KKJ(;TzjWJLzO}Qs!34
z$!xNRfkchk-u6tIWy%V`6TfP=Vd8Cg_9^{O>ZInvs&-`p*uma73kyL!f}eY_HBQ-F
z$07-jxO|OV(j_)L%ea3pqSjR$^&D+Pi8g%uz138fR(BA@^~1l=UpVuWYf^s=;7w)G
z52p>NOru>vokkD2D40a0U|JPAm>cz3Ofwd9q*l`+ECr@xadr@q8D|;V?F*poa~gbR
zfq=0AdttfWH9b3P4K?BCYalWy%FPvewzXD~lu(i?@1pFLEIf2~npZ@3e*_n1qBPPx
z?=zt|me}sodQXaMrE1uo=Qn@m#?G$3iBW*%HLLL4BX<m&{O(QEe*a16B`vuGCx59k
z$W1wgT@ImPLK+`dLfEFq+u%Sv;Nwfb(<^^OG=W<|d9eG+fNOoh<N3J@353mFde-E6
z(R9F!ut85S6_XhOV}Me#7-&%bl27J6hqlk`;8DbRJhGxLz+t<UvgH?eF3I`*Q2~}s
zc|`L)9cU*g(pwk5=mbbCX}i(AO2FWLV}12g9b(mm-`-8|=&9$=iP36~RL@V?eG^k9
zWF|zvva0Z&=-F^-96ZYTI75AF+(+xY1sA6pP(LBuja$WGwoxEf?X%iiDeO_?Qxd#e
zI<1CaL;1JcTIrvc(!HJKEuuP#ys18kuQbqA5A-C3J^lgRn}HtPDx|msCDBLOP+k+?
zMOS*mW^FLu>B&VroHdsBcYBGuy~8>MVvd^^Ct&xrJ&<I}Xhych+vOp41pB?2A)ryD
z``H&1BQez)MnxiTxGYnXQx1U*JSSzRDmdaq=Bt<6{Uu)~yWDY}t`HNQ(ofY9aDOAw
z@xwqiWvB{KBFtgi5NC8@S$pF&tTSkFtqEf<SW|JaINMvBq$YOUOcw^T1zKtzoT%Ug
zwAjFX;wRC$XDyG1gWLobYqFT=*_)NX_Pftw{rTC1)3gTztYG=;KABI7{AoWocNROI
zMUVAITUmY%0a-qa_>j|U*ifBl)3r?2NQYZbhP6A&S%N@Ghrf)W#?T*DULNQB;+3|>
zW^ly6@!A=_doHp5lE~Mx%3_)!g23rWm)74yi`6<JWFf0h#nm3BZxyc0c}YB~Nmlc&
zA@}$`)NjoiEDk$N$A)f1%w)Mbi?X3k^d$Q|ZhoreG#UZ>Y<qd^EyqFOrWdnocCA?1
zCfB7<`US*`)F3SRQW-jhi_^Cx{SMGc8z<<sR>(QH11q!0tTp7a5D$0TD)`n4)rl#`
z7x>alsAP0cdWK0WbUyvjXEX-PV=y_8Lh-L};vpoNJVr>Qq#Y0K+w^{!$a()F1ajqN
zrrjxBh~kq(!6LeeldkcFh8KOQu{Vy56#SN{;MRCXZ4PG2&*uo`zJyVg)j7;rkm@)>
zT8~tgNQrLq`aYwD-HAJZ`gH1YIPq`*1~*U>rMg^OnDvJOPR%DF^%2**BJ7J@mo1;R
z)CgEecGksswW^kddFF*f;C_+KyU?BQnuox>isk9avN9%T`OMG>?))cM3BMM0SgEIF
z1}F#A{poWc(46oMvUXugY;^yI9HFqP<$JqklmDan1q+P<ZTP#d1IDLY6M-H1J%u^!
zM?iE52ehVdX8w8AxJRiU<21fj5K|~mENIu<6y4gf%b%4_a!5Rfbxw?Ecdq#gx7`B(
zu{ehtRxmRFHpPPHQ{z53p#r-VxHUDggN5bit*;|yQ`B0eAgciP5wy)@!6fhNaO<%-
zwPj)HQH7Kzd%kCLkb-I6;W^CiK%3(zC!ppkdQkoK16K`)7?%B#1XEEw@+&8`#4c~E
zI4>-ihB$QfE6GczF*Czrzg!HW|AgOszd+Mkj2(QOtB*QIY~YB$FL!WWTHv~)P+UFG
z2t5Em0=%uyNy_YcF+Ao`yn3DBdfja8j5Xmn-*#F!J?)|Oq+hRjK}5S~nw9)LHY<s3
z5z_n7h;~*n%d;xrAi(A7`ip4lY_?~ixR5JYAQ}brH>lbA?zmXfGDXL*+g5LA(pxts
zmSc-jz~HL;g4|(^9e%~YiFetM4LCQ)@QuDEVV3~NAg^I(!!!sn_)BakykkaMdnn+W
za^W^Mecu4J6BOWLH~`q+pGeVw=W6VXp5*(ow*pjny&G~TjoYXx70I{GwAOJXK-WI<
zM-O1Ld^=2g0;+no=3s@fWJ%LTZS!#{lrf8)`2+hkM)Dg4x_(iF{$+}al@rl;{`lK;
zf1&>CCN0s$%n$h~1$ZzRDE*tb>(!k?pJ=h6fI8TtA?|w&t^p?)WPJf<i#?{_V929n
zI<0%iQ(Aec+tBc!c$6)xX5v_Im*qVOl<<TrBGzyi1mTn%sv4#IrUbFAgu4os#r%9d
zU3-;rmcb(DBG=Db->2#)t^m%O?kjU{V>ROdY609Ydb8W5|AUj2g)HQoFl_fhe6Y~~
zF8px4I1?UGo74Y${t`Ba;#>Rzo4R2gI32ZYd~L<_IN!aPqN;K&RKhz?z(^e&B5LDz
zJ|iwO)D{#Yx#QsyzS>f8Rm3^PiMusTb*Jk?Gh`w9mzs~8g#x(`OC+RKUyla@N1*GL
z^q*I`<WH~J?0&N0?w|0@<uQ2EHcOZRfhqz&Nj+oV5xgs1&h7p3z9%H|ptyd5U%~AW
z7ORnZOSrYJV$RMI+hT7c3K{@tru%Q~Gott%IS%w+fGGd$zRs`M3wyqxE1tIk0AEGr
zQBgyL6Y*0S7iWz~!5x?TwbOvNj`sE%98g$5y$P)!wYa^ET+0Ol0>2}XPjY-g13GYu
zQe){|IPYNEp~EN&2jmm};ja;hCK+v5{)Ei}lH!x6d3fZGp_8Nx<cj2s0_qdrFhBu5
zi#oY*v|LI?fG=>UI#<!^?v8D)7{IHEoJC*UY8qhq=JBu6TN}JVUv7!HJCC~Dk>pEp
zD=?;781AIXlHsO)%U2I$EVN_CawFQW3l4Be=SDDgoGY!q)n(TPfx!j|Hq<#QxR0Bl
z&azhrkd&*wT24|^r!hS$V-+RY!5!=ZvFNbv8xtETbhtULfPm+|108>^40dMGoRKnj
zYyl$HUB0giy0@kS_2~Z1Fd;Abxw}TOr@D?W{7_cwv`HcWN&>&fSOdVq>CbPqF9Ek4
zJcj{1fBc=ePW$oqshoU2^Zv>^J}+IY9;NFQ*TPIKDk5MQtOOx@utr1&VD!Ey7G2ZR
zC7eZcL@d=uwErBTWwieegp^x)F(c@87T1+7URcL)>^U$ciW?2zXWnhI{!Riv;J^Mg
z*r4+LXzLpGjtSsutrnZgwb#W9H*FIEDpFOv7c<+m3F+QyCaI?ZhU0a@XOCu&kWT#E
z{~^$J+RE$G$w=zCqy-k$V_Gwc+?uF2*OP?Jjc5Xg>HYuCLLPvQd3JfuLS98ycccQy
z?b->-<!&Cq16fKliG}QDpHJRP(=6G&Apy|MgtHX5snioz!f1D9@lkK=CZkSWj#3u9
zzfB2f3rTvR0bn(c%^ME}5<A+~3+3D<N%o;u%O&3rB6MI=tYX5`GN6TwxNEAoYlXSm
z6KP<I0Z4RZtN77<#Lc2(!&LSYly~9Nx`#`9HXu=#C1Mw}=Q0z62>mx9G6OxEn8QJ%
zhzGI$O%FE9gsz<yXunLGuf2A+6<_X-&sE!G6594QrZwyna9&z4+w9l{@Tpr<07G|n
zPllwSfIL<|Kxo~pojz^ZorVE{ScS<U<>qOgXmZf#K<<$Br&T27mAD^AYRGEh)?90I
znK%3pJai?5sOtaiqd#m)Zl5a0F;efZy(%9iZKs;#>}uc0Eo?Kd1Am`VWOZ<qJ?Iy%
zc`Y%V=Zo8$SP82y$xcRo*=AcXKo``vZj^T-;Tq$3@A9vf)`ks6apDsO023}<2Y7Y&
z*2dgYFQB|e1;N9DH%A=O5av`2kPe$%-a2)CQvo=vXN<&2xl5odH~^V^x-C||vI}oD
zbSVmwSe3rm;Vf?TsC}*F7;OnhU=u3Zeju%MV30A%$SgY%%%M}dQqR0Q1O@o+1>CXA
z1@L9{LI~O=8h3+~eNz3JuZ+tXfQgjJ_axoo*Zal-08erThtBEzB00U-wS}{9W}|&`
z$X3PO#r$SJd&o3>edo#IT2oV0_>P#&*n*^02oMEScQt6`mmvC^QjrJa&!*)g)wuq6
zyx$X6AUffw{m@wOfI^q5#kcD!;Z0)h1h1EUD+++@MBpCESCD)$9LF2{Eix0d2nOkD
z&}TI%NGvhPx(NgtiD41hk<0bOu<I*--W9%auQgrs>HtQG2`T=jLLX~L<>We@4Een-
zr8PY%{;`d`bvuhlo<41}Q1n&`sG!X6N-_(6=sZd9Zi3HchSpmJXqxW>l(U*EDr@&@
zEv${+>!H+@vLO`w^A-H?!WhGMk51yPI8tJiO!kRW7n4xyx%yF>{>W4~(@)6z%lTUa
zvJC$Nbk?~Iq2;WMMJf}2oT>*9bQ|iN!{OJ$JKv&ZvrGff!@mK_yO_U>%ie<UPYMk(
z=rL1nJTx0TsJlRw9iUaHE5R9i*XRLS%ZF9OvpsKSsD-x7Cc+ObXEGeJy*{rtZD#ok
z>K@+oX|ZtAwr#ArH1;rB=#U_PwN#<y8#&|x9AT4j$M!bB!b|-wKX8Zd`f|;{vy8Cr
zT^zt{PowKaT&c*Ndlc;wx8%9eeRt2uv)9Oz?oG?7_i<{7a&CjH(xpyx6GbHuUuKP0
zyE6Mwl2>}JsW}dixe|ppE2DWxI&S4ww*9Wsg5k0^3q?P&cnI{Zt1p2<M;0h_s9f1~
zo}~Q9qi5!UH%ctKW!=m<@dv4hDJmDFWtczER}8R8$^n#n5`nm8327Ei$a+!(<}+Bu
zw^Cxu9%L@qSxTtEMP30QGXBuQ&>JMrW!k$-kc1&8yFImgcKJK3IM=%&#U6xM0D5Y@
zo*A*?31my>nXpFpq3r15<HMK9LZ<x6n(-$#Q|9|Kpak@NY@64_F~8*`CxB10&Yuv~
zr-!(;)pZ?NkW7}f-2HhMs=^JbOA|rnbqseE&T=&J3M1^-eK4&H+HcrU-{*Qrt``Z|
zlmh?r56oMc!BZ%XI7BbNT%&cUA2zFk@_>VIpo?s?zw@f|rAO#Y-zk0xFt+-RIjN*L
zSH<6kdfNg~6M4a|anl9u#1asmpK5Jos1?*2H7x;O{MX&pqwYl4=os;5C5g{I5@X|T
z^%I$lZQp=2#!yYWGm+-dT4O=h&fESd{){X^h6@CDkJ01~_7q2rVr}-Q@5_^`C)Es-
z=Kwz1MF}U)#eL~*u6*G*ku<q0rrh{Kb_S#S)Jf_%(C0apl%ezlcKD2TQ3hc=p;*7L
z;KWW%TR1f-&#KjeI@%XHMvD*G2WWfmZYn(?Gill0;SILP-+3nqyAC8e?&qP=s^bs4
z!{X6Wc|Ir$t@Gc-RGmS9Do&1Nd5O#-Pf`rfBCE8_Ni0tFvZUT1#DhJsHO=#96f%1{
zHK}U`NyYCfn3)!NGyN>etf<H>WN(up6fpk2<fLU$`K1cnEyIzmDl0}U8I$6)Jmi+q
z))E&=IO<4BHw;wFOqI$$BVZ+nh{u6B@5?1TdB$xqyw`*qmJl41?7&g$H6$Pj<)ob?
zFYqY2n*oL?KtiaQa|X^=itNKxq1`Hp4n?+?P6pp;0!flxIl%q$`P{F^Ci9kTOmj_!
zuI^-ptb^~2v9H)67s4*%*pm9kh<lqItlp~5{X}uG(_?rQ2z?i5{cPB0oS2Fw$6Xkc
zL1fwI_3!(p<|NCu?NW<8OUP%EDOv`emjFusI#7Wqs|zgrsG9A>wiKPTQY|+HplGM4
znJR|4Su54?q9#s?R?zQYs;%EJOShXu6Ow6{ctz6Wzl0LJmz^4QLw9FHMCuO@aNR!v
zkP}~KurDtqe4t$$kP)2y)t}lLn~;B;8@o4|G8&nb;-?b~bWfmTwV6eY6js-4dhfAN
z5GSiYcF4(l6R2Qc(cKMT5GmaN-jOZ&>y{xl$ISv+T-}DPTaKzb&?r744p?$mBBRbz
zymlTQ;JF;Ch^R@Sg!d!#F#TFO`{=z2IBaUQlUJl5!4YV(StOE3=Pw|X3jFs9l95_G
zd7EL6n*_TcitA&Fg#beNQYUXaPXUO`$2sQV&gz9Y3Oe~r$8u|@LbybYD($O6*6lc;
zlU;OW6+>uoCrB#Yh}*n6*Af8}&w2qKLO>$FS_4h5NLFuW!5zZf(91Hocr^${eGR_r
zgKHU04!ttLV<!~kW;!ZZj|LJlKqcN(upFXw@P<`#nY=43dwODTR#-#EeP9D;+Dc7%
zk9vV-!Y8xU6k78b<D~gXMZH`Bd5AA{Z(BBCVnIa=;2hIm0uj2fkLybOq+A>$+dX8o
z1gHWT^ZwfPQ1iQ#PptF8-KGs|c-J&xr*D4`J`JwKZp|rbMkF5-4a}j;89ag#=gz3g
z+GR-qQr?23OqkD;-Y($~SHn|-venuN53PD?l~9cDq!?FKoEXrdjzz!en$BpwfFgde
z+Thg&z?cHpAnyZX#M#<Xm&e9_SMp0;9Cd07G2Ir%&}}1g8$?-2spV;-jZcoHe&){)
zpk9jW3s@WZwe=YzAXfOO_r~hh+QuA$#VQFH`v7ABdm}%#o5lhWrhex=jS1UUc+8@^
zS1AUYy9lu5Ws6VNO3@4Ecog;p`TEY!H)T8L-)0ocj+ZR9yQORlDi0bX%<xO_iLQ;0
zezA-y*-8)UmnB2$mmSor(jy<sFtCvE3neGM&z5+VyL>hAOH(&SMz%cO!}#0J<nD%u
zmdF~FE`qAXD%!rkbT`5)uSuD|yRo+vf)?;Ai?Fv@<al;cM%BGY#moRd@l}x|A-48>
z<x<kl!{VN+j~m_UMv6)!#f%XQ$8It2K6#u5pksK90rcQ}W_l>J%jaM;5d&yKugC%A
z6D`putZ5^kVl4Ete6iinxSq3Y(S0FRI+;PnL|GXVR&C^wg(B2*CIR>nwa8g|nY*Yy
z{t<gDqWLKIHQe=}L+=irGe*34TWYBtn?n=;n@9rG?_X+XmeR%LxPC+RU;5&FQxICP
zMP=e>Dv<WXAo1409~Or=j3<uQmE5CE^M(g5q+QHgbiZDv+`9I%cQBZ5x|A{*{P?H{
zmunc%X7BZIJ@S{51ST`p(SiG=qcM@3%6$J1ZS^LI`m=CeF8F&)=}ep|I>RhrdT9OV
zd*3+VEl)8KHS6dme;2V?p9z@<gm(Z&eg~xctQ0~IpqK`PC#VZk9k1>Min5;0fvpr$
z`KjuFAY)z}n5Q!!?;x2nC`}#;tY#E?QDZMl)_9;r4<m?K?}wKKFJ&&KRD{~4Q1X)&
zHUBJ2($i=RT7iP90hvI0ZNoAk{02tNJsjxybKyn6FtCX9n-D^*DS(`f)lUZ-^G=P$
zM#_J3mcGeaQ<i2sl4&p)+v&H@AhwlqWC5S4n69@4;H0u($hN&z#MSEb`?l4G0VRZq
zwbWmt8So!K^>kr6g{5^%<-f<%Gg#H!pMtI#e+O`$z`X^JG*l2z>;i<2o!_Wau`#PI
z`PCuE{P;uanF3<j*|f}m#4gQwpSe{5^wcZuZKtQxCi&7M0m;p|Z*GadM0D#9VAHxH
zILsqJ^!@7(e8c@<7SnNuAF>S?(2OPc!7GiqR_}u5Ay;kzMicyf>;y2U|1LoO>$T}=
zz}U^RZfSr0`fpm}ulfK&e>SXi`ycoE*9+!>9>W#6pfQ%o{?jcKSkeD{ozAb_|Mv?}
zK>(xj6+Y?zm;L|kf_82|C2jxQ&G0`?9?+s#1O7uxtVjIk)&0Hvzpekb^Z$MJ|6}y{
kJM{e>`TuVc-|GH>Si-{xKXJ;22Y?@~+q%D@Z`wZjU(8sPF#rGn

diff --git a/docs/figures/widedeep_logo.png b/docs/figures/widedeep_logo.png
deleted file mode 100644
index 2e7fbe6eb765aaf238395a8c79c0bc07d81ae517..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 85253
zcmeFZcUV)~*De~6Do7Cpr3FDjq$<6`Mnt43AVm;RdJPbINhm5C5uzZygd!jyz1M&U
zNSEHb^b&eax$(D6+~@ngd!PI7dCq}{hsDa8bIm#CJKph*F{g$-)>6H6k@+G30Jx;~
z@V*WJKr#jZTzE-AhPzXy=sphsTwJhIR(`CetjzJ)4P<TSXaxW~42y{)*VFA_4qTZz
zc>n(WD~gAd4U_|v9JdZ0e=>MNsm781DxU7jX6M4i!i$`ohRsjuNev7XM}iHDGscRA
zzGtL;2`6Q_;}@hlHtjXzw}L+IPC=rM;fU@8sTBm^y|TBC9k&K4hk6uOIQ5QoE6>eR
z7X;De$E13S7usYSO*$!QX^H;U=!M1|dl89>&z3UpA0lV5`9Dbhk^%y5Q_f!rN`}97
z;k#$Ln^npRcpOUKF#nZx?b_^v_#4WX?^i?L4RW7!UCVhVk|^!Wz_u1rOb_tB4C7J1
z5NH{)YE**~{YJ~n`hq1)kzdK1^eekRy>_7RWv1@$ot^B9O0Rn8{CEMDWM4NwB2F~U
zldxB^pzK1L9Sj~3R5J$ran^f#n7)G3*{GP@_;)XYsiPh;UJL8iu*7a#S(5V^MctEe
zjA{*bJ$Gf~CAQ#B5IR;%wj}D4Fsu;=dw#!}^A=t6NI`09RgJL|oxyDs%a;2t=Fa}S
z#yxHEs|FwFGl#-onPg%8pSVxmTzP9@qg^V?b;rB2O*J=;mow&+>K4(At}8mG!lnbO
zQxu<XlEn3vMsK7ysnhKCyvFIrc0MozMF2Y6@#V*(0alO|znyjUfNA{ow4p9==0^wc
z`xdDRag&<RwfeLjHL}Jg+E*O3RNnwHbJUJREcXHLxlBHa=)cph6xHU|VpuAc{2swj
zT79|kkUBOcg3WEOZ5#RRgExy5MCitHF7NXr_Mf*As@obJ9|g0h1GE1Eh_Y*}z4HVp
z@&eLIB8uKGX|4XS5TfQFzGx26OrxR>x?cR(9+=p<iK;`A;#H9Cd%!Z0JY5h!#|5h9
z%f*+k1ivh%cO;83=lM)@-h9J_L^OzDj=CC97|1e5<4sYk1oWnS)8r>itVBn{sqFrc
zVIbt)4fXFdFO}Kf1FsOP{6+cr-bXSu4$^XxXP2%jXFa}E&XV^U5*(>)lcxTd#~rvH
zM3yG;!{!=kv^m+A3rS5PFT}fpLgsC~1(V5y0{7>Mz2!`Z3|h$NADmG8lQv%PY7v@$
zaYZpVE&g`E$E$B6_LN?yUw)E%`?~%0Jc_50c@Ks&pFQ6CZ2Z}Ih-00BIXpYl$d<31
zMVgK+WI2;5EiQdDov;7AKVlWUYP`yRKzGLS=3RfA;Y*XdCK|Hbm&)jL|K@GqY-Mdp
zZl7vCcrjy9V#qymW#iqYZ`|gCCE+C-#giqM#`GO|Dw$;7wakmU#!7mbT#F3tYQDNq
zxj?g^wdlNX<^tTk<1JUuHUotJ9XDq%-?dA=ld~hS+qcU_a;G_d?hyqS1%=P6ORwZ!
zF@E*<3;5#I=Le#<Y8f)a*1pAjyZBOWS9X_vx8UvDw?S{iAH8~OX!ZKh>&_Q3;dkWj
zz#oc6k#>ZBysz_qB>L#XGqqziP={qH<#>4V!`>T3Jfl&4QSkQ)c>_-ip4Q|M4=?3C
z&5OvBcxv{9;)=z^(0i~bZQdN#ER!5+9ZEfhJlYu5=MM@p*mFFegbuITSK7a@H?rRt
zqWSz>XCr6cmRnRIF;HJIKRqSuaNvo%axHC&wwdOvvVu0!!u)zRwIEl@lWxgxtLD_X
zgF|dX7Q;-r2DxxOq;*C-w}zldbWM=YE1$SipTu6#UeWpTiox}q>Et6Ni`qAFZyskp
z%v4&8Y_rjf&4ydQu@1A2UbYY@kA1EGYUHm1wr499WH}<)b($;MLWvN|vkOj!lYhzR
z_?37@96jB7!S*}_z?R#qpHe0rQmgS?Xxe@H-n4G*1z&eLTDcO_gc-XACVmgb(3p-q
z^Aw9qYYrRa>wE`Z8(yohYw^tAu6?;@E9~v~P}Gton>qG+<#n3tYu8^h3*5~U@io^F
zwq^-`S9Pt51uA@3EKJ16V!(0$l=q@jtVg0iB*5~4_`ayIxxSO<B&$QOc%SgBGo*#M
z(xsfvNYBWm>XkqWe{1ZdfQ*f~b#?dq?$4?HRw|_nhD*<ejSdY(3Y5PX7)cl*6757q
zT&+90j}m4bS)J}V9gWGBXN{AM7nVAe1X$R-Xs^mIt#i_@Jgzcxo^`}dG!H1P7Pri&
za!ubIa*eP^S~TFX;YrYK7z`}?0507ZUl9}!Ve2(*Z(I#_-SsrucKf;%ogeKGxj{dw
zF!g2X-V{ysVc)%!TgeqR`bBj?GJOG6epTdMT;mpTs#Q6SJbh&o`?+!HaZL_Iqu@v>
zo)qx={m#iY<K=+v^AK2mpAKB#>?0@;^Z^tK>=9}!8bF32f}n}e6zt<bi>8IZ5k(Di
z7jrdpCWFwfl@Yhl;N*~4vJ@(AQdLp|GKEVe6dzufzxtZ4l%Dh#utz*qAYu9n+1Kh9
zNnhDRlimnXU~dt#(6c|ey30Qx?xtqxcSX(~G8FXrxPPXfB?H(N-+KGy#7o{1wUUYw
z;DZ0+qs7`qn>L{cL{wS~hnB6@w_9!8+nTJYixM{b$&EMLKYjgF2i9B>O!Tu$9jWj3
z;aSmL-d_f4w%u&M1$>D4O11cO|BY@n@1)L~6M2*XDvnr4DZyNERgnLxytaq7faaDq
zSbOrLILF?Vm~gUI`GrTj8wX7XDQm!cU+*1Ry|Zq!61SW$iz<s$w~eCisBxU$R2fgv
zQMXGk_Y2wx?xA{P*<9M-_E6qnCb>nCIV=C7NUPV|R1?@#-V}_=f{Fe4r^<(=tM;iX
zGZXbIr<hX(!3RRiLe9n+2+O6Y)U4#Jx;UjSo!ANi2@})edXP%fTQpx7pNPJxu~k`~
z`*7W`YrgSC_lx>h4j@cB&!4B;r++DSh?q=FnaxBnL1#-wAX_69vu{aM;=-NYc7-Fp
zEEd(FLbhnHD!ePyem+~$Y}XPQF%}UO;iWFCeiWA%vmAHid~tfnu<Z`t9lt6c%(D6t
z&5}$NUDfSMcY{}Yq9Cz}TGx%ZM^Bbg?jZBtI898BIG-0kFGerZ6tNx3cz7}*QT6Ke
z8#CL1(-Fhm!%JdgiBpL-elL&9vEMNzo<$=w3k_z+YDXSO*c7yIq^9yH@F?h#hf7L^
z^PKrWGfSvHt=OpXFw)sv{0fjP^lGRX>LY4!nyXoLBC-@Vf^|ggS6Uocs6;%rFkO5c
z{W-TzD!pdk`^Ro2V^ypL!Vp`Ee1!?ii(coENajqvBPr%9jXiM5FZbBdm`IqZJawY&
zYlAlg2rihkpU&+w37wcpPWIX<1oWd9!qf}nBF(JLoNE`{vv%_qhb=}~i`X_?)@|#z
z{Toi-bVbDW8l~ho`_9OMyN*X*Pfkr5Y<D*f`baEqHxzy<-1ZChLm!JE5s>%i=0E4(
zs>Z9Tz;a<-a9f33%rI*7jdRX+v|OW{0sMhnz*rYD;4}$UB@{r~_r0$Zz9Cn%RkfjT
zcTT}&u{D_~KqyN{m5d5%a2_zpS7xAqoj+_JPCrR+94#Hq1dTKd`_~~;TIwq2cjkGJ
zWiv4U+h?CoU9mhzIY)i2a5#(~6@0F<JB)JZs+t4A9+W*$&62$is=umu<ARnZ>0FQe
zCj~7)AmjpIAvp}=YmycQ4R$5QhVl6BxVjRjWm5&NwgO(6TrhXIH88-5;r$L(k|gb~
zBO04AmBQTG)AOsNG`p$&Nz6x9`fP+6U_uEnQ-7eUYX3kLF~ZRyPm(4E$|wRdtSbRL
zFHO9Wemw;h_o6^4bNgZtLnnRL-&xH8Fj81bnTu@00T03raH2QX%23T(Qxk9pcTE8x
zCAti_fV(2XeFKP?0i^%A1^^xru@J6xh;ILWh8O?{wF8j+e&!kO_ph(_xF1~W-@h+J
zy#bKp{<@6&d6h<tf0|@0?E?P#!b{v|03}^zH8tFCT?;oWD`&7R$UW-&RypnlrOQJj
zFaW@K>(>vFn$FD~0D#!tPS4QYP*X$N0^}t4!V>h-O3>TM<ySudS#N3FrIVHW3l485
zM`y6Kw;bnxo{+{}|GF*2$?=~@+#TdN4K*KgD1+RrIK%`+1%)}~FLH2j$huitOY7W!
z@VhzgFF8(IcXt<QAt5g>FF`L6L6Dn`&|N7hDIwu|Lig?o;GPfw`#8J5@D^|ebN$l^
zzMuP6U<)@p7k4|5GsmxfU%Ui)xXW>J{(2GN>z{F2dE5PYlQa1Dv~Uv?`t^y>T|r@?
z|EZa~o%R1qvtOV5)9gR<`sa1BzdDnC3bt}n201xdIlIgMvAFDi-t@nG_-CB|G<<C5
zZRKcq-wxLjjGL4E-Fu?KznlHnkN(G|Mt_=0ii`id<-b1pSIb{>kk)py!wvMsFC)s|
zl@<DL_x|hiI#ytiqsOnupE}#Q%isIm<iFhh-AY#I*Sh>`o&EvAf9~Q?Ab(L-h=3^h
zi|f`_)&PJaK<&Pgo;T5E168r(xY6wB^ooZWSWG1GdW7lf>-Uy7Qi+K4sjY6uu|1rh
zdz5A(SxF`H^!=+(FE>ASmFQ}{)QQiF|8_Gys(SP0$LH0zT`p@x5qA>JQ&21jkPxdq
z<T^-BkP$ha0V8WxlDGY5pKTzHN6$Cq!M$5Eb?3?GNs+42XS2^{A*ds$+?h;Wohp|&
zg(85ME)YP%0U-Lnm;Z;@VfF8(>P&R0@xK?mn?AnBdeXZ31(scHEj^1|b@ajFB44Io
zyG-Vqt%JL}EDIZ(rs`lpOTVozR5Wp_?7+OdDnOVSIL#7Kb*x%c`{Jno!nv0vHnM$j
zw_E2Au8PHpNybLrq+U?Q11+day(!qv*waJXkYbZcHIx?fbG3KgEWY7jF05Z5JZ{U+
z#L*g?Whq*Db*=kis{Ow7)fj_BhhVJb0m{~9W7Gy_dOF;}cO)<AYq@g#K*k{T`c{Oz
zO<j!xQFZQ-`b7eeYdO+9)S1q$;r{~og&XEXVrB;N;?nE^!>c10uin(UW6jK!BUKSU
z4gr<E<Pc7~mkaE?_WH4(cspa6dsU~VQoBdAP6swcp;TWY9>C1lB20a@(2MP!Q19}p
z{lVp<8)fH%AV<PW%&kZ|?X9iOEmR4&0K^w=ybrcN-fnPqFF&-gfj90;Si>96zuQXp
z+$}ml8KJw4!aiNSR~pweZZe6oobc{lcHRp+-9LxHtHn!U@tE$D<rpN;2{z-<T#0tm
zxk>1HE}fFj+|Sls@-KpYUtBJ3n$XQOCdZB(L|0*tR!`CG;CMc=CeFA@j`-1)qsD8`
z<FF@fGq$KRRfs2Q^$_u*xzeoMJPGO<bBRC)$w>gBN-$Ro5#bIsOz}*s!cP^M3R!5y
zTAwA<nWYk2yv-eN7e_C~pAIIq8ZzIkTf7Ixyt3_eOvL^y$sg=kkvoIKoVPJ)Df?wD
zbuA`$@m$kI$I(<tE$4p&Pd!byc+7(;95Pq71KM|0Q?Z!w=G~cFh^c<>d+eGsCu7<X
z@ao;^%;|pQv*I4EH-9WXr-#FsR;%(YeAjf>xQVE~Oef3l516GuP_5W+dj)g@y0+;L
ztg=k6?x)*zif`!Y*E(Z&bw{D#b4>{>4p}Rs6y;(_b}PJFa1xmzik0wI#i5FWzSw+%
zYY^73)6m{me}H`2e^;e`q4UXIiSd^;35Yq&Q40ld46@$?V%yrrhzE&FU&If{iKP6d
zeSQlW0h=7cbi|B?lPU7x^E#M3WaAtim2~$mETJRYSK_3_sg&<&gMa;(`kY3e$?#>w
z;kSxo=Z38?P#MQ05Bgx7z}A5jpDH3Qej=a&K-|KKvtd|3`-B7#vN{fZd0K-s%nyDU
zy9>N6iS#r=Z?)55Hnv8mQ5%f5bzY|2<)OdHf6nc)_up9qOC-%t36l;=ryrZ0g`$th
zozd_?a#bYjM1_!FLv+FJe6LLE*>ZnR12!BMA*jv({l{uetvrf>TMM(wg@5dp93Ci_
zTM0gxU%@W#+H9kG&Mu_7GBw^G3Y8MrfE9^BSJO(xCbFkR9iL9Ryz3mX^=n4)NE9`K
zR;b{pQU32oe^pDD;%7+xF0jaUj4*!y3E5xfM+0af@116wSqV95i0t#y1>@MrEkpwO
z$bgMZ&-+I60@3#Z1|vigD2AexD%qnVP{aBMXz!rog|&w!-2bdlno?8T{Xc*L5XU74
zLPsb1fm3IxncCBsM*m!=uBIz0W%gP0Zyi(5-awZhBedA3&r7DgWDmFeC5(GsvHsJQ
z`E8O3{Rhf5o`np5949!Cbd<gBm=BI($_!Z)>SdD6hwG~cm9on2zkcBxTtL$2HZFI?
zq-(~sFx9gKu@KET65T&tP=(lY@B%hqApM-L@yoQwz|rJ+io9z;@f%D{ft1~*{ySHX
zA?Je#a|-Ur8v!xTjEL?8ipRwoJ}Z4)PqzTN+1ms<K7^rSwnpX8j!(XS{;2$$C#$DO
zI``(>%x)Tx6TtcAK~w5879ggXYBu0xZBoHyVWNWg0PqZ~{-NUKyVEfmtj40%iIalw
zlB(A%<|n*@WAeo}RrcT04eC-=?8@x&vNEB^!=LWjZ6c~dCm^bK1@iNx^?Y`L;tF|U
zcw9n1^t*`jj8?Yw%;daTnfqs(d5M|n)16_VfHOY+@1A_6+_d;vE^(%b4K3B)DJ1)2
z){i)|7H(rFa%LoCB=5{ti9e=eaUYK^=$IcC^4(K!_wxger426V;qhX4Bk15cE8i5b
zF8Gh+F=W7vO|BM;(MCZo7RD%Ko<-YE<kb(i*01a4DIgZ$E_%sd^p^TMK)$lmHRt*8
z(}EV6W5kO{F;YBL2wox*NEhrOb-jVdsiOJcB>dG!6LknwC_*Aht0?vO3>`|92=tQe
zliyL0LF9v=dwoubg>_nZsE`w$1IPSvOeT(HcYQ<X2uE&ZX!PjxX%iTjplKt+tCRh=
z!C6P&!aBVul%&f58Y6ervip*5-DTJO2_FWOK`XM;m>t_yS5%Ktu$rsK#|4+`6pm(Z
z!snRr#t3YBOnDOR-~~CQRk)YZ<GC4rAV78{*n*LPWfID6pspka%gLfh;xT2AKZ>{C
zL59pW%tE)JvXFU9??|Qr#;>Ip&u2}yINrw^#mz}qli*FlAuLIJ^mD=ii8|T`9?m_Z
zNsGx!N!0^$4i7rU!uSxZ9_m4rBwAZh+O~XtQ#lj2<KsF<n&57_DUq+C%L8ALXOpR=
zu<-3&*ZwceroZjOlo%K*&veEq>GS6`qQ72aQ3XAWhT58?L5G>#-_Clk8<+dk6g1k!
z!6svlha^xA+`X9*;qkeq-Txug4(u|}=TIWD2!yq)+`}t|CY4)hHRy~~g_TPLS0Gbu
z3U)9sJv-EfdV6{wFeAK4LF@iBhn9Y)V>wUTXkVuuxPrE*pgjHL3RpvAk5|J{c$?qi
zY`0gS0e*~voJ7QqOuL1yz4%3?j>Q4D66wDKM!1(_=hQcVe8Q}l-R=WdjV?dS?^Uck
zd@%Lg#t+M}Cq*A-E&c8_oZc$(sUV${&!1Ex47ZhYu1oNb(gFlgDdGmZh{gq|Oxo0j
za{(T0i3h{=`hA~yV|oTFZ*gF7j>hHusG+dqyVFFX$DZ!`R-)5S8D^*F<Csk@r2p`W
z(fh5m%BT=L*JRMm8Qd4@Oj(e{1N53~P{_)*|ESVclkn!mYu{Wo?(jv3zxbvQy<=|i
z^NNv*Uw3{)PX4Y!E%aar)9#F}CVhhF4|c4Tf&d6Z6$AuzkV>j*(f{$})TYhm5TP0D
zSh#{=&_l$ST{9aD={=)6tCQ!Zw&GZ0glW|t&X>b%{Pfz{#K5J!F*8uTZ8}Ok!!Z4=
z_{{lv+WwLrex!j-a};B`ni6Iw5!w(>U(w0I)(yTEy`v!}&)xo<N^$#H{ZOp+PTlTn
z?M4d_2<hcs^bp?`=Z60G-S?67Hvb}7Ll-BvbFMz@!NFLt!`nckL}EKr8spMa^^$WC
z`)xBZzRDp+?8R`K%h64a&rwGT?i&G~v+(8fv1Mo4-%Qc;fMJd+gC)REQVI`_H(GOZ
zBN#Kt&O$?x3k~~u!s|4{J6)zaw4XcV;|yd_5%*`l@n(a&In}{xTfTE%{@w?x)7ter
zwC<^du<s}zRD6F76}?1_pFm*Kdy4L(W3zJ<a^~!tsNAFBJ^IHX^GZO0Rjr>5U*9<&
z7hD0~__*8mP&{ImZw51DfHa*%zg~ayA?Wwon_E!Y%2kx`d=LAh25wy5fxT%w@#}-E
z?rb+3Yc)VrUVrqpSc*xlyioDvEw*hZ$x(A#9#d$5h=RRM<u+gc%{56%fv~@w+XCkJ
z|Bw%BP|M^oHiLXrb~~#;42jU6Gd{G@?Ya%J%ZG^-ZJzM>Ulr+dq(H3g9-K|%La@+2
z<sVPwX85u9BD(Jb7!OqdaR}!eg86I{1X*t#-yJG6?h>O4>p>snOv-7DXuQ5NG5rRs
zvUL12u6}uclef~w-9`;>tT9<)+2%^0<5wd1s9-)va-`sK4(!clrxT&WXgA;O)7MUp
z2^_EfWWRX0r<*5@61s^n2A)k}t(8VUksw|Zw1Xo%5%+pe9ZxbL_+8+3adFvsErjYV
zLr6VRa1^GYeQv}dpgPGf;A7Si<4Kv&IUMbLVsQH#e^9MAiXFOdfgG>#*)q!>Tp`8B
z)CexcK&M4#N*)q?Jj9W=?xQOCGZZ=dR3kR>J4b8J+Z)Tn+>>rUY+VDK8>-o%wB~}#
zWu}nDf({$F|H@!dCG73l${T`u925S9@)l=`oD29)T_{=E*&!+-)ytx?2U!BkD^Bdp
zi69v>xcW)>qZDiIq@F&=&Ri~<=?IZ4@8E{#Gsnv~hD&mnkYvKcnoPYZ)ih}evYZ7?
z43?7`iN4?v>y@-Ab#3USLpk_;KhN@037ZyZWZAV-E~j3*$fvx0<IG#NJBahJNo?7g
zW!G^JA-Gu6;}Qj<<B&1}Hq%HVw}A~S%_$)gdPHngW4Gb*Rffcp9s27p11=s(%8>f*
z6d9T@LfXp~RcTG7Pu>K1(hIWuW6R=x5{hD{gRS|Wcr0EcCc<u4#L1s7U@t>?bcjT<
z7Jl%n%3v~{q)L{(j5a9ar>!z0fmEf|9JXM}GkcS+;8&%Gqv>b;_2Zy2eC5FWI|+@d
znZHXF#8-3nC46A-e&Whiz~0b>zdG~>UI~pgyhA8aXb4_FjJuzG!-SukMQuw65v)i}
zWbF3ujN?mm6BXQho@slg_I&#kykj+yu~M*XTp726_=2kRs)Fu51^EpdRLpvP!T0ec
z1fX495F*|FiXd=a2L?<bJv>|lFk6Em4kc6fNA#JTPR#Tns<e?a0V};JIZ1ERsPPCm
zqo6Z42Kq(fZvaRhoL3!TCv0W4`tp1p@?*=~N_)N}@7F|3_}F8!#2NKk5H2h<i1WT#
zthzf8zy9K(iYSql<>ae2#D8L`A`TOi=LezkVC>0{Mgb@5bTO)r!^Zx^nm$?fOIEYp
zwAi3s7GUEMZo`rd*4}P(fuLTBwSM`i)*0*XRRjbnF-XcwVi;7}P4=rNKFOvm`@rH(
z)nod!t$P%5t(z;>DaBEt?Vwt4HS{}-joO!Fe&Ih3Zyrr?QX26m%~edZAwD{oIBD&h
z!mO(IP5XoheLrw~=FMGDdYYRnzdSinIi8{2T~*;!rUH)#IZRrI%io<HsqA44c-dIs
zlq7KW&EI%Z-1s2uw6fBva+P7<$!XFN2k@<S8pzLKP{8`+R2u=yzLYy8lfI{5cf6tA
zsLzGT^vwo@C}d}+J<{5rRTyM&Y-@8WF?F(RfmWkvfUE)er1oB+ldr4Y{%O>-{3~;o
zIt~^gID{ez4J|EVS26(fj4~|-X{N4VI>*Z28yjrvtPLdfFs9`8p%(m^?`hpOODPF|
zu~T<EASchgew_5%4#n2Qjj|Sh$dqpNgW{R@D%eZmY{3SENSFT-`kGOoC()y)6q(hN
z-5{}%XoV8GgGg!E0{ZqCr=ql`?R4-{8c`1985?Oy$;3-fVMpSk9i(ZJsxZ)mukS?+
zH|A#|dF!73iskMV>_e_SH}MkxgcCM)R<sObv6UteB>!9AH8E2-7R|@AUj8E#5}W$_
z`|GmXZ*^x@$7jf!9@WbeC+<0yaeSEYZMt3Qm00o)`~F08dzz(T<jT&wu0Z@s2Oa*h
z8v8cuRlN5SF-XcvVkBe@t?!mc2i;rIAszTje&P`eYJMeXq)w>epeXSsrlc$QUp?3g
z4v&Bt30)9TP`?U<VkcJQeKuO_s%Y7z_HX6z4D?Q&St_GeLSlI+Q=G?LAkn)gHZwbw
zIf(g#@H9sJ>jiO8FphH{m%wWnI!U#;ykH0Rs{Uo@;XA8wUHh0kP_{i!T_>0q6PE}_
zGjEL=&v`n?vIG03y)NQi{@_8N(EY7TV&wSf1t1P<ie`V9kI)v-HM(Z9Z%_3;xABo&
z2BY|RNN(=}bnSGy!S6Wi>`gR&!P7vvwBwAscb7nkqCw}!u$d@gYexG0w0QCf9hdG4
z)|f2&bbdekl_8y^<TEeOEk9cr*5#CKCaIz2*4IA}|HUA*UXcWysz^Ci=e1LG+a;FR
z8lwZb>u!BRWVk1)fn@gGQ<`rPs=9qLGNh9`q42ep?dfyqlvVTpcVFjo`Ge6mjMuvC
zX!3g|W&d8r=4!r)X#th;wkj~VSiig{3pu8Z#2hHrU5Wdx<i{w8QQw;%Op$33<ISH=
zS6zjj%O6mc8YDfpu#M>9a>6`$GPxPPAs{nrJtJ-wm$G>k`|hP7YQ}1!7<h&4Tse{u
z<qtG)5cehPohKMfkccYq-BxC*_IZx<;k3iypuy>>DAk9tXv0&=FHd5Q-7s;%HYY88
zVvpT3hTe{D0$K3O?!eK+;OG$8djqdWuJH#ITrnA3$|3(T5N`DGj&Wx*zXa{)PXfca
zK|1+iN9Ka2yuu;SkilDXP%H-BzLmGVDj$t^wjV%TEJl|sKg8oh(Y&2rkEoQ47WTSg
z7^lxArgZ<Nz;eg1g094Mbt9WY#`Cw<6(CUnM!W;payO~oujB5<;71gwsA<j<pcJd(
zBRdxzEPn&o#V(-Bf_D2Hw<|#rq}ffIGTT7j4$8}VK!Z20{4aFF%cU6c(+uHuIX|*B
zAiAL)p!@i9r(ZV7`=K^XSDAT+<E2Zx@I~fvA57_99TYjYpEUaDI=&wW1+E{3WGO-^
zjqVy7>n2i~jCSV}hqn3UuDeA?>AaX7hvB~Y-2=@hJU{V+UWMU8;bD6s7hb>$yAXRt
zvTwG{C19g$-+lb%Htf6!%Um+M=ClyL)gTzVD^yPQ4e2c%V64mak<d#ht_li~1Yf}$
zheJ4#oo)M_?3`|fJv)iI%DKi$cd)G~!$w2=1R4C`R2!v21~C#<Qy^Osprj@Q_XDj!
zfHCsB6bIfO%D0N4NLxR@e1B!t$S+Gx4=+ZYFxbYHC>l?E6!PHBUOqL1za3%6zH(ee
z@lTxS5jRuen>OG|KqE44G=${jF{Ps>w=plpN5H3*(RH4YM@>B;Ru<<%ug7)f_*I5=
z=mf^qg$3lsW0~_C^$dft=BJvQYTrwoZatD2N^SG_U7A1RX@apH^d)oQp(Om6m{FPU
zN%3V9KH#B7f7!R2lVJDzPW4}>Wo<t?<jp;MDp;qvblK$2X}K|q{F^p&msctw>pi=H
zgEiV^`8ENEP4dkW?Aj^HVdbY)g+8|YuX_f<W3C35nep$`LpepWw&NhJ0O*OD#G_lB
zq|cu7|M508oVT5Q>M0?3n`jUhlv<!w@!+EoWues7D&w0yi;lNO^apC4oTFKC#O;$b
zalRJQL?9C8U!-)worV&xb6hBY0umAHc9~a<)R$27k4c9<TI)-tWdm3-JI0^j^G5=O
ztp8vTCp~a7Ipsh~9B=;+PV*Ocv1DiKamxF&!OK4qLUw9%uP5cMZ&GdU>)P{Hq}(z<
z-OAvWhzb4HS6IRLs_8x6q(7u^-kHL+3?NvAgJEoDJb{f3xXh7|#Ge)`uFcOM#mN=h
zm#a`<Ed}B-_FWg493q2q0TkA3?3G$?Mtq_&#T#NZ0*B6-$6yXTI^SqDrAqF%4=xLT
z4DHu>EMO4hOB*dZv8)}l03`Za^j4@=HZ{WWJAO0fzmWxXa?|04%penTOq=anPX_0y
zU3pZVvwS4?M#mNCNzEM%@*}DWgqIG7Az=t100p-n<W`YH1fln?sn=1wM>{=*L6y3)
z6=meb5b%;6TSotP_e+f9%qhJ{#num~pTl2)n#uSiOpalWvNAOYEJ>J%y1Ac?=a-z}
zJ)6HaV8>x^+sy`Yl+_b~eWk&7>eKgP?`&Al>gm6QF6#u@DdtIvB|bfJZy_GLbTm6T
ze)h12`ZB&JUv3-_<u0ta;7v{)*ThvPvBgBwxlGSGc?HptRckV?UZ*!3RWuR9_W6q_
zX>2(drNXJoS)k`yOM*J`<OUWBz5iX`1rP^Z#<?GlQzR)KOKFtdh3|9%dqSiyZi?!X
ze{^J|cF5NaDATeU_A}QM&oW&}inDUa&TGfs<g!_u%-*Z3tY~3yJZ0Sd9V1s$06_XJ
zHYswvYB&E(5{aCWN<P!IWK7=$QdGdNR6wP^#h4X1Dul0j=t_7`9^9j{Ld}mB!X4@i
z6qeq;AjEPgPAQKlfFlX)fQ!G?u8SNhecbGZ`+%f|XQEEa^kOmU&)??zq+nMLXKrT2
zHqUn465q)`1h4e2`i#;Of)n}|)%?3NT!!fVP9vz|3hSeE&axQ5Gn&uW(=MCfs)c;n
zXWs-WJ_kG5ndJ$|yPBQIjJ??LTL|l)b}y;NZ!`apA(H2u_&0Y2QnJpBO|mt(W0ffL
zrNU66#v0S>+P1^uLc?42+@pSlm0dZ725guQx;-m<J1nqvU^!G5!AU44;aCgDGa@U0
z6P6cO6;_M{)|7&UUbpXsJQbsq8`n2lr%NOcMPbXchSv4;{rEQQDjmO?*}iE)PW1M|
zMn}~NfvfRX;F|p=#ZK^cZWjxi6`oC*VT>jWHVi!He>k8j`#L8wEvwQwwqs(R7V#Nw
zU^0Dvoi~(_8Df6*g6)>##P@Q6ySS<jtqVED7|BE1_%UEV{*CM9*2bXG@>~JTKJ}zw
z#kim}f<>YbRcXdVu>ZylT;4A?C3T0;&#huHW;ALHLRvl{Yy6h?d*+>SJx}uSJ1Z~4
zJuBigak5+tv60zd_Jr|IB#iq7uQ&-+JVpS-z<|xp?bRAT@-@4Pi=m`nWf>Vm9FNp~
z)Rb1m6xipp%_$f{j2Aonw$9O!H%ST7{=XYLK9KaUZ3WM*&Y~P--`;Jb6N7*p{gS86
zm1Aw=M%S@Eo>!y4^28{cCbu5k+x6R<sK7Q05jGiv4u7Tg@<*(Ma36HJx<u?$2C~M!
zsYV2UpRGEichr=)_b6JxV2)N{w248Y2$j6?>W?I+#}(8zEc!m+<Mp-MK?MZ`@R>x7
z2daatoJm>j`KVAT*%fixv>u%fdLKdK2K`$gu~{0d(q5A3^$7WSX@cvR2atFNI|Xb?
z-TPy0b09;*;rhlX#3211w)*A8^l_#yyg$X&ead{Pdl23|lb@ieeb^oCp?gd`5{eKB
zPcKaRS_f1wmM}yy99>jCIALYROPcx1-l($}M4_Tl#+;JK?4zA8ojLD~^NN{h5q4cf
zSwl0LCpN+t7LW7tN3lUy{?vQVD0b-k4@*|Q6ObO?v~7E!YaH!*c=rjm+J**x>OfhU
zmAvWf)aN%>P}l-}h+;?7qWLatKQNL0XHMo8I6*bpHg!>p1V5e|^u@5lX`i0Me1+J;
z@zmN!&0?_pR-XPZ@$#JO5nlwMCPP$9M&hmWJHVR=7QZ?LYia54Z|4Ykl*g|L{;#RI
z_tMb_G_ld3=l0JzBQ{(0*&=;n=cg<ng9-gDxt%vyj?)GV&IArssRT1!;z&dIIXC6H
z5E2_jb3+pAvhkl1snGe?=*2azSZ(dxC7ACr%wPHvZMm3Jsh?#}s`;ZIJ3v;%nt{S;
zr(7LYRSiHOn_~T6MFV{l>+eVo2)e;gJoo*;W3&)^6-wNrK)fz=>UFnE*N$CDUA!pY
zsU|||6YLEmEh^%N%ZXgT;ri^2>eyP+-y$w3f&=G_T+SDG@SzU3gCYyDT7loGj=lb~
zDPaXYE`~+We6AVhvcnWPp7mCCF3Z=jLv388P8-(SDu7WWgc7r%0!|2QAjp%62w3Ab
z*S^tkGLdfptv*YwxI)*doxe2qzIm5t*b<bjpR)dJ^CWz6X~8SN<{UlRzNk%8J?XOc
z4;F&{dwuW+g{&)j-AsMdQK{5E{It{XUY>5=nPH^QFyGUZf$Zt79_flOj~?+7*8P4a
zY@$>26aIGpkXe=@Gj#8pFamEmaO39OBN5nwmka<RKjt;vqN(X~H96Pi+4>`4><(1Y
z`DFn8$(EB-gA3SZ1%1Gb55D5zO-I%#G&cAOBY@<M+T10HB5WPpZ+kA&*R309zSpEZ
z71a|Fk&`qeULoXBI&a4CP(x4>Az@ax2ResY3>l~p*je*evXtb)BuS2c1DW>Rx;Kg(
zhTO)e2mRe#DwU)Uy7hk9Be%dNvGQ@@#60r0R;hiN(>26a$H}&Z-_a*70=u%PDyn~p
zkh!Zx@!J=034KJ9G0unUA;oED)fwbI0}6g6Sh@IYpzOmXofYxfojz*L{I0Lag3%%@
zsh{tb<g6WjcZzQ8=C9qmHD?z=f`=wH#qb%(qYh?<p$l|*IRlR%*0B?@8TL<&#A@VK
zC)O){%ud3i1{=Ilg^rr3RXnWFslGox(etb7yT46oL#X-&Qq~VcZ@9_!HjAjoXcLva
z>}$SP=G_2FPwBLWL9Kb&Q%&zfKnMuBZ|C5d4!*V+6mjzxazRu)1kLi8(vWUR<(%^z
zx!W`*`a1aCed@Frf12c;2fE=UK0m%sa!-yq-hx4W?6-pV{T#iHLnJDzpq`%tf;Zm$
zQ(813AztcBWt!KMQpK+YhcGp9*|_6MfY4O!p}BZ%YKzUYQlHxe0#N2S75V;1>`USg
zgBhro$fr|?qJYg^3FEoL%pA7Y()bQW)PI$Zy5N}vv;{Gc@W;EI6tcklb@Lu-o5gUa
zj%L><$ZqPkPUoFPH6yIKtiudm)XbygvFjB;&T8%__*7UtPSG&SX<N<B8owNh=C&lb
z;{PuB@iFv3gqEh!%67E((3*I(vXZ88??V#oS>Tqu242~YIsT%L@#D}keB3loBjNvi
z&#%400qX4F;3oW-Bm}W3RO`E?O=rSVPYw9zBn#84rXx}*ZLNRp|3?Or8XSSx*fAYu
z4Swfl-#}P5x;2aR*!4w<lBsS#7L<cNQ_!q&ktw!>Z*mfO2(}`;av!H8CYNWqdg<_d
zO~l|m1d-eGFepTQ=8BI&QmGS(!V<^5-uldJ+3_8MeX3%7QQ$P!!PDt@$LRp{_IA~|
z?z^SlD7*+Op8r(>Ke@DUfS>9$aU$B%dJp7!mD%`&uee;uQgi8@_tv{i0j^LrU15Rc
z0#~+IBC05fMEGo<kk8NM(Xjx8gyRzU@HhX2s{{fjTWigd1qgI@)4Vx0L}+^Av=l}1
zj3(aa<89L3O|hKQwJ(l+;;J!Vsl4S=qumy$$Z5cM1PjywB#Vh4>P%M0!^@VqapEQ3
z47lSY)c_J_HnxEMb7bQ|UtZ40f>fVuw{=pg?SAP>%<@F^l5IP*05wzZb!anF?$rt%
zGtZm+i%^7W#c}u6%PF4TKefc0#K1-bb{6upx!KF-h5~H4ONzFz)hea)k%jSbHwy+M
zJ8Qk;=kIqsXy$o%#)7s}nFbP+kRVM^_2oKAo1O%GY6l1kx{NzSXN>RxVVM*H4(7VF
zmD7s*?VPmoTNFYCv@P79P5YcYpG|z%xIEF5BT#*mFnFZKhPPLedLYt#a>}nI0bj2G
z{K7s37*&`*Is*p#tyu8zg|}wNU^XYFl%3djcMB4%9UzxHPRRAOc#74E^|W5^;q;Ep
zQkC3wKbI{&%0iX*cnQb^3ydXVLUv18;605)*qr$2<|LN_0(!o9KDelQ_|$=3wS3KC
zTy{O0SRpHY;&v;7n=Zk<71ZU4O5G?rY2#bm)gDph#-9^uQo)hR7UzsuFJ8S7QCuUU
zDs5O_IRYtwp`P{8oHn#&rTc#;o%6~G8J+UBVl_xmb+e7jXAuZ*i+VJy_opHken8@h
zIFUNNI@!|kXa21uD2vy~#-r~tRqd6Z$~Y3*tINgq@A&0GXFRjhSTPNI=Xv=s)YD)G
z9X{TF3^}*J@FQw{!Hm)Xf2<65M(9yfs-<67-zMT9J#*YWLorckPZrMTcSjQb{{MGD
zzkMHi(Ak58SUPMV&$rC+z?^fVd+1Qu<;rV9ieE{(Yl1=Oam)sEZKfYNouqlZ{v+|V
zpX@^BRWk?q`%{<YW7H?sEiUoZO@udep71=p%7eIcl;SXYV)*$Z9<bFKB>qePJ}>4%
zNii1fsEyu4$H}{aq#H~X^0JpwIprRqgTWB%Tyki=$!ILVsC3k6X_0Yh7wQASH1L~w
zZC!<0;~cBxRU~0+PVg#GLABtpitF7!wD_3>cg|`|Xc}=o+1C@9s?}dgl0lPXC1`pi
zAV_M#(v<`6?=HCVP?tAuSAh?~7u|6nXTa`ryzPzZ5>qB@1DbFXyFT?uC=7SNZ%p8}
zWp9})RW{JF&R|!73eqQM7r%ZX>6xP2e;N|?B)MRmJJy07>qC4A0@JR92OQ5F3LOAH
z#e*JF;>oV-K8{$dYLiR|)i1i+xWS5_e(MiGq9a=Tu&%z>3P>RSyDQ`CYvLj|jqk;h
zk6yWXXKSbLt{S-qI|MW~fyU0%fDau6sy+Ljl;Yv#;zN1D9E8`OxLkfN>aw?;=RwY_
z7j_#4)TgiTPd}9zmF?B%bg(WAR^?SFT+etOzth*Xs)A~Kd3-htu`fl#ja;1WSQED5
zO-ykTKoHrm)QP9oKt)a$kobAMnamXSbR`mI3I{$9zag>|QyU)px9>+yCCQ2>d6iCE
z^@TkU{mIq?#Jw5m6AM2H%;$jFiF5Qt!eO^-WJFk{f1lV6QrGw;eRhi`YL1qLCK}F;
zQSDR%x(2*G+Qp|J^D`{(JH}1tk4`gvU((qy39~Cs$cE#MV`o_<m#xcQ+=9QSUK|*R
zJ^k=6JD78juy<RVVfJaa>yNh`eLXNyhnC~@6_ASscA(hM4TKo8Ho_M<Q6(!qaW=cf
zmCnmO{z(~60zqAxxD{J!M3@k$jZRXj2?Cg3SB24HF(9_-a~d;hab=4;e1~k`>OF5K
z4GeDfh`Yjjq+?>*Dy1Kkw|a)-ZiG(=lnU)Z7e}_W8y(I&tBlOID$fb4C#-=p^c$bq
z8hbo}a0u5bBn4oKSf|lpHaYDeb;&C#_Ua8m8LoGM2GLFsC;7N0?1bgsropz!>9XpB
zN#tU#7$W_5l`-fA!{6!|FQVtx@tI^`6M%AT&@p-i216{N2hqTiK$h9Q^LxzkV_tm$
zYd<^9e0xTE6-t{XcDqD#2-#f_ZktK`Jxh<(pSUncVVJhn$&;fxi_UL9pEXd4IJloE
zrB9ndnsj6^TytY2)PI<JOfV5US$TxX7mD9R3*b(#4%gs?3=T@TW0jZK@bMeRY<eW9
z^KtpRa%aa-^vTRI+tMsu!TQI3;sFb*o`Q>?#Q6R?%{aw=C}=?XZCE1x1d5Mzs;LR5
zC>7H<(+)MQIR2<|+$(6y1AKiCo+HE?uwJy*gUtc%?p^X1&$RrWZD~eN1B;uO70eam
z@H^fA!<k!gBC2vHgs`-4ABauvq<Q6mD)N9<u^Y#_pPu==>UsGjGhr!4`tFikg#2Bk
ze6D8*swkogewxug;?-d9fW%fdO}vm-CaCGarZtj*sFefmA%Z1w@;oJwbfWF}LFTBx
z@4=h`YHl|K=_8!Pc(XD29gy*<ksV8oA=CbhzE#%r`T3#`C}k7l1xnpSH_g}{oHH7R
zGYxqX6Fe%giIvE#=HEN+igVp0{9~0KHln`gB7-5w6ZH5z)Ws5bgDtKpo>SMfSG#s;
zDSPh`+PssmC)S_&`T2RY2k4p;_FGa_A%;mpHPdX~u0muM!i*pDkH2trf2u1Q1g-@T
z+i=Y_SBCVj3cy*}0yb20$Y&219M9Z+(*z3d9iBefx)&fm#QaLU5_oQCch~!);LuaJ
z!t(*_y0;rrGjj{*KWvG_Q7Lo4k4T#w!Vx?GNh}XtoM0C+LhfHMp-v1W?JR5%W)5gB
z8zhIW)3&$Io=$D99)jT<n(mKCdJHmrfA(yFmq?v@=ds_tH?|#AC$SrY(FM>-i3v;v
zu8?GNdSXNToxr|)zw*IW#qmGRjoTb*B6hCR_Z@(6%x_iYWC}qcZYYmFO&IoMtG^^*
z+Gs3MyWn+RT$d@E<^ve45^@~Y@e}KfI|+~7j);V=wAmhC->*|w+*(?FLOAYu=U2g=
zROJYtYU6Sa7eQi1?@95{{3GNt&U{zBR8@<TBio8(+1bqC5tW5(k=di!gS!<4VOWl}
zpGTIWb!ApWWexo!Y+zI}TDZB);6Wx+4|EnISFD%_@o0Wa*kJgV#s2T*KiS>?Zg%*R
z+~mFFXBql0Kch3`3R=VNlb88>Yik*Ax1Gh0c2te)t=N5m`Bd1I<<elgnm~IVAZbnW
zU2CBj)xoc-@O58}4fVm=%)2Tp89}{uL#IDCX2!;=8w!laRB%ZZ5NtIzU}3G!f_Rht
zSWiE%a{H;72xdoF;Q-hM^G!aWFI|2zxo^1U@eZ!lEU`?B`-b(TC{$v`Sk{~Gc#YGT
zZqZ4<rHvbYR{k!<B45KZ|Ly6z#!jdieG4aA@7Wq(2zUR_E?c+py9GkLa9>vU$X=PD
z)7%N;!qP#&WVy72$NODdaS5p<G7IA$dPjH6q>Rb1Q_aLVdF`jZ!ejZeD0|{na&$HG
zUkd41z3Gr$uGQe(vhLOxSsla+lnc8O&pTj@Vl10N9-Bu@$(n*w<<Y@Xj)yB$Gy9`H
z=&zpg+sK05P<V%VI>dx4z$r>7@qTo6&Fe(>6n>HH!>kpRcBPdBYIOZ1r12qCeM06^
zanBd<XXx#*5mt(!V{psB#O3kbKCf$#gbQb`H0xK7-UwOna-V5`Z9aLj_Sd5qsPAmA
zcUz<$UVR6A2d^>v5`ccHWihN~5^^S1yXVmtb9^_wG^T#k$D%67pVzO^-gxhXocn`O
zH|;C9ne`}3Vz%~?6t_pgipsg_%4$7riqOF9oy`=ov2w*Qo<v5TXB8jId${3xIl)=K
z4&k^po85GqXv^#27CH$dK_x3D=}$Ojyn)>3%yIfM;kXwJf7sMVS#For!)?E+`sdO6
zUZ`bQ@tKYXZ9s+T#naq6lO2N?v}Q^bZ_hx7o)3yocws4UF77b;+rS`IB7fRibjndj
z>d*Qe;Z@cY9%8nm{nrb|N-8--;$~ky!647>$fxQDRlW3z&KQ0+=biL3+8Tsn2!BTd
zA>+DKufnw&ifK7{()L~W@hAR88)AlHVF)vITc2z~t)RaJ$`E<-0r{6|(Hec(C-1t4
zOb95S#kgMY>=RB0=aqu`8<;SoU@lY6tb3^YpWs(CGt>^H4wZPdt%g<4?^z@s4cffz
z?XPj?f36O_lP^EfmkzG+_@GrgmV1=IcsIKKm8Q>yg<R;88~?_ldQC60Wc5(5=}wVR
z6D|4^L<#!>C5$^I8n`QR>4TNycay0o(XXfvbKfvuQC%n34~w^VxhTu?<H}>p^939_
zMelTv*Z0zFSC((;B~n~fK+VT}d-2x?#+T0y24mhNou>>I8MJ|S_YX>M*(S=TSUL#t
ziakH^WJg=V4(n4_WaApNP|vT?7Db157b$yqp-7A~Oubg^Id}N>R0_Gt;Fs(7{myqr
zTV?Oe6$Nqc1Ov~~vBHqm2--!See!*tjnME~$llc#Oi8eCpR3Pc?+;!5yLSXO9ysJp
znLhgpgcN=#TWz28K&0E9SwaI!{p5tCiKSIn(5wC*h~uifn9T>x9_;J1#tFriVbNjo
zu<JdW;OkO{rKB1InUL{i__$;Zsu}mMYJM$O^GOG)8P#z%fA0%C)~u~@irsPAHVLsT
z9ENc1s5CjM-6O9v?{m6R-42B0Z19AAsbUh{`rQ9>EwOviSZ*Z6OJhT`mx}gPZ@*6!
z+CTk3|8c)vpp(9QVB!TUkEU+bR!f-=_8i50UE}UyCo*IwR&16|X>f6x10a%j;LS>a
z1<K_j&WC`77bdm3>()iy0V6|qxg5S3os68GB+w3yJIF&1A$C(;Qb=<Bw1VEz#aoz!
zw}`h!{)^lCCu(L|ndg@@r7iB}XxlLNW+vV3IYvZYN|Et!Ppuf$R6y#C;+!hGO?kRu
z`^-7FPP44hInG#YW1oH3>q&WueOXV{YlX8+Z1-_ktx!L7qHQygc2Q`N2lu^_*RDNl
z#$@cPFW@m=AN8gEm{l<!ZdqosHnC?1$@tnw3+<uZ>3a6Q&~4&14iq2d2ZhSg)XqNH
zapZ#z14?rn<hxc)z)@}S+uQr^R~iMyrq^JsCt-b?y-Q2BrpYH}wf4q^>W8_{>2(gm
zd}e`|oqW+CNImigl$v-kbS-dq@FQJY`vdvItINP;vjVkv1@RY{9r?*|Yvl}RRxtL_
zmm;=+zcm~lvI$Q#q)bR&OVZWxL(MaJL_LkVdSx?8ZzUJVpqbH;)==11-do{6LG(ik
zT@Q2J7QG~BG`hy%D1s%GfKz@v3vTUdjA;!oveA*xTD$dh^w7FR0UYuRl%j3qZM1ED
z<pbqDuTHsCNa{oN_W6DRv~+u7=unxxzs|3sOFPL70eU2ml&u@1Tfv+Yd#aM1mqLBg
z0y5QB5}UTZr6g)}_BrqekHG{_-P7GJ<nyYlDVbm%4mQIh&ei%!fH6r_lOgCy=&tlE
z3IF)UBdEIqR;*-f$IEah|5nt`5Xg{%j!maVl+LOGP8plzwi%34E|TwSi{1aJkwq&s
z62iAHv}jYbXrmn4aMhus59szGpz-PEy?wEhzI{WzXW#5TG}ORVwiw6H_dx79eR~-W
zW|pAT$qZa_cGAaSi2FQX#t+>7aoMM^uKw>8PQQJBrV3-rc4%r7r&jpp{!GXZA8n$_
zJxisFQp{EeqTv>@toNO<$KsW%+Tk+eL)WnUPfpT-XWG}(uH%OK!?5GGPc@HfFf)il
z&_gCZMU5t!akMoKs4jOZMCi?8UkTAL?tsJd;>pr8pRvW%QPBAQH#nqJLqV#@)Z*Ov
zf@4qqfLRxsdU^dzn%i)j(~Q_x{ow@7EnP9pRoGj4)DK!T&pUQ;>sLx#mV@RQZo_Xd
z%Gh-54=8bvy`ykaTe#kqg^Ha)#S4_=`3e2CBFTLA&8qhOM~g2KE3L;TJF~A~6BVP?
z%49_$VhPS}w$k93-j5F^G>ywPovd`<LN(JO)Sgdj)1=t+)-pZfLT48k9aZ8ErXjfJ
zG-{`$X^Phl!0Qg$a<8((_n19yR`pahD}aM%)trVbe`s5~M$w~l^wq$Qxeh;Q&*+Jk
z;gYH%kfBVH#diPM&mH_SkUXSERPC!R5|7A7gC)!!(wKkBiwu$x{dOL4ui=H6W##FU
zH-zQ3n6>i<<s=Jw>^x4wo0xUnd#9{YbkVwYIxm%3bXI;fi^k>+YF%*Tb+@<KWYYKk
zsZo8gNVg#g_kU6_xw+~#<V@tO<FvrRe4^qnc|#i!DfJX<juK@T0EeGQ3g8e<+Xj<$
zUVVzr%->aRU%RQv^Ze*bqfnAgQ6b~!2>-H(su<1O$(^E_`;3|o;u7Fy#=yz<@$1h*
z^Kw1Rpg><!ck1VLqvz;j+z|=q@QJ@qt#L;$c{Fc(Xu4N>Fq<bm+Kkm7&3w0xTBw&F
zvoP(KcxoW?74hbx!a*j@zz<?{&O0II`xkN~W53<yS)M0Hh$^JRH3BiwQpFg<=`-bk
z{QMzg4CqPSc$C@XX2f|Cb7jT;TX7i-PffXNtp+OUp|7vf;PnHm(U03@W?Ry2AO91=
zYa!d<2RGS?N7}5|q{CJ`=wU*sd|EQJ#Sw$>kU?$6Pn2G=lwMy;O&1Y{v1G7vosQE4
zMAsod#<Ocau8ls9<SaK7a7MdG{yQp>CsDcGUtB1FuTvI;m@McV>8F)A<Nq%Jmc7Q*
zotKZo1A3PQ(;Jv9d<q3HlzptxWJN<9JIM}yYA0C^n~S$x?njI5bbZdPaoAIjIBtBB
z>QI*<|8(%VvY@HMcQ4lz@VTRlra|v6A5nM7vk$$?M>S-a;no7lL0n9+ZZ4q~%#oM$
z_JQzubQtE3U2vFTgaUQv<J?0as|x!ep)uDK*6+sb>h76i%rTZ|m%Ag{>t)YB9gf+G
zye<Ob{!zj7&!y5a4ZDLuAKVFOTtHJ(xcsQyZpeD*MR%r}Xo6_MtM~1dpIOvI?YWov
z(4Xg-2fQ11r(QNn_kA>4klwAL5z&A=f8;l?)Eh8cXpA*wnU#fC?#UM$X-SMRiw5lU
zjV0D6TAd6hXIQ^vM_(s8Gm9cZ^Bgm+De*2MQOT_*A3O#&g?jFt<w~jaeDk+&K1mYR
zJL?#$^^2%imS>yoIs!e@YKykj=DF&h(Pklo+TUW4Mrfpm8i*!v7zF%yq`B44Q$~|y
z5_QOW*5PL~xFOXG9^d!|jR%{>b#9+fR-O$#uW#gypo-CZA|15}b_xMv!bZBt)aIDV
z-$3#psyK?cN$fhmK-M<PPwiT)Gi>lzV`;O{uKnI;_bc%}$`!hKo;{_TxKrOrTS-=^
z6#bby;`(}~xsqaCrRVH<wf_%k?;X|D)_w~g6+}fvMXEy15e1RnC6p)#NS7`h=^g3P
z5)}od7?CCfh|(bxX+dg~DmC<8lolZL5D6h6cgOR-_x}F4;~V3SZ{!a$c6Qi{wf9=j
zeCC|b!~1ei)EEZzC03U@X_y%nda~u6NnU&-=d&=ML|NdW7rZJWO*ATQ&>wJg)<JGG
z>IaFf7`=NAAUD?4uSjI}w>%V><x8GgOy<f=&S9E4noDt8V@+uM<U^4B0z(w6d0gNl
z0NDm$oRF4hY3ebL#z^Zy9dL?zgMAAL*jGt)W4luEW;{(Ka#dZ|m`;=RaK7J6v}Ye+
z>Qc)LNKV~aI0hU_u2*zrWj^nj6IrK;X^`!Z;s~jD+)Rnl<n--%o!#=Y(vyYZT%=L6
zx_YYu(B*hg{KrED?jsz;?WbkEgrGt{=%1R#se-7_8(x4fw1WYvDZSdMV4!lJrn#uj
zDaHvoum1Btz=={<gl9r_k5%QJ4S1`%h+$Cl=K*7uS8KvpG6l!y%VMi6QkR3r7KNw1
zfgK)Jryw2xb;+TJ-{Y@sU1tg`1N@CGYkN}a<1I+b(s;*RYLRtSJRp##u)e_H?>_+c
zd5uMe#b~k`FJGJ5*o+G&B!g)&dgMFlpDS^obq*Xo<gD+uWBay~wx&W@;vtljL2L*y
zDCIXW38Fb&PDFq9__FN2O59rr)iDj~r+fiGh=}x*TP_e0EL@${a%>6rg;Ha_otjl@
ze!4G|@+%rxP9auy6xVm~IAkxV7da>N7_wCHxBpY#wC<Hs5M>5DJA_-vjFRm!LyNyO
zmG!{EuA{eaY4&C-ZJvaY(j<ot#u%s%Z6b1AD|a|N$&LaMhR?>6v;70gwwbiKHwux#
zx!Zb>8|yel!o?40ikUadyWIBJ>5_h}5beX6u?R<CajN~T$qGPJqxfr2GJeQ*uxtzc
zJD89$JL=Q>V=RY3rN^VVGq#^R%5E1;8-gShv&b5SPDaXV<2%vwT*Ru-UC{|XAn$V-
zS*moHI?4FkK^@4om@<O}SM-o}J+ae`U%PrZZ8!3qbS-vrnYHnleS6t*qvqVXIWYIJ
z{V7bzwY4_w?$`vxfd0Y*Ace4_RF{3KcmX_s%-o=1@h(zY+Cx6mE1(W6ZwK}PCNXtn
zT6gY{#ovi&K5Hpdz}N_y;?DDaB>iYCo8%(BBX#R0k39V5RgS(!=!d)`4r#Zz;ikK-
zN}u%`w4}a7a}C$05?c5^5yGcA!y2}5%(uP<0I3(+EmT=A(Lidnu4ilcqt6&h&#j6V
zMv|$yEb|$hw<^H;M$x{xOCwMVLA3E6Q$wsLEg;W6;FXwP$^qx~g^J1HvvmWtcCidM
z;2(YV3^IqDn^F;hgZbb*fDTL0=#@Jx1b*XOPhY*OQKlh9>ctPf6-2mkLACkzva@w{
zk)9g@<B8JgLG-w%u6bFS#C-p-9$z^3K7Rz!;opIY!nvk<HVUuUk9xBHep*Cn4Gc_Y
zv*me^p?zp5F<R3xQtp2{J!ZVLc@#(F&@H_+ScCs*bMYFOMZE1!zKt%V=%37dWc}zJ
ze*mX*rw&8se0yDeeCye0jal-ULkkw9@7p>&fiGSGFycOh8QjU$yAL8`2ad2@^<8_q
z43i30{seF>N;DrRW~=|o0vHMNV+#{R_gp27yd3nAR=UUym0H}a0FWQ)eL_K@HcDAd
z5w$1KZLPK=_ZQwWhP8`K6z6-%mY)xzFqeK~fEI}F7JoZO&1kvv^iHYYm5rT9+Z&gp
z4S?KwCHeb1KsxxpoP%GH3${UtE^RyP127&wwS*c&Y&VLQ;z#(8NcA$~byLPTQ;Gu(
zHTh;x{IJxbF7_bxaBODqW<(`I!mZ-;QQ>{uVVS#C9x`|k+hFVmjIT076P}PNhN==1
zxjk-V2<w%a?0#8(C#)*UtL!3;q0)ZHE?H=E^lnl7kTDa4#Pvdpb=9S;+|hSJh*~i8
zFpn*|m6UMHo*H%7^GF=p8_#Bby$)G8e_J<j9~t)}aQK(0W6|r7{4BUU`skOn85&H?
zh!!j1IX-)dQpult+04@j!c+VVIu2|i>6eaH6A3N&I(m{P2Hw^ipFKUX&*Rw~llIJ~
zqK_j@K(^{uRZ|<f$htL(b~f}ZFC|V}RcIA=la<Uu#W9?~%f*2Nj<gm=IE_{z0wb*y
zRhqds#Wi6hMu+#(F3jXh?*W2+Qwwk$uTf%WPYIYHX3oQy9BBm?pqfVJeD@)Y`r~i$
zleEun(vu;`4U67_4N&GVHjE3WH~6#C1Q<;~5ZZ3HZt_Wumy=Q=YHIEq;IT7eBD>+0
z&;ynICKaD)8L!%aTo+((;FsfhTYxup{f9aKbSAS?w`-D{d(=7vhs^TlI%-`HPDo7x
zXmQL;1pJoVb!YvUO?SB9h<mu+jIhA>$v*%(7&zzkyf4SyN!vdW!!T`?d5+nTa^XB3
zk>lFVBy>cvj)j~8P!rQ)f!p(42dLfc*@agXPwXhSKY<7IPZ8Z?gPKPvkNT~h^43}J
z2zgB*mg&jKF1FS*JSi)=$9!63%4hr-#kM6FraHXu)|Im;p&M<kO{YIz1>dJ%U8H5#
zy!_sW!;z6`_k3esno&ZZNle<c<mmb9*+$Ivry=`%A_FkJO-cV`dvze>KV;rV)`|ML
z-|vl@g6X{`yifMhsSz3#OjgRT5M61C@7Bse?g8(1G1n;p$RnMcclNxx))#fxBC6(?
z<!acrLwd{CkLBRCWqyt5kG7+MWeBOx@*@vK$AS|4&}4>&UM&4F2F%L+BrnhT@3KPJ
z1)LaDsmA%_<Zgk}OFPjZa#uwDq`tAK7H#m8Q39s~RQ%72^k(3?fQhL^Ja>t$`&FK@
z`Qpfc<hRUw9}gDSc*lDN!`TOU0n0T#wmUBG6SxC83G}Eo<XElI*G6$8uW3vYG~VLa
z2?d!2wS>OTeV}U11~eUFZdt8?{CI$_$P7pP29EUwk;WNrjQ<c`G3zQ<-s|>PVwjLj
zu7+yb`X#`QZpKO_>ah2$$m;r$6u3xfq66^z@7dR(Dzu6@`ZHr5_*SvJjg6ho3lKD{
zv`sQ^VQ+g}hfsCE)6!R0$s70FxPCDSqepTR4%|Uf>0YyQ)xWkMtq;nUZ$@`wYX*B#
z7+f^&^in^%R=)RH6({~;xII2xpznU`mz(PyHYXw1F#UjV?O}4C-d~xi4UazFd=TGQ
zdk-g2vtVUGTx8?%;$fN?!oHa*njU7&i;dGrOe#0RNKQ6p*7k2U>COSPE3duB>_e%t
zOUr{S)ac#F-NeII9P^6)7z>93H3?f8JKpGiw+@#NJacRDv2{f=GpfU7mB4N<vCDZ5
zvl19&z{_o9Ef(Rf6j9840qKOS-|j=}HgszZc7+D`863(HYr6e_ShO~gpuGj1lK;z3
zvTxJxQrFbi*rvjbtdLen?ANXnP;;Z#r~YtR;?Vrs&R$5ml74?fy2eMyN60$Sl!h<2
z_#Cmc$6c&!ShXSJB^x45ufF>wP`aNiD>ESsK=8@@vujo6<HD%f`r(xjnI3?KtaVeV
z_ZpMwajz||2B_l_MR@A`f~y2lU#b0Gx5<)#Xyea1;Y-0p%y$<LXuSA{=(y`VDVCYO
z9O)A=?83GJ2Y*emGbVWQw^Gu^*-JoU70@3sOaHZfo;YK5@g(`Ynt)j%dnwu;F*QfE
zwHURW+&nfLdJH1WRhz>hlmY#*zo@t+(l=5<O+trehyDt@gVIgu0^lE^U^dNL4$mwg
zJDC(3sjl27sM`>fi`GJH_P&F?VB@lOhSsB%hdztPk82{=VLEH7Jq0{`bFi93AgR0-
z(lK{Z$nS+M|KI#t?Rk;a!vYgl`=N_?9Iu*K!%(q}viGCr=ilZ(Pq2UeCPsKFY(JQv
zSUe}SgK%LxN}Z6t`yIZEDzz=bnvm~7N(ol&1%u^3UG8zhtk?(+mDfg6Zy=+nhO3?B
zMW{Y5?%e$cu!p0XJf%{`O~|M{p#2L?krj6Wb2DB#Koc@K)G7oJ(@_;VizXZql}s{X
z$_zl*R9X$M_$#%1P5N6(mWuZC4Zhp5LT0D#n|}oyrrlfPH$J9_=*E1bukp{cB{>D|
z>i~5B`r{7;k~Yp+0+N1U^M;tnVr}g;c5BGq&<*^KJ=OHRb<ue3a!%>XYf~L$bU3hF
zk_h~wTC-CraCH2#Sy>VOfpAj(;^(z<4PS2)V}LWfO7goS;&H`|LwegVwHg}GNz|=i
zyw1D-I%Ib4S$VSA)F^e^a1vN4Ix@dinbtbb2_u?Wi_dh4x%Y2Hjf`D;rt9{GrYJzF
zGaQP7dag<+^&YVONpErC$TQSeyGE@zO5=j*%DiOC5RAM5N&Z{m2Wdeu;E-Z+tKD?m
zt-sgHX}oiDx7<g^@rGV@e&x`FY1kXtng-$H`i`>UbW7zSC+A`h6Yg@8j2zuf(b)U|
zC9TDp-@a6E2iVZM>K#vcQ<=ktXhM=S_N}%fBSz<M$;$<bdT$2ShXK9^ZKV`)qB2?f
z01h;O6innD^-Y{gSx@>A1+!<y2q)%V*iV$;WZLH42RS^FPWbkKpU#@*Ur5zh7b=U-
z1~@`GQ#T|L$_Z)02i70LpBZN=@(99@D?95I|9%h)X9l>g7f6&!{YRJKq9jx%ej@oF
z!A5GvK()RrA^0p*_eShdKxxIX|4c#Eya4&oj~Y%~-;(|&TO)VpFa^hY5KiRhA6*J;
z#31D>()wsIF)-()@SkQzm*S-bKOtFXJKI(MU}Q(i$TeoVV$_Gwl6!KjC?8TfCP_P;
z$9vf$Y}hoFs<R$sQ;c_~zuyJ1Loo9Zo;ANIt~GOi4$bCxM?)y?UL*B|US;nCy@-b-
zt`VH^Gy+|}!5gSR`DNyj1N1<xp9Wv}k|%d4FTACtkJ(sRn{ggbZI(6}FB`cgvrbW%
z-9hFAzck>C)EoIuPbXI){tVm>!iyT5eB7R#kW<&*7JffNWl+B$yDISIEZG2^7_jAL
z3{iRfKEFPN(M?8k>OwWm-(Yj6{L{Dc#*noKb+N%Q`D=-8ch^01EQH(jJO<>!n4jQn
z>2E85N#$gF&g(2)%kEcdF>+tFp$68o=(_*}*;fT-q_E?MA|Cu+@|`5q)bk@mdR!8=
zBEyJ>*Xwg8=vu*+fsR2QP0cz?ny|r|jJyzr8Z+F7jree$Cqf&Q$lsBeC%!Q|+8<W8
z+ml@^*Rd(tYp)`en-(NQjI0-h!1_CDI1`#)no0jGPjSQWUzX%puclXmsRrQQ8MU?9
zs`0$Fsi6!ri?GqUP!5s{J6+*JDyl3se<NM)WB}cg%naSwzrN_Qa@qVdvLyCLn;`Hb
znWRPQlFl~mABNrN8C@Zim#1#;3f_NH(PXZ9U*p;Vr9V#=I?-*jH#c`&)bx_`PJ<jk
zUhgk*uRj3*i)Q@&Ot_}fYRKR}z#MI>;$rt}&xO^VkcFti8~aPUdhDfdeeUn!P0&uT
zu${;~O=_jzpq9r4zW-bht&!z~a6;06fME_Hsoa4xy1l7>v3sKzXj5+MHn#s>Z2oRi
z5g%mnA7N9()EUFG9U+Ukw`y8JQ1i3`yi4A?*E|=ng$X$A%kbCqeDN3hvwt;e^8{bB
zBQVB`87dMnr$d`5Y!%)5FLgtF%F+K`olYDLx-i7Uxus_F6&GFlq7Ng1mqQhf1=IkZ
zRea`r<Nd|rMo{&Hl1SgW7;_NQ{7(7ZJYv|BmB&9*1DCId&JH-{DM_0jw3-}r*~rWW
zi35tK%=`n9uck2(Yjq8}!KjMB>vjZ9jYnE@^E<9aHbv7$%JEnJ+5UT1Us(Ukm8z2_
z87q%l{(V+u%Os&mFC^m-0w2X`Hy@5#wE!B;;4g~7!4Gf0vc+}XXFranj3aD`Yv;3P
z0jgb}{Kn$$XQcPibp7}<UJ2MjM&mp=Dj(nMefLJyN95uX{`qHu|7MAA>;eBuoqn%%
z8PmM9EKcR8eN<uEM>ts%I@=fkj4tsvqpqHz&A7p2@6#LQ_xr9b+7ixM>4^*3B`Q@%
z+76igJc+4S$g;b9_cLhUXpL_*yHWx=zF75`K@Z5v^heEJirzs;OZ_}-unHA;`+j<%
zUZw1H2%=9vd!*nu2{%0iNSLQFy{H~Dfc@5n;JtM4hS=EO@jzYy0^8<d*d_oW_VCax
zVPS11zkjjg!@&dFCVr{okR~RG*@KAz{R@%DiI3^fQ4-e6!rY7OMe>SGtrAs5CX;dL
z<C)}0p1eL@ma=6v%LZv|TZhO|E94aS9ZZy>sm1j>M&yyuqY*8&@Tknsgph~*k##Oa
z{{zM4(G*}W(d~J2cibTT+}Y(~>x!ZhY3oqed11579sN63S@Lvk`us*OLU*Jj9<pd5
zLfukOL4Y{@3HXz_|6cdamA8@Zi32fC40Hh9&t~}<i+OT~(weLXImYDR*0lx~j&GP%
zy_mm2eJC?aXQo!>CNz*6rac9lJefI!*dhYJT@!~H0+}>tf5b|#BY3;Yqu<Q%1ML`m
z+vo3Hb>2al-`xGG4seQd_(p>-eKkbz^or+aui{~5N@klP%F%7&rx|gyO<QbLpvbAL
z{4+99XBAlHUzGjit@0010f(Yj3R1%KS5gPk+E4Kvs^%OPW-EvWdYhED&WC|0xo6Ef
zZcrU%0*_+yN96{W%%$r#K8}^9b8OScr@S)Pe$J}dxsoEvzWrU{NAH~Cp<_~%3XCtg
z+w_U~q<vGkF#wRVl?U`gEbjsP-OlG)iDpUYihT4TAGVx+j}Xc*viR8A3ckNGZ+Fn;
zojZ@kiV+HwS|KfC(!{Z&|B%86?f+yAVarvPbgj&e<LB_88wXxCmYoNe{{W__fB+CS
z2tCCh??);EzBSJZkSlSRe8#xiM3H4{dV#FtbLhI<nW#v*{v4lN#soX{F96{pp9Iv@
zDyUZEV+}D`*?x*yhp;Y2HeY(OMZ%~iwgW9aTdyzrlLb??D&STC=aAM#Zn^2;Rr{IY
zFaCWfZ+}}B0!O^;(-#vVi->5aCUvhcb573YhieTloQ74zcsfSfmU8VIRehpO*tL+J
zT;hbtm`XzV<;7`H%HJvGST1}PqzbGMs_XwE{x2uJhu^QTRNso+Fb%SFPqfi$x>7^%
zbBo1%zm(eI_=6Is?qj(i{5!~4I*n^UK==Gm`#G|s&r6%9e+Tex6*#Toc=id5Q<g94
zrUfPpafKYdYYLMk+ee(MFS@EjnyU%E|DZLuPnRU5cYx_MUff@QFt8L$)E^d)BmDcP
z>Z{g&mw?WlM@m%}?lo5JdFfIGK#d49+Fe^{v)Pe8Xa}`)89!oLbgIkn%HQ+Ro@d8r
znwJZzy<;(}<2s@s44J#3Z<IXbJK%VIZ*7?PjIZ=|>5=VhK00$F(QpDhM9v5d0QNq7
z-t3*Dx~u4Q0JI~-X0E>kYpwIHkKZzBvi->9lK5OW2VHEIZmM47t8M_~+jQ`l>-(Gz
z*qyIjW08j3b~^q+$iHiB^$~~$<I*s8wV?Fe-2oZVUj^YdK4cH{wdd8#Bc*}NwrLA2
z5VxDUVs1?)cmd5^K%WLUAE?gye`V=Zu<h+eHsgS6c0I=3w%XJS`bzR^GpOxaan+Or
z!-(LhWFtnp^JQI5YYjsD3Lt!w1bK1jO7nMM(_N!x^9o1@@uOM=@mD(FI{62?d~@%U
zaa6cS^Rk_J|B)rys{i*M6Ia!;od4eLSoh*~#2IK=o*H?#URhRJ6B(3Al;9_R1`H?n
z0P-;`Q_K)BdH{^YP@-#4`mNCq{8`VAlM|;T$4}E5KfSPB5HE<o!L^^GdHYnF!vi37
z|0+xmh*qb6i-}EGT@7)dI8x%$KMp_CCq!3IXj)jmx6R`S7b>#0o&Cq&+6_-^zIBge
zHUq?@!D6J<=o+f$iZ1(^iuv>+V%C|DdHZX^eg10#4g1H%(<yEJ5&aZqrhw*_x&{R&
zx@|zQB=0ROp{)L`Xcyji8?ELw<72>cEHkba_cI#G?iLA1jDuerAM(+mXKXb+HLmC-
z{lYhI$tkF2<}#zFVnUnrf|^3!wgkCQq-1=xZjo+1d3Y_kjQj&o#vKRE%_1sUxG)<M
zbYSoZCx*gA`wX}}FD36;DL{y@JZQI*yPnH@wSdDjgC@iB7EDAR2iMupo?3q>>3vX>
znCm-cZn>ml8^J#W3)5m}F`(y3q4Zh>o})lP1FFJnk4*~$JUOIN4_;D@jwlBB`Z&zx
z=eDJ3H{Wd84g}0HttFcKaOW7hB?HC@c`-%0abhSi6|gS+Wk$Cyh)~<7IYgQt2_jC#
z^0-*6yElkUZYW&j4g#3bbR*H$@-D_d>I5eQ*ZJD2o0)wD8TzY(B2mst`vI6<Xp048
zT(^;}&y*guY+dmAm+Pe$IlQ{z0xKSjVvAyD(li*$9NT(hyIGQL6Z^V*0RM<v_PW20
zMO_rOQWaP?VFQQexuh>%o@WPtrGVcwk+MgR$88%Fw{0cyC+?04LTb|Y8LQg`R(8ea
zIZpkHFzqOg$F~VL2`)C;U!Y%~;{`yS;5oyA&hbxE=^@F<77Ema7V90L!A0NO#`UC&
z$;jBef(yFbtDkCFLq-K_R`%B{GF-;=H!D+S(y)m~R<oc0o6pY<>l(c4_pEC5zv!-u
zUHXkY`2Eh1c0JVQ%!=|aUuCf)=NGjz;x&Ef4FInXzz;ZS8noS?3X9S3w10W}5>UAC
z*Ee_p$o4z5;(gN2`&z}oF(!{OlzLDyK(MjiML}~BqK0m)y3X}3O#%z~VJ9(!UDn!W
z!&$Md;lr)OuWQQRw*#OHBlo8Oxn!+5A8th=cWl)!&#ut+IS#D)eJ(cY?p1utz&Z-T
zVfCB;ajz-bNCNPHbhYUC8zt?C5!nB~LbaGP%4`=97nhUC6-womL(lXE^BWBTvqa6M
zz+oq4UZapaATZan=Q8CP!08{Y37~Dwnm04jWdSR<f4uB+@;cZADAL#cThhR0&FQC^
z?OL~grZPZveV2goJ@PVvDnB<J^?2}5w7q<<!qkM9#`*je^2|di8`I|8u3SZcycWH@
z_PzZVLYkHES$C8dKo6<f15w(isW;DYMUoBejCZpE^}exz!OL;EWM4+&V%W&w%+?`#
z1ko8}GseD2Y86}8;9di(4HI6iC8lYN*!z3#u_qYa<lOXvR(K&p0+jWSs*iGyR(LC{
zXH=_E5mG$aFvsfkaDYu-ne>a6N)T07Azp>3EM4EePye_b1T_#?E#SXYCw&y(O$Trz
znx$DGqC?tJY7j!kPczmx_3rEH%dv$>@;QZ`v-$*G1n7}$-e5j!3&cMru1#GwPjy*<
zH-bKQp~zUO-V~kEaG2|H8HCqb9t5Qj;>Ott&@Ixiz7?yk^;g*`PjA~%fF~>s|I6g0
zzQCrL<L_=~E*hV6{UbT@3L`ePQh@Euzxoq5tLIZYKyh(AF2rfCU|U`iBVYXJDb%%r
z<RxybwwF&ky;nB0*@NERebm%#;laLUY<(71U@Fq}E7jMS{2m%pH~b)7Es@KA1Lb`g
zG|Ng3=)1WmkAg}rAKM0ku%THqxaBRvqr<?W4Iuem?v)wHv*)^yD6+C>!rq`mihY#_
zxb>p>-T7E44^BddPE*T2zyABO#D}-Xd{-_w$3f2aW+>0JP?Eo{I=#~$(dk816zs%G
z7u%^zsyj=7ltei(&(<@SGBqxLuKsv@dyx?}ELLVB3wdj&AKErwbjIO0hn=z#_~8t3
z|7E_A#85%iO7X5xoON;^g5k)#vpQ>}kwiD)9f%)@vj6mx0U}1De_NJSCMq<3$hko+
zX!9OXYZr-Ye8WtUJh5vK10BGcSKk8yMq9;#aQa)XKY9t#YIsa;M8h;`>L0_ubffRM
zISoH)w3gcT@<L6eJpUk9;#sFzf0M#|z5*a^G(QtY4o{1$i<mkMpIudcW`6hrm^R1D
zt*Ub{O*fGuq!bt;VQs1tU=-lfRMME&Y~ppQvpWSbqHM#Pjkz1}K8V|n0P*kJ@P3y(
zH9NeWd6-#}gIBw-b~4#a>3By7tO8)tec9Vs_BfE!NuqhhRFVs&mrU2{?T#Pfzr<%k
zP~TAq9i^KqR8&b$yI-#y-sJztP=d#4_m(abbd`&Zrb-U{szj#%jfxqzS%o(qAJ7T3
znrymG=nEYc^cq_P@PIu}QNW~5plD@|Q2=+%z)19Zk}TfD#&-%Z5~PcEM7pCx-uLw!
z8;VM57~=g*%(6?nu-w2G|77fTl#-P*8VM_B0S=>p?b-^qy2FlYm$`JK1WA!rn9?%)
zIDMpv5PMi<vBJ&nVP0lxLNe1PwJ<<q_l@2<zfMae_h=x;1rHntm0s9Z(X#X*LP>i?
z4GYR5!N{M%rqF`!r$yZW=!Zt4+A(@k{U=k9h||UebO{?!&+P(jG6UH0K&!S+*;{n#
z5gvScdc-VA-O~NiR1z4FQ~*&F98HcN8q(16(&GjM9eFSh8wFl%p4Z>d%$U|ZzNS3k
zk16t+?a)69BQ-Tb!B@6l&Sq;$AMT~^v-iZgF}IE9Z~@rfJ8&-VBam_ddVCWp@pL!x
zD`z>OTk*<Bk(@A~=0`(IAUDs<Nwlc{A$Ceu&h>E>MT^67B_a<K5B(EySaC$)htX?E
zjRf)qOJ3<2i)p{<`i&aW(Y<1k%7iBLg-qn_z|3q-H>~D3IY&QMyU%e_d9o%&e;NWF
zr#`@06A8onwVtz%lROzKNcO;dxMtT<+xapsTjDfL{%Z>D<>OR;rQ`k+(ACbcH1fkO
zxj**g#w1zkEJ<i6?cZmo@h?DupR1o5Iw=*;uy!W2D(zfRS%{sFhkvY(4IC5s@Gg*F
zl*PB?m_XQyw}1?znJGmzQLLcLsk3@3H+(&7P#!$~cv>3*F^Oj}ncQUcY*yYK-jh=(
zf`_FGJ6DfHdZzHlHsu=o#A!8jucTEY9sj3$@=4RTe~kT|?%+ZyD?_3V6N#Ot%P>}D
zv)<@_9iD#<-ZpOS^O4EJa^<!4(#WXBR8}$lCZ1!x6VUwV>etolW_iT~2Hhb6Yeycq
zW+tArd=YR<gmi*o@@GX2<^fHGLo_|I?Y4;h9#`1tT@ia>z6LmljlrZyxyb0KLK|S?
z><qoX!9h1R0jxC33RY6oeb>(Ix<Kg<@)OlDmS60pV3cizSfYxJX@}h#50zMB#^D*q
zkb}TzipDSL!UZ4Bb;t(=f#_vV&p&1pf*Bn^d+DrO^~JIPV>whC4*^UP#E@~^jQhol
zPPI7rv3W#Am>B`>mi1f36}htWsDnNXg|k4mk;5y+1@U0-1*I_($}fCC2fYckqkfSG
z?ocauy82`F$Xr?O&-bugVOUA;tzRua=FOy$awswJ9c=AsXnxW#aX2jsp`f1VI&^k_
zKmaj>uZioI+T#7+7D`r6!O+xkF~K(}qHTa}QJVM<b{exD|C}vnR*0e-FvacW8I8Go
zC;SEn-I7OlFS#fwqvI2MK{B8fcuNb;Xbd!4hB@!+f|zZf+)STu`=Gz@yAd?{(!&{>
z>`VxUR*-+P5=aW-BpG62;r`AIo*Yebb@jmL$C1bk4qlb<C_sm+9(dc&ap&n;46uqj
z+GG*0wV1l3D$Sd2=<3}vF)s?Au|71$>7^7+Ew2|_P4-C4^1w(rm?0Osg)t5}8g7su
zUEE7CSaDjxUI~0c9q1%?9X-*oOyBi1$Wr!R;!#>QvNubCyj=7eni^T(yYvaZvb6OR
zWb_@Pgki)C?++(LAsW<%MODd;Vh)Bz;YumMGrSGR$AxQvt-@W^j{8hR+6PESa;n&I
zx@YrTMg8?>JzkYcVgW#RI_1cC=$;*d%mmynzWCDrqBTnC4@Mxm9+ot46eOPXNLgwe
zk`7%uHGS>3J3(&S9VI<g;x^;5*NSP8>kNHEaleweoP9h{{Ul#MNP#~ve`z>4uHSn4
ziLwd~HGa5u#i|&+_!q*hHLDdnpX_AaPJq_`Lv)ItR^+j~dMj^jk_v`nzWbUpdM}*$
z@2=|z1cH+B^UAjFBHzhRs>WHML5Rvcxd82q^T*oXq{d_Wed<u#9Y@(x%aqjN_DV{M
zLMUvm$420|XBPG6L!W~!5?^*m0>WN3DBN~uW&D}YOnMK{1N;Ra&Epf>OnSJm(Q;zD
z_7otQjr}WCP5eizFfZqN3>guY^`qw2T>noEF1=Wr#4vO;SSH%4YD+c)6;hn!)%qzv
zZk=(F_et#iy>bt#wHs{O{9VlL#@W1;U-Cfw(5$56mg?Pm{?P@!AY~PD2PGjO;dD%T
zH0~p0ObT}Qs0>Xin7Cs*8@7Fc2w=@}x4uQ&`yPwK&IpGP%|nw%;Qs2prd*xCQY~D7
z<LmBzqFFKOBZIORT0VuDWTIfLnorE&$S9b)li46EWE@06DVR%~4WIJ#7Zwx5oa>H}
z0)VM0rl`v$b1NQ0vaW{xvZ+~dXxctOI|=Nh(Z|DJXXC|J9n5k`@w=_}?lSywKmsnv
z3vo`59~t09^Zb9m>tNx@NgU~)M0$@G71R`X-XboV+UBe*I(!j=!?M6XQ)@m*vjq;l
zea5j-eVqev>x0Vb*AtmX#*ti^Qo%&=;NV-Uz~WFQAt#1hpSFayJQ}+zu(8(nOA~*V
z`8eeM<hhksySXQzJ2|wh4URa7h>nF0J*|ubF@x1(>)g0QX>NPZR{cUE^Z!oE_T;^;
zDZ4o_9iluH`y7R?7fdmgrAVD1X`Y=eaIN2h?RPZ}th(x~+w>M(KLG@`A&ksCnNred
zmghBDr*X9Qx0kXbdPsF5a5^9Ue5}Z`qB*Txx$FhZf;A?spj$3YO@Xq1x^^t{nVMeg
z-cBTPe@KVJjely8h3p0Z;KD?;ko`z#ZmM(rL#aV%!hT}PTkB@W=U0sNoCu4e-S`Y+
zkkRsPwC%`C>EXa)hYsTqfZybByI*e{p_A;huAz*77^o*<(j1aCSmvKBlHa|k4|!|x
z-r4kH5KtT-_4V`7npyH9e3!i5IKC`9nNR(R3~`h#i?sS})eErWj`5BW1F#W&zx2u8
z11>V0bGbrZs#|uSPgcIqb3lJXN&-u1r6e9b+!AxS25E#;{I@druNSd;pBDeJWYxb|
zEcn%QynBm%IeDy<eY_prEd*G30nO2bwWnLi<eI({V4rzxYEv`?$QEuYMb(%89H#%(
zXY+>Cg@wwE>5wvkKS}FaMM<M!&3(x1nt-H@*W7k?%P;wf;AZL~3&v^RU%4i;R0c3z
z0kuEo)C}Fco?46Fin|3!;bHw0vmMtFx95HOPt%xlBLSD+Qmst?uf}Tto}pWpU9a0w
z?t)JE8iguvTLhDEzNAKq>FC_yQU1{<KlGS?TXyBtBJ4x5)bRCFkjtyeYKdtQ5qpj_
ze@WC<;;c|T^~|cG|9zZ*%%-F1XF||&X$JuI`Y=pwa?v~D2ue~`Flf&xhIV-74PuB8
zGJe2ABYf!DY}Z{2Ik*AIBO&3>zX~cL<;Wazc{>XKj>BhPz+3BAu=2@;{QdB`9{hG=
zr-#?JfX}wx%G{@`2vb}rYbvGG+dV3vZ;cT~%2b7v-mt2uSmn^gLDR`TPgIa)ohYrq
z4(B3?(kTNh*B|e4L#=PwsSto}Asi>j{bN{{2GnzzHOL~?72}B#<+5I~qx42x@Q<nL
zQ=Y}DkX3<ozLmTL;B#R9-;&>ixaIccgMk~9t!!_@4OD-5v+lEEorobAGRbUiU&t}O
zHO{LdePfLD0PszW`Z!nW0c9)Vfff&jqYlm=yi)ecNZ}j?v+Z&Py#s*8IF)i<GJKUk
z^YjG2;7*>uwzb;p6zQ?f#&q=mtS|$hX+&G2^pLX=50J_lfF#7?t0#@4p#%CO8ok<l
zz@Py|fR%WeF!T_i8;JOXd?t$ODL{$(v>DCWwp{aH#!Z_?R+#V6OpZo6Y<J3i9ywYr
zGj*5e54nl1eX;k7R!Q9N%8tI-SQvlDCk^nzflc<-6f|%h1@*9$0;HMZ2frJLJ1}S_
zvaDs<9e}LK9yVL&cmLC3<4=v>H`U8KqlHbX^26EEO*K{Mb^#t1;GUR3SZ)nOKUGO|
z-787A3epKX6H?a8_%HDufA&kr!GZVwmuqwakYze-Xh)Y8_AFfB_-nwNwFH#@%B&q9
z*<6EgG|)zZD^VF&7m9qAbuC3-`BkGz<yv4_=1WWA#-?|iv&gax&_k`GFrJgcD>9+x
zmQ|we*y+NO5igR$>&DF2$f+_ZrJfMk>8D7%!k8oLQDm*5dC_!K8@9X_Q5yXh!n@4&
zUN6@6{rpsD>?Zp%ek`jMviG=6c*-G+^ch(g4eUaP698`jyj4qo`};p{|8vwTfZ99X
zH=0}-+JWB7nB<m7^|L`42;kVud%z#yss5G}DJD{#iezjM^{aFJT(17s*@$|$M5%|8
z*c1dD9O2RUWFw-kLJqq$cc(n69~}IrAPo@HD{t=^FBMqw4eT3_%ix>hV0|TB1zq{(
zK&mu6a)#<-!@BS*qFpXoiFg4j!-~6nC_O2SZ<_Ip1e^uCPfeSPHi~?9P-%DJKj_~A
zEQQx7xU@IXReQQl`Vn}9t@8RETf()gm`aMe+W1wD`MNQWjZo&37{A(|<+VFI+T`02
z*}4$a$fny7a0?~jy|V{h!vPb4JOFPOu}W9=0Nu(6tDPKvxm(W2+8$wz{wg+^v@qW<
zCm9>!Yjh4ms#)WkvbU)>NM9R`P@0<F`_o?c5=&k}NH5Apkn(dpAMP3(vZn5quR&u2
z4x9fu3}n@hxYL0Dost|gzdn5ZYS-t=#^BgbeT;*>?#pSknNJZo3$3On?m2Q_NMVH%
zJ}LaNMAs9TY@|vdo<SGqdrJ5BfItH~L6BbIH?uq#2!W`*)HVY|z~Pg{y+1~8ZvJ+S
zb;9spxK6?;TsN<~wHFhZkUAB@J^h2%%NOQe9&`r<_<(EYnRc%TB>$3|F9)KL0rE`|
znVBkpS5<6#Xa$HoAh#R&L$LwC*7&1Ylf3J{?y3zj{uFT}Gj00PO&)3E<$fnn3nkm0
zmHrEvmX>lyZ>MMI4Omd8UaL(RbQhF2gkoz#9pwiB=j>Vy2H2ZaR9vB4`Je@<8=;{B
z5bxsjSWgF7z&%jw1w3~efKWo{Rl7MgyOJp|*2NBlW3)l$R0nu+MBUAk)$2MeO9XHa
z=3UMMiV)Y0E#7)MqAZuKJafP`@qv|qQ*nZSY8A6TUwC1^m1M%c#LaKWEt7rD?bcXV
zzO0r5!qtHS_yRBYh#Jmn&nh&Su^!=kjKgQwYDK+1&yA;zA{vIuEW#F8007LV6quIX
zHRG#_%+n9Pn;%%TJWQ7n>+}b#4+z83U#+b5PspB#@hkF@`9?!F$vuiQF+m4mXK*u2
zb=$Oe3XmK-R~duhw<?HzoJAFLjLNS6bCr#o$ljpCn6QTM+14O=qHY+1g`#7JpgT`Z
zxTeWpLUoE(#vkuHND8rrnFeWjr2@he`mp4DZ4XeiQ5@$L(Wnl=8<gv{`xr0}ro#7N
zjemVI5@U8HI{hV2gdN61VuYqUHFU?zV`X8x1|X;gxoDk>R_DqFq<>6sx5d#$7(+mR
zlfYJ2WM^FjW0qK2^|#*>1HZ2mRw?T5$n(!@E}z3(xIO>$`}VajvMRr7Pb^;(B}JSX
zQ@>4-{S@BHi1GP0b>{W_=_G@)lMSM|T`9@c=r8Q}gH4sTZE}LG*VDKmul9H{8zbc%
zk*`Z<a-h0FFk)&k`Lr0d;h7F18`l&GI2>)FXDX=S`Y;YswCESA05>OoAaA?jpvPSZ
zRRuIk>cMg5H5u6En(FPKy8l$vzyqxwmD^FQvcw~fZtHMD>R=_*S4TMN7E;@VC-Jsi
zH!Vep=E3Z|`*ZT$O$_@`fLMHAtoyA+f%9yJAgKOYL6_^-`-y;Yz(A^_`j@@(-M@gv
z9T^Dy)I6-F$}us8Wzgo6>FPrG5$_)iZgc8irHboNHT3@Qz47dD>dW1nD*$KH{qy*V
zlHzaHEzk^PE1b4-7djVub3aS*#$B-n(QoB*Q>w~R*~i^K1WAWXocxkw*=wCRW%80s
zQ~YWo5-vFw34}waH_s?mqxjj*Nz3^FzRJLmeg!W)r}>embqYam8XQI5sp9u6jn4?4
z8j}Dd;K!`!G4(&E00Ft=tfdDZE)yMi9(Z8JUX|7)(7&?MPww(+RJMjGTFyTSpJ_-P
zloUD|d$<mBHe8!v!I+5n8@din;X|h3{AQBQm7&cA$eOv0k%Os_--<=$E+sSmbFnny
zI&bqg(_2RqPY<h?nTsek8>ZKZziRVhT_S){D58=}?*aor2ke%x00F|KrjqS|gP7yb
zUG?@8Qq(K(X5waq{Px`=&&|~Tz9~4KUns!KJE#t$yjP9zk7ez35T9{P4#552I`$GO
zwJIVgZDt;@*xDJfzIO7M-h<$WkF(-uV{()!|MNNk1i+?<W)Xq#Pbe`(FKiWqYVbpS
zn4E{0^NMKuWy=+>->5Gn#-55Y)QuzK1oF=G2>g}glPb|c$4Fh|bvb_B0m9rr10lCm
zA;<yj!GmZ05T_SjP1m+%#)5^Can*~p;i7oFXS?@dA?C`DaKEX`mddGJD|;D1y(w6y
z5g?D#(6i(NjsYV`Bqj`x{QwRFlj0b$R4r4v?9WA(FGu$7k*kl%%X>ZFdbSqJpRilP
z^b_9U_V$P${fgMG*(SLW3w@OB88H!5V`VAj!JFwBQu%$42*-4x3RFsnAZChjHGa!y
z8ir~&FW1RHJtt23*O#Ldi5z!!P!QiTNRapA0jw&>ylsJj+Ie#<@K;G%hRqG5lThaK
z9iR?SAV<f1%eTdoi=Ex}T!Mxz8Qhfv`gfY|axpvEhz~bA^lCB1*WO&;&K!I>O{4f9
zxu=s~Gf`qZ(?u`t`&R@62S)?t^d-u5LM>+*0ah40aQIM=#m25jSIoW$!fTb;<Nt2U
zB5;}(rU*<3&(o6RkK<~msUOc$T|%zPesJajBt%Zg{<lg`?GL-dzzVgMfTkax@ko%n
zmcQQAc<s@}!ho7l55^xH-%AZdc%sNJPeR^48P#X)W^l01?s~3^^aM9H>+4-2d<X8-
zYXB-;Y$YGQV9;m3U}Hz<E2ycYR)1M}(6ijGUP)dMczb!tZ=LA`Nv7}3X+YO~7EVuy
zN8W#<<|y08M#>(*)1{E<4iu|HacE-@#g%;B*$?u=jb|XDeXRL<ZEBDwT5o%_3)<<7
z4|q4NtsrYzaIo%(ai|IecW&nKAWDbB0xRb$Cc&ADzlYxSjjc6-C$}d&Gy8qR+&Z06
z`*E8MUfqjLQySEoZ5MX?#foqL-@n#`Y+~tcsfSW(R!09`{@PRxTq(3%?>GIz{xddH
zz?GWJ1diwzmiHFh1g<oPN~F>+{C^p&Y0!-9vlpe(igOko0$(yv;8&7U=~6a*KF!hK
zcHHr>3M!}F5F;-9{O5S}%|w~RVKz2kiX@K3p(%r(JQu!^x~QsP*NB%#8ck~I2e);i
zAh5N2d~(R=?<(?%Wm&L~h~b)B9#&<E!*zOkMzjQf=ak_zb3K|Vv}U$Sxt5qFzq0S+
zw0^94q-J-N?gDFGkq4Uf*31b(YO(T2OFi1Y;^3q#uB0@hzt^8Ew+AZP`poP}Gw2DN
zox1L<%Ml$B6Y}hnSK!jPIyHH;<s@{qfOvs;nT!Sh#`&swrYI*4)0Lv(NNE*%&XzII
zL8@@Sujm|y1oy9&W<=N5p@~WRfdlE|3_?R9-tlkmTz)|9ws!)mX^{CzA^C15OiSs{
zR@D>D6J0v1Lx9cx3K%zc>z`Dww!3BriYxxBClPjSXv3mcOEY_uVt9iVWn7cddU;%L
zTglJbN(t6|qoa%(6`Hw>J^$D_s*95YKQJCR`R&oU#m*^>-nNL>yAHrXjm2pD(_Uat
zPrW1$wywS1L1|ARudvH~*n%=sp=2c>`d%4HfvUJJLIxZiJTL8c`9}CPW(323_Vi3l
zU;0dt0MDEJJbYQf><P6nhXy~6xGZ(tTNA$Q5x<+L+Aat(#fmQ%T)NeuxcSIdyc<F?
zF_&%v3_<XYb95e-9)d!k1AHn?0+eiA0Rdid6gJ?W`)H}Jmichq$VBsQ;UulwWJtU2
z<+M7rTm3G(@=ldvl%;f2kXdWK{>6>ND8)&jTxx4NCQmLG8HPKxF~Nw|5g1wyvO0zn
zB{OwqGJ9?yI0mlU-j2eC08i9OpkUh<^NE(^`vNdg81bl_%JIJHw0QiTYJ|Ny7Abvs
zE&K8ovbF)Xz85zWfasmZ>@{fbf8w<qe4Qw-?snv5trtO3$^qe>Ua2L*jn@+W$xD31
zUHO3HADo#q(M-6tozeC{Wa+lLmWGXNV5Zd;IJD6hoN9#eePL^0Fk7iy!Ud3D73`JD
z^u0!fBM!-{4BEnONyQ=KkYg>YyZ)CKKD9}%q*7D;V}{qleH{<0MR3+tcqtE&9O$pV
zFj%mgUz$Dm7!VULt|c0^g+IL%a4g<GfMp^QPMysX<Zk%p%sv~b-53hE-CNLOo!Xwa
zniQvR8=zNhN6N*yuanp^>6R=mjw$hq#^k~JAB*WH0RA`nj5xI8Lh=#Q0lZ-rg8xhA
zrxfqsJ<u?e>S6Qy;pP{fhgSTE9{(u`Achz~cBq=i#NH0*Z;K#h57C0s1`%Y>fMb($
z+}UK+vF`OWL}gjnbZy^wuSNgzZ>Rr$@BhRjmhf8cBahgq@ORdB9z1;PSn!KF{kB-o
zt$WK?d9IljSglXiD&7>-b8^aQ{yPa)x|q&J$$eJ|aMkdmXs-muMUDN79oXaM5$P6^
zIT5Hm=r{+zhK>V`zeDP}xAsa-)bickKD83Acd#s4wb_%r-BXEmLXVU)NWwF=4!`!I
z#mt(;dX!Tq-xY30yhAHT&ym0W7D5h3)b)5C`K`BN)1SSr5e>tmC*XR6ZeuFtEUA;f
zm-}r!J4L_LRrG|#w<qX}Q_kJdJ50Tc4K+4Jx7!Ylu-x1|f$2C27$0J;rue?xU3~J%
z>g!eYIHXta7n>v}V0Zt*J93{B!h1UA7f`DB^J8hlhhzCAKXI1mnBkOi=<m^?;`0SO
zVhD$b-VJ}AqYNA=`Bfby=(xFzuavm#@G+C%TKqvgb-t{~taiRW-D~i$)%saoCkZ@&
zEnmZ}$hQEqDa7JTUMpA7qSRo*+4DRPg|gd06$S-Vr+Y_^J&^UI6aza~U=av1%!sW*
z0^<h6s4zRjhVF&fX1WU(P?CX{RY$woEF@+3*`w?*_t3jqwkwULFa61%_teuIyozWx
zl+cK{ZkGsR=W0(zty<!Co~CW^^tKJn##V|@*GKg|lEiso!m`R-w5bQ{0!__jKY(>1
zBr?+SIOJt4di{jAxfip(NEO7s)s}t5LY^qZ_e;Eh@m+@3Wh1LTo%fkhwoW&VcUYNU
zT#843MyhcLv6JEE;?N{$3svYebBDg$XNo`W5?cp$c(FkIf47MH>jg%|ZuR6eEv3Mq
zPxsh}_aRaJZI(7M+yZh(MX6edzdL2ZG=HF7BjSH+9!-$dMBs_n&jLY#d}s>6l)FEW
zE8?9YQ>XZb{G!{mJ)+*^bMyV1>dsE6_sW~@+b#1rg`t=Vbnp}MR(dG$(5$A9_5hbG
zPYoZ<I#Tgrdx9J;BeFp{FMI(I`GUZldw~4+RiXB=cJv3ai%OhgPPgiklJ^a3b5l8i
z%^I)^twQh<-$)hjF2o^cAPZ3_gv(_2-S5;|zsFe;Utx1A$6VYzho6re$3Yta%oZL4
zkG)2FJ9JGuX;aVPW)ID8v~J#L47v$esDup%25KJW#5=egEj<+h@z}|yvA^n1#W#Ga
zTLmLWyUI&pxtK&JU|hKj5R~xQBhSE-ylj=_xz_vdpT?C&UPVT9r^YXuU1UBZRVj0G
zw_AC+=QO7K$%!ZG=%|}ElLg_EKh5NY|L*-U_P$N%C`s%Z<xVu?E3QfI56_&CdrpzI
z%U_<0^I581Dc>+^_S`{TXTRj^=<#qN8Th>ypvkQ#TjD$~Xf{^rA67OoVnT~mm0wFh
zKjNEWeP5ExDO}{J`pS2b5m#ZcInakyS6<DRL4_*@*@3b$mw3&bn(%5E;QN6!^aI(}
z+TqQ=iV9Y34+lUpZ%Wf16XUXswPH;g?ePx!Z$(26i^SajWIaR%7QlqX$hhdT`r_Z)
zydd$*ObOEdOnFDO)uHK0Qc{+{bj4U+%{poO%JLhsxK_k&k?_yja%6F~JQCPq`&SeF
zM;c*Q>6fnRr&Wn<5^gQ~9JbFswC#CRFZ!;q#~5SU?|v$&9OF=0mcNG2j>&Jn^q<mf
zPRR64_>o5DvM+7v@E_h`=+VOXhv5JA0w4?@y7c8JT?IR}4Y(*8pa;@ZYQ4EFt4pC`
z-+;J_#TOL2?h61MG1g;m<D;xa-w~Z632*E_AO@oyN!L6-L?4&K#nwe{S+isJHCWb7
z`TODJMQp)l(c~TGicg>?pMZ8b;!r=@n|~^1gz5l-F!ov2W0~EZ-tUCgQY5JOU1zw~
zWuit+s7`sEdxWB{^71$Yc4t{&j2xM1fgmrO#1yj?IW8we)Hsf~m^^8^NVIsWJK#8m
zBtD7&I*4>s8-S19F{>G2*o?3PGc8KU;SQs0d8XR9H}*SiY9YOQ-)gbTz6Y&=Ab(Mf
z*mZ-4?VyAQ^!K0r+b-dNWeXX2?mIy>%-J0qDRF;SDl%Mee&cvDUEzmoWs$#+0I04c
z>C?zN>mku|`XWF^l{KuNIQTRYC@`oZFFln2UH=OpDCbKFpKULX_7jg@q(tThkW`dv
ztgteN$WR{QA3t0ii2FiM2dQpG`zy>?=^?u>QaI%Z*KPOf1Nl7Fr73l$h_-4GKfV9n
z{J6_?SOo!&K{HvNOi}x4x`MQ$--uO#2QrJQ@q)B0H`a8~4DQ0O_t#HB_3<uI01)~f
z7)k~YV)ge4)+e$=ImxFjy{)Zd%k-sLureZr<aMEh9)I1b6GWMbZ~7nZL24X6wsyZd
z&M}MOg2ddIl5#SMA)jB2*0^|Z&5{i7qTjDZXUhKFz`A(Eivx?W#yL`4xe3RC<Q;IC
zdBn1gMWGz)y4lGc)@W7g<2!t%HQb#))6ca2lKHG3Z3tD&SKH(W|3qNOgo<*il0&k?
z1O0#v5jNJDI^=z}4I}aCZ99vEuW3)tyW3;Y*Vg5{PcqO|#Uh~NNVNK^<rkW1ihwKE
zkLG&7ESBj@`*|F)edKc9J{3lKXC6Gm{=UaHekd|@|81zy+d-r7>cSNKQgsgvm?Q+b
z&Z-%q@F3BxY=SHt;WzRTE%;qf4_G$ixN+Zutpt1C`fsN5>5mAvu6ofH@WLRP5$$iX
zYRK8am&B<e<4&SyQF(a~uy6{221AOXS1pfSfMB$A{8`z6Bk?ONfA{mF%N$$C;Be@h
zSgF;Vea7HuzsNU%39C1Dzi`RL@4Qx=;vwdYg*<-c5m;z@@4Hxl!^K$WX+aXfdr&$e
zbTeADGZ(vnSiO&1r3B-{>BI#G{*(`ZlD75YZ6r1W2Zr+2^9>xjg!>;pvb9?fESR(h
za*xaTKkoy6oc!1A;>Twg>6MN2r<QmE_3kau_V97XYAkDnS#a~kSDSXBrjPDV3Nz=0
zA$zzhiN{)0{gSz4WPjMc^MwP`PA|!e{~VCx_w;bC<R_f3knM;L=?3R%oB96ece5Wy
z^gQId2SYANCVKulxT|NOb!uaP5lwRQHD4XL3&QgN#S|dp!_<Qd7PL2VEEe^1*ViY<
zwr31szpnByLRU4Yk;hg0%zbg3kSRrtzd0c}(N|wQrTsdti({RE^h`W0$3=q;FHk%k
zK6kdYli%CCc{EocN{2rHRl|P*liW6uBr{<NV$)4$NUv5my&=R3e8i9MY&Lsbi>me3
z_AYR+;`gxi;Zco9d~9BtHF|K7vKL~T^bW!wIM^&3>_F{a=k5|^@?y0=T=v(|Nt&K3
zw7viF0=>Mj;KIc?P|ru8D+RUn;0!Q%=BhaiSe64A)HUbuQVwyXQP2lj+LMU|5=953
zm#EPc)V#nyE&^Sr<96hf*)Q>Xaz!$4GZjc{CtMIcNod>uw-ZZyoF|XJc2B$D(RYzR
zZrY(dZIxe5t<=MD9(nqw4gKx`XF6<D^*L!xivgK*1y(jy&<j)Y|8$~dYSYkfFM~N{
z%UbLqkYfM;9rOQn7V15dc!2%o@ctS(7bSH&L?nOBp-Sg<%-c_yoa=w8gU}u9|0At3
z8v3oYP=?hXT<`_eTLU>2d~W;W946VP_L^D97t3Awd&?s<#j}z*^Zi(y#E`dyiV@7r
z$Z}i(@3H^!16)R`DR7SNt*igSA4W4heh8VvvIT9Nv~+ut@i1JsAUA0@Wq42C$I|L#
z&c<GWxzXem+inN{G%sI(t^Ct8?Q^+iO4j?1j_B&sk5VW9>?@U@0QY7T9K;=Y+6(&4
zCX1=ZSGso;l5EVfUMT`1g1C4pLS3NALmB#jG_3Y5VlW6r^E%8L#a@@$1}#qpZN=k8
z$~{Y2rbTo;59eOYH<SBQh392~|F4b>BXN`$MNqIX>AK+?2$lVwN7BMSyh$mX$p(<e
zKy{lZF<AseEr^$(drM{@Utviu7q*ac7#Mr<#pwXmk+s@WKROm&E02Gvi28%O3t2D%
z`@<?7*8rPOmNb7n2jTl=faantgF)oe2-EF_@BI7?>kloYQ9Ru&^1o1JmyCqD?*fel
zTZx|I(KpvQ_;*kY4EBRsiHTb5bl<*A(bdwniCl@IdQPCu&aK3bat83v^XopPar)AB
zb`)C1aM((m(|dRiS9I|rtbwI^Z?Eu6o9Nn-QTtBM9V7IYk#gZHsNf}zOQ&J4=B~K#
zT+}sP{jRQ<l&60>DGwC$zgT<ku%@!MeN<6IMMXtHq^SsqH0ja;7C^d)(jf{W9aNeK
zNyLt{p+hJ^s?s987ZpTmsG$>;5<-y@0wg4GR-Esh$vE$KUFZCM=dYP7hMl$dUh66M
zeLv3=>}|(=q?zJ$x>w)ovqy=iVmW_`Fp0II4pXyhJFpcP;4GI?QZ@Dm^8JUiKOScV
zpJcngzsrQ$hvs?u3bLPj<G3|Q4Cz{+>@Qb;@~0hU!|-u_{z!|$zj|c0SFSHy8KRak
zeNek>`Oc7UC%%N~;kc9K;wii*62QHhnj;Qgc=<`QY`1XRu-11TO8xiv6!UxN5U_s>
zIjQantbF<S8W)GVX_)u$+PUd>A(Sj*x9N!9^BsQKZnbeq7=m)2FSld~2vBWz=BtOq
ze8)N(?r1b|im~sEaEF6%IiTaa={4Do|5+rk#5i6gM+E}waA-`pJ?-?5Ex-Cre2&eH
zzj}gxTW3E})v#7ivd*RS@ymRtmW=c)?5j{=m8=&POnIaG0H!0cFPlFl_C3^8Sd;~%
z1=|IuLA8>M?4#&yPw4y4J_`ydfxX~$t)lvbM}O&(u%F)H(MqS-Hv0w(jJslcEy;VG
zALul{R{$`lX{@EnWUr9H@OTVVb@Ktr{FP$9z?xtAk1_N<JTzvh7vjX*QDWYvm@mKT
zR~|fPrA=K`s#jK<o7u1I6(}xo-yfC_Nd)tKqdEGw-X}&=>tE-_nIKB_xJhlNY&n=U
zsdD8z*c0R%{wKhcy(bNKeq{BfJ?S2Bl}0}^A7_0Ho>f7Nej{HRxZ9O=#Au)<EW~9P
zGP1R_W{j{G(wy$G+#v1#GSxkqbH(Uq=}Z8EdHxlybzYhJI3=BEm5s6%_Oe-ASL?^b
zRPj+vuX-c5MNF0XY}<y;i$f<Z%~i+&Gi;6fUzM3;H_?xI_Ugeu)nlXl6^I!d^TntK
zBa>TPDy>?BkM4aV-?2X(F-qP#GT_#w)gAa)xBXLa(CSO?%KVAwYzXzSwHbezgk;7S
z564J|NKBKe`=@w$o9+=UPy-V1va`J3H^_H}+jXn8Lk&@yIvN@+rluolQj1>}Zf)c8
zhBxcD_5G@dd8GH+;Xt?~pE{^dERJh7Bi;c|<0CB&A6~7B@Ehg36L<WNGGfJ@=n`r-
zr+nV!xo7u6jmNo3qWd;r*l;=N;WUEQCpxsX60thUqlB0Y?QEW*MyzC37J7V$VNrA=
z%~_J*cSFS>iYljl0+uzBh-JfZK4$W>3T5;>1J%3?lKlvvp{sS6sXHP<hqWpZIih{u
z8EelB!r6<PF<9YOC%iXiRw3^r2QoeCa+sz*8_^<MFX@yI+ApPU0a2_5|8O1($-M~_
zF|=;s#6Hb1ZM*)OdS4JiIgI+RAAeMJmu`E<KM5ZN=+Q<R$^Q0Itq8@K{2E`w!Jidy
zQPY0SsQ&O1T-rgxWu;4MOh;<%2%@W~)@SrS2b;}Gko(*3)(?ap8dW<VJhYBck`5>L
z)fkUxQL}GWwoWNG_AkC_xoP7bU0^&%kclzohbvRw>u_y5!P1<LbZMga<g!9diCagy
z{Oqr&@sD7@it6reFeM`2vB;~HG%;PfXgSd~cw(xx%u9-mn;v>QZv&jIF17okFO=?;
z#-JJde)7L%{)syJxoVkue-ZzRq;pV*oUL2<2%9AuA<-P=#;aNH(@-80&cQ8y-f6_5
z`l_Odk4V;*GBNi|!fKEZA?nQGiLMGMw@BnV+p|v5#E%2yu|?P>lZvS{JTPA+V1|5U
z+cqKtlGyl6Z#CIRtJBFa`bXcpzXZ#DwE!`O>6+Tvj^5Nw)AihT#hl#v5+9CRq8>lP
zw``=;f*^e}ccprN|8P=sGM6P!t)};f85!LqvFjk|`+5dk^Q*5XJuig3Zz;AL&f2=C
z2o!~=x5U85d=#xiTkG4K+am()4KlX}{xU*99d~Vai4Z?)XJIOqPphp_|NS}EdeLKx
z^=6M*n~xeF{Vq;&-bvz=U$W$I&QAjwFDKZH+IBsTa*K=nAi?pY1wdap=)9EpknD8A
z`>%HOpl|b%3==Y|^hIIh%8EIy_>haa-gJpsiC8UpE@iDh-58fq(`Yg@yg5$I(foo)
z)26KENx9V^*Td;@)kn#TbBh5j8|c?59s>UVY4Z*i`=o(wZ=YbQ(w*fi)%m{ontWPO
zA6mBYOMt6OOC5A-u>r$#Z^-wybLh~$vLr<2Y58e87lY1`Vad${^*q}-t1=6#<e}-Z
z8}&Vhxrv2x!g#lZI2a3@9Dvp>Zt&sNUHi{)BYKN(R;H8-vU9E_9qcUj4~GxNVUEA!
zzZoJR4Y2ypnB8B@pu1(c0zw6(fhAoKJ&h7=SlpzxKX^H%bgOk{byJCEpvIqV&oTI=
z&>q9^z|p5tE49h^_hoR0hwY$xvu@AHsfT^ih=a0QZ>}|hbmbNwbT<2CDD7(0D&a(N
zO5feaGQK>V0NI)ra*Mb%UO&=VjMt_;M5(-ug0^}<pWt;jcwO;aMSo49I<+H^`^am;
zQ;4j2aJcjfN6*)@xxo%d--5y|n#;n~KZA!p>Crr;{<nP>KnUSp#F}iQkX2e+*O*G2
z84js645`gEsY5LCiMWs7=GKmb{WSt<=iZu^4bp^=<|X<&*u0UI6%#_M%+x(V+7<jL
zzW0-+pr9RMcUidLG@95Zta_0quZOcXMnkqgf^>X+MrCilM{JDig+hs8wwL4ZdAV?*
z6LUHC;Twux&jDq)`_|fjhN9A6z%Hh-XTBxx#5ct9J;&t-TAWPpTcOI;23iP^I*f7s
z(U$gyRrr29v<b5iZ;#l;4rLt~olcUMX(st?dL}3RoY+($iAJ=GW&7K7;yyxp4WQ5w
zE8%PyBy5hjhF4M?YhL%8H9Z5~=Xwo=n!8cSJ7B>vSTNNx{@1_^Fy}`-6;d=OgMeiq
zW_;_0OT%!7Q$e=hjp6x-rUYc^J6f39+sF1U<yu|+YM0tGObH-$QmxQj1BAJ`t7>ub
zrX7>35%4b}NanH!u0t|^4Q6@Lz`A$l&L-5-tj$R-6Y_e1#RK*tV>%b=U6XnhCSzg#
zX|WjIc=t=JcT%sCJd%TXE(QYg(PQs0bu%AP%wJ3OG*ZX+d>ryGzZN!ZcAHxxmiC`M
z|6nFV%}aIi9J3TxFI38muW3qj{G73qNbt&0BPi2rC6|!P!0Z@Kt&wCjCf`~Dd@e~w
zNWXg(d#THB^i>!6cCv@l;u(HcSI)q@3JhC!{nV1tR!wT0M(m5Sl^Q-mPvfkp3F75-
z_Nv#C%#X96D}{Y!20mba2`#eFWO0rF4C(FtQqpY;8B_yu02wd%sgJ!H+rE$4=ILh{
zyy8FS;`pAq&e2jrAb$3^>ge~5e9MGWdnM6*rNz?O2>SY)>4|_$)DF3cUHW_MDXRm>
z)j^@#YFxedzWS}7m{I9UC4N-G9sfpV8o#ZK`Gaxbz~~!wJ*l}fNQj)J6MMN5xjocv
z%El`A%q;8Y%RAvLlopGV2{r|qwbE8m&s;8}!lV|P?S`-MB1%2HBrdZuQ^*bQtr!l3
zNhi|?roxEz?-hS5+ufRXR!Ar_(84UYIA+#3amu)|tZa!RY)PadM${0t!AsCOi7a}n
zP4PN193tiPCT0E5Q()H?v5Ks&{xEK}|J*q+Xp3jBPlJ)o-DvaP=Yn~in(Tc={kZyR
zufmSSCWTqGqCn`>o#eZ;lE}h60B%hDbT7oot|{QdFW);uRe$ObY<62&_YVrg^DIAm
zIQVYcr=j7-S>H>`<r$p)JleFFP_CeZ)J8#q)^#B_$D$|lk4Ju7j`|V#q~z{>?O}jJ
zzwZ={G%P31``u8JSPb`G86h7$tbmxjz2l}2!=YSKVoHp9(UYWFZ!s%9GWz&GvI<Qs
z#4>KGrnfBY?bhPAIP}14FElIO4c7ikXHm2^R3-LHS3$x(gVv1~_tavO%Dt88QF`;&
zVTZY*UtjCnuBNWY)hTswyPC4wBN%Rp;eS0eo-mh{SB)l#{q+<dU13^q5qsqTKjwLJ
zI+?GDC&sA$=tWzvn?9Uy_hzw8?s%Tk<#+7|entP#eLPZ1=Vj`Z2OWm}=zl;|exrCk
z9WHpkapv8;1a&klSnOY0U*UA>E6R#w-Aa1N(T+F-qW4CD8A9z`llT;1>w7fbSs{ZO
zGs1lWWQQoyu!mNtq_c#M&#9h2t@$i7UtFRE82fW7l6n7qpbiHghF>s^qBf+!9{Peh
zIpeQ|)=5|1;W!dbf^5n`Jf!R+hdx$A@MD%CmU%pH-yK2)HN!9ryVAYY?e{hc?c}T4
z@$=1GlKJ*k;rhxt#jp=zA3{@GAq<|_MmJSYuS8Xk2Z`}06t`a+I8TbAe7tifBT13u
zI**>5b>Jt)&EbiB32@=8ML^%6w*Nc?e9uq#M_nJ~f9y7hx(q`p{fPUFHXt7F&J8M3
z`)kn6-;=F?+bKWOz%*iM?QX#F_hDf;>EE20#6DQ-N7MciHv>Zz{iOb+ZSI$g$@x8s
zi<)W^o?{NSidznyh!-e1nPEtJas8*&1kX&v&^co|(7x?ZpsIu(*y1u-jaq;|%=lPr
zFZLl>@EYvz6$SbqIaePiD8%0}y}HuY-EX_S6?;$hfu-#`exC29r7I1m=d=jjmc41E
z<Svf5_XqXc${Z;V#GXJck_z2ji{YAcU}~DXZt+ab8iC~yS?RdJ@C+U|k3T;0hatz{
z%iH$Z47(A1X36Ltx72BogG-H9McLLgs$~PC(g(Mz)Dg&h^1^=0fWB8Kx#d*U*N|;>
z&<bwOV<TSHrnCBg@(5q%N0)Yj-KKt*li{KYXHuC9%g$xV$*6un%=EI_cdVn2NksK8
ziI^UJ(_6oC>6e8$pFp=TlNo>8m)*&WEsOirSY1u8-{ohxUb#T3MrqX(la#^I8zyDb
zA|m1i8cxxr(W<&)q!;_)uPFOU+sR;c1QEL)9>qmIy0$g~Sk-Z6X7IvxL|8LS;1~^2
z7uz2r-z4hz_@?Cc@EJ?%3vZg&`?IID{8n7_!zj$WR3DPSSo?5>25>AqszMz-#PJ89
zZ+XC%i{;t%QCr)}=BQbQ#T~BT+^OI7!@9(_^kT_mOo4v>PmXMoK=bokdKcdVbrpBv
zwUN2(=~oB~)X@tAlff=GC=#&8_Nxn0ZWSQZe$vUx{Zmz=<_gf;?M#vUez1&qU})E0
zrr}9L$KHTwmeq~O4(ZB1TML>g&Ha8*RJAJ-`{);HAAVeBoL*O#B!mfbnkWtM+WjY=
z;`eOfov-nKhPKlK+PS@rpUFk>S=0UmJy9pp!j9}D-{h0#b|t6Er%U+<&LgO|hg?E4
z>l9!2Dlhkv!*F?$^e3Pu6gog0Khi~34o$9e7}PoL*@-Yjm~Ma{*#BmJ*WV}Qo96Bk
zYNkHh+;DU`+9GasYcJH6SA&)UCY)`pPncpJQL{-dD~?42DfAC|GfHhmnG6zpKyZ=k
zx4jqVlx_DPVsnwuIO>Yvu7@2@5xEn%a6omIAv73`SNC7SV7A%vNq37c#>Gz-dy4qK
zvyv9^3DSnln%-tjRBK18mEIxmA@asc;IE-9rev2XXcpKd=O2D)#Xd0HdsJwYEc6I(
zbF3<3VE9x$@q#K<V#EIEt&<sk&sQi@F0ee_^AZopzV4i+V<6<1Yet@M^G~*T1}Pwd
zviUKWscJi<9BKbZ`B;^YWfFak#z3pDm%HjgV8#wUX_feg8B-^InDVPud95`bRg9Gt
zPZ_Guu#}tqyvRk9715WXCqC))#OD~`UUq{G+^wYc2aa%tW6(h0E(^nc^F4Ezy#$+@
z)XBDR-lnyttIWtD$jE#%sB+;3(976Q4`+6CtYMa-<o9^6<~~eHlGh!{JfIJ`Z&A^(
z7zl8zM8$)#tJ6P;%bvpv#n!D|<j*9lic}(t1h&Igq2D(&3ir|Hpw=4GUnA&G-M3>x
zK6<ng;by=uhoUUamHwbcxNF9~)s}TScrUS6i!#!SF-kU#UMQt|-+CK;&%x?4L<ATV
zXY(C@uuSw^5gUVb8#z3fcwvTFHm*ep1=AA#{;26XKfLF2UdPFW@K-{FUhJ1!b~n(4
z34SPDgdW(-`Jtj-<8yPZDjpxKwWDARws-Vd!<X4t{9R)W5D$detlGEhSs<L|KVPF<
z&vJs(emq$OzpZ@@#y?MFJKnkCKRDFTKIC&_ySItp-A(`6&iYqG(EXNa<exK`8iMyZ
zK27DinsV-<Tdt6jn<z?t>V5r+RFtbHWsDruG;<J$WgWt_(@8bubl%iq=9f~{{C4$@
zGu3}bg@-HH0@+kk%f%yUOn<cYTy+nMLsKnR1a)a&a^hbdgzNLL-F#`^HTfnMVs7?s
zn>2C@R)kw@R8fN7Nhf}?jqWsUkIM$BQJjE1<0KwQ!9Vy~w`08YZt6UX#;$eewQ(Rs
zg40Y2%$hHX>%xr8hH7N$b!<NgBdt#L3O9#2vCf#D?IK_1O%CrRGx@1MWiH!!eMpeO
zz_{#tWF&53B;<xHpDGuaybs}Mw_whGk1F7&9wJ3uI|v3iJm=KsbMBdlf2~O4rJe2R
zjRL0?vyhF~hV9F+u`PbYvL&F=VR}`o#~#P9Q_-hWkIQX~%Djx1J%{e_5nUL1G1m%^
zEpt(GwH3^8*IC%`$rFly8rXoaA_0u?<~la)xOvz%AFX|0%eIZ%m2;Q9FT)ijh5isW
zu(;6XmtCxuZT-{utNlu)%BJV=6kUCiSl@~nc%0Eu^_V4#%tNdUejs5yJ~f2ul`zdK
zv&yPzqyJ5TnOmivUz`KNpxUZ%_+i3|1ZFGAr_#1`N_xsVg{HXhu*td2zO}k$J=b+@
zx@@Kb=5G~C|F~Q+t_8m`k8uKeqoZG4w@#`-Cw9gv4DM04k7HP&!s)|NLYSnS{+iQu
zhbDx|Y{>)WW}+EzTPC&lKce8sn^|14=GV&&M&`3!dfpIvZ)BRwy?~^nI4C!WddiiZ
z4}!WXFN|Kn*L+;-F?#ll!9H2(t4AGFbvE5Z4=T~#r93V{2-=bmYarvs^&8&mjyD9E
zFuW;VR<mQ%U*i|ZM~9J~D-3qZ++|~2p}^7+R!57oD?gAY-)Z^MitDrZVMbQYPCGjv
z^aK5ds&f;i5c0)>o>nOch4!y>j{_WrA59csP7$>WT;Nf2W?0o|$P@m~IZNlv2iw?v
z!5pJEZD4Y|C|y06r!FwY?n@&B7%@URp=uBaZE?a5PLJm<($%9C>+VjA=lP*WYTumQ
z{CmRX;R=?(BD1W8iFCbzlKa^Co{GqD4SB_?LBXuERu~@<;PE)5^mM$GG@?p9Bv%Ph
z1Edz_S=OU7Dyh|o5eZYKWSk9nbY<^ib~9{n-044rzDYTWlBE^R%Wp~`5(f#iz$!t0
z5^LnAUGUt1M!Rmkhiv*;cEFD+78xGuUp57GKW&4T)%bfc^^xC#9)jF#XLt_7L-aj+
zUC434H$R{C-c-K)cB`17#ptRbt6(4WPS6YajfL54rjyhlq3#Tp5Bp#bKOonA!~e{m
zJU%R}yX!FI=WxNDYN@%)Is-M?nIzV<wOi=6hXJbIKqh7J{>@j)`xGuz=|RUEyJw;V
zHYX1Nre;?%+O>=9(Fano+^5<M=8(FR$wI96%xlh$B&=5^Wms~5xF>*|!@xt!W7cjY
zxY3@!eu+qJAt?pG^k;fGGD<g9X%%Y$o3%a<lVLC5KFPRb6v5gB7_wMl|NjhS1@Uji
zuE4cYL;XSOrt6vHVW3w+Sn!?hV>__4X`!Zl!!8QAE)FIc%B0z8L?Jz|W0|9}GN+us
z7r1Z?a-MOoggrrA@W|zQ-&^Kf((87)e0qedGKms&%WhYFjP_+|F(>iiw3e9qgw&SC
zj7w69<M2$>oy%Lc%{Vt<J74BuPtUE~Icfp5?vL*Ywj6_<UFi}GZ<)5&@kT^>yz8Sc
zM*E}t8b3vaZ-}PlTn{*(UUxe$F<vY;Tznr;_?ZV;qb^5<>vi9qdwnqYkyXDizc!}h
zEm3QH#_E$zf>kEbJ3KDi>~=Mei1PElQvAb@1OvmT{A$h)8_tlFoo79(5ZlNV@e3hT
z<k)dtnh*J#LnzOz!_&d;jMM2Y*|L5<dNMy-IFh+9D)Hxd+VS1q_Go}GTHPJHZFExW
z_8O^?QVYn_uf}h>ptE6(BYZA?A|Y#EcB9W)7NqPVefP~k<wSl*Io^0J%1l}P_Meck
zo4f?CldSI*1i6Fe*Fy~3$z3D0w$_NhM>k6>rRkG-O<@+nQmnjR=oPxOE~9j>sdd;}
z2=&majP6F{M4BZa0@>(mP0S?ZPo#QM$axRkbwSkyxc%RJ8G9hb+pu=xEqaaC3oc8(
zfeY>8k+3lf4w|88>WO_VZE@-Y8?CMnBQ}-}hC*?*Wrb)L%hr0+=dKCQ1n55IB!jE&
zJzM>c)4o$Oy<bC4f3Tz}_ejM4?a0unvZc+>%YB^tHq6;_A#Xq%ms-30ktaZVz5!7*
zY18u8!lxN?ay3KozV++8BFlJtYQ!^Cak>!0P7vSM=QLckKdlB<gfx5fqS4PX`42n5
zdWW?E9@>z$4yq1R?TWXGe$E%KJ>)`W7+P@lWS2(D*`g+_<gZ}-o9`!%a!f7EjSy<`
z*9c~_rk{S?5t@!3I_FmJd{Mr65MN=nmN+*TfX0D4N)hpA+$JJ{Dm$-Vi7v-r4|<Jb
zmy1%&-nCDaogbL4aa%J|J4|!hA{}T(SU??T7hPHhhD{viS3~C1Y&tfz;{8$}u?w!9
zkOJ}MZB`|RK(~5dV}>YfDR*B9S|OGkoRvvNP3My4eB8LioAR>tXz`uG^7r=E-%`u9
zxtGy(7AdC0Ju2*Qj=XUKm|EcV3)ebhR+I-jm(%4Lp3>Ofk`{-&cOv--I)%e`V>Qwu
zgX<;ELxY{}9JxZ<-|nh&2;QT!S^iq*wx&l<VOY0UtDK0~Oz+#O+~$%&LM{ZPL{`Jx
zkj_x{iyj|B?b=cpgI)Dg9A~^n<#$3|MP5s@RH!uNd~MQIZH{?*&hrVa4{F95x?FQy
z&FS{HwOBpN?Z=k-wG=ALte?lgOijdcZByBkee!$m)_;QLN8uh(Rd?Eaa|m2udOgp?
zkZo`tUdm`(tJe2XwVXwSKdBsoqt~P(R-dm__s2a(D{Yl?Q?D|kf*06t@m;pUti4?d
z3Yi;%7)_mSB)RhMM}4hG`G56Knq1`=@GG7SzhW6WCiVel%G1bT+ZZxT^;1V~2a10@
z#fUm7?43!J-I5Q@?-biiF7L31Sx5~R;rvFdu-V7c#s7c!sQ&RZY}!tAL40${Gl2$!
zG+$gefp@D7Px7@eu?%}KE+0p7YI1CY<J~;XQ6?c}^S=MzKH8e~TX1j>?-?(G0%;2l
z?-|kyu8Q6K>X&U_It6~}|Ln4VuLqqXg#y^m_f{LmeRNfwt!I1K)I1t%x{IH$pgN#2
zqYOllxNgJ9&JSmSfDhL_IV9y4|403x)Gvs3>%{@X<A4j<&mY5Z9e1r%@l<hE7bzE)
z{F@0xL0-!uDXC1j*tJh+@pcQzqiwP9oHt+K!2dhZj#r1=Su;;}5@WsOQ{`7>s5QkI
z;I+UDPYL3*LsX70enO;{Y<qmG8<P9`Fzwi##r`NhnlR&bloTOk{7~|N+ryJh$lH&K
za!$F?JtgabTs~XP@qlLskz!BA5KTx3*BkC0VLJ?(td7quo7kU9Y`9<Asg;wcFsS*@
zD4Mv>Rljj;zoA*Quhp78LPyd?L@}0frsV!S7jKNlJoG!Ku+qloJg@k?kC4TR<aoy^
zgCSIvkTU)&S?xV)2k9x>oaw9a%9k@<_QlQT3V5hYR}fH(6{E*A7FO2*8T)X?u;xr^
z&`V)=c$JUM!l>>h&%1i*r_fs(*br~&Ui!m17&a3n126_LVAK%_o#Zz&Mz69CFdWMD
zBTKvda4X~W5d)^tRNdIkSvJ-O=QFkh(mqY(lT~{v<Of_j{1<85<nlV9#e$Y%yhn0J
zQ7s%c3jRb3KU$qq=WA)YOU&cn>2rgm_uX8t&fFKlhV0#u>$@m6qbbl}w?2d#yr(PA
z&CuT;4nkIHOT4TuFyTYJ#1uNgr%Xs<E~`xm<!4x=>CqWs+qv9(q|#T-D_SU9{<asX
zUj`wu#?a^Ole7)Oe6z>2i;MX1FjO4bcU&rJy?1g5*l3Ku){<OtX`R~wdJW{c+t!;c
zDO&|gV6*);TUUn3d*!C^KzMJ_@b$QAJsa@?#l@9*PxH|HO}|wmoXYqM=&g@Q7~eb2
z2m9fgDCuzX0Yn&}4yET*I*upT;%hyifetzV7QYf+#qe?zgj2Qql<rve)*=>VDHj)W
z*NqX0>KKCL$_Rw`U>wkf$*x9|)PV<jN$olKq9%hnh<omq%~-;g!!V)DqcoV3E3ZkI
zEMLL(w8=pLQS|+*%|9atgM!ufWQV~=YWZ2KZCI>=-5W_bd3bjDY&3-)>lHtjlq*AD
z0PfxEtAQQ%wUk?IR(aU7P6FPpxl+8}dTIjw+D0Ri#$XFapQ&%^GV?EBTAi3Ek{o@u
z`QkzuTa}wy%c2Wro8D*W>5D%lLFkbz?>j=2{j$iHB2Wc`%ql-bpY(D1LORL=AcBC(
z`5h5FeI18?$KE~sj^tJ<EiTTs*pwS1YdF2S=bFKQV0lya?Rvq>uniI=;dFp3Z`-%k
z7I}kEzNmYSkLT!4v3X=7-Mdb(E;t}SWA)4*^LrR}2jA{l+jx$q)ps`Ax$Hd@xCM;+
zy}?wUtvJ%pGx;PSM_~E<E)E&DMCtkKL~qV3DZsjj^DlrIm|w<~Rw{7em7R<5Mdh9!
zhHa0YEo4u=V_|?&jG4U8*l?Q)3n8-{bFonz{@6#bJ2x}K2GU-po9MzM*KXceY?qvr
z`fg_*0yz0Ov&q8QRw<{aNxp1>ot5imJQI}4@C%<-_KfV!nP+EMcGGUG)0^y6muIuj
z{5$YoFhm*!fEKM{lO633DZJ$T=r7qT-d~*x_95G87qSsO!p)-OjlF=1qqjcB`WWD}
z@M0f$L2kleHsJz+Q@$0^U+^WJf@*Pi&o)rrHv8GbP|aLdpvUoHCepl1xS7-vvr%)i
zu1vARo6k2{24A}hokqN^0$Idiq3Wl9N6-~JcS=4PoE6-g9c+!zhPmXn>Bx`A-5-wy
z3MY;?v<wtiq+xUTXMidt$S!L~No*40<*)QwHs7yy1Nmn;$D3L`yybB9*XG54hmnC(
zr$;+P*rp$AZeL@YJ%1w!4o1D6Gw?GtwFgwnk$u;fSjWW9a3y}wj7z7t0X_2hwUkr$
zAV?f*)IBC#zGb7>2zY@^z}pBAiaZ#=D)R2CeQ!>ta-Lq{ARl$eEfRc55zYBxd#6{Y
zy<^RqoS(WUyK8d)3H?36MzDv>w5P|Epl{QO50s!H&=lLig@+mQhyQ*@j8sDM-VY6H
z2BwLgFaCZ)jM!1q4%Xk1AGFXnw4wc6$#7YmPqlSQelMVIuulQWK#3nx3I!2z3P+x9
zoWq<Knt#CWg?D0+ca?|3T6O-K{hTa`AZ*TxZI6j%E)#Vsl4Q8UIQAvUEgQa!k;S4Q
zHOK2cZ1C+A?oImzVnPXEpSd^W?#No32?>ZJ^St;=Os!U1cETQJ0C%eZ-0hhQH7LZ}
z&oWKa<J~f=?^qf3xiS+)*9-Z`-*}vDzETf@=S&O$7!6SW^Sj(FDTTmz3R<@0q;i`$
z91T9Bk6!Ev=c@hmFuB6Fqrx}5PoDL^ya$D@m0i`n7FNL*Bpauc6!R}OXULMi$K92(
znQ)Xo;r})XQmNA^CczzrwZ<5}1vTV*Lw@1~Ui=%Dl01EwZ6+meC^q)%qO@A$x}V%&
z-8EzA2@aooyXqUH7e0?pSfsXmlMvd<KJF=NKINDnujQ!8j}3Ku;I?c$lPh0t-Hd^9
zjX7uAeJ<NG0w9wGLbkU3YXIG!HXHi8Z_|^8w!Mv_y!Ob<1e3&T67ol#l%`jB=Hll)
zliC6vuHns^?}CIwlcv*PB`qq`3TdwV#oqlAsc0Oozr>DpcF*-rgoSw~$lXPRdF!ny
zcG{cvufuKb0Pf62VkKnGl{RT-t~-Rk7-!D;p9}iLQQVuG05T=9r}Za;eYeq9H(a+5
zX4#w~)KOIYf>v(EMtb{Pfo;6FhIDs*Q--YJt5cr;afd^n#e(AXap{6t3^h!M{}VhF
zb;klSyv+8i92_O_HF3$%A;3G5x$K`rT43DKcgD|^Tqx(FF!J_}Zs)gFVy}`2y~UzK
zv)OgmKN=pZ!-l5;J=%?s?u>H*`2g!ihOHnECI>4kalR;28aVRBH9*7bfa%))zxUD7
zWhQTe9(#<EeJ;wqODo}%qm;f)+)bDIhdR}xM&4|F3c|#~(rwtrEsge#0RF6>ZvAzr
zEF0LC45%zQ06G~stk(KMVR&6&iB+zwF|d92m6^84<+$nVyEd6+;)a|P9Fs?}s}`ED
z4ZqXfx@w;G;h+@4o9xs-_6}0XcL%=z7bMT0?fh>=%$dXWV%5B(PUsjir;}u`lVSmL
zW!RgfEI2cB{ijRp4OQ(XZ}m1x4tbx;?!L<$F2wrkp#GjXb~}qq<@MfStWmNio^&#m
z`04xpWYZ)8i1}^qOEEA8#-k>zL0Y6YFHa=|5sg?ORI0ePNj`ASak!r^W(90O_h(Z|
zgo&m{unDLd54ap9m6Uv<_ZJB`Qxm!G5PJd9Jei86_*1GIFia!NhliN|4qO{xfL&sa
zkd_M)cTFsTQFF)H@W*fU99gV<M+BEXy_5+@8femWx&`S2?8wy<YrRr#CGl))bU5ok
zGXJ$DuOo=bZ?czt7$2=DliN=5;f$7hXIBYa*+Nnjxz_x=rp=FVvPI-9@c1m0pfoOi
z$H$p^D4t<q(=N$tZ;8j1l=!oV$5%)q<Nr#kPw$kBrj?g`bP_KRvEJ@~QD$KyfUPJ!
zL^9=A8o^@35-jPuF$>mo%A}aS2X8Z;-XO2^!ezR@Xl({$e8gQfhCr(P>^Bx~@mvwN
z+%FOyg(AlC%~&g135Mi)O9xhedgE)(G#_mGUTzPrQxb)5mqq54<O^E*A*!=%#<mia
zwr0&{YK|7wxlO@J4d9#0^*!T;ZGa5>&Cy#jrNmF|83sPXp=bORDD7iYbFq8bQAJ#y
zFV+B^fVjkY2+j8BqDq?kGg#S*T2vNX@U}_WQn7SFIUp@D(W;C)Ao2;k9i@P`Pd4%E
z96`axn{CX)#)6k0xbhC*7ciTX8!-)hiS5pkUy|JZ{3K5cD%TrV88)Fgyk*wuQ7RHg
z!V~1DPh@YJ0pUOWxA;^OlW@UO4wiJe5fL)iJuufDH$^p^+dP5(;9eZQ5CD;knWEZ+
zO#IJ}+zctJE#j%>yh7Vt2@Y1CeujP>_FKB9p{|y-o}uM7gK)*giJ;~tfra+X;3wF9
zc?p;uT7~_Xo6|&l{4}lZlY4UTNZ~D?X_Z<vOJ3pZWKfUUz>$b+#|@DF3-FH?T<R7|
zL$b}_up@`QoB9QJus)>6j!=Tu3bOERi%tSWvCgm)^l^{^LUPVPF3>&XnFH-#)cVK~
zGY-wZqN3N+EIyW-H|Ga4#mF~*n#JKC9mfY`L>U6r{s?vAq~XSg&L7tS#b^Eel>P>)
znTzn*!n`%oa4O8(FD!h86J8<Sz1yqK3JMj4Dlbdrf~$Np-8(-cJa~ZeO*HLULy;we
zi{(6Wj;+7Y3ER#0!Cs+!dfTy(O{(t|&XnSJxV<5S6bxLqDN;o_>tOV9T(u_t_z_w_
zfZFzr5H;3s2g^_o(m5E;>)5_WMc8uVC{DLZm24M+T_udJAe1gnlFt>#L8tJZd~UcP
z0=Ckfyp@wx*6H1*FiX{PAv<o%U_9r{EbEMt%EL;hCN>7f{KHFpi$a*oqISGL^tZvi
z@4;llW@%BW;3u*ZWm_K52fe-)<cl95o`~6Q<kJ6{w$+$(6~6M(BMv9ZT4BHo^BUf1
zUW_Wa@diRZw2|+x6bHhd-zfrh_&XB|OkdHOLt?Me!RGV%`aru?*t!mB&MA<e-FAbM
z&WCT8baowsIj_b`)4?!xyl7(gz)+qWjxb(&qYlypg1+C7HmRM4C;JX$6L){Av#OhF
zo9cs_yB7|x)b3JCo(ORZmIUAc_C<clOo`jt)M^U+7%|5!x5IzID=&YzCMIjE)jk*#
zD_Km<rD{4Rfu#GoJ&M6ZND3T|dV$!+2AL%AeqHa*v(XN&3CNErHR$M*C3W2U&Rm}p
zmw^&Oecz+c(YQ*muQ@p+{3@9IZ3ms)n8C9gRk3bl8gY8gC=~l<vfFlNhF*Wo;WYEa
zp5z$GGU*2cS+=xXlFl1SjCOMe&v)TMz2VEjK(Rga<$X8oVPu{f@1Ep1sW><U1X@SO
zy$^w`&8{Vd!K?pNSC7KfG*QgSdoA}Tnd)gH9GkvE%x8n!tL)`hlyE&B9Uhso@-Kw=
z$rjJ<=tdCVoF|lHCx?iLv#JWOI{E!92kzvSB@T9z^}T^mEd6;4BctHMloIWjH8yTw
zY1*M;+^tuT8MiSFILu45tUZ_j?qahSu7no@i7ScSP>wHxfkNGP51$wmcKDTL)m%Ta
zn?fE0fO5dV#)bgP!2P?D3{P0$^z}GD74pq=TdaxeVzA^y^o<y_eII9G3=4IQB;egK
z2$0OAeqo0hzeRe=hUq4!Fkv=<q5YR1hwH`VvpRXt78!hZ&XGttR~xJBrc2X;#E4Tf
z!V|cWol5OalDetd5f>G*I&QA66?V}Ro^+ziPt}pC>Mww1{B3RJKrOYT)L+)m?c8=%
z(8kQO@?>&vO46{YIO)5vVB>BDg0Ph_q^o4JNF4T150!KVKL)xM&5dv*hj)<+b{KMg
zXz)3CTA4APKOShdBPhHl&o;)@!~3f81m?ETfL3sjn$O{n_67nY90TuAxA*B<J6rrw
zr(FN^?gqz0pr55HmG56#1f`gsiwO@F$0BgY-N6n>{R&~+0atFDp9_vDR1zMrO@J#>
zZ(xygHTk2uNXjJ<Z11=7_b8!suuZXBbGNKz-v>Ik$IDx_v#@|V0F|xRZO{D|0}o)1
zEdzM?pLEK?-hh(&q8Z89#-ve%UpF@2#0Im{Vn|b}420(WP?;4VG#*9X0TD-ZXVz*9
z*VA#uJs=~fw+30{f|vU!!<(*;dlIX;71Q*G*%=W}jv=aHu(PgBQ1pQLt6<8<)5TK7
zEI4jrl*v6eu<upv02hI;1zdy|!~*X>T>S}SdKLZq4WAm9rfi_LV-t{($PZ4J-nr|D
zEe49go2O4~zJtDNK<fyopQXtzf@)S)Tgqy*lv71K8@;D7IS&p$^;t#(xP`Y`)(jS!
z`=}%9slszNO<!UKBg`a%<p)ra1Jt6!pzzjrem3nWiJ27+;{R@Ku|3bDg#uCurwmaN
z7Tn50`oH?ofb_JnnLt&4l1&#ghW^3Jh=2sCTQhZkt`u^adwR{y4(14Lb50V@pSy9C
zWU5J?{e<or>@k`4ayzLXEDU>SfT}U6#tm`Pu}litSC%|x24@-Z4LZX1w~1cT-TO%D
zE4BkG+gXP!V0|!zHftWd0Z8ej@nn>oRGUR`8AM{>?gIKxd`-W$#{yU8&I&GxZB+H{
z<aYg;YsPTzYM0g7zCGT+PJ6-6p#@zMh(#$4M)fxWaXI_u(j(1<t2)C=>JBNr>~*X<
z>I>~~b#Ae6zo)ndF3L}oX2&~N7O<~NtUI`TM3z92Ugp^B)?yqXpHewq>I4G0e-eq0
z6a!aMt>J3em5Y2%<l#O+I2cQ9N<AY_U3i-WV1O`GSNN4?a}aH&C})NCu5{D;V*klJ
zi%%RZE8hK{;W6BoWO92Tw)Og>R>Fs-q<lz;WZhAAT_<md>G0w!O8CYUsmU6gSKm~W
z(<iN7(%6j$sfLFZqIv-*b&AeOjrx~o_{M}Q3>KVefis*Z_e-aQzGVtpoDI&`AC5i*
zZ5b#rF?eIh?&Fct5J&xyxdl4XXq+%t!*<+u2~BI+gM=s+7#;$O#-H7O3*T3Xg*585
zXAW)uQnjw1>EcjNGtzyWP;{)k*ZXY;t5BP^nD-k8?dq^JkStuVJfR^WB}&e0Z$CD%
z%Kv@9!2g~3#n^N)C(pi!<ShZcI8WD`o^b8WB*pDEG~zR4Ol+Wx$4}ccd0_4~$13r5
zkz>Y@R<fYmy*Dic6TI!ebuGO6O7>~Xif5zTTtM@lK17&0=-h+p2TeSV#rL+r;@iC+
zQa;=`5B{U{LeFs8XL7apV2k%dBn5NiD*%xyhaD-_7wz39W%1iQfk2jy%7xXTRQ;1q
zHZzT^KRR;o?=5=YlZl3T2k6?8R?iRvIefhi=o2(hz0~U$Hd^%4D=BIFe5eq=-^ay>
z<`lRl;C*nF;#S%Cf~bUU48I>Q!GSI?0|?U7Gn@Z3qVZnQ{5~EA5Fct-f6(=M^}+8Y
zc!U;QMk5>qQ8u^{;&z5%XW!~cBNd1L^a8*Zf`||Y;NWcoB|Q4IJIOiVpT?<9<3>tF
zT)lN_z%YbN@*1CgeoKt>9dABjO3@xZ8GE&NwI#u>=K>(N?ZCcqn4z+ArHBA>`!`uy
z_!ADzDvgLog8BO6e62h#xy3C`sAO1~))qu)u~-sR05nrh48={XQDN->x<*kjb6N)+
zT&<$;=pgf<S;}MyI_`h{_O>#qG~Cqfmq@OEI%!RD(7_+fdR~Fh+ktrIg-5^?-7&c4
z`KAT(`)%sQ(=T_)*PT(?alV?7n>@W!JcGpQ+Bs6?7uEsa<Pn&AXc?y{zq#OffV3@9
z9}udQ)!mnK+S3PB+_9BniZacc8@hqpZR1dBkShT@o@Y2RzvxHuYpO~SC+XjQ4GJFi
zn)om#>*tv$r{MtyOZ3h~V|NP#tdt|x-d_a6A2)fbqu)oa>(k2d!ipi|m;EUpes-<)
z#EnUE$}=1Pte3z*nkDS3p4`)gbw6t17`Vb;#{pbEr=r!Eq1Ky3kY{AH)iL0bS?n8l
zZF2=9<10GdD<2A|uXu{YO$Y#+V`Cp<k#CJ&7$e`>KcUv`HCMjc@*Q+l`m-*lvZ`XS
zS*Q76Wx}~Aw5!9Ss<-3-0wAd*y&(aH5Sw+M7~6NZWj%dRu8hPt$=Tm{9P}1`yXRHH
zpynmpwN5I>wO#D2zOt3q<=HJ|&G%aM33GmpVs9(Gif8qcDGYrYTu{80AXm&YfhIy{
zY)~MGpGzpxVlcU!N4w5mB0h^(aV-&u%k*nJ3Q%SH(AA?#=jZIutzZbPUq&^kqgDtH
z#C3<y)qR3zCp%!GD09cRf(+ow^`v>Ueqn=;xZxUuyc1NLq+D3lC(t@al<F{xWL}4A
z&T_o3>PT&cYW>{MPp~trF+P&+n8n9s{tx`&1I1&(BgE>I1+W)!B-iyHbt}~Y84Dvb
zb|gT`t^Zh~7W|)Fp=@DfVPgIN;tJIBZNTII(<C!k=kZ7YjC%MDDLVc2JeQjH&6L~w
zdsbEa$R2xt?s8I`b&tW<fu5%8YEt_jiAd^wxp(i)0}6VjSFv-j-KhB?1ubpumEq}j
zAJ&<)y)lRTqrUNevg!Q2OKSX{i`LrESdhX<N6+v58@c_)vD@~tP3))E$*Zik{FJD>
zZco$5Z?-Qdz|K>toGqYa1Wl6BUv#ukO@OH0(M!xC<nICyB+zyJZ<Q#AgPk))_zP=H
zjuuvM$du1I`88>n=T(&IMK|3{p)5t63nLi|yhNk{DMi0}h($B9?}X1iJ??^O+Aex*
zbOpb;Uj_P!X~gg4{7!~-`s^-AHQ6GI$`_jT1$3LJei8LZFOP|oq?cT0MQCut>U11C
zv%>&dd>8!@g7_FOuhpfjbqT*p?4B46Ta~y^G`%YM<f{b36aRTlo%P<Zyin@*wkd;w
zlcZkAELx@gvOMttpcSS(`I6-wpe$3#)5z48Jhtp&p@rgmYpYRnHD_q9%Ei;3%I~30
z0p7{D7$~;&zk~!82Y2f8t?d5fWbP48m~zQ9%99?5_)KV4I&zX9&f3X0i*9e6lw2W}
zrq~19lwP0nzIoYx3vhgGTFs!}#OQjk5%9MNHAxU3{3Zk8-8aJ-7>B#GiYXJD&x`6q
zEFUbUD8EsOSQ!6t&K(GE{b+hKyRk*$U|eUm!nI(r;YI6eXT8K!LjDyqTp2JxY~AG8
zje=!x)_xPjvNMs|E?L|)e)c9*F<Uq#<+Y}fTOrfUy#2_*7jfY29Zu;ooUxy0LAnk+
zh0uOUv+bs8s_Eh=yLR3&j4x2;fhv&Sl-{9xHx^_RL{j=kek5>ouilGPXqGTxhUpPY
zO>sbd+~$A;z`44t^|h|-2^J>Qmt1+cxeT`TbA|Bfg{9&zHh0GE@sGGU%!{w*ZPj_L
zC?%3?nQPTFi8PHZ>|j5D{(l2(2aa}GIxcHkIKSk4u{EL4Ymq=L95<mJ0_DRg{h&v|
zFra+Nd30U*UoC>b$a(=5E#4&QzfRwdK%BOEu6^PC*u2ihTt|s0(dP4-mTQ?nHlxC7
z`L;FBVp(~aD8g2rAcM4e5FA|AI5{Tgw?-B1;)JCUZ?$Rnj`}0{*rSwZK^AT!>wcdx
zHodf)x}#3PsO{C*8*fEYx$xs@fz6*(>EH;0c;_0axm0a#K#Goj4-KR2tYhg&c|J`M
zYyW@s(b&f-eN2g`C>F0y^LCXNE-oxpD}HEB2j9y8#vW%6K>K?x0FVFYNAiQfliY5Y
z=VLgH8oR4lTvR<>ib+4NPgW1^oe!Y5?Ml7r=aV-c1kJoaD4uWyk~eKx5vI4V3`8$w
z5zQ6Ku!fl%pD@BB^yaJ3f(OkMWhHVcb&n2E2JHD{Kfn;X5ckczg?hexwBhjz<M|rM
zRQG!)^9Ue-%xMUyoF=keAfUJEDnaGvHBZ0!7W*M|#bTIK4l(B-fU@)K2^Z!R<Lv8y
z{7m!Y81#}GWH|C$$9Af@Ze}Ev!?4?~qvP=R(0%dY3&mC`p(7)UGnDb|1J%A^I8t?f
zfx~9r_E1GF4Wu6z_(vQnRk+K`ap>7<>MbAi82AS5fB%Lv-OdZOZZZ7sPwmw02w&88
zPCqOFmAxeu2!ar`0@}v;XSN7S8oF~)cuMsdWV;`-xdAbx*)gf2AqcC@;+KE^W6ny7
zE^2%BBIDn})RGbX<Lpx)<9niYi(%6y9{tMxB+0}xc|E)?Ij+MAg@nOUC^4_eHKJ;B
zwk2{70F(6O)5qrChAid)nF3U#Zo1bEQ{|&;^|Dj+2=R{&0$3cv7d|c3P}Cs*d!!Wz
zPkY<)JfPOw%00(!(z6m)3-+%m)4kOKs>#uTtUNhO&!72i#BN?<!N7AOv{`!Yv)<#m
zl{{2}6F;uZ14Wd;xOs9ZLDA(2t7~Iq`!TDeY2Odl2cpLq;F+5ECqnFgBS^h>Duzlt
zXtUWw&C(_R$?sNZ_D!w9Ei<bqx7Jkx>D4}=)iJ3K5}=L90^Ss|02{t@&2!P=2?*eR
zD~_{e^0WP+ZMjHn?1ih9<Pz3I^kGID?YrSIZFz>Wm62J7CLFaw1v62iQv(?k&l{qT
zamVoq&6`O-!<&}3`9LR(Y0<P@--}!ro-_z{GTu`+O@;I*<zmq>cRQN{a`Zo}jGq4i
zs$(QqWo~^3I4f|P$Q(lcGrH5ffwx2gniq(_|5d*E;aXORkc(eL2J`b4rx&wtA*h$q
zbOz*O(YX;Pp*H)QC-sGjR$B_~Zd_JX4(eoo;04jc2dzwLa|}vowH{&!A#HR{>Fd>8
z<d9d$5l4z|nFw<j9gAzLRhtDua(+Kt3{K~+YwO8*H^}*cyWMwg`<d$NunfZ=Z<VgO
zK>T6KD9dM^aW33u@W;uR*-3=~%;Tzy9sepM(BJ!rDRtWM=#Tt$e2v}RViZL*Aw{<n
zZM67N615oObuU!N^oMF(5;T4MN|f72LDX7gr`6hO@$UKy%X-Qf&|!ps^Ge<5VuKoL
z@r4t(hvzK?49CS?{Zt)3;a<3ud6!nAQOrQ!8T)!0OWmdja~pfnejY8_{#|fwwW!sw
zEZy}?LrG_DB_VOG1Qfjk1duk1xN`h1HIh7iowI8;$6N)CIA++6nL9g1tO);sPQkWE
z1{FopuBS!cfL_=izvGtE$HfPHqkS@cuT_A2GK??k<(rM5<a8h)oq6zcAJ^8!=k;eO
zc3w9bvX2UZKr?Ak;u6B%ejc}?iuutsY9)GQ3TiDYY9f3%SL3~&4RFc(XxAVz@&Eux
zk`wCqA2c78Mx-kMK_v<6sUlW}owa%RbyIN<5AB#$uxnZAi`Id-!jf?lJIfFbZ0q*8
z6{d_rufI~mwmJr9iQZY)u%u+{wNe(km4zmQzJk>xH_O@q8@zAbdgx7se=*Jjhr51Q
zC>L=I+1?T_&!~NSZ~N%PErnm@I(PL$4M31i4iPG4vMJ*acUm|hDFea<g=O)RowhaK
z#*=-0y^%l*^INH4>=U8*x!l7GqN#3``5O~&t;O3Pw1PVB<OvX+<sBs4`w}bn&Z`q#
zIzf(5;zjVnwwY2A=q<4WvXdd@D6k&}a&L101)3_c%@~kGVbQ#dBvFdG#ogXvq|l*4
z?_hSRw~kYO4~-B<^zwla1~Am;@Ae2IZ8$<^q9!PlXMvD>4|rkQ>Ur<(dd#c(FOGA5
zdw&!Z8+}igX2^xa_c^NOt2}9Dn~ssNq*ZC_$9+P^;Uvt}+CdWw!&%`r2gj43K15g!
zh={p!CeZYO9-#gjA^1^-Zgc;MvW*<+ntVt*IYSYDFYOXk=G?si)&3&+he1ADJv~01
z5;bn7Xs>Vq;J)fm+V!#?-{cHKs9M&ouk`md6QHTa{~o}f|B}Zws?^lhW~KXIPRSdc
zJ6oG!X_fcUNG(-7;Ar_$zoIJjvhdN)=LhvUzOE7yU=N?>sPQua2e^+$EnX@hE&(U>
zJEEQ{zQawv$sYL?Fod_Wc?F>LmF<OPZ$N)mnDvzPw3%g$;=&0srNcg~^j1=D7-8=G
zqIC0cPP<d{*8vX+&%FE}Nj`c|Q@=PxtcksWvSuCJ6ZG~xjxVvJRJB%vOPvRoS|KI2
z@r=ZUw{BKgZu%yS+|4!1=DkA0>~8;(=L+nP!)2mM9fzfsR{}bGq1r~cbDQ|cJrtCZ
z=dT0@)O_?ga?NKK1K=XSWWRF6!{6~}nm*h3T4?~B+r|9#{TjPu5X|KT<?&}axIrMu
zyhcKl9Uw_n7kOC#rrdKv;eOIc)cA2Yfa?B*wQTUZ<ZEnM#B9R@M^_XVS3eU25^7ww
z3PLF{&-@aCh>M-*9<J>mddI*o8OzS&YV<+2KvspWBhq#04qy2gZPnFN9IdTjnq{FJ
zH><s1t=aLiTXF;4Imq5V^hQlj#eMeFMyHR1WZYBe71`m`mdh3Dk687anrxcijt*oc
zZ97Ww&(rmMTr^NKo*lgLX683?#b7rd$5WlpXV7~dzs;AE7xG;3D|}}@t00szqrh5!
zo@e3%kYqXoP9fxhvnGlsXL15;e|jR-d|t)0mD?4+VZ_Z4y1DPJVwRDUoX(_tWpx(h
zhm)h-<@uIwwQRm`l8dK^a&Bk~ddj<Wm+{Dz<gx)7eSR=2Ws(PA^-cIY!9Wog+1d{h
zDr&)f;+~P?$A(xK&Q$H*#@Q=Qk^JrbO5#@^72_c0o5!xubk562i4u**>wKq7MLNh=
zqhe%`0rS2lsLG^;sBv7b7<jm6ZmWvA={wZJ)GF5<)t<?74DJcEFK37d6;6G7bze?{
zZ9b)j{B{&R#FJZ7HV<U^Wvj+9(u?f+1$n>Tzs`oYh*w>a7JgthK+xas4RcvrM9~!n
z%NoaiEvonx?jx4EWv$lT`NPt7^!{vOW2>g#H%=?Q)#5)+CaFiUe#@*+YmtIIJWYGL
zTM3$)PJH=e7G8I_4$RE`7h89Fr=bo60%>J+nbO*x`W|3@xI1wN`HuC%&DVCf#>?o9
zV?pfO)75xT-*j-bW!Y}{syRaG82zS!2oP};r*R1E(cj7k_63$b;RtrIIJ}(p+5~HR
zC!-+jk+f;b^xWknByP6#M*Wj#xB(ym+|1GiuP7VP6v%O<JxbyJhiL(>YPiVpTXY%Z
zKd}b)%co45oh@5I3f46{{mLi*WD7JTlJZArNzxQFBj#pOz%WntWpb_yNmR3QC7klq
z==$rZc*o6W)Y%~YjjP71?}duHVa`uMX)l=HP6j2-tV>?|phr}HqXmN1&A)ad4f39e
zi&~ck@qhQa6KZ^0j45V)PpgX?XtP6a(INX?H$680WS0J+1V=}0S#I~Hf&Cy@qUB48
zUnS+R5wJI6Q%c_w-@GMxQ2hEE-#?IeeTWR;rr%^pxqJJZ%>5`5#jO;%F3R!i;<C~2
zePnf|<RmTQgk&Rf>o+w{13r3yvQJ8X1C&v>nA`R224&QAGs#77PVwK$sDl99`J;R=
z+JS@&d$oqW8bdw}vGVT7?|)C9h@iKI>c$)+0IBF?BScZ7*OCP!>j6(Y0puPL&RiC(
zUc}9?-Kr0t(-oi?+!V`*kr&AqcNJyT5_`KrDg{N2z~ORJ_KAmXsRPXfH`4WrB`*<v
zAuBYN_;c&e_CWOBptl1Tev_a`G9j<WcCfnW_!oB6W12k*rz#*XDQU*#DF><h<FaIA
zPC9pceakc~IL|#KJdmzu@JlgvU=crY7sPQF%jUTljDg1PRi;<dD0swYwt3Rr+sqFt
z4U$CTrkvy!ux(*lYLOUF6Pnht==iIdEXMy=Gue{P^%CC~C?L4|XZ`Euk#mw)W?*{_
z2h#!<geSaL%oFoUCirZ0(zZU?n}<Ayz80dlrv=XXYLtd^dUJ_3Fs%o`K@b3@wdTK+
z{QSIn$`E3A`f8{syS&=Qa4QYP@c}F1QFgqZGy0vRj{P%S7W{nDG78XZiY|I5(bF^m
z03mISc&DHw*J2Fz{cW0d?1t~#sow*MMf+o!W%3Wl$fZ$kb(zVzz!i11J&#*%Kjk8^
zMz!yQk5D24gk0%yyPF;|6<>ShPfQ9(j6MO0(Qo!He{Z970>@B^7sqy7^gWN^Ptd6R
z3j0=P%iy>0ojdE#ygF<+s85rfYQF$PJO+d)8PIqA09{}XngTDXj~rvjc+|gf9$l(-
zwogmT7dv*#Qs44{C_GN2p#OnuUWrltL(nGqFls7&=2K*$ONARHWLMlX^X!Ell@gKG
z(FR-I;H<gQ-co7yjUde-o416=lg)-+HY}VZT+!R!FK>}7$thg;n(F^j)F~-+n8>b=
zRk>sAKRGWX;nVg7+xi~r*sv%9mVH10EW2|y>T^@T{T=jW7j~N8R-XCCx~Vsqrm{zt
zZ>9H8r*dI!SE}3Ywq*_OU3q$gSlV=PmNinI7zpi;eWW+q1sbY;ikCI&9(i>coS2Hv
z@=V_saUhnw6aiyo6<?~0#l~nOIiANdms~rJGHH8g9@Jy2O|_l6`0Pf3j^e$9B_NMM
zk=Tp4sDZ#m%QR<vd^kR)3igqVSDk1B>wch!`>mozbGPk-yix~0T3)SNiE)-(X;loU
zCpd=_Ne5lRh?Q(5%2ijQUNjuyreu&dqNF;!g-4QIxbg=@CX_Z$(k7M4`7hsUsD=?R
z%L+w10o?Rkow5GjNo{jnhE<tZqxU7nEYabVLElS#i%&9K?-57H*}W%I+D9qRriyT#
z(QE7f4{6^S*5tOWt1MKIWko<l!6i*aK)N6h8z3NHK{`Z1dXwH0QBjem^d6<Elu$z_
zHb83VAw&pD2|Y?e3naNSuCve1!hP>O&vSkeBPo1k&N1Hc7UQZZvcjUUC(aZ`YqTel
zYbY&UwoDLQZ6p;Fx$(?mMPPlI=yf-92a4-%nFPyXva#QU;Ij5L&%oC8;|rSd2<h7`
z{^dVzOvU!WGokQ@kCxn<;?<Xj)fKLO!p!q#kZw^wKY@7`Bj&T|H#pIc7PN-mu)1g&
zv!y<$kbX>6LWWt&7Jd8q$ZwZh|Bd8Tp133SZ-m&#u#eAvxHqF5CC26d$pO1fk9R)^
zmEa5<J%6P!9dHaTwVyq5&fDrjdiOe0Ebx0ZP{gq*Zf9Gm4I86j1-~R^Q63auZP5ti
zG^6z+Rr%z96kHGs$(rG**f<^H>4LBb$LPK$Cn%`6QyZzH4JR5^n*@I235)FKx-c1P
zVT75a5;gwXNGCYi$AgOj(vL~Z>!N}vfy0Yv%1n*TuqjR5928YvBPGxQ3}RsB#!)U8
zE$@>o($BR4cI%IW86Og*FX@C5o_Qn`4FM}ru24Useb1jLb1Hrb@U?+R=GXkQO1@$8
zGVlP_S6MZGsx{0i4B5s8M?A2KP%gj0__N_D;Vn7)SVJl!=7?H#Z$ZlaO)tK+fz7T;
znAL)@*Y`DI@p8d>hBNK5EANot{TD2P!L<nX?@5;s=VWFP4~*Xdri2#-<%~Xms|#+<
zhY7?i1+AR3AMZlNQHjOx>&P2YYP>^@x{>}YR|lT^hjF+Iq|M+<25Lb2<N~l@!`*}<
z;%abqZY#9gFcqw6n&~k8cAg3ZkzI87yleY9+|F7bn61N6L7RujZJn;D5$F82V>Si8
zOqL1eT1RW7%K~PFIbMh!0(yPnH)RGaADDFD$GwY21XT~KJ;OT0<VMvba4H(qbHe%4
zVk9EOs(jwe-O*2Z86t(TU1@dlasei}lQkXcrmoSQGAlKzNjUL>&Qf5kn?#h|xL2vn
zlpiix*dYp0N6po|AR)mx`{5KD3uDPV_02NHLQY(d08rzJb<kO{4$b2RwVuLd5Mce=
z8)OR<@%?a-k`}QIFKX!uX^c=6;0=p0ULkrDG8@W(!hIHUaL6W(rc5&t&|6hqSql0S
zDCak5^9i<aX5I^68qQ;8T82pl05}X3eev|)4ED82c1rmitQ<2q8Dd2jhC&C4%t(X6
z?4FgY_93PSEs$r>T8@QRB*v`3`&SwZF`x#A<%AcN2}{xo0U2!z+bxQ-A5lo$H4Fc#
zXhtjCgOtIQXYOBaVuP6;=)Gd3JMiU*j7pf8#$ml;6#<33A}xQ7-5C)oD$q*5hAwOf
z6afY~fqC->Shn&o_Ii$+3tAwTCZi<y^XFV@{NTN=6~axUeE)u=_tIlNSZ~j}G*elv
zK8m)ZYcm)$00{!Kb^)H56@V@P@x0HYy~5ggak2TBUe)WlbhR_!`wAj<wQjr5>2iIu
zge3z+!%<Kl6sb@VSuq$;h6c(P6<k1ARZf?>n#T!N!Bp}(yD5WA!XhY^e!V+@=&zOu
zv0Oa@2i~~z5A1ZjAtGH+Q>afg{Lbm9Ur^x&%C$^b;Y<F(+JREg(|ql_CrT#ExIRgY
ziT))M5z1~>CNTo%gnBK}_A4C!ZWOnLChhNL@4R(L?~?OqW=oj@(N?hqn|F(#I%dpM
zdb*kGsN9w4JfWJt{3aW~Z2FbQMP7dAN#ge-;ra*xz?^4`S0A&h|J-Mw&=v8QPj6Y`
zvGuoTt3hG?se;3D?T$(Ksj_TZAhg&%D4*)oS15_7iVpbPH3^b~>b@!_9a~scvfv&@
z7MLyfK(Y44&>~tb=BpjzsjJku3f5U<uUe9QF}aRhKq;o*3te!^n%~EAF2&eYL+^W0
zwY9p=ivf4N1fqY!B+x7{sR_4J1HjEZ_~GP6^fj_Fky7Vy8sVjnQDNKwRCba?mz<DX
zzP(3RONV1LoYpXYHHg>-2pcUi!>rAe$QTmmpxx50LUolIPG1$c&;@BWB4x(_F;pR9
z|G?vY0H^Y16o}+B$^5%u{;2|*_Zc?&mk+-+sI!zS=yv;&^u;3KdE>^sw~Ul<Tp=$^
z?CfBc_Rm~{uK73{o`!~B&S<M-S5f*`z)XYKYw7*_dSE$^+c`G(ak)HW(Ukn-p*YZ&
z__7J!W?2Rn4K6^>w?T_$&4Uf|n1!`O6eaaYh88HSuf}QFVj>;0mpA5_RW+n{=>`sH
z?MQcmP=4Gj@}(aV9P?`4r1g-Sr?JRC31EN>%rN}@z=7td%WqIa<9{pyI2)prXR1{`
zpP9M3>Bfg$|LOw+ztO`3BnexnYi?S1+;5^Lt1w35aeeC(sLd7JO-93-ZAj!b`;f_T
zm-PiemHzJpltLtK=Id1Oyd*jVMNl)qmgmxgmg5SM61@Np*&@LcU`stYqgT;T)iZ&e
z;!GHKxt-_T)lSrdxdKP#j#eY9T(3sWL4);58V_vQ1t>pfR(s<&qKo=-y}Rt2T9FIy
z)NijUGPpA^4PAqAMa9*<%81*?(WnCNsqrBvf)24hcyt2myRzmZZNnt&e@bXzMdDsO
z5PDyg+MT5AUI1(wN!ft?kw_&qA+arQf(MP^k&PVz=JM-SsT7|o+Tt>4SwRkg|0+}k
zer=_WI1`BKha+LVb92vCfkDPUaTyjmjK4XsBWqa5C4Oc($t;jd@p0g0wxt$#<(QsR
zUzUK-qa+p3RLe%{oO_5VxgR_E7@^_1U}#xL^;mb98~4)d&Z)H1qoh_4%sd*KLpm`V
z@Gu|q5#P!Wjl=Xx5{_g-bpDDZ&iKz78%_O`+SHs$+kv`q4=4TaX5zgr=cdFhIY82V
z5o7zoA>(@gPS}-u8775#V5`dP!@NuBIR(0M1Q1sONkMGPDpOK$HAAR?WN`{CGXI2Q
zeAx%9_-65(R@V!39eieEZQh|XpKR8+bYcSoA6@#X{DfM-Ze*o~LASdDbLh8)RS7={
zwmJy2$_gt|K!}qX^(#<j1t75*FsqhFbEF1|g@T(vw6tZlP22w^&MRNd<Cc}9xdP77
zl-8a`?nfHR7D$Rs**@uV2MK^j8ELb1MZ&IA-zmIkNyPYwI<X4Sq@(khjC%7zRB$vy
zZ}aOQPO;oGFk&VV^V6-IY+h=v-U4ovvoVrJ;1Odd%x!#`c{kX%39_6zEBy=H+YH$5
zEtk9NZJRF;RPf5+rT84x1?cS@uwyJ=rhawjPF>-dN$JU#NEw&uq*&Q(I6vvC@v{sf
z3BGmy4H}g`ejUzuv3C=B=8dbK{sa^*ZtiNG(LAtg-3-K-1X)3B!WMmB0a>r4e`$9+
z0UP6JqtH<4hhVY~2R|;W5N-m1r13=HZyTP~_3{C8=jEu(mR-I{kj8~u;suq_a3>Jo
zr9Es;O%YzdUEomG7^Z_biWMgUGVn=YMd$7{HmZ=v<U^bm0!~g_WTKBR(88-mw{_>8
z(#<I8uR8G#!_BXE_bBls(e<snti-LgfiS_hj?H$zrloJy{b@iO*tyYC(#X^X3cV(S
zs8UIwzuLkOlwe!-s8@8<+YtYjTI{eiRWEjbbWl4(t)?&!Oa`=QCKAF6?>;fGqc~e7
zq=Ysqc&^_(>NR`{^2SQ-DXidjd(^i5R}P+V#EBdq9<E=##TL%a>ALge3B8%Z<>z7=
z8Oj06=||sAW`|^xHS<e<*z>5Mb%;?*%c7ZE@vmyh(AX;*>p!L$UcR&Ii<@0N_(j^{
zwEzab)_*W4kx<^EEa2yxPJ#IC9L*qjQLdJ#wS=GizbyPN2akV1h3^n&x!2r9$B=`R
zK~f708#K2vs?znp|7mX`R9>Zyq7|<WE;RV0|L=dI+X9a^Ec?&<HB`BgipKx<KSLf*
z{!ZK%9TS)tb-GT?il7BO@(yyJ*@w2Ll;;=7(XAqi|DM_uC5@^VAmoN=`UF^TG32~{
z%W5LMBhG%R)vuzKr=8oy-0%kr=&)(ZX5~+#?CE9XPw1fih<?$dz9c{Uwhh8yQ~7B5
zW~N#fuDX0YhfyQ<V{tR{wbsK0ex1)O%{uGDJ5RT9qK=)dX1VX_J41`mwX2Pdf`oM=
zbJE-##ho2BL}_ds4RnWRRK<|y$~RiZJ5Rbd5Js`s@oLm?AQYMp7cObaTU}lr6wuS+
z)tZ!i(tt#^&|E8*8yoc*ns*}fz#A2<2F<1u?nLZ<_2<7UhaYpCYfm8b(dQwwv~ozZ
ziWv1MQPF(ErVyWn@o+1zgf!Pm{%@W_;RajacGt{H1*K{DEJAum^C108n`;m8TorCu
z$Yz8W&@|GDXflKUd<uW8u!YibuF9GwI33efw9nkxrecy4)$<s~>Ncy(G2Kj^o>Cd*
zaBu1nf|=KrBxl3s1zefRH={Un4?+*d0cf!q=f_ny)Mps}VssToi(haCkq5zrjmlJG
zIjtA&JbJ*V7Z`gnZI1Twn*@Z$G~2mk-&Iig9B@friWS8*IFEq&TlN$!+kX=p)#m{T
zK~UVuU(RF|gLefhj`KH(EaF-Z>)61Kz22l``G+;F&URtEF2q`u#*l){Ip<U+x`Y&(
zFx>d{=$63eFD#qv9@2>ttSX8Tnj=UGHDK&lG#Kdf2B4aeH2BTu?(qW#z4+VbrJf(w
zU#(K;8|*8|8Vnd|IpBGS^?DaArrB4+d}4pQ)<(2PkS&$OJ!Hd6+i2*a97t<GeNjwY
z{XQAP`X6uzBYSnFLGhpqVO&F=WffWAbhz=#p1c0sGusV{n4z!bAV#iwUXhAa4$n~h
zDI+Xb4v~$pZ8_hyOc;ieY?|%Vr<RjasA^QT2xPn4I1Ne6R5+<_ch8P6NepTD7}x>Z
zmrh(>rzjr~l9QFIteIC_D^gA&oYQ^idenS{^_pa36H>TgRLFY-(V89h<fen`ho0bE
z(OoXhz39@12tF6Po8EFO11+a{_2lwftm2yA1;9zu^^Ei*L75>Ho|-72X(AKfFhp*z
zl&Oi6aByfS(R}SUE6pg5;3*MPf$QcK3FyVD$qO~>?mwN@uQp7&3(&asymZu+kjx*$
z8p@&`i>1Vs7i-%j1lRv6$EP<<!iZ+tB^l#dtm^Qz1NT*~emZR{`lO=Cd^PsyX}FJ{
z`L*Ls4c(CON6-&1phbwWBPNkfSNhmG)6aMcc<2e7oRId7>e_Mg<^`gLEOwU3C3X_~
z0PIRKSXP8Z?e|>VQC&*J28>T--Mk<797WnNP^=ilSvcEOD7YWnrN=w0#*HOd#@Q1X
z%nP1Cye>0c|6(?!8n&W}c4i3EGMvYO_1n*EC6@EMhjYz2!`8tpavH(syp*^@Cmb&!
zp)WxDR#gOARAr<?Edq@vK2{!y6^l7jI6!kw6}GYk8?5^<9y^nb;wdm#C$3f6SeT#q
zu!Vj4!<x-D00(Q&8*$@9xWA7eC`}%zz~9b7*WeV@oBlutT;g+<Z%wCIR2-8NC(hJD
z4yxE^Vye5AYwOCozz8;{wUMQAb-VbU>$+(Y{{8SJ%f_MNvYT!zt+naTeX`;AdXorI
zh}$y5spW<33&{?$vNP?Zyw-9Odd3OXYLf)Az1T9e#6tKplsZ+j5qSl3I;tOjH^#R;
zFeTt3siHGnXtGgW7j(0;H2P(wbaT7V-02n*Dxcs>?TKZ-wFEx}IuE$u>^k56YrKgz
zC#rrxLFyb?L8&nIO`hwJ2%}hK{h6V}w6$ets08MZZs>`#O6EbR>2p)Z);d#;J5V#^
zmuHm~wED~e;c~iu+Irr;C2^&A=_fn>wg_$?%X#YLa45Z*2{Fe@BA!U0@rL>E@w-wl
zZ&1RGU4q16fyi=2P&mg-joe6+rxwEOW&%j(Fwt_2&?GcjeMD=}mI^^~%Wb%Bu6X;6
zVz{?9?B)l%_0YUd!-3}YlLHaTE!9!<n6H7F*2{~Vg-QIpdRf|-52Kp2jEp!BDDor#
zXh!Go%WqmXX42WsRUz>#*D0N2yJYw9K@Em&S=ss%3s<4mxoRV+hI#3Rqg8IsA>SJZ
zWhi{1=r&~fh$+(HW==x~y&!dIBXB^$HW5&=LE8EHP(UNHfh;Mq)upiN#4DjTsql(w
z6IbYVPwT=a$2HfB_)W^Hb(N@?#=HBD>7xk*Rh-fw0N(=}qpI^d`zV7iyOj}^a_V&e
zONokM$15Dgaj_g#J&t&BiwX|uQ{}RSWhYl3_D3AXx?*8eiG-by114S$DsTOF3dl>+
zsV(V8A>z9t*6*w`b!REWT_;p=pMlC*<U%cr<I~rK2XQ9^p{9)qW~PO8xws+mM4pQb
zh5Rp{Bww1IL32U84%-r5ItF7(H>Faalav$f8~zJ@7`0cmJA?UtG={(ZD1(X_6<)}2
z7XIKHE!4nrY~8NDQoU=qbFlY<ZBH+@>WKkE^KHVIco9VvMUngXR7_yq=u#ZUX{ivV
z=@96Quy4_3D4sV94=BqNH*c$J21Uq`==mN0Kug-PE}<uHDN0IOB>JdP57iX+5mY<c
zBQ5+kP$5FF@3Mgsk%Xr+b<VQ`BzQ&UD$saqJPB~4ZIo*oPI!$h;8&bs!-O|cgx}2i
z(;v9MSJ<EA8U&>Tjr9=ABx%@d!byJLh^xwdnq5+Oz;JYw{8D>#nLk7d-&}2+G@fjM
z8W;j!LpfTRYJtjy+lYz1jNsQqw4>mm2m)~+ksUv91t-LE_S!i=hdrRr7Iik7tU^Bz
zES41o4_gOAV@jp)Q(Y8B-)3;My~&($%cZ!m>Xa*j9`qWQIr5?LpiG%TTfE;iX?5-N
zoYycSsZkIU-i{Kin;ovlm@r?RXw^~vMfaMuTA3$qWpPur@c{VnE#@k=?Dfe@+oESn
zJ8jLWd~8~()@ki1kk&!+s0b0y``J+I;@qXRJTwLVa;{J$2$FEw7^t@1FjA(<uVe?h
zc!5B<jW>dL=3n~P{m;Uv8kh5r5HTK=pnfg>Ck2va6&!sIO2sObk$Thr*aw6baK5-x
zmwIx81+K5=*$!X$JoxC=NaLuArY{oQ*)AQkmTOI`%H=LyD0M|JQyjeFUUUH0ptxZ(
zFVQX-Q`C}G3)qQ~^Hu^hAzbpn`5zIapR=;)555XFR#>4QPd1OzF(JE^S5(D;4I%qQ
zo3F+w_iP5xg4Z6lT=e0K!B)L)T^-0l+M))vkq0(xZOkd?tfMU-D}zFI1}eS-vEOxw
zqL_Z6PcU5Lf|3xV5h>9wGF1G}rQb^bxD=awvPsP35HGFts14P#utoiU^RsNW97T$3
zEkzx%IkURlfSQi6r6L0sOT+0Vjprdzq}zD+?-QlN&rP@fFa0@USc~aM@y}Eb7N3J3
zitJZQ8am%jMg2GS6S{i5oBn(R{=tF9_fsK?R~H`Vcx~R%Wxmw4R1&z-4h;W)9;Mje
zJ<xJ_;Dw0%=FgeU&4PGdymcz(Mp5Vg`%xwqr<h2JEN|*15Q}9>^Gi!xP)(@hDKA6k
z^8fcE`$*9bicchhSe$75WkO>pVaG-cs|1`0-h2WyQBSoy)K4U3TA`0z*%MG7d!=p`
z8lx+ZAh4KP!u&A#SG^6j>!<$7@S)H71-XJ{;&=65wS%W@k)&?>!%Hw}Nu75^_09eb
zFHQW?&(OXZQ;sJ@61pqwCA>W66&&jUk_{A_HOK;CA6U)@FITXQZ&@I<UY)sNrqaPF
zBRN_Ta3Eh`ZQ85t8T~su#8JZx7|K}NxiAN_sc~|YdOzko*`DdsOr(J~%KR&c1xKqs
z5E{GpRJfcZRz2x)>du+RX4)D4L--J#KWP~+9y4+F>>Ix3h$W}XePTCy<-*z&<)$d*
z*w2?d7Ux!!$v6$7SZJ4CmdJRUy>~jkbbcrg@$%giC+ej)j<xvW;-GN;HF75jnHK+{
z+A8x7d0Tr{Zbjve$W;xv*HUcWSyzkEIB2#k>fW*0lTu16>D*J^X)$_ykhHj8b*QsI
zh8(6SERQ%>>;*VWeL4?;0*(6LSKfAxuP+|(;md}DA4$zAu8$O#DOR=|uIht~o0h%=
zVYCvP9d{pBx^*7a<8AY3Y_m~U-{lk|;|4rF4&lDosOS373!+IfGt=q$o_jk!t5V4#
z@JNrkahtT2>6Nn8UZv)08ggZIqj5BLE4JZ(Dzy6K$v?ska$0ZA&Q+`DNiH5mX6Ejr
zRc?EPI0OtP$_HhF8o(Y9Fp{30J=>BEtWG{3O(xnDd&RWXwSD>>4SM*1u~M8H&X-P)
z;$1gPL^mbetvZa}L;Zw2<DI6svTQRDS)x|}&aOJLHP5_YDTwNUH08}mqpWuI;mDhA
zG`)bwpm#iAh-1Y__Hca8-=>=0@;TLf_<NTiHSG%Uoo)W;ZD%I$IJ}_xsuetb9zuZ|
zK;{}c3q#z~?Fr(v4X+KW%?&?zp8nRu&C=D^t8G8%;x4^?8_RPn57Ym8@6O~xLjS#v
zhk~gIHe{I3qD{KA>0)gsl<Ve5NERk!-YbJ1_PC}V<knf|0JX=T<$lNXcn?l3EvBk&
z(2ELiycuNNq;pCIaoy!uZ6_Tu`*lO$uJ&)48Jh2vMm)Cl5YxP>>f8iuj&U=a?m3jq
zpbWDQulZj;h5u6^hBU4Lg@|EYZO(vA@5^V<==<Og&m~WLrQ<ECqDrH~fum99?@uk<
zR*Y`7dDaL^FUB{6Vw>${xN=I=|LD8nIe=zBYpYYktF&o<;f278Bk4A%!jR~QkP+W|
zBbtnl>(b;tWx<9NM9g?#h0`rdd}1YLp%jUaF2qmA<mHAQNK=>zsS_+3E^Lx?MvPlS
ztB#pg^)N`Y7DV5E&4;dsOjlSOt$rM2vpU||T(&bCZRZ>ZCrRT=4bBt@(KNQh1`S@z
z&=qFSCi>iyH(lFErv&#O2`Md9TnoRH)|b1IiuHH1HqHK~n&5Rp%DRVq!<o`lHasT2
zJL<+;O9r@&w~Q$B*z2dee^}!_jV#p^k;-}MCbQwf``16038RAD;-VHx=aVH7csoi=
zOX}@tP*;X*f`8t#2|9D$uAQhzWrV<fTA<60`W$W->`TPBmX)d{9xI>p5}+}7(f(FQ
zskp6G1It?IFTo}<G&<`|w)^i=^nK9&p`ard>7cqPnxQOD$GCzT?p#v#ZHq`B++b;l
z8r23ALo!{H4m^A|mM9+=XbkgrT_e4e&6=7_l3b7BvDpw%@(7GuL>qa4a?HGCWqF$5
z?A39y&d*|<`D#If|JjTFC!gJ**wRrEWb=~wk#jQOll~_!`X7E_o<zp;>EA{wYhuGj
ze~?L->O#f!O>;$*)2b#s?g7|1j=U}vb?!0J!VAPzx(H}#GBp2yIrH{v-s?3ZCKMW&
zA5)HQO-+oNJd9K?)rTKAWqyyh{bkY#8=CY|!MP`gGw%pIOvqqa;p-FM2b#Xx@rZ9v
zPxI3F%`VcI+^S_uBCRN7&Zf3%rJ&m93{5CyQysQ&6mzm@U3Ka5t(a?r9mA&9`s!s)
z*f-5HJh{utc!B)Q;pdjCrz8U>RX<^lIHG38sRuc?id2)hyURXC6q}$7{xHP(;K~ed
z>7J^KqC4w{?cXfiM?UPDz1K!L`m!z7w#WOt1Za8El1_ZX`Zv5ggwZ?Wl{w6R5eT}y
zq9z7J4{l9EQu=GGDTP<~v~xLTa=s9jFn*$|l)^N`DUH@Nm&e84fo5=5$>pb=MrR`i
zonbTQ$0t;i$d9_x^wGTr5%slK9&Q~4H?dcA|A6e7%+ABMVZ)wlA>UOA{4BfXyqxHg
z4Zr&b`s0V0=IU1onVN2c<uOcOc8V_zK=<E~t5IF+^JP185^rE@kO^yj9CYDb5K%3b
zfS^$GgD1J%^DoJNj*b6O6v*$bTU_bCap)rBqc-vfcbF^fjqT>3{H%?K{uS$u^m8*J
zP8~DN(>Ui;1w}%ZmkfMG^))#!iPTAc*(UZuF;;5h$-ns${-E@DaB|n`!zOLc01)nc
zCP}cUv$TxjE`Orj#OSeN`EApMVftzG(qc2Zk-N_i+h0In_yg|#_6#LOIBj_SbcWJL
z<0Qa_-qJ1Y;qbqk_1fV>9=yT8_2*8sbb%!k=E+B7y--Frz*d|4=S)0?r=jPj6O<bu
z`2xAv_-@DXX#G{6z!eqjNvJ76ctrJ9cCf(4o&e#QLSjCDi@$Q2-WKn**UX{Cb9X*E
z`Gd3+IpR5>7@KLLv$PLr#$g<n<`rBkQdfVC^y)!}|4B{~7;SDTS7IOVI_W%BmAX1<
zAo=#LV?o$qY2(5cC2jax5rw-+rX&)8aDBzvfB0+;6f>thMP#JBe_n<-)2W1Y&Pd}3
zo?THD={9_;E@@fu0AI^#6-?nPO?+Uu1D5pywH|%p<+g3=e%e<q8ivr~S1uwQWFHND
zwN#Why)p01{HAx+3LbRHIeH7SE8Wd}nw=rF#QwW=k9r$iNgCpq*n1_QG&a~)!j~jH
z;Y7=U+wDYW#geA&hg1PW^(HotYO&xaAFz_Dc&*R;yym}1uL<tjPSDEBti+LRp~eTl
zL}28%<qoPZJv7WdWl+gu4sSoTX<Hwu)FdJV5Xm*Q0>nq>2?@}(ojB{RA8kmcH}(YF
zl_RM>#WeX*=kZBnGrR{~@(+~#!BSe7AN;~ufx0JQW7t6bnp10&BtCL8Iix20k*zaM
zimGJf-YU&?uZt_yg^VGeHJ`f2J7$Z`QT7~F+YRP^1?!TzSHJRgdM&thHydBs^-q2|
zQsf0)7JWuhVJ{<BUAfY~-8JwnJ^k=*nWa8R`uq1gVHY9Wdw!fqy{E7FGAcFh=iy)Y
zk)V&#C87rdn;7tL!7Our4+|&(_(BQ>_AR}g0XiHtQY5XAHXW#t<@$g)p<M~?+YRsJ
zgqeO@m}`+^j}cQdPKmQYI{xkK7v0qY_hqJRavA(CHF)(kGw5|9Z#M3AEh_zD0cSQP
z?wd~LfNy2uM4I(9lsE9l`Q8|LIh%4{W$_cCVlltY7t{Ft0x@`{V|{+))>r`KxA$Gt
zyf7D^dae9z3QFbXjvQ|d(;gneWyHLR?`4J{@8-MYspMTDvnA<qI}yquO$tRngKuzV
zi2n#)xwm}#Pb`$!?3l*Bm05h7JzkmIxgPuI>fQ}`DOzH(5>L{(gc)=RNb7OGdt&4Y
z?TvaM(1y{oTHm2Uy*&ShWqN8Lsal<6wFS#D@ejV+ulR2N_kDU_h;1_SE0i|hT=}&m
z>1e81xdhdVkLsMUQL+PpCu9)*$lf0P@R**Nm&9iW&kuLi?THg*pIUc#OO17S&H0O>
zxrpmdXg*_ro=1e1ooBhOum`{F<6XB?rHOu3nxke{j~sbD_uTK=O5N+7HIlV6`yexS
z2cP6|I~t6hg{H*qiKuW8v@dr2I|Cr|%`5VWs8DjSZjkZ1|HFmTvGcoFE(2cn`2BrH
zL>Ung3A1GEX0Y%)d+JV-I>r6WXN?XZKTP?8HihIWuJGt7(Im+{dV4`n^j`tKG-Wx@
zOp?Tp;Pwto7&+UPHR@#l+VcyR{^ETHFV{NnOM6#3!kKgQ5~KcpOmJe8zAXp(9pcs7
z*BSszjeDQh(fS%hO%MU%XVAUQcUpCz!}yB9w`|tH%Mp%q{PA8+f0a&sH>wL!?lJmf
z2GLdcIxNdnLw%*;7+U+g65)%*)0G*VQx$2UOFQi22J*XMZ`-vjhZV}Lx`vOz&gn3N
zOm+g8BVqD~ItVi=GB@$6zikS8LO0)Nsd&4q9oe`YGg)HN{@Se^07v@+J};eLDi_ma
zYDZ(7(X*wS(Zt8A?zbz>6s7?1bES@cdd`C~I9>`}Hqj)8y-1<(!kpEoJW54B_{v=O
zWdU=5S%j6#veD(fkDp)zmMafMs9)_06sbyfF9+(DxF7MSDGZq(697o?+@-+#>-k<c
zH~a^tN>`1>FA^8NK7Yzjqo|^T;4=080Ji++JA*-hW9c`a%P6y8|LrHz(%q5>G6l%C
z{;DxWs`2q3=5zVh{BDJP-pA!rj$eIB&EKO}tl(Jmg?h;z`f)iICfm=~5Nw|wO`F-s
zVPu5~0lR9lfQSd|k?VdFU)E~;2%FU{DK1je%VzGj8k0GzJCCPO_hY)QcQu*i!<Dhy
zF+o4lM1F=joAv8_8Z=km>@}IRoQvk{BzfrjGc<Uo_A5{$Vb+@yOIZW<%`>a^0xdf5
zVs34|#+`PKXTCqbRMXhOz11BZDKfSrsgT!fI521;8?KqiC$6T1{f3oKaNGmC80R2+
zAX!m-S9^fpFW8uA1cp(1@mr|p%lGzeI|p{)A4QGdj<l{+wz8!v2~Jr{sE-2oCgqM2
z(A5U+T~VAR&8J6h1ZwaD4Tcq=SW9j6I#FckJ7*T_)J%!vBs5WQwVSr;30)L1t8B+=
zEm^yTpC%#bKZ1-P<#vC9odaORV2}ITshg?Ib8lTdEd=1bEv$}xY>po5H_Gi1y0Vg@
z-R}$(H|F#<?JJLLo^C|;q0^yjlGIZd#Okvgr@R~Jg~d1iDAwi%9;+OG11r$beDkYY
zvq%=Nyl<`Cyq3!Mih2M=z4y40nqp(Uq&InJ14%KsJ3KAT0V~@F?7v&B`8OZtaow4k
zNLN-i>8h&(L24H4PcUT59hvUOe}`+fkEG9?ioqCmCtbPS#Z?uIb%XCVi`6;}S$PfF
z%pAnrp=Hm=#-zk?6DRKIxi1KULwE`?V*VY7iUHQZ9y`L)n}2<LLGMk-gjUa>LBY(a
zoTYY`z!L@ZzLH4;&Z@7Vi98{5AUQq#P{GYGL|N&<$dp5}Hc&8Z?q7CnPCr^OG2Zas
zBC-L{$sJNxVVw{}Ux3L<7G1HOdnix5qQhGxs@Y#`5sq?k?tM(><k-yYHQ>cO<^$`G
zeP=5t*nrvp#m*=1`E4NLcjA?JM7oK2Gf%85E<bSZX+^#`gK$Lv%AKd0$}5&IBT|4T
z&Gh6`Y(?lz^{>mai-E3PA8L&X?jU9>>pwGog!H1VL*!sjPpD$AJ_H_(fvjIO>cQad
zfQA>1Hxpb&doQ&$&N{d%SVf?YTFtF<WmBFaIF8Qb??1BfTr6q5J-ztm`QCd2?J-El
zf33o~+iB9c`aRw2joUC)zhCs>4}`q0DoxEWlpPb@2g}j&Q$c<_&Oy=UZqsvV;H14%
z_g$9LUDZG-_dLx@>8mwdV7X|yc+`WMVJj}oOkd8e(xIUJ_cO1hRH8WaYzp?^jGxXf
z_2x_Id&$whiQo0_LBB?v&i-cC!_?A1fjURQ^=~IQiKA2NKLUUk+$}v3rsS1%m0#fa
zyyH!*aPj#YS=;f1B%Zmop*(+T#8uo_m558!y*C<QH#QCGXl=goT`s9{GtPPVI?yH_
z&8YI4C5Lwj#986(L>PX|tl)C+=3NJKSosSy4*%DQh0g<j#vPNnk%(8NdY3o$-+rS+
z!X;TvLobHwj__K&L;nTwO1AD1@$&qr;P^Z-=2!Nv^x!%KaOzo#r)_NL=m!VY%yI_i
zV_@Z{-E_0gD3eL1xz!Fly6}tp!$0Q5w@GZCh|;!|?@Ukk)=SHoUiidR{<Qn(KXeUp
z8fO7~*K*mNtlp^ZcrkkU8b{IaP1~9z<R0Zg&Kjo~!sBh|wo4$QSzbL`r~P&JJ<X1E
z7aem);oF9o#^E0$&SrS>=FMZWShuMFUP4KL`t@M|njg3DW$^<$i+I2n->8g<P?R+J
zh&~*Lu<ASO9OM$0Mvv;`&GwrT59t`{zL%9gCGJ)G#&tr$@k!<?ATsx?#|JcuzvtQn
znC+G-N1Cl5%*-UY<p$L;{I<&;ll5t|h~gNQ!vxYP2gdi@P>cqCZw;pP`Rypv^=bII
z{PTsk3exi*<Ea%3UE5ToFBqA<jWlXJ20PZJTOcwd)zsR>GI{|Xa3dzt&~JTCSA1Lv
zr=$y#`(x<*LYpd*hfaH0^~ie#HY+$XBO92h$PO8#0+Kh^?-IY~5BMC)=tKeEJ(L1j
zn)!)?FpbMWsVSnt;EvC|h|!w+0Fo2)Y0>aJ^y>QVjSLTJgJZf>A0ps{WIc1WxAOQl
zmdrv(zRWo-cpiRRgzu(?(pSg!xe~(_?9~Zqn-gZcXf&NCIr+Wx#gnzu3ULBr_4iH@
zlT`M1Y3F6;yQH_2o-tH@piHZPi8-uK?>C5vBX`&*ef~Z-<~Ozea+?9_<(>1KFBVxo
zY8527F6YAnpj>D7NG421aJkk91utHPubFZ|r?onPM87jV(?>7u7eD-$jFzeWGI+Q(
z5>%aiDQ=HWQ3Avj9)H`MA8}CJtxY@rVD`{aW4vbyJ%xYMv)RWS(sUjS+LP8>nS<9B
zW*R6c!{*3=6jg%pf9!A$bkk}S#Rola>S@Q0-MZHJ()bl>xRtYd{wAV7oSu+-_O{rM
zIg0RyKfk_H<)ecwb*d&=Udcs&Z$Ejo801Qfd$p+w$zfJ0exV3}=xp)Cj^c2*aj>ew
zq^EYF{N9jRdzt^KpU<h3k9iuC0>RWz?zbQ}#D&`(0`-!V)fQ7xVro@Sy0G;<>xVOt
zr=Vt_Bs!(<%j(Rvi)~UO(?`Uy$4JmhB1UV%^IEa3pH8tTg4#4xzl(|l=WWuW2oY|r
z)n`kW73U6piOmTIN>!%%H9|^rCRq9X@{WUBs}X6DlANA+&fdgWJE~>;=6LH{nJRH8
zeb%1y%VaMSC6RlDacvhcNY1}=WZkSH95mCnZ9gF$g+bj60>TwL$lu7p(1-a<noJ)D
zZad4Ub%Y~cI&A8znRZSt`Y};lxOZ5irgD@0=8=lv$&|{QzP(X&+qV)%DV&>%%W}6?
z!Qs_$&n8Y>1J>JIV6P<<dS%|hrw**K+)yyd?a~(_mLJMD91{&+s=KM9$658g?~xM@
z5qDMj)#oW+YMSt-BIVYk?666JIv*-MF3Kx*O2&)&82|~%OuKRJqwu{0ke#+NWSq~8
z{c0$?ffBfCw}@(E9~j|Ab?l1z6PtA2i7}oUkIPuSEY)Xxl=AYpDSUf01I$<QABzdt
zPu;Wem{cfQSoRV19v#XiH-G1w2K0vWmhmiKn$8yL&7CP^(04*FBCg&fg2y&&exFQ4
z4E=|$^|M&h+bT2>&OL-~c`vgVKeYmFXuHe6&-1jWT!F-G)0;uWRw5R#xp6cXLN4Q%
z5*!$j|1;NE>BJcHr*?#n^wnFMdUNfY=UsecUZSJpHH&5g0c^#_WG)VNy*UDQuiimW
z8|gGbNqpc4|5Ra^J%z*^|FGiaOJVic_$##jg7#c@`ZZmy-s(F&aBV|I=%dZEPhF0{
zS8^|{?-T0xr+$*Qzf&OSWStF?8P~HN^$e#dk<P<R-}GJJt@;0M(Ey^fg_EMjF>%}M
zh>rfN+_ti_*gmJ+;?SuQ%FW<YS99kLodJ}D6OrZlb5n(OC^r&bDFtezYwS;~`hI2m
zPS=;*&D@tk++b5BTza_Gz*mXQF*Np;!`%-?Pflt%wRf7mdG`Xfbk&?_ut1Rx?l|>F
zC?ShU%#%nRyke^#-6+<1gG|2xR5L<qnAHKl+A!p^%&j4K7q}{@%Z=$5%x4D{TKcn~
z?qeU2S8mdb@nr$e3Y_V?AT)+5+Vmr|;&~puJ}<Z^U_im~?-F@^zKtproPqBh)ldgC
z-YtsrU7<@4v}!M6qXMl;T5|$Ny~m65V!rFr4;sOp+e38r)4nB@=hsnFL;~cC*3-nL
zygVuC={Yk+%zangCi55j+2A+!^V17f`3@H|pBQ$cJ(mir<Na^N7M+=XXONBku7Eeo
zgMh%kTc2fTZub_p^2l3Go$*q!eDzKpS(~SKlT5$ctxOL7;o}AkCNfL#Ew&4qch+yf
zB-Im4+S82O-r3*cm3fDKx9R4E*hl?@$+|9fIk5CjdSnrm-?jVRc1(KwPQC+nxJsu1
z2cB%^Z`rR!F|*pya{9N{KuZQ!=5}u(U40;q2N#I;18)}TS2bT~{i|!OG1h%UmvD{s
zCB3?)F^M8+?KlH?`9o;m!;~}NsL(vWD(>ylw)uHqUp2<Z&B9I}S^VfJHRbOk<cM-S
zpV$+Sf@!bBdYNDHT*n0zt3Of%FgSbS2rCT!>_P)>MQ_#VkXyUb_exRfgy)Fj;(K)}
z#IsETp)|p_Ec4D~1lK%1^b{bym3KD!l=+tx%I7`s%{KU_%xHi+8<rVbmX;QPdH2uu
z^*%RS?aZQkz0HX29%I{)!c}!_mD|LKCVWl%4>?80E~Wt1f8tx2nW3G&QbTe$e@L!S
z6WXcoFyfdvkv{|Om$|QB0Ypo4Bw4al*Yz(hSYDVy4(8mQKy*BhXzR|zQ(r&vRp^24
zKP@8dbnA97Aq6Tb+w!WlBSqPALUA@#UaMmllO?8l;IJZQUydItoE>lakivSrc|teG
zE??cId@A03u=0L{YvsJUSxN0Y3`H@RAFCR~h)ia;x{Wro|Cws>3+0Rjp-N1~)hsz+
zLX~iEYzLFtzon`YX7dTH32ypwxj<>I*iIniZN1spq|USGQeQ108X*SZb9OjuCh==j
z;rM1bgqj>jK*UY0?T!G4bcz&-8NEma2sg)>I+nuH|G;+X64#J&Hv!s;#+)yHD6i@e
zEhKH$t!lw03k~=qr0+_)-v=XIk!jN;e0M*ox4!^G_?6<If?6=W0w*xD9G36moj=nh
z7c3{h?)sba$<O9tr2&*<aywRSAuftbo|+WLpOVb8nF`yT3rLLz^`5_<R#w&mw(=K=
zVV%?D`MLG;u$2X7(Z>1v-sc~cm)~k=MbCZM*6v>PR&Ouv2)K&ltfmP=;RiMaDK>8-
zQMdCPVwIJH^`n_LM^8ZRqpqhjz=hBr4{kuwt8f<3%>2L?u8NX#t$s6tcbA;aW1Zfe
z^fevGEem;mOSxiw4iOEtzTENM3)`&;-|kw^XB67={g%bvLERBIU%F~x&x(?aX>%%9
zSjTtW0gJt^9pP<nWKB(WXhyiRp=DFC^<BY;_EM|QDdXZ%4L{2L;y;2>@QHTVe-H0K
z%;=@}2BtHMYEssQRtIhL`-k$ihMiLA3Y=_c>E$>Lxo<mwb3yE`BXGU6<HQlBkRLb$
zfd0TQu#c7MBEfm4wcNX)m@yZvw$U5m>S>D}y`xhcGQk))M~%?=iw!tn#{I%U_)Df7
z=@744%XUqRzMcOChlsOf6W5FBQVyV<-*U}~vpFW@?Wk5%OEE&|OGuC@vyo}MUdqL1
zhP{Hj-UE`i#vbTw{*x%C%wGL1P>)vpv{paI^qS%JhP`@UuE$6p*-m5mi;n|mXveUb
z37ox^Yu>lkzp|;lCL_<0>Y@Zgi%ZkeE<A>^Ke)c1bjYhWvb5WDC_9c7(Gy5L^q7L|
z>WtI~eA`H}>ibNwO>55Bn@M<XRWYd)f6X^9u#!m!;A%(AP8gHpikn>kZnuRrb1Kw(
z6~VjLHasuk0<||BVNo!C+Gky@yEngonyEstS@;F-Fk{}a75rxEz=}kRXandijpeSt
zb&8!FIca23=czJYSKDN#jjIWm&7_S{XlC>)bb0g-3A1S*Yg$KaBj!;wacML;5R&TF
zF@`mvsH7Cl6xG<cdSx~Oi@bH4OeMP(Dvl}S0aLgwF(}7hQA`8aUWtV-#ku(Wyn&a$
z-Zc4)66>m0-2>?{Nfy1Xt1JoAIPC=@05OL;(>6S>b!qox#YQ<S^(rfCbf4e#v*rqM
zojcr_^lA=x-AH+W$uhg}c@}Cch(M<#(duz|I+n;B9$nYC>_q2}Z$gawWl)r-(1a+3
zxKQ?x>VV|sQWsD#R16h{b0Hl^m%Zm!t(l98hJ&8PYcCQxQ1`bX`Cj%!WO&|(wYLRL
zNp6ElIp)U-xm^V}4(h_!5}GVGXMa3`6{fx9)l;nPnCbX_J<cBK7`5{Cpo%-F)0b7;
zy``mJ{uc485So6T{840&ohfn$LP*myQ%^B5Dd!;VoBdsvY5T-M^8SxMd%Q)3lF#OI
zakt-AOhh!~b@oyQJ-P$34$60`B2RN()5nk{zx}6WY9CubXrk3UFVj!E_Z8xj@Xn%X
z#p!rj;xxLXFnKp2I_18tap23MPIw}EQf9c3`<D2j&CDj?WoeNZr(9EVHm)PzsBEUK
z#&W)5;fF*NGGI9w!7r|Iiuk3&(2F0|f(?H;``dgmS^v1StRioW;1{R6iQf~8R$jwT
zu%Vw$g!z>9#$-H_1chba)WjV?AZLPP{6P3QhC{)-79Y@AdcpnG0-^QSbsx1UHUztf
z^vc_{NRB6E8%Tj$6F^Via<A_c*)cOrqJd;WBi1u`JQYO=y76Hgqu6kNVXt@=B<A%g
zc=GbX5Bz|^6+hb)@vWS{MXlpfSJYIH-z};}o{uc`5Izt3Mhqm9kW1!$hmC{8^UO&;
z?W@KLmCIID!P%pXb3E_{VhZUWc_#Q;9LR9Tk5@Hd-Gj7CLIn7au~hOfwNU?YTAY9z
zRf)6yJX#YJ4f41(be$Nck)gQ+13f@jD<BRn(ga%DbpRT0xiy;Q)r(i}IPWQ4%H4r~
zH`)+hgi45^lRA-o?UA!O@Z|>*LaD*-dYNhrxhtEWWsFiai7D@&|NU-=?lHg1puZv=
z6lS#r9b{jM>CO+kOl6dPvEmgy{pPjXO3TpKaL`vADOGnV+B0XLsL>ZL@8#K+p8j_-
zv$wi)ruz(Ow-6{Ew>W`3y`cAMGUrPy|JzRmuJ4-iOu7n8s*p!w<LH-?s-B4@cAnfY
z+Vd7KG->=YQqG~)aZyjcGB8d09Ao@*a0!vG9%Jk^wRUk|To?S@_eqU|Tiqn8{<`iy
zgg%OKO5o0#5xR5q5EvnCPi;@@p76T*xBkE@zD46TnK+cGh9QkIl-mrnUfu~h!V}7J
z3BeQJmq*m7ZKDu`$?1$La?qk>R}^*O8b3q1A$`(sahe>607DUoUoo(^_9D?=D{-QB
ziJW6O2QazNrCsZuSW=k_Jp@5*_Ed}4^%&7or$&6VpV|f~J!#$_6CzyX^Twt+z2*Ap
z_3j$0T|HkNRDfP(Z2`(%5Sf>jg|D<zHrODB2e^e7E070N{gD5G>YK~1e)GupZ2DM|
zPP=pt_TCNt^n+TBGR8Gm-WA0&3qeuId&=?mvX^horyB#~)(-8rMHX*S3b@0x`BcTf
zi;U@dNB@3%QX{<!1+Y-v7p!*^Ahw$?Eg?tKi8-llj1_lAo217}tHgJ?PfzHuh0*UG
z*oRK-9%7CsVG@B8>1V=4<pW;*+Vy~^2%8GA+_~AF#IzCti|s$9!PR?0F%@f4F#-wV
zs>7ar;R^YEI<gDoTZ7>W31I(u7kV!m<bG3_m5Z}}*ROR8I1gl#C6_J7H;AbtC4B#`
zTL7Ez(E~c$;vuv3k+YI!6$yzq6AhN*P1+Z4M$a5->VX~t{3Sf~lw>wE!gJ19F+J{M
z{n=BtQxpYkMUT@c0iwbuFcpw@mZDOH{7^3W+wQi9^W65=uLBN@Zde%CwZkMpkA5lF
z{yVjfTyCdnP6kWrz1Jgh9yF#^FfE2zWrghJdDO-AHZy$A9z(8Mf%Fn;fW>{_7Eb8Z
z_s}_s%YD{Ypxc+N^+wAsoN8?FoUyb9DnKCj{7Y_L7%sp5<;tm)M0x)+8`nJahU&yp
zy9?&Wl{JVl*Sgn%XB%SNh(uTQ`}`j`=(?Qola;)-j=HN@{fOz+5W_B|sR#(qrJ9O%
z4fvW`wotgsW}a6lfLgq!Cj5cP-gZFQw6`FxP=FEC?;bwSFIYXxJCH!Q?yQZH{Zet=
z@4~Zi_xGLAfCG498)WW;lbtv22&L3gx@9swIl}*Q873kCFzVLA-<v_~R(8*kKL(tp
zbY}Mt5?dp`2YA-wE%MPEh3gt_#tC53zGcKJ$leR*q1-LUto(KS9LjpWe9yOiO+|+{
zT!d71v=~K(p#Iu?UCm>ISh$=&HuM}=-#>;Nssg_jGJWW%xy%ag$&aVsnZLF#<Drd2
zvpRmidTzCk#fD4F&NA>SN!lu>i%c2D7wu<;Wh^=O_5{_HEmV0~Z^k&C<;J%>o5F$j
zL}@@0nI9c8@ALL4J3jXuj-Lf~YUf-tSe-!*wA&#TdP{DYSWlz7Ww`OVce6`A*PlY~
z5y`hCb;YkjMJZXzK4$L?79Wr|hryz~iHM)MahMY&tWv=u@7pwEqaT=9;^CLRX4{9o
zLMcu}J78-MdPX4WMrk%l_m0uNVdv~e4_)j_t-0H1N9kw{YxSh<j^riy6+4naNQelS
zZ1L&Etj&)c9MMpRJzDrJCsyZ@<Va?I*rXOMBG|OVG1x5%-AC!MHuoY=TDg_vGL}j%
zTi2Gj`KOXF%!<*ix%R&P)60E%*6r!_Zu-#!<k8U(u^u2<5r)(OC^8HIcE*23-MDGK
zlDyvb{$roxdiY+o<*Z4I5UMEhnlA2V;|D)A+eoJ#4^T(F{0vs2&UEkHI?r{num1vM
z4&WHZBy;>SWBd<yUnj@O?SrvU+XJbyjrLGM^8NIu2)2nfX^QmG`e#oGC|F@e&!Abt
zWcu0s0>LL1eVWVwvFUV7F+ZX~>xe2=^SAL)?1jMHANgO_a}0dcGuPHX;|gOF5g+&-
z$4!AA{(zje(}m!a=9ryE>J0&9ar@SQ)pG#iWmsjz-N9!J$+HaPHZ|E?7v6@M4d2w)
zGm1!bDUie7!qL-hbfqhm{Xo8q$K>|vG7du=j4FQBz1e`|-)whGv(cffQ*A28(7*Fv
zqXV*IA+anzg^0zF)o7OA08L_xLxcnyqxX58dPQP;fOlY^p3<C8gaJ~Nfll0+02Yg^
zIOMX;#i&Z=VhM;h14})lObSvlS$KP<%Q*T(;KAb~EG@^P#;b|e-oB$g8A~aPgAvkE
zx70;(6K@u3B+>D!H=cuY)Y>CBW1qOxlXF#0+`gmL6u`TC!5O+W1svw3m=Z9J<G4L6
zH-jEW4tr`<%-Ux&pQ(w|_A(jf6joHcv(P}ZQ3n0|UVV5ZsF>#u`2iFYzx?&I3e=yB
z<H=M+8p0lRX_rVI%wz4|#5o?bcb6U9$5oYly`7}g+<stAst>I#Ts9rByzDoI{SqmW
zVlyFtxqPPOaLhBhSkIInpd$WoF0@PPb`*Bwy^&TLsgJi%RFI2s0n9XCM1&-m#Ql>V
z!pHqgsYhzA89#YWb;D%<^vI0V2>#TBg}R?|PkVfS1+v}X8+--KJ2K)%F%t8G$>5O2
z)RssslsANf@4U`_m*tQGRdmJizSL{@Sqaer+(sEb#JhY^7U%>NxGN7W3p3*P=*i|6
zbZ5~mP`978nW9jH`aHcJ^5{j?DXRL>sa1Ok25wcG;ia;^uvu7C0KV@Q_xmKkzG|;{
z6W__W_(xP8J>h)SKyAD3wI%6@ozzbTWp7o0hG0TI@N!@HSufAw^rJ<@1{!nNYm-R=
zg&2f$tFK?sfj2lg(T)n-)N|h10ko4Y-FA-Jv{t|E75uU-uX~&OcA44-VsdxEU~2xJ
zJ~U8IKyvaE^Bpre{9$;zlYAKR<zw`cm+bg`A9`5Ed}Dd+)>&GY4M)^X#UFpQb4J@t
zVnG>V%H05aeyJHvT~j{@%J{)6$W~yCf|+iSL6#8s^KEQve3Qy;#nk=Y;rW!`hP1mA
z&Qn%{>W2V*|M1B=7R`{qT4?>~lJ&)e+HK}E1&dGD=wA)-YD*Tm@DG0-PT03u_+p(`
zPi(5A^+f$kFVDsbV=bl`Z<{Lg?PBe2z#26v6aRPJ5Gt_${=t*@isP#f^^DMObJJEw
zlje@p4LgO=@r7epif}7)(D%9{{n}B*VyJl|7zEn=SHN4u`yYP~`#}A@C4u7dE6Fnu
zY-^ePXLxS2?E&Mb7H(6wT;3I2W9$~Q73MSI*}W;ae9Li<0)PnY*y5%p{+`JG>S$h!
z&4~R*k;5POVvrRQpOdmPX?F5wrCriJGph<I-SZk)aon`@>6AagWFxLp7OYtLkEMTM
zxT3_7aVCcApGk5eJ$$mL|NWsFs~Ponj`|nx85>9jUa+B!z^V7>XSRu8(k5VEh6Dq#
z{$JbBg$Twb5q{0o;Ps5id5rPYy_82&9zU6|v)a5vPe9f;^7F0;`HoMQYR1Zm5XLxs
zZ=D-s+{2pXo;dzPoa3{AXVSQJlhQPB5)pVj_sd(P5550i-XdZ!!MUpfHP!V^?(IbX
z7|b#-Y(tFCKsl0IcniQII5#B52NhLlF=|~q=~dq{Pc@ncE+rIc_6{4+b76j>LlG_K
z-pAe%*p^_)in%5~;N7<F-ajLn^hv1DGLnmJU64Ah)-41Z>5B%QAAhax3JE-}kr%+>
zP@BOyMX7N%6IQNz49K-@Nhpf!Y<bYDto1#p(EB(AmQ{`mfE=DQqg(WmO7FFsQj7zC
zjFconJ#@6WN9u`zyW5lxRmP_U0*=EefOROP+Ua9z|EuE$mFsIV6A&E#50ro}t)=nD
zJgoJyXG7L6A~s`Xoog(SI@M~;i7WB<cLNyv<93L)P21p{@vems2ieQo(j7Bd`x@vI
zsd+FYE>O?;P{Y0q`;#*}GWbHKm(x^?lP(Gvl<Z+^cVjsjR#%QgQu1t)c03c*X*Vx6
zf`;5IT5InNFl(D+B#GnuFEKiH!T2w~{PYB-GdE-`Z<#O5Z0L>_Owk&ccYXfa7+}V}
z?*qlLzVN+WEZq`l5guRH-#zd+WoLd^T1Y@iG1jE=lxA96bf@CL3?cd1%Q8JX8!J+q
zH2LtRb@M~FMdo!cdFG+R@C#iAOyzG9lAw}`J25r@MpFe2uIoDwZj}L_Kkz1?tiIa2
z>x{Ib!d0cgEb<jk3C*<26(w&cS6smp)rOrT6M5)q$&S&7@PL+asN{LTFNsi|sj>6u
zr{bJ@nMC9d?mGZ-@Gk5xFiZp-p#L;XoUky~^LH#t6nb{OAyHw}!uw|7<3ho;`yV6e
zJG#ocL2E@whh94C+Ww|f5U8!=cr`m(9YLPpeief|$wbp*+3G=lIey`QJSln@$KqD{
z-jbt`H{;ZS(td>tk~(ocQ5&&GcTY}BgaWz>yYSB(S6xW1?ev`I-ZKB5iH|lUP|YOG
z;bV#&FM|;%d9pr>9NcN@|BAfeOxj0WZ)E#V#Fb|kEYm7?JZ~1WA6bGn-NH#t>U$#Y
zmrEgHPq%;X|KLf~qzZ)Jua|>p+fowOXODw=VJ6&O!qg&I@5rwWgRI(=LVSeu`oE3G
zzjxMX-sVFHj>XCsxWaVa8!f6ghg1!J#rR4*p+{Z741RTi^PpEy&52;nC@a#>9I2gN
z_a`x{Cr?m4ozMJU!E&2!8nNjYHVOEm`6@Ng?PZJhbBe18xsXki_W=djtVl7g0m+q)
zR%hY_2Ti@s093ddcluS~Mo%Hngg>nO;o!}R-N|epi4Q7W2um$`I?_^I?3?5|nT<YK
z(diOc@Qj%?i+vU;*}&abv|2m4QII+uhy>}fBzPfka#AOUxE*tDS9(q!EJXz$g%_n{
zzMOmRi8E@Hge~nGzWUp(bos*SU~NwqTG!CMVtM)Uh&^`J+RNeD?V!<z2FE}-rbaW7
z6?PpI1;pZA2cql6qWYJ5024avld6z9CPv;L74_SAL?|@%Q2QT!@ASBCxxSt!Y_>TM
zj`>}4BT=g*>zswCN$mG9Ym#f)96(Dy0z)|2G|tMO&F6lc6L$-nV+vYrW_6ZI7~a`Q
zE+fkz8Ar;^S#!y^h4LNvQqb=0V#?4}nr&CB<2qvXLwY;Kol9VCz@;e~mh+ll0Bp;(
zZUO2>YHK%9BIUVFOw<>=^RJvt3&8JK5mSP(Y@GzXJEkFdRj<R+s$d6*0)drv24`A)
z*JsdH?3(JNW`n@wniThY#pA%l#!>Hz<W+&eYiG_Isk??CzK4tyv;@m{!8#VRZ`56b
zt8e6RyAMXr{h#)(H6H4;4R2yIuO)2~D^WSD!<Z1>acD|}VT?o0#|<&0)+m-3wu6$a
z7$!3gW1TCSa!7>1E9*QMhf-~lnTQ-_SZ2b!&t!XlvueM-pLRYypZ>r9{GR`PKlgLp
z*L_`EH6fctSj8X{=1z6^KYFTgH=vx=CG8Z=kp@Y;V;GobqATFsT%6HEj(2~7GW;UB
zX80hiHJ%mqp+m1H)(5zUym>3hUS&lcb{JLqT!z2UxYeSk7}G%L;!uCe`!$A{wX~g~
z&KlPpVv90(?V-quegx&5uH1^UG@$&A66Y^|6xqP+oT$IRXA!DS#<;`*!BYkJqp$Wq
z7MMu_AaYDxZo<x;$BOpEAdA&v3q`#4Mn^gAFH#yYovEOV6#xPm_SE+Zs1Y}HNxf$s
z+0sc{MhZV!lBTsy(q`p3UgMzI2RR-gBtVsx6Y8QL4={tL60Vb{;?zs@Ft*8)5dD|!
zZ&jaFpp&wk(HZsqR88*Ka;ZXy|08p{Mr(IRSU6TBIzl9ea;<=^ZRm5f@eycgpUj()
z5a2Uc3$viL3Rn8<s)E?~y|+ovL$Kw>4~p|9oIKx^P*OUzHij|GmHfnl3>N}y^;47U
zRjLbYV0`}*zuXil{m;fnlSB(|T>Ca)wEiFsQZ?Rrc!=$c52<$k22{i%3q?5R7Fk!o
zL1nq2avMGwNl(Vy@1+io(YCIn>{-p0HxW3z&R8oV#8T5jF!3BrN>z|!z7p|C$%D5O
ziP_bA@Mk`f(PWzD>=6U)RpO|?!S1OCBZV07?Mi~ENNkz}g&Gx)`}VZzmH~NzzwIrI
z?6sj%2yIU6;Hb-bQOsA>7=!uqtHJ2lN!XeB3rK{MVSzXZ3jjKQ67*LyMD&e4Qvpy8
zK~b=abH+juvDKax5OumX*C!qD^@m1KA|_3Q(;bvu7H3L$Eo$s`*t`dD3M2XU`xEav
zC75Rg>1doSveJf!qq6cC1;_H(*!8|Bdn=~h53672u!W!Z&j|A8#+G-KqVpL)Pq%p{
zp&Hr^Yfr^_UAxnRMi1I#88TYs)<lGGS==2Y5Xf5L_GKoK{S%oXqu&5nXDc}JNH%hY
z8w4WZ=bQLLIif2P4Juc#dJXOBNh;H=qs;re@{-q+9T{I!RTmux!{2L=G>wUP3K2g7
zglx`&T%ObfiUHUci-7|-cLuqaai;T@61>osD{2t8&<-s$g)wwDX38gw8RHqzZoz9&
zEV$z1&{_{JMXG^bc;;>Hf+cut$}p$kofdl@XaxW2Zy``(L@M5#>!Ba%oU1Ohv2(v4
z+dg$At8A2xU?iqndYr60X3K@yjMdyF=cg|KRhWav@!U(zPeVpGMvn@@cgO(h1@JJt
zBD~pv&Q=&JRIX{3<I;uB#oQow<`u;&V-{VtFb3R^ecP#;8H;J=8K+CgZgsS)KzS0|
z@+ZR0Ugccw{q0?hb+ilTbMgte4V#(aq{el5%AOsm4z%retFenEsYb`uc5!c+rR6Hc
zlb7a8$x#=VW%>@0_86(Y#Y9hA0>mPBfouZgFgi>6D+REjBV#nll{p?40XmO0c<*$@
z>mId{Fk)IGDvXm_K6rKg;ufm+fgIfW8e%=t8JYg{nhvNvd+SvA-5%o&HCWynnWr~X
z*8|kTbmatvD&|!3>Pcw_7WAVzjRntx{cxk@jb5rJ_#MC{-H>9Sx5xKX1NR>AihK|R
z>=9vBiv`+Io46*RbAGVeH5|?h6hA?Eb4~}!eaSowYMW}w^ZOxC?(OLBLX9N=m9YGZ
z7YeC_p)bu?j8wh&pa;Y8zbFqem+6?i3ak6#m%Hu>L_Tw8`|^Yur+>DGjzlHd5QFv9
zYc8x2=U(5E;bzfTN@Fd>*^V_-BdV$2Oh`YJRK@I@06XuhH#aArqpBYh0VVeoF#oPx
zYGV1?yI%@+6bi}^b})fUsZ!rCl9pC1!Z~batje~vk^0|L$pPA<?gc>=H-}^VZJmd_
z_@xq~V6A{Xn_aAKIG5=;|Lj6x^A}p8??orft)e^7W`+H5*@tD$>7I8rcFSh4$F)|Z
zre(OQ(uQ3bG}a;C^9{O)ABYc#2$E(M5&wocu@SSXIuIdt%nRdJbYOS_x}ksNV$~eT
z6M2L`1>aJ*W4OemD!$`AYY6N8^tQJFxPE8=slZB2khpv>IzHpJo?!DhlOH#4kdyjH
zFP}g8SCoJGz)k5)WagAlqdf&(hFKtzzbQ?8=o*m^rOJZmLDu}syQBH}8p6-lJ!$OO
zFhFSHJuy^merQyQ5jg!;l7(6HU^t$T7BVG15nX|-QhQxot`9>$2#5<Ui9;b(zMnT5
zM%&X@T)?-mW<~>42X8=i;JZ8}^!{CtRtcdzY)aLBE`E~rFlH;o^v?4V>};_NH%VeO
zP2!G!MYVH2PN!5B7Np~N$~LatYI?B*6fc0H&M#gP`8RcqCbl>)`;~~Gl0tk_tY}a&
zVK=eX?#wWI^4&O86&1X{JRx0<I}y&hyq+ktiq-!Q>Bn)scY%a=?AxWJo<#f|M!EK{
zQ)!nwlg=?0YbbvX4D|+%q|<(fE_|t#3KN2FOZCQ@njH(@$Ejb5_;%`8v!1hg=rd=Z
zh(3g$qzq7t{BfVo`&}rN!R`)*!$YZi*}(6(K92AEY|+b(W#pXQ_l2<nMl)8(()kTi
zV#wP3m$W@@CmpiZgLC}`3G2{Mc;-4V<y1<jFmHHjFpzJ>oG1$!u#}IhJ`0`+Jb^Xj
zb1?v9OxPPM5Nn{ZSw1js3gsyq8x7cn8l)))cfOZ!97i>h;=3TyAa(6rExZxsG2lhm
z3rbt$`AZZl_T`uI5YRSIp0gUMwT-=X!s@=)#nc=Dg!Dzikvce<2NMemP#z$Vn~=^v
zB2HQQykYA`B03DNqQ^8#7!AMa4KhytrEP0c%PS*yJPr}<0!6z!eh9~InBj6}SW*iA
zrY(zrBn}v`n`}gn3#T9H@_B9ToMgA=__8B0+m-%Pc;3GPD*1yY?g(j`FTdeSCzT~a
ztXLrHGy_#-zO{hkIh;}pp0Cxh86Q=o!01RQOso2&wFj@tqA}4WYdrYz%)r(4WRcZB
zmhevjE_5!xJ3AY|qoUDoWk3d_Wxh3HJ#GaV_Li@|s^H<Z|7Lh#w3Wug;Lh>EOJzw*
z^?BB?3Tv_}V9H&;p}*t@@426J7{H^(eTDY|>he0YNQn&CV!<rs1wlJb+qQwKJBMGN
z>pEN52<S4Kt%du9poqlc10_hDd-*Rdq28?>>TS5+2B=$f@z;Z26REfl%vJTG*KOSH
zZOC1{R>VptoY?k5ZnfEuip6)co40IeP+i!r^iv%mZG9#J%@U8cG+i-xO7PDb9f-dr
z$2rk~)(x_E{{VfUHrs3ww_?6#BZ&Z1N6m13S*CwnP92wY=qTOeDnFpxj9(3WnaUwa
zKG`3SWof!WySuEzn5{(5+xu#U%$_^YYyq0t@@yE5k>y=i8CmW3?|*Uae<wM+bjqH8
zbN?cBQ1Mk}$!<hZF&~2{y)*JSQZ{4Po735T@_ZgRfFH53ElDXUDq^RZ6OZ8VqrH#8
zJ3>$wn~W6<Xpv7Mk8Xv0=GeBQPw0hJW%d!eWQG}?AbqgbB%R%&WkA^dG06(o+Nkk8
zM>AB|%J;lq;jpNz{S&Z{;}4cOar+P?f!Gr04ga(Ji}bfZr2MlK=z{#;mofMz-Zo#?
W@A}3t`E6f;pOv|-**#O=i~j{8Q%T4G

diff --git a/docs/index.rst b/docs/index.rst
deleted file mode 100644
index 32c3a33a..00000000
--- a/docs/index.rst
+++ /dev/null
@@ -1,160 +0,0 @@
-pytorch-widedeep
-================
-
-*A flexible package to combine tabular data with text and images using wide and deep models*
-
-Below there is an introduction to the architectures one can build using
-``pytorch-widedeep``. If you prefer to learn about the utilities and
-components go straight to corresponding sections in the Documentation.
-
-Documentation
--------------
-
-.. toctree::
-    :maxdepth: 1
-
-    Installation <installation>
-    Quick Start <quick_start>
-    Utilities <utils/index>
-    Preprocessing <preprocessing>
-    Model Components <model_components>
-    Bayesian Models <bayesian_models>
-    Metrics <metrics>
-    Losses <losses>
-    Dataloaders <dataloaders>
-    Callbacks <callbacks>
-    The Trainer <trainer>
-    Tab2Vec <tab2vec>
-    Examples <examples>
-
-
-Introduction
-------------
-``pytorch-widedeep`` is based on Google's `Wide and Deep Algorithm
-<https://arxiv.org/abs/1606.07792>`_, adjusted for multi-modal datasets
-
-
-In general terms, ``pytorch-widedeep`` is a package to use deep learning with
-tabular and multimodal data. In particular, is intended to facilitate the
-combination of text and images with corresponding tabular data using wide and
-deep models. With that in mind there are a number of architectures that can
-be implemented with just a few lines of code. The main components of those
-architectures are shown in the Figure below:
-
-.. image:: figures/widedeep_arch.png
-   :width: 700px
-   :align: center
-
-The dashed boxes in the figure represent optional, overall components, and the
-dashed lines indicate the corresponding connections, depending on whether or
-not certain components are present. For example, the dashed, blue-arrows
-indicate that the ``deeptabular``, ``deeptext`` and ``deepimage`` components
-are connected directly to the output neuron or neurons (depending on whether
-we are performing a binary classification or regression, or a multi-class
-classification) if the optional ``deephead`` is not present. The components
-within the faded-pink rectangle are concatenated.
-
-Note that it is not possible to illustrate the number of possible
-architectures and components available in ``pytorch-widedeep`` in one Figure.
-Therefore, for more details on possible architectures (and more) please, read
-this documentation, or see the `Examples
-<https://github.com/jrzaurin/pytorch-widedeep/tree/master/examples>`_ folders
-in the repo.
-
-In math terms, and following the notation in the `paper
-<https://arxiv.org/abs/1606.07792>`_, the expression for the architecture
-without a ``deephead`` component can be formulated as:
-
-
-.. image:: figures/architecture_1_math.png
-   :width: 600px
-   :align: center
-
-
-Where *'W'* are the weight matrices applied to the wide model and to the final
-activations of the deep models, :math:`a` are these final activations, and
-:math:`{\phi}` (x) are the cross product transformations of the original
-features *'x'*. In case you are wondering what are *"cross product
-transformations"*, here is a quote taken directly from the paper: *"For binary
-features, a cross-product transformation (e.g., “AND(gender=female,
-language=en)”) is 1 if and only if the constituent features (“gender=female”
-and “language=en”) are all 1, and 0 otherwise".* Finally, :math:`{\sigma}` (.)
-is the activation function.
-
-While if there is a ``deephead`` component, the previous expression turns
-into:
-
-.. image:: figures/architecture_2_math.png
-   :width: 350px
-   :align: center
-
-
-It is important to emphasize that **each individual component, wide,
-deeptabular, deeptext and deepimage, can be used independently and in
-isolation**. For example, one could use only ``wide``, which is in simply a
-linear model. In fact, one of the most interesting functionalities in
-``pytorch-widedeep`` would be the use of the ``deeptabular`` component on its
-own, i.e. what one might normally refer as Deep Learning for Tabular Data.
-Currently, ``pytorch-widedeep`` offers the following different models for
-that component:
-
-0. **Wide**: a simple linear model where the nonlinearities are captured via
-cross-product transformations, as explained before.
-
-1. **TabMlp**: a simple MLP that receives embeddings representing the
-categorical features, concatenated with the continuous features, which can
-also be embedded.
-
-2. **TabResnet**: similar to the previous model but the embeddings are
-passed through a series of ResNet blocks built with dense layers.
-
-3. **TabNet**: details on TabNet can be found in `TabNet: Attentive
-Interpretable Tabular Learning <https://arxiv.org/abs/1908.07442>`_
-
-The ``Tabformer`` family, i.e. Transformers for Tabular data:
-
-4. **TabTransformer**: details on the TabTransformer can be found in
-`TabTransformer: Tabular Data Modeling Using Contextual Embeddings
-<https://arxiv.org/pdf/2012.06678.pdf>`_.
-
-5. **SAINT**: Details on SAINT can be found in `SAINT: Improved Neural
-Networks for Tabular Data via Row Attention and Contrastive Pre-Training
-<https://arxiv.org/abs/2106.01342>`_.
-
-6. **FT-Transformer**: details on the FT-Transformer can be found in
-`Revisiting Deep Learning Models for Tabular Data
-<https://arxiv.org/abs/2106.11959>`_.
-
-7. **TabFastFormer**: adaptation of the FastFormer for tabular data. Details
-on the Fasformer can be found in `FastFormers: Highly Efficient Transformer
-Models for Natural Language Understanding
-<https://arxiv.org/abs/2010.13382>`_
-
-8. **TabPerceiver**: adaptation of the Perceiver for tabular data. Details on
-the Perceiver can be found in `Perceiver: General Perception with Iterative
-Attention <https://arxiv.org/abs/2103.03206>`_
-
-And probabilistic DL models for tabular data based on
-`Weight Uncertainty in Neural Networks <https://arxiv.org/abs/1505.05424>`_:
-
-9. **BayesianWide**: Probabilistic adaptation of the `Wide` model.
-
-10. **BayesianTabMlp**: Probabilistic adaptation of the `TabMlp` model
-
-Note that while there are scientific publications for the TabTransformer,
-SAINT and FT-Transformer, the TabFasfFormer and TabPerceiver are our own
-adaptation of those algorithms for tabular data. For details on these models
-and their options please see the examples in the Examples folder and the
-documentation.
-
-Finally, it is perfectly possible to use custom models as long as the the
-custom models have an attribute called ``output_dim`` with the size of the
-last layer of activations, so that ``WideDeep`` can be constructed. Again,
-examples on how to use custom components can be found in the Examples
-folder.
-
-Indices and tables
-==================
-
-* :ref:`genindex`
-* :ref:`search`
diff --git a/docs/installation.rst b/docs/installation.rst
deleted file mode 100644
index c63af2fd..00000000
--- a/docs/installation.rst
+++ /dev/null
@@ -1,44 +0,0 @@
-Installation
-************
-This section explains how to install ``pytorch-widedeep``.
-
-For the latest stable release, execute:
-
-.. code::
-
-    pip install pytorch-widedeep
-
-For the bleeding-edge version, execute:
-
-.. code::
-
-	pip install git+https://github.com/jrzaurin/pytorch-widedeep.git
-
-
-For developer install
-
-.. code::
-
-	# Clone the repository
-	git clone https://github.com/jrzaurin/pytorch-widedeep
-	cd pytorch-widedeep
-
-	# Install in dev mode
-	pip install -e .
-
-Dependencies
-------------
-* pandas
-* numpy
-* scipy
-* scikit-learn
-* gensim
-* spacy
-* opencv-contrib-python
-* imutils
-* tqdm
-* torch
-* torchvision
-* einops
-* wrapt
-* torchmetrics
diff --git a/docs/losses.rst b/docs/losses.rst
deleted file mode 100644
index 6ce8e769..00000000
--- a/docs/losses.rst
+++ /dev/null
@@ -1,65 +0,0 @@
-Losses
-======
-
-``pytorch-widedeep`` accepts a number of losses and objectives that can be
-passed to the ``Trainer`` class via the parameter ``objective``
-(see ``pytorch-widedeep.training.Trainer``). For most cases the loss function
-that ``pytorch-widedeep`` will use internally is already implemented in
-Pytorch.
-
-In addition, ``pytorch-widedeep`` implements a series of  "custom" loss
-functions. These are described below for completion since, as mentioned
-before, they are used internally by the ``Trainer``. Of course, onen could
-always use them on their own and can be imported as:
-
-.. code-block:: python
-
-	from pytorch_widedeep.losses import FocalLoss
-
-.. note:: Losses in this module expect the predictions and ground truth to have the
-	same dimensions for regression and binary classification problems
-	:math:`(N_{samples}, 1)`. In the case of multiclass classification problems
-	the ground truth is expected to be a 1D tensor with the corresponding
-	classes. See Examples below
-
-.. autoclass:: pytorch_widedeep.losses.MSELoss
-	:members:
-
-.. autoclass:: pytorch_widedeep.losses.MSLELoss
-	:members:
-
-.. autoclass:: pytorch_widedeep.losses.RMSELoss
-	:members:
-
-.. autoclass:: pytorch_widedeep.losses.RMSLELoss
-	:members:
-
-.. autoclass:: pytorch_widedeep.losses.QuantileLoss
-	:members:
-
-.. autoclass:: pytorch_widedeep.losses.FocalLoss
-	:members:
-
-.. autoclass:: pytorch_widedeep.losses.BayesianSELoss
-	:members:
-
-.. autoclass:: pytorch_widedeep.losses.TweedieLoss
-	:members:
-
-.. autoclass:: pytorch_widedeep.losses.ZILNLoss
-	:members:
-
-.. autoclass:: pytorch_widedeep.losses.L1Loss
-	:members:
-
-.. autoclass:: pytorch_widedeep.losses.FocalR_L1Loss
-	:members:
-
-.. autoclass:: pytorch_widedeep.losses.FocalR_MSELoss
-	:members:
-
-.. autoclass:: pytorch_widedeep.losses.FocalR_RMSELoss
-	:members:
-
-.. autoclass:: pytorch_widedeep.losses.HuberLoss
-	:members:
diff --git a/docs/make.bat b/docs/make.bat
deleted file mode 100644
index 922152e9..00000000
--- a/docs/make.bat
+++ /dev/null
@@ -1,35 +0,0 @@
-@ECHO OFF
-
-pushd %~dp0
-
-REM Command file for Sphinx documentation
-
-if "%SPHINXBUILD%" == "" (
-	set SPHINXBUILD=sphinx-build
-)
-set SOURCEDIR=.
-set BUILDDIR=_build
-
-if "%1" == "" goto help
-
-%SPHINXBUILD% >NUL 2>NUL
-if errorlevel 9009 (
-	echo.
-	echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
-	echo.installed, then set the SPHINXBUILD environment variable to point
-	echo.to the full path of the 'sphinx-build' executable. Alternatively you
-	echo.may add the Sphinx directory to PATH.
-	echo.
-	echo.If you don't have Sphinx installed, grab it from
-	echo.http://sphinx-doc.org/
-	exit /b 1
-)
-
-%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
-goto end
-
-:help
-%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
-
-:end
-popd
diff --git a/docs/metrics.rst b/docs/metrics.rst
deleted file mode 100644
index 0d0e84c4..00000000
--- a/docs/metrics.rst
+++ /dev/null
@@ -1,51 +0,0 @@
-Metrics
-=======
-
-.. note:: Metrics in this module expect the predictions and ground truth to have the
-	same dimensions for regression and binary classification problems: :math:`(N_{samples}, 1)`.
-	In the case of multiclass classification problems the ground truth is expected to be
-	a 1D tensor with the corresponding classes. See Examples below
-
-We have added the possibility of using the metrics available at the
-`torchmetrics <https://torchmetrics.readthedocs.io/en/latest/>`_ library.
-Note that this library is still in its early versions and therefore this
-option should be used with caution. To use ``torchmetrics`` simply import
-them and use them as any of the ``pytorch-widedeep`` metrics described
-below.
-
-.. code-block:: python
-
-	from torchmetrics import Accuracy, Precision
-
-	accuracy = Accuracy(average=None, num_classes=2)
-	precision = Precision(average='micro', num_classes=2)
-
-	trainer = Trainer(model, objective="binary", metrics=[accuracy, precision])
-
-A functioning example for ``pytorch-widedeep`` using ``torchmetrics`` can be
-found in the `Examples folder <https://github.com/jrzaurin/pytorch-widedeep/blob/master/examples>`_.
-
-
-.. autoclass:: pytorch_widedeep.metrics.Accuracy
-	:members:
-	:undoc-members:
-
-.. autoclass:: pytorch_widedeep.metrics.Precision
-	:members:
-	:undoc-members:
-
-.. autoclass:: pytorch_widedeep.metrics.Recall
-	:members:
-	:undoc-members:
-
-.. autoclass:: pytorch_widedeep.metrics.FBetaScore
-	:members:
-	:undoc-members:
-
-.. autoclass:: pytorch_widedeep.metrics.F1Score
-	:members:
-	:undoc-members:
-
-.. autoclass:: pytorch_widedeep.metrics.R2Score
-	:members:
-	:undoc-members:
diff --git a/docs/model_components.rst b/docs/model_components.rst
deleted file mode 100644
index 00a3a1ba..00000000
--- a/docs/model_components.rst
+++ /dev/null
@@ -1,71 +0,0 @@
-The ``models`` module
-======================
-
-This module contains the models that can be used as the four main components
-that will comprise a Wide and Deep model (``wide``, ``deeptabular``,
-``deeptext``, ``deepimage``), as well as the ``WideDeep`` "constructor"
-class. Note that each of the four components can be used independently.
-
-.. autoclass:: pytorch_widedeep.models.tabular.linear.wide.Wide
-    :exclude-members: forward
-    :members:
-
-.. autoclass:: pytorch_widedeep.models.tabular.mlp.tab_mlp.TabMlp
-    :exclude-members: forward
-    :members:
-
-.. autoclass:: pytorch_widedeep.models.tabular.mlp.context_attention_mlp.ContextAttentionMLP
-    :exclude-members: forward
-    :members:
-
-.. autoclass:: pytorch_widedeep.models.tabular.mlp.self_attention_mlp.SelfAttentionMLP
-    :exclude-members: forward
-    :members:
-
-.. autoclass:: pytorch_widedeep.models.tabular.resnet.tab_resnet.TabResnet
-    :exclude-members: forward
-    :members:
-
-.. autoclass:: pytorch_widedeep.models.tabular.tabnet.tab_net.TabNet
-    :exclude-members: forward
-    :members:
-
-.. autoclass:: pytorch_widedeep.models.tabular.transformers.tab_transformer.TabTransformer
-    :exclude-members: forward
-    :members:
-
-.. autoclass:: pytorch_widedeep.models.tabular.transformers.saint.SAINT
-    :exclude-members: forward
-    :members:
-
-.. autoclass:: pytorch_widedeep.models.tabular.transformers.ft_transformer.FTTransformer
-    :exclude-members: forward
-    :members:
-
-.. autoclass:: pytorch_widedeep.models.tabular.transformers.tab_perceiver.TabPerceiver
-    :exclude-members: forward
-    :members:
-
-.. autoclass:: pytorch_widedeep.models.tabular.transformers.tab_fastformer.TabFastFormer
-    :exclude-members: forward
-    :members:
-
-.. autoclass:: pytorch_widedeep.models.text.attentive_rnn.BasicRNN
-    :exclude-members: forward
-    :members:
-
-.. autoclass:: pytorch_widedeep.models.text.attentive_rnn.AttentiveRNN
-    :exclude-members: forward
-    :members:
-
-.. autoclass:: pytorch_widedeep.models.text.stacked_attentive_rnn.StackedAttentiveRNN
-    :exclude-members: forward
-    :members:
-
-.. autoclass:: pytorch_widedeep.models.image.vision.Vision
-    :exclude-members: forward
-    :members:
-
-.. autoclass:: pytorch_widedeep.models.wide_deep.WideDeep
-    :exclude-members: forward
-    :members:
diff --git a/docs/preprocessing.rst b/docs/preprocessing.rst
deleted file mode 100644
index a068db8c..00000000
--- a/docs/preprocessing.rst
+++ /dev/null
@@ -1,22 +0,0 @@
-The ``preprocessing`` module
-============================
-
-This module contains the classes that are used to prepare the data before
-being passed to the models. There is one Preprocessor per data mode or
-model component: ``wide``, ``deeptabular``, ``deepimage`` and ``deeptext``.
-
-.. autoclass:: pytorch_widedeep.preprocessing.WidePreprocessor
-	:members:
-	:undoc-members:
-
-.. autoclass:: pytorch_widedeep.preprocessing.TabPreprocessor
-	:members:
-	:undoc-members:
-
-.. autoclass:: pytorch_widedeep.preprocessing.TextPreprocessor
-	:members:
-	:undoc-members:
-
-.. autoclass:: pytorch_widedeep.preprocessing.ImagePreprocessor
-	:members:
-	:undoc-members:
diff --git a/docs/quick_start.rst b/docs/quick_start.rst
deleted file mode 100644
index e21d618e..00000000
--- a/docs/quick_start.rst
+++ /dev/null
@@ -1,134 +0,0 @@
-Quick Start
-***********
-
-This is an example of a binary classification with the `adult census
-<https://www.kaggle.com/wenruliu/adult-income-dataset?select=adult.csv>`__
-dataset using a combination of a wide and deep model (in this case a so called
-``deeptabular`` model) with defaults settings.
-
-
-Read and split the dataset
---------------------------
-
-.. code-block:: python
-
-    import pandas as pd
-    import numpy as np
-    from sklearn.model_selection import train_test_split
-    from pytorch_widedeep.datasets import load_adult
-
-    df = load_adult(as_frame=True)
-    df["income_label"] = (df["income"].apply(lambda x: ">50K" in x)).astype(int)
-    df.drop("income", axis=1, inplace=True)
-    df_train, df_test = train_test_split(df, test_size=0.2, stratify=df.income_label)
-
-
-
-Prepare the wide and deep columns
----------------------------------
-
-.. code-block:: python
-
-    from pytorch_widedeep import Trainer
-    from pytorch_widedeep.preprocessing import WidePreprocessor, TabPreprocessor
-    from pytorch_widedeep.models import Wide, TabMlp, WideDeep
-    from pytorch_widedeep.metrics import Accuracy
-
-    # Define the 'column set up'
-    wide_cols = [
-        "education",
-        "relationship",
-        "workclass",
-        "occupation",
-        "native-country",
-        "gender",
-    ]
-    crossed_cols = [("education", "occupation"), ("native-country", "occupation")]
-
-    cat_embed_cols = [
-        "workclass",
-        "education",
-        "marital-status",
-        "occupation",
-        "relationship",
-        "race",
-        "gender",
-        "capital-gain",
-        "capital-loss",
-        "native-country",
-    ]
-    continuous_cols = ["age", "hours-per-week"]
-    target = "income_label"
-    target = df_train[target].values
-
-Preprocessing and model components definition
----------------------------------------------
-
-.. code-block:: python
-
-    wide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols)
-    X_wide = wide_preprocessor.fit_transform(df_train)
-
-    tab_preprocessor = TabPreprocessor(
-        cat_embed_cols=cat_embed_cols, continuous_cols=continuous_cols  # type: ignore[arg-type]
-    )
-    X_tab = tab_preprocessor.fit_transform(df_train)
-
-    # build the model
-    wide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1)
-    tab_mlp = TabMlp(
-        column_idx=tab_preprocessor.column_idx,
-        cat_embed_input=tab_preprocessor.cat_embed_input,
-        continuous_cols=continuous_cols,
-    )
-    model = WideDeep(wide=wide, deeptabular=tab_mlp)
-
-
-Fit and predict
--------------------------------
-
-.. code-block:: python
-
-    # train and validate
-    trainer = Trainer(model, objective="binary", metrics=[Accuracy])
-    trainer.fit(
-        X_wide=X_wide,
-        X_tab=X_tab,
-        target=target,
-        n_epochs=5,
-        batch_size=256,
-    )
-
-    # predict on test
-    X_wide_te = wide_preprocessor.transform(df_test)
-    X_tab_te = tab_preprocessor.transform(df_test)
-    preds = trainer.predict(X_wide=X_wide_te, X_tab=X_tab_te)
-
-
-Save and load
--------------------------------
-
-.. code-block:: python
-
-    # Option 1: this will also save training history and lr history if the
-    # LRHistory callback is used
-    trainer.save(path="model_weights", save_state_dict=True)
-
-    # Option 2: save as any other torch model
-    torch.save(model.state_dict(), "model_weights/wd_model.pt")
-
-    # From here in advance, Option 1 or 2 are the same. I assume the user has
-    # prepared the data and defined the new model components:
-    # 1. Build the model
-    model_new = WideDeep(wide=wide, deeptabular=tab_mlp)
-    model_new.load_state_dict(torch.load("model_weights/wd_model.pt"))
-
-    # 2. Instantiate the trainer
-    trainer_new = Trainer(model_new, objective="binary")
-
-    # 3. Either start the fit or directly predict
-    preds = trainer_new.predict(X_wide=X_wide, X_tab=X_tab)
-
-Of course, one can do **much more**. See the Examples folder in the repo, this
-documentation or the companion posts for a better understanding of the content
-of the package and its functionalities.
diff --git a/docs/requirements.txt b/docs/requirements.txt
deleted file mode 100644
index 03877a58..00000000
--- a/docs/requirements.txt
+++ /dev/null
@@ -1,20 +0,0 @@
-sphinx
-sphinx_rtd_theme
-recommonmark
-sphinx-markdown-tables
-sphinx-copybutton
-sphinx-autodoc-typehints
-pandas
-numpy
-scipy
-scikit-learn
-gensim
-spacy
-opencv-contrib-python
-imutils
-tqdm
-torch
-torchvision
-einops
-wrapt
-torchmetrics
diff --git a/docs/tab2vec.rst b/docs/tab2vec.rst
deleted file mode 100644
index 8daf940c..00000000
--- a/docs/tab2vec.rst
+++ /dev/null
@@ -1,7 +0,0 @@
-Tab2Vec
-=======
-
-.. autoclass:: pytorch_widedeep.tab2vec.Tab2Vec
-	:members:
-	:undoc-members:
-
diff --git a/docs/trainer.rst b/docs/trainer.rst
deleted file mode 100644
index 808e2882..00000000
--- a/docs/trainer.rst
+++ /dev/null
@@ -1,23 +0,0 @@
-Training multimodal Deep Learning Models
-========================================
-
-Here is the documentation for the ``Trainer`` class, that will do all the heavy lifting.
-
-Trainer is also available from ``pytorch-widedeep`` directly, for example, one could do:
-
-.. code-block:: python
-
-	from pytorch-widedeep.training import Trainer
-
-
-or also:
-
-.. code-block:: python
-
-	from pytorch-widedeep import Trainer
-
-
-.. autoclass:: pytorch_widedeep.training.Trainer
-	:exclude-members: forward
-	:members:
-	:undoc-members:
diff --git a/docs/utils/deeptabular_utils.rst b/docs/utils/deeptabular_utils.rst
deleted file mode 100644
index 25f8ba4e..00000000
--- a/docs/utils/deeptabular_utils.rst
+++ /dev/null
@@ -1,6 +0,0 @@
-deeptabular utils
-=================
-
-.. autoclass:: pytorch_widedeep.utils.deeptabular_utils.LabelEncoder
-	:members:
-	:undoc-members:
diff --git a/docs/utils/fastai_transforms.rst b/docs/utils/fastai_transforms.rst
deleted file mode 100644
index f18d60b0..00000000
--- a/docs/utils/fastai_transforms.rst
+++ /dev/null
@@ -1,19 +0,0 @@
-Fastai transforms
-=================
-
-I have directly copied and pasted part of the ``transforms.py`` module from
-the ``fastai`` library. The reason to do such a thing is because
-``pytorch_widedeep`` only needs the ``Tokenizer`` and the ``Vocab`` classes
-there. This way I avoid extra dependencies. Credit for all the code in the
-``fastai_transforms`` module in this ``pytorch-widedeep`` package goes to
-Jeremy Howard and the `fastai` team. I only include the documentation here for
-completion, but I strongly advise the user to read the ``fastai``
-`documentation <https://docs.fast.ai/>`_.
-
-.. autoclass:: pytorch_widedeep.utils.fastai_transforms.Tokenizer
-	:members:
-	:undoc-members:
-
-.. autoclass:: pytorch_widedeep.utils.fastai_transforms.Vocab
-	:members:
-	:undoc-members:
diff --git a/docs/utils/image_utils.rst b/docs/utils/image_utils.rst
deleted file mode 100644
index aedb1f61..00000000
--- a/docs/utils/image_utils.rst
+++ /dev/null
@@ -1,20 +0,0 @@
-Image utils
-===========
-:class:`SimplePreprocessor
-<pytorch_widedeep.utils.image_utils.SimplePreprocessor>` and
-:class:`AspectAwarePreprocessor
-<pytorch_widedeep.utils.image_utils.AspectAwarePreprocessor>` are directly
-taked from the great series of Books `Deep Learning for Computer Vision
-<https://www.pyimagesearch.com/deep-learning-computer-vision-python-book/>`_
-by `Adrian <https://www.pyimagesearch.com/>`_. Therefore, all credit for the
-code in the ``image_utils`` module goes to `Adrian Rosebrock
-<https://www.pyimagesearch.com/>`_.
-
-.. autoclass:: pytorch_widedeep.utils.image_utils.AspectAwarePreprocessor
-	:members:
-	:undoc-members:
-
-.. autoclass:: pytorch_widedeep.utils.image_utils.SimplePreprocessor
-	:members:
-	:undoc-members:
-
diff --git a/docs/utils/index.rst b/docs/utils/index.rst
deleted file mode 100644
index 72df9e03..00000000
--- a/docs/utils/index.rst
+++ /dev/null
@@ -1,23 +0,0 @@
-The ``utils`` module
-====================
-
-These are a series of utilities that might be useful for a number of
-preprocessing tasks, even not directly related to ``pytorch-widedeep``. All
-the classes and functions discussed here are available directly from the
-``utils`` module. For example, the ``LabelEncoder`` within the
-``deeptabular_utils`` submodule can be imported as:
-
-.. code-block:: python
-
-	from pytorch_widedeep.utils import LabelEncoder
-
-
-Objects
--------
-
-.. toctree::
-
-    deeptabular_utils
-    fastai_transforms
-    image_utils
-    text_utils
\ No newline at end of file
diff --git a/docs/utils/text_utils.rst b/docs/utils/text_utils.rst
deleted file mode 100644
index 1ad57478..00000000
--- a/docs/utils/text_utils.rst
+++ /dev/null
@@ -1,15 +0,0 @@
-Text utils
-=================
-Collection of helper function that facilitate processing text.
-
-.. autofunction:: pytorch_widedeep.utils.text_utils.simple_preprocess
-	:noindex:
-
-.. autofunction:: pytorch_widedeep.utils.text_utils.get_texts
-	:noindex:
-
-.. autofunction:: pytorch_widedeep.utils.text_utils.pad_sequences
-	:noindex:
-
-.. autofunction:: pytorch_widedeep.utils.text_utils.build_embeddings_matrix
-	:noindex:
\ No newline at end of file

From d2435c924e56b4c6c95a2c11bbe7e9886d59962d Mon Sep 17 00:00:00 2001
From: Javier <jrzaurin@gmail.com>
Date: Mon, 21 Oct 2024 12:43:55 +0100
Subject: [PATCH 17/24] Added versions to some packages in the requirements
 file since it was failing to build in Ubuntu 20.04

---
 requirements.txt | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/requirements.txt b/requirements.txt
index 98f5d5d0..3a8cbb3e 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -4,16 +4,16 @@ scipy>=1.7.3,<=1.12.0
 scikit-learn>=1.0.2
 gensim
 spacy
-opencv-contrib-python
+opencv-contrib-python>=4.10.0.84
 imutils
 tqdm
 torch >= 2.0.0
 torchvision >= 0.15.0
 einops
 wrapt
-torchmetrics
-pyarrow
-fastparquet>=0.8.1
-transformers
-sentence-transformers
-sentencepiece
\ No newline at end of file
+torchmetrics>=1.4.0
+pyarrow>=17.0.0
+fastparquet>=2024.5.0
+transformers>=4.44.0
+sentence-transformers>=3.1.0
+sentencepiece>=0.2.0
\ No newline at end of file

From 5babda9fe55f579166914d957a09c22c71606a15 Mon Sep 17 00:00:00 2001
From: Javier <jrzaurin@gmail.com>
Date: Thu, 24 Oct 2024 14:43:21 +0100
Subject: [PATCH 18/24] Added unused code to the datasets. Maybe used in the
 future

---
 .coveragerc                              |   5 +
 pytorch_widedeep/metrics.py              |  26 ++--
 pytorch_widedeep/training/_wd_dataset.py | 146 ++++++++++++++++++++++-
 3 files changed, 159 insertions(+), 18 deletions(-)

diff --git a/.coveragerc b/.coveragerc
index 5e85af5d..745be0ba 100644
--- a/.coveragerc
+++ b/.coveragerc
@@ -5,6 +5,11 @@ omit =
     pytorch_widedeep/bayesian_models/bayesian_nn/modules/*
 
 [report]
+exclude_lines =
+    pragma: no cover
+    def __repr__
+    class GroupWideDeepDataset
+    def group_collate_fn
 exclude_also =
     def __repr__
 omit =
diff --git a/pytorch_widedeep/metrics.py b/pytorch_widedeep/metrics.py
index 772f4c57..673cb95a 100644
--- a/pytorch_widedeep/metrics.py
+++ b/pytorch_widedeep/metrics.py
@@ -442,8 +442,8 @@ class NDCG_at_k(Metric):
     >>> from pytorch_widedeep.metrics import NDCG_at_k
     >>>
     >>> ndcg = NDCG_at_k(k=10)
-    >>> y_pred = torch.rand(100, 10)
-    >>> y_true = torch.randint(100, 10)
+    >>> y_pred = torch.rand(100, 5)
+    >>> y_true = torch.randint(2, (100,))
     >>> score = ndcg(y_pred, y_true)
     """
 
@@ -535,8 +535,8 @@ class BinaryNDCG_at_k(Metric):
     >>> from pytorch_widedeep.metrics import BinaryNDCG_at_k
     >>>
     >>> ndcg = BinaryNDCG_at_k(k=10)
-    >>> y_pred = torch.rand(100, 10)
-    >>> y_true = torch.randint(0, 2, (100, 10))
+    >>> y_pred = torch.randint(2, (100,))
+    >>> y_true = torch.randint(2, (100,))
     >>> score = ndcg(y_pred, y_true)
     """
 
@@ -622,8 +622,8 @@ class MAP_at_k(Metric):
     >>> from pytorch_widedeep.metrics import MAP_at_k
     >>>
     >>> map_at_k = MAP_at_k(k=10)
-    >>> y_pred = torch.rand(100, 10)
-    >>> y_true = torch.randint(0, 2, (100, 10))
+    >>> y_pred = torch.randint(2, (100,))
+    >>> y_true = torch.randint(2, (100,))
     >>> score = map_at_k(y_pred, y_true)
     """
 
@@ -688,9 +688,9 @@ class HitRatio_at_k(Metric):
     >>> from pytorch_widedeep.metrics import HitRatio_at_k
     >>>
     >>> hr_at_k = HitRatio_at_k(k=10)
-    >>> y_pred = torch.rand(100, 10)
-    >>> y_true = torch.randint(0, 2, (100, 10))
-    >>> score = hr_at_k(y_pred, y_true
+    >>> y_pred = torch.randint(2, (100,))
+    >>> y_true = torch.randint(2, (100,))
+    >>> score = hr_at_k(y_pred, y_true)
     """
 
     @alias(
@@ -747,8 +747,8 @@ class Precision_at_k(Metric):
     >>> from pytorch_widedeep.metrics import Precision_at_k
     >>>
     >>> prec_at_k = Precision_at_k(k=10)
-    >>> y_pred = torch.rand(100, 10)
-    >>> y_true = torch.randint(0, 2, (100, 10))
+    >>> y_pred = torch.randint(2, (100,))
+    >>> y_true = torch.randint(2, (100,))
     >>> score = prec_at_k(y_pred, y_true)
     """
 
@@ -805,8 +805,8 @@ class Recall_at_k(Metric):
     >>> from pytorch_widedeep.metrics import Recall_at_k
     >>>
     >>> rec_at_k = Recall_at_k(k=10)
-    >>> y_pred = torch.rand(100, 10)
-    >>> y_true = torch.randint(0, 2, (100, 10))
+    >>> y_pred = torch.randint(2, (100,))
+    >>> y_true = torch.randint(2, (100,))
     >>> score = rec_at_k(y_pred, y_true)
     """
 
diff --git a/pytorch_widedeep/training/_wd_dataset.py b/pytorch_widedeep/training/_wd_dataset.py
index f86ac84e..217d9f73 100644
--- a/pytorch_widedeep/training/_wd_dataset.py
+++ b/pytorch_widedeep/training/_wd_dataset.py
@@ -1,4 +1,4 @@
-from typing import List, Union
+from typing import Dict, List, Tuple, Union
 
 import numpy as np
 import torch
@@ -72,21 +72,20 @@ def __getitem__(self, idx: int):  # noqa: C901
         if self.X_img is not None:
             if isinstance(self.X_img, list):
                 x.deepimage = [
-                    self._prepare_images(self.X_img[i], idx)
+                    self._prepare_images(self.X_img[i][idx])
                     for i in range(len(self.X_img))
                 ]
             else:
-                x.deepimage = self._prepare_images(self.X_img, idx)
+                x.deepimage = self._prepare_images(self.X_img[idx])
         if self.Y is None:
             return x
         else:
             y = self.Y[idx]
             return x, y
 
-    def _prepare_images(self, imgs: np.ndarray, idx: int):
+    def _prepare_images(self, xdi: np.ndarray):
         # if an image dataset is used, make sure is in the right format to
         # be ingested by the conv layers
-        xdi = imgs[idx]
         # if int must be uint8
         if "int" in str(xdi.dtype) and "uint8" != str(xdi.dtype):
             xdi = xdi.astype("uint8")
@@ -129,3 +128,140 @@ def __len__(self):
                 return len(self.X_img[0])
             else:
                 return len(self.X_img)
+
+
+# This code is currently not used in the library. It will eventually be used
+# if I decide to support datasets for recommendation system with a varying
+# number of items per user. Allowing this in the current implementation
+# brings a series of issues that as of right now I prefer not to address.
+class GroupWideDeepDataset(Dataset):
+    def __init__(
+        self,
+        group_sizes: np.ndarray,
+        X_wide: Optional[np.ndarray] = None,
+        X_tab: Optional[Union[np.ndarray, List[np.ndarray]]] = None,
+        X_text: Optional[Union[np.ndarray, List[np.ndarray]]] = None,
+        X_img: Optional[Union[np.ndarray, List[np.ndarray]]] = None,
+        target: Optional[np.ndarray] = None,
+        transforms: Optional[Union[Transforms, Compose]] = None,
+    ):
+        super(GroupWideDeepDataset, self).__init__()
+        self.group_sizes = group_sizes
+        self.group_cumsum = np.cumsum([0] + self.group_sizes)
+        self.group_cumsum = np.insert(self.group_cumsum, 0, 0)
+        self.num_groups = len(self.group_sizes)
+
+        self.X_wide = X_wide
+        self.X_tab = X_tab
+        self.X_text = X_text
+        self.X_img = X_img
+
+        self.transforms = transforms
+        if self.transforms:
+            if isinstance(self.transforms, Compose):
+                self.transforms_names = [
+                    tr.__class__.__name__ for tr in self.transforms.transforms
+                ]
+            else:
+                self.transforms_names = [self.transforms.__class__.__name__]
+        else:
+            self.transforms_names = []
+
+        self.Y = target
+
+    def __getitem__(self, idx: int):
+        print(idx)
+        start_idx = self.group_cumsum[idx]
+        end_idx = self.group_cumsum[idx + 1]
+
+        x = Bunch()
+        if self.X_wide is not None:
+            x.wide = self.X_wide[start_idx:end_idx]
+        if self.X_tab is not None:
+            if isinstance(self.X_tab, list):
+                x.deeptabular = [
+                    self.X_tab[i][start_idx:end_idx] for i in range(len(self.X_tab))
+                ]
+            else:
+                x.deeptabular = self.X_tab[start_idx:end_idx]
+        if self.X_text is not None:
+            if isinstance(self.X_text, list):
+                x.deeptext = [
+                    self.X_text[i][start_idx:end_idx] for i in range(len(self.X_text))
+                ]
+            else:
+                x.deeptext = self.X_text[start_idx:end_idx]
+        if self.X_img is not None:
+            if isinstance(self.X_img, list):
+                x.deepimage = [
+                    self._prepare_images(self.X_img[i][start_idx:end_idx])
+                    for i in range(len(self.X_img))
+                ]
+            else:
+                x.deepimage = self._prepare_images(self.X_img[start_idx:end_idx])
+
+        if self.Y is None:
+            return x
+        else:
+            y = self.Y[start_idx:end_idx]
+            return x, y
+
+    def _prepare_images(self, xdi: np.ndarray):
+        if "int" in str(xdi.dtype) and "uint8" != str(xdi.dtype):
+            xdi = xdi.astype("uint8")
+        if "float" in str(xdi.dtype) and "float32" != str(xdi.dtype):
+            xdi = xdi.astype("float32")
+        if not self.transforms or "ToTensor" not in self.transforms_names:
+            # first dim will be the size of the group, so ndim == 3 implies
+            # 2D images
+            if xdi.ndim == 3:
+                xdi = xdi[:, :, :, None]
+            # Group Size, Height, Width, Channel -> Group Size, Channel,
+            # Height, Width
+            xdi = xdi.transpose(0, 3, 1, 2)
+            if "int" in str(xdi.dtype):
+                xdi = (xdi / xdi.max()).astype("float32")
+        # if ToTensor() is included, simply apply transforms
+        if "ToTensor" in self.transforms_names:
+            xdi = self.transforms(xdi)
+        # else apply transforms on the result of calling torch.tensor on
+        # xdi after all the previous manipulation
+        elif self.transforms:
+            xdi = self.transforms(torch.tensor(xdi))
+        return xdi
+
+    def __len__(self):
+        return self.num_groups
+
+
+def group_collate_fn(batch: List[Tuple[Bunch, np.ndarray]]):
+
+    all_targets: List[np.ndarray] = []
+    all_features: Dict[str, List[np.ndarray] | List[List[np.ndarray]]] = {
+        "wide": [],
+        "deeptabular": [],
+        "deeptext": [],
+        "deepimage": [],
+    }
+    for features, targets in batch:
+        for key in all_features.keys():
+            if key in features:
+                all_features[key].append(features[key])
+        all_targets.append(targets)
+    all_features = {k: v for k, v in all_features.items() if v}
+
+    batch_features: Dict[str, torch.Tensor | List[torch.Tensor]] = {}
+
+    for key, value in all_features.items():
+        if isinstance(value[0], list):
+            value = [
+                np.concatenate([v[i] for v in value], axis=0)
+                for i in range(len(value[0]))
+            ]
+            batch_features[key] = [torch.from_numpy(v) for v in value]
+        else:
+            batch_features[key] = torch.from_numpy(np.concatenate(value, axis=0))
+
+    batch_targets = torch.from_numpy(np.concatenate(all_targets, axis=0))
+
+    return batch_features, batch_targets

From 722f00c336ea79ade29a24e38df46316d03854b5 Mon Sep 17 00:00:00 2001
From: Javier <jrzaurin@gmail.com>
Date: Tue, 29 Oct 2024 13:03:42 +0000
Subject: [PATCH 19/24] Added a few rec algos

---
 pytorch_widedeep/losses.py                  |   2 +-
 pytorch_widedeep/losses_multitarget.py      |   2 +-
 pytorch_widedeep/models/rec/_layers.py      |  81 --------
 pytorch_widedeep/models/rec/autoint.py      |  94 +++++++++
 pytorch_widedeep/models/rec/autoint_plus.py | 152 ++++++++++++++
 pytorch_widedeep/models/rec/dcn.py          |  99 +++++++++
 pytorch_widedeep/models/rec/dcnv2.py        | 217 ++++++++++++++++++++
 pytorch_widedeep/models/rec/dcnv3.py        |   0
 pytorch_widedeep/models/rec/deepfm.py       |   1 -
 pytorch_widedeep/models/rec/din.py          |  50 ++++-
 pytorch_widedeep/models/rec/gdcn.py         | 143 +++++++++++++
 pytorch_widedeep/models/rec/xdeepfm.py      |  32 ++-
 tests/test_rec/test_layers.py               |   7 +-
 13 files changed, 788 insertions(+), 92 deletions(-)
 delete mode 100644 pytorch_widedeep/models/rec/_layers.py
 create mode 100644 pytorch_widedeep/models/rec/autoint.py
 create mode 100644 pytorch_widedeep/models/rec/autoint_plus.py
 create mode 100644 pytorch_widedeep/models/rec/dcn.py
 create mode 100644 pytorch_widedeep/models/rec/dcnv2.py
 create mode 100644 pytorch_widedeep/models/rec/dcnv3.py
 create mode 100644 pytorch_widedeep/models/rec/gdcn.py

diff --git a/pytorch_widedeep/losses.py b/pytorch_widedeep/losses.py
index 5f5a5769..e5851df5 100644
--- a/pytorch_widedeep/losses.py
+++ b/pytorch_widedeep/losses.py
@@ -1,6 +1,6 @@
 import torch
-import torch.nn as nn
 import torch.nn.functional as F
+from torch import nn
 
 from pytorch_widedeep.wdtypes import List, Tuple, Union, Tensor, Literal, Optional
 
diff --git a/pytorch_widedeep/losses_multitarget.py b/pytorch_widedeep/losses_multitarget.py
index 5d8bed61..05b00333 100644
--- a/pytorch_widedeep/losses_multitarget.py
+++ b/pytorch_widedeep/losses_multitarget.py
@@ -1,6 +1,6 @@
 import torch
-import torch.nn as nn
 import torch.nn.functional as F
+from torch import nn
 
 from pytorch_widedeep.wdtypes import List, Tuple, Union, Tensor, Literal, Optional
 from pytorch_widedeep.utils.general_utils import alias
diff --git a/pytorch_widedeep/models/rec/_layers.py b/pytorch_widedeep/models/rec/_layers.py
deleted file mode 100644
index 32132ab3..00000000
--- a/pytorch_widedeep/models/rec/_layers.py
+++ /dev/null
@@ -1,81 +0,0 @@
-from typing import List, Literal, Optional
-
-import torch
-import torch.nn.functional as F
-from torch import Tensor, nn
-
-
-class Dice(nn.Module):
-    def __init__(self, input_dim: int):
-        super(Dice, self).__init__()
-        self.bn = nn.BatchNorm1d(input_dim, eps=1e-9)
-        self.alpha = nn.Parameter(torch.zeros(input_dim))
-
-    def forward(self, X: Tensor) -> Tensor:
-        # This implementation assumes X has n_dim = 3
-        x_p = self.bn(X.transpose(1, 2)).transpose(1, 2)
-        p = torch.sigmoid(x_p)
-        return p * X + (1 - p) * self.alpha * X
-
-
-class ActivationUnit(nn.Module):
-    def __init__(
-        self,
-        embed_dim: int,
-        activation: Literal["prelu", "dice"],
-        proj_dim: Optional[int] = None,
-    ):
-        super(ActivationUnit, self).__init__()
-        self.proj_dim = proj_dim if proj_dim is not None else embed_dim
-        self.linear_in = nn.Linear(embed_dim * 4, self.proj_dim)
-        if activation == "prelu":
-            self.activation: nn.PReLU | Dice = nn.PReLU()
-        elif activation == "dice":
-            self.activation = Dice(self.proj_dim)
-        self.linear_out = nn.Linear(self.proj_dim, 1)
-
-    def forward(self, item: Tensor, user_behavior: Tensor) -> Tensor:
-        # in this implementation:
-        # item: [batch_size, 1, embedding_dim]
-        # user_behavior: [batch_size, seq_len, embedding_dim]
-        item_expanded = item.expand(-1, user_behavior.size(1), -1)
-        attn_input = torch.cat(
-            [
-                item_expanded,
-                user_behavior,
-                item_expanded - user_behavior,
-                item_expanded * user_behavior,
-            ],
-            dim=-1,
-        )
-        attn_output = self.activation(self.linear_in(attn_input))
-        attn_output = self.linear_out(attn_output).squeeze(-1)
-        return F.softmax(attn_output, dim=1)
-
-
-class CompressedInteractionNetwork(nn.Module):
-    def __init__(self, num_cols: int, cin_layer_dims: List[int]):
-        super(CompressedInteractionNetwork, self).__init__()
-        self.num_cols = num_cols
-        self.cin_layer_dims = cin_layer_dims
-        self.cin_layers = nn.ModuleList()
-
-        prev_layer_dim = num_cols
-        for layer_dim in cin_layer_dims:
-            self.cin_layers.append(
-                nn.Conv1d(prev_layer_dim * num_cols, layer_dim, kernel_size=1)
-            )
-            prev_layer_dim = layer_dim
-
-    def forward(self, X: Tensor) -> Tensor:
-        batch_size, embed_dim = X.shape[0], X.shape[-1]
-        prev_x = X
-        cin_outs = []
-        for cin_layer in self.cin_layers:
-            x_i = torch.einsum("b m d, b h d  -> b m h d", X, prev_x)
-            x_i = x_i.reshape(batch_size, self.num_cols * prev_x.shape[1], embed_dim)
-            x_i = cin_layer(x_i)
-            cin_outs.append(x_i.sum(2))
-            prev_x = x_i
-
-        return torch.cat(cin_outs, dim=1)
diff --git a/pytorch_widedeep/models/rec/autoint.py b/pytorch_widedeep/models/rec/autoint.py
new file mode 100644
index 00000000..84caf558
--- /dev/null
+++ b/pytorch_widedeep/models/rec/autoint.py
@@ -0,0 +1,94 @@
+from typing import Dict, List, Tuple, Literal, Optional
+
+import torch
+from torch import nn
+
+from pytorch_widedeep.models.tabular._base_tabular_model import (
+    BaseTabularModelWithAttention,
+)
+
+
+class AutoInt(BaseTabularModelWithAttention):
+    def __init__(
+        self,
+        column_idx: Dict[str, int],
+        input_dim: int,
+        *,
+        num_heads: int = 4,
+        num_layers: int = 2,
+        reduction: Literal["mean", "cat"] = "mean",
+        cat_embed_input: Optional[List[Tuple[str, int]]],
+        cat_embed_dropout: Optional[float],
+        use_cat_bias: Optional[bool],
+        cat_embed_activation: Optional[str],
+        shared_embed: Optional[bool],
+        add_shared_embed: Optional[bool],
+        frac_shared_embed: Optional[float],
+        continuous_cols: Optional[List[str]],
+        cont_norm_layer: Optional[Literal["batchnorm", "layernorm"]],
+        embed_continuous_method: Optional[Literal["standard", "piecewise", "periodic"]],
+        cont_embed_dropout: Optional[float],
+        cont_embed_activation: Optional[str],
+        quantization_setup: Optional[Dict[str, List[float]]],
+        n_frequencies: Optional[int],
+        sigma: Optional[float],
+        share_last_layer: Optional[bool],
+        full_embed_dropout: Optional[bool],
+    ):
+        super(AutoInt, self).__init__(
+            column_idx=column_idx,
+            input_dim=input_dim,
+            cat_embed_input=cat_embed_input,
+            cat_embed_dropout=cat_embed_dropout,
+            use_cat_bias=use_cat_bias,
+            cat_embed_activation=cat_embed_activation,
+            shared_embed=shared_embed,
+            add_shared_embed=add_shared_embed,
+            frac_shared_embed=frac_shared_embed,
+            continuous_cols=continuous_cols,
+            cont_norm_layer=cont_norm_layer,
+            embed_continuous_method=embed_continuous_method,
+            cont_embed_dropout=cont_embed_dropout,
+            cont_embed_activation=cont_embed_activation,
+            quantization_setup=quantization_setup,
+            n_frequencies=n_frequencies,
+            sigma=sigma,
+            share_last_layer=share_last_layer,
+            full_embed_dropout=full_embed_dropout,
+        )
+
+        self.num_heads = num_heads
+        self.num_layers = num_layers
+        self.reduction = reduction
+
+        self.attention_layers = nn.ModuleList(
+            [
+                nn.MultiheadAttention(
+                    embed_dim=input_dim,
+                    num_heads=num_heads,
+                    batch_first=True,
+                )
+                for _ in range(num_layers)
+            ]
+        )
+
+    def forward(self, X: torch.Tensor) -> torch.Tensor:
+        x = self._get_embeddings(X)
+
+        for layer in self.attention_layers:
+            attn_output, _ = layer(x, x, x)
+            x = attn_output + x
+
+        if self.reduction == "mean":
+            out = x.mean(dim=1)
+        else:
+            out = x.view(x.size(0), -1)
+
+        return out
+
+    @property
+    def output_dim(self) -> int:
+        if self.reduction == "mean":
+            return self.input_dim
+        else:
+            return self.input_dim * len(self.column_idx)
diff --git a/pytorch_widedeep/models/rec/autoint_plus.py b/pytorch_widedeep/models/rec/autoint_plus.py
new file mode 100644
index 00000000..22110fd7
--- /dev/null
+++ b/pytorch_widedeep/models/rec/autoint_plus.py
@@ -0,0 +1,152 @@
+from typing import Dict, List, Tuple, Union, Literal, Optional
+
+import torch
+from torch import nn
+
+from pytorch_widedeep.models.tabular.mlp._layers import MLP
+from pytorch_widedeep.models.tabular._base_tabular_model import (
+    BaseTabularModelWithAttention,
+)
+
+
+class AutoIntPlus(BaseTabularModelWithAttention):
+    def __init__(
+        self,
+        column_idx: Dict[str, int],
+        input_dim: int,
+        *,
+        num_heads: int = 4,
+        num_layers: int = 2,
+        reduction: Literal["mean", "cat"] = "mean",
+        structure: Literal["stacked", "parallel"] = "parallel",
+        gated: bool = True,
+        cat_embed_input: Optional[List[Tuple[str, int]]],
+        cat_embed_dropout: Optional[float],
+        use_cat_bias: Optional[bool],
+        cat_embed_activation: Optional[str],
+        shared_embed: Optional[bool],
+        add_shared_embed: Optional[bool],
+        frac_shared_embed: Optional[float],
+        continuous_cols: Optional[List[str]],
+        cont_norm_layer: Optional[Literal["batchnorm", "layernorm"]],
+        embed_continuous_method: Optional[Literal["standard", "piecewise", "periodic"]],
+        cont_embed_dropout: Optional[float],
+        cont_embed_activation: Optional[str],
+        quantization_setup: Optional[Dict[str, List[float]]],
+        n_frequencies: Optional[int],
+        sigma: Optional[float],
+        share_last_layer: Optional[bool],
+        full_embed_dropout: Optional[bool],
+        mlp_hidden_dims: List[int] = [200, 100],
+        mlp_activation: str = "relu",
+        mlp_dropout: Union[float, List[float]] = 0.1,
+        mlp_batchnorm: bool = False,
+        mlp_batchnorm_last: bool = False,
+        mlp_linear_first: bool = True,
+    ):
+        super(AutoIntPlus, self).__init__(
+            column_idx=column_idx,
+            input_dim=input_dim,
+            cat_embed_input=cat_embed_input,
+            cat_embed_dropout=cat_embed_dropout,
+            use_cat_bias=use_cat_bias,
+            cat_embed_activation=cat_embed_activation,
+            shared_embed=shared_embed,
+            add_shared_embed=add_shared_embed,
+            frac_shared_embed=frac_shared_embed,
+            continuous_cols=continuous_cols,
+            cont_norm_layer=cont_norm_layer,
+            embed_continuous_method=embed_continuous_method,
+            cont_embed_dropout=cont_embed_dropout,
+            cont_embed_activation=cont_embed_activation,
+            quantization_setup=quantization_setup,
+            n_frequencies=n_frequencies,
+            sigma=sigma,
+            share_last_layer=share_last_layer,
+            full_embed_dropout=full_embed_dropout,
+        )
+
+        self.input_dim = input_dim
+        self.num_heads = num_heads
+        self.num_layers = num_layers
+        self.reduction = reduction
+        self.structure = structure
+        self.gated = gated
+
+        self.attention_layers = nn.ModuleList(
+            [
+                nn.MultiheadAttention(
+                    embed_dim=input_dim,
+                    num_heads=num_heads,
+                    batch_first=True,
+                )
+                for _ in range(num_layers)
+            ]
+        )
+
+        if self.structure == "parallel":
+            deep_network_inp_dim = input_dim * len(column_idx)
+            if self.gated:
+                self.gate = nn.Linear(input_dim + mlp_hidden_dims[-1], input_dim)
+        else:  # structure == "stacked"
+            if self.reduction == "mean":
+                deep_network_inp_dim = input_dim
+            else:
+                deep_network_inp_dim = input_dim * len(column_idx)
+
+        mlp_hidden_dims = [deep_network_inp_dim] + mlp_hidden_dims
+        self.deep_network = MLP(
+            mlp_hidden_dims,
+            mlp_activation,
+            mlp_dropout,
+            mlp_batchnorm,
+            mlp_batchnorm_last,
+            mlp_linear_first,
+        )
+
+    def forward(self, X: torch.Tensor) -> torch.Tensor:
+        x = self._get_embeddings(X)
+        attn_output = self._apply_attention_layers(x)
+        reduced_attn_output = self._reduce_attention_output(attn_output)
+        if self.structure == "parallel":
+            return self._parallel_output(reduced_attn_output, x)
+        else:  # structure == "stacked"
+            return self.deep_network(reduced_attn_output)
+
+    def _apply_attention_layers(self, x: torch.Tensor) -> torch.Tensor:
+        attn_output = x
+        for layer in self.attention_layers:
+            layer_out, _ = layer(attn_output, attn_output, attn_output)
+            attn_output = layer_out + attn_output
+        return attn_output
+
+    def _reduce_attention_output(self, attn_output: torch.Tensor) -> torch.Tensor:
+        return (
+            attn_output.mean(dim=1)
+            if self.reduction == "mean"
+            else attn_output.flatten(1)
+        )
+
+    def _parallel_output(
+        self, attn_output: torch.Tensor, x: torch.Tensor
+    ) -> torch.Tensor:
+        deep_output = self.deep_network(x)
+
+        if not self.gated:
+            return torch.cat([attn_output, deep_output], dim=1)
+
+        combined_features = torch.cat([attn_output, deep_output], dim=-1)
+        gate_weights = torch.sigmoid(self.gate(combined_features))
+        return gate_weights * attn_output + (1 - gate_weights) * deep_output
+
+    @property
+    def output_dim(self) -> int:
+        if self.structure == "parallel":
+            if self.gated:
+                return self.deep_network.output_dim
+            else:
+                return self.deep_network.output_dim + self.input_dim * len(
+                    self.column_idx
+                )
+        else:
+            return self.deep_network.output_dim
diff --git a/pytorch_widedeep/models/rec/dcn.py b/pytorch_widedeep/models/rec/dcn.py
new file mode 100644
index 00000000..372ccf71
--- /dev/null
+++ b/pytorch_widedeep/models/rec/dcn.py
@@ -0,0 +1,99 @@
+from typing import Dict, List, Tuple, Union, Literal, Optional
+
+import torch
+from torch import nn
+
+from pytorch_widedeep.models.tabular.mlp._layers import MLP
+from pytorch_widedeep.models.tabular._base_tabular_model import (
+    BaseTabularModelWithoutAttention,
+)
+
+
+class CrossNetwork(nn.Module):
+    def __init__(self, input_dim: int, num_layers: int):
+        super().__init__()
+        self.num_layers = num_layers
+        self.cross_layers = nn.ModuleList(
+            [nn.Linear(input_dim, input_dim, bias=True) for _ in range(num_layers)]
+        )
+
+    def forward(self, X: torch.Tensor) -> torch.Tensor:
+        x0 = X
+        xi = X
+        for layer in self.cross_layers:
+            xi = x0 * layer(xi) + xi
+        return xi
+
+
+class DeepCrossNetwork(BaseTabularModelWithoutAttention):
+    def __init__(
+        self,
+        column_idx: Dict[str, int],
+        *,
+        n_cross_layers: int = 3,
+        cat_embed_input: Optional[List[Tuple[str, int, int]]] = None,
+        cat_embed_dropout: Optional[float] = None,
+        use_cat_bias: Optional[bool] = None,
+        cat_embed_activation: Optional[str] = None,
+        continuous_cols: Optional[List[str]] = None,
+        cont_norm_layer: Optional[Literal["batchnorm", "layernorm"]] = None,
+        embed_continuous: Optional[bool] = None,
+        embed_continuous_method: Optional[
+            Literal["standard", "piecewise", "periodic"]
+        ] = None,
+        cont_embed_dim: Optional[int] = None,
+        cont_embed_dropout: Optional[float] = None,
+        cont_embed_activation: Optional[str] = None,
+        quantization_setup: Optional[Dict[str, List[float]]] = None,
+        n_frequencies: Optional[int] = None,
+        sigma: Optional[float] = None,
+        share_last_layer: Optional[bool] = None,
+        full_embed_dropout: Optional[bool] = None,
+        mlp_hidden_dims: List[int] = [200, 100],
+        mlp_activation: str = "relu",
+        mlp_dropout: Union[float, List[float]] = 0.1,
+        mlp_batchnorm: bool = False,
+        mlp_batchnorm_last: bool = False,
+        mlp_linear_first: bool = True,
+    ):
+        super(DeepCrossNetwork, self).__init__(
+            column_idx=column_idx,
+            cat_embed_input=cat_embed_input,
+            cat_embed_dropout=cat_embed_dropout,
+            use_cat_bias=use_cat_bias,
+            cat_embed_activation=cat_embed_activation,
+            continuous_cols=continuous_cols,
+            cont_norm_layer=cont_norm_layer,
+            embed_continuous=embed_continuous,
+            embed_continuous_method=embed_continuous_method,
+            cont_embed_dim=cont_embed_dim,
+            cont_embed_dropout=cont_embed_dropout,
+            cont_embed_activation=cont_embed_activation,
+            quantization_setup=quantization_setup,
+            n_frequencies=n_frequencies,
+            sigma=sigma,
+            share_last_layer=share_last_layer,
+            full_embed_dropout=full_embed_dropout,
+        )
+
+        embeddings_output_dim = self.cat_out_dim + self.cont_out_dim
+        mlp_hidden_dims = [embeddings_output_dim] + mlp_hidden_dims
+        self.deep_network = MLP(
+            mlp_hidden_dims,
+            mlp_activation,
+            mlp_dropout,
+            mlp_batchnorm,
+            mlp_batchnorm_last,
+            mlp_linear_first,
+        )
+        self.cross_network = CrossNetwork(embeddings_output_dim, n_cross_layers)
+
+    def forward(self, X: torch.Tensor) -> torch.Tensor:
+        x = self._get_embeddings(X)
+        cross_output = self.cross_network(x)
+        deep_output = self.deep_network(x)
+        return torch.cat([cross_output, deep_output], dim=1)
+
+    @property
+    def output_dim(self) -> int:
+        return self.mlp_hidden_dims[-1] + self.cat_out_dim + self.cont_out_dim
diff --git a/pytorch_widedeep/models/rec/dcnv2.py b/pytorch_widedeep/models/rec/dcnv2.py
new file mode 100644
index 00000000..e646320d
--- /dev/null
+++ b/pytorch_widedeep/models/rec/dcnv2.py
@@ -0,0 +1,217 @@
+from typing import Dict, List, Tuple, Union, Literal, Optional
+
+import torch
+from torch import nn
+
+from pytorch_widedeep.models.tabular.mlp._layers import MLP
+from pytorch_widedeep.models.tabular._base_tabular_model import (
+    BaseTabularModelWithoutAttention,
+)
+
+
+class CrossLayerV2(nn.Module):
+    """
+    Single Cross Layer implementing equation 4 from the paper
+    E_i(x_l) = x_0 ⊙ (U_l^i · g(C_l^i · g(V_l^iT x_l)) + b_l)
+    """
+
+    def __init__(
+        self,
+        input_dim: int,
+        low_rank: Optional[int] = None,
+        num_experts: int = 1,
+        expert_dropout: float = 0.0,
+    ):
+        super().__init__()
+        self.input_dim = input_dim
+        self.low_rank = low_rank
+        self.num_experts = num_experts
+        self.expert_activation = nn.ReLU()
+
+        if low_rank is None:
+            self.W = nn.Parameter(torch.empty(num_experts, input_dim, input_dim))
+        else:
+            self.U = nn.Parameter(torch.empty(num_experts, input_dim, low_rank))
+            self.C = nn.Parameter(torch.empty(num_experts, low_rank, low_rank))
+            self.V = nn.Parameter(torch.empty(num_experts, low_rank, input_dim))
+
+        self.bias = nn.Parameter(torch.zeros(input_dim))
+
+        # Expert gate if using multiple experts
+        if num_experts > 1:
+            self.expert_gate = nn.Linear(input_dim, num_experts)
+
+        self.dropout = nn.Dropout(expert_dropout)
+        self.reset_parameters()
+
+    def reset_parameters(self):
+        if self.low_rank is None:
+            nn.init.xavier_uniform_(self.W)
+        else:
+            nn.init.xavier_uniform_(self.U)
+            nn.init.xavier_uniform_(self.C)
+            nn.init.xavier_uniform_(self.V)
+
+    def get_expert_output(self, x: torch.Tensor) -> torch.Tensor:
+        if self.low_rank is None:
+            expert_output = torch.einsum("bi,eij,bj->bei", x, self.W, x)
+        else:
+            v_x = torch.einsum("eij,bj->bei", self.V, x)
+            v_x = self.expert_activation(v_x)
+
+            c_x = torch.einsum("eij,bej->bei", self.C, v_x)
+            c_x = self.expert_activation(c_x)
+
+            expert_output = torch.einsum("eij,bej->bei", self.U, c_x)
+
+        return self.dropout(expert_output)
+
+    def forward(self, x0: torch.Tensor, xi: torch.Tensor) -> torch.Tensor:
+        expert_output = self.get_expert_output(xi)
+
+        if self.num_experts > 1:
+            gate_values = torch.softmax(self.expert_gate(xi), dim=1)
+            gate_values = gate_values.unsqueeze(2)
+            combined_output = torch.sum(expert_output * gate_values, dim=1)
+        else:
+            combined_output = expert_output.squeeze(1)
+
+        return x0 * combined_output + self.bias + xi
+
+
+class CrossNetworkV2(nn.Module):
+    """
+    Cross Network made up of multiple cross layers
+    """
+
+    def __init__(
+        self,
+        input_dim: int,
+        num_layers: int,
+        low_rank: Optional[int] = None,
+        num_experts: int = 1,
+        expert_dropout: float = 0.0,
+    ):
+        super().__init__()
+        self.input_dim = input_dim
+        self.num_layers = num_layers
+
+        self.cross_layers = nn.ModuleList(
+            [
+                CrossLayerV2(
+                    input_dim=input_dim,
+                    low_rank=low_rank,
+                    num_experts=num_experts,
+                    expert_dropout=expert_dropout,
+                )
+                for _ in range(num_layers)
+            ]
+        )
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        x0 = x
+        xi = x
+
+        for layer in self.cross_layers:
+            xi = layer(x0, xi)
+
+        return xi
+
+
+class DeepCrossNetworkV2(BaseTabularModelWithoutAttention):
+    def __init__(
+        self,
+        column_idx: Dict[str, int],
+        *,
+        num_cross_layers: int = 3,
+        num_experts: int = 4,
+        low_rank: Optional[int] = None,
+        expert_dropout: float = 0.0,
+        structure: Literal["stacked", "parallel"] = "parallel",
+        cat_embed_input: Optional[List[Tuple[str, int, int]]] = None,
+        cat_embed_dropout: Optional[float] = None,
+        use_cat_bias: Optional[bool] = None,
+        cat_embed_activation: Optional[str] = None,
+        continuous_cols: Optional[List[str]] = None,
+        cont_norm_layer: Optional[Literal["batchnorm", "layernorm"]] = None,
+        embed_continuous: Optional[bool] = None,
+        embed_continuous_method: Optional[
+            Literal["standard", "piecewise", "periodic"]
+        ] = None,
+        cont_embed_dim: Optional[int] = None,
+        cont_embed_dropout: Optional[float] = None,
+        cont_embed_activation: Optional[str] = None,
+        quantization_setup: Optional[Dict[str, List[float]]] = None,
+        n_frequencies: Optional[int] = None,
+        sigma: Optional[float] = None,
+        share_last_layer: Optional[bool] = None,
+        full_embed_dropout: Optional[bool] = None,
+        mlp_hidden_dims: List[int] = [200, 100],
+        mlp_activation: str = "relu",
+        mlp_dropout: Union[float, List[float]] = 0.1,
+        mlp_batchnorm: bool = False,
+        mlp_batchnorm_last: bool = False,
+        mlp_linear_first: bool = True,
+    ):
+        super(DeepCrossNetworkV2, self).__init__(
+            column_idx=column_idx,
+            cat_embed_input=cat_embed_input,
+            cat_embed_dropout=cat_embed_dropout,
+            use_cat_bias=use_cat_bias,
+            cat_embed_activation=cat_embed_activation,
+            continuous_cols=continuous_cols,
+            cont_norm_layer=cont_norm_layer,
+            embed_continuous=embed_continuous,
+            embed_continuous_method=embed_continuous_method,
+            cont_embed_dim=cont_embed_dim,
+            cont_embed_dropout=cont_embed_dropout,
+            cont_embed_activation=cont_embed_activation,
+            quantization_setup=quantization_setup,
+            n_frequencies=n_frequencies,
+            sigma=sigma,
+            share_last_layer=share_last_layer,
+            full_embed_dropout=full_embed_dropout,
+        )
+
+        self.num_cross_layers = num_cross_layers
+        self.num_experts = num_experts
+        self.low_rank = low_rank
+        self.expert_dropout = expert_dropout
+        self.structure = structure
+
+        embeddings_output_dim = self.cat_out_dim + self.cont_out_dim
+        mlp_hidden_dims = [embeddings_output_dim] + mlp_hidden_dims
+        self.deep_network = MLP(
+            mlp_hidden_dims,
+            mlp_activation,
+            mlp_dropout,
+            mlp_batchnorm,
+            mlp_batchnorm_last,
+            mlp_linear_first,
+        )
+
+        self.cross_network = CrossNetworkV2(
+            input_dim=embeddings_output_dim,
+            num_layers=num_cross_layers,
+            low_rank=low_rank,
+            num_experts=num_experts,
+            expert_dropout=expert_dropout,
+        )
+
+    def forward(self, X: torch.Tensor) -> torch.Tensor:
+        x = self._get_embeddings(X)
+        cross_output = self.cross_network(x)
+
+        if self.structure == "stacked":
+            deep_output = self.deep_network(cross_output)
+            return deep_output
+        else:  # parallel
+            deep_output = self.deep_network(x)
+            return torch.cat([cross_output, deep_output], dim=1)
+
+    @property
+    def output_dim(self) -> int:
+        if self.structure == "stacked":
+            return self.mlp_hidden_dims[-1]
+        else:  # parallel
+            return self.mlp_hidden_dims[-1] + self.cat_out_dim + self.cont_out_dim
diff --git a/pytorch_widedeep/models/rec/dcnv3.py b/pytorch_widedeep/models/rec/dcnv3.py
new file mode 100644
index 00000000..e69de29b
diff --git a/pytorch_widedeep/models/rec/deepfm.py b/pytorch_widedeep/models/rec/deepfm.py
index 98801e0d..95f9896b 100644
--- a/pytorch_widedeep/models/rec/deepfm.py
+++ b/pytorch_widedeep/models/rec/deepfm.py
@@ -161,7 +161,6 @@ def __init__(
         cat_embed_activation: Optional[str] = None,
         continuous_cols: Optional[List[str]] = None,
         cont_norm_layer: Optional[Literal["batchnorm", "layernorm"]] = None,
-        embed_continuous: Optional[bool] = None,
         embed_continuous_method: Optional[
             Literal["piecewise", "periodic"]
         ] = "piecewise",
diff --git a/pytorch_widedeep/models/rec/din.py b/pytorch_widedeep/models/rec/din.py
index 3c42a975..94ed0911 100644
--- a/pytorch_widedeep/models/rec/din.py
+++ b/pytorch_widedeep/models/rec/din.py
@@ -1,9 +1,9 @@
 from typing import Any, Dict, List, Tuple, Literal, Optional
 
 import torch
+import torch.nn.functional as F
 from torch import Tensor, nn
 
-from pytorch_widedeep.models.rec._layers import ActivationUnit
 from pytorch_widedeep.models.tabular.mlp._layers import MLP
 from pytorch_widedeep.models._base_wd_model_component import BaseWDModelComponent
 from pytorch_widedeep.models.tabular._base_tabular_model import (
@@ -12,6 +12,54 @@
 )
 
 
+class Dice(nn.Module):
+    def __init__(self, input_dim: int):
+        super(Dice, self).__init__()
+        self.bn = nn.BatchNorm1d(input_dim, eps=1e-9)
+        self.alpha = nn.Parameter(torch.zeros(input_dim))
+
+    def forward(self, X: Tensor) -> Tensor:
+        # This implementation assumes X has n_dim = 3
+        x_p = self.bn(X.transpose(1, 2)).transpose(1, 2)
+        p = torch.sigmoid(x_p)
+        return p * X + (1 - p) * self.alpha * X
+
+
+class ActivationUnit(nn.Module):
+    def __init__(
+        self,
+        embed_dim: int,
+        activation: Literal["prelu", "dice"],
+        proj_dim: Optional[int] = None,
+    ):
+        super(ActivationUnit, self).__init__()
+        self.proj_dim = proj_dim if proj_dim is not None else embed_dim
+        self.linear_in = nn.Linear(embed_dim * 4, self.proj_dim)
+        if activation == "prelu":
+            self.activation: nn.PReLU | Dice = nn.PReLU()
+        elif activation == "dice":
+            self.activation = Dice(self.proj_dim)
+        self.linear_out = nn.Linear(self.proj_dim, 1)
+
+    def forward(self, item: Tensor, user_behavior: Tensor) -> Tensor:
+        # in this implementation:
+        # item: [batch_size, 1, embedding_dim]
+        # user_behavior: [batch_size, seq_len, embedding_dim]
+        item_expanded = item.expand(-1, user_behavior.size(1), -1)
+        attn_input = torch.cat(
+            [
+                item_expanded,
+                user_behavior,
+                item_expanded - user_behavior,
+                item_expanded * user_behavior,
+            ],
+            dim=-1,
+        )
+        attn_output = self.activation(self.linear_in(attn_input))
+        attn_output = self.linear_out(attn_output).squeeze(-1)
+        return F.softmax(attn_output, dim=1)
+
+
 class DeepInterestNetwork(BaseWDModelComponent):
     """
     Adaptation of the Deep Interest Network (DIN) for recommendation systems
diff --git a/pytorch_widedeep/models/rec/gdcn.py b/pytorch_widedeep/models/rec/gdcn.py
new file mode 100644
index 00000000..41c1c3f4
--- /dev/null
+++ b/pytorch_widedeep/models/rec/gdcn.py
@@ -0,0 +1,143 @@
+from typing import Dict, List, Tuple, Union, Literal, Optional
+
+import torch
+from torch import nn
+
+from pytorch_widedeep.models.tabular.mlp._layers import MLP
+from pytorch_widedeep.models.tabular._base_tabular_model import (
+    BaseTabularModelWithoutAttention,
+)
+
+
+class GatedCrossLayer(nn.Module):
+    """
+    Single Gated Cross Layer implementing:
+    c_(i+1) = c_0 ⊙ (W^c × c_i + b) ⊙ σ(W^g × c_i) + c_i
+    """
+
+    def __init__(self, input_dim: int):
+        super().__init__()
+        self.input_dim = input_dim
+        self.cross_weight = nn.Parameter(torch.empty(input_dim, input_dim))
+        self.gate_weight = nn.Parameter(torch.empty(input_dim, input_dim))
+        self.bias = nn.Parameter(torch.zeros(input_dim))
+        self.reset_parameters()
+
+    def reset_parameters(self):
+        """Initialize layer parameters"""
+        nn.init.xavier_uniform_(self.cross_weight)
+        nn.init.xavier_uniform_(self.gate_weight)
+
+    def forward(self, x0: torch.Tensor, xi: torch.Tensor) -> torch.Tensor:
+        cross_term = torch.einsum("bi,ij->bj", xi, self.cross_weight)
+        cross_term = cross_term + self.bias
+        cross_term = x0 * cross_term
+        gate_term = torch.einsum("bi,ij->bj", xi, self.gate_weight)
+        gate_term = torch.sigmoid(gate_term)
+        return cross_term * gate_term + xi
+
+
+class GatedCrossNetwork(nn.Module):
+    def __init__(self, input_dim: int, num_layers: int):
+        super().__init__()
+        self.num_layers = num_layers
+        self.input_dim = input_dim
+        self.layers = nn.ModuleList(
+            [GatedCrossLayer(input_dim) for _ in range(num_layers)]
+        )
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        x0 = x
+        xi = x
+
+        for layer in self.layers:
+            xi = layer(x0, xi)
+
+        return xi
+
+
+class GatedCrossDeepNetwork(BaseTabularModelWithoutAttention):
+    def __init__(
+        self,
+        column_idx: Dict[str, int],
+        *,
+        num_cross_layers: int = 3,
+        structure: Literal["stacked", "parallel"] = "parallel",
+        cat_embed_input: Optional[List[Tuple[str, int, int]]] = None,
+        cat_embed_dropout: Optional[float] = None,
+        use_cat_bias: Optional[bool] = None,
+        cat_embed_activation: Optional[str] = None,
+        continuous_cols: Optional[List[str]] = None,
+        cont_norm_layer: Optional[Literal["batchnorm", "layernorm"]] = None,
+        embed_continuous: Optional[bool] = None,
+        embed_continuous_method: Optional[
+            Literal["standard", "piecewise", "periodic"]
+        ] = None,
+        cont_embed_dim: Optional[int] = None,
+        cont_embed_dropout: Optional[float] = None,
+        cont_embed_activation: Optional[str] = None,
+        quantization_setup: Optional[Dict[str, List[float]]] = None,
+        n_frequencies: Optional[int] = None,
+        sigma: Optional[float] = None,
+        share_last_layer: Optional[bool] = None,
+        full_embed_dropout: Optional[bool] = None,
+        mlp_hidden_dims: List[int] = [200, 100],
+        mlp_activation: str = "relu",
+        mlp_dropout: Union[float, List[float]] = 0.1,
+        mlp_batchnorm: bool = False,
+        mlp_batchnorm_last: bool = False,
+        mlp_linear_first: bool = True,
+    ):
+        super(GatedCrossDeepNetwork, self).__init__(
+            column_idx=column_idx,
+            cat_embed_input=cat_embed_input,
+            cat_embed_dropout=cat_embed_dropout,
+            use_cat_bias=use_cat_bias,
+            cat_embed_activation=cat_embed_activation,
+            continuous_cols=continuous_cols,
+            cont_norm_layer=cont_norm_layer,
+            embed_continuous=embed_continuous,
+            embed_continuous_method=embed_continuous_method,
+            cont_embed_dim=cont_embed_dim,
+            cont_embed_dropout=cont_embed_dropout,
+            cont_embed_activation=cont_embed_activation,
+            quantization_setup=quantization_setup,
+            n_frequencies=n_frequencies,
+            sigma=sigma,
+            share_last_layer=share_last_layer,
+            full_embed_dropout=full_embed_dropout,
+        )
+
+        self.num_cross_layers = num_cross_layers
+        self.structure = structure
+
+        embeddings_output_dim = self.cat_out_dim + self.cont_out_dim
+        mlp_hidden_dims = [embeddings_output_dim] + mlp_hidden_dims
+        self.deep_network = MLP(
+            mlp_hidden_dims,
+            mlp_activation,
+            mlp_dropout,
+            mlp_batchnorm,
+            mlp_batchnorm_last,
+            mlp_linear_first,
+        )
+
+        self.cross_network = GatedCrossNetwork(embeddings_output_dim, num_cross_layers)
+
+    def forward(self, X: torch.Tensor) -> torch.Tensor:
+        x = self._get_embeddings(X)
+        cross_output = self.cross_network(x)
+
+        if self.structure == "stacked":
+            deep_output = self.deep_network(cross_output)
+            return deep_output
+        else:  # parallel
+            deep_output = self.deep_network(x)
+            return torch.cat([cross_output, deep_output], dim=1)
+
+    @property
+    def output_dim(self) -> int:
+        if self.structure == "stacked":
+            return self.mlp_hidden_dims[-1]
+        else:  # parallel
+            return self.mlp_hidden_dims[-1] + self.cat_out_dim + self.cont_out_dim
diff --git a/pytorch_widedeep/models/rec/xdeepfm.py b/pytorch_widedeep/models/rec/xdeepfm.py
index 3a89e796..8834e707 100644
--- a/pytorch_widedeep/models/rec/xdeepfm.py
+++ b/pytorch_widedeep/models/rec/xdeepfm.py
@@ -1,14 +1,42 @@
 from typing import Dict, List, Tuple, Literal, Optional
 
-from torch import Tensor
+import torch
+from torch import Tensor, nn
 
-from pytorch_widedeep.models.rec._layers import CompressedInteractionNetwork
 from pytorch_widedeep.models.tabular.mlp._layers import MLP
 from pytorch_widedeep.models.tabular._base_tabular_model import (
     BaseTabularModelWithAttention,
 )
 
 
+class CompressedInteractionNetwork(nn.Module):
+    def __init__(self, num_cols: int, cin_layer_dims: List[int]):
+        super(CompressedInteractionNetwork, self).__init__()
+        self.num_cols = num_cols
+        self.cin_layer_dims = cin_layer_dims
+        self.cin_layers = nn.ModuleList()
+
+        prev_layer_dim = num_cols
+        for layer_dim in cin_layer_dims:
+            self.cin_layers.append(
+                nn.Conv1d(prev_layer_dim * num_cols, layer_dim, kernel_size=1)
+            )
+            prev_layer_dim = layer_dim
+
+    def forward(self, X: Tensor) -> Tensor:
+        batch_size, embed_dim = X.shape[0], X.shape[-1]
+        prev_x = X
+        cin_outs = []
+        for cin_layer in self.cin_layers:
+            x_i = torch.einsum("b m d, b h d  -> b m h d", X, prev_x)
+            x_i = x_i.reshape(batch_size, self.num_cols * prev_x.shape[1], embed_dim)
+            x_i = cin_layer(x_i)
+            cin_outs.append(x_i.sum(2))
+            prev_x = x_i
+
+        return torch.cat(cin_outs, dim=1)
+
+
 class ExtremeDeepFactorizationMachine(BaseTabularModelWithAttention):
     """
     Adaptation of 'xDeepFM implementation: xDeepFM: Combining Explicit and
diff --git a/tests/test_rec/test_layers.py b/tests/test_rec/test_layers.py
index ab0884fb..1bf725a8 100644
--- a/tests/test_rec/test_layers.py
+++ b/tests/test_rec/test_layers.py
@@ -1,11 +1,8 @@
 import torch
 import pytest
 
-from pytorch_widedeep.models.rec._layers import (
-    Dice,
-    ActivationUnit,
-    CompressedInteractionNetwork,
-)
+from pytorch_widedeep.models.rec.din import Dice, ActivationUnit
+from pytorch_widedeep.models.rec.xdeepfm import CompressedInteractionNetwork
 
 
 @pytest.fixture

From d26af146b320951b26fc3f9424ebd335541a38bb Mon Sep 17 00:00:00 2001
From: Javier <jrzaurin@gmail.com>
Date: Thu, 31 Oct 2024 17:45:24 +0000
Subject: [PATCH 20/24] All test passed. Docs added. Docs Need to be reviewed

---
 pytorch_widedeep/models/rec/__init__.py       |   5 +
 pytorch_widedeep/models/rec/autoint.py        | 123 ++++++++--
 pytorch_widedeep/models/rec/autoint_plus.py   | 230 ++++++++++++++----
 .../models/rec/basic_transformer.py           |   2 +-
 pytorch_widedeep/models/rec/dcn.py            | 121 ++++++++-
 pytorch_widedeep/models/rec/dcnv2.py          | 150 ++++++++++--
 pytorch_widedeep/models/rec/dcnv3.py          |   0
 pytorch_widedeep/models/rec/gdcn.py           | 136 ++++++++++-
 requirements.txt                              |  12 +-
 tests/test_rec/test_autoint.py                | 105 ++++++++
 tests/test_rec/test_autoint_plus.py           | 173 +++++++++++++
 tests/test_rec/test_dcn.py                    |  96 ++++++++
 tests/test_rec/test_dcnv2.py                  | 156 ++++++++++++
 tests/test_rec/test_gdcn.py                   | 132 ++++++++++
 14 files changed, 1334 insertions(+), 107 deletions(-)
 delete mode 100644 pytorch_widedeep/models/rec/dcnv3.py
 create mode 100644 tests/test_rec/test_autoint.py
 create mode 100644 tests/test_rec/test_autoint_plus.py
 create mode 100644 tests/test_rec/test_dcn.py
 create mode 100644 tests/test_rec/test_dcnv2.py
 create mode 100644 tests/test_rec/test_gdcn.py

diff --git a/pytorch_widedeep/models/rec/__init__.py b/pytorch_widedeep/models/rec/__init__.py
index 53a92c83..3289d8f8 100644
--- a/pytorch_widedeep/models/rec/__init__.py
+++ b/pytorch_widedeep/models/rec/__init__.py
@@ -1,5 +1,10 @@
+from pytorch_widedeep.models.rec.dcn import DeepCrossNetwork
 from pytorch_widedeep.models.rec.din import DeepInterestNetwork
+from pytorch_widedeep.models.rec.gdcn import GatedDeepCrossNetwork
+from pytorch_widedeep.models.rec.dcnv2 import DeepCrossNetworkV2
 from pytorch_widedeep.models.rec.deepfm import DeepFactorizationMachine
+from pytorch_widedeep.models.rec.autoint import AutoInt
 from pytorch_widedeep.models.rec.deepffm import DeepFieldAwareFactorizationMachine
 from pytorch_widedeep.models.rec.xdeepfm import ExtremeDeepFactorizationMachine
+from pytorch_widedeep.models.rec.autoint_plus import AutoIntPlus
 from pytorch_widedeep.models.rec.basic_transformer import Transformer
diff --git a/pytorch_widedeep/models/rec/autoint.py b/pytorch_widedeep/models/rec/autoint.py
index 84caf558..95265c69 100644
--- a/pytorch_widedeep/models/rec/autoint.py
+++ b/pytorch_widedeep/models/rec/autoint.py
@@ -9,31 +9,110 @@
 
 
 class AutoInt(BaseTabularModelWithAttention):
+    r"""Defines an `AutoInt` model that can be used as the `deeptabular` component
+    of a Wide & Deep model or independently by itself.
+
+    This class implements the [AutoInt: Automatic Feature Interaction Learning via Self-Attentive
+    Neural Networks](https://arxiv.org/abs/1810.11921) architecture, which learns feature
+    interactions through multi-head self-attention networks.
+
+    Parameters
+    ----------
+    column_idx: Dict
+        Dict containing the index of the columns that will be passed through
+        the `AutoInt` model. Required to slice the tensors. e.g.
+        _{'education': 0, 'relationship': 1, 'workclass': 2, ...}_.
+    input_dim: int
+        Dimension of the input embeddings
+    num_heads: int, default = 4
+        Number of attention heads
+    num_layers: int, default = 2
+        Number of interacting layers (attention + residual)
+    reduction: str, default = "mean"
+        How to reduce the output of the attention layers. Options are:
+        _'mean'_: mean of attention outputs
+        _'cat'_: concatenation of attention outputs
+    cat_embed_input: List, Optional, default = None
+        List of Tuples with the column name, number of unique values and
+        embedding dimension. e.g. _[(education, 11, 32), ...]_
+    cat_embed_dropout: float, Optional, default = None
+        Categorical embeddings dropout. If `None`, it will default to 0.
+    use_cat_bias: bool, Optional, default = None,
+        Boolean indicating if bias will be used for the categorical embeddings
+    cat_embed_activation: Optional, str, default = None,
+        Activation function for the categorical embeddings
+    continuous_cols: List, Optional, default = None
+        List with the name of the numeric (aka continuous) columns
+    cont_norm_layer: str, Optional, default = None
+        Type of normalization layer applied to the continuous features.
+        Options are: _'layernorm'_ and _'batchnorm'_
+    embed_continuous_method: Optional, str, default = None,
+        Method to use to embed the continuous features. Options are:
+        _'standard'_, _'periodic'_ or _'piecewise'_
+    cont_embed_dropout: float, Optional, default = None,
+        Dropout for the continuous embeddings
+    cont_embed_activation: Optional, str, default = None,
+        Activation function for the continuous embeddings
+    quantization_setup: Dict[str, List[float]], Optional, default = None,
+        Setup for the piecewise embeddings quantization
+    n_frequencies: int, Optional, default = None,
+        Number of frequencies for periodic embeddings
+    sigma: float, Optional, default = None,
+        Sigma parameter for periodic embeddings
+    share_last_layer: bool, Optional, default = None,
+        Whether to share the last layer in periodic embeddings
+    full_embed_dropout: bool, Optional, default = None,
+        If True, drops the entire embedding for a column
+
+    Attributes
+    ----------
+    attention_layers: nn.ModuleList
+        List of multi-head attention layers
+
+    Examples
+    --------
+    >>> import torch
+    >>> from pytorch_widedeep.models.rec import AutoInt
+    >>> X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)
+    >>> colnames = ["a", "b", "c", "d", "e"]
+    >>> cat_embed_input = [(u, i, j) for u, i, j in zip(colnames[:4], [4] * 4, [8] * 4)]
+    >>> column_idx = {k: v for v, k in enumerate(colnames)}
+    >>> model = AutoInt(
+    ...     column_idx=column_idx,
+    ...     input_dim=32,
+    ...     cat_embed_input=cat_embed_input,
+    ...     continuous_cols=["e"],
+    ...     embed_continuous_method="standard",
+    ...     num_heads=4,
+    ...     num_layers=2
+    ... )
+    >>> out = model(X_tab)
+    """
+
     def __init__(
         self,
+        *,
         column_idx: Dict[str, int],
         input_dim: int,
-        *,
         num_heads: int = 4,
         num_layers: int = 2,
         reduction: Literal["mean", "cat"] = "mean",
-        cat_embed_input: Optional[List[Tuple[str, int]]],
-        cat_embed_dropout: Optional[float],
-        use_cat_bias: Optional[bool],
-        cat_embed_activation: Optional[str],
-        shared_embed: Optional[bool],
-        add_shared_embed: Optional[bool],
-        frac_shared_embed: Optional[float],
-        continuous_cols: Optional[List[str]],
-        cont_norm_layer: Optional[Literal["batchnorm", "layernorm"]],
-        embed_continuous_method: Optional[Literal["standard", "piecewise", "periodic"]],
-        cont_embed_dropout: Optional[float],
-        cont_embed_activation: Optional[str],
-        quantization_setup: Optional[Dict[str, List[float]]],
-        n_frequencies: Optional[int],
-        sigma: Optional[float],
-        share_last_layer: Optional[bool],
-        full_embed_dropout: Optional[bool],
+        cat_embed_input: Optional[List[Tuple[str, int]]] = None,
+        cat_embed_dropout: Optional[float] = None,
+        use_cat_bias: Optional[bool] = None,
+        cat_embed_activation: Optional[str] = None,
+        continuous_cols: Optional[List[str]] = None,
+        cont_norm_layer: Optional[Literal["batchnorm", "layernorm"]] = None,
+        embed_continuous_method: Optional[
+            Literal["standard", "piecewise", "periodic"]
+        ] = None,
+        cont_embed_dropout: Optional[float] = None,
+        cont_embed_activation: Optional[str] = None,
+        quantization_setup: Optional[Dict[str, List[float]]] = None,
+        n_frequencies: Optional[int] = None,
+        sigma: Optional[float] = None,
+        share_last_layer: Optional[bool] = None,
+        full_embed_dropout: Optional[bool] = None,
     ):
         super(AutoInt, self).__init__(
             column_idx=column_idx,
@@ -42,9 +121,9 @@ def __init__(
             cat_embed_dropout=cat_embed_dropout,
             use_cat_bias=use_cat_bias,
             cat_embed_activation=cat_embed_activation,
-            shared_embed=shared_embed,
-            add_shared_embed=add_shared_embed,
-            frac_shared_embed=frac_shared_embed,
+            shared_embed=None,
+            add_shared_embed=None,
+            frac_shared_embed=None,
             continuous_cols=continuous_cols,
             cont_norm_layer=cont_norm_layer,
             embed_continuous_method=embed_continuous_method,
@@ -82,7 +161,7 @@ def forward(self, X: torch.Tensor) -> torch.Tensor:
         if self.reduction == "mean":
             out = x.mean(dim=1)
         else:
-            out = x.view(x.size(0), -1)
+            out = x.reshape(x.size(0), -1)
 
         return out
 
diff --git a/pytorch_widedeep/models/rec/autoint_plus.py b/pytorch_widedeep/models/rec/autoint_plus.py
index 22110fd7..6e6d9588 100644
--- a/pytorch_widedeep/models/rec/autoint_plus.py
+++ b/pytorch_widedeep/models/rec/autoint_plus.py
@@ -10,34 +10,140 @@
 
 
 class AutoIntPlus(BaseTabularModelWithAttention):
+    r"""Defines an `AutoIntPlus` model that can be used as the `deeptabular` component
+    of a Wide & Deep model or independently by itself.
+
+    This class implements an enhanced version of the [AutoInt](https://arxiv.org/abs/1810.11921)
+    architecture, adding a parallel or stacked deep network and an optional gating mechanism
+    to control the contribution of the attention-based and MLP branches.
+
+    Parameters
+    ----------
+    column_idx: Dict
+        Dict containing the index of the columns that will be passed through
+        the model. Required to slice the tensors. e.g.
+        _{'education': 0, 'relationship': 1, 'workclass': 2, ...}_.
+    input_dim: int
+        Dimension of the input embeddings
+    num_heads: int, default = 4
+        Number of attention heads
+    num_layers: int, default = 2
+        Number of interacting layers (attention + residual)
+    reduction: str, default = "mean"
+        How to reduce the output of the attention layers. Options are:
+        _'mean'_: mean of attention outputs
+        _'cat'_: concatenation of attention outputs
+    structure: str, default = "parallel"
+        Structure of the model. Either _'parallel'_ or _'stacked'_. If _'parallel'_,
+        the output will be the concatenation of the attention and deep networks.
+        If _'stacked'_, the attention output will be fed into the deep network.
+    gated: bool, default = True
+        If True and structure is 'parallel', uses a gating mechanism to combine
+        the attention and deep networks. Note: requires reduction='mean'.
+    cat_embed_input: List, Optional, default = None
+        List of Tuples with the column name, number of unique values and
+        embedding dimension. e.g. _[(education, 11, 32), ...]_
+    cat_embed_dropout: float, Optional, default = None
+        Categorical embeddings dropout. If `None`, it will default to 0.
+    use_cat_bias: bool, Optional, default = None,
+        Boolean indicating if bias will be used for the categorical embeddings
+    cat_embed_activation: Optional, str, default = None,
+        Activation function for the categorical embeddings
+    continuous_cols: List, Optional, default = None
+        List with the name of the numeric (aka continuous) columns
+    cont_norm_layer: str, Optional, default = None
+        Type of normalization layer applied to the continuous features.
+        Options are: _'layernorm'_ and _'batchnorm'_
+    embed_continuous_method: Optional, str, default = None,
+        Method to use to embed the continuous features. Options are:
+        _'standard'_, _'periodic'_ or _'piecewise'_
+    cont_embed_dropout: float, Optional, default = None,
+        Dropout for the continuous embeddings
+    cont_embed_activation: Optional, str, default = None,
+        Activation function for the continuous embeddings
+    quantization_setup: Dict[str, List[float]], Optional, default = None,
+        Setup for the piecewise embeddings quantization
+    n_frequencies: int, Optional, default = None,
+        Number of frequencies for periodic embeddings
+    sigma: float, Optional, default = None,
+        Sigma parameter for periodic embeddings
+    share_last_layer: bool, Optional, default = None,
+        Whether to share the last layer in periodic embeddings
+    full_embed_dropout: bool, Optional, default = None,
+        If True, drops the entire embedding for a column
+    mlp_hidden_dims: List, default = [100, 100]
+        List with the number of neurons per dense layer in the deep network
+    mlp_activation: str, default = "relu"
+        Activation function for the dense layers
+    mlp_dropout: float or List, default = 0.1
+        Dropout between the dense layers
+    mlp_batchnorm: bool, default = False
+        If True, applies batch normalization in the dense layers
+    mlp_batchnorm_last: bool, default = False
+        If True, applies batch normalization in the last dense layer
+    mlp_linear_first: bool, default = True
+        If True, applies the order: [Linear -> Activation -> BatchNorm -> Dropout]
+        If False: [BatchNorm -> Dropout -> Linear -> Activation]
+
+    Attributes
+    ----------
+    attention_layers: nn.ModuleList
+        List of multi-head attention layers
+    deep_network: nn.Module
+        The deep network component (MLP)
+    gate: nn.Module, optional
+        The gating network (if gated=True)
+
+    Examples
+    --------
+    >>> import torch
+    >>> from pytorch_widedeep.models.rec import AutoIntPlus
+    >>> X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)
+    >>> colnames = ["a", "b", "c", "d", "e"]
+    >>> cat_embed_input = [(u, i, j) for u, i, j in zip(colnames[:4], [4] * 4, [8] * 4)]
+    >>> column_idx = {k: v for v, k in enumerate(colnames)}
+    >>> model = AutoIntPlus(
+    ...     column_idx=column_idx,
+    ...     input_dim=32,
+    ...     cat_embed_input=cat_embed_input,
+    ...     continuous_cols=["e"],
+    ...     embed_continuous_method="standard",
+    ...     num_heads=4,
+    ...     num_layers=2,
+    ...     structure="parallel",
+    ...     gated=True,
+    ...     mlp_hidden_dims=[64, 32]
+    ... )
+    >>> out = model(X_tab)
+    """
+
     def __init__(
         self,
+        *,
         column_idx: Dict[str, int],
         input_dim: int,
-        *,
         num_heads: int = 4,
         num_layers: int = 2,
         reduction: Literal["mean", "cat"] = "mean",
         structure: Literal["stacked", "parallel"] = "parallel",
         gated: bool = True,
-        cat_embed_input: Optional[List[Tuple[str, int]]],
-        cat_embed_dropout: Optional[float],
-        use_cat_bias: Optional[bool],
-        cat_embed_activation: Optional[str],
-        shared_embed: Optional[bool],
-        add_shared_embed: Optional[bool],
-        frac_shared_embed: Optional[float],
-        continuous_cols: Optional[List[str]],
-        cont_norm_layer: Optional[Literal["batchnorm", "layernorm"]],
-        embed_continuous_method: Optional[Literal["standard", "piecewise", "periodic"]],
-        cont_embed_dropout: Optional[float],
-        cont_embed_activation: Optional[str],
-        quantization_setup: Optional[Dict[str, List[float]]],
-        n_frequencies: Optional[int],
-        sigma: Optional[float],
-        share_last_layer: Optional[bool],
-        full_embed_dropout: Optional[bool],
-        mlp_hidden_dims: List[int] = [200, 100],
+        cat_embed_input: Optional[List[Tuple[str, int]]] = None,
+        cat_embed_dropout: Optional[float] = None,
+        use_cat_bias: Optional[bool] = None,
+        cat_embed_activation: Optional[str] = None,
+        continuous_cols: Optional[List[str]] = None,
+        cont_norm_layer: Optional[Literal["batchnorm", "layernorm"]] = None,
+        embed_continuous_method: Optional[
+            Literal["standard", "piecewise", "periodic"]
+        ] = None,
+        cont_embed_dropout: Optional[float] = None,
+        cont_embed_activation: Optional[str] = None,
+        quantization_setup: Optional[Dict[str, List[float]]] = None,
+        n_frequencies: Optional[int] = None,
+        sigma: Optional[float] = None,
+        share_last_layer: Optional[bool] = None,
+        full_embed_dropout: Optional[bool] = None,
+        mlp_hidden_dims: List[int] = [100, 100],
         mlp_activation: str = "relu",
         mlp_dropout: Union[float, List[float]] = 0.1,
         mlp_batchnorm: bool = False,
@@ -51,9 +157,9 @@ def __init__(
             cat_embed_dropout=cat_embed_dropout,
             use_cat_bias=use_cat_bias,
             cat_embed_activation=cat_embed_activation,
-            shared_embed=shared_embed,
-            add_shared_embed=add_shared_embed,
-            frac_shared_embed=frac_shared_embed,
+            shared_embed=None,
+            add_shared_embed=None,
+            frac_shared_embed=None,
             continuous_cols=continuous_cols,
             cont_norm_layer=cont_norm_layer,
             embed_continuous_method=embed_continuous_method,
@@ -73,35 +179,35 @@ def __init__(
         self.structure = structure
         self.gated = gated
 
+        self.mlp_hidden_dims = mlp_hidden_dims
+        self.mlp_activation = mlp_activation
+        self.mlp_dropout = mlp_dropout
+        self.mlp_batchnorm = mlp_batchnorm
+        self.mlp_batchnorm_last = mlp_batchnorm_last
+        self.mlp_linear_first = mlp_linear_first
+
         self.attention_layers = nn.ModuleList(
             [
                 nn.MultiheadAttention(
-                    embed_dim=input_dim,
-                    num_heads=num_heads,
+                    embed_dim=self.input_dim,
+                    num_heads=self.num_heads,
                     batch_first=True,
                 )
                 for _ in range(num_layers)
             ]
         )
 
-        if self.structure == "parallel":
-            deep_network_inp_dim = input_dim * len(column_idx)
-            if self.gated:
-                self.gate = nn.Linear(input_dim + mlp_hidden_dims[-1], input_dim)
-        else:  # structure == "stacked"
-            if self.reduction == "mean":
-                deep_network_inp_dim = input_dim
-            else:
-                deep_network_inp_dim = input_dim * len(column_idx)
+        if self.gated:
+            self.gate = self._build_gate()
 
-        mlp_hidden_dims = [deep_network_inp_dim] + mlp_hidden_dims
+        deep_network_inp_dim = self._get_deep_network_input_dim()
         self.deep_network = MLP(
-            mlp_hidden_dims,
-            mlp_activation,
-            mlp_dropout,
-            mlp_batchnorm,
-            mlp_batchnorm_last,
-            mlp_linear_first,
+            [deep_network_inp_dim] + self.mlp_hidden_dims,
+            self.mlp_activation,
+            self.mlp_dropout,
+            self.mlp_batchnorm,
+            self.mlp_batchnorm_last,
+            self.mlp_linear_first,
         )
 
     def forward(self, X: torch.Tensor) -> torch.Tensor:
@@ -113,6 +219,35 @@ def forward(self, X: torch.Tensor) -> torch.Tensor:
         else:  # structure == "stacked"
             return self.deep_network(reduced_attn_output)
 
+    def _get_deep_network_input_dim(self) -> int:
+        if self.structure == "parallel":
+            return self.input_dim * len(self.column_idx)
+        elif self.reduction == "mean":  # structure == "stacked"
+            return self.input_dim
+        else:
+            return self.input_dim * len(self.column_idx)
+
+    def _build_gate(self) -> nn.Linear:
+        self._setup_gating()
+        return nn.Linear(self.input_dim * 2, self.input_dim)
+
+    def _setup_gating(self):
+        if self.structure == "stacked":
+            raise ValueError(
+                "Gating is not supported for stacked structure. Set `gated=False`."
+            )
+        if self.reduction != "mean":
+            raise ValueError(
+                "When using a gated structure, the reduction must be 'mean'."
+            )
+        if self.mlp_hidden_dims[-1] != self.input_dim:
+            self.mlp_hidden_dims = self.mlp_hidden_dims + [self.input_dim]
+            UserWarning(
+                "When using a gated structure, the last hidden layer of "
+                "the MLP must have the same dimension as the input. "
+                "The last hidden layer has been set to the input dimension."
+            )
+
     def _apply_attention_layers(self, x: torch.Tensor) -> torch.Tensor:
         attn_output = x
         for layer in self.attention_layers:
@@ -130,7 +265,7 @@ def _reduce_attention_output(self, attn_output: torch.Tensor) -> torch.Tensor:
     def _parallel_output(
         self, attn_output: torch.Tensor, x: torch.Tensor
     ) -> torch.Tensor:
-        deep_output = self.deep_network(x)
+        deep_output = self.deep_network(x.reshape(x.size(0), -1))
 
         if not self.gated:
             return torch.cat([attn_output, deep_output], dim=1)
@@ -143,10 +278,13 @@ def _parallel_output(
     def output_dim(self) -> int:
         if self.structure == "parallel":
             if self.gated:
-                return self.deep_network.output_dim
+                return self.input_dim
             else:
-                return self.deep_network.output_dim + self.input_dim * len(
-                    self.column_idx
-                )
+                if self.reduction == "mean":
+                    return self.mlp_hidden_dims[-1] + self.input_dim
+                else:
+                    return self.mlp_hidden_dims[-1] + self.input_dim * len(
+                        self.column_idx
+                    )
         else:
-            return self.deep_network.output_dim
+            return self.mlp_hidden_dims[-1]
diff --git a/pytorch_widedeep/models/rec/basic_transformer.py b/pytorch_widedeep/models/rec/basic_transformer.py
index 1ea2c7fe..f32eb694 100644
--- a/pytorch_widedeep/models/rec/basic_transformer.py
+++ b/pytorch_widedeep/models/rec/basic_transformer.py
@@ -83,6 +83,7 @@ class Transformer(nn.Module):
     @alias("seq_length", ["max_length", "maxlen"])
     def __init__(
         self,
+        *,
         vocab_size: int,
         seq_length: int,
         input_dim: int,
@@ -96,7 +97,6 @@ def __init__(
         use_flash_attention: bool = False,
         padding_idx: int = 0,
         with_cls_token: bool = False,
-        *,  # from here on pos encoding args
         with_pos_encoding: bool = True,
         pos_encoding_dropout: float = 0.1,
         pos_encoder: Optional[nn.Module] = None,
diff --git a/pytorch_widedeep/models/rec/dcn.py b/pytorch_widedeep/models/rec/dcn.py
index 372ccf71..a985f106 100644
--- a/pytorch_widedeep/models/rec/dcn.py
+++ b/pytorch_widedeep/models/rec/dcn.py
@@ -26,10 +26,109 @@ def forward(self, X: torch.Tensor) -> torch.Tensor:
 
 
 class DeepCrossNetwork(BaseTabularModelWithoutAttention):
+    r"""Defines a `DeepCrossNetwork` model that can be used as the `deeptabular`
+    component of a Wide & Deep model or independently by itself.
+
+    This class implements the [Deep & Cross Network for Ad Click Predictions](https://arxiv.org/abs/1708.05123)
+    architecture, which automatically combines features to generate feature interactions
+    in an explicit fashion and at each layer.
+
+    The cross layer implements the following equation:
+
+    $$x_{l+1} = x_0 \odot (W_l x_l + b_l) + x_l$$
+
+    where:
+
+    * $\odot$ represents element-wise multiplication
+    * $x_l$, $x_{l+1}$ are the outputs from the $l^{th}$ and $(l+1)^{th}$ cross layers
+    * $W_l$, $b_l$ are the weight and bias parameters
+
+    Parameters
+    ----------
+    column_idx: Dict
+        Dict containing the index of the columns that will be passed through
+        the `DeepCrossNetwork` model. Required to slice the tensors. e.g.
+        _{'education': 0, 'relationship': 1, 'workclass': 2, ...}_.
+    n_cross_layers: int, default = 3
+        Number of cross layers in the cross network
+    cat_embed_input: List, Optional, default = None
+        List of Tuples with the column name, number of unique values and
+        embedding dimension. e.g. _[(education, 11, 32), ...]_
+    cat_embed_dropout: float, Optional, default = None
+        Categorical embeddings dropout. If `None`, it will default to 0.
+    use_cat_bias: bool, Optional, default = None,
+        Boolean indicating if bias will be used for the categorical embeddings
+    cat_embed_activation: Optional, str, default = None,
+        Activation function for the categorical embeddings
+    continuous_cols: List, Optional, default = None
+        List with the name of the numeric (aka continuous) columns
+    cont_norm_layer: str, Optional, default = None
+        Type of normalization layer applied to the continuous features.
+        Options are: _'layernorm'_ and _'batchnorm'_
+    embed_continuous: bool, Optional, default = None,
+        Boolean indicating if the continuous columns will be embedded
+    embed_continuous_method: Optional, str, default = None,
+        Method to use to embed the continuous features. Options are:
+        _'standard'_, _'periodic'_ or _'piecewise'_
+    cont_embed_dim: int, Optional, default = None,
+        Size of the continuous embeddings
+    cont_embed_dropout: float, Optional, default = None,
+        Dropout for the continuous embeddings
+    cont_embed_activation: Optional, str, default = None,
+        Activation function for the continuous embeddings
+    quantization_setup: Dict[str, List[float]], Optional, default = None,
+        Setup for the piecewise embeddings quantization
+    n_frequencies: int, Optional, default = None,
+        Number of frequencies for periodic embeddings
+    sigma: float, Optional, default = None,
+        Sigma parameter for periodic embeddings
+    share_last_layer: bool, Optional, default = None,
+        Whether to share the last layer in periodic embeddings
+    full_embed_dropout: bool, Optional, default = None,
+        If True, drops the entire embedding for a column
+    mlp_hidden_dims: List, default = [200, 100]
+        List with the number of neurons per dense layer in the deep network
+    mlp_activation: str, default = "relu"
+        Activation function for the dense layers
+    mlp_dropout: float or List, default = 0.1
+        Dropout between the dense layers
+    mlp_batchnorm: bool, default = False
+        If True, applies batch normalization in the dense layers
+    mlp_batchnorm_last: bool, default = False
+        If True, applies batch normalization in the last dense layer
+    mlp_linear_first: bool, default = True
+        If True, applies the order: [Linear -> Activation -> BatchNorm -> Dropout]
+        If False: [BatchNorm -> Dropout -> Linear -> Activation]
+
+    Attributes
+    ----------
+    cross_network: nn.Module
+        The cross network component
+    deep_network: nn.Module
+        The deep network component (MLP)
+
+    Examples
+    --------
+    >>> import torch
+    >>> from pytorch_widedeep.models.rec import DeepCrossNetwork
+    >>> X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)
+    >>> colnames = ["a", "b", "c", "d", "e"]
+    >>> cat_embed_input = [(u, i, j) for u, i, j in zip(colnames[:4], [4] * 4, [8] * 4)]
+    >>> column_idx = {k: v for v, k in enumerate(colnames)}
+    >>> model = DeepCrossNetwork(
+    ...     column_idx=column_idx,
+    ...     cat_embed_input=cat_embed_input,
+    ...     continuous_cols=["e"],
+    ...     n_cross_layers=2,
+    ...     mlp_hidden_dims=[16, 8]
+    ... )
+    >>> out = model(X_tab)
+    """
+
     def __init__(
         self,
-        column_idx: Dict[str, int],
         *,
+        column_idx: Dict[str, int],
         n_cross_layers: int = 3,
         cat_embed_input: Optional[List[Tuple[str, int, int]]] = None,
         cat_embed_dropout: Optional[float] = None,
@@ -76,15 +175,21 @@ def __init__(
             full_embed_dropout=full_embed_dropout,
         )
 
+        self.mlp_hidden_dims = mlp_hidden_dims
+        self.mlp_activation = mlp_activation
+        self.mlp_dropout = mlp_dropout
+        self.mlp_batchnorm = mlp_batchnorm
+        self.mlp_batchnorm_last = mlp_batchnorm_last
+        self.mlp_linear_first = mlp_linear_first
+
         embeddings_output_dim = self.cat_out_dim + self.cont_out_dim
-        mlp_hidden_dims = [embeddings_output_dim] + mlp_hidden_dims
         self.deep_network = MLP(
-            mlp_hidden_dims,
-            mlp_activation,
-            mlp_dropout,
-            mlp_batchnorm,
-            mlp_batchnorm_last,
-            mlp_linear_first,
+            [embeddings_output_dim] + self.mlp_hidden_dims,
+            self.mlp_activation,
+            self.mlp_dropout,
+            self.mlp_batchnorm,
+            self.mlp_batchnorm_last,
+            self.mlp_linear_first,
         )
         self.cross_network = CrossNetwork(embeddings_output_dim, n_cross_layers)
 
diff --git a/pytorch_widedeep/models/rec/dcnv2.py b/pytorch_widedeep/models/rec/dcnv2.py
index e646320d..cb82823c 100644
--- a/pytorch_widedeep/models/rec/dcnv2.py
+++ b/pytorch_widedeep/models/rec/dcnv2.py
@@ -17,6 +17,7 @@ class CrossLayerV2(nn.Module):
 
     def __init__(
         self,
+        *,
         input_dim: int,
         low_rank: Optional[int] = None,
         num_experts: int = 1,
@@ -119,13 +120,128 @@ def forward(self, x: torch.Tensor) -> torch.Tensor:
 
 
 class DeepCrossNetworkV2(BaseTabularModelWithoutAttention):
+    r"""Defines a `DeepCrossNetworkV2` model that can be used as the `deeptabular`
+    component of a Wide & Deep model or independently by itself.
+
+    This class implements the
+    [DCN V2: Improved Deep & Cross Network and Practical Lessons for Web-scale Learning to Rank Systems](https://arxiv.org/pdf/2008.13535)
+    architecture, which enhances the original DCN by introducing a more expressive cross
+    network that uses multiple experts and matrix decomposition techniques to improve
+    model capacity while maintaining computational efficiency.
+
+    The cross layer implements the following equation:
+
+    $$E_i(x_l) = x_0 \odot (U_l^i \cdot g(C_l^i \cdot g(V_l^iT x_l)) + b_l)$$
+
+    where:
+
+    * $\odot$ represents element-wise multiplication
+    * $U_l^i$, $C_l^i$, $V_l^i$ are the decomposed weight matrices for expert $i$ at layer $l$
+    * $g$ is the activation function (ReLU)
+    * $b_l$ is the bias term
+
+    Parameters
+    ----------
+    column_idx: Dict
+        Dict containing the index of the columns that will be passed through
+        the `DeepCrossNetworkV2` model. Required to slice the tensors. e.g.
+        _{'education': 0, 'relationship': 1, 'workclass': 2, ...}_.
+    num_cross_layers: int, default = 2
+        Number of cross layers in the cross network
+    low_rank: int, Optional, default = None
+        Rank of the weight matrix decomposition. If None, full-rank weights are used
+    num_experts: int, default = 2
+        Number of expert networks in mixture of experts
+    expert_dropout: float, default = 0.0
+        Dropout rate for expert outputs
+    structure: str, default = "parallel"
+        Structure of the model. Either _'parallel'_ or _'stacked'_. If _'parallel'_,
+        the output will be the concatenation of the cross network and deep network
+        outputs. If _'stacked'_, the cross network output will be fed into the deep
+        network.
+    cat_embed_input: List, Optional, default = None
+        List of Tuples with the column name, number of unique values and
+        embedding dimension. e.g. _[(education, 11, 32), ...]_
+    cat_embed_dropout: float, Optional, default = None
+        Categorical embeddings dropout. If `None`, it will default to 0.
+    use_cat_bias: bool, Optional, default = None,
+        Boolean indicating if bias will be used for the categorical embeddings
+    cat_embed_activation: Optional, str, default = None,
+        Activation function for the categorical embeddings
+    continuous_cols: List, Optional, default = None
+        List with the name of the numeric (aka continuous) columns
+    cont_norm_layer: str, Optional, default = None,
+        Type of normalization layer applied to the continuous features.
+        Options are: _'layernorm'_ and _'batchnorm'_
+    embed_continuous: bool, Optional, default = None,
+        Boolean indicating if the continuous columns will be embedded
+    embed_continuous_method: Optional, str, default = None,
+        Method to use to embed the continuous features. Options are:
+        _'standard'_, _'periodic'_ or _'piecewise'_
+    cont_embed_dim: int, Optional, default = None,
+        Size of the continuous embeddings
+    cont_embed_dropout: float, Optional, default = None,
+        Dropout for the continuous embeddings
+    cont_embed_activation: Optional, str, default = None,
+        Activation function for the continuous embeddings
+    quantization_setup: Dict[str, List[float]], Optional, default = None,
+        Setup for the piecewise embeddings quantization
+    n_frequencies: int, Optional, default = None,
+        Number of frequencies for periodic embeddings
+    sigma: float, Optional, default = None,
+        Sigma parameter for periodic embeddings
+    share_last_layer: bool, Optional, default = None,
+        Whether to share the last layer in periodic embeddings
+    full_embed_dropout: bool, Optional, default = None,
+        If True, drops the entire embedding for a column
+    mlp_hidden_dims: List, default = [200, 100]
+        List with the number of neurons per dense layer in the deep network
+    mlp_activation: str, default = "relu"
+        Activation function for the dense layers
+    mlp_dropout: float or List, default = 0.1
+        Dropout between the dense layers
+    mlp_batchnorm: bool, default = False
+        If True, applies batch normalization in the dense layers
+    mlp_batchnorm_last: bool, default = False
+        If True, applies batch normalization in the last dense layer
+    mlp_linear_first: bool, default = True
+        If True, applies the order: [Linear -> Activation -> BatchNorm -> Dropout]
+        If False: [BatchNorm -> Dropout -> Linear -> Activation]
+
+    Attributes
+    ----------
+    cross_network: nn.Module
+        The cross network component with mixture of experts
+    deep_network: nn.Module
+        The deep network component (MLP)
+
+    Examples
+    --------
+    >>> import torch
+    >>> from pytorch_widedeep.models.rec import DeepCrossNetworkV2
+    >>> X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)
+    >>> colnames = ["a", "b", "c", "d", "e"]
+    >>> cat_embed_input = [(u, i, j) for u, i, j in zip(colnames[:4], [4] * 4, [8] * 4)]
+    >>> column_idx = {k: v for v, k in enumerate(colnames)}
+    >>> model = DeepCrossNetworkV2(
+    ...     column_idx=column_idx,
+    ...     cat_embed_input=cat_embed_input,
+    ...     continuous_cols=["e"],
+    ...     num_cross_layers=2,
+    ...     low_rank=32,
+    ...     num_experts=4,
+    ...     mlp_hidden_dims=[16, 8]
+    ... )
+    >>> out = model(X_tab)
+    """
+
     def __init__(
         self,
-        column_idx: Dict[str, int],
         *,
-        num_cross_layers: int = 3,
-        num_experts: int = 4,
+        column_idx: Dict[str, int],
+        num_cross_layers: int = 2,
         low_rank: Optional[int] = None,
+        num_experts: int = 2,
         expert_dropout: float = 0.0,
         structure: Literal["stacked", "parallel"] = "parallel",
         cat_embed_input: Optional[List[Tuple[str, int, int]]] = None,
@@ -179,23 +295,29 @@ def __init__(
         self.expert_dropout = expert_dropout
         self.structure = structure
 
+        self.mlp_hidden_dims = mlp_hidden_dims
+        self.mlp_activation = mlp_activation
+        self.mlp_dropout = mlp_dropout
+        self.mlp_batchnorm = mlp_batchnorm
+        self.mlp_batchnorm_last = mlp_batchnorm_last
+        self.mlp_linear_first = mlp_linear_first
+
         embeddings_output_dim = self.cat_out_dim + self.cont_out_dim
-        mlp_hidden_dims = [embeddings_output_dim] + mlp_hidden_dims
         self.deep_network = MLP(
-            mlp_hidden_dims,
-            mlp_activation,
-            mlp_dropout,
-            mlp_batchnorm,
-            mlp_batchnorm_last,
-            mlp_linear_first,
+            [embeddings_output_dim] + self.mlp_hidden_dims,
+            self.mlp_activation,
+            self.mlp_dropout,
+            self.mlp_batchnorm,
+            self.mlp_batchnorm_last,
+            self.mlp_linear_first,
         )
 
         self.cross_network = CrossNetworkV2(
             input_dim=embeddings_output_dim,
-            num_layers=num_cross_layers,
-            low_rank=low_rank,
-            num_experts=num_experts,
-            expert_dropout=expert_dropout,
+            num_layers=self.num_cross_layers,
+            low_rank=self.low_rank,
+            num_experts=self.num_experts,
+            expert_dropout=self.expert_dropout,
         )
 
     def forward(self, X: torch.Tensor) -> torch.Tensor:
diff --git a/pytorch_widedeep/models/rec/dcnv3.py b/pytorch_widedeep/models/rec/dcnv3.py
deleted file mode 100644
index e69de29b..00000000
diff --git a/pytorch_widedeep/models/rec/gdcn.py b/pytorch_widedeep/models/rec/gdcn.py
index 41c1c3f4..10f0a9a7 100644
--- a/pytorch_widedeep/models/rec/gdcn.py
+++ b/pytorch_widedeep/models/rec/gdcn.py
@@ -56,11 +56,121 @@ def forward(self, x: torch.Tensor) -> torch.Tensor:
         return xi
 
 
-class GatedCrossDeepNetwork(BaseTabularModelWithoutAttention):
+class GatedDeepCrossNetwork(BaseTabularModelWithoutAttention):
+    r"""Defines a `GatedDeepCrossNetwork` model that can be used as the `deeptabular`
+    component of a Wide & Deep model or independently by itself.
+
+    This class implements the Gated Deep & Cross Network (GDCN) architecture
+    as described in the paper
+    [Towards Deeper, Lighter and Interpretable Cross Network for CTR Prediction](https://arxiv.org/pdf/2311.04635).
+    The GDCN enhances the original DCN by introducing a gating mechanism in
+    the cross network. The gating mechanism controls feature interactions by
+    learning which interactions are more important.
+
+    The cross layer implements the following equation:
+
+    $$c_{i+1} = c_0 \odot (W^c \times c_i + b) \odot \sigma(W^g \times c_i) + c_i$$
+
+    where:
+
+    * $\odot$ represents element-wise multiplication
+    * $W^c$ and $W^g$ are the cross and gate weight matrices respectively
+    * $\sigma$ is the sigmoid activation function
+
+    Parameters
+    ----------
+    column_idx: Dict
+        Dict containing the index of the columns that will be passed through
+        the `GatedDeepCrossNetwork` model. Required to slice the tensors. e.g.
+        _{'education': 0, 'relationship': 1, 'workclass': 2, ...}_.
+    num_cross_layers: int, default = 3
+        Number of cross layers in the cross network
+    structure: str, default = "parallel"
+        Structure of the model. Either _'parallel'_ or _'stacked'_. If _'parallel'_,
+        the output will be the concatenation of the cross network and deep network
+        outputs. If _'stacked'_, the cross network output will be fed into the deep
+        network.
+    cat_embed_input: List, Optional, default = None
+        List of Tuples with the column name, number of unique values and
+        embedding dimension. e.g. _[(education, 11, 32), ...]_
+    cat_embed_dropout: float, Optional, default = None
+        Categorical embeddings dropout. If `None`, it will default to 0.
+    use_cat_bias: bool, Optional, default = None,
+        Boolean indicating if bias will be used for the categorical embeddings.
+        If `None`, it will default to 'False'.
+    cat_embed_activation: Optional, str, default = None,
+        Activation function for the categorical embeddings, if any. Currently
+        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported
+    continuous_cols: List, Optional, default = None
+        List with the name of the numeric (aka continuous) columns
+    cont_norm_layer: str, Optional, default = None
+        Type of normalization layer applied to the continuous features.
+        Options are: _'layernorm'_ and _'batchnorm'_. if `None`, no
+        normalization layer will be used.
+    embed_continuous: bool, Optional, default = None,
+        Boolean indicating if the continuous columns will be embedded
+    embed_continuous_method: Optional, str, default = None,
+        Method to use to embed the continuous features. Options are:
+        _'standard'_, _'periodic'_ or _'piecewise'_.
+    cont_embed_dim: int, Optional, default = None,
+        Size of the continuous embeddings
+    cont_embed_dropout: float, Optional, default = None,
+        Dropout for the continuous embeddings
+    cont_embed_activation: Optional, str, default = None,
+        Activation function for the continuous embeddings
+    quantization_setup: Dict[str, List[float]], Optional, default = None,
+        Setup for the piecewise embeddings quantization
+    n_frequencies: int, Optional, default = None,
+        Number of frequencies for periodic embeddings
+    sigma: float, Optional, default = None,
+        Sigma parameter for periodic embeddings
+    share_last_layer: bool, Optional, default = None,
+        Whether to share the last layer in periodic embeddings
+    full_embed_dropout: bool, Optional, default = None,
+        If True, drops the entire embedding for a column
+    mlp_hidden_dims: List, default = [200, 100]
+        List with the number of neurons per dense layer in the deep network
+    mlp_activation: str, default = "relu"
+        Activation function for the dense layers
+    mlp_dropout: float or List, default = 0.1
+        Dropout between the dense layers
+    mlp_batchnorm: bool, default = False
+        If True, applies batch normalization in the dense layers
+    mlp_batchnorm_last: bool, default = False
+        If True, applies batch normalization in the last dense layer
+    mlp_linear_first: bool, default = True
+        If True, applies the order: [Linear -> Activation -> BatchNorm -> Dropout]
+        If False: [BatchNorm -> Dropout -> Linear -> Activation]
+
+    Attributes
+    ----------
+    cross_network: nn.Module
+        The gated cross network component
+    deep_network: nn.Module
+        The deep network component (MLP)
+
+    Examples
+    --------
+    >>> import torch
+    >>> from pytorch_widedeep.models.rec import GatedDeepCrossNetwork
+    >>> X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)
+    >>> colnames = ["a", "b", "c", "d", "e"]
+    >>> cat_embed_input = [(u, i, j) for u, i, j in zip(colnames[:4], [4] * 4, [8] * 4)]
+    >>> column_idx = {k: v for v, k in enumerate(colnames)}
+    >>> model = GatedDeepCrossNetwork(
+    ...     column_idx=column_idx,
+    ...     cat_embed_input=cat_embed_input,
+    ...     continuous_cols=["e"],
+    ...     num_cross_layers=2,
+    ...     mlp_hidden_dims=[16, 8]
+    ... )
+    >>> out = model(X_tab)
+    """
+
     def __init__(
         self,
-        column_idx: Dict[str, int],
         *,
+        column_idx: Dict[str, int],
         num_cross_layers: int = 3,
         structure: Literal["stacked", "parallel"] = "parallel",
         cat_embed_input: Optional[List[Tuple[str, int, int]]] = None,
@@ -88,7 +198,7 @@ def __init__(
         mlp_batchnorm_last: bool = False,
         mlp_linear_first: bool = True,
     ):
-        super(GatedCrossDeepNetwork, self).__init__(
+        super(GatedDeepCrossNetwork, self).__init__(
             column_idx=column_idx,
             cat_embed_input=cat_embed_input,
             cat_embed_dropout=cat_embed_dropout,
@@ -111,15 +221,21 @@ def __init__(
         self.num_cross_layers = num_cross_layers
         self.structure = structure
 
+        self.mlp_hidden_dims = mlp_hidden_dims
+        self.mlp_activation = mlp_activation
+        self.mlp_dropout = mlp_dropout
+        self.mlp_batchnorm = mlp_batchnorm
+        self.mlp_batchnorm_last = mlp_batchnorm_last
+        self.mlp_linear_first = mlp_linear_first
+
         embeddings_output_dim = self.cat_out_dim + self.cont_out_dim
-        mlp_hidden_dims = [embeddings_output_dim] + mlp_hidden_dims
         self.deep_network = MLP(
-            mlp_hidden_dims,
-            mlp_activation,
-            mlp_dropout,
-            mlp_batchnorm,
-            mlp_batchnorm_last,
-            mlp_linear_first,
+            [embeddings_output_dim] + self.mlp_hidden_dims,
+            self.mlp_activation,
+            self.mlp_dropout,
+            self.mlp_batchnorm,
+            self.mlp_batchnorm_last,
+            self.mlp_linear_first,
         )
 
         self.cross_network = GatedCrossNetwork(embeddings_output_dim, num_cross_layers)
diff --git a/requirements.txt b/requirements.txt
index 3a8cbb3e..95257a66 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -4,16 +4,16 @@ scipy>=1.7.3,<=1.12.0
 scikit-learn>=1.0.2
 gensim
 spacy
-opencv-contrib-python>=4.10.0.84
+opencv-contrib-python>=4.9.0.80
 imutils
 tqdm
 torch >= 2.0.0
 torchvision >= 0.15.0
 einops
 wrapt
-torchmetrics>=1.4.0
-pyarrow>=17.0.0
-fastparquet>=2024.5.0
-transformers>=4.44.0
-sentence-transformers>=3.1.0
+torchmetrics>=1.3.1
+pyarrow>=15.0.0
+fastparquet>=2024.2.0
+transformers>=4.37.0
+sentence-transformers>=2.3.0
 sentencepiece>=0.2.0
\ No newline at end of file
diff --git a/tests/test_rec/test_autoint.py b/tests/test_rec/test_autoint.py
new file mode 100644
index 00000000..0294236b
--- /dev/null
+++ b/tests/test_rec/test_autoint.py
@@ -0,0 +1,105 @@
+import torch
+import pytest
+
+from pytorch_widedeep import Trainer
+from pytorch_widedeep.models import Wide, WideDeep
+from pytorch_widedeep.models.rec import AutoInt
+from pytorch_widedeep.preprocessing import TabPreprocessor, WidePreprocessor
+
+from .utils_test_rec import create_train_val_test_data
+
+train, valid, test = create_train_val_test_data()
+
+
+@pytest.mark.parametrize("reduction", ["mean", "cat"])
+def test_autoint_reduction(reduction, cat_embed_cols, continuous_cols):
+    tab_preprocessor = TabPreprocessor(
+        cat_embed_cols=cat_embed_cols,
+        continuous_cols=continuous_cols,
+        with_attention=True,
+    )
+
+    X_tab = tab_preprocessor.fit_transform(train)
+    X_tab_tnsr = torch.tensor(X_tab, dtype=torch.float32)
+
+    model = AutoInt(
+        column_idx=tab_preprocessor.column_idx,
+        input_dim=16,
+        cat_embed_input=tab_preprocessor.cat_embed_input,  # type: ignore[arg-type]
+        num_heads=2,
+        num_layers=2,
+        reduction=reduction,
+        continuous_cols=continuous_cols,
+        embed_continuous_method="standard",
+    )
+
+    res = model(X_tab_tnsr)
+
+    assert res.shape == (X_tab_tnsr.shape[0], model.output_dim)
+
+
+def test_autoint_with_wide_component(cat_embed_cols):
+    tab_preprocessor = TabPreprocessor(
+        cat_embed_cols=cat_embed_cols,
+        with_attention=True,
+    )
+
+    X_tab = tab_preprocessor.fit_transform(train)
+    X_tab_tnsr = torch.tensor(X_tab, dtype=torch.float32)
+
+    wide_preprocessor = WidePreprocessor(wide_cols=cat_embed_cols)
+    X_wide = wide_preprocessor.fit_transform(train)
+    X_wide_tnsr = torch.tensor(X_wide, dtype=torch.float32)
+
+    auto_int = AutoInt(
+        column_idx=tab_preprocessor.column_idx,
+        input_dim=16,
+        cat_embed_input=tab_preprocessor.cat_embed_input,  # type: ignore[arg-type]
+        num_heads=2,
+    )
+
+    linear = Wide(input_dim=X_wide.max())
+
+    model = WideDeep(wide=linear, deeptabular=auto_int)
+
+    X_inp = {"wide": X_wide_tnsr, "deeptabular": X_tab_tnsr}
+
+    out = model(X_inp)
+
+    assert out.shape[0] == X_tab_tnsr.shape[0]
+    assert out.shape[1] == 1
+
+
+def test_autoint_full_process(cat_embed_cols):
+
+    tab_preprocessor = TabPreprocessor(
+        cat_embed_cols=cat_embed_cols,
+        for_mf=True,
+    )
+
+    X_tab_tr = tab_preprocessor.fit_transform(train)
+    X_tab_val = tab_preprocessor.transform(valid)
+    X_tab_te = tab_preprocessor.transform(test)
+    y_tr = train["purchased"].values
+    y_val = valid["purchased"].values
+
+    auto_int = AutoInt(
+        column_idx=tab_preprocessor.column_idx,
+        input_dim=16,
+        cat_embed_input=tab_preprocessor.cat_embed_input,  # type: ignore[arg-type]
+        num_heads=2,
+    )
+    model = WideDeep(deeptabular=auto_int)
+
+    X_train = {"X_tab": X_tab_tr, "target": y_tr}
+    X_val = {"X_tab": X_tab_val, "target": y_val}
+    trainer = Trainer(model=model, objective="binary", verbose=0)
+    trainer.fit(X_train=X_train, X_val=X_val, n_epochs=1)
+    preds = trainer.predict(X_tab=X_tab_te)
+
+    assert preds.shape[0] == X_tab_te.shape[0]
+    assert (
+        trainer.history is not None
+        and "train_loss" in trainer.history
+        and "val_loss" in trainer.history
+    )
diff --git a/tests/test_rec/test_autoint_plus.py b/tests/test_rec/test_autoint_plus.py
new file mode 100644
index 00000000..3819d9d0
--- /dev/null
+++ b/tests/test_rec/test_autoint_plus.py
@@ -0,0 +1,173 @@
+import torch
+import pytest
+
+from pytorch_widedeep import Trainer
+from pytorch_widedeep.models import Wide, WideDeep
+from pytorch_widedeep.models.rec import AutoIntPlus
+from pytorch_widedeep.preprocessing import TabPreprocessor, WidePreprocessor
+
+from .utils_test_rec import create_train_val_test_data
+
+train, valid, test = create_train_val_test_data()
+
+
+@pytest.mark.parametrize(
+    "reduction, structure, gated",
+    [
+        ("cat", "stacked", False),
+        ("mean", "stacked", False),
+        ("cat", "parallel", False),
+        ("mean", "parallel", True),
+        ("mean", "parallel", False),
+    ],
+)
+def test_autoint_plus_reduction(
+    reduction, structure, gated, cat_embed_cols, continuous_cols
+):
+    tab_preprocessor = TabPreprocessor(
+        cat_embed_cols=cat_embed_cols,
+        continuous_cols=continuous_cols,
+        with_attention=True,
+    )
+
+    X_tab = tab_preprocessor.fit_transform(train)
+    X_tab_tnsr = torch.tensor(X_tab, dtype=torch.float32)
+
+    model = AutoIntPlus(
+        column_idx=tab_preprocessor.column_idx,
+        input_dim=16,
+        cat_embed_input=tab_preprocessor.cat_embed_input,
+        num_heads=2,
+        num_layers=2,
+        reduction=reduction,
+        structure=structure,
+        gated=gated,
+        continuous_cols=continuous_cols,
+        embed_continuous_method="standard",
+    )
+
+    res = model(X_tab_tnsr)
+
+    assert res.shape == (X_tab_tnsr.shape[0], model.output_dim)
+
+
+def test_autoint_plus_gating_validation_errors(cat_embed_cols, continuous_cols):
+    tab_preprocessor = TabPreprocessor(
+        cat_embed_cols=cat_embed_cols,
+        continuous_cols=continuous_cols,
+        with_attention=True,
+    )
+
+    _ = tab_preprocessor.fit_transform(train)
+
+    # Test 1: ValueError when using gated with stacked structure
+    with pytest.raises(
+        ValueError, match="Gating is not supported for stacked structure"
+    ):
+        AutoIntPlus(
+            column_idx=tab_preprocessor.column_idx,
+            input_dim=16,
+            cat_embed_input=tab_preprocessor.cat_embed_input,
+            structure="stacked",
+            gated=True,
+            continuous_cols=continuous_cols,
+            embed_continuous_method="standard",
+        )
+
+    # Test 2: ValueError when using gated with reduction="cat"
+    with pytest.raises(
+        ValueError, match="When using a gated structure, the reduction must be 'mean'"
+    ):
+        AutoIntPlus(
+            column_idx=tab_preprocessor.column_idx,
+            input_dim=16,
+            cat_embed_input=tab_preprocessor.cat_embed_input,
+            structure="parallel",
+            reduction="cat",
+            gated=True,
+            continuous_cols=continuous_cols,
+            embed_continuous_method="standard",
+        )
+
+
+def test_autoint_plus_with_wide_component(cat_embed_cols):
+    tab_preprocessor = TabPreprocessor(
+        cat_embed_cols=cat_embed_cols,
+        with_attention=True,
+    )
+
+    X_tab = tab_preprocessor.fit_transform(train)
+    X_tab_tnsr = torch.tensor(X_tab, dtype=torch.float32)
+
+    wide_preprocessor = WidePreprocessor(wide_cols=cat_embed_cols)
+    X_wide = wide_preprocessor.fit_transform(train)
+    X_wide_tnsr = torch.tensor(X_wide, dtype=torch.float32)
+
+    auto_int_plus = AutoIntPlus(
+        column_idx=tab_preprocessor.column_idx,
+        input_dim=16,
+        num_heads=2,
+        num_layers=2,
+        cat_embed_input=tab_preprocessor.cat_embed_input,
+        structure="parallel",
+        gated=True,
+    )
+
+    linear = Wide(input_dim=X_wide.max())
+
+    model = WideDeep(wide=linear, deeptabular=auto_int_plus)
+
+    X_inp = {"wide": X_wide_tnsr, "deeptabular": X_tab_tnsr}
+
+    out = model(X_inp)
+
+    assert out.shape[0] == X_tab_tnsr.shape[0]
+    assert out.shape[1] == 1
+
+
+@pytest.mark.parametrize(
+    "structure, gated, reduction",
+    [
+        ("parallel", True, "mean"),
+        ("stacked", False, "cat"),
+    ],
+)
+def test_autoint_plus_full_process(structure, gated, reduction, cat_embed_cols):
+    tab_preprocessor = TabPreprocessor(
+        cat_embed_cols=cat_embed_cols,
+        for_mf=True,
+    )
+
+    X_tab_tr = tab_preprocessor.fit_transform(train)
+    X_tab_val = tab_preprocessor.transform(valid)
+    X_tab_te = tab_preprocessor.transform(test)
+    y_tr = train["purchased"].values
+    y_val = valid["purchased"].values
+
+    auto_int_plus = AutoIntPlus(
+        column_idx=tab_preprocessor.column_idx,
+        input_dim=16,
+        cat_embed_input=tab_preprocessor.cat_embed_input,  # type: ignore[arg-type]
+        num_heads=2,
+        num_layers=2,
+        structure=structure,
+        gated=gated,
+        reduction=reduction,
+    )
+
+    model = WideDeep(deeptabular=auto_int_plus)
+
+    X_train = {"X_tab": X_tab_tr, "target": y_tr}
+    X_val = {"X_tab": X_tab_val, "target": y_val}
+
+    trainer = Trainer(model=model, objective="binary", verbose=0)
+    trainer.fit(X_train=X_train, X_val=X_val, n_epochs=1)
+
+    preds = trainer.predict(X_tab=X_tab_te)
+
+    assert preds.shape[0] == X_tab_te.shape[0]
+    assert (
+        trainer.history is not None
+        and "train_loss" in trainer.history
+        and "val_loss" in trainer.history
+    )
diff --git a/tests/test_rec/test_dcn.py b/tests/test_rec/test_dcn.py
new file mode 100644
index 00000000..c83ea432
--- /dev/null
+++ b/tests/test_rec/test_dcn.py
@@ -0,0 +1,96 @@
+import torch
+
+from pytorch_widedeep import Trainer
+from pytorch_widedeep.models import Wide, WideDeep
+from pytorch_widedeep.models.rec import DeepCrossNetwork
+from pytorch_widedeep.preprocessing import TabPreprocessor, WidePreprocessor
+
+from .utils_test_rec import create_train_val_test_data
+
+train, valid, test = create_train_val_test_data()
+
+
+def test_dcn_forward(cat_embed_cols, continuous_cols):
+    tab_preprocessor = TabPreprocessor(
+        cat_embed_cols=cat_embed_cols,
+        continuous_cols=continuous_cols,
+    )
+
+    X_tab = tab_preprocessor.fit_transform(train)
+    X_tab_tnsr = torch.tensor(X_tab, dtype=torch.float32)
+
+    model = DeepCrossNetwork(
+        column_idx=tab_preprocessor.column_idx,
+        cat_embed_input=tab_preprocessor.cat_embed_input,  # type: ignore[arg-type]
+        continuous_cols=continuous_cols,
+        n_cross_layers=2,
+        mlp_hidden_dims=[16, 8],
+    )
+
+    res = model(X_tab_tnsr)
+
+    assert res.shape == (X_tab_tnsr.shape[0], model.output_dim)
+
+
+def test_dcn_with_wide_component(cat_embed_cols):
+    tab_preprocessor = TabPreprocessor(
+        cat_embed_cols=cat_embed_cols,
+    )
+
+    X_tab = tab_preprocessor.fit_transform(train)
+    X_tab_tnsr = torch.tensor(X_tab, dtype=torch.float32)
+
+    wide_preprocessor = WidePreprocessor(wide_cols=cat_embed_cols)
+    X_wide = wide_preprocessor.fit_transform(train)
+    X_wide_tnsr = torch.tensor(X_wide, dtype=torch.float32)
+
+    dcn = DeepCrossNetwork(
+        column_idx=tab_preprocessor.column_idx,
+        cat_embed_input=tab_preprocessor.cat_embed_input,  # type: ignore[arg-type]
+        n_cross_layers=2,
+        mlp_hidden_dims=[16, 8],
+    )
+
+    linear = Wide(input_dim=X_wide.max())
+
+    model = WideDeep(wide=linear, deeptabular=dcn)
+
+    X_inp = {"wide": X_wide_tnsr, "deeptabular": X_tab_tnsr}
+
+    out = model(X_inp)
+
+    assert out.shape[0] == X_tab_tnsr.shape[0]
+    assert out.shape[1] == 1
+
+
+def test_dcn_full_process(cat_embed_cols):
+    tab_preprocessor = TabPreprocessor(
+        cat_embed_cols=cat_embed_cols,
+    )
+
+    X_tab_tr = tab_preprocessor.fit_transform(train)
+    X_tab_val = tab_preprocessor.transform(valid)
+    X_tab_te = tab_preprocessor.transform(test)
+    y_tr = train["purchased"].values
+    y_val = valid["purchased"].values
+
+    dcn = DeepCrossNetwork(
+        column_idx=tab_preprocessor.column_idx,
+        cat_embed_input=tab_preprocessor.cat_embed_input,  # type: ignore[arg-type]
+        n_cross_layers=2,
+        mlp_hidden_dims=[16, 8],
+    )
+    model = WideDeep(deeptabular=dcn)
+
+    X_train = {"X_tab": X_tab_tr, "target": y_tr}
+    X_val = {"X_tab": X_tab_val, "target": y_val}
+    trainer = Trainer(model=model, objective="binary", verbose=0)
+    trainer.fit(X_train=X_train, X_val=X_val, n_epochs=1)
+    preds = trainer.predict(X_tab=X_tab_te)
+
+    assert preds.shape[0] == X_tab_te.shape[0]
+    assert (
+        trainer.history is not None
+        and "train_loss" in trainer.history
+        and "val_loss" in trainer.history
+    )
diff --git a/tests/test_rec/test_dcnv2.py b/tests/test_rec/test_dcnv2.py
new file mode 100644
index 00000000..3122ef0b
--- /dev/null
+++ b/tests/test_rec/test_dcnv2.py
@@ -0,0 +1,156 @@
+import torch
+import pytest
+
+from pytorch_widedeep import Trainer
+from pytorch_widedeep.models import WideDeep
+from pytorch_widedeep.preprocessing import TabPreprocessor
+from pytorch_widedeep.models.rec.dcnv2 import (
+    CrossLayerV2,
+    CrossNetworkV2,
+    DeepCrossNetworkV2,
+)
+
+from .utils_test_rec import create_train_val_test_data
+
+train, valid, test = create_train_val_test_data()
+
+
+@pytest.mark.parametrize(
+    "input_dim,low_rank,num_experts,expert_dropout",
+    [
+        (16, None, 1, 0.0),  # Basic case without low-rank
+        (16, 8, 1, 0.0),  # With low-rank
+        (16, None, 4, 0.1),  # Multiple experts
+        (16, 8, 4, 0.1),  # Low-rank with multiple experts
+    ],
+)
+def test_cross_layer_v2(input_dim, low_rank, num_experts, expert_dropout):
+    batch_size = 32
+    x = torch.randn(batch_size, input_dim)
+
+    layer = CrossLayerV2(
+        input_dim=input_dim,
+        low_rank=low_rank,
+        num_experts=num_experts,
+        expert_dropout=expert_dropout,
+    )
+
+    # Test forward pass
+    output = layer(x, x)
+    assert output.shape == (batch_size, input_dim)
+
+    # Test expert output shape
+    expert_output = layer.get_expert_output(x)
+    assert expert_output.shape == (batch_size, num_experts, input_dim)
+
+    if num_experts > 1:
+        assert hasattr(layer, "expert_gate")
+        gate_output = layer.expert_gate(x)
+        assert gate_output.shape == (batch_size, num_experts)
+
+
+@pytest.mark.parametrize(
+    "input_dim,num_layers,low_rank,num_experts",
+    [
+        (16, 2, None, 1),  # Basic case
+        (16, 3, 8, 1),  # With low-rank
+        (16, 2, None, 4),  # Multiple experts
+        (16, 3, 8, 4),  # Low-rank with multiple experts
+    ],
+)
+def test_cross_network_v2(input_dim, num_layers, low_rank, num_experts):
+    batch_size = 32
+    x = torch.randn(batch_size, input_dim)
+
+    network = CrossNetworkV2(
+        input_dim=input_dim,
+        num_layers=num_layers,
+        low_rank=low_rank,
+        num_experts=num_experts,
+    )
+
+    # Test network structure
+    assert len(network.cross_layers) == num_layers
+
+    # Test forward pass
+    output = network(x)
+    assert output.shape == (batch_size, input_dim)
+
+
+@pytest.mark.parametrize(
+    "structure,num_experts,low_rank",
+    [
+        ("parallel", 1, None),  # Basic parallel
+        ("stacked", 1, None),  # Basic stacked
+        ("parallel", 2, 4),  # Parallel with experts and low-rank
+        ("stacked", 2, 4),  # Stacked with experts and low-rank
+    ],
+)
+def test_dcn_v2_structure(
+    structure, num_experts, low_rank, cat_embed_cols, continuous_cols
+):
+    tab_preprocessor = TabPreprocessor(
+        cat_embed_cols=cat_embed_cols,
+        continuous_cols=continuous_cols,
+    )
+
+    X_tab = tab_preprocessor.fit_transform(train)
+    X_tab_tnsr = torch.tensor(X_tab, dtype=torch.float32)
+
+    model = DeepCrossNetworkV2(
+        column_idx=tab_preprocessor.column_idx,
+        low_rank=low_rank,
+        num_experts=num_experts,
+        structure=structure,
+        cat_embed_input=tab_preprocessor.cat_embed_input,  # type: ignore[arg-type]
+        continuous_cols=continuous_cols,
+        mlp_hidden_dims=[16, 8],
+    )
+
+    out = model(X_tab_tnsr)
+    assert out.shape == (X_tab_tnsr.shape[0], model.output_dim)
+
+    # Test output dimension property
+    if structure == "stacked":
+        assert model.output_dim == model.mlp_hidden_dims[-1]
+    else:  # parallel
+        assert model.output_dim == (
+            model.mlp_hidden_dims[-1] + model.cat_out_dim + model.cont_out_dim
+        )
+
+
+def test_dcn_v2_full_process(cat_embed_cols, continuous_cols):
+    tab_preprocessor = TabPreprocessor(
+        cat_embed_cols=cat_embed_cols,
+        continuous_cols=continuous_cols,
+    )
+
+    X_tab_tr = tab_preprocessor.fit_transform(train)
+    X_tab_val = tab_preprocessor.transform(valid)
+    X_tab_te = tab_preprocessor.transform(test)
+    y_tr = train["purchased"].values
+    y_val = valid["purchased"].values
+
+    dcn = DeepCrossNetworkV2(
+        column_idx=tab_preprocessor.column_idx,
+        num_cross_layers=2,
+        num_experts=2,
+        low_rank=4,
+        cat_embed_input=tab_preprocessor.cat_embed_input,  # type: ignore[arg-type]
+        continuous_cols=continuous_cols,
+        mlp_hidden_dims=[16, 8],
+    )
+    model = WideDeep(deeptabular=dcn)
+
+    X_train = {"X_tab": X_tab_tr, "target": y_tr}
+    X_val = {"X_tab": X_tab_val, "target": y_val}
+    trainer = Trainer(model=model, objective="binary", verbose=0)
+    trainer.fit(X_train=X_train, X_val=X_val, n_epochs=1)
+    preds = trainer.predict(X_tab=X_tab_te)
+
+    assert preds.shape[0] == X_tab_te.shape[0]
+    assert (
+        trainer.history is not None
+        and "train_loss" in trainer.history
+        and "val_loss" in trainer.history
+    )
diff --git a/tests/test_rec/test_gdcn.py b/tests/test_rec/test_gdcn.py
new file mode 100644
index 00000000..ed2afe88
--- /dev/null
+++ b/tests/test_rec/test_gdcn.py
@@ -0,0 +1,132 @@
+import torch
+import pytest
+
+from pytorch_widedeep import Trainer
+from pytorch_widedeep.models import WideDeep
+from pytorch_widedeep.preprocessing import TabPreprocessor
+from pytorch_widedeep.models.rec.gdcn import (
+    GatedCrossLayer,
+    GatedCrossNetwork,
+    GatedDeepCrossNetwork,
+)
+
+from .utils_test_rec import create_train_val_test_data
+
+train, valid, test = create_train_val_test_data()
+
+
+def test_gated_cross_layer():
+    input_dim = 16
+    batch_size = 32
+
+    layer = GatedCrossLayer(input_dim=input_dim)
+
+    # Test input tensors
+    x0 = torch.randn(batch_size, input_dim)
+    xi = torch.randn(batch_size, input_dim)
+
+    # Test forward pass
+    output = layer(x0, xi)
+
+    # Check output shape
+    assert output.shape == (batch_size, input_dim)
+
+    # Check parameter shapes
+    assert layer.cross_weight.shape == (input_dim, input_dim)
+    assert layer.gate_weight.shape == (input_dim, input_dim)
+    assert layer.bias.shape == (input_dim,)
+
+
+@pytest.mark.parametrize("num_layers", [2, 3, 4])
+def test_gated_cross_network(num_layers):
+    input_dim = 16
+    batch_size = 32
+
+    network = GatedCrossNetwork(input_dim=input_dim, num_layers=num_layers)
+
+    # Test input tensor
+    x = torch.randn(batch_size, input_dim)
+
+    # Test forward pass
+    output = network(x)
+
+    # Check output shape
+    assert output.shape == (batch_size, input_dim)
+
+    # Check number of layers
+    assert len(network.layers) == num_layers
+
+    # Check each layer is a GatedCrossLayer
+    for layer in network.layers:
+        assert isinstance(layer, GatedCrossLayer)
+
+
+@pytest.mark.parametrize(
+    "structure,num_layers",
+    [
+        ("parallel", 2),
+        ("stacked", 2),
+        ("parallel", 3),
+        ("stacked", 3),
+    ],
+)
+def test_gdcn_structure(structure, num_layers, cat_embed_cols, continuous_cols):
+    tab_preprocessor = TabPreprocessor(
+        cat_embed_cols=cat_embed_cols, continuous_cols=continuous_cols
+    )
+
+    X_tab = tab_preprocessor.fit_transform(train)
+    X_tab_tnsr = torch.tensor(X_tab, dtype=torch.float32)
+
+    model = GatedDeepCrossNetwork(
+        column_idx=tab_preprocessor.column_idx,
+        num_cross_layers=num_layers,
+        structure=structure,
+        cat_embed_input=tab_preprocessor.cat_embed_input,  # type: ignore[arg-type]
+        continuous_cols=continuous_cols,
+        mlp_hidden_dims=[16, 8],
+    )
+
+    out = model(X_tab_tnsr)
+    assert out.shape == (X_tab_tnsr.shape[0], model.output_dim)
+
+    # Test output dimension property
+    if structure == "stacked":
+        assert model.output_dim == model.mlp_hidden_dims[-1]
+    else:  # parallel
+        assert model.output_dim == (
+            model.mlp_hidden_dims[-1] + model.cat_out_dim + model.cont_out_dim
+        )
+
+
+def test_gdcn_full_process(cat_embed_cols):
+    tab_preprocessor = TabPreprocessor(
+        cat_embed_cols=cat_embed_cols,
+    )
+
+    X_tab_tr = tab_preprocessor.fit_transform(train)
+    X_tab_val = tab_preprocessor.transform(valid)
+    X_tab_te = tab_preprocessor.transform(test)
+    y_tr = train["purchased"].values
+    y_val = valid["purchased"].values
+
+    gdcn = GatedDeepCrossNetwork(
+        column_idx=tab_preprocessor.column_idx,
+        cat_embed_input=tab_preprocessor.cat_embed_input,  # type: ignore[arg-type]
+        num_cross_layers=2,
+        mlp_hidden_dims=[16, 8],
+    )
+    model = WideDeep(deeptabular=gdcn)
+
+    X_train = {"X_tab": X_tab_tr, "target": y_tr}
+    X_val = {"X_tab": X_tab_val, "target": y_val}
+    trainer = Trainer(model=model, objective="binary", verbose=0)
+    trainer.fit(X_train=X_train, X_val=X_val, n_epochs=1)
+    preds = trainer.predict(X_tab=X_tab_te)
+
+    assert preds.shape[0] == X_tab_te.shape[0]
+    assert (
+        trainer.history is not None
+        and "train_loss" in trainer.history
+        and "val_loss" in trainer.history
+    )

From 3b603c9f6e564919bbc67ad42ee75a79a5df6e99 Mon Sep 17 00:00:00 2001
From: Javier <jrzaurin@gmail.com>
Date: Sat, 2 Nov 2024 13:05:11 +0000
Subject: [PATCH 21/24] Reviewed the docs. Tested installation for all
 supported python versions. Fixed a few bugs

---
 examples/scripts/movielens_din.py             |   193 +-
 mkdocs/mkdocs.yml                             |    26 +-
 mkdocs/site/404.html                          |    16 +-
 mkdocs/site/contributing.html                 |    18 +-
 mkdocs/site/index.html                        |    21 +-
 mkdocs/site/index.md                          |     2 +-
 mkdocs/site/installation.html                 |    16 +-
 mkdocs/site/objects.inv                       |   Bin 2007 -> 2066 bytes
 .../pytorch-widedeep/bayesian_models.html     |  1405 +-
 .../site/pytorch-widedeep/bayesian_models.md  |     8 -
 .../pytorch-widedeep/bayesian_trainer.html    |  1864 +-
 mkdocs/site/pytorch-widedeep/callbacks.html   |  1561 +-
 mkdocs/site/pytorch-widedeep/dataloaders.html |   228 +-
 mkdocs/site/pytorch-widedeep/losses.html      |  4985 +++-
 mkdocs/site/pytorch-widedeep/metrics.html     |   976 +-
 .../pytorch-widedeep/model_components.html    | 23998 +++++++++++-----
 .../site/pytorch-widedeep/model_components.md |     8 -
 .../site/pytorch-widedeep/preprocessing.html  | 13471 ++++++---
 .../self_supervised_pretraining.html          |  1974 +-
 mkdocs/site/pytorch-widedeep/tab2vec.html     |   989 +-
 mkdocs/site/pytorch-widedeep/trainer.html     |  3765 ++-
 .../utils/deeptabular_utils.html              |   831 +-
 .../utils/fastai_transforms.html              |  1142 +-
 .../utils/fastai_transforms.md                |     4 +-
 .../pytorch-widedeep/utils/image_utils.html   |   676 +-
 mkdocs/site/pytorch-widedeep/utils/index.html |    16 +-
 .../pytorch-widedeep/utils/text_utils.html    |   618 +-
 mkdocs/site/quick_start.html                  |    16 +-
 mkdocs/site/search/search_index.json          |     2 +-
 mkdocs/site/sitemap.xml                       |    88 +-
 mkdocs/site/sitemap.xml.gz                    |   Bin 795 -> 796 bytes
 mkdocs/sources/index.md                       |     2 +-
 .../pytorch-widedeep/bayesian_models.md       |     8 -
 .../pytorch-widedeep/load_from_folder.md      |    19 +-
 .../pytorch-widedeep/model_components.md      |     8 -
 .../pytorch-widedeep/the_rec_module.md        |    27 +-
 .../utils/fastai_transforms.md                |     4 +-
 .../image/image_from_folder.py                |    16 +
 .../tabular/tabular_from_folder.py            |    15 +
 .../load_from_folder/text/text_from_folder.py |    17 +-
 .../wd_dataset_from_folder.py                 |    14 +-
 pytorch_widedeep/metrics.py                   |    16 +
 pytorch_widedeep/models/model_fusion.py       |   206 +-
 pytorch_widedeep/models/rec/autoint.py        |    62 +-
 pytorch_widedeep/models/rec/autoint_plus.py   |    85 +-
 pytorch_widedeep/models/rec/dcn.py            |    87 +-
 pytorch_widedeep/models/rec/dcnv2.py          |    88 +-
 pytorch_widedeep/models/rec/deepfm.py         |     9 +-
 pytorch_widedeep/models/rec/din.py            |    12 +-
 pytorch_widedeep/models/rec/gdcn.py           |    84 +-
 pytorch_widedeep/models/rec/xdeepfm.py        |     9 +-
 .../preprocessing/din_preprocessor.py         |    89 +-
 .../preprocessing/tab_preprocessor.py         |    77 +-
 .../_base_contrastive_denoising_trainer.py    |     6 +-
 .../_base_encoder_decoder_trainer.py          |     6 +-
 .../contrastive_denoising_trainer.py          |     5 +-
 .../encoder_decoder_trainer.py                |     5 +-
 pytorch_widedeep/tab2vec.py                   |     2 +-
 pytorch_widedeep/training/_base_trainer.py    |    10 +-
 pytorch_widedeep/training/_wd_dataset.py      |     4 +-
 pytorch_widedeep/training/trainer.py          |     2 +-
 pytorch_widedeep/utils/image_utils.py         |    38 +-
 pytorch_widedeep/utils/text_utils.py          |    14 +-
 requirements.txt                              |     1 -
 tests/test_data_utils/test_du_image.py        |    63 +
 tests/test_rec/test_autoint.py                |    49 +
 tests/test_rec/test_autoint_plus.py           |    56 +
 tests/test_rec/test_deepfm.py                 |     4 +-
 tests/test_rec/test_xdeepfm.py                |     4 +-
 69 files changed, 44244 insertions(+), 15896 deletions(-)

diff --git a/examples/scripts/movielens_din.py b/examples/scripts/movielens_din.py
index 551c5dec..97e18d17 100644
--- a/examples/scripts/movielens_din.py
+++ b/examples/scripts/movielens_din.py
@@ -1,21 +1,12 @@
-# DIN is a "special" model and the data needs a very particular preparation
-# process. Therefore, the library does not have a dedicated preprocessor for
-# this specific model. Below is a detail explanation and if someone wants to
-# use this algo I would suggest to wrap it all in an object prior to pass the
-# data to the model
-
 import re
-from functools import partial
 
-import numpy as np
 import pandas as pd
-from sklearn.preprocessing import LabelEncoder
 
 from pytorch_widedeep import Trainer
 from pytorch_widedeep.models import WideDeep
 from pytorch_widedeep.metrics import Accuracy
 from pytorch_widedeep.datasets import load_movielens100k
-from pytorch_widedeep.preprocessing import TabPreprocessor
+from pytorch_widedeep.preprocessing import DINPreprocessor
 from pytorch_widedeep.models.rec.din import DeepInterestNetwork
 
 
@@ -25,58 +16,6 @@ def clean_genre_list(genre_list):
     )
 
 
-def label_encode_column(df, column_name):
-    le = LabelEncoder()
-    df[column_name] = le.fit_transform(df[column_name])
-    return df, le
-
-
-def create_sequences(group, seq_len=5):
-    movies = group["movie_id"].tolist()
-    genres = group["genre_list"].tolist()
-    ratings = group["rating"].tolist()
-
-    sequences = []
-    for i in range(len(movies) - seq_len):
-        user_movies_sequence = movies[i : i + seq_len]
-        genres_sequence = genres[i : i + seq_len]
-        ratings_sequence = ratings[i : i + seq_len]
-        target_item = movies[i + seq_len]
-        target_item_rating = ratings[i + seq_len]
-
-        sequences.append(
-            {
-                "user_id": group.name,
-                "user_movies_sequence": user_movies_sequence,
-                "genres_sequence": genres_sequence,
-                "ratings_sequence": ratings_sequence,
-                "target_item": target_item,
-                "target_item_rating": target_item_rating,
-            }
-        )
-
-    seq_df = pd.DataFrame(sequences)
-    non_seq_cols = group.drop_duplicates(["user_id"]).drop(
-        ["movie_id", "genre_list", "rating", "timestamp"], axis=1
-    )
-
-    return pd.merge(seq_df, non_seq_cols, on="user_id")
-
-
-def preprocess_movie_data(df, seq_len=5):
-    df_sorted = df.sort_values(["user_id", "timestamp"])
-
-    partial_create_sequences = partial(create_sequences, seq_len=seq_len)
-
-    result_df = (
-        df_sorted.groupby("user_id")
-        .apply(partial_create_sequences)
-        .reset_index(drop=True)
-    )
-
-    return result_df
-
-
 if __name__ == "__main__":
 
     data, users, items = load_movielens100k(as_frame=True)
@@ -123,123 +62,25 @@ def preprocess_movie_data(df, seq_len=5):
 
     df["rating"] = df["rating"].apply(lambda x: 1 if x >= 4 else 0)
 
-    # Up until here, everything is quite standard. Now, some columns will be
-    # treated as sequences, and therefore, they need to be tokenized
-    # and "numericalised"/label encoded
-    df, user_le = label_encode_column(df, "user_id")
-    df, item_le = label_encode_column(df, "movie_id")
-    df, genre_le = label_encode_column(df, "genre_list")
-
-    # Internally all models for tabular data in this libray use padding idx =
-    # 0 for unseen values, while sklearn's LabelEncoder starts at 0.
-    # Therefore we need to add 1 to the encoded values to leave 0 for
-    # unknown/unseen values
-    df["movie_id"] = df["movie_id"] + 1
-    df["genre_list"] = df["genre_list"] + 1
-
-    # The explanation as to why we do this with the ratings will come later
-    df["rating"] = df["rating"] + 1
-
-    # we build sequences of 5 movies. Our goal will be predicting whether the
-    # next movie will be reviewed positively or negatively
-    df = df.sort_values(by=["timestamp"]).reset_index(drop=True)
-    seq_df = preprocess_movie_data(df, seq_len=5)
-    # target back to 0/1
-    seq_df["target_item_rating"] = seq_df["target_item_rating"] - 1
-
-    X_target_item = np.array(seq_df.target_item.tolist()).reshape(-1, 1)
-
-    # in reality, all users here have more than 5 reviews, so we have complete
-    # sequences, but there is a change that this does not happen in a given
-    # dataset, so one would have to pad with padding idx (0 in this case)
-    seq_len = 5
-    X_user_behaviour = np.array(
-        [
-            lst + [0] * (seq_len - len(lst))
-            for lst in seq_df.user_movies_sequence.tolist()
-        ]
-    )
-    X_ratings = np.array(
-        [lst + [0] * (seq_len - len(lst)) for lst in seq_df.ratings_sequence.tolist()]
-    )
-    X_genres = np.array(
-        [lst + [0] * (seq_len - len(lst)) for lst in seq_df.genres_sequence.tolist()]
-    )
-
-    # At this point we have the target item as an array of shape (N obs, 1),
-    # and all columns that are going to be treated as sequences stored in
-    # arrays of shape (N obs, seq_len). The rest of the columns are going to
-    # be treated as ANY other "standard" tabular dataset
-    other_cols = ["user_id", "age", "gender", "occupation"]
-    df_other_feat = seq_df[other_cols]
-    tab_preprocessor = TabPreprocessor(cat_embed_cols=other_cols)
-    X_other_feats = tab_preprocessor.fit_transform(df_other_feat)
-
-    X_all = np.concatenate(
-        [X_other_feats, X_target_item, X_user_behaviour, X_ratings, X_genres], axis=1
-    )
-
-    # Now, all the model components in this library they take a tensor
-    # (just one) as input. Therefore, if we want to treat some data
-    # differently, we need to slice the tensor internally. For this to happen
-    # we need to "tell" the algorithm which column is which. DIN has data of
-    # two different natures: sequences and standard tabular (i.e. everything
-    # else). For the sequences, lets simply define columns with what they are
-    # an an index (but you can define then with whichever string you want)
-    user_behaviour_cols = [f"item_{i+1}" for i in range(5)]
-    genres_cols = [f"genre_{i+1}" for i in range(5)]
-    ratings_cols = [f"rating_{i+1}" for i in range(5)]
-
-    # Then all columns in the datasets are, in order of appearance in X_all:
-    all_cols = (
-        [el[0] for el in tab_preprocessor.cat_embed_input]  # tabular cols
-        + ["target_item"]  # target item
-        + user_behaviour_cols  # user behaviour seq cols
-        + ratings_cols  # ratings seq cols
-        + genres_cols  # genres seq cols
+    din_preprocessor = DINPreprocessor(
+        user_id_col="user_id",
+        item_embed_col="movie_id",
+        target_col="rating",
+        max_seq_length=5,
+        action_col="rating",
+        other_seq_embed_cols=["genre_list"],
+        cat_embed_cols=["user_id", "age", "gender", "occupation"],
     )
-    column_idx = {k: i for i, k in enumerate(all_cols)}
-
-    # Now we need to define the so called "configs". For the sequence columns these will consist of:
-    # - the column names
-    # - the maximum value in the column (to define the number of embeddings)
-    # - the embedding dim
-    user_behavior_confiq = (
-        user_behaviour_cols,
-        X_user_behaviour.max(),
-        32,
-    )
-
-    # Again, the explanation to this will come when we instantiate the model
-    # (also, please, read the docs)
-    rating_seq_config = (ratings_cols, 2)
-
-    # all the other sequence columns that are not user behaviour or an action
-    # related to the items that define the user behaviour will be refer
-    # as "other sequence columns" and will be pass as elements of a list
-    other_seq_cols_confiq = [(genres_cols, X_genres.max(), 16)]
 
-    # And finally, the config for the remaining, tabular columns
-    other_cols_config = tab_preprocessor.cat_embed_input
+    X_din, y = din_preprocessor.fit_transform(df)
 
-    # Now, one of the params of the DeepInterestNetwork is action_seq_config.
-    # This 'action' can be, for example a rating, or purchased/not-purchased.
-    # The way that this 'action' will be used is the following: this action
-    # will **always** be learned as a 1d embedding and will be combined with
-    # the user behaviour. For example, imagine that the action is
-    # purchased/not-purchased. then per item in the user behaviour sequence
-    # there will be a binary action to learn 0/1. Such action will be
-    # represented by a float number (so 3 floats will be learned, one for
-    # purchased, one for not-purchased and one for padding) that will
-    # multiply the corresponding item embedding in the user behaviour
-    # sequence.
     din = DeepInterestNetwork(
-        column_idx=column_idx,
+        column_idx=din_preprocessor.din_columns_idx,
         target_item_col="target_item",
-        user_behavior_confiq=user_behavior_confiq,
-        action_seq_config=rating_seq_config,
-        other_seq_cols_confiq=other_seq_cols_confiq,
-        cat_embed_input=other_cols_config,
+        user_behavior_confiq=din_preprocessor.user_behaviour_config,
+        action_seq_config=din_preprocessor.action_seq_config,
+        other_seq_cols_confiq=din_preprocessor.other_seq_config,
+        cat_embed_input=din_preprocessor.tab_preprocessor.cat_embed_input,
         mlp_hidden_dims=[128, 64],
     )
 
@@ -250,8 +91,8 @@ def preprocess_movie_data(df, seq_len=5):
 
     # in the real world you would have to split the data into train, val and test
     trainer.fit(
-        X_tab=X_all,
-        target=seq_df.target_item_rating.values,
+        X_tab=X_din,
+        target=y,
         n_epochs=5,
         batch_size=512,
     )
diff --git a/mkdocs/mkdocs.yml b/mkdocs/mkdocs.yml
index 7cdd8ee8..9e719c56 100644
--- a/mkdocs/mkdocs.yml
+++ b/mkdocs/mkdocs.yml
@@ -53,14 +53,14 @@ nav:
         - 12_ZILNLoss_origkeras_vs_pytorch_widedeep: examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.ipynb
         - 13_model_uncertainty_prediction: examples/13_model_uncertainty_prediction.ipynb
         - 14_bayesian_models: examples/14_bayesian_models.ipynb
-        - 15_Self-Supervised Pre-Training pt 1: examples/16_Self_Supervised_Pretraning_pt1.ipynb
-        - 15_Self-Supervised Pre-Training pt 2: examples/16_Self_Supervised_Pretraning_pt2.ipynb
-        - 16_Usign-a-custom-hugging-face-model: examples/17_Usign_a_custom_hugging_face_model.ipynb
-        - 17_feature_importance_via_attention_weights: examples/18_feature_importance_via_attention_weights.ipynb
-        - 18_wide_and_deep_for_recsys_pt1: examples/19_wide_and_deep_for_recsys_pt1.ipynb
-        - 18_wide_and_deep_for_recsys_pt2: examples/19_wide_and_deep_for_recsys_pt2.ipynb
-        - 19_load_from_folder_functionality: examples/20_load_from_folder_functionality.ipynb
-        - 20-Using-huggingface-within-widedeep: examples/21_Using_huggingface_within_widedeep.ipynb
+        - 15_Self-Supervised Pre-Training pt 1: examples/15_Self_Supervised_Pretraning_pt1.ipynb
+        - 15_Self-Supervised Pre-Training pt 2: examples/15_Self_Supervised_Pretraning_pt2.ipynb
+        - 16_Usign-a-custom-hugging-face-model: examples/16_Usign_a_custom_hugging_face_model.ipynb
+        - 17_feature_importance_via_attention_weights: examples/17_feature_importance_via_attention_weights.ipynb
+        - 18_wide_and_deep_for_recsys_pt1: examples/18_wide_and_deep_for_recsys_pt1.ipynb
+        - 18_wide_and_deep_for_recsys_pt2: examples/18_wide_and_deep_for_recsys_pt2.ipynb
+        - 19_load_from_folder_functionality: examples/19_load_from_folder_functionality.ipynb
+        - 20-Using-huggingface-within-widedeep: examples/20_Using_huggingface_within_widedeep.ipynb
     - Contributing: contributing.md
 
 theme:
@@ -122,21 +122,21 @@ plugins:
                     members_order: source
                     show_source: True
                     show_submodules: True
-                    merge_init_into_class: True
                     separate_signature: True
                     show_root_full_path: False
                     show_root_heading: True
-                    docstring_section_style: list
-                    show_root_toc_entry: False
+                    docstring_section_style: table
+                    show_root_toc_entry: True
+                    generate_full_source: true
+                    docstring_style: numpy
+                    # merge_init_into_class: True
                     # show_root_heading: True
                     # show_object_full_path: False
                     # separate_signature: True
                     # merge_init_into_class: True
                     # heading_level: 2
-                    # show_source: True
                     # show_category_heading: true
                     # docstring_section_style: table
-                    docstring_style: numpy
                     filters:
                        - "!^_"  # exclude all members starting with _
 ## Custom theme additions
diff --git a/mkdocs/site/404.html b/mkdocs/site/404.html
index acaae39f..b12bf418 100644
--- a/mkdocs/site/404.html
+++ b/mkdocs/site/404.html
@@ -1320,7 +1320,7 @@
   
   
     <li class="md-nav__item">
-      <a href="/examples/16_Self_Supervised_Pretraning_pt1.ipynb" class="md-nav__link">
+      <a href="/examples/15_Self_Supervised_Pretraning_pt1.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1341,7 +1341,7 @@
   
   
     <li class="md-nav__item">
-      <a href="/examples/16_Self_Supervised_Pretraning_pt2.ipynb" class="md-nav__link">
+      <a href="/examples/15_Self_Supervised_Pretraning_pt2.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1362,7 +1362,7 @@
   
   
     <li class="md-nav__item">
-      <a href="/examples/17_Usign_a_custom_hugging_face_model.ipynb" class="md-nav__link">
+      <a href="/examples/16_Usign_a_custom_hugging_face_model.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1383,7 +1383,7 @@
   
   
     <li class="md-nav__item">
-      <a href="/examples/18_feature_importance_via_attention_weights.ipynb" class="md-nav__link">
+      <a href="/examples/17_feature_importance_via_attention_weights.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1404,7 +1404,7 @@
   
   
     <li class="md-nav__item">
-      <a href="/examples/19_wide_and_deep_for_recsys_pt1.ipynb" class="md-nav__link">
+      <a href="/examples/18_wide_and_deep_for_recsys_pt1.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1425,7 +1425,7 @@
   
   
     <li class="md-nav__item">
-      <a href="/examples/19_wide_and_deep_for_recsys_pt2.ipynb" class="md-nav__link">
+      <a href="/examples/18_wide_and_deep_for_recsys_pt2.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1446,7 +1446,7 @@
   
   
     <li class="md-nav__item">
-      <a href="/examples/20_load_from_folder_functionality.ipynb" class="md-nav__link">
+      <a href="/examples/19_load_from_folder_functionality.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1467,7 +1467,7 @@
   
   
     <li class="md-nav__item">
-      <a href="/examples/21_Using_huggingface_within_widedeep.ipynb" class="md-nav__link">
+      <a href="/examples/20_Using_huggingface_within_widedeep.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
diff --git a/mkdocs/site/contributing.html b/mkdocs/site/contributing.html
index 62ae46dc..0f4867ab 100644
--- a/mkdocs/site/contributing.html
+++ b/mkdocs/site/contributing.html
@@ -11,7 +11,7 @@
         <link rel="canonical" href="https://pytorch-widedeep.readthedocs.io/contributing.html">
       
       
-        <link rel="prev" href="examples/14_bayesian_models.html">
+        <link rel="prev" href="examples/20_Using_huggingface_within_widedeep.html">
       
       
       
@@ -1326,7 +1326,7 @@
   
   
     <li class="md-nav__item">
-      <a href="examples/16_Self_Supervised_Pretraning_pt1.ipynb" class="md-nav__link">
+      <a href="examples/15_Self_Supervised_Pretraning_pt1.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1347,7 +1347,7 @@
   
   
     <li class="md-nav__item">
-      <a href="examples/16_Self_Supervised_Pretraning_pt2.ipynb" class="md-nav__link">
+      <a href="examples/15_Self_Supervised_Pretraning_pt2.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1368,7 +1368,7 @@
   
   
     <li class="md-nav__item">
-      <a href="examples/17_Usign_a_custom_hugging_face_model.ipynb" class="md-nav__link">
+      <a href="examples/16_Usign_a_custom_hugging_face_model.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1389,7 +1389,7 @@
   
   
     <li class="md-nav__item">
-      <a href="examples/18_feature_importance_via_attention_weights.ipynb" class="md-nav__link">
+      <a href="examples/17_feature_importance_via_attention_weights.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1410,7 +1410,7 @@
   
   
     <li class="md-nav__item">
-      <a href="examples/19_wide_and_deep_for_recsys_pt1.ipynb" class="md-nav__link">
+      <a href="examples/18_wide_and_deep_for_recsys_pt1.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1431,7 +1431,7 @@
   
   
     <li class="md-nav__item">
-      <a href="examples/19_wide_and_deep_for_recsys_pt2.ipynb" class="md-nav__link">
+      <a href="examples/18_wide_and_deep_for_recsys_pt2.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1452,7 +1452,7 @@
   
   
     <li class="md-nav__item">
-      <a href="examples/20_load_from_folder_functionality.ipynb" class="md-nav__link">
+      <a href="examples/19_load_from_folder_functionality.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1473,7 +1473,7 @@
   
   
     <li class="md-nav__item">
-      <a href="examples/21_Using_huggingface_within_widedeep.ipynb" class="md-nav__link">
+      <a href="examples/20_Using_huggingface_within_widedeep.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
diff --git a/mkdocs/site/index.html b/mkdocs/site/index.html
index 666a04ab..71873d46 100644
--- a/mkdocs/site/index.html
+++ b/mkdocs/site/index.html
@@ -1341,7 +1341,7 @@
   
   
     <li class="md-nav__item">
-      <a href="examples/16_Self_Supervised_Pretraning_pt1.ipynb" class="md-nav__link">
+      <a href="examples/15_Self_Supervised_Pretraning_pt1.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1362,7 +1362,7 @@
   
   
     <li class="md-nav__item">
-      <a href="examples/16_Self_Supervised_Pretraning_pt2.ipynb" class="md-nav__link">
+      <a href="examples/15_Self_Supervised_Pretraning_pt2.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1383,7 +1383,7 @@
   
   
     <li class="md-nav__item">
-      <a href="examples/17_Usign_a_custom_hugging_face_model.ipynb" class="md-nav__link">
+      <a href="examples/16_Usign_a_custom_hugging_face_model.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1404,7 +1404,7 @@
   
   
     <li class="md-nav__item">
-      <a href="examples/18_feature_importance_via_attention_weights.ipynb" class="md-nav__link">
+      <a href="examples/17_feature_importance_via_attention_weights.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1425,7 +1425,7 @@
   
   
     <li class="md-nav__item">
-      <a href="examples/19_wide_and_deep_for_recsys_pt1.ipynb" class="md-nav__link">
+      <a href="examples/18_wide_and_deep_for_recsys_pt1.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1446,7 +1446,7 @@
   
   
     <li class="md-nav__item">
-      <a href="examples/19_wide_and_deep_for_recsys_pt2.ipynb" class="md-nav__link">
+      <a href="examples/18_wide_and_deep_for_recsys_pt2.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1467,7 +1467,7 @@
   
   
     <li class="md-nav__item">
-      <a href="examples/20_load_from_folder_functionality.ipynb" class="md-nav__link">
+      <a href="examples/19_load_from_folder_functionality.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1488,7 +1488,7 @@
   
   
     <li class="md-nav__item">
-      <a href="examples/21_Using_huggingface_within_widedeep.ipynb" class="md-nav__link">
+      <a href="examples/20_Using_huggingface_within_widedeep.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1572,7 +1572,7 @@ <h1 id="pytorch-widedeep"><strong>pytorch-widedeep</strong><a class="headerlink"
 <ul>
 <li><a href="#pytorch-widedeep"><strong>pytorch-widedeep</strong></a><ul>
 <li><a href="#introduction">Introduction</a></li>
-<li><a href="#Architectures">Architectures</a></li>
+<li><a href="#architectures">Architectures</a></li>
 <li><a href="#the-deeptabular-component">The <code>deeptabular</code> component</a></li>
 <li><a href="#the-rec-module">The <code>rec</code> module</a></li>
 <li><a href="#text-and-images">Text and Images</a></li>
@@ -2382,7 +2382,8 @@ <h4 id="apa">APA<a class="headerlink" href="#apa" title="Permanent link">&para;<
       
         <a href="mailto:jrzaurin@gmail.com">Javier</a>, 
         <a href="mailto:mulinka.pavol@gmail.com">Pavol Mulinka</a>, 
-        <a href="mailto:javierrodriguezzaurin@javiers-macbook-pro.local">Javier Rodriguez Zaurin</a>
+        <a href="mailto:javierrodriguezzaurin@javiers-macbook-pro.local">Javier Rodriguez Zaurin</a>, 
+        <a href="mailto:not.committed.yet">Not Committed Yet</a>
     </nav>
   </span>
 
diff --git a/mkdocs/site/index.md b/mkdocs/site/index.md
index 77d61195..fb6aebfb 100644
--- a/mkdocs/site/index.md
+++ b/mkdocs/site/index.md
@@ -31,7 +31,7 @@ The content of this document is organized as follows:
 
 - [**pytorch-widedeep**](#pytorch-widedeep)
     - [Introduction](#introduction)
-    - [Architectures](#Architectures)
+    - [Architectures](#architectures)
     - [The ``deeptabular`` component](#the-deeptabular-component)
     - [The ``rec`` module](#the-rec-module)
     - [Text and Images](#text-and-images)
diff --git a/mkdocs/site/installation.html b/mkdocs/site/installation.html
index f321b41e..9394411d 100644
--- a/mkdocs/site/installation.html
+++ b/mkdocs/site/installation.html
@@ -1382,7 +1382,7 @@
   
   
     <li class="md-nav__item">
-      <a href="examples/16_Self_Supervised_Pretraning_pt1.ipynb" class="md-nav__link">
+      <a href="examples/15_Self_Supervised_Pretraning_pt1.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1403,7 +1403,7 @@
   
   
     <li class="md-nav__item">
-      <a href="examples/16_Self_Supervised_Pretraning_pt2.ipynb" class="md-nav__link">
+      <a href="examples/15_Self_Supervised_Pretraning_pt2.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1424,7 +1424,7 @@
   
   
     <li class="md-nav__item">
-      <a href="examples/17_Usign_a_custom_hugging_face_model.ipynb" class="md-nav__link">
+      <a href="examples/16_Usign_a_custom_hugging_face_model.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1445,7 +1445,7 @@
   
   
     <li class="md-nav__item">
-      <a href="examples/18_feature_importance_via_attention_weights.ipynb" class="md-nav__link">
+      <a href="examples/17_feature_importance_via_attention_weights.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1466,7 +1466,7 @@
   
   
     <li class="md-nav__item">
-      <a href="examples/19_wide_and_deep_for_recsys_pt1.ipynb" class="md-nav__link">
+      <a href="examples/18_wide_and_deep_for_recsys_pt1.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1487,7 +1487,7 @@
   
   
     <li class="md-nav__item">
-      <a href="examples/19_wide_and_deep_for_recsys_pt2.ipynb" class="md-nav__link">
+      <a href="examples/18_wide_and_deep_for_recsys_pt2.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1508,7 +1508,7 @@
   
   
     <li class="md-nav__item">
-      <a href="examples/20_load_from_folder_functionality.ipynb" class="md-nav__link">
+      <a href="examples/19_load_from_folder_functionality.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1529,7 +1529,7 @@
   
   
     <li class="md-nav__item">
-      <a href="examples/21_Using_huggingface_within_widedeep.ipynb" class="md-nav__link">
+      <a href="examples/20_Using_huggingface_within_widedeep.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
diff --git a/mkdocs/site/objects.inv b/mkdocs/site/objects.inv
index 2208df9aa6290a7b647eaabf35faa393dd5d954b..dd3a1ce2f70da77e8e1c82a403f462903b602f72 100644
GIT binary patch
delta 1956
zcmV;V2V3~p50Vg&gnwLHZ{j!<e&1g)((dbIG=1DRrX9AGDC$DhmG&hv&Iwq}ZEdHQ
z`Soi%2?PSfp*~Juz&7#uKA(&4q<EJ(r<?GVMGyfL!HT?tU?dCEJc2|7lB~*v@LhSr
zGHB;@&EQ9!`$uKI6z_l1gb4BFyGk1IUmNk?4#jU|n*9F9mw&In6@!VSN$Vl?t0`!R
zJd)LYQaGYQNs_DxeHOt_!jt!<%!`6$>$B2#b!_C$7Pm~OU*8?dyw}aT-T-|ra+b*t
zOOA*nNlqf*BDhiMnMvQJE0U0mLUb0OUHLSGej<!{o`!LrsAmI~k~LJHd*ye!WU30L
zCVv*XBh)>9uzv;_G6|_B5|<g3EYDnXX#$}B-uq3ty9aGpGZ<YTx*zhP5O|a3V_y}j
z;FExRF~n;W>hKmIJ2tpc6>h5EFf<|rh;5g@{5ewvJ;F_mfg*!A=dXlE{W`Rhegj#~
zgc?Ac@!gZ?n(ohxs*!}>fcbL9xhvK+@oBDxHD`Rc(tl8BF2dP&w^SR6NObQVhTFbK
zB*F^>#tS6Mx8=%Vq-~|Srg@je`TW|QR$KC1Q*{_!IAP_|JCVHdemQf=afU+j?~-Ja
zCE$!`TcB&QPWvRR&S>G(?h7o#KfVG)%$?m^@?2B>_ii?K#<LOTm}HoiiDZ&+wSoxl
zb^6hyuYWaP7BfmTfU8!VLR-L^gTg~YQjvX#G|3{@jZ8suMny28wB&@I-{ol@wig#D
z<{<kFZ!dTHeMxf;j#$>xoRj<kk~rnL73iAk0|&~qYdK=NmFJr1;>x8$YiZ6&g5Irk
z#nA|JMzPvSu67MyDib|fSYB5PX`U8&2ARAJA%DnAS(Gx2nAz+hQgSy~RgJhbAZAl}
zT8f@s9A_wKtx_QrkW%WM@nE99?|OE~<px4kM;{iH2@H}S7D*`@kO!4^4zwbI(NJ=d
z2{j8y^)}!x?HnzlS)e!euDKfb<{)2l{)_?ga@B!(x^fSs@^PFRs40Uany6V6ObJz0
z*nclG0k|i0!!p2uu93aqvC44D4MFvI63cdL?IC{=2@$%Rywy7V`6fA};6;_q!9#r#
zY7Rhq4?c(YP^VyQC;FEo>Vs68XXb1{^|-3Cf-uLc*H!7HDZHOOAhmiBhhX=_E3oxO
z3b%vBWu&l%t~MJ1Nqj(Msfs)q$O7G`X@6|!_%KRy8NuQbf=i$g+E$<5AT?2qQJGq$
zYsfmOP!}fU9}!)<XRvYAtiBuI8g-7Q;G<aRsy&K|tkm5ETm-S)o19M{ZKvlR@O^dA
z_Jq*Aq0K}Py#icL?&gm^EYX4+Me%KXuX@(87ZV|y^(k3RRc`7n1kT})Vu`baynmaf
z08W9uR74({j4#6+M<2~BC+K)L%HD<!S$Q5b&NCRzIA<uK8A#u>+m!1y%hoZW5blxM
z#MKSr;C5>EEnVAY5O~Kd2%6woCK{X<2vs(iG`Yomj?w-eNOWi1a;SD%HU1Bs!fb9A
z3fI96sE~v{Llo}odRdivv9dJ-e}6`|r-Wr7oexbxbF0>s{tFHT&#6B8&enQ|INX)w
zJh-`=KeX~^QTIjJLaH8Y5Ak|aX3w`%&!{5<29a5e_HF5+Ktnbfbj;*fxVvF=6b39)
zGfDv?v-l+Fp&)6IAh}@m=bb$s=HENW@ood_!w;=^AagKls!kuT*H*YT=zri=cR7E7
z7qAi9cL0gkzuF*&zP;*@gQq^Wmj^b!K8-vQUaA<||2oK@0u<5ht*}Gy2Jf)&<@)_h
zJ`n@dX@*&$b)!%(*N_9k5*aTlYzY*EOGc8=Bau<mKc?o7VPp3K7A!pVM;1(+GccD<
zL|FiTVFDuEM3(6)`brQdLVs@!ZU3M9sDJF4ejW-`M7A>B(g+&SeY$W=Y5%XkBk1HP
zfw8nfcLoZh#zNK4ij<WXU0p!ZjU#($=yA)8*61CbA+4{r*?3s(^zm)Gdx8PCmB+!+
znl*+Ydx#uqt6I=qy#UqgxT`0i=kE^J&z_F^18Fvsfs2INkFCSSyMGkj*k${sCX4QW
z_nN2Le8c72=-ma3a;%6^pHM0|@)hv`D98(7;MlmpaB)_VHe9NufiIVAXU`AFS5kpX
zeG%*(0ma5i28iwW)5(70%GMd}`=pN$>$snLeg@)09FI9O0?!7<1=l0tfycN8`w;I=
zKp@aH80T0va&4l<l7E9ukQ~6;#KbW4-ViusVCnG#S=c!E2_|;_d>J1L_dUhx)F9!G
zTN_~rLz{RSNa<Ag=+{FC^Eb`rC4*JTjI1E37Zzt{E#~oWT>+;DJ{w4znGGJY{XzqY
z!{E(CBpw6yvym8}Yd~L6Cv2x_5j^J4ko6v+S9smpAHWOh-GA7ye)&Uxb8+DHiIYG4
zr<{_NCsiA)fN~%bJh#xkM{@^xVK?HAKvt%}86}DGFW=bOi&^~c9Tr>H)pe&TJrM;^
zX<p}GeFj+Jwxu{gY@Jfswfj>1CDRAoo*LT)Lv5!y>93Njk|j|H=?bDqP2)tE63N+{
q9cD|;07(5FWgVRrm>>m-LIMAkpk~IXK*E?V9BQs(VE+S+c8>L^F2Rcc

delta 1896
zcmV-u2bcJg5Z4cognyh(bE7yAhVT0;rfT;(Oihk^%OvAUsvK9=PR&;Bg_1>MYXgFn
zL_X%%uOz@<*hV;NF_#2n==W)Lw*U#dd{>2_>-d!=kN}j?lDvatgvWW2KqezamQ_Z?
zwmoAUcK6Mh(YI#rZ@u%WeE*YXM9NQJ^`<rdxi$ag(EM8E*?;eUeER%bIT$3*b`H{?
z+J+Y7i7X$o(h~|LS+*qfSw`Q9$le#KC`-myXRU9~aj`p{-!Z8_efMbd!MJs^2KrnU
zjH?e@PKY8|K@t!$y4BlLyM3Q8Nk%w@<gCE%$)_dE5pgPtJWh*D9~-fptf2n8*M4tW
zCc0r__s>dqjDHT$Z<DV1zxSC=v5^T(wS_={H)a3qyRRDph`%PW$sGx*<NB>W;kNz^
z14K%YoAG4veX1LJjN3B?nvBvyyb@unC#^VDtl%H9eha){QjbpFc=s&&#{JHz9;U=C
z*ni%byJmfZPYOL|dgI+#L&2QK)31I}TZ>4z4<5^H-+v?$@En17j)Z(yEj?D+#+q-;
zd!81v8-KmJW6w9L$Lhi<E1%hkl$8&QsZYi!3gllE;fiJ84RqI_Z&<Hm64qce_Zs&(
zR^Xpr0TSjf?mPB;qyBS0oq6MIjd_NQ^D0wJ5wQXlJ(%q$yM1GLS;{Ce0=~RBr8b9^
z0Ii3nq<<^>Q0bZ{zMafL2}WgfO=%?vJwJue9_(%|(&V7}0`G2j@^wKA0iGxuYu+Kh
zfg)ZxHwJy9J_?}Bq?RYrjXmF>^DCbn+F0`r31%wH7o#=i4Y8i4tf$0Y>Ja^Kh`c#1
zrbS*B9JsnnAu1|WRw_<dZp4UE!jD#cMp{`BJAY$2sbtSAjyILiUPYxS{sr2f3xfr9
z%|=Ti8I6@7T<WWJZl>tA+h;pets&MH%tUyVft#AyPt!aylW-FTS#quK4`4#5K85|X
zm$nB&*Ng*>bfX+Zk8vw@&lTueRx?nqg?RR^^VT@_{Y?qTA&7tN^m4PRV;6IJ7jcH?
z=YJ^~*Za!L70sd$Ex4V&)&;Il))IE9=%#MHZW|w_k4Rn*Vu-d!yaHRTmGnC{T?U0U
zQtr}7$kGFpg>DKIAP4o3p>e?p5tMde)nP%^r_cz#HH!sEF6s*h*AIn8vf0$CbGsE@
z5xsZMXybOX{@)5Wsx!0>KB|S@wMSKvJAZY1A1R|$?cHNeo?P!T1L%hupxuGcvtgG+
z62?`uxW1n~g^0wC+$bHts~@zVP3@&bs`fF9EGK$z608Ky>5meLvp~T^Q-YwtUTToX
zb|+L|j;oIrmJ>L^gR(a-L6(6p8s`Z{3(grNv;dixxz^Qc#rP^E6yiOoby~j-8GqeP
z?6j5d*bEZygax4|1n08Vd4X8(M%V4$d^W@Izef^188-~|grU*@X0|Zh?MmS$xD}O(
z&}T^E{k~q*twEyfOb{;U?nqbxGW)S@XkX&=)&He{QWVszwDOgiwvD$fg@|tNXOBDk
zXI1x2xk~C0Y)|oKUGe9;Nnq8H1%IQ&K8*HF>7qyjA5A)CYAoIDWpoq*#`PVggpnzJ
z5%pM-yv&eNu>K;%9*6xV1aiH*$cD&6Ck|u=`%Kjt<IS@bZVWoQ^{azFA&A%r{vAl-
zjjt}sap<f%WC%3I_HtnBo7Ko8<)xdk^RJ`)ONb(-y_0qvJm4LQP^sTf@_!2vpx$QK
zC$w%An#VQdf^byEs|r_uk_g2}76$|wrTfS1`D5DHqkscNpz+9o#5;j~=|ol~h!-Xy
zF)lK0PBE_p2_ntZ(B|u)Pv*m(+s{LT8f0VBFGkdw?&}NBNc-P39idN-0*uugbQ_>J
zX(LpBtf8E;=;{JUKZzW~(SPHS8O`Y}&JgeGX*LnpGks#?cTX|kdU;$N&8;y4*#mOK
zSN%YD^#as9$6fsZdj8$v=GfD9e_+k!X5b>D?$_4g=6#xO<FfhH8HeG2`;#?I_=eke
z>31J8I<XQ)LrUr5D3&CIpd>GVq2rPQ!_7HW+Hk9m0lwU_TRcBtUw>N#q0Eb5?+7X`
zNix7}C!bD^8&|H$=sqTW1gw*O{`suLhZv6~GXiHT<Ae1`dEgN@Vjtq}6a)gU)i_7l
z+I6WKM+TQ7IgoX!iDBx!C2&aKnDGNCToU{QiCaEj#^aE_r>IU{B>b`MDhwfXsi%QR
zr^?6t8iBOmoos(fu77Tsk|ku#gT>jq7JK-Yu7T4RJ{L)sI~zP@_XiCmhQX7GNE`#^
zvylkUd%(P)PT01mW%N`$1Mgizul2fTe?Tv4W@F>#w+8y%#gR8FPT}-_6qGCjb+y%s
zr~slMu!rs=nm@@4hY^1Oyvji^N-~{ap|wpGv-CZ<EOuYluVFjY+t;!Ls_btpIIjWL
zy1Q09DDJFMx%qvm{(_kW?m*6V#ZdjUEB*E6vSL{hL%xJ0(YJ9j&WRH2%}sO1&H%|3
iC_C~?Op%f#v4sCB(09hDLehpV9vZG=V*dl6XInq3U$8C!

diff --git a/mkdocs/site/pytorch-widedeep/bayesian_models.html b/mkdocs/site/pytorch-widedeep/bayesian_models.html
index 77dd456f..9b055c9c 100644
--- a/mkdocs/site/pytorch-widedeep/bayesian_models.html
+++ b/mkdocs/site/pytorch-widedeep/bayesian_models.html
@@ -1396,7 +1396,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/16_Self_Supervised_Pretraning_pt1.ipynb" class="md-nav__link">
+      <a href="../examples/15_Self_Supervised_Pretraning_pt1.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1417,7 +1417,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/16_Self_Supervised_Pretraning_pt2.ipynb" class="md-nav__link">
+      <a href="../examples/15_Self_Supervised_Pretraning_pt2.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1438,7 +1438,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/17_Usign_a_custom_hugging_face_model.ipynb" class="md-nav__link">
+      <a href="../examples/16_Usign_a_custom_hugging_face_model.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1459,7 +1459,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/18_feature_importance_via_attention_weights.ipynb" class="md-nav__link">
+      <a href="../examples/17_feature_importance_via_attention_weights.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1480,7 +1480,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/19_wide_and_deep_for_recsys_pt1.ipynb" class="md-nav__link">
+      <a href="../examples/18_wide_and_deep_for_recsys_pt1.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1501,7 +1501,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/19_wide_and_deep_for_recsys_pt2.ipynb" class="md-nav__link">
+      <a href="../examples/18_wide_and_deep_for_recsys_pt2.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1522,7 +1522,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/20_load_from_folder_functionality.ipynb" class="md-nav__link">
+      <a href="../examples/19_load_from_folder_functionality.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1543,7 +1543,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/21_Using_huggingface_within_widedeep.ipynb" class="md-nav__link">
+      <a href="../examples/20_Using_huggingface_within_widedeep.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1620,16 +1620,7 @@ <h2 id="pytorch_widedeep.bayesian_models.tabular.bayesian_linear.bayesian_wide.B
 
 
 <a href="#pytorch_widedeep.bayesian_models.tabular.bayesian_linear.bayesian_wide.BayesianWide" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="nf">BayesianWide</span><span class="p">(</span>
-    <span class="n">input_dim</span><span class="p">,</span>
-    <span class="n">pred_dim</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span>
-    <span class="n">prior_sigma_1</span><span class="o">=</span><span class="mf">1.0</span><span class="p">,</span>
-    <span class="n">prior_sigma_2</span><span class="o">=</span><span class="mf">0.002</span><span class="p">,</span>
-    <span class="n">prior_pi</span><span class="o">=</span><span class="mf">0.8</span><span class="p">,</span>
-    <span class="n">posterior_mu_init</span><span class="o">=</span><span class="mf">0.0</span><span class="p">,</span>
-    <span class="n">posterior_rho_init</span><span class="o">=-</span><span class="mf">7.0</span><span class="p">,</span>
-<span class="p">)</span>
-</code></pre></div>
+
 
     <div class="doc doc-contents first">
             <p class="doc doc-class-bases">
@@ -1641,36 +1632,55 @@ <h2 id="pytorch_widedeep.bayesian_models.tabular.bayesian_linear.bayesian_wide.B
 
 
 <p><span class="doc-section-title">Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>input_dim</code></b>
-              (<code>int</code>)
-          –
-          <div class="doc-md-description">
-            <p>size of the Embedding layer. <code>input_dim</code> is the summation of all the
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+          <th>Default</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>input_dim</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>size of the Embedding layer. <code>input_dim</code> is the summation of all the
 individual values for all the features that go through the wide
 component. For example, if the wide component receives 2 features with
 5 individual values each, <code>input_dim = 10</code></p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>pred_dim</code></b>
-              (<code>int</code>, default:
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>pred_dim</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>size of the ouput tensor containing the predictions</p>
+              </div>
+            </td>
+            <td>
                   <code>1</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>size of the ouput tensor containing the predictions</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>prior_sigma_1</code></b>
-              (<code>float</code>, default:
-                  <code>1.0</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>The prior weight distribution is a scaled mixture of two Gaussian
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>prior_sigma_1</code></td>
+            <td>
+                  <code>float</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>The prior weight distribution is a scaled mixture of two Gaussian
 densities:</p>
 <div class="arithmatex">\[
    \begin{aligned}
@@ -1680,39 +1690,51 @@ <h2 id="pytorch_widedeep.bayesian_models.tabular.bayesian_linear.bayesian_wide.B
 <p><code>prior_sigma_1</code> is the prior of the sigma parameter for the first of the two
 Gaussians that will be mixed to produce the prior weight
 distribution.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>prior_sigma_2</code></b>
-              (<code>float</code>, default:
-                  <code>0.002</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Prior of the sigma parameter for the second of the two Gaussian
+              </div>
+            </td>
+            <td>
+                  <code>1.0</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>prior_sigma_2</code></td>
+            <td>
+                  <code>float</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Prior of the sigma parameter for the second of the two Gaussian
 distributions that will be mixed to produce the prior weight
 distribution</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>prior_pi</code></b>
-              (<code>float</code>, default:
-                  <code>0.8</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Scaling factor that will be used to mix the Gaussians to produce the
+              </div>
+            </td>
+            <td>
+                  <code>0.002</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>prior_pi</code></td>
+            <td>
+                  <code>float</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Scaling factor that will be used to mix the Gaussians to produce the
 prior weight distribution</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>posterior_mu_init</code></b>
-              (<code>float</code>, default:
-                  <code>0.0</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>The posterior sample of the weights is defined as:</p>
+              </div>
+            </td>
+            <td>
+                  <code>0.8</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>posterior_mu_init</code></td>
+            <td>
+                  <code>float</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>The posterior sample of the weights is defined as:</p>
 <div class="arithmatex">\[
    \begin{aligned}
    \mathbf{w} &amp;= \mu + log(1 + exp(\rho))
@@ -1727,34 +1749,55 @@ <h2 id="pytorch_widedeep.bayesian_models.tabular.bayesian_linear.bayesian_wide.B
 \]</div>
 <p><span class="arithmatex">\(\mu\)</span> is initialised using a normal distributtion with mean
 <code>posterior_mu_init</code> and std equal to 0.1.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>posterior_rho_init</code></b>
-              (<code>float</code>, default:
-                  <code>-7.0</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>As in the case of <span class="arithmatex">\(\mu\)</span>, <span class="arithmatex">\(\rho\)</span> is initialised using a
+              </div>
+            </td>
+            <td>
+                  <code>0.0</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>posterior_rho_init</code></td>
+            <td>
+                  <code>float</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>As in the case of <span class="arithmatex">\(\mu\)</span>, <span class="arithmatex">\(\rho\)</span> is initialised using a
 normal distributtion with mean <code>posterior_rho_init</code> and std equal to
 0.1.</p>
-          </div>
-        </li>
-    </ul>
+              </div>
+            </td>
+            <td>
+                  <code>-7.0</code>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
 
 <p><span class="doc-section-title">Attributes:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code><span title="pytorch_widedeep.bayesian_models.tabular.bayesian_linear.bayesian_wide.BayesianWide.bayesian_wide_linear">bayesian_wide_linear</span></code></b>
-              (<code><span title="torch.nn.Module">Module</span></code>)
-          –
-          <div class="doc-md-description">
-            <p>the linear layer that comprises the wide branch of the model</p>
-          </div>
-        </li>
-    </ul>
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code><span title="pytorch_widedeep.bayesian_models.tabular.bayesian_linear.bayesian_wide.BayesianWide.bayesian_wide_linear">bayesian_wide_linear</span></code></td>
+            <td>
+                  <code><span title="torch.nn.Module">Module</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>the linear layer that comprises the wide branch of the model</p>
+              </div>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
 
 <p><span class="doc-section-title">Examples:</span></p>
@@ -1765,9 +1808,81 @@ <h2 id="pytorch_widedeep.bayesian_models.tabular.bayesian_linear.bayesian_wide.B
 <span class="gp">&gt;&gt;&gt; </span><span class="n">out</span> <span class="o">=</span> <span class="n">wide</span><span class="p">(</span><span class="n">X</span><span class="p">)</span>
 </code></pre></div>
 
-                  <details class="quote">
-                    <summary>Source code in <code>pytorch_widedeep/bayesian_models/tabular/bayesian_linear/bayesian_wide.py</code></summary>
-                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 81</span>
+              <details class="quote">
+                <summary>Source code in <code>pytorch_widedeep/bayesian_models/tabular/bayesian_linear/bayesian_wide.py</code></summary>
+                <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">  9</span>
+<span class="normal"> 10</span>
+<span class="normal"> 11</span>
+<span class="normal"> 12</span>
+<span class="normal"> 13</span>
+<span class="normal"> 14</span>
+<span class="normal"> 15</span>
+<span class="normal"> 16</span>
+<span class="normal"> 17</span>
+<span class="normal"> 18</span>
+<span class="normal"> 19</span>
+<span class="normal"> 20</span>
+<span class="normal"> 21</span>
+<span class="normal"> 22</span>
+<span class="normal"> 23</span>
+<span class="normal"> 24</span>
+<span class="normal"> 25</span>
+<span class="normal"> 26</span>
+<span class="normal"> 27</span>
+<span class="normal"> 28</span>
+<span class="normal"> 29</span>
+<span class="normal"> 30</span>
+<span class="normal"> 31</span>
+<span class="normal"> 32</span>
+<span class="normal"> 33</span>
+<span class="normal"> 34</span>
+<span class="normal"> 35</span>
+<span class="normal"> 36</span>
+<span class="normal"> 37</span>
+<span class="normal"> 38</span>
+<span class="normal"> 39</span>
+<span class="normal"> 40</span>
+<span class="normal"> 41</span>
+<span class="normal"> 42</span>
+<span class="normal"> 43</span>
+<span class="normal"> 44</span>
+<span class="normal"> 45</span>
+<span class="normal"> 46</span>
+<span class="normal"> 47</span>
+<span class="normal"> 48</span>
+<span class="normal"> 49</span>
+<span class="normal"> 50</span>
+<span class="normal"> 51</span>
+<span class="normal"> 52</span>
+<span class="normal"> 53</span>
+<span class="normal"> 54</span>
+<span class="normal"> 55</span>
+<span class="normal"> 56</span>
+<span class="normal"> 57</span>
+<span class="normal"> 58</span>
+<span class="normal"> 59</span>
+<span class="normal"> 60</span>
+<span class="normal"> 61</span>
+<span class="normal"> 62</span>
+<span class="normal"> 63</span>
+<span class="normal"> 64</span>
+<span class="normal"> 65</span>
+<span class="normal"> 66</span>
+<span class="normal"> 67</span>
+<span class="normal"> 68</span>
+<span class="normal"> 69</span>
+<span class="normal"> 70</span>
+<span class="normal"> 71</span>
+<span class="normal"> 72</span>
+<span class="normal"> 73</span>
+<span class="normal"> 74</span>
+<span class="normal"> 75</span>
+<span class="normal"> 76</span>
+<span class="normal"> 77</span>
+<span class="normal"> 78</span>
+<span class="normal"> 79</span>
+<span class="normal"> 80</span>
+<span class="normal"> 81</span>
 <span class="normal"> 82</span>
 <span class="normal"> 83</span>
 <span class="normal"> 84</span>
@@ -1789,31 +1904,111 @@ <h2 id="pytorch_widedeep.bayesian_models.tabular.bayesian_linear.bayesian_wide.B
 <span class="normal">100</span>
 <span class="normal">101</span>
 <span class="normal">102</span>
-<span class="normal">103</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
-    <span class="bp">self</span><span class="p">,</span>
-    <span class="n">input_dim</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span>
-    <span class="n">pred_dim</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1</span><span class="p">,</span>
-    <span class="n">prior_sigma_1</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">1.0</span><span class="p">,</span>
-    <span class="n">prior_sigma_2</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.002</span><span class="p">,</span>
-    <span class="n">prior_pi</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.8</span><span class="p">,</span>
-    <span class="n">posterior_mu_init</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.0</span><span class="p">,</span>
-    <span class="n">posterior_rho_init</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="o">-</span><span class="mf">7.0</span><span class="p">,</span>
-<span class="p">):</span>
-    <span class="nb">super</span><span class="p">(</span><span class="n">BayesianWide</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
-    <span class="c1">#  Embeddings: val + 1 because 0 is reserved for padding/unseen cateogories.</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">bayesian_wide_linear</span> <span class="o">=</span> <span class="n">bnn</span><span class="o">.</span><span class="n">BayesianEmbedding</span><span class="p">(</span>
-        <span class="n">n_embed</span><span class="o">=</span><span class="n">input_dim</span> <span class="o">+</span> <span class="mi">1</span><span class="p">,</span>
-        <span class="n">embed_dim</span><span class="o">=</span><span class="n">pred_dim</span><span class="p">,</span>
-        <span class="n">padding_idx</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span>
-        <span class="n">prior_sigma_1</span><span class="o">=</span><span class="n">prior_sigma_1</span><span class="p">,</span>
-        <span class="n">prior_sigma_2</span><span class="o">=</span><span class="n">prior_sigma_2</span><span class="p">,</span>
-        <span class="n">prior_pi</span><span class="o">=</span><span class="n">prior_pi</span><span class="p">,</span>
-        <span class="n">posterior_mu_init</span><span class="o">=</span><span class="n">posterior_mu_init</span><span class="p">,</span>
-        <span class="n">posterior_rho_init</span><span class="o">=</span><span class="n">posterior_rho_init</span><span class="p">,</span>
-    <span class="p">)</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">bias</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Parameter</span><span class="p">(</span><span class="n">torch</span><span class="o">.</span><span class="n">zeros</span><span class="p">(</span><span class="n">pred_dim</span><span class="p">))</span>
+<span class="normal">103</span>
+<span class="normal">104</span>
+<span class="normal">105</span>
+<span class="normal">106</span>
+<span class="normal">107</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">class</span> <span class="nc">BayesianWide</span><span class="p">(</span><span class="n">BaseBayesianModel</span><span class="p">):</span>
+<span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Defines a `Wide` model. This is a linear model where the</span>
+<span class="sd">    non-linearlities are captured via crossed-columns</span>
+
+<span class="sd">    Parameters</span>
+<span class="sd">    ----------</span>
+<span class="sd">    input_dim: int</span>
+<span class="sd">        size of the Embedding layer. `input_dim` is the summation of all the</span>
+<span class="sd">        individual values for all the features that go through the wide</span>
+<span class="sd">        component. For example, if the wide component receives 2 features with</span>
+<span class="sd">        5 individual values each, `input_dim = 10`</span>
+<span class="sd">    pred_dim: int</span>
+<span class="sd">        size of the ouput tensor containing the predictions</span>
+<span class="sd">    prior_sigma_1: float, default = 1.0</span>
+<span class="sd">        The prior weight distribution is a scaled mixture of two Gaussian</span>
+<span class="sd">        densities:</span>
+
+<span class="sd">        $$</span>
+<span class="sd">           \begin{aligned}</span>
+<span class="sd">           P(\mathbf{w}) = \prod_{i=j} \pi N (\mathbf{w}_j | 0, \sigma_{1}^{2}) + (1 - \pi) N (\mathbf{w}_j | 0, \sigma_{2}^{2})</span>
+<span class="sd">           \end{aligned}</span>
+<span class="sd">        $$</span>
+
+<span class="sd">        `prior_sigma_1` is the prior of the sigma parameter for the first of the two</span>
+<span class="sd">        Gaussians that will be mixed to produce the prior weight</span>
+<span class="sd">        distribution.</span>
+<span class="sd">    prior_sigma_2: float, default = 0.002</span>
+<span class="sd">        Prior of the sigma parameter for the second of the two Gaussian</span>
+<span class="sd">        distributions that will be mixed to produce the prior weight</span>
+<span class="sd">        distribution</span>
+<span class="sd">    prior_pi: float, default = 0.8</span>
+<span class="sd">        Scaling factor that will be used to mix the Gaussians to produce the</span>
+<span class="sd">        prior weight distribution</span>
+<span class="sd">    posterior_mu_init: float = 0.0</span>
+<span class="sd">        The posterior sample of the weights is defined as:</span>
+
+<span class="sd">        $$</span>
+<span class="sd">           \begin{aligned}</span>
+<span class="sd">           \mathbf{w} &amp;= \mu + log(1 + exp(\rho))</span>
+<span class="sd">           \end{aligned}</span>
+<span class="sd">        $$</span>
+
+<span class="sd">        where:</span>
+
+<span class="sd">        $$</span>
+<span class="sd">           \begin{aligned}</span>
+<span class="sd">           \mathcal{N}(x\vert \mu, \sigma) &amp;= \frac{1}{\sqrt{2\pi}\sigma}e^{-\frac{(x-\mu)^2}{2\sigma^2}}\\</span>
+<span class="sd">           \log{\mathcal{N}(x\vert \mu, \sigma)} &amp;= -\log{\sqrt{2\pi}} -\log{\sigma} -\frac{(x-\mu)^2}{2\sigma^2}\\</span>
+<span class="sd">           \end{aligned}</span>
+<span class="sd">        $$</span>
+
+<span class="sd">        $\mu$ is initialised using a normal distributtion with mean</span>
+<span class="sd">        `posterior_mu_init` and std equal to 0.1.</span>
+<span class="sd">    posterior_rho_init: float = -7.0</span>
+<span class="sd">        As in the case of $\mu$, $\rho$ is initialised using a</span>
+<span class="sd">        normal distributtion with mean `posterior_rho_init` and std equal to</span>
+<span class="sd">        0.1.</span>
+
+<span class="sd">    Attributes</span>
+<span class="sd">    -----------</span>
+<span class="sd">    bayesian_wide_linear: nn.Module</span>
+<span class="sd">        the linear layer that comprises the wide branch of the model</span>
+
+<span class="sd">    Examples</span>
+<span class="sd">    --------</span>
+<span class="sd">    &gt;&gt;&gt; import torch</span>
+<span class="sd">    &gt;&gt;&gt; from pytorch_widedeep.bayesian_models import BayesianWide</span>
+<span class="sd">    &gt;&gt;&gt; X = torch.empty(4, 4).random_(6)</span>
+<span class="sd">    &gt;&gt;&gt; wide = BayesianWide(input_dim=int(X.max().item()), pred_dim=1)</span>
+<span class="sd">    &gt;&gt;&gt; out = wide(X)</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+
+    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span>
+        <span class="n">input_dim</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span>
+        <span class="n">pred_dim</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1</span><span class="p">,</span>
+        <span class="n">prior_sigma_1</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">1.0</span><span class="p">,</span>
+        <span class="n">prior_sigma_2</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.002</span><span class="p">,</span>
+        <span class="n">prior_pi</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.8</span><span class="p">,</span>
+        <span class="n">posterior_mu_init</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.0</span><span class="p">,</span>
+        <span class="n">posterior_rho_init</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="o">-</span><span class="mf">7.0</span><span class="p">,</span>
+    <span class="p">):</span>
+        <span class="nb">super</span><span class="p">(</span><span class="n">BayesianWide</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
+        <span class="c1">#  Embeddings: val + 1 because 0 is reserved for padding/unseen cateogories.</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">bayesian_wide_linear</span> <span class="o">=</span> <span class="n">bnn</span><span class="o">.</span><span class="n">BayesianEmbedding</span><span class="p">(</span>
+            <span class="n">n_embed</span><span class="o">=</span><span class="n">input_dim</span> <span class="o">+</span> <span class="mi">1</span><span class="p">,</span>
+            <span class="n">embed_dim</span><span class="o">=</span><span class="n">pred_dim</span><span class="p">,</span>
+            <span class="n">padding_idx</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span>
+            <span class="n">prior_sigma_1</span><span class="o">=</span><span class="n">prior_sigma_1</span><span class="p">,</span>
+            <span class="n">prior_sigma_2</span><span class="o">=</span><span class="n">prior_sigma_2</span><span class="p">,</span>
+            <span class="n">prior_pi</span><span class="o">=</span><span class="n">prior_pi</span><span class="p">,</span>
+            <span class="n">posterior_mu_init</span><span class="o">=</span><span class="n">posterior_mu_init</span><span class="p">,</span>
+            <span class="n">posterior_rho_init</span><span class="o">=</span><span class="n">posterior_rho_init</span><span class="p">,</span>
+        <span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">bias</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Parameter</span><span class="p">(</span><span class="n">torch</span><span class="o">.</span><span class="n">zeros</span><span class="p">(</span><span class="n">pred_dim</span><span class="p">))</span>
+
+    <span class="k">def</span> <span class="nf">forward</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">X</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tensor</span><span class="p">:</span>
+        <span class="n">out</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">bayesian_wide_linear</span><span class="p">(</span><span class="n">X</span><span class="o">.</span><span class="n">long</span><span class="p">())</span><span class="o">.</span><span class="n">sum</span><span class="p">(</span><span class="n">dim</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">bias</span>
+        <span class="k">return</span> <span class="n">out</span>
 </code></pre></div></td></tr></table></div>
-                  </details>
+              </details>
 
 
 
@@ -1844,28 +2039,7 @@ <h2 id="pytorch_widedeep.bayesian_models.tabular.bayesian_mlp.bayesian_tab_mlp.B
 
 
 <a href="#pytorch_widedeep.bayesian_models.tabular.bayesian_mlp.bayesian_tab_mlp.BayesianTabMlp" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="nf">BayesianTabMlp</span><span class="p">(</span>
-    <span class="n">column_idx</span><span class="p">,</span>
-    <span class="o">*</span><span class="p">,</span>
-    <span class="n">cat_embed_input</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">cat_embed_activation</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">continuous_cols</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">embed_continuous</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">cont_embed_dim</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">cont_embed_dropout</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">cont_embed_activation</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">use_cont_bias</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">cont_norm_layer</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">mlp_hidden_dims</span><span class="o">=</span><span class="p">[</span><span class="mi">200</span><span class="p">,</span> <span class="mi">100</span><span class="p">],</span>
-    <span class="n">mlp_activation</span><span class="o">=</span><span class="s2">&quot;leaky_relu&quot;</span><span class="p">,</span>
-    <span class="n">prior_sigma_1</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span>
-    <span class="n">prior_sigma_2</span><span class="o">=</span><span class="mf">0.002</span><span class="p">,</span>
-    <span class="n">prior_pi</span><span class="o">=</span><span class="mf">0.8</span><span class="p">,</span>
-    <span class="n">posterior_mu_init</span><span class="o">=</span><span class="mf">0.0</span><span class="p">,</span>
-    <span class="n">posterior_rho_init</span><span class="o">=-</span><span class="mf">7.0</span><span class="p">,</span>
-    <span class="n">pred_dim</span><span class="o">=</span><span class="mi">1</span>
-<span class="p">)</span>
-</code></pre></div>
+
 
     <div class="doc doc-contents first">
             <p class="doc doc-class-bases">
@@ -1879,141 +2053,200 @@ <h2 id="pytorch_widedeep.bayesian_models.tabular.bayesian_mlp.bayesian_tab_mlp.B
 
 
 <p><span class="doc-section-title">Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>column_idx</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, int]</code>)
-          –
-          <div class="doc-md-description">
-            <p>Dict containing the index of the columns that will be passed through
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+          <th>Default</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>column_idx</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, int]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Dict containing the index of the columns that will be passed through
 the <code>TabMlp</code> model. Required to slice the tensors. e.g. <em>{'education':
 0, 'relationship': 1, 'workclass': 2, ...}</em></p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>cat_embed_input</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.wdtypes.Tuple">Tuple</span>[str, int, int]]]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>List of Tuples with the column name, number of unique values and
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>cat_embed_input</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.wdtypes.Tuple">Tuple</span>[str, int, int]]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>List of Tuples with the column name, number of unique values and
 embedding dimension. e.g. <em>[(education, 11, 32), ...]</em></p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>cat_embed_activation</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>, default:
+              </div>
+            </td>
+            <td>
                   <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Activation function for the categorical embeddings, if any. Currently
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>cat_embed_activation</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Activation function for the categorical embeddings, if any. Currently
 <em>'tanh'</em>, <em>'relu'</em>, <em>'leaky_relu'</em> and <em>'gelu'</em> are supported</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>continuous_cols</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[str]]</code>, default:
+              </div>
+            </td>
+            <td>
                   <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>List with the name of the numeric (aka continuous) columns</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>cont_norm_layer</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[batchnorm, layernorm]]</code>, default:
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>continuous_cols</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[str]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>List with the name of the numeric (aka continuous) columns</p>
+              </div>
+            </td>
+            <td>
                   <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Type of normalization layer applied to the continuous features. Options
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>cont_norm_layer</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[batchnorm, layernorm]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Type of normalization layer applied to the continuous features. Options
 are: 'layernorm', 'batchnorm' or None.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>embed_continuous</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
+              </div>
+            </td>
+            <td>
                   <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Boolean indicating if the continuous columns will be embedded
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>embed_continuous</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Boolean indicating if the continuous columns will be embedded
 (i.e. passed each through a linear layer with or without activation)</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>cont_embed_dim</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[int]</code>, default:
+              </div>
+            </td>
+            <td>
                   <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Size of the continuous embeddings</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>cont_embed_dropout</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>cont_embed_dim</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[int]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Size of the continuous embeddings</p>
+              </div>
+            </td>
+            <td>
                   <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Dropout for the continuous embeddings</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>use_cont_bias</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>cont_embed_dropout</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Dropout for the continuous embeddings</p>
+              </div>
+            </td>
+            <td>
                   <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Boolean indicating if bias will be used for the continuous embeddings</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>cont_embed_activation</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>, default:
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>use_cont_bias</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Boolean indicating if bias will be used for the continuous embeddings</p>
+              </div>
+            </td>
+            <td>
                   <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Activation function for the continuous embeddings if any. Currently
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>cont_embed_activation</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Activation function for the continuous embeddings if any. Currently
 <em>'tanh'</em>, <em>'relu'</em>, <em>'leaky_relu'</em> and <em>'gelu'</em> are supported</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>mlp_hidden_dims</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.List">List</span>[int]</code>, default:
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>mlp_hidden_dims</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.List">List</span>[int]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>List with the number of neurons per dense layer in the mlp.</p>
+              </div>
+            </td>
+            <td>
                   <code>[200, 100]</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>List with the number of neurons per dense layer in the mlp.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>mlp_activation</code></b>
-              (<code>str</code>, default:
-                  <code>&#39;leaky_relu&#39;</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Activation function for the dense layers of the MLP. Currently
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>mlp_activation</code></td>
+            <td>
+                  <code>str</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Activation function for the dense layers of the MLP. Currently
 <em>'tanh'</em>, <em>'relu'</em>, <em>'leaky_relu'</em> and <em>'gelu'</em> are supported</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>prior_sigma_1</code></b>
-              (<code>float</code>, default:
-                  <code>1</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>The prior weight distribution is a scaled mixture of two Gaussian
+              </div>
+            </td>
+            <td>
+                  <code>&#39;leaky_relu&#39;</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>prior_sigma_1</code></td>
+            <td>
+                  <code>float</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>The prior weight distribution is a scaled mixture of two Gaussian
 densities:</p>
 <div class="arithmatex">\[
    \begin{aligned}
@@ -2023,40 +2256,52 @@ <h2 id="pytorch_widedeep.bayesian_models.tabular.bayesian_mlp.bayesian_tab_mlp.B
 <p><code>prior_sigma_1</code> is the prior of the sigma parameter for the first of the two
 Gaussians that will be mixed to produce the prior weight
 distribution.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>prior_sigma_2</code></b>
-              (<code>float</code>, default:
-                  <code>0.002</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Prior of the sigma parameter for the second of the two Gaussian
+              </div>
+            </td>
+            <td>
+                  <code>1</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>prior_sigma_2</code></td>
+            <td>
+                  <code>float</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Prior of the sigma parameter for the second of the two Gaussian
 distributions that will be mixed to produce the prior weight
 distribution for each Bayesian linear and embedding layer</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>prior_pi</code></b>
-              (<code>float</code>, default:
-                  <code>0.8</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Scaling factor that will be used to mix the Gaussians to produce the
+              </div>
+            </td>
+            <td>
+                  <code>0.002</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>prior_pi</code></td>
+            <td>
+                  <code>float</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Scaling factor that will be used to mix the Gaussians to produce the
 prior weight distribution ffor each Bayesian linear and embedding
 layer</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>posterior_mu_init</code></b>
-              (<code>float</code>, default:
-                  <code>0.0</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>The posterior sample of the weights is defined as:</p>
+              </div>
+            </td>
+            <td>
+                  <code>0.8</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>posterior_mu_init</code></td>
+            <td>
+                  <code>float</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>The posterior sample of the weights is defined as:</p>
 <p>$$
    \begin{aligned}
    \mathbf{w} &amp;= \mu + log(1 + exp(\rho))
@@ -2071,43 +2316,67 @@ <h2 id="pytorch_widedeep.bayesian_models.tabular.bayesian_mlp.bayesian_tab_mlp.B
 \]</div>
 <p><span class="arithmatex">\(\mu\)</span> is initialised using a normal distributtion with mean
 <code>posterior_mu_init</code> and std equal to 0.1.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>posterior_rho_init</code></b>
-              (<code>float</code>, default:
-                  <code>-7.0</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>As in the case of <span class="arithmatex">\(\mu\)</span>, <span class="arithmatex">\(\rho\)</span> is initialised using a
+              </div>
+            </td>
+            <td>
+                  <code>0.0</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>posterior_rho_init</code></td>
+            <td>
+                  <code>float</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>As in the case of <span class="arithmatex">\(\mu\)</span>, <span class="arithmatex">\(\rho\)</span> is initialised using a
 normal distributtion with mean <code>posterior_rho_init</code> and std equal to
 0.1.</p>
-          </div>
-        </li>
-    </ul>
+              </div>
+            </td>
+            <td>
+                  <code>-7.0</code>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
 
 <p><span class="doc-section-title">Attributes:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code><span title="pytorch_widedeep.bayesian_models.tabular.bayesian_mlp.bayesian_tab_mlp.BayesianTabMlp.bayesian_cat_and_cont_embed">bayesian_cat_and_cont_embed</span></code></b>
-              (<code><span title="torch.nn.Module">Module</span></code>)
-          –
-          <div class="doc-md-description">
-            <p>This is the module that processes the categorical and continuous columns</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code><span title="pytorch_widedeep.bayesian_models.tabular.bayesian_mlp.bayesian_tab_mlp.BayesianTabMlp.bayesian_tab_mlp">bayesian_tab_mlp</span></code></b>
-              (<code><span title="torch.nn.Sequential">Sequential</span></code>)
-          –
-          <div class="doc-md-description">
-            <p>mlp model that will receive the concatenation of the embeddings and
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code><span title="pytorch_widedeep.bayesian_models.tabular.bayesian_mlp.bayesian_tab_mlp.BayesianTabMlp.bayesian_cat_and_cont_embed">bayesian_cat_and_cont_embed</span></code></td>
+            <td>
+                  <code><span title="torch.nn.Module">Module</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>This is the module that processes the categorical and continuous columns</p>
+              </div>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code><span title="pytorch_widedeep.bayesian_models.tabular.bayesian_mlp.bayesian_tab_mlp.BayesianTabMlp.bayesian_tab_mlp">bayesian_tab_mlp</span></code></td>
+            <td>
+                  <code><span title="torch.nn.Sequential">Sequential</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>mlp model that will receive the concatenation of the embeddings and
 the continuous columns</p>
-          </div>
-        </li>
-    </ul>
+              </div>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
 
 <p><span class="doc-section-title">Examples:</span></p>
@@ -2122,9 +2391,116 @@ <h2 id="pytorch_widedeep.bayesian_models.tabular.bayesian_mlp.bayesian_tab_mlp.B
 <span class="gp">&gt;&gt;&gt; </span><span class="n">out</span> <span class="o">=</span> <span class="n">model</span><span class="p">(</span><span class="n">X_tab</span><span class="p">)</span>
 </code></pre></div>
 
-                  <details class="quote">
-                    <summary>Source code in <code>pytorch_widedeep/bayesian_models/tabular/bayesian_mlp/bayesian_tab_mlp.py</code></summary>
-                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">123</span>
+              <details class="quote">
+                <summary>Source code in <code>pytorch_widedeep/bayesian_models/tabular/bayesian_mlp/bayesian_tab_mlp.py</code></summary>
+                <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 16</span>
+<span class="normal"> 17</span>
+<span class="normal"> 18</span>
+<span class="normal"> 19</span>
+<span class="normal"> 20</span>
+<span class="normal"> 21</span>
+<span class="normal"> 22</span>
+<span class="normal"> 23</span>
+<span class="normal"> 24</span>
+<span class="normal"> 25</span>
+<span class="normal"> 26</span>
+<span class="normal"> 27</span>
+<span class="normal"> 28</span>
+<span class="normal"> 29</span>
+<span class="normal"> 30</span>
+<span class="normal"> 31</span>
+<span class="normal"> 32</span>
+<span class="normal"> 33</span>
+<span class="normal"> 34</span>
+<span class="normal"> 35</span>
+<span class="normal"> 36</span>
+<span class="normal"> 37</span>
+<span class="normal"> 38</span>
+<span class="normal"> 39</span>
+<span class="normal"> 40</span>
+<span class="normal"> 41</span>
+<span class="normal"> 42</span>
+<span class="normal"> 43</span>
+<span class="normal"> 44</span>
+<span class="normal"> 45</span>
+<span class="normal"> 46</span>
+<span class="normal"> 47</span>
+<span class="normal"> 48</span>
+<span class="normal"> 49</span>
+<span class="normal"> 50</span>
+<span class="normal"> 51</span>
+<span class="normal"> 52</span>
+<span class="normal"> 53</span>
+<span class="normal"> 54</span>
+<span class="normal"> 55</span>
+<span class="normal"> 56</span>
+<span class="normal"> 57</span>
+<span class="normal"> 58</span>
+<span class="normal"> 59</span>
+<span class="normal"> 60</span>
+<span class="normal"> 61</span>
+<span class="normal"> 62</span>
+<span class="normal"> 63</span>
+<span class="normal"> 64</span>
+<span class="normal"> 65</span>
+<span class="normal"> 66</span>
+<span class="normal"> 67</span>
+<span class="normal"> 68</span>
+<span class="normal"> 69</span>
+<span class="normal"> 70</span>
+<span class="normal"> 71</span>
+<span class="normal"> 72</span>
+<span class="normal"> 73</span>
+<span class="normal"> 74</span>
+<span class="normal"> 75</span>
+<span class="normal"> 76</span>
+<span class="normal"> 77</span>
+<span class="normal"> 78</span>
+<span class="normal"> 79</span>
+<span class="normal"> 80</span>
+<span class="normal"> 81</span>
+<span class="normal"> 82</span>
+<span class="normal"> 83</span>
+<span class="normal"> 84</span>
+<span class="normal"> 85</span>
+<span class="normal"> 86</span>
+<span class="normal"> 87</span>
+<span class="normal"> 88</span>
+<span class="normal"> 89</span>
+<span class="normal"> 90</span>
+<span class="normal"> 91</span>
+<span class="normal"> 92</span>
+<span class="normal"> 93</span>
+<span class="normal"> 94</span>
+<span class="normal"> 95</span>
+<span class="normal"> 96</span>
+<span class="normal"> 97</span>
+<span class="normal"> 98</span>
+<span class="normal"> 99</span>
+<span class="normal">100</span>
+<span class="normal">101</span>
+<span class="normal">102</span>
+<span class="normal">103</span>
+<span class="normal">104</span>
+<span class="normal">105</span>
+<span class="normal">106</span>
+<span class="normal">107</span>
+<span class="normal">108</span>
+<span class="normal">109</span>
+<span class="normal">110</span>
+<span class="normal">111</span>
+<span class="normal">112</span>
+<span class="normal">113</span>
+<span class="normal">114</span>
+<span class="normal">115</span>
+<span class="normal">116</span>
+<span class="normal">117</span>
+<span class="normal">118</span>
+<span class="normal">119</span>
+<span class="normal">120</span>
+<span class="normal">121</span>
+<span class="normal">122</span>
+<span class="normal">123</span>
 <span class="normal">124</span>
 <span class="normal">125</span>
 <span class="normal">126</span>
@@ -2242,127 +2618,278 @@ <h2 id="pytorch_widedeep.bayesian_models.tabular.bayesian_mlp.bayesian_tab_mlp.B
 <span class="normal">238</span>
 <span class="normal">239</span>
 <span class="normal">240</span>
-<span class="normal">241</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
-    <span class="bp">self</span><span class="p">,</span>
-    <span class="n">column_idx</span><span class="p">:</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">],</span>
-    <span class="o">*</span><span class="p">,</span>
-    <span class="n">cat_embed_input</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="n">Tuple</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">,</span> <span class="nb">int</span><span class="p">]]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">cat_embed_activation</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">continuous_cols</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">embed_continuous</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">cont_embed_dim</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">cont_embed_dropout</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">cont_embed_activation</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">use_cont_bias</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">cont_norm_layer</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Literal</span><span class="p">[</span><span class="s2">&quot;batchnorm&quot;</span><span class="p">,</span> <span class="s2">&quot;layernorm&quot;</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">mlp_hidden_dims</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="p">[</span><span class="mi">200</span><span class="p">,</span> <span class="mi">100</span><span class="p">],</span>
-    <span class="n">mlp_activation</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;leaky_relu&quot;</span><span class="p">,</span>
-    <span class="n">prior_sigma_1</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mi">1</span><span class="p">,</span>
-    <span class="n">prior_sigma_2</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.002</span><span class="p">,</span>
-    <span class="n">prior_pi</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.8</span><span class="p">,</span>
-    <span class="n">posterior_mu_init</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.0</span><span class="p">,</span>
-    <span class="n">posterior_rho_init</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="o">-</span><span class="mf">7.0</span><span class="p">,</span>
-    <span class="n">pred_dim</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span>  <span class="c1"># Bayesian models will require their own trainer and need the output layer</span>
-<span class="p">):</span>
-    <span class="nb">super</span><span class="p">(</span><span class="n">BayesianTabMlp</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
-
-    <span class="bp">self</span><span class="o">.</span><span class="n">column_idx</span> <span class="o">=</span> <span class="n">column_idx</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">cat_embed_input</span> <span class="o">=</span> <span class="n">cat_embed_input</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">cat_embed_activation</span> <span class="o">=</span> <span class="n">cat_embed_activation</span>
-
-    <span class="bp">self</span><span class="o">.</span><span class="n">continuous_cols</span> <span class="o">=</span> <span class="n">continuous_cols</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">cont_norm_layer</span> <span class="o">=</span> <span class="n">cont_norm_layer</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">embed_continuous</span> <span class="o">=</span> <span class="n">embed_continuous</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">cont_embed_dim</span> <span class="o">=</span> <span class="n">cont_embed_dim</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">cont_embed_dropout</span> <span class="o">=</span> <span class="n">cont_embed_dropout</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">use_cont_bias</span> <span class="o">=</span> <span class="n">use_cont_bias</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">cont_embed_activation</span> <span class="o">=</span> <span class="n">cont_embed_activation</span>
-
-    <span class="bp">self</span><span class="o">.</span><span class="n">mlp_hidden_dims</span> <span class="o">=</span> <span class="n">mlp_hidden_dims</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">mlp_activation</span> <span class="o">=</span> <span class="n">mlp_activation</span>
-
-    <span class="bp">self</span><span class="o">.</span><span class="n">prior_sigma_1</span> <span class="o">=</span> <span class="n">prior_sigma_1</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">prior_sigma_2</span> <span class="o">=</span> <span class="n">prior_sigma_2</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">prior_pi</span> <span class="o">=</span> <span class="n">prior_pi</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">posterior_mu_init</span> <span class="o">=</span> <span class="n">posterior_mu_init</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">posterior_rho_init</span> <span class="o">=</span> <span class="n">posterior_rho_init</span>
-
-    <span class="bp">self</span><span class="o">.</span><span class="n">pred_dim</span> <span class="o">=</span> <span class="n">pred_dim</span>
-
-    <span class="n">allowed_activations</span> <span class="o">=</span> <span class="p">[</span><span class="s2">&quot;relu&quot;</span><span class="p">,</span> <span class="s2">&quot;leaky_relu&quot;</span><span class="p">,</span> <span class="s2">&quot;tanh&quot;</span><span class="p">,</span> <span class="s2">&quot;gelu&quot;</span><span class="p">]</span>
-    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_activation</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">allowed_activations</span><span class="p">:</span>
-        <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
-            <span class="s2">&quot;Currently, only the following activation functions are supported &quot;</span>
-            <span class="s2">&quot;for the Bayesian MLP&#39;s dense layers: </span><span class="si">{}</span><span class="s2">. Got &#39;</span><span class="si">{}</span><span class="s2">&#39; instead&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
-                <span class="s2">&quot;, &quot;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">allowed_activations</span><span class="p">),</span>
-                <span class="bp">self</span><span class="o">.</span><span class="n">mlp_activation</span><span class="p">,</span>
+<span class="normal">241</span>
+<span class="normal">242</span>
+<span class="normal">243</span>
+<span class="normal">244</span>
+<span class="normal">245</span>
+<span class="normal">246</span>
+<span class="normal">247</span>
+<span class="normal">248</span>
+<span class="normal">249</span>
+<span class="normal">250</span>
+<span class="normal">251</span>
+<span class="normal">252</span>
+<span class="normal">253</span>
+<span class="normal">254</span>
+<span class="normal">255</span>
+<span class="normal">256</span>
+<span class="normal">257</span>
+<span class="normal">258</span>
+<span class="normal">259</span>
+<span class="normal">260</span>
+<span class="normal">261</span>
+<span class="normal">262</span>
+<span class="normal">263</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">class</span> <span class="nc">BayesianTabMlp</span><span class="p">(</span><span class="n">BaseBayesianModel</span><span class="p">):</span>
+<span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Defines a `BayesianTabMlp` model.</span>
+
+<span class="sd">    This class combines embedding representations of the categorical features</span>
+<span class="sd">    with numerical (aka continuous) features, embedded or not. These are then</span>
+<span class="sd">    passed through a series of probabilistic dense layers (i.e. a MLP).</span>
+
+<span class="sd">    Parameters</span>
+<span class="sd">    ----------</span>
+<span class="sd">    column_idx: Dict</span>
+<span class="sd">        Dict containing the index of the columns that will be passed through</span>
+<span class="sd">        the `TabMlp` model. Required to slice the tensors. e.g. _{&#39;education&#39;:</span>
+<span class="sd">        0, &#39;relationship&#39;: 1, &#39;workclass&#39;: 2, ...}_</span>
+<span class="sd">    cat_embed_input: List, Optional, default = None</span>
+<span class="sd">        List of Tuples with the column name, number of unique values and</span>
+<span class="sd">        embedding dimension. e.g. _[(education, 11, 32), ...]_</span>
+<span class="sd">    cat_embed_activation: Optional, str, default = None,</span>
+<span class="sd">        Activation function for the categorical embeddings, if any. Currently</span>
+<span class="sd">        _&#39;tanh&#39;_, _&#39;relu&#39;_, _&#39;leaky_relu&#39;_ and _&#39;gelu&#39;_ are supported</span>
+<span class="sd">    continuous_cols: List, Optional, default = None</span>
+<span class="sd">        List with the name of the numeric (aka continuous) columns</span>
+<span class="sd">    cont_norm_layer: str, default =  &quot;batchnorm&quot;</span>
+<span class="sd">        Type of normalization layer applied to the continuous features. Options</span>
+<span class="sd">        are: &#39;layernorm&#39;, &#39;batchnorm&#39; or None.</span>
+<span class="sd">    embed_continuous: bool, default = False,</span>
+<span class="sd">        Boolean indicating if the continuous columns will be embedded</span>
+<span class="sd">        (i.e. passed each through a linear layer with or without activation)</span>
+<span class="sd">    cont_embed_dim: int, default = 32,</span>
+<span class="sd">        Size of the continuous embeddings</span>
+<span class="sd">    cont_embed_dropout: float, default = 0.1,</span>
+<span class="sd">        Dropout for the continuous embeddings</span>
+<span class="sd">    use_cont_bias: bool, default = True,</span>
+<span class="sd">        Boolean indicating if bias will be used for the continuous embeddings</span>
+<span class="sd">    cont_embed_activation: Optional, str, default = None,</span>
+<span class="sd">        Activation function for the continuous embeddings if any. Currently</span>
+<span class="sd">        _&#39;tanh&#39;_, _&#39;relu&#39;_, _&#39;leaky_relu&#39;_ and _&#39;gelu&#39;_ are supported</span>
+<span class="sd">    mlp_hidden_dims: List, default = [200, 100]</span>
+<span class="sd">        List with the number of neurons per dense layer in the mlp.</span>
+<span class="sd">    mlp_activation: str, default = &quot;relu&quot;</span>
+<span class="sd">        Activation function for the dense layers of the MLP. Currently</span>
+<span class="sd">        _&#39;tanh&#39;_, _&#39;relu&#39;_, _&#39;leaky_relu&#39;_ and _&#39;gelu&#39;_ are supported</span>
+<span class="sd">    prior_sigma_1: float, default = 1.0</span>
+<span class="sd">        The prior weight distribution is a scaled mixture of two Gaussian</span>
+<span class="sd">        densities:</span>
+
+<span class="sd">        $$</span>
+<span class="sd">           \begin{aligned}</span>
+<span class="sd">           P(\mathbf{w}) = \prod_{i=j} \pi N (\mathbf{w}_j | 0, \sigma_{1}^{2}) + (1 - \pi) N (\mathbf{w}_j | 0, \sigma_{2}^{2})</span>
+<span class="sd">           \end{aligned}</span>
+<span class="sd">        $$</span>
+
+<span class="sd">        `prior_sigma_1` is the prior of the sigma parameter for the first of the two</span>
+<span class="sd">        Gaussians that will be mixed to produce the prior weight</span>
+<span class="sd">        distribution.</span>
+<span class="sd">    prior_sigma_2: float, default = 0.002</span>
+<span class="sd">        Prior of the sigma parameter for the second of the two Gaussian</span>
+<span class="sd">        distributions that will be mixed to produce the prior weight</span>
+<span class="sd">        distribution for each Bayesian linear and embedding layer</span>
+<span class="sd">    prior_pi: float, default = 0.8</span>
+<span class="sd">        Scaling factor that will be used to mix the Gaussians to produce the</span>
+<span class="sd">        prior weight distribution ffor each Bayesian linear and embedding</span>
+<span class="sd">        layer</span>
+<span class="sd">    posterior_mu_init: float = 0.0</span>
+<span class="sd">        The posterior sample of the weights is defined as:</span>
+
+<span class="sd">        $$</span>
+<span class="sd">           \begin{aligned}</span>
+<span class="sd">           \mathbf{w} &amp;= \mu + log(1 + exp(\rho))</span>
+<span class="sd">           \end{aligned}</span>
+<span class="sd">        $$</span>
+<span class="sd">        where:</span>
+
+<span class="sd">        $$</span>
+<span class="sd">           \begin{aligned}</span>
+<span class="sd">           \mathcal{N}(x\vert \mu, \sigma) &amp;= \frac{1}{\sqrt{2\pi}\sigma}e^{-\frac{(x-\mu)^2}{2\sigma^2}}\\</span>
+<span class="sd">           \log{\mathcal{N}(x\vert \mu, \sigma)} &amp;= -\log{\sqrt{2\pi}} -\log{\sigma} -\frac{(x-\mu)^2}{2\sigma^2}\\</span>
+<span class="sd">           \end{aligned}</span>
+<span class="sd">        $$</span>
+
+<span class="sd">        $\mu$ is initialised using a normal distributtion with mean</span>
+<span class="sd">        `posterior_mu_init` and std equal to 0.1.</span>
+<span class="sd">    posterior_rho_init: float = -7.0</span>
+<span class="sd">        As in the case of $\mu$, $\rho$ is initialised using a</span>
+<span class="sd">        normal distributtion with mean `posterior_rho_init` and std equal to</span>
+<span class="sd">        0.1.</span>
+
+<span class="sd">    Attributes</span>
+<span class="sd">    ----------</span>
+<span class="sd">    bayesian_cat_and_cont_embed: nn.Module</span>
+<span class="sd">        This is the module that processes the categorical and continuous columns</span>
+<span class="sd">    bayesian_tab_mlp: nn.Sequential</span>
+<span class="sd">        mlp model that will receive the concatenation of the embeddings and</span>
+<span class="sd">        the continuous columns</span>
+
+<span class="sd">    Examples</span>
+<span class="sd">    --------</span>
+<span class="sd">    &gt;&gt;&gt; import torch</span>
+<span class="sd">    &gt;&gt;&gt; from pytorch_widedeep.bayesian_models import BayesianTabMlp</span>
+<span class="sd">    &gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)</span>
+<span class="sd">    &gt;&gt;&gt; colnames = [&#39;a&#39;, &#39;b&#39;, &#39;c&#39;, &#39;d&#39;, &#39;e&#39;]</span>
+<span class="sd">    &gt;&gt;&gt; cat_embed_input = [(u,i,j) for u,i,j in zip(colnames[:4], [4]*4, [8]*4)]</span>
+<span class="sd">    &gt;&gt;&gt; column_idx = {k:v for v,k in enumerate(colnames)}</span>
+<span class="sd">    &gt;&gt;&gt; model = BayesianTabMlp(mlp_hidden_dims=[8,4], column_idx=column_idx, cat_embed_input=cat_embed_input,</span>
+<span class="sd">    ... continuous_cols = [&#39;e&#39;])</span>
+<span class="sd">    &gt;&gt;&gt; out = model(X_tab)</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+
+    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span>
+        <span class="n">column_idx</span><span class="p">:</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">],</span>
+        <span class="o">*</span><span class="p">,</span>
+        <span class="n">cat_embed_input</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="n">Tuple</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">,</span> <span class="nb">int</span><span class="p">]]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">cat_embed_activation</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">continuous_cols</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">embed_continuous</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">cont_embed_dim</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">cont_embed_dropout</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">cont_embed_activation</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">use_cont_bias</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">cont_norm_layer</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Literal</span><span class="p">[</span><span class="s2">&quot;batchnorm&quot;</span><span class="p">,</span> <span class="s2">&quot;layernorm&quot;</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">mlp_hidden_dims</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="p">[</span><span class="mi">200</span><span class="p">,</span> <span class="mi">100</span><span class="p">],</span>
+        <span class="n">mlp_activation</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;leaky_relu&quot;</span><span class="p">,</span>
+        <span class="n">prior_sigma_1</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mi">1</span><span class="p">,</span>
+        <span class="n">prior_sigma_2</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.002</span><span class="p">,</span>
+        <span class="n">prior_pi</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.8</span><span class="p">,</span>
+        <span class="n">posterior_mu_init</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.0</span><span class="p">,</span>
+        <span class="n">posterior_rho_init</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="o">-</span><span class="mf">7.0</span><span class="p">,</span>
+        <span class="n">pred_dim</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span>  <span class="c1"># Bayesian models will require their own trainer and need the output layer</span>
+    <span class="p">):</span>
+        <span class="nb">super</span><span class="p">(</span><span class="n">BayesianTabMlp</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">column_idx</span> <span class="o">=</span> <span class="n">column_idx</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">cat_embed_input</span> <span class="o">=</span> <span class="n">cat_embed_input</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">cat_embed_activation</span> <span class="o">=</span> <span class="n">cat_embed_activation</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">continuous_cols</span> <span class="o">=</span> <span class="n">continuous_cols</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">cont_norm_layer</span> <span class="o">=</span> <span class="n">cont_norm_layer</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">embed_continuous</span> <span class="o">=</span> <span class="n">embed_continuous</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">cont_embed_dim</span> <span class="o">=</span> <span class="n">cont_embed_dim</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">cont_embed_dropout</span> <span class="o">=</span> <span class="n">cont_embed_dropout</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">use_cont_bias</span> <span class="o">=</span> <span class="n">use_cont_bias</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">cont_embed_activation</span> <span class="o">=</span> <span class="n">cont_embed_activation</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">mlp_hidden_dims</span> <span class="o">=</span> <span class="n">mlp_hidden_dims</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">mlp_activation</span> <span class="o">=</span> <span class="n">mlp_activation</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">prior_sigma_1</span> <span class="o">=</span> <span class="n">prior_sigma_1</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">prior_sigma_2</span> <span class="o">=</span> <span class="n">prior_sigma_2</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">prior_pi</span> <span class="o">=</span> <span class="n">prior_pi</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">posterior_mu_init</span> <span class="o">=</span> <span class="n">posterior_mu_init</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">posterior_rho_init</span> <span class="o">=</span> <span class="n">posterior_rho_init</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">pred_dim</span> <span class="o">=</span> <span class="n">pred_dim</span>
+
+        <span class="n">allowed_activations</span> <span class="o">=</span> <span class="p">[</span><span class="s2">&quot;relu&quot;</span><span class="p">,</span> <span class="s2">&quot;leaky_relu&quot;</span><span class="p">,</span> <span class="s2">&quot;tanh&quot;</span><span class="p">,</span> <span class="s2">&quot;gelu&quot;</span><span class="p">]</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_activation</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">allowed_activations</span><span class="p">:</span>
+            <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
+                <span class="s2">&quot;Currently, only the following activation functions are supported &quot;</span>
+                <span class="s2">&quot;for the Bayesian MLP&#39;s dense layers: </span><span class="si">{}</span><span class="s2">. Got &#39;</span><span class="si">{}</span><span class="s2">&#39; instead&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
+                    <span class="s2">&quot;, &quot;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">allowed_activations</span><span class="p">),</span>
+                    <span class="bp">self</span><span class="o">.</span><span class="n">mlp_activation</span><span class="p">,</span>
+                <span class="p">)</span>
             <span class="p">)</span>
-        <span class="p">)</span>
 
-    <span class="c1"># Categorical</span>
-    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">cat_embed_input</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">cat_embed</span> <span class="o">=</span> <span class="n">BayesianDiffSizeCatEmbeddings</span><span class="p">(</span>
-            <span class="n">column_idx</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">column_idx</span><span class="p">,</span>
-            <span class="n">embed_input</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">cat_embed_input</span><span class="p">,</span>
-            <span class="n">prior_sigma_1</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">prior_sigma_1</span><span class="p">,</span>
-            <span class="n">prior_sigma_2</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">prior_sigma_2</span><span class="p">,</span>
-            <span class="n">prior_pi</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">prior_pi</span><span class="p">,</span>
-            <span class="n">posterior_mu_init</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">posterior_mu_init</span><span class="p">,</span>
-            <span class="n">posterior_rho_init</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">posterior_rho_init</span><span class="p">,</span>
-            <span class="n">activation_fn</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">cat_embed_activation</span><span class="p">,</span>
-        <span class="p">)</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">cat_out_dim</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">sum</span><span class="p">([</span><span class="n">embed</span><span class="p">[</span><span class="mi">2</span><span class="p">]</span> <span class="k">for</span> <span class="n">embed</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">cat_embed_input</span><span class="p">]))</span>
-    <span class="k">else</span><span class="p">:</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">cat_out_dim</span> <span class="o">=</span> <span class="mi">0</span>
-
-    <span class="c1"># Continuous</span>
-    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">continuous_cols</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">cont_idx</span> <span class="o">=</span> <span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">column_idx</span><span class="p">[</span><span class="n">col</span><span class="p">]</span> <span class="k">for</span> <span class="n">col</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">continuous_cols</span><span class="p">]</span>
-        <span class="k">if</span> <span class="n">cont_norm_layer</span> <span class="o">==</span> <span class="s2">&quot;layernorm&quot;</span><span class="p">:</span>
-            <span class="bp">self</span><span class="o">.</span><span class="n">cont_norm</span><span class="p">:</span> <span class="n">NormLayers</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">LayerNorm</span><span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">continuous_cols</span><span class="p">))</span>
-        <span class="k">elif</span> <span class="n">cont_norm_layer</span> <span class="o">==</span> <span class="s2">&quot;batchnorm&quot;</span><span class="p">:</span>
-            <span class="bp">self</span><span class="o">.</span><span class="n">cont_norm</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">BatchNorm1d</span><span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">continuous_cols</span><span class="p">))</span>
-        <span class="k">else</span><span class="p">:</span>
-            <span class="bp">self</span><span class="o">.</span><span class="n">cont_norm</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Identity</span><span class="p">()</span>
-        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">embed_continuous</span><span class="p">:</span>
-            <span class="k">assert</span> <span class="bp">self</span><span class="o">.</span><span class="n">cont_embed_dim</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">,</span> <span class="p">(</span>
-                <span class="s2">&quot;If &#39;embed_continuous&#39; is True, &#39;cont_embed_dim&#39; must be &quot;</span>
-                <span class="s2">&quot;provided&quot;</span>
-            <span class="p">)</span>
-            <span class="bp">self</span><span class="o">.</span><span class="n">cont_embed</span> <span class="o">=</span> <span class="n">BayesianContEmbeddings</span><span class="p">(</span>
-                <span class="n">n_cont_cols</span><span class="o">=</span><span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">continuous_cols</span><span class="p">),</span>
-                <span class="n">embed_dim</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">cont_embed_dim</span><span class="p">,</span>
+        <span class="c1"># Categorical</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">cat_embed_input</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">cat_embed</span> <span class="o">=</span> <span class="n">BayesianDiffSizeCatEmbeddings</span><span class="p">(</span>
+                <span class="n">column_idx</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">column_idx</span><span class="p">,</span>
+                <span class="n">embed_input</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">cat_embed_input</span><span class="p">,</span>
                 <span class="n">prior_sigma_1</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">prior_sigma_1</span><span class="p">,</span>
                 <span class="n">prior_sigma_2</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">prior_sigma_2</span><span class="p">,</span>
                 <span class="n">prior_pi</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">prior_pi</span><span class="p">,</span>
                 <span class="n">posterior_mu_init</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">posterior_mu_init</span><span class="p">,</span>
                 <span class="n">posterior_rho_init</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">posterior_rho_init</span><span class="p">,</span>
-                <span class="n">use_bias</span><span class="o">=</span><span class="p">(</span>
-                    <span class="kc">False</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">use_cont_bias</span> <span class="ow">is</span> <span class="kc">None</span> <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">use_cont_bias</span>
-                <span class="p">),</span>
-                <span class="n">activation_fn</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">cont_embed_activation</span><span class="p">,</span>
+                <span class="n">activation_fn</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">cat_embed_activation</span><span class="p">,</span>
             <span class="p">)</span>
-            <span class="bp">self</span><span class="o">.</span><span class="n">cont_out_dim</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">continuous_cols</span><span class="p">)</span> <span class="o">*</span> <span class="bp">self</span><span class="o">.</span><span class="n">cont_embed_dim</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">cat_out_dim</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">sum</span><span class="p">([</span><span class="n">embed</span><span class="p">[</span><span class="mi">2</span><span class="p">]</span> <span class="k">for</span> <span class="n">embed</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">cat_embed_input</span><span class="p">]))</span>
         <span class="k">else</span><span class="p">:</span>
-            <span class="bp">self</span><span class="o">.</span><span class="n">cont_out_dim</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">continuous_cols</span><span class="p">)</span>
-    <span class="k">else</span><span class="p">:</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">cont_out_dim</span> <span class="o">=</span> <span class="mi">0</span>
-
-    <span class="bp">self</span><span class="o">.</span><span class="n">output_dim</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">cat_out_dim</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">cont_out_dim</span>
-
-    <span class="n">mlp_hidden_dims</span> <span class="o">=</span> <span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">output_dim</span><span class="p">]</span> <span class="o">+</span> <span class="n">mlp_hidden_dims</span> <span class="o">+</span> <span class="p">[</span><span class="n">pred_dim</span><span class="p">]</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">bayesian_tab_mlp</span> <span class="o">=</span> <span class="n">BayesianMLP</span><span class="p">(</span>
-        <span class="n">mlp_hidden_dims</span><span class="p">,</span>
-        <span class="n">mlp_activation</span><span class="p">,</span>
-        <span class="kc">True</span><span class="p">,</span>  <span class="c1"># use_bias</span>
-        <span class="n">prior_sigma_1</span><span class="p">,</span>
-        <span class="n">prior_sigma_2</span><span class="p">,</span>
-        <span class="n">prior_pi</span><span class="p">,</span>
-        <span class="n">posterior_mu_init</span><span class="p">,</span>
-        <span class="n">posterior_rho_init</span><span class="p">,</span>
-    <span class="p">)</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">cat_out_dim</span> <span class="o">=</span> <span class="mi">0</span>
+
+        <span class="c1"># Continuous</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">continuous_cols</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">cont_idx</span> <span class="o">=</span> <span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">column_idx</span><span class="p">[</span><span class="n">col</span><span class="p">]</span> <span class="k">for</span> <span class="n">col</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">continuous_cols</span><span class="p">]</span>
+            <span class="k">if</span> <span class="n">cont_norm_layer</span> <span class="o">==</span> <span class="s2">&quot;layernorm&quot;</span><span class="p">:</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">cont_norm</span><span class="p">:</span> <span class="n">NormLayers</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">LayerNorm</span><span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">continuous_cols</span><span class="p">))</span>
+            <span class="k">elif</span> <span class="n">cont_norm_layer</span> <span class="o">==</span> <span class="s2">&quot;batchnorm&quot;</span><span class="p">:</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">cont_norm</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">BatchNorm1d</span><span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">continuous_cols</span><span class="p">))</span>
+            <span class="k">else</span><span class="p">:</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">cont_norm</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Identity</span><span class="p">()</span>
+            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">embed_continuous</span><span class="p">:</span>
+                <span class="k">assert</span> <span class="bp">self</span><span class="o">.</span><span class="n">cont_embed_dim</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">,</span> <span class="p">(</span>
+                    <span class="s2">&quot;If &#39;embed_continuous&#39; is True, &#39;cont_embed_dim&#39; must be &quot;</span>
+                    <span class="s2">&quot;provided&quot;</span>
+                <span class="p">)</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">cont_embed</span> <span class="o">=</span> <span class="n">BayesianContEmbeddings</span><span class="p">(</span>
+                    <span class="n">n_cont_cols</span><span class="o">=</span><span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">continuous_cols</span><span class="p">),</span>
+                    <span class="n">embed_dim</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">cont_embed_dim</span><span class="p">,</span>
+                    <span class="n">prior_sigma_1</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">prior_sigma_1</span><span class="p">,</span>
+                    <span class="n">prior_sigma_2</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">prior_sigma_2</span><span class="p">,</span>
+                    <span class="n">prior_pi</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">prior_pi</span><span class="p">,</span>
+                    <span class="n">posterior_mu_init</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">posterior_mu_init</span><span class="p">,</span>
+                    <span class="n">posterior_rho_init</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">posterior_rho_init</span><span class="p">,</span>
+                    <span class="n">use_bias</span><span class="o">=</span><span class="p">(</span>
+                        <span class="kc">False</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">use_cont_bias</span> <span class="ow">is</span> <span class="kc">None</span> <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">use_cont_bias</span>
+                    <span class="p">),</span>
+                    <span class="n">activation_fn</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">cont_embed_activation</span><span class="p">,</span>
+                <span class="p">)</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">cont_out_dim</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">continuous_cols</span><span class="p">)</span> <span class="o">*</span> <span class="bp">self</span><span class="o">.</span><span class="n">cont_embed_dim</span>
+            <span class="k">else</span><span class="p">:</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">cont_out_dim</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">continuous_cols</span><span class="p">)</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">cont_out_dim</span> <span class="o">=</span> <span class="mi">0</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">output_dim</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">cat_out_dim</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">cont_out_dim</span>
+
+        <span class="n">mlp_hidden_dims</span> <span class="o">=</span> <span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">output_dim</span><span class="p">]</span> <span class="o">+</span> <span class="n">mlp_hidden_dims</span> <span class="o">+</span> <span class="p">[</span><span class="n">pred_dim</span><span class="p">]</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">bayesian_tab_mlp</span> <span class="o">=</span> <span class="n">BayesianMLP</span><span class="p">(</span>
+            <span class="n">mlp_hidden_dims</span><span class="p">,</span>
+            <span class="n">mlp_activation</span><span class="p">,</span>
+            <span class="kc">True</span><span class="p">,</span>  <span class="c1"># use_bias</span>
+            <span class="n">prior_sigma_1</span><span class="p">,</span>
+            <span class="n">prior_sigma_2</span><span class="p">,</span>
+            <span class="n">prior_pi</span><span class="p">,</span>
+            <span class="n">posterior_mu_init</span><span class="p">,</span>
+            <span class="n">posterior_rho_init</span><span class="p">,</span>
+        <span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">forward</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">X</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tensor</span><span class="p">:</span>
+        <span class="n">x</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_embeddings</span><span class="p">(</span><span class="n">X</span><span class="p">)</span>
+        <span class="n">x</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">bayesian_tab_mlp</span><span class="p">(</span><span class="n">x</span><span class="p">)</span>
+        <span class="k">return</span> <span class="n">x</span>
+
+    <span class="k">def</span> <span class="nf">_get_embeddings</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">X</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tensor</span><span class="p">:</span>
+        <span class="n">tensors_to_concat</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="n">Tensor</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">cat_embed_input</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="n">x_cat</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">cat_embed</span><span class="p">(</span><span class="n">X</span><span class="p">)</span>
+            <span class="n">tensors_to_concat</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">x_cat</span><span class="p">)</span>
+
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">continuous_cols</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="n">x_cont</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">cont_norm</span><span class="p">((</span><span class="n">X</span><span class="p">[:,</span> <span class="bp">self</span><span class="o">.</span><span class="n">cont_idx</span><span class="p">]</span><span class="o">.</span><span class="n">float</span><span class="p">()))</span>
+            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">embed_continuous</span><span class="p">:</span>
+                <span class="n">x_cont</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">cont_embed</span><span class="p">(</span><span class="n">x_cont</span><span class="p">)</span>
+                <span class="n">x_cont</span> <span class="o">=</span> <span class="n">einops</span><span class="o">.</span><span class="n">rearrange</span><span class="p">(</span><span class="n">x_cont</span><span class="p">,</span> <span class="s2">&quot;b s d -&gt; b (s d)&quot;</span><span class="p">)</span>
+            <span class="n">tensors_to_concat</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">x_cont</span><span class="p">)</span>
+
+        <span class="n">x</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">cat</span><span class="p">(</span><span class="n">tensors_to_concat</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span>
+
+        <span class="k">return</span> <span class="n">x</span>
 </code></pre></div></td></tr></table></div>
-                  </details>
+              </details>
 
 
 
diff --git a/mkdocs/site/pytorch-widedeep/bayesian_models.md b/mkdocs/site/pytorch-widedeep/bayesian_models.md
index 2b48c267..63ea1837 100644
--- a/mkdocs/site/pytorch-widedeep/bayesian_models.md
+++ b/mkdocs/site/pytorch-widedeep/bayesian_models.md
@@ -10,13 +10,5 @@ The models in this module are based on the publication:
 
 
 ::: pytorch_widedeep.bayesian_models.tabular.bayesian_linear.bayesian_wide.BayesianWide
-    selection:
-        filters:
-            - "!^_"  # exclude all members starting with _
-            - "!^forward$"
 
 ::: pytorch_widedeep.bayesian_models.tabular.bayesian_mlp.bayesian_tab_mlp.BayesianTabMlp
-    selection:
-        filters:
-            - "!^_"  # exclude all members starting with _
-            - "!^forward$"
diff --git a/mkdocs/site/pytorch-widedeep/bayesian_trainer.html b/mkdocs/site/pytorch-widedeep/bayesian_trainer.html
index 2b758b4b..aa29a595 100644
--- a/mkdocs/site/pytorch-widedeep/bayesian_trainer.html
+++ b/mkdocs/site/pytorch-widedeep/bayesian_trainer.html
@@ -1387,7 +1387,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/16_Self_Supervised_Pretraning_pt1.ipynb" class="md-nav__link">
+      <a href="../examples/15_Self_Supervised_Pretraning_pt1.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1408,7 +1408,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/16_Self_Supervised_Pretraning_pt2.ipynb" class="md-nav__link">
+      <a href="../examples/15_Self_Supervised_Pretraning_pt2.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1429,7 +1429,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/17_Usign_a_custom_hugging_face_model.ipynb" class="md-nav__link">
+      <a href="../examples/16_Usign_a_custom_hugging_face_model.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1450,7 +1450,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/18_feature_importance_via_attention_weights.ipynb" class="md-nav__link">
+      <a href="../examples/17_feature_importance_via_attention_weights.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1471,7 +1471,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/19_wide_and_deep_for_recsys_pt1.ipynb" class="md-nav__link">
+      <a href="../examples/18_wide_and_deep_for_recsys_pt1.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1492,7 +1492,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/19_wide_and_deep_for_recsys_pt2.ipynb" class="md-nav__link">
+      <a href="../examples/18_wide_and_deep_for_recsys_pt2.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1513,7 +1513,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/20_load_from_folder_functionality.ipynb" class="md-nav__link">
+      <a href="../examples/19_load_from_folder_functionality.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1534,7 +1534,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/21_Using_huggingface_within_widedeep.ipynb" class="md-nav__link">
+      <a href="../examples/20_Using_huggingface_within_widedeep.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1605,19 +1605,7 @@ <h2 id="pytorch_widedeep.training.BayesianTrainer" class="doc doc-heading">
 
 
 <a href="#pytorch_widedeep.training.BayesianTrainer" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="nf">BayesianTrainer</span><span class="p">(</span>
-    <span class="n">model</span><span class="p">,</span>
-    <span class="n">objective</span><span class="p">,</span>
-    <span class="n">custom_loss_function</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">optimizer</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">lr_scheduler</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">callbacks</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">metrics</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">verbose</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span>
-    <span class="n">seed</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span>
-    <span class="o">**</span><span class="n">kwargs</span>
-<span class="p">)</span>
-</code></pre></div>
+
 
     <div class="doc doc-contents first">
             <p class="doc doc-class-bases">
@@ -1631,84 +1619,121 @@ <h2 id="pytorch_widedeep.training.BayesianTrainer" class="doc doc-heading">
 
 
 <p><span class="doc-section-title">Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>model</code></b>
-              (<code><span title="pytorch_widedeep.bayesian_models._base_bayesian_model.BaseBayesianModel">BaseBayesianModel</span></code>)
-          –
-          <div class="doc-md-description">
-            <p>An object of class <code>BaseBayesianModel</code>. See the <code>Model Components</code>
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+          <th>Default</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>model</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.bayesian_models._base_bayesian_model.BaseBayesianModel">BaseBayesianModel</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>An object of class <code>BaseBayesianModel</code>. See the <code>Model Components</code>
 section here in the docs.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>objective</code></b>
-              (<code>str</code>)
-          –
-          <div class="doc-md-description">
-            <p>Defines the objective, loss or cost function.<br/>
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>objective</code></td>
+            <td>
+                  <code>str</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Defines the objective, loss or cost function.<br/>
 Param aliases: <code>loss_function</code>, <code>loss_fn</code>, <code>loss</code>,
 <code>cost_function</code>, <code>cost_fn</code>, <code>cost</code><br/>
 Possible values are: <em>'binary'</em>, <em>'multiclass'</em>, <em>'regression'</em></p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>custom_loss_function</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Module">Module</span>]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>If none of the loss functions available suits the user, it is possible
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>custom_loss_function</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Module">Module</span>]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>If none of the loss functions available suits the user, it is possible
 to pass a custom loss function. See for example
 <code>pytorch_widedeep.losses.FocalLoss</code> for the required structure of the
 object or the Examples folder in the repo.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>optimizer</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Optimizer">Optimizer</span>]</code>, default:
+              </div>
+            </td>
+            <td>
                   <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>An instance of Pytorch's <code>Optimizer</code> object(e.g. <code>torch.optim.Adam
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>optimizer</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Optimizer">Optimizer</span>]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>An instance of Pytorch's <code>Optimizer</code> object(e.g. <code>torch.optim.Adam
 ()</code>). if no optimizer is passed it will default to <code>AdamW</code>.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>lr_scheduler</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.LRScheduler">LRScheduler</span>]</code>, default:
+              </div>
+            </td>
+            <td>
                   <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>An instance of Pytorch's <code>LRScheduler</code> object
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>lr_scheduler</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.LRScheduler">LRScheduler</span>]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>An instance of Pytorch's <code>LRScheduler</code> object
 (e.g <code>torch.optim.lr_scheduler.StepLR(opt, step_size=5)</code>).</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>callbacks</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.callbacks.Callback">Callback</span>]]</code>, default:
+              </div>
+            </td>
+            <td>
                   <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>List with <code>Callback</code> objects. The three callbacks available in
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>callbacks</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.callbacks.Callback">Callback</span>]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>List with <code>Callback</code> objects. The three callbacks available in
 <code>pytorch-widedeep</code> are: <code>LRHistory</code>, <code>ModelCheckpoint</code> and
 <code>EarlyStopping</code>. This can also be a custom callback. See
 <code>pytorch_widedeep.callbacks.Callback</code> or the Examples folder in the
 repo.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>metrics</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.metrics.Metric">Metric</span>], <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="torchmetrics.Metric">Metric</span>]]]</code>, default:
+              </div>
+            </td>
+            <td>
                   <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <ul>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>metrics</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.metrics.Metric">Metric</span>], <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="torchmetrics.Metric">Metric</span>]]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <ul>
 <li>List of objects of type <code>Metric</code>. Metrics available are:
   <code>Accuracy</code>, <code>Precision</code>, <code>Recall</code>, <code>FBetaScore</code>,
   <code>F1Score</code> and <code>R2Score</code>. This can also be a custom metric as
@@ -1720,38 +1745,61 @@ <h2 id="pytorch_widedeep.training.BayesianTrainer" class="doc doc-heading">
   long as it is an object of type</code>Metric<code>.
   See</code>the <a href="(https://lightning.ai/docs/torchmetrics)">instructions</a></li>
 </ul>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>verbose</code></b>
-              (<code>int</code>, default:
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>verbose</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Setting it to 0 will print nothing during training.</p>
+              </div>
+            </td>
+            <td>
                   <code>1</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Setting it to 0 will print nothing during training.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>seed</code></b>
-              (<code>int</code>, default:
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>seed</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Random seed to be used internally for train_test_split</p>
+              </div>
+            </td>
+            <td>
                   <code>1</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Random seed to be used internally for train_test_split</p>
-          </div>
-        </li>
-    </ul>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
 
 <p><span class="doc-section-title">Other Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>**kwargs</code></b>
-          –
-          <div class="doc-md-description">
-            <p>Other infrequently used arguments that can also be passed as kwargs are:</p>
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>**kwargs</code></td>
+            <td>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Other infrequently used arguments that can also be passed as kwargs are:</p>
 <ul>
 <li>
 <p><strong>device</strong>: <code>str</code><br/>
@@ -1774,28 +1822,121 @@ <h2 id="pytorch_widedeep.training.BayesianTrainer" class="doc doc-heading">
     learning rate is a bit particular.</p>
 </li>
 </ul>
-          </div>
-        </li>
-    </ul>
+              </div>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
 
 <p><span class="doc-section-title">Attributes:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code><span title="pytorch_widedeep.training.BayesianTrainer.cyclic_lr">cyclic_lr</span></code></b>
-              (<code>bool</code>)
-          –
-          <div class="doc-md-description">
-            <p>Attribute that indicates if  the lr_scheduler is cyclic_lr
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code><span title="pytorch_widedeep.training.BayesianTrainer.cyclic_lr">cyclic_lr</span></code></td>
+            <td>
+                  <code>bool</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Attribute that indicates if  the lr_scheduler is cyclic_lr
 (i.e. <code>CyclicLR</code> or <code>OneCycleLR</code>). See <code>Pytorch schedulers
 &lt;https://pytorch.org/docs/stable/optim.html&gt;</code>_.</p>
-          </div>
-        </li>
-    </ul>
-
-                  <details class="quote">
-                    <summary>Source code in <code>pytorch_widedeep/training/bayesian_trainer.py</code></summary>
-                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">112</span>
+              </div>
+            </td>
+          </tr>
+      </tbody>
+    </table>
+
+              <details class="quote">
+                <summary>Source code in <code>pytorch_widedeep/training/bayesian_trainer.py</code></summary>
+                <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 33</span>
+<span class="normal"> 34</span>
+<span class="normal"> 35</span>
+<span class="normal"> 36</span>
+<span class="normal"> 37</span>
+<span class="normal"> 38</span>
+<span class="normal"> 39</span>
+<span class="normal"> 40</span>
+<span class="normal"> 41</span>
+<span class="normal"> 42</span>
+<span class="normal"> 43</span>
+<span class="normal"> 44</span>
+<span class="normal"> 45</span>
+<span class="normal"> 46</span>
+<span class="normal"> 47</span>
+<span class="normal"> 48</span>
+<span class="normal"> 49</span>
+<span class="normal"> 50</span>
+<span class="normal"> 51</span>
+<span class="normal"> 52</span>
+<span class="normal"> 53</span>
+<span class="normal"> 54</span>
+<span class="normal"> 55</span>
+<span class="normal"> 56</span>
+<span class="normal"> 57</span>
+<span class="normal"> 58</span>
+<span class="normal"> 59</span>
+<span class="normal"> 60</span>
+<span class="normal"> 61</span>
+<span class="normal"> 62</span>
+<span class="normal"> 63</span>
+<span class="normal"> 64</span>
+<span class="normal"> 65</span>
+<span class="normal"> 66</span>
+<span class="normal"> 67</span>
+<span class="normal"> 68</span>
+<span class="normal"> 69</span>
+<span class="normal"> 70</span>
+<span class="normal"> 71</span>
+<span class="normal"> 72</span>
+<span class="normal"> 73</span>
+<span class="normal"> 74</span>
+<span class="normal"> 75</span>
+<span class="normal"> 76</span>
+<span class="normal"> 77</span>
+<span class="normal"> 78</span>
+<span class="normal"> 79</span>
+<span class="normal"> 80</span>
+<span class="normal"> 81</span>
+<span class="normal"> 82</span>
+<span class="normal"> 83</span>
+<span class="normal"> 84</span>
+<span class="normal"> 85</span>
+<span class="normal"> 86</span>
+<span class="normal"> 87</span>
+<span class="normal"> 88</span>
+<span class="normal"> 89</span>
+<span class="normal"> 90</span>
+<span class="normal"> 91</span>
+<span class="normal"> 92</span>
+<span class="normal"> 93</span>
+<span class="normal"> 94</span>
+<span class="normal"> 95</span>
+<span class="normal"> 96</span>
+<span class="normal"> 97</span>
+<span class="normal"> 98</span>
+<span class="normal"> 99</span>
+<span class="normal">100</span>
+<span class="normal">101</span>
+<span class="normal">102</span>
+<span class="normal">103</span>
+<span class="normal">104</span>
+<span class="normal">105</span>
+<span class="normal">106</span>
+<span class="normal">107</span>
+<span class="normal">108</span>
+<span class="normal">109</span>
+<span class="normal">110</span>
+<span class="normal">111</span>
+<span class="normal">112</span>
 <span class="normal">113</span>
 <span class="normal">114</span>
 <span class="normal">115</span>
@@ -1823,37 +1964,852 @@ <h2 id="pytorch_widedeep.training.BayesianTrainer" class="doc doc-heading">
 <span class="normal">137</span>
 <span class="normal">138</span>
 <span class="normal">139</span>
-<span class="normal">140</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="nd">@alias</span><span class="p">(</span>  <span class="c1"># noqa: C901</span>
-    <span class="s2">&quot;objective&quot;</span><span class="p">,</span>
-    <span class="p">[</span><span class="s2">&quot;loss_function&quot;</span><span class="p">,</span> <span class="s2">&quot;loss_fn&quot;</span><span class="p">,</span> <span class="s2">&quot;loss&quot;</span><span class="p">,</span> <span class="s2">&quot;cost_function&quot;</span><span class="p">,</span> <span class="s2">&quot;cost_fn&quot;</span><span class="p">,</span> <span class="s2">&quot;cost&quot;</span><span class="p">],</span>
-<span class="p">)</span>
-<span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
-    <span class="bp">self</span><span class="p">,</span>
-    <span class="n">model</span><span class="p">:</span> <span class="n">BaseBayesianModel</span><span class="p">,</span>
-    <span class="n">objective</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
-    <span class="n">custom_loss_function</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Module</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">optimizer</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Optimizer</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">lr_scheduler</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">LRScheduler</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">callbacks</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="n">Callback</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">metrics</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Union</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="n">Metric</span><span class="p">],</span> <span class="n">List</span><span class="p">[</span><span class="n">TorchMetric</span><span class="p">]]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">verbose</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1</span><span class="p">,</span>
-    <span class="n">seed</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1</span><span class="p">,</span>
-    <span class="o">**</span><span class="n">kwargs</span><span class="p">,</span>
-<span class="p">):</span>
-    <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span>
-        <span class="n">model</span><span class="o">=</span><span class="n">model</span><span class="p">,</span>
-        <span class="n">objective</span><span class="o">=</span><span class="n">objective</span><span class="p">,</span>
-        <span class="n">custom_loss_function</span><span class="o">=</span><span class="n">custom_loss_function</span><span class="p">,</span>
-        <span class="n">optimizer</span><span class="o">=</span><span class="n">optimizer</span><span class="p">,</span>
-        <span class="n">lr_scheduler</span><span class="o">=</span><span class="n">lr_scheduler</span><span class="p">,</span>
-        <span class="n">callbacks</span><span class="o">=</span><span class="n">callbacks</span><span class="p">,</span>
-        <span class="n">metrics</span><span class="o">=</span><span class="n">metrics</span><span class="p">,</span>
-        <span class="n">verbose</span><span class="o">=</span><span class="n">verbose</span><span class="p">,</span>
-        <span class="n">seed</span><span class="o">=</span><span class="n">seed</span><span class="p">,</span>
-        <span class="o">**</span><span class="n">kwargs</span><span class="p">,</span>
+<span class="normal">140</span>
+<span class="normal">141</span>
+<span class="normal">142</span>
+<span class="normal">143</span>
+<span class="normal">144</span>
+<span class="normal">145</span>
+<span class="normal">146</span>
+<span class="normal">147</span>
+<span class="normal">148</span>
+<span class="normal">149</span>
+<span class="normal">150</span>
+<span class="normal">151</span>
+<span class="normal">152</span>
+<span class="normal">153</span>
+<span class="normal">154</span>
+<span class="normal">155</span>
+<span class="normal">156</span>
+<span class="normal">157</span>
+<span class="normal">158</span>
+<span class="normal">159</span>
+<span class="normal">160</span>
+<span class="normal">161</span>
+<span class="normal">162</span>
+<span class="normal">163</span>
+<span class="normal">164</span>
+<span class="normal">165</span>
+<span class="normal">166</span>
+<span class="normal">167</span>
+<span class="normal">168</span>
+<span class="normal">169</span>
+<span class="normal">170</span>
+<span class="normal">171</span>
+<span class="normal">172</span>
+<span class="normal">173</span>
+<span class="normal">174</span>
+<span class="normal">175</span>
+<span class="normal">176</span>
+<span class="normal">177</span>
+<span class="normal">178</span>
+<span class="normal">179</span>
+<span class="normal">180</span>
+<span class="normal">181</span>
+<span class="normal">182</span>
+<span class="normal">183</span>
+<span class="normal">184</span>
+<span class="normal">185</span>
+<span class="normal">186</span>
+<span class="normal">187</span>
+<span class="normal">188</span>
+<span class="normal">189</span>
+<span class="normal">190</span>
+<span class="normal">191</span>
+<span class="normal">192</span>
+<span class="normal">193</span>
+<span class="normal">194</span>
+<span class="normal">195</span>
+<span class="normal">196</span>
+<span class="normal">197</span>
+<span class="normal">198</span>
+<span class="normal">199</span>
+<span class="normal">200</span>
+<span class="normal">201</span>
+<span class="normal">202</span>
+<span class="normal">203</span>
+<span class="normal">204</span>
+<span class="normal">205</span>
+<span class="normal">206</span>
+<span class="normal">207</span>
+<span class="normal">208</span>
+<span class="normal">209</span>
+<span class="normal">210</span>
+<span class="normal">211</span>
+<span class="normal">212</span>
+<span class="normal">213</span>
+<span class="normal">214</span>
+<span class="normal">215</span>
+<span class="normal">216</span>
+<span class="normal">217</span>
+<span class="normal">218</span>
+<span class="normal">219</span>
+<span class="normal">220</span>
+<span class="normal">221</span>
+<span class="normal">222</span>
+<span class="normal">223</span>
+<span class="normal">224</span>
+<span class="normal">225</span>
+<span class="normal">226</span>
+<span class="normal">227</span>
+<span class="normal">228</span>
+<span class="normal">229</span>
+<span class="normal">230</span>
+<span class="normal">231</span>
+<span class="normal">232</span>
+<span class="normal">233</span>
+<span class="normal">234</span>
+<span class="normal">235</span>
+<span class="normal">236</span>
+<span class="normal">237</span>
+<span class="normal">238</span>
+<span class="normal">239</span>
+<span class="normal">240</span>
+<span class="normal">241</span>
+<span class="normal">242</span>
+<span class="normal">243</span>
+<span class="normal">244</span>
+<span class="normal">245</span>
+<span class="normal">246</span>
+<span class="normal">247</span>
+<span class="normal">248</span>
+<span class="normal">249</span>
+<span class="normal">250</span>
+<span class="normal">251</span>
+<span class="normal">252</span>
+<span class="normal">253</span>
+<span class="normal">254</span>
+<span class="normal">255</span>
+<span class="normal">256</span>
+<span class="normal">257</span>
+<span class="normal">258</span>
+<span class="normal">259</span>
+<span class="normal">260</span>
+<span class="normal">261</span>
+<span class="normal">262</span>
+<span class="normal">263</span>
+<span class="normal">264</span>
+<span class="normal">265</span>
+<span class="normal">266</span>
+<span class="normal">267</span>
+<span class="normal">268</span>
+<span class="normal">269</span>
+<span class="normal">270</span>
+<span class="normal">271</span>
+<span class="normal">272</span>
+<span class="normal">273</span>
+<span class="normal">274</span>
+<span class="normal">275</span>
+<span class="normal">276</span>
+<span class="normal">277</span>
+<span class="normal">278</span>
+<span class="normal">279</span>
+<span class="normal">280</span>
+<span class="normal">281</span>
+<span class="normal">282</span>
+<span class="normal">283</span>
+<span class="normal">284</span>
+<span class="normal">285</span>
+<span class="normal">286</span>
+<span class="normal">287</span>
+<span class="normal">288</span>
+<span class="normal">289</span>
+<span class="normal">290</span>
+<span class="normal">291</span>
+<span class="normal">292</span>
+<span class="normal">293</span>
+<span class="normal">294</span>
+<span class="normal">295</span>
+<span class="normal">296</span>
+<span class="normal">297</span>
+<span class="normal">298</span>
+<span class="normal">299</span>
+<span class="normal">300</span>
+<span class="normal">301</span>
+<span class="normal">302</span>
+<span class="normal">303</span>
+<span class="normal">304</span>
+<span class="normal">305</span>
+<span class="normal">306</span>
+<span class="normal">307</span>
+<span class="normal">308</span>
+<span class="normal">309</span>
+<span class="normal">310</span>
+<span class="normal">311</span>
+<span class="normal">312</span>
+<span class="normal">313</span>
+<span class="normal">314</span>
+<span class="normal">315</span>
+<span class="normal">316</span>
+<span class="normal">317</span>
+<span class="normal">318</span>
+<span class="normal">319</span>
+<span class="normal">320</span>
+<span class="normal">321</span>
+<span class="normal">322</span>
+<span class="normal">323</span>
+<span class="normal">324</span>
+<span class="normal">325</span>
+<span class="normal">326</span>
+<span class="normal">327</span>
+<span class="normal">328</span>
+<span class="normal">329</span>
+<span class="normal">330</span>
+<span class="normal">331</span>
+<span class="normal">332</span>
+<span class="normal">333</span>
+<span class="normal">334</span>
+<span class="normal">335</span>
+<span class="normal">336</span>
+<span class="normal">337</span>
+<span class="normal">338</span>
+<span class="normal">339</span>
+<span class="normal">340</span>
+<span class="normal">341</span>
+<span class="normal">342</span>
+<span class="normal">343</span>
+<span class="normal">344</span>
+<span class="normal">345</span>
+<span class="normal">346</span>
+<span class="normal">347</span>
+<span class="normal">348</span>
+<span class="normal">349</span>
+<span class="normal">350</span>
+<span class="normal">351</span>
+<span class="normal">352</span>
+<span class="normal">353</span>
+<span class="normal">354</span>
+<span class="normal">355</span>
+<span class="normal">356</span>
+<span class="normal">357</span>
+<span class="normal">358</span>
+<span class="normal">359</span>
+<span class="normal">360</span>
+<span class="normal">361</span>
+<span class="normal">362</span>
+<span class="normal">363</span>
+<span class="normal">364</span>
+<span class="normal">365</span>
+<span class="normal">366</span>
+<span class="normal">367</span>
+<span class="normal">368</span>
+<span class="normal">369</span>
+<span class="normal">370</span>
+<span class="normal">371</span>
+<span class="normal">372</span>
+<span class="normal">373</span>
+<span class="normal">374</span>
+<span class="normal">375</span>
+<span class="normal">376</span>
+<span class="normal">377</span>
+<span class="normal">378</span>
+<span class="normal">379</span>
+<span class="normal">380</span>
+<span class="normal">381</span>
+<span class="normal">382</span>
+<span class="normal">383</span>
+<span class="normal">384</span>
+<span class="normal">385</span>
+<span class="normal">386</span>
+<span class="normal">387</span>
+<span class="normal">388</span>
+<span class="normal">389</span>
+<span class="normal">390</span>
+<span class="normal">391</span>
+<span class="normal">392</span>
+<span class="normal">393</span>
+<span class="normal">394</span>
+<span class="normal">395</span>
+<span class="normal">396</span>
+<span class="normal">397</span>
+<span class="normal">398</span>
+<span class="normal">399</span>
+<span class="normal">400</span>
+<span class="normal">401</span>
+<span class="normal">402</span>
+<span class="normal">403</span>
+<span class="normal">404</span>
+<span class="normal">405</span>
+<span class="normal">406</span>
+<span class="normal">407</span>
+<span class="normal">408</span>
+<span class="normal">409</span>
+<span class="normal">410</span>
+<span class="normal">411</span>
+<span class="normal">412</span>
+<span class="normal">413</span>
+<span class="normal">414</span>
+<span class="normal">415</span>
+<span class="normal">416</span>
+<span class="normal">417</span>
+<span class="normal">418</span>
+<span class="normal">419</span>
+<span class="normal">420</span>
+<span class="normal">421</span>
+<span class="normal">422</span>
+<span class="normal">423</span>
+<span class="normal">424</span>
+<span class="normal">425</span>
+<span class="normal">426</span>
+<span class="normal">427</span>
+<span class="normal">428</span>
+<span class="normal">429</span>
+<span class="normal">430</span>
+<span class="normal">431</span>
+<span class="normal">432</span>
+<span class="normal">433</span>
+<span class="normal">434</span>
+<span class="normal">435</span>
+<span class="normal">436</span>
+<span class="normal">437</span>
+<span class="normal">438</span>
+<span class="normal">439</span>
+<span class="normal">440</span>
+<span class="normal">441</span>
+<span class="normal">442</span>
+<span class="normal">443</span>
+<span class="normal">444</span>
+<span class="normal">445</span>
+<span class="normal">446</span>
+<span class="normal">447</span>
+<span class="normal">448</span>
+<span class="normal">449</span>
+<span class="normal">450</span>
+<span class="normal">451</span>
+<span class="normal">452</span>
+<span class="normal">453</span>
+<span class="normal">454</span>
+<span class="normal">455</span>
+<span class="normal">456</span>
+<span class="normal">457</span>
+<span class="normal">458</span>
+<span class="normal">459</span>
+<span class="normal">460</span>
+<span class="normal">461</span>
+<span class="normal">462</span>
+<span class="normal">463</span>
+<span class="normal">464</span>
+<span class="normal">465</span>
+<span class="normal">466</span>
+<span class="normal">467</span>
+<span class="normal">468</span>
+<span class="normal">469</span>
+<span class="normal">470</span>
+<span class="normal">471</span>
+<span class="normal">472</span>
+<span class="normal">473</span>
+<span class="normal">474</span>
+<span class="normal">475</span>
+<span class="normal">476</span>
+<span class="normal">477</span>
+<span class="normal">478</span>
+<span class="normal">479</span>
+<span class="normal">480</span>
+<span class="normal">481</span>
+<span class="normal">482</span>
+<span class="normal">483</span>
+<span class="normal">484</span>
+<span class="normal">485</span>
+<span class="normal">486</span>
+<span class="normal">487</span>
+<span class="normal">488</span>
+<span class="normal">489</span>
+<span class="normal">490</span>
+<span class="normal">491</span>
+<span class="normal">492</span>
+<span class="normal">493</span>
+<span class="normal">494</span>
+<span class="normal">495</span>
+<span class="normal">496</span>
+<span class="normal">497</span>
+<span class="normal">498</span>
+<span class="normal">499</span>
+<span class="normal">500</span>
+<span class="normal">501</span>
+<span class="normal">502</span>
+<span class="normal">503</span>
+<span class="normal">504</span>
+<span class="normal">505</span>
+<span class="normal">506</span>
+<span class="normal">507</span>
+<span class="normal">508</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">class</span> <span class="nc">BayesianTrainer</span><span class="p">(</span><span class="n">BaseBayesianTrainer</span><span class="p">):</span>
+<span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Class to set the of attributes that will be used during the</span>
+<span class="sd">    training process.</span>
+
+<span class="sd">    Both the Bayesian models and the Trainer in this repo are based on the paper:</span>
+<span class="sd">    [Weight Uncertainty in Neural Networks](https://arxiv.org/pdf/1505.05424.pdf).</span>
+
+<span class="sd">    Parameters</span>
+<span class="sd">    ----------</span>
+<span class="sd">    model: `BaseBayesianModel`</span>
+<span class="sd">        An object of class `BaseBayesianModel`. See the `Model Components`</span>
+<span class="sd">        section here in the docs.</span>
+<span class="sd">    objective: str</span>
+<span class="sd">        Defines the objective, loss or cost function.&lt;br/&gt;</span>
+<span class="sd">        Param aliases: `loss_function`, `loss_fn`, `loss`,</span>
+<span class="sd">        `cost_function`, `cost_fn`, `cost`&lt;br/&gt;</span>
+<span class="sd">        Possible values are: _&#39;binary&#39;_, _&#39;multiclass&#39;_, _&#39;regression&#39;_</span>
+<span class="sd">    custom_loss_function: `nn.Module`, optional, default = None</span>
+<span class="sd">        If none of the loss functions available suits the user, it is possible</span>
+<span class="sd">        to pass a custom loss function. See for example</span>
+<span class="sd">        `pytorch_widedeep.losses.FocalLoss` for the required structure of the</span>
+<span class="sd">        object or the Examples folder in the repo.</span>
+<span class="sd">    optimizer: `Optimzer`, optional, default= None</span>
+<span class="sd">        An instance of Pytorch&#39;s `Optimizer` object(e.g. `torch.optim.Adam</span>
+<span class="sd">        ()`). if no optimizer is passed it will default to `AdamW`.</span>
+<span class="sd">    lr_scheduler: `LRScheduler`, optional, default=None</span>
+<span class="sd">        An instance of Pytorch&#39;s `LRScheduler` object</span>
+<span class="sd">        (e.g `torch.optim.lr_scheduler.StepLR(opt, step_size=5)`).</span>
+<span class="sd">    callbacks: List, optional, default=None</span>
+<span class="sd">        List with `Callback` objects. The three callbacks available in</span>
+<span class="sd">        `pytorch-widedeep` are: `LRHistory`, `ModelCheckpoint` and</span>
+<span class="sd">        `EarlyStopping`. This can also be a custom callback. See</span>
+<span class="sd">        `pytorch_widedeep.callbacks.Callback` or the Examples folder in the</span>
+<span class="sd">        repo.</span>
+<span class="sd">    metrics: List, optional, default=None</span>
+<span class="sd">        - List of objects of type `Metric`. Metrics available are:</span>
+<span class="sd">          `Accuracy`, `Precision`, `Recall`, `FBetaScore`,</span>
+<span class="sd">          `F1Score` and `R2Score`. This can also be a custom metric as</span>
+<span class="sd">          long as it is an object of type `Metric`. See</span>
+<span class="sd">          `pytorch_widedeep.metrics.Metric` or the Examples folder in the repo</span>
+<span class="sd">        - List of objects of type `torchmetrics.Metric`. This can be any</span>
+<span class="sd">          metric from torchmetrics library [Examples](https://lightning.ai/docs/torchmetrics)</span>
+<span class="sd">          classification-metrics&gt;`_. It can also be a torchmetric custom metric as</span>
+<span class="sd">          long as it is an object of type `Metric`.</span>
+<span class="sd">          See `the [instructions]((https://lightning.ai/docs/torchmetrics))</span>
+<span class="sd">    verbose: int, default=1</span>
+<span class="sd">        Setting it to 0 will print nothing during training.</span>
+<span class="sd">    seed: int, default=1</span>
+<span class="sd">        Random seed to be used internally for train_test_split</span>
+
+<span class="sd">    Other Parameters</span>
+<span class="sd">    ----------------</span>
+<span class="sd">    **kwargs: dict</span>
+<span class="sd">        Other infrequently used arguments that can also be passed as kwargs are:</span>
+
+<span class="sd">        - **device**: `str`&lt;br/&gt;</span>
+<span class="sd">            string indicating the device. One of _&#39;cpu&#39;_, _&#39;gpu&#39;_ or &#39;mps&#39; if</span>
+<span class="sd">            run on a Mac with Apple silicon or AMD GPU(s)</span>
+
+<span class="sd">        - **num_workers**: `int`&lt;br/&gt;</span>
+<span class="sd">            number of workers to be used internally by the data loaders</span>
+
+<span class="sd">        - **class_weight**: `List[float]`&lt;br/&gt;</span>
+<span class="sd">            This is the `weight` or `pos_weight` parameter in</span>
+<span class="sd">            `CrossEntropyLoss` and `BCEWithLogitsLoss`, depending on whether</span>
+
+<span class="sd">        - **reducelronplateau_criterion**: `str`</span>
+<span class="sd">            This sets the criterion that will be used by the lr scheduler to</span>
+<span class="sd">            take a step: One of _&#39;loss&#39;_ or _&#39;metric&#39;_. The ReduceLROnPlateau</span>
+<span class="sd">            learning rate is a bit particular.</span>
+
+<span class="sd">    Attributes</span>
+<span class="sd">    ----------</span>
+<span class="sd">    cyclic_lr: bool</span>
+<span class="sd">        Attribute that indicates if  the lr_scheduler is cyclic_lr</span>
+<span class="sd">        (i.e. `CyclicLR` or `OneCycleLR`). See `Pytorch schedulers</span>
+<span class="sd">        &lt;https://pytorch.org/docs/stable/optim.html&gt;`_.</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+
+    <span class="nd">@alias</span><span class="p">(</span>  <span class="c1"># noqa: C901</span>
+        <span class="s2">&quot;objective&quot;</span><span class="p">,</span>
+        <span class="p">[</span><span class="s2">&quot;loss_function&quot;</span><span class="p">,</span> <span class="s2">&quot;loss_fn&quot;</span><span class="p">,</span> <span class="s2">&quot;loss&quot;</span><span class="p">,</span> <span class="s2">&quot;cost_function&quot;</span><span class="p">,</span> <span class="s2">&quot;cost_fn&quot;</span><span class="p">,</span> <span class="s2">&quot;cost&quot;</span><span class="p">],</span>
     <span class="p">)</span>
+    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span>
+        <span class="n">model</span><span class="p">:</span> <span class="n">BaseBayesianModel</span><span class="p">,</span>
+        <span class="n">objective</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
+        <span class="n">custom_loss_function</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Module</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">optimizer</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Optimizer</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">lr_scheduler</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">LRScheduler</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">callbacks</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="n">Callback</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">metrics</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Union</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="n">Metric</span><span class="p">],</span> <span class="n">List</span><span class="p">[</span><span class="n">TorchMetric</span><span class="p">]]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">verbose</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1</span><span class="p">,</span>
+        <span class="n">seed</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1</span><span class="p">,</span>
+        <span class="o">**</span><span class="n">kwargs</span><span class="p">,</span>
+    <span class="p">):</span>
+        <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span>
+            <span class="n">model</span><span class="o">=</span><span class="n">model</span><span class="p">,</span>
+            <span class="n">objective</span><span class="o">=</span><span class="n">objective</span><span class="p">,</span>
+            <span class="n">custom_loss_function</span><span class="o">=</span><span class="n">custom_loss_function</span><span class="p">,</span>
+            <span class="n">optimizer</span><span class="o">=</span><span class="n">optimizer</span><span class="p">,</span>
+            <span class="n">lr_scheduler</span><span class="o">=</span><span class="n">lr_scheduler</span><span class="p">,</span>
+            <span class="n">callbacks</span><span class="o">=</span><span class="n">callbacks</span><span class="p">,</span>
+            <span class="n">metrics</span><span class="o">=</span><span class="n">metrics</span><span class="p">,</span>
+            <span class="n">verbose</span><span class="o">=</span><span class="n">verbose</span><span class="p">,</span>
+            <span class="n">seed</span><span class="o">=</span><span class="n">seed</span><span class="p">,</span>
+            <span class="o">**</span><span class="n">kwargs</span><span class="p">,</span>
+        <span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">fit</span><span class="p">(</span>  <span class="c1"># noqa: C901</span>
+        <span class="bp">self</span><span class="p">,</span>
+        <span class="n">X_tab</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">,</span>
+        <span class="n">target</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">,</span>
+        <span class="n">X_tab_val</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">target_val</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">val_split</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">n_epochs</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1</span><span class="p">,</span>
+        <span class="n">validation_freq</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1</span><span class="p">,</span>
+        <span class="n">batch_size</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">32</span><span class="p">,</span>
+        <span class="n">n_train_samples</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">2</span><span class="p">,</span>
+        <span class="n">n_val_samples</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">2</span><span class="p">,</span>
+    <span class="p">):</span>
+<span class="w">        </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Fit method.</span>
+
+<span class="sd">        Parameters</span>
+<span class="sd">        ----------</span>
+<span class="sd">        X_tab: np.ndarray,</span>
+<span class="sd">            tabular dataset</span>
+<span class="sd">        target: np.ndarray</span>
+<span class="sd">            target values</span>
+<span class="sd">        X_tab_val: np.ndarray, Optional, default = None</span>
+<span class="sd">            validation data</span>
+<span class="sd">        target_val: np.ndarray, Optional, default = None</span>
+<span class="sd">            validation target values</span>
+<span class="sd">        val_split: float, Optional. default=None</span>
+<span class="sd">            An alterative to passing the validation set is to use a train/val</span>
+<span class="sd">            split fraction via `val_split`</span>
+<span class="sd">        n_epochs: int, default=1</span>
+<span class="sd">            number of epochs</span>
+<span class="sd">        validation_freq: int, default=1</span>
+<span class="sd">            epochs validation frequency</span>
+<span class="sd">        batch_size: int, default=32</span>
+<span class="sd">            batch size</span>
+<span class="sd">        n_train_samples: int, default=2</span>
+<span class="sd">            number of samples to average over during the training process.</span>
+<span class="sd">            See [Weight Uncertainty in Neural Networks](https://arxiv.org/pdf/1505.05424.pdf) for details.</span>
+<span class="sd">        n_val_samples: int, default=2</span>
+<span class="sd">            number of samples to average over during the validation process.</span>
+<span class="sd">            See [Weight Uncertainty in Neural Networks](https://arxiv.org/pdf/1505.05424.pdf) for details.</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">batch_size</span> <span class="o">=</span> <span class="n">batch_size</span>
+
+        <span class="n">train_set</span><span class="p">,</span> <span class="n">eval_set</span> <span class="o">=</span> <span class="n">tabular_train_val_split</span><span class="p">(</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">seed</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">objective</span><span class="p">,</span> <span class="n">X_tab</span><span class="p">,</span> <span class="n">target</span><span class="p">,</span> <span class="n">X_tab_val</span><span class="p">,</span> <span class="n">target_val</span><span class="p">,</span> <span class="n">val_split</span>
+        <span class="p">)</span>
+        <span class="n">train_loader</span> <span class="o">=</span> <span class="n">DataLoader</span><span class="p">(</span>
+            <span class="n">dataset</span><span class="o">=</span><span class="n">train_set</span><span class="p">,</span> <span class="n">batch_size</span><span class="o">=</span><span class="n">batch_size</span><span class="p">,</span> <span class="n">num_workers</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">num_workers</span>
+        <span class="p">)</span>
+        <span class="n">train_steps</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">train_loader</span><span class="p">)</span>
+
+        <span class="k">if</span> <span class="n">eval_set</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="n">eval_loader</span> <span class="o">=</span> <span class="n">DataLoader</span><span class="p">(</span>
+                <span class="n">dataset</span><span class="o">=</span><span class="n">eval_set</span><span class="p">,</span>
+                <span class="n">batch_size</span><span class="o">=</span><span class="n">batch_size</span><span class="p">,</span>
+                <span class="n">num_workers</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">num_workers</span><span class="p">,</span>
+                <span class="n">shuffle</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
+            <span class="p">)</span>
+            <span class="n">eval_steps</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">eval_loader</span><span class="p">)</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">callback_container</span><span class="o">.</span><span class="n">on_train_begin</span><span class="p">(</span>
+            <span class="p">{</span>
+                <span class="s2">&quot;batch_size&quot;</span><span class="p">:</span> <span class="n">batch_size</span><span class="p">,</span>
+                <span class="s2">&quot;train_steps&quot;</span><span class="p">:</span> <span class="n">train_steps</span><span class="p">,</span>
+                <span class="s2">&quot;n_epochs&quot;</span><span class="p">:</span> <span class="n">n_epochs</span><span class="p">,</span>
+            <span class="p">}</span>
+        <span class="p">)</span>
+        <span class="k">for</span> <span class="n">epoch</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">n_epochs</span><span class="p">):</span>
+            <span class="n">epoch_logs</span><span class="p">:</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="p">{}</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">callback_container</span><span class="o">.</span><span class="n">on_epoch_begin</span><span class="p">(</span><span class="n">epoch</span><span class="p">,</span> <span class="n">logs</span><span class="o">=</span><span class="n">epoch_logs</span><span class="p">)</span>
+
+            <span class="bp">self</span><span class="o">.</span><span class="n">train_running_loss</span> <span class="o">=</span> <span class="mf">0.0</span>
+            <span class="k">with</span> <span class="n">trange</span><span class="p">(</span><span class="n">train_steps</span><span class="p">,</span> <span class="n">disable</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">verbose</span> <span class="o">!=</span> <span class="mi">1</span><span class="p">)</span> <span class="k">as</span> <span class="n">t</span><span class="p">:</span>
+                <span class="k">for</span> <span class="n">batch_idx</span><span class="p">,</span> <span class="p">(</span><span class="n">X</span><span class="p">,</span> <span class="n">y</span><span class="p">)</span> <span class="ow">in</span> <span class="nb">zip</span><span class="p">(</span><span class="n">t</span><span class="p">,</span> <span class="n">train_loader</span><span class="p">):</span>
+                    <span class="n">t</span><span class="o">.</span><span class="n">set_description</span><span class="p">(</span><span class="s2">&quot;epoch </span><span class="si">%i</span><span class="s2">&quot;</span> <span class="o">%</span> <span class="p">(</span><span class="n">epoch</span> <span class="o">+</span> <span class="mi">1</span><span class="p">))</span>
+                    <span class="n">train_score</span><span class="p">,</span> <span class="n">train_loss</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_train_step</span><span class="p">(</span>
+                        <span class="n">X</span><span class="p">,</span> <span class="n">y</span><span class="p">,</span> <span class="n">n_train_samples</span><span class="p">,</span> <span class="n">train_steps</span><span class="p">,</span> <span class="n">batch_idx</span>
+                    <span class="p">)</span>
+                    <span class="n">print_loss_and_metric</span><span class="p">(</span><span class="n">t</span><span class="p">,</span> <span class="n">train_loss</span><span class="p">,</span> <span class="n">train_score</span><span class="p">)</span>
+                    <span class="bp">self</span><span class="o">.</span><span class="n">callback_container</span><span class="o">.</span><span class="n">on_batch_end</span><span class="p">(</span><span class="n">batch</span><span class="o">=</span><span class="n">batch_idx</span><span class="p">)</span>
+            <span class="n">epoch_logs</span> <span class="o">=</span> <span class="n">save_epoch_logs</span><span class="p">(</span><span class="n">epoch_logs</span><span class="p">,</span> <span class="n">train_loss</span><span class="p">,</span> <span class="n">train_score</span><span class="p">,</span> <span class="s2">&quot;train&quot;</span><span class="p">)</span>
+
+            <span class="n">on_epoch_end_metric</span> <span class="o">=</span> <span class="kc">None</span>
+            <span class="k">if</span> <span class="n">eval_set</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">and</span> <span class="n">epoch</span> <span class="o">%</span> <span class="n">validation_freq</span> <span class="o">==</span> <span class="p">(</span>
+                <span class="n">validation_freq</span> <span class="o">-</span> <span class="mi">1</span>
+            <span class="p">):</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">callback_container</span><span class="o">.</span><span class="n">on_eval_begin</span><span class="p">()</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">valid_running_loss</span> <span class="o">=</span> <span class="mf">0.0</span>
+                <span class="k">with</span> <span class="n">trange</span><span class="p">(</span><span class="n">eval_steps</span><span class="p">,</span> <span class="n">disable</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">verbose</span> <span class="o">!=</span> <span class="mi">1</span><span class="p">)</span> <span class="k">as</span> <span class="n">v</span><span class="p">:</span>
+                    <span class="k">for</span> <span class="n">i</span><span class="p">,</span> <span class="p">(</span><span class="n">X</span><span class="p">,</span> <span class="n">y</span><span class="p">)</span> <span class="ow">in</span> <span class="nb">zip</span><span class="p">(</span><span class="n">v</span><span class="p">,</span> <span class="n">eval_loader</span><span class="p">):</span>
+                        <span class="n">v</span><span class="o">.</span><span class="n">set_description</span><span class="p">(</span><span class="s2">&quot;valid&quot;</span><span class="p">)</span>
+                        <span class="n">val_score</span><span class="p">,</span> <span class="n">val_loss</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_eval_step</span><span class="p">(</span>
+                            <span class="n">X</span><span class="p">,</span> <span class="n">y</span><span class="p">,</span> <span class="n">n_val_samples</span><span class="p">,</span> <span class="n">train_steps</span><span class="p">,</span> <span class="n">i</span>
+                        <span class="p">)</span>
+                        <span class="n">print_loss_and_metric</span><span class="p">(</span><span class="n">v</span><span class="p">,</span> <span class="n">val_loss</span><span class="p">,</span> <span class="n">val_score</span><span class="p">)</span>
+                <span class="n">epoch_logs</span> <span class="o">=</span> <span class="n">save_epoch_logs</span><span class="p">(</span><span class="n">epoch_logs</span><span class="p">,</span> <span class="n">val_loss</span><span class="p">,</span> <span class="n">val_score</span><span class="p">,</span> <span class="s2">&quot;val&quot;</span><span class="p">)</span>
+
+                <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">reducelronplateau</span><span class="p">:</span>
+                    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">reducelronplateau_criterion</span> <span class="o">==</span> <span class="s2">&quot;loss&quot;</span><span class="p">:</span>
+                        <span class="n">on_epoch_end_metric</span> <span class="o">=</span> <span class="n">val_loss</span>
+                    <span class="k">else</span><span class="p">:</span>
+                        <span class="n">on_epoch_end_metric</span> <span class="o">=</span> <span class="n">val_score</span><span class="p">[</span>
+                            <span class="bp">self</span><span class="o">.</span><span class="n">reducelronplateau_criterion</span>
+                        <span class="p">]</span>
+
+            <span class="bp">self</span><span class="o">.</span><span class="n">callback_container</span><span class="o">.</span><span class="n">on_epoch_end</span><span class="p">(</span><span class="n">epoch</span><span class="p">,</span> <span class="n">epoch_logs</span><span class="p">,</span> <span class="n">on_epoch_end_metric</span><span class="p">)</span>
+
+            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">early_stop</span><span class="p">:</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">callback_container</span><span class="o">.</span><span class="n">on_train_end</span><span class="p">(</span><span class="n">epoch_logs</span><span class="p">)</span>
+                <span class="k">break</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">callback_container</span><span class="o">.</span><span class="n">on_train_end</span><span class="p">(</span><span class="n">epoch_logs</span><span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">_restore_best_weights</span><span class="p">()</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">model</span><span class="o">.</span><span class="n">train</span><span class="p">()</span>
+
+    <span class="k">def</span> <span class="nf">predict</span><span class="p">(</span>  <span class="c1"># type: ignore[return]</span>
+        <span class="bp">self</span><span class="p">,</span>
+        <span class="n">X_tab</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">,</span>
+        <span class="n">n_samples</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">5</span><span class="p">,</span>
+        <span class="n">return_samples</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
+        <span class="n">batch_size</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">256</span><span class="p">,</span>
+    <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">:</span>
+<span class="w">        </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Returns the predictions</span>
+
+<span class="sd">        Parameters</span>
+<span class="sd">        ----------</span>
+<span class="sd">        X_tab: np.ndarray,</span>
+<span class="sd">            tabular dataset</span>
+<span class="sd">        n_samples: int, default=5</span>
+<span class="sd">            number of samples that will be either returned or averaged to</span>
+<span class="sd">            produce an overal prediction</span>
+<span class="sd">        return_samples: bool, default = False</span>
+<span class="sd">            Boolean indicating whether the n samples will be averaged or directly returned</span>
+<span class="sd">        batch_size: int, default = 256</span>
+<span class="sd">            batch size</span>
+
+<span class="sd">        Returns</span>
+<span class="sd">        -------</span>
+<span class="sd">        np.ndarray:</span>
+<span class="sd">            array with the predictions</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+
+        <span class="n">preds_l</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_predict</span><span class="p">(</span><span class="n">X_tab</span><span class="p">,</span> <span class="n">n_samples</span><span class="p">,</span> <span class="n">return_samples</span><span class="p">,</span> <span class="n">batch_size</span><span class="p">)</span>
+        <span class="n">preds</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">hstack</span><span class="p">(</span><span class="n">preds_l</span><span class="p">)</span> <span class="k">if</span> <span class="n">return_samples</span> <span class="k">else</span> <span class="n">np</span><span class="o">.</span><span class="n">vstack</span><span class="p">(</span><span class="n">preds_l</span><span class="p">)</span>
+        <span class="n">axis</span> <span class="o">=</span> <span class="mi">2</span> <span class="k">if</span> <span class="n">return_samples</span> <span class="k">else</span> <span class="mi">1</span>
+
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">objective</span> <span class="o">==</span> <span class="s2">&quot;regression&quot;</span><span class="p">:</span>
+            <span class="k">return</span> <span class="n">preds</span><span class="o">.</span><span class="n">squeeze</span><span class="p">(</span><span class="n">axis</span><span class="p">)</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">objective</span> <span class="o">==</span> <span class="s2">&quot;binary&quot;</span><span class="p">:</span>
+            <span class="k">return</span> <span class="p">(</span><span class="n">preds</span><span class="o">.</span><span class="n">squeeze</span><span class="p">(</span><span class="n">axis</span><span class="p">)</span> <span class="o">&gt;</span> <span class="mf">0.5</span><span class="p">)</span><span class="o">.</span><span class="n">astype</span><span class="p">(</span><span class="s2">&quot;int&quot;</span><span class="p">)</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">objective</span> <span class="o">==</span> <span class="s2">&quot;multiclass&quot;</span><span class="p">:</span>
+            <span class="k">return</span> <span class="n">np</span><span class="o">.</span><span class="n">argmax</span><span class="p">(</span><span class="n">preds</span><span class="p">,</span> <span class="n">axis</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">predict_proba</span><span class="p">(</span>  <span class="c1"># type: ignore[return]</span>
+        <span class="bp">self</span><span class="p">,</span>
+        <span class="n">X_tab</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">,</span>
+        <span class="n">n_samples</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">5</span><span class="p">,</span>
+        <span class="n">return_samples</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
+        <span class="n">batch_size</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">256</span><span class="p">,</span>
+    <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">:</span>
+<span class="w">        </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Returns the predicted probabilities</span>
+
+<span class="sd">        Parameters</span>
+<span class="sd">        ----------</span>
+<span class="sd">        X_tab: np.ndarray,</span>
+<span class="sd">            tabular dataset</span>
+<span class="sd">        n_samples: int, default=5</span>
+<span class="sd">            number of samples that will be either returned or averaged to</span>
+<span class="sd">            produce an overal prediction</span>
+<span class="sd">        return_samples: bool, default = False</span>
+<span class="sd">            Boolean indicating whether the n samples will be averaged or directly returned</span>
+<span class="sd">        batch_size: int, default = 256</span>
+<span class="sd">            batch size</span>
+
+<span class="sd">        Returns</span>
+<span class="sd">        -------</span>
+<span class="sd">        np.ndarray</span>
+<span class="sd">            array with the probabilities per class</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">preds_l</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_predict</span><span class="p">(</span><span class="n">X_tab</span><span class="p">,</span> <span class="n">n_samples</span><span class="p">,</span> <span class="n">return_samples</span><span class="p">,</span> <span class="n">batch_size</span><span class="p">)</span>
+        <span class="n">preds</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">hstack</span><span class="p">(</span><span class="n">preds_l</span><span class="p">)</span> <span class="k">if</span> <span class="n">return_samples</span> <span class="k">else</span> <span class="n">np</span><span class="o">.</span><span class="n">vstack</span><span class="p">(</span><span class="n">preds_l</span><span class="p">)</span>
+
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">objective</span> <span class="o">==</span> <span class="s2">&quot;binary&quot;</span><span class="p">:</span>
+            <span class="k">if</span> <span class="n">return_samples</span><span class="p">:</span>
+                <span class="n">preds</span> <span class="o">=</span> <span class="n">preds</span><span class="o">.</span><span class="n">squeeze</span><span class="p">(</span><span class="mi">2</span><span class="p">)</span>
+                <span class="n">probs</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">zeros</span><span class="p">([</span><span class="n">n_samples</span><span class="p">,</span> <span class="n">preds</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">1</span><span class="p">],</span> <span class="mi">2</span><span class="p">])</span>
+                <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">n_samples</span><span class="p">):</span>
+                    <span class="n">probs</span><span class="p">[</span><span class="n">i</span><span class="p">,</span> <span class="p">:,</span> <span class="mi">0</span><span class="p">]</span> <span class="o">=</span> <span class="mi">1</span> <span class="o">-</span> <span class="n">preds</span><span class="p">[</span><span class="n">i</span><span class="p">]</span>
+                    <span class="n">probs</span><span class="p">[</span><span class="n">i</span><span class="p">,</span> <span class="p">:,</span> <span class="mi">1</span><span class="p">]</span> <span class="o">=</span> <span class="n">preds</span><span class="p">[</span><span class="n">i</span><span class="p">]</span>
+            <span class="k">else</span><span class="p">:</span>
+                <span class="n">preds</span> <span class="o">=</span> <span class="n">preds</span><span class="o">.</span><span class="n">squeeze</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span>
+                <span class="n">probs</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">zeros</span><span class="p">([</span><span class="n">preds</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="mi">2</span><span class="p">])</span>
+                <span class="n">probs</span><span class="p">[:,</span> <span class="mi">0</span><span class="p">]</span> <span class="o">=</span> <span class="mi">1</span> <span class="o">-</span> <span class="n">preds</span>
+                <span class="n">probs</span><span class="p">[:,</span> <span class="mi">1</span><span class="p">]</span> <span class="o">=</span> <span class="n">preds</span>
+            <span class="k">return</span> <span class="n">probs</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">objective</span> <span class="o">==</span> <span class="s2">&quot;multiclass&quot;</span><span class="p">:</span>
+            <span class="k">return</span> <span class="n">preds</span>
+
+    <span class="k">def</span> <span class="nf">save</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span>
+        <span class="n">path</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
+        <span class="n">save_state_dict</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
+        <span class="n">model_filename</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;bayesian_model.pt&quot;</span><span class="p">,</span>
+    <span class="p">):</span>
+<span class="w">        </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Saves the model, training and evaluation history to disk</span>
+
+<span class="sd">        The `Trainer` class is built so that it &#39;just&#39; trains a model. With</span>
+<span class="sd">        that in mind, all the torch related parameters (such as optimizers or</span>
+<span class="sd">        learning rate schedulers) have to be defined externally and then</span>
+<span class="sd">        passed to the `Trainer`. As a result, the `Trainer` does not</span>
+<span class="sd">        generate any attribute or additional data products that need to be</span>
+<span class="sd">        saved other than the `model` object itself, which can be saved as</span>
+<span class="sd">        any other torch model (e.g. `torch.save(model, path)`).</span>
+
+<span class="sd">        Parameters</span>
+<span class="sd">        ----------</span>
+<span class="sd">        path: str</span>
+<span class="sd">            path to the directory where the model and the feature importance</span>
+<span class="sd">            attribute will be saved.</span>
+<span class="sd">        save_state_dict: bool, default = False</span>
+<span class="sd">            Boolean indicating whether to save directly the model or the</span>
+<span class="sd">            model&#39;s state dictionary</span>
+<span class="sd">        model_filename: str, Optional, default = &quot;wd_model.pt&quot;</span>
+<span class="sd">            filename where the model weights will be store</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+
+        <span class="n">save_dir</span> <span class="o">=</span> <span class="n">Path</span><span class="p">(</span><span class="n">path</span><span class="p">)</span>
+        <span class="n">history_dir</span> <span class="o">=</span> <span class="n">save_dir</span> <span class="o">/</span> <span class="s2">&quot;history&quot;</span>
+        <span class="n">history_dir</span><span class="o">.</span><span class="n">mkdir</span><span class="p">(</span><span class="n">exist_ok</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">parents</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
+
+        <span class="c1"># the trainer is run with the History Callback by default</span>
+        <span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">history_dir</span> <span class="o">/</span> <span class="s2">&quot;train_eval_history.json&quot;</span><span class="p">,</span> <span class="s2">&quot;w&quot;</span><span class="p">)</span> <span class="k">as</span> <span class="n">teh</span><span class="p">:</span>
+            <span class="n">json</span><span class="o">.</span><span class="n">dump</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">history</span><span class="p">,</span> <span class="n">teh</span><span class="p">)</span>  <span class="c1"># type: ignore[attr-defined]</span>
+
+        <span class="n">has_lr_history</span> <span class="o">=</span> <span class="nb">any</span><span class="p">(</span>
+            <span class="p">[</span><span class="n">clbk</span><span class="o">.</span><span class="vm">__class__</span><span class="o">.</span><span class="vm">__name__</span> <span class="o">==</span> <span class="s2">&quot;LRHistory&quot;</span> <span class="k">for</span> <span class="n">clbk</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">callbacks</span><span class="p">]</span>
+        <span class="p">)</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">lr_scheduler</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">and</span> <span class="n">has_lr_history</span><span class="p">:</span>
+            <span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">history_dir</span> <span class="o">/</span> <span class="s2">&quot;lr_history.json&quot;</span><span class="p">,</span> <span class="s2">&quot;w&quot;</span><span class="p">)</span> <span class="k">as</span> <span class="n">lrh</span><span class="p">:</span>
+                <span class="n">json</span><span class="o">.</span><span class="n">dump</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">lr_history</span><span class="p">,</span> <span class="n">lrh</span><span class="p">)</span>  <span class="c1"># type: ignore[attr-defined]</span>
+
+        <span class="n">model_path</span> <span class="o">=</span> <span class="n">save_dir</span> <span class="o">/</span> <span class="n">model_filename</span>
+        <span class="k">if</span> <span class="n">save_state_dict</span><span class="p">:</span>
+            <span class="n">torch</span><span class="o">.</span><span class="n">save</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">model</span><span class="o">.</span><span class="n">state_dict</span><span class="p">(),</span> <span class="n">model_path</span><span class="p">)</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="n">torch</span><span class="o">.</span><span class="n">save</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">model</span><span class="p">,</span> <span class="n">model_path</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">_train_step</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span>
+        <span class="n">X_tab</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">,</span>
+        <span class="n">target</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">,</span>
+        <span class="n">n_samples</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span>
+        <span class="n">n_batches</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span>
+        <span class="n">batch_idx</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span>
+    <span class="p">):</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">model</span><span class="o">.</span><span class="n">train</span><span class="p">()</span>
+
+        <span class="n">X</span> <span class="o">=</span> <span class="n">to_device</span><span class="p">(</span><span class="n">X_tab</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">device</span><span class="p">)</span>
+        <span class="n">y</span> <span class="o">=</span> <span class="n">target</span><span class="o">.</span><span class="n">view</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span><span class="o">.</span><span class="n">float</span><span class="p">()</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">objective</span> <span class="o">!=</span> <span class="s2">&quot;multiclass&quot;</span> <span class="k">else</span> <span class="n">target</span>
+        <span class="n">y</span> <span class="o">=</span> <span class="n">to_device</span><span class="p">(</span><span class="n">y</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">device</span><span class="p">)</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">optimizer</span><span class="o">.</span><span class="n">zero_grad</span><span class="p">()</span>
+        <span class="n">y_pred</span><span class="p">,</span> <span class="n">loss</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">model</span><span class="o">.</span><span class="n">sample_elbo</span><span class="p">(</span><span class="n">X</span><span class="p">,</span> <span class="n">y</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">loss_fn</span><span class="p">,</span> <span class="n">n_samples</span><span class="p">,</span> <span class="n">n_batches</span><span class="p">)</span>  <span class="c1"># type: ignore[arg-type]</span>
+
+        <span class="n">y_pred</span> <span class="o">=</span> <span class="n">y_pred</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="n">dim</span><span class="o">=</span><span class="mi">0</span><span class="p">)</span>
+        <span class="n">score</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_score</span><span class="p">(</span><span class="n">y_pred</span><span class="p">,</span> <span class="n">y</span><span class="p">)</span>
+
+        <span class="n">loss</span><span class="o">.</span><span class="n">backward</span><span class="p">()</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">optimizer</span><span class="o">.</span><span class="n">step</span><span class="p">()</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">train_running_loss</span> <span class="o">+=</span> <span class="n">loss</span><span class="o">.</span><span class="n">item</span><span class="p">()</span>
+        <span class="n">avg_loss</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">train_running_loss</span> <span class="o">/</span> <span class="p">(</span><span class="n">batch_idx</span> <span class="o">+</span> <span class="mi">1</span><span class="p">)</span>
+
+        <span class="k">return</span> <span class="n">score</span><span class="p">,</span> <span class="n">avg_loss</span>
+
+    <span class="k">def</span> <span class="nf">_eval_step</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span>
+        <span class="n">X_tab</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">,</span>
+        <span class="n">target</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">,</span>
+        <span class="n">n_samples</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span>
+        <span class="n">n_batches</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span>
+        <span class="n">batch_idx</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span>
+    <span class="p">):</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">model</span><span class="o">.</span><span class="n">eval</span><span class="p">()</span>
+        <span class="k">with</span> <span class="n">torch</span><span class="o">.</span><span class="n">no_grad</span><span class="p">():</span>
+            <span class="n">X</span> <span class="o">=</span> <span class="n">to_device</span><span class="p">(</span><span class="n">X_tab</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">device</span><span class="p">)</span>
+            <span class="n">y</span> <span class="o">=</span> <span class="n">target</span><span class="o">.</span><span class="n">view</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span><span class="o">.</span><span class="n">float</span><span class="p">()</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">objective</span> <span class="o">!=</span> <span class="s2">&quot;multiclass&quot;</span> <span class="k">else</span> <span class="n">target</span>
+            <span class="n">y</span> <span class="o">=</span> <span class="n">to_device</span><span class="p">(</span><span class="n">y</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">device</span><span class="p">)</span>
+
+            <span class="n">y_pred</span><span class="p">,</span> <span class="n">loss</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">model</span><span class="o">.</span><span class="n">sample_elbo</span><span class="p">(</span>
+                <span class="n">X</span><span class="p">,</span>  <span class="c1"># type: ignore[arg-type]</span>
+                <span class="n">y</span><span class="p">,</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">loss_fn</span><span class="p">,</span>
+                <span class="n">n_samples</span><span class="p">,</span>
+                <span class="n">n_batches</span><span class="p">,</span>
+            <span class="p">)</span>
+            <span class="n">y_pred</span> <span class="o">=</span> <span class="n">y_pred</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="n">dim</span><span class="o">=</span><span class="mi">0</span><span class="p">)</span>
+            <span class="n">score</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_score</span><span class="p">(</span><span class="n">y_pred</span><span class="p">,</span> <span class="n">y</span><span class="p">)</span>
+
+            <span class="bp">self</span><span class="o">.</span><span class="n">valid_running_loss</span> <span class="o">+=</span> <span class="n">loss</span><span class="o">.</span><span class="n">item</span><span class="p">()</span>
+            <span class="n">avg_loss</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">valid_running_loss</span> <span class="o">/</span> <span class="p">(</span><span class="n">batch_idx</span> <span class="o">+</span> <span class="mi">1</span><span class="p">)</span>
+
+        <span class="k">return</span> <span class="n">score</span><span class="p">,</span> <span class="n">avg_loss</span>
+
+    <span class="k">def</span> <span class="nf">_get_score</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">y_pred</span><span class="p">,</span> <span class="n">y</span><span class="p">):</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">metric</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">objective</span> <span class="o">==</span> <span class="s2">&quot;regression&quot;</span><span class="p">:</span>
+                <span class="n">score</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">metric</span><span class="p">(</span><span class="n">y_pred</span><span class="p">,</span> <span class="n">y</span><span class="p">)</span>
+            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">objective</span> <span class="o">==</span> <span class="s2">&quot;binary&quot;</span><span class="p">:</span>
+                <span class="n">score</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">metric</span><span class="p">(</span><span class="n">torch</span><span class="o">.</span><span class="n">sigmoid</span><span class="p">(</span><span class="n">y_pred</span><span class="p">),</span> <span class="n">y</span><span class="p">)</span>
+            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">objective</span> <span class="o">==</span> <span class="s2">&quot;multiclass&quot;</span><span class="p">:</span>
+                <span class="n">score</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">metric</span><span class="p">(</span><span class="n">F</span><span class="o">.</span><span class="n">softmax</span><span class="p">(</span><span class="n">y_pred</span><span class="p">,</span> <span class="n">dim</span><span class="o">=</span><span class="mi">1</span><span class="p">),</span> <span class="n">y</span><span class="p">)</span>
+            <span class="k">return</span> <span class="n">score</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="k">return</span> <span class="kc">None</span>
+
+    <span class="k">def</span> <span class="nf">_predict</span><span class="p">(</span>  <span class="c1"># noqa: C901</span>
+        <span class="bp">self</span><span class="p">,</span>
+        <span class="n">X_tab</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">,</span>
+        <span class="n">n_samples</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">5</span><span class="p">,</span>
+        <span class="n">return_samples</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
+        <span class="n">batch_size</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">256</span><span class="p">,</span>
+    <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">List</span><span class="p">:</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">batch_size</span> <span class="o">=</span> <span class="n">batch_size</span>
+
+        <span class="n">test_set</span> <span class="o">=</span> <span class="n">TensorDataset</span><span class="p">(</span><span class="n">torch</span><span class="o">.</span><span class="n">from_numpy</span><span class="p">(</span><span class="n">X_tab</span><span class="p">))</span>
+        <span class="n">test_loader</span> <span class="o">=</span> <span class="n">DataLoader</span><span class="p">(</span>
+            <span class="n">dataset</span><span class="o">=</span><span class="n">test_set</span><span class="p">,</span>
+            <span class="n">batch_size</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">batch_size</span><span class="p">,</span>
+            <span class="n">num_workers</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">num_workers</span><span class="p">,</span>
+            <span class="n">shuffle</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
+        <span class="p">)</span>
+        <span class="n">test_steps</span> <span class="o">=</span> <span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">test_loader</span><span class="o">.</span><span class="n">dataset</span><span class="p">)</span> <span class="o">//</span> <span class="n">test_loader</span><span class="o">.</span><span class="n">batch_size</span><span class="p">)</span> <span class="o">+</span> <span class="mi">1</span>  <span class="c1"># type: ignore[arg-type]</span>
+
+        <span class="n">preds_l</span> <span class="o">=</span> <span class="p">[]</span>
+        <span class="k">with</span> <span class="n">torch</span><span class="o">.</span><span class="n">no_grad</span><span class="p">():</span>
+            <span class="k">with</span> <span class="n">trange</span><span class="p">(</span><span class="n">test_steps</span><span class="p">,</span> <span class="n">disable</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">verbose</span> <span class="o">!=</span> <span class="mi">1</span><span class="p">)</span> <span class="k">as</span> <span class="n">tt</span><span class="p">:</span>
+                <span class="k">for</span> <span class="n">_</span><span class="p">,</span> <span class="n">Xl</span> <span class="ow">in</span> <span class="nb">zip</span><span class="p">(</span><span class="n">tt</span><span class="p">,</span> <span class="n">test_loader</span><span class="p">):</span>
+                    <span class="n">tt</span><span class="o">.</span><span class="n">set_description</span><span class="p">(</span><span class="s2">&quot;predict&quot;</span><span class="p">)</span>
+
+                    <span class="k">try</span><span class="p">:</span>
+                        <span class="n">X</span> <span class="o">=</span> <span class="n">Xl</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">to</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">device</span><span class="p">)</span>
+                    <span class="k">except</span> <span class="ne">TypeError</span><span class="p">:</span>
+                        <span class="n">X</span> <span class="o">=</span> <span class="n">Xl</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">to</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">device</span><span class="p">,</span> <span class="n">dtype</span><span class="o">=</span><span class="n">torch</span><span class="o">.</span><span class="n">float32</span><span class="p">)</span>
+
+                    <span class="k">if</span> <span class="n">return_samples</span><span class="p">:</span>
+                        <span class="n">preds</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">stack</span><span class="p">([</span><span class="bp">self</span><span class="o">.</span><span class="n">model</span><span class="p">(</span><span class="n">X</span><span class="p">)</span> <span class="k">for</span> <span class="n">_</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">n_samples</span><span class="p">)])</span>
+                    <span class="k">else</span><span class="p">:</span>
+                        <span class="bp">self</span><span class="o">.</span><span class="n">model</span><span class="o">.</span><span class="n">eval</span><span class="p">()</span>
+                        <span class="n">preds</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">model</span><span class="p">(</span><span class="n">X</span><span class="p">)</span>
+
+                    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">objective</span> <span class="o">==</span> <span class="s2">&quot;binary&quot;</span><span class="p">:</span>
+                        <span class="n">preds</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">sigmoid</span><span class="p">(</span><span class="n">preds</span><span class="p">)</span>
+                    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">objective</span> <span class="o">==</span> <span class="s2">&quot;multiclass&quot;</span><span class="p">:</span>
+                        <span class="n">preds</span> <span class="o">=</span> <span class="p">(</span>
+                            <span class="n">F</span><span class="o">.</span><span class="n">softmax</span><span class="p">(</span><span class="n">preds</span><span class="p">,</span> <span class="n">dim</span><span class="o">=</span><span class="mi">2</span><span class="p">)</span>
+                            <span class="k">if</span> <span class="n">return_samples</span>
+                            <span class="k">else</span> <span class="n">F</span><span class="o">.</span><span class="n">softmax</span><span class="p">(</span><span class="n">preds</span><span class="p">,</span> <span class="n">dim</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span>
+                        <span class="p">)</span>
+
+                    <span class="n">preds_cpu</span> <span class="o">=</span> <span class="n">preds</span><span class="o">.</span><span class="n">cpu</span><span class="p">()</span><span class="o">.</span><span class="n">data</span><span class="o">.</span><span class="n">numpy</span><span class="p">()</span>
+                    <span class="n">preds_l</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">preds_cpu</span><span class="p">)</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">model</span><span class="o">.</span><span class="n">train</span><span class="p">()</span>
+
+        <span class="k">return</span> <span class="n">preds_l</span>
 </code></pre></div></td></tr></table></div>
-                  </details>
+              </details>
 
 
 
@@ -1895,107 +2851,161 @@ <h3 id="pytorch_widedeep.training.BayesianTrainer.fit" class="doc doc-heading">
 
 
 <p><span class="doc-section-title">Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>X_tab</code></b>
-              (<code><span title="numpy.ndarray">ndarray</span></code>)
-          –
-          <div class="doc-md-description">
-            <p>tabular dataset</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>target</code></b>
-              (<code><span title="numpy.ndarray">ndarray</span></code>)
-          –
-          <div class="doc-md-description">
-            <p>target values</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>X_tab_val</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="numpy.ndarray">ndarray</span>]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>validation data</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>target_val</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="numpy.ndarray">ndarray</span>]</code>, default:
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+          <th>Default</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>X_tab</code></td>
+            <td>
+                  <code><span title="numpy.ndarray">ndarray</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>tabular dataset</p>
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>target</code></td>
+            <td>
+                  <code><span title="numpy.ndarray">ndarray</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>target values</p>
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>X_tab_val</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="numpy.ndarray">ndarray</span>]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>validation data</p>
+              </div>
+            </td>
+            <td>
                   <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>validation target values</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>val_split</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>target_val</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="numpy.ndarray">ndarray</span>]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>validation target values</p>
+              </div>
+            </td>
+            <td>
                   <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>An alterative to passing the validation set is to use a train/val
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>val_split</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>An alterative to passing the validation set is to use a train/val
 split fraction via <code>val_split</code></p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>n_epochs</code></b>
-              (<code>int</code>, default:
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>n_epochs</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>number of epochs</p>
+              </div>
+            </td>
+            <td>
                   <code>1</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>number of epochs</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>validation_freq</code></b>
-              (<code>int</code>, default:
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>validation_freq</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>epochs validation frequency</p>
+              </div>
+            </td>
+            <td>
                   <code>1</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>epochs validation frequency</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>batch_size</code></b>
-              (<code>int</code>, default:
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>batch_size</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>batch size</p>
+              </div>
+            </td>
+            <td>
                   <code>32</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>batch size</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>n_train_samples</code></b>
-              (<code>int</code>, default:
-                  <code>2</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>number of samples to average over during the training process.
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>n_train_samples</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>number of samples to average over during the training process.
 See <a href="https://arxiv.org/pdf/1505.05424.pdf">Weight Uncertainty in Neural Networks</a> for details.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>n_val_samples</code></b>
-              (<code>int</code>, default:
+              </div>
+            </td>
+            <td>
                   <code>2</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>number of samples to average over during the validation process.
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>n_val_samples</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>number of samples to average over during the validation process.
 See <a href="https://arxiv.org/pdf/1505.05424.pdf">Weight Uncertainty in Neural Networks</a> for details.</p>
-          </div>
-        </li>
-    </ul>
+              </div>
+            </td>
+            <td>
+                  <code>2</code>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
             <details class="quote">
               <summary>Source code in <code>pytorch_widedeep/training/bayesian_trainer.py</code></summary>
@@ -2253,59 +3263,98 @@ <h3 id="pytorch_widedeep.training.BayesianTrainer.predict" class="doc doc-headin
 
 
 <p><span class="doc-section-title">Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>X_tab</code></b>
-              (<code><span title="numpy.ndarray">ndarray</span></code>)
-          –
-          <div class="doc-md-description">
-            <p>tabular dataset</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>n_samples</code></b>
-              (<code>int</code>, default:
-                  <code>5</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>number of samples that will be either returned or averaged to
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+          <th>Default</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>X_tab</code></td>
+            <td>
+                  <code><span title="numpy.ndarray">ndarray</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>tabular dataset</p>
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>n_samples</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>number of samples that will be either returned or averaged to
 produce an overal prediction</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>return_samples</code></b>
-              (<code>bool</code>, default:
+              </div>
+            </td>
+            <td>
+                  <code>5</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>return_samples</code></td>
+            <td>
+                  <code>bool</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Boolean indicating whether the n samples will be averaged or directly returned</p>
+              </div>
+            </td>
+            <td>
                   <code>False</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Boolean indicating whether the n samples will be averaged or directly returned</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>batch_size</code></b>
-              (<code>int</code>, default:
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>batch_size</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>batch size</p>
+              </div>
+            </td>
+            <td>
                   <code>256</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>batch size</p>
-          </div>
-        </li>
-    </ul>
-
-
-<p><span class="doc-section-title">Returns:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-              <code>np.ndarray:</code>
-          –
-          <div class="doc-md-description">
-            <p>array with the predictions</p>
-          </div>
-        </li>
-    </ul>
+            </td>
+          </tr>
+      </tbody>
+    </table>
+
+
+    <p><span class="doc-section-title">Returns:</span></p>
+    <table>
+      <thead>
+        <tr>
+          <th>Type</th>
+          <th>Description</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td>
+                  <code>np.ndarray:</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>array with the predictions</p>
+              </div>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
             <details class="quote">
               <summary>Source code in <code>pytorch_widedeep/training/bayesian_trainer.py</code></summary>
@@ -2407,59 +3456,98 @@ <h3 id="pytorch_widedeep.training.BayesianTrainer.predict_proba" class="doc doc-
 
 
 <p><span class="doc-section-title">Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>X_tab</code></b>
-              (<code><span title="numpy.ndarray">ndarray</span></code>)
-          –
-          <div class="doc-md-description">
-            <p>tabular dataset</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>n_samples</code></b>
-              (<code>int</code>, default:
-                  <code>5</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>number of samples that will be either returned or averaged to
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+          <th>Default</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>X_tab</code></td>
+            <td>
+                  <code><span title="numpy.ndarray">ndarray</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>tabular dataset</p>
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>n_samples</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>number of samples that will be either returned or averaged to
 produce an overal prediction</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>return_samples</code></b>
-              (<code>bool</code>, default:
+              </div>
+            </td>
+            <td>
+                  <code>5</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>return_samples</code></td>
+            <td>
+                  <code>bool</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Boolean indicating whether the n samples will be averaged or directly returned</p>
+              </div>
+            </td>
+            <td>
                   <code>False</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Boolean indicating whether the n samples will be averaged or directly returned</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>batch_size</code></b>
-              (<code>int</code>, default:
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>batch_size</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>batch size</p>
+              </div>
+            </td>
+            <td>
                   <code>256</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>batch size</p>
-          </div>
-        </li>
-    </ul>
-
-
-<p><span class="doc-section-title">Returns:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-              <code><span title="numpy.ndarray">ndarray</span></code>
-          –
-          <div class="doc-md-description">
-            <p>array with the probabilities per class</p>
-          </div>
-        </li>
-    </ul>
+            </td>
+          </tr>
+      </tbody>
+    </table>
+
+
+    <p><span class="doc-section-title">Returns:</span></p>
+    <table>
+      <thead>
+        <tr>
+          <th>Type</th>
+          <th>Description</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td>
+                  <code><span title="numpy.ndarray">ndarray</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>array with the probabilities per class</p>
+              </div>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
             <details class="quote">
               <summary>Source code in <code>pytorch_widedeep/training/bayesian_trainer.py</code></summary>
@@ -2584,38 +3672,62 @@ <h3 id="pytorch_widedeep.training.BayesianTrainer.save" class="doc doc-heading">
 
 
 <p><span class="doc-section-title">Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>path</code></b>
-              (<code>str</code>)
-          –
-          <div class="doc-md-description">
-            <p>path to the directory where the model and the feature importance
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+          <th>Default</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>path</code></td>
+            <td>
+                  <code>str</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>path to the directory where the model and the feature importance
 attribute will be saved.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>save_state_dict</code></b>
-              (<code>bool</code>, default:
-                  <code>False</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Boolean indicating whether to save directly the model or the
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>save_state_dict</code></td>
+            <td>
+                  <code>bool</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Boolean indicating whether to save directly the model or the
 model's state dictionary</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>model_filename</code></b>
-              (<code>str</code>, default:
+              </div>
+            </td>
+            <td>
+                  <code>False</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>model_filename</code></td>
+            <td>
+                  <code>str</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>filename where the model weights will be store</p>
+              </div>
+            </td>
+            <td>
                   <code>&#39;bayesian_model.pt&#39;</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>filename where the model weights will be store</p>
-          </div>
-        </li>
-    </ul>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
             <details class="quote">
               <summary>Source code in <code>pytorch_widedeep/training/bayesian_trainer.py</code></summary>
diff --git a/mkdocs/site/pytorch-widedeep/callbacks.html b/mkdocs/site/pytorch-widedeep/callbacks.html
index 883b2386..ed2cfa86 100644
--- a/mkdocs/site/pytorch-widedeep/callbacks.html
+++ b/mkdocs/site/pytorch-widedeep/callbacks.html
@@ -1405,7 +1405,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/16_Self_Supervised_Pretraning_pt1.ipynb" class="md-nav__link">
+      <a href="../examples/15_Self_Supervised_Pretraning_pt1.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1426,7 +1426,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/16_Self_Supervised_Pretraning_pt2.ipynb" class="md-nav__link">
+      <a href="../examples/15_Self_Supervised_Pretraning_pt2.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1447,7 +1447,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/17_Usign_a_custom_hugging_face_model.ipynb" class="md-nav__link">
+      <a href="../examples/16_Usign_a_custom_hugging_face_model.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1468,7 +1468,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/18_feature_importance_via_attention_weights.ipynb" class="md-nav__link">
+      <a href="../examples/17_feature_importance_via_attention_weights.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1489,7 +1489,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/19_wide_and_deep_for_recsys_pt1.ipynb" class="md-nav__link">
+      <a href="../examples/18_wide_and_deep_for_recsys_pt1.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1510,7 +1510,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/19_wide_and_deep_for_recsys_pt2.ipynb" class="md-nav__link">
+      <a href="../examples/18_wide_and_deep_for_recsys_pt2.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1531,7 +1531,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/20_load_from_folder_functionality.ipynb" class="md-nav__link">
+      <a href="../examples/19_load_from_folder_functionality.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1552,7 +1552,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/21_Using_huggingface_within_widedeep.ipynb" class="md-nav__link">
+      <a href="../examples/20_Using_huggingface_within_widedeep.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1628,8 +1628,7 @@ <h2 id="pytorch_widedeep.callbacks.LRHistory" class="doc doc-heading">
 
 
 <a href="#pytorch_widedeep.callbacks.LRHistory" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="nf">LRHistory</span><span class="p">(</span><span class="n">n_epochs</span><span class="p">)</span>
-</code></pre></div>
+
 
     <div class="doc doc-contents first">
             <p class="doc doc-class-bases">
@@ -1643,16 +1642,32 @@ <h2 id="pytorch_widedeep.callbacks.LRHistory" class="doc doc-heading">
 
 
 <p><span class="doc-section-title">Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>n_epochs</code></b>
-              (<code>int</code>)
-          –
-          <div class="doc-md-description">
-            <p>number of training epochs</p>
-          </div>
-        </li>
-    </ul>
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+          <th>Default</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>n_epochs</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>number of training epochs</p>
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
 
 <p><span class="doc-section-title">Examples:</span></p>
@@ -1668,15 +1683,257 @@ <h2 id="pytorch_widedeep.callbacks.LRHistory" class="doc doc-heading">
 <span class="gp">&gt;&gt;&gt; </span><span class="n">trainer</span> <span class="o">=</span> <span class="n">Trainer</span><span class="p">(</span><span class="n">model</span><span class="p">,</span> <span class="n">objective</span><span class="o">=</span><span class="s2">&quot;regression&quot;</span><span class="p">,</span> <span class="n">callbacks</span><span class="o">=</span><span class="p">[</span><span class="n">LRHistory</span><span class="p">(</span><span class="n">n_epochs</span><span class="o">=</span><span class="mi">10</span><span class="p">)])</span>
 </code></pre></div>
 
-                  <details class="quote">
-                    <summary>Source code in <code>pytorch_widedeep/callbacks.py</code></summary>
-                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">277</span>
+              <details class="quote">
+                <summary>Source code in <code>pytorch_widedeep/callbacks.py</code></summary>
+                <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">251</span>
+<span class="normal">252</span>
+<span class="normal">253</span>
+<span class="normal">254</span>
+<span class="normal">255</span>
+<span class="normal">256</span>
+<span class="normal">257</span>
+<span class="normal">258</span>
+<span class="normal">259</span>
+<span class="normal">260</span>
+<span class="normal">261</span>
+<span class="normal">262</span>
+<span class="normal">263</span>
+<span class="normal">264</span>
+<span class="normal">265</span>
+<span class="normal">266</span>
+<span class="normal">267</span>
+<span class="normal">268</span>
+<span class="normal">269</span>
+<span class="normal">270</span>
+<span class="normal">271</span>
+<span class="normal">272</span>
+<span class="normal">273</span>
+<span class="normal">274</span>
+<span class="normal">275</span>
+<span class="normal">276</span>
+<span class="normal">277</span>
 <span class="normal">278</span>
-<span class="normal">279</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">n_epochs</span><span class="p">:</span> <span class="nb">int</span><span class="p">):</span>
-    <span class="nb">super</span><span class="p">(</span><span class="n">LRHistory</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">n_epochs</span> <span class="o">=</span> <span class="n">n_epochs</span>
+<span class="normal">279</span>
+<span class="normal">280</span>
+<span class="normal">281</span>
+<span class="normal">282</span>
+<span class="normal">283</span>
+<span class="normal">284</span>
+<span class="normal">285</span>
+<span class="normal">286</span>
+<span class="normal">287</span>
+<span class="normal">288</span>
+<span class="normal">289</span>
+<span class="normal">290</span>
+<span class="normal">291</span>
+<span class="normal">292</span>
+<span class="normal">293</span>
+<span class="normal">294</span>
+<span class="normal">295</span>
+<span class="normal">296</span>
+<span class="normal">297</span>
+<span class="normal">298</span>
+<span class="normal">299</span>
+<span class="normal">300</span>
+<span class="normal">301</span>
+<span class="normal">302</span>
+<span class="normal">303</span>
+<span class="normal">304</span>
+<span class="normal">305</span>
+<span class="normal">306</span>
+<span class="normal">307</span>
+<span class="normal">308</span>
+<span class="normal">309</span>
+<span class="normal">310</span>
+<span class="normal">311</span>
+<span class="normal">312</span>
+<span class="normal">313</span>
+<span class="normal">314</span>
+<span class="normal">315</span>
+<span class="normal">316</span>
+<span class="normal">317</span>
+<span class="normal">318</span>
+<span class="normal">319</span>
+<span class="normal">320</span>
+<span class="normal">321</span>
+<span class="normal">322</span>
+<span class="normal">323</span>
+<span class="normal">324</span>
+<span class="normal">325</span>
+<span class="normal">326</span>
+<span class="normal">327</span>
+<span class="normal">328</span>
+<span class="normal">329</span>
+<span class="normal">330</span>
+<span class="normal">331</span>
+<span class="normal">332</span>
+<span class="normal">333</span>
+<span class="normal">334</span>
+<span class="normal">335</span>
+<span class="normal">336</span>
+<span class="normal">337</span>
+<span class="normal">338</span>
+<span class="normal">339</span>
+<span class="normal">340</span>
+<span class="normal">341</span>
+<span class="normal">342</span>
+<span class="normal">343</span>
+<span class="normal">344</span>
+<span class="normal">345</span>
+<span class="normal">346</span>
+<span class="normal">347</span>
+<span class="normal">348</span>
+<span class="normal">349</span>
+<span class="normal">350</span>
+<span class="normal">351</span>
+<span class="normal">352</span>
+<span class="normal">353</span>
+<span class="normal">354</span>
+<span class="normal">355</span>
+<span class="normal">356</span>
+<span class="normal">357</span>
+<span class="normal">358</span>
+<span class="normal">359</span>
+<span class="normal">360</span>
+<span class="normal">361</span>
+<span class="normal">362</span>
+<span class="normal">363</span>
+<span class="normal">364</span>
+<span class="normal">365</span>
+<span class="normal">366</span>
+<span class="normal">367</span>
+<span class="normal">368</span>
+<span class="normal">369</span>
+<span class="normal">370</span>
+<span class="normal">371</span>
+<span class="normal">372</span>
+<span class="normal">373</span>
+<span class="normal">374</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">class</span> <span class="nc">LRHistory</span><span class="p">(</span><span class="n">Callback</span><span class="p">):</span>
+<span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Saves the learning rates during training in the `lr_history` attribute</span>
+<span class="sd">    of the `Trainer`.</span>
+
+<span class="sd">    Callbacks are passed as input parameters to the `Trainer` class. See</span>
+<span class="sd">    `pytorch_widedeep.trainer.Trainer`</span>
+
+<span class="sd">    Parameters</span>
+<span class="sd">    ----------</span>
+<span class="sd">    n_epochs: int</span>
+<span class="sd">        number of training epochs</span>
+
+<span class="sd">    Examples</span>
+<span class="sd">    --------</span>
+<span class="sd">    &gt;&gt;&gt; from pytorch_widedeep.callbacks import LRHistory</span>
+<span class="sd">    &gt;&gt;&gt; from pytorch_widedeep.models import TabMlp, Wide, WideDeep</span>
+<span class="sd">    &gt;&gt;&gt; from pytorch_widedeep.training import Trainer</span>
+<span class="sd">    &gt;&gt;&gt;</span>
+<span class="sd">    &gt;&gt;&gt; embed_input = [(u, i, j) for u, i, j in zip([&quot;a&quot;, &quot;b&quot;, &quot;c&quot;][:4], [4] * 3, [8] * 3)]</span>
+<span class="sd">    &gt;&gt;&gt; column_idx = {k: v for v, k in enumerate([&quot;a&quot;, &quot;b&quot;, &quot;c&quot;])}</span>
+<span class="sd">    &gt;&gt;&gt; wide = Wide(10, 1)</span>
+<span class="sd">    &gt;&gt;&gt; deep = TabMlp(mlp_hidden_dims=[8, 4], column_idx=column_idx, cat_embed_input=embed_input)</span>
+<span class="sd">    &gt;&gt;&gt; model = WideDeep(wide, deep)</span>
+<span class="sd">    &gt;&gt;&gt; trainer = Trainer(model, objective=&quot;regression&quot;, callbacks=[LRHistory(n_epochs=10)])</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+
+    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">n_epochs</span><span class="p">:</span> <span class="nb">int</span><span class="p">):</span>
+        <span class="nb">super</span><span class="p">(</span><span class="n">LRHistory</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">n_epochs</span> <span class="o">=</span> <span class="n">n_epochs</span>
+
+    <span class="k">def</span> <span class="nf">on_epoch_begin</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">epoch</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span> <span class="n">logs</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Dict</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">):</span>
+        <span class="k">if</span> <span class="n">epoch</span> <span class="o">==</span> <span class="mi">0</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">trainer</span><span class="o">.</span><span class="n">lr_scheduler</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">trainer</span><span class="o">.</span><span class="n">lr_history</span> <span class="o">=</span> <span class="p">{}</span>
+            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_multiple_scheduler</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">trainer</span><span class="o">.</span><span class="n">lr_scheduler</span><span class="p">):</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">_save_group_lr_mulitple_scheduler</span><span class="p">(</span><span class="n">step_location</span><span class="o">=</span><span class="s2">&quot;on_epoch_begin&quot;</span><span class="p">)</span>
+            <span class="k">else</span><span class="p">:</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">_save_group_lr</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">trainer</span><span class="o">.</span><span class="n">optimizer</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">on_batch_end</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">batch</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span> <span class="n">logs</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Dict</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">):</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">trainer</span><span class="o">.</span><span class="n">lr_scheduler</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_multiple_scheduler</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">trainer</span><span class="o">.</span><span class="n">lr_scheduler</span><span class="p">):</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">_save_group_lr_mulitple_scheduler</span><span class="p">(</span><span class="n">step_location</span><span class="o">=</span><span class="s2">&quot;on_batch_end&quot;</span><span class="p">)</span>
+            <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">trainer</span><span class="o">.</span><span class="n">cyclic_lr</span><span class="p">:</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">_save_group_lr</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">trainer</span><span class="o">.</span><span class="n">optimizer</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">on_epoch_end</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span> <span class="n">epoch</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span> <span class="n">logs</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Dict</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> <span class="n">metric</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span>
+    <span class="p">):</span>
+        <span class="k">if</span> <span class="n">epoch</span> <span class="o">!=</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">n_epochs</span> <span class="o">-</span> <span class="mi">1</span><span class="p">)</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">trainer</span><span class="o">.</span><span class="n">lr_scheduler</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_multiple_scheduler</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">trainer</span><span class="o">.</span><span class="n">lr_scheduler</span><span class="p">):</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">_save_group_lr_mulitple_scheduler</span><span class="p">(</span><span class="n">step_location</span><span class="o">=</span><span class="s2">&quot;on_epoch_end&quot;</span><span class="p">)</span>
+            <span class="k">elif</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">trainer</span><span class="o">.</span><span class="n">cyclic_lr</span><span class="p">:</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">_save_group_lr</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">trainer</span><span class="o">.</span><span class="n">optimizer</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">_save_group_lr_mulitple_scheduler</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">step_location</span><span class="p">:</span> <span class="nb">str</span><span class="p">):</span>
+        <span class="k">for</span> <span class="n">model_name</span><span class="p">,</span> <span class="n">optimizer</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">trainer</span><span class="o">.</span><span class="n">optimizer</span><span class="o">.</span><span class="n">_optimizers</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
+            <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">optimizer</span><span class="p">,</span> <span class="nb">list</span><span class="p">):</span>
+                <span class="c1"># then, if it has schedulers, we assume it has to have the</span>
+                <span class="c1"># same number of schedulers as optimizers</span>
+                <span class="k">for</span> <span class="n">i</span><span class="p">,</span> <span class="n">opt</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="n">optimizer</span><span class="p">):</span>
+                    <span class="k">if</span> <span class="p">(</span>
+                        <span class="n">step_location</span> <span class="o">==</span> <span class="s2">&quot;on_epoch_begin&quot;</span>
+                        <span class="ow">or</span> <span class="p">(</span>
+                            <span class="n">step_location</span> <span class="o">==</span> <span class="s2">&quot;on_batch_end&quot;</span>
+                            <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">_has_cyclic_scheduler</span><span class="p">(</span><span class="n">model_name</span><span class="p">)</span>
+                        <span class="p">)</span>
+                        <span class="ow">or</span> <span class="p">(</span>
+                            <span class="n">step_location</span> <span class="o">==</span> <span class="s2">&quot;on_epoch_end&quot;</span>
+                            <span class="ow">and</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">_has_cyclic_scheduler</span><span class="p">(</span><span class="n">model_name</span><span class="p">)</span>
+                        <span class="p">)</span>
+                    <span class="p">):</span>
+                        <span class="bp">self</span><span class="o">.</span><span class="n">_save_group_lr</span><span class="p">(</span><span class="n">opt</span><span class="p">,</span> <span class="n">model_name</span><span class="p">,</span> <span class="s2">&quot;_&quot;</span><span class="o">.</span><span class="n">join</span><span class="p">([</span><span class="s2">&quot;opt&quot;</span><span class="p">,</span> <span class="nb">str</span><span class="p">(</span><span class="n">i</span><span class="p">)]))</span>
+                <span class="k">else</span><span class="p">:</span>
+                    <span class="c1"># do nothing</span>
+                    <span class="k">pass</span>
+            <span class="k">else</span><span class="p">:</span>
+                <span class="k">if</span> <span class="p">(</span>
+                    <span class="n">step_location</span> <span class="o">==</span> <span class="s2">&quot;on_epoch_begin&quot;</span>
+                    <span class="ow">or</span> <span class="p">(</span>
+                        <span class="n">step_location</span> <span class="o">==</span> <span class="s2">&quot;on_batch_end&quot;</span>
+                        <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">_has_cyclic_scheduler</span><span class="p">(</span><span class="n">model_name</span><span class="p">)</span>
+                    <span class="p">)</span>
+                    <span class="ow">or</span> <span class="p">(</span>
+                        <span class="n">step_location</span> <span class="o">==</span> <span class="s2">&quot;on_epoch_end&quot;</span>
+                        <span class="ow">and</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">_has_cyclic_scheduler</span><span class="p">(</span><span class="n">model_name</span><span class="p">)</span>
+                    <span class="p">)</span>
+                <span class="p">):</span>
+                    <span class="bp">self</span><span class="o">.</span><span class="n">_save_group_lr</span><span class="p">(</span><span class="n">optimizer</span><span class="p">,</span> <span class="n">model_name</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">_save_group_lr</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span>
+        <span class="n">opt</span><span class="p">:</span> <span class="n">Optimizer</span><span class="p">,</span>
+        <span class="n">suffix</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">model_name</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+    <span class="p">):</span>
+        <span class="n">suffix</span> <span class="o">=</span> <span class="n">suffix</span> <span class="ow">or</span> <span class="s2">&quot;&quot;</span>
+        <span class="n">model_name</span> <span class="o">=</span> <span class="n">model_name</span> <span class="ow">or</span> <span class="s2">&quot;&quot;</span>
+        <span class="k">for</span> <span class="n">group_idx</span><span class="p">,</span> <span class="n">group</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="n">opt</span><span class="o">.</span><span class="n">param_groups</span><span class="p">):</span>
+            <span class="n">group_name</span> <span class="o">=</span> <span class="p">(</span><span class="s2">&quot;_&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">join</span><span class="p">(</span>
+                <span class="p">[</span><span class="n">x</span> <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="p">[</span><span class="s2">&quot;lr&quot;</span><span class="p">,</span> <span class="n">model_name</span><span class="p">,</span> <span class="n">suffix</span><span class="p">,</span> <span class="nb">str</span><span class="p">(</span><span class="n">group_idx</span><span class="p">)]</span> <span class="k">if</span> <span class="n">x</span><span class="p">]</span>
+            <span class="p">)</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">trainer</span><span class="o">.</span><span class="n">lr_history</span><span class="o">.</span><span class="n">setdefault</span><span class="p">(</span><span class="n">group_name</span><span class="p">,</span> <span class="p">[])</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">group</span><span class="p">[</span><span class="s2">&quot;lr&quot;</span><span class="p">])</span>
+
+    <span class="nd">@staticmethod</span>
+    <span class="k">def</span> <span class="nf">_multiple_scheduler</span><span class="p">(</span><span class="n">scheduler</span><span class="p">:</span> <span class="n">LRScheduler</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">bool</span><span class="p">:</span>
+        <span class="k">return</span> <span class="n">scheduler</span><span class="o">.</span><span class="vm">__class__</span><span class="o">.</span><span class="vm">__name__</span> <span class="o">==</span> <span class="s2">&quot;MultipleLRScheduler&quot;</span>
+
+    <span class="k">def</span> <span class="nf">_has_cyclic_scheduler</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">model_name</span><span class="p">:</span> <span class="nb">str</span><span class="p">):</span>
+        <span class="k">if</span> <span class="n">model_name</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">trainer</span><span class="o">.</span><span class="n">lr_scheduler</span><span class="o">.</span><span class="n">_schedulers</span><span class="p">:</span>
+            <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">trainer</span><span class="o">.</span><span class="n">lr_scheduler</span><span class="o">.</span><span class="n">_schedulers</span><span class="p">[</span><span class="n">model_name</span><span class="p">],</span> <span class="nb">list</span><span class="p">):</span>
+                <span class="k">return</span> <span class="nb">any</span><span class="p">(</span>
+                    <span class="p">[</span>
+                        <span class="bp">self</span><span class="o">.</span><span class="n">_is_cyclic</span><span class="p">(</span><span class="n">s</span><span class="p">)</span>
+                        <span class="k">for</span> <span class="n">s</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">trainer</span><span class="o">.</span><span class="n">lr_scheduler</span><span class="o">.</span><span class="n">_schedulers</span><span class="p">[</span><span class="n">model_name</span><span class="p">]</span>
+                    <span class="p">]</span>
+                <span class="p">)</span>
+            <span class="k">else</span><span class="p">:</span>
+                <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_is_cyclic</span><span class="p">(</span>
+                    <span class="bp">self</span><span class="o">.</span><span class="n">trainer</span><span class="o">.</span><span class="n">lr_scheduler</span><span class="o">.</span><span class="n">_schedulers</span><span class="p">[</span><span class="n">model_name</span><span class="p">]</span>
+                <span class="p">)</span>
+
+    <span class="nd">@staticmethod</span>
+    <span class="k">def</span> <span class="nf">_is_cyclic</span><span class="p">(</span><span class="n">scheduler</span><span class="p">:</span> <span class="n">LRScheduler</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">bool</span><span class="p">:</span>
+        <span class="k">return</span> <span class="s2">&quot;cycl&quot;</span> <span class="ow">in</span> <span class="n">scheduler</span><span class="o">.</span><span class="vm">__class__</span><span class="o">.</span><span class="vm">__name__</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span>
 </code></pre></div></td></tr></table></div>
-                  </details>
+              </details>
 
 
 
@@ -1707,17 +1964,7 @@ <h2 id="pytorch_widedeep.callbacks.ModelCheckpoint" class="doc doc-heading">
 
 
 <a href="#pytorch_widedeep.callbacks.ModelCheckpoint" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="nf">ModelCheckpoint</span><span class="p">(</span>
-    <span class="n">filepath</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">monitor</span><span class="o">=</span><span class="s2">&quot;val_loss&quot;</span><span class="p">,</span>
-    <span class="n">min_delta</span><span class="o">=</span><span class="mf">0.0</span><span class="p">,</span>
-    <span class="n">verbose</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span>
-    <span class="n">save_best_only</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
-    <span class="n">mode</span><span class="o">=</span><span class="s2">&quot;auto&quot;</span><span class="p">,</span>
-    <span class="n">period</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span>
-    <span class="n">max_save</span><span class="o">=-</span><span class="mi">1</span><span class="p">,</span>
-<span class="p">)</span>
-</code></pre></div>
+
 
     <div class="doc doc-contents first">
             <p class="doc doc-class-bases">
@@ -1732,135 +1979,195 @@ <h2 id="pytorch_widedeep.callbacks.ModelCheckpoint" class="doc doc-heading">
 
 
 <p><span class="doc-section-title">Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>filepath</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Full path to save the output weights. It must contain only the root of
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+          <th>Default</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>filepath</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Full path to save the output weights. It must contain only the root of
 the filenames. Epoch number and <code>.pt</code> extension (for pytorch) will be
 added. e.g. <code>filepath="path/to/output_weights/weights_out"</code> And the
 saved files in that directory will be named:
 <em>'weights_out_1.pt', 'weights_out_2.pt', ...</em>. If set to <code>None</code> the
 class just report best metric and best_epoch.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>monitor</code></b>
-              (<code>str</code>, default:
-                  <code>&#39;val_loss&#39;</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>quantity to monitor. Typically <em>'val_loss'</em> or metric name
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>monitor</code></td>
+            <td>
+                  <code>str</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>quantity to monitor. Typically <em>'val_loss'</em> or metric name
 (e.g. <em>'val_acc'</em>)</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>min_delta</code></b>
-              (<code>float</code>, default:
-                  <code>0.0</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>minimum change in the monitored quantity to qualify as an
+              </div>
+            </td>
+            <td>
+                  <code>&#39;val_loss&#39;</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>min_delta</code></td>
+            <td>
+                  <code>float</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>minimum change in the monitored quantity to qualify as an
 improvement, i.e. an absolute change of less than min_delta, will
 count as no improvement.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>verbose</code></b>
-              (<code>int</code>, default:
+              </div>
+            </td>
+            <td>
+                  <code>0.0</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>verbose</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>verbosity mode</p>
+              </div>
+            </td>
+            <td>
                   <code>0</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>verbosity mode</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>save_best_only</code></b>
-              (<code>bool</code>, default:
-                  <code>False</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>the latest best model according to the quantity monitored will not be
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>save_best_only</code></td>
+            <td>
+                  <code>bool</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>the latest best model according to the quantity monitored will not be
 overwritten.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>mode</code></b>
-              (<code>str</code>, default:
-                  <code>&#39;auto&#39;</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>If <code>save_best_only=True</code>, the decision to overwrite the current save
+              </div>
+            </td>
+            <td>
+                  <code>False</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>mode</code></td>
+            <td>
+                  <code>str</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>If <code>save_best_only=True</code>, the decision to overwrite the current save
 file is made based on either the maximization or the minimization of
 the monitored quantity. For <em>'acc'</em>, this should be <em>'max'</em>, for
 <em>'loss'</em> this should be <em>'min'</em>, etc. In '<em>auto'</em> mode, the
 direction is automatically inferred from the name of the monitored
 quantity.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>period</code></b>
-              (<code>int</code>, default:
+              </div>
+            </td>
+            <td>
+                  <code>&#39;auto&#39;</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>period</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Interval (number of epochs) between checkpoints.</p>
+              </div>
+            </td>
+            <td>
                   <code>1</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Interval (number of epochs) between checkpoints.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>max_save</code></b>
-              (<code>int</code>, default:
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>max_save</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Maximum number of outputs to save. If -1 will save all outputs</p>
+              </div>
+            </td>
+            <td>
                   <code>-1</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Maximum number of outputs to save. If -1 will save all outputs</p>
-          </div>
-        </li>
-    </ul>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
 
 <p><span class="doc-section-title">Attributes:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code><span title="pytorch_widedeep.callbacks.ModelCheckpoint.best">best</span></code></b>
-              (<code>float</code>)
-          –
-          <div class="doc-md-description">
-            <p>best metric</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code><span title="pytorch_widedeep.callbacks.ModelCheckpoint.best_epoch">best_epoch</span></code></b>
-              (<code>int</code>)
-          –
-          <div class="doc-md-description">
-            <p>best epoch</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code><span title="pytorch_widedeep.callbacks.ModelCheckpoint.best_state_dict">best_state_dict</span></code></b>
-              (<code>dict</code>)
-          –
-          <div class="doc-md-description">
-            <p>best model state dictionary.<br/>
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code><span title="pytorch_widedeep.callbacks.ModelCheckpoint.best">best</span></code></td>
+            <td>
+                  <code>float</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>best metric</p>
+              </div>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code><span title="pytorch_widedeep.callbacks.ModelCheckpoint.best_epoch">best_epoch</span></code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>best epoch</p>
+              </div>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code><span title="pytorch_widedeep.callbacks.ModelCheckpoint.best_state_dict">best_state_dict</span></code></td>
+            <td>
+                  <code>dict</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>best model state dictionary.<br/>
 To restore model to its best state use <code>Trainer.model.load_state_dict
 (model_checkpoint.best_state_dict)</code> where <code>model_checkpoint</code> is an
 instance of the class <code>ModelCheckpoint</code>. See the Examples folder in
 the repo or the Examples section in this documentation for details</p>
-          </div>
-        </li>
-    </ul>
+              </div>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
 
 <p><span class="doc-section-title">Examples:</span></p>
@@ -1876,9 +2183,78 @@ <h2 id="pytorch_widedeep.callbacks.ModelCheckpoint" class="doc doc-heading">
 <span class="gp">&gt;&gt;&gt; </span><span class="n">trainer</span> <span class="o">=</span> <span class="n">Trainer</span><span class="p">(</span><span class="n">model</span><span class="p">,</span> <span class="n">objective</span><span class="o">=</span><span class="s2">&quot;regression&quot;</span><span class="p">,</span> <span class="n">callbacks</span><span class="o">=</span><span class="p">[</span><span class="n">ModelCheckpoint</span><span class="p">(</span><span class="n">filepath</span><span class="o">=</span><span class="s1">&#39;checkpoints/weights_out&#39;</span><span class="p">)])</span>
 </code></pre></div>
 
-                  <details class="quote">
-                    <summary>Source code in <code>pytorch_widedeep/callbacks.py</code></summary>
-                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">446</span>
+              <details class="quote">
+                <summary>Source code in <code>pytorch_widedeep/callbacks.py</code></summary>
+                <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">377</span>
+<span class="normal">378</span>
+<span class="normal">379</span>
+<span class="normal">380</span>
+<span class="normal">381</span>
+<span class="normal">382</span>
+<span class="normal">383</span>
+<span class="normal">384</span>
+<span class="normal">385</span>
+<span class="normal">386</span>
+<span class="normal">387</span>
+<span class="normal">388</span>
+<span class="normal">389</span>
+<span class="normal">390</span>
+<span class="normal">391</span>
+<span class="normal">392</span>
+<span class="normal">393</span>
+<span class="normal">394</span>
+<span class="normal">395</span>
+<span class="normal">396</span>
+<span class="normal">397</span>
+<span class="normal">398</span>
+<span class="normal">399</span>
+<span class="normal">400</span>
+<span class="normal">401</span>
+<span class="normal">402</span>
+<span class="normal">403</span>
+<span class="normal">404</span>
+<span class="normal">405</span>
+<span class="normal">406</span>
+<span class="normal">407</span>
+<span class="normal">408</span>
+<span class="normal">409</span>
+<span class="normal">410</span>
+<span class="normal">411</span>
+<span class="normal">412</span>
+<span class="normal">413</span>
+<span class="normal">414</span>
+<span class="normal">415</span>
+<span class="normal">416</span>
+<span class="normal">417</span>
+<span class="normal">418</span>
+<span class="normal">419</span>
+<span class="normal">420</span>
+<span class="normal">421</span>
+<span class="normal">422</span>
+<span class="normal">423</span>
+<span class="normal">424</span>
+<span class="normal">425</span>
+<span class="normal">426</span>
+<span class="normal">427</span>
+<span class="normal">428</span>
+<span class="normal">429</span>
+<span class="normal">430</span>
+<span class="normal">431</span>
+<span class="normal">432</span>
+<span class="normal">433</span>
+<span class="normal">434</span>
+<span class="normal">435</span>
+<span class="normal">436</span>
+<span class="normal">437</span>
+<span class="normal">438</span>
+<span class="normal">439</span>
+<span class="normal">440</span>
+<span class="normal">441</span>
+<span class="normal">442</span>
+<span class="normal">443</span>
+<span class="normal">444</span>
+<span class="normal">445</span>
+<span class="normal">446</span>
 <span class="normal">447</span>
 <span class="normal">448</span>
 <span class="normal">449</span>
@@ -1940,71 +2316,278 @@ <h2 id="pytorch_widedeep.callbacks.ModelCheckpoint" class="doc doc-heading">
 <span class="normal">505</span>
 <span class="normal">506</span>
 <span class="normal">507</span>
-<span class="normal">508</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
-    <span class="bp">self</span><span class="p">,</span>
-    <span class="n">filepath</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">monitor</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;val_loss&quot;</span><span class="p">,</span>
-    <span class="n">min_delta</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.0</span><span class="p">,</span>
-    <span class="n">verbose</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">0</span><span class="p">,</span>
-    <span class="n">save_best_only</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
-    <span class="n">mode</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;auto&quot;</span><span class="p">,</span>
-    <span class="n">period</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1</span><span class="p">,</span>
-    <span class="n">max_save</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="o">-</span><span class="mi">1</span><span class="p">,</span>
-<span class="p">):</span>
-    <span class="nb">super</span><span class="p">(</span><span class="n">ModelCheckpoint</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
-
-    <span class="bp">self</span><span class="o">.</span><span class="n">filepath</span> <span class="o">=</span> <span class="n">filepath</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">monitor</span> <span class="o">=</span> <span class="n">monitor</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">min_delta</span> <span class="o">=</span> <span class="n">min_delta</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">verbose</span> <span class="o">=</span> <span class="n">verbose</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">save_best_only</span> <span class="o">=</span> <span class="n">save_best_only</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">mode</span> <span class="o">=</span> <span class="n">mode</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">period</span> <span class="o">=</span> <span class="n">period</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">max_save</span> <span class="o">=</span> <span class="n">max_save</span>
-
-    <span class="bp">self</span><span class="o">.</span><span class="n">epochs_since_last_save</span> <span class="o">=</span> <span class="mi">0</span>
-
-    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">filepath</span><span class="p">:</span>
-        <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">filepath</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s2">&quot;/&quot;</span><span class="p">)[:</span><span class="o">-</span><span class="mi">1</span><span class="p">])</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
-            <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
-                <span class="s2">&quot;&#39;filepath&#39; must be the full path to save the output weights,&quot;</span>
-                <span class="s2">&quot; including the root of the filenames. e.g. &#39;checkpoints/weights_out&#39;&quot;</span>
+<span class="normal">508</span>
+<span class="normal">509</span>
+<span class="normal">510</span>
+<span class="normal">511</span>
+<span class="normal">512</span>
+<span class="normal">513</span>
+<span class="normal">514</span>
+<span class="normal">515</span>
+<span class="normal">516</span>
+<span class="normal">517</span>
+<span class="normal">518</span>
+<span class="normal">519</span>
+<span class="normal">520</span>
+<span class="normal">521</span>
+<span class="normal">522</span>
+<span class="normal">523</span>
+<span class="normal">524</span>
+<span class="normal">525</span>
+<span class="normal">526</span>
+<span class="normal">527</span>
+<span class="normal">528</span>
+<span class="normal">529</span>
+<span class="normal">530</span>
+<span class="normal">531</span>
+<span class="normal">532</span>
+<span class="normal">533</span>
+<span class="normal">534</span>
+<span class="normal">535</span>
+<span class="normal">536</span>
+<span class="normal">537</span>
+<span class="normal">538</span>
+<span class="normal">539</span>
+<span class="normal">540</span>
+<span class="normal">541</span>
+<span class="normal">542</span>
+<span class="normal">543</span>
+<span class="normal">544</span>
+<span class="normal">545</span>
+<span class="normal">546</span>
+<span class="normal">547</span>
+<span class="normal">548</span>
+<span class="normal">549</span>
+<span class="normal">550</span>
+<span class="normal">551</span>
+<span class="normal">552</span>
+<span class="normal">553</span>
+<span class="normal">554</span>
+<span class="normal">555</span>
+<span class="normal">556</span>
+<span class="normal">557</span>
+<span class="normal">558</span>
+<span class="normal">559</span>
+<span class="normal">560</span>
+<span class="normal">561</span>
+<span class="normal">562</span>
+<span class="normal">563</span>
+<span class="normal">564</span>
+<span class="normal">565</span>
+<span class="normal">566</span>
+<span class="normal">567</span>
+<span class="normal">568</span>
+<span class="normal">569</span>
+<span class="normal">570</span>
+<span class="normal">571</span>
+<span class="normal">572</span>
+<span class="normal">573</span>
+<span class="normal">574</span>
+<span class="normal">575</span>
+<span class="normal">576</span>
+<span class="normal">577</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">class</span> <span class="nc">ModelCheckpoint</span><span class="p">(</span><span class="n">Callback</span><span class="p">):</span>
+<span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Saves the model after every epoch.</span>
+
+<span class="sd">    This class is almost identical to the corresponding keras class.</span>
+<span class="sd">    Therefore, **credit** to the Keras Team.</span>
+
+<span class="sd">    Callbacks are passed as input parameters to the `Trainer` class. See</span>
+<span class="sd">    `pytorch_widedeep.trainer.Trainer`</span>
+
+<span class="sd">    Parameters</span>
+<span class="sd">    ----------</span>
+<span class="sd">    filepath: str, default=None</span>
+<span class="sd">        Full path to save the output weights. It must contain only the root of</span>
+<span class="sd">        the filenames. Epoch number and `.pt` extension (for pytorch) will be</span>
+<span class="sd">        added. e.g. `filepath=&quot;path/to/output_weights/weights_out&quot;` And the</span>
+<span class="sd">        saved files in that directory will be named:</span>
+<span class="sd">        _&#39;weights_out_1.pt&#39;, &#39;weights_out_2.pt&#39;, ..._. If set to `None` the</span>
+<span class="sd">        class just report best metric and best_epoch.</span>
+<span class="sd">    monitor: str, default=&quot;loss&quot;</span>
+<span class="sd">        quantity to monitor. Typically _&#39;val_loss&#39;_ or metric name</span>
+<span class="sd">        (e.g. _&#39;val_acc&#39;_)</span>
+<span class="sd">    min_delta: float, default=0.</span>
+<span class="sd">        minimum change in the monitored quantity to qualify as an</span>
+<span class="sd">        improvement, i.e. an absolute change of less than min_delta, will</span>
+<span class="sd">        count as no improvement.</span>
+<span class="sd">    verbose:int, default=0</span>
+<span class="sd">        verbosity mode</span>
+<span class="sd">    save_best_only: bool, default=False,</span>
+<span class="sd">        the latest best model according to the quantity monitored will not be</span>
+<span class="sd">        overwritten.</span>
+<span class="sd">    mode: str, default=&quot;auto&quot;</span>
+<span class="sd">        If `save_best_only=True`, the decision to overwrite the current save</span>
+<span class="sd">        file is made based on either the maximization or the minimization of</span>
+<span class="sd">        the monitored quantity. For _&#39;acc&#39;_, this should be _&#39;max&#39;_, for</span>
+<span class="sd">        _&#39;loss&#39;_ this should be _&#39;min&#39;_, etc. In &#39;_auto&#39;_ mode, the</span>
+<span class="sd">        direction is automatically inferred from the name of the monitored</span>
+<span class="sd">        quantity.</span>
+<span class="sd">    period: int, default=1</span>
+<span class="sd">        Interval (number of epochs) between checkpoints.</span>
+<span class="sd">    max_save: int, default=-1</span>
+<span class="sd">        Maximum number of outputs to save. If -1 will save all outputs</span>
+
+<span class="sd">    Attributes</span>
+<span class="sd">    ----------</span>
+<span class="sd">    best: float</span>
+<span class="sd">        best metric</span>
+<span class="sd">    best_epoch: int</span>
+<span class="sd">        best epoch</span>
+<span class="sd">    best_state_dict: dict</span>
+<span class="sd">        best model state dictionary.&lt;br/&gt;</span>
+<span class="sd">        To restore model to its best state use `Trainer.model.load_state_dict</span>
+<span class="sd">        (model_checkpoint.best_state_dict)` where `model_checkpoint` is an</span>
+<span class="sd">        instance of the class `ModelCheckpoint`. See the Examples folder in</span>
+<span class="sd">        the repo or the Examples section in this documentation for details</span>
+
+<span class="sd">    Examples</span>
+<span class="sd">    --------</span>
+<span class="sd">    &gt;&gt;&gt; from pytorch_widedeep.callbacks import ModelCheckpoint</span>
+<span class="sd">    &gt;&gt;&gt; from pytorch_widedeep.models import TabMlp, Wide, WideDeep</span>
+<span class="sd">    &gt;&gt;&gt; from pytorch_widedeep.training import Trainer</span>
+<span class="sd">    &gt;&gt;&gt;</span>
+<span class="sd">    &gt;&gt;&gt; embed_input = [(u, i, j) for u, i, j in zip([&quot;a&quot;, &quot;b&quot;, &quot;c&quot;][:4], [4] * 3, [8] * 3)]</span>
+<span class="sd">    &gt;&gt;&gt; column_idx = {k: v for v, k in enumerate([&quot;a&quot;, &quot;b&quot;, &quot;c&quot;])}</span>
+<span class="sd">    &gt;&gt;&gt; wide = Wide(10, 1)</span>
+<span class="sd">    &gt;&gt;&gt; deep = TabMlp(mlp_hidden_dims=[8, 4], column_idx=column_idx, cat_embed_input=embed_input)</span>
+<span class="sd">    &gt;&gt;&gt; model = WideDeep(wide, deep)</span>
+<span class="sd">    &gt;&gt;&gt; trainer = Trainer(model, objective=&quot;regression&quot;, callbacks=[ModelCheckpoint(filepath=&#39;checkpoints/weights_out&#39;)])</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+
+    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span>
+        <span class="n">filepath</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">monitor</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;val_loss&quot;</span><span class="p">,</span>
+        <span class="n">min_delta</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.0</span><span class="p">,</span>
+        <span class="n">verbose</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">0</span><span class="p">,</span>
+        <span class="n">save_best_only</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
+        <span class="n">mode</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;auto&quot;</span><span class="p">,</span>
+        <span class="n">period</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1</span><span class="p">,</span>
+        <span class="n">max_save</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="o">-</span><span class="mi">1</span><span class="p">,</span>
+    <span class="p">):</span>
+        <span class="nb">super</span><span class="p">(</span><span class="n">ModelCheckpoint</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">filepath</span> <span class="o">=</span> <span class="n">filepath</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">monitor</span> <span class="o">=</span> <span class="n">monitor</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">min_delta</span> <span class="o">=</span> <span class="n">min_delta</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">verbose</span> <span class="o">=</span> <span class="n">verbose</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">save_best_only</span> <span class="o">=</span> <span class="n">save_best_only</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">mode</span> <span class="o">=</span> <span class="n">mode</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">period</span> <span class="o">=</span> <span class="n">period</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">max_save</span> <span class="o">=</span> <span class="n">max_save</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">epochs_since_last_save</span> <span class="o">=</span> <span class="mi">0</span>
+
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">filepath</span><span class="p">:</span>
+            <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">filepath</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s2">&quot;/&quot;</span><span class="p">)[:</span><span class="o">-</span><span class="mi">1</span><span class="p">])</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
+                <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
+                    <span class="s2">&quot;&#39;filepath&#39; must be the full path to save the output weights,&quot;</span>
+                    <span class="s2">&quot; including the root of the filenames. e.g. &#39;checkpoints/weights_out&#39;&quot;</span>
+                <span class="p">)</span>
+
+            <span class="n">root_dir</span> <span class="o">=</span> <span class="p">(</span><span class="s2">&quot;/&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">filepath</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s2">&quot;/&quot;</span><span class="p">)[:</span><span class="o">-</span><span class="mi">1</span><span class="p">])</span>
+            <span class="k">if</span> <span class="ow">not</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">exists</span><span class="p">(</span><span class="n">root_dir</span><span class="p">):</span>
+                <span class="n">os</span><span class="o">.</span><span class="n">makedirs</span><span class="p">(</span><span class="n">root_dir</span><span class="p">)</span>
+
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">max_save</span> <span class="o">&gt;</span> <span class="mi">0</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">old_files</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
+
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mode</span> <span class="ow">not</span> <span class="ow">in</span> <span class="p">[</span><span class="s2">&quot;auto&quot;</span><span class="p">,</span> <span class="s2">&quot;min&quot;</span><span class="p">,</span> <span class="s2">&quot;max&quot;</span><span class="p">]:</span>
+            <span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span>
+                <span class="s2">&quot;ModelCheckpoint mode </span><span class="si">%s</span><span class="s2"> is unknown, &quot;</span>
+                <span class="s2">&quot;fallback to auto mode.&quot;</span> <span class="o">%</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">mode</span><span class="p">),</span>
+                <span class="ne">RuntimeWarning</span><span class="p">,</span>
             <span class="p">)</span>
-
-        <span class="n">root_dir</span> <span class="o">=</span> <span class="p">(</span><span class="s2">&quot;/&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">filepath</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s2">&quot;/&quot;</span><span class="p">)[:</span><span class="o">-</span><span class="mi">1</span><span class="p">])</span>
-        <span class="k">if</span> <span class="ow">not</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">exists</span><span class="p">(</span><span class="n">root_dir</span><span class="p">):</span>
-            <span class="n">os</span><span class="o">.</span><span class="n">makedirs</span><span class="p">(</span><span class="n">root_dir</span><span class="p">)</span>
-
-    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">max_save</span> <span class="o">&gt;</span> <span class="mi">0</span><span class="p">:</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">old_files</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
-
-    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mode</span> <span class="ow">not</span> <span class="ow">in</span> <span class="p">[</span><span class="s2">&quot;auto&quot;</span><span class="p">,</span> <span class="s2">&quot;min&quot;</span><span class="p">,</span> <span class="s2">&quot;max&quot;</span><span class="p">]:</span>
-        <span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span>
-            <span class="s2">&quot;ModelCheckpoint mode </span><span class="si">%s</span><span class="s2"> is unknown, &quot;</span>
-            <span class="s2">&quot;fallback to auto mode.&quot;</span> <span class="o">%</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">mode</span><span class="p">),</span>
-            <span class="ne">RuntimeWarning</span><span class="p">,</span>
-        <span class="p">)</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">mode</span> <span class="o">=</span> <span class="s2">&quot;auto&quot;</span>
-    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mode</span> <span class="o">==</span> <span class="s2">&quot;min&quot;</span><span class="p">:</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">monitor_op</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">less</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">best</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">Inf</span>
-    <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">mode</span> <span class="o">==</span> <span class="s2">&quot;max&quot;</span><span class="p">:</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">monitor_op</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">greater</span>  <span class="c1"># type: ignore[assignment]</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">best</span> <span class="o">=</span> <span class="o">-</span><span class="n">np</span><span class="o">.</span><span class="n">Inf</span>
-    <span class="k">else</span><span class="p">:</span>
-        <span class="k">if</span> <span class="n">_is_metric</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">monitor</span><span class="p">):</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">mode</span> <span class="o">=</span> <span class="s2">&quot;auto&quot;</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mode</span> <span class="o">==</span> <span class="s2">&quot;min&quot;</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">monitor_op</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">less</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">best</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">Inf</span>
+        <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">mode</span> <span class="o">==</span> <span class="s2">&quot;max&quot;</span><span class="p">:</span>
             <span class="bp">self</span><span class="o">.</span><span class="n">monitor_op</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">greater</span>  <span class="c1"># type: ignore[assignment]</span>
             <span class="bp">self</span><span class="o">.</span><span class="n">best</span> <span class="o">=</span> <span class="o">-</span><span class="n">np</span><span class="o">.</span><span class="n">Inf</span>
         <span class="k">else</span><span class="p">:</span>
-            <span class="bp">self</span><span class="o">.</span><span class="n">monitor_op</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">less</span>
-            <span class="bp">self</span><span class="o">.</span><span class="n">best</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">Inf</span>
-
-    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">monitor_op</span> <span class="o">==</span> <span class="n">np</span><span class="o">.</span><span class="n">greater</span><span class="p">:</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">min_delta</span> <span class="o">*=</span> <span class="mi">1</span>
-    <span class="k">else</span><span class="p">:</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">min_delta</span> <span class="o">*=</span> <span class="o">-</span><span class="mi">1</span>
+            <span class="k">if</span> <span class="n">_is_metric</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">monitor</span><span class="p">):</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">monitor_op</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">greater</span>  <span class="c1"># type: ignore[assignment]</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">best</span> <span class="o">=</span> <span class="o">-</span><span class="n">np</span><span class="o">.</span><span class="n">Inf</span>
+            <span class="k">else</span><span class="p">:</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">monitor_op</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">less</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">best</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">Inf</span>
+
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">monitor_op</span> <span class="o">==</span> <span class="n">np</span><span class="o">.</span><span class="n">greater</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">min_delta</span> <span class="o">*=</span> <span class="mi">1</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">min_delta</span> <span class="o">*=</span> <span class="o">-</span><span class="mi">1</span>
+
+    <span class="k">def</span> <span class="nf">on_epoch_end</span><span class="p">(</span>  <span class="c1"># noqa: C901</span>
+        <span class="bp">self</span><span class="p">,</span> <span class="n">epoch</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span> <span class="n">logs</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Dict</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> <span class="n">metric</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span>
+    <span class="p">):</span>
+        <span class="n">logs</span> <span class="o">=</span> <span class="n">logs</span> <span class="ow">or</span> <span class="p">{}</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">epochs_since_last_save</span> <span class="o">+=</span> <span class="mi">1</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">epochs_since_last_save</span> <span class="o">&gt;=</span> <span class="bp">self</span><span class="o">.</span><span class="n">period</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">epochs_since_last_save</span> <span class="o">=</span> <span class="mi">0</span>
+            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">filepath</span><span class="p">:</span>
+                <span class="n">filepath</span> <span class="o">=</span> <span class="s2">&quot;</span><span class="si">{}</span><span class="s2">_</span><span class="si">{}</span><span class="s2">.p&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">filepath</span><span class="p">,</span> <span class="n">epoch</span> <span class="o">+</span> <span class="mi">1</span><span class="p">)</span>
+            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">save_best_only</span><span class="p">:</span>
+                <span class="n">current</span> <span class="o">=</span> <span class="n">logs</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">monitor</span><span class="p">)</span>
+                <span class="k">if</span> <span class="n">current</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+                    <span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span>
+                        <span class="s2">&quot;Can save best model only with </span><span class="si">%s</span><span class="s2"> available, &quot;</span>
+                        <span class="s2">&quot;skipping.&quot;</span> <span class="o">%</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">monitor</span><span class="p">),</span>
+                        <span class="ne">RuntimeWarning</span><span class="p">,</span>
+                    <span class="p">)</span>
+                <span class="k">else</span><span class="p">:</span>
+                    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">monitor_op</span><span class="p">(</span><span class="n">current</span> <span class="o">-</span> <span class="bp">self</span><span class="o">.</span><span class="n">min_delta</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">best</span><span class="p">):</span>
+                        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">verbose</span> <span class="o">&gt;</span> <span class="mi">0</span><span class="p">:</span>
+                            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">filepath</span><span class="p">:</span>
+                                <span class="nb">print</span><span class="p">(</span>
+                                    <span class="sa">f</span><span class="s2">&quot;</span><span class="se">\n</span><span class="s2">Epoch </span><span class="si">{</span><span class="n">epoch</span><span class="w"> </span><span class="o">+</span><span class="w"> </span><span class="mi">1</span><span class="si">}</span><span class="s2">: </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">monitor</span><span class="si">}</span><span class="s2"> improved from </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">best</span><span class="si">:</span><span class="s2">.5f</span><span class="si">}</span><span class="s2"> to </span><span class="si">{</span><span class="n">current</span><span class="si">:</span><span class="s2">.5f</span><span class="si">}</span><span class="s2"> &quot;</span>
+                                    <span class="sa">f</span><span class="s2">&quot;Saving model to </span><span class="si">{</span><span class="n">filepath</span><span class="si">}</span><span class="s2">&quot;</span>
+                                <span class="p">)</span>
+                            <span class="k">else</span><span class="p">:</span>
+                                <span class="nb">print</span><span class="p">(</span>
+                                    <span class="sa">f</span><span class="s2">&quot;</span><span class="se">\n</span><span class="s2">Epoch </span><span class="si">{</span><span class="n">epoch</span><span class="w"> </span><span class="o">+</span><span class="w"> </span><span class="mi">1</span><span class="si">}</span><span class="s2">: </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">monitor</span><span class="si">}</span><span class="s2"> improved from </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">best</span><span class="si">:</span><span class="s2">.5f</span><span class="si">}</span><span class="s2"> to </span><span class="si">{</span><span class="n">current</span><span class="si">:</span><span class="s2">.5f</span><span class="si">}</span><span class="s2"> &quot;</span>
+                                <span class="p">)</span>
+                        <span class="bp">self</span><span class="o">.</span><span class="n">best</span> <span class="o">=</span> <span class="n">current</span>
+                        <span class="bp">self</span><span class="o">.</span><span class="n">best_epoch</span> <span class="o">=</span> <span class="n">epoch</span>
+                        <span class="bp">self</span><span class="o">.</span><span class="n">best_state_dict</span> <span class="o">=</span> <span class="n">copy</span><span class="o">.</span><span class="n">deepcopy</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">model</span><span class="o">.</span><span class="n">state_dict</span><span class="p">())</span>
+                        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">filepath</span><span class="p">:</span>
+                            <span class="n">torch</span><span class="o">.</span><span class="n">save</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">best_state_dict</span><span class="p">,</span> <span class="n">filepath</span><span class="p">)</span>
+                            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">max_save</span> <span class="o">&gt;</span> <span class="mi">0</span><span class="p">:</span>
+                                <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">old_files</span><span class="p">)</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">max_save</span><span class="p">:</span>
+                                    <span class="k">try</span><span class="p">:</span>
+                                        <span class="n">os</span><span class="o">.</span><span class="n">remove</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">old_files</span><span class="p">[</span><span class="mi">0</span><span class="p">])</span>
+                                    <span class="k">except</span> <span class="ne">FileNotFoundError</span><span class="p">:</span>
+                                        <span class="k">pass</span>
+                                    <span class="bp">self</span><span class="o">.</span><span class="n">old_files</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">old_files</span><span class="p">[</span><span class="mi">1</span><span class="p">:]</span>
+                                <span class="bp">self</span><span class="o">.</span><span class="n">old_files</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">filepath</span><span class="p">)</span>
+                    <span class="k">else</span><span class="p">:</span>
+                        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">verbose</span> <span class="o">&gt;</span> <span class="mi">0</span><span class="p">:</span>
+                            <span class="nb">print</span><span class="p">(</span>
+                                <span class="sa">f</span><span class="s2">&quot;</span><span class="se">\n</span><span class="s2">Epoch </span><span class="si">{</span><span class="n">epoch</span><span class="w"> </span><span class="o">+</span><span class="w"> </span><span class="mi">1</span><span class="si">}</span><span class="s2">: </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">monitor</span><span class="si">}</span><span class="s2"> did not improve from </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">best</span><span class="si">:</span><span class="s2">.5f</span><span class="si">}</span><span class="s2"> &quot;</span>
+                                <span class="sa">f</span><span class="s2">&quot; considering a &#39;min_delta&#39; improvement of </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">min_delta</span><span class="si">:</span><span class="s2">.5f</span><span class="si">}</span><span class="s2">&quot;</span>
+                            <span class="p">)</span>
+            <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">save_best_only</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">filepath</span><span class="p">:</span>
+                <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">verbose</span> <span class="o">&gt;</span> <span class="mi">0</span><span class="p">:</span>
+                    <span class="nb">print</span><span class="p">(</span><span class="s2">&quot;</span><span class="se">\n</span><span class="s2">Epoch </span><span class="si">%05d</span><span class="s2">: saving model to </span><span class="si">%s</span><span class="s2">&quot;</span> <span class="o">%</span> <span class="p">(</span><span class="n">epoch</span> <span class="o">+</span> <span class="mi">1</span><span class="p">,</span> <span class="n">filepath</span><span class="p">))</span>
+                <span class="n">torch</span><span class="o">.</span><span class="n">save</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">model</span><span class="o">.</span><span class="n">state_dict</span><span class="p">(),</span> <span class="n">filepath</span><span class="p">)</span>
+                <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">max_save</span> <span class="o">&gt;</span> <span class="mi">0</span><span class="p">:</span>
+                    <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">old_files</span><span class="p">)</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">max_save</span><span class="p">:</span>
+                        <span class="k">try</span><span class="p">:</span>
+                            <span class="n">os</span><span class="o">.</span><span class="n">remove</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">old_files</span><span class="p">[</span><span class="mi">0</span><span class="p">])</span>
+                        <span class="k">except</span> <span class="ne">FileNotFoundError</span><span class="p">:</span>
+                            <span class="k">pass</span>
+                        <span class="bp">self</span><span class="o">.</span><span class="n">old_files</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">old_files</span><span class="p">[</span><span class="mi">1</span><span class="p">:]</span>
+                    <span class="bp">self</span><span class="o">.</span><span class="n">old_files</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">filepath</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">__getstate__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="n">d</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="vm">__dict__</span>
+        <span class="n">self_dict</span> <span class="o">=</span> <span class="p">{</span><span class="n">k</span><span class="p">:</span> <span class="n">d</span><span class="p">[</span><span class="n">k</span><span class="p">]</span> <span class="k">for</span> <span class="n">k</span> <span class="ow">in</span> <span class="n">d</span> <span class="k">if</span> <span class="n">k</span> <span class="ow">not</span> <span class="ow">in</span> <span class="p">[</span><span class="s2">&quot;trainer&quot;</span><span class="p">,</span> <span class="s2">&quot;model&quot;</span><span class="p">]}</span>
+        <span class="k">return</span> <span class="n">self_dict</span>
+
+    <span class="k">def</span> <span class="nf">__setstate__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">state</span><span class="p">):</span>
+        <span class="bp">self</span><span class="o">.</span><span class="vm">__dict__</span> <span class="o">=</span> <span class="n">state</span>
 </code></pre></div></td></tr></table></div>
-                  </details>
+              </details>
 
 
 
@@ -2035,16 +2618,7 @@ <h2 id="pytorch_widedeep.callbacks.EarlyStopping" class="doc doc-heading">
 
 
 <a href="#pytorch_widedeep.callbacks.EarlyStopping" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="nf">EarlyStopping</span><span class="p">(</span>
-    <span class="n">monitor</span><span class="o">=</span><span class="s2">&quot;val_loss&quot;</span><span class="p">,</span>
-    <span class="n">min_delta</span><span class="o">=</span><span class="mf">0.0</span><span class="p">,</span>
-    <span class="n">patience</span><span class="o">=</span><span class="mi">10</span><span class="p">,</span>
-    <span class="n">verbose</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span>
-    <span class="n">mode</span><span class="o">=</span><span class="s2">&quot;auto&quot;</span><span class="p">,</span>
-    <span class="n">baseline</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">restore_best_weights</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
-<span class="p">)</span>
-</code></pre></div>
+
 
     <div class="doc doc-contents first">
             <p class="doc doc-class-bases">
@@ -2059,110 +2633,163 @@ <h2 id="pytorch_widedeep.callbacks.EarlyStopping" class="doc doc-heading">
 
 
 <p><span class="doc-section-title">Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>monitor</code></b>
-              (<code>str</code>, default:
-                  <code>&#39;val_loss&#39;</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Quantity to monitor. Typically <em>'val_loss'</em> or metric name
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+          <th>Default</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>monitor</code></td>
+            <td>
+                  <code>str</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Quantity to monitor. Typically <em>'val_loss'</em> or metric name
 (e.g. <em>'val_acc'</em>)</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>min_delta</code></b>
-              (<code>float</code>, default:
-                  <code>0.0</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>minimum change in the monitored quantity to qualify as an
+              </div>
+            </td>
+            <td>
+                  <code>&#39;val_loss&#39;</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>min_delta</code></td>
+            <td>
+                  <code>float</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>minimum change in the monitored quantity to qualify as an
 improvement, i.e. an absolute change of less than min_delta, will
 count as no improvement.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>patience</code></b>
-              (<code>int</code>, default:
-                  <code>10</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Number of epochs that produced the monitored quantity with no
+              </div>
+            </td>
+            <td>
+                  <code>0.0</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>patience</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Number of epochs that produced the monitored quantity with no
 improvement after which training will be stopped.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>verbose</code></b>
-              (<code>int</code>, default:
+              </div>
+            </td>
+            <td>
+                  <code>10</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>verbose</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>verbosity mode.</p>
+              </div>
+            </td>
+            <td>
                   <code>0</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>verbosity mode.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>mode</code></b>
-              (<code>str</code>, default:
-                  <code>&#39;auto&#39;</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>one of <em>{'auto', 'min', 'max'}</em>. In <em>'min'</em> mode, training will
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>mode</code></td>
+            <td>
+                  <code>str</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>one of <em>{'auto', 'min', 'max'}</em>. In <em>'min'</em> mode, training will
 stop when the quantity monitored has stopped decreasing; in <em>'max'</em>
 mode it will stop when the quantity monitored has stopped increasing;
 in <em>'auto'</em> mode, the direction is automatically inferred from the
 name of the monitored quantity.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>baseline</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Baseline value for the monitored quantity to reach. Training will
+              </div>
+            </td>
+            <td>
+                  <code>&#39;auto&#39;</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>baseline</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Baseline value for the monitored quantity to reach. Training will
 stop if the model does not show improvement over the baseline.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>restore_best_weights</code></b>
-              (<code>bool</code>, default:
-                  <code>False</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Whether to restore model weights from the epoch with the best
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>restore_best_weights</code></td>
+            <td>
+                  <code>bool</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Whether to restore model weights from the epoch with the best
 value of the monitored quantity. If <code>False</code>, the model weights
 obtained at the last step of training are used.</p>
-          </div>
-        </li>
-    </ul>
+              </div>
+            </td>
+            <td>
+                  <code>False</code>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
 
 <p><span class="doc-section-title">Attributes:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code><span title="pytorch_widedeep.callbacks.EarlyStopping.best">best</span></code></b>
-              (<code>float</code>)
-          –
-          <div class="doc-md-description">
-            <p>best metric</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code><span title="pytorch_widedeep.callbacks.EarlyStopping.stopped_epoch">stopped_epoch</span></code></b>
-              (<code>int</code>)
-          –
-          <div class="doc-md-description">
-            <p>epoch when the training stopped</p>
-          </div>
-        </li>
-    </ul>
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code><span title="pytorch_widedeep.callbacks.EarlyStopping.best">best</span></code></td>
+            <td>
+                  <code>float</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>best metric</p>
+              </div>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code><span title="pytorch_widedeep.callbacks.EarlyStopping.stopped_epoch">stopped_epoch</span></code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>epoch when the training stopped</p>
+              </div>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
 
 <p><span class="doc-section-title">Examples:</span></p>
@@ -2178,9 +2805,67 @@ <h2 id="pytorch_widedeep.callbacks.EarlyStopping" class="doc doc-heading">
 <span class="gp">&gt;&gt;&gt; </span><span class="n">trainer</span> <span class="o">=</span> <span class="n">Trainer</span><span class="p">(</span><span class="n">model</span><span class="p">,</span> <span class="n">objective</span><span class="o">=</span><span class="s2">&quot;regression&quot;</span><span class="p">,</span> <span class="n">callbacks</span><span class="o">=</span><span class="p">[</span><span class="n">EarlyStopping</span><span class="p">(</span><span class="n">patience</span><span class="o">=</span><span class="mi">10</span><span class="p">)])</span>
 </code></pre></div>
 
-                  <details class="quote">
-                    <summary>Source code in <code>pytorch_widedeep/callbacks.py</code></summary>
-                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">638</span>
+              <details class="quote">
+                <summary>Source code in <code>pytorch_widedeep/callbacks.py</code></summary>
+                <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">580</span>
+<span class="normal">581</span>
+<span class="normal">582</span>
+<span class="normal">583</span>
+<span class="normal">584</span>
+<span class="normal">585</span>
+<span class="normal">586</span>
+<span class="normal">587</span>
+<span class="normal">588</span>
+<span class="normal">589</span>
+<span class="normal">590</span>
+<span class="normal">591</span>
+<span class="normal">592</span>
+<span class="normal">593</span>
+<span class="normal">594</span>
+<span class="normal">595</span>
+<span class="normal">596</span>
+<span class="normal">597</span>
+<span class="normal">598</span>
+<span class="normal">599</span>
+<span class="normal">600</span>
+<span class="normal">601</span>
+<span class="normal">602</span>
+<span class="normal">603</span>
+<span class="normal">604</span>
+<span class="normal">605</span>
+<span class="normal">606</span>
+<span class="normal">607</span>
+<span class="normal">608</span>
+<span class="normal">609</span>
+<span class="normal">610</span>
+<span class="normal">611</span>
+<span class="normal">612</span>
+<span class="normal">613</span>
+<span class="normal">614</span>
+<span class="normal">615</span>
+<span class="normal">616</span>
+<span class="normal">617</span>
+<span class="normal">618</span>
+<span class="normal">619</span>
+<span class="normal">620</span>
+<span class="normal">621</span>
+<span class="normal">622</span>
+<span class="normal">623</span>
+<span class="normal">624</span>
+<span class="normal">625</span>
+<span class="normal">626</span>
+<span class="normal">627</span>
+<span class="normal">628</span>
+<span class="normal">629</span>
+<span class="normal">630</span>
+<span class="normal">631</span>
+<span class="normal">632</span>
+<span class="normal">633</span>
+<span class="normal">634</span>
+<span class="normal">635</span>
+<span class="normal">636</span>
+<span class="normal">637</span>
+<span class="normal">638</span>
 <span class="normal">639</span>
 <span class="normal">640</span>
 <span class="normal">641</span>
@@ -2225,54 +2910,226 @@ <h2 id="pytorch_widedeep.callbacks.EarlyStopping" class="doc doc-heading">
 <span class="normal">680</span>
 <span class="normal">681</span>
 <span class="normal">682</span>
-<span class="normal">683</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
-    <span class="bp">self</span><span class="p">,</span>
-    <span class="n">monitor</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;val_loss&quot;</span><span class="p">,</span>
-    <span class="n">min_delta</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.0</span><span class="p">,</span>
-    <span class="n">patience</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">10</span><span class="p">,</span>
-    <span class="n">verbose</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">0</span><span class="p">,</span>
-    <span class="n">mode</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;auto&quot;</span><span class="p">,</span>
-    <span class="n">baseline</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">restore_best_weights</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
-<span class="p">):</span>
-    <span class="nb">super</span><span class="p">(</span><span class="n">EarlyStopping</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
-
-    <span class="bp">self</span><span class="o">.</span><span class="n">monitor</span> <span class="o">=</span> <span class="n">monitor</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">min_delta</span> <span class="o">=</span> <span class="n">min_delta</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">patience</span> <span class="o">=</span> <span class="n">patience</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">verbose</span> <span class="o">=</span> <span class="n">verbose</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">mode</span> <span class="o">=</span> <span class="n">mode</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">baseline</span> <span class="o">=</span> <span class="n">baseline</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">restore_best_weights</span> <span class="o">=</span> <span class="n">restore_best_weights</span>
-
-    <span class="bp">self</span><span class="o">.</span><span class="n">wait</span> <span class="o">=</span> <span class="mi">0</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">stopped_epoch</span> <span class="o">=</span> <span class="mi">0</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">state_dict</span> <span class="o">=</span> <span class="kc">None</span>
-
-    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mode</span> <span class="ow">not</span> <span class="ow">in</span> <span class="p">[</span><span class="s2">&quot;auto&quot;</span><span class="p">,</span> <span class="s2">&quot;min&quot;</span><span class="p">,</span> <span class="s2">&quot;max&quot;</span><span class="p">]:</span>
-        <span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span>
-            <span class="s2">&quot;EarlyStopping mode </span><span class="si">%s</span><span class="s2"> is unknown, &quot;</span>
-            <span class="s2">&quot;fallback to auto mode.&quot;</span> <span class="o">%</span> <span class="bp">self</span><span class="o">.</span><span class="n">mode</span><span class="p">,</span>
-            <span class="ne">RuntimeWarning</span><span class="p">,</span>
-        <span class="p">)</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">mode</span> <span class="o">=</span> <span class="s2">&quot;auto&quot;</span>
-
-    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mode</span> <span class="o">==</span> <span class="s2">&quot;min&quot;</span><span class="p">:</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">monitor_op</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">less</span>
-    <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">mode</span> <span class="o">==</span> <span class="s2">&quot;max&quot;</span><span class="p">:</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">monitor_op</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">greater</span>  <span class="c1"># type: ignore[assignment]</span>
-    <span class="k">else</span><span class="p">:</span>
-        <span class="k">if</span> <span class="n">_is_metric</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">monitor</span><span class="p">):</span>
+<span class="normal">683</span>
+<span class="normal">684</span>
+<span class="normal">685</span>
+<span class="normal">686</span>
+<span class="normal">687</span>
+<span class="normal">688</span>
+<span class="normal">689</span>
+<span class="normal">690</span>
+<span class="normal">691</span>
+<span class="normal">692</span>
+<span class="normal">693</span>
+<span class="normal">694</span>
+<span class="normal">695</span>
+<span class="normal">696</span>
+<span class="normal">697</span>
+<span class="normal">698</span>
+<span class="normal">699</span>
+<span class="normal">700</span>
+<span class="normal">701</span>
+<span class="normal">702</span>
+<span class="normal">703</span>
+<span class="normal">704</span>
+<span class="normal">705</span>
+<span class="normal">706</span>
+<span class="normal">707</span>
+<span class="normal">708</span>
+<span class="normal">709</span>
+<span class="normal">710</span>
+<span class="normal">711</span>
+<span class="normal">712</span>
+<span class="normal">713</span>
+<span class="normal">714</span>
+<span class="normal">715</span>
+<span class="normal">716</span>
+<span class="normal">717</span>
+<span class="normal">718</span>
+<span class="normal">719</span>
+<span class="normal">720</span>
+<span class="normal">721</span>
+<span class="normal">722</span>
+<span class="normal">723</span>
+<span class="normal">724</span>
+<span class="normal">725</span>
+<span class="normal">726</span>
+<span class="normal">727</span>
+<span class="normal">728</span>
+<span class="normal">729</span>
+<span class="normal">730</span>
+<span class="normal">731</span>
+<span class="normal">732</span>
+<span class="normal">733</span>
+<span class="normal">734</span>
+<span class="normal">735</span>
+<span class="normal">736</span>
+<span class="normal">737</span>
+<span class="normal">738</span>
+<span class="normal">739</span>
+<span class="normal">740</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">class</span> <span class="nc">EarlyStopping</span><span class="p">(</span><span class="n">Callback</span><span class="p">):</span>
+<span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Stop training when a monitored quantity has stopped improving.</span>
+
+<span class="sd">    This class is almost identical to the corresponding keras class.</span>
+<span class="sd">    Therefore, **credit** to the Keras Team.</span>
+
+<span class="sd">    Callbacks are passed as input parameters to the `Trainer` class. See</span>
+<span class="sd">    `pytorch_widedeep.trainer.Trainer`</span>
+
+<span class="sd">    Parameters</span>
+<span class="sd">    -----------</span>
+<span class="sd">    monitor: str, default=&#39;val_loss&#39;.</span>
+<span class="sd">        Quantity to monitor. Typically _&#39;val_loss&#39;_ or metric name</span>
+<span class="sd">        (e.g. _&#39;val_acc&#39;_)</span>
+<span class="sd">    min_delta: float, default=0.</span>
+<span class="sd">        minimum change in the monitored quantity to qualify as an</span>
+<span class="sd">        improvement, i.e. an absolute change of less than min_delta, will</span>
+<span class="sd">        count as no improvement.</span>
+<span class="sd">    patience: int, default=10.</span>
+<span class="sd">        Number of epochs that produced the monitored quantity with no</span>
+<span class="sd">        improvement after which training will be stopped.</span>
+<span class="sd">    verbose: int.</span>
+<span class="sd">        verbosity mode.</span>
+<span class="sd">    mode: str, default=&#39;auto&#39;</span>
+<span class="sd">        one of _{&#39;auto&#39;, &#39;min&#39;, &#39;max&#39;}_. In _&#39;min&#39;_ mode, training will</span>
+<span class="sd">        stop when the quantity monitored has stopped decreasing; in _&#39;max&#39;_</span>
+<span class="sd">        mode it will stop when the quantity monitored has stopped increasing;</span>
+<span class="sd">        in _&#39;auto&#39;_ mode, the direction is automatically inferred from the</span>
+<span class="sd">        name of the monitored quantity.</span>
+<span class="sd">    baseline: float, Optional. default=None.</span>
+<span class="sd">        Baseline value for the monitored quantity to reach. Training will</span>
+<span class="sd">        stop if the model does not show improvement over the baseline.</span>
+<span class="sd">    restore_best_weights: bool, default=None</span>
+<span class="sd">        Whether to restore model weights from the epoch with the best</span>
+<span class="sd">        value of the monitored quantity. If `False`, the model weights</span>
+<span class="sd">        obtained at the last step of training are used.</span>
+
+<span class="sd">    Attributes</span>
+<span class="sd">    ----------</span>
+<span class="sd">    best: float</span>
+<span class="sd">        best metric</span>
+<span class="sd">    stopped_epoch: int</span>
+<span class="sd">        epoch when the training stopped</span>
+
+<span class="sd">    Examples</span>
+<span class="sd">    --------</span>
+<span class="sd">    &gt;&gt;&gt; from pytorch_widedeep.callbacks import EarlyStopping</span>
+<span class="sd">    &gt;&gt;&gt; from pytorch_widedeep.models import TabMlp, Wide, WideDeep</span>
+<span class="sd">    &gt;&gt;&gt; from pytorch_widedeep.training import Trainer</span>
+<span class="sd">    &gt;&gt;&gt;</span>
+<span class="sd">    &gt;&gt;&gt; embed_input = [(u, i, j) for u, i, j in zip([&quot;a&quot;, &quot;b&quot;, &quot;c&quot;][:4], [4] * 3, [8] * 3)]</span>
+<span class="sd">    &gt;&gt;&gt; column_idx = {k: v for v, k in enumerate([&quot;a&quot;, &quot;b&quot;, &quot;c&quot;])}</span>
+<span class="sd">    &gt;&gt;&gt; wide = Wide(10, 1)</span>
+<span class="sd">    &gt;&gt;&gt; deep = TabMlp(mlp_hidden_dims=[8, 4], column_idx=column_idx, cat_embed_input=embed_input)</span>
+<span class="sd">    &gt;&gt;&gt; model = WideDeep(wide, deep)</span>
+<span class="sd">    &gt;&gt;&gt; trainer = Trainer(model, objective=&quot;regression&quot;, callbacks=[EarlyStopping(patience=10)])</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+
+    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span>
+        <span class="n">monitor</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;val_loss&quot;</span><span class="p">,</span>
+        <span class="n">min_delta</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.0</span><span class="p">,</span>
+        <span class="n">patience</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">10</span><span class="p">,</span>
+        <span class="n">verbose</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">0</span><span class="p">,</span>
+        <span class="n">mode</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;auto&quot;</span><span class="p">,</span>
+        <span class="n">baseline</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">restore_best_weights</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
+    <span class="p">):</span>
+        <span class="nb">super</span><span class="p">(</span><span class="n">EarlyStopping</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">monitor</span> <span class="o">=</span> <span class="n">monitor</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">min_delta</span> <span class="o">=</span> <span class="n">min_delta</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">patience</span> <span class="o">=</span> <span class="n">patience</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">verbose</span> <span class="o">=</span> <span class="n">verbose</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">mode</span> <span class="o">=</span> <span class="n">mode</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">baseline</span> <span class="o">=</span> <span class="n">baseline</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">restore_best_weights</span> <span class="o">=</span> <span class="n">restore_best_weights</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">wait</span> <span class="o">=</span> <span class="mi">0</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">stopped_epoch</span> <span class="o">=</span> <span class="mi">0</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">state_dict</span> <span class="o">=</span> <span class="kc">None</span>
+
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mode</span> <span class="ow">not</span> <span class="ow">in</span> <span class="p">[</span><span class="s2">&quot;auto&quot;</span><span class="p">,</span> <span class="s2">&quot;min&quot;</span><span class="p">,</span> <span class="s2">&quot;max&quot;</span><span class="p">]:</span>
+            <span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span>
+                <span class="s2">&quot;EarlyStopping mode </span><span class="si">%s</span><span class="s2"> is unknown, &quot;</span>
+                <span class="s2">&quot;fallback to auto mode.&quot;</span> <span class="o">%</span> <span class="bp">self</span><span class="o">.</span><span class="n">mode</span><span class="p">,</span>
+                <span class="ne">RuntimeWarning</span><span class="p">,</span>
+            <span class="p">)</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">mode</span> <span class="o">=</span> <span class="s2">&quot;auto&quot;</span>
+
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mode</span> <span class="o">==</span> <span class="s2">&quot;min&quot;</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">monitor_op</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">less</span>
+        <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">mode</span> <span class="o">==</span> <span class="s2">&quot;max&quot;</span><span class="p">:</span>
             <span class="bp">self</span><span class="o">.</span><span class="n">monitor_op</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">greater</span>  <span class="c1"># type: ignore[assignment]</span>
         <span class="k">else</span><span class="p">:</span>
-            <span class="bp">self</span><span class="o">.</span><span class="n">monitor_op</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">less</span>
+            <span class="k">if</span> <span class="n">_is_metric</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">monitor</span><span class="p">):</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">monitor_op</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">greater</span>  <span class="c1"># type: ignore[assignment]</span>
+            <span class="k">else</span><span class="p">:</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">monitor_op</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">less</span>
+
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">monitor_op</span> <span class="o">==</span> <span class="n">np</span><span class="o">.</span><span class="n">greater</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">min_delta</span> <span class="o">*=</span> <span class="mi">1</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">min_delta</span> <span class="o">*=</span> <span class="o">-</span><span class="mi">1</span>
+
+    <span class="k">def</span> <span class="nf">on_train_begin</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">logs</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Dict</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">):</span>
+        <span class="c1"># Allow instances to be re-used</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">wait</span> <span class="o">=</span> <span class="mi">0</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">stopped_epoch</span> <span class="o">=</span> <span class="mi">0</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">baseline</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">best</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">baseline</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">best</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">Inf</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">monitor_op</span> <span class="o">==</span> <span class="n">np</span><span class="o">.</span><span class="n">less</span> <span class="k">else</span> <span class="o">-</span><span class="n">np</span><span class="o">.</span><span class="n">Inf</span>
+
+    <span class="k">def</span> <span class="nf">on_epoch_end</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span> <span class="n">epoch</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span> <span class="n">logs</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Dict</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> <span class="n">metric</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span>
+    <span class="p">):</span>
+        <span class="n">current</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_monitor_value</span><span class="p">(</span><span class="n">logs</span><span class="p">)</span>
+        <span class="k">if</span> <span class="n">current</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="k">return</span>
+
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">monitor_op</span><span class="p">(</span><span class="n">current</span> <span class="o">-</span> <span class="bp">self</span><span class="o">.</span><span class="n">min_delta</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">best</span><span class="p">):</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">best</span> <span class="o">=</span> <span class="n">current</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">wait</span> <span class="o">=</span> <span class="mi">0</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">best_epoch</span> <span class="o">=</span> <span class="n">epoch</span>
+            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">restore_best_weights</span><span class="p">:</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">state_dict</span> <span class="o">=</span> <span class="n">copy</span><span class="o">.</span><span class="n">deepcopy</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">model</span><span class="o">.</span><span class="n">state_dict</span><span class="p">())</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">wait</span> <span class="o">+=</span> <span class="mi">1</span>
+            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">wait</span> <span class="o">&gt;=</span> <span class="bp">self</span><span class="o">.</span><span class="n">patience</span><span class="p">:</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">stopped_epoch</span> <span class="o">=</span> <span class="n">epoch</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">trainer</span><span class="o">.</span><span class="n">early_stop</span> <span class="o">=</span> <span class="kc">True</span>
+
+    <span class="k">def</span> <span class="nf">on_train_end</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">logs</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Dict</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">):</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">stopped_epoch</span> <span class="o">&gt;</span> <span class="mi">0</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">verbose</span> <span class="o">&gt;</span> <span class="mi">0</span><span class="p">:</span>
+            <span class="nb">print</span><span class="p">(</span>
+                <span class="sa">f</span><span class="s2">&quot;Best Epoch: </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">best_epoch</span><span class="w"> </span><span class="o">+</span><span class="w"> </span><span class="mi">1</span><span class="si">}</span><span class="s2">. Best </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">monitor</span><span class="si">}</span><span class="s2">: </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">best</span><span class="si">:</span><span class="s2">.5f</span><span class="si">}</span><span class="s2">&quot;</span>
+            <span class="p">)</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">restore_best_weights</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">state_dict</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">verbose</span> <span class="o">&gt;</span> <span class="mi">0</span><span class="p">:</span>
+                <span class="nb">print</span><span class="p">(</span><span class="s2">&quot;Restoring model weights from the end of the best epoch&quot;</span><span class="p">)</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">model</span><span class="o">.</span><span class="n">load_state_dict</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">state_dict</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">get_monitor_value</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">logs</span><span class="p">):</span>
+        <span class="n">monitor_value</span> <span class="o">=</span> <span class="n">logs</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">monitor</span><span class="p">)</span>
+        <span class="k">if</span> <span class="n">monitor_value</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span>
+                <span class="s2">&quot;Early stopping conditioned on metric `</span><span class="si">%s</span><span class="s2">` &quot;</span>
+                <span class="s2">&quot;which is not available. Available metrics are: </span><span class="si">%s</span><span class="s2">&quot;</span>
+                <span class="o">%</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">monitor</span><span class="p">,</span> <span class="s2">&quot;,&quot;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="nb">list</span><span class="p">(</span><span class="n">logs</span><span class="o">.</span><span class="n">keys</span><span class="p">()))),</span>
+                <span class="ne">RuntimeWarning</span><span class="p">,</span>
+            <span class="p">)</span>
+        <span class="k">return</span> <span class="n">monitor_value</span>
+
+    <span class="k">def</span> <span class="nf">__getstate__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="n">d</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="vm">__dict__</span>
+        <span class="n">self_dict</span> <span class="o">=</span> <span class="p">{</span><span class="n">k</span><span class="p">:</span> <span class="n">d</span><span class="p">[</span><span class="n">k</span><span class="p">]</span> <span class="k">for</span> <span class="n">k</span> <span class="ow">in</span> <span class="n">d</span> <span class="k">if</span> <span class="n">k</span> <span class="ow">not</span> <span class="ow">in</span> <span class="p">[</span><span class="s2">&quot;trainer&quot;</span><span class="p">,</span> <span class="s2">&quot;model&quot;</span><span class="p">]}</span>
+        <span class="k">return</span> <span class="n">self_dict</span>
 
-    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">monitor_op</span> <span class="o">==</span> <span class="n">np</span><span class="o">.</span><span class="n">greater</span><span class="p">:</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">min_delta</span> <span class="o">*=</span> <span class="mi">1</span>
-    <span class="k">else</span><span class="p">:</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">min_delta</span> <span class="o">*=</span> <span class="o">-</span><span class="mi">1</span>
+    <span class="k">def</span> <span class="nf">__setstate__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">state</span><span class="p">):</span>
+        <span class="bp">self</span><span class="o">.</span><span class="vm">__dict__</span> <span class="o">=</span> <span class="n">state</span>
 </code></pre></div></td></tr></table></div>
-                  </details>
+              </details>
 
 
 
diff --git a/mkdocs/site/pytorch-widedeep/dataloaders.html b/mkdocs/site/pytorch-widedeep/dataloaders.html
index 3453782e..357c490d 100644
--- a/mkdocs/site/pytorch-widedeep/dataloaders.html
+++ b/mkdocs/site/pytorch-widedeep/dataloaders.html
@@ -1387,7 +1387,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/16_Self_Supervised_Pretraning_pt1.ipynb" class="md-nav__link">
+      <a href="../examples/15_Self_Supervised_Pretraning_pt1.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1408,7 +1408,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/16_Self_Supervised_Pretraning_pt2.ipynb" class="md-nav__link">
+      <a href="../examples/15_Self_Supervised_Pretraning_pt2.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1429,7 +1429,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/17_Usign_a_custom_hugging_face_model.ipynb" class="md-nav__link">
+      <a href="../examples/16_Usign_a_custom_hugging_face_model.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1450,7 +1450,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/18_feature_importance_via_attention_weights.ipynb" class="md-nav__link">
+      <a href="../examples/17_feature_importance_via_attention_weights.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1471,7 +1471,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/19_wide_and_deep_for_recsys_pt1.ipynb" class="md-nav__link">
+      <a href="../examples/18_wide_and_deep_for_recsys_pt1.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1492,7 +1492,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/19_wide_and_deep_for_recsys_pt2.ipynb" class="md-nav__link">
+      <a href="../examples/18_wide_and_deep_for_recsys_pt2.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1513,7 +1513,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/20_load_from_folder_functionality.ipynb" class="md-nav__link">
+      <a href="../examples/19_load_from_folder_functionality.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1534,7 +1534,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/21_Using_huggingface_within_widedeep.ipynb" class="md-nav__link">
+      <a href="../examples/20_Using_huggingface_within_widedeep.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1610,8 +1610,7 @@ <h2 id="pytorch_widedeep.dataloaders.DataLoaderImbalanced" class="doc doc-headin
 
 
 <a href="#pytorch_widedeep.dataloaders.DataLoaderImbalanced" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="nf">DataLoaderImbalanced</span><span class="p">(</span><span class="n">dataset</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
-</code></pre></div>
+
 
     <div class="doc doc-contents first">
             <p class="doc doc-class-bases">
@@ -1624,34 +1623,61 @@ <h2 id="pytorch_widedeep.dataloaders.DataLoaderImbalanced" class="doc doc-headin
 
 
 <p><span class="doc-section-title">Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>dataset</code></b>
-              (<code><span title="typing.Optional">Optional</span>[<span title="pytorch_widedeep.training._wd_dataset.WideDeepDataset">WideDeepDataset</span>]</code>, default:
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+          <th>Default</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>dataset</code></td>
+            <td>
+                  <code><span title="typing.Optional">Optional</span>[<span title="pytorch_widedeep.training._wd_dataset.WideDeepDataset">WideDeepDataset</span>]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>see <code>pytorch_widedeep.training._wd_dataset</code></p>
+              </div>
+            </td>
+            <td>
                   <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>see <code>pytorch_widedeep.training._wd_dataset</code></p>
-          </div>
-        </li>
-    </ul>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
 
 <p><span class="doc-section-title">Other Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>*args</code></b>
-          –
-          <div class="doc-md-description">
-            <p>Positional arguments to be passed to the parent CustomDataLoader.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>**kwargs</code></b>
-          –
-          <div class="doc-md-description">
-            <p>This can include any parameter that can be passed to the <em>'standard'</em>
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>*args</code></td>
+            <td>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Positional arguments to be passed to the parent CustomDataLoader.</p>
+              </div>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>**kwargs</code></td>
+            <td>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>This can include any parameter that can be passed to the <em>'standard'</em>
 pytorch
 <a href="https://pytorch.org/docs/stable/data.html#torch.utils.data.DataLoader">DataLoader</a>
 and that is not already explicitely passed to the class. In addition,
@@ -1662,13 +1688,45 @@ <h2 id="pytorch_widedeep.dataloaders.DataLoaderImbalanced" class="doc doc-headin
 <div class="arithmatex">\[
 minority \space class \space count \times number \space of \space classes \times oversample\_mul
 \]</div>
-          </div>
-        </li>
-    </ul>
-
-                  <details class="quote">
-                    <summary>Source code in <code>pytorch_widedeep/dataloaders.py</code></summary>
-                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">109</span>
+              </div>
+            </td>
+          </tr>
+      </tbody>
+    </table>
+
+              <details class="quote">
+                <summary>Source code in <code>pytorch_widedeep/dataloaders.py</code></summary>
+                <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 79</span>
+<span class="normal"> 80</span>
+<span class="normal"> 81</span>
+<span class="normal"> 82</span>
+<span class="normal"> 83</span>
+<span class="normal"> 84</span>
+<span class="normal"> 85</span>
+<span class="normal"> 86</span>
+<span class="normal"> 87</span>
+<span class="normal"> 88</span>
+<span class="normal"> 89</span>
+<span class="normal"> 90</span>
+<span class="normal"> 91</span>
+<span class="normal"> 92</span>
+<span class="normal"> 93</span>
+<span class="normal"> 94</span>
+<span class="normal"> 95</span>
+<span class="normal"> 96</span>
+<span class="normal"> 97</span>
+<span class="normal"> 98</span>
+<span class="normal"> 99</span>
+<span class="normal">100</span>
+<span class="normal">101</span>
+<span class="normal">102</span>
+<span class="normal">103</span>
+<span class="normal">104</span>
+<span class="normal">105</span>
+<span class="normal">106</span>
+<span class="normal">107</span>
+<span class="normal">108</span>
+<span class="normal">109</span>
 <span class="normal">110</span>
 <span class="normal">111</span>
 <span class="normal">112</span>
@@ -1676,17 +1734,85 @@ <h2 id="pytorch_widedeep.dataloaders.DataLoaderImbalanced" class="doc doc-headin
 <span class="normal">114</span>
 <span class="normal">115</span>
 <span class="normal">116</span>
-<span class="normal">117</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">dataset</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">WideDeepDataset</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">args</span> <span class="o">=</span> <span class="n">args</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">kwargs</span> <span class="o">=</span> <span class="n">kwargs</span>
-
-    <span class="k">if</span> <span class="n">dataset</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">_setup_sampler</span><span class="p">(</span><span class="n">dataset</span><span class="p">)</span>
-        <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">dataset</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="n">sampler</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">sampler</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
-    <span class="k">else</span><span class="p">:</span>
-        <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
+<span class="normal">117</span>
+<span class="normal">118</span>
+<span class="normal">119</span>
+<span class="normal">120</span>
+<span class="normal">121</span>
+<span class="normal">122</span>
+<span class="normal">123</span>
+<span class="normal">124</span>
+<span class="normal">125</span>
+<span class="normal">126</span>
+<span class="normal">127</span>
+<span class="normal">128</span>
+<span class="normal">129</span>
+<span class="normal">130</span>
+<span class="normal">131</span>
+<span class="normal">132</span>
+<span class="normal">133</span>
+<span class="normal">134</span>
+<span class="normal">135</span>
+<span class="normal">136</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">class</span> <span class="nc">DataLoaderImbalanced</span><span class="p">(</span><span class="n">CustomDataLoader</span><span class="p">):</span>
+<span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Class to load and shuffle batches with adjusted weights for imbalanced</span>
+<span class="sd">    datasets. If the classes do not begin from 0 remapping is necessary. See</span>
+<span class="sd">    [here](https://towardsdatascience.com/pytorch-tabular-multiclass-classification-9f8211a123ab).</span>
+
+<span class="sd">    Parameters</span>
+<span class="sd">    ----------</span>
+<span class="sd">    dataset: `WideDeepDataset`</span>
+<span class="sd">        see `pytorch_widedeep.training._wd_dataset`</span>
+
+<span class="sd">    Other Parameters</span>
+<span class="sd">    ----------------</span>
+<span class="sd">    *args: Any</span>
+<span class="sd">        Positional arguments to be passed to the parent CustomDataLoader.</span>
+<span class="sd">    **kwargs: Dict</span>
+<span class="sd">        This can include any parameter that can be passed to the _&#39;standard&#39;_</span>
+<span class="sd">        pytorch</span>
+<span class="sd">        [DataLoader](https://pytorch.org/docs/stable/data.html#torch.utils.data.DataLoader)</span>
+<span class="sd">        and that is not already explicitely passed to the class. In addition,</span>
+<span class="sd">        the dictionary can also include the extra parameter `oversample_mul` which</span>
+<span class="sd">        will multiply the number of samples of the minority class to be sampled by</span>
+<span class="sd">        the [`WeightedRandomSampler`](https://pytorch.org/docs/stable/data.html#torch.utils.data.WeightedRandomSampler).</span>
+
+<span class="sd">        In other words, the `num_samples` param in `WeightedRandomSampler` will be defined as:</span>
+
+<span class="sd">        $$</span>
+<span class="sd">        minority \space class \space count \times number \space of \space classes \times oversample\_mul</span>
+<span class="sd">        $$</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+
+    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">dataset</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">WideDeepDataset</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">args</span> <span class="o">=</span> <span class="n">args</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">kwargs</span> <span class="o">=</span> <span class="n">kwargs</span>
+
+        <span class="k">if</span> <span class="n">dataset</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">_setup_sampler</span><span class="p">(</span><span class="n">dataset</span><span class="p">)</span>
+            <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">dataset</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="n">sampler</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">sampler</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
+
+    <span class="k">def</span> <span class="nf">set_dataset</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">dataset</span><span class="p">:</span> <span class="n">WideDeepDataset</span><span class="p">):</span>
+        <span class="n">sampler</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_setup_sampler</span><span class="p">(</span><span class="n">dataset</span><span class="p">)</span>
+        <span class="c1"># update the kwargs with the new sampler</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">kwargs</span><span class="p">[</span><span class="s2">&quot;sampler&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">sampler</span>
+        <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="n">set_dataset</span><span class="p">(</span><span class="n">dataset</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">_setup_sampler</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">dataset</span><span class="p">:</span> <span class="n">WideDeepDataset</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">WeightedRandomSampler</span><span class="p">:</span>
+        <span class="k">assert</span> <span class="n">dataset</span><span class="o">.</span><span class="n">Y</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">,</span> <span class="p">(</span>
+            <span class="s2">&quot;The &#39;dataset&#39; instance of WideDeepDataset must contain a &quot;</span>
+            <span class="s2">&quot;target array &#39;Y&#39;&quot;</span>
+        <span class="p">)</span>
+
+        <span class="n">oversample_mul</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">kwargs</span><span class="o">.</span><span class="n">pop</span><span class="p">(</span><span class="s2">&quot;oversample_mul&quot;</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span>
+        <span class="n">weights</span><span class="p">,</span> <span class="n">minor_cls_cnt</span><span class="p">,</span> <span class="n">num_clss</span> <span class="o">=</span> <span class="n">get_class_weights</span><span class="p">(</span><span class="n">dataset</span><span class="p">)</span>
+        <span class="n">num_samples</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">minor_cls_cnt</span> <span class="o">*</span> <span class="n">num_clss</span> <span class="o">*</span> <span class="n">oversample_mul</span><span class="p">)</span>
+        <span class="n">samples_weight</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">([</span><span class="n">weights</span><span class="p">[</span><span class="n">i</span><span class="p">]</span> <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="n">dataset</span><span class="o">.</span><span class="n">Y</span><span class="p">]))</span>
+        <span class="n">sampler</span> <span class="o">=</span> <span class="n">WeightedRandomSampler</span><span class="p">(</span><span class="n">samples_weight</span><span class="p">,</span> <span class="n">num_samples</span><span class="p">,</span> <span class="n">replacement</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
+        <span class="k">return</span> <span class="n">sampler</span>
 </code></pre></div></td></tr></table></div>
-                  </details>
+              </details>
 
 
 
diff --git a/mkdocs/site/pytorch-widedeep/losses.html b/mkdocs/site/pytorch-widedeep/losses.html
index 91b430e8..6147a17f 100644
--- a/mkdocs/site/pytorch-widedeep/losses.html
+++ b/mkdocs/site/pytorch-widedeep/losses.html
@@ -1558,7 +1558,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/16_Self_Supervised_Pretraning_pt1.ipynb" class="md-nav__link">
+      <a href="../examples/15_Self_Supervised_Pretraning_pt1.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1579,7 +1579,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/16_Self_Supervised_Pretraning_pt2.ipynb" class="md-nav__link">
+      <a href="../examples/15_Self_Supervised_Pretraning_pt2.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1600,7 +1600,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/17_Usign_a_custom_hugging_face_model.ipynb" class="md-nav__link">
+      <a href="../examples/16_Usign_a_custom_hugging_face_model.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1621,7 +1621,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/18_feature_importance_via_attention_weights.ipynb" class="md-nav__link">
+      <a href="../examples/17_feature_importance_via_attention_weights.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1642,7 +1642,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/19_wide_and_deep_for_recsys_pt1.ipynb" class="md-nav__link">
+      <a href="../examples/18_wide_and_deep_for_recsys_pt1.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1663,7 +1663,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/19_wide_and_deep_for_recsys_pt2.ipynb" class="md-nav__link">
+      <a href="../examples/18_wide_and_deep_for_recsys_pt2.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1684,7 +1684,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/20_load_from_folder_functionality.ipynb" class="md-nav__link">
+      <a href="../examples/19_load_from_folder_functionality.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1705,7 +1705,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/21_Using_huggingface_within_widedeep.ipynb" class="md-nav__link">
+      <a href="../examples/20_Using_huggingface_within_widedeep.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1793,8 +1793,7 @@ <h2 id="pytorch_widedeep.losses.MSELoss" class="doc doc-heading">
 
 
 <a href="#pytorch_widedeep.losses.MSELoss" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="nf">MSELoss</span><span class="p">()</span>
-</code></pre></div>
+
 
     <div class="doc doc-contents first">
             <p class="doc doc-class-bases">
@@ -1803,13 +1802,75 @@ <h2 id="pytorch_widedeep.losses.MSELoss" class="doc doc-heading">
 
         <p>Mean square error loss</p>
 
-                  <details class="quote">
-                    <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
-                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">16</span>
-<span class="normal">17</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
-    <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
+              <details class="quote">
+                <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
+                <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">10</span>
+<span class="normal">11</span>
+<span class="normal">12</span>
+<span class="normal">13</span>
+<span class="normal">14</span>
+<span class="normal">15</span>
+<span class="normal">16</span>
+<span class="normal">17</span>
+<span class="normal">18</span>
+<span class="normal">19</span>
+<span class="normal">20</span>
+<span class="normal">21</span>
+<span class="normal">22</span>
+<span class="normal">23</span>
+<span class="normal">24</span>
+<span class="normal">25</span>
+<span class="normal">26</span>
+<span class="normal">27</span>
+<span class="normal">28</span>
+<span class="normal">29</span>
+<span class="normal">30</span>
+<span class="normal">31</span>
+<span class="normal">32</span>
+<span class="normal">33</span>
+<span class="normal">34</span>
+<span class="normal">35</span>
+<span class="normal">36</span>
+<span class="normal">37</span>
+<span class="normal">38</span>
+<span class="normal">39</span>
+<span class="normal">40</span>
+<span class="normal">41</span>
+<span class="normal">42</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">class</span> <span class="nc">MSELoss</span><span class="p">(</span><span class="n">nn</span><span class="o">.</span><span class="n">Module</span><span class="p">):</span>
+<span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Mean square error loss&quot;&quot;&quot;</span>
+
+    <span class="c1"># legacy code from when we used to support FDS-LDS and this class could</span>
+    <span class="c1"># taked the corresponding params. At this stage probably you want to use</span>
+    <span class="c1"># torch.nn.MSELoss</span>
+    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
+
+    <span class="k">def</span> <span class="nf">forward</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span>
+        <span class="nb">input</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">,</span>
+        <span class="n">target</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">,</span>
+    <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tensor</span><span class="p">:</span>
+<span class="w">        </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Parameters</span>
+<span class="sd">        ----------</span>
+<span class="sd">        input: Tensor</span>
+<span class="sd">            Input tensor with predictions</span>
+<span class="sd">        target: Tensor</span>
+<span class="sd">            Target tensor with the actual values</span>
+
+<span class="sd">        Examples</span>
+<span class="sd">        --------</span>
+<span class="sd">        &gt;&gt;&gt; import torch</span>
+<span class="sd">        &gt;&gt;&gt; from pytorch_widedeep.losses import MSELoss</span>
+<span class="sd">        &gt;&gt;&gt;</span>
+<span class="sd">        &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)</span>
+<span class="sd">        &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)</span>
+<span class="sd">        &gt;&gt;&gt; loss = MSELoss()(input, target)</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">loss</span> <span class="o">=</span> <span class="p">(</span><span class="nb">input</span> <span class="o">-</span> <span class="n">target</span><span class="p">)</span> <span class="o">**</span> <span class="mi">2</span>
+        <span class="k">return</span> <span class="n">torch</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="n">loss</span><span class="p">)</span>
 </code></pre></div></td></tr></table></div>
-                  </details>
+              </details>
 
 
 
@@ -1839,24 +1900,46 @@ <h3 id="pytorch_widedeep.losses.MSELoss.forward" class="doc doc-heading">
 
 
 <p><span class="doc-section-title">Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>input</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>)
-          –
-          <div class="doc-md-description">
-            <p>Input tensor with predictions</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>target</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>)
-          –
-          <div class="doc-md-description">
-            <p>Target tensor with the actual values</p>
-          </div>
-        </li>
-    </ul>
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+          <th>Default</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>input</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Input tensor with predictions</p>
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>target</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Target tensor with the actual values</p>
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
 
 <p><span class="doc-section-title">Examples:</span></p>
@@ -1940,8 +2023,7 @@ <h2 id="pytorch_widedeep.losses.MSLELoss" class="doc doc-heading">
 
 
 <a href="#pytorch_widedeep.losses.MSLELoss" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="nf">MSLELoss</span><span class="p">()</span>
-</code></pre></div>
+
 
     <div class="doc doc-contents first">
             <p class="doc doc-class-bases">
@@ -1950,13 +2032,83 @@ <h2 id="pytorch_widedeep.losses.MSLELoss" class="doc doc-heading">
 
         <p>Mean square log error loss</p>
 
-                  <details class="quote">
-                    <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
-                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">48</span>
-<span class="normal">49</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
-    <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
+              <details class="quote">
+                <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
+                <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">45</span>
+<span class="normal">46</span>
+<span class="normal">47</span>
+<span class="normal">48</span>
+<span class="normal">49</span>
+<span class="normal">50</span>
+<span class="normal">51</span>
+<span class="normal">52</span>
+<span class="normal">53</span>
+<span class="normal">54</span>
+<span class="normal">55</span>
+<span class="normal">56</span>
+<span class="normal">57</span>
+<span class="normal">58</span>
+<span class="normal">59</span>
+<span class="normal">60</span>
+<span class="normal">61</span>
+<span class="normal">62</span>
+<span class="normal">63</span>
+<span class="normal">64</span>
+<span class="normal">65</span>
+<span class="normal">66</span>
+<span class="normal">67</span>
+<span class="normal">68</span>
+<span class="normal">69</span>
+<span class="normal">70</span>
+<span class="normal">71</span>
+<span class="normal">72</span>
+<span class="normal">73</span>
+<span class="normal">74</span>
+<span class="normal">75</span>
+<span class="normal">76</span>
+<span class="normal">77</span>
+<span class="normal">78</span>
+<span class="normal">79</span>
+<span class="normal">80</span>
+<span class="normal">81</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">class</span> <span class="nc">MSLELoss</span><span class="p">(</span><span class="n">nn</span><span class="o">.</span><span class="n">Module</span><span class="p">):</span>
+<span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Mean square log error loss&quot;&quot;&quot;</span>
+
+    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
+
+    <span class="k">def</span> <span class="nf">forward</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span>
+        <span class="nb">input</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">,</span>
+        <span class="n">target</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">,</span>
+    <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tensor</span><span class="p">:</span>
+<span class="w">        </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Parameters</span>
+<span class="sd">        ----------</span>
+<span class="sd">        input: Tensor</span>
+<span class="sd">            Input tensor with predictions (not probabilities)</span>
+<span class="sd">        target: Tensor</span>
+<span class="sd">            Target tensor with the actual classes</span>
+
+<span class="sd">        Examples</span>
+<span class="sd">        --------</span>
+<span class="sd">        &gt;&gt;&gt; import torch</span>
+<span class="sd">        &gt;&gt;&gt; from pytorch_widedeep.losses import MSLELoss</span>
+<span class="sd">        &gt;&gt;&gt;</span>
+<span class="sd">        &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)</span>
+<span class="sd">        &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)</span>
+<span class="sd">        &gt;&gt;&gt; loss = MSLELoss()(input, target)</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="k">assert</span> <span class="p">(</span>
+            <span class="nb">input</span><span class="o">.</span><span class="n">min</span><span class="p">()</span> <span class="o">&gt;=</span> <span class="mi">0</span>
+        <span class="p">),</span> <span class="s2">&quot;&quot;&quot;All input values must be &gt;=0, if your model is predicting</span>
+<span class="s2">            values &lt;0 try to enforce positive values by activation function</span>
+<span class="s2">            on last layer with `trainer.enforce_positive_output=True`&quot;&quot;&quot;</span>
+        <span class="k">assert</span> <span class="n">target</span><span class="o">.</span><span class="n">min</span><span class="p">()</span> <span class="o">&gt;=</span> <span class="mi">0</span><span class="p">,</span> <span class="s2">&quot;All target values must be &gt;=0&quot;</span>
+
+        <span class="n">loss</span> <span class="o">=</span> <span class="p">(</span><span class="n">torch</span><span class="o">.</span><span class="n">log</span><span class="p">(</span><span class="nb">input</span> <span class="o">+</span> <span class="mi">1</span><span class="p">)</span> <span class="o">-</span> <span class="n">torch</span><span class="o">.</span><span class="n">log</span><span class="p">(</span><span class="n">target</span> <span class="o">+</span> <span class="mi">1</span><span class="p">))</span> <span class="o">**</span> <span class="mi">2</span>
+        <span class="k">return</span> <span class="n">torch</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="n">loss</span><span class="p">)</span>
 </code></pre></div></td></tr></table></div>
-                  </details>
+              </details>
 
 
 
@@ -1986,24 +2138,46 @@ <h3 id="pytorch_widedeep.losses.MSLELoss.forward" class="doc doc-heading">
 
 
 <p><span class="doc-section-title">Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>input</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>)
-          –
-          <div class="doc-md-description">
-            <p>Input tensor with predictions (not probabilities)</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>target</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>)
-          –
-          <div class="doc-md-description">
-            <p>Target tensor with the actual classes</p>
-          </div>
-        </li>
-    </ul>
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+          <th>Default</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>input</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Input tensor with predictions (not probabilities)</p>
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>target</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Target tensor with the actual classes</p>
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
 
 <p><span class="doc-section-title">Examples:</span></p>
@@ -2101,8 +2275,7 @@ <h2 id="pytorch_widedeep.losses.RMSELoss" class="doc doc-heading">
 
 
 <a href="#pytorch_widedeep.losses.RMSELoss" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="nf">RMSELoss</span><span class="p">()</span>
-</code></pre></div>
+
 
     <div class="doc doc-contents first">
             <p class="doc doc-class-bases">
@@ -2111,13 +2284,67 @@ <h2 id="pytorch_widedeep.losses.RMSELoss" class="doc doc-heading">
 
         <p>Root mean square error loss</p>
 
-                  <details class="quote">
-                    <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
-                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">90</span>
-<span class="normal">91</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
-    <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
+              <details class="quote">
+                <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
+                <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 84</span>
+<span class="normal"> 85</span>
+<span class="normal"> 86</span>
+<span class="normal"> 87</span>
+<span class="normal"> 88</span>
+<span class="normal"> 89</span>
+<span class="normal"> 90</span>
+<span class="normal"> 91</span>
+<span class="normal"> 92</span>
+<span class="normal"> 93</span>
+<span class="normal"> 94</span>
+<span class="normal"> 95</span>
+<span class="normal"> 96</span>
+<span class="normal"> 97</span>
+<span class="normal"> 98</span>
+<span class="normal"> 99</span>
+<span class="normal">100</span>
+<span class="normal">101</span>
+<span class="normal">102</span>
+<span class="normal">103</span>
+<span class="normal">104</span>
+<span class="normal">105</span>
+<span class="normal">106</span>
+<span class="normal">107</span>
+<span class="normal">108</span>
+<span class="normal">109</span>
+<span class="normal">110</span>
+<span class="normal">111</span>
+<span class="normal">112</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">class</span> <span class="nc">RMSELoss</span><span class="p">(</span><span class="n">nn</span><span class="o">.</span><span class="n">Module</span><span class="p">):</span>
+<span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Root mean square error loss&quot;&quot;&quot;</span>
+
+    <span class="c1"># legacy code from when we used to support FDS-LDS and this class could</span>
+    <span class="c1"># taked the corresponding params. At this stage probably you want to use</span>
+    <span class="c1"># torch.sqrt(nn.MSELoss)</span>
+    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
+
+    <span class="k">def</span> <span class="nf">forward</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="nb">input</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">,</span> <span class="n">target</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tensor</span><span class="p">:</span>
+<span class="w">        </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Parameters</span>
+<span class="sd">        ----------</span>
+<span class="sd">        input: Tensor</span>
+<span class="sd">            Input tensor with predictions (not probabilities)</span>
+<span class="sd">        target: Tensor</span>
+<span class="sd">            Target tensor with the actual classes</span>
+
+<span class="sd">        Examples</span>
+<span class="sd">        --------</span>
+<span class="sd">        &gt;&gt;&gt; import torch</span>
+<span class="sd">        &gt;&gt;&gt; from pytorch_widedeep.losses import RMSELoss</span>
+<span class="sd">        &gt;&gt;&gt;</span>
+<span class="sd">        &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)</span>
+<span class="sd">        &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)</span>
+<span class="sd">        &gt;&gt;&gt; loss = RMSELoss()(input, target)</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">loss</span> <span class="o">=</span> <span class="p">(</span><span class="nb">input</span> <span class="o">-</span> <span class="n">target</span><span class="p">)</span> <span class="o">**</span> <span class="mi">2</span>
+        <span class="k">return</span> <span class="n">torch</span><span class="o">.</span><span class="n">sqrt</span><span class="p">(</span><span class="n">torch</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="n">loss</span><span class="p">))</span>
 </code></pre></div></td></tr></table></div>
-                  </details>
+              </details>
 
 
 
@@ -2147,24 +2374,46 @@ <h3 id="pytorch_widedeep.losses.RMSELoss.forward" class="doc doc-heading">
 
 
 <p><span class="doc-section-title">Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>input</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>)
-          –
-          <div class="doc-md-description">
-            <p>Input tensor with predictions (not probabilities)</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>target</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>)
-          –
-          <div class="doc-md-description">
-            <p>Target tensor with the actual classes</p>
-          </div>
-        </li>
-    </ul>
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+          <th>Default</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>input</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Input tensor with predictions (not probabilities)</p>
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>target</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Target tensor with the actual classes</p>
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
 
 <p><span class="doc-section-title">Examples:</span></p>
@@ -2240,8 +2489,7 @@ <h2 id="pytorch_widedeep.losses.RMSLELoss" class="doc doc-heading">
 
 
 <a href="#pytorch_widedeep.losses.RMSLELoss" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="nf">RMSLELoss</span><span class="p">()</span>
-</code></pre></div>
+
 
     <div class="doc doc-contents first">
             <p class="doc doc-class-bases">
@@ -2250,13 +2498,75 @@ <h2 id="pytorch_widedeep.losses.RMSLELoss" class="doc doc-heading">
 
         <p>Root mean square log error loss</p>
 
-                  <details class="quote">
-                    <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
-                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">118</span>
-<span class="normal">119</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
-    <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
+              <details class="quote">
+                <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
+                <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">115</span>
+<span class="normal">116</span>
+<span class="normal">117</span>
+<span class="normal">118</span>
+<span class="normal">119</span>
+<span class="normal">120</span>
+<span class="normal">121</span>
+<span class="normal">122</span>
+<span class="normal">123</span>
+<span class="normal">124</span>
+<span class="normal">125</span>
+<span class="normal">126</span>
+<span class="normal">127</span>
+<span class="normal">128</span>
+<span class="normal">129</span>
+<span class="normal">130</span>
+<span class="normal">131</span>
+<span class="normal">132</span>
+<span class="normal">133</span>
+<span class="normal">134</span>
+<span class="normal">135</span>
+<span class="normal">136</span>
+<span class="normal">137</span>
+<span class="normal">138</span>
+<span class="normal">139</span>
+<span class="normal">140</span>
+<span class="normal">141</span>
+<span class="normal">142</span>
+<span class="normal">143</span>
+<span class="normal">144</span>
+<span class="normal">145</span>
+<span class="normal">146</span>
+<span class="normal">147</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">class</span> <span class="nc">RMSLELoss</span><span class="p">(</span><span class="n">nn</span><span class="o">.</span><span class="n">Module</span><span class="p">):</span>
+<span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Root mean square log error loss&quot;&quot;&quot;</span>
+
+    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
+
+    <span class="k">def</span> <span class="nf">forward</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="nb">input</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">,</span> <span class="n">target</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tensor</span><span class="p">:</span>
+<span class="w">        </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Parameters</span>
+<span class="sd">        ----------</span>
+<span class="sd">        input: Tensor</span>
+<span class="sd">            Input tensor with predictions (not probabilities)</span>
+<span class="sd">        target: Tensor</span>
+<span class="sd">            Target tensor with the actual classes</span>
+
+<span class="sd">        Examples</span>
+<span class="sd">        --------</span>
+<span class="sd">        &gt;&gt;&gt; import torch</span>
+<span class="sd">        &gt;&gt;&gt; from pytorch_widedeep.losses import RMSLELoss</span>
+<span class="sd">        &gt;&gt;&gt;</span>
+<span class="sd">        &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)</span>
+<span class="sd">        &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)</span>
+<span class="sd">        &gt;&gt;&gt; loss = RMSLELoss()(input, target)</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="k">assert</span> <span class="p">(</span>
+            <span class="nb">input</span><span class="o">.</span><span class="n">min</span><span class="p">()</span> <span class="o">&gt;=</span> <span class="mi">0</span>
+        <span class="p">),</span> <span class="s2">&quot;&quot;&quot;All input values must be &gt;=0, if your model is predicting</span>
+<span class="s2">            values &lt;0 try to enforce positive values by activation function</span>
+<span class="s2">            on last layer with `trainer.enforce_positive_output=True`&quot;&quot;&quot;</span>
+        <span class="k">assert</span> <span class="n">target</span><span class="o">.</span><span class="n">min</span><span class="p">()</span> <span class="o">&gt;=</span> <span class="mi">0</span><span class="p">,</span> <span class="s2">&quot;All target values must be &gt;=0&quot;</span>
+
+        <span class="n">loss</span> <span class="o">=</span> <span class="p">(</span><span class="n">torch</span><span class="o">.</span><span class="n">log</span><span class="p">(</span><span class="nb">input</span> <span class="o">+</span> <span class="mi">1</span><span class="p">)</span> <span class="o">-</span> <span class="n">torch</span><span class="o">.</span><span class="n">log</span><span class="p">(</span><span class="n">target</span> <span class="o">+</span> <span class="mi">1</span><span class="p">))</span> <span class="o">**</span> <span class="mi">2</span>
+        <span class="k">return</span> <span class="n">torch</span><span class="o">.</span><span class="n">sqrt</span><span class="p">(</span><span class="n">torch</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="n">loss</span><span class="p">))</span>
 </code></pre></div></td></tr></table></div>
-                  </details>
+              </details>
 
 
 
@@ -2286,24 +2596,46 @@ <h3 id="pytorch_widedeep.losses.RMSLELoss.forward" class="doc doc-heading">
 
 
 <p><span class="doc-section-title">Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>input</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>)
-          –
-          <div class="doc-md-description">
-            <p>Input tensor with predictions (not probabilities)</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>target</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>)
-          –
-          <div class="doc-md-description">
-            <p>Target tensor with the actual classes</p>
-          </div>
-        </li>
-    </ul>
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+          <th>Default</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>input</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Input tensor with predictions (not probabilities)</p>
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>target</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Target tensor with the actual classes</p>
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
 
 <p><span class="doc-section-title">Examples:</span></p>
@@ -2393,10 +2725,7 @@ <h2 id="pytorch_widedeep.losses.QuantileLoss" class="doc doc-heading">
 
 
 <a href="#pytorch_widedeep.losses.QuantileLoss" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="nf">QuantileLoss</span><span class="p">(</span>
-    <span class="n">quantiles</span><span class="o">=</span><span class="p">[</span><span class="mf">0.02</span><span class="p">,</span> <span class="mf">0.1</span><span class="p">,</span> <span class="mf">0.25</span><span class="p">,</span> <span class="mf">0.5</span><span class="p">,</span> <span class="mf">0.75</span><span class="p">,</span> <span class="mf">0.9</span><span class="p">,</span> <span class="mf">0.98</span><span class="p">]</span>
-<span class="p">)</span>
-</code></pre></div>
+
 
     <div class="doc doc-contents first">
             <p class="doc doc-class-bases">
@@ -2412,34 +2741,154 @@ <h2 id="pytorch_widedeep.losses.QuantileLoss" class="doc doc-heading">
 
 
 <p><span class="doc-section-title">Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>quantiles</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.List">List</span>[float]</code>, default:
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+          <th>Default</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>quantiles</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.List">List</span>[float]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>List of quantiles</p>
+              </div>
+            </td>
+            <td>
                   <code>[0.02, 0.1, 0.25, 0.5, 0.75, 0.9, 0.98]</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>List of quantiles</p>
-          </div>
-        </li>
-    </ul>
-
-                  <details class="quote">
-                    <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
-                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">166</span>
+            </td>
+          </tr>
+      </tbody>
+    </table>
+
+              <details class="quote">
+                <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
+                <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">150</span>
+<span class="normal">151</span>
+<span class="normal">152</span>
+<span class="normal">153</span>
+<span class="normal">154</span>
+<span class="normal">155</span>
+<span class="normal">156</span>
+<span class="normal">157</span>
+<span class="normal">158</span>
+<span class="normal">159</span>
+<span class="normal">160</span>
+<span class="normal">161</span>
+<span class="normal">162</span>
+<span class="normal">163</span>
+<span class="normal">164</span>
+<span class="normal">165</span>
+<span class="normal">166</span>
 <span class="normal">167</span>
 <span class="normal">168</span>
 <span class="normal">169</span>
 <span class="normal">170</span>
-<span class="normal">171</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
-    <span class="bp">self</span><span class="p">,</span>
-    <span class="n">quantiles</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="p">[</span><span class="mf">0.02</span><span class="p">,</span> <span class="mf">0.1</span><span class="p">,</span> <span class="mf">0.25</span><span class="p">,</span> <span class="mf">0.5</span><span class="p">,</span> <span class="mf">0.75</span><span class="p">,</span> <span class="mf">0.9</span><span class="p">,</span> <span class="mf">0.98</span><span class="p">],</span>
-<span class="p">):</span>
-    <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">quantiles</span> <span class="o">=</span> <span class="n">quantiles</span>
+<span class="normal">171</span>
+<span class="normal">172</span>
+<span class="normal">173</span>
+<span class="normal">174</span>
+<span class="normal">175</span>
+<span class="normal">176</span>
+<span class="normal">177</span>
+<span class="normal">178</span>
+<span class="normal">179</span>
+<span class="normal">180</span>
+<span class="normal">181</span>
+<span class="normal">182</span>
+<span class="normal">183</span>
+<span class="normal">184</span>
+<span class="normal">185</span>
+<span class="normal">186</span>
+<span class="normal">187</span>
+<span class="normal">188</span>
+<span class="normal">189</span>
+<span class="normal">190</span>
+<span class="normal">191</span>
+<span class="normal">192</span>
+<span class="normal">193</span>
+<span class="normal">194</span>
+<span class="normal">195</span>
+<span class="normal">196</span>
+<span class="normal">197</span>
+<span class="normal">198</span>
+<span class="normal">199</span>
+<span class="normal">200</span>
+<span class="normal">201</span>
+<span class="normal">202</span>
+<span class="normal">203</span>
+<span class="normal">204</span>
+<span class="normal">205</span>
+<span class="normal">206</span>
+<span class="normal">207</span>
+<span class="normal">208</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">class</span> <span class="nc">QuantileLoss</span><span class="p">(</span><span class="n">nn</span><span class="o">.</span><span class="n">Module</span><span class="p">):</span>
+<span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Quantile loss defined as:</span>
+
+<span class="sd">    $$</span>
+<span class="sd">    Loss = max(q \times (y-y_{pred}), (1-q) \times (y_{pred}-y))</span>
+<span class="sd">    $$</span>
+
+<span class="sd">    All credits go to the implementation at</span>
+<span class="sd">    [pytorch-forecasting](https://pytorch-forecasting.readthedocs.io/en/latest/_modules/pytorch_forecasting/metrics.html#QuantileLoss).</span>
+
+<span class="sd">    Parameters</span>
+<span class="sd">    ----------</span>
+<span class="sd">    quantiles: List, default = [0.02, 0.1, 0.25, 0.5, 0.75, 0.9, 0.98]</span>
+<span class="sd">        List of quantiles</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+
+    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span>
+        <span class="n">quantiles</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="p">[</span><span class="mf">0.02</span><span class="p">,</span> <span class="mf">0.1</span><span class="p">,</span> <span class="mf">0.25</span><span class="p">,</span> <span class="mf">0.5</span><span class="p">,</span> <span class="mf">0.75</span><span class="p">,</span> <span class="mf">0.9</span><span class="p">,</span> <span class="mf">0.98</span><span class="p">],</span>
+    <span class="p">):</span>
+        <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">quantiles</span> <span class="o">=</span> <span class="n">quantiles</span>
+
+    <span class="k">def</span> <span class="nf">forward</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="nb">input</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">,</span> <span class="n">target</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tensor</span><span class="p">:</span>
+<span class="w">        </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Parameters</span>
+<span class="sd">        ----------</span>
+<span class="sd">        input: Tensor</span>
+<span class="sd">            Input tensor with predictions</span>
+<span class="sd">        target: Tensor</span>
+<span class="sd">            Target tensor with the actual values</span>
+
+<span class="sd">        Examples</span>
+<span class="sd">        --------</span>
+<span class="sd">        &gt;&gt;&gt; import torch</span>
+<span class="sd">        &gt;&gt;&gt;</span>
+<span class="sd">        &gt;&gt;&gt; from pytorch_widedeep.losses import QuantileLoss</span>
+<span class="sd">        &gt;&gt;&gt;</span>
+<span class="sd">        &gt;&gt;&gt; # REGRESSION</span>
+<span class="sd">        &gt;&gt;&gt; target = torch.tensor([[0.6, 1.5]]).view(-1, 1)</span>
+<span class="sd">        &gt;&gt;&gt; input = torch.tensor([[.1, .2,], [.4, .5]])</span>
+<span class="sd">        &gt;&gt;&gt; qloss = QuantileLoss([0.25, 0.75])</span>
+<span class="sd">        &gt;&gt;&gt; loss = qloss(input, target)</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+
+        <span class="k">assert</span> <span class="nb">input</span><span class="o">.</span><span class="n">shape</span> <span class="o">==</span> <span class="n">torch</span><span class="o">.</span><span class="n">Size</span><span class="p">([</span><span class="n">target</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">quantiles</span><span class="p">)]),</span> <span class="p">(</span>
+            <span class="s2">&quot;The input and target have inconsistent shape. The dimension of the prediction &quot;</span>
+            <span class="s2">&quot;of the model that is using QuantileLoss must be equal to number of quantiles, &quot;</span>
+            <span class="sa">f</span><span class="s2">&quot;i.e. </span><span class="si">{</span><span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">quantiles</span><span class="p">)</span><span class="si">}</span><span class="s2">.&quot;</span>
+        <span class="p">)</span>
+        <span class="n">target</span> <span class="o">=</span> <span class="n">target</span><span class="o">.</span><span class="n">view</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span><span class="o">.</span><span class="n">float</span><span class="p">()</span>
+        <span class="n">losses</span> <span class="o">=</span> <span class="p">[]</span>
+        <span class="k">for</span> <span class="n">i</span><span class="p">,</span> <span class="n">q</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">quantiles</span><span class="p">):</span>
+            <span class="n">errors</span> <span class="o">=</span> <span class="n">target</span> <span class="o">-</span> <span class="nb">input</span><span class="p">[</span><span class="o">...</span><span class="p">,</span> <span class="n">i</span><span class="p">]</span>
+            <span class="n">losses</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">torch</span><span class="o">.</span><span class="n">max</span><span class="p">((</span><span class="n">q</span> <span class="o">-</span> <span class="mi">1</span><span class="p">)</span> <span class="o">*</span> <span class="n">errors</span><span class="p">,</span> <span class="n">q</span> <span class="o">*</span> <span class="n">errors</span><span class="p">)</span><span class="o">.</span><span class="n">unsqueeze</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">))</span>
+
+        <span class="n">loss</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">cat</span><span class="p">(</span><span class="n">losses</span><span class="p">,</span> <span class="n">dim</span><span class="o">=</span><span class="mi">2</span><span class="p">)</span>
+
+        <span class="k">return</span> <span class="n">torch</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="n">loss</span><span class="p">)</span>
 </code></pre></div></td></tr></table></div>
-                  </details>
+              </details>
 
 
 
@@ -2469,24 +2918,46 @@ <h3 id="pytorch_widedeep.losses.QuantileLoss.forward" class="doc doc-heading">
 
 
 <p><span class="doc-section-title">Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>input</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>)
-          –
-          <div class="doc-md-description">
-            <p>Input tensor with predictions</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>target</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>)
-          –
-          <div class="doc-md-description">
-            <p>Target tensor with the actual values</p>
-          </div>
-        </li>
-    </ul>
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+          <th>Default</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>input</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Input tensor with predictions</p>
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>target</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Target tensor with the actual values</p>
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
 
 <p><span class="doc-section-title">Examples:</span></p>
@@ -2597,8 +3068,7 @@ <h2 id="pytorch_widedeep.losses.FocalLoss" class="doc doc-heading">
 
 
 <a href="#pytorch_widedeep.losses.FocalLoss" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="nf">FocalLoss</span><span class="p">(</span><span class="n">alpha</span><span class="o">=</span><span class="mf">0.25</span><span class="p">,</span> <span class="n">gamma</span><span class="o">=</span><span class="mf">1.0</span><span class="p">)</span>
-</code></pre></div>
+
 
     <div class="doc doc-contents first">
             <p class="doc doc-class-bases">
@@ -2617,44 +3087,196 @@ <h2 id="pytorch_widedeep.losses.FocalLoss" class="doc doc-heading">
 
 
 <p><span class="doc-section-title">Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>alpha</code></b>
-              (<code>float</code>, default:
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+          <th>Default</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>alpha</code></td>
+            <td>
+                  <code>float</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Focal Loss <code>alpha</code> parameter</p>
+              </div>
+            </td>
+            <td>
                   <code>0.25</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Focal Loss <code>alpha</code> parameter</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>gamma</code></b>
-              (<code>float</code>, default:
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>gamma</code></td>
+            <td>
+                  <code>float</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Focal Loss <code>gamma</code> parameter</p>
+              </div>
+            </td>
+            <td>
                   <code>1.0</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Focal Loss <code>gamma</code> parameter</p>
-          </div>
-        </li>
-    </ul>
-
-                  <details class="quote">
-                    <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
-                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">233</span>
+            </td>
+          </tr>
+      </tbody>
+    </table>
+
+              <details class="quote">
+                <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
+                <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">211</span>
+<span class="normal">212</span>
+<span class="normal">213</span>
+<span class="normal">214</span>
+<span class="normal">215</span>
+<span class="normal">216</span>
+<span class="normal">217</span>
+<span class="normal">218</span>
+<span class="normal">219</span>
+<span class="normal">220</span>
+<span class="normal">221</span>
+<span class="normal">222</span>
+<span class="normal">223</span>
+<span class="normal">224</span>
+<span class="normal">225</span>
+<span class="normal">226</span>
+<span class="normal">227</span>
+<span class="normal">228</span>
+<span class="normal">229</span>
+<span class="normal">230</span>
+<span class="normal">231</span>
+<span class="normal">232</span>
+<span class="normal">233</span>
 <span class="normal">234</span>
 <span class="normal">235</span>
-<span class="normal">236</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">alpha</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.25</span><span class="p">,</span> <span class="n">gamma</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">1.0</span><span class="p">):</span>
-    <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">alpha</span> <span class="o">=</span> <span class="n">alpha</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">gamma</span> <span class="o">=</span> <span class="n">gamma</span>
-</code></pre></div></td></tr></table></div>
-                  </details>
+<span class="normal">236</span>
+<span class="normal">237</span>
+<span class="normal">238</span>
+<span class="normal">239</span>
+<span class="normal">240</span>
+<span class="normal">241</span>
+<span class="normal">242</span>
+<span class="normal">243</span>
+<span class="normal">244</span>
+<span class="normal">245</span>
+<span class="normal">246</span>
+<span class="normal">247</span>
+<span class="normal">248</span>
+<span class="normal">249</span>
+<span class="normal">250</span>
+<span class="normal">251</span>
+<span class="normal">252</span>
+<span class="normal">253</span>
+<span class="normal">254</span>
+<span class="normal">255</span>
+<span class="normal">256</span>
+<span class="normal">257</span>
+<span class="normal">258</span>
+<span class="normal">259</span>
+<span class="normal">260</span>
+<span class="normal">261</span>
+<span class="normal">262</span>
+<span class="normal">263</span>
+<span class="normal">264</span>
+<span class="normal">265</span>
+<span class="normal">266</span>
+<span class="normal">267</span>
+<span class="normal">268</span>
+<span class="normal">269</span>
+<span class="normal">270</span>
+<span class="normal">271</span>
+<span class="normal">272</span>
+<span class="normal">273</span>
+<span class="normal">274</span>
+<span class="normal">275</span>
+<span class="normal">276</span>
+<span class="normal">277</span>
+<span class="normal">278</span>
+<span class="normal">279</span>
+<span class="normal">280</span>
+<span class="normal">281</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">class</span> <span class="nc">FocalLoss</span><span class="p">(</span><span class="n">nn</span><span class="o">.</span><span class="n">Module</span><span class="p">):</span>
+<span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Implementation of the [Focal loss](https://arxiv.org/pdf/1708.02002.pdf)</span>
+<span class="sd">    for both binary and multiclass classification:</span>
 
+<span class="sd">    $$</span>
+<span class="sd">    FL(p_t) = \alpha (1 - p_t)^{\gamma} log(p_t)</span>
+<span class="sd">    $$</span>
 
+<span class="sd">    where, for a case of a binary classification problem</span>
 
-  <div class="doc doc-children">
+<span class="sd">    $$</span>
+<span class="sd">    \begin{equation} p_t= \begin{cases}p, &amp; \text{if $y=1$}.\\1-p, &amp; \text{otherwise}. \end{cases} \end{equation}</span>
+<span class="sd">    $$</span>
+
+<span class="sd">    Parameters</span>
+<span class="sd">    ----------</span>
+<span class="sd">    alpha: float</span>
+<span class="sd">        Focal Loss `alpha` parameter</span>
+<span class="sd">    gamma: float</span>
+<span class="sd">        Focal Loss `gamma` parameter</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+
+    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">alpha</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.25</span><span class="p">,</span> <span class="n">gamma</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">1.0</span><span class="p">):</span>
+        <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">alpha</span> <span class="o">=</span> <span class="n">alpha</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">gamma</span> <span class="o">=</span> <span class="n">gamma</span>
+
+    <span class="k">def</span> <span class="nf">_get_weight</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">p</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">,</span> <span class="n">t</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tensor</span><span class="p">:</span>
+        <span class="n">pt</span> <span class="o">=</span> <span class="n">p</span> <span class="o">*</span> <span class="n">t</span> <span class="o">+</span> <span class="p">(</span><span class="mi">1</span> <span class="o">-</span> <span class="n">p</span><span class="p">)</span> <span class="o">*</span> <span class="p">(</span><span class="mi">1</span> <span class="o">-</span> <span class="n">t</span><span class="p">)</span>  <span class="c1"># type: ignore</span>
+        <span class="n">w</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">alpha</span> <span class="o">*</span> <span class="n">t</span> <span class="o">+</span> <span class="p">(</span><span class="mi">1</span> <span class="o">-</span> <span class="bp">self</span><span class="o">.</span><span class="n">alpha</span><span class="p">)</span> <span class="o">*</span> <span class="p">(</span><span class="mi">1</span> <span class="o">-</span> <span class="n">t</span><span class="p">)</span>  <span class="c1"># type: ignore</span>
+        <span class="k">return</span> <span class="p">(</span><span class="n">w</span> <span class="o">*</span> <span class="p">(</span><span class="mi">1</span> <span class="o">-</span> <span class="n">pt</span><span class="p">)</span><span class="o">.</span><span class="n">pow</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">gamma</span><span class="p">))</span><span class="o">.</span><span class="n">detach</span><span class="p">()</span>  <span class="c1"># type: ignore</span>
+
+    <span class="k">def</span> <span class="nf">forward</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="nb">input</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">,</span> <span class="n">target</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tensor</span><span class="p">:</span>
+<span class="w">        </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Parameters</span>
+<span class="sd">        ----------</span>
+<span class="sd">        input: Tensor</span>
+<span class="sd">            Input tensor with predictions (not probabilities)</span>
+<span class="sd">        target: Tensor</span>
+<span class="sd">            Target tensor with the actual classes</span>
+
+<span class="sd">        Examples</span>
+<span class="sd">        --------</span>
+<span class="sd">        &gt;&gt;&gt; import torch</span>
+<span class="sd">        &gt;&gt;&gt;</span>
+<span class="sd">        &gt;&gt;&gt; from pytorch_widedeep.losses import FocalLoss</span>
+<span class="sd">        &gt;&gt;&gt;</span>
+<span class="sd">        &gt;&gt;&gt; # BINARY</span>
+<span class="sd">        &gt;&gt;&gt; target = torch.tensor([0, 1, 0, 1]).view(-1, 1)</span>
+<span class="sd">        &gt;&gt;&gt; input = torch.tensor([[0.6, 0.7, 0.3, 0.8]]).t()</span>
+<span class="sd">        &gt;&gt;&gt; loss = FocalLoss()(input, target)</span>
+<span class="sd">        &gt;&gt;&gt;</span>
+<span class="sd">        &gt;&gt;&gt; # MULTICLASS</span>
+<span class="sd">        &gt;&gt;&gt; target = torch.tensor([1, 0, 2]).view(-1, 1)</span>
+<span class="sd">        &gt;&gt;&gt; input = torch.tensor([[0.2, 0.5, 0.3], [0.8, 0.1, 0.1], [0.7, 0.2, 0.1]])</span>
+<span class="sd">        &gt;&gt;&gt; loss = FocalLoss()(input, target)</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">input_prob</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">sigmoid</span><span class="p">(</span><span class="nb">input</span><span class="p">)</span>
+        <span class="k">if</span> <span class="nb">input</span><span class="o">.</span><span class="n">size</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span> <span class="o">==</span> <span class="mi">1</span><span class="p">:</span>
+            <span class="n">input_prob</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">cat</span><span class="p">([</span><span class="mi">1</span> <span class="o">-</span> <span class="n">input_prob</span><span class="p">,</span> <span class="n">input_prob</span><span class="p">],</span> <span class="n">axis</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span>  <span class="c1"># type: ignore</span>
+            <span class="n">num_class</span> <span class="o">=</span> <span class="mi">2</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="n">num_class</span> <span class="o">=</span> <span class="n">input_prob</span><span class="o">.</span><span class="n">size</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span>
+        <span class="n">binary_target</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">eye</span><span class="p">(</span><span class="n">num_class</span><span class="p">)[</span><span class="n">target</span><span class="o">.</span><span class="n">squeeze</span><span class="p">()</span><span class="o">.</span><span class="n">cpu</span><span class="p">()</span><span class="o">.</span><span class="n">long</span><span class="p">()]</span>
+        <span class="n">binary_target</span> <span class="o">=</span> <span class="n">binary_target</span><span class="o">.</span><span class="n">to</span><span class="p">(</span><span class="nb">input</span><span class="o">.</span><span class="n">device</span><span class="p">)</span>
+        <span class="n">binary_target</span> <span class="o">=</span> <span class="n">binary_target</span><span class="o">.</span><span class="n">contiguous</span><span class="p">()</span>
+        <span class="n">weight</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_weight</span><span class="p">(</span><span class="n">input_prob</span><span class="p">,</span> <span class="n">binary_target</span><span class="p">)</span>
+
+        <span class="k">return</span> <span class="n">F</span><span class="o">.</span><span class="n">binary_cross_entropy</span><span class="p">(</span>
+            <span class="n">input_prob</span><span class="p">,</span> <span class="n">binary_target</span><span class="p">,</span> <span class="n">weight</span><span class="p">,</span> <span class="n">reduction</span><span class="o">=</span><span class="s2">&quot;mean&quot;</span>
+        <span class="p">)</span>
+</code></pre></div></td></tr></table></div>
+              </details>
+
+
+
+  <div class="doc doc-children">
 
 
 
@@ -2680,24 +3302,46 @@ <h3 id="pytorch_widedeep.losses.FocalLoss.forward" class="doc doc-heading">
 
 
 <p><span class="doc-section-title">Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>input</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>)
-          –
-          <div class="doc-md-description">
-            <p>Input tensor with predictions (not probabilities)</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>target</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>)
-          –
-          <div class="doc-md-description">
-            <p>Target tensor with the actual classes</p>
-          </div>
-        </li>
-    </ul>
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+          <th>Default</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>input</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Input tensor with predictions (not probabilities)</p>
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>target</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Target tensor with the actual classes</p>
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
 
 <p><span class="doc-section-title">Examples:</span></p>
@@ -2818,8 +3462,7 @@ <h2 id="pytorch_widedeep.losses.BayesianSELoss" class="doc doc-heading">
 
 
 <a href="#pytorch_widedeep.losses.BayesianSELoss" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="nf">BayesianSELoss</span><span class="p">()</span>
-</code></pre></div>
+
 
     <div class="doc doc-contents first">
             <p class="doc doc-class-bases">
@@ -2830,13 +3473,63 @@ <h2 id="pytorch_widedeep.losses.BayesianSELoss" class="doc doc-heading">
 the original publication
 <a href="https://arxiv.org/abs/1505.05424">Weight Uncertainty in Neural Networks</a>.</p>
 
-                  <details class="quote">
-                    <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
-                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">310</span>
-<span class="normal">311</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
-    <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
+              <details class="quote">
+                <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
+                <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">304</span>
+<span class="normal">305</span>
+<span class="normal">306</span>
+<span class="normal">307</span>
+<span class="normal">308</span>
+<span class="normal">309</span>
+<span class="normal">310</span>
+<span class="normal">311</span>
+<span class="normal">312</span>
+<span class="normal">313</span>
+<span class="normal">314</span>
+<span class="normal">315</span>
+<span class="normal">316</span>
+<span class="normal">317</span>
+<span class="normal">318</span>
+<span class="normal">319</span>
+<span class="normal">320</span>
+<span class="normal">321</span>
+<span class="normal">322</span>
+<span class="normal">323</span>
+<span class="normal">324</span>
+<span class="normal">325</span>
+<span class="normal">326</span>
+<span class="normal">327</span>
+<span class="normal">328</span>
+<span class="normal">329</span>
+<span class="normal">330</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">class</span> <span class="nc">BayesianSELoss</span><span class="p">(</span><span class="n">nn</span><span class="o">.</span><span class="n">Module</span><span class="p">):</span>
+<span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Squared Loss (log Gaussian) for the case of a regression as specified in</span>
+<span class="sd">    the original publication</span>
+<span class="sd">    [Weight Uncertainty in Neural Networks](https://arxiv.org/abs/1505.05424).</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+
+    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
+
+    <span class="k">def</span> <span class="nf">forward</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="nb">input</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">,</span> <span class="n">target</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tensor</span><span class="p">:</span>
+<span class="w">        </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Parameters</span>
+<span class="sd">        ----------</span>
+<span class="sd">        input: Tensor</span>
+<span class="sd">            Input tensor with predictions (not probabilities)</span>
+<span class="sd">        target: Tensor</span>
+<span class="sd">            Target tensor with the actual classes</span>
+
+<span class="sd">        Examples</span>
+<span class="sd">        --------</span>
+<span class="sd">        &gt;&gt;&gt; import torch</span>
+<span class="sd">        &gt;&gt;&gt; from pytorch_widedeep.losses import BayesianSELoss</span>
+<span class="sd">        &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)</span>
+<span class="sd">        &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)</span>
+<span class="sd">        &gt;&gt;&gt; loss = BayesianSELoss()(input, target)</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="k">return</span> <span class="p">(</span><span class="mf">0.5</span> <span class="o">*</span> <span class="p">(</span><span class="nb">input</span> <span class="o">-</span> <span class="n">target</span><span class="p">)</span> <span class="o">**</span> <span class="mi">2</span><span class="p">)</span><span class="o">.</span><span class="n">sum</span><span class="p">()</span>
 </code></pre></div></td></tr></table></div>
-                  </details>
+              </details>
 
 
 
@@ -2866,24 +3559,46 @@ <h3 id="pytorch_widedeep.losses.BayesianSELoss.forward" class="doc doc-heading">
 
 
 <p><span class="doc-section-title">Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>input</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>)
-          –
-          <div class="doc-md-description">
-            <p>Input tensor with predictions (not probabilities)</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>target</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>)
-          –
-          <div class="doc-md-description">
-            <p>Target tensor with the actual classes</p>
-          </div>
-        </li>
-    </ul>
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+          <th>Default</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>input</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Input tensor with predictions (not probabilities)</p>
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>target</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Target tensor with the actual classes</p>
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
 
 <p><span class="doc-section-title">Examples:</span></p>
@@ -2954,8 +3669,7 @@ <h2 id="pytorch_widedeep.losses.TweedieLoss" class="doc doc-heading">
 
 
 <a href="#pytorch_widedeep.losses.TweedieLoss" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="nf">TweedieLoss</span><span class="p">()</span>
-</code></pre></div>
+
 
     <div class="doc doc-contents first">
             <p class="doc doc-class-bases">
@@ -2967,13 +3681,107 @@ <h2 id="pytorch_widedeep.losses.TweedieLoss" class="doc doc-heading">
 <a href="https://towardsdatascience.com/tweedie-loss-function-for-right-skewed-data-2c5ca470678f">this post</a>
 and the <a href="https://arxiv.org/abs/1811.10192">original publication</a> for details.</p>
 
-                  <details class="quote">
-                    <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
-                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">342</span>
-<span class="normal">343</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
-    <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
+              <details class="quote">
+                <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
+                <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">333</span>
+<span class="normal">334</span>
+<span class="normal">335</span>
+<span class="normal">336</span>
+<span class="normal">337</span>
+<span class="normal">338</span>
+<span class="normal">339</span>
+<span class="normal">340</span>
+<span class="normal">341</span>
+<span class="normal">342</span>
+<span class="normal">343</span>
+<span class="normal">344</span>
+<span class="normal">345</span>
+<span class="normal">346</span>
+<span class="normal">347</span>
+<span class="normal">348</span>
+<span class="normal">349</span>
+<span class="normal">350</span>
+<span class="normal">351</span>
+<span class="normal">352</span>
+<span class="normal">353</span>
+<span class="normal">354</span>
+<span class="normal">355</span>
+<span class="normal">356</span>
+<span class="normal">357</span>
+<span class="normal">358</span>
+<span class="normal">359</span>
+<span class="normal">360</span>
+<span class="normal">361</span>
+<span class="normal">362</span>
+<span class="normal">363</span>
+<span class="normal">364</span>
+<span class="normal">365</span>
+<span class="normal">366</span>
+<span class="normal">367</span>
+<span class="normal">368</span>
+<span class="normal">369</span>
+<span class="normal">370</span>
+<span class="normal">371</span>
+<span class="normal">372</span>
+<span class="normal">373</span>
+<span class="normal">374</span>
+<span class="normal">375</span>
+<span class="normal">376</span>
+<span class="normal">377</span>
+<span class="normal">378</span>
+<span class="normal">379</span>
+<span class="normal">380</span>
+<span class="normal">381</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">class</span> <span class="nc">TweedieLoss</span><span class="p">(</span><span class="n">nn</span><span class="o">.</span><span class="n">Module</span><span class="p">):</span>
+<span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">    Tweedie loss for extremely unbalanced zero-inflated data</span>
+
+<span class="sd">    All credits go to Wenbo Shi. See</span>
+<span class="sd">    [this post](https://towardsdatascience.com/tweedie-loss-function-for-right-skewed-data-2c5ca470678f)</span>
+<span class="sd">    and the [original publication](https://arxiv.org/abs/1811.10192) for details.</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+
+    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
+
+    <span class="k">def</span> <span class="nf">forward</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span>
+        <span class="nb">input</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">,</span>
+        <span class="n">target</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">,</span>
+        <span class="n">p</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">1.5</span><span class="p">,</span>
+    <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tensor</span><span class="p">:</span>
+<span class="w">        </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Parameters</span>
+<span class="sd">        ----------</span>
+<span class="sd">        input: Tensor</span>
+<span class="sd">            Input tensor with predictions</span>
+<span class="sd">        target: Tensor</span>
+<span class="sd">            Target tensor with the actual values</span>
+<span class="sd">        p: float, default = 1.5</span>
+<span class="sd">            the power to be used to compute the loss. See the original</span>
+<span class="sd">            publication for details</span>
+
+<span class="sd">        Examples</span>
+<span class="sd">        --------</span>
+<span class="sd">        &gt;&gt;&gt; import torch</span>
+<span class="sd">        &gt;&gt;&gt; from pytorch_widedeep.losses import TweedieLoss</span>
+<span class="sd">        &gt;&gt;&gt;</span>
+<span class="sd">        &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)</span>
+<span class="sd">        &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)</span>
+<span class="sd">        &gt;&gt;&gt; loss = TweedieLoss()(input, target)</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+
+        <span class="k">assert</span> <span class="p">(</span>
+            <span class="nb">input</span><span class="o">.</span><span class="n">min</span><span class="p">()</span> <span class="o">&gt;</span> <span class="mi">0</span>
+        <span class="p">),</span> <span class="s2">&quot;&quot;&quot;All input values must be &gt;=0, if your model is predicting</span>
+<span class="s2">            values &lt;0 try to enforce positive values by activation function</span>
+<span class="s2">            on last layer with `trainer.enforce_positive_output=True`&quot;&quot;&quot;</span>
+        <span class="k">assert</span> <span class="n">target</span><span class="o">.</span><span class="n">min</span><span class="p">()</span> <span class="o">&gt;=</span> <span class="mi">0</span><span class="p">,</span> <span class="s2">&quot;All target values must be &gt;=0&quot;</span>
+        <span class="n">loss</span> <span class="o">=</span> <span class="o">-</span><span class="n">target</span> <span class="o">*</span> <span class="n">torch</span><span class="o">.</span><span class="n">pow</span><span class="p">(</span><span class="nb">input</span><span class="p">,</span> <span class="mi">1</span> <span class="o">-</span> <span class="n">p</span><span class="p">)</span> <span class="o">/</span> <span class="p">(</span><span class="mi">1</span> <span class="o">-</span> <span class="n">p</span><span class="p">)</span> <span class="o">+</span> <span class="n">torch</span><span class="o">.</span><span class="n">pow</span><span class="p">(</span><span class="nb">input</span><span class="p">,</span> <span class="mi">2</span> <span class="o">-</span> <span class="n">p</span><span class="p">)</span> <span class="o">/</span> <span class="p">(</span>
+            <span class="mi">2</span> <span class="o">-</span> <span class="n">p</span>
+        <span class="p">)</span>
+        <span class="k">return</span> <span class="n">torch</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="n">loss</span><span class="p">)</span>
 </code></pre></div></td></tr></table></div>
-                  </details>
+              </details>
 
 
 
@@ -3003,35 +3811,61 @@ <h3 id="pytorch_widedeep.losses.TweedieLoss.forward" class="doc doc-heading">
 
 
 <p><span class="doc-section-title">Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>input</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>)
-          –
-          <div class="doc-md-description">
-            <p>Input tensor with predictions</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>target</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>)
-          –
-          <div class="doc-md-description">
-            <p>Target tensor with the actual values</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>p</code></b>
-              (<code>float</code>, default:
-                  <code>1.5</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>the power to be used to compute the loss. See the original
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+          <th>Default</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>input</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Input tensor with predictions</p>
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>target</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Target tensor with the actual values</p>
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>p</code></td>
+            <td>
+                  <code>float</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>the power to be used to compute the loss. See the original
 publication for details</p>
-          </div>
-        </li>
-    </ul>
+              </div>
+            </td>
+            <td>
+                  <code>1.5</code>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
 
 <p><span class="doc-section-title">Examples:</span></p>
@@ -3141,8 +3975,7 @@ <h2 id="pytorch_widedeep.losses.ZILNLoss" class="doc doc-heading">
 
 
 <a href="#pytorch_widedeep.losses.ZILNLoss" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="nf">ZILNLoss</span><span class="p">()</span>
-</code></pre></div>
+
 
     <div class="doc doc-contents first">
             <p class="doc doc-class-bases">
@@ -3154,13 +3987,169 @@ <h2 id="pytorch_widedeep.losses.ZILNLoss" class="doc doc-heading">
 and the corresponding
 <a href="https://github.com/google/lifetime_value/blob/master/lifetime_value/zero_inflated_lognormal.py">code</a>.</p>
 
-                  <details class="quote">
-                    <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
-                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">392</span>
-<span class="normal">393</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
-    <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
+              <details class="quote">
+                <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
+                <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">384</span>
+<span class="normal">385</span>
+<span class="normal">386</span>
+<span class="normal">387</span>
+<span class="normal">388</span>
+<span class="normal">389</span>
+<span class="normal">390</span>
+<span class="normal">391</span>
+<span class="normal">392</span>
+<span class="normal">393</span>
+<span class="normal">394</span>
+<span class="normal">395</span>
+<span class="normal">396</span>
+<span class="normal">397</span>
+<span class="normal">398</span>
+<span class="normal">399</span>
+<span class="normal">400</span>
+<span class="normal">401</span>
+<span class="normal">402</span>
+<span class="normal">403</span>
+<span class="normal">404</span>
+<span class="normal">405</span>
+<span class="normal">406</span>
+<span class="normal">407</span>
+<span class="normal">408</span>
+<span class="normal">409</span>
+<span class="normal">410</span>
+<span class="normal">411</span>
+<span class="normal">412</span>
+<span class="normal">413</span>
+<span class="normal">414</span>
+<span class="normal">415</span>
+<span class="normal">416</span>
+<span class="normal">417</span>
+<span class="normal">418</span>
+<span class="normal">419</span>
+<span class="normal">420</span>
+<span class="normal">421</span>
+<span class="normal">422</span>
+<span class="normal">423</span>
+<span class="normal">424</span>
+<span class="normal">425</span>
+<span class="normal">426</span>
+<span class="normal">427</span>
+<span class="normal">428</span>
+<span class="normal">429</span>
+<span class="normal">430</span>
+<span class="normal">431</span>
+<span class="normal">432</span>
+<span class="normal">433</span>
+<span class="normal">434</span>
+<span class="normal">435</span>
+<span class="normal">436</span>
+<span class="normal">437</span>
+<span class="normal">438</span>
+<span class="normal">439</span>
+<span class="normal">440</span>
+<span class="normal">441</span>
+<span class="normal">442</span>
+<span class="normal">443</span>
+<span class="normal">444</span>
+<span class="normal">445</span>
+<span class="normal">446</span>
+<span class="normal">447</span>
+<span class="normal">448</span>
+<span class="normal">449</span>
+<span class="normal">450</span>
+<span class="normal">451</span>
+<span class="normal">452</span>
+<span class="normal">453</span>
+<span class="normal">454</span>
+<span class="normal">455</span>
+<span class="normal">456</span>
+<span class="normal">457</span>
+<span class="normal">458</span>
+<span class="normal">459</span>
+<span class="normal">460</span>
+<span class="normal">461</span>
+<span class="normal">462</span>
+<span class="normal">463</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">class</span> <span class="nc">ZILNLoss</span><span class="p">(</span><span class="n">nn</span><span class="o">.</span><span class="n">Module</span><span class="p">):</span>
+<span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Adjusted implementation of the Zero Inflated LogNormal Loss</span>
+
+<span class="sd">    See [A Deep Probabilistic Model for Customer Lifetime Value Prediction](https://arxiv.org/pdf/1912.07753.pdf)</span>
+<span class="sd">    and the corresponding</span>
+<span class="sd">    [code](https://github.com/google/lifetime_value/blob/master/lifetime_value/zero_inflated_lognormal.py).</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+
+    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
+
+    <span class="k">def</span> <span class="nf">forward</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="nb">input</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">,</span> <span class="n">target</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tensor</span><span class="p">:</span>
+<span class="w">        </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Parameters</span>
+<span class="sd">        ----------</span>
+<span class="sd">        input: Tensor</span>
+<span class="sd">            Input tensor with predictions with spape (N,3), where N is the batch size</span>
+<span class="sd">        target: Tensor</span>
+<span class="sd">            Target tensor with the actual target values</span>
+
+<span class="sd">        Examples</span>
+<span class="sd">        --------</span>
+<span class="sd">        &gt;&gt;&gt; import torch</span>
+<span class="sd">        &gt;&gt;&gt; from pytorch_widedeep.losses import ZILNLoss</span>
+<span class="sd">        &gt;&gt;&gt;</span>
+<span class="sd">        &gt;&gt;&gt; target = torch.tensor([[0., 1.5]]).view(-1, 1)</span>
+<span class="sd">        &gt;&gt;&gt; input = torch.tensor([[.1, .2, .3], [.4, .5, .6]])</span>
+<span class="sd">        &gt;&gt;&gt; loss = ZILNLoss()(input, target)</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">positive</span> <span class="o">=</span> <span class="n">target</span> <span class="o">&gt;</span> <span class="mi">0</span>
+        <span class="n">positive</span> <span class="o">=</span> <span class="n">positive</span><span class="o">.</span><span class="n">float</span><span class="p">()</span>
+
+        <span class="k">assert</span> <span class="nb">input</span><span class="o">.</span><span class="n">shape</span> <span class="o">==</span> <span class="n">torch</span><span class="o">.</span><span class="n">Size</span><span class="p">([</span><span class="n">target</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="mi">3</span><span class="p">]),</span> <span class="p">(</span>
+            <span class="s2">&quot;Wrong shape of the &#39;input&#39; tensor. The pred_dim of the &quot;</span>
+            <span class="s2">&quot;model that is using ZILNLoss must be equal to 3.&quot;</span>
+        <span class="p">)</span>
+
+        <span class="n">positive_input</span> <span class="o">=</span> <span class="nb">input</span><span class="p">[</span><span class="o">...</span><span class="p">,</span> <span class="p">:</span><span class="mi">1</span><span class="p">]</span>
+
+        <span class="n">classification_loss</span> <span class="o">=</span> <span class="n">F</span><span class="o">.</span><span class="n">binary_cross_entropy_with_logits</span><span class="p">(</span>
+            <span class="n">positive_input</span><span class="p">,</span> <span class="n">positive</span><span class="p">,</span> <span class="n">reduction</span><span class="o">=</span><span class="s2">&quot;none&quot;</span>
+        <span class="p">)</span><span class="o">.</span><span class="n">flatten</span><span class="p">()</span>
+
+        <span class="n">loc</span> <span class="o">=</span> <span class="nb">input</span><span class="p">[</span><span class="o">...</span><span class="p">,</span> <span class="mi">1</span><span class="p">:</span><span class="mi">2</span><span class="p">]</span>
+
+        <span class="c1"># when using max the two input tensors (input and other) have to be of</span>
+        <span class="c1"># the same type</span>
+        <span class="n">max_input</span> <span class="o">=</span> <span class="n">F</span><span class="o">.</span><span class="n">softplus</span><span class="p">(</span><span class="nb">input</span><span class="p">[</span><span class="o">...</span><span class="p">,</span> <span class="mi">2</span><span class="p">:])</span>
+        <span class="n">max_other</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_eps</span><span class="p">(</span><span class="n">max_input</span><span class="p">)</span>
+        <span class="n">scale</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">max</span><span class="p">(</span><span class="n">max_input</span><span class="p">,</span> <span class="n">max_other</span><span class="p">)</span>
+        <span class="n">safe_labels</span> <span class="o">=</span> <span class="n">positive</span> <span class="o">*</span> <span class="n">target</span> <span class="o">+</span> <span class="p">(</span><span class="mi">1</span> <span class="o">-</span> <span class="n">positive</span><span class="p">)</span> <span class="o">*</span> <span class="n">torch</span><span class="o">.</span><span class="n">ones_like</span><span class="p">(</span><span class="n">target</span><span class="p">)</span>
+
+        <span class="n">regression_loss</span> <span class="o">=</span> <span class="o">-</span><span class="n">torch</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span>
+            <span class="n">positive</span>
+            <span class="o">*</span> <span class="n">torch</span><span class="o">.</span><span class="n">distributions</span><span class="o">.</span><span class="n">log_normal</span><span class="o">.</span><span class="n">LogNormal</span><span class="p">(</span><span class="n">loc</span><span class="o">=</span><span class="n">loc</span><span class="p">,</span> <span class="n">scale</span><span class="o">=</span><span class="n">scale</span><span class="p">)</span><span class="o">.</span><span class="n">log_prob</span><span class="p">(</span>
+                <span class="n">safe_labels</span>
+            <span class="p">),</span>
+            <span class="n">dim</span><span class="o">=-</span><span class="mi">1</span><span class="p">,</span>
+        <span class="p">)</span>
+
+        <span class="k">return</span> <span class="n">torch</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="n">classification_loss</span> <span class="o">+</span> <span class="n">regression_loss</span><span class="p">)</span>
+
+    <span class="nd">@staticmethod</span>
+    <span class="k">def</span> <span class="nf">get_eps</span><span class="p">(</span><span class="n">max_input</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tensor</span><span class="p">:</span>
+        <span class="k">if</span> <span class="n">max_input</span><span class="o">.</span><span class="n">device</span><span class="o">.</span><span class="n">type</span> <span class="o">==</span> <span class="s2">&quot;mps&quot;</span><span class="p">:</span>
+            <span class="c1"># For MPS, use float32 and then convert to the input type</span>
+            <span class="n">eps</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">finfo</span><span class="p">(</span><span class="n">torch</span><span class="o">.</span><span class="n">float32</span><span class="p">)</span><span class="o">.</span><span class="n">eps</span>
+            <span class="n">max_other</span> <span class="o">=</span> <span class="p">(</span>
+                <span class="n">torch</span><span class="o">.</span><span class="n">sqrt</span><span class="p">(</span><span class="n">torch</span><span class="o">.</span><span class="n">tensor</span><span class="p">([</span><span class="n">eps</span><span class="p">],</span> <span class="n">device</span><span class="o">=</span><span class="s2">&quot;cpu&quot;</span><span class="p">))</span>
+                <span class="o">.</span><span class="n">to</span><span class="p">(</span><span class="n">max_input</span><span class="o">.</span><span class="n">device</span><span class="p">)</span>
+                <span class="o">.</span><span class="n">to</span><span class="p">(</span><span class="n">max_input</span><span class="o">.</span><span class="n">dtype</span><span class="p">)</span>
+            <span class="p">)</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="c1"># For other devices, use the original approach</span>
+            <span class="n">eps</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">finfo</span><span class="p">(</span><span class="n">torch</span><span class="o">.</span><span class="n">double</span><span class="p">)</span><span class="o">.</span><span class="n">eps</span>
+            <span class="n">max_other</span> <span class="o">=</span> <span class="p">(</span>
+                <span class="n">torch</span><span class="o">.</span><span class="n">sqrt</span><span class="p">(</span><span class="n">torch</span><span class="o">.</span><span class="n">tensor</span><span class="p">([</span><span class="n">eps</span><span class="p">]))</span><span class="o">.</span><span class="n">to</span><span class="p">(</span><span class="n">max_input</span><span class="o">.</span><span class="n">device</span><span class="p">)</span><span class="o">.</span><span class="n">to</span><span class="p">(</span><span class="n">max_input</span><span class="o">.</span><span class="n">dtype</span><span class="p">)</span>
+            <span class="p">)</span>
+
+        <span class="k">return</span> <span class="n">max_other</span>
 </code></pre></div></td></tr></table></div>
-                  </details>
+              </details>
 
 
 
@@ -3190,24 +4179,46 @@ <h3 id="pytorch_widedeep.losses.ZILNLoss.forward" class="doc doc-heading">
 
 
 <p><span class="doc-section-title">Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>input</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>)
-          –
-          <div class="doc-md-description">
-            <p>Input tensor with predictions with spape (N,3), where N is the batch size</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>target</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>)
-          –
-          <div class="doc-md-description">
-            <p>Target tensor with the actual target values</p>
-          </div>
-        </li>
-    </ul>
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+          <th>Default</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>input</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Input tensor with predictions with spape (N,3), where N is the batch size</p>
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>target</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Target tensor with the actual target values</p>
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
 
 <p><span class="doc-section-title">Examples:</span></p>
@@ -3343,8 +4354,7 @@ <h2 id="pytorch_widedeep.losses.L1Loss" class="doc doc-heading">
 
 
 <a href="#pytorch_widedeep.losses.L1Loss" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="nf">L1Loss</span><span class="p">()</span>
-</code></pre></div>
+
 
     <div class="doc doc-contents first">
             <p class="doc doc-class-bases">
@@ -3353,13 +4363,69 @@ <h2 id="pytorch_widedeep.losses.L1Loss" class="doc doc-heading">
 
         <p>L1 loss</p>
 
-                  <details class="quote">
-                    <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
-                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">472</span>
-<span class="normal">473</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
-    <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
+              <details class="quote">
+                <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
+                <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">466</span>
+<span class="normal">467</span>
+<span class="normal">468</span>
+<span class="normal">469</span>
+<span class="normal">470</span>
+<span class="normal">471</span>
+<span class="normal">472</span>
+<span class="normal">473</span>
+<span class="normal">474</span>
+<span class="normal">475</span>
+<span class="normal">476</span>
+<span class="normal">477</span>
+<span class="normal">478</span>
+<span class="normal">479</span>
+<span class="normal">480</span>
+<span class="normal">481</span>
+<span class="normal">482</span>
+<span class="normal">483</span>
+<span class="normal">484</span>
+<span class="normal">485</span>
+<span class="normal">486</span>
+<span class="normal">487</span>
+<span class="normal">488</span>
+<span class="normal">489</span>
+<span class="normal">490</span>
+<span class="normal">491</span>
+<span class="normal">492</span>
+<span class="normal">493</span>
+<span class="normal">494</span>
+<span class="normal">495</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">class</span> <span class="nc">L1Loss</span><span class="p">(</span><span class="n">nn</span><span class="o">.</span><span class="n">Module</span><span class="p">):</span>
+<span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;L1 loss&quot;&quot;&quot;</span>
+
+    <span class="c1"># legacy code from when we used to support FDS-LDS and this class could</span>
+    <span class="c1"># taked the corresponding params. At this stage probably you want to use</span>
+    <span class="c1"># torch.nn.L1Loss</span>
+    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
+
+    <span class="k">def</span> <span class="nf">forward</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="nb">input</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">,</span> <span class="n">target</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tensor</span><span class="p">:</span>
+<span class="w">        </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Parameters</span>
+<span class="sd">        ----------</span>
+<span class="sd">        input: Tensor</span>
+<span class="sd">            Input tensor with predictions</span>
+<span class="sd">        target: Tensor</span>
+<span class="sd">            Target tensor with the actual values</span>
+
+<span class="sd">        Examples</span>
+<span class="sd">        --------</span>
+<span class="sd">        &gt;&gt;&gt; import torch</span>
+<span class="sd">        &gt;&gt;&gt;</span>
+<span class="sd">        &gt;&gt;&gt; from pytorch_widedeep.losses import L1Loss</span>
+<span class="sd">        &gt;&gt;&gt;</span>
+<span class="sd">        &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)</span>
+<span class="sd">        &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)</span>
+<span class="sd">        &gt;&gt;&gt; loss = L1Loss()(input, target)</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">loss</span> <span class="o">=</span> <span class="n">F</span><span class="o">.</span><span class="n">l1_loss</span><span class="p">(</span><span class="nb">input</span><span class="p">,</span> <span class="n">target</span><span class="p">,</span> <span class="n">reduction</span><span class="o">=</span><span class="s2">&quot;none&quot;</span><span class="p">)</span>
+        <span class="k">return</span> <span class="n">torch</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="n">loss</span><span class="p">)</span>
 </code></pre></div></td></tr></table></div>
-                  </details>
+              </details>
 
 
 
@@ -3389,24 +4455,46 @@ <h3 id="pytorch_widedeep.losses.L1Loss.forward" class="doc doc-heading">
 
 
 <p><span class="doc-section-title">Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>input</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>)
-          –
-          <div class="doc-md-description">
-            <p>Input tensor with predictions</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>target</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>)
-          –
-          <div class="doc-md-description">
-            <p>Target tensor with the actual values</p>
-          </div>
-        </li>
-    </ul>
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+          <th>Default</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>input</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Input tensor with predictions</p>
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>target</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Target tensor with the actual values</p>
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
 
 <p><span class="doc-section-title">Examples:</span></p>
@@ -3485,8 +4573,7 @@ <h2 id="pytorch_widedeep.losses.FocalR_L1Loss" class="doc doc-heading">
 
 
 <a href="#pytorch_widedeep.losses.FocalR_L1Loss" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="nf">FocalR_L1Loss</span><span class="p">(</span><span class="n">beta</span><span class="o">=</span><span class="mf">0.2</span><span class="p">,</span> <span class="n">gamma</span><span class="o">=</span><span class="mf">1.0</span><span class="p">,</span> <span class="n">activation_fn</span><span class="o">=</span><span class="s1">&#39;sigmoid&#39;</span><span class="p">)</span>
-</code></pre></div>
+
 
     <div class="doc doc-contents first">
             <p class="doc doc-class-bases">
@@ -3498,44 +4585,83 @@ <h2 id="pytorch_widedeep.losses.FocalR_L1Loss" class="doc doc-heading">
 
 
 <p><span class="doc-section-title">Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>beta</code></b>
-              (<code>float</code>, default:
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+          <th>Default</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>beta</code></td>
+            <td>
+                  <code>float</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Focal Loss <code>beta</code> parameter in their implementation</p>
+              </div>
+            </td>
+            <td>
                   <code>0.2</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Focal Loss <code>beta</code> parameter in their implementation</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>gamma</code></b>
-              (<code>float</code>, default:
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>gamma</code></td>
+            <td>
+                  <code>float</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Focal Loss <code>gamma</code> parameter</p>
+              </div>
+            </td>
+            <td>
                   <code>1.0</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Focal Loss <code>gamma</code> parameter</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>activation_fn</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[sigmoid, tanh]</code>, default:
-                  <code>&#39;sigmoid&#39;</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Activation function to be used during the computation of the loss.
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>activation_fn</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[sigmoid, tanh]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Activation function to be used during the computation of the loss.
 Possible values are <em>'sigmoid'</em> and <em>'tanh'</em>. See the original
 publication for details.</p>
-          </div>
-        </li>
-    </ul>
-
-                  <details class="quote">
-                    <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
-                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">515</span>
+              </div>
+            </td>
+            <td>
+                  <code>&#39;sigmoid&#39;</code>
+            </td>
+          </tr>
+      </tbody>
+    </table>
+
+              <details class="quote">
+                <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
+                <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">498</span>
+<span class="normal">499</span>
+<span class="normal">500</span>
+<span class="normal">501</span>
+<span class="normal">502</span>
+<span class="normal">503</span>
+<span class="normal">504</span>
+<span class="normal">505</span>
+<span class="normal">506</span>
+<span class="normal">507</span>
+<span class="normal">508</span>
+<span class="normal">509</span>
+<span class="normal">510</span>
+<span class="normal">511</span>
+<span class="normal">512</span>
+<span class="normal">513</span>
+<span class="normal">514</span>
+<span class="normal">515</span>
 <span class="normal">516</span>
 <span class="normal">517</span>
 <span class="normal">518</span>
@@ -3544,18 +4670,107 @@ <h2 id="pytorch_widedeep.losses.FocalR_L1Loss" class="doc doc-heading">
 <span class="normal">521</span>
 <span class="normal">522</span>
 <span class="normal">523</span>
-<span class="normal">524</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
-    <span class="bp">self</span><span class="p">,</span>
-    <span class="n">beta</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.2</span><span class="p">,</span>
-    <span class="n">gamma</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">1.0</span><span class="p">,</span>
-    <span class="n">activation_fn</span><span class="p">:</span> <span class="n">Literal</span><span class="p">[</span><span class="s2">&quot;sigmoid&quot;</span><span class="p">,</span> <span class="s2">&quot;tanh&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="s2">&quot;sigmoid&quot;</span><span class="p">,</span>
-<span class="p">):</span>
-    <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">beta</span> <span class="o">=</span> <span class="n">beta</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">gamma</span> <span class="o">=</span> <span class="n">gamma</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">activation_fn</span> <span class="o">=</span> <span class="n">activation_fn</span>
+<span class="normal">524</span>
+<span class="normal">525</span>
+<span class="normal">526</span>
+<span class="normal">527</span>
+<span class="normal">528</span>
+<span class="normal">529</span>
+<span class="normal">530</span>
+<span class="normal">531</span>
+<span class="normal">532</span>
+<span class="normal">533</span>
+<span class="normal">534</span>
+<span class="normal">535</span>
+<span class="normal">536</span>
+<span class="normal">537</span>
+<span class="normal">538</span>
+<span class="normal">539</span>
+<span class="normal">540</span>
+<span class="normal">541</span>
+<span class="normal">542</span>
+<span class="normal">543</span>
+<span class="normal">544</span>
+<span class="normal">545</span>
+<span class="normal">546</span>
+<span class="normal">547</span>
+<span class="normal">548</span>
+<span class="normal">549</span>
+<span class="normal">550</span>
+<span class="normal">551</span>
+<span class="normal">552</span>
+<span class="normal">553</span>
+<span class="normal">554</span>
+<span class="normal">555</span>
+<span class="normal">556</span>
+<span class="normal">557</span>
+<span class="normal">558</span>
+<span class="normal">559</span>
+<span class="normal">560</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">class</span> <span class="nc">FocalR_L1Loss</span><span class="p">(</span><span class="n">nn</span><span class="o">.</span><span class="n">Module</span><span class="p">):</span>
+<span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Focal-R L1 loss</span>
+
+<span class="sd">    Based on [Delving into Deep Imbalanced Regression](https://arxiv.org/abs/2102.09554).</span>
+
+<span class="sd">    Parameters</span>
+<span class="sd">    ----------</span>
+<span class="sd">    beta: float</span>
+<span class="sd">        Focal Loss `beta` parameter in their implementation</span>
+<span class="sd">    gamma: float</span>
+<span class="sd">        Focal Loss `gamma` parameter</span>
+<span class="sd">    activation_fn: str, default = &quot;sigmoid&quot;</span>
+<span class="sd">        Activation function to be used during the computation of the loss.</span>
+<span class="sd">        Possible values are _&#39;sigmoid&#39;_ and _&#39;tanh&#39;_. See the original</span>
+<span class="sd">        publication for details.</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+
+    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span>
+        <span class="n">beta</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.2</span><span class="p">,</span>
+        <span class="n">gamma</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">1.0</span><span class="p">,</span>
+        <span class="n">activation_fn</span><span class="p">:</span> <span class="n">Literal</span><span class="p">[</span><span class="s2">&quot;sigmoid&quot;</span><span class="p">,</span> <span class="s2">&quot;tanh&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="s2">&quot;sigmoid&quot;</span><span class="p">,</span>
+    <span class="p">):</span>
+        <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">beta</span> <span class="o">=</span> <span class="n">beta</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">gamma</span> <span class="o">=</span> <span class="n">gamma</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">activation_fn</span> <span class="o">=</span> <span class="n">activation_fn</span>
+
+    <span class="k">def</span> <span class="nf">forward</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span>
+        <span class="nb">input</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">,</span>
+        <span class="n">target</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">,</span>
+    <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tensor</span><span class="p">:</span>
+<span class="w">        </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Parameters</span>
+<span class="sd">        ----------</span>
+<span class="sd">        input: Tensor</span>
+<span class="sd">            Input tensor with predictions (not probabilities)</span>
+<span class="sd">        target: Tensor</span>
+<span class="sd">            Target tensor with the actual classes</span>
+
+<span class="sd">        Examples</span>
+<span class="sd">        --------</span>
+<span class="sd">        &gt;&gt;&gt; import torch</span>
+<span class="sd">        &gt;&gt;&gt;</span>
+<span class="sd">        &gt;&gt;&gt; from pytorch_widedeep.losses import FocalR_L1Loss</span>
+<span class="sd">        &gt;&gt;&gt;</span>
+<span class="sd">        &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)</span>
+<span class="sd">        &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)</span>
+<span class="sd">        &gt;&gt;&gt; loss = FocalR_L1Loss()(input, target)</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">loss</span> <span class="o">=</span> <span class="n">F</span><span class="o">.</span><span class="n">l1_loss</span><span class="p">(</span><span class="nb">input</span><span class="p">,</span> <span class="n">target</span><span class="p">,</span> <span class="n">reduction</span><span class="o">=</span><span class="s2">&quot;none&quot;</span><span class="p">)</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">activation_fn</span> <span class="o">==</span> <span class="s2">&quot;tanh&quot;</span><span class="p">:</span>
+            <span class="n">loss</span> <span class="o">*=</span> <span class="p">(</span><span class="n">torch</span><span class="o">.</span><span class="n">tanh</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">beta</span> <span class="o">*</span> <span class="n">torch</span><span class="o">.</span><span class="n">abs</span><span class="p">(</span><span class="nb">input</span> <span class="o">-</span> <span class="n">target</span><span class="p">)))</span> <span class="o">**</span> <span class="bp">self</span><span class="o">.</span><span class="n">gamma</span>
+        <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">activation_fn</span> <span class="o">==</span> <span class="s2">&quot;sigmoid&quot;</span><span class="p">:</span>
+            <span class="n">loss</span> <span class="o">*=</span> <span class="p">(</span>
+                <span class="mi">2</span> <span class="o">*</span> <span class="n">torch</span><span class="o">.</span><span class="n">sigmoid</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">beta</span> <span class="o">*</span> <span class="n">torch</span><span class="o">.</span><span class="n">abs</span><span class="p">(</span><span class="nb">input</span> <span class="o">-</span> <span class="n">target</span><span class="p">))</span> <span class="o">-</span> <span class="mi">1</span>
+            <span class="p">)</span> <span class="o">**</span> <span class="bp">self</span><span class="o">.</span><span class="n">gamma</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="ne">ValueError</span><span class="p">(</span>
+                <span class="s2">&quot;Incorrect activation function value - must be in [&#39;sigmoid&#39;, &#39;tanh&#39;]&quot;</span>
+            <span class="p">)</span>
+        <span class="k">return</span> <span class="n">torch</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="n">loss</span><span class="p">)</span>
 </code></pre></div></td></tr></table></div>
-                  </details>
+              </details>
 
 
 
@@ -3585,24 +4800,46 @@ <h3 id="pytorch_widedeep.losses.FocalR_L1Loss.forward" class="doc doc-heading">
 
 
 <p><span class="doc-section-title">Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>input</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>)
-          –
-          <div class="doc-md-description">
-            <p>Input tensor with predictions (not probabilities)</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>target</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>)
-          –
-          <div class="doc-md-description">
-            <p>Target tensor with the actual classes</p>
-          </div>
-        </li>
-    </ul>
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+          <th>Default</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>input</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Input tensor with predictions (not probabilities)</p>
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>target</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Target tensor with the actual classes</p>
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
 
 <p><span class="doc-section-title">Examples:</span></p>
@@ -3709,10 +4946,7 @@ <h2 id="pytorch_widedeep.losses.FocalR_MSELoss" class="doc doc-heading">
 
 
 <a href="#pytorch_widedeep.losses.FocalR_MSELoss" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="nf">FocalR_MSELoss</span><span class="p">(</span>
-    <span class="n">beta</span><span class="o">=</span><span class="mf">0.2</span><span class="p">,</span> <span class="n">gamma</span><span class="o">=</span><span class="mf">1.0</span><span class="p">,</span> <span class="n">activation_fn</span><span class="o">=</span><span class="s2">&quot;sigmoid&quot;</span>
-<span class="p">)</span>
-</code></pre></div>
+
 
     <div class="doc doc-contents first">
             <p class="doc doc-class-bases">
@@ -3724,44 +4958,83 @@ <h2 id="pytorch_widedeep.losses.FocalR_MSELoss" class="doc doc-heading">
 
 
 <p><span class="doc-section-title">Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>beta</code></b>
-              (<code>float</code>, default:
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+          <th>Default</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>beta</code></td>
+            <td>
+                  <code>float</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Focal Loss <code>beta</code> parameter in their implementation</p>
+              </div>
+            </td>
+            <td>
                   <code>0.2</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Focal Loss <code>beta</code> parameter in their implementation</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>gamma</code></b>
-              (<code>float</code>, default:
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>gamma</code></td>
+            <td>
+                  <code>float</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Focal Loss <code>gamma</code> parameter</p>
+              </div>
+            </td>
+            <td>
                   <code>1.0</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Focal Loss <code>gamma</code> parameter</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>activation_fn</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[sigmoid, tanh]</code>, default:
-                  <code>&#39;sigmoid&#39;</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Activation function to be used during the computation of the loss.
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>activation_fn</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[sigmoid, tanh]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Activation function to be used during the computation of the loss.
 Possible values are <em>'sigmoid'</em> and <em>'tanh'</em>. See the original
 publication for details.</p>
-          </div>
-        </li>
-    </ul>
-
-                  <details class="quote">
-                    <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
-                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">580</span>
+              </div>
+            </td>
+            <td>
+                  <code>&#39;sigmoid&#39;</code>
+            </td>
+          </tr>
+      </tbody>
+    </table>
+
+              <details class="quote">
+                <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
+                <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">563</span>
+<span class="normal">564</span>
+<span class="normal">565</span>
+<span class="normal">566</span>
+<span class="normal">567</span>
+<span class="normal">568</span>
+<span class="normal">569</span>
+<span class="normal">570</span>
+<span class="normal">571</span>
+<span class="normal">572</span>
+<span class="normal">573</span>
+<span class="normal">574</span>
+<span class="normal">575</span>
+<span class="normal">576</span>
+<span class="normal">577</span>
+<span class="normal">578</span>
+<span class="normal">579</span>
+<span class="normal">580</span>
 <span class="normal">581</span>
 <span class="normal">582</span>
 <span class="normal">583</span>
@@ -3770,18 +5043,107 @@ <h2 id="pytorch_widedeep.losses.FocalR_MSELoss" class="doc doc-heading">
 <span class="normal">586</span>
 <span class="normal">587</span>
 <span class="normal">588</span>
-<span class="normal">589</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
-    <span class="bp">self</span><span class="p">,</span>
-    <span class="n">beta</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.2</span><span class="p">,</span>
-    <span class="n">gamma</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">1.0</span><span class="p">,</span>
-    <span class="n">activation_fn</span><span class="p">:</span> <span class="n">Literal</span><span class="p">[</span><span class="s2">&quot;sigmoid&quot;</span><span class="p">,</span> <span class="s2">&quot;tanh&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="s2">&quot;sigmoid&quot;</span><span class="p">,</span>
-<span class="p">):</span>
-    <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">beta</span> <span class="o">=</span> <span class="n">beta</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">gamma</span> <span class="o">=</span> <span class="n">gamma</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">activation_fn</span> <span class="o">=</span> <span class="n">activation_fn</span>
+<span class="normal">589</span>
+<span class="normal">590</span>
+<span class="normal">591</span>
+<span class="normal">592</span>
+<span class="normal">593</span>
+<span class="normal">594</span>
+<span class="normal">595</span>
+<span class="normal">596</span>
+<span class="normal">597</span>
+<span class="normal">598</span>
+<span class="normal">599</span>
+<span class="normal">600</span>
+<span class="normal">601</span>
+<span class="normal">602</span>
+<span class="normal">603</span>
+<span class="normal">604</span>
+<span class="normal">605</span>
+<span class="normal">606</span>
+<span class="normal">607</span>
+<span class="normal">608</span>
+<span class="normal">609</span>
+<span class="normal">610</span>
+<span class="normal">611</span>
+<span class="normal">612</span>
+<span class="normal">613</span>
+<span class="normal">614</span>
+<span class="normal">615</span>
+<span class="normal">616</span>
+<span class="normal">617</span>
+<span class="normal">618</span>
+<span class="normal">619</span>
+<span class="normal">620</span>
+<span class="normal">621</span>
+<span class="normal">622</span>
+<span class="normal">623</span>
+<span class="normal">624</span>
+<span class="normal">625</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">class</span> <span class="nc">FocalR_MSELoss</span><span class="p">(</span><span class="n">nn</span><span class="o">.</span><span class="n">Module</span><span class="p">):</span>
+<span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Focal-R MSE loss</span>
+
+<span class="sd">    Based on [Delving into Deep Imbalanced Regression](https://arxiv.org/abs/2102.09554).</span>
+
+<span class="sd">    Parameters</span>
+<span class="sd">    ----------</span>
+<span class="sd">    beta: float</span>
+<span class="sd">        Focal Loss `beta` parameter in their implementation</span>
+<span class="sd">    gamma: float</span>
+<span class="sd">        Focal Loss `gamma` parameter</span>
+<span class="sd">    activation_fn: str, default = &quot;sigmoid&quot;</span>
+<span class="sd">        Activation function to be used during the computation of the loss.</span>
+<span class="sd">        Possible values are _&#39;sigmoid&#39;_ and _&#39;tanh&#39;_. See the original</span>
+<span class="sd">        publication for details.</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+
+    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span>
+        <span class="n">beta</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.2</span><span class="p">,</span>
+        <span class="n">gamma</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">1.0</span><span class="p">,</span>
+        <span class="n">activation_fn</span><span class="p">:</span> <span class="n">Literal</span><span class="p">[</span><span class="s2">&quot;sigmoid&quot;</span><span class="p">,</span> <span class="s2">&quot;tanh&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="s2">&quot;sigmoid&quot;</span><span class="p">,</span>
+    <span class="p">):</span>
+        <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">beta</span> <span class="o">=</span> <span class="n">beta</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">gamma</span> <span class="o">=</span> <span class="n">gamma</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">activation_fn</span> <span class="o">=</span> <span class="n">activation_fn</span>
+
+    <span class="k">def</span> <span class="nf">forward</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span>
+        <span class="nb">input</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">,</span>
+        <span class="n">target</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">,</span>
+    <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tensor</span><span class="p">:</span>
+<span class="w">        </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Parameters</span>
+<span class="sd">        ----------</span>
+<span class="sd">        input: Tensor</span>
+<span class="sd">            Input tensor with predictions (not probabilities)</span>
+<span class="sd">        target: Tensor</span>
+<span class="sd">            Target tensor with the actual classes</span>
+
+<span class="sd">        Examples</span>
+<span class="sd">        --------</span>
+<span class="sd">        &gt;&gt;&gt; import torch</span>
+<span class="sd">        &gt;&gt;&gt;</span>
+<span class="sd">        &gt;&gt;&gt; from pytorch_widedeep.losses import FocalR_MSELoss</span>
+<span class="sd">        &gt;&gt;&gt;</span>
+<span class="sd">        &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)</span>
+<span class="sd">        &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)</span>
+<span class="sd">        &gt;&gt;&gt; loss = FocalR_MSELoss()(input, target)</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">loss</span> <span class="o">=</span> <span class="p">(</span><span class="nb">input</span> <span class="o">-</span> <span class="n">target</span><span class="p">)</span> <span class="o">**</span> <span class="mi">2</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">activation_fn</span> <span class="o">==</span> <span class="s2">&quot;tanh&quot;</span><span class="p">:</span>
+            <span class="n">loss</span> <span class="o">*=</span> <span class="p">(</span><span class="n">torch</span><span class="o">.</span><span class="n">tanh</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">beta</span> <span class="o">*</span> <span class="n">torch</span><span class="o">.</span><span class="n">abs</span><span class="p">(</span><span class="nb">input</span> <span class="o">-</span> <span class="n">target</span><span class="p">)))</span> <span class="o">**</span> <span class="bp">self</span><span class="o">.</span><span class="n">gamma</span>
+        <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">activation_fn</span> <span class="o">==</span> <span class="s2">&quot;sigmoid&quot;</span><span class="p">:</span>
+            <span class="n">loss</span> <span class="o">*=</span> <span class="p">(</span>
+                <span class="mi">2</span> <span class="o">*</span> <span class="n">torch</span><span class="o">.</span><span class="n">sigmoid</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">beta</span> <span class="o">*</span> <span class="n">torch</span><span class="o">.</span><span class="n">abs</span><span class="p">((</span><span class="nb">input</span> <span class="o">-</span> <span class="n">target</span><span class="p">)</span> <span class="o">**</span> <span class="mi">2</span><span class="p">))</span> <span class="o">-</span> <span class="mi">1</span>
+            <span class="p">)</span> <span class="o">**</span> <span class="bp">self</span><span class="o">.</span><span class="n">gamma</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="ne">ValueError</span><span class="p">(</span>
+                <span class="s2">&quot;Incorrect activation function value - must be in [&#39;sigmoid&#39;, &#39;tanh&#39;]&quot;</span>
+            <span class="p">)</span>
+        <span class="k">return</span> <span class="n">torch</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="n">loss</span><span class="p">)</span>
 </code></pre></div></td></tr></table></div>
-                  </details>
+              </details>
 
 
 
@@ -3811,24 +5173,46 @@ <h3 id="pytorch_widedeep.losses.FocalR_MSELoss.forward" class="doc doc-heading">
 
 
 <p><span class="doc-section-title">Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>input</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>)
-          –
-          <div class="doc-md-description">
-            <p>Input tensor with predictions (not probabilities)</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>target</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>)
-          –
-          <div class="doc-md-description">
-            <p>Target tensor with the actual classes</p>
-          </div>
-        </li>
-    </ul>
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+          <th>Default</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>input</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Input tensor with predictions (not probabilities)</p>
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>target</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Target tensor with the actual classes</p>
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
 
 <p><span class="doc-section-title">Examples:</span></p>
@@ -3935,10 +5319,7 @@ <h2 id="pytorch_widedeep.losses.FocalR_RMSELoss" class="doc doc-heading">
 
 
 <a href="#pytorch_widedeep.losses.FocalR_RMSELoss" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="nf">FocalR_RMSELoss</span><span class="p">(</span>
-    <span class="n">beta</span><span class="o">=</span><span class="mf">0.2</span><span class="p">,</span> <span class="n">gamma</span><span class="o">=</span><span class="mf">1.0</span><span class="p">,</span> <span class="n">activation_fn</span><span class="o">=</span><span class="s2">&quot;sigmoid&quot;</span>
-<span class="p">)</span>
-</code></pre></div>
+
 
     <div class="doc doc-contents first">
             <p class="doc doc-class-bases">
@@ -3950,44 +5331,83 @@ <h2 id="pytorch_widedeep.losses.FocalR_RMSELoss" class="doc doc-heading">
 
 
 <p><span class="doc-section-title">Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>beta</code></b>
-              (<code>float</code>, default:
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+          <th>Default</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>beta</code></td>
+            <td>
+                  <code>float</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Focal Loss <code>beta</code> parameter in their implementation</p>
+              </div>
+            </td>
+            <td>
                   <code>0.2</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Focal Loss <code>beta</code> parameter in their implementation</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>gamma</code></b>
-              (<code>float</code>, default:
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>gamma</code></td>
+            <td>
+                  <code>float</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Focal Loss <code>gamma</code> parameter</p>
+              </div>
+            </td>
+            <td>
                   <code>1.0</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Focal Loss <code>gamma</code> parameter</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>activation_fn</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[sigmoid, tanh]</code>, default:
-                  <code>&#39;sigmoid&#39;</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Activation function to be used during the computation of the loss.
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>activation_fn</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[sigmoid, tanh]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Activation function to be used during the computation of the loss.
 Possible values are <em>'sigmoid'</em> and <em>'tanh'</em>. See the original
 publication for details.</p>
-          </div>
-        </li>
-    </ul>
-
-                  <details class="quote">
-                    <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
-                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">645</span>
+              </div>
+            </td>
+            <td>
+                  <code>&#39;sigmoid&#39;</code>
+            </td>
+          </tr>
+      </tbody>
+    </table>
+
+              <details class="quote">
+                <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
+                <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">628</span>
+<span class="normal">629</span>
+<span class="normal">630</span>
+<span class="normal">631</span>
+<span class="normal">632</span>
+<span class="normal">633</span>
+<span class="normal">634</span>
+<span class="normal">635</span>
+<span class="normal">636</span>
+<span class="normal">637</span>
+<span class="normal">638</span>
+<span class="normal">639</span>
+<span class="normal">640</span>
+<span class="normal">641</span>
+<span class="normal">642</span>
+<span class="normal">643</span>
+<span class="normal">644</span>
+<span class="normal">645</span>
 <span class="normal">646</span>
 <span class="normal">647</span>
 <span class="normal">648</span>
@@ -3996,18 +5416,107 @@ <h2 id="pytorch_widedeep.losses.FocalR_RMSELoss" class="doc doc-heading">
 <span class="normal">651</span>
 <span class="normal">652</span>
 <span class="normal">653</span>
-<span class="normal">654</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
-    <span class="bp">self</span><span class="p">,</span>
-    <span class="n">beta</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.2</span><span class="p">,</span>
-    <span class="n">gamma</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">1.0</span><span class="p">,</span>
-    <span class="n">activation_fn</span><span class="p">:</span> <span class="n">Literal</span><span class="p">[</span><span class="s2">&quot;sigmoid&quot;</span><span class="p">,</span> <span class="s2">&quot;tanh&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="s2">&quot;sigmoid&quot;</span><span class="p">,</span>
-<span class="p">):</span>
-    <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">beta</span> <span class="o">=</span> <span class="n">beta</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">gamma</span> <span class="o">=</span> <span class="n">gamma</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">activation_fn</span> <span class="o">=</span> <span class="n">activation_fn</span>
+<span class="normal">654</span>
+<span class="normal">655</span>
+<span class="normal">656</span>
+<span class="normal">657</span>
+<span class="normal">658</span>
+<span class="normal">659</span>
+<span class="normal">660</span>
+<span class="normal">661</span>
+<span class="normal">662</span>
+<span class="normal">663</span>
+<span class="normal">664</span>
+<span class="normal">665</span>
+<span class="normal">666</span>
+<span class="normal">667</span>
+<span class="normal">668</span>
+<span class="normal">669</span>
+<span class="normal">670</span>
+<span class="normal">671</span>
+<span class="normal">672</span>
+<span class="normal">673</span>
+<span class="normal">674</span>
+<span class="normal">675</span>
+<span class="normal">676</span>
+<span class="normal">677</span>
+<span class="normal">678</span>
+<span class="normal">679</span>
+<span class="normal">680</span>
+<span class="normal">681</span>
+<span class="normal">682</span>
+<span class="normal">683</span>
+<span class="normal">684</span>
+<span class="normal">685</span>
+<span class="normal">686</span>
+<span class="normal">687</span>
+<span class="normal">688</span>
+<span class="normal">689</span>
+<span class="normal">690</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">class</span> <span class="nc">FocalR_RMSELoss</span><span class="p">(</span><span class="n">nn</span><span class="o">.</span><span class="n">Module</span><span class="p">):</span>
+<span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Focal-R RMSE loss</span>
+
+<span class="sd">    Based on [Delving into Deep Imbalanced Regression](https://arxiv.org/abs/2102.09554).</span>
+
+<span class="sd">    Parameters</span>
+<span class="sd">    ----------</span>
+<span class="sd">    beta: float</span>
+<span class="sd">        Focal Loss `beta` parameter in their implementation</span>
+<span class="sd">    gamma: float</span>
+<span class="sd">        Focal Loss `gamma` parameter</span>
+<span class="sd">    activation_fn: str, default = &quot;sigmoid&quot;</span>
+<span class="sd">        Activation function to be used during the computation of the loss.</span>
+<span class="sd">        Possible values are _&#39;sigmoid&#39;_ and _&#39;tanh&#39;_. See the original</span>
+<span class="sd">        publication for details.</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+
+    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span>
+        <span class="n">beta</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.2</span><span class="p">,</span>
+        <span class="n">gamma</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">1.0</span><span class="p">,</span>
+        <span class="n">activation_fn</span><span class="p">:</span> <span class="n">Literal</span><span class="p">[</span><span class="s2">&quot;sigmoid&quot;</span><span class="p">,</span> <span class="s2">&quot;tanh&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="s2">&quot;sigmoid&quot;</span><span class="p">,</span>
+    <span class="p">):</span>
+        <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">beta</span> <span class="o">=</span> <span class="n">beta</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">gamma</span> <span class="o">=</span> <span class="n">gamma</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">activation_fn</span> <span class="o">=</span> <span class="n">activation_fn</span>
+
+    <span class="k">def</span> <span class="nf">forward</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span>
+        <span class="nb">input</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">,</span>
+        <span class="n">target</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">,</span>
+    <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tensor</span><span class="p">:</span>
+<span class="w">        </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Parameters</span>
+<span class="sd">        ----------</span>
+<span class="sd">        input: Tensor</span>
+<span class="sd">            Input tensor with predictions (not probabilities)</span>
+<span class="sd">        target: Tensor</span>
+<span class="sd">            Target tensor with the actual classes</span>
+
+<span class="sd">        Examples</span>
+<span class="sd">        --------</span>
+<span class="sd">        &gt;&gt;&gt; import torch</span>
+<span class="sd">        &gt;&gt;&gt;</span>
+<span class="sd">        &gt;&gt;&gt; from pytorch_widedeep.losses import FocalR_RMSELoss</span>
+<span class="sd">        &gt;&gt;&gt;</span>
+<span class="sd">        &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)</span>
+<span class="sd">        &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)</span>
+<span class="sd">        &gt;&gt;&gt; loss = FocalR_RMSELoss()(input, target)</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">loss</span> <span class="o">=</span> <span class="p">(</span><span class="nb">input</span> <span class="o">-</span> <span class="n">target</span><span class="p">)</span> <span class="o">**</span> <span class="mi">2</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">activation_fn</span> <span class="o">==</span> <span class="s2">&quot;tanh&quot;</span><span class="p">:</span>
+            <span class="n">loss</span> <span class="o">*=</span> <span class="p">(</span><span class="n">torch</span><span class="o">.</span><span class="n">tanh</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">beta</span> <span class="o">*</span> <span class="n">torch</span><span class="o">.</span><span class="n">abs</span><span class="p">(</span><span class="nb">input</span> <span class="o">-</span> <span class="n">target</span><span class="p">)))</span> <span class="o">**</span> <span class="bp">self</span><span class="o">.</span><span class="n">gamma</span>
+        <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">activation_fn</span> <span class="o">==</span> <span class="s2">&quot;sigmoid&quot;</span><span class="p">:</span>
+            <span class="n">loss</span> <span class="o">*=</span> <span class="p">(</span>
+                <span class="mi">2</span> <span class="o">*</span> <span class="n">torch</span><span class="o">.</span><span class="n">sigmoid</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">beta</span> <span class="o">*</span> <span class="n">torch</span><span class="o">.</span><span class="n">abs</span><span class="p">((</span><span class="nb">input</span> <span class="o">-</span> <span class="n">target</span><span class="p">)</span> <span class="o">**</span> <span class="mi">2</span><span class="p">))</span> <span class="o">-</span> <span class="mi">1</span>
+            <span class="p">)</span> <span class="o">**</span> <span class="bp">self</span><span class="o">.</span><span class="n">gamma</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="ne">ValueError</span><span class="p">(</span>
+                <span class="s2">&quot;Incorrect activation function value - must be in [&#39;sigmoid&#39;, &#39;tanh&#39;]&quot;</span>
+            <span class="p">)</span>
+        <span class="k">return</span> <span class="n">torch</span><span class="o">.</span><span class="n">sqrt</span><span class="p">(</span><span class="n">torch</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="n">loss</span><span class="p">))</span>
 </code></pre></div></td></tr></table></div>
-                  </details>
+              </details>
 
 
 
@@ -4037,24 +5546,46 @@ <h3 id="pytorch_widedeep.losses.FocalR_RMSELoss.forward" class="doc doc-heading"
 
 
 <p><span class="doc-section-title">Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>input</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>)
-          –
-          <div class="doc-md-description">
-            <p>Input tensor with predictions (not probabilities)</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>target</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>)
-          –
-          <div class="doc-md-description">
-            <p>Target tensor with the actual classes</p>
-          </div>
-        </li>
-    </ul>
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+          <th>Default</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>input</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Input tensor with predictions (not probabilities)</p>
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>target</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Target tensor with the actual classes</p>
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
 
 <p><span class="doc-section-title">Examples:</span></p>
@@ -4161,8 +5692,7 @@ <h2 id="pytorch_widedeep.losses.HuberLoss" class="doc doc-heading">
 
 
 <a href="#pytorch_widedeep.losses.HuberLoss" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="nf">HuberLoss</span><span class="p">(</span><span class="n">beta</span><span class="o">=</span><span class="mf">0.2</span><span class="p">)</span>
-</code></pre></div>
+
 
     <div class="doc doc-contents first">
             <p class="doc doc-class-bases">
@@ -4172,15 +5702,87 @@ <h2 id="pytorch_widedeep.losses.HuberLoss" class="doc doc-heading">
         <p>Hubbler Loss</p>
 <p>Based on <a href="https://arxiv.org/abs/2102.09554">Delving into Deep Imbalanced Regression</a>.</p>
 
-                  <details class="quote">
-                    <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
-                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">699</span>
+              <details class="quote">
+                <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
+                <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">693</span>
+<span class="normal">694</span>
+<span class="normal">695</span>
+<span class="normal">696</span>
+<span class="normal">697</span>
+<span class="normal">698</span>
+<span class="normal">699</span>
 <span class="normal">700</span>
-<span class="normal">701</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">beta</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.2</span><span class="p">):</span>
-    <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">beta</span> <span class="o">=</span> <span class="n">beta</span>
+<span class="normal">701</span>
+<span class="normal">702</span>
+<span class="normal">703</span>
+<span class="normal">704</span>
+<span class="normal">705</span>
+<span class="normal">706</span>
+<span class="normal">707</span>
+<span class="normal">708</span>
+<span class="normal">709</span>
+<span class="normal">710</span>
+<span class="normal">711</span>
+<span class="normal">712</span>
+<span class="normal">713</span>
+<span class="normal">714</span>
+<span class="normal">715</span>
+<span class="normal">716</span>
+<span class="normal">717</span>
+<span class="normal">718</span>
+<span class="normal">719</span>
+<span class="normal">720</span>
+<span class="normal">721</span>
+<span class="normal">722</span>
+<span class="normal">723</span>
+<span class="normal">724</span>
+<span class="normal">725</span>
+<span class="normal">726</span>
+<span class="normal">727</span>
+<span class="normal">728</span>
+<span class="normal">729</span>
+<span class="normal">730</span>
+<span class="normal">731</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">class</span> <span class="nc">HuberLoss</span><span class="p">(</span><span class="n">nn</span><span class="o">.</span><span class="n">Module</span><span class="p">):</span>
+<span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Hubbler Loss</span>
+
+<span class="sd">    Based on [Delving into Deep Imbalanced Regression](https://arxiv.org/abs/2102.09554).</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+
+    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">beta</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.2</span><span class="p">):</span>
+        <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">beta</span> <span class="o">=</span> <span class="n">beta</span>
+
+    <span class="k">def</span> <span class="nf">forward</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span>
+        <span class="nb">input</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">,</span>
+        <span class="n">target</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">,</span>
+    <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tensor</span><span class="p">:</span>
+<span class="w">        </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Parameters</span>
+<span class="sd">        ----------</span>
+<span class="sd">        input: Tensor</span>
+<span class="sd">            Input tensor with predictions (not probabilities)</span>
+<span class="sd">        target: Tensor</span>
+<span class="sd">            Target tensor with the actual classes</span>
+
+<span class="sd">        Examples</span>
+<span class="sd">        --------</span>
+<span class="sd">        &gt;&gt;&gt; import torch</span>
+<span class="sd">        &gt;&gt;&gt;</span>
+<span class="sd">        &gt;&gt;&gt; from pytorch_widedeep.losses import HuberLoss</span>
+<span class="sd">        &gt;&gt;&gt;</span>
+<span class="sd">        &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)</span>
+<span class="sd">        &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)</span>
+<span class="sd">        &gt;&gt;&gt; loss = HuberLoss()(input, target)</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">l1_loss</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">abs</span><span class="p">(</span><span class="nb">input</span> <span class="o">-</span> <span class="n">target</span><span class="p">)</span>
+        <span class="n">cond</span> <span class="o">=</span> <span class="n">l1_loss</span> <span class="o">&lt;</span> <span class="bp">self</span><span class="o">.</span><span class="n">beta</span>
+        <span class="n">loss</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">where</span><span class="p">(</span>
+            <span class="n">cond</span><span class="p">,</span> <span class="mf">0.5</span> <span class="o">*</span> <span class="n">l1_loss</span><span class="o">**</span><span class="mi">2</span> <span class="o">/</span> <span class="bp">self</span><span class="o">.</span><span class="n">beta</span><span class="p">,</span> <span class="n">l1_loss</span> <span class="o">-</span> <span class="mf">0.5</span> <span class="o">*</span> <span class="bp">self</span><span class="o">.</span><span class="n">beta</span>
+        <span class="p">)</span>
+        <span class="k">return</span> <span class="n">torch</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="n">loss</span><span class="p">)</span>
 </code></pre></div></td></tr></table></div>
-                  </details>
+              </details>
 
 
 
@@ -4210,24 +5812,46 @@ <h3 id="pytorch_widedeep.losses.HuberLoss.forward" class="doc doc-heading">
 
 
 <p><span class="doc-section-title">Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>input</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>)
-          –
-          <div class="doc-md-description">
-            <p>Input tensor with predictions (not probabilities)</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>target</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>)
-          –
-          <div class="doc-md-description">
-            <p>Target tensor with the actual classes</p>
-          </div>
-        </li>
-    </ul>
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+          <th>Default</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>input</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Input tensor with predictions (not probabilities)</p>
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>target</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Target tensor with the actual classes</p>
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
 
 <p><span class="doc-section-title">Examples:</span></p>
@@ -4322,8 +5946,7 @@ <h2 id="pytorch_widedeep.losses.InfoNCELoss" class="doc doc-heading">
 
 
 <a href="#pytorch_widedeep.losses.InfoNCELoss" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="nf">InfoNCELoss</span><span class="p">(</span><span class="n">temperature</span><span class="o">=</span><span class="mf">0.1</span><span class="p">,</span> <span class="n">reduction</span><span class="o">=</span><span class="s1">&#39;mean&#39;</span><span class="p">)</span>
-</code></pre></div>
+
 
     <div class="doc doc-contents first">
             <p class="doc doc-class-bases">
@@ -4342,42 +5965,170 @@ <h2 id="pytorch_widedeep.losses.InfoNCELoss" class="doc doc-heading">
 
 
 <p><span class="doc-section-title">Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>temperature</code></b>
-              (<code>float</code>, default:
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+          <th>Default</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>temperature</code></td>
+            <td>
+                  <code>float</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>The logits are divided by the temperature before computing the loss value</p>
+              </div>
+            </td>
+            <td>
                   <code>0.1</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>The logits are divided by the temperature before computing the loss value</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>reduction</code></b>
-              (<code>str</code>, default:
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>reduction</code></td>
+            <td>
+                  <code>str</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Loss reduction method</p>
+              </div>
+            </td>
+            <td>
                   <code>&#39;mean&#39;</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Loss reduction method</p>
-          </div>
-        </li>
-    </ul>
-
-                  <details class="quote">
-                    <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
-                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">756</span>
+            </td>
+          </tr>
+      </tbody>
+    </table>
+
+              <details class="quote">
+                <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
+                <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">734</span>
+<span class="normal">735</span>
+<span class="normal">736</span>
+<span class="normal">737</span>
+<span class="normal">738</span>
+<span class="normal">739</span>
+<span class="normal">740</span>
+<span class="normal">741</span>
+<span class="normal">742</span>
+<span class="normal">743</span>
+<span class="normal">744</span>
+<span class="normal">745</span>
+<span class="normal">746</span>
+<span class="normal">747</span>
+<span class="normal">748</span>
+<span class="normal">749</span>
+<span class="normal">750</span>
+<span class="normal">751</span>
+<span class="normal">752</span>
+<span class="normal">753</span>
+<span class="normal">754</span>
+<span class="normal">755</span>
+<span class="normal">756</span>
 <span class="normal">757</span>
 <span class="normal">758</span>
 <span class="normal">759</span>
-<span class="normal">760</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">temperature</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.1</span><span class="p">,</span> <span class="n">reduction</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;mean&quot;</span><span class="p">):</span>
-    <span class="nb">super</span><span class="p">(</span><span class="n">InfoNCELoss</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
+<span class="normal">760</span>
+<span class="normal">761</span>
+<span class="normal">762</span>
+<span class="normal">763</span>
+<span class="normal">764</span>
+<span class="normal">765</span>
+<span class="normal">766</span>
+<span class="normal">767</span>
+<span class="normal">768</span>
+<span class="normal">769</span>
+<span class="normal">770</span>
+<span class="normal">771</span>
+<span class="normal">772</span>
+<span class="normal">773</span>
+<span class="normal">774</span>
+<span class="normal">775</span>
+<span class="normal">776</span>
+<span class="normal">777</span>
+<span class="normal">778</span>
+<span class="normal">779</span>
+<span class="normal">780</span>
+<span class="normal">781</span>
+<span class="normal">782</span>
+<span class="normal">783</span>
+<span class="normal">784</span>
+<span class="normal">785</span>
+<span class="normal">786</span>
+<span class="normal">787</span>
+<span class="normal">788</span>
+<span class="normal">789</span>
+<span class="normal">790</span>
+<span class="normal">791</span>
+<span class="normal">792</span>
+<span class="normal">793</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">class</span> <span class="nc">InfoNCELoss</span><span class="p">(</span><span class="n">nn</span><span class="o">.</span><span class="n">Module</span><span class="p">):</span>
+<span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;InfoNCE Loss. Loss applied during the Contrastive Denoising Self</span>
+<span class="sd">    Supervised Pre-training routine available in this library</span>
+
+<span class="sd">    :information_source: **NOTE**: This loss is in principle not exposed to</span>
+<span class="sd">     the user, as it is used internally in the library, but it is included</span>
+<span class="sd">     here for completion.</span>
+
+<span class="sd">    See [SAINT: Improved Neural Networks for Tabular Data via Row Attention</span>
+<span class="sd">    and Contrastive Pre-Training](https://arxiv.org/abs/2106.01342) and</span>
+<span class="sd">    references therein</span>
+
+<span class="sd">    Partially inspired by the code in this [repo](https://github.com/RElbers/info-nce-pytorch)</span>
+
+<span class="sd">    Parameters</span>
+<span class="sd">    ----------</span>
+<span class="sd">    temperature: float, default = 0.1</span>
+<span class="sd">        The logits are divided by the temperature before computing the loss value</span>
+<span class="sd">    reduction: str, default = &quot;mean&quot;</span>
+<span class="sd">        Loss reduction method</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+
+    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">temperature</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.1</span><span class="p">,</span> <span class="n">reduction</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;mean&quot;</span><span class="p">):</span>
+        <span class="nb">super</span><span class="p">(</span><span class="n">InfoNCELoss</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
 
-    <span class="bp">self</span><span class="o">.</span><span class="n">temperature</span> <span class="o">=</span> <span class="n">temperature</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">reduction</span> <span class="o">=</span> <span class="n">reduction</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">temperature</span> <span class="o">=</span> <span class="n">temperature</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">reduction</span> <span class="o">=</span> <span class="n">reduction</span>
+
+    <span class="k">def</span> <span class="nf">forward</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">g_projs</span><span class="p">:</span> <span class="n">Tuple</span><span class="p">[</span><span class="n">Tensor</span><span class="p">,</span> <span class="n">Tensor</span><span class="p">])</span> <span class="o">-&gt;</span> <span class="n">Tensor</span><span class="p">:</span>
+<span class="w">        </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Parameters</span>
+<span class="sd">        ----------</span>
+<span class="sd">        g_projs: Tuple</span>
+<span class="sd">            Tuple with the two tensors corresponding to the output of the two</span>
+<span class="sd">            projection heads, as described &#39;SAINT: Improved Neural Networks</span>
+<span class="sd">            for Tabular Data via Row Attention and Contrastive Pre-Training&#39;.</span>
+
+<span class="sd">        Examples</span>
+<span class="sd">        --------</span>
+<span class="sd">        &gt;&gt;&gt; import torch</span>
+<span class="sd">        &gt;&gt;&gt; from pytorch_widedeep.losses import InfoNCELoss</span>
+<span class="sd">        &gt;&gt;&gt; g_projs = (torch.rand(3, 5, 16), torch.rand(3, 5, 16))</span>
+<span class="sd">        &gt;&gt;&gt; loss = InfoNCELoss()</span>
+<span class="sd">        &gt;&gt;&gt; res = loss(g_projs)</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">z</span><span class="p">,</span> <span class="n">z_</span> <span class="o">=</span> <span class="n">g_projs</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="n">g_projs</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span>
+
+        <span class="n">norm_z</span> <span class="o">=</span> <span class="n">F</span><span class="o">.</span><span class="n">normalize</span><span class="p">(</span><span class="n">z</span><span class="p">,</span> <span class="n">dim</span><span class="o">=-</span><span class="mi">1</span><span class="p">)</span><span class="o">.</span><span class="n">flatten</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span>
+        <span class="n">norm_z_</span> <span class="o">=</span> <span class="n">F</span><span class="o">.</span><span class="n">normalize</span><span class="p">(</span><span class="n">z_</span><span class="p">,</span> <span class="n">dim</span><span class="o">=-</span><span class="mi">1</span><span class="p">)</span><span class="o">.</span><span class="n">flatten</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span>
+
+        <span class="n">logits</span> <span class="o">=</span> <span class="p">(</span><span class="n">norm_z</span> <span class="o">@</span> <span class="n">norm_z_</span><span class="o">.</span><span class="n">t</span><span class="p">())</span> <span class="o">/</span> <span class="bp">self</span><span class="o">.</span><span class="n">temperature</span>
+        <span class="n">logits_</span> <span class="o">=</span> <span class="p">(</span><span class="n">norm_z_</span> <span class="o">@</span> <span class="n">norm_z</span><span class="o">.</span><span class="n">t</span><span class="p">())</span> <span class="o">/</span> <span class="bp">self</span><span class="o">.</span><span class="n">temperature</span>
+
+        <span class="c1"># the target/labels are the entries on the diagonal</span>
+        <span class="n">target</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">arange</span><span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">norm_z</span><span class="p">),</span> <span class="n">device</span><span class="o">=</span><span class="n">norm_z</span><span class="o">.</span><span class="n">device</span><span class="p">)</span>
+
+        <span class="n">loss</span> <span class="o">=</span> <span class="n">F</span><span class="o">.</span><span class="n">cross_entropy</span><span class="p">(</span><span class="n">logits</span><span class="p">,</span> <span class="n">target</span><span class="p">,</span> <span class="n">reduction</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">reduction</span><span class="p">)</span>
+        <span class="n">loss_</span> <span class="o">=</span> <span class="n">F</span><span class="o">.</span><span class="n">cross_entropy</span><span class="p">(</span><span class="n">logits_</span><span class="p">,</span> <span class="n">target</span><span class="p">,</span> <span class="n">reduction</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">reduction</span><span class="p">)</span>
+
+        <span class="k">return</span> <span class="p">(</span><span class="n">loss</span> <span class="o">+</span> <span class="n">loss_</span><span class="p">)</span> <span class="o">/</span> <span class="mf">2.0</span>
 </code></pre></div></td></tr></table></div>
-                  </details>
+              </details>
 
 
 
@@ -4407,18 +6158,34 @@ <h3 id="pytorch_widedeep.losses.InfoNCELoss.forward" class="doc doc-heading">
 
 
 <p><span class="doc-section-title">Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>g_projs</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Tuple">Tuple</span>[<span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span>, <span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span>]</code>)
-          –
-          <div class="doc-md-description">
-            <p>Tuple with the two tensors corresponding to the output of the two
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+          <th>Default</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>g_projs</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Tuple">Tuple</span>[<span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span>, <span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span>]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Tuple with the two tensors corresponding to the output of the two
 projection heads, as described 'SAINT: Improved Neural Networks
 for Tabular Data via Row Attention and Contrastive Pre-Training'.</p>
-          </div>
-        </li>
-    </ul>
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
 
 <p><span class="doc-section-title">Examples:</span></p>
@@ -4517,10 +6284,7 @@ <h2 id="pytorch_widedeep.losses.DenoisingLoss" class="doc doc-heading">
 
 
 <a href="#pytorch_widedeep.losses.DenoisingLoss" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="nf">DenoisingLoss</span><span class="p">(</span>
-    <span class="n">lambda_cat</span><span class="o">=</span><span class="mf">1.0</span><span class="p">,</span> <span class="n">lambda_cont</span><span class="o">=</span><span class="mf">1.0</span><span class="p">,</span> <span class="n">reduction</span><span class="o">=</span><span class="s2">&quot;mean&quot;</span>
-<span class="p">)</span>
-</code></pre></div>
+
 
     <div class="doc doc-contents first">
             <p class="doc doc-class-bases">
@@ -4538,60 +6302,286 @@ <h2 id="pytorch_widedeep.losses.DenoisingLoss" class="doc doc-heading">
 
 
 <p><span class="doc-section-title">Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>lambda_cat</code></b>
-              (<code>float</code>, default:
-                  <code>1.0</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Multiplicative factor that will be applied to loss associated to the
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+          <th>Default</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>lambda_cat</code></td>
+            <td>
+                  <code>float</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Multiplicative factor that will be applied to loss associated to the
 categorical features</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>lambda_cont</code></b>
-              (<code>float</code>, default:
+              </div>
+            </td>
+            <td>
                   <code>1.0</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Multiplicative factor that will be applied to loss associated to the
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>lambda_cont</code></td>
+            <td>
+                  <code>float</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Multiplicative factor that will be applied to loss associated to the
 continuous features</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>reduction</code></b>
-              (<code>str</code>, default:
+              </div>
+            </td>
+            <td>
+                  <code>1.0</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>reduction</code></td>
+            <td>
+                  <code>str</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Loss reduction method</p>
+              </div>
+            </td>
+            <td>
                   <code>&#39;mean&#39;</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Loss reduction method</p>
-          </div>
-        </li>
-    </ul>
-
-                  <details class="quote">
-                    <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
-                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">820</span>
+            </td>
+          </tr>
+      </tbody>
+    </table>
+
+              <details class="quote">
+                <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
+                <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">796</span>
+<span class="normal">797</span>
+<span class="normal">798</span>
+<span class="normal">799</span>
+<span class="normal">800</span>
+<span class="normal">801</span>
+<span class="normal">802</span>
+<span class="normal">803</span>
+<span class="normal">804</span>
+<span class="normal">805</span>
+<span class="normal">806</span>
+<span class="normal">807</span>
+<span class="normal">808</span>
+<span class="normal">809</span>
+<span class="normal">810</span>
+<span class="normal">811</span>
+<span class="normal">812</span>
+<span class="normal">813</span>
+<span class="normal">814</span>
+<span class="normal">815</span>
+<span class="normal">816</span>
+<span class="normal">817</span>
+<span class="normal">818</span>
+<span class="normal">819</span>
+<span class="normal">820</span>
 <span class="normal">821</span>
 <span class="normal">822</span>
 <span class="normal">823</span>
 <span class="normal">824</span>
 <span class="normal">825</span>
 <span class="normal">826</span>
-<span class="normal">827</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
-    <span class="bp">self</span><span class="p">,</span> <span class="n">lambda_cat</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">1.0</span><span class="p">,</span> <span class="n">lambda_cont</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">1.0</span><span class="p">,</span> <span class="n">reduction</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;mean&quot;</span>
-<span class="p">):</span>
-    <span class="nb">super</span><span class="p">(</span><span class="n">DenoisingLoss</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
-
-    <span class="bp">self</span><span class="o">.</span><span class="n">lambda_cat</span> <span class="o">=</span> <span class="n">lambda_cat</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">lambda_cont</span> <span class="o">=</span> <span class="n">lambda_cont</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">reduction</span> <span class="o">=</span> <span class="n">reduction</span>
-</code></pre></div></td></tr></table></div>
-                  </details>
+<span class="normal">827</span>
+<span class="normal">828</span>
+<span class="normal">829</span>
+<span class="normal">830</span>
+<span class="normal">831</span>
+<span class="normal">832</span>
+<span class="normal">833</span>
+<span class="normal">834</span>
+<span class="normal">835</span>
+<span class="normal">836</span>
+<span class="normal">837</span>
+<span class="normal">838</span>
+<span class="normal">839</span>
+<span class="normal">840</span>
+<span class="normal">841</span>
+<span class="normal">842</span>
+<span class="normal">843</span>
+<span class="normal">844</span>
+<span class="normal">845</span>
+<span class="normal">846</span>
+<span class="normal">847</span>
+<span class="normal">848</span>
+<span class="normal">849</span>
+<span class="normal">850</span>
+<span class="normal">851</span>
+<span class="normal">852</span>
+<span class="normal">853</span>
+<span class="normal">854</span>
+<span class="normal">855</span>
+<span class="normal">856</span>
+<span class="normal">857</span>
+<span class="normal">858</span>
+<span class="normal">859</span>
+<span class="normal">860</span>
+<span class="normal">861</span>
+<span class="normal">862</span>
+<span class="normal">863</span>
+<span class="normal">864</span>
+<span class="normal">865</span>
+<span class="normal">866</span>
+<span class="normal">867</span>
+<span class="normal">868</span>
+<span class="normal">869</span>
+<span class="normal">870</span>
+<span class="normal">871</span>
+<span class="normal">872</span>
+<span class="normal">873</span>
+<span class="normal">874</span>
+<span class="normal">875</span>
+<span class="normal">876</span>
+<span class="normal">877</span>
+<span class="normal">878</span>
+<span class="normal">879</span>
+<span class="normal">880</span>
+<span class="normal">881</span>
+<span class="normal">882</span>
+<span class="normal">883</span>
+<span class="normal">884</span>
+<span class="normal">885</span>
+<span class="normal">886</span>
+<span class="normal">887</span>
+<span class="normal">888</span>
+<span class="normal">889</span>
+<span class="normal">890</span>
+<span class="normal">891</span>
+<span class="normal">892</span>
+<span class="normal">893</span>
+<span class="normal">894</span>
+<span class="normal">895</span>
+<span class="normal">896</span>
+<span class="normal">897</span>
+<span class="normal">898</span>
+<span class="normal">899</span>
+<span class="normal">900</span>
+<span class="normal">901</span>
+<span class="normal">902</span>
+<span class="normal">903</span>
+<span class="normal">904</span>
+<span class="normal">905</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">class</span> <span class="nc">DenoisingLoss</span><span class="p">(</span><span class="n">nn</span><span class="o">.</span><span class="n">Module</span><span class="p">):</span>
+<span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Denoising Loss. Loss applied during the Contrastive Denoising Self</span>
+<span class="sd">    Supervised Pre-training routine available in this library</span>
+
+<span class="sd">    :information_source: **NOTE**: This loss is in principle not exposed to</span>
+<span class="sd">     the user, as it is used internally in the library, but it is included</span>
+<span class="sd">     here for completion.</span>
+
+<span class="sd">    See [SAINT: Improved Neural Networks for Tabular Data via Row Attention</span>
+<span class="sd">    and Contrastive Pre-Training](https://arxiv.org/abs/2106.01342) and</span>
+<span class="sd">    references therein</span>
+
+<span class="sd">    Parameters</span>
+<span class="sd">    ----------</span>
+<span class="sd">    lambda_cat: float, default = 1.</span>
+<span class="sd">        Multiplicative factor that will be applied to loss associated to the</span>
+<span class="sd">        categorical features</span>
+<span class="sd">    lambda_cont: float, default = 1.</span>
+<span class="sd">        Multiplicative factor that will be applied to loss associated to the</span>
+<span class="sd">        continuous features</span>
+<span class="sd">    reduction: str, default = &quot;mean&quot;</span>
+<span class="sd">        Loss reduction method</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+
+    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span> <span class="n">lambda_cat</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">1.0</span><span class="p">,</span> <span class="n">lambda_cont</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">1.0</span><span class="p">,</span> <span class="n">reduction</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;mean&quot;</span>
+    <span class="p">):</span>
+        <span class="nb">super</span><span class="p">(</span><span class="n">DenoisingLoss</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">lambda_cat</span> <span class="o">=</span> <span class="n">lambda_cat</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">lambda_cont</span> <span class="o">=</span> <span class="n">lambda_cont</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">reduction</span> <span class="o">=</span> <span class="n">reduction</span>
+
+    <span class="k">def</span> <span class="nf">forward</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span>
+        <span class="n">x_cat_and_cat_</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span>
+            <span class="n">Union</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="n">Tuple</span><span class="p">[</span><span class="n">Tensor</span><span class="p">,</span> <span class="n">Tensor</span><span class="p">]],</span> <span class="n">Tuple</span><span class="p">[</span><span class="n">Tensor</span><span class="p">,</span> <span class="n">Tensor</span><span class="p">]]</span>
+        <span class="p">],</span>
+        <span class="n">x_cont_and_cont_</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span>
+            <span class="n">Union</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="n">Tuple</span><span class="p">[</span><span class="n">Tensor</span><span class="p">,</span> <span class="n">Tensor</span><span class="p">]],</span> <span class="n">Tuple</span><span class="p">[</span><span class="n">Tensor</span><span class="p">,</span> <span class="n">Tensor</span><span class="p">]]</span>
+        <span class="p">],</span>
+    <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tensor</span><span class="p">:</span>
+<span class="w">        </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Parameters</span>
+<span class="sd">        ----------</span>
+<span class="sd">        x_cat_and_cat_: tuple of Tensors or lists of tuples</span>
+<span class="sd">            Tuple of tensors containing the raw input features and their</span>
+<span class="sd">            encodings, referred in the SAINT paper as $x$ and $x&#39;&#39;$</span>
+<span class="sd">            respectively. If one denoising MLP is used per categorical</span>
+<span class="sd">            feature `x_cat_and_cat_` will be a list of tuples, one per</span>
+<span class="sd">            categorical feature</span>
+<span class="sd">        x_cont_and_cont_: tuple of Tensors or lists of tuples</span>
+<span class="sd">            same as `x_cat_and_cat_` but for continuous columns</span>
+
+<span class="sd">        Examples</span>
+<span class="sd">        --------</span>
+<span class="sd">        &gt;&gt;&gt; import torch</span>
+<span class="sd">        &gt;&gt;&gt; from pytorch_widedeep.losses import DenoisingLoss</span>
+<span class="sd">        &gt;&gt;&gt; x_cat_and_cat_ = (torch.empty(3).random_(3).long(), torch.randn(3, 3))</span>
+<span class="sd">        &gt;&gt;&gt; x_cont_and_cont_ = (torch.randn(3, 1), torch.randn(3, 1))</span>
+<span class="sd">        &gt;&gt;&gt; loss = DenoisingLoss()</span>
+<span class="sd">        &gt;&gt;&gt; res = loss(x_cat_and_cat_, x_cont_and_cont_)</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+
+        <span class="n">loss_cat</span> <span class="o">=</span> <span class="p">(</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">_compute_cat_loss</span><span class="p">(</span><span class="n">x_cat_and_cat_</span><span class="p">)</span>
+            <span class="k">if</span> <span class="n">x_cat_and_cat_</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span>
+            <span class="k">else</span> <span class="n">torch</span><span class="o">.</span><span class="n">tensor</span><span class="p">(</span><span class="mf">0.0</span><span class="p">)</span>
+        <span class="p">)</span>
+        <span class="n">loss_cont</span> <span class="o">=</span> <span class="p">(</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">_compute_cont_loss</span><span class="p">(</span><span class="n">x_cont_and_cont_</span><span class="p">)</span>
+            <span class="k">if</span> <span class="n">x_cont_and_cont_</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span>
+            <span class="k">else</span> <span class="n">torch</span><span class="o">.</span><span class="n">tensor</span><span class="p">(</span><span class="mf">0.0</span><span class="p">)</span>
+        <span class="p">)</span>
+
+        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">lambda_cat</span> <span class="o">*</span> <span class="n">loss_cat</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">lambda_cont</span> <span class="o">*</span> <span class="n">loss_cont</span>
+
+    <span class="k">def</span> <span class="nf">_compute_cat_loss</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span> <span class="n">x_cat_and_cat_</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="n">Tuple</span><span class="p">[</span><span class="n">Tensor</span><span class="p">,</span> <span class="n">Tensor</span><span class="p">]],</span> <span class="n">Tuple</span><span class="p">[</span><span class="n">Tensor</span><span class="p">,</span> <span class="n">Tensor</span><span class="p">]]</span>
+    <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tensor</span><span class="p">:</span>
+        <span class="n">loss_cat</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">tensor</span><span class="p">(</span><span class="mf">0.0</span><span class="p">,</span> <span class="n">device</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_get_device</span><span class="p">(</span><span class="n">x_cat_and_cat_</span><span class="p">))</span>
+        <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">x_cat_and_cat_</span><span class="p">,</span> <span class="nb">list</span><span class="p">):</span>
+            <span class="k">for</span> <span class="n">x</span><span class="p">,</span> <span class="n">x_</span> <span class="ow">in</span> <span class="n">x_cat_and_cat_</span><span class="p">:</span>
+                <span class="n">loss_cat</span> <span class="o">+=</span> <span class="n">F</span><span class="o">.</span><span class="n">cross_entropy</span><span class="p">(</span><span class="n">x_</span><span class="p">,</span> <span class="n">x</span><span class="p">,</span> <span class="n">reduction</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">reduction</span><span class="p">)</span>
+        <span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">x_cat_and_cat_</span><span class="p">,</span> <span class="nb">tuple</span><span class="p">):</span>
+            <span class="n">x</span><span class="p">,</span> <span class="n">x_</span> <span class="o">=</span> <span class="n">x_cat_and_cat_</span>
+            <span class="n">loss_cat</span> <span class="o">+=</span> <span class="n">F</span><span class="o">.</span><span class="n">cross_entropy</span><span class="p">(</span><span class="n">x_</span><span class="p">,</span> <span class="n">x</span><span class="p">,</span> <span class="n">reduction</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">reduction</span><span class="p">)</span>
+
+        <span class="k">return</span> <span class="n">loss_cat</span>
+
+    <span class="k">def</span> <span class="nf">_compute_cont_loss</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">x_cont_and_cont_</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tensor</span><span class="p">:</span>
+        <span class="n">loss_cont</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">tensor</span><span class="p">(</span><span class="mf">0.0</span><span class="p">,</span> <span class="n">device</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_get_device</span><span class="p">(</span><span class="n">x_cont_and_cont_</span><span class="p">))</span>
+        <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">x_cont_and_cont_</span><span class="p">,</span> <span class="nb">list</span><span class="p">):</span>
+            <span class="k">for</span> <span class="n">x</span><span class="p">,</span> <span class="n">x_</span> <span class="ow">in</span> <span class="n">x_cont_and_cont_</span><span class="p">:</span>
+                <span class="n">loss_cont</span> <span class="o">+=</span> <span class="n">F</span><span class="o">.</span><span class="n">mse_loss</span><span class="p">(</span><span class="n">x_</span><span class="p">,</span> <span class="n">x</span><span class="p">,</span> <span class="n">reduction</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">reduction</span><span class="p">)</span>
+        <span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">x_cont_and_cont_</span><span class="p">,</span> <span class="nb">tuple</span><span class="p">):</span>
+            <span class="n">x</span><span class="p">,</span> <span class="n">x_</span> <span class="o">=</span> <span class="n">x_cont_and_cont_</span>
+            <span class="n">loss_cont</span> <span class="o">+=</span> <span class="n">F</span><span class="o">.</span><span class="n">mse_loss</span><span class="p">(</span><span class="n">x_</span><span class="p">,</span> <span class="n">x</span><span class="p">,</span> <span class="n">reduction</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">reduction</span><span class="p">)</span>
+
+        <span class="k">return</span> <span class="n">loss_cont</span>
+
+    <span class="nd">@staticmethod</span>
+    <span class="k">def</span> <span class="nf">_get_device</span><span class="p">(</span>
+        <span class="n">x_and_x_</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="n">Tuple</span><span class="p">[</span><span class="n">Tensor</span><span class="p">,</span> <span class="n">Tensor</span><span class="p">]],</span> <span class="n">Tuple</span><span class="p">[</span><span class="n">Tensor</span><span class="p">,</span> <span class="n">Tensor</span><span class="p">]]</span>
+    <span class="p">):</span>
+        <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">x_and_x_</span><span class="p">,</span> <span class="nb">tuple</span><span class="p">):</span>
+            <span class="n">device</span> <span class="o">=</span> <span class="n">x_and_x_</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">device</span>
+        <span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">x_and_x_</span><span class="p">,</span> <span class="nb">list</span><span class="p">):</span>
+            <span class="n">device</span> <span class="o">=</span> <span class="n">x_and_x_</span><span class="p">[</span><span class="mi">0</span><span class="p">][</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">device</span>
+        <span class="k">return</span> <span class="n">device</span>
+</code></pre></div></td></tr></table></div>
+              </details>
 
 
 
@@ -4621,28 +6611,50 @@ <h3 id="pytorch_widedeep.losses.DenoisingLoss.forward" class="doc doc-heading">
 
 
 <p><span class="doc-section-title">Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>x_cat_and_cat_</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.wdtypes.Tuple">Tuple</span>[<span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span>, <span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span>]], <span title="pytorch_widedeep.wdtypes.Tuple">Tuple</span>[<span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span>, <span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span>]]]</code>)
-          –
-          <div class="doc-md-description">
-            <p>Tuple of tensors containing the raw input features and their
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+          <th>Default</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>x_cat_and_cat_</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.wdtypes.Tuple">Tuple</span>[<span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span>, <span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span>]], <span title="pytorch_widedeep.wdtypes.Tuple">Tuple</span>[<span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span>, <span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span>]]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Tuple of tensors containing the raw input features and their
 encodings, referred in the SAINT paper as <span class="arithmatex">\(x\)</span> and <span class="arithmatex">\(x''\)</span>
 respectively. If one denoising MLP is used per categorical
 feature <code>x_cat_and_cat_</code> will be a list of tuples, one per
 categorical feature</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>x_cont_and_cont_</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.wdtypes.Tuple">Tuple</span>[<span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span>, <span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span>]], <span title="pytorch_widedeep.wdtypes.Tuple">Tuple</span>[<span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span>, <span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span>]]]</code>)
-          –
-          <div class="doc-md-description">
-            <p>same as <code>x_cat_and_cat_</code> but for continuous columns</p>
-          </div>
-        </li>
-    </ul>
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>x_cont_and_cont_</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.wdtypes.Tuple">Tuple</span>[<span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span>, <span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span>]], <span title="pytorch_widedeep.wdtypes.Tuple">Tuple</span>[<span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span>, <span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span>]]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>same as <code>x_cat_and_cat_</code> but for continuous columns</p>
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
 
 <p><span class="doc-section-title">Examples:</span></p>
@@ -4764,8 +6776,7 @@ <h2 id="pytorch_widedeep.losses.EncoderDecoderLoss" class="doc doc-heading">
 
 
 <a href="#pytorch_widedeep.losses.EncoderDecoderLoss" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="nf">EncoderDecoderLoss</span><span class="p">(</span><span class="n">eps</span><span class="o">=</span><span class="mf">1e-09</span><span class="p">)</span>
-</code></pre></div>
+
 
     <div class="doc doc-contents first">
             <p class="doc doc-class-bases">
@@ -4784,28 +6795,162 @@ <h2 id="pytorch_widedeep.losses.EncoderDecoderLoss" class="doc doc-heading">
 
 
 <p><span class="doc-section-title">Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>eps</code></b>
-              (<code>float</code>, default:
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+          <th>Default</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>eps</code></td>
+            <td>
+                  <code>float</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Simply a small number to avoid dividing by zero</p>
+              </div>
+            </td>
+            <td>
                   <code>1e-09</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Simply a small number to avoid dividing by zero</p>
-          </div>
-        </li>
-    </ul>
-
-                  <details class="quote">
-                    <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
-                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">927</span>
+            </td>
+          </tr>
+      </tbody>
+    </table>
+
+              <details class="quote">
+                <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
+                <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">908</span>
+<span class="normal">909</span>
+<span class="normal">910</span>
+<span class="normal">911</span>
+<span class="normal">912</span>
+<span class="normal">913</span>
+<span class="normal">914</span>
+<span class="normal">915</span>
+<span class="normal">916</span>
+<span class="normal">917</span>
+<span class="normal">918</span>
+<span class="normal">919</span>
+<span class="normal">920</span>
+<span class="normal">921</span>
+<span class="normal">922</span>
+<span class="normal">923</span>
+<span class="normal">924</span>
+<span class="normal">925</span>
+<span class="normal">926</span>
+<span class="normal">927</span>
 <span class="normal">928</span>
-<span class="normal">929</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">eps</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">1e-9</span><span class="p">):</span>
-    <span class="nb">super</span><span class="p">(</span><span class="n">EncoderDecoderLoss</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">eps</span> <span class="o">=</span> <span class="n">eps</span>
+<span class="normal">929</span>
+<span class="normal">930</span>
+<span class="normal">931</span>
+<span class="normal">932</span>
+<span class="normal">933</span>
+<span class="normal">934</span>
+<span class="normal">935</span>
+<span class="normal">936</span>
+<span class="normal">937</span>
+<span class="normal">938</span>
+<span class="normal">939</span>
+<span class="normal">940</span>
+<span class="normal">941</span>
+<span class="normal">942</span>
+<span class="normal">943</span>
+<span class="normal">944</span>
+<span class="normal">945</span>
+<span class="normal">946</span>
+<span class="normal">947</span>
+<span class="normal">948</span>
+<span class="normal">949</span>
+<span class="normal">950</span>
+<span class="normal">951</span>
+<span class="normal">952</span>
+<span class="normal">953</span>
+<span class="normal">954</span>
+<span class="normal">955</span>
+<span class="normal">956</span>
+<span class="normal">957</span>
+<span class="normal">958</span>
+<span class="normal">959</span>
+<span class="normal">960</span>
+<span class="normal">961</span>
+<span class="normal">962</span>
+<span class="normal">963</span>
+<span class="normal">964</span>
+<span class="normal">965</span>
+<span class="normal">966</span>
+<span class="normal">967</span>
+<span class="normal">968</span>
+<span class="normal">969</span>
+<span class="normal">970</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">class</span> <span class="nc">EncoderDecoderLoss</span><span class="p">(</span><span class="n">nn</span><span class="o">.</span><span class="n">Module</span><span class="p">):</span>
+<span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;&#39;_Standard_&#39; Encoder Decoder Loss. Loss applied during the Endoder-Decoder</span>
+<span class="sd">     Self-Supervised Pre-Training routine available in this library</span>
+
+<span class="sd">    :information_source: **NOTE**: This loss is in principle not exposed to</span>
+<span class="sd">     the user, as it is used internally in the library, but it is included</span>
+<span class="sd">     here for completion.</span>
+
+<span class="sd">    The implementation of this lost is based on that at the</span>
+<span class="sd">    [tabnet repo](https://github.com/dreamquark-ai/tabnet), which is in itself an</span>
+<span class="sd">    adaptation of that in the original paper [TabNet: Attentive</span>
+<span class="sd">    Interpretable Tabular Learning](https://arxiv.org/abs/1908.07442).</span>
+
+<span class="sd">    Parameters</span>
+<span class="sd">    ----------</span>
+<span class="sd">    eps: float</span>
+<span class="sd">        Simply a small number to avoid dividing by zero</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+
+    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">eps</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">1e-9</span><span class="p">):</span>
+        <span class="nb">super</span><span class="p">(</span><span class="n">EncoderDecoderLoss</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">eps</span> <span class="o">=</span> <span class="n">eps</span>
+
+    <span class="k">def</span> <span class="nf">forward</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">x_true</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">,</span> <span class="n">x_pred</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">,</span> <span class="n">mask</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tensor</span><span class="p">:</span>
+<span class="w">        </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Parameters</span>
+<span class="sd">        ----------</span>
+<span class="sd">        x_true: Tensor</span>
+<span class="sd">            Embeddings of the input data</span>
+<span class="sd">        x_pred: Tensor</span>
+<span class="sd">            Reconstructed embeddings</span>
+<span class="sd">        mask: Tensor</span>
+<span class="sd">            Mask with 1s indicated that the reconstruction, and therefore the</span>
+<span class="sd">            loss, is based on those features.</span>
+
+<span class="sd">        Examples</span>
+<span class="sd">        --------</span>
+<span class="sd">        &gt;&gt;&gt; import torch</span>
+<span class="sd">        &gt;&gt;&gt; from pytorch_widedeep.losses import EncoderDecoderLoss</span>
+<span class="sd">        &gt;&gt;&gt; x_true = torch.rand(3, 3)</span>
+<span class="sd">        &gt;&gt;&gt; x_pred = torch.rand(3, 3)</span>
+<span class="sd">        &gt;&gt;&gt; mask = torch.empty(3, 3).random_(2)</span>
+<span class="sd">        &gt;&gt;&gt; loss = EncoderDecoderLoss()</span>
+<span class="sd">        &gt;&gt;&gt; res = loss(x_true, x_pred, mask)</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+
+        <span class="n">errors</span> <span class="o">=</span> <span class="n">x_pred</span> <span class="o">-</span> <span class="n">x_true</span>
+
+        <span class="n">reconstruction_errors</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">mul</span><span class="p">(</span><span class="n">errors</span><span class="p">,</span> <span class="n">mask</span><span class="p">)</span> <span class="o">**</span> <span class="mi">2</span>
+
+        <span class="n">x_true_means</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="n">x_true</span><span class="p">,</span> <span class="n">dim</span><span class="o">=</span><span class="mi">0</span><span class="p">)</span>
+        <span class="n">x_true_means</span><span class="p">[</span><span class="n">x_true_means</span> <span class="o">==</span> <span class="mi">0</span><span class="p">]</span> <span class="o">=</span> <span class="mi">1</span>
+
+        <span class="n">x_true_stds</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">std</span><span class="p">(</span><span class="n">x_true</span><span class="p">,</span> <span class="n">dim</span><span class="o">=</span><span class="mi">0</span><span class="p">)</span> <span class="o">**</span> <span class="mi">2</span>
+        <span class="n">x_true_stds</span><span class="p">[</span><span class="n">x_true_stds</span> <span class="o">==</span> <span class="mi">0</span><span class="p">]</span> <span class="o">=</span> <span class="n">x_true_means</span><span class="p">[</span><span class="n">x_true_stds</span> <span class="o">==</span> <span class="mi">0</span><span class="p">]</span>
+
+        <span class="n">features_loss</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">matmul</span><span class="p">(</span><span class="n">reconstruction_errors</span><span class="p">,</span> <span class="mi">1</span> <span class="o">/</span> <span class="n">x_true_stds</span><span class="p">)</span>
+        <span class="n">nb_reconstructed_variables</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">sum</span><span class="p">(</span><span class="n">mask</span><span class="p">,</span> <span class="n">dim</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span>
+        <span class="n">features_loss_norm</span> <span class="o">=</span> <span class="n">features_loss</span> <span class="o">/</span> <span class="p">(</span><span class="n">nb_reconstructed_variables</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">eps</span><span class="p">)</span>
+
+        <span class="n">loss</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="n">features_loss_norm</span><span class="p">)</span>
+
+        <span class="k">return</span> <span class="n">loss</span>
 </code></pre></div></td></tr></table></div>
-                  </details>
+              </details>
 
 
 
@@ -4835,33 +6980,61 @@ <h3 id="pytorch_widedeep.losses.EncoderDecoderLoss.forward" class="doc doc-headi
 
 
 <p><span class="doc-section-title">Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>x_true</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>)
-          –
-          <div class="doc-md-description">
-            <p>Embeddings of the input data</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>x_pred</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>)
-          –
-          <div class="doc-md-description">
-            <p>Reconstructed embeddings</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>mask</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>)
-          –
-          <div class="doc-md-description">
-            <p>Mask with 1s indicated that the reconstruction, and therefore the
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+          <th>Default</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>x_true</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Embeddings of the input data</p>
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>x_pred</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Reconstructed embeddings</p>
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>mask</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Mask with 1s indicated that the reconstruction, and therefore the
 loss, is based on those features.</p>
-          </div>
-        </li>
-    </ul>
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
 
 <p><span class="doc-section-title">Examples:</span></p>
@@ -4978,8 +7151,7 @@ <h2 id="pytorch_widedeep.losses_multitarget.MultiTargetRegressionLoss" class="do
 
 
 <a href="#pytorch_widedeep.losses_multitarget.MultiTargetRegressionLoss" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="nf">MultiTargetRegressionLoss</span><span class="p">(</span><span class="n">weights</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">reduction</span><span class="o">=</span><span class="s1">&#39;mean&#39;</span><span class="p">)</span>
-</code></pre></div>
+
 
     <div class="doc doc-contents first">
             <p class="doc doc-class-bases">
@@ -4992,33 +7164,51 @@ <h2 id="pytorch_widedeep.losses_multitarget.MultiTargetRegressionLoss" class="do
 
 
 <p><span class="doc-section-title">Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>weights</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[float]]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>List of weights to apply to the loss associated to each target. The
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+          <th>Default</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>weights</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[float]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>List of weights to apply to the loss associated to each target. The
 length of the list must match the number of targets.
 Alias: 'target_weights'</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>reduction</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[mean, sum]</code>, default:
-                  <code>&#39;mean&#39;</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Specifies the reduction to apply to the loss associated to each
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>reduction</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[mean, sum]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Specifies the reduction to apply to the loss associated to each
 target: 'mean' | 'sum'. Note that this is NOT the same as the
 reduction in the MSELoss. This reduction is applied after the loss
 for each target has been computed. Alias: 'target_reduction'</p>
-          </div>
-        </li>
-    </ul>
+              </div>
+            </td>
+            <td>
+                  <code>&#39;mean&#39;</code>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
 
 <p><span class="doc-section-title">Examples:</span></p>
@@ -5030,9 +7220,37 @@ <h2 id="pytorch_widedeep.losses_multitarget.MultiTargetRegressionLoss" class="do
 <span class="gp">&gt;&gt;&gt; </span><span class="n">output</span> <span class="o">=</span> <span class="n">loss</span><span class="p">(</span><span class="nb">input</span><span class="p">,</span> <span class="n">target</span><span class="p">)</span>
 </code></pre></div>
 
-                  <details class="quote">
-                    <summary>Source code in <code>pytorch_widedeep/losses_multitarget.py</code></summary>
-                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">39</span>
+              <details class="quote">
+                <summary>Source code in <code>pytorch_widedeep/losses_multitarget.py</code></summary>
+                <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">11</span>
+<span class="normal">12</span>
+<span class="normal">13</span>
+<span class="normal">14</span>
+<span class="normal">15</span>
+<span class="normal">16</span>
+<span class="normal">17</span>
+<span class="normal">18</span>
+<span class="normal">19</span>
+<span class="normal">20</span>
+<span class="normal">21</span>
+<span class="normal">22</span>
+<span class="normal">23</span>
+<span class="normal">24</span>
+<span class="normal">25</span>
+<span class="normal">26</span>
+<span class="normal">27</span>
+<span class="normal">28</span>
+<span class="normal">29</span>
+<span class="normal">30</span>
+<span class="normal">31</span>
+<span class="normal">32</span>
+<span class="normal">33</span>
+<span class="normal">34</span>
+<span class="normal">35</span>
+<span class="normal">36</span>
+<span class="normal">37</span>
+<span class="normal">38</span>
+<span class="normal">39</span>
 <span class="normal">40</span>
 <span class="normal">41</span>
 <span class="normal">42</span>
@@ -5045,22 +7263,88 @@ <h2 id="pytorch_widedeep.losses_multitarget.MultiTargetRegressionLoss" class="do
 <span class="normal">49</span>
 <span class="normal">50</span>
 <span class="normal">51</span>
-<span class="normal">52</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="nd">@alias</span><span class="p">(</span><span class="s2">&quot;reduction&quot;</span><span class="p">,</span> <span class="p">[</span><span class="s2">&quot;target_reduction&quot;</span><span class="p">])</span>
-<span class="nd">@alias</span><span class="p">(</span><span class="s2">&quot;weights&quot;</span><span class="p">,</span> <span class="p">[</span><span class="s2">&quot;target_weights&quot;</span><span class="p">])</span>
-<span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
-    <span class="bp">self</span><span class="p">,</span>
-    <span class="n">weights</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">float</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">reduction</span><span class="p">:</span> <span class="n">Literal</span><span class="p">[</span><span class="s2">&quot;mean&quot;</span><span class="p">,</span> <span class="s2">&quot;sum&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="s2">&quot;mean&quot;</span><span class="p">,</span>
-<span class="p">):</span>
-    <span class="nb">super</span><span class="p">(</span><span class="n">MultiTargetRegressionLoss</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
+<span class="normal">52</span>
+<span class="normal">53</span>
+<span class="normal">54</span>
+<span class="normal">55</span>
+<span class="normal">56</span>
+<span class="normal">57</span>
+<span class="normal">58</span>
+<span class="normal">59</span>
+<span class="normal">60</span>
+<span class="normal">61</span>
+<span class="normal">62</span>
+<span class="normal">63</span>
+<span class="normal">64</span>
+<span class="normal">65</span>
+<span class="normal">66</span>
+<span class="normal">67</span>
+<span class="normal">68</span>
+<span class="normal">69</span>
+<span class="normal">70</span>
+<span class="normal">71</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">class</span> <span class="nc">MultiTargetRegressionLoss</span><span class="p">(</span><span class="n">nn</span><span class="o">.</span><span class="n">Module</span><span class="p">):</span>
+<span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">    This class is a wrapper around the Pytorch MSELoss. It allows for multi-target</span>
+<span class="sd">    regression problems. The user can provide a list of weights to apply to each</span>
+<span class="sd">    target. The loss can be either the sum or the mean of the individual losses</span>
 
-    <span class="bp">self</span><span class="o">.</span><span class="n">weights</span> <span class="o">=</span> <span class="n">weights</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">reduction</span> <span class="o">=</span> <span class="n">reduction</span>
+<span class="sd">    Parameters</span>
+<span class="sd">    ----------</span>
+<span class="sd">    weights: Optional[List[float], default = None]</span>
+<span class="sd">        List of weights to apply to the loss associated to each target. The</span>
+<span class="sd">        length of the list must match the number of targets.</span>
+<span class="sd">        Alias: &#39;target_weights&#39;</span>
+<span class="sd">    reduction: Literal[&quot;mean&quot;, &quot;sum&quot;], default = &quot;mean</span>
+<span class="sd">        Specifies the reduction to apply to the loss associated to each</span>
+<span class="sd">        target: &#39;mean&#39; | &#39;sum&#39;. Note that this is NOT the same as the</span>
+<span class="sd">        reduction in the MSELoss. This reduction is applied after the loss</span>
+<span class="sd">        for each target has been computed. Alias: &#39;target_reduction&#39;</span>
 
-    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">reduction</span> <span class="ow">not</span> <span class="ow">in</span> <span class="p">[</span><span class="s2">&quot;mean&quot;</span><span class="p">,</span> <span class="s2">&quot;sum&quot;</span><span class="p">]:</span>
-        <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">&quot;reduction must be either &#39;mean&#39; or &#39;sum&#39;&quot;</span><span class="p">)</span>
+<span class="sd">    Examples</span>
+<span class="sd">    --------</span>
+<span class="sd">    &gt;&gt;&gt; import torch</span>
+<span class="sd">    &gt;&gt;&gt; from pytorch_widedeep.losses_multitarget import MultiTargetRegressionLoss</span>
+<span class="sd">    &gt;&gt;&gt; input = torch.randn(3, 2)</span>
+<span class="sd">    &gt;&gt;&gt; target = torch.randn(3, 2)</span>
+<span class="sd">    &gt;&gt;&gt; loss = MultiTargetRegressionLoss(weights=[0.5, 0.5], reduction=&quot;mean&quot;)</span>
+<span class="sd">    &gt;&gt;&gt; output = loss(input, target)</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+
+    <span class="nd">@alias</span><span class="p">(</span><span class="s2">&quot;reduction&quot;</span><span class="p">,</span> <span class="p">[</span><span class="s2">&quot;target_reduction&quot;</span><span class="p">])</span>
+    <span class="nd">@alias</span><span class="p">(</span><span class="s2">&quot;weights&quot;</span><span class="p">,</span> <span class="p">[</span><span class="s2">&quot;target_weights&quot;</span><span class="p">])</span>
+    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span>
+        <span class="n">weights</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">float</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">reduction</span><span class="p">:</span> <span class="n">Literal</span><span class="p">[</span><span class="s2">&quot;mean&quot;</span><span class="p">,</span> <span class="s2">&quot;sum&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="s2">&quot;mean&quot;</span><span class="p">,</span>
+    <span class="p">):</span>
+        <span class="nb">super</span><span class="p">(</span><span class="n">MultiTargetRegressionLoss</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">weights</span> <span class="o">=</span> <span class="n">weights</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">reduction</span> <span class="o">=</span> <span class="n">reduction</span>
+
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">reduction</span> <span class="ow">not</span> <span class="ow">in</span> <span class="p">[</span><span class="s2">&quot;mean&quot;</span><span class="p">,</span> <span class="s2">&quot;sum&quot;</span><span class="p">]:</span>
+            <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">&quot;reduction must be either &#39;mean&#39; or &#39;sum&#39;&quot;</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">forward</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="nb">input</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">,</span> <span class="n">target</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tensor</span><span class="p">:</span>
+
+        <span class="k">assert</span> <span class="nb">input</span><span class="o">.</span><span class="n">size</span><span class="p">()</span> <span class="o">==</span> <span class="n">target</span><span class="o">.</span><span class="n">size</span><span class="p">()</span>
+
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">weights</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+
+            <span class="k">assert</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">weights</span><span class="p">)</span> <span class="o">==</span> <span class="nb">input</span><span class="o">.</span><span class="n">size</span><span class="p">(</span><span class="mi">1</span><span class="p">),</span> <span class="p">(</span>
+                <span class="s2">&quot;The number of weights must match the number of targets. &quot;</span>
+                <span class="sa">f</span><span class="s2">&quot;Got </span><span class="si">{</span><span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">weights</span><span class="p">)</span><span class="si">}</span><span class="s2"> weights and </span><span class="si">{</span><span class="nb">input</span><span class="o">.</span><span class="n">size</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span><span class="si">}</span><span class="s2"> targets&quot;</span>
+            <span class="p">)</span>
+
+            <span class="n">loss</span> <span class="o">=</span> <span class="n">F</span><span class="o">.</span><span class="n">mse_loss</span><span class="p">(</span><span class="nb">input</span><span class="p">,</span> <span class="n">target</span><span class="p">,</span> <span class="n">reduction</span><span class="o">=</span><span class="s2">&quot;none&quot;</span><span class="p">)</span> <span class="o">*</span> <span class="n">torch</span><span class="o">.</span><span class="n">tensor</span><span class="p">(</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">weights</span>
+            <span class="p">)</span><span class="o">.</span><span class="n">to</span><span class="p">(</span><span class="nb">input</span><span class="o">.</span><span class="n">device</span><span class="p">)</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="n">loss</span> <span class="o">=</span> <span class="n">F</span><span class="o">.</span><span class="n">mse_loss</span><span class="p">(</span><span class="nb">input</span><span class="p">,</span> <span class="n">target</span><span class="p">,</span> <span class="n">reduction</span><span class="o">=</span><span class="s2">&quot;none&quot;</span><span class="p">)</span>
+
+        <span class="k">return</span> <span class="n">loss</span><span class="o">.</span><span class="n">mean</span><span class="p">()</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">reduction</span> <span class="o">==</span> <span class="s2">&quot;mean&quot;</span> <span class="k">else</span> <span class="n">loss</span><span class="o">.</span><span class="n">sum</span><span class="p">()</span>
 </code></pre></div></td></tr></table></div>
-                  </details>
+              </details>
 
 
 
@@ -5091,14 +7375,7 @@ <h2 id="pytorch_widedeep.losses_multitarget.MultiTargetClassificationLoss" class
 
 
 <a href="#pytorch_widedeep.losses_multitarget.MultiTargetClassificationLoss" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="nf">MultiTargetClassificationLoss</span><span class="p">(</span>
-    <span class="n">binary_config</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">multiclass_config</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">weights</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">reduction</span><span class="o">=</span><span class="s2">&quot;mean&quot;</span><span class="p">,</span>
-    <span class="n">binary_trick</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
-<span class="p">)</span>
-</code></pre></div>
+
 
     <div class="doc doc-contents first">
             <p class="doc doc-class-bases">
@@ -5112,67 +7389,92 @@ <h2 id="pytorch_widedeep.losses_multitarget.MultiTargetClassificationLoss" class
 
 
 <p><span class="doc-section-title">Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>binary_config</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[int, <span title="pytorch_widedeep.wdtypes.Tuple">Tuple</span>[int, float]]]]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>List of integers with the index of the target for binary
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+          <th>Default</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>binary_config</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[int, <span title="pytorch_widedeep.wdtypes.Tuple">Tuple</span>[int, float]]]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>List of integers with the index of the target for binary
 classification or tuples with two elements: the index of the targets
 or binary classification and the positive weight for binary
 classification</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>multiclass_config</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="pytorch_widedeep.wdtypes.Tuple">Tuple</span>[int, int], <span title="pytorch_widedeep.wdtypes.Tuple">Tuple</span>[int, int, <span title="pytorch_widedeep.wdtypes.List">List</span>[float]]]]]</code>, default:
+              </div>
+            </td>
+            <td>
                   <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>List of tuples with two or three elements: the index of the target and the
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>multiclass_config</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="pytorch_widedeep.wdtypes.Tuple">Tuple</span>[int, int], <span title="pytorch_widedeep.wdtypes.Tuple">Tuple</span>[int, int, <span title="pytorch_widedeep.wdtypes.List">List</span>[float]]]]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>List of tuples with two or three elements: the index of the target and the
 number of classes for multiclass classification, or a tuple with the index of
 the target, the number of classes and a list of weights to apply to each class
 (i.e. the 'weight' parameter in the cross_entropy loss)</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>weights</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[float]]</code>, default:
+              </div>
+            </td>
+            <td>
                   <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>List of weights to apply to the loss associated to each target. The
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>weights</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[float]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>List of weights to apply to the loss associated to each target. The
 length of the list must match the number of targets.
 Alias: 'target_weights'</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>reduction</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[mean, sum]</code>, default:
-                  <code>&#39;mean&#39;</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Specifies the reduction to apply to the loss associated to each
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>reduction</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[mean, sum]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Specifies the reduction to apply to the loss associated to each
 target: 'mean' | 'sum'. Note that this is NOT the same as the
 reduction in the cross_entropy loss or the
 binary_cross_entropy_with_logits. This reduction is applied after the
 loss for each target has been computed. Alias: 'target_reduction'</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>binary_trick</code></b>
-              (<code>bool</code>, default:
-                  <code>False</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>If True, each target will be considered independently and the loss
+              </div>
+            </td>
+            <td>
+                  <code>&#39;mean&#39;</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>binary_trick</code></td>
+            <td>
+                  <code>bool</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>If True, each target will be considered independently and the loss
 will be computed as binary_cross_entropy_with_logits. This is a
 faster implementation. Note that the 'weights' parameter is not
 compatible with binary_trick=True. Also note that if
@@ -5186,9 +7488,14 @@ <h2 id="pytorch_widedeep.losses_multitarget.MultiTargetClassificationLoss" class
   WideDeep class should be adjusted accordingly (adding 2 to per
   binary target). For example, in a problem with a binary target and
   a 4 class multiclassification target, the pred_dim should be 6.</p>
-          </div>
-        </li>
-    </ul>
+              </div>
+            </td>
+            <td>
+                  <code>False</code>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
 
 <p><span class="doc-section-title">Examples:</span></p>
@@ -5206,9 +7513,72 @@ <h2 id="pytorch_widedeep.losses_multitarget.MultiTargetClassificationLoss" class
 <span class="gp">&gt;&gt;&gt; </span><span class="n">output_3</span> <span class="o">=</span> <span class="n">loss_3</span><span class="p">(</span><span class="n">input_binary_trick</span><span class="p">,</span> <span class="n">target</span><span class="p">)</span>
 </code></pre></div>
 
-                  <details class="quote">
-                    <summary>Source code in <code>pytorch_widedeep/losses_multitarget.py</code></summary>
-                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">137</span>
+              <details class="quote">
+                <summary>Source code in <code>pytorch_widedeep/losses_multitarget.py</code></summary>
+                <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 74</span>
+<span class="normal"> 75</span>
+<span class="normal"> 76</span>
+<span class="normal"> 77</span>
+<span class="normal"> 78</span>
+<span class="normal"> 79</span>
+<span class="normal"> 80</span>
+<span class="normal"> 81</span>
+<span class="normal"> 82</span>
+<span class="normal"> 83</span>
+<span class="normal"> 84</span>
+<span class="normal"> 85</span>
+<span class="normal"> 86</span>
+<span class="normal"> 87</span>
+<span class="normal"> 88</span>
+<span class="normal"> 89</span>
+<span class="normal"> 90</span>
+<span class="normal"> 91</span>
+<span class="normal"> 92</span>
+<span class="normal"> 93</span>
+<span class="normal"> 94</span>
+<span class="normal"> 95</span>
+<span class="normal"> 96</span>
+<span class="normal"> 97</span>
+<span class="normal"> 98</span>
+<span class="normal"> 99</span>
+<span class="normal">100</span>
+<span class="normal">101</span>
+<span class="normal">102</span>
+<span class="normal">103</span>
+<span class="normal">104</span>
+<span class="normal">105</span>
+<span class="normal">106</span>
+<span class="normal">107</span>
+<span class="normal">108</span>
+<span class="normal">109</span>
+<span class="normal">110</span>
+<span class="normal">111</span>
+<span class="normal">112</span>
+<span class="normal">113</span>
+<span class="normal">114</span>
+<span class="normal">115</span>
+<span class="normal">116</span>
+<span class="normal">117</span>
+<span class="normal">118</span>
+<span class="normal">119</span>
+<span class="normal">120</span>
+<span class="normal">121</span>
+<span class="normal">122</span>
+<span class="normal">123</span>
+<span class="normal">124</span>
+<span class="normal">125</span>
+<span class="normal">126</span>
+<span class="normal">127</span>
+<span class="normal">128</span>
+<span class="normal">129</span>
+<span class="normal">130</span>
+<span class="normal">131</span>
+<span class="normal">132</span>
+<span class="normal">133</span>
+<span class="normal">134</span>
+<span class="normal">135</span>
+<span class="normal">136</span>
+<span class="normal">137</span>
 <span class="normal">138</span>
 <span class="normal">139</span>
 <span class="normal">140</span>
@@ -5255,56 +7625,333 @@ <h2 id="pytorch_widedeep.losses_multitarget.MultiTargetClassificationLoss" class
 <span class="normal">181</span>
 <span class="normal">182</span>
 <span class="normal">183</span>
-<span class="normal">184</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="nd">@alias</span><span class="p">(</span><span class="s2">&quot;reduction&quot;</span><span class="p">,</span> <span class="p">[</span><span class="s2">&quot;target_reduction&quot;</span><span class="p">])</span>
-<span class="nd">@alias</span><span class="p">(</span><span class="s2">&quot;weights&quot;</span><span class="p">,</span> <span class="p">[</span><span class="s2">&quot;target_weights&quot;</span><span class="p">])</span>
-<span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>  <span class="c1"># noqa: C901</span>
-    <span class="bp">self</span><span class="p">,</span>
-    <span class="n">binary_config</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="n">Union</span><span class="p">[</span><span class="nb">int</span><span class="p">,</span> <span class="n">Tuple</span><span class="p">[</span><span class="nb">int</span><span class="p">,</span> <span class="nb">float</span><span class="p">]]]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">multiclass_config</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span>
-        <span class="n">List</span><span class="p">[</span><span class="n">Union</span><span class="p">[</span><span class="n">Tuple</span><span class="p">[</span><span class="nb">int</span><span class="p">,</span> <span class="nb">int</span><span class="p">],</span> <span class="n">Tuple</span><span class="p">[</span><span class="nb">int</span><span class="p">,</span> <span class="nb">int</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="nb">float</span><span class="p">]]]]</span>
-    <span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">weights</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">float</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">reduction</span><span class="p">:</span> <span class="n">Literal</span><span class="p">[</span><span class="s2">&quot;mean&quot;</span><span class="p">,</span> <span class="s2">&quot;sum&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="s2">&quot;mean&quot;</span><span class="p">,</span>
-    <span class="n">binary_trick</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
-<span class="p">):</span>
-    <span class="nb">super</span><span class="p">(</span><span class="n">MultiTargetClassificationLoss</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
-
-    <span class="k">if</span> <span class="n">reduction</span> <span class="ow">not</span> <span class="ow">in</span> <span class="p">[</span><span class="s2">&quot;mean&quot;</span><span class="p">,</span> <span class="s2">&quot;sum&quot;</span><span class="p">]:</span>
-        <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">&quot;reduction must be either &#39;mean&#39; or &#39;sum&#39;&quot;</span><span class="p">)</span>
-
-    <span class="bp">self</span><span class="o">.</span><span class="n">binary_config</span> <span class="o">=</span> <span class="n">binary_config</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">multiclass_config</span> <span class="o">=</span> <span class="n">multiclass_config</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">weights</span> <span class="o">=</span> <span class="n">weights</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">reduction</span> <span class="o">=</span> <span class="n">reduction</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">binary_trick</span> <span class="o">=</span> <span class="n">binary_trick</span>
-
-    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">weights</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
-        <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">weights</span><span class="p">)</span> <span class="o">!=</span> <span class="p">(</span>
-            <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">binary_config</span><span class="p">)</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">binary_config</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="k">else</span> <span class="mi">0</span>
-        <span class="p">)</span> <span class="o">+</span> <span class="p">(</span>
-            <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">multiclass_config</span><span class="p">)</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">multiclass_config</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="k">else</span> <span class="mi">0</span>
-        <span class="p">):</span>
-            <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
-                <span class="s2">&quot;The number of weights must match the number of binary and multiclass targets&quot;</span>
+<span class="normal">184</span>
+<span class="normal">185</span>
+<span class="normal">186</span>
+<span class="normal">187</span>
+<span class="normal">188</span>
+<span class="normal">189</span>
+<span class="normal">190</span>
+<span class="normal">191</span>
+<span class="normal">192</span>
+<span class="normal">193</span>
+<span class="normal">194</span>
+<span class="normal">195</span>
+<span class="normal">196</span>
+<span class="normal">197</span>
+<span class="normal">198</span>
+<span class="normal">199</span>
+<span class="normal">200</span>
+<span class="normal">201</span>
+<span class="normal">202</span>
+<span class="normal">203</span>
+<span class="normal">204</span>
+<span class="normal">205</span>
+<span class="normal">206</span>
+<span class="normal">207</span>
+<span class="normal">208</span>
+<span class="normal">209</span>
+<span class="normal">210</span>
+<span class="normal">211</span>
+<span class="normal">212</span>
+<span class="normal">213</span>
+<span class="normal">214</span>
+<span class="normal">215</span>
+<span class="normal">216</span>
+<span class="normal">217</span>
+<span class="normal">218</span>
+<span class="normal">219</span>
+<span class="normal">220</span>
+<span class="normal">221</span>
+<span class="normal">222</span>
+<span class="normal">223</span>
+<span class="normal">224</span>
+<span class="normal">225</span>
+<span class="normal">226</span>
+<span class="normal">227</span>
+<span class="normal">228</span>
+<span class="normal">229</span>
+<span class="normal">230</span>
+<span class="normal">231</span>
+<span class="normal">232</span>
+<span class="normal">233</span>
+<span class="normal">234</span>
+<span class="normal">235</span>
+<span class="normal">236</span>
+<span class="normal">237</span>
+<span class="normal">238</span>
+<span class="normal">239</span>
+<span class="normal">240</span>
+<span class="normal">241</span>
+<span class="normal">242</span>
+<span class="normal">243</span>
+<span class="normal">244</span>
+<span class="normal">245</span>
+<span class="normal">246</span>
+<span class="normal">247</span>
+<span class="normal">248</span>
+<span class="normal">249</span>
+<span class="normal">250</span>
+<span class="normal">251</span>
+<span class="normal">252</span>
+<span class="normal">253</span>
+<span class="normal">254</span>
+<span class="normal">255</span>
+<span class="normal">256</span>
+<span class="normal">257</span>
+<span class="normal">258</span>
+<span class="normal">259</span>
+<span class="normal">260</span>
+<span class="normal">261</span>
+<span class="normal">262</span>
+<span class="normal">263</span>
+<span class="normal">264</span>
+<span class="normal">265</span>
+<span class="normal">266</span>
+<span class="normal">267</span>
+<span class="normal">268</span>
+<span class="normal">269</span>
+<span class="normal">270</span>
+<span class="normal">271</span>
+<span class="normal">272</span>
+<span class="normal">273</span>
+<span class="normal">274</span>
+<span class="normal">275</span>
+<span class="normal">276</span>
+<span class="normal">277</span>
+<span class="normal">278</span>
+<span class="normal">279</span>
+<span class="normal">280</span>
+<span class="normal">281</span>
+<span class="normal">282</span>
+<span class="normal">283</span>
+<span class="normal">284</span>
+<span class="normal">285</span>
+<span class="normal">286</span>
+<span class="normal">287</span>
+<span class="normal">288</span>
+<span class="normal">289</span>
+<span class="normal">290</span>
+<span class="normal">291</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">class</span> <span class="nc">MultiTargetClassificationLoss</span><span class="p">(</span><span class="n">nn</span><span class="o">.</span><span class="n">Module</span><span class="p">):</span>
+<span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">    This class is a wrapper around the Pytorch binary_cross_entropy_with_logits and</span>
+<span class="sd">    cross_entropy losses. It allows for multi-target classification problems. The</span>
+<span class="sd">    user can provide a list of weights to apply to each target. The loss can be</span>
+<span class="sd">    either the sum or the mean of the individual losses</span>
+
+<span class="sd">    Parameters</span>
+<span class="sd">    ----------</span>
+<span class="sd">    binary_config: Optional[List[int | Tuple[int, float]]], default = None</span>
+<span class="sd">        List of integers with the index of the target for binary</span>
+<span class="sd">        classification or tuples with two elements: the index of the targets</span>
+<span class="sd">        or binary classification and the positive weight for binary</span>
+<span class="sd">        classification</span>
+<span class="sd">    multiclass_config: Optional[Tuple[int, int] | Tuple[int, int, List[float]]], default = None</span>
+<span class="sd">        List of tuples with two or three elements: the index of the target and the</span>
+<span class="sd">        number of classes for multiclass classification, or a tuple with the index of</span>
+<span class="sd">        the target, the number of classes and a list of weights to apply to each class</span>
+<span class="sd">        (i.e. the &#39;weight&#39; parameter in the cross_entropy loss)</span>
+<span class="sd">    weights: Optional[List[float], default = None]</span>
+<span class="sd">        List of weights to apply to the loss associated to each target. The</span>
+<span class="sd">        length of the list must match the number of targets.</span>
+<span class="sd">        Alias: &#39;target_weights&#39;</span>
+<span class="sd">    reduction: Literal[&quot;mean&quot;, &quot;sum&quot;], default = &quot;sum</span>
+<span class="sd">        Specifies the reduction to apply to the loss associated to each</span>
+<span class="sd">        target: &#39;mean&#39; | &#39;sum&#39;. Note that this is NOT the same as the</span>
+<span class="sd">        reduction in the cross_entropy loss or the</span>
+<span class="sd">        binary_cross_entropy_with_logits. This reduction is applied after the</span>
+<span class="sd">        loss for each target has been computed. Alias: &#39;target_reduction&#39;</span>
+<span class="sd">    binary_trick: bool, default = False</span>
+<span class="sd">        If True, each target will be considered independently and the loss</span>
+<span class="sd">        will be computed as binary_cross_entropy_with_logits. This is a</span>
+<span class="sd">        faster implementation. Note that the &#39;weights&#39; parameter is not</span>
+<span class="sd">        compatible with binary_trick=True. Also note that if</span>
+<span class="sd">        binary_trick=True, the &#39;binary_config&#39; must be a list of integers and</span>
+<span class="sd">        the &#39;multiclass_config&#39; must be a list of tuples with two integers:</span>
+<span class="sd">        the index of the target and the number of classes. Finally, if</span>
+<span class="sd">        binary_trick=True, the binary targets must be the first targets in</span>
+<span class="sd">        the target tensor.</span>
+
+<span class="sd">        :information_source: **NOTE**: When using the binary_trick, the binary targets are</span>
+<span class="sd">          considered as 2 classes. Therefore, the pred_dim parametere of the</span>
+<span class="sd">          WideDeep class should be adjusted accordingly (adding 2 to per</span>
+<span class="sd">          binary target). For example, in a problem with a binary target and</span>
+<span class="sd">          a 4 class multiclassification target, the pred_dim should be 6.</span>
+
+
+<span class="sd">    Examples</span>
+<span class="sd">    --------</span>
+<span class="sd">    &gt;&gt;&gt; import torch</span>
+<span class="sd">    &gt;&gt;&gt; from pytorch_widedeep.losses_multitarget import MultiTargetClassificationLoss</span>
+<span class="sd">    &gt;&gt;&gt; input = torch.randn(5, 4)</span>
+<span class="sd">    &gt;&gt;&gt; input_binary_trick = torch.randn(5, 5)</span>
+<span class="sd">    &gt;&gt;&gt; target = torch.stack([torch.tensor([0, 1, 0, 1, 1]), torch.tensor([0, 1, 2, 0, 2])], 1)</span>
+<span class="sd">    &gt;&gt;&gt; loss_1 = MultiTargetClassificationLoss(binary_config=[0], multiclass_config=[(1, 3)], reduction=&quot;mean&quot;)</span>
+<span class="sd">    &gt;&gt;&gt; output_1 = loss_1(input, target)</span>
+<span class="sd">    &gt;&gt;&gt; loss_2 = MultiTargetClassificationLoss(binary_config=[(0, 0.5)], multiclass_config=[(1, 3, [1., 2., 3.])],</span>
+<span class="sd">    ... reduction=&quot;sum&quot;, weights=[0.5, 0.5])</span>
+<span class="sd">    &gt;&gt;&gt; output_2 = loss_2(input, target)</span>
+<span class="sd">    &gt;&gt;&gt; loss_3 = MultiTargetClassificationLoss(binary_config=[0], multiclass_config=[(1, 3)], binary_trick=True)</span>
+<span class="sd">    &gt;&gt;&gt; output_3 = loss_3(input_binary_trick, target)</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+
+    <span class="nd">@alias</span><span class="p">(</span><span class="s2">&quot;reduction&quot;</span><span class="p">,</span> <span class="p">[</span><span class="s2">&quot;target_reduction&quot;</span><span class="p">])</span>
+    <span class="nd">@alias</span><span class="p">(</span><span class="s2">&quot;weights&quot;</span><span class="p">,</span> <span class="p">[</span><span class="s2">&quot;target_weights&quot;</span><span class="p">])</span>
+    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>  <span class="c1"># noqa: C901</span>
+        <span class="bp">self</span><span class="p">,</span>
+        <span class="n">binary_config</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="n">Union</span><span class="p">[</span><span class="nb">int</span><span class="p">,</span> <span class="n">Tuple</span><span class="p">[</span><span class="nb">int</span><span class="p">,</span> <span class="nb">float</span><span class="p">]]]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">multiclass_config</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span>
+            <span class="n">List</span><span class="p">[</span><span class="n">Union</span><span class="p">[</span><span class="n">Tuple</span><span class="p">[</span><span class="nb">int</span><span class="p">,</span> <span class="nb">int</span><span class="p">],</span> <span class="n">Tuple</span><span class="p">[</span><span class="nb">int</span><span class="p">,</span> <span class="nb">int</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="nb">float</span><span class="p">]]]]</span>
+        <span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">weights</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">float</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">reduction</span><span class="p">:</span> <span class="n">Literal</span><span class="p">[</span><span class="s2">&quot;mean&quot;</span><span class="p">,</span> <span class="s2">&quot;sum&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="s2">&quot;mean&quot;</span><span class="p">,</span>
+        <span class="n">binary_trick</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
+    <span class="p">):</span>
+        <span class="nb">super</span><span class="p">(</span><span class="n">MultiTargetClassificationLoss</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
+
+        <span class="k">if</span> <span class="n">reduction</span> <span class="ow">not</span> <span class="ow">in</span> <span class="p">[</span><span class="s2">&quot;mean&quot;</span><span class="p">,</span> <span class="s2">&quot;sum&quot;</span><span class="p">]:</span>
+            <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">&quot;reduction must be either &#39;mean&#39; or &#39;sum&#39;&quot;</span><span class="p">)</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">binary_config</span> <span class="o">=</span> <span class="n">binary_config</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">multiclass_config</span> <span class="o">=</span> <span class="n">multiclass_config</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">weights</span> <span class="o">=</span> <span class="n">weights</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">reduction</span> <span class="o">=</span> <span class="n">reduction</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">binary_trick</span> <span class="o">=</span> <span class="n">binary_trick</span>
+
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">weights</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">weights</span><span class="p">)</span> <span class="o">!=</span> <span class="p">(</span>
+                <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">binary_config</span><span class="p">)</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">binary_config</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="k">else</span> <span class="mi">0</span>
+            <span class="p">)</span> <span class="o">+</span> <span class="p">(</span>
+                <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">multiclass_config</span><span class="p">)</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">multiclass_config</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="k">else</span> <span class="mi">0</span>
+            <span class="p">):</span>
+                <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
+                    <span class="s2">&quot;The number of weights must match the number of binary and multiclass targets&quot;</span>
+                <span class="p">)</span>
+
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">binary_trick</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">_check_inputs_with_binary_trick</span><span class="p">()</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">_binary_config</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="n">binary_config</span>  <span class="c1"># type: ignore[assignment]</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">_multiclass_config</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="n">Tuple</span><span class="p">[</span><span class="nb">int</span><span class="p">,</span> <span class="nb">int</span><span class="p">]]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">multiclass_config</span>  <span class="c1"># type: ignore[assignment]</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">binary_config_with_pos_weights</span> <span class="o">=</span> <span class="p">(</span>
+                <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_set_binary_config_without_binary_trick</span><span class="p">())</span>
+                <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">binary_config</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span>
+                <span class="k">else</span> <span class="kc">None</span>
+            <span class="p">)</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">multiclass_config_with_weights</span> <span class="o">=</span> <span class="p">(</span>
+                <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_set_multiclass_config_without_binary_trick</span><span class="p">())</span>
+                <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">multiclass_config</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span>
+                <span class="k">else</span> <span class="kc">None</span>
             <span class="p">)</span>
 
-    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">binary_trick</span><span class="p">:</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">_check_inputs_with_binary_trick</span><span class="p">()</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">_binary_config</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="n">binary_config</span>  <span class="c1"># type: ignore[assignment]</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">_multiclass_config</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="n">Tuple</span><span class="p">[</span><span class="nb">int</span><span class="p">,</span> <span class="nb">int</span><span class="p">]]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">multiclass_config</span>  <span class="c1"># type: ignore[assignment]</span>
-    <span class="k">else</span><span class="p">:</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">binary_config_with_pos_weights</span> <span class="o">=</span> <span class="p">(</span>
-            <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_set_binary_config_without_binary_trick</span><span class="p">())</span>
-            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">binary_config</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span>
-            <span class="k">else</span> <span class="kc">None</span>
+    <span class="k">def</span> <span class="nf">forward</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="nb">input</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">,</span> <span class="n">target</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tensor</span><span class="p">:</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">binary_trick</span><span class="p">:</span>
+            <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_forward_binary_trick</span><span class="p">(</span><span class="nb">input</span><span class="p">,</span> <span class="n">target</span><span class="p">)</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_forward_without_binary_trick</span><span class="p">(</span><span class="nb">input</span><span class="p">,</span> <span class="n">target</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">_forward_binary_trick</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="nb">input</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">,</span> <span class="n">target</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tensor</span><span class="p">:</span>
+        <span class="n">binary_target_tensors</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="n">Tensor</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_binary_config</span><span class="p">:</span>
+            <span class="k">for</span> <span class="n">idx</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_binary_config</span><span class="p">:</span>
+                <span class="n">binary_target_tensors</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
+                    <span class="n">torch</span><span class="o">.</span><span class="n">eye</span><span class="p">(</span><span class="mi">2</span><span class="p">)[</span><span class="n">target</span><span class="p">[:,</span> <span class="n">idx</span><span class="p">]</span><span class="o">.</span><span class="n">long</span><span class="p">()]</span><span class="o">.</span><span class="n">to</span><span class="p">(</span><span class="nb">input</span><span class="o">.</span><span class="n">device</span><span class="p">)</span>
+                <span class="p">)</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_multiclass_config</span><span class="p">:</span>
+            <span class="k">for</span> <span class="n">idx</span><span class="p">,</span> <span class="n">n_classes</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_multiclass_config</span><span class="p">:</span>
+                <span class="n">binary_target_tensors</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
+                    <span class="n">torch</span><span class="o">.</span><span class="n">eye</span><span class="p">(</span><span class="n">n_classes</span><span class="p">)[</span><span class="n">target</span><span class="p">[:,</span> <span class="n">idx</span><span class="p">]</span><span class="o">.</span><span class="n">long</span><span class="p">()]</span><span class="o">.</span><span class="n">to</span><span class="p">(</span><span class="nb">input</span><span class="o">.</span><span class="n">device</span><span class="p">)</span>
+                <span class="p">)</span>
+        <span class="n">binary_target</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">cat</span><span class="p">(</span><span class="n">binary_target_tensors</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span>
+        <span class="k">return</span> <span class="n">F</span><span class="o">.</span><span class="n">binary_cross_entropy_with_logits</span><span class="p">(</span><span class="nb">input</span><span class="p">,</span> <span class="n">binary_target</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">_forward_without_binary_trick</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="nb">input</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">,</span> <span class="n">target</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tensor</span><span class="p">:</span>
+        <span class="n">losses</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="n">Tensor</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">binary_config_with_pos_weights</span><span class="p">:</span>
+            <span class="k">for</span> <span class="n">idx</span><span class="p">,</span> <span class="n">bpos_weight</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">binary_config_with_pos_weights</span><span class="p">:</span>
+                <span class="n">_loss</span> <span class="o">=</span> <span class="n">F</span><span class="o">.</span><span class="n">binary_cross_entropy_with_logits</span><span class="p">(</span>
+                    <span class="nb">input</span><span class="p">[:,</span> <span class="n">idx</span><span class="p">],</span>
+                    <span class="n">target</span><span class="p">[:,</span> <span class="n">idx</span><span class="p">]</span><span class="o">.</span><span class="n">float</span><span class="p">(),</span>
+                    <span class="n">pos_weight</span><span class="o">=</span><span class="p">(</span>
+                        <span class="n">torch</span><span class="o">.</span><span class="n">tensor</span><span class="p">(</span><span class="n">bpos_weight</span><span class="p">)</span><span class="o">.</span><span class="n">to</span><span class="p">(</span><span class="nb">input</span><span class="o">.</span><span class="n">device</span><span class="p">)</span>
+                        <span class="k">if</span> <span class="n">bpos_weight</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span>
+                        <span class="k">else</span> <span class="kc">None</span>
+                    <span class="p">),</span>
+                <span class="p">)</span>
+                <span class="n">losses</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_loss</span><span class="p">)</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">multiclass_config_with_weights</span><span class="p">:</span>
+            <span class="k">for</span> <span class="n">idx</span><span class="p">,</span> <span class="n">n_classes</span><span class="p">,</span> <span class="n">mpos_weight</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">multiclass_config_with_weights</span><span class="p">:</span>
+                <span class="n">_loss</span> <span class="o">=</span> <span class="n">F</span><span class="o">.</span><span class="n">cross_entropy</span><span class="p">(</span>
+                    <span class="nb">input</span><span class="p">[:,</span> <span class="n">idx</span> <span class="p">:</span> <span class="n">idx</span> <span class="o">+</span> <span class="n">n_classes</span><span class="p">],</span>
+                    <span class="n">target</span><span class="p">[:,</span> <span class="n">idx</span><span class="p">]</span><span class="o">.</span><span class="n">long</span><span class="p">(),</span>
+                    <span class="n">weight</span><span class="o">=</span><span class="p">(</span>
+                        <span class="n">torch</span><span class="o">.</span><span class="n">tensor</span><span class="p">(</span><span class="n">mpos_weight</span><span class="p">)</span><span class="o">.</span><span class="n">to</span><span class="p">(</span><span class="nb">input</span><span class="o">.</span><span class="n">device</span><span class="p">)</span>
+                        <span class="k">if</span> <span class="n">mpos_weight</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span>
+                        <span class="k">else</span> <span class="kc">None</span>
+                    <span class="p">),</span>
+                <span class="p">)</span>
+                <span class="n">losses</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_loss</span><span class="p">)</span>
+
+            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">weights</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+                <span class="n">losses</span> <span class="o">=</span> <span class="p">[</span><span class="n">l</span> <span class="o">*</span> <span class="n">w</span> <span class="k">for</span> <span class="n">l</span><span class="p">,</span> <span class="n">w</span> <span class="ow">in</span> <span class="nb">zip</span><span class="p">(</span><span class="n">losses</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">weights</span><span class="p">)]</span>  <span class="c1"># noqa: E741</span>
+
+        <span class="k">return</span> <span class="p">(</span>
+            <span class="n">torch</span><span class="o">.</span><span class="n">stack</span><span class="p">(</span><span class="n">losses</span><span class="p">)</span><span class="o">.</span><span class="n">sum</span><span class="p">()</span>
+            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">reduction</span> <span class="o">==</span> <span class="s2">&quot;sum&quot;</span>
+            <span class="k">else</span> <span class="n">torch</span><span class="o">.</span><span class="n">stack</span><span class="p">(</span><span class="n">losses</span><span class="p">)</span><span class="o">.</span><span class="n">mean</span><span class="p">()</span>
         <span class="p">)</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">multiclass_config_with_weights</span> <span class="o">=</span> <span class="p">(</span>
-            <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_set_multiclass_config_without_binary_trick</span><span class="p">())</span>
-            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">multiclass_config</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span>
-            <span class="k">else</span> <span class="kc">None</span>
+
+    <span class="k">def</span> <span class="nf">_check_inputs_with_binary_trick</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">binary_config</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="k">if</span> <span class="nb">any</span><span class="p">(</span><span class="nb">isinstance</span><span class="p">(</span><span class="n">bc</span><span class="p">,</span> <span class="nb">tuple</span><span class="p">)</span> <span class="k">for</span> <span class="n">bc</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">binary_config</span><span class="p">):</span>
+                <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
+                    <span class="s2">&quot;binary_trick=True is only compatible with binary_config as a list of integers&quot;</span>
+                <span class="p">)</span>
+
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">multiclass_config</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="k">if</span> <span class="ow">not</span> <span class="nb">all</span><span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">mc</span><span class="p">)</span> <span class="o">==</span> <span class="mi">2</span> <span class="k">for</span> <span class="n">mc</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">multiclass_config</span><span class="p">):</span>
+                <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
+                    <span class="s2">&quot;binary_trick=True is only compatible with multiclass_config as a list of &quot;</span>
+                    <span class="s2">&quot;tuples with two integers: the index of the target and the number of classes&quot;</span>
+                <span class="p">)</span>
+
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">binary_config</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">multiclass_config</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="n">last_binary_idx</span> <span class="o">=</span> <span class="p">(</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">binary_config</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">][</span><span class="mi">0</span><span class="p">]</span>
+                <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">binary_config</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">],</span> <span class="nb">tuple</span><span class="p">)</span>
+                <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">binary_config</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span>
+            <span class="p">)</span>
+            <span class="k">if</span> <span class="n">last_binary_idx</span> <span class="o">&gt;=</span> <span class="bp">self</span><span class="o">.</span><span class="n">multiclass_config</span><span class="p">[</span><span class="mi">0</span><span class="p">][</span><span class="mi">0</span><span class="p">]:</span>
+                <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
+                    <span class="s2">&quot;When using binary_trick=True, the binary targets must be the first targets&quot;</span>
+                    <span class="s2">&quot; in the target tensor&quot;</span>
+                <span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">_set_binary_config_without_binary_trick</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span>
+    <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">List</span><span class="p">[</span><span class="n">Tuple</span><span class="p">[</span><span class="nb">int</span><span class="p">,</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]]]:</span>
+        <span class="n">binary_config_with_pos_weights</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="n">Tuple</span><span class="p">[</span><span class="nb">int</span><span class="p">,</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]]]</span> <span class="o">=</span> <span class="p">[]</span>
+        <span class="k">for</span> <span class="n">bc</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">binary_config</span><span class="p">:</span>
+            <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">bc</span><span class="p">,</span> <span class="nb">tuple</span><span class="p">):</span>
+                <span class="n">binary_config_with_pos_weights</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">bc</span><span class="p">)</span>
+            <span class="k">else</span><span class="p">:</span>
+                <span class="n">binary_config_with_pos_weights</span><span class="o">.</span><span class="n">append</span><span class="p">((</span><span class="n">bc</span><span class="p">,</span> <span class="kc">None</span><span class="p">))</span>
+        <span class="k">return</span> <span class="n">binary_config_with_pos_weights</span>
+
+    <span class="k">def</span> <span class="nf">_set_multiclass_config_without_binary_trick</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span>
+    <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">List</span><span class="p">[</span><span class="n">Tuple</span><span class="p">[</span><span class="nb">int</span><span class="p">,</span> <span class="nb">int</span><span class="p">,</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">float</span><span class="p">]]]]:</span>
+        <span class="n">multiclass_config_with_weights</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="n">Tuple</span><span class="p">[</span><span class="nb">int</span><span class="p">,</span> <span class="nb">int</span><span class="p">,</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">float</span><span class="p">]]]]</span> <span class="o">=</span> <span class="p">(</span>
+            <span class="p">[]</span>
         <span class="p">)</span>
+        <span class="k">for</span> <span class="n">mc</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">multiclass_config</span><span class="p">:</span>
+            <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">mc</span><span class="p">)</span> <span class="o">==</span> <span class="mi">3</span><span class="p">:</span>
+                <span class="n">multiclass_config_with_weights</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">mc</span><span class="p">)</span>  <span class="c1"># type: ignore[arg-type]</span>
+            <span class="k">else</span><span class="p">:</span>
+                <span class="n">multiclass_config_with_weights</span><span class="o">.</span><span class="n">append</span><span class="p">((</span><span class="n">mc</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="n">mc</span><span class="p">[</span><span class="mi">1</span><span class="p">],</span> <span class="kc">None</span><span class="p">))</span>
+        <span class="k">return</span> <span class="n">multiclass_config_with_weights</span>
 </code></pre></div></td></tr></table></div>
-                  </details>
+              </details>
 
 
 
@@ -5335,15 +7982,7 @@ <h2 id="pytorch_widedeep.losses_multitarget.MutilTargetRegressionAndClassificati
 
 
 <a href="#pytorch_widedeep.losses_multitarget.MutilTargetRegressionAndClassificationLoss" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="nf">MutilTargetRegressionAndClassificationLoss</span><span class="p">(</span>
-    <span class="n">regression_config</span><span class="o">=</span><span class="p">[],</span>
-    <span class="n">binary_config</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">multiclass_config</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">weights</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">reduction</span><span class="o">=</span><span class="s2">&quot;mean&quot;</span><span class="p">,</span>
-    <span class="n">binary_trick</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
-<span class="p">)</span>
-</code></pre></div>
+
 
     <div class="doc doc-contents first">
             <p class="doc doc-class-bases">
@@ -5357,76 +7996,105 @@ <h2 id="pytorch_widedeep.losses_multitarget.MutilTargetRegressionAndClassificati
 
 
 <p><span class="doc-section-title">Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>regression_config</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.List">List</span>[int]</code>, default:
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+          <th>Default</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>regression_config</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.List">List</span>[int]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>List of integers with the indices of the regression targets</p>
+              </div>
+            </td>
+            <td>
                   <code>[]</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>List of integers with the indices of the regression targets</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>binary_config</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[int, <span title="pytorch_widedeep.wdtypes.Tuple">Tuple</span>[int, float]]]]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>List of integers with the index of the target for binary
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>binary_config</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[int, <span title="pytorch_widedeep.wdtypes.Tuple">Tuple</span>[int, float]]]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>List of integers with the index of the target for binary
 classification or tuples with two elements: the index of the targets
 or binary classification and the positive weight for binary
 classification</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>multiclass_config</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="pytorch_widedeep.wdtypes.Tuple">Tuple</span>[int, int], <span title="pytorch_widedeep.wdtypes.Tuple">Tuple</span>[int, int, <span title="pytorch_widedeep.wdtypes.List">List</span>[float]]]]]</code>, default:
+              </div>
+            </td>
+            <td>
                   <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>List of tuples with two or three elements: the index of the target and the
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>multiclass_config</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="pytorch_widedeep.wdtypes.Tuple">Tuple</span>[int, int], <span title="pytorch_widedeep.wdtypes.Tuple">Tuple</span>[int, int, <span title="pytorch_widedeep.wdtypes.List">List</span>[float]]]]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>List of tuples with two or three elements: the index of the target and the
 number of classes for multiclass classification, or a tuple with the index of
 the target, the number of classes and a list of weights to apply to each class
 (i.e. the 'weight' parameter in the cross_entropy loss)</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>weights</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[float]]</code>, default:
+              </div>
+            </td>
+            <td>
                   <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>List of weights to apply to the loss associated to each target. The
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>weights</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[float]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>List of weights to apply to the loss associated to each target. The
 length of the list must match the number of targets.
 Alias: 'target_weights'</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>reduction</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[mean, sum]</code>, default:
-                  <code>&#39;mean&#39;</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Specifies the reduction to apply to the output: 'mean' | 'sum'. Note
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>reduction</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[mean, sum]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Specifies the reduction to apply to the output: 'mean' | 'sum'. Note
 that this is NOT the same as the reduction in the cross_entropy loss,
 the binary_cross_entropy_with_logits or the MSELoss. This reduction
 is applied after each target has been computed. Alias: 'target_reduction'</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>binary_trick</code></b>
-              (<code>bool</code>, default:
-                  <code>False</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>If True, each target will be considered independently and the loss
+              </div>
+            </td>
+            <td>
+                  <code>&#39;mean&#39;</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>binary_trick</code></td>
+            <td>
+                  <code>bool</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>If True, each target will be considered independently and the loss
 will be computed as binary_cross_entropy_with_logits. This is a
 faster implementation. Note that the 'weights' parameter is not
 compatible with binary_trick=True. Also note that if
@@ -5440,9 +8108,14 @@ <h2 id="pytorch_widedeep.losses_multitarget.MutilTargetRegressionAndClassificati
   WideDeep class should be adjusted accordingly (adding 2 to per
   binary target). For example, in a problem with a binary target and
   a 4 class multiclassification target, the pred_dim should be 6.</p>
-          </div>
-        </li>
-    </ul>
+              </div>
+            </td>
+            <td>
+                  <code>False</code>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
 
 <p><span class="doc-section-title">Examples:</span></p>
@@ -5455,9 +8128,67 @@ <h2 id="pytorch_widedeep.losses_multitarget.MutilTargetRegressionAndClassificati
 <span class="gp">&gt;&gt;&gt; </span><span class="n">output</span> <span class="o">=</span> <span class="n">loss</span><span class="p">(</span><span class="nb">input</span><span class="p">,</span> <span class="n">target</span><span class="p">)</span>
 </code></pre></div>
 
-                  <details class="quote">
-                    <summary>Source code in <code>pytorch_widedeep/losses_multitarget.py</code></summary>
-                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">352</span>
+              <details class="quote">
+                <summary>Source code in <code>pytorch_widedeep/losses_multitarget.py</code></summary>
+                <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">294</span>
+<span class="normal">295</span>
+<span class="normal">296</span>
+<span class="normal">297</span>
+<span class="normal">298</span>
+<span class="normal">299</span>
+<span class="normal">300</span>
+<span class="normal">301</span>
+<span class="normal">302</span>
+<span class="normal">303</span>
+<span class="normal">304</span>
+<span class="normal">305</span>
+<span class="normal">306</span>
+<span class="normal">307</span>
+<span class="normal">308</span>
+<span class="normal">309</span>
+<span class="normal">310</span>
+<span class="normal">311</span>
+<span class="normal">312</span>
+<span class="normal">313</span>
+<span class="normal">314</span>
+<span class="normal">315</span>
+<span class="normal">316</span>
+<span class="normal">317</span>
+<span class="normal">318</span>
+<span class="normal">319</span>
+<span class="normal">320</span>
+<span class="normal">321</span>
+<span class="normal">322</span>
+<span class="normal">323</span>
+<span class="normal">324</span>
+<span class="normal">325</span>
+<span class="normal">326</span>
+<span class="normal">327</span>
+<span class="normal">328</span>
+<span class="normal">329</span>
+<span class="normal">330</span>
+<span class="normal">331</span>
+<span class="normal">332</span>
+<span class="normal">333</span>
+<span class="normal">334</span>
+<span class="normal">335</span>
+<span class="normal">336</span>
+<span class="normal">337</span>
+<span class="normal">338</span>
+<span class="normal">339</span>
+<span class="normal">340</span>
+<span class="normal">341</span>
+<span class="normal">342</span>
+<span class="normal">343</span>
+<span class="normal">344</span>
+<span class="normal">345</span>
+<span class="normal">346</span>
+<span class="normal">347</span>
+<span class="normal">348</span>
+<span class="normal">349</span>
+<span class="normal">350</span>
+<span class="normal">351</span>
+<span class="normal">352</span>
 <span class="normal">353</span>
 <span class="normal">354</span>
 <span class="normal">355</span>
@@ -5529,81 +8260,371 @@ <h2 id="pytorch_widedeep.losses_multitarget.MutilTargetRegressionAndClassificati
 <span class="normal">421</span>
 <span class="normal">422</span>
 <span class="normal">423</span>
-<span class="normal">424</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="nd">@alias</span><span class="p">(</span><span class="s2">&quot;reduction&quot;</span><span class="p">,</span> <span class="p">[</span><span class="s2">&quot;target_reduction&quot;</span><span class="p">])</span>
-<span class="nd">@alias</span><span class="p">(</span><span class="s2">&quot;weights&quot;</span><span class="p">,</span> <span class="p">[</span><span class="s2">&quot;target_weights&quot;</span><span class="p">])</span>
-<span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>  <span class="c1"># noqa: C901</span>
-    <span class="bp">self</span><span class="p">,</span>
-    <span class="n">regression_config</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="p">[],</span>
-    <span class="n">binary_config</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="n">Union</span><span class="p">[</span><span class="nb">int</span><span class="p">,</span> <span class="n">Tuple</span><span class="p">[</span><span class="nb">int</span><span class="p">,</span> <span class="nb">float</span><span class="p">]]]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">multiclass_config</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span>
-        <span class="n">List</span><span class="p">[</span><span class="n">Union</span><span class="p">[</span><span class="n">Tuple</span><span class="p">[</span><span class="nb">int</span><span class="p">,</span> <span class="nb">int</span><span class="p">],</span> <span class="n">Tuple</span><span class="p">[</span><span class="nb">int</span><span class="p">,</span> <span class="nb">int</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="nb">float</span><span class="p">]]]]</span>
-    <span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">weights</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">float</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">reduction</span><span class="p">:</span> <span class="n">Literal</span><span class="p">[</span><span class="s2">&quot;mean&quot;</span><span class="p">,</span> <span class="s2">&quot;sum&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="s2">&quot;mean&quot;</span><span class="p">,</span>
-    <span class="n">binary_trick</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
-<span class="p">):</span>
-
-    <span class="nb">super</span><span class="p">(</span><span class="n">MutilTargetRegressionAndClassificationLoss</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
-
-    <span class="bp">self</span><span class="o">.</span><span class="n">regression_config</span> <span class="o">=</span> <span class="n">regression_config</span>
-
-    <span class="k">assert</span> <span class="n">binary_config</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">or</span> <span class="n">multiclass_config</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">,</span> <span class="p">(</span>
-        <span class="s2">&quot;Either binary_config or multiclass_config must be provided. &quot;</span>
-        <span class="s2">&quot;Otherwise, use the MultiTargetRegressionLoss&quot;</span>
-    <span class="p">)</span>
+<span class="normal">424</span>
+<span class="normal">425</span>
+<span class="normal">426</span>
+<span class="normal">427</span>
+<span class="normal">428</span>
+<span class="normal">429</span>
+<span class="normal">430</span>
+<span class="normal">431</span>
+<span class="normal">432</span>
+<span class="normal">433</span>
+<span class="normal">434</span>
+<span class="normal">435</span>
+<span class="normal">436</span>
+<span class="normal">437</span>
+<span class="normal">438</span>
+<span class="normal">439</span>
+<span class="normal">440</span>
+<span class="normal">441</span>
+<span class="normal">442</span>
+<span class="normal">443</span>
+<span class="normal">444</span>
+<span class="normal">445</span>
+<span class="normal">446</span>
+<span class="normal">447</span>
+<span class="normal">448</span>
+<span class="normal">449</span>
+<span class="normal">450</span>
+<span class="normal">451</span>
+<span class="normal">452</span>
+<span class="normal">453</span>
+<span class="normal">454</span>
+<span class="normal">455</span>
+<span class="normal">456</span>
+<span class="normal">457</span>
+<span class="normal">458</span>
+<span class="normal">459</span>
+<span class="normal">460</span>
+<span class="normal">461</span>
+<span class="normal">462</span>
+<span class="normal">463</span>
+<span class="normal">464</span>
+<span class="normal">465</span>
+<span class="normal">466</span>
+<span class="normal">467</span>
+<span class="normal">468</span>
+<span class="normal">469</span>
+<span class="normal">470</span>
+<span class="normal">471</span>
+<span class="normal">472</span>
+<span class="normal">473</span>
+<span class="normal">474</span>
+<span class="normal">475</span>
+<span class="normal">476</span>
+<span class="normal">477</span>
+<span class="normal">478</span>
+<span class="normal">479</span>
+<span class="normal">480</span>
+<span class="normal">481</span>
+<span class="normal">482</span>
+<span class="normal">483</span>
+<span class="normal">484</span>
+<span class="normal">485</span>
+<span class="normal">486</span>
+<span class="normal">487</span>
+<span class="normal">488</span>
+<span class="normal">489</span>
+<span class="normal">490</span>
+<span class="normal">491</span>
+<span class="normal">492</span>
+<span class="normal">493</span>
+<span class="normal">494</span>
+<span class="normal">495</span>
+<span class="normal">496</span>
+<span class="normal">497</span>
+<span class="normal">498</span>
+<span class="normal">499</span>
+<span class="normal">500</span>
+<span class="normal">501</span>
+<span class="normal">502</span>
+<span class="normal">503</span>
+<span class="normal">504</span>
+<span class="normal">505</span>
+<span class="normal">506</span>
+<span class="normal">507</span>
+<span class="normal">508</span>
+<span class="normal">509</span>
+<span class="normal">510</span>
+<span class="normal">511</span>
+<span class="normal">512</span>
+<span class="normal">513</span>
+<span class="normal">514</span>
+<span class="normal">515</span>
+<span class="normal">516</span>
+<span class="normal">517</span>
+<span class="normal">518</span>
+<span class="normal">519</span>
+<span class="normal">520</span>
+<span class="normal">521</span>
+<span class="normal">522</span>
+<span class="normal">523</span>
+<span class="normal">524</span>
+<span class="normal">525</span>
+<span class="normal">526</span>
+<span class="normal">527</span>
+<span class="normal">528</span>
+<span class="normal">529</span>
+<span class="normal">530</span>
+<span class="normal">531</span>
+<span class="normal">532</span>
+<span class="normal">533</span>
+<span class="normal">534</span>
+<span class="normal">535</span>
+<span class="normal">536</span>
+<span class="normal">537</span>
+<span class="normal">538</span>
+<span class="normal">539</span>
+<span class="normal">540</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">class</span> <span class="nc">MutilTargetRegressionAndClassificationLoss</span><span class="p">(</span><span class="n">nn</span><span class="o">.</span><span class="n">Module</span><span class="p">):</span>
+<span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">    This class is a wrapper around the MultiTargetRegressionLoss and the</span>
+<span class="sd">    MultiTargetClassificationLoss. It allows for multi-target regression and</span>
+<span class="sd">    classification problems. The user can provide a list of weights to apply to</span>
+<span class="sd">    each target. The loss can be either the sum or the mean of the individual losses</span>
+
+<span class="sd">    Parameters</span>
+<span class="sd">    ----------</span>
+<span class="sd">    regression_config: List[int], default = []</span>
+<span class="sd">        List of integers with the indices of the regression targets</span>
+<span class="sd">    binary_config: Optional[List[int | Tuple[int, float]]], default = None</span>
+<span class="sd">        List of integers with the index of the target for binary</span>
+<span class="sd">        classification or tuples with two elements: the index of the targets</span>
+<span class="sd">        or binary classification and the positive weight for binary</span>
+<span class="sd">        classification</span>
+<span class="sd">    multiclass_config: Optional[Tuple[int, int] | Tuple[int, int, List[float]]], default = None</span>
+<span class="sd">        List of tuples with two or three elements: the index of the target and the</span>
+<span class="sd">        number of classes for multiclass classification, or a tuple with the index of</span>
+<span class="sd">        the target, the number of classes and a list of weights to apply to each class</span>
+<span class="sd">        (i.e. the &#39;weight&#39; parameter in the cross_entropy loss)</span>
+<span class="sd">    weights: Optional[List[float], default = None]</span>
+<span class="sd">        List of weights to apply to the loss associated to each target. The</span>
+<span class="sd">        length of the list must match the number of targets.</span>
+<span class="sd">        Alias: &#39;target_weights&#39;</span>
+<span class="sd">    reduction: Literal[&quot;mean&quot;, &quot;sum&quot;], default = &quot;sum</span>
+<span class="sd">        Specifies the reduction to apply to the output: &#39;mean&#39; | &#39;sum&#39;. Note</span>
+<span class="sd">        that this is NOT the same as the reduction in the cross_entropy loss,</span>
+<span class="sd">        the binary_cross_entropy_with_logits or the MSELoss. This reduction</span>
+<span class="sd">        is applied after each target has been computed. Alias: &#39;target_reduction&#39;</span>
+<span class="sd">    binary_trick: bool, default = False</span>
+<span class="sd">        If True, each target will be considered independently and the loss</span>
+<span class="sd">        will be computed as binary_cross_entropy_with_logits. This is a</span>
+<span class="sd">        faster implementation. Note that the &#39;weights&#39; parameter is not</span>
+<span class="sd">        compatible with binary_trick=True. Also note that if</span>
+<span class="sd">        binary_trick=True, the &#39;binary_config&#39; must be a list of integers and</span>
+<span class="sd">        the &#39;multiclass_config&#39; must be a list of tuples with two integers:</span>
+<span class="sd">        the index of the target and the number of classes. Finally, if</span>
+<span class="sd">        binary_trick=True, the binary targets must be the first targets in</span>
+<span class="sd">        the target tensor.</span>
+
+<span class="sd">        :information_source: **NOTE**: When using the binary_trick, the binary targets are</span>
+<span class="sd">          considered as 2 classes. Therefore, the pred_dim parametere of the</span>
+<span class="sd">          WideDeep class should be adjusted accordingly (adding 2 to per</span>
+<span class="sd">          binary target). For example, in a problem with a binary target and</span>
+<span class="sd">          a 4 class multiclassification target, the pred_dim should be 6.</span>
 
-    <span class="k">if</span> <span class="n">binary_trick</span><span class="p">:</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">_check_inputs_with_binary_trick</span><span class="p">(</span>
-            <span class="n">regression_config</span><span class="p">,</span> <span class="n">binary_config</span><span class="p">,</span> <span class="n">multiclass_config</span>
+<span class="sd">    Examples</span>
+<span class="sd">    --------</span>
+<span class="sd">    &gt;&gt;&gt; import torch</span>
+<span class="sd">    &gt;&gt;&gt; from pytorch_widedeep.losses_multitarget import MutilTargetRegressionAndClassificationLoss</span>
+<span class="sd">    &gt;&gt;&gt; input = torch.randn(5, 5)</span>
+<span class="sd">    &gt;&gt;&gt; target = torch.stack([torch.randn(5), torch.tensor([0, 1, 0, 1, 1]), torch.tensor([0, 1, 2, 0, 2])], 1)</span>
+<span class="sd">    &gt;&gt;&gt; loss = MutilTargetRegressionAndClassificationLoss(regression_config=[0], binary_config=[2],</span>
+<span class="sd">    ... multiclass_config=[(2, 3)], reduction=&quot;mean&quot;)</span>
+<span class="sd">    &gt;&gt;&gt; output = loss(input, target)</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+
+    <span class="nd">@alias</span><span class="p">(</span><span class="s2">&quot;reduction&quot;</span><span class="p">,</span> <span class="p">[</span><span class="s2">&quot;target_reduction&quot;</span><span class="p">])</span>
+    <span class="nd">@alias</span><span class="p">(</span><span class="s2">&quot;weights&quot;</span><span class="p">,</span> <span class="p">[</span><span class="s2">&quot;target_weights&quot;</span><span class="p">])</span>
+    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>  <span class="c1"># noqa: C901</span>
+        <span class="bp">self</span><span class="p">,</span>
+        <span class="n">regression_config</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="p">[],</span>
+        <span class="n">binary_config</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="n">Union</span><span class="p">[</span><span class="nb">int</span><span class="p">,</span> <span class="n">Tuple</span><span class="p">[</span><span class="nb">int</span><span class="p">,</span> <span class="nb">float</span><span class="p">]]]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">multiclass_config</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span>
+            <span class="n">List</span><span class="p">[</span><span class="n">Union</span><span class="p">[</span><span class="n">Tuple</span><span class="p">[</span><span class="nb">int</span><span class="p">,</span> <span class="nb">int</span><span class="p">],</span> <span class="n">Tuple</span><span class="p">[</span><span class="nb">int</span><span class="p">,</span> <span class="nb">int</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="nb">float</span><span class="p">]]]]</span>
+        <span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">weights</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">float</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">reduction</span><span class="p">:</span> <span class="n">Literal</span><span class="p">[</span><span class="s2">&quot;mean&quot;</span><span class="p">,</span> <span class="s2">&quot;sum&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="s2">&quot;mean&quot;</span><span class="p">,</span>
+        <span class="n">binary_trick</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
+    <span class="p">):</span>
+
+        <span class="nb">super</span><span class="p">(</span><span class="n">MutilTargetRegressionAndClassificationLoss</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">regression_config</span> <span class="o">=</span> <span class="n">regression_config</span>
+
+        <span class="k">assert</span> <span class="n">binary_config</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">or</span> <span class="n">multiclass_config</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">,</span> <span class="p">(</span>
+            <span class="s2">&quot;Either binary_config or multiclass_config must be provided. &quot;</span>
+            <span class="s2">&quot;Otherwise, use the MultiTargetRegressionLoss&quot;</span>
         <span class="p">)</span>
 
-    <span class="k">if</span> <span class="n">weights</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
-        <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">weights</span><span class="p">)</span> <span class="o">!=</span> <span class="p">(</span>
-            <span class="nb">len</span><span class="p">(</span><span class="n">regression_config</span><span class="p">)</span>
-            <span class="o">+</span> <span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">binary_config</span><span class="p">)</span> <span class="k">if</span> <span class="n">binary_config</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="k">else</span> <span class="mi">0</span><span class="p">)</span>
-            <span class="o">+</span> <span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">multiclass_config</span><span class="p">)</span> <span class="k">if</span> <span class="n">multiclass_config</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="k">else</span> <span class="mi">0</span><span class="p">)</span>
-        <span class="p">):</span>
-            <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
-                <span class="s2">&quot;The number of weights must match the number of regression, binary and multiclass targets&quot;</span>
+        <span class="k">if</span> <span class="n">binary_trick</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">_check_inputs_with_binary_trick</span><span class="p">(</span>
+                <span class="n">regression_config</span><span class="p">,</span> <span class="n">binary_config</span><span class="p">,</span> <span class="n">multiclass_config</span>
             <span class="p">)</span>
 
-        <span class="bp">self</span><span class="o">.</span><span class="n">weights_regression</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_prepare_weights_for_regression_targets</span><span class="p">(</span>
-            <span class="n">weights</span><span class="p">,</span> <span class="n">regression_config</span>
-        <span class="p">)</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">weights_binary</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_prepare_weights_per_binary_targets</span><span class="p">(</span>
-            <span class="n">weights</span><span class="p">,</span> <span class="n">binary_config</span>
+        <span class="k">if</span> <span class="n">weights</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">weights</span><span class="p">)</span> <span class="o">!=</span> <span class="p">(</span>
+                <span class="nb">len</span><span class="p">(</span><span class="n">regression_config</span><span class="p">)</span>
+                <span class="o">+</span> <span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">binary_config</span><span class="p">)</span> <span class="k">if</span> <span class="n">binary_config</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="k">else</span> <span class="mi">0</span><span class="p">)</span>
+                <span class="o">+</span> <span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">multiclass_config</span><span class="p">)</span> <span class="k">if</span> <span class="n">multiclass_config</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="k">else</span> <span class="mi">0</span><span class="p">)</span>
+            <span class="p">):</span>
+                <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
+                    <span class="s2">&quot;The number of weights must match the number of regression, binary and multiclass targets&quot;</span>
+                <span class="p">)</span>
+
+            <span class="bp">self</span><span class="o">.</span><span class="n">weights_regression</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_prepare_weights_for_regression_targets</span><span class="p">(</span>
+                <span class="n">weights</span><span class="p">,</span> <span class="n">regression_config</span>
+            <span class="p">)</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">weights_binary</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_prepare_weights_per_binary_targets</span><span class="p">(</span>
+                <span class="n">weights</span><span class="p">,</span> <span class="n">binary_config</span>
+            <span class="p">)</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">weights_multiclass</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_prepare_weights_per_multiclass_targets</span><span class="p">(</span>
+                <span class="n">weights</span><span class="p">,</span> <span class="n">multiclass_config</span>
+            <span class="p">)</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">weights</span> <span class="o">=</span> <span class="n">weights</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">weights_regression</span> <span class="o">=</span> <span class="kc">None</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">weights_binary</span> <span class="o">=</span> <span class="kc">None</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">weights_multiclass</span> <span class="o">=</span> <span class="kc">None</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">multi_target_regression_loss</span> <span class="o">=</span> <span class="n">MultiTargetRegressionLoss</span><span class="p">(</span>
+            <span class="n">weights</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">weights_regression</span><span class="p">,</span> <span class="n">reduction</span><span class="o">=</span><span class="n">reduction</span>
         <span class="p">)</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">weights_multiclass</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_prepare_weights_per_multiclass_targets</span><span class="p">(</span>
-            <span class="n">weights</span><span class="p">,</span> <span class="n">multiclass_config</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">multi_target_classification_loss</span> <span class="o">=</span> <span class="n">MultiTargetClassificationLoss</span><span class="p">(</span>
+            <span class="n">binary_config</span><span class="o">=</span><span class="n">binary_config</span><span class="p">,</span>
+            <span class="n">multiclass_config</span><span class="o">=</span><span class="n">multiclass_config</span><span class="p">,</span>
+            <span class="n">weights</span><span class="o">=</span><span class="p">(</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">weights_binary</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">weights_multiclass</span>
+                <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">weights_binary</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span>
+                <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">weights_multiclass</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span>
+                <span class="k">else</span> <span class="p">(</span>
+                    <span class="bp">self</span><span class="o">.</span><span class="n">weights_binary</span>
+                    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">weights_binary</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span>
+                    <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">weights_multiclass</span>
+                <span class="p">)</span>
+            <span class="p">),</span>
+            <span class="n">reduction</span><span class="o">=</span><span class="n">reduction</span><span class="p">,</span>
+            <span class="n">binary_trick</span><span class="o">=</span><span class="n">binary_trick</span><span class="p">,</span>
         <span class="p">)</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">weights</span> <span class="o">=</span> <span class="n">weights</span>
-    <span class="k">else</span><span class="p">:</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">weights_regression</span> <span class="o">=</span> <span class="kc">None</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">weights_binary</span> <span class="o">=</span> <span class="kc">None</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">weights_multiclass</span> <span class="o">=</span> <span class="kc">None</span>
 
-    <span class="bp">self</span><span class="o">.</span><span class="n">multi_target_regression_loss</span> <span class="o">=</span> <span class="n">MultiTargetRegressionLoss</span><span class="p">(</span>
-        <span class="n">weights</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">weights_regression</span><span class="p">,</span> <span class="n">reduction</span><span class="o">=</span><span class="n">reduction</span>
-    <span class="p">)</span>
+    <span class="k">def</span> <span class="nf">forward</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="nb">input</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">,</span> <span class="n">target</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tensor</span><span class="p">:</span>
 
-    <span class="bp">self</span><span class="o">.</span><span class="n">multi_target_classification_loss</span> <span class="o">=</span> <span class="n">MultiTargetClassificationLoss</span><span class="p">(</span>
-        <span class="n">binary_config</span><span class="o">=</span><span class="n">binary_config</span><span class="p">,</span>
-        <span class="n">multiclass_config</span><span class="o">=</span><span class="n">multiclass_config</span><span class="p">,</span>
-        <span class="n">weights</span><span class="o">=</span><span class="p">(</span>
-            <span class="bp">self</span><span class="o">.</span><span class="n">weights_binary</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">weights_multiclass</span>
-            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">weights_binary</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span>
-            <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">weights_multiclass</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span>
-            <span class="k">else</span> <span class="p">(</span>
-                <span class="bp">self</span><span class="o">.</span><span class="n">weights_binary</span>
-                <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">weights_binary</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span>
-                <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">weights_multiclass</span>
+        <span class="n">regression_loss</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">multi_target_regression_loss</span><span class="p">(</span>
+            <span class="nb">input</span><span class="p">[:,</span> <span class="bp">self</span><span class="o">.</span><span class="n">regression_config</span><span class="p">],</span>
+            <span class="n">target</span><span class="p">[:,</span> <span class="bp">self</span><span class="o">.</span><span class="n">regression_config</span><span class="p">],</span>
+        <span class="p">)</span>
+
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">multi_target_classification_loss</span><span class="o">.</span><span class="n">binary_trick</span><span class="p">:</span>
+            <span class="n">classification_loss</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">multi_target_classification_loss</span><span class="p">(</span>
+                <span class="nb">input</span><span class="p">[:,</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">regression_config</span><span class="p">)</span> <span class="p">:],</span> <span class="n">target</span>
             <span class="p">)</span>
-        <span class="p">),</span>
-        <span class="n">reduction</span><span class="o">=</span><span class="n">reduction</span><span class="p">,</span>
-        <span class="n">binary_trick</span><span class="o">=</span><span class="n">binary_trick</span><span class="p">,</span>
-    <span class="p">)</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="n">classification_loss</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">multi_target_classification_loss</span><span class="p">(</span><span class="nb">input</span><span class="p">,</span> <span class="n">target</span><span class="p">)</span>
+
+        <span class="k">return</span> <span class="n">regression_loss</span> <span class="o">+</span> <span class="n">classification_loss</span>
+
+    <span class="k">def</span> <span class="nf">_check_inputs_with_binary_trick</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span>
+        <span class="n">regression_config</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="nb">int</span><span class="p">],</span>
+        <span class="n">binary_config</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="n">Union</span><span class="p">[</span><span class="nb">int</span><span class="p">,</span> <span class="n">Tuple</span><span class="p">[</span><span class="nb">int</span><span class="p">,</span> <span class="nb">float</span><span class="p">]]]],</span>
+        <span class="n">multiclass_config</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span>
+            <span class="n">List</span><span class="p">[</span><span class="n">Union</span><span class="p">[</span><span class="n">Tuple</span><span class="p">[</span><span class="nb">int</span><span class="p">,</span> <span class="nb">int</span><span class="p">],</span> <span class="n">Tuple</span><span class="p">[</span><span class="nb">int</span><span class="p">,</span> <span class="nb">int</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="nb">float</span><span class="p">]]]]</span>
+        <span class="p">],</span>
+    <span class="p">)</span> <span class="o">-&gt;</span> <span class="kc">None</span><span class="p">:</span>
+
+        <span class="n">error_msg</span> <span class="o">=</span> <span class="s2">&quot;When using binary_trick=True, the targets order must be: regression, binary and multiclass&quot;</span>
+
+        <span class="n">first_regression_idx</span> <span class="o">=</span> <span class="n">regression_config</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
+        <span class="n">last_regression_idx</span> <span class="o">=</span> <span class="n">regression_config</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span>
+        <span class="k">if</span> <span class="n">first_regression_idx</span> <span class="o">!=</span> <span class="mi">0</span><span class="p">:</span>
+            <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="n">error_msg</span><span class="p">)</span>
+
+        <span class="k">if</span> <span class="n">binary_config</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">and</span> <span class="n">multiclass_config</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="n">first_binary_idx</span> <span class="o">=</span> <span class="p">(</span>
+                <span class="n">binary_config</span><span class="p">[</span><span class="mi">0</span><span class="p">][</span><span class="mi">0</span><span class="p">]</span>
+                <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">binary_config</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="nb">tuple</span><span class="p">)</span>
+                <span class="k">else</span> <span class="n">binary_config</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
+            <span class="p">)</span>
+            <span class="n">last_binary_idx</span> <span class="o">=</span> <span class="p">(</span>
+                <span class="n">binary_config</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">][</span><span class="mi">0</span><span class="p">]</span>
+                <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">binary_config</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">],</span> <span class="nb">tuple</span><span class="p">)</span>
+                <span class="k">else</span> <span class="n">binary_config</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span>
+            <span class="p">)</span>
+            <span class="n">first_multiclass_idx</span> <span class="o">=</span> <span class="n">multiclass_config</span><span class="p">[</span><span class="mi">0</span><span class="p">][</span><span class="mi">0</span><span class="p">]</span>
+
+            <span class="k">if</span> <span class="p">(</span><span class="n">first_binary_idx</span> <span class="o">!=</span> <span class="n">last_regression_idx</span> <span class="o">+</span> <span class="mi">1</span><span class="p">)</span> <span class="ow">or</span> <span class="p">(</span>
+                <span class="n">last_binary_idx</span> <span class="o">&gt;=</span> <span class="n">first_multiclass_idx</span>
+            <span class="p">):</span>
+                <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="n">error_msg</span><span class="p">)</span>
+        <span class="k">elif</span> <span class="n">binary_config</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="n">first_binary_idx</span> <span class="o">=</span> <span class="p">(</span>
+                <span class="n">binary_config</span><span class="p">[</span><span class="mi">0</span><span class="p">][</span><span class="mi">0</span><span class="p">]</span>
+                <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">binary_config</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="nb">tuple</span><span class="p">)</span>
+                <span class="k">else</span> <span class="n">binary_config</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
+            <span class="p">)</span>
+            <span class="k">if</span> <span class="n">first_binary_idx</span> <span class="o">!=</span> <span class="n">last_regression_idx</span> <span class="o">+</span> <span class="mi">1</span><span class="p">:</span>
+                <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="n">error_msg</span><span class="p">)</span>
+        <span class="k">elif</span> <span class="n">multiclass_config</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="n">first_multiclass_idx</span> <span class="o">=</span> <span class="n">multiclass_config</span><span class="p">[</span><span class="mi">0</span><span class="p">][</span><span class="mi">0</span><span class="p">]</span>
+            <span class="k">if</span> <span class="n">first_multiclass_idx</span> <span class="o">!=</span> <span class="n">last_regression_idx</span> <span class="o">+</span> <span class="mi">1</span><span class="p">:</span>
+                <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="n">error_msg</span><span class="p">)</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
+                <span class="s2">&quot;Either binary_config or multiclass_config must be provided. &quot;</span>
+                <span class="s2">&quot;Otherwise, use the MultiTargetRegressionLoss&quot;</span>
+            <span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">_prepare_weights_for_regression_targets</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span>
+        <span class="n">weights</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="nb">float</span><span class="p">],</span>
+        <span class="n">regression_config</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="nb">int</span><span class="p">],</span>
+    <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">List</span><span class="p">[</span><span class="nb">float</span><span class="p">]:</span>
+
+        <span class="n">weights_regression</span> <span class="o">=</span> <span class="p">[</span>
+            <span class="n">w</span> <span class="k">for</span> <span class="n">idx</span><span class="p">,</span> <span class="n">w</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="n">weights</span><span class="p">)</span> <span class="k">if</span> <span class="n">idx</span> <span class="ow">in</span> <span class="n">regression_config</span>
+        <span class="p">]</span>
+
+        <span class="k">return</span> <span class="n">weights_regression</span>
+
+    <span class="k">def</span> <span class="nf">_prepare_weights_per_binary_targets</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span>
+        <span class="n">weights</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="nb">float</span><span class="p">],</span>
+        <span class="n">binary_config</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="n">Union</span><span class="p">[</span><span class="nb">int</span><span class="p">,</span> <span class="n">Tuple</span><span class="p">[</span><span class="nb">int</span><span class="p">,</span> <span class="nb">float</span><span class="p">]]]],</span>
+    <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">float</span><span class="p">]]:</span>
+
+        <span class="k">if</span> <span class="n">binary_config</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="n">binary_idx</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
+            <span class="k">for</span> <span class="n">bc</span> <span class="ow">in</span> <span class="n">binary_config</span><span class="p">:</span>
+                <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">bc</span><span class="p">,</span> <span class="nb">tuple</span><span class="p">):</span>
+                    <span class="n">binary_idx</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">bc</span><span class="p">[</span><span class="mi">0</span><span class="p">])</span>
+                <span class="k">else</span><span class="p">:</span>
+                    <span class="n">binary_idx</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">bc</span><span class="p">)</span>
+            <span class="n">weights_binary</span> <span class="o">=</span> <span class="p">[</span><span class="n">w</span> <span class="k">for</span> <span class="n">idx</span><span class="p">,</span> <span class="n">w</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="n">weights</span><span class="p">)</span> <span class="k">if</span> <span class="n">idx</span> <span class="ow">in</span> <span class="n">binary_idx</span><span class="p">]</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="n">weights_binary</span> <span class="o">=</span> <span class="kc">None</span>
+
+        <span class="k">return</span> <span class="n">weights_binary</span>
+
+    <span class="k">def</span> <span class="nf">_prepare_weights_per_multiclass_targets</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span>
+        <span class="n">weights</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="nb">float</span><span class="p">],</span>
+        <span class="n">multiclass_config</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span>
+            <span class="n">List</span><span class="p">[</span><span class="n">Union</span><span class="p">[</span><span class="n">Tuple</span><span class="p">[</span><span class="nb">int</span><span class="p">,</span> <span class="nb">int</span><span class="p">],</span> <span class="n">Tuple</span><span class="p">[</span><span class="nb">int</span><span class="p">,</span> <span class="nb">int</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="nb">float</span><span class="p">]]]]</span>
+        <span class="p">],</span>
+    <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">float</span><span class="p">]]:</span>
+
+        <span class="k">if</span> <span class="n">multiclass_config</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="n">multiclass_idx</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="p">[</span><span class="n">mc</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="k">for</span> <span class="n">mc</span> <span class="ow">in</span> <span class="n">multiclass_config</span><span class="p">]</span>
+            <span class="n">weights_multiclass</span> <span class="o">=</span> <span class="p">[</span>
+                <span class="n">w</span> <span class="k">for</span> <span class="n">idx</span><span class="p">,</span> <span class="n">w</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="n">weights</span><span class="p">)</span> <span class="k">if</span> <span class="n">idx</span> <span class="ow">in</span> <span class="n">multiclass_idx</span>
+            <span class="p">]</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="n">weights_multiclass</span> <span class="o">=</span> <span class="kc">None</span>
+
+        <span class="k">return</span> <span class="n">weights_multiclass</span>
 </code></pre></div></td></tr></table></div>
-                  </details>
+              </details>
 
 
 
diff --git a/mkdocs/site/pytorch-widedeep/metrics.html b/mkdocs/site/pytorch-widedeep/metrics.html
index 2835b15b..ca85170a 100644
--- a/mkdocs/site/pytorch-widedeep/metrics.html
+++ b/mkdocs/site/pytorch-widedeep/metrics.html
@@ -1432,7 +1432,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/16_Self_Supervised_Pretraning_pt1.ipynb" class="md-nav__link">
+      <a href="../examples/15_Self_Supervised_Pretraning_pt1.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1453,7 +1453,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/16_Self_Supervised_Pretraning_pt2.ipynb" class="md-nav__link">
+      <a href="../examples/15_Self_Supervised_Pretraning_pt2.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1474,7 +1474,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/17_Usign_a_custom_hugging_face_model.ipynb" class="md-nav__link">
+      <a href="../examples/16_Usign_a_custom_hugging_face_model.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1495,7 +1495,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/18_feature_importance_via_attention_weights.ipynb" class="md-nav__link">
+      <a href="../examples/17_feature_importance_via_attention_weights.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1516,7 +1516,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/19_wide_and_deep_for_recsys_pt1.ipynb" class="md-nav__link">
+      <a href="../examples/18_wide_and_deep_for_recsys_pt1.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1537,7 +1537,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/19_wide_and_deep_for_recsys_pt2.ipynb" class="md-nav__link">
+      <a href="../examples/18_wide_and_deep_for_recsys_pt2.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1558,7 +1558,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/20_load_from_folder_functionality.ipynb" class="md-nav__link">
+      <a href="../examples/19_load_from_folder_functionality.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1579,7 +1579,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/21_Using_huggingface_within_widedeep.ipynb" class="md-nav__link">
+      <a href="../examples/20_Using_huggingface_within_widedeep.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1674,8 +1674,7 @@ <h2 id="pytorch_widedeep.metrics.Accuracy" class="doc doc-heading">
 
 
 <a href="#pytorch_widedeep.metrics.Accuracy" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="nf">Accuracy</span><span class="p">(</span><span class="n">top_k</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span>
-</code></pre></div>
+
 
     <div class="doc doc-contents first">
             <p class="doc doc-class-bases">
@@ -1686,19 +1685,33 @@ <h2 id="pytorch_widedeep.metrics.Accuracy" class="doc doc-heading">
 
 
 <p><span class="doc-section-title">Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>top_k</code></b>
-              (<code>int</code>, default:
-                  <code>1</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Accuracy will be computed using the top k most likely classes in
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+          <th>Default</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>top_k</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Accuracy will be computed using the top k most likely classes in
 multiclass problems</p>
-          </div>
-        </li>
-    </ul>
+              </div>
+            </td>
+            <td>
+                  <code>1</code>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
 
 <p><span class="doc-section-title">Examples:</span></p>
@@ -1719,23 +1732,123 @@ <h2 id="pytorch_widedeep.metrics.Accuracy" class="doc doc-heading">
 <span class="go">array(0.66666667)</span>
 </code></pre></div>
 
-                  <details class="quote">
-                    <summary>Source code in <code>pytorch_widedeep/metrics.py</code></summary>
-                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">76</span>
-<span class="normal">77</span>
-<span class="normal">78</span>
-<span class="normal">79</span>
-<span class="normal">80</span>
-<span class="normal">81</span>
-<span class="normal">82</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">top_k</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1</span><span class="p">):</span>
-    <span class="nb">super</span><span class="p">(</span><span class="n">Accuracy</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
-
-    <span class="bp">self</span><span class="o">.</span><span class="n">top_k</span> <span class="o">=</span> <span class="n">top_k</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">correct_count</span> <span class="o">=</span> <span class="mi">0</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">total_count</span> <span class="o">=</span> <span class="mi">0</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">_name</span> <span class="o">=</span> <span class="s2">&quot;acc&quot;</span>
+              <details class="quote">
+                <summary>Source code in <code>pytorch_widedeep/metrics.py</code></summary>
+                <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 48</span>
+<span class="normal"> 49</span>
+<span class="normal"> 50</span>
+<span class="normal"> 51</span>
+<span class="normal"> 52</span>
+<span class="normal"> 53</span>
+<span class="normal"> 54</span>
+<span class="normal"> 55</span>
+<span class="normal"> 56</span>
+<span class="normal"> 57</span>
+<span class="normal"> 58</span>
+<span class="normal"> 59</span>
+<span class="normal"> 60</span>
+<span class="normal"> 61</span>
+<span class="normal"> 62</span>
+<span class="normal"> 63</span>
+<span class="normal"> 64</span>
+<span class="normal"> 65</span>
+<span class="normal"> 66</span>
+<span class="normal"> 67</span>
+<span class="normal"> 68</span>
+<span class="normal"> 69</span>
+<span class="normal"> 70</span>
+<span class="normal"> 71</span>
+<span class="normal"> 72</span>
+<span class="normal"> 73</span>
+<span class="normal"> 74</span>
+<span class="normal"> 75</span>
+<span class="normal"> 76</span>
+<span class="normal"> 77</span>
+<span class="normal"> 78</span>
+<span class="normal"> 79</span>
+<span class="normal"> 80</span>
+<span class="normal"> 81</span>
+<span class="normal"> 82</span>
+<span class="normal"> 83</span>
+<span class="normal"> 84</span>
+<span class="normal"> 85</span>
+<span class="normal"> 86</span>
+<span class="normal"> 87</span>
+<span class="normal"> 88</span>
+<span class="normal"> 89</span>
+<span class="normal"> 90</span>
+<span class="normal"> 91</span>
+<span class="normal"> 92</span>
+<span class="normal"> 93</span>
+<span class="normal"> 94</span>
+<span class="normal"> 95</span>
+<span class="normal"> 96</span>
+<span class="normal"> 97</span>
+<span class="normal"> 98</span>
+<span class="normal"> 99</span>
+<span class="normal">100</span>
+<span class="normal">101</span>
+<span class="normal">102</span>
+<span class="normal">103</span>
+<span class="normal">104</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">class</span> <span class="nc">Accuracy</span><span class="p">(</span><span class="n">Metric</span><span class="p">):</span>
+<span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Class to calculate the accuracy for both binary and categorical problems</span>
+
+<span class="sd">    Parameters</span>
+<span class="sd">    ----------</span>
+<span class="sd">    top_k: int, default = 1</span>
+<span class="sd">        Accuracy will be computed using the top k most likely classes in</span>
+<span class="sd">        multiclass problems</span>
+
+<span class="sd">    Examples</span>
+<span class="sd">    --------</span>
+<span class="sd">    &gt;&gt;&gt; import torch</span>
+<span class="sd">    &gt;&gt;&gt;</span>
+<span class="sd">    &gt;&gt;&gt; from pytorch_widedeep.metrics import Accuracy</span>
+<span class="sd">    &gt;&gt;&gt;</span>
+<span class="sd">    &gt;&gt;&gt; acc = Accuracy()</span>
+<span class="sd">    &gt;&gt;&gt; y_true = torch.tensor([0, 1, 0, 1]).view(-1, 1)</span>
+<span class="sd">    &gt;&gt;&gt; y_pred = torch.tensor([[0.3, 0.2, 0.6, 0.7]]).view(-1, 1)</span>
+<span class="sd">    &gt;&gt;&gt; acc(y_pred, y_true)</span>
+<span class="sd">    array(0.5)</span>
+<span class="sd">    &gt;&gt;&gt;</span>
+<span class="sd">    &gt;&gt;&gt; acc = Accuracy(top_k=2)</span>
+<span class="sd">    &gt;&gt;&gt; y_true = torch.tensor([0, 1, 2])</span>
+<span class="sd">    &gt;&gt;&gt; y_pred = torch.tensor([[0.3, 0.5, 0.2], [0.1, 0.1, 0.8], [0.1, 0.5, 0.4]])</span>
+<span class="sd">    &gt;&gt;&gt; acc(y_pred, y_true)</span>
+<span class="sd">    array(0.66666667)</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+
+    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">top_k</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1</span><span class="p">):</span>
+        <span class="nb">super</span><span class="p">(</span><span class="n">Accuracy</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">top_k</span> <span class="o">=</span> <span class="n">top_k</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">correct_count</span> <span class="o">=</span> <span class="mi">0</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">total_count</span> <span class="o">=</span> <span class="mi">0</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">_name</span> <span class="o">=</span> <span class="s2">&quot;acc&quot;</span>
+
+    <span class="k">def</span> <span class="nf">reset</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+<span class="w">        </span><span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        resets counters to 0</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">correct_count</span> <span class="o">=</span> <span class="mi">0</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">total_count</span> <span class="o">=</span> <span class="mi">0</span>
+
+    <span class="k">def</span> <span class="fm">__call__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">y_pred</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">,</span> <span class="n">y_true</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">:</span>
+        <span class="n">num_classes</span> <span class="o">=</span> <span class="n">y_pred</span><span class="o">.</span><span class="n">size</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span>
+
+        <span class="k">if</span> <span class="n">num_classes</span> <span class="o">==</span> <span class="mi">1</span><span class="p">:</span>
+            <span class="n">y_pred</span> <span class="o">=</span> <span class="n">y_pred</span><span class="o">.</span><span class="n">round</span><span class="p">()</span>
+            <span class="n">y_true</span> <span class="o">=</span> <span class="n">y_true</span>
+        <span class="k">elif</span> <span class="n">num_classes</span> <span class="o">&gt;</span> <span class="mi">1</span><span class="p">:</span>
+            <span class="n">y_pred</span> <span class="o">=</span> <span class="n">y_pred</span><span class="o">.</span><span class="n">topk</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">top_k</span><span class="p">,</span> <span class="mi">1</span><span class="p">)[</span><span class="mi">1</span><span class="p">]</span>
+            <span class="n">y_true</span> <span class="o">=</span> <span class="n">y_true</span><span class="o">.</span><span class="n">view</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span><span class="o">.</span><span class="n">expand_as</span><span class="p">(</span><span class="n">y_pred</span><span class="p">)</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">correct_count</span> <span class="o">+=</span> <span class="n">y_pred</span><span class="o">.</span><span class="n">eq</span><span class="p">(</span><span class="n">y_true</span><span class="p">)</span><span class="o">.</span><span class="n">sum</span><span class="p">()</span><span class="o">.</span><span class="n">item</span><span class="p">()</span>  <span class="c1"># type: ignore[assignment]</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">total_count</span> <span class="o">+=</span> <span class="nb">len</span><span class="p">(</span><span class="n">y_pred</span><span class="p">)</span>
+        <span class="n">accuracy</span> <span class="o">=</span> <span class="nb">float</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">correct_count</span><span class="p">)</span> <span class="o">/</span> <span class="nb">float</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">total_count</span><span class="p">)</span>
+        <span class="k">return</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">(</span><span class="n">accuracy</span><span class="p">)</span>
 </code></pre></div></td></tr></table></div>
-                  </details>
+              </details>
 
 
 
@@ -1800,8 +1913,7 @@ <h2 id="pytorch_widedeep.metrics.Precision" class="doc doc-heading">
 
 
 <a href="#pytorch_widedeep.metrics.Precision" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="nf">Precision</span><span class="p">(</span><span class="n">average</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
-</code></pre></div>
+
 
     <div class="doc doc-contents first">
             <p class="doc doc-class-bases">
@@ -1812,19 +1924,33 @@ <h2 id="pytorch_widedeep.metrics.Precision" class="doc doc-heading">
 
 
 <p><span class="doc-section-title">Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>average</code></b>
-              (<code>bool</code>, default:
-                  <code>True</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>This applies only to multiclass problems. if <code>True</code> calculate
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+          <th>Default</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>average</code></td>
+            <td>
+                  <code>bool</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>This applies only to multiclass problems. if <code>True</code> calculate
 precision for each label, and finds their unweighted mean.</p>
-          </div>
-        </li>
-    </ul>
+              </div>
+            </td>
+            <td>
+                  <code>True</code>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
 
 <p><span class="doc-section-title">Examples:</span></p>
@@ -1845,25 +1971,137 @@ <h2 id="pytorch_widedeep.metrics.Precision" class="doc doc-heading">
 <span class="go">array(0.33333334)</span>
 </code></pre></div>
 
-                  <details class="quote">
-                    <summary>Source code in <code>pytorch_widedeep/metrics.py</code></summary>
-                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">135</span>
+              <details class="quote">
+                <summary>Source code in <code>pytorch_widedeep/metrics.py</code></summary>
+                <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">107</span>
+<span class="normal">108</span>
+<span class="normal">109</span>
+<span class="normal">110</span>
+<span class="normal">111</span>
+<span class="normal">112</span>
+<span class="normal">113</span>
+<span class="normal">114</span>
+<span class="normal">115</span>
+<span class="normal">116</span>
+<span class="normal">117</span>
+<span class="normal">118</span>
+<span class="normal">119</span>
+<span class="normal">120</span>
+<span class="normal">121</span>
+<span class="normal">122</span>
+<span class="normal">123</span>
+<span class="normal">124</span>
+<span class="normal">125</span>
+<span class="normal">126</span>
+<span class="normal">127</span>
+<span class="normal">128</span>
+<span class="normal">129</span>
+<span class="normal">130</span>
+<span class="normal">131</span>
+<span class="normal">132</span>
+<span class="normal">133</span>
+<span class="normal">134</span>
+<span class="normal">135</span>
 <span class="normal">136</span>
 <span class="normal">137</span>
 <span class="normal">138</span>
 <span class="normal">139</span>
 <span class="normal">140</span>
 <span class="normal">141</span>
-<span class="normal">142</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">average</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">):</span>
-    <span class="nb">super</span><span class="p">(</span><span class="n">Precision</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
+<span class="normal">142</span>
+<span class="normal">143</span>
+<span class="normal">144</span>
+<span class="normal">145</span>
+<span class="normal">146</span>
+<span class="normal">147</span>
+<span class="normal">148</span>
+<span class="normal">149</span>
+<span class="normal">150</span>
+<span class="normal">151</span>
+<span class="normal">152</span>
+<span class="normal">153</span>
+<span class="normal">154</span>
+<span class="normal">155</span>
+<span class="normal">156</span>
+<span class="normal">157</span>
+<span class="normal">158</span>
+<span class="normal">159</span>
+<span class="normal">160</span>
+<span class="normal">161</span>
+<span class="normal">162</span>
+<span class="normal">163</span>
+<span class="normal">164</span>
+<span class="normal">165</span>
+<span class="normal">166</span>
+<span class="normal">167</span>
+<span class="normal">168</span>
+<span class="normal">169</span>
+<span class="normal">170</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">class</span> <span class="nc">Precision</span><span class="p">(</span><span class="n">Metric</span><span class="p">):</span>
+<span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Class to calculate the precision for both binary and categorical problems</span>
+
+<span class="sd">    Parameters</span>
+<span class="sd">    ----------</span>
+<span class="sd">    average: bool, default = True</span>
+<span class="sd">        This applies only to multiclass problems. if ``True`` calculate</span>
+<span class="sd">        precision for each label, and finds their unweighted mean.</span>
+
+<span class="sd">    Examples</span>
+<span class="sd">    --------</span>
+<span class="sd">    &gt;&gt;&gt; import torch</span>
+<span class="sd">    &gt;&gt;&gt;</span>
+<span class="sd">    &gt;&gt;&gt; from pytorch_widedeep.metrics import Precision</span>
+<span class="sd">    &gt;&gt;&gt;</span>
+<span class="sd">    &gt;&gt;&gt; prec = Precision()</span>
+<span class="sd">    &gt;&gt;&gt; y_true = torch.tensor([0, 1, 0, 1]).view(-1, 1)</span>
+<span class="sd">    &gt;&gt;&gt; y_pred = torch.tensor([[0.3, 0.2, 0.6, 0.7]]).view(-1, 1)</span>
+<span class="sd">    &gt;&gt;&gt; prec(y_pred, y_true)</span>
+<span class="sd">    array(0.5)</span>
+<span class="sd">    &gt;&gt;&gt;</span>
+<span class="sd">    &gt;&gt;&gt; prec = Precision(average=True)</span>
+<span class="sd">    &gt;&gt;&gt; y_true = torch.tensor([0, 1, 2])</span>
+<span class="sd">    &gt;&gt;&gt; y_pred = torch.tensor([[0.7, 0.1, 0.2], [0.1, 0.1, 0.8], [0.1, 0.5, 0.4]])</span>
+<span class="sd">    &gt;&gt;&gt; prec(y_pred, y_true)</span>
+<span class="sd">    array(0.33333334)</span>
+<span class="sd">    &quot;&quot;&quot;</span>
 
-    <span class="bp">self</span><span class="o">.</span><span class="n">average</span> <span class="o">=</span> <span class="n">average</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">true_positives</span> <span class="o">=</span> <span class="mi">0</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">all_positives</span> <span class="o">=</span> <span class="mi">0</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">eps</span> <span class="o">=</span> <span class="mf">1e-20</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">_name</span> <span class="o">=</span> <span class="s2">&quot;prec&quot;</span>
+    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">average</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">):</span>
+        <span class="nb">super</span><span class="p">(</span><span class="n">Precision</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">average</span> <span class="o">=</span> <span class="n">average</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">true_positives</span> <span class="o">=</span> <span class="mi">0</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">all_positives</span> <span class="o">=</span> <span class="mi">0</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">eps</span> <span class="o">=</span> <span class="mf">1e-20</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">_name</span> <span class="o">=</span> <span class="s2">&quot;prec&quot;</span>
+
+    <span class="k">def</span> <span class="nf">reset</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+<span class="w">        </span><span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        resets counters to 0</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">true_positives</span> <span class="o">=</span> <span class="mi">0</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">all_positives</span> <span class="o">=</span> <span class="mi">0</span>
+
+    <span class="k">def</span> <span class="fm">__call__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">y_pred</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">,</span> <span class="n">y_true</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">:</span>
+        <span class="n">num_class</span> <span class="o">=</span> <span class="n">y_pred</span><span class="o">.</span><span class="n">size</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span>
+
+        <span class="k">if</span> <span class="n">num_class</span> <span class="o">==</span> <span class="mi">1</span><span class="p">:</span>
+            <span class="n">y_pred</span> <span class="o">=</span> <span class="n">y_pred</span><span class="o">.</span><span class="n">round</span><span class="p">()</span>
+            <span class="n">y_true</span> <span class="o">=</span> <span class="n">y_true</span>
+        <span class="k">elif</span> <span class="n">num_class</span> <span class="o">&gt;</span> <span class="mi">1</span><span class="p">:</span>
+            <span class="n">y_true</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">eye</span><span class="p">(</span><span class="n">num_class</span><span class="p">)[</span><span class="n">y_true</span><span class="o">.</span><span class="n">squeeze</span><span class="p">()</span><span class="o">.</span><span class="n">cpu</span><span class="p">()</span><span class="o">.</span><span class="n">long</span><span class="p">()]</span>
+            <span class="n">y_pred</span> <span class="o">=</span> <span class="n">y_pred</span><span class="o">.</span><span class="n">topk</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">)[</span><span class="mi">1</span><span class="p">]</span><span class="o">.</span><span class="n">view</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">)</span>
+            <span class="n">y_pred</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">eye</span><span class="p">(</span><span class="n">num_class</span><span class="p">)[</span><span class="n">y_pred</span><span class="o">.</span><span class="n">cpu</span><span class="p">()</span><span class="o">.</span><span class="n">long</span><span class="p">()]</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">true_positives</span> <span class="o">+=</span> <span class="p">(</span><span class="n">y_true</span> <span class="o">*</span> <span class="n">y_pred</span><span class="p">)</span><span class="o">.</span><span class="n">sum</span><span class="p">(</span><span class="n">dim</span><span class="o">=</span><span class="mi">0</span><span class="p">)</span>  <span class="c1"># type:ignore</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">all_positives</span> <span class="o">+=</span> <span class="n">y_pred</span><span class="o">.</span><span class="n">sum</span><span class="p">(</span><span class="n">dim</span><span class="o">=</span><span class="mi">0</span><span class="p">)</span>  <span class="c1"># type:ignore</span>
+
+        <span class="n">precision</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">true_positives</span> <span class="o">/</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">all_positives</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">eps</span><span class="p">)</span>
+
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">average</span><span class="p">:</span>
+            <span class="k">return</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">(</span><span class="n">precision</span><span class="o">.</span><span class="n">mean</span><span class="p">()</span><span class="o">.</span><span class="n">item</span><span class="p">())</span>  <span class="c1"># type:ignore</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="k">return</span> <span class="n">precision</span><span class="o">.</span><span class="n">detach</span><span class="p">()</span><span class="o">.</span><span class="n">cpu</span><span class="p">()</span><span class="o">.</span><span class="n">numpy</span><span class="p">()</span>  <span class="c1"># type: ignore[attr-defined]</span>
 </code></pre></div></td></tr></table></div>
-                  </details>
+              </details>
 
 
 
@@ -1928,8 +2166,7 @@ <h2 id="pytorch_widedeep.metrics.Recall" class="doc doc-heading">
 
 
 <a href="#pytorch_widedeep.metrics.Recall" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="nf">Recall</span><span class="p">(</span><span class="n">average</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
-</code></pre></div>
+
 
     <div class="doc doc-contents first">
             <p class="doc doc-class-bases">
@@ -1940,19 +2177,33 @@ <h2 id="pytorch_widedeep.metrics.Recall" class="doc doc-heading">
 
 
 <p><span class="doc-section-title">Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>average</code></b>
-              (<code>bool</code>, default:
-                  <code>True</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>This applies only to multiclass problems. if <code>True</code> calculate recall
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+          <th>Default</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>average</code></td>
+            <td>
+                  <code>bool</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>This applies only to multiclass problems. if <code>True</code> calculate recall
 for each label, and finds their unweighted mean.</p>
-          </div>
-        </li>
-    </ul>
+              </div>
+            </td>
+            <td>
+                  <code>True</code>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
 
 <p><span class="doc-section-title">Examples:</span></p>
@@ -1973,25 +2224,137 @@ <h2 id="pytorch_widedeep.metrics.Recall" class="doc doc-heading">
 <span class="go">array(0.33333334)</span>
 </code></pre></div>
 
-                  <details class="quote">
-                    <summary>Source code in <code>pytorch_widedeep/metrics.py</code></summary>
-                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">201</span>
+              <details class="quote">
+                <summary>Source code in <code>pytorch_widedeep/metrics.py</code></summary>
+                <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">173</span>
+<span class="normal">174</span>
+<span class="normal">175</span>
+<span class="normal">176</span>
+<span class="normal">177</span>
+<span class="normal">178</span>
+<span class="normal">179</span>
+<span class="normal">180</span>
+<span class="normal">181</span>
+<span class="normal">182</span>
+<span class="normal">183</span>
+<span class="normal">184</span>
+<span class="normal">185</span>
+<span class="normal">186</span>
+<span class="normal">187</span>
+<span class="normal">188</span>
+<span class="normal">189</span>
+<span class="normal">190</span>
+<span class="normal">191</span>
+<span class="normal">192</span>
+<span class="normal">193</span>
+<span class="normal">194</span>
+<span class="normal">195</span>
+<span class="normal">196</span>
+<span class="normal">197</span>
+<span class="normal">198</span>
+<span class="normal">199</span>
+<span class="normal">200</span>
+<span class="normal">201</span>
 <span class="normal">202</span>
 <span class="normal">203</span>
 <span class="normal">204</span>
 <span class="normal">205</span>
 <span class="normal">206</span>
 <span class="normal">207</span>
-<span class="normal">208</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">average</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">):</span>
-    <span class="nb">super</span><span class="p">(</span><span class="n">Recall</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
+<span class="normal">208</span>
+<span class="normal">209</span>
+<span class="normal">210</span>
+<span class="normal">211</span>
+<span class="normal">212</span>
+<span class="normal">213</span>
+<span class="normal">214</span>
+<span class="normal">215</span>
+<span class="normal">216</span>
+<span class="normal">217</span>
+<span class="normal">218</span>
+<span class="normal">219</span>
+<span class="normal">220</span>
+<span class="normal">221</span>
+<span class="normal">222</span>
+<span class="normal">223</span>
+<span class="normal">224</span>
+<span class="normal">225</span>
+<span class="normal">226</span>
+<span class="normal">227</span>
+<span class="normal">228</span>
+<span class="normal">229</span>
+<span class="normal">230</span>
+<span class="normal">231</span>
+<span class="normal">232</span>
+<span class="normal">233</span>
+<span class="normal">234</span>
+<span class="normal">235</span>
+<span class="normal">236</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">class</span> <span class="nc">Recall</span><span class="p">(</span><span class="n">Metric</span><span class="p">):</span>
+<span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Class to calculate the recall for both binary and categorical problems</span>
+
+<span class="sd">    Parameters</span>
+<span class="sd">    ----------</span>
+<span class="sd">    average: bool, default = True</span>
+<span class="sd">        This applies only to multiclass problems. if ``True`` calculate recall</span>
+<span class="sd">        for each label, and finds their unweighted mean.</span>
+
+<span class="sd">    Examples</span>
+<span class="sd">    --------</span>
+<span class="sd">    &gt;&gt;&gt; import torch</span>
+<span class="sd">    &gt;&gt;&gt;</span>
+<span class="sd">    &gt;&gt;&gt; from pytorch_widedeep.metrics import Recall</span>
+<span class="sd">    &gt;&gt;&gt;</span>
+<span class="sd">    &gt;&gt;&gt; rec = Recall()</span>
+<span class="sd">    &gt;&gt;&gt; y_true = torch.tensor([0, 1, 0, 1]).view(-1, 1)</span>
+<span class="sd">    &gt;&gt;&gt; y_pred = torch.tensor([[0.3, 0.2, 0.6, 0.7]]).view(-1, 1)</span>
+<span class="sd">    &gt;&gt;&gt; rec(y_pred, y_true)</span>
+<span class="sd">    array(0.5)</span>
+<span class="sd">    &gt;&gt;&gt;</span>
+<span class="sd">    &gt;&gt;&gt; rec = Recall(average=True)</span>
+<span class="sd">    &gt;&gt;&gt; y_true = torch.tensor([0, 1, 2])</span>
+<span class="sd">    &gt;&gt;&gt; y_pred = torch.tensor([[0.7, 0.1, 0.2], [0.1, 0.1, 0.8], [0.1, 0.5, 0.4]])</span>
+<span class="sd">    &gt;&gt;&gt; rec(y_pred, y_true)</span>
+<span class="sd">    array(0.33333334)</span>
+<span class="sd">    &quot;&quot;&quot;</span>
 
-    <span class="bp">self</span><span class="o">.</span><span class="n">average</span> <span class="o">=</span> <span class="n">average</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">true_positives</span> <span class="o">=</span> <span class="mi">0</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">actual_positives</span> <span class="o">=</span> <span class="mi">0</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">eps</span> <span class="o">=</span> <span class="mf">1e-20</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">_name</span> <span class="o">=</span> <span class="s2">&quot;rec&quot;</span>
+    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">average</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">):</span>
+        <span class="nb">super</span><span class="p">(</span><span class="n">Recall</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">average</span> <span class="o">=</span> <span class="n">average</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">true_positives</span> <span class="o">=</span> <span class="mi">0</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">actual_positives</span> <span class="o">=</span> <span class="mi">0</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">eps</span> <span class="o">=</span> <span class="mf">1e-20</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">_name</span> <span class="o">=</span> <span class="s2">&quot;rec&quot;</span>
+
+    <span class="k">def</span> <span class="nf">reset</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+<span class="w">        </span><span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        resets counters to 0</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">true_positives</span> <span class="o">=</span> <span class="mi">0</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">actual_positives</span> <span class="o">=</span> <span class="mi">0</span>
+
+    <span class="k">def</span> <span class="fm">__call__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">y_pred</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">,</span> <span class="n">y_true</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">:</span>
+        <span class="n">num_class</span> <span class="o">=</span> <span class="n">y_pred</span><span class="o">.</span><span class="n">size</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span>
+
+        <span class="k">if</span> <span class="n">num_class</span> <span class="o">==</span> <span class="mi">1</span><span class="p">:</span>
+            <span class="n">y_pred</span> <span class="o">=</span> <span class="n">y_pred</span><span class="o">.</span><span class="n">round</span><span class="p">()</span>
+            <span class="n">y_true</span> <span class="o">=</span> <span class="n">y_true</span>
+        <span class="k">elif</span> <span class="n">num_class</span> <span class="o">&gt;</span> <span class="mi">1</span><span class="p">:</span>
+            <span class="n">y_true</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">eye</span><span class="p">(</span><span class="n">num_class</span><span class="p">)[</span><span class="n">y_true</span><span class="o">.</span><span class="n">squeeze</span><span class="p">()</span><span class="o">.</span><span class="n">cpu</span><span class="p">()</span><span class="o">.</span><span class="n">long</span><span class="p">()]</span>
+            <span class="n">y_pred</span> <span class="o">=</span> <span class="n">y_pred</span><span class="o">.</span><span class="n">topk</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">)[</span><span class="mi">1</span><span class="p">]</span><span class="o">.</span><span class="n">view</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">)</span>
+            <span class="n">y_pred</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">eye</span><span class="p">(</span><span class="n">num_class</span><span class="p">)[</span><span class="n">y_pred</span><span class="o">.</span><span class="n">cpu</span><span class="p">()</span><span class="o">.</span><span class="n">long</span><span class="p">()]</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">true_positives</span> <span class="o">+=</span> <span class="p">(</span><span class="n">y_true</span> <span class="o">*</span> <span class="n">y_pred</span><span class="p">)</span><span class="o">.</span><span class="n">sum</span><span class="p">(</span><span class="n">dim</span><span class="o">=</span><span class="mi">0</span><span class="p">)</span>  <span class="c1"># type: ignore</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">actual_positives</span> <span class="o">+=</span> <span class="n">y_true</span><span class="o">.</span><span class="n">sum</span><span class="p">(</span><span class="n">dim</span><span class="o">=</span><span class="mi">0</span><span class="p">)</span>  <span class="c1"># type: ignore</span>
+
+        <span class="n">recall</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">true_positives</span> <span class="o">/</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">actual_positives</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">eps</span><span class="p">)</span>
+
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">average</span><span class="p">:</span>
+            <span class="k">return</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">(</span><span class="n">recall</span><span class="o">.</span><span class="n">mean</span><span class="p">()</span><span class="o">.</span><span class="n">item</span><span class="p">())</span>  <span class="c1"># type:ignore</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="k">return</span> <span class="n">recall</span><span class="o">.</span><span class="n">detach</span><span class="p">()</span><span class="o">.</span><span class="n">cpu</span><span class="p">()</span><span class="o">.</span><span class="n">numpy</span><span class="p">()</span>  <span class="c1"># type: ignore[attr-defined]</span>
 </code></pre></div></td></tr></table></div>
-                  </details>
+              </details>
 
 
 
@@ -2056,8 +2419,7 @@ <h2 id="pytorch_widedeep.metrics.FBetaScore" class="doc doc-heading">
 
 
 <a href="#pytorch_widedeep.metrics.FBetaScore" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="nf">FBetaScore</span><span class="p">(</span><span class="n">beta</span><span class="p">,</span> <span class="n">average</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
-</code></pre></div>
+
 
     <div class="doc doc-contents first">
             <p class="doc doc-class-bases">
@@ -2071,27 +2433,47 @@ <h2 id="pytorch_widedeep.metrics.FBetaScore" class="doc doc-heading">
 
 
 <p><span class="doc-section-title">Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>beta</code></b>
-              (<code>int</code>)
-          –
-          <div class="doc-md-description">
-            <p>Coefficient to control the balance between precision and recall</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>average</code></b>
-              (<code>bool</code>, default:
-                  <code>True</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>This applies only to multiclass problems. if <code>True</code> calculate fbeta
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+          <th>Default</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>beta</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Coefficient to control the balance between precision and recall</p>
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>average</code></td>
+            <td>
+                  <code>bool</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>This applies only to multiclass problems. if <code>True</code> calculate fbeta
 for each label, and find their unweighted mean.</p>
-          </div>
-        </li>
-    </ul>
+              </div>
+            </td>
+            <td>
+                  <code>True</code>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
 
 <p><span class="doc-section-title">Examples:</span></p>
@@ -2112,9 +2494,43 @@ <h2 id="pytorch_widedeep.metrics.FBetaScore" class="doc doc-heading">
 <span class="go">array(0.33333334)</span>
 </code></pre></div>
 
-                  <details class="quote">
-                    <summary>Source code in <code>pytorch_widedeep/metrics.py</code></summary>
-                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">273</span>
+              <details class="quote">
+                <summary>Source code in <code>pytorch_widedeep/metrics.py</code></summary>
+                <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">239</span>
+<span class="normal">240</span>
+<span class="normal">241</span>
+<span class="normal">242</span>
+<span class="normal">243</span>
+<span class="normal">244</span>
+<span class="normal">245</span>
+<span class="normal">246</span>
+<span class="normal">247</span>
+<span class="normal">248</span>
+<span class="normal">249</span>
+<span class="normal">250</span>
+<span class="normal">251</span>
+<span class="normal">252</span>
+<span class="normal">253</span>
+<span class="normal">254</span>
+<span class="normal">255</span>
+<span class="normal">256</span>
+<span class="normal">257</span>
+<span class="normal">258</span>
+<span class="normal">259</span>
+<span class="normal">260</span>
+<span class="normal">261</span>
+<span class="normal">262</span>
+<span class="normal">263</span>
+<span class="normal">264</span>
+<span class="normal">265</span>
+<span class="normal">266</span>
+<span class="normal">267</span>
+<span class="normal">268</span>
+<span class="normal">269</span>
+<span class="normal">270</span>
+<span class="normal">271</span>
+<span class="normal">272</span>
+<span class="normal">273</span>
 <span class="normal">274</span>
 <span class="normal">275</span>
 <span class="normal">276</span>
@@ -2122,17 +2538,89 @@ <h2 id="pytorch_widedeep.metrics.FBetaScore" class="doc doc-heading">
 <span class="normal">278</span>
 <span class="normal">279</span>
 <span class="normal">280</span>
-<span class="normal">281</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">beta</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span> <span class="n">average</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">):</span>
-    <span class="nb">super</span><span class="p">(</span><span class="n">FBetaScore</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
-
-    <span class="bp">self</span><span class="o">.</span><span class="n">beta</span> <span class="o">=</span> <span class="n">beta</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">average</span> <span class="o">=</span> <span class="n">average</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">precision</span> <span class="o">=</span> <span class="n">Precision</span><span class="p">(</span><span class="n">average</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">recall</span> <span class="o">=</span> <span class="n">Recall</span><span class="p">(</span><span class="n">average</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">eps</span> <span class="o">=</span> <span class="mf">1e-20</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">_name</span> <span class="o">=</span> <span class="s2">&quot;&quot;</span><span class="o">.</span><span class="n">join</span><span class="p">([</span><span class="s2">&quot;f&quot;</span><span class="p">,</span> <span class="nb">str</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">beta</span><span class="p">)])</span>
+<span class="normal">281</span>
+<span class="normal">282</span>
+<span class="normal">283</span>
+<span class="normal">284</span>
+<span class="normal">285</span>
+<span class="normal">286</span>
+<span class="normal">287</span>
+<span class="normal">288</span>
+<span class="normal">289</span>
+<span class="normal">290</span>
+<span class="normal">291</span>
+<span class="normal">292</span>
+<span class="normal">293</span>
+<span class="normal">294</span>
+<span class="normal">295</span>
+<span class="normal">296</span>
+<span class="normal">297</span>
+<span class="normal">298</span>
+<span class="normal">299</span>
+<span class="normal">300</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">class</span> <span class="nc">FBetaScore</span><span class="p">(</span><span class="n">Metric</span><span class="p">):</span>
+<span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Class to calculate the fbeta score for both binary and categorical problems</span>
+
+<span class="sd">    $$</span>
+<span class="sd">    F_{\beta} = ((1 + {\beta}^2) * \frac{(precision * recall)}{({\beta}^2 * precision + recall)}</span>
+<span class="sd">    $$</span>
+
+<span class="sd">    Parameters</span>
+<span class="sd">    ----------</span>
+<span class="sd">    beta: int</span>
+<span class="sd">        Coefficient to control the balance between precision and recall</span>
+<span class="sd">    average: bool, default = True</span>
+<span class="sd">        This applies only to multiclass problems. if ``True`` calculate fbeta</span>
+<span class="sd">        for each label, and find their unweighted mean.</span>
+
+<span class="sd">    Examples</span>
+<span class="sd">    --------</span>
+<span class="sd">    &gt;&gt;&gt; import torch</span>
+<span class="sd">    &gt;&gt;&gt;</span>
+<span class="sd">    &gt;&gt;&gt; from pytorch_widedeep.metrics import FBetaScore</span>
+<span class="sd">    &gt;&gt;&gt;</span>
+<span class="sd">    &gt;&gt;&gt; fbeta = FBetaScore(beta=2)</span>
+<span class="sd">    &gt;&gt;&gt; y_true = torch.tensor([0, 1, 0, 1]).view(-1, 1)</span>
+<span class="sd">    &gt;&gt;&gt; y_pred = torch.tensor([[0.3, 0.2, 0.6, 0.7]]).view(-1, 1)</span>
+<span class="sd">    &gt;&gt;&gt; fbeta(y_pred, y_true)</span>
+<span class="sd">    array(0.5)</span>
+<span class="sd">    &gt;&gt;&gt;</span>
+<span class="sd">    &gt;&gt;&gt; fbeta = FBetaScore(beta=2)</span>
+<span class="sd">    &gt;&gt;&gt; y_true = torch.tensor([0, 1, 2])</span>
+<span class="sd">    &gt;&gt;&gt; y_pred = torch.tensor([[0.7, 0.1, 0.2], [0.1, 0.1, 0.8], [0.1, 0.5, 0.4]])</span>
+<span class="sd">    &gt;&gt;&gt; fbeta(y_pred, y_true)</span>
+<span class="sd">    array(0.33333334)</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+
+    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">beta</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span> <span class="n">average</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">):</span>
+        <span class="nb">super</span><span class="p">(</span><span class="n">FBetaScore</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">beta</span> <span class="o">=</span> <span class="n">beta</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">average</span> <span class="o">=</span> <span class="n">average</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">precision</span> <span class="o">=</span> <span class="n">Precision</span><span class="p">(</span><span class="n">average</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">recall</span> <span class="o">=</span> <span class="n">Recall</span><span class="p">(</span><span class="n">average</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">eps</span> <span class="o">=</span> <span class="mf">1e-20</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">_name</span> <span class="o">=</span> <span class="s2">&quot;&quot;</span><span class="o">.</span><span class="n">join</span><span class="p">([</span><span class="s2">&quot;f&quot;</span><span class="p">,</span> <span class="nb">str</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">beta</span><span class="p">)])</span>
+
+    <span class="k">def</span> <span class="nf">reset</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+<span class="w">        </span><span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        resets precision and recall</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">precision</span><span class="o">.</span><span class="n">reset</span><span class="p">()</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">recall</span><span class="o">.</span><span class="n">reset</span><span class="p">()</span>
+
+    <span class="k">def</span> <span class="fm">__call__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">y_pred</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">,</span> <span class="n">y_true</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">:</span>
+        <span class="n">prec</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">precision</span><span class="p">(</span><span class="n">y_pred</span><span class="p">,</span> <span class="n">y_true</span><span class="p">)</span>
+        <span class="n">rec</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">recall</span><span class="p">(</span><span class="n">y_pred</span><span class="p">,</span> <span class="n">y_true</span><span class="p">)</span>
+        <span class="n">beta2</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">beta</span><span class="o">**</span><span class="mi">2</span>
+
+        <span class="n">fbeta</span> <span class="o">=</span> <span class="p">((</span><span class="mi">1</span> <span class="o">+</span> <span class="n">beta2</span><span class="p">)</span> <span class="o">*</span> <span class="n">prec</span> <span class="o">*</span> <span class="n">rec</span><span class="p">)</span> <span class="o">/</span> <span class="p">(</span><span class="n">beta2</span> <span class="o">*</span> <span class="n">prec</span> <span class="o">+</span> <span class="n">rec</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">eps</span><span class="p">)</span>
+
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">average</span><span class="p">:</span>
+            <span class="k">return</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">(</span><span class="n">fbeta</span><span class="o">.</span><span class="n">mean</span><span class="p">()</span><span class="o">.</span><span class="n">item</span><span class="p">())</span>  <span class="c1"># type: ignore[attr-defined]</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="k">return</span> <span class="n">fbeta</span>
 </code></pre></div></td></tr></table></div>
-                  </details>
+              </details>
 
 
 
@@ -2197,8 +2685,7 @@ <h2 id="pytorch_widedeep.metrics.F1Score" class="doc doc-heading">
 
 
 <a href="#pytorch_widedeep.metrics.F1Score" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="nf">F1Score</span><span class="p">(</span><span class="n">average</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
-</code></pre></div>
+
 
     <div class="doc doc-contents first">
             <p class="doc doc-class-bases">
@@ -2209,19 +2696,33 @@ <h2 id="pytorch_widedeep.metrics.F1Score" class="doc doc-heading">
 
 
 <p><span class="doc-section-title">Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>average</code></b>
-              (<code>bool</code>, default:
-                  <code>True</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>This applies only to multiclass problems. if <code>True</code> calculate f1 for
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+          <th>Default</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>average</code></td>
+            <td>
+                  <code>bool</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>This applies only to multiclass problems. if <code>True</code> calculate f1 for
 each label, and find their unweighted mean.</p>
-          </div>
-        </li>
-    </ul>
+              </div>
+            </td>
+            <td>
+                  <code>True</code>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
 
 <p><span class="doc-section-title">Examples:</span></p>
@@ -2242,21 +2743,95 @@ <h2 id="pytorch_widedeep.metrics.F1Score" class="doc doc-heading">
 <span class="go">array(0.33333334)</span>
 </code></pre></div>
 
-                  <details class="quote">
-                    <summary>Source code in <code>pytorch_widedeep/metrics.py</code></summary>
-                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">331</span>
+              <details class="quote">
+                <summary>Source code in <code>pytorch_widedeep/metrics.py</code></summary>
+                <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">303</span>
+<span class="normal">304</span>
+<span class="normal">305</span>
+<span class="normal">306</span>
+<span class="normal">307</span>
+<span class="normal">308</span>
+<span class="normal">309</span>
+<span class="normal">310</span>
+<span class="normal">311</span>
+<span class="normal">312</span>
+<span class="normal">313</span>
+<span class="normal">314</span>
+<span class="normal">315</span>
+<span class="normal">316</span>
+<span class="normal">317</span>
+<span class="normal">318</span>
+<span class="normal">319</span>
+<span class="normal">320</span>
+<span class="normal">321</span>
+<span class="normal">322</span>
+<span class="normal">323</span>
+<span class="normal">324</span>
+<span class="normal">325</span>
+<span class="normal">326</span>
+<span class="normal">327</span>
+<span class="normal">328</span>
+<span class="normal">329</span>
+<span class="normal">330</span>
+<span class="normal">331</span>
 <span class="normal">332</span>
 <span class="normal">333</span>
 <span class="normal">334</span>
 <span class="normal">335</span>
-<span class="normal">336</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">average</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">):</span>
-    <span class="nb">super</span><span class="p">(</span><span class="n">F1Score</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
+<span class="normal">336</span>
+<span class="normal">337</span>
+<span class="normal">338</span>
+<span class="normal">339</span>
+<span class="normal">340</span>
+<span class="normal">341</span>
+<span class="normal">342</span>
+<span class="normal">343</span>
+<span class="normal">344</span>
+<span class="normal">345</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">class</span> <span class="nc">F1Score</span><span class="p">(</span><span class="n">Metric</span><span class="p">):</span>
+<span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Class to calculate the f1 score for both binary and categorical problems</span>
+
+<span class="sd">    Parameters</span>
+<span class="sd">    ----------</span>
+<span class="sd">    average: bool, default = True</span>
+<span class="sd">        This applies only to multiclass problems. if ``True`` calculate f1 for</span>
+<span class="sd">        each label, and find their unweighted mean.</span>
+
+<span class="sd">    Examples</span>
+<span class="sd">    --------</span>
+<span class="sd">    &gt;&gt;&gt; import torch</span>
+<span class="sd">    &gt;&gt;&gt;</span>
+<span class="sd">    &gt;&gt;&gt; from pytorch_widedeep.metrics import F1Score</span>
+<span class="sd">    &gt;&gt;&gt;</span>
+<span class="sd">    &gt;&gt;&gt; f1 = F1Score()</span>
+<span class="sd">    &gt;&gt;&gt; y_true = torch.tensor([0, 1, 0, 1]).view(-1, 1)</span>
+<span class="sd">    &gt;&gt;&gt; y_pred = torch.tensor([[0.3, 0.2, 0.6, 0.7]]).view(-1, 1)</span>
+<span class="sd">    &gt;&gt;&gt; f1(y_pred, y_true)</span>
+<span class="sd">    array(0.5)</span>
+<span class="sd">    &gt;&gt;&gt;</span>
+<span class="sd">    &gt;&gt;&gt; f1 = F1Score()</span>
+<span class="sd">    &gt;&gt;&gt; y_true = torch.tensor([0, 1, 2])</span>
+<span class="sd">    &gt;&gt;&gt; y_pred = torch.tensor([[0.7, 0.1, 0.2], [0.1, 0.1, 0.8], [0.1, 0.5, 0.4]])</span>
+<span class="sd">    &gt;&gt;&gt; f1(y_pred, y_true)</span>
+<span class="sd">    array(0.33333334)</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+
+    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">average</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">):</span>
+        <span class="nb">super</span><span class="p">(</span><span class="n">F1Score</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
 
-    <span class="bp">self</span><span class="o">.</span><span class="n">average</span> <span class="o">=</span> <span class="n">average</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">f1</span> <span class="o">=</span> <span class="n">FBetaScore</span><span class="p">(</span><span class="n">beta</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">average</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">average</span><span class="p">)</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">_name</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">f1</span><span class="o">.</span><span class="n">_name</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">average</span> <span class="o">=</span> <span class="n">average</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">f1</span> <span class="o">=</span> <span class="n">FBetaScore</span><span class="p">(</span><span class="n">beta</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">average</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">average</span><span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">_name</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">f1</span><span class="o">.</span><span class="n">_name</span>
+
+    <span class="k">def</span> <span class="nf">reset</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+<span class="w">        </span><span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        resets counters to 0</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">f1</span><span class="o">.</span><span class="n">reset</span><span class="p">()</span>
+
+    <span class="k">def</span> <span class="fm">__call__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">y_pred</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">,</span> <span class="n">y_true</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">:</span>
+        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">f1</span><span class="p">(</span><span class="n">y_pred</span><span class="p">,</span> <span class="n">y_true</span><span class="p">)</span>
 </code></pre></div></td></tr></table></div>
-                  </details>
+              </details>
 
 
 
@@ -2319,8 +2894,7 @@ <h2 id="pytorch_widedeep.metrics.R2Score" class="doc doc-heading">
 
 
 <a href="#pytorch_widedeep.metrics.R2Score" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="nf">R2Score</span><span class="p">()</span>
-</code></pre></div>
+
 
     <div class="doc doc-contents first">
             <p class="doc doc-class-bases">
@@ -2348,23 +2922,109 @@ <h2 id="pytorch_widedeep.metrics.R2Score" class="doc doc-heading">
 <span class="go">array(0.94860814)</span>
 </code></pre></div>
 
-                  <details class="quote">
-                    <summary>Source code in <code>pytorch_widedeep/metrics.py</code></summary>
-                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">373</span>
+              <details class="quote">
+                <summary>Source code in <code>pytorch_widedeep/metrics.py</code></summary>
+                <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">348</span>
+<span class="normal">349</span>
+<span class="normal">350</span>
+<span class="normal">351</span>
+<span class="normal">352</span>
+<span class="normal">353</span>
+<span class="normal">354</span>
+<span class="normal">355</span>
+<span class="normal">356</span>
+<span class="normal">357</span>
+<span class="normal">358</span>
+<span class="normal">359</span>
+<span class="normal">360</span>
+<span class="normal">361</span>
+<span class="normal">362</span>
+<span class="normal">363</span>
+<span class="normal">364</span>
+<span class="normal">365</span>
+<span class="normal">366</span>
+<span class="normal">367</span>
+<span class="normal">368</span>
+<span class="normal">369</span>
+<span class="normal">370</span>
+<span class="normal">371</span>
+<span class="normal">372</span>
+<span class="normal">373</span>
 <span class="normal">374</span>
 <span class="normal">375</span>
 <span class="normal">376</span>
 <span class="normal">377</span>
 <span class="normal">378</span>
-<span class="normal">379</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">numerator</span> <span class="o">=</span> <span class="mi">0</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">denominator</span> <span class="o">=</span> <span class="mi">0</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">num_examples</span> <span class="o">=</span> <span class="mi">0</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">y_true_sum</span> <span class="o">=</span> <span class="mi">0</span>
+<span class="normal">379</span>
+<span class="normal">380</span>
+<span class="normal">381</span>
+<span class="normal">382</span>
+<span class="normal">383</span>
+<span class="normal">384</span>
+<span class="normal">385</span>
+<span class="normal">386</span>
+<span class="normal">387</span>
+<span class="normal">388</span>
+<span class="normal">389</span>
+<span class="normal">390</span>
+<span class="normal">391</span>
+<span class="normal">392</span>
+<span class="normal">393</span>
+<span class="normal">394</span>
+<span class="normal">395</span>
+<span class="normal">396</span>
+<span class="normal">397</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">class</span> <span class="nc">R2Score</span><span class="p">(</span><span class="n">Metric</span><span class="p">):</span>
+<span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">    Calculates R-Squared, the</span>
+<span class="sd">    [coefficient of determination](https://en.wikipedia.org/wiki/Coefficient_of_determination&gt;):</span>
+
+<span class="sd">    $$</span>
+<span class="sd">    R^2 = 1 - \frac{\sum_{j=1}^n(y_j - \hat{y_j})^2}{\sum_{j=1}^n(y_j - \bar{y})^2}</span>
+<span class="sd">    $$</span>
+
+<span class="sd">    where $\hat{y_j}$ is the ground truth, $y_j$ is the predicted value and</span>
+<span class="sd">    $\bar{y}$ is the mean of the ground truth.</span>
+
+<span class="sd">    Examples</span>
+<span class="sd">    --------</span>
+<span class="sd">    &gt;&gt;&gt; import torch</span>
+<span class="sd">    &gt;&gt;&gt;</span>
+<span class="sd">    &gt;&gt;&gt; from pytorch_widedeep.metrics import R2Score</span>
+<span class="sd">    &gt;&gt;&gt;</span>
+<span class="sd">    &gt;&gt;&gt; r2 = R2Score()</span>
+<span class="sd">    &gt;&gt;&gt; y_true = torch.tensor([3, -0.5, 2, 7]).view(-1, 1)</span>
+<span class="sd">    &gt;&gt;&gt; y_pred = torch.tensor([2.5, 0.0, 2, 8]).view(-1, 1)</span>
+<span class="sd">    &gt;&gt;&gt; r2(y_pred, y_true)</span>
+<span class="sd">    array(0.94860814)</span>
+<span class="sd">    &quot;&quot;&quot;</span>
 
-    <span class="bp">self</span><span class="o">.</span><span class="n">_name</span> <span class="o">=</span> <span class="s2">&quot;r2&quot;</span>
+    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">numerator</span> <span class="o">=</span> <span class="mi">0</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">denominator</span> <span class="o">=</span> <span class="mi">0</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">num_examples</span> <span class="o">=</span> <span class="mi">0</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">y_true_sum</span> <span class="o">=</span> <span class="mi">0</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">_name</span> <span class="o">=</span> <span class="s2">&quot;r2&quot;</span>
+
+    <span class="k">def</span> <span class="nf">reset</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+<span class="w">        </span><span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        resets counters to 0</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">numerator</span> <span class="o">=</span> <span class="mi">0</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">denominator</span> <span class="o">=</span> <span class="mi">0</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">num_examples</span> <span class="o">=</span> <span class="mi">0</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">y_true_sum</span> <span class="o">=</span> <span class="mi">0</span>
+
+    <span class="k">def</span> <span class="fm">__call__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">y_pred</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">,</span> <span class="n">y_true</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">:</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">numerator</span> <span class="o">+=</span> <span class="p">((</span><span class="n">y_pred</span> <span class="o">-</span> <span class="n">y_true</span><span class="p">)</span> <span class="o">**</span> <span class="mi">2</span><span class="p">)</span><span class="o">.</span><span class="n">sum</span><span class="p">()</span><span class="o">.</span><span class="n">item</span><span class="p">()</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">num_examples</span> <span class="o">+=</span> <span class="n">y_true</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">y_true_sum</span> <span class="o">+=</span> <span class="n">y_true</span><span class="o">.</span><span class="n">sum</span><span class="p">()</span><span class="o">.</span><span class="n">item</span><span class="p">()</span>
+        <span class="n">y_true_avg</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">y_true_sum</span> <span class="o">/</span> <span class="bp">self</span><span class="o">.</span><span class="n">num_examples</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">denominator</span> <span class="o">+=</span> <span class="p">((</span><span class="n">y_true</span> <span class="o">-</span> <span class="n">y_true_avg</span><span class="p">)</span> <span class="o">**</span> <span class="mi">2</span><span class="p">)</span><span class="o">.</span><span class="n">sum</span><span class="p">()</span><span class="o">.</span><span class="n">item</span><span class="p">()</span>
+        <span class="k">return</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">((</span><span class="mi">1</span> <span class="o">-</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">numerator</span> <span class="o">/</span> <span class="bp">self</span><span class="o">.</span><span class="n">denominator</span><span class="p">)))</span>
 </code></pre></div></td></tr></table></div>
-                  </details>
+              </details>
 
 
 
diff --git a/mkdocs/site/pytorch-widedeep/model_components.html b/mkdocs/site/pytorch-widedeep/model_components.html
index 13519665..4138c3dc 100644
--- a/mkdocs/site/pytorch-widedeep/model_components.html
+++ b/mkdocs/site/pytorch-widedeep/model_components.html
@@ -1567,7 +1567,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/16_Self_Supervised_Pretraning_pt1.ipynb" class="md-nav__link">
+      <a href="../examples/15_Self_Supervised_Pretraning_pt1.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1588,7 +1588,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/16_Self_Supervised_Pretraning_pt2.ipynb" class="md-nav__link">
+      <a href="../examples/15_Self_Supervised_Pretraning_pt2.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1609,7 +1609,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/17_Usign_a_custom_hugging_face_model.ipynb" class="md-nav__link">
+      <a href="../examples/16_Usign_a_custom_hugging_face_model.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1630,7 +1630,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/18_feature_importance_via_attention_weights.ipynb" class="md-nav__link">
+      <a href="../examples/17_feature_importance_via_attention_weights.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1651,7 +1651,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/19_wide_and_deep_for_recsys_pt1.ipynb" class="md-nav__link">
+      <a href="../examples/18_wide_and_deep_for_recsys_pt1.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1672,7 +1672,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/19_wide_and_deep_for_recsys_pt2.ipynb" class="md-nav__link">
+      <a href="../examples/18_wide_and_deep_for_recsys_pt2.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1693,7 +1693,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/20_load_from_folder_functionality.ipynb" class="md-nav__link">
+      <a href="../examples/19_load_from_folder_functionality.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1714,7 +1714,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/21_Using_huggingface_within_widedeep.ipynb" class="md-nav__link">
+      <a href="../examples/20_Using_huggingface_within_widedeep.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1791,8 +1791,7 @@ <h2 id="pytorch_widedeep.models.tabular.linear.wide.Wide" class="doc doc-heading
 
 
 <a href="#pytorch_widedeep.models.tabular.linear.wide.Wide" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="nf">Wide</span><span class="p">(</span><span class="n">input_dim</span><span class="p">,</span> <span class="n">pred_dim</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span>
-</code></pre></div>
+
 
     <div class="doc doc-contents first">
             <p class="doc doc-class-bases">
@@ -1805,46 +1804,78 @@ <h2 id="pytorch_widedeep.models.tabular.linear.wide.Wide" class="doc doc-heading
 
 
 <p><span class="doc-section-title">Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>input_dim</code></b>
-              (<code>int</code>)
-          –
-          <div class="doc-md-description">
-            <p>size of the Linear layer (implemented via an Embedding layer).
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+          <th>Default</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>input_dim</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>size of the Linear layer (implemented via an Embedding layer).
 <code>input_dim</code> is the summation of all the individual values for all the
 features that go through the wide model. For example, if the wide
 model receives 2 features with 5 individual values each, <code>input_dim =
 10</code></p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>pred_dim</code></b>
-              (<code>int</code>, default:
-                  <code>1</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>size of the ouput tensor containing the predictions. Note that unlike
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>pred_dim</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>size of the ouput tensor containing the predictions. Note that unlike
 all the other models, the wide model is connected directly to the
 output neuron(s) when used to build a Wide and Deep model. Therefore,
 it requires the <code>pred_dim</code> parameter.</p>
-          </div>
-        </li>
-    </ul>
+              </div>
+            </td>
+            <td>
+                  <code>1</code>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
 
 <p><span class="doc-section-title">Attributes:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code><span title="pytorch_widedeep.models.tabular.linear.wide.Wide.wide_linear">wide_linear</span></code></b>
-              (<code><span title="torch.nn.Module">Module</span></code>)
-          –
-          <div class="doc-md-description">
-            <p>the linear layer that comprises the wide branch of the model</p>
-          </div>
-        </li>
-    </ul>
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code><span title="pytorch_widedeep.models.tabular.linear.wide.Wide.wide_linear">wide_linear</span></code></td>
+            <td>
+                  <code><span title="torch.nn.Module">Module</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>the linear layer that comprises the wide branch of the model</p>
+              </div>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
 
 <p><span class="doc-section-title">Examples:</span></p>
@@ -1855,9 +1886,42 @@ <h2 id="pytorch_widedeep.models.tabular.linear.wide.Wide" class="doc doc-heading
 <span class="gp">&gt;&gt;&gt; </span><span class="n">out</span> <span class="o">=</span> <span class="n">wide</span><span class="p">(</span><span class="n">X</span><span class="p">)</span>
 </code></pre></div>
 
-                  <details class="quote">
-                    <summary>Source code in <code>pytorch_widedeep/models/tabular/linear/wide.py</code></summary>
-                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">43</span>
+              <details class="quote">
+                <summary>Source code in <code>pytorch_widedeep/models/tabular/linear/wide.py</code></summary>
+                <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">10</span>
+<span class="normal">11</span>
+<span class="normal">12</span>
+<span class="normal">13</span>
+<span class="normal">14</span>
+<span class="normal">15</span>
+<span class="normal">16</span>
+<span class="normal">17</span>
+<span class="normal">18</span>
+<span class="normal">19</span>
+<span class="normal">20</span>
+<span class="normal">21</span>
+<span class="normal">22</span>
+<span class="normal">23</span>
+<span class="normal">24</span>
+<span class="normal">25</span>
+<span class="normal">26</span>
+<span class="normal">27</span>
+<span class="normal">28</span>
+<span class="normal">29</span>
+<span class="normal">30</span>
+<span class="normal">31</span>
+<span class="normal">32</span>
+<span class="normal">33</span>
+<span class="normal">34</span>
+<span class="normal">35</span>
+<span class="normal">36</span>
+<span class="normal">37</span>
+<span class="normal">38</span>
+<span class="normal">39</span>
+<span class="normal">40</span>
+<span class="normal">41</span>
+<span class="normal">42</span>
+<span class="normal">43</span>
 <span class="normal">44</span>
 <span class="normal">45</span>
 <span class="normal">46</span>
@@ -1868,20 +1932,83 @@ <h2 id="pytorch_widedeep.models.tabular.linear.wide.Wide" class="doc doc-heading
 <span class="normal">51</span>
 <span class="normal">52</span>
 <span class="normal">53</span>
-<span class="normal">54</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="nd">@alias</span><span class="p">(</span><span class="s2">&quot;pred_dim&quot;</span><span class="p">,</span> <span class="p">[</span><span class="s2">&quot;pred_size&quot;</span><span class="p">,</span> <span class="s2">&quot;num_class&quot;</span><span class="p">])</span>
-<span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">input_dim</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span> <span class="n">pred_dim</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1</span><span class="p">):</span>
-    <span class="nb">super</span><span class="p">(</span><span class="n">Wide</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
-
-    <span class="bp">self</span><span class="o">.</span><span class="n">input_dim</span> <span class="o">=</span> <span class="n">input_dim</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">pred_dim</span> <span class="o">=</span> <span class="n">pred_dim</span>
-
-    <span class="c1"># Embeddings: val + 1 because 0 is reserved for padding/unseen cateogories.</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">wide_linear</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Embedding</span><span class="p">(</span><span class="n">input_dim</span> <span class="o">+</span> <span class="mi">1</span><span class="p">,</span> <span class="n">pred_dim</span><span class="p">,</span> <span class="n">padding_idx</span><span class="o">=</span><span class="mi">0</span><span class="p">)</span>
-    <span class="c1"># (Sum(Embedding) + bias) is equivalent to (OneHotVector + Linear)</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">bias</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Parameter</span><span class="p">(</span><span class="n">torch</span><span class="o">.</span><span class="n">zeros</span><span class="p">(</span><span class="n">pred_dim</span><span class="p">))</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">_reset_parameters</span><span class="p">()</span>
+<span class="normal">54</span>
+<span class="normal">55</span>
+<span class="normal">56</span>
+<span class="normal">57</span>
+<span class="normal">58</span>
+<span class="normal">59</span>
+<span class="normal">60</span>
+<span class="normal">61</span>
+<span class="normal">62</span>
+<span class="normal">63</span>
+<span class="normal">64</span>
+<span class="normal">65</span>
+<span class="normal">66</span>
+<span class="normal">67</span>
+<span class="normal">68</span>
+<span class="normal">69</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">class</span> <span class="nc">Wide</span><span class="p">(</span><span class="n">nn</span><span class="o">.</span><span class="n">Module</span><span class="p">):</span>
+<span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Defines a `Wide` (linear) model where the non-linearities are</span>
+<span class="sd">    captured via the so-called crossed-columns. This can be used as the</span>
+<span class="sd">    `wide` component of a Wide &amp; Deep model.</span>
+
+<span class="sd">    Parameters</span>
+<span class="sd">    -----------</span>
+<span class="sd">    input_dim: int</span>
+<span class="sd">        size of the Linear layer (implemented via an Embedding layer).</span>
+<span class="sd">        `input_dim` is the summation of all the individual values for all the</span>
+<span class="sd">        features that go through the wide model. For example, if the wide</span>
+<span class="sd">        model receives 2 features with 5 individual values each, `input_dim =</span>
+<span class="sd">        10`</span>
+<span class="sd">    pred_dim: int, default = 1</span>
+<span class="sd">        size of the ouput tensor containing the predictions. Note that unlike</span>
+<span class="sd">        all the other models, the wide model is connected directly to the</span>
+<span class="sd">        output neuron(s) when used to build a Wide and Deep model. Therefore,</span>
+<span class="sd">        it requires the `pred_dim` parameter.</span>
+
+<span class="sd">    Attributes</span>
+<span class="sd">    -----------</span>
+<span class="sd">    wide_linear: nn.Module</span>
+<span class="sd">        the linear layer that comprises the wide branch of the model</span>
+
+<span class="sd">    Examples</span>
+<span class="sd">    --------</span>
+<span class="sd">    &gt;&gt;&gt; import torch</span>
+<span class="sd">    &gt;&gt;&gt; from pytorch_widedeep.models import Wide</span>
+<span class="sd">    &gt;&gt;&gt; X = torch.empty(4, 4).random_(20)</span>
+<span class="sd">    &gt;&gt;&gt; wide = Wide(input_dim=int(X.max().item()), pred_dim=1)</span>
+<span class="sd">    &gt;&gt;&gt; out = wide(X)</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+
+    <span class="nd">@alias</span><span class="p">(</span><span class="s2">&quot;pred_dim&quot;</span><span class="p">,</span> <span class="p">[</span><span class="s2">&quot;pred_size&quot;</span><span class="p">,</span> <span class="s2">&quot;num_class&quot;</span><span class="p">])</span>
+    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">input_dim</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span> <span class="n">pred_dim</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1</span><span class="p">):</span>
+        <span class="nb">super</span><span class="p">(</span><span class="n">Wide</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">input_dim</span> <span class="o">=</span> <span class="n">input_dim</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">pred_dim</span> <span class="o">=</span> <span class="n">pred_dim</span>
+
+        <span class="c1"># Embeddings: val + 1 because 0 is reserved for padding/unseen cateogories.</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">wide_linear</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Embedding</span><span class="p">(</span><span class="n">input_dim</span> <span class="o">+</span> <span class="mi">1</span><span class="p">,</span> <span class="n">pred_dim</span><span class="p">,</span> <span class="n">padding_idx</span><span class="o">=</span><span class="mi">0</span><span class="p">)</span>
+        <span class="c1"># (Sum(Embedding) + bias) is equivalent to (OneHotVector + Linear)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">bias</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Parameter</span><span class="p">(</span><span class="n">torch</span><span class="o">.</span><span class="n">zeros</span><span class="p">(</span><span class="n">pred_dim</span><span class="p">))</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">_reset_parameters</span><span class="p">()</span>
+
+    <span class="k">def</span> <span class="nf">_reset_parameters</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="kc">None</span><span class="p">:</span>
+<span class="w">        </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;initialize Embedding and bias like nn.Linear. See [original</span>
+<span class="sd">        implementation](https://pytorch.org/docs/stable/_modules/torch/nn/modules/linear.html#Linear).</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">nn</span><span class="o">.</span><span class="n">init</span><span class="o">.</span><span class="n">kaiming_uniform_</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">wide_linear</span><span class="o">.</span><span class="n">weight</span><span class="p">,</span> <span class="n">a</span><span class="o">=</span><span class="n">math</span><span class="o">.</span><span class="n">sqrt</span><span class="p">(</span><span class="mi">5</span><span class="p">))</span>
+        <span class="n">fan_in</span><span class="p">,</span> <span class="n">_</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">init</span><span class="o">.</span><span class="n">_calculate_fan_in_and_fan_out</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">wide_linear</span><span class="o">.</span><span class="n">weight</span><span class="p">)</span>
+        <span class="n">bound</span> <span class="o">=</span> <span class="mi">1</span> <span class="o">/</span> <span class="n">math</span><span class="o">.</span><span class="n">sqrt</span><span class="p">(</span><span class="n">fan_in</span><span class="p">)</span>
+        <span class="n">nn</span><span class="o">.</span><span class="n">init</span><span class="o">.</span><span class="n">uniform_</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">bias</span><span class="p">,</span> <span class="o">-</span><span class="n">bound</span><span class="p">,</span> <span class="n">bound</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">forward</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">X</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tensor</span><span class="p">:</span>
+<span class="w">        </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Forward pass. Simply connecting the Embedding layer with the ouput</span>
+<span class="sd">        neuron(s)&quot;&quot;&quot;</span>
+        <span class="n">out</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">wide_linear</span><span class="p">(</span><span class="n">X</span><span class="o">.</span><span class="n">long</span><span class="p">())</span><span class="o">.</span><span class="n">sum</span><span class="p">(</span><span class="n">dim</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">bias</span>
+        <span class="k">return</span> <span class="n">out</span>
 </code></pre></div></td></tr></table></div>
-                  </details>
+              </details>
 
 
 
@@ -1945,33 +2072,7 @@ <h2 id="pytorch_widedeep.models.tabular.mlp.tab_mlp.TabMlp" class="doc doc-headi
 
 
 <a href="#pytorch_widedeep.models.tabular.mlp.tab_mlp.TabMlp" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="nf">TabMlp</span><span class="p">(</span>
-    <span class="n">column_idx</span><span class="p">,</span>
-    <span class="o">*</span><span class="p">,</span>
-    <span class="n">cat_embed_input</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">cat_embed_dropout</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">use_cat_bias</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">cat_embed_activation</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">continuous_cols</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">cont_norm_layer</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">embed_continuous</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">embed_continuous_method</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">cont_embed_dim</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">cont_embed_dropout</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">cont_embed_activation</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">quantization_setup</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">n_frequencies</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">sigma</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">share_last_layer</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">full_embed_dropout</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">mlp_hidden_dims</span><span class="o">=</span><span class="p">[</span><span class="mi">200</span><span class="p">,</span> <span class="mi">100</span><span class="p">],</span>
-    <span class="n">mlp_activation</span><span class="o">=</span><span class="s2">&quot;relu&quot;</span><span class="p">,</span>
-    <span class="n">mlp_dropout</span><span class="o">=</span><span class="mf">0.1</span><span class="p">,</span>
-    <span class="n">mlp_batchnorm</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
-    <span class="n">mlp_batchnorm_last</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
-    <span class="n">mlp_linear_first</span><span class="o">=</span><span class="kc">True</span>
-<span class="p">)</span>
-</code></pre></div>
+
 
     <div class="doc doc-contents first">
             <p class="doc doc-class-bases">
@@ -1989,106 +2090,149 @@ <h2 id="pytorch_widedeep.models.tabular.mlp.tab_mlp.TabMlp" class="doc doc-headi
 
 
 <p><span class="doc-section-title">Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>column_idx</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, int]</code>)
-          –
-          <div class="doc-md-description">
-            <p>Dict containing the index of the columns that will be passed through
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+          <th>Default</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>column_idx</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, int]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Dict containing the index of the columns that will be passed through
 the <code>TabMlp</code> model. Required to slice the tensors. e.g. <em>{'education':
 0, 'relationship': 1, 'workclass': 2, ...}</em>.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>cat_embed_input</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.wdtypes.Tuple">Tuple</span>[str, int, int]]]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>List of Tuples with the column name, number of unique values and
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>cat_embed_input</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.wdtypes.Tuple">Tuple</span>[str, int, int]]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>List of Tuples with the column name, number of unique values and
 embedding dimension. e.g. <em>[(education, 11, 32), ...]</em></p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>cat_embed_dropout</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Categorical embeddings dropout. If <code>None</code>, it will default
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>cat_embed_dropout</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Categorical embeddings dropout. If <code>None</code>, it will default
 to 0.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>use_cat_bias</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Boolean indicating if bias will be used for the categorical embeddings.
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>use_cat_bias</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Boolean indicating if bias will be used for the categorical embeddings.
 If <code>None</code>, it will default to 'False'.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>cat_embed_activation</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Activation function for the categorical embeddings, if any. Currently
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>cat_embed_activation</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Activation function for the categorical embeddings, if any. Currently
 <em>'tanh'</em>, <em>'relu'</em>, <em>'leaky_relu'</em> and <em>'gelu'</em> are supported</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>continuous_cols</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[str]]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>List with the name of the numeric (aka continuous) columns</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>cont_norm_layer</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[batchnorm, layernorm]]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Type of normalization layer applied to the continuous features.
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>continuous_cols</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[str]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>List with the name of the numeric (aka continuous) columns</p>
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>cont_norm_layer</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[batchnorm, layernorm]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Type of normalization layer applied to the continuous features.
 Options are: <em>'layernorm'</em> and <em>'batchnorm'</em>. if <code>None</code>, no
 normalization layer will be used.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>embed_continuous</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Boolean indicating if the continuous columns will be embedded using
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>embed_continuous</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Boolean indicating if the continuous columns will be embedded using
 one of the available methods: <em>'standard'</em>, <em>'periodic'</em>
 or <em>'piecewise'</em>. If <code>None</code>, it will default to 'False'.<br/>
 <img alt="ℹ️" class="emojione" src="https://cdnjs.cloudflare.com/ajax/libs/emojione/2.2.7/assets/png/2139.png" title=":information_source:" /> <strong>NOTE</strong>: This parameter is deprecated and it
  will be removed in future releases. Please, use the
  <code>embed_continuous_method</code> parameter instead.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>embed_continuous_method</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[standard, piecewise, periodic]]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Method to use to embed the continuous features. Options are:
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>embed_continuous_method</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[standard, piecewise, periodic]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Method to use to embed the continuous features. Options are:
 <em>'standard'</em>, <em>'periodic'</em> or <em>'piecewise'</em>. The <em>'standard'</em>
 embedding method is based on the FT-Transformer implementation
 presented in the paper: <a href="https://arxiv.org/abs/2106.11959v5">Revisiting Deep Learning Models for
@@ -2096,189 +2240,262 @@ <h2 id="pytorch_widedeep.models.tabular.mlp.tab_mlp.TabMlp" class="doc doc-headi
 and_'piecewise'_ methods were presented in the paper: <a href="https://arxiv.org/abs/2203.05556">On Embeddings for
 Numerical Features in Tabular Deep Learning</a>.
 Please, read the papers for details.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>cont_embed_dim</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[int]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Size of the continuous embeddings. If the continuous columns are
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>cont_embed_dim</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[int]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Size of the continuous embeddings. If the continuous columns are
 embedded, <code>cont_embed_dim</code> must be passed.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>cont_embed_dropout</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>cont_embed_activation</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Activation function for the continuous embeddings if any. Currently
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>cont_embed_dropout</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p>
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>cont_embed_activation</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Activation function for the continuous embeddings if any. Currently
 <em>'tanh'</em>, <em>'relu'</em>, <em>'leaky_relu'</em> and <em>'gelu'</em> are supported.
 If <code>None</code>, no activation function will be applied.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>quantization_setup</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, <span title="pytorch_widedeep.wdtypes.List">List</span>[float]]]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>This parameter is used when the <em>'piecewise'</em> method is used to embed
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>quantization_setup</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, <span title="pytorch_widedeep.wdtypes.List">List</span>[float]]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>This parameter is used when the <em>'piecewise'</em> method is used to embed
 the continuous cols. It is a dict where keys are the name of the continuous
 columns and values are lists with the boundaries for the quantization
 of the continuous_cols. See the examples for details. If
 If the <em>'piecewise'</em> method is used, this parameter is required.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>n_frequencies</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[int]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>This is the so called <em>'k'</em> in their paper <a href="https://arxiv.org/abs/2203.05556">On Embeddings for
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>n_frequencies</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[int]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>This is the so called <em>'k'</em> in their paper <a href="https://arxiv.org/abs/2203.05556">On Embeddings for
 Numerical Features in Tabular Deep Learning</a>,
 and is the number of 'frequencies' that will be used to represent each
 continuous column. See their Eq 2 in the paper for details. If
 the <em>'periodic'</em> method is used, this parameter is required.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>sigma</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>This is the sigma parameter in the paper mentioned when describing the
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>sigma</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>This is the sigma parameter in the paper mentioned when describing the
 previous parameters and it is used to initialise the 'frequency
 weights'. See their Eq 2 in the paper for details. If
 the <em>'periodic'</em> method is used, this parameter is required.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>share_last_layer</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>This parameter is not present in the before mentioned paper but it is implemented in
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>share_last_layer</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>This parameter is not present in the before mentioned paper but it is implemented in
 the <a href="https://github.com/yandex-research/rtdl-num-embeddings/tree/main">official repo</a>.
 If <code>True</code> the linear layer that turns the frequencies into embeddings
 will be shared across the continuous columns. If <code>False</code> a different
 linear layer will be used for each continuous column.
 If the <em>'periodic'</em> method is used, this parameter is required.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>full_embed_dropout</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>If <code>True</code>, the full embedding corresponding to a column will be masked
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>full_embed_dropout</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>If <code>True</code>, the full embedding corresponding to a column will be masked
 out/dropout. If <code>None</code>, it will default to <code>False</code>.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>mlp_hidden_dims</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.List">List</span>[int]</code>, default:
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>mlp_hidden_dims</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.List">List</span>[int]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>List with the number of neurons per dense layer in the mlp.</p>
+              </div>
+            </td>
+            <td>
                   <code>[200, 100]</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>List with the number of neurons per dense layer in the mlp.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>mlp_activation</code></b>
-              (<code>str</code>, default:
-                  <code>&#39;relu&#39;</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Activation function for the dense layers of the MLP. Currently
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>mlp_activation</code></td>
+            <td>
+                  <code>str</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Activation function for the dense layers of the MLP. Currently
 <em>'tanh'</em>, <em>'relu'</em>, <em>'leaky_relu'</em> and <em>'gelu'</em> are supported</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>mlp_dropout</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Union">Union</span>[float, <span title="pytorch_widedeep.wdtypes.List">List</span>[float]]</code>, default:
-                  <code>0.1</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>float or List of floats with the dropout between the dense layers.
+              </div>
+            </td>
+            <td>
+                  <code>&#39;relu&#39;</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>mlp_dropout</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Union">Union</span>[float, <span title="pytorch_widedeep.wdtypes.List">List</span>[float]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>float or List of floats with the dropout between the dense layers.
 e.g: <em>[0.5,0.5]</em></p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>mlp_batchnorm</code></b>
-              (<code>bool</code>, default:
-                  <code>False</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Boolean indicating whether or not batch normalization will be applied
+              </div>
+            </td>
+            <td>
+                  <code>0.1</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>mlp_batchnorm</code></td>
+            <td>
+                  <code>bool</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Boolean indicating whether or not batch normalization will be applied
 to the dense layers</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>mlp_batchnorm_last</code></b>
-              (<code>bool</code>, default:
+              </div>
+            </td>
+            <td>
                   <code>False</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Boolean indicating whether or not batch normalization will be applied
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>mlp_batchnorm_last</code></td>
+            <td>
+                  <code>bool</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Boolean indicating whether or not batch normalization will be applied
 to the last of the dense layers</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>mlp_linear_first</code></b>
-              (<code>bool</code>, default:
-                  <code>True</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Boolean indicating the order of the operations in the dense
+              </div>
+            </td>
+            <td>
+                  <code>False</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>mlp_linear_first</code></td>
+            <td>
+                  <code>bool</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Boolean indicating the order of the operations in the dense
 layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt;
 LIN -&gt; ACT]</code></p>
-          </div>
-        </li>
-    </ul>
+              </div>
+            </td>
+            <td>
+                  <code>True</code>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
 
 <p><span class="doc-section-title">Attributes:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code><span title="pytorch_widedeep.models.tabular.mlp.tab_mlp.TabMlp.encoder">encoder</span></code></b>
-              (<code><span title="torch.nn.Module">Module</span></code>)
-          –
-          <div class="doc-md-description">
-            <p>mlp model that will receive the concatenation of the embeddings and
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code><span title="pytorch_widedeep.models.tabular.mlp.tab_mlp.TabMlp.encoder">encoder</span></code></td>
+            <td>
+                  <code><span title="torch.nn.Module">Module</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>mlp model that will receive the concatenation of the embeddings and
 the continuous columns</p>
-          </div>
-        </li>
-    </ul>
+              </div>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
 
 <p><span class="doc-section-title">Examples:</span></p>
@@ -2293,9 +2510,135 @@ <h2 id="pytorch_widedeep.models.tabular.mlp.tab_mlp.TabMlp" class="doc doc-headi
 <span class="gp">&gt;&gt;&gt; </span><span class="n">out</span> <span class="o">=</span> <span class="n">model</span><span class="p">(</span><span class="n">X_tab</span><span class="p">)</span>
 </code></pre></div>
 
-                  <details class="quote">
-                    <summary>Source code in <code>pytorch_widedeep/models/tabular/mlp/tab_mlp.py</code></summary>
-                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">136</span>
+              <details class="quote">
+                <summary>Source code in <code>pytorch_widedeep/models/tabular/mlp/tab_mlp.py</code></summary>
+                <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 10</span>
+<span class="normal"> 11</span>
+<span class="normal"> 12</span>
+<span class="normal"> 13</span>
+<span class="normal"> 14</span>
+<span class="normal"> 15</span>
+<span class="normal"> 16</span>
+<span class="normal"> 17</span>
+<span class="normal"> 18</span>
+<span class="normal"> 19</span>
+<span class="normal"> 20</span>
+<span class="normal"> 21</span>
+<span class="normal"> 22</span>
+<span class="normal"> 23</span>
+<span class="normal"> 24</span>
+<span class="normal"> 25</span>
+<span class="normal"> 26</span>
+<span class="normal"> 27</span>
+<span class="normal"> 28</span>
+<span class="normal"> 29</span>
+<span class="normal"> 30</span>
+<span class="normal"> 31</span>
+<span class="normal"> 32</span>
+<span class="normal"> 33</span>
+<span class="normal"> 34</span>
+<span class="normal"> 35</span>
+<span class="normal"> 36</span>
+<span class="normal"> 37</span>
+<span class="normal"> 38</span>
+<span class="normal"> 39</span>
+<span class="normal"> 40</span>
+<span class="normal"> 41</span>
+<span class="normal"> 42</span>
+<span class="normal"> 43</span>
+<span class="normal"> 44</span>
+<span class="normal"> 45</span>
+<span class="normal"> 46</span>
+<span class="normal"> 47</span>
+<span class="normal"> 48</span>
+<span class="normal"> 49</span>
+<span class="normal"> 50</span>
+<span class="normal"> 51</span>
+<span class="normal"> 52</span>
+<span class="normal"> 53</span>
+<span class="normal"> 54</span>
+<span class="normal"> 55</span>
+<span class="normal"> 56</span>
+<span class="normal"> 57</span>
+<span class="normal"> 58</span>
+<span class="normal"> 59</span>
+<span class="normal"> 60</span>
+<span class="normal"> 61</span>
+<span class="normal"> 62</span>
+<span class="normal"> 63</span>
+<span class="normal"> 64</span>
+<span class="normal"> 65</span>
+<span class="normal"> 66</span>
+<span class="normal"> 67</span>
+<span class="normal"> 68</span>
+<span class="normal"> 69</span>
+<span class="normal"> 70</span>
+<span class="normal"> 71</span>
+<span class="normal"> 72</span>
+<span class="normal"> 73</span>
+<span class="normal"> 74</span>
+<span class="normal"> 75</span>
+<span class="normal"> 76</span>
+<span class="normal"> 77</span>
+<span class="normal"> 78</span>
+<span class="normal"> 79</span>
+<span class="normal"> 80</span>
+<span class="normal"> 81</span>
+<span class="normal"> 82</span>
+<span class="normal"> 83</span>
+<span class="normal"> 84</span>
+<span class="normal"> 85</span>
+<span class="normal"> 86</span>
+<span class="normal"> 87</span>
+<span class="normal"> 88</span>
+<span class="normal"> 89</span>
+<span class="normal"> 90</span>
+<span class="normal"> 91</span>
+<span class="normal"> 92</span>
+<span class="normal"> 93</span>
+<span class="normal"> 94</span>
+<span class="normal"> 95</span>
+<span class="normal"> 96</span>
+<span class="normal"> 97</span>
+<span class="normal"> 98</span>
+<span class="normal"> 99</span>
+<span class="normal">100</span>
+<span class="normal">101</span>
+<span class="normal">102</span>
+<span class="normal">103</span>
+<span class="normal">104</span>
+<span class="normal">105</span>
+<span class="normal">106</span>
+<span class="normal">107</span>
+<span class="normal">108</span>
+<span class="normal">109</span>
+<span class="normal">110</span>
+<span class="normal">111</span>
+<span class="normal">112</span>
+<span class="normal">113</span>
+<span class="normal">114</span>
+<span class="normal">115</span>
+<span class="normal">116</span>
+<span class="normal">117</span>
+<span class="normal">118</span>
+<span class="normal">119</span>
+<span class="normal">120</span>
+<span class="normal">121</span>
+<span class="normal">122</span>
+<span class="normal">123</span>
+<span class="normal">124</span>
+<span class="normal">125</span>
+<span class="normal">126</span>
+<span class="normal">127</span>
+<span class="normal">128</span>
+<span class="normal">129</span>
+<span class="normal">130</span>
+<span class="normal">131</span>
+<span class="normal">132</span>
+<span class="normal">133</span>
+<span class="normal">134</span>
+<span class="normal">135</span>
+<span class="normal">136</span>
 <span class="normal">137</span>
 <span class="normal">138</span>
 <span class="normal">139</span>
@@ -2362,76 +2705,222 @@ <h2 id="pytorch_widedeep.models.tabular.mlp.tab_mlp.TabMlp" class="doc doc-headi
 <span class="normal">200</span>
 <span class="normal">201</span>
 <span class="normal">202</span>
-<span class="normal">203</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
-    <span class="bp">self</span><span class="p">,</span>
-    <span class="n">column_idx</span><span class="p">:</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">],</span>
-    <span class="o">*</span><span class="p">,</span>
-    <span class="n">cat_embed_input</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="n">Tuple</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">,</span> <span class="nb">int</span><span class="p">]]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">cat_embed_dropout</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">use_cat_bias</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">cat_embed_activation</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">continuous_cols</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">cont_norm_layer</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Literal</span><span class="p">[</span><span class="s2">&quot;batchnorm&quot;</span><span class="p">,</span> <span class="s2">&quot;layernorm&quot;</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">embed_continuous</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">embed_continuous_method</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span>
-        <span class="n">Literal</span><span class="p">[</span><span class="s2">&quot;standard&quot;</span><span class="p">,</span> <span class="s2">&quot;piecewise&quot;</span><span class="p">,</span> <span class="s2">&quot;periodic&quot;</span><span class="p">]</span>
-    <span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">cont_embed_dim</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">cont_embed_dropout</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">cont_embed_activation</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">quantization_setup</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="nb">float</span><span class="p">]]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">n_frequencies</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">sigma</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">share_last_layer</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">full_embed_dropout</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">mlp_hidden_dims</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="p">[</span><span class="mi">200</span><span class="p">,</span> <span class="mi">100</span><span class="p">],</span>
-    <span class="n">mlp_activation</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;relu&quot;</span><span class="p">,</span>
-    <span class="n">mlp_dropout</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="nb">float</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="nb">float</span><span class="p">]]</span> <span class="o">=</span> <span class="mf">0.1</span><span class="p">,</span>
-    <span class="n">mlp_batchnorm</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
-    <span class="n">mlp_batchnorm_last</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
-    <span class="n">mlp_linear_first</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">,</span>
-<span class="p">):</span>
-    <span class="nb">super</span><span class="p">(</span><span class="n">TabMlp</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span>
-        <span class="n">column_idx</span><span class="o">=</span><span class="n">column_idx</span><span class="p">,</span>
-        <span class="n">cat_embed_input</span><span class="o">=</span><span class="n">cat_embed_input</span><span class="p">,</span>
-        <span class="n">cat_embed_dropout</span><span class="o">=</span><span class="n">cat_embed_dropout</span><span class="p">,</span>
-        <span class="n">use_cat_bias</span><span class="o">=</span><span class="n">use_cat_bias</span><span class="p">,</span>
-        <span class="n">cat_embed_activation</span><span class="o">=</span><span class="n">cat_embed_activation</span><span class="p">,</span>
-        <span class="n">continuous_cols</span><span class="o">=</span><span class="n">continuous_cols</span><span class="p">,</span>
-        <span class="n">cont_norm_layer</span><span class="o">=</span><span class="n">cont_norm_layer</span><span class="p">,</span>
-        <span class="n">embed_continuous</span><span class="o">=</span><span class="n">embed_continuous</span><span class="p">,</span>
-        <span class="n">embed_continuous_method</span><span class="o">=</span><span class="n">embed_continuous_method</span><span class="p">,</span>
-        <span class="n">cont_embed_dim</span><span class="o">=</span><span class="n">cont_embed_dim</span><span class="p">,</span>
-        <span class="n">cont_embed_dropout</span><span class="o">=</span><span class="n">cont_embed_dropout</span><span class="p">,</span>
-        <span class="n">cont_embed_activation</span><span class="o">=</span><span class="n">cont_embed_activation</span><span class="p">,</span>
-        <span class="n">quantization_setup</span><span class="o">=</span><span class="n">quantization_setup</span><span class="p">,</span>
-        <span class="n">n_frequencies</span><span class="o">=</span><span class="n">n_frequencies</span><span class="p">,</span>
-        <span class="n">sigma</span><span class="o">=</span><span class="n">sigma</span><span class="p">,</span>
-        <span class="n">share_last_layer</span><span class="o">=</span><span class="n">share_last_layer</span><span class="p">,</span>
-        <span class="n">full_embed_dropout</span><span class="o">=</span><span class="n">full_embed_dropout</span><span class="p">,</span>
-    <span class="p">)</span>
+<span class="normal">203</span>
+<span class="normal">204</span>
+<span class="normal">205</span>
+<span class="normal">206</span>
+<span class="normal">207</span>
+<span class="normal">208</span>
+<span class="normal">209</span>
+<span class="normal">210</span>
+<span class="normal">211</span>
+<span class="normal">212</span>
+<span class="normal">213</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">class</span> <span class="nc">TabMlp</span><span class="p">(</span><span class="n">BaseTabularModelWithoutAttention</span><span class="p">):</span>
+<span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Defines a `TabMlp` model that can be used as the `deeptabular`</span>
+<span class="sd">    component of a Wide &amp; Deep model or independently by itself.</span>
+
+<span class="sd">    This class combines embedding representations of the categorical features</span>
+<span class="sd">    with numerical (aka continuous) features, embedded or not. These are then</span>
+<span class="sd">    passed through a series of dense layers (i.e. a MLP).</span>
+
+<span class="sd">    Most of the parameters for this class are `Optional` since the use of</span>
+<span class="sd">    categorical or continuous is in fact optional (i.e. one can use</span>
+<span class="sd">    categorical features only, continuous features only or both).</span>
+
+<span class="sd">    Parameters</span>
+<span class="sd">    ----------</span>
+<span class="sd">    column_idx: Dict</span>
+<span class="sd">        Dict containing the index of the columns that will be passed through</span>
+<span class="sd">        the `TabMlp` model. Required to slice the tensors. e.g. _{&#39;education&#39;:</span>
+<span class="sd">        0, &#39;relationship&#39;: 1, &#39;workclass&#39;: 2, ...}_.</span>
+<span class="sd">    cat_embed_input: List, Optional, default = None</span>
+<span class="sd">        List of Tuples with the column name, number of unique values and</span>
+<span class="sd">        embedding dimension. e.g. _[(education, 11, 32), ...]_</span>
+<span class="sd">    cat_embed_dropout: float, Optional, default = None</span>
+<span class="sd">        Categorical embeddings dropout. If `None`, it will default</span>
+<span class="sd">        to 0.</span>
+<span class="sd">    use_cat_bias: bool, Optional, default = None,</span>
+<span class="sd">        Boolean indicating if bias will be used for the categorical embeddings.</span>
+<span class="sd">        If `None`, it will default to &#39;False&#39;.</span>
+<span class="sd">    cat_embed_activation: Optional, str, default = None,</span>
+<span class="sd">        Activation function for the categorical embeddings, if any. Currently</span>
+<span class="sd">        _&#39;tanh&#39;_, _&#39;relu&#39;_, _&#39;leaky_relu&#39;_ and _&#39;gelu&#39;_ are supported</span>
+<span class="sd">    continuous_cols: List, Optional, default = None</span>
+<span class="sd">        List with the name of the numeric (aka continuous) columns</span>
+<span class="sd">    cont_norm_layer: str, Optional, default =  None</span>
+<span class="sd">        Type of normalization layer applied to the continuous features.</span>
+<span class="sd">        Options are: _&#39;layernorm&#39;_ and _&#39;batchnorm&#39;_. if `None`, no</span>
+<span class="sd">        normalization layer will be used.</span>
+<span class="sd">    embed_continuous: bool, Optional, default = None,</span>
+<span class="sd">        Boolean indicating if the continuous columns will be embedded using</span>
+<span class="sd">        one of the available methods: _&#39;standard&#39;_, _&#39;periodic&#39;_</span>
+<span class="sd">        or _&#39;piecewise&#39;_. If `None`, it will default to &#39;False&#39;.&lt;br/&gt;</span>
+<span class="sd">        :information_source: **NOTE**: This parameter is deprecated and it</span>
+<span class="sd">         will be removed in future releases. Please, use the</span>
+<span class="sd">         `embed_continuous_method` parameter instead.</span>
+<span class="sd">    embed_continuous_method: Optional, str, default = None,</span>
+<span class="sd">        Method to use to embed the continuous features. Options are:</span>
+<span class="sd">        _&#39;standard&#39;_, _&#39;periodic&#39;_ or _&#39;piecewise&#39;_. The _&#39;standard&#39;_</span>
+<span class="sd">        embedding method is based on the FT-Transformer implementation</span>
+<span class="sd">        presented in the paper: [Revisiting Deep Learning Models for</span>
+<span class="sd">        Tabular Data](https://arxiv.org/abs/2106.11959v5). The _&#39;periodic&#39;_</span>
+<span class="sd">        and_&#39;piecewise&#39;_ methods were presented in the paper: [On Embeddings for</span>
+<span class="sd">        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556).</span>
+<span class="sd">        Please, read the papers for details.</span>
+<span class="sd">    cont_embed_dim: int, Optional, default = None,</span>
+<span class="sd">        Size of the continuous embeddings. If the continuous columns are</span>
+<span class="sd">        embedded, `cont_embed_dim` must be passed.</span>
+<span class="sd">    cont_embed_dropout: float, Optional, default = None,</span>
+<span class="sd">        Dropout for the continuous embeddings. If `None`, it will default to 0.0</span>
+<span class="sd">    cont_embed_activation: Optional, str, default = None,</span>
+<span class="sd">        Activation function for the continuous embeddings if any. Currently</span>
+<span class="sd">        _&#39;tanh&#39;_, _&#39;relu&#39;_, _&#39;leaky_relu&#39;_ and _&#39;gelu&#39;_ are supported.</span>
+<span class="sd">        If `None`, no activation function will be applied.</span>
+<span class="sd">    quantization_setup: Dict[str, List[float]], Optional, default = None,</span>
+<span class="sd">        This parameter is used when the _&#39;piecewise&#39;_ method is used to embed</span>
+<span class="sd">        the continuous cols. It is a dict where keys are the name of the continuous</span>
+<span class="sd">        columns and values are lists with the boundaries for the quantization</span>
+<span class="sd">        of the continuous_cols. See the examples for details. If</span>
+<span class="sd">        If the _&#39;piecewise&#39;_ method is used, this parameter is required.</span>
+<span class="sd">    n_frequencies: int, Optional, default = None,</span>
+<span class="sd">        This is the so called _&#39;k&#39;_ in their paper [On Embeddings for</span>
+<span class="sd">        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556),</span>
+<span class="sd">        and is the number of &#39;frequencies&#39; that will be used to represent each</span>
+<span class="sd">        continuous column. See their Eq 2 in the paper for details. If</span>
+<span class="sd">        the _&#39;periodic&#39;_ method is used, this parameter is required.</span>
+<span class="sd">    sigma: float, Optional, default = None,</span>
+<span class="sd">        This is the sigma parameter in the paper mentioned when describing the</span>
+<span class="sd">        previous parameters and it is used to initialise the &#39;frequency</span>
+<span class="sd">        weights&#39;. See their Eq 2 in the paper for details. If</span>
+<span class="sd">        the _&#39;periodic&#39;_ method is used, this parameter is required.</span>
+<span class="sd">    share_last_layer: bool, Optional, default = None,</span>
+<span class="sd">        This parameter is not present in the before mentioned paper but it is implemented in</span>
+<span class="sd">        the [official repo](https://github.com/yandex-research/rtdl-num-embeddings/tree/main).</span>
+<span class="sd">        If `True` the linear layer that turns the frequencies into embeddings</span>
+<span class="sd">        will be shared across the continuous columns. If `False` a different</span>
+<span class="sd">        linear layer will be used for each continuous column.</span>
+<span class="sd">        If the _&#39;periodic&#39;_ method is used, this parameter is required.</span>
+<span class="sd">    full_embed_dropout: bool, Optional, default = None,</span>
+<span class="sd">        If `True`, the full embedding corresponding to a column will be masked</span>
+<span class="sd">        out/dropout. If `None`, it will default to `False`.</span>
+<span class="sd">    mlp_hidden_dims: List, default = [200, 100]</span>
+<span class="sd">        List with the number of neurons per dense layer in the mlp.</span>
+<span class="sd">    mlp_activation: str, default = &quot;relu&quot;</span>
+<span class="sd">        Activation function for the dense layers of the MLP. Currently</span>
+<span class="sd">        _&#39;tanh&#39;_, _&#39;relu&#39;_, _&#39;leaky_relu&#39;_ and _&#39;gelu&#39;_ are supported</span>
+<span class="sd">    mlp_dropout: float or List, default = 0.1</span>
+<span class="sd">        float or List of floats with the dropout between the dense layers.</span>
+<span class="sd">        e.g: _[0.5,0.5]_</span>
+<span class="sd">    mlp_batchnorm: bool, default = False</span>
+<span class="sd">        Boolean indicating whether or not batch normalization will be applied</span>
+<span class="sd">        to the dense layers</span>
+<span class="sd">    mlp_batchnorm_last: bool, default = False</span>
+<span class="sd">        Boolean indicating whether or not batch normalization will be applied</span>
+<span class="sd">        to the last of the dense layers</span>
+<span class="sd">    mlp_linear_first: bool, default = False</span>
+<span class="sd">        Boolean indicating the order of the operations in the dense</span>
+<span class="sd">        layer. If `True: [LIN -&gt; ACT -&gt; BN -&gt; DP]`. If `False: [BN -&gt; DP -&gt;</span>
+<span class="sd">        LIN -&gt; ACT]`</span>
+
+<span class="sd">    Attributes</span>
+<span class="sd">    ----------</span>
+<span class="sd">    encoder: nn.Module</span>
+<span class="sd">        mlp model that will receive the concatenation of the embeddings and</span>
+<span class="sd">        the continuous columns</span>
+
+<span class="sd">    Examples</span>
+<span class="sd">    --------</span>
+<span class="sd">    &gt;&gt;&gt; import torch</span>
+<span class="sd">    &gt;&gt;&gt; from pytorch_widedeep.models import TabMlp</span>
+<span class="sd">    &gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)</span>
+<span class="sd">    &gt;&gt;&gt; colnames = [&quot;a&quot;, &quot;b&quot;, &quot;c&quot;, &quot;d&quot;, &quot;e&quot;]</span>
+<span class="sd">    &gt;&gt;&gt; cat_embed_input = [(u, i, j) for u, i, j in zip(colnames[:4], [4] * 4, [8] * 4)]</span>
+<span class="sd">    &gt;&gt;&gt; column_idx = {k: v for v, k in enumerate(colnames)}</span>
+<span class="sd">    &gt;&gt;&gt; model = TabMlp(mlp_hidden_dims=[8, 4], column_idx=column_idx, cat_embed_input=cat_embed_input,</span>
+<span class="sd">    ... continuous_cols=[&quot;e&quot;])</span>
+<span class="sd">    &gt;&gt;&gt; out = model(X_tab)</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+
+    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span>
+        <span class="n">column_idx</span><span class="p">:</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">],</span>
+        <span class="o">*</span><span class="p">,</span>
+        <span class="n">cat_embed_input</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="n">Tuple</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">,</span> <span class="nb">int</span><span class="p">]]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">cat_embed_dropout</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">use_cat_bias</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">cat_embed_activation</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">continuous_cols</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">cont_norm_layer</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Literal</span><span class="p">[</span><span class="s2">&quot;batchnorm&quot;</span><span class="p">,</span> <span class="s2">&quot;layernorm&quot;</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">embed_continuous</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">embed_continuous_method</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span>
+            <span class="n">Literal</span><span class="p">[</span><span class="s2">&quot;standard&quot;</span><span class="p">,</span> <span class="s2">&quot;piecewise&quot;</span><span class="p">,</span> <span class="s2">&quot;periodic&quot;</span><span class="p">]</span>
+        <span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">cont_embed_dim</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">cont_embed_dropout</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">cont_embed_activation</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">quantization_setup</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="nb">float</span><span class="p">]]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">n_frequencies</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">sigma</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">share_last_layer</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">full_embed_dropout</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">mlp_hidden_dims</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="p">[</span><span class="mi">200</span><span class="p">,</span> <span class="mi">100</span><span class="p">],</span>
+        <span class="n">mlp_activation</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;relu&quot;</span><span class="p">,</span>
+        <span class="n">mlp_dropout</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="nb">float</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="nb">float</span><span class="p">]]</span> <span class="o">=</span> <span class="mf">0.1</span><span class="p">,</span>
+        <span class="n">mlp_batchnorm</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
+        <span class="n">mlp_batchnorm_last</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
+        <span class="n">mlp_linear_first</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">,</span>
+    <span class="p">):</span>
+        <span class="nb">super</span><span class="p">(</span><span class="n">TabMlp</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span>
+            <span class="n">column_idx</span><span class="o">=</span><span class="n">column_idx</span><span class="p">,</span>
+            <span class="n">cat_embed_input</span><span class="o">=</span><span class="n">cat_embed_input</span><span class="p">,</span>
+            <span class="n">cat_embed_dropout</span><span class="o">=</span><span class="n">cat_embed_dropout</span><span class="p">,</span>
+            <span class="n">use_cat_bias</span><span class="o">=</span><span class="n">use_cat_bias</span><span class="p">,</span>
+            <span class="n">cat_embed_activation</span><span class="o">=</span><span class="n">cat_embed_activation</span><span class="p">,</span>
+            <span class="n">continuous_cols</span><span class="o">=</span><span class="n">continuous_cols</span><span class="p">,</span>
+            <span class="n">cont_norm_layer</span><span class="o">=</span><span class="n">cont_norm_layer</span><span class="p">,</span>
+            <span class="n">embed_continuous</span><span class="o">=</span><span class="n">embed_continuous</span><span class="p">,</span>
+            <span class="n">embed_continuous_method</span><span class="o">=</span><span class="n">embed_continuous_method</span><span class="p">,</span>
+            <span class="n">cont_embed_dim</span><span class="o">=</span><span class="n">cont_embed_dim</span><span class="p">,</span>
+            <span class="n">cont_embed_dropout</span><span class="o">=</span><span class="n">cont_embed_dropout</span><span class="p">,</span>
+            <span class="n">cont_embed_activation</span><span class="o">=</span><span class="n">cont_embed_activation</span><span class="p">,</span>
+            <span class="n">quantization_setup</span><span class="o">=</span><span class="n">quantization_setup</span><span class="p">,</span>
+            <span class="n">n_frequencies</span><span class="o">=</span><span class="n">n_frequencies</span><span class="p">,</span>
+            <span class="n">sigma</span><span class="o">=</span><span class="n">sigma</span><span class="p">,</span>
+            <span class="n">share_last_layer</span><span class="o">=</span><span class="n">share_last_layer</span><span class="p">,</span>
+            <span class="n">full_embed_dropout</span><span class="o">=</span><span class="n">full_embed_dropout</span><span class="p">,</span>
+        <span class="p">)</span>
 
-    <span class="bp">self</span><span class="o">.</span><span class="n">mlp_hidden_dims</span> <span class="o">=</span> <span class="n">mlp_hidden_dims</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">mlp_activation</span> <span class="o">=</span> <span class="n">mlp_activation</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">mlp_dropout</span> <span class="o">=</span> <span class="n">mlp_dropout</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">mlp_batchnorm</span> <span class="o">=</span> <span class="n">mlp_batchnorm</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">mlp_batchnorm_last</span> <span class="o">=</span> <span class="n">mlp_batchnorm_last</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">mlp_linear_first</span> <span class="o">=</span> <span class="n">mlp_linear_first</span>
-
-    <span class="c1"># Embeddings are instantiated at the base model</span>
-    <span class="c1"># Mlp</span>
-    <span class="n">mlp_input_dim</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">cat_out_dim</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">cont_out_dim</span>
-    <span class="n">mlp_hidden_dims</span> <span class="o">=</span> <span class="p">[</span><span class="n">mlp_input_dim</span><span class="p">]</span> <span class="o">+</span> <span class="n">mlp_hidden_dims</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">encoder</span> <span class="o">=</span> <span class="n">MLP</span><span class="p">(</span>
-        <span class="n">mlp_hidden_dims</span><span class="p">,</span>
-        <span class="n">mlp_activation</span><span class="p">,</span>
-        <span class="n">mlp_dropout</span><span class="p">,</span>
-        <span class="n">mlp_batchnorm</span><span class="p">,</span>
-        <span class="n">mlp_batchnorm_last</span><span class="p">,</span>
-        <span class="n">mlp_linear_first</span><span class="p">,</span>
-    <span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">mlp_hidden_dims</span> <span class="o">=</span> <span class="n">mlp_hidden_dims</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">mlp_activation</span> <span class="o">=</span> <span class="n">mlp_activation</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">mlp_dropout</span> <span class="o">=</span> <span class="n">mlp_dropout</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">mlp_batchnorm</span> <span class="o">=</span> <span class="n">mlp_batchnorm</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">mlp_batchnorm_last</span> <span class="o">=</span> <span class="n">mlp_batchnorm_last</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">mlp_linear_first</span> <span class="o">=</span> <span class="n">mlp_linear_first</span>
+
+        <span class="c1"># Embeddings are instantiated at the base model</span>
+        <span class="c1"># Mlp</span>
+        <span class="n">mlp_input_dim</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">cat_out_dim</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">cont_out_dim</span>
+        <span class="n">mlp_hidden_dims</span> <span class="o">=</span> <span class="p">[</span><span class="n">mlp_input_dim</span><span class="p">]</span> <span class="o">+</span> <span class="n">mlp_hidden_dims</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">encoder</span> <span class="o">=</span> <span class="n">MLP</span><span class="p">(</span>
+            <span class="n">mlp_hidden_dims</span><span class="p">,</span>
+            <span class="n">mlp_activation</span><span class="p">,</span>
+            <span class="n">mlp_dropout</span><span class="p">,</span>
+            <span class="n">mlp_batchnorm</span><span class="p">,</span>
+            <span class="n">mlp_batchnorm_last</span><span class="p">,</span>
+            <span class="n">mlp_linear_first</span><span class="p">,</span>
+        <span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">forward</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">X</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tensor</span><span class="p">:</span>
+        <span class="n">x</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_embeddings</span><span class="p">(</span><span class="n">X</span><span class="p">)</span>
+        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">encoder</span><span class="p">(</span><span class="n">x</span><span class="p">)</span>
+
+    <span class="nd">@property</span>
+    <span class="k">def</span> <span class="nf">output_dim</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">int</span><span class="p">:</span>
+<span class="w">        </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;The output dimension of the model. This is a required property</span>
+<span class="sd">        neccesary to build the `WideDeep` class&quot;&quot;&quot;</span>
+        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_hidden_dims</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span>
 </code></pre></div></td></tr></table></div>
-                  </details>
+              </details>
 
 
 
@@ -2486,16 +2975,7 @@ <h2 id="pytorch_widedeep.models.tabular.mlp.tab_mlp.TabMlpDecoder" class="doc do
 
 
 <a href="#pytorch_widedeep.models.tabular.mlp.tab_mlp.TabMlpDecoder" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="nf">TabMlpDecoder</span><span class="p">(</span>
-    <span class="n">embed_dim</span><span class="p">,</span>
-    <span class="n">mlp_hidden_dims</span><span class="o">=</span><span class="p">[</span><span class="mi">100</span><span class="p">,</span> <span class="mi">200</span><span class="p">],</span>
-    <span class="n">mlp_activation</span><span class="o">=</span><span class="s2">&quot;relu&quot;</span><span class="p">,</span>
-    <span class="n">mlp_dropout</span><span class="o">=</span><span class="mf">0.1</span><span class="p">,</span>
-    <span class="n">mlp_batchnorm</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
-    <span class="n">mlp_batchnorm_last</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
-    <span class="n">mlp_linear_first</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
-<span class="p">)</span>
-</code></pre></div>
+
 
     <div class="doc doc-contents first">
             <p class="doc doc-class-bases">
@@ -2511,95 +2991,147 @@ <h2 id="pytorch_widedeep.models.tabular.mlp.tab_mlp.TabMlpDecoder" class="doc do
 
 
 <p><span class="doc-section-title">Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>embed_dim</code></b>
-              (<code>int</code>)
-          –
-          <div class="doc-md-description">
-            <p>Size of the embeddings tensor that needs to be reconstructed.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>mlp_hidden_dims</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.List">List</span>[int]</code>, default:
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+          <th>Default</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>embed_dim</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Size of the embeddings tensor that needs to be reconstructed.</p>
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>mlp_hidden_dims</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.List">List</span>[int]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>List with the number of neurons per dense layer in the mlp.</p>
+              </div>
+            </td>
+            <td>
                   <code>[100, 200]</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>List with the number of neurons per dense layer in the mlp.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>mlp_activation</code></b>
-              (<code>str</code>, default:
-                  <code>&#39;relu&#39;</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Activation function for the dense layers of the MLP. Currently
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>mlp_activation</code></td>
+            <td>
+                  <code>str</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Activation function for the dense layers of the MLP. Currently
 <em>'tanh'</em>, <em>'relu'</em>, <em>'leaky_relu'</em> and <em>'gelu'</em> are supported</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>mlp_dropout</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Union">Union</span>[float, <span title="pytorch_widedeep.wdtypes.List">List</span>[float]]</code>, default:
-                  <code>0.1</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>float or List of floats with the dropout between the dense layers.
+              </div>
+            </td>
+            <td>
+                  <code>&#39;relu&#39;</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>mlp_dropout</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Union">Union</span>[float, <span title="pytorch_widedeep.wdtypes.List">List</span>[float]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>float or List of floats with the dropout between the dense layers.
 e.g: <em>[0.5,0.5]</em></p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>mlp_batchnorm</code></b>
-              (<code>bool</code>, default:
-                  <code>False</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Boolean indicating whether or not batch normalization will be applied
+              </div>
+            </td>
+            <td>
+                  <code>0.1</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>mlp_batchnorm</code></td>
+            <td>
+                  <code>bool</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Boolean indicating whether or not batch normalization will be applied
 to the dense layers</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>mlp_batchnorm_last</code></b>
-              (<code>bool</code>, default:
+              </div>
+            </td>
+            <td>
                   <code>False</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Boolean indicating whether or not batch normalization will be applied
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>mlp_batchnorm_last</code></td>
+            <td>
+                  <code>bool</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Boolean indicating whether or not batch normalization will be applied
 to the last of the dense layers</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>mlp_linear_first</code></b>
-              (<code>bool</code>, default:
-                  <code>True</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Boolean indicating the order of the operations in the dense
+              </div>
+            </td>
+            <td>
+                  <code>False</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>mlp_linear_first</code></td>
+            <td>
+                  <code>bool</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Boolean indicating the order of the operations in the dense
 layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt;
 LIN -&gt; ACT]</code></p>
-          </div>
-        </li>
-    </ul>
+              </div>
+            </td>
+            <td>
+                  <code>True</code>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
 
 <p><span class="doc-section-title">Attributes:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code><span title="pytorch_widedeep.models.tabular.mlp.tab_mlp.TabMlpDecoder.decoder">decoder</span></code></b>
-              (<code><span title="torch.nn.Module">Module</span></code>)
-          –
-          <div class="doc-md-description">
-            <p>mlp model that will receive the output of the encoder</p>
-          </div>
-        </li>
-    </ul>
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code><span title="pytorch_widedeep.models.tabular.mlp.tab_mlp.TabMlpDecoder.decoder">decoder</span></code></td>
+            <td>
+                  <code><span title="torch.nn.Module">Module</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>mlp model that will receive the output of the encoder</p>
+              </div>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
 
 <p><span class="doc-section-title">Examples:</span></p>
@@ -2612,9 +3144,57 @@ <h2 id="pytorch_widedeep.models.tabular.mlp.tab_mlp.TabMlpDecoder" class="doc do
 <span class="go">torch.Size([3, 32])</span>
 </code></pre></div>
 
-                  <details class="quote">
-                    <summary>Source code in <code>pytorch_widedeep/models/tabular/mlp/tab_mlp.py</code></summary>
-                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">271</span>
+              <details class="quote">
+                <summary>Source code in <code>pytorch_widedeep/models/tabular/mlp/tab_mlp.py</code></summary>
+                <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">223</span>
+<span class="normal">224</span>
+<span class="normal">225</span>
+<span class="normal">226</span>
+<span class="normal">227</span>
+<span class="normal">228</span>
+<span class="normal">229</span>
+<span class="normal">230</span>
+<span class="normal">231</span>
+<span class="normal">232</span>
+<span class="normal">233</span>
+<span class="normal">234</span>
+<span class="normal">235</span>
+<span class="normal">236</span>
+<span class="normal">237</span>
+<span class="normal">238</span>
+<span class="normal">239</span>
+<span class="normal">240</span>
+<span class="normal">241</span>
+<span class="normal">242</span>
+<span class="normal">243</span>
+<span class="normal">244</span>
+<span class="normal">245</span>
+<span class="normal">246</span>
+<span class="normal">247</span>
+<span class="normal">248</span>
+<span class="normal">249</span>
+<span class="normal">250</span>
+<span class="normal">251</span>
+<span class="normal">252</span>
+<span class="normal">253</span>
+<span class="normal">254</span>
+<span class="normal">255</span>
+<span class="normal">256</span>
+<span class="normal">257</span>
+<span class="normal">258</span>
+<span class="normal">259</span>
+<span class="normal">260</span>
+<span class="normal">261</span>
+<span class="normal">262</span>
+<span class="normal">263</span>
+<span class="normal">264</span>
+<span class="normal">265</span>
+<span class="normal">266</span>
+<span class="normal">267</span>
+<span class="normal">268</span>
+<span class="normal">269</span>
+<span class="normal">270</span>
+<span class="normal">271</span>
 <span class="normal">272</span>
 <span class="normal">273</span>
 <span class="normal">274</span>
@@ -2642,37 +3222,91 @@ <h2 id="pytorch_widedeep.models.tabular.mlp.tab_mlp.TabMlpDecoder" class="doc do
 <span class="normal">296</span>
 <span class="normal">297</span>
 <span class="normal">298</span>
-<span class="normal">299</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
-    <span class="bp">self</span><span class="p">,</span>
-    <span class="n">embed_dim</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span>
-    <span class="n">mlp_hidden_dims</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="p">[</span><span class="mi">100</span><span class="p">,</span> <span class="mi">200</span><span class="p">],</span>
-    <span class="n">mlp_activation</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;relu&quot;</span><span class="p">,</span>
-    <span class="n">mlp_dropout</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="nb">float</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="nb">float</span><span class="p">]]</span> <span class="o">=</span> <span class="mf">0.1</span><span class="p">,</span>
-    <span class="n">mlp_batchnorm</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
-    <span class="n">mlp_batchnorm_last</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
-    <span class="n">mlp_linear_first</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">,</span>
-<span class="p">):</span>
-    <span class="nb">super</span><span class="p">(</span><span class="n">TabMlpDecoder</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
-
-    <span class="bp">self</span><span class="o">.</span><span class="n">embed_dim</span> <span class="o">=</span> <span class="n">embed_dim</span>
-
-    <span class="bp">self</span><span class="o">.</span><span class="n">mlp_hidden_dims</span> <span class="o">=</span> <span class="n">mlp_hidden_dims</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">mlp_activation</span> <span class="o">=</span> <span class="n">mlp_activation</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">mlp_dropout</span> <span class="o">=</span> <span class="n">mlp_dropout</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">mlp_batchnorm</span> <span class="o">=</span> <span class="n">mlp_batchnorm</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">mlp_batchnorm_last</span> <span class="o">=</span> <span class="n">mlp_batchnorm_last</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">mlp_linear_first</span> <span class="o">=</span> <span class="n">mlp_linear_first</span>
-
-    <span class="bp">self</span><span class="o">.</span><span class="n">decoder</span> <span class="o">=</span> <span class="n">MLP</span><span class="p">(</span>
-        <span class="n">mlp_hidden_dims</span> <span class="o">+</span> <span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">embed_dim</span><span class="p">],</span>
-        <span class="n">mlp_activation</span><span class="p">,</span>
-        <span class="n">mlp_dropout</span><span class="p">,</span>
-        <span class="n">mlp_batchnorm</span><span class="p">,</span>
-        <span class="n">mlp_batchnorm_last</span><span class="p">,</span>
-        <span class="n">mlp_linear_first</span><span class="p">,</span>
-    <span class="p">)</span>
+<span class="normal">299</span>
+<span class="normal">300</span>
+<span class="normal">301</span>
+<span class="normal">302</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">class</span> <span class="nc">TabMlpDecoder</span><span class="p">(</span><span class="n">nn</span><span class="o">.</span><span class="n">Module</span><span class="p">):</span>
+<span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Companion decoder model for the `TabMlp` model (which can be considered</span>
+<span class="sd">    an encoder itself).</span>
+
+<span class="sd">    This class is designed to be used with the `EncoderDecoderTrainer` when</span>
+<span class="sd">    using self-supervised pre-training (see the corresponding section in the</span>
+<span class="sd">    docs). The `TabMlpDecoder` will receive the output from the MLP</span>
+<span class="sd">    and &#39;_reconstruct_&#39; the embeddings.</span>
+
+<span class="sd">    Parameters</span>
+<span class="sd">    ----------</span>
+<span class="sd">    embed_dim: int</span>
+<span class="sd">        Size of the embeddings tensor that needs to be reconstructed.</span>
+<span class="sd">    mlp_hidden_dims: List, default = [200, 100]</span>
+<span class="sd">        List with the number of neurons per dense layer in the mlp.</span>
+<span class="sd">    mlp_activation: str, default = &quot;relu&quot;</span>
+<span class="sd">        Activation function for the dense layers of the MLP. Currently</span>
+<span class="sd">        _&#39;tanh&#39;_, _&#39;relu&#39;_, _&#39;leaky_relu&#39;_ and _&#39;gelu&#39;_ are supported</span>
+<span class="sd">    mlp_dropout: float or List, default = 0.1</span>
+<span class="sd">        float or List of floats with the dropout between the dense layers.</span>
+<span class="sd">        e.g: _[0.5,0.5]_</span>
+<span class="sd">    mlp_batchnorm: bool, default = False</span>
+<span class="sd">        Boolean indicating whether or not batch normalization will be applied</span>
+<span class="sd">        to the dense layers</span>
+<span class="sd">    mlp_batchnorm_last: bool, default = False</span>
+<span class="sd">        Boolean indicating whether or not batch normalization will be applied</span>
+<span class="sd">        to the last of the dense layers</span>
+<span class="sd">    mlp_linear_first: bool, default = False</span>
+<span class="sd">        Boolean indicating the order of the operations in the dense</span>
+<span class="sd">        layer. If `True: [LIN -&gt; ACT -&gt; BN -&gt; DP]`. If `False: [BN -&gt; DP -&gt;</span>
+<span class="sd">        LIN -&gt; ACT]`</span>
+
+<span class="sd">    Attributes</span>
+<span class="sd">    ----------</span>
+<span class="sd">    decoder: nn.Module</span>
+<span class="sd">        mlp model that will receive the output of the encoder</span>
+
+<span class="sd">    Examples</span>
+<span class="sd">    --------</span>
+<span class="sd">    &gt;&gt;&gt; import torch</span>
+<span class="sd">    &gt;&gt;&gt; from pytorch_widedeep.models import TabMlpDecoder</span>
+<span class="sd">    &gt;&gt;&gt; x_inp = torch.rand(3, 8)</span>
+<span class="sd">    &gt;&gt;&gt; decoder = TabMlpDecoder(embed_dim=32, mlp_hidden_dims=[8,16])</span>
+<span class="sd">    &gt;&gt;&gt; res = decoder(x_inp)</span>
+<span class="sd">    &gt;&gt;&gt; res.shape</span>
+<span class="sd">    torch.Size([3, 32])</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+
+    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span>
+        <span class="n">embed_dim</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span>
+        <span class="n">mlp_hidden_dims</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="p">[</span><span class="mi">100</span><span class="p">,</span> <span class="mi">200</span><span class="p">],</span>
+        <span class="n">mlp_activation</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;relu&quot;</span><span class="p">,</span>
+        <span class="n">mlp_dropout</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="nb">float</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="nb">float</span><span class="p">]]</span> <span class="o">=</span> <span class="mf">0.1</span><span class="p">,</span>
+        <span class="n">mlp_batchnorm</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
+        <span class="n">mlp_batchnorm_last</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
+        <span class="n">mlp_linear_first</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">,</span>
+    <span class="p">):</span>
+        <span class="nb">super</span><span class="p">(</span><span class="n">TabMlpDecoder</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">embed_dim</span> <span class="o">=</span> <span class="n">embed_dim</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">mlp_hidden_dims</span> <span class="o">=</span> <span class="n">mlp_hidden_dims</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">mlp_activation</span> <span class="o">=</span> <span class="n">mlp_activation</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">mlp_dropout</span> <span class="o">=</span> <span class="n">mlp_dropout</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">mlp_batchnorm</span> <span class="o">=</span> <span class="n">mlp_batchnorm</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">mlp_batchnorm_last</span> <span class="o">=</span> <span class="n">mlp_batchnorm_last</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">mlp_linear_first</span> <span class="o">=</span> <span class="n">mlp_linear_first</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">decoder</span> <span class="o">=</span> <span class="n">MLP</span><span class="p">(</span>
+            <span class="n">mlp_hidden_dims</span> <span class="o">+</span> <span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">embed_dim</span><span class="p">],</span>
+            <span class="n">mlp_activation</span><span class="p">,</span>
+            <span class="n">mlp_dropout</span><span class="p">,</span>
+            <span class="n">mlp_batchnorm</span><span class="p">,</span>
+            <span class="n">mlp_batchnorm_last</span><span class="p">,</span>
+            <span class="n">mlp_linear_first</span><span class="p">,</span>
+        <span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">forward</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">X</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tensor</span><span class="p">:</span>
+        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">decoder</span><span class="p">(</span><span class="n">X</span><span class="p">)</span>
 </code></pre></div></td></tr></table></div>
-                  </details>
+              </details>
 
 
 
@@ -2703,36 +3337,7 @@ <h2 id="pytorch_widedeep.models.tabular.resnet.tab_resnet.TabResnet" class="doc
 
 
 <a href="#pytorch_widedeep.models.tabular.resnet.tab_resnet.TabResnet" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="nf">TabResnet</span><span class="p">(</span>
-    <span class="n">column_idx</span><span class="p">,</span>
-    <span class="o">*</span><span class="p">,</span>
-    <span class="n">cat_embed_input</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">cat_embed_dropout</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">use_cat_bias</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">cat_embed_activation</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">continuous_cols</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">cont_norm_layer</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">embed_continuous</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">embed_continuous_method</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">cont_embed_dim</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">cont_embed_dropout</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">cont_embed_activation</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">quantization_setup</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">n_frequencies</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">sigma</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">share_last_layer</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">full_embed_dropout</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">blocks_dims</span><span class="o">=</span><span class="p">[</span><span class="mi">200</span><span class="p">,</span> <span class="mi">100</span><span class="p">,</span> <span class="mi">100</span><span class="p">],</span>
-    <span class="n">blocks_dropout</span><span class="o">=</span><span class="mf">0.1</span><span class="p">,</span>
-    <span class="n">simplify_blocks</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
-    <span class="n">mlp_hidden_dims</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">mlp_activation</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">mlp_dropout</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">mlp_batchnorm</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">mlp_batchnorm_last</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">mlp_linear_first</span><span class="o">=</span><span class="kc">None</span>
-<span class="p">)</span>
-</code></pre></div>
+
 
     <div class="doc doc-contents first">
             <p class="doc doc-class-bases">
@@ -2752,106 +3357,149 @@ <h2 id="pytorch_widedeep.models.tabular.resnet.tab_resnet.TabResnet" class="doc
 
 
 <p><span class="doc-section-title">Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>column_idx</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, int]</code>)
-          –
-          <div class="doc-md-description">
-            <p>Dict containing the index of the columns that will be passed through
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+          <th>Default</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>column_idx</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, int]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Dict containing the index of the columns that will be passed through
 the <code>TabMlp</code> model. Required to slice the tensors. e.g. <em>{'education':
 0, 'relationship': 1, 'workclass': 2, ...}</em>.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>cat_embed_input</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.wdtypes.Tuple">Tuple</span>[str, int, int]]]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>List of Tuples with the column name, number of unique values and
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>cat_embed_input</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.wdtypes.Tuple">Tuple</span>[str, int, int]]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>List of Tuples with the column name, number of unique values and
 embedding dimension. e.g. <em>[(education, 11, 32), ...]</em></p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>cat_embed_dropout</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Categorical embeddings dropout. If <code>None</code>, it will default
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>cat_embed_dropout</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Categorical embeddings dropout. If <code>None</code>, it will default
 to 0.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>use_cat_bias</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Boolean indicating if bias will be used for the categorical embeddings.
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>use_cat_bias</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Boolean indicating if bias will be used for the categorical embeddings.
 If <code>None</code>, it will default to 'False'.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>cat_embed_activation</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Activation function for the categorical embeddings, if any. Currently
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>cat_embed_activation</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Activation function for the categorical embeddings, if any. Currently
 <em>'tanh'</em>, <em>'relu'</em>, <em>'leaky_relu'</em> and <em>'gelu'</em> are supported</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>continuous_cols</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[str]]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>List with the name of the numeric (aka continuous) columns</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>cont_norm_layer</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[batchnorm, layernorm]]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Type of normalization layer applied to the continuous features.
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>continuous_cols</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[str]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>List with the name of the numeric (aka continuous) columns</p>
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>cont_norm_layer</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[batchnorm, layernorm]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Type of normalization layer applied to the continuous features.
 Options are: <em>'layernorm'</em> and <em>'batchnorm'</em>. if <code>None</code>, no
 normalization layer will be used.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>embed_continuous</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Boolean indicating if the continuous columns will be embedded using
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>embed_continuous</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Boolean indicating if the continuous columns will be embedded using
 one of the available methods: <em>'standard'</em>, <em>'periodic'</em>
 or <em>'piecewise'</em>. If <code>None</code>, it will default to 'False'.<br/>
 <img alt="ℹ️" class="emojione" src="https://cdnjs.cloudflare.com/ajax/libs/emojione/2.2.7/assets/png/2139.png" title=":information_source:" /> <strong>NOTE</strong>: This parameter is deprecated and it
  will be removed in future releases. Please, use the
  <code>embed_continuous_method</code> parameter instead.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>embed_continuous_method</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[standard, piecewise, periodic]]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Method to use to embed the continuous features. Options are:
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>embed_continuous_method</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[standard, piecewise, periodic]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Method to use to embed the continuous features. Options are:
 <em>'standard'</em>, <em>'periodic'</em> or <em>'piecewise'</em>. The <em>'standard'</em>
 embedding method is based on the FT-Transformer implementation
 presented in the paper: <a href="https://arxiv.org/abs/2106.11959v5">Revisiting Deep Learning Models for
@@ -2859,247 +3507,335 @@ <h2 id="pytorch_widedeep.models.tabular.resnet.tab_resnet.TabResnet" class="doc
 and_'piecewise'_ methods were presented in the paper: <a href="https://arxiv.org/abs/2203.05556">On Embeddings for
 Numerical Features in Tabular Deep Learning</a>.
 Please, read the papers for details.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>cont_embed_dim</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[int]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Size of the continuous embeddings. If the continuous columns are
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>cont_embed_dim</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[int]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Size of the continuous embeddings. If the continuous columns are
 embedded, <code>cont_embed_dim</code> must be passed.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>cont_embed_dropout</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>cont_embed_activation</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Activation function for the continuous embeddings if any. Currently
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>cont_embed_dropout</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p>
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>cont_embed_activation</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Activation function for the continuous embeddings if any. Currently
 <em>'tanh'</em>, <em>'relu'</em>, <em>'leaky_relu'</em> and <em>'gelu'</em> are supported.
 If <code>None</code>, no activation function will be applied.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>quantization_setup</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, <span title="pytorch_widedeep.wdtypes.List">List</span>[float]]]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>This parameter is used when the <em>'piecewise'</em> method is used to embed
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>quantization_setup</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, <span title="pytorch_widedeep.wdtypes.List">List</span>[float]]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>This parameter is used when the <em>'piecewise'</em> method is used to embed
 the continuous cols. It is a dict where keys are the name of the continuous
 columns and values are lists with the boundaries for the quantization
 of the continuous_cols. See the examples for details. If
 If the <em>'piecewise'</em> method is used, this parameter is required.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>n_frequencies</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[int]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>This is the so called <em>'k'</em> in their paper <a href="https://arxiv.org/abs/2203.05556">On Embeddings for
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>n_frequencies</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[int]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>This is the so called <em>'k'</em> in their paper <a href="https://arxiv.org/abs/2203.05556">On Embeddings for
 Numerical Features in Tabular Deep Learning</a>,
 and is the number of 'frequencies' that will be used to represent each
 continuous column. See their Eq 2 in the paper for details. If
 the <em>'periodic'</em> method is used, this parameter is required.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>sigma</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>This is the sigma parameter in the paper mentioned when describing the
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>sigma</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>This is the sigma parameter in the paper mentioned when describing the
 previous parameters and it is used to initialise the 'frequency
 weights'. See their Eq 2 in the paper for details. If
 the <em>'periodic'</em> method is used, this parameter is required.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>share_last_layer</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>This parameter is not present in the before mentioned paper but it is implemented in
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>share_last_layer</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>This parameter is not present in the before mentioned paper but it is implemented in
 the <a href="https://github.com/yandex-research/rtdl-num-embeddings/tree/main">official repo</a>.
 If <code>True</code> the linear layer that turns the frequencies into embeddings
 will be shared across the continuous columns. If <code>False</code> a different
 linear layer will be used for each continuous column.
 If the <em>'periodic'</em> method is used, this parameter is required.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>full_embed_dropout</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>If <code>True</code>, the full embedding corresponding to a column will be masked
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>full_embed_dropout</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>If <code>True</code>, the full embedding corresponding to a column will be masked
 out/dropout. If <code>None</code>, it will default to <code>False</code>.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>blocks_dims</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.List">List</span>[int]</code>, default:
-                  <code>[200, 100, 100]</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>List of integers that define the input and output units of each block.
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>blocks_dims</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.List">List</span>[int]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>List of integers that define the input and output units of each block.
 For example: <em>[200, 100, 100]</em> will generate 2 blocks. The first will
 receive a tensor of size 200 and output a tensor of size 100, and the
 second will receive a tensor of size 100 and output a tensor of size
 100. See <code>pytorch_widedeep.models.tab_resnet._layers</code> for
 details on the structure of each block.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>blocks_dropout</code></b>
-              (<code>float</code>, default:
+              </div>
+            </td>
+            <td>
+                  <code>[200, 100, 100]</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>blocks_dropout</code></td>
+            <td>
+                  <code>float</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Block's internal dropout.</p>
+              </div>
+            </td>
+            <td>
                   <code>0.1</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Block's internal dropout.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>simplify_blocks</code></b>
-              (<code>bool</code>, default:
-                  <code>False</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Boolean indicating if the simplest possible residual blocks (<code>X -&gt; [
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>simplify_blocks</code></td>
+            <td>
+                  <code>bool</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Boolean indicating if the simplest possible residual blocks (<code>X -&gt; [
 [LIN, BN, ACT]  + X ]</code>) will be used instead of a standard one
 (<code>X -&gt; [ [LIN1, BN1, ACT1] -&gt; [LIN2, BN2]  + X ]</code>).</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>mlp_hidden_dims</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[int]]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>List with the number of neurons per dense layer in the MLP. e.g:
+              </div>
+            </td>
+            <td>
+                  <code>False</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>mlp_hidden_dims</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[int]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>List with the number of neurons per dense layer in the MLP. e.g:
 <em>[64, 32]</em>. If <code>None</code> the  output of the Resnet Blocks will be
 connected directly to the output neuron(s).</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>mlp_activation</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Activation function for the dense layers of the MLP. Currently
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>mlp_activation</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Activation function for the dense layers of the MLP. Currently
 <em>'tanh'</em>, <em>'relu'</em>, <em>'leaky'_relu' and _'gelu'</em> are supported.
 If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it
 will default to <em>'relu'</em>.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>mlp_dropout</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>float with the dropout between the dense layers of the MLP.
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>mlp_dropout</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>float with the dropout between the dense layers of the MLP.
 If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it
 will default to 0.0.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>mlp_batchnorm</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Boolean indicating whether or not batch normalization will be applied
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>mlp_batchnorm</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Boolean indicating whether or not batch normalization will be applied
 to the dense layers
 If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it
 will default to False.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>mlp_batchnorm_last</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Boolean indicating whether or not batch normalization will be applied
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>mlp_batchnorm_last</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Boolean indicating whether or not batch normalization will be applied
 to the last of the dense layers
 If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it
 will default to False.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>mlp_linear_first</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Boolean indicating the order of the operations in the dense
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>mlp_linear_first</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Boolean indicating the order of the operations in the dense
 layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt;
 LIN -&gt; ACT]</code>
 If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it
 will default to <code>True</code>.</p>
-          </div>
-        </li>
-    </ul>
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
 
 <p><span class="doc-section-title">Attributes:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code><span title="pytorch_widedeep.models.tabular.resnet.tab_resnet.TabResnet.encoder">encoder</span></code></b>
-              (<code><span title="torch.nn.Module">Module</span></code>)
-          –
-          <div class="doc-md-description">
-            <p>deep dense Resnet model that will receive the concatenation of the
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code><span title="pytorch_widedeep.models.tabular.resnet.tab_resnet.TabResnet.encoder">encoder</span></code></td>
+            <td>
+                  <code><span title="torch.nn.Module">Module</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>deep dense Resnet model that will receive the concatenation of the
 embeddings and the continuous columns</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code><span title="pytorch_widedeep.models.tabular.resnet.tab_resnet.TabResnet.mlp">mlp</span></code></b>
-              (<code><span title="torch.nn.Module">Module</span></code>)
-          –
-          <div class="doc-md-description">
-            <p>if <code>mlp_hidden_dims</code> is <code>True</code>, this attribute will be an mlp
+              </div>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code><span title="pytorch_widedeep.models.tabular.resnet.tab_resnet.TabResnet.mlp">mlp</span></code></td>
+            <td>
+                  <code><span title="torch.nn.Module">Module</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>if <code>mlp_hidden_dims</code> is <code>True</code>, this attribute will be an mlp
 model that will receive the results of the concatenation of the
 embeddings and the continuous columns -- if present --.</p>
-          </div>
-        </li>
-    </ul>
+              </div>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
 
 <p><span class="doc-section-title">Examples:</span></p>
@@ -3114,9 +3850,165 @@ <h2 id="pytorch_widedeep.models.tabular.resnet.tab_resnet.TabResnet" class="doc
 <span class="gp">&gt;&gt;&gt; </span><span class="n">out</span> <span class="o">=</span> <span class="n">model</span><span class="p">(</span><span class="n">X_deep</span><span class="p">)</span>
 </code></pre></div>
 
-                  <details class="quote">
-                    <summary>Source code in <code>pytorch_widedeep/models/tabular/resnet/tab_resnet.py</code></summary>
-                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">167</span>
+              <details class="quote">
+                <summary>Source code in <code>pytorch_widedeep/models/tabular/resnet/tab_resnet.py</code></summary>
+                <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 11</span>
+<span class="normal"> 12</span>
+<span class="normal"> 13</span>
+<span class="normal"> 14</span>
+<span class="normal"> 15</span>
+<span class="normal"> 16</span>
+<span class="normal"> 17</span>
+<span class="normal"> 18</span>
+<span class="normal"> 19</span>
+<span class="normal"> 20</span>
+<span class="normal"> 21</span>
+<span class="normal"> 22</span>
+<span class="normal"> 23</span>
+<span class="normal"> 24</span>
+<span class="normal"> 25</span>
+<span class="normal"> 26</span>
+<span class="normal"> 27</span>
+<span class="normal"> 28</span>
+<span class="normal"> 29</span>
+<span class="normal"> 30</span>
+<span class="normal"> 31</span>
+<span class="normal"> 32</span>
+<span class="normal"> 33</span>
+<span class="normal"> 34</span>
+<span class="normal"> 35</span>
+<span class="normal"> 36</span>
+<span class="normal"> 37</span>
+<span class="normal"> 38</span>
+<span class="normal"> 39</span>
+<span class="normal"> 40</span>
+<span class="normal"> 41</span>
+<span class="normal"> 42</span>
+<span class="normal"> 43</span>
+<span class="normal"> 44</span>
+<span class="normal"> 45</span>
+<span class="normal"> 46</span>
+<span class="normal"> 47</span>
+<span class="normal"> 48</span>
+<span class="normal"> 49</span>
+<span class="normal"> 50</span>
+<span class="normal"> 51</span>
+<span class="normal"> 52</span>
+<span class="normal"> 53</span>
+<span class="normal"> 54</span>
+<span class="normal"> 55</span>
+<span class="normal"> 56</span>
+<span class="normal"> 57</span>
+<span class="normal"> 58</span>
+<span class="normal"> 59</span>
+<span class="normal"> 60</span>
+<span class="normal"> 61</span>
+<span class="normal"> 62</span>
+<span class="normal"> 63</span>
+<span class="normal"> 64</span>
+<span class="normal"> 65</span>
+<span class="normal"> 66</span>
+<span class="normal"> 67</span>
+<span class="normal"> 68</span>
+<span class="normal"> 69</span>
+<span class="normal"> 70</span>
+<span class="normal"> 71</span>
+<span class="normal"> 72</span>
+<span class="normal"> 73</span>
+<span class="normal"> 74</span>
+<span class="normal"> 75</span>
+<span class="normal"> 76</span>
+<span class="normal"> 77</span>
+<span class="normal"> 78</span>
+<span class="normal"> 79</span>
+<span class="normal"> 80</span>
+<span class="normal"> 81</span>
+<span class="normal"> 82</span>
+<span class="normal"> 83</span>
+<span class="normal"> 84</span>
+<span class="normal"> 85</span>
+<span class="normal"> 86</span>
+<span class="normal"> 87</span>
+<span class="normal"> 88</span>
+<span class="normal"> 89</span>
+<span class="normal"> 90</span>
+<span class="normal"> 91</span>
+<span class="normal"> 92</span>
+<span class="normal"> 93</span>
+<span class="normal"> 94</span>
+<span class="normal"> 95</span>
+<span class="normal"> 96</span>
+<span class="normal"> 97</span>
+<span class="normal"> 98</span>
+<span class="normal"> 99</span>
+<span class="normal">100</span>
+<span class="normal">101</span>
+<span class="normal">102</span>
+<span class="normal">103</span>
+<span class="normal">104</span>
+<span class="normal">105</span>
+<span class="normal">106</span>
+<span class="normal">107</span>
+<span class="normal">108</span>
+<span class="normal">109</span>
+<span class="normal">110</span>
+<span class="normal">111</span>
+<span class="normal">112</span>
+<span class="normal">113</span>
+<span class="normal">114</span>
+<span class="normal">115</span>
+<span class="normal">116</span>
+<span class="normal">117</span>
+<span class="normal">118</span>
+<span class="normal">119</span>
+<span class="normal">120</span>
+<span class="normal">121</span>
+<span class="normal">122</span>
+<span class="normal">123</span>
+<span class="normal">124</span>
+<span class="normal">125</span>
+<span class="normal">126</span>
+<span class="normal">127</span>
+<span class="normal">128</span>
+<span class="normal">129</span>
+<span class="normal">130</span>
+<span class="normal">131</span>
+<span class="normal">132</span>
+<span class="normal">133</span>
+<span class="normal">134</span>
+<span class="normal">135</span>
+<span class="normal">136</span>
+<span class="normal">137</span>
+<span class="normal">138</span>
+<span class="normal">139</span>
+<span class="normal">140</span>
+<span class="normal">141</span>
+<span class="normal">142</span>
+<span class="normal">143</span>
+<span class="normal">144</span>
+<span class="normal">145</span>
+<span class="normal">146</span>
+<span class="normal">147</span>
+<span class="normal">148</span>
+<span class="normal">149</span>
+<span class="normal">150</span>
+<span class="normal">151</span>
+<span class="normal">152</span>
+<span class="normal">153</span>
+<span class="normal">154</span>
+<span class="normal">155</span>
+<span class="normal">156</span>
+<span class="normal">157</span>
+<span class="normal">158</span>
+<span class="normal">159</span>
+<span class="normal">160</span>
+<span class="normal">161</span>
+<span class="normal">162</span>
+<span class="normal">163</span>
+<span class="normal">164</span>
+<span class="normal">165</span>
+<span class="normal">166</span>
+<span class="normal">167</span>
 <span class="normal">168</span>
 <span class="normal">169</span>
 <span class="normal">170</span>
@@ -3212,105 +4104,297 @@ <h2 id="pytorch_widedeep.models.tabular.resnet.tab_resnet.TabResnet" class="doc
 <span class="normal">260</span>
 <span class="normal">261</span>
 <span class="normal">262</span>
-<span class="normal">263</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
-    <span class="bp">self</span><span class="p">,</span>
-    <span class="n">column_idx</span><span class="p">:</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">],</span>
-    <span class="o">*</span><span class="p">,</span>
-    <span class="n">cat_embed_input</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="n">Tuple</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">,</span> <span class="nb">int</span><span class="p">]]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">cat_embed_dropout</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">use_cat_bias</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">cat_embed_activation</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">continuous_cols</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">cont_norm_layer</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Literal</span><span class="p">[</span><span class="s2">&quot;batchnorm&quot;</span><span class="p">,</span> <span class="s2">&quot;layernorm&quot;</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">embed_continuous</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">embed_continuous_method</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span>
-        <span class="n">Literal</span><span class="p">[</span><span class="s2">&quot;standard&quot;</span><span class="p">,</span> <span class="s2">&quot;piecewise&quot;</span><span class="p">,</span> <span class="s2">&quot;periodic&quot;</span><span class="p">]</span>
-    <span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">cont_embed_dim</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">cont_embed_dropout</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">cont_embed_activation</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">quantization_setup</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="nb">float</span><span class="p">]]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">n_frequencies</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">sigma</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">share_last_layer</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">full_embed_dropout</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">blocks_dims</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="p">[</span><span class="mi">200</span><span class="p">,</span> <span class="mi">100</span><span class="p">,</span> <span class="mi">100</span><span class="p">],</span>
-    <span class="n">blocks_dropout</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.1</span><span class="p">,</span>
-    <span class="n">simplify_blocks</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
-    <span class="n">mlp_hidden_dims</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">int</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">mlp_activation</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">mlp_dropout</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">mlp_batchnorm</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">mlp_batchnorm_last</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">mlp_linear_first</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-<span class="p">):</span>
-    <span class="nb">super</span><span class="p">(</span><span class="n">TabResnet</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span>
-        <span class="n">column_idx</span><span class="o">=</span><span class="n">column_idx</span><span class="p">,</span>
-        <span class="n">cat_embed_input</span><span class="o">=</span><span class="n">cat_embed_input</span><span class="p">,</span>
-        <span class="n">cat_embed_dropout</span><span class="o">=</span><span class="n">cat_embed_dropout</span><span class="p">,</span>
-        <span class="n">use_cat_bias</span><span class="o">=</span><span class="n">use_cat_bias</span><span class="p">,</span>
-        <span class="n">cat_embed_activation</span><span class="o">=</span><span class="n">cat_embed_activation</span><span class="p">,</span>
-        <span class="n">continuous_cols</span><span class="o">=</span><span class="n">continuous_cols</span><span class="p">,</span>
-        <span class="n">cont_norm_layer</span><span class="o">=</span><span class="n">cont_norm_layer</span><span class="p">,</span>
-        <span class="n">embed_continuous</span><span class="o">=</span><span class="n">embed_continuous</span><span class="p">,</span>
-        <span class="n">embed_continuous_method</span><span class="o">=</span><span class="n">embed_continuous_method</span><span class="p">,</span>
-        <span class="n">cont_embed_dim</span><span class="o">=</span><span class="n">cont_embed_dim</span><span class="p">,</span>
-        <span class="n">cont_embed_dropout</span><span class="o">=</span><span class="n">cont_embed_dropout</span><span class="p">,</span>
-        <span class="n">cont_embed_activation</span><span class="o">=</span><span class="n">cont_embed_activation</span><span class="p">,</span>
-        <span class="n">quantization_setup</span><span class="o">=</span><span class="n">quantization_setup</span><span class="p">,</span>
-        <span class="n">n_frequencies</span><span class="o">=</span><span class="n">n_frequencies</span><span class="p">,</span>
-        <span class="n">sigma</span><span class="o">=</span><span class="n">sigma</span><span class="p">,</span>
-        <span class="n">share_last_layer</span><span class="o">=</span><span class="n">share_last_layer</span><span class="p">,</span>
-        <span class="n">full_embed_dropout</span><span class="o">=</span><span class="n">full_embed_dropout</span><span class="p">,</span>
-    <span class="p">)</span>
-
-    <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">blocks_dims</span><span class="p">)</span> <span class="o">&lt;</span> <span class="mi">2</span><span class="p">:</span>
-        <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
-            <span class="s2">&quot;&#39;blocks&#39; must contain at least two elements, e.g. [256, 128]&quot;</span>
+<span class="normal">263</span>
+<span class="normal">264</span>
+<span class="normal">265</span>
+<span class="normal">266</span>
+<span class="normal">267</span>
+<span class="normal">268</span>
+<span class="normal">269</span>
+<span class="normal">270</span>
+<span class="normal">271</span>
+<span class="normal">272</span>
+<span class="normal">273</span>
+<span class="normal">274</span>
+<span class="normal">275</span>
+<span class="normal">276</span>
+<span class="normal">277</span>
+<span class="normal">278</span>
+<span class="normal">279</span>
+<span class="normal">280</span>
+<span class="normal">281</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">class</span> <span class="nc">TabResnet</span><span class="p">(</span><span class="n">BaseTabularModelWithoutAttention</span><span class="p">):</span>
+<span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Defines a `TabResnet` model that can be used as the `deeptabular`</span>
+<span class="sd">    component of a Wide &amp; Deep model or independently by itself.</span>
+
+<span class="sd">    This class combines embedding representations of the categorical features</span>
+<span class="sd">    with numerical (aka continuous) features, embedded or not. These are then</span>
+<span class="sd">    passed through a series of Resnet blocks. See</span>
+<span class="sd">    `pytorch_widedeep.models.tab_resnet._layers` for details on the</span>
+<span class="sd">    structure of each block.</span>
+
+<span class="sd">    Most of the parameters for this class are `Optional` since the use of</span>
+<span class="sd">    categorical or continuous is in fact optional (i.e. one can use</span>
+<span class="sd">    categorical features only, continuous features only or both).</span>
+
+<span class="sd">    Parameters</span>
+<span class="sd">    ----------</span>
+<span class="sd">    column_idx: Dict</span>
+<span class="sd">        Dict containing the index of the columns that will be passed through</span>
+<span class="sd">        the `TabMlp` model. Required to slice the tensors. e.g. _{&#39;education&#39;:</span>
+<span class="sd">        0, &#39;relationship&#39;: 1, &#39;workclass&#39;: 2, ...}_.</span>
+<span class="sd">    cat_embed_input: List, Optional, default = None</span>
+<span class="sd">        List of Tuples with the column name, number of unique values and</span>
+<span class="sd">        embedding dimension. e.g. _[(education, 11, 32), ...]_</span>
+<span class="sd">    cat_embed_dropout: float, Optional, default = None</span>
+<span class="sd">        Categorical embeddings dropout. If `None`, it will default</span>
+<span class="sd">        to 0.</span>
+<span class="sd">    use_cat_bias: bool, Optional, default = None,</span>
+<span class="sd">        Boolean indicating if bias will be used for the categorical embeddings.</span>
+<span class="sd">        If `None`, it will default to &#39;False&#39;.</span>
+<span class="sd">    cat_embed_activation: Optional, str, default = None,</span>
+<span class="sd">        Activation function for the categorical embeddings, if any. Currently</span>
+<span class="sd">        _&#39;tanh&#39;_, _&#39;relu&#39;_, _&#39;leaky_relu&#39;_ and _&#39;gelu&#39;_ are supported</span>
+<span class="sd">    continuous_cols: List, Optional, default = None</span>
+<span class="sd">        List with the name of the numeric (aka continuous) columns</span>
+<span class="sd">    cont_norm_layer: str, Optional, default =  None</span>
+<span class="sd">        Type of normalization layer applied to the continuous features.</span>
+<span class="sd">        Options are: _&#39;layernorm&#39;_ and _&#39;batchnorm&#39;_. if `None`, no</span>
+<span class="sd">        normalization layer will be used.</span>
+<span class="sd">    embed_continuous: bool, Optional, default = None,</span>
+<span class="sd">        Boolean indicating if the continuous columns will be embedded using</span>
+<span class="sd">        one of the available methods: _&#39;standard&#39;_, _&#39;periodic&#39;_</span>
+<span class="sd">        or _&#39;piecewise&#39;_. If `None`, it will default to &#39;False&#39;.&lt;br/&gt;</span>
+<span class="sd">        :information_source: **NOTE**: This parameter is deprecated and it</span>
+<span class="sd">         will be removed in future releases. Please, use the</span>
+<span class="sd">         `embed_continuous_method` parameter instead.</span>
+<span class="sd">    embed_continuous_method: Optional, str, default = None,</span>
+<span class="sd">        Method to use to embed the continuous features. Options are:</span>
+<span class="sd">        _&#39;standard&#39;_, _&#39;periodic&#39;_ or _&#39;piecewise&#39;_. The _&#39;standard&#39;_</span>
+<span class="sd">        embedding method is based on the FT-Transformer implementation</span>
+<span class="sd">        presented in the paper: [Revisiting Deep Learning Models for</span>
+<span class="sd">        Tabular Data](https://arxiv.org/abs/2106.11959v5). The _&#39;periodic&#39;_</span>
+<span class="sd">        and_&#39;piecewise&#39;_ methods were presented in the paper: [On Embeddings for</span>
+<span class="sd">        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556).</span>
+<span class="sd">        Please, read the papers for details.</span>
+<span class="sd">    cont_embed_dim: int, Optional, default = None,</span>
+<span class="sd">        Size of the continuous embeddings. If the continuous columns are</span>
+<span class="sd">        embedded, `cont_embed_dim` must be passed.</span>
+<span class="sd">    cont_embed_dropout: float, Optional, default = None,</span>
+<span class="sd">        Dropout for the continuous embeddings. If `None`, it will default to 0.0</span>
+<span class="sd">    cont_embed_activation: Optional, str, default = None,</span>
+<span class="sd">        Activation function for the continuous embeddings if any. Currently</span>
+<span class="sd">        _&#39;tanh&#39;_, _&#39;relu&#39;_, _&#39;leaky_relu&#39;_ and _&#39;gelu&#39;_ are supported.</span>
+<span class="sd">        If `None`, no activation function will be applied.</span>
+<span class="sd">    quantization_setup: Dict[str, List[float]], Optional, default = None,</span>
+<span class="sd">        This parameter is used when the _&#39;piecewise&#39;_ method is used to embed</span>
+<span class="sd">        the continuous cols. It is a dict where keys are the name of the continuous</span>
+<span class="sd">        columns and values are lists with the boundaries for the quantization</span>
+<span class="sd">        of the continuous_cols. See the examples for details. If</span>
+<span class="sd">        If the _&#39;piecewise&#39;_ method is used, this parameter is required.</span>
+<span class="sd">    n_frequencies: int, Optional, default = None,</span>
+<span class="sd">        This is the so called _&#39;k&#39;_ in their paper [On Embeddings for</span>
+<span class="sd">        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556),</span>
+<span class="sd">        and is the number of &#39;frequencies&#39; that will be used to represent each</span>
+<span class="sd">        continuous column. See their Eq 2 in the paper for details. If</span>
+<span class="sd">        the _&#39;periodic&#39;_ method is used, this parameter is required.</span>
+<span class="sd">    sigma: float, Optional, default = None,</span>
+<span class="sd">        This is the sigma parameter in the paper mentioned when describing the</span>
+<span class="sd">        previous parameters and it is used to initialise the &#39;frequency</span>
+<span class="sd">        weights&#39;. See their Eq 2 in the paper for details. If</span>
+<span class="sd">        the _&#39;periodic&#39;_ method is used, this parameter is required.</span>
+<span class="sd">    share_last_layer: bool, Optional, default = None,</span>
+<span class="sd">        This parameter is not present in the before mentioned paper but it is implemented in</span>
+<span class="sd">        the [official repo](https://github.com/yandex-research/rtdl-num-embeddings/tree/main).</span>
+<span class="sd">        If `True` the linear layer that turns the frequencies into embeddings</span>
+<span class="sd">        will be shared across the continuous columns. If `False` a different</span>
+<span class="sd">        linear layer will be used for each continuous column.</span>
+<span class="sd">        If the _&#39;periodic&#39;_ method is used, this parameter is required.</span>
+<span class="sd">    full_embed_dropout: bool, Optional, default = None,</span>
+<span class="sd">        If `True`, the full embedding corresponding to a column will be masked</span>
+<span class="sd">        out/dropout. If `None`, it will default to `False`.</span>
+<span class="sd">    blocks_dims: List, default = [200, 100, 100]</span>
+<span class="sd">        List of integers that define the input and output units of each block.</span>
+<span class="sd">        For example: _[200, 100, 100]_ will generate 2 blocks. The first will</span>
+<span class="sd">        receive a tensor of size 200 and output a tensor of size 100, and the</span>
+<span class="sd">        second will receive a tensor of size 100 and output a tensor of size</span>
+<span class="sd">        100. See `pytorch_widedeep.models.tab_resnet._layers` for</span>
+<span class="sd">        details on the structure of each block.</span>
+<span class="sd">    blocks_dropout: float, default =  0.1</span>
+<span class="sd">        Block&#39;s internal dropout.</span>
+<span class="sd">    simplify_blocks: bool, default = False,</span>
+<span class="sd">        Boolean indicating if the simplest possible residual blocks (`X -&gt; [</span>
+<span class="sd">        [LIN, BN, ACT]  + X ]`) will be used instead of a standard one</span>
+<span class="sd">        (`X -&gt; [ [LIN1, BN1, ACT1] -&gt; [LIN2, BN2]  + X ]`).</span>
+<span class="sd">    mlp_hidden_dims: List, Optional, default = None</span>
+<span class="sd">        List with the number of neurons per dense layer in the MLP. e.g:</span>
+<span class="sd">        _[64, 32]_. If `None` the  output of the Resnet Blocks will be</span>
+<span class="sd">        connected directly to the output neuron(s).</span>
+<span class="sd">    mlp_activation: str, Optional, default = None</span>
+<span class="sd">        Activation function for the dense layers of the MLP. Currently</span>
+<span class="sd">        _&#39;tanh&#39;_, _&#39;relu&#39;_, _&#39;leaky&#39;_relu&#39; and _&#39;gelu&#39;_ are supported.</span>
+<span class="sd">        If &#39;mlp_hidden_dims&#39; is not `None` and this parameter is `None`, it</span>
+<span class="sd">        will default to _&#39;relu&#39;_.</span>
+<span class="sd">    mlp_dropout: float, Optional, default = None</span>
+<span class="sd">        float with the dropout between the dense layers of the MLP.</span>
+<span class="sd">        If &#39;mlp_hidden_dims&#39; is not `None` and this parameter is `None`, it</span>
+<span class="sd">        will default to 0.0.</span>
+<span class="sd">    mlp_batchnorm: bool, Optional, default = None</span>
+<span class="sd">        Boolean indicating whether or not batch normalization will be applied</span>
+<span class="sd">        to the dense layers</span>
+<span class="sd">        If &#39;mlp_hidden_dims&#39; is not `None` and this parameter is `None`, it</span>
+<span class="sd">        will default to False.</span>
+<span class="sd">    mlp_batchnorm_last: bool, Optional, default = None</span>
+<span class="sd">        Boolean indicating whether or not batch normalization will be applied</span>
+<span class="sd">        to the last of the dense layers</span>
+<span class="sd">        If &#39;mlp_hidden_dims&#39; is not `None` and this parameter is `None`, it</span>
+<span class="sd">        will default to False.</span>
+<span class="sd">    mlp_linear_first: bool, Optional, default = None</span>
+<span class="sd">        Boolean indicating the order of the operations in the dense</span>
+<span class="sd">        layer. If `True: [LIN -&gt; ACT -&gt; BN -&gt; DP]`. If `False: [BN -&gt; DP -&gt;</span>
+<span class="sd">        LIN -&gt; ACT]`</span>
+<span class="sd">        If &#39;mlp_hidden_dims&#39; is not `None` and this parameter is `None`, it</span>
+<span class="sd">        will default to `True`.</span>
+
+<span class="sd">    Attributes</span>
+<span class="sd">    ----------</span>
+<span class="sd">    encoder: nn.Module</span>
+<span class="sd">        deep dense Resnet model that will receive the concatenation of the</span>
+<span class="sd">        embeddings and the continuous columns</span>
+<span class="sd">    mlp: nn.Module</span>
+<span class="sd">        if `mlp_hidden_dims` is `True`, this attribute will be an mlp</span>
+<span class="sd">        model that will receive the results of the concatenation of the</span>
+<span class="sd">        embeddings and the continuous columns -- if present --.</span>
+
+<span class="sd">    Examples</span>
+<span class="sd">    --------</span>
+<span class="sd">    &gt;&gt;&gt; import torch</span>
+<span class="sd">    &gt;&gt;&gt; from pytorch_widedeep.models import TabResnet</span>
+<span class="sd">    &gt;&gt;&gt; X_deep = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)</span>
+<span class="sd">    &gt;&gt;&gt; colnames = [&#39;a&#39;, &#39;b&#39;, &#39;c&#39;, &#39;d&#39;, &#39;e&#39;]</span>
+<span class="sd">    &gt;&gt;&gt; cat_embed_input = [(u,i,j) for u,i,j in zip(colnames[:4], [4]*4, [8]*4)]</span>
+<span class="sd">    &gt;&gt;&gt; column_idx = {k:v for v,k in enumerate(colnames)}</span>
+<span class="sd">    &gt;&gt;&gt; model = TabResnet(blocks_dims=[16,4], column_idx=column_idx, cat_embed_input=cat_embed_input,</span>
+<span class="sd">    ... continuous_cols = [&#39;e&#39;])</span>
+<span class="sd">    &gt;&gt;&gt; out = model(X_deep)</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+
+    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span>
+        <span class="n">column_idx</span><span class="p">:</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">],</span>
+        <span class="o">*</span><span class="p">,</span>
+        <span class="n">cat_embed_input</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="n">Tuple</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">,</span> <span class="nb">int</span><span class="p">]]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">cat_embed_dropout</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">use_cat_bias</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">cat_embed_activation</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">continuous_cols</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">cont_norm_layer</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Literal</span><span class="p">[</span><span class="s2">&quot;batchnorm&quot;</span><span class="p">,</span> <span class="s2">&quot;layernorm&quot;</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">embed_continuous</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">embed_continuous_method</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span>
+            <span class="n">Literal</span><span class="p">[</span><span class="s2">&quot;standard&quot;</span><span class="p">,</span> <span class="s2">&quot;piecewise&quot;</span><span class="p">,</span> <span class="s2">&quot;periodic&quot;</span><span class="p">]</span>
+        <span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">cont_embed_dim</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">cont_embed_dropout</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">cont_embed_activation</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">quantization_setup</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="nb">float</span><span class="p">]]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">n_frequencies</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">sigma</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">share_last_layer</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">full_embed_dropout</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">blocks_dims</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="p">[</span><span class="mi">200</span><span class="p">,</span> <span class="mi">100</span><span class="p">,</span> <span class="mi">100</span><span class="p">],</span>
+        <span class="n">blocks_dropout</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.1</span><span class="p">,</span>
+        <span class="n">simplify_blocks</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
+        <span class="n">mlp_hidden_dims</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">int</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">mlp_activation</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">mlp_dropout</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">mlp_batchnorm</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">mlp_batchnorm_last</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">mlp_linear_first</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+    <span class="p">):</span>
+        <span class="nb">super</span><span class="p">(</span><span class="n">TabResnet</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span>
+            <span class="n">column_idx</span><span class="o">=</span><span class="n">column_idx</span><span class="p">,</span>
+            <span class="n">cat_embed_input</span><span class="o">=</span><span class="n">cat_embed_input</span><span class="p">,</span>
+            <span class="n">cat_embed_dropout</span><span class="o">=</span><span class="n">cat_embed_dropout</span><span class="p">,</span>
+            <span class="n">use_cat_bias</span><span class="o">=</span><span class="n">use_cat_bias</span><span class="p">,</span>
+            <span class="n">cat_embed_activation</span><span class="o">=</span><span class="n">cat_embed_activation</span><span class="p">,</span>
+            <span class="n">continuous_cols</span><span class="o">=</span><span class="n">continuous_cols</span><span class="p">,</span>
+            <span class="n">cont_norm_layer</span><span class="o">=</span><span class="n">cont_norm_layer</span><span class="p">,</span>
+            <span class="n">embed_continuous</span><span class="o">=</span><span class="n">embed_continuous</span><span class="p">,</span>
+            <span class="n">embed_continuous_method</span><span class="o">=</span><span class="n">embed_continuous_method</span><span class="p">,</span>
+            <span class="n">cont_embed_dim</span><span class="o">=</span><span class="n">cont_embed_dim</span><span class="p">,</span>
+            <span class="n">cont_embed_dropout</span><span class="o">=</span><span class="n">cont_embed_dropout</span><span class="p">,</span>
+            <span class="n">cont_embed_activation</span><span class="o">=</span><span class="n">cont_embed_activation</span><span class="p">,</span>
+            <span class="n">quantization_setup</span><span class="o">=</span><span class="n">quantization_setup</span><span class="p">,</span>
+            <span class="n">n_frequencies</span><span class="o">=</span><span class="n">n_frequencies</span><span class="p">,</span>
+            <span class="n">sigma</span><span class="o">=</span><span class="n">sigma</span><span class="p">,</span>
+            <span class="n">share_last_layer</span><span class="o">=</span><span class="n">share_last_layer</span><span class="p">,</span>
+            <span class="n">full_embed_dropout</span><span class="o">=</span><span class="n">full_embed_dropout</span><span class="p">,</span>
         <span class="p">)</span>
 
-    <span class="bp">self</span><span class="o">.</span><span class="n">blocks_dims</span> <span class="o">=</span> <span class="n">blocks_dims</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">blocks_dropout</span> <span class="o">=</span> <span class="n">blocks_dropout</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">simplify_blocks</span> <span class="o">=</span> <span class="n">simplify_blocks</span>
+        <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">blocks_dims</span><span class="p">)</span> <span class="o">&lt;</span> <span class="mi">2</span><span class="p">:</span>
+            <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
+                <span class="s2">&quot;&#39;blocks&#39; must contain at least two elements, e.g. [256, 128]&quot;</span>
+            <span class="p">)</span>
 
-    <span class="bp">self</span><span class="o">.</span><span class="n">mlp_hidden_dims</span> <span class="o">=</span> <span class="n">mlp_hidden_dims</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">mlp_activation</span> <span class="o">=</span> <span class="n">mlp_activation</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">mlp_dropout</span> <span class="o">=</span> <span class="n">mlp_dropout</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">mlp_batchnorm</span> <span class="o">=</span> <span class="n">mlp_batchnorm</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">mlp_batchnorm_last</span> <span class="o">=</span> <span class="n">mlp_batchnorm_last</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">mlp_linear_first</span> <span class="o">=</span> <span class="n">mlp_linear_first</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">blocks_dims</span> <span class="o">=</span> <span class="n">blocks_dims</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">blocks_dropout</span> <span class="o">=</span> <span class="n">blocks_dropout</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">simplify_blocks</span> <span class="o">=</span> <span class="n">simplify_blocks</span>
 
-    <span class="c1"># Embeddings are instantiated at the base model</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">mlp_hidden_dims</span> <span class="o">=</span> <span class="n">mlp_hidden_dims</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">mlp_activation</span> <span class="o">=</span> <span class="n">mlp_activation</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">mlp_dropout</span> <span class="o">=</span> <span class="n">mlp_dropout</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">mlp_batchnorm</span> <span class="o">=</span> <span class="n">mlp_batchnorm</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">mlp_batchnorm_last</span> <span class="o">=</span> <span class="n">mlp_batchnorm_last</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">mlp_linear_first</span> <span class="o">=</span> <span class="n">mlp_linear_first</span>
 
-    <span class="c1"># Resnet</span>
-    <span class="n">dense_resnet_input_dim</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">cat_out_dim</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">cont_out_dim</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">encoder</span> <span class="o">=</span> <span class="n">DenseResnet</span><span class="p">(</span>
-        <span class="n">dense_resnet_input_dim</span><span class="p">,</span> <span class="n">blocks_dims</span><span class="p">,</span> <span class="n">blocks_dropout</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">simplify_blocks</span>
-    <span class="p">)</span>
+        <span class="c1"># Embeddings are instantiated at the base model</span>
 
-    <span class="c1"># Mlp: adding an MLP on top of the Resnet blocks is optional and</span>
-    <span class="c1"># therefore all related params are optional</span>
-    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_hidden_dims</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">mlp</span> <span class="o">=</span> <span class="n">MLP</span><span class="p">(</span>
-            <span class="n">d_hidden</span><span class="o">=</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">blocks_dims</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">]]</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_hidden_dims</span><span class="p">,</span>
-            <span class="n">activation</span><span class="o">=</span><span class="p">(</span>
-                <span class="s2">&quot;relu&quot;</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_activation</span> <span class="ow">is</span> <span class="kc">None</span> <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_activation</span>
-            <span class="p">),</span>
-            <span class="n">dropout</span><span class="o">=</span><span class="mf">0.0</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_dropout</span> <span class="ow">is</span> <span class="kc">None</span> <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_dropout</span><span class="p">,</span>
-            <span class="n">batchnorm</span><span class="o">=</span><span class="kc">False</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_batchnorm</span> <span class="ow">is</span> <span class="kc">None</span> <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_batchnorm</span><span class="p">,</span>
-            <span class="n">batchnorm_last</span><span class="o">=</span><span class="p">(</span>
-                <span class="kc">False</span>
-                <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_batchnorm_last</span> <span class="ow">is</span> <span class="kc">None</span>
-                <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_batchnorm_last</span>
-            <span class="p">),</span>
-            <span class="n">linear_first</span><span class="o">=</span><span class="p">(</span>
-                <span class="kc">True</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_linear_first</span> <span class="ow">is</span> <span class="kc">None</span> <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_linear_first</span>
-            <span class="p">),</span>
+        <span class="c1"># Resnet</span>
+        <span class="n">dense_resnet_input_dim</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">cat_out_dim</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">cont_out_dim</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">encoder</span> <span class="o">=</span> <span class="n">DenseResnet</span><span class="p">(</span>
+            <span class="n">dense_resnet_input_dim</span><span class="p">,</span> <span class="n">blocks_dims</span><span class="p">,</span> <span class="n">blocks_dropout</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">simplify_blocks</span>
+        <span class="p">)</span>
+
+        <span class="c1"># Mlp: adding an MLP on top of the Resnet blocks is optional and</span>
+        <span class="c1"># therefore all related params are optional</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_hidden_dims</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">mlp</span> <span class="o">=</span> <span class="n">MLP</span><span class="p">(</span>
+                <span class="n">d_hidden</span><span class="o">=</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">blocks_dims</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">]]</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_hidden_dims</span><span class="p">,</span>
+                <span class="n">activation</span><span class="o">=</span><span class="p">(</span>
+                    <span class="s2">&quot;relu&quot;</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_activation</span> <span class="ow">is</span> <span class="kc">None</span> <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_activation</span>
+                <span class="p">),</span>
+                <span class="n">dropout</span><span class="o">=</span><span class="mf">0.0</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_dropout</span> <span class="ow">is</span> <span class="kc">None</span> <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_dropout</span><span class="p">,</span>
+                <span class="n">batchnorm</span><span class="o">=</span><span class="kc">False</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_batchnorm</span> <span class="ow">is</span> <span class="kc">None</span> <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_batchnorm</span><span class="p">,</span>
+                <span class="n">batchnorm_last</span><span class="o">=</span><span class="p">(</span>
+                    <span class="kc">False</span>
+                    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_batchnorm_last</span> <span class="ow">is</span> <span class="kc">None</span>
+                    <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_batchnorm_last</span>
+                <span class="p">),</span>
+                <span class="n">linear_first</span><span class="o">=</span><span class="p">(</span>
+                    <span class="kc">True</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_linear_first</span> <span class="ow">is</span> <span class="kc">None</span> <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_linear_first</span>
+                <span class="p">),</span>
+            <span class="p">)</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">mlp</span> <span class="o">=</span> <span class="kc">None</span>
+
+    <span class="k">def</span> <span class="nf">forward</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">X</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tensor</span><span class="p">:</span>
+        <span class="n">x</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_embeddings</span><span class="p">(</span><span class="n">X</span><span class="p">)</span>
+        <span class="n">x</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">encoder</span><span class="p">(</span><span class="n">x</span><span class="p">)</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="n">x</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp</span><span class="p">(</span><span class="n">x</span><span class="p">)</span>
+        <span class="k">return</span> <span class="n">x</span>
+
+    <span class="nd">@property</span>
+    <span class="k">def</span> <span class="nf">output_dim</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">int</span><span class="p">:</span>
+<span class="w">        </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;The output dimension of the model. This is a required property</span>
+<span class="sd">        neccesary to build the `WideDeep` class</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="k">return</span> <span class="p">(</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">mlp_hidden_dims</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span>
+            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_hidden_dims</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span>
+            <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">blocks_dims</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span>
         <span class="p">)</span>
-    <span class="k">else</span><span class="p">:</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">mlp</span> <span class="o">=</span> <span class="kc">None</span>
 </code></pre></div></td></tr></table></div>
-                  </details>
+              </details>
 
 
 
@@ -3365,19 +4449,7 @@ <h2 id="pytorch_widedeep.models.tabular.resnet.tab_resnet.TabResnetDecoder" clas
 
 
 <a href="#pytorch_widedeep.models.tabular.resnet.tab_resnet.TabResnetDecoder" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="nf">TabResnetDecoder</span><span class="p">(</span>
-    <span class="n">embed_dim</span><span class="p">,</span>
-    <span class="n">blocks_dims</span><span class="o">=</span><span class="p">[</span><span class="mi">100</span><span class="p">,</span> <span class="mi">100</span><span class="p">,</span> <span class="mi">200</span><span class="p">],</span>
-    <span class="n">blocks_dropout</span><span class="o">=</span><span class="mf">0.1</span><span class="p">,</span>
-    <span class="n">simplify_blocks</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
-    <span class="n">mlp_hidden_dims</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">mlp_activation</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">mlp_dropout</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">mlp_batchnorm</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">mlp_batchnorm_last</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">mlp_linear_first</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-<span class="p">)</span>
-</code></pre></div>
+
 
     <div class="doc doc-contents first">
             <p class="doc doc-class-bases">
@@ -3393,154 +4465,221 @@ <h2 id="pytorch_widedeep.models.tabular.resnet.tab_resnet.TabResnetDecoder" clas
 
 
 <p><span class="doc-section-title">Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>embed_dim</code></b>
-              (<code>int</code>)
-          –
-          <div class="doc-md-description">
-            <p>Size of the embeddings tensor to be reconstructed.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>blocks_dims</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.List">List</span>[int]</code>, default:
-                  <code>[100, 100, 200]</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>List of integers that define the input and output units of each block.
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+          <th>Default</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>embed_dim</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Size of the embeddings tensor to be reconstructed.</p>
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>blocks_dims</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.List">List</span>[int]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>List of integers that define the input and output units of each block.
 For example: <em>[200, 100, 100]</em> will generate 2 blocks. The first will
 receive a tensor of size 200 and output a tensor of size 100, and the
 second will receive a tensor of size 100 and output a tensor of size
 100. See <code>pytorch_widedeep.models.tab_resnet._layers</code> for
 details on the structure of each block.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>blocks_dropout</code></b>
-              (<code>float</code>, default:
+              </div>
+            </td>
+            <td>
+                  <code>[100, 100, 200]</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>blocks_dropout</code></td>
+            <td>
+                  <code>float</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Block's internal dropout.</p>
+              </div>
+            </td>
+            <td>
                   <code>0.1</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Block's internal dropout.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>simplify_blocks</code></b>
-              (<code>bool</code>, default:
-                  <code>False</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Boolean indicating if the simplest possible residual blocks (<code>X -&gt; [
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>simplify_blocks</code></td>
+            <td>
+                  <code>bool</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Boolean indicating if the simplest possible residual blocks (<code>X -&gt; [
 [LIN, BN, ACT]  + X ]</code>) will be used instead of a standard one
 (<code>X -&gt; [ [LIN1, BN1, ACT1] -&gt; [LIN2, BN2]  + X ]</code>).</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>mlp_hidden_dims</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[int]]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>List with the number of neurons per dense layer in the MLP. e.g:
+              </div>
+            </td>
+            <td>
+                  <code>False</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>mlp_hidden_dims</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[int]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>List with the number of neurons per dense layer in the MLP. e.g:
 <em>[64, 32]</em>. If <code>None</code> the  output of the Resnet Blocks will be
 connected directly to the output neuron(s).</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>mlp_activation</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Activation function for the dense layers of the MLP. Currently
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>mlp_activation</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Activation function for the dense layers of the MLP. Currently
 <em>'tanh'</em>, <em>'relu'</em>, <em>'leaky'_relu' and _'gelu'</em> are supported.
 If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it
 will default to <em>'relu'</em>.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>mlp_dropout</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>float with the dropout between the dense layers of the MLP.
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>mlp_dropout</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>float with the dropout between the dense layers of the MLP.
 If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it
 will default to 0.0.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>mlp_batchnorm</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Boolean indicating whether or not batch normalization will be applied
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>mlp_batchnorm</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Boolean indicating whether or not batch normalization will be applied
 to the dense layers
 If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it
 will default to False.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>mlp_batchnorm_last</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Boolean indicating whether or not batch normalization will be applied
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>mlp_batchnorm_last</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Boolean indicating whether or not batch normalization will be applied
 to the last of the dense layers
 If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it
 will default to False.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>mlp_linear_first</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Boolean indicating the order of the operations in the dense
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>mlp_linear_first</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Boolean indicating the order of the operations in the dense
 layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt;
 LIN -&gt; ACT]</code>
 If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it
 will default to <code>True</code>.</p>
-          </div>
-        </li>
-    </ul>
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
 
 <p><span class="doc-section-title">Attributes:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code><span title="pytorch_widedeep.models.tabular.resnet.tab_resnet.TabResnetDecoder.decoder">decoder</span></code></b>
-              (<code><span title="torch.nn.Module">Module</span></code>)
-          –
-          <div class="doc-md-description">
-            <p>deep dense Resnet model that will receive the output of the encoder IF
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code><span title="pytorch_widedeep.models.tabular.resnet.tab_resnet.TabResnetDecoder.decoder">decoder</span></code></td>
+            <td>
+                  <code><span title="torch.nn.Module">Module</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>deep dense Resnet model that will receive the output of the encoder IF
 <code>mlp_hidden_dims</code> is None</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code><span title="pytorch_widedeep.models.tabular.resnet.tab_resnet.TabResnetDecoder.mlp">mlp</span></code></b>
-              (<code><span title="torch.nn.Module">Module</span></code>)
-          –
-          <div class="doc-md-description">
-            <p>if <code>mlp_hidden_dims</code> is not None, the overall decoder will consist
+              </div>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code><span title="pytorch_widedeep.models.tabular.resnet.tab_resnet.TabResnetDecoder.mlp">mlp</span></code></td>
+            <td>
+                  <code><span title="torch.nn.Module">Module</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>if <code>mlp_hidden_dims</code> is not None, the overall decoder will consist
 in an MLP that will receive the output of the encoder followed by the
 deep dense Resnet.</p>
-          </div>
-        </li>
-    </ul>
+              </div>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
 
 <p><span class="doc-section-title">Examples:</span></p>
@@ -3553,9 +4692,86 @@ <h2 id="pytorch_widedeep.models.tabular.resnet.tab_resnet.TabResnetDecoder" clas
 <span class="go">torch.Size([3, 32])</span>
 </code></pre></div>
 
-                  <details class="quote">
-                    <summary>Source code in <code>pytorch_widedeep/models/tabular/resnet/tab_resnet.py</code></summary>
-                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">361</span>
+              <details class="quote">
+                <summary>Source code in <code>pytorch_widedeep/models/tabular/resnet/tab_resnet.py</code></summary>
+                <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">284</span>
+<span class="normal">285</span>
+<span class="normal">286</span>
+<span class="normal">287</span>
+<span class="normal">288</span>
+<span class="normal">289</span>
+<span class="normal">290</span>
+<span class="normal">291</span>
+<span class="normal">292</span>
+<span class="normal">293</span>
+<span class="normal">294</span>
+<span class="normal">295</span>
+<span class="normal">296</span>
+<span class="normal">297</span>
+<span class="normal">298</span>
+<span class="normal">299</span>
+<span class="normal">300</span>
+<span class="normal">301</span>
+<span class="normal">302</span>
+<span class="normal">303</span>
+<span class="normal">304</span>
+<span class="normal">305</span>
+<span class="normal">306</span>
+<span class="normal">307</span>
+<span class="normal">308</span>
+<span class="normal">309</span>
+<span class="normal">310</span>
+<span class="normal">311</span>
+<span class="normal">312</span>
+<span class="normal">313</span>
+<span class="normal">314</span>
+<span class="normal">315</span>
+<span class="normal">316</span>
+<span class="normal">317</span>
+<span class="normal">318</span>
+<span class="normal">319</span>
+<span class="normal">320</span>
+<span class="normal">321</span>
+<span class="normal">322</span>
+<span class="normal">323</span>
+<span class="normal">324</span>
+<span class="normal">325</span>
+<span class="normal">326</span>
+<span class="normal">327</span>
+<span class="normal">328</span>
+<span class="normal">329</span>
+<span class="normal">330</span>
+<span class="normal">331</span>
+<span class="normal">332</span>
+<span class="normal">333</span>
+<span class="normal">334</span>
+<span class="normal">335</span>
+<span class="normal">336</span>
+<span class="normal">337</span>
+<span class="normal">338</span>
+<span class="normal">339</span>
+<span class="normal">340</span>
+<span class="normal">341</span>
+<span class="normal">342</span>
+<span class="normal">343</span>
+<span class="normal">344</span>
+<span class="normal">345</span>
+<span class="normal">346</span>
+<span class="normal">347</span>
+<span class="normal">348</span>
+<span class="normal">349</span>
+<span class="normal">350</span>
+<span class="normal">351</span>
+<span class="normal">352</span>
+<span class="normal">353</span>
+<span class="normal">354</span>
+<span class="normal">355</span>
+<span class="normal">356</span>
+<span class="normal">357</span>
+<span class="normal">358</span>
+<span class="normal">359</span>
+<span class="normal">360</span>
+<span class="normal">361</span>
 <span class="normal">362</span>
 <span class="normal">363</span>
 <span class="normal">364</span>
@@ -3617,71 +4833,156 @@ <h2 id="pytorch_widedeep.models.tabular.resnet.tab_resnet.TabResnetDecoder" clas
 <span class="normal">420</span>
 <span class="normal">421</span>
 <span class="normal">422</span>
-<span class="normal">423</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
-    <span class="bp">self</span><span class="p">,</span>
-    <span class="n">embed_dim</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span>
-    <span class="n">blocks_dims</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="p">[</span><span class="mi">100</span><span class="p">,</span> <span class="mi">100</span><span class="p">,</span> <span class="mi">200</span><span class="p">],</span>
-    <span class="n">blocks_dropout</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.1</span><span class="p">,</span>
-    <span class="n">simplify_blocks</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
-    <span class="n">mlp_hidden_dims</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">int</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">mlp_activation</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">mlp_dropout</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">mlp_batchnorm</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">mlp_batchnorm_last</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">mlp_linear_first</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-<span class="p">):</span>
-    <span class="nb">super</span><span class="p">(</span><span class="n">TabResnetDecoder</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
-
-    <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">blocks_dims</span><span class="p">)</span> <span class="o">&lt;</span> <span class="mi">2</span><span class="p">:</span>
-        <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
-            <span class="s2">&quot;&#39;blocks&#39; must contain at least two elements, e.g. [256, 128]&quot;</span>
-        <span class="p">)</span>
+<span class="normal">423</span>
+<span class="normal">424</span>
+<span class="normal">425</span>
+<span class="normal">426</span>
+<span class="normal">427</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">class</span> <span class="nc">TabResnetDecoder</span><span class="p">(</span><span class="n">nn</span><span class="o">.</span><span class="n">Module</span><span class="p">):</span>
+<span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Companion decoder model for the `TabResnet` model (which can be</span>
+<span class="sd">    considered an encoder itself)</span>
+
+<span class="sd">    This class is designed to be used with the `EncoderDecoderTrainer` when</span>
+<span class="sd">    using self-supervised pre-training (see the corresponding section in the</span>
+<span class="sd">    docs). This class will receive the output from the ResNet blocks or the</span>
+<span class="sd">    MLP(if present) and &#39;_reconstruct_&#39; the embeddings.</span>
+
+<span class="sd">    Parameters</span>
+<span class="sd">    ----------</span>
+<span class="sd">    embed_dim: int</span>
+<span class="sd">        Size of the embeddings tensor to be reconstructed.</span>
+<span class="sd">    blocks_dims: List, default = [200, 100, 100]</span>
+<span class="sd">        List of integers that define the input and output units of each block.</span>
+<span class="sd">        For example: _[200, 100, 100]_ will generate 2 blocks. The first will</span>
+<span class="sd">        receive a tensor of size 200 and output a tensor of size 100, and the</span>
+<span class="sd">        second will receive a tensor of size 100 and output a tensor of size</span>
+<span class="sd">        100. See `pytorch_widedeep.models.tab_resnet._layers` for</span>
+<span class="sd">        details on the structure of each block.</span>
+<span class="sd">    blocks_dropout: float, default =  0.1</span>
+<span class="sd">        Block&#39;s internal dropout.</span>
+<span class="sd">    simplify_blocks: bool, default = False,</span>
+<span class="sd">        Boolean indicating if the simplest possible residual blocks (`X -&gt; [</span>
+<span class="sd">        [LIN, BN, ACT]  + X ]`) will be used instead of a standard one</span>
+<span class="sd">        (`X -&gt; [ [LIN1, BN1, ACT1] -&gt; [LIN2, BN2]  + X ]`).</span>
+<span class="sd">    mlp_hidden_dims: List, Optional, default = None</span>
+<span class="sd">        List with the number of neurons per dense layer in the MLP. e.g:</span>
+<span class="sd">        _[64, 32]_. If `None` the  output of the Resnet Blocks will be</span>
+<span class="sd">        connected directly to the output neuron(s).</span>
+<span class="sd">    mlp_activation: str, Optional, default = None</span>
+<span class="sd">        Activation function for the dense layers of the MLP. Currently</span>
+<span class="sd">        _&#39;tanh&#39;_, _&#39;relu&#39;_, _&#39;leaky&#39;_relu&#39; and _&#39;gelu&#39;_ are supported.</span>
+<span class="sd">        If &#39;mlp_hidden_dims&#39; is not `None` and this parameter is `None`, it</span>
+<span class="sd">        will default to _&#39;relu&#39;_.</span>
+<span class="sd">    mlp_dropout: float, Optional, default = None</span>
+<span class="sd">        float with the dropout between the dense layers of the MLP.</span>
+<span class="sd">        If &#39;mlp_hidden_dims&#39; is not `None` and this parameter is `None`, it</span>
+<span class="sd">        will default to 0.0.</span>
+<span class="sd">    mlp_batchnorm: bool, Optional, default = None</span>
+<span class="sd">        Boolean indicating whether or not batch normalization will be applied</span>
+<span class="sd">        to the dense layers</span>
+<span class="sd">        If &#39;mlp_hidden_dims&#39; is not `None` and this parameter is `None`, it</span>
+<span class="sd">        will default to False.</span>
+<span class="sd">    mlp_batchnorm_last: bool, Optional, default = None</span>
+<span class="sd">        Boolean indicating whether or not batch normalization will be applied</span>
+<span class="sd">        to the last of the dense layers</span>
+<span class="sd">        If &#39;mlp_hidden_dims&#39; is not `None` and this parameter is `None`, it</span>
+<span class="sd">        will default to False.</span>
+<span class="sd">    mlp_linear_first: bool, Optional, default = None</span>
+<span class="sd">        Boolean indicating the order of the operations in the dense</span>
+<span class="sd">        layer. If `True: [LIN -&gt; ACT -&gt; BN -&gt; DP]`. If `False: [BN -&gt; DP -&gt;</span>
+<span class="sd">        LIN -&gt; ACT]`</span>
+<span class="sd">        If &#39;mlp_hidden_dims&#39; is not `None` and this parameter is `None`, it</span>
+<span class="sd">        will default to `True`.</span>
+
+<span class="sd">    Attributes</span>
+<span class="sd">    ----------</span>
+<span class="sd">    decoder: nn.Module</span>
+<span class="sd">        deep dense Resnet model that will receive the output of the encoder IF</span>
+<span class="sd">        `mlp_hidden_dims` is None</span>
+<span class="sd">    mlp: nn.Module</span>
+<span class="sd">        if `mlp_hidden_dims` is not None, the overall decoder will consist</span>
+<span class="sd">        in an MLP that will receive the output of the encoder followed by the</span>
+<span class="sd">        deep dense Resnet.</span>
+
+<span class="sd">    Examples</span>
+<span class="sd">    --------</span>
+<span class="sd">    &gt;&gt;&gt; import torch</span>
+<span class="sd">    &gt;&gt;&gt; from pytorch_widedeep.models import TabResnetDecoder</span>
+<span class="sd">    &gt;&gt;&gt; x_inp = torch.rand(3, 8)</span>
+<span class="sd">    &gt;&gt;&gt; decoder = TabResnetDecoder(embed_dim=32, blocks_dims=[8, 16, 16])</span>
+<span class="sd">    &gt;&gt;&gt; res = decoder(x_inp)</span>
+<span class="sd">    &gt;&gt;&gt; res.shape</span>
+<span class="sd">    torch.Size([3, 32])</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+
+    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span>
+        <span class="n">embed_dim</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span>
+        <span class="n">blocks_dims</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="p">[</span><span class="mi">100</span><span class="p">,</span> <span class="mi">100</span><span class="p">,</span> <span class="mi">200</span><span class="p">],</span>
+        <span class="n">blocks_dropout</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.1</span><span class="p">,</span>
+        <span class="n">simplify_blocks</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
+        <span class="n">mlp_hidden_dims</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">int</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">mlp_activation</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">mlp_dropout</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">mlp_batchnorm</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">mlp_batchnorm_last</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">mlp_linear_first</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+    <span class="p">):</span>
+        <span class="nb">super</span><span class="p">(</span><span class="n">TabResnetDecoder</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
+
+        <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">blocks_dims</span><span class="p">)</span> <span class="o">&lt;</span> <span class="mi">2</span><span class="p">:</span>
+            <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
+                <span class="s2">&quot;&#39;blocks&#39; must contain at least two elements, e.g. [256, 128]&quot;</span>
+            <span class="p">)</span>
 
-    <span class="bp">self</span><span class="o">.</span><span class="n">embed_dim</span> <span class="o">=</span> <span class="n">embed_dim</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">embed_dim</span> <span class="o">=</span> <span class="n">embed_dim</span>
 
-    <span class="bp">self</span><span class="o">.</span><span class="n">blocks_dims</span> <span class="o">=</span> <span class="n">blocks_dims</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">blocks_dropout</span> <span class="o">=</span> <span class="n">blocks_dropout</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">simplify_blocks</span> <span class="o">=</span> <span class="n">simplify_blocks</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">blocks_dims</span> <span class="o">=</span> <span class="n">blocks_dims</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">blocks_dropout</span> <span class="o">=</span> <span class="n">blocks_dropout</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">simplify_blocks</span> <span class="o">=</span> <span class="n">simplify_blocks</span>
 
-    <span class="bp">self</span><span class="o">.</span><span class="n">mlp_hidden_dims</span> <span class="o">=</span> <span class="n">mlp_hidden_dims</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">mlp_activation</span> <span class="o">=</span> <span class="n">mlp_activation</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">mlp_dropout</span> <span class="o">=</span> <span class="n">mlp_dropout</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">mlp_batchnorm</span> <span class="o">=</span> <span class="n">mlp_batchnorm</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">mlp_batchnorm_last</span> <span class="o">=</span> <span class="n">mlp_batchnorm_last</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">mlp_linear_first</span> <span class="o">=</span> <span class="n">mlp_linear_first</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">mlp_hidden_dims</span> <span class="o">=</span> <span class="n">mlp_hidden_dims</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">mlp_activation</span> <span class="o">=</span> <span class="n">mlp_activation</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">mlp_dropout</span> <span class="o">=</span> <span class="n">mlp_dropout</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">mlp_batchnorm</span> <span class="o">=</span> <span class="n">mlp_batchnorm</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">mlp_batchnorm_last</span> <span class="o">=</span> <span class="n">mlp_batchnorm_last</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">mlp_linear_first</span> <span class="o">=</span> <span class="n">mlp_linear_first</span>
 
-    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_hidden_dims</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">mlp</span> <span class="o">=</span> <span class="n">MLP</span><span class="p">(</span>
-            <span class="n">d_hidden</span><span class="o">=</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">mlp_first_hidden_dim</span><span class="p">]</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_hidden_dims</span><span class="p">,</span>
-            <span class="n">activation</span><span class="o">=</span><span class="p">(</span>
-                <span class="s2">&quot;relu&quot;</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_activation</span> <span class="ow">is</span> <span class="kc">None</span> <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_activation</span>
-            <span class="p">),</span>
-            <span class="n">dropout</span><span class="o">=</span><span class="mf">0.0</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_dropout</span> <span class="ow">is</span> <span class="kc">None</span> <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_dropout</span><span class="p">,</span>
-            <span class="n">batchnorm</span><span class="o">=</span><span class="kc">False</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_batchnorm</span> <span class="ow">is</span> <span class="kc">None</span> <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_batchnorm</span><span class="p">,</span>
-            <span class="n">batchnorm_last</span><span class="o">=</span><span class="p">(</span>
-                <span class="kc">False</span>
-                <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_batchnorm_last</span> <span class="ow">is</span> <span class="kc">None</span>
-                <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_batchnorm_last</span>
-            <span class="p">),</span>
-            <span class="n">linear_first</span><span class="o">=</span><span class="p">(</span>
-                <span class="kc">True</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_linear_first</span> <span class="ow">is</span> <span class="kc">None</span> <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_linear_first</span>
-            <span class="p">),</span>
-        <span class="p">)</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">decoder</span> <span class="o">=</span> <span class="n">DenseResnet</span><span class="p">(</span>
-            <span class="bp">self</span><span class="o">.</span><span class="n">mlp_hidden_dims</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">],</span>
-            <span class="n">blocks_dims</span><span class="p">,</span>
-            <span class="n">blocks_dropout</span><span class="p">,</span>
-            <span class="bp">self</span><span class="o">.</span><span class="n">simplify_blocks</span><span class="p">,</span>
-        <span class="p">)</span>
-    <span class="k">else</span><span class="p">:</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">mlp</span> <span class="o">=</span> <span class="kc">None</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">decoder</span> <span class="o">=</span> <span class="n">DenseResnet</span><span class="p">(</span>
-            <span class="n">blocks_dims</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="n">blocks_dims</span><span class="p">,</span> <span class="n">blocks_dropout</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">simplify_blocks</span>
-        <span class="p">)</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_hidden_dims</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">mlp</span> <span class="o">=</span> <span class="n">MLP</span><span class="p">(</span>
+                <span class="n">d_hidden</span><span class="o">=</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">mlp_first_hidden_dim</span><span class="p">]</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_hidden_dims</span><span class="p">,</span>
+                <span class="n">activation</span><span class="o">=</span><span class="p">(</span>
+                    <span class="s2">&quot;relu&quot;</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_activation</span> <span class="ow">is</span> <span class="kc">None</span> <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_activation</span>
+                <span class="p">),</span>
+                <span class="n">dropout</span><span class="o">=</span><span class="mf">0.0</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_dropout</span> <span class="ow">is</span> <span class="kc">None</span> <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_dropout</span><span class="p">,</span>
+                <span class="n">batchnorm</span><span class="o">=</span><span class="kc">False</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_batchnorm</span> <span class="ow">is</span> <span class="kc">None</span> <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_batchnorm</span><span class="p">,</span>
+                <span class="n">batchnorm_last</span><span class="o">=</span><span class="p">(</span>
+                    <span class="kc">False</span>
+                    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_batchnorm_last</span> <span class="ow">is</span> <span class="kc">None</span>
+                    <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_batchnorm_last</span>
+                <span class="p">),</span>
+                <span class="n">linear_first</span><span class="o">=</span><span class="p">(</span>
+                    <span class="kc">True</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_linear_first</span> <span class="ow">is</span> <span class="kc">None</span> <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_linear_first</span>
+                <span class="p">),</span>
+            <span class="p">)</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">decoder</span> <span class="o">=</span> <span class="n">DenseResnet</span><span class="p">(</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">mlp_hidden_dims</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">],</span>
+                <span class="n">blocks_dims</span><span class="p">,</span>
+                <span class="n">blocks_dropout</span><span class="p">,</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">simplify_blocks</span><span class="p">,</span>
+            <span class="p">)</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">mlp</span> <span class="o">=</span> <span class="kc">None</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">decoder</span> <span class="o">=</span> <span class="n">DenseResnet</span><span class="p">(</span>
+                <span class="n">blocks_dims</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="n">blocks_dims</span><span class="p">,</span> <span class="n">blocks_dropout</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">simplify_blocks</span>
+            <span class="p">)</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">reconstruction_layer</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Linear</span><span class="p">(</span><span class="n">blocks_dims</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">],</span> <span class="n">embed_dim</span><span class="p">,</span> <span class="n">bias</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
 
-    <span class="bp">self</span><span class="o">.</span><span class="n">reconstruction_layer</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Linear</span><span class="p">(</span><span class="n">blocks_dims</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">],</span> <span class="n">embed_dim</span><span class="p">,</span> <span class="n">bias</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
+    <span class="k">def</span> <span class="nf">forward</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">X</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tensor</span><span class="p">:</span>
+        <span class="n">x</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp</span><span class="p">(</span><span class="n">X</span><span class="p">)</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="k">else</span> <span class="n">X</span>
+        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">reconstruction_layer</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">decoder</span><span class="p">(</span><span class="n">x</span><span class="p">))</span>
 </code></pre></div></td></tr></table></div>
-                  </details>
+              </details>
 
 
 
@@ -3712,39 +5013,7 @@ <h2 id="pytorch_widedeep.models.tabular.tabnet.tab_net.TabNet" class="doc doc-he
 
 
 <a href="#pytorch_widedeep.models.tabular.tabnet.tab_net.TabNet" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="nf">TabNet</span><span class="p">(</span>
-    <span class="n">column_idx</span><span class="p">,</span>
-    <span class="o">*</span><span class="p">,</span>
-    <span class="n">cat_embed_input</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">cat_embed_dropout</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">use_cat_bias</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">cat_embed_activation</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">continuous_cols</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">cont_norm_layer</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">embed_continuous</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">embed_continuous_method</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">cont_embed_dim</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">cont_embed_dropout</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">cont_embed_activation</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">quantization_setup</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">n_frequencies</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">sigma</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">share_last_layer</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">full_embed_dropout</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">n_steps</span><span class="o">=</span><span class="mi">3</span><span class="p">,</span>
-    <span class="n">step_dim</span><span class="o">=</span><span class="mi">8</span><span class="p">,</span>
-    <span class="n">attn_dim</span><span class="o">=</span><span class="mi">8</span><span class="p">,</span>
-    <span class="n">dropout</span><span class="o">=</span><span class="mf">0.0</span><span class="p">,</span>
-    <span class="n">n_glu_step_dependent</span><span class="o">=</span><span class="mi">2</span><span class="p">,</span>
-    <span class="n">n_glu_shared</span><span class="o">=</span><span class="mi">2</span><span class="p">,</span>
-    <span class="n">ghost_bn</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
-    <span class="n">virtual_batch_size</span><span class="o">=</span><span class="mi">128</span><span class="p">,</span>
-    <span class="n">momentum</span><span class="o">=</span><span class="mf">0.02</span><span class="p">,</span>
-    <span class="n">gamma</span><span class="o">=</span><span class="mf">1.3</span><span class="p">,</span>
-    <span class="n">epsilon</span><span class="o">=</span><span class="mf">1e-15</span><span class="p">,</span>
-    <span class="n">mask_type</span><span class="o">=</span><span class="s2">&quot;sparsemax&quot;</span>
-<span class="p">)</span>
-</code></pre></div>
+
 
     <div class="doc doc-contents first">
             <p class="doc doc-class-bases">
@@ -3761,106 +5030,149 @@ <h2 id="pytorch_widedeep.models.tabular.tabnet.tab_net.TabNet" class="doc doc-he
 
 
 <p><span class="doc-section-title">Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>column_idx</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, int]</code>)
-          –
-          <div class="doc-md-description">
-            <p>Dict containing the index of the columns that will be passed through
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+          <th>Default</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>column_idx</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, int]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Dict containing the index of the columns that will be passed through
 the <code>TabMlp</code> model. Required to slice the tensors. e.g. <em>{'education':
 0, 'relationship': 1, 'workclass': 2, ...}</em>.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>cat_embed_input</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.wdtypes.Tuple">Tuple</span>[str, int, int]]]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>List of Tuples with the column name, number of unique values and
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>cat_embed_input</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.wdtypes.Tuple">Tuple</span>[str, int, int]]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>List of Tuples with the column name, number of unique values and
 embedding dimension. e.g. <em>[(education, 11, 32), ...]</em></p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>cat_embed_dropout</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Categorical embeddings dropout. If <code>None</code>, it will default
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>cat_embed_dropout</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Categorical embeddings dropout. If <code>None</code>, it will default
 to 0.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>use_cat_bias</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Boolean indicating if bias will be used for the categorical embeddings.
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>use_cat_bias</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Boolean indicating if bias will be used for the categorical embeddings.
 If <code>None</code>, it will default to 'False'.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>cat_embed_activation</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Activation function for the categorical embeddings, if any. Currently
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>cat_embed_activation</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Activation function for the categorical embeddings, if any. Currently
 <em>'tanh'</em>, <em>'relu'</em>, <em>'leaky_relu'</em> and <em>'gelu'</em> are supported</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>continuous_cols</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[str]]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>List with the name of the numeric (aka continuous) columns</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>cont_norm_layer</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[batchnorm, layernorm]]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Type of normalization layer applied to the continuous features.
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>continuous_cols</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[str]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>List with the name of the numeric (aka continuous) columns</p>
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>cont_norm_layer</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[batchnorm, layernorm]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Type of normalization layer applied to the continuous features.
 Options are: <em>'layernorm'</em> and <em>'batchnorm'</em>. if <code>None</code>, no
 normalization layer will be used.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>embed_continuous</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Boolean indicating if the continuous columns will be embedded using
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>embed_continuous</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Boolean indicating if the continuous columns will be embedded using
 one of the available methods: <em>'standard'</em>, <em>'periodic'</em>
 or <em>'piecewise'</em>. If <code>None</code>, it will default to 'False'.<br/>
 <img alt="ℹ️" class="emojione" src="https://cdnjs.cloudflare.com/ajax/libs/emojione/2.2.7/assets/png/2139.png" title=":information_source:" /> <strong>NOTE</strong>: This parameter is deprecated and it
  will be removed in future releases. Please, use the
  <code>embed_continuous_method</code> parameter instead.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>embed_continuous_method</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[standard, piecewise, periodic]]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Method to use to embed the continuous features. Options are:
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>embed_continuous_method</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[standard, piecewise, periodic]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Method to use to embed the continuous features. Options are:
 <em>'standard'</em>, <em>'periodic'</em> or <em>'piecewise'</em>. The <em>'standard'</em>
 embedding method is based on the FT-Transformer implementation
 presented in the paper: <a href="https://arxiv.org/abs/2106.11959v5">Revisiting Deep Learning Models for
@@ -3868,252 +5180,349 @@ <h2 id="pytorch_widedeep.models.tabular.tabnet.tab_net.TabNet" class="doc doc-he
 and_'piecewise'_ methods were presented in the paper: <a href="https://arxiv.org/abs/2203.05556">On Embeddings for
 Numerical Features in Tabular Deep Learning</a>.
 Please, read the papers for details.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>cont_embed_dim</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[int]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Size of the continuous embeddings. If the continuous columns are
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>cont_embed_dim</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[int]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Size of the continuous embeddings. If the continuous columns are
 embedded, <code>cont_embed_dim</code> must be passed.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>cont_embed_dropout</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>cont_embed_activation</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Activation function for the continuous embeddings if any. Currently
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>cont_embed_dropout</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p>
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>cont_embed_activation</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Activation function for the continuous embeddings if any. Currently
 <em>'tanh'</em>, <em>'relu'</em>, <em>'leaky_relu'</em> and <em>'gelu'</em> are supported.
 If <code>None</code>, no activation function will be applied.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>quantization_setup</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, <span title="pytorch_widedeep.wdtypes.List">List</span>[float]]]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>This parameter is used when the <em>'piecewise'</em> method is used to embed
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>quantization_setup</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, <span title="pytorch_widedeep.wdtypes.List">List</span>[float]]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>This parameter is used when the <em>'piecewise'</em> method is used to embed
 the continuous cols. It is a dict where keys are the name of the continuous
 columns and values are lists with the boundaries for the quantization
 of the continuous_cols. See the examples for details. If
 If the <em>'piecewise'</em> method is used, this parameter is required.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>n_frequencies</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[int]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>This is the so called <em>'k'</em> in their paper <a href="https://arxiv.org/abs/2203.05556">On Embeddings for
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>n_frequencies</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[int]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>This is the so called <em>'k'</em> in their paper <a href="https://arxiv.org/abs/2203.05556">On Embeddings for
 Numerical Features in Tabular Deep Learning</a>,
 and is the number of 'frequencies' that will be used to represent each
 continuous column. See their Eq 2 in the paper for details. If
 the <em>'periodic'</em> method is used, this parameter is required.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>sigma</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>This is the sigma parameter in the paper mentioned when describing the
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>sigma</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>This is the sigma parameter in the paper mentioned when describing the
 previous parameters and it is used to initialise the 'frequency
 weights'. See their Eq 2 in the paper for details. If
 the <em>'periodic'</em> method is used, this parameter is required.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>share_last_layer</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>This parameter is not present in the before mentioned paper but it is implemented in
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>share_last_layer</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>This parameter is not present in the before mentioned paper but it is implemented in
 the <a href="https://github.com/yandex-research/rtdl-num-embeddings/tree/main">official repo</a>.
 If <code>True</code> the linear layer that turns the frequencies into embeddings
 will be shared across the continuous columns. If <code>False</code> a different
 linear layer will be used for each continuous column.
 If the <em>'periodic'</em> method is used, this parameter is required.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>full_embed_dropout</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>If <code>True</code>, the full embedding corresponding to a column will be masked
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>full_embed_dropout</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>If <code>True</code>, the full embedding corresponding to a column will be masked
 out/dropout. If <code>None</code>, it will default to <code>False</code>.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>n_steps</code></b>
-              (<code>int</code>, default:
-                  <code>3</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>number of decision steps. For a better understanding of the function
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>n_steps</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>number of decision steps. For a better understanding of the function
 of <code>n_steps</code> and the upcoming parameters, please see the
 <a href="https://arxiv.org/abs/1908.07442">paper</a>.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>step_dim</code></b>
-              (<code>int</code>, default:
-                  <code>8</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Step's output dimension. This is the output dimension that
+              </div>
+            </td>
+            <td>
+                  <code>3</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>step_dim</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Step's output dimension. This is the output dimension that
 <code>WideDeep</code> will collect and connect to the output neuron(s).</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>attn_dim</code></b>
-              (<code>int</code>, default:
+              </div>
+            </td>
+            <td>
                   <code>8</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Attention dimension</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>dropout</code></b>
-              (<code>float</code>, default:
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>attn_dim</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Attention dimension</p>
+              </div>
+            </td>
+            <td>
+                  <code>8</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>dropout</code></td>
+            <td>
+                  <code>float</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>GLU block's internal dropout</p>
+              </div>
+            </td>
+            <td>
                   <code>0.0</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>GLU block's internal dropout</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>n_glu_step_dependent</code></b>
-              (<code>int</code>, default:
-                  <code>2</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>number of GLU Blocks (<code>[FC -&gt; BN -&gt; GLU]</code>) that are step dependent</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>n_glu_shared</code></b>
-              (<code>int</code>, default:
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>n_glu_step_dependent</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>number of GLU Blocks (<code>[FC -&gt; BN -&gt; GLU]</code>) that are step dependent</p>
+              </div>
+            </td>
+            <td>
                   <code>2</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>number of GLU Blocks (<code>[FC -&gt; BN -&gt; GLU]</code>) that will be shared
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>n_glu_shared</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>number of GLU Blocks (<code>[FC -&gt; BN -&gt; GLU]</code>) that will be shared
 across decision steps</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>ghost_bn</code></b>
-              (<code>bool</code>, default:
-                  <code>True</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Boolean indicating if <a href="https://arxiv.org/abs/1705.08741">Ghost Batch Normalization</a>
+              </div>
+            </td>
+            <td>
+                  <code>2</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>ghost_bn</code></td>
+            <td>
+                  <code>bool</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Boolean indicating if <a href="https://arxiv.org/abs/1705.08741">Ghost Batch Normalization</a>
 will be used.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>virtual_batch_size</code></b>
-              (<code>int</code>, default:
+              </div>
+            </td>
+            <td>
+                  <code>True</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>virtual_batch_size</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Batch size when using Ghost Batch Normalization</p>
+              </div>
+            </td>
+            <td>
                   <code>128</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Batch size when using Ghost Batch Normalization</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>momentum</code></b>
-              (<code>float</code>, default:
-                  <code>0.02</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Ghost Batch Normalization's momentum. The dreamquark-ai advises for
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>momentum</code></td>
+            <td>
+                  <code>float</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Ghost Batch Normalization's momentum. The dreamquark-ai advises for
 very low values. However high values are used in the original
 publication. During our tests higher values lead to better results</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>gamma</code></b>
-              (<code>float</code>, default:
-                  <code>1.3</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Relaxation parameter in the paper. When gamma = 1, a feature is
+              </div>
+            </td>
+            <td>
+                  <code>0.02</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>gamma</code></td>
+            <td>
+                  <code>float</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Relaxation parameter in the paper. When gamma = 1, a feature is
 enforced to be used only at one decision step. As gamma increases,
 more flexibility is provided to use a feature at multiple decision
 steps</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>epsilon</code></b>
-              (<code>float</code>, default:
+              </div>
+            </td>
+            <td>
+                  <code>1.3</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>epsilon</code></td>
+            <td>
+                  <code>float</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Float to avoid log(0). Always keep low</p>
+              </div>
+            </td>
+            <td>
                   <code>1e-15</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Float to avoid log(0). Always keep low</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>mask_type</code></b>
-              (<code>str</code>, default:
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>mask_type</code></td>
+            <td>
+                  <code>str</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Mask function to use. Either <em>'sparsemax'</em> or <em>'entmax'</em></p>
+              </div>
+            </td>
+            <td>
                   <code>&#39;sparsemax&#39;</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Mask function to use. Either <em>'sparsemax'</em> or <em>'entmax'</em></p>
-          </div>
-        </li>
-    </ul>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
 
 <p><span class="doc-section-title">Attributes:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code><span title="pytorch_widedeep.models.tabular.tabnet.tab_net.TabNet.encoder">encoder</span></code></b>
-              (<code><span title="torch.nn.Module">Module</span></code>)
-          –
-          <div class="doc-md-description">
-            <p>the TabNet encoder. For details see the <a href="https://arxiv.org/abs/1908.07442">original publication</a>.</p>
-          </div>
-        </li>
-    </ul>
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code><span title="pytorch_widedeep.models.tabular.tabnet.tab_net.TabNet.encoder">encoder</span></code></td>
+            <td>
+                  <code><span title="torch.nn.Module">Module</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>the TabNet encoder. For details see the <a href="https://arxiv.org/abs/1908.07442">original publication</a>.</p>
+              </div>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
 
 <p><span class="doc-section-title">Examples:</span></p>
@@ -4125,73 +5534,209 @@ <h2 id="pytorch_widedeep.models.tabular.tabnet.tab_net.TabNet" class="doc doc-he
 <span class="gp">&gt;&gt;&gt; </span><span class="n">out</span> <span class="o">=</span> <span class="n">model</span><span class="p">(</span><span class="n">X_tab</span><span class="p">)</span>
 </code></pre></div>
 
-                  <details class="quote">
-                    <summary>Source code in <code>pytorch_widedeep/models/tabular/tabnet/tab_net.py</code></summary>
-                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">151</span>
-<span class="normal">152</span>
-<span class="normal">153</span>
-<span class="normal">154</span>
-<span class="normal">155</span>
-<span class="normal">156</span>
-<span class="normal">157</span>
-<span class="normal">158</span>
-<span class="normal">159</span>
-<span class="normal">160</span>
-<span class="normal">161</span>
-<span class="normal">162</span>
-<span class="normal">163</span>
-<span class="normal">164</span>
-<span class="normal">165</span>
-<span class="normal">166</span>
-<span class="normal">167</span>
-<span class="normal">168</span>
-<span class="normal">169</span>
-<span class="normal">170</span>
-<span class="normal">171</span>
-<span class="normal">172</span>
-<span class="normal">173</span>
-<span class="normal">174</span>
-<span class="normal">175</span>
-<span class="normal">176</span>
-<span class="normal">177</span>
-<span class="normal">178</span>
-<span class="normal">179</span>
-<span class="normal">180</span>
-<span class="normal">181</span>
-<span class="normal">182</span>
-<span class="normal">183</span>
-<span class="normal">184</span>
-<span class="normal">185</span>
-<span class="normal">186</span>
-<span class="normal">187</span>
-<span class="normal">188</span>
-<span class="normal">189</span>
-<span class="normal">190</span>
-<span class="normal">191</span>
-<span class="normal">192</span>
-<span class="normal">193</span>
-<span class="normal">194</span>
-<span class="normal">195</span>
-<span class="normal">196</span>
-<span class="normal">197</span>
-<span class="normal">198</span>
-<span class="normal">199</span>
-<span class="normal">200</span>
-<span class="normal">201</span>
-<span class="normal">202</span>
-<span class="normal">203</span>
-<span class="normal">204</span>
-<span class="normal">205</span>
-<span class="normal">206</span>
-<span class="normal">207</span>
-<span class="normal">208</span>
-<span class="normal">209</span>
-<span class="normal">210</span>
-<span class="normal">211</span>
-<span class="normal">212</span>
-<span class="normal">213</span>
-<span class="normal">214</span>
-<span class="normal">215</span>
+              <details class="quote">
+                <summary>Source code in <code>pytorch_widedeep/models/tabular/tabnet/tab_net.py</code></summary>
+                <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 15</span>
+<span class="normal"> 16</span>
+<span class="normal"> 17</span>
+<span class="normal"> 18</span>
+<span class="normal"> 19</span>
+<span class="normal"> 20</span>
+<span class="normal"> 21</span>
+<span class="normal"> 22</span>
+<span class="normal"> 23</span>
+<span class="normal"> 24</span>
+<span class="normal"> 25</span>
+<span class="normal"> 26</span>
+<span class="normal"> 27</span>
+<span class="normal"> 28</span>
+<span class="normal"> 29</span>
+<span class="normal"> 30</span>
+<span class="normal"> 31</span>
+<span class="normal"> 32</span>
+<span class="normal"> 33</span>
+<span class="normal"> 34</span>
+<span class="normal"> 35</span>
+<span class="normal"> 36</span>
+<span class="normal"> 37</span>
+<span class="normal"> 38</span>
+<span class="normal"> 39</span>
+<span class="normal"> 40</span>
+<span class="normal"> 41</span>
+<span class="normal"> 42</span>
+<span class="normal"> 43</span>
+<span class="normal"> 44</span>
+<span class="normal"> 45</span>
+<span class="normal"> 46</span>
+<span class="normal"> 47</span>
+<span class="normal"> 48</span>
+<span class="normal"> 49</span>
+<span class="normal"> 50</span>
+<span class="normal"> 51</span>
+<span class="normal"> 52</span>
+<span class="normal"> 53</span>
+<span class="normal"> 54</span>
+<span class="normal"> 55</span>
+<span class="normal"> 56</span>
+<span class="normal"> 57</span>
+<span class="normal"> 58</span>
+<span class="normal"> 59</span>
+<span class="normal"> 60</span>
+<span class="normal"> 61</span>
+<span class="normal"> 62</span>
+<span class="normal"> 63</span>
+<span class="normal"> 64</span>
+<span class="normal"> 65</span>
+<span class="normal"> 66</span>
+<span class="normal"> 67</span>
+<span class="normal"> 68</span>
+<span class="normal"> 69</span>
+<span class="normal"> 70</span>
+<span class="normal"> 71</span>
+<span class="normal"> 72</span>
+<span class="normal"> 73</span>
+<span class="normal"> 74</span>
+<span class="normal"> 75</span>
+<span class="normal"> 76</span>
+<span class="normal"> 77</span>
+<span class="normal"> 78</span>
+<span class="normal"> 79</span>
+<span class="normal"> 80</span>
+<span class="normal"> 81</span>
+<span class="normal"> 82</span>
+<span class="normal"> 83</span>
+<span class="normal"> 84</span>
+<span class="normal"> 85</span>
+<span class="normal"> 86</span>
+<span class="normal"> 87</span>
+<span class="normal"> 88</span>
+<span class="normal"> 89</span>
+<span class="normal"> 90</span>
+<span class="normal"> 91</span>
+<span class="normal"> 92</span>
+<span class="normal"> 93</span>
+<span class="normal"> 94</span>
+<span class="normal"> 95</span>
+<span class="normal"> 96</span>
+<span class="normal"> 97</span>
+<span class="normal"> 98</span>
+<span class="normal"> 99</span>
+<span class="normal">100</span>
+<span class="normal">101</span>
+<span class="normal">102</span>
+<span class="normal">103</span>
+<span class="normal">104</span>
+<span class="normal">105</span>
+<span class="normal">106</span>
+<span class="normal">107</span>
+<span class="normal">108</span>
+<span class="normal">109</span>
+<span class="normal">110</span>
+<span class="normal">111</span>
+<span class="normal">112</span>
+<span class="normal">113</span>
+<span class="normal">114</span>
+<span class="normal">115</span>
+<span class="normal">116</span>
+<span class="normal">117</span>
+<span class="normal">118</span>
+<span class="normal">119</span>
+<span class="normal">120</span>
+<span class="normal">121</span>
+<span class="normal">122</span>
+<span class="normal">123</span>
+<span class="normal">124</span>
+<span class="normal">125</span>
+<span class="normal">126</span>
+<span class="normal">127</span>
+<span class="normal">128</span>
+<span class="normal">129</span>
+<span class="normal">130</span>
+<span class="normal">131</span>
+<span class="normal">132</span>
+<span class="normal">133</span>
+<span class="normal">134</span>
+<span class="normal">135</span>
+<span class="normal">136</span>
+<span class="normal">137</span>
+<span class="normal">138</span>
+<span class="normal">139</span>
+<span class="normal">140</span>
+<span class="normal">141</span>
+<span class="normal">142</span>
+<span class="normal">143</span>
+<span class="normal">144</span>
+<span class="normal">145</span>
+<span class="normal">146</span>
+<span class="normal">147</span>
+<span class="normal">148</span>
+<span class="normal">149</span>
+<span class="normal">150</span>
+<span class="normal">151</span>
+<span class="normal">152</span>
+<span class="normal">153</span>
+<span class="normal">154</span>
+<span class="normal">155</span>
+<span class="normal">156</span>
+<span class="normal">157</span>
+<span class="normal">158</span>
+<span class="normal">159</span>
+<span class="normal">160</span>
+<span class="normal">161</span>
+<span class="normal">162</span>
+<span class="normal">163</span>
+<span class="normal">164</span>
+<span class="normal">165</span>
+<span class="normal">166</span>
+<span class="normal">167</span>
+<span class="normal">168</span>
+<span class="normal">169</span>
+<span class="normal">170</span>
+<span class="normal">171</span>
+<span class="normal">172</span>
+<span class="normal">173</span>
+<span class="normal">174</span>
+<span class="normal">175</span>
+<span class="normal">176</span>
+<span class="normal">177</span>
+<span class="normal">178</span>
+<span class="normal">179</span>
+<span class="normal">180</span>
+<span class="normal">181</span>
+<span class="normal">182</span>
+<span class="normal">183</span>
+<span class="normal">184</span>
+<span class="normal">185</span>
+<span class="normal">186</span>
+<span class="normal">187</span>
+<span class="normal">188</span>
+<span class="normal">189</span>
+<span class="normal">190</span>
+<span class="normal">191</span>
+<span class="normal">192</span>
+<span class="normal">193</span>
+<span class="normal">194</span>
+<span class="normal">195</span>
+<span class="normal">196</span>
+<span class="normal">197</span>
+<span class="normal">198</span>
+<span class="normal">199</span>
+<span class="normal">200</span>
+<span class="normal">201</span>
+<span class="normal">202</span>
+<span class="normal">203</span>
+<span class="normal">204</span>
+<span class="normal">205</span>
+<span class="normal">206</span>
+<span class="normal">207</span>
+<span class="normal">208</span>
+<span class="normal">209</span>
+<span class="normal">210</span>
+<span class="normal">211</span>
+<span class="normal">212</span>
+<span class="normal">213</span>
+<span class="normal">214</span>
+<span class="normal">215</span>
 <span class="normal">216</span>
 <span class="normal">217</span>
 <span class="normal">218</span>
@@ -4213,95 +5758,269 @@ <h2 id="pytorch_widedeep.models.tabular.tabnet.tab_net.TabNet" class="doc doc-he
 <span class="normal">234</span>
 <span class="normal">235</span>
 <span class="normal">236</span>
-<span class="normal">237</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
-    <span class="bp">self</span><span class="p">,</span>
-    <span class="n">column_idx</span><span class="p">:</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">],</span>
-    <span class="o">*</span><span class="p">,</span>
-    <span class="n">cat_embed_input</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="n">Tuple</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">,</span> <span class="nb">int</span><span class="p">]]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">cat_embed_dropout</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">use_cat_bias</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">cat_embed_activation</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">continuous_cols</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">cont_norm_layer</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Literal</span><span class="p">[</span><span class="s2">&quot;batchnorm&quot;</span><span class="p">,</span> <span class="s2">&quot;layernorm&quot;</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">embed_continuous</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">embed_continuous_method</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span>
-        <span class="n">Literal</span><span class="p">[</span><span class="s2">&quot;standard&quot;</span><span class="p">,</span> <span class="s2">&quot;piecewise&quot;</span><span class="p">,</span> <span class="s2">&quot;periodic&quot;</span><span class="p">]</span>
-    <span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">cont_embed_dim</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">cont_embed_dropout</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">cont_embed_activation</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">quantization_setup</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="nb">float</span><span class="p">]]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">n_frequencies</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">sigma</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">share_last_layer</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">full_embed_dropout</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">n_steps</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">3</span><span class="p">,</span>
-    <span class="n">step_dim</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">8</span><span class="p">,</span>
-    <span class="n">attn_dim</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">8</span><span class="p">,</span>
-    <span class="n">dropout</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.0</span><span class="p">,</span>
-    <span class="n">n_glu_step_dependent</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">2</span><span class="p">,</span>
-    <span class="n">n_glu_shared</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">2</span><span class="p">,</span>
-    <span class="n">ghost_bn</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">,</span>
-    <span class="n">virtual_batch_size</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">128</span><span class="p">,</span>
-    <span class="n">momentum</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.02</span><span class="p">,</span>
-    <span class="n">gamma</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">1.3</span><span class="p">,</span>
-    <span class="n">epsilon</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">1e-15</span><span class="p">,</span>
-    <span class="n">mask_type</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;sparsemax&quot;</span><span class="p">,</span>
-<span class="p">):</span>
-    <span class="nb">super</span><span class="p">(</span><span class="n">TabNet</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span>
-        <span class="n">column_idx</span><span class="o">=</span><span class="n">column_idx</span><span class="p">,</span>
-        <span class="n">cat_embed_input</span><span class="o">=</span><span class="n">cat_embed_input</span><span class="p">,</span>
-        <span class="n">cat_embed_dropout</span><span class="o">=</span><span class="n">cat_embed_dropout</span><span class="p">,</span>
-        <span class="n">use_cat_bias</span><span class="o">=</span><span class="n">use_cat_bias</span><span class="p">,</span>
-        <span class="n">cat_embed_activation</span><span class="o">=</span><span class="n">cat_embed_activation</span><span class="p">,</span>
-        <span class="n">continuous_cols</span><span class="o">=</span><span class="n">continuous_cols</span><span class="p">,</span>
-        <span class="n">cont_norm_layer</span><span class="o">=</span><span class="n">cont_norm_layer</span><span class="p">,</span>
-        <span class="n">embed_continuous</span><span class="o">=</span><span class="n">embed_continuous</span><span class="p">,</span>
-        <span class="n">embed_continuous_method</span><span class="o">=</span><span class="n">embed_continuous_method</span><span class="p">,</span>
-        <span class="n">cont_embed_dim</span><span class="o">=</span><span class="n">cont_embed_dim</span><span class="p">,</span>
-        <span class="n">cont_embed_dropout</span><span class="o">=</span><span class="n">cont_embed_dropout</span><span class="p">,</span>
-        <span class="n">cont_embed_activation</span><span class="o">=</span><span class="n">cont_embed_activation</span><span class="p">,</span>
-        <span class="n">quantization_setup</span><span class="o">=</span><span class="n">quantization_setup</span><span class="p">,</span>
-        <span class="n">n_frequencies</span><span class="o">=</span><span class="n">n_frequencies</span><span class="p">,</span>
-        <span class="n">sigma</span><span class="o">=</span><span class="n">sigma</span><span class="p">,</span>
-        <span class="n">share_last_layer</span><span class="o">=</span><span class="n">share_last_layer</span><span class="p">,</span>
-        <span class="n">full_embed_dropout</span><span class="o">=</span><span class="n">full_embed_dropout</span><span class="p">,</span>
-    <span class="p">)</span>
+<span class="normal">237</span>
+<span class="normal">238</span>
+<span class="normal">239</span>
+<span class="normal">240</span>
+<span class="normal">241</span>
+<span class="normal">242</span>
+<span class="normal">243</span>
+<span class="normal">244</span>
+<span class="normal">245</span>
+<span class="normal">246</span>
+<span class="normal">247</span>
+<span class="normal">248</span>
+<span class="normal">249</span>
+<span class="normal">250</span>
+<span class="normal">251</span>
+<span class="normal">252</span>
+<span class="normal">253</span>
+<span class="normal">254</span>
+<span class="normal">255</span>
+<span class="normal">256</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">class</span> <span class="nc">TabNet</span><span class="p">(</span><span class="n">BaseTabularModelWithoutAttention</span><span class="p">):</span>
+<span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Defines a [TabNet model](https://arxiv.org/abs/1908.07442) that</span>
+<span class="sd">    can be used as the `deeptabular` component of a Wide &amp; Deep model or</span>
+<span class="sd">    independently by itself.</span>
+
+<span class="sd">    The implementation in this library is fully based on that</span>
+<span class="sd">    [here](https://github.com/dreamquark-ai/tabnet) by the dreamquark-ai team,</span>
+<span class="sd">    simply adapted so that it can work within the `WideDeep` frame.</span>
+<span class="sd">    Therefore, **ALL CREDIT TO THE DREAMQUARK-AI TEAM**.</span>
+
+<span class="sd">    Parameters</span>
+<span class="sd">    ----------</span>
+<span class="sd">    column_idx: Dict</span>
+<span class="sd">        Dict containing the index of the columns that will be passed through</span>
+<span class="sd">        the `TabMlp` model. Required to slice the tensors. e.g. _{&#39;education&#39;:</span>
+<span class="sd">        0, &#39;relationship&#39;: 1, &#39;workclass&#39;: 2, ...}_.</span>
+<span class="sd">    cat_embed_input: List, Optional, default = None</span>
+<span class="sd">        List of Tuples with the column name, number of unique values and</span>
+<span class="sd">        embedding dimension. e.g. _[(education, 11, 32), ...]_</span>
+<span class="sd">    cat_embed_dropout: float, Optional, default = None</span>
+<span class="sd">        Categorical embeddings dropout. If `None`, it will default</span>
+<span class="sd">        to 0.</span>
+<span class="sd">    use_cat_bias: bool, Optional, default = None,</span>
+<span class="sd">        Boolean indicating if bias will be used for the categorical embeddings.</span>
+<span class="sd">        If `None`, it will default to &#39;False&#39;.</span>
+<span class="sd">    cat_embed_activation: Optional, str, default = None,</span>
+<span class="sd">        Activation function for the categorical embeddings, if any. Currently</span>
+<span class="sd">        _&#39;tanh&#39;_, _&#39;relu&#39;_, _&#39;leaky_relu&#39;_ and _&#39;gelu&#39;_ are supported</span>
+<span class="sd">    continuous_cols: List, Optional, default = None</span>
+<span class="sd">        List with the name of the numeric (aka continuous) columns</span>
+<span class="sd">    cont_norm_layer: str, Optional, default =  None</span>
+<span class="sd">        Type of normalization layer applied to the continuous features.</span>
+<span class="sd">        Options are: _&#39;layernorm&#39;_ and _&#39;batchnorm&#39;_. if `None`, no</span>
+<span class="sd">        normalization layer will be used.</span>
+<span class="sd">    embed_continuous: bool, Optional, default = None,</span>
+<span class="sd">        Boolean indicating if the continuous columns will be embedded using</span>
+<span class="sd">        one of the available methods: _&#39;standard&#39;_, _&#39;periodic&#39;_</span>
+<span class="sd">        or _&#39;piecewise&#39;_. If `None`, it will default to &#39;False&#39;.&lt;br/&gt;</span>
+<span class="sd">        :information_source: **NOTE**: This parameter is deprecated and it</span>
+<span class="sd">         will be removed in future releases. Please, use the</span>
+<span class="sd">         `embed_continuous_method` parameter instead.</span>
+<span class="sd">    embed_continuous_method: Optional, str, default = None,</span>
+<span class="sd">        Method to use to embed the continuous features. Options are:</span>
+<span class="sd">        _&#39;standard&#39;_, _&#39;periodic&#39;_ or _&#39;piecewise&#39;_. The _&#39;standard&#39;_</span>
+<span class="sd">        embedding method is based on the FT-Transformer implementation</span>
+<span class="sd">        presented in the paper: [Revisiting Deep Learning Models for</span>
+<span class="sd">        Tabular Data](https://arxiv.org/abs/2106.11959v5). The _&#39;periodic&#39;_</span>
+<span class="sd">        and_&#39;piecewise&#39;_ methods were presented in the paper: [On Embeddings for</span>
+<span class="sd">        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556).</span>
+<span class="sd">        Please, read the papers for details.</span>
+<span class="sd">    cont_embed_dim: int, Optional, default = None,</span>
+<span class="sd">        Size of the continuous embeddings. If the continuous columns are</span>
+<span class="sd">        embedded, `cont_embed_dim` must be passed.</span>
+<span class="sd">    cont_embed_dropout: float, Optional, default = None,</span>
+<span class="sd">        Dropout for the continuous embeddings. If `None`, it will default to 0.0</span>
+<span class="sd">    cont_embed_activation: Optional, str, default = None,</span>
+<span class="sd">        Activation function for the continuous embeddings if any. Currently</span>
+<span class="sd">        _&#39;tanh&#39;_, _&#39;relu&#39;_, _&#39;leaky_relu&#39;_ and _&#39;gelu&#39;_ are supported.</span>
+<span class="sd">        If `None`, no activation function will be applied.</span>
+<span class="sd">    quantization_setup: Dict[str, List[float]], Optional, default = None,</span>
+<span class="sd">        This parameter is used when the _&#39;piecewise&#39;_ method is used to embed</span>
+<span class="sd">        the continuous cols. It is a dict where keys are the name of the continuous</span>
+<span class="sd">        columns and values are lists with the boundaries for the quantization</span>
+<span class="sd">        of the continuous_cols. See the examples for details. If</span>
+<span class="sd">        If the _&#39;piecewise&#39;_ method is used, this parameter is required.</span>
+<span class="sd">    n_frequencies: int, Optional, default = None,</span>
+<span class="sd">        This is the so called _&#39;k&#39;_ in their paper [On Embeddings for</span>
+<span class="sd">        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556),</span>
+<span class="sd">        and is the number of &#39;frequencies&#39; that will be used to represent each</span>
+<span class="sd">        continuous column. See their Eq 2 in the paper for details. If</span>
+<span class="sd">        the _&#39;periodic&#39;_ method is used, this parameter is required.</span>
+<span class="sd">    sigma: float, Optional, default = None,</span>
+<span class="sd">        This is the sigma parameter in the paper mentioned when describing the</span>
+<span class="sd">        previous parameters and it is used to initialise the &#39;frequency</span>
+<span class="sd">        weights&#39;. See their Eq 2 in the paper for details. If</span>
+<span class="sd">        the _&#39;periodic&#39;_ method is used, this parameter is required.</span>
+<span class="sd">    share_last_layer: bool, Optional, default = None,</span>
+<span class="sd">        This parameter is not present in the before mentioned paper but it is implemented in</span>
+<span class="sd">        the [official repo](https://github.com/yandex-research/rtdl-num-embeddings/tree/main).</span>
+<span class="sd">        If `True` the linear layer that turns the frequencies into embeddings</span>
+<span class="sd">        will be shared across the continuous columns. If `False` a different</span>
+<span class="sd">        linear layer will be used for each continuous column.</span>
+<span class="sd">        If the _&#39;periodic&#39;_ method is used, this parameter is required.</span>
+<span class="sd">    full_embed_dropout: bool, Optional, default = None,</span>
+<span class="sd">        If `True`, the full embedding corresponding to a column will be masked</span>
+<span class="sd">        out/dropout. If `None`, it will default to `False`.</span>
+<span class="sd">    n_steps: int, default = 3</span>
+<span class="sd">        number of decision steps. For a better understanding of the function</span>
+<span class="sd">        of `n_steps` and the upcoming parameters, please see the</span>
+<span class="sd">        [paper](https://arxiv.org/abs/1908.07442).</span>
+<span class="sd">    step_dim: int, default = 8</span>
+<span class="sd">        Step&#39;s output dimension. This is the output dimension that</span>
+<span class="sd">        `WideDeep` will collect and connect to the output neuron(s).</span>
+<span class="sd">    attn_dim: int, default = 8</span>
+<span class="sd">        Attention dimension</span>
+<span class="sd">    dropout: float, default = 0.0</span>
+<span class="sd">        GLU block&#39;s internal dropout</span>
+<span class="sd">    n_glu_step_dependent: int, default = 2</span>
+<span class="sd">        number of GLU Blocks (`[FC -&gt; BN -&gt; GLU]`) that are step dependent</span>
+<span class="sd">    n_glu_shared: int, default = 2</span>
+<span class="sd">        number of GLU Blocks (`[FC -&gt; BN -&gt; GLU]`) that will be shared</span>
+<span class="sd">        across decision steps</span>
+<span class="sd">    ghost_bn: bool, default=True</span>
+<span class="sd">        Boolean indicating if [Ghost Batch Normalization](https://arxiv.org/abs/1705.08741)</span>
+<span class="sd">        will be used.</span>
+<span class="sd">    virtual_batch_size: int, default = 128</span>
+<span class="sd">        Batch size when using Ghost Batch Normalization</span>
+<span class="sd">    momentum: float, default = 0.02</span>
+<span class="sd">        Ghost Batch Normalization&#39;s momentum. The dreamquark-ai advises for</span>
+<span class="sd">        very low values. However high values are used in the original</span>
+<span class="sd">        publication. During our tests higher values lead to better results</span>
+<span class="sd">    gamma: float, default = 1.3</span>
+<span class="sd">        Relaxation parameter in the paper. When gamma = 1, a feature is</span>
+<span class="sd">        enforced to be used only at one decision step. As gamma increases,</span>
+<span class="sd">        more flexibility is provided to use a feature at multiple decision</span>
+<span class="sd">        steps</span>
+<span class="sd">    epsilon: float, default = 1e-15</span>
+<span class="sd">        Float to avoid log(0). Always keep low</span>
+<span class="sd">    mask_type: str, default = &quot;sparsemax&quot;</span>
+<span class="sd">        Mask function to use. Either _&#39;sparsemax&#39;_ or _&#39;entmax&#39;_</span>
+
+<span class="sd">    Attributes</span>
+<span class="sd">    ----------</span>
+<span class="sd">    encoder: nn.Module</span>
+<span class="sd">        the TabNet encoder. For details see the [original publication](https://arxiv.org/abs/1908.07442).</span>
+
+<span class="sd">    Examples</span>
+<span class="sd">    --------</span>
+<span class="sd">    &gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)</span>
+<span class="sd">    &gt;&gt;&gt; colnames = [&quot;a&quot;, &quot;b&quot;, &quot;c&quot;, &quot;d&quot;, &quot;e&quot;]</span>
+<span class="sd">    &gt;&gt;&gt; cat_embed_input = [(u, i, j) for u, i, j in zip(colnames[:4], [4] * 4, [8] * 4)]</span>
+<span class="sd">    &gt;&gt;&gt; column_idx = {k: v for v, k in enumerate(colnames)}</span>
+<span class="sd">    &gt;&gt;&gt; model = TabNet(column_idx=column_idx, cat_embed_input=cat_embed_input, continuous_cols=[&quot;e&quot;])</span>
+<span class="sd">    &gt;&gt;&gt; out = model(X_tab)</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+
+    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span>
+        <span class="n">column_idx</span><span class="p">:</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">],</span>
+        <span class="o">*</span><span class="p">,</span>
+        <span class="n">cat_embed_input</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="n">Tuple</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">,</span> <span class="nb">int</span><span class="p">]]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">cat_embed_dropout</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">use_cat_bias</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">cat_embed_activation</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">continuous_cols</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">cont_norm_layer</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Literal</span><span class="p">[</span><span class="s2">&quot;batchnorm&quot;</span><span class="p">,</span> <span class="s2">&quot;layernorm&quot;</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">embed_continuous</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">embed_continuous_method</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span>
+            <span class="n">Literal</span><span class="p">[</span><span class="s2">&quot;standard&quot;</span><span class="p">,</span> <span class="s2">&quot;piecewise&quot;</span><span class="p">,</span> <span class="s2">&quot;periodic&quot;</span><span class="p">]</span>
+        <span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">cont_embed_dim</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">cont_embed_dropout</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">cont_embed_activation</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">quantization_setup</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="nb">float</span><span class="p">]]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">n_frequencies</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">sigma</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">share_last_layer</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">full_embed_dropout</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">n_steps</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">3</span><span class="p">,</span>
+        <span class="n">step_dim</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">8</span><span class="p">,</span>
+        <span class="n">attn_dim</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">8</span><span class="p">,</span>
+        <span class="n">dropout</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.0</span><span class="p">,</span>
+        <span class="n">n_glu_step_dependent</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">2</span><span class="p">,</span>
+        <span class="n">n_glu_shared</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">2</span><span class="p">,</span>
+        <span class="n">ghost_bn</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">,</span>
+        <span class="n">virtual_batch_size</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">128</span><span class="p">,</span>
+        <span class="n">momentum</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.02</span><span class="p">,</span>
+        <span class="n">gamma</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">1.3</span><span class="p">,</span>
+        <span class="n">epsilon</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">1e-15</span><span class="p">,</span>
+        <span class="n">mask_type</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;sparsemax&quot;</span><span class="p">,</span>
+    <span class="p">):</span>
+        <span class="nb">super</span><span class="p">(</span><span class="n">TabNet</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span>
+            <span class="n">column_idx</span><span class="o">=</span><span class="n">column_idx</span><span class="p">,</span>
+            <span class="n">cat_embed_input</span><span class="o">=</span><span class="n">cat_embed_input</span><span class="p">,</span>
+            <span class="n">cat_embed_dropout</span><span class="o">=</span><span class="n">cat_embed_dropout</span><span class="p">,</span>
+            <span class="n">use_cat_bias</span><span class="o">=</span><span class="n">use_cat_bias</span><span class="p">,</span>
+            <span class="n">cat_embed_activation</span><span class="o">=</span><span class="n">cat_embed_activation</span><span class="p">,</span>
+            <span class="n">continuous_cols</span><span class="o">=</span><span class="n">continuous_cols</span><span class="p">,</span>
+            <span class="n">cont_norm_layer</span><span class="o">=</span><span class="n">cont_norm_layer</span><span class="p">,</span>
+            <span class="n">embed_continuous</span><span class="o">=</span><span class="n">embed_continuous</span><span class="p">,</span>
+            <span class="n">embed_continuous_method</span><span class="o">=</span><span class="n">embed_continuous_method</span><span class="p">,</span>
+            <span class="n">cont_embed_dim</span><span class="o">=</span><span class="n">cont_embed_dim</span><span class="p">,</span>
+            <span class="n">cont_embed_dropout</span><span class="o">=</span><span class="n">cont_embed_dropout</span><span class="p">,</span>
+            <span class="n">cont_embed_activation</span><span class="o">=</span><span class="n">cont_embed_activation</span><span class="p">,</span>
+            <span class="n">quantization_setup</span><span class="o">=</span><span class="n">quantization_setup</span><span class="p">,</span>
+            <span class="n">n_frequencies</span><span class="o">=</span><span class="n">n_frequencies</span><span class="p">,</span>
+            <span class="n">sigma</span><span class="o">=</span><span class="n">sigma</span><span class="p">,</span>
+            <span class="n">share_last_layer</span><span class="o">=</span><span class="n">share_last_layer</span><span class="p">,</span>
+            <span class="n">full_embed_dropout</span><span class="o">=</span><span class="n">full_embed_dropout</span><span class="p">,</span>
+        <span class="p">)</span>
 
-    <span class="bp">self</span><span class="o">.</span><span class="n">n_steps</span> <span class="o">=</span> <span class="n">n_steps</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">step_dim</span> <span class="o">=</span> <span class="n">step_dim</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">attn_dim</span> <span class="o">=</span> <span class="n">attn_dim</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">dropout</span> <span class="o">=</span> <span class="n">dropout</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">n_glu_step_dependent</span> <span class="o">=</span> <span class="n">n_glu_step_dependent</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">n_glu_shared</span> <span class="o">=</span> <span class="n">n_glu_shared</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">ghost_bn</span> <span class="o">=</span> <span class="n">ghost_bn</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">virtual_batch_size</span> <span class="o">=</span> <span class="n">virtual_batch_size</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">momentum</span> <span class="o">=</span> <span class="n">momentum</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">gamma</span> <span class="o">=</span> <span class="n">gamma</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">epsilon</span> <span class="o">=</span> <span class="n">epsilon</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">mask_type</span> <span class="o">=</span> <span class="n">mask_type</span>
-
-    <span class="c1"># Embeddings are instantiated at the base model</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">embed_out_dim</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">cat_out_dim</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">cont_out_dim</span>
-
-    <span class="c1"># TabNet</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">encoder</span> <span class="o">=</span> <span class="n">TabNetEncoder</span><span class="p">(</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">embed_out_dim</span><span class="p">,</span>
-        <span class="n">n_steps</span><span class="p">,</span>
-        <span class="n">step_dim</span><span class="p">,</span>
-        <span class="n">attn_dim</span><span class="p">,</span>
-        <span class="n">dropout</span><span class="p">,</span>
-        <span class="n">n_glu_step_dependent</span><span class="p">,</span>
-        <span class="n">n_glu_shared</span><span class="p">,</span>
-        <span class="n">ghost_bn</span><span class="p">,</span>
-        <span class="n">virtual_batch_size</span><span class="p">,</span>
-        <span class="n">momentum</span><span class="p">,</span>
-        <span class="n">gamma</span><span class="p">,</span>
-        <span class="n">epsilon</span><span class="p">,</span>
-        <span class="n">mask_type</span><span class="p">,</span>
-    <span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">n_steps</span> <span class="o">=</span> <span class="n">n_steps</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">step_dim</span> <span class="o">=</span> <span class="n">step_dim</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">attn_dim</span> <span class="o">=</span> <span class="n">attn_dim</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">dropout</span> <span class="o">=</span> <span class="n">dropout</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">n_glu_step_dependent</span> <span class="o">=</span> <span class="n">n_glu_step_dependent</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">n_glu_shared</span> <span class="o">=</span> <span class="n">n_glu_shared</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">ghost_bn</span> <span class="o">=</span> <span class="n">ghost_bn</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">virtual_batch_size</span> <span class="o">=</span> <span class="n">virtual_batch_size</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">momentum</span> <span class="o">=</span> <span class="n">momentum</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">gamma</span> <span class="o">=</span> <span class="n">gamma</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">epsilon</span> <span class="o">=</span> <span class="n">epsilon</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">mask_type</span> <span class="o">=</span> <span class="n">mask_type</span>
+
+        <span class="c1"># Embeddings are instantiated at the base model</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">embed_out_dim</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">cat_out_dim</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">cont_out_dim</span>
+
+        <span class="c1"># TabNet</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">encoder</span> <span class="o">=</span> <span class="n">TabNetEncoder</span><span class="p">(</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">embed_out_dim</span><span class="p">,</span>
+            <span class="n">n_steps</span><span class="p">,</span>
+            <span class="n">step_dim</span><span class="p">,</span>
+            <span class="n">attn_dim</span><span class="p">,</span>
+            <span class="n">dropout</span><span class="p">,</span>
+            <span class="n">n_glu_step_dependent</span><span class="p">,</span>
+            <span class="n">n_glu_shared</span><span class="p">,</span>
+            <span class="n">ghost_bn</span><span class="p">,</span>
+            <span class="n">virtual_batch_size</span><span class="p">,</span>
+            <span class="n">momentum</span><span class="p">,</span>
+            <span class="n">gamma</span><span class="p">,</span>
+            <span class="n">epsilon</span><span class="p">,</span>
+            <span class="n">mask_type</span><span class="p">,</span>
+        <span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">forward</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span> <span class="n">X</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">,</span> <span class="n">prior</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Tensor</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span>
+    <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tuple</span><span class="p">[</span><span class="n">Tensor</span><span class="p">,</span> <span class="n">Tensor</span><span class="p">]:</span>
+        <span class="n">x</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_embeddings</span><span class="p">(</span><span class="n">X</span><span class="p">)</span>
+        <span class="n">steps_output</span><span class="p">,</span> <span class="n">M_loss</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">encoder</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">prior</span><span class="p">)</span>
+        <span class="n">res</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">sum</span><span class="p">(</span><span class="n">torch</span><span class="o">.</span><span class="n">stack</span><span class="p">(</span><span class="n">steps_output</span><span class="p">,</span> <span class="n">dim</span><span class="o">=</span><span class="mi">0</span><span class="p">),</span> <span class="n">dim</span><span class="o">=</span><span class="mi">0</span><span class="p">)</span>
+        <span class="k">return</span> <span class="p">(</span><span class="n">res</span><span class="p">,</span> <span class="n">M_loss</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">forward_masks</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">X</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tuple</span><span class="p">[</span><span class="n">Tensor</span><span class="p">,</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">int</span><span class="p">,</span> <span class="n">Tensor</span><span class="p">]]:</span>
+        <span class="n">x</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_embeddings</span><span class="p">(</span><span class="n">X</span><span class="p">)</span>
+        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">encoder</span><span class="o">.</span><span class="n">forward_masks</span><span class="p">(</span><span class="n">x</span><span class="p">)</span>
+
+    <span class="nd">@property</span>
+    <span class="k">def</span> <span class="nf">output_dim</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">int</span><span class="p">:</span>
+<span class="w">        </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;The output dimension of the model. This is a required property</span>
+<span class="sd">        neccesary to build the `WideDeep` class</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">step_dim</span>
 </code></pre></div></td></tr></table></div>
-                  </details>
+              </details>
 
 
 
@@ -4356,18 +6075,7 @@ <h2 id="pytorch_widedeep.models.tabular.tabnet.tab_net.TabNetDecoder" class="doc
 
 
 <a href="#pytorch_widedeep.models.tabular.tabnet.tab_net.TabNetDecoder" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="nf">TabNetDecoder</span><span class="p">(</span>
-    <span class="n">embed_dim</span><span class="p">,</span>
-    <span class="n">n_steps</span><span class="o">=</span><span class="mi">3</span><span class="p">,</span>
-    <span class="n">step_dim</span><span class="o">=</span><span class="mi">8</span><span class="p">,</span>
-    <span class="n">dropout</span><span class="o">=</span><span class="mf">0.0</span><span class="p">,</span>
-    <span class="n">n_glu_step_dependent</span><span class="o">=</span><span class="mi">2</span><span class="p">,</span>
-    <span class="n">n_glu_shared</span><span class="o">=</span><span class="mi">2</span><span class="p">,</span>
-    <span class="n">ghost_bn</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
-    <span class="n">virtual_batch_size</span><span class="o">=</span><span class="mi">128</span><span class="p">,</span>
-    <span class="n">momentum</span><span class="o">=</span><span class="mf">0.02</span><span class="p">,</span>
-<span class="p">)</span>
-</code></pre></div>
+
 
     <div class="doc doc-contents first">
             <p class="doc doc-class-bases">
@@ -4384,117 +6092,177 @@ <h2 id="pytorch_widedeep.models.tabular.tabnet.tab_net.TabNetDecoder" class="doc
 
 
 <p><span class="doc-section-title">Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>embed_dim</code></b>
-              (<code>int</code>)
-          –
-          <div class="doc-md-description">
-            <p>Size of the embeddings tensor to be reconstructed.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>n_steps</code></b>
-              (<code>int</code>, default:
-                  <code>3</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>number of decision steps. For a better understanding of the function
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+          <th>Default</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>embed_dim</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Size of the embeddings tensor to be reconstructed.</p>
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>n_steps</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>number of decision steps. For a better understanding of the function
 of <code>n_steps</code> and the upcoming parameters, please see the
 <a href="https://arxiv.org/abs/1908.07442">paper</a>.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>step_dim</code></b>
-              (<code>int</code>, default:
-                  <code>8</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Step's output dimension. This is the output dimension that
+              </div>
+            </td>
+            <td>
+                  <code>3</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>step_dim</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Step's output dimension. This is the output dimension that
 <code>WideDeep</code> will collect and connect to the output neuron(s).</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>dropout</code></b>
-              (<code>float</code>, default:
+              </div>
+            </td>
+            <td>
+                  <code>8</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>dropout</code></td>
+            <td>
+                  <code>float</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>GLU block's internal dropout</p>
+              </div>
+            </td>
+            <td>
                   <code>0.0</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>GLU block's internal dropout</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>n_glu_step_dependent</code></b>
-              (<code>int</code>, default:
-                  <code>2</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>number of GLU Blocks (<code>[FC -&gt; BN -&gt; GLU]</code>) that are step dependent</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>n_glu_shared</code></b>
-              (<code>int</code>, default:
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>n_glu_step_dependent</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>number of GLU Blocks (<code>[FC -&gt; BN -&gt; GLU]</code>) that are step dependent</p>
+              </div>
+            </td>
+            <td>
                   <code>2</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>number of GLU Blocks (<code>[FC -&gt; BN -&gt; GLU]</code>) that will be shared
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>n_glu_shared</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>number of GLU Blocks (<code>[FC -&gt; BN -&gt; GLU]</code>) that will be shared
 across decision steps</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>ghost_bn</code></b>
-              (<code>bool</code>, default:
-                  <code>True</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Boolean indicating if <a href="https://arxiv.org/abs/1705.08741">Ghost Batch Normalization</a>
+              </div>
+            </td>
+            <td>
+                  <code>2</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>ghost_bn</code></td>
+            <td>
+                  <code>bool</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Boolean indicating if <a href="https://arxiv.org/abs/1705.08741">Ghost Batch Normalization</a>
 will be used.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>virtual_batch_size</code></b>
-              (<code>int</code>, default:
+              </div>
+            </td>
+            <td>
+                  <code>True</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>virtual_batch_size</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Batch size when using Ghost Batch Normalization</p>
+              </div>
+            </td>
+            <td>
                   <code>128</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Batch size when using Ghost Batch Normalization</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>momentum</code></b>
-              (<code>float</code>, default:
-                  <code>0.02</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Ghost Batch Normalization's momentum. The dreamquark-ai advises for
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>momentum</code></td>
+            <td>
+                  <code>float</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Ghost Batch Normalization's momentum. The dreamquark-ai advises for
 very low values. However high values are used in the original
 publication. During our tests higher values lead to better results</p>
-          </div>
-        </li>
-    </ul>
+              </div>
+            </td>
+            <td>
+                  <code>0.02</code>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
 
 <p><span class="doc-section-title">Attributes:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code><span title="pytorch_widedeep.models.tabular.tabnet.tab_net.TabNetDecoder.decoder">decoder</span></code></b>
-              (<code><span title="torch.nn.Module">Module</span></code>)
-          –
-          <div class="doc-md-description">
-            <p>decoder that will receive the output from the encoder's steps and will
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code><span title="pytorch_widedeep.models.tabular.tabnet.tab_net.TabNetDecoder.decoder">decoder</span></code></td>
+            <td>
+                  <code><span title="torch.nn.Module">Module</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>decoder that will receive the output from the encoder's steps and will
 reconstruct the embeddings</p>
-          </div>
-        </li>
-    </ul>
+              </div>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
 
 <p><span class="doc-section-title">Examples:</span></p>
@@ -4507,101 +6275,227 @@ <h2 id="pytorch_widedeep.models.tabular.tabnet.tab_net.TabNetDecoder" class="doc
 <span class="go">torch.Size([3, 32])</span>
 </code></pre></div>
 
-                  <details class="quote">
-                    <summary>Source code in <code>pytorch_widedeep/models/tabular/tabnet/tab_net.py</code></summary>
-                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">335</span>
-<span class="normal">336</span>
-<span class="normal">337</span>
-<span class="normal">338</span>
-<span class="normal">339</span>
-<span class="normal">340</span>
-<span class="normal">341</span>
-<span class="normal">342</span>
-<span class="normal">343</span>
-<span class="normal">344</span>
-<span class="normal">345</span>
-<span class="normal">346</span>
-<span class="normal">347</span>
-<span class="normal">348</span>
-<span class="normal">349</span>
-<span class="normal">350</span>
-<span class="normal">351</span>
-<span class="normal">352</span>
-<span class="normal">353</span>
-<span class="normal">354</span>
-<span class="normal">355</span>
-<span class="normal">356</span>
-<span class="normal">357</span>
-<span class="normal">358</span>
-<span class="normal">359</span>
-<span class="normal">360</span>
-<span class="normal">361</span>
-<span class="normal">362</span>
-<span class="normal">363</span>
-<span class="normal">364</span>
-<span class="normal">365</span>
-<span class="normal">366</span>
-<span class="normal">367</span>
-<span class="normal">368</span>
-<span class="normal">369</span>
-<span class="normal">370</span>
-<span class="normal">371</span>
-<span class="normal">372</span>
-<span class="normal">373</span>
-<span class="normal">374</span>
-<span class="normal">375</span>
-<span class="normal">376</span>
-<span class="normal">377</span>
-<span class="normal">378</span>
-<span class="normal">379</span>
-<span class="normal">380</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
-    <span class="bp">self</span><span class="p">,</span>
-    <span class="n">embed_dim</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span>
-    <span class="n">n_steps</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">3</span><span class="p">,</span>
-    <span class="n">step_dim</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">8</span><span class="p">,</span>
-    <span class="n">dropout</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.0</span><span class="p">,</span>
-    <span class="n">n_glu_step_dependent</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">2</span><span class="p">,</span>
-    <span class="n">n_glu_shared</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">2</span><span class="p">,</span>
-    <span class="n">ghost_bn</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">,</span>
-    <span class="n">virtual_batch_size</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">128</span><span class="p">,</span>
-    <span class="n">momentum</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.02</span><span class="p">,</span>
-<span class="p">):</span>
-    <span class="nb">super</span><span class="p">(</span><span class="n">TabNetDecoder</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
-
-    <span class="bp">self</span><span class="o">.</span><span class="n">n_steps</span> <span class="o">=</span> <span class="n">n_steps</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">step_dim</span> <span class="o">=</span> <span class="n">step_dim</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">dropout</span> <span class="o">=</span> <span class="n">dropout</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">n_glu_step_dependent</span> <span class="o">=</span> <span class="n">n_glu_step_dependent</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">n_glu_shared</span> <span class="o">=</span> <span class="n">n_glu_shared</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">ghost_bn</span> <span class="o">=</span> <span class="n">ghost_bn</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">virtual_batch_size</span> <span class="o">=</span> <span class="n">virtual_batch_size</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">momentum</span> <span class="o">=</span> <span class="n">momentum</span>
-
-    <span class="n">shared_layers</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">ModuleList</span><span class="p">()</span>
-    <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">n_glu_shared</span><span class="p">):</span>
-        <span class="k">if</span> <span class="n">i</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
-            <span class="n">shared_layers</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">nn</span><span class="o">.</span><span class="n">Linear</span><span class="p">(</span><span class="n">step_dim</span><span class="p">,</span> <span class="mi">2</span> <span class="o">*</span> <span class="n">step_dim</span><span class="p">,</span> <span class="n">bias</span><span class="o">=</span><span class="kc">False</span><span class="p">))</span>
-        <span class="k">else</span><span class="p">:</span>
-            <span class="n">shared_layers</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">nn</span><span class="o">.</span><span class="n">Linear</span><span class="p">(</span><span class="n">step_dim</span><span class="p">,</span> <span class="mi">2</span> <span class="o">*</span> <span class="n">step_dim</span><span class="p">,</span> <span class="n">bias</span><span class="o">=</span><span class="kc">False</span><span class="p">))</span>
-
-    <span class="bp">self</span><span class="o">.</span><span class="n">decoder</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">ModuleList</span><span class="p">()</span>
-    <span class="k">for</span> <span class="n">step</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">n_steps</span><span class="p">):</span>
-        <span class="n">transformer</span> <span class="o">=</span> <span class="n">FeatTransformer</span><span class="p">(</span>
-            <span class="n">step_dim</span><span class="p">,</span>
-            <span class="n">step_dim</span><span class="p">,</span>
-            <span class="n">dropout</span><span class="p">,</span>
-            <span class="n">shared_layers</span><span class="p">,</span>
-            <span class="n">n_glu_step_dependent</span><span class="p">,</span>
-            <span class="n">ghost_bn</span><span class="p">,</span>
-            <span class="n">virtual_batch_size</span><span class="p">,</span>
-            <span class="n">momentum</span><span class="o">=</span><span class="n">momentum</span><span class="p">,</span>
-        <span class="p">)</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">decoder</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">transformer</span><span class="p">)</span>
-
-    <span class="bp">self</span><span class="o">.</span><span class="n">reconstruction_layer</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Linear</span><span class="p">(</span><span class="n">step_dim</span><span class="p">,</span> <span class="n">embed_dim</span><span class="p">,</span> <span class="n">bias</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
-    <span class="n">initialize_non_glu</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">reconstruction_layer</span><span class="p">,</span> <span class="n">step_dim</span><span class="p">,</span> <span class="n">embed_dim</span><span class="p">)</span>
+              <details class="quote">
+                <summary>Source code in <code>pytorch_widedeep/models/tabular/tabnet/tab_net.py</code></summary>
+                <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">280</span>
+<span class="normal">281</span>
+<span class="normal">282</span>
+<span class="normal">283</span>
+<span class="normal">284</span>
+<span class="normal">285</span>
+<span class="normal">286</span>
+<span class="normal">287</span>
+<span class="normal">288</span>
+<span class="normal">289</span>
+<span class="normal">290</span>
+<span class="normal">291</span>
+<span class="normal">292</span>
+<span class="normal">293</span>
+<span class="normal">294</span>
+<span class="normal">295</span>
+<span class="normal">296</span>
+<span class="normal">297</span>
+<span class="normal">298</span>
+<span class="normal">299</span>
+<span class="normal">300</span>
+<span class="normal">301</span>
+<span class="normal">302</span>
+<span class="normal">303</span>
+<span class="normal">304</span>
+<span class="normal">305</span>
+<span class="normal">306</span>
+<span class="normal">307</span>
+<span class="normal">308</span>
+<span class="normal">309</span>
+<span class="normal">310</span>
+<span class="normal">311</span>
+<span class="normal">312</span>
+<span class="normal">313</span>
+<span class="normal">314</span>
+<span class="normal">315</span>
+<span class="normal">316</span>
+<span class="normal">317</span>
+<span class="normal">318</span>
+<span class="normal">319</span>
+<span class="normal">320</span>
+<span class="normal">321</span>
+<span class="normal">322</span>
+<span class="normal">323</span>
+<span class="normal">324</span>
+<span class="normal">325</span>
+<span class="normal">326</span>
+<span class="normal">327</span>
+<span class="normal">328</span>
+<span class="normal">329</span>
+<span class="normal">330</span>
+<span class="normal">331</span>
+<span class="normal">332</span>
+<span class="normal">333</span>
+<span class="normal">334</span>
+<span class="normal">335</span>
+<span class="normal">336</span>
+<span class="normal">337</span>
+<span class="normal">338</span>
+<span class="normal">339</span>
+<span class="normal">340</span>
+<span class="normal">341</span>
+<span class="normal">342</span>
+<span class="normal">343</span>
+<span class="normal">344</span>
+<span class="normal">345</span>
+<span class="normal">346</span>
+<span class="normal">347</span>
+<span class="normal">348</span>
+<span class="normal">349</span>
+<span class="normal">350</span>
+<span class="normal">351</span>
+<span class="normal">352</span>
+<span class="normal">353</span>
+<span class="normal">354</span>
+<span class="normal">355</span>
+<span class="normal">356</span>
+<span class="normal">357</span>
+<span class="normal">358</span>
+<span class="normal">359</span>
+<span class="normal">360</span>
+<span class="normal">361</span>
+<span class="normal">362</span>
+<span class="normal">363</span>
+<span class="normal">364</span>
+<span class="normal">365</span>
+<span class="normal">366</span>
+<span class="normal">367</span>
+<span class="normal">368</span>
+<span class="normal">369</span>
+<span class="normal">370</span>
+<span class="normal">371</span>
+<span class="normal">372</span>
+<span class="normal">373</span>
+<span class="normal">374</span>
+<span class="normal">375</span>
+<span class="normal">376</span>
+<span class="normal">377</span>
+<span class="normal">378</span>
+<span class="normal">379</span>
+<span class="normal">380</span>
+<span class="normal">381</span>
+<span class="normal">382</span>
+<span class="normal">383</span>
+<span class="normal">384</span>
+<span class="normal">385</span>
+<span class="normal">386</span>
+<span class="normal">387</span>
+<span class="normal">388</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">class</span> <span class="nc">TabNetDecoder</span><span class="p">(</span><span class="n">nn</span><span class="o">.</span><span class="n">Module</span><span class="p">):</span>
+<span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Companion decoder model for the `TabNet` model (which can be</span>
+<span class="sd">    considered an encoder itself)</span>
+
+<span class="sd">    This class is designed to be used with the `EncoderDecoderTrainer` when</span>
+<span class="sd">    using self-supervised pre-training (see the corresponding section in the</span>
+<span class="sd">    docs). This class will receive the output from the `TabNet` encoder</span>
+<span class="sd">    (i.e. the output from the so called &#39;steps&#39;) and &#39;_reconstruct_&#39; the</span>
+<span class="sd">    embeddings.</span>
+
+<span class="sd">    Parameters</span>
+<span class="sd">    ----------</span>
+<span class="sd">    embed_dim: int</span>
+<span class="sd">        Size of the embeddings tensor to be reconstructed.</span>
+<span class="sd">    n_steps: int, default = 3</span>
+<span class="sd">        number of decision steps. For a better understanding of the function</span>
+<span class="sd">        of `n_steps` and the upcoming parameters, please see the</span>
+<span class="sd">        [paper](https://arxiv.org/abs/1908.07442).</span>
+<span class="sd">    step_dim: int, default = 8</span>
+<span class="sd">        Step&#39;s output dimension. This is the output dimension that</span>
+<span class="sd">        `WideDeep` will collect and connect to the output neuron(s).</span>
+<span class="sd">    dropout: float, default = 0.0</span>
+<span class="sd">        GLU block&#39;s internal dropout</span>
+<span class="sd">    n_glu_step_dependent: int, default = 2</span>
+<span class="sd">        number of GLU Blocks (`[FC -&gt; BN -&gt; GLU]`) that are step dependent</span>
+<span class="sd">    n_glu_shared: int, default = 2</span>
+<span class="sd">        number of GLU Blocks (`[FC -&gt; BN -&gt; GLU]`) that will be shared</span>
+<span class="sd">        across decision steps</span>
+<span class="sd">    ghost_bn: bool, default=True</span>
+<span class="sd">        Boolean indicating if [Ghost Batch Normalization](https://arxiv.org/abs/1705.08741)</span>
+<span class="sd">        will be used.</span>
+<span class="sd">    virtual_batch_size: int, default = 128</span>
+<span class="sd">        Batch size when using Ghost Batch Normalization</span>
+<span class="sd">    momentum: float, default = 0.02</span>
+<span class="sd">        Ghost Batch Normalization&#39;s momentum. The dreamquark-ai advises for</span>
+<span class="sd">        very low values. However high values are used in the original</span>
+<span class="sd">        publication. During our tests higher values lead to better results</span>
+
+<span class="sd">    Attributes</span>
+<span class="sd">    ----------</span>
+<span class="sd">    decoder: nn.Module</span>
+<span class="sd">        decoder that will receive the output from the encoder&#39;s steps and will</span>
+<span class="sd">        reconstruct the embeddings</span>
+
+<span class="sd">    Examples</span>
+<span class="sd">    --------</span>
+<span class="sd">    &gt;&gt;&gt; import torch</span>
+<span class="sd">    &gt;&gt;&gt; from pytorch_widedeep.models import TabNetDecoder</span>
+<span class="sd">    &gt;&gt;&gt; x_inp = [torch.rand(3, 8), torch.rand(3, 8), torch.rand(3, 8)]</span>
+<span class="sd">    &gt;&gt;&gt; decoder = TabNetDecoder(embed_dim=32, ghost_bn=False)</span>
+<span class="sd">    &gt;&gt;&gt; res = decoder(x_inp)</span>
+<span class="sd">    &gt;&gt;&gt; res.shape</span>
+<span class="sd">    torch.Size([3, 32])</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+
+    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span>
+        <span class="n">embed_dim</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span>
+        <span class="n">n_steps</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">3</span><span class="p">,</span>
+        <span class="n">step_dim</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">8</span><span class="p">,</span>
+        <span class="n">dropout</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.0</span><span class="p">,</span>
+        <span class="n">n_glu_step_dependent</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">2</span><span class="p">,</span>
+        <span class="n">n_glu_shared</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">2</span><span class="p">,</span>
+        <span class="n">ghost_bn</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">,</span>
+        <span class="n">virtual_batch_size</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">128</span><span class="p">,</span>
+        <span class="n">momentum</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.02</span><span class="p">,</span>
+    <span class="p">):</span>
+        <span class="nb">super</span><span class="p">(</span><span class="n">TabNetDecoder</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">n_steps</span> <span class="o">=</span> <span class="n">n_steps</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">step_dim</span> <span class="o">=</span> <span class="n">step_dim</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">dropout</span> <span class="o">=</span> <span class="n">dropout</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">n_glu_step_dependent</span> <span class="o">=</span> <span class="n">n_glu_step_dependent</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">n_glu_shared</span> <span class="o">=</span> <span class="n">n_glu_shared</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">ghost_bn</span> <span class="o">=</span> <span class="n">ghost_bn</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">virtual_batch_size</span> <span class="o">=</span> <span class="n">virtual_batch_size</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">momentum</span> <span class="o">=</span> <span class="n">momentum</span>
+
+        <span class="n">shared_layers</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">ModuleList</span><span class="p">()</span>
+        <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">n_glu_shared</span><span class="p">):</span>
+            <span class="k">if</span> <span class="n">i</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
+                <span class="n">shared_layers</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">nn</span><span class="o">.</span><span class="n">Linear</span><span class="p">(</span><span class="n">step_dim</span><span class="p">,</span> <span class="mi">2</span> <span class="o">*</span> <span class="n">step_dim</span><span class="p">,</span> <span class="n">bias</span><span class="o">=</span><span class="kc">False</span><span class="p">))</span>
+            <span class="k">else</span><span class="p">:</span>
+                <span class="n">shared_layers</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">nn</span><span class="o">.</span><span class="n">Linear</span><span class="p">(</span><span class="n">step_dim</span><span class="p">,</span> <span class="mi">2</span> <span class="o">*</span> <span class="n">step_dim</span><span class="p">,</span> <span class="n">bias</span><span class="o">=</span><span class="kc">False</span><span class="p">))</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">decoder</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">ModuleList</span><span class="p">()</span>
+        <span class="k">for</span> <span class="n">step</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">n_steps</span><span class="p">):</span>
+            <span class="n">transformer</span> <span class="o">=</span> <span class="n">FeatTransformer</span><span class="p">(</span>
+                <span class="n">step_dim</span><span class="p">,</span>
+                <span class="n">step_dim</span><span class="p">,</span>
+                <span class="n">dropout</span><span class="p">,</span>
+                <span class="n">shared_layers</span><span class="p">,</span>
+                <span class="n">n_glu_step_dependent</span><span class="p">,</span>
+                <span class="n">ghost_bn</span><span class="p">,</span>
+                <span class="n">virtual_batch_size</span><span class="p">,</span>
+                <span class="n">momentum</span><span class="o">=</span><span class="n">momentum</span><span class="p">,</span>
+            <span class="p">)</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">decoder</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">transformer</span><span class="p">)</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">reconstruction_layer</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Linear</span><span class="p">(</span><span class="n">step_dim</span><span class="p">,</span> <span class="n">embed_dim</span><span class="p">,</span> <span class="n">bias</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
+        <span class="n">initialize_non_glu</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">reconstruction_layer</span><span class="p">,</span> <span class="n">step_dim</span><span class="p">,</span> <span class="n">embed_dim</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">forward</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">X</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="n">Tensor</span><span class="p">])</span> <span class="o">-&gt;</span> <span class="n">Tensor</span><span class="p">:</span>
+        <span class="n">out</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">tensor</span><span class="p">(</span><span class="mf">0.0</span><span class="p">)</span>
+        <span class="k">for</span> <span class="n">i</span><span class="p">,</span> <span class="n">x</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="n">X</span><span class="p">):</span>
+            <span class="n">x</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">decoder</span><span class="p">[</span><span class="n">i</span><span class="p">](</span><span class="n">x</span><span class="p">)</span>
+            <span class="n">out</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">out</span><span class="p">,</span> <span class="n">x</span><span class="p">)</span>
+        <span class="n">out</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">reconstruction_layer</span><span class="p">(</span><span class="n">out</span><span class="p">)</span>
+        <span class="k">return</span> <span class="n">out</span>
 </code></pre></div></td></tr></table></div>
-                  </details>
+              </details>
 
 
 
@@ -4632,33 +6526,7 @@ <h2 id="pytorch_widedeep.models.tabular.mlp.context_attention_mlp.ContextAttenti
 
 
 <a href="#pytorch_widedeep.models.tabular.mlp.context_attention_mlp.ContextAttentionMLP" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="nf">ContextAttentionMLP</span><span class="p">(</span>
-    <span class="n">column_idx</span><span class="p">,</span>
-    <span class="o">*</span><span class="p">,</span>
-    <span class="n">cat_embed_input</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">cat_embed_dropout</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">use_cat_bias</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">cat_embed_activation</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">shared_embed</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">add_shared_embed</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">frac_shared_embed</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">continuous_cols</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">cont_norm_layer</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">embed_continuous_method</span><span class="o">=</span><span class="s2">&quot;standard&quot;</span><span class="p">,</span>
-    <span class="n">cont_embed_dropout</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">cont_embed_activation</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">quantization_setup</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">n_frequencies</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">sigma</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">share_last_layer</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">full_embed_dropout</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">input_dim</span><span class="o">=</span><span class="mi">32</span><span class="p">,</span>
-    <span class="n">attn_dropout</span><span class="o">=</span><span class="mf">0.2</span><span class="p">,</span>
-    <span class="n">with_addnorm</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
-    <span class="n">attn_activation</span><span class="o">=</span><span class="s2">&quot;leaky_relu&quot;</span><span class="p">,</span>
-    <span class="n">n_blocks</span><span class="o">=</span><span class="mi">3</span>
-<span class="p">)</span>
-</code></pre></div>
+
 
     <div class="doc doc-contents first">
             <p class="doc doc-class-bases">
@@ -4682,132 +6550,183 @@ <h2 id="pytorch_widedeep.models.tabular.mlp.context_attention_mlp.ContextAttenti
 
 
 <p><span class="doc-section-title">Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>column_idx</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, int]</code>)
-          –
-          <div class="doc-md-description">
-            <p>Dict containing the index of the columns that will be passed through
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+          <th>Default</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>column_idx</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, int]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Dict containing the index of the columns that will be passed through
 the <code>TabMlp</code> model. Required to slice the tensors. e.g. <em>{'education':
 0, 'relationship': 1, 'workclass': 2, ...}</em>.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>cat_embed_input</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.wdtypes.Tuple">Tuple</span>[str, int]]]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>List of Tuples with the column name and number of unique values and
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>cat_embed_input</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.wdtypes.Tuple">Tuple</span>[str, int]]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>List of Tuples with the column name and number of unique values and
 embedding dimension. e.g. <em>[(education, 11), ...]</em></p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>cat_embed_dropout</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Categorical embeddings dropout. If <code>None</code>, it will default
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>cat_embed_dropout</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Categorical embeddings dropout. If <code>None</code>, it will default
 to 0.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>use_cat_bias</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Boolean indicating if bias will be used for the categorical embeddings.
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>use_cat_bias</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Boolean indicating if bias will be used for the categorical embeddings.
 If <code>None</code>, it will default to 'False'.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>cat_embed_activation</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Activation function for the categorical embeddings, if any. Currently
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>cat_embed_activation</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Activation function for the categorical embeddings, if any. Currently
 <em>'tanh'</em>, <em>'relu'</em>, <em>'leaky_relu'</em> and <em>'gelu'</em> are supported</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>shared_embed</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Boolean indicating if the embeddings will be "shared". The idea behind <code>shared_embed</code> is
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>shared_embed</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Boolean indicating if the embeddings will be "shared". The idea behind <code>shared_embed</code> is
 described in the Appendix A in the <a href="https://arxiv.org/abs/2012.06678">TabTransformer paper</a>:
 <em>'The goal of having column embedding is to enable the model to
 distinguish the classes in one column from those in the other
 columns'</em>. In other words, the idea is to let the model learn which
 column is embedded at the time. See: <code>pytorch_widedeep.models.transformers._layers.SharedEmbeddings</code>.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>add_shared_embed</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>The two embedding sharing strategies are: 1) add the shared embeddings
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>add_shared_embed</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>The two embedding sharing strategies are: 1) add the shared embeddings
 to the column embeddings or 2) to replace the first
 <code>frac_shared_embed</code> with the shared embeddings.
 See <code>pytorch_widedeep.models.embeddings_layers.SharedEmbeddings</code>
 If 'None' is passed, it will default to 'False'.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>frac_shared_embed</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>The fraction of embeddings that will be shared (if <code>add_shared_embed
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>frac_shared_embed</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>The fraction of embeddings that will be shared (if <code>add_shared_embed
 = False</code>) by all the different categories for one particular
 column. If 'None' is passed, it will default to 0.0.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>continuous_cols</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[str]]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>List with the name of the numeric (aka continuous) columns</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>cont_norm_layer</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[batchnorm, layernorm]]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Type of normalization layer applied to the continuous features.
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>continuous_cols</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[str]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>List with the name of the numeric (aka continuous) columns</p>
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>cont_norm_layer</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[batchnorm, layernorm]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Type of normalization layer applied to the continuous features.
 Options are: <em>'layernorm'</em> and <em>'batchnorm'</em>. if <code>None</code>, no
 normalization layer will be used.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>embed_continuous_method</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[standard, piecewise, periodic]]</code>, default:
-                  <code>&#39;standard&#39;</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Method to use to embed the continuous features. Options are:
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>embed_continuous_method</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[standard, piecewise, periodic]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Method to use to embed the continuous features. Options are:
 <em>'standard'</em>, <em>'periodic'</em> or <em>'piecewise'</em>. The <em>'standard'</em>
 embedding method is based on the FT-Transformer implementation
 presented in the paper: <a href="https://arxiv.org/abs/2106.11959v5">Revisiting Deep Learning Models for
@@ -4815,165 +6734,230 @@ <h2 id="pytorch_widedeep.models.tabular.mlp.context_attention_mlp.ContextAttenti
 and_'piecewise'_ methods were presented in the paper: <a href="https://arxiv.org/abs/2203.05556">On Embeddings for
 Numerical Features in Tabular Deep Learning</a>.
 Please, read the papers for details.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>cont_embed_dropout</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>cont_embed_activation</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Activation function for the continuous embeddings if any. Currently
+              </div>
+            </td>
+            <td>
+                  <code>&#39;standard&#39;</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>cont_embed_dropout</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p>
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>cont_embed_activation</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Activation function for the continuous embeddings if any. Currently
 <em>'tanh'</em>, <em>'relu'</em>, <em>'leaky_relu'</em> and <em>'gelu'</em> are supported.
 If <code>None</code>, no activation function will be applied.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>quantization_setup</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, <span title="pytorch_widedeep.wdtypes.List">List</span>[float]]]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>This parameter is used when the <em>'piecewise'</em> method is used to embed
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>quantization_setup</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, <span title="pytorch_widedeep.wdtypes.List">List</span>[float]]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>This parameter is used when the <em>'piecewise'</em> method is used to embed
 the continuous cols. It is a dict where keys are the name of the continuous
 columns and values are lists with the boundaries for the quantization
 of the continuous_cols. See the examples for details. If
 If the <em>'piecewise'</em> method is used, this parameter is required.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>n_frequencies</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[int]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>This is the so called <em>'k'</em> in their paper <a href="https://arxiv.org/abs/2203.05556">On Embeddings for
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>n_frequencies</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[int]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>This is the so called <em>'k'</em> in their paper <a href="https://arxiv.org/abs/2203.05556">On Embeddings for
 Numerical Features in Tabular Deep Learning</a>,
 and is the number of 'frequencies' that will be used to represent each
 continuous column. See their Eq 2 in the paper for details. If
 the <em>'periodic'</em> method is used, this parameter is required.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>sigma</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>This is the sigma parameter in the paper mentioned when describing the
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>sigma</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>This is the sigma parameter in the paper mentioned when describing the
 previous parameters and it is used to initialise the 'frequency
 weights'. See their Eq 2 in the paper for details. If
 the <em>'periodic'</em> method is used, this parameter is required.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>share_last_layer</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>This parameter is not present in the before mentioned paper but it is implemented in
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>share_last_layer</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>This parameter is not present in the before mentioned paper but it is implemented in
 the <a href="https://github.com/yandex-research/rtdl-num-embeddings/tree/main">official repo</a>.
 If <code>True</code> the linear layer that turns the frequencies into embeddings
 will be shared across the continuous columns. If <code>False</code> a different
 linear layer will be used for each continuous column.
 If the <em>'periodic'</em> method is used, this parameter is required.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>full_embed_dropout</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>If <code>True</code>, the full embedding corresponding to a column will be masked
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>full_embed_dropout</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>If <code>True</code>, the full embedding corresponding to a column will be masked
 out/dropout. If <code>None</code>, it will default to <code>False</code>.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>input_dim</code></b>
-              (<code>int</code>, default:
-                  <code>32</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>The so-called <em>dimension of the model</em>. Is the number of embeddings
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>input_dim</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>The so-called <em>dimension of the model</em>. Is the number of embeddings
 used to encode the categorical and/or continuous columns</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>attn_dropout</code></b>
-              (<code>float</code>, default:
+              </div>
+            </td>
+            <td>
+                  <code>32</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>attn_dropout</code></td>
+            <td>
+                  <code>float</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Dropout for each attention block</p>
+              </div>
+            </td>
+            <td>
                   <code>0.2</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Dropout for each attention block</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>with_addnorm</code></b>
-              (<code>bool</code>, default:
-                  <code>False</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Boolean indicating if residual connections will be used in the
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>with_addnorm</code></td>
+            <td>
+                  <code>bool</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Boolean indicating if residual connections will be used in the
 attention blocks</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>attn_activation</code></b>
-              (<code>str</code>, default:
-                  <code>&#39;leaky_relu&#39;</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>String indicating the activation function to be applied to the dense
+              </div>
+            </td>
+            <td>
+                  <code>False</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>attn_activation</code></td>
+            <td>
+                  <code>str</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>String indicating the activation function to be applied to the dense
 layer in each attention encoder. <em>'tanh'</em>, <em>'relu'</em>, <em>'leaky_relu'</em>
 and <em>'gelu'</em> are supported.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>n_blocks</code></b>
-              (<code>int</code>, default:
+              </div>
+            </td>
+            <td>
+                  <code>&#39;leaky_relu&#39;</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>n_blocks</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Number of attention blocks</p>
+              </div>
+            </td>
+            <td>
                   <code>3</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Number of attention blocks</p>
-          </div>
-        </li>
-    </ul>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
 
 <p><span class="doc-section-title">Attributes:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code><span title="pytorch_widedeep.models.tabular.mlp.context_attention_mlp.ContextAttentionMLP.encoder">encoder</span></code></b>
-              (<code><span title="torch.nn.Module">Module</span></code>)
-          –
-          <div class="doc-md-description">
-            <p>Sequence of attention encoders.</p>
-          </div>
-        </li>
-    </ul>
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code><span title="pytorch_widedeep.models.tabular.mlp.context_attention_mlp.ContextAttentionMLP.encoder">encoder</span></code></td>
+            <td>
+                  <code><span title="torch.nn.Module">Module</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Sequence of attention encoders.</p>
+              </div>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
 
 <p><span class="doc-section-title">Examples:</span></p>
@@ -4987,155 +6971,479 @@ <h2 id="pytorch_widedeep.models.tabular.mlp.context_attention_mlp.ContextAttenti
 <span class="gp">&gt;&gt;&gt; </span><span class="n">out</span> <span class="o">=</span> <span class="n">model</span><span class="p">(</span><span class="n">X_tab</span><span class="p">)</span>
 </code></pre></div>
 
-                  <details class="quote">
-                    <summary>Source code in <code>pytorch_widedeep/models/tabular/mlp/context_attention_mlp.py</code></summary>
-                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">143</span>
-<span class="normal">144</span>
-<span class="normal">145</span>
-<span class="normal">146</span>
-<span class="normal">147</span>
-<span class="normal">148</span>
-<span class="normal">149</span>
-<span class="normal">150</span>
-<span class="normal">151</span>
-<span class="normal">152</span>
-<span class="normal">153</span>
-<span class="normal">154</span>
-<span class="normal">155</span>
-<span class="normal">156</span>
-<span class="normal">157</span>
-<span class="normal">158</span>
-<span class="normal">159</span>
-<span class="normal">160</span>
-<span class="normal">161</span>
-<span class="normal">162</span>
-<span class="normal">163</span>
-<span class="normal">164</span>
-<span class="normal">165</span>
-<span class="normal">166</span>
-<span class="normal">167</span>
-<span class="normal">168</span>
-<span class="normal">169</span>
-<span class="normal">170</span>
-<span class="normal">171</span>
-<span class="normal">172</span>
-<span class="normal">173</span>
-<span class="normal">174</span>
-<span class="normal">175</span>
-<span class="normal">176</span>
-<span class="normal">177</span>
-<span class="normal">178</span>
-<span class="normal">179</span>
-<span class="normal">180</span>
-<span class="normal">181</span>
-<span class="normal">182</span>
-<span class="normal">183</span>
-<span class="normal">184</span>
-<span class="normal">185</span>
-<span class="normal">186</span>
-<span class="normal">187</span>
-<span class="normal">188</span>
-<span class="normal">189</span>
-<span class="normal">190</span>
-<span class="normal">191</span>
-<span class="normal">192</span>
-<span class="normal">193</span>
-<span class="normal">194</span>
-<span class="normal">195</span>
-<span class="normal">196</span>
-<span class="normal">197</span>
-<span class="normal">198</span>
-<span class="normal">199</span>
-<span class="normal">200</span>
-<span class="normal">201</span>
-<span class="normal">202</span>
-<span class="normal">203</span>
-<span class="normal">204</span>
-<span class="normal">205</span>
-<span class="normal">206</span>
-<span class="normal">207</span>
-<span class="normal">208</span>
-<span class="normal">209</span>
+              <details class="quote">
+                <summary>Source code in <code>pytorch_widedeep/models/tabular/mlp/context_attention_mlp.py</code></summary>
+                <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 10</span>
+<span class="normal"> 11</span>
+<span class="normal"> 12</span>
+<span class="normal"> 13</span>
+<span class="normal"> 14</span>
+<span class="normal"> 15</span>
+<span class="normal"> 16</span>
+<span class="normal"> 17</span>
+<span class="normal"> 18</span>
+<span class="normal"> 19</span>
+<span class="normal"> 20</span>
+<span class="normal"> 21</span>
+<span class="normal"> 22</span>
+<span class="normal"> 23</span>
+<span class="normal"> 24</span>
+<span class="normal"> 25</span>
+<span class="normal"> 26</span>
+<span class="normal"> 27</span>
+<span class="normal"> 28</span>
+<span class="normal"> 29</span>
+<span class="normal"> 30</span>
+<span class="normal"> 31</span>
+<span class="normal"> 32</span>
+<span class="normal"> 33</span>
+<span class="normal"> 34</span>
+<span class="normal"> 35</span>
+<span class="normal"> 36</span>
+<span class="normal"> 37</span>
+<span class="normal"> 38</span>
+<span class="normal"> 39</span>
+<span class="normal"> 40</span>
+<span class="normal"> 41</span>
+<span class="normal"> 42</span>
+<span class="normal"> 43</span>
+<span class="normal"> 44</span>
+<span class="normal"> 45</span>
+<span class="normal"> 46</span>
+<span class="normal"> 47</span>
+<span class="normal"> 48</span>
+<span class="normal"> 49</span>
+<span class="normal"> 50</span>
+<span class="normal"> 51</span>
+<span class="normal"> 52</span>
+<span class="normal"> 53</span>
+<span class="normal"> 54</span>
+<span class="normal"> 55</span>
+<span class="normal"> 56</span>
+<span class="normal"> 57</span>
+<span class="normal"> 58</span>
+<span class="normal"> 59</span>
+<span class="normal"> 60</span>
+<span class="normal"> 61</span>
+<span class="normal"> 62</span>
+<span class="normal"> 63</span>
+<span class="normal"> 64</span>
+<span class="normal"> 65</span>
+<span class="normal"> 66</span>
+<span class="normal"> 67</span>
+<span class="normal"> 68</span>
+<span class="normal"> 69</span>
+<span class="normal"> 70</span>
+<span class="normal"> 71</span>
+<span class="normal"> 72</span>
+<span class="normal"> 73</span>
+<span class="normal"> 74</span>
+<span class="normal"> 75</span>
+<span class="normal"> 76</span>
+<span class="normal"> 77</span>
+<span class="normal"> 78</span>
+<span class="normal"> 79</span>
+<span class="normal"> 80</span>
+<span class="normal"> 81</span>
+<span class="normal"> 82</span>
+<span class="normal"> 83</span>
+<span class="normal"> 84</span>
+<span class="normal"> 85</span>
+<span class="normal"> 86</span>
+<span class="normal"> 87</span>
+<span class="normal"> 88</span>
+<span class="normal"> 89</span>
+<span class="normal"> 90</span>
+<span class="normal"> 91</span>
+<span class="normal"> 92</span>
+<span class="normal"> 93</span>
+<span class="normal"> 94</span>
+<span class="normal"> 95</span>
+<span class="normal"> 96</span>
+<span class="normal"> 97</span>
+<span class="normal"> 98</span>
+<span class="normal"> 99</span>
+<span class="normal">100</span>
+<span class="normal">101</span>
+<span class="normal">102</span>
+<span class="normal">103</span>
+<span class="normal">104</span>
+<span class="normal">105</span>
+<span class="normal">106</span>
+<span class="normal">107</span>
+<span class="normal">108</span>
+<span class="normal">109</span>
+<span class="normal">110</span>
+<span class="normal">111</span>
+<span class="normal">112</span>
+<span class="normal">113</span>
+<span class="normal">114</span>
+<span class="normal">115</span>
+<span class="normal">116</span>
+<span class="normal">117</span>
+<span class="normal">118</span>
+<span class="normal">119</span>
+<span class="normal">120</span>
+<span class="normal">121</span>
+<span class="normal">122</span>
+<span class="normal">123</span>
+<span class="normal">124</span>
+<span class="normal">125</span>
+<span class="normal">126</span>
+<span class="normal">127</span>
+<span class="normal">128</span>
+<span class="normal">129</span>
+<span class="normal">130</span>
+<span class="normal">131</span>
+<span class="normal">132</span>
+<span class="normal">133</span>
+<span class="normal">134</span>
+<span class="normal">135</span>
+<span class="normal">136</span>
+<span class="normal">137</span>
+<span class="normal">138</span>
+<span class="normal">139</span>
+<span class="normal">140</span>
+<span class="normal">141</span>
+<span class="normal">142</span>
+<span class="normal">143</span>
+<span class="normal">144</span>
+<span class="normal">145</span>
+<span class="normal">146</span>
+<span class="normal">147</span>
+<span class="normal">148</span>
+<span class="normal">149</span>
+<span class="normal">150</span>
+<span class="normal">151</span>
+<span class="normal">152</span>
+<span class="normal">153</span>
+<span class="normal">154</span>
+<span class="normal">155</span>
+<span class="normal">156</span>
+<span class="normal">157</span>
+<span class="normal">158</span>
+<span class="normal">159</span>
+<span class="normal">160</span>
+<span class="normal">161</span>
+<span class="normal">162</span>
+<span class="normal">163</span>
+<span class="normal">164</span>
+<span class="normal">165</span>
+<span class="normal">166</span>
+<span class="normal">167</span>
+<span class="normal">168</span>
+<span class="normal">169</span>
+<span class="normal">170</span>
+<span class="normal">171</span>
+<span class="normal">172</span>
+<span class="normal">173</span>
+<span class="normal">174</span>
+<span class="normal">175</span>
+<span class="normal">176</span>
+<span class="normal">177</span>
+<span class="normal">178</span>
+<span class="normal">179</span>
+<span class="normal">180</span>
+<span class="normal">181</span>
+<span class="normal">182</span>
+<span class="normal">183</span>
+<span class="normal">184</span>
+<span class="normal">185</span>
+<span class="normal">186</span>
+<span class="normal">187</span>
+<span class="normal">188</span>
+<span class="normal">189</span>
+<span class="normal">190</span>
+<span class="normal">191</span>
+<span class="normal">192</span>
+<span class="normal">193</span>
+<span class="normal">194</span>
+<span class="normal">195</span>
+<span class="normal">196</span>
+<span class="normal">197</span>
+<span class="normal">198</span>
+<span class="normal">199</span>
+<span class="normal">200</span>
+<span class="normal">201</span>
+<span class="normal">202</span>
+<span class="normal">203</span>
+<span class="normal">204</span>
+<span class="normal">205</span>
+<span class="normal">206</span>
+<span class="normal">207</span>
+<span class="normal">208</span>
+<span class="normal">209</span>
 <span class="normal">210</span>
 <span class="normal">211</span>
 <span class="normal">212</span>
 <span class="normal">213</span>
 <span class="normal">214</span>
-<span class="normal">215</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
-    <span class="bp">self</span><span class="p">,</span>
-    <span class="n">column_idx</span><span class="p">:</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">],</span>
-    <span class="o">*</span><span class="p">,</span>
-    <span class="n">cat_embed_input</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="n">Tuple</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">]]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">cat_embed_dropout</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">use_cat_bias</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">cat_embed_activation</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">shared_embed</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">add_shared_embed</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">frac_shared_embed</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">continuous_cols</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">cont_norm_layer</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Literal</span><span class="p">[</span><span class="s2">&quot;batchnorm&quot;</span><span class="p">,</span> <span class="s2">&quot;layernorm&quot;</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">embed_continuous_method</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span>
-        <span class="n">Literal</span><span class="p">[</span><span class="s2">&quot;standard&quot;</span><span class="p">,</span> <span class="s2">&quot;piecewise&quot;</span><span class="p">,</span> <span class="s2">&quot;periodic&quot;</span><span class="p">]</span>
-    <span class="p">]</span> <span class="o">=</span> <span class="s2">&quot;standard&quot;</span><span class="p">,</span>
-    <span class="n">cont_embed_dropout</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">cont_embed_activation</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">quantization_setup</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="nb">float</span><span class="p">]]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">n_frequencies</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">sigma</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">share_last_layer</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">full_embed_dropout</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">input_dim</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">32</span><span class="p">,</span>
-    <span class="n">attn_dropout</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.2</span><span class="p">,</span>
-    <span class="n">with_addnorm</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
-    <span class="n">attn_activation</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;leaky_relu&quot;</span><span class="p">,</span>
-    <span class="n">n_blocks</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">3</span><span class="p">,</span>
-<span class="p">):</span>
-    <span class="nb">super</span><span class="p">(</span><span class="n">ContextAttentionMLP</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span>
-        <span class="n">column_idx</span><span class="o">=</span><span class="n">column_idx</span><span class="p">,</span>
-        <span class="n">cat_embed_input</span><span class="o">=</span><span class="n">cat_embed_input</span><span class="p">,</span>
-        <span class="n">cat_embed_dropout</span><span class="o">=</span><span class="n">cat_embed_dropout</span><span class="p">,</span>
-        <span class="n">use_cat_bias</span><span class="o">=</span><span class="n">use_cat_bias</span><span class="p">,</span>
-        <span class="n">cat_embed_activation</span><span class="o">=</span><span class="n">cat_embed_activation</span><span class="p">,</span>
-        <span class="n">shared_embed</span><span class="o">=</span><span class="n">shared_embed</span><span class="p">,</span>
-        <span class="n">add_shared_embed</span><span class="o">=</span><span class="n">add_shared_embed</span><span class="p">,</span>
-        <span class="n">frac_shared_embed</span><span class="o">=</span><span class="n">frac_shared_embed</span><span class="p">,</span>
-        <span class="n">continuous_cols</span><span class="o">=</span><span class="n">continuous_cols</span><span class="p">,</span>
-        <span class="n">cont_norm_layer</span><span class="o">=</span><span class="n">cont_norm_layer</span><span class="p">,</span>
-        <span class="n">embed_continuous_method</span><span class="o">=</span><span class="n">embed_continuous_method</span><span class="p">,</span>
-        <span class="n">cont_embed_dropout</span><span class="o">=</span><span class="n">cont_embed_dropout</span><span class="p">,</span>
-        <span class="n">cont_embed_activation</span><span class="o">=</span><span class="n">cont_embed_activation</span><span class="p">,</span>
-        <span class="n">input_dim</span><span class="o">=</span><span class="n">input_dim</span><span class="p">,</span>
-        <span class="n">quantization_setup</span><span class="o">=</span><span class="n">quantization_setup</span><span class="p">,</span>
-        <span class="n">n_frequencies</span><span class="o">=</span><span class="n">n_frequencies</span><span class="p">,</span>
-        <span class="n">sigma</span><span class="o">=</span><span class="n">sigma</span><span class="p">,</span>
-        <span class="n">share_last_layer</span><span class="o">=</span><span class="n">share_last_layer</span><span class="p">,</span>
-        <span class="n">full_embed_dropout</span><span class="o">=</span><span class="n">full_embed_dropout</span><span class="p">,</span>
-    <span class="p">)</span>
+<span class="normal">215</span>
+<span class="normal">216</span>
+<span class="normal">217</span>
+<span class="normal">218</span>
+<span class="normal">219</span>
+<span class="normal">220</span>
+<span class="normal">221</span>
+<span class="normal">222</span>
+<span class="normal">223</span>
+<span class="normal">224</span>
+<span class="normal">225</span>
+<span class="normal">226</span>
+<span class="normal">227</span>
+<span class="normal">228</span>
+<span class="normal">229</span>
+<span class="normal">230</span>
+<span class="normal">231</span>
+<span class="normal">232</span>
+<span class="normal">233</span>
+<span class="normal">234</span>
+<span class="normal">235</span>
+<span class="normal">236</span>
+<span class="normal">237</span>
+<span class="normal">238</span>
+<span class="normal">239</span>
+<span class="normal">240</span>
+<span class="normal">241</span>
+<span class="normal">242</span>
+<span class="normal">243</span>
+<span class="normal">244</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">class</span> <span class="nc">ContextAttentionMLP</span><span class="p">(</span><span class="n">BaseTabularModelWithAttention</span><span class="p">):</span>
+<span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Defines a `ContextAttentionMLP` model that can be used as the</span>
+<span class="sd">    `deeptabular` component of a Wide &amp; Deep model or independently by</span>
+<span class="sd">    itself.</span>
+
+<span class="sd">    This class combines embedding representations of the categorical features</span>
+<span class="sd">    with numerical (aka continuous) features that are also embedded. These</span>
+<span class="sd">    are then passed through a series of attention blocks. Each attention</span>
+<span class="sd">    block is comprised by a `ContextAttentionEncoder`. Such encoder is in</span>
+<span class="sd">    part inspired by the attention mechanism described in</span>
+<span class="sd">    [Hierarchical Attention Networks for Document</span>
+<span class="sd">    Classification](https://www.cs.cmu.edu/~./hovy/papers/16HLT-hierarchical-attention-networks.pdf).</span>
+<span class="sd">    See `pytorch_widedeep.models.tabular.mlp._attention_layers` for details.</span>
+
+<span class="sd">    Most of the parameters for this class are `Optional` since the use of</span>
+<span class="sd">    categorical or continuous is in fact optional (i.e. one can use</span>
+<span class="sd">    categorical features only, continuous features only or both).</span>
+
+<span class="sd">    Parameters</span>
+<span class="sd">    ----------</span>
+<span class="sd">    column_idx: Dict</span>
+<span class="sd">        Dict containing the index of the columns that will be passed through</span>
+<span class="sd">        the `TabMlp` model. Required to slice the tensors. e.g. _{&#39;education&#39;:</span>
+<span class="sd">        0, &#39;relationship&#39;: 1, &#39;workclass&#39;: 2, ...}_.</span>
+<span class="sd">    cat_embed_input: List, Optional, default = None</span>
+<span class="sd">        List of Tuples with the column name and number of unique values and</span>
+<span class="sd">        embedding dimension. e.g. _[(education, 11), ...]_</span>
+<span class="sd">    cat_embed_dropout: float, Optional, default = None</span>
+<span class="sd">        Categorical embeddings dropout. If `None`, it will default</span>
+<span class="sd">        to 0.</span>
+<span class="sd">    use_cat_bias: bool, Optional, default = None,</span>
+<span class="sd">        Boolean indicating if bias will be used for the categorical embeddings.</span>
+<span class="sd">        If `None`, it will default to &#39;False&#39;.</span>
+<span class="sd">    cat_embed_activation: Optional, str, default = None,</span>
+<span class="sd">        Activation function for the categorical embeddings, if any. Currently</span>
+<span class="sd">        _&#39;tanh&#39;_, _&#39;relu&#39;_, _&#39;leaky_relu&#39;_ and _&#39;gelu&#39;_ are supported</span>
+<span class="sd">    shared_embed: bool, Optional, default = None</span>
+<span class="sd">        Boolean indicating if the embeddings will be &quot;shared&quot;. The idea behind `shared_embed` is</span>
+<span class="sd">        described in the Appendix A in the [TabTransformer paper](https://arxiv.org/abs/2012.06678):</span>
+<span class="sd">        _&#39;The goal of having column embedding is to enable the model to</span>
+<span class="sd">        distinguish the classes in one column from those in the other</span>
+<span class="sd">        columns&#39;_. In other words, the idea is to let the model learn which</span>
+<span class="sd">        column is embedded at the time. See: `pytorch_widedeep.models.transformers._layers.SharedEmbeddings`.</span>
+<span class="sd">    add_shared_embed: bool, Optional, default = None</span>
+<span class="sd">        The two embedding sharing strategies are: 1) add the shared embeddings</span>
+<span class="sd">        to the column embeddings or 2) to replace the first</span>
+<span class="sd">        `frac_shared_embed` with the shared embeddings.</span>
+<span class="sd">        See `pytorch_widedeep.models.embeddings_layers.SharedEmbeddings`</span>
+<span class="sd">        If &#39;None&#39; is passed, it will default to &#39;False&#39;.</span>
+<span class="sd">    frac_shared_embed: float, Optional, default = None</span>
+<span class="sd">        The fraction of embeddings that will be shared (if `add_shared_embed</span>
+<span class="sd">        = False`) by all the different categories for one particular</span>
+<span class="sd">        column. If &#39;None&#39; is passed, it will default to 0.0.</span>
+<span class="sd">    continuous_cols: List, Optional, default = None</span>
+<span class="sd">        List with the name of the numeric (aka continuous) columns</span>
+<span class="sd">    cont_norm_layer: str, Optional, default =  None</span>
+<span class="sd">        Type of normalization layer applied to the continuous features.</span>
+<span class="sd">        Options are: _&#39;layernorm&#39;_ and _&#39;batchnorm&#39;_. if `None`, no</span>
+<span class="sd">        normalization layer will be used.</span>
+<span class="sd">    embed_continuous_method: Optional, str, default = None,</span>
+<span class="sd">        Method to use to embed the continuous features. Options are:</span>
+<span class="sd">        _&#39;standard&#39;_, _&#39;periodic&#39;_ or _&#39;piecewise&#39;_. The _&#39;standard&#39;_</span>
+<span class="sd">        embedding method is based on the FT-Transformer implementation</span>
+<span class="sd">        presented in the paper: [Revisiting Deep Learning Models for</span>
+<span class="sd">        Tabular Data](https://arxiv.org/abs/2106.11959v5). The _&#39;periodic&#39;_</span>
+<span class="sd">        and_&#39;piecewise&#39;_ methods were presented in the paper: [On Embeddings for</span>
+<span class="sd">        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556).</span>
+<span class="sd">        Please, read the papers for details.</span>
+<span class="sd">    cont_embed_dropout: float, Optional, default = None,</span>
+<span class="sd">        Dropout for the continuous embeddings. If `None`, it will default to 0.0</span>
+<span class="sd">    cont_embed_activation: Optional, str, default = None,</span>
+<span class="sd">        Activation function for the continuous embeddings if any. Currently</span>
+<span class="sd">        _&#39;tanh&#39;_, _&#39;relu&#39;_, _&#39;leaky_relu&#39;_ and _&#39;gelu&#39;_ are supported.</span>
+<span class="sd">        If `None`, no activation function will be applied.</span>
+<span class="sd">    quantization_setup: Dict[str, List[float]], Optional, default = None,</span>
+<span class="sd">        This parameter is used when the _&#39;piecewise&#39;_ method is used to embed</span>
+<span class="sd">        the continuous cols. It is a dict where keys are the name of the continuous</span>
+<span class="sd">        columns and values are lists with the boundaries for the quantization</span>
+<span class="sd">        of the continuous_cols. See the examples for details. If</span>
+<span class="sd">        If the _&#39;piecewise&#39;_ method is used, this parameter is required.</span>
+<span class="sd">    n_frequencies: int, Optional, default = None,</span>
+<span class="sd">        This is the so called _&#39;k&#39;_ in their paper [On Embeddings for</span>
+<span class="sd">        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556),</span>
+<span class="sd">        and is the number of &#39;frequencies&#39; that will be used to represent each</span>
+<span class="sd">        continuous column. See their Eq 2 in the paper for details. If</span>
+<span class="sd">        the _&#39;periodic&#39;_ method is used, this parameter is required.</span>
+<span class="sd">    sigma: float, Optional, default = None,</span>
+<span class="sd">        This is the sigma parameter in the paper mentioned when describing the</span>
+<span class="sd">        previous parameters and it is used to initialise the &#39;frequency</span>
+<span class="sd">        weights&#39;. See their Eq 2 in the paper for details. If</span>
+<span class="sd">        the _&#39;periodic&#39;_ method is used, this parameter is required.</span>
+<span class="sd">    share_last_layer: bool, Optional, default = None,</span>
+<span class="sd">        This parameter is not present in the before mentioned paper but it is implemented in</span>
+<span class="sd">        the [official repo](https://github.com/yandex-research/rtdl-num-embeddings/tree/main).</span>
+<span class="sd">        If `True` the linear layer that turns the frequencies into embeddings</span>
+<span class="sd">        will be shared across the continuous columns. If `False` a different</span>
+<span class="sd">        linear layer will be used for each continuous column.</span>
+<span class="sd">        If the _&#39;periodic&#39;_ method is used, this parameter is required.</span>
+<span class="sd">    full_embed_dropout: bool, Optional, default = None,</span>
+<span class="sd">        If `True`, the full embedding corresponding to a column will be masked</span>
+<span class="sd">        out/dropout. If `None`, it will default to `False`.</span>
+<span class="sd">    input_dim: int, default = 32</span>
+<span class="sd">        The so-called *dimension of the model*. Is the number of embeddings</span>
+<span class="sd">        used to encode the categorical and/or continuous columns</span>
+<span class="sd">    attn_dropout: float, default = 0.2</span>
+<span class="sd">        Dropout for each attention block</span>
+<span class="sd">    with_addnorm: bool = False,</span>
+<span class="sd">        Boolean indicating if residual connections will be used in the</span>
+<span class="sd">        attention blocks</span>
+<span class="sd">    attn_activation: str, default = &quot;leaky_relu&quot;</span>
+<span class="sd">        String indicating the activation function to be applied to the dense</span>
+<span class="sd">        layer in each attention encoder. _&#39;tanh&#39;_, _&#39;relu&#39;_, _&#39;leaky_relu&#39;_</span>
+<span class="sd">        and _&#39;gelu&#39;_ are supported.</span>
+<span class="sd">    n_blocks: int, default = 3</span>
+<span class="sd">        Number of attention blocks</span>
+
+<span class="sd">    Attributes</span>
+<span class="sd">    ----------</span>
+<span class="sd">    encoder: nn.Module</span>
+<span class="sd">        Sequence of attention encoders.</span>
+
+<span class="sd">    Examples</span>
+<span class="sd">    --------</span>
+<span class="sd">    &gt;&gt;&gt; import torch</span>
+<span class="sd">    &gt;&gt;&gt; from pytorch_widedeep.models import ContextAttentionMLP</span>
+<span class="sd">    &gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)</span>
+<span class="sd">    &gt;&gt;&gt; colnames = [&#39;a&#39;, &#39;b&#39;, &#39;c&#39;, &#39;d&#39;, &#39;e&#39;]</span>
+<span class="sd">    &gt;&gt;&gt; cat_embed_input = [(u,i,j) for u,i,j in zip(colnames[:4], [4]*4, [8]*4)]</span>
+<span class="sd">    &gt;&gt;&gt; column_idx = {k:v for v,k in enumerate(colnames)}</span>
+<span class="sd">    &gt;&gt;&gt; model = ContextAttentionMLP(column_idx=column_idx, cat_embed_input=cat_embed_input, continuous_cols = [&#39;e&#39;])</span>
+<span class="sd">    &gt;&gt;&gt; out = model(X_tab)</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+
+    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span>
+        <span class="n">column_idx</span><span class="p">:</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">],</span>
+        <span class="o">*</span><span class="p">,</span>
+        <span class="n">cat_embed_input</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="n">Tuple</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">]]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">cat_embed_dropout</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">use_cat_bias</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">cat_embed_activation</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">shared_embed</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">add_shared_embed</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">frac_shared_embed</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">continuous_cols</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">cont_norm_layer</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Literal</span><span class="p">[</span><span class="s2">&quot;batchnorm&quot;</span><span class="p">,</span> <span class="s2">&quot;layernorm&quot;</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">embed_continuous_method</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span>
+            <span class="n">Literal</span><span class="p">[</span><span class="s2">&quot;standard&quot;</span><span class="p">,</span> <span class="s2">&quot;piecewise&quot;</span><span class="p">,</span> <span class="s2">&quot;periodic&quot;</span><span class="p">]</span>
+        <span class="p">]</span> <span class="o">=</span> <span class="s2">&quot;standard&quot;</span><span class="p">,</span>
+        <span class="n">cont_embed_dropout</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">cont_embed_activation</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">quantization_setup</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="nb">float</span><span class="p">]]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">n_frequencies</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">sigma</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">share_last_layer</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">full_embed_dropout</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">input_dim</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">32</span><span class="p">,</span>
+        <span class="n">attn_dropout</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.2</span><span class="p">,</span>
+        <span class="n">with_addnorm</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
+        <span class="n">attn_activation</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;leaky_relu&quot;</span><span class="p">,</span>
+        <span class="n">n_blocks</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">3</span><span class="p">,</span>
+    <span class="p">):</span>
+        <span class="nb">super</span><span class="p">(</span><span class="n">ContextAttentionMLP</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span>
+            <span class="n">column_idx</span><span class="o">=</span><span class="n">column_idx</span><span class="p">,</span>
+            <span class="n">cat_embed_input</span><span class="o">=</span><span class="n">cat_embed_input</span><span class="p">,</span>
+            <span class="n">cat_embed_dropout</span><span class="o">=</span><span class="n">cat_embed_dropout</span><span class="p">,</span>
+            <span class="n">use_cat_bias</span><span class="o">=</span><span class="n">use_cat_bias</span><span class="p">,</span>
+            <span class="n">cat_embed_activation</span><span class="o">=</span><span class="n">cat_embed_activation</span><span class="p">,</span>
+            <span class="n">shared_embed</span><span class="o">=</span><span class="n">shared_embed</span><span class="p">,</span>
+            <span class="n">add_shared_embed</span><span class="o">=</span><span class="n">add_shared_embed</span><span class="p">,</span>
+            <span class="n">frac_shared_embed</span><span class="o">=</span><span class="n">frac_shared_embed</span><span class="p">,</span>
+            <span class="n">continuous_cols</span><span class="o">=</span><span class="n">continuous_cols</span><span class="p">,</span>
+            <span class="n">cont_norm_layer</span><span class="o">=</span><span class="n">cont_norm_layer</span><span class="p">,</span>
+            <span class="n">embed_continuous_method</span><span class="o">=</span><span class="n">embed_continuous_method</span><span class="p">,</span>
+            <span class="n">cont_embed_dropout</span><span class="o">=</span><span class="n">cont_embed_dropout</span><span class="p">,</span>
+            <span class="n">cont_embed_activation</span><span class="o">=</span><span class="n">cont_embed_activation</span><span class="p">,</span>
+            <span class="n">input_dim</span><span class="o">=</span><span class="n">input_dim</span><span class="p">,</span>
+            <span class="n">quantization_setup</span><span class="o">=</span><span class="n">quantization_setup</span><span class="p">,</span>
+            <span class="n">n_frequencies</span><span class="o">=</span><span class="n">n_frequencies</span><span class="p">,</span>
+            <span class="n">sigma</span><span class="o">=</span><span class="n">sigma</span><span class="p">,</span>
+            <span class="n">share_last_layer</span><span class="o">=</span><span class="n">share_last_layer</span><span class="p">,</span>
+            <span class="n">full_embed_dropout</span><span class="o">=</span><span class="n">full_embed_dropout</span><span class="p">,</span>
+        <span class="p">)</span>
 
-    <span class="bp">self</span><span class="o">.</span><span class="n">attn_dropout</span> <span class="o">=</span> <span class="n">attn_dropout</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">with_addnorm</span> <span class="o">=</span> <span class="n">with_addnorm</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">attn_activation</span> <span class="o">=</span> <span class="n">attn_activation</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">n_blocks</span> <span class="o">=</span> <span class="n">n_blocks</span>
-
-    <span class="bp">self</span><span class="o">.</span><span class="n">with_cls_token</span> <span class="o">=</span> <span class="s2">&quot;cls_token&quot;</span> <span class="ow">in</span> <span class="n">column_idx</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">n_cat</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">cat_embed_input</span><span class="p">)</span> <span class="k">if</span> <span class="n">cat_embed_input</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="k">else</span> <span class="mi">0</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">n_cont</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">continuous_cols</span><span class="p">)</span> <span class="k">if</span> <span class="n">continuous_cols</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="k">else</span> <span class="mi">0</span>
-
-    <span class="c1"># Embeddings are instantiated at the base model</span>
-    <span class="c1"># Attention Blocks</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">encoder</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Sequential</span><span class="p">()</span>
-    <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">n_blocks</span><span class="p">):</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">encoder</span><span class="o">.</span><span class="n">add_module</span><span class="p">(</span>
-            <span class="s2">&quot;attention_block&quot;</span> <span class="o">+</span> <span class="nb">str</span><span class="p">(</span><span class="n">i</span><span class="p">),</span>
-            <span class="n">ContextAttentionEncoder</span><span class="p">(</span>
-                <span class="n">input_dim</span><span class="p">,</span>
-                <span class="n">attn_dropout</span><span class="p">,</span>
-                <span class="n">with_addnorm</span><span class="p">,</span>
-                <span class="n">attn_activation</span><span class="p">,</span>
-            <span class="p">),</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">attn_dropout</span> <span class="o">=</span> <span class="n">attn_dropout</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">with_addnorm</span> <span class="o">=</span> <span class="n">with_addnorm</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">attn_activation</span> <span class="o">=</span> <span class="n">attn_activation</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">n_blocks</span> <span class="o">=</span> <span class="n">n_blocks</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">with_cls_token</span> <span class="o">=</span> <span class="s2">&quot;cls_token&quot;</span> <span class="ow">in</span> <span class="n">column_idx</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">n_cat</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">cat_embed_input</span><span class="p">)</span> <span class="k">if</span> <span class="n">cat_embed_input</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="k">else</span> <span class="mi">0</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">n_cont</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">continuous_cols</span><span class="p">)</span> <span class="k">if</span> <span class="n">continuous_cols</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="k">else</span> <span class="mi">0</span>
+
+        <span class="c1"># Embeddings are instantiated at the base model</span>
+        <span class="c1"># Attention Blocks</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">encoder</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Sequential</span><span class="p">()</span>
+        <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">n_blocks</span><span class="p">):</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">encoder</span><span class="o">.</span><span class="n">add_module</span><span class="p">(</span>
+                <span class="s2">&quot;attention_block&quot;</span> <span class="o">+</span> <span class="nb">str</span><span class="p">(</span><span class="n">i</span><span class="p">),</span>
+                <span class="n">ContextAttentionEncoder</span><span class="p">(</span>
+                    <span class="n">input_dim</span><span class="p">,</span>
+                    <span class="n">attn_dropout</span><span class="p">,</span>
+                    <span class="n">with_addnorm</span><span class="p">,</span>
+                    <span class="n">attn_activation</span><span class="p">,</span>
+                <span class="p">),</span>
+            <span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">forward</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">X</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tensor</span><span class="p">:</span>
+        <span class="n">x</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_embeddings</span><span class="p">(</span><span class="n">X</span><span class="p">)</span>
+        <span class="n">x</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">encoder</span><span class="p">(</span><span class="n">x</span><span class="p">)</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">with_cls_token</span><span class="p">:</span>
+            <span class="n">out</span> <span class="o">=</span> <span class="n">x</span><span class="p">[:,</span> <span class="mi">0</span><span class="p">,</span> <span class="p">:]</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="n">out</span> <span class="o">=</span> <span class="n">x</span><span class="o">.</span><span class="n">flatten</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span>
+        <span class="k">return</span> <span class="n">out</span>
+
+    <span class="nd">@property</span>
+    <span class="k">def</span> <span class="nf">output_dim</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">int</span><span class="p">:</span>
+<span class="w">        </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;The output dimension of the model. This is a required property</span>
+<span class="sd">        neccesary to build the `WideDeep` class</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="k">return</span> <span class="p">(</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">input_dim</span>
+            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">with_cls_token</span>
+            <span class="k">else</span> <span class="p">((</span><span class="bp">self</span><span class="o">.</span><span class="n">n_cat</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">n_cont</span><span class="p">)</span> <span class="o">*</span> <span class="bp">self</span><span class="o">.</span><span class="n">input_dim</span><span class="p">)</span>
         <span class="p">)</span>
+
+    <span class="nd">@property</span>
+    <span class="k">def</span> <span class="nf">attention_weights</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">List</span><span class="p">[</span><span class="n">Tensor</span><span class="p">]:</span>
+<span class="w">        </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;List with the attention weights per block</span>
+
+<span class="sd">        The shape of the attention weights is $(N, F)$, where $N$ is the batch</span>
+<span class="sd">        size and $F$ is the number of features/columns in the dataset</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="k">return</span> <span class="p">[</span><span class="n">blk</span><span class="o">.</span><span class="n">attn</span><span class="o">.</span><span class="n">attn_weights</span> <span class="k">for</span> <span class="n">blk</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">encoder</span><span class="p">]</span>
 </code></pre></div></td></tr></table></div>
-                  </details>
+              </details>
 
 
 
@@ -5215,35 +7523,7 @@ <h2 id="pytorch_widedeep.models.tabular.mlp.self_attention_mlp.SelfAttentionMLP"
 
 
 <a href="#pytorch_widedeep.models.tabular.mlp.self_attention_mlp.SelfAttentionMLP" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="nf">SelfAttentionMLP</span><span class="p">(</span>
-    <span class="n">column_idx</span><span class="p">,</span>
-    <span class="o">*</span><span class="p">,</span>
-    <span class="n">cat_embed_input</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">cat_embed_dropout</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">use_cat_bias</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">cat_embed_activation</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">shared_embed</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">add_shared_embed</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">frac_shared_embed</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">continuous_cols</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">cont_norm_layer</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">embed_continuous_method</span><span class="o">=</span><span class="s2">&quot;standard&quot;</span><span class="p">,</span>
-    <span class="n">cont_embed_dropout</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">cont_embed_activation</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">quantization_setup</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">n_frequencies</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">sigma</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">share_last_layer</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">full_embed_dropout</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">input_dim</span><span class="o">=</span><span class="mi">32</span><span class="p">,</span>
-    <span class="n">attn_dropout</span><span class="o">=</span><span class="mf">0.2</span><span class="p">,</span>
-    <span class="n">n_heads</span><span class="o">=</span><span class="mi">8</span><span class="p">,</span>
-    <span class="n">use_bias</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
-    <span class="n">with_addnorm</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
-    <span class="n">attn_activation</span><span class="o">=</span><span class="s2">&quot;leaky_relu&quot;</span><span class="p">,</span>
-    <span class="n">n_blocks</span><span class="o">=</span><span class="mi">3</span>
-<span class="p">)</span>
-</code></pre></div>
+
 
     <div class="doc doc-contents first">
             <p class="doc doc-class-bases">
@@ -5279,132 +7559,183 @@ <h2 id="pytorch_widedeep.models.tabular.mlp.self_attention_mlp.SelfAttentionMLP"
 
 
 <p><span class="doc-section-title">Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>column_idx</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, int]</code>)
-          –
-          <div class="doc-md-description">
-            <p>Dict containing the index of the columns that will be passed through
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+          <th>Default</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>column_idx</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, int]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Dict containing the index of the columns that will be passed through
 the <code>TabMlp</code> model. Required to slice the tensors. e.g. <em>{'education':
 0, 'relationship': 1, 'workclass': 2, ...}</em>.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>cat_embed_input</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.wdtypes.Tuple">Tuple</span>[str, int]]]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>List of Tuples with the column name and number of unique values and
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>cat_embed_input</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.wdtypes.Tuple">Tuple</span>[str, int]]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>List of Tuples with the column name and number of unique values and
 embedding dimension. e.g. <em>[(education, 11), ...]</em></p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>cat_embed_dropout</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Categorical embeddings dropout. If <code>None</code>, it will default
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>cat_embed_dropout</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Categorical embeddings dropout. If <code>None</code>, it will default
 to 0.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>use_cat_bias</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Boolean indicating if bias will be used for the categorical embeddings.
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>use_cat_bias</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Boolean indicating if bias will be used for the categorical embeddings.
 If <code>None</code>, it will default to 'False'.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>cat_embed_activation</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Activation function for the categorical embeddings, if any. Currently
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>cat_embed_activation</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Activation function for the categorical embeddings, if any. Currently
 <em>'tanh'</em>, <em>'relu'</em>, <em>'leaky_relu'</em> and <em>'gelu'</em> are supported</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>shared_embed</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Boolean indicating if the embeddings will be "shared". The idea behind <code>shared_embed</code> is
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>shared_embed</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Boolean indicating if the embeddings will be "shared". The idea behind <code>shared_embed</code> is
 described in the Appendix A in the <a href="https://arxiv.org/abs/2012.06678">TabTransformer paper</a>:
 <em>'The goal of having column embedding is to enable the model to
 distinguish the classes in one column from those in the other
 columns'</em>. In other words, the idea is to let the model learn which
 column is embedded at the time. See: <code>pytorch_widedeep.models.transformers._layers.SharedEmbeddings</code>.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>add_shared_embed</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>The two embedding sharing strategies are: 1) add the shared embeddings
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>add_shared_embed</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>The two embedding sharing strategies are: 1) add the shared embeddings
 to the column embeddings or 2) to replace the first
 <code>frac_shared_embed</code> with the shared embeddings.
 See <code>pytorch_widedeep.models.embeddings_layers.SharedEmbeddings</code>
 If 'None' is passed, it will default to 'False'.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>frac_shared_embed</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>The fraction of embeddings that will be shared (if <code>add_shared_embed
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>frac_shared_embed</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>The fraction of embeddings that will be shared (if <code>add_shared_embed
 = False</code>) by all the different categories for one particular
 column. If 'None' is passed, it will default to 0.0.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>continuous_cols</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[str]]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>List with the name of the numeric (aka continuous) columns</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>cont_norm_layer</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[batchnorm, layernorm]]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Type of normalization layer applied to the continuous features.
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>continuous_cols</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[str]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>List with the name of the numeric (aka continuous) columns</p>
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>cont_norm_layer</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[batchnorm, layernorm]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Type of normalization layer applied to the continuous features.
 Options are: <em>'layernorm'</em> and <em>'batchnorm'</em>. if <code>None</code>, no
 normalization layer will be used.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>embed_continuous_method</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[standard, piecewise, periodic]]</code>, default:
-                  <code>&#39;standard&#39;</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Method to use to embed the continuous features. Options are:
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>embed_continuous_method</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[standard, piecewise, periodic]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Method to use to embed the continuous features. Options are:
 <em>'standard'</em>, <em>'periodic'</em> or <em>'piecewise'</em>. The <em>'standard'</em>
 embedding method is based on the FT-Transformer implementation
 presented in the paper: <a href="https://arxiv.org/abs/2106.11959v5">Revisiting Deep Learning Models for
@@ -5412,193 +7743,269 @@ <h2 id="pytorch_widedeep.models.tabular.mlp.self_attention_mlp.SelfAttentionMLP"
 and_'piecewise'_ methods were presented in the paper: <a href="https://arxiv.org/abs/2203.05556">On Embeddings for
 Numerical Features in Tabular Deep Learning</a>.
 Please, read the papers for details.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>cont_embed_dropout</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>cont_embed_activation</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Activation function for the continuous embeddings if any. Currently
+              </div>
+            </td>
+            <td>
+                  <code>&#39;standard&#39;</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>cont_embed_dropout</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p>
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>cont_embed_activation</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Activation function for the continuous embeddings if any. Currently
 <em>'tanh'</em>, <em>'relu'</em>, <em>'leaky_relu'</em> and <em>'gelu'</em> are supported.
 If <code>None</code>, no activation function will be applied.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>quantization_setup</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, <span title="pytorch_widedeep.wdtypes.List">List</span>[float]]]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>This parameter is used when the <em>'piecewise'</em> method is used to embed
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>quantization_setup</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, <span title="pytorch_widedeep.wdtypes.List">List</span>[float]]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>This parameter is used when the <em>'piecewise'</em> method is used to embed
 the continuous cols. It is a dict where keys are the name of the continuous
 columns and values are lists with the boundaries for the quantization
 of the continuous_cols. See the examples for details. If
 If the <em>'piecewise'</em> method is used, this parameter is required.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>n_frequencies</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[int]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>This is the so called <em>'k'</em> in their paper <a href="https://arxiv.org/abs/2203.05556">On Embeddings for
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>n_frequencies</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[int]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>This is the so called <em>'k'</em> in their paper <a href="https://arxiv.org/abs/2203.05556">On Embeddings for
 Numerical Features in Tabular Deep Learning</a>,
 and is the number of 'frequencies' that will be used to represent each
 continuous column. See their Eq 2 in the paper for details. If
 the <em>'periodic'</em> method is used, this parameter is required.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>sigma</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>This is the sigma parameter in the paper mentioned when describing the
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>sigma</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>This is the sigma parameter in the paper mentioned when describing the
 previous parameters and it is used to initialise the 'frequency
 weights'. See their Eq 2 in the paper for details. If
 the <em>'periodic'</em> method is used, this parameter is required.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>share_last_layer</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>This parameter is not present in the before mentioned paper but it is implemented in
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>share_last_layer</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>This parameter is not present in the before mentioned paper but it is implemented in
 the <a href="https://github.com/yandex-research/rtdl-num-embeddings/tree/main">official repo</a>.
 If <code>True</code> the linear layer that turns the frequencies into embeddings
 will be shared across the continuous columns. If <code>False</code> a different
 linear layer will be used for each continuous column.
 If the <em>'periodic'</em> method is used, this parameter is required.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>full_embed_dropout</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>If <code>True</code>, the full embedding corresponding to a column will be masked
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>full_embed_dropout</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>If <code>True</code>, the full embedding corresponding to a column will be masked
 out/dropout. If <code>None</code>, it will default to <code>False</code>.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>input_dim</code></b>
-              (<code>int</code>, default:
-                  <code>32</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>The so-called <em>dimension of the model</em>. Is the number of
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>input_dim</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>The so-called <em>dimension of the model</em>. Is the number of
 embeddings used to encode the categorical and/or continuous columns</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>attn_dropout</code></b>
-              (<code>float</code>, default:
+              </div>
+            </td>
+            <td>
+                  <code>32</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>attn_dropout</code></td>
+            <td>
+                  <code>float</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Dropout for each attention block</p>
+              </div>
+            </td>
+            <td>
                   <code>0.2</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Dropout for each attention block</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>n_heads</code></b>
-              (<code>int</code>, default:
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>n_heads</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Number of attention heads per attention block.</p>
+              </div>
+            </td>
+            <td>
                   <code>8</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Number of attention heads per attention block.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>use_bias</code></b>
-              (<code>bool</code>, default:
-                  <code>False</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Boolean indicating whether or not to use bias in the Q, K projection
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>use_bias</code></td>
+            <td>
+                  <code>bool</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Boolean indicating whether or not to use bias in the Q, K projection
 layers.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>with_addnorm</code></b>
-              (<code>bool</code>, default:
+              </div>
+            </td>
+            <td>
                   <code>False</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Boolean indicating if residual connections will be used in the attention blocks</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>attn_activation</code></b>
-              (<code>str</code>, default:
-                  <code>&#39;leaky_relu&#39;</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>String indicating the activation function to be applied to the dense
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>with_addnorm</code></td>
+            <td>
+                  <code>bool</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Boolean indicating if residual connections will be used in the attention blocks</p>
+              </div>
+            </td>
+            <td>
+                  <code>False</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>attn_activation</code></td>
+            <td>
+                  <code>str</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>String indicating the activation function to be applied to the dense
 layer in each attention encoder. <em>'tanh'</em>, <em>'relu'</em>, <em>'leaky_relu'</em>
 and <em>'gelu'</em> are supported.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>n_blocks</code></b>
-              (<code>int</code>, default:
+              </div>
+            </td>
+            <td>
+                  <code>&#39;leaky_relu&#39;</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>n_blocks</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Number of attention blocks</p>
+              </div>
+            </td>
+            <td>
                   <code>3</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Number of attention blocks</p>
-          </div>
-        </li>
-    </ul>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
 
 <p><span class="doc-section-title">Attributes:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code><span title="pytorch_widedeep.models.tabular.mlp.self_attention_mlp.SelfAttentionMLP.cat_and_cont_embed">cat_and_cont_embed</span></code></b>
-              (<code><span title="torch.nn.Module">Module</span></code>)
-          –
-          <div class="doc-md-description">
-            <p>This is the module that processes the categorical and continuous columns</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code><span title="pytorch_widedeep.models.tabular.mlp.self_attention_mlp.SelfAttentionMLP.encoder">encoder</span></code></b>
-              (<code><span title="torch.nn.Module">Module</span></code>)
-          –
-          <div class="doc-md-description">
-            <p>Sequence of attention encoders.</p>
-          </div>
-        </li>
-    </ul>
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code><span title="pytorch_widedeep.models.tabular.mlp.self_attention_mlp.SelfAttentionMLP.cat_and_cont_embed">cat_and_cont_embed</span></code></td>
+            <td>
+                  <code><span title="torch.nn.Module">Module</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>This is the module that processes the categorical and continuous columns</p>
+              </div>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code><span title="pytorch_widedeep.models.tabular.mlp.self_attention_mlp.SelfAttentionMLP.encoder">encoder</span></code></td>
+            <td>
+                  <code><span title="torch.nn.Module">Module</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Sequence of attention encoders.</p>
+              </div>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
 
 <p><span class="doc-section-title">Examples:</span></p>
@@ -5612,9 +8019,162 @@ <h2 id="pytorch_widedeep.models.tabular.mlp.self_attention_mlp.SelfAttentionMLP"
 <span class="gp">&gt;&gt;&gt; </span><span class="n">out</span> <span class="o">=</span> <span class="n">model</span><span class="p">(</span><span class="n">X_tab</span><span class="p">)</span>
 </code></pre></div>
 
-                  <details class="quote">
-                    <summary>Source code in <code>pytorch_widedeep/models/tabular/mlp/self_attention_mlp.py</code></summary>
-                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">163</span>
+              <details class="quote">
+                <summary>Source code in <code>pytorch_widedeep/models/tabular/mlp/self_attention_mlp.py</code></summary>
+                <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 10</span>
+<span class="normal"> 11</span>
+<span class="normal"> 12</span>
+<span class="normal"> 13</span>
+<span class="normal"> 14</span>
+<span class="normal"> 15</span>
+<span class="normal"> 16</span>
+<span class="normal"> 17</span>
+<span class="normal"> 18</span>
+<span class="normal"> 19</span>
+<span class="normal"> 20</span>
+<span class="normal"> 21</span>
+<span class="normal"> 22</span>
+<span class="normal"> 23</span>
+<span class="normal"> 24</span>
+<span class="normal"> 25</span>
+<span class="normal"> 26</span>
+<span class="normal"> 27</span>
+<span class="normal"> 28</span>
+<span class="normal"> 29</span>
+<span class="normal"> 30</span>
+<span class="normal"> 31</span>
+<span class="normal"> 32</span>
+<span class="normal"> 33</span>
+<span class="normal"> 34</span>
+<span class="normal"> 35</span>
+<span class="normal"> 36</span>
+<span class="normal"> 37</span>
+<span class="normal"> 38</span>
+<span class="normal"> 39</span>
+<span class="normal"> 40</span>
+<span class="normal"> 41</span>
+<span class="normal"> 42</span>
+<span class="normal"> 43</span>
+<span class="normal"> 44</span>
+<span class="normal"> 45</span>
+<span class="normal"> 46</span>
+<span class="normal"> 47</span>
+<span class="normal"> 48</span>
+<span class="normal"> 49</span>
+<span class="normal"> 50</span>
+<span class="normal"> 51</span>
+<span class="normal"> 52</span>
+<span class="normal"> 53</span>
+<span class="normal"> 54</span>
+<span class="normal"> 55</span>
+<span class="normal"> 56</span>
+<span class="normal"> 57</span>
+<span class="normal"> 58</span>
+<span class="normal"> 59</span>
+<span class="normal"> 60</span>
+<span class="normal"> 61</span>
+<span class="normal"> 62</span>
+<span class="normal"> 63</span>
+<span class="normal"> 64</span>
+<span class="normal"> 65</span>
+<span class="normal"> 66</span>
+<span class="normal"> 67</span>
+<span class="normal"> 68</span>
+<span class="normal"> 69</span>
+<span class="normal"> 70</span>
+<span class="normal"> 71</span>
+<span class="normal"> 72</span>
+<span class="normal"> 73</span>
+<span class="normal"> 74</span>
+<span class="normal"> 75</span>
+<span class="normal"> 76</span>
+<span class="normal"> 77</span>
+<span class="normal"> 78</span>
+<span class="normal"> 79</span>
+<span class="normal"> 80</span>
+<span class="normal"> 81</span>
+<span class="normal"> 82</span>
+<span class="normal"> 83</span>
+<span class="normal"> 84</span>
+<span class="normal"> 85</span>
+<span class="normal"> 86</span>
+<span class="normal"> 87</span>
+<span class="normal"> 88</span>
+<span class="normal"> 89</span>
+<span class="normal"> 90</span>
+<span class="normal"> 91</span>
+<span class="normal"> 92</span>
+<span class="normal"> 93</span>
+<span class="normal"> 94</span>
+<span class="normal"> 95</span>
+<span class="normal"> 96</span>
+<span class="normal"> 97</span>
+<span class="normal"> 98</span>
+<span class="normal"> 99</span>
+<span class="normal">100</span>
+<span class="normal">101</span>
+<span class="normal">102</span>
+<span class="normal">103</span>
+<span class="normal">104</span>
+<span class="normal">105</span>
+<span class="normal">106</span>
+<span class="normal">107</span>
+<span class="normal">108</span>
+<span class="normal">109</span>
+<span class="normal">110</span>
+<span class="normal">111</span>
+<span class="normal">112</span>
+<span class="normal">113</span>
+<span class="normal">114</span>
+<span class="normal">115</span>
+<span class="normal">116</span>
+<span class="normal">117</span>
+<span class="normal">118</span>
+<span class="normal">119</span>
+<span class="normal">120</span>
+<span class="normal">121</span>
+<span class="normal">122</span>
+<span class="normal">123</span>
+<span class="normal">124</span>
+<span class="normal">125</span>
+<span class="normal">126</span>
+<span class="normal">127</span>
+<span class="normal">128</span>
+<span class="normal">129</span>
+<span class="normal">130</span>
+<span class="normal">131</span>
+<span class="normal">132</span>
+<span class="normal">133</span>
+<span class="normal">134</span>
+<span class="normal">135</span>
+<span class="normal">136</span>
+<span class="normal">137</span>
+<span class="normal">138</span>
+<span class="normal">139</span>
+<span class="normal">140</span>
+<span class="normal">141</span>
+<span class="normal">142</span>
+<span class="normal">143</span>
+<span class="normal">144</span>
+<span class="normal">145</span>
+<span class="normal">146</span>
+<span class="normal">147</span>
+<span class="normal">148</span>
+<span class="normal">149</span>
+<span class="normal">150</span>
+<span class="normal">151</span>
+<span class="normal">152</span>
+<span class="normal">153</span>
+<span class="normal">154</span>
+<span class="normal">155</span>
+<span class="normal">156</span>
+<span class="normal">157</span>
+<span class="normal">158</span>
+<span class="normal">159</span>
+<span class="normal">160</span>
+<span class="normal">161</span>
+<span class="normal">162</span>
+<span class="normal">163</span>
 <span class="normal">164</span>
 <span class="normal">165</span>
 <span class="normal">166</span>
@@ -5692,87 +8252,300 @@ <h2 id="pytorch_widedeep.models.tabular.mlp.self_attention_mlp.SelfAttentionMLP"
 <span class="normal">238</span>
 <span class="normal">239</span>
 <span class="normal">240</span>
-<span class="normal">241</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
-    <span class="bp">self</span><span class="p">,</span>
-    <span class="n">column_idx</span><span class="p">:</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">],</span>
-    <span class="o">*</span><span class="p">,</span>
-    <span class="n">cat_embed_input</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="n">Tuple</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">]]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">cat_embed_dropout</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">use_cat_bias</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">cat_embed_activation</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">shared_embed</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">add_shared_embed</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">frac_shared_embed</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">continuous_cols</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">cont_norm_layer</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Literal</span><span class="p">[</span><span class="s2">&quot;batchnorm&quot;</span><span class="p">,</span> <span class="s2">&quot;layernorm&quot;</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">embed_continuous_method</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span>
-        <span class="n">Literal</span><span class="p">[</span><span class="s2">&quot;standard&quot;</span><span class="p">,</span> <span class="s2">&quot;piecewise&quot;</span><span class="p">,</span> <span class="s2">&quot;periodic&quot;</span><span class="p">]</span>
-    <span class="p">]</span> <span class="o">=</span> <span class="s2">&quot;standard&quot;</span><span class="p">,</span>
-    <span class="n">cont_embed_dropout</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">cont_embed_activation</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">quantization_setup</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="nb">float</span><span class="p">]]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">n_frequencies</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">sigma</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">share_last_layer</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">full_embed_dropout</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">input_dim</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">32</span><span class="p">,</span>
-    <span class="n">attn_dropout</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.2</span><span class="p">,</span>
-    <span class="n">n_heads</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">8</span><span class="p">,</span>
-    <span class="n">use_bias</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
-    <span class="n">with_addnorm</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
-    <span class="n">attn_activation</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;leaky_relu&quot;</span><span class="p">,</span>
-    <span class="n">n_blocks</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">3</span><span class="p">,</span>
-<span class="p">):</span>
-    <span class="nb">super</span><span class="p">(</span><span class="n">SelfAttentionMLP</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span>
-        <span class="n">column_idx</span><span class="o">=</span><span class="n">column_idx</span><span class="p">,</span>
-        <span class="n">input_dim</span><span class="o">=</span><span class="n">input_dim</span><span class="p">,</span>
-        <span class="n">cat_embed_input</span><span class="o">=</span><span class="n">cat_embed_input</span><span class="p">,</span>
-        <span class="n">cat_embed_dropout</span><span class="o">=</span><span class="n">cat_embed_dropout</span><span class="p">,</span>
-        <span class="n">use_cat_bias</span><span class="o">=</span><span class="n">use_cat_bias</span><span class="p">,</span>
-        <span class="n">cat_embed_activation</span><span class="o">=</span><span class="n">cat_embed_activation</span><span class="p">,</span>
-        <span class="n">shared_embed</span><span class="o">=</span><span class="n">shared_embed</span><span class="p">,</span>
-        <span class="n">add_shared_embed</span><span class="o">=</span><span class="n">add_shared_embed</span><span class="p">,</span>
-        <span class="n">frac_shared_embed</span><span class="o">=</span><span class="n">frac_shared_embed</span><span class="p">,</span>
-        <span class="n">continuous_cols</span><span class="o">=</span><span class="n">continuous_cols</span><span class="p">,</span>
-        <span class="n">cont_norm_layer</span><span class="o">=</span><span class="n">cont_norm_layer</span><span class="p">,</span>
-        <span class="n">embed_continuous_method</span><span class="o">=</span><span class="n">embed_continuous_method</span><span class="p">,</span>
-        <span class="n">cont_embed_dropout</span><span class="o">=</span><span class="n">cont_embed_dropout</span><span class="p">,</span>
-        <span class="n">cont_embed_activation</span><span class="o">=</span><span class="n">cont_embed_activation</span><span class="p">,</span>
-        <span class="n">quantization_setup</span><span class="o">=</span><span class="n">quantization_setup</span><span class="p">,</span>
-        <span class="n">n_frequencies</span><span class="o">=</span><span class="n">n_frequencies</span><span class="p">,</span>
-        <span class="n">sigma</span><span class="o">=</span><span class="n">sigma</span><span class="p">,</span>
-        <span class="n">share_last_layer</span><span class="o">=</span><span class="n">share_last_layer</span><span class="p">,</span>
-        <span class="n">full_embed_dropout</span><span class="o">=</span><span class="n">full_embed_dropout</span><span class="p">,</span>
-    <span class="p">)</span>
+<span class="normal">241</span>
+<span class="normal">242</span>
+<span class="normal">243</span>
+<span class="normal">244</span>
+<span class="normal">245</span>
+<span class="normal">246</span>
+<span class="normal">247</span>
+<span class="normal">248</span>
+<span class="normal">249</span>
+<span class="normal">250</span>
+<span class="normal">251</span>
+<span class="normal">252</span>
+<span class="normal">253</span>
+<span class="normal">254</span>
+<span class="normal">255</span>
+<span class="normal">256</span>
+<span class="normal">257</span>
+<span class="normal">258</span>
+<span class="normal">259</span>
+<span class="normal">260</span>
+<span class="normal">261</span>
+<span class="normal">262</span>
+<span class="normal">263</span>
+<span class="normal">264</span>
+<span class="normal">265</span>
+<span class="normal">266</span>
+<span class="normal">267</span>
+<span class="normal">268</span>
+<span class="normal">269</span>
+<span class="normal">270</span>
+<span class="normal">271</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">class</span> <span class="nc">SelfAttentionMLP</span><span class="p">(</span><span class="n">BaseTabularModelWithAttention</span><span class="p">):</span>
+<span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Defines a `SelfAttentionMLP` model that can be used as the</span>
+<span class="sd">    deeptabular component of a Wide &amp; Deep model or independently by</span>
+<span class="sd">    itself.</span>
+
+<span class="sd">    This class combines embedding representations of the categorical features</span>
+<span class="sd">    with numerical (aka continuous) features that are also embedded. These</span>
+<span class="sd">    are then passed through a series of attention blocks. Each attention</span>
+<span class="sd">    block is comprised by what we would refer as a simplified</span>
+<span class="sd">    `SelfAttentionEncoder`. See</span>
+<span class="sd">    `pytorch_widedeep.models.tabular.mlp._attention_layers` for details. The</span>
+<span class="sd">    reason to use a simplified version of self attention is because we</span>
+<span class="sd">    observed that the &#39;_standard_&#39; attention mechanism used in the</span>
+<span class="sd">    TabTransformer has a notable tendency to overfit.</span>
+
+<span class="sd">    In more detail, this model only uses Q and K (and not V). If we think</span>
+<span class="sd">    about it as in terms of text (and intuitively), the Softmax(QK^T) is the</span>
+<span class="sd">    attention mechanism that tells us how much, at each position in the input</span>
+<span class="sd">    sentence, each word is represented or &#39;expressed&#39;. We refer to that</span>
+<span class="sd">    as &#39;attention weights&#39;. These attention weighst are normally multiplied</span>
+<span class="sd">    by a Value matrix to further strength the focus on the words that each</span>
+<span class="sd">    word should be attending to (again, intuitively).</span>
+
+<span class="sd">    In this implementation we skip this last multiplication and instead we</span>
+<span class="sd">    multiply the attention weights directly by the input tensor. This is a</span>
+<span class="sd">    simplification that we expect is beneficial in terms of avoiding</span>
+<span class="sd">    overfitting for tabular data.</span>
+
+<span class="sd">    Most of the parameters for this class are `Optional` since the use of</span>
+<span class="sd">    categorical or continuous is in fact optional (i.e. one can use</span>
+<span class="sd">    categorical features only, continuous features only or both).</span>
+
+<span class="sd">    Parameters</span>
+<span class="sd">    ----------</span>
+<span class="sd">    column_idx: Dict</span>
+<span class="sd">        Dict containing the index of the columns that will be passed through</span>
+<span class="sd">        the `TabMlp` model. Required to slice the tensors. e.g. _{&#39;education&#39;:</span>
+<span class="sd">        0, &#39;relationship&#39;: 1, &#39;workclass&#39;: 2, ...}_.</span>
+<span class="sd">    cat_embed_input: List, Optional, default = None</span>
+<span class="sd">        List of Tuples with the column name and number of unique values and</span>
+<span class="sd">        embedding dimension. e.g. _[(education, 11), ...]_</span>
+<span class="sd">    cat_embed_dropout: float, Optional, default = None</span>
+<span class="sd">        Categorical embeddings dropout. If `None`, it will default</span>
+<span class="sd">        to 0.</span>
+<span class="sd">    use_cat_bias: bool, Optional, default = None,</span>
+<span class="sd">        Boolean indicating if bias will be used for the categorical embeddings.</span>
+<span class="sd">        If `None`, it will default to &#39;False&#39;.</span>
+<span class="sd">    cat_embed_activation: Optional, str, default = None,</span>
+<span class="sd">        Activation function for the categorical embeddings, if any. Currently</span>
+<span class="sd">        _&#39;tanh&#39;_, _&#39;relu&#39;_, _&#39;leaky_relu&#39;_ and _&#39;gelu&#39;_ are supported</span>
+<span class="sd">    shared_embed: bool, Optional, default = None</span>
+<span class="sd">        Boolean indicating if the embeddings will be &quot;shared&quot;. The idea behind `shared_embed` is</span>
+<span class="sd">        described in the Appendix A in the [TabTransformer paper](https://arxiv.org/abs/2012.06678):</span>
+<span class="sd">        _&#39;The goal of having column embedding is to enable the model to</span>
+<span class="sd">        distinguish the classes in one column from those in the other</span>
+<span class="sd">        columns&#39;_. In other words, the idea is to let the model learn which</span>
+<span class="sd">        column is embedded at the time. See: `pytorch_widedeep.models.transformers._layers.SharedEmbeddings`.</span>
+<span class="sd">    add_shared_embed: bool, Optional, default = None</span>
+<span class="sd">        The two embedding sharing strategies are: 1) add the shared embeddings</span>
+<span class="sd">        to the column embeddings or 2) to replace the first</span>
+<span class="sd">        `frac_shared_embed` with the shared embeddings.</span>
+<span class="sd">        See `pytorch_widedeep.models.embeddings_layers.SharedEmbeddings`</span>
+<span class="sd">        If &#39;None&#39; is passed, it will default to &#39;False&#39;.</span>
+<span class="sd">    frac_shared_embed: float, Optional, default = None</span>
+<span class="sd">        The fraction of embeddings that will be shared (if `add_shared_embed</span>
+<span class="sd">        = False`) by all the different categories for one particular</span>
+<span class="sd">        column. If &#39;None&#39; is passed, it will default to 0.0.</span>
+<span class="sd">    continuous_cols: List, Optional, default = None</span>
+<span class="sd">        List with the name of the numeric (aka continuous) columns</span>
+<span class="sd">    cont_norm_layer: str, Optional, default =  None</span>
+<span class="sd">        Type of normalization layer applied to the continuous features.</span>
+<span class="sd">        Options are: _&#39;layernorm&#39;_ and _&#39;batchnorm&#39;_. if `None`, no</span>
+<span class="sd">        normalization layer will be used.</span>
+<span class="sd">    embed_continuous_method: Optional, str, default = None,</span>
+<span class="sd">        Method to use to embed the continuous features. Options are:</span>
+<span class="sd">        _&#39;standard&#39;_, _&#39;periodic&#39;_ or _&#39;piecewise&#39;_. The _&#39;standard&#39;_</span>
+<span class="sd">        embedding method is based on the FT-Transformer implementation</span>
+<span class="sd">        presented in the paper: [Revisiting Deep Learning Models for</span>
+<span class="sd">        Tabular Data](https://arxiv.org/abs/2106.11959v5). The _&#39;periodic&#39;_</span>
+<span class="sd">        and_&#39;piecewise&#39;_ methods were presented in the paper: [On Embeddings for</span>
+<span class="sd">        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556).</span>
+<span class="sd">        Please, read the papers for details.</span>
+<span class="sd">    cont_embed_dropout: float, Optional, default = None,</span>
+<span class="sd">        Dropout for the continuous embeddings. If `None`, it will default to 0.0</span>
+<span class="sd">    cont_embed_activation: Optional, str, default = None,</span>
+<span class="sd">        Activation function for the continuous embeddings if any. Currently</span>
+<span class="sd">        _&#39;tanh&#39;_, _&#39;relu&#39;_, _&#39;leaky_relu&#39;_ and _&#39;gelu&#39;_ are supported.</span>
+<span class="sd">        If `None`, no activation function will be applied.</span>
+<span class="sd">    quantization_setup: Dict[str, List[float]], Optional, default = None,</span>
+<span class="sd">        This parameter is used when the _&#39;piecewise&#39;_ method is used to embed</span>
+<span class="sd">        the continuous cols. It is a dict where keys are the name of the continuous</span>
+<span class="sd">        columns and values are lists with the boundaries for the quantization</span>
+<span class="sd">        of the continuous_cols. See the examples for details. If</span>
+<span class="sd">        If the _&#39;piecewise&#39;_ method is used, this parameter is required.</span>
+<span class="sd">    n_frequencies: int, Optional, default = None,</span>
+<span class="sd">        This is the so called _&#39;k&#39;_ in their paper [On Embeddings for</span>
+<span class="sd">        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556),</span>
+<span class="sd">        and is the number of &#39;frequencies&#39; that will be used to represent each</span>
+<span class="sd">        continuous column. See their Eq 2 in the paper for details. If</span>
+<span class="sd">        the _&#39;periodic&#39;_ method is used, this parameter is required.</span>
+<span class="sd">    sigma: float, Optional, default = None,</span>
+<span class="sd">        This is the sigma parameter in the paper mentioned when describing the</span>
+<span class="sd">        previous parameters and it is used to initialise the &#39;frequency</span>
+<span class="sd">        weights&#39;. See their Eq 2 in the paper for details. If</span>
+<span class="sd">        the _&#39;periodic&#39;_ method is used, this parameter is required.</span>
+<span class="sd">    share_last_layer: bool, Optional, default = None,</span>
+<span class="sd">        This parameter is not present in the before mentioned paper but it is implemented in</span>
+<span class="sd">        the [official repo](https://github.com/yandex-research/rtdl-num-embeddings/tree/main).</span>
+<span class="sd">        If `True` the linear layer that turns the frequencies into embeddings</span>
+<span class="sd">        will be shared across the continuous columns. If `False` a different</span>
+<span class="sd">        linear layer will be used for each continuous column.</span>
+<span class="sd">        If the _&#39;periodic&#39;_ method is used, this parameter is required.</span>
+<span class="sd">    full_embed_dropout: bool, Optional, default = None,</span>
+<span class="sd">        If `True`, the full embedding corresponding to a column will be masked</span>
+<span class="sd">        out/dropout. If `None`, it will default to `False`.</span>
+<span class="sd">    input_dim: int, default = 32</span>
+<span class="sd">        The so-called *dimension of the model*. Is the number of</span>
+<span class="sd">        embeddings used to encode the categorical and/or continuous columns</span>
+<span class="sd">    attn_dropout: float, default = 0.2</span>
+<span class="sd">        Dropout for each attention block</span>
+<span class="sd">    n_heads: int, default = 8</span>
+<span class="sd">        Number of attention heads per attention block.</span>
+<span class="sd">    use_bias: bool, default = False</span>
+<span class="sd">        Boolean indicating whether or not to use bias in the Q, K projection</span>
+<span class="sd">        layers.</span>
+<span class="sd">    with_addnorm: bool = False,</span>
+<span class="sd">        Boolean indicating if residual connections will be used in the attention blocks</span>
+<span class="sd">    attn_activation: str, default = &quot;leaky_relu&quot;</span>
+<span class="sd">        String indicating the activation function to be applied to the dense</span>
+<span class="sd">        layer in each attention encoder. _&#39;tanh&#39;_, _&#39;relu&#39;_, _&#39;leaky_relu&#39;_</span>
+<span class="sd">        and _&#39;gelu&#39;_ are supported.</span>
+<span class="sd">    n_blocks: int, default = 3</span>
+<span class="sd">        Number of attention blocks</span>
+
+<span class="sd">    Attributes</span>
+<span class="sd">    ----------</span>
+<span class="sd">    cat_and_cont_embed: nn.Module</span>
+<span class="sd">        This is the module that processes the categorical and continuous columns</span>
+<span class="sd">    encoder: nn.Module</span>
+<span class="sd">        Sequence of attention encoders.</span>
+
+<span class="sd">    Examples</span>
+<span class="sd">    --------</span>
+<span class="sd">    &gt;&gt;&gt; import torch</span>
+<span class="sd">    &gt;&gt;&gt; from pytorch_widedeep.models import SelfAttentionMLP</span>
+<span class="sd">    &gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)</span>
+<span class="sd">    &gt;&gt;&gt; colnames = [&#39;a&#39;, &#39;b&#39;, &#39;c&#39;, &#39;d&#39;, &#39;e&#39;]</span>
+<span class="sd">    &gt;&gt;&gt; cat_embed_input = [(u,i,j) for u,i,j in zip(colnames[:4], [4]*4, [8]*4)]</span>
+<span class="sd">    &gt;&gt;&gt; column_idx = {k:v for v,k in enumerate(colnames)}</span>
+<span class="sd">    &gt;&gt;&gt; model = SelfAttentionMLP(column_idx=column_idx, cat_embed_input=cat_embed_input, continuous_cols = [&#39;e&#39;])</span>
+<span class="sd">    &gt;&gt;&gt; out = model(X_tab)</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+
+    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span>
+        <span class="n">column_idx</span><span class="p">:</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">],</span>
+        <span class="o">*</span><span class="p">,</span>
+        <span class="n">cat_embed_input</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="n">Tuple</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">]]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">cat_embed_dropout</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">use_cat_bias</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">cat_embed_activation</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">shared_embed</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">add_shared_embed</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">frac_shared_embed</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">continuous_cols</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">cont_norm_layer</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Literal</span><span class="p">[</span><span class="s2">&quot;batchnorm&quot;</span><span class="p">,</span> <span class="s2">&quot;layernorm&quot;</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">embed_continuous_method</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span>
+            <span class="n">Literal</span><span class="p">[</span><span class="s2">&quot;standard&quot;</span><span class="p">,</span> <span class="s2">&quot;piecewise&quot;</span><span class="p">,</span> <span class="s2">&quot;periodic&quot;</span><span class="p">]</span>
+        <span class="p">]</span> <span class="o">=</span> <span class="s2">&quot;standard&quot;</span><span class="p">,</span>
+        <span class="n">cont_embed_dropout</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">cont_embed_activation</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">quantization_setup</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="nb">float</span><span class="p">]]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">n_frequencies</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">sigma</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">share_last_layer</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">full_embed_dropout</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">input_dim</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">32</span><span class="p">,</span>
+        <span class="n">attn_dropout</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.2</span><span class="p">,</span>
+        <span class="n">n_heads</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">8</span><span class="p">,</span>
+        <span class="n">use_bias</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
+        <span class="n">with_addnorm</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
+        <span class="n">attn_activation</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;leaky_relu&quot;</span><span class="p">,</span>
+        <span class="n">n_blocks</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">3</span><span class="p">,</span>
+    <span class="p">):</span>
+        <span class="nb">super</span><span class="p">(</span><span class="n">SelfAttentionMLP</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span>
+            <span class="n">column_idx</span><span class="o">=</span><span class="n">column_idx</span><span class="p">,</span>
+            <span class="n">input_dim</span><span class="o">=</span><span class="n">input_dim</span><span class="p">,</span>
+            <span class="n">cat_embed_input</span><span class="o">=</span><span class="n">cat_embed_input</span><span class="p">,</span>
+            <span class="n">cat_embed_dropout</span><span class="o">=</span><span class="n">cat_embed_dropout</span><span class="p">,</span>
+            <span class="n">use_cat_bias</span><span class="o">=</span><span class="n">use_cat_bias</span><span class="p">,</span>
+            <span class="n">cat_embed_activation</span><span class="o">=</span><span class="n">cat_embed_activation</span><span class="p">,</span>
+            <span class="n">shared_embed</span><span class="o">=</span><span class="n">shared_embed</span><span class="p">,</span>
+            <span class="n">add_shared_embed</span><span class="o">=</span><span class="n">add_shared_embed</span><span class="p">,</span>
+            <span class="n">frac_shared_embed</span><span class="o">=</span><span class="n">frac_shared_embed</span><span class="p">,</span>
+            <span class="n">continuous_cols</span><span class="o">=</span><span class="n">continuous_cols</span><span class="p">,</span>
+            <span class="n">cont_norm_layer</span><span class="o">=</span><span class="n">cont_norm_layer</span><span class="p">,</span>
+            <span class="n">embed_continuous_method</span><span class="o">=</span><span class="n">embed_continuous_method</span><span class="p">,</span>
+            <span class="n">cont_embed_dropout</span><span class="o">=</span><span class="n">cont_embed_dropout</span><span class="p">,</span>
+            <span class="n">cont_embed_activation</span><span class="o">=</span><span class="n">cont_embed_activation</span><span class="p">,</span>
+            <span class="n">quantization_setup</span><span class="o">=</span><span class="n">quantization_setup</span><span class="p">,</span>
+            <span class="n">n_frequencies</span><span class="o">=</span><span class="n">n_frequencies</span><span class="p">,</span>
+            <span class="n">sigma</span><span class="o">=</span><span class="n">sigma</span><span class="p">,</span>
+            <span class="n">share_last_layer</span><span class="o">=</span><span class="n">share_last_layer</span><span class="p">,</span>
+            <span class="n">full_embed_dropout</span><span class="o">=</span><span class="n">full_embed_dropout</span><span class="p">,</span>
+        <span class="p">)</span>
 
-    <span class="bp">self</span><span class="o">.</span><span class="n">attn_dropout</span> <span class="o">=</span> <span class="n">attn_dropout</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">n_heads</span> <span class="o">=</span> <span class="n">n_heads</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">use_bias</span> <span class="o">=</span> <span class="n">use_bias</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">with_addnorm</span> <span class="o">=</span> <span class="n">with_addnorm</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">attn_activation</span> <span class="o">=</span> <span class="n">attn_activation</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">n_blocks</span> <span class="o">=</span> <span class="n">n_blocks</span>
-
-    <span class="bp">self</span><span class="o">.</span><span class="n">with_cls_token</span> <span class="o">=</span> <span class="s2">&quot;cls_token&quot;</span> <span class="ow">in</span> <span class="n">column_idx</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">n_cat</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">cat_embed_input</span><span class="p">)</span> <span class="k">if</span> <span class="n">cat_embed_input</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="k">else</span> <span class="mi">0</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">n_cont</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">continuous_cols</span><span class="p">)</span> <span class="k">if</span> <span class="n">continuous_cols</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="k">else</span> <span class="mi">0</span>
-
-    <span class="c1"># Embeddings are instantiated at the base model</span>
-    <span class="c1"># Attention Blocks</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">encoder</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Sequential</span><span class="p">()</span>
-    <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">n_blocks</span><span class="p">):</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">encoder</span><span class="o">.</span><span class="n">add_module</span><span class="p">(</span>
-            <span class="s2">&quot;attention_block&quot;</span> <span class="o">+</span> <span class="nb">str</span><span class="p">(</span><span class="n">i</span><span class="p">),</span>
-            <span class="n">SelfAttentionEncoder</span><span class="p">(</span>
-                <span class="n">input_dim</span><span class="p">,</span>
-                <span class="n">attn_dropout</span><span class="p">,</span>
-                <span class="n">use_bias</span><span class="p">,</span>
-                <span class="n">n_heads</span><span class="p">,</span>
-                <span class="n">with_addnorm</span><span class="p">,</span>
-                <span class="n">attn_activation</span><span class="p">,</span>
-            <span class="p">),</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">attn_dropout</span> <span class="o">=</span> <span class="n">attn_dropout</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">n_heads</span> <span class="o">=</span> <span class="n">n_heads</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">use_bias</span> <span class="o">=</span> <span class="n">use_bias</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">with_addnorm</span> <span class="o">=</span> <span class="n">with_addnorm</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">attn_activation</span> <span class="o">=</span> <span class="n">attn_activation</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">n_blocks</span> <span class="o">=</span> <span class="n">n_blocks</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">with_cls_token</span> <span class="o">=</span> <span class="s2">&quot;cls_token&quot;</span> <span class="ow">in</span> <span class="n">column_idx</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">n_cat</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">cat_embed_input</span><span class="p">)</span> <span class="k">if</span> <span class="n">cat_embed_input</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="k">else</span> <span class="mi">0</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">n_cont</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">continuous_cols</span><span class="p">)</span> <span class="k">if</span> <span class="n">continuous_cols</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="k">else</span> <span class="mi">0</span>
+
+        <span class="c1"># Embeddings are instantiated at the base model</span>
+        <span class="c1"># Attention Blocks</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">encoder</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Sequential</span><span class="p">()</span>
+        <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">n_blocks</span><span class="p">):</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">encoder</span><span class="o">.</span><span class="n">add_module</span><span class="p">(</span>
+                <span class="s2">&quot;attention_block&quot;</span> <span class="o">+</span> <span class="nb">str</span><span class="p">(</span><span class="n">i</span><span class="p">),</span>
+                <span class="n">SelfAttentionEncoder</span><span class="p">(</span>
+                    <span class="n">input_dim</span><span class="p">,</span>
+                    <span class="n">attn_dropout</span><span class="p">,</span>
+                    <span class="n">use_bias</span><span class="p">,</span>
+                    <span class="n">n_heads</span><span class="p">,</span>
+                    <span class="n">with_addnorm</span><span class="p">,</span>
+                    <span class="n">attn_activation</span><span class="p">,</span>
+                <span class="p">),</span>
+            <span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">forward</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">X</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tensor</span><span class="p">:</span>
+        <span class="n">x</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_embeddings</span><span class="p">(</span><span class="n">X</span><span class="p">)</span>
+        <span class="n">x</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">encoder</span><span class="p">(</span><span class="n">x</span><span class="p">)</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">with_cls_token</span><span class="p">:</span>
+            <span class="n">out</span> <span class="o">=</span> <span class="n">x</span><span class="p">[:,</span> <span class="mi">0</span><span class="p">,</span> <span class="p">:]</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="n">out</span> <span class="o">=</span> <span class="n">x</span><span class="o">.</span><span class="n">flatten</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span>
+        <span class="k">return</span> <span class="n">out</span>
+
+    <span class="nd">@property</span>
+    <span class="k">def</span> <span class="nf">output_dim</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">int</span><span class="p">:</span>
+<span class="w">        </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;The output dimension of the model. This is a required property</span>
+<span class="sd">        neccesary to build the WideDeep class</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="k">return</span> <span class="p">(</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">input_dim</span>
+            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">with_cls_token</span>
+            <span class="k">else</span> <span class="p">((</span><span class="bp">self</span><span class="o">.</span><span class="n">n_cat</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">n_cont</span><span class="p">)</span> <span class="o">*</span> <span class="bp">self</span><span class="o">.</span><span class="n">input_dim</span><span class="p">)</span>
         <span class="p">)</span>
+
+    <span class="nd">@property</span>
+    <span class="k">def</span> <span class="nf">attention_weights</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">List</span><span class="p">[</span><span class="n">Tensor</span><span class="p">]:</span>
+<span class="w">        </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;List with the attention weights per block</span>
+
+<span class="sd">        The shape of the attention weights is $(N, H, F, F)$, where $N$ is the</span>
+<span class="sd">        batch size, $H$ is the number of attention heads and $F$ is the</span>
+<span class="sd">        number of features/columns in the dataset</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="k">return</span> <span class="p">[</span><span class="n">blk</span><span class="o">.</span><span class="n">attn</span><span class="o">.</span><span class="n">attn_weights</span> <span class="k">for</span> <span class="n">blk</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">encoder</span><span class="p">]</span>
 </code></pre></div></td></tr></table></div>
-                  </details>
+              </details>
 
 
 
@@ -5853,45 +8626,7 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_transformer.TabTransfor
 
 
 <a href="#pytorch_widedeep.models.tabular.transformers.tab_transformer.TabTransformer" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="nf">TabTransformer</span><span class="p">(</span>
-    <span class="n">column_idx</span><span class="p">,</span>
-    <span class="o">*</span><span class="p">,</span>
-    <span class="n">cat_embed_input</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">cat_embed_dropout</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">use_cat_bias</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">cat_embed_activation</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">shared_embed</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">add_shared_embed</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">frac_shared_embed</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">continuous_cols</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">cont_norm_layer</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">embed_continuous</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">embed_continuous_method</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">cont_embed_dropout</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">cont_embed_activation</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">quantization_setup</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">n_frequencies</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">sigma</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">share_last_layer</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">full_embed_dropout</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">input_dim</span><span class="o">=</span><span class="mi">32</span><span class="p">,</span>
-    <span class="n">n_heads</span><span class="o">=</span><span class="mi">8</span><span class="p">,</span>
-    <span class="n">use_qkv_bias</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
-    <span class="n">n_blocks</span><span class="o">=</span><span class="mi">4</span><span class="p">,</span>
-    <span class="n">attn_dropout</span><span class="o">=</span><span class="mf">0.2</span><span class="p">,</span>
-    <span class="n">ff_dropout</span><span class="o">=</span><span class="mf">0.1</span><span class="p">,</span>
-    <span class="n">ff_factor</span><span class="o">=</span><span class="mi">4</span><span class="p">,</span>
-    <span class="n">transformer_activation</span><span class="o">=</span><span class="s2">&quot;gelu&quot;</span><span class="p">,</span>
-    <span class="n">use_linear_attention</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
-    <span class="n">use_flash_attention</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
-    <span class="n">mlp_hidden_dims</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">mlp_activation</span><span class="o">=</span><span class="s2">&quot;relu&quot;</span><span class="p">,</span>
-    <span class="n">mlp_dropout</span><span class="o">=</span><span class="mf">0.1</span><span class="p">,</span>
-    <span class="n">mlp_batchnorm</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
-    <span class="n">mlp_batchnorm_last</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
-    <span class="n">mlp_linear_first</span><span class="o">=</span><span class="kc">True</span>
-<span class="p">)</span>
-</code></pre></div>
+
 
     <div class="doc doc-contents first">
             <p class="doc doc-class-bases">
@@ -5913,132 +8648,183 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_transformer.TabTransfor
 
 
 <p><span class="doc-section-title">Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>column_idx</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, int]</code>)
-          –
-          <div class="doc-md-description">
-            <p>Dict containing the index of the columns that will be passed through
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+          <th>Default</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>column_idx</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, int]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Dict containing the index of the columns that will be passed through
 the <code>TabMlp</code> model. Required to slice the tensors. e.g. <em>{'education':
 0, 'relationship': 1, 'workclass': 2, ...}</em>.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>cat_embed_input</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.wdtypes.Tuple">Tuple</span>[str, int]]]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>List of Tuples with the column name and number of unique values and
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>cat_embed_input</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.wdtypes.Tuple">Tuple</span>[str, int]]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>List of Tuples with the column name and number of unique values and
 embedding dimension. e.g. <em>[(education, 11), ...]</em></p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>cat_embed_dropout</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Categorical embeddings dropout. If <code>None</code>, it will default
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>cat_embed_dropout</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Categorical embeddings dropout. If <code>None</code>, it will default
 to 0.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>use_cat_bias</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Boolean indicating if bias will be used for the categorical embeddings.
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>use_cat_bias</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Boolean indicating if bias will be used for the categorical embeddings.
 If <code>None</code>, it will default to 'False'.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>cat_embed_activation</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Activation function for the categorical embeddings, if any. Currently
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>cat_embed_activation</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Activation function for the categorical embeddings, if any. Currently
 <em>'tanh'</em>, <em>'relu'</em>, <em>'leaky_relu'</em> and <em>'gelu'</em> are supported</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>shared_embed</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Boolean indicating if the embeddings will be "shared". The idea behind <code>shared_embed</code> is
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>shared_embed</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Boolean indicating if the embeddings will be "shared". The idea behind <code>shared_embed</code> is
 described in the Appendix A in the <a href="https://arxiv.org/abs/2012.06678">TabTransformer paper</a>:
 <em>'The goal of having column embedding is to enable the model to
 distinguish the classes in one column from those in the other
 columns'</em>. In other words, the idea is to let the model learn which
 column is embedded at the time. See: <code>pytorch_widedeep.models.transformers._layers.SharedEmbeddings</code>.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>add_shared_embed</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>The two embedding sharing strategies are: 1) add the shared embeddings
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>add_shared_embed</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>The two embedding sharing strategies are: 1) add the shared embeddings
 to the column embeddings or 2) to replace the first
 <code>frac_shared_embed</code> with the shared embeddings.
 See <code>pytorch_widedeep.models.embeddings_layers.SharedEmbeddings</code>
 If 'None' is passed, it will default to 'False'.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>frac_shared_embed</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>The fraction of embeddings that will be shared (if <code>add_shared_embed
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>frac_shared_embed</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>The fraction of embeddings that will be shared (if <code>add_shared_embed
 = False</code>) by all the different categories for one particular
 column. If 'None' is passed, it will default to 0.0.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>continuous_cols</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[str]]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>List with the name of the numeric (aka continuous) columns</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>cont_norm_layer</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[batchnorm, layernorm]]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Type of normalization layer applied to the continuous features.
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>continuous_cols</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[str]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>List with the name of the numeric (aka continuous) columns</p>
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>cont_norm_layer</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[batchnorm, layernorm]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Type of normalization layer applied to the continuous features.
 Options are: <em>'layernorm'</em> and <em>'batchnorm'</em>. if <code>None</code>, no
 normalization layer will be used.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>embed_continuous_method</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[standard, piecewise, periodic]]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Method to use to embed the continuous features. Options are:
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>embed_continuous_method</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[standard, piecewise, periodic]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Method to use to embed the continuous features. Options are:
 <em>'standard'</em>, <em>'periodic'</em> or <em>'piecewise'</em>. The <em>'standard'</em>
 embedding method is based on the FT-Transformer implementation
 presented in the paper: <a href="https://arxiv.org/abs/2106.11959v5">Revisiting Deep Learning Models for
@@ -6046,307 +8832,419 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_transformer.TabTransfor
 and_'piecewise'_ methods were presented in the paper: <a href="https://arxiv.org/abs/2203.05556">On Embeddings for
 Numerical Features in Tabular Deep Learning</a>.
 Please, read the papers for details.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>cont_embed_dropout</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>cont_embed_activation</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Activation function for the continuous embeddings if any. Currently
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>cont_embed_dropout</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p>
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>cont_embed_activation</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Activation function for the continuous embeddings if any. Currently
 <em>'tanh'</em>, <em>'relu'</em>, <em>'leaky_relu'</em> and <em>'gelu'</em> are supported.
 If <code>None</code>, no activation function will be applied.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>quantization_setup</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, <span title="pytorch_widedeep.wdtypes.List">List</span>[float]]]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>This parameter is used when the <em>'piecewise'</em> method is used to embed
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>quantization_setup</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, <span title="pytorch_widedeep.wdtypes.List">List</span>[float]]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>This parameter is used when the <em>'piecewise'</em> method is used to embed
 the continuous cols. It is a dict where keys are the name of the continuous
 columns and values are lists with the boundaries for the quantization
 of the continuous_cols. See the examples for details. If
 If the <em>'piecewise'</em> method is used, this parameter is required.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>n_frequencies</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[int]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>This is the so called <em>'k'</em> in their paper <a href="https://arxiv.org/abs/2203.05556">On Embeddings for
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>n_frequencies</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[int]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>This is the so called <em>'k'</em> in their paper <a href="https://arxiv.org/abs/2203.05556">On Embeddings for
 Numerical Features in Tabular Deep Learning</a>,
 and is the number of 'frequencies' that will be used to represent each
 continuous column. See their Eq 2 in the paper for details. If
 the <em>'periodic'</em> method is used, this parameter is required.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>sigma</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>This is the sigma parameter in the paper mentioned when describing the
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>sigma</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>This is the sigma parameter in the paper mentioned when describing the
 previous parameters and it is used to initialise the 'frequency
 weights'. See their Eq 2 in the paper for details. If
 the <em>'periodic'</em> method is used, this parameter is required.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>share_last_layer</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>This parameter is not present in the before mentioned paper but it is implemented in
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>share_last_layer</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>This parameter is not present in the before mentioned paper but it is implemented in
 the <a href="https://github.com/yandex-research/rtdl-num-embeddings/tree/main">official repo</a>.
 If <code>True</code> the linear layer that turns the frequencies into embeddings
 will be shared across the continuous columns. If <code>False</code> a different
 linear layer will be used for each continuous column.
 If the <em>'periodic'</em> method is used, this parameter is required.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>full_embed_dropout</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>If <code>True</code>, the full embedding corresponding to a column will be masked
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>full_embed_dropout</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>If <code>True</code>, the full embedding corresponding to a column will be masked
 out/dropout. If <code>None</code>, it will default to <code>False</code>.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>input_dim</code></b>
-              (<code>int</code>, default:
-                  <code>32</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>The so-called <em>dimension of the model</em>. Is the number of
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>input_dim</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>The so-called <em>dimension of the model</em>. Is the number of
 embeddings used to encode the categorical and/or continuous columns</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>n_heads</code></b>
-              (<code>int</code>, default:
+              </div>
+            </td>
+            <td>
+                  <code>32</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>n_heads</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Number of attention heads per Transformer block</p>
+              </div>
+            </td>
+            <td>
                   <code>8</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Number of attention heads per Transformer block</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>use_qkv_bias</code></b>
-              (<code>bool</code>, default:
-                  <code>False</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Boolean indicating whether or not to use bias in the Q, K, and V
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>use_qkv_bias</code></td>
+            <td>
+                  <code>bool</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Boolean indicating whether or not to use bias in the Q, K, and V
 projection layers.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>n_blocks</code></b>
-              (<code>int</code>, default:
+              </div>
+            </td>
+            <td>
+                  <code>False</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>n_blocks</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Number of Transformer blocks</p>
+              </div>
+            </td>
+            <td>
                   <code>4</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Number of Transformer blocks</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>attn_dropout</code></b>
-              (<code>float</code>, default:
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>attn_dropout</code></td>
+            <td>
+                  <code>float</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Dropout that will be applied to the Multi-Head Attention layers</p>
+              </div>
+            </td>
+            <td>
                   <code>0.2</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Dropout that will be applied to the Multi-Head Attention layers</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>ff_dropout</code></b>
-              (<code>float</code>, default:
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>ff_dropout</code></td>
+            <td>
+                  <code>float</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Dropout that will be applied to the FeedForward network</p>
+              </div>
+            </td>
+            <td>
                   <code>0.1</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Dropout that will be applied to the FeedForward network</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>ff_factor</code></b>
-              (<code>int</code>, default:
-                  <code>4</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Multiplicative factor applied to the first layer of the FF network in
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>ff_factor</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Multiplicative factor applied to the first layer of the FF network in
 each Transformer block, This is normally set to 4.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>transformer_activation</code></b>
-              (<code>str</code>, default:
-                  <code>&#39;gelu&#39;</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Transformer Encoder activation function. <em>'tanh'</em>, <em>'relu'</em>,
+              </div>
+            </td>
+            <td>
+                  <code>4</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>transformer_activation</code></td>
+            <td>
+                  <code>str</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Transformer Encoder activation function. <em>'tanh'</em>, <em>'relu'</em>,
 <em>'leaky_relu'</em>, <em>'gelu'</em>, <em>'geglu'</em> and <em>'reglu'</em> are supported</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>use_linear_attention</code></b>
-              (<code>bool</code>, default:
-                  <code>False</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Boolean indicating if Linear Attention (from <a href="https://arxiv.org/abs/2006.16236">Transformers are RNNs:
+              </div>
+            </td>
+            <td>
+                  <code>&#39;gelu&#39;</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>use_linear_attention</code></td>
+            <td>
+                  <code>bool</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Boolean indicating if Linear Attention (from <a href="https://arxiv.org/abs/2006.16236">Transformers are RNNs:
 Fast Autoregressive Transformers with Linear Attention</a>)
 will be used. The inclusing of this mode of attention is inspired by
 <a href="https://www.uber.com/en-GB/blog/deepeta-how-uber-predicts-arrival-times/">this post</a>,
 where the Uber team finds that this attention mechanism leads to the
 best results for their tabular data.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>use_flash_attention</code></b>
-              (<code>bool</code>, default:
+              </div>
+            </td>
+            <td>
                   <code>False</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Boolean indicating if
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>use_flash_attention</code></td>
+            <td>
+                  <code>bool</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Boolean indicating if
 <a href="https://pytorch.org/docs/stable/generated/torch.nn.functional.scaled_dot_product_attention.html">Flash Attention</a>
 will be used. <br/></p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>mlp_hidden_dims</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[int]]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>List with the number of neurons per dense layer in the MLP. e.g:
+              </div>
+            </td>
+            <td>
+                  <code>False</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>mlp_hidden_dims</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[int]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>List with the number of neurons per dense layer in the MLP. e.g:
 <em>[64, 32]</em>. If not provided no MLP on top of the final
 Transformer block will be used.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>mlp_activation</code></b>
-              (<code>str</code>, default:
-                  <code>&#39;relu&#39;</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Activation function for the dense layers of the MLP. Currently
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>mlp_activation</code></td>
+            <td>
+                  <code>str</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Activation function for the dense layers of the MLP. Currently
 <em>'tanh'</em>, <em>'relu'</em>, <em>'leaky'_relu' and _'gelu'</em> are supported.
 If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it
 will default to <em>'relu'</em>.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>mlp_dropout</code></b>
-              (<code>float</code>, default:
-                  <code>0.1</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>float with the dropout between the dense layers of the MLP.
+              </div>
+            </td>
+            <td>
+                  <code>&#39;relu&#39;</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>mlp_dropout</code></td>
+            <td>
+                  <code>float</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>float with the dropout between the dense layers of the MLP.
 If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it
 will default to 0.0.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>mlp_batchnorm</code></b>
-              (<code>bool</code>, default:
-                  <code>False</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Boolean indicating whether or not batch normalization will be applied
+              </div>
+            </td>
+            <td>
+                  <code>0.1</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>mlp_batchnorm</code></td>
+            <td>
+                  <code>bool</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Boolean indicating whether or not batch normalization will be applied
 to the dense layers
 If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it
 will default to False.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>mlp_batchnorm_last</code></b>
-              (<code>bool</code>, default:
+              </div>
+            </td>
+            <td>
                   <code>False</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Boolean indicating whether or not batch normalization will be applied
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>mlp_batchnorm_last</code></td>
+            <td>
+                  <code>bool</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Boolean indicating whether or not batch normalization will be applied
 to the last of the dense layers
 If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it
 will default to False.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>mlp_linear_first</code></b>
-              (<code>bool</code>, default:
-                  <code>True</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Boolean indicating the order of the operations in the dense
+              </div>
+            </td>
+            <td>
+                  <code>False</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>mlp_linear_first</code></td>
+            <td>
+                  <code>bool</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Boolean indicating the order of the operations in the dense
 layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt;
 LIN -&gt; ACT]</code>
 If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it
 will default to <code>True</code>.</p>
-          </div>
-        </li>
-    </ul>
+              </div>
+            </td>
+            <td>
+                  <code>True</code>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
 
 <p><span class="doc-section-title">Attributes:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code><span title="pytorch_widedeep.models.tabular.transformers.tab_transformer.TabTransformer.encoder">encoder</span></code></b>
-              (<code><span title="torch.nn.Module">Module</span></code>)
-          –
-          <div class="doc-md-description">
-            <p>Sequence of Transformer blocks</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code><span title="pytorch_widedeep.models.tabular.transformers.tab_transformer.TabTransformer.mlp">mlp</span></code></b>
-              (<code><span title="torch.nn.Module">Module</span></code>)
-          –
-          <div class="doc-md-description">
-            <p>MLP component in the model</p>
-          </div>
-        </li>
-    </ul>
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code><span title="pytorch_widedeep.models.tabular.transformers.tab_transformer.TabTransformer.encoder">encoder</span></code></td>
+            <td>
+                  <code><span title="torch.nn.Module">Module</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Sequence of Transformer blocks</p>
+              </div>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code><span title="pytorch_widedeep.models.tabular.transformers.tab_transformer.TabTransformer.mlp">mlp</span></code></td>
+            <td>
+                  <code><span title="torch.nn.Module">Module</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>MLP component in the model</p>
+              </div>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
 
 <p><span class="doc-section-title">Examples:</span></p>
@@ -6361,10 +9259,190 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_transformer.TabTransfor
 <span class="gp">&gt;&gt;&gt; </span><span class="n">out</span> <span class="o">=</span> <span class="n">model</span><span class="p">(</span><span class="n">X_tab</span><span class="p">)</span>
 </code></pre></div>
 
-                  <details class="quote">
-                    <summary>Source code in <code>pytorch_widedeep/models/tabular/transformers/tab_transformer.py</code></summary>
-                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">192</span>
-<span class="normal">193</span>
+              <details class="quote">
+                <summary>Source code in <code>pytorch_widedeep/models/tabular/transformers/tab_transformer.py</code></summary>
+                <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 12</span>
+<span class="normal"> 13</span>
+<span class="normal"> 14</span>
+<span class="normal"> 15</span>
+<span class="normal"> 16</span>
+<span class="normal"> 17</span>
+<span class="normal"> 18</span>
+<span class="normal"> 19</span>
+<span class="normal"> 20</span>
+<span class="normal"> 21</span>
+<span class="normal"> 22</span>
+<span class="normal"> 23</span>
+<span class="normal"> 24</span>
+<span class="normal"> 25</span>
+<span class="normal"> 26</span>
+<span class="normal"> 27</span>
+<span class="normal"> 28</span>
+<span class="normal"> 29</span>
+<span class="normal"> 30</span>
+<span class="normal"> 31</span>
+<span class="normal"> 32</span>
+<span class="normal"> 33</span>
+<span class="normal"> 34</span>
+<span class="normal"> 35</span>
+<span class="normal"> 36</span>
+<span class="normal"> 37</span>
+<span class="normal"> 38</span>
+<span class="normal"> 39</span>
+<span class="normal"> 40</span>
+<span class="normal"> 41</span>
+<span class="normal"> 42</span>
+<span class="normal"> 43</span>
+<span class="normal"> 44</span>
+<span class="normal"> 45</span>
+<span class="normal"> 46</span>
+<span class="normal"> 47</span>
+<span class="normal"> 48</span>
+<span class="normal"> 49</span>
+<span class="normal"> 50</span>
+<span class="normal"> 51</span>
+<span class="normal"> 52</span>
+<span class="normal"> 53</span>
+<span class="normal"> 54</span>
+<span class="normal"> 55</span>
+<span class="normal"> 56</span>
+<span class="normal"> 57</span>
+<span class="normal"> 58</span>
+<span class="normal"> 59</span>
+<span class="normal"> 60</span>
+<span class="normal"> 61</span>
+<span class="normal"> 62</span>
+<span class="normal"> 63</span>
+<span class="normal"> 64</span>
+<span class="normal"> 65</span>
+<span class="normal"> 66</span>
+<span class="normal"> 67</span>
+<span class="normal"> 68</span>
+<span class="normal"> 69</span>
+<span class="normal"> 70</span>
+<span class="normal"> 71</span>
+<span class="normal"> 72</span>
+<span class="normal"> 73</span>
+<span class="normal"> 74</span>
+<span class="normal"> 75</span>
+<span class="normal"> 76</span>
+<span class="normal"> 77</span>
+<span class="normal"> 78</span>
+<span class="normal"> 79</span>
+<span class="normal"> 80</span>
+<span class="normal"> 81</span>
+<span class="normal"> 82</span>
+<span class="normal"> 83</span>
+<span class="normal"> 84</span>
+<span class="normal"> 85</span>
+<span class="normal"> 86</span>
+<span class="normal"> 87</span>
+<span class="normal"> 88</span>
+<span class="normal"> 89</span>
+<span class="normal"> 90</span>
+<span class="normal"> 91</span>
+<span class="normal"> 92</span>
+<span class="normal"> 93</span>
+<span class="normal"> 94</span>
+<span class="normal"> 95</span>
+<span class="normal"> 96</span>
+<span class="normal"> 97</span>
+<span class="normal"> 98</span>
+<span class="normal"> 99</span>
+<span class="normal">100</span>
+<span class="normal">101</span>
+<span class="normal">102</span>
+<span class="normal">103</span>
+<span class="normal">104</span>
+<span class="normal">105</span>
+<span class="normal">106</span>
+<span class="normal">107</span>
+<span class="normal">108</span>
+<span class="normal">109</span>
+<span class="normal">110</span>
+<span class="normal">111</span>
+<span class="normal">112</span>
+<span class="normal">113</span>
+<span class="normal">114</span>
+<span class="normal">115</span>
+<span class="normal">116</span>
+<span class="normal">117</span>
+<span class="normal">118</span>
+<span class="normal">119</span>
+<span class="normal">120</span>
+<span class="normal">121</span>
+<span class="normal">122</span>
+<span class="normal">123</span>
+<span class="normal">124</span>
+<span class="normal">125</span>
+<span class="normal">126</span>
+<span class="normal">127</span>
+<span class="normal">128</span>
+<span class="normal">129</span>
+<span class="normal">130</span>
+<span class="normal">131</span>
+<span class="normal">132</span>
+<span class="normal">133</span>
+<span class="normal">134</span>
+<span class="normal">135</span>
+<span class="normal">136</span>
+<span class="normal">137</span>
+<span class="normal">138</span>
+<span class="normal">139</span>
+<span class="normal">140</span>
+<span class="normal">141</span>
+<span class="normal">142</span>
+<span class="normal">143</span>
+<span class="normal">144</span>
+<span class="normal">145</span>
+<span class="normal">146</span>
+<span class="normal">147</span>
+<span class="normal">148</span>
+<span class="normal">149</span>
+<span class="normal">150</span>
+<span class="normal">151</span>
+<span class="normal">152</span>
+<span class="normal">153</span>
+<span class="normal">154</span>
+<span class="normal">155</span>
+<span class="normal">156</span>
+<span class="normal">157</span>
+<span class="normal">158</span>
+<span class="normal">159</span>
+<span class="normal">160</span>
+<span class="normal">161</span>
+<span class="normal">162</span>
+<span class="normal">163</span>
+<span class="normal">164</span>
+<span class="normal">165</span>
+<span class="normal">166</span>
+<span class="normal">167</span>
+<span class="normal">168</span>
+<span class="normal">169</span>
+<span class="normal">170</span>
+<span class="normal">171</span>
+<span class="normal">172</span>
+<span class="normal">173</span>
+<span class="normal">174</span>
+<span class="normal">175</span>
+<span class="normal">176</span>
+<span class="normal">177</span>
+<span class="normal">178</span>
+<span class="normal">179</span>
+<span class="normal">180</span>
+<span class="normal">181</span>
+<span class="normal">182</span>
+<span class="normal">183</span>
+<span class="normal">184</span>
+<span class="normal">185</span>
+<span class="normal">186</span>
+<span class="normal">187</span>
+<span class="normal">188</span>
+<span class="normal">189</span>
+<span class="normal">190</span>
+<span class="normal">191</span>
+<span class="normal">192</span>
+<span class="normal">193</span>
 <span class="normal">194</span>
 <span class="normal">195</span>
 <span class="normal">196</span>
@@ -6515,161 +9593,519 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_transformer.TabTransfor
 <span class="normal">341</span>
 <span class="normal">342</span>
 <span class="normal">343</span>
-<span class="normal">344</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
-    <span class="bp">self</span><span class="p">,</span>
-    <span class="n">column_idx</span><span class="p">:</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">],</span>
-    <span class="o">*</span><span class="p">,</span>
-    <span class="n">cat_embed_input</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="n">Tuple</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">]]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">cat_embed_dropout</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">use_cat_bias</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">cat_embed_activation</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">shared_embed</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">add_shared_embed</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">frac_shared_embed</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">continuous_cols</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">cont_norm_layer</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Literal</span><span class="p">[</span><span class="s2">&quot;batchnorm&quot;</span><span class="p">,</span> <span class="s2">&quot;layernorm&quot;</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">embed_continuous</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">embed_continuous_method</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span>
-        <span class="n">Literal</span><span class="p">[</span><span class="s2">&quot;standard&quot;</span><span class="p">,</span> <span class="s2">&quot;piecewise&quot;</span><span class="p">,</span> <span class="s2">&quot;periodic&quot;</span><span class="p">]</span>
-    <span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">cont_embed_dropout</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">cont_embed_activation</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">quantization_setup</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="nb">float</span><span class="p">]]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">n_frequencies</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">sigma</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">share_last_layer</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">full_embed_dropout</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">input_dim</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">32</span><span class="p">,</span>
-    <span class="n">n_heads</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">8</span><span class="p">,</span>
-    <span class="n">use_qkv_bias</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
-    <span class="n">n_blocks</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">4</span><span class="p">,</span>
-    <span class="n">attn_dropout</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.2</span><span class="p">,</span>
-    <span class="n">ff_dropout</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.1</span><span class="p">,</span>
-    <span class="n">ff_factor</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">4</span><span class="p">,</span>
-    <span class="n">transformer_activation</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;gelu&quot;</span><span class="p">,</span>
-    <span class="n">use_linear_attention</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
-    <span class="n">use_flash_attention</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
-    <span class="n">mlp_hidden_dims</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">int</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">mlp_activation</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;relu&quot;</span><span class="p">,</span>
-    <span class="n">mlp_dropout</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.1</span><span class="p">,</span>
-    <span class="n">mlp_batchnorm</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
-    <span class="n">mlp_batchnorm_last</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
-    <span class="n">mlp_linear_first</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">,</span>
-<span class="p">):</span>
-    <span class="nb">super</span><span class="p">(</span><span class="n">TabTransformer</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span>
-        <span class="n">column_idx</span><span class="o">=</span><span class="n">column_idx</span><span class="p">,</span>
-        <span class="n">cat_embed_input</span><span class="o">=</span><span class="n">cat_embed_input</span><span class="p">,</span>
-        <span class="n">cat_embed_dropout</span><span class="o">=</span><span class="n">cat_embed_dropout</span><span class="p">,</span>
-        <span class="n">use_cat_bias</span><span class="o">=</span><span class="n">use_cat_bias</span><span class="p">,</span>
-        <span class="n">cat_embed_activation</span><span class="o">=</span><span class="n">cat_embed_activation</span><span class="p">,</span>
-        <span class="n">shared_embed</span><span class="o">=</span><span class="n">shared_embed</span><span class="p">,</span>
-        <span class="n">add_shared_embed</span><span class="o">=</span><span class="n">add_shared_embed</span><span class="p">,</span>
-        <span class="n">frac_shared_embed</span><span class="o">=</span><span class="n">frac_shared_embed</span><span class="p">,</span>
-        <span class="n">continuous_cols</span><span class="o">=</span><span class="n">continuous_cols</span><span class="p">,</span>
-        <span class="n">cont_norm_layer</span><span class="o">=</span><span class="n">cont_norm_layer</span><span class="p">,</span>
-        <span class="c1"># UGLY hack to be able to use the base class __init__ method</span>
-        <span class="n">embed_continuous_method</span><span class="o">=</span><span class="p">(</span>
-            <span class="s2">&quot;standard&quot;</span>
-            <span class="k">if</span> <span class="n">embed_continuous_method</span> <span class="ow">is</span> <span class="kc">None</span>
-            <span class="k">else</span> <span class="n">embed_continuous_method</span>
-        <span class="p">),</span>
-        <span class="n">cont_embed_dropout</span><span class="o">=</span><span class="n">cont_embed_dropout</span><span class="p">,</span>
-        <span class="n">cont_embed_activation</span><span class="o">=</span><span class="n">cont_embed_activation</span><span class="p">,</span>
-        <span class="n">quantization_setup</span><span class="o">=</span><span class="n">quantization_setup</span><span class="p">,</span>
-        <span class="n">n_frequencies</span><span class="o">=</span><span class="n">n_frequencies</span><span class="p">,</span>
-        <span class="n">sigma</span><span class="o">=</span><span class="n">sigma</span><span class="p">,</span>
-        <span class="n">share_last_layer</span><span class="o">=</span><span class="n">share_last_layer</span><span class="p">,</span>
-        <span class="n">input_dim</span><span class="o">=</span><span class="n">input_dim</span><span class="p">,</span>
-        <span class="n">full_embed_dropout</span><span class="o">=</span><span class="n">full_embed_dropout</span><span class="p">,</span>
-    <span class="p">)</span>
-
-    <span class="c1"># we overwrite the embed_continuous_method parameter that was set</span>
-    <span class="c1"># to &#39;standard&#39; if embed_continuous_method is None. In other words,</span>
-    <span class="c1"># this will already have the right value if the user provided a value</span>
-    <span class="c1"># for embed_continuous_method, otherwise will be &quot;standard&quot; when it</span>
-    <span class="c1"># should be None</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">embed_continuous_method</span> <span class="o">=</span> <span class="n">embed_continuous_method</span>
+<span class="normal">344</span>
+<span class="normal">345</span>
+<span class="normal">346</span>
+<span class="normal">347</span>
+<span class="normal">348</span>
+<span class="normal">349</span>
+<span class="normal">350</span>
+<span class="normal">351</span>
+<span class="normal">352</span>
+<span class="normal">353</span>
+<span class="normal">354</span>
+<span class="normal">355</span>
+<span class="normal">356</span>
+<span class="normal">357</span>
+<span class="normal">358</span>
+<span class="normal">359</span>
+<span class="normal">360</span>
+<span class="normal">361</span>
+<span class="normal">362</span>
+<span class="normal">363</span>
+<span class="normal">364</span>
+<span class="normal">365</span>
+<span class="normal">366</span>
+<span class="normal">367</span>
+<span class="normal">368</span>
+<span class="normal">369</span>
+<span class="normal">370</span>
+<span class="normal">371</span>
+<span class="normal">372</span>
+<span class="normal">373</span>
+<span class="normal">374</span>
+<span class="normal">375</span>
+<span class="normal">376</span>
+<span class="normal">377</span>
+<span class="normal">378</span>
+<span class="normal">379</span>
+<span class="normal">380</span>
+<span class="normal">381</span>
+<span class="normal">382</span>
+<span class="normal">383</span>
+<span class="normal">384</span>
+<span class="normal">385</span>
+<span class="normal">386</span>
+<span class="normal">387</span>
+<span class="normal">388</span>
+<span class="normal">389</span>
+<span class="normal">390</span>
+<span class="normal">391</span>
+<span class="normal">392</span>
+<span class="normal">393</span>
+<span class="normal">394</span>
+<span class="normal">395</span>
+<span class="normal">396</span>
+<span class="normal">397</span>
+<span class="normal">398</span>
+<span class="normal">399</span>
+<span class="normal">400</span>
+<span class="normal">401</span>
+<span class="normal">402</span>
+<span class="normal">403</span>
+<span class="normal">404</span>
+<span class="normal">405</span>
+<span class="normal">406</span>
+<span class="normal">407</span>
+<span class="normal">408</span>
+<span class="normal">409</span>
+<span class="normal">410</span>
+<span class="normal">411</span>
+<span class="normal">412</span>
+<span class="normal">413</span>
+<span class="normal">414</span>
+<span class="normal">415</span>
+<span class="normal">416</span>
+<span class="normal">417</span>
+<span class="normal">418</span>
+<span class="normal">419</span>
+<span class="normal">420</span>
+<span class="normal">421</span>
+<span class="normal">422</span>
+<span class="normal">423</span>
+<span class="normal">424</span>
+<span class="normal">425</span>
+<span class="normal">426</span>
+<span class="normal">427</span>
+<span class="normal">428</span>
+<span class="normal">429</span>
+<span class="normal">430</span>
+<span class="normal">431</span>
+<span class="normal">432</span>
+<span class="normal">433</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">class</span> <span class="nc">TabTransformer</span><span class="p">(</span><span class="n">BaseTabularModelWithAttention</span><span class="p">):</span>
+<span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Defines our adptation of the</span>
+<span class="sd">    [TabTransformer model](https://arxiv.org/abs/2012.06678)</span>
+<span class="sd">    that can be used as the `deeptabular` component of a</span>
+<span class="sd">    Wide &amp; Deep model or independently by itself.</span>
+
+<span class="sd">    Most of the parameters for this class are `Optional` since the use of</span>
+<span class="sd">    categorical or continuous is in fact optional (i.e. one can use</span>
+<span class="sd">    categorical features only, continuous features only or both).</span>
+
+<span class="sd">    :information_source: **NOTE**:</span>
+<span class="sd">    This is an enhanced adaptation of the model described in the paper. It can</span>
+<span class="sd">    be considered as the flagship of our transformer family of models for</span>
+<span class="sd">    tabular data and offers mutiple, additional features relative to the</span>
+<span class="sd">    original publication(and some other models in the library)</span>
+
+<span class="sd">    Parameters</span>
+<span class="sd">    ----------</span>
+<span class="sd">    column_idx: Dict</span>
+<span class="sd">        Dict containing the index of the columns that will be passed through</span>
+<span class="sd">        the `TabMlp` model. Required to slice the tensors. e.g. _{&#39;education&#39;:</span>
+<span class="sd">        0, &#39;relationship&#39;: 1, &#39;workclass&#39;: 2, ...}_.</span>
+<span class="sd">    cat_embed_input: List, Optional, default = None</span>
+<span class="sd">        List of Tuples with the column name and number of unique values and</span>
+<span class="sd">        embedding dimension. e.g. _[(education, 11), ...]_</span>
+<span class="sd">    cat_embed_dropout: float, Optional, default = None</span>
+<span class="sd">        Categorical embeddings dropout. If `None`, it will default</span>
+<span class="sd">        to 0.</span>
+<span class="sd">    use_cat_bias: bool, Optional, default = None,</span>
+<span class="sd">        Boolean indicating if bias will be used for the categorical embeddings.</span>
+<span class="sd">        If `None`, it will default to &#39;False&#39;.</span>
+<span class="sd">    cat_embed_activation: Optional, str, default = None,</span>
+<span class="sd">        Activation function for the categorical embeddings, if any. Currently</span>
+<span class="sd">        _&#39;tanh&#39;_, _&#39;relu&#39;_, _&#39;leaky_relu&#39;_ and _&#39;gelu&#39;_ are supported</span>
+<span class="sd">    shared_embed: bool, Optional, default = None</span>
+<span class="sd">        Boolean indicating if the embeddings will be &quot;shared&quot;. The idea behind `shared_embed` is</span>
+<span class="sd">        described in the Appendix A in the [TabTransformer paper](https://arxiv.org/abs/2012.06678):</span>
+<span class="sd">        _&#39;The goal of having column embedding is to enable the model to</span>
+<span class="sd">        distinguish the classes in one column from those in the other</span>
+<span class="sd">        columns&#39;_. In other words, the idea is to let the model learn which</span>
+<span class="sd">        column is embedded at the time. See: `pytorch_widedeep.models.transformers._layers.SharedEmbeddings`.</span>
+<span class="sd">    add_shared_embed: bool, Optional, default = None</span>
+<span class="sd">        The two embedding sharing strategies are: 1) add the shared embeddings</span>
+<span class="sd">        to the column embeddings or 2) to replace the first</span>
+<span class="sd">        `frac_shared_embed` with the shared embeddings.</span>
+<span class="sd">        See `pytorch_widedeep.models.embeddings_layers.SharedEmbeddings`</span>
+<span class="sd">        If &#39;None&#39; is passed, it will default to &#39;False&#39;.</span>
+<span class="sd">    frac_shared_embed: float, Optional, default = None</span>
+<span class="sd">        The fraction of embeddings that will be shared (if `add_shared_embed</span>
+<span class="sd">        = False`) by all the different categories for one particular</span>
+<span class="sd">        column. If &#39;None&#39; is passed, it will default to 0.0.</span>
+<span class="sd">    continuous_cols: List, Optional, default = None</span>
+<span class="sd">        List with the name of the numeric (aka continuous) columns</span>
+<span class="sd">    cont_norm_layer: str, Optional, default =  None</span>
+<span class="sd">        Type of normalization layer applied to the continuous features.</span>
+<span class="sd">        Options are: _&#39;layernorm&#39;_ and _&#39;batchnorm&#39;_. if `None`, no</span>
+<span class="sd">        normalization layer will be used.</span>
+<span class="sd">    embed_continuous_method: Optional, str, default = None,</span>
+<span class="sd">        Method to use to embed the continuous features. Options are:</span>
+<span class="sd">        _&#39;standard&#39;_, _&#39;periodic&#39;_ or _&#39;piecewise&#39;_. The _&#39;standard&#39;_</span>
+<span class="sd">        embedding method is based on the FT-Transformer implementation</span>
+<span class="sd">        presented in the paper: [Revisiting Deep Learning Models for</span>
+<span class="sd">        Tabular Data](https://arxiv.org/abs/2106.11959v5). The _&#39;periodic&#39;_</span>
+<span class="sd">        and_&#39;piecewise&#39;_ methods were presented in the paper: [On Embeddings for</span>
+<span class="sd">        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556).</span>
+<span class="sd">        Please, read the papers for details.</span>
+<span class="sd">    cont_embed_dropout: float, Optional, default = None,</span>
+<span class="sd">        Dropout for the continuous embeddings. If `None`, it will default to 0.0</span>
+<span class="sd">    cont_embed_activation: Optional, str, default = None,</span>
+<span class="sd">        Activation function for the continuous embeddings if any. Currently</span>
+<span class="sd">        _&#39;tanh&#39;_, _&#39;relu&#39;_, _&#39;leaky_relu&#39;_ and _&#39;gelu&#39;_ are supported.</span>
+<span class="sd">        If `None`, no activation function will be applied.</span>
+<span class="sd">    quantization_setup: Dict[str, List[float]], Optional, default = None,</span>
+<span class="sd">        This parameter is used when the _&#39;piecewise&#39;_ method is used to embed</span>
+<span class="sd">        the continuous cols. It is a dict where keys are the name of the continuous</span>
+<span class="sd">        columns and values are lists with the boundaries for the quantization</span>
+<span class="sd">        of the continuous_cols. See the examples for details. If</span>
+<span class="sd">        If the _&#39;piecewise&#39;_ method is used, this parameter is required.</span>
+<span class="sd">    n_frequencies: int, Optional, default = None,</span>
+<span class="sd">        This is the so called _&#39;k&#39;_ in their paper [On Embeddings for</span>
+<span class="sd">        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556),</span>
+<span class="sd">        and is the number of &#39;frequencies&#39; that will be used to represent each</span>
+<span class="sd">        continuous column. See their Eq 2 in the paper for details. If</span>
+<span class="sd">        the _&#39;periodic&#39;_ method is used, this parameter is required.</span>
+<span class="sd">    sigma: float, Optional, default = None,</span>
+<span class="sd">        This is the sigma parameter in the paper mentioned when describing the</span>
+<span class="sd">        previous parameters and it is used to initialise the &#39;frequency</span>
+<span class="sd">        weights&#39;. See their Eq 2 in the paper for details. If</span>
+<span class="sd">        the _&#39;periodic&#39;_ method is used, this parameter is required.</span>
+<span class="sd">    share_last_layer: bool, Optional, default = None,</span>
+<span class="sd">        This parameter is not present in the before mentioned paper but it is implemented in</span>
+<span class="sd">        the [official repo](https://github.com/yandex-research/rtdl-num-embeddings/tree/main).</span>
+<span class="sd">        If `True` the linear layer that turns the frequencies into embeddings</span>
+<span class="sd">        will be shared across the continuous columns. If `False` a different</span>
+<span class="sd">        linear layer will be used for each continuous column.</span>
+<span class="sd">        If the _&#39;periodic&#39;_ method is used, this parameter is required.</span>
+<span class="sd">    full_embed_dropout: bool, Optional, default = None,</span>
+<span class="sd">        If `True`, the full embedding corresponding to a column will be masked</span>
+<span class="sd">        out/dropout. If `None`, it will default to `False`.</span>
+<span class="sd">    input_dim: int, default = 32</span>
+<span class="sd">        The so-called *dimension of the model*. Is the number of</span>
+<span class="sd">        embeddings used to encode the categorical and/or continuous columns</span>
+<span class="sd">    n_heads: int, default = 8</span>
+<span class="sd">        Number of attention heads per Transformer block</span>
+<span class="sd">    use_qkv_bias: bool, default = False</span>
+<span class="sd">        Boolean indicating whether or not to use bias in the Q, K, and V</span>
+<span class="sd">        projection layers.</span>
+<span class="sd">    n_blocks: int, default = 4</span>
+<span class="sd">        Number of Transformer blocks</span>
+<span class="sd">    attn_dropout: float, default = 0.2</span>
+<span class="sd">        Dropout that will be applied to the Multi-Head Attention layers</span>
+<span class="sd">    ff_dropout: float, default = 0.1</span>
+<span class="sd">        Dropout that will be applied to the FeedForward network</span>
+<span class="sd">    ff_factor: float, default = 4</span>
+<span class="sd">        Multiplicative factor applied to the first layer of the FF network in</span>
+<span class="sd">        each Transformer block, This is normally set to 4.</span>
+<span class="sd">    transformer_activation: str, default = &quot;gelu&quot;</span>
+<span class="sd">        Transformer Encoder activation function. _&#39;tanh&#39;_, _&#39;relu&#39;_,</span>
+<span class="sd">        _&#39;leaky_relu&#39;_, _&#39;gelu&#39;_, _&#39;geglu&#39;_ and _&#39;reglu&#39;_ are supported</span>
+<span class="sd">    use_linear_attention: Boolean, default = False,</span>
+<span class="sd">        Boolean indicating if Linear Attention (from [Transformers are RNNs:</span>
+<span class="sd">        Fast Autoregressive Transformers with Linear Attention](https://arxiv.org/abs/2006.16236))</span>
+<span class="sd">        will be used. The inclusing of this mode of attention is inspired by</span>
+<span class="sd">        [this post](https://www.uber.com/en-GB/blog/deepeta-how-uber-predicts-arrival-times/),</span>
+<span class="sd">        where the Uber team finds that this attention mechanism leads to the</span>
+<span class="sd">        best results for their tabular data.</span>
+<span class="sd">    use_flash_attention: Boolean, default = False,</span>
+<span class="sd">        Boolean indicating if</span>
+<span class="sd">        [Flash Attention](https://pytorch.org/docs/stable/generated/torch.nn.functional.scaled_dot_product_attention.html)</span>
+<span class="sd">        will be used. &lt;br/&gt;</span>
+<span class="sd">    mlp_hidden_dims: List, Optional, default = None</span>
+<span class="sd">        List with the number of neurons per dense layer in the MLP. e.g:</span>
+<span class="sd">        _[64, 32]_. If not provided no MLP on top of the final</span>
+<span class="sd">        Transformer block will be used.</span>
+<span class="sd">    mlp_activation: str, Optional, default = None</span>
+<span class="sd">        Activation function for the dense layers of the MLP. Currently</span>
+<span class="sd">        _&#39;tanh&#39;_, _&#39;relu&#39;_, _&#39;leaky&#39;_relu&#39; and _&#39;gelu&#39;_ are supported.</span>
+<span class="sd">        If &#39;mlp_hidden_dims&#39; is not `None` and this parameter is `None`, it</span>
+<span class="sd">        will default to _&#39;relu&#39;_.</span>
+<span class="sd">    mlp_dropout: float, Optional, default = None</span>
+<span class="sd">        float with the dropout between the dense layers of the MLP.</span>
+<span class="sd">        If &#39;mlp_hidden_dims&#39; is not `None` and this parameter is `None`, it</span>
+<span class="sd">        will default to 0.0.</span>
+<span class="sd">    mlp_batchnorm: bool, Optional, default = None</span>
+<span class="sd">        Boolean indicating whether or not batch normalization will be applied</span>
+<span class="sd">        to the dense layers</span>
+<span class="sd">        If &#39;mlp_hidden_dims&#39; is not `None` and this parameter is `None`, it</span>
+<span class="sd">        will default to False.</span>
+<span class="sd">    mlp_batchnorm_last: bool, Optional, default = None</span>
+<span class="sd">        Boolean indicating whether or not batch normalization will be applied</span>
+<span class="sd">        to the last of the dense layers</span>
+<span class="sd">        If &#39;mlp_hidden_dims&#39; is not `None` and this parameter is `None`, it</span>
+<span class="sd">        will default to False.</span>
+<span class="sd">    mlp_linear_first: bool, Optional, default = None</span>
+<span class="sd">        Boolean indicating the order of the operations in the dense</span>
+<span class="sd">        layer. If `True: [LIN -&gt; ACT -&gt; BN -&gt; DP]`. If `False: [BN -&gt; DP -&gt;</span>
+<span class="sd">        LIN -&gt; ACT]`</span>
+<span class="sd">        If &#39;mlp_hidden_dims&#39; is not `None` and this parameter is `None`, it</span>
+<span class="sd">        will default to `True`.</span>
+
+<span class="sd">    Attributes</span>
+<span class="sd">    ----------</span>
+<span class="sd">    encoder: nn.Module</span>
+<span class="sd">        Sequence of Transformer blocks</span>
+<span class="sd">    mlp: nn.Module</span>
+<span class="sd">        MLP component in the model</span>
+
+<span class="sd">    Examples</span>
+<span class="sd">    --------</span>
+<span class="sd">    &gt;&gt;&gt; import torch</span>
+<span class="sd">    &gt;&gt;&gt; from pytorch_widedeep.models import TabTransformer</span>
+<span class="sd">    &gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)</span>
+<span class="sd">    &gt;&gt;&gt; colnames = [&#39;a&#39;, &#39;b&#39;, &#39;c&#39;, &#39;d&#39;, &#39;e&#39;]</span>
+<span class="sd">    &gt;&gt;&gt; cat_embed_input = [(u,i) for u,i in zip(colnames[:4], [4]*4)]</span>
+<span class="sd">    &gt;&gt;&gt; continuous_cols = [&#39;e&#39;]</span>
+<span class="sd">    &gt;&gt;&gt; column_idx = {k:v for v,k in enumerate(colnames)}</span>
+<span class="sd">    &gt;&gt;&gt; model = TabTransformer(column_idx=column_idx, cat_embed_input=cat_embed_input, continuous_cols=continuous_cols)</span>
+<span class="sd">    &gt;&gt;&gt; out = model(X_tab)</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+
+    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span>
+        <span class="n">column_idx</span><span class="p">:</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">],</span>
+        <span class="o">*</span><span class="p">,</span>
+        <span class="n">cat_embed_input</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="n">Tuple</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">]]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">cat_embed_dropout</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">use_cat_bias</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">cat_embed_activation</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">shared_embed</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">add_shared_embed</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">frac_shared_embed</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">continuous_cols</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">cont_norm_layer</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Literal</span><span class="p">[</span><span class="s2">&quot;batchnorm&quot;</span><span class="p">,</span> <span class="s2">&quot;layernorm&quot;</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">embed_continuous</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">embed_continuous_method</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span>
+            <span class="n">Literal</span><span class="p">[</span><span class="s2">&quot;standard&quot;</span><span class="p">,</span> <span class="s2">&quot;piecewise&quot;</span><span class="p">,</span> <span class="s2">&quot;periodic&quot;</span><span class="p">]</span>
+        <span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">cont_embed_dropout</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">cont_embed_activation</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">quantization_setup</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="nb">float</span><span class="p">]]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">n_frequencies</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">sigma</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">share_last_layer</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">full_embed_dropout</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">input_dim</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">32</span><span class="p">,</span>
+        <span class="n">n_heads</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">8</span><span class="p">,</span>
+        <span class="n">use_qkv_bias</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
+        <span class="n">n_blocks</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">4</span><span class="p">,</span>
+        <span class="n">attn_dropout</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.2</span><span class="p">,</span>
+        <span class="n">ff_dropout</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.1</span><span class="p">,</span>
+        <span class="n">ff_factor</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">4</span><span class="p">,</span>
+        <span class="n">transformer_activation</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;gelu&quot;</span><span class="p">,</span>
+        <span class="n">use_linear_attention</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
+        <span class="n">use_flash_attention</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
+        <span class="n">mlp_hidden_dims</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">int</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">mlp_activation</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;relu&quot;</span><span class="p">,</span>
+        <span class="n">mlp_dropout</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.1</span><span class="p">,</span>
+        <span class="n">mlp_batchnorm</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
+        <span class="n">mlp_batchnorm_last</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
+        <span class="n">mlp_linear_first</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">,</span>
+    <span class="p">):</span>
+        <span class="nb">super</span><span class="p">(</span><span class="n">TabTransformer</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span>
+            <span class="n">column_idx</span><span class="o">=</span><span class="n">column_idx</span><span class="p">,</span>
+            <span class="n">cat_embed_input</span><span class="o">=</span><span class="n">cat_embed_input</span><span class="p">,</span>
+            <span class="n">cat_embed_dropout</span><span class="o">=</span><span class="n">cat_embed_dropout</span><span class="p">,</span>
+            <span class="n">use_cat_bias</span><span class="o">=</span><span class="n">use_cat_bias</span><span class="p">,</span>
+            <span class="n">cat_embed_activation</span><span class="o">=</span><span class="n">cat_embed_activation</span><span class="p">,</span>
+            <span class="n">shared_embed</span><span class="o">=</span><span class="n">shared_embed</span><span class="p">,</span>
+            <span class="n">add_shared_embed</span><span class="o">=</span><span class="n">add_shared_embed</span><span class="p">,</span>
+            <span class="n">frac_shared_embed</span><span class="o">=</span><span class="n">frac_shared_embed</span><span class="p">,</span>
+            <span class="n">continuous_cols</span><span class="o">=</span><span class="n">continuous_cols</span><span class="p">,</span>
+            <span class="n">cont_norm_layer</span><span class="o">=</span><span class="n">cont_norm_layer</span><span class="p">,</span>
+            <span class="c1"># UGLY hack to be able to use the base class __init__ method</span>
+            <span class="n">embed_continuous_method</span><span class="o">=</span><span class="p">(</span>
+                <span class="s2">&quot;standard&quot;</span>
+                <span class="k">if</span> <span class="n">embed_continuous_method</span> <span class="ow">is</span> <span class="kc">None</span>
+                <span class="k">else</span> <span class="n">embed_continuous_method</span>
+            <span class="p">),</span>
+            <span class="n">cont_embed_dropout</span><span class="o">=</span><span class="n">cont_embed_dropout</span><span class="p">,</span>
+            <span class="n">cont_embed_activation</span><span class="o">=</span><span class="n">cont_embed_activation</span><span class="p">,</span>
+            <span class="n">quantization_setup</span><span class="o">=</span><span class="n">quantization_setup</span><span class="p">,</span>
+            <span class="n">n_frequencies</span><span class="o">=</span><span class="n">n_frequencies</span><span class="p">,</span>
+            <span class="n">sigma</span><span class="o">=</span><span class="n">sigma</span><span class="p">,</span>
+            <span class="n">share_last_layer</span><span class="o">=</span><span class="n">share_last_layer</span><span class="p">,</span>
+            <span class="n">input_dim</span><span class="o">=</span><span class="n">input_dim</span><span class="p">,</span>
+            <span class="n">full_embed_dropout</span><span class="o">=</span><span class="n">full_embed_dropout</span><span class="p">,</span>
+        <span class="p">)</span>
 
-    <span class="k">if</span> <span class="n">embed_continuous</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">embed_continuous</span> <span class="o">=</span> <span class="n">embed_continuous</span>
-        <span class="k">if</span> <span class="n">embed_continuous</span> <span class="ow">and</span> <span class="n">embed_continuous_method</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+        <span class="c1"># we overwrite the embed_continuous_method parameter that was set</span>
+        <span class="c1"># to &#39;standard&#39; if embed_continuous_method is None. In other words,</span>
+        <span class="c1"># this will already have the right value if the user provided a value</span>
+        <span class="c1"># for embed_continuous_method, otherwise will be &quot;standard&quot; when it</span>
+        <span class="c1"># should be None</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">embed_continuous_method</span> <span class="o">=</span> <span class="n">embed_continuous_method</span>
+
+        <span class="k">if</span> <span class="n">embed_continuous</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">embed_continuous</span> <span class="o">=</span> <span class="n">embed_continuous</span>
+            <span class="k">if</span> <span class="n">embed_continuous</span> <span class="ow">and</span> <span class="n">embed_continuous_method</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+                <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
+                    <span class="s2">&quot;If &#39;embed_continuous&#39; is True, &#39;embed_continuous_method&#39; must be &quot;</span>
+                    <span class="s2">&quot;one of &#39;standard&#39;, &#39;piecewise&#39; or &#39;periodic&#39;.&quot;</span>
+                <span class="p">)</span>
+        <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">embed_continuous_method</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">embed_continuous</span> <span class="o">=</span> <span class="kc">True</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">embed_continuous</span> <span class="o">=</span> <span class="kc">False</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">n_heads</span> <span class="o">=</span> <span class="n">n_heads</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">use_qkv_bias</span> <span class="o">=</span> <span class="n">use_qkv_bias</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">n_blocks</span> <span class="o">=</span> <span class="n">n_blocks</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">attn_dropout</span> <span class="o">=</span> <span class="n">attn_dropout</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">ff_dropout</span> <span class="o">=</span> <span class="n">ff_dropout</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">transformer_activation</span> <span class="o">=</span> <span class="n">transformer_activation</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">use_linear_attention</span> <span class="o">=</span> <span class="n">use_linear_attention</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">use_flash_attention</span> <span class="o">=</span> <span class="n">use_flash_attention</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">ff_factor</span> <span class="o">=</span> <span class="n">ff_factor</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">mlp_hidden_dims</span> <span class="o">=</span> <span class="n">mlp_hidden_dims</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">mlp_activation</span> <span class="o">=</span> <span class="n">mlp_activation</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">mlp_dropout</span> <span class="o">=</span> <span class="n">mlp_dropout</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">mlp_batchnorm</span> <span class="o">=</span> <span class="n">mlp_batchnorm</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">mlp_batchnorm_last</span> <span class="o">=</span> <span class="n">mlp_batchnorm_last</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">mlp_linear_first</span> <span class="o">=</span> <span class="n">mlp_linear_first</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">with_cls_token</span> <span class="o">=</span> <span class="s2">&quot;cls_token&quot;</span> <span class="ow">in</span> <span class="n">column_idx</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">n_cat</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">cat_embed_input</span><span class="p">)</span> <span class="k">if</span> <span class="n">cat_embed_input</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="k">else</span> <span class="mi">0</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">n_cont</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">continuous_cols</span><span class="p">)</span> <span class="k">if</span> <span class="n">continuous_cols</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="k">else</span> <span class="mi">0</span>
+
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">n_cont</span> <span class="ow">and</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">n_cat</span> <span class="ow">and</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">embed_continuous</span><span class="p">:</span>
             <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
-                <span class="s2">&quot;If &#39;embed_continuous&#39; is True, &#39;embed_continuous_method&#39; must be &quot;</span>
-                <span class="s2">&quot;one of &#39;standard&#39;, &#39;piecewise&#39; or &#39;periodic&#39;.&quot;</span>
+                <span class="s2">&quot;If only continuous features are used &#39;embed_continuous&#39; must be set to &#39;True&#39;&quot;</span>
             <span class="p">)</span>
-    <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">embed_continuous_method</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">embed_continuous</span> <span class="o">=</span> <span class="kc">True</span>
-    <span class="k">else</span><span class="p">:</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">embed_continuous</span> <span class="o">=</span> <span class="kc">False</span>
-
-    <span class="bp">self</span><span class="o">.</span><span class="n">n_heads</span> <span class="o">=</span> <span class="n">n_heads</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">use_qkv_bias</span> <span class="o">=</span> <span class="n">use_qkv_bias</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">n_blocks</span> <span class="o">=</span> <span class="n">n_blocks</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">attn_dropout</span> <span class="o">=</span> <span class="n">attn_dropout</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">ff_dropout</span> <span class="o">=</span> <span class="n">ff_dropout</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">transformer_activation</span> <span class="o">=</span> <span class="n">transformer_activation</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">use_linear_attention</span> <span class="o">=</span> <span class="n">use_linear_attention</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">use_flash_attention</span> <span class="o">=</span> <span class="n">use_flash_attention</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">ff_factor</span> <span class="o">=</span> <span class="n">ff_factor</span>
-
-    <span class="bp">self</span><span class="o">.</span><span class="n">mlp_hidden_dims</span> <span class="o">=</span> <span class="n">mlp_hidden_dims</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">mlp_activation</span> <span class="o">=</span> <span class="n">mlp_activation</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">mlp_dropout</span> <span class="o">=</span> <span class="n">mlp_dropout</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">mlp_batchnorm</span> <span class="o">=</span> <span class="n">mlp_batchnorm</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">mlp_batchnorm_last</span> <span class="o">=</span> <span class="n">mlp_batchnorm_last</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">mlp_linear_first</span> <span class="o">=</span> <span class="n">mlp_linear_first</span>
-
-    <span class="bp">self</span><span class="o">.</span><span class="n">with_cls_token</span> <span class="o">=</span> <span class="s2">&quot;cls_token&quot;</span> <span class="ow">in</span> <span class="n">column_idx</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">n_cat</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">cat_embed_input</span><span class="p">)</span> <span class="k">if</span> <span class="n">cat_embed_input</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="k">else</span> <span class="mi">0</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">n_cont</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">continuous_cols</span><span class="p">)</span> <span class="k">if</span> <span class="n">continuous_cols</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="k">else</span> <span class="mi">0</span>
-
-    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">n_cont</span> <span class="ow">and</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">n_cat</span> <span class="ow">and</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">embed_continuous</span><span class="p">:</span>
-        <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
-            <span class="s2">&quot;If only continuous features are used &#39;embed_continuous&#39; must be set to &#39;True&#39;&quot;</span>
-        <span class="p">)</span>
 
-    <span class="c1"># Embeddings are instantiated at the base model</span>
-    <span class="c1"># Transformer blocks</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">encoder</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Sequential</span><span class="p">()</span>
-    <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">n_blocks</span><span class="p">):</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">encoder</span><span class="o">.</span><span class="n">add_module</span><span class="p">(</span>
-            <span class="s2">&quot;transformer_block&quot;</span> <span class="o">+</span> <span class="nb">str</span><span class="p">(</span><span class="n">i</span><span class="p">),</span>
-            <span class="n">TransformerEncoder</span><span class="p">(</span>
-                <span class="n">input_dim</span><span class="p">,</span>
-                <span class="n">n_heads</span><span class="p">,</span>
-                <span class="n">use_qkv_bias</span><span class="p">,</span>
-                <span class="n">attn_dropout</span><span class="p">,</span>
-                <span class="n">ff_dropout</span><span class="p">,</span>
-                <span class="n">ff_factor</span><span class="p">,</span>
-                <span class="n">transformer_activation</span><span class="p">,</span>
-                <span class="n">use_linear_attention</span><span class="p">,</span>
-                <span class="n">use_flash_attention</span><span class="p">,</span>
-            <span class="p">),</span>
-        <span class="p">)</span>
+        <span class="c1"># Embeddings are instantiated at the base model</span>
+        <span class="c1"># Transformer blocks</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">encoder</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Sequential</span><span class="p">()</span>
+        <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">n_blocks</span><span class="p">):</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">encoder</span><span class="o">.</span><span class="n">add_module</span><span class="p">(</span>
+                <span class="s2">&quot;transformer_block&quot;</span> <span class="o">+</span> <span class="nb">str</span><span class="p">(</span><span class="n">i</span><span class="p">),</span>
+                <span class="n">TransformerEncoder</span><span class="p">(</span>
+                    <span class="n">input_dim</span><span class="p">,</span>
+                    <span class="n">n_heads</span><span class="p">,</span>
+                    <span class="n">use_qkv_bias</span><span class="p">,</span>
+                    <span class="n">attn_dropout</span><span class="p">,</span>
+                    <span class="n">ff_dropout</span><span class="p">,</span>
+                    <span class="n">ff_factor</span><span class="p">,</span>
+                    <span class="n">transformer_activation</span><span class="p">,</span>
+                    <span class="n">use_linear_attention</span><span class="p">,</span>
+                    <span class="n">use_flash_attention</span><span class="p">,</span>
+                <span class="p">),</span>
+            <span class="p">)</span>
 
-    <span class="bp">self</span><span class="o">.</span><span class="n">mlp_first_hidden_dim</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_mlp_first_hidden_dim</span><span class="p">()</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">mlp_first_hidden_dim</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_mlp_first_hidden_dim</span><span class="p">()</span>
 
-    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_hidden_dims</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">mlp</span> <span class="o">=</span> <span class="n">MLP</span><span class="p">(</span>
-            <span class="n">d_hidden</span><span class="o">=</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">mlp_first_hidden_dim</span><span class="p">]</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_hidden_dims</span><span class="p">,</span>
-            <span class="n">activation</span><span class="o">=</span><span class="p">(</span>
-                <span class="s2">&quot;relu&quot;</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_activation</span> <span class="ow">is</span> <span class="kc">None</span> <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_activation</span>
-            <span class="p">),</span>
-            <span class="n">dropout</span><span class="o">=</span><span class="mf">0.0</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_dropout</span> <span class="ow">is</span> <span class="kc">None</span> <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_dropout</span><span class="p">,</span>
-            <span class="n">batchnorm</span><span class="o">=</span><span class="kc">False</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_batchnorm</span> <span class="ow">is</span> <span class="kc">None</span> <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_batchnorm</span><span class="p">,</span>
-            <span class="n">batchnorm_last</span><span class="o">=</span><span class="p">(</span>
-                <span class="kc">False</span>
-                <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_batchnorm_last</span> <span class="ow">is</span> <span class="kc">None</span>
-                <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_batchnorm_last</span>
-            <span class="p">),</span>
-            <span class="n">linear_first</span><span class="o">=</span><span class="p">(</span>
-                <span class="kc">False</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_linear_first</span> <span class="ow">is</span> <span class="kc">None</span> <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_linear_first</span>
-            <span class="p">),</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_hidden_dims</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">mlp</span> <span class="o">=</span> <span class="n">MLP</span><span class="p">(</span>
+                <span class="n">d_hidden</span><span class="o">=</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">mlp_first_hidden_dim</span><span class="p">]</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_hidden_dims</span><span class="p">,</span>
+                <span class="n">activation</span><span class="o">=</span><span class="p">(</span>
+                    <span class="s2">&quot;relu&quot;</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_activation</span> <span class="ow">is</span> <span class="kc">None</span> <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_activation</span>
+                <span class="p">),</span>
+                <span class="n">dropout</span><span class="o">=</span><span class="mf">0.0</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_dropout</span> <span class="ow">is</span> <span class="kc">None</span> <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_dropout</span><span class="p">,</span>
+                <span class="n">batchnorm</span><span class="o">=</span><span class="kc">False</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_batchnorm</span> <span class="ow">is</span> <span class="kc">None</span> <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_batchnorm</span><span class="p">,</span>
+                <span class="n">batchnorm_last</span><span class="o">=</span><span class="p">(</span>
+                    <span class="kc">False</span>
+                    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_batchnorm_last</span> <span class="ow">is</span> <span class="kc">None</span>
+                    <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_batchnorm_last</span>
+                <span class="p">),</span>
+                <span class="n">linear_first</span><span class="o">=</span><span class="p">(</span>
+                    <span class="kc">False</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_linear_first</span> <span class="ow">is</span> <span class="kc">None</span> <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_linear_first</span>
+                <span class="p">),</span>
+            <span class="p">)</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">mlp</span> <span class="o">=</span> <span class="kc">None</span>
+
+    <span class="k">def</span> <span class="nf">forward</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">X</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tensor</span><span class="p">:</span>
+        <span class="n">x_embed</span><span class="p">,</span> <span class="n">x_cont_not_embed</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_embeddings_tt</span><span class="p">(</span><span class="n">X</span><span class="p">)</span>
+        <span class="n">x</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">encoder</span><span class="p">(</span><span class="n">x_embed</span><span class="p">)</span>
+
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">with_cls_token</span><span class="p">:</span>
+            <span class="n">x</span> <span class="o">=</span> <span class="n">x</span><span class="p">[:,</span> <span class="mi">0</span><span class="p">,</span> <span class="p">:]</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="n">x</span> <span class="o">=</span> <span class="n">x</span><span class="o">.</span><span class="n">flatten</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span>
+
+        <span class="k">if</span> <span class="n">x_cont_not_embed</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="n">x</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">cat</span><span class="p">([</span><span class="n">x</span><span class="p">,</span> <span class="n">x_cont_not_embed</span><span class="p">],</span> <span class="mi">1</span><span class="p">)</span>
+
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="n">x</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp</span><span class="p">(</span><span class="n">x</span><span class="p">)</span>
+        <span class="k">return</span> <span class="n">x</span>
+
+    <span class="k">def</span> <span class="nf">_mlp_first_hidden_dim</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">int</span><span class="p">:</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">with_cls_token</span><span class="p">:</span>
+            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">embed_continuous</span><span class="p">:</span>
+                <span class="n">attn_output_dim</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">input_dim</span>
+            <span class="k">else</span><span class="p">:</span>
+                <span class="n">attn_output_dim</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">input_dim</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">n_cont</span>
+        <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">embed_continuous</span><span class="p">:</span>
+            <span class="n">attn_output_dim</span> <span class="o">=</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">n_cat</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">n_cont</span><span class="p">)</span> <span class="o">*</span> <span class="bp">self</span><span class="o">.</span><span class="n">input_dim</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="n">attn_output_dim</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">n_cat</span> <span class="o">*</span> <span class="bp">self</span><span class="o">.</span><span class="n">input_dim</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">n_cont</span>
+
+        <span class="k">return</span> <span class="n">attn_output_dim</span>
+
+    <span class="nd">@property</span>
+    <span class="k">def</span> <span class="nf">output_dim</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">int</span><span class="p">:</span>
+<span class="w">        </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;The output dimension of the model. This is a required property</span>
+<span class="sd">        neccesary to build the `WideDeep` class</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="k">return</span> <span class="p">(</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">mlp_hidden_dims</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span>
+            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_hidden_dims</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span>
+            <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_first_hidden_dim</span>
         <span class="p">)</span>
-    <span class="k">else</span><span class="p">:</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">mlp</span> <span class="o">=</span> <span class="kc">None</span>
+
+    <span class="nd">@property</span>
+    <span class="k">def</span> <span class="nf">attention_weights</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">List</span><span class="p">[</span><span class="n">Tensor</span><span class="p">]:</span>
+<span class="w">        </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;List with the attention weights per block</span>
+
+<span class="sd">        The shape of the attention weights is $(N, H, F, F)$, where $N$ is the</span>
+<span class="sd">        batch size, $H$ is the number of attention heads and $F$ is the</span>
+<span class="sd">        number of features/columns in the dataset</span>
+
+<span class="sd">        :information_source: **NOTE**:</span>
+<span class="sd">        if flash attention or linear attention</span>
+<span class="sd">        are used, no attention weights are saved during the training process</span>
+<span class="sd">        and calling this property will throw a ValueError</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">use_flash_attention</span> <span class="ow">or</span> <span class="bp">self</span><span class="o">.</span><span class="n">use_linear_attention</span><span class="p">:</span>
+            <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
+                <span class="s2">&quot;Extraction of the attention weights is not supported for &quot;</span>
+                <span class="s2">&quot;linear or flash attention&quot;</span>
+            <span class="p">)</span>
+
+        <span class="k">return</span> <span class="p">[</span><span class="n">blk</span><span class="o">.</span><span class="n">attn</span><span class="o">.</span><span class="n">attn_weights</span> <span class="k">for</span> <span class="n">blk</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">encoder</span><span class="p">]</span>
+
+    <span class="k">def</span> <span class="nf">_get_embeddings_tt</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">X</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tuple</span><span class="p">[</span><span class="n">Tensor</span><span class="p">,</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Tensor</span><span class="p">]]:</span>  <span class="c1"># type: ignore[override]</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">n_cont</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">embed_continuous</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">n_cat</span><span class="p">:</span>
+            <span class="c1"># there are continuous and categorical features and the continuous</span>
+            <span class="c1"># features are embedded</span>
+            <span class="n">x_embed</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_embeddings</span><span class="p">(</span><span class="n">X</span><span class="p">)</span>
+            <span class="n">x_cont_not_embed</span> <span class="o">=</span> <span class="kc">None</span>
+        <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">n_cont</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">embed_continuous</span><span class="p">:</span>
+            <span class="c1"># there are only continuous features and they are embedded</span>
+            <span class="n">x_embed</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">cont_norm</span><span class="p">((</span><span class="n">X</span><span class="p">[:,</span> <span class="bp">self</span><span class="o">.</span><span class="n">cont_idx</span><span class="p">]</span><span class="o">.</span><span class="n">float</span><span class="p">()))</span>
+            <span class="n">x_embed</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">cont_embed</span><span class="p">(</span><span class="n">x_embed</span><span class="p">)</span>
+            <span class="n">x_cont_not_embed</span> <span class="o">=</span> <span class="kc">None</span>
+        <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">n_cont</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">n_cat</span><span class="p">:</span>
+            <span class="c1"># there are continuous and categorical features but the continuous</span>
+            <span class="c1"># features are not embedded</span>
+            <span class="n">x_embed</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">cat_embed</span><span class="p">(</span><span class="n">X</span><span class="p">)</span>
+            <span class="n">x_cont_not_embed</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">cont_norm</span><span class="p">((</span><span class="n">X</span><span class="p">[:,</span> <span class="bp">self</span><span class="o">.</span><span class="n">cont_idx</span><span class="p">]</span><span class="o">.</span><span class="n">float</span><span class="p">()))</span>
+        <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">n_cat</span><span class="p">:</span>
+            <span class="c1"># there are only categorical features</span>
+            <span class="n">x_embed</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">cat_embed</span><span class="p">(</span><span class="n">X</span><span class="p">)</span>
+            <span class="n">x_cont_not_embed</span> <span class="o">=</span> <span class="kc">None</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
+                <span class="s2">&quot;Using only continuous features that are not embedded is &quot;</span>
+                <span class="s2">&quot;is not supported in this implementation. Please set &#39;embed_continuous_method&#39; &quot;</span>
+                <span class="s2">&quot;to one of [&#39;standard&#39;, &#39;piecewise&#39;, &#39;periodic&#39;]&quot;</span>
+            <span class="p">)</span>
+        <span class="k">return</span> <span class="n">x_embed</span><span class="p">,</span> <span class="n">x_cont_not_embed</span>
 </code></pre></div></td></tr></table></div>
-                  </details>
+              </details>
 
 
 
@@ -6754,42 +10190,7 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.saint.SAINT" class="doc doc
 
 
 <a href="#pytorch_widedeep.models.tabular.transformers.saint.SAINT" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="nf">SAINT</span><span class="p">(</span>
-    <span class="n">column_idx</span><span class="p">,</span>
-    <span class="o">*</span><span class="p">,</span>
-    <span class="n">cat_embed_input</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">cat_embed_dropout</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">use_cat_bias</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">cat_embed_activation</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">shared_embed</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">add_shared_embed</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">frac_shared_embed</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">continuous_cols</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">cont_norm_layer</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">embed_continuous_method</span><span class="o">=</span><span class="s2">&quot;standard&quot;</span><span class="p">,</span>
-    <span class="n">cont_embed_dropout</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">cont_embed_activation</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">quantization_setup</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">n_frequencies</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">sigma</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">share_last_layer</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">full_embed_dropout</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">input_dim</span><span class="o">=</span><span class="mi">32</span><span class="p">,</span>
-    <span class="n">use_qkv_bias</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
-    <span class="n">n_heads</span><span class="o">=</span><span class="mi">8</span><span class="p">,</span>
-    <span class="n">n_blocks</span><span class="o">=</span><span class="mi">2</span><span class="p">,</span>
-    <span class="n">attn_dropout</span><span class="o">=</span><span class="mf">0.1</span><span class="p">,</span>
-    <span class="n">ff_dropout</span><span class="o">=</span><span class="mf">0.2</span><span class="p">,</span>
-    <span class="n">ff_factor</span><span class="o">=</span><span class="mi">4</span><span class="p">,</span>
-    <span class="n">transformer_activation</span><span class="o">=</span><span class="s2">&quot;gelu&quot;</span><span class="p">,</span>
-    <span class="n">mlp_hidden_dims</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">mlp_activation</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">mlp_dropout</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">mlp_batchnorm</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">mlp_batchnorm_last</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">mlp_linear_first</span><span class="o">=</span><span class="kc">None</span>
-<span class="p">)</span>
-</code></pre></div>
+
 
     <div class="doc doc-contents first">
             <p class="doc doc-class-bases">
@@ -6807,132 +10208,183 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.saint.SAINT" class="doc doc
 
 
 <p><span class="doc-section-title">Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>column_idx</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, int]</code>)
-          –
-          <div class="doc-md-description">
-            <p>Dict containing the index of the columns that will be passed through
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+          <th>Default</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>column_idx</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, int]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Dict containing the index of the columns that will be passed through
 the <code>TabMlp</code> model. Required to slice the tensors. e.g. <em>{'education':
 0, 'relationship': 1, 'workclass': 2, ...}</em>.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>cat_embed_input</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.wdtypes.Tuple">Tuple</span>[str, int]]]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>List of Tuples with the column name and number of unique values and
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>cat_embed_input</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.wdtypes.Tuple">Tuple</span>[str, int]]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>List of Tuples with the column name and number of unique values and
 embedding dimension. e.g. <em>[(education, 11), ...]</em></p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>cat_embed_dropout</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Categorical embeddings dropout. If <code>None</code>, it will default
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>cat_embed_dropout</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Categorical embeddings dropout. If <code>None</code>, it will default
 to 0.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>use_cat_bias</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Boolean indicating if bias will be used for the categorical embeddings.
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>use_cat_bias</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Boolean indicating if bias will be used for the categorical embeddings.
 If <code>None</code>, it will default to 'False'.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>cat_embed_activation</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Activation function for the categorical embeddings, if any. Currently
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>cat_embed_activation</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Activation function for the categorical embeddings, if any. Currently
 <em>'tanh'</em>, <em>'relu'</em>, <em>'leaky_relu'</em> and <em>'gelu'</em> are supported</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>shared_embed</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Boolean indicating if the embeddings will be "shared". The idea behind <code>shared_embed</code> is
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>shared_embed</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Boolean indicating if the embeddings will be "shared". The idea behind <code>shared_embed</code> is
 described in the Appendix A in the <a href="https://arxiv.org/abs/2012.06678">TabTransformer paper</a>:
 <em>'The goal of having column embedding is to enable the model to
 distinguish the classes in one column from those in the other
 columns'</em>. In other words, the idea is to let the model learn which
 column is embedded at the time. See: <code>pytorch_widedeep.models.transformers._layers.SharedEmbeddings</code>.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>add_shared_embed</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>The two embedding sharing strategies are: 1) add the shared embeddings
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>add_shared_embed</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>The two embedding sharing strategies are: 1) add the shared embeddings
 to the column embeddings or 2) to replace the first
 <code>frac_shared_embed</code> with the shared embeddings.
 See <code>pytorch_widedeep.models.embeddings_layers.SharedEmbeddings</code>
 If 'None' is passed, it will default to 'False'.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>frac_shared_embed</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>The fraction of embeddings that will be shared (if <code>add_shared_embed
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>frac_shared_embed</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>The fraction of embeddings that will be shared (if <code>add_shared_embed
 = False</code>) by all the different categories for one particular
 column. If 'None' is passed, it will default to 0.0.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>continuous_cols</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[str]]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>List with the name of the numeric (aka continuous) columns</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>cont_norm_layer</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[batchnorm, layernorm]]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Type of normalization layer applied to the continuous features.
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>continuous_cols</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[str]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>List with the name of the numeric (aka continuous) columns</p>
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>cont_norm_layer</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[batchnorm, layernorm]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Type of normalization layer applied to the continuous features.
 Options are: <em>'layernorm'</em> and <em>'batchnorm'</em>. if <code>None</code>, no
 normalization layer will be used.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>embed_continuous_method</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[standard, piecewise, periodic]]</code>, default:
-                  <code>&#39;standard&#39;</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Method to use to embed the continuous features. Options are:
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>embed_continuous_method</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[standard, piecewise, periodic]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Method to use to embed the continuous features. Options are:
 <em>'standard'</em>, <em>'periodic'</em> or <em>'piecewise'</em>. The <em>'standard'</em>
 embedding method is based on the FT-Transformer implementation
 presented in the paper: <a href="https://arxiv.org/abs/2106.11959v5">Revisiting Deep Learning Models for
@@ -6940,281 +10392,385 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.saint.SAINT" class="doc doc
 and_'piecewise'_ methods were presented in the paper: <a href="https://arxiv.org/abs/2203.05556">On Embeddings for
 Numerical Features in Tabular Deep Learning</a>.
 Please, read the papers for details.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>cont_embed_dropout</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>cont_embed_activation</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Activation function for the continuous embeddings if any. Currently
+              </div>
+            </td>
+            <td>
+                  <code>&#39;standard&#39;</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>cont_embed_dropout</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p>
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>cont_embed_activation</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Activation function for the continuous embeddings if any. Currently
 <em>'tanh'</em>, <em>'relu'</em>, <em>'leaky_relu'</em> and <em>'gelu'</em> are supported.
 If <code>None</code>, no activation function will be applied.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>quantization_setup</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, <span title="pytorch_widedeep.wdtypes.List">List</span>[float]]]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>This parameter is used when the <em>'piecewise'</em> method is used to embed
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>quantization_setup</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, <span title="pytorch_widedeep.wdtypes.List">List</span>[float]]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>This parameter is used when the <em>'piecewise'</em> method is used to embed
 the continuous cols. It is a dict where keys are the name of the continuous
 columns and values are lists with the boundaries for the quantization
 of the continuous_cols. See the examples for details. If
 If the <em>'piecewise'</em> method is used, this parameter is required.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>n_frequencies</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[int]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>This is the so called <em>'k'</em> in their paper <a href="https://arxiv.org/abs/2203.05556">On Embeddings for
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>n_frequencies</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[int]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>This is the so called <em>'k'</em> in their paper <a href="https://arxiv.org/abs/2203.05556">On Embeddings for
 Numerical Features in Tabular Deep Learning</a>,
 and is the number of 'frequencies' that will be used to represent each
 continuous column. See their Eq 2 in the paper for details. If
 the <em>'periodic'</em> method is used, this parameter is required.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>sigma</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>This is the sigma parameter in the paper mentioned when describing the
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>sigma</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>This is the sigma parameter in the paper mentioned when describing the
 previous parameters and it is used to initialise the 'frequency
 weights'. See their Eq 2 in the paper for details. If
 the <em>'periodic'</em> method is used, this parameter is required.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>share_last_layer</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>This parameter is not present in the before mentioned paper but it is implemented in
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>share_last_layer</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>This parameter is not present in the before mentioned paper but it is implemented in
 the <a href="https://github.com/yandex-research/rtdl-num-embeddings/tree/main">official repo</a>.
 If <code>True</code> the linear layer that turns the frequencies into embeddings
 will be shared across the continuous columns. If <code>False</code> a different
 linear layer will be used for each continuous column.
 If the <em>'periodic'</em> method is used, this parameter is required.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>full_embed_dropout</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>If <code>True</code>, the full embedding corresponding to a column will be masked
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>full_embed_dropout</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>If <code>True</code>, the full embedding corresponding to a column will be masked
 out/dropout. If <code>None</code>, it will default to <code>False</code>.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>input_dim</code></b>
-              (<code>int</code>, default:
-                  <code>32</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>The so-called <em>dimension of the model</em>. Is the number of
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>input_dim</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>The so-called <em>dimension of the model</em>. Is the number of
 embeddings used to encode the categorical and/or continuous columns</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>n_heads</code></b>
-              (<code>int</code>, default:
+              </div>
+            </td>
+            <td>
+                  <code>32</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>n_heads</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Number of attention heads per Transformer block</p>
+              </div>
+            </td>
+            <td>
                   <code>8</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Number of attention heads per Transformer block</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>use_qkv_bias</code></b>
-              (<code>bool</code>, default:
-                  <code>False</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Boolean indicating whether or not to use bias in the Q, K, and V
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>use_qkv_bias</code></td>
+            <td>
+                  <code>bool</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Boolean indicating whether or not to use bias in the Q, K, and V
 projection layers</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>n_blocks</code></b>
-              (<code>int</code>, default:
+              </div>
+            </td>
+            <td>
+                  <code>False</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>n_blocks</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Number of SAINT-Transformer blocks.</p>
+              </div>
+            </td>
+            <td>
                   <code>2</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Number of SAINT-Transformer blocks.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>attn_dropout</code></b>
-              (<code>float</code>, default:
-                  <code>0.1</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Dropout that will be applied to the Multi-Head Attention column and
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>attn_dropout</code></td>
+            <td>
+                  <code>float</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Dropout that will be applied to the Multi-Head Attention column and
 row layers</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>ff_dropout</code></b>
-              (<code>float</code>, default:
+              </div>
+            </td>
+            <td>
+                  <code>0.1</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>ff_dropout</code></td>
+            <td>
+                  <code>float</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Dropout that will be applied to the FeedForward network</p>
+              </div>
+            </td>
+            <td>
                   <code>0.2</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Dropout that will be applied to the FeedForward network</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>ff_factor</code></b>
-              (<code>int</code>, default:
-                  <code>4</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Multiplicative factor applied to the first layer of the FF network in
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>ff_factor</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Multiplicative factor applied to the first layer of the FF network in
 each Transformer block, This is normally set to 4.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>transformer_activation</code></b>
-              (<code>str</code>, default:
-                  <code>&#39;gelu&#39;</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Transformer Encoder activation function. <em>'tanh'</em>, <em>'relu'</em>,
+              </div>
+            </td>
+            <td>
+                  <code>4</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>transformer_activation</code></td>
+            <td>
+                  <code>str</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Transformer Encoder activation function. <em>'tanh'</em>, <em>'relu'</em>,
 <em>'leaky_relu'</em>, <em>'gelu'</em>, <em>'geglu'</em> and <em>'reglu'</em> are supported</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>mlp_hidden_dims</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[int]]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>List with the number of neurons per dense layer in the MLP. e.g:
+              </div>
+            </td>
+            <td>
+                  <code>&#39;gelu&#39;</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>mlp_hidden_dims</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[int]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>List with the number of neurons per dense layer in the MLP. e.g:
 <em>[64, 32]</em>. If not provided no MLP on top of the final
 Transformer block will be used.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>mlp_activation</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Activation function for the dense layers of the MLP. Currently
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>mlp_activation</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Activation function for the dense layers of the MLP. Currently
 <em>'tanh'</em>, <em>'relu'</em>, <em>'leaky'_relu' and _'gelu'</em> are supported.
 If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it
 will default to <em>'relu'</em>.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>mlp_dropout</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>float with the dropout between the dense layers of the MLP.
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>mlp_dropout</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>float with the dropout between the dense layers of the MLP.
 If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it
 will default to 0.0.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>mlp_batchnorm</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Boolean indicating whether or not batch normalization will be applied
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>mlp_batchnorm</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Boolean indicating whether or not batch normalization will be applied
 to the dense layers
 If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it
 will default to False.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>mlp_batchnorm_last</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Boolean indicating whether or not batch normalization will be applied
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>mlp_batchnorm_last</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Boolean indicating whether or not batch normalization will be applied
 to the last of the dense layers
 If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it
 will default to False.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>mlp_linear_first</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Boolean indicating the order of the operations in the dense
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>mlp_linear_first</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Boolean indicating the order of the operations in the dense
 layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt;
 LIN -&gt; ACT]</code>
 If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it
 will default to <code>True</code>.</p>
-          </div>
-        </li>
-    </ul>
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
 
 <p><span class="doc-section-title">Attributes:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code><span title="pytorch_widedeep.models.tabular.transformers.saint.SAINT.encoder">encoder</span></code></b>
-              (<code><span title="torch.nn.Module">Module</span></code>)
-          –
-          <div class="doc-md-description">
-            <p>Sequence of SAINT-Transformer blocks</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code><span title="pytorch_widedeep.models.tabular.transformers.saint.SAINT.mlp">mlp</span></code></b>
-              (<code><span title="torch.nn.Module">Module</span></code>)
-          –
-          <div class="doc-md-description">
-            <p>MLP component in the model</p>
-          </div>
-        </li>
-    </ul>
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code><span title="pytorch_widedeep.models.tabular.transformers.saint.SAINT.encoder">encoder</span></code></td>
+            <td>
+                  <code><span title="torch.nn.Module">Module</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Sequence of SAINT-Transformer blocks</p>
+              </div>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code><span title="pytorch_widedeep.models.tabular.transformers.saint.SAINT.mlp">mlp</span></code></td>
+            <td>
+                  <code><span title="torch.nn.Module">Module</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>MLP component in the model</p>
+              </div>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
 
 <p><span class="doc-section-title">Examples:</span></p>
@@ -7229,9 +10785,175 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.saint.SAINT" class="doc doc
 <span class="gp">&gt;&gt;&gt; </span><span class="n">out</span> <span class="o">=</span> <span class="n">model</span><span class="p">(</span><span class="n">X_tab</span><span class="p">)</span>
 </code></pre></div>
 
-                  <details class="quote">
-                    <summary>Source code in <code>pytorch_widedeep/models/tabular/transformers/saint.py</code></summary>
-                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">177</span>
+              <details class="quote">
+                <summary>Source code in <code>pytorch_widedeep/models/tabular/transformers/saint.py</code></summary>
+                <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 11</span>
+<span class="normal"> 12</span>
+<span class="normal"> 13</span>
+<span class="normal"> 14</span>
+<span class="normal"> 15</span>
+<span class="normal"> 16</span>
+<span class="normal"> 17</span>
+<span class="normal"> 18</span>
+<span class="normal"> 19</span>
+<span class="normal"> 20</span>
+<span class="normal"> 21</span>
+<span class="normal"> 22</span>
+<span class="normal"> 23</span>
+<span class="normal"> 24</span>
+<span class="normal"> 25</span>
+<span class="normal"> 26</span>
+<span class="normal"> 27</span>
+<span class="normal"> 28</span>
+<span class="normal"> 29</span>
+<span class="normal"> 30</span>
+<span class="normal"> 31</span>
+<span class="normal"> 32</span>
+<span class="normal"> 33</span>
+<span class="normal"> 34</span>
+<span class="normal"> 35</span>
+<span class="normal"> 36</span>
+<span class="normal"> 37</span>
+<span class="normal"> 38</span>
+<span class="normal"> 39</span>
+<span class="normal"> 40</span>
+<span class="normal"> 41</span>
+<span class="normal"> 42</span>
+<span class="normal"> 43</span>
+<span class="normal"> 44</span>
+<span class="normal"> 45</span>
+<span class="normal"> 46</span>
+<span class="normal"> 47</span>
+<span class="normal"> 48</span>
+<span class="normal"> 49</span>
+<span class="normal"> 50</span>
+<span class="normal"> 51</span>
+<span class="normal"> 52</span>
+<span class="normal"> 53</span>
+<span class="normal"> 54</span>
+<span class="normal"> 55</span>
+<span class="normal"> 56</span>
+<span class="normal"> 57</span>
+<span class="normal"> 58</span>
+<span class="normal"> 59</span>
+<span class="normal"> 60</span>
+<span class="normal"> 61</span>
+<span class="normal"> 62</span>
+<span class="normal"> 63</span>
+<span class="normal"> 64</span>
+<span class="normal"> 65</span>
+<span class="normal"> 66</span>
+<span class="normal"> 67</span>
+<span class="normal"> 68</span>
+<span class="normal"> 69</span>
+<span class="normal"> 70</span>
+<span class="normal"> 71</span>
+<span class="normal"> 72</span>
+<span class="normal"> 73</span>
+<span class="normal"> 74</span>
+<span class="normal"> 75</span>
+<span class="normal"> 76</span>
+<span class="normal"> 77</span>
+<span class="normal"> 78</span>
+<span class="normal"> 79</span>
+<span class="normal"> 80</span>
+<span class="normal"> 81</span>
+<span class="normal"> 82</span>
+<span class="normal"> 83</span>
+<span class="normal"> 84</span>
+<span class="normal"> 85</span>
+<span class="normal"> 86</span>
+<span class="normal"> 87</span>
+<span class="normal"> 88</span>
+<span class="normal"> 89</span>
+<span class="normal"> 90</span>
+<span class="normal"> 91</span>
+<span class="normal"> 92</span>
+<span class="normal"> 93</span>
+<span class="normal"> 94</span>
+<span class="normal"> 95</span>
+<span class="normal"> 96</span>
+<span class="normal"> 97</span>
+<span class="normal"> 98</span>
+<span class="normal"> 99</span>
+<span class="normal">100</span>
+<span class="normal">101</span>
+<span class="normal">102</span>
+<span class="normal">103</span>
+<span class="normal">104</span>
+<span class="normal">105</span>
+<span class="normal">106</span>
+<span class="normal">107</span>
+<span class="normal">108</span>
+<span class="normal">109</span>
+<span class="normal">110</span>
+<span class="normal">111</span>
+<span class="normal">112</span>
+<span class="normal">113</span>
+<span class="normal">114</span>
+<span class="normal">115</span>
+<span class="normal">116</span>
+<span class="normal">117</span>
+<span class="normal">118</span>
+<span class="normal">119</span>
+<span class="normal">120</span>
+<span class="normal">121</span>
+<span class="normal">122</span>
+<span class="normal">123</span>
+<span class="normal">124</span>
+<span class="normal">125</span>
+<span class="normal">126</span>
+<span class="normal">127</span>
+<span class="normal">128</span>
+<span class="normal">129</span>
+<span class="normal">130</span>
+<span class="normal">131</span>
+<span class="normal">132</span>
+<span class="normal">133</span>
+<span class="normal">134</span>
+<span class="normal">135</span>
+<span class="normal">136</span>
+<span class="normal">137</span>
+<span class="normal">138</span>
+<span class="normal">139</span>
+<span class="normal">140</span>
+<span class="normal">141</span>
+<span class="normal">142</span>
+<span class="normal">143</span>
+<span class="normal">144</span>
+<span class="normal">145</span>
+<span class="normal">146</span>
+<span class="normal">147</span>
+<span class="normal">148</span>
+<span class="normal">149</span>
+<span class="normal">150</span>
+<span class="normal">151</span>
+<span class="normal">152</span>
+<span class="normal">153</span>
+<span class="normal">154</span>
+<span class="normal">155</span>
+<span class="normal">156</span>
+<span class="normal">157</span>
+<span class="normal">158</span>
+<span class="normal">159</span>
+<span class="normal">160</span>
+<span class="normal">161</span>
+<span class="normal">162</span>
+<span class="normal">163</span>
+<span class="normal">164</span>
+<span class="normal">165</span>
+<span class="normal">166</span>
+<span class="normal">167</span>
+<span class="normal">168</span>
+<span class="normal">169</span>
+<span class="normal">170</span>
+<span class="normal">171</span>
+<span class="normal">172</span>
+<span class="normal">173</span>
+<span class="normal">174</span>
+<span class="normal">175</span>
+<span class="normal">176</span>
+<span class="normal">177</span>
 <span class="normal">178</span>
 <span class="normal">179</span>
 <span class="normal">180</span>
@@ -7353,131 +11075,385 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.saint.SAINT" class="doc doc
 <span class="normal">296</span>
 <span class="normal">297</span>
 <span class="normal">298</span>
-<span class="normal">299</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
-    <span class="bp">self</span><span class="p">,</span>
-    <span class="n">column_idx</span><span class="p">:</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">],</span>
-    <span class="o">*</span><span class="p">,</span>
-    <span class="n">cat_embed_input</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="n">Tuple</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">]]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">cat_embed_dropout</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">use_cat_bias</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">cat_embed_activation</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">shared_embed</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">add_shared_embed</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">frac_shared_embed</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">continuous_cols</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">cont_norm_layer</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Literal</span><span class="p">[</span><span class="s2">&quot;batchnorm&quot;</span><span class="p">,</span> <span class="s2">&quot;layernorm&quot;</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">embed_continuous_method</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span>
-        <span class="n">Literal</span><span class="p">[</span><span class="s2">&quot;standard&quot;</span><span class="p">,</span> <span class="s2">&quot;piecewise&quot;</span><span class="p">,</span> <span class="s2">&quot;periodic&quot;</span><span class="p">]</span>
-    <span class="p">]</span> <span class="o">=</span> <span class="s2">&quot;standard&quot;</span><span class="p">,</span>
-    <span class="n">cont_embed_dropout</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">cont_embed_activation</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">quantization_setup</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="nb">float</span><span class="p">]]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">n_frequencies</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">sigma</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">share_last_layer</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">full_embed_dropout</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">input_dim</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">32</span><span class="p">,</span>
-    <span class="n">use_qkv_bias</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
-    <span class="n">n_heads</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">8</span><span class="p">,</span>
-    <span class="n">n_blocks</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">2</span><span class="p">,</span>
-    <span class="n">attn_dropout</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.1</span><span class="p">,</span>
-    <span class="n">ff_dropout</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.2</span><span class="p">,</span>
-    <span class="n">ff_factor</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">4</span><span class="p">,</span>
-    <span class="n">transformer_activation</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;gelu&quot;</span><span class="p">,</span>
-    <span class="n">mlp_hidden_dims</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">int</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">mlp_activation</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">mlp_dropout</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">mlp_batchnorm</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">mlp_batchnorm_last</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">mlp_linear_first</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-<span class="p">):</span>
-    <span class="nb">super</span><span class="p">(</span><span class="n">SAINT</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span>
-        <span class="n">column_idx</span><span class="o">=</span><span class="n">column_idx</span><span class="p">,</span>
-        <span class="n">cat_embed_input</span><span class="o">=</span><span class="n">cat_embed_input</span><span class="p">,</span>
-        <span class="n">cat_embed_dropout</span><span class="o">=</span><span class="n">cat_embed_dropout</span><span class="p">,</span>
-        <span class="n">use_cat_bias</span><span class="o">=</span><span class="n">use_cat_bias</span><span class="p">,</span>
-        <span class="n">cat_embed_activation</span><span class="o">=</span><span class="n">cat_embed_activation</span><span class="p">,</span>
-        <span class="n">shared_embed</span><span class="o">=</span><span class="n">shared_embed</span><span class="p">,</span>
-        <span class="n">add_shared_embed</span><span class="o">=</span><span class="n">add_shared_embed</span><span class="p">,</span>
-        <span class="n">frac_shared_embed</span><span class="o">=</span><span class="n">frac_shared_embed</span><span class="p">,</span>
-        <span class="n">continuous_cols</span><span class="o">=</span><span class="n">continuous_cols</span><span class="p">,</span>
-        <span class="n">cont_norm_layer</span><span class="o">=</span><span class="n">cont_norm_layer</span><span class="p">,</span>
-        <span class="n">embed_continuous_method</span><span class="o">=</span><span class="n">embed_continuous_method</span><span class="p">,</span>
-        <span class="n">cont_embed_dropout</span><span class="o">=</span><span class="n">cont_embed_dropout</span><span class="p">,</span>
-        <span class="n">cont_embed_activation</span><span class="o">=</span><span class="n">cont_embed_activation</span><span class="p">,</span>
-        <span class="n">input_dim</span><span class="o">=</span><span class="n">input_dim</span><span class="p">,</span>
-        <span class="n">quantization_setup</span><span class="o">=</span><span class="n">quantization_setup</span><span class="p">,</span>
-        <span class="n">n_frequencies</span><span class="o">=</span><span class="n">n_frequencies</span><span class="p">,</span>
-        <span class="n">sigma</span><span class="o">=</span><span class="n">sigma</span><span class="p">,</span>
-        <span class="n">share_last_layer</span><span class="o">=</span><span class="n">share_last_layer</span><span class="p">,</span>
-        <span class="n">full_embed_dropout</span><span class="o">=</span><span class="n">full_embed_dropout</span><span class="p">,</span>
-    <span class="p">)</span>
-
-    <span class="bp">self</span><span class="o">.</span><span class="n">use_qkv_bias</span> <span class="o">=</span> <span class="n">use_qkv_bias</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">n_heads</span> <span class="o">=</span> <span class="n">n_heads</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">n_blocks</span> <span class="o">=</span> <span class="n">n_blocks</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">attn_dropout</span> <span class="o">=</span> <span class="n">attn_dropout</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">ff_dropout</span> <span class="o">=</span> <span class="n">ff_dropout</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">ff_factor</span> <span class="o">=</span> <span class="n">ff_factor</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">transformer_activation</span> <span class="o">=</span> <span class="n">transformer_activation</span>
-
-    <span class="bp">self</span><span class="o">.</span><span class="n">mlp_hidden_dims</span> <span class="o">=</span> <span class="n">mlp_hidden_dims</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">mlp_activation</span> <span class="o">=</span> <span class="n">mlp_activation</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">mlp_dropout</span> <span class="o">=</span> <span class="n">mlp_dropout</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">mlp_batchnorm</span> <span class="o">=</span> <span class="n">mlp_batchnorm</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">mlp_batchnorm_last</span> <span class="o">=</span> <span class="n">mlp_batchnorm_last</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">mlp_linear_first</span> <span class="o">=</span> <span class="n">mlp_linear_first</span>
-
-    <span class="bp">self</span><span class="o">.</span><span class="n">with_cls_token</span> <span class="o">=</span> <span class="s2">&quot;cls_token&quot;</span> <span class="ow">in</span> <span class="n">column_idx</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">n_cat</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">cat_embed_input</span><span class="p">)</span> <span class="k">if</span> <span class="n">cat_embed_input</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="k">else</span> <span class="mi">0</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">n_cont</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">continuous_cols</span><span class="p">)</span> <span class="k">if</span> <span class="n">continuous_cols</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="k">else</span> <span class="mi">0</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">n_feats</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">n_cat</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">n_cont</span>
-
-    <span class="c1"># Embeddings are instantiated at the base model</span>
-    <span class="c1"># Transformer blocks</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">encoder</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Sequential</span><span class="p">()</span>
-    <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">n_blocks</span><span class="p">):</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">encoder</span><span class="o">.</span><span class="n">add_module</span><span class="p">(</span>
-            <span class="s2">&quot;saint_block&quot;</span> <span class="o">+</span> <span class="nb">str</span><span class="p">(</span><span class="n">i</span><span class="p">),</span>
-            <span class="n">SaintEncoder</span><span class="p">(</span>
-                <span class="n">input_dim</span><span class="p">,</span>
-                <span class="n">n_heads</span><span class="p">,</span>
-                <span class="n">use_qkv_bias</span><span class="p">,</span>
-                <span class="n">attn_dropout</span><span class="p">,</span>
-                <span class="n">ff_dropout</span><span class="p">,</span>
-                <span class="n">ff_factor</span><span class="p">,</span>
-                <span class="n">transformer_activation</span><span class="p">,</span>
-                <span class="bp">self</span><span class="o">.</span><span class="n">n_feats</span><span class="p">,</span>
-            <span class="p">),</span>
+<span class="normal">299</span>
+<span class="normal">300</span>
+<span class="normal">301</span>
+<span class="normal">302</span>
+<span class="normal">303</span>
+<span class="normal">304</span>
+<span class="normal">305</span>
+<span class="normal">306</span>
+<span class="normal">307</span>
+<span class="normal">308</span>
+<span class="normal">309</span>
+<span class="normal">310</span>
+<span class="normal">311</span>
+<span class="normal">312</span>
+<span class="normal">313</span>
+<span class="normal">314</span>
+<span class="normal">315</span>
+<span class="normal">316</span>
+<span class="normal">317</span>
+<span class="normal">318</span>
+<span class="normal">319</span>
+<span class="normal">320</span>
+<span class="normal">321</span>
+<span class="normal">322</span>
+<span class="normal">323</span>
+<span class="normal">324</span>
+<span class="normal">325</span>
+<span class="normal">326</span>
+<span class="normal">327</span>
+<span class="normal">328</span>
+<span class="normal">329</span>
+<span class="normal">330</span>
+<span class="normal">331</span>
+<span class="normal">332</span>
+<span class="normal">333</span>
+<span class="normal">334</span>
+<span class="normal">335</span>
+<span class="normal">336</span>
+<span class="normal">337</span>
+<span class="normal">338</span>
+<span class="normal">339</span>
+<span class="normal">340</span>
+<span class="normal">341</span>
+<span class="normal">342</span>
+<span class="normal">343</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">class</span> <span class="nc">SAINT</span><span class="p">(</span><span class="n">BaseTabularModelWithAttention</span><span class="p">):</span>
+<span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Defines a [SAINT model](https://arxiv.org/abs/2106.01342) that</span>
+<span class="sd">    can be used as the `deeptabular` component of a Wide &amp; Deep model or</span>
+<span class="sd">    independently by itself.</span>
+
+<span class="sd">    Most of the parameters for this class are `Optional` since the use of</span>
+<span class="sd">    categorical or continuous is in fact optional (i.e. one can use</span>
+<span class="sd">    categorical features only, continuous features only or both).</span>
+
+<span class="sd">    :information_source: **NOTE**: This is an slightly modified and enhanced</span>
+<span class="sd">     version of the model described in the paper,</span>
+
+<span class="sd">    Parameters</span>
+<span class="sd">    ----------</span>
+<span class="sd">    column_idx: Dict</span>
+<span class="sd">        Dict containing the index of the columns that will be passed through</span>
+<span class="sd">        the `TabMlp` model. Required to slice the tensors. e.g. _{&#39;education&#39;:</span>
+<span class="sd">        0, &#39;relationship&#39;: 1, &#39;workclass&#39;: 2, ...}_.</span>
+<span class="sd">    cat_embed_input: List, Optional, default = None</span>
+<span class="sd">        List of Tuples with the column name and number of unique values and</span>
+<span class="sd">        embedding dimension. e.g. _[(education, 11), ...]_</span>
+<span class="sd">    cat_embed_dropout: float, Optional, default = None</span>
+<span class="sd">        Categorical embeddings dropout. If `None`, it will default</span>
+<span class="sd">        to 0.</span>
+<span class="sd">    use_cat_bias: bool, Optional, default = None,</span>
+<span class="sd">        Boolean indicating if bias will be used for the categorical embeddings.</span>
+<span class="sd">        If `None`, it will default to &#39;False&#39;.</span>
+<span class="sd">    cat_embed_activation: Optional, str, default = None,</span>
+<span class="sd">        Activation function for the categorical embeddings, if any. Currently</span>
+<span class="sd">        _&#39;tanh&#39;_, _&#39;relu&#39;_, _&#39;leaky_relu&#39;_ and _&#39;gelu&#39;_ are supported</span>
+<span class="sd">    shared_embed: bool, Optional, default = None</span>
+<span class="sd">        Boolean indicating if the embeddings will be &quot;shared&quot;. The idea behind `shared_embed` is</span>
+<span class="sd">        described in the Appendix A in the [TabTransformer paper](https://arxiv.org/abs/2012.06678):</span>
+<span class="sd">        _&#39;The goal of having column embedding is to enable the model to</span>
+<span class="sd">        distinguish the classes in one column from those in the other</span>
+<span class="sd">        columns&#39;_. In other words, the idea is to let the model learn which</span>
+<span class="sd">        column is embedded at the time. See: `pytorch_widedeep.models.transformers._layers.SharedEmbeddings`.</span>
+<span class="sd">    add_shared_embed: bool, Optional, default = None</span>
+<span class="sd">        The two embedding sharing strategies are: 1) add the shared embeddings</span>
+<span class="sd">        to the column embeddings or 2) to replace the first</span>
+<span class="sd">        `frac_shared_embed` with the shared embeddings.</span>
+<span class="sd">        See `pytorch_widedeep.models.embeddings_layers.SharedEmbeddings`</span>
+<span class="sd">        If &#39;None&#39; is passed, it will default to &#39;False&#39;.</span>
+<span class="sd">    frac_shared_embed: float, Optional, default = None</span>
+<span class="sd">        The fraction of embeddings that will be shared (if `add_shared_embed</span>
+<span class="sd">        = False`) by all the different categories for one particular</span>
+<span class="sd">        column. If &#39;None&#39; is passed, it will default to 0.0.</span>
+<span class="sd">    continuous_cols: List, Optional, default = None</span>
+<span class="sd">        List with the name of the numeric (aka continuous) columns</span>
+<span class="sd">    cont_norm_layer: str, Optional, default =  None</span>
+<span class="sd">        Type of normalization layer applied to the continuous features.</span>
+<span class="sd">        Options are: _&#39;layernorm&#39;_ and _&#39;batchnorm&#39;_. if `None`, no</span>
+<span class="sd">        normalization layer will be used.</span>
+<span class="sd">    embed_continuous_method: Optional, str, default = None,</span>
+<span class="sd">        Method to use to embed the continuous features. Options are:</span>
+<span class="sd">        _&#39;standard&#39;_, _&#39;periodic&#39;_ or _&#39;piecewise&#39;_. The _&#39;standard&#39;_</span>
+<span class="sd">        embedding method is based on the FT-Transformer implementation</span>
+<span class="sd">        presented in the paper: [Revisiting Deep Learning Models for</span>
+<span class="sd">        Tabular Data](https://arxiv.org/abs/2106.11959v5). The _&#39;periodic&#39;_</span>
+<span class="sd">        and_&#39;piecewise&#39;_ methods were presented in the paper: [On Embeddings for</span>
+<span class="sd">        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556).</span>
+<span class="sd">        Please, read the papers for details.</span>
+<span class="sd">    cont_embed_dropout: float, Optional, default = None,</span>
+<span class="sd">        Dropout for the continuous embeddings. If `None`, it will default to 0.0</span>
+<span class="sd">    cont_embed_activation: Optional, str, default = None,</span>
+<span class="sd">        Activation function for the continuous embeddings if any. Currently</span>
+<span class="sd">        _&#39;tanh&#39;_, _&#39;relu&#39;_, _&#39;leaky_relu&#39;_ and _&#39;gelu&#39;_ are supported.</span>
+<span class="sd">        If `None`, no activation function will be applied.</span>
+<span class="sd">    quantization_setup: Dict[str, List[float]], Optional, default = None,</span>
+<span class="sd">        This parameter is used when the _&#39;piecewise&#39;_ method is used to embed</span>
+<span class="sd">        the continuous cols. It is a dict where keys are the name of the continuous</span>
+<span class="sd">        columns and values are lists with the boundaries for the quantization</span>
+<span class="sd">        of the continuous_cols. See the examples for details. If</span>
+<span class="sd">        If the _&#39;piecewise&#39;_ method is used, this parameter is required.</span>
+<span class="sd">    n_frequencies: int, Optional, default = None,</span>
+<span class="sd">        This is the so called _&#39;k&#39;_ in their paper [On Embeddings for</span>
+<span class="sd">        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556),</span>
+<span class="sd">        and is the number of &#39;frequencies&#39; that will be used to represent each</span>
+<span class="sd">        continuous column. See their Eq 2 in the paper for details. If</span>
+<span class="sd">        the _&#39;periodic&#39;_ method is used, this parameter is required.</span>
+<span class="sd">    sigma: float, Optional, default = None,</span>
+<span class="sd">        This is the sigma parameter in the paper mentioned when describing the</span>
+<span class="sd">        previous parameters and it is used to initialise the &#39;frequency</span>
+<span class="sd">        weights&#39;. See their Eq 2 in the paper for details. If</span>
+<span class="sd">        the _&#39;periodic&#39;_ method is used, this parameter is required.</span>
+<span class="sd">    share_last_layer: bool, Optional, default = None,</span>
+<span class="sd">        This parameter is not present in the before mentioned paper but it is implemented in</span>
+<span class="sd">        the [official repo](https://github.com/yandex-research/rtdl-num-embeddings/tree/main).</span>
+<span class="sd">        If `True` the linear layer that turns the frequencies into embeddings</span>
+<span class="sd">        will be shared across the continuous columns. If `False` a different</span>
+<span class="sd">        linear layer will be used for each continuous column.</span>
+<span class="sd">        If the _&#39;periodic&#39;_ method is used, this parameter is required.</span>
+<span class="sd">    full_embed_dropout: bool, Optional, default = None,</span>
+<span class="sd">        If `True`, the full embedding corresponding to a column will be masked</span>
+<span class="sd">        out/dropout. If `None`, it will default to `False`.</span>
+<span class="sd">    input_dim: int, default = 32</span>
+<span class="sd">        The so-called *dimension of the model*. Is the number of</span>
+<span class="sd">        embeddings used to encode the categorical and/or continuous columns</span>
+<span class="sd">    n_heads: int, default = 8</span>
+<span class="sd">        Number of attention heads per Transformer block</span>
+<span class="sd">    use_qkv_bias: bool, default = False</span>
+<span class="sd">        Boolean indicating whether or not to use bias in the Q, K, and V</span>
+<span class="sd">        projection layers</span>
+<span class="sd">    n_blocks: int, default = 2</span>
+<span class="sd">        Number of SAINT-Transformer blocks.</span>
+<span class="sd">    attn_dropout: float, default = 0.2</span>
+<span class="sd">        Dropout that will be applied to the Multi-Head Attention column and</span>
+<span class="sd">        row layers</span>
+<span class="sd">    ff_dropout: float, default = 0.1</span>
+<span class="sd">        Dropout that will be applied to the FeedForward network</span>
+<span class="sd">    ff_factor: float, default = 4</span>
+<span class="sd">        Multiplicative factor applied to the first layer of the FF network in</span>
+<span class="sd">        each Transformer block, This is normally set to 4.</span>
+<span class="sd">    transformer_activation: str, default = &quot;gelu&quot;</span>
+<span class="sd">        Transformer Encoder activation function. _&#39;tanh&#39;_, _&#39;relu&#39;_,</span>
+<span class="sd">        _&#39;leaky_relu&#39;_, _&#39;gelu&#39;_, _&#39;geglu&#39;_ and _&#39;reglu&#39;_ are supported</span>
+<span class="sd">    mlp_hidden_dims: List, Optional, default = None</span>
+<span class="sd">        List with the number of neurons per dense layer in the MLP. e.g:</span>
+<span class="sd">        _[64, 32]_. If not provided no MLP on top of the final</span>
+<span class="sd">        Transformer block will be used.</span>
+<span class="sd">    mlp_activation: str, Optional, default = None</span>
+<span class="sd">        Activation function for the dense layers of the MLP. Currently</span>
+<span class="sd">        _&#39;tanh&#39;_, _&#39;relu&#39;_, _&#39;leaky&#39;_relu&#39; and _&#39;gelu&#39;_ are supported.</span>
+<span class="sd">        If &#39;mlp_hidden_dims&#39; is not `None` and this parameter is `None`, it</span>
+<span class="sd">        will default to _&#39;relu&#39;_.</span>
+<span class="sd">    mlp_dropout: float, Optional, default = None</span>
+<span class="sd">        float with the dropout between the dense layers of the MLP.</span>
+<span class="sd">        If &#39;mlp_hidden_dims&#39; is not `None` and this parameter is `None`, it</span>
+<span class="sd">        will default to 0.0.</span>
+<span class="sd">    mlp_batchnorm: bool, Optional, default = None</span>
+<span class="sd">        Boolean indicating whether or not batch normalization will be applied</span>
+<span class="sd">        to the dense layers</span>
+<span class="sd">        If &#39;mlp_hidden_dims&#39; is not `None` and this parameter is `None`, it</span>
+<span class="sd">        will default to False.</span>
+<span class="sd">    mlp_batchnorm_last: bool, Optional, default = None</span>
+<span class="sd">        Boolean indicating whether or not batch normalization will be applied</span>
+<span class="sd">        to the last of the dense layers</span>
+<span class="sd">        If &#39;mlp_hidden_dims&#39; is not `None` and this parameter is `None`, it</span>
+<span class="sd">        will default to False.</span>
+<span class="sd">    mlp_linear_first: bool, Optional, default = None</span>
+<span class="sd">        Boolean indicating the order of the operations in the dense</span>
+<span class="sd">        layer. If `True: [LIN -&gt; ACT -&gt; BN -&gt; DP]`. If `False: [BN -&gt; DP -&gt;</span>
+<span class="sd">        LIN -&gt; ACT]`</span>
+<span class="sd">        If &#39;mlp_hidden_dims&#39; is not `None` and this parameter is `None`, it</span>
+<span class="sd">        will default to `True`.</span>
+
+<span class="sd">    Attributes</span>
+<span class="sd">    ----------</span>
+<span class="sd">    encoder: nn.Module</span>
+<span class="sd">        Sequence of SAINT-Transformer blocks</span>
+<span class="sd">    mlp: nn.Module</span>
+<span class="sd">        MLP component in the model</span>
+
+<span class="sd">    Examples</span>
+<span class="sd">    --------</span>
+<span class="sd">    &gt;&gt;&gt; import torch</span>
+<span class="sd">    &gt;&gt;&gt; from pytorch_widedeep.models import SAINT</span>
+<span class="sd">    &gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)</span>
+<span class="sd">    &gt;&gt;&gt; colnames = [&#39;a&#39;, &#39;b&#39;, &#39;c&#39;, &#39;d&#39;, &#39;e&#39;]</span>
+<span class="sd">    &gt;&gt;&gt; cat_embed_input = [(u,i) for u,i in zip(colnames[:4], [4]*4)]</span>
+<span class="sd">    &gt;&gt;&gt; continuous_cols = [&#39;e&#39;]</span>
+<span class="sd">    &gt;&gt;&gt; column_idx = {k:v for v,k in enumerate(colnames)}</span>
+<span class="sd">    &gt;&gt;&gt; model = SAINT(column_idx=column_idx, cat_embed_input=cat_embed_input, continuous_cols=continuous_cols)</span>
+<span class="sd">    &gt;&gt;&gt; out = model(X_tab)</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+
+    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span>
+        <span class="n">column_idx</span><span class="p">:</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">],</span>
+        <span class="o">*</span><span class="p">,</span>
+        <span class="n">cat_embed_input</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="n">Tuple</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">]]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">cat_embed_dropout</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">use_cat_bias</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">cat_embed_activation</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">shared_embed</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">add_shared_embed</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">frac_shared_embed</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">continuous_cols</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">cont_norm_layer</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Literal</span><span class="p">[</span><span class="s2">&quot;batchnorm&quot;</span><span class="p">,</span> <span class="s2">&quot;layernorm&quot;</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">embed_continuous_method</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span>
+            <span class="n">Literal</span><span class="p">[</span><span class="s2">&quot;standard&quot;</span><span class="p">,</span> <span class="s2">&quot;piecewise&quot;</span><span class="p">,</span> <span class="s2">&quot;periodic&quot;</span><span class="p">]</span>
+        <span class="p">]</span> <span class="o">=</span> <span class="s2">&quot;standard&quot;</span><span class="p">,</span>
+        <span class="n">cont_embed_dropout</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">cont_embed_activation</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">quantization_setup</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="nb">float</span><span class="p">]]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">n_frequencies</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">sigma</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">share_last_layer</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">full_embed_dropout</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">input_dim</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">32</span><span class="p">,</span>
+        <span class="n">use_qkv_bias</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
+        <span class="n">n_heads</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">8</span><span class="p">,</span>
+        <span class="n">n_blocks</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">2</span><span class="p">,</span>
+        <span class="n">attn_dropout</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.1</span><span class="p">,</span>
+        <span class="n">ff_dropout</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.2</span><span class="p">,</span>
+        <span class="n">ff_factor</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">4</span><span class="p">,</span>
+        <span class="n">transformer_activation</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;gelu&quot;</span><span class="p">,</span>
+        <span class="n">mlp_hidden_dims</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">int</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">mlp_activation</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">mlp_dropout</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">mlp_batchnorm</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">mlp_batchnorm_last</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">mlp_linear_first</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+    <span class="p">):</span>
+        <span class="nb">super</span><span class="p">(</span><span class="n">SAINT</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span>
+            <span class="n">column_idx</span><span class="o">=</span><span class="n">column_idx</span><span class="p">,</span>
+            <span class="n">cat_embed_input</span><span class="o">=</span><span class="n">cat_embed_input</span><span class="p">,</span>
+            <span class="n">cat_embed_dropout</span><span class="o">=</span><span class="n">cat_embed_dropout</span><span class="p">,</span>
+            <span class="n">use_cat_bias</span><span class="o">=</span><span class="n">use_cat_bias</span><span class="p">,</span>
+            <span class="n">cat_embed_activation</span><span class="o">=</span><span class="n">cat_embed_activation</span><span class="p">,</span>
+            <span class="n">shared_embed</span><span class="o">=</span><span class="n">shared_embed</span><span class="p">,</span>
+            <span class="n">add_shared_embed</span><span class="o">=</span><span class="n">add_shared_embed</span><span class="p">,</span>
+            <span class="n">frac_shared_embed</span><span class="o">=</span><span class="n">frac_shared_embed</span><span class="p">,</span>
+            <span class="n">continuous_cols</span><span class="o">=</span><span class="n">continuous_cols</span><span class="p">,</span>
+            <span class="n">cont_norm_layer</span><span class="o">=</span><span class="n">cont_norm_layer</span><span class="p">,</span>
+            <span class="n">embed_continuous_method</span><span class="o">=</span><span class="n">embed_continuous_method</span><span class="p">,</span>
+            <span class="n">cont_embed_dropout</span><span class="o">=</span><span class="n">cont_embed_dropout</span><span class="p">,</span>
+            <span class="n">cont_embed_activation</span><span class="o">=</span><span class="n">cont_embed_activation</span><span class="p">,</span>
+            <span class="n">input_dim</span><span class="o">=</span><span class="n">input_dim</span><span class="p">,</span>
+            <span class="n">quantization_setup</span><span class="o">=</span><span class="n">quantization_setup</span><span class="p">,</span>
+            <span class="n">n_frequencies</span><span class="o">=</span><span class="n">n_frequencies</span><span class="p">,</span>
+            <span class="n">sigma</span><span class="o">=</span><span class="n">sigma</span><span class="p">,</span>
+            <span class="n">share_last_layer</span><span class="o">=</span><span class="n">share_last_layer</span><span class="p">,</span>
+            <span class="n">full_embed_dropout</span><span class="o">=</span><span class="n">full_embed_dropout</span><span class="p">,</span>
         <span class="p">)</span>
 
-    <span class="bp">self</span><span class="o">.</span><span class="n">mlp_first_hidden_dim</span> <span class="o">=</span> <span class="p">(</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">input_dim</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">with_cls_token</span> <span class="k">else</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">n_feats</span> <span class="o">*</span> <span class="bp">self</span><span class="o">.</span><span class="n">input_dim</span><span class="p">)</span>
-    <span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">use_qkv_bias</span> <span class="o">=</span> <span class="n">use_qkv_bias</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">n_heads</span> <span class="o">=</span> <span class="n">n_heads</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">n_blocks</span> <span class="o">=</span> <span class="n">n_blocks</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">attn_dropout</span> <span class="o">=</span> <span class="n">attn_dropout</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">ff_dropout</span> <span class="o">=</span> <span class="n">ff_dropout</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">ff_factor</span> <span class="o">=</span> <span class="n">ff_factor</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">transformer_activation</span> <span class="o">=</span> <span class="n">transformer_activation</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">mlp_hidden_dims</span> <span class="o">=</span> <span class="n">mlp_hidden_dims</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">mlp_activation</span> <span class="o">=</span> <span class="n">mlp_activation</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">mlp_dropout</span> <span class="o">=</span> <span class="n">mlp_dropout</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">mlp_batchnorm</span> <span class="o">=</span> <span class="n">mlp_batchnorm</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">mlp_batchnorm_last</span> <span class="o">=</span> <span class="n">mlp_batchnorm_last</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">mlp_linear_first</span> <span class="o">=</span> <span class="n">mlp_linear_first</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">with_cls_token</span> <span class="o">=</span> <span class="s2">&quot;cls_token&quot;</span> <span class="ow">in</span> <span class="n">column_idx</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">n_cat</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">cat_embed_input</span><span class="p">)</span> <span class="k">if</span> <span class="n">cat_embed_input</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="k">else</span> <span class="mi">0</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">n_cont</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">continuous_cols</span><span class="p">)</span> <span class="k">if</span> <span class="n">continuous_cols</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="k">else</span> <span class="mi">0</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">n_feats</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">n_cat</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">n_cont</span>
+
+        <span class="c1"># Embeddings are instantiated at the base model</span>
+        <span class="c1"># Transformer blocks</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">encoder</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Sequential</span><span class="p">()</span>
+        <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">n_blocks</span><span class="p">):</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">encoder</span><span class="o">.</span><span class="n">add_module</span><span class="p">(</span>
+                <span class="s2">&quot;saint_block&quot;</span> <span class="o">+</span> <span class="nb">str</span><span class="p">(</span><span class="n">i</span><span class="p">),</span>
+                <span class="n">SaintEncoder</span><span class="p">(</span>
+                    <span class="n">input_dim</span><span class="p">,</span>
+                    <span class="n">n_heads</span><span class="p">,</span>
+                    <span class="n">use_qkv_bias</span><span class="p">,</span>
+                    <span class="n">attn_dropout</span><span class="p">,</span>
+                    <span class="n">ff_dropout</span><span class="p">,</span>
+                    <span class="n">ff_factor</span><span class="p">,</span>
+                    <span class="n">transformer_activation</span><span class="p">,</span>
+                    <span class="bp">self</span><span class="o">.</span><span class="n">n_feats</span><span class="p">,</span>
+                <span class="p">),</span>
+            <span class="p">)</span>
 
-    <span class="c1"># Mlp: adding an MLP on top of the Resnet blocks is optional and</span>
-    <span class="c1"># therefore all related params are optional</span>
-    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_hidden_dims</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">mlp</span> <span class="o">=</span> <span class="n">MLP</span><span class="p">(</span>
-            <span class="n">d_hidden</span><span class="o">=</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">mlp_first_hidden_dim</span><span class="p">]</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_hidden_dims</span><span class="p">,</span>
-            <span class="n">activation</span><span class="o">=</span><span class="p">(</span>
-                <span class="s2">&quot;relu&quot;</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_activation</span> <span class="ow">is</span> <span class="kc">None</span> <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_activation</span>
-            <span class="p">),</span>
-            <span class="n">dropout</span><span class="o">=</span><span class="mf">0.0</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_dropout</span> <span class="ow">is</span> <span class="kc">None</span> <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_dropout</span><span class="p">,</span>
-            <span class="n">batchnorm</span><span class="o">=</span><span class="kc">False</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_batchnorm</span> <span class="ow">is</span> <span class="kc">None</span> <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_batchnorm</span><span class="p">,</span>
-            <span class="n">batchnorm_last</span><span class="o">=</span><span class="p">(</span>
-                <span class="kc">False</span>
-                <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_batchnorm_last</span> <span class="ow">is</span> <span class="kc">None</span>
-                <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_batchnorm_last</span>
-            <span class="p">),</span>
-            <span class="n">linear_first</span><span class="o">=</span><span class="p">(</span>
-                <span class="kc">False</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_linear_first</span> <span class="ow">is</span> <span class="kc">None</span> <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_linear_first</span>
-            <span class="p">),</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">mlp_first_hidden_dim</span> <span class="o">=</span> <span class="p">(</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">input_dim</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">with_cls_token</span> <span class="k">else</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">n_feats</span> <span class="o">*</span> <span class="bp">self</span><span class="o">.</span><span class="n">input_dim</span><span class="p">)</span>
+        <span class="p">)</span>
+
+        <span class="c1"># Mlp: adding an MLP on top of the Resnet blocks is optional and</span>
+        <span class="c1"># therefore all related params are optional</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_hidden_dims</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">mlp</span> <span class="o">=</span> <span class="n">MLP</span><span class="p">(</span>
+                <span class="n">d_hidden</span><span class="o">=</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">mlp_first_hidden_dim</span><span class="p">]</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_hidden_dims</span><span class="p">,</span>
+                <span class="n">activation</span><span class="o">=</span><span class="p">(</span>
+                    <span class="s2">&quot;relu&quot;</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_activation</span> <span class="ow">is</span> <span class="kc">None</span> <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_activation</span>
+                <span class="p">),</span>
+                <span class="n">dropout</span><span class="o">=</span><span class="mf">0.0</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_dropout</span> <span class="ow">is</span> <span class="kc">None</span> <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_dropout</span><span class="p">,</span>
+                <span class="n">batchnorm</span><span class="o">=</span><span class="kc">False</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_batchnorm</span> <span class="ow">is</span> <span class="kc">None</span> <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_batchnorm</span><span class="p">,</span>
+                <span class="n">batchnorm_last</span><span class="o">=</span><span class="p">(</span>
+                    <span class="kc">False</span>
+                    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_batchnorm_last</span> <span class="ow">is</span> <span class="kc">None</span>
+                    <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_batchnorm_last</span>
+                <span class="p">),</span>
+                <span class="n">linear_first</span><span class="o">=</span><span class="p">(</span>
+                    <span class="kc">False</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_linear_first</span> <span class="ow">is</span> <span class="kc">None</span> <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_linear_first</span>
+                <span class="p">),</span>
+            <span class="p">)</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">mlp</span> <span class="o">=</span> <span class="kc">None</span>
+
+    <span class="k">def</span> <span class="nf">forward</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">X</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tensor</span><span class="p">:</span>
+        <span class="n">x</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_embeddings</span><span class="p">(</span><span class="n">X</span><span class="p">)</span>
+        <span class="n">x</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">encoder</span><span class="p">(</span><span class="n">x</span><span class="p">)</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">with_cls_token</span><span class="p">:</span>
+            <span class="n">x</span> <span class="o">=</span> <span class="n">x</span><span class="p">[:,</span> <span class="mi">0</span><span class="p">,</span> <span class="p">:]</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="n">x</span> <span class="o">=</span> <span class="n">x</span><span class="o">.</span><span class="n">flatten</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="n">x</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp</span><span class="p">(</span><span class="n">x</span><span class="p">)</span>
+        <span class="k">return</span> <span class="n">x</span>
+
+    <span class="nd">@property</span>
+    <span class="k">def</span> <span class="nf">output_dim</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">int</span><span class="p">:</span>
+<span class="w">        </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;The output dimension of the model. This is a required property</span>
+<span class="sd">        neccesary to build the `WideDeep` class</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="k">return</span> <span class="p">(</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">mlp_hidden_dims</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span>
+            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_hidden_dims</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span>
+            <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_first_hidden_dim</span>
         <span class="p">)</span>
-    <span class="k">else</span><span class="p">:</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">mlp</span> <span class="o">=</span> <span class="kc">None</span>
+
+    <span class="nd">@property</span>
+    <span class="k">def</span> <span class="nf">attention_weights</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">List</span><span class="p">[</span><span class="n">Tuple</span><span class="p">[</span><span class="n">Tensor</span><span class="p">,</span> <span class="n">Tensor</span><span class="p">]]:</span>
+<span class="w">        </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;List with the attention weights. Each element of the list is a tuple</span>
+<span class="sd">        where the first and the second elements are the column and row</span>
+<span class="sd">        attention weights respectively</span>
+
+<span class="sd">        The shape of the attention weights is:</span>
+
+<span class="sd">        - column attention: $(N, H, F, F)$</span>
+
+<span class="sd">        - row attention: $(1, H, N, N)$</span>
+
+<span class="sd">        where $N$ is the batch size, $H$ is the number of heads and $F$ is the</span>
+<span class="sd">        number of features/columns in the dataset</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">attention_weights</span> <span class="o">=</span> <span class="p">[]</span>
+        <span class="k">for</span> <span class="n">blk</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">encoder</span><span class="p">:</span>
+            <span class="n">attention_weights</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
+                <span class="p">(</span><span class="n">blk</span><span class="o">.</span><span class="n">col_attn</span><span class="o">.</span><span class="n">attn_weights</span><span class="p">,</span> <span class="n">blk</span><span class="o">.</span><span class="n">row_attn</span><span class="o">.</span><span class="n">attn_weights</span><span class="p">)</span>
+            <span class="p">)</span>
+        <span class="k">return</span> <span class="n">attention_weights</span>
 </code></pre></div></td></tr></table></div>
-                  </details>
+              </details>
 
 
 
@@ -7568,44 +11544,7 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.ft_transformer.FTTransforme
 
 
 <a href="#pytorch_widedeep.models.tabular.transformers.ft_transformer.FTTransformer" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="nf">FTTransformer</span><span class="p">(</span>
-    <span class="n">column_idx</span><span class="p">,</span>
-    <span class="o">*</span><span class="p">,</span>
-    <span class="n">cat_embed_input</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">cat_embed_dropout</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">use_cat_bias</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">cat_embed_activation</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">shared_embed</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">add_shared_embed</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">frac_shared_embed</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">continuous_cols</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">cont_norm_layer</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">embed_continuous_method</span><span class="o">=</span><span class="s2">&quot;standard&quot;</span><span class="p">,</span>
-    <span class="n">cont_embed_dropout</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">cont_embed_activation</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">quantization_setup</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">n_frequencies</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">sigma</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">share_last_layer</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">full_embed_dropout</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">input_dim</span><span class="o">=</span><span class="mi">64</span><span class="p">,</span>
-    <span class="n">kv_compression_factor</span><span class="o">=</span><span class="mf">0.5</span><span class="p">,</span>
-    <span class="n">kv_sharing</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
-    <span class="n">use_qkv_bias</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
-    <span class="n">n_heads</span><span class="o">=</span><span class="mi">8</span><span class="p">,</span>
-    <span class="n">n_blocks</span><span class="o">=</span><span class="mi">4</span><span class="p">,</span>
-    <span class="n">attn_dropout</span><span class="o">=</span><span class="mf">0.2</span><span class="p">,</span>
-    <span class="n">ff_dropout</span><span class="o">=</span><span class="mf">0.1</span><span class="p">,</span>
-    <span class="n">ff_factor</span><span class="o">=</span><span class="mf">1.33</span><span class="p">,</span>
-    <span class="n">transformer_activation</span><span class="o">=</span><span class="s2">&quot;reglu&quot;</span><span class="p">,</span>
-    <span class="n">mlp_hidden_dims</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">mlp_activation</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">mlp_dropout</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">mlp_batchnorm</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">mlp_batchnorm_last</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">mlp_linear_first</span><span class="o">=</span><span class="kc">None</span>
-<span class="p">)</span>
-</code></pre></div>
+
 
     <div class="doc doc-contents first">
             <p class="doc doc-class-bases">
@@ -7621,132 +11560,183 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.ft_transformer.FTTransforme
 
 
 <p><span class="doc-section-title">Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>column_idx</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, int]</code>)
-          –
-          <div class="doc-md-description">
-            <p>Dict containing the index of the columns that will be passed through
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+          <th>Default</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>column_idx</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, int]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Dict containing the index of the columns that will be passed through
 the <code>TabMlp</code> model. Required to slice the tensors. e.g. <em>{'education':
 0, 'relationship': 1, 'workclass': 2, ...}</em>.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>cat_embed_input</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.wdtypes.Tuple">Tuple</span>[str, int]]]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>List of Tuples with the column name and number of unique values and
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>cat_embed_input</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.wdtypes.Tuple">Tuple</span>[str, int]]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>List of Tuples with the column name and number of unique values and
 embedding dimension. e.g. <em>[(education, 11), ...]</em></p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>cat_embed_dropout</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Categorical embeddings dropout. If <code>None</code>, it will default
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>cat_embed_dropout</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Categorical embeddings dropout. If <code>None</code>, it will default
 to 0.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>use_cat_bias</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Boolean indicating if bias will be used for the categorical embeddings.
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>use_cat_bias</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Boolean indicating if bias will be used for the categorical embeddings.
 If <code>None</code>, it will default to 'False'.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>cat_embed_activation</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Activation function for the categorical embeddings, if any. Currently
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>cat_embed_activation</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Activation function for the categorical embeddings, if any. Currently
 <em>'tanh'</em>, <em>'relu'</em>, <em>'leaky_relu'</em> and <em>'gelu'</em> are supported</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>shared_embed</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Boolean indicating if the embeddings will be "shared". The idea behind <code>shared_embed</code> is
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>shared_embed</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Boolean indicating if the embeddings will be "shared". The idea behind <code>shared_embed</code> is
 described in the Appendix A in the <a href="https://arxiv.org/abs/2012.06678">TabTransformer paper</a>:
 <em>'The goal of having column embedding is to enable the model to
 distinguish the classes in one column from those in the other
 columns'</em>. In other words, the idea is to let the model learn which
 column is embedded at the time. See: <code>pytorch_widedeep.models.transformers._layers.SharedEmbeddings</code>.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>add_shared_embed</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>The two embedding sharing strategies are: 1) add the shared embeddings
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>add_shared_embed</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>The two embedding sharing strategies are: 1) add the shared embeddings
 to the column embeddings or 2) to replace the first
 <code>frac_shared_embed</code> with the shared embeddings.
 See <code>pytorch_widedeep.models.embeddings_layers.SharedEmbeddings</code>
 If 'None' is passed, it will default to 'False'.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>frac_shared_embed</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>The fraction of embeddings that will be shared (if <code>add_shared_embed
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>frac_shared_embed</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>The fraction of embeddings that will be shared (if <code>add_shared_embed
 = False</code>) by all the different categories for one particular
 column. If 'None' is passed, it will default to 0.0.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>continuous_cols</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[str]]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>List with the name of the numeric (aka continuous) columns</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>cont_norm_layer</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[batchnorm, layernorm]]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Type of normalization layer applied to the continuous features.
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>continuous_cols</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[str]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>List with the name of the numeric (aka continuous) columns</p>
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>cont_norm_layer</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[batchnorm, layernorm]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Type of normalization layer applied to the continuous features.
 Options are: <em>'layernorm'</em> and <em>'batchnorm'</em>. if <code>None</code>, no
 normalization layer will be used.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>embed_continuous_method</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[standard, piecewise, periodic]]</code>, default:
-                  <code>&#39;standard&#39;</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Method to use to embed the continuous features. Options are:
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>embed_continuous_method</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[standard, piecewise, periodic]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Method to use to embed the continuous features. Options are:
 <em>'standard'</em>, <em>'periodic'</em> or <em>'piecewise'</em>. The <em>'standard'</em>
 embedding method is based on the FT-Transformer implementation
 presented in the paper: <a href="https://arxiv.org/abs/2106.11959v5">Revisiting Deep Learning Models for
@@ -7754,308 +11744,420 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.ft_transformer.FTTransforme
 and_'piecewise'_ methods were presented in the paper: <a href="https://arxiv.org/abs/2203.05556">On Embeddings for
 Numerical Features in Tabular Deep Learning</a>.
 Please, read the papers for details.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>cont_embed_dropout</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>cont_embed_activation</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Activation function for the continuous embeddings if any. Currently
+              </div>
+            </td>
+            <td>
+                  <code>&#39;standard&#39;</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>cont_embed_dropout</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p>
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>cont_embed_activation</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Activation function for the continuous embeddings if any. Currently
 <em>'tanh'</em>, <em>'relu'</em>, <em>'leaky_relu'</em> and <em>'gelu'</em> are supported.
 If <code>None</code>, no activation function will be applied.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>quantization_setup</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, <span title="pytorch_widedeep.wdtypes.List">List</span>[float]]]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>This parameter is used when the <em>'piecewise'</em> method is used to embed
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>quantization_setup</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, <span title="pytorch_widedeep.wdtypes.List">List</span>[float]]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>This parameter is used when the <em>'piecewise'</em> method is used to embed
 the continuous cols. It is a dict where keys are the name of the continuous
 columns and values are lists with the boundaries for the quantization
 of the continuous_cols. See the examples for details. If
 If the <em>'piecewise'</em> method is used, this parameter is required.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>n_frequencies</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[int]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>This is the so called <em>'k'</em> in their paper <a href="https://arxiv.org/abs/2203.05556">On Embeddings for
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>n_frequencies</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[int]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>This is the so called <em>'k'</em> in their paper <a href="https://arxiv.org/abs/2203.05556">On Embeddings for
 Numerical Features in Tabular Deep Learning</a>,
 and is the number of 'frequencies' that will be used to represent each
 continuous column. See their Eq 2 in the paper for details. If
 the <em>'periodic'</em> method is used, this parameter is required.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>sigma</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>This is the sigma parameter in the paper mentioned when describing the
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>sigma</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>This is the sigma parameter in the paper mentioned when describing the
 previous parameters and it is used to initialise the 'frequency
 weights'. See their Eq 2 in the paper for details. If
 the <em>'periodic'</em> method is used, this parameter is required.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>share_last_layer</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>This parameter is not present in the before mentioned paper but it is implemented in
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>share_last_layer</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>This parameter is not present in the before mentioned paper but it is implemented in
 the <a href="https://github.com/yandex-research/rtdl-num-embeddings/tree/main">official repo</a>.
 If <code>True</code> the linear layer that turns the frequencies into embeddings
 will be shared across the continuous columns. If <code>False</code> a different
 linear layer will be used for each continuous column.
 If the <em>'periodic'</em> method is used, this parameter is required.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>full_embed_dropout</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>If <code>True</code>, the full embedding corresponding to a column will be masked
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>full_embed_dropout</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>If <code>True</code>, the full embedding corresponding to a column will be masked
 out/dropout. If <code>None</code>, it will default to <code>False</code>.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>input_dim</code></b>
-              (<code>int</code>, default:
-                  <code>64</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>The so-called <em>dimension of the model</em>. Is the number of embeddings used to encode
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>input_dim</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>The so-called <em>dimension of the model</em>. Is the number of embeddings used to encode
 the categorical and/or continuous columns.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>kv_compression_factor</code></b>
-              (<code>float</code>, default:
-                  <code>0.5</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>By default, the FTTransformer uses Linear Attention
+              </div>
+            </td>
+            <td>
+                  <code>64</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>kv_compression_factor</code></td>
+            <td>
+                  <code>float</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>By default, the FTTransformer uses Linear Attention
 (See <a href="https://arxiv.org/abs/2006.04768&gt;">Linformer: Self-Attention with Linear Complexity</a> ).
 The compression factor that will be used to reduce the input sequence
 length. If we denote the resulting sequence length as
 <span class="arithmatex">\(k = int(kv_{compression \space factor} \times s)\)</span>
 where <span class="arithmatex">\(s\)</span> is the input sequence length.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>kv_sharing</code></b>
-              (<code>bool</code>, default:
-                  <code>False</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Boolean indicating if the <span class="arithmatex">\(E\)</span> and <span class="arithmatex">\(F\)</span> projection matrices
+              </div>
+            </td>
+            <td>
+                  <code>0.5</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>kv_sharing</code></td>
+            <td>
+                  <code>bool</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Boolean indicating if the <span class="arithmatex">\(E\)</span> and <span class="arithmatex">\(F\)</span> projection matrices
 will share weights.  See <a href="https://arxiv.org/abs/2006.04768">Linformer: Self-Attention with Linear
 Complexity</a> for details</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>n_heads</code></b>
-              (<code>int</code>, default:
-                  <code>8</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Number of attention heads per FTTransformer block</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>use_qkv_bias</code></b>
-              (<code>bool</code>, default:
+              </div>
+            </td>
+            <td>
                   <code>False</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Boolean indicating whether or not to use bias in the Q, K, and V
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>n_heads</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Number of attention heads per FTTransformer block</p>
+              </div>
+            </td>
+            <td>
+                  <code>8</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>use_qkv_bias</code></td>
+            <td>
+                  <code>bool</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Boolean indicating whether or not to use bias in the Q, K, and V
 projection layers</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>n_blocks</code></b>
-              (<code>int</code>, default:
+              </div>
+            </td>
+            <td>
+                  <code>False</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>n_blocks</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Number of FTTransformer blocks</p>
+              </div>
+            </td>
+            <td>
                   <code>4</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Number of FTTransformer blocks</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>attn_dropout</code></b>
-              (<code>float</code>, default:
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>attn_dropout</code></td>
+            <td>
+                  <code>float</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Dropout that will be applied to the Linear-Attention layers</p>
+              </div>
+            </td>
+            <td>
                   <code>0.2</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Dropout that will be applied to the Linear-Attention layers</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>ff_dropout</code></b>
-              (<code>float</code>, default:
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>ff_dropout</code></td>
+            <td>
+                  <code>float</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Dropout that will be applied to the FeedForward network</p>
+              </div>
+            </td>
+            <td>
                   <code>0.1</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Dropout that will be applied to the FeedForward network</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>ff_factor</code></b>
-              (<code>float</code>, default:
-                  <code>1.33</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Multiplicative factor applied to the first layer of the FF network in
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>ff_factor</code></td>
+            <td>
+                  <code>float</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Multiplicative factor applied to the first layer of the FF network in
 each Transformer block, This is normally set to 4, but they use 4/3
 in the paper.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>transformer_activation</code></b>
-              (<code>str</code>, default:
-                  <code>&#39;reglu&#39;</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Transformer Encoder activation function. <em>'tanh'</em>, <em>'relu'</em>,
+              </div>
+            </td>
+            <td>
+                  <code>1.33</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>transformer_activation</code></td>
+            <td>
+                  <code>str</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Transformer Encoder activation function. <em>'tanh'</em>, <em>'relu'</em>,
 <em>'leaky_relu'</em>, <em>'gelu'</em>, <em>'geglu'</em> and <em>'reglu'</em> are supported</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>mlp_hidden_dims</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[int]]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>List with the number of neurons per dense layer in the MLP. e.g:
+              </div>
+            </td>
+            <td>
+                  <code>&#39;reglu&#39;</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>mlp_hidden_dims</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[int]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>List with the number of neurons per dense layer in the MLP. e.g:
 <em>[64, 32]</em>. If not provided no MLP on top of the final
 FTTransformer block will be used.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>mlp_activation</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Activation function for the dense layers of the MLP. Currently
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>mlp_activation</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Activation function for the dense layers of the MLP. Currently
 <em>'tanh'</em>, <em>'relu'</em>, <em>'leaky'_relu' and _'gelu'</em> are supported.
 If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it
 will default to <em>'relu'</em>.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>mlp_dropout</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>float with the dropout between the dense layers of the MLP.
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>mlp_dropout</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>float with the dropout between the dense layers of the MLP.
 If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it
 will default to 0.0.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>mlp_batchnorm</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Boolean indicating whether or not batch normalization will be applied
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>mlp_batchnorm</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Boolean indicating whether or not batch normalization will be applied
 to the dense layers
 If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it
 will default to False.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>mlp_batchnorm_last</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Boolean indicating whether or not batch normalization will be applied
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>mlp_batchnorm_last</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Boolean indicating whether or not batch normalization will be applied
 to the last of the dense layers
 If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it
 will default to False.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>mlp_linear_first</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Boolean indicating the order of the operations in the dense
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>mlp_linear_first</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Boolean indicating the order of the operations in the dense
 layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt;
 LIN -&gt; ACT]</code>
 If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it
 will default to <code>True</code>.</p>
-          </div>
-        </li>
-    </ul>
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
 
 <p><span class="doc-section-title">Attributes:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code><span title="pytorch_widedeep.models.tabular.transformers.ft_transformer.FTTransformer.encoder">encoder</span></code></b>
-              (<code><span title="torch.nn.Module">Module</span></code>)
-          –
-          <div class="doc-md-description">
-            <p>Sequence of FTTransformer blocks</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code><span title="pytorch_widedeep.models.tabular.transformers.ft_transformer.FTTransformer.mlp">mlp</span></code></b>
-              (<code><span title="torch.nn.Module">Module</span></code>)
-          –
-          <div class="doc-md-description">
-            <p>MLP component in the model</p>
-          </div>
-        </li>
-    </ul>
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code><span title="pytorch_widedeep.models.tabular.transformers.ft_transformer.FTTransformer.encoder">encoder</span></code></td>
+            <td>
+                  <code><span title="torch.nn.Module">Module</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Sequence of FTTransformer blocks</p>
+              </div>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code><span title="pytorch_widedeep.models.tabular.transformers.ft_transformer.FTTransformer.mlp">mlp</span></code></td>
+            <td>
+                  <code><span title="torch.nn.Module">Module</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>MLP component in the model</p>
+              </div>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
 
 <p><span class="doc-section-title">Examples:</span></p>
@@ -8070,9 +12172,183 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.ft_transformer.FTTransforme
 <span class="gp">&gt;&gt;&gt; </span><span class="n">out</span> <span class="o">=</span> <span class="n">model</span><span class="p">(</span><span class="n">X_tab</span><span class="p">)</span>
 </code></pre></div>
 
-                  <details class="quote">
-                    <summary>Source code in <code>pytorch_widedeep/models/tabular/transformers/ft_transformer.py</code></summary>
-                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">185</span>
+              <details class="quote">
+                <summary>Source code in <code>pytorch_widedeep/models/tabular/transformers/ft_transformer.py</code></summary>
+                <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 11</span>
+<span class="normal"> 12</span>
+<span class="normal"> 13</span>
+<span class="normal"> 14</span>
+<span class="normal"> 15</span>
+<span class="normal"> 16</span>
+<span class="normal"> 17</span>
+<span class="normal"> 18</span>
+<span class="normal"> 19</span>
+<span class="normal"> 20</span>
+<span class="normal"> 21</span>
+<span class="normal"> 22</span>
+<span class="normal"> 23</span>
+<span class="normal"> 24</span>
+<span class="normal"> 25</span>
+<span class="normal"> 26</span>
+<span class="normal"> 27</span>
+<span class="normal"> 28</span>
+<span class="normal"> 29</span>
+<span class="normal"> 30</span>
+<span class="normal"> 31</span>
+<span class="normal"> 32</span>
+<span class="normal"> 33</span>
+<span class="normal"> 34</span>
+<span class="normal"> 35</span>
+<span class="normal"> 36</span>
+<span class="normal"> 37</span>
+<span class="normal"> 38</span>
+<span class="normal"> 39</span>
+<span class="normal"> 40</span>
+<span class="normal"> 41</span>
+<span class="normal"> 42</span>
+<span class="normal"> 43</span>
+<span class="normal"> 44</span>
+<span class="normal"> 45</span>
+<span class="normal"> 46</span>
+<span class="normal"> 47</span>
+<span class="normal"> 48</span>
+<span class="normal"> 49</span>
+<span class="normal"> 50</span>
+<span class="normal"> 51</span>
+<span class="normal"> 52</span>
+<span class="normal"> 53</span>
+<span class="normal"> 54</span>
+<span class="normal"> 55</span>
+<span class="normal"> 56</span>
+<span class="normal"> 57</span>
+<span class="normal"> 58</span>
+<span class="normal"> 59</span>
+<span class="normal"> 60</span>
+<span class="normal"> 61</span>
+<span class="normal"> 62</span>
+<span class="normal"> 63</span>
+<span class="normal"> 64</span>
+<span class="normal"> 65</span>
+<span class="normal"> 66</span>
+<span class="normal"> 67</span>
+<span class="normal"> 68</span>
+<span class="normal"> 69</span>
+<span class="normal"> 70</span>
+<span class="normal"> 71</span>
+<span class="normal"> 72</span>
+<span class="normal"> 73</span>
+<span class="normal"> 74</span>
+<span class="normal"> 75</span>
+<span class="normal"> 76</span>
+<span class="normal"> 77</span>
+<span class="normal"> 78</span>
+<span class="normal"> 79</span>
+<span class="normal"> 80</span>
+<span class="normal"> 81</span>
+<span class="normal"> 82</span>
+<span class="normal"> 83</span>
+<span class="normal"> 84</span>
+<span class="normal"> 85</span>
+<span class="normal"> 86</span>
+<span class="normal"> 87</span>
+<span class="normal"> 88</span>
+<span class="normal"> 89</span>
+<span class="normal"> 90</span>
+<span class="normal"> 91</span>
+<span class="normal"> 92</span>
+<span class="normal"> 93</span>
+<span class="normal"> 94</span>
+<span class="normal"> 95</span>
+<span class="normal"> 96</span>
+<span class="normal"> 97</span>
+<span class="normal"> 98</span>
+<span class="normal"> 99</span>
+<span class="normal">100</span>
+<span class="normal">101</span>
+<span class="normal">102</span>
+<span class="normal">103</span>
+<span class="normal">104</span>
+<span class="normal">105</span>
+<span class="normal">106</span>
+<span class="normal">107</span>
+<span class="normal">108</span>
+<span class="normal">109</span>
+<span class="normal">110</span>
+<span class="normal">111</span>
+<span class="normal">112</span>
+<span class="normal">113</span>
+<span class="normal">114</span>
+<span class="normal">115</span>
+<span class="normal">116</span>
+<span class="normal">117</span>
+<span class="normal">118</span>
+<span class="normal">119</span>
+<span class="normal">120</span>
+<span class="normal">121</span>
+<span class="normal">122</span>
+<span class="normal">123</span>
+<span class="normal">124</span>
+<span class="normal">125</span>
+<span class="normal">126</span>
+<span class="normal">127</span>
+<span class="normal">128</span>
+<span class="normal">129</span>
+<span class="normal">130</span>
+<span class="normal">131</span>
+<span class="normal">132</span>
+<span class="normal">133</span>
+<span class="normal">134</span>
+<span class="normal">135</span>
+<span class="normal">136</span>
+<span class="normal">137</span>
+<span class="normal">138</span>
+<span class="normal">139</span>
+<span class="normal">140</span>
+<span class="normal">141</span>
+<span class="normal">142</span>
+<span class="normal">143</span>
+<span class="normal">144</span>
+<span class="normal">145</span>
+<span class="normal">146</span>
+<span class="normal">147</span>
+<span class="normal">148</span>
+<span class="normal">149</span>
+<span class="normal">150</span>
+<span class="normal">151</span>
+<span class="normal">152</span>
+<span class="normal">153</span>
+<span class="normal">154</span>
+<span class="normal">155</span>
+<span class="normal">156</span>
+<span class="normal">157</span>
+<span class="normal">158</span>
+<span class="normal">159</span>
+<span class="normal">160</span>
+<span class="normal">161</span>
+<span class="normal">162</span>
+<span class="normal">163</span>
+<span class="normal">164</span>
+<span class="normal">165</span>
+<span class="normal">166</span>
+<span class="normal">167</span>
+<span class="normal">168</span>
+<span class="normal">169</span>
+<span class="normal">170</span>
+<span class="normal">171</span>
+<span class="normal">172</span>
+<span class="normal">173</span>
+<span class="normal">174</span>
+<span class="normal">175</span>
+<span class="normal">176</span>
+<span class="normal">177</span>
+<span class="normal">178</span>
+<span class="normal">179</span>
+<span class="normal">180</span>
+<span class="normal">181</span>
+<span class="normal">182</span>
+<span class="normal">183</span>
+<span class="normal">184</span>
+<span class="normal">185</span>
 <span class="normal">186</span>
 <span class="normal">187</span>
 <span class="normal">188</span>
@@ -8203,140 +12479,380 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.ft_transformer.FTTransforme
 <span class="normal">313</span>
 <span class="normal">314</span>
 <span class="normal">315</span>
-<span class="normal">316</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
-    <span class="bp">self</span><span class="p">,</span>
-    <span class="n">column_idx</span><span class="p">:</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">],</span>
-    <span class="o">*</span><span class="p">,</span>
-    <span class="n">cat_embed_input</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="n">Tuple</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">]]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">cat_embed_dropout</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">use_cat_bias</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">cat_embed_activation</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">shared_embed</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">add_shared_embed</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">frac_shared_embed</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">continuous_cols</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">cont_norm_layer</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Literal</span><span class="p">[</span><span class="s2">&quot;batchnorm&quot;</span><span class="p">,</span> <span class="s2">&quot;layernorm&quot;</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">embed_continuous_method</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span>
-        <span class="n">Literal</span><span class="p">[</span><span class="s2">&quot;standard&quot;</span><span class="p">,</span> <span class="s2">&quot;piecewise&quot;</span><span class="p">,</span> <span class="s2">&quot;periodic&quot;</span><span class="p">]</span>
-    <span class="p">]</span> <span class="o">=</span> <span class="s2">&quot;standard&quot;</span><span class="p">,</span>
-    <span class="n">cont_embed_dropout</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">cont_embed_activation</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">quantization_setup</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="nb">float</span><span class="p">]]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">n_frequencies</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">sigma</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">share_last_layer</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">full_embed_dropout</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">input_dim</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">64</span><span class="p">,</span>
-    <span class="n">kv_compression_factor</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.5</span><span class="p">,</span>
-    <span class="n">kv_sharing</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
-    <span class="n">use_qkv_bias</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
-    <span class="n">n_heads</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">8</span><span class="p">,</span>
-    <span class="n">n_blocks</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">4</span><span class="p">,</span>
-    <span class="n">attn_dropout</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.2</span><span class="p">,</span>
-    <span class="n">ff_dropout</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.1</span><span class="p">,</span>
-    <span class="n">ff_factor</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">1.33</span><span class="p">,</span>
-    <span class="n">transformer_activation</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;reglu&quot;</span><span class="p">,</span>
-    <span class="n">mlp_hidden_dims</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">int</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">mlp_activation</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">mlp_dropout</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">mlp_batchnorm</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">mlp_batchnorm_last</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">mlp_linear_first</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-<span class="p">):</span>
-    <span class="nb">super</span><span class="p">(</span><span class="n">FTTransformer</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span>
-        <span class="n">column_idx</span><span class="o">=</span><span class="n">column_idx</span><span class="p">,</span>
-        <span class="n">cat_embed_input</span><span class="o">=</span><span class="n">cat_embed_input</span><span class="p">,</span>
-        <span class="n">cat_embed_dropout</span><span class="o">=</span><span class="n">cat_embed_dropout</span><span class="p">,</span>
-        <span class="n">use_cat_bias</span><span class="o">=</span><span class="n">use_cat_bias</span><span class="p">,</span>
-        <span class="n">cat_embed_activation</span><span class="o">=</span><span class="n">cat_embed_activation</span><span class="p">,</span>
-        <span class="n">shared_embed</span><span class="o">=</span><span class="n">shared_embed</span><span class="p">,</span>
-        <span class="n">add_shared_embed</span><span class="o">=</span><span class="n">add_shared_embed</span><span class="p">,</span>
-        <span class="n">frac_shared_embed</span><span class="o">=</span><span class="n">frac_shared_embed</span><span class="p">,</span>
-        <span class="n">continuous_cols</span><span class="o">=</span><span class="n">continuous_cols</span><span class="p">,</span>
-        <span class="n">cont_norm_layer</span><span class="o">=</span><span class="n">cont_norm_layer</span><span class="p">,</span>
-        <span class="n">embed_continuous_method</span><span class="o">=</span><span class="n">embed_continuous_method</span><span class="p">,</span>
-        <span class="n">cont_embed_dropout</span><span class="o">=</span><span class="n">cont_embed_dropout</span><span class="p">,</span>
-        <span class="n">cont_embed_activation</span><span class="o">=</span><span class="n">cont_embed_activation</span><span class="p">,</span>
-        <span class="n">input_dim</span><span class="o">=</span><span class="n">input_dim</span><span class="p">,</span>
-        <span class="n">quantization_setup</span><span class="o">=</span><span class="n">quantization_setup</span><span class="p">,</span>
-        <span class="n">n_frequencies</span><span class="o">=</span><span class="n">n_frequencies</span><span class="p">,</span>
-        <span class="n">sigma</span><span class="o">=</span><span class="n">sigma</span><span class="p">,</span>
-        <span class="n">share_last_layer</span><span class="o">=</span><span class="n">share_last_layer</span><span class="p">,</span>
-        <span class="n">full_embed_dropout</span><span class="o">=</span><span class="n">full_embed_dropout</span><span class="p">,</span>
-    <span class="p">)</span>
-
-    <span class="bp">self</span><span class="o">.</span><span class="n">kv_compression_factor</span> <span class="o">=</span> <span class="n">kv_compression_factor</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">kv_sharing</span> <span class="o">=</span> <span class="n">kv_sharing</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">use_qkv_bias</span> <span class="o">=</span> <span class="n">use_qkv_bias</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">n_heads</span> <span class="o">=</span> <span class="n">n_heads</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">n_blocks</span> <span class="o">=</span> <span class="n">n_blocks</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">attn_dropout</span> <span class="o">=</span> <span class="n">attn_dropout</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">ff_dropout</span> <span class="o">=</span> <span class="n">ff_dropout</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">ff_factor</span> <span class="o">=</span> <span class="n">ff_factor</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">transformer_activation</span> <span class="o">=</span> <span class="n">transformer_activation</span>
-
-    <span class="bp">self</span><span class="o">.</span><span class="n">mlp_hidden_dims</span> <span class="o">=</span> <span class="n">mlp_hidden_dims</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">mlp_activation</span> <span class="o">=</span> <span class="n">mlp_activation</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">mlp_dropout</span> <span class="o">=</span> <span class="n">mlp_dropout</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">mlp_batchnorm</span> <span class="o">=</span> <span class="n">mlp_batchnorm</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">mlp_batchnorm_last</span> <span class="o">=</span> <span class="n">mlp_batchnorm_last</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">mlp_linear_first</span> <span class="o">=</span> <span class="n">mlp_linear_first</span>
-
-    <span class="bp">self</span><span class="o">.</span><span class="n">with_cls_token</span> <span class="o">=</span> <span class="s2">&quot;cls_token&quot;</span> <span class="ow">in</span> <span class="n">column_idx</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">n_cat</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">cat_embed_input</span><span class="p">)</span> <span class="k">if</span> <span class="n">cat_embed_input</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="k">else</span> <span class="mi">0</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">n_cont</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">continuous_cols</span><span class="p">)</span> <span class="k">if</span> <span class="n">continuous_cols</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="k">else</span> <span class="mi">0</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">n_feats</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">n_cat</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">n_cont</span>
-
-    <span class="c1"># Embeddings are instantiated at the base model</span>
-    <span class="c1"># Transformer blocks</span>
-    <span class="n">is_first</span> <span class="o">=</span> <span class="kc">True</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">encoder</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Sequential</span><span class="p">()</span>
-    <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">n_blocks</span><span class="p">):</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">encoder</span><span class="o">.</span><span class="n">add_module</span><span class="p">(</span>
-            <span class="s2">&quot;fttransformer_block&quot;</span> <span class="o">+</span> <span class="nb">str</span><span class="p">(</span><span class="n">i</span><span class="p">),</span>
-            <span class="n">FTTransformerEncoder</span><span class="p">(</span>
-                <span class="n">input_dim</span><span class="p">,</span>
-                <span class="bp">self</span><span class="o">.</span><span class="n">n_feats</span><span class="p">,</span>
-                <span class="n">n_heads</span><span class="p">,</span>
-                <span class="n">use_qkv_bias</span><span class="p">,</span>
-                <span class="n">attn_dropout</span><span class="p">,</span>
-                <span class="n">ff_dropout</span><span class="p">,</span>
-                <span class="n">ff_factor</span><span class="p">,</span>
-                <span class="n">kv_compression_factor</span><span class="p">,</span>
-                <span class="n">kv_sharing</span><span class="p">,</span>
-                <span class="n">transformer_activation</span><span class="p">,</span>
-                <span class="n">is_first</span><span class="p">,</span>
-            <span class="p">),</span>
-        <span class="p">)</span>
-        <span class="n">is_first</span> <span class="o">=</span> <span class="kc">False</span>
-
-    <span class="bp">self</span><span class="o">.</span><span class="n">mlp_first_hidden_dim</span> <span class="o">=</span> <span class="p">(</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">input_dim</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">with_cls_token</span> <span class="k">else</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">n_feats</span> <span class="o">*</span> <span class="bp">self</span><span class="o">.</span><span class="n">input_dim</span><span class="p">)</span>
-    <span class="p">)</span>
-
-    <span class="c1"># Mlp: adding an MLP on top of the Resnet blocks is optional and</span>
-    <span class="c1"># therefore all related params are optional</span>
-    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_hidden_dims</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">mlp</span> <span class="o">=</span> <span class="n">MLP</span><span class="p">(</span>
-            <span class="n">d_hidden</span><span class="o">=</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">mlp_first_hidden_dim</span><span class="p">]</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_hidden_dims</span><span class="p">,</span>
-            <span class="n">activation</span><span class="o">=</span><span class="p">(</span>
-                <span class="s2">&quot;relu&quot;</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_activation</span> <span class="ow">is</span> <span class="kc">None</span> <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_activation</span>
-            <span class="p">),</span>
-            <span class="n">dropout</span><span class="o">=</span><span class="mf">0.0</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_dropout</span> <span class="ow">is</span> <span class="kc">None</span> <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_dropout</span><span class="p">,</span>
-            <span class="n">batchnorm</span><span class="o">=</span><span class="kc">False</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_batchnorm</span> <span class="ow">is</span> <span class="kc">None</span> <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_batchnorm</span><span class="p">,</span>
-            <span class="n">batchnorm_last</span><span class="o">=</span><span class="p">(</span>
-                <span class="kc">False</span>
-                <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_batchnorm_last</span> <span class="ow">is</span> <span class="kc">None</span>
-                <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_batchnorm_last</span>
-            <span class="p">),</span>
-            <span class="n">linear_first</span><span class="o">=</span><span class="p">(</span>
-                <span class="kc">False</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_linear_first</span> <span class="ow">is</span> <span class="kc">None</span> <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_linear_first</span>
-            <span class="p">),</span>
+<span class="normal">316</span>
+<span class="normal">317</span>
+<span class="normal">318</span>
+<span class="normal">319</span>
+<span class="normal">320</span>
+<span class="normal">321</span>
+<span class="normal">322</span>
+<span class="normal">323</span>
+<span class="normal">324</span>
+<span class="normal">325</span>
+<span class="normal">326</span>
+<span class="normal">327</span>
+<span class="normal">328</span>
+<span class="normal">329</span>
+<span class="normal">330</span>
+<span class="normal">331</span>
+<span class="normal">332</span>
+<span class="normal">333</span>
+<span class="normal">334</span>
+<span class="normal">335</span>
+<span class="normal">336</span>
+<span class="normal">337</span>
+<span class="normal">338</span>
+<span class="normal">339</span>
+<span class="normal">340</span>
+<span class="normal">341</span>
+<span class="normal">342</span>
+<span class="normal">343</span>
+<span class="normal">344</span>
+<span class="normal">345</span>
+<span class="normal">346</span>
+<span class="normal">347</span>
+<span class="normal">348</span>
+<span class="normal">349</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">class</span> <span class="nc">FTTransformer</span><span class="p">(</span><span class="n">BaseTabularModelWithAttention</span><span class="p">):</span>
+<span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Defines a [FTTransformer model](https://arxiv.org/abs/2106.11959) that</span>
+<span class="sd">    can be used as the `deeptabular` component of a Wide &amp; Deep model or</span>
+<span class="sd">    independently by itself.</span>
+
+<span class="sd">    Most of the parameters for this class are `Optional` since the use of</span>
+<span class="sd">    categorical or continuous is in fact optional (i.e. one can use</span>
+<span class="sd">    categorical features only, continuous features only or both).</span>
+
+<span class="sd">    Parameters</span>
+<span class="sd">    ----------</span>
+<span class="sd">    column_idx: Dict</span>
+<span class="sd">        Dict containing the index of the columns that will be passed through</span>
+<span class="sd">        the `TabMlp` model. Required to slice the tensors. e.g. _{&#39;education&#39;:</span>
+<span class="sd">        0, &#39;relationship&#39;: 1, &#39;workclass&#39;: 2, ...}_.</span>
+<span class="sd">    cat_embed_input: List, Optional, default = None</span>
+<span class="sd">        List of Tuples with the column name and number of unique values and</span>
+<span class="sd">        embedding dimension. e.g. _[(education, 11), ...]_</span>
+<span class="sd">    cat_embed_dropout: float, Optional, default = None</span>
+<span class="sd">        Categorical embeddings dropout. If `None`, it will default</span>
+<span class="sd">        to 0.</span>
+<span class="sd">    use_cat_bias: bool, Optional, default = None,</span>
+<span class="sd">        Boolean indicating if bias will be used for the categorical embeddings.</span>
+<span class="sd">        If `None`, it will default to &#39;False&#39;.</span>
+<span class="sd">    cat_embed_activation: Optional, str, default = None,</span>
+<span class="sd">        Activation function for the categorical embeddings, if any. Currently</span>
+<span class="sd">        _&#39;tanh&#39;_, _&#39;relu&#39;_, _&#39;leaky_relu&#39;_ and _&#39;gelu&#39;_ are supported</span>
+<span class="sd">    shared_embed: bool, Optional, default = None</span>
+<span class="sd">        Boolean indicating if the embeddings will be &quot;shared&quot;. The idea behind `shared_embed` is</span>
+<span class="sd">        described in the Appendix A in the [TabTransformer paper](https://arxiv.org/abs/2012.06678):</span>
+<span class="sd">        _&#39;The goal of having column embedding is to enable the model to</span>
+<span class="sd">        distinguish the classes in one column from those in the other</span>
+<span class="sd">        columns&#39;_. In other words, the idea is to let the model learn which</span>
+<span class="sd">        column is embedded at the time. See: `pytorch_widedeep.models.transformers._layers.SharedEmbeddings`.</span>
+<span class="sd">    add_shared_embed: bool, Optional, default = None</span>
+<span class="sd">        The two embedding sharing strategies are: 1) add the shared embeddings</span>
+<span class="sd">        to the column embeddings or 2) to replace the first</span>
+<span class="sd">        `frac_shared_embed` with the shared embeddings.</span>
+<span class="sd">        See `pytorch_widedeep.models.embeddings_layers.SharedEmbeddings`</span>
+<span class="sd">        If &#39;None&#39; is passed, it will default to &#39;False&#39;.</span>
+<span class="sd">    frac_shared_embed: float, Optional, default = None</span>
+<span class="sd">        The fraction of embeddings that will be shared (if `add_shared_embed</span>
+<span class="sd">        = False`) by all the different categories for one particular</span>
+<span class="sd">        column. If &#39;None&#39; is passed, it will default to 0.0.</span>
+<span class="sd">    continuous_cols: List, Optional, default = None</span>
+<span class="sd">        List with the name of the numeric (aka continuous) columns</span>
+<span class="sd">    cont_norm_layer: str, Optional, default =  None</span>
+<span class="sd">        Type of normalization layer applied to the continuous features.</span>
+<span class="sd">        Options are: _&#39;layernorm&#39;_ and _&#39;batchnorm&#39;_. if `None`, no</span>
+<span class="sd">        normalization layer will be used.</span>
+<span class="sd">    embed_continuous_method: Optional, str, default = None,</span>
+<span class="sd">        Method to use to embed the continuous features. Options are:</span>
+<span class="sd">        _&#39;standard&#39;_, _&#39;periodic&#39;_ or _&#39;piecewise&#39;_. The _&#39;standard&#39;_</span>
+<span class="sd">        embedding method is based on the FT-Transformer implementation</span>
+<span class="sd">        presented in the paper: [Revisiting Deep Learning Models for</span>
+<span class="sd">        Tabular Data](https://arxiv.org/abs/2106.11959v5). The _&#39;periodic&#39;_</span>
+<span class="sd">        and_&#39;piecewise&#39;_ methods were presented in the paper: [On Embeddings for</span>
+<span class="sd">        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556).</span>
+<span class="sd">        Please, read the papers for details.</span>
+<span class="sd">    cont_embed_dropout: float, Optional, default = None,</span>
+<span class="sd">        Dropout for the continuous embeddings. If `None`, it will default to 0.0</span>
+<span class="sd">    cont_embed_activation: Optional, str, default = None,</span>
+<span class="sd">        Activation function for the continuous embeddings if any. Currently</span>
+<span class="sd">        _&#39;tanh&#39;_, _&#39;relu&#39;_, _&#39;leaky_relu&#39;_ and _&#39;gelu&#39;_ are supported.</span>
+<span class="sd">        If `None`, no activation function will be applied.</span>
+<span class="sd">    quantization_setup: Dict[str, List[float]], Optional, default = None,</span>
+<span class="sd">        This parameter is used when the _&#39;piecewise&#39;_ method is used to embed</span>
+<span class="sd">        the continuous cols. It is a dict where keys are the name of the continuous</span>
+<span class="sd">        columns and values are lists with the boundaries for the quantization</span>
+<span class="sd">        of the continuous_cols. See the examples for details. If</span>
+<span class="sd">        If the _&#39;piecewise&#39;_ method is used, this parameter is required.</span>
+<span class="sd">    n_frequencies: int, Optional, default = None,</span>
+<span class="sd">        This is the so called _&#39;k&#39;_ in their paper [On Embeddings for</span>
+<span class="sd">        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556),</span>
+<span class="sd">        and is the number of &#39;frequencies&#39; that will be used to represent each</span>
+<span class="sd">        continuous column. See their Eq 2 in the paper for details. If</span>
+<span class="sd">        the _&#39;periodic&#39;_ method is used, this parameter is required.</span>
+<span class="sd">    sigma: float, Optional, default = None,</span>
+<span class="sd">        This is the sigma parameter in the paper mentioned when describing the</span>
+<span class="sd">        previous parameters and it is used to initialise the &#39;frequency</span>
+<span class="sd">        weights&#39;. See their Eq 2 in the paper for details. If</span>
+<span class="sd">        the _&#39;periodic&#39;_ method is used, this parameter is required.</span>
+<span class="sd">    share_last_layer: bool, Optional, default = None,</span>
+<span class="sd">        This parameter is not present in the before mentioned paper but it is implemented in</span>
+<span class="sd">        the [official repo](https://github.com/yandex-research/rtdl-num-embeddings/tree/main).</span>
+<span class="sd">        If `True` the linear layer that turns the frequencies into embeddings</span>
+<span class="sd">        will be shared across the continuous columns. If `False` a different</span>
+<span class="sd">        linear layer will be used for each continuous column.</span>
+<span class="sd">        If the _&#39;periodic&#39;_ method is used, this parameter is required.</span>
+<span class="sd">    full_embed_dropout: bool, Optional, default = None,</span>
+<span class="sd">        If `True`, the full embedding corresponding to a column will be masked</span>
+<span class="sd">        out/dropout. If `None`, it will default to `False`.</span>
+<span class="sd">    input_dim: int, default = 64</span>
+<span class="sd">        The so-called *dimension of the model*. Is the number of embeddings used to encode</span>
+<span class="sd">        the categorical and/or continuous columns.</span>
+<span class="sd">    kv_compression_factor: int, default = 0.5</span>
+<span class="sd">        By default, the FTTransformer uses Linear Attention</span>
+<span class="sd">        (See [Linformer: Self-Attention with Linear Complexity](https://arxiv.org/abs/2006.04768&gt;) ).</span>
+<span class="sd">        The compression factor that will be used to reduce the input sequence</span>
+<span class="sd">        length. If we denote the resulting sequence length as</span>
+<span class="sd">        $k = int(kv_{compression \space factor} \times s)$</span>
+<span class="sd">        where $s$ is the input sequence length.</span>
+<span class="sd">    kv_sharing: bool, default = False</span>
+<span class="sd">        Boolean indicating if the $E$ and $F$ projection matrices</span>
+<span class="sd">        will share weights.  See [Linformer: Self-Attention with Linear</span>
+<span class="sd">        Complexity](https://arxiv.org/abs/2006.04768) for details</span>
+<span class="sd">    n_heads: int, default = 8</span>
+<span class="sd">        Number of attention heads per FTTransformer block</span>
+<span class="sd">    use_qkv_bias: bool, default = False</span>
+<span class="sd">        Boolean indicating whether or not to use bias in the Q, K, and V</span>
+<span class="sd">        projection layers</span>
+<span class="sd">    n_blocks: int, default = 4</span>
+<span class="sd">        Number of FTTransformer blocks</span>
+<span class="sd">    attn_dropout: float, default = 0.2</span>
+<span class="sd">        Dropout that will be applied to the Linear-Attention layers</span>
+<span class="sd">    ff_dropout: float, default = 0.1</span>
+<span class="sd">        Dropout that will be applied to the FeedForward network</span>
+<span class="sd">    ff_factor: float, default = 4 / 3</span>
+<span class="sd">        Multiplicative factor applied to the first layer of the FF network in</span>
+<span class="sd">        each Transformer block, This is normally set to 4, but they use 4/3</span>
+<span class="sd">        in the paper.</span>
+<span class="sd">    transformer_activation: str, default = &quot;gelu&quot;</span>
+<span class="sd">        Transformer Encoder activation function. _&#39;tanh&#39;_, _&#39;relu&#39;_,</span>
+<span class="sd">        _&#39;leaky_relu&#39;_, _&#39;gelu&#39;_, _&#39;geglu&#39;_ and _&#39;reglu&#39;_ are supported</span>
+<span class="sd">    mlp_hidden_dims: List, Optional, default = None</span>
+<span class="sd">        List with the number of neurons per dense layer in the MLP. e.g:</span>
+<span class="sd">        _[64, 32]_. If not provided no MLP on top of the final</span>
+<span class="sd">        FTTransformer block will be used.</span>
+<span class="sd">    mlp_activation: str, Optional, default = None</span>
+<span class="sd">        Activation function for the dense layers of the MLP. Currently</span>
+<span class="sd">        _&#39;tanh&#39;_, _&#39;relu&#39;_, _&#39;leaky&#39;_relu&#39; and _&#39;gelu&#39;_ are supported.</span>
+<span class="sd">        If &#39;mlp_hidden_dims&#39; is not `None` and this parameter is `None`, it</span>
+<span class="sd">        will default to _&#39;relu&#39;_.</span>
+<span class="sd">    mlp_dropout: float, Optional, default = None</span>
+<span class="sd">        float with the dropout between the dense layers of the MLP.</span>
+<span class="sd">        If &#39;mlp_hidden_dims&#39; is not `None` and this parameter is `None`, it</span>
+<span class="sd">        will default to 0.0.</span>
+<span class="sd">    mlp_batchnorm: bool, Optional, default = None</span>
+<span class="sd">        Boolean indicating whether or not batch normalization will be applied</span>
+<span class="sd">        to the dense layers</span>
+<span class="sd">        If &#39;mlp_hidden_dims&#39; is not `None` and this parameter is `None`, it</span>
+<span class="sd">        will default to False.</span>
+<span class="sd">    mlp_batchnorm_last: bool, Optional, default = None</span>
+<span class="sd">        Boolean indicating whether or not batch normalization will be applied</span>
+<span class="sd">        to the last of the dense layers</span>
+<span class="sd">        If &#39;mlp_hidden_dims&#39; is not `None` and this parameter is `None`, it</span>
+<span class="sd">        will default to False.</span>
+<span class="sd">    mlp_linear_first: bool, Optional, default = None</span>
+<span class="sd">        Boolean indicating the order of the operations in the dense</span>
+<span class="sd">        layer. If `True: [LIN -&gt; ACT -&gt; BN -&gt; DP]`. If `False: [BN -&gt; DP -&gt;</span>
+<span class="sd">        LIN -&gt; ACT]`</span>
+<span class="sd">        If &#39;mlp_hidden_dims&#39; is not `None` and this parameter is `None`, it</span>
+<span class="sd">        will default to `True`.</span>
+
+<span class="sd">    Attributes</span>
+<span class="sd">    ----------</span>
+<span class="sd">    encoder: nn.Module</span>
+<span class="sd">        Sequence of FTTransformer blocks</span>
+<span class="sd">    mlp: nn.Module</span>
+<span class="sd">        MLP component in the model</span>
+
+<span class="sd">    Examples</span>
+<span class="sd">    --------</span>
+<span class="sd">    &gt;&gt;&gt; import torch</span>
+<span class="sd">    &gt;&gt;&gt; from pytorch_widedeep.models import FTTransformer</span>
+<span class="sd">    &gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)</span>
+<span class="sd">    &gt;&gt;&gt; colnames = [&#39;a&#39;, &#39;b&#39;, &#39;c&#39;, &#39;d&#39;, &#39;e&#39;]</span>
+<span class="sd">    &gt;&gt;&gt; cat_embed_input = [(u,i) for u,i in zip(colnames[:4], [4]*4)]</span>
+<span class="sd">    &gt;&gt;&gt; continuous_cols = [&#39;e&#39;]</span>
+<span class="sd">    &gt;&gt;&gt; column_idx = {k:v for v,k in enumerate(colnames)}</span>
+<span class="sd">    &gt;&gt;&gt; model = FTTransformer(column_idx=column_idx, cat_embed_input=cat_embed_input, continuous_cols=continuous_cols)</span>
+<span class="sd">    &gt;&gt;&gt; out = model(X_tab)</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+
+    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span>
+        <span class="n">column_idx</span><span class="p">:</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">],</span>
+        <span class="o">*</span><span class="p">,</span>
+        <span class="n">cat_embed_input</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="n">Tuple</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">]]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">cat_embed_dropout</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">use_cat_bias</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">cat_embed_activation</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">shared_embed</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">add_shared_embed</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">frac_shared_embed</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">continuous_cols</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">cont_norm_layer</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Literal</span><span class="p">[</span><span class="s2">&quot;batchnorm&quot;</span><span class="p">,</span> <span class="s2">&quot;layernorm&quot;</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">embed_continuous_method</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span>
+            <span class="n">Literal</span><span class="p">[</span><span class="s2">&quot;standard&quot;</span><span class="p">,</span> <span class="s2">&quot;piecewise&quot;</span><span class="p">,</span> <span class="s2">&quot;periodic&quot;</span><span class="p">]</span>
+        <span class="p">]</span> <span class="o">=</span> <span class="s2">&quot;standard&quot;</span><span class="p">,</span>
+        <span class="n">cont_embed_dropout</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">cont_embed_activation</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">quantization_setup</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="nb">float</span><span class="p">]]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">n_frequencies</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">sigma</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">share_last_layer</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">full_embed_dropout</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">input_dim</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">64</span><span class="p">,</span>
+        <span class="n">kv_compression_factor</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.5</span><span class="p">,</span>
+        <span class="n">kv_sharing</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
+        <span class="n">use_qkv_bias</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
+        <span class="n">n_heads</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">8</span><span class="p">,</span>
+        <span class="n">n_blocks</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">4</span><span class="p">,</span>
+        <span class="n">attn_dropout</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.2</span><span class="p">,</span>
+        <span class="n">ff_dropout</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.1</span><span class="p">,</span>
+        <span class="n">ff_factor</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">1.33</span><span class="p">,</span>
+        <span class="n">transformer_activation</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;reglu&quot;</span><span class="p">,</span>
+        <span class="n">mlp_hidden_dims</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">int</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">mlp_activation</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">mlp_dropout</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">mlp_batchnorm</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">mlp_batchnorm_last</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">mlp_linear_first</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+    <span class="p">):</span>
+        <span class="nb">super</span><span class="p">(</span><span class="n">FTTransformer</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span>
+            <span class="n">column_idx</span><span class="o">=</span><span class="n">column_idx</span><span class="p">,</span>
+            <span class="n">cat_embed_input</span><span class="o">=</span><span class="n">cat_embed_input</span><span class="p">,</span>
+            <span class="n">cat_embed_dropout</span><span class="o">=</span><span class="n">cat_embed_dropout</span><span class="p">,</span>
+            <span class="n">use_cat_bias</span><span class="o">=</span><span class="n">use_cat_bias</span><span class="p">,</span>
+            <span class="n">cat_embed_activation</span><span class="o">=</span><span class="n">cat_embed_activation</span><span class="p">,</span>
+            <span class="n">shared_embed</span><span class="o">=</span><span class="n">shared_embed</span><span class="p">,</span>
+            <span class="n">add_shared_embed</span><span class="o">=</span><span class="n">add_shared_embed</span><span class="p">,</span>
+            <span class="n">frac_shared_embed</span><span class="o">=</span><span class="n">frac_shared_embed</span><span class="p">,</span>
+            <span class="n">continuous_cols</span><span class="o">=</span><span class="n">continuous_cols</span><span class="p">,</span>
+            <span class="n">cont_norm_layer</span><span class="o">=</span><span class="n">cont_norm_layer</span><span class="p">,</span>
+            <span class="n">embed_continuous_method</span><span class="o">=</span><span class="n">embed_continuous_method</span><span class="p">,</span>
+            <span class="n">cont_embed_dropout</span><span class="o">=</span><span class="n">cont_embed_dropout</span><span class="p">,</span>
+            <span class="n">cont_embed_activation</span><span class="o">=</span><span class="n">cont_embed_activation</span><span class="p">,</span>
+            <span class="n">input_dim</span><span class="o">=</span><span class="n">input_dim</span><span class="p">,</span>
+            <span class="n">quantization_setup</span><span class="o">=</span><span class="n">quantization_setup</span><span class="p">,</span>
+            <span class="n">n_frequencies</span><span class="o">=</span><span class="n">n_frequencies</span><span class="p">,</span>
+            <span class="n">sigma</span><span class="o">=</span><span class="n">sigma</span><span class="p">,</span>
+            <span class="n">share_last_layer</span><span class="o">=</span><span class="n">share_last_layer</span><span class="p">,</span>
+            <span class="n">full_embed_dropout</span><span class="o">=</span><span class="n">full_embed_dropout</span><span class="p">,</span>
+        <span class="p">)</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">kv_compression_factor</span> <span class="o">=</span> <span class="n">kv_compression_factor</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">kv_sharing</span> <span class="o">=</span> <span class="n">kv_sharing</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">use_qkv_bias</span> <span class="o">=</span> <span class="n">use_qkv_bias</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">n_heads</span> <span class="o">=</span> <span class="n">n_heads</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">n_blocks</span> <span class="o">=</span> <span class="n">n_blocks</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">attn_dropout</span> <span class="o">=</span> <span class="n">attn_dropout</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">ff_dropout</span> <span class="o">=</span> <span class="n">ff_dropout</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">ff_factor</span> <span class="o">=</span> <span class="n">ff_factor</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">transformer_activation</span> <span class="o">=</span> <span class="n">transformer_activation</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">mlp_hidden_dims</span> <span class="o">=</span> <span class="n">mlp_hidden_dims</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">mlp_activation</span> <span class="o">=</span> <span class="n">mlp_activation</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">mlp_dropout</span> <span class="o">=</span> <span class="n">mlp_dropout</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">mlp_batchnorm</span> <span class="o">=</span> <span class="n">mlp_batchnorm</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">mlp_batchnorm_last</span> <span class="o">=</span> <span class="n">mlp_batchnorm_last</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">mlp_linear_first</span> <span class="o">=</span> <span class="n">mlp_linear_first</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">with_cls_token</span> <span class="o">=</span> <span class="s2">&quot;cls_token&quot;</span> <span class="ow">in</span> <span class="n">column_idx</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">n_cat</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">cat_embed_input</span><span class="p">)</span> <span class="k">if</span> <span class="n">cat_embed_input</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="k">else</span> <span class="mi">0</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">n_cont</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">continuous_cols</span><span class="p">)</span> <span class="k">if</span> <span class="n">continuous_cols</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="k">else</span> <span class="mi">0</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">n_feats</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">n_cat</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">n_cont</span>
+
+        <span class="c1"># Embeddings are instantiated at the base model</span>
+        <span class="c1"># Transformer blocks</span>
+        <span class="n">is_first</span> <span class="o">=</span> <span class="kc">True</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">encoder</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Sequential</span><span class="p">()</span>
+        <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">n_blocks</span><span class="p">):</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">encoder</span><span class="o">.</span><span class="n">add_module</span><span class="p">(</span>
+                <span class="s2">&quot;fttransformer_block&quot;</span> <span class="o">+</span> <span class="nb">str</span><span class="p">(</span><span class="n">i</span><span class="p">),</span>
+                <span class="n">FTTransformerEncoder</span><span class="p">(</span>
+                    <span class="n">input_dim</span><span class="p">,</span>
+                    <span class="bp">self</span><span class="o">.</span><span class="n">n_feats</span><span class="p">,</span>
+                    <span class="n">n_heads</span><span class="p">,</span>
+                    <span class="n">use_qkv_bias</span><span class="p">,</span>
+                    <span class="n">attn_dropout</span><span class="p">,</span>
+                    <span class="n">ff_dropout</span><span class="p">,</span>
+                    <span class="n">ff_factor</span><span class="p">,</span>
+                    <span class="n">kv_compression_factor</span><span class="p">,</span>
+                    <span class="n">kv_sharing</span><span class="p">,</span>
+                    <span class="n">transformer_activation</span><span class="p">,</span>
+                    <span class="n">is_first</span><span class="p">,</span>
+                <span class="p">),</span>
+            <span class="p">)</span>
+            <span class="n">is_first</span> <span class="o">=</span> <span class="kc">False</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">mlp_first_hidden_dim</span> <span class="o">=</span> <span class="p">(</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">input_dim</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">with_cls_token</span> <span class="k">else</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">n_feats</span> <span class="o">*</span> <span class="bp">self</span><span class="o">.</span><span class="n">input_dim</span><span class="p">)</span>
+        <span class="p">)</span>
+
+        <span class="c1"># Mlp: adding an MLP on top of the Resnet blocks is optional and</span>
+        <span class="c1"># therefore all related params are optional</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_hidden_dims</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">mlp</span> <span class="o">=</span> <span class="n">MLP</span><span class="p">(</span>
+                <span class="n">d_hidden</span><span class="o">=</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">mlp_first_hidden_dim</span><span class="p">]</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_hidden_dims</span><span class="p">,</span>
+                <span class="n">activation</span><span class="o">=</span><span class="p">(</span>
+                    <span class="s2">&quot;relu&quot;</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_activation</span> <span class="ow">is</span> <span class="kc">None</span> <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_activation</span>
+                <span class="p">),</span>
+                <span class="n">dropout</span><span class="o">=</span><span class="mf">0.0</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_dropout</span> <span class="ow">is</span> <span class="kc">None</span> <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_dropout</span><span class="p">,</span>
+                <span class="n">batchnorm</span><span class="o">=</span><span class="kc">False</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_batchnorm</span> <span class="ow">is</span> <span class="kc">None</span> <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_batchnorm</span><span class="p">,</span>
+                <span class="n">batchnorm_last</span><span class="o">=</span><span class="p">(</span>
+                    <span class="kc">False</span>
+                    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_batchnorm_last</span> <span class="ow">is</span> <span class="kc">None</span>
+                    <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_batchnorm_last</span>
+                <span class="p">),</span>
+                <span class="n">linear_first</span><span class="o">=</span><span class="p">(</span>
+                    <span class="kc">False</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_linear_first</span> <span class="ow">is</span> <span class="kc">None</span> <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_linear_first</span>
+                <span class="p">),</span>
+            <span class="p">)</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">mlp</span> <span class="o">=</span> <span class="kc">None</span>
+
+    <span class="k">def</span> <span class="nf">forward</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">X</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tensor</span><span class="p">:</span>
+        <span class="n">x</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_embeddings</span><span class="p">(</span><span class="n">X</span><span class="p">)</span>
+        <span class="n">x</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">encoder</span><span class="p">(</span><span class="n">x</span><span class="p">)</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">with_cls_token</span><span class="p">:</span>
+            <span class="n">x</span> <span class="o">=</span> <span class="n">x</span><span class="p">[:,</span> <span class="mi">0</span><span class="p">,</span> <span class="p">:]</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="n">x</span> <span class="o">=</span> <span class="n">x</span><span class="o">.</span><span class="n">flatten</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="n">x</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp</span><span class="p">(</span><span class="n">x</span><span class="p">)</span>
+        <span class="k">return</span> <span class="n">x</span>
+
+    <span class="nd">@property</span>
+    <span class="k">def</span> <span class="nf">output_dim</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">int</span><span class="p">:</span>
+<span class="w">        </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;The output dimension of the model. This is a required property</span>
+<span class="sd">        neccesary to build the `WideDeep` class</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="k">return</span> <span class="p">(</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">mlp_hidden_dims</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span>
+            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_hidden_dims</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span>
+            <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_first_hidden_dim</span>
         <span class="p">)</span>
-    <span class="k">else</span><span class="p">:</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">mlp</span> <span class="o">=</span> <span class="kc">None</span>
+
+    <span class="nd">@property</span>
+    <span class="k">def</span> <span class="nf">attention_weights</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">List</span><span class="p">[</span><span class="n">Tensor</span><span class="p">]:</span>
+<span class="w">        </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;List with the attention weights per block</span>
+
+<span class="sd">        The shape of the attention weights is: $(N, H, F, k)$, where $N$ is</span>
+<span class="sd">        the batch size, $H$ is the number of attention heads, $F$ is the</span>
+<span class="sd">        number of features/columns and $k$ is the reduced sequence length or</span>
+<span class="sd">        dimension, i.e. $k = int(kv_{compression \space factor} \times s)$</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="k">return</span> <span class="p">[</span><span class="n">blk</span><span class="o">.</span><span class="n">attn</span><span class="o">.</span><span class="n">attn_weights</span> <span class="k">for</span> <span class="n">blk</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">encoder</span><span class="p">]</span>
 </code></pre></div></td></tr></table></div>
-                  </details>
+              </details>
 
 
 
@@ -8418,47 +12934,7 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_perceiver.TabPerceiver"
 
 
 <a href="#pytorch_widedeep.models.tabular.transformers.tab_perceiver.TabPerceiver" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="nf">TabPerceiver</span><span class="p">(</span>
-    <span class="n">column_idx</span><span class="p">,</span>
-    <span class="o">*</span><span class="p">,</span>
-    <span class="n">cat_embed_input</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">cat_embed_dropout</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">use_cat_bias</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">cat_embed_activation</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">shared_embed</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">add_shared_embed</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">frac_shared_embed</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">continuous_cols</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">cont_norm_layer</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">embed_continuous_method</span><span class="o">=</span><span class="s2">&quot;standard&quot;</span><span class="p">,</span>
-    <span class="n">cont_embed_dropout</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">cont_embed_activation</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">quantization_setup</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">n_frequencies</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">sigma</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">share_last_layer</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">full_embed_dropout</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">input_dim</span><span class="o">=</span><span class="mi">32</span><span class="p">,</span>
-    <span class="n">n_cross_attns</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span>
-    <span class="n">n_cross_attn_heads</span><span class="o">=</span><span class="mi">4</span><span class="p">,</span>
-    <span class="n">n_latents</span><span class="o">=</span><span class="mi">16</span><span class="p">,</span>
-    <span class="n">latent_dim</span><span class="o">=</span><span class="mi">128</span><span class="p">,</span>
-    <span class="n">n_latent_heads</span><span class="o">=</span><span class="mi">4</span><span class="p">,</span>
-    <span class="n">n_latent_blocks</span><span class="o">=</span><span class="mi">4</span><span class="p">,</span>
-    <span class="n">n_perceiver_blocks</span><span class="o">=</span><span class="mi">4</span><span class="p">,</span>
-    <span class="n">share_weights</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
-    <span class="n">attn_dropout</span><span class="o">=</span><span class="mf">0.1</span><span class="p">,</span>
-    <span class="n">ff_dropout</span><span class="o">=</span><span class="mf">0.1</span><span class="p">,</span>
-    <span class="n">ff_factor</span><span class="o">=</span><span class="mi">4</span><span class="p">,</span>
-    <span class="n">transformer_activation</span><span class="o">=</span><span class="s2">&quot;geglu&quot;</span><span class="p">,</span>
-    <span class="n">mlp_hidden_dims</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">mlp_activation</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">mlp_dropout</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">mlp_batchnorm</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">mlp_batchnorm_last</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">mlp_linear_first</span><span class="o">=</span><span class="kc">None</span>
-<span class="p">)</span>
-</code></pre></div>
+
 
     <div class="doc doc-contents first">
             <p class="doc doc-class-bases">
@@ -8479,132 +12955,183 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_perceiver.TabPerceiver"
 
 
 <p><span class="doc-section-title">Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>column_idx</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, int]</code>)
-          –
-          <div class="doc-md-description">
-            <p>Dict containing the index of the columns that will be passed through
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+          <th>Default</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>column_idx</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, int]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Dict containing the index of the columns that will be passed through
 the <code>TabMlp</code> model. Required to slice the tensors. e.g. <em>{'education':
 0, 'relationship': 1, 'workclass': 2, ...}</em>.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>cat_embed_input</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.wdtypes.Tuple">Tuple</span>[str, int]]]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>List of Tuples with the column name and number of unique values and
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>cat_embed_input</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.wdtypes.Tuple">Tuple</span>[str, int]]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>List of Tuples with the column name and number of unique values and
 embedding dimension. e.g. <em>[(education, 11), ...]</em></p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>cat_embed_dropout</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Categorical embeddings dropout. If <code>None</code>, it will default
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>cat_embed_dropout</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Categorical embeddings dropout. If <code>None</code>, it will default
 to 0.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>use_cat_bias</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Boolean indicating if bias will be used for the categorical embeddings.
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>use_cat_bias</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Boolean indicating if bias will be used for the categorical embeddings.
 If <code>None</code>, it will default to 'False'.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>cat_embed_activation</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Activation function for the categorical embeddings, if any. Currently
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>cat_embed_activation</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Activation function for the categorical embeddings, if any. Currently
 <em>'tanh'</em>, <em>'relu'</em>, <em>'leaky_relu'</em> and <em>'gelu'</em> are supported</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>shared_embed</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Boolean indicating if the embeddings will be "shared". The idea behind <code>shared_embed</code> is
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>shared_embed</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Boolean indicating if the embeddings will be "shared". The idea behind <code>shared_embed</code> is
 described in the Appendix A in the <a href="https://arxiv.org/abs/2012.06678">TabTransformer paper</a>:
 <em>'The goal of having column embedding is to enable the model to
 distinguish the classes in one column from those in the other
 columns'</em>. In other words, the idea is to let the model learn which
 column is embedded at the time. See: <code>pytorch_widedeep.models.transformers._layers.SharedEmbeddings</code>.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>add_shared_embed</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>The two embedding sharing strategies are: 1) add the shared embeddings
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>add_shared_embed</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>The two embedding sharing strategies are: 1) add the shared embeddings
 to the column embeddings or 2) to replace the first
 <code>frac_shared_embed</code> with the shared embeddings.
 See <code>pytorch_widedeep.models.embeddings_layers.SharedEmbeddings</code>
 If 'None' is passed, it will default to 'False'.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>frac_shared_embed</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>The fraction of embeddings that will be shared (if <code>add_shared_embed
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>frac_shared_embed</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>The fraction of embeddings that will be shared (if <code>add_shared_embed
 = False</code>) by all the different categories for one particular
 column. If 'None' is passed, it will default to 0.0.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>continuous_cols</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[str]]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>List with the name of the numeric (aka continuous) columns</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>cont_norm_layer</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[batchnorm, layernorm]]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Type of normalization layer applied to the continuous features.
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>continuous_cols</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[str]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>List with the name of the numeric (aka continuous) columns</p>
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>cont_norm_layer</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[batchnorm, layernorm]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Type of normalization layer applied to the continuous features.
 Options are: <em>'layernorm'</em> and <em>'batchnorm'</em>. if <code>None</code>, no
 normalization layer will be used.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>embed_continuous_method</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[standard, piecewise, periodic]]</code>, default:
-                  <code>&#39;standard&#39;</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Method to use to embed the continuous features. Options are:
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>embed_continuous_method</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[standard, piecewise, periodic]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Method to use to embed the continuous features. Options are:
 <em>'standard'</em>, <em>'periodic'</em> or <em>'piecewise'</em>. The <em>'standard'</em>
 embedding method is based on the FT-Transformer implementation
 presented in the paper: <a href="https://arxiv.org/abs/2106.11959v5">Revisiting Deep Learning Models for
@@ -8612,350 +13139,477 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_perceiver.TabPerceiver"
 and_'piecewise'_ methods were presented in the paper: <a href="https://arxiv.org/abs/2203.05556">On Embeddings for
 Numerical Features in Tabular Deep Learning</a>.
 Please, read the papers for details.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>cont_embed_dropout</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>cont_embed_activation</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Activation function for the continuous embeddings if any. Currently
+              </div>
+            </td>
+            <td>
+                  <code>&#39;standard&#39;</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>cont_embed_dropout</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p>
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>cont_embed_activation</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Activation function for the continuous embeddings if any. Currently
 <em>'tanh'</em>, <em>'relu'</em>, <em>'leaky_relu'</em> and <em>'gelu'</em> are supported.
 If <code>None</code>, no activation function will be applied.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>quantization_setup</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, <span title="pytorch_widedeep.wdtypes.List">List</span>[float]]]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>This parameter is used when the <em>'piecewise'</em> method is used to embed
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>quantization_setup</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, <span title="pytorch_widedeep.wdtypes.List">List</span>[float]]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>This parameter is used when the <em>'piecewise'</em> method is used to embed
 the continuous cols. It is a dict where keys are the name of the continuous
 columns and values are lists with the boundaries for the quantization
 of the continuous_cols. See the examples for details. If
 If the <em>'piecewise'</em> method is used, this parameter is required.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>n_frequencies</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[int]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>This is the so called <em>'k'</em> in their paper <a href="https://arxiv.org/abs/2203.05556">On Embeddings for
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>n_frequencies</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[int]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>This is the so called <em>'k'</em> in their paper <a href="https://arxiv.org/abs/2203.05556">On Embeddings for
 Numerical Features in Tabular Deep Learning</a>,
 and is the number of 'frequencies' that will be used to represent each
 continuous column. See their Eq 2 in the paper for details. If
 the <em>'periodic'</em> method is used, this parameter is required.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>sigma</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>This is the sigma parameter in the paper mentioned when describing the
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>sigma</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>This is the sigma parameter in the paper mentioned when describing the
 previous parameters and it is used to initialise the 'frequency
 weights'. See their Eq 2 in the paper for details. If
 the <em>'periodic'</em> method is used, this parameter is required.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>share_last_layer</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>This parameter is not present in the before mentioned paper but it is implemented in
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>share_last_layer</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>This parameter is not present in the before mentioned paper but it is implemented in
 the <a href="https://github.com/yandex-research/rtdl-num-embeddings/tree/main">official repo</a>.
 If <code>True</code> the linear layer that turns the frequencies into embeddings
 will be shared across the continuous columns. If <code>False</code> a different
 linear layer will be used for each continuous column.
 If the <em>'periodic'</em> method is used, this parameter is required.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>full_embed_dropout</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>If <code>True</code>, the full embedding corresponding to a column will be masked
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>full_embed_dropout</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>If <code>True</code>, the full embedding corresponding to a column will be masked
 out/dropout. If <code>None</code>, it will default to <code>False</code>.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>input_dim</code></b>
-              (<code>int</code>, default:
-                  <code>32</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>The so-called <em>dimension of the model</em>. Is the number of embeddings
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>input_dim</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>The so-called <em>dimension of the model</em>. Is the number of embeddings
 used to encode the categorical and/or continuous columns.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>n_cross_attns</code></b>
-              (<code>int</code>, default:
-                  <code>1</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Number of times each perceiver block will cross attend to the input
+              </div>
+            </td>
+            <td>
+                  <code>32</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>n_cross_attns</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Number of times each perceiver block will cross attend to the input
 data (i.e. number of cross attention components per perceiver block).
 This should normally be 1. However, in the paper they describe some
 architectures (normally computer vision-related problems) where the
 Perceiver attends multiple times to the input array. Therefore, maybe
 multiple cross attention to the input array is also useful in some
 cases for tabular data <img alt="🤷" class="emojione" src="https://cdnjs.cloudflare.com/ajax/libs/emojione/2.2.7/assets/png/1f937.png" title=":shrug:" /> .</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>n_cross_attn_heads</code></b>
-              (<code>int</code>, default:
+              </div>
+            </td>
+            <td>
+                  <code>1</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>n_cross_attn_heads</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Number of attention heads for the cross attention component</p>
+              </div>
+            </td>
+            <td>
                   <code>4</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Number of attention heads for the cross attention component</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>n_latents</code></b>
-              (<code>int</code>, default:
-                  <code>16</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Number of latents. This is the <span class="arithmatex">\(N\)</span> parameter in the paper. As
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>n_latents</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Number of latents. This is the <span class="arithmatex">\(N\)</span> parameter in the paper. As
 indicated in the paper, this number should be significantly lower
 than <span class="arithmatex">\(M\)</span> (the number of columns in the dataset). Setting <span class="arithmatex">\(N\)</span> closer
 to <span class="arithmatex">\(M\)</span> defies the main purpose of the Perceiver, which is to overcome
 the transformer quadratic bottleneck</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>latent_dim</code></b>
-              (<code>int</code>, default:
+              </div>
+            </td>
+            <td>
+                  <code>16</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>latent_dim</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Latent dimension.</p>
+              </div>
+            </td>
+            <td>
                   <code>128</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Latent dimension.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>n_latent_heads</code></b>
-              (<code>int</code>, default:
-                  <code>4</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Number of attention heads per Latent Transformer</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>n_latent_blocks</code></b>
-              (<code>int</code>, default:
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>n_latent_heads</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Number of attention heads per Latent Transformer</p>
+              </div>
+            </td>
+            <td>
                   <code>4</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Number of transformer encoder blocks (normalised MHA + normalised FF)
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>n_latent_blocks</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Number of transformer encoder blocks (normalised MHA + normalised FF)
 per Latent Transformer</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>n_perceiver_blocks</code></b>
-              (<code>int</code>, default:
+              </div>
+            </td>
+            <td>
                   <code>4</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Number of Perceiver blocks defined as [Cross Attention + Latent
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>n_perceiver_blocks</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Number of Perceiver blocks defined as [Cross Attention + Latent
 Transformer]</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>share_weights</code></b>
-              (<code>bool</code>, default:
-                  <code>False</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Boolean indicating if the weights will be shared between Perceiver
+              </div>
+            </td>
+            <td>
+                  <code>4</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>share_weights</code></td>
+            <td>
+                  <code>bool</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Boolean indicating if the weights will be shared between Perceiver
 blocks</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>attn_dropout</code></b>
-              (<code>float</code>, default:
+              </div>
+            </td>
+            <td>
+                  <code>False</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>attn_dropout</code></td>
+            <td>
+                  <code>float</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Dropout that will be applied to the Multi-Head Attention layers</p>
+              </div>
+            </td>
+            <td>
                   <code>0.1</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Dropout that will be applied to the Multi-Head Attention layers</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>ff_dropout</code></b>
-              (<code>float</code>, default:
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>ff_dropout</code></td>
+            <td>
+                  <code>float</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Dropout that will be applied to the FeedForward network</p>
+              </div>
+            </td>
+            <td>
                   <code>0.1</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Dropout that will be applied to the FeedForward network</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>ff_factor</code></b>
-              (<code>int</code>, default:
-                  <code>4</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Multiplicative factor applied to the first layer of the FF network in
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>ff_factor</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Multiplicative factor applied to the first layer of the FF network in
 each Transformer block, This is normally set to 4.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>transformer_activation</code></b>
-              (<code>str</code>, default:
-                  <code>&#39;geglu&#39;</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Transformer Encoder activation function. <em>'tanh'</em>, <em>'relu'</em>,
+              </div>
+            </td>
+            <td>
+                  <code>4</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>transformer_activation</code></td>
+            <td>
+                  <code>str</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Transformer Encoder activation function. <em>'tanh'</em>, <em>'relu'</em>,
 <em>'leaky_relu'</em>, <em>'gelu'</em>, <em>'geglu'</em> and <em>'reglu'</em> are supported</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>mlp_hidden_dims</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[int]]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>List with the number of neurons per dense layer in the MLP. e.g:
+              </div>
+            </td>
+            <td>
+                  <code>&#39;geglu&#39;</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>mlp_hidden_dims</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[int]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>List with the number of neurons per dense layer in the MLP. e.g:
 <em>[64, 32]</em>. If not provided no MLP on top of the final
 Transformer block will be used.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>mlp_activation</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Activation function for the dense layers of the MLP. Currently
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>mlp_activation</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Activation function for the dense layers of the MLP. Currently
 <em>'tanh'</em>, <em>'relu'</em>, <em>'leaky'_relu' and _'gelu'</em> are supported.
 If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it
 will default to <em>'relu'</em>.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>mlp_dropout</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>float with the dropout between the dense layers of the MLP.
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>mlp_dropout</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>float with the dropout between the dense layers of the MLP.
 If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it
 will default to 0.0.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>mlp_batchnorm</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Boolean indicating whether or not batch normalization will be applied
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>mlp_batchnorm</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Boolean indicating whether or not batch normalization will be applied
 to the dense layers
 If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it
 will default to False.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>mlp_batchnorm_last</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Boolean indicating whether or not batch normalization will be applied
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>mlp_batchnorm_last</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Boolean indicating whether or not batch normalization will be applied
 to the last of the dense layers
 If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it
 will default to False.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>mlp_linear_first</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Boolean indicating the order of the operations in the dense
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>mlp_linear_first</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Boolean indicating the order of the operations in the dense
 layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt;
 LIN -&gt; ACT]</code>
 If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it
 will default to <code>True</code>.</p>
-          </div>
-        </li>
-    </ul>
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
 
 <p><span class="doc-section-title">Attributes:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code><span title="pytorch_widedeep.models.tabular.transformers.tab_perceiver.TabPerceiver.encoder">encoder</span></code></b>
-              (<code><span title="torch.nn.ModuleDict">ModuleDict</span></code>)
-          –
-          <div class="doc-md-description">
-            <p>ModuleDict with the Perceiver blocks</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code><span title="pytorch_widedeep.models.tabular.transformers.tab_perceiver.TabPerceiver.latents">latents</span></code></b>
-              (<code><span title="torch.nn.Parameter">Parameter</span></code>)
-          –
-          <div class="doc-md-description">
-            <p>Latents that will be used for prediction</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code><span title="pytorch_widedeep.models.tabular.transformers.tab_perceiver.TabPerceiver.mlp">mlp</span></code></b>
-              (<code><span title="torch.nn.Module">Module</span></code>)
-          –
-          <div class="doc-md-description">
-            <p>MLP component in the model</p>
-          </div>
-        </li>
-    </ul>
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code><span title="pytorch_widedeep.models.tabular.transformers.tab_perceiver.TabPerceiver.encoder">encoder</span></code></td>
+            <td>
+                  <code><span title="torch.nn.ModuleDict">ModuleDict</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>ModuleDict with the Perceiver blocks</p>
+              </div>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code><span title="pytorch_widedeep.models.tabular.transformers.tab_perceiver.TabPerceiver.latents">latents</span></code></td>
+            <td>
+                  <code><span title="torch.nn.Parameter">Parameter</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Latents that will be used for prediction</p>
+              </div>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code><span title="pytorch_widedeep.models.tabular.transformers.tab_perceiver.TabPerceiver.mlp">mlp</span></code></td>
+            <td>
+                  <code><span title="torch.nn.Module">Module</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>MLP component in the model</p>
+              </div>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
 
 <p><span class="doc-section-title">Examples:</span></p>
@@ -8972,81 +13626,275 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_perceiver.TabPerceiver"
 <span class="gp">&gt;&gt;&gt; </span><span class="n">out</span> <span class="o">=</span> <span class="n">model</span><span class="p">(</span><span class="n">X_tab</span><span class="p">)</span>
 </code></pre></div>
 
-                  <details class="quote">
-                    <summary>Source code in <code>pytorch_widedeep/models/tabular/transformers/tab_perceiver.py</code></summary>
-                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">207</span>
-<span class="normal">208</span>
-<span class="normal">209</span>
-<span class="normal">210</span>
-<span class="normal">211</span>
-<span class="normal">212</span>
-<span class="normal">213</span>
-<span class="normal">214</span>
-<span class="normal">215</span>
-<span class="normal">216</span>
-<span class="normal">217</span>
-<span class="normal">218</span>
-<span class="normal">219</span>
-<span class="normal">220</span>
-<span class="normal">221</span>
-<span class="normal">222</span>
-<span class="normal">223</span>
-<span class="normal">224</span>
-<span class="normal">225</span>
-<span class="normal">226</span>
-<span class="normal">227</span>
-<span class="normal">228</span>
-<span class="normal">229</span>
-<span class="normal">230</span>
-<span class="normal">231</span>
-<span class="normal">232</span>
-<span class="normal">233</span>
-<span class="normal">234</span>
-<span class="normal">235</span>
-<span class="normal">236</span>
-<span class="normal">237</span>
-<span class="normal">238</span>
-<span class="normal">239</span>
-<span class="normal">240</span>
-<span class="normal">241</span>
-<span class="normal">242</span>
-<span class="normal">243</span>
-<span class="normal">244</span>
-<span class="normal">245</span>
-<span class="normal">246</span>
-<span class="normal">247</span>
-<span class="normal">248</span>
-<span class="normal">249</span>
-<span class="normal">250</span>
-<span class="normal">251</span>
-<span class="normal">252</span>
-<span class="normal">253</span>
-<span class="normal">254</span>
-<span class="normal">255</span>
-<span class="normal">256</span>
-<span class="normal">257</span>
-<span class="normal">258</span>
-<span class="normal">259</span>
-<span class="normal">260</span>
-<span class="normal">261</span>
-<span class="normal">262</span>
-<span class="normal">263</span>
-<span class="normal">264</span>
-<span class="normal">265</span>
-<span class="normal">266</span>
-<span class="normal">267</span>
-<span class="normal">268</span>
-<span class="normal">269</span>
-<span class="normal">270</span>
-<span class="normal">271</span>
-<span class="normal">272</span>
-<span class="normal">273</span>
-<span class="normal">274</span>
-<span class="normal">275</span>
-<span class="normal">276</span>
-<span class="normal">277</span>
-<span class="normal">278</span>
-<span class="normal">279</span>
+              <details class="quote">
+                <summary>Source code in <code>pytorch_widedeep/models/tabular/transformers/tab_perceiver.py</code></summary>
+                <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 13</span>
+<span class="normal"> 14</span>
+<span class="normal"> 15</span>
+<span class="normal"> 16</span>
+<span class="normal"> 17</span>
+<span class="normal"> 18</span>
+<span class="normal"> 19</span>
+<span class="normal"> 20</span>
+<span class="normal"> 21</span>
+<span class="normal"> 22</span>
+<span class="normal"> 23</span>
+<span class="normal"> 24</span>
+<span class="normal"> 25</span>
+<span class="normal"> 26</span>
+<span class="normal"> 27</span>
+<span class="normal"> 28</span>
+<span class="normal"> 29</span>
+<span class="normal"> 30</span>
+<span class="normal"> 31</span>
+<span class="normal"> 32</span>
+<span class="normal"> 33</span>
+<span class="normal"> 34</span>
+<span class="normal"> 35</span>
+<span class="normal"> 36</span>
+<span class="normal"> 37</span>
+<span class="normal"> 38</span>
+<span class="normal"> 39</span>
+<span class="normal"> 40</span>
+<span class="normal"> 41</span>
+<span class="normal"> 42</span>
+<span class="normal"> 43</span>
+<span class="normal"> 44</span>
+<span class="normal"> 45</span>
+<span class="normal"> 46</span>
+<span class="normal"> 47</span>
+<span class="normal"> 48</span>
+<span class="normal"> 49</span>
+<span class="normal"> 50</span>
+<span class="normal"> 51</span>
+<span class="normal"> 52</span>
+<span class="normal"> 53</span>
+<span class="normal"> 54</span>
+<span class="normal"> 55</span>
+<span class="normal"> 56</span>
+<span class="normal"> 57</span>
+<span class="normal"> 58</span>
+<span class="normal"> 59</span>
+<span class="normal"> 60</span>
+<span class="normal"> 61</span>
+<span class="normal"> 62</span>
+<span class="normal"> 63</span>
+<span class="normal"> 64</span>
+<span class="normal"> 65</span>
+<span class="normal"> 66</span>
+<span class="normal"> 67</span>
+<span class="normal"> 68</span>
+<span class="normal"> 69</span>
+<span class="normal"> 70</span>
+<span class="normal"> 71</span>
+<span class="normal"> 72</span>
+<span class="normal"> 73</span>
+<span class="normal"> 74</span>
+<span class="normal"> 75</span>
+<span class="normal"> 76</span>
+<span class="normal"> 77</span>
+<span class="normal"> 78</span>
+<span class="normal"> 79</span>
+<span class="normal"> 80</span>
+<span class="normal"> 81</span>
+<span class="normal"> 82</span>
+<span class="normal"> 83</span>
+<span class="normal"> 84</span>
+<span class="normal"> 85</span>
+<span class="normal"> 86</span>
+<span class="normal"> 87</span>
+<span class="normal"> 88</span>
+<span class="normal"> 89</span>
+<span class="normal"> 90</span>
+<span class="normal"> 91</span>
+<span class="normal"> 92</span>
+<span class="normal"> 93</span>
+<span class="normal"> 94</span>
+<span class="normal"> 95</span>
+<span class="normal"> 96</span>
+<span class="normal"> 97</span>
+<span class="normal"> 98</span>
+<span class="normal"> 99</span>
+<span class="normal">100</span>
+<span class="normal">101</span>
+<span class="normal">102</span>
+<span class="normal">103</span>
+<span class="normal">104</span>
+<span class="normal">105</span>
+<span class="normal">106</span>
+<span class="normal">107</span>
+<span class="normal">108</span>
+<span class="normal">109</span>
+<span class="normal">110</span>
+<span class="normal">111</span>
+<span class="normal">112</span>
+<span class="normal">113</span>
+<span class="normal">114</span>
+<span class="normal">115</span>
+<span class="normal">116</span>
+<span class="normal">117</span>
+<span class="normal">118</span>
+<span class="normal">119</span>
+<span class="normal">120</span>
+<span class="normal">121</span>
+<span class="normal">122</span>
+<span class="normal">123</span>
+<span class="normal">124</span>
+<span class="normal">125</span>
+<span class="normal">126</span>
+<span class="normal">127</span>
+<span class="normal">128</span>
+<span class="normal">129</span>
+<span class="normal">130</span>
+<span class="normal">131</span>
+<span class="normal">132</span>
+<span class="normal">133</span>
+<span class="normal">134</span>
+<span class="normal">135</span>
+<span class="normal">136</span>
+<span class="normal">137</span>
+<span class="normal">138</span>
+<span class="normal">139</span>
+<span class="normal">140</span>
+<span class="normal">141</span>
+<span class="normal">142</span>
+<span class="normal">143</span>
+<span class="normal">144</span>
+<span class="normal">145</span>
+<span class="normal">146</span>
+<span class="normal">147</span>
+<span class="normal">148</span>
+<span class="normal">149</span>
+<span class="normal">150</span>
+<span class="normal">151</span>
+<span class="normal">152</span>
+<span class="normal">153</span>
+<span class="normal">154</span>
+<span class="normal">155</span>
+<span class="normal">156</span>
+<span class="normal">157</span>
+<span class="normal">158</span>
+<span class="normal">159</span>
+<span class="normal">160</span>
+<span class="normal">161</span>
+<span class="normal">162</span>
+<span class="normal">163</span>
+<span class="normal">164</span>
+<span class="normal">165</span>
+<span class="normal">166</span>
+<span class="normal">167</span>
+<span class="normal">168</span>
+<span class="normal">169</span>
+<span class="normal">170</span>
+<span class="normal">171</span>
+<span class="normal">172</span>
+<span class="normal">173</span>
+<span class="normal">174</span>
+<span class="normal">175</span>
+<span class="normal">176</span>
+<span class="normal">177</span>
+<span class="normal">178</span>
+<span class="normal">179</span>
+<span class="normal">180</span>
+<span class="normal">181</span>
+<span class="normal">182</span>
+<span class="normal">183</span>
+<span class="normal">184</span>
+<span class="normal">185</span>
+<span class="normal">186</span>
+<span class="normal">187</span>
+<span class="normal">188</span>
+<span class="normal">189</span>
+<span class="normal">190</span>
+<span class="normal">191</span>
+<span class="normal">192</span>
+<span class="normal">193</span>
+<span class="normal">194</span>
+<span class="normal">195</span>
+<span class="normal">196</span>
+<span class="normal">197</span>
+<span class="normal">198</span>
+<span class="normal">199</span>
+<span class="normal">200</span>
+<span class="normal">201</span>
+<span class="normal">202</span>
+<span class="normal">203</span>
+<span class="normal">204</span>
+<span class="normal">205</span>
+<span class="normal">206</span>
+<span class="normal">207</span>
+<span class="normal">208</span>
+<span class="normal">209</span>
+<span class="normal">210</span>
+<span class="normal">211</span>
+<span class="normal">212</span>
+<span class="normal">213</span>
+<span class="normal">214</span>
+<span class="normal">215</span>
+<span class="normal">216</span>
+<span class="normal">217</span>
+<span class="normal">218</span>
+<span class="normal">219</span>
+<span class="normal">220</span>
+<span class="normal">221</span>
+<span class="normal">222</span>
+<span class="normal">223</span>
+<span class="normal">224</span>
+<span class="normal">225</span>
+<span class="normal">226</span>
+<span class="normal">227</span>
+<span class="normal">228</span>
+<span class="normal">229</span>
+<span class="normal">230</span>
+<span class="normal">231</span>
+<span class="normal">232</span>
+<span class="normal">233</span>
+<span class="normal">234</span>
+<span class="normal">235</span>
+<span class="normal">236</span>
+<span class="normal">237</span>
+<span class="normal">238</span>
+<span class="normal">239</span>
+<span class="normal">240</span>
+<span class="normal">241</span>
+<span class="normal">242</span>
+<span class="normal">243</span>
+<span class="normal">244</span>
+<span class="normal">245</span>
+<span class="normal">246</span>
+<span class="normal">247</span>
+<span class="normal">248</span>
+<span class="normal">249</span>
+<span class="normal">250</span>
+<span class="normal">251</span>
+<span class="normal">252</span>
+<span class="normal">253</span>
+<span class="normal">254</span>
+<span class="normal">255</span>
+<span class="normal">256</span>
+<span class="normal">257</span>
+<span class="normal">258</span>
+<span class="normal">259</span>
+<span class="normal">260</span>
+<span class="normal">261</span>
+<span class="normal">262</span>
+<span class="normal">263</span>
+<span class="normal">264</span>
+<span class="normal">265</span>
+<span class="normal">266</span>
+<span class="normal">267</span>
+<span class="normal">268</span>
+<span class="normal">269</span>
+<span class="normal">270</span>
+<span class="normal">271</span>
+<span class="normal">272</span>
+<span class="normal">273</span>
+<span class="normal">274</span>
+<span class="normal">275</span>
+<span class="normal">276</span>
+<span class="normal">277</span>
+<span class="normal">278</span>
+<span class="normal">279</span>
 <span class="normal">280</span>
 <span class="normal">281</span>
 <span class="normal">282</span>
@@ -9097,172 +13945,600 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_perceiver.TabPerceiver"
 <span class="normal">327</span>
 <span class="normal">328</span>
 <span class="normal">329</span>
-<span class="normal">330</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
-    <span class="bp">self</span><span class="p">,</span>
-    <span class="n">column_idx</span><span class="p">:</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">],</span>
-    <span class="o">*</span><span class="p">,</span>
-    <span class="n">cat_embed_input</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="n">Tuple</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">]]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">cat_embed_dropout</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">use_cat_bias</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">cat_embed_activation</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">shared_embed</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">add_shared_embed</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">frac_shared_embed</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">continuous_cols</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">cont_norm_layer</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Literal</span><span class="p">[</span><span class="s2">&quot;batchnorm&quot;</span><span class="p">,</span> <span class="s2">&quot;layernorm&quot;</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">embed_continuous_method</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span>
-        <span class="n">Literal</span><span class="p">[</span><span class="s2">&quot;standard&quot;</span><span class="p">,</span> <span class="s2">&quot;piecewise&quot;</span><span class="p">,</span> <span class="s2">&quot;periodic&quot;</span><span class="p">]</span>
-    <span class="p">]</span> <span class="o">=</span> <span class="s2">&quot;standard&quot;</span><span class="p">,</span>
-    <span class="n">cont_embed_dropout</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">cont_embed_activation</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">quantization_setup</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="nb">float</span><span class="p">]]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">n_frequencies</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">sigma</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">share_last_layer</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">full_embed_dropout</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">input_dim</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">32</span><span class="p">,</span>
-    <span class="n">n_cross_attns</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1</span><span class="p">,</span>
-    <span class="n">n_cross_attn_heads</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">4</span><span class="p">,</span>
-    <span class="n">n_latents</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">16</span><span class="p">,</span>
-    <span class="n">latent_dim</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">128</span><span class="p">,</span>
-    <span class="n">n_latent_heads</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">4</span><span class="p">,</span>
-    <span class="n">n_latent_blocks</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">4</span><span class="p">,</span>
-    <span class="n">n_perceiver_blocks</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">4</span><span class="p">,</span>
-    <span class="n">share_weights</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
-    <span class="n">attn_dropout</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.1</span><span class="p">,</span>
-    <span class="n">ff_dropout</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.1</span><span class="p">,</span>
-    <span class="n">ff_factor</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">4</span><span class="p">,</span>
-    <span class="n">transformer_activation</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;geglu&quot;</span><span class="p">,</span>
-    <span class="n">mlp_hidden_dims</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">int</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">mlp_activation</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">mlp_dropout</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">mlp_batchnorm</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">mlp_batchnorm_last</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">mlp_linear_first</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-<span class="p">):</span>
-    <span class="nb">super</span><span class="p">(</span><span class="n">TabPerceiver</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span>
-        <span class="n">column_idx</span><span class="o">=</span><span class="n">column_idx</span><span class="p">,</span>
-        <span class="n">cat_embed_input</span><span class="o">=</span><span class="n">cat_embed_input</span><span class="p">,</span>
-        <span class="n">cat_embed_dropout</span><span class="o">=</span><span class="n">cat_embed_dropout</span><span class="p">,</span>
-        <span class="n">use_cat_bias</span><span class="o">=</span><span class="n">use_cat_bias</span><span class="p">,</span>
-        <span class="n">cat_embed_activation</span><span class="o">=</span><span class="n">cat_embed_activation</span><span class="p">,</span>
-        <span class="n">shared_embed</span><span class="o">=</span><span class="n">shared_embed</span><span class="p">,</span>
-        <span class="n">add_shared_embed</span><span class="o">=</span><span class="n">add_shared_embed</span><span class="p">,</span>
-        <span class="n">frac_shared_embed</span><span class="o">=</span><span class="n">frac_shared_embed</span><span class="p">,</span>
-        <span class="n">continuous_cols</span><span class="o">=</span><span class="n">continuous_cols</span><span class="p">,</span>
-        <span class="n">cont_norm_layer</span><span class="o">=</span><span class="n">cont_norm_layer</span><span class="p">,</span>
-        <span class="n">embed_continuous_method</span><span class="o">=</span><span class="n">embed_continuous_method</span><span class="p">,</span>
-        <span class="n">cont_embed_dropout</span><span class="o">=</span><span class="n">cont_embed_dropout</span><span class="p">,</span>
-        <span class="n">cont_embed_activation</span><span class="o">=</span><span class="n">cont_embed_activation</span><span class="p">,</span>
-        <span class="n">input_dim</span><span class="o">=</span><span class="n">input_dim</span><span class="p">,</span>
-        <span class="n">quantization_setup</span><span class="o">=</span><span class="n">quantization_setup</span><span class="p">,</span>
-        <span class="n">n_frequencies</span><span class="o">=</span><span class="n">n_frequencies</span><span class="p">,</span>
-        <span class="n">sigma</span><span class="o">=</span><span class="n">sigma</span><span class="p">,</span>
-        <span class="n">share_last_layer</span><span class="o">=</span><span class="n">share_last_layer</span><span class="p">,</span>
-        <span class="n">full_embed_dropout</span><span class="o">=</span><span class="n">full_embed_dropout</span><span class="p">,</span>
-    <span class="p">)</span>
-
-    <span class="bp">self</span><span class="o">.</span><span class="n">n_cross_attns</span> <span class="o">=</span> <span class="n">n_cross_attns</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">n_cross_attn_heads</span> <span class="o">=</span> <span class="n">n_cross_attn_heads</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">n_latents</span> <span class="o">=</span> <span class="n">n_latents</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">latent_dim</span> <span class="o">=</span> <span class="n">latent_dim</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">n_latent_heads</span> <span class="o">=</span> <span class="n">n_latent_heads</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">n_latent_blocks</span> <span class="o">=</span> <span class="n">n_latent_blocks</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">n_perceiver_blocks</span> <span class="o">=</span> <span class="n">n_perceiver_blocks</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">share_weights</span> <span class="o">=</span> <span class="n">share_weights</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">attn_dropout</span> <span class="o">=</span> <span class="n">attn_dropout</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">ff_dropout</span> <span class="o">=</span> <span class="n">ff_dropout</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">ff_factor</span> <span class="o">=</span> <span class="n">ff_factor</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">transformer_activation</span> <span class="o">=</span> <span class="n">transformer_activation</span>
-
-    <span class="bp">self</span><span class="o">.</span><span class="n">mlp_hidden_dims</span> <span class="o">=</span> <span class="n">mlp_hidden_dims</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">mlp_activation</span> <span class="o">=</span> <span class="n">mlp_activation</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">mlp_dropout</span> <span class="o">=</span> <span class="n">mlp_dropout</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">mlp_batchnorm</span> <span class="o">=</span> <span class="n">mlp_batchnorm</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">mlp_batchnorm_last</span> <span class="o">=</span> <span class="n">mlp_batchnorm_last</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">mlp_linear_first</span> <span class="o">=</span> <span class="n">mlp_linear_first</span>
-
-    <span class="c1"># Embeddings are instantiated at the base model</span>
-    <span class="c1"># Transformer blocks</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">latents</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">init</span><span class="o">.</span><span class="n">trunc_normal_</span><span class="p">(</span>
-        <span class="n">nn</span><span class="o">.</span><span class="n">Parameter</span><span class="p">(</span><span class="n">torch</span><span class="o">.</span><span class="n">empty</span><span class="p">(</span><span class="n">n_latents</span><span class="p">,</span> <span class="n">latent_dim</span><span class="p">))</span>
-    <span class="p">)</span>
-
-    <span class="bp">self</span><span class="o">.</span><span class="n">encoder</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">ModuleDict</span><span class="p">()</span>
-    <span class="n">first_perceiver_block</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_build_perceiver_block</span><span class="p">()</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">encoder</span><span class="p">[</span><span class="s2">&quot;perceiver_block0&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">first_perceiver_block</span>
-
-    <span class="k">if</span> <span class="n">share_weights</span><span class="p">:</span>
-        <span class="k">for</span> <span class="n">n</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="n">n_perceiver_blocks</span><span class="p">):</span>
-            <span class="bp">self</span><span class="o">.</span><span class="n">encoder</span><span class="p">[</span><span class="s2">&quot;perceiver_block&quot;</span> <span class="o">+</span> <span class="nb">str</span><span class="p">(</span><span class="n">n</span><span class="p">)]</span> <span class="o">=</span> <span class="n">first_perceiver_block</span>
-    <span class="k">else</span><span class="p">:</span>
-        <span class="k">for</span> <span class="n">n</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="n">n_perceiver_blocks</span><span class="p">):</span>
-            <span class="bp">self</span><span class="o">.</span><span class="n">encoder</span><span class="p">[</span><span class="s2">&quot;perceiver_block&quot;</span> <span class="o">+</span> <span class="nb">str</span><span class="p">(</span><span class="n">n</span><span class="p">)]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_build_perceiver_block</span><span class="p">()</span>
-
-    <span class="bp">self</span><span class="o">.</span><span class="n">mlp_first_hidden_dim</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">latent_dim</span>
-
-    <span class="c1"># Mlp</span>
-    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_hidden_dims</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">mlp</span> <span class="o">=</span> <span class="n">MLP</span><span class="p">(</span>
-            <span class="n">d_hidden</span><span class="o">=</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">mlp_first_hidden_dim</span><span class="p">]</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_hidden_dims</span><span class="p">,</span>
-            <span class="n">activation</span><span class="o">=</span><span class="p">(</span>
-                <span class="s2">&quot;relu&quot;</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_activation</span> <span class="ow">is</span> <span class="kc">None</span> <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_activation</span>
-            <span class="p">),</span>
-            <span class="n">dropout</span><span class="o">=</span><span class="mf">0.0</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_dropout</span> <span class="ow">is</span> <span class="kc">None</span> <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_dropout</span><span class="p">,</span>
-            <span class="n">batchnorm</span><span class="o">=</span><span class="kc">False</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_batchnorm</span> <span class="ow">is</span> <span class="kc">None</span> <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_batchnorm</span><span class="p">,</span>
-            <span class="n">batchnorm_last</span><span class="o">=</span><span class="p">(</span>
-                <span class="kc">False</span>
-                <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_batchnorm_last</span> <span class="ow">is</span> <span class="kc">None</span>
-                <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_batchnorm_last</span>
-            <span class="p">),</span>
-            <span class="n">linear_first</span><span class="o">=</span><span class="p">(</span>
-                <span class="kc">False</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_linear_first</span> <span class="ow">is</span> <span class="kc">None</span> <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_linear_first</span>
-            <span class="p">),</span>
-        <span class="p">)</span>
-    <span class="k">else</span><span class="p">:</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">mlp</span> <span class="o">=</span> <span class="kc">None</span>
-</code></pre></div></td></tr></table></div>
-                  </details>
-
-
-
-  <div class="doc doc-children">
-
-
-
-
-
-
-
-<div class="doc doc-object doc-attribute">
-
-
-
-<h3 id="pytorch_widedeep.models.tabular.transformers.tab_perceiver.TabPerceiver.output_dim" class="doc doc-heading">
-            <span class="doc doc-object-name doc-attribute-name">output_dim</span>
-
-
-  <span class="doc doc-labels">
-      <small class="doc doc-label doc-label-property"><code>property</code></small>
-  </span>
-
-<a href="#pytorch_widedeep.models.tabular.transformers.tab_perceiver.TabPerceiver.output_dim" class="headerlink" title="Permanent link">&para;</a></h3>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">output_dim</span>
-</code></pre></div>
-
-    <div class="doc doc-contents ">
-
-        <p>The output dimension of the model. This is a required property
-neccesary to build the <code>WideDeep</code> class</p>
-    </div>
-
-</div>
-
-<div class="doc doc-object doc-attribute">
-
-
-
-<h3 id="pytorch_widedeep.models.tabular.transformers.tab_perceiver.TabPerceiver.attention_weights" class="doc doc-heading">
+<span class="normal">330</span>
+<span class="normal">331</span>
+<span class="normal">332</span>
+<span class="normal">333</span>
+<span class="normal">334</span>
+<span class="normal">335</span>
+<span class="normal">336</span>
+<span class="normal">337</span>
+<span class="normal">338</span>
+<span class="normal">339</span>
+<span class="normal">340</span>
+<span class="normal">341</span>
+<span class="normal">342</span>
+<span class="normal">343</span>
+<span class="normal">344</span>
+<span class="normal">345</span>
+<span class="normal">346</span>
+<span class="normal">347</span>
+<span class="normal">348</span>
+<span class="normal">349</span>
+<span class="normal">350</span>
+<span class="normal">351</span>
+<span class="normal">352</span>
+<span class="normal">353</span>
+<span class="normal">354</span>
+<span class="normal">355</span>
+<span class="normal">356</span>
+<span class="normal">357</span>
+<span class="normal">358</span>
+<span class="normal">359</span>
+<span class="normal">360</span>
+<span class="normal">361</span>
+<span class="normal">362</span>
+<span class="normal">363</span>
+<span class="normal">364</span>
+<span class="normal">365</span>
+<span class="normal">366</span>
+<span class="normal">367</span>
+<span class="normal">368</span>
+<span class="normal">369</span>
+<span class="normal">370</span>
+<span class="normal">371</span>
+<span class="normal">372</span>
+<span class="normal">373</span>
+<span class="normal">374</span>
+<span class="normal">375</span>
+<span class="normal">376</span>
+<span class="normal">377</span>
+<span class="normal">378</span>
+<span class="normal">379</span>
+<span class="normal">380</span>
+<span class="normal">381</span>
+<span class="normal">382</span>
+<span class="normal">383</span>
+<span class="normal">384</span>
+<span class="normal">385</span>
+<span class="normal">386</span>
+<span class="normal">387</span>
+<span class="normal">388</span>
+<span class="normal">389</span>
+<span class="normal">390</span>
+<span class="normal">391</span>
+<span class="normal">392</span>
+<span class="normal">393</span>
+<span class="normal">394</span>
+<span class="normal">395</span>
+<span class="normal">396</span>
+<span class="normal">397</span>
+<span class="normal">398</span>
+<span class="normal">399</span>
+<span class="normal">400</span>
+<span class="normal">401</span>
+<span class="normal">402</span>
+<span class="normal">403</span>
+<span class="normal">404</span>
+<span class="normal">405</span>
+<span class="normal">406</span>
+<span class="normal">407</span>
+<span class="normal">408</span>
+<span class="normal">409</span>
+<span class="normal">410</span>
+<span class="normal">411</span>
+<span class="normal">412</span>
+<span class="normal">413</span>
+<span class="normal">414</span>
+<span class="normal">415</span>
+<span class="normal">416</span>
+<span class="normal">417</span>
+<span class="normal">418</span>
+<span class="normal">419</span>
+<span class="normal">420</span>
+<span class="normal">421</span>
+<span class="normal">422</span>
+<span class="normal">423</span>
+<span class="normal">424</span>
+<span class="normal">425</span>
+<span class="normal">426</span>
+<span class="normal">427</span>
+<span class="normal">428</span>
+<span class="normal">429</span>
+<span class="normal">430</span>
+<span class="normal">431</span>
+<span class="normal">432</span>
+<span class="normal">433</span>
+<span class="normal">434</span>
+<span class="normal">435</span>
+<span class="normal">436</span>
+<span class="normal">437</span>
+<span class="normal">438</span>
+<span class="normal">439</span>
+<span class="normal">440</span>
+<span class="normal">441</span>
+<span class="normal">442</span>
+<span class="normal">443</span>
+<span class="normal">444</span>
+<span class="normal">445</span>
+<span class="normal">446</span>
+<span class="normal">447</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">class</span> <span class="nc">TabPerceiver</span><span class="p">(</span><span class="n">BaseTabularModelWithAttention</span><span class="p">):</span>
+<span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Defines an adaptation of a [Perceiver](https://arxiv.org/abs/2103.03206)</span>
+<span class="sd">     that can be used as the `deeptabular` component of a Wide &amp; Deep model</span>
+<span class="sd">     or independently by itself.</span>
+
+<span class="sd">    Most of the parameters for this class are `Optional` since the use of</span>
+<span class="sd">    categorical or continuous is in fact optional (i.e. one can use</span>
+<span class="sd">    categorical features only, continuous features only or both).</span>
+
+<span class="sd">    :information_source: **NOTE**: while there are scientific publications for</span>
+<span class="sd">     the `TabTransformer`, `SAINT` and `FTTransformer`, the `TabPerceiver`</span>
+<span class="sd">     and the `TabFastFormer` are our own adaptations of the</span>
+<span class="sd">     [Perceiver](https://arxiv.org/abs/2103.03206) and the</span>
+<span class="sd">     [FastFormer](https://arxiv.org/abs/2108.09084) for tabular data.</span>
+
+<span class="sd">    Parameters</span>
+<span class="sd">    ----------</span>
+<span class="sd">    column_idx: Dict</span>
+<span class="sd">        Dict containing the index of the columns that will be passed through</span>
+<span class="sd">        the `TabMlp` model. Required to slice the tensors. e.g. _{&#39;education&#39;:</span>
+<span class="sd">        0, &#39;relationship&#39;: 1, &#39;workclass&#39;: 2, ...}_.</span>
+<span class="sd">    cat_embed_input: List, Optional, default = None</span>
+<span class="sd">        List of Tuples with the column name and number of unique values and</span>
+<span class="sd">        embedding dimension. e.g. _[(education, 11), ...]_</span>
+<span class="sd">    cat_embed_dropout: float, Optional, default = None</span>
+<span class="sd">        Categorical embeddings dropout. If `None`, it will default</span>
+<span class="sd">        to 0.</span>
+<span class="sd">    use_cat_bias: bool, Optional, default = None,</span>
+<span class="sd">        Boolean indicating if bias will be used for the categorical embeddings.</span>
+<span class="sd">        If `None`, it will default to &#39;False&#39;.</span>
+<span class="sd">    cat_embed_activation: Optional, str, default = None,</span>
+<span class="sd">        Activation function for the categorical embeddings, if any. Currently</span>
+<span class="sd">        _&#39;tanh&#39;_, _&#39;relu&#39;_, _&#39;leaky_relu&#39;_ and _&#39;gelu&#39;_ are supported</span>
+<span class="sd">    shared_embed: bool, Optional, default = None</span>
+<span class="sd">        Boolean indicating if the embeddings will be &quot;shared&quot;. The idea behind `shared_embed` is</span>
+<span class="sd">        described in the Appendix A in the [TabTransformer paper](https://arxiv.org/abs/2012.06678):</span>
+<span class="sd">        _&#39;The goal of having column embedding is to enable the model to</span>
+<span class="sd">        distinguish the classes in one column from those in the other</span>
+<span class="sd">        columns&#39;_. In other words, the idea is to let the model learn which</span>
+<span class="sd">        column is embedded at the time. See: `pytorch_widedeep.models.transformers._layers.SharedEmbeddings`.</span>
+<span class="sd">    add_shared_embed: bool, Optional, default = None</span>
+<span class="sd">        The two embedding sharing strategies are: 1) add the shared embeddings</span>
+<span class="sd">        to the column embeddings or 2) to replace the first</span>
+<span class="sd">        `frac_shared_embed` with the shared embeddings.</span>
+<span class="sd">        See `pytorch_widedeep.models.embeddings_layers.SharedEmbeddings`</span>
+<span class="sd">        If &#39;None&#39; is passed, it will default to &#39;False&#39;.</span>
+<span class="sd">    frac_shared_embed: float, Optional, default = None</span>
+<span class="sd">        The fraction of embeddings that will be shared (if `add_shared_embed</span>
+<span class="sd">        = False`) by all the different categories for one particular</span>
+<span class="sd">        column. If &#39;None&#39; is passed, it will default to 0.0.</span>
+<span class="sd">    continuous_cols: List, Optional, default = None</span>
+<span class="sd">        List with the name of the numeric (aka continuous) columns</span>
+<span class="sd">    cont_norm_layer: str, Optional, default =  None</span>
+<span class="sd">        Type of normalization layer applied to the continuous features.</span>
+<span class="sd">        Options are: _&#39;layernorm&#39;_ and _&#39;batchnorm&#39;_. if `None`, no</span>
+<span class="sd">        normalization layer will be used.</span>
+<span class="sd">    embed_continuous_method: Optional, str, default = None,</span>
+<span class="sd">        Method to use to embed the continuous features. Options are:</span>
+<span class="sd">        _&#39;standard&#39;_, _&#39;periodic&#39;_ or _&#39;piecewise&#39;_. The _&#39;standard&#39;_</span>
+<span class="sd">        embedding method is based on the FT-Transformer implementation</span>
+<span class="sd">        presented in the paper: [Revisiting Deep Learning Models for</span>
+<span class="sd">        Tabular Data](https://arxiv.org/abs/2106.11959v5). The _&#39;periodic&#39;_</span>
+<span class="sd">        and_&#39;piecewise&#39;_ methods were presented in the paper: [On Embeddings for</span>
+<span class="sd">        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556).</span>
+<span class="sd">        Please, read the papers for details.</span>
+<span class="sd">    cont_embed_dropout: float, Optional, default = None,</span>
+<span class="sd">        Dropout for the continuous embeddings. If `None`, it will default to 0.0</span>
+<span class="sd">    cont_embed_activation: Optional, str, default = None,</span>
+<span class="sd">        Activation function for the continuous embeddings if any. Currently</span>
+<span class="sd">        _&#39;tanh&#39;_, _&#39;relu&#39;_, _&#39;leaky_relu&#39;_ and _&#39;gelu&#39;_ are supported.</span>
+<span class="sd">        If `None`, no activation function will be applied.</span>
+<span class="sd">    quantization_setup: Dict[str, List[float]], Optional, default = None,</span>
+<span class="sd">        This parameter is used when the _&#39;piecewise&#39;_ method is used to embed</span>
+<span class="sd">        the continuous cols. It is a dict where keys are the name of the continuous</span>
+<span class="sd">        columns and values are lists with the boundaries for the quantization</span>
+<span class="sd">        of the continuous_cols. See the examples for details. If</span>
+<span class="sd">        If the _&#39;piecewise&#39;_ method is used, this parameter is required.</span>
+<span class="sd">    n_frequencies: int, Optional, default = None,</span>
+<span class="sd">        This is the so called _&#39;k&#39;_ in their paper [On Embeddings for</span>
+<span class="sd">        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556),</span>
+<span class="sd">        and is the number of &#39;frequencies&#39; that will be used to represent each</span>
+<span class="sd">        continuous column. See their Eq 2 in the paper for details. If</span>
+<span class="sd">        the _&#39;periodic&#39;_ method is used, this parameter is required.</span>
+<span class="sd">    sigma: float, Optional, default = None,</span>
+<span class="sd">        This is the sigma parameter in the paper mentioned when describing the</span>
+<span class="sd">        previous parameters and it is used to initialise the &#39;frequency</span>
+<span class="sd">        weights&#39;. See their Eq 2 in the paper for details. If</span>
+<span class="sd">        the _&#39;periodic&#39;_ method is used, this parameter is required.</span>
+<span class="sd">    share_last_layer: bool, Optional, default = None,</span>
+<span class="sd">        This parameter is not present in the before mentioned paper but it is implemented in</span>
+<span class="sd">        the [official repo](https://github.com/yandex-research/rtdl-num-embeddings/tree/main).</span>
+<span class="sd">        If `True` the linear layer that turns the frequencies into embeddings</span>
+<span class="sd">        will be shared across the continuous columns. If `False` a different</span>
+<span class="sd">        linear layer will be used for each continuous column.</span>
+<span class="sd">        If the _&#39;periodic&#39;_ method is used, this parameter is required.</span>
+<span class="sd">    full_embed_dropout: bool, Optional, default = None,</span>
+<span class="sd">        If `True`, the full embedding corresponding to a column will be masked</span>
+<span class="sd">        out/dropout. If `None`, it will default to `False`.</span>
+<span class="sd">    input_dim: int, default = 32</span>
+<span class="sd">        The so-called *dimension of the model*. Is the number of embeddings</span>
+<span class="sd">        used to encode the categorical and/or continuous columns.</span>
+<span class="sd">    n_cross_attns: int, default = 1</span>
+<span class="sd">        Number of times each perceiver block will cross attend to the input</span>
+<span class="sd">        data (i.e. number of cross attention components per perceiver block).</span>
+<span class="sd">        This should normally be 1. However, in the paper they describe some</span>
+<span class="sd">        architectures (normally computer vision-related problems) where the</span>
+<span class="sd">        Perceiver attends multiple times to the input array. Therefore, maybe</span>
+<span class="sd">        multiple cross attention to the input array is also useful in some</span>
+<span class="sd">        cases for tabular data :shrug: .</span>
+<span class="sd">    n_cross_attn_heads: int, default = 4</span>
+<span class="sd">        Number of attention heads for the cross attention component</span>
+<span class="sd">    n_latents: int, default = 16</span>
+<span class="sd">        Number of latents. This is the $N$ parameter in the paper. As</span>
+<span class="sd">        indicated in the paper, this number should be significantly lower</span>
+<span class="sd">        than $M$ (the number of columns in the dataset). Setting $N$ closer</span>
+<span class="sd">        to $M$ defies the main purpose of the Perceiver, which is to overcome</span>
+<span class="sd">        the transformer quadratic bottleneck</span>
+<span class="sd">    latent_dim: int, default = 128</span>
+<span class="sd">        Latent dimension.</span>
+<span class="sd">    n_latent_heads: int, default = 4</span>
+<span class="sd">        Number of attention heads per Latent Transformer</span>
+<span class="sd">    n_latent_blocks: int, default = 4</span>
+<span class="sd">        Number of transformer encoder blocks (normalised MHA + normalised FF)</span>
+<span class="sd">        per Latent Transformer</span>
+<span class="sd">    n_perceiver_blocks: int, default = 4</span>
+<span class="sd">        Number of Perceiver blocks defined as [Cross Attention + Latent</span>
+<span class="sd">        Transformer]</span>
+<span class="sd">    share_weights: Boolean, default = False</span>
+<span class="sd">        Boolean indicating if the weights will be shared between Perceiver</span>
+<span class="sd">        blocks</span>
+<span class="sd">    attn_dropout: float, default = 0.2</span>
+<span class="sd">        Dropout that will be applied to the Multi-Head Attention layers</span>
+<span class="sd">    ff_dropout: float, default = 0.1</span>
+<span class="sd">        Dropout that will be applied to the FeedForward network</span>
+<span class="sd">    ff_factor: float, default = 4</span>
+<span class="sd">        Multiplicative factor applied to the first layer of the FF network in</span>
+<span class="sd">        each Transformer block, This is normally set to 4.</span>
+<span class="sd">    transformer_activation: str, default = &quot;gelu&quot;</span>
+<span class="sd">        Transformer Encoder activation function. _&#39;tanh&#39;_, _&#39;relu&#39;_,</span>
+<span class="sd">        _&#39;leaky_relu&#39;_, _&#39;gelu&#39;_, _&#39;geglu&#39;_ and _&#39;reglu&#39;_ are supported</span>
+<span class="sd">    mlp_hidden_dims: List, Optional, default = None</span>
+<span class="sd">        List with the number of neurons per dense layer in the MLP. e.g:</span>
+<span class="sd">        _[64, 32]_. If not provided no MLP on top of the final</span>
+<span class="sd">        Transformer block will be used.</span>
+<span class="sd">    mlp_activation: str, Optional, default = None</span>
+<span class="sd">        Activation function for the dense layers of the MLP. Currently</span>
+<span class="sd">        _&#39;tanh&#39;_, _&#39;relu&#39;_, _&#39;leaky&#39;_relu&#39; and _&#39;gelu&#39;_ are supported.</span>
+<span class="sd">        If &#39;mlp_hidden_dims&#39; is not `None` and this parameter is `None`, it</span>
+<span class="sd">        will default to _&#39;relu&#39;_.</span>
+<span class="sd">    mlp_dropout: float, Optional, default = None</span>
+<span class="sd">        float with the dropout between the dense layers of the MLP.</span>
+<span class="sd">        If &#39;mlp_hidden_dims&#39; is not `None` and this parameter is `None`, it</span>
+<span class="sd">        will default to 0.0.</span>
+<span class="sd">    mlp_batchnorm: bool, Optional, default = None</span>
+<span class="sd">        Boolean indicating whether or not batch normalization will be applied</span>
+<span class="sd">        to the dense layers</span>
+<span class="sd">        If &#39;mlp_hidden_dims&#39; is not `None` and this parameter is `None`, it</span>
+<span class="sd">        will default to False.</span>
+<span class="sd">    mlp_batchnorm_last: bool, Optional, default = None</span>
+<span class="sd">        Boolean indicating whether or not batch normalization will be applied</span>
+<span class="sd">        to the last of the dense layers</span>
+<span class="sd">        If &#39;mlp_hidden_dims&#39; is not `None` and this parameter is `None`, it</span>
+<span class="sd">        will default to False.</span>
+<span class="sd">    mlp_linear_first: bool, Optional, default = None</span>
+<span class="sd">        Boolean indicating the order of the operations in the dense</span>
+<span class="sd">        layer. If `True: [LIN -&gt; ACT -&gt; BN -&gt; DP]`. If `False: [BN -&gt; DP -&gt;</span>
+<span class="sd">        LIN -&gt; ACT]`</span>
+<span class="sd">        If &#39;mlp_hidden_dims&#39; is not `None` and this parameter is `None`, it</span>
+<span class="sd">        will default to `True`.</span>
+
+<span class="sd">    Attributes</span>
+<span class="sd">    ----------</span>
+<span class="sd">    encoder: nn.ModuleDict</span>
+<span class="sd">        ModuleDict with the Perceiver blocks</span>
+<span class="sd">    latents: nn.Parameter</span>
+<span class="sd">        Latents that will be used for prediction</span>
+<span class="sd">    mlp: nn.Module</span>
+<span class="sd">        MLP component in the model</span>
+
+<span class="sd">    Examples</span>
+<span class="sd">    --------</span>
+<span class="sd">    &gt;&gt;&gt; import torch</span>
+<span class="sd">    &gt;&gt;&gt; from pytorch_widedeep.models import TabPerceiver</span>
+<span class="sd">    &gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)</span>
+<span class="sd">    &gt;&gt;&gt; colnames = [&#39;a&#39;, &#39;b&#39;, &#39;c&#39;, &#39;d&#39;, &#39;e&#39;]</span>
+<span class="sd">    &gt;&gt;&gt; cat_embed_input = [(u,i) for u,i in zip(colnames[:4], [4]*4)]</span>
+<span class="sd">    &gt;&gt;&gt; continuous_cols = [&#39;e&#39;]</span>
+<span class="sd">    &gt;&gt;&gt; column_idx = {k:v for v,k in enumerate(colnames)}</span>
+<span class="sd">    &gt;&gt;&gt; model = TabPerceiver(column_idx=column_idx, cat_embed_input=cat_embed_input,</span>
+<span class="sd">    ... continuous_cols=continuous_cols, n_latents=2, latent_dim=16,</span>
+<span class="sd">    ... n_perceiver_blocks=2)</span>
+<span class="sd">    &gt;&gt;&gt; out = model(X_tab)</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+
+    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span>
+        <span class="n">column_idx</span><span class="p">:</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">],</span>
+        <span class="o">*</span><span class="p">,</span>
+        <span class="n">cat_embed_input</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="n">Tuple</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">]]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">cat_embed_dropout</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">use_cat_bias</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">cat_embed_activation</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">shared_embed</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">add_shared_embed</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">frac_shared_embed</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">continuous_cols</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">cont_norm_layer</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Literal</span><span class="p">[</span><span class="s2">&quot;batchnorm&quot;</span><span class="p">,</span> <span class="s2">&quot;layernorm&quot;</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">embed_continuous_method</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span>
+            <span class="n">Literal</span><span class="p">[</span><span class="s2">&quot;standard&quot;</span><span class="p">,</span> <span class="s2">&quot;piecewise&quot;</span><span class="p">,</span> <span class="s2">&quot;periodic&quot;</span><span class="p">]</span>
+        <span class="p">]</span> <span class="o">=</span> <span class="s2">&quot;standard&quot;</span><span class="p">,</span>
+        <span class="n">cont_embed_dropout</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">cont_embed_activation</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">quantization_setup</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="nb">float</span><span class="p">]]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">n_frequencies</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">sigma</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">share_last_layer</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">full_embed_dropout</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">input_dim</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">32</span><span class="p">,</span>
+        <span class="n">n_cross_attns</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1</span><span class="p">,</span>
+        <span class="n">n_cross_attn_heads</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">4</span><span class="p">,</span>
+        <span class="n">n_latents</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">16</span><span class="p">,</span>
+        <span class="n">latent_dim</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">128</span><span class="p">,</span>
+        <span class="n">n_latent_heads</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">4</span><span class="p">,</span>
+        <span class="n">n_latent_blocks</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">4</span><span class="p">,</span>
+        <span class="n">n_perceiver_blocks</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">4</span><span class="p">,</span>
+        <span class="n">share_weights</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
+        <span class="n">attn_dropout</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.1</span><span class="p">,</span>
+        <span class="n">ff_dropout</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.1</span><span class="p">,</span>
+        <span class="n">ff_factor</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">4</span><span class="p">,</span>
+        <span class="n">transformer_activation</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;geglu&quot;</span><span class="p">,</span>
+        <span class="n">mlp_hidden_dims</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">int</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">mlp_activation</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">mlp_dropout</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">mlp_batchnorm</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">mlp_batchnorm_last</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">mlp_linear_first</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+    <span class="p">):</span>
+        <span class="nb">super</span><span class="p">(</span><span class="n">TabPerceiver</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span>
+            <span class="n">column_idx</span><span class="o">=</span><span class="n">column_idx</span><span class="p">,</span>
+            <span class="n">cat_embed_input</span><span class="o">=</span><span class="n">cat_embed_input</span><span class="p">,</span>
+            <span class="n">cat_embed_dropout</span><span class="o">=</span><span class="n">cat_embed_dropout</span><span class="p">,</span>
+            <span class="n">use_cat_bias</span><span class="o">=</span><span class="n">use_cat_bias</span><span class="p">,</span>
+            <span class="n">cat_embed_activation</span><span class="o">=</span><span class="n">cat_embed_activation</span><span class="p">,</span>
+            <span class="n">shared_embed</span><span class="o">=</span><span class="n">shared_embed</span><span class="p">,</span>
+            <span class="n">add_shared_embed</span><span class="o">=</span><span class="n">add_shared_embed</span><span class="p">,</span>
+            <span class="n">frac_shared_embed</span><span class="o">=</span><span class="n">frac_shared_embed</span><span class="p">,</span>
+            <span class="n">continuous_cols</span><span class="o">=</span><span class="n">continuous_cols</span><span class="p">,</span>
+            <span class="n">cont_norm_layer</span><span class="o">=</span><span class="n">cont_norm_layer</span><span class="p">,</span>
+            <span class="n">embed_continuous_method</span><span class="o">=</span><span class="n">embed_continuous_method</span><span class="p">,</span>
+            <span class="n">cont_embed_dropout</span><span class="o">=</span><span class="n">cont_embed_dropout</span><span class="p">,</span>
+            <span class="n">cont_embed_activation</span><span class="o">=</span><span class="n">cont_embed_activation</span><span class="p">,</span>
+            <span class="n">input_dim</span><span class="o">=</span><span class="n">input_dim</span><span class="p">,</span>
+            <span class="n">quantization_setup</span><span class="o">=</span><span class="n">quantization_setup</span><span class="p">,</span>
+            <span class="n">n_frequencies</span><span class="o">=</span><span class="n">n_frequencies</span><span class="p">,</span>
+            <span class="n">sigma</span><span class="o">=</span><span class="n">sigma</span><span class="p">,</span>
+            <span class="n">share_last_layer</span><span class="o">=</span><span class="n">share_last_layer</span><span class="p">,</span>
+            <span class="n">full_embed_dropout</span><span class="o">=</span><span class="n">full_embed_dropout</span><span class="p">,</span>
+        <span class="p">)</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">n_cross_attns</span> <span class="o">=</span> <span class="n">n_cross_attns</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">n_cross_attn_heads</span> <span class="o">=</span> <span class="n">n_cross_attn_heads</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">n_latents</span> <span class="o">=</span> <span class="n">n_latents</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">latent_dim</span> <span class="o">=</span> <span class="n">latent_dim</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">n_latent_heads</span> <span class="o">=</span> <span class="n">n_latent_heads</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">n_latent_blocks</span> <span class="o">=</span> <span class="n">n_latent_blocks</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">n_perceiver_blocks</span> <span class="o">=</span> <span class="n">n_perceiver_blocks</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">share_weights</span> <span class="o">=</span> <span class="n">share_weights</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">attn_dropout</span> <span class="o">=</span> <span class="n">attn_dropout</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">ff_dropout</span> <span class="o">=</span> <span class="n">ff_dropout</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">ff_factor</span> <span class="o">=</span> <span class="n">ff_factor</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">transformer_activation</span> <span class="o">=</span> <span class="n">transformer_activation</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">mlp_hidden_dims</span> <span class="o">=</span> <span class="n">mlp_hidden_dims</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">mlp_activation</span> <span class="o">=</span> <span class="n">mlp_activation</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">mlp_dropout</span> <span class="o">=</span> <span class="n">mlp_dropout</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">mlp_batchnorm</span> <span class="o">=</span> <span class="n">mlp_batchnorm</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">mlp_batchnorm_last</span> <span class="o">=</span> <span class="n">mlp_batchnorm_last</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">mlp_linear_first</span> <span class="o">=</span> <span class="n">mlp_linear_first</span>
+
+        <span class="c1"># Embeddings are instantiated at the base model</span>
+        <span class="c1"># Transformer blocks</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">latents</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">init</span><span class="o">.</span><span class="n">trunc_normal_</span><span class="p">(</span>
+            <span class="n">nn</span><span class="o">.</span><span class="n">Parameter</span><span class="p">(</span><span class="n">torch</span><span class="o">.</span><span class="n">empty</span><span class="p">(</span><span class="n">n_latents</span><span class="p">,</span> <span class="n">latent_dim</span><span class="p">))</span>
+        <span class="p">)</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">encoder</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">ModuleDict</span><span class="p">()</span>
+        <span class="n">first_perceiver_block</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_build_perceiver_block</span><span class="p">()</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">encoder</span><span class="p">[</span><span class="s2">&quot;perceiver_block0&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">first_perceiver_block</span>
+
+        <span class="k">if</span> <span class="n">share_weights</span><span class="p">:</span>
+            <span class="k">for</span> <span class="n">n</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="n">n_perceiver_blocks</span><span class="p">):</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">encoder</span><span class="p">[</span><span class="s2">&quot;perceiver_block&quot;</span> <span class="o">+</span> <span class="nb">str</span><span class="p">(</span><span class="n">n</span><span class="p">)]</span> <span class="o">=</span> <span class="n">first_perceiver_block</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="k">for</span> <span class="n">n</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="n">n_perceiver_blocks</span><span class="p">):</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">encoder</span><span class="p">[</span><span class="s2">&quot;perceiver_block&quot;</span> <span class="o">+</span> <span class="nb">str</span><span class="p">(</span><span class="n">n</span><span class="p">)]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_build_perceiver_block</span><span class="p">()</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">mlp_first_hidden_dim</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">latent_dim</span>
+
+        <span class="c1"># Mlp</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_hidden_dims</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">mlp</span> <span class="o">=</span> <span class="n">MLP</span><span class="p">(</span>
+                <span class="n">d_hidden</span><span class="o">=</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">mlp_first_hidden_dim</span><span class="p">]</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_hidden_dims</span><span class="p">,</span>
+                <span class="n">activation</span><span class="o">=</span><span class="p">(</span>
+                    <span class="s2">&quot;relu&quot;</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_activation</span> <span class="ow">is</span> <span class="kc">None</span> <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_activation</span>
+                <span class="p">),</span>
+                <span class="n">dropout</span><span class="o">=</span><span class="mf">0.0</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_dropout</span> <span class="ow">is</span> <span class="kc">None</span> <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_dropout</span><span class="p">,</span>
+                <span class="n">batchnorm</span><span class="o">=</span><span class="kc">False</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_batchnorm</span> <span class="ow">is</span> <span class="kc">None</span> <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_batchnorm</span><span class="p">,</span>
+                <span class="n">batchnorm_last</span><span class="o">=</span><span class="p">(</span>
+                    <span class="kc">False</span>
+                    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_batchnorm_last</span> <span class="ow">is</span> <span class="kc">None</span>
+                    <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_batchnorm_last</span>
+                <span class="p">),</span>
+                <span class="n">linear_first</span><span class="o">=</span><span class="p">(</span>
+                    <span class="kc">False</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_linear_first</span> <span class="ow">is</span> <span class="kc">None</span> <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_linear_first</span>
+                <span class="p">),</span>
+            <span class="p">)</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">mlp</span> <span class="o">=</span> <span class="kc">None</span>
+
+    <span class="k">def</span> <span class="nf">forward</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">X</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tensor</span><span class="p">:</span>
+        <span class="n">x_emb</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_embeddings</span><span class="p">(</span><span class="n">X</span><span class="p">)</span>
+
+        <span class="n">x</span> <span class="o">=</span> <span class="n">einops</span><span class="o">.</span><span class="n">repeat</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">latents</span><span class="p">,</span> <span class="s2">&quot;n d -&gt; b n d&quot;</span><span class="p">,</span> <span class="n">b</span><span class="o">=</span><span class="n">X</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">0</span><span class="p">])</span>
+
+        <span class="k">for</span> <span class="n">n</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">n_perceiver_blocks</span><span class="p">):</span>
+            <span class="n">cross_attns</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">encoder</span><span class="p">[</span><span class="s2">&quot;perceiver_block&quot;</span> <span class="o">+</span> <span class="nb">str</span><span class="p">(</span><span class="n">n</span><span class="p">)][</span><span class="s2">&quot;cross_attns&quot;</span><span class="p">]</span>
+            <span class="n">latent_transformer</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">encoder</span><span class="p">[</span><span class="s2">&quot;perceiver_block&quot;</span> <span class="o">+</span> <span class="nb">str</span><span class="p">(</span><span class="n">n</span><span class="p">)][</span>
+                <span class="s2">&quot;latent_transformer&quot;</span>
+            <span class="p">]</span>
+            <span class="k">for</span> <span class="n">cross_attn</span> <span class="ow">in</span> <span class="n">cross_attns</span><span class="p">:</span>
+                <span class="n">x</span> <span class="o">=</span> <span class="n">cross_attn</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">x_emb</span><span class="p">)</span>
+            <span class="n">x</span> <span class="o">=</span> <span class="n">latent_transformer</span><span class="p">(</span><span class="n">x</span><span class="p">)</span>
+
+        <span class="c1"># average along the latent index axis</span>
+        <span class="n">x</span> <span class="o">=</span> <span class="n">x</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="n">dim</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span>
+
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="n">x</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp</span><span class="p">(</span><span class="n">x</span><span class="p">)</span>
+
+        <span class="k">return</span> <span class="n">x</span>
+
+    <span class="nd">@property</span>
+    <span class="k">def</span> <span class="nf">output_dim</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">int</span><span class="p">:</span>
+<span class="w">        </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;The output dimension of the model. This is a required property</span>
+<span class="sd">        neccesary to build the `WideDeep` class</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="k">return</span> <span class="p">(</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">mlp_hidden_dims</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span>
+            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_hidden_dims</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span>
+            <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_first_hidden_dim</span>
+        <span class="p">)</span>
+
+    <span class="nd">@property</span>
+    <span class="k">def</span> <span class="nf">attention_weights</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">List</span><span class="p">:</span>
+<span class="w">        </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;List with the attention weights. If the weights are not shared</span>
+<span class="sd">        between perceiver blocks each element of the list will be a list</span>
+<span class="sd">        itself containing the Cross Attention and Latent Transformer</span>
+<span class="sd">        attention weights respectively</span>
+
+<span class="sd">        The shape of the attention weights is:</span>
+
+<span class="sd">        - Cross Attention: $(N, C, L, F)$</span>
+
+<span class="sd">        - Latent Attention: $(N, T, L, L)$</span>
+
+<span class="sd">        WHere $N$ is the batch size, $C$ is the number of Cross Attention</span>
+<span class="sd">        heads, $L$ is the number of Latents, $F$ is the number of</span>
+<span class="sd">        features/columns in the dataset and $T$ is the number of Latent</span>
+<span class="sd">        Attention heads</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">share_weights</span><span class="p">:</span>
+            <span class="n">cross_attns</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">encoder</span><span class="p">[</span><span class="s2">&quot;perceiver_block0&quot;</span><span class="p">][</span><span class="s2">&quot;cross_attns&quot;</span><span class="p">]</span>
+            <span class="n">latent_transformer</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">encoder</span><span class="p">[</span><span class="s2">&quot;perceiver_block0&quot;</span><span class="p">][</span><span class="s2">&quot;latent_transformer&quot;</span><span class="p">]</span>
+            <span class="n">attention_weights</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_extract_attn_weights</span><span class="p">(</span>
+                <span class="n">cross_attns</span><span class="p">,</span> <span class="n">latent_transformer</span>
+            <span class="p">)</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="n">attention_weights</span> <span class="o">=</span> <span class="p">[]</span>
+            <span class="k">for</span> <span class="n">n</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">n_perceiver_blocks</span><span class="p">):</span>
+                <span class="n">cross_attns</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">encoder</span><span class="p">[</span><span class="s2">&quot;perceiver_block&quot;</span> <span class="o">+</span> <span class="nb">str</span><span class="p">(</span><span class="n">n</span><span class="p">)][</span><span class="s2">&quot;cross_attns&quot;</span><span class="p">]</span>
+                <span class="n">latent_transformer</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">encoder</span><span class="p">[</span><span class="s2">&quot;perceiver_block&quot;</span> <span class="o">+</span> <span class="nb">str</span><span class="p">(</span><span class="n">n</span><span class="p">)][</span>
+                    <span class="s2">&quot;latent_transformer&quot;</span>
+                <span class="p">]</span>
+                <span class="n">attention_weights</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
+                    <span class="bp">self</span><span class="o">.</span><span class="n">_extract_attn_weights</span><span class="p">(</span><span class="n">cross_attns</span><span class="p">,</span> <span class="n">latent_transformer</span><span class="p">)</span>
+                <span class="p">)</span>
+        <span class="k">return</span> <span class="n">attention_weights</span>
+
+    <span class="k">def</span> <span class="nf">_build_perceiver_block</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">nn</span><span class="o">.</span><span class="n">ModuleDict</span><span class="p">:</span>
+        <span class="n">perceiver_block</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">ModuleDict</span><span class="p">()</span>
+
+        <span class="c1"># Cross Attention</span>
+        <span class="n">cross_attns</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">ModuleList</span><span class="p">()</span>
+        <span class="k">for</span> <span class="n">_</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">n_cross_attns</span><span class="p">):</span>
+            <span class="n">cross_attns</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
+                <span class="n">PerceiverEncoder</span><span class="p">(</span>
+                    <span class="bp">self</span><span class="o">.</span><span class="n">input_dim</span><span class="p">,</span>
+                    <span class="bp">self</span><span class="o">.</span><span class="n">n_cross_attn_heads</span><span class="p">,</span>
+                    <span class="kc">False</span><span class="p">,</span>  <span class="c1"># use_bias</span>
+                    <span class="bp">self</span><span class="o">.</span><span class="n">attn_dropout</span><span class="p">,</span>
+                    <span class="bp">self</span><span class="o">.</span><span class="n">ff_dropout</span><span class="p">,</span>
+                    <span class="bp">self</span><span class="o">.</span><span class="n">ff_factor</span><span class="p">,</span>
+                    <span class="bp">self</span><span class="o">.</span><span class="n">transformer_activation</span><span class="p">,</span>
+                    <span class="bp">self</span><span class="o">.</span><span class="n">latent_dim</span><span class="p">,</span>  <span class="c1"># q_dim,</span>
+                <span class="p">),</span>
+            <span class="p">)</span>
+        <span class="n">perceiver_block</span><span class="p">[</span><span class="s2">&quot;cross_attns&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">cross_attns</span>
+
+        <span class="c1"># Latent Transformer</span>
+        <span class="n">latent_transformer</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Sequential</span><span class="p">()</span>
+        <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">n_latent_blocks</span><span class="p">):</span>
+            <span class="n">latent_transformer</span><span class="o">.</span><span class="n">add_module</span><span class="p">(</span>
+                <span class="s2">&quot;latent_block&quot;</span> <span class="o">+</span> <span class="nb">str</span><span class="p">(</span><span class="n">i</span><span class="p">),</span>
+                <span class="n">PerceiverEncoder</span><span class="p">(</span>
+                    <span class="bp">self</span><span class="o">.</span><span class="n">latent_dim</span><span class="p">,</span>  <span class="c1"># input_dim</span>
+                    <span class="bp">self</span><span class="o">.</span><span class="n">n_latent_heads</span><span class="p">,</span>
+                    <span class="kc">False</span><span class="p">,</span>  <span class="c1"># use_bias</span>
+                    <span class="bp">self</span><span class="o">.</span><span class="n">attn_dropout</span><span class="p">,</span>
+                    <span class="bp">self</span><span class="o">.</span><span class="n">ff_dropout</span><span class="p">,</span>
+                    <span class="bp">self</span><span class="o">.</span><span class="n">ff_factor</span><span class="p">,</span>
+                    <span class="bp">self</span><span class="o">.</span><span class="n">transformer_activation</span><span class="p">,</span>
+                <span class="p">),</span>
+            <span class="p">)</span>
+        <span class="n">perceiver_block</span><span class="p">[</span><span class="s2">&quot;latent_transformer&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">latent_transformer</span>
+
+        <span class="k">return</span> <span class="n">perceiver_block</span>
+
+    <span class="nd">@staticmethod</span>
+    <span class="k">def</span> <span class="nf">_extract_attn_weights</span><span class="p">(</span><span class="n">cross_attns</span><span class="p">,</span> <span class="n">latent_transformer</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">List</span><span class="p">:</span>
+        <span class="n">attention_weights</span> <span class="o">=</span> <span class="p">[]</span>
+        <span class="k">for</span> <span class="n">cross_attn</span> <span class="ow">in</span> <span class="n">cross_attns</span><span class="p">:</span>
+            <span class="n">attention_weights</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">cross_attn</span><span class="o">.</span><span class="n">attn</span><span class="o">.</span><span class="n">attn_weights</span><span class="p">)</span>
+        <span class="k">for</span> <span class="n">latent_block</span> <span class="ow">in</span> <span class="n">latent_transformer</span><span class="p">:</span>
+            <span class="n">attention_weights</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">latent_block</span><span class="o">.</span><span class="n">attn</span><span class="o">.</span><span class="n">attn_weights</span><span class="p">)</span>
+        <span class="k">return</span> <span class="n">attention_weights</span>
+</code></pre></div></td></tr></table></div>
+              </details>
+
+
+
+  <div class="doc doc-children">
+
+
+
+
+
+
+
+<div class="doc doc-object doc-attribute">
+
+
+
+<h3 id="pytorch_widedeep.models.tabular.transformers.tab_perceiver.TabPerceiver.output_dim" class="doc doc-heading">
+            <span class="doc doc-object-name doc-attribute-name">output_dim</span>
+
+
+  <span class="doc doc-labels">
+      <small class="doc doc-label doc-label-property"><code>property</code></small>
+  </span>
+
+<a href="#pytorch_widedeep.models.tabular.transformers.tab_perceiver.TabPerceiver.output_dim" class="headerlink" title="Permanent link">&para;</a></h3>
+<div class="doc-signature highlight"><pre><span></span><code><span class="n">output_dim</span>
+</code></pre></div>
+
+    <div class="doc doc-contents ">
+
+        <p>The output dimension of the model. This is a required property
+neccesary to build the <code>WideDeep</code> class</p>
+    </div>
+
+</div>
+
+<div class="doc doc-object doc-attribute">
+
+
+
+<h3 id="pytorch_widedeep.models.tabular.transformers.tab_perceiver.TabPerceiver.attention_weights" class="doc doc-heading">
             <span class="doc doc-object-name doc-attribute-name">attention_weights</span>
 
 
@@ -9316,44 +14592,7 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_fastformer.TabFastForme
 
 
 <a href="#pytorch_widedeep.models.tabular.transformers.tab_fastformer.TabFastFormer" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="nf">TabFastFormer</span><span class="p">(</span>
-    <span class="n">column_idx</span><span class="p">,</span>
-    <span class="o">*</span><span class="p">,</span>
-    <span class="n">cat_embed_input</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">cat_embed_dropout</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">use_cat_bias</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">cat_embed_activation</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">shared_embed</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">add_shared_embed</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">frac_shared_embed</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">continuous_cols</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">cont_norm_layer</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">embed_continuous_method</span><span class="o">=</span><span class="s2">&quot;standard&quot;</span><span class="p">,</span>
-    <span class="n">cont_embed_dropout</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">cont_embed_activation</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">quantization_setup</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">n_frequencies</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">sigma</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">share_last_layer</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">full_embed_dropout</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">input_dim</span><span class="o">=</span><span class="mi">32</span><span class="p">,</span>
-    <span class="n">n_heads</span><span class="o">=</span><span class="mi">8</span><span class="p">,</span>
-    <span class="n">use_bias</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
-    <span class="n">n_blocks</span><span class="o">=</span><span class="mi">4</span><span class="p">,</span>
-    <span class="n">attn_dropout</span><span class="o">=</span><span class="mf">0.1</span><span class="p">,</span>
-    <span class="n">ff_dropout</span><span class="o">=</span><span class="mf">0.2</span><span class="p">,</span>
-    <span class="n">ff_factor</span><span class="o">=</span><span class="mi">4</span><span class="p">,</span>
-    <span class="n">share_qv_weights</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
-    <span class="n">share_weights</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
-    <span class="n">transformer_activation</span><span class="o">=</span><span class="s2">&quot;relu&quot;</span><span class="p">,</span>
-    <span class="n">mlp_hidden_dims</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">mlp_activation</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">mlp_dropout</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">mlp_batchnorm</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">mlp_batchnorm_last</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">mlp_linear_first</span><span class="o">=</span><span class="kc">None</span>
-<span class="p">)</span>
-</code></pre></div>
+
 
     <div class="doc doc-contents first">
             <p class="doc doc-class-bases">
@@ -9374,132 +14613,183 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_fastformer.TabFastForme
 
 
 <p><span class="doc-section-title">Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>column_idx</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, int]</code>)
-          –
-          <div class="doc-md-description">
-            <p>Dict containing the index of the columns that will be passed through
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+          <th>Default</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>column_idx</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, int]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Dict containing the index of the columns that will be passed through
 the <code>TabMlp</code> model. Required to slice the tensors. e.g. <em>{'education':
 0, 'relationship': 1, 'workclass': 2, ...}</em>.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>cat_embed_input</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.wdtypes.Tuple">Tuple</span>[str, int]]]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>List of Tuples with the column name and number of unique values and
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>cat_embed_input</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.wdtypes.Tuple">Tuple</span>[str, int]]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>List of Tuples with the column name and number of unique values and
 embedding dimension. e.g. <em>[(education, 11), ...]</em></p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>cat_embed_dropout</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Categorical embeddings dropout. If <code>None</code>, it will default
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>cat_embed_dropout</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Categorical embeddings dropout. If <code>None</code>, it will default
 to 0.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>use_cat_bias</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Boolean indicating if bias will be used for the categorical embeddings.
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>use_cat_bias</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Boolean indicating if bias will be used for the categorical embeddings.
 If <code>None</code>, it will default to 'False'.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>cat_embed_activation</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Activation function for the categorical embeddings, if any. Currently
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>cat_embed_activation</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Activation function for the categorical embeddings, if any. Currently
 <em>'tanh'</em>, <em>'relu'</em>, <em>'leaky_relu'</em> and <em>'gelu'</em> are supported</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>shared_embed</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Boolean indicating if the embeddings will be "shared". The idea behind <code>shared_embed</code> is
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>shared_embed</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Boolean indicating if the embeddings will be "shared". The idea behind <code>shared_embed</code> is
 described in the Appendix A in the <a href="https://arxiv.org/abs/2012.06678">TabTransformer paper</a>:
 <em>'The goal of having column embedding is to enable the model to
 distinguish the classes in one column from those in the other
 columns'</em>. In other words, the idea is to let the model learn which
 column is embedded at the time. See: <code>pytorch_widedeep.models.transformers._layers.SharedEmbeddings</code>.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>add_shared_embed</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>The two embedding sharing strategies are: 1) add the shared embeddings
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>add_shared_embed</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>The two embedding sharing strategies are: 1) add the shared embeddings
 to the column embeddings or 2) to replace the first
 <code>frac_shared_embed</code> with the shared embeddings.
 See <code>pytorch_widedeep.models.embeddings_layers.SharedEmbeddings</code>
 If 'None' is passed, it will default to 'False'.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>frac_shared_embed</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>The fraction of embeddings that will be shared (if <code>add_shared_embed
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>frac_shared_embed</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>The fraction of embeddings that will be shared (if <code>add_shared_embed
 = False</code>) by all the different categories for one particular
 column. If 'None' is passed, it will default to 0.0.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>continuous_cols</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[str]]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>List with the name of the numeric (aka continuous) columns</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>cont_norm_layer</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[batchnorm, layernorm]]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Type of normalization layer applied to the continuous features.
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>continuous_cols</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[str]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>List with the name of the numeric (aka continuous) columns</p>
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>cont_norm_layer</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[batchnorm, layernorm]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Type of normalization layer applied to the continuous features.
 Options are: <em>'layernorm'</em> and <em>'batchnorm'</em>. if <code>None</code>, no
 normalization layer will be used.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>embed_continuous_method</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[standard, piecewise, periodic]]</code>, default:
-                  <code>&#39;standard&#39;</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Method to use to embed the continuous features. Options are:
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>embed_continuous_method</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[standard, piecewise, periodic]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Method to use to embed the continuous features. Options are:
 <em>'standard'</em>, <em>'periodic'</em> or <em>'piecewise'</em>. The <em>'standard'</em>
 embedding method is based on the FT-Transformer implementation
 presented in the paper: <a href="https://arxiv.org/abs/2106.11959v5">Revisiting Deep Learning Models for
@@ -9507,316 +14797,432 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_fastformer.TabFastForme
 and_'piecewise'_ methods were presented in the paper: <a href="https://arxiv.org/abs/2203.05556">On Embeddings for
 Numerical Features in Tabular Deep Learning</a>.
 Please, read the papers for details.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>cont_embed_dropout</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>cont_embed_activation</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Activation function for the continuous embeddings if any. Currently
+              </div>
+            </td>
+            <td>
+                  <code>&#39;standard&#39;</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>cont_embed_dropout</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p>
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>cont_embed_activation</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Activation function for the continuous embeddings if any. Currently
 <em>'tanh'</em>, <em>'relu'</em>, <em>'leaky_relu'</em> and <em>'gelu'</em> are supported.
 If <code>None</code>, no activation function will be applied.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>quantization_setup</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, <span title="pytorch_widedeep.wdtypes.List">List</span>[float]]]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>This parameter is used when the <em>'piecewise'</em> method is used to embed
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>quantization_setup</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, <span title="pytorch_widedeep.wdtypes.List">List</span>[float]]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>This parameter is used when the <em>'piecewise'</em> method is used to embed
 the continuous cols. It is a dict where keys are the name of the continuous
 columns and values are lists with the boundaries for the quantization
 of the continuous_cols. See the examples for details. If
 If the <em>'piecewise'</em> method is used, this parameter is required.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>n_frequencies</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[int]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>This is the so called <em>'k'</em> in their paper <a href="https://arxiv.org/abs/2203.05556">On Embeddings for
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>n_frequencies</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[int]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>This is the so called <em>'k'</em> in their paper <a href="https://arxiv.org/abs/2203.05556">On Embeddings for
 Numerical Features in Tabular Deep Learning</a>,
 and is the number of 'frequencies' that will be used to represent each
 continuous column. See their Eq 2 in the paper for details. If
 the <em>'periodic'</em> method is used, this parameter is required.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>sigma</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>This is the sigma parameter in the paper mentioned when describing the
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>sigma</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>This is the sigma parameter in the paper mentioned when describing the
 previous parameters and it is used to initialise the 'frequency
 weights'. See their Eq 2 in the paper for details. If
 the <em>'periodic'</em> method is used, this parameter is required.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>share_last_layer</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>This parameter is not present in the before mentioned paper but it is implemented in
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>share_last_layer</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>This parameter is not present in the before mentioned paper but it is implemented in
 the <a href="https://github.com/yandex-research/rtdl-num-embeddings/tree/main">official repo</a>.
 If <code>True</code> the linear layer that turns the frequencies into embeddings
 will be shared across the continuous columns. If <code>False</code> a different
 linear layer will be used for each continuous column.
 If the <em>'periodic'</em> method is used, this parameter is required.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>full_embed_dropout</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>If <code>True</code>, the full embedding corresponding to a column will be masked
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>full_embed_dropout</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>If <code>True</code>, the full embedding corresponding to a column will be masked
 out/dropout. If <code>None</code>, it will default to <code>False</code>.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>input_dim</code></b>
-              (<code>int</code>, default:
-                  <code>32</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>The so-called <em>dimension of the model</em>. Is the number of
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>input_dim</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>The so-called <em>dimension of the model</em>. Is the number of
 embeddings used to encode the categorical and/or continuous columns</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>n_heads</code></b>
-              (<code>int</code>, default:
+              </div>
+            </td>
+            <td>
+                  <code>32</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>n_heads</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Number of attention heads per FastFormer block</p>
+              </div>
+            </td>
+            <td>
                   <code>8</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Number of attention heads per FastFormer block</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>use_bias</code></b>
-              (<code>bool</code>, default:
-                  <code>False</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Boolean indicating whether or not to use bias in the Q, K, and V
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>use_bias</code></td>
+            <td>
+                  <code>bool</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Boolean indicating whether or not to use bias in the Q, K, and V
 projection layers</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>n_blocks</code></b>
-              (<code>int</code>, default:
+              </div>
+            </td>
+            <td>
+                  <code>False</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>n_blocks</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Number of FastFormer blocks</p>
+              </div>
+            </td>
+            <td>
                   <code>4</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Number of FastFormer blocks</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>attn_dropout</code></b>
-              (<code>float</code>, default:
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>attn_dropout</code></td>
+            <td>
+                  <code>float</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Dropout that will be applied to the Additive Attention layers</p>
+              </div>
+            </td>
+            <td>
                   <code>0.1</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Dropout that will be applied to the Additive Attention layers</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>ff_dropout</code></b>
-              (<code>float</code>, default:
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>ff_dropout</code></td>
+            <td>
+                  <code>float</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Dropout that will be applied to the FeedForward network</p>
+              </div>
+            </td>
+            <td>
                   <code>0.2</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Dropout that will be applied to the FeedForward network</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>ff_factor</code></b>
-              (<code>int</code>, default:
-                  <code>4</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Multiplicative factor applied to the first layer of the FF network in
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>ff_factor</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Multiplicative factor applied to the first layer of the FF network in
 each Transformer block, This is normally set to 4.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>share_qv_weights</code></b>
-              (<code>bool</code>, default:
-                  <code>False</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Following the paper, this is a boolean indicating if the Value (<span class="arithmatex">\(V\)</span>) and
+              </div>
+            </td>
+            <td>
+                  <code>4</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>share_qv_weights</code></td>
+            <td>
+                  <code>bool</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Following the paper, this is a boolean indicating if the Value (<span class="arithmatex">\(V\)</span>) and
 the Query (<span class="arithmatex">\(Q\)</span>) transformation parameters will be shared.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>share_weights</code></b>
-              (<code>bool</code>, default:
+              </div>
+            </td>
+            <td>
                   <code>False</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>In addition to sharing the <span class="arithmatex">\(V\)</span> and <span class="arithmatex">\(Q\)</span> transformation parameters, the
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>share_weights</code></td>
+            <td>
+                  <code>bool</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>In addition to sharing the <span class="arithmatex">\(V\)</span> and <span class="arithmatex">\(Q\)</span> transformation parameters, the
 parameters across different Fastformer layers can also be shared.
 Please, see
 <code>pytorch_widedeep/models/tabular/transformers/tab_fastformer.py</code> for
 details</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>transformer_activation</code></b>
-              (<code>str</code>, default:
-                  <code>&#39;relu&#39;</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Transformer Encoder activation function. <em>'tanh'</em>, <em>'relu'</em>,
+              </div>
+            </td>
+            <td>
+                  <code>False</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>transformer_activation</code></td>
+            <td>
+                  <code>str</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Transformer Encoder activation function. <em>'tanh'</em>, <em>'relu'</em>,
 <em>'leaky_relu'</em>, <em>'gelu'</em>, <em>'geglu'</em> and <em>'reglu'</em> are supported</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>mlp_hidden_dims</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[int]]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>MLP hidden dimensions. If not provided no MLP on top of the final
+              </div>
+            </td>
+            <td>
+                  <code>&#39;relu&#39;</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>mlp_hidden_dims</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[int]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>MLP hidden dimensions. If not provided no MLP on top of the final
 FTTransformer block will be used</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>mlp_hidden_dims</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[int]]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>List with the number of neurons per dense layer in the MLP. e.g:
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>mlp_hidden_dims</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[int]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>List with the number of neurons per dense layer in the MLP. e.g:
 <em>[64, 32]</em>. If not provided no MLP on top of the final
 Transformer block will be used.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>mlp_activation</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Activation function for the dense layers of the MLP. Currently
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>mlp_activation</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Activation function for the dense layers of the MLP. Currently
 <em>'tanh'</em>, <em>'relu'</em>, <em>'leaky'_relu' and _'gelu'</em> are supported.
 If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it
 will default to <em>'relu'</em>.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>mlp_dropout</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>float with the dropout between the dense layers of the MLP.
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>mlp_dropout</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>float with the dropout between the dense layers of the MLP.
 If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it
 will default to 0.0.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>mlp_batchnorm</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Boolean indicating whether or not batch normalization will be applied
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>mlp_batchnorm</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Boolean indicating whether or not batch normalization will be applied
 to the dense layers
 If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it
 will default to False.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>mlp_batchnorm_last</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Boolean indicating whether or not batch normalization will be applied
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>mlp_batchnorm_last</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Boolean indicating whether or not batch normalization will be applied
 to the last of the dense layers
 If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it
 will default to False.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>mlp_linear_first</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Boolean indicating the order of the operations in the dense
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>mlp_linear_first</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Boolean indicating the order of the operations in the dense
 layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt;
 LIN -&gt; ACT]</code>
 If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it
 will default to <code>True</code>.</p>
-          </div>
-        </li>
-    </ul>
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
 
 <p><span class="doc-section-title">Attributes:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code><span title="pytorch_widedeep.models.tabular.transformers.tab_fastformer.TabFastFormer.encoder">encoder</span></code></b>
-              (<code><span title="torch.nn.Module">Module</span></code>)
-          –
-          <div class="doc-md-description">
-            <p>Sequence of FasFormer blocks.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code><span title="pytorch_widedeep.models.tabular.transformers.tab_fastformer.TabFastFormer.mlp">mlp</span></code></b>
-              (<code><span title="torch.nn.Module">Module</span></code>)
-          –
-          <div class="doc-md-description">
-            <p>MLP component in the model</p>
-          </div>
-        </li>
-    </ul>
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code><span title="pytorch_widedeep.models.tabular.transformers.tab_fastformer.TabFastFormer.encoder">encoder</span></code></td>
+            <td>
+                  <code><span title="torch.nn.Module">Module</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Sequence of FasFormer blocks.</p>
+              </div>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code><span title="pytorch_widedeep.models.tabular.transformers.tab_fastformer.TabFastFormer.mlp">mlp</span></code></td>
+            <td>
+                  <code><span title="torch.nn.Module">Module</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>MLP component in the model</p>
+              </div>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
 
 <p><span class="doc-section-title">Examples:</span></p>
@@ -9831,9 +15237,189 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_fastformer.TabFastForme
 <span class="gp">&gt;&gt;&gt; </span><span class="n">out</span> <span class="o">=</span> <span class="n">model</span><span class="p">(</span><span class="n">X_tab</span><span class="p">)</span>
 </code></pre></div>
 
-                  <details class="quote">
-                    <summary>Source code in <code>pytorch_widedeep/models/tabular/transformers/tab_fastformer.py</code></summary>
-                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">191</span>
+              <details class="quote">
+                <summary>Source code in <code>pytorch_widedeep/models/tabular/transformers/tab_fastformer.py</code></summary>
+                <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 11</span>
+<span class="normal"> 12</span>
+<span class="normal"> 13</span>
+<span class="normal"> 14</span>
+<span class="normal"> 15</span>
+<span class="normal"> 16</span>
+<span class="normal"> 17</span>
+<span class="normal"> 18</span>
+<span class="normal"> 19</span>
+<span class="normal"> 20</span>
+<span class="normal"> 21</span>
+<span class="normal"> 22</span>
+<span class="normal"> 23</span>
+<span class="normal"> 24</span>
+<span class="normal"> 25</span>
+<span class="normal"> 26</span>
+<span class="normal"> 27</span>
+<span class="normal"> 28</span>
+<span class="normal"> 29</span>
+<span class="normal"> 30</span>
+<span class="normal"> 31</span>
+<span class="normal"> 32</span>
+<span class="normal"> 33</span>
+<span class="normal"> 34</span>
+<span class="normal"> 35</span>
+<span class="normal"> 36</span>
+<span class="normal"> 37</span>
+<span class="normal"> 38</span>
+<span class="normal"> 39</span>
+<span class="normal"> 40</span>
+<span class="normal"> 41</span>
+<span class="normal"> 42</span>
+<span class="normal"> 43</span>
+<span class="normal"> 44</span>
+<span class="normal"> 45</span>
+<span class="normal"> 46</span>
+<span class="normal"> 47</span>
+<span class="normal"> 48</span>
+<span class="normal"> 49</span>
+<span class="normal"> 50</span>
+<span class="normal"> 51</span>
+<span class="normal"> 52</span>
+<span class="normal"> 53</span>
+<span class="normal"> 54</span>
+<span class="normal"> 55</span>
+<span class="normal"> 56</span>
+<span class="normal"> 57</span>
+<span class="normal"> 58</span>
+<span class="normal"> 59</span>
+<span class="normal"> 60</span>
+<span class="normal"> 61</span>
+<span class="normal"> 62</span>
+<span class="normal"> 63</span>
+<span class="normal"> 64</span>
+<span class="normal"> 65</span>
+<span class="normal"> 66</span>
+<span class="normal"> 67</span>
+<span class="normal"> 68</span>
+<span class="normal"> 69</span>
+<span class="normal"> 70</span>
+<span class="normal"> 71</span>
+<span class="normal"> 72</span>
+<span class="normal"> 73</span>
+<span class="normal"> 74</span>
+<span class="normal"> 75</span>
+<span class="normal"> 76</span>
+<span class="normal"> 77</span>
+<span class="normal"> 78</span>
+<span class="normal"> 79</span>
+<span class="normal"> 80</span>
+<span class="normal"> 81</span>
+<span class="normal"> 82</span>
+<span class="normal"> 83</span>
+<span class="normal"> 84</span>
+<span class="normal"> 85</span>
+<span class="normal"> 86</span>
+<span class="normal"> 87</span>
+<span class="normal"> 88</span>
+<span class="normal"> 89</span>
+<span class="normal"> 90</span>
+<span class="normal"> 91</span>
+<span class="normal"> 92</span>
+<span class="normal"> 93</span>
+<span class="normal"> 94</span>
+<span class="normal"> 95</span>
+<span class="normal"> 96</span>
+<span class="normal"> 97</span>
+<span class="normal"> 98</span>
+<span class="normal"> 99</span>
+<span class="normal">100</span>
+<span class="normal">101</span>
+<span class="normal">102</span>
+<span class="normal">103</span>
+<span class="normal">104</span>
+<span class="normal">105</span>
+<span class="normal">106</span>
+<span class="normal">107</span>
+<span class="normal">108</span>
+<span class="normal">109</span>
+<span class="normal">110</span>
+<span class="normal">111</span>
+<span class="normal">112</span>
+<span class="normal">113</span>
+<span class="normal">114</span>
+<span class="normal">115</span>
+<span class="normal">116</span>
+<span class="normal">117</span>
+<span class="normal">118</span>
+<span class="normal">119</span>
+<span class="normal">120</span>
+<span class="normal">121</span>
+<span class="normal">122</span>
+<span class="normal">123</span>
+<span class="normal">124</span>
+<span class="normal">125</span>
+<span class="normal">126</span>
+<span class="normal">127</span>
+<span class="normal">128</span>
+<span class="normal">129</span>
+<span class="normal">130</span>
+<span class="normal">131</span>
+<span class="normal">132</span>
+<span class="normal">133</span>
+<span class="normal">134</span>
+<span class="normal">135</span>
+<span class="normal">136</span>
+<span class="normal">137</span>
+<span class="normal">138</span>
+<span class="normal">139</span>
+<span class="normal">140</span>
+<span class="normal">141</span>
+<span class="normal">142</span>
+<span class="normal">143</span>
+<span class="normal">144</span>
+<span class="normal">145</span>
+<span class="normal">146</span>
+<span class="normal">147</span>
+<span class="normal">148</span>
+<span class="normal">149</span>
+<span class="normal">150</span>
+<span class="normal">151</span>
+<span class="normal">152</span>
+<span class="normal">153</span>
+<span class="normal">154</span>
+<span class="normal">155</span>
+<span class="normal">156</span>
+<span class="normal">157</span>
+<span class="normal">158</span>
+<span class="normal">159</span>
+<span class="normal">160</span>
+<span class="normal">161</span>
+<span class="normal">162</span>
+<span class="normal">163</span>
+<span class="normal">164</span>
+<span class="normal">165</span>
+<span class="normal">166</span>
+<span class="normal">167</span>
+<span class="normal">168</span>
+<span class="normal">169</span>
+<span class="normal">170</span>
+<span class="normal">171</span>
+<span class="normal">172</span>
+<span class="normal">173</span>
+<span class="normal">174</span>
+<span class="normal">175</span>
+<span class="normal">176</span>
+<span class="normal">177</span>
+<span class="normal">178</span>
+<span class="normal">179</span>
+<span class="normal">180</span>
+<span class="normal">181</span>
+<span class="normal">182</span>
+<span class="normal">183</span>
+<span class="normal">184</span>
+<span class="normal">185</span>
+<span class="normal">186</span>
+<span class="normal">187</span>
+<span class="normal">188</span>
+<span class="normal">189</span>
+<span class="normal">190</span>
+<span class="normal">191</span>
 <span class="normal">192</span>
 <span class="normal">193</span>
 <span class="normal">194</span>
@@ -9975,151 +15561,407 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_fastformer.TabFastForme
 <span class="normal">330</span>
 <span class="normal">331</span>
 <span class="normal">332</span>
-<span class="normal">333</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
-    <span class="bp">self</span><span class="p">,</span>
-    <span class="n">column_idx</span><span class="p">:</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">],</span>
-    <span class="o">*</span><span class="p">,</span>
-    <span class="n">cat_embed_input</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="n">Tuple</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">]]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">cat_embed_dropout</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">use_cat_bias</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">cat_embed_activation</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">shared_embed</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">add_shared_embed</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">frac_shared_embed</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">continuous_cols</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">cont_norm_layer</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Literal</span><span class="p">[</span><span class="s2">&quot;batchnorm&quot;</span><span class="p">,</span> <span class="s2">&quot;layernorm&quot;</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">embed_continuous_method</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span>
-        <span class="n">Literal</span><span class="p">[</span><span class="s2">&quot;standard&quot;</span><span class="p">,</span> <span class="s2">&quot;piecewise&quot;</span><span class="p">,</span> <span class="s2">&quot;periodic&quot;</span><span class="p">]</span>
-    <span class="p">]</span> <span class="o">=</span> <span class="s2">&quot;standard&quot;</span><span class="p">,</span>
-    <span class="n">cont_embed_dropout</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">cont_embed_activation</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">quantization_setup</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="nb">float</span><span class="p">]]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">n_frequencies</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">sigma</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">share_last_layer</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">full_embed_dropout</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">input_dim</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">32</span><span class="p">,</span>
-    <span class="n">n_heads</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">8</span><span class="p">,</span>
-    <span class="n">use_bias</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
-    <span class="n">n_blocks</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">4</span><span class="p">,</span>
-    <span class="n">attn_dropout</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.1</span><span class="p">,</span>
-    <span class="n">ff_dropout</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.2</span><span class="p">,</span>
-    <span class="n">ff_factor</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">4</span><span class="p">,</span>
-    <span class="n">share_qv_weights</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
-    <span class="n">share_weights</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
-    <span class="n">transformer_activation</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;relu&quot;</span><span class="p">,</span>
-    <span class="n">mlp_hidden_dims</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">int</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">mlp_activation</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">mlp_dropout</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">mlp_batchnorm</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">mlp_batchnorm_last</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">mlp_linear_first</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-<span class="p">):</span>
-    <span class="nb">super</span><span class="p">(</span><span class="n">TabFastFormer</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span>
-        <span class="n">column_idx</span><span class="o">=</span><span class="n">column_idx</span><span class="p">,</span>
-        <span class="n">cat_embed_input</span><span class="o">=</span><span class="n">cat_embed_input</span><span class="p">,</span>
-        <span class="n">cat_embed_dropout</span><span class="o">=</span><span class="n">cat_embed_dropout</span><span class="p">,</span>
-        <span class="n">use_cat_bias</span><span class="o">=</span><span class="n">use_cat_bias</span><span class="p">,</span>
-        <span class="n">cat_embed_activation</span><span class="o">=</span><span class="n">cat_embed_activation</span><span class="p">,</span>
-        <span class="n">shared_embed</span><span class="o">=</span><span class="n">shared_embed</span><span class="p">,</span>
-        <span class="n">add_shared_embed</span><span class="o">=</span><span class="n">add_shared_embed</span><span class="p">,</span>
-        <span class="n">frac_shared_embed</span><span class="o">=</span><span class="n">frac_shared_embed</span><span class="p">,</span>
-        <span class="n">continuous_cols</span><span class="o">=</span><span class="n">continuous_cols</span><span class="p">,</span>
-        <span class="n">cont_norm_layer</span><span class="o">=</span><span class="n">cont_norm_layer</span><span class="p">,</span>
-        <span class="n">embed_continuous_method</span><span class="o">=</span><span class="n">embed_continuous_method</span><span class="p">,</span>
-        <span class="n">cont_embed_dropout</span><span class="o">=</span><span class="n">cont_embed_dropout</span><span class="p">,</span>
-        <span class="n">cont_embed_activation</span><span class="o">=</span><span class="n">cont_embed_activation</span><span class="p">,</span>
-        <span class="n">input_dim</span><span class="o">=</span><span class="n">input_dim</span><span class="p">,</span>
-        <span class="n">quantization_setup</span><span class="o">=</span><span class="n">quantization_setup</span><span class="p">,</span>
-        <span class="n">n_frequencies</span><span class="o">=</span><span class="n">n_frequencies</span><span class="p">,</span>
-        <span class="n">sigma</span><span class="o">=</span><span class="n">sigma</span><span class="p">,</span>
-        <span class="n">share_last_layer</span><span class="o">=</span><span class="n">share_last_layer</span><span class="p">,</span>
-        <span class="n">full_embed_dropout</span><span class="o">=</span><span class="n">full_embed_dropout</span><span class="p">,</span>
-    <span class="p">)</span>
+<span class="normal">333</span>
+<span class="normal">334</span>
+<span class="normal">335</span>
+<span class="normal">336</span>
+<span class="normal">337</span>
+<span class="normal">338</span>
+<span class="normal">339</span>
+<span class="normal">340</span>
+<span class="normal">341</span>
+<span class="normal">342</span>
+<span class="normal">343</span>
+<span class="normal">344</span>
+<span class="normal">345</span>
+<span class="normal">346</span>
+<span class="normal">347</span>
+<span class="normal">348</span>
+<span class="normal">349</span>
+<span class="normal">350</span>
+<span class="normal">351</span>
+<span class="normal">352</span>
+<span class="normal">353</span>
+<span class="normal">354</span>
+<span class="normal">355</span>
+<span class="normal">356</span>
+<span class="normal">357</span>
+<span class="normal">358</span>
+<span class="normal">359</span>
+<span class="normal">360</span>
+<span class="normal">361</span>
+<span class="normal">362</span>
+<span class="normal">363</span>
+<span class="normal">364</span>
+<span class="normal">365</span>
+<span class="normal">366</span>
+<span class="normal">367</span>
+<span class="normal">368</span>
+<span class="normal">369</span>
+<span class="normal">370</span>
+<span class="normal">371</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">class</span> <span class="nc">TabFastFormer</span><span class="p">(</span><span class="n">BaseTabularModelWithAttention</span><span class="p">):</span>
+<span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Defines an adaptation of a [FastFormer](https://arxiv.org/abs/2108.09084)</span>
+<span class="sd">    that can be used as the `deeptabular` component of a Wide &amp; Deep model</span>
+<span class="sd">    or independently by itself.</span>
+
+<span class="sd">    Most of the parameters for this class are `Optional` since the use of</span>
+<span class="sd">    categorical or continuous is in fact optional (i.e. one can use</span>
+<span class="sd">    categorical features only, continuous features only or both).</span>
+
+<span class="sd">    :information_source: **NOTE**: while there are scientific publications for</span>
+<span class="sd">     the `TabTransformer`, `SAINT` and `FTTransformer`, the `TabPerceiver`</span>
+<span class="sd">     and the `TabFastFormer` are our own adaptations of the</span>
+<span class="sd">     [Perceiver](https://arxiv.org/abs/2103.03206) and the</span>
+<span class="sd">     [FastFormer](https://arxiv.org/abs/2108.09084) for tabular data.</span>
+
+<span class="sd">    Parameters</span>
+<span class="sd">    ----------</span>
+<span class="sd">    column_idx: Dict</span>
+<span class="sd">        Dict containing the index of the columns that will be passed through</span>
+<span class="sd">        the `TabMlp` model. Required to slice the tensors. e.g. _{&#39;education&#39;:</span>
+<span class="sd">        0, &#39;relationship&#39;: 1, &#39;workclass&#39;: 2, ...}_.</span>
+<span class="sd">    cat_embed_input: List, Optional, default = None</span>
+<span class="sd">        List of Tuples with the column name and number of unique values and</span>
+<span class="sd">        embedding dimension. e.g. _[(education, 11), ...]_</span>
+<span class="sd">    cat_embed_dropout: float, Optional, default = None</span>
+<span class="sd">        Categorical embeddings dropout. If `None`, it will default</span>
+<span class="sd">        to 0.</span>
+<span class="sd">    use_cat_bias: bool, Optional, default = None,</span>
+<span class="sd">        Boolean indicating if bias will be used for the categorical embeddings.</span>
+<span class="sd">        If `None`, it will default to &#39;False&#39;.</span>
+<span class="sd">    cat_embed_activation: Optional, str, default = None,</span>
+<span class="sd">        Activation function for the categorical embeddings, if any. Currently</span>
+<span class="sd">        _&#39;tanh&#39;_, _&#39;relu&#39;_, _&#39;leaky_relu&#39;_ and _&#39;gelu&#39;_ are supported</span>
+<span class="sd">    shared_embed: bool, Optional, default = None</span>
+<span class="sd">        Boolean indicating if the embeddings will be &quot;shared&quot;. The idea behind `shared_embed` is</span>
+<span class="sd">        described in the Appendix A in the [TabTransformer paper](https://arxiv.org/abs/2012.06678):</span>
+<span class="sd">        _&#39;The goal of having column embedding is to enable the model to</span>
+<span class="sd">        distinguish the classes in one column from those in the other</span>
+<span class="sd">        columns&#39;_. In other words, the idea is to let the model learn which</span>
+<span class="sd">        column is embedded at the time. See: `pytorch_widedeep.models.transformers._layers.SharedEmbeddings`.</span>
+<span class="sd">    add_shared_embed: bool, Optional, default = None</span>
+<span class="sd">        The two embedding sharing strategies are: 1) add the shared embeddings</span>
+<span class="sd">        to the column embeddings or 2) to replace the first</span>
+<span class="sd">        `frac_shared_embed` with the shared embeddings.</span>
+<span class="sd">        See `pytorch_widedeep.models.embeddings_layers.SharedEmbeddings`</span>
+<span class="sd">        If &#39;None&#39; is passed, it will default to &#39;False&#39;.</span>
+<span class="sd">    frac_shared_embed: float, Optional, default = None</span>
+<span class="sd">        The fraction of embeddings that will be shared (if `add_shared_embed</span>
+<span class="sd">        = False`) by all the different categories for one particular</span>
+<span class="sd">        column. If &#39;None&#39; is passed, it will default to 0.0.</span>
+<span class="sd">    continuous_cols: List, Optional, default = None</span>
+<span class="sd">        List with the name of the numeric (aka continuous) columns</span>
+<span class="sd">    cont_norm_layer: str, Optional, default =  None</span>
+<span class="sd">        Type of normalization layer applied to the continuous features.</span>
+<span class="sd">        Options are: _&#39;layernorm&#39;_ and _&#39;batchnorm&#39;_. if `None`, no</span>
+<span class="sd">        normalization layer will be used.</span>
+<span class="sd">    embed_continuous_method: Optional, str, default = None,</span>
+<span class="sd">        Method to use to embed the continuous features. Options are:</span>
+<span class="sd">        _&#39;standard&#39;_, _&#39;periodic&#39;_ or _&#39;piecewise&#39;_. The _&#39;standard&#39;_</span>
+<span class="sd">        embedding method is based on the FT-Transformer implementation</span>
+<span class="sd">        presented in the paper: [Revisiting Deep Learning Models for</span>
+<span class="sd">        Tabular Data](https://arxiv.org/abs/2106.11959v5). The _&#39;periodic&#39;_</span>
+<span class="sd">        and_&#39;piecewise&#39;_ methods were presented in the paper: [On Embeddings for</span>
+<span class="sd">        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556).</span>
+<span class="sd">        Please, read the papers for details.</span>
+<span class="sd">    cont_embed_dropout: float, Optional, default = None,</span>
+<span class="sd">        Dropout for the continuous embeddings. If `None`, it will default to 0.0</span>
+<span class="sd">    cont_embed_activation: Optional, str, default = None,</span>
+<span class="sd">        Activation function for the continuous embeddings if any. Currently</span>
+<span class="sd">        _&#39;tanh&#39;_, _&#39;relu&#39;_, _&#39;leaky_relu&#39;_ and _&#39;gelu&#39;_ are supported.</span>
+<span class="sd">        If `None`, no activation function will be applied.</span>
+<span class="sd">    quantization_setup: Dict[str, List[float]], Optional, default = None,</span>
+<span class="sd">        This parameter is used when the _&#39;piecewise&#39;_ method is used to embed</span>
+<span class="sd">        the continuous cols. It is a dict where keys are the name of the continuous</span>
+<span class="sd">        columns and values are lists with the boundaries for the quantization</span>
+<span class="sd">        of the continuous_cols. See the examples for details. If</span>
+<span class="sd">        If the _&#39;piecewise&#39;_ method is used, this parameter is required.</span>
+<span class="sd">    n_frequencies: int, Optional, default = None,</span>
+<span class="sd">        This is the so called _&#39;k&#39;_ in their paper [On Embeddings for</span>
+<span class="sd">        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556),</span>
+<span class="sd">        and is the number of &#39;frequencies&#39; that will be used to represent each</span>
+<span class="sd">        continuous column. See their Eq 2 in the paper for details. If</span>
+<span class="sd">        the _&#39;periodic&#39;_ method is used, this parameter is required.</span>
+<span class="sd">    sigma: float, Optional, default = None,</span>
+<span class="sd">        This is the sigma parameter in the paper mentioned when describing the</span>
+<span class="sd">        previous parameters and it is used to initialise the &#39;frequency</span>
+<span class="sd">        weights&#39;. See their Eq 2 in the paper for details. If</span>
+<span class="sd">        the _&#39;periodic&#39;_ method is used, this parameter is required.</span>
+<span class="sd">    share_last_layer: bool, Optional, default = None,</span>
+<span class="sd">        This parameter is not present in the before mentioned paper but it is implemented in</span>
+<span class="sd">        the [official repo](https://github.com/yandex-research/rtdl-num-embeddings/tree/main).</span>
+<span class="sd">        If `True` the linear layer that turns the frequencies into embeddings</span>
+<span class="sd">        will be shared across the continuous columns. If `False` a different</span>
+<span class="sd">        linear layer will be used for each continuous column.</span>
+<span class="sd">        If the _&#39;periodic&#39;_ method is used, this parameter is required.</span>
+<span class="sd">    full_embed_dropout: bool, Optional, default = None,</span>
+<span class="sd">        If `True`, the full embedding corresponding to a column will be masked</span>
+<span class="sd">        out/dropout. If `None`, it will default to `False`.</span>
+<span class="sd">    input_dim: int, default = 32</span>
+<span class="sd">        The so-called *dimension of the model*. Is the number of</span>
+<span class="sd">        embeddings used to encode the categorical and/or continuous columns</span>
+<span class="sd">    n_heads: int, default = 8</span>
+<span class="sd">        Number of attention heads per FastFormer block</span>
+<span class="sd">    use_bias: bool, default = False</span>
+<span class="sd">        Boolean indicating whether or not to use bias in the Q, K, and V</span>
+<span class="sd">        projection layers</span>
+<span class="sd">    n_blocks: int, default = 4</span>
+<span class="sd">        Number of FastFormer blocks</span>
+<span class="sd">    attn_dropout: float, default = 0.2</span>
+<span class="sd">        Dropout that will be applied to the Additive Attention layers</span>
+<span class="sd">    ff_dropout: float, default = 0.1</span>
+<span class="sd">        Dropout that will be applied to the FeedForward network</span>
+<span class="sd">    ff_factor: float, default = 4</span>
+<span class="sd">        Multiplicative factor applied to the first layer of the FF network in</span>
+<span class="sd">        each Transformer block, This is normally set to 4.</span>
+<span class="sd">    share_qv_weights: bool, default = False</span>
+<span class="sd">        Following the paper, this is a boolean indicating if the Value ($V$) and</span>
+<span class="sd">        the Query ($Q$) transformation parameters will be shared.</span>
+<span class="sd">    share_weights: bool, default = False</span>
+<span class="sd">        In addition to sharing the $V$ and $Q$ transformation parameters, the</span>
+<span class="sd">        parameters across different Fastformer layers can also be shared.</span>
+<span class="sd">        Please, see</span>
+<span class="sd">        `pytorch_widedeep/models/tabular/transformers/tab_fastformer.py` for</span>
+<span class="sd">        details</span>
+<span class="sd">    transformer_activation: str, default = &quot;gelu&quot;</span>
+<span class="sd">        Transformer Encoder activation function. _&#39;tanh&#39;_, _&#39;relu&#39;_,</span>
+<span class="sd">        _&#39;leaky_relu&#39;_, _&#39;gelu&#39;_, _&#39;geglu&#39;_ and _&#39;reglu&#39;_ are supported</span>
+<span class="sd">    mlp_hidden_dims: List, Optional, default = None</span>
+<span class="sd">        MLP hidden dimensions. If not provided no MLP on top of the final</span>
+<span class="sd">        FTTransformer block will be used</span>
+<span class="sd">    mlp_hidden_dims: List, Optional, default = None</span>
+<span class="sd">        List with the number of neurons per dense layer in the MLP. e.g:</span>
+<span class="sd">        _[64, 32]_. If not provided no MLP on top of the final</span>
+<span class="sd">        Transformer block will be used.</span>
+<span class="sd">    mlp_activation: str, Optional, default = None</span>
+<span class="sd">        Activation function for the dense layers of the MLP. Currently</span>
+<span class="sd">        _&#39;tanh&#39;_, _&#39;relu&#39;_, _&#39;leaky&#39;_relu&#39; and _&#39;gelu&#39;_ are supported.</span>
+<span class="sd">        If &#39;mlp_hidden_dims&#39; is not `None` and this parameter is `None`, it</span>
+<span class="sd">        will default to _&#39;relu&#39;_.</span>
+<span class="sd">    mlp_dropout: float, Optional, default = None</span>
+<span class="sd">        float with the dropout between the dense layers of the MLP.</span>
+<span class="sd">        If &#39;mlp_hidden_dims&#39; is not `None` and this parameter is `None`, it</span>
+<span class="sd">        will default to 0.0.</span>
+<span class="sd">    mlp_batchnorm: bool, Optional, default = None</span>
+<span class="sd">        Boolean indicating whether or not batch normalization will be applied</span>
+<span class="sd">        to the dense layers</span>
+<span class="sd">        If &#39;mlp_hidden_dims&#39; is not `None` and this parameter is `None`, it</span>
+<span class="sd">        will default to False.</span>
+<span class="sd">    mlp_batchnorm_last: bool, Optional, default = None</span>
+<span class="sd">        Boolean indicating whether or not batch normalization will be applied</span>
+<span class="sd">        to the last of the dense layers</span>
+<span class="sd">        If &#39;mlp_hidden_dims&#39; is not `None` and this parameter is `None`, it</span>
+<span class="sd">        will default to False.</span>
+<span class="sd">    mlp_linear_first: bool, Optional, default = None</span>
+<span class="sd">        Boolean indicating the order of the operations in the dense</span>
+<span class="sd">        layer. If `True: [LIN -&gt; ACT -&gt; BN -&gt; DP]`. If `False: [BN -&gt; DP -&gt;</span>
+<span class="sd">        LIN -&gt; ACT]`</span>
+<span class="sd">        If &#39;mlp_hidden_dims&#39; is not `None` and this parameter is `None`, it</span>
+<span class="sd">        will default to `True`.</span>
+
+<span class="sd">    Attributes</span>
+<span class="sd">    ----------</span>
+<span class="sd">    encoder: nn.Module</span>
+<span class="sd">        Sequence of FasFormer blocks.</span>
+<span class="sd">    mlp: nn.Module</span>
+<span class="sd">        MLP component in the model</span>
+
+<span class="sd">    Examples</span>
+<span class="sd">    --------</span>
+<span class="sd">    &gt;&gt;&gt; import torch</span>
+<span class="sd">    &gt;&gt;&gt; from pytorch_widedeep.models import TabFastFormer</span>
+<span class="sd">    &gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)</span>
+<span class="sd">    &gt;&gt;&gt; colnames = [&#39;a&#39;, &#39;b&#39;, &#39;c&#39;, &#39;d&#39;, &#39;e&#39;]</span>
+<span class="sd">    &gt;&gt;&gt; cat_embed_input = [(u,i) for u,i in zip(colnames[:4], [4]*4)]</span>
+<span class="sd">    &gt;&gt;&gt; continuous_cols = [&#39;e&#39;]</span>
+<span class="sd">    &gt;&gt;&gt; column_idx = {k:v for v,k in enumerate(colnames)}</span>
+<span class="sd">    &gt;&gt;&gt; model = TabFastFormer(column_idx=column_idx, cat_embed_input=cat_embed_input, continuous_cols=continuous_cols)</span>
+<span class="sd">    &gt;&gt;&gt; out = model(X_tab)</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+
+    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span>
+        <span class="n">column_idx</span><span class="p">:</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">],</span>
+        <span class="o">*</span><span class="p">,</span>
+        <span class="n">cat_embed_input</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="n">Tuple</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">]]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">cat_embed_dropout</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">use_cat_bias</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">cat_embed_activation</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">shared_embed</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">add_shared_embed</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">frac_shared_embed</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">continuous_cols</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">cont_norm_layer</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Literal</span><span class="p">[</span><span class="s2">&quot;batchnorm&quot;</span><span class="p">,</span> <span class="s2">&quot;layernorm&quot;</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">embed_continuous_method</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span>
+            <span class="n">Literal</span><span class="p">[</span><span class="s2">&quot;standard&quot;</span><span class="p">,</span> <span class="s2">&quot;piecewise&quot;</span><span class="p">,</span> <span class="s2">&quot;periodic&quot;</span><span class="p">]</span>
+        <span class="p">]</span> <span class="o">=</span> <span class="s2">&quot;standard&quot;</span><span class="p">,</span>
+        <span class="n">cont_embed_dropout</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">cont_embed_activation</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">quantization_setup</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="nb">float</span><span class="p">]]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">n_frequencies</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">sigma</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">share_last_layer</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">full_embed_dropout</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">input_dim</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">32</span><span class="p">,</span>
+        <span class="n">n_heads</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">8</span><span class="p">,</span>
+        <span class="n">use_bias</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
+        <span class="n">n_blocks</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">4</span><span class="p">,</span>
+        <span class="n">attn_dropout</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.1</span><span class="p">,</span>
+        <span class="n">ff_dropout</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.2</span><span class="p">,</span>
+        <span class="n">ff_factor</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">4</span><span class="p">,</span>
+        <span class="n">share_qv_weights</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
+        <span class="n">share_weights</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
+        <span class="n">transformer_activation</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;relu&quot;</span><span class="p">,</span>
+        <span class="n">mlp_hidden_dims</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">int</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">mlp_activation</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">mlp_dropout</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">mlp_batchnorm</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">mlp_batchnorm_last</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">mlp_linear_first</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+    <span class="p">):</span>
+        <span class="nb">super</span><span class="p">(</span><span class="n">TabFastFormer</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span>
+            <span class="n">column_idx</span><span class="o">=</span><span class="n">column_idx</span><span class="p">,</span>
+            <span class="n">cat_embed_input</span><span class="o">=</span><span class="n">cat_embed_input</span><span class="p">,</span>
+            <span class="n">cat_embed_dropout</span><span class="o">=</span><span class="n">cat_embed_dropout</span><span class="p">,</span>
+            <span class="n">use_cat_bias</span><span class="o">=</span><span class="n">use_cat_bias</span><span class="p">,</span>
+            <span class="n">cat_embed_activation</span><span class="o">=</span><span class="n">cat_embed_activation</span><span class="p">,</span>
+            <span class="n">shared_embed</span><span class="o">=</span><span class="n">shared_embed</span><span class="p">,</span>
+            <span class="n">add_shared_embed</span><span class="o">=</span><span class="n">add_shared_embed</span><span class="p">,</span>
+            <span class="n">frac_shared_embed</span><span class="o">=</span><span class="n">frac_shared_embed</span><span class="p">,</span>
+            <span class="n">continuous_cols</span><span class="o">=</span><span class="n">continuous_cols</span><span class="p">,</span>
+            <span class="n">cont_norm_layer</span><span class="o">=</span><span class="n">cont_norm_layer</span><span class="p">,</span>
+            <span class="n">embed_continuous_method</span><span class="o">=</span><span class="n">embed_continuous_method</span><span class="p">,</span>
+            <span class="n">cont_embed_dropout</span><span class="o">=</span><span class="n">cont_embed_dropout</span><span class="p">,</span>
+            <span class="n">cont_embed_activation</span><span class="o">=</span><span class="n">cont_embed_activation</span><span class="p">,</span>
+            <span class="n">input_dim</span><span class="o">=</span><span class="n">input_dim</span><span class="p">,</span>
+            <span class="n">quantization_setup</span><span class="o">=</span><span class="n">quantization_setup</span><span class="p">,</span>
+            <span class="n">n_frequencies</span><span class="o">=</span><span class="n">n_frequencies</span><span class="p">,</span>
+            <span class="n">sigma</span><span class="o">=</span><span class="n">sigma</span><span class="p">,</span>
+            <span class="n">share_last_layer</span><span class="o">=</span><span class="n">share_last_layer</span><span class="p">,</span>
+            <span class="n">full_embed_dropout</span><span class="o">=</span><span class="n">full_embed_dropout</span><span class="p">,</span>
+        <span class="p">)</span>
 
-    <span class="bp">self</span><span class="o">.</span><span class="n">n_heads</span> <span class="o">=</span> <span class="n">n_heads</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">use_bias</span> <span class="o">=</span> <span class="n">use_bias</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">n_blocks</span> <span class="o">=</span> <span class="n">n_blocks</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">attn_dropout</span> <span class="o">=</span> <span class="n">attn_dropout</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">ff_dropout</span> <span class="o">=</span> <span class="n">ff_dropout</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">ff_factor</span> <span class="o">=</span> <span class="n">ff_factor</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">share_qv_weights</span> <span class="o">=</span> <span class="n">share_qv_weights</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">share_weights</span> <span class="o">=</span> <span class="n">share_weights</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">transformer_activation</span> <span class="o">=</span> <span class="n">transformer_activation</span>
-
-    <span class="bp">self</span><span class="o">.</span><span class="n">mlp_hidden_dims</span> <span class="o">=</span> <span class="n">mlp_hidden_dims</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">mlp_activation</span> <span class="o">=</span> <span class="n">mlp_activation</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">mlp_dropout</span> <span class="o">=</span> <span class="n">mlp_dropout</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">mlp_batchnorm</span> <span class="o">=</span> <span class="n">mlp_batchnorm</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">mlp_batchnorm_last</span> <span class="o">=</span> <span class="n">mlp_batchnorm_last</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">mlp_linear_first</span> <span class="o">=</span> <span class="n">mlp_linear_first</span>
-
-    <span class="bp">self</span><span class="o">.</span><span class="n">with_cls_token</span> <span class="o">=</span> <span class="s2">&quot;cls_token&quot;</span> <span class="ow">in</span> <span class="n">column_idx</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">n_cat</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">cat_embed_input</span><span class="p">)</span> <span class="k">if</span> <span class="n">cat_embed_input</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="k">else</span> <span class="mi">0</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">n_cont</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">continuous_cols</span><span class="p">)</span> <span class="k">if</span> <span class="n">continuous_cols</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="k">else</span> <span class="mi">0</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">n_feats</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">n_cat</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">n_cont</span>
-
-    <span class="c1"># Embeddings are instantiated at the base model</span>
-    <span class="c1"># Transformer blocks</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">encoder</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Sequential</span><span class="p">()</span>
-    <span class="n">first_fastformer_block</span> <span class="o">=</span> <span class="n">FastFormerEncoder</span><span class="p">(</span>
-        <span class="n">input_dim</span><span class="p">,</span>
-        <span class="n">n_heads</span><span class="p">,</span>
-        <span class="n">use_bias</span><span class="p">,</span>
-        <span class="n">attn_dropout</span><span class="p">,</span>
-        <span class="n">ff_dropout</span><span class="p">,</span>
-        <span class="n">ff_factor</span><span class="p">,</span>
-        <span class="n">share_qv_weights</span><span class="p">,</span>
-        <span class="n">transformer_activation</span><span class="p">,</span>
-    <span class="p">)</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">encoder</span><span class="o">.</span><span class="n">add_module</span><span class="p">(</span><span class="s2">&quot;fastformer_block0&quot;</span><span class="p">,</span> <span class="n">first_fastformer_block</span><span class="p">)</span>
-    <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="n">n_blocks</span><span class="p">):</span>
-        <span class="k">if</span> <span class="n">share_weights</span><span class="p">:</span>
-            <span class="bp">self</span><span class="o">.</span><span class="n">encoder</span><span class="o">.</span><span class="n">add_module</span><span class="p">(</span>
-                <span class="s2">&quot;fastformer_block&quot;</span> <span class="o">+</span> <span class="nb">str</span><span class="p">(</span><span class="n">i</span><span class="p">),</span> <span class="n">first_fastformer_block</span>
-            <span class="p">)</span>
-        <span class="k">else</span><span class="p">:</span>
-            <span class="bp">self</span><span class="o">.</span><span class="n">encoder</span><span class="o">.</span><span class="n">add_module</span><span class="p">(</span>
-                <span class="s2">&quot;fastformer_block&quot;</span> <span class="o">+</span> <span class="nb">str</span><span class="p">(</span><span class="n">i</span><span class="p">),</span>
-                <span class="n">FastFormerEncoder</span><span class="p">(</span>
-                    <span class="n">input_dim</span><span class="p">,</span>
-                    <span class="n">n_heads</span><span class="p">,</span>
-                    <span class="n">use_bias</span><span class="p">,</span>
-                    <span class="n">attn_dropout</span><span class="p">,</span>
-                    <span class="n">ff_dropout</span><span class="p">,</span>
-                    <span class="n">ff_factor</span><span class="p">,</span>
-                    <span class="n">share_qv_weights</span><span class="p">,</span>
-                    <span class="n">transformer_activation</span><span class="p">,</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">n_heads</span> <span class="o">=</span> <span class="n">n_heads</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">use_bias</span> <span class="o">=</span> <span class="n">use_bias</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">n_blocks</span> <span class="o">=</span> <span class="n">n_blocks</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">attn_dropout</span> <span class="o">=</span> <span class="n">attn_dropout</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">ff_dropout</span> <span class="o">=</span> <span class="n">ff_dropout</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">ff_factor</span> <span class="o">=</span> <span class="n">ff_factor</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">share_qv_weights</span> <span class="o">=</span> <span class="n">share_qv_weights</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">share_weights</span> <span class="o">=</span> <span class="n">share_weights</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">transformer_activation</span> <span class="o">=</span> <span class="n">transformer_activation</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">mlp_hidden_dims</span> <span class="o">=</span> <span class="n">mlp_hidden_dims</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">mlp_activation</span> <span class="o">=</span> <span class="n">mlp_activation</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">mlp_dropout</span> <span class="o">=</span> <span class="n">mlp_dropout</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">mlp_batchnorm</span> <span class="o">=</span> <span class="n">mlp_batchnorm</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">mlp_batchnorm_last</span> <span class="o">=</span> <span class="n">mlp_batchnorm_last</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">mlp_linear_first</span> <span class="o">=</span> <span class="n">mlp_linear_first</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">with_cls_token</span> <span class="o">=</span> <span class="s2">&quot;cls_token&quot;</span> <span class="ow">in</span> <span class="n">column_idx</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">n_cat</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">cat_embed_input</span><span class="p">)</span> <span class="k">if</span> <span class="n">cat_embed_input</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="k">else</span> <span class="mi">0</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">n_cont</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">continuous_cols</span><span class="p">)</span> <span class="k">if</span> <span class="n">continuous_cols</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="k">else</span> <span class="mi">0</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">n_feats</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">n_cat</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">n_cont</span>
+
+        <span class="c1"># Embeddings are instantiated at the base model</span>
+        <span class="c1"># Transformer blocks</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">encoder</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Sequential</span><span class="p">()</span>
+        <span class="n">first_fastformer_block</span> <span class="o">=</span> <span class="n">FastFormerEncoder</span><span class="p">(</span>
+            <span class="n">input_dim</span><span class="p">,</span>
+            <span class="n">n_heads</span><span class="p">,</span>
+            <span class="n">use_bias</span><span class="p">,</span>
+            <span class="n">attn_dropout</span><span class="p">,</span>
+            <span class="n">ff_dropout</span><span class="p">,</span>
+            <span class="n">ff_factor</span><span class="p">,</span>
+            <span class="n">share_qv_weights</span><span class="p">,</span>
+            <span class="n">transformer_activation</span><span class="p">,</span>
+        <span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">encoder</span><span class="o">.</span><span class="n">add_module</span><span class="p">(</span><span class="s2">&quot;fastformer_block0&quot;</span><span class="p">,</span> <span class="n">first_fastformer_block</span><span class="p">)</span>
+        <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="n">n_blocks</span><span class="p">):</span>
+            <span class="k">if</span> <span class="n">share_weights</span><span class="p">:</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">encoder</span><span class="o">.</span><span class="n">add_module</span><span class="p">(</span>
+                    <span class="s2">&quot;fastformer_block&quot;</span> <span class="o">+</span> <span class="nb">str</span><span class="p">(</span><span class="n">i</span><span class="p">),</span> <span class="n">first_fastformer_block</span>
+                <span class="p">)</span>
+            <span class="k">else</span><span class="p">:</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">encoder</span><span class="o">.</span><span class="n">add_module</span><span class="p">(</span>
+                    <span class="s2">&quot;fastformer_block&quot;</span> <span class="o">+</span> <span class="nb">str</span><span class="p">(</span><span class="n">i</span><span class="p">),</span>
+                    <span class="n">FastFormerEncoder</span><span class="p">(</span>
+                        <span class="n">input_dim</span><span class="p">,</span>
+                        <span class="n">n_heads</span><span class="p">,</span>
+                        <span class="n">use_bias</span><span class="p">,</span>
+                        <span class="n">attn_dropout</span><span class="p">,</span>
+                        <span class="n">ff_dropout</span><span class="p">,</span>
+                        <span class="n">ff_factor</span><span class="p">,</span>
+                        <span class="n">share_qv_weights</span><span class="p">,</span>
+                        <span class="n">transformer_activation</span><span class="p">,</span>
+                    <span class="p">),</span>
+                <span class="p">)</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">mlp_first_hidden_dim</span> <span class="o">=</span> <span class="p">(</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">input_dim</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">with_cls_token</span> <span class="k">else</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">n_feats</span> <span class="o">*</span> <span class="bp">self</span><span class="o">.</span><span class="n">input_dim</span><span class="p">)</span>
+        <span class="p">)</span>
+
+        <span class="c1"># Mlp: adding an MLP on top of the Resnet blocks is optional and</span>
+        <span class="c1"># therefore all related params are optional</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_hidden_dims</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">mlp</span> <span class="o">=</span> <span class="n">MLP</span><span class="p">(</span>
+                <span class="n">d_hidden</span><span class="o">=</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">mlp_first_hidden_dim</span><span class="p">]</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_hidden_dims</span><span class="p">,</span>
+                <span class="n">activation</span><span class="o">=</span><span class="p">(</span>
+                    <span class="s2">&quot;relu&quot;</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_activation</span> <span class="ow">is</span> <span class="kc">None</span> <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_activation</span>
+                <span class="p">),</span>
+                <span class="n">dropout</span><span class="o">=</span><span class="mf">0.0</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_dropout</span> <span class="ow">is</span> <span class="kc">None</span> <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_dropout</span><span class="p">,</span>
+                <span class="n">batchnorm</span><span class="o">=</span><span class="kc">False</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_batchnorm</span> <span class="ow">is</span> <span class="kc">None</span> <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_batchnorm</span><span class="p">,</span>
+                <span class="n">batchnorm_last</span><span class="o">=</span><span class="p">(</span>
+                    <span class="kc">False</span>
+                    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_batchnorm_last</span> <span class="ow">is</span> <span class="kc">None</span>
+                    <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_batchnorm_last</span>
+                <span class="p">),</span>
+                <span class="n">linear_first</span><span class="o">=</span><span class="p">(</span>
+                    <span class="kc">False</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_linear_first</span> <span class="ow">is</span> <span class="kc">None</span> <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_linear_first</span>
                 <span class="p">),</span>
             <span class="p">)</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">mlp</span> <span class="o">=</span> <span class="kc">None</span>
 
-    <span class="bp">self</span><span class="o">.</span><span class="n">mlp_first_hidden_dim</span> <span class="o">=</span> <span class="p">(</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">input_dim</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">with_cls_token</span> <span class="k">else</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">n_feats</span> <span class="o">*</span> <span class="bp">self</span><span class="o">.</span><span class="n">input_dim</span><span class="p">)</span>
-    <span class="p">)</span>
-
-    <span class="c1"># Mlp: adding an MLP on top of the Resnet blocks is optional and</span>
-    <span class="c1"># therefore all related params are optional</span>
-    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_hidden_dims</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">mlp</span> <span class="o">=</span> <span class="n">MLP</span><span class="p">(</span>
-            <span class="n">d_hidden</span><span class="o">=</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">mlp_first_hidden_dim</span><span class="p">]</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_hidden_dims</span><span class="p">,</span>
-            <span class="n">activation</span><span class="o">=</span><span class="p">(</span>
-                <span class="s2">&quot;relu&quot;</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_activation</span> <span class="ow">is</span> <span class="kc">None</span> <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_activation</span>
-            <span class="p">),</span>
-            <span class="n">dropout</span><span class="o">=</span><span class="mf">0.0</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_dropout</span> <span class="ow">is</span> <span class="kc">None</span> <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_dropout</span><span class="p">,</span>
-            <span class="n">batchnorm</span><span class="o">=</span><span class="kc">False</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_batchnorm</span> <span class="ow">is</span> <span class="kc">None</span> <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_batchnorm</span><span class="p">,</span>
-            <span class="n">batchnorm_last</span><span class="o">=</span><span class="p">(</span>
-                <span class="kc">False</span>
-                <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_batchnorm_last</span> <span class="ow">is</span> <span class="kc">None</span>
-                <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_batchnorm_last</span>
-            <span class="p">),</span>
-            <span class="n">linear_first</span><span class="o">=</span><span class="p">(</span>
-                <span class="kc">False</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_linear_first</span> <span class="ow">is</span> <span class="kc">None</span> <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_linear_first</span>
-            <span class="p">),</span>
+    <span class="k">def</span> <span class="nf">forward</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">X</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tensor</span><span class="p">:</span>
+        <span class="n">x</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_embeddings</span><span class="p">(</span><span class="n">X</span><span class="p">)</span>
+        <span class="n">x</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">encoder</span><span class="p">(</span><span class="n">x</span><span class="p">)</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">with_cls_token</span><span class="p">:</span>
+            <span class="n">x</span> <span class="o">=</span> <span class="n">x</span><span class="p">[:,</span> <span class="mi">0</span><span class="p">,</span> <span class="p">:]</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="n">x</span> <span class="o">=</span> <span class="n">x</span><span class="o">.</span><span class="n">flatten</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="n">x</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp</span><span class="p">(</span><span class="n">x</span><span class="p">)</span>
+        <span class="k">return</span> <span class="n">x</span>
+
+    <span class="nd">@property</span>
+    <span class="k">def</span> <span class="nf">output_dim</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">int</span><span class="p">:</span>
+<span class="w">        </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;The output dimension of the model. This is a required property</span>
+<span class="sd">        neccesary to build the `WideDeep` class</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="k">return</span> <span class="p">(</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">mlp_hidden_dims</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span>
+            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_hidden_dims</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span>
+            <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">mlp_first_hidden_dim</span>
         <span class="p">)</span>
-    <span class="k">else</span><span class="p">:</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">mlp</span> <span class="o">=</span> <span class="kc">None</span>
+
+    <span class="nd">@property</span>
+    <span class="k">def</span> <span class="nf">attention_weights</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">List</span><span class="p">[</span><span class="n">Tuple</span><span class="p">[</span><span class="n">Tensor</span><span class="p">,</span> <span class="n">Tensor</span><span class="p">]]:</span>
+<span class="w">        </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;List with the attention weights. Each element of the list is a</span>
+<span class="sd">        tuple where the first and second elements are the $\alpha$</span>
+<span class="sd">        and $\beta$ attention weights in the paper.</span>
+
+<span class="sd">        The shape of the attention weights is $(N, H, F)$ where $N$ is the</span>
+<span class="sd">        batch size, $H$ is the number of attention heads and $F$ is the</span>
+<span class="sd">        number of features/columns in the dataset</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">share_weights</span><span class="p">:</span>
+            <span class="n">attention_weights</span> <span class="o">=</span> <span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">encoder</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">attn</span><span class="o">.</span><span class="n">attn_weight</span><span class="p">]</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="n">attention_weights</span> <span class="o">=</span> <span class="p">[</span><span class="n">blk</span><span class="o">.</span><span class="n">attn</span><span class="o">.</span><span class="n">attn_weights</span> <span class="k">for</span> <span class="n">blk</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">encoder</span><span class="p">]</span>
+        <span class="k">return</span> <span class="n">attention_weights</span>
 </code></pre></div></td></tr></table></div>
-                  </details>
+              </details>
 
 
 
@@ -10202,26 +16044,7 @@ <h2 id="pytorch_widedeep.models.text.rnns.basic_rnn.BasicRNN" class="doc doc-hea
 
 
 <a href="#pytorch_widedeep.models.text.rnns.basic_rnn.BasicRNN" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="nf">BasicRNN</span><span class="p">(</span>
-    <span class="n">vocab_size</span><span class="p">,</span>
-    <span class="n">embed_dim</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">embed_matrix</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">embed_trainable</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
-    <span class="n">rnn_type</span><span class="o">=</span><span class="s2">&quot;lstm&quot;</span><span class="p">,</span>
-    <span class="n">hidden_dim</span><span class="o">=</span><span class="mi">64</span><span class="p">,</span>
-    <span class="n">n_layers</span><span class="o">=</span><span class="mi">3</span><span class="p">,</span>
-    <span class="n">rnn_dropout</span><span class="o">=</span><span class="mf">0.0</span><span class="p">,</span>
-    <span class="n">bidirectional</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
-    <span class="n">use_hidden_state</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
-    <span class="n">padding_idx</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span>
-    <span class="n">head_hidden_dims</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">head_activation</span><span class="o">=</span><span class="s2">&quot;relu&quot;</span><span class="p">,</span>
-    <span class="n">head_dropout</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">head_batchnorm</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
-    <span class="n">head_batchnorm_last</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
-    <span class="n">head_linear_first</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
-<span class="p">)</span>
-</code></pre></div>
+
 
     <div class="doc doc-contents first">
             <p class="doc doc-class-bases">
@@ -10236,216 +16059,314 @@ <h2 id="pytorch_widedeep.models.text.rnns.basic_rnn.BasicRNN" class="doc doc-hea
 
 
 <p><span class="doc-section-title">Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>vocab_size</code></b>
-              (<code>int</code>)
-          –
-          <div class="doc-md-description">
-            <p>Number of words in the vocabulary</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>embed_dim</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[int]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Dimension of the word embeddings if non-pretained word vectors are
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+          <th>Default</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>vocab_size</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Number of words in the vocabulary</p>
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>embed_dim</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[int]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Dimension of the word embeddings if non-pretained word vectors are
 used</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>embed_matrix</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="numpy.ndarray">ndarray</span>]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Pretrained word embeddings</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>embed_trainable</code></b>
-              (<code>bool</code>, default:
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>embed_matrix</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="numpy.ndarray">ndarray</span>]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Pretrained word embeddings</p>
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>embed_trainable</code></td>
+            <td>
+                  <code>bool</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Boolean indicating if the pretrained embeddings are trainable</p>
+              </div>
+            </td>
+            <td>
                   <code>True</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Boolean indicating if the pretrained embeddings are trainable</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>rnn_type</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[lstm, gru]</code>, default:
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>rnn_type</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[lstm, gru]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>String indicating the type of RNN to use. One of <em>'lstm'</em> or <em>'gru'</em></p>
+              </div>
+            </td>
+            <td>
                   <code>&#39;lstm&#39;</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>String indicating the type of RNN to use. One of <em>'lstm'</em> or <em>'gru'</em></p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>hidden_dim</code></b>
-              (<code>int</code>, default:
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>hidden_dim</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Hidden dim of the RNN</p>
+              </div>
+            </td>
+            <td>
                   <code>64</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Hidden dim of the RNN</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>n_layers</code></b>
-              (<code>int</code>, default:
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>n_layers</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Number of recurrent layers</p>
+              </div>
+            </td>
+            <td>
                   <code>3</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Number of recurrent layers</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>rnn_dropout</code></b>
-              (<code>float</code>, default:
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>rnn_dropout</code></td>
+            <td>
+                  <code>float</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Dropout for each RNN layer except the last layer</p>
+              </div>
+            </td>
+            <td>
                   <code>0.0</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Dropout for each RNN layer except the last layer</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>bidirectional</code></b>
-              (<code>bool</code>, default:
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>bidirectional</code></td>
+            <td>
+                  <code>bool</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Boolean indicating whether the staked RNNs are bidirectional</p>
+              </div>
+            </td>
+            <td>
                   <code>False</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Boolean indicating whether the staked RNNs are bidirectional</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>use_hidden_state</code></b>
-              (<code>bool</code>, default:
-                  <code>True</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Boolean indicating whether to use the final hidden state or the RNN's
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>use_hidden_state</code></td>
+            <td>
+                  <code>bool</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Boolean indicating whether to use the final hidden state or the RNN's
 output as predicting features. Typically the former is used.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>padding_idx</code></b>
-              (<code>int</code>, default:
-                  <code>1</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>index of the padding token in the padded-tokenised sequences. The
+              </div>
+            </td>
+            <td>
+                  <code>True</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>padding_idx</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>index of the padding token in the padded-tokenised sequences. The
 <code>TextPreprocessor</code> class within this library uses fastai's tokenizer
 where the token index 0 is reserved for the <em>'unknown'</em> word token.
 Therefore, the default value is set to 1.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>head_hidden_dims</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[int]]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>List with the sizes of the dense layers in the head e.g: <em>[128, 64]</em></p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>head_activation</code></b>
-              (<code>str</code>, default:
-                  <code>&#39;relu&#39;</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Activation function for the dense layers in the head. Currently
+              </div>
+            </td>
+            <td>
+                  <code>1</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>head_hidden_dims</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[int]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>List with the sizes of the dense layers in the head e.g: <em>[128, 64]</em></p>
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>head_activation</code></td>
+            <td>
+                  <code>str</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Activation function for the dense layers in the head. Currently
 <em>'tanh'</em>, <em>'relu'</em>, <em>'leaky_relu'</em> and <em>'gelu'</em> are supported</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>head_dropout</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Dropout of the dense layers in the head</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>head_batchnorm</code></b>
-              (<code>bool</code>, default:
-                  <code>False</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Boolean indicating whether or not to include batch normalization in
+              </div>
+            </td>
+            <td>
+                  <code>&#39;relu&#39;</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>head_dropout</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Dropout of the dense layers in the head</p>
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>head_batchnorm</code></td>
+            <td>
+                  <code>bool</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Boolean indicating whether or not to include batch normalization in
 the dense layers that form the <em>'rnn_mlp'</em></p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>head_batchnorm_last</code></b>
-              (<code>bool</code>, default:
+              </div>
+            </td>
+            <td>
                   <code>False</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Boolean indicating whether or not to apply batch normalization to the
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>head_batchnorm_last</code></td>
+            <td>
+                  <code>bool</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Boolean indicating whether or not to apply batch normalization to the
 last of the dense layers in the head</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>head_linear_first</code></b>
-              (<code>bool</code>, default:
+              </div>
+            </td>
+            <td>
                   <code>False</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Boolean indicating whether the order of the operations in the dense
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>head_linear_first</code></td>
+            <td>
+                  <code>bool</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Boolean indicating whether the order of the operations in the dense
 layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt;
 LIN -&gt; ACT]</code></p>
-          </div>
-        </li>
-    </ul>
+              </div>
+            </td>
+            <td>
+                  <code>False</code>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
 
 <p><span class="doc-section-title">Attributes:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code><span title="pytorch_widedeep.models.text.rnns.basic_rnn.BasicRNN.word_embed">word_embed</span></code></b>
-              (<code><span title="torch.nn.Module">Module</span></code>)
-          –
-          <div class="doc-md-description">
-            <p>word embedding matrix</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code><span title="pytorch_widedeep.models.text.rnns.basic_rnn.BasicRNN.rnn">rnn</span></code></b>
-              (<code><span title="torch.nn.Module">Module</span></code>)
-          –
-          <div class="doc-md-description">
-            <p>Stack of RNNs</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code><span title="pytorch_widedeep.models.text.rnns.basic_rnn.BasicRNN.rnn_mlp">rnn_mlp</span></code></b>
-              (<code><span title="torch.nn.Module">Module</span></code>)
-          –
-          <div class="doc-md-description">
-            <p>Stack of dense layers on top of the RNN. This will only exists if
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code><span title="pytorch_widedeep.models.text.rnns.basic_rnn.BasicRNN.word_embed">word_embed</span></code></td>
+            <td>
+                  <code><span title="torch.nn.Module">Module</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>word embedding matrix</p>
+              </div>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code><span title="pytorch_widedeep.models.text.rnns.basic_rnn.BasicRNN.rnn">rnn</span></code></td>
+            <td>
+                  <code><span title="torch.nn.Module">Module</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Stack of RNNs</p>
+              </div>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code><span title="pytorch_widedeep.models.text.rnns.basic_rnn.BasicRNN.rnn_mlp">rnn_mlp</span></code></td>
+            <td>
+                  <code><span title="torch.nn.Module">Module</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Stack of dense layers on top of the RNN. This will only exists if
 <code>head_layers_dim</code> is not None</p>
-          </div>
-        </li>
-    </ul>
+              </div>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
 
 <p><span class="doc-section-title">Examples:</span></p>
@@ -10456,9 +16377,83 @@ <h2 id="pytorch_widedeep.models.text.rnns.basic_rnn.BasicRNN" class="doc doc-hea
 <span class="gp">&gt;&gt;&gt; </span><span class="n">out</span> <span class="o">=</span> <span class="n">model</span><span class="p">(</span><span class="n">X_text</span><span class="p">)</span>
 </code></pre></div>
 
-                  <details class="quote">
-                    <summary>Source code in <code>pytorch_widedeep/models/text/rnns/basic_rnn.py</code></summary>
-                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 86</span>
+              <details class="quote">
+                <summary>Source code in <code>pytorch_widedeep/models/text/rnns/basic_rnn.py</code></summary>
+                <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 12</span>
+<span class="normal"> 13</span>
+<span class="normal"> 14</span>
+<span class="normal"> 15</span>
+<span class="normal"> 16</span>
+<span class="normal"> 17</span>
+<span class="normal"> 18</span>
+<span class="normal"> 19</span>
+<span class="normal"> 20</span>
+<span class="normal"> 21</span>
+<span class="normal"> 22</span>
+<span class="normal"> 23</span>
+<span class="normal"> 24</span>
+<span class="normal"> 25</span>
+<span class="normal"> 26</span>
+<span class="normal"> 27</span>
+<span class="normal"> 28</span>
+<span class="normal"> 29</span>
+<span class="normal"> 30</span>
+<span class="normal"> 31</span>
+<span class="normal"> 32</span>
+<span class="normal"> 33</span>
+<span class="normal"> 34</span>
+<span class="normal"> 35</span>
+<span class="normal"> 36</span>
+<span class="normal"> 37</span>
+<span class="normal"> 38</span>
+<span class="normal"> 39</span>
+<span class="normal"> 40</span>
+<span class="normal"> 41</span>
+<span class="normal"> 42</span>
+<span class="normal"> 43</span>
+<span class="normal"> 44</span>
+<span class="normal"> 45</span>
+<span class="normal"> 46</span>
+<span class="normal"> 47</span>
+<span class="normal"> 48</span>
+<span class="normal"> 49</span>
+<span class="normal"> 50</span>
+<span class="normal"> 51</span>
+<span class="normal"> 52</span>
+<span class="normal"> 53</span>
+<span class="normal"> 54</span>
+<span class="normal"> 55</span>
+<span class="normal"> 56</span>
+<span class="normal"> 57</span>
+<span class="normal"> 58</span>
+<span class="normal"> 59</span>
+<span class="normal"> 60</span>
+<span class="normal"> 61</span>
+<span class="normal"> 62</span>
+<span class="normal"> 63</span>
+<span class="normal"> 64</span>
+<span class="normal"> 65</span>
+<span class="normal"> 66</span>
+<span class="normal"> 67</span>
+<span class="normal"> 68</span>
+<span class="normal"> 69</span>
+<span class="normal"> 70</span>
+<span class="normal"> 71</span>
+<span class="normal"> 72</span>
+<span class="normal"> 73</span>
+<span class="normal"> 74</span>
+<span class="normal"> 75</span>
+<span class="normal"> 76</span>
+<span class="normal"> 77</span>
+<span class="normal"> 78</span>
+<span class="normal"> 79</span>
+<span class="normal"> 80</span>
+<span class="normal"> 81</span>
+<span class="normal"> 82</span>
+<span class="normal"> 83</span>
+<span class="normal"> 84</span>
+<span class="normal"> 85</span>
+<span class="normal"> 86</span>
 <span class="normal"> 87</span>
 <span class="normal"> 88</span>
 <span class="normal"> 89</span>
@@ -10566,117 +16561,331 @@ <h2 id="pytorch_widedeep.models.text.rnns.basic_rnn.BasicRNN" class="doc doc-hea
 <span class="normal">191</span>
 <span class="normal">192</span>
 <span class="normal">193</span>
-<span class="normal">194</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
-    <span class="bp">self</span><span class="p">,</span>
-    <span class="n">vocab_size</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span>
-    <span class="n">embed_dim</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">embed_matrix</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">embed_trainable</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">,</span>
-    <span class="n">rnn_type</span><span class="p">:</span> <span class="n">Literal</span><span class="p">[</span><span class="s2">&quot;lstm&quot;</span><span class="p">,</span> <span class="s2">&quot;gru&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="s2">&quot;lstm&quot;</span><span class="p">,</span>
-    <span class="n">hidden_dim</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">64</span><span class="p">,</span>
-    <span class="n">n_layers</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">3</span><span class="p">,</span>
-    <span class="n">rnn_dropout</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.0</span><span class="p">,</span>
-    <span class="n">bidirectional</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
-    <span class="n">use_hidden_state</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">,</span>
-    <span class="n">padding_idx</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1</span><span class="p">,</span>
-    <span class="n">head_hidden_dims</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">int</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">head_activation</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;relu&quot;</span><span class="p">,</span>
-    <span class="n">head_dropout</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">head_batchnorm</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
-    <span class="n">head_batchnorm_last</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
-    <span class="n">head_linear_first</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
-<span class="p">):</span>
-    <span class="nb">super</span><span class="p">(</span><span class="n">BasicRNN</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
-
-    <span class="k">if</span> <span class="n">embed_dim</span> <span class="ow">is</span> <span class="kc">None</span> <span class="ow">and</span> <span class="n">embed_matrix</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
-        <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
-            <span class="s2">&quot;If no &#39;embed_matrix&#39; is passed, the embedding dimension must&quot;</span>
-            <span class="s2">&quot;be specified with &#39;embed_dim&#39;&quot;</span>
-        <span class="p">)</span>
+<span class="normal">194</span>
+<span class="normal">195</span>
+<span class="normal">196</span>
+<span class="normal">197</span>
+<span class="normal">198</span>
+<span class="normal">199</span>
+<span class="normal">200</span>
+<span class="normal">201</span>
+<span class="normal">202</span>
+<span class="normal">203</span>
+<span class="normal">204</span>
+<span class="normal">205</span>
+<span class="normal">206</span>
+<span class="normal">207</span>
+<span class="normal">208</span>
+<span class="normal">209</span>
+<span class="normal">210</span>
+<span class="normal">211</span>
+<span class="normal">212</span>
+<span class="normal">213</span>
+<span class="normal">214</span>
+<span class="normal">215</span>
+<span class="normal">216</span>
+<span class="normal">217</span>
+<span class="normal">218</span>
+<span class="normal">219</span>
+<span class="normal">220</span>
+<span class="normal">221</span>
+<span class="normal">222</span>
+<span class="normal">223</span>
+<span class="normal">224</span>
+<span class="normal">225</span>
+<span class="normal">226</span>
+<span class="normal">227</span>
+<span class="normal">228</span>
+<span class="normal">229</span>
+<span class="normal">230</span>
+<span class="normal">231</span>
+<span class="normal">232</span>
+<span class="normal">233</span>
+<span class="normal">234</span>
+<span class="normal">235</span>
+<span class="normal">236</span>
+<span class="normal">237</span>
+<span class="normal">238</span>
+<span class="normal">239</span>
+<span class="normal">240</span>
+<span class="normal">241</span>
+<span class="normal">242</span>
+<span class="normal">243</span>
+<span class="normal">244</span>
+<span class="normal">245</span>
+<span class="normal">246</span>
+<span class="normal">247</span>
+<span class="normal">248</span>
+<span class="normal">249</span>
+<span class="normal">250</span>
+<span class="normal">251</span>
+<span class="normal">252</span>
+<span class="normal">253</span>
+<span class="normal">254</span>
+<span class="normal">255</span>
+<span class="normal">256</span>
+<span class="normal">257</span>
+<span class="normal">258</span>
+<span class="normal">259</span>
+<span class="normal">260</span>
+<span class="normal">261</span>
+<span class="normal">262</span>
+<span class="normal">263</span>
+<span class="normal">264</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">class</span> <span class="nc">BasicRNN</span><span class="p">(</span><span class="n">BaseWDModelComponent</span><span class="p">):</span>
+<span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Standard text classifier/regressor comprised by a stack of RNNs</span>
+<span class="sd">    (LSTMs or GRUs) that can be used as the `deeptext` component of a Wide &amp;</span>
+<span class="sd">    Deep model or independently by itself.</span>
+
+<span class="sd">    In addition, there is the option to add a Fully Connected (FC) set of</span>
+<span class="sd">    dense layers on top of the stack of RNNs</span>
+
+<span class="sd">    Parameters</span>
+<span class="sd">    ----------</span>
+<span class="sd">    vocab_size: int</span>
+<span class="sd">        Number of words in the vocabulary</span>
+<span class="sd">    embed_dim: int, Optional, default = None</span>
+<span class="sd">        Dimension of the word embeddings if non-pretained word vectors are</span>
+<span class="sd">        used</span>
+<span class="sd">    embed_matrix: np.ndarray, Optional, default = None</span>
+<span class="sd">        Pretrained word embeddings</span>
+<span class="sd">    embed_trainable: bool, default = True</span>
+<span class="sd">        Boolean indicating if the pretrained embeddings are trainable</span>
+<span class="sd">    rnn_type: str, default = &#39;lstm&#39;</span>
+<span class="sd">        String indicating the type of RNN to use. One of _&#39;lstm&#39;_ or _&#39;gru&#39;_</span>
+<span class="sd">    hidden_dim: int, default = 64</span>
+<span class="sd">        Hidden dim of the RNN</span>
+<span class="sd">    n_layers: int, default = 3</span>
+<span class="sd">        Number of recurrent layers</span>
+<span class="sd">    rnn_dropout: float, default = 0.1</span>
+<span class="sd">        Dropout for each RNN layer except the last layer</span>
+<span class="sd">    bidirectional: bool, default = True</span>
+<span class="sd">        Boolean indicating whether the staked RNNs are bidirectional</span>
+<span class="sd">    use_hidden_state: str, default = True</span>
+<span class="sd">        Boolean indicating whether to use the final hidden state or the RNN&#39;s</span>
+<span class="sd">        output as predicting features. Typically the former is used.</span>
+<span class="sd">    padding_idx: int, default = 1</span>
+<span class="sd">        index of the padding token in the padded-tokenised sequences. The</span>
+<span class="sd">        `TextPreprocessor` class within this library uses fastai&#39;s tokenizer</span>
+<span class="sd">        where the token index 0 is reserved for the _&#39;unknown&#39;_ word token.</span>
+<span class="sd">        Therefore, the default value is set to 1.</span>
+<span class="sd">    head_hidden_dims: List, Optional, default = None</span>
+<span class="sd">        List with the sizes of the dense layers in the head e.g: _[128, 64]_</span>
+<span class="sd">    head_activation: str, default = &quot;relu&quot;</span>
+<span class="sd">        Activation function for the dense layers in the head. Currently</span>
+<span class="sd">        _&#39;tanh&#39;_, _&#39;relu&#39;_, _&#39;leaky_relu&#39;_ and _&#39;gelu&#39;_ are supported</span>
+<span class="sd">    head_dropout: float, Optional, default = None</span>
+<span class="sd">        Dropout of the dense layers in the head</span>
+<span class="sd">    head_batchnorm: bool, default = False</span>
+<span class="sd">        Boolean indicating whether or not to include batch normalization in</span>
+<span class="sd">        the dense layers that form the _&#39;rnn_mlp&#39;_</span>
+<span class="sd">    head_batchnorm_last: bool, default = False</span>
+<span class="sd">        Boolean indicating whether or not to apply batch normalization to the</span>
+<span class="sd">        last of the dense layers in the head</span>
+<span class="sd">    head_linear_first: bool, default = False</span>
+<span class="sd">        Boolean indicating whether the order of the operations in the dense</span>
+<span class="sd">        layer. If `True: [LIN -&gt; ACT -&gt; BN -&gt; DP]`. If `False: [BN -&gt; DP -&gt;</span>
+<span class="sd">        LIN -&gt; ACT]`</span>
+
+<span class="sd">    Attributes</span>
+<span class="sd">    ----------</span>
+<span class="sd">    word_embed: nn.Module</span>
+<span class="sd">        word embedding matrix</span>
+<span class="sd">    rnn: nn.Module</span>
+<span class="sd">        Stack of RNNs</span>
+<span class="sd">    rnn_mlp: nn.Module</span>
+<span class="sd">        Stack of dense layers on top of the RNN. This will only exists if</span>
+<span class="sd">        `head_layers_dim` is not None</span>
+
+<span class="sd">    Examples</span>
+<span class="sd">    --------</span>
+<span class="sd">    &gt;&gt;&gt; import torch</span>
+<span class="sd">    &gt;&gt;&gt; from pytorch_widedeep.models import BasicRNN</span>
+<span class="sd">    &gt;&gt;&gt; X_text = torch.cat((torch.zeros([5,1]), torch.empty(5, 4).random_(1,4)), axis=1)</span>
+<span class="sd">    &gt;&gt;&gt; model = BasicRNN(vocab_size=4, hidden_dim=4, n_layers=2, padding_idx=0, embed_dim=4)</span>
+<span class="sd">    &gt;&gt;&gt; out = model(X_text)</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+
+    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span>
+        <span class="n">vocab_size</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span>
+        <span class="n">embed_dim</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">embed_matrix</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">embed_trainable</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">,</span>
+        <span class="n">rnn_type</span><span class="p">:</span> <span class="n">Literal</span><span class="p">[</span><span class="s2">&quot;lstm&quot;</span><span class="p">,</span> <span class="s2">&quot;gru&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="s2">&quot;lstm&quot;</span><span class="p">,</span>
+        <span class="n">hidden_dim</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">64</span><span class="p">,</span>
+        <span class="n">n_layers</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">3</span><span class="p">,</span>
+        <span class="n">rnn_dropout</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.0</span><span class="p">,</span>
+        <span class="n">bidirectional</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
+        <span class="n">use_hidden_state</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">,</span>
+        <span class="n">padding_idx</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1</span><span class="p">,</span>
+        <span class="n">head_hidden_dims</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">int</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">head_activation</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;relu&quot;</span><span class="p">,</span>
+        <span class="n">head_dropout</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">head_batchnorm</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
+        <span class="n">head_batchnorm_last</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
+        <span class="n">head_linear_first</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
+    <span class="p">):</span>
+        <span class="nb">super</span><span class="p">(</span><span class="n">BasicRNN</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
 
-    <span class="k">if</span> <span class="n">rnn_type</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="ow">not</span> <span class="ow">in</span> <span class="p">[</span><span class="s2">&quot;lstm&quot;</span><span class="p">,</span> <span class="s2">&quot;gru&quot;</span><span class="p">]:</span>
-        <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
-            <span class="sa">f</span><span class="s2">&quot;&#39;rnn_type&#39; must be &#39;lstm&#39; or &#39;gru&#39;, got </span><span class="si">{</span><span class="n">rnn_type</span><span class="si">}</span><span class="s2"> instead&quot;</span>
-        <span class="p">)</span>
+        <span class="k">if</span> <span class="n">embed_dim</span> <span class="ow">is</span> <span class="kc">None</span> <span class="ow">and</span> <span class="n">embed_matrix</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
+                <span class="s2">&quot;If no &#39;embed_matrix&#39; is passed, the embedding dimension must&quot;</span>
+                <span class="s2">&quot;be specified with &#39;embed_dim&#39;&quot;</span>
+            <span class="p">)</span>
 
-    <span class="k">if</span> <span class="p">(</span>
-        <span class="n">embed_dim</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span>
-        <span class="ow">and</span> <span class="n">embed_matrix</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span>
-        <span class="ow">and</span> <span class="ow">not</span> <span class="n">embed_dim</span> <span class="o">==</span> <span class="n">embed_matrix</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span>
-    <span class="p">):</span>
-        <span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span>
-            <span class="s2">&quot;the input embedding dimension </span><span class="si">{}</span><span class="s2"> and the dimension of the &quot;</span>
-            <span class="s2">&quot;pretrained embeddings </span><span class="si">{}</span><span class="s2"> do not match. The pretrained embeddings &quot;</span>
-            <span class="s2">&quot;dimension (</span><span class="si">{}</span><span class="s2">) will be used&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
-                <span class="n">embed_dim</span><span class="p">,</span> <span class="n">embed_matrix</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">1</span><span class="p">],</span> <span class="n">embed_matrix</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span>
-            <span class="p">),</span>
-            <span class="ne">UserWarning</span><span class="p">,</span>
-        <span class="p">)</span>
+        <span class="k">if</span> <span class="n">rnn_type</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="ow">not</span> <span class="ow">in</span> <span class="p">[</span><span class="s2">&quot;lstm&quot;</span><span class="p">,</span> <span class="s2">&quot;gru&quot;</span><span class="p">]:</span>
+            <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
+                <span class="sa">f</span><span class="s2">&quot;&#39;rnn_type&#39; must be &#39;lstm&#39; or &#39;gru&#39;, got </span><span class="si">{</span><span class="n">rnn_type</span><span class="si">}</span><span class="s2"> instead&quot;</span>
+            <span class="p">)</span>
 
-    <span class="bp">self</span><span class="o">.</span><span class="n">vocab_size</span> <span class="o">=</span> <span class="n">vocab_size</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">embed_trainable</span> <span class="o">=</span> <span class="n">embed_trainable</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">embed_dim</span> <span class="o">=</span> <span class="n">embed_dim</span>
-
-    <span class="bp">self</span><span class="o">.</span><span class="n">rnn_type</span> <span class="o">=</span> <span class="n">rnn_type</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">hidden_dim</span> <span class="o">=</span> <span class="n">hidden_dim</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">n_layers</span> <span class="o">=</span> <span class="n">n_layers</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">rnn_dropout</span> <span class="o">=</span> <span class="n">rnn_dropout</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">bidirectional</span> <span class="o">=</span> <span class="n">bidirectional</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">use_hidden_state</span> <span class="o">=</span> <span class="n">use_hidden_state</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">padding_idx</span> <span class="o">=</span> <span class="n">padding_idx</span>
-
-    <span class="bp">self</span><span class="o">.</span><span class="n">head_hidden_dims</span> <span class="o">=</span> <span class="n">head_hidden_dims</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">head_activation</span> <span class="o">=</span> <span class="n">head_activation</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">head_dropout</span> <span class="o">=</span> <span class="n">head_dropout</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">head_batchnorm</span> <span class="o">=</span> <span class="n">head_batchnorm</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">head_batchnorm_last</span> <span class="o">=</span> <span class="n">head_batchnorm_last</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">head_linear_first</span> <span class="o">=</span> <span class="n">head_linear_first</span>
-
-    <span class="c1"># Embeddings</span>
-    <span class="k">if</span> <span class="n">embed_matrix</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">word_embed</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">embed_dim</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_set_embeddings</span><span class="p">(</span><span class="n">embed_matrix</span><span class="p">)</span>
-    <span class="k">else</span><span class="p">:</span>
-        <span class="k">assert</span> <span class="bp">self</span><span class="o">.</span><span class="n">embed_dim</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">word_embed</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Embedding</span><span class="p">(</span>
-            <span class="bp">self</span><span class="o">.</span><span class="n">vocab_size</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">embed_dim</span><span class="p">,</span> <span class="n">padding_idx</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">padding_idx</span>
-        <span class="p">)</span>
+        <span class="k">if</span> <span class="p">(</span>
+            <span class="n">embed_dim</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span>
+            <span class="ow">and</span> <span class="n">embed_matrix</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span>
+            <span class="ow">and</span> <span class="ow">not</span> <span class="n">embed_dim</span> <span class="o">==</span> <span class="n">embed_matrix</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span>
+        <span class="p">):</span>
+            <span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span>
+                <span class="s2">&quot;the input embedding dimension </span><span class="si">{}</span><span class="s2"> and the dimension of the &quot;</span>
+                <span class="s2">&quot;pretrained embeddings </span><span class="si">{}</span><span class="s2"> do not match. The pretrained embeddings &quot;</span>
+                <span class="s2">&quot;dimension (</span><span class="si">{}</span><span class="s2">) will be used&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
+                    <span class="n">embed_dim</span><span class="p">,</span> <span class="n">embed_matrix</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">1</span><span class="p">],</span> <span class="n">embed_matrix</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span>
+                <span class="p">),</span>
+                <span class="ne">UserWarning</span><span class="p">,</span>
+            <span class="p">)</span>
 
-    <span class="c1"># RNN</span>
-    <span class="n">rnn_params</span> <span class="o">=</span> <span class="p">{</span>
-        <span class="s2">&quot;input_size&quot;</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">embed_dim</span><span class="p">,</span>
-        <span class="s2">&quot;hidden_size&quot;</span><span class="p">:</span> <span class="n">hidden_dim</span><span class="p">,</span>
-        <span class="s2">&quot;num_layers&quot;</span><span class="p">:</span> <span class="n">n_layers</span><span class="p">,</span>
-        <span class="s2">&quot;bidirectional&quot;</span><span class="p">:</span> <span class="n">bidirectional</span><span class="p">,</span>
-        <span class="s2">&quot;dropout&quot;</span><span class="p">:</span> <span class="n">rnn_dropout</span><span class="p">,</span>
-        <span class="s2">&quot;batch_first&quot;</span><span class="p">:</span> <span class="kc">True</span><span class="p">,</span>
-    <span class="p">}</span>
-    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">rnn_type</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="o">==</span> <span class="s2">&quot;lstm&quot;</span><span class="p">:</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">rnn</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="n">nn</span><span class="o">.</span><span class="n">LSTM</span><span class="p">,</span> <span class="n">nn</span><span class="o">.</span><span class="n">GRU</span><span class="p">]</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">LSTM</span><span class="p">(</span><span class="o">**</span><span class="n">rnn_params</span><span class="p">)</span>
-    <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">rnn_type</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="o">==</span> <span class="s2">&quot;gru&quot;</span><span class="p">:</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">rnn</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">GRU</span><span class="p">(</span><span class="o">**</span><span class="n">rnn_params</span><span class="p">)</span>
-    <span class="k">else</span><span class="p">:</span>
-        <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
-            <span class="sa">f</span><span class="s2">&quot;&#39;rnn_type&#39; must be &#39;lstm&#39; or &#39;gru&#39;, got </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">rnn_type</span><span class="si">}</span><span class="s2"> instead&quot;</span>
-        <span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">vocab_size</span> <span class="o">=</span> <span class="n">vocab_size</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">embed_trainable</span> <span class="o">=</span> <span class="n">embed_trainable</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">embed_dim</span> <span class="o">=</span> <span class="n">embed_dim</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">rnn_type</span> <span class="o">=</span> <span class="n">rnn_type</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">hidden_dim</span> <span class="o">=</span> <span class="n">hidden_dim</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">n_layers</span> <span class="o">=</span> <span class="n">n_layers</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">rnn_dropout</span> <span class="o">=</span> <span class="n">rnn_dropout</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">bidirectional</span> <span class="o">=</span> <span class="n">bidirectional</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">use_hidden_state</span> <span class="o">=</span> <span class="n">use_hidden_state</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">padding_idx</span> <span class="o">=</span> <span class="n">padding_idx</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">head_hidden_dims</span> <span class="o">=</span> <span class="n">head_hidden_dims</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">head_activation</span> <span class="o">=</span> <span class="n">head_activation</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">head_dropout</span> <span class="o">=</span> <span class="n">head_dropout</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">head_batchnorm</span> <span class="o">=</span> <span class="n">head_batchnorm</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">head_batchnorm_last</span> <span class="o">=</span> <span class="n">head_batchnorm_last</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">head_linear_first</span> <span class="o">=</span> <span class="n">head_linear_first</span>
+
+        <span class="c1"># Embeddings</span>
+        <span class="k">if</span> <span class="n">embed_matrix</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">word_embed</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">embed_dim</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_set_embeddings</span><span class="p">(</span><span class="n">embed_matrix</span><span class="p">)</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="k">assert</span> <span class="bp">self</span><span class="o">.</span><span class="n">embed_dim</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">word_embed</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Embedding</span><span class="p">(</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">vocab_size</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">embed_dim</span><span class="p">,</span> <span class="n">padding_idx</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">padding_idx</span>
+            <span class="p">)</span>
+
+        <span class="c1"># RNN</span>
+        <span class="n">rnn_params</span> <span class="o">=</span> <span class="p">{</span>
+            <span class="s2">&quot;input_size&quot;</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">embed_dim</span><span class="p">,</span>
+            <span class="s2">&quot;hidden_size&quot;</span><span class="p">:</span> <span class="n">hidden_dim</span><span class="p">,</span>
+            <span class="s2">&quot;num_layers&quot;</span><span class="p">:</span> <span class="n">n_layers</span><span class="p">,</span>
+            <span class="s2">&quot;bidirectional&quot;</span><span class="p">:</span> <span class="n">bidirectional</span><span class="p">,</span>
+            <span class="s2">&quot;dropout&quot;</span><span class="p">:</span> <span class="n">rnn_dropout</span><span class="p">,</span>
+            <span class="s2">&quot;batch_first&quot;</span><span class="p">:</span> <span class="kc">True</span><span class="p">,</span>
+        <span class="p">}</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">rnn_type</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="o">==</span> <span class="s2">&quot;lstm&quot;</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">rnn</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="n">nn</span><span class="o">.</span><span class="n">LSTM</span><span class="p">,</span> <span class="n">nn</span><span class="o">.</span><span class="n">GRU</span><span class="p">]</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">LSTM</span><span class="p">(</span><span class="o">**</span><span class="n">rnn_params</span><span class="p">)</span>
+        <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">rnn_type</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="o">==</span> <span class="s2">&quot;gru&quot;</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">rnn</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">GRU</span><span class="p">(</span><span class="o">**</span><span class="n">rnn_params</span><span class="p">)</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
+                <span class="sa">f</span><span class="s2">&quot;&#39;rnn_type&#39; must be &#39;lstm&#39; or &#39;gru&#39;, got </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">rnn_type</span><span class="si">}</span><span class="s2"> instead&quot;</span>
+            <span class="p">)</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">rnn_output_dim</span> <span class="o">=</span> <span class="n">hidden_dim</span> <span class="o">*</span> <span class="mi">2</span> <span class="k">if</span> <span class="n">bidirectional</span> <span class="k">else</span> <span class="n">hidden_dim</span>
+
+        <span class="c1"># FC-Head (Mlp)</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">head_hidden_dims</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="n">head_hidden_dims</span> <span class="o">=</span> <span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">rnn_output_dim</span><span class="p">]</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">head_hidden_dims</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">rnn_mlp</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="n">MLP</span><span class="p">,</span> <span class="n">nn</span><span class="o">.</span><span class="n">Identity</span><span class="p">]</span> <span class="o">=</span> <span class="n">MLP</span><span class="p">(</span>
+                <span class="n">head_hidden_dims</span><span class="p">,</span>
+                <span class="n">head_activation</span><span class="p">,</span>
+                <span class="n">head_dropout</span><span class="p">,</span>
+                <span class="n">head_batchnorm</span><span class="p">,</span>
+                <span class="n">head_batchnorm_last</span><span class="p">,</span>
+                <span class="n">head_linear_first</span><span class="p">,</span>
+            <span class="p">)</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="c1"># simple hack to add readability in the forward pass</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">rnn_mlp</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Identity</span><span class="p">()</span>
 
-    <span class="bp">self</span><span class="o">.</span><span class="n">rnn_output_dim</span> <span class="o">=</span> <span class="n">hidden_dim</span> <span class="o">*</span> <span class="mi">2</span> <span class="k">if</span> <span class="n">bidirectional</span> <span class="k">else</span> <span class="n">hidden_dim</span>
+    <span class="k">def</span> <span class="nf">forward</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">X</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tensor</span><span class="p">:</span>
+        <span class="n">embed</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">word_embed</span><span class="p">(</span><span class="n">X</span><span class="o">.</span><span class="n">long</span><span class="p">())</span>
 
-    <span class="c1"># FC-Head (Mlp)</span>
-    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">head_hidden_dims</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
-        <span class="n">head_hidden_dims</span> <span class="o">=</span> <span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">rnn_output_dim</span><span class="p">]</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">head_hidden_dims</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">rnn_mlp</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="n">MLP</span><span class="p">,</span> <span class="n">nn</span><span class="o">.</span><span class="n">Identity</span><span class="p">]</span> <span class="o">=</span> <span class="n">MLP</span><span class="p">(</span>
-            <span class="n">head_hidden_dims</span><span class="p">,</span>
-            <span class="n">head_activation</span><span class="p">,</span>
-            <span class="n">head_dropout</span><span class="p">,</span>
-            <span class="n">head_batchnorm</span><span class="p">,</span>
-            <span class="n">head_batchnorm_last</span><span class="p">,</span>
-            <span class="n">head_linear_first</span><span class="p">,</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">rnn_type</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="o">==</span> <span class="s2">&quot;lstm&quot;</span><span class="p">:</span>
+            <span class="n">o</span><span class="p">,</span> <span class="p">(</span><span class="n">h</span><span class="p">,</span> <span class="n">c</span><span class="p">)</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">rnn</span><span class="p">(</span><span class="n">embed</span><span class="p">)</span>
+        <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">rnn_type</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="o">==</span> <span class="s2">&quot;gru&quot;</span><span class="p">:</span>
+            <span class="n">o</span><span class="p">,</span> <span class="n">h</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">rnn</span><span class="p">(</span><span class="n">embed</span><span class="p">)</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
+                <span class="sa">f</span><span class="s2">&quot;&#39;rnn_type&#39; must be &#39;lstm&#39; or &#39;gru&#39;, got </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">rnn_type</span><span class="si">}</span><span class="s2"> instead&quot;</span>
+            <span class="p">)</span>
+
+        <span class="n">processed_outputs</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_process_rnn_outputs</span><span class="p">(</span><span class="n">o</span><span class="p">,</span> <span class="n">h</span><span class="p">)</span>
+
+        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">rnn_mlp</span><span class="p">(</span><span class="n">processed_outputs</span><span class="p">)</span>
+
+    <span class="nd">@property</span>
+    <span class="k">def</span> <span class="nf">output_dim</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">int</span><span class="p">:</span>
+<span class="w">        </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;The output dimension of the model. This is a required property</span>
+<span class="sd">        neccesary to build the `WideDeep` class</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="k">return</span> <span class="p">(</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">head_hidden_dims</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span>
+            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">head_hidden_dims</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span>
+            <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">rnn_output_dim</span>
         <span class="p">)</span>
-    <span class="k">else</span><span class="p">:</span>
-        <span class="c1"># simple hack to add readability in the forward pass</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">rnn_mlp</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Identity</span><span class="p">()</span>
+
+    <span class="k">def</span> <span class="nf">_set_embeddings</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span> <span class="n">embed_matrix</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span>
+    <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tuple</span><span class="p">[</span><span class="n">nn</span><span class="o">.</span><span class="n">Module</span><span class="p">,</span> <span class="nb">int</span><span class="p">]:</span>
+        <span class="k">if</span> <span class="n">embed_matrix</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="k">assert</span> <span class="p">(</span>
+                <span class="n">embed_matrix</span><span class="o">.</span><span class="n">dtype</span> <span class="o">==</span> <span class="s2">&quot;float32&quot;</span>
+            <span class="p">),</span> <span class="s2">&quot;&#39;embed_matrix&#39; must be of dtype &#39;float32&#39;, got dtype &#39;</span><span class="si">{}</span><span class="s2">&#39;&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
+                <span class="nb">str</span><span class="p">(</span><span class="n">embed_matrix</span><span class="o">.</span><span class="n">dtype</span><span class="p">)</span>
+            <span class="p">)</span>
+            <span class="n">word_embed</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Embedding</span><span class="p">(</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">vocab_size</span><span class="p">,</span> <span class="n">embed_matrix</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">1</span><span class="p">],</span> <span class="n">padding_idx</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">padding_idx</span>
+            <span class="p">)</span>
+            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">embed_trainable</span><span class="p">:</span>
+                <span class="n">word_embed</span><span class="o">.</span><span class="n">weight</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Parameter</span><span class="p">(</span>
+                    <span class="n">torch</span><span class="o">.</span><span class="n">tensor</span><span class="p">(</span><span class="n">embed_matrix</span><span class="p">),</span> <span class="n">requires_grad</span><span class="o">=</span><span class="kc">True</span>
+                <span class="p">)</span>
+            <span class="k">else</span><span class="p">:</span>
+                <span class="n">word_embed</span><span class="o">.</span><span class="n">weight</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Parameter</span><span class="p">(</span>
+                    <span class="n">torch</span><span class="o">.</span><span class="n">tensor</span><span class="p">(</span><span class="n">embed_matrix</span><span class="p">),</span> <span class="n">requires_grad</span><span class="o">=</span><span class="kc">False</span>
+                <span class="p">)</span>
+            <span class="n">embed_dim</span> <span class="o">=</span> <span class="n">embed_matrix</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="k">assert</span> <span class="bp">self</span><span class="o">.</span><span class="n">embed_dim</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span>
+            <span class="n">word_embed</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Embedding</span><span class="p">(</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">vocab_size</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">embed_dim</span><span class="p">,</span> <span class="n">padding_idx</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">padding_idx</span>
+            <span class="p">)</span>
+            <span class="n">embed_dim</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">embed_dim</span>
+
+        <span class="k">return</span> <span class="n">word_embed</span><span class="p">,</span> <span class="n">embed_dim</span>
+
+    <span class="k">def</span> <span class="nf">_process_rnn_outputs</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">output</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">,</span> <span class="n">hidden</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tensor</span><span class="p">:</span>
+        <span class="n">output</span> <span class="o">=</span> <span class="n">output</span><span class="o">.</span><span class="n">permute</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="mi">2</span><span class="p">)</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">bidirectional</span><span class="p">:</span>
+            <span class="n">processed_outputs</span> <span class="o">=</span> <span class="p">(</span>
+                <span class="n">torch</span><span class="o">.</span><span class="n">cat</span><span class="p">((</span><span class="n">hidden</span><span class="p">[</span><span class="o">-</span><span class="mi">2</span><span class="p">],</span> <span class="n">hidden</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">]),</span> <span class="n">dim</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span>
+                <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">use_hidden_state</span>
+                <span class="k">else</span> <span class="n">output</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span>
+            <span class="p">)</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="n">processed_outputs</span> <span class="o">=</span> <span class="n">hidden</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">use_hidden_state</span> <span class="k">else</span> <span class="n">output</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span>
+
+        <span class="k">return</span> <span class="n">processed_outputs</span>
 </code></pre></div></td></tr></table></div>
-                  </details>
+              </details>
 
 
 
@@ -10731,28 +16940,7 @@ <h2 id="pytorch_widedeep.models.text.rnns.attentive_rnn.AttentiveRNN" class="doc
 
 
 <a href="#pytorch_widedeep.models.text.rnns.attentive_rnn.AttentiveRNN" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="nf">AttentiveRNN</span><span class="p">(</span>
-    <span class="n">vocab_size</span><span class="p">,</span>
-    <span class="n">embed_dim</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">embed_matrix</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">embed_trainable</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
-    <span class="n">rnn_type</span><span class="o">=</span><span class="s2">&quot;lstm&quot;</span><span class="p">,</span>
-    <span class="n">hidden_dim</span><span class="o">=</span><span class="mi">64</span><span class="p">,</span>
-    <span class="n">n_layers</span><span class="o">=</span><span class="mi">3</span><span class="p">,</span>
-    <span class="n">rnn_dropout</span><span class="o">=</span><span class="mf">0.1</span><span class="p">,</span>
-    <span class="n">bidirectional</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
-    <span class="n">use_hidden_state</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
-    <span class="n">padding_idx</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span>
-    <span class="n">attn_concatenate</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
-    <span class="n">attn_dropout</span><span class="o">=</span><span class="mf">0.1</span><span class="p">,</span>
-    <span class="n">head_hidden_dims</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">head_activation</span><span class="o">=</span><span class="s2">&quot;relu&quot;</span><span class="p">,</span>
-    <span class="n">head_dropout</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">head_batchnorm</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
-    <span class="n">head_batchnorm_last</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
-    <span class="n">head_linear_first</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
-<span class="p">)</span>
-</code></pre></div>
+
 
     <div class="doc doc-contents first">
             <p class="doc doc-class-bases">
@@ -10768,238 +16956,344 @@ <h2 id="pytorch_widedeep.models.text.rnns.attentive_rnn.AttentiveRNN" class="doc
 
 
 <p><span class="doc-section-title">Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>vocab_size</code></b>
-              (<code>int</code>)
-          –
-          <div class="doc-md-description">
-            <p>Number of words in the vocabulary</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>embed_dim</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[int]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Dimension of the word embeddings if non-pretained word vectors are
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+          <th>Default</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>vocab_size</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Number of words in the vocabulary</p>
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>embed_dim</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[int]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Dimension of the word embeddings if non-pretained word vectors are
 used</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>embed_matrix</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="numpy.ndarray">ndarray</span>]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Pretrained word embeddings</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>embed_trainable</code></b>
-              (<code>bool</code>, default:
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>embed_matrix</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="numpy.ndarray">ndarray</span>]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Pretrained word embeddings</p>
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>embed_trainable</code></td>
+            <td>
+                  <code>bool</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Boolean indicating if the pretrained embeddings are trainable</p>
+              </div>
+            </td>
+            <td>
                   <code>True</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Boolean indicating if the pretrained embeddings are trainable</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>rnn_type</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[lstm, gru]</code>, default:
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>rnn_type</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[lstm, gru]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>String indicating the type of RNN to use. One of <em>'lstm'</em> or <em>'gru'</em></p>
+              </div>
+            </td>
+            <td>
                   <code>&#39;lstm&#39;</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>String indicating the type of RNN to use. One of <em>'lstm'</em> or <em>'gru'</em></p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>hidden_dim</code></b>
-              (<code>int</code>, default:
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>hidden_dim</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Hidden dim of the RNN</p>
+              </div>
+            </td>
+            <td>
                   <code>64</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Hidden dim of the RNN</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>n_layers</code></b>
-              (<code>int</code>, default:
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>n_layers</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Number of recurrent layers</p>
+              </div>
+            </td>
+            <td>
                   <code>3</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Number of recurrent layers</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>rnn_dropout</code></b>
-              (<code>float</code>, default:
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>rnn_dropout</code></td>
+            <td>
+                  <code>float</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Dropout for each RNN layer except the last layer</p>
+              </div>
+            </td>
+            <td>
                   <code>0.1</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Dropout for each RNN layer except the last layer</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>bidirectional</code></b>
-              (<code>bool</code>, default:
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>bidirectional</code></td>
+            <td>
+                  <code>bool</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Boolean indicating whether the staked RNNs are bidirectional</p>
+              </div>
+            </td>
+            <td>
                   <code>False</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Boolean indicating whether the staked RNNs are bidirectional</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>use_hidden_state</code></b>
-              (<code>bool</code>, default:
-                  <code>True</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Boolean indicating whether to use the final hidden state or the RNN's
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>use_hidden_state</code></td>
+            <td>
+                  <code>bool</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Boolean indicating whether to use the final hidden state or the RNN's
 output as predicting features. Typically the former is used.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>padding_idx</code></b>
-              (<code>int</code>, default:
-                  <code>1</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>index of the padding token in the padded-tokenised sequences. The
+              </div>
+            </td>
+            <td>
+                  <code>True</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>padding_idx</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>index of the padding token in the padded-tokenised sequences. The
 <code>TextPreprocessor</code> class within this library uses fastai's
 tokenizer where the token index 0 is reserved for the <em>'unknown'</em>
 word token. Therefore, the default value is set to 1.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>attn_concatenate</code></b>
-              (<code>bool</code>, default:
-                  <code>True</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Boolean indicating if the input to the attention mechanism will be the
+              </div>
+            </td>
+            <td>
+                  <code>1</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>attn_concatenate</code></td>
+            <td>
+                  <code>bool</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Boolean indicating if the input to the attention mechanism will be the
 output of the RNN or the output of the RNN concatenated with the last
 hidden state.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>attn_dropout</code></b>
-              (<code>float</code>, default:
+              </div>
+            </td>
+            <td>
+                  <code>True</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>attn_dropout</code></td>
+            <td>
+                  <code>float</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Internal dropout for the attention mechanism</p>
+              </div>
+            </td>
+            <td>
                   <code>0.1</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Internal dropout for the attention mechanism</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>head_hidden_dims</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[int]]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>List with the sizes of the dense layers in the head e.g: <em>[128, 64]</em></p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>head_activation</code></b>
-              (<code>str</code>, default:
-                  <code>&#39;relu&#39;</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Activation function for the dense layers in the head. Currently
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>head_hidden_dims</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[int]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>List with the sizes of the dense layers in the head e.g: <em>[128, 64]</em></p>
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>head_activation</code></td>
+            <td>
+                  <code>str</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Activation function for the dense layers in the head. Currently
 <em>'tanh'</em>, <em>'relu'</em>, <em>'leaky_relu'</em> and <em>'gelu'</em> are supported</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>head_dropout</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Dropout of the dense layers in the head</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>head_batchnorm</code></b>
-              (<code>bool</code>, default:
-                  <code>False</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Boolean indicating whether or not to include batch normalization in
+              </div>
+            </td>
+            <td>
+                  <code>&#39;relu&#39;</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>head_dropout</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Dropout of the dense layers in the head</p>
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>head_batchnorm</code></td>
+            <td>
+                  <code>bool</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Boolean indicating whether or not to include batch normalization in
 the dense layers that form the <em>'rnn_mlp'</em></p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>head_batchnorm_last</code></b>
-              (<code>bool</code>, default:
+              </div>
+            </td>
+            <td>
                   <code>False</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Boolean indicating whether or not to apply batch normalization to the
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>head_batchnorm_last</code></td>
+            <td>
+                  <code>bool</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Boolean indicating whether or not to apply batch normalization to the
 last of the dense layers in the head</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>head_linear_first</code></b>
-              (<code>bool</code>, default:
+              </div>
+            </td>
+            <td>
                   <code>False</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Boolean indicating whether the order of the operations in the dense
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>head_linear_first</code></td>
+            <td>
+                  <code>bool</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Boolean indicating whether the order of the operations in the dense
 layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt;
 LIN -&gt; ACT]</code></p>
-          </div>
-        </li>
-    </ul>
+              </div>
+            </td>
+            <td>
+                  <code>False</code>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
 
 <p><span class="doc-section-title">Attributes:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code><span title="pytorch_widedeep.models.text.rnns.attentive_rnn.AttentiveRNN.word_embed">word_embed</span></code></b>
-              (<code><span title="nn.Module">Module</span></code>)
-          –
-          <div class="doc-md-description">
-            <p>word embedding matrix</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code><span title="pytorch_widedeep.models.text.rnns.attentive_rnn.AttentiveRNN.rnn">rnn</span></code></b>
-              (<code><span title="nn.Module">Module</span></code>)
-          –
-          <div class="doc-md-description">
-            <p>Stack of RNNs</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code><span title="pytorch_widedeep.models.text.rnns.attentive_rnn.AttentiveRNN.rnn_mlp">rnn_mlp</span></code></b>
-              (<code><span title="nn.Module">Module</span></code>)
-          –
-          <div class="doc-md-description">
-            <p>Stack of dense layers on top of the RNN. This will only exists if
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code><span title="pytorch_widedeep.models.text.rnns.attentive_rnn.AttentiveRNN.word_embed">word_embed</span></code></td>
+            <td>
+                  <code><span title="nn.Module">Module</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>word embedding matrix</p>
+              </div>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code><span title="pytorch_widedeep.models.text.rnns.attentive_rnn.AttentiveRNN.rnn">rnn</span></code></td>
+            <td>
+                  <code><span title="nn.Module">Module</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Stack of RNNs</p>
+              </div>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code><span title="pytorch_widedeep.models.text.rnns.attentive_rnn.AttentiveRNN.rnn_mlp">rnn_mlp</span></code></td>
+            <td>
+                  <code><span title="nn.Module">Module</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Stack of dense layers on top of the RNN. This will only exists if
 <code>head_layers_dim</code> is not <code>None</code></p>
-          </div>
-        </li>
-    </ul>
+              </div>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
 
 <p><span class="doc-section-title">Examples:</span></p>
@@ -11010,9 +17304,90 @@ <h2 id="pytorch_widedeep.models.text.rnns.attentive_rnn.AttentiveRNN" class="doc
 <span class="gp">&gt;&gt;&gt; </span><span class="n">out</span> <span class="o">=</span> <span class="n">model</span><span class="p">(</span><span class="n">X_text</span><span class="p">)</span>
 </code></pre></div>
 
-                  <details class="quote">
-                    <summary>Source code in <code>pytorch_widedeep/models/text/rnns/attentive_rnn.py</code></summary>
-                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 91</span>
+              <details class="quote">
+                <summary>Source code in <code>pytorch_widedeep/models/text/rnns/attentive_rnn.py</code></summary>
+                <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 10</span>
+<span class="normal"> 11</span>
+<span class="normal"> 12</span>
+<span class="normal"> 13</span>
+<span class="normal"> 14</span>
+<span class="normal"> 15</span>
+<span class="normal"> 16</span>
+<span class="normal"> 17</span>
+<span class="normal"> 18</span>
+<span class="normal"> 19</span>
+<span class="normal"> 20</span>
+<span class="normal"> 21</span>
+<span class="normal"> 22</span>
+<span class="normal"> 23</span>
+<span class="normal"> 24</span>
+<span class="normal"> 25</span>
+<span class="normal"> 26</span>
+<span class="normal"> 27</span>
+<span class="normal"> 28</span>
+<span class="normal"> 29</span>
+<span class="normal"> 30</span>
+<span class="normal"> 31</span>
+<span class="normal"> 32</span>
+<span class="normal"> 33</span>
+<span class="normal"> 34</span>
+<span class="normal"> 35</span>
+<span class="normal"> 36</span>
+<span class="normal"> 37</span>
+<span class="normal"> 38</span>
+<span class="normal"> 39</span>
+<span class="normal"> 40</span>
+<span class="normal"> 41</span>
+<span class="normal"> 42</span>
+<span class="normal"> 43</span>
+<span class="normal"> 44</span>
+<span class="normal"> 45</span>
+<span class="normal"> 46</span>
+<span class="normal"> 47</span>
+<span class="normal"> 48</span>
+<span class="normal"> 49</span>
+<span class="normal"> 50</span>
+<span class="normal"> 51</span>
+<span class="normal"> 52</span>
+<span class="normal"> 53</span>
+<span class="normal"> 54</span>
+<span class="normal"> 55</span>
+<span class="normal"> 56</span>
+<span class="normal"> 57</span>
+<span class="normal"> 58</span>
+<span class="normal"> 59</span>
+<span class="normal"> 60</span>
+<span class="normal"> 61</span>
+<span class="normal"> 62</span>
+<span class="normal"> 63</span>
+<span class="normal"> 64</span>
+<span class="normal"> 65</span>
+<span class="normal"> 66</span>
+<span class="normal"> 67</span>
+<span class="normal"> 68</span>
+<span class="normal"> 69</span>
+<span class="normal"> 70</span>
+<span class="normal"> 71</span>
+<span class="normal"> 72</span>
+<span class="normal"> 73</span>
+<span class="normal"> 74</span>
+<span class="normal"> 75</span>
+<span class="normal"> 76</span>
+<span class="normal"> 77</span>
+<span class="normal"> 78</span>
+<span class="normal"> 79</span>
+<span class="normal"> 80</span>
+<span class="normal"> 81</span>
+<span class="normal"> 82</span>
+<span class="normal"> 83</span>
+<span class="normal"> 84</span>
+<span class="normal"> 85</span>
+<span class="normal"> 86</span>
+<span class="normal"> 87</span>
+<span class="normal"> 88</span>
+<span class="normal"> 89</span>
+<span class="normal"> 90</span>
+<span class="normal"> 91</span>
 <span class="normal"> 92</span>
 <span class="normal"> 93</span>
 <span class="normal"> 94</span>
@@ -11080,77 +17455,208 @@ <h2 id="pytorch_widedeep.models.text.rnns.attentive_rnn.AttentiveRNN" class="doc
 <span class="normal">156</span>
 <span class="normal">157</span>
 <span class="normal">158</span>
-<span class="normal">159</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
-    <span class="bp">self</span><span class="p">,</span>
-    <span class="n">vocab_size</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span>
-    <span class="n">embed_dim</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">embed_matrix</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">embed_trainable</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">,</span>
-    <span class="n">rnn_type</span><span class="p">:</span> <span class="n">Literal</span><span class="p">[</span><span class="s2">&quot;lstm&quot;</span><span class="p">,</span> <span class="s2">&quot;gru&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="s2">&quot;lstm&quot;</span><span class="p">,</span>
-    <span class="n">hidden_dim</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">64</span><span class="p">,</span>
-    <span class="n">n_layers</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">3</span><span class="p">,</span>
-    <span class="n">rnn_dropout</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.1</span><span class="p">,</span>
-    <span class="n">bidirectional</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
-    <span class="n">use_hidden_state</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">,</span>
-    <span class="n">padding_idx</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1</span><span class="p">,</span>
-    <span class="n">attn_concatenate</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">,</span>
-    <span class="n">attn_dropout</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.1</span><span class="p">,</span>
-    <span class="n">head_hidden_dims</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">int</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">head_activation</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;relu&quot;</span><span class="p">,</span>
-    <span class="n">head_dropout</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">head_batchnorm</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
-    <span class="n">head_batchnorm_last</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
-    <span class="n">head_linear_first</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
-<span class="p">):</span>
-    <span class="nb">super</span><span class="p">(</span><span class="n">AttentiveRNN</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span>
-        <span class="n">vocab_size</span><span class="o">=</span><span class="n">vocab_size</span><span class="p">,</span>
-        <span class="n">embed_dim</span><span class="o">=</span><span class="n">embed_dim</span><span class="p">,</span>
-        <span class="n">embed_matrix</span><span class="o">=</span><span class="n">embed_matrix</span><span class="p">,</span>
-        <span class="n">embed_trainable</span><span class="o">=</span><span class="n">embed_trainable</span><span class="p">,</span>
-        <span class="n">rnn_type</span><span class="o">=</span><span class="n">rnn_type</span><span class="p">,</span>
-        <span class="n">hidden_dim</span><span class="o">=</span><span class="n">hidden_dim</span><span class="p">,</span>
-        <span class="n">n_layers</span><span class="o">=</span><span class="n">n_layers</span><span class="p">,</span>
-        <span class="n">rnn_dropout</span><span class="o">=</span><span class="n">rnn_dropout</span><span class="p">,</span>
-        <span class="n">bidirectional</span><span class="o">=</span><span class="n">bidirectional</span><span class="p">,</span>
-        <span class="n">use_hidden_state</span><span class="o">=</span><span class="n">use_hidden_state</span><span class="p">,</span>
-        <span class="n">padding_idx</span><span class="o">=</span><span class="n">padding_idx</span><span class="p">,</span>
-        <span class="n">head_hidden_dims</span><span class="o">=</span><span class="n">head_hidden_dims</span><span class="p">,</span>
-        <span class="n">head_activation</span><span class="o">=</span><span class="n">head_activation</span><span class="p">,</span>
-        <span class="n">head_dropout</span><span class="o">=</span><span class="n">head_dropout</span><span class="p">,</span>
-        <span class="n">head_batchnorm</span><span class="o">=</span><span class="n">head_batchnorm</span><span class="p">,</span>
-        <span class="n">head_batchnorm_last</span><span class="o">=</span><span class="n">head_batchnorm_last</span><span class="p">,</span>
-        <span class="n">head_linear_first</span><span class="o">=</span><span class="n">head_linear_first</span><span class="p">,</span>
-    <span class="p">)</span>
-
-    <span class="c1"># Embeddings and RNN defined in the BasicRNN inherited class</span>
+<span class="normal">159</span>
+<span class="normal">160</span>
+<span class="normal">161</span>
+<span class="normal">162</span>
+<span class="normal">163</span>
+<span class="normal">164</span>
+<span class="normal">165</span>
+<span class="normal">166</span>
+<span class="normal">167</span>
+<span class="normal">168</span>
+<span class="normal">169</span>
+<span class="normal">170</span>
+<span class="normal">171</span>
+<span class="normal">172</span>
+<span class="normal">173</span>
+<span class="normal">174</span>
+<span class="normal">175</span>
+<span class="normal">176</span>
+<span class="normal">177</span>
+<span class="normal">178</span>
+<span class="normal">179</span>
+<span class="normal">180</span>
+<span class="normal">181</span>
+<span class="normal">182</span>
+<span class="normal">183</span>
+<span class="normal">184</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">class</span> <span class="nc">AttentiveRNN</span><span class="p">(</span><span class="n">BasicRNN</span><span class="p">):</span>
+<span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Text classifier/regressor comprised by a stack of RNNs</span>
+<span class="sd">    (LSTMs or GRUs) plus an attention layer. This model can be used as the</span>
+<span class="sd">    `deeptext` component of a Wide &amp; Deep model or independently by</span>
+<span class="sd">    itself.</span>
+
+<span class="sd">    In addition, there is the option to add a Fully Connected (FC) set of dense</span>
+<span class="sd">    layers on top of attention layer</span>
+
+<span class="sd">    Parameters</span>
+<span class="sd">    ----------</span>
+<span class="sd">    vocab_size: int</span>
+<span class="sd">        Number of words in the vocabulary</span>
+<span class="sd">    embed_dim: int, Optional, default = None</span>
+<span class="sd">        Dimension of the word embeddings if non-pretained word vectors are</span>
+<span class="sd">        used</span>
+<span class="sd">    embed_matrix: np.ndarray, Optional, default = None</span>
+<span class="sd">        Pretrained word embeddings</span>
+<span class="sd">    embed_trainable: bool, default = True</span>
+<span class="sd">        Boolean indicating if the pretrained embeddings are trainable</span>
+<span class="sd">    rnn_type: str, default = &#39;lstm&#39;</span>
+<span class="sd">        String indicating the type of RNN to use. One of _&#39;lstm&#39;_ or _&#39;gru&#39;_</span>
+<span class="sd">    hidden_dim: int, default = 64</span>
+<span class="sd">        Hidden dim of the RNN</span>
+<span class="sd">    n_layers: int, default = 3</span>
+<span class="sd">        Number of recurrent layers</span>
+<span class="sd">    rnn_dropout: float, default = 0.1</span>
+<span class="sd">        Dropout for each RNN layer except the last layer</span>
+<span class="sd">    bidirectional: bool, default = True</span>
+<span class="sd">        Boolean indicating whether the staked RNNs are bidirectional</span>
+<span class="sd">    use_hidden_state: str, default = True</span>
+<span class="sd">        Boolean indicating whether to use the final hidden state or the RNN&#39;s</span>
+<span class="sd">        output as predicting features. Typically the former is used.</span>
+<span class="sd">    padding_idx: int, default = 1</span>
+<span class="sd">        index of the padding token in the padded-tokenised sequences. The</span>
+<span class="sd">        `TextPreprocessor` class within this library uses fastai&#39;s</span>
+<span class="sd">        tokenizer where the token index 0 is reserved for the _&#39;unknown&#39;_</span>
+<span class="sd">        word token. Therefore, the default value is set to 1.</span>
+<span class="sd">    attn_concatenate: bool, default = True</span>
+<span class="sd">        Boolean indicating if the input to the attention mechanism will be the</span>
+<span class="sd">        output of the RNN or the output of the RNN concatenated with the last</span>
+<span class="sd">        hidden state.</span>
+<span class="sd">    attn_dropout: float, default = 0.1</span>
+<span class="sd">        Internal dropout for the attention mechanism</span>
+<span class="sd">    head_hidden_dims: List, Optional, default = None</span>
+<span class="sd">        List with the sizes of the dense layers in the head e.g: _[128, 64]_</span>
+<span class="sd">    head_activation: str, default = &quot;relu&quot;</span>
+<span class="sd">        Activation function for the dense layers in the head. Currently</span>
+<span class="sd">        _&#39;tanh&#39;_, _&#39;relu&#39;_, _&#39;leaky_relu&#39;_ and _&#39;gelu&#39;_ are supported</span>
+<span class="sd">    head_dropout: float, Optional, default = None</span>
+<span class="sd">        Dropout of the dense layers in the head</span>
+<span class="sd">    head_batchnorm: bool, default = False</span>
+<span class="sd">        Boolean indicating whether or not to include batch normalization in</span>
+<span class="sd">        the dense layers that form the _&#39;rnn_mlp&#39;_</span>
+<span class="sd">    head_batchnorm_last: bool, default = False</span>
+<span class="sd">        Boolean indicating whether or not to apply batch normalization to the</span>
+<span class="sd">        last of the dense layers in the head</span>
+<span class="sd">    head_linear_first: bool, default = False</span>
+<span class="sd">        Boolean indicating whether the order of the operations in the dense</span>
+<span class="sd">        layer. If `True: [LIN -&gt; ACT -&gt; BN -&gt; DP]`. If `False: [BN -&gt; DP -&gt;</span>
+<span class="sd">        LIN -&gt; ACT]`</span>
+
+<span class="sd">    Attributes</span>
+<span class="sd">    ----------</span>
+<span class="sd">    word_embed: nn.Module</span>
+<span class="sd">        word embedding matrix</span>
+<span class="sd">    rnn: nn.Module</span>
+<span class="sd">        Stack of RNNs</span>
+<span class="sd">    rnn_mlp: nn.Module</span>
+<span class="sd">        Stack of dense layers on top of the RNN. This will only exists if</span>
+<span class="sd">        `head_layers_dim` is not `None`</span>
+
+<span class="sd">    Examples</span>
+<span class="sd">    --------</span>
+<span class="sd">    &gt;&gt;&gt; import torch</span>
+<span class="sd">    &gt;&gt;&gt; from pytorch_widedeep.models import AttentiveRNN</span>
+<span class="sd">    &gt;&gt;&gt; X_text = torch.cat((torch.zeros([5,1]), torch.empty(5, 4).random_(1,4)), axis=1)</span>
+<span class="sd">    &gt;&gt;&gt; model = AttentiveRNN(vocab_size=4, hidden_dim=4, n_layers=2, padding_idx=0, embed_dim=4)</span>
+<span class="sd">    &gt;&gt;&gt; out = model(X_text)</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+
+    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span>
+        <span class="n">vocab_size</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span>
+        <span class="n">embed_dim</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">embed_matrix</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">embed_trainable</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">,</span>
+        <span class="n">rnn_type</span><span class="p">:</span> <span class="n">Literal</span><span class="p">[</span><span class="s2">&quot;lstm&quot;</span><span class="p">,</span> <span class="s2">&quot;gru&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="s2">&quot;lstm&quot;</span><span class="p">,</span>
+        <span class="n">hidden_dim</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">64</span><span class="p">,</span>
+        <span class="n">n_layers</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">3</span><span class="p">,</span>
+        <span class="n">rnn_dropout</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.1</span><span class="p">,</span>
+        <span class="n">bidirectional</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
+        <span class="n">use_hidden_state</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">,</span>
+        <span class="n">padding_idx</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1</span><span class="p">,</span>
+        <span class="n">attn_concatenate</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">,</span>
+        <span class="n">attn_dropout</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.1</span><span class="p">,</span>
+        <span class="n">head_hidden_dims</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">int</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">head_activation</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;relu&quot;</span><span class="p">,</span>
+        <span class="n">head_dropout</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">head_batchnorm</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
+        <span class="n">head_batchnorm_last</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
+        <span class="n">head_linear_first</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
+    <span class="p">):</span>
+        <span class="nb">super</span><span class="p">(</span><span class="n">AttentiveRNN</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span>
+            <span class="n">vocab_size</span><span class="o">=</span><span class="n">vocab_size</span><span class="p">,</span>
+            <span class="n">embed_dim</span><span class="o">=</span><span class="n">embed_dim</span><span class="p">,</span>
+            <span class="n">embed_matrix</span><span class="o">=</span><span class="n">embed_matrix</span><span class="p">,</span>
+            <span class="n">embed_trainable</span><span class="o">=</span><span class="n">embed_trainable</span><span class="p">,</span>
+            <span class="n">rnn_type</span><span class="o">=</span><span class="n">rnn_type</span><span class="p">,</span>
+            <span class="n">hidden_dim</span><span class="o">=</span><span class="n">hidden_dim</span><span class="p">,</span>
+            <span class="n">n_layers</span><span class="o">=</span><span class="n">n_layers</span><span class="p">,</span>
+            <span class="n">rnn_dropout</span><span class="o">=</span><span class="n">rnn_dropout</span><span class="p">,</span>
+            <span class="n">bidirectional</span><span class="o">=</span><span class="n">bidirectional</span><span class="p">,</span>
+            <span class="n">use_hidden_state</span><span class="o">=</span><span class="n">use_hidden_state</span><span class="p">,</span>
+            <span class="n">padding_idx</span><span class="o">=</span><span class="n">padding_idx</span><span class="p">,</span>
+            <span class="n">head_hidden_dims</span><span class="o">=</span><span class="n">head_hidden_dims</span><span class="p">,</span>
+            <span class="n">head_activation</span><span class="o">=</span><span class="n">head_activation</span><span class="p">,</span>
+            <span class="n">head_dropout</span><span class="o">=</span><span class="n">head_dropout</span><span class="p">,</span>
+            <span class="n">head_batchnorm</span><span class="o">=</span><span class="n">head_batchnorm</span><span class="p">,</span>
+            <span class="n">head_batchnorm_last</span><span class="o">=</span><span class="n">head_batchnorm_last</span><span class="p">,</span>
+            <span class="n">head_linear_first</span><span class="o">=</span><span class="n">head_linear_first</span><span class="p">,</span>
+        <span class="p">)</span>
 
-    <span class="c1"># Attention</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">attn_concatenate</span> <span class="o">=</span> <span class="n">attn_concatenate</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">attn_dropout</span> <span class="o">=</span> <span class="n">attn_dropout</span>
+        <span class="c1"># Embeddings and RNN defined in the BasicRNN inherited class</span>
 
-    <span class="k">if</span> <span class="n">bidirectional</span> <span class="ow">and</span> <span class="n">attn_concatenate</span><span class="p">:</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">rnn_output_dim</span> <span class="o">=</span> <span class="n">hidden_dim</span> <span class="o">*</span> <span class="mi">4</span>
-    <span class="k">elif</span> <span class="n">bidirectional</span> <span class="ow">or</span> <span class="n">attn_concatenate</span><span class="p">:</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">rnn_output_dim</span> <span class="o">=</span> <span class="n">hidden_dim</span> <span class="o">*</span> <span class="mi">2</span>
-    <span class="k">else</span><span class="p">:</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">rnn_output_dim</span> <span class="o">=</span> <span class="n">hidden_dim</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">attn</span> <span class="o">=</span> <span class="n">ContextAttention</span><span class="p">(</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">rnn_output_dim</span><span class="p">,</span> <span class="n">attn_dropout</span><span class="p">,</span> <span class="n">sum_along_seq</span><span class="o">=</span><span class="kc">True</span>
-    <span class="p">)</span>
+        <span class="c1"># Attention</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">attn_concatenate</span> <span class="o">=</span> <span class="n">attn_concatenate</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">attn_dropout</span> <span class="o">=</span> <span class="n">attn_dropout</span>
 
-    <span class="c1"># FC-Head (Mlp)</span>
-    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">head_hidden_dims</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
-        <span class="n">head_hidden_dims</span> <span class="o">=</span> <span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">rnn_output_dim</span><span class="p">]</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">head_hidden_dims</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">rnn_mlp</span> <span class="o">=</span> <span class="n">MLP</span><span class="p">(</span>
-            <span class="n">head_hidden_dims</span><span class="p">,</span>
-            <span class="n">head_activation</span><span class="p">,</span>
-            <span class="n">head_dropout</span><span class="p">,</span>
-            <span class="n">head_batchnorm</span><span class="p">,</span>
-            <span class="n">head_batchnorm_last</span><span class="p">,</span>
-            <span class="n">head_linear_first</span><span class="p">,</span>
+        <span class="k">if</span> <span class="n">bidirectional</span> <span class="ow">and</span> <span class="n">attn_concatenate</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">rnn_output_dim</span> <span class="o">=</span> <span class="n">hidden_dim</span> <span class="o">*</span> <span class="mi">4</span>
+        <span class="k">elif</span> <span class="n">bidirectional</span> <span class="ow">or</span> <span class="n">attn_concatenate</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">rnn_output_dim</span> <span class="o">=</span> <span class="n">hidden_dim</span> <span class="o">*</span> <span class="mi">2</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">rnn_output_dim</span> <span class="o">=</span> <span class="n">hidden_dim</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">attn</span> <span class="o">=</span> <span class="n">ContextAttention</span><span class="p">(</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">rnn_output_dim</span><span class="p">,</span> <span class="n">attn_dropout</span><span class="p">,</span> <span class="n">sum_along_seq</span><span class="o">=</span><span class="kc">True</span>
         <span class="p">)</span>
+
+        <span class="c1"># FC-Head (Mlp)</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">head_hidden_dims</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="n">head_hidden_dims</span> <span class="o">=</span> <span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">rnn_output_dim</span><span class="p">]</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">head_hidden_dims</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">rnn_mlp</span> <span class="o">=</span> <span class="n">MLP</span><span class="p">(</span>
+                <span class="n">head_hidden_dims</span><span class="p">,</span>
+                <span class="n">head_activation</span><span class="p">,</span>
+                <span class="n">head_dropout</span><span class="p">,</span>
+                <span class="n">head_batchnorm</span><span class="p">,</span>
+                <span class="n">head_batchnorm_last</span><span class="p">,</span>
+                <span class="n">head_linear_first</span><span class="p">,</span>
+            <span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">_process_rnn_outputs</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">output</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">,</span> <span class="n">hidden</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tensor</span><span class="p">:</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">attn_concatenate</span><span class="p">:</span>
+            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">bidirectional</span><span class="p">:</span>
+                <span class="n">bi_hidden</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">cat</span><span class="p">((</span><span class="n">hidden</span><span class="p">[</span><span class="o">-</span><span class="mi">2</span><span class="p">],</span> <span class="n">hidden</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">]),</span> <span class="n">dim</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span>
+                <span class="n">attn_inp</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">cat</span><span class="p">(</span>
+                    <span class="p">[</span><span class="n">output</span><span class="p">,</span> <span class="n">bi_hidden</span><span class="o">.</span><span class="n">unsqueeze</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span><span class="o">.</span><span class="n">expand_as</span><span class="p">(</span><span class="n">output</span><span class="p">)],</span> <span class="n">dim</span><span class="o">=</span><span class="mi">2</span>
+                <span class="p">)</span>
+            <span class="k">else</span><span class="p">:</span>
+                <span class="n">attn_inp</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">cat</span><span class="p">(</span>
+                    <span class="p">[</span><span class="n">output</span><span class="p">,</span> <span class="n">hidden</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span><span class="o">.</span><span class="n">unsqueeze</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span><span class="o">.</span><span class="n">expand_as</span><span class="p">(</span><span class="n">output</span><span class="p">)],</span> <span class="n">dim</span><span class="o">=</span><span class="mi">2</span>
+                <span class="p">)</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="n">attn_inp</span> <span class="o">=</span> <span class="n">output</span>
+
+        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">attn</span><span class="p">(</span><span class="n">attn_inp</span><span class="p">)</span>
+
+    <span class="nd">@property</span>
+    <span class="k">def</span> <span class="nf">attention_weights</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">List</span><span class="p">:</span>
+<span class="w">        </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;List with the attention weights</span>
+
+<span class="sd">        The shape of the attention weights is $(N, S)$, where $N$ is the batch</span>
+<span class="sd">        size and $S$ is the length of the sequence</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">attn</span><span class="o">.</span><span class="n">attn_weights</span>
 </code></pre></div></td></tr></table></div>
-                  </details>
+              </details>
 
 
 
@@ -11206,27 +17712,7 @@ <h2 id="pytorch_widedeep.models.text.rnns.stacked_attentive_rnn.StackedAttentive
 
 
 <a href="#pytorch_widedeep.models.text.rnns.stacked_attentive_rnn.StackedAttentiveRNN" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="nf">StackedAttentiveRNN</span><span class="p">(</span>
-    <span class="n">vocab_size</span><span class="p">,</span>
-    <span class="n">embed_dim</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">embed_matrix</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">embed_trainable</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
-    <span class="n">rnn_type</span><span class="o">=</span><span class="s2">&quot;lstm&quot;</span><span class="p">,</span>
-    <span class="n">hidden_dim</span><span class="o">=</span><span class="mi">64</span><span class="p">,</span>
-    <span class="n">bidirectional</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
-    <span class="n">padding_idx</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span>
-    <span class="n">n_blocks</span><span class="o">=</span><span class="mi">3</span><span class="p">,</span>
-    <span class="n">attn_concatenate</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
-    <span class="n">attn_dropout</span><span class="o">=</span><span class="mf">0.1</span><span class="p">,</span>
-    <span class="n">with_addnorm</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
-    <span class="n">head_hidden_dims</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">head_activation</span><span class="o">=</span><span class="s2">&quot;relu&quot;</span><span class="p">,</span>
-    <span class="n">head_dropout</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">head_batchnorm</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
-    <span class="n">head_batchnorm_last</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
-    <span class="n">head_linear_first</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
-<span class="p">)</span>
-</code></pre></div>
+
 
     <div class="doc doc-contents first">
             <p class="doc doc-class-bases">
@@ -11241,229 +17727,331 @@ <h2 id="pytorch_widedeep.models.text.rnns.stacked_attentive_rnn.StackedAttentive
 
 
 <p><span class="doc-section-title">Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>vocab_size</code></b>
-              (<code>int</code>)
-          –
-          <div class="doc-md-description">
-            <p>Number of words in the vocabulary</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>embed_dim</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[int]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Dimension of the word embeddings if non-pretained word vectors are
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+          <th>Default</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>vocab_size</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Number of words in the vocabulary</p>
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>embed_dim</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[int]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Dimension of the word embeddings if non-pretained word vectors are
 used</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>embed_matrix</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="numpy.ndarray">ndarray</span>]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Pretrained word embeddings</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>embed_trainable</code></b>
-              (<code>bool</code>, default:
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>embed_matrix</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="numpy.ndarray">ndarray</span>]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Pretrained word embeddings</p>
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>embed_trainable</code></td>
+            <td>
+                  <code>bool</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Boolean indicating if the pretrained embeddings are trainable</p>
+              </div>
+            </td>
+            <td>
                   <code>True</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Boolean indicating if the pretrained embeddings are trainable</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>rnn_type</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[lstm, gru]</code>, default:
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>rnn_type</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[lstm, gru]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>String indicating the type of RNN to use. One of 'lstm' or 'gru'</p>
+              </div>
+            </td>
+            <td>
                   <code>&#39;lstm&#39;</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>String indicating the type of RNN to use. One of 'lstm' or 'gru'</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>hidden_dim</code></b>
-              (<code>int</code>, default:
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>hidden_dim</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Hidden dim of the RNN</p>
+              </div>
+            </td>
+            <td>
                   <code>64</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Hidden dim of the RNN</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>bidirectional</code></b>
-              (<code>bool</code>, default:
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>bidirectional</code></td>
+            <td>
+                  <code>bool</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Boolean indicating whether the staked RNNs are bidirectional</p>
+              </div>
+            </td>
+            <td>
                   <code>False</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Boolean indicating whether the staked RNNs are bidirectional</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>padding_idx</code></b>
-              (<code>int</code>, default:
-                  <code>1</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>index of the padding token in the padded-tokenised sequences. The
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>padding_idx</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>index of the padding token in the padded-tokenised sequences. The
 <code>TextPreprocessor</code> class within this library uses fastai's
 tokenizer where the token index 0 is reserved for the <em>'unknown'</em>
 word token. Therefore, the default value is set to 1.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>n_blocks</code></b>
-              (<code>int</code>, default:
-                  <code>3</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Number of attention blocks. Each block is comprised by an RNN and a
+              </div>
+            </td>
+            <td>
+                  <code>1</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>n_blocks</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Number of attention blocks. Each block is comprised by an RNN and a
 Context Attention Encoder</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>attn_concatenate</code></b>
-              (<code>bool</code>, default:
-                  <code>False</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Boolean indicating if the input to the attention mechanism will be the
+              </div>
+            </td>
+            <td>
+                  <code>3</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>attn_concatenate</code></td>
+            <td>
+                  <code>bool</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Boolean indicating if the input to the attention mechanism will be the
 output of the RNN or the output of the RNN concatenated with the last
 hidden state or simply</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>attn_dropout</code></b>
-              (<code>float</code>, default:
-                  <code>0.1</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Internal dropout for the attention mechanism</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>with_addnorm</code></b>
-              (<code>bool</code>, default:
+              </div>
+            </td>
+            <td>
                   <code>False</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Boolean indicating if the output of each block will be added to the
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>attn_dropout</code></td>
+            <td>
+                  <code>float</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Internal dropout for the attention mechanism</p>
+              </div>
+            </td>
+            <td>
+                  <code>0.1</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>with_addnorm</code></td>
+            <td>
+                  <code>bool</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Boolean indicating if the output of each block will be added to the
 input and normalised</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>head_hidden_dims</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[int]]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>List with the sizes of the dense layers in the head e.g: [128, 64]</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>head_activation</code></b>
-              (<code>str</code>, default:
-                  <code>&#39;relu&#39;</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Activation function for the dense layers in the head. Currently
-<em>'tanh'</em>, <em>'relu'</em>, <em>'leaky_relu'</em> and <em>'gelu'</em> are supported</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>head_dropout</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Dropout of the dense layers in the head</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>head_batchnorm</code></b>
-              (<code>bool</code>, default:
+              </div>
+            </td>
+            <td>
                   <code>False</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Boolean indicating whether or not to include batch normalization in
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>head_hidden_dims</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[int]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>List with the sizes of the dense layers in the head e.g: [128, 64]</p>
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>head_activation</code></td>
+            <td>
+                  <code>str</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Activation function for the dense layers in the head. Currently
+<em>'tanh'</em>, <em>'relu'</em>, <em>'leaky_relu'</em> and <em>'gelu'</em> are supported</p>
+              </div>
+            </td>
+            <td>
+                  <code>&#39;relu&#39;</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>head_dropout</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Dropout of the dense layers in the head</p>
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>head_batchnorm</code></td>
+            <td>
+                  <code>bool</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Boolean indicating whether or not to include batch normalization in
 the dense layers that form the <em>'rnn_mlp'</em></p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>head_batchnorm_last</code></b>
-              (<code>bool</code>, default:
+              </div>
+            </td>
+            <td>
                   <code>False</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Boolean indicating whether or not to apply batch normalization to the
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>head_batchnorm_last</code></td>
+            <td>
+                  <code>bool</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Boolean indicating whether or not to apply batch normalization to the
 last of the dense layers in the head</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>head_linear_first</code></b>
-              (<code>bool</code>, default:
+              </div>
+            </td>
+            <td>
                   <code>False</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Boolean indicating whether the order of the operations in the dense
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>head_linear_first</code></td>
+            <td>
+                  <code>bool</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Boolean indicating whether the order of the operations in the dense
 layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt;
 LIN -&gt; ACT]</code></p>
-          </div>
-        </li>
-    </ul>
+              </div>
+            </td>
+            <td>
+                  <code>False</code>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
 
 <p><span class="doc-section-title">Attributes:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code><span title="pytorch_widedeep.models.text.rnns.stacked_attentive_rnn.StackedAttentiveRNN.word_embed">word_embed</span></code></b>
-              (<code><span title="torch.nn.Module">Module</span></code>)
-          –
-          <div class="doc-md-description">
-            <p>word embedding matrix</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code><span title="pytorch_widedeep.models.text.rnns.stacked_attentive_rnn.StackedAttentiveRNN.rnn">rnn</span></code></b>
-              (<code><span title="torch.nn.Module">Module</span></code>)
-          –
-          <div class="doc-md-description">
-            <p>Stack of RNNs</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code><span title="pytorch_widedeep.models.text.rnns.stacked_attentive_rnn.StackedAttentiveRNN.rnn_mlp">rnn_mlp</span></code></b>
-              (<code><span title="torch.nn.Module">Module</span></code>)
-          –
-          <div class="doc-md-description">
-            <p>Stack of dense layers on top of the RNN. This will only exists if
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code><span title="pytorch_widedeep.models.text.rnns.stacked_attentive_rnn.StackedAttentiveRNN.word_embed">word_embed</span></code></td>
+            <td>
+                  <code><span title="torch.nn.Module">Module</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>word embedding matrix</p>
+              </div>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code><span title="pytorch_widedeep.models.text.rnns.stacked_attentive_rnn.StackedAttentiveRNN.rnn">rnn</span></code></td>
+            <td>
+                  <code><span title="torch.nn.Module">Module</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Stack of RNNs</p>
+              </div>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code><span title="pytorch_widedeep.models.text.rnns.stacked_attentive_rnn.StackedAttentiveRNN.rnn_mlp">rnn_mlp</span></code></td>
+            <td>
+                  <code><span title="torch.nn.Module">Module</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Stack of dense layers on top of the RNN. This will only exists if
 <code>head_layers_dim</code> is not <code>None</code></p>
-          </div>
-        </li>
-    </ul>
+              </div>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
 
 <p><span class="doc-section-title">Examples:</span></p>
@@ -11474,9 +18062,88 @@ <h2 id="pytorch_widedeep.models.text.rnns.stacked_attentive_rnn.StackedAttentive
 <span class="gp">&gt;&gt;&gt; </span><span class="n">out</span> <span class="o">=</span> <span class="n">model</span><span class="p">(</span><span class="n">X_text</span><span class="p">)</span>
 </code></pre></div>
 
-                  <details class="quote">
-                    <summary>Source code in <code>pytorch_widedeep/models/text/rnns/stacked_attentive_rnn.py</code></summary>
-                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 92</span>
+              <details class="quote">
+                <summary>Source code in <code>pytorch_widedeep/models/text/rnns/stacked_attentive_rnn.py</code></summary>
+                <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 13</span>
+<span class="normal"> 14</span>
+<span class="normal"> 15</span>
+<span class="normal"> 16</span>
+<span class="normal"> 17</span>
+<span class="normal"> 18</span>
+<span class="normal"> 19</span>
+<span class="normal"> 20</span>
+<span class="normal"> 21</span>
+<span class="normal"> 22</span>
+<span class="normal"> 23</span>
+<span class="normal"> 24</span>
+<span class="normal"> 25</span>
+<span class="normal"> 26</span>
+<span class="normal"> 27</span>
+<span class="normal"> 28</span>
+<span class="normal"> 29</span>
+<span class="normal"> 30</span>
+<span class="normal"> 31</span>
+<span class="normal"> 32</span>
+<span class="normal"> 33</span>
+<span class="normal"> 34</span>
+<span class="normal"> 35</span>
+<span class="normal"> 36</span>
+<span class="normal"> 37</span>
+<span class="normal"> 38</span>
+<span class="normal"> 39</span>
+<span class="normal"> 40</span>
+<span class="normal"> 41</span>
+<span class="normal"> 42</span>
+<span class="normal"> 43</span>
+<span class="normal"> 44</span>
+<span class="normal"> 45</span>
+<span class="normal"> 46</span>
+<span class="normal"> 47</span>
+<span class="normal"> 48</span>
+<span class="normal"> 49</span>
+<span class="normal"> 50</span>
+<span class="normal"> 51</span>
+<span class="normal"> 52</span>
+<span class="normal"> 53</span>
+<span class="normal"> 54</span>
+<span class="normal"> 55</span>
+<span class="normal"> 56</span>
+<span class="normal"> 57</span>
+<span class="normal"> 58</span>
+<span class="normal"> 59</span>
+<span class="normal"> 60</span>
+<span class="normal"> 61</span>
+<span class="normal"> 62</span>
+<span class="normal"> 63</span>
+<span class="normal"> 64</span>
+<span class="normal"> 65</span>
+<span class="normal"> 66</span>
+<span class="normal"> 67</span>
+<span class="normal"> 68</span>
+<span class="normal"> 69</span>
+<span class="normal"> 70</span>
+<span class="normal"> 71</span>
+<span class="normal"> 72</span>
+<span class="normal"> 73</span>
+<span class="normal"> 74</span>
+<span class="normal"> 75</span>
+<span class="normal"> 76</span>
+<span class="normal"> 77</span>
+<span class="normal"> 78</span>
+<span class="normal"> 79</span>
+<span class="normal"> 80</span>
+<span class="normal"> 81</span>
+<span class="normal"> 82</span>
+<span class="normal"> 83</span>
+<span class="normal"> 84</span>
+<span class="normal"> 85</span>
+<span class="normal"> 86</span>
+<span class="normal"> 87</span>
+<span class="normal"> 88</span>
+<span class="normal"> 89</span>
+<span class="normal"> 90</span>
+<span class="normal"> 91</span>
+<span class="normal"> 92</span>
 <span class="normal"> 93</span>
 <span class="normal"> 94</span>
 <span class="normal"> 95</span>
@@ -11597,130 +18264,349 @@ <h2 id="pytorch_widedeep.models.text.rnns.stacked_attentive_rnn.StackedAttentive
 <span class="normal">210</span>
 <span class="normal">211</span>
 <span class="normal">212</span>
-<span class="normal">213</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
-    <span class="bp">self</span><span class="p">,</span>
-    <span class="n">vocab_size</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span>
-    <span class="n">embed_dim</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">embed_matrix</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">embed_trainable</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">,</span>
-    <span class="n">rnn_type</span><span class="p">:</span> <span class="n">Literal</span><span class="p">[</span><span class="s2">&quot;lstm&quot;</span><span class="p">,</span> <span class="s2">&quot;gru&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="s2">&quot;lstm&quot;</span><span class="p">,</span>
-    <span class="n">hidden_dim</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">64</span><span class="p">,</span>
-    <span class="n">bidirectional</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
-    <span class="n">padding_idx</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1</span><span class="p">,</span>
-    <span class="n">n_blocks</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">3</span><span class="p">,</span>
-    <span class="n">attn_concatenate</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
-    <span class="n">attn_dropout</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.1</span><span class="p">,</span>
-    <span class="n">with_addnorm</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
-    <span class="n">head_hidden_dims</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">int</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">head_activation</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;relu&quot;</span><span class="p">,</span>
-    <span class="n">head_dropout</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">head_batchnorm</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
-    <span class="n">head_batchnorm_last</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
-    <span class="n">head_linear_first</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
-<span class="p">):</span>
-    <span class="nb">super</span><span class="p">(</span><span class="n">StackedAttentiveRNN</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
-
-    <span class="k">if</span> <span class="p">(</span>
-        <span class="n">embed_dim</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span>
-        <span class="ow">and</span> <span class="n">embed_matrix</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span>
-        <span class="ow">and</span> <span class="ow">not</span> <span class="n">embed_dim</span> <span class="o">==</span> <span class="n">embed_matrix</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span>
+<span class="normal">213</span>
+<span class="normal">214</span>
+<span class="normal">215</span>
+<span class="normal">216</span>
+<span class="normal">217</span>
+<span class="normal">218</span>
+<span class="normal">219</span>
+<span class="normal">220</span>
+<span class="normal">221</span>
+<span class="normal">222</span>
+<span class="normal">223</span>
+<span class="normal">224</span>
+<span class="normal">225</span>
+<span class="normal">226</span>
+<span class="normal">227</span>
+<span class="normal">228</span>
+<span class="normal">229</span>
+<span class="normal">230</span>
+<span class="normal">231</span>
+<span class="normal">232</span>
+<span class="normal">233</span>
+<span class="normal">234</span>
+<span class="normal">235</span>
+<span class="normal">236</span>
+<span class="normal">237</span>
+<span class="normal">238</span>
+<span class="normal">239</span>
+<span class="normal">240</span>
+<span class="normal">241</span>
+<span class="normal">242</span>
+<span class="normal">243</span>
+<span class="normal">244</span>
+<span class="normal">245</span>
+<span class="normal">246</span>
+<span class="normal">247</span>
+<span class="normal">248</span>
+<span class="normal">249</span>
+<span class="normal">250</span>
+<span class="normal">251</span>
+<span class="normal">252</span>
+<span class="normal">253</span>
+<span class="normal">254</span>
+<span class="normal">255</span>
+<span class="normal">256</span>
+<span class="normal">257</span>
+<span class="normal">258</span>
+<span class="normal">259</span>
+<span class="normal">260</span>
+<span class="normal">261</span>
+<span class="normal">262</span>
+<span class="normal">263</span>
+<span class="normal">264</span>
+<span class="normal">265</span>
+<span class="normal">266</span>
+<span class="normal">267</span>
+<span class="normal">268</span>
+<span class="normal">269</span>
+<span class="normal">270</span>
+<span class="normal">271</span>
+<span class="normal">272</span>
+<span class="normal">273</span>
+<span class="normal">274</span>
+<span class="normal">275</span>
+<span class="normal">276</span>
+<span class="normal">277</span>
+<span class="normal">278</span>
+<span class="normal">279</span>
+<span class="normal">280</span>
+<span class="normal">281</span>
+<span class="normal">282</span>
+<span class="normal">283</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">class</span> <span class="nc">StackedAttentiveRNN</span><span class="p">(</span><span class="n">BaseWDModelComponent</span><span class="p">):</span>
+<span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Text classifier/regressor comprised by a stack of blocks:</span>
+<span class="sd">    `[RNN + Attention]`. This can be used as the `deeptext` component of a</span>
+<span class="sd">    Wide &amp; Deep model or independently by itself.</span>
+
+<span class="sd">    In addition, there is the option to add a Fully Connected (FC) set of</span>
+<span class="sd">    dense layers on top of the attentiob blocks</span>
+
+<span class="sd">    Parameters</span>
+<span class="sd">    ----------</span>
+<span class="sd">    vocab_size: int</span>
+<span class="sd">        Number of words in the vocabulary</span>
+<span class="sd">    embed_dim: int, Optional, default = None</span>
+<span class="sd">        Dimension of the word embeddings if non-pretained word vectors are</span>
+<span class="sd">        used</span>
+<span class="sd">    embed_matrix: np.ndarray, Optional, default = None</span>
+<span class="sd">        Pretrained word embeddings</span>
+<span class="sd">    embed_trainable: bool, default = True</span>
+<span class="sd">        Boolean indicating if the pretrained embeddings are trainable</span>
+<span class="sd">    rnn_type: str, default = &#39;lstm&#39;</span>
+<span class="sd">        String indicating the type of RNN to use. One of &#39;lstm&#39; or &#39;gru&#39;</span>
+<span class="sd">    hidden_dim: int, default = 64</span>
+<span class="sd">        Hidden dim of the RNN</span>
+<span class="sd">    bidirectional: bool, default = True</span>
+<span class="sd">        Boolean indicating whether the staked RNNs are bidirectional</span>
+<span class="sd">    padding_idx: int, default = 1</span>
+<span class="sd">        index of the padding token in the padded-tokenised sequences. The</span>
+<span class="sd">        `TextPreprocessor` class within this library uses fastai&#39;s</span>
+<span class="sd">        tokenizer where the token index 0 is reserved for the _&#39;unknown&#39;_</span>
+<span class="sd">        word token. Therefore, the default value is set to 1.</span>
+<span class="sd">    n_blocks: int, default = 3</span>
+<span class="sd">        Number of attention blocks. Each block is comprised by an RNN and a</span>
+<span class="sd">        Context Attention Encoder</span>
+<span class="sd">    attn_concatenate: bool, default = True</span>
+<span class="sd">        Boolean indicating if the input to the attention mechanism will be the</span>
+<span class="sd">        output of the RNN or the output of the RNN concatenated with the last</span>
+<span class="sd">        hidden state or simply</span>
+<span class="sd">    attn_dropout: float, default = 0.1</span>
+<span class="sd">        Internal dropout for the attention mechanism</span>
+<span class="sd">    with_addnorm: bool, default = False</span>
+<span class="sd">        Boolean indicating if the output of each block will be added to the</span>
+<span class="sd">        input and normalised</span>
+<span class="sd">    head_hidden_dims: List, Optional, default = None</span>
+<span class="sd">        List with the sizes of the dense layers in the head e.g: [128, 64]</span>
+<span class="sd">    head_activation: str, default = &quot;relu&quot;</span>
+<span class="sd">        Activation function for the dense layers in the head. Currently</span>
+<span class="sd">        _&#39;tanh&#39;_, _&#39;relu&#39;_, _&#39;leaky_relu&#39;_ and _&#39;gelu&#39;_ are supported</span>
+<span class="sd">    head_dropout: float, Optional, default = None</span>
+<span class="sd">        Dropout of the dense layers in the head</span>
+<span class="sd">    head_batchnorm: bool, default = False</span>
+<span class="sd">        Boolean indicating whether or not to include batch normalization in</span>
+<span class="sd">        the dense layers that form the _&#39;rnn_mlp&#39;_</span>
+<span class="sd">    head_batchnorm_last: bool, default = False</span>
+<span class="sd">        Boolean indicating whether or not to apply batch normalization to the</span>
+<span class="sd">        last of the dense layers in the head</span>
+<span class="sd">    head_linear_first: bool, default = False</span>
+<span class="sd">        Boolean indicating whether the order of the operations in the dense</span>
+<span class="sd">        layer. If `True: [LIN -&gt; ACT -&gt; BN -&gt; DP]`. If `False: [BN -&gt; DP -&gt;</span>
+<span class="sd">        LIN -&gt; ACT]`</span>
+
+<span class="sd">    Attributes</span>
+<span class="sd">    ----------</span>
+<span class="sd">    word_embed: nn.Module</span>
+<span class="sd">        word embedding matrix</span>
+<span class="sd">    rnn: nn.Module</span>
+<span class="sd">        Stack of RNNs</span>
+<span class="sd">    rnn_mlp: nn.Module</span>
+<span class="sd">        Stack of dense layers on top of the RNN. This will only exists if</span>
+<span class="sd">        `head_layers_dim` is not `None`</span>
+
+<span class="sd">    Examples</span>
+<span class="sd">    --------</span>
+<span class="sd">    &gt;&gt;&gt; import torch</span>
+<span class="sd">    &gt;&gt;&gt; from pytorch_widedeep.models import StackedAttentiveRNN</span>
+<span class="sd">    &gt;&gt;&gt; X_text = torch.cat((torch.zeros([5,1]), torch.empty(5, 4).random_(1,4)), axis=1)</span>
+<span class="sd">    &gt;&gt;&gt; model = StackedAttentiveRNN(vocab_size=4, hidden_dim=4, padding_idx=0, embed_dim=4)</span>
+<span class="sd">    &gt;&gt;&gt; out = model(X_text)</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+
+    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span>
+        <span class="n">vocab_size</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span>
+        <span class="n">embed_dim</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">embed_matrix</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">embed_trainable</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">,</span>
+        <span class="n">rnn_type</span><span class="p">:</span> <span class="n">Literal</span><span class="p">[</span><span class="s2">&quot;lstm&quot;</span><span class="p">,</span> <span class="s2">&quot;gru&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="s2">&quot;lstm&quot;</span><span class="p">,</span>
+        <span class="n">hidden_dim</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">64</span><span class="p">,</span>
+        <span class="n">bidirectional</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
+        <span class="n">padding_idx</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1</span><span class="p">,</span>
+        <span class="n">n_blocks</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">3</span><span class="p">,</span>
+        <span class="n">attn_concatenate</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
+        <span class="n">attn_dropout</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.1</span><span class="p">,</span>
+        <span class="n">with_addnorm</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
+        <span class="n">head_hidden_dims</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">int</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">head_activation</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;relu&quot;</span><span class="p">,</span>
+        <span class="n">head_dropout</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">head_batchnorm</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
+        <span class="n">head_batchnorm_last</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
+        <span class="n">head_linear_first</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
     <span class="p">):</span>
-        <span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span>
-            <span class="s2">&quot;the input embedding dimension </span><span class="si">{}</span><span class="s2"> and the dimension of the &quot;</span>
-            <span class="s2">&quot;pretrained embeddings </span><span class="si">{}</span><span class="s2"> do not match. The pretrained embeddings &quot;</span>
-            <span class="s2">&quot;dimension (</span><span class="si">{}</span><span class="s2">) will be used&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
-                <span class="n">embed_dim</span><span class="p">,</span> <span class="n">embed_matrix</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">1</span><span class="p">],</span> <span class="n">embed_matrix</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span>
-            <span class="p">),</span>
-            <span class="ne">UserWarning</span><span class="p">,</span>
-        <span class="p">)</span>
+        <span class="nb">super</span><span class="p">(</span><span class="n">StackedAttentiveRNN</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
+
+        <span class="k">if</span> <span class="p">(</span>
+            <span class="n">embed_dim</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span>
+            <span class="ow">and</span> <span class="n">embed_matrix</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span>
+            <span class="ow">and</span> <span class="ow">not</span> <span class="n">embed_dim</span> <span class="o">==</span> <span class="n">embed_matrix</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span>
+        <span class="p">):</span>
+            <span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span>
+                <span class="s2">&quot;the input embedding dimension </span><span class="si">{}</span><span class="s2"> and the dimension of the &quot;</span>
+                <span class="s2">&quot;pretrained embeddings </span><span class="si">{}</span><span class="s2"> do not match. The pretrained embeddings &quot;</span>
+                <span class="s2">&quot;dimension (</span><span class="si">{}</span><span class="s2">) will be used&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
+                    <span class="n">embed_dim</span><span class="p">,</span> <span class="n">embed_matrix</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">1</span><span class="p">],</span> <span class="n">embed_matrix</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span>
+                <span class="p">),</span>
+                <span class="ne">UserWarning</span><span class="p">,</span>
+            <span class="p">)</span>
 
-    <span class="k">if</span> <span class="n">rnn_type</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="ow">not</span> <span class="ow">in</span> <span class="p">[</span><span class="s2">&quot;lstm&quot;</span><span class="p">,</span> <span class="s2">&quot;gru&quot;</span><span class="p">]:</span>
-        <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
-            <span class="sa">f</span><span class="s2">&quot;&#39;rnn_type&#39; must be &#39;lstm&#39; or &#39;gru&#39;, got </span><span class="si">{</span><span class="n">rnn_type</span><span class="si">}</span><span class="s2"> instead&quot;</span>
-        <span class="p">)</span>
+        <span class="k">if</span> <span class="n">rnn_type</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="ow">not</span> <span class="ow">in</span> <span class="p">[</span><span class="s2">&quot;lstm&quot;</span><span class="p">,</span> <span class="s2">&quot;gru&quot;</span><span class="p">]:</span>
+            <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
+                <span class="sa">f</span><span class="s2">&quot;&#39;rnn_type&#39; must be &#39;lstm&#39; or &#39;gru&#39;, got </span><span class="si">{</span><span class="n">rnn_type</span><span class="si">}</span><span class="s2"> instead&quot;</span>
+            <span class="p">)</span>
 
-    <span class="bp">self</span><span class="o">.</span><span class="n">vocab_size</span> <span class="o">=</span> <span class="n">vocab_size</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">embed_trainable</span> <span class="o">=</span> <span class="n">embed_trainable</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">embed_dim</span> <span class="o">=</span> <span class="n">embed_dim</span>
-
-    <span class="bp">self</span><span class="o">.</span><span class="n">rnn_type</span> <span class="o">=</span> <span class="n">rnn_type</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">hidden_dim</span> <span class="o">=</span> <span class="n">hidden_dim</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">bidirectional</span> <span class="o">=</span> <span class="n">bidirectional</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">padding_idx</span> <span class="o">=</span> <span class="n">padding_idx</span>
-
-    <span class="bp">self</span><span class="o">.</span><span class="n">n_blocks</span> <span class="o">=</span> <span class="n">n_blocks</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">attn_concatenate</span> <span class="o">=</span> <span class="n">attn_concatenate</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">attn_dropout</span> <span class="o">=</span> <span class="n">attn_dropout</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">with_addnorm</span> <span class="o">=</span> <span class="n">with_addnorm</span>
-
-    <span class="bp">self</span><span class="o">.</span><span class="n">head_hidden_dims</span> <span class="o">=</span> <span class="n">head_hidden_dims</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">head_activation</span> <span class="o">=</span> <span class="n">head_activation</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">head_dropout</span> <span class="o">=</span> <span class="n">head_dropout</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">head_batchnorm</span> <span class="o">=</span> <span class="n">head_batchnorm</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">head_batchnorm_last</span> <span class="o">=</span> <span class="n">head_batchnorm_last</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">head_linear_first</span> <span class="o">=</span> <span class="n">head_linear_first</span>
-
-    <span class="c1"># Embeddings</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">word_embed</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">embed_dim</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_set_embeddings</span><span class="p">(</span><span class="n">embed_matrix</span><span class="p">)</span>
-
-    <span class="c1"># Linear Projection: if embed_dim is different that the input of the</span>
-    <span class="c1"># attention blocks we add a linear projection</span>
-    <span class="k">if</span> <span class="n">bidirectional</span> <span class="ow">and</span> <span class="n">attn_concatenate</span><span class="p">:</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">rnn_output_dim</span> <span class="o">=</span> <span class="n">hidden_dim</span> <span class="o">*</span> <span class="mi">4</span>
-    <span class="k">elif</span> <span class="n">bidirectional</span> <span class="ow">or</span> <span class="n">attn_concatenate</span><span class="p">:</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">rnn_output_dim</span> <span class="o">=</span> <span class="n">hidden_dim</span> <span class="o">*</span> <span class="mi">2</span>
-    <span class="k">else</span><span class="p">:</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">rnn_output_dim</span> <span class="o">=</span> <span class="n">hidden_dim</span>
-
-    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">rnn_output_dim</span> <span class="o">!=</span> <span class="bp">self</span><span class="o">.</span><span class="n">embed_dim</span><span class="p">:</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">embed_proj</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="n">nn</span><span class="o">.</span><span class="n">Linear</span><span class="p">,</span> <span class="n">nn</span><span class="o">.</span><span class="n">Identity</span><span class="p">]</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Linear</span><span class="p">(</span>
-            <span class="bp">self</span><span class="o">.</span><span class="n">embed_dim</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">rnn_output_dim</span>
-        <span class="p">)</span>
-    <span class="k">else</span><span class="p">:</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">embed_proj</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Identity</span><span class="p">()</span>
-
-    <span class="c1"># RNN</span>
-    <span class="n">rnn_params</span> <span class="o">=</span> <span class="p">{</span>
-        <span class="s2">&quot;input_size&quot;</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">rnn_output_dim</span><span class="p">,</span>
-        <span class="s2">&quot;hidden_size&quot;</span><span class="p">:</span> <span class="n">hidden_dim</span><span class="p">,</span>
-        <span class="s2">&quot;bidirectional&quot;</span><span class="p">:</span> <span class="n">bidirectional</span><span class="p">,</span>
-        <span class="s2">&quot;batch_first&quot;</span><span class="p">:</span> <span class="kc">True</span><span class="p">,</span>
-    <span class="p">}</span>
-    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">rnn_type</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="o">==</span> <span class="s2">&quot;lstm&quot;</span><span class="p">:</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">rnn</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="n">nn</span><span class="o">.</span><span class="n">LSTM</span><span class="p">,</span> <span class="n">nn</span><span class="o">.</span><span class="n">GRU</span><span class="p">]</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">LSTM</span><span class="p">(</span><span class="o">**</span><span class="n">rnn_params</span><span class="p">)</span>
-    <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">rnn_type</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="o">==</span> <span class="s2">&quot;gru&quot;</span><span class="p">:</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">rnn</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">GRU</span><span class="p">(</span><span class="o">**</span><span class="n">rnn_params</span><span class="p">)</span>
-
-    <span class="c1"># FC-Head (Mlp)</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">attention_blks</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">ModuleList</span><span class="p">()</span>
-    <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">n_blocks</span><span class="p">):</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">attention_blks</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
-            <span class="n">ContextAttentionEncoder</span><span class="p">(</span>
-                <span class="bp">self</span><span class="o">.</span><span class="n">rnn</span><span class="p">,</span>
-                <span class="bp">self</span><span class="o">.</span><span class="n">rnn_output_dim</span><span class="p">,</span>
-                <span class="n">attn_dropout</span><span class="p">,</span>
-                <span class="n">attn_concatenate</span><span class="p">,</span>
-                <span class="n">with_addnorm</span><span class="o">=</span><span class="n">with_addnorm</span> <span class="k">if</span> <span class="n">i</span> <span class="o">!=</span> <span class="n">n_blocks</span> <span class="o">-</span> <span class="mi">1</span> <span class="k">else</span> <span class="kc">False</span><span class="p">,</span>
-                <span class="n">sum_along_seq</span><span class="o">=</span><span class="n">i</span> <span class="o">==</span> <span class="n">n_blocks</span> <span class="o">-</span> <span class="mi">1</span><span class="p">,</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">vocab_size</span> <span class="o">=</span> <span class="n">vocab_size</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">embed_trainable</span> <span class="o">=</span> <span class="n">embed_trainable</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">embed_dim</span> <span class="o">=</span> <span class="n">embed_dim</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">rnn_type</span> <span class="o">=</span> <span class="n">rnn_type</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">hidden_dim</span> <span class="o">=</span> <span class="n">hidden_dim</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">bidirectional</span> <span class="o">=</span> <span class="n">bidirectional</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">padding_idx</span> <span class="o">=</span> <span class="n">padding_idx</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">n_blocks</span> <span class="o">=</span> <span class="n">n_blocks</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">attn_concatenate</span> <span class="o">=</span> <span class="n">attn_concatenate</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">attn_dropout</span> <span class="o">=</span> <span class="n">attn_dropout</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">with_addnorm</span> <span class="o">=</span> <span class="n">with_addnorm</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">head_hidden_dims</span> <span class="o">=</span> <span class="n">head_hidden_dims</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">head_activation</span> <span class="o">=</span> <span class="n">head_activation</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">head_dropout</span> <span class="o">=</span> <span class="n">head_dropout</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">head_batchnorm</span> <span class="o">=</span> <span class="n">head_batchnorm</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">head_batchnorm_last</span> <span class="o">=</span> <span class="n">head_batchnorm_last</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">head_linear_first</span> <span class="o">=</span> <span class="n">head_linear_first</span>
+
+        <span class="c1"># Embeddings</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">word_embed</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">embed_dim</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_set_embeddings</span><span class="p">(</span><span class="n">embed_matrix</span><span class="p">)</span>
+
+        <span class="c1"># Linear Projection: if embed_dim is different that the input of the</span>
+        <span class="c1"># attention blocks we add a linear projection</span>
+        <span class="k">if</span> <span class="n">bidirectional</span> <span class="ow">and</span> <span class="n">attn_concatenate</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">rnn_output_dim</span> <span class="o">=</span> <span class="n">hidden_dim</span> <span class="o">*</span> <span class="mi">4</span>
+        <span class="k">elif</span> <span class="n">bidirectional</span> <span class="ow">or</span> <span class="n">attn_concatenate</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">rnn_output_dim</span> <span class="o">=</span> <span class="n">hidden_dim</span> <span class="o">*</span> <span class="mi">2</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">rnn_output_dim</span> <span class="o">=</span> <span class="n">hidden_dim</span>
+
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">rnn_output_dim</span> <span class="o">!=</span> <span class="bp">self</span><span class="o">.</span><span class="n">embed_dim</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">embed_proj</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="n">nn</span><span class="o">.</span><span class="n">Linear</span><span class="p">,</span> <span class="n">nn</span><span class="o">.</span><span class="n">Identity</span><span class="p">]</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Linear</span><span class="p">(</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">embed_dim</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">rnn_output_dim</span>
+            <span class="p">)</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">embed_proj</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Identity</span><span class="p">()</span>
+
+        <span class="c1"># RNN</span>
+        <span class="n">rnn_params</span> <span class="o">=</span> <span class="p">{</span>
+            <span class="s2">&quot;input_size&quot;</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">rnn_output_dim</span><span class="p">,</span>
+            <span class="s2">&quot;hidden_size&quot;</span><span class="p">:</span> <span class="n">hidden_dim</span><span class="p">,</span>
+            <span class="s2">&quot;bidirectional&quot;</span><span class="p">:</span> <span class="n">bidirectional</span><span class="p">,</span>
+            <span class="s2">&quot;batch_first&quot;</span><span class="p">:</span> <span class="kc">True</span><span class="p">,</span>
+        <span class="p">}</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">rnn_type</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="o">==</span> <span class="s2">&quot;lstm&quot;</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">rnn</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="n">nn</span><span class="o">.</span><span class="n">LSTM</span><span class="p">,</span> <span class="n">nn</span><span class="o">.</span><span class="n">GRU</span><span class="p">]</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">LSTM</span><span class="p">(</span><span class="o">**</span><span class="n">rnn_params</span><span class="p">)</span>
+        <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">rnn_type</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="o">==</span> <span class="s2">&quot;gru&quot;</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">rnn</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">GRU</span><span class="p">(</span><span class="o">**</span><span class="n">rnn_params</span><span class="p">)</span>
+
+        <span class="c1"># FC-Head (Mlp)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">attention_blks</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">ModuleList</span><span class="p">()</span>
+        <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">n_blocks</span><span class="p">):</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">attention_blks</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
+                <span class="n">ContextAttentionEncoder</span><span class="p">(</span>
+                    <span class="bp">self</span><span class="o">.</span><span class="n">rnn</span><span class="p">,</span>
+                    <span class="bp">self</span><span class="o">.</span><span class="n">rnn_output_dim</span><span class="p">,</span>
+                    <span class="n">attn_dropout</span><span class="p">,</span>
+                    <span class="n">attn_concatenate</span><span class="p">,</span>
+                    <span class="n">with_addnorm</span><span class="o">=</span><span class="n">with_addnorm</span> <span class="k">if</span> <span class="n">i</span> <span class="o">!=</span> <span class="n">n_blocks</span> <span class="o">-</span> <span class="mi">1</span> <span class="k">else</span> <span class="kc">False</span><span class="p">,</span>
+                    <span class="n">sum_along_seq</span><span class="o">=</span><span class="n">i</span> <span class="o">==</span> <span class="n">n_blocks</span> <span class="o">-</span> <span class="mi">1</span><span class="p">,</span>
+                <span class="p">)</span>
             <span class="p">)</span>
-        <span class="p">)</span>
 
-    <span class="c1"># Mlp</span>
-    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">head_hidden_dims</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
-        <span class="n">head_hidden_dims</span> <span class="o">=</span> <span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">rnn_output_dim</span><span class="p">]</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">head_hidden_dims</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">rnn_mlp</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="n">MLP</span><span class="p">,</span> <span class="n">nn</span><span class="o">.</span><span class="n">Identity</span><span class="p">]</span> <span class="o">=</span> <span class="n">MLP</span><span class="p">(</span>
-            <span class="n">head_hidden_dims</span><span class="p">,</span>
-            <span class="n">head_activation</span><span class="p">,</span>
-            <span class="n">head_dropout</span><span class="p">,</span>
-            <span class="n">head_batchnorm</span><span class="p">,</span>
-            <span class="n">head_batchnorm_last</span><span class="p">,</span>
-            <span class="n">head_linear_first</span><span class="p">,</span>
+        <span class="c1"># Mlp</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">head_hidden_dims</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="n">head_hidden_dims</span> <span class="o">=</span> <span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">rnn_output_dim</span><span class="p">]</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">head_hidden_dims</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">rnn_mlp</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="n">MLP</span><span class="p">,</span> <span class="n">nn</span><span class="o">.</span><span class="n">Identity</span><span class="p">]</span> <span class="o">=</span> <span class="n">MLP</span><span class="p">(</span>
+                <span class="n">head_hidden_dims</span><span class="p">,</span>
+                <span class="n">head_activation</span><span class="p">,</span>
+                <span class="n">head_dropout</span><span class="p">,</span>
+                <span class="n">head_batchnorm</span><span class="p">,</span>
+                <span class="n">head_batchnorm_last</span><span class="p">,</span>
+                <span class="n">head_linear_first</span><span class="p">,</span>
+            <span class="p">)</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="c1"># simple hack to add readability in the forward pass</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">rnn_mlp</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Identity</span><span class="p">()</span>
+
+    <span class="k">def</span> <span class="nf">forward</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">X</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tensor</span><span class="p">:</span>  <span class="c1"># type: ignore</span>
+        <span class="n">x</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">embed_proj</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">word_embed</span><span class="p">(</span><span class="n">X</span><span class="o">.</span><span class="n">long</span><span class="p">()))</span>
+
+        <span class="n">h</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">init</span><span class="o">.</span><span class="n">zeros_</span><span class="p">(</span>
+            <span class="n">torch</span><span class="o">.</span><span class="n">Tensor</span><span class="p">(</span><span class="mi">2</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">bidirectional</span> <span class="k">else</span> <span class="mi">1</span><span class="p">,</span> <span class="n">X</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="bp">self</span><span class="o">.</span><span class="n">hidden_dim</span><span class="p">)</span>
+        <span class="p">)</span><span class="o">.</span><span class="n">to</span><span class="p">(</span><span class="n">x</span><span class="o">.</span><span class="n">device</span><span class="p">)</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">rnn_type</span> <span class="o">==</span> <span class="s2">&quot;lstm&quot;</span><span class="p">:</span>
+            <span class="n">c</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">init</span><span class="o">.</span><span class="n">zeros_</span><span class="p">(</span>
+                <span class="n">torch</span><span class="o">.</span><span class="n">Tensor</span><span class="p">(</span>
+                    <span class="mi">2</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">bidirectional</span> <span class="k">else</span> <span class="mi">1</span><span class="p">,</span> <span class="n">X</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="bp">self</span><span class="o">.</span><span class="n">hidden_dim</span>
+                <span class="p">)</span>
+            <span class="p">)</span><span class="o">.</span><span class="n">to</span><span class="p">(</span><span class="n">x</span><span class="o">.</span><span class="n">device</span><span class="p">)</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="n">c</span> <span class="o">=</span> <span class="kc">None</span>
+
+        <span class="k">for</span> <span class="n">blk</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">attention_blks</span><span class="p">:</span>
+            <span class="n">x</span><span class="p">,</span> <span class="n">h</span><span class="p">,</span> <span class="n">c</span> <span class="o">=</span> <span class="n">blk</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">h</span><span class="p">,</span> <span class="n">c</span><span class="p">)</span>
+
+        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">rnn_mlp</span><span class="p">(</span><span class="n">x</span><span class="p">)</span>
+
+    <span class="nd">@property</span>
+    <span class="k">def</span> <span class="nf">output_dim</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">int</span><span class="p">:</span>
+<span class="w">        </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;The output dimension of the model. This is a required property</span>
+<span class="sd">        neccesary to build the `WideDeep` class</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="k">return</span> <span class="p">(</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">head_hidden_dims</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span>
+            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">head_hidden_dims</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span>
+            <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">rnn_output_dim</span>
         <span class="p">)</span>
-    <span class="k">else</span><span class="p">:</span>
-        <span class="c1"># simple hack to add readability in the forward pass</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">rnn_mlp</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Identity</span><span class="p">()</span>
+
+    <span class="nd">@property</span>
+    <span class="k">def</span> <span class="nf">attention_weights</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">List</span><span class="p">:</span>
+<span class="w">        </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;List with the attention weights per block</span>
+
+<span class="sd">        The shape of the attention weights is $(N, S)$ Where $N$ is the batch</span>
+<span class="sd">        size and $S$ is the length of the sequence</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="k">return</span> <span class="p">[</span><span class="n">blk</span><span class="o">.</span><span class="n">attn</span><span class="o">.</span><span class="n">attn_weights</span> <span class="k">for</span> <span class="n">blk</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">attention_blks</span><span class="p">]</span>
+
+    <span class="k">def</span> <span class="nf">_set_embeddings</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span> <span class="n">embed_matrix</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="n">Any</span><span class="p">,</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">]</span>
+    <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tuple</span><span class="p">[</span><span class="n">nn</span><span class="o">.</span><span class="n">Module</span><span class="p">,</span> <span class="nb">int</span><span class="p">]:</span>
+        <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">embed_matrix</span><span class="p">,</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">):</span>
+            <span class="k">assert</span> <span class="p">(</span>
+                <span class="n">embed_matrix</span><span class="o">.</span><span class="n">dtype</span> <span class="o">==</span> <span class="s2">&quot;float32&quot;</span>
+            <span class="p">),</span> <span class="s2">&quot;&#39;embed_matrix&#39; must be of dtype &#39;float32&#39;, got dtype &#39;</span><span class="si">{}</span><span class="s2">&#39;&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
+                <span class="nb">str</span><span class="p">(</span><span class="n">embed_matrix</span><span class="o">.</span><span class="n">dtype</span><span class="p">)</span>
+            <span class="p">)</span>
+            <span class="n">word_embed</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Embedding</span><span class="p">(</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">vocab_size</span><span class="p">,</span> <span class="n">embed_matrix</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">1</span><span class="p">],</span> <span class="n">padding_idx</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">padding_idx</span>
+            <span class="p">)</span>
+            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">embed_trainable</span><span class="p">:</span>
+                <span class="n">word_embed</span><span class="o">.</span><span class="n">weight</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Parameter</span><span class="p">(</span>
+                    <span class="n">torch</span><span class="o">.</span><span class="n">tensor</span><span class="p">(</span><span class="n">embed_matrix</span><span class="p">),</span> <span class="n">requires_grad</span><span class="o">=</span><span class="kc">True</span>
+                <span class="p">)</span>
+            <span class="k">else</span><span class="p">:</span>
+                <span class="n">word_embed</span><span class="o">.</span><span class="n">weight</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Parameter</span><span class="p">(</span>
+                    <span class="n">torch</span><span class="o">.</span><span class="n">tensor</span><span class="p">(</span><span class="n">embed_matrix</span><span class="p">),</span> <span class="n">requires_grad</span><span class="o">=</span><span class="kc">False</span>
+                <span class="p">)</span>
+            <span class="n">embed_dim</span> <span class="o">=</span> <span class="n">embed_matrix</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="k">assert</span> <span class="bp">self</span><span class="o">.</span><span class="n">embed_dim</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span>
+            <span class="n">word_embed</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Embedding</span><span class="p">(</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">vocab_size</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">embed_dim</span><span class="p">,</span> <span class="n">padding_idx</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">padding_idx</span>
+            <span class="p">)</span>
+            <span class="n">embed_dim</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">embed_dim</span>
+
+        <span class="k">return</span> <span class="n">word_embed</span><span class="p">,</span> <span class="n">embed_dim</span>
 </code></pre></div></td></tr></table></div>
-                  </details>
+              </details>
 
 
 
@@ -11800,20 +18686,7 @@ <h2 id="pytorch_widedeep.models.text.huggingface_transformers.hf_model.HFModel"
 
 
 <a href="#pytorch_widedeep.models.text.huggingface_transformers.hf_model.HFModel" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="nf">HFModel</span><span class="p">(</span>
-    <span class="n">model_name</span><span class="p">,</span>
-    <span class="n">use_cls_token</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
-    <span class="n">trainable_parameters</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">head_hidden_dims</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">head_activation</span><span class="o">=</span><span class="s2">&quot;relu&quot;</span><span class="p">,</span>
-    <span class="n">head_dropout</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">head_batchnorm</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
-    <span class="n">head_batchnorm_last</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
-    <span class="n">head_linear_first</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
-    <span class="n">verbose</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
-    <span class="o">**</span><span class="n">kwargs</span>
-<span class="p">)</span>
-</code></pre></div>
+
 
     <div class="doc doc-contents first">
             <p class="doc doc-class-bases">
@@ -11831,136 +18704,206 @@ <h2 id="pytorch_widedeep.models.text.huggingface_transformers.hf_model.HFModel"
 
 
 <p><span class="doc-section-title">Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>model_name</code></b>
-              (<code>str</code>)
-          –
-          <div class="doc-md-description">
-            <p>The model name from the transformers library e.g. 'bert-base-uncased'.
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+          <th>Default</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>model_name</code></td>
+            <td>
+                  <code>str</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>The model name from the transformers library e.g. 'bert-base-uncased'.
 Currently supported models are those from the families: BERT, RoBERTa,
 DistilBERT, ALBERT and ELECTRA.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>use_cls_token</code></b>
-              (<code>bool</code>, default:
-                  <code>True</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Boolean indicating whether to use the [CLS] token or the mean of the
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>use_cls_token</code></td>
+            <td>
+                  <code>bool</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Boolean indicating whether to use the [CLS] token or the mean of the
 sequence of hidden states as the sentence embedding</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>trainable_parameters</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[str]]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>List with the names of the model parameters that will be trained. If
+              </div>
+            </td>
+            <td>
+                  <code>True</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>trainable_parameters</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[str]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>List with the names of the model parameters that will be trained. If
 None, none of the parameters will be trainable</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>head_hidden_dims</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[int]]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>List with the sizes of the dense layers in the head e.g: <em>[128, 64]</em></p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>head_activation</code></b>
-              (<code>str</code>, default:
-                  <code>&#39;relu&#39;</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Activation function for the dense layers in the head. Currently
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>head_hidden_dims</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[int]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>List with the sizes of the dense layers in the head e.g: <em>[128, 64]</em></p>
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>head_activation</code></td>
+            <td>
+                  <code>str</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Activation function for the dense layers in the head. Currently
 <em>'tanh'</em>, <em>'relu'</em>, <em>'leaky_relu'</em> and <em>'gelu'</em> are supported</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>head_dropout</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Dropout of the dense layers in the head</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>head_batchnorm</code></b>
-              (<code>bool</code>, default:
-                  <code>False</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Boolean indicating whether or not to include batch normalization in the
+              </div>
+            </td>
+            <td>
+                  <code>&#39;relu&#39;</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>head_dropout</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Dropout of the dense layers in the head</p>
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>head_batchnorm</code></td>
+            <td>
+                  <code>bool</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Boolean indicating whether or not to include batch normalization in the
 dense layers that form the head</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>head_batchnorm_last</code></b>
-              (<code>bool</code>, default:
+              </div>
+            </td>
+            <td>
                   <code>False</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Boolean indicating whether or not to apply batch normalization to the
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>head_batchnorm_last</code></td>
+            <td>
+                  <code>bool</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Boolean indicating whether or not to apply batch normalization to the
 last of the dense layers in the head</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>head_linear_first</code></b>
-              (<code>bool</code>, default:
+              </div>
+            </td>
+            <td>
                   <code>False</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Boolean indicating whether the order of the operations in the dense
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>head_linear_first</code></td>
+            <td>
+                  <code>bool</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Boolean indicating whether the order of the operations in the dense
 layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt;
 LIN -&gt; ACT]</code></p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>verbose</code></b>
-              (<code>bool</code>, default:
+              </div>
+            </td>
+            <td>
                   <code>False</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>If True, it will print information about the model</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>**kwargs</code></b>
-          –
-          <div class="doc-md-description">
-            <p>Additional kwargs to be passed to the model</p>
-          </div>
-        </li>
-    </ul>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>verbose</code></td>
+            <td>
+                  <code>bool</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>If True, it will print information about the model</p>
+              </div>
+            </td>
+            <td>
+                  <code>False</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>**kwargs</code></td>
+            <td>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Additional kwargs to be passed to the model</p>
+              </div>
+            </td>
+            <td>
+                  <code>{}</code>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
 
 <p><span class="doc-section-title">Attributes:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code><span title="pytorch_widedeep.models.text.huggingface_transformers.hf_model.HFModel.head">head</span></code></b>
-              (<code><span title="nn.Module">Module</span></code>)
-          –
-          <div class="doc-md-description">
-            <p>Stack of dense layers on top of the transformer. This will only exists
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code><span title="pytorch_widedeep.models.text.huggingface_transformers.hf_model.HFModel.head">head</span></code></td>
+            <td>
+                  <code><span title="nn.Module">Module</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Stack of dense layers on top of the transformer. This will only exists
 if <code>head_layers_dim</code> is not None</p>
-          </div>
-        </li>
-    </ul>
+              </div>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
 
 <p><span class="doc-section-title">Examples:</span></p>
@@ -11971,9 +18914,69 @@ <h2 id="pytorch_widedeep.models.text.huggingface_transformers.hf_model.HFModel"
 <span class="gp">&gt;&gt;&gt; </span><span class="n">out</span> <span class="o">=</span> <span class="n">model</span><span class="p">(</span><span class="n">X_text</span><span class="p">)</span>
 </code></pre></div>
 
-                  <details class="quote">
-                    <summary>Source code in <code>pytorch_widedeep/models/text/huggingface_transformers/hf_model.py</code></summary>
-                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 72</span>
+              <details class="quote">
+                <summary>Source code in <code>pytorch_widedeep/models/text/huggingface_transformers/hf_model.py</code></summary>
+                <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 12</span>
+<span class="normal"> 13</span>
+<span class="normal"> 14</span>
+<span class="normal"> 15</span>
+<span class="normal"> 16</span>
+<span class="normal"> 17</span>
+<span class="normal"> 18</span>
+<span class="normal"> 19</span>
+<span class="normal"> 20</span>
+<span class="normal"> 21</span>
+<span class="normal"> 22</span>
+<span class="normal"> 23</span>
+<span class="normal"> 24</span>
+<span class="normal"> 25</span>
+<span class="normal"> 26</span>
+<span class="normal"> 27</span>
+<span class="normal"> 28</span>
+<span class="normal"> 29</span>
+<span class="normal"> 30</span>
+<span class="normal"> 31</span>
+<span class="normal"> 32</span>
+<span class="normal"> 33</span>
+<span class="normal"> 34</span>
+<span class="normal"> 35</span>
+<span class="normal"> 36</span>
+<span class="normal"> 37</span>
+<span class="normal"> 38</span>
+<span class="normal"> 39</span>
+<span class="normal"> 40</span>
+<span class="normal"> 41</span>
+<span class="normal"> 42</span>
+<span class="normal"> 43</span>
+<span class="normal"> 44</span>
+<span class="normal"> 45</span>
+<span class="normal"> 46</span>
+<span class="normal"> 47</span>
+<span class="normal"> 48</span>
+<span class="normal"> 49</span>
+<span class="normal"> 50</span>
+<span class="normal"> 51</span>
+<span class="normal"> 52</span>
+<span class="normal"> 53</span>
+<span class="normal"> 54</span>
+<span class="normal"> 55</span>
+<span class="normal"> 56</span>
+<span class="normal"> 57</span>
+<span class="normal"> 58</span>
+<span class="normal"> 59</span>
+<span class="normal"> 60</span>
+<span class="normal"> 61</span>
+<span class="normal"> 62</span>
+<span class="normal"> 63</span>
+<span class="normal"> 64</span>
+<span class="normal"> 65</span>
+<span class="normal"> 66</span>
+<span class="normal"> 67</span>
+<span class="normal"> 68</span>
+<span class="normal"> 69</span>
+<span class="normal"> 70</span>
+<span class="normal"> 71</span>
+<span class="normal"> 72</span>
 <span class="normal"> 73</span>
 <span class="normal"> 74</span>
 <span class="normal"> 75</span>
@@ -12031,97 +19034,261 @@ <h2 id="pytorch_widedeep.models.text.huggingface_transformers.hf_model.HFModel"
 <span class="normal">127</span>
 <span class="normal">128</span>
 <span class="normal">129</span>
-<span class="normal">130</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="nd">@alias</span><span class="p">(</span><span class="s2">&quot;use_cls_token&quot;</span><span class="p">,</span> <span class="p">[</span><span class="s2">&quot;use_special_token&quot;</span><span class="p">])</span>
-<span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
-    <span class="bp">self</span><span class="p">,</span>
-    <span class="n">model_name</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
-    <span class="n">use_cls_token</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">,</span>
-    <span class="n">trainable_parameters</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">head_hidden_dims</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">int</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">head_activation</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;relu&quot;</span><span class="p">,</span>
-    <span class="n">head_dropout</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">head_batchnorm</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
-    <span class="n">head_batchnorm_last</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
-    <span class="n">head_linear_first</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
-    <span class="n">verbose</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
-    <span class="o">**</span><span class="n">kwargs</span><span class="p">,</span>
-<span class="p">):</span>
-    <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
-
-    <span class="c1"># TO DO: add warning regarging ELECTRA as ELECTRA does not have a cls</span>
-    <span class="c1"># token.  Research what happens with ELECTRA</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">model_name</span> <span class="o">=</span> <span class="n">model_name</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">use_cls_token</span> <span class="o">=</span> <span class="n">use_cls_token</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">trainable_parameters</span> <span class="o">=</span> <span class="n">trainable_parameters</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">head_hidden_dims</span> <span class="o">=</span> <span class="n">head_hidden_dims</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">head_activation</span> <span class="o">=</span> <span class="n">head_activation</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">head_dropout</span> <span class="o">=</span> <span class="n">head_dropout</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">head_batchnorm</span> <span class="o">=</span> <span class="n">head_batchnorm</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">head_batchnorm_last</span> <span class="o">=</span> <span class="n">head_batchnorm_last</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">head_linear_first</span> <span class="o">=</span> <span class="n">head_linear_first</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">verbose</span> <span class="o">=</span> <span class="n">verbose</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">kwargs</span> <span class="o">=</span> <span class="n">kwargs</span>
-
-    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">verbose</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">use_cls_token</span><span class="p">:</span>
-        <span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span>
-            <span class="s2">&quot;The model will use the [CLS] token. Make sure the tokenizer &quot;</span>
-            <span class="s2">&quot;was run with add_special_tokens=True&quot;</span><span class="p">,</span>
-            <span class="ne">UserWarning</span><span class="p">,</span>
-        <span class="p">)</span>
-
-    <span class="bp">self</span><span class="o">.</span><span class="n">model_class</span> <span class="o">=</span> <span class="n">get_model_class</span><span class="p">(</span><span class="n">model_name</span><span class="p">)</span>
-
-    <span class="bp">self</span><span class="o">.</span><span class="n">config</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">model</span> <span class="o">=</span> <span class="n">get_config_and_model</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">model_name</span><span class="p">)</span>
-
-    <span class="bp">self</span><span class="o">.</span><span class="n">output_attention_weights</span> <span class="o">=</span> <span class="n">kwargs</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;output_attentions&quot;</span><span class="p">,</span> <span class="kc">False</span><span class="p">)</span>
-
-    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">trainable_parameters</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
-        <span class="k">for</span> <span class="n">n</span><span class="p">,</span> <span class="n">p</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">model</span><span class="o">.</span><span class="n">named_parameters</span><span class="p">():</span>
-            <span class="n">p</span><span class="o">.</span><span class="n">requires_grad</span> <span class="o">=</span> <span class="nb">any</span><span class="p">([</span><span class="n">tl</span> <span class="ow">in</span> <span class="n">n</span> <span class="k">for</span> <span class="n">tl</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">trainable_parameters</span><span class="p">])</span>
-
-    <span class="c1"># FC-Head (Mlp). Note that the FC head will always be trainable</span>
-    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">head_hidden_dims</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
-        <span class="n">head_hidden_dims</span> <span class="o">=</span> <span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">config</span><span class="o">.</span><span class="n">hidden_size</span><span class="p">]</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">head_hidden_dims</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">head</span> <span class="o">=</span> <span class="n">MLP</span><span class="p">(</span>
-            <span class="n">head_hidden_dims</span><span class="p">,</span>
-            <span class="n">head_activation</span><span class="p">,</span>
-            <span class="n">head_dropout</span><span class="p">,</span>
-            <span class="n">head_batchnorm</span><span class="p">,</span>
-            <span class="n">head_batchnorm_last</span><span class="p">,</span>
-            <span class="n">head_linear_first</span><span class="p">,</span>
-        <span class="p">)</span>
-</code></pre></div></td></tr></table></div>
-                  </details>
-
-
-
-  <div class="doc doc-children">
-
-
-
-
-
-
-
-<div class="doc doc-object doc-attribute">
-
-
-
-<h3 id="pytorch_widedeep.models.text.huggingface_transformers.hf_model.HFModel.attention_weight" class="doc doc-heading">
-            <span class="doc doc-object-name doc-attribute-name">attention_weight</span>
-
-
-  <span class="doc doc-labels">
-      <small class="doc doc-label doc-label-property"><code>property</code></small>
-  </span>
-
-<a href="#pytorch_widedeep.models.text.huggingface_transformers.hf_model.HFModel.attention_weight" class="headerlink" title="Permanent link">&para;</a></h3>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">attention_weight</span>
-</code></pre></div>
-
-    <div class="doc doc-contents ">
-
-        <p>Returns the attention weights if the model was created with the
+<span class="normal">130</span>
+<span class="normal">131</span>
+<span class="normal">132</span>
+<span class="normal">133</span>
+<span class="normal">134</span>
+<span class="normal">135</span>
+<span class="normal">136</span>
+<span class="normal">137</span>
+<span class="normal">138</span>
+<span class="normal">139</span>
+<span class="normal">140</span>
+<span class="normal">141</span>
+<span class="normal">142</span>
+<span class="normal">143</span>
+<span class="normal">144</span>
+<span class="normal">145</span>
+<span class="normal">146</span>
+<span class="normal">147</span>
+<span class="normal">148</span>
+<span class="normal">149</span>
+<span class="normal">150</span>
+<span class="normal">151</span>
+<span class="normal">152</span>
+<span class="normal">153</span>
+<span class="normal">154</span>
+<span class="normal">155</span>
+<span class="normal">156</span>
+<span class="normal">157</span>
+<span class="normal">158</span>
+<span class="normal">159</span>
+<span class="normal">160</span>
+<span class="normal">161</span>
+<span class="normal">162</span>
+<span class="normal">163</span>
+<span class="normal">164</span>
+<span class="normal">165</span>
+<span class="normal">166</span>
+<span class="normal">167</span>
+<span class="normal">168</span>
+<span class="normal">169</span>
+<span class="normal">170</span>
+<span class="normal">171</span>
+<span class="normal">172</span>
+<span class="normal">173</span>
+<span class="normal">174</span>
+<span class="normal">175</span>
+<span class="normal">176</span>
+<span class="normal">177</span>
+<span class="normal">178</span>
+<span class="normal">179</span>
+<span class="normal">180</span>
+<span class="normal">181</span>
+<span class="normal">182</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">class</span> <span class="nc">HFModel</span><span class="p">(</span><span class="n">BaseWDModelComponent</span><span class="p">):</span>
+<span class="w">    </span><span class="sd">&quot;&quot;&quot;This class is a wrapper around the Hugging Face transformers library. It</span>
+<span class="sd">    can be used as the text component of a Wide &amp; Deep model or independently</span>
+<span class="sd">    by itself.</span>
+
+<span class="sd">    At the moment only models from the families BERT, RoBERTa, DistilBERT,</span>
+<span class="sd">    ALBERT and ELECTRA are supported. This is because this library is</span>
+<span class="sd">    designed to address classification and regression tasks and these are the</span>
+<span class="sd">    most &#39;popular&#39; encoder-only models, which have proved to be those that</span>
+<span class="sd">    work best for these tasks.</span>
+
+<span class="sd">    Parameters</span>
+<span class="sd">    ----------</span>
+<span class="sd">    model_name: str</span>
+<span class="sd">        The model name from the transformers library e.g. &#39;bert-base-uncased&#39;.</span>
+<span class="sd">        Currently supported models are those from the families: BERT, RoBERTa,</span>
+<span class="sd">        DistilBERT, ALBERT and ELECTRA.</span>
+<span class="sd">    use_cls_token: bool, default = True</span>
+<span class="sd">        Boolean indicating whether to use the [CLS] token or the mean of the</span>
+<span class="sd">        sequence of hidden states as the sentence embedding</span>
+<span class="sd">    trainable_parameters: List, Optional, default = None</span>
+<span class="sd">        List with the names of the model parameters that will be trained. If</span>
+<span class="sd">        None, none of the parameters will be trainable</span>
+<span class="sd">    head_hidden_dims: List, Optional, default = None</span>
+<span class="sd">        List with the sizes of the dense layers in the head e.g: _[128, 64]_</span>
+<span class="sd">    head_activation: str, default = &quot;relu&quot;</span>
+<span class="sd">        Activation function for the dense layers in the head. Currently</span>
+<span class="sd">        _&#39;tanh&#39;_, _&#39;relu&#39;_, _&#39;leaky_relu&#39;_ and _&#39;gelu&#39;_ are supported</span>
+<span class="sd">    head_dropout: float, Optional, default = None</span>
+<span class="sd">        Dropout of the dense layers in the head</span>
+<span class="sd">    head_batchnorm: bool, default = False</span>
+<span class="sd">        Boolean indicating whether or not to include batch normalization in the</span>
+<span class="sd">        dense layers that form the head</span>
+<span class="sd">    head_batchnorm_last: bool, default = False</span>
+<span class="sd">        Boolean indicating whether or not to apply batch normalization to the</span>
+<span class="sd">        last of the dense layers in the head</span>
+<span class="sd">    head_linear_first: bool, default = False</span>
+<span class="sd">        Boolean indicating whether the order of the operations in the dense</span>
+<span class="sd">        layer. If `True: [LIN -&gt; ACT -&gt; BN -&gt; DP]`. If `False: [BN -&gt; DP -&gt;</span>
+<span class="sd">        LIN -&gt; ACT]`</span>
+<span class="sd">    verbose: bool, default = False</span>
+<span class="sd">        If True, it will print information about the model</span>
+<span class="sd">    **kwargs</span>
+<span class="sd">        Additional kwargs to be passed to the model</span>
+
+<span class="sd">    Attributes</span>
+<span class="sd">    ----------</span>
+<span class="sd">    head: nn.Module</span>
+<span class="sd">        Stack of dense layers on top of the transformer. This will only exists</span>
+<span class="sd">        if `head_layers_dim` is not None</span>
+
+<span class="sd">    Examples</span>
+<span class="sd">    --------</span>
+<span class="sd">    &gt;&gt;&gt; import torch</span>
+<span class="sd">    &gt;&gt;&gt; from pytorch_widedeep.models import HFModel</span>
+<span class="sd">    &gt;&gt;&gt; X_text = torch.cat((torch.zeros([5,1]), torch.empty(5, 4).random_(1,4)), axis=1).long()</span>
+<span class="sd">    &gt;&gt;&gt; model = HFModel(model_name=&#39;bert-base-uncased&#39;)</span>
+<span class="sd">    &gt;&gt;&gt; out = model(X_text)</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+
+    <span class="nd">@alias</span><span class="p">(</span><span class="s2">&quot;use_cls_token&quot;</span><span class="p">,</span> <span class="p">[</span><span class="s2">&quot;use_special_token&quot;</span><span class="p">])</span>
+    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span>
+        <span class="n">model_name</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
+        <span class="n">use_cls_token</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">,</span>
+        <span class="n">trainable_parameters</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">head_hidden_dims</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">int</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">head_activation</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;relu&quot;</span><span class="p">,</span>
+        <span class="n">head_dropout</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">head_batchnorm</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
+        <span class="n">head_batchnorm_last</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
+        <span class="n">head_linear_first</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
+        <span class="n">verbose</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
+        <span class="o">**</span><span class="n">kwargs</span><span class="p">,</span>
+    <span class="p">):</span>
+        <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
+
+        <span class="c1"># TO DO: add warning regarging ELECTRA as ELECTRA does not have a cls</span>
+        <span class="c1"># token.  Research what happens with ELECTRA</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">model_name</span> <span class="o">=</span> <span class="n">model_name</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">use_cls_token</span> <span class="o">=</span> <span class="n">use_cls_token</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">trainable_parameters</span> <span class="o">=</span> <span class="n">trainable_parameters</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">head_hidden_dims</span> <span class="o">=</span> <span class="n">head_hidden_dims</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">head_activation</span> <span class="o">=</span> <span class="n">head_activation</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">head_dropout</span> <span class="o">=</span> <span class="n">head_dropout</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">head_batchnorm</span> <span class="o">=</span> <span class="n">head_batchnorm</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">head_batchnorm_last</span> <span class="o">=</span> <span class="n">head_batchnorm_last</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">head_linear_first</span> <span class="o">=</span> <span class="n">head_linear_first</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">verbose</span> <span class="o">=</span> <span class="n">verbose</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">kwargs</span> <span class="o">=</span> <span class="n">kwargs</span>
+
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">verbose</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">use_cls_token</span><span class="p">:</span>
+            <span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span>
+                <span class="s2">&quot;The model will use the [CLS] token. Make sure the tokenizer &quot;</span>
+                <span class="s2">&quot;was run with add_special_tokens=True&quot;</span><span class="p">,</span>
+                <span class="ne">UserWarning</span><span class="p">,</span>
+            <span class="p">)</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">model_class</span> <span class="o">=</span> <span class="n">get_model_class</span><span class="p">(</span><span class="n">model_name</span><span class="p">)</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">config</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">model</span> <span class="o">=</span> <span class="n">get_config_and_model</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">model_name</span><span class="p">)</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">output_attention_weights</span> <span class="o">=</span> <span class="n">kwargs</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;output_attentions&quot;</span><span class="p">,</span> <span class="kc">False</span><span class="p">)</span>
+
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">trainable_parameters</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="k">for</span> <span class="n">n</span><span class="p">,</span> <span class="n">p</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">model</span><span class="o">.</span><span class="n">named_parameters</span><span class="p">():</span>
+                <span class="n">p</span><span class="o">.</span><span class="n">requires_grad</span> <span class="o">=</span> <span class="nb">any</span><span class="p">([</span><span class="n">tl</span> <span class="ow">in</span> <span class="n">n</span> <span class="k">for</span> <span class="n">tl</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">trainable_parameters</span><span class="p">])</span>
+
+        <span class="c1"># FC-Head (Mlp). Note that the FC head will always be trainable</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">head_hidden_dims</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="n">head_hidden_dims</span> <span class="o">=</span> <span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">config</span><span class="o">.</span><span class="n">hidden_size</span><span class="p">]</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">head_hidden_dims</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">head</span> <span class="o">=</span> <span class="n">MLP</span><span class="p">(</span>
+                <span class="n">head_hidden_dims</span><span class="p">,</span>
+                <span class="n">head_activation</span><span class="p">,</span>
+                <span class="n">head_dropout</span><span class="p">,</span>
+                <span class="n">head_batchnorm</span><span class="p">,</span>
+                <span class="n">head_batchnorm_last</span><span class="p">,</span>
+                <span class="n">head_linear_first</span><span class="p">,</span>
+            <span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">forward</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">X</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tensor</span><span class="p">:</span>
+
+        <span class="c1"># this is inefficient since the attention mask is returned by the</span>
+        <span class="c1"># tokenizer, but all models in this library use a forward pass that</span>
+        <span class="c1"># takes ONLY an input tensor. A fix will be addressed in a future</span>
+        <span class="n">attn_mask</span> <span class="o">=</span> <span class="p">(</span><span class="n">X</span> <span class="o">!=</span> <span class="mi">0</span><span class="p">)</span><span class="o">.</span><span class="n">type</span><span class="p">(</span><span class="n">torch</span><span class="o">.</span><span class="n">int8</span><span class="p">)</span>
+
+        <span class="n">output</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">model</span><span class="p">(</span><span class="n">input_ids</span><span class="o">=</span><span class="n">X</span><span class="p">,</span> <span class="n">attention_mask</span><span class="o">=</span><span class="n">attn_mask</span><span class="p">,</span> <span class="o">**</span><span class="bp">self</span><span class="o">.</span><span class="n">kwargs</span><span class="p">)</span>
+
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">output_attention_weights</span><span class="p">:</span>
+            <span class="c1"># TO CONSIDER: attention weights as a returned object and not an</span>
+            <span class="c1"># attribute</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">attn_weights</span> <span class="o">=</span> <span class="n">output</span><span class="p">[</span><span class="s2">&quot;attentions&quot;</span><span class="p">]</span>
+
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">use_cls_token</span><span class="p">:</span>
+            <span class="n">output</span> <span class="o">=</span> <span class="n">output</span><span class="p">[</span><span class="mi">0</span><span class="p">][:,</span> <span class="mi">0</span><span class="p">,</span> <span class="p">:]</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="c1"># Here one can choose to flatten, but unless the sequence length</span>
+            <span class="c1"># is very small, flatten will result in a very large output</span>
+            <span class="c1"># tensor.</span>
+            <span class="n">output</span> <span class="o">=</span> <span class="n">output</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="n">dim</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span>
+
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">head_hidden_dims</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="n">output</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">head</span><span class="p">(</span><span class="n">output</span><span class="p">)</span>
+
+        <span class="k">return</span> <span class="n">output</span>
+
+    <span class="nd">@property</span>
+    <span class="k">def</span> <span class="nf">output_dim</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">int</span><span class="p">:</span>
+        <span class="k">return</span> <span class="p">(</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">head_hidden_dims</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span>
+            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">head_hidden_dims</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span>
+            <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">config</span><span class="o">.</span><span class="n">hidden_size</span>
+        <span class="p">)</span>
+
+    <span class="nd">@property</span>
+    <span class="k">def</span> <span class="nf">attention_weight</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tensor</span><span class="p">:</span>
+<span class="w">        </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Returns the attention weights if the model was created with the</span>
+<span class="sd">        output_attention_weights=True argument. If not, it will raise an</span>
+<span class="sd">        AttributeError.</span>
+
+<span class="sd">        The shape of the attention weights is $(N, H, F, F)$, where $N$ is the</span>
+<span class="sd">        batch size, $H$ is the number of attention heads and $F$ is the</span>
+<span class="sd">        sequence length.</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">output_attention_weights</span><span class="p">:</span>
+            <span class="k">raise</span> <span class="ne">AttributeError</span><span class="p">(</span>
+                <span class="s2">&quot;The output_attention_weights attribute was not set to True when creating the model object &quot;</span>
+                <span class="s2">&quot;Please pass an output_attention_weights=True argument when creating the HFModel object&quot;</span>
+            <span class="p">)</span>
+        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">attn_weights</span>
+</code></pre></div></td></tr></table></div>
+              </details>
+
+
+
+  <div class="doc doc-children">
+
+
+
+
+
+
+
+<div class="doc doc-object doc-attribute">
+
+
+
+<h3 id="pytorch_widedeep.models.text.huggingface_transformers.hf_model.HFModel.attention_weight" class="doc doc-heading">
+            <span class="doc doc-object-name doc-attribute-name">attention_weight</span>
+
+
+  <span class="doc doc-labels">
+      <small class="doc doc-label doc-label-property"><code>property</code></small>
+  </span>
+
+<a href="#pytorch_widedeep.models.text.huggingface_transformers.hf_model.HFModel.attention_weight" class="headerlink" title="Permanent link">&para;</a></h3>
+<div class="doc-signature highlight"><pre><span></span><code><span class="n">attention_weight</span>
+</code></pre></div>
+
+    <div class="doc doc-contents ">
+
+        <p>Returns the attention weights if the model was created with the
 output_attention_weights=True argument. If not, it will raise an
 AttributeError.</p>
 <p>The shape of the attention weights is <span class="arithmatex">\((N, H, F, F)\)</span>, where <span class="arithmatex">\(N\)</span> is the
@@ -12150,21 +19317,7 @@ <h2 id="pytorch_widedeep.models.image.vision.Vision" class="doc doc-heading">
 
 
 <a href="#pytorch_widedeep.models.image.vision.Vision" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="nf">Vision</span><span class="p">(</span>
-    <span class="n">pretrained_model_setup</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">n_trainable</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">trainable_params</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">channel_sizes</span><span class="o">=</span><span class="p">[</span><span class="mi">64</span><span class="p">,</span> <span class="mi">128</span><span class="p">,</span> <span class="mi">256</span><span class="p">,</span> <span class="mi">512</span><span class="p">],</span>
-    <span class="n">kernel_sizes</span><span class="o">=</span><span class="p">[</span><span class="mi">7</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="mi">3</span><span class="p">],</span>
-    <span class="n">strides</span><span class="o">=</span><span class="p">[</span><span class="mi">2</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">],</span>
-    <span class="n">head_hidden_dims</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">head_activation</span><span class="o">=</span><span class="s2">&quot;relu&quot;</span><span class="p">,</span>
-    <span class="n">head_dropout</span><span class="o">=</span><span class="mf">0.1</span><span class="p">,</span>
-    <span class="n">head_batchnorm</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
-    <span class="n">head_batchnorm_last</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
-    <span class="n">head_linear_first</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
-<span class="p">)</span>
-</code></pre></div>
+
 
     <div class="doc doc-contents first">
             <p class="doc doc-class-bases">
@@ -12185,15 +19338,24 @@ <h2 id="pytorch_widedeep.models.image.vision.Vision" class="doc doc-heading">
 
 
 <p><span class="doc-section-title">Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>pretrained_model_setup</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Union">Union</span>[str, <span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, <span title="pytorch_widedeep.wdtypes.Union">Union</span>[str, <span title="pytorch_widedeep.wdtypes.WeightsEnum">WeightsEnum</span>]]]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Name of the pretrained model. Should be a variant of the following
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+          <th>Default</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>pretrained_model_setup</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Union">Union</span>[str, <span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, <span title="pytorch_widedeep.wdtypes.Union">Union</span>[str, <span title="pytorch_widedeep.wdtypes.WeightsEnum">WeightsEnum</span>]]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Name of the pretrained model. Should be a variant of the following
 architectures: <em>'resnet'</em>, <em>'shufflenet'</em>, <em>'resnext'</em>,
 <em>'wide_resnet'</em>, <em>'regnet'</em>, <em>'densenet'</em>, <em>'mobilenetv3'</em>,
 <em>'mobilenetv2'</em>, <em>'mnasnet'</em>, <em>'efficientnet'</em> and <em>'squeezenet'</em>. if
@@ -12203,149 +19365,210 @@ <h2 id="pytorch_widedeep.models.image.vision.Vision" class="doc doc-heading">
 also be dictionary with the name of the model and the weights (e.g.
 <code>{'resnet50': ResNet50_Weights.DEFAULT}</code> or
 <code>{'resnet50': "IMAGENET1K_V2"}</code>). <br/> Aliased as <code>pretrained_model_name</code>.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>n_trainable</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[int]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Number of trainable layers starting from the layer closer to the
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>n_trainable</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[int]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Number of trainable layers starting from the layer closer to the
 output neuron(s). Note that this number DOES NOT take into account
 the so-called <em>'head'</em> which is ALWAYS trainable. If
 <code>trainable_params</code> is not None this parameter will be ignored</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>trainable_params</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[str]]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>List of strings containing the names (or substring within the name) of
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>trainable_params</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[str]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>List of strings containing the names (or substring within the name) of
 the parameters that will be trained. For example, if we use a
 <em>'resnet18'</em> pretrained model and we set <code>trainable_params =
 ['layer4']</code> only the parameters of <em>'layer4'</em> of the network
 (and the head, as mentioned before) will be trained. Note that
 setting this or the previous parameter involves some knowledge of
 the architecture used.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>channel_sizes</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.List">List</span>[int]</code>, default:
-                  <code>[64, 128, 256, 512]</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>List of integers with the channel sizes of a CNN in case we choose not
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>channel_sizes</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.List">List</span>[int]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>List of integers with the channel sizes of a CNN in case we choose not
 to use a pretrained model</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>kernel_sizes</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Union">Union</span>[int, <span title="pytorch_widedeep.wdtypes.List">List</span>[int]]</code>, default:
+              </div>
+            </td>
+            <td>
+                  <code>[64, 128, 256, 512]</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>kernel_sizes</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Union">Union</span>[int, <span title="pytorch_widedeep.wdtypes.List">List</span>[int]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>List of integers with the kernel sizes of a CNN in case we choose not
+to use a pretrained model. Must be of length equal to <code>len(channel_sizes) - 1</code>.</p>
+              </div>
+            </td>
+            <td>
                   <code>[7, 3, 3, 3]</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>List of integers with the kernel sizes of a CNN in case we choose not
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>strides</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Union">Union</span>[int, <span title="pytorch_widedeep.wdtypes.List">List</span>[int]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>List of integers with the stride sizes of a CNN in case we choose not
 to use a pretrained model. Must be of length equal to <code>len(channel_sizes) - 1</code>.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>strides</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Union">Union</span>[int, <span title="pytorch_widedeep.wdtypes.List">List</span>[int]]</code>, default:
+              </div>
+            </td>
+            <td>
                   <code>[2, 1, 1, 1]</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>List of integers with the stride sizes of a CNN in case we choose not
-to use a pretrained model. Must be of length equal to <code>len(channel_sizes) - 1</code>.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>head_hidden_dims</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[int]]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>List with the number of neurons per dense layer in the head. e.g: <em>[64,32]</em></p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>head_activation</code></b>
-              (<code>str</code>, default:
-                  <code>&#39;relu&#39;</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Activation function for the dense layers in the head. Currently
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>head_hidden_dims</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[int]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>List with the number of neurons per dense layer in the head. e.g: <em>[64,32]</em></p>
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>head_activation</code></td>
+            <td>
+                  <code>str</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Activation function for the dense layers in the head. Currently
 <em>'tanh'</em>, <em>'relu'</em>, <em>'leaky_relu'</em> and <em>'gelu'</em> are supported</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>head_dropout</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Union">Union</span>[float, <span title="pytorch_widedeep.wdtypes.List">List</span>[float]]</code>, default:
+              </div>
+            </td>
+            <td>
+                  <code>&#39;relu&#39;</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>head_dropout</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Union">Union</span>[float, <span title="pytorch_widedeep.wdtypes.List">List</span>[float]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>float indicating the dropout between the dense layers.</p>
+              </div>
+            </td>
+            <td>
                   <code>0.1</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>float indicating the dropout between the dense layers.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>head_batchnorm</code></b>
-              (<code>bool</code>, default:
-                  <code>False</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Boolean indicating whether or not batch normalization will be applied
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>head_batchnorm</code></td>
+            <td>
+                  <code>bool</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Boolean indicating whether or not batch normalization will be applied
 to the dense layers</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>head_batchnorm_last</code></b>
-              (<code>bool</code>, default:
+              </div>
+            </td>
+            <td>
                   <code>False</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Boolean indicating whether or not batch normalization will be applied
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>head_batchnorm_last</code></td>
+            <td>
+                  <code>bool</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Boolean indicating whether or not batch normalization will be applied
 to the last of the dense layers</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>head_linear_first</code></b>
-              (<code>bool</code>, default:
+              </div>
+            </td>
+            <td>
                   <code>False</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Boolean indicating the order of the operations in the dense
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>head_linear_first</code></td>
+            <td>
+                  <code>bool</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Boolean indicating the order of the operations in the dense
 layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt;
 LIN -&gt; ACT]</code></p>
-          </div>
-        </li>
-    </ul>
+              </div>
+            </td>
+            <td>
+                  <code>False</code>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
 
 <p><span class="doc-section-title">Attributes:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code><span title="pytorch_widedeep.models.image.vision.Vision.features">features</span></code></b>
-              (<code><span title="torch.nn.Module">Module</span></code>)
-          –
-          <div class="doc-md-description">
-            <p>The pretrained model or Standard CNN plus the optional head</p>
-          </div>
-        </li>
-    </ul>
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code><span title="pytorch_widedeep.models.image.vision.Vision.features">features</span></code></td>
+            <td>
+                  <code><span title="torch.nn.Module">Module</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>The pretrained model or Standard CNN plus the optional head</p>
+              </div>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
 
 <p><span class="doc-section-title">Examples:</span></p>
@@ -12356,9 +19579,90 @@ <h2 id="pytorch_widedeep.models.image.vision.Vision" class="doc doc-heading">
 <span class="gp">&gt;&gt;&gt; </span><span class="n">out</span> <span class="o">=</span> <span class="n">model</span><span class="p">(</span><span class="n">X_img</span><span class="p">)</span>
 </code></pre></div>
 
-                  <details class="quote">
-                    <summary>Source code in <code>pytorch_widedeep/models/image/vision.py</code></summary>
-                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">123</span>
+              <details class="quote">
+                <summary>Source code in <code>pytorch_widedeep/models/image/vision.py</code></summary>
+                <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 42</span>
+<span class="normal"> 43</span>
+<span class="normal"> 44</span>
+<span class="normal"> 45</span>
+<span class="normal"> 46</span>
+<span class="normal"> 47</span>
+<span class="normal"> 48</span>
+<span class="normal"> 49</span>
+<span class="normal"> 50</span>
+<span class="normal"> 51</span>
+<span class="normal"> 52</span>
+<span class="normal"> 53</span>
+<span class="normal"> 54</span>
+<span class="normal"> 55</span>
+<span class="normal"> 56</span>
+<span class="normal"> 57</span>
+<span class="normal"> 58</span>
+<span class="normal"> 59</span>
+<span class="normal"> 60</span>
+<span class="normal"> 61</span>
+<span class="normal"> 62</span>
+<span class="normal"> 63</span>
+<span class="normal"> 64</span>
+<span class="normal"> 65</span>
+<span class="normal"> 66</span>
+<span class="normal"> 67</span>
+<span class="normal"> 68</span>
+<span class="normal"> 69</span>
+<span class="normal"> 70</span>
+<span class="normal"> 71</span>
+<span class="normal"> 72</span>
+<span class="normal"> 73</span>
+<span class="normal"> 74</span>
+<span class="normal"> 75</span>
+<span class="normal"> 76</span>
+<span class="normal"> 77</span>
+<span class="normal"> 78</span>
+<span class="normal"> 79</span>
+<span class="normal"> 80</span>
+<span class="normal"> 81</span>
+<span class="normal"> 82</span>
+<span class="normal"> 83</span>
+<span class="normal"> 84</span>
+<span class="normal"> 85</span>
+<span class="normal"> 86</span>
+<span class="normal"> 87</span>
+<span class="normal"> 88</span>
+<span class="normal"> 89</span>
+<span class="normal"> 90</span>
+<span class="normal"> 91</span>
+<span class="normal"> 92</span>
+<span class="normal"> 93</span>
+<span class="normal"> 94</span>
+<span class="normal"> 95</span>
+<span class="normal"> 96</span>
+<span class="normal"> 97</span>
+<span class="normal"> 98</span>
+<span class="normal"> 99</span>
+<span class="normal">100</span>
+<span class="normal">101</span>
+<span class="normal">102</span>
+<span class="normal">103</span>
+<span class="normal">104</span>
+<span class="normal">105</span>
+<span class="normal">106</span>
+<span class="normal">107</span>
+<span class="normal">108</span>
+<span class="normal">109</span>
+<span class="normal">110</span>
+<span class="normal">111</span>
+<span class="normal">112</span>
+<span class="normal">113</span>
+<span class="normal">114</span>
+<span class="normal">115</span>
+<span class="normal">116</span>
+<span class="normal">117</span>
+<span class="normal">118</span>
+<span class="normal">119</span>
+<span class="normal">120</span>
+<span class="normal">121</span>
+<span class="normal">122</span>
+<span class="normal">123</span>
 <span class="normal">124</span>
 <span class="normal">125</span>
 <span class="normal">126</span>
@@ -12407,127 +19711,481 @@ <h2 id="pytorch_widedeep.models.image.vision.Vision" class="doc doc-heading">
 <span class="normal">169</span>
 <span class="normal">170</span>
 <span class="normal">171</span>
-<span class="normal">172</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="nd">@alias</span><span class="p">(</span><span class="s2">&quot;pretrained_model_setup&quot;</span><span class="p">,</span> <span class="p">[</span><span class="s2">&quot;pretrained_model_name&quot;</span><span class="p">])</span>
-<span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
-    <span class="bp">self</span><span class="p">,</span>
-    <span class="n">pretrained_model_setup</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Union</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">WeightsEnum</span><span class="p">]]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">n_trainable</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">trainable_params</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">channel_sizes</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="p">[</span><span class="mi">64</span><span class="p">,</span> <span class="mi">128</span><span class="p">,</span> <span class="mi">256</span><span class="p">,</span> <span class="mi">512</span><span class="p">],</span>
-    <span class="n">kernel_sizes</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="nb">int</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="nb">int</span><span class="p">]]</span> <span class="o">=</span> <span class="p">[</span><span class="mi">7</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="mi">3</span><span class="p">],</span>
-    <span class="n">strides</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="nb">int</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="nb">int</span><span class="p">]]</span> <span class="o">=</span> <span class="p">[</span><span class="mi">2</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">],</span>
-    <span class="n">head_hidden_dims</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">int</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">head_activation</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;relu&quot;</span><span class="p">,</span>
-    <span class="n">head_dropout</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="nb">float</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="nb">float</span><span class="p">]]</span> <span class="o">=</span> <span class="mf">0.1</span><span class="p">,</span>
-    <span class="n">head_batchnorm</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
-    <span class="n">head_batchnorm_last</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
-    <span class="n">head_linear_first</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
-<span class="p">):</span>
-    <span class="nb">super</span><span class="p">(</span><span class="n">Vision</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
-
-    <span class="bp">self</span><span class="o">.</span><span class="n">_check_pretrained_model_setup</span><span class="p">(</span>
-        <span class="n">pretrained_model_setup</span><span class="p">,</span> <span class="n">n_trainable</span><span class="p">,</span> <span class="n">trainable_params</span>
-    <span class="p">)</span>
-
-    <span class="bp">self</span><span class="o">.</span><span class="n">pretrained_model_setup</span> <span class="o">=</span> <span class="n">pretrained_model_setup</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">n_trainable</span> <span class="o">=</span> <span class="n">n_trainable</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">trainable_params</span> <span class="o">=</span> <span class="n">trainable_params</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">channel_sizes</span> <span class="o">=</span> <span class="n">channel_sizes</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">kernel_sizes</span> <span class="o">=</span> <span class="n">kernel_sizes</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">strides</span> <span class="o">=</span> <span class="n">strides</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">head_hidden_dims</span> <span class="o">=</span> <span class="n">head_hidden_dims</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">head_activation</span> <span class="o">=</span> <span class="n">head_activation</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">head_dropout</span> <span class="o">=</span> <span class="n">head_dropout</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">head_batchnorm</span> <span class="o">=</span> <span class="n">head_batchnorm</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">head_batchnorm_last</span> <span class="o">=</span> <span class="n">head_batchnorm_last</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">head_linear_first</span> <span class="o">=</span> <span class="n">head_linear_first</span>
-
-    <span class="bp">self</span><span class="o">.</span><span class="n">features</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">backbone_output_dim</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_features</span><span class="p">()</span>
-
-    <span class="k">if</span> <span class="n">pretrained_model_setup</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">_freeze</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">features</span><span class="p">)</span>
-
-    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">head_hidden_dims</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
-        <span class="n">head_hidden_dims</span> <span class="o">=</span> <span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">backbone_output_dim</span><span class="p">]</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">head_hidden_dims</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">vision_mlp</span> <span class="o">=</span> <span class="n">MLP</span><span class="p">(</span>
-            <span class="n">head_hidden_dims</span><span class="p">,</span>
-            <span class="bp">self</span><span class="o">.</span><span class="n">head_activation</span><span class="p">,</span>
-            <span class="bp">self</span><span class="o">.</span><span class="n">head_dropout</span><span class="p">,</span>
-            <span class="bp">self</span><span class="o">.</span><span class="n">head_batchnorm</span><span class="p">,</span>
-            <span class="bp">self</span><span class="o">.</span><span class="n">head_batchnorm_last</span><span class="p">,</span>
-            <span class="bp">self</span><span class="o">.</span><span class="n">head_linear_first</span><span class="p">,</span>
-        <span class="p">)</span>
-</code></pre></div></td></tr></table></div>
-                  </details>
-
-
-
-  <div class="doc doc-children">
-
-
-
-
-
-
-
-<div class="doc doc-object doc-attribute">
-
-
-
-<h3 id="pytorch_widedeep.models.image.vision.Vision.output_dim" class="doc doc-heading">
-            <span class="doc doc-object-name doc-attribute-name">output_dim</span>
-
-
-  <span class="doc doc-labels">
-      <small class="doc doc-label doc-label-property"><code>property</code></small>
-  </span>
-
-<a href="#pytorch_widedeep.models.image.vision.Vision.output_dim" class="headerlink" title="Permanent link">&para;</a></h3>
-<div class="doc-signature highlight"><pre><span></span><code><span class="n">output_dim</span>
-</code></pre></div>
-
-    <div class="doc doc-contents ">
-
-        <p>The output dimension of the model. This is a required property
-neccesary to build the <code>WideDeep</code> class</p>
-    </div>
-
-</div>
-
-
-
-
-
-  </div>
-
-    </div>
-
-</div>
-
-<div class="doc doc-object doc-class">
-
-
-
-<h2 id="pytorch_widedeep.models.model_fusion.ModelFuser" class="doc doc-heading">
-            <span class="doc doc-object-name doc-class-name">ModelFuser</span>
-
-
-<a href="#pytorch_widedeep.models.model_fusion.ModelFuser" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="nf">ModelFuser</span><span class="p">(</span>
-    <span class="n">models</span><span class="p">,</span>
-    <span class="o">*</span><span class="p">,</span>
-    <span class="n">fusion_method</span><span class="p">,</span>
-    <span class="n">projection_method</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">custom_head</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">head_hidden_dims</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">head_activation</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">head_dropout</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">head_batchnorm</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">head_batchnorm_last</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">head_linear_first</span><span class="o">=</span><span class="kc">None</span>
-<span class="p">)</span>
-</code></pre></div>
-
+<span class="normal">172</span>
+<span class="normal">173</span>
+<span class="normal">174</span>
+<span class="normal">175</span>
+<span class="normal">176</span>
+<span class="normal">177</span>
+<span class="normal">178</span>
+<span class="normal">179</span>
+<span class="normal">180</span>
+<span class="normal">181</span>
+<span class="normal">182</span>
+<span class="normal">183</span>
+<span class="normal">184</span>
+<span class="normal">185</span>
+<span class="normal">186</span>
+<span class="normal">187</span>
+<span class="normal">188</span>
+<span class="normal">189</span>
+<span class="normal">190</span>
+<span class="normal">191</span>
+<span class="normal">192</span>
+<span class="normal">193</span>
+<span class="normal">194</span>
+<span class="normal">195</span>
+<span class="normal">196</span>
+<span class="normal">197</span>
+<span class="normal">198</span>
+<span class="normal">199</span>
+<span class="normal">200</span>
+<span class="normal">201</span>
+<span class="normal">202</span>
+<span class="normal">203</span>
+<span class="normal">204</span>
+<span class="normal">205</span>
+<span class="normal">206</span>
+<span class="normal">207</span>
+<span class="normal">208</span>
+<span class="normal">209</span>
+<span class="normal">210</span>
+<span class="normal">211</span>
+<span class="normal">212</span>
+<span class="normal">213</span>
+<span class="normal">214</span>
+<span class="normal">215</span>
+<span class="normal">216</span>
+<span class="normal">217</span>
+<span class="normal">218</span>
+<span class="normal">219</span>
+<span class="normal">220</span>
+<span class="normal">221</span>
+<span class="normal">222</span>
+<span class="normal">223</span>
+<span class="normal">224</span>
+<span class="normal">225</span>
+<span class="normal">226</span>
+<span class="normal">227</span>
+<span class="normal">228</span>
+<span class="normal">229</span>
+<span class="normal">230</span>
+<span class="normal">231</span>
+<span class="normal">232</span>
+<span class="normal">233</span>
+<span class="normal">234</span>
+<span class="normal">235</span>
+<span class="normal">236</span>
+<span class="normal">237</span>
+<span class="normal">238</span>
+<span class="normal">239</span>
+<span class="normal">240</span>
+<span class="normal">241</span>
+<span class="normal">242</span>
+<span class="normal">243</span>
+<span class="normal">244</span>
+<span class="normal">245</span>
+<span class="normal">246</span>
+<span class="normal">247</span>
+<span class="normal">248</span>
+<span class="normal">249</span>
+<span class="normal">250</span>
+<span class="normal">251</span>
+<span class="normal">252</span>
+<span class="normal">253</span>
+<span class="normal">254</span>
+<span class="normal">255</span>
+<span class="normal">256</span>
+<span class="normal">257</span>
+<span class="normal">258</span>
+<span class="normal">259</span>
+<span class="normal">260</span>
+<span class="normal">261</span>
+<span class="normal">262</span>
+<span class="normal">263</span>
+<span class="normal">264</span>
+<span class="normal">265</span>
+<span class="normal">266</span>
+<span class="normal">267</span>
+<span class="normal">268</span>
+<span class="normal">269</span>
+<span class="normal">270</span>
+<span class="normal">271</span>
+<span class="normal">272</span>
+<span class="normal">273</span>
+<span class="normal">274</span>
+<span class="normal">275</span>
+<span class="normal">276</span>
+<span class="normal">277</span>
+<span class="normal">278</span>
+<span class="normal">279</span>
+<span class="normal">280</span>
+<span class="normal">281</span>
+<span class="normal">282</span>
+<span class="normal">283</span>
+<span class="normal">284</span>
+<span class="normal">285</span>
+<span class="normal">286</span>
+<span class="normal">287</span>
+<span class="normal">288</span>
+<span class="normal">289</span>
+<span class="normal">290</span>
+<span class="normal">291</span>
+<span class="normal">292</span>
+<span class="normal">293</span>
+<span class="normal">294</span>
+<span class="normal">295</span>
+<span class="normal">296</span>
+<span class="normal">297</span>
+<span class="normal">298</span>
+<span class="normal">299</span>
+<span class="normal">300</span>
+<span class="normal">301</span>
+<span class="normal">302</span>
+<span class="normal">303</span>
+<span class="normal">304</span>
+<span class="normal">305</span>
+<span class="normal">306</span>
+<span class="normal">307</span>
+<span class="normal">308</span>
+<span class="normal">309</span>
+<span class="normal">310</span>
+<span class="normal">311</span>
+<span class="normal">312</span>
+<span class="normal">313</span>
+<span class="normal">314</span>
+<span class="normal">315</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">class</span> <span class="nc">Vision</span><span class="p">(</span><span class="n">BaseWDModelComponent</span><span class="p">):</span>
+<span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Defines a standard image classifier/regressor using a pretrained</span>
+<span class="sd">    network or a sequence of convolution layers that can be used as the</span>
+<span class="sd">    `deepimage` component of a Wide &amp; Deep model or independently by</span>
+<span class="sd">    itself.</span>
+
+<span class="sd">    :information_source: **NOTE**: this class represents the integration</span>
+<span class="sd">     between `pytorch-widedeep` and `torchvision`. New architectures will be</span>
+<span class="sd">     available as they are added to `torchvision`. In a distant future we aim</span>
+<span class="sd">     to bring transformer-based architectures as well. However, simple</span>
+<span class="sd">     CNN-based architectures (and even MLP-based) seem to produce SoTA</span>
+<span class="sd">     results. For the time being, we describe below the options available</span>
+<span class="sd">     through this class</span>
+
+<span class="sd">    Parameters</span>
+<span class="sd">    ----------</span>
+<span class="sd">    pretrained_model_setup: Optional, str or dict, default = None</span>
+<span class="sd">        Name of the pretrained model. Should be a variant of the following</span>
+<span class="sd">        architectures: _&#39;resnet&#39;_, _&#39;shufflenet&#39;_, _&#39;resnext&#39;_,</span>
+<span class="sd">        _&#39;wide_resnet&#39;_, _&#39;regnet&#39;_, _&#39;densenet&#39;_, _&#39;mobilenetv3&#39;_,</span>
+<span class="sd">        _&#39;mobilenetv2&#39;_, _&#39;mnasnet&#39;_, _&#39;efficientnet&#39;_ and _&#39;squeezenet&#39;_. if</span>
+<span class="sd">        `pretrained_model_setup = None` a basic, fully trainable CNN will be</span>
+<span class="sd">        used. Alternatively, since Torchvision 0.13 one can use pretrained</span>
+<span class="sd">        models with different weigths. Therefore, `pretrained_model_setup` can</span>
+<span class="sd">        also be dictionary with the name of the model and the weights (e.g.</span>
+<span class="sd">        `{&#39;resnet50&#39;: ResNet50_Weights.DEFAULT}` or</span>
+<span class="sd">        `{&#39;resnet50&#39;: &quot;IMAGENET1K_V2&quot;}`). &lt;br/&gt; Aliased as `pretrained_model_name`.</span>
+<span class="sd">    n_trainable: Optional, int, default = None</span>
+<span class="sd">        Number of trainable layers starting from the layer closer to the</span>
+<span class="sd">        output neuron(s). Note that this number DOES NOT take into account</span>
+<span class="sd">        the so-called _&#39;head&#39;_ which is ALWAYS trainable. If</span>
+<span class="sd">        `trainable_params` is not None this parameter will be ignored</span>
+<span class="sd">    trainable_params: Optional, list, default = None</span>
+<span class="sd">        List of strings containing the names (or substring within the name) of</span>
+<span class="sd">        the parameters that will be trained. For example, if we use a</span>
+<span class="sd">        _&#39;resnet18&#39;_ pretrained model and we set `trainable_params =</span>
+<span class="sd">        [&#39;layer4&#39;]` only the parameters of _&#39;layer4&#39;_ of the network</span>
+<span class="sd">        (and the head, as mentioned before) will be trained. Note that</span>
+<span class="sd">        setting this or the previous parameter involves some knowledge of</span>
+<span class="sd">        the architecture used.</span>
+<span class="sd">    channel_sizes: list, default = [64, 128, 256, 512]</span>
+<span class="sd">        List of integers with the channel sizes of a CNN in case we choose not</span>
+<span class="sd">        to use a pretrained model</span>
+<span class="sd">    kernel_sizes: list or int, default = 3</span>
+<span class="sd">        List of integers with the kernel sizes of a CNN in case we choose not</span>
+<span class="sd">        to use a pretrained model. Must be of length equal to `len(channel_sizes) - 1`.</span>
+<span class="sd">    strides: list or int, default = 1</span>
+<span class="sd">        List of integers with the stride sizes of a CNN in case we choose not</span>
+<span class="sd">        to use a pretrained model. Must be of length equal to `len(channel_sizes) - 1`.</span>
+<span class="sd">    head_hidden_dims: Optional, list, default = None</span>
+<span class="sd">        List with the number of neurons per dense layer in the head. e.g: _[64,32]_</span>
+<span class="sd">    head_activation: str, default = &quot;relu&quot;</span>
+<span class="sd">        Activation function for the dense layers in the head. Currently</span>
+<span class="sd">        _&#39;tanh&#39;_, _&#39;relu&#39;_, _&#39;leaky_relu&#39;_ and _&#39;gelu&#39;_ are supported</span>
+<span class="sd">    head_dropout: float, default = 0.1</span>
+<span class="sd">        float indicating the dropout between the dense layers.</span>
+<span class="sd">    head_batchnorm: bool, default = False</span>
+<span class="sd">        Boolean indicating whether or not batch normalization will be applied</span>
+<span class="sd">        to the dense layers</span>
+<span class="sd">    head_batchnorm_last: bool, default = False</span>
+<span class="sd">        Boolean indicating whether or not batch normalization will be applied</span>
+<span class="sd">        to the last of the dense layers</span>
+<span class="sd">    head_linear_first: bool, default = False</span>
+<span class="sd">        Boolean indicating the order of the operations in the dense</span>
+<span class="sd">        layer. If `True: [LIN -&gt; ACT -&gt; BN -&gt; DP]`. If `False: [BN -&gt; DP -&gt;</span>
+<span class="sd">        LIN -&gt; ACT]`</span>
+
+<span class="sd">    Attributes</span>
+<span class="sd">    ----------</span>
+<span class="sd">    features: nn.Module</span>
+<span class="sd">        The pretrained model or Standard CNN plus the optional head</span>
+
+<span class="sd">    Examples</span>
+<span class="sd">    --------</span>
+<span class="sd">    &gt;&gt;&gt; import torch</span>
+<span class="sd">    &gt;&gt;&gt; from pytorch_widedeep.models import Vision</span>
+<span class="sd">    &gt;&gt;&gt; X_img = torch.rand((2,3,224,224))</span>
+<span class="sd">    &gt;&gt;&gt; model = Vision(channel_sizes=[64, 128], kernel_sizes = [3, 3], strides=[1, 1], head_hidden_dims=[32, 8])</span>
+<span class="sd">    &gt;&gt;&gt; out = model(X_img)</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+
+    <span class="nd">@alias</span><span class="p">(</span><span class="s2">&quot;pretrained_model_setup&quot;</span><span class="p">,</span> <span class="p">[</span><span class="s2">&quot;pretrained_model_name&quot;</span><span class="p">])</span>
+    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span>
+        <span class="n">pretrained_model_setup</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Union</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">WeightsEnum</span><span class="p">]]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">n_trainable</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">trainable_params</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">channel_sizes</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="p">[</span><span class="mi">64</span><span class="p">,</span> <span class="mi">128</span><span class="p">,</span> <span class="mi">256</span><span class="p">,</span> <span class="mi">512</span><span class="p">],</span>
+        <span class="n">kernel_sizes</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="nb">int</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="nb">int</span><span class="p">]]</span> <span class="o">=</span> <span class="p">[</span><span class="mi">7</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="mi">3</span><span class="p">],</span>
+        <span class="n">strides</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="nb">int</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="nb">int</span><span class="p">]]</span> <span class="o">=</span> <span class="p">[</span><span class="mi">2</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">],</span>
+        <span class="n">head_hidden_dims</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">int</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">head_activation</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;relu&quot;</span><span class="p">,</span>
+        <span class="n">head_dropout</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="nb">float</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="nb">float</span><span class="p">]]</span> <span class="o">=</span> <span class="mf">0.1</span><span class="p">,</span>
+        <span class="n">head_batchnorm</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
+        <span class="n">head_batchnorm_last</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
+        <span class="n">head_linear_first</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
+    <span class="p">):</span>
+        <span class="nb">super</span><span class="p">(</span><span class="n">Vision</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">_check_pretrained_model_setup</span><span class="p">(</span>
+            <span class="n">pretrained_model_setup</span><span class="p">,</span> <span class="n">n_trainable</span><span class="p">,</span> <span class="n">trainable_params</span>
+        <span class="p">)</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">pretrained_model_setup</span> <span class="o">=</span> <span class="n">pretrained_model_setup</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">n_trainable</span> <span class="o">=</span> <span class="n">n_trainable</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">trainable_params</span> <span class="o">=</span> <span class="n">trainable_params</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">channel_sizes</span> <span class="o">=</span> <span class="n">channel_sizes</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">kernel_sizes</span> <span class="o">=</span> <span class="n">kernel_sizes</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">strides</span> <span class="o">=</span> <span class="n">strides</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">head_hidden_dims</span> <span class="o">=</span> <span class="n">head_hidden_dims</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">head_activation</span> <span class="o">=</span> <span class="n">head_activation</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">head_dropout</span> <span class="o">=</span> <span class="n">head_dropout</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">head_batchnorm</span> <span class="o">=</span> <span class="n">head_batchnorm</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">head_batchnorm_last</span> <span class="o">=</span> <span class="n">head_batchnorm_last</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">head_linear_first</span> <span class="o">=</span> <span class="n">head_linear_first</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">features</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">backbone_output_dim</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_features</span><span class="p">()</span>
+
+        <span class="k">if</span> <span class="n">pretrained_model_setup</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">_freeze</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">features</span><span class="p">)</span>
+
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">head_hidden_dims</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="n">head_hidden_dims</span> <span class="o">=</span> <span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">backbone_output_dim</span><span class="p">]</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">head_hidden_dims</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">vision_mlp</span> <span class="o">=</span> <span class="n">MLP</span><span class="p">(</span>
+                <span class="n">head_hidden_dims</span><span class="p">,</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">head_activation</span><span class="p">,</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">head_dropout</span><span class="p">,</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">head_batchnorm</span><span class="p">,</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">head_batchnorm_last</span><span class="p">,</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">head_linear_first</span><span class="p">,</span>
+            <span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">forward</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">X</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tensor</span><span class="p">:</span>
+        <span class="n">x</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">features</span><span class="p">(</span><span class="n">X</span><span class="p">)</span>
+
+        <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">x</span><span class="o">.</span><span class="n">shape</span><span class="p">)</span> <span class="o">&gt;</span> <span class="mi">2</span><span class="p">:</span>
+            <span class="k">if</span> <span class="n">x</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">2</span><span class="p">]</span> <span class="o">&gt;</span> <span class="mi">1</span><span class="p">:</span>
+                <span class="n">x</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">functional</span><span class="o">.</span><span class="n">adaptive_avg_pool2d</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">))</span>
+            <span class="n">x</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">flatten</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span>
+
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">head_hidden_dims</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="n">x</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">vision_mlp</span><span class="p">(</span><span class="n">x</span><span class="p">)</span>
+
+        <span class="k">return</span> <span class="n">x</span>
+
+    <span class="nd">@property</span>
+    <span class="k">def</span> <span class="nf">output_dim</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">int</span><span class="p">:</span>
+<span class="w">        </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;The output dimension of the model. This is a required property</span>
+<span class="sd">        neccesary to build the `WideDeep` class</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="k">return</span> <span class="p">(</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">head_hidden_dims</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span>
+            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">head_hidden_dims</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span>
+            <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">backbone_output_dim</span>
+        <span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">_get_features</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tuple</span><span class="p">[</span><span class="n">nn</span><span class="o">.</span><span class="n">Module</span><span class="p">,</span> <span class="nb">int</span><span class="p">]:</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">pretrained_model_setup</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">pretrained_model_setup</span><span class="p">,</span> <span class="nb">str</span><span class="p">):</span>
+                <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">pretrained_model_setup</span> <span class="ow">in</span> <span class="n">allowed_pretrained_models</span><span class="o">.</span><span class="n">keys</span><span class="p">():</span>
+                    <span class="n">model</span> <span class="o">=</span> <span class="n">allowed_pretrained_models</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">pretrained_model_setup</span><span class="p">]</span>
+                    <span class="n">pretrained_model</span> <span class="o">=</span> <span class="n">torchvision</span><span class="o">.</span><span class="n">models</span><span class="o">.</span><span class="vm">__dict__</span><span class="p">[</span><span class="n">model</span><span class="p">](</span>
+                        <span class="n">weights</span><span class="o">=</span><span class="n">torchvision</span><span class="o">.</span><span class="n">models</span><span class="o">.</span><span class="n">get_model_weights</span><span class="p">(</span><span class="n">model</span><span class="p">)</span><span class="o">.</span><span class="n">DEFAULT</span>
+                    <span class="p">)</span>
+                    <span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span>
+                        <span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">pretrained_model_setup</span><span class="si">}</span><span class="s2"> defaulting to </span><span class="si">{</span><span class="n">model</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">,</span>
+                        <span class="ne">UserWarning</span><span class="p">,</span>
+                    <span class="p">)</span>
+                <span class="k">else</span><span class="p">:</span>
+                    <span class="n">pretrained_model</span> <span class="o">=</span> <span class="n">torchvision</span><span class="o">.</span><span class="n">models</span><span class="o">.</span><span class="vm">__dict__</span><span class="p">[</span>
+                        <span class="bp">self</span><span class="o">.</span><span class="n">pretrained_model_setup</span>
+                    <span class="p">](</span><span class="n">weights</span><span class="o">=</span><span class="s2">&quot;IMAGENET1K_V1&quot;</span><span class="p">)</span>
+
+            <span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">pretrained_model_setup</span><span class="p">,</span> <span class="n">Dict</span><span class="p">):</span>
+                <span class="n">model_name</span> <span class="o">=</span> <span class="nb">next</span><span class="p">(</span><span class="nb">iter</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">pretrained_model_setup</span><span class="p">))</span>
+                <span class="n">model_weights</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">pretrained_model_setup</span><span class="p">[</span><span class="n">model_name</span><span class="p">]</span>
+
+                <span class="k">if</span> <span class="n">model_name</span> <span class="ow">in</span> <span class="n">allowed_pretrained_models</span><span class="o">.</span><span class="n">keys</span><span class="p">():</span>
+                    <span class="n">model_name</span> <span class="o">=</span> <span class="n">allowed_pretrained_models</span><span class="p">[</span><span class="n">model_name</span><span class="p">]</span>
+
+                <span class="n">pretrained_model</span> <span class="o">=</span> <span class="n">torchvision</span><span class="o">.</span><span class="n">models</span><span class="o">.</span><span class="vm">__dict__</span><span class="p">[</span><span class="n">model_name</span><span class="p">](</span>
+                    <span class="n">weights</span><span class="o">=</span><span class="n">model_weights</span>
+                <span class="p">)</span>
+            <span class="n">output_dim</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_backbone_output_dim</span><span class="p">(</span><span class="n">pretrained_model</span><span class="p">)</span>
+            <span class="n">features</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Sequential</span><span class="p">(</span><span class="o">*</span><span class="p">(</span><span class="nb">list</span><span class="p">(</span><span class="n">pretrained_model</span><span class="o">.</span><span class="n">children</span><span class="p">())[:</span><span class="o">-</span><span class="mi">1</span><span class="p">]))</span>
+
+        <span class="k">else</span><span class="p">:</span>
+            <span class="n">features</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_basic_cnn</span><span class="p">()</span>
+            <span class="n">output_dim</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">channel_sizes</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span>
+
+        <span class="k">return</span> <span class="n">features</span><span class="p">,</span> <span class="n">output_dim</span>
+
+    <span class="k">def</span> <span class="nf">_basic_cnn</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="n">channel_sizes</span> <span class="o">=</span> <span class="p">[</span><span class="mi">3</span><span class="p">]</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">channel_sizes</span>
+        <span class="n">kernel_sizes</span> <span class="o">=</span> <span class="p">(</span>
+            <span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">kernel_sizes</span><span class="p">]</span> <span class="o">*</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">channel_sizes</span><span class="p">)</span>
+            <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">kernel_sizes</span><span class="p">,</span> <span class="nb">int</span><span class="p">)</span>
+            <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">kernel_sizes</span>
+        <span class="p">)</span>
+        <span class="n">strides</span> <span class="o">=</span> <span class="p">(</span>
+            <span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">strides</span><span class="p">]</span> <span class="o">*</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">channel_sizes</span><span class="p">)</span>
+            <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">strides</span><span class="p">,</span> <span class="nb">int</span><span class="p">)</span>
+            <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">strides</span>
+        <span class="p">)</span>
+
+        <span class="n">BasicCNN</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Sequential</span><span class="p">()</span>
+        <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="nb">len</span><span class="p">(</span><span class="n">channel_sizes</span><span class="p">)):</span>
+            <span class="n">BasicCNN</span><span class="o">.</span><span class="n">add_module</span><span class="p">(</span>
+                <span class="s2">&quot;conv_layer_</span><span class="si">{}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">i</span> <span class="o">-</span> <span class="mi">1</span><span class="p">),</span>
+                <span class="n">conv_layer</span><span class="p">(</span>
+                    <span class="n">channel_sizes</span><span class="p">[</span><span class="n">i</span> <span class="o">-</span> <span class="mi">1</span><span class="p">],</span>
+                    <span class="n">channel_sizes</span><span class="p">[</span><span class="n">i</span><span class="p">],</span>
+                    <span class="n">kernel_sizes</span><span class="p">[</span><span class="n">i</span> <span class="o">-</span> <span class="mi">1</span><span class="p">],</span>
+                    <span class="n">strides</span><span class="p">[</span><span class="n">i</span> <span class="o">-</span> <span class="mi">1</span><span class="p">],</span>
+                    <span class="n">maxpool</span><span class="o">=</span><span class="n">i</span> <span class="o">==</span> <span class="mi">1</span><span class="p">,</span>
+                    <span class="n">adaptiveavgpool</span><span class="o">=</span><span class="n">i</span> <span class="o">==</span> <span class="nb">len</span><span class="p">(</span><span class="n">channel_sizes</span><span class="p">)</span> <span class="o">-</span> <span class="mi">1</span><span class="p">,</span>
+                <span class="p">),</span>
+            <span class="p">)</span>
+        <span class="k">return</span> <span class="n">BasicCNN</span>
+
+    <span class="k">def</span> <span class="nf">_freeze</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">features</span><span class="p">):</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">trainable_params</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="k">for</span> <span class="n">name</span><span class="p">,</span> <span class="n">param</span> <span class="ow">in</span> <span class="n">features</span><span class="o">.</span><span class="n">named_parameters</span><span class="p">():</span>
+                <span class="k">for</span> <span class="n">tl</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">trainable_params</span><span class="p">:</span>
+                    <span class="n">param</span><span class="o">.</span><span class="n">requires_grad</span> <span class="o">=</span> <span class="n">tl</span> <span class="ow">in</span> <span class="n">name</span>
+        <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">n_trainable</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="k">for</span> <span class="n">i</span><span class="p">,</span> <span class="p">(</span><span class="n">name</span><span class="p">,</span> <span class="n">param</span><span class="p">)</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span>
+                <span class="nb">reversed</span><span class="p">(</span><span class="nb">list</span><span class="p">(</span><span class="n">features</span><span class="o">.</span><span class="n">named_parameters</span><span class="p">()))</span>
+            <span class="p">):</span>
+                <span class="n">param</span><span class="o">.</span><span class="n">requires_grad</span> <span class="o">=</span> <span class="n">i</span> <span class="o">&lt;</span> <span class="bp">self</span><span class="o">.</span><span class="n">n_trainable</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span>
+                <span class="s2">&quot;Both &#39;trainable_params&#39; and &#39;n_trainable&#39; are &#39;None&#39; and the entire network will be trained&quot;</span><span class="p">,</span>
+                <span class="ne">UserWarning</span><span class="p">,</span>
+            <span class="p">)</span>
+
+    <span class="nd">@staticmethod</span>
+    <span class="k">def</span> <span class="nf">get_backbone_output_dim</span><span class="p">(</span><span class="n">features</span><span class="p">):</span>
+        <span class="k">try</span><span class="p">:</span>
+            <span class="k">return</span> <span class="n">features</span><span class="o">.</span><span class="n">fc</span><span class="o">.</span><span class="n">in_features</span>
+        <span class="k">except</span> <span class="ne">AttributeError</span><span class="p">:</span>
+            <span class="k">try</span><span class="p">:</span>
+                <span class="n">features</span><span class="o">.</span><span class="n">classifier</span><span class="o">.</span><span class="vm">__dict__</span><span class="p">[</span><span class="s2">&quot;_modules&quot;</span><span class="p">][</span><span class="s2">&quot;0&quot;</span><span class="p">]</span><span class="o">.</span><span class="n">in_features</span>
+            <span class="k">except</span> <span class="ne">AttributeError</span><span class="p">:</span>
+                <span class="k">try</span><span class="p">:</span>
+                    <span class="k">return</span> <span class="n">features</span><span class="o">.</span><span class="n">classifier</span><span class="o">.</span><span class="vm">__dict__</span><span class="p">[</span><span class="s2">&quot;_modules&quot;</span><span class="p">][</span><span class="s2">&quot;1&quot;</span><span class="p">]</span><span class="o">.</span><span class="n">in_features</span>
+                <span class="k">except</span> <span class="ne">AttributeError</span><span class="p">:</span>
+                    <span class="k">return</span> <span class="n">features</span><span class="o">.</span><span class="n">classifier</span><span class="o">.</span><span class="vm">__dict__</span><span class="p">[</span><span class="s2">&quot;_modules&quot;</span><span class="p">][</span><span class="s2">&quot;1&quot;</span><span class="p">]</span><span class="o">.</span><span class="n">in_channels</span>
+
+    <span class="nd">@staticmethod</span>
+    <span class="k">def</span> <span class="nf">_check_pretrained_model_setup</span><span class="p">(</span>
+        <span class="n">pretrained_model_setup</span><span class="p">,</span> <span class="n">n_trainable</span><span class="p">,</span> <span class="n">trainable_params</span>
+    <span class="p">):</span>
+        <span class="k">if</span> <span class="n">pretrained_model_setup</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">pretrained_model_setup</span><span class="p">,</span> <span class="nb">str</span><span class="p">):</span>
+                <span class="n">pretrained_model_name</span> <span class="o">=</span> <span class="n">pretrained_model_setup</span>
+            <span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">pretrained_model_setup</span><span class="p">,</span> <span class="n">Dict</span><span class="p">):</span>
+                <span class="n">pretrained_model_name</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="n">pretrained_model_setup</span><span class="o">.</span><span class="n">keys</span><span class="p">())[</span><span class="mi">0</span><span class="p">]</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="n">pretrained_model_name</span> <span class="o">=</span> <span class="kc">None</span>
+
+        <span class="k">if</span> <span class="n">pretrained_model_name</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="n">valid_pretrained_model_name</span> <span class="o">=</span> <span class="nb">any</span><span class="p">(</span>
+                <span class="p">[</span><span class="n">name</span> <span class="ow">in</span> <span class="n">pretrained_model_name</span> <span class="k">for</span> <span class="n">name</span> <span class="ow">in</span> <span class="n">allowed_pretrained_models</span><span class="p">]</span>
+            <span class="p">)</span>
+            <span class="k">if</span> <span class="ow">not</span> <span class="n">valid_pretrained_model_name</span><span class="p">:</span>
+                <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
+                    <span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="n">pretrained_model_setup</span><span class="si">}</span><span class="s2"> is not among the allowed pretrained models.&quot;</span>
+                    <span class="sa">f</span><span class="s2">&quot; These are </span><span class="si">{</span><span class="n">allowed_pretrained_models</span><span class="o">.</span><span class="n">keys</span><span class="p">()</span><span class="si">}</span><span class="s2">. Please choose a variant of these architectures&quot;</span>
+                <span class="p">)</span>
+            <span class="k">if</span> <span class="n">n_trainable</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">and</span> <span class="n">trainable_params</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+                <span class="k">raise</span> <span class="ne">UserWarning</span><span class="p">(</span>
+                    <span class="s2">&quot;Both &#39;n_trainable&#39; and &#39;trainable_params&#39; are not None. &#39;trainable_params&#39; will be used&quot;</span>
+                <span class="p">)</span>
+</code></pre></div></td></tr></table></div>
+              </details>
+
+
+
+  <div class="doc doc-children">
+
+
+
+
+
+
+
+<div class="doc doc-object doc-attribute">
+
+
+
+<h3 id="pytorch_widedeep.models.image.vision.Vision.output_dim" class="doc doc-heading">
+            <span class="doc doc-object-name doc-attribute-name">output_dim</span>
+
+
+  <span class="doc doc-labels">
+      <small class="doc doc-label doc-label-property"><code>property</code></small>
+  </span>
+
+<a href="#pytorch_widedeep.models.image.vision.Vision.output_dim" class="headerlink" title="Permanent link">&para;</a></h3>
+<div class="doc-signature highlight"><pre><span></span><code><span class="n">output_dim</span>
+</code></pre></div>
+
+    <div class="doc doc-contents ">
+
+        <p>The output dimension of the model. This is a required property
+neccesary to build the <code>WideDeep</code> class</p>
+    </div>
+
+</div>
+
+
+
+
+
+  </div>
+
+    </div>
+
+</div>
+
+<div class="doc doc-object doc-class">
+
+
+
+<h2 id="pytorch_widedeep.models.model_fusion.ModelFuser" class="doc doc-heading">
+            <span class="doc doc-object-name doc-class-name">ModelFuser</span>
+
+
+<a href="#pytorch_widedeep.models.model_fusion.ModelFuser" class="headerlink" title="Permanent link">&para;</a></h2>
+
+
     <div class="doc doc-contents first">
             <p class="doc doc-class-bases">
               Bases: <code><span title="pytorch_widedeep.models._base_wd_model_component.BaseWDModelComponent">BaseWDModelComponent</span></code></p>
@@ -12539,131 +20197,197 @@ <h2 id="pytorch_widedeep.models.model_fusion.ModelFuser" class="doc doc-heading"
 
 
 <p><span class="doc-section-title">Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>models</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.models._base_wd_model_component.BaseWDModelComponent">BaseWDModelComponent</span>]</code>)
-          –
-          <div class="doc-md-description">
-            <p>List of models whose outputs will be fused</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>fusion_method</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[concatenate, mean, max, sum, mult, dot, <span title="pytorch_widedeep.models.model_fusion.ModelFuser.head">head</span>], <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[concatenate, mean, max, sum, mult, <span title="pytorch_widedeep.models.model_fusion.ModelFuser.head">head</span>]]]</code>)
-          –
-          <div class="doc-md-description">
-            <p>Method to fuse the output of the models. It can be one of
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+          <th>Default</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>models</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.models._base_wd_model_component.BaseWDModelComponent">BaseWDModelComponent</span>]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>List of models whose outputs will be fused</p>
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>fusion_method</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[concatenate, mean, max, sum, mult, dot, <span title="pytorch_widedeep.models.model_fusion.ModelFuser.head">head</span>], <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[concatenate, mean, max, sum, mult, <span title="pytorch_widedeep.models.model_fusion.ModelFuser.head">head</span>]]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Method to fuse the output of the models. It can be one of
 ['concatenate', 'mean', 'max', 'sum', 'mult', 'dot', 'head'] or a
 list of those, but 'dot'. If a list is provided the output of the
 models will be fused using all the methods in the list and the final
 output will be the concatenation of the outputs of each method</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>projection_method</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[min, max, mean]]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>If the fusion_method is not 'concatenate', this parameter will
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>projection_method</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[min, max, mean]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>If the fusion_method is not 'concatenate', this parameter will
 determine how to project the output of the models to a common
 dimension. It can be one of ['min', 'max', 'mean']. Default is None</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>custom_head</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="pytorch_widedeep.models._base_wd_model_component.BaseWDModelComponent">BaseWDModelComponent</span>, <span title="torch.nn.Module">Module</span>]]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Custom head to be used to fuse the output of the models. If provided,
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>custom_head</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="pytorch_widedeep.models._base_wd_model_component.BaseWDModelComponent">BaseWDModelComponent</span>, <span title="torch.nn.Module">Module</span>]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Custom head to be used to fuse the output of the models. If provided,
 this will take precedence over head_hidden_dims. Also, if
 provided, 'projection_method' will be ignored.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>head_hidden_dims</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[int]]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>List with the number of neurons per layer in the custom head. If
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>head_hidden_dims</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[int]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>List with the number of neurons per layer in the custom head. If
 custom_head is provided, this parameter will be ignored</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>head_activation</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Activation function to be used in the custom head. Default is None</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>head_dropout</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Dropout to be used in the custom head. Default is None</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>head_batchnorm</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Whether to use batchnorm in the custom head. Default is None</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>head_batchnorm_last</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Whether or not batch normalization will be applied to the last of the
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>head_activation</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Activation function to be used in the custom head. Default is None</p>
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>head_dropout</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Dropout to be used in the custom head. Default is None</p>
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>head_batchnorm</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Whether to use batchnorm in the custom head. Default is None</p>
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>head_batchnorm_last</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Whether or not batch normalization will be applied to the last of the
 dense layers</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>head_linear_first</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Boolean indicating the order of the operations in the dense
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>head_linear_first</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Boolean indicating the order of the operations in the dense
 layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt;
 LIN -&gt; ACT]</code></p>
-          </div>
-        </li>
-    </ul>
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
 
 <p><span class="doc-section-title">Attributes:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code><span title="pytorch_widedeep.models.model_fusion.ModelFuser.head">head</span></code></b>
-              (<code><span title="torch.nn.Module">Module</span> or <span title="pytorch_widedeep.models._base_wd_model_component.BaseWDModelComponent">BaseWDModelComponent</span></code>)
-          –
-          <div class="doc-md-description">
-            <p>Custom head to be used to fuse the output of the models. If
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code><span title="pytorch_widedeep.models.model_fusion.ModelFuser.head">head</span></code></td>
+            <td>
+                  <code><span title="torch.nn.Module">Module</span> or <span title="pytorch_widedeep.models._base_wd_model_component.BaseWDModelComponent">BaseWDModelComponent</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Custom head to be used to fuse the output of the models. If
 custom_head is provided, this will take precedence over
 head_hidden_dims</p>
-          </div>
-        </li>
-    </ul>
+              </div>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
 
 <p><span class="doc-section-title">Examples:</span></p>
@@ -12708,9 +20432,100 @@ <h2 id="pytorch_widedeep.models.model_fusion.ModelFuser" class="doc doc-heading"
 <span class="gp">&gt;&gt;&gt; </span><span class="n">out</span> <span class="o">=</span> <span class="n">fused_model</span><span class="p">([</span><span class="n">X_text1_tnsr</span><span class="p">,</span> <span class="n">X_text2_tnsr</span><span class="p">])</span>
 </code></pre></div>
 
-                  <details class="quote">
-                    <summary>Source code in <code>pytorch_widedeep/models/model_fusion.py</code></summary>
-                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">101</span>
+              <details class="quote">
+                <summary>Source code in <code>pytorch_widedeep/models/model_fusion.py</code></summary>
+                <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 10</span>
+<span class="normal"> 11</span>
+<span class="normal"> 12</span>
+<span class="normal"> 13</span>
+<span class="normal"> 14</span>
+<span class="normal"> 15</span>
+<span class="normal"> 16</span>
+<span class="normal"> 17</span>
+<span class="normal"> 18</span>
+<span class="normal"> 19</span>
+<span class="normal"> 20</span>
+<span class="normal"> 21</span>
+<span class="normal"> 22</span>
+<span class="normal"> 23</span>
+<span class="normal"> 24</span>
+<span class="normal"> 25</span>
+<span class="normal"> 26</span>
+<span class="normal"> 27</span>
+<span class="normal"> 28</span>
+<span class="normal"> 29</span>
+<span class="normal"> 30</span>
+<span class="normal"> 31</span>
+<span class="normal"> 32</span>
+<span class="normal"> 33</span>
+<span class="normal"> 34</span>
+<span class="normal"> 35</span>
+<span class="normal"> 36</span>
+<span class="normal"> 37</span>
+<span class="normal"> 38</span>
+<span class="normal"> 39</span>
+<span class="normal"> 40</span>
+<span class="normal"> 41</span>
+<span class="normal"> 42</span>
+<span class="normal"> 43</span>
+<span class="normal"> 44</span>
+<span class="normal"> 45</span>
+<span class="normal"> 46</span>
+<span class="normal"> 47</span>
+<span class="normal"> 48</span>
+<span class="normal"> 49</span>
+<span class="normal"> 50</span>
+<span class="normal"> 51</span>
+<span class="normal"> 52</span>
+<span class="normal"> 53</span>
+<span class="normal"> 54</span>
+<span class="normal"> 55</span>
+<span class="normal"> 56</span>
+<span class="normal"> 57</span>
+<span class="normal"> 58</span>
+<span class="normal"> 59</span>
+<span class="normal"> 60</span>
+<span class="normal"> 61</span>
+<span class="normal"> 62</span>
+<span class="normal"> 63</span>
+<span class="normal"> 64</span>
+<span class="normal"> 65</span>
+<span class="normal"> 66</span>
+<span class="normal"> 67</span>
+<span class="normal"> 68</span>
+<span class="normal"> 69</span>
+<span class="normal"> 70</span>
+<span class="normal"> 71</span>
+<span class="normal"> 72</span>
+<span class="normal"> 73</span>
+<span class="normal"> 74</span>
+<span class="normal"> 75</span>
+<span class="normal"> 76</span>
+<span class="normal"> 77</span>
+<span class="normal"> 78</span>
+<span class="normal"> 79</span>
+<span class="normal"> 80</span>
+<span class="normal"> 81</span>
+<span class="normal"> 82</span>
+<span class="normal"> 83</span>
+<span class="normal"> 84</span>
+<span class="normal"> 85</span>
+<span class="normal"> 86</span>
+<span class="normal"> 87</span>
+<span class="normal"> 88</span>
+<span class="normal"> 89</span>
+<span class="normal"> 90</span>
+<span class="normal"> 91</span>
+<span class="normal"> 92</span>
+<span class="normal"> 93</span>
+<span class="normal"> 94</span>
+<span class="normal"> 95</span>
+<span class="normal"> 96</span>
+<span class="normal"> 97</span>
+<span class="normal"> 98</span>
+<span class="normal"> 99</span>
+<span class="normal">100</span>
+<span class="normal">101</span>
 <span class="normal">102</span>
 <span class="normal">103</span>
 <span class="normal">104</span>
@@ -12787,86 +20602,567 @@ <h2 id="pytorch_widedeep.models.model_fusion.ModelFuser" class="doc doc-heading"
 <span class="normal">175</span>
 <span class="normal">176</span>
 <span class="normal">177</span>
-<span class="normal">178</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
-    <span class="bp">self</span><span class="p">,</span>
-    <span class="n">models</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="n">BaseWDModelComponent</span><span class="p">],</span>
-    <span class="o">*</span><span class="p">,</span>
-    <span class="n">fusion_method</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span>
-        <span class="n">Literal</span><span class="p">[</span>
+<span class="normal">178</span>
+<span class="normal">179</span>
+<span class="normal">180</span>
+<span class="normal">181</span>
+<span class="normal">182</span>
+<span class="normal">183</span>
+<span class="normal">184</span>
+<span class="normal">185</span>
+<span class="normal">186</span>
+<span class="normal">187</span>
+<span class="normal">188</span>
+<span class="normal">189</span>
+<span class="normal">190</span>
+<span class="normal">191</span>
+<span class="normal">192</span>
+<span class="normal">193</span>
+<span class="normal">194</span>
+<span class="normal">195</span>
+<span class="normal">196</span>
+<span class="normal">197</span>
+<span class="normal">198</span>
+<span class="normal">199</span>
+<span class="normal">200</span>
+<span class="normal">201</span>
+<span class="normal">202</span>
+<span class="normal">203</span>
+<span class="normal">204</span>
+<span class="normal">205</span>
+<span class="normal">206</span>
+<span class="normal">207</span>
+<span class="normal">208</span>
+<span class="normal">209</span>
+<span class="normal">210</span>
+<span class="normal">211</span>
+<span class="normal">212</span>
+<span class="normal">213</span>
+<span class="normal">214</span>
+<span class="normal">215</span>
+<span class="normal">216</span>
+<span class="normal">217</span>
+<span class="normal">218</span>
+<span class="normal">219</span>
+<span class="normal">220</span>
+<span class="normal">221</span>
+<span class="normal">222</span>
+<span class="normal">223</span>
+<span class="normal">224</span>
+<span class="normal">225</span>
+<span class="normal">226</span>
+<span class="normal">227</span>
+<span class="normal">228</span>
+<span class="normal">229</span>
+<span class="normal">230</span>
+<span class="normal">231</span>
+<span class="normal">232</span>
+<span class="normal">233</span>
+<span class="normal">234</span>
+<span class="normal">235</span>
+<span class="normal">236</span>
+<span class="normal">237</span>
+<span class="normal">238</span>
+<span class="normal">239</span>
+<span class="normal">240</span>
+<span class="normal">241</span>
+<span class="normal">242</span>
+<span class="normal">243</span>
+<span class="normal">244</span>
+<span class="normal">245</span>
+<span class="normal">246</span>
+<span class="normal">247</span>
+<span class="normal">248</span>
+<span class="normal">249</span>
+<span class="normal">250</span>
+<span class="normal">251</span>
+<span class="normal">252</span>
+<span class="normal">253</span>
+<span class="normal">254</span>
+<span class="normal">255</span>
+<span class="normal">256</span>
+<span class="normal">257</span>
+<span class="normal">258</span>
+<span class="normal">259</span>
+<span class="normal">260</span>
+<span class="normal">261</span>
+<span class="normal">262</span>
+<span class="normal">263</span>
+<span class="normal">264</span>
+<span class="normal">265</span>
+<span class="normal">266</span>
+<span class="normal">267</span>
+<span class="normal">268</span>
+<span class="normal">269</span>
+<span class="normal">270</span>
+<span class="normal">271</span>
+<span class="normal">272</span>
+<span class="normal">273</span>
+<span class="normal">274</span>
+<span class="normal">275</span>
+<span class="normal">276</span>
+<span class="normal">277</span>
+<span class="normal">278</span>
+<span class="normal">279</span>
+<span class="normal">280</span>
+<span class="normal">281</span>
+<span class="normal">282</span>
+<span class="normal">283</span>
+<span class="normal">284</span>
+<span class="normal">285</span>
+<span class="normal">286</span>
+<span class="normal">287</span>
+<span class="normal">288</span>
+<span class="normal">289</span>
+<span class="normal">290</span>
+<span class="normal">291</span>
+<span class="normal">292</span>
+<span class="normal">293</span>
+<span class="normal">294</span>
+<span class="normal">295</span>
+<span class="normal">296</span>
+<span class="normal">297</span>
+<span class="normal">298</span>
+<span class="normal">299</span>
+<span class="normal">300</span>
+<span class="normal">301</span>
+<span class="normal">302</span>
+<span class="normal">303</span>
+<span class="normal">304</span>
+<span class="normal">305</span>
+<span class="normal">306</span>
+<span class="normal">307</span>
+<span class="normal">308</span>
+<span class="normal">309</span>
+<span class="normal">310</span>
+<span class="normal">311</span>
+<span class="normal">312</span>
+<span class="normal">313</span>
+<span class="normal">314</span>
+<span class="normal">315</span>
+<span class="normal">316</span>
+<span class="normal">317</span>
+<span class="normal">318</span>
+<span class="normal">319</span>
+<span class="normal">320</span>
+<span class="normal">321</span>
+<span class="normal">322</span>
+<span class="normal">323</span>
+<span class="normal">324</span>
+<span class="normal">325</span>
+<span class="normal">326</span>
+<span class="normal">327</span>
+<span class="normal">328</span>
+<span class="normal">329</span>
+<span class="normal">330</span>
+<span class="normal">331</span>
+<span class="normal">332</span>
+<span class="normal">333</span>
+<span class="normal">334</span>
+<span class="normal">335</span>
+<span class="normal">336</span>
+<span class="normal">337</span>
+<span class="normal">338</span>
+<span class="normal">339</span>
+<span class="normal">340</span>
+<span class="normal">341</span>
+<span class="normal">342</span>
+<span class="normal">343</span>
+<span class="normal">344</span>
+<span class="normal">345</span>
+<span class="normal">346</span>
+<span class="normal">347</span>
+<span class="normal">348</span>
+<span class="normal">349</span>
+<span class="normal">350</span>
+<span class="normal">351</span>
+<span class="normal">352</span>
+<span class="normal">353</span>
+<span class="normal">354</span>
+<span class="normal">355</span>
+<span class="normal">356</span>
+<span class="normal">357</span>
+<span class="normal">358</span>
+<span class="normal">359</span>
+<span class="normal">360</span>
+<span class="normal">361</span>
+<span class="normal">362</span>
+<span class="normal">363</span>
+<span class="normal">364</span>
+<span class="normal">365</span>
+<span class="normal">366</span>
+<span class="normal">367</span>
+<span class="normal">368</span>
+<span class="normal">369</span>
+<span class="normal">370</span>
+<span class="normal">371</span>
+<span class="normal">372</span>
+<span class="normal">373</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">class</span> <span class="nc">ModelFuser</span><span class="p">(</span><span class="n">BaseWDModelComponent</span><span class="p">):</span>
+<span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">    This class is a wrapper around a list of models that are associated to the</span>
+<span class="sd">    different text and/or image columns (and datasets) The class is designed</span>
+<span class="sd">    to &#39;fuse&#39; the models using a variety of methods.</span>
+
+<span class="sd">    Parameters</span>
+<span class="sd">    ----------</span>
+<span class="sd">    models: List[BaseWDModelComponent]</span>
+<span class="sd">        List of models whose outputs will be fused</span>
+<span class="sd">    fusion_method: Union[str, List[str]]</span>
+<span class="sd">        Method to fuse the output of the models. It can be one of</span>
+<span class="sd">        [&#39;concatenate&#39;, &#39;mean&#39;, &#39;max&#39;, &#39;sum&#39;, &#39;mult&#39;, &#39;dot&#39;, &#39;head&#39;] or a</span>
+<span class="sd">        list of those, but &#39;dot&#39;. If a list is provided the output of the</span>
+<span class="sd">        models will be fused using all the methods in the list and the final</span>
+<span class="sd">        output will be the concatenation of the outputs of each method</span>
+<span class="sd">    projection_method: Optional[str]</span>
+<span class="sd">        If the fusion_method is not &#39;concatenate&#39;, this parameter will</span>
+<span class="sd">        determine how to project the output of the models to a common</span>
+<span class="sd">        dimension. It can be one of [&#39;min&#39;, &#39;max&#39;, &#39;mean&#39;]. Default is None</span>
+<span class="sd">    custom_head: Optional[BaseWDModelComponent | nn.Module]</span>
+<span class="sd">        Custom head to be used to fuse the output of the models. If provided,</span>
+<span class="sd">        this will take precedence over head_hidden_dims. Also, if</span>
+<span class="sd">        provided, &#39;projection_method&#39; will be ignored.</span>
+<span class="sd">    head_hidden_dims: Optional[List[int]]</span>
+<span class="sd">        List with the number of neurons per layer in the custom head. If</span>
+<span class="sd">        custom_head is provided, this parameter will be ignored</span>
+<span class="sd">    head_activation: Optional[str]</span>
+<span class="sd">        Activation function to be used in the custom head. Default is None</span>
+<span class="sd">    head_dropout: Optional[float]</span>
+<span class="sd">        Dropout to be used in the custom head. Default is None</span>
+<span class="sd">    head_batchnorm: Optional[bool]</span>
+<span class="sd">        Whether to use batchnorm in the custom head. Default is None</span>
+<span class="sd">    head_batchnorm_last: Optional[bool]</span>
+<span class="sd">        Whether or not batch normalization will be applied to the last of the</span>
+<span class="sd">        dense layers</span>
+<span class="sd">    head_linear_first: Optional[bool]</span>
+<span class="sd">        Boolean indicating the order of the operations in the dense</span>
+<span class="sd">        layer. If `True: [LIN -&gt; ACT -&gt; BN -&gt; DP]`. If `False: [BN -&gt; DP -&gt;</span>
+<span class="sd">        LIN -&gt; ACT]`</span>
+
+<span class="sd">    Attributes</span>
+<span class="sd">    ----------</span>
+<span class="sd">    head: nn.Module or BaseWDModelComponent</span>
+<span class="sd">        Custom head to be used to fuse the output of the models. If</span>
+<span class="sd">        custom_head is provided, this will take precedence over</span>
+<span class="sd">        head_hidden_dims</span>
+
+<span class="sd">    Examples</span>
+<span class="sd">    --------</span>
+<span class="sd">    &gt;&gt;&gt; from pytorch_widedeep.preprocessing import TextPreprocessor</span>
+<span class="sd">    &gt;&gt;&gt; from pytorch_widedeep.models import BasicRNN, ModelFuser</span>
+<span class="sd">    &gt;&gt;&gt; import torch</span>
+<span class="sd">    &gt;&gt;&gt; import pandas as pd</span>
+<span class="sd">    &gt;&gt;&gt;</span>
+<span class="sd">    &gt;&gt;&gt; df = pd.DataFrame({&#39;text_col1&#39;: [&#39;hello world&#39;, &#39;this is a test&#39;],</span>
+<span class="sd">    ... &#39;text_col2&#39;: [&#39;goodbye world&#39;, &#39;this is another test&#39;]})</span>
+<span class="sd">    &gt;&gt;&gt; text_preprocessor_1 = TextPreprocessor(</span>
+<span class="sd">    ...     text_col=&quot;text_col1&quot;,</span>
+<span class="sd">    ...     max_vocab=10,</span>
+<span class="sd">    ...     min_freq=1,</span>
+<span class="sd">    ...     maxlen=5,</span>
+<span class="sd">    ...     n_cpus=1,</span>
+<span class="sd">    ...     verbose=0)</span>
+<span class="sd">    &gt;&gt;&gt; text_preprocessor_2 = TextPreprocessor(</span>
+<span class="sd">    ...     text_col=&quot;text_col2&quot;,</span>
+<span class="sd">    ...     max_vocab=10,</span>
+<span class="sd">    ...     min_freq=1,</span>
+<span class="sd">    ...     maxlen=5,</span>
+<span class="sd">    ...     n_cpus=1,</span>
+<span class="sd">    ...     verbose=0)</span>
+<span class="sd">    &gt;&gt;&gt; X_text1 = text_preprocessor_1.fit_transform(df)</span>
+<span class="sd">    &gt;&gt;&gt; X_text2 = text_preprocessor_2.fit_transform(df)</span>
+<span class="sd">    &gt;&gt;&gt; X_text1_tnsr = torch.from_numpy(X_text1)</span>
+<span class="sd">    &gt;&gt;&gt; X_text2_tnsr = torch.from_numpy(X_text2)</span>
+<span class="sd">    &gt;&gt;&gt; rnn1 = BasicRNN(</span>
+<span class="sd">    ...     vocab_size=len(text_preprocessor_1.vocab.itos),</span>
+<span class="sd">    ...     embed_dim=4,</span>
+<span class="sd">    ...     hidden_dim=4,</span>
+<span class="sd">    ...     n_layers=1,</span>
+<span class="sd">    ...     bidirectional=False)</span>
+<span class="sd">    &gt;&gt;&gt; rnn2 = BasicRNN(</span>
+<span class="sd">    ...     vocab_size=len(text_preprocessor_2.vocab.itos),</span>
+<span class="sd">    ...     embed_dim=4,</span>
+<span class="sd">    ...     hidden_dim=4,</span>
+<span class="sd">    ...     n_layers=1,</span>
+<span class="sd">    ...     bidirectional=False)</span>
+<span class="sd">    &gt;&gt;&gt; fused_model = ModelFuser(models=[rnn1, rnn2], fusion_method=&#39;concatenate&#39;)</span>
+<span class="sd">    &gt;&gt;&gt; out = fused_model([X_text1_tnsr, X_text2_tnsr])</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+
+    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span>
+        <span class="n">models</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="n">BaseWDModelComponent</span><span class="p">],</span>
+        <span class="o">*</span><span class="p">,</span>
+        <span class="n">fusion_method</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span>
+            <span class="n">Literal</span><span class="p">[</span>
+                <span class="s2">&quot;concatenate&quot;</span><span class="p">,</span>
+                <span class="s2">&quot;mean&quot;</span><span class="p">,</span>
+                <span class="s2">&quot;max&quot;</span><span class="p">,</span>
+                <span class="s2">&quot;sum&quot;</span><span class="p">,</span>
+                <span class="s2">&quot;mult&quot;</span><span class="p">,</span>
+                <span class="s2">&quot;dot&quot;</span><span class="p">,</span>
+                <span class="s2">&quot;head&quot;</span><span class="p">,</span>
+            <span class="p">],</span>
+            <span class="n">List</span><span class="p">[</span><span class="n">Literal</span><span class="p">[</span><span class="s2">&quot;concatenate&quot;</span><span class="p">,</span> <span class="s2">&quot;mean&quot;</span><span class="p">,</span> <span class="s2">&quot;max&quot;</span><span class="p">,</span> <span class="s2">&quot;sum&quot;</span><span class="p">,</span> <span class="s2">&quot;mult&quot;</span><span class="p">,</span> <span class="s2">&quot;head&quot;</span><span class="p">]],</span>
+        <span class="p">],</span>
+        <span class="n">projection_method</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Literal</span><span class="p">[</span><span class="s2">&quot;min&quot;</span><span class="p">,</span> <span class="s2">&quot;max&quot;</span><span class="p">,</span> <span class="s2">&quot;mean&quot;</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">custom_head</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Union</span><span class="p">[</span><span class="n">BaseWDModelComponent</span><span class="p">,</span> <span class="n">nn</span><span class="o">.</span><span class="n">Module</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">head_hidden_dims</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">int</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">head_activation</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">head_dropout</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">head_batchnorm</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">head_batchnorm_last</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">head_linear_first</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+    <span class="p">)</span> <span class="o">-&gt;</span> <span class="kc">None</span><span class="p">:</span>
+        <span class="nb">super</span><span class="p">(</span><span class="n">ModelFuser</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">models</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">ModuleList</span><span class="p">(</span><span class="n">models</span><span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">fusion_method</span> <span class="o">=</span> <span class="n">fusion_method</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">projection_method</span> <span class="o">=</span> <span class="n">projection_method</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">all_output_dim_equal</span> <span class="o">=</span> <span class="nb">all</span><span class="p">(</span>
+            <span class="n">model</span><span class="o">.</span><span class="n">output_dim</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">models</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">output_dim</span> <span class="k">for</span> <span class="n">model</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">models</span>
+        <span class="p">)</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">check_input_parameters</span><span class="p">()</span>
+
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">fusion_method</span> <span class="o">==</span> <span class="s2">&quot;head&quot;</span><span class="p">:</span>
+            <span class="k">assert</span> <span class="p">(</span>
+                <span class="n">head_hidden_dims</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">or</span> <span class="n">custom_head</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span>
+            <span class="p">),</span> <span class="s2">&quot;When using &#39;head&#39; as fusion_method, either head_hidden_dims or custom_head must be provided&quot;</span>
+            <span class="k">if</span> <span class="n">custom_head</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+                <span class="c1"># custom_head takes precedence over head_hidden_dims (in case</span>
+                <span class="c1"># both are provided)</span>
+                <span class="k">assert</span> <span class="nb">hasattr</span><span class="p">(</span>
+                    <span class="n">custom_head</span><span class="p">,</span> <span class="s2">&quot;output_dim&quot;</span>
+                <span class="p">),</span> <span class="s2">&quot;custom_head must have an &#39;output_dim&#39; property&quot;</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">head</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="n">BaseWDModelComponent</span><span class="p">,</span> <span class="n">nn</span><span class="o">.</span><span class="n">Module</span><span class="p">]</span> <span class="o">=</span> <span class="n">custom_head</span>
+            <span class="k">else</span><span class="p">:</span>
+                <span class="k">assert</span> <span class="n">head_hidden_dims</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">head_hidden_dims</span> <span class="o">=</span> <span class="n">head_hidden_dims</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">head_activation</span> <span class="o">=</span> <span class="n">head_activation</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">head_dropout</span> <span class="o">=</span> <span class="n">head_dropout</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">head_batchnorm</span> <span class="o">=</span> <span class="n">head_batchnorm</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">head_batchnorm_last</span> <span class="o">=</span> <span class="n">head_batchnorm_last</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">head_linear_first</span> <span class="o">=</span> <span class="n">head_linear_first</span>
+
+                <span class="bp">self</span><span class="o">.</span><span class="n">head</span> <span class="o">=</span> <span class="n">MLP</span><span class="p">(</span>
+                    <span class="n">d_hidden</span><span class="o">=</span><span class="p">[</span><span class="nb">sum</span><span class="p">([</span><span class="n">model</span><span class="o">.</span><span class="n">output_dim</span> <span class="k">for</span> <span class="n">model</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">models</span><span class="p">])]</span>
+                    <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">head_hidden_dims</span><span class="p">,</span>
+                    <span class="n">activation</span><span class="o">=</span><span class="p">(</span>
+                        <span class="s2">&quot;relu&quot;</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">head_activation</span> <span class="ow">is</span> <span class="kc">None</span> <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">head_activation</span>
+                    <span class="p">),</span>
+                    <span class="n">dropout</span><span class="o">=</span><span class="mf">0.0</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">head_dropout</span> <span class="ow">is</span> <span class="kc">None</span> <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">head_dropout</span><span class="p">,</span>
+                    <span class="n">batchnorm</span><span class="o">=</span><span class="p">(</span>
+                        <span class="kc">False</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">head_batchnorm</span> <span class="ow">is</span> <span class="kc">None</span> <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">head_batchnorm</span>
+                    <span class="p">),</span>
+                    <span class="n">batchnorm_last</span><span class="o">=</span><span class="p">(</span>
+                        <span class="kc">False</span>
+                        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">head_batchnorm_last</span> <span class="ow">is</span> <span class="kc">None</span>
+                        <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">head_batchnorm_last</span>
+                    <span class="p">),</span>
+                    <span class="n">linear_first</span><span class="o">=</span><span class="p">(</span>
+                        <span class="kc">True</span>
+                        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">head_linear_first</span> <span class="ow">is</span> <span class="kc">None</span>
+                        <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">head_linear_first</span>
+                    <span class="p">),</span>
+                <span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">forward</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">X</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="n">Tensor</span><span class="p">])</span> <span class="o">-&gt;</span> <span class="n">Tensor</span><span class="p">:</span>  <span class="c1"># noqa: C901</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">fusion_method</span> <span class="o">==</span> <span class="s2">&quot;head&quot;</span><span class="p">:</span>
+            <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_head_fusion</span><span class="p">(</span><span class="n">X</span><span class="p">)</span>
+        <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">fusion_method</span> <span class="o">==</span> <span class="s2">&quot;dot&quot;</span><span class="p">:</span>
+            <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_dot_fusion</span><span class="p">(</span><span class="n">X</span><span class="p">)</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_other_fusions</span><span class="p">(</span><span class="n">X</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">_head_fusion</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">X</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="n">Tensor</span><span class="p">])</span> <span class="o">-&gt;</span> <span class="n">Tensor</span><span class="p">:</span>
+        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">head</span><span class="p">(</span><span class="n">torch</span><span class="o">.</span><span class="n">cat</span><span class="p">([</span><span class="n">model</span><span class="p">(</span><span class="n">x</span><span class="p">)</span> <span class="k">for</span> <span class="n">model</span><span class="p">,</span> <span class="n">x</span> <span class="ow">in</span> <span class="nb">zip</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">models</span><span class="p">,</span> <span class="n">X</span><span class="p">)],</span> <span class="o">-</span><span class="mi">1</span><span class="p">))</span>
+
+    <span class="k">def</span> <span class="nf">_dot_fusion</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">X</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="n">Tensor</span><span class="p">])</span> <span class="o">-&gt;</span> <span class="n">Tensor</span><span class="p">:</span>
+        <span class="k">assert</span> <span class="p">(</span>
+            <span class="nb">len</span><span class="p">(</span><span class="n">X</span><span class="p">)</span> <span class="o">==</span> <span class="mi">2</span>
+        <span class="p">),</span> <span class="s2">&quot;When using &#39;dot&#39; as fusion_method, only two models can be fused&quot;</span>
+        <span class="n">outputs</span> <span class="o">=</span> <span class="p">[</span><span class="n">model</span><span class="p">(</span><span class="n">x</span><span class="p">)</span> <span class="k">for</span> <span class="n">model</span><span class="p">,</span> <span class="n">x</span> <span class="ow">in</span> <span class="nb">zip</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">models</span><span class="p">,</span> <span class="n">X</span><span class="p">)]</span>
+        <span class="k">return</span> <span class="n">torch</span><span class="o">.</span><span class="n">bmm</span><span class="p">(</span><span class="n">outputs</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span><span class="o">.</span><span class="n">unsqueeze</span><span class="p">(</span><span class="mi">1</span><span class="p">),</span> <span class="n">outputs</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">unsqueeze</span><span class="p">(</span><span class="mi">2</span><span class="p">))</span><span class="o">.</span><span class="n">view</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">_other_fusions</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">X</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="n">Tensor</span><span class="p">])</span> <span class="o">-&gt;</span> <span class="n">Tensor</span><span class="p">:</span>
+        <span class="n">fusion_methods</span> <span class="o">=</span> <span class="p">(</span>
+            <span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">fusion_method</span><span class="p">]</span>
+            <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">fusion_method</span><span class="p">,</span> <span class="nb">str</span><span class="p">)</span>
+            <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">fusion_method</span>
+        <span class="p">)</span>
+        <span class="n">fused_outputs</span> <span class="o">=</span> <span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">_apply_fusion_method</span><span class="p">(</span><span class="n">fm</span><span class="p">,</span> <span class="n">X</span><span class="p">)</span> <span class="k">for</span> <span class="n">fm</span> <span class="ow">in</span> <span class="n">fusion_methods</span><span class="p">]</span>  <span class="c1"># type: ignore[attr-defined]</span>
+        <span class="k">return</span> <span class="p">(</span>
+            <span class="n">fused_outputs</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">fused_outputs</span><span class="p">)</span> <span class="o">==</span> <span class="mi">1</span> <span class="k">else</span> <span class="n">torch</span><span class="o">.</span><span class="n">cat</span><span class="p">(</span><span class="n">fused_outputs</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span>
+        <span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">_apply_fusion_method</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">fusion_method</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">X</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="n">Tensor</span><span class="p">])</span> <span class="o">-&gt;</span> <span class="n">Tensor</span><span class="p">:</span>
+        <span class="k">if</span> <span class="n">fusion_method</span> <span class="o">==</span> <span class="s2">&quot;concatenate&quot;</span><span class="p">:</span>
+            <span class="k">return</span> <span class="n">torch</span><span class="o">.</span><span class="n">cat</span><span class="p">([</span><span class="n">model</span><span class="p">(</span><span class="n">x</span><span class="p">)</span> <span class="k">for</span> <span class="n">model</span><span class="p">,</span> <span class="n">x</span> <span class="ow">in</span> <span class="nb">zip</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">models</span><span class="p">,</span> <span class="n">X</span><span class="p">)],</span> <span class="mi">1</span><span class="p">)</span>
+
+        <span class="n">model_outputs</span> <span class="o">=</span> <span class="p">[</span><span class="n">model</span><span class="p">(</span><span class="n">x</span><span class="p">)</span> <span class="k">for</span> <span class="n">model</span><span class="p">,</span> <span class="n">x</span> <span class="ow">in</span> <span class="nb">zip</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">models</span><span class="p">,</span> <span class="n">X</span><span class="p">)]</span>
+        <span class="n">projections</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_project</span><span class="p">(</span><span class="n">model_outputs</span><span class="p">)</span>
+        <span class="n">stacked</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">stack</span><span class="p">(</span><span class="n">projections</span><span class="p">,</span> <span class="o">-</span><span class="mi">1</span><span class="p">)</span>
+
+        <span class="k">if</span> <span class="n">fusion_method</span> <span class="o">==</span> <span class="s2">&quot;mean&quot;</span><span class="p">:</span>
+            <span class="k">return</span> <span class="n">torch</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="n">stacked</span><span class="p">,</span> <span class="o">-</span><span class="mi">1</span><span class="p">)</span>
+        <span class="k">elif</span> <span class="n">fusion_method</span> <span class="o">==</span> <span class="s2">&quot;max&quot;</span><span class="p">:</span>
+            <span class="k">return</span> <span class="n">torch</span><span class="o">.</span><span class="n">max</span><span class="p">(</span><span class="n">stacked</span><span class="p">,</span> <span class="o">-</span><span class="mi">1</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span>
+        <span class="k">elif</span> <span class="n">fusion_method</span> <span class="o">==</span> <span class="s2">&quot;min&quot;</span><span class="p">:</span>
+            <span class="k">return</span> <span class="n">torch</span><span class="o">.</span><span class="n">min</span><span class="p">(</span><span class="n">stacked</span><span class="p">,</span> <span class="o">-</span><span class="mi">1</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span>
+        <span class="k">elif</span> <span class="n">fusion_method</span> <span class="o">==</span> <span class="s2">&quot;sum&quot;</span><span class="p">:</span>
+            <span class="k">return</span> <span class="n">torch</span><span class="o">.</span><span class="n">sum</span><span class="p">(</span><span class="n">stacked</span><span class="p">,</span> <span class="o">-</span><span class="mi">1</span><span class="p">)</span>
+        <span class="k">elif</span> <span class="n">fusion_method</span> <span class="o">==</span> <span class="s2">&quot;mult&quot;</span><span class="p">:</span>
+            <span class="k">return</span> <span class="n">torch</span><span class="o">.</span><span class="n">prod</span><span class="p">(</span><span class="n">stacked</span><span class="p">,</span> <span class="o">-</span><span class="mi">1</span><span class="p">)</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Unsupported fusion method: </span><span class="si">{</span><span class="n">fusion_method</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">_project</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">X</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="n">Tensor</span><span class="p">])</span> <span class="o">-&gt;</span> <span class="n">List</span><span class="p">[</span><span class="n">Tensor</span><span class="p">]:</span>
+<span class="w">        </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Projects the output of the models to a common dimension.&quot;&quot;&quot;</span>
+
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">all_output_dim_equal</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">projection_method</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="k">return</span> <span class="n">X</span>
+
+        <span class="n">output_dims</span> <span class="o">=</span> <span class="p">[</span><span class="n">model</span><span class="o">.</span><span class="n">output_dim</span> <span class="k">for</span> <span class="n">model</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">models</span><span class="p">]</span>
+
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">projection_method</span> <span class="o">==</span> <span class="s2">&quot;min&quot;</span><span class="p">:</span>
+            <span class="n">proj_dim</span> <span class="o">=</span> <span class="nb">min</span><span class="p">(</span><span class="n">output_dims</span><span class="p">)</span>
+            <span class="n">idx</span> <span class="o">=</span> <span class="n">output_dims</span><span class="o">.</span><span class="n">index</span><span class="p">(</span><span class="n">proj_dim</span><span class="p">)</span>
+        <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">projection_method</span> <span class="o">==</span> <span class="s2">&quot;max&quot;</span><span class="p">:</span>
+            <span class="n">proj_dim</span> <span class="o">=</span> <span class="nb">max</span><span class="p">(</span><span class="n">output_dims</span><span class="p">)</span>
+            <span class="n">idx</span> <span class="o">=</span> <span class="n">output_dims</span><span class="o">.</span><span class="n">index</span><span class="p">(</span><span class="n">proj_dim</span><span class="p">)</span>
+        <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">projection_method</span> <span class="o">==</span> <span class="s2">&quot;mean&quot;</span><span class="p">:</span>
+            <span class="n">proj_dim</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="nb">sum</span><span class="p">(</span><span class="n">output_dims</span><span class="p">)</span> <span class="o">/</span> <span class="nb">len</span><span class="p">(</span><span class="n">output_dims</span><span class="p">))</span>
+            <span class="n">idx</span> <span class="o">=</span> <span class="kc">None</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">&quot;projection_method must be one of [&#39;min&#39;, &#39;max&#39;, &#39;mean&#39;]&quot;</span><span class="p">)</span>
+
+        <span class="n">x_proj</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="n">Tensor</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
+        <span class="k">for</span> <span class="n">i</span><span class="p">,</span> <span class="n">x</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="n">X</span><span class="p">):</span>
+            <span class="k">if</span> <span class="n">i</span> <span class="o">==</span> <span class="n">idx</span><span class="p">:</span>
+                <span class="n">x_proj</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">x</span><span class="p">)</span>
+            <span class="k">else</span><span class="p">:</span>
+                <span class="n">x_proj</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
+                    <span class="n">nn</span><span class="o">.</span><span class="n">Linear</span><span class="p">(</span><span class="n">output_dims</span><span class="p">[</span><span class="n">i</span><span class="p">],</span> <span class="n">proj_dim</span><span class="p">,</span> <span class="n">bias</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">device</span><span class="o">=</span><span class="n">x</span><span class="o">.</span><span class="n">device</span><span class="p">)(</span><span class="n">x</span><span class="p">)</span>
+                <span class="p">)</span>
+
+        <span class="k">return</span> <span class="n">x_proj</span>
+
+    <span class="nd">@property</span>
+    <span class="k">def</span> <span class="nf">output_dim</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">int</span><span class="p">:</span>
+<span class="w">        </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Returns the output dimension of the model.&quot;&quot;&quot;</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">fusion_method</span> <span class="o">==</span> <span class="s2">&quot;head&quot;</span><span class="p">:</span>
+            <span class="n">output_dim</span> <span class="o">=</span> <span class="p">(</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">head_hidden_dims</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span>
+                <span class="k">if</span> <span class="nb">hasattr</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="s2">&quot;head_hidden_dims&quot;</span><span class="p">)</span>
+                <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">head</span><span class="o">.</span><span class="n">output_dim</span>
+            <span class="p">)</span>
+        <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">fusion_method</span> <span class="o">==</span> <span class="s2">&quot;dot&quot;</span><span class="p">:</span>
+            <span class="n">output_dim</span> <span class="o">=</span> <span class="mi">1</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="n">output_dim</span> <span class="o">=</span> <span class="mi">0</span>
+            <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">fusion_method</span><span class="p">,</span> <span class="nb">str</span><span class="p">):</span>
+                <span class="n">fusion_methods</span> <span class="o">=</span> <span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">fusion_method</span><span class="p">]</span>
+            <span class="k">else</span><span class="p">:</span>
+                <span class="n">fusion_methods</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">fusion_method</span>  <span class="c1"># type: ignore</span>
+            <span class="k">for</span> <span class="n">fm</span> <span class="ow">in</span> <span class="n">fusion_methods</span><span class="p">:</span>
+                <span class="k">if</span> <span class="n">fm</span> <span class="o">==</span> <span class="s2">&quot;concatenate&quot;</span><span class="p">:</span>
+                    <span class="n">output_dim</span> <span class="o">+=</span> <span class="nb">sum</span><span class="p">([</span><span class="n">model</span><span class="o">.</span><span class="n">output_dim</span> <span class="k">for</span> <span class="n">model</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">models</span><span class="p">])</span>
+                <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">projection_method</span> <span class="o">==</span> <span class="s2">&quot;mean&quot;</span><span class="p">:</span>
+                    <span class="n">output_dim</span> <span class="o">+=</span> <span class="nb">int</span><span class="p">(</span>
+                        <span class="nb">sum</span><span class="p">([</span><span class="n">model</span><span class="o">.</span><span class="n">output_dim</span> <span class="k">for</span> <span class="n">model</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">models</span><span class="p">])</span>
+                        <span class="o">/</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">models</span><span class="p">)</span>
+                    <span class="p">)</span>
+                <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">projection_method</span> <span class="o">==</span> <span class="s2">&quot;min&quot;</span><span class="p">:</span>
+                    <span class="n">output_dim</span> <span class="o">+=</span> <span class="nb">min</span><span class="p">([</span><span class="n">model</span><span class="o">.</span><span class="n">output_dim</span> <span class="k">for</span> <span class="n">model</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">models</span><span class="p">])</span>
+                <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">projection_method</span> <span class="o">==</span> <span class="s2">&quot;max&quot;</span><span class="p">:</span>
+                    <span class="n">output_dim</span> <span class="o">+=</span> <span class="nb">max</span><span class="p">([</span><span class="n">model</span><span class="o">.</span><span class="n">output_dim</span> <span class="k">for</span> <span class="n">model</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">models</span><span class="p">])</span>
+                <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">all_output_dim_equal</span><span class="p">:</span>
+                    <span class="n">output_dim</span> <span class="o">+=</span> <span class="bp">self</span><span class="o">.</span><span class="n">models</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">output_dim</span>
+                <span class="k">else</span><span class="p">:</span>
+                    <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
+                        <span class="s2">&quot;projection_method must be one of [&#39;min&#39;, &#39;max&#39;, &#39;mean&#39;]&quot;</span>
+                    <span class="p">)</span>
+
+        <span class="k">return</span> <span class="n">output_dim</span>
+
+    <span class="k">def</span> <span class="nf">check_input_parameters</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">_validate_tabnet</span><span class="p">()</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">_validate_fusion_methods</span><span class="p">()</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">_validate_projection_requirements</span><span class="p">()</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">_validate_head_dot_exclusivity</span><span class="p">()</span>
+
+    <span class="k">def</span> <span class="nf">_validate_tabnet</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="k">if</span> <span class="nb">any</span><span class="p">(</span><span class="nb">isinstance</span><span class="p">(</span><span class="n">model</span><span class="p">,</span> <span class="n">TabNet</span><span class="p">)</span> <span class="k">for</span> <span class="n">model</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">models</span><span class="p">):</span>
+            <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
+                <span class="s2">&quot;TabNet is not supported in ModelFuser. &quot;</span>
+                <span class="s2">&quot;Please, use another model for tabular data&quot;</span>
+            <span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">_validate_fusion_methods</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="n">valid_methods</span> <span class="o">=</span> <span class="p">[</span>
             <span class="s2">&quot;concatenate&quot;</span><span class="p">,</span>
-            <span class="s2">&quot;mean&quot;</span><span class="p">,</span>
+            <span class="s2">&quot;min&quot;</span><span class="p">,</span>
             <span class="s2">&quot;max&quot;</span><span class="p">,</span>
+            <span class="s2">&quot;mean&quot;</span><span class="p">,</span>
             <span class="s2">&quot;sum&quot;</span><span class="p">,</span>
             <span class="s2">&quot;mult&quot;</span><span class="p">,</span>
             <span class="s2">&quot;dot&quot;</span><span class="p">,</span>
             <span class="s2">&quot;head&quot;</span><span class="p">,</span>
-        <span class="p">],</span>
-        <span class="n">List</span><span class="p">[</span><span class="n">Literal</span><span class="p">[</span><span class="s2">&quot;concatenate&quot;</span><span class="p">,</span> <span class="s2">&quot;mean&quot;</span><span class="p">,</span> <span class="s2">&quot;max&quot;</span><span class="p">,</span> <span class="s2">&quot;sum&quot;</span><span class="p">,</span> <span class="s2">&quot;mult&quot;</span><span class="p">,</span> <span class="s2">&quot;head&quot;</span><span class="p">]],</span>
-    <span class="p">],</span>
-    <span class="n">projection_method</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Literal</span><span class="p">[</span><span class="s2">&quot;min&quot;</span><span class="p">,</span> <span class="s2">&quot;max&quot;</span><span class="p">,</span> <span class="s2">&quot;mean&quot;</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">custom_head</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Union</span><span class="p">[</span><span class="n">BaseWDModelComponent</span><span class="p">,</span> <span class="n">nn</span><span class="o">.</span><span class="n">Module</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">head_hidden_dims</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">int</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">head_activation</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">head_dropout</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">head_batchnorm</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">head_batchnorm_last</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">head_linear_first</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-<span class="p">)</span> <span class="o">-&gt;</span> <span class="kc">None</span><span class="p">:</span>
-    <span class="nb">super</span><span class="p">(</span><span class="n">ModelFuser</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
-
-    <span class="bp">self</span><span class="o">.</span><span class="n">models</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">ModuleList</span><span class="p">(</span><span class="n">models</span><span class="p">)</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">fusion_method</span> <span class="o">=</span> <span class="n">fusion_method</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">projection_method</span> <span class="o">=</span> <span class="n">projection_method</span>
-
-    <span class="bp">self</span><span class="o">.</span><span class="n">all_output_dim_equal</span> <span class="o">=</span> <span class="nb">all</span><span class="p">(</span>
-        <span class="n">model</span><span class="o">.</span><span class="n">output_dim</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">models</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">output_dim</span> <span class="k">for</span> <span class="n">model</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">models</span>
-    <span class="p">)</span>
-
-    <span class="bp">self</span><span class="o">.</span><span class="n">check_input_parameters</span><span class="p">()</span>
+        <span class="p">]</span>
+
+        <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">fusion_method</span><span class="p">,</span> <span class="nb">str</span><span class="p">):</span>
+            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">fusion_method</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">valid_methods</span><span class="p">:</span>
+                <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
+                    <span class="s2">&quot;fusion_method must be one of [&#39;concatenate&#39;, &#39;mean&#39;, &#39;max&#39;, &#39;sum&#39;, &#39;mult&#39;, &#39;dot&#39;, &#39;head&#39;] &quot;</span>
+                    <span class="s2">&quot;or a list of any those but &#39;dot&#39;&quot;</span>
+                <span class="p">)</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="k">if</span> <span class="ow">not</span> <span class="nb">all</span><span class="p">(</span><span class="n">fm</span> <span class="ow">in</span> <span class="n">valid_methods</span> <span class="k">for</span> <span class="n">fm</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">fusion_method</span><span class="p">):</span>
+                <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
+                    <span class="s2">&quot;fusion_method must be one of [&#39;concatenate&#39;, &#39;mean&#39;, &#39;max&#39;, &#39;sum&#39;, &#39;mult&#39;, &#39;dot&#39;, &#39;head&#39;] &quot;</span>
+                    <span class="s2">&quot;or a list of those but &#39;dot&#39;&quot;</span>
+                <span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">_validate_projection_requirements</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="n">needs_projection</span> <span class="o">=</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">all_output_dim_equal</span>
+        <span class="n">has_size_dependent_method</span> <span class="o">=</span> <span class="kc">False</span>
+
+        <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">fusion_method</span><span class="p">,</span> <span class="nb">str</span><span class="p">):</span>
+            <span class="n">has_size_dependent_method</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">fusion_method</span> <span class="ow">in</span> <span class="p">[</span><span class="s2">&quot;min&quot;</span><span class="p">,</span> <span class="s2">&quot;max&quot;</span><span class="p">,</span> <span class="s2">&quot;mean&quot;</span><span class="p">]</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="n">has_size_dependent_method</span> <span class="o">=</span> <span class="nb">all</span><span class="p">(</span>
+                <span class="n">fm</span> <span class="ow">in</span> <span class="p">[</span><span class="s2">&quot;min&quot;</span><span class="p">,</span> <span class="s2">&quot;max&quot;</span><span class="p">,</span> <span class="s2">&quot;mean&quot;</span><span class="p">]</span> <span class="k">for</span> <span class="n">fm</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">fusion_method</span>
+            <span class="p">)</span>
 
-    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">fusion_method</span> <span class="o">==</span> <span class="s2">&quot;head&quot;</span><span class="p">:</span>
-        <span class="k">assert</span> <span class="p">(</span>
-            <span class="n">head_hidden_dims</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">or</span> <span class="n">custom_head</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span>
-        <span class="p">),</span> <span class="s2">&quot;When using &#39;head&#39; as fusion_method, either head_hidden_dims or custom_head must be provided&quot;</span>
-        <span class="k">if</span> <span class="n">custom_head</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
-            <span class="c1"># custom_head takes precedence over head_hidden_dims (in case</span>
-            <span class="c1"># both are provided)</span>
-            <span class="k">assert</span> <span class="nb">hasattr</span><span class="p">(</span>
-                <span class="n">custom_head</span><span class="p">,</span> <span class="s2">&quot;output_dim&quot;</span>
-            <span class="p">),</span> <span class="s2">&quot;custom_head must have an &#39;output_dim&#39; property&quot;</span>
-            <span class="bp">self</span><span class="o">.</span><span class="n">head</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="n">BaseWDModelComponent</span><span class="p">,</span> <span class="n">nn</span><span class="o">.</span><span class="n">Module</span><span class="p">]</span> <span class="o">=</span> <span class="n">custom_head</span>
+        <span class="k">if</span> <span class="n">has_size_dependent_method</span> <span class="ow">and</span> <span class="n">needs_projection</span><span class="p">:</span>
+            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">projection_method</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+                <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
+                    <span class="s2">&quot;If &#39;fusion_method&#39; is not &#39;concatenate&#39; or &#39;head&#39;, &quot;</span>
+                    <span class="s2">&quot;and the output dimensions of the models are not equal, &quot;</span>
+                    <span class="s2">&quot;&#39;projection_method&#39; must be provided&quot;</span>
+                <span class="p">)</span>
+            <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">projection_method</span> <span class="ow">not</span> <span class="ow">in</span> <span class="p">[</span><span class="s2">&quot;min&quot;</span><span class="p">,</span> <span class="s2">&quot;max&quot;</span><span class="p">,</span> <span class="s2">&quot;mean&quot;</span><span class="p">]:</span>
+                <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
+                    <span class="s2">&quot;projection_method must be one of [&#39;min&#39;, &#39;max&#39;, &#39;mean&#39;]&quot;</span>
+                <span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">_validate_head_dot_exclusivity</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">fusion_method</span><span class="p">,</span> <span class="nb">list</span><span class="p">):</span>
+            <span class="k">if</span> <span class="nb">any</span><span class="p">(</span><span class="n">method</span> <span class="ow">in</span> <span class="p">[</span><span class="s2">&quot;head&quot;</span><span class="p">,</span> <span class="s2">&quot;dot&quot;</span><span class="p">]</span> <span class="k">for</span> <span class="n">method</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">fusion_method</span><span class="p">):</span>
+                <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
+                    <span class="s2">&quot;When using &#39;head&#39; or &#39;dot&#39; as fusion_method, no other method should be provided&quot;</span>
+                <span class="p">)</span>
+
+    <span class="k">def</span> <span class="fm">__repr__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">projection_method</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="n">proj</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">projection_method</span><span class="si">}</span><span class="s2">&quot;</span>
         <span class="k">else</span><span class="p">:</span>
-            <span class="k">assert</span> <span class="n">head_hidden_dims</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span>
-            <span class="bp">self</span><span class="o">.</span><span class="n">head_hidden_dims</span> <span class="o">=</span> <span class="n">head_hidden_dims</span>
-            <span class="bp">self</span><span class="o">.</span><span class="n">head_activation</span> <span class="o">=</span> <span class="n">head_activation</span>
-            <span class="bp">self</span><span class="o">.</span><span class="n">head_dropout</span> <span class="o">=</span> <span class="n">head_dropout</span>
-            <span class="bp">self</span><span class="o">.</span><span class="n">head_batchnorm</span> <span class="o">=</span> <span class="n">head_batchnorm</span>
-            <span class="bp">self</span><span class="o">.</span><span class="n">head_batchnorm_last</span> <span class="o">=</span> <span class="n">head_batchnorm_last</span>
-            <span class="bp">self</span><span class="o">.</span><span class="n">head_linear_first</span> <span class="o">=</span> <span class="n">head_linear_first</span>
+            <span class="n">proj</span> <span class="o">=</span> <span class="s2">&quot;&quot;</span>
 
-            <span class="bp">self</span><span class="o">.</span><span class="n">head</span> <span class="o">=</span> <span class="n">MLP</span><span class="p">(</span>
-                <span class="n">d_hidden</span><span class="o">=</span><span class="p">[</span><span class="nb">sum</span><span class="p">([</span><span class="n">model</span><span class="o">.</span><span class="n">output_dim</span> <span class="k">for</span> <span class="n">model</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">models</span><span class="p">])]</span>
-                <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">head_hidden_dims</span><span class="p">,</span>
-                <span class="n">activation</span><span class="o">=</span><span class="p">(</span>
-                    <span class="s2">&quot;relu&quot;</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">head_activation</span> <span class="ow">is</span> <span class="kc">None</span> <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">head_activation</span>
-                <span class="p">),</span>
-                <span class="n">dropout</span><span class="o">=</span><span class="mf">0.0</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">head_dropout</span> <span class="ow">is</span> <span class="kc">None</span> <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">head_dropout</span><span class="p">,</span>
-                <span class="n">batchnorm</span><span class="o">=</span><span class="p">(</span>
-                    <span class="kc">False</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">head_batchnorm</span> <span class="ow">is</span> <span class="kc">None</span> <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">head_batchnorm</span>
-                <span class="p">),</span>
-                <span class="n">batchnorm_last</span><span class="o">=</span><span class="p">(</span>
-                    <span class="kc">False</span>
-                    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">head_batchnorm_last</span> <span class="ow">is</span> <span class="kc">None</span>
-                    <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">head_batchnorm_last</span>
-                <span class="p">),</span>
-                <span class="n">linear_first</span><span class="o">=</span><span class="p">(</span>
-                    <span class="kc">True</span>
-                    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">head_linear_first</span> <span class="ow">is</span> <span class="kc">None</span>
-                    <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">head_linear_first</span>
-                <span class="p">),</span>
-            <span class="p">)</span>
+        <span class="k">return</span> <span class="sa">f</span><span class="s2">&quot;Fusion method: </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">fusion_method</span><span class="si">}</span><span class="s2">. Projection method: </span><span class="si">{</span><span class="n">proj</span><span class="si">}</span><span class="se">\n</span><span class="s2">Fused Models:</span><span class="se">\n</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">models</span><span class="si">}</span><span class="s2">&quot;</span>
 </code></pre></div></td></tr></table></div>
-                  </details>
+              </details>
 
 
 
@@ -12903,24 +21199,521 @@ <h3 id="pytorch_widedeep.models.model_fusion.ModelFuser.output_dim" class="doc d
 
 
 
-<div class="doc doc-object doc-function">
 
 
-<h3 id="pytorch_widedeep.models.model_fusion.ModelFuser.project" class="doc doc-heading">
-            <span class="doc doc-object-name doc-function-name">project</span>
+  </div>
 
+    </div>
 
-<a href="#pytorch_widedeep.models.model_fusion.ModelFuser.project" class="headerlink" title="Permanent link">&para;</a></h3>
-<div class="doc-signature highlight"><pre><span></span><code><span class="nf">project</span><span class="p">(</span><span class="n">X</span><span class="p">)</span>
-</code></pre></div>
+</div>
 
-    <div class="doc doc-contents ">
+<div class="doc doc-object doc-class">
 
-        <p>Projects the output of the models to a common dimension.</p>
 
-            <details class="quote">
-              <summary>Source code in <code>pytorch_widedeep/models/model_fusion.py</code></summary>
-              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">232</span>
+
+<h2 id="pytorch_widedeep.models.wide_deep.WideDeep" class="doc doc-heading">
+            <span class="doc doc-object-name doc-class-name">WideDeep</span>
+
+
+<a href="#pytorch_widedeep.models.wide_deep.WideDeep" class="headerlink" title="Permanent link">&para;</a></h2>
+
+
+    <div class="doc doc-contents first">
+            <p class="doc doc-class-bases">
+              Bases: <code><span title="torch.nn.Module">Module</span></code></p>
+
+
+        <p>Main collector class that combines all <code>wide</code>, <code>deeptabular</code>
+<code>deeptext</code> and <code>deepimage</code> models.</p>
+<p>Note that all models described so far in this library must be passed to
+the <code>WideDeep</code> class once constructed. This is because the models output
+the last layer before the prediction layer. Such prediction layer is
+added by the <code>WideDeep</code> class as it collects the components for every
+data mode.</p>
+<p>There are two options to combine these models that correspond to the
+two main architectures that <code>pytorch-widedeep</code> can build.</p>
+<ul>
+<li>
+<p>Directly connecting the output of the model components to an ouput neuron(s).</p>
+</li>
+<li>
+<p>Adding a <code>Fully-Connected Head</code> (FC-Head) on top of the deep models.
+  This FC-Head will combine the output form the <code>deeptabular</code>, <code>deeptext</code> and
+  <code>deepimage</code> and will be then connected to the output neuron(s).</p>
+</li>
+</ul>
+
+
+<p><span class="doc-section-title">Parameters:</span></p>
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+          <th>Default</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>wide</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="torch.nn.Module">Module</span>]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p><code>Wide</code> model. This is a linear model where the non-linearities are
+captured via crossed-columns.</p>
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>deeptabular</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="pytorch_widedeep.models._base_wd_model_component.BaseWDModelComponent">BaseWDModelComponent</span>, <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.models._base_wd_model_component.BaseWDModelComponent">BaseWDModelComponent</span>]]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Currently this library implements a number of possible architectures
+for the <code>deeptabular</code> component. See the documenation of the
+package. Note that <code>deeptabular</code> can be a list of models. This is
+useful when using multiple tabular inputs (e.g. for example in the
+context of a two-tower model for recommendation systems)</p>
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>deeptext</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="pytorch_widedeep.models._base_wd_model_component.BaseWDModelComponent">BaseWDModelComponent</span>, <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.models._base_wd_model_component.BaseWDModelComponent">BaseWDModelComponent</span>]]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Currently this library implements a number of possible architectures
+for the <code>deeptext</code> component. See the documenation of the
+package. Note that <code>deeptext</code> can be a list of models. This is useful
+when using multiple text inputs.</p>
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>deepimage</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="pytorch_widedeep.models._base_wd_model_component.BaseWDModelComponent">BaseWDModelComponent</span>, <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.models._base_wd_model_component.BaseWDModelComponent">BaseWDModelComponent</span>]]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Currently this library uses <code>torchvision</code> and implements a number of
+possible architectures for the <code>deepimage</code> component. See the
+documenation of the package. Note that <code>deepimage</code> can be a list of
+models. This is useful when using multiple image inputs.</p>
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>deephead</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.models._base_wd_model_component.BaseWDModelComponent">BaseWDModelComponent</span>]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Alternatively, the user can pass a custom model that will receive the
+output of the deep component. If <code>deephead</code> is not None all the
+previous fc-head parameters will be ignored</p>
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>head_hidden_dims</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[int]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>List with the sizes of the dense layers in the head e.g: [128, 64]</p>
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>head_activation</code></td>
+            <td>
+                  <code>str</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Activation function for the dense layers in the head. Currently
+<code>'tanh'</code>, <code>'relu'</code>, <code>'leaky_relu'</code> and <code>'gelu'</code> are supported</p>
+              </div>
+            </td>
+            <td>
+                  <code>&#39;relu&#39;</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>head_dropout</code></td>
+            <td>
+                  <code>float</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Dropout of the dense layers in the head</p>
+              </div>
+            </td>
+            <td>
+                  <code>0.1</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>head_batchnorm</code></td>
+            <td>
+                  <code>bool</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Boolean indicating whether or not to include batch normalization in
+the dense layers that form the <code>'rnn_mlp'</code></p>
+              </div>
+            </td>
+            <td>
+                  <code>False</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>head_batchnorm_last</code></td>
+            <td>
+                  <code>bool</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Boolean indicating whether or not to apply batch normalization to the
+last of the dense layers in the head</p>
+              </div>
+            </td>
+            <td>
+                  <code>False</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>head_linear_first</code></td>
+            <td>
+                  <code>bool</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Boolean indicating whether the order of the operations in the dense
+layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt;
+LIN -&gt; ACT]</code></p>
+              </div>
+            </td>
+            <td>
+                  <code>True</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>enforce_positive</code></td>
+            <td>
+                  <code>bool</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Boolean indicating if the output from the final layer must be
+positive. This is important if you are using loss functions with
+non-negative input restrictions, e.g. RMSLE, or if you know your
+predictions are bounded in between 0 and inf</p>
+              </div>
+            </td>
+            <td>
+                  <code>False</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>enforce_positive_activation</code></td>
+            <td>
+                  <code>str</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Activation function to enforce that the final layer has a positive
+output. <code>'softplus'</code> or <code>'relu'</code> are supported.</p>
+              </div>
+            </td>
+            <td>
+                  <code>&#39;softplus&#39;</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>pred_dim</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Size of the final wide and deep output layer containing the
+predictions. <code>1</code> for regression and binary classification or number
+of classes for multiclass classification.</p>
+              </div>
+            </td>
+            <td>
+                  <code>1</code>
+            </td>
+          </tr>
+      </tbody>
+    </table>
+
+
+<p><span class="doc-section-title">Examples:</span></p>
+    <div class="highlight"><pre><span></span><code><span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">pytorch_widedeep.models</span> <span class="kn">import</span> <span class="n">TabResnet</span><span class="p">,</span> <span class="n">Vision</span><span class="p">,</span> <span class="n">BasicRNN</span><span class="p">,</span> <span class="n">Wide</span><span class="p">,</span> <span class="n">WideDeep</span>
+<span class="gp">&gt;&gt;&gt; </span><span class="n">embed_input</span> <span class="o">=</span> <span class="p">[(</span><span class="n">u</span><span class="p">,</span> <span class="n">i</span><span class="p">,</span> <span class="n">j</span><span class="p">)</span> <span class="k">for</span> <span class="n">u</span><span class="p">,</span> <span class="n">i</span><span class="p">,</span> <span class="n">j</span> <span class="ow">in</span> <span class="nb">zip</span><span class="p">([</span><span class="s2">&quot;a&quot;</span><span class="p">,</span> <span class="s2">&quot;b&quot;</span><span class="p">,</span> <span class="s2">&quot;c&quot;</span><span class="p">][:</span><span class="mi">4</span><span class="p">],</span> <span class="p">[</span><span class="mi">4</span><span class="p">]</span> <span class="o">*</span> <span class="mi">3</span><span class="p">,</span> <span class="p">[</span><span class="mi">8</span><span class="p">]</span> <span class="o">*</span> <span class="mi">3</span><span class="p">)]</span>
+<span class="gp">&gt;&gt;&gt; </span><span class="n">column_idx</span> <span class="o">=</span> <span class="p">{</span><span class="n">k</span><span class="p">:</span> <span class="n">v</span> <span class="k">for</span> <span class="n">v</span><span class="p">,</span> <span class="n">k</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">([</span><span class="s2">&quot;a&quot;</span><span class="p">,</span> <span class="s2">&quot;b&quot;</span><span class="p">,</span> <span class="s2">&quot;c&quot;</span><span class="p">])}</span>
+<span class="gp">&gt;&gt;&gt; </span><span class="n">wide</span> <span class="o">=</span> <span class="n">Wide</span><span class="p">(</span><span class="mi">10</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span>
+<span class="gp">&gt;&gt;&gt; </span><span class="n">deeptabular</span> <span class="o">=</span> <span class="n">TabResnet</span><span class="p">(</span><span class="n">blocks_dims</span><span class="o">=</span><span class="p">[</span><span class="mi">8</span><span class="p">,</span> <span class="mi">4</span><span class="p">],</span> <span class="n">column_idx</span><span class="o">=</span><span class="n">column_idx</span><span class="p">,</span> <span class="n">cat_embed_input</span><span class="o">=</span><span class="n">embed_input</span><span class="p">)</span>
+<span class="gp">&gt;&gt;&gt; </span><span class="n">deeptext</span> <span class="o">=</span> <span class="n">BasicRNN</span><span class="p">(</span><span class="n">vocab_size</span><span class="o">=</span><span class="mi">10</span><span class="p">,</span> <span class="n">embed_dim</span><span class="o">=</span><span class="mi">4</span><span class="p">,</span> <span class="n">padding_idx</span><span class="o">=</span><span class="mi">0</span><span class="p">)</span>
+<span class="gp">&gt;&gt;&gt; </span><span class="n">deepimage</span> <span class="o">=</span> <span class="n">Vision</span><span class="p">()</span>
+<span class="gp">&gt;&gt;&gt; </span><span class="n">model</span> <span class="o">=</span> <span class="n">WideDeep</span><span class="p">(</span><span class="n">wide</span><span class="o">=</span><span class="n">wide</span><span class="p">,</span> <span class="n">deeptabular</span><span class="o">=</span><span class="n">deeptabular</span><span class="p">,</span> <span class="n">deeptext</span><span class="o">=</span><span class="n">deeptext</span><span class="p">,</span> <span class="n">deepimage</span><span class="o">=</span><span class="n">deepimage</span><span class="p">)</span>
+</code></pre></div>
+    <p><img alt="ℹ️" class="emojione" src="https://cdnjs.cloudflare.com/ajax/libs/emojione/2.2.7/assets/png/2139.png" title=":information_source:" /> <strong>NOTE</strong>: It is possible to use custom components to
+ build Wide &amp; Deep models. Simply, build them and pass them as the
+ corresponding parameters. Note that the custom models MUST return a last
+ layer of activations(i.e. not the final prediction) so that  these
+ activations are collected by <code>WideDeep</code> and combined accordingly. In
+ addition, the models MUST also contain an attribute <code>output_dim</code> with
+ the size of these last layers of activations. See for example
+ <code>pytorch_widedeep.models.tab_mlp.TabMlp</code></p>
+
+              <details class="quote">
+                <summary>Source code in <code>pytorch_widedeep/models/wide_deep.py</code></summary>
+                <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 19</span>
+<span class="normal"> 20</span>
+<span class="normal"> 21</span>
+<span class="normal"> 22</span>
+<span class="normal"> 23</span>
+<span class="normal"> 24</span>
+<span class="normal"> 25</span>
+<span class="normal"> 26</span>
+<span class="normal"> 27</span>
+<span class="normal"> 28</span>
+<span class="normal"> 29</span>
+<span class="normal"> 30</span>
+<span class="normal"> 31</span>
+<span class="normal"> 32</span>
+<span class="normal"> 33</span>
+<span class="normal"> 34</span>
+<span class="normal"> 35</span>
+<span class="normal"> 36</span>
+<span class="normal"> 37</span>
+<span class="normal"> 38</span>
+<span class="normal"> 39</span>
+<span class="normal"> 40</span>
+<span class="normal"> 41</span>
+<span class="normal"> 42</span>
+<span class="normal"> 43</span>
+<span class="normal"> 44</span>
+<span class="normal"> 45</span>
+<span class="normal"> 46</span>
+<span class="normal"> 47</span>
+<span class="normal"> 48</span>
+<span class="normal"> 49</span>
+<span class="normal"> 50</span>
+<span class="normal"> 51</span>
+<span class="normal"> 52</span>
+<span class="normal"> 53</span>
+<span class="normal"> 54</span>
+<span class="normal"> 55</span>
+<span class="normal"> 56</span>
+<span class="normal"> 57</span>
+<span class="normal"> 58</span>
+<span class="normal"> 59</span>
+<span class="normal"> 60</span>
+<span class="normal"> 61</span>
+<span class="normal"> 62</span>
+<span class="normal"> 63</span>
+<span class="normal"> 64</span>
+<span class="normal"> 65</span>
+<span class="normal"> 66</span>
+<span class="normal"> 67</span>
+<span class="normal"> 68</span>
+<span class="normal"> 69</span>
+<span class="normal"> 70</span>
+<span class="normal"> 71</span>
+<span class="normal"> 72</span>
+<span class="normal"> 73</span>
+<span class="normal"> 74</span>
+<span class="normal"> 75</span>
+<span class="normal"> 76</span>
+<span class="normal"> 77</span>
+<span class="normal"> 78</span>
+<span class="normal"> 79</span>
+<span class="normal"> 80</span>
+<span class="normal"> 81</span>
+<span class="normal"> 82</span>
+<span class="normal"> 83</span>
+<span class="normal"> 84</span>
+<span class="normal"> 85</span>
+<span class="normal"> 86</span>
+<span class="normal"> 87</span>
+<span class="normal"> 88</span>
+<span class="normal"> 89</span>
+<span class="normal"> 90</span>
+<span class="normal"> 91</span>
+<span class="normal"> 92</span>
+<span class="normal"> 93</span>
+<span class="normal"> 94</span>
+<span class="normal"> 95</span>
+<span class="normal"> 96</span>
+<span class="normal"> 97</span>
+<span class="normal"> 98</span>
+<span class="normal"> 99</span>
+<span class="normal">100</span>
+<span class="normal">101</span>
+<span class="normal">102</span>
+<span class="normal">103</span>
+<span class="normal">104</span>
+<span class="normal">105</span>
+<span class="normal">106</span>
+<span class="normal">107</span>
+<span class="normal">108</span>
+<span class="normal">109</span>
+<span class="normal">110</span>
+<span class="normal">111</span>
+<span class="normal">112</span>
+<span class="normal">113</span>
+<span class="normal">114</span>
+<span class="normal">115</span>
+<span class="normal">116</span>
+<span class="normal">117</span>
+<span class="normal">118</span>
+<span class="normal">119</span>
+<span class="normal">120</span>
+<span class="normal">121</span>
+<span class="normal">122</span>
+<span class="normal">123</span>
+<span class="normal">124</span>
+<span class="normal">125</span>
+<span class="normal">126</span>
+<span class="normal">127</span>
+<span class="normal">128</span>
+<span class="normal">129</span>
+<span class="normal">130</span>
+<span class="normal">131</span>
+<span class="normal">132</span>
+<span class="normal">133</span>
+<span class="normal">134</span>
+<span class="normal">135</span>
+<span class="normal">136</span>
+<span class="normal">137</span>
+<span class="normal">138</span>
+<span class="normal">139</span>
+<span class="normal">140</span>
+<span class="normal">141</span>
+<span class="normal">142</span>
+<span class="normal">143</span>
+<span class="normal">144</span>
+<span class="normal">145</span>
+<span class="normal">146</span>
+<span class="normal">147</span>
+<span class="normal">148</span>
+<span class="normal">149</span>
+<span class="normal">150</span>
+<span class="normal">151</span>
+<span class="normal">152</span>
+<span class="normal">153</span>
+<span class="normal">154</span>
+<span class="normal">155</span>
+<span class="normal">156</span>
+<span class="normal">157</span>
+<span class="normal">158</span>
+<span class="normal">159</span>
+<span class="normal">160</span>
+<span class="normal">161</span>
+<span class="normal">162</span>
+<span class="normal">163</span>
+<span class="normal">164</span>
+<span class="normal">165</span>
+<span class="normal">166</span>
+<span class="normal">167</span>
+<span class="normal">168</span>
+<span class="normal">169</span>
+<span class="normal">170</span>
+<span class="normal">171</span>
+<span class="normal">172</span>
+<span class="normal">173</span>
+<span class="normal">174</span>
+<span class="normal">175</span>
+<span class="normal">176</span>
+<span class="normal">177</span>
+<span class="normal">178</span>
+<span class="normal">179</span>
+<span class="normal">180</span>
+<span class="normal">181</span>
+<span class="normal">182</span>
+<span class="normal">183</span>
+<span class="normal">184</span>
+<span class="normal">185</span>
+<span class="normal">186</span>
+<span class="normal">187</span>
+<span class="normal">188</span>
+<span class="normal">189</span>
+<span class="normal">190</span>
+<span class="normal">191</span>
+<span class="normal">192</span>
+<span class="normal">193</span>
+<span class="normal">194</span>
+<span class="normal">195</span>
+<span class="normal">196</span>
+<span class="normal">197</span>
+<span class="normal">198</span>
+<span class="normal">199</span>
+<span class="normal">200</span>
+<span class="normal">201</span>
+<span class="normal">202</span>
+<span class="normal">203</span>
+<span class="normal">204</span>
+<span class="normal">205</span>
+<span class="normal">206</span>
+<span class="normal">207</span>
+<span class="normal">208</span>
+<span class="normal">209</span>
+<span class="normal">210</span>
+<span class="normal">211</span>
+<span class="normal">212</span>
+<span class="normal">213</span>
+<span class="normal">214</span>
+<span class="normal">215</span>
+<span class="normal">216</span>
+<span class="normal">217</span>
+<span class="normal">218</span>
+<span class="normal">219</span>
+<span class="normal">220</span>
+<span class="normal">221</span>
+<span class="normal">222</span>
+<span class="normal">223</span>
+<span class="normal">224</span>
+<span class="normal">225</span>
+<span class="normal">226</span>
+<span class="normal">227</span>
+<span class="normal">228</span>
+<span class="normal">229</span>
+<span class="normal">230</span>
+<span class="normal">231</span>
+<span class="normal">232</span>
 <span class="normal">233</span>
 <span class="normal">234</span>
 <span class="normal">235</span>
@@ -12949,406 +21742,717 @@ <h3 id="pytorch_widedeep.models.model_fusion.ModelFuser.project" class="doc doc-
 <span class="normal">258</span>
 <span class="normal">259</span>
 <span class="normal">260</span>
-<span class="normal">261</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">project</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">X</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="n">Tensor</span><span class="p">])</span> <span class="o">-&gt;</span> <span class="n">List</span><span class="p">[</span><span class="n">Tensor</span><span class="p">]:</span>
-<span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Projects the output of the models to a common dimension.&quot;&quot;&quot;</span>
-
-    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">all_output_dim_equal</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">projection_method</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
-        <span class="k">return</span> <span class="n">X</span>
-
-    <span class="n">output_dims</span> <span class="o">=</span> <span class="p">[</span><span class="n">model</span><span class="o">.</span><span class="n">output_dim</span> <span class="k">for</span> <span class="n">model</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">models</span><span class="p">]</span>
-
-    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">projection_method</span> <span class="o">==</span> <span class="s2">&quot;min&quot;</span><span class="p">:</span>
-        <span class="n">proj_dim</span> <span class="o">=</span> <span class="nb">min</span><span class="p">(</span><span class="n">output_dims</span><span class="p">)</span>
-        <span class="n">idx</span> <span class="o">=</span> <span class="n">output_dims</span><span class="o">.</span><span class="n">index</span><span class="p">(</span><span class="n">proj_dim</span><span class="p">)</span>
-    <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">projection_method</span> <span class="o">==</span> <span class="s2">&quot;max&quot;</span><span class="p">:</span>
-        <span class="n">proj_dim</span> <span class="o">=</span> <span class="nb">max</span><span class="p">(</span><span class="n">output_dims</span><span class="p">)</span>
-        <span class="n">idx</span> <span class="o">=</span> <span class="n">output_dims</span><span class="o">.</span><span class="n">index</span><span class="p">(</span><span class="n">proj_dim</span><span class="p">)</span>
-    <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">projection_method</span> <span class="o">==</span> <span class="s2">&quot;mean&quot;</span><span class="p">:</span>
-        <span class="n">proj_dim</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="nb">sum</span><span class="p">(</span><span class="n">output_dims</span><span class="p">)</span> <span class="o">/</span> <span class="nb">len</span><span class="p">(</span><span class="n">output_dims</span><span class="p">))</span>
-        <span class="n">idx</span> <span class="o">=</span> <span class="kc">None</span>
-    <span class="k">else</span><span class="p">:</span>
-        <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">&quot;projection_method must be one of [&#39;min&#39;, &#39;max&#39;, &#39;mean&#39;]&quot;</span><span class="p">)</span>
-
-    <span class="n">x_proj</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="n">Tensor</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
-    <span class="k">for</span> <span class="n">i</span><span class="p">,</span> <span class="n">x</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="n">X</span><span class="p">):</span>
-        <span class="k">if</span> <span class="n">i</span> <span class="o">==</span> <span class="n">idx</span><span class="p">:</span>
-            <span class="n">x_proj</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">x</span><span class="p">)</span>
-        <span class="k">else</span><span class="p">:</span>
-            <span class="n">x_proj</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
-                <span class="n">nn</span><span class="o">.</span><span class="n">Linear</span><span class="p">(</span><span class="n">output_dims</span><span class="p">[</span><span class="n">i</span><span class="p">],</span> <span class="n">proj_dim</span><span class="p">,</span> <span class="n">bias</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">device</span><span class="o">=</span><span class="n">x</span><span class="o">.</span><span class="n">device</span><span class="p">)(</span><span class="n">x</span><span class="p">)</span>
-            <span class="p">)</span>
-
-    <span class="k">return</span> <span class="n">x_proj</span>
-</code></pre></div></td></tr></table></div>
-            </details>
-    </div>
+<span class="normal">261</span>
+<span class="normal">262</span>
+<span class="normal">263</span>
+<span class="normal">264</span>
+<span class="normal">265</span>
+<span class="normal">266</span>
+<span class="normal">267</span>
+<span class="normal">268</span>
+<span class="normal">269</span>
+<span class="normal">270</span>
+<span class="normal">271</span>
+<span class="normal">272</span>
+<span class="normal">273</span>
+<span class="normal">274</span>
+<span class="normal">275</span>
+<span class="normal">276</span>
+<span class="normal">277</span>
+<span class="normal">278</span>
+<span class="normal">279</span>
+<span class="normal">280</span>
+<span class="normal">281</span>
+<span class="normal">282</span>
+<span class="normal">283</span>
+<span class="normal">284</span>
+<span class="normal">285</span>
+<span class="normal">286</span>
+<span class="normal">287</span>
+<span class="normal">288</span>
+<span class="normal">289</span>
+<span class="normal">290</span>
+<span class="normal">291</span>
+<span class="normal">292</span>
+<span class="normal">293</span>
+<span class="normal">294</span>
+<span class="normal">295</span>
+<span class="normal">296</span>
+<span class="normal">297</span>
+<span class="normal">298</span>
+<span class="normal">299</span>
+<span class="normal">300</span>
+<span class="normal">301</span>
+<span class="normal">302</span>
+<span class="normal">303</span>
+<span class="normal">304</span>
+<span class="normal">305</span>
+<span class="normal">306</span>
+<span class="normal">307</span>
+<span class="normal">308</span>
+<span class="normal">309</span>
+<span class="normal">310</span>
+<span class="normal">311</span>
+<span class="normal">312</span>
+<span class="normal">313</span>
+<span class="normal">314</span>
+<span class="normal">315</span>
+<span class="normal">316</span>
+<span class="normal">317</span>
+<span class="normal">318</span>
+<span class="normal">319</span>
+<span class="normal">320</span>
+<span class="normal">321</span>
+<span class="normal">322</span>
+<span class="normal">323</span>
+<span class="normal">324</span>
+<span class="normal">325</span>
+<span class="normal">326</span>
+<span class="normal">327</span>
+<span class="normal">328</span>
+<span class="normal">329</span>
+<span class="normal">330</span>
+<span class="normal">331</span>
+<span class="normal">332</span>
+<span class="normal">333</span>
+<span class="normal">334</span>
+<span class="normal">335</span>
+<span class="normal">336</span>
+<span class="normal">337</span>
+<span class="normal">338</span>
+<span class="normal">339</span>
+<span class="normal">340</span>
+<span class="normal">341</span>
+<span class="normal">342</span>
+<span class="normal">343</span>
+<span class="normal">344</span>
+<span class="normal">345</span>
+<span class="normal">346</span>
+<span class="normal">347</span>
+<span class="normal">348</span>
+<span class="normal">349</span>
+<span class="normal">350</span>
+<span class="normal">351</span>
+<span class="normal">352</span>
+<span class="normal">353</span>
+<span class="normal">354</span>
+<span class="normal">355</span>
+<span class="normal">356</span>
+<span class="normal">357</span>
+<span class="normal">358</span>
+<span class="normal">359</span>
+<span class="normal">360</span>
+<span class="normal">361</span>
+<span class="normal">362</span>
+<span class="normal">363</span>
+<span class="normal">364</span>
+<span class="normal">365</span>
+<span class="normal">366</span>
+<span class="normal">367</span>
+<span class="normal">368</span>
+<span class="normal">369</span>
+<span class="normal">370</span>
+<span class="normal">371</span>
+<span class="normal">372</span>
+<span class="normal">373</span>
+<span class="normal">374</span>
+<span class="normal">375</span>
+<span class="normal">376</span>
+<span class="normal">377</span>
+<span class="normal">378</span>
+<span class="normal">379</span>
+<span class="normal">380</span>
+<span class="normal">381</span>
+<span class="normal">382</span>
+<span class="normal">383</span>
+<span class="normal">384</span>
+<span class="normal">385</span>
+<span class="normal">386</span>
+<span class="normal">387</span>
+<span class="normal">388</span>
+<span class="normal">389</span>
+<span class="normal">390</span>
+<span class="normal">391</span>
+<span class="normal">392</span>
+<span class="normal">393</span>
+<span class="normal">394</span>
+<span class="normal">395</span>
+<span class="normal">396</span>
+<span class="normal">397</span>
+<span class="normal">398</span>
+<span class="normal">399</span>
+<span class="normal">400</span>
+<span class="normal">401</span>
+<span class="normal">402</span>
+<span class="normal">403</span>
+<span class="normal">404</span>
+<span class="normal">405</span>
+<span class="normal">406</span>
+<span class="normal">407</span>
+<span class="normal">408</span>
+<span class="normal">409</span>
+<span class="normal">410</span>
+<span class="normal">411</span>
+<span class="normal">412</span>
+<span class="normal">413</span>
+<span class="normal">414</span>
+<span class="normal">415</span>
+<span class="normal">416</span>
+<span class="normal">417</span>
+<span class="normal">418</span>
+<span class="normal">419</span>
+<span class="normal">420</span>
+<span class="normal">421</span>
+<span class="normal">422</span>
+<span class="normal">423</span>
+<span class="normal">424</span>
+<span class="normal">425</span>
+<span class="normal">426</span>
+<span class="normal">427</span>
+<span class="normal">428</span>
+<span class="normal">429</span>
+<span class="normal">430</span>
+<span class="normal">431</span>
+<span class="normal">432</span>
+<span class="normal">433</span>
+<span class="normal">434</span>
+<span class="normal">435</span>
+<span class="normal">436</span>
+<span class="normal">437</span>
+<span class="normal">438</span>
+<span class="normal">439</span>
+<span class="normal">440</span>
+<span class="normal">441</span>
+<span class="normal">442</span>
+<span class="normal">443</span>
+<span class="normal">444</span>
+<span class="normal">445</span>
+<span class="normal">446</span>
+<span class="normal">447</span>
+<span class="normal">448</span>
+<span class="normal">449</span>
+<span class="normal">450</span>
+<span class="normal">451</span>
+<span class="normal">452</span>
+<span class="normal">453</span>
+<span class="normal">454</span>
+<span class="normal">455</span>
+<span class="normal">456</span>
+<span class="normal">457</span>
+<span class="normal">458</span>
+<span class="normal">459</span>
+<span class="normal">460</span>
+<span class="normal">461</span>
+<span class="normal">462</span>
+<span class="normal">463</span>
+<span class="normal">464</span>
+<span class="normal">465</span>
+<span class="normal">466</span>
+<span class="normal">467</span>
+<span class="normal">468</span>
+<span class="normal">469</span>
+<span class="normal">470</span>
+<span class="normal">471</span>
+<span class="normal">472</span>
+<span class="normal">473</span>
+<span class="normal">474</span>
+<span class="normal">475</span>
+<span class="normal">476</span>
+<span class="normal">477</span>
+<span class="normal">478</span>
+<span class="normal">479</span>
+<span class="normal">480</span>
+<span class="normal">481</span>
+<span class="normal">482</span>
+<span class="normal">483</span>
+<span class="normal">484</span>
+<span class="normal">485</span>
+<span class="normal">486</span>
+<span class="normal">487</span>
+<span class="normal">488</span>
+<span class="normal">489</span>
+<span class="normal">490</span>
+<span class="normal">491</span>
+<span class="normal">492</span>
+<span class="normal">493</span>
+<span class="normal">494</span>
+<span class="normal">495</span>
+<span class="normal">496</span>
+<span class="normal">497</span>
+<span class="normal">498</span>
+<span class="normal">499</span>
+<span class="normal">500</span>
+<span class="normal">501</span>
+<span class="normal">502</span>
+<span class="normal">503</span>
+<span class="normal">504</span>
+<span class="normal">505</span>
+<span class="normal">506</span>
+<span class="normal">507</span>
+<span class="normal">508</span>
+<span class="normal">509</span>
+<span class="normal">510</span>
+<span class="normal">511</span>
+<span class="normal">512</span>
+<span class="normal">513</span>
+<span class="normal">514</span>
+<span class="normal">515</span>
+<span class="normal">516</span>
+<span class="normal">517</span>
+<span class="normal">518</span>
+<span class="normal">519</span>
+<span class="normal">520</span>
+<span class="normal">521</span>
+<span class="normal">522</span>
+<span class="normal">523</span>
+<span class="normal">524</span>
+<span class="normal">525</span>
+<span class="normal">526</span>
+<span class="normal">527</span>
+<span class="normal">528</span>
+<span class="normal">529</span>
+<span class="normal">530</span>
+<span class="normal">531</span>
+<span class="normal">532</span>
+<span class="normal">533</span>
+<span class="normal">534</span>
+<span class="normal">535</span>
+<span class="normal">536</span>
+<span class="normal">537</span>
+<span class="normal">538</span>
+<span class="normal">539</span>
+<span class="normal">540</span>
+<span class="normal">541</span>
+<span class="normal">542</span>
+<span class="normal">543</span>
+<span class="normal">544</span>
+<span class="normal">545</span>
+<span class="normal">546</span>
+<span class="normal">547</span>
+<span class="normal">548</span>
+<span class="normal">549</span>
+<span class="normal">550</span>
+<span class="normal">551</span>
+<span class="normal">552</span>
+<span class="normal">553</span>
+<span class="normal">554</span>
+<span class="normal">555</span>
+<span class="normal">556</span>
+<span class="normal">557</span>
+<span class="normal">558</span>
+<span class="normal">559</span>
+<span class="normal">560</span>
+<span class="normal">561</span>
+<span class="normal">562</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">class</span> <span class="nc">WideDeep</span><span class="p">(</span><span class="n">nn</span><span class="o">.</span><span class="n">Module</span><span class="p">):</span>
+<span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Main collector class that combines all `wide`, `deeptabular`</span>
+<span class="sd">    `deeptext` and `deepimage` models.</span>
+
+<span class="sd">    Note that all models described so far in this library must be passed to</span>
+<span class="sd">    the `WideDeep` class once constructed. This is because the models output</span>
+<span class="sd">    the last layer before the prediction layer. Such prediction layer is</span>
+<span class="sd">    added by the `WideDeep` class as it collects the components for every</span>
+<span class="sd">    data mode.</span>
+
+<span class="sd">    There are two options to combine these models that correspond to the</span>
+<span class="sd">    two main architectures that `pytorch-widedeep` can build.</span>
+
+<span class="sd">    - Directly connecting the output of the model components to an ouput neuron(s).</span>
+
+<span class="sd">    - Adding a `Fully-Connected Head` (FC-Head) on top of the deep models.</span>
+<span class="sd">      This FC-Head will combine the output form the `deeptabular`, `deeptext` and</span>
+<span class="sd">      `deepimage` and will be then connected to the output neuron(s).</span>
+
+<span class="sd">    Parameters</span>
+<span class="sd">    ----------</span>
+<span class="sd">    wide: nn.Module, Optional, default = None</span>
+<span class="sd">        `Wide` model. This is a linear model where the non-linearities are</span>
+<span class="sd">        captured via crossed-columns.</span>
+<span class="sd">    deeptabular: BaseWDModelComponent, Optional, default = None</span>
+<span class="sd">        Currently this library implements a number of possible architectures</span>
+<span class="sd">        for the `deeptabular` component. See the documenation of the</span>
+<span class="sd">        package. Note that `deeptabular` can be a list of models. This is</span>
+<span class="sd">        useful when using multiple tabular inputs (e.g. for example in the</span>
+<span class="sd">        context of a two-tower model for recommendation systems)</span>
+<span class="sd">    deeptext: BaseWDModelComponent | List[BaseWDModelComponent], Optional, default = None</span>
+<span class="sd">        Currently this library implements a number of possible architectures</span>
+<span class="sd">        for the `deeptext` component. See the documenation of the</span>
+<span class="sd">        package. Note that `deeptext` can be a list of models. This is useful</span>
+<span class="sd">        when using multiple text inputs.</span>
+<span class="sd">    deepimage: BaseWDModelComponent | List[BaseWDModelComponent], Optional, default = None</span>
+<span class="sd">        Currently this library uses `torchvision` and implements a number of</span>
+<span class="sd">        possible architectures for the `deepimage` component. See the</span>
+<span class="sd">        documenation of the package. Note that `deepimage` can be a list of</span>
+<span class="sd">        models. This is useful when using multiple image inputs.</span>
+<span class="sd">    deephead: BaseWDModelComponent, Optional, default = None</span>
+<span class="sd">        Alternatively, the user can pass a custom model that will receive the</span>
+<span class="sd">        output of the deep component. If `deephead` is not None all the</span>
+<span class="sd">        previous fc-head parameters will be ignored</span>
+<span class="sd">    head_hidden_dims: List, Optional, default = None</span>
+<span class="sd">        List with the sizes of the dense layers in the head e.g: [128, 64]</span>
+<span class="sd">    head_activation: str, default = &quot;relu&quot;</span>
+<span class="sd">        Activation function for the dense layers in the head. Currently</span>
+<span class="sd">        `&#39;tanh&#39;`, `&#39;relu&#39;`, `&#39;leaky_relu&#39;` and `&#39;gelu&#39;` are supported</span>
+<span class="sd">    head_dropout: float, Optional, default = None</span>
+<span class="sd">        Dropout of the dense layers in the head</span>
+<span class="sd">    head_batchnorm: bool, default = False</span>
+<span class="sd">        Boolean indicating whether or not to include batch normalization in</span>
+<span class="sd">        the dense layers that form the `&#39;rnn_mlp&#39;`</span>
+<span class="sd">    head_batchnorm_last: bool, default = False</span>
+<span class="sd">        Boolean indicating whether or not to apply batch normalization to the</span>
+<span class="sd">        last of the dense layers in the head</span>
+<span class="sd">    head_linear_first: bool, default = False</span>
+<span class="sd">        Boolean indicating whether the order of the operations in the dense</span>
+<span class="sd">        layer. If `True: [LIN -&gt; ACT -&gt; BN -&gt; DP]`. If `False: [BN -&gt; DP -&gt;</span>
+<span class="sd">        LIN -&gt; ACT]`</span>
+<span class="sd">    enforce_positive: bool, default = False</span>
+<span class="sd">        Boolean indicating if the output from the final layer must be</span>
+<span class="sd">        positive. This is important if you are using loss functions with</span>
+<span class="sd">        non-negative input restrictions, e.g. RMSLE, or if you know your</span>
+<span class="sd">        predictions are bounded in between 0 and inf</span>
+<span class="sd">    enforce_positive_activation: str, default = &quot;softplus&quot;</span>
+<span class="sd">        Activation function to enforce that the final layer has a positive</span>
+<span class="sd">        output. `&#39;softplus&#39;` or `&#39;relu&#39;` are supported.</span>
+<span class="sd">    pred_dim: int, default = 1</span>
+<span class="sd">        Size of the final wide and deep output layer containing the</span>
+<span class="sd">        predictions. `1` for regression and binary classification or number</span>
+<span class="sd">        of classes for multiclass classification.</span>
+
+
+<span class="sd">    Examples</span>
+<span class="sd">    --------</span>
+
+<span class="sd">    &gt;&gt;&gt; from pytorch_widedeep.models import TabResnet, Vision, BasicRNN, Wide, WideDeep</span>
+<span class="sd">    &gt;&gt;&gt; embed_input = [(u, i, j) for u, i, j in zip([&quot;a&quot;, &quot;b&quot;, &quot;c&quot;][:4], [4] * 3, [8] * 3)]</span>
+<span class="sd">    &gt;&gt;&gt; column_idx = {k: v for v, k in enumerate([&quot;a&quot;, &quot;b&quot;, &quot;c&quot;])}</span>
+<span class="sd">    &gt;&gt;&gt; wide = Wide(10, 1)</span>
+<span class="sd">    &gt;&gt;&gt; deeptabular = TabResnet(blocks_dims=[8, 4], column_idx=column_idx, cat_embed_input=embed_input)</span>
+<span class="sd">    &gt;&gt;&gt; deeptext = BasicRNN(vocab_size=10, embed_dim=4, padding_idx=0)</span>
+<span class="sd">    &gt;&gt;&gt; deepimage = Vision()</span>
+<span class="sd">    &gt;&gt;&gt; model = WideDeep(wide=wide, deeptabular=deeptabular, deeptext=deeptext, deepimage=deepimage)</span>
+
+
+<span class="sd">    :information_source: **NOTE**: It is possible to use custom components to</span>
+<span class="sd">     build Wide &amp; Deep models. Simply, build them and pass them as the</span>
+<span class="sd">     corresponding parameters. Note that the custom models MUST return a last</span>
+<span class="sd">     layer of activations(i.e. not the final prediction) so that  these</span>
+<span class="sd">     activations are collected by `WideDeep` and combined accordingly. In</span>
+<span class="sd">     addition, the models MUST also contain an attribute `output_dim` with</span>
+<span class="sd">     the size of these last layers of activations. See for example</span>
+<span class="sd">     `pytorch_widedeep.models.tab_mlp.TabMlp`</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+
+    <span class="nd">@alias</span><span class="p">(</span>  <span class="c1"># noqa: C901</span>
+        <span class="s2">&quot;pred_dim&quot;</span><span class="p">,</span>
+        <span class="p">[</span><span class="s2">&quot;num_class&quot;</span><span class="p">,</span> <span class="s2">&quot;pred_size&quot;</span><span class="p">],</span>
+    <span class="p">)</span>
+    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span>
+        <span class="n">wide</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">nn</span><span class="o">.</span><span class="n">Module</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">deeptabular</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span>
+            <span class="n">Union</span><span class="p">[</span><span class="n">BaseWDModelComponent</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="n">BaseWDModelComponent</span><span class="p">]]</span>
+        <span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">deeptext</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span>
+            <span class="n">Union</span><span class="p">[</span><span class="n">BaseWDModelComponent</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="n">BaseWDModelComponent</span><span class="p">]]</span>
+        <span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">deepimage</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span>
+            <span class="n">Union</span><span class="p">[</span><span class="n">BaseWDModelComponent</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="n">BaseWDModelComponent</span><span class="p">]]</span>
+        <span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">deephead</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">BaseWDModelComponent</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">head_hidden_dims</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">int</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">head_activation</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;relu&quot;</span><span class="p">,</span>
+        <span class="n">head_dropout</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.1</span><span class="p">,</span>
+        <span class="n">head_batchnorm</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
+        <span class="n">head_batchnorm_last</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
+        <span class="n">head_linear_first</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">,</span>
+        <span class="n">enforce_positive</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
+        <span class="n">enforce_positive_activation</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;softplus&quot;</span><span class="p">,</span>
+        <span class="n">pred_dim</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1</span><span class="p">,</span>
+    <span class="p">):</span>
+        <span class="nb">super</span><span class="p">(</span><span class="n">WideDeep</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
 
-</div>
+        <span class="bp">self</span><span class="o">.</span><span class="n">_check_inputs</span><span class="p">(</span>
+            <span class="n">wide</span><span class="p">,</span>
+            <span class="n">deeptabular</span><span class="p">,</span>
+            <span class="n">deeptext</span><span class="p">,</span>
+            <span class="n">deepimage</span><span class="p">,</span>
+            <span class="n">deephead</span><span class="p">,</span>
+            <span class="n">head_hidden_dims</span><span class="p">,</span>
+            <span class="n">pred_dim</span><span class="p">,</span>
+        <span class="p">)</span>
 
+        <span class="c1"># this attribute will be eventually over-written by the Trainer&#39;s</span>
+        <span class="c1"># device. Acts here as a &#39;placeholder&#39;.</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">wd_device</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span>
 
+        <span class="c1"># required as attribute just in case we pass a deephead</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">pred_dim</span> <span class="o">=</span> <span class="n">pred_dim</span>
 
-  </div>
+        <span class="bp">self</span><span class="o">.</span><span class="n">enforce_positive</span> <span class="o">=</span> <span class="n">enforce_positive</span>
 
-    </div>
+        <span class="c1"># better to set this attribute already here</span>
+        <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">deeptabular</span><span class="p">,</span> <span class="nb">list</span><span class="p">):</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">is_tabnet</span> <span class="o">=</span> <span class="kc">False</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">is_tabnet</span> <span class="o">=</span> <span class="n">deeptabular</span><span class="o">.</span><span class="vm">__class__</span><span class="o">.</span><span class="vm">__name__</span> <span class="o">==</span> <span class="s2">&quot;TabNet&quot;</span>
+
+        <span class="c1"># The main 5 components of the wide and deep assemble: wide,</span>
+        <span class="c1"># deeptabular, deeptext, deepimage and deephead</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">with_deephead</span> <span class="o">=</span> <span class="n">deephead</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">or</span> <span class="n">head_hidden_dims</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span>
+        <span class="k">if</span> <span class="n">deephead</span> <span class="ow">is</span> <span class="kc">None</span> <span class="ow">and</span> <span class="n">head_hidden_dims</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">deephead</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_build_deephead</span><span class="p">(</span>
+                <span class="n">deeptabular</span><span class="p">,</span>
+                <span class="n">deeptext</span><span class="p">,</span>
+                <span class="n">deepimage</span><span class="p">,</span>
+                <span class="n">head_hidden_dims</span><span class="p">,</span>
+                <span class="n">head_activation</span><span class="p">,</span>
+                <span class="n">head_dropout</span><span class="p">,</span>
+                <span class="n">head_batchnorm</span><span class="p">,</span>
+                <span class="n">head_batchnorm_last</span><span class="p">,</span>
+                <span class="n">head_linear_first</span><span class="p">,</span>
+            <span class="p">)</span>
+        <span class="k">elif</span> <span class="n">deephead</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">deephead</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Sequential</span><span class="p">(</span>
+                <span class="n">deephead</span><span class="p">,</span> <span class="n">nn</span><span class="o">.</span><span class="n">Linear</span><span class="p">(</span><span class="n">deephead</span><span class="o">.</span><span class="n">output_dim</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">pred_dim</span><span class="p">)</span>
+            <span class="p">)</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="c1"># for consistency with other components we default to None</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">deephead</span> <span class="o">=</span> <span class="kc">None</span>
 
-</div>
+        <span class="bp">self</span><span class="o">.</span><span class="n">wide</span> <span class="o">=</span> <span class="n">wide</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">deeptabular</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">deeptext</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">deepimage</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_set_model_components</span><span class="p">(</span>
+            <span class="n">deeptabular</span><span class="p">,</span> <span class="n">deeptext</span><span class="p">,</span> <span class="n">deepimage</span>
+        <span class="p">)</span>
 
-<div class="doc doc-object doc-class">
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">enforce_positive</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">enf_pos</span> <span class="o">=</span> <span class="n">get_activation_fn</span><span class="p">(</span><span class="n">enforce_positive_activation</span><span class="p">)</span>
 
+    <span class="k">def</span> <span class="nf">forward</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span>
+        <span class="n">X</span><span class="p">:</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Union</span><span class="p">[</span><span class="n">Tensor</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="n">Tensor</span><span class="p">]]],</span>
+        <span class="n">y</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Tensor</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+    <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Union</span><span class="p">[</span><span class="n">Tensor</span><span class="p">,</span> <span class="n">Tuple</span><span class="p">[</span><span class="n">Tensor</span><span class="p">,</span> <span class="n">Tensor</span><span class="p">]]:</span>
 
+        <span class="n">wide_out</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_forward_wide</span><span class="p">(</span><span class="n">X</span><span class="p">)</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">with_deephead</span><span class="p">:</span>
+            <span class="n">deep</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_forward_deephead</span><span class="p">(</span><span class="n">X</span><span class="p">,</span> <span class="n">wide_out</span><span class="p">)</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="n">deep</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_forward_deep</span><span class="p">(</span><span class="n">X</span><span class="p">,</span> <span class="n">wide_out</span><span class="p">)</span>
 
-<h2 id="pytorch_widedeep.models.wide_deep.WideDeep" class="doc doc-heading">
-            <span class="doc doc-object-name doc-class-name">WideDeep</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">enforce_positive</span><span class="p">:</span>
+            <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">enf_pos</span><span class="p">(</span><span class="n">deep</span><span class="p">)</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="k">return</span> <span class="n">deep</span>
+
+    <span class="k">def</span> <span class="nf">_build_deephead</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span>
+        <span class="n">deeptabular</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Union</span><span class="p">[</span><span class="n">BaseWDModelComponent</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="n">BaseWDModelComponent</span><span class="p">]]],</span>
+        <span class="n">deeptext</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Union</span><span class="p">[</span><span class="n">BaseWDModelComponent</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="n">BaseWDModelComponent</span><span class="p">]]],</span>
+        <span class="n">deepimage</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Union</span><span class="p">[</span><span class="n">BaseWDModelComponent</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="n">BaseWDModelComponent</span><span class="p">]]],</span>
+        <span class="n">head_hidden_dims</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="nb">int</span><span class="p">],</span>
+        <span class="n">head_activation</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
+        <span class="n">head_dropout</span><span class="p">:</span> <span class="nb">float</span><span class="p">,</span>
+        <span class="n">head_batchnorm</span><span class="p">:</span> <span class="nb">bool</span><span class="p">,</span>
+        <span class="n">head_batchnorm_last</span><span class="p">:</span> <span class="nb">bool</span><span class="p">,</span>
+        <span class="n">head_linear_first</span><span class="p">:</span> <span class="nb">bool</span><span class="p">,</span>
+    <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">nn</span><span class="o">.</span><span class="n">Sequential</span><span class="p">:</span>
+        <span class="n">deep_dim</span> <span class="o">=</span> <span class="mi">0</span>
+        <span class="k">if</span> <span class="n">deeptabular</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">deeptabular</span><span class="p">,</span> <span class="nb">list</span><span class="p">):</span>
+                <span class="k">for</span> <span class="n">dt</span> <span class="ow">in</span> <span class="n">deeptabular</span><span class="p">:</span>
+                    <span class="n">deep_dim</span> <span class="o">+=</span> <span class="n">dt</span><span class="o">.</span><span class="n">output_dim</span>
+            <span class="k">else</span><span class="p">:</span>
+                <span class="n">deep_dim</span> <span class="o">+=</span> <span class="n">deeptabular</span><span class="o">.</span><span class="n">output_dim</span>
+        <span class="k">if</span> <span class="n">deeptext</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">deeptext</span><span class="p">,</span> <span class="nb">list</span><span class="p">):</span>
+                <span class="k">for</span> <span class="n">dt</span> <span class="ow">in</span> <span class="n">deeptext</span><span class="p">:</span>
+                    <span class="n">deep_dim</span> <span class="o">+=</span> <span class="n">dt</span><span class="o">.</span><span class="n">output_dim</span>
+            <span class="k">else</span><span class="p">:</span>
+                <span class="n">deep_dim</span> <span class="o">+=</span> <span class="n">deeptext</span><span class="o">.</span><span class="n">output_dim</span>
+        <span class="k">if</span> <span class="n">deepimage</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">deepimage</span><span class="p">,</span> <span class="nb">list</span><span class="p">):</span>
+                <span class="k">for</span> <span class="n">di</span> <span class="ow">in</span> <span class="n">deepimage</span><span class="p">:</span>
+                    <span class="n">deep_dim</span> <span class="o">+=</span> <span class="n">di</span><span class="o">.</span><span class="n">output_dim</span>
+            <span class="k">else</span><span class="p">:</span>
+                <span class="n">deep_dim</span> <span class="o">+=</span> <span class="n">deepimage</span><span class="o">.</span><span class="n">output_dim</span>
+
+        <span class="n">head_hidden_dims</span> <span class="o">=</span> <span class="p">[</span><span class="n">deep_dim</span><span class="p">]</span> <span class="o">+</span> <span class="n">head_hidden_dims</span>
+        <span class="n">deephead</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Sequential</span><span class="p">(</span>
+            <span class="n">MLP</span><span class="p">(</span>
+                <span class="n">head_hidden_dims</span><span class="p">,</span>
+                <span class="n">head_activation</span><span class="p">,</span>
+                <span class="n">head_dropout</span><span class="p">,</span>
+                <span class="n">head_batchnorm</span><span class="p">,</span>
+                <span class="n">head_batchnorm_last</span><span class="p">,</span>
+                <span class="n">head_linear_first</span><span class="p">,</span>
+            <span class="p">),</span>
+            <span class="n">nn</span><span class="o">.</span><span class="n">Linear</span><span class="p">(</span><span class="n">head_hidden_dims</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">],</span> <span class="bp">self</span><span class="o">.</span><span class="n">pred_dim</span><span class="p">),</span>
+        <span class="p">)</span>
 
+        <span class="k">return</span> <span class="n">deephead</span>
+
+    <span class="k">def</span> <span class="nf">_set_model_components</span><span class="p">(</span>  <span class="c1"># noqa: C901</span>
+        <span class="bp">self</span><span class="p">,</span>
+        <span class="n">deeptabular</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Union</span><span class="p">[</span><span class="n">BaseWDModelComponent</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="n">BaseWDModelComponent</span><span class="p">]]],</span>
+        <span class="n">deeptext</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Union</span><span class="p">[</span><span class="n">BaseWDModelComponent</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="n">BaseWDModelComponent</span><span class="p">]]],</span>
+        <span class="n">deepimage</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Union</span><span class="p">[</span><span class="n">BaseWDModelComponent</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="n">BaseWDModelComponent</span><span class="p">]]],</span>
+    <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tuple</span><span class="p">[</span>
+        <span class="n">Optional</span><span class="p">[</span><span class="n">Union</span><span class="p">[</span><span class="n">nn</span><span class="o">.</span><span class="n">ModuleList</span><span class="p">,</span> <span class="n">WDModel</span><span class="p">]],</span>
+        <span class="n">Optional</span><span class="p">[</span><span class="n">Union</span><span class="p">[</span><span class="n">nn</span><span class="o">.</span><span class="n">ModuleList</span><span class="p">,</span> <span class="n">WDModel</span><span class="p">]],</span>
+        <span class="n">Optional</span><span class="p">[</span><span class="n">Union</span><span class="p">[</span><span class="n">nn</span><span class="o">.</span><span class="n">ModuleList</span><span class="p">,</span> <span class="n">WDModel</span><span class="p">]],</span>
+    <span class="p">]:</span>
+        <span class="k">if</span> <span class="n">deeptabular</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="n">deeptabular_</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_set_model_component</span><span class="p">(</span><span class="n">deeptabular</span><span class="p">,</span> <span class="n">is_deeptabular</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="n">deeptabular_</span> <span class="o">=</span> <span class="kc">None</span>
 
-<a href="#pytorch_widedeep.models.wide_deep.WideDeep" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="nf">WideDeep</span><span class="p">(</span>
-    <span class="n">wide</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">deeptabular</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">deeptext</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">deepimage</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">deephead</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">head_hidden_dims</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">head_activation</span><span class="o">=</span><span class="s2">&quot;relu&quot;</span><span class="p">,</span>
-    <span class="n">head_dropout</span><span class="o">=</span><span class="mf">0.1</span><span class="p">,</span>
-    <span class="n">head_batchnorm</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
-    <span class="n">head_batchnorm_last</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
-    <span class="n">head_linear_first</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
-    <span class="n">enforce_positive</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
-    <span class="n">enforce_positive_activation</span><span class="o">=</span><span class="s2">&quot;softplus&quot;</span><span class="p">,</span>
-    <span class="n">pred_dim</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span>
-<span class="p">)</span>
-</code></pre></div>
+        <span class="k">if</span> <span class="n">deeptext</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="n">deeptext_</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_set_model_component</span><span class="p">(</span><span class="n">deeptext</span><span class="p">)</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="n">deeptext_</span> <span class="o">=</span> <span class="kc">None</span>
 
-    <div class="doc doc-contents first">
-            <p class="doc doc-class-bases">
-              Bases: <code><span title="torch.nn.Module">Module</span></code></p>
+        <span class="k">if</span> <span class="n">deepimage</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="n">deepimage_</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_set_model_component</span><span class="p">(</span><span class="n">deepimage</span><span class="p">)</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="n">deepimage_</span> <span class="o">=</span> <span class="kc">None</span>
 
+        <span class="k">return</span> <span class="n">deeptabular_</span><span class="p">,</span> <span class="n">deeptext_</span><span class="p">,</span> <span class="n">deepimage_</span>
 
-        <p>Main collector class that combines all <code>wide</code>, <code>deeptabular</code>
-<code>deeptext</code> and <code>deepimage</code> models.</p>
-<p>Note that all models described so far in this library must be passed to
-the <code>WideDeep</code> class once constructed. This is because the models output
-the last layer before the prediction layer. Such prediction layer is
-added by the <code>WideDeep</code> class as it collects the components for every
-data mode.</p>
-<p>There are two options to combine these models that correspond to the
-two main architectures that <code>pytorch-widedeep</code> can build.</p>
-<ul>
-<li>
-<p>Directly connecting the output of the model components to an ouput neuron(s).</p>
-</li>
-<li>
-<p>Adding a <code>Fully-Connected Head</code> (FC-Head) on top of the deep models.
-  This FC-Head will combine the output form the <code>deeptabular</code>, <code>deeptext</code> and
-  <code>deepimage</code> and will be then connected to the output neuron(s).</p>
-</li>
-</ul>
+    <span class="k">def</span> <span class="nf">_forward_wide</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">X</span><span class="p">:</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Union</span><span class="p">[</span><span class="n">Tensor</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="n">Tensor</span><span class="p">]]])</span> <span class="o">-&gt;</span> <span class="n">Tensor</span><span class="p">:</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">wide</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="n">out</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">wide</span><span class="p">(</span><span class="n">X</span><span class="p">[</span><span class="s2">&quot;wide&quot;</span><span class="p">])</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="n">first_model_mode</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="n">X</span><span class="o">.</span><span class="n">keys</span><span class="p">())[</span><span class="mi">0</span><span class="p">]</span>
+            <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">X</span><span class="p">[</span><span class="n">first_model_mode</span><span class="p">],</span> <span class="nb">list</span><span class="p">):</span>
+                <span class="n">batch_size</span> <span class="o">=</span> <span class="n">X</span><span class="p">[</span><span class="n">first_model_mode</span><span class="p">][</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">size</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span>
+            <span class="k">else</span><span class="p">:</span>
+                <span class="n">batch_size</span> <span class="o">=</span> <span class="n">X</span><span class="p">[</span><span class="n">first_model_mode</span><span class="p">]</span><span class="o">.</span><span class="n">size</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span>  <span class="c1"># type: ignore[union-attr]</span>
+            <span class="n">out</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">zeros</span><span class="p">(</span><span class="n">batch_size</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">pred_dim</span><span class="p">)</span><span class="o">.</span><span class="n">to</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">wd_device</span><span class="p">)</span>
+
+        <span class="k">return</span> <span class="n">out</span>
+
+    <span class="k">def</span> <span class="nf">_forward_deep</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span> <span class="n">X</span><span class="p">:</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Union</span><span class="p">[</span><span class="n">Tensor</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="n">Tensor</span><span class="p">]]],</span> <span class="n">wide_out</span><span class="p">:</span> <span class="n">Tensor</span>
+    <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Union</span><span class="p">[</span><span class="n">Tensor</span><span class="p">,</span> <span class="n">Tuple</span><span class="p">[</span><span class="n">Tensor</span><span class="p">,</span> <span class="n">Tensor</span><span class="p">]]:</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">deeptabular</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">is_tabnet</span><span class="p">:</span>
+                <span class="n">tab_out</span><span class="p">,</span> <span class="n">M_loss</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">deeptabular</span><span class="p">(</span><span class="n">X</span><span class="p">[</span><span class="s2">&quot;deeptabular&quot;</span><span class="p">])</span>
+                <span class="n">wide_out</span><span class="o">.</span><span class="n">add_</span><span class="p">(</span><span class="n">tab_out</span><span class="p">)</span>
+            <span class="k">else</span><span class="p">:</span>
+                <span class="n">wide_out</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_forward_component</span><span class="p">(</span>
+                    <span class="n">X</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">deeptabular</span><span class="p">,</span> <span class="s2">&quot;deeptabular&quot;</span><span class="p">,</span> <span class="n">wide_out</span>
+                <span class="p">)</span>
+
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">deeptext</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="n">wide_out</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_forward_component</span><span class="p">(</span><span class="n">X</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">deeptext</span><span class="p">,</span> <span class="s2">&quot;deeptext&quot;</span><span class="p">,</span> <span class="n">wide_out</span><span class="p">)</span>
+
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">deepimage</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="n">wide_out</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_forward_component</span><span class="p">(</span><span class="n">X</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">deepimage</span><span class="p">,</span> <span class="s2">&quot;deepimage&quot;</span><span class="p">,</span> <span class="n">wide_out</span><span class="p">)</span>
+
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">is_tabnet</span><span class="p">:</span>
+            <span class="n">res</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="n">Tensor</span><span class="p">,</span> <span class="n">Tuple</span><span class="p">[</span><span class="n">Tensor</span><span class="p">,</span> <span class="n">Tensor</span><span class="p">]]</span> <span class="o">=</span> <span class="p">(</span><span class="n">wide_out</span><span class="p">,</span> <span class="n">M_loss</span><span class="p">)</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="n">res</span> <span class="o">=</span> <span class="n">wide_out</span>
+
+        <span class="k">return</span> <span class="n">res</span>
+
+    <span class="k">def</span> <span class="nf">_forward_deephead</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span> <span class="n">X</span><span class="p">:</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Union</span><span class="p">[</span><span class="n">Tensor</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="n">Tensor</span><span class="p">]]],</span> <span class="n">wide_out</span><span class="p">:</span> <span class="n">Tensor</span>
+    <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Union</span><span class="p">[</span><span class="n">Tensor</span><span class="p">,</span> <span class="n">Tuple</span><span class="p">[</span><span class="n">Tensor</span><span class="p">,</span> <span class="n">Tensor</span><span class="p">]]:</span>
+        <span class="n">deepside</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">FloatTensor</span><span class="p">()</span><span class="o">.</span><span class="n">to</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">wd_device</span><span class="p">)</span>
+
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">deeptabular</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">is_tabnet</span><span class="p">:</span>
+                <span class="n">deepside</span><span class="p">,</span> <span class="n">M_loss</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">deeptabular</span><span class="p">(</span><span class="n">X</span><span class="p">[</span><span class="s2">&quot;deeptabular&quot;</span><span class="p">])</span>
+            <span class="k">else</span><span class="p">:</span>
+                <span class="n">deepside</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_forward_component_with_head</span><span class="p">(</span>
+                    <span class="n">X</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">deeptabular</span><span class="p">,</span> <span class="s2">&quot;deeptabular&quot;</span><span class="p">,</span> <span class="n">deepside</span>
+                <span class="p">)</span>
+
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">deeptext</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="n">deepside</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_forward_component_with_head</span><span class="p">(</span>
+                <span class="n">X</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">deeptext</span><span class="p">,</span> <span class="s2">&quot;deeptext&quot;</span><span class="p">,</span> <span class="n">deepside</span>
+            <span class="p">)</span>
 
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">deepimage</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="n">deepside</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_forward_component_with_head</span><span class="p">(</span>
+                <span class="n">X</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">deepimage</span><span class="p">,</span> <span class="s2">&quot;deepimage&quot;</span><span class="p">,</span> <span class="n">deepside</span>
+            <span class="p">)</span>
 
-<p><span class="doc-section-title">Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>wide</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="torch.nn.Module">Module</span>]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p><code>Wide</code> model. This is a linear model where the non-linearities are
-captured via crossed-columns.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>deeptabular</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="pytorch_widedeep.models._base_wd_model_component.BaseWDModelComponent">BaseWDModelComponent</span>, <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.models._base_wd_model_component.BaseWDModelComponent">BaseWDModelComponent</span>]]]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Currently this library implements a number of possible architectures
-for the <code>deeptabular</code> component. See the documenation of the
-package. Note that <code>deeptabular</code> can be a list of models. This is
-useful when using multiple tabular inputs (e.g. for example in the
-context of a two-tower model for recommendation systems)</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>deeptext</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="pytorch_widedeep.models._base_wd_model_component.BaseWDModelComponent">BaseWDModelComponent</span>, <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.models._base_wd_model_component.BaseWDModelComponent">BaseWDModelComponent</span>]]]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Currently this library implements a number of possible architectures
-for the <code>deeptext</code> component. See the documenation of the
-package. Note that <code>deeptext</code> can be a list of models. This is useful
-when using multiple text inputs.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>deepimage</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="pytorch_widedeep.models._base_wd_model_component.BaseWDModelComponent">BaseWDModelComponent</span>, <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.models._base_wd_model_component.BaseWDModelComponent">BaseWDModelComponent</span>]]]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Currently this library uses <code>torchvision</code> and implements a number of
-possible architectures for the <code>deepimage</code> component. See the
-documenation of the package. Note that <code>deepimage</code> can be a list of
-models. This is useful when using multiple image inputs.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>deephead</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.models._base_wd_model_component.BaseWDModelComponent">BaseWDModelComponent</span>]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Alternatively, the user can pass a custom model that will receive the
-output of the deep component. If <code>deephead</code> is not None all the
-previous fc-head parameters will be ignored</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>head_hidden_dims</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[int]]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>List with the sizes of the dense layers in the head e.g: [128, 64]</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>head_activation</code></b>
-              (<code>str</code>, default:
-                  <code>&#39;relu&#39;</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Activation function for the dense layers in the head. Currently
-<code>'tanh'</code>, <code>'relu'</code>, <code>'leaky_relu'</code> and <code>'gelu'</code> are supported</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>head_dropout</code></b>
-              (<code>float</code>, default:
-                  <code>0.1</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Dropout of the dense layers in the head</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>head_batchnorm</code></b>
-              (<code>bool</code>, default:
-                  <code>False</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Boolean indicating whether or not to include batch normalization in
-the dense layers that form the <code>'rnn_mlp'</code></p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>head_batchnorm_last</code></b>
-              (<code>bool</code>, default:
-                  <code>False</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Boolean indicating whether or not to apply batch normalization to the
-last of the dense layers in the head</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>head_linear_first</code></b>
-              (<code>bool</code>, default:
-                  <code>True</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Boolean indicating whether the order of the operations in the dense
-layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt;
-LIN -&gt; ACT]</code></p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>enforce_positive</code></b>
-              (<code>bool</code>, default:
-                  <code>False</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Boolean indicating if the output from the final layer must be
-positive. This is important if you are using loss functions with
-non-negative input restrictions, e.g. RMSLE, or if you know your
-predictions are bounded in between 0 and inf</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>enforce_positive_activation</code></b>
-              (<code>str</code>, default:
-                  <code>&#39;softplus&#39;</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Activation function to enforce that the final layer has a positive
-output. <code>'softplus'</code> or <code>'relu'</code> are supported.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>pred_dim</code></b>
-              (<code>int</code>, default:
-                  <code>1</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Size of the final wide and deep output layer containing the
-predictions. <code>1</code> for regression and binary classification or number
-of classes for multiclass classification.</p>
-          </div>
-        </li>
-    </ul>
+        <span class="c1"># assertion to avoid type issues</span>
+        <span class="k">assert</span> <span class="bp">self</span><span class="o">.</span><span class="n">deephead</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span>
+        <span class="n">deepside_out</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">deephead</span><span class="p">(</span><span class="n">deepside</span><span class="p">)</span>
 
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">is_tabnet</span><span class="p">:</span>
+            <span class="n">res</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="n">Tensor</span><span class="p">,</span> <span class="n">Tuple</span><span class="p">[</span><span class="n">Tensor</span><span class="p">,</span> <span class="n">Tensor</span><span class="p">]]</span> <span class="o">=</span> <span class="p">(</span>
+                <span class="n">wide_out</span><span class="o">.</span><span class="n">add_</span><span class="p">(</span><span class="n">deepside_out</span><span class="p">),</span>
+                <span class="n">M_loss</span><span class="p">,</span>
+            <span class="p">)</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="n">res</span> <span class="o">=</span> <span class="n">wide_out</span><span class="o">.</span><span class="n">add_</span><span class="p">(</span><span class="n">deepside_out</span><span class="p">)</span>
+
+        <span class="k">return</span> <span class="n">res</span>
+
+    <span class="k">def</span> <span class="nf">_forward_component</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span>
+        <span class="n">X</span><span class="p">:</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Union</span><span class="p">[</span><span class="n">Tensor</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="n">Tensor</span><span class="p">]]],</span>
+        <span class="n">component</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="n">nn</span><span class="o">.</span><span class="n">ModuleList</span><span class="p">,</span> <span class="n">WDModel</span><span class="p">],</span>
+        <span class="n">component_type</span><span class="p">:</span> <span class="n">Literal</span><span class="p">[</span><span class="s2">&quot;deeptabular&quot;</span><span class="p">,</span> <span class="s2">&quot;deeptext&quot;</span><span class="p">,</span> <span class="s2">&quot;deepimage&quot;</span><span class="p">],</span>
+        <span class="n">wide_out</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">,</span>
+    <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tensor</span><span class="p">:</span>
+        <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">component</span><span class="p">,</span> <span class="n">nn</span><span class="o">.</span><span class="n">ModuleList</span><span class="p">):</span>
+            <span class="n">component_out</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">add</span><span class="p">(</span>  <span class="c1"># type: ignore[call-overload]</span>
+                <span class="o">*</span><span class="p">[</span><span class="n">cp</span><span class="p">(</span><span class="n">X</span><span class="p">[</span><span class="n">component_type</span><span class="p">][</span><span class="n">i</span><span class="p">])</span> <span class="k">for</span> <span class="n">i</span><span class="p">,</span> <span class="n">cp</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="n">component</span><span class="p">)]</span>
+            <span class="p">)</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="n">component_out</span> <span class="o">=</span> <span class="n">component</span><span class="p">(</span><span class="n">X</span><span class="p">[</span><span class="n">component_type</span><span class="p">])</span>
+
+        <span class="k">return</span> <span class="n">wide_out</span><span class="o">.</span><span class="n">add_</span><span class="p">(</span><span class="n">component_out</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">_forward_component_with_head</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span>
+        <span class="n">X</span><span class="p">:</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Union</span><span class="p">[</span><span class="n">Tensor</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="n">Tensor</span><span class="p">]]],</span>
+        <span class="n">component</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="n">nn</span><span class="o">.</span><span class="n">ModuleList</span><span class="p">,</span> <span class="n">WDModel</span><span class="p">],</span>
+        <span class="n">component_type</span><span class="p">:</span> <span class="n">Literal</span><span class="p">[</span><span class="s2">&quot;deeptabular&quot;</span><span class="p">,</span> <span class="s2">&quot;deeptext&quot;</span><span class="p">,</span> <span class="s2">&quot;deepimage&quot;</span><span class="p">],</span>
+        <span class="n">deepside</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">,</span>
+    <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tensor</span><span class="p">:</span>
+        <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">component</span><span class="p">,</span> <span class="n">nn</span><span class="o">.</span><span class="n">ModuleList</span><span class="p">):</span>
+            <span class="n">component_out</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">cat</span><span class="p">(</span>  <span class="c1"># type: ignore[call-overload]</span>
+                <span class="p">[</span><span class="n">cp</span><span class="p">(</span><span class="n">X</span><span class="p">[</span><span class="n">component_type</span><span class="p">][</span><span class="n">i</span><span class="p">])</span> <span class="k">for</span> <span class="n">i</span><span class="p">,</span> <span class="n">cp</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="n">component</span><span class="p">)],</span> <span class="n">axis</span><span class="o">=</span><span class="mi">1</span>
+            <span class="p">)</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="n">component_out</span> <span class="o">=</span> <span class="n">component</span><span class="p">(</span><span class="n">X</span><span class="p">[</span><span class="n">component_type</span><span class="p">])</span>
+
+        <span class="k">return</span> <span class="n">torch</span><span class="o">.</span><span class="n">cat</span><span class="p">([</span><span class="n">deepside</span><span class="p">,</span> <span class="n">component_out</span><span class="p">],</span> <span class="n">axis</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span>  <span class="c1"># type: ignore[call-overload]</span>
+
+    <span class="k">def</span> <span class="nf">_set_model_component</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span>
+        <span class="n">component</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="n">BaseWDModelComponent</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="n">BaseWDModelComponent</span><span class="p">]],</span>
+        <span class="n">is_deeptabular</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
+    <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Union</span><span class="p">[</span><span class="n">nn</span><span class="o">.</span><span class="n">ModuleList</span><span class="p">,</span> <span class="n">WDModel</span><span class="p">]:</span>
+        <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">component</span><span class="p">,</span> <span class="nb">list</span><span class="p">):</span>
+            <span class="n">component_</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Union</span><span class="p">[</span><span class="n">nn</span><span class="o">.</span><span class="n">ModuleList</span><span class="p">,</span> <span class="n">WDModel</span><span class="p">]]</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">ModuleList</span><span class="p">()</span>
+            <span class="k">for</span> <span class="n">cp</span> <span class="ow">in</span> <span class="n">component</span><span class="p">:</span>
+                <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">with_deephead</span> <span class="ow">or</span> <span class="n">cp</span><span class="o">.</span><span class="n">output_dim</span> <span class="o">==</span> <span class="mi">1</span><span class="p">:</span>
+                    <span class="n">component_</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">cp</span><span class="p">)</span>
+                <span class="k">else</span><span class="p">:</span>
+                    <span class="n">component_</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
+                        <span class="n">nn</span><span class="o">.</span><span class="n">Sequential</span><span class="p">(</span><span class="n">cp</span><span class="p">,</span> <span class="n">nn</span><span class="o">.</span><span class="n">Linear</span><span class="p">(</span><span class="n">cp</span><span class="o">.</span><span class="n">output_dim</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">pred_dim</span><span class="p">))</span>
+                    <span class="p">)</span>
+        <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">with_deephead</span> <span class="ow">or</span> <span class="n">component</span><span class="o">.</span><span class="n">output_dim</span> <span class="o">==</span> <span class="mi">1</span><span class="p">:</span>
+            <span class="n">component_</span> <span class="o">=</span> <span class="n">component</span>
+        <span class="k">elif</span> <span class="n">is_deeptabular</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">is_tabnet</span><span class="p">:</span>
+            <span class="n">component_</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Sequential</span><span class="p">(</span>
+                <span class="n">component</span><span class="p">,</span> <span class="n">TabNetPredLayer</span><span class="p">(</span><span class="n">component</span><span class="o">.</span><span class="n">output_dim</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">pred_dim</span><span class="p">)</span>
+            <span class="p">)</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="n">component_</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Sequential</span><span class="p">(</span>
+                <span class="n">component</span><span class="p">,</span> <span class="n">nn</span><span class="o">.</span><span class="n">Linear</span><span class="p">(</span><span class="n">component</span><span class="o">.</span><span class="n">output_dim</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">pred_dim</span><span class="p">)</span>
+            <span class="p">)</span>
 
-<p><span class="doc-section-title">Examples:</span></p>
-    <div class="highlight"><pre><span></span><code><span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">pytorch_widedeep.models</span> <span class="kn">import</span> <span class="n">TabResnet</span><span class="p">,</span> <span class="n">Vision</span><span class="p">,</span> <span class="n">BasicRNN</span><span class="p">,</span> <span class="n">Wide</span><span class="p">,</span> <span class="n">WideDeep</span>
-<span class="gp">&gt;&gt;&gt; </span><span class="n">embed_input</span> <span class="o">=</span> <span class="p">[(</span><span class="n">u</span><span class="p">,</span> <span class="n">i</span><span class="p">,</span> <span class="n">j</span><span class="p">)</span> <span class="k">for</span> <span class="n">u</span><span class="p">,</span> <span class="n">i</span><span class="p">,</span> <span class="n">j</span> <span class="ow">in</span> <span class="nb">zip</span><span class="p">([</span><span class="s2">&quot;a&quot;</span><span class="p">,</span> <span class="s2">&quot;b&quot;</span><span class="p">,</span> <span class="s2">&quot;c&quot;</span><span class="p">][:</span><span class="mi">4</span><span class="p">],</span> <span class="p">[</span><span class="mi">4</span><span class="p">]</span> <span class="o">*</span> <span class="mi">3</span><span class="p">,</span> <span class="p">[</span><span class="mi">8</span><span class="p">]</span> <span class="o">*</span> <span class="mi">3</span><span class="p">)]</span>
-<span class="gp">&gt;&gt;&gt; </span><span class="n">column_idx</span> <span class="o">=</span> <span class="p">{</span><span class="n">k</span><span class="p">:</span> <span class="n">v</span> <span class="k">for</span> <span class="n">v</span><span class="p">,</span> <span class="n">k</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">([</span><span class="s2">&quot;a&quot;</span><span class="p">,</span> <span class="s2">&quot;b&quot;</span><span class="p">,</span> <span class="s2">&quot;c&quot;</span><span class="p">])}</span>
-<span class="gp">&gt;&gt;&gt; </span><span class="n">wide</span> <span class="o">=</span> <span class="n">Wide</span><span class="p">(</span><span class="mi">10</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span>
-<span class="gp">&gt;&gt;&gt; </span><span class="n">deeptabular</span> <span class="o">=</span> <span class="n">TabResnet</span><span class="p">(</span><span class="n">blocks_dims</span><span class="o">=</span><span class="p">[</span><span class="mi">8</span><span class="p">,</span> <span class="mi">4</span><span class="p">],</span> <span class="n">column_idx</span><span class="o">=</span><span class="n">column_idx</span><span class="p">,</span> <span class="n">cat_embed_input</span><span class="o">=</span><span class="n">embed_input</span><span class="p">)</span>
-<span class="gp">&gt;&gt;&gt; </span><span class="n">deeptext</span> <span class="o">=</span> <span class="n">BasicRNN</span><span class="p">(</span><span class="n">vocab_size</span><span class="o">=</span><span class="mi">10</span><span class="p">,</span> <span class="n">embed_dim</span><span class="o">=</span><span class="mi">4</span><span class="p">,</span> <span class="n">padding_idx</span><span class="o">=</span><span class="mi">0</span><span class="p">)</span>
-<span class="gp">&gt;&gt;&gt; </span><span class="n">deepimage</span> <span class="o">=</span> <span class="n">Vision</span><span class="p">()</span>
-<span class="gp">&gt;&gt;&gt; </span><span class="n">model</span> <span class="o">=</span> <span class="n">WideDeep</span><span class="p">(</span><span class="n">wide</span><span class="o">=</span><span class="n">wide</span><span class="p">,</span> <span class="n">deeptabular</span><span class="o">=</span><span class="n">deeptabular</span><span class="p">,</span> <span class="n">deeptext</span><span class="o">=</span><span class="n">deeptext</span><span class="p">,</span> <span class="n">deepimage</span><span class="o">=</span><span class="n">deepimage</span><span class="p">)</span>
-</code></pre></div>
-    <p><img alt="ℹ️" class="emojione" src="https://cdnjs.cloudflare.com/ajax/libs/emojione/2.2.7/assets/png/2139.png" title=":information_source:" /> <strong>NOTE</strong>: It is possible to use custom components to
- build Wide &amp; Deep models. Simply, build them and pass them as the
- corresponding parameters. Note that the custom models MUST return a last
- layer of activations(i.e. not the final prediction) so that  these
- activations are collected by <code>WideDeep</code> and combined accordingly. In
- addition, the models MUST also contain an attribute <code>output_dim</code> with
- the size of these last layers of activations. See for example
- <code>pytorch_widedeep.models.tab_mlp.TabMlp</code></p>
+        <span class="k">return</span> <span class="n">component_</span>
 
-                  <details class="quote">
-                    <summary>Source code in <code>pytorch_widedeep/models/wide_deep.py</code></summary>
-                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">117</span>
-<span class="normal">118</span>
-<span class="normal">119</span>
-<span class="normal">120</span>
-<span class="normal">121</span>
-<span class="normal">122</span>
-<span class="normal">123</span>
-<span class="normal">124</span>
-<span class="normal">125</span>
-<span class="normal">126</span>
-<span class="normal">127</span>
-<span class="normal">128</span>
-<span class="normal">129</span>
-<span class="normal">130</span>
-<span class="normal">131</span>
-<span class="normal">132</span>
-<span class="normal">133</span>
-<span class="normal">134</span>
-<span class="normal">135</span>
-<span class="normal">136</span>
-<span class="normal">137</span>
-<span class="normal">138</span>
-<span class="normal">139</span>
-<span class="normal">140</span>
-<span class="normal">141</span>
-<span class="normal">142</span>
-<span class="normal">143</span>
-<span class="normal">144</span>
-<span class="normal">145</span>
-<span class="normal">146</span>
-<span class="normal">147</span>
-<span class="normal">148</span>
-<span class="normal">149</span>
-<span class="normal">150</span>
-<span class="normal">151</span>
-<span class="normal">152</span>
-<span class="normal">153</span>
-<span class="normal">154</span>
-<span class="normal">155</span>
-<span class="normal">156</span>
-<span class="normal">157</span>
-<span class="normal">158</span>
-<span class="normal">159</span>
-<span class="normal">160</span>
-<span class="normal">161</span>
-<span class="normal">162</span>
-<span class="normal">163</span>
-<span class="normal">164</span>
-<span class="normal">165</span>
-<span class="normal">166</span>
-<span class="normal">167</span>
-<span class="normal">168</span>
-<span class="normal">169</span>
-<span class="normal">170</span>
-<span class="normal">171</span>
-<span class="normal">172</span>
-<span class="normal">173</span>
-<span class="normal">174</span>
-<span class="normal">175</span>
-<span class="normal">176</span>
-<span class="normal">177</span>
-<span class="normal">178</span>
-<span class="normal">179</span>
-<span class="normal">180</span>
-<span class="normal">181</span>
-<span class="normal">182</span>
-<span class="normal">183</span>
-<span class="normal">184</span>
-<span class="normal">185</span>
-<span class="normal">186</span>
-<span class="normal">187</span>
-<span class="normal">188</span>
-<span class="normal">189</span>
-<span class="normal">190</span>
-<span class="normal">191</span>
-<span class="normal">192</span>
-<span class="normal">193</span>
-<span class="normal">194</span>
-<span class="normal">195</span>
-<span class="normal">196</span>
-<span class="normal">197</span>
-<span class="normal">198</span>
-<span class="normal">199</span>
-<span class="normal">200</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="nd">@alias</span><span class="p">(</span>  <span class="c1"># noqa: C901</span>
-    <span class="s2">&quot;pred_dim&quot;</span><span class="p">,</span>
-    <span class="p">[</span><span class="s2">&quot;num_class&quot;</span><span class="p">,</span> <span class="s2">&quot;pred_size&quot;</span><span class="p">],</span>
-<span class="p">)</span>
-<span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
-    <span class="bp">self</span><span class="p">,</span>
-    <span class="n">wide</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">nn</span><span class="o">.</span><span class="n">Module</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">deeptabular</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span>
-        <span class="n">Union</span><span class="p">[</span><span class="n">BaseWDModelComponent</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="n">BaseWDModelComponent</span><span class="p">]]</span>
-    <span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">deeptext</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span>
-        <span class="n">Union</span><span class="p">[</span><span class="n">BaseWDModelComponent</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="n">BaseWDModelComponent</span><span class="p">]]</span>
-    <span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">deepimage</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span>
-        <span class="n">Union</span><span class="p">[</span><span class="n">BaseWDModelComponent</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="n">BaseWDModelComponent</span><span class="p">]]</span>
-    <span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">deephead</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">BaseWDModelComponent</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">head_hidden_dims</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">int</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">head_activation</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;relu&quot;</span><span class="p">,</span>
-    <span class="n">head_dropout</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.1</span><span class="p">,</span>
-    <span class="n">head_batchnorm</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
-    <span class="n">head_batchnorm_last</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
-    <span class="n">head_linear_first</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">,</span>
-    <span class="n">enforce_positive</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
-    <span class="n">enforce_positive_activation</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;softplus&quot;</span><span class="p">,</span>
-    <span class="n">pred_dim</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1</span><span class="p">,</span>
-<span class="p">):</span>
-    <span class="nb">super</span><span class="p">(</span><span class="n">WideDeep</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
-
-    <span class="bp">self</span><span class="o">.</span><span class="n">_check_inputs</span><span class="p">(</span>
+    <span class="nd">@staticmethod</span>  <span class="c1"># noqa: C901</span>
+    <span class="k">def</span> <span class="nf">_check_inputs</span><span class="p">(</span>  <span class="c1"># noqa: C901</span>
         <span class="n">wide</span><span class="p">,</span>
         <span class="n">deeptabular</span><span class="p">,</span>
         <span class="n">deeptext</span><span class="p">,</span>
@@ -13356,55 +22460,135 @@ <h2 id="pytorch_widedeep.models.wide_deep.WideDeep" class="doc doc-heading">
         <span class="n">deephead</span><span class="p">,</span>
         <span class="n">head_hidden_dims</span><span class="p">,</span>
         <span class="n">pred_dim</span><span class="p">,</span>
-    <span class="p">)</span>
-
-    <span class="c1"># this attribute will be eventually over-written by the Trainer&#39;s</span>
-    <span class="c1"># device. Acts here as a &#39;placeholder&#39;.</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">wd_device</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span>
-
-    <span class="c1"># required as attribute just in case we pass a deephead</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">pred_dim</span> <span class="o">=</span> <span class="n">pred_dim</span>
-
-    <span class="bp">self</span><span class="o">.</span><span class="n">enforce_positive</span> <span class="o">=</span> <span class="n">enforce_positive</span>
-
-    <span class="c1"># better to set this attribute already here</span>
-    <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">deeptabular</span><span class="p">,</span> <span class="nb">list</span><span class="p">):</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">is_tabnet</span> <span class="o">=</span> <span class="kc">False</span>
-    <span class="k">else</span><span class="p">:</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">is_tabnet</span> <span class="o">=</span> <span class="n">deeptabular</span><span class="o">.</span><span class="vm">__class__</span><span class="o">.</span><span class="vm">__name__</span> <span class="o">==</span> <span class="s2">&quot;TabNet&quot;</span>
-
-    <span class="c1"># The main 5 components of the wide and deep assemble: wide,</span>
-    <span class="c1"># deeptabular, deeptext, deepimage and deephead</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">with_deephead</span> <span class="o">=</span> <span class="n">deephead</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">or</span> <span class="n">head_hidden_dims</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span>
-    <span class="k">if</span> <span class="n">deephead</span> <span class="ow">is</span> <span class="kc">None</span> <span class="ow">and</span> <span class="n">head_hidden_dims</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">deephead</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_build_deephead</span><span class="p">(</span>
-            <span class="n">deeptabular</span><span class="p">,</span>
-            <span class="n">deeptext</span><span class="p">,</span>
-            <span class="n">deepimage</span><span class="p">,</span>
-            <span class="n">head_hidden_dims</span><span class="p">,</span>
-            <span class="n">head_activation</span><span class="p">,</span>
-            <span class="n">head_dropout</span><span class="p">,</span>
-            <span class="n">head_batchnorm</span><span class="p">,</span>
-            <span class="n">head_batchnorm_last</span><span class="p">,</span>
-            <span class="n">head_linear_first</span><span class="p">,</span>
-        <span class="p">)</span>
-    <span class="k">elif</span> <span class="n">deephead</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">deephead</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Sequential</span><span class="p">(</span>
-            <span class="n">deephead</span><span class="p">,</span> <span class="n">nn</span><span class="o">.</span><span class="n">Linear</span><span class="p">(</span><span class="n">deephead</span><span class="o">.</span><span class="n">output_dim</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">pred_dim</span><span class="p">)</span>
-        <span class="p">)</span>
-    <span class="k">else</span><span class="p">:</span>
-        <span class="c1"># for consistency with other components we default to None</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">deephead</span> <span class="o">=</span> <span class="kc">None</span>
+    <span class="p">):</span>
+        <span class="k">if</span> <span class="n">wide</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="k">assert</span> <span class="n">wide</span><span class="o">.</span><span class="n">wide_linear</span><span class="o">.</span><span class="n">weight</span><span class="o">.</span><span class="n">size</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span> <span class="o">==</span> <span class="n">pred_dim</span><span class="p">,</span> <span class="p">(</span>
+                <span class="s2">&quot;the &#39;pred_dim&#39; of the wide component (</span><span class="si">{}</span><span class="s2">) must be equal to the &#39;pred_dim&#39; &quot;</span>
+                <span class="s2">&quot;of the deep component and the overall model itself (</span><span class="si">{}</span><span class="s2">)&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
+                    <span class="n">wide</span><span class="o">.</span><span class="n">wide_linear</span><span class="o">.</span><span class="n">weight</span><span class="o">.</span><span class="n">size</span><span class="p">(</span><span class="mi">1</span><span class="p">),</span> <span class="n">pred_dim</span>
+                <span class="p">)</span>
+            <span class="p">)</span>
 
-    <span class="bp">self</span><span class="o">.</span><span class="n">wide</span> <span class="o">=</span> <span class="n">wide</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">deeptabular</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">deeptext</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">deepimage</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_set_model_components</span><span class="p">(</span>
-        <span class="n">deeptabular</span><span class="p">,</span> <span class="n">deeptext</span><span class="p">,</span> <span class="n">deepimage</span>
-    <span class="p">)</span>
+        <span class="k">if</span> <span class="n">deeptabular</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="n">err_msg</span> <span class="o">=</span> <span class="p">(</span>
+                <span class="s2">&quot;deeptabular model must have an &#39;output_dim&#39; attribute or property.&quot;</span>
+            <span class="p">)</span>
+            <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">deeptabular</span><span class="p">,</span> <span class="nb">list</span><span class="p">):</span>
+                <span class="n">all_have_output_dim</span> <span class="o">=</span> <span class="nb">all</span><span class="p">(</span>
+                    <span class="nb">hasattr</span><span class="p">(</span><span class="n">dt</span><span class="p">,</span> <span class="s2">&quot;output_dim&quot;</span><span class="p">)</span> <span class="k">for</span> <span class="n">dt</span> <span class="ow">in</span> <span class="n">deeptabular</span>
+                <span class="p">)</span>
+                <span class="k">if</span> <span class="ow">not</span> <span class="n">all_have_output_dim</span><span class="p">:</span>
+                    <span class="k">raise</span> <span class="ne">AttributeError</span><span class="p">(</span><span class="n">err_msg</span><span class="p">)</span>
+            <span class="k">else</span><span class="p">:</span>
+                <span class="k">if</span> <span class="ow">not</span> <span class="nb">hasattr</span><span class="p">(</span><span class="n">deeptabular</span><span class="p">,</span> <span class="s2">&quot;output_dim&quot;</span><span class="p">):</span>
+                    <span class="k">raise</span> <span class="ne">AttributeError</span><span class="p">(</span><span class="n">err_msg</span><span class="p">)</span>
+                <span class="c1"># the following assertion is thought for those cases where we</span>
+                <span class="c1"># use fusion with &#39;dot product&#39; so that the output_dim will</span>
+                <span class="c1"># be 1 and the pred_dim is not 1</span>
+                <span class="k">if</span> <span class="n">deeptabular</span><span class="o">.</span><span class="n">output_dim</span> <span class="o">==</span> <span class="mi">1</span><span class="p">:</span>
+                    <span class="k">assert</span> <span class="n">pred_dim</span> <span class="o">==</span> <span class="mi">1</span><span class="p">,</span> <span class="s2">&quot;If &#39;output_dim&#39; is 1, &#39;pred_dim&#39; must be 1&quot;</span>
+
+        <span class="k">if</span> <span class="n">deeptabular</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="n">is_tabnet</span> <span class="o">=</span> <span class="kc">False</span>
+            <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">deeptabular</span><span class="p">,</span> <span class="nb">list</span><span class="p">):</span>
+                <span class="n">is_any_tabnet</span> <span class="o">=</span> <span class="nb">any</span><span class="p">(</span>
+                    <span class="n">dt</span><span class="o">.</span><span class="vm">__class__</span><span class="o">.</span><span class="vm">__name__</span> <span class="o">==</span> <span class="s2">&quot;TabNet&quot;</span> <span class="k">for</span> <span class="n">dt</span> <span class="ow">in</span> <span class="n">deeptabular</span>
+                <span class="p">)</span>
+                <span class="k">if</span> <span class="n">is_any_tabnet</span><span class="p">:</span>
+                    <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
+                        <span class="s2">&quot;Currently TabNet is not supported as a component of a multiple &quot;</span>
+                        <span class="s2">&quot;tabular component model.&quot;</span>
+                    <span class="p">)</span>
+            <span class="k">else</span><span class="p">:</span>
+                <span class="n">is_tabnet</span> <span class="o">=</span> <span class="n">deeptabular</span><span class="o">.</span><span class="vm">__class__</span><span class="o">.</span><span class="vm">__name__</span> <span class="o">==</span> <span class="s2">&quot;TabNet&quot;</span>
+            <span class="n">has_wide_text_or_image</span> <span class="o">=</span> <span class="p">(</span>
+                <span class="n">wide</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">or</span> <span class="n">deeptext</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">or</span> <span class="n">deepimage</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span>
+            <span class="p">)</span>
+            <span class="k">if</span> <span class="n">is_tabnet</span> <span class="ow">and</span> <span class="n">has_wide_text_or_image</span><span class="p">:</span>
+                <span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span>
+                    <span class="s2">&quot;&#39;WideDeep&#39; is a model comprised by multiple components and the &#39;deeptabular&#39;&quot;</span>
+                    <span class="s2">&quot; component is &#39;TabNet&#39;. We recommend using &#39;TabNet&#39; in isolation.&quot;</span>
+                    <span class="s2">&quot; The reasons are: i)&#39;TabNet&#39; uses sparse regularization which partially losses&quot;</span>
+                    <span class="s2">&quot; its purpose when used in combination with other components.&quot;</span>
+                    <span class="s2">&quot; If you still want to use a multiple component model with &#39;TabNet&#39;,&quot;</span>
+                    <span class="s2">&quot; consider setting &#39;lambda_sparse&#39; to 0 during training. ii) The feature&quot;</span>
+                    <span class="s2">&quot; importances will be computed only for TabNet but the model will comprise multiple&quot;</span>
+                    <span class="s2">&quot; components. Therefore, such importances will partially lose their &#39;meaning&#39;.&quot;</span><span class="p">,</span>
+                    <span class="ne">UserWarning</span><span class="p">,</span>
+                <span class="p">)</span>
+
+        <span class="k">if</span> <span class="n">deeptext</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="n">err_msg</span> <span class="o">=</span> <span class="s2">&quot;deeptext model must have an &#39;output_dim&#39; attribute or property.&quot;</span>
+            <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">deeptext</span><span class="p">,</span> <span class="nb">list</span><span class="p">):</span>
+                <span class="n">all_have_output_dim</span> <span class="o">=</span> <span class="nb">all</span><span class="p">(</span><span class="nb">hasattr</span><span class="p">(</span><span class="n">dt</span><span class="p">,</span> <span class="s2">&quot;output_dim&quot;</span><span class="p">)</span> <span class="k">for</span> <span class="n">dt</span> <span class="ow">in</span> <span class="n">deeptext</span><span class="p">)</span>
+                <span class="k">if</span> <span class="ow">not</span> <span class="n">all_have_output_dim</span><span class="p">:</span>
+                    <span class="k">raise</span> <span class="ne">AttributeError</span><span class="p">(</span><span class="n">err_msg</span><span class="p">)</span>
+            <span class="k">else</span><span class="p">:</span>
+                <span class="k">if</span> <span class="ow">not</span> <span class="nb">hasattr</span><span class="p">(</span><span class="n">deeptext</span><span class="p">,</span> <span class="s2">&quot;output_dim&quot;</span><span class="p">):</span>
+                    <span class="k">raise</span> <span class="ne">AttributeError</span><span class="p">(</span><span class="n">err_msg</span><span class="p">)</span>
+                <span class="k">if</span> <span class="n">deeptext</span><span class="o">.</span><span class="n">output_dim</span> <span class="o">==</span> <span class="mi">1</span><span class="p">:</span>
+                    <span class="k">assert</span> <span class="n">pred_dim</span> <span class="o">==</span> <span class="mi">1</span><span class="p">,</span> <span class="s2">&quot;If &#39;output_dim&#39; is 1, &#39;pred_dim&#39; must be 1&quot;</span>
+
+        <span class="k">if</span> <span class="n">deepimage</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="n">err_msg</span> <span class="o">=</span> <span class="s2">&quot;deepimage model must have an &#39;output_dim&#39; attribute or property.&quot;</span>
+            <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">deepimage</span><span class="p">,</span> <span class="nb">list</span><span class="p">):</span>
+                <span class="n">all_have_output_dim</span> <span class="o">=</span> <span class="nb">all</span><span class="p">(</span><span class="nb">hasattr</span><span class="p">(</span><span class="n">di</span><span class="p">,</span> <span class="s2">&quot;output_dim&quot;</span><span class="p">)</span> <span class="k">for</span> <span class="n">di</span> <span class="ow">in</span> <span class="n">deepimage</span><span class="p">)</span>
+                <span class="k">if</span> <span class="ow">not</span> <span class="n">all_have_output_dim</span><span class="p">:</span>
+                    <span class="k">raise</span> <span class="ne">AttributeError</span><span class="p">(</span><span class="n">err_msg</span><span class="p">)</span>
+            <span class="k">else</span><span class="p">:</span>
+                <span class="k">if</span> <span class="ow">not</span> <span class="nb">hasattr</span><span class="p">(</span><span class="n">deepimage</span><span class="p">,</span> <span class="s2">&quot;output_dim&quot;</span><span class="p">):</span>
+                    <span class="k">raise</span> <span class="ne">AttributeError</span><span class="p">(</span><span class="n">err_msg</span><span class="p">)</span>
+                <span class="k">if</span> <span class="n">deepimage</span><span class="o">.</span><span class="n">output_dim</span> <span class="o">==</span> <span class="mi">1</span><span class="p">:</span>
+                    <span class="k">assert</span> <span class="n">pred_dim</span> <span class="o">==</span> <span class="mi">1</span><span class="p">,</span> <span class="s2">&quot;If &#39;output_dim&#39; is 1, &#39;pred_dim&#39; must be 1&quot;</span>
+
+        <span class="k">if</span> <span class="n">deephead</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">and</span> <span class="n">head_hidden_dims</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
+                <span class="s2">&quot;both &#39;deephead&#39; and &#39;head_hidden_dims&#39; are not None. Use one of the other, but not both&quot;</span>
+            <span class="p">)</span>
+        <span class="k">if</span> <span class="p">(</span>
+            <span class="n">head_hidden_dims</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span>
+            <span class="ow">and</span> <span class="ow">not</span> <span class="n">deeptabular</span>
+            <span class="ow">and</span> <span class="ow">not</span> <span class="n">deeptext</span>
+            <span class="ow">and</span> <span class="ow">not</span> <span class="n">deepimage</span>
+        <span class="p">):</span>
+            <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
+                <span class="s2">&quot;if &#39;head_hidden_dims&#39; is not None, at least one deep component must be used&quot;</span>
+            <span class="p">)</span>
 
-    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">enforce_positive</span><span class="p">:</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">enf_pos</span> <span class="o">=</span> <span class="n">get_activation_fn</span><span class="p">(</span><span class="n">enforce_positive_activation</span><span class="p">)</span>
+        <span class="k">if</span> <span class="n">deephead</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="k">if</span> <span class="ow">not</span> <span class="nb">hasattr</span><span class="p">(</span><span class="n">deephead</span><span class="p">,</span> <span class="s2">&quot;output_dim&quot;</span><span class="p">):</span>
+                <span class="k">raise</span> <span class="ne">AttributeError</span><span class="p">(</span>
+                    <span class="s2">&quot;As any other custom model passed to &#39;WideDeep&#39;, &#39;deephead&#39; must have an &quot;</span>
+                    <span class="s2">&quot;&#39;output_dim&#39; attribute or property. &quot;</span>
+                <span class="p">)</span>
+            <span class="n">deephead_inp_feat</span> <span class="o">=</span> <span class="nb">next</span><span class="p">(</span><span class="n">deephead</span><span class="o">.</span><span class="n">parameters</span><span class="p">())</span><span class="o">.</span><span class="n">size</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span>
+            <span class="n">output_dim</span> <span class="o">=</span> <span class="mi">0</span>
+            <span class="k">if</span> <span class="n">deeptabular</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+                <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">deeptabular</span><span class="p">,</span> <span class="nb">list</span><span class="p">):</span>
+                    <span class="k">for</span> <span class="n">dt</span> <span class="ow">in</span> <span class="n">deeptabular</span><span class="p">:</span>
+                        <span class="n">output_dim</span> <span class="o">+=</span> <span class="n">dt</span><span class="o">.</span><span class="n">output_dim</span>
+                <span class="k">else</span><span class="p">:</span>
+                    <span class="n">output_dim</span> <span class="o">+=</span> <span class="n">deeptabular</span><span class="o">.</span><span class="n">output_dim</span>
+            <span class="k">if</span> <span class="n">deeptext</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+                <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">deeptext</span><span class="p">,</span> <span class="nb">list</span><span class="p">):</span>
+                    <span class="k">for</span> <span class="n">dt</span> <span class="ow">in</span> <span class="n">deeptext</span><span class="p">:</span>
+                        <span class="n">output_dim</span> <span class="o">+=</span> <span class="n">dt</span><span class="o">.</span><span class="n">output_dim</span>
+                <span class="k">else</span><span class="p">:</span>
+                    <span class="n">output_dim</span> <span class="o">+=</span> <span class="n">deeptext</span><span class="o">.</span><span class="n">output_dim</span>
+            <span class="k">if</span> <span class="n">deepimage</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+                <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">deepimage</span><span class="p">,</span> <span class="nb">list</span><span class="p">):</span>
+                    <span class="k">for</span> <span class="n">di</span> <span class="ow">in</span> <span class="n">deepimage</span><span class="p">:</span>
+                        <span class="n">output_dim</span> <span class="o">+=</span> <span class="n">di</span><span class="o">.</span><span class="n">output_dim</span>
+                <span class="k">else</span><span class="p">:</span>
+                    <span class="n">output_dim</span> <span class="o">+=</span> <span class="n">deepimage</span><span class="o">.</span><span class="n">output_dim</span>
+            <span class="k">if</span> <span class="n">deephead_inp_feat</span> <span class="o">!=</span> <span class="n">output_dim</span><span class="p">:</span>
+                <span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span>
+                    <span class="s2">&quot;A custom &#39;deephead&#39; is used and it seems that the input features &quot;</span>
+                    <span class="s2">&quot;do not match the output of the deep components&quot;</span><span class="p">,</span>
+                    <span class="ne">UserWarning</span><span class="p">,</span>
+                <span class="p">)</span>
 </code></pre></div></td></tr></table></div>
-                  </details>
+              </details>
 
 
 
diff --git a/mkdocs/site/pytorch-widedeep/model_components.md b/mkdocs/site/pytorch-widedeep/model_components.md
index ab42e7c9..c59332aa 100644
--- a/mkdocs/site/pytorch-widedeep/model_components.md
+++ b/mkdocs/site/pytorch-widedeep/model_components.md
@@ -12,7 +12,6 @@ self-supervised pre-training with tabular data.
     selection:
         filters:
             - "!^forward$"
-            - "!^_reset_parameters$"
 
 ::: pytorch_widedeep.models.tabular.mlp.tab_mlp.TabMlp
     selection:
@@ -82,42 +81,35 @@ self-supervised pre-training with tabular data.
 ::: pytorch_widedeep.models.text.rnns.basic_rnn.BasicRNN
     selection:
         filters:
-            - "!^_"  # exclude all members starting with _
             - "!^forward$"
 
 ::: pytorch_widedeep.models.text.rnns.attentive_rnn.AttentiveRNN
     selection:
         filters:
-            - "!^_"  # exclude all members starting with _
             - "!^forward$"
 
 ::: pytorch_widedeep.models.text.rnns.stacked_attentive_rnn.StackedAttentiveRNN
     selection:
         filters:
-            - "!^_"  # exclude all members starting with _
             - "!^forward$"
 
 ::: pytorch_widedeep.models.text.huggingface_transformers.hf_model.HFModel
     selection:
         filters:
-            - "!^_"  # exclude all members starting with _
             - "!^forward$"
 
 
 ::: pytorch_widedeep.models.image.vision.Vision
     selection:
         filters:
-            - "!^_"  # exclude all members starting with _
             - "!^forward$"
 
 ::: pytorch_widedeep.models.model_fusion.ModelFuser
     selection:
         filters:
-            - "!^_"  # exclude all members starting with _
             - "!^forward$"
 
 ::: pytorch_widedeep.models.wide_deep.WideDeep
     selection:
         filters:
-            - "!^_"  # exclude all members starting with _
             - "!^forward$"
diff --git a/mkdocs/site/pytorch-widedeep/preprocessing.html b/mkdocs/site/pytorch-widedeep/preprocessing.html
index 5f013907..217aa63f 100644
--- a/mkdocs/site/pytorch-widedeep/preprocessing.html
+++ b/mkdocs/site/pytorch-widedeep/preprocessing.html
@@ -1486,7 +1486,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/16_Self_Supervised_Pretraning_pt1.ipynb" class="md-nav__link">
+      <a href="../examples/15_Self_Supervised_Pretraning_pt1.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1507,7 +1507,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/16_Self_Supervised_Pretraning_pt2.ipynb" class="md-nav__link">
+      <a href="../examples/15_Self_Supervised_Pretraning_pt2.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1528,7 +1528,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/17_Usign_a_custom_hugging_face_model.ipynb" class="md-nav__link">
+      <a href="../examples/16_Usign_a_custom_hugging_face_model.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1549,7 +1549,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/18_feature_importance_via_attention_weights.ipynb" class="md-nav__link">
+      <a href="../examples/17_feature_importance_via_attention_weights.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1570,7 +1570,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/19_wide_and_deep_for_recsys_pt1.ipynb" class="md-nav__link">
+      <a href="../examples/18_wide_and_deep_for_recsys_pt1.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1591,7 +1591,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/19_wide_and_deep_for_recsys_pt2.ipynb" class="md-nav__link">
+      <a href="../examples/18_wide_and_deep_for_recsys_pt2.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1612,7 +1612,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/20_load_from_folder_functionality.ipynb" class="md-nav__link">
+      <a href="../examples/19_load_from_folder_functionality.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1633,7 +1633,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/21_Using_huggingface_within_widedeep.ipynb" class="md-nav__link">
+      <a href="../examples/20_Using_huggingface_within_widedeep.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1711,8 +1711,7 @@ <h2 id="pytorch_widedeep.preprocessing.wide_preprocessor.WidePreprocessor" class
 
 
 <a href="#pytorch_widedeep.preprocessing.wide_preprocessor.WidePreprocessor" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="nf">WidePreprocessor</span><span class="p">(</span><span class="n">wide_cols</span><span class="p">,</span> <span class="n">crossed_cols</span><span class="o">=</span><span class="kc">None</span><span class="p">)</span>
-</code></pre></div>
+
 
     <div class="doc doc-contents first">
             <p class="doc doc-class-bases">
@@ -1728,68 +1727,109 @@ <h2 id="pytorch_widedeep.preprocessing.wide_preprocessor.WidePreprocessor" class
 
 
 <p><span class="doc-section-title">Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>wide_cols</code></b>
-              (<code><span title="typing.List">List</span>[str]</code>)
-          –
-          <div class="doc-md-description">
-            <p>List of strings with the name of the columns that will label
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+          <th>Default</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>wide_cols</code></td>
+            <td>
+                  <code><span title="typing.List">List</span>[str]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>List of strings with the name of the columns that will label
 encoded and passed through the <code>wide</code> component</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>crossed_cols</code></b>
-              (<code><span title="typing.Optional">Optional</span>[<span title="typing.List">List</span>[<span title="typing.Tuple">Tuple</span>[str, str]]]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>List of Tuples with the name of the columns that will be <code>'crossed'</code>
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>crossed_cols</code></td>
+            <td>
+                  <code><span title="typing.Optional">Optional</span>[<span title="typing.List">List</span>[<span title="typing.Tuple">Tuple</span>[str, str]]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>List of Tuples with the name of the columns that will be <code>'crossed'</code>
 and then label encoded. e.g. <em>[('education', 'occupation'), ...]</em>. For
 binary features, a cross-product transformation is 1 if and only if
 the constituent features are all 1, and 0 otherwise.</p>
-          </div>
-        </li>
-    </ul>
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
 
 <p><span class="doc-section-title">Attributes:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code><span title="pytorch_widedeep.preprocessing.wide_preprocessor.WidePreprocessor.wide_crossed_cols">wide_crossed_cols</span></code></b>
-              (<code><span title="typing.List">List</span></code>)
-          –
-          <div class="doc-md-description">
-            <p>List with the names of all columns that will be label encoded</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code><span title="pytorch_widedeep.preprocessing.wide_preprocessor.WidePreprocessor.encoding_dict">encoding_dict</span></code></b>
-              (<code>Dict</code>)
-          –
-          <div class="doc-md-description">
-            <p>Dictionary where the keys are the result of pasting <code>colname + '_' +
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code><span title="pytorch_widedeep.preprocessing.wide_preprocessor.WidePreprocessor.wide_crossed_cols">wide_crossed_cols</span></code></td>
+            <td>
+                  <code><span title="typing.List">List</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>List with the names of all columns that will be label encoded</p>
+              </div>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code><span title="pytorch_widedeep.preprocessing.wide_preprocessor.WidePreprocessor.encoding_dict">encoding_dict</span></code></td>
+            <td>
+                  <code>Dict</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Dictionary where the keys are the result of pasting <code>colname + '_' +
 column value</code> and the values are the corresponding mapped integer.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code><span title="pytorch_widedeep.preprocessing.wide_preprocessor.WidePreprocessor.inverse_encoding_dict">inverse_encoding_dict</span></code></b>
-              (<code>Dict</code>)
-          –
-          <div class="doc-md-description">
-            <p>the inverse encoding dictionary</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code><span title="pytorch_widedeep.preprocessing.wide_preprocessor.WidePreprocessor.wide_dim">wide_dim</span></code></b>
-              (<code>int</code>)
-          –
-          <div class="doc-md-description">
-            <p>Dimension of the wide model (i.e. dim of the linear layer)</p>
-          </div>
-        </li>
-    </ul>
+              </div>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code><span title="pytorch_widedeep.preprocessing.wide_preprocessor.WidePreprocessor.inverse_encoding_dict">inverse_encoding_dict</span></code></td>
+            <td>
+                  <code>Dict</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>the inverse encoding dictionary</p>
+              </div>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code><span title="pytorch_widedeep.preprocessing.wide_preprocessor.WidePreprocessor.wide_dim">wide_dim</span></code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Dimension of the wide model (i.e. dim of the linear layer)</p>
+              </div>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
 
 <p><span class="doc-section-title">Examples:</span></p>
@@ -1813,27 +1853,399 @@ <h2 id="pytorch_widedeep.preprocessing.wide_preprocessor.WidePreprocessor" class
 <span class="go">2     g        g-l</span>
 </code></pre></div>
 
-                  <details class="quote">
-                    <summary>Source code in <code>pytorch_widedeep/preprocessing/wide_preprocessor.py</code></summary>
-                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">66</span>
-<span class="normal">67</span>
-<span class="normal">68</span>
-<span class="normal">69</span>
-<span class="normal">70</span>
-<span class="normal">71</span>
-<span class="normal">72</span>
-<span class="normal">73</span>
-<span class="normal">74</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
-    <span class="bp">self</span><span class="p">,</span> <span class="n">wide_cols</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">],</span> <span class="n">crossed_cols</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="n">Tuple</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">]]]</span> <span class="o">=</span> <span class="kc">None</span>
-<span class="p">):</span>
-    <span class="nb">super</span><span class="p">(</span><span class="n">WidePreprocessor</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
-
-    <span class="bp">self</span><span class="o">.</span><span class="n">wide_cols</span> <span class="o">=</span> <span class="n">wide_cols</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">crossed_cols</span> <span class="o">=</span> <span class="n">crossed_cols</span>
-
-    <span class="bp">self</span><span class="o">.</span><span class="n">is_fitted</span> <span class="o">=</span> <span class="kc">False</span>
+              <details class="quote">
+                <summary>Source code in <code>pytorch_widedeep/preprocessing/wide_preprocessor.py</code></summary>
+                <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 12</span>
+<span class="normal"> 13</span>
+<span class="normal"> 14</span>
+<span class="normal"> 15</span>
+<span class="normal"> 16</span>
+<span class="normal"> 17</span>
+<span class="normal"> 18</span>
+<span class="normal"> 19</span>
+<span class="normal"> 20</span>
+<span class="normal"> 21</span>
+<span class="normal"> 22</span>
+<span class="normal"> 23</span>
+<span class="normal"> 24</span>
+<span class="normal"> 25</span>
+<span class="normal"> 26</span>
+<span class="normal"> 27</span>
+<span class="normal"> 28</span>
+<span class="normal"> 29</span>
+<span class="normal"> 30</span>
+<span class="normal"> 31</span>
+<span class="normal"> 32</span>
+<span class="normal"> 33</span>
+<span class="normal"> 34</span>
+<span class="normal"> 35</span>
+<span class="normal"> 36</span>
+<span class="normal"> 37</span>
+<span class="normal"> 38</span>
+<span class="normal"> 39</span>
+<span class="normal"> 40</span>
+<span class="normal"> 41</span>
+<span class="normal"> 42</span>
+<span class="normal"> 43</span>
+<span class="normal"> 44</span>
+<span class="normal"> 45</span>
+<span class="normal"> 46</span>
+<span class="normal"> 47</span>
+<span class="normal"> 48</span>
+<span class="normal"> 49</span>
+<span class="normal"> 50</span>
+<span class="normal"> 51</span>
+<span class="normal"> 52</span>
+<span class="normal"> 53</span>
+<span class="normal"> 54</span>
+<span class="normal"> 55</span>
+<span class="normal"> 56</span>
+<span class="normal"> 57</span>
+<span class="normal"> 58</span>
+<span class="normal"> 59</span>
+<span class="normal"> 60</span>
+<span class="normal"> 61</span>
+<span class="normal"> 62</span>
+<span class="normal"> 63</span>
+<span class="normal"> 64</span>
+<span class="normal"> 65</span>
+<span class="normal"> 66</span>
+<span class="normal"> 67</span>
+<span class="normal"> 68</span>
+<span class="normal"> 69</span>
+<span class="normal"> 70</span>
+<span class="normal"> 71</span>
+<span class="normal"> 72</span>
+<span class="normal"> 73</span>
+<span class="normal"> 74</span>
+<span class="normal"> 75</span>
+<span class="normal"> 76</span>
+<span class="normal"> 77</span>
+<span class="normal"> 78</span>
+<span class="normal"> 79</span>
+<span class="normal"> 80</span>
+<span class="normal"> 81</span>
+<span class="normal"> 82</span>
+<span class="normal"> 83</span>
+<span class="normal"> 84</span>
+<span class="normal"> 85</span>
+<span class="normal"> 86</span>
+<span class="normal"> 87</span>
+<span class="normal"> 88</span>
+<span class="normal"> 89</span>
+<span class="normal"> 90</span>
+<span class="normal"> 91</span>
+<span class="normal"> 92</span>
+<span class="normal"> 93</span>
+<span class="normal"> 94</span>
+<span class="normal"> 95</span>
+<span class="normal"> 96</span>
+<span class="normal"> 97</span>
+<span class="normal"> 98</span>
+<span class="normal"> 99</span>
+<span class="normal">100</span>
+<span class="normal">101</span>
+<span class="normal">102</span>
+<span class="normal">103</span>
+<span class="normal">104</span>
+<span class="normal">105</span>
+<span class="normal">106</span>
+<span class="normal">107</span>
+<span class="normal">108</span>
+<span class="normal">109</span>
+<span class="normal">110</span>
+<span class="normal">111</span>
+<span class="normal">112</span>
+<span class="normal">113</span>
+<span class="normal">114</span>
+<span class="normal">115</span>
+<span class="normal">116</span>
+<span class="normal">117</span>
+<span class="normal">118</span>
+<span class="normal">119</span>
+<span class="normal">120</span>
+<span class="normal">121</span>
+<span class="normal">122</span>
+<span class="normal">123</span>
+<span class="normal">124</span>
+<span class="normal">125</span>
+<span class="normal">126</span>
+<span class="normal">127</span>
+<span class="normal">128</span>
+<span class="normal">129</span>
+<span class="normal">130</span>
+<span class="normal">131</span>
+<span class="normal">132</span>
+<span class="normal">133</span>
+<span class="normal">134</span>
+<span class="normal">135</span>
+<span class="normal">136</span>
+<span class="normal">137</span>
+<span class="normal">138</span>
+<span class="normal">139</span>
+<span class="normal">140</span>
+<span class="normal">141</span>
+<span class="normal">142</span>
+<span class="normal">143</span>
+<span class="normal">144</span>
+<span class="normal">145</span>
+<span class="normal">146</span>
+<span class="normal">147</span>
+<span class="normal">148</span>
+<span class="normal">149</span>
+<span class="normal">150</span>
+<span class="normal">151</span>
+<span class="normal">152</span>
+<span class="normal">153</span>
+<span class="normal">154</span>
+<span class="normal">155</span>
+<span class="normal">156</span>
+<span class="normal">157</span>
+<span class="normal">158</span>
+<span class="normal">159</span>
+<span class="normal">160</span>
+<span class="normal">161</span>
+<span class="normal">162</span>
+<span class="normal">163</span>
+<span class="normal">164</span>
+<span class="normal">165</span>
+<span class="normal">166</span>
+<span class="normal">167</span>
+<span class="normal">168</span>
+<span class="normal">169</span>
+<span class="normal">170</span>
+<span class="normal">171</span>
+<span class="normal">172</span>
+<span class="normal">173</span>
+<span class="normal">174</span>
+<span class="normal">175</span>
+<span class="normal">176</span>
+<span class="normal">177</span>
+<span class="normal">178</span>
+<span class="normal">179</span>
+<span class="normal">180</span>
+<span class="normal">181</span>
+<span class="normal">182</span>
+<span class="normal">183</span>
+<span class="normal">184</span>
+<span class="normal">185</span>
+<span class="normal">186</span>
+<span class="normal">187</span>
+<span class="normal">188</span>
+<span class="normal">189</span>
+<span class="normal">190</span>
+<span class="normal">191</span>
+<span class="normal">192</span>
+<span class="normal">193</span>
+<span class="normal">194</span>
+<span class="normal">195</span>
+<span class="normal">196</span>
+<span class="normal">197</span>
+<span class="normal">198</span>
+<span class="normal">199</span>
+<span class="normal">200</span>
+<span class="normal">201</span>
+<span class="normal">202</span>
+<span class="normal">203</span>
+<span class="normal">204</span>
+<span class="normal">205</span>
+<span class="normal">206</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">class</span> <span class="nc">WidePreprocessor</span><span class="p">(</span><span class="n">BasePreprocessor</span><span class="p">):</span>
+<span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Preprocessor to prepare the wide input dataset</span>
+
+<span class="sd">    This Preprocessor prepares the data for the wide, linear component.</span>
+<span class="sd">    This linear model is implemented via an Embedding layer that is</span>
+<span class="sd">    connected to the output neuron. `WidePreprocessor` numerically</span>
+<span class="sd">    encodes all the unique values of all categorical columns `wide_cols +</span>
+<span class="sd">    crossed_cols`. See the Example below.</span>
+
+<span class="sd">    Parameters</span>
+<span class="sd">    ----------</span>
+<span class="sd">    wide_cols: List</span>
+<span class="sd">        List of strings with the name of the columns that will label</span>
+<span class="sd">        encoded and passed through the `wide` component</span>
+<span class="sd">    crossed_cols: List, default = None</span>
+<span class="sd">        List of Tuples with the name of the columns that will be `&#39;crossed&#39;`</span>
+<span class="sd">        and then label encoded. e.g. _[(&#39;education&#39;, &#39;occupation&#39;), ...]_. For</span>
+<span class="sd">        binary features, a cross-product transformation is 1 if and only if</span>
+<span class="sd">        the constituent features are all 1, and 0 otherwise.</span>
+
+<span class="sd">    Attributes</span>
+<span class="sd">    ----------</span>
+<span class="sd">    wide_crossed_cols: List</span>
+<span class="sd">        List with the names of all columns that will be label encoded</span>
+<span class="sd">    encoding_dict: Dict</span>
+<span class="sd">        Dictionary where the keys are the result of pasting `colname + &#39;_&#39; +</span>
+<span class="sd">        column value` and the values are the corresponding mapped integer.</span>
+<span class="sd">    inverse_encoding_dict: Dict</span>
+<span class="sd">        the inverse encoding dictionary</span>
+<span class="sd">    wide_dim: int</span>
+<span class="sd">        Dimension of the wide model (i.e. dim of the linear layer)</span>
+
+<span class="sd">    Examples</span>
+<span class="sd">    --------</span>
+<span class="sd">    &gt;&gt;&gt; import pandas as pd</span>
+<span class="sd">    &gt;&gt;&gt; from pytorch_widedeep.preprocessing import WidePreprocessor</span>
+<span class="sd">    &gt;&gt;&gt; df = pd.DataFrame({&#39;color&#39;: [&#39;r&#39;, &#39;b&#39;, &#39;g&#39;], &#39;size&#39;: [&#39;s&#39;, &#39;n&#39;, &#39;l&#39;]})</span>
+<span class="sd">    &gt;&gt;&gt; wide_cols = [&#39;color&#39;]</span>
+<span class="sd">    &gt;&gt;&gt; crossed_cols = [(&#39;color&#39;, &#39;size&#39;)]</span>
+<span class="sd">    &gt;&gt;&gt; wide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols)</span>
+<span class="sd">    &gt;&gt;&gt; X_wide = wide_preprocessor.fit_transform(df)</span>
+<span class="sd">    &gt;&gt;&gt; X_wide</span>
+<span class="sd">    array([[1, 4],</span>
+<span class="sd">           [2, 5],</span>
+<span class="sd">           [3, 6]])</span>
+<span class="sd">    &gt;&gt;&gt; wide_preprocessor.encoding_dict</span>
+<span class="sd">    {&#39;color_r&#39;: 1, &#39;color_b&#39;: 2, &#39;color_g&#39;: 3, &#39;color_size_r-s&#39;: 4, &#39;color_size_b-n&#39;: 5, &#39;color_size_g-l&#39;: 6}</span>
+<span class="sd">    &gt;&gt;&gt; wide_preprocessor.inverse_transform(X_wide)</span>
+<span class="sd">      color color_size</span>
+<span class="sd">    0     r        r-s</span>
+<span class="sd">    1     b        b-n</span>
+<span class="sd">    2     g        g-l</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+
+    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span> <span class="n">wide_cols</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">],</span> <span class="n">crossed_cols</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="n">Tuple</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">]]]</span> <span class="o">=</span> <span class="kc">None</span>
+    <span class="p">):</span>
+        <span class="nb">super</span><span class="p">(</span><span class="n">WidePreprocessor</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">wide_cols</span> <span class="o">=</span> <span class="n">wide_cols</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">crossed_cols</span> <span class="o">=</span> <span class="n">crossed_cols</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">is_fitted</span> <span class="o">=</span> <span class="kc">False</span>
+
+    <span class="k">def</span> <span class="nf">fit</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="s2">&quot;WidePreprocessor&quot;</span><span class="p">:</span>
+<span class="w">        </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Fits the Preprocessor and creates required attributes</span>
+
+<span class="sd">        Parameters</span>
+<span class="sd">        ----------</span>
+<span class="sd">        df: pd.DataFrame</span>
+<span class="sd">            Input pandas dataframe</span>
+
+<span class="sd">        Returns</span>
+<span class="sd">        -------</span>
+<span class="sd">        WidePreprocessor</span>
+<span class="sd">            `WidePreprocessor` fitted object</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">df_wide</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_prepare_wide</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">wide_crossed_cols</span> <span class="o">=</span> <span class="n">df_wide</span><span class="o">.</span><span class="n">columns</span><span class="o">.</span><span class="n">tolist</span><span class="p">()</span>
+        <span class="n">glob_feature_list</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_make_global_feature_list</span><span class="p">(</span>
+            <span class="n">df_wide</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">wide_crossed_cols</span><span class="p">]</span>
+        <span class="p">)</span>
+        <span class="c1"># leave 0 for padding/&quot;unseen&quot; categories</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">encoding_dict</span> <span class="o">=</span> <span class="p">{</span><span class="n">v</span><span class="p">:</span> <span class="n">i</span> <span class="o">+</span> <span class="mi">1</span> <span class="k">for</span> <span class="n">i</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="n">glob_feature_list</span><span class="p">)}</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">wide_dim</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">encoding_dict</span><span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">inverse_encoding_dict</span> <span class="o">=</span> <span class="p">{</span><span class="n">k</span><span class="p">:</span> <span class="n">v</span> <span class="k">for</span> <span class="n">v</span><span class="p">,</span> <span class="n">k</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">encoding_dict</span><span class="o">.</span><span class="n">items</span><span class="p">()}</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">inverse_encoding_dict</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">=</span> <span class="s2">&quot;unseen&quot;</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">is_fitted</span> <span class="o">=</span> <span class="kc">True</span>
+
+        <span class="k">return</span> <span class="bp">self</span>
+
+    <span class="k">def</span> <span class="nf">transform</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">:</span>
+<span class="w">        </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Parameters</span>
+<span class="sd">        ----------</span>
+<span class="sd">        df: pd.DataFrame</span>
+<span class="sd">            Input pandas dataframe</span>
+
+<span class="sd">        Returns</span>
+<span class="sd">        -------</span>
+<span class="sd">        np.ndarray</span>
+<span class="sd">            transformed input dataframe</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">check_is_fitted</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">attributes</span><span class="o">=</span><span class="p">[</span><span class="s2">&quot;encoding_dict&quot;</span><span class="p">])</span>
+        <span class="n">df_wide</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_prepare_wide</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
+        <span class="n">encoded</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">zeros</span><span class="p">([</span><span class="nb">len</span><span class="p">(</span><span class="n">df_wide</span><span class="p">),</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">wide_crossed_cols</span><span class="p">)])</span>
+        <span class="k">for</span> <span class="n">col_i</span><span class="p">,</span> <span class="n">col</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">wide_crossed_cols</span><span class="p">):</span>
+            <span class="n">encoded</span><span class="p">[:,</span> <span class="n">col_i</span><span class="p">]</span> <span class="o">=</span> <span class="n">df_wide</span><span class="p">[</span><span class="n">col</span><span class="p">]</span><span class="o">.</span><span class="n">apply</span><span class="p">(</span>
+                <span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="p">(</span>
+                    <span class="bp">self</span><span class="o">.</span><span class="n">encoding_dict</span><span class="p">[</span><span class="n">col</span> <span class="o">+</span> <span class="s2">&quot;_&quot;</span> <span class="o">+</span> <span class="nb">str</span><span class="p">(</span><span class="n">x</span><span class="p">)]</span>
+                    <span class="k">if</span> <span class="n">col</span> <span class="o">+</span> <span class="s2">&quot;_&quot;</span> <span class="o">+</span> <span class="nb">str</span><span class="p">(</span><span class="n">x</span><span class="p">)</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">encoding_dict</span>
+                    <span class="k">else</span> <span class="mi">0</span>
+                <span class="p">)</span>
+            <span class="p">)</span>
+        <span class="k">return</span> <span class="n">encoded</span><span class="o">.</span><span class="n">astype</span><span class="p">(</span><span class="s2">&quot;int64&quot;</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">transform_sample</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">:</span>
+        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">transform</span><span class="p">(</span><span class="n">df</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span>
+
+    <span class="k">def</span> <span class="nf">inverse_transform</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">encoded</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">:</span>
+<span class="w">        </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Takes as input the output from the `transform` method and it will</span>
+<span class="sd">        return the original values.</span>
+
+<span class="sd">        Parameters</span>
+<span class="sd">        ----------</span>
+<span class="sd">        encoded: np.ndarray</span>
+<span class="sd">            numpy array with the encoded values that are the output from the</span>
+<span class="sd">            `transform` method</span>
+
+<span class="sd">        Returns</span>
+<span class="sd">        -------</span>
+<span class="sd">        pd.DataFrame</span>
+<span class="sd">            Pandas dataframe with the original values</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">decoded</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">(</span><span class="n">encoded</span><span class="p">,</span> <span class="n">columns</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">wide_crossed_cols</span><span class="p">)</span>
+
+        <span class="k">if</span> <span class="n">pd</span><span class="o">.</span><span class="n">__version__</span> <span class="o">&gt;=</span> <span class="s2">&quot;2.1.0&quot;</span><span class="p">:</span>
+            <span class="n">decoded</span> <span class="o">=</span> <span class="n">decoded</span><span class="o">.</span><span class="n">map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">inverse_encoding_dict</span><span class="p">[</span><span class="n">x</span><span class="p">])</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="n">decoded</span> <span class="o">=</span> <span class="n">decoded</span><span class="o">.</span><span class="n">applymap</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">inverse_encoding_dict</span><span class="p">[</span><span class="n">x</span><span class="p">])</span>
+
+        <span class="k">for</span> <span class="n">col</span> <span class="ow">in</span> <span class="n">decoded</span><span class="o">.</span><span class="n">columns</span><span class="p">:</span>
+            <span class="n">rm_str</span> <span class="o">=</span> <span class="s2">&quot;&quot;</span><span class="o">.</span><span class="n">join</span><span class="p">([</span><span class="n">col</span><span class="p">,</span> <span class="s2">&quot;_&quot;</span><span class="p">])</span>
+            <span class="n">decoded</span><span class="p">[</span><span class="n">col</span><span class="p">]</span> <span class="o">=</span> <span class="n">decoded</span><span class="p">[</span><span class="n">col</span><span class="p">]</span><span class="o">.</span><span class="n">apply</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">x</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="n">rm_str</span><span class="p">,</span> <span class="s2">&quot;&quot;</span><span class="p">))</span>
+        <span class="k">return</span> <span class="n">decoded</span>
+
+    <span class="k">def</span> <span class="nf">fit_transform</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">:</span>
+<span class="w">        </span><span class="sd">&quot;&quot;&quot;Combines `fit` and `transform`</span>
+
+<span class="sd">        Parameters</span>
+<span class="sd">        ----------</span>
+<span class="sd">        df: pd.DataFrame</span>
+<span class="sd">            Input pandas dataframe</span>
+
+<span class="sd">        Returns</span>
+<span class="sd">        -------</span>
+<span class="sd">        np.ndarray</span>
+<span class="sd">            transformed input dataframe</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">df</span><span class="p">)</span><span class="o">.</span><span class="n">transform</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">_make_global_feature_list</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">List</span><span class="p">:</span>
+        <span class="n">glob_feature_list</span> <span class="o">=</span> <span class="p">[]</span>
+        <span class="k">for</span> <span class="n">column</span> <span class="ow">in</span> <span class="n">df</span><span class="o">.</span><span class="n">columns</span><span class="p">:</span>
+            <span class="n">glob_feature_list</span> <span class="o">+=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_make_column_feature_list</span><span class="p">(</span><span class="n">df</span><span class="p">[</span><span class="n">column</span><span class="p">])</span>
+        <span class="k">return</span> <span class="n">glob_feature_list</span>
+
+    <span class="k">def</span> <span class="nf">_make_column_feature_list</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">s</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">Series</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">List</span><span class="p">:</span>
+        <span class="k">return</span> <span class="p">[</span><span class="n">s</span><span class="o">.</span><span class="n">name</span> <span class="o">+</span> <span class="s2">&quot;_&quot;</span> <span class="o">+</span> <span class="nb">str</span><span class="p">(</span><span class="n">x</span><span class="p">)</span> <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="n">s</span><span class="o">.</span><span class="n">unique</span><span class="p">()]</span>
+
+    <span class="k">def</span> <span class="nf">_cross_cols</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">):</span>
+        <span class="n">df_cc</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
+        <span class="n">crossed_colnames</span> <span class="o">=</span> <span class="p">[]</span>
+        <span class="k">for</span> <span class="n">cols</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">crossed_cols</span><span class="p">:</span>
+            <span class="k">for</span> <span class="n">c</span> <span class="ow">in</span> <span class="n">cols</span><span class="p">:</span>
+                <span class="n">df_cc</span><span class="p">[</span><span class="n">c</span><span class="p">]</span> <span class="o">=</span> <span class="n">df_cc</span><span class="p">[</span><span class="n">c</span><span class="p">]</span><span class="o">.</span><span class="n">astype</span><span class="p">(</span><span class="s2">&quot;str&quot;</span><span class="p">)</span>
+            <span class="n">colname</span> <span class="o">=</span> <span class="s2">&quot;_&quot;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">cols</span><span class="p">)</span>
+            <span class="n">df_cc</span><span class="p">[</span><span class="n">colname</span><span class="p">]</span> <span class="o">=</span> <span class="n">df_cc</span><span class="p">[</span><span class="nb">list</span><span class="p">(</span><span class="n">cols</span><span class="p">)]</span><span class="o">.</span><span class="n">apply</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="s2">&quot;-&quot;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">x</span><span class="p">),</span> <span class="n">axis</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span>
+            <span class="n">crossed_colnames</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">colname</span><span class="p">)</span>
+        <span class="k">return</span> <span class="n">df_cc</span><span class="p">[</span><span class="n">crossed_colnames</span><span class="p">]</span>
+
+    <span class="k">def</span> <span class="nf">_prepare_wide</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">):</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">crossed_cols</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="n">df_cc</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_cross_cols</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
+            <span class="k">return</span> <span class="n">pd</span><span class="o">.</span><span class="n">concat</span><span class="p">([</span><span class="n">df</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">wide_cols</span><span class="p">],</span> <span class="n">df_cc</span><span class="p">],</span> <span class="n">axis</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="k">return</span> <span class="n">df</span><span class="o">.</span><span class="n">copy</span><span class="p">()[</span><span class="bp">self</span><span class="o">.</span><span class="n">wide_cols</span><span class="p">]</span>
+
+    <span class="k">def</span> <span class="fm">__repr__</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">str</span><span class="p">:</span>
+        <span class="n">list_of_params</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[</span><span class="s2">&quot;wide_cols=</span><span class="si">{wide_cols}</span><span class="s2">&quot;</span><span class="p">]</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">crossed_cols</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="n">list_of_params</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">&quot;crossed_cols=</span><span class="si">{crossed_cols}</span><span class="s2">&quot;</span><span class="p">)</span>
+        <span class="n">all_params</span> <span class="o">=</span> <span class="s2">&quot;, &quot;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">list_of_params</span><span class="p">)</span>
+        <span class="k">return</span> <span class="sa">f</span><span class="s2">&quot;WidePreprocessor(</span><span class="si">{</span><span class="n">all_params</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="bp">self</span><span class="o">.</span><span class="vm">__dict__</span><span class="p">)</span><span class="si">}</span><span class="s2">)&quot;</span>
 </code></pre></div></td></tr></table></div>
-                  </details>
+              </details>
 
 
 
@@ -1864,28 +2276,55 @@ <h3 id="pytorch_widedeep.preprocessing.wide_preprocessor.WidePreprocessor.fit" c
 
 
 <p><span class="doc-section-title">Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>df</code></b>
-              (<code><span title="pandas.DataFrame">DataFrame</span></code>)
-          –
-          <div class="doc-md-description">
-            <p>Input pandas dataframe</p>
-          </div>
-        </li>
-    </ul>
-
-
-<p><span class="doc-section-title">Returns:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-              <code><a class="autorefs autorefs-internal" title="pytorch_widedeep.preprocessing.wide_preprocessor.WidePreprocessor" href="#pytorch_widedeep.preprocessing.wide_preprocessor.WidePreprocessor">WidePreprocessor</a></code>
-          –
-          <div class="doc-md-description">
-            <p><code>WidePreprocessor</code> fitted object</p>
-          </div>
-        </li>
-    </ul>
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+          <th>Default</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>df</code></td>
+            <td>
+                  <code><span title="pandas.DataFrame">DataFrame</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Input pandas dataframe</p>
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+      </tbody>
+    </table>
+
+
+    <p><span class="doc-section-title">Returns:</span></p>
+    <table>
+      <thead>
+        <tr>
+          <th>Type</th>
+          <th>Description</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td>
+                  <code><a class="autorefs autorefs-internal" title="pytorch_widedeep.preprocessing.wide_preprocessor.WidePreprocessor" href="#pytorch_widedeep.preprocessing.wide_preprocessor.WidePreprocessor">WidePreprocessor</a></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p><code>WidePreprocessor</code> fitted object</p>
+              </div>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
             <details class="quote">
               <summary>Source code in <code>pytorch_widedeep/preprocessing/wide_preprocessor.py</code></summary>
@@ -1964,28 +2403,55 @@ <h3 id="pytorch_widedeep.preprocessing.wide_preprocessor.WidePreprocessor.transf
 
 
 <p><span class="doc-section-title">Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>df</code></b>
-              (<code><span title="pandas.DataFrame">DataFrame</span></code>)
-          –
-          <div class="doc-md-description">
-            <p>Input pandas dataframe</p>
-          </div>
-        </li>
-    </ul>
-
-
-<p><span class="doc-section-title">Returns:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-              <code><span title="numpy.ndarray">ndarray</span></code>
-          –
-          <div class="doc-md-description">
-            <p>transformed input dataframe</p>
-          </div>
-        </li>
-    </ul>
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+          <th>Default</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>df</code></td>
+            <td>
+                  <code><span title="pandas.DataFrame">DataFrame</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Input pandas dataframe</p>
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+      </tbody>
+    </table>
+
+
+    <p><span class="doc-section-title">Returns:</span></p>
+    <table>
+      <thead>
+        <tr>
+          <th>Type</th>
+          <th>Description</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td>
+                  <code><span title="numpy.ndarray">ndarray</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>transformed input dataframe</p>
+              </div>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
             <details class="quote">
               <summary>Source code in <code>pytorch_widedeep/preprocessing/wide_preprocessor.py</code></summary>
@@ -2060,29 +2526,56 @@ <h3 id="pytorch_widedeep.preprocessing.wide_preprocessor.WidePreprocessor.invers
 
 
 <p><span class="doc-section-title">Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>encoded</code></b>
-              (<code><span title="numpy.ndarray">ndarray</span></code>)
-          –
-          <div class="doc-md-description">
-            <p>numpy array with the encoded values that are the output from the
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+          <th>Default</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>encoded</code></td>
+            <td>
+                  <code><span title="numpy.ndarray">ndarray</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>numpy array with the encoded values that are the output from the
 <code>transform</code> method</p>
-          </div>
-        </li>
-    </ul>
-
-
-<p><span class="doc-section-title">Returns:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-              <code><span title="pandas.DataFrame">DataFrame</span></code>
-          –
-          <div class="doc-md-description">
-            <p>Pandas dataframe with the original values</p>
-          </div>
-        </li>
-    </ul>
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+      </tbody>
+    </table>
+
+
+    <p><span class="doc-section-title">Returns:</span></p>
+    <table>
+      <thead>
+        <tr>
+          <th>Type</th>
+          <th>Description</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td>
+                  <code><span title="pandas.DataFrame">DataFrame</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Pandas dataframe with the original values</p>
+              </div>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
             <details class="quote">
               <summary>Source code in <code>pytorch_widedeep/preprocessing/wide_preprocessor.py</code></summary>
@@ -2160,28 +2653,55 @@ <h3 id="pytorch_widedeep.preprocessing.wide_preprocessor.WidePreprocessor.fit_tr
 
 
 <p><span class="doc-section-title">Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>df</code></b>
-              (<code><span title="pandas.DataFrame">DataFrame</span></code>)
-          –
-          <div class="doc-md-description">
-            <p>Input pandas dataframe</p>
-          </div>
-        </li>
-    </ul>
-
-
-<p><span class="doc-section-title">Returns:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-              <code><span title="numpy.ndarray">ndarray</span></code>
-          –
-          <div class="doc-md-description">
-            <p>transformed input dataframe</p>
-          </div>
-        </li>
-    </ul>
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+          <th>Default</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>df</code></td>
+            <td>
+                  <code><span title="pandas.DataFrame">DataFrame</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Input pandas dataframe</p>
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+      </tbody>
+    </table>
+
+
+    <p><span class="doc-section-title">Returns:</span></p>
+    <table>
+      <thead>
+        <tr>
+          <th>Type</th>
+          <th>Description</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td>
+                  <code><span title="numpy.ndarray">ndarray</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>transformed input dataframe</p>
+              </div>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
             <details class="quote">
               <summary>Source code in <code>pytorch_widedeep/preprocessing/wide_preprocessor.py</code></summary>
@@ -2235,24 +2755,7 @@ <h2 id="pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor" class="
 
 
 <a href="#pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="nf">TabPreprocessor</span><span class="p">(</span>
-    <span class="n">cat_embed_cols</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">continuous_cols</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">quantization_setup</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">cols_to_scale</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">auto_embed_dim</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
-    <span class="n">embedding_rule</span><span class="o">=</span><span class="s2">&quot;fastai_new&quot;</span><span class="p">,</span>
-    <span class="n">default_embed_dim</span><span class="o">=</span><span class="mi">16</span><span class="p">,</span>
-    <span class="n">with_attention</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
-    <span class="n">with_cls_token</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
-    <span class="n">shared_embed</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
-    <span class="n">verbose</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span>
-    <span class="o">*</span><span class="p">,</span>
-    <span class="n">scale</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
-    <span class="n">already_standard</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="o">**</span><span class="n">kwargs</span>
-<span class="p">)</span>
-</code></pre></div>
+
 
     <div class="doc doc-contents first">
             <p class="doc doc-class-bases">
@@ -2263,77 +2766,107 @@ <h2 id="pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor" class="
 
 
 <p><span class="doc-section-title">Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>cat_embed_cols</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[str], <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.wdtypes.Tuple">Tuple</span>[str, int]]]]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>List containing the name of the categorical columns that will be
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+          <th>Default</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>cat_embed_cols</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[str], <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.wdtypes.Tuple">Tuple</span>[str, int]]]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>List containing the name of the categorical columns that will be
 represented by embeddings (e.g. <em>['education', 'relationship', ...]</em>) or
 a Tuple with the name and the embedding dimension (e.g.: <em>[
-('education',32), ('relationship',16), ...]</em>)</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>continuous_cols</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[str]]</code>, default:
+('education',32), ('relationship',16), ...]</em>).<br/> Param alias: <code>embed_cols</code></p>
+              </div>
+            </td>
+            <td>
                   <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>List with the name of the continuous cols</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>quantization_setup</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[int, <span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, <span title="pytorch_widedeep.wdtypes.Union">Union</span>[int, <span title="pytorch_widedeep.wdtypes.List">List</span>[float]]]]]</code>, default:
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>continuous_cols</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[str]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>List with the name of the continuous cols</p>
+              </div>
+            </td>
+            <td>
                   <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Continuous columns can be turned into categorical via <code>pd.cut</code>. If
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>quantization_setup</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[int, <span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, <span title="pytorch_widedeep.wdtypes.Union">Union</span>[int, <span title="pytorch_widedeep.wdtypes.List">List</span>[float]]]]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Continuous columns can be turned into categorical via <code>pd.cut</code>. If
 <code>quantization_setup</code> is an <code>int</code>, all continuous columns will be
 quantized using this value as the number of bins. Alternatively, a
 dictionary where the keys are the column names to quantize and the
 values are the either integers indicating the number of bins or a
-list of scalars indicating the bin edges can also be used.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>cols_to_scale</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[str], str]]</code>, default:
+list of scalars indicating the bin edges can also be used.<br/>
+Param alias: <code>cols_and_bins</code></p>
+              </div>
+            </td>
+            <td>
                   <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>List with the names of the columns that will be standarised via
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>cols_to_scale</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[str], str]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>List with the names of the columns that will be standarised via
 sklearn's <code>StandardScaler</code>. It can also be the string <code>'all'</code> in
 which case all the continuous cols will be scaled.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>auto_embed_dim</code></b>
-              (<code>bool</code>, default:
-                  <code>True</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Boolean indicating whether the embedding dimensions will be
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>auto_embed_dim</code></td>
+            <td>
+                  <code>bool</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Boolean indicating whether the embedding dimensions will be
 automatically defined via rule of thumb. See <code>embedding_rule</code>
 below.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>embedding_rule</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[google, fastai_old, fastai_new]</code>, default:
-                  <code>&#39;fastai_new&#39;</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>If <code>auto_embed_dim=True</code>, this is the choice of embedding rule of
+              </div>
+            </td>
+            <td>
+                  <code>True</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>embedding_rule</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[google, fastai_old, fastai_new]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>If <code>auto_embed_dim=True</code>, this is the choice of embedding rule of
 thumb. Choices are:</p>
 <ul>
 <li>
@@ -2346,28 +2879,36 @@ <h2 id="pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor" class="
 <p><em>google</em>: <span class="arithmatex">\(min(600, round(n_{cat}^{0.24}))\)</span></p>
 </li>
 </ul>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>default_embed_dim</code></b>
-              (<code>int</code>, default:
-                  <code>16</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Dimension for the embeddings if the embedding dimension is not
+              </div>
+            </td>
+            <td>
+                  <code>&#39;fastai_new&#39;</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>default_embed_dim</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Dimension for the embeddings if the embedding dimension is not
 provided in the <code>cat_embed_cols</code> parameter and <code>auto_embed_dim</code> is
 set to <code>False</code>.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>with_attention</code></b>
-              (<code>bool</code>, default:
-                  <code>False</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Boolean indicating whether the preprocessed data will be passed to an
+              </div>
+            </td>
+            <td>
+                  <code>16</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>with_attention</code></td>
+            <td>
+                  <code>bool</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Boolean indicating whether the preprocessed data will be passed to an
 attention-based model (more precisely a model where all embeddings
 must have the same dimensions). If <code>True</code>, the param <code>cat_embed_cols</code>
 must just be a list containing just the categorical column names:
@@ -2375,159 +2916,272 @@ <h2 id="pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor" class="
 <em>['education', 'relationship', ...]</em>. This is because they will all be
  encoded using embeddings of the same dim, which will be specified
  later when the model is defined. <br/> Param alias:
- <code>for_transformer</code></p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>with_cls_token</code></b>
-              (<code>bool</code>, default:
+ <code>for_transformer</code>, <code>for_matrix_factorization</code>, <code>for_mf</code></p>
+              </div>
+            </td>
+            <td>
                   <code>False</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Boolean indicating if a <code>'[CLS]'</code> token will be added to the dataset
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>with_cls_token</code></td>
+            <td>
+                  <code>bool</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Boolean indicating if a <code>'[CLS]'</code> token will be added to the dataset
 when using attention-based models. The final hidden state
 corresponding to this token is used as the aggregated representation
 for classification and regression tasks. If not, the categorical
 and/or continuous embeddings will be concatenated before being passed
 to the final MLP (if present).</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>shared_embed</code></b>
-              (<code>bool</code>, default:
+              </div>
+            </td>
+            <td>
                   <code>False</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Boolean indicating if the embeddings will be "shared" when using
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>shared_embed</code></td>
+            <td>
+                  <code>bool</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Boolean indicating if the embeddings will be "shared" when using
 attention-based models. The idea behind <code>shared_embed</code> is
 described in the Appendix A in the <a href="https://arxiv.org/abs/2012.06678">TabTransformer paper</a>:
 <em>'The goal of having column embedding is to enable the model to
 distinguish the classes in one column from those in the other
 columns'</em>. In other words, the idea is to let the model learn which
 column is embedded at the time. See: <code>pytorch_widedeep.models.transformers._layers.SharedEmbeddings</code>.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>verbose</code></b>
-              (<code>int</code>, default:
-                  <code>1</code>
-)
-          –
-          <div class="doc-md-description">
-            
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>scale</code></b>
-              (<code>bool</code>, default:
+              </div>
+            </td>
+            <td>
                   <code>False</code>
-)
-          –
-          <div class="doc-md-description">
-            <p><img alt="ℹ️" class="emojione" src="https://cdnjs.cloudflare.com/ajax/libs/emojione/2.2.7/assets/png/2139.png" title=":information_source:" /> <strong>note</strong>: this arg will be removed in upcoming
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>verbose</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                
+              </div>
+            </td>
+            <td>
+                  <code>1</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>scale</code></td>
+            <td>
+                  <code>bool</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p><img alt="ℹ️" class="emojione" src="https://cdnjs.cloudflare.com/ajax/libs/emojione/2.2.7/assets/png/2139.png" title=":information_source:" /> <strong>note</strong>: this arg will be removed in upcoming
  releases. Please use <code>cols_to_scale</code> instead. <br/> Bool indicating
  whether or not to scale/standarise continuous cols. It is important
  to emphasize that all the DL models for tabular data in the library
  also include the possibility of normalising the input continuous
  features via a <code>BatchNorm</code> or a <code>LayerNorm</code>. <br/> Param alias:
  <code>scale_cont_cols</code>.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>already_standard</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[str]]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p><img alt="ℹ️" class="emojione" src="https://cdnjs.cloudflare.com/ajax/libs/emojione/2.2.7/assets/png/2139.png" title=":information_source:" /> <strong>note</strong>: this arg will be removed in upcoming
+              </div>
+            </td>
+            <td>
+                  <code>False</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>already_standard</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[str]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p><img alt="ℹ️" class="emojione" src="https://cdnjs.cloudflare.com/ajax/libs/emojione/2.2.7/assets/png/2139.png" title=":information_source:" /> <strong>note</strong>: this arg will be removed in upcoming
  releases. Please use <code>cols_to_scale</code> instead. <br/> List with the
  name of the continuous cols that do not need to be
  scaled/standarised.</p>
-          </div>
-        </li>
-    </ul>
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
 
 <p><span class="doc-section-title">Other Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>**kwargs</code></b>
-          –
-          <div class="doc-md-description">
-            <p><code>pd.cut</code> and <code>StandardScaler</code> related args</p>
-          </div>
-        </li>
-    </ul>
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>**kwargs</code></td>
+            <td>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p><code>pd.cut</code> and <code>StandardScaler</code> related args</p>
+              </div>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
 
 <p><span class="doc-section-title">Attributes:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code><span title="pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor.embed_dim">embed_dim</span></code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Dict">Dict</span></code>)
-          –
-          <div class="doc-md-description">
-            <p>Dictionary where keys are the embed cols and values are the embedding
-dimensions. If <code>with_attention</code> is set to <code>True</code> this attribute
-is not generated during the <code>fit</code> process</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code><span title="pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor.label_encoder">label_encoder</span></code></b>
-              (<code><a class="autorefs autorefs-internal" title="pytorch_widedeep.utils.deeptabular_utils.LabelEncoder" href="utils/deeptabular_utils.html#pytorch_widedeep.utils.deeptabular_utils.LabelEncoder">LabelEncoder</a></code>)
-          –
-          <div class="doc-md-description">
-            <p>see <code>pytorch_widedeep.utils.dense_utils.LabelEncder</code></p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code><span title="pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor.cat_embed_input">cat_embed_input</span></code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.List">List</span></code>)
-          –
-          <div class="doc-md-description">
-            <p>List of Tuples with the column name, number of individual values for
-that column and, If <code>with_attention</code> is set to <code>False</code>, the
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code><span title="pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor.cat_cols">cat_cols</span></code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.List">List</span>[str]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>List containing the names of the categorical columns after processing.</p>
+              </div>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code><span title="pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor.cols_and_bins">cols_and_bins</span></code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, <span title="pytorch_widedeep.wdtypes.Union">Union</span>[int, <span title="pytorch_widedeep.wdtypes.List">List</span>[float]]]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Dictionary containing the quantization setup after processing if
+quantization is requested. This is derived from the
+quantization_setup parameter.</p>
+              </div>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code><span title="pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor.quant_args">quant_args</span></code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Dict">Dict</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Dictionary containing arguments passed to pandas.cut() function for quantization.</p>
+              </div>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code><span title="pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor.scale_args">scale_args</span></code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Dict">Dict</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Dictionary containing arguments passed to StandardScaler.</p>
+              </div>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code><span title="pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor.label_encoder">label_encoder</span></code></td>
+            <td>
+                  <code><a class="autorefs autorefs-internal" title="pytorch_widedeep.utils.deeptabular_utils.LabelEncoder" href="utils/deeptabular_utils.html#pytorch_widedeep.utils.deeptabular_utils.LabelEncoder">LabelEncoder</a></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Instance of LabelEncoder used to encode categorical variables.
+See <code>pytorch_widedeep.utils.dense_utils.LabelEncoder</code>.</p>
+              </div>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code><span title="pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor.cat_embed_input">cat_embed_input</span></code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.List">List</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>List of tuples with the column name, number of individual values for
+that column and, if <code>with_attention</code> is set to <code>False</code>, the
 corresponding embeddings dim, e.g. <em>[('education', 16, 10),
 ('relationship', 6, 8), ...]</em>.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code><span title="pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor.standardize_cols">standardize_cols</span></code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.List">List</span></code>)
-          –
-          <div class="doc-md-description">
-            <p>List of the columns that will be standarized</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code><span title="pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor.scaler">scaler</span></code></b>
-              (<code><span title="sklearn.preprocessing.StandardScaler">StandardScaler</span></code>)
-          –
-          <div class="doc-md-description">
-            <p>an instance of <code>sklearn.preprocessing.StandardScaler</code></p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code><span title="pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor.column_idx">column_idx</span></code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Dict">Dict</span></code>)
-          –
-          <div class="doc-md-description">
-            <p>Dictionary where keys are column names and values are column indexes.
-This is neccesary to slice tensors</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code><span title="pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor.quantizer">quantizer</span></code></b>
-              (<code><a class="autorefs autorefs-internal" title="pytorch_widedeep.preprocessing.tab_preprocessor.Quantizer" href="#pytorch_widedeep.preprocessing.tab_preprocessor.Quantizer">Quantizer</a></code>)
-          –
-          <div class="doc-md-description">
-            <p>an instance of <code>Quantizer</code></p>
-          </div>
-        </li>
-    </ul>
+              </div>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code><span title="pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor.standardize_cols">standardize_cols</span></code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.List">List</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>List of the columns that will be standardized.</p>
+              </div>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code><span title="pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor.scaler">scaler</span></code></td>
+            <td>
+                  <code><span title="sklearn.preprocessing.StandardScaler">StandardScaler</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>An instance of <code>sklearn.preprocessing.StandardScaler</code> if standardization
+is requested.</p>
+              </div>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code><span title="pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor.column_idx">column_idx</span></code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Dict">Dict</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Dictionary where keys are column names and values are column indexes.
+This is necessary to slice tensors.</p>
+              </div>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code><span title="pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor.quantizer">quantizer</span></code></td>
+            <td>
+                  <code><a class="autorefs autorefs-internal" title="pytorch_widedeep.preprocessing.tab_preprocessor.Quantizer" href="#pytorch_widedeep.preprocessing.tab_preprocessor.Quantizer">Quantizer</a></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>An instance of <code>Quantizer</code> if quantization is requested.</p>
+              </div>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code><span title="pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor.is_fitted">is_fitted</span></code></td>
+            <td>
+                  <code>bool</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Boolean indicating if the preprocessor has been fitted.</p>
+              </div>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
 
 <p><span class="doc-section-title">Examples:</span></p>
@@ -2553,9 +3207,141 @@ <h2 id="pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor" class="
 <span class="gp">&gt;&gt;&gt; </span><span class="n">ft_cont_df2</span> <span class="o">=</span> <span class="n">tab_preprocessor2</span><span class="o">.</span><span class="n">fit_transform</span><span class="p">(</span><span class="n">cont_df</span><span class="p">)</span>
 </code></pre></div>
 
-                  <details class="quote">
-                    <summary>Source code in <code>pytorch_widedeep/preprocessing/tab_preprocessor.py</code></summary>
-                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">242</span>
+              <details class="quote">
+                <summary>Source code in <code>pytorch_widedeep/preprocessing/tab_preprocessor.py</code></summary>
+                <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">110</span>
+<span class="normal">111</span>
+<span class="normal">112</span>
+<span class="normal">113</span>
+<span class="normal">114</span>
+<span class="normal">115</span>
+<span class="normal">116</span>
+<span class="normal">117</span>
+<span class="normal">118</span>
+<span class="normal">119</span>
+<span class="normal">120</span>
+<span class="normal">121</span>
+<span class="normal">122</span>
+<span class="normal">123</span>
+<span class="normal">124</span>
+<span class="normal">125</span>
+<span class="normal">126</span>
+<span class="normal">127</span>
+<span class="normal">128</span>
+<span class="normal">129</span>
+<span class="normal">130</span>
+<span class="normal">131</span>
+<span class="normal">132</span>
+<span class="normal">133</span>
+<span class="normal">134</span>
+<span class="normal">135</span>
+<span class="normal">136</span>
+<span class="normal">137</span>
+<span class="normal">138</span>
+<span class="normal">139</span>
+<span class="normal">140</span>
+<span class="normal">141</span>
+<span class="normal">142</span>
+<span class="normal">143</span>
+<span class="normal">144</span>
+<span class="normal">145</span>
+<span class="normal">146</span>
+<span class="normal">147</span>
+<span class="normal">148</span>
+<span class="normal">149</span>
+<span class="normal">150</span>
+<span class="normal">151</span>
+<span class="normal">152</span>
+<span class="normal">153</span>
+<span class="normal">154</span>
+<span class="normal">155</span>
+<span class="normal">156</span>
+<span class="normal">157</span>
+<span class="normal">158</span>
+<span class="normal">159</span>
+<span class="normal">160</span>
+<span class="normal">161</span>
+<span class="normal">162</span>
+<span class="normal">163</span>
+<span class="normal">164</span>
+<span class="normal">165</span>
+<span class="normal">166</span>
+<span class="normal">167</span>
+<span class="normal">168</span>
+<span class="normal">169</span>
+<span class="normal">170</span>
+<span class="normal">171</span>
+<span class="normal">172</span>
+<span class="normal">173</span>
+<span class="normal">174</span>
+<span class="normal">175</span>
+<span class="normal">176</span>
+<span class="normal">177</span>
+<span class="normal">178</span>
+<span class="normal">179</span>
+<span class="normal">180</span>
+<span class="normal">181</span>
+<span class="normal">182</span>
+<span class="normal">183</span>
+<span class="normal">184</span>
+<span class="normal">185</span>
+<span class="normal">186</span>
+<span class="normal">187</span>
+<span class="normal">188</span>
+<span class="normal">189</span>
+<span class="normal">190</span>
+<span class="normal">191</span>
+<span class="normal">192</span>
+<span class="normal">193</span>
+<span class="normal">194</span>
+<span class="normal">195</span>
+<span class="normal">196</span>
+<span class="normal">197</span>
+<span class="normal">198</span>
+<span class="normal">199</span>
+<span class="normal">200</span>
+<span class="normal">201</span>
+<span class="normal">202</span>
+<span class="normal">203</span>
+<span class="normal">204</span>
+<span class="normal">205</span>
+<span class="normal">206</span>
+<span class="normal">207</span>
+<span class="normal">208</span>
+<span class="normal">209</span>
+<span class="normal">210</span>
+<span class="normal">211</span>
+<span class="normal">212</span>
+<span class="normal">213</span>
+<span class="normal">214</span>
+<span class="normal">215</span>
+<span class="normal">216</span>
+<span class="normal">217</span>
+<span class="normal">218</span>
+<span class="normal">219</span>
+<span class="normal">220</span>
+<span class="normal">221</span>
+<span class="normal">222</span>
+<span class="normal">223</span>
+<span class="normal">224</span>
+<span class="normal">225</span>
+<span class="normal">226</span>
+<span class="normal">227</span>
+<span class="normal">228</span>
+<span class="normal">229</span>
+<span class="normal">230</span>
+<span class="normal">231</span>
+<span class="normal">232</span>
+<span class="normal">233</span>
+<span class="normal">234</span>
+<span class="normal">235</span>
+<span class="normal">236</span>
+<span class="normal">237</span>
+<span class="normal">238</span>
+<span class="normal">239</span>
+<span class="normal">240</span>
+<span class="normal">241</span>
+<span class="normal">242</span>
 <span class="normal">243</span>
 <span class="normal">244</span>
 <span class="normal">245</span>
@@ -2612,122 +3398,9 @@ <h2 id="pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor" class="
 <span class="normal">296</span>
 <span class="normal">297</span>
 <span class="normal">298</span>
-<span class="normal">299</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="nd">@alias</span><span class="p">(</span><span class="s2">&quot;with_attention&quot;</span><span class="p">,</span> <span class="p">[</span><span class="s2">&quot;for_transformer&quot;</span><span class="p">,</span> <span class="s2">&quot;for_matrix_factorization&quot;</span><span class="p">,</span> <span class="s2">&quot;for_mf&quot;</span><span class="p">])</span>
-<span class="nd">@alias</span><span class="p">(</span><span class="s2">&quot;cat_embed_cols&quot;</span><span class="p">,</span> <span class="p">[</span><span class="s2">&quot;embed_cols&quot;</span><span class="p">])</span>
-<span class="nd">@alias</span><span class="p">(</span><span class="s2">&quot;scale&quot;</span><span class="p">,</span> <span class="p">[</span><span class="s2">&quot;scale_cont_cols&quot;</span><span class="p">])</span>
-<span class="nd">@alias</span><span class="p">(</span><span class="s2">&quot;quantization_setup&quot;</span><span class="p">,</span> <span class="p">[</span><span class="s2">&quot;cols_and_bins&quot;</span><span class="p">])</span>
-<span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
-    <span class="bp">self</span><span class="p">,</span>
-    <span class="n">cat_embed_cols</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Union</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">],</span> <span class="n">List</span><span class="p">[</span><span class="n">Tuple</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">]]]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">continuous_cols</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">quantization_setup</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span>
-        <span class="n">Union</span><span class="p">[</span><span class="nb">int</span><span class="p">,</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Union</span><span class="p">[</span><span class="nb">int</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="nb">float</span><span class="p">]]]]</span>
-    <span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">cols_to_scale</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Union</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">],</span> <span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">auto_embed_dim</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">,</span>
-    <span class="n">embedding_rule</span><span class="p">:</span> <span class="n">Literal</span><span class="p">[</span><span class="s2">&quot;google&quot;</span><span class="p">,</span> <span class="s2">&quot;fastai_old&quot;</span><span class="p">,</span> <span class="s2">&quot;fastai_new&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="s2">&quot;fastai_new&quot;</span><span class="p">,</span>
-    <span class="n">default_embed_dim</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">16</span><span class="p">,</span>
-    <span class="n">with_attention</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
-    <span class="n">with_cls_token</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
-    <span class="n">shared_embed</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
-    <span class="n">verbose</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1</span><span class="p">,</span>
-    <span class="o">*</span><span class="p">,</span>
-    <span class="n">scale</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
-    <span class="n">already_standard</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="o">**</span><span class="n">kwargs</span><span class="p">,</span>
-<span class="p">):</span>
-    <span class="nb">super</span><span class="p">(</span><span class="n">TabPreprocessor</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
-
-    <span class="bp">self</span><span class="o">.</span><span class="n">continuous_cols</span> <span class="o">=</span> <span class="n">continuous_cols</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">quantization_setup</span> <span class="o">=</span> <span class="n">quantization_setup</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">cols_to_scale</span> <span class="o">=</span> <span class="n">cols_to_scale</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">scale</span> <span class="o">=</span> <span class="n">scale</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">already_standard</span> <span class="o">=</span> <span class="n">already_standard</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">auto_embed_dim</span> <span class="o">=</span> <span class="n">auto_embed_dim</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">embedding_rule</span> <span class="o">=</span> <span class="n">embedding_rule</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">default_embed_dim</span> <span class="o">=</span> <span class="n">default_embed_dim</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">with_attention</span> <span class="o">=</span> <span class="n">with_attention</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">with_cls_token</span> <span class="o">=</span> <span class="n">with_cls_token</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">shared_embed</span> <span class="o">=</span> <span class="n">shared_embed</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">verbose</span> <span class="o">=</span> <span class="n">verbose</span>
-
-    <span class="bp">self</span><span class="o">.</span><span class="n">quant_args</span> <span class="o">=</span> <span class="p">{</span>
-        <span class="n">k</span><span class="p">:</span> <span class="n">v</span> <span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="n">kwargs</span><span class="o">.</span><span class="n">items</span><span class="p">()</span> <span class="k">if</span> <span class="n">k</span> <span class="ow">in</span> <span class="n">pd</span><span class="o">.</span><span class="n">cut</span><span class="o">.</span><span class="vm">__code__</span><span class="o">.</span><span class="n">co_varnames</span>
-    <span class="p">}</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">scale_args</span> <span class="o">=</span> <span class="p">{</span>
-        <span class="n">k</span><span class="p">:</span> <span class="n">v</span> <span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="n">kwargs</span><span class="o">.</span><span class="n">items</span><span class="p">()</span> <span class="k">if</span> <span class="n">k</span> <span class="ow">in</span> <span class="n">StandardScaler</span><span class="p">()</span><span class="o">.</span><span class="n">get_params</span><span class="p">()</span>
-    <span class="p">}</span>
-
-    <span class="bp">self</span><span class="o">.</span><span class="n">_check_inputs</span><span class="p">(</span><span class="n">cat_embed_cols</span><span class="p">)</span>
-
-    <span class="k">if</span> <span class="n">with_cls_token</span><span class="p">:</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">cat_embed_cols</span> <span class="o">=</span> <span class="p">(</span>
-            <span class="p">[</span><span class="s2">&quot;cls_token&quot;</span><span class="p">]</span> <span class="o">+</span> <span class="n">cat_embed_cols</span>  <span class="c1"># type: ignore[operator]</span>
-            <span class="k">if</span> <span class="n">cat_embed_cols</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span>
-            <span class="k">else</span> <span class="p">[</span><span class="s2">&quot;cls_token&quot;</span><span class="p">]</span>
-        <span class="p">)</span>
-    <span class="k">else</span><span class="p">:</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">cat_embed_cols</span> <span class="o">=</span> <span class="n">cat_embed_cols</span>  <span class="c1"># type: ignore[assignment]</span>
-
-    <span class="bp">self</span><span class="o">.</span><span class="n">is_fitted</span> <span class="o">=</span> <span class="kc">False</span>
-</code></pre></div></td></tr></table></div>
-                  </details>
-
-
-
-  <div class="doc doc-children">
-
-
-
-
-
-
-
-
-
-<div class="doc doc-object doc-function">
-
-
-<h3 id="pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor.fit" class="doc doc-heading">
-            <span class="doc doc-object-name doc-function-name">fit</span>
-
-
-<a href="#pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor.fit" class="headerlink" title="Permanent link">&para;</a></h3>
-<div class="doc-signature highlight"><pre><span></span><code><span class="nf">fit</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
-</code></pre></div>
-
-    <div class="doc doc-contents ">
-
-        <p>Fits the Preprocessor and creates required attributes</p>
-
-
-<p><span class="doc-section-title">Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>df</code></b>
-              (<code><span title="pandas.DataFrame">DataFrame</span></code>)
-          –
-          <div class="doc-md-description">
-            <p>Input pandas dataframe</p>
-          </div>
-        </li>
-    </ul>
-
-
-<p><span class="doc-section-title">Returns:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-              <code><a class="autorefs autorefs-internal" title="pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor" href="#pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor">TabPreprocessor</a></code>
-          –
-          <div class="doc-md-description">
-            <p><code>TabPreprocessor</code> fitted object</p>
-          </div>
-        </li>
-    </ul>
-
-            <details class="quote">
-              <summary>Source code in <code>pytorch_widedeep/preprocessing/tab_preprocessor.py</code></summary>
-              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">301</span>
+<span class="normal">299</span>
+<span class="normal">300</span>
+<span class="normal">301</span>
 <span class="normal">302</span>
 <span class="normal">303</span>
 <span class="normal">304</span>
@@ -2800,129 +3473,9 @@ <h3 id="pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor.fit" cla
 <span class="normal">371</span>
 <span class="normal">372</span>
 <span class="normal">373</span>
-<span class="normal">374</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">fit</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">BasePreprocessor</span><span class="p">:</span>  <span class="c1"># noqa: C901</span>
-<span class="w">    </span><span class="sd">&quot;&quot;&quot;Fits the Preprocessor and creates required attributes</span>
-
-<span class="sd">    Parameters</span>
-<span class="sd">    ----------</span>
-<span class="sd">    df: pd.DataFrame</span>
-<span class="sd">        Input pandas dataframe</span>
-
-<span class="sd">    Returns</span>
-<span class="sd">    -------</span>
-<span class="sd">    TabPreprocessor</span>
-<span class="sd">        `TabPreprocessor` fitted object</span>
-<span class="sd">    &quot;&quot;&quot;</span>
-
-    <span class="n">df_adj</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_insert_cls_token</span><span class="p">(</span><span class="n">df</span><span class="p">)</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">with_cls_token</span> <span class="k">else</span> <span class="n">df</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
-
-    <span class="bp">self</span><span class="o">.</span><span class="n">column_idx</span><span class="p">:</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="p">{}</span>
-
-    <span class="c1"># Categorical embeddings logic</span>
-    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">cat_embed_cols</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">or</span> <span class="bp">self</span><span class="o">.</span><span class="n">quantization_setup</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">cat_embed_input</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="n">Union</span><span class="p">[</span><span class="n">Tuple</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">],</span> <span class="n">Tuple</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">,</span> <span class="nb">int</span><span class="p">]]]</span> <span class="o">=</span> <span class="p">(</span>
-            <span class="p">[]</span>
-        <span class="p">)</span>
-
-    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">cat_embed_cols</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
-        <span class="n">df_cat</span><span class="p">,</span> <span class="n">cat_embed_dim</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_prepare_categorical</span><span class="p">(</span><span class="n">df_adj</span><span class="p">)</span>
-
-        <span class="bp">self</span><span class="o">.</span><span class="n">label_encoder</span> <span class="o">=</span> <span class="n">LabelEncoder</span><span class="p">(</span>
-            <span class="n">columns_to_encode</span><span class="o">=</span><span class="n">df_cat</span><span class="o">.</span><span class="n">columns</span><span class="o">.</span><span class="n">tolist</span><span class="p">(),</span>
-            <span class="n">shared_embed</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">shared_embed</span><span class="p">,</span>
-            <span class="n">with_attention</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">with_attention</span><span class="p">,</span>
-        <span class="p">)</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">label_encoder</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">df_cat</span><span class="p">)</span>
-
-        <span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">label_encoder</span><span class="o">.</span><span class="n">encoding_dict</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
-            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">with_attention</span><span class="p">:</span>
-                <span class="bp">self</span><span class="o">.</span><span class="n">cat_embed_input</span><span class="o">.</span><span class="n">append</span><span class="p">((</span><span class="n">k</span><span class="p">,</span> <span class="nb">len</span><span class="p">(</span><span class="n">v</span><span class="p">)))</span>
-            <span class="k">else</span><span class="p">:</span>
-                <span class="bp">self</span><span class="o">.</span><span class="n">cat_embed_input</span><span class="o">.</span><span class="n">append</span><span class="p">((</span><span class="n">k</span><span class="p">,</span> <span class="nb">len</span><span class="p">(</span><span class="n">v</span><span class="p">),</span> <span class="n">cat_embed_dim</span><span class="p">[</span><span class="n">k</span><span class="p">]))</span>
-
-        <span class="bp">self</span><span class="o">.</span><span class="n">column_idx</span><span class="o">.</span><span class="n">update</span><span class="p">({</span><span class="n">k</span><span class="p">:</span> <span class="n">v</span> <span class="k">for</span> <span class="n">v</span><span class="p">,</span> <span class="n">k</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="n">df_cat</span><span class="o">.</span><span class="n">columns</span><span class="p">)})</span>
-
-    <span class="c1"># Continuous columns logic</span>
-    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">continuous_cols</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
-        <span class="n">df_cont</span><span class="p">,</span> <span class="n">cont_embed_dim</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_prepare_continuous</span><span class="p">(</span><span class="n">df_adj</span><span class="p">)</span>
-
-        <span class="c1"># Standardization logic</span>
-        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">standardize_cols</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
-            <span class="bp">self</span><span class="o">.</span><span class="n">scaler</span> <span class="o">=</span> <span class="n">StandardScaler</span><span class="p">(</span><span class="o">**</span><span class="bp">self</span><span class="o">.</span><span class="n">scale_args</span><span class="p">)</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span>
-                <span class="n">df_cont</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">standardize_cols</span><span class="p">]</span><span class="o">.</span><span class="n">values</span>
-            <span class="p">)</span>
-        <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">verbose</span><span class="p">:</span>
-            <span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span><span class="s2">&quot;Continuous columns will not be normalised&quot;</span><span class="p">)</span>
-
-        <span class="c1"># Quantization logic</span>
-        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">cols_and_bins</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
-            <span class="c1"># we do not run &#39;Quantizer.fit&#39; here since in the wild case</span>
-            <span class="c1"># someone wants standardization and quantization for the same</span>
-            <span class="c1"># columns, the Quantizer will run on the scaled data</span>
-            <span class="bp">self</span><span class="o">.</span><span class="n">quantizer</span> <span class="o">=</span> <span class="n">Quantizer</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">cols_and_bins</span><span class="p">,</span> <span class="o">**</span><span class="bp">self</span><span class="o">.</span><span class="n">quant_args</span><span class="p">)</span>
-
-            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">with_attention</span><span class="p">:</span>
-                <span class="k">for</span> <span class="n">col</span><span class="p">,</span> <span class="n">n_cat</span><span class="p">,</span> <span class="n">_</span> <span class="ow">in</span> <span class="n">cont_embed_dim</span><span class="p">:</span>
-                    <span class="bp">self</span><span class="o">.</span><span class="n">cat_embed_input</span><span class="o">.</span><span class="n">append</span><span class="p">((</span><span class="n">col</span><span class="p">,</span> <span class="n">n_cat</span><span class="p">))</span>
-            <span class="k">else</span><span class="p">:</span>
-                <span class="bp">self</span><span class="o">.</span><span class="n">cat_embed_input</span><span class="o">.</span><span class="n">extend</span><span class="p">(</span><span class="n">cont_embed_dim</span><span class="p">)</span>
-
-        <span class="bp">self</span><span class="o">.</span><span class="n">column_idx</span><span class="o">.</span><span class="n">update</span><span class="p">(</span>
-            <span class="p">{</span><span class="n">k</span><span class="p">:</span> <span class="n">v</span> <span class="o">+</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">column_idx</span><span class="p">)</span> <span class="k">for</span> <span class="n">v</span><span class="p">,</span> <span class="n">k</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="n">df_cont</span><span class="p">)}</span>
-        <span class="p">)</span>
-
-    <span class="bp">self</span><span class="o">.</span><span class="n">is_fitted</span> <span class="o">=</span> <span class="kc">True</span>
-
-    <span class="k">return</span> <span class="bp">self</span>
-</code></pre></div></td></tr></table></div>
-            </details>
-    </div>
-
-</div>
-
-<div class="doc doc-object doc-function">
-
-
-<h3 id="pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor.transform" class="doc doc-heading">
-            <span class="doc doc-object-name doc-function-name">transform</span>
-
-
-<a href="#pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor.transform" class="headerlink" title="Permanent link">&para;</a></h3>
-<div class="doc-signature highlight"><pre><span></span><code><span class="nf">transform</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
-</code></pre></div>
-
-    <div class="doc doc-contents ">
-
-        <p>Returns the processed <code>dataframe</code> as a np.ndarray</p>
-
-
-<p><span class="doc-section-title">Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>df</code></b>
-              (<code><span title="pandas.DataFrame">DataFrame</span></code>)
-          –
-          <div class="doc-md-description">
-            <p>Input pandas dataframe</p>
-          </div>
-        </li>
-    </ul>
-
-
-<p><span class="doc-section-title">Returns:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-              <code><span title="numpy.ndarray">ndarray</span></code>
-          –
-          <div class="doc-md-description">
-            <p>transformed input dataframe</p>
-          </div>
-        </li>
-    </ul>
-
-            <details class="quote">
-              <summary>Source code in <code>pytorch_widedeep/preprocessing/tab_preprocessor.py</code></summary>
-              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">376</span>
+<span class="normal">374</span>
+<span class="normal">375</span>
+<span class="normal">376</span>
 <span class="normal">377</span>
 <span class="normal">378</span>
 <span class="normal">379</span>
@@ -2965,100 +3518,12 @@ <h3 id="pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor.transfor
 <span class="normal">416</span>
 <span class="normal">417</span>
 <span class="normal">418</span>
-<span class="normal">419</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">transform</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">:</span>  <span class="c1"># noqa: C901</span>
-<span class="w">    </span><span class="sd">&quot;&quot;&quot;Returns the processed `dataframe` as a np.ndarray</span>
-
-<span class="sd">    Parameters</span>
-<span class="sd">    ----------</span>
-<span class="sd">    df: pd.DataFrame</span>
-<span class="sd">        Input pandas dataframe</span>
-
-<span class="sd">    Returns</span>
-<span class="sd">    -------</span>
-<span class="sd">    np.ndarray</span>
-<span class="sd">        transformed input dataframe</span>
-<span class="sd">    &quot;&quot;&quot;</span>
-    <span class="n">check_is_fitted</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">condition</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">is_fitted</span><span class="p">)</span>
-
-    <span class="n">df_adj</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_insert_cls_token</span><span class="p">(</span><span class="n">df</span><span class="p">)</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">with_cls_token</span> <span class="k">else</span> <span class="n">df</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
-
-    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">cat_embed_cols</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
-        <span class="n">df_cat</span> <span class="o">=</span> <span class="n">df_adj</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">cat_cols</span><span class="p">]</span>
-        <span class="n">df_cat</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">label_encoder</span><span class="o">.</span><span class="n">transform</span><span class="p">(</span><span class="n">df_cat</span><span class="p">)</span>
-    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">continuous_cols</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
-        <span class="n">df_cont</span> <span class="o">=</span> <span class="n">df_adj</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">continuous_cols</span><span class="p">]</span>
-        <span class="c1"># Standardization logic</span>
-        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">standardize_cols</span><span class="p">:</span>
-            <span class="n">df_cont</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">standardize_cols</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">scaler</span><span class="o">.</span><span class="n">transform</span><span class="p">(</span>
-                <span class="n">df_cont</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">standardize_cols</span><span class="p">]</span><span class="o">.</span><span class="n">values</span>
-            <span class="p">)</span>
-        <span class="c1"># Quantization logic</span>
-        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">cols_and_bins</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
-            <span class="c1"># Adjustment so I don&#39;t have to override the method</span>
-            <span class="c1"># in &#39;ChunkTabPreprocessor&#39;</span>
-            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">quantizer</span><span class="o">.</span><span class="n">is_fitted</span><span class="p">:</span>
-                <span class="n">df_cont</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">quantizer</span><span class="o">.</span><span class="n">transform</span><span class="p">(</span><span class="n">df_cont</span><span class="p">)</span>
-            <span class="k">else</span><span class="p">:</span>
-                <span class="n">df_cont</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">quantizer</span><span class="o">.</span><span class="n">fit_transform</span><span class="p">(</span><span class="n">df_cont</span><span class="p">)</span>
-    <span class="k">try</span><span class="p">:</span>
-        <span class="n">df_deep</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">concat</span><span class="p">([</span><span class="n">df_cat</span><span class="p">,</span> <span class="n">df_cont</span><span class="p">],</span> <span class="n">axis</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span>
-    <span class="k">except</span> <span class="ne">NameError</span><span class="p">:</span>
-        <span class="k">try</span><span class="p">:</span>
-            <span class="n">df_deep</span> <span class="o">=</span> <span class="n">df_cat</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
-        <span class="k">except</span> <span class="ne">NameError</span><span class="p">:</span>
-            <span class="n">df_deep</span> <span class="o">=</span> <span class="n">df_cont</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
-
-    <span class="k">return</span> <span class="n">df_deep</span><span class="o">.</span><span class="n">values</span>
-</code></pre></div></td></tr></table></div>
-            </details>
-    </div>
-
-</div>
-
-<div class="doc doc-object doc-function">
-
-
-<h3 id="pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor.inverse_transform" class="doc doc-heading">
-            <span class="doc doc-object-name doc-function-name">inverse_transform</span>
-
-
-<a href="#pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor.inverse_transform" class="headerlink" title="Permanent link">&para;</a></h3>
-<div class="doc-signature highlight"><pre><span></span><code><span class="nf">inverse_transform</span><span class="p">(</span><span class="n">encoded</span><span class="p">)</span>
-</code></pre></div>
-
-    <div class="doc doc-contents ">
-
-        <p>Takes as input the output from the <code>transform</code> method and it will
-return the original values.</p>
-
-
-<p><span class="doc-section-title">Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>encoded</code></b>
-              (<code><span title="numpy.ndarray">ndarray</span></code>)
-          –
-          <div class="doc-md-description">
-            <p>array with the output of the <code>transform</code> method</p>
-          </div>
-        </li>
-    </ul>
-
-
-<p><span class="doc-section-title">Returns:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-              <code><span title="pandas.DataFrame">DataFrame</span></code>
-          –
-          <div class="doc-md-description">
-            <p>Pandas dataframe with the original values</p>
-          </div>
-        </li>
-    </ul>
-
-            <details class="quote">
-              <summary>Source code in <code>pytorch_widedeep/preprocessing/tab_preprocessor.py</code></summary>
-              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">424</span>
+<span class="normal">419</span>
+<span class="normal">420</span>
+<span class="normal">421</span>
+<span class="normal">422</span>
+<span class="normal">423</span>
+<span class="normal">424</span>
 <span class="normal">425</span>
 <span class="normal">426</span>
 <span class="normal">427</span>
@@ -3097,95 +3562,9 @@ <h3 id="pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor.inverse_
 <span class="normal">460</span>
 <span class="normal">461</span>
 <span class="normal">462</span>
-<span class="normal">463</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">inverse_transform</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">encoded</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">:</span>  <span class="c1"># noqa: C901</span>
-<span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Takes as input the output from the `transform` method and it will</span>
-<span class="sd">    return the original values.</span>
-
-<span class="sd">    Parameters</span>
-<span class="sd">    ----------</span>
-<span class="sd">    encoded: np.ndarray</span>
-<span class="sd">        array with the output of the `transform` method</span>
-
-<span class="sd">    Returns</span>
-<span class="sd">    -------</span>
-<span class="sd">    pd.DataFrame</span>
-<span class="sd">        Pandas dataframe with the original values</span>
-<span class="sd">    &quot;&quot;&quot;</span>
-    <span class="n">decoded</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">(</span><span class="n">encoded</span><span class="p">,</span> <span class="n">columns</span><span class="o">=</span><span class="nb">list</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">column_idx</span><span class="o">.</span><span class="n">keys</span><span class="p">()))</span>
-    <span class="c1"># embeddings back to original category</span>
-    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">cat_embed_cols</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
-        <span class="n">decoded</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">label_encoder</span><span class="o">.</span><span class="n">inverse_transform</span><span class="p">(</span><span class="n">decoded</span><span class="p">)</span>
-    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">continuous_cols</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
-        <span class="c1"># quantized cols to the mid point</span>
-        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">cols_and_bins</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
-            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">verbose</span><span class="p">:</span>
-                <span class="nb">print</span><span class="p">(</span>
-                    <span class="s2">&quot;Note that quantized cols will be turned into the mid point of &quot;</span>
-                    <span class="s2">&quot;the corresponding bin&quot;</span>
-                <span class="p">)</span>
-            <span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">quantizer</span><span class="o">.</span><span class="n">inversed_bins</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
-                <span class="n">decoded</span><span class="p">[</span><span class="n">k</span><span class="p">]</span> <span class="o">=</span> <span class="n">decoded</span><span class="p">[</span><span class="n">k</span><span class="p">]</span><span class="o">.</span><span class="n">map</span><span class="p">(</span><span class="n">v</span><span class="p">)</span>
-        <span class="c1"># continuous_cols back to non-standarised</span>
-        <span class="k">try</span><span class="p">:</span>
-            <span class="n">decoded</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">standardize_cols</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">scaler</span><span class="o">.</span><span class="n">inverse_transform</span><span class="p">(</span>
-                <span class="n">decoded</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">standardize_cols</span><span class="p">]</span>
-            <span class="p">)</span>
-        <span class="k">except</span> <span class="ne">Exception</span><span class="p">:</span>  <span class="c1"># KeyError:</span>
-            <span class="k">pass</span>
-
-    <span class="k">if</span> <span class="s2">&quot;cls_token&quot;</span> <span class="ow">in</span> <span class="n">decoded</span><span class="o">.</span><span class="n">columns</span><span class="p">:</span>
-        <span class="n">decoded</span><span class="o">.</span><span class="n">drop</span><span class="p">(</span><span class="s2">&quot;cls_token&quot;</span><span class="p">,</span> <span class="n">axis</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">inplace</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
-
-    <span class="k">return</span> <span class="n">decoded</span>
-</code></pre></div></td></tr></table></div>
-            </details>
-    </div>
-
-</div>
-
-<div class="doc doc-object doc-function">
-
-
-<h3 id="pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor.fit_transform" class="doc doc-heading">
-            <span class="doc doc-object-name doc-function-name">fit_transform</span>
-
-
-<a href="#pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor.fit_transform" class="headerlink" title="Permanent link">&para;</a></h3>
-<div class="doc-signature highlight"><pre><span></span><code><span class="nf">fit_transform</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
-</code></pre></div>
-
-    <div class="doc doc-contents ">
-
-        <p>Combines <code>fit</code> and <code>transform</code></p>
-
-
-<p><span class="doc-section-title">Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>df</code></b>
-              (<code><span title="pandas.DataFrame">DataFrame</span></code>)
-          –
-          <div class="doc-md-description">
-            <p>Input pandas dataframe</p>
-          </div>
-        </li>
-    </ul>
-
-
-<p><span class="doc-section-title">Returns:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-              <code><span title="numpy.ndarray">ndarray</span></code>
-          –
-          <div class="doc-md-description">
-            <p>transformed input dataframe</p>
-          </div>
-        </li>
-    </ul>
-
-            <details class="quote">
-              <summary>Source code in <code>pytorch_widedeep/preprocessing/tab_preprocessor.py</code></summary>
-              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">465</span>
+<span class="normal">463</span>
+<span class="normal">464</span>
+<span class="normal">465</span>
 <span class="normal">466</span>
 <span class="normal">467</span>
 <span class="normal">468</span>
@@ -3198,91 +3577,843 @@ <h3 id="pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor.fit_tran
 <span class="normal">475</span>
 <span class="normal">476</span>
 <span class="normal">477</span>
-<span class="normal">478</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">fit_transform</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">:</span>
-<span class="w">    </span><span class="sd">&quot;&quot;&quot;Combines `fit` and `transform`</span>
+<span class="normal">478</span>
+<span class="normal">479</span>
+<span class="normal">480</span>
+<span class="normal">481</span>
+<span class="normal">482</span>
+<span class="normal">483</span>
+<span class="normal">484</span>
+<span class="normal">485</span>
+<span class="normal">486</span>
+<span class="normal">487</span>
+<span class="normal">488</span>
+<span class="normal">489</span>
+<span class="normal">490</span>
+<span class="normal">491</span>
+<span class="normal">492</span>
+<span class="normal">493</span>
+<span class="normal">494</span>
+<span class="normal">495</span>
+<span class="normal">496</span>
+<span class="normal">497</span>
+<span class="normal">498</span>
+<span class="normal">499</span>
+<span class="normal">500</span>
+<span class="normal">501</span>
+<span class="normal">502</span>
+<span class="normal">503</span>
+<span class="normal">504</span>
+<span class="normal">505</span>
+<span class="normal">506</span>
+<span class="normal">507</span>
+<span class="normal">508</span>
+<span class="normal">509</span>
+<span class="normal">510</span>
+<span class="normal">511</span>
+<span class="normal">512</span>
+<span class="normal">513</span>
+<span class="normal">514</span>
+<span class="normal">515</span>
+<span class="normal">516</span>
+<span class="normal">517</span>
+<span class="normal">518</span>
+<span class="normal">519</span>
+<span class="normal">520</span>
+<span class="normal">521</span>
+<span class="normal">522</span>
+<span class="normal">523</span>
+<span class="normal">524</span>
+<span class="normal">525</span>
+<span class="normal">526</span>
+<span class="normal">527</span>
+<span class="normal">528</span>
+<span class="normal">529</span>
+<span class="normal">530</span>
+<span class="normal">531</span>
+<span class="normal">532</span>
+<span class="normal">533</span>
+<span class="normal">534</span>
+<span class="normal">535</span>
+<span class="normal">536</span>
+<span class="normal">537</span>
+<span class="normal">538</span>
+<span class="normal">539</span>
+<span class="normal">540</span>
+<span class="normal">541</span>
+<span class="normal">542</span>
+<span class="normal">543</span>
+<span class="normal">544</span>
+<span class="normal">545</span>
+<span class="normal">546</span>
+<span class="normal">547</span>
+<span class="normal">548</span>
+<span class="normal">549</span>
+<span class="normal">550</span>
+<span class="normal">551</span>
+<span class="normal">552</span>
+<span class="normal">553</span>
+<span class="normal">554</span>
+<span class="normal">555</span>
+<span class="normal">556</span>
+<span class="normal">557</span>
+<span class="normal">558</span>
+<span class="normal">559</span>
+<span class="normal">560</span>
+<span class="normal">561</span>
+<span class="normal">562</span>
+<span class="normal">563</span>
+<span class="normal">564</span>
+<span class="normal">565</span>
+<span class="normal">566</span>
+<span class="normal">567</span>
+<span class="normal">568</span>
+<span class="normal">569</span>
+<span class="normal">570</span>
+<span class="normal">571</span>
+<span class="normal">572</span>
+<span class="normal">573</span>
+<span class="normal">574</span>
+<span class="normal">575</span>
+<span class="normal">576</span>
+<span class="normal">577</span>
+<span class="normal">578</span>
+<span class="normal">579</span>
+<span class="normal">580</span>
+<span class="normal">581</span>
+<span class="normal">582</span>
+<span class="normal">583</span>
+<span class="normal">584</span>
+<span class="normal">585</span>
+<span class="normal">586</span>
+<span class="normal">587</span>
+<span class="normal">588</span>
+<span class="normal">589</span>
+<span class="normal">590</span>
+<span class="normal">591</span>
+<span class="normal">592</span>
+<span class="normal">593</span>
+<span class="normal">594</span>
+<span class="normal">595</span>
+<span class="normal">596</span>
+<span class="normal">597</span>
+<span class="normal">598</span>
+<span class="normal">599</span>
+<span class="normal">600</span>
+<span class="normal">601</span>
+<span class="normal">602</span>
+<span class="normal">603</span>
+<span class="normal">604</span>
+<span class="normal">605</span>
+<span class="normal">606</span>
+<span class="normal">607</span>
+<span class="normal">608</span>
+<span class="normal">609</span>
+<span class="normal">610</span>
+<span class="normal">611</span>
+<span class="normal">612</span>
+<span class="normal">613</span>
+<span class="normal">614</span>
+<span class="normal">615</span>
+<span class="normal">616</span>
+<span class="normal">617</span>
+<span class="normal">618</span>
+<span class="normal">619</span>
+<span class="normal">620</span>
+<span class="normal">621</span>
+<span class="normal">622</span>
+<span class="normal">623</span>
+<span class="normal">624</span>
+<span class="normal">625</span>
+<span class="normal">626</span>
+<span class="normal">627</span>
+<span class="normal">628</span>
+<span class="normal">629</span>
+<span class="normal">630</span>
+<span class="normal">631</span>
+<span class="normal">632</span>
+<span class="normal">633</span>
+<span class="normal">634</span>
+<span class="normal">635</span>
+<span class="normal">636</span>
+<span class="normal">637</span>
+<span class="normal">638</span>
+<span class="normal">639</span>
+<span class="normal">640</span>
+<span class="normal">641</span>
+<span class="normal">642</span>
+<span class="normal">643</span>
+<span class="normal">644</span>
+<span class="normal">645</span>
+<span class="normal">646</span>
+<span class="normal">647</span>
+<span class="normal">648</span>
+<span class="normal">649</span>
+<span class="normal">650</span>
+<span class="normal">651</span>
+<span class="normal">652</span>
+<span class="normal">653</span>
+<span class="normal">654</span>
+<span class="normal">655</span>
+<span class="normal">656</span>
+<span class="normal">657</span>
+<span class="normal">658</span>
+<span class="normal">659</span>
+<span class="normal">660</span>
+<span class="normal">661</span>
+<span class="normal">662</span>
+<span class="normal">663</span>
+<span class="normal">664</span>
+<span class="normal">665</span>
+<span class="normal">666</span>
+<span class="normal">667</span>
+<span class="normal">668</span>
+<span class="normal">669</span>
+<span class="normal">670</span>
+<span class="normal">671</span>
+<span class="normal">672</span>
+<span class="normal">673</span>
+<span class="normal">674</span>
+<span class="normal">675</span>
+<span class="normal">676</span>
+<span class="normal">677</span>
+<span class="normal">678</span>
+<span class="normal">679</span>
+<span class="normal">680</span>
+<span class="normal">681</span>
+<span class="normal">682</span>
+<span class="normal">683</span>
+<span class="normal">684</span>
+<span class="normal">685</span>
+<span class="normal">686</span>
+<span class="normal">687</span>
+<span class="normal">688</span>
+<span class="normal">689</span>
+<span class="normal">690</span>
+<span class="normal">691</span>
+<span class="normal">692</span>
+<span class="normal">693</span>
+<span class="normal">694</span>
+<span class="normal">695</span>
+<span class="normal">696</span>
+<span class="normal">697</span>
+<span class="normal">698</span>
+<span class="normal">699</span>
+<span class="normal">700</span>
+<span class="normal">701</span>
+<span class="normal">702</span>
+<span class="normal">703</span>
+<span class="normal">704</span>
+<span class="normal">705</span>
+<span class="normal">706</span>
+<span class="normal">707</span>
+<span class="normal">708</span>
+<span class="normal">709</span>
+<span class="normal">710</span>
+<span class="normal">711</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">class</span> <span class="nc">TabPreprocessor</span><span class="p">(</span><span class="n">BasePreprocessor</span><span class="p">):</span>
+<span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Preprocessor to prepare the `deeptabular` component input dataset</span>
 
 <span class="sd">    Parameters</span>
 <span class="sd">    ----------</span>
-<span class="sd">    df: pd.DataFrame</span>
-<span class="sd">        Input pandas dataframe</span>
-
-<span class="sd">    Returns</span>
-<span class="sd">    -------</span>
-<span class="sd">    np.ndarray</span>
-<span class="sd">        transformed input dataframe</span>
+<span class="sd">    cat_embed_cols: List, default = None</span>
+<span class="sd">        List containing the name of the categorical columns that will be</span>
+<span class="sd">        represented by embeddings (e.g. _[&#39;education&#39;, &#39;relationship&#39;, ...]_) or</span>
+<span class="sd">        a Tuple with the name and the embedding dimension (e.g.: _[</span>
+<span class="sd">        (&#39;education&#39;,32), (&#39;relationship&#39;,16), ...]_).&lt;br/&gt; Param alias: `embed_cols`</span>
+<span class="sd">    continuous_cols: List, default = None</span>
+<span class="sd">        List with the name of the continuous cols</span>
+<span class="sd">    quantization_setup: int or Dict, default = None</span>
+<span class="sd">        Continuous columns can be turned into categorical via `pd.cut`. If</span>
+<span class="sd">        `quantization_setup` is an `int`, all continuous columns will be</span>
+<span class="sd">        quantized using this value as the number of bins. Alternatively, a</span>
+<span class="sd">        dictionary where the keys are the column names to quantize and the</span>
+<span class="sd">        values are the either integers indicating the number of bins or a</span>
+<span class="sd">        list of scalars indicating the bin edges can also be used.&lt;br/&gt;</span>
+<span class="sd">        Param alias: `cols_and_bins`</span>
+<span class="sd">    cols_to_scale: List or str, default = None,</span>
+<span class="sd">        List with the names of the columns that will be standarised via</span>
+<span class="sd">        sklearn&#39;s `StandardScaler`. It can also be the string `&#39;all&#39;` in</span>
+<span class="sd">        which case all the continuous cols will be scaled.</span>
+<span class="sd">    auto_embed_dim: bool, default = True</span>
+<span class="sd">        Boolean indicating whether the embedding dimensions will be</span>
+<span class="sd">        automatically defined via rule of thumb. See `embedding_rule`</span>
+<span class="sd">        below.</span>
+<span class="sd">    embedding_rule: str, default = &#39;fastai_new&#39;</span>
+<span class="sd">        If `auto_embed_dim=True`, this is the choice of embedding rule of</span>
+<span class="sd">        thumb. Choices are:</span>
+
+<span class="sd">        - _fastai_new_: $min(600, round(1.6 \times n_{cat}^{0.56}))$</span>
+
+<span class="sd">        - _fastai_old_: $min(50, (n_{cat}//{2})+1)$</span>
+
+<span class="sd">        - _google_: $min(600, round(n_{cat}^{0.24}))$</span>
+<span class="sd">    default_embed_dim: int, default=16</span>
+<span class="sd">        Dimension for the embeddings if the embedding dimension is not</span>
+<span class="sd">        provided in the `cat_embed_cols` parameter and `auto_embed_dim` is</span>
+<span class="sd">        set to `False`.</span>
+<span class="sd">    with_attention: bool, default = False</span>
+<span class="sd">        Boolean indicating whether the preprocessed data will be passed to an</span>
+<span class="sd">        attention-based model (more precisely a model where all embeddings</span>
+<span class="sd">        must have the same dimensions). If `True`, the param `cat_embed_cols`</span>
+<span class="sd">        must just be a list containing just the categorical column names:</span>
+<span class="sd">        e.g.</span>
+<span class="sd">        _[&#39;education&#39;, &#39;relationship&#39;, ...]_. This is because they will all be</span>
+<span class="sd">         encoded using embeddings of the same dim, which will be specified</span>
+<span class="sd">         later when the model is defined. &lt;br/&gt; Param alias:</span>
+<span class="sd">         `for_transformer`, `for_matrix_factorization`, `for_mf`</span>
+<span class="sd">    with_cls_token: bool, default = False</span>
+<span class="sd">        Boolean indicating if a `&#39;[CLS]&#39;` token will be added to the dataset</span>
+<span class="sd">        when using attention-based models. The final hidden state</span>
+<span class="sd">        corresponding to this token is used as the aggregated representation</span>
+<span class="sd">        for classification and regression tasks. If not, the categorical</span>
+<span class="sd">        and/or continuous embeddings will be concatenated before being passed</span>
+<span class="sd">        to the final MLP (if present).</span>
+<span class="sd">    shared_embed: bool, default = False</span>
+<span class="sd">        Boolean indicating if the embeddings will be &quot;shared&quot; when using</span>
+<span class="sd">        attention-based models. The idea behind `shared_embed` is</span>
+<span class="sd">        described in the Appendix A in the [TabTransformer paper](https://arxiv.org/abs/2012.06678):</span>
+<span class="sd">        _&#39;The goal of having column embedding is to enable the model to</span>
+<span class="sd">        distinguish the classes in one column from those in the other</span>
+<span class="sd">        columns&#39;_. In other words, the idea is to let the model learn which</span>
+<span class="sd">        column is embedded at the time. See: `pytorch_widedeep.models.transformers._layers.SharedEmbeddings`.</span>
+<span class="sd">    verbose: int, default = 1</span>
+<span class="sd">    scale: bool, default = False</span>
+<span class="sd">        :information_source: **note**: this arg will be removed in upcoming</span>
+<span class="sd">         releases. Please use `cols_to_scale` instead. &lt;br/&gt; Bool indicating</span>
+<span class="sd">         whether or not to scale/standarise continuous cols. It is important</span>
+<span class="sd">         to emphasize that all the DL models for tabular data in the library</span>
+<span class="sd">         also include the possibility of normalising the input continuous</span>
+<span class="sd">         features via a `BatchNorm` or a `LayerNorm`. &lt;br/&gt; Param alias:</span>
+<span class="sd">         `scale_cont_cols`.</span>
+<span class="sd">    already_standard: List, default = None</span>
+<span class="sd">        :information_source: **note**: this arg will be removed in upcoming</span>
+<span class="sd">         releases. Please use `cols_to_scale` instead. &lt;br/&gt; List with the</span>
+<span class="sd">         name of the continuous cols that do not need to be</span>
+<span class="sd">         scaled/standarised.</span>
+
+<span class="sd">    Other Parameters</span>
+<span class="sd">    ----------------</span>
+<span class="sd">    **kwargs: dict</span>
+<span class="sd">        `pd.cut` and `StandardScaler` related args</span>
+
+<span class="sd">    Attributes</span>
+<span class="sd">    ----------</span>
+<span class="sd">    cat_cols: List[str]</span>
+<span class="sd">        List containing the names of the categorical columns after processing.</span>
+<span class="sd">    cols_and_bins: Optional[Dict[str, Union[int, List[float]]]]</span>
+<span class="sd">        Dictionary containing the quantization setup after processing if</span>
+<span class="sd">        quantization is requested. This is derived from the</span>
+<span class="sd">        quantization_setup parameter.</span>
+<span class="sd">    quant_args: Dict</span>
+<span class="sd">        Dictionary containing arguments passed to pandas.cut() function for quantization.</span>
+<span class="sd">    scale_args: Dict</span>
+<span class="sd">        Dictionary containing arguments passed to StandardScaler.</span>
+<span class="sd">    label_encoder: LabelEncoder</span>
+<span class="sd">        Instance of LabelEncoder used to encode categorical variables.</span>
+<span class="sd">        See `pytorch_widedeep.utils.dense_utils.LabelEncoder`.</span>
+<span class="sd">    cat_embed_input: List</span>
+<span class="sd">        List of tuples with the column name, number of individual values for</span>
+<span class="sd">        that column and, if `with_attention` is set to `False`, the</span>
+<span class="sd">        corresponding embeddings dim, e.g. _[(&#39;education&#39;, 16, 10),</span>
+<span class="sd">        (&#39;relationship&#39;, 6, 8), ...]_.</span>
+<span class="sd">    standardize_cols: List</span>
+<span class="sd">        List of the columns that will be standardized.</span>
+<span class="sd">    scaler: StandardScaler</span>
+<span class="sd">        An instance of `sklearn.preprocessing.StandardScaler` if standardization</span>
+<span class="sd">        is requested.</span>
+<span class="sd">    column_idx: Dict</span>
+<span class="sd">        Dictionary where keys are column names and values are column indexes.</span>
+<span class="sd">        This is necessary to slice tensors.</span>
+<span class="sd">    quantizer: Quantizer</span>
+<span class="sd">        An instance of `Quantizer` if quantization is requested.</span>
+<span class="sd">    is_fitted: bool</span>
+<span class="sd">        Boolean indicating if the preprocessor has been fitted.</span>
+
+<span class="sd">    Examples</span>
+<span class="sd">    --------</span>
+<span class="sd">    &gt;&gt;&gt; import pandas as pd</span>
+<span class="sd">    &gt;&gt;&gt; import numpy as np</span>
+<span class="sd">    &gt;&gt;&gt; from pytorch_widedeep.preprocessing import TabPreprocessor</span>
+<span class="sd">    &gt;&gt;&gt; df = pd.DataFrame({&#39;color&#39;: [&#39;r&#39;, &#39;b&#39;, &#39;g&#39;], &#39;size&#39;: [&#39;s&#39;, &#39;n&#39;, &#39;l&#39;], &#39;age&#39;: [25, 40, 55]})</span>
+<span class="sd">    &gt;&gt;&gt; cat_embed_cols = [(&#39;color&#39;,5), (&#39;size&#39;,5)]</span>
+<span class="sd">    &gt;&gt;&gt; cont_cols = [&#39;age&#39;]</span>
+<span class="sd">    &gt;&gt;&gt; deep_preprocessor = TabPreprocessor(cat_embed_cols=cat_embed_cols, continuous_cols=cont_cols)</span>
+<span class="sd">    &gt;&gt;&gt; X_tab = deep_preprocessor.fit_transform(df)</span>
+<span class="sd">    &gt;&gt;&gt; deep_preprocessor.cat_embed_cols</span>
+<span class="sd">    [(&#39;color&#39;, 5), (&#39;size&#39;, 5)]</span>
+<span class="sd">    &gt;&gt;&gt; deep_preprocessor.column_idx</span>
+<span class="sd">    {&#39;color&#39;: 0, &#39;size&#39;: 1, &#39;age&#39;: 2}</span>
+<span class="sd">    &gt;&gt;&gt; cont_df = pd.DataFrame({&quot;col1&quot;: np.random.rand(10), &quot;col2&quot;: np.random.rand(10) + 1})</span>
+<span class="sd">    &gt;&gt;&gt; cont_cols = [&quot;col1&quot;, &quot;col2&quot;]</span>
+<span class="sd">    &gt;&gt;&gt; tab_preprocessor = TabPreprocessor(continuous_cols=cont_cols, quantization_setup=3)</span>
+<span class="sd">    &gt;&gt;&gt; ft_cont_df = tab_preprocessor.fit_transform(cont_df)</span>
+<span class="sd">    &gt;&gt;&gt; # or...</span>
+<span class="sd">    &gt;&gt;&gt; quantization_setup = {&#39;col1&#39;: [0., 0.4, 1.], &#39;col2&#39;: [1., 1.4, 2.]}</span>
+<span class="sd">    &gt;&gt;&gt; tab_preprocessor2 = TabPreprocessor(continuous_cols=cont_cols, quantization_setup=quantization_setup)</span>
+<span class="sd">    &gt;&gt;&gt; ft_cont_df2 = tab_preprocessor2.fit_transform(cont_df)</span>
 <span class="sd">    &quot;&quot;&quot;</span>
-    <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">df</span><span class="p">)</span><span class="o">.</span><span class="n">transform</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
-</code></pre></div></td></tr></table></div>
-            </details>
-    </div>
 
-</div>
+    <span class="nd">@alias</span><span class="p">(</span><span class="s2">&quot;with_attention&quot;</span><span class="p">,</span> <span class="p">[</span><span class="s2">&quot;for_transformer&quot;</span><span class="p">,</span> <span class="s2">&quot;for_matrix_factorization&quot;</span><span class="p">,</span> <span class="s2">&quot;for_mf&quot;</span><span class="p">])</span>
+    <span class="nd">@alias</span><span class="p">(</span><span class="s2">&quot;cat_embed_cols&quot;</span><span class="p">,</span> <span class="p">[</span><span class="s2">&quot;embed_cols&quot;</span><span class="p">])</span>
+    <span class="nd">@alias</span><span class="p">(</span><span class="s2">&quot;scale&quot;</span><span class="p">,</span> <span class="p">[</span><span class="s2">&quot;scale_cont_cols&quot;</span><span class="p">])</span>
+    <span class="nd">@alias</span><span class="p">(</span><span class="s2">&quot;quantization_setup&quot;</span><span class="p">,</span> <span class="p">[</span><span class="s2">&quot;cols_and_bins&quot;</span><span class="p">])</span>
+    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span>
+        <span class="n">cat_embed_cols</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Union</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">],</span> <span class="n">List</span><span class="p">[</span><span class="n">Tuple</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">]]]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">continuous_cols</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">quantization_setup</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span>
+            <span class="n">Union</span><span class="p">[</span><span class="nb">int</span><span class="p">,</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Union</span><span class="p">[</span><span class="nb">int</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="nb">float</span><span class="p">]]]]</span>
+        <span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">cols_to_scale</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Union</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">],</span> <span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">auto_embed_dim</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">,</span>
+        <span class="n">embedding_rule</span><span class="p">:</span> <span class="n">Literal</span><span class="p">[</span><span class="s2">&quot;google&quot;</span><span class="p">,</span> <span class="s2">&quot;fastai_old&quot;</span><span class="p">,</span> <span class="s2">&quot;fastai_new&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="s2">&quot;fastai_new&quot;</span><span class="p">,</span>
+        <span class="n">default_embed_dim</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">16</span><span class="p">,</span>
+        <span class="n">with_attention</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
+        <span class="n">with_cls_token</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
+        <span class="n">shared_embed</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
+        <span class="n">verbose</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1</span><span class="p">,</span>
+        <span class="o">*</span><span class="p">,</span>
+        <span class="n">scale</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
+        <span class="n">already_standard</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="o">**</span><span class="n">kwargs</span><span class="p">,</span>
+    <span class="p">):</span>
+        <span class="nb">super</span><span class="p">(</span><span class="n">TabPreprocessor</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">continuous_cols</span> <span class="o">=</span> <span class="n">continuous_cols</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">quantization_setup</span> <span class="o">=</span> <span class="n">quantization_setup</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">cols_to_scale</span> <span class="o">=</span> <span class="n">cols_to_scale</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">scale</span> <span class="o">=</span> <span class="n">scale</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">already_standard</span> <span class="o">=</span> <span class="n">already_standard</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">auto_embed_dim</span> <span class="o">=</span> <span class="n">auto_embed_dim</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">embedding_rule</span> <span class="o">=</span> <span class="n">embedding_rule</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">default_embed_dim</span> <span class="o">=</span> <span class="n">default_embed_dim</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">with_attention</span> <span class="o">=</span> <span class="n">with_attention</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">with_cls_token</span> <span class="o">=</span> <span class="n">with_cls_token</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">shared_embed</span> <span class="o">=</span> <span class="n">shared_embed</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">verbose</span> <span class="o">=</span> <span class="n">verbose</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">quant_args</span> <span class="o">=</span> <span class="p">{</span>
+            <span class="n">k</span><span class="p">:</span> <span class="n">v</span> <span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="n">kwargs</span><span class="o">.</span><span class="n">items</span><span class="p">()</span> <span class="k">if</span> <span class="n">k</span> <span class="ow">in</span> <span class="n">pd</span><span class="o">.</span><span class="n">cut</span><span class="o">.</span><span class="vm">__code__</span><span class="o">.</span><span class="n">co_varnames</span>
+        <span class="p">}</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">scale_args</span> <span class="o">=</span> <span class="p">{</span>
+            <span class="n">k</span><span class="p">:</span> <span class="n">v</span> <span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="n">kwargs</span><span class="o">.</span><span class="n">items</span><span class="p">()</span> <span class="k">if</span> <span class="n">k</span> <span class="ow">in</span> <span class="n">StandardScaler</span><span class="p">()</span><span class="o">.</span><span class="n">get_params</span><span class="p">()</span>
+        <span class="p">}</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">_check_inputs</span><span class="p">(</span><span class="n">cat_embed_cols</span><span class="p">)</span>
+
+        <span class="k">if</span> <span class="n">with_cls_token</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">cat_embed_cols</span> <span class="o">=</span> <span class="p">(</span>
+                <span class="p">[</span><span class="s2">&quot;cls_token&quot;</span><span class="p">]</span> <span class="o">+</span> <span class="n">cat_embed_cols</span>  <span class="c1"># type: ignore[operator]</span>
+                <span class="k">if</span> <span class="n">cat_embed_cols</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span>
+                <span class="k">else</span> <span class="p">[</span><span class="s2">&quot;cls_token&quot;</span><span class="p">]</span>
+            <span class="p">)</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">cat_embed_cols</span> <span class="o">=</span> <span class="n">cat_embed_cols</span>  <span class="c1"># type: ignore[assignment]</span>
 
+        <span class="bp">self</span><span class="o">.</span><span class="n">is_fitted</span> <span class="o">=</span> <span class="kc">False</span>
 
+    <span class="k">def</span> <span class="nf">fit</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">BasePreprocessor</span><span class="p">:</span>  <span class="c1"># noqa: C901</span>
+<span class="w">        </span><span class="sd">&quot;&quot;&quot;Fits the Preprocessor and creates required attributes</span>
 
-  </div>
+<span class="sd">        Parameters</span>
+<span class="sd">        ----------</span>
+<span class="sd">        df: pd.DataFrame</span>
+<span class="sd">            Input pandas dataframe</span>
 
-    </div>
+<span class="sd">        Returns</span>
+<span class="sd">        -------</span>
+<span class="sd">        TabPreprocessor</span>
+<span class="sd">            `TabPreprocessor` fitted object</span>
+<span class="sd">        &quot;&quot;&quot;</span>
 
-</div>
+        <span class="n">df_adj</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_insert_cls_token</span><span class="p">(</span><span class="n">df</span><span class="p">)</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">with_cls_token</span> <span class="k">else</span> <span class="n">df</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
 
-<div class="doc doc-object doc-class">
+        <span class="bp">self</span><span class="o">.</span><span class="n">column_idx</span><span class="p">:</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="p">{}</span>
 
+        <span class="c1"># Categorical embeddings logic</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">cat_embed_cols</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">or</span> <span class="bp">self</span><span class="o">.</span><span class="n">quantization_setup</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">cat_embed_input</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="n">Union</span><span class="p">[</span><span class="n">Tuple</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">],</span> <span class="n">Tuple</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">,</span> <span class="nb">int</span><span class="p">]]]</span> <span class="o">=</span> <span class="p">(</span>
+                <span class="p">[]</span>
+            <span class="p">)</span>
 
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">cat_embed_cols</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="n">df_cat</span><span class="p">,</span> <span class="n">cat_embed_dim</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_prepare_categorical</span><span class="p">(</span><span class="n">df_adj</span><span class="p">)</span>
 
-<h2 id="pytorch_widedeep.preprocessing.tab_preprocessor.Quantizer" class="doc doc-heading">
-            <span class="doc doc-object-name doc-class-name">Quantizer</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">label_encoder</span> <span class="o">=</span> <span class="n">LabelEncoder</span><span class="p">(</span>
+                <span class="n">columns_to_encode</span><span class="o">=</span><span class="n">df_cat</span><span class="o">.</span><span class="n">columns</span><span class="o">.</span><span class="n">tolist</span><span class="p">(),</span>
+                <span class="n">shared_embed</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">shared_embed</span><span class="p">,</span>
+                <span class="n">with_attention</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">with_attention</span><span class="p">,</span>
+            <span class="p">)</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">label_encoder</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">df_cat</span><span class="p">)</span>
 
+            <span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">label_encoder</span><span class="o">.</span><span class="n">encoding_dict</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
+                <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">with_attention</span><span class="p">:</span>
+                    <span class="bp">self</span><span class="o">.</span><span class="n">cat_embed_input</span><span class="o">.</span><span class="n">append</span><span class="p">((</span><span class="n">k</span><span class="p">,</span> <span class="nb">len</span><span class="p">(</span><span class="n">v</span><span class="p">)))</span>
+                <span class="k">else</span><span class="p">:</span>
+                    <span class="bp">self</span><span class="o">.</span><span class="n">cat_embed_input</span><span class="o">.</span><span class="n">append</span><span class="p">((</span><span class="n">k</span><span class="p">,</span> <span class="nb">len</span><span class="p">(</span><span class="n">v</span><span class="p">),</span> <span class="n">cat_embed_dim</span><span class="p">[</span><span class="n">k</span><span class="p">]))</span>
 
-<a href="#pytorch_widedeep.preprocessing.tab_preprocessor.Quantizer" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="nf">Quantizer</span><span class="p">(</span><span class="n">quantization_setup</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
-</code></pre></div>
+            <span class="bp">self</span><span class="o">.</span><span class="n">column_idx</span><span class="o">.</span><span class="n">update</span><span class="p">({</span><span class="n">k</span><span class="p">:</span> <span class="n">v</span> <span class="k">for</span> <span class="n">v</span><span class="p">,</span> <span class="n">k</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="n">df_cat</span><span class="o">.</span><span class="n">columns</span><span class="p">)})</span>
 
-    <div class="doc doc-contents first">
+        <span class="c1"># Continuous columns logic</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">continuous_cols</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="n">df_cont</span><span class="p">,</span> <span class="n">cont_embed_dim</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_prepare_continuous</span><span class="p">(</span><span class="n">df_adj</span><span class="p">)</span>
 
+            <span class="c1"># Standardization logic</span>
+            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">standardize_cols</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">scaler</span> <span class="o">=</span> <span class="n">StandardScaler</span><span class="p">(</span><span class="o">**</span><span class="bp">self</span><span class="o">.</span><span class="n">scale_args</span><span class="p">)</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span>
+                    <span class="n">df_cont</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">standardize_cols</span><span class="p">]</span><span class="o">.</span><span class="n">values</span>
+                <span class="p">)</span>
+            <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">verbose</span><span class="p">:</span>
+                <span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span><span class="s2">&quot;Continuous columns will not be normalised&quot;</span><span class="p">)</span>
+
+            <span class="c1"># Quantization logic</span>
+            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">cols_and_bins</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+                <span class="c1"># we do not run &#39;Quantizer.fit&#39; here since in the wild case</span>
+                <span class="c1"># someone wants standardization and quantization for the same</span>
+                <span class="c1"># columns, the Quantizer will run on the scaled data</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">quantizer</span> <span class="o">=</span> <span class="n">Quantizer</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">cols_and_bins</span><span class="p">,</span> <span class="o">**</span><span class="bp">self</span><span class="o">.</span><span class="n">quant_args</span><span class="p">)</span>
+
+                <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">with_attention</span><span class="p">:</span>
+                    <span class="k">for</span> <span class="n">col</span><span class="p">,</span> <span class="n">n_cat</span><span class="p">,</span> <span class="n">_</span> <span class="ow">in</span> <span class="n">cont_embed_dim</span><span class="p">:</span>
+                        <span class="bp">self</span><span class="o">.</span><span class="n">cat_embed_input</span><span class="o">.</span><span class="n">append</span><span class="p">((</span><span class="n">col</span><span class="p">,</span> <span class="n">n_cat</span><span class="p">))</span>
+                <span class="k">else</span><span class="p">:</span>
+                    <span class="bp">self</span><span class="o">.</span><span class="n">cat_embed_input</span><span class="o">.</span><span class="n">extend</span><span class="p">(</span><span class="n">cont_embed_dim</span><span class="p">)</span>
+
+            <span class="bp">self</span><span class="o">.</span><span class="n">column_idx</span><span class="o">.</span><span class="n">update</span><span class="p">(</span>
+                <span class="p">{</span><span class="n">k</span><span class="p">:</span> <span class="n">v</span> <span class="o">+</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">column_idx</span><span class="p">)</span> <span class="k">for</span> <span class="n">v</span><span class="p">,</span> <span class="n">k</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="n">df_cont</span><span class="p">)}</span>
+            <span class="p">)</span>
 
-        <p>Helper class to perform the quantization of continuous columns. It is
-included in this docs for completion, since depending on the value of the
-parameter <code>'quantization_setup'</code> of the <code>TabPreprocessor</code> class, that
-class might have an attribute of type <code>Quantizer</code>. However, this class is
-designed to always run internally within the <code>TabPreprocessor</code> class.</p>
+        <span class="bp">self</span><span class="o">.</span><span class="n">is_fitted</span> <span class="o">=</span> <span class="kc">True</span>
+
+        <span class="k">return</span> <span class="bp">self</span>
+
+    <span class="k">def</span> <span class="nf">transform</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">:</span>  <span class="c1"># noqa: C901</span>
+<span class="w">        </span><span class="sd">&quot;&quot;&quot;Returns the processed `dataframe` as a np.ndarray</span>
+
+<span class="sd">        Parameters</span>
+<span class="sd">        ----------</span>
+<span class="sd">        df: pd.DataFrame</span>
+<span class="sd">            Input pandas dataframe</span>
+
+<span class="sd">        Returns</span>
+<span class="sd">        -------</span>
+<span class="sd">        np.ndarray</span>
+<span class="sd">            transformed input dataframe</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">check_is_fitted</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">condition</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">is_fitted</span><span class="p">)</span>
+
+        <span class="n">df_adj</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_insert_cls_token</span><span class="p">(</span><span class="n">df</span><span class="p">)</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">with_cls_token</span> <span class="k">else</span> <span class="n">df</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
+
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">cat_embed_cols</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="n">df_cat</span> <span class="o">=</span> <span class="n">df_adj</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">cat_cols</span><span class="p">]</span>
+            <span class="n">df_cat</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">label_encoder</span><span class="o">.</span><span class="n">transform</span><span class="p">(</span><span class="n">df_cat</span><span class="p">)</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">continuous_cols</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="n">df_cont</span> <span class="o">=</span> <span class="n">df_adj</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">continuous_cols</span><span class="p">]</span>
+            <span class="c1"># Standardization logic</span>
+            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">standardize_cols</span><span class="p">:</span>
+                <span class="n">df_cont</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">standardize_cols</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">scaler</span><span class="o">.</span><span class="n">transform</span><span class="p">(</span>
+                    <span class="n">df_cont</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">standardize_cols</span><span class="p">]</span><span class="o">.</span><span class="n">values</span>
+                <span class="p">)</span>
+            <span class="c1"># Quantization logic</span>
+            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">cols_and_bins</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+                <span class="c1"># Adjustment so I don&#39;t have to override the method</span>
+                <span class="c1"># in &#39;ChunkTabPreprocessor&#39;</span>
+                <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">quantizer</span><span class="o">.</span><span class="n">is_fitted</span><span class="p">:</span>
+                    <span class="n">df_cont</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">quantizer</span><span class="o">.</span><span class="n">transform</span><span class="p">(</span><span class="n">df_cont</span><span class="p">)</span>
+                <span class="k">else</span><span class="p">:</span>
+                    <span class="n">df_cont</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">quantizer</span><span class="o">.</span><span class="n">fit_transform</span><span class="p">(</span><span class="n">df_cont</span><span class="p">)</span>
+        <span class="k">try</span><span class="p">:</span>
+            <span class="n">df_deep</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">concat</span><span class="p">([</span><span class="n">df_cat</span><span class="p">,</span> <span class="n">df_cont</span><span class="p">],</span> <span class="n">axis</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span>
+        <span class="k">except</span> <span class="ne">NameError</span><span class="p">:</span>
+            <span class="k">try</span><span class="p">:</span>
+                <span class="n">df_deep</span> <span class="o">=</span> <span class="n">df_cat</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
+            <span class="k">except</span> <span class="ne">NameError</span><span class="p">:</span>
+                <span class="n">df_deep</span> <span class="o">=</span> <span class="n">df_cont</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
+
+        <span class="k">return</span> <span class="n">df_deep</span><span class="o">.</span><span class="n">values</span>
+
+    <span class="k">def</span> <span class="nf">transform_sample</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">:</span>
+        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">transform</span><span class="p">(</span><span class="n">df</span><span class="p">)</span><span class="o">.</span><span class="n">astype</span><span class="p">(</span><span class="s2">&quot;float&quot;</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span>
+
+    <span class="k">def</span> <span class="nf">inverse_transform</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">encoded</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">:</span>  <span class="c1"># noqa: C901</span>
+<span class="w">        </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Takes as input the output from the `transform` method and it will</span>
+<span class="sd">        return the original values.</span>
+
+<span class="sd">        Parameters</span>
+<span class="sd">        ----------</span>
+<span class="sd">        encoded: np.ndarray</span>
+<span class="sd">            array with the output of the `transform` method</span>
+
+<span class="sd">        Returns</span>
+<span class="sd">        -------</span>
+<span class="sd">        pd.DataFrame</span>
+<span class="sd">            Pandas dataframe with the original values</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">decoded</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">(</span><span class="n">encoded</span><span class="p">,</span> <span class="n">columns</span><span class="o">=</span><span class="nb">list</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">column_idx</span><span class="o">.</span><span class="n">keys</span><span class="p">()))</span>
+        <span class="c1"># embeddings back to original category</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">cat_embed_cols</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="n">decoded</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">label_encoder</span><span class="o">.</span><span class="n">inverse_transform</span><span class="p">(</span><span class="n">decoded</span><span class="p">)</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">continuous_cols</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="c1"># quantized cols to the mid point</span>
+            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">cols_and_bins</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+                <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">verbose</span><span class="p">:</span>
+                    <span class="nb">print</span><span class="p">(</span>
+                        <span class="s2">&quot;Note that quantized cols will be turned into the mid point of &quot;</span>
+                        <span class="s2">&quot;the corresponding bin&quot;</span>
+                    <span class="p">)</span>
+                <span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">quantizer</span><span class="o">.</span><span class="n">inversed_bins</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
+                    <span class="n">decoded</span><span class="p">[</span><span class="n">k</span><span class="p">]</span> <span class="o">=</span> <span class="n">decoded</span><span class="p">[</span><span class="n">k</span><span class="p">]</span><span class="o">.</span><span class="n">map</span><span class="p">(</span><span class="n">v</span><span class="p">)</span>
+            <span class="c1"># continuous_cols back to non-standarised</span>
+            <span class="k">try</span><span class="p">:</span>
+                <span class="n">decoded</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">standardize_cols</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">scaler</span><span class="o">.</span><span class="n">inverse_transform</span><span class="p">(</span>
+                    <span class="n">decoded</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">standardize_cols</span><span class="p">]</span>
+                <span class="p">)</span>
+            <span class="k">except</span> <span class="ne">Exception</span><span class="p">:</span>  <span class="c1"># KeyError:</span>
+                <span class="k">pass</span>
+
+        <span class="k">if</span> <span class="s2">&quot;cls_token&quot;</span> <span class="ow">in</span> <span class="n">decoded</span><span class="o">.</span><span class="n">columns</span><span class="p">:</span>
+            <span class="n">decoded</span><span class="o">.</span><span class="n">drop</span><span class="p">(</span><span class="s2">&quot;cls_token&quot;</span><span class="p">,</span> <span class="n">axis</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">inplace</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
+
+        <span class="k">return</span> <span class="n">decoded</span>
+
+    <span class="k">def</span> <span class="nf">fit_transform</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">:</span>
+<span class="w">        </span><span class="sd">&quot;&quot;&quot;Combines `fit` and `transform`</span>
+
+<span class="sd">        Parameters</span>
+<span class="sd">        ----------</span>
+<span class="sd">        df: pd.DataFrame</span>
+<span class="sd">            Input pandas dataframe</span>
+
+<span class="sd">        Returns</span>
+<span class="sd">        -------</span>
+<span class="sd">        np.ndarray</span>
+<span class="sd">            transformed input dataframe</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">df</span><span class="p">)</span><span class="o">.</span><span class="n">transform</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">_insert_cls_token</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">:</span>
+        <span class="n">df_cls</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
+        <span class="n">df_cls</span><span class="o">.</span><span class="n">insert</span><span class="p">(</span><span class="n">loc</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span> <span class="n">column</span><span class="o">=</span><span class="s2">&quot;cls_token&quot;</span><span class="p">,</span> <span class="n">value</span><span class="o">=</span><span class="s2">&quot;[CLS]&quot;</span><span class="p">)</span>
+        <span class="k">return</span> <span class="n">df_cls</span>
+
+    <span class="k">def</span> <span class="nf">_prepare_categorical</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span>
+    <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tuple</span><span class="p">[</span><span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">,</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">]]:</span>
+        <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">cat_embed_cols</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="nb">tuple</span><span class="p">):</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">cat_cols</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[</span><span class="n">emb</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="k">for</span> <span class="n">emb</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">cat_embed_cols</span><span class="p">]</span>
+            <span class="n">cat_embed_dim</span><span class="p">:</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">cat_embed_cols</span><span class="p">)</span>  <span class="c1"># type: ignore</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">cat_cols</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">cat_embed_cols</span>  <span class="c1"># type: ignore[assignment]</span>
+            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">auto_embed_dim</span><span class="p">:</span>
+                <span class="k">assert</span> <span class="nb">isinstance</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">cat_embed_cols</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="nb">str</span><span class="p">),</span> <span class="p">(</span>
+                    <span class="s2">&quot;If &#39;auto_embed_dim&#39; is &#39;True&#39; and &#39;with_attention&#39; is &#39;False&#39;, &quot;</span>
+                    <span class="s2">&quot;&#39;cat_embed_cols&#39; must be a list of strings with the columns to &quot;</span>
+                    <span class="s2">&quot;be encoded as embeddings.&quot;</span>
+                <span class="p">)</span>
+                <span class="n">n_cats</span><span class="p">:</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="p">{</span><span class="n">col</span><span class="p">:</span> <span class="n">df</span><span class="p">[</span><span class="n">col</span><span class="p">]</span><span class="o">.</span><span class="n">nunique</span><span class="p">()</span> <span class="k">for</span> <span class="n">col</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">cat_embed_cols</span><span class="p">}</span>  <span class="c1"># type: ignore[misc]</span>
+                <span class="n">cat_embed_dim</span> <span class="o">=</span> <span class="p">{</span>
+                    <span class="n">col</span><span class="p">:</span> <span class="n">embed_sz_rule</span><span class="p">(</span><span class="n">n_cat</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">embedding_rule</span><span class="p">)</span>  <span class="c1"># type: ignore[misc]</span>
+                    <span class="k">for</span> <span class="n">col</span><span class="p">,</span> <span class="n">n_cat</span> <span class="ow">in</span> <span class="n">n_cats</span><span class="o">.</span><span class="n">items</span><span class="p">()</span>
+                <span class="p">}</span>
+            <span class="k">else</span><span class="p">:</span>
+                <span class="n">cat_embed_dim</span> <span class="o">=</span> <span class="p">{</span>
+                    <span class="n">e</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">default_embed_dim</span> <span class="k">for</span> <span class="n">e</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">cat_embed_cols</span>  <span class="c1"># type: ignore[misc]</span>
+                <span class="p">}</span>  <span class="c1"># type: ignore</span>
+        <span class="k">return</span> <span class="n">df</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">cat_cols</span><span class="p">],</span> <span class="n">cat_embed_dim</span>
+
+    <span class="k">def</span> <span class="nf">_prepare_continuous</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span>
+    <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tuple</span><span class="p">[</span><span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">,</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="n">Tuple</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">,</span> <span class="nb">int</span><span class="p">]]]]:</span>
+        <span class="c1"># Standardization logic</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">cols_to_scale</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">standardize_cols</span> <span class="o">=</span> <span class="p">(</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">cols_to_scale</span>
+                <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">cols_to_scale</span> <span class="o">!=</span> <span class="s2">&quot;all&quot;</span>
+                <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">continuous_cols</span>
+            <span class="p">)</span>
+        <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">scale</span><span class="p">:</span>
+            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">already_standard</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">standardize_cols</span> <span class="o">=</span> <span class="p">[</span>
+                    <span class="n">c</span> <span class="k">for</span> <span class="n">c</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">continuous_cols</span> <span class="k">if</span> <span class="n">c</span> <span class="ow">not</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">already_standard</span>
+                <span class="p">]</span>
+            <span class="k">else</span><span class="p">:</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">standardize_cols</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">continuous_cols</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">standardize_cols</span> <span class="o">=</span> <span class="kc">None</span>
 
+        <span class="c1"># Quantization logic</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">quantization_setup</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="c1"># the quantized columns are then treated as categorical</span>
+            <span class="n">quant_cont_embed_input</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="n">Tuple</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">,</span> <span class="nb">int</span><span class="p">]]]</span> <span class="o">=</span> <span class="p">[]</span>
+            <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">quantization_setup</span><span class="p">,</span> <span class="nb">int</span><span class="p">):</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">cols_and_bins</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Union</span><span class="p">[</span><span class="nb">int</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="nb">float</span><span class="p">]]]]</span> <span class="o">=</span> <span class="p">{}</span>
+                <span class="k">for</span> <span class="n">col</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">continuous_cols</span><span class="p">:</span>
+                    <span class="bp">self</span><span class="o">.</span><span class="n">cols_and_bins</span><span class="p">[</span><span class="n">col</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">quantization_setup</span>
+                    <span class="n">quant_cont_embed_input</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
+                        <span class="p">(</span>
+                            <span class="n">col</span><span class="p">,</span>
+                            <span class="bp">self</span><span class="o">.</span><span class="n">quantization_setup</span><span class="p">,</span>
+                            <span class="p">(</span>
+                                <span class="n">embed_sz_rule</span><span class="p">(</span>
+                                    <span class="bp">self</span><span class="o">.</span><span class="n">quantization_setup</span> <span class="o">+</span> <span class="mi">1</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">embedding_rule</span>  <span class="c1"># type: ignore[arg-type]</span>
+                                <span class="p">)</span>
+                                <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">auto_embed_dim</span>
+                                <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">default_embed_dim</span>
+                            <span class="p">),</span>
+                        <span class="p">)</span>
+                    <span class="p">)</span>
+            <span class="k">else</span><span class="p">:</span>
+                <span class="k">for</span> <span class="n">col</span><span class="p">,</span> <span class="n">val</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">quantization_setup</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
+                    <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">val</span><span class="p">,</span> <span class="nb">int</span><span class="p">):</span>
+                        <span class="n">quant_cont_embed_input</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
+                            <span class="p">(</span>
+                                <span class="n">col</span><span class="p">,</span>
+                                <span class="n">val</span><span class="p">,</span>
+                                <span class="p">(</span>
+                                    <span class="n">embed_sz_rule</span><span class="p">(</span>
+                                        <span class="n">val</span> <span class="o">+</span> <span class="mi">1</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">embedding_rule</span>  <span class="c1"># type: ignore[arg-type]</span>
+                                    <span class="p">)</span>
+                                    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">auto_embed_dim</span>
+                                    <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">default_embed_dim</span>
+                                <span class="p">),</span>
+                            <span class="p">)</span>
+                        <span class="p">)</span>
+                    <span class="k">else</span><span class="p">:</span>
+                        <span class="n">quant_cont_embed_input</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
+                            <span class="p">(</span>
+                                <span class="n">col</span><span class="p">,</span>
+                                <span class="nb">len</span><span class="p">(</span><span class="n">val</span><span class="p">)</span> <span class="o">-</span> <span class="mi">1</span><span class="p">,</span>
+                                <span class="p">(</span>
+                                    <span class="n">embed_sz_rule</span><span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">val</span><span class="p">),</span> <span class="bp">self</span><span class="o">.</span><span class="n">embedding_rule</span><span class="p">)</span>  <span class="c1"># type: ignore[arg-type]</span>
+                                    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">auto_embed_dim</span>
+                                    <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">default_embed_dim</span>
+                                <span class="p">),</span>
+                            <span class="p">)</span>
+                        <span class="p">)</span>
+
+                <span class="bp">self</span><span class="o">.</span><span class="n">cols_and_bins</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">quantization_setup</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">cols_and_bins</span> <span class="o">=</span> <span class="kc">None</span>
+            <span class="n">quant_cont_embed_input</span> <span class="o">=</span> <span class="kc">None</span>
+
+        <span class="k">return</span> <span class="n">df</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">continuous_cols</span><span class="p">],</span> <span class="n">quant_cont_embed_input</span>
+
+    <span class="k">def</span> <span class="nf">_check_inputs</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">cat_embed_cols</span><span class="p">):</span>  <span class="c1"># noqa: C901</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">scale</span> <span class="ow">or</span> <span class="bp">self</span><span class="o">.</span><span class="n">already_standard</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span>
+                <span class="s2">&quot;&#39;scale&#39; and &#39;already_standard&#39; will be deprecated in the next release. &quot;</span>
+                <span class="s2">&quot;Please use &#39;cols_to_scale&#39; instead&quot;</span><span class="p">,</span>
+                <span class="ne">DeprecationWarning</span><span class="p">,</span>
+                <span class="n">stacklevel</span><span class="o">=</span><span class="mi">2</span><span class="p">,</span>
+            <span class="p">)</span>
 
-<p><span class="doc-section-title">Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>quantization_setup</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, <span title="pytorch_widedeep.wdtypes.Union">Union</span>[int, <span title="pytorch_widedeep.wdtypes.List">List</span>[float]]]</code>)
-          –
-          <div class="doc-md-description">
-            <p>Dictionary where the keys are the column names to quantize and the
-values are the either integers indicating the number of bins or a
-list of scalars indicating the bin edges.</p>
-          </div>
-        </li>
-    </ul>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">scale</span><span class="p">:</span>
+            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">already_standard</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+                <span class="n">standardize_cols</span> <span class="o">=</span> <span class="p">[</span>
+                    <span class="n">c</span> <span class="k">for</span> <span class="n">c</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">continuous_cols</span> <span class="k">if</span> <span class="n">c</span> <span class="ow">not</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">already_standard</span>
+                <span class="p">]</span>
+            <span class="k">else</span><span class="p">:</span>
+                <span class="n">standardize_cols</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">continuous_cols</span>
+        <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">cols_to_scale</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="n">standardize_cols</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">cols_to_scale</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="n">standardize_cols</span> <span class="o">=</span> <span class="kc">None</span>
+
+        <span class="k">if</span> <span class="n">standardize_cols</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">quantization_setup</span><span class="p">,</span> <span class="nb">int</span><span class="p">):</span>
+                <span class="n">cols_to_quantize_and_standardize</span> <span class="o">=</span> <span class="p">[</span>
+                    <span class="n">c</span> <span class="k">for</span> <span class="n">c</span> <span class="ow">in</span> <span class="n">standardize_cols</span> <span class="k">if</span> <span class="n">c</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">continuous_cols</span>
+                <span class="p">]</span>
+            <span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">quantization_setup</span><span class="p">,</span> <span class="nb">dict</span><span class="p">):</span>
+                <span class="n">cols_to_quantize_and_standardize</span> <span class="o">=</span> <span class="p">[</span>
+                    <span class="n">c</span> <span class="k">for</span> <span class="n">c</span> <span class="ow">in</span> <span class="n">standardize_cols</span> <span class="k">if</span> <span class="n">c</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">quantization_setup</span>
+                <span class="p">]</span>
+            <span class="k">else</span><span class="p">:</span>
+                <span class="n">cols_to_quantize_and_standardize</span> <span class="o">=</span> <span class="kc">None</span>
+            <span class="k">if</span> <span class="n">cols_to_quantize_and_standardize</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+                <span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span>
+                    <span class="sa">f</span><span class="s2">&quot;the following columns: </span><span class="si">{</span><span class="n">cols_to_quantize_and_standardize</span><span class="si">}</span><span class="s2"> will be first scaled&quot;</span>
+                    <span class="s2">&quot; using a StandardScaler and then quantized. Make sure this is what you really want&quot;</span>
+                <span class="p">)</span>
+
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">with_cls_token</span> <span class="ow">and</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">with_attention</span><span class="p">:</span>
+            <span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span>
+                <span class="s2">&quot;If &#39;with_cls_token&#39; is set to &#39;True&#39;, &#39;with_attention&#39; will be automatically &quot;</span>
+                <span class="s2">&quot;to &#39;True&#39; if is &#39;False&#39;&quot;</span><span class="p">,</span>
+                <span class="ne">UserWarning</span><span class="p">,</span>
+            <span class="p">)</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">with_attention</span> <span class="o">=</span> <span class="kc">True</span>
+
+        <span class="k">if</span> <span class="p">(</span><span class="n">cat_embed_cols</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">)</span> <span class="ow">and</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">continuous_cols</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">):</span>
+            <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
+                <span class="s2">&quot;&#39;cat_embed_cols&#39; and &#39;continuous_cols&#39; are &#39;None&#39;. Please, define at least one of the two.&quot;</span>
+            <span class="p">)</span>
 
-                  <details class="quote">
-                    <summary>Source code in <code>pytorch_widedeep/preprocessing/tab_preprocessor.py</code></summary>
-                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">54</span>
-<span class="normal">55</span>
-<span class="normal">56</span>
-<span class="normal">57</span>
-<span class="normal">58</span>
-<span class="normal">59</span>
-<span class="normal">60</span>
-<span class="normal">61</span>
-<span class="normal">62</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
-    <span class="bp">self</span><span class="p">,</span>
-    <span class="n">quantization_setup</span><span class="p">:</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Union</span><span class="p">[</span><span class="nb">int</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="nb">float</span><span class="p">]]],</span>
-    <span class="o">**</span><span class="n">kwargs</span><span class="p">,</span>
-<span class="p">):</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">quantization_setup</span> <span class="o">=</span> <span class="n">quantization_setup</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">quant_args</span> <span class="o">=</span> <span class="n">kwargs</span>
-
-    <span class="bp">self</span><span class="o">.</span><span class="n">is_fitted</span> <span class="o">=</span> <span class="kc">False</span>
+        <span class="k">if</span> <span class="p">(</span>
+            <span class="n">cat_embed_cols</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span>
+            <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">continuous_cols</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span>
+            <span class="ow">and</span> <span class="nb">len</span><span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">intersect1d</span><span class="p">(</span><span class="n">cat_embed_cols</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">continuous_cols</span><span class="p">))</span> <span class="o">&gt;</span> <span class="mi">0</span>
+        <span class="p">):</span>
+            <span class="n">overlapping_cols</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span>
+                <span class="n">np</span><span class="o">.</span><span class="n">intersect1d</span><span class="p">(</span><span class="n">cat_embed_cols</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">continuous_cols</span><span class="p">)</span>
+            <span class="p">)</span>
+            <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
+                <span class="s2">&quot;Currently passing columns as both categorical and continuum is not supported.&quot;</span>
+                <span class="s2">&quot; Please, choose one or the other for the following columns: </span><span class="si">{}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
+                    <span class="s2">&quot;, &quot;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">overlapping_cols</span><span class="p">)</span>
+                <span class="p">)</span>
+            <span class="p">)</span>
+        <span class="n">transformer_error_message</span> <span class="o">=</span> <span class="p">(</span>
+            <span class="s2">&quot;If with_attention is &#39;True&#39; cat_embed_cols must be a list &quot;</span>
+            <span class="s2">&quot; of strings with the columns to be encoded as embeddings.&quot;</span>
+        <span class="p">)</span>
+        <span class="k">if</span> <span class="p">(</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">with_attention</span>
+            <span class="ow">and</span> <span class="n">cat_embed_cols</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span>
+            <span class="ow">and</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">cat_embed_cols</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="nb">tuple</span><span class="p">)</span>
+        <span class="p">):</span>
+            <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="n">transformer_error_message</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="fm">__repr__</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">str</span><span class="p">:</span>  <span class="c1"># noqa: C901</span>
+        <span class="n">list_of_params</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">cat_embed_cols</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="n">list_of_params</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">&quot;cat_embed_cols=</span><span class="si">{cat_embed_cols}</span><span class="s2">&quot;</span><span class="p">)</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">continuous_cols</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="n">list_of_params</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">&quot;continuous_cols=</span><span class="si">{continuous_cols}</span><span class="s2">&quot;</span><span class="p">)</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">quantization_setup</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="n">list_of_params</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">&quot;quantization_setup=</span><span class="si">{quantization_setup}</span><span class="s2">&quot;</span><span class="p">)</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">cols_to_scale</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="n">list_of_params</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">&quot;cols_to_scale=</span><span class="si">{cols_to_scale}</span><span class="s2">&quot;</span><span class="p">)</span>
+        <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">auto_embed_dim</span><span class="p">:</span>
+            <span class="n">list_of_params</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">&quot;auto_embed_dim=</span><span class="si">{auto_embed_dim}</span><span class="s2">&quot;</span><span class="p">)</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">embedding_rule</span> <span class="o">!=</span> <span class="s2">&quot;fastai_new&quot;</span><span class="p">:</span>
+            <span class="n">list_of_params</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">&quot;embedding_rule=&#39;</span><span class="si">{embedding_rule}</span><span class="s2">&#39;&quot;</span><span class="p">)</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">default_embed_dim</span> <span class="o">!=</span> <span class="mi">16</span><span class="p">:</span>
+            <span class="n">list_of_params</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">&quot;default_embed_dim=</span><span class="si">{default_embed_dim}</span><span class="s2">&quot;</span><span class="p">)</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">with_attention</span><span class="p">:</span>
+            <span class="n">list_of_params</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">&quot;with_attention=</span><span class="si">{with_attention}</span><span class="s2">&quot;</span><span class="p">)</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">with_cls_token</span><span class="p">:</span>
+            <span class="n">list_of_params</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">&quot;with_cls_token=</span><span class="si">{with_cls_token}</span><span class="s2">&quot;</span><span class="p">)</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">shared_embed</span><span class="p">:</span>
+            <span class="n">list_of_params</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">&quot;shared_embed=</span><span class="si">{shared_embed}</span><span class="s2">&quot;</span><span class="p">)</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">verbose</span> <span class="o">!=</span> <span class="mi">1</span><span class="p">:</span>
+            <span class="n">list_of_params</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">&quot;verbose=</span><span class="si">{verbose}</span><span class="s2">&quot;</span><span class="p">)</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">scale</span><span class="p">:</span>
+            <span class="n">list_of_params</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">&quot;scale=</span><span class="si">{scale}</span><span class="s2">&quot;</span><span class="p">)</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">already_standard</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="n">list_of_params</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">&quot;already_standard=</span><span class="si">{already_standard}</span><span class="s2">&quot;</span><span class="p">)</span>
+        <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">quant_args</span><span class="p">)</span> <span class="o">&gt;</span> <span class="mi">0</span><span class="p">:</span>
+            <span class="n">list_of_params</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
+                <span class="s2">&quot;, &quot;</span><span class="o">.</span><span class="n">join</span><span class="p">([</span><span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="n">k</span><span class="si">}</span><span class="s2">&quot;</span> <span class="o">+</span> <span class="s2">&quot;=&quot;</span> <span class="o">+</span> <span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="n">v</span><span class="si">}</span><span class="s2">&quot;</span> <span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">quant_args</span><span class="o">.</span><span class="n">items</span><span class="p">()])</span>
+            <span class="p">)</span>
+        <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">scale_args</span><span class="p">)</span> <span class="o">&gt;</span> <span class="mi">0</span><span class="p">:</span>
+            <span class="n">list_of_params</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
+                <span class="s2">&quot;, &quot;</span><span class="o">.</span><span class="n">join</span><span class="p">([</span><span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="n">k</span><span class="si">}</span><span class="s2">&quot;</span> <span class="o">+</span> <span class="s2">&quot;=&quot;</span> <span class="o">+</span> <span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="n">v</span><span class="si">}</span><span class="s2">&quot;</span> <span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">scale_args</span><span class="o">.</span><span class="n">items</span><span class="p">()])</span>
+            <span class="p">)</span>
+        <span class="n">all_params</span> <span class="o">=</span> <span class="s2">&quot;, &quot;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">list_of_params</span><span class="p">)</span>
+        <span class="k">return</span> <span class="sa">f</span><span class="s2">&quot;TabPreprocessor(</span><span class="si">{</span><span class="n">all_params</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="bp">self</span><span class="o">.</span><span class="vm">__dict__</span><span class="p">)</span><span class="si">}</span><span class="s2">)&quot;</span>
 </code></pre></div></td></tr></table></div>
-                  </details>
+              </details>
 
 
 
@@ -3296,341 +4427,342 @@ <h2 id="pytorch_widedeep.preprocessing.tab_preprocessor.Quantizer" class="doc do
 
 
 
+<div class="doc doc-object doc-function">
 
 
-  </div>
-
-    </div>
-
-</div>
-
-<div class="doc doc-object doc-class">
-
-
-
-<h2 id="pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor" class="doc doc-heading">
-            <span class="doc doc-object-name doc-class-name">TextPreprocessor</span>
+<h3 id="pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor.fit" class="doc doc-heading">
+            <span class="doc doc-object-name doc-function-name">fit</span>
 
 
-<a href="#pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="nf">TextPreprocessor</span><span class="p">(</span>
-    <span class="n">text_col</span><span class="p">,</span>
-    <span class="n">max_vocab</span><span class="o">=</span><span class="mi">30000</span><span class="p">,</span>
-    <span class="n">min_freq</span><span class="o">=</span><span class="mi">5</span><span class="p">,</span>
-    <span class="n">maxlen</span><span class="o">=</span><span class="mi">80</span><span class="p">,</span>
-    <span class="n">pad_first</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
-    <span class="n">pad_idx</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span>
-    <span class="n">already_processed</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
-    <span class="n">word_vectors_path</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">n_cpus</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">verbose</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span>
-<span class="p">)</span>
+<a href="#pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor.fit" class="headerlink" title="Permanent link">&para;</a></h3>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">fit</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
 </code></pre></div>
 
-    <div class="doc doc-contents first">
-            <p class="doc doc-class-bases">
-              Bases: <code><span title="pytorch_widedeep.preprocessing.base_preprocessor.BasePreprocessor">BasePreprocessor</span></code></p>
-
+    <div class="doc doc-contents ">
 
-        <p>Preprocessor to prepare the <code>deeptext</code> input dataset</p>
+        <p>Fits the Preprocessor and creates required attributes</p>
 
 
 <p><span class="doc-section-title">Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>text_col</code></b>
-              (<code>str</code>)
-          –
-          <div class="doc-md-description">
-            <p>column in the input dataframe containing the texts</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>max_vocab</code></b>
-              (<code>int</code>, default:
-                  <code>30000</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Maximum number of tokens in the vocabulary</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>min_freq</code></b>
-              (<code>int</code>, default:
-                  <code>5</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Minimum frequency for a token to be part of the vocabulary</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>maxlen</code></b>
-              (<code>int</code>, default:
-                  <code>80</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Maximum length of the tokenized sequences</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>pad_first</code></b>
-              (<code>bool</code>, default:
-                  <code>True</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Indicates whether the padding index will be added at the beginning or the
-end of the sequences</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>pad_idx</code></b>
-              (<code>int</code>, default:
-                  <code>1</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>padding index. Fastai's Tokenizer leaves 0 for the 'unknown' token.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>already_processed</code></b>
-              (<code><span title="typing.Optional">Optional</span>[bool]</code>, default:
-                  <code>False</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Boolean indicating if the sequence of elements is already processed or
-prepared. If this is the case, this Preprocessor will simply tokenize
-and pad the sequence. <br/></p>
-<div class="highlight"><pre><span></span><code>Param aliases: `not_text`. &lt;br/&gt;
-</code></pre></div>
-<p>This parameter is thought for those cases where the input sequences
-are already fully processed or are directly not text (e.g. IDs)</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>word_vectors_path</code></b>
-              (<code><span title="typing.Optional">Optional</span>[str]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Path to the pretrained word vectors</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>n_cpus</code></b>
-              (<code><span title="typing.Optional">Optional</span>[int]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>number of CPUs to used during the tokenization process</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>verbose</code></b>
-              (<code>int</code>, default:
-                  <code>1</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Enable verbose output.</p>
-          </div>
-        </li>
-    </ul>
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+          <th>Default</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>df</code></td>
+            <td>
+                  <code><span title="pandas.DataFrame">DataFrame</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Input pandas dataframe</p>
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+      </tbody>
+    </table>
+
+
+    <p><span class="doc-section-title">Returns:</span></p>
+    <table>
+      <thead>
+        <tr>
+          <th>Type</th>
+          <th>Description</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td>
+                  <code><a class="autorefs autorefs-internal" title="pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor" href="#pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor">TabPreprocessor</a></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p><code>TabPreprocessor</code> fitted object</p>
+              </div>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
+            <details class="quote">
+              <summary>Source code in <code>pytorch_widedeep/preprocessing/tab_preprocessor.py</code></summary>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">312</span>
+<span class="normal">313</span>
+<span class="normal">314</span>
+<span class="normal">315</span>
+<span class="normal">316</span>
+<span class="normal">317</span>
+<span class="normal">318</span>
+<span class="normal">319</span>
+<span class="normal">320</span>
+<span class="normal">321</span>
+<span class="normal">322</span>
+<span class="normal">323</span>
+<span class="normal">324</span>
+<span class="normal">325</span>
+<span class="normal">326</span>
+<span class="normal">327</span>
+<span class="normal">328</span>
+<span class="normal">329</span>
+<span class="normal">330</span>
+<span class="normal">331</span>
+<span class="normal">332</span>
+<span class="normal">333</span>
+<span class="normal">334</span>
+<span class="normal">335</span>
+<span class="normal">336</span>
+<span class="normal">337</span>
+<span class="normal">338</span>
+<span class="normal">339</span>
+<span class="normal">340</span>
+<span class="normal">341</span>
+<span class="normal">342</span>
+<span class="normal">343</span>
+<span class="normal">344</span>
+<span class="normal">345</span>
+<span class="normal">346</span>
+<span class="normal">347</span>
+<span class="normal">348</span>
+<span class="normal">349</span>
+<span class="normal">350</span>
+<span class="normal">351</span>
+<span class="normal">352</span>
+<span class="normal">353</span>
+<span class="normal">354</span>
+<span class="normal">355</span>
+<span class="normal">356</span>
+<span class="normal">357</span>
+<span class="normal">358</span>
+<span class="normal">359</span>
+<span class="normal">360</span>
+<span class="normal">361</span>
+<span class="normal">362</span>
+<span class="normal">363</span>
+<span class="normal">364</span>
+<span class="normal">365</span>
+<span class="normal">366</span>
+<span class="normal">367</span>
+<span class="normal">368</span>
+<span class="normal">369</span>
+<span class="normal">370</span>
+<span class="normal">371</span>
+<span class="normal">372</span>
+<span class="normal">373</span>
+<span class="normal">374</span>
+<span class="normal">375</span>
+<span class="normal">376</span>
+<span class="normal">377</span>
+<span class="normal">378</span>
+<span class="normal">379</span>
+<span class="normal">380</span>
+<span class="normal">381</span>
+<span class="normal">382</span>
+<span class="normal">383</span>
+<span class="normal">384</span>
+<span class="normal">385</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">fit</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">BasePreprocessor</span><span class="p">:</span>  <span class="c1"># noqa: C901</span>
+<span class="w">    </span><span class="sd">&quot;&quot;&quot;Fits the Preprocessor and creates required attributes</span>
 
-<p><span class="doc-section-title">Attributes:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code><span title="pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor.vocab">vocab</span></code></b>
-              (<code><a class="autorefs autorefs-internal" title="pytorch_widedeep.utils.fastai_transforms.Vocab" href="utils/fastai_transforms.html#pytorch_widedeep.utils.fastai_transforms.Vocab">Vocab</a></code>)
-          –
-          <div class="doc-md-description">
-            <p>an instance of <code>pytorch_widedeep.utils.fastai_transforms.Vocab</code></p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code><span title="pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor.embedding_matrix">embedding_matrix</span></code></b>
-              (<code><span title="numpy.ndarray">ndarray</span></code>)
-          –
-          <div class="doc-md-description">
-            <p>Array with the pretrained embeddings</p>
-          </div>
-        </li>
-    </ul>
+<span class="sd">    Parameters</span>
+<span class="sd">    ----------</span>
+<span class="sd">    df: pd.DataFrame</span>
+<span class="sd">        Input pandas dataframe</span>
 
+<span class="sd">    Returns</span>
+<span class="sd">    -------</span>
+<span class="sd">    TabPreprocessor</span>
+<span class="sd">        `TabPreprocessor` fitted object</span>
+<span class="sd">    &quot;&quot;&quot;</span>
 
-<p><span class="doc-section-title">Examples:</span></p>
-    <div class="highlight"><pre><span></span><code><span class="gp">&gt;&gt;&gt; </span><span class="kn">import</span> <span class="nn">pandas</span> <span class="k">as</span> <span class="nn">pd</span>
-<span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">pytorch_widedeep.preprocessing</span> <span class="kn">import</span> <span class="n">TextPreprocessor</span>
-<span class="gp">&gt;&gt;&gt; </span><span class="n">df_train</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">({</span><span class="s1">&#39;text_column&#39;</span><span class="p">:</span> <span class="p">[</span><span class="s2">&quot;life is like a box of chocolates&quot;</span><span class="p">,</span>
-<span class="gp">... </span><span class="s2">&quot;You never know what you&#39;re gonna get&quot;</span><span class="p">]})</span>
-<span class="gp">&gt;&gt;&gt; </span><span class="n">text_preprocessor</span> <span class="o">=</span> <span class="n">TextPreprocessor</span><span class="p">(</span><span class="n">text_col</span><span class="o">=</span><span class="s1">&#39;text_column&#39;</span><span class="p">,</span> <span class="n">max_vocab</span><span class="o">=</span><span class="mi">25</span><span class="p">,</span> <span class="n">min_freq</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">maxlen</span><span class="o">=</span><span class="mi">10</span><span class="p">)</span>
-<span class="gp">&gt;&gt;&gt; </span><span class="n">text_preprocessor</span><span class="o">.</span><span class="n">fit_transform</span><span class="p">(</span><span class="n">df_train</span><span class="p">)</span>
-<span class="go">The vocabulary contains 24 tokens</span>
-<span class="go">array([[ 1,  1,  1,  1, 10, 11, 12, 13, 14, 15],</span>
-<span class="go">       [ 5,  9, 16, 17, 18,  9, 19, 20, 21, 22]], dtype=int32)</span>
-<span class="gp">&gt;&gt;&gt; </span><span class="n">df_te</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">({</span><span class="s1">&#39;text_column&#39;</span><span class="p">:</span> <span class="p">[</span><span class="s1">&#39;you never know what is in the box&#39;</span><span class="p">]})</span>
-<span class="gp">&gt;&gt;&gt; </span><span class="n">text_preprocessor</span><span class="o">.</span><span class="n">transform</span><span class="p">(</span><span class="n">df_te</span><span class="p">)</span>
-<span class="go">array([[ 1,  1,  9, 16, 17, 18, 11,  0,  0, 13]], dtype=int32)</span>
-</code></pre></div>
+    <span class="n">df_adj</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_insert_cls_token</span><span class="p">(</span><span class="n">df</span><span class="p">)</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">with_cls_token</span> <span class="k">else</span> <span class="n">df</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
 
-                  <details class="quote">
-                    <summary>Source code in <code>pytorch_widedeep/preprocessing/text_preprocessor.py</code></summary>
-                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 79</span>
-<span class="normal"> 80</span>
-<span class="normal"> 81</span>
-<span class="normal"> 82</span>
-<span class="normal"> 83</span>
-<span class="normal"> 84</span>
-<span class="normal"> 85</span>
-<span class="normal"> 86</span>
-<span class="normal"> 87</span>
-<span class="normal"> 88</span>
-<span class="normal"> 89</span>
-<span class="normal"> 90</span>
-<span class="normal"> 91</span>
-<span class="normal"> 92</span>
-<span class="normal"> 93</span>
-<span class="normal"> 94</span>
-<span class="normal"> 95</span>
-<span class="normal"> 96</span>
-<span class="normal"> 97</span>
-<span class="normal"> 98</span>
-<span class="normal"> 99</span>
-<span class="normal">100</span>
-<span class="normal">101</span>
-<span class="normal">102</span>
-<span class="normal">103</span>
-<span class="normal">104</span>
-<span class="normal">105</span>
-<span class="normal">106</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="nd">@alias</span><span class="p">(</span><span class="s2">&quot;already_processed&quot;</span><span class="p">,</span> <span class="p">[</span><span class="s2">&quot;not_text&quot;</span><span class="p">])</span>
-<span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
-    <span class="bp">self</span><span class="p">,</span>
-    <span class="n">text_col</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
-    <span class="n">max_vocab</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">30000</span><span class="p">,</span>
-    <span class="n">min_freq</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">5</span><span class="p">,</span>
-    <span class="n">maxlen</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">80</span><span class="p">,</span>
-    <span class="n">pad_first</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">,</span>
-    <span class="n">pad_idx</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1</span><span class="p">,</span>
-    <span class="n">already_processed</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
-    <span class="n">word_vectors_path</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">n_cpus</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">verbose</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1</span><span class="p">,</span>
-<span class="p">):</span>
-    <span class="nb">super</span><span class="p">(</span><span class="n">TextPreprocessor</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
-
-    <span class="bp">self</span><span class="o">.</span><span class="n">text_col</span> <span class="o">=</span> <span class="n">text_col</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">max_vocab</span> <span class="o">=</span> <span class="n">max_vocab</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">min_freq</span> <span class="o">=</span> <span class="n">min_freq</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">maxlen</span> <span class="o">=</span> <span class="n">maxlen</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">pad_first</span> <span class="o">=</span> <span class="n">pad_first</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">pad_idx</span> <span class="o">=</span> <span class="n">pad_idx</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">already_processed</span> <span class="o">=</span> <span class="n">already_processed</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">word_vectors_path</span> <span class="o">=</span> <span class="n">word_vectors_path</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">verbose</span> <span class="o">=</span> <span class="n">verbose</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">n_cpus</span> <span class="o">=</span> <span class="n">n_cpus</span> <span class="k">if</span> <span class="n">n_cpus</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="k">else</span> <span class="n">os</span><span class="o">.</span><span class="n">cpu_count</span><span class="p">()</span>
-
-    <span class="bp">self</span><span class="o">.</span><span class="n">is_fitted</span> <span class="o">=</span> <span class="kc">False</span>
-</code></pre></div></td></tr></table></div>
-                  </details>
+    <span class="bp">self</span><span class="o">.</span><span class="n">column_idx</span><span class="p">:</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="p">{}</span>
 
+    <span class="c1"># Categorical embeddings logic</span>
+    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">cat_embed_cols</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">or</span> <span class="bp">self</span><span class="o">.</span><span class="n">quantization_setup</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">cat_embed_input</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="n">Union</span><span class="p">[</span><span class="n">Tuple</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">],</span> <span class="n">Tuple</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">,</span> <span class="nb">int</span><span class="p">]]]</span> <span class="o">=</span> <span class="p">(</span>
+            <span class="p">[]</span>
+        <span class="p">)</span>
 
+    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">cat_embed_cols</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+        <span class="n">df_cat</span><span class="p">,</span> <span class="n">cat_embed_dim</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_prepare_categorical</span><span class="p">(</span><span class="n">df_adj</span><span class="p">)</span>
 
-  <div class="doc doc-children">
+        <span class="bp">self</span><span class="o">.</span><span class="n">label_encoder</span> <span class="o">=</span> <span class="n">LabelEncoder</span><span class="p">(</span>
+            <span class="n">columns_to_encode</span><span class="o">=</span><span class="n">df_cat</span><span class="o">.</span><span class="n">columns</span><span class="o">.</span><span class="n">tolist</span><span class="p">(),</span>
+            <span class="n">shared_embed</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">shared_embed</span><span class="p">,</span>
+            <span class="n">with_attention</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">with_attention</span><span class="p">,</span>
+        <span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">label_encoder</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">df_cat</span><span class="p">)</span>
+
+        <span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">label_encoder</span><span class="o">.</span><span class="n">encoding_dict</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
+            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">with_attention</span><span class="p">:</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">cat_embed_input</span><span class="o">.</span><span class="n">append</span><span class="p">((</span><span class="n">k</span><span class="p">,</span> <span class="nb">len</span><span class="p">(</span><span class="n">v</span><span class="p">)))</span>
+            <span class="k">else</span><span class="p">:</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">cat_embed_input</span><span class="o">.</span><span class="n">append</span><span class="p">((</span><span class="n">k</span><span class="p">,</span> <span class="nb">len</span><span class="p">(</span><span class="n">v</span><span class="p">),</span> <span class="n">cat_embed_dim</span><span class="p">[</span><span class="n">k</span><span class="p">]))</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">column_idx</span><span class="o">.</span><span class="n">update</span><span class="p">({</span><span class="n">k</span><span class="p">:</span> <span class="n">v</span> <span class="k">for</span> <span class="n">v</span><span class="p">,</span> <span class="n">k</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="n">df_cat</span><span class="o">.</span><span class="n">columns</span><span class="p">)})</span>
 
+    <span class="c1"># Continuous columns logic</span>
+    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">continuous_cols</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+        <span class="n">df_cont</span><span class="p">,</span> <span class="n">cont_embed_dim</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_prepare_continuous</span><span class="p">(</span><span class="n">df_adj</span><span class="p">)</span>
 
+        <span class="c1"># Standardization logic</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">standardize_cols</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">scaler</span> <span class="o">=</span> <span class="n">StandardScaler</span><span class="p">(</span><span class="o">**</span><span class="bp">self</span><span class="o">.</span><span class="n">scale_args</span><span class="p">)</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span>
+                <span class="n">df_cont</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">standardize_cols</span><span class="p">]</span><span class="o">.</span><span class="n">values</span>
+            <span class="p">)</span>
+        <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">verbose</span><span class="p">:</span>
+            <span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span><span class="s2">&quot;Continuous columns will not be normalised&quot;</span><span class="p">)</span>
 
+        <span class="c1"># Quantization logic</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">cols_and_bins</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="c1"># we do not run &#39;Quantizer.fit&#39; here since in the wild case</span>
+            <span class="c1"># someone wants standardization and quantization for the same</span>
+            <span class="c1"># columns, the Quantizer will run on the scaled data</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">quantizer</span> <span class="o">=</span> <span class="n">Quantizer</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">cols_and_bins</span><span class="p">,</span> <span class="o">**</span><span class="bp">self</span><span class="o">.</span><span class="n">quant_args</span><span class="p">)</span>
 
+            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">with_attention</span><span class="p">:</span>
+                <span class="k">for</span> <span class="n">col</span><span class="p">,</span> <span class="n">n_cat</span><span class="p">,</span> <span class="n">_</span> <span class="ow">in</span> <span class="n">cont_embed_dim</span><span class="p">:</span>
+                    <span class="bp">self</span><span class="o">.</span><span class="n">cat_embed_input</span><span class="o">.</span><span class="n">append</span><span class="p">((</span><span class="n">col</span><span class="p">,</span> <span class="n">n_cat</span><span class="p">))</span>
+            <span class="k">else</span><span class="p">:</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">cat_embed_input</span><span class="o">.</span><span class="n">extend</span><span class="p">(</span><span class="n">cont_embed_dim</span><span class="p">)</span>
 
+        <span class="bp">self</span><span class="o">.</span><span class="n">column_idx</span><span class="o">.</span><span class="n">update</span><span class="p">(</span>
+            <span class="p">{</span><span class="n">k</span><span class="p">:</span> <span class="n">v</span> <span class="o">+</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">column_idx</span><span class="p">)</span> <span class="k">for</span> <span class="n">v</span><span class="p">,</span> <span class="n">k</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="n">df_cont</span><span class="p">)}</span>
+        <span class="p">)</span>
 
+    <span class="bp">self</span><span class="o">.</span><span class="n">is_fitted</span> <span class="o">=</span> <span class="kc">True</span>
 
+    <span class="k">return</span> <span class="bp">self</span>
+</code></pre></div></td></tr></table></div>
+            </details>
+    </div>
 
+</div>
 
 <div class="doc doc-object doc-function">
 
 
-<h3 id="pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor.fit" class="doc doc-heading">
-            <span class="doc doc-object-name doc-function-name">fit</span>
+<h3 id="pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor.transform" class="doc doc-heading">
+            <span class="doc doc-object-name doc-function-name">transform</span>
 
 
-<a href="#pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor.fit" class="headerlink" title="Permanent link">&para;</a></h3>
-<div class="doc-signature highlight"><pre><span></span><code><span class="nf">fit</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
+<a href="#pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor.transform" class="headerlink" title="Permanent link">&para;</a></h3>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">transform</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
 </code></pre></div>
 
     <div class="doc doc-contents ">
 
-        <p>Builds the vocabulary</p>
+        <p>Returns the processed <code>dataframe</code> as a np.ndarray</p>
 
 
 <p><span class="doc-section-title">Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>df</code></b>
-              (<code><span title="pandas.DataFrame">DataFrame</span></code>)
-          –
-          <div class="doc-md-description">
-            <p>Input pandas dataframe</p>
-          </div>
-        </li>
-    </ul>
-
-
-<p><span class="doc-section-title">Returns:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-              <code><a class="autorefs autorefs-internal" title="pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor" href="#pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor">TextPreprocessor</a></code>
-          –
-          <div class="doc-md-description">
-            <p><code>TextPreprocessor</code> fitted object</p>
-          </div>
-        </li>
-    </ul>
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+          <th>Default</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>df</code></td>
+            <td>
+                  <code><span title="pandas.DataFrame">DataFrame</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Input pandas dataframe</p>
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+      </tbody>
+    </table>
+
+
+    <p><span class="doc-section-title">Returns:</span></p>
+    <table>
+      <thead>
+        <tr>
+          <th>Type</th>
+          <th>Description</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td>
+                  <code><span title="numpy.ndarray">ndarray</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>transformed input dataframe</p>
+              </div>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
             <details class="quote">
-              <summary>Source code in <code>pytorch_widedeep/preprocessing/text_preprocessor.py</code></summary>
-              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">108</span>
-<span class="normal">109</span>
-<span class="normal">110</span>
-<span class="normal">111</span>
-<span class="normal">112</span>
-<span class="normal">113</span>
-<span class="normal">114</span>
-<span class="normal">115</span>
-<span class="normal">116</span>
-<span class="normal">117</span>
-<span class="normal">118</span>
-<span class="normal">119</span>
-<span class="normal">120</span>
-<span class="normal">121</span>
-<span class="normal">122</span>
-<span class="normal">123</span>
-<span class="normal">124</span>
-<span class="normal">125</span>
-<span class="normal">126</span>
-<span class="normal">127</span>
-<span class="normal">128</span>
-<span class="normal">129</span>
-<span class="normal">130</span>
-<span class="normal">131</span>
-<span class="normal">132</span>
-<span class="normal">133</span>
-<span class="normal">134</span>
-<span class="normal">135</span>
-<span class="normal">136</span>
-<span class="normal">137</span>
-<span class="normal">138</span>
-<span class="normal">139</span>
-<span class="normal">140</span>
-<span class="normal">141</span>
-<span class="normal">142</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">fit</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">BasePreprocessor</span><span class="p">:</span>
-<span class="w">    </span><span class="sd">&quot;&quot;&quot;Builds the vocabulary</span>
+              <summary>Source code in <code>pytorch_widedeep/preprocessing/tab_preprocessor.py</code></summary>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">387</span>
+<span class="normal">388</span>
+<span class="normal">389</span>
+<span class="normal">390</span>
+<span class="normal">391</span>
+<span class="normal">392</span>
+<span class="normal">393</span>
+<span class="normal">394</span>
+<span class="normal">395</span>
+<span class="normal">396</span>
+<span class="normal">397</span>
+<span class="normal">398</span>
+<span class="normal">399</span>
+<span class="normal">400</span>
+<span class="normal">401</span>
+<span class="normal">402</span>
+<span class="normal">403</span>
+<span class="normal">404</span>
+<span class="normal">405</span>
+<span class="normal">406</span>
+<span class="normal">407</span>
+<span class="normal">408</span>
+<span class="normal">409</span>
+<span class="normal">410</span>
+<span class="normal">411</span>
+<span class="normal">412</span>
+<span class="normal">413</span>
+<span class="normal">414</span>
+<span class="normal">415</span>
+<span class="normal">416</span>
+<span class="normal">417</span>
+<span class="normal">418</span>
+<span class="normal">419</span>
+<span class="normal">420</span>
+<span class="normal">421</span>
+<span class="normal">422</span>
+<span class="normal">423</span>
+<span class="normal">424</span>
+<span class="normal">425</span>
+<span class="normal">426</span>
+<span class="normal">427</span>
+<span class="normal">428</span>
+<span class="normal">429</span>
+<span class="normal">430</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">transform</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">:</span>  <span class="c1"># noqa: C901</span>
+<span class="w">    </span><span class="sd">&quot;&quot;&quot;Returns the processed `dataframe` as a np.ndarray</span>
 
 <span class="sd">    Parameters</span>
 <span class="sd">    ----------</span>
@@ -3639,31 +4771,40 @@ <h3 id="pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor.fit" c
 
 <span class="sd">    Returns</span>
 <span class="sd">    -------</span>
-<span class="sd">    TextPreprocessor</span>
-<span class="sd">        `TextPreprocessor` fitted object</span>
+<span class="sd">    np.ndarray</span>
+<span class="sd">        transformed input dataframe</span>
 <span class="sd">    &quot;&quot;&quot;</span>
-    <span class="n">texts</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_read_texts</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
-
-    <span class="n">tokens</span> <span class="o">=</span> <span class="n">get_texts</span><span class="p">(</span><span class="n">texts</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">already_processed</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">n_cpus</span><span class="p">)</span>
-
-    <span class="bp">self</span><span class="o">.</span><span class="n">vocab</span><span class="p">:</span> <span class="n">TVocab</span> <span class="o">=</span> <span class="n">Vocab</span><span class="p">(</span>
-        <span class="n">max_vocab</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">max_vocab</span><span class="p">,</span>
-        <span class="n">min_freq</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">min_freq</span><span class="p">,</span>
-        <span class="n">pad_idx</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">pad_idx</span><span class="p">,</span>
-    <span class="p">)</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span>
-        <span class="n">tokens</span><span class="p">,</span>
-    <span class="p">)</span>
+    <span class="n">check_is_fitted</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">condition</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">is_fitted</span><span class="p">)</span>
 
-    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">verbose</span><span class="p">:</span>
-        <span class="nb">print</span><span class="p">(</span><span class="s2">&quot;The vocabulary contains </span><span class="si">{}</span><span class="s2"> tokens&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">vocab</span><span class="o">.</span><span class="n">stoi</span><span class="p">)))</span>
-    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">word_vectors_path</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">embedding_matrix</span> <span class="o">=</span> <span class="n">build_embeddings_matrix</span><span class="p">(</span>
-            <span class="bp">self</span><span class="o">.</span><span class="n">vocab</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">word_vectors_path</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">min_freq</span>
-        <span class="p">)</span>
+    <span class="n">df_adj</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_insert_cls_token</span><span class="p">(</span><span class="n">df</span><span class="p">)</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">with_cls_token</span> <span class="k">else</span> <span class="n">df</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
 
-    <span class="bp">self</span><span class="o">.</span><span class="n">is_fitted</span> <span class="o">=</span> <span class="kc">True</span>
+    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">cat_embed_cols</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+        <span class="n">df_cat</span> <span class="o">=</span> <span class="n">df_adj</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">cat_cols</span><span class="p">]</span>
+        <span class="n">df_cat</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">label_encoder</span><span class="o">.</span><span class="n">transform</span><span class="p">(</span><span class="n">df_cat</span><span class="p">)</span>
+    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">continuous_cols</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+        <span class="n">df_cont</span> <span class="o">=</span> <span class="n">df_adj</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">continuous_cols</span><span class="p">]</span>
+        <span class="c1"># Standardization logic</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">standardize_cols</span><span class="p">:</span>
+            <span class="n">df_cont</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">standardize_cols</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">scaler</span><span class="o">.</span><span class="n">transform</span><span class="p">(</span>
+                <span class="n">df_cont</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">standardize_cols</span><span class="p">]</span><span class="o">.</span><span class="n">values</span>
+            <span class="p">)</span>
+        <span class="c1"># Quantization logic</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">cols_and_bins</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="c1"># Adjustment so I don&#39;t have to override the method</span>
+            <span class="c1"># in &#39;ChunkTabPreprocessor&#39;</span>
+            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">quantizer</span><span class="o">.</span><span class="n">is_fitted</span><span class="p">:</span>
+                <span class="n">df_cont</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">quantizer</span><span class="o">.</span><span class="n">transform</span><span class="p">(</span><span class="n">df_cont</span><span class="p">)</span>
+            <span class="k">else</span><span class="p">:</span>
+                <span class="n">df_cont</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">quantizer</span><span class="o">.</span><span class="n">fit_transform</span><span class="p">(</span><span class="n">df_cont</span><span class="p">)</span>
+    <span class="k">try</span><span class="p">:</span>
+        <span class="n">df_deep</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">concat</span><span class="p">([</span><span class="n">df_cat</span><span class="p">,</span> <span class="n">df_cont</span><span class="p">],</span> <span class="n">axis</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span>
+    <span class="k">except</span> <span class="ne">NameError</span><span class="p">:</span>
+        <span class="k">try</span><span class="p">:</span>
+            <span class="n">df_deep</span> <span class="o">=</span> <span class="n">df_cat</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
+        <span class="k">except</span> <span class="ne">NameError</span><span class="p">:</span>
+            <span class="n">df_deep</span> <span class="o">=</span> <span class="n">df_cont</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
 
-    <span class="k">return</span> <span class="bp">self</span>
+    <span class="k">return</span> <span class="n">df_deep</span><span class="o">.</span><span class="n">values</span>
 </code></pre></div></td></tr></table></div>
             </details>
     </div>
@@ -3673,78 +4814,152 @@ <h3 id="pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor.fit" c
 <div class="doc doc-object doc-function">
 
 
-<h3 id="pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor.transform" class="doc doc-heading">
-            <span class="doc doc-object-name doc-function-name">transform</span>
+<h3 id="pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor.inverse_transform" class="doc doc-heading">
+            <span class="doc doc-object-name doc-function-name">inverse_transform</span>
 
 
-<a href="#pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor.transform" class="headerlink" title="Permanent link">&para;</a></h3>
-<div class="doc-signature highlight"><pre><span></span><code><span class="nf">transform</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
+<a href="#pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor.inverse_transform" class="headerlink" title="Permanent link">&para;</a></h3>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">inverse_transform</span><span class="p">(</span><span class="n">encoded</span><span class="p">)</span>
 </code></pre></div>
 
     <div class="doc doc-contents ">
 
-        <p>Returns the padded, <em>'numericalised'</em> sequences</p>
+        <p>Takes as input the output from the <code>transform</code> method and it will
+return the original values.</p>
 
 
 <p><span class="doc-section-title">Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>df</code></b>
-              (<code><span title="pandas.DataFrame">DataFrame</span></code>)
-          –
-          <div class="doc-md-description">
-            <p>Input pandas dataframe</p>
-          </div>
-        </li>
-    </ul>
-
-
-<p><span class="doc-section-title">Returns:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-              <code><span title="numpy.ndarray">ndarray</span></code>
-          –
-          <div class="doc-md-description">
-            <p>Padded, <em>'numericalised'</em> sequences</p>
-          </div>
-        </li>
-    </ul>
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+          <th>Default</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>encoded</code></td>
+            <td>
+                  <code><span title="numpy.ndarray">ndarray</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>array with the output of the <code>transform</code> method</p>
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+      </tbody>
+    </table>
+
+
+    <p><span class="doc-section-title">Returns:</span></p>
+    <table>
+      <thead>
+        <tr>
+          <th>Type</th>
+          <th>Description</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td>
+                  <code><span title="pandas.DataFrame">DataFrame</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Pandas dataframe with the original values</p>
+              </div>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
             <details class="quote">
-              <summary>Source code in <code>pytorch_widedeep/preprocessing/text_preprocessor.py</code></summary>
-              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">144</span>
-<span class="normal">145</span>
-<span class="normal">146</span>
-<span class="normal">147</span>
-<span class="normal">148</span>
-<span class="normal">149</span>
-<span class="normal">150</span>
-<span class="normal">151</span>
-<span class="normal">152</span>
-<span class="normal">153</span>
-<span class="normal">154</span>
-<span class="normal">155</span>
-<span class="normal">156</span>
-<span class="normal">157</span>
-<span class="normal">158</span>
-<span class="normal">159</span>
-<span class="normal">160</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">transform</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">:</span>
-<span class="w">    </span><span class="sd">&quot;&quot;&quot;Returns the padded, _&#39;numericalised&#39;_ sequences</span>
+              <summary>Source code in <code>pytorch_widedeep/preprocessing/tab_preprocessor.py</code></summary>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">435</span>
+<span class="normal">436</span>
+<span class="normal">437</span>
+<span class="normal">438</span>
+<span class="normal">439</span>
+<span class="normal">440</span>
+<span class="normal">441</span>
+<span class="normal">442</span>
+<span class="normal">443</span>
+<span class="normal">444</span>
+<span class="normal">445</span>
+<span class="normal">446</span>
+<span class="normal">447</span>
+<span class="normal">448</span>
+<span class="normal">449</span>
+<span class="normal">450</span>
+<span class="normal">451</span>
+<span class="normal">452</span>
+<span class="normal">453</span>
+<span class="normal">454</span>
+<span class="normal">455</span>
+<span class="normal">456</span>
+<span class="normal">457</span>
+<span class="normal">458</span>
+<span class="normal">459</span>
+<span class="normal">460</span>
+<span class="normal">461</span>
+<span class="normal">462</span>
+<span class="normal">463</span>
+<span class="normal">464</span>
+<span class="normal">465</span>
+<span class="normal">466</span>
+<span class="normal">467</span>
+<span class="normal">468</span>
+<span class="normal">469</span>
+<span class="normal">470</span>
+<span class="normal">471</span>
+<span class="normal">472</span>
+<span class="normal">473</span>
+<span class="normal">474</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">inverse_transform</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">encoded</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">:</span>  <span class="c1"># noqa: C901</span>
+<span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Takes as input the output from the `transform` method and it will</span>
+<span class="sd">    return the original values.</span>
 
 <span class="sd">    Parameters</span>
 <span class="sd">    ----------</span>
-<span class="sd">    df: pd.DataFrame</span>
-<span class="sd">        Input pandas dataframe</span>
+<span class="sd">    encoded: np.ndarray</span>
+<span class="sd">        array with the output of the `transform` method</span>
 
 <span class="sd">    Returns</span>
 <span class="sd">    -------</span>
-<span class="sd">    np.ndarray</span>
-<span class="sd">        Padded, _&#39;numericalised&#39;_ sequences</span>
+<span class="sd">    pd.DataFrame</span>
+<span class="sd">        Pandas dataframe with the original values</span>
 <span class="sd">    &quot;&quot;&quot;</span>
-    <span class="n">check_is_fitted</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">attributes</span><span class="o">=</span><span class="p">[</span><span class="s2">&quot;vocab&quot;</span><span class="p">])</span>
-    <span class="n">texts</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_read_texts</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
-    <span class="n">tokens</span> <span class="o">=</span> <span class="n">get_texts</span><span class="p">(</span><span class="n">texts</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">already_processed</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">n_cpus</span><span class="p">)</span>
-    <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_pad_sequences</span><span class="p">(</span><span class="n">tokens</span><span class="p">)</span>
+    <span class="n">decoded</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">(</span><span class="n">encoded</span><span class="p">,</span> <span class="n">columns</span><span class="o">=</span><span class="nb">list</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">column_idx</span><span class="o">.</span><span class="n">keys</span><span class="p">()))</span>
+    <span class="c1"># embeddings back to original category</span>
+    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">cat_embed_cols</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+        <span class="n">decoded</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">label_encoder</span><span class="o">.</span><span class="n">inverse_transform</span><span class="p">(</span><span class="n">decoded</span><span class="p">)</span>
+    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">continuous_cols</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+        <span class="c1"># quantized cols to the mid point</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">cols_and_bins</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">verbose</span><span class="p">:</span>
+                <span class="nb">print</span><span class="p">(</span>
+                    <span class="s2">&quot;Note that quantized cols will be turned into the mid point of &quot;</span>
+                    <span class="s2">&quot;the corresponding bin&quot;</span>
+                <span class="p">)</span>
+            <span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">quantizer</span><span class="o">.</span><span class="n">inversed_bins</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
+                <span class="n">decoded</span><span class="p">[</span><span class="n">k</span><span class="p">]</span> <span class="o">=</span> <span class="n">decoded</span><span class="p">[</span><span class="n">k</span><span class="p">]</span><span class="o">.</span><span class="n">map</span><span class="p">(</span><span class="n">v</span><span class="p">)</span>
+        <span class="c1"># continuous_cols back to non-standarised</span>
+        <span class="k">try</span><span class="p">:</span>
+            <span class="n">decoded</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">standardize_cols</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">scaler</span><span class="o">.</span><span class="n">inverse_transform</span><span class="p">(</span>
+                <span class="n">decoded</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">standardize_cols</span><span class="p">]</span>
+            <span class="p">)</span>
+        <span class="k">except</span> <span class="ne">Exception</span><span class="p">:</span>  <span class="c1"># KeyError:</span>
+            <span class="k">pass</span>
+
+    <span class="k">if</span> <span class="s2">&quot;cls_token&quot;</span> <span class="ow">in</span> <span class="n">decoded</span><span class="o">.</span><span class="n">columns</span><span class="p">:</span>
+        <span class="n">decoded</span><span class="o">.</span><span class="n">drop</span><span class="p">(</span><span class="s2">&quot;cls_token&quot;</span><span class="p">,</span> <span class="n">axis</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">inplace</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
+
+    <span class="k">return</span> <span class="n">decoded</span>
 </code></pre></div></td></tr></table></div>
             </details>
     </div>
@@ -3754,233 +4969,317 @@ <h3 id="pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor.transf
 <div class="doc doc-object doc-function">
 
 
-<h3 id="pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor.transform_sample" class="doc doc-heading">
-            <span class="doc doc-object-name doc-function-name">transform_sample</span>
+<h3 id="pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor.fit_transform" class="doc doc-heading">
+            <span class="doc doc-object-name doc-function-name">fit_transform</span>
 
 
-<a href="#pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor.transform_sample" class="headerlink" title="Permanent link">&para;</a></h3>
-<div class="doc-signature highlight"><pre><span></span><code><span class="nf">transform_sample</span><span class="p">(</span><span class="n">text</span><span class="p">)</span>
+<a href="#pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor.fit_transform" class="headerlink" title="Permanent link">&para;</a></h3>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">fit_transform</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
 </code></pre></div>
 
     <div class="doc doc-contents ">
 
-        <p>Returns the padded, <em>'numericalised'</em> sequence</p>
+        <p>Combines <code>fit</code> and <code>transform</code></p>
 
 
 <p><span class="doc-section-title">Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>text</code></b>
-              (<code>str</code>)
-          –
-          <div class="doc-md-description">
-            <p>text to be tokenized and padded</p>
-          </div>
-        </li>
-    </ul>
-
-
-<p><span class="doc-section-title">Returns:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-              <code><span title="numpy.ndarray">ndarray</span></code>
-          –
-          <div class="doc-md-description">
-            <p>Padded, <em>'numericalised'</em> sequence</p>
-          </div>
-        </li>
-    </ul>
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+          <th>Default</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>df</code></td>
+            <td>
+                  <code><span title="pandas.DataFrame">DataFrame</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Input pandas dataframe</p>
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+      </tbody>
+    </table>
+
+
+    <p><span class="doc-section-title">Returns:</span></p>
+    <table>
+      <thead>
+        <tr>
+          <th>Type</th>
+          <th>Description</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td>
+                  <code><span title="numpy.ndarray">ndarray</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>transformed input dataframe</p>
+              </div>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
             <details class="quote">
-              <summary>Source code in <code>pytorch_widedeep/preprocessing/text_preprocessor.py</code></summary>
-              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">162</span>
-<span class="normal">163</span>
-<span class="normal">164</span>
-<span class="normal">165</span>
-<span class="normal">166</span>
-<span class="normal">167</span>
-<span class="normal">168</span>
-<span class="normal">169</span>
-<span class="normal">170</span>
-<span class="normal">171</span>
-<span class="normal">172</span>
-<span class="normal">173</span>
-<span class="normal">174</span>
-<span class="normal">175</span>
-<span class="normal">176</span>
-<span class="normal">177</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">transform_sample</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">text</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">:</span>
-<span class="w">    </span><span class="sd">&quot;&quot;&quot;Returns the padded, _&#39;numericalised&#39;_ sequence</span>
+              <summary>Source code in <code>pytorch_widedeep/preprocessing/tab_preprocessor.py</code></summary>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">476</span>
+<span class="normal">477</span>
+<span class="normal">478</span>
+<span class="normal">479</span>
+<span class="normal">480</span>
+<span class="normal">481</span>
+<span class="normal">482</span>
+<span class="normal">483</span>
+<span class="normal">484</span>
+<span class="normal">485</span>
+<span class="normal">486</span>
+<span class="normal">487</span>
+<span class="normal">488</span>
+<span class="normal">489</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">fit_transform</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">:</span>
+<span class="w">    </span><span class="sd">&quot;&quot;&quot;Combines `fit` and `transform`</span>
 
 <span class="sd">    Parameters</span>
 <span class="sd">    ----------</span>
-<span class="sd">    text: str</span>
-<span class="sd">        text to be tokenized and padded</span>
+<span class="sd">    df: pd.DataFrame</span>
+<span class="sd">        Input pandas dataframe</span>
 
 <span class="sd">    Returns</span>
 <span class="sd">    -------</span>
 <span class="sd">    np.ndarray</span>
-<span class="sd">        Padded, _&#39;numericalised&#39;_ sequence</span>
+<span class="sd">        transformed input dataframe</span>
 <span class="sd">    &quot;&quot;&quot;</span>
-    <span class="n">check_is_fitted</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">attributes</span><span class="o">=</span><span class="p">[</span><span class="s2">&quot;vocab&quot;</span><span class="p">])</span>
-    <span class="n">tokens</span> <span class="o">=</span> <span class="n">get_texts</span><span class="p">([</span><span class="n">text</span><span class="p">],</span> <span class="bp">self</span><span class="o">.</span><span class="n">already_processed</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">n_cpus</span><span class="p">)</span>
-    <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_pad_sequences</span><span class="p">(</span><span class="n">tokens</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span>
+    <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">df</span><span class="p">)</span><span class="o">.</span><span class="n">transform</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
 </code></pre></div></td></tr></table></div>
             </details>
     </div>
 
 </div>
 
-<div class="doc doc-object doc-function">
 
 
-<h3 id="pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor.fit_transform" class="doc doc-heading">
-            <span class="doc doc-object-name doc-function-name">fit_transform</span>
+  </div>
 
+    </div>
 
-<a href="#pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor.fit_transform" class="headerlink" title="Permanent link">&para;</a></h3>
-<div class="doc-signature highlight"><pre><span></span><code><span class="nf">fit_transform</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
-</code></pre></div>
+</div>
 
-    <div class="doc doc-contents ">
+<div class="doc doc-object doc-class">
 
-        <p>Combines <code>fit</code> and <code>transform</code></p>
 
 
-<p><span class="doc-section-title">Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>df</code></b>
-              (<code><span title="pandas.DataFrame">DataFrame</span></code>)
-          –
-          <div class="doc-md-description">
-            <p>Input pandas dataframe</p>
-          </div>
-        </li>
-    </ul>
+<h2 id="pytorch_widedeep.preprocessing.tab_preprocessor.Quantizer" class="doc doc-heading">
+            <span class="doc doc-object-name doc-class-name">Quantizer</span>
 
 
-<p><span class="doc-section-title">Returns:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-              <code><span title="numpy.ndarray">ndarray</span></code>
-          –
-          <div class="doc-md-description">
-            <p>Padded, <em>'numericalised'</em> sequences</p>
-          </div>
-        </li>
-    </ul>
+<a href="#pytorch_widedeep.preprocessing.tab_preprocessor.Quantizer" class="headerlink" title="Permanent link">&para;</a></h2>
 
-            <details class="quote">
-              <summary>Source code in <code>pytorch_widedeep/preprocessing/text_preprocessor.py</code></summary>
-              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">179</span>
-<span class="normal">180</span>
-<span class="normal">181</span>
-<span class="normal">182</span>
-<span class="normal">183</span>
-<span class="normal">184</span>
-<span class="normal">185</span>
-<span class="normal">186</span>
-<span class="normal">187</span>
-<span class="normal">188</span>
-<span class="normal">189</span>
-<span class="normal">190</span>
-<span class="normal">191</span>
-<span class="normal">192</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">fit_transform</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">:</span>
-<span class="w">    </span><span class="sd">&quot;&quot;&quot;Combines `fit` and `transform`</span>
+
+    <div class="doc doc-contents first">
+
+
+        <p>Helper class to perform the quantization of continuous columns. It is
+included in this docs for completion, since depending on the value of the
+parameter <code>'quantization_setup'</code> of the <code>TabPreprocessor</code> class, that
+class might have an attribute of type <code>Quantizer</code>. However, this class is
+designed to always run internally within the <code>TabPreprocessor</code> class.</p>
+
+
+<p><span class="doc-section-title">Parameters:</span></p>
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+          <th>Default</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>quantization_setup</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, <span title="pytorch_widedeep.wdtypes.Union">Union</span>[int, <span title="pytorch_widedeep.wdtypes.List">List</span>[float]]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Dictionary where the keys are the column names to quantize and the
+values are the either integers indicating the number of bins or a
+list of scalars indicating the bin edges.</p>
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+      </tbody>
+    </table>
+
+              <details class="quote">
+                <summary>Source code in <code>pytorch_widedeep/preprocessing/tab_preprocessor.py</code></summary>
+                <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 39</span>
+<span class="normal"> 40</span>
+<span class="normal"> 41</span>
+<span class="normal"> 42</span>
+<span class="normal"> 43</span>
+<span class="normal"> 44</span>
+<span class="normal"> 45</span>
+<span class="normal"> 46</span>
+<span class="normal"> 47</span>
+<span class="normal"> 48</span>
+<span class="normal"> 49</span>
+<span class="normal"> 50</span>
+<span class="normal"> 51</span>
+<span class="normal"> 52</span>
+<span class="normal"> 53</span>
+<span class="normal"> 54</span>
+<span class="normal"> 55</span>
+<span class="normal"> 56</span>
+<span class="normal"> 57</span>
+<span class="normal"> 58</span>
+<span class="normal"> 59</span>
+<span class="normal"> 60</span>
+<span class="normal"> 61</span>
+<span class="normal"> 62</span>
+<span class="normal"> 63</span>
+<span class="normal"> 64</span>
+<span class="normal"> 65</span>
+<span class="normal"> 66</span>
+<span class="normal"> 67</span>
+<span class="normal"> 68</span>
+<span class="normal"> 69</span>
+<span class="normal"> 70</span>
+<span class="normal"> 71</span>
+<span class="normal"> 72</span>
+<span class="normal"> 73</span>
+<span class="normal"> 74</span>
+<span class="normal"> 75</span>
+<span class="normal"> 76</span>
+<span class="normal"> 77</span>
+<span class="normal"> 78</span>
+<span class="normal"> 79</span>
+<span class="normal"> 80</span>
+<span class="normal"> 81</span>
+<span class="normal"> 82</span>
+<span class="normal"> 83</span>
+<span class="normal"> 84</span>
+<span class="normal"> 85</span>
+<span class="normal"> 86</span>
+<span class="normal"> 87</span>
+<span class="normal"> 88</span>
+<span class="normal"> 89</span>
+<span class="normal"> 90</span>
+<span class="normal"> 91</span>
+<span class="normal"> 92</span>
+<span class="normal"> 93</span>
+<span class="normal"> 94</span>
+<span class="normal"> 95</span>
+<span class="normal"> 96</span>
+<span class="normal"> 97</span>
+<span class="normal"> 98</span>
+<span class="normal"> 99</span>
+<span class="normal">100</span>
+<span class="normal">101</span>
+<span class="normal">102</span>
+<span class="normal">103</span>
+<span class="normal">104</span>
+<span class="normal">105</span>
+<span class="normal">106</span>
+<span class="normal">107</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">class</span> <span class="nc">Quantizer</span><span class="p">:</span>
+<span class="w">    </span><span class="sd">&quot;&quot;&quot;Helper class to perform the quantization of continuous columns. It is</span>
+<span class="sd">    included in this docs for completion, since depending on the value of the</span>
+<span class="sd">    parameter `&#39;quantization_setup&#39;` of the `TabPreprocessor` class, that</span>
+<span class="sd">    class might have an attribute of type `Quantizer`. However, this class is</span>
+<span class="sd">    designed to always run internally within the `TabPreprocessor` class.</span>
 
 <span class="sd">    Parameters</span>
 <span class="sd">    ----------</span>
-<span class="sd">    df: pd.DataFrame</span>
-<span class="sd">        Input pandas dataframe</span>
-
-<span class="sd">    Returns</span>
-<span class="sd">    -------</span>
-<span class="sd">    np.ndarray</span>
-<span class="sd">        Padded, _&#39;numericalised&#39;_ sequences</span>
+<span class="sd">    quantization_setup: Dict, default = None</span>
+<span class="sd">        Dictionary where the keys are the column names to quantize and the</span>
+<span class="sd">        values are the either integers indicating the number of bins or a</span>
+<span class="sd">        list of scalars indicating the bin edges.</span>
 <span class="sd">    &quot;&quot;&quot;</span>
-    <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">df</span><span class="p">)</span><span class="o">.</span><span class="n">transform</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
-</code></pre></div></td></tr></table></div>
-            </details>
-    </div>
 
-</div>
+    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span>
+        <span class="n">quantization_setup</span><span class="p">:</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Union</span><span class="p">[</span><span class="nb">int</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="nb">float</span><span class="p">]]],</span>
+        <span class="o">**</span><span class="n">kwargs</span><span class="p">,</span>
+    <span class="p">):</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">quantization_setup</span> <span class="o">=</span> <span class="n">quantization_setup</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">quant_args</span> <span class="o">=</span> <span class="n">kwargs</span>
 
-<div class="doc doc-object doc-function">
+        <span class="bp">self</span><span class="o">.</span><span class="n">is_fitted</span> <span class="o">=</span> <span class="kc">False</span>
 
+    <span class="k">def</span> <span class="nf">fit</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="s2">&quot;Quantizer&quot;</span><span class="p">:</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">bins</span><span class="p">:</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="nb">float</span><span class="p">]]</span> <span class="o">=</span> <span class="p">{}</span>
+        <span class="k">for</span> <span class="n">col</span><span class="p">,</span> <span class="n">bins</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">quantization_setup</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
+            <span class="n">_</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">bins</span><span class="p">[</span><span class="n">col</span><span class="p">]</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">cut</span><span class="p">(</span>
+                <span class="n">df</span><span class="p">[</span><span class="n">col</span><span class="p">],</span> <span class="n">bins</span><span class="p">,</span> <span class="n">retbins</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">labels</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="o">**</span><span class="bp">self</span><span class="o">.</span><span class="n">quant_args</span>
+            <span class="p">)</span>
 
-<h3 id="pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor.inverse_transform" class="doc doc-heading">
-            <span class="doc doc-object-name doc-function-name">inverse_transform</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">inversed_bins</span><span class="p">:</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">int</span><span class="p">,</span> <span class="nb">float</span><span class="p">]]</span> <span class="o">=</span> <span class="p">{}</span>
+        <span class="k">for</span> <span class="n">col</span><span class="p">,</span> <span class="n">bins</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">bins</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">inversed_bins</span><span class="p">[</span><span class="n">col</span><span class="p">]</span> <span class="o">=</span> <span class="p">{</span>
+                <span class="n">k</span><span class="p">:</span> <span class="n">v</span>
+                <span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="nb">list</span><span class="p">(</span>
+                    <span class="nb">zip</span><span class="p">(</span>
+                        <span class="nb">range</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="nb">len</span><span class="p">(</span><span class="n">bins</span><span class="p">)</span> <span class="o">+</span> <span class="mi">1</span><span class="p">),</span>
+                        <span class="p">[(</span><span class="n">a</span> <span class="o">+</span> <span class="n">b</span><span class="p">)</span> <span class="o">/</span> <span class="mf">2.0</span> <span class="k">for</span> <span class="n">a</span><span class="p">,</span> <span class="n">b</span> <span class="ow">in</span> <span class="nb">zip</span><span class="p">(</span><span class="n">bins</span><span class="p">,</span> <span class="n">bins</span><span class="p">[</span><span class="mi">1</span><span class="p">:])],</span>
+                    <span class="p">)</span>
+                <span class="p">)</span>
+            <span class="p">}</span>
+            <span class="c1"># 0</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">inversed_bins</span><span class="p">[</span><span class="n">col</span><span class="p">][</span><span class="mi">0</span><span class="p">]</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">nan</span>
 
+        <span class="bp">self</span><span class="o">.</span><span class="n">is_fitted</span> <span class="o">=</span> <span class="kc">True</span>
 
-<a href="#pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor.inverse_transform" class="headerlink" title="Permanent link">&para;</a></h3>
-<div class="doc-signature highlight"><pre><span></span><code><span class="nf">inverse_transform</span><span class="p">(</span><span class="n">padded_seq</span><span class="p">)</span>
-</code></pre></div>
+        <span class="k">return</span> <span class="bp">self</span>
 
-    <div class="doc doc-contents ">
+    <span class="k">def</span> <span class="nf">transform</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">:</span>
+        <span class="n">check_is_fitted</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">condition</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">is_fitted</span><span class="p">)</span>
 
-        <p>Returns the original text plus the added 'special' tokens</p>
+        <span class="n">dfc</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
+        <span class="k">for</span> <span class="n">col</span><span class="p">,</span> <span class="n">bins</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">bins</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
+            <span class="n">dfc</span><span class="p">[</span><span class="n">col</span><span class="p">]</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">cut</span><span class="p">(</span><span class="n">dfc</span><span class="p">[</span><span class="n">col</span><span class="p">],</span> <span class="n">bins</span><span class="p">,</span> <span class="n">labels</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="o">**</span><span class="bp">self</span><span class="o">.</span><span class="n">quant_args</span><span class="p">)</span>
+            <span class="c1"># 0 will be left for numbers outside the bins, i.e. smaller than</span>
+            <span class="c1"># the smaller boundary or larger than the largest boundary</span>
+            <span class="n">dfc</span><span class="p">[</span><span class="n">col</span><span class="p">]</span> <span class="o">=</span> <span class="n">dfc</span><span class="p">[</span><span class="n">col</span><span class="p">]</span> <span class="o">+</span> <span class="mi">1</span>
+            <span class="n">dfc</span><span class="p">[</span><span class="n">col</span><span class="p">]</span> <span class="o">=</span> <span class="n">dfc</span><span class="p">[</span><span class="n">col</span><span class="p">]</span><span class="o">.</span><span class="n">fillna</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span>
+            <span class="n">dfc</span><span class="p">[</span><span class="n">col</span><span class="p">]</span> <span class="o">=</span> <span class="n">dfc</span><span class="p">[</span><span class="n">col</span><span class="p">]</span><span class="o">.</span><span class="n">astype</span><span class="p">(</span><span class="nb">int</span><span class="p">)</span>
+
+        <span class="k">return</span> <span class="n">dfc</span>
+
+    <span class="k">def</span> <span class="nf">fit_transform</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">:</span>
+        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">df</span><span class="p">)</span><span class="o">.</span><span class="n">transform</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="fm">__repr__</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">str</span><span class="p">:</span>
+        <span class="k">return</span> <span class="sa">f</span><span class="s2">&quot;Quantizer(quantization_setup=</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">quantization_setup</span><span class="si">}</span><span class="s2">)&quot;</span>
+</code></pre></div></td></tr></table></div>
+              </details>
+
+
+
+  <div class="doc doc-children">
 
 
-<p><span class="doc-section-title">Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>padded_seq</code></b>
-              (<code><span title="numpy.ndarray">ndarray</span></code>)
-          –
-          <div class="doc-md-description">
-            <p>array with the output of the <code>transform</code> method</p>
-          </div>
-        </li>
-    </ul>
 
 
-<p><span class="doc-section-title">Returns:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-              <code><span title="pandas.DataFrame">DataFrame</span></code>
-          –
-          <div class="doc-md-description">
-            <p>Pandas dataframe with the original text plus the added 'special' tokens</p>
-          </div>
-        </li>
-    </ul>
 
-            <details class="quote">
-              <summary>Source code in <code>pytorch_widedeep/preprocessing/text_preprocessor.py</code></summary>
-              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">194</span>
-<span class="normal">195</span>
-<span class="normal">196</span>
-<span class="normal">197</span>
-<span class="normal">198</span>
-<span class="normal">199</span>
-<span class="normal">200</span>
-<span class="normal">201</span>
-<span class="normal">202</span>
-<span class="normal">203</span>
-<span class="normal">204</span>
-<span class="normal">205</span>
-<span class="normal">206</span>
-<span class="normal">207</span>
-<span class="normal">208</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">inverse_transform</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">padded_seq</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">:</span>
-<span class="w">    </span><span class="sd">&quot;&quot;&quot;Returns the original text plus the added &#39;special&#39; tokens</span>
 
-<span class="sd">    Parameters</span>
-<span class="sd">    ----------</span>
-<span class="sd">    padded_seq: np.ndarray</span>
-<span class="sd">        array with the output of the `transform` method</span>
 
-<span class="sd">    Returns</span>
-<span class="sd">    -------</span>
-<span class="sd">    pd.DataFrame</span>
-<span class="sd">        Pandas dataframe with the original text plus the added &#39;special&#39; tokens</span>
-<span class="sd">    &quot;&quot;&quot;</span>
-    <span class="n">texts</span> <span class="o">=</span> <span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">vocab</span><span class="o">.</span><span class="n">inverse_transform</span><span class="p">(</span><span class="n">num</span><span class="p">)</span> <span class="k">for</span> <span class="n">num</span> <span class="ow">in</span> <span class="n">padded_seq</span><span class="p">]</span>
-    <span class="k">return</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">({</span><span class="bp">self</span><span class="o">.</span><span class="n">text_col</span><span class="p">:</span> <span class="n">texts</span><span class="p">})</span>
-</code></pre></div></td></tr></table></div>
-            </details>
-    </div>
 
-</div>
 
 
 
@@ -3994,167 +5293,299 @@ <h3 id="pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor.invers
 
 
 
-<h2 id="pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor" class="doc doc-heading">
-            <span class="doc doc-object-name doc-class-name">HFPreprocessor</span>
+<h2 id="pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor" class="doc doc-heading">
+            <span class="doc doc-object-name doc-class-name">TextPreprocessor</span>
 
 
-<a href="#pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="nf">HFPreprocessor</span><span class="p">(</span>
-    <span class="n">model_name</span><span class="p">,</span>
-    <span class="o">*</span><span class="p">,</span>
-    <span class="n">use_fast_tokenizer</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
-    <span class="n">text_col</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">root_dir</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">num_workers</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">preprocessing_rules</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">tokenizer_params</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">encode_params</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="o">**</span><span class="n">kwargs</span>
-<span class="p">)</span>
-</code></pre></div>
+<a href="#pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor" class="headerlink" title="Permanent link">&para;</a></h2>
+
 
     <div class="doc doc-contents first">
             <p class="doc doc-class-bases">
               Bases: <code><span title="pytorch_widedeep.preprocessing.base_preprocessor.BasePreprocessor">BasePreprocessor</span></code></p>
 
 
-        <p>Text processor to prepare the <code>deeptext</code> input dataset that is a
-wrapper around HuggingFace's tokenizers.</p>
-<p>Following the main phylosophy of the <code>pytorch-widedeep</code> library, this
-class is designed to be as flexible as possible. Therefore, it is coded
-so that the user can use it as one would use any HuggingFace tokenizers,
-or following the API call 'protocol' of the rest of the library.</p>
+        <p>Preprocessor to prepare the <code>deeptext</code> input dataset</p>
 
 
 <p><span class="doc-section-title">Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>model_name</code></b>
-              (<code>str</code>)
-          –
-          <div class="doc-md-description">
-            <p>The model name from the transformers library e.g. <em>'bert-base-uncased'</em>.
-Currently supported models are those from the families: BERT, RoBERTa,
-DistilBERT, ALBERT and ELECTRA.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>use_fast_tokenizer</code></b>
-              (<code>bool</code>, default:
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+          <th>Default</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>text_col</code></td>
+            <td>
+                  <code>str</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>column in the input dataframe containing the texts</p>
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>max_vocab</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Maximum number of tokens in the vocabulary</p>
+              </div>
+            </td>
+            <td>
+                  <code>30000</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>min_freq</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Minimum frequency for a token to be part of the vocabulary</p>
+              </div>
+            </td>
+            <td>
+                  <code>5</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>maxlen</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Maximum length of the tokenized sequences</p>
+              </div>
+            </td>
+            <td>
+                  <code>80</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>pad_first</code></td>
+            <td>
+                  <code>bool</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Indicates whether the padding index will be added at the beginning or the
+end of the sequences</p>
+              </div>
+            </td>
+            <td>
+                  <code>True</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>pad_idx</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>padding index. Fastai's Tokenizer leaves 0 for the 'unknown' token.</p>
+              </div>
+            </td>
+            <td>
+                  <code>1</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>already_processed</code></td>
+            <td>
+                  <code><span title="typing.Optional">Optional</span>[bool]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Boolean indicating if the sequence of elements is already processed or
+prepared. If this is the case, this Preprocessor will simply tokenize
+and pad the sequence. <br/></p>
+<div class="highlight"><pre><span></span><code>Param aliases: `not_text`. &lt;br/&gt;
+</code></pre></div>
+<p>This parameter is thought for those cases where the input sequences
+are already fully processed or are directly not text (e.g. IDs)</p>
+              </div>
+            </td>
+            <td>
                   <code>False</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Whether to use the fast tokenizer from HuggingFace or not</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>text_col</code></b>
-              (<code><span title="typing.Optional">Optional</span>[str]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>The column in the input dataframe containing the text data. If this
-tokenizer is used via the <code>fit</code> and <code>transform</code> methods, this
-argument is mandatory. If the tokenizer is used via the <code>encode</code>
-method, this argument is not needed since the input text is passed
-directly to the <code>encode</code> method.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>num_workers</code></b>
-              (<code><span title="typing.Optional">Optional</span>[int]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Number of workers to use when preprocessing the text data. If not
-None, and <code>use_fast_tokenizer</code> is False, the text data will be
-preprocessed in parallel using the number of workers specified. If
-<code>use_fast_tokenizer</code> is True, this argument is ignored.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>preprocessing_rules</code></b>
-              (<code><span title="typing.Optional">Optional</span>[<span title="typing.List">List</span>[<span title="typing.Callable">Callable</span>[[str], str]]]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>A list of functions to be applied to the text data before encoding.
-This can be useful to clean the text data before encoding. For
-example, removing html tags, special characters, etc.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>tokenizer_params</code></b>
-              (<code><span title="typing.Optional">Optional</span>[<span title="typing.Dict">Dict</span>[str, <span title="typing.Any">Any</span>]]</code>, default:
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>word_vectors_path</code></td>
+            <td>
+                  <code><span title="typing.Optional">Optional</span>[str]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Path to the pretrained word vectors</p>
+              </div>
+            </td>
+            <td>
                   <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Additional parameters to be passed to the HuggingFace's
-<code>PreTrainedTokenizer</code>. Parameters to the <code>PreTrainedTokenizer</code>
-can also be passed via the <code>**kwargs</code> argument</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>encode_params</code></b>
-              (<code><span title="typing.Optional">Optional</span>[<span title="typing.Dict">Dict</span>[str, <span title="typing.Any">Any</span>]]</code>, default:
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>n_cpus</code></td>
+            <td>
+                  <code><span title="typing.Optional">Optional</span>[int]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>number of CPUs to used during the tokenization process</p>
+              </div>
+            </td>
+            <td>
                   <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Additional parameters to be passed to the <code>batch_encode_plus</code> method
-of the HuggingFace's <code>PreTrainedTokenizer</code>. If the <code>fit</code> and <code>transform</code>
-methods are used, the <code>encode_params</code> dict parameter is mandatory. If
-the <code>encode</code> method is used, this parameter is not needed since the
-input text is passed directly to the <code>encode</code> method.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>**kwargs</code></b>
-          –
-          <div class="doc-md-description">
-            <p>Additional kwargs to be passed to the model, in particular to the
-<code>PreTrainedTokenizer</code> class.</p>
-          </div>
-        </li>
-    </ul>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>verbose</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Enable verbose output.</p>
+              </div>
+            </td>
+            <td>
+                  <code>1</code>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
 
 <p><span class="doc-section-title">Attributes:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code><span title="pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.is_fitted">is_fitted</span></code></b>
-              (<code>bool</code>)
-          –
-          <div class="doc-md-description">
-            <p>Boolean indicating if the preprocessor has been fitted. This is a
-HuggingFacea tokenizer, so it is always considered fitted and this
-attribute is manually set to True internally. This parameter exists
-for consistency with the rest of the library and because is needed
-for some functionality in the library.</p>
-          </div>
-        </li>
-    </ul>
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code><span title="pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor.vocab">vocab</span></code></td>
+            <td>
+                  <code><a class="autorefs autorefs-internal" title="pytorch_widedeep.utils.fastai_transforms.Vocab" href="utils/fastai_transforms.html#pytorch_widedeep.utils.fastai_transforms.Vocab">Vocab</a></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>an instance of <code>pytorch_widedeep.utils.fastai_transforms.Vocab</code></p>
+              </div>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code><span title="pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor.embedding_matrix">embedding_matrix</span></code></td>
+            <td>
+                  <code><span title="numpy.ndarray">ndarray</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Array with the pretrained embeddings</p>
+              </div>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
 
 <p><span class="doc-section-title">Examples:</span></p>
     <div class="highlight"><pre><span></span><code><span class="gp">&gt;&gt;&gt; </span><span class="kn">import</span> <span class="nn">pandas</span> <span class="k">as</span> <span class="nn">pd</span>
-<span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">pytorch_widedeep.preprocessing</span> <span class="kn">import</span> <span class="n">HFPreprocessor</span>
-<span class="gp">&gt;&gt;&gt; </span><span class="n">df</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">({</span><span class="s2">&quot;text&quot;</span><span class="p">:</span> <span class="p">[</span><span class="s2">&quot;this is the first text&quot;</span><span class="p">,</span> <span class="s2">&quot;this is the second text&quot;</span><span class="p">]})</span>
-<span class="gp">&gt;&gt;&gt; </span><span class="n">hf_processor_1</span> <span class="o">=</span> <span class="n">HFPreprocessor</span><span class="p">(</span><span class="n">model_name</span><span class="o">=</span><span class="s2">&quot;bert-base-uncased&quot;</span><span class="p">,</span> <span class="n">text_col</span><span class="o">=</span><span class="s2">&quot;text&quot;</span><span class="p">)</span>
-<span class="gp">&gt;&gt;&gt; </span><span class="n">X_text_1</span> <span class="o">=</span> <span class="n">hf_processor_1</span><span class="o">.</span><span class="n">fit_transform</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
-<span class="gp">&gt;&gt;&gt; </span><span class="n">texts</span> <span class="o">=</span> <span class="p">[</span><span class="s2">&quot;this is a new text&quot;</span><span class="p">,</span> <span class="s2">&quot;this is another text&quot;</span><span class="p">]</span>
-<span class="gp">&gt;&gt;&gt; </span><span class="n">hf_processor_2</span> <span class="o">=</span> <span class="n">HFPreprocessor</span><span class="p">(</span><span class="n">model_name</span><span class="o">=</span><span class="s2">&quot;bert-base-uncased&quot;</span><span class="p">)</span>
-<span class="gp">&gt;&gt;&gt; </span><span class="n">X_text_2</span> <span class="o">=</span> <span class="n">hf_processor_2</span><span class="o">.</span><span class="n">encode</span><span class="p">(</span><span class="n">texts</span><span class="p">,</span> <span class="n">max_length</span><span class="o">=</span><span class="mi">10</span><span class="p">,</span> <span class="n">padding</span><span class="o">=</span><span class="s2">&quot;max_length&quot;</span><span class="p">,</span> <span class="n">truncation</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
+<span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">pytorch_widedeep.preprocessing</span> <span class="kn">import</span> <span class="n">TextPreprocessor</span>
+<span class="gp">&gt;&gt;&gt; </span><span class="n">df_train</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">({</span><span class="s1">&#39;text_column&#39;</span><span class="p">:</span> <span class="p">[</span><span class="s2">&quot;life is like a box of chocolates&quot;</span><span class="p">,</span>
+<span class="gp">... </span><span class="s2">&quot;You never know what you&#39;re gonna get&quot;</span><span class="p">]})</span>
+<span class="gp">&gt;&gt;&gt; </span><span class="n">text_preprocessor</span> <span class="o">=</span> <span class="n">TextPreprocessor</span><span class="p">(</span><span class="n">text_col</span><span class="o">=</span><span class="s1">&#39;text_column&#39;</span><span class="p">,</span> <span class="n">max_vocab</span><span class="o">=</span><span class="mi">25</span><span class="p">,</span> <span class="n">min_freq</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">maxlen</span><span class="o">=</span><span class="mi">10</span><span class="p">)</span>
+<span class="gp">&gt;&gt;&gt; </span><span class="n">text_preprocessor</span><span class="o">.</span><span class="n">fit_transform</span><span class="p">(</span><span class="n">df_train</span><span class="p">)</span>
+<span class="go">The vocabulary contains 24 tokens</span>
+<span class="go">array([[ 1,  1,  1,  1, 10, 11, 12, 13, 14, 15],</span>
+<span class="go">       [ 5,  9, 16, 17, 18,  9, 19, 20, 21, 22]], dtype=int32)</span>
+<span class="gp">&gt;&gt;&gt; </span><span class="n">df_te</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">({</span><span class="s1">&#39;text_column&#39;</span><span class="p">:</span> <span class="p">[</span><span class="s1">&#39;you never know what is in the box&#39;</span><span class="p">]})</span>
+<span class="gp">&gt;&gt;&gt; </span><span class="n">text_preprocessor</span><span class="o">.</span><span class="n">transform</span><span class="p">(</span><span class="n">df_te</span><span class="p">)</span>
+<span class="go">array([[ 1,  1,  9, 16, 17, 18, 11,  0,  0, 13]], dtype=int32)</span>
 </code></pre></div>
 
-                  <details class="quote">
-                    <summary>Source code in <code>pytorch_widedeep/preprocessing/hf_preprocessor.py</code></summary>
-                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 84</span>
+              <details class="quote">
+                <summary>Source code in <code>pytorch_widedeep/preprocessing/text_preprocessor.py</code></summary>
+                <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 22</span>
+<span class="normal"> 23</span>
+<span class="normal"> 24</span>
+<span class="normal"> 25</span>
+<span class="normal"> 26</span>
+<span class="normal"> 27</span>
+<span class="normal"> 28</span>
+<span class="normal"> 29</span>
+<span class="normal"> 30</span>
+<span class="normal"> 31</span>
+<span class="normal"> 32</span>
+<span class="normal"> 33</span>
+<span class="normal"> 34</span>
+<span class="normal"> 35</span>
+<span class="normal"> 36</span>
+<span class="normal"> 37</span>
+<span class="normal"> 38</span>
+<span class="normal"> 39</span>
+<span class="normal"> 40</span>
+<span class="normal"> 41</span>
+<span class="normal"> 42</span>
+<span class="normal"> 43</span>
+<span class="normal"> 44</span>
+<span class="normal"> 45</span>
+<span class="normal"> 46</span>
+<span class="normal"> 47</span>
+<span class="normal"> 48</span>
+<span class="normal"> 49</span>
+<span class="normal"> 50</span>
+<span class="normal"> 51</span>
+<span class="normal"> 52</span>
+<span class="normal"> 53</span>
+<span class="normal"> 54</span>
+<span class="normal"> 55</span>
+<span class="normal"> 56</span>
+<span class="normal"> 57</span>
+<span class="normal"> 58</span>
+<span class="normal"> 59</span>
+<span class="normal"> 60</span>
+<span class="normal"> 61</span>
+<span class="normal"> 62</span>
+<span class="normal"> 63</span>
+<span class="normal"> 64</span>
+<span class="normal"> 65</span>
+<span class="normal"> 66</span>
+<span class="normal"> 67</span>
+<span class="normal"> 68</span>
+<span class="normal"> 69</span>
+<span class="normal"> 70</span>
+<span class="normal"> 71</span>
+<span class="normal"> 72</span>
+<span class="normal"> 73</span>
+<span class="normal"> 74</span>
+<span class="normal"> 75</span>
+<span class="normal"> 76</span>
+<span class="normal"> 77</span>
+<span class="normal"> 78</span>
+<span class="normal"> 79</span>
+<span class="normal"> 80</span>
+<span class="normal"> 81</span>
+<span class="normal"> 82</span>
+<span class="normal"> 83</span>
+<span class="normal"> 84</span>
 <span class="normal"> 85</span>
 <span class="normal"> 86</span>
 <span class="normal"> 87</span>
@@ -4191,114 +5622,9 @@ <h2 id="pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor" class="do
 <span class="normal">118</span>
 <span class="normal">119</span>
 <span class="normal">120</span>
-<span class="normal">121</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
-    <span class="bp">self</span><span class="p">,</span>
-    <span class="n">model_name</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
-    <span class="o">*</span><span class="p">,</span>
-    <span class="n">use_fast_tokenizer</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
-    <span class="n">text_col</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">root_dir</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">num_workers</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">preprocessing_rules</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="n">Callable</span><span class="p">[[</span><span class="nb">str</span><span class="p">],</span> <span class="nb">str</span><span class="p">]]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">tokenizer_params</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">encode_params</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="o">**</span><span class="n">kwargs</span><span class="p">,</span>
-<span class="p">):</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">model_name</span> <span class="o">=</span> <span class="n">model_name</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">use_fast_tokenizer</span> <span class="o">=</span> <span class="n">use_fast_tokenizer</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">text_col</span> <span class="o">=</span> <span class="n">text_col</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">root_dir</span> <span class="o">=</span> <span class="n">root_dir</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">num_workers</span> <span class="o">=</span> <span class="n">num_workers</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">preprocessing_rules</span> <span class="o">=</span> <span class="n">preprocessing_rules</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">tokenizer_params</span> <span class="o">=</span> <span class="n">tokenizer_params</span> <span class="k">if</span> <span class="n">tokenizer_params</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="k">else</span> <span class="p">{}</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">encode_params</span> <span class="o">=</span> <span class="n">encode_params</span> <span class="k">if</span> <span class="n">encode_params</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="k">else</span> <span class="p">{}</span>
-
-    <span class="bp">self</span><span class="o">.</span><span class="n">_multiprocessing</span> <span class="o">=</span> <span class="p">(</span>
-        <span class="n">num_workers</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">and</span> <span class="n">num_workers</span> <span class="o">&gt;</span> <span class="mi">1</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">use_fast_tokenizer</span>
-    <span class="p">)</span>
-
-    <span class="k">if</span> <span class="n">kwargs</span><span class="p">:</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">tokenizer_params</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="n">kwargs</span><span class="p">)</span>
-
-    <span class="bp">self</span><span class="o">.</span><span class="n">tokenizer</span> <span class="o">=</span> <span class="n">get_tokenizer</span><span class="p">(</span>
-        <span class="n">model_name</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">model_name</span><span class="p">,</span>
-        <span class="n">use_fast_tokenizer</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">use_fast_tokenizer</span><span class="p">,</span>
-        <span class="o">**</span><span class="bp">self</span><span class="o">.</span><span class="n">tokenizer_params</span><span class="p">,</span>
-    <span class="p">)</span>
-
-    <span class="c1"># A HuggingFace tokenizer is already trained, since we need this</span>
-    <span class="c1"># attribute elsewhere in the library, we simply set it to True</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">is_fitted</span> <span class="o">=</span> <span class="kc">True</span>
-</code></pre></div></td></tr></table></div>
-                  </details>
-
-
-
-  <div class="doc doc-children">
-
-
-
-
-
-
-
-
-
-<div class="doc doc-object doc-function">
-
-
-<h3 id="pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.encode" class="doc doc-heading">
-            <span class="doc doc-object-name doc-function-name">encode</span>
-
-
-<a href="#pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.encode" class="headerlink" title="Permanent link">&para;</a></h3>
-<div class="doc-signature highlight"><pre><span></span><code><span class="nf">encode</span><span class="p">(</span><span class="n">texts</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
-</code></pre></div>
-
-    <div class="doc doc-contents ">
-
-        <p>Encodes a list of texts. The method is a wrapper around the
-<code>batch_encode_plus</code> method of the HuggingFace's tokenizer.</p>
-<p>if 'use_fast_tokenizer' is True, the method will use the <code>batch_encode_plus</code></p>
-
-
-<p><span class="doc-section-title">Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>texts</code></b>
-              (<code><span title="typing.List">List</span>[str]</code>)
-          –
-          <div class="doc-md-description">
-            <p>List of texts to be encoded</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>**kwargs</code></b>
-          –
-          <div class="doc-md-description">
-            <p>Additional parameters to be passed to the <code>batch_encode_plus</code> method
-of the HuggingFace's tokenizer. If the 'encode_params' dict was passed
-when instantiating the class, that dictionaly will be updated with
-the kwargs passed here.</p>
-          </div>
-        </li>
-    </ul>
-
-
-<p><span class="doc-section-title">Returns:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-              <code><span title="numpy.array">array</span></code>
-          –
-          <div class="doc-md-description">
-            <p>The encoded texts</p>
-          </div>
-        </li>
-    </ul>
-
-            <details class="quote">
-              <summary>Source code in <code>pytorch_widedeep/preprocessing/hf_preprocessor.py</code></summary>
-              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">123</span>
+<span class="normal">121</span>
+<span class="normal">122</span>
+<span class="normal">123</span>
 <span class="normal">124</span>
 <span class="normal">125</span>
 <span class="normal">126</span>
@@ -4357,125 +5683,9 @@ <h3 id="pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.encode" cl
 <span class="normal">179</span>
 <span class="normal">180</span>
 <span class="normal">181</span>
-<span class="normal">182</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">encode</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">texts</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">],</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">npt</span><span class="o">.</span><span class="n">NDArray</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">int64</span><span class="p">]:</span>
-<span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
-<span class="sd">    Encodes a list of texts. The method is a wrapper around the</span>
-<span class="sd">    `batch_encode_plus` method of the HuggingFace&#39;s tokenizer.</span>
-
-<span class="sd">    if &#39;use_fast_tokenizer&#39; is True, the method will use the `batch_encode_plus`</span>
-
-<span class="sd">    Parameters</span>
-<span class="sd">    ----------</span>
-<span class="sd">    texts: List[str]</span>
-<span class="sd">        List of texts to be encoded</span>
-<span class="sd">    **kwargs</span>
-<span class="sd">        Additional parameters to be passed to the `batch_encode_plus` method</span>
-<span class="sd">        of the HuggingFace&#39;s tokenizer. If the &#39;encode_params&#39; dict was passed</span>
-<span class="sd">        when instantiating the class, that dictionaly will be updated with</span>
-<span class="sd">        the kwargs passed here.</span>
-
-<span class="sd">    Returns</span>
-<span class="sd">    -------</span>
-<span class="sd">    np.array</span>
-<span class="sd">        The encoded texts</span>
-<span class="sd">    &quot;&quot;&quot;</span>
-    <span class="k">if</span> <span class="n">kwargs</span><span class="p">:</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">encode_params</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="n">kwargs</span><span class="p">)</span>
-
-    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">preprocessing_rules</span><span class="p">:</span>
-        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_multiprocessing</span><span class="p">:</span>
-            <span class="n">texts</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_process_text_parallel</span><span class="p">(</span><span class="n">texts</span><span class="p">)</span>
-        <span class="k">else</span><span class="p">:</span>
-            <span class="n">texts</span> <span class="o">=</span> <span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">_preprocess_text</span><span class="p">(</span><span class="n">text</span><span class="p">)</span> <span class="k">for</span> <span class="n">text</span> <span class="ow">in</span> <span class="n">texts</span><span class="p">]</span>
-
-    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_multiprocessing</span><span class="p">:</span>
-        <span class="n">input_ids</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_encode_paralell</span><span class="p">(</span><span class="n">texts</span><span class="p">,</span> <span class="o">**</span><span class="bp">self</span><span class="o">.</span><span class="n">encode_params</span><span class="p">)</span>
-    <span class="k">else</span><span class="p">:</span>
-        <span class="n">encoded_texts</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">tokenizer</span><span class="o">.</span><span class="n">batch_encode_plus</span><span class="p">(</span>
-            <span class="n">texts</span><span class="p">,</span>
-            <span class="o">**</span><span class="bp">self</span><span class="o">.</span><span class="n">encode_params</span><span class="p">,</span>
-        <span class="p">)</span>
-        <span class="n">input_ids</span> <span class="o">=</span> <span class="n">encoded_texts</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;input_ids&quot;</span><span class="p">)</span>
-
-    <span class="bp">self</span><span class="o">.</span><span class="n">is_fitted</span> <span class="o">=</span> <span class="kc">True</span>
-
-    <span class="k">try</span><span class="p">:</span>
-        <span class="n">output</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">(</span><span class="n">input_ids</span><span class="p">)</span>
-    <span class="k">except</span> <span class="ne">ValueError</span><span class="p">:</span>
-        <span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span>
-            <span class="s2">&quot;Padding and Truncating parameters were not passed and all input arrays &quot;</span>
-            <span class="s2">&quot;do not have the same shape. Padding to the longest sequence. &quot;</span>
-            <span class="s2">&quot;Padding will be done with the index of the pad token for the model&quot;</span><span class="p">,</span>
-            <span class="ne">UserWarning</span><span class="p">,</span>
-        <span class="p">)</span>
-        <span class="n">max_len</span> <span class="o">=</span> <span class="nb">max</span><span class="p">([</span><span class="nb">len</span><span class="p">(</span><span class="n">ids</span><span class="p">)</span> <span class="k">for</span> <span class="n">ids</span> <span class="ow">in</span> <span class="n">input_ids</span><span class="p">])</span>
-        <span class="n">output</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">(</span>
-            <span class="p">[</span>
-                <span class="n">np</span><span class="o">.</span><span class="n">pad</span><span class="p">(</span><span class="n">ids</span><span class="p">,</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">tokenizer</span><span class="o">.</span><span class="n">pad_token_id</span><span class="p">,</span> <span class="n">max_len</span> <span class="o">-</span> <span class="nb">len</span><span class="p">(</span><span class="n">ids</span><span class="p">)))</span>
-                <span class="k">for</span> <span class="n">ids</span> <span class="ow">in</span> <span class="n">input_ids</span>
-            <span class="p">]</span>
-        <span class="p">)</span>
-
-    <span class="k">return</span> <span class="n">output</span>
-</code></pre></div></td></tr></table></div>
-            </details>
-    </div>
-
-</div>
-
-<div class="doc doc-object doc-function">
-
-
-<h3 id="pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.decode" class="doc doc-heading">
-            <span class="doc doc-object-name doc-function-name">decode</span>
-
-
-<a href="#pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.decode" class="headerlink" title="Permanent link">&para;</a></h3>
-<div class="doc-signature highlight"><pre><span></span><code><span class="nf">decode</span><span class="p">(</span><span class="n">input_ids</span><span class="p">,</span> <span class="n">skip_special_tokens</span><span class="p">)</span>
-</code></pre></div>
-
-    <div class="doc doc-contents ">
-
-        <p>Decodes a list of input_ids. The method is a wrapper around the
-<code>convert_ids_to_tokens</code> and <code>convert_tokens_to_string</code> methods of the
-HuggingFace's tokenizer.</p>
-
-
-<p><span class="doc-section-title">Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>input_ids</code></b>
-              (<code><span title="numpy.typing.NDArray">NDArray</span>[<span title="numpy.int64">int64</span>]</code>)
-          –
-          <div class="doc-md-description">
-            <p>The input_ids to be decoded</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>skip_special_tokens</code></b>
-              (<code>bool</code>)
-          –
-          <div class="doc-md-description">
-            <p>Whether to skip the special tokens or not</p>
-          </div>
-        </li>
-    </ul>
-
-
-<p><span class="doc-section-title">Returns:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-              <code><span title="typing.List">List</span>[str]</code>
-          –
-          <div class="doc-md-description">
-            <p>The decoded texts</p>
-          </div>
-        </li>
-    </ul>
-
-            <details class="quote">
-              <summary>Source code in <code>pytorch_widedeep/preprocessing/hf_preprocessor.py</code></summary>
-              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">184</span>
+<span class="normal">182</span>
+<span class="normal">183</span>
+<span class="normal">184</span>
 <span class="normal">185</span>
 <span class="normal">186</span>
 <span class="normal">187</span>
@@ -4501,75 +5711,9 @@ <h3 id="pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.decode" cl
 <span class="normal">207</span>
 <span class="normal">208</span>
 <span class="normal">209</span>
-<span class="normal">210</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">decode</span><span class="p">(</span>
-    <span class="bp">self</span><span class="p">,</span> <span class="n">input_ids</span><span class="p">:</span> <span class="n">npt</span><span class="o">.</span><span class="n">NDArray</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">int64</span><span class="p">],</span> <span class="n">skip_special_tokens</span><span class="p">:</span> <span class="nb">bool</span>
-<span class="p">)</span> <span class="o">-&gt;</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]:</span>
-<span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
-<span class="sd">    Decodes a list of input_ids. The method is a wrapper around the</span>
-<span class="sd">    `convert_ids_to_tokens` and `convert_tokens_to_string` methods of the</span>
-<span class="sd">    HuggingFace&#39;s tokenizer.</span>
-
-<span class="sd">    Parameters</span>
-<span class="sd">    ----------</span>
-<span class="sd">    input_ids: npt.NDArray[np.int64]</span>
-<span class="sd">        The input_ids to be decoded</span>
-<span class="sd">    skip_special_tokens: bool</span>
-<span class="sd">        Whether to skip the special tokens or not</span>
-
-<span class="sd">    Returns</span>
-<span class="sd">    -------</span>
-<span class="sd">    List[str]</span>
-<span class="sd">        The decoded texts</span>
-<span class="sd">    &quot;&quot;&quot;</span>
-    <span class="n">texts</span> <span class="o">=</span> <span class="p">[</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">tokenizer</span><span class="o">.</span><span class="n">convert_tokens_to_string</span><span class="p">(</span>
-            <span class="bp">self</span><span class="o">.</span><span class="n">tokenizer</span><span class="o">.</span><span class="n">convert_ids_to_tokens</span><span class="p">(</span><span class="n">input_ids</span><span class="p">[</span><span class="n">i</span><span class="p">],</span> <span class="n">skip_special_tokens</span><span class="p">)</span>
-        <span class="p">)</span>
-        <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">input_ids</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">0</span><span class="p">])</span>
-    <span class="p">]</span>
-    <span class="k">return</span> <span class="n">texts</span>
-</code></pre></div></td></tr></table></div>
-            </details>
-    </div>
-
-</div>
-
-<div class="doc doc-object doc-function">
-
-
-<h3 id="pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.fit" class="doc doc-heading">
-            <span class="doc doc-object-name doc-function-name">fit</span>
-
-
-<a href="#pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.fit" class="headerlink" title="Permanent link">&para;</a></h3>
-<div class="doc-signature highlight"><pre><span></span><code><span class="nf">fit</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
-</code></pre></div>
-
-    <div class="doc doc-contents ">
-
-        <p>This method is included for consistency with the rest of the library
-in general and with the <code>BasePreprocessor</code> in particular. HuggingFace's
-tokenizers and models are already trained. Therefore, the 'fit' method
-here does nothing other than checking that the 'text_col' parameter is
-not <code>None</code>.</p>
-
-
-<p><span class="doc-section-title">Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>df</code></b>
-              (<code><span title="pandas.DataFrame">DataFrame</span></code>)
-          –
-          <div class="doc-md-description">
-            <p>The dataframe containing the text data in the column specified by
-the 'text_col' parameter</p>
-          </div>
-        </li>
-    </ul>
-
-            <details class="quote">
-              <summary>Source code in <code>pytorch_widedeep/preprocessing/hf_preprocessor.py</code></summary>
-              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">212</span>
+<span class="normal">210</span>
+<span class="normal">211</span>
+<span class="normal">212</span>
 <span class="normal">213</span>
 <span class="normal">214</span>
 <span class="normal">215</span>
@@ -4588,79 +5732,9 @@ <h3 id="pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.fit" class
 <span class="normal">228</span>
 <span class="normal">229</span>
 <span class="normal">230</span>
-<span class="normal">231</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">fit</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="s2">&quot;HFPreprocessor&quot;</span><span class="p">:</span>
-<span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
-<span class="sd">    This method is included for consistency with the rest of the library</span>
-<span class="sd">    in general and with the `BasePreprocessor` in particular. HuggingFace&#39;s</span>
-<span class="sd">    tokenizers and models are already trained. Therefore, the &#39;fit&#39; method</span>
-<span class="sd">    here does nothing other than checking that the &#39;text_col&#39; parameter is</span>
-<span class="sd">    not `None`.</span>
-
-<span class="sd">    Parameters</span>
-<span class="sd">    ----------</span>
-<span class="sd">    df: pd.DataFrame</span>
-<span class="sd">        The dataframe containing the text data in the column specified by</span>
-<span class="sd">        the &#39;text_col&#39; parameter</span>
-<span class="sd">    &quot;&quot;&quot;</span>
-    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">text_col</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
-        <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
-            <span class="s2">&quot;&#39;text_col&#39; is None. Please specify the column name containing the text data&quot;</span>
-            <span class="s2">&quot; if you want to use the &#39;fit&#39; method&quot;</span>
-        <span class="p">)</span>
-    <span class="k">return</span> <span class="bp">self</span>
-</code></pre></div></td></tr></table></div>
-            </details>
-    </div>
-
-</div>
-
-<div class="doc doc-object doc-function">
-
-
-<h3 id="pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.transform" class="doc doc-heading">
-            <span class="doc doc-object-name doc-function-name">transform</span>
-
-
-<a href="#pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.transform" class="headerlink" title="Permanent link">&para;</a></h3>
-<div class="doc-signature highlight"><pre><span></span><code><span class="nf">transform</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
-</code></pre></div>
-
-    <div class="doc doc-contents ">
-
-        <p>Encodes the text data in the input dataframe. This method simply
-calls the <code>encode</code> method under the hood. Similar to the <code>fit</code> method,
-this method is included for consistency with the rest of the library
-in general and with the <code>BasePreprocessor</code> in particular.</p>
-
-
-<p><span class="doc-section-title">Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>df</code></b>
-              (<code><span title="pandas.DataFrame">DataFrame</span></code>)
-          –
-          <div class="doc-md-description">
-            <p>The dataframe containing the text data in the column specified by
-the 'text_col' parameter</p>
-          </div>
-        </li>
-    </ul>
-
-
-<p><span class="doc-section-title">Returns:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-              <code><span title="numpy.array">array</span></code>
-          –
-          <div class="doc-md-description">
-            <p>The encoded texts</p>
-          </div>
-        </li>
-    </ul>
-
-            <details class="quote">
-              <summary>Source code in <code>pytorch_widedeep/preprocessing/hf_preprocessor.py</code></summary>
-              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">233</span>
+<span class="normal">231</span>
+<span class="normal">232</span>
+<span class="normal">233</span>
 <span class="normal">234</span>
 <span class="normal">235</span>
 <span class="normal">236</span>
@@ -4686,120 +5760,401 @@ <h3 id="pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.transform"
 <span class="normal">256</span>
 <span class="normal">257</span>
 <span class="normal">258</span>
-<span class="normal">259</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">transform</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">npt</span><span class="o">.</span><span class="n">NDArray</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">int64</span><span class="p">]:</span>
-<span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
-<span class="sd">    Encodes the text data in the input dataframe. This method simply</span>
-<span class="sd">    calls the `encode` method under the hood. Similar to the `fit` method,</span>
-<span class="sd">    this method is included for consistency with the rest of the library</span>
-<span class="sd">    in general and with the `BasePreprocessor` in particular.</span>
+<span class="normal">259</span>
+<span class="normal">260</span>
+<span class="normal">261</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">class</span> <span class="nc">TextPreprocessor</span><span class="p">(</span><span class="n">BasePreprocessor</span><span class="p">):</span>
+<span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Preprocessor to prepare the ``deeptext`` input dataset</span>
 
 <span class="sd">    Parameters</span>
 <span class="sd">    ----------</span>
-<span class="sd">    df: pd.DataFrame</span>
-<span class="sd">        The dataframe containing the text data in the column specified by</span>
-<span class="sd">        the &#39;text_col&#39; parameter</span>
-
-<span class="sd">    Returns</span>
-<span class="sd">    -------</span>
-<span class="sd">    np.array</span>
-<span class="sd">        The encoded texts</span>
+<span class="sd">    text_col: str</span>
+<span class="sd">        column in the input dataframe containing the texts</span>
+<span class="sd">    max_vocab: int, default=30000</span>
+<span class="sd">        Maximum number of tokens in the vocabulary</span>
+<span class="sd">    min_freq: int, default=5</span>
+<span class="sd">        Minimum frequency for a token to be part of the vocabulary</span>
+<span class="sd">    maxlen: int, default=80</span>
+<span class="sd">        Maximum length of the tokenized sequences</span>
+<span class="sd">    pad_first: bool,  default = True</span>
+<span class="sd">        Indicates whether the padding index will be added at the beginning or the</span>
+<span class="sd">        end of the sequences</span>
+<span class="sd">    pad_idx: int, default = 1</span>
+<span class="sd">        padding index. Fastai&#39;s Tokenizer leaves 0 for the &#39;unknown&#39; token.</span>
+<span class="sd">    already_processed: bool, Optional, default = False</span>
+<span class="sd">        Boolean indicating if the sequence of elements is already processed or</span>
+<span class="sd">        prepared. If this is the case, this Preprocessor will simply tokenize</span>
+<span class="sd">        and pad the sequence. &lt;br/&gt;</span>
+
+<span class="sd">            Param aliases: `not_text`. &lt;br/&gt;</span>
+
+<span class="sd">        This parameter is thought for those cases where the input sequences</span>
+<span class="sd">        are already fully processed or are directly not text (e.g. IDs)</span>
+<span class="sd">    word_vectors_path: str, Optional</span>
+<span class="sd">        Path to the pretrained word vectors</span>
+<span class="sd">    n_cpus: int, Optional, default = None</span>
+<span class="sd">        number of CPUs to used during the tokenization process</span>
+<span class="sd">    verbose: int, default 1</span>
+<span class="sd">        Enable verbose output.</span>
+
+<span class="sd">    Attributes</span>
+<span class="sd">    ----------</span>
+<span class="sd">    vocab: Vocab</span>
+<span class="sd">        an instance of `pytorch_widedeep.utils.fastai_transforms.Vocab`</span>
+<span class="sd">    embedding_matrix: np.ndarray</span>
+<span class="sd">        Array with the pretrained embeddings</span>
+
+<span class="sd">    Examples</span>
+<span class="sd">    ---------</span>
+<span class="sd">    &gt;&gt;&gt; import pandas as pd</span>
+<span class="sd">    &gt;&gt;&gt; from pytorch_widedeep.preprocessing import TextPreprocessor</span>
+<span class="sd">    &gt;&gt;&gt; df_train = pd.DataFrame({&#39;text_column&#39;: [&quot;life is like a box of chocolates&quot;,</span>
+<span class="sd">    ... &quot;You never know what you&#39;re gonna get&quot;]})</span>
+<span class="sd">    &gt;&gt;&gt; text_preprocessor = TextPreprocessor(text_col=&#39;text_column&#39;, max_vocab=25, min_freq=1, maxlen=10)</span>
+<span class="sd">    &gt;&gt;&gt; text_preprocessor.fit_transform(df_train)</span>
+<span class="sd">    The vocabulary contains 24 tokens</span>
+<span class="sd">    array([[ 1,  1,  1,  1, 10, 11, 12, 13, 14, 15],</span>
+<span class="sd">           [ 5,  9, 16, 17, 18,  9, 19, 20, 21, 22]], dtype=int32)</span>
+<span class="sd">    &gt;&gt;&gt; df_te = pd.DataFrame({&#39;text_column&#39;: [&#39;you never know what is in the box&#39;]})</span>
+<span class="sd">    &gt;&gt;&gt; text_preprocessor.transform(df_te)</span>
+<span class="sd">    array([[ 1,  1,  9, 16, 17, 18, 11,  0,  0, 13]], dtype=int32)</span>
 <span class="sd">    &quot;&quot;&quot;</span>
-    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">text_col</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
-        <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
-            <span class="s2">&quot;&#39;text_col&#39; is None. Please specify the column name containing the text data&quot;</span>
-            <span class="s2">&quot; if you want to use the &#39;fit&#39; method&quot;</span>
+
+    <span class="nd">@alias</span><span class="p">(</span><span class="s2">&quot;already_processed&quot;</span><span class="p">,</span> <span class="p">[</span><span class="s2">&quot;not_text&quot;</span><span class="p">])</span>
+    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span>
+        <span class="n">text_col</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
+        <span class="n">max_vocab</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">30000</span><span class="p">,</span>
+        <span class="n">min_freq</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">5</span><span class="p">,</span>
+        <span class="n">maxlen</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">80</span><span class="p">,</span>
+        <span class="n">pad_first</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">,</span>
+        <span class="n">pad_idx</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1</span><span class="p">,</span>
+        <span class="n">already_processed</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
+        <span class="n">word_vectors_path</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">n_cpus</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">verbose</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1</span><span class="p">,</span>
+    <span class="p">):</span>
+        <span class="nb">super</span><span class="p">(</span><span class="n">TextPreprocessor</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">text_col</span> <span class="o">=</span> <span class="n">text_col</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">max_vocab</span> <span class="o">=</span> <span class="n">max_vocab</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">min_freq</span> <span class="o">=</span> <span class="n">min_freq</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">maxlen</span> <span class="o">=</span> <span class="n">maxlen</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">pad_first</span> <span class="o">=</span> <span class="n">pad_first</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">pad_idx</span> <span class="o">=</span> <span class="n">pad_idx</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">already_processed</span> <span class="o">=</span> <span class="n">already_processed</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">word_vectors_path</span> <span class="o">=</span> <span class="n">word_vectors_path</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">verbose</span> <span class="o">=</span> <span class="n">verbose</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">n_cpus</span> <span class="o">=</span> <span class="n">n_cpus</span> <span class="k">if</span> <span class="n">n_cpus</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="k">else</span> <span class="n">os</span><span class="o">.</span><span class="n">cpu_count</span><span class="p">()</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">is_fitted</span> <span class="o">=</span> <span class="kc">False</span>
+
+    <span class="k">def</span> <span class="nf">fit</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">BasePreprocessor</span><span class="p">:</span>
+<span class="w">        </span><span class="sd">&quot;&quot;&quot;Builds the vocabulary</span>
+
+<span class="sd">        Parameters</span>
+<span class="sd">        ----------</span>
+<span class="sd">        df: pd.DataFrame</span>
+<span class="sd">            Input pandas dataframe</span>
+
+<span class="sd">        Returns</span>
+<span class="sd">        -------</span>
+<span class="sd">        TextPreprocessor</span>
+<span class="sd">            `TextPreprocessor` fitted object</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">texts</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_read_texts</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
+
+        <span class="n">tokens</span> <span class="o">=</span> <span class="n">get_texts</span><span class="p">(</span><span class="n">texts</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">already_processed</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">n_cpus</span><span class="p">)</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">vocab</span><span class="p">:</span> <span class="n">TVocab</span> <span class="o">=</span> <span class="n">Vocab</span><span class="p">(</span>
+            <span class="n">max_vocab</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">max_vocab</span><span class="p">,</span>
+            <span class="n">min_freq</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">min_freq</span><span class="p">,</span>
+            <span class="n">pad_idx</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">pad_idx</span><span class="p">,</span>
+        <span class="p">)</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span>
+            <span class="n">tokens</span><span class="p">,</span>
         <span class="p">)</span>
 
-    <span class="n">texts</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_read_texts</span><span class="p">(</span><span class="n">df</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">root_dir</span><span class="p">)</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">verbose</span><span class="p">:</span>
+            <span class="nb">print</span><span class="p">(</span><span class="s2">&quot;The vocabulary contains </span><span class="si">{}</span><span class="s2"> tokens&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">vocab</span><span class="o">.</span><span class="n">stoi</span><span class="p">)))</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">word_vectors_path</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">embedding_matrix</span> <span class="o">=</span> <span class="n">build_embeddings_matrix</span><span class="p">(</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">vocab</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">word_vectors_path</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">min_freq</span>
+            <span class="p">)</span>
 
-    <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">encode</span><span class="p">(</span><span class="n">texts</span><span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">is_fitted</span> <span class="o">=</span> <span class="kc">True</span>
+
+        <span class="k">return</span> <span class="bp">self</span>
+
+    <span class="k">def</span> <span class="nf">transform</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">:</span>
+<span class="w">        </span><span class="sd">&quot;&quot;&quot;Returns the padded, _&#39;numericalised&#39;_ sequences</span>
+
+<span class="sd">        Parameters</span>
+<span class="sd">        ----------</span>
+<span class="sd">        df: pd.DataFrame</span>
+<span class="sd">            Input pandas dataframe</span>
+
+<span class="sd">        Returns</span>
+<span class="sd">        -------</span>
+<span class="sd">        np.ndarray</span>
+<span class="sd">            Padded, _&#39;numericalised&#39;_ sequences</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">check_is_fitted</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">attributes</span><span class="o">=</span><span class="p">[</span><span class="s2">&quot;vocab&quot;</span><span class="p">])</span>
+        <span class="n">texts</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_read_texts</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
+        <span class="n">tokens</span> <span class="o">=</span> <span class="n">get_texts</span><span class="p">(</span><span class="n">texts</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">already_processed</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">n_cpus</span><span class="p">)</span>
+        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_pad_sequences</span><span class="p">(</span><span class="n">tokens</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">transform_sample</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">text</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">:</span>
+<span class="w">        </span><span class="sd">&quot;&quot;&quot;Returns the padded, _&#39;numericalised&#39;_ sequence</span>
+
+<span class="sd">        Parameters</span>
+<span class="sd">        ----------</span>
+<span class="sd">        text: str</span>
+<span class="sd">            text to be tokenized and padded</span>
+
+<span class="sd">        Returns</span>
+<span class="sd">        -------</span>
+<span class="sd">        np.ndarray</span>
+<span class="sd">            Padded, _&#39;numericalised&#39;_ sequence</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">check_is_fitted</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">attributes</span><span class="o">=</span><span class="p">[</span><span class="s2">&quot;vocab&quot;</span><span class="p">])</span>
+        <span class="n">tokens</span> <span class="o">=</span> <span class="n">get_texts</span><span class="p">([</span><span class="n">text</span><span class="p">],</span> <span class="bp">self</span><span class="o">.</span><span class="n">already_processed</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">n_cpus</span><span class="p">)</span>
+        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_pad_sequences</span><span class="p">(</span><span class="n">tokens</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span>
+
+    <span class="k">def</span> <span class="nf">fit_transform</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">:</span>
+<span class="w">        </span><span class="sd">&quot;&quot;&quot;Combines `fit` and `transform`</span>
+
+<span class="sd">        Parameters</span>
+<span class="sd">        ----------</span>
+<span class="sd">        df: pd.DataFrame</span>
+<span class="sd">            Input pandas dataframe</span>
+
+<span class="sd">        Returns</span>
+<span class="sd">        -------</span>
+<span class="sd">        np.ndarray</span>
+<span class="sd">            Padded, _&#39;numericalised&#39;_ sequences</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">df</span><span class="p">)</span><span class="o">.</span><span class="n">transform</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">inverse_transform</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">padded_seq</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">:</span>
+<span class="w">        </span><span class="sd">&quot;&quot;&quot;Returns the original text plus the added &#39;special&#39; tokens</span>
+
+<span class="sd">        Parameters</span>
+<span class="sd">        ----------</span>
+<span class="sd">        padded_seq: np.ndarray</span>
+<span class="sd">            array with the output of the `transform` method</span>
+
+<span class="sd">        Returns</span>
+<span class="sd">        -------</span>
+<span class="sd">        pd.DataFrame</span>
+<span class="sd">            Pandas dataframe with the original text plus the added &#39;special&#39; tokens</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">texts</span> <span class="o">=</span> <span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">vocab</span><span class="o">.</span><span class="n">inverse_transform</span><span class="p">(</span><span class="n">num</span><span class="p">)</span> <span class="k">for</span> <span class="n">num</span> <span class="ow">in</span> <span class="n">padded_seq</span><span class="p">]</span>
+        <span class="k">return</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">({</span><span class="bp">self</span><span class="o">.</span><span class="n">text_col</span><span class="p">:</span> <span class="n">texts</span><span class="p">})</span>
+
+    <span class="k">def</span> <span class="nf">_pad_sequences</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">tokens</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]])</span> <span class="o">-&gt;</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">:</span>
+        <span class="n">sequences</span> <span class="o">=</span> <span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">vocab</span><span class="o">.</span><span class="n">transform</span><span class="p">(</span><span class="n">t</span><span class="p">)</span> <span class="k">for</span> <span class="n">t</span> <span class="ow">in</span> <span class="n">tokens</span><span class="p">]</span>
+        <span class="n">padded_seq</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">(</span>
+            <span class="p">[</span>
+                <span class="n">pad_sequences</span><span class="p">(</span>
+                    <span class="n">s</span><span class="p">,</span>
+                    <span class="n">maxlen</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">maxlen</span><span class="p">,</span>
+                    <span class="n">pad_first</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">pad_first</span><span class="p">,</span>
+                    <span class="n">pad_idx</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">pad_idx</span><span class="p">,</span>
+                <span class="p">)</span>
+                <span class="k">for</span> <span class="n">s</span> <span class="ow">in</span> <span class="n">sequences</span>
+            <span class="p">]</span>
+        <span class="p">)</span>
+        <span class="k">return</span> <span class="n">padded_seq</span>
+
+    <span class="k">def</span> <span class="nf">_read_texts</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">,</span> <span class="n">root_dir</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span>
+    <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]:</span>
+        <span class="k">if</span> <span class="n">root_dir</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="k">if</span> <span class="ow">not</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">exists</span><span class="p">(</span><span class="n">root_dir</span><span class="p">):</span>
+                <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
+                    <span class="s2">&quot;root_dir does not exist. Please create it before fitting the preprocessor&quot;</span>
+                <span class="p">)</span>
+            <span class="n">texts_fnames</span> <span class="o">=</span> <span class="n">df</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">text_col</span><span class="p">]</span><span class="o">.</span><span class="n">tolist</span><span class="p">()</span>
+            <span class="n">texts</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
+            <span class="k">for</span> <span class="n">texts_fname</span> <span class="ow">in</span> <span class="n">texts_fnames</span><span class="p">:</span>
+                <span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">root_dir</span><span class="p">,</span> <span class="n">texts_fname</span><span class="p">),</span> <span class="s2">&quot;r&quot;</span><span class="p">)</span> <span class="k">as</span> <span class="n">f</span><span class="p">:</span>
+                    <span class="n">texts</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">read</span><span class="p">()</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s2">&quot;</span><span class="se">\n</span><span class="s2">&quot;</span><span class="p">,</span> <span class="s2">&quot;&quot;</span><span class="p">))</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="n">texts</span> <span class="o">=</span> <span class="n">df</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">text_col</span><span class="p">]</span><span class="o">.</span><span class="n">tolist</span><span class="p">()</span>
+
+        <span class="k">return</span> <span class="n">texts</span>
+
+    <span class="k">def</span> <span class="nf">_load_vocab</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">vocab</span><span class="p">:</span> <span class="n">TVocab</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="kc">None</span><span class="p">:</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">vocab</span> <span class="o">=</span> <span class="n">vocab</span>
+
+    <span class="k">def</span> <span class="fm">__repr__</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">str</span><span class="p">:</span>
+        <span class="n">list_of_params</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[</span><span class="s2">&quot;text_col=</span><span class="si">{text_col}</span><span class="s2">&quot;</span><span class="p">]</span>
+        <span class="n">list_of_params</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">&quot;max_vocab=</span><span class="si">{max_vocab}</span><span class="s2">&quot;</span><span class="p">)</span>
+        <span class="n">list_of_params</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">&quot;min_freq=</span><span class="si">{min_freq}</span><span class="s2">&quot;</span><span class="p">)</span>
+        <span class="n">list_of_params</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">&quot;maxlen=</span><span class="si">{maxlen}</span><span class="s2">&quot;</span><span class="p">)</span>
+        <span class="n">list_of_params</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">&quot;pad_first=</span><span class="si">{pad_first}</span><span class="s2">&quot;</span><span class="p">)</span>
+        <span class="n">list_of_params</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">&quot;pad_idx=</span><span class="si">{pad_idx}</span><span class="s2">&quot;</span><span class="p">)</span>
+        <span class="n">list_of_params</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">&quot;already_processed=</span><span class="si">{already_processed}</span><span class="s2">&quot;</span><span class="p">)</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">word_vectors_path</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="n">list_of_params</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">&quot;word_vectors_path=</span><span class="si">{word_vectors_path}</span><span class="s2">&quot;</span><span class="p">)</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">n_cpus</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="n">list_of_params</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">&quot;n_cpus=</span><span class="si">{n_cpus}</span><span class="s2">&quot;</span><span class="p">)</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">verbose</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="n">list_of_params</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">&quot;verbose=</span><span class="si">{verbose}</span><span class="s2">&quot;</span><span class="p">)</span>
+        <span class="n">all_params</span> <span class="o">=</span> <span class="s2">&quot;, &quot;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">list_of_params</span><span class="p">)</span>
+        <span class="k">return</span> <span class="sa">f</span><span class="s2">&quot;TextPreprocessor(</span><span class="si">{</span><span class="n">all_params</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="bp">self</span><span class="o">.</span><span class="vm">__dict__</span><span class="p">)</span><span class="si">}</span><span class="s2">)&quot;</span>
 </code></pre></div></td></tr></table></div>
-            </details>
-    </div>
+              </details>
+
+
+
+  <div class="doc doc-children">
+
+
+
+
+
+
+
 
-</div>
 
 <div class="doc doc-object doc-function">
 
 
-<h3 id="pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.transform_sample" class="doc doc-heading">
-            <span class="doc doc-object-name doc-function-name">transform_sample</span>
+<h3 id="pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor.fit" class="doc doc-heading">
+            <span class="doc doc-object-name doc-function-name">fit</span>
 
 
-<a href="#pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.transform_sample" class="headerlink" title="Permanent link">&para;</a></h3>
-<div class="doc-signature highlight"><pre><span></span><code><span class="nf">transform_sample</span><span class="p">(</span><span class="n">text</span><span class="p">)</span>
+<a href="#pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor.fit" class="headerlink" title="Permanent link">&para;</a></h3>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">fit</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
 </code></pre></div>
 
     <div class="doc doc-contents ">
 
-        <p>Encodes a single text sample.</p>
+        <p>Builds the vocabulary</p>
 
 
 <p><span class="doc-section-title">Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>text</code></b>
-              (<code>str</code>)
-          –
-          <div class="doc-md-description">
-            <p>The text sample to be encoded</p>
-          </div>
-        </li>
-    </ul>
-
-
-<p><span class="doc-section-title">Returns:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-              <code><span title="numpy.array">array</span></code>
-          –
-          <div class="doc-md-description">
-            <p>The encoded text</p>
-          </div>
-        </li>
-    </ul>
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+          <th>Default</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>df</code></td>
+            <td>
+                  <code><span title="pandas.DataFrame">DataFrame</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Input pandas dataframe</p>
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+      </tbody>
+    </table>
+
+
+    <p><span class="doc-section-title">Returns:</span></p>
+    <table>
+      <thead>
+        <tr>
+          <th>Type</th>
+          <th>Description</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td>
+                  <code><a class="autorefs autorefs-internal" title="pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor" href="#pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor">TextPreprocessor</a></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p><code>TextPreprocessor</code> fitted object</p>
+              </div>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
             <details class="quote">
-              <summary>Source code in <code>pytorch_widedeep/preprocessing/hf_preprocessor.py</code></summary>
-              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">261</span>
-<span class="normal">262</span>
-<span class="normal">263</span>
-<span class="normal">264</span>
-<span class="normal">265</span>
-<span class="normal">266</span>
-<span class="normal">267</span>
-<span class="normal">268</span>
-<span class="normal">269</span>
-<span class="normal">270</span>
-<span class="normal">271</span>
-<span class="normal">272</span>
-<span class="normal">273</span>
-<span class="normal">274</span>
-<span class="normal">275</span>
-<span class="normal">276</span>
-<span class="normal">277</span>
-<span class="normal">278</span>
-<span class="normal">279</span>
-<span class="normal">280</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">transform_sample</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">text</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">npt</span><span class="o">.</span><span class="n">NDArray</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">int64</span><span class="p">]:</span>
-<span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
-<span class="sd">    Encodes a single text sample.</span>
+              <summary>Source code in <code>pytorch_widedeep/preprocessing/text_preprocessor.py</code></summary>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">108</span>
+<span class="normal">109</span>
+<span class="normal">110</span>
+<span class="normal">111</span>
+<span class="normal">112</span>
+<span class="normal">113</span>
+<span class="normal">114</span>
+<span class="normal">115</span>
+<span class="normal">116</span>
+<span class="normal">117</span>
+<span class="normal">118</span>
+<span class="normal">119</span>
+<span class="normal">120</span>
+<span class="normal">121</span>
+<span class="normal">122</span>
+<span class="normal">123</span>
+<span class="normal">124</span>
+<span class="normal">125</span>
+<span class="normal">126</span>
+<span class="normal">127</span>
+<span class="normal">128</span>
+<span class="normal">129</span>
+<span class="normal">130</span>
+<span class="normal">131</span>
+<span class="normal">132</span>
+<span class="normal">133</span>
+<span class="normal">134</span>
+<span class="normal">135</span>
+<span class="normal">136</span>
+<span class="normal">137</span>
+<span class="normal">138</span>
+<span class="normal">139</span>
+<span class="normal">140</span>
+<span class="normal">141</span>
+<span class="normal">142</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">fit</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">BasePreprocessor</span><span class="p">:</span>
+<span class="w">    </span><span class="sd">&quot;&quot;&quot;Builds the vocabulary</span>
 
 <span class="sd">    Parameters</span>
 <span class="sd">    ----------</span>
-<span class="sd">    text: str</span>
-<span class="sd">        The text sample to be encoded</span>
+<span class="sd">    df: pd.DataFrame</span>
+<span class="sd">        Input pandas dataframe</span>
 
 <span class="sd">    Returns</span>
 <span class="sd">    -------</span>
-<span class="sd">    np.array</span>
-<span class="sd">        The encoded text</span>
+<span class="sd">    TextPreprocessor</span>
+<span class="sd">        `TextPreprocessor` fitted object</span>
 <span class="sd">    &quot;&quot;&quot;</span>
+    <span class="n">texts</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_read_texts</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
 
-    <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">is_fitted</span><span class="p">:</span>
-        <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
-            <span class="s2">&quot;The `encode` (or `fit`) method must be called before calling `transform_sample`&quot;</span>
+    <span class="n">tokens</span> <span class="o">=</span> <span class="n">get_texts</span><span class="p">(</span><span class="n">texts</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">already_processed</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">n_cpus</span><span class="p">)</span>
+
+    <span class="bp">self</span><span class="o">.</span><span class="n">vocab</span><span class="p">:</span> <span class="n">TVocab</span> <span class="o">=</span> <span class="n">Vocab</span><span class="p">(</span>
+        <span class="n">max_vocab</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">max_vocab</span><span class="p">,</span>
+        <span class="n">min_freq</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">min_freq</span><span class="p">,</span>
+        <span class="n">pad_idx</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">pad_idx</span><span class="p">,</span>
+    <span class="p">)</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span>
+        <span class="n">tokens</span><span class="p">,</span>
+    <span class="p">)</span>
+
+    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">verbose</span><span class="p">:</span>
+        <span class="nb">print</span><span class="p">(</span><span class="s2">&quot;The vocabulary contains </span><span class="si">{}</span><span class="s2"> tokens&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">vocab</span><span class="o">.</span><span class="n">stoi</span><span class="p">)))</span>
+    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">word_vectors_path</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">embedding_matrix</span> <span class="o">=</span> <span class="n">build_embeddings_matrix</span><span class="p">(</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">vocab</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">word_vectors_path</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">min_freq</span>
         <span class="p">)</span>
-    <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">encode</span><span class="p">([</span><span class="n">text</span><span class="p">])[</span><span class="mi">0</span><span class="p">]</span>
+
+    <span class="bp">self</span><span class="o">.</span><span class="n">is_fitted</span> <span class="o">=</span> <span class="kc">True</span>
+
+    <span class="k">return</span> <span class="bp">self</span>
 </code></pre></div></td></tr></table></div>
             </details>
     </div>
@@ -4809,77 +6164,105 @@ <h3 id="pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.transform_
 <div class="doc doc-object doc-function">
 
 
-<h3 id="pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.fit_transform" class="doc doc-heading">
-            <span class="doc doc-object-name doc-function-name">fit_transform</span>
+<h3 id="pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor.transform" class="doc doc-heading">
+            <span class="doc doc-object-name doc-function-name">transform</span>
 
 
-<a href="#pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.fit_transform" class="headerlink" title="Permanent link">&para;</a></h3>
-<div class="doc-signature highlight"><pre><span></span><code><span class="nf">fit_transform</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
+<a href="#pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor.transform" class="headerlink" title="Permanent link">&para;</a></h3>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">transform</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
 </code></pre></div>
 
     <div class="doc doc-contents ">
 
-        <p>Encodes the text data in the input dataframe.</p>
+        <p>Returns the padded, <em>'numericalised'</em> sequences</p>
 
 
 <p><span class="doc-section-title">Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>df</code></b>
-              (<code><span title="pandas.DataFrame">DataFrame</span></code>)
-          –
-          <div class="doc-md-description">
-            <p>The dataframe containing the text data in the column specified by
-the 'text_col' parameter</p>
-          </div>
-        </li>
-    </ul>
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+          <th>Default</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>df</code></td>
+            <td>
+                  <code><span title="pandas.DataFrame">DataFrame</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Input pandas dataframe</p>
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+      </tbody>
+    </table>
+
+
+    <p><span class="doc-section-title">Returns:</span></p>
+    <table>
+      <thead>
+        <tr>
+          <th>Type</th>
+          <th>Description</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td>
+                  <code><span title="numpy.ndarray">ndarray</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Padded, <em>'numericalised'</em> sequences</p>
+              </div>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
+            <details class="quote">
+              <summary>Source code in <code>pytorch_widedeep/preprocessing/text_preprocessor.py</code></summary>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">144</span>
+<span class="normal">145</span>
+<span class="normal">146</span>
+<span class="normal">147</span>
+<span class="normal">148</span>
+<span class="normal">149</span>
+<span class="normal">150</span>
+<span class="normal">151</span>
+<span class="normal">152</span>
+<span class="normal">153</span>
+<span class="normal">154</span>
+<span class="normal">155</span>
+<span class="normal">156</span>
+<span class="normal">157</span>
+<span class="normal">158</span>
+<span class="normal">159</span>
+<span class="normal">160</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">transform</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">:</span>
+<span class="w">    </span><span class="sd">&quot;&quot;&quot;Returns the padded, _&#39;numericalised&#39;_ sequences</span>
 
-<p><span class="doc-section-title">Returns:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-              <code><span title="numpy.array">array</span></code>
-          –
-          <div class="doc-md-description">
-            <p>The encoded texts</p>
-          </div>
-        </li>
-    </ul>
-
-            <details class="quote">
-              <summary>Source code in <code>pytorch_widedeep/preprocessing/hf_preprocessor.py</code></summary>
-              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">282</span>
-<span class="normal">283</span>
-<span class="normal">284</span>
-<span class="normal">285</span>
-<span class="normal">286</span>
-<span class="normal">287</span>
-<span class="normal">288</span>
-<span class="normal">289</span>
-<span class="normal">290</span>
-<span class="normal">291</span>
-<span class="normal">292</span>
-<span class="normal">293</span>
-<span class="normal">294</span>
-<span class="normal">295</span>
-<span class="normal">296</span>
-<span class="normal">297</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">fit_transform</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">npt</span><span class="o">.</span><span class="n">NDArray</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">int64</span><span class="p">]:</span>
-<span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
-<span class="sd">    Encodes the text data in the input dataframe.</span>
-
-<span class="sd">    Parameters</span>
-<span class="sd">    ----------</span>
-<span class="sd">    df: pd.DataFrame</span>
-<span class="sd">        The dataframe containing the text data in the column specified by</span>
-<span class="sd">        the &#39;text_col&#39; parameter</span>
+<span class="sd">    Parameters</span>
+<span class="sd">    ----------</span>
+<span class="sd">    df: pd.DataFrame</span>
+<span class="sd">        Input pandas dataframe</span>
 
 <span class="sd">    Returns</span>
 <span class="sd">    -------</span>
-<span class="sd">    np.array</span>
-<span class="sd">        The encoded texts</span>
+<span class="sd">    np.ndarray</span>
+<span class="sd">        Padded, _&#39;numericalised&#39;_ sequences</span>
 <span class="sd">    &quot;&quot;&quot;</span>
-    <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">df</span><span class="p">)</span><span class="o">.</span><span class="n">transform</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
+    <span class="n">check_is_fitted</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">attributes</span><span class="o">=</span><span class="p">[</span><span class="s2">&quot;vocab&quot;</span><span class="p">])</span>
+    <span class="n">texts</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_read_texts</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
+    <span class="n">tokens</span> <span class="o">=</span> <span class="n">get_texts</span><span class="p">(</span><span class="n">texts</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">already_processed</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">n_cpus</span><span class="p">)</span>
+    <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_pad_sequences</span><span class="p">(</span><span class="n">tokens</span><span class="p">)</span>
 </code></pre></div></td></tr></table></div>
             </details>
     </div>
@@ -4889,331 +6272,617 @@ <h3 id="pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.fit_transf
 <div class="doc doc-object doc-function">
 
 
-<h3 id="pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.inverse_transform" class="doc doc-heading">
-            <span class="doc doc-object-name doc-function-name">inverse_transform</span>
+<h3 id="pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor.transform_sample" class="doc doc-heading">
+            <span class="doc doc-object-name doc-function-name">transform_sample</span>
 
 
-<a href="#pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.inverse_transform" class="headerlink" title="Permanent link">&para;</a></h3>
-<div class="doc-signature highlight"><pre><span></span><code><span class="nf">inverse_transform</span><span class="p">(</span><span class="n">input_ids</span><span class="p">,</span> <span class="n">skip_special_tokens</span><span class="p">)</span>
+<a href="#pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor.transform_sample" class="headerlink" title="Permanent link">&para;</a></h3>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">transform_sample</span><span class="p">(</span><span class="n">text</span><span class="p">)</span>
 </code></pre></div>
 
     <div class="doc doc-contents ">
 
-        <p>Decodes a list of input_ids. The method simply calls the <code>decode</code> method
-under the hood.</p>
+        <p>Returns the padded, <em>'numericalised'</em> sequence</p>
 
 
 <p><span class="doc-section-title">Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>input_ids</code></b>
-              (<code><span title="numpy.typing.NDArray">NDArray</span>[<span title="numpy.int64">int64</span>]</code>)
-          –
-          <div class="doc-md-description">
-            <p>The input_ids to be decoded</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>skip_special_tokens</code></b>
-              (<code>bool</code>)
-          –
-          <div class="doc-md-description">
-            <p>Whether to skip the special tokens or not</p>
-          </div>
-        </li>
-    </ul>
-
-
-<p><span class="doc-section-title">Returns:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-              <code><span title="typing.List">List</span>[str]</code>
-          –
-          <div class="doc-md-description">
-            <p>The decoded texts</p>
-          </div>
-        </li>
-    </ul>
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+          <th>Default</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>text</code></td>
+            <td>
+                  <code>str</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>text to be tokenized and padded</p>
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+      </tbody>
+    </table>
+
+
+    <p><span class="doc-section-title">Returns:</span></p>
+    <table>
+      <thead>
+        <tr>
+          <th>Type</th>
+          <th>Description</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td>
+                  <code><span title="numpy.ndarray">ndarray</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Padded, <em>'numericalised'</em> sequence</p>
+              </div>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
             <details class="quote">
-              <summary>Source code in <code>pytorch_widedeep/preprocessing/hf_preprocessor.py</code></summary>
-              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">299</span>
-<span class="normal">300</span>
-<span class="normal">301</span>
-<span class="normal">302</span>
-<span class="normal">303</span>
-<span class="normal">304</span>
-<span class="normal">305</span>
-<span class="normal">306</span>
-<span class="normal">307</span>
-<span class="normal">308</span>
-<span class="normal">309</span>
-<span class="normal">310</span>
-<span class="normal">311</span>
-<span class="normal">312</span>
-<span class="normal">313</span>
-<span class="normal">314</span>
-<span class="normal">315</span>
-<span class="normal">316</span>
-<span class="normal">317</span>
-<span class="normal">318</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">inverse_transform</span><span class="p">(</span>
-    <span class="bp">self</span><span class="p">,</span> <span class="n">input_ids</span><span class="p">:</span> <span class="n">npt</span><span class="o">.</span><span class="n">NDArray</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">int64</span><span class="p">],</span> <span class="n">skip_special_tokens</span><span class="p">:</span> <span class="nb">bool</span>
-<span class="p">)</span> <span class="o">-&gt;</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]:</span>
-<span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
-<span class="sd">    Decodes a list of input_ids. The method simply calls the `decode` method</span>
-<span class="sd">    under the hood.</span>
+              <summary>Source code in <code>pytorch_widedeep/preprocessing/text_preprocessor.py</code></summary>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">162</span>
+<span class="normal">163</span>
+<span class="normal">164</span>
+<span class="normal">165</span>
+<span class="normal">166</span>
+<span class="normal">167</span>
+<span class="normal">168</span>
+<span class="normal">169</span>
+<span class="normal">170</span>
+<span class="normal">171</span>
+<span class="normal">172</span>
+<span class="normal">173</span>
+<span class="normal">174</span>
+<span class="normal">175</span>
+<span class="normal">176</span>
+<span class="normal">177</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">transform_sample</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">text</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">:</span>
+<span class="w">    </span><span class="sd">&quot;&quot;&quot;Returns the padded, _&#39;numericalised&#39;_ sequence</span>
 
 <span class="sd">    Parameters</span>
 <span class="sd">    ----------</span>
-<span class="sd">    input_ids: npt.NDArray[np.int64]</span>
-<span class="sd">        The input_ids to be decoded</span>
-<span class="sd">    skip_special_tokens: bool</span>
-<span class="sd">        Whether to skip the special tokens or not</span>
+<span class="sd">    text: str</span>
+<span class="sd">        text to be tokenized and padded</span>
 
 <span class="sd">    Returns</span>
 <span class="sd">    -------</span>
-<span class="sd">    List[str]</span>
-<span class="sd">        The decoded texts</span>
+<span class="sd">    np.ndarray</span>
+<span class="sd">        Padded, _&#39;numericalised&#39;_ sequence</span>
 <span class="sd">    &quot;&quot;&quot;</span>
-    <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">decode</span><span class="p">(</span><span class="n">input_ids</span><span class="p">,</span> <span class="n">skip_special_tokens</span><span class="p">)</span>
+    <span class="n">check_is_fitted</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">attributes</span><span class="o">=</span><span class="p">[</span><span class="s2">&quot;vocab&quot;</span><span class="p">])</span>
+    <span class="n">tokens</span> <span class="o">=</span> <span class="n">get_texts</span><span class="p">([</span><span class="n">text</span><span class="p">],</span> <span class="bp">self</span><span class="o">.</span><span class="n">already_processed</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">n_cpus</span><span class="p">)</span>
+    <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_pad_sequences</span><span class="p">(</span><span class="n">tokens</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span>
 </code></pre></div></td></tr></table></div>
             </details>
     </div>
 
 </div>
 
+<div class="doc doc-object doc-function">
 
 
-  </div>
-
-    </div>
-
-</div>
+<h3 id="pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor.fit_transform" class="doc doc-heading">
+            <span class="doc doc-object-name doc-function-name">fit_transform</span>
 
-<div class="doc doc-object doc-class">
 
+<a href="#pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor.fit_transform" class="headerlink" title="Permanent link">&para;</a></h3>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">fit_transform</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
+</code></pre></div>
 
+    <div class="doc doc-contents ">
 
-<h2 id="pytorch_widedeep.preprocessing.image_preprocessor.ImagePreprocessor" class="doc doc-heading">
-            <span class="doc doc-object-name doc-class-name">ImagePreprocessor</span>
+        <p>Combines <code>fit</code> and <code>transform</code></p>
 
 
-<a href="#pytorch_widedeep.preprocessing.image_preprocessor.ImagePreprocessor" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="nf">ImagePreprocessor</span><span class="p">(</span>
-    <span class="n">img_col</span><span class="p">,</span> <span class="n">img_path</span><span class="p">,</span> <span class="n">width</span><span class="o">=</span><span class="mi">224</span><span class="p">,</span> <span class="n">height</span><span class="o">=</span><span class="mi">224</span><span class="p">,</span> <span class="n">verbose</span><span class="o">=</span><span class="mi">1</span>
-<span class="p">)</span>
-</code></pre></div>
+<p><span class="doc-section-title">Parameters:</span></p>
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+          <th>Default</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>df</code></td>
+            <td>
+                  <code><span title="pandas.DataFrame">DataFrame</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Input pandas dataframe</p>
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+      </tbody>
+    </table>
+
+
+    <p><span class="doc-section-title">Returns:</span></p>
+    <table>
+      <thead>
+        <tr>
+          <th>Type</th>
+          <th>Description</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td>
+                  <code><span title="numpy.ndarray">ndarray</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Padded, <em>'numericalised'</em> sequences</p>
+              </div>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
-    <div class="doc doc-contents first">
-            <p class="doc doc-class-bases">
-              Bases: <code><span title="pytorch_widedeep.preprocessing.base_preprocessor.BasePreprocessor">BasePreprocessor</span></code></p>
+            <details class="quote">
+              <summary>Source code in <code>pytorch_widedeep/preprocessing/text_preprocessor.py</code></summary>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">179</span>
+<span class="normal">180</span>
+<span class="normal">181</span>
+<span class="normal">182</span>
+<span class="normal">183</span>
+<span class="normal">184</span>
+<span class="normal">185</span>
+<span class="normal">186</span>
+<span class="normal">187</span>
+<span class="normal">188</span>
+<span class="normal">189</span>
+<span class="normal">190</span>
+<span class="normal">191</span>
+<span class="normal">192</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">fit_transform</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">:</span>
+<span class="w">    </span><span class="sd">&quot;&quot;&quot;Combines `fit` and `transform`</span>
 
+<span class="sd">    Parameters</span>
+<span class="sd">    ----------</span>
+<span class="sd">    df: pd.DataFrame</span>
+<span class="sd">        Input pandas dataframe</span>
 
-        <p>Preprocessor to prepare the <code>deepimage</code> input dataset.</p>
-<p>The Preprocessing consists simply on resizing according to their
-aspect ratio</p>
+<span class="sd">    Returns</span>
+<span class="sd">    -------</span>
+<span class="sd">    np.ndarray</span>
+<span class="sd">        Padded, _&#39;numericalised&#39;_ sequences</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">df</span><span class="p">)</span><span class="o">.</span><span class="n">transform</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
+</code></pre></div></td></tr></table></div>
+            </details>
+    </div>
 
+</div>
 
-<p><span class="doc-section-title">Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>img_col</code></b>
-              (<code>str</code>)
-          –
-          <div class="doc-md-description">
-            <p>name of the column with the images filenames</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>img_path</code></b>
-              (<code>str</code>)
-          –
-          <div class="doc-md-description">
-            <p>path to the dicrectory where the images are stored</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>width</code></b>
-              (<code>int</code>, default:
-                  <code>224</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>width of the resulting processed image.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>height</code></b>
-              (<code>int</code>, default:
-                  <code>224</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>width of the resulting processed image.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>verbose</code></b>
-              (<code>int</code>, default:
-                  <code>1</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Enable verbose output.</p>
-          </div>
-        </li>
-    </ul>
+<div class="doc doc-object doc-function">
 
 
-<p><span class="doc-section-title">Attributes:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code><span title="pytorch_widedeep.preprocessing.image_preprocessor.ImagePreprocessor.aap">aap</span></code></b>
-              (<code><a class="autorefs autorefs-internal" title="pytorch_widedeep.utils.image_utils.AspectAwarePreprocessor" href="utils/image_utils.html#pytorch_widedeep.utils.image_utils.AspectAwarePreprocessor">AspectAwarePreprocessor</a></code>)
-          –
-          <div class="doc-md-description">
-            <p>an instance of <code>pytorch_widedeep.utils.image_utils.AspectAwarePreprocessor</code></p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code><span title="pytorch_widedeep.preprocessing.image_preprocessor.ImagePreprocessor.spp">spp</span></code></b>
-              (<code><a class="autorefs autorefs-internal" title="pytorch_widedeep.utils.image_utils.SimplePreprocessor" href="utils/image_utils.html#pytorch_widedeep.utils.image_utils.SimplePreprocessor">SimplePreprocessor</a></code>)
-          –
-          <div class="doc-md-description">
-            <p>an instance of <code>pytorch_widedeep.utils.image_utils.SimplePreprocessor</code></p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code><span title="pytorch_widedeep.preprocessing.image_preprocessor.ImagePreprocessor.normalise_metrics">normalise_metrics</span></code></b>
-              (<code>Dict</code>)
-          –
-          <div class="doc-md-description">
-            <p>Dict containing the normalisation metrics of the image dataset, i.e.
-mean and std for the R, G and B channels</p>
-          </div>
-        </li>
-    </ul>
+<h3 id="pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor.inverse_transform" class="doc doc-heading">
+            <span class="doc doc-object-name doc-function-name">inverse_transform</span>
 
 
-<p><span class="doc-section-title">Examples:</span></p>
-    <div class="highlight"><pre><span></span><code><span class="gp">&gt;&gt;&gt; </span><span class="kn">import</span> <span class="nn">pandas</span> <span class="k">as</span> <span class="nn">pd</span>
-<span class="gp">&gt;&gt;&gt;</span>
-<span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">pytorch_widedeep.preprocessing</span> <span class="kn">import</span> <span class="n">ImagePreprocessor</span>
-<span class="gp">&gt;&gt;&gt;</span>
-<span class="gp">&gt;&gt;&gt; </span><span class="n">path_to_image1</span> <span class="o">=</span> <span class="s1">&#39;tests/test_data_utils/images/galaxy1.png&#39;</span>
-<span class="gp">&gt;&gt;&gt; </span><span class="n">path_to_image2</span> <span class="o">=</span> <span class="s1">&#39;tests/test_data_utils/images/galaxy2.png&#39;</span>
-<span class="gp">&gt;&gt;&gt;</span>
-<span class="gp">&gt;&gt;&gt; </span><span class="n">df_train</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">({</span><span class="s1">&#39;images_column&#39;</span><span class="p">:</span> <span class="p">[</span><span class="n">path_to_image1</span><span class="p">]})</span>
-<span class="gp">&gt;&gt;&gt; </span><span class="n">df_test</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">({</span><span class="s1">&#39;images_column&#39;</span><span class="p">:</span> <span class="p">[</span><span class="n">path_to_image2</span><span class="p">]})</span>
-<span class="gp">&gt;&gt;&gt; </span><span class="n">img_preprocessor</span> <span class="o">=</span> <span class="n">ImagePreprocessor</span><span class="p">(</span><span class="n">img_col</span><span class="o">=</span><span class="s1">&#39;images_column&#39;</span><span class="p">,</span> <span class="n">img_path</span><span class="o">=</span><span class="s1">&#39;.&#39;</span><span class="p">,</span> <span class="n">verbose</span><span class="o">=</span><span class="mi">0</span><span class="p">)</span>
-<span class="gp">&gt;&gt;&gt; </span><span class="n">resized_images</span> <span class="o">=</span> <span class="n">img_preprocessor</span><span class="o">.</span><span class="n">fit_transform</span><span class="p">(</span><span class="n">df_train</span><span class="p">)</span>
-<span class="gp">&gt;&gt;&gt; </span><span class="n">new_resized_images</span> <span class="o">=</span> <span class="n">img_preprocessor</span><span class="o">.</span><span class="n">transform</span><span class="p">(</span><span class="n">df_train</span><span class="p">)</span>
+<a href="#pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor.inverse_transform" class="headerlink" title="Permanent link">&para;</a></h3>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">inverse_transform</span><span class="p">(</span><span class="n">padded_seq</span><span class="p">)</span>
 </code></pre></div>
-    <p><img alt="ℹ️" class="emojione" src="https://cdnjs.cloudflare.com/ajax/libs/emojione/2.2.7/assets/png/2139.png" title=":information_source:" /> <strong>NOTE</strong>:
-Normalising metrics will only be computed when the <code>fit_transform</code>
-method is run. Running <code>transform</code> only will not change the computed
-metrics and running <code>fit</code> only simply instantiates the resizing
-functions.</p>
 
-                  <details class="quote">
-                    <summary>Source code in <code>pytorch_widedeep/preprocessing/image_preprocessor.py</code></summary>
-                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">66</span>
-<span class="normal">67</span>
-<span class="normal">68</span>
-<span class="normal">69</span>
-<span class="normal">70</span>
-<span class="normal">71</span>
-<span class="normal">72</span>
-<span class="normal">73</span>
-<span class="normal">74</span>
-<span class="normal">75</span>
-<span class="normal">76</span>
-<span class="normal">77</span>
-<span class="normal">78</span>
-<span class="normal">79</span>
-<span class="normal">80</span>
-<span class="normal">81</span>
-<span class="normal">82</span>
-<span class="normal">83</span>
-<span class="normal">84</span>
-<span class="normal">85</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
-    <span class="bp">self</span><span class="p">,</span>
-    <span class="n">img_col</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
-    <span class="n">img_path</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
-    <span class="n">width</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">224</span><span class="p">,</span>
-    <span class="n">height</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">224</span><span class="p">,</span>
-    <span class="n">verbose</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1</span><span class="p">,</span>
-<span class="p">):</span>
-    <span class="nb">super</span><span class="p">(</span><span class="n">ImagePreprocessor</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
-
-    <span class="bp">self</span><span class="o">.</span><span class="n">img_col</span> <span class="o">=</span> <span class="n">img_col</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">img_path</span> <span class="o">=</span> <span class="n">img_path</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">width</span> <span class="o">=</span> <span class="n">width</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">height</span> <span class="o">=</span> <span class="n">height</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">verbose</span> <span class="o">=</span> <span class="n">verbose</span>
-
-    <span class="bp">self</span><span class="o">.</span><span class="n">aap</span> <span class="o">=</span> <span class="n">AspectAwarePreprocessor</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">width</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">height</span><span class="p">)</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">spp</span> <span class="o">=</span> <span class="n">SimplePreprocessor</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">width</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">height</span><span class="p">)</span>
-
-    <span class="bp">self</span><span class="o">.</span><span class="n">compute_normalising_computed</span> <span class="o">=</span> <span class="kc">False</span>
-</code></pre></div></td></tr></table></div>
-                  </details>
+    <div class="doc doc-contents ">
 
+        <p>Returns the original text plus the added 'special' tokens</p>
 
 
-  <div class="doc doc-children">
+<p><span class="doc-section-title">Parameters:</span></p>
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+          <th>Default</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>padded_seq</code></td>
+            <td>
+                  <code><span title="numpy.ndarray">ndarray</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>array with the output of the <code>transform</code> method</p>
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+      </tbody>
+    </table>
+
+
+    <p><span class="doc-section-title">Returns:</span></p>
+    <table>
+      <thead>
+        <tr>
+          <th>Type</th>
+          <th>Description</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td>
+                  <code><span title="pandas.DataFrame">DataFrame</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Pandas dataframe with the original text plus the added 'special' tokens</p>
+              </div>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
+            <details class="quote">
+              <summary>Source code in <code>pytorch_widedeep/preprocessing/text_preprocessor.py</code></summary>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">194</span>
+<span class="normal">195</span>
+<span class="normal">196</span>
+<span class="normal">197</span>
+<span class="normal">198</span>
+<span class="normal">199</span>
+<span class="normal">200</span>
+<span class="normal">201</span>
+<span class="normal">202</span>
+<span class="normal">203</span>
+<span class="normal">204</span>
+<span class="normal">205</span>
+<span class="normal">206</span>
+<span class="normal">207</span>
+<span class="normal">208</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">inverse_transform</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">padded_seq</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">:</span>
+<span class="w">    </span><span class="sd">&quot;&quot;&quot;Returns the original text plus the added &#39;special&#39; tokens</span>
 
+<span class="sd">    Parameters</span>
+<span class="sd">    ----------</span>
+<span class="sd">    padded_seq: np.ndarray</span>
+<span class="sd">        array with the output of the `transform` method</span>
 
+<span class="sd">    Returns</span>
+<span class="sd">    -------</span>
+<span class="sd">    pd.DataFrame</span>
+<span class="sd">        Pandas dataframe with the original text plus the added &#39;special&#39; tokens</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="n">texts</span> <span class="o">=</span> <span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">vocab</span><span class="o">.</span><span class="n">inverse_transform</span><span class="p">(</span><span class="n">num</span><span class="p">)</span> <span class="k">for</span> <span class="n">num</span> <span class="ow">in</span> <span class="n">padded_seq</span><span class="p">]</span>
+    <span class="k">return</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">({</span><span class="bp">self</span><span class="o">.</span><span class="n">text_col</span><span class="p">:</span> <span class="n">texts</span><span class="p">})</span>
+</code></pre></div></td></tr></table></div>
+            </details>
+    </div>
 
+</div>
 
 
 
+  </div>
 
+    </div>
 
-<div class="doc doc-object doc-function">
+</div>
 
+<div class="doc doc-object doc-class">
 
-<h3 id="pytorch_widedeep.preprocessing.image_preprocessor.ImagePreprocessor.transform" class="doc doc-heading">
-            <span class="doc doc-object-name doc-function-name">transform</span>
 
 
-<a href="#pytorch_widedeep.preprocessing.image_preprocessor.ImagePreprocessor.transform" class="headerlink" title="Permanent link">&para;</a></h3>
-<div class="doc-signature highlight"><pre><span></span><code><span class="nf">transform</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
-</code></pre></div>
+<h2 id="pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor" class="doc doc-heading">
+            <span class="doc doc-object-name doc-class-name">HFPreprocessor</span>
 
-    <div class="doc doc-contents ">
 
-        <p>Resizes the images to the input height and width.</p>
+<a href="#pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor" class="headerlink" title="Permanent link">&para;</a></h2>
+
+
+    <div class="doc doc-contents first">
+            <p class="doc doc-class-bases">
+              Bases: <code><span title="pytorch_widedeep.preprocessing.base_preprocessor.BasePreprocessor">BasePreprocessor</span></code></p>
+
+
+        <p>Text processor to prepare the <code>deeptext</code> input dataset that is a
+wrapper around HuggingFace's tokenizers.</p>
+<p>Following the main phylosophy of the <code>pytorch-widedeep</code> library, this
+class is designed to be as flexible as possible. Therefore, it is coded
+so that the user can use it as one would use any HuggingFace tokenizers,
+or following the API call 'protocol' of the rest of the library.</p>
 
 
 <p><span class="doc-section-title">Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>df</code></b>
-              (<code><span title="pandas.DataFrame">DataFrame</span></code>)
-          –
-          <div class="doc-md-description">
-            <p>Input pandas dataframe with the <code>img_col</code></p>
-          </div>
-        </li>
-    </ul>
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+          <th>Default</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>model_name</code></td>
+            <td>
+                  <code>str</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>The model name from the transformers library e.g. <em>'bert-base-uncased'</em>.
+Currently supported models are those from the families: BERT, RoBERTa,
+DistilBERT, ALBERT and ELECTRA.</p>
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>use_fast_tokenizer</code></td>
+            <td>
+                  <code>bool</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Whether to use the fast tokenizer from HuggingFace or not</p>
+              </div>
+            </td>
+            <td>
+                  <code>False</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>text_col</code></td>
+            <td>
+                  <code><span title="typing.Optional">Optional</span>[str]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>The column in the input dataframe containing the text data. If this
+tokenizer is used via the <code>fit</code> and <code>transform</code> methods, this
+argument is mandatory. If the tokenizer is used via the <code>encode</code>
+method, this argument is not needed since the input text is passed
+directly to the <code>encode</code> method.</p>
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>num_workers</code></td>
+            <td>
+                  <code><span title="typing.Optional">Optional</span>[int]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Number of workers to use when preprocessing the text data. If not
+None, and <code>use_fast_tokenizer</code> is False, the text data will be
+preprocessed in parallel using the number of workers specified. If
+<code>use_fast_tokenizer</code> is True, this argument is ignored.</p>
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>preprocessing_rules</code></td>
+            <td>
+                  <code><span title="typing.Optional">Optional</span>[<span title="typing.List">List</span>[<span title="typing.Callable">Callable</span>[[str], str]]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>A list of functions to be applied to the text data before encoding.
+This can be useful to clean the text data before encoding. For
+example, removing html tags, special characters, etc.</p>
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>tokenizer_params</code></td>
+            <td>
+                  <code><span title="typing.Optional">Optional</span>[<span title="typing.Dict">Dict</span>[str, <span title="typing.Any">Any</span>]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Additional parameters to be passed to the HuggingFace's
+<code>PreTrainedTokenizer</code>. Parameters to the <code>PreTrainedTokenizer</code>
+can also be passed via the <code>**kwargs</code> argument</p>
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>encode_params</code></td>
+            <td>
+                  <code><span title="typing.Optional">Optional</span>[<span title="typing.Dict">Dict</span>[str, <span title="typing.Any">Any</span>]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Additional parameters to be passed to the <code>batch_encode_plus</code> method
+of the HuggingFace's <code>PreTrainedTokenizer</code>. If the <code>fit</code> and <code>transform</code>
+methods are used, the <code>encode_params</code> dict parameter is mandatory. If
+the <code>encode</code> method is used, this parameter is not needed since the
+input text is passed directly to the <code>encode</code> method.</p>
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>**kwargs</code></td>
+            <td>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Additional kwargs to be passed to the model, in particular to the
+<code>PreTrainedTokenizer</code> class.</p>
+              </div>
+            </td>
+            <td>
+                  <code>{}</code>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
 
-<p><span class="doc-section-title">Returns:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-              <code><span title="numpy.ndarray">ndarray</span></code>
-          –
-          <div class="doc-md-description">
-            <p>Resized images to the input height and width</p>
-          </div>
-        </li>
-    </ul>
+<p><span class="doc-section-title">Attributes:</span></p>
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code><span title="pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.is_fitted">is_fitted</span></code></td>
+            <td>
+                  <code>bool</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Boolean indicating if the preprocessor has been fitted. This is a
+HuggingFacea tokenizer, so it is always considered fitted and this
+attribute is manually set to True internally. This parameter exists
+for consistency with the rest of the library and because is needed
+for some functionality in the library.</p>
+              </div>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
-            <details class="quote">
-              <summary>Source code in <code>pytorch_widedeep/preprocessing/image_preprocessor.py</code></summary>
-              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">100</span>
+
+<p><span class="doc-section-title">Examples:</span></p>
+    <div class="highlight"><pre><span></span><code><span class="gp">&gt;&gt;&gt; </span><span class="kn">import</span> <span class="nn">pandas</span> <span class="k">as</span> <span class="nn">pd</span>
+<span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">pytorch_widedeep.preprocessing</span> <span class="kn">import</span> <span class="n">HFPreprocessor</span>
+<span class="gp">&gt;&gt;&gt; </span><span class="n">df</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">({</span><span class="s2">&quot;text&quot;</span><span class="p">:</span> <span class="p">[</span><span class="s2">&quot;this is the first text&quot;</span><span class="p">,</span> <span class="s2">&quot;this is the second text&quot;</span><span class="p">]})</span>
+<span class="gp">&gt;&gt;&gt; </span><span class="n">hf_processor_1</span> <span class="o">=</span> <span class="n">HFPreprocessor</span><span class="p">(</span><span class="n">model_name</span><span class="o">=</span><span class="s2">&quot;bert-base-uncased&quot;</span><span class="p">,</span> <span class="n">text_col</span><span class="o">=</span><span class="s2">&quot;text&quot;</span><span class="p">)</span>
+<span class="gp">&gt;&gt;&gt; </span><span class="n">X_text_1</span> <span class="o">=</span> <span class="n">hf_processor_1</span><span class="o">.</span><span class="n">fit_transform</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
+<span class="gp">&gt;&gt;&gt; </span><span class="n">texts</span> <span class="o">=</span> <span class="p">[</span><span class="s2">&quot;this is a new text&quot;</span><span class="p">,</span> <span class="s2">&quot;this is another text&quot;</span><span class="p">]</span>
+<span class="gp">&gt;&gt;&gt; </span><span class="n">hf_processor_2</span> <span class="o">=</span> <span class="n">HFPreprocessor</span><span class="p">(</span><span class="n">model_name</span><span class="o">=</span><span class="s2">&quot;bert-base-uncased&quot;</span><span class="p">)</span>
+<span class="gp">&gt;&gt;&gt; </span><span class="n">X_text_2</span> <span class="o">=</span> <span class="n">hf_processor_2</span><span class="o">.</span><span class="n">encode</span><span class="p">(</span><span class="n">texts</span><span class="p">,</span> <span class="n">max_length</span><span class="o">=</span><span class="mi">10</span><span class="p">,</span> <span class="n">padding</span><span class="o">=</span><span class="s2">&quot;max_length&quot;</span><span class="p">,</span> <span class="n">truncation</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
+</code></pre></div>
+
+              <details class="quote">
+                <summary>Source code in <code>pytorch_widedeep/preprocessing/hf_preprocessor.py</code></summary>
+                <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 17</span>
+<span class="normal"> 18</span>
+<span class="normal"> 19</span>
+<span class="normal"> 20</span>
+<span class="normal"> 21</span>
+<span class="normal"> 22</span>
+<span class="normal"> 23</span>
+<span class="normal"> 24</span>
+<span class="normal"> 25</span>
+<span class="normal"> 26</span>
+<span class="normal"> 27</span>
+<span class="normal"> 28</span>
+<span class="normal"> 29</span>
+<span class="normal"> 30</span>
+<span class="normal"> 31</span>
+<span class="normal"> 32</span>
+<span class="normal"> 33</span>
+<span class="normal"> 34</span>
+<span class="normal"> 35</span>
+<span class="normal"> 36</span>
+<span class="normal"> 37</span>
+<span class="normal"> 38</span>
+<span class="normal"> 39</span>
+<span class="normal"> 40</span>
+<span class="normal"> 41</span>
+<span class="normal"> 42</span>
+<span class="normal"> 43</span>
+<span class="normal"> 44</span>
+<span class="normal"> 45</span>
+<span class="normal"> 46</span>
+<span class="normal"> 47</span>
+<span class="normal"> 48</span>
+<span class="normal"> 49</span>
+<span class="normal"> 50</span>
+<span class="normal"> 51</span>
+<span class="normal"> 52</span>
+<span class="normal"> 53</span>
+<span class="normal"> 54</span>
+<span class="normal"> 55</span>
+<span class="normal"> 56</span>
+<span class="normal"> 57</span>
+<span class="normal"> 58</span>
+<span class="normal"> 59</span>
+<span class="normal"> 60</span>
+<span class="normal"> 61</span>
+<span class="normal"> 62</span>
+<span class="normal"> 63</span>
+<span class="normal"> 64</span>
+<span class="normal"> 65</span>
+<span class="normal"> 66</span>
+<span class="normal"> 67</span>
+<span class="normal"> 68</span>
+<span class="normal"> 69</span>
+<span class="normal"> 70</span>
+<span class="normal"> 71</span>
+<span class="normal"> 72</span>
+<span class="normal"> 73</span>
+<span class="normal"> 74</span>
+<span class="normal"> 75</span>
+<span class="normal"> 76</span>
+<span class="normal"> 77</span>
+<span class="normal"> 78</span>
+<span class="normal"> 79</span>
+<span class="normal"> 80</span>
+<span class="normal"> 81</span>
+<span class="normal"> 82</span>
+<span class="normal"> 83</span>
+<span class="normal"> 84</span>
+<span class="normal"> 85</span>
+<span class="normal"> 86</span>
+<span class="normal"> 87</span>
+<span class="normal"> 88</span>
+<span class="normal"> 89</span>
+<span class="normal"> 90</span>
+<span class="normal"> 91</span>
+<span class="normal"> 92</span>
+<span class="normal"> 93</span>
+<span class="normal"> 94</span>
+<span class="normal"> 95</span>
+<span class="normal"> 96</span>
+<span class="normal"> 97</span>
+<span class="normal"> 98</span>
+<span class="normal"> 99</span>
+<span class="normal">100</span>
 <span class="normal">101</span>
 <span class="normal">102</span>
 <span class="normal">103</span>
@@ -5277,349 +6946,9 @@ <h3 id="pytorch_widedeep.preprocessing.image_preprocessor.ImagePreprocessor.tran
 <span class="normal">161</span>
 <span class="normal">162</span>
 <span class="normal">163</span>
-<span class="normal">164</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">transform</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">:</span>
-<span class="w">    </span><span class="sd">&quot;&quot;&quot;Resizes the images to the input height and width.</span>
-
-
-<span class="sd">    Parameters</span>
-<span class="sd">    ----------</span>
-<span class="sd">    df: pd.DataFrame</span>
-<span class="sd">        Input pandas dataframe with the `img_col`</span>
-
-<span class="sd">    Returns</span>
-<span class="sd">    -------</span>
-<span class="sd">    np.ndarray</span>
-<span class="sd">        Resized images to the input height and width</span>
-<span class="sd">    &quot;&quot;&quot;</span>
-    <span class="n">image_list</span> <span class="o">=</span> <span class="n">df</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">img_col</span><span class="p">]</span><span class="o">.</span><span class="n">tolist</span><span class="p">()</span>
-    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">verbose</span><span class="p">:</span>
-        <span class="nb">print</span><span class="p">(</span><span class="s2">&quot;Reading Images from </span><span class="si">{}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">img_path</span><span class="p">))</span>
-    <span class="n">imgs</span> <span class="o">=</span> <span class="p">[</span><span class="n">cv2</span><span class="o">.</span><span class="n">imread</span><span class="p">(</span><span class="s2">&quot;/&quot;</span><span class="o">.</span><span class="n">join</span><span class="p">([</span><span class="bp">self</span><span class="o">.</span><span class="n">img_path</span><span class="p">,</span> <span class="n">img</span><span class="p">]))</span> <span class="k">for</span> <span class="n">img</span> <span class="ow">in</span> <span class="n">image_list</span><span class="p">]</span>
-
-    <span class="c1"># finding images with different height and width</span>
-    <span class="n">aspect</span> <span class="o">=</span> <span class="p">[(</span><span class="n">im</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="n">im</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">1</span><span class="p">])</span> <span class="k">for</span> <span class="n">im</span> <span class="ow">in</span> <span class="n">imgs</span><span class="p">]</span>
-    <span class="n">aspect_r</span> <span class="o">=</span> <span class="p">[</span><span class="n">a</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">/</span> <span class="n">a</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span> <span class="k">for</span> <span class="n">a</span> <span class="ow">in</span> <span class="n">aspect</span><span class="p">]</span>
-    <span class="n">diff_idx</span> <span class="o">=</span> <span class="p">[</span><span class="n">i</span> <span class="k">for</span> <span class="n">i</span><span class="p">,</span> <span class="n">r</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="n">aspect_r</span><span class="p">)</span> <span class="k">if</span> <span class="n">r</span> <span class="o">!=</span> <span class="mf">1.0</span><span class="p">]</span>
-
-    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">verbose</span><span class="p">:</span>
-        <span class="nb">print</span><span class="p">(</span><span class="s2">&quot;Resizing&quot;</span><span class="p">)</span>
-    <span class="n">resized_imgs</span> <span class="o">=</span> <span class="p">[]</span>
-    <span class="k">for</span> <span class="n">i</span><span class="p">,</span> <span class="n">img</span> <span class="ow">in</span> <span class="n">tqdm</span><span class="p">(</span><span class="nb">enumerate</span><span class="p">(</span><span class="n">imgs</span><span class="p">),</span> <span class="n">total</span><span class="o">=</span><span class="nb">len</span><span class="p">(</span><span class="n">imgs</span><span class="p">),</span> <span class="n">disable</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">verbose</span> <span class="o">!=</span> <span class="mi">1</span><span class="p">):</span>
-        <span class="k">if</span> <span class="n">i</span> <span class="ow">in</span> <span class="n">diff_idx</span><span class="p">:</span>
-            <span class="n">resized_imgs</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">aap</span><span class="o">.</span><span class="n">preprocess</span><span class="p">(</span><span class="n">img</span><span class="p">))</span>
-        <span class="k">else</span><span class="p">:</span>
-            <span class="c1"># if aspect ratio is 1:1, no need for AspectAwarePreprocessor</span>
-            <span class="n">resized_imgs</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">spp</span><span class="o">.</span><span class="n">preprocess</span><span class="p">(</span><span class="n">img</span><span class="p">))</span>
-
-    <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">compute_normalising_computed</span><span class="p">:</span>
-        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">verbose</span><span class="p">:</span>
-            <span class="nb">print</span><span class="p">(</span><span class="s2">&quot;Computing normalisation metrics&quot;</span><span class="p">)</span>
-        <span class="c1"># mean and std deviation will only be computed when the fit method</span>
-        <span class="c1"># is called</span>
-        <span class="n">mean_R</span><span class="p">,</span> <span class="n">mean_G</span><span class="p">,</span> <span class="n">mean_B</span> <span class="o">=</span> <span class="p">[],</span> <span class="p">[],</span> <span class="p">[]</span>
-        <span class="n">std_R</span><span class="p">,</span> <span class="n">std_G</span><span class="p">,</span> <span class="n">std_B</span> <span class="o">=</span> <span class="p">[],</span> <span class="p">[],</span> <span class="p">[]</span>
-        <span class="k">for</span> <span class="n">rsz_img</span> <span class="ow">in</span> <span class="n">resized_imgs</span><span class="p">:</span>
-            <span class="p">(</span><span class="n">mean_b</span><span class="p">,</span> <span class="n">mean_g</span><span class="p">,</span> <span class="n">mean_r</span><span class="p">),</span> <span class="p">(</span><span class="n">std_b</span><span class="p">,</span> <span class="n">std_g</span><span class="p">,</span> <span class="n">std_r</span><span class="p">)</span> <span class="o">=</span> <span class="n">cv2</span><span class="o">.</span><span class="n">meanStdDev</span><span class="p">(</span>
-                <span class="n">rsz_img</span>
-            <span class="p">)</span>
-            <span class="n">mean_R</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">mean_r</span><span class="p">)</span>
-            <span class="n">mean_G</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">mean_g</span><span class="p">)</span>
-            <span class="n">mean_B</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">mean_b</span><span class="p">)</span>
-            <span class="n">std_R</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">std_r</span><span class="p">)</span>
-            <span class="n">std_G</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">std_g</span><span class="p">)</span>
-            <span class="n">std_B</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">std_b</span><span class="p">)</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">normalise_metrics</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">(</span>
-            <span class="n">mean</span><span class="o">=</span><span class="p">{</span>
-                <span class="s2">&quot;R&quot;</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="n">mean_R</span><span class="p">)</span> <span class="o">/</span> <span class="mf">255.0</span><span class="p">,</span>
-                <span class="s2">&quot;G&quot;</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="n">mean_G</span><span class="p">)</span> <span class="o">/</span> <span class="mf">255.0</span><span class="p">,</span>
-                <span class="s2">&quot;B&quot;</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="n">mean_B</span><span class="p">)</span> <span class="o">/</span> <span class="mf">255.0</span><span class="p">,</span>
-            <span class="p">},</span>
-            <span class="n">std</span><span class="o">=</span><span class="p">{</span>
-                <span class="s2">&quot;R&quot;</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="n">std_R</span><span class="p">)</span> <span class="o">/</span> <span class="mf">255.0</span><span class="p">,</span>
-                <span class="s2">&quot;G&quot;</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="n">std_G</span><span class="p">)</span> <span class="o">/</span> <span class="mf">255.0</span><span class="p">,</span>
-                <span class="s2">&quot;B&quot;</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="n">std_B</span><span class="p">)</span> <span class="o">/</span> <span class="mf">255.0</span><span class="p">,</span>
-            <span class="p">},</span>
-        <span class="p">)</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">compute_normalising_computed</span> <span class="o">=</span> <span class="kc">True</span>
-    <span class="k">return</span> <span class="n">np</span><span class="o">.</span><span class="n">asarray</span><span class="p">(</span><span class="n">resized_imgs</span><span class="p">)</span>
-</code></pre></div></td></tr></table></div>
-            </details>
-    </div>
-
-</div>
-
-<div class="doc doc-object doc-function">
-
-
-<h3 id="pytorch_widedeep.preprocessing.image_preprocessor.ImagePreprocessor.fit_transform" class="doc doc-heading">
-            <span class="doc doc-object-name doc-function-name">fit_transform</span>
-
-
-<a href="#pytorch_widedeep.preprocessing.image_preprocessor.ImagePreprocessor.fit_transform" class="headerlink" title="Permanent link">&para;</a></h3>
-<div class="doc-signature highlight"><pre><span></span><code><span class="nf">fit_transform</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
-</code></pre></div>
-
-    <div class="doc doc-contents ">
-
-        <p>Combines <code>fit</code> and <code>transform</code></p>
-
-
-<p><span class="doc-section-title">Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>df</code></b>
-              (<code><span title="pandas.DataFrame">DataFrame</span></code>)
-          –
-          <div class="doc-md-description">
-            <p>Input pandas dataframe</p>
-          </div>
-        </li>
-    </ul>
-
-
-<p><span class="doc-section-title">Returns:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-              <code><span title="numpy.ndarray">ndarray</span></code>
-          –
-          <div class="doc-md-description">
-            <p>Resized images to the input height and width</p>
-          </div>
-        </li>
-    </ul>
-
-            <details class="quote">
-              <summary>Source code in <code>pytorch_widedeep/preprocessing/image_preprocessor.py</code></summary>
-              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">166</span>
-<span class="normal">167</span>
-<span class="normal">168</span>
-<span class="normal">169</span>
-<span class="normal">170</span>
-<span class="normal">171</span>
-<span class="normal">172</span>
-<span class="normal">173</span>
-<span class="normal">174</span>
-<span class="normal">175</span>
-<span class="normal">176</span>
-<span class="normal">177</span>
-<span class="normal">178</span>
-<span class="normal">179</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">fit_transform</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">:</span>
-<span class="w">    </span><span class="sd">&quot;&quot;&quot;Combines `fit` and `transform`</span>
-
-<span class="sd">    Parameters</span>
-<span class="sd">    ----------</span>
-<span class="sd">    df: pd.DataFrame</span>
-<span class="sd">        Input pandas dataframe</span>
-
-<span class="sd">    Returns</span>
-<span class="sd">    -------</span>
-<span class="sd">    np.ndarray</span>
-<span class="sd">        Resized images to the input height and width</span>
-<span class="sd">    &quot;&quot;&quot;</span>
-    <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">df</span><span class="p">)</span><span class="o">.</span><span class="n">transform</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
-</code></pre></div></td></tr></table></div>
-            </details>
-    </div>
-
-</div>
-
-
-
-  </div>
-
-    </div>
-
-</div>
-
-<div class="doc doc-object doc-class">
-
-
-
-<h2 id="pytorch_widedeep.preprocessing.din_preprocessor.DINPreprocessor" class="doc doc-heading">
-            <span class="doc doc-object-name doc-class-name">DINPreprocessor</span>
-
-
-<a href="#pytorch_widedeep.preprocessing.din_preprocessor.DINPreprocessor" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="nf">DINPreprocessor</span><span class="p">(</span>
-    <span class="o">*</span><span class="p">,</span>
-    <span class="n">user_id_col</span><span class="p">,</span>
-    <span class="n">item_embed_col</span><span class="p">,</span>
-    <span class="n">target_col</span><span class="p">,</span>
-    <span class="n">max_seq_length</span><span class="p">,</span>
-    <span class="n">action_col</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">other_seq_embed_cols</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">cat_embed_cols</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">continuous_cols</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">quantization_setup</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">cols_to_scale</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">auto_embed_dim</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
-    <span class="n">embedding_rule</span><span class="o">=</span><span class="s2">&quot;fastai_new&quot;</span><span class="p">,</span>
-    <span class="n">default_embed_dim</span><span class="o">=</span><span class="mi">16</span><span class="p">,</span>
-    <span class="n">verbose</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span>
-    <span class="n">scale</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
-    <span class="n">already_standard</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="o">**</span><span class="n">kwargs</span>
-<span class="p">)</span>
-</code></pre></div>
-
-    <div class="doc doc-contents first">
-            <p class="doc doc-class-bases">
-              Bases: <code><span title="pytorch_widedeep.preprocessing.base_preprocessor.BasePreprocessor">BasePreprocessor</span></code></p>
-
-
-        <p>Preprocessor for Deep Interest Network (DIN) models.</p>
-<p>This preprocessor handles the preparation of data for DIN models,
-including sequence building, label encoding, and handling of various
-types of input columns (categorical, continuous, and sequential).</p>
-
-
-<details class="parameters:" open>
-  <summary></summary>
-  <p>user_id_col : str
-    Name of the column containing user IDs.
-item_embed_col : Union[str, Tuple[str, int]]
-    Name of the column containing item IDs to be embedded, or a tuple of
-    (column_name, embedding_dim).
-target_col : str
-    Name of the column containing the target variable.
-max_seq_length : int
-    Maximum length of sequences to be created.
-action_col : Optional[str], default=None
-    Name of the column containing user actions (if applicable).
-other_seq_embed_cols : Optional[List[str] | List[Tuple[str, int]]], default=None
-    List of other columns to be treated as sequences.
-cat_embed_cols : Optional[Union[List[str], List[Tuple[str, int]]]], default=None
-    List of categorical columns to be represented by embeddings.
-continuous_cols: List, default = None
-    List with the name of the continuous cols.
-quantization_setup: int or Dict[str, Union[int, List[float]]], default=None
-    Continuous columns can be turned into categorical via <code>pd.cut</code>.
-cols_to_scale: List or str, default = None,
-    List with the names of the columns that will be standardized via
-    sklearn's <code>StandardScaler</code>.
-auto_embed_dim: bool, default = True
-    Boolean indicating whether the embedding dimensions will be
-    automatically defined via rule of thumb.
-embedding_rule: str, default = 'fastai_new'
-    Rule of thumb for embedding size.
-default_embed_dim: int, default=16
-    Default dimension for the embeddings.
-verbose : int, default=1
-    Verbosity level.
-scale: bool, default = False
-    Boolean indicating whether or not to scale/standardize continuous cols.
-already_standard: List, default = None
-    List with the name of the continuous cols that do not need to be
-    scaled/standardized.
-**kwargs :
-    Additional keyword arguments to be passed to the TabPreprocessor.</p>
-</details>
-
-<details class="attributes:" open>
-  <summary></summary>
-  <p>is_fitted : bool
-    Whether the preprocessor has been fitted.
-has_standard_tab_data : bool
-    Whether the data includes standard tabular data.
-tab_preprocessor : TabPreprocessor
-    Preprocessor for standard tabular data.
-din_columns_idx : Dict[str, int]
-    Dictionary mapping column names to their indices in the processed data.
-item_le : LabelEncoder
-    Label encoder for item IDs.
-n_items : int
-    Number of unique items.
-user_behaviour_config : Tuple[List[str], int, int]
-    Configuration for user behavior sequences.
-action_le : LabelEncoder
-    Label encoder for action column (if applicable).
-n_actions : int
-    Number of unique actions (if applicable).
-action_seq_config : Tuple[List[str], int]
-    Configuration for action sequences (if applicable).
-other_seq_le : LabelEncoder
-    Label encoder for other sequence columns (if applicable).
-n_other_seq_cols : Dict[str, int]
-    Number of unique values in each other sequence column.
-other_seq_config : List[Tuple[List[str], int, int]]
-    Configuration for other sequence columns.</p>
-</details>
-
-<details class="examples:" open>
-  <summary></summary>
-  <blockquote>
-<blockquote>
-<blockquote>
-<p>import pandas as pd
-from pytorch_widedeep.preprocessing import DINPreprocessor
-data = {
-...    'user_id': [1, 1, 1, 2, 2, 2, 3, 3, 3],
-...    'item_id': [101, 102, 103, 101, 103, 104, 102, 103, 104],
-...    'timestamp': [1, 2, 3, 1, 2, 3, 1, 2, 3],
-...    'category': ['A', 'B', 'A', 'B', 'A', 'C', 'B', 'A', 'C'],
-...    'price': [10.5, 15.0, 12.0, 10.5, 12.0, 20.0, 15.0, 12.0, 20.0],
-...    'rating': [0, 1, 0, 1, 0, 1, 0, 1, 0]
-... }
-df = pd.DataFrame(data)
-din_preprocessor = DINPreprocessor(
-...     user_id_col='user_id',
-...     item_embed_col='item_id',
-...     target_col='rating',
-...     max_seq_length=2,
-...     action_col='rating',
-...     other_seq_embed_cols=['category'],
-...     cat_embed_cols=['user_id'],
-...     continuous_cols=['price'],
-...     cols_to_scale=['price']
-... )
-X, y = din_preprocessor.fit_transform(df)</p>
-</blockquote>
-</blockquote>
-</blockquote>
-</details>
-                  <details class="quote">
-                    <summary>Source code in <code>pytorch_widedeep/preprocessing/din_preprocessor.py</code></summary>
-                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">126</span>
-<span class="normal">127</span>
-<span class="normal">128</span>
-<span class="normal">129</span>
-<span class="normal">130</span>
-<span class="normal">131</span>
-<span class="normal">132</span>
-<span class="normal">133</span>
-<span class="normal">134</span>
-<span class="normal">135</span>
-<span class="normal">136</span>
-<span class="normal">137</span>
-<span class="normal">138</span>
-<span class="normal">139</span>
-<span class="normal">140</span>
-<span class="normal">141</span>
-<span class="normal">142</span>
-<span class="normal">143</span>
-<span class="normal">144</span>
-<span class="normal">145</span>
-<span class="normal">146</span>
-<span class="normal">147</span>
-<span class="normal">148</span>
-<span class="normal">149</span>
-<span class="normal">150</span>
-<span class="normal">151</span>
-<span class="normal">152</span>
-<span class="normal">153</span>
-<span class="normal">154</span>
-<span class="normal">155</span>
-<span class="normal">156</span>
-<span class="normal">157</span>
-<span class="normal">158</span>
-<span class="normal">159</span>
-<span class="normal">160</span>
-<span class="normal">161</span>
-<span class="normal">162</span>
-<span class="normal">163</span>
-<span class="normal">164</span>
-<span class="normal">165</span>
-<span class="normal">166</span>
+<span class="normal">164</span>
+<span class="normal">165</span>
+<span class="normal">166</span>
 <span class="normal">167</span>
 <span class="normal">168</span>
 <span class="normal">169</span>
@@ -5637,67 +6966,3938 @@ <h2 id="pytorch_widedeep.preprocessing.din_preprocessor.DINPreprocessor" class="
 <span class="normal">181</span>
 <span class="normal">182</span>
 <span class="normal">183</span>
-<span class="normal">184</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="nd">@alias</span><span class="p">(</span><span class="s2">&quot;item_embed_col&quot;</span><span class="p">,</span> <span class="p">[</span><span class="s2">&quot;item_id_col&quot;</span><span class="p">])</span>
-<span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
-    <span class="bp">self</span><span class="p">,</span>
-    <span class="o">*</span><span class="p">,</span>
-    <span class="n">user_id_col</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
-    <span class="n">item_embed_col</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Tuple</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">]],</span>
-    <span class="n">target_col</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
-    <span class="n">max_seq_length</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span>
-    <span class="n">action_col</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">other_seq_embed_cols</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Union</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">],</span> <span class="n">List</span><span class="p">[</span><span class="n">Tuple</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">]]]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">cat_embed_cols</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Union</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">],</span> <span class="n">List</span><span class="p">[</span><span class="n">Tuple</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">]]]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">continuous_cols</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">quantization_setup</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span>
-        <span class="n">Union</span><span class="p">[</span><span class="nb">int</span><span class="p">,</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Union</span><span class="p">[</span><span class="nb">int</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="nb">float</span><span class="p">]]]]</span>
-    <span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">cols_to_scale</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Union</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">],</span> <span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">auto_embed_dim</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">,</span>
-    <span class="n">embedding_rule</span><span class="p">:</span> <span class="n">Literal</span><span class="p">[</span><span class="s2">&quot;google&quot;</span><span class="p">,</span> <span class="s2">&quot;fastai_old&quot;</span><span class="p">,</span> <span class="s2">&quot;fastai_new&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="s2">&quot;fastai_new&quot;</span><span class="p">,</span>
-    <span class="n">default_embed_dim</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">16</span><span class="p">,</span>
-    <span class="n">verbose</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1</span><span class="p">,</span>
-    <span class="n">scale</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
-    <span class="n">already_standard</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="o">**</span><span class="n">kwargs</span><span class="p">,</span>
-<span class="p">):</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">user_id_col</span> <span class="o">=</span> <span class="n">user_id_col</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">item_embed_col</span> <span class="o">=</span> <span class="n">item_embed_col</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">max_seq_length</span> <span class="o">=</span> <span class="n">max_seq_length</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">target_col</span> <span class="o">=</span> <span class="n">target_col</span> <span class="k">if</span> <span class="n">target_col</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="k">else</span> <span class="s2">&quot;target&quot;</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">action_col</span> <span class="o">=</span> <span class="n">action_col</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">other_seq_embed_cols</span> <span class="o">=</span> <span class="n">other_seq_embed_cols</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">cat_embed_cols</span> <span class="o">=</span> <span class="n">cat_embed_cols</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">continuous_cols</span> <span class="o">=</span> <span class="n">continuous_cols</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">quantization_setup</span> <span class="o">=</span> <span class="n">quantization_setup</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">cols_to_scale</span> <span class="o">=</span> <span class="n">cols_to_scale</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">auto_embed_dim</span> <span class="o">=</span> <span class="n">auto_embed_dim</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">embedding_rule</span> <span class="o">=</span> <span class="n">embedding_rule</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">default_embed_dim</span> <span class="o">=</span> <span class="n">default_embed_dim</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">verbose</span> <span class="o">=</span> <span class="n">verbose</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">scale</span> <span class="o">=</span> <span class="n">scale</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">already_standard</span> <span class="o">=</span> <span class="n">already_standard</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">kwargs</span> <span class="o">=</span> <span class="n">kwargs</span>
-
-    <span class="bp">self</span><span class="o">.</span><span class="n">has_standard_tab_data</span> <span class="o">=</span> <span class="nb">bool</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">cat_embed_cols</span> <span class="ow">or</span> <span class="bp">self</span><span class="o">.</span><span class="n">continuous_cols</span><span class="p">)</span>
-    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">has_standard_tab_data</span><span class="p">:</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">tab_preprocessor</span> <span class="o">=</span> <span class="n">TabPreprocessor</span><span class="p">(</span>
-            <span class="n">cat_embed_cols</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">cat_embed_cols</span><span class="p">,</span>
-            <span class="n">continuous_cols</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">continuous_cols</span><span class="p">,</span>
-            <span class="n">quantization_setup</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">quantization_setup</span><span class="p">,</span>
-            <span class="n">cols_to_scale</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">cols_to_scale</span><span class="p">,</span>
-            <span class="n">auto_embed_dim</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">auto_embed_dim</span><span class="p">,</span>
-            <span class="n">embedding_rule</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">embedding_rule</span><span class="p">,</span>
-            <span class="n">default_embed_dim</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">default_embed_dim</span><span class="p">,</span>
-            <span class="n">verbose</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">verbose</span><span class="p">,</span>
-            <span class="n">scale</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">scale</span><span class="p">,</span>
-            <span class="n">already_standard</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">already_standard</span><span class="p">,</span>
-            <span class="o">**</span><span class="bp">self</span><span class="o">.</span><span class="n">kwargs</span><span class="p">,</span>
+<span class="normal">184</span>
+<span class="normal">185</span>
+<span class="normal">186</span>
+<span class="normal">187</span>
+<span class="normal">188</span>
+<span class="normal">189</span>
+<span class="normal">190</span>
+<span class="normal">191</span>
+<span class="normal">192</span>
+<span class="normal">193</span>
+<span class="normal">194</span>
+<span class="normal">195</span>
+<span class="normal">196</span>
+<span class="normal">197</span>
+<span class="normal">198</span>
+<span class="normal">199</span>
+<span class="normal">200</span>
+<span class="normal">201</span>
+<span class="normal">202</span>
+<span class="normal">203</span>
+<span class="normal">204</span>
+<span class="normal">205</span>
+<span class="normal">206</span>
+<span class="normal">207</span>
+<span class="normal">208</span>
+<span class="normal">209</span>
+<span class="normal">210</span>
+<span class="normal">211</span>
+<span class="normal">212</span>
+<span class="normal">213</span>
+<span class="normal">214</span>
+<span class="normal">215</span>
+<span class="normal">216</span>
+<span class="normal">217</span>
+<span class="normal">218</span>
+<span class="normal">219</span>
+<span class="normal">220</span>
+<span class="normal">221</span>
+<span class="normal">222</span>
+<span class="normal">223</span>
+<span class="normal">224</span>
+<span class="normal">225</span>
+<span class="normal">226</span>
+<span class="normal">227</span>
+<span class="normal">228</span>
+<span class="normal">229</span>
+<span class="normal">230</span>
+<span class="normal">231</span>
+<span class="normal">232</span>
+<span class="normal">233</span>
+<span class="normal">234</span>
+<span class="normal">235</span>
+<span class="normal">236</span>
+<span class="normal">237</span>
+<span class="normal">238</span>
+<span class="normal">239</span>
+<span class="normal">240</span>
+<span class="normal">241</span>
+<span class="normal">242</span>
+<span class="normal">243</span>
+<span class="normal">244</span>
+<span class="normal">245</span>
+<span class="normal">246</span>
+<span class="normal">247</span>
+<span class="normal">248</span>
+<span class="normal">249</span>
+<span class="normal">250</span>
+<span class="normal">251</span>
+<span class="normal">252</span>
+<span class="normal">253</span>
+<span class="normal">254</span>
+<span class="normal">255</span>
+<span class="normal">256</span>
+<span class="normal">257</span>
+<span class="normal">258</span>
+<span class="normal">259</span>
+<span class="normal">260</span>
+<span class="normal">261</span>
+<span class="normal">262</span>
+<span class="normal">263</span>
+<span class="normal">264</span>
+<span class="normal">265</span>
+<span class="normal">266</span>
+<span class="normal">267</span>
+<span class="normal">268</span>
+<span class="normal">269</span>
+<span class="normal">270</span>
+<span class="normal">271</span>
+<span class="normal">272</span>
+<span class="normal">273</span>
+<span class="normal">274</span>
+<span class="normal">275</span>
+<span class="normal">276</span>
+<span class="normal">277</span>
+<span class="normal">278</span>
+<span class="normal">279</span>
+<span class="normal">280</span>
+<span class="normal">281</span>
+<span class="normal">282</span>
+<span class="normal">283</span>
+<span class="normal">284</span>
+<span class="normal">285</span>
+<span class="normal">286</span>
+<span class="normal">287</span>
+<span class="normal">288</span>
+<span class="normal">289</span>
+<span class="normal">290</span>
+<span class="normal">291</span>
+<span class="normal">292</span>
+<span class="normal">293</span>
+<span class="normal">294</span>
+<span class="normal">295</span>
+<span class="normal">296</span>
+<span class="normal">297</span>
+<span class="normal">298</span>
+<span class="normal">299</span>
+<span class="normal">300</span>
+<span class="normal">301</span>
+<span class="normal">302</span>
+<span class="normal">303</span>
+<span class="normal">304</span>
+<span class="normal">305</span>
+<span class="normal">306</span>
+<span class="normal">307</span>
+<span class="normal">308</span>
+<span class="normal">309</span>
+<span class="normal">310</span>
+<span class="normal">311</span>
+<span class="normal">312</span>
+<span class="normal">313</span>
+<span class="normal">314</span>
+<span class="normal">315</span>
+<span class="normal">316</span>
+<span class="normal">317</span>
+<span class="normal">318</span>
+<span class="normal">319</span>
+<span class="normal">320</span>
+<span class="normal">321</span>
+<span class="normal">322</span>
+<span class="normal">323</span>
+<span class="normal">324</span>
+<span class="normal">325</span>
+<span class="normal">326</span>
+<span class="normal">327</span>
+<span class="normal">328</span>
+<span class="normal">329</span>
+<span class="normal">330</span>
+<span class="normal">331</span>
+<span class="normal">332</span>
+<span class="normal">333</span>
+<span class="normal">334</span>
+<span class="normal">335</span>
+<span class="normal">336</span>
+<span class="normal">337</span>
+<span class="normal">338</span>
+<span class="normal">339</span>
+<span class="normal">340</span>
+<span class="normal">341</span>
+<span class="normal">342</span>
+<span class="normal">343</span>
+<span class="normal">344</span>
+<span class="normal">345</span>
+<span class="normal">346</span>
+<span class="normal">347</span>
+<span class="normal">348</span>
+<span class="normal">349</span>
+<span class="normal">350</span>
+<span class="normal">351</span>
+<span class="normal">352</span>
+<span class="normal">353</span>
+<span class="normal">354</span>
+<span class="normal">355</span>
+<span class="normal">356</span>
+<span class="normal">357</span>
+<span class="normal">358</span>
+<span class="normal">359</span>
+<span class="normal">360</span>
+<span class="normal">361</span>
+<span class="normal">362</span>
+<span class="normal">363</span>
+<span class="normal">364</span>
+<span class="normal">365</span>
+<span class="normal">366</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">class</span> <span class="nc">HFPreprocessor</span><span class="p">(</span><span class="n">BasePreprocessor</span><span class="p">):</span>
+<span class="w">    </span><span class="sd">&quot;&quot;&quot;Text processor to prepare the ``deeptext`` input dataset that is a</span>
+<span class="sd">    wrapper around HuggingFace&#39;s tokenizers.</span>
+
+<span class="sd">    Following the main phylosophy of the `pytorch-widedeep` library, this</span>
+<span class="sd">    class is designed to be as flexible as possible. Therefore, it is coded</span>
+<span class="sd">    so that the user can use it as one would use any HuggingFace tokenizers,</span>
+<span class="sd">    or following the API call &#39;protocol&#39; of the rest of the library.</span>
+
+<span class="sd">    Parameters</span>
+<span class="sd">    ----------</span>
+<span class="sd">    model_name: str</span>
+<span class="sd">        The model name from the transformers library e.g. _&#39;bert-base-uncased&#39;_.</span>
+<span class="sd">        Currently supported models are those from the families: BERT, RoBERTa,</span>
+<span class="sd">        DistilBERT, ALBERT and ELECTRA.</span>
+<span class="sd">    use_fast_tokenizer: bool, default = False</span>
+<span class="sd">        Whether to use the fast tokenizer from HuggingFace or not</span>
+<span class="sd">    text_col: Optional[str], default = None</span>
+<span class="sd">        The column in the input dataframe containing the text data. If this</span>
+<span class="sd">        tokenizer is used via the `fit` and `transform` methods, this</span>
+<span class="sd">        argument is mandatory. If the tokenizer is used via the `encode`</span>
+<span class="sd">        method, this argument is not needed since the input text is passed</span>
+<span class="sd">        directly to the `encode` method.</span>
+<span class="sd">    num_workers: Optional[int], default = None</span>
+<span class="sd">        Number of workers to use when preprocessing the text data. If not</span>
+<span class="sd">        None, and `use_fast_tokenizer` is False, the text data will be</span>
+<span class="sd">        preprocessed in parallel using the number of workers specified. If</span>
+<span class="sd">        `use_fast_tokenizer` is True, this argument is ignored.</span>
+<span class="sd">    preprocessing_rules: Optional[List[Callable[[str], str]]], default = None</span>
+<span class="sd">        A list of functions to be applied to the text data before encoding.</span>
+<span class="sd">        This can be useful to clean the text data before encoding. For</span>
+<span class="sd">        example, removing html tags, special characters, etc.</span>
+<span class="sd">    tokenizer_params: Optional[Dict[str, Any]], default = None</span>
+<span class="sd">        Additional parameters to be passed to the HuggingFace&#39;s</span>
+<span class="sd">        `PreTrainedTokenizer`. Parameters to the `PreTrainedTokenizer`</span>
+<span class="sd">        can also be passed via the `**kwargs` argument</span>
+<span class="sd">    encode_params: Optional[Dict[str, Any]], default = None</span>
+<span class="sd">        Additional parameters to be passed to the `batch_encode_plus` method</span>
+<span class="sd">        of the HuggingFace&#39;s `PreTrainedTokenizer`. If the `fit` and `transform`</span>
+<span class="sd">        methods are used, the `encode_params` dict parameter is mandatory. If</span>
+<span class="sd">        the `encode` method is used, this parameter is not needed since the</span>
+<span class="sd">        input text is passed directly to the `encode` method.</span>
+<span class="sd">    **kwargs</span>
+<span class="sd">        Additional kwargs to be passed to the model, in particular to the</span>
+<span class="sd">        `PreTrainedTokenizer` class.</span>
+
+<span class="sd">    Attributes</span>
+<span class="sd">    ----------</span>
+<span class="sd">    is_fitted: bool</span>
+<span class="sd">        Boolean indicating if the preprocessor has been fitted. This is a</span>
+<span class="sd">        HuggingFacea tokenizer, so it is always considered fitted and this</span>
+<span class="sd">        attribute is manually set to True internally. This parameter exists</span>
+<span class="sd">        for consistency with the rest of the library and because is needed</span>
+<span class="sd">        for some functionality in the library.</span>
+
+<span class="sd">    Examples</span>
+<span class="sd">    --------</span>
+<span class="sd">    &gt;&gt;&gt; import pandas as pd</span>
+<span class="sd">    &gt;&gt;&gt; from pytorch_widedeep.preprocessing import HFPreprocessor</span>
+<span class="sd">    &gt;&gt;&gt; df = pd.DataFrame({&quot;text&quot;: [&quot;this is the first text&quot;, &quot;this is the second text&quot;]})</span>
+<span class="sd">    &gt;&gt;&gt; hf_processor_1 = HFPreprocessor(model_name=&quot;bert-base-uncased&quot;, text_col=&quot;text&quot;)</span>
+<span class="sd">    &gt;&gt;&gt; X_text_1 = hf_processor_1.fit_transform(df)</span>
+<span class="sd">    &gt;&gt;&gt; texts = [&quot;this is a new text&quot;, &quot;this is another text&quot;]</span>
+<span class="sd">    &gt;&gt;&gt; hf_processor_2 = HFPreprocessor(model_name=&quot;bert-base-uncased&quot;)</span>
+<span class="sd">    &gt;&gt;&gt; X_text_2 = hf_processor_2.encode(texts, max_length=10, padding=&quot;max_length&quot;, truncation=True)</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+
+    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span>
+        <span class="n">model_name</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
+        <span class="o">*</span><span class="p">,</span>
+        <span class="n">use_fast_tokenizer</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
+        <span class="n">text_col</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">root_dir</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">num_workers</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">preprocessing_rules</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="n">Callable</span><span class="p">[[</span><span class="nb">str</span><span class="p">],</span> <span class="nb">str</span><span class="p">]]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">tokenizer_params</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">encode_params</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="o">**</span><span class="n">kwargs</span><span class="p">,</span>
+    <span class="p">):</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">model_name</span> <span class="o">=</span> <span class="n">model_name</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">use_fast_tokenizer</span> <span class="o">=</span> <span class="n">use_fast_tokenizer</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">text_col</span> <span class="o">=</span> <span class="n">text_col</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">root_dir</span> <span class="o">=</span> <span class="n">root_dir</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">num_workers</span> <span class="o">=</span> <span class="n">num_workers</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">preprocessing_rules</span> <span class="o">=</span> <span class="n">preprocessing_rules</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">tokenizer_params</span> <span class="o">=</span> <span class="n">tokenizer_params</span> <span class="k">if</span> <span class="n">tokenizer_params</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="k">else</span> <span class="p">{}</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">encode_params</span> <span class="o">=</span> <span class="n">encode_params</span> <span class="k">if</span> <span class="n">encode_params</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="k">else</span> <span class="p">{}</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">_multiprocessing</span> <span class="o">=</span> <span class="p">(</span>
+            <span class="n">num_workers</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">and</span> <span class="n">num_workers</span> <span class="o">&gt;</span> <span class="mi">1</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">use_fast_tokenizer</span>
+        <span class="p">)</span>
+
+        <span class="k">if</span> <span class="n">kwargs</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">tokenizer_params</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="n">kwargs</span><span class="p">)</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">tokenizer</span> <span class="o">=</span> <span class="n">get_tokenizer</span><span class="p">(</span>
+            <span class="n">model_name</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">model_name</span><span class="p">,</span>
+            <span class="n">use_fast_tokenizer</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">use_fast_tokenizer</span><span class="p">,</span>
+            <span class="o">**</span><span class="bp">self</span><span class="o">.</span><span class="n">tokenizer_params</span><span class="p">,</span>
+        <span class="p">)</span>
+
+        <span class="c1"># A HuggingFace tokenizer is already trained, since we need this</span>
+        <span class="c1"># attribute elsewhere in the library, we simply set it to True</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">is_fitted</span> <span class="o">=</span> <span class="kc">True</span>
+
+    <span class="k">def</span> <span class="nf">encode</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">texts</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">],</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">npt</span><span class="o">.</span><span class="n">NDArray</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">int64</span><span class="p">]:</span>
+<span class="w">        </span><span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Encodes a list of texts. The method is a wrapper around the</span>
+<span class="sd">        `batch_encode_plus` method of the HuggingFace&#39;s tokenizer.</span>
+
+<span class="sd">        if &#39;use_fast_tokenizer&#39; is True, the method will use the `batch_encode_plus`</span>
+
+<span class="sd">        Parameters</span>
+<span class="sd">        ----------</span>
+<span class="sd">        texts: List[str]</span>
+<span class="sd">            List of texts to be encoded</span>
+<span class="sd">        **kwargs</span>
+<span class="sd">            Additional parameters to be passed to the `batch_encode_plus` method</span>
+<span class="sd">            of the HuggingFace&#39;s tokenizer. If the &#39;encode_params&#39; dict was passed</span>
+<span class="sd">            when instantiating the class, that dictionaly will be updated with</span>
+<span class="sd">            the kwargs passed here.</span>
+
+<span class="sd">        Returns</span>
+<span class="sd">        -------</span>
+<span class="sd">        np.array</span>
+<span class="sd">            The encoded texts</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="k">if</span> <span class="n">kwargs</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">encode_params</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="n">kwargs</span><span class="p">)</span>
+
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">preprocessing_rules</span><span class="p">:</span>
+            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_multiprocessing</span><span class="p">:</span>
+                <span class="n">texts</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_process_text_parallel</span><span class="p">(</span><span class="n">texts</span><span class="p">)</span>
+            <span class="k">else</span><span class="p">:</span>
+                <span class="n">texts</span> <span class="o">=</span> <span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">_preprocess_text</span><span class="p">(</span><span class="n">text</span><span class="p">)</span> <span class="k">for</span> <span class="n">text</span> <span class="ow">in</span> <span class="n">texts</span><span class="p">]</span>
+
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_multiprocessing</span><span class="p">:</span>
+            <span class="n">input_ids</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_encode_paralell</span><span class="p">(</span><span class="n">texts</span><span class="p">,</span> <span class="o">**</span><span class="bp">self</span><span class="o">.</span><span class="n">encode_params</span><span class="p">)</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="n">encoded_texts</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">tokenizer</span><span class="o">.</span><span class="n">batch_encode_plus</span><span class="p">(</span>
+                <span class="n">texts</span><span class="p">,</span>
+                <span class="o">**</span><span class="bp">self</span><span class="o">.</span><span class="n">encode_params</span><span class="p">,</span>
+            <span class="p">)</span>
+            <span class="n">input_ids</span> <span class="o">=</span> <span class="n">encoded_texts</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;input_ids&quot;</span><span class="p">)</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">is_fitted</span> <span class="o">=</span> <span class="kc">True</span>
+
+        <span class="k">try</span><span class="p">:</span>
+            <span class="n">output</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">(</span><span class="n">input_ids</span><span class="p">)</span>
+        <span class="k">except</span> <span class="ne">ValueError</span><span class="p">:</span>
+            <span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span>
+                <span class="s2">&quot;Padding and Truncating parameters were not passed and all input arrays &quot;</span>
+                <span class="s2">&quot;do not have the same shape. Padding to the longest sequence. &quot;</span>
+                <span class="s2">&quot;Padding will be done with the index of the pad token for the model&quot;</span><span class="p">,</span>
+                <span class="ne">UserWarning</span><span class="p">,</span>
+            <span class="p">)</span>
+            <span class="n">max_len</span> <span class="o">=</span> <span class="nb">max</span><span class="p">([</span><span class="nb">len</span><span class="p">(</span><span class="n">ids</span><span class="p">)</span> <span class="k">for</span> <span class="n">ids</span> <span class="ow">in</span> <span class="n">input_ids</span><span class="p">])</span>
+            <span class="n">output</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">(</span>
+                <span class="p">[</span>
+                    <span class="n">np</span><span class="o">.</span><span class="n">pad</span><span class="p">(</span><span class="n">ids</span><span class="p">,</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">tokenizer</span><span class="o">.</span><span class="n">pad_token_id</span><span class="p">,</span> <span class="n">max_len</span> <span class="o">-</span> <span class="nb">len</span><span class="p">(</span><span class="n">ids</span><span class="p">)))</span>
+                    <span class="k">for</span> <span class="n">ids</span> <span class="ow">in</span> <span class="n">input_ids</span>
+                <span class="p">]</span>
+            <span class="p">)</span>
+
+        <span class="k">return</span> <span class="n">output</span>
+
+    <span class="k">def</span> <span class="nf">decode</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span> <span class="n">input_ids</span><span class="p">:</span> <span class="n">npt</span><span class="o">.</span><span class="n">NDArray</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">int64</span><span class="p">],</span> <span class="n">skip_special_tokens</span><span class="p">:</span> <span class="nb">bool</span>
+    <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]:</span>
+<span class="w">        </span><span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Decodes a list of input_ids. The method is a wrapper around the</span>
+<span class="sd">        `convert_ids_to_tokens` and `convert_tokens_to_string` methods of the</span>
+<span class="sd">        HuggingFace&#39;s tokenizer.</span>
+
+<span class="sd">        Parameters</span>
+<span class="sd">        ----------</span>
+<span class="sd">        input_ids: npt.NDArray[np.int64]</span>
+<span class="sd">            The input_ids to be decoded</span>
+<span class="sd">        skip_special_tokens: bool</span>
+<span class="sd">            Whether to skip the special tokens or not</span>
+
+<span class="sd">        Returns</span>
+<span class="sd">        -------</span>
+<span class="sd">        List[str]</span>
+<span class="sd">            The decoded texts</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">texts</span> <span class="o">=</span> <span class="p">[</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">tokenizer</span><span class="o">.</span><span class="n">convert_tokens_to_string</span><span class="p">(</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">tokenizer</span><span class="o">.</span><span class="n">convert_ids_to_tokens</span><span class="p">(</span><span class="n">input_ids</span><span class="p">[</span><span class="n">i</span><span class="p">],</span> <span class="n">skip_special_tokens</span><span class="p">)</span>
+            <span class="p">)</span>
+            <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">input_ids</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">0</span><span class="p">])</span>
+        <span class="p">]</span>
+        <span class="k">return</span> <span class="n">texts</span>
+
+    <span class="k">def</span> <span class="nf">fit</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="s2">&quot;HFPreprocessor&quot;</span><span class="p">:</span>
+<span class="w">        </span><span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        This method is included for consistency with the rest of the library</span>
+<span class="sd">        in general and with the `BasePreprocessor` in particular. HuggingFace&#39;s</span>
+<span class="sd">        tokenizers and models are already trained. Therefore, the &#39;fit&#39; method</span>
+<span class="sd">        here does nothing other than checking that the &#39;text_col&#39; parameter is</span>
+<span class="sd">        not `None`.</span>
+
+<span class="sd">        Parameters</span>
+<span class="sd">        ----------</span>
+<span class="sd">        df: pd.DataFrame</span>
+<span class="sd">            The dataframe containing the text data in the column specified by</span>
+<span class="sd">            the &#39;text_col&#39; parameter</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">text_col</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
+                <span class="s2">&quot;&#39;text_col&#39; is None. Please specify the column name containing the text data&quot;</span>
+                <span class="s2">&quot; if you want to use the &#39;fit&#39; method&quot;</span>
+            <span class="p">)</span>
+        <span class="k">return</span> <span class="bp">self</span>
+
+    <span class="k">def</span> <span class="nf">transform</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">npt</span><span class="o">.</span><span class="n">NDArray</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">int64</span><span class="p">]:</span>
+<span class="w">        </span><span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Encodes the text data in the input dataframe. This method simply</span>
+<span class="sd">        calls the `encode` method under the hood. Similar to the `fit` method,</span>
+<span class="sd">        this method is included for consistency with the rest of the library</span>
+<span class="sd">        in general and with the `BasePreprocessor` in particular.</span>
+
+<span class="sd">        Parameters</span>
+<span class="sd">        ----------</span>
+<span class="sd">        df: pd.DataFrame</span>
+<span class="sd">            The dataframe containing the text data in the column specified by</span>
+<span class="sd">            the &#39;text_col&#39; parameter</span>
+
+<span class="sd">        Returns</span>
+<span class="sd">        -------</span>
+<span class="sd">        np.array</span>
+<span class="sd">            The encoded texts</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">text_col</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
+                <span class="s2">&quot;&#39;text_col&#39; is None. Please specify the column name containing the text data&quot;</span>
+                <span class="s2">&quot; if you want to use the &#39;fit&#39; method&quot;</span>
+            <span class="p">)</span>
+
+        <span class="n">texts</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_read_texts</span><span class="p">(</span><span class="n">df</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">root_dir</span><span class="p">)</span>
+
+        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">encode</span><span class="p">(</span><span class="n">texts</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">transform_sample</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">text</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">npt</span><span class="o">.</span><span class="n">NDArray</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">int64</span><span class="p">]:</span>
+<span class="w">        </span><span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Encodes a single text sample.</span>
+
+<span class="sd">        Parameters</span>
+<span class="sd">        ----------</span>
+<span class="sd">        text: str</span>
+<span class="sd">            The text sample to be encoded</span>
+
+<span class="sd">        Returns</span>
+<span class="sd">        -------</span>
+<span class="sd">        np.array</span>
+<span class="sd">            The encoded text</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+
+        <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">is_fitted</span><span class="p">:</span>
+            <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
+                <span class="s2">&quot;The `encode` (or `fit`) method must be called before calling `transform_sample`&quot;</span>
+            <span class="p">)</span>
+        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">encode</span><span class="p">([</span><span class="n">text</span><span class="p">])[</span><span class="mi">0</span><span class="p">]</span>
+
+    <span class="k">def</span> <span class="nf">fit_transform</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">npt</span><span class="o">.</span><span class="n">NDArray</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">int64</span><span class="p">]:</span>
+<span class="w">        </span><span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Encodes the text data in the input dataframe.</span>
+
+<span class="sd">        Parameters</span>
+<span class="sd">        ----------</span>
+<span class="sd">        df: pd.DataFrame</span>
+<span class="sd">            The dataframe containing the text data in the column specified by</span>
+<span class="sd">            the &#39;text_col&#39; parameter</span>
+
+<span class="sd">        Returns</span>
+<span class="sd">        -------</span>
+<span class="sd">        np.array</span>
+<span class="sd">            The encoded texts</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">df</span><span class="p">)</span><span class="o">.</span><span class="n">transform</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">inverse_transform</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span> <span class="n">input_ids</span><span class="p">:</span> <span class="n">npt</span><span class="o">.</span><span class="n">NDArray</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">int64</span><span class="p">],</span> <span class="n">skip_special_tokens</span><span class="p">:</span> <span class="nb">bool</span>
+    <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]:</span>
+<span class="w">        </span><span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Decodes a list of input_ids. The method simply calls the `decode` method</span>
+<span class="sd">        under the hood.</span>
+
+<span class="sd">        Parameters</span>
+<span class="sd">        ----------</span>
+<span class="sd">        input_ids: npt.NDArray[np.int64]</span>
+<span class="sd">            The input_ids to be decoded</span>
+<span class="sd">        skip_special_tokens: bool</span>
+<span class="sd">            Whether to skip the special tokens or not</span>
+
+<span class="sd">        Returns</span>
+<span class="sd">        -------</span>
+<span class="sd">        List[str]</span>
+<span class="sd">            The decoded texts</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">decode</span><span class="p">(</span><span class="n">input_ids</span><span class="p">,</span> <span class="n">skip_special_tokens</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">_process_text_parallel</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">texts</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">])</span> <span class="o">-&gt;</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]:</span>
+        <span class="n">num_processes</span> <span class="o">=</span> <span class="p">(</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">num_workers</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">num_workers</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="k">else</span> <span class="n">os</span><span class="o">.</span><span class="n">cpu_count</span><span class="p">()</span>
+        <span class="p">)</span>
+        <span class="k">with</span> <span class="n">ProcessPoolExecutor</span><span class="p">(</span><span class="n">max_workers</span><span class="o">=</span><span class="n">num_processes</span><span class="p">)</span> <span class="k">as</span> <span class="n">executor</span><span class="p">:</span>
+            <span class="n">processed_texts</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="n">executor</span><span class="o">.</span><span class="n">map</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_preprocess_text</span><span class="p">,</span> <span class="n">texts</span><span class="p">))</span>
+        <span class="k">return</span> <span class="n">processed_texts</span>
+
+    <span class="k">def</span> <span class="nf">_preprocess_text</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">text</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">str</span><span class="p">:</span>
+        <span class="k">for</span> <span class="n">rule</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">preprocessing_rules</span><span class="p">:</span>
+            <span class="n">text</span> <span class="o">=</span> <span class="n">rule</span><span class="p">(</span><span class="n">text</span><span class="p">)</span>
+        <span class="k">return</span> <span class="n">text</span>
+
+    <span class="k">def</span> <span class="nf">_encode_paralell</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">texts</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">],</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">List</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">int</span><span class="p">]]:</span>
+        <span class="n">num_processes</span> <span class="o">=</span> <span class="p">(</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">num_workers</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">num_workers</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="k">else</span> <span class="n">os</span><span class="o">.</span><span class="n">cpu_count</span><span class="p">()</span>
+        <span class="p">)</span>
+        <span class="n">ptokenizer</span> <span class="o">=</span> <span class="n">partial</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">tokenizer</span><span class="o">.</span><span class="n">encode</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+        <span class="k">with</span> <span class="n">ProcessPoolExecutor</span><span class="p">(</span><span class="n">max_workers</span><span class="o">=</span><span class="n">num_processes</span><span class="p">)</span> <span class="k">as</span> <span class="n">executor</span><span class="p">:</span>
+            <span class="n">encoded_texts</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="n">executor</span><span class="o">.</span><span class="n">map</span><span class="p">(</span><span class="n">ptokenizer</span><span class="p">,</span> <span class="n">texts</span><span class="p">))</span>
+        <span class="k">return</span> <span class="n">encoded_texts</span>
+
+    <span class="k">def</span> <span class="nf">_read_texts</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">,</span> <span class="n">root_dir</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span>
+    <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]:</span>
+        <span class="k">if</span> <span class="n">root_dir</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="k">if</span> <span class="ow">not</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">exists</span><span class="p">(</span><span class="n">root_dir</span><span class="p">):</span>
+                <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
+                    <span class="s2">&quot;root_dir does not exist. Please create it before fitting the preprocessor&quot;</span>
+                <span class="p">)</span>
+            <span class="n">texts_fnames</span> <span class="o">=</span> <span class="n">df</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">text_col</span><span class="p">]</span><span class="o">.</span><span class="n">tolist</span><span class="p">()</span>
+            <span class="n">texts</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
+            <span class="k">for</span> <span class="n">texts_fname</span> <span class="ow">in</span> <span class="n">texts_fnames</span><span class="p">:</span>
+                <span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">root_dir</span><span class="p">,</span> <span class="n">texts_fname</span><span class="p">),</span> <span class="s2">&quot;r&quot;</span><span class="p">)</span> <span class="k">as</span> <span class="n">f</span><span class="p">:</span>
+                    <span class="n">texts</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">read</span><span class="p">()</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s2">&quot;</span><span class="se">\n</span><span class="s2">&quot;</span><span class="p">,</span> <span class="s2">&quot;&quot;</span><span class="p">))</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="n">texts</span> <span class="o">=</span> <span class="n">df</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">text_col</span><span class="p">]</span><span class="o">.</span><span class="n">tolist</span><span class="p">()</span>
+
+        <span class="k">return</span> <span class="n">texts</span>
+
+    <span class="k">def</span> <span class="fm">__repr__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="k">return</span> <span class="p">(</span>
+            <span class="sa">f</span><span class="s2">&quot;HFPreprocessor(text_col=</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">text_col</span><span class="si">}</span><span class="s2">, model_name=</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">model_name</span><span class="si">}</span><span class="s2">, &quot;</span>
+            <span class="sa">f</span><span class="s2">&quot;use_fast_tokenizer=</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">use_fast_tokenizer</span><span class="si">}</span><span class="s2">, num_workers=</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">num_workers</span><span class="si">}</span><span class="s2">, &quot;</span>
+            <span class="sa">f</span><span class="s2">&quot;preprocessing_rules=</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">preprocessing_rules</span><span class="si">}</span><span class="s2">, tokenizer_params=</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">tokenizer_params</span><span class="si">}</span><span class="s2">, &quot;</span>
+            <span class="sa">f</span><span class="s2">&quot;encode_params=</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">encode_params</span><span class="si">}</span><span class="s2">)&quot;</span>
+        <span class="p">)</span>
+</code></pre></div></td></tr></table></div>
+              </details>
+
+
+
+  <div class="doc doc-children">
+
+
+
+
+
+
+
+
+
+<div class="doc doc-object doc-function">
+
+
+<h3 id="pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.encode" class="doc doc-heading">
+            <span class="doc doc-object-name doc-function-name">encode</span>
+
+
+<a href="#pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.encode" class="headerlink" title="Permanent link">&para;</a></h3>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">encode</span><span class="p">(</span><span class="n">texts</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+</code></pre></div>
+
+    <div class="doc doc-contents ">
+
+        <p>Encodes a list of texts. The method is a wrapper around the
+<code>batch_encode_plus</code> method of the HuggingFace's tokenizer.</p>
+<p>if 'use_fast_tokenizer' is True, the method will use the <code>batch_encode_plus</code></p>
+
+
+<p><span class="doc-section-title">Parameters:</span></p>
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+          <th>Default</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>texts</code></td>
+            <td>
+                  <code><span title="typing.List">List</span>[str]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>List of texts to be encoded</p>
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>**kwargs</code></td>
+            <td>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Additional parameters to be passed to the <code>batch_encode_plus</code> method
+of the HuggingFace's tokenizer. If the 'encode_params' dict was passed
+when instantiating the class, that dictionaly will be updated with
+the kwargs passed here.</p>
+              </div>
+            </td>
+            <td>
+                  <code>{}</code>
+            </td>
+          </tr>
+      </tbody>
+    </table>
+
+
+    <p><span class="doc-section-title">Returns:</span></p>
+    <table>
+      <thead>
+        <tr>
+          <th>Type</th>
+          <th>Description</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td>
+                  <code><span title="numpy.array">array</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>The encoded texts</p>
+              </div>
+            </td>
+          </tr>
+      </tbody>
+    </table>
+
+            <details class="quote">
+              <summary>Source code in <code>pytorch_widedeep/preprocessing/hf_preprocessor.py</code></summary>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">123</span>
+<span class="normal">124</span>
+<span class="normal">125</span>
+<span class="normal">126</span>
+<span class="normal">127</span>
+<span class="normal">128</span>
+<span class="normal">129</span>
+<span class="normal">130</span>
+<span class="normal">131</span>
+<span class="normal">132</span>
+<span class="normal">133</span>
+<span class="normal">134</span>
+<span class="normal">135</span>
+<span class="normal">136</span>
+<span class="normal">137</span>
+<span class="normal">138</span>
+<span class="normal">139</span>
+<span class="normal">140</span>
+<span class="normal">141</span>
+<span class="normal">142</span>
+<span class="normal">143</span>
+<span class="normal">144</span>
+<span class="normal">145</span>
+<span class="normal">146</span>
+<span class="normal">147</span>
+<span class="normal">148</span>
+<span class="normal">149</span>
+<span class="normal">150</span>
+<span class="normal">151</span>
+<span class="normal">152</span>
+<span class="normal">153</span>
+<span class="normal">154</span>
+<span class="normal">155</span>
+<span class="normal">156</span>
+<span class="normal">157</span>
+<span class="normal">158</span>
+<span class="normal">159</span>
+<span class="normal">160</span>
+<span class="normal">161</span>
+<span class="normal">162</span>
+<span class="normal">163</span>
+<span class="normal">164</span>
+<span class="normal">165</span>
+<span class="normal">166</span>
+<span class="normal">167</span>
+<span class="normal">168</span>
+<span class="normal">169</span>
+<span class="normal">170</span>
+<span class="normal">171</span>
+<span class="normal">172</span>
+<span class="normal">173</span>
+<span class="normal">174</span>
+<span class="normal">175</span>
+<span class="normal">176</span>
+<span class="normal">177</span>
+<span class="normal">178</span>
+<span class="normal">179</span>
+<span class="normal">180</span>
+<span class="normal">181</span>
+<span class="normal">182</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">encode</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">texts</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">],</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">npt</span><span class="o">.</span><span class="n">NDArray</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">int64</span><span class="p">]:</span>
+<span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">    Encodes a list of texts. The method is a wrapper around the</span>
+<span class="sd">    `batch_encode_plus` method of the HuggingFace&#39;s tokenizer.</span>
+
+<span class="sd">    if &#39;use_fast_tokenizer&#39; is True, the method will use the `batch_encode_plus`</span>
+
+<span class="sd">    Parameters</span>
+<span class="sd">    ----------</span>
+<span class="sd">    texts: List[str]</span>
+<span class="sd">        List of texts to be encoded</span>
+<span class="sd">    **kwargs</span>
+<span class="sd">        Additional parameters to be passed to the `batch_encode_plus` method</span>
+<span class="sd">        of the HuggingFace&#39;s tokenizer. If the &#39;encode_params&#39; dict was passed</span>
+<span class="sd">        when instantiating the class, that dictionaly will be updated with</span>
+<span class="sd">        the kwargs passed here.</span>
+
+<span class="sd">    Returns</span>
+<span class="sd">    -------</span>
+<span class="sd">    np.array</span>
+<span class="sd">        The encoded texts</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="k">if</span> <span class="n">kwargs</span><span class="p">:</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">encode_params</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="n">kwargs</span><span class="p">)</span>
+
+    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">preprocessing_rules</span><span class="p">:</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_multiprocessing</span><span class="p">:</span>
+            <span class="n">texts</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_process_text_parallel</span><span class="p">(</span><span class="n">texts</span><span class="p">)</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="n">texts</span> <span class="o">=</span> <span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">_preprocess_text</span><span class="p">(</span><span class="n">text</span><span class="p">)</span> <span class="k">for</span> <span class="n">text</span> <span class="ow">in</span> <span class="n">texts</span><span class="p">]</span>
+
+    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_multiprocessing</span><span class="p">:</span>
+        <span class="n">input_ids</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_encode_paralell</span><span class="p">(</span><span class="n">texts</span><span class="p">,</span> <span class="o">**</span><span class="bp">self</span><span class="o">.</span><span class="n">encode_params</span><span class="p">)</span>
+    <span class="k">else</span><span class="p">:</span>
+        <span class="n">encoded_texts</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">tokenizer</span><span class="o">.</span><span class="n">batch_encode_plus</span><span class="p">(</span>
+            <span class="n">texts</span><span class="p">,</span>
+            <span class="o">**</span><span class="bp">self</span><span class="o">.</span><span class="n">encode_params</span><span class="p">,</span>
+        <span class="p">)</span>
+        <span class="n">input_ids</span> <span class="o">=</span> <span class="n">encoded_texts</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;input_ids&quot;</span><span class="p">)</span>
+
+    <span class="bp">self</span><span class="o">.</span><span class="n">is_fitted</span> <span class="o">=</span> <span class="kc">True</span>
+
+    <span class="k">try</span><span class="p">:</span>
+        <span class="n">output</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">(</span><span class="n">input_ids</span><span class="p">)</span>
+    <span class="k">except</span> <span class="ne">ValueError</span><span class="p">:</span>
+        <span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span>
+            <span class="s2">&quot;Padding and Truncating parameters were not passed and all input arrays &quot;</span>
+            <span class="s2">&quot;do not have the same shape. Padding to the longest sequence. &quot;</span>
+            <span class="s2">&quot;Padding will be done with the index of the pad token for the model&quot;</span><span class="p">,</span>
+            <span class="ne">UserWarning</span><span class="p">,</span>
+        <span class="p">)</span>
+        <span class="n">max_len</span> <span class="o">=</span> <span class="nb">max</span><span class="p">([</span><span class="nb">len</span><span class="p">(</span><span class="n">ids</span><span class="p">)</span> <span class="k">for</span> <span class="n">ids</span> <span class="ow">in</span> <span class="n">input_ids</span><span class="p">])</span>
+        <span class="n">output</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">(</span>
+            <span class="p">[</span>
+                <span class="n">np</span><span class="o">.</span><span class="n">pad</span><span class="p">(</span><span class="n">ids</span><span class="p">,</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">tokenizer</span><span class="o">.</span><span class="n">pad_token_id</span><span class="p">,</span> <span class="n">max_len</span> <span class="o">-</span> <span class="nb">len</span><span class="p">(</span><span class="n">ids</span><span class="p">)))</span>
+                <span class="k">for</span> <span class="n">ids</span> <span class="ow">in</span> <span class="n">input_ids</span>
+            <span class="p">]</span>
+        <span class="p">)</span>
+
+    <span class="k">return</span> <span class="n">output</span>
+</code></pre></div></td></tr></table></div>
+            </details>
+    </div>
+
+</div>
+
+<div class="doc doc-object doc-function">
+
+
+<h3 id="pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.decode" class="doc doc-heading">
+            <span class="doc doc-object-name doc-function-name">decode</span>
+
+
+<a href="#pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.decode" class="headerlink" title="Permanent link">&para;</a></h3>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">decode</span><span class="p">(</span><span class="n">input_ids</span><span class="p">,</span> <span class="n">skip_special_tokens</span><span class="p">)</span>
+</code></pre></div>
+
+    <div class="doc doc-contents ">
+
+        <p>Decodes a list of input_ids. The method is a wrapper around the
+<code>convert_ids_to_tokens</code> and <code>convert_tokens_to_string</code> methods of the
+HuggingFace's tokenizer.</p>
+
+
+<p><span class="doc-section-title">Parameters:</span></p>
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+          <th>Default</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>input_ids</code></td>
+            <td>
+                  <code><span title="numpy.typing.NDArray">NDArray</span>[<span title="numpy.int64">int64</span>]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>The input_ids to be decoded</p>
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>skip_special_tokens</code></td>
+            <td>
+                  <code>bool</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Whether to skip the special tokens or not</p>
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+      </tbody>
+    </table>
+
+
+    <p><span class="doc-section-title">Returns:</span></p>
+    <table>
+      <thead>
+        <tr>
+          <th>Type</th>
+          <th>Description</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td>
+                  <code><span title="typing.List">List</span>[str]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>The decoded texts</p>
+              </div>
+            </td>
+          </tr>
+      </tbody>
+    </table>
+
+            <details class="quote">
+              <summary>Source code in <code>pytorch_widedeep/preprocessing/hf_preprocessor.py</code></summary>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">184</span>
+<span class="normal">185</span>
+<span class="normal">186</span>
+<span class="normal">187</span>
+<span class="normal">188</span>
+<span class="normal">189</span>
+<span class="normal">190</span>
+<span class="normal">191</span>
+<span class="normal">192</span>
+<span class="normal">193</span>
+<span class="normal">194</span>
+<span class="normal">195</span>
+<span class="normal">196</span>
+<span class="normal">197</span>
+<span class="normal">198</span>
+<span class="normal">199</span>
+<span class="normal">200</span>
+<span class="normal">201</span>
+<span class="normal">202</span>
+<span class="normal">203</span>
+<span class="normal">204</span>
+<span class="normal">205</span>
+<span class="normal">206</span>
+<span class="normal">207</span>
+<span class="normal">208</span>
+<span class="normal">209</span>
+<span class="normal">210</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">decode</span><span class="p">(</span>
+    <span class="bp">self</span><span class="p">,</span> <span class="n">input_ids</span><span class="p">:</span> <span class="n">npt</span><span class="o">.</span><span class="n">NDArray</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">int64</span><span class="p">],</span> <span class="n">skip_special_tokens</span><span class="p">:</span> <span class="nb">bool</span>
+<span class="p">)</span> <span class="o">-&gt;</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]:</span>
+<span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">    Decodes a list of input_ids. The method is a wrapper around the</span>
+<span class="sd">    `convert_ids_to_tokens` and `convert_tokens_to_string` methods of the</span>
+<span class="sd">    HuggingFace&#39;s tokenizer.</span>
+
+<span class="sd">    Parameters</span>
+<span class="sd">    ----------</span>
+<span class="sd">    input_ids: npt.NDArray[np.int64]</span>
+<span class="sd">        The input_ids to be decoded</span>
+<span class="sd">    skip_special_tokens: bool</span>
+<span class="sd">        Whether to skip the special tokens or not</span>
+
+<span class="sd">    Returns</span>
+<span class="sd">    -------</span>
+<span class="sd">    List[str]</span>
+<span class="sd">        The decoded texts</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="n">texts</span> <span class="o">=</span> <span class="p">[</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">tokenizer</span><span class="o">.</span><span class="n">convert_tokens_to_string</span><span class="p">(</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">tokenizer</span><span class="o">.</span><span class="n">convert_ids_to_tokens</span><span class="p">(</span><span class="n">input_ids</span><span class="p">[</span><span class="n">i</span><span class="p">],</span> <span class="n">skip_special_tokens</span><span class="p">)</span>
+        <span class="p">)</span>
+        <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">input_ids</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">0</span><span class="p">])</span>
+    <span class="p">]</span>
+    <span class="k">return</span> <span class="n">texts</span>
+</code></pre></div></td></tr></table></div>
+            </details>
+    </div>
+
+</div>
+
+<div class="doc doc-object doc-function">
+
+
+<h3 id="pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.fit" class="doc doc-heading">
+            <span class="doc doc-object-name doc-function-name">fit</span>
+
+
+<a href="#pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.fit" class="headerlink" title="Permanent link">&para;</a></h3>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">fit</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
+</code></pre></div>
+
+    <div class="doc doc-contents ">
+
+        <p>This method is included for consistency with the rest of the library
+in general and with the <code>BasePreprocessor</code> in particular. HuggingFace's
+tokenizers and models are already trained. Therefore, the 'fit' method
+here does nothing other than checking that the 'text_col' parameter is
+not <code>None</code>.</p>
+
+
+<p><span class="doc-section-title">Parameters:</span></p>
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+          <th>Default</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>df</code></td>
+            <td>
+                  <code><span title="pandas.DataFrame">DataFrame</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>The dataframe containing the text data in the column specified by
+the 'text_col' parameter</p>
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+      </tbody>
+    </table>
+
+            <details class="quote">
+              <summary>Source code in <code>pytorch_widedeep/preprocessing/hf_preprocessor.py</code></summary>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">212</span>
+<span class="normal">213</span>
+<span class="normal">214</span>
+<span class="normal">215</span>
+<span class="normal">216</span>
+<span class="normal">217</span>
+<span class="normal">218</span>
+<span class="normal">219</span>
+<span class="normal">220</span>
+<span class="normal">221</span>
+<span class="normal">222</span>
+<span class="normal">223</span>
+<span class="normal">224</span>
+<span class="normal">225</span>
+<span class="normal">226</span>
+<span class="normal">227</span>
+<span class="normal">228</span>
+<span class="normal">229</span>
+<span class="normal">230</span>
+<span class="normal">231</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">fit</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="s2">&quot;HFPreprocessor&quot;</span><span class="p">:</span>
+<span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">    This method is included for consistency with the rest of the library</span>
+<span class="sd">    in general and with the `BasePreprocessor` in particular. HuggingFace&#39;s</span>
+<span class="sd">    tokenizers and models are already trained. Therefore, the &#39;fit&#39; method</span>
+<span class="sd">    here does nothing other than checking that the &#39;text_col&#39; parameter is</span>
+<span class="sd">    not `None`.</span>
+
+<span class="sd">    Parameters</span>
+<span class="sd">    ----------</span>
+<span class="sd">    df: pd.DataFrame</span>
+<span class="sd">        The dataframe containing the text data in the column specified by</span>
+<span class="sd">        the &#39;text_col&#39; parameter</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">text_col</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+        <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
+            <span class="s2">&quot;&#39;text_col&#39; is None. Please specify the column name containing the text data&quot;</span>
+            <span class="s2">&quot; if you want to use the &#39;fit&#39; method&quot;</span>
+        <span class="p">)</span>
+    <span class="k">return</span> <span class="bp">self</span>
+</code></pre></div></td></tr></table></div>
+            </details>
+    </div>
+
+</div>
+
+<div class="doc doc-object doc-function">
+
+
+<h3 id="pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.transform" class="doc doc-heading">
+            <span class="doc doc-object-name doc-function-name">transform</span>
+
+
+<a href="#pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.transform" class="headerlink" title="Permanent link">&para;</a></h3>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">transform</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
+</code></pre></div>
+
+    <div class="doc doc-contents ">
+
+        <p>Encodes the text data in the input dataframe. This method simply
+calls the <code>encode</code> method under the hood. Similar to the <code>fit</code> method,
+this method is included for consistency with the rest of the library
+in general and with the <code>BasePreprocessor</code> in particular.</p>
+
+
+<p><span class="doc-section-title">Parameters:</span></p>
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+          <th>Default</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>df</code></td>
+            <td>
+                  <code><span title="pandas.DataFrame">DataFrame</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>The dataframe containing the text data in the column specified by
+the 'text_col' parameter</p>
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+      </tbody>
+    </table>
+
+
+    <p><span class="doc-section-title">Returns:</span></p>
+    <table>
+      <thead>
+        <tr>
+          <th>Type</th>
+          <th>Description</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td>
+                  <code><span title="numpy.array">array</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>The encoded texts</p>
+              </div>
+            </td>
+          </tr>
+      </tbody>
+    </table>
+
+            <details class="quote">
+              <summary>Source code in <code>pytorch_widedeep/preprocessing/hf_preprocessor.py</code></summary>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">233</span>
+<span class="normal">234</span>
+<span class="normal">235</span>
+<span class="normal">236</span>
+<span class="normal">237</span>
+<span class="normal">238</span>
+<span class="normal">239</span>
+<span class="normal">240</span>
+<span class="normal">241</span>
+<span class="normal">242</span>
+<span class="normal">243</span>
+<span class="normal">244</span>
+<span class="normal">245</span>
+<span class="normal">246</span>
+<span class="normal">247</span>
+<span class="normal">248</span>
+<span class="normal">249</span>
+<span class="normal">250</span>
+<span class="normal">251</span>
+<span class="normal">252</span>
+<span class="normal">253</span>
+<span class="normal">254</span>
+<span class="normal">255</span>
+<span class="normal">256</span>
+<span class="normal">257</span>
+<span class="normal">258</span>
+<span class="normal">259</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">transform</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">npt</span><span class="o">.</span><span class="n">NDArray</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">int64</span><span class="p">]:</span>
+<span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">    Encodes the text data in the input dataframe. This method simply</span>
+<span class="sd">    calls the `encode` method under the hood. Similar to the `fit` method,</span>
+<span class="sd">    this method is included for consistency with the rest of the library</span>
+<span class="sd">    in general and with the `BasePreprocessor` in particular.</span>
+
+<span class="sd">    Parameters</span>
+<span class="sd">    ----------</span>
+<span class="sd">    df: pd.DataFrame</span>
+<span class="sd">        The dataframe containing the text data in the column specified by</span>
+<span class="sd">        the &#39;text_col&#39; parameter</span>
+
+<span class="sd">    Returns</span>
+<span class="sd">    -------</span>
+<span class="sd">    np.array</span>
+<span class="sd">        The encoded texts</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">text_col</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+        <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
+            <span class="s2">&quot;&#39;text_col&#39; is None. Please specify the column name containing the text data&quot;</span>
+            <span class="s2">&quot; if you want to use the &#39;fit&#39; method&quot;</span>
+        <span class="p">)</span>
+
+    <span class="n">texts</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_read_texts</span><span class="p">(</span><span class="n">df</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">root_dir</span><span class="p">)</span>
+
+    <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">encode</span><span class="p">(</span><span class="n">texts</span><span class="p">)</span>
+</code></pre></div></td></tr></table></div>
+            </details>
+    </div>
+
+</div>
+
+<div class="doc doc-object doc-function">
+
+
+<h3 id="pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.transform_sample" class="doc doc-heading">
+            <span class="doc doc-object-name doc-function-name">transform_sample</span>
+
+
+<a href="#pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.transform_sample" class="headerlink" title="Permanent link">&para;</a></h3>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">transform_sample</span><span class="p">(</span><span class="n">text</span><span class="p">)</span>
+</code></pre></div>
+
+    <div class="doc doc-contents ">
+
+        <p>Encodes a single text sample.</p>
+
+
+<p><span class="doc-section-title">Parameters:</span></p>
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+          <th>Default</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>text</code></td>
+            <td>
+                  <code>str</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>The text sample to be encoded</p>
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+      </tbody>
+    </table>
+
+
+    <p><span class="doc-section-title">Returns:</span></p>
+    <table>
+      <thead>
+        <tr>
+          <th>Type</th>
+          <th>Description</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td>
+                  <code><span title="numpy.array">array</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>The encoded text</p>
+              </div>
+            </td>
+          </tr>
+      </tbody>
+    </table>
+
+            <details class="quote">
+              <summary>Source code in <code>pytorch_widedeep/preprocessing/hf_preprocessor.py</code></summary>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">261</span>
+<span class="normal">262</span>
+<span class="normal">263</span>
+<span class="normal">264</span>
+<span class="normal">265</span>
+<span class="normal">266</span>
+<span class="normal">267</span>
+<span class="normal">268</span>
+<span class="normal">269</span>
+<span class="normal">270</span>
+<span class="normal">271</span>
+<span class="normal">272</span>
+<span class="normal">273</span>
+<span class="normal">274</span>
+<span class="normal">275</span>
+<span class="normal">276</span>
+<span class="normal">277</span>
+<span class="normal">278</span>
+<span class="normal">279</span>
+<span class="normal">280</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">transform_sample</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">text</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">npt</span><span class="o">.</span><span class="n">NDArray</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">int64</span><span class="p">]:</span>
+<span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">    Encodes a single text sample.</span>
+
+<span class="sd">    Parameters</span>
+<span class="sd">    ----------</span>
+<span class="sd">    text: str</span>
+<span class="sd">        The text sample to be encoded</span>
+
+<span class="sd">    Returns</span>
+<span class="sd">    -------</span>
+<span class="sd">    np.array</span>
+<span class="sd">        The encoded text</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+
+    <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">is_fitted</span><span class="p">:</span>
+        <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
+            <span class="s2">&quot;The `encode` (or `fit`) method must be called before calling `transform_sample`&quot;</span>
+        <span class="p">)</span>
+    <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">encode</span><span class="p">([</span><span class="n">text</span><span class="p">])[</span><span class="mi">0</span><span class="p">]</span>
+</code></pre></div></td></tr></table></div>
+            </details>
+    </div>
+
+</div>
+
+<div class="doc doc-object doc-function">
+
+
+<h3 id="pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.fit_transform" class="doc doc-heading">
+            <span class="doc doc-object-name doc-function-name">fit_transform</span>
+
+
+<a href="#pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.fit_transform" class="headerlink" title="Permanent link">&para;</a></h3>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">fit_transform</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
+</code></pre></div>
+
+    <div class="doc doc-contents ">
+
+        <p>Encodes the text data in the input dataframe.</p>
+
+
+<p><span class="doc-section-title">Parameters:</span></p>
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+          <th>Default</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>df</code></td>
+            <td>
+                  <code><span title="pandas.DataFrame">DataFrame</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>The dataframe containing the text data in the column specified by
+the 'text_col' parameter</p>
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+      </tbody>
+    </table>
+
+
+    <p><span class="doc-section-title">Returns:</span></p>
+    <table>
+      <thead>
+        <tr>
+          <th>Type</th>
+          <th>Description</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td>
+                  <code><span title="numpy.array">array</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>The encoded texts</p>
+              </div>
+            </td>
+          </tr>
+      </tbody>
+    </table>
+
+            <details class="quote">
+              <summary>Source code in <code>pytorch_widedeep/preprocessing/hf_preprocessor.py</code></summary>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">282</span>
+<span class="normal">283</span>
+<span class="normal">284</span>
+<span class="normal">285</span>
+<span class="normal">286</span>
+<span class="normal">287</span>
+<span class="normal">288</span>
+<span class="normal">289</span>
+<span class="normal">290</span>
+<span class="normal">291</span>
+<span class="normal">292</span>
+<span class="normal">293</span>
+<span class="normal">294</span>
+<span class="normal">295</span>
+<span class="normal">296</span>
+<span class="normal">297</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">fit_transform</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">npt</span><span class="o">.</span><span class="n">NDArray</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">int64</span><span class="p">]:</span>
+<span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">    Encodes the text data in the input dataframe.</span>
+
+<span class="sd">    Parameters</span>
+<span class="sd">    ----------</span>
+<span class="sd">    df: pd.DataFrame</span>
+<span class="sd">        The dataframe containing the text data in the column specified by</span>
+<span class="sd">        the &#39;text_col&#39; parameter</span>
+
+<span class="sd">    Returns</span>
+<span class="sd">    -------</span>
+<span class="sd">    np.array</span>
+<span class="sd">        The encoded texts</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">df</span><span class="p">)</span><span class="o">.</span><span class="n">transform</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
+</code></pre></div></td></tr></table></div>
+            </details>
+    </div>
+
+</div>
+
+<div class="doc doc-object doc-function">
+
+
+<h3 id="pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.inverse_transform" class="doc doc-heading">
+            <span class="doc doc-object-name doc-function-name">inverse_transform</span>
+
+
+<a href="#pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.inverse_transform" class="headerlink" title="Permanent link">&para;</a></h3>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">inverse_transform</span><span class="p">(</span><span class="n">input_ids</span><span class="p">,</span> <span class="n">skip_special_tokens</span><span class="p">)</span>
+</code></pre></div>
+
+    <div class="doc doc-contents ">
+
+        <p>Decodes a list of input_ids. The method simply calls the <code>decode</code> method
+under the hood.</p>
+
+
+<p><span class="doc-section-title">Parameters:</span></p>
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+          <th>Default</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>input_ids</code></td>
+            <td>
+                  <code><span title="numpy.typing.NDArray">NDArray</span>[<span title="numpy.int64">int64</span>]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>The input_ids to be decoded</p>
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>skip_special_tokens</code></td>
+            <td>
+                  <code>bool</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Whether to skip the special tokens or not</p>
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+      </tbody>
+    </table>
+
+
+    <p><span class="doc-section-title">Returns:</span></p>
+    <table>
+      <thead>
+        <tr>
+          <th>Type</th>
+          <th>Description</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td>
+                  <code><span title="typing.List">List</span>[str]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>The decoded texts</p>
+              </div>
+            </td>
+          </tr>
+      </tbody>
+    </table>
+
+            <details class="quote">
+              <summary>Source code in <code>pytorch_widedeep/preprocessing/hf_preprocessor.py</code></summary>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">299</span>
+<span class="normal">300</span>
+<span class="normal">301</span>
+<span class="normal">302</span>
+<span class="normal">303</span>
+<span class="normal">304</span>
+<span class="normal">305</span>
+<span class="normal">306</span>
+<span class="normal">307</span>
+<span class="normal">308</span>
+<span class="normal">309</span>
+<span class="normal">310</span>
+<span class="normal">311</span>
+<span class="normal">312</span>
+<span class="normal">313</span>
+<span class="normal">314</span>
+<span class="normal">315</span>
+<span class="normal">316</span>
+<span class="normal">317</span>
+<span class="normal">318</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">inverse_transform</span><span class="p">(</span>
+    <span class="bp">self</span><span class="p">,</span> <span class="n">input_ids</span><span class="p">:</span> <span class="n">npt</span><span class="o">.</span><span class="n">NDArray</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">int64</span><span class="p">],</span> <span class="n">skip_special_tokens</span><span class="p">:</span> <span class="nb">bool</span>
+<span class="p">)</span> <span class="o">-&gt;</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]:</span>
+<span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">    Decodes a list of input_ids. The method simply calls the `decode` method</span>
+<span class="sd">    under the hood.</span>
+
+<span class="sd">    Parameters</span>
+<span class="sd">    ----------</span>
+<span class="sd">    input_ids: npt.NDArray[np.int64]</span>
+<span class="sd">        The input_ids to be decoded</span>
+<span class="sd">    skip_special_tokens: bool</span>
+<span class="sd">        Whether to skip the special tokens or not</span>
+
+<span class="sd">    Returns</span>
+<span class="sd">    -------</span>
+<span class="sd">    List[str]</span>
+<span class="sd">        The decoded texts</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">decode</span><span class="p">(</span><span class="n">input_ids</span><span class="p">,</span> <span class="n">skip_special_tokens</span><span class="p">)</span>
+</code></pre></div></td></tr></table></div>
+            </details>
+    </div>
+
+</div>
+
+
+
+  </div>
+
+    </div>
+
+</div>
+
+<div class="doc doc-object doc-class">
+
+
+
+<h2 id="pytorch_widedeep.preprocessing.image_preprocessor.ImagePreprocessor" class="doc doc-heading">
+            <span class="doc doc-object-name doc-class-name">ImagePreprocessor</span>
+
+
+<a href="#pytorch_widedeep.preprocessing.image_preprocessor.ImagePreprocessor" class="headerlink" title="Permanent link">&para;</a></h2>
+
+
+    <div class="doc doc-contents first">
+            <p class="doc doc-class-bases">
+              Bases: <code><span title="pytorch_widedeep.preprocessing.base_preprocessor.BasePreprocessor">BasePreprocessor</span></code></p>
+
+
+        <p>Preprocessor to prepare the <code>deepimage</code> input dataset.</p>
+<p>The Preprocessing consists simply on resizing according to their
+aspect ratio</p>
+
+
+<p><span class="doc-section-title">Parameters:</span></p>
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+          <th>Default</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>img_col</code></td>
+            <td>
+                  <code>str</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>name of the column with the images filenames</p>
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>img_path</code></td>
+            <td>
+                  <code>str</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>path to the dicrectory where the images are stored</p>
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>width</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>width of the resulting processed image.</p>
+              </div>
+            </td>
+            <td>
+                  <code>224</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>height</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>width of the resulting processed image.</p>
+              </div>
+            </td>
+            <td>
+                  <code>224</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>verbose</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Enable verbose output.</p>
+              </div>
+            </td>
+            <td>
+                  <code>1</code>
+            </td>
+          </tr>
+      </tbody>
+    </table>
+
+
+<p><span class="doc-section-title">Attributes:</span></p>
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code><span title="pytorch_widedeep.preprocessing.image_preprocessor.ImagePreprocessor.aap">aap</span></code></td>
+            <td>
+                  <code><a class="autorefs autorefs-internal" title="pytorch_widedeep.utils.image_utils.AspectAwarePreprocessor" href="utils/image_utils.html#pytorch_widedeep.utils.image_utils.AspectAwarePreprocessor">AspectAwarePreprocessor</a></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>an instance of <code>pytorch_widedeep.utils.image_utils.AspectAwarePreprocessor</code></p>
+              </div>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code><span title="pytorch_widedeep.preprocessing.image_preprocessor.ImagePreprocessor.spp">spp</span></code></td>
+            <td>
+                  <code><a class="autorefs autorefs-internal" title="pytorch_widedeep.utils.image_utils.SimplePreprocessor" href="utils/image_utils.html#pytorch_widedeep.utils.image_utils.SimplePreprocessor">SimplePreprocessor</a></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>an instance of <code>pytorch_widedeep.utils.image_utils.SimplePreprocessor</code></p>
+              </div>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code><span title="pytorch_widedeep.preprocessing.image_preprocessor.ImagePreprocessor.normalise_metrics">normalise_metrics</span></code></td>
+            <td>
+                  <code>Dict</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Dict containing the normalisation metrics of the image dataset, i.e.
+mean and std for the R, G and B channels</p>
+              </div>
+            </td>
+          </tr>
+      </tbody>
+    </table>
+
+
+<p><span class="doc-section-title">Examples:</span></p>
+    <div class="highlight"><pre><span></span><code><span class="gp">&gt;&gt;&gt; </span><span class="kn">import</span> <span class="nn">pandas</span> <span class="k">as</span> <span class="nn">pd</span>
+<span class="gp">&gt;&gt;&gt;</span>
+<span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">pytorch_widedeep.preprocessing</span> <span class="kn">import</span> <span class="n">ImagePreprocessor</span>
+<span class="gp">&gt;&gt;&gt;</span>
+<span class="gp">&gt;&gt;&gt; </span><span class="n">path_to_image1</span> <span class="o">=</span> <span class="s1">&#39;tests/test_data_utils/images/galaxy1.png&#39;</span>
+<span class="gp">&gt;&gt;&gt; </span><span class="n">path_to_image2</span> <span class="o">=</span> <span class="s1">&#39;tests/test_data_utils/images/galaxy2.png&#39;</span>
+<span class="gp">&gt;&gt;&gt;</span>
+<span class="gp">&gt;&gt;&gt; </span><span class="n">df_train</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">({</span><span class="s1">&#39;images_column&#39;</span><span class="p">:</span> <span class="p">[</span><span class="n">path_to_image1</span><span class="p">]})</span>
+<span class="gp">&gt;&gt;&gt; </span><span class="n">df_test</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">({</span><span class="s1">&#39;images_column&#39;</span><span class="p">:</span> <span class="p">[</span><span class="n">path_to_image2</span><span class="p">]})</span>
+<span class="gp">&gt;&gt;&gt; </span><span class="n">img_preprocessor</span> <span class="o">=</span> <span class="n">ImagePreprocessor</span><span class="p">(</span><span class="n">img_col</span><span class="o">=</span><span class="s1">&#39;images_column&#39;</span><span class="p">,</span> <span class="n">img_path</span><span class="o">=</span><span class="s1">&#39;.&#39;</span><span class="p">,</span> <span class="n">verbose</span><span class="o">=</span><span class="mi">0</span><span class="p">)</span>
+<span class="gp">&gt;&gt;&gt; </span><span class="n">resized_images</span> <span class="o">=</span> <span class="n">img_preprocessor</span><span class="o">.</span><span class="n">fit_transform</span><span class="p">(</span><span class="n">df_train</span><span class="p">)</span>
+<span class="gp">&gt;&gt;&gt; </span><span class="n">new_resized_images</span> <span class="o">=</span> <span class="n">img_preprocessor</span><span class="o">.</span><span class="n">transform</span><span class="p">(</span><span class="n">df_train</span><span class="p">)</span>
+</code></pre></div>
+    <p><img alt="ℹ️" class="emojione" src="https://cdnjs.cloudflare.com/ajax/libs/emojione/2.2.7/assets/png/2139.png" title=":information_source:" /> <strong>NOTE</strong>:
+Normalising metrics will only be computed when the <code>fit_transform</code>
+method is run. Running <code>transform</code> only will not change the computed
+metrics and running <code>fit</code> only simply instantiates the resizing
+functions.</p>
+
+              <details class="quote">
+                <summary>Source code in <code>pytorch_widedeep/preprocessing/image_preprocessor.py</code></summary>
+                <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 15</span>
+<span class="normal"> 16</span>
+<span class="normal"> 17</span>
+<span class="normal"> 18</span>
+<span class="normal"> 19</span>
+<span class="normal"> 20</span>
+<span class="normal"> 21</span>
+<span class="normal"> 22</span>
+<span class="normal"> 23</span>
+<span class="normal"> 24</span>
+<span class="normal"> 25</span>
+<span class="normal"> 26</span>
+<span class="normal"> 27</span>
+<span class="normal"> 28</span>
+<span class="normal"> 29</span>
+<span class="normal"> 30</span>
+<span class="normal"> 31</span>
+<span class="normal"> 32</span>
+<span class="normal"> 33</span>
+<span class="normal"> 34</span>
+<span class="normal"> 35</span>
+<span class="normal"> 36</span>
+<span class="normal"> 37</span>
+<span class="normal"> 38</span>
+<span class="normal"> 39</span>
+<span class="normal"> 40</span>
+<span class="normal"> 41</span>
+<span class="normal"> 42</span>
+<span class="normal"> 43</span>
+<span class="normal"> 44</span>
+<span class="normal"> 45</span>
+<span class="normal"> 46</span>
+<span class="normal"> 47</span>
+<span class="normal"> 48</span>
+<span class="normal"> 49</span>
+<span class="normal"> 50</span>
+<span class="normal"> 51</span>
+<span class="normal"> 52</span>
+<span class="normal"> 53</span>
+<span class="normal"> 54</span>
+<span class="normal"> 55</span>
+<span class="normal"> 56</span>
+<span class="normal"> 57</span>
+<span class="normal"> 58</span>
+<span class="normal"> 59</span>
+<span class="normal"> 60</span>
+<span class="normal"> 61</span>
+<span class="normal"> 62</span>
+<span class="normal"> 63</span>
+<span class="normal"> 64</span>
+<span class="normal"> 65</span>
+<span class="normal"> 66</span>
+<span class="normal"> 67</span>
+<span class="normal"> 68</span>
+<span class="normal"> 69</span>
+<span class="normal"> 70</span>
+<span class="normal"> 71</span>
+<span class="normal"> 72</span>
+<span class="normal"> 73</span>
+<span class="normal"> 74</span>
+<span class="normal"> 75</span>
+<span class="normal"> 76</span>
+<span class="normal"> 77</span>
+<span class="normal"> 78</span>
+<span class="normal"> 79</span>
+<span class="normal"> 80</span>
+<span class="normal"> 81</span>
+<span class="normal"> 82</span>
+<span class="normal"> 83</span>
+<span class="normal"> 84</span>
+<span class="normal"> 85</span>
+<span class="normal"> 86</span>
+<span class="normal"> 87</span>
+<span class="normal"> 88</span>
+<span class="normal"> 89</span>
+<span class="normal"> 90</span>
+<span class="normal"> 91</span>
+<span class="normal"> 92</span>
+<span class="normal"> 93</span>
+<span class="normal"> 94</span>
+<span class="normal"> 95</span>
+<span class="normal"> 96</span>
+<span class="normal"> 97</span>
+<span class="normal"> 98</span>
+<span class="normal"> 99</span>
+<span class="normal">100</span>
+<span class="normal">101</span>
+<span class="normal">102</span>
+<span class="normal">103</span>
+<span class="normal">104</span>
+<span class="normal">105</span>
+<span class="normal">106</span>
+<span class="normal">107</span>
+<span class="normal">108</span>
+<span class="normal">109</span>
+<span class="normal">110</span>
+<span class="normal">111</span>
+<span class="normal">112</span>
+<span class="normal">113</span>
+<span class="normal">114</span>
+<span class="normal">115</span>
+<span class="normal">116</span>
+<span class="normal">117</span>
+<span class="normal">118</span>
+<span class="normal">119</span>
+<span class="normal">120</span>
+<span class="normal">121</span>
+<span class="normal">122</span>
+<span class="normal">123</span>
+<span class="normal">124</span>
+<span class="normal">125</span>
+<span class="normal">126</span>
+<span class="normal">127</span>
+<span class="normal">128</span>
+<span class="normal">129</span>
+<span class="normal">130</span>
+<span class="normal">131</span>
+<span class="normal">132</span>
+<span class="normal">133</span>
+<span class="normal">134</span>
+<span class="normal">135</span>
+<span class="normal">136</span>
+<span class="normal">137</span>
+<span class="normal">138</span>
+<span class="normal">139</span>
+<span class="normal">140</span>
+<span class="normal">141</span>
+<span class="normal">142</span>
+<span class="normal">143</span>
+<span class="normal">144</span>
+<span class="normal">145</span>
+<span class="normal">146</span>
+<span class="normal">147</span>
+<span class="normal">148</span>
+<span class="normal">149</span>
+<span class="normal">150</span>
+<span class="normal">151</span>
+<span class="normal">152</span>
+<span class="normal">153</span>
+<span class="normal">154</span>
+<span class="normal">155</span>
+<span class="normal">156</span>
+<span class="normal">157</span>
+<span class="normal">158</span>
+<span class="normal">159</span>
+<span class="normal">160</span>
+<span class="normal">161</span>
+<span class="normal">162</span>
+<span class="normal">163</span>
+<span class="normal">164</span>
+<span class="normal">165</span>
+<span class="normal">166</span>
+<span class="normal">167</span>
+<span class="normal">168</span>
+<span class="normal">169</span>
+<span class="normal">170</span>
+<span class="normal">171</span>
+<span class="normal">172</span>
+<span class="normal">173</span>
+<span class="normal">174</span>
+<span class="normal">175</span>
+<span class="normal">176</span>
+<span class="normal">177</span>
+<span class="normal">178</span>
+<span class="normal">179</span>
+<span class="normal">180</span>
+<span class="normal">181</span>
+<span class="normal">182</span>
+<span class="normal">183</span>
+<span class="normal">184</span>
+<span class="normal">185</span>
+<span class="normal">186</span>
+<span class="normal">187</span>
+<span class="normal">188</span>
+<span class="normal">189</span>
+<span class="normal">190</span>
+<span class="normal">191</span>
+<span class="normal">192</span>
+<span class="normal">193</span>
+<span class="normal">194</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">class</span> <span class="nc">ImagePreprocessor</span><span class="p">(</span><span class="n">BasePreprocessor</span><span class="p">):</span>
+<span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Preprocessor to prepare the ``deepimage`` input dataset.</span>
+
+<span class="sd">    The Preprocessing consists simply on resizing according to their</span>
+<span class="sd">    aspect ratio</span>
+
+<span class="sd">    Parameters</span>
+<span class="sd">    ----------</span>
+<span class="sd">    img_col: str</span>
+<span class="sd">        name of the column with the images filenames</span>
+<span class="sd">    img_path: str</span>
+<span class="sd">        path to the dicrectory where the images are stored</span>
+<span class="sd">    width: int, default=224</span>
+<span class="sd">        width of the resulting processed image.</span>
+<span class="sd">    height: int, default=224</span>
+<span class="sd">        width of the resulting processed image.</span>
+<span class="sd">    verbose: int, default 1</span>
+<span class="sd">        Enable verbose output.</span>
+
+<span class="sd">    Attributes</span>
+<span class="sd">    ----------</span>
+<span class="sd">    aap: AspectAwarePreprocessor</span>
+<span class="sd">        an instance of `pytorch_widedeep.utils.image_utils.AspectAwarePreprocessor`</span>
+<span class="sd">    spp: SimplePreprocessor</span>
+<span class="sd">        an instance of `pytorch_widedeep.utils.image_utils.SimplePreprocessor`</span>
+<span class="sd">    normalise_metrics: Dict</span>
+<span class="sd">        Dict containing the normalisation metrics of the image dataset, i.e.</span>
+<span class="sd">        mean and std for the R, G and B channels</span>
+
+<span class="sd">    Examples</span>
+<span class="sd">    --------</span>
+<span class="sd">    &gt;&gt;&gt; import pandas as pd</span>
+<span class="sd">    &gt;&gt;&gt;</span>
+<span class="sd">    &gt;&gt;&gt; from pytorch_widedeep.preprocessing import ImagePreprocessor</span>
+<span class="sd">    &gt;&gt;&gt;</span>
+<span class="sd">    &gt;&gt;&gt; path_to_image1 = &#39;tests/test_data_utils/images/galaxy1.png&#39;</span>
+<span class="sd">    &gt;&gt;&gt; path_to_image2 = &#39;tests/test_data_utils/images/galaxy2.png&#39;</span>
+<span class="sd">    &gt;&gt;&gt;</span>
+<span class="sd">    &gt;&gt;&gt; df_train = pd.DataFrame({&#39;images_column&#39;: [path_to_image1]})</span>
+<span class="sd">    &gt;&gt;&gt; df_test = pd.DataFrame({&#39;images_column&#39;: [path_to_image2]})</span>
+<span class="sd">    &gt;&gt;&gt; img_preprocessor = ImagePreprocessor(img_col=&#39;images_column&#39;, img_path=&#39;.&#39;, verbose=0)</span>
+<span class="sd">    &gt;&gt;&gt; resized_images = img_preprocessor.fit_transform(df_train)</span>
+<span class="sd">    &gt;&gt;&gt; new_resized_images = img_preprocessor.transform(df_train)</span>
+
+<span class="sd">    :information_source: **NOTE**:</span>
+<span class="sd">    Normalising metrics will only be computed when the ``fit_transform``</span>
+<span class="sd">    method is run. Running ``transform`` only will not change the computed</span>
+<span class="sd">    metrics and running ``fit`` only simply instantiates the resizing</span>
+<span class="sd">    functions.</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+
+    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span>
+        <span class="n">img_col</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
+        <span class="n">img_path</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
+        <span class="n">width</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">224</span><span class="p">,</span>
+        <span class="n">height</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">224</span><span class="p">,</span>
+        <span class="n">verbose</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1</span><span class="p">,</span>
+    <span class="p">):</span>
+        <span class="nb">super</span><span class="p">(</span><span class="n">ImagePreprocessor</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">img_col</span> <span class="o">=</span> <span class="n">img_col</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">img_path</span> <span class="o">=</span> <span class="n">img_path</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">width</span> <span class="o">=</span> <span class="n">width</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">height</span> <span class="o">=</span> <span class="n">height</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">verbose</span> <span class="o">=</span> <span class="n">verbose</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">aap</span> <span class="o">=</span> <span class="n">AspectAwarePreprocessor</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">width</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">height</span><span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">spp</span> <span class="o">=</span> <span class="n">SimplePreprocessor</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">width</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">height</span><span class="p">)</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">compute_normalising_computed</span> <span class="o">=</span> <span class="kc">False</span>
+
+    <span class="k">def</span> <span class="nf">fit</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">BasePreprocessor</span><span class="p">:</span>
+        <span class="c1"># simply include this method to comply with the BasePreprocessor and</span>
+        <span class="c1"># for consistency with the rest of the preprocessors</span>
+        <span class="k">return</span> <span class="bp">self</span>
+
+    <span class="k">def</span> <span class="nf">transform_sample</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">img</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">:</span>
+        <span class="n">aspect_r</span> <span class="o">=</span> <span class="n">img</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">/</span> <span class="n">img</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span>
+        <span class="k">if</span> <span class="n">aspect_r</span> <span class="o">!=</span> <span class="mf">1.0</span><span class="p">:</span>
+            <span class="n">resized_img</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">aap</span><span class="o">.</span><span class="n">preprocess</span><span class="p">(</span><span class="n">img</span><span class="p">)</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="n">resized_img</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">spp</span><span class="o">.</span><span class="n">preprocess</span><span class="p">(</span><span class="n">img</span><span class="p">)</span>
+        <span class="k">return</span> <span class="n">resized_img</span>
+
+    <span class="k">def</span> <span class="nf">transform</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">:</span>
+<span class="w">        </span><span class="sd">&quot;&quot;&quot;Resizes the images to the input height and width.</span>
+
+
+<span class="sd">        Parameters</span>
+<span class="sd">        ----------</span>
+<span class="sd">        df: pd.DataFrame</span>
+<span class="sd">            Input pandas dataframe with the `img_col`</span>
+
+<span class="sd">        Returns</span>
+<span class="sd">        -------</span>
+<span class="sd">        np.ndarray</span>
+<span class="sd">            Resized images to the input height and width</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">image_list</span> <span class="o">=</span> <span class="n">df</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">img_col</span><span class="p">]</span><span class="o">.</span><span class="n">tolist</span><span class="p">()</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">verbose</span><span class="p">:</span>
+            <span class="nb">print</span><span class="p">(</span><span class="s2">&quot;Reading Images from </span><span class="si">{}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">img_path</span><span class="p">))</span>
+        <span class="n">imgs</span> <span class="o">=</span> <span class="p">[</span><span class="n">cv2</span><span class="o">.</span><span class="n">imread</span><span class="p">(</span><span class="s2">&quot;/&quot;</span><span class="o">.</span><span class="n">join</span><span class="p">([</span><span class="bp">self</span><span class="o">.</span><span class="n">img_path</span><span class="p">,</span> <span class="n">img</span><span class="p">]))</span> <span class="k">for</span> <span class="n">img</span> <span class="ow">in</span> <span class="n">image_list</span><span class="p">]</span>
+
+        <span class="c1"># finding images with different height and width</span>
+        <span class="n">aspect</span> <span class="o">=</span> <span class="p">[(</span><span class="n">im</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="n">im</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">1</span><span class="p">])</span> <span class="k">for</span> <span class="n">im</span> <span class="ow">in</span> <span class="n">imgs</span><span class="p">]</span>
+        <span class="n">aspect_r</span> <span class="o">=</span> <span class="p">[</span><span class="n">a</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">/</span> <span class="n">a</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span> <span class="k">for</span> <span class="n">a</span> <span class="ow">in</span> <span class="n">aspect</span><span class="p">]</span>
+        <span class="n">diff_idx</span> <span class="o">=</span> <span class="p">[</span><span class="n">i</span> <span class="k">for</span> <span class="n">i</span><span class="p">,</span> <span class="n">r</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="n">aspect_r</span><span class="p">)</span> <span class="k">if</span> <span class="n">r</span> <span class="o">!=</span> <span class="mf">1.0</span><span class="p">]</span>
+
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">verbose</span><span class="p">:</span>
+            <span class="nb">print</span><span class="p">(</span><span class="s2">&quot;Resizing&quot;</span><span class="p">)</span>
+        <span class="n">resized_imgs</span> <span class="o">=</span> <span class="p">[]</span>
+        <span class="k">for</span> <span class="n">i</span><span class="p">,</span> <span class="n">img</span> <span class="ow">in</span> <span class="n">tqdm</span><span class="p">(</span><span class="nb">enumerate</span><span class="p">(</span><span class="n">imgs</span><span class="p">),</span> <span class="n">total</span><span class="o">=</span><span class="nb">len</span><span class="p">(</span><span class="n">imgs</span><span class="p">),</span> <span class="n">disable</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">verbose</span> <span class="o">!=</span> <span class="mi">1</span><span class="p">):</span>
+            <span class="k">if</span> <span class="n">i</span> <span class="ow">in</span> <span class="n">diff_idx</span><span class="p">:</span>
+                <span class="n">resized_imgs</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">aap</span><span class="o">.</span><span class="n">preprocess</span><span class="p">(</span><span class="n">img</span><span class="p">))</span>
+            <span class="k">else</span><span class="p">:</span>
+                <span class="c1"># if aspect ratio is 1:1, no need for AspectAwarePreprocessor</span>
+                <span class="n">resized_imgs</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">spp</span><span class="o">.</span><span class="n">preprocess</span><span class="p">(</span><span class="n">img</span><span class="p">))</span>
+
+        <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">compute_normalising_computed</span><span class="p">:</span>
+            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">verbose</span><span class="p">:</span>
+                <span class="nb">print</span><span class="p">(</span><span class="s2">&quot;Computing normalisation metrics&quot;</span><span class="p">)</span>
+            <span class="c1"># mean and std deviation will only be computed when the fit method</span>
+            <span class="c1"># is called</span>
+            <span class="n">mean_R</span><span class="p">,</span> <span class="n">mean_G</span><span class="p">,</span> <span class="n">mean_B</span> <span class="o">=</span> <span class="p">[],</span> <span class="p">[],</span> <span class="p">[]</span>
+            <span class="n">std_R</span><span class="p">,</span> <span class="n">std_G</span><span class="p">,</span> <span class="n">std_B</span> <span class="o">=</span> <span class="p">[],</span> <span class="p">[],</span> <span class="p">[]</span>
+            <span class="k">for</span> <span class="n">rsz_img</span> <span class="ow">in</span> <span class="n">resized_imgs</span><span class="p">:</span>
+                <span class="p">(</span><span class="n">mean_b</span><span class="p">,</span> <span class="n">mean_g</span><span class="p">,</span> <span class="n">mean_r</span><span class="p">),</span> <span class="p">(</span><span class="n">std_b</span><span class="p">,</span> <span class="n">std_g</span><span class="p">,</span> <span class="n">std_r</span><span class="p">)</span> <span class="o">=</span> <span class="n">cv2</span><span class="o">.</span><span class="n">meanStdDev</span><span class="p">(</span>
+                    <span class="n">rsz_img</span>
+                <span class="p">)</span>
+                <span class="n">mean_R</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">mean_r</span><span class="p">)</span>
+                <span class="n">mean_G</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">mean_g</span><span class="p">)</span>
+                <span class="n">mean_B</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">mean_b</span><span class="p">)</span>
+                <span class="n">std_R</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">std_r</span><span class="p">)</span>
+                <span class="n">std_G</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">std_g</span><span class="p">)</span>
+                <span class="n">std_B</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">std_b</span><span class="p">)</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">normalise_metrics</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">(</span>
+                <span class="n">mean</span><span class="o">=</span><span class="p">{</span>
+                    <span class="s2">&quot;R&quot;</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="n">mean_R</span><span class="p">)</span> <span class="o">/</span> <span class="mf">255.0</span><span class="p">,</span>
+                    <span class="s2">&quot;G&quot;</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="n">mean_G</span><span class="p">)</span> <span class="o">/</span> <span class="mf">255.0</span><span class="p">,</span>
+                    <span class="s2">&quot;B&quot;</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="n">mean_B</span><span class="p">)</span> <span class="o">/</span> <span class="mf">255.0</span><span class="p">,</span>
+                <span class="p">},</span>
+                <span class="n">std</span><span class="o">=</span><span class="p">{</span>
+                    <span class="s2">&quot;R&quot;</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="n">std_R</span><span class="p">)</span> <span class="o">/</span> <span class="mf">255.0</span><span class="p">,</span>
+                    <span class="s2">&quot;G&quot;</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="n">std_G</span><span class="p">)</span> <span class="o">/</span> <span class="mf">255.0</span><span class="p">,</span>
+                    <span class="s2">&quot;B&quot;</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="n">std_B</span><span class="p">)</span> <span class="o">/</span> <span class="mf">255.0</span><span class="p">,</span>
+                <span class="p">},</span>
+            <span class="p">)</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">compute_normalising_computed</span> <span class="o">=</span> <span class="kc">True</span>
+        <span class="k">return</span> <span class="n">np</span><span class="o">.</span><span class="n">asarray</span><span class="p">(</span><span class="n">resized_imgs</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">fit_transform</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">:</span>
+<span class="w">        </span><span class="sd">&quot;&quot;&quot;Combines `fit` and `transform`</span>
+
+<span class="sd">        Parameters</span>
+<span class="sd">        ----------</span>
+<span class="sd">        df: pd.DataFrame</span>
+<span class="sd">            Input pandas dataframe</span>
+
+<span class="sd">        Returns</span>
+<span class="sd">        -------</span>
+<span class="sd">        np.ndarray</span>
+<span class="sd">            Resized images to the input height and width</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">df</span><span class="p">)</span><span class="o">.</span><span class="n">transform</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">inverse_transform</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">transformed_image</span><span class="p">):</span>
+        <span class="k">raise</span> <span class="ne">NotImplementedError</span><span class="p">(</span>
+            <span class="s2">&quot;&#39;inverse_transform&#39; method is not implemented for &#39;ImagePreprocessor&#39;&quot;</span>
+        <span class="p">)</span>
+
+    <span class="k">def</span> <span class="fm">__repr__</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">str</span><span class="p">:</span>
+        <span class="n">list_of_params</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
+        <span class="n">list_of_params</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">&quot;img_col=</span><span class="si">{img_col}</span><span class="s2">&quot;</span><span class="p">)</span>
+        <span class="n">list_of_params</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">&quot;img_path=</span><span class="si">{img_path}</span><span class="s2">&quot;</span><span class="p">)</span>
+        <span class="n">list_of_params</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">&quot;width=</span><span class="si">{width}</span><span class="s2">&quot;</span><span class="p">)</span>
+        <span class="n">list_of_params</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">&quot;height=</span><span class="si">{height}</span><span class="s2">&quot;</span><span class="p">)</span>
+        <span class="n">list_of_params</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">&quot;verbose=</span><span class="si">{verbose}</span><span class="s2">&quot;</span><span class="p">)</span>
+        <span class="n">all_params</span> <span class="o">=</span> <span class="s2">&quot;, &quot;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">list_of_params</span><span class="p">)</span>
+        <span class="k">return</span> <span class="sa">f</span><span class="s2">&quot;ImagePreprocessor(</span><span class="si">{</span><span class="n">all_params</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="bp">self</span><span class="o">.</span><span class="vm">__dict__</span><span class="p">)</span><span class="si">}</span><span class="s2">)&quot;</span>
+</code></pre></div></td></tr></table></div>
+              </details>
+
+
+
+  <div class="doc doc-children">
+
+
+
+
+
+
+
+
+
+<div class="doc doc-object doc-function">
+
+
+<h3 id="pytorch_widedeep.preprocessing.image_preprocessor.ImagePreprocessor.transform" class="doc doc-heading">
+            <span class="doc doc-object-name doc-function-name">transform</span>
+
+
+<a href="#pytorch_widedeep.preprocessing.image_preprocessor.ImagePreprocessor.transform" class="headerlink" title="Permanent link">&para;</a></h3>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">transform</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
+</code></pre></div>
+
+    <div class="doc doc-contents ">
+
+        <p>Resizes the images to the input height and width.</p>
+
+
+<p><span class="doc-section-title">Parameters:</span></p>
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+          <th>Default</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>df</code></td>
+            <td>
+                  <code><span title="pandas.DataFrame">DataFrame</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Input pandas dataframe with the <code>img_col</code></p>
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+      </tbody>
+    </table>
+
+
+    <p><span class="doc-section-title">Returns:</span></p>
+    <table>
+      <thead>
+        <tr>
+          <th>Type</th>
+          <th>Description</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td>
+                  <code><span title="numpy.ndarray">ndarray</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Resized images to the input height and width</p>
+              </div>
+            </td>
+          </tr>
+      </tbody>
+    </table>
+
+            <details class="quote">
+              <summary>Source code in <code>pytorch_widedeep/preprocessing/image_preprocessor.py</code></summary>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">100</span>
+<span class="normal">101</span>
+<span class="normal">102</span>
+<span class="normal">103</span>
+<span class="normal">104</span>
+<span class="normal">105</span>
+<span class="normal">106</span>
+<span class="normal">107</span>
+<span class="normal">108</span>
+<span class="normal">109</span>
+<span class="normal">110</span>
+<span class="normal">111</span>
+<span class="normal">112</span>
+<span class="normal">113</span>
+<span class="normal">114</span>
+<span class="normal">115</span>
+<span class="normal">116</span>
+<span class="normal">117</span>
+<span class="normal">118</span>
+<span class="normal">119</span>
+<span class="normal">120</span>
+<span class="normal">121</span>
+<span class="normal">122</span>
+<span class="normal">123</span>
+<span class="normal">124</span>
+<span class="normal">125</span>
+<span class="normal">126</span>
+<span class="normal">127</span>
+<span class="normal">128</span>
+<span class="normal">129</span>
+<span class="normal">130</span>
+<span class="normal">131</span>
+<span class="normal">132</span>
+<span class="normal">133</span>
+<span class="normal">134</span>
+<span class="normal">135</span>
+<span class="normal">136</span>
+<span class="normal">137</span>
+<span class="normal">138</span>
+<span class="normal">139</span>
+<span class="normal">140</span>
+<span class="normal">141</span>
+<span class="normal">142</span>
+<span class="normal">143</span>
+<span class="normal">144</span>
+<span class="normal">145</span>
+<span class="normal">146</span>
+<span class="normal">147</span>
+<span class="normal">148</span>
+<span class="normal">149</span>
+<span class="normal">150</span>
+<span class="normal">151</span>
+<span class="normal">152</span>
+<span class="normal">153</span>
+<span class="normal">154</span>
+<span class="normal">155</span>
+<span class="normal">156</span>
+<span class="normal">157</span>
+<span class="normal">158</span>
+<span class="normal">159</span>
+<span class="normal">160</span>
+<span class="normal">161</span>
+<span class="normal">162</span>
+<span class="normal">163</span>
+<span class="normal">164</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">transform</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">:</span>
+<span class="w">    </span><span class="sd">&quot;&quot;&quot;Resizes the images to the input height and width.</span>
+
+
+<span class="sd">    Parameters</span>
+<span class="sd">    ----------</span>
+<span class="sd">    df: pd.DataFrame</span>
+<span class="sd">        Input pandas dataframe with the `img_col`</span>
+
+<span class="sd">    Returns</span>
+<span class="sd">    -------</span>
+<span class="sd">    np.ndarray</span>
+<span class="sd">        Resized images to the input height and width</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="n">image_list</span> <span class="o">=</span> <span class="n">df</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">img_col</span><span class="p">]</span><span class="o">.</span><span class="n">tolist</span><span class="p">()</span>
+    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">verbose</span><span class="p">:</span>
+        <span class="nb">print</span><span class="p">(</span><span class="s2">&quot;Reading Images from </span><span class="si">{}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">img_path</span><span class="p">))</span>
+    <span class="n">imgs</span> <span class="o">=</span> <span class="p">[</span><span class="n">cv2</span><span class="o">.</span><span class="n">imread</span><span class="p">(</span><span class="s2">&quot;/&quot;</span><span class="o">.</span><span class="n">join</span><span class="p">([</span><span class="bp">self</span><span class="o">.</span><span class="n">img_path</span><span class="p">,</span> <span class="n">img</span><span class="p">]))</span> <span class="k">for</span> <span class="n">img</span> <span class="ow">in</span> <span class="n">image_list</span><span class="p">]</span>
+
+    <span class="c1"># finding images with different height and width</span>
+    <span class="n">aspect</span> <span class="o">=</span> <span class="p">[(</span><span class="n">im</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="n">im</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">1</span><span class="p">])</span> <span class="k">for</span> <span class="n">im</span> <span class="ow">in</span> <span class="n">imgs</span><span class="p">]</span>
+    <span class="n">aspect_r</span> <span class="o">=</span> <span class="p">[</span><span class="n">a</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">/</span> <span class="n">a</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span> <span class="k">for</span> <span class="n">a</span> <span class="ow">in</span> <span class="n">aspect</span><span class="p">]</span>
+    <span class="n">diff_idx</span> <span class="o">=</span> <span class="p">[</span><span class="n">i</span> <span class="k">for</span> <span class="n">i</span><span class="p">,</span> <span class="n">r</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="n">aspect_r</span><span class="p">)</span> <span class="k">if</span> <span class="n">r</span> <span class="o">!=</span> <span class="mf">1.0</span><span class="p">]</span>
+
+    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">verbose</span><span class="p">:</span>
+        <span class="nb">print</span><span class="p">(</span><span class="s2">&quot;Resizing&quot;</span><span class="p">)</span>
+    <span class="n">resized_imgs</span> <span class="o">=</span> <span class="p">[]</span>
+    <span class="k">for</span> <span class="n">i</span><span class="p">,</span> <span class="n">img</span> <span class="ow">in</span> <span class="n">tqdm</span><span class="p">(</span><span class="nb">enumerate</span><span class="p">(</span><span class="n">imgs</span><span class="p">),</span> <span class="n">total</span><span class="o">=</span><span class="nb">len</span><span class="p">(</span><span class="n">imgs</span><span class="p">),</span> <span class="n">disable</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">verbose</span> <span class="o">!=</span> <span class="mi">1</span><span class="p">):</span>
+        <span class="k">if</span> <span class="n">i</span> <span class="ow">in</span> <span class="n">diff_idx</span><span class="p">:</span>
+            <span class="n">resized_imgs</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">aap</span><span class="o">.</span><span class="n">preprocess</span><span class="p">(</span><span class="n">img</span><span class="p">))</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="c1"># if aspect ratio is 1:1, no need for AspectAwarePreprocessor</span>
+            <span class="n">resized_imgs</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">spp</span><span class="o">.</span><span class="n">preprocess</span><span class="p">(</span><span class="n">img</span><span class="p">))</span>
+
+    <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">compute_normalising_computed</span><span class="p">:</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">verbose</span><span class="p">:</span>
+            <span class="nb">print</span><span class="p">(</span><span class="s2">&quot;Computing normalisation metrics&quot;</span><span class="p">)</span>
+        <span class="c1"># mean and std deviation will only be computed when the fit method</span>
+        <span class="c1"># is called</span>
+        <span class="n">mean_R</span><span class="p">,</span> <span class="n">mean_G</span><span class="p">,</span> <span class="n">mean_B</span> <span class="o">=</span> <span class="p">[],</span> <span class="p">[],</span> <span class="p">[]</span>
+        <span class="n">std_R</span><span class="p">,</span> <span class="n">std_G</span><span class="p">,</span> <span class="n">std_B</span> <span class="o">=</span> <span class="p">[],</span> <span class="p">[],</span> <span class="p">[]</span>
+        <span class="k">for</span> <span class="n">rsz_img</span> <span class="ow">in</span> <span class="n">resized_imgs</span><span class="p">:</span>
+            <span class="p">(</span><span class="n">mean_b</span><span class="p">,</span> <span class="n">mean_g</span><span class="p">,</span> <span class="n">mean_r</span><span class="p">),</span> <span class="p">(</span><span class="n">std_b</span><span class="p">,</span> <span class="n">std_g</span><span class="p">,</span> <span class="n">std_r</span><span class="p">)</span> <span class="o">=</span> <span class="n">cv2</span><span class="o">.</span><span class="n">meanStdDev</span><span class="p">(</span>
+                <span class="n">rsz_img</span>
+            <span class="p">)</span>
+            <span class="n">mean_R</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">mean_r</span><span class="p">)</span>
+            <span class="n">mean_G</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">mean_g</span><span class="p">)</span>
+            <span class="n">mean_B</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">mean_b</span><span class="p">)</span>
+            <span class="n">std_R</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">std_r</span><span class="p">)</span>
+            <span class="n">std_G</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">std_g</span><span class="p">)</span>
+            <span class="n">std_B</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">std_b</span><span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">normalise_metrics</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">(</span>
+            <span class="n">mean</span><span class="o">=</span><span class="p">{</span>
+                <span class="s2">&quot;R&quot;</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="n">mean_R</span><span class="p">)</span> <span class="o">/</span> <span class="mf">255.0</span><span class="p">,</span>
+                <span class="s2">&quot;G&quot;</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="n">mean_G</span><span class="p">)</span> <span class="o">/</span> <span class="mf">255.0</span><span class="p">,</span>
+                <span class="s2">&quot;B&quot;</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="n">mean_B</span><span class="p">)</span> <span class="o">/</span> <span class="mf">255.0</span><span class="p">,</span>
+            <span class="p">},</span>
+            <span class="n">std</span><span class="o">=</span><span class="p">{</span>
+                <span class="s2">&quot;R&quot;</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="n">std_R</span><span class="p">)</span> <span class="o">/</span> <span class="mf">255.0</span><span class="p">,</span>
+                <span class="s2">&quot;G&quot;</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="n">std_G</span><span class="p">)</span> <span class="o">/</span> <span class="mf">255.0</span><span class="p">,</span>
+                <span class="s2">&quot;B&quot;</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="n">std_B</span><span class="p">)</span> <span class="o">/</span> <span class="mf">255.0</span><span class="p">,</span>
+            <span class="p">},</span>
+        <span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">compute_normalising_computed</span> <span class="o">=</span> <span class="kc">True</span>
+    <span class="k">return</span> <span class="n">np</span><span class="o">.</span><span class="n">asarray</span><span class="p">(</span><span class="n">resized_imgs</span><span class="p">)</span>
+</code></pre></div></td></tr></table></div>
+            </details>
+    </div>
+
+</div>
+
+<div class="doc doc-object doc-function">
+
+
+<h3 id="pytorch_widedeep.preprocessing.image_preprocessor.ImagePreprocessor.fit_transform" class="doc doc-heading">
+            <span class="doc doc-object-name doc-function-name">fit_transform</span>
+
+
+<a href="#pytorch_widedeep.preprocessing.image_preprocessor.ImagePreprocessor.fit_transform" class="headerlink" title="Permanent link">&para;</a></h3>
+<div class="doc-signature highlight"><pre><span></span><code><span class="nf">fit_transform</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
+</code></pre></div>
+
+    <div class="doc doc-contents ">
+
+        <p>Combines <code>fit</code> and <code>transform</code></p>
+
+
+<p><span class="doc-section-title">Parameters:</span></p>
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+          <th>Default</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>df</code></td>
+            <td>
+                  <code><span title="pandas.DataFrame">DataFrame</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Input pandas dataframe</p>
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+      </tbody>
+    </table>
+
+
+    <p><span class="doc-section-title">Returns:</span></p>
+    <table>
+      <thead>
+        <tr>
+          <th>Type</th>
+          <th>Description</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td>
+                  <code><span title="numpy.ndarray">ndarray</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Resized images to the input height and width</p>
+              </div>
+            </td>
+          </tr>
+      </tbody>
+    </table>
+
+            <details class="quote">
+              <summary>Source code in <code>pytorch_widedeep/preprocessing/image_preprocessor.py</code></summary>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">166</span>
+<span class="normal">167</span>
+<span class="normal">168</span>
+<span class="normal">169</span>
+<span class="normal">170</span>
+<span class="normal">171</span>
+<span class="normal">172</span>
+<span class="normal">173</span>
+<span class="normal">174</span>
+<span class="normal">175</span>
+<span class="normal">176</span>
+<span class="normal">177</span>
+<span class="normal">178</span>
+<span class="normal">179</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">fit_transform</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">:</span>
+<span class="w">    </span><span class="sd">&quot;&quot;&quot;Combines `fit` and `transform`</span>
+
+<span class="sd">    Parameters</span>
+<span class="sd">    ----------</span>
+<span class="sd">    df: pd.DataFrame</span>
+<span class="sd">        Input pandas dataframe</span>
+
+<span class="sd">    Returns</span>
+<span class="sd">    -------</span>
+<span class="sd">    np.ndarray</span>
+<span class="sd">        Resized images to the input height and width</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">df</span><span class="p">)</span><span class="o">.</span><span class="n">transform</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
+</code></pre></div></td></tr></table></div>
+            </details>
+    </div>
+
+</div>
+
+
+
+  </div>
+
+    </div>
+
+</div>
+
+<div class="doc doc-object doc-class">
+
+
+
+<h2 id="pytorch_widedeep.preprocessing.din_preprocessor.DINPreprocessor" class="doc doc-heading">
+            <span class="doc doc-object-name doc-class-name">DINPreprocessor</span>
+
+
+<a href="#pytorch_widedeep.preprocessing.din_preprocessor.DINPreprocessor" class="headerlink" title="Permanent link">&para;</a></h2>
+
+
+    <div class="doc doc-contents first">
+            <p class="doc doc-class-bases">
+              Bases: <code><span title="pytorch_widedeep.preprocessing.base_preprocessor.BasePreprocessor">BasePreprocessor</span></code></p>
+
+
+        <p>Preprocessor for Deep Interest Network (DIN) models.</p>
+<p>This preprocessor handles the preparation of data for DIN models,
+including sequence building, label encoding, and handling of various
+types of input columns (categorical, continuous, and sequential).</p>
+
+
+<p><span class="doc-section-title">Parameters:</span></p>
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+          <th>Default</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>user_id_col</code></td>
+            <td>
+                  <code>str</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Name of the column containing user IDs.</p>
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>item_embed_col</code></td>
+            <td>
+                  <code><span title="typing.Union">Union</span>[str, <span title="typing.Tuple">Tuple</span>[str, int]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Name of the column containing item IDs to be embedded, or a tuple of
+(column_name, embedding_dim). If embedding_dim is not provided, it
+will be automatically calculated using a rule of thumb specified by
+<code>embedding_rule</code>. Aliased as <code>item_id_col</code>.</p>
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>target_col</code></td>
+            <td>
+                  <code>str</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Name of the column containing the target variable.</p>
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>max_seq_length</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Maximum length of sequences to be created.</p>
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>action_col</code></td>
+            <td>
+                  <code><span title="typing.Optional">Optional</span>[str]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Name of the column containing user actions (if applicable), for
+example a rating, or purchased/not-purchased, etc). This 'action_col'
+can also be the same as the 'target_col'.</p>
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>other_seq_embed_cols</code></td>
+            <td>
+                  <code><span title="typing.Optional">Optional</span>[<span title="typing.List">List</span>[str] | <span title="typing.List">List</span>[<span title="typing.Tuple">Tuple</span>[str, int]]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>List of other columns to be treated as sequences. Each element in the
+list can be a string with the name of the column or a tuple with the
+name of the column and the embedding dimension.</p>
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>cat_embed_cols</code></td>
+            <td>
+                  <code><span title="typing.Optional">Optional</span>[<span title="typing.Union">Union</span>[<span title="typing.List">List</span>[str], <span title="typing.List">List</span>[<span title="typing.Tuple">Tuple</span>[str, int]]]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>This and the following parameters are identical to those used in the
+TabPreprocessor.<br>
+List containing the name of the categorical columns that will be
+represented by embeddings (e.g. <em>['education', 'relationship', ...]</em>) or
+a Tuple with the name and the embedding dimension (e.g.: <em>[
+('education',32), ('relationship',16), ...]</em>)</p>
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>continuous_cols</code></td>
+            <td>
+                  <code><span title="typing.Optional">Optional</span>[<span title="typing.List">List</span>[str]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>List with the name of the continuous cols</p>
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>quantization_setup</code></td>
+            <td>
+                  <code><span title="typing.Optional">Optional</span>[<span title="typing.Union">Union</span>[int, <span title="typing.Dict">Dict</span>[str, <span title="typing.Union">Union</span>[int, <span title="typing.List">List</span>[float]]]]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Continuous columns can be turned into categorical via <code>pd.cut</code>. If
+<code>quantization_setup</code> is an <code>int</code>, all continuous columns will be
+quantized using this value as the number of bins. Alternatively, a
+dictionary where the keys are the column names to quantize and the
+values are the either integers indicating the number of bins or a
+list of scalars indicating the bin edges can also be used.</p>
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>cols_to_scale</code></td>
+            <td>
+                  <code><span title="typing.Optional">Optional</span>[<span title="typing.Union">Union</span>[<span title="typing.List">List</span>[str], str]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>List with the names of the columns that will be standarised via
+sklearn's <code>StandardScaler</code>. It can also be the string <code>'all'</code> in
+which case all the continuous cols will be scaled.</p>
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>auto_embed_dim</code></td>
+            <td>
+                  <code>bool</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Boolean indicating whether the embedding dimensions will be
+automatically defined via rule of thumb. See <code>embedding_rule</code>
+below.</p>
+              </div>
+            </td>
+            <td>
+                  <code>True</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>embedding_rule</code></td>
+            <td>
+                  <code><span title="typing.Literal">Literal</span>[&#39;google&#39;, &#39;fastai_old&#39;, &#39;fastai_new&#39;]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>If <code>auto_embed_dim=True</code>, this is the choice of embedding rule of
+thumb. Choices are:</p>
+<ul>
+<li>
+<p><em>fastai_new</em>: <span class="arithmatex">\(min(600, round(1.6     imes n_{cat}^{0.56}))\)</span></p>
+</li>
+<li>
+<p><em>fastai_old</em>: <span class="arithmatex">\(min(50, (n_{cat}//{2})+1)\)</span></p>
+</li>
+<li>
+<p><em>google</em>: <span class="arithmatex">\(min(600, round(n_{cat}^{0.24}))\)</span></p>
+</li>
+</ul>
+              </div>
+            </td>
+            <td>
+                  <code>&#39;fastai_new&#39;</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>default_embed_dim</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Dimension for the embeddings if the embedding dimension is not
+provided in the <code>cat_embed_cols</code> parameter and <code>auto_embed_dim</code> is
+set to <code>False</code>.</p>
+              </div>
+            </td>
+            <td>
+                  <code>16</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>verbose</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Verbosity level.</p>
+              </div>
+            </td>
+            <td>
+                  <code>1</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>scale</code></td>
+            <td>
+                  <code>bool</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p><img alt="ℹ️" class="emojione" src="https://cdnjs.cloudflare.com/ajax/libs/emojione/2.2.7/assets/png/2139.png" title=":information_source:" /> <strong>note</strong>: this arg will be removed in upcoming
+ releases. Please use <code>cols_to_scale</code> instead. <br/> Bool indicating
+ whether or not to scale/standarise continuous cols. It is important
+ to emphasize that all the DL models for tabular data in the library
+ also include the possibility of normalising the input continuous
+ features via a <code>BatchNorm</code> or a <code>LayerNorm</code>. <br/> Param alias:
+ <code>scale_cont_cols</code>.</p>
+              </div>
+            </td>
+            <td>
+                  <code>False</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>already_standard</code></td>
+            <td>
+                  <code><span title="typing.Optional">Optional</span>[<span title="typing.List">List</span>[str]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p><img alt="ℹ️" class="emojione" src="https://cdnjs.cloudflare.com/ajax/libs/emojione/2.2.7/assets/png/2139.png" title=":information_source:" /> <strong>note</strong>: this arg will be removed in upcoming
+ releases. Please use <code>cols_to_scale</code> instead. <br/> List with the
+ name of the continuous cols that do not need to be
+ scaled/standarised.</p>
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+      </tbody>
+    </table>
+
+
+<p><span class="doc-section-title">Other Parameters:</span></p>
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>**kwargs</code></td>
+            <td>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p><code>pd.cut</code> and <code>StandardScaler</code> related args</p>
+              </div>
+            </td>
+          </tr>
+      </tbody>
+    </table>
+
+
+<p><span class="doc-section-title">Attributes:</span></p>
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code><span title="pytorch_widedeep.preprocessing.din_preprocessor.DINPreprocessor.is_fitted">is_fitted</span></code></td>
+            <td>
+                  <code>bool</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Whether the preprocessor has been fitted.</p>
+              </div>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code><span title="pytorch_widedeep.preprocessing.din_preprocessor.DINPreprocessor.has_standard_tab_data">has_standard_tab_data</span></code></td>
+            <td>
+                  <code>bool</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Whether the data includes standard tabular data.</p>
+              </div>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code><span title="pytorch_widedeep.preprocessing.din_preprocessor.DINPreprocessor.tab_preprocessor">tab_preprocessor</span></code></td>
+            <td>
+                  <code><a class="autorefs autorefs-internal" title="pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor" href="#pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor">TabPreprocessor</a></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Preprocessor for standard tabular data (if applicable).</p>
+              </div>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code><span title="pytorch_widedeep.preprocessing.din_preprocessor.DINPreprocessor.din_columns_idx">din_columns_idx</span></code></td>
+            <td>
+                  <code><span title="typing.Dict">Dict</span>[str, int]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Dictionary mapping column names to their indices in the processed data.</p>
+              </div>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code><span title="pytorch_widedeep.preprocessing.din_preprocessor.DINPreprocessor.item_le">item_le</span></code></td>
+            <td>
+                  <code><a class="autorefs autorefs-internal" title="pytorch_widedeep.utils.deeptabular_utils.LabelEncoder" href="utils/deeptabular_utils.html#pytorch_widedeep.utils.deeptabular_utils.LabelEncoder">LabelEncoder</a></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Label encoder for item IDs.</p>
+              </div>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code><span title="pytorch_widedeep.preprocessing.din_preprocessor.DINPreprocessor.n_items">n_items</span></code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Number of unique items.</p>
+              </div>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code><span title="pytorch_widedeep.preprocessing.din_preprocessor.DINPreprocessor.user_behaviour_config">user_behaviour_config</span></code></td>
+            <td>
+                  <code><span title="typing.Tuple">Tuple</span>[<span title="typing.List">List</span>[str], int, int]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Configuration for user behavior sequences.</p>
+              </div>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code><span title="pytorch_widedeep.preprocessing.din_preprocessor.DINPreprocessor.action_le">action_le</span></code></td>
+            <td>
+                  <code><a class="autorefs autorefs-internal" title="pytorch_widedeep.utils.deeptabular_utils.LabelEncoder" href="utils/deeptabular_utils.html#pytorch_widedeep.utils.deeptabular_utils.LabelEncoder">LabelEncoder</a></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Label encoder for action column (if applicable).</p>
+              </div>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code><span title="pytorch_widedeep.preprocessing.din_preprocessor.DINPreprocessor.n_actions">n_actions</span></code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Number of unique actions (if applicable).</p>
+              </div>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code><span title="pytorch_widedeep.preprocessing.din_preprocessor.DINPreprocessor.action_seq_config">action_seq_config</span></code></td>
+            <td>
+                  <code><span title="typing.Tuple">Tuple</span>[<span title="typing.List">List</span>[str], int]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Configuration for action sequences (if applicable).</p>
+              </div>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code><span title="pytorch_widedeep.preprocessing.din_preprocessor.DINPreprocessor.other_seq_le">other_seq_le</span></code></td>
+            <td>
+                  <code><a class="autorefs autorefs-internal" title="pytorch_widedeep.utils.deeptabular_utils.LabelEncoder" href="utils/deeptabular_utils.html#pytorch_widedeep.utils.deeptabular_utils.LabelEncoder">LabelEncoder</a></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Label encoder for other sequence columns (if applicable).</p>
+              </div>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code><span title="pytorch_widedeep.preprocessing.din_preprocessor.DINPreprocessor.n_other_seq_cols">n_other_seq_cols</span></code></td>
+            <td>
+                  <code><span title="typing.Dict">Dict</span>[str, int]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Number of unique values in each other sequence column.</p>
+              </div>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code><span title="pytorch_widedeep.preprocessing.din_preprocessor.DINPreprocessor.other_seq_config">other_seq_config</span></code></td>
+            <td>
+                  <code><span title="typing.List">List</span>[<span title="typing.Tuple">Tuple</span>[<span title="typing.List">List</span>[str], int, int]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Configuration for other sequence columns.</p>
+              </div>
+            </td>
+          </tr>
+      </tbody>
+    </table>
+
+
+<p><span class="doc-section-title">Examples:</span></p>
+    <div class="highlight"><pre><span></span><code><span class="gp">&gt;&gt;&gt; </span><span class="kn">import</span> <span class="nn">pandas</span> <span class="k">as</span> <span class="nn">pd</span>
+<span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">pytorch_widedeep.preprocessing</span> <span class="kn">import</span> <span class="n">DINPreprocessor</span>
+<span class="gp">&gt;&gt;&gt; </span><span class="n">data</span> <span class="o">=</span> <span class="p">{</span>
+<span class="gp">... </span>   <span class="s1">&#39;user_id&#39;</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="mi">3</span><span class="p">],</span>
+<span class="gp">... </span>   <span class="s1">&#39;item_id&#39;</span><span class="p">:</span> <span class="p">[</span><span class="mi">101</span><span class="p">,</span> <span class="mi">102</span><span class="p">,</span> <span class="mi">103</span><span class="p">,</span> <span class="mi">101</span><span class="p">,</span> <span class="mi">103</span><span class="p">,</span> <span class="mi">104</span><span class="p">,</span> <span class="mi">102</span><span class="p">,</span> <span class="mi">103</span><span class="p">,</span> <span class="mi">104</span><span class="p">],</span>
+<span class="gp">... </span>   <span class="s1">&#39;timestamp&#39;</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">],</span>
+<span class="gp">... </span>   <span class="s1">&#39;category&#39;</span><span class="p">:</span> <span class="p">[</span><span class="s1">&#39;A&#39;</span><span class="p">,</span> <span class="s1">&#39;B&#39;</span><span class="p">,</span> <span class="s1">&#39;A&#39;</span><span class="p">,</span> <span class="s1">&#39;B&#39;</span><span class="p">,</span> <span class="s1">&#39;A&#39;</span><span class="p">,</span> <span class="s1">&#39;C&#39;</span><span class="p">,</span> <span class="s1">&#39;B&#39;</span><span class="p">,</span> <span class="s1">&#39;A&#39;</span><span class="p">,</span> <span class="s1">&#39;C&#39;</span><span class="p">],</span>
+<span class="gp">... </span>   <span class="s1">&#39;price&#39;</span><span class="p">:</span> <span class="p">[</span><span class="mf">10.5</span><span class="p">,</span> <span class="mf">15.0</span><span class="p">,</span> <span class="mf">12.0</span><span class="p">,</span> <span class="mf">10.5</span><span class="p">,</span> <span class="mf">12.0</span><span class="p">,</span> <span class="mf">20.0</span><span class="p">,</span> <span class="mf">15.0</span><span class="p">,</span> <span class="mf">12.0</span><span class="p">,</span> <span class="mf">20.0</span><span class="p">],</span>
+<span class="gp">... </span>   <span class="s1">&#39;rating&#39;</span><span class="p">:</span> <span class="p">[</span><span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">0</span><span class="p">]</span>
+<span class="gp">... </span><span class="p">}</span>
+<span class="gp">&gt;&gt;&gt; </span><span class="n">df</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">(</span><span class="n">data</span><span class="p">)</span>
+<span class="gp">&gt;&gt;&gt; </span><span class="n">din_preprocessor</span> <span class="o">=</span> <span class="n">DINPreprocessor</span><span class="p">(</span>
+<span class="gp">... </span>    <span class="n">user_id_col</span><span class="o">=</span><span class="s1">&#39;user_id&#39;</span><span class="p">,</span>
+<span class="gp">... </span>    <span class="n">item_embed_col</span><span class="o">=</span><span class="s1">&#39;item_id&#39;</span><span class="p">,</span>
+<span class="gp">... </span>    <span class="n">target_col</span><span class="o">=</span><span class="s1">&#39;rating&#39;</span><span class="p">,</span>
+<span class="gp">... </span>    <span class="n">max_seq_length</span><span class="o">=</span><span class="mi">2</span><span class="p">,</span>
+<span class="gp">... </span>    <span class="n">action_col</span><span class="o">=</span><span class="s1">&#39;rating&#39;</span><span class="p">,</span>
+<span class="gp">... </span>    <span class="n">other_seq_embed_cols</span><span class="o">=</span><span class="p">[</span><span class="s1">&#39;category&#39;</span><span class="p">],</span>
+<span class="gp">... </span>    <span class="n">cat_embed_cols</span><span class="o">=</span><span class="p">[</span><span class="s1">&#39;user_id&#39;</span><span class="p">],</span>
+<span class="gp">... </span>    <span class="n">continuous_cols</span><span class="o">=</span><span class="p">[</span><span class="s1">&#39;price&#39;</span><span class="p">],</span>
+<span class="gp">... </span>    <span class="n">cols_to_scale</span><span class="o">=</span><span class="p">[</span><span class="s1">&#39;price&#39;</span><span class="p">]</span>
+<span class="gp">... </span><span class="p">)</span>
+<span class="gp">&gt;&gt;&gt; </span><span class="n">X</span><span class="p">,</span> <span class="n">y</span> <span class="o">=</span> <span class="n">din_preprocessor</span><span class="o">.</span><span class="n">fit_transform</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
+</code></pre></div>
+
+              <details class="quote">
+                <summary>Source code in <code>pytorch_widedeep/preprocessing/din_preprocessor.py</code></summary>
+                <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 21</span>
+<span class="normal"> 22</span>
+<span class="normal"> 23</span>
+<span class="normal"> 24</span>
+<span class="normal"> 25</span>
+<span class="normal"> 26</span>
+<span class="normal"> 27</span>
+<span class="normal"> 28</span>
+<span class="normal"> 29</span>
+<span class="normal"> 30</span>
+<span class="normal"> 31</span>
+<span class="normal"> 32</span>
+<span class="normal"> 33</span>
+<span class="normal"> 34</span>
+<span class="normal"> 35</span>
+<span class="normal"> 36</span>
+<span class="normal"> 37</span>
+<span class="normal"> 38</span>
+<span class="normal"> 39</span>
+<span class="normal"> 40</span>
+<span class="normal"> 41</span>
+<span class="normal"> 42</span>
+<span class="normal"> 43</span>
+<span class="normal"> 44</span>
+<span class="normal"> 45</span>
+<span class="normal"> 46</span>
+<span class="normal"> 47</span>
+<span class="normal"> 48</span>
+<span class="normal"> 49</span>
+<span class="normal"> 50</span>
+<span class="normal"> 51</span>
+<span class="normal"> 52</span>
+<span class="normal"> 53</span>
+<span class="normal"> 54</span>
+<span class="normal"> 55</span>
+<span class="normal"> 56</span>
+<span class="normal"> 57</span>
+<span class="normal"> 58</span>
+<span class="normal"> 59</span>
+<span class="normal"> 60</span>
+<span class="normal"> 61</span>
+<span class="normal"> 62</span>
+<span class="normal"> 63</span>
+<span class="normal"> 64</span>
+<span class="normal"> 65</span>
+<span class="normal"> 66</span>
+<span class="normal"> 67</span>
+<span class="normal"> 68</span>
+<span class="normal"> 69</span>
+<span class="normal"> 70</span>
+<span class="normal"> 71</span>
+<span class="normal"> 72</span>
+<span class="normal"> 73</span>
+<span class="normal"> 74</span>
+<span class="normal"> 75</span>
+<span class="normal"> 76</span>
+<span class="normal"> 77</span>
+<span class="normal"> 78</span>
+<span class="normal"> 79</span>
+<span class="normal"> 80</span>
+<span class="normal"> 81</span>
+<span class="normal"> 82</span>
+<span class="normal"> 83</span>
+<span class="normal"> 84</span>
+<span class="normal"> 85</span>
+<span class="normal"> 86</span>
+<span class="normal"> 87</span>
+<span class="normal"> 88</span>
+<span class="normal"> 89</span>
+<span class="normal"> 90</span>
+<span class="normal"> 91</span>
+<span class="normal"> 92</span>
+<span class="normal"> 93</span>
+<span class="normal"> 94</span>
+<span class="normal"> 95</span>
+<span class="normal"> 96</span>
+<span class="normal"> 97</span>
+<span class="normal"> 98</span>
+<span class="normal"> 99</span>
+<span class="normal">100</span>
+<span class="normal">101</span>
+<span class="normal">102</span>
+<span class="normal">103</span>
+<span class="normal">104</span>
+<span class="normal">105</span>
+<span class="normal">106</span>
+<span class="normal">107</span>
+<span class="normal">108</span>
+<span class="normal">109</span>
+<span class="normal">110</span>
+<span class="normal">111</span>
+<span class="normal">112</span>
+<span class="normal">113</span>
+<span class="normal">114</span>
+<span class="normal">115</span>
+<span class="normal">116</span>
+<span class="normal">117</span>
+<span class="normal">118</span>
+<span class="normal">119</span>
+<span class="normal">120</span>
+<span class="normal">121</span>
+<span class="normal">122</span>
+<span class="normal">123</span>
+<span class="normal">124</span>
+<span class="normal">125</span>
+<span class="normal">126</span>
+<span class="normal">127</span>
+<span class="normal">128</span>
+<span class="normal">129</span>
+<span class="normal">130</span>
+<span class="normal">131</span>
+<span class="normal">132</span>
+<span class="normal">133</span>
+<span class="normal">134</span>
+<span class="normal">135</span>
+<span class="normal">136</span>
+<span class="normal">137</span>
+<span class="normal">138</span>
+<span class="normal">139</span>
+<span class="normal">140</span>
+<span class="normal">141</span>
+<span class="normal">142</span>
+<span class="normal">143</span>
+<span class="normal">144</span>
+<span class="normal">145</span>
+<span class="normal">146</span>
+<span class="normal">147</span>
+<span class="normal">148</span>
+<span class="normal">149</span>
+<span class="normal">150</span>
+<span class="normal">151</span>
+<span class="normal">152</span>
+<span class="normal">153</span>
+<span class="normal">154</span>
+<span class="normal">155</span>
+<span class="normal">156</span>
+<span class="normal">157</span>
+<span class="normal">158</span>
+<span class="normal">159</span>
+<span class="normal">160</span>
+<span class="normal">161</span>
+<span class="normal">162</span>
+<span class="normal">163</span>
+<span class="normal">164</span>
+<span class="normal">165</span>
+<span class="normal">166</span>
+<span class="normal">167</span>
+<span class="normal">168</span>
+<span class="normal">169</span>
+<span class="normal">170</span>
+<span class="normal">171</span>
+<span class="normal">172</span>
+<span class="normal">173</span>
+<span class="normal">174</span>
+<span class="normal">175</span>
+<span class="normal">176</span>
+<span class="normal">177</span>
+<span class="normal">178</span>
+<span class="normal">179</span>
+<span class="normal">180</span>
+<span class="normal">181</span>
+<span class="normal">182</span>
+<span class="normal">183</span>
+<span class="normal">184</span>
+<span class="normal">185</span>
+<span class="normal">186</span>
+<span class="normal">187</span>
+<span class="normal">188</span>
+<span class="normal">189</span>
+<span class="normal">190</span>
+<span class="normal">191</span>
+<span class="normal">192</span>
+<span class="normal">193</span>
+<span class="normal">194</span>
+<span class="normal">195</span>
+<span class="normal">196</span>
+<span class="normal">197</span>
+<span class="normal">198</span>
+<span class="normal">199</span>
+<span class="normal">200</span>
+<span class="normal">201</span>
+<span class="normal">202</span>
+<span class="normal">203</span>
+<span class="normal">204</span>
+<span class="normal">205</span>
+<span class="normal">206</span>
+<span class="normal">207</span>
+<span class="normal">208</span>
+<span class="normal">209</span>
+<span class="normal">210</span>
+<span class="normal">211</span>
+<span class="normal">212</span>
+<span class="normal">213</span>
+<span class="normal">214</span>
+<span class="normal">215</span>
+<span class="normal">216</span>
+<span class="normal">217</span>
+<span class="normal">218</span>
+<span class="normal">219</span>
+<span class="normal">220</span>
+<span class="normal">221</span>
+<span class="normal">222</span>
+<span class="normal">223</span>
+<span class="normal">224</span>
+<span class="normal">225</span>
+<span class="normal">226</span>
+<span class="normal">227</span>
+<span class="normal">228</span>
+<span class="normal">229</span>
+<span class="normal">230</span>
+<span class="normal">231</span>
+<span class="normal">232</span>
+<span class="normal">233</span>
+<span class="normal">234</span>
+<span class="normal">235</span>
+<span class="normal">236</span>
+<span class="normal">237</span>
+<span class="normal">238</span>
+<span class="normal">239</span>
+<span class="normal">240</span>
+<span class="normal">241</span>
+<span class="normal">242</span>
+<span class="normal">243</span>
+<span class="normal">244</span>
+<span class="normal">245</span>
+<span class="normal">246</span>
+<span class="normal">247</span>
+<span class="normal">248</span>
+<span class="normal">249</span>
+<span class="normal">250</span>
+<span class="normal">251</span>
+<span class="normal">252</span>
+<span class="normal">253</span>
+<span class="normal">254</span>
+<span class="normal">255</span>
+<span class="normal">256</span>
+<span class="normal">257</span>
+<span class="normal">258</span>
+<span class="normal">259</span>
+<span class="normal">260</span>
+<span class="normal">261</span>
+<span class="normal">262</span>
+<span class="normal">263</span>
+<span class="normal">264</span>
+<span class="normal">265</span>
+<span class="normal">266</span>
+<span class="normal">267</span>
+<span class="normal">268</span>
+<span class="normal">269</span>
+<span class="normal">270</span>
+<span class="normal">271</span>
+<span class="normal">272</span>
+<span class="normal">273</span>
+<span class="normal">274</span>
+<span class="normal">275</span>
+<span class="normal">276</span>
+<span class="normal">277</span>
+<span class="normal">278</span>
+<span class="normal">279</span>
+<span class="normal">280</span>
+<span class="normal">281</span>
+<span class="normal">282</span>
+<span class="normal">283</span>
+<span class="normal">284</span>
+<span class="normal">285</span>
+<span class="normal">286</span>
+<span class="normal">287</span>
+<span class="normal">288</span>
+<span class="normal">289</span>
+<span class="normal">290</span>
+<span class="normal">291</span>
+<span class="normal">292</span>
+<span class="normal">293</span>
+<span class="normal">294</span>
+<span class="normal">295</span>
+<span class="normal">296</span>
+<span class="normal">297</span>
+<span class="normal">298</span>
+<span class="normal">299</span>
+<span class="normal">300</span>
+<span class="normal">301</span>
+<span class="normal">302</span>
+<span class="normal">303</span>
+<span class="normal">304</span>
+<span class="normal">305</span>
+<span class="normal">306</span>
+<span class="normal">307</span>
+<span class="normal">308</span>
+<span class="normal">309</span>
+<span class="normal">310</span>
+<span class="normal">311</span>
+<span class="normal">312</span>
+<span class="normal">313</span>
+<span class="normal">314</span>
+<span class="normal">315</span>
+<span class="normal">316</span>
+<span class="normal">317</span>
+<span class="normal">318</span>
+<span class="normal">319</span>
+<span class="normal">320</span>
+<span class="normal">321</span>
+<span class="normal">322</span>
+<span class="normal">323</span>
+<span class="normal">324</span>
+<span class="normal">325</span>
+<span class="normal">326</span>
+<span class="normal">327</span>
+<span class="normal">328</span>
+<span class="normal">329</span>
+<span class="normal">330</span>
+<span class="normal">331</span>
+<span class="normal">332</span>
+<span class="normal">333</span>
+<span class="normal">334</span>
+<span class="normal">335</span>
+<span class="normal">336</span>
+<span class="normal">337</span>
+<span class="normal">338</span>
+<span class="normal">339</span>
+<span class="normal">340</span>
+<span class="normal">341</span>
+<span class="normal">342</span>
+<span class="normal">343</span>
+<span class="normal">344</span>
+<span class="normal">345</span>
+<span class="normal">346</span>
+<span class="normal">347</span>
+<span class="normal">348</span>
+<span class="normal">349</span>
+<span class="normal">350</span>
+<span class="normal">351</span>
+<span class="normal">352</span>
+<span class="normal">353</span>
+<span class="normal">354</span>
+<span class="normal">355</span>
+<span class="normal">356</span>
+<span class="normal">357</span>
+<span class="normal">358</span>
+<span class="normal">359</span>
+<span class="normal">360</span>
+<span class="normal">361</span>
+<span class="normal">362</span>
+<span class="normal">363</span>
+<span class="normal">364</span>
+<span class="normal">365</span>
+<span class="normal">366</span>
+<span class="normal">367</span>
+<span class="normal">368</span>
+<span class="normal">369</span>
+<span class="normal">370</span>
+<span class="normal">371</span>
+<span class="normal">372</span>
+<span class="normal">373</span>
+<span class="normal">374</span>
+<span class="normal">375</span>
+<span class="normal">376</span>
+<span class="normal">377</span>
+<span class="normal">378</span>
+<span class="normal">379</span>
+<span class="normal">380</span>
+<span class="normal">381</span>
+<span class="normal">382</span>
+<span class="normal">383</span>
+<span class="normal">384</span>
+<span class="normal">385</span>
+<span class="normal">386</span>
+<span class="normal">387</span>
+<span class="normal">388</span>
+<span class="normal">389</span>
+<span class="normal">390</span>
+<span class="normal">391</span>
+<span class="normal">392</span>
+<span class="normal">393</span>
+<span class="normal">394</span>
+<span class="normal">395</span>
+<span class="normal">396</span>
+<span class="normal">397</span>
+<span class="normal">398</span>
+<span class="normal">399</span>
+<span class="normal">400</span>
+<span class="normal">401</span>
+<span class="normal">402</span>
+<span class="normal">403</span>
+<span class="normal">404</span>
+<span class="normal">405</span>
+<span class="normal">406</span>
+<span class="normal">407</span>
+<span class="normal">408</span>
+<span class="normal">409</span>
+<span class="normal">410</span>
+<span class="normal">411</span>
+<span class="normal">412</span>
+<span class="normal">413</span>
+<span class="normal">414</span>
+<span class="normal">415</span>
+<span class="normal">416</span>
+<span class="normal">417</span>
+<span class="normal">418</span>
+<span class="normal">419</span>
+<span class="normal">420</span>
+<span class="normal">421</span>
+<span class="normal">422</span>
+<span class="normal">423</span>
+<span class="normal">424</span>
+<span class="normal">425</span>
+<span class="normal">426</span>
+<span class="normal">427</span>
+<span class="normal">428</span>
+<span class="normal">429</span>
+<span class="normal">430</span>
+<span class="normal">431</span>
+<span class="normal">432</span>
+<span class="normal">433</span>
+<span class="normal">434</span>
+<span class="normal">435</span>
+<span class="normal">436</span>
+<span class="normal">437</span>
+<span class="normal">438</span>
+<span class="normal">439</span>
+<span class="normal">440</span>
+<span class="normal">441</span>
+<span class="normal">442</span>
+<span class="normal">443</span>
+<span class="normal">444</span>
+<span class="normal">445</span>
+<span class="normal">446</span>
+<span class="normal">447</span>
+<span class="normal">448</span>
+<span class="normal">449</span>
+<span class="normal">450</span>
+<span class="normal">451</span>
+<span class="normal">452</span>
+<span class="normal">453</span>
+<span class="normal">454</span>
+<span class="normal">455</span>
+<span class="normal">456</span>
+<span class="normal">457</span>
+<span class="normal">458</span>
+<span class="normal">459</span>
+<span class="normal">460</span>
+<span class="normal">461</span>
+<span class="normal">462</span>
+<span class="normal">463</span>
+<span class="normal">464</span>
+<span class="normal">465</span>
+<span class="normal">466</span>
+<span class="normal">467</span>
+<span class="normal">468</span>
+<span class="normal">469</span>
+<span class="normal">470</span>
+<span class="normal">471</span>
+<span class="normal">472</span>
+<span class="normal">473</span>
+<span class="normal">474</span>
+<span class="normal">475</span>
+<span class="normal">476</span>
+<span class="normal">477</span>
+<span class="normal">478</span>
+<span class="normal">479</span>
+<span class="normal">480</span>
+<span class="normal">481</span>
+<span class="normal">482</span>
+<span class="normal">483</span>
+<span class="normal">484</span>
+<span class="normal">485</span>
+<span class="normal">486</span>
+<span class="normal">487</span>
+<span class="normal">488</span>
+<span class="normal">489</span>
+<span class="normal">490</span>
+<span class="normal">491</span>
+<span class="normal">492</span>
+<span class="normal">493</span>
+<span class="normal">494</span>
+<span class="normal">495</span>
+<span class="normal">496</span>
+<span class="normal">497</span>
+<span class="normal">498</span>
+<span class="normal">499</span>
+<span class="normal">500</span>
+<span class="normal">501</span>
+<span class="normal">502</span>
+<span class="normal">503</span>
+<span class="normal">504</span>
+<span class="normal">505</span>
+<span class="normal">506</span>
+<span class="normal">507</span>
+<span class="normal">508</span>
+<span class="normal">509</span>
+<span class="normal">510</span>
+<span class="normal">511</span>
+<span class="normal">512</span>
+<span class="normal">513</span>
+<span class="normal">514</span>
+<span class="normal">515</span>
+<span class="normal">516</span>
+<span class="normal">517</span>
+<span class="normal">518</span>
+<span class="normal">519</span>
+<span class="normal">520</span>
+<span class="normal">521</span>
+<span class="normal">522</span>
+<span class="normal">523</span>
+<span class="normal">524</span>
+<span class="normal">525</span>
+<span class="normal">526</span>
+<span class="normal">527</span>
+<span class="normal">528</span>
+<span class="normal">529</span>
+<span class="normal">530</span>
+<span class="normal">531</span>
+<span class="normal">532</span>
+<span class="normal">533</span>
+<span class="normal">534</span>
+<span class="normal">535</span>
+<span class="normal">536</span>
+<span class="normal">537</span>
+<span class="normal">538</span>
+<span class="normal">539</span>
+<span class="normal">540</span>
+<span class="normal">541</span>
+<span class="normal">542</span>
+<span class="normal">543</span>
+<span class="normal">544</span>
+<span class="normal">545</span>
+<span class="normal">546</span>
+<span class="normal">547</span>
+<span class="normal">548</span>
+<span class="normal">549</span>
+<span class="normal">550</span>
+<span class="normal">551</span>
+<span class="normal">552</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">class</span> <span class="nc">DINPreprocessor</span><span class="p">(</span><span class="n">BasePreprocessor</span><span class="p">):</span>
+<span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">    Preprocessor for Deep Interest Network (DIN) models.</span>
+
+<span class="sd">    This preprocessor handles the preparation of data for DIN models,</span>
+<span class="sd">    including sequence building, label encoding, and handling of various</span>
+<span class="sd">    types of input columns (categorical, continuous, and sequential).</span>
+
+<span class="sd">    Parameters</span>
+<span class="sd">    ----------</span>
+<span class="sd">    user_id_col : str</span>
+<span class="sd">        Name of the column containing user IDs.</span>
+<span class="sd">    item_embed_col : Union[str, Tuple[str, int]]</span>
+<span class="sd">        Name of the column containing item IDs to be embedded, or a tuple of</span>
+<span class="sd">        (column_name, embedding_dim). If embedding_dim is not provided, it</span>
+<span class="sd">        will be automatically calculated using a rule of thumb specified by</span>
+<span class="sd">        `embedding_rule`. Aliased as `item_id_col`.</span>
+<span class="sd">    target_col : str</span>
+<span class="sd">        Name of the column containing the target variable.</span>
+<span class="sd">    max_seq_length : int</span>
+<span class="sd">        Maximum length of sequences to be created.</span>
+<span class="sd">    action_col : Optional[str], default=None</span>
+<span class="sd">        Name of the column containing user actions (if applicable), for</span>
+<span class="sd">        example a rating, or purchased/not-purchased, etc). This &#39;action_col&#39;</span>
+<span class="sd">        can also be the same as the &#39;target_col&#39;.</span>
+<span class="sd">    other_seq_embed_cols : Optional[List[str] | List[Tuple[str, int]]], default=None</span>
+<span class="sd">        List of other columns to be treated as sequences. Each element in the</span>
+<span class="sd">        list can be a string with the name of the column or a tuple with the</span>
+<span class="sd">        name of the column and the embedding dimension.</span>
+
+<span class="sd">    cat_embed_cols : Optional[Union[List[str], List[Tuple[str, int]]]], default=None</span>
+<span class="sd">        This and the following parameters are identical to those used in the</span>
+<span class="sd">        TabPreprocessor.&lt;br&gt;</span>
+<span class="sd">        List containing the name of the categorical columns that will be</span>
+<span class="sd">        represented by embeddings (e.g. _[&#39;education&#39;, &#39;relationship&#39;, ...]_) or</span>
+<span class="sd">        a Tuple with the name and the embedding dimension (e.g.: _[</span>
+<span class="sd">        (&#39;education&#39;,32), (&#39;relationship&#39;,16), ...]_)</span>
+<span class="sd">    continuous_cols: List, default = None</span>
+<span class="sd">        List with the name of the continuous cols</span>
+<span class="sd">    quantization_setup: int or Dict, default = None</span>
+<span class="sd">        Continuous columns can be turned into categorical via `pd.cut`. If</span>
+<span class="sd">        `quantization_setup` is an `int`, all continuous columns will be</span>
+<span class="sd">        quantized using this value as the number of bins. Alternatively, a</span>
+<span class="sd">        dictionary where the keys are the column names to quantize and the</span>
+<span class="sd">        values are the either integers indicating the number of bins or a</span>
+<span class="sd">        list of scalars indicating the bin edges can also be used.</span>
+<span class="sd">    cols_to_scale: List or str, default = None,</span>
+<span class="sd">        List with the names of the columns that will be standarised via</span>
+<span class="sd">        sklearn&#39;s `StandardScaler`. It can also be the string `&#39;all&#39;` in</span>
+<span class="sd">        which case all the continuous cols will be scaled.</span>
+<span class="sd">    auto_embed_dim: bool, default = True</span>
+<span class="sd">        Boolean indicating whether the embedding dimensions will be</span>
+<span class="sd">        automatically defined via rule of thumb. See `embedding_rule`</span>
+<span class="sd">        below.</span>
+<span class="sd">    embedding_rule: str, default = &#39;fastai_new&#39;</span>
+<span class="sd">        If `auto_embed_dim=True`, this is the choice of embedding rule of</span>
+<span class="sd">        thumb. Choices are:</span>
+
+<span class="sd">        - _fastai_new_: $min(600, round(1.6 \times n_{cat}^{0.56}))$</span>
+
+<span class="sd">        - _fastai_old_: $min(50, (n_{cat}//{2})+1)$</span>
+
+<span class="sd">        - _google_: $min(600, round(n_{cat}^{0.24}))$</span>
+<span class="sd">    default_embed_dim: int, default=16</span>
+<span class="sd">        Dimension for the embeddings if the embedding dimension is not</span>
+<span class="sd">        provided in the `cat_embed_cols` parameter and `auto_embed_dim` is</span>
+<span class="sd">        set to `False`.</span>
+<span class="sd">    verbose : int, default=1</span>
+<span class="sd">        Verbosity level.</span>
+<span class="sd">    scale: bool, default = False</span>
+<span class="sd">        :information_source: **note**: this arg will be removed in upcoming</span>
+<span class="sd">         releases. Please use `cols_to_scale` instead. &lt;br/&gt; Bool indicating</span>
+<span class="sd">         whether or not to scale/standarise continuous cols. It is important</span>
+<span class="sd">         to emphasize that all the DL models for tabular data in the library</span>
+<span class="sd">         also include the possibility of normalising the input continuous</span>
+<span class="sd">         features via a `BatchNorm` or a `LayerNorm`. &lt;br/&gt; Param alias:</span>
+<span class="sd">         `scale_cont_cols`.</span>
+<span class="sd">    already_standard: List, default = None</span>
+<span class="sd">        :information_source: **note**: this arg will be removed in upcoming</span>
+<span class="sd">         releases. Please use `cols_to_scale` instead. &lt;br/&gt; List with the</span>
+<span class="sd">         name of the continuous cols that do not need to be</span>
+<span class="sd">         scaled/standarised.</span>
+
+<span class="sd">    Other Parameters</span>
+<span class="sd">    ----------------</span>
+<span class="sd">    **kwargs: dict</span>
+<span class="sd">        `pd.cut` and `StandardScaler` related args</span>
+
+<span class="sd">    Attributes</span>
+<span class="sd">    ----------</span>
+<span class="sd">    is_fitted : bool</span>
+<span class="sd">        Whether the preprocessor has been fitted.</span>
+<span class="sd">    has_standard_tab_data : bool</span>
+<span class="sd">        Whether the data includes standard tabular data.</span>
+<span class="sd">    tab_preprocessor : TabPreprocessor</span>
+<span class="sd">        Preprocessor for standard tabular data (if applicable).</span>
+<span class="sd">    din_columns_idx : Dict[str, int]</span>
+<span class="sd">        Dictionary mapping column names to their indices in the processed data.</span>
+<span class="sd">    item_le : LabelEncoder</span>
+<span class="sd">        Label encoder for item IDs.</span>
+<span class="sd">    n_items : int</span>
+<span class="sd">        Number of unique items.</span>
+<span class="sd">    user_behaviour_config : Tuple[List[str], int, int]</span>
+<span class="sd">        Configuration for user behavior sequences.</span>
+<span class="sd">    action_le : LabelEncoder</span>
+<span class="sd">        Label encoder for action column (if applicable).</span>
+<span class="sd">    n_actions : int</span>
+<span class="sd">        Number of unique actions (if applicable).</span>
+<span class="sd">    action_seq_config : Tuple[List[str], int]</span>
+<span class="sd">        Configuration for action sequences (if applicable).</span>
+<span class="sd">    other_seq_le : LabelEncoder</span>
+<span class="sd">        Label encoder for other sequence columns (if applicable).</span>
+<span class="sd">    n_other_seq_cols : Dict[str, int]</span>
+<span class="sd">        Number of unique values in each other sequence column.</span>
+<span class="sd">    other_seq_config : List[Tuple[List[str], int, int]]</span>
+<span class="sd">        Configuration for other sequence columns.</span>
+
+<span class="sd">    Examples</span>
+<span class="sd">    --------</span>
+<span class="sd">    &gt;&gt;&gt; import pandas as pd</span>
+<span class="sd">    &gt;&gt;&gt; from pytorch_widedeep.preprocessing import DINPreprocessor</span>
+<span class="sd">    &gt;&gt;&gt; data = {</span>
+<span class="sd">    ...    &#39;user_id&#39;: [1, 1, 1, 2, 2, 2, 3, 3, 3],</span>
+<span class="sd">    ...    &#39;item_id&#39;: [101, 102, 103, 101, 103, 104, 102, 103, 104],</span>
+<span class="sd">    ...    &#39;timestamp&#39;: [1, 2, 3, 1, 2, 3, 1, 2, 3],</span>
+<span class="sd">    ...    &#39;category&#39;: [&#39;A&#39;, &#39;B&#39;, &#39;A&#39;, &#39;B&#39;, &#39;A&#39;, &#39;C&#39;, &#39;B&#39;, &#39;A&#39;, &#39;C&#39;],</span>
+<span class="sd">    ...    &#39;price&#39;: [10.5, 15.0, 12.0, 10.5, 12.0, 20.0, 15.0, 12.0, 20.0],</span>
+<span class="sd">    ...    &#39;rating&#39;: [0, 1, 0, 1, 0, 1, 0, 1, 0]</span>
+<span class="sd">    ... }</span>
+<span class="sd">    &gt;&gt;&gt; df = pd.DataFrame(data)</span>
+<span class="sd">    &gt;&gt;&gt; din_preprocessor = DINPreprocessor(</span>
+<span class="sd">    ...     user_id_col=&#39;user_id&#39;,</span>
+<span class="sd">    ...     item_embed_col=&#39;item_id&#39;,</span>
+<span class="sd">    ...     target_col=&#39;rating&#39;,</span>
+<span class="sd">    ...     max_seq_length=2,</span>
+<span class="sd">    ...     action_col=&#39;rating&#39;,</span>
+<span class="sd">    ...     other_seq_embed_cols=[&#39;category&#39;],</span>
+<span class="sd">    ...     cat_embed_cols=[&#39;user_id&#39;],</span>
+<span class="sd">    ...     continuous_cols=[&#39;price&#39;],</span>
+<span class="sd">    ...     cols_to_scale=[&#39;price&#39;]</span>
+<span class="sd">    ... )</span>
+<span class="sd">    &gt;&gt;&gt; X, y = din_preprocessor.fit_transform(df)</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+
+    <span class="nd">@alias</span><span class="p">(</span><span class="s2">&quot;item_embed_col&quot;</span><span class="p">,</span> <span class="p">[</span><span class="s2">&quot;item_id_col&quot;</span><span class="p">])</span>
+    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span>
+        <span class="o">*</span><span class="p">,</span>
+        <span class="n">user_id_col</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
+        <span class="n">item_embed_col</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Tuple</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">]],</span>
+        <span class="n">target_col</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
+        <span class="n">max_seq_length</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span>
+        <span class="n">action_col</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">other_seq_embed_cols</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Union</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">],</span> <span class="n">List</span><span class="p">[</span><span class="n">Tuple</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">]]]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">cat_embed_cols</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Union</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">],</span> <span class="n">List</span><span class="p">[</span><span class="n">Tuple</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">]]]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">continuous_cols</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">quantization_setup</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span>
+            <span class="n">Union</span><span class="p">[</span><span class="nb">int</span><span class="p">,</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Union</span><span class="p">[</span><span class="nb">int</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="nb">float</span><span class="p">]]]]</span>
+        <span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">cols_to_scale</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Union</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">],</span> <span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">auto_embed_dim</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">,</span>
+        <span class="n">embedding_rule</span><span class="p">:</span> <span class="n">Literal</span><span class="p">[</span><span class="s2">&quot;google&quot;</span><span class="p">,</span> <span class="s2">&quot;fastai_old&quot;</span><span class="p">,</span> <span class="s2">&quot;fastai_new&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="s2">&quot;fastai_new&quot;</span><span class="p">,</span>
+        <span class="n">default_embed_dim</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">16</span><span class="p">,</span>
+        <span class="n">verbose</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1</span><span class="p">,</span>
+        <span class="n">scale</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
+        <span class="n">already_standard</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="o">**</span><span class="n">kwargs</span><span class="p">,</span>
+    <span class="p">):</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">user_id_col</span> <span class="o">=</span> <span class="n">user_id_col</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">item_embed_col</span> <span class="o">=</span> <span class="n">item_embed_col</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">max_seq_length</span> <span class="o">=</span> <span class="n">max_seq_length</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">target_col</span> <span class="o">=</span> <span class="n">target_col</span> <span class="k">if</span> <span class="n">target_col</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="k">else</span> <span class="s2">&quot;target&quot;</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">action_col</span> <span class="o">=</span> <span class="n">action_col</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">other_seq_embed_cols</span> <span class="o">=</span> <span class="n">other_seq_embed_cols</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">cat_embed_cols</span> <span class="o">=</span> <span class="n">cat_embed_cols</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">continuous_cols</span> <span class="o">=</span> <span class="n">continuous_cols</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">quantization_setup</span> <span class="o">=</span> <span class="n">quantization_setup</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">cols_to_scale</span> <span class="o">=</span> <span class="n">cols_to_scale</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">auto_embed_dim</span> <span class="o">=</span> <span class="n">auto_embed_dim</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">embedding_rule</span> <span class="o">=</span> <span class="n">embedding_rule</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">default_embed_dim</span> <span class="o">=</span> <span class="n">default_embed_dim</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">verbose</span> <span class="o">=</span> <span class="n">verbose</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">scale</span> <span class="o">=</span> <span class="n">scale</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">already_standard</span> <span class="o">=</span> <span class="n">already_standard</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">kwargs</span> <span class="o">=</span> <span class="n">kwargs</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">has_standard_tab_data</span> <span class="o">=</span> <span class="nb">bool</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">cat_embed_cols</span> <span class="ow">or</span> <span class="bp">self</span><span class="o">.</span><span class="n">continuous_cols</span><span class="p">)</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">has_standard_tab_data</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">tab_preprocessor</span> <span class="o">=</span> <span class="n">TabPreprocessor</span><span class="p">(</span>
+                <span class="n">cat_embed_cols</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">cat_embed_cols</span><span class="p">,</span>
+                <span class="n">continuous_cols</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">continuous_cols</span><span class="p">,</span>
+                <span class="n">quantization_setup</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">quantization_setup</span><span class="p">,</span>
+                <span class="n">cols_to_scale</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">cols_to_scale</span><span class="p">,</span>
+                <span class="n">auto_embed_dim</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">auto_embed_dim</span><span class="p">,</span>
+                <span class="n">embedding_rule</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">embedding_rule</span><span class="p">,</span>
+                <span class="n">default_embed_dim</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">default_embed_dim</span><span class="p">,</span>
+                <span class="n">verbose</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">verbose</span><span class="p">,</span>
+                <span class="n">scale</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">scale</span><span class="p">,</span>
+                <span class="n">already_standard</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">already_standard</span><span class="p">,</span>
+                <span class="o">**</span><span class="bp">self</span><span class="o">.</span><span class="n">kwargs</span><span class="p">,</span>
+            <span class="p">)</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">is_fitted</span> <span class="o">=</span> <span class="kc">False</span>
+
+    <span class="k">def</span> <span class="nf">fit</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="s2">&quot;DINPreprocessor&quot;</span><span class="p">:</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">has_standard_tab_data</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">tab_preprocessor</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">din_columns_idx</span> <span class="o">=</span> <span class="p">{</span>
+                <span class="n">col</span><span class="p">:</span> <span class="n">i</span> <span class="k">for</span> <span class="n">i</span><span class="p">,</span> <span class="n">col</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">tab_preprocessor</span><span class="o">.</span><span class="n">column_idx</span><span class="o">.</span><span class="n">keys</span><span class="p">())</span>
+            <span class="p">}</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">din_columns_idx</span> <span class="o">=</span> <span class="p">{}</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">_fit_label_encoders</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">is_fitted</span> <span class="o">=</span> <span class="kc">True</span>
+        <span class="k">return</span> <span class="bp">self</span>
+
+    <span class="k">def</span> <span class="nf">transform</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tuple</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">,</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">]:</span>
+        <span class="n">_df</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_pre_transform</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
+        <span class="n">df_w_sequences</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_build_sequences</span><span class="p">(</span><span class="n">_df</span><span class="p">)</span>
+        <span class="n">X_all</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_concatenate_features</span><span class="p">(</span><span class="n">df_w_sequences</span><span class="p">)</span>
+        <span class="k">return</span> <span class="n">X_all</span><span class="p">,</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">(</span><span class="n">df_w_sequences</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">target_col</span><span class="p">]</span><span class="o">.</span><span class="n">tolist</span><span class="p">())</span>
+
+    <span class="k">def</span> <span class="nf">fit_transform</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tuple</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">,</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">]:</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
+        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">transform</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">inverse_transform</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">X</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">:</span>
+        <span class="k">raise</span> <span class="ne">NotImplementedError</span><span class="p">(</span>
+            <span class="s2">&quot;inverse_transform is not implemented for this preprocessor&quot;</span>
+        <span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">_fit_label_encoders</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="kc">None</span><span class="p">:</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">item_le</span> <span class="o">=</span> <span class="n">LabelEncoder</span><span class="p">(</span><span class="n">columns_to_encode</span><span class="o">=</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">_get_item_col</span><span class="p">()])</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">item_le</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">n_items</span> <span class="o">=</span> <span class="nb">max</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">item_le</span><span class="o">.</span><span class="n">encoding_dict</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">_get_item_col</span><span class="p">()]</span><span class="o">.</span><span class="n">values</span><span class="p">())</span>
+        <span class="n">user_behaviour_embed_size</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_embed_size</span><span class="p">(</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">item_embed_col</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">n_items</span>
+        <span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">user_behaviour_config</span> <span class="o">=</span> <span class="p">(</span>
+            <span class="p">[</span><span class="sa">f</span><span class="s2">&quot;item_</span><span class="si">{</span><span class="n">i</span><span class="o">+</span><span class="mi">1</span><span class="si">}</span><span class="s2">&quot;</span> <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">max_seq_length</span><span class="p">)],</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">n_items</span><span class="p">,</span>
+            <span class="n">user_behaviour_embed_size</span><span class="p">,</span>
+        <span class="p">)</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">_update_din_columns_idx</span><span class="p">(</span><span class="s2">&quot;item&quot;</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">max_seq_length</span><span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">din_columns_idx</span><span class="p">[</span><span class="s2">&quot;target_item&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">din_columns_idx</span><span class="p">)</span>
+
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">action_col</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">_fit_action_label_encoder</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">other_seq_embed_cols</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">_fit_other_seq_label_encoders</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">_fit_action_label_encoder</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="kc">None</span><span class="p">:</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">action_le</span> <span class="o">=</span> <span class="n">LabelEncoder</span><span class="p">(</span><span class="n">columns_to_encode</span><span class="o">=</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">action_col</span><span class="p">])</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">action_le</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">n_actions</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">action_le</span><span class="o">.</span><span class="n">encoding_dict</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">action_col</span><span class="p">])</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">action_seq_config</span> <span class="o">=</span> <span class="p">(</span>
+            <span class="p">[</span><span class="sa">f</span><span class="s2">&quot;action_</span><span class="si">{</span><span class="n">i</span><span class="o">+</span><span class="mi">1</span><span class="si">}</span><span class="s2">&quot;</span> <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">max_seq_length</span><span class="p">)],</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">n_actions</span><span class="p">,</span>
+        <span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">_update_din_columns_idx</span><span class="p">(</span><span class="s2">&quot;action&quot;</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">max_seq_length</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">_other_seq_cols_float_warning</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="kc">None</span><span class="p">:</span>
+        <span class="n">other_seq_cols</span> <span class="o">=</span> <span class="p">[</span>
+            <span class="n">col</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">col</span><span class="p">,</span> <span class="nb">tuple</span><span class="p">)</span> <span class="k">else</span> <span class="n">col</span>
+            <span class="k">for</span> <span class="n">col</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">other_seq_embed_cols</span>
+        <span class="p">]</span>
+        <span class="k">for</span> <span class="n">col</span> <span class="ow">in</span> <span class="n">other_seq_cols</span><span class="p">:</span>
+            <span class="k">if</span> <span class="n">df</span><span class="p">[</span><span class="n">col</span><span class="p">]</span><span class="o">.</span><span class="n">dtype</span> <span class="o">==</span> <span class="nb">float</span><span class="p">:</span>
+                <span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span>
+                    <span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="n">col</span><span class="si">}</span><span class="s2"> is a float column. It will be converted to integers. &quot;</span>
+                    <span class="s2">&quot;If this is not what you want, please convert it beforehand.&quot;</span><span class="p">,</span>
+                    <span class="ne">UserWarning</span><span class="p">,</span>
+                <span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">_convert_other_seq_cols_to_int</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="kc">None</span><span class="p">:</span>
+        <span class="n">other_seq_cols</span> <span class="o">=</span> <span class="p">[</span>
+            <span class="n">col</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">col</span><span class="p">,</span> <span class="nb">tuple</span><span class="p">)</span> <span class="k">else</span> <span class="n">col</span>
+            <span class="k">for</span> <span class="n">col</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">other_seq_embed_cols</span>
+        <span class="p">]</span>
+        <span class="k">for</span> <span class="n">col</span> <span class="ow">in</span> <span class="n">other_seq_cols</span><span class="p">:</span>
+            <span class="k">if</span> <span class="n">df</span><span class="p">[</span><span class="n">col</span><span class="p">]</span><span class="o">.</span><span class="n">dtype</span> <span class="o">==</span> <span class="nb">float</span><span class="p">:</span>
+                <span class="n">df</span><span class="p">[</span><span class="n">col</span><span class="p">]</span> <span class="o">=</span> <span class="n">df</span><span class="p">[</span><span class="n">col</span><span class="p">]</span><span class="o">.</span><span class="n">astype</span><span class="p">(</span><span class="nb">int</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">_fit_other_seq_label_encoders</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="kc">None</span><span class="p">:</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">_other_seq_cols_float_warning</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">_convert_other_seq_cols_to_int</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">other_seq_le</span> <span class="o">=</span> <span class="n">LabelEncoder</span><span class="p">(</span>
+            <span class="n">columns_to_encode</span><span class="o">=</span><span class="p">[</span>
+                <span class="n">col</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">col</span><span class="p">,</span> <span class="nb">tuple</span><span class="p">)</span> <span class="k">else</span> <span class="n">col</span>
+                <span class="k">for</span> <span class="n">col</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">other_seq_embed_cols</span>
+            <span class="p">]</span>
+        <span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">other_seq_le</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
+        <span class="n">other_seq_cols</span> <span class="o">=</span> <span class="p">[</span>
+            <span class="n">col</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">col</span><span class="p">,</span> <span class="nb">tuple</span><span class="p">)</span> <span class="k">else</span> <span class="n">col</span>
+            <span class="k">for</span> <span class="n">col</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">other_seq_embed_cols</span>
+        <span class="p">]</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">n_other_seq_cols</span> <span class="o">=</span> <span class="p">{</span>
+            <span class="n">col</span><span class="p">:</span> <span class="nb">max</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">other_seq_le</span><span class="o">.</span><span class="n">encoding_dict</span><span class="p">[</span><span class="n">col</span><span class="p">]</span><span class="o">.</span><span class="n">values</span><span class="p">())</span>
+            <span class="k">for</span> <span class="n">col</span> <span class="ow">in</span> <span class="n">other_seq_cols</span>
+        <span class="p">}</span>
+        <span class="n">other_seq_embed_sizes</span> <span class="o">=</span> <span class="p">{</span>
+            <span class="n">col</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_embed_size</span><span class="p">(</span><span class="n">col</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">n_other_seq_cols</span><span class="p">[</span><span class="n">col</span><span class="p">])</span>
+            <span class="k">for</span> <span class="n">col</span> <span class="ow">in</span> <span class="n">other_seq_cols</span>
+        <span class="p">}</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">other_seq_config</span> <span class="o">=</span> <span class="p">[</span>
+            <span class="p">(</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">_get_seq_col_names</span><span class="p">(</span><span class="n">col</span><span class="p">),</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">n_other_seq_cols</span><span class="p">[</span><span class="n">col</span><span class="p">],</span>
+                <span class="n">other_seq_embed_sizes</span><span class="p">[</span><span class="n">col</span><span class="p">],</span>
+            <span class="p">)</span>
+            <span class="k">for</span> <span class="n">col</span> <span class="ow">in</span> <span class="n">other_seq_cols</span>
+        <span class="p">]</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">_update_din_columns_idx_for_other_seq</span><span class="p">(</span><span class="n">other_seq_cols</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">_get_item_col</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">str</span><span class="p">:</span>
+        <span class="k">return</span> <span class="p">(</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">item_embed_col</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
+            <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">item_embed_col</span><span class="p">,</span> <span class="nb">tuple</span><span class="p">)</span>
+            <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">item_embed_col</span>
+        <span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">_get_embed_size</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">col</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Tuple</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">]],</span> <span class="n">n_unique</span><span class="p">:</span> <span class="nb">int</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">int</span><span class="p">:</span>
+        <span class="k">return</span> <span class="n">col</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span> <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">col</span><span class="p">,</span> <span class="nb">tuple</span><span class="p">)</span> <span class="k">else</span> <span class="n">embed_sz_rule</span><span class="p">(</span><span class="n">n_unique</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">_get_seq_col_names</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">col</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]:</span>
+        <span class="k">return</span> <span class="p">[</span><span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="n">col</span><span class="si">}</span><span class="s2">_</span><span class="si">{</span><span class="n">i</span><span class="o">+</span><span class="mi">1</span><span class="si">}</span><span class="s2">&quot;</span> <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">max_seq_length</span><span class="p">)]</span>
+
+    <span class="k">def</span> <span class="nf">_update_din_columns_idx</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">prefix</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">length</span><span class="p">:</span> <span class="nb">int</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="kc">None</span><span class="p">:</span>
+        <span class="n">current_len</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">din_columns_idx</span><span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">din_columns_idx</span><span class="o">.</span><span class="n">update</span><span class="p">(</span>
+            <span class="p">{</span><span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="n">prefix</span><span class="si">}</span><span class="s2">_</span><span class="si">{</span><span class="n">i</span><span class="o">+</span><span class="mi">1</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">:</span> <span class="n">i</span> <span class="o">+</span> <span class="n">current_len</span> <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">length</span><span class="p">)}</span>
+        <span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">_update_din_columns_idx_for_other_seq</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">other_seq_cols</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">])</span> <span class="o">-&gt;</span> <span class="kc">None</span><span class="p">:</span>
+        <span class="n">current_len</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">din_columns_idx</span><span class="p">)</span>
+        <span class="k">for</span> <span class="n">col</span> <span class="ow">in</span> <span class="n">other_seq_cols</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">din_columns_idx</span><span class="o">.</span><span class="n">update</span><span class="p">(</span>
+                <span class="p">{</span><span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="n">col</span><span class="si">}</span><span class="s2">_</span><span class="si">{</span><span class="n">i</span><span class="o">+</span><span class="mi">1</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">:</span> <span class="n">i</span> <span class="o">+</span> <span class="n">current_len</span> <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">max_seq_length</span><span class="p">)}</span>
+            <span class="p">)</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">din_columns_idx</span><span class="p">[</span><span class="sa">f</span><span class="s2">&quot;target_</span><span class="si">{</span><span class="n">col</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">din_columns_idx</span><span class="p">)</span>
+            <span class="n">current_len</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">din_columns_idx</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">_pre_transform</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">:</span>
+        <span class="n">check_is_fitted</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">attributes</span><span class="o">=</span><span class="p">[</span><span class="s2">&quot;din_columns_idx&quot;</span><span class="p">])</span>
+        <span class="n">df</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">item_le</span><span class="o">.</span><span class="n">transform</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">action_col</span><span class="p">:</span>
+            <span class="n">df</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">action_le</span><span class="o">.</span><span class="n">transform</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">other_seq_embed_cols</span><span class="p">:</span>
+            <span class="n">df</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">other_seq_le</span><span class="o">.</span><span class="n">transform</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
+        <span class="k">return</span> <span class="n">df</span>
+
+    <span class="k">def</span> <span class="nf">_build_sequences</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">:</span>
+        <span class="c1"># df.columns.tolist() -&gt; to avoid annoying pandas warning</span>
+        <span class="k">return</span> <span class="p">(</span>
+            <span class="n">df</span><span class="o">.</span><span class="n">groupby</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">user_id_col</span><span class="p">)[</span><span class="n">df</span><span class="o">.</span><span class="n">columns</span><span class="o">.</span><span class="n">tolist</span><span class="p">()]</span>  <span class="c1"># type: ignore[index]</span>
+            <span class="o">.</span><span class="n">apply</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_group_sequences</span><span class="p">)</span>
+            <span class="o">.</span><span class="n">reset_index</span><span class="p">(</span><span class="n">drop</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
+        <span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">_group_sequences</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">group</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">:</span>
+        <span class="n">item_col</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_item_col</span><span class="p">()</span>
+        <span class="n">items</span> <span class="o">=</span> <span class="n">group</span><span class="p">[</span><span class="n">item_col</span><span class="p">]</span><span class="o">.</span><span class="n">tolist</span><span class="p">()</span>
+        <span class="n">targets</span> <span class="o">=</span> <span class="n">group</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">target_col</span><span class="p">]</span><span class="o">.</span><span class="n">tolist</span><span class="p">()</span>
+        <span class="n">drop_cols</span> <span class="o">=</span> <span class="p">[</span><span class="n">item_col</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">target_col</span><span class="p">]</span>
+
+        <span class="c1"># sequences cannot be built with user, item pairs with only one</span>
+        <span class="c1"># interaction</span>
+        <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">items</span><span class="p">)</span> <span class="o">&lt;=</span> <span class="mi">1</span><span class="p">:</span>
+            <span class="k">return</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">()</span>
+
+        <span class="n">sequences</span> <span class="o">=</span> <span class="p">[</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">_create_sequence</span><span class="p">(</span><span class="n">group</span><span class="p">,</span> <span class="n">items</span><span class="p">,</span> <span class="n">targets</span><span class="p">,</span> <span class="n">i</span><span class="p">,</span> <span class="n">drop_cols</span><span class="p">)</span>
+            <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="nb">max</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="nb">len</span><span class="p">(</span><span class="n">items</span><span class="p">)</span> <span class="o">-</span> <span class="bp">self</span><span class="o">.</span><span class="n">max_seq_length</span><span class="p">))</span>
+        <span class="p">]</span>
+
+        <span class="n">seq_df</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">(</span><span class="n">sequences</span><span class="p">)</span>
+        <span class="n">non_seq_cols</span> <span class="o">=</span> <span class="n">group</span><span class="o">.</span><span class="n">drop_duplicates</span><span class="p">([</span><span class="bp">self</span><span class="o">.</span><span class="n">user_id_col</span><span class="p">])</span><span class="o">.</span><span class="n">drop</span><span class="p">(</span><span class="n">drop_cols</span><span class="p">,</span> <span class="n">axis</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span>
+        <span class="k">return</span> <span class="n">pd</span><span class="o">.</span><span class="n">merge</span><span class="p">(</span><span class="n">seq_df</span><span class="p">,</span> <span class="n">non_seq_cols</span><span class="p">,</span> <span class="n">on</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">user_id_col</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">_create_sequence</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span>
+        <span class="n">group</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">,</span>
+        <span class="n">items</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="nb">int</span><span class="p">],</span>
+        <span class="n">targets</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="nb">int</span><span class="p">],</span>
+        <span class="n">i</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span>
+        <span class="n">drop_cols</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">],</span>
+    <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Union</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="nb">int</span><span class="p">]]]:</span>
+        <span class="n">end_idx</span> <span class="o">=</span> <span class="nb">min</span><span class="p">(</span><span class="n">i</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">max_seq_length</span><span class="p">,</span> <span class="nb">len</span><span class="p">(</span><span class="n">items</span><span class="p">)</span> <span class="o">-</span> <span class="mi">1</span><span class="p">)</span>
+        <span class="n">item_sequences</span> <span class="o">=</span> <span class="n">items</span><span class="p">[</span><span class="n">i</span><span class="p">:</span><span class="n">end_idx</span><span class="p">]</span>
+        <span class="n">target_item</span> <span class="o">=</span> <span class="n">items</span><span class="p">[</span><span class="n">end_idx</span><span class="p">]</span>
+        <span class="n">target</span> <span class="o">=</span> <span class="n">targets</span><span class="p">[</span><span class="n">end_idx</span><span class="p">]</span>
+
+        <span class="n">sequence</span> <span class="o">=</span> <span class="p">{</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">user_id_col</span><span class="p">:</span> <span class="n">group</span><span class="o">.</span><span class="n">name</span><span class="p">,</span>
+            <span class="s2">&quot;items_sequence&quot;</span><span class="p">:</span> <span class="n">item_sequences</span><span class="p">,</span>
+            <span class="s2">&quot;target_item&quot;</span><span class="p">:</span> <span class="n">target_item</span><span class="p">,</span>
+        <span class="p">}</span>
+
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">action_col</span><span class="p">:</span>
+            <span class="n">sequence</span><span class="o">.</span><span class="n">update</span><span class="p">(</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">_create_action_sequence</span><span class="p">(</span><span class="n">group</span><span class="p">,</span> <span class="n">targets</span><span class="p">,</span> <span class="n">i</span><span class="p">,</span> <span class="n">target</span><span class="p">,</span> <span class="n">drop_cols</span><span class="p">)</span>
+            <span class="p">)</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="n">sequence</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">target_col</span><span class="p">]</span> <span class="o">=</span> <span class="n">target</span>
+            <span class="n">drop_cols</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">target_col</span><span class="p">)</span>
+
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">other_seq_embed_cols</span><span class="p">:</span>
+            <span class="n">sequence</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_create_other_seq_sequence</span><span class="p">(</span><span class="n">group</span><span class="p">,</span> <span class="n">i</span><span class="p">,</span> <span class="n">drop_cols</span><span class="p">))</span>
+
+        <span class="k">return</span> <span class="n">sequence</span>
+
+    <span class="k">def</span> <span class="nf">_create_action_sequence</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span>
+        <span class="n">group</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">,</span>
+        <span class="n">targets</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="nb">int</span><span class="p">],</span>
+        <span class="n">i</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span>
+        <span class="n">target</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span>
+        <span class="n">drop_cols</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">],</span>
+    <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Union</span><span class="p">[</span><span class="nb">int</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="nb">int</span><span class="p">]]]:</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">action_col</span> <span class="o">!=</span> <span class="bp">self</span><span class="o">.</span><span class="n">target_col</span><span class="p">:</span>
+            <span class="n">actions</span> <span class="o">=</span> <span class="n">group</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">action_col</span><span class="p">]</span><span class="o">.</span><span class="n">tolist</span><span class="p">()</span>
+            <span class="n">action_sequences</span> <span class="o">=</span> <span class="p">(</span>
+                <span class="n">actions</span><span class="p">[</span><span class="n">i</span> <span class="p">:</span> <span class="n">i</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">max_seq_length</span><span class="p">]</span>
+                <span class="k">if</span> <span class="n">i</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">max_seq_length</span> <span class="o">&lt;</span> <span class="nb">len</span><span class="p">(</span><span class="n">actions</span><span class="p">)</span>
+                <span class="k">else</span> <span class="n">actions</span><span class="p">[</span><span class="n">i</span><span class="p">:</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span>
+            <span class="p">)</span>
+            <span class="n">drop_cols</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">action_col</span><span class="p">)</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="n">target</span> <span class="o">-=</span> <span class="mi">1</span>  <span class="c1"># the &#39;transform&#39; method adds 1 as it saves 0 for padding</span>
+            <span class="n">action_sequences</span> <span class="o">=</span> <span class="p">(</span>
+                <span class="n">targets</span><span class="p">[</span><span class="n">i</span> <span class="p">:</span> <span class="n">i</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">max_seq_length</span><span class="p">]</span>
+                <span class="k">if</span> <span class="n">i</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">max_seq_length</span> <span class="o">&lt;</span> <span class="nb">len</span><span class="p">(</span><span class="n">targets</span><span class="p">)</span>
+                <span class="k">else</span> <span class="n">targets</span><span class="p">[</span><span class="n">i</span><span class="p">:</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span>
+            <span class="p">)</span>
+
+        <span class="k">return</span> <span class="p">{</span><span class="bp">self</span><span class="o">.</span><span class="n">target_col</span><span class="p">:</span> <span class="n">target</span><span class="p">,</span> <span class="s2">&quot;actions_sequence&quot;</span><span class="p">:</span> <span class="n">action_sequences</span><span class="p">}</span>
+
+    <span class="k">def</span> <span class="nf">_create_other_seq_sequence</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span> <span class="n">group</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">,</span> <span class="n">i</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span> <span class="n">drop_cols</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span>
+    <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Union</span><span class="p">[</span><span class="nb">int</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="nb">int</span><span class="p">]]]:</span>
+        <span class="n">other_seq_cols</span> <span class="o">=</span> <span class="p">[</span>
+            <span class="n">col</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">col</span><span class="p">,</span> <span class="nb">tuple</span><span class="p">)</span> <span class="k">else</span> <span class="n">col</span>
+            <span class="k">for</span> <span class="n">col</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">other_seq_embed_cols</span>
+        <span class="p">]</span>
+        <span class="n">drop_cols</span> <span class="o">+=</span> <span class="n">other_seq_cols</span>
+        <span class="n">other_seqs</span> <span class="o">=</span> <span class="p">{</span><span class="n">col</span><span class="p">:</span> <span class="n">group</span><span class="p">[</span><span class="n">col</span><span class="p">]</span><span class="o">.</span><span class="n">tolist</span><span class="p">()</span> <span class="k">for</span> <span class="n">col</span> <span class="ow">in</span> <span class="n">other_seq_cols</span><span class="p">}</span>
+
+        <span class="n">other_seqs_sequences</span><span class="p">:</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Union</span><span class="p">[</span><span class="nb">int</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="nb">int</span><span class="p">]]]</span> <span class="o">=</span> <span class="p">{}</span>
+        <span class="k">for</span> <span class="n">col</span> <span class="ow">in</span> <span class="n">other_seq_cols</span><span class="p">:</span>
+            <span class="n">other_seqs_sequences</span><span class="p">[</span><span class="n">col</span><span class="p">]</span> <span class="o">=</span> <span class="p">(</span>
+                <span class="n">other_seqs</span><span class="p">[</span><span class="n">col</span><span class="p">][</span><span class="n">i</span> <span class="p">:</span> <span class="n">i</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">max_seq_length</span><span class="p">]</span>
+                <span class="k">if</span> <span class="n">i</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">max_seq_length</span> <span class="o">&lt;</span> <span class="nb">len</span><span class="p">(</span><span class="n">other_seqs</span><span class="p">[</span><span class="n">col</span><span class="p">])</span>
+                <span class="k">else</span> <span class="n">other_seqs</span><span class="p">[</span><span class="n">col</span><span class="p">][</span><span class="n">i</span><span class="p">:</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span>
+            <span class="p">)</span>
+
+        <span class="n">sequence</span><span class="p">:</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Union</span><span class="p">[</span><span class="nb">int</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="nb">int</span><span class="p">]]]</span> <span class="o">=</span> <span class="p">{}</span>
+        <span class="k">for</span> <span class="n">col</span> <span class="ow">in</span> <span class="n">other_seq_cols</span><span class="p">:</span>
+            <span class="n">sequence</span><span class="p">[</span><span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="n">col</span><span class="si">}</span><span class="s2">_sequence&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">other_seqs_sequences</span><span class="p">[</span><span class="n">col</span><span class="p">]</span>
+            <span class="n">sequence</span><span class="p">[</span><span class="sa">f</span><span class="s2">&quot;target_</span><span class="si">{</span><span class="n">col</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="p">(</span>
+                <span class="n">other_seqs</span><span class="p">[</span><span class="n">col</span><span class="p">][</span><span class="n">i</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">max_seq_length</span><span class="p">]</span>
+                <span class="k">if</span> <span class="n">i</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">max_seq_length</span> <span class="o">&lt;</span> <span class="nb">len</span><span class="p">(</span><span class="n">other_seqs</span><span class="p">[</span><span class="n">col</span><span class="p">])</span>
+                <span class="k">else</span> <span class="n">other_seqs</span><span class="p">[</span><span class="n">col</span><span class="p">][</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span>
+            <span class="p">)</span>
+
+        <span class="k">return</span> <span class="n">sequence</span>
+
+    <span class="k">def</span> <span class="nf">_concatenate_features</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df_w_sequences</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">:</span>
+        <span class="n">user_behaviour_seq</span> <span class="o">=</span> <span class="n">df_w_sequences</span><span class="p">[</span><span class="s2">&quot;items_sequence&quot;</span><span class="p">]</span><span class="o">.</span><span class="n">tolist</span><span class="p">()</span>
+        <span class="n">X_user_behaviour</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">vstack</span><span class="p">(</span>
+            <span class="p">[</span>
+                <span class="n">pad_sequences</span><span class="p">(</span><span class="n">seq</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">max_seq_length</span><span class="p">,</span> <span class="n">pad_idx</span><span class="o">=</span><span class="mi">0</span><span class="p">)</span>
+                <span class="k">for</span> <span class="n">seq</span> <span class="ow">in</span> <span class="n">user_behaviour_seq</span>
+            <span class="p">]</span>
+        <span class="p">)</span>
+        <span class="n">X_target_item</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">(</span><span class="n">df_w_sequences</span><span class="p">[</span><span class="s2">&quot;target_item&quot;</span><span class="p">]</span><span class="o">.</span><span class="n">tolist</span><span class="p">())</span><span class="o">.</span><span class="n">reshape</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span>
+        <span class="n">X_all</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">concatenate</span><span class="p">([</span><span class="n">X_user_behaviour</span><span class="p">,</span> <span class="n">X_target_item</span><span class="p">],</span> <span class="n">axis</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span>
+
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">has_standard_tab_data</span><span class="p">:</span>
+            <span class="n">X_tab</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">tab_preprocessor</span><span class="o">.</span><span class="n">transform</span><span class="p">(</span><span class="n">df_w_sequences</span><span class="p">)</span>
+            <span class="n">X_all</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">concatenate</span><span class="p">([</span><span class="n">X_tab</span><span class="p">,</span> <span class="n">X_all</span><span class="p">],</span> <span class="n">axis</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span>
+
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">action_col</span><span class="p">:</span>
+            <span class="n">action_seq</span> <span class="o">=</span> <span class="n">df_w_sequences</span><span class="p">[</span><span class="s2">&quot;actions_sequence&quot;</span><span class="p">]</span><span class="o">.</span><span class="n">tolist</span><span class="p">()</span>
+            <span class="n">X_actions</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">vstack</span><span class="p">(</span>
+                <span class="p">[</span>
+                    <span class="n">pad_sequences</span><span class="p">(</span><span class="n">seq</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">max_seq_length</span><span class="p">,</span> <span class="n">pad_idx</span><span class="o">=</span><span class="mi">0</span><span class="p">)</span>
+                    <span class="k">for</span> <span class="n">seq</span> <span class="ow">in</span> <span class="n">action_seq</span>
+                <span class="p">]</span>
+            <span class="p">)</span>
+            <span class="n">X_all</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">concatenate</span><span class="p">([</span><span class="n">X_all</span><span class="p">,</span> <span class="n">X_actions</span><span class="p">],</span> <span class="n">axis</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span>
+
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">other_seq_embed_cols</span><span class="p">:</span>
+            <span class="n">X_all</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_concatenate_other_seq_features</span><span class="p">(</span><span class="n">df_w_sequences</span><span class="p">,</span> <span class="n">X_all</span><span class="p">)</span>
+
+        <span class="k">assert</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">din_columns_idx</span><span class="p">)</span> <span class="o">==</span> <span class="n">X_all</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">1</span><span class="p">],</span> <span class="p">(</span>
+            <span class="sa">f</span><span class="s2">&quot;Something went wrong. The number of columns in the final array &quot;</span>
+            <span class="sa">f</span><span class="s2">&quot;(</span><span class="si">{</span><span class="n">X_all</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span><span class="si">}</span><span class="s2">) is different from the number of columns in &quot;</span>
+            <span class="sa">f</span><span class="s2">&quot;self.din_columns_idx (</span><span class="si">{</span><span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">din_columns_idx</span><span class="p">)</span><span class="si">}</span><span class="s2">)&quot;</span>
         <span class="p">)</span>
 
-    <span class="bp">self</span><span class="o">.</span><span class="n">is_fitted</span> <span class="o">=</span> <span class="kc">False</span>
+        <span class="k">return</span> <span class="n">X_all</span>
+
+    <span class="k">def</span> <span class="nf">_concatenate_other_seq_features</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span> <span class="n">df_w_sequences</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">,</span> <span class="n">X_all</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span>
+    <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">:</span>
+        <span class="n">other_seq_cols</span> <span class="o">=</span> <span class="p">[</span>
+            <span class="n">col</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">col</span><span class="p">,</span> <span class="nb">tuple</span><span class="p">)</span> <span class="k">else</span> <span class="n">col</span>
+            <span class="k">for</span> <span class="n">col</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">other_seq_embed_cols</span>
+        <span class="p">]</span>
+        <span class="n">other_seq</span> <span class="o">=</span> <span class="p">{</span>
+            <span class="n">col</span><span class="p">:</span> <span class="n">df_w_sequences</span><span class="p">[</span><span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="n">col</span><span class="si">}</span><span class="s2">_sequence&quot;</span><span class="p">]</span><span class="o">.</span><span class="n">tolist</span><span class="p">()</span> <span class="k">for</span> <span class="n">col</span> <span class="ow">in</span> <span class="n">other_seq_cols</span>
+        <span class="p">}</span>
+        <span class="n">other_seq_target</span> <span class="o">=</span> <span class="p">{</span>
+            <span class="n">col</span><span class="p">:</span> <span class="n">df_w_sequences</span><span class="p">[</span><span class="sa">f</span><span class="s2">&quot;target_</span><span class="si">{</span><span class="n">col</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">]</span><span class="o">.</span><span class="n">tolist</span><span class="p">()</span> <span class="k">for</span> <span class="n">col</span> <span class="ow">in</span> <span class="n">other_seq_cols</span>
+        <span class="p">}</span>
+        <span class="n">X_other_seq_arrays</span> <span class="o">=</span> <span class="p">[]</span>
+
+        <span class="k">for</span> <span class="n">col</span> <span class="ow">in</span> <span class="n">other_seq_cols</span><span class="p">:</span>
+            <span class="n">X_other_seq_arrays</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
+                <span class="n">np</span><span class="o">.</span><span class="n">vstack</span><span class="p">(</span>
+                    <span class="p">[</span>
+                        <span class="n">pad_sequences</span><span class="p">(</span><span class="n">s</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">max_seq_length</span><span class="p">,</span> <span class="n">pad_idx</span><span class="o">=</span><span class="mi">0</span><span class="p">)</span>
+                        <span class="k">for</span> <span class="n">s</span> <span class="ow">in</span> <span class="n">other_seq</span><span class="p">[</span><span class="n">col</span><span class="p">]</span>
+                    <span class="p">]</span>
+                <span class="p">)</span>
+            <span class="p">)</span>
+            <span class="n">X_other_seq_arrays</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">(</span><span class="n">other_seq_target</span><span class="p">[</span><span class="n">col</span><span class="p">])</span><span class="o">.</span><span class="n">reshape</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">))</span>
+
+        <span class="k">return</span> <span class="n">np</span><span class="o">.</span><span class="n">concatenate</span><span class="p">([</span><span class="n">X_all</span><span class="p">]</span> <span class="o">+</span> <span class="n">X_other_seq_arrays</span><span class="p">,</span> <span class="n">axis</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span>
 </code></pre></div></td></tr></table></div>
-                  </details>
+              </details>
 
 
 
@@ -5735,10 +10935,7 @@ <h2 id="pytorch_widedeep.preprocessing.wide_preprocessor.ChunkWidePreprocessor"
 
 
 <a href="#pytorch_widedeep.preprocessing.wide_preprocessor.ChunkWidePreprocessor" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="nf">ChunkWidePreprocessor</span><span class="p">(</span>
-    <span class="n">wide_cols</span><span class="p">,</span> <span class="n">n_chunks</span><span class="p">,</span> <span class="n">crossed_cols</span><span class="o">=</span><span class="kc">None</span>
-<span class="p">)</span>
-</code></pre></div>
+
 
     <div class="doc doc-contents first">
             <p class="doc doc-class-bases">
@@ -5754,68 +10951,109 @@ <h2 id="pytorch_widedeep.preprocessing.wide_preprocessor.ChunkWidePreprocessor"
 
 
 <p><span class="doc-section-title">Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>wide_cols</code></b>
-              (<code><span title="typing.List">List</span>[str]</code>)
-          –
-          <div class="doc-md-description">
-            <p>List of strings with the name of the columns that will label
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+          <th>Default</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>wide_cols</code></td>
+            <td>
+                  <code><span title="typing.List">List</span>[str]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>List of strings with the name of the columns that will label
 encoded and passed through the <code>wide</code> component</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>crossed_cols</code></b>
-              (<code><span title="typing.Optional">Optional</span>[<span title="typing.List">List</span>[<span title="typing.Tuple">Tuple</span>[str, str]]]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>List of Tuples with the name of the columns that will be <code>'crossed'</code>
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>crossed_cols</code></td>
+            <td>
+                  <code><span title="typing.Optional">Optional</span>[<span title="typing.List">List</span>[<span title="typing.Tuple">Tuple</span>[str, str]]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>List of Tuples with the name of the columns that will be <code>'crossed'</code>
 and then label encoded. e.g. <em>[('education', 'occupation'), ...]</em>. For
 binary features, a cross-product transformation is 1 if and only if
 the constituent features are all 1, and 0 otherwise.</p>
-          </div>
-        </li>
-    </ul>
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
 
 <p><span class="doc-section-title">Attributes:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code><span title="pytorch_widedeep.preprocessing.wide_preprocessor.ChunkWidePreprocessor.wide_crossed_cols">wide_crossed_cols</span></code></b>
-              (<code><span title="typing.List">List</span></code>)
-          –
-          <div class="doc-md-description">
-            <p>List with the names of all columns that will be label encoded</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code><span title="pytorch_widedeep.preprocessing.wide_preprocessor.ChunkWidePreprocessor.encoding_dict">encoding_dict</span></code></b>
-              (<code>Dict</code>)
-          –
-          <div class="doc-md-description">
-            <p>Dictionary where the keys are the result of pasting <code>colname + '_' +
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code><span title="pytorch_widedeep.preprocessing.wide_preprocessor.ChunkWidePreprocessor.wide_crossed_cols">wide_crossed_cols</span></code></td>
+            <td>
+                  <code><span title="typing.List">List</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>List with the names of all columns that will be label encoded</p>
+              </div>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code><span title="pytorch_widedeep.preprocessing.wide_preprocessor.ChunkWidePreprocessor.encoding_dict">encoding_dict</span></code></td>
+            <td>
+                  <code>Dict</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Dictionary where the keys are the result of pasting <code>colname + '_' +
 column value</code> and the values are the corresponding mapped integer.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code><span title="pytorch_widedeep.preprocessing.wide_preprocessor.ChunkWidePreprocessor.inverse_encoding_dict">inverse_encoding_dict</span></code></b>
-              (<code>Dict</code>)
-          –
-          <div class="doc-md-description">
-            <p>the inverse encoding dictionary</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code><span title="pytorch_widedeep.preprocessing.wide_preprocessor.ChunkWidePreprocessor.wide_dim">wide_dim</span></code></b>
-              (<code>int</code>)
-          –
-          <div class="doc-md-description">
-            <p>Dimension of the wide model (i.e. dim of the linear layer)</p>
-          </div>
-        </li>
-    </ul>
+              </div>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code><span title="pytorch_widedeep.preprocessing.wide_preprocessor.ChunkWidePreprocessor.inverse_encoding_dict">inverse_encoding_dict</span></code></td>
+            <td>
+                  <code>Dict</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>the inverse encoding dictionary</p>
+              </div>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code><span title="pytorch_widedeep.preprocessing.wide_preprocessor.ChunkWidePreprocessor.wide_dim">wide_dim</span></code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Dimension of the wide model (i.e. dim of the linear layer)</p>
+              </div>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
 
 <p><span class="doc-section-title">Examples:</span></p>
@@ -5829,9 +11067,53 @@ <h2 id="pytorch_widedeep.preprocessing.wide_preprocessor.ChunkWidePreprocessor"
 <span class="gp">&gt;&gt;&gt; </span><span class="n">X_wide</span> <span class="o">=</span> <span class="n">chunk_wide_preprocessor</span><span class="o">.</span><span class="n">fit_transform</span><span class="p">(</span><span class="n">chunk</span><span class="p">)</span>
 </code></pre></div>
 
-                  <details class="quote">
-                    <summary>Source code in <code>pytorch_widedeep/preprocessing/wide_preprocessor.py</code></summary>
-                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">253</span>
+              <details class="quote">
+                <summary>Source code in <code>pytorch_widedeep/preprocessing/wide_preprocessor.py</code></summary>
+                <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">209</span>
+<span class="normal">210</span>
+<span class="normal">211</span>
+<span class="normal">212</span>
+<span class="normal">213</span>
+<span class="normal">214</span>
+<span class="normal">215</span>
+<span class="normal">216</span>
+<span class="normal">217</span>
+<span class="normal">218</span>
+<span class="normal">219</span>
+<span class="normal">220</span>
+<span class="normal">221</span>
+<span class="normal">222</span>
+<span class="normal">223</span>
+<span class="normal">224</span>
+<span class="normal">225</span>
+<span class="normal">226</span>
+<span class="normal">227</span>
+<span class="normal">228</span>
+<span class="normal">229</span>
+<span class="normal">230</span>
+<span class="normal">231</span>
+<span class="normal">232</span>
+<span class="normal">233</span>
+<span class="normal">234</span>
+<span class="normal">235</span>
+<span class="normal">236</span>
+<span class="normal">237</span>
+<span class="normal">238</span>
+<span class="normal">239</span>
+<span class="normal">240</span>
+<span class="normal">241</span>
+<span class="normal">242</span>
+<span class="normal">243</span>
+<span class="normal">244</span>
+<span class="normal">245</span>
+<span class="normal">246</span>
+<span class="normal">247</span>
+<span class="normal">248</span>
+<span class="normal">249</span>
+<span class="normal">250</span>
+<span class="normal">251</span>
+<span class="normal">252</span>
+<span class="normal">253</span>
 <span class="normal">254</span>
 <span class="normal">255</span>
 <span class="normal">256</span>
@@ -5843,21 +11125,169 @@ <h2 id="pytorch_widedeep.preprocessing.wide_preprocessor.ChunkWidePreprocessor"
 <span class="normal">262</span>
 <span class="normal">263</span>
 <span class="normal">264</span>
-<span class="normal">265</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
-    <span class="bp">self</span><span class="p">,</span>
-    <span class="n">wide_cols</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">],</span>
-    <span class="n">n_chunks</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span>
-    <span class="n">crossed_cols</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="n">Tuple</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">]]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-<span class="p">):</span>
-    <span class="nb">super</span><span class="p">(</span><span class="n">ChunkWidePreprocessor</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">wide_cols</span><span class="p">,</span> <span class="n">crossed_cols</span><span class="p">)</span>
+<span class="normal">265</span>
+<span class="normal">266</span>
+<span class="normal">267</span>
+<span class="normal">268</span>
+<span class="normal">269</span>
+<span class="normal">270</span>
+<span class="normal">271</span>
+<span class="normal">272</span>
+<span class="normal">273</span>
+<span class="normal">274</span>
+<span class="normal">275</span>
+<span class="normal">276</span>
+<span class="normal">277</span>
+<span class="normal">278</span>
+<span class="normal">279</span>
+<span class="normal">280</span>
+<span class="normal">281</span>
+<span class="normal">282</span>
+<span class="normal">283</span>
+<span class="normal">284</span>
+<span class="normal">285</span>
+<span class="normal">286</span>
+<span class="normal">287</span>
+<span class="normal">288</span>
+<span class="normal">289</span>
+<span class="normal">290</span>
+<span class="normal">291</span>
+<span class="normal">292</span>
+<span class="normal">293</span>
+<span class="normal">294</span>
+<span class="normal">295</span>
+<span class="normal">296</span>
+<span class="normal">297</span>
+<span class="normal">298</span>
+<span class="normal">299</span>
+<span class="normal">300</span>
+<span class="normal">301</span>
+<span class="normal">302</span>
+<span class="normal">303</span>
+<span class="normal">304</span>
+<span class="normal">305</span>
+<span class="normal">306</span>
+<span class="normal">307</span>
+<span class="normal">308</span>
+<span class="normal">309</span>
+<span class="normal">310</span>
+<span class="normal">311</span>
+<span class="normal">312</span>
+<span class="normal">313</span>
+<span class="normal">314</span>
+<span class="normal">315</span>
+<span class="normal">316</span>
+<span class="normal">317</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">class</span> <span class="nc">ChunkWidePreprocessor</span><span class="p">(</span><span class="n">WidePreprocessor</span><span class="p">):</span>
+<span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Preprocessor to prepare the wide input dataset</span>
+
+<span class="sd">    This Preprocessor prepares the data for the wide, linear component.</span>
+<span class="sd">    This linear model is implemented via an Embedding layer that is</span>
+<span class="sd">    connected to the output neuron. `ChunkWidePreprocessor` numerically</span>
+<span class="sd">    encodes all the unique values of all categorical columns `wide_cols +</span>
+<span class="sd">    crossed_cols`. See the Example below.</span>
+
+<span class="sd">    Parameters</span>
+<span class="sd">    ----------</span>
+<span class="sd">    wide_cols: List</span>
+<span class="sd">        List of strings with the name of the columns that will label</span>
+<span class="sd">        encoded and passed through the `wide` component</span>
+<span class="sd">    crossed_cols: List, default = None</span>
+<span class="sd">        List of Tuples with the name of the columns that will be `&#39;crossed&#39;`</span>
+<span class="sd">        and then label encoded. e.g. _[(&#39;education&#39;, &#39;occupation&#39;), ...]_. For</span>
+<span class="sd">        binary features, a cross-product transformation is 1 if and only if</span>
+<span class="sd">        the constituent features are all 1, and 0 otherwise.</span>
+
+<span class="sd">    Attributes</span>
+<span class="sd">    ----------</span>
+<span class="sd">    wide_crossed_cols: List</span>
+<span class="sd">        List with the names of all columns that will be label encoded</span>
+<span class="sd">    encoding_dict: Dict</span>
+<span class="sd">        Dictionary where the keys are the result of pasting `colname + &#39;_&#39; +</span>
+<span class="sd">        column value` and the values are the corresponding mapped integer.</span>
+<span class="sd">    inverse_encoding_dict: Dict</span>
+<span class="sd">        the inverse encoding dictionary</span>
+<span class="sd">    wide_dim: int</span>
+<span class="sd">        Dimension of the wide model (i.e. dim of the linear layer)</span>
+
+<span class="sd">    Examples</span>
+<span class="sd">    --------</span>
+<span class="sd">    &gt;&gt;&gt; import pandas as pd</span>
+<span class="sd">    &gt;&gt;&gt; from pytorch_widedeep.preprocessing import ChunkWidePreprocessor</span>
+<span class="sd">    &gt;&gt;&gt; chunk = pd.DataFrame({&#39;color&#39;: [&#39;r&#39;, &#39;b&#39;, &#39;g&#39;], &#39;size&#39;: [&#39;s&#39;, &#39;n&#39;, &#39;l&#39;]})</span>
+<span class="sd">    &gt;&gt;&gt; wide_cols = [&#39;color&#39;]</span>
+<span class="sd">    &gt;&gt;&gt; crossed_cols = [(&#39;color&#39;, &#39;size&#39;)]</span>
+<span class="sd">    &gt;&gt;&gt; chunk_wide_preprocessor = ChunkWidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols,</span>
+<span class="sd">    ... n_chunks=1)</span>
+<span class="sd">    &gt;&gt;&gt; X_wide = chunk_wide_preprocessor.fit_transform(chunk)</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+
+    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span>
+        <span class="n">wide_cols</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">],</span>
+        <span class="n">n_chunks</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span>
+        <span class="n">crossed_cols</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="n">Tuple</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">]]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+    <span class="p">):</span>
+        <span class="nb">super</span><span class="p">(</span><span class="n">ChunkWidePreprocessor</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">wide_cols</span><span class="p">,</span> <span class="n">crossed_cols</span><span class="p">)</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">n_chunks</span> <span class="o">=</span> <span class="n">n_chunks</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">chunk_counter</span> <span class="o">=</span> <span class="mi">0</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">is_fitted</span> <span class="o">=</span> <span class="kc">False</span>
+
+    <span class="k">def</span> <span class="nf">partial_fit</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">chunk</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="s2">&quot;ChunkWidePreprocessor&quot;</span><span class="p">:</span>
+<span class="w">        </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Fits the Preprocessor and creates required attributes</span>
+
+<span class="sd">        Parameters</span>
+<span class="sd">        ----------</span>
+<span class="sd">        chunk: pd.DataFrame</span>
+<span class="sd">            Input pandas dataframe</span>
+
+<span class="sd">        Returns</span>
+<span class="sd">        -------</span>
+<span class="sd">        ChunkWidePreprocessor</span>
+<span class="sd">            `ChunkWidePreprocessor` fitted object</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">df_wide</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_prepare_wide</span><span class="p">(</span><span class="n">chunk</span><span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">wide_crossed_cols</span> <span class="o">=</span> <span class="n">df_wide</span><span class="o">.</span><span class="n">columns</span><span class="o">.</span><span class="n">tolist</span><span class="p">()</span>
+
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">chunk_counter</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">glob_feature_set</span> <span class="o">=</span> <span class="nb">set</span><span class="p">(</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">_make_global_feature_list</span><span class="p">(</span><span class="n">df_wide</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">wide_crossed_cols</span><span class="p">])</span>
+            <span class="p">)</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">glob_feature_set</span><span class="o">.</span><span class="n">update</span><span class="p">(</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">_make_global_feature_list</span><span class="p">(</span><span class="n">df_wide</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">wide_crossed_cols</span><span class="p">])</span>
+            <span class="p">)</span>
 
-    <span class="bp">self</span><span class="o">.</span><span class="n">n_chunks</span> <span class="o">=</span> <span class="n">n_chunks</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">chunk_counter</span> <span class="o">+=</span> <span class="mi">1</span>
 
-    <span class="bp">self</span><span class="o">.</span><span class="n">chunk_counter</span> <span class="o">=</span> <span class="mi">0</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">chunk_counter</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">n_chunks</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">encoding_dict</span> <span class="o">=</span> <span class="p">{</span><span class="n">v</span><span class="p">:</span> <span class="n">i</span> <span class="o">+</span> <span class="mi">1</span> <span class="k">for</span> <span class="n">i</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">glob_feature_set</span><span class="p">)}</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">wide_dim</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">encoding_dict</span><span class="p">)</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">inverse_encoding_dict</span> <span class="o">=</span> <span class="p">{</span><span class="n">k</span><span class="p">:</span> <span class="n">v</span> <span class="k">for</span> <span class="n">v</span><span class="p">,</span> <span class="n">k</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">encoding_dict</span><span class="o">.</span><span class="n">items</span><span class="p">()}</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">inverse_encoding_dict</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">=</span> <span class="s2">&quot;unseen&quot;</span>
 
-    <span class="bp">self</span><span class="o">.</span><span class="n">is_fitted</span> <span class="o">=</span> <span class="kc">False</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">is_fitted</span> <span class="o">=</span> <span class="kc">True</span>
+
+        <span class="k">return</span> <span class="bp">self</span>
+
+    <span class="k">def</span> <span class="nf">fit</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="s2">&quot;ChunkWidePreprocessor&quot;</span><span class="p">:</span>
+<span class="w">        </span><span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Runs `partial_fit`. This is just to override the fit method in the base</span>
+<span class="sd">        class. This class is not designed or thought to run fit</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">partial_fit</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="fm">__repr__</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">str</span><span class="p">:</span>
+        <span class="n">list_of_params</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[</span><span class="s2">&quot;wide_cols=</span><span class="si">{wide_cols}</span><span class="s2">&quot;</span><span class="p">]</span>
+        <span class="n">list_of_params</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">&quot;n_chunks=</span><span class="si">{n_chunks}</span><span class="s2">&quot;</span><span class="p">)</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">crossed_cols</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="n">list_of_params</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">&quot;crossed_cols=</span><span class="si">{crossed_cols}</span><span class="s2">&quot;</span><span class="p">)</span>
+        <span class="n">all_params</span> <span class="o">=</span> <span class="s2">&quot;, &quot;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">list_of_params</span><span class="p">)</span>
+        <span class="k">return</span> <span class="sa">f</span><span class="s2">&quot;WidePreprocessor(</span><span class="si">{</span><span class="n">all_params</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="bp">self</span><span class="o">.</span><span class="vm">__dict__</span><span class="p">)</span><span class="si">}</span><span class="s2">)&quot;</span>
 </code></pre></div></td></tr></table></div>
-                  </details>
+              </details>
 
 
 
@@ -5888,28 +11318,55 @@ <h3 id="pytorch_widedeep.preprocessing.wide_preprocessor.ChunkWidePreprocessor.p
 
 
 <p><span class="doc-section-title">Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>chunk</code></b>
-              (<code><span title="pandas.DataFrame">DataFrame</span></code>)
-          –
-          <div class="doc-md-description">
-            <p>Input pandas dataframe</p>
-          </div>
-        </li>
-    </ul>
-
-
-<p><span class="doc-section-title">Returns:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-              <code><a class="autorefs autorefs-internal" title="pytorch_widedeep.preprocessing.wide_preprocessor.ChunkWidePreprocessor" href="#pytorch_widedeep.preprocessing.wide_preprocessor.ChunkWidePreprocessor">ChunkWidePreprocessor</a></code>
-          –
-          <div class="doc-md-description">
-            <p><code>ChunkWidePreprocessor</code> fitted object</p>
-          </div>
-        </li>
-    </ul>
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+          <th>Default</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>chunk</code></td>
+            <td>
+                  <code><span title="pandas.DataFrame">DataFrame</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Input pandas dataframe</p>
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+      </tbody>
+    </table>
+
+
+    <p><span class="doc-section-title">Returns:</span></p>
+    <table>
+      <thead>
+        <tr>
+          <th>Type</th>
+          <th>Description</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td>
+                  <code><a class="autorefs autorefs-internal" title="pytorch_widedeep.preprocessing.wide_preprocessor.ChunkWidePreprocessor" href="#pytorch_widedeep.preprocessing.wide_preprocessor.ChunkWidePreprocessor">ChunkWidePreprocessor</a></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p><code>ChunkWidePreprocessor</code> fitted object</p>
+              </div>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
             <details class="quote">
               <summary>Source code in <code>pytorch_widedeep/preprocessing/wide_preprocessor.py</code></summary>
@@ -6042,23 +11499,7 @@ <h2 id="pytorch_widedeep.preprocessing.tab_preprocessor.ChunkTabPreprocessor" cl
 
 
 <a href="#pytorch_widedeep.preprocessing.tab_preprocessor.ChunkTabPreprocessor" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="nf">ChunkTabPreprocessor</span><span class="p">(</span>
-    <span class="n">n_chunks</span><span class="p">,</span>
-    <span class="n">cat_embed_cols</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">continuous_cols</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">cols_and_bins</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">cols_to_scale</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">default_embed_dim</span><span class="o">=</span><span class="mi">16</span><span class="p">,</span>
-    <span class="n">with_attention</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
-    <span class="n">with_cls_token</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
-    <span class="n">shared_embed</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
-    <span class="n">verbose</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span>
-    <span class="o">*</span><span class="p">,</span>
-    <span class="n">scale</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
-    <span class="n">already_standard</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="o">**</span><span class="n">kwargs</span>
-<span class="p">)</span>
-</code></pre></div>
+
 
     <div class="doc doc-contents first">
             <p class="doc doc-class-bases">
@@ -6069,82 +11510,118 @@ <h2 id="pytorch_widedeep.preprocessing.tab_preprocessor.ChunkTabPreprocessor" cl
 
 
 <p><span class="doc-section-title">Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>n_chunks</code></b>
-              (<code>int</code>)
-          –
-          <div class="doc-md-description">
-            <p>Number of chunks that the tabular dataset is divided by.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>cat_embed_cols</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[str], <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.wdtypes.Tuple">Tuple</span>[str, int]]]]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>List containing the name of the categorical columns that will be
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+          <th>Default</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>n_chunks</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Number of chunks that the tabular dataset is divided by.</p>
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>cat_embed_cols</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[str], <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.wdtypes.Tuple">Tuple</span>[str, int]]]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>List containing the name of the categorical columns that will be
 represented by embeddings (e.g. <em>['education', 'relationship', ...]</em>) or
 a Tuple with the name and the embedding dimension (e.g.: <em>[
-('education',32), ('relationship',16), ...]</em>)</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>continuous_cols</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[str]]</code>, default:
+('education',32), ('relationship',16), ...]</em>). <br/> Param alias:
+<code>embed_cols</code></p>
+              </div>
+            </td>
+            <td>
                   <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>List with the name of the continuous cols</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>cols_and_bins</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, <span title="pytorch_widedeep.wdtypes.List">List</span>[float]]]</code>, default:
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>continuous_cols</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[str]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>List with the name of the continuous cols</p>
+              </div>
+            </td>
+            <td>
                   <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Continuous columns can be turned into categorical via
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>cols_and_bins</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, <span title="pytorch_widedeep.wdtypes.List">List</span>[float]]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Continuous columns can be turned into categorical via
 <code>pd.cut</code>. 'cols_and_bins' is dictionary where the keys are the column
 names to quantize and the values are a list of scalars indicating the
-bin edges.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>cols_to_scale</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[str], str]]</code>, default:
+bin edges. <br/> Param alias: <code>quantization_setup</code></p>
+              </div>
+            </td>
+            <td>
                   <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>List with the names of the columns that will be standarised via
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>cols_to_scale</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[str], str]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>List with the names of the columns that will be standarised via
 sklearn's <code>StandardScaler</code></p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>default_embed_dim</code></b>
-              (<code>int</code>, default:
-                  <code>16</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Dimension for the embeddings if the embed_dim is not provided in the
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>default_embed_dim</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Dimension for the embeddings if the embed_dim is not provided in the
 <code>cat_embed_cols</code> parameter and <code>auto_embed_dim</code> is set to
 <code>False</code>.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>with_attention</code></b>
-              (<code>bool</code>, default:
-                  <code>False</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Boolean indicating whether the preprocessed data will be passed to an
+              </div>
+            </td>
+            <td>
+                  <code>16</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>with_attention</code></td>
+            <td>
+                  <code>bool</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Boolean indicating whether the preprocessed data will be passed to an
 attention-based model (more precisely a model where all embeddings
 must have the same dimensions). If <code>True</code>, the param <code>cat_embed_cols</code>
 must just be a list containing just the categorical column names:
@@ -6153,159 +11630,271 @@ <h2 id="pytorch_widedeep.preprocessing.tab_preprocessor.ChunkTabPreprocessor" cl
  encoded using embeddings of the same dim, which will be specified
  later when the model is defined. <br/> Param alias:
  <code>for_transformer</code></p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>with_cls_token</code></b>
-              (<code>bool</code>, default:
+              </div>
+            </td>
+            <td>
                   <code>False</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Boolean indicating if a <code>'[CLS]'</code> token will be added to the dataset
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>with_cls_token</code></td>
+            <td>
+                  <code>bool</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Boolean indicating if a <code>'[CLS]'</code> token will be added to the dataset
 when using attention-based models. The final hidden state
 corresponding to this token is used as the aggregated representation
 for classification and regression tasks. If not, the categorical
 (and continuous embeddings if present) will be concatenated before
 being passed to the final MLP (if present).</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>shared_embed</code></b>
-              (<code>bool</code>, default:
+              </div>
+            </td>
+            <td>
                   <code>False</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Boolean indicating if the embeddings will be "shared" when using
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>shared_embed</code></td>
+            <td>
+                  <code>bool</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Boolean indicating if the embeddings will be "shared" when using
 attention-based models. The idea behind <code>shared_embed</code> is
 described in the Appendix A in the <a href="https://arxiv.org/abs/2012.06678">TabTransformer paper</a>:
 <em>'The goal of having column embedding is to enable the model to
 distinguish the classes in one column from those in the other
 columns'</em>. In other words, the idea is to let the model learn which
 column is embedded at the time. See: <code>pytorch_widedeep.models.transformers._layers.SharedEmbeddings</code>.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>verbose</code></b>
-              (<code>int</code>, default:
-                  <code>1</code>
-)
-          –
-          <div class="doc-md-description">
-            
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>scale</code></b>
-              (<code>bool</code>, default:
+              </div>
+            </td>
+            <td>
                   <code>False</code>
-)
-          –
-          <div class="doc-md-description">
-            <p><img alt="ℹ️" class="emojione" src="https://cdnjs.cloudflare.com/ajax/libs/emojione/2.2.7/assets/png/2139.png" title=":information_source:" /> <strong>note</strong>: this arg will be removed in upcoming
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>verbose</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                
+              </div>
+            </td>
+            <td>
+                  <code>1</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>scale</code></td>
+            <td>
+                  <code>bool</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p><img alt="ℹ️" class="emojione" src="https://cdnjs.cloudflare.com/ajax/libs/emojione/2.2.7/assets/png/2139.png" title=":information_source:" /> <strong>note</strong>: this arg will be removed in upcoming
  releases. Please use <code>cols_to_scale</code> instead. <br/> Bool indicating
  whether or not to scale/standarise continuous cols. It is important
  to emphasize that all the DL models for tabular data in the library
  also include the possibility of normalising the input continuous
  features via a <code>BatchNorm</code> or a <code>LayerNorm</code>. <br/> Param alias:
  <code>scale_cont_cols</code>.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>already_standard</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[str]]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p><img alt="ℹ️" class="emojione" src="https://cdnjs.cloudflare.com/ajax/libs/emojione/2.2.7/assets/png/2139.png" title=":information_source:" /> <strong>note</strong>: this arg will be removed in upcoming
+              </div>
+            </td>
+            <td>
+                  <code>False</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>already_standard</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[str]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p><img alt="ℹ️" class="emojione" src="https://cdnjs.cloudflare.com/ajax/libs/emojione/2.2.7/assets/png/2139.png" title=":information_source:" /> <strong>note</strong>: this arg will be removed in upcoming
  releases. Please use <code>cols_to_scale</code> instead. <br/> List with the
  name of the continuous cols that do not need to be
  scaled/standarised.</p>
-          </div>
-        </li>
-    </ul>
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
 
 <p><span class="doc-section-title">Other Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>**kwargs</code></b>
-          –
-          <div class="doc-md-description">
-            <p><code>pd.cut</code> and <code>StandardScaler</code> related args</p>
-          </div>
-        </li>
-    </ul>
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>**kwargs</code></td>
+            <td>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p><code>pd.cut</code> and <code>StandardScaler</code> related args</p>
+              </div>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
 
 <p><span class="doc-section-title">Attributes:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code><span title="pytorch_widedeep.preprocessing.tab_preprocessor.ChunkTabPreprocessor.embed_dim">embed_dim</span></code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Dict">Dict</span></code>)
-          –
-          <div class="doc-md-description">
-            <p>Dictionary where keys are the embed cols and values are the embedding
-dimensions. If <code>with_attention</code> is set to <code>True</code> this attribute
-is not generated during the <code>fit</code> process</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code><span title="pytorch_widedeep.preprocessing.tab_preprocessor.ChunkTabPreprocessor.label_encoder">label_encoder</span></code></b>
-              (<code><a class="autorefs autorefs-internal" title="pytorch_widedeep.utils.deeptabular_utils.LabelEncoder" href="utils/deeptabular_utils.html#pytorch_widedeep.utils.deeptabular_utils.LabelEncoder">LabelEncoder</a></code>)
-          –
-          <div class="doc-md-description">
-            <p>see <code>pytorch_widedeep.utils.dense_utils.LabelEncder</code></p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code><span title="pytorch_widedeep.preprocessing.tab_preprocessor.ChunkTabPreprocessor.cat_embed_input">cat_embed_input</span></code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.List">List</span></code>)
-          –
-          <div class="doc-md-description">
-            <p>List of Tuples with the column name, number of individual values for
-that column and, If <code>with_attention</code> is set to <code>False</code>, the
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code><span title="pytorch_widedeep.preprocessing.tab_preprocessor.ChunkTabPreprocessor.cat_cols">cat_cols</span></code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.List">List</span>[str]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>List containing the names of the categorical columns after processing.</p>
+              </div>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code><span title="pytorch_widedeep.preprocessing.tab_preprocessor.ChunkTabPreprocessor.cols_and_bins">cols_and_bins</span></code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, <span title="pytorch_widedeep.wdtypes.Union">Union</span>[int, <span title="pytorch_widedeep.wdtypes.List">List</span>[float]]]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Dictionary containing the quantization setup after processing if
+quantization is requested. This is derived from the
+quantization_setup parameter.</p>
+              </div>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code><span title="pytorch_widedeep.preprocessing.tab_preprocessor.ChunkTabPreprocessor.quant_args">quant_args</span></code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Dict">Dict</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Dictionary containing arguments passed to pandas.cut() function for quantization.</p>
+              </div>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code><span title="pytorch_widedeep.preprocessing.tab_preprocessor.ChunkTabPreprocessor.scale_args">scale_args</span></code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Dict">Dict</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Dictionary containing arguments passed to StandardScaler.</p>
+              </div>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code><span title="pytorch_widedeep.preprocessing.tab_preprocessor.ChunkTabPreprocessor.label_encoder">label_encoder</span></code></td>
+            <td>
+                  <code><a class="autorefs autorefs-internal" title="pytorch_widedeep.utils.deeptabular_utils.LabelEncoder" href="utils/deeptabular_utils.html#pytorch_widedeep.utils.deeptabular_utils.LabelEncoder">LabelEncoder</a></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Instance of LabelEncoder used to encode categorical variables.
+See <code>pytorch_widedeep.utils.dense_utils.LabelEncoder</code>.</p>
+              </div>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code><span title="pytorch_widedeep.preprocessing.tab_preprocessor.ChunkTabPreprocessor.cat_embed_input">cat_embed_input</span></code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.List">List</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>List of tuples with the column name, number of individual values for
+that column and, if <code>with_attention</code> is set to <code>False</code>, the
 corresponding embeddings dim, e.g. <em>[('education', 16, 10),
 ('relationship', 6, 8), ...]</em>.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code><span title="pytorch_widedeep.preprocessing.tab_preprocessor.ChunkTabPreprocessor.standardize_cols">standardize_cols</span></code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.List">List</span></code>)
-          –
-          <div class="doc-md-description">
-            <p>List of the columns that will be standarized</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code><span title="pytorch_widedeep.preprocessing.tab_preprocessor.ChunkTabPreprocessor.scaler">scaler</span></code></b>
-              (<code><span title="sklearn.preprocessing.StandardScaler">StandardScaler</span></code>)
-          –
-          <div class="doc-md-description">
-            <p>an instance of <code>sklearn.preprocessing.StandardScaler</code>
-if 'cols_to_scale' is not None or 'scale' is 'True'</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code><span title="pytorch_widedeep.preprocessing.tab_preprocessor.ChunkTabPreprocessor.column_idx">column_idx</span></code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Dict">Dict</span></code>)
-          –
-          <div class="doc-md-description">
-            <p>Dictionary where keys are column names and values are column indexes.
-This is neccesary to slice tensors</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code><span title="pytorch_widedeep.preprocessing.tab_preprocessor.ChunkTabPreprocessor.quantizer">quantizer</span></code></b>
-              (<code><a class="autorefs autorefs-internal" title="pytorch_widedeep.preprocessing.tab_preprocessor.Quantizer" href="#pytorch_widedeep.preprocessing.tab_preprocessor.Quantizer">Quantizer</a></code>)
-          –
-          <div class="doc-md-description">
-            <p>an instance of <code>Quantizer</code></p>
-          </div>
-        </li>
-    </ul>
+              </div>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code><span title="pytorch_widedeep.preprocessing.tab_preprocessor.ChunkTabPreprocessor.standardize_cols">standardize_cols</span></code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.List">List</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>List of the columns that will be standardized.</p>
+              </div>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code><span title="pytorch_widedeep.preprocessing.tab_preprocessor.ChunkTabPreprocessor.scaler">scaler</span></code></td>
+            <td>
+                  <code><span title="sklearn.preprocessing.StandardScaler">StandardScaler</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>An instance of <code>sklearn.preprocessing.StandardScaler</code> if standardization
+is requested.</p>
+              </div>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code><span title="pytorch_widedeep.preprocessing.tab_preprocessor.ChunkTabPreprocessor.column_idx">column_idx</span></code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Dict">Dict</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Dictionary where keys are column names and values are column indexes.
+This is necessary to slice tensors.</p>
+              </div>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code><span title="pytorch_widedeep.preprocessing.tab_preprocessor.ChunkTabPreprocessor.quantizer">quantizer</span></code></td>
+            <td>
+                  <code><a class="autorefs autorefs-internal" title="pytorch_widedeep.preprocessing.tab_preprocessor.Quantizer" href="#pytorch_widedeep.preprocessing.tab_preprocessor.Quantizer">Quantizer</a></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>An instance of <code>Quantizer</code> if quantization is requested.</p>
+              </div>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code><span title="pytorch_widedeep.preprocessing.tab_preprocessor.ChunkTabPreprocessor.is_fitted">is_fitted</span></code></td>
+            <td>
+                  <code>bool</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Boolean indicating if the preprocessor has been fitted.</p>
+              </div>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
 
 <p><span class="doc-section-title">Examples:</span></p>
@@ -6327,103 +11916,711 @@ <h2 id="pytorch_widedeep.preprocessing.tab_preprocessor.ChunkTabPreprocessor" cl
 <span class="go">{&#39;cat_col&#39;: 0, &#39;cont_col&#39;: 1}</span>
 </code></pre></div>
 
-                  <details class="quote">
-                    <summary>Source code in <code>pytorch_widedeep/preprocessing/tab_preprocessor.py</code></summary>
-                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">818</span>
-<span class="normal">819</span>
-<span class="normal">820</span>
-<span class="normal">821</span>
-<span class="normal">822</span>
-<span class="normal">823</span>
-<span class="normal">824</span>
-<span class="normal">825</span>
-<span class="normal">826</span>
-<span class="normal">827</span>
-<span class="normal">828</span>
-<span class="normal">829</span>
-<span class="normal">830</span>
-<span class="normal">831</span>
-<span class="normal">832</span>
-<span class="normal">833</span>
-<span class="normal">834</span>
-<span class="normal">835</span>
-<span class="normal">836</span>
-<span class="normal">837</span>
-<span class="normal">838</span>
-<span class="normal">839</span>
-<span class="normal">840</span>
-<span class="normal">841</span>
-<span class="normal">842</span>
-<span class="normal">843</span>
-<span class="normal">844</span>
-<span class="normal">845</span>
-<span class="normal">846</span>
-<span class="normal">847</span>
-<span class="normal">848</span>
-<span class="normal">849</span>
-<span class="normal">850</span>
-<span class="normal">851</span>
-<span class="normal">852</span>
-<span class="normal">853</span>
-<span class="normal">854</span>
-<span class="normal">855</span>
-<span class="normal">856</span>
-<span class="normal">857</span>
-<span class="normal">858</span>
-<span class="normal">859</span>
-<span class="normal">860</span>
-<span class="normal">861</span>
-<span class="normal">862</span>
-<span class="normal">863</span>
-<span class="normal">864</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="nd">@alias</span><span class="p">(</span><span class="s2">&quot;with_attention&quot;</span><span class="p">,</span> <span class="p">[</span><span class="s2">&quot;for_transformer&quot;</span><span class="p">])</span>
-<span class="nd">@alias</span><span class="p">(</span><span class="s2">&quot;cat_embed_cols&quot;</span><span class="p">,</span> <span class="p">[</span><span class="s2">&quot;embed_cols&quot;</span><span class="p">])</span>
-<span class="nd">@alias</span><span class="p">(</span><span class="s2">&quot;scale&quot;</span><span class="p">,</span> <span class="p">[</span><span class="s2">&quot;scale_cont_cols&quot;</span><span class="p">])</span>
-<span class="nd">@alias</span><span class="p">(</span><span class="s2">&quot;cols_and_bins&quot;</span><span class="p">,</span> <span class="p">[</span><span class="s2">&quot;quantization_setup&quot;</span><span class="p">])</span>
-<span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
-    <span class="bp">self</span><span class="p">,</span>
-    <span class="n">n_chunks</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span>
-    <span class="n">cat_embed_cols</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Union</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">],</span> <span class="n">List</span><span class="p">[</span><span class="n">Tuple</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">]]]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">continuous_cols</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">cols_and_bins</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="nb">float</span><span class="p">]]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">cols_to_scale</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Union</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">],</span> <span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">default_embed_dim</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">16</span><span class="p">,</span>
-    <span class="n">with_attention</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
-    <span class="n">with_cls_token</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
-    <span class="n">shared_embed</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
-    <span class="n">verbose</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1</span><span class="p">,</span>
-    <span class="o">*</span><span class="p">,</span>
-    <span class="n">scale</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
-    <span class="n">already_standard</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="o">**</span><span class="n">kwargs</span><span class="p">,</span>
-<span class="p">):</span>
-    <span class="nb">super</span><span class="p">(</span><span class="n">ChunkTabPreprocessor</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span>
-        <span class="n">cat_embed_cols</span><span class="o">=</span><span class="n">cat_embed_cols</span><span class="p">,</span>
-        <span class="n">continuous_cols</span><span class="o">=</span><span class="n">continuous_cols</span><span class="p">,</span>
-        <span class="n">quantization_setup</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-        <span class="n">cols_to_scale</span><span class="o">=</span><span class="n">cols_to_scale</span><span class="p">,</span>
-        <span class="n">auto_embed_dim</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
-        <span class="n">embedding_rule</span><span class="o">=</span><span class="s2">&quot;google&quot;</span><span class="p">,</span>  <span class="c1"># does not matter, irrelevant</span>
-        <span class="n">default_embed_dim</span><span class="o">=</span><span class="n">default_embed_dim</span><span class="p">,</span>
-        <span class="n">with_attention</span><span class="o">=</span><span class="n">with_attention</span><span class="p">,</span>
-        <span class="n">with_cls_token</span><span class="o">=</span><span class="n">with_cls_token</span><span class="p">,</span>
-        <span class="n">shared_embed</span><span class="o">=</span><span class="n">shared_embed</span><span class="p">,</span>
-        <span class="n">verbose</span><span class="o">=</span><span class="n">verbose</span><span class="p">,</span>
-        <span class="n">scale</span><span class="o">=</span><span class="n">scale</span><span class="p">,</span>
-        <span class="n">already_standard</span><span class="o">=</span><span class="n">already_standard</span><span class="p">,</span>
+              <details class="quote">
+                <summary>Source code in <code>pytorch_widedeep/preprocessing/tab_preprocessor.py</code></summary>
+                <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 714</span>
+<span class="normal"> 715</span>
+<span class="normal"> 716</span>
+<span class="normal"> 717</span>
+<span class="normal"> 718</span>
+<span class="normal"> 719</span>
+<span class="normal"> 720</span>
+<span class="normal"> 721</span>
+<span class="normal"> 722</span>
+<span class="normal"> 723</span>
+<span class="normal"> 724</span>
+<span class="normal"> 725</span>
+<span class="normal"> 726</span>
+<span class="normal"> 727</span>
+<span class="normal"> 728</span>
+<span class="normal"> 729</span>
+<span class="normal"> 730</span>
+<span class="normal"> 731</span>
+<span class="normal"> 732</span>
+<span class="normal"> 733</span>
+<span class="normal"> 734</span>
+<span class="normal"> 735</span>
+<span class="normal"> 736</span>
+<span class="normal"> 737</span>
+<span class="normal"> 738</span>
+<span class="normal"> 739</span>
+<span class="normal"> 740</span>
+<span class="normal"> 741</span>
+<span class="normal"> 742</span>
+<span class="normal"> 743</span>
+<span class="normal"> 744</span>
+<span class="normal"> 745</span>
+<span class="normal"> 746</span>
+<span class="normal"> 747</span>
+<span class="normal"> 748</span>
+<span class="normal"> 749</span>
+<span class="normal"> 750</span>
+<span class="normal"> 751</span>
+<span class="normal"> 752</span>
+<span class="normal"> 753</span>
+<span class="normal"> 754</span>
+<span class="normal"> 755</span>
+<span class="normal"> 756</span>
+<span class="normal"> 757</span>
+<span class="normal"> 758</span>
+<span class="normal"> 759</span>
+<span class="normal"> 760</span>
+<span class="normal"> 761</span>
+<span class="normal"> 762</span>
+<span class="normal"> 763</span>
+<span class="normal"> 764</span>
+<span class="normal"> 765</span>
+<span class="normal"> 766</span>
+<span class="normal"> 767</span>
+<span class="normal"> 768</span>
+<span class="normal"> 769</span>
+<span class="normal"> 770</span>
+<span class="normal"> 771</span>
+<span class="normal"> 772</span>
+<span class="normal"> 773</span>
+<span class="normal"> 774</span>
+<span class="normal"> 775</span>
+<span class="normal"> 776</span>
+<span class="normal"> 777</span>
+<span class="normal"> 778</span>
+<span class="normal"> 779</span>
+<span class="normal"> 780</span>
+<span class="normal"> 781</span>
+<span class="normal"> 782</span>
+<span class="normal"> 783</span>
+<span class="normal"> 784</span>
+<span class="normal"> 785</span>
+<span class="normal"> 786</span>
+<span class="normal"> 787</span>
+<span class="normal"> 788</span>
+<span class="normal"> 789</span>
+<span class="normal"> 790</span>
+<span class="normal"> 791</span>
+<span class="normal"> 792</span>
+<span class="normal"> 793</span>
+<span class="normal"> 794</span>
+<span class="normal"> 795</span>
+<span class="normal"> 796</span>
+<span class="normal"> 797</span>
+<span class="normal"> 798</span>
+<span class="normal"> 799</span>
+<span class="normal"> 800</span>
+<span class="normal"> 801</span>
+<span class="normal"> 802</span>
+<span class="normal"> 803</span>
+<span class="normal"> 804</span>
+<span class="normal"> 805</span>
+<span class="normal"> 806</span>
+<span class="normal"> 807</span>
+<span class="normal"> 808</span>
+<span class="normal"> 809</span>
+<span class="normal"> 810</span>
+<span class="normal"> 811</span>
+<span class="normal"> 812</span>
+<span class="normal"> 813</span>
+<span class="normal"> 814</span>
+<span class="normal"> 815</span>
+<span class="normal"> 816</span>
+<span class="normal"> 817</span>
+<span class="normal"> 818</span>
+<span class="normal"> 819</span>
+<span class="normal"> 820</span>
+<span class="normal"> 821</span>
+<span class="normal"> 822</span>
+<span class="normal"> 823</span>
+<span class="normal"> 824</span>
+<span class="normal"> 825</span>
+<span class="normal"> 826</span>
+<span class="normal"> 827</span>
+<span class="normal"> 828</span>
+<span class="normal"> 829</span>
+<span class="normal"> 830</span>
+<span class="normal"> 831</span>
+<span class="normal"> 832</span>
+<span class="normal"> 833</span>
+<span class="normal"> 834</span>
+<span class="normal"> 835</span>
+<span class="normal"> 836</span>
+<span class="normal"> 837</span>
+<span class="normal"> 838</span>
+<span class="normal"> 839</span>
+<span class="normal"> 840</span>
+<span class="normal"> 841</span>
+<span class="normal"> 842</span>
+<span class="normal"> 843</span>
+<span class="normal"> 844</span>
+<span class="normal"> 845</span>
+<span class="normal"> 846</span>
+<span class="normal"> 847</span>
+<span class="normal"> 848</span>
+<span class="normal"> 849</span>
+<span class="normal"> 850</span>
+<span class="normal"> 851</span>
+<span class="normal"> 852</span>
+<span class="normal"> 853</span>
+<span class="normal"> 854</span>
+<span class="normal"> 855</span>
+<span class="normal"> 856</span>
+<span class="normal"> 857</span>
+<span class="normal"> 858</span>
+<span class="normal"> 859</span>
+<span class="normal"> 860</span>
+<span class="normal"> 861</span>
+<span class="normal"> 862</span>
+<span class="normal"> 863</span>
+<span class="normal"> 864</span>
+<span class="normal"> 865</span>
+<span class="normal"> 866</span>
+<span class="normal"> 867</span>
+<span class="normal"> 868</span>
+<span class="normal"> 869</span>
+<span class="normal"> 870</span>
+<span class="normal"> 871</span>
+<span class="normal"> 872</span>
+<span class="normal"> 873</span>
+<span class="normal"> 874</span>
+<span class="normal"> 875</span>
+<span class="normal"> 876</span>
+<span class="normal"> 877</span>
+<span class="normal"> 878</span>
+<span class="normal"> 879</span>
+<span class="normal"> 880</span>
+<span class="normal"> 881</span>
+<span class="normal"> 882</span>
+<span class="normal"> 883</span>
+<span class="normal"> 884</span>
+<span class="normal"> 885</span>
+<span class="normal"> 886</span>
+<span class="normal"> 887</span>
+<span class="normal"> 888</span>
+<span class="normal"> 889</span>
+<span class="normal"> 890</span>
+<span class="normal"> 891</span>
+<span class="normal"> 892</span>
+<span class="normal"> 893</span>
+<span class="normal"> 894</span>
+<span class="normal"> 895</span>
+<span class="normal"> 896</span>
+<span class="normal"> 897</span>
+<span class="normal"> 898</span>
+<span class="normal"> 899</span>
+<span class="normal"> 900</span>
+<span class="normal"> 901</span>
+<span class="normal"> 902</span>
+<span class="normal"> 903</span>
+<span class="normal"> 904</span>
+<span class="normal"> 905</span>
+<span class="normal"> 906</span>
+<span class="normal"> 907</span>
+<span class="normal"> 908</span>
+<span class="normal"> 909</span>
+<span class="normal"> 910</span>
+<span class="normal"> 911</span>
+<span class="normal"> 912</span>
+<span class="normal"> 913</span>
+<span class="normal"> 914</span>
+<span class="normal"> 915</span>
+<span class="normal"> 916</span>
+<span class="normal"> 917</span>
+<span class="normal"> 918</span>
+<span class="normal"> 919</span>
+<span class="normal"> 920</span>
+<span class="normal"> 921</span>
+<span class="normal"> 922</span>
+<span class="normal"> 923</span>
+<span class="normal"> 924</span>
+<span class="normal"> 925</span>
+<span class="normal"> 926</span>
+<span class="normal"> 927</span>
+<span class="normal"> 928</span>
+<span class="normal"> 929</span>
+<span class="normal"> 930</span>
+<span class="normal"> 931</span>
+<span class="normal"> 932</span>
+<span class="normal"> 933</span>
+<span class="normal"> 934</span>
+<span class="normal"> 935</span>
+<span class="normal"> 936</span>
+<span class="normal"> 937</span>
+<span class="normal"> 938</span>
+<span class="normal"> 939</span>
+<span class="normal"> 940</span>
+<span class="normal"> 941</span>
+<span class="normal"> 942</span>
+<span class="normal"> 943</span>
+<span class="normal"> 944</span>
+<span class="normal"> 945</span>
+<span class="normal"> 946</span>
+<span class="normal"> 947</span>
+<span class="normal"> 948</span>
+<span class="normal"> 949</span>
+<span class="normal"> 950</span>
+<span class="normal"> 951</span>
+<span class="normal"> 952</span>
+<span class="normal"> 953</span>
+<span class="normal"> 954</span>
+<span class="normal"> 955</span>
+<span class="normal"> 956</span>
+<span class="normal"> 957</span>
+<span class="normal"> 958</span>
+<span class="normal"> 959</span>
+<span class="normal"> 960</span>
+<span class="normal"> 961</span>
+<span class="normal"> 962</span>
+<span class="normal"> 963</span>
+<span class="normal"> 964</span>
+<span class="normal"> 965</span>
+<span class="normal"> 966</span>
+<span class="normal"> 967</span>
+<span class="normal"> 968</span>
+<span class="normal"> 969</span>
+<span class="normal"> 970</span>
+<span class="normal"> 971</span>
+<span class="normal"> 972</span>
+<span class="normal"> 973</span>
+<span class="normal"> 974</span>
+<span class="normal"> 975</span>
+<span class="normal"> 976</span>
+<span class="normal"> 977</span>
+<span class="normal"> 978</span>
+<span class="normal"> 979</span>
+<span class="normal"> 980</span>
+<span class="normal"> 981</span>
+<span class="normal"> 982</span>
+<span class="normal"> 983</span>
+<span class="normal"> 984</span>
+<span class="normal"> 985</span>
+<span class="normal"> 986</span>
+<span class="normal"> 987</span>
+<span class="normal"> 988</span>
+<span class="normal"> 989</span>
+<span class="normal"> 990</span>
+<span class="normal"> 991</span>
+<span class="normal"> 992</span>
+<span class="normal"> 993</span>
+<span class="normal"> 994</span>
+<span class="normal"> 995</span>
+<span class="normal"> 996</span>
+<span class="normal"> 997</span>
+<span class="normal"> 998</span>
+<span class="normal"> 999</span>
+<span class="normal">1000</span>
+<span class="normal">1001</span>
+<span class="normal">1002</span>
+<span class="normal">1003</span>
+<span class="normal">1004</span>
+<span class="normal">1005</span>
+<span class="normal">1006</span>
+<span class="normal">1007</span>
+<span class="normal">1008</span>
+<span class="normal">1009</span>
+<span class="normal">1010</span>
+<span class="normal">1011</span>
+<span class="normal">1012</span>
+<span class="normal">1013</span>
+<span class="normal">1014</span>
+<span class="normal">1015</span>
+<span class="normal">1016</span>
+<span class="normal">1017</span>
+<span class="normal">1018</span>
+<span class="normal">1019</span>
+<span class="normal">1020</span>
+<span class="normal">1021</span>
+<span class="normal">1022</span>
+<span class="normal">1023</span>
+<span class="normal">1024</span>
+<span class="normal">1025</span>
+<span class="normal">1026</span>
+<span class="normal">1027</span>
+<span class="normal">1028</span>
+<span class="normal">1029</span>
+<span class="normal">1030</span>
+<span class="normal">1031</span>
+<span class="normal">1032</span>
+<span class="normal">1033</span>
+<span class="normal">1034</span>
+<span class="normal">1035</span>
+<span class="normal">1036</span>
+<span class="normal">1037</span>
+<span class="normal">1038</span>
+<span class="normal">1039</span>
+<span class="normal">1040</span>
+<span class="normal">1041</span>
+<span class="normal">1042</span>
+<span class="normal">1043</span>
+<span class="normal">1044</span>
+<span class="normal">1045</span>
+<span class="normal">1046</span>
+<span class="normal">1047</span>
+<span class="normal">1048</span>
+<span class="normal">1049</span>
+<span class="normal">1050</span>
+<span class="normal">1051</span>
+<span class="normal">1052</span>
+<span class="normal">1053</span>
+<span class="normal">1054</span>
+<span class="normal">1055</span>
+<span class="normal">1056</span>
+<span class="normal">1057</span>
+<span class="normal">1058</span>
+<span class="normal">1059</span>
+<span class="normal">1060</span>
+<span class="normal">1061</span>
+<span class="normal">1062</span>
+<span class="normal">1063</span>
+<span class="normal">1064</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">class</span> <span class="nc">ChunkTabPreprocessor</span><span class="p">(</span><span class="n">TabPreprocessor</span><span class="p">):</span>
+<span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Preprocessor to prepare the `deeptabular` component input dataset</span>
+
+<span class="sd">    Parameters</span>
+<span class="sd">    ----------</span>
+<span class="sd">    n_chunks: int</span>
+<span class="sd">        Number of chunks that the tabular dataset is divided by.</span>
+<span class="sd">    cat_embed_cols: List, default = None</span>
+<span class="sd">        List containing the name of the categorical columns that will be</span>
+<span class="sd">        represented by embeddings (e.g. _[&#39;education&#39;, &#39;relationship&#39;, ...]_) or</span>
+<span class="sd">        a Tuple with the name and the embedding dimension (e.g.: _[</span>
+<span class="sd">        (&#39;education&#39;,32), (&#39;relationship&#39;,16), ...]_). &lt;br/&gt; Param alias:</span>
+<span class="sd">        `embed_cols`</span>
+<span class="sd">    continuous_cols: List, default = None</span>
+<span class="sd">        List with the name of the continuous cols</span>
+<span class="sd">    cols_and_bins: Dict, default = None</span>
+<span class="sd">        Continuous columns can be turned into categorical via</span>
+<span class="sd">        `pd.cut`. &#39;cols_and_bins&#39; is dictionary where the keys are the column</span>
+<span class="sd">        names to quantize and the values are a list of scalars indicating the</span>
+<span class="sd">        bin edges. &lt;br/&gt; Param alias: `quantization_setup`</span>
+<span class="sd">    cols_to_scale: List, default = None,</span>
+<span class="sd">        List with the names of the columns that will be standarised via</span>
+<span class="sd">        sklearn&#39;s `StandardScaler`</span>
+<span class="sd">    default_embed_dim: int, default=16</span>
+<span class="sd">        Dimension for the embeddings if the embed_dim is not provided in the</span>
+<span class="sd">        `cat_embed_cols` parameter and `auto_embed_dim` is set to</span>
+<span class="sd">        `False`.</span>
+<span class="sd">    with_attention: bool, default = False</span>
+<span class="sd">        Boolean indicating whether the preprocessed data will be passed to an</span>
+<span class="sd">        attention-based model (more precisely a model where all embeddings</span>
+<span class="sd">        must have the same dimensions). If `True`, the param `cat_embed_cols`</span>
+<span class="sd">        must just be a list containing just the categorical column names:</span>
+<span class="sd">        e.g.</span>
+<span class="sd">        _[&#39;education&#39;, &#39;relationship&#39;, ...]_. This is because they will all be</span>
+<span class="sd">         encoded using embeddings of the same dim, which will be specified</span>
+<span class="sd">         later when the model is defined. &lt;br/&gt; Param alias:</span>
+<span class="sd">         `for_transformer`</span>
+<span class="sd">    with_cls_token: bool, default = False</span>
+<span class="sd">        Boolean indicating if a `&#39;[CLS]&#39;` token will be added to the dataset</span>
+<span class="sd">        when using attention-based models. The final hidden state</span>
+<span class="sd">        corresponding to this token is used as the aggregated representation</span>
+<span class="sd">        for classification and regression tasks. If not, the categorical</span>
+<span class="sd">        (and continuous embeddings if present) will be concatenated before</span>
+<span class="sd">        being passed to the final MLP (if present).</span>
+<span class="sd">    shared_embed: bool, default = False</span>
+<span class="sd">        Boolean indicating if the embeddings will be &quot;shared&quot; when using</span>
+<span class="sd">        attention-based models. The idea behind `shared_embed` is</span>
+<span class="sd">        described in the Appendix A in the [TabTransformer paper](https://arxiv.org/abs/2012.06678):</span>
+<span class="sd">        _&#39;The goal of having column embedding is to enable the model to</span>
+<span class="sd">        distinguish the classes in one column from those in the other</span>
+<span class="sd">        columns&#39;_. In other words, the idea is to let the model learn which</span>
+<span class="sd">        column is embedded at the time. See: `pytorch_widedeep.models.transformers._layers.SharedEmbeddings`.</span>
+<span class="sd">    verbose: int, default = 1</span>
+<span class="sd">    scale: bool, default = False</span>
+<span class="sd">        :information_source: **note**: this arg will be removed in upcoming</span>
+<span class="sd">         releases. Please use `cols_to_scale` instead. &lt;br/&gt; Bool indicating</span>
+<span class="sd">         whether or not to scale/standarise continuous cols. It is important</span>
+<span class="sd">         to emphasize that all the DL models for tabular data in the library</span>
+<span class="sd">         also include the possibility of normalising the input continuous</span>
+<span class="sd">         features via a `BatchNorm` or a `LayerNorm`. &lt;br/&gt; Param alias:</span>
+<span class="sd">         `scale_cont_cols`.</span>
+<span class="sd">    already_standard: List, default = None</span>
+<span class="sd">        :information_source: **note**: this arg will be removed in upcoming</span>
+<span class="sd">         releases. Please use `cols_to_scale` instead. &lt;br/&gt; List with the</span>
+<span class="sd">         name of the continuous cols that do not need to be</span>
+<span class="sd">         scaled/standarised.</span>
+
+<span class="sd">    Other Parameters</span>
+<span class="sd">    ----------------</span>
+<span class="sd">    **kwargs: dict</span>
+<span class="sd">        `pd.cut` and `StandardScaler` related args</span>
+
+<span class="sd">    Attributes</span>
+<span class="sd">    ----------</span>
+<span class="sd">    cat_cols: List[str]</span>
+<span class="sd">        List containing the names of the categorical columns after processing.</span>
+<span class="sd">    cols_and_bins: Optional[Dict[str, Union[int, List[float]]]]</span>
+<span class="sd">        Dictionary containing the quantization setup after processing if</span>
+<span class="sd">        quantization is requested. This is derived from the</span>
+<span class="sd">        quantization_setup parameter.</span>
+<span class="sd">    quant_args: Dict</span>
+<span class="sd">        Dictionary containing arguments passed to pandas.cut() function for quantization.</span>
+<span class="sd">    scale_args: Dict</span>
+<span class="sd">        Dictionary containing arguments passed to StandardScaler.</span>
+<span class="sd">    label_encoder: LabelEncoder</span>
+<span class="sd">        Instance of LabelEncoder used to encode categorical variables.</span>
+<span class="sd">        See `pytorch_widedeep.utils.dense_utils.LabelEncoder`.</span>
+<span class="sd">    cat_embed_input: List</span>
+<span class="sd">        List of tuples with the column name, number of individual values for</span>
+<span class="sd">        that column and, if `with_attention` is set to `False`, the</span>
+<span class="sd">        corresponding embeddings dim, e.g. _[(&#39;education&#39;, 16, 10),</span>
+<span class="sd">        (&#39;relationship&#39;, 6, 8), ...]_.</span>
+<span class="sd">    standardize_cols: List</span>
+<span class="sd">        List of the columns that will be standardized.</span>
+<span class="sd">    scaler: StandardScaler</span>
+<span class="sd">        An instance of `sklearn.preprocessing.StandardScaler` if standardization</span>
+<span class="sd">        is requested.</span>
+<span class="sd">    column_idx: Dict</span>
+<span class="sd">        Dictionary where keys are column names and values are column indexes.</span>
+<span class="sd">        This is necessary to slice tensors.</span>
+<span class="sd">    quantizer: Quantizer</span>
+<span class="sd">        An instance of `Quantizer` if quantization is requested.</span>
+<span class="sd">    is_fitted: bool</span>
+<span class="sd">        Boolean indicating if the preprocessor has been fitted.</span>
+
+<span class="sd">    Examples</span>
+<span class="sd">    --------</span>
+<span class="sd">    &gt;&gt;&gt; import pandas as pd</span>
+<span class="sd">    &gt;&gt;&gt; import numpy as np</span>
+<span class="sd">    &gt;&gt;&gt; from pytorch_widedeep.preprocessing import ChunkTabPreprocessor</span>
+<span class="sd">    &gt;&gt;&gt; np.random.seed(42)</span>
+<span class="sd">    &gt;&gt;&gt; chunk_df = pd.DataFrame({&#39;cat_col&#39;: np.random.choice([&#39;A&#39;, &#39;B&#39;, &#39;C&#39;], size=8),</span>
+<span class="sd">    ... &#39;cont_col&#39;: np.random.uniform(1, 100, size=8)})</span>
+<span class="sd">    &gt;&gt;&gt; cat_embed_cols = [(&#39;cat_col&#39;,4)]</span>
+<span class="sd">    &gt;&gt;&gt; cont_cols = [&#39;cont_col&#39;]</span>
+<span class="sd">    &gt;&gt;&gt; tab_preprocessor = ChunkTabPreprocessor(</span>
+<span class="sd">    ... n_chunks=1, cat_embed_cols=cat_embed_cols, continuous_cols=cont_cols</span>
+<span class="sd">    ... )</span>
+<span class="sd">    &gt;&gt;&gt; X_tab = tab_preprocessor.fit_transform(chunk_df)</span>
+<span class="sd">    &gt;&gt;&gt; tab_preprocessor.cat_embed_cols</span>
+<span class="sd">    [(&#39;cat_col&#39;, 4)]</span>
+<span class="sd">    &gt;&gt;&gt; tab_preprocessor.column_idx</span>
+<span class="sd">    {&#39;cat_col&#39;: 0, &#39;cont_col&#39;: 1}</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+
+    <span class="nd">@alias</span><span class="p">(</span><span class="s2">&quot;with_attention&quot;</span><span class="p">,</span> <span class="p">[</span><span class="s2">&quot;for_transformer&quot;</span><span class="p">])</span>
+    <span class="nd">@alias</span><span class="p">(</span><span class="s2">&quot;cat_embed_cols&quot;</span><span class="p">,</span> <span class="p">[</span><span class="s2">&quot;embed_cols&quot;</span><span class="p">])</span>
+    <span class="nd">@alias</span><span class="p">(</span><span class="s2">&quot;scale&quot;</span><span class="p">,</span> <span class="p">[</span><span class="s2">&quot;scale_cont_cols&quot;</span><span class="p">])</span>
+    <span class="nd">@alias</span><span class="p">(</span><span class="s2">&quot;cols_and_bins&quot;</span><span class="p">,</span> <span class="p">[</span><span class="s2">&quot;quantization_setup&quot;</span><span class="p">])</span>
+    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span>
+        <span class="n">n_chunks</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span>
+        <span class="n">cat_embed_cols</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Union</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">],</span> <span class="n">List</span><span class="p">[</span><span class="n">Tuple</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">]]]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">continuous_cols</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">cols_and_bins</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="nb">float</span><span class="p">]]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">cols_to_scale</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Union</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">],</span> <span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">default_embed_dim</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">16</span><span class="p">,</span>
+        <span class="n">with_attention</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
+        <span class="n">with_cls_token</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
+        <span class="n">shared_embed</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
+        <span class="n">verbose</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1</span><span class="p">,</span>
+        <span class="o">*</span><span class="p">,</span>
+        <span class="n">scale</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
+        <span class="n">already_standard</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
         <span class="o">**</span><span class="n">kwargs</span><span class="p">,</span>
-    <span class="p">)</span>
+    <span class="p">):</span>
+        <span class="nb">super</span><span class="p">(</span><span class="n">ChunkTabPreprocessor</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span>
+            <span class="n">cat_embed_cols</span><span class="o">=</span><span class="n">cat_embed_cols</span><span class="p">,</span>
+            <span class="n">continuous_cols</span><span class="o">=</span><span class="n">continuous_cols</span><span class="p">,</span>
+            <span class="n">quantization_setup</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+            <span class="n">cols_to_scale</span><span class="o">=</span><span class="n">cols_to_scale</span><span class="p">,</span>
+            <span class="n">auto_embed_dim</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
+            <span class="n">embedding_rule</span><span class="o">=</span><span class="s2">&quot;google&quot;</span><span class="p">,</span>  <span class="c1"># does not matter, irrelevant</span>
+            <span class="n">default_embed_dim</span><span class="o">=</span><span class="n">default_embed_dim</span><span class="p">,</span>
+            <span class="n">with_attention</span><span class="o">=</span><span class="n">with_attention</span><span class="p">,</span>
+            <span class="n">with_cls_token</span><span class="o">=</span><span class="n">with_cls_token</span><span class="p">,</span>
+            <span class="n">shared_embed</span><span class="o">=</span><span class="n">shared_embed</span><span class="p">,</span>
+            <span class="n">verbose</span><span class="o">=</span><span class="n">verbose</span><span class="p">,</span>
+            <span class="n">scale</span><span class="o">=</span><span class="n">scale</span><span class="p">,</span>
+            <span class="n">already_standard</span><span class="o">=</span><span class="n">already_standard</span><span class="p">,</span>
+            <span class="o">**</span><span class="n">kwargs</span><span class="p">,</span>
+        <span class="p">)</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">n_chunks</span> <span class="o">=</span> <span class="n">n_chunks</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">chunk_counter</span> <span class="o">=</span> <span class="mi">0</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">cols_and_bins</span> <span class="o">=</span> <span class="n">cols_and_bins</span>  <span class="c1"># type: ignore[assignment]</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">cols_and_bins</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">quantizer</span> <span class="o">=</span> <span class="n">Quantizer</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">cols_and_bins</span><span class="p">,</span> <span class="o">**</span><span class="bp">self</span><span class="o">.</span><span class="n">quant_args</span><span class="p">)</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">embed_prepared</span> <span class="o">=</span> <span class="kc">False</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">continuous_prepared</span> <span class="o">=</span> <span class="kc">False</span>
+
+    <span class="k">def</span> <span class="nf">partial_fit</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="s2">&quot;ChunkTabPreprocessor&quot;</span><span class="p">:</span>  <span class="c1"># noqa: C901</span>
+        <span class="c1"># df here, and throughout the class, is a chunk of the original df</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">chunk_counter</span> <span class="o">+=</span> <span class="mi">1</span>
+
+        <span class="n">df_adj</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_insert_cls_token</span><span class="p">(</span><span class="n">df</span><span class="p">)</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">with_cls_token</span> <span class="k">else</span> <span class="n">df</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">column_idx</span><span class="p">:</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="p">{}</span>
+
+        <span class="c1"># Categorical embeddings logic</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">cat_embed_cols</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">embed_prepared</span><span class="p">:</span>
+                <span class="n">df_cat</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">cat_embed_dim</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_prepare_categorical</span><span class="p">(</span><span class="n">df_adj</span><span class="p">)</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">label_encoder</span> <span class="o">=</span> <span class="n">LabelEncoder</span><span class="p">(</span>
+                    <span class="n">columns_to_encode</span><span class="o">=</span><span class="n">df_cat</span><span class="o">.</span><span class="n">columns</span><span class="o">.</span><span class="n">tolist</span><span class="p">(),</span>
+                    <span class="n">shared_embed</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">shared_embed</span><span class="p">,</span>
+                    <span class="n">with_attention</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">with_attention</span><span class="p">,</span>
+                <span class="p">)</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">label_encoder</span><span class="o">.</span><span class="n">partial_fit</span><span class="p">(</span><span class="n">df_cat</span><span class="p">)</span>
+            <span class="k">else</span><span class="p">:</span>
+                <span class="n">df_cat</span> <span class="o">=</span> <span class="n">df_adj</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">cat_cols</span><span class="p">]</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">label_encoder</span><span class="o">.</span><span class="n">partial_fit</span><span class="p">(</span><span class="n">df_cat</span><span class="p">)</span>
+
+            <span class="bp">self</span><span class="o">.</span><span class="n">column_idx</span><span class="o">.</span><span class="n">update</span><span class="p">({</span><span class="n">k</span><span class="p">:</span> <span class="n">v</span> <span class="k">for</span> <span class="n">v</span><span class="p">,</span> <span class="n">k</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="n">df_cat</span><span class="o">.</span><span class="n">columns</span><span class="p">)})</span>
+
+        <span class="c1"># Continuous columns logic</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">continuous_cols</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">continuous_prepared</span><span class="p">:</span>
+                <span class="n">df_cont</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">cont_embed_dim</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_prepare_continuous</span><span class="p">(</span><span class="n">df_adj</span><span class="p">)</span>
+            <span class="k">else</span><span class="p">:</span>
+                <span class="n">df_cont</span> <span class="o">=</span> <span class="n">df</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">continuous_cols</span><span class="p">]</span>
+
+            <span class="bp">self</span><span class="o">.</span><span class="n">column_idx</span><span class="o">.</span><span class="n">update</span><span class="p">(</span>
+                <span class="p">{</span><span class="n">k</span><span class="p">:</span> <span class="n">v</span> <span class="o">+</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">column_idx</span><span class="p">)</span> <span class="k">for</span> <span class="n">v</span><span class="p">,</span> <span class="n">k</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="n">df_cont</span><span class="o">.</span><span class="n">columns</span><span class="p">)}</span>
+            <span class="p">)</span>
+
+            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">standardize_cols</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">scaler</span><span class="o">.</span><span class="n">partial_fit</span><span class="p">(</span><span class="n">df_cont</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">standardize_cols</span><span class="p">]</span><span class="o">.</span><span class="n">values</span><span class="p">)</span>
+
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">chunk_counter</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">n_chunks</span><span class="p">:</span>
+            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">cat_embed_cols</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">or</span> <span class="bp">self</span><span class="o">.</span><span class="n">cols_and_bins</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">cat_embed_input</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span>
+                    <span class="n">Union</span><span class="p">[</span><span class="n">Tuple</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">],</span> <span class="n">Tuple</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">,</span> <span class="nb">int</span><span class="p">]]</span>
+                <span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
+
+            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">cat_embed_cols</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+                <span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">label_encoder</span><span class="o">.</span><span class="n">encoding_dict</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
+                    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">with_attention</span><span class="p">:</span>
+                        <span class="bp">self</span><span class="o">.</span><span class="n">cat_embed_input</span><span class="o">.</span><span class="n">append</span><span class="p">((</span><span class="n">k</span><span class="p">,</span> <span class="nb">len</span><span class="p">(</span><span class="n">v</span><span class="p">)))</span>
+                    <span class="k">else</span><span class="p">:</span>
+                        <span class="bp">self</span><span class="o">.</span><span class="n">cat_embed_input</span><span class="o">.</span><span class="n">append</span><span class="p">((</span><span class="n">k</span><span class="p">,</span> <span class="nb">len</span><span class="p">(</span><span class="n">v</span><span class="p">),</span> <span class="bp">self</span><span class="o">.</span><span class="n">cat_embed_dim</span><span class="p">[</span><span class="n">k</span><span class="p">]))</span>
+
+            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">cols_and_bins</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+                <span class="k">assert</span> <span class="bp">self</span><span class="o">.</span><span class="n">cont_embed_dim</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span>  <span class="c1"># just to make mypy happy</span>
+                <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">with_attention</span><span class="p">:</span>
+                    <span class="k">for</span> <span class="n">col</span><span class="p">,</span> <span class="n">n_cat</span><span class="p">,</span> <span class="n">_</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">cont_embed_dim</span><span class="p">:</span>
+                        <span class="bp">self</span><span class="o">.</span><span class="n">cat_embed_input</span><span class="o">.</span><span class="n">append</span><span class="p">((</span><span class="n">col</span><span class="p">,</span> <span class="n">n_cat</span><span class="p">))</span>
+                <span class="k">else</span><span class="p">:</span>
+                    <span class="bp">self</span><span class="o">.</span><span class="n">cat_embed_input</span><span class="o">.</span><span class="n">extend</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">cont_embed_dim</span><span class="p">)</span>
+
+            <span class="bp">self</span><span class="o">.</span><span class="n">is_fitted</span> <span class="o">=</span> <span class="kc">True</span>
+
+        <span class="k">return</span> <span class="bp">self</span>
+
+    <span class="k">def</span> <span class="nf">fit</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="s2">&quot;ChunkTabPreprocessor&quot;</span><span class="p">:</span>
+        <span class="c1"># just to override the fit method in the base class. This class is not</span>
+        <span class="c1"># designed or thought to run fit</span>
+        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">partial_fit</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">_prepare_categorical</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span>
+    <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tuple</span><span class="p">[</span><span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">,</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">]]:</span>
+        <span class="c1"># When dealing with chunks we will NOT support the option of</span>
+        <span class="c1"># automatically define embeddings as this implies going through the</span>
+        <span class="c1"># entire dataset</span>
+        <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">cat_embed_cols</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="nb">tuple</span><span class="p">):</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">cat_cols</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[</span><span class="n">emb</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="k">for</span> <span class="n">emb</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">cat_embed_cols</span><span class="p">]</span>
+            <span class="n">cat_embed_dim</span><span class="p">:</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">cat_embed_cols</span><span class="p">)</span>  <span class="c1"># type: ignore</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">cat_cols</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">cat_embed_cols</span>  <span class="c1"># type: ignore[assignment]</span>
+            <span class="n">cat_embed_dim</span> <span class="o">=</span> <span class="p">{</span>
+                <span class="n">e</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">default_embed_dim</span> <span class="k">for</span> <span class="n">e</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">cat_embed_cols</span>  <span class="c1"># type: ignore[misc]</span>
+            <span class="p">}</span>  <span class="c1"># type: ignore</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">embed_prepared</span> <span class="o">=</span> <span class="kc">True</span>
+
+        <span class="k">return</span> <span class="n">df</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">cat_cols</span><span class="p">],</span> <span class="n">cat_embed_dim</span>
+
+    <span class="k">def</span> <span class="nf">_prepare_continuous</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span>
+    <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tuple</span><span class="p">[</span><span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">,</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="n">Tuple</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">,</span> <span class="nb">int</span><span class="p">]]]]:</span>
+        <span class="c1"># Standardization logic</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">cols_to_scale</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">standardize_cols</span> <span class="o">=</span> <span class="p">(</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">cols_to_scale</span>
+                <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">cols_to_scale</span> <span class="o">!=</span> <span class="s2">&quot;all&quot;</span>
+                <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">continuous_cols</span>
+            <span class="p">)</span>
+        <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">scale</span><span class="p">:</span>
+            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">already_standard</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">standardize_cols</span> <span class="o">=</span> <span class="p">[</span>
+                    <span class="n">c</span>
+                    <span class="k">for</span> <span class="n">c</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">continuous_cols</span>  <span class="c1"># type: ignore[misc]</span>
+                    <span class="k">if</span> <span class="n">c</span> <span class="ow">not</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">already_standard</span>
+                <span class="p">]</span>
+            <span class="k">else</span><span class="p">:</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">standardize_cols</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">continuous_cols</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">standardize_cols</span> <span class="o">=</span> <span class="kc">None</span>
+
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">standardize_cols</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">scaler</span> <span class="o">=</span> <span class="n">StandardScaler</span><span class="p">(</span><span class="o">**</span><span class="bp">self</span><span class="o">.</span><span class="n">scale_args</span><span class="p">)</span>
+        <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">verbose</span><span class="p">:</span>
+            <span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span><span class="s2">&quot;Continuous columns will not be normalised&quot;</span><span class="p">)</span>
+
+        <span class="c1"># Quantization logic</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">cols_and_bins</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="c1"># the quantized columns are then treated as categorical</span>
+            <span class="n">quant_cont_embed_input</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="n">Tuple</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">,</span> <span class="nb">int</span><span class="p">]]]</span> <span class="o">=</span> <span class="p">[]</span>
+            <span class="k">for</span> <span class="n">col</span><span class="p">,</span> <span class="n">val</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">cols_and_bins</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
+                <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">val</span><span class="p">,</span> <span class="nb">int</span><span class="p">):</span>
+                    <span class="n">quant_cont_embed_input</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
+                        <span class="p">(</span>
+                            <span class="n">col</span><span class="p">,</span>
+                            <span class="n">val</span><span class="p">,</span>
+                            <span class="bp">self</span><span class="o">.</span><span class="n">default_embed_dim</span><span class="p">,</span>
+                        <span class="p">)</span>
+                    <span class="p">)</span>
+                <span class="k">else</span><span class="p">:</span>
+                    <span class="n">quant_cont_embed_input</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
+                        <span class="p">(</span>
+                            <span class="n">col</span><span class="p">,</span>
+                            <span class="nb">len</span><span class="p">(</span><span class="n">val</span><span class="p">)</span> <span class="o">-</span> <span class="mi">1</span><span class="p">,</span>
+                            <span class="bp">self</span><span class="o">.</span><span class="n">default_embed_dim</span><span class="p">,</span>
+                        <span class="p">)</span>
+                    <span class="p">)</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="n">quant_cont_embed_input</span> <span class="o">=</span> <span class="kc">None</span>
 
-    <span class="bp">self</span><span class="o">.</span><span class="n">n_chunks</span> <span class="o">=</span> <span class="n">n_chunks</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">chunk_counter</span> <span class="o">=</span> <span class="mi">0</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">continuous_prepared</span> <span class="o">=</span> <span class="kc">True</span>
 
-    <span class="bp">self</span><span class="o">.</span><span class="n">cols_and_bins</span> <span class="o">=</span> <span class="n">cols_and_bins</span>  <span class="c1"># type: ignore[assignment]</span>
-    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">cols_and_bins</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">quantizer</span> <span class="o">=</span> <span class="n">Quantizer</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">cols_and_bins</span><span class="p">,</span> <span class="o">**</span><span class="bp">self</span><span class="o">.</span><span class="n">quant_args</span><span class="p">)</span>
+        <span class="k">return</span> <span class="n">df</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">continuous_cols</span><span class="p">],</span> <span class="n">quant_cont_embed_input</span>
 
-    <span class="bp">self</span><span class="o">.</span><span class="n">embed_prepared</span> <span class="o">=</span> <span class="kc">False</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">continuous_prepared</span> <span class="o">=</span> <span class="kc">False</span>
+    <span class="k">def</span> <span class="fm">__repr__</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">str</span><span class="p">:</span>  <span class="c1"># noqa: C901</span>
+        <span class="n">list_of_params</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">n_chunks</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="n">list_of_params</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">&quot;n_chunks=</span><span class="si">{n_chunks}</span><span class="s2">&quot;</span><span class="p">)</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">cat_embed_cols</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="n">list_of_params</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">&quot;cat_embed_cols=</span><span class="si">{cat_embed_cols}</span><span class="s2">&quot;</span><span class="p">)</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">continuous_cols</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="n">list_of_params</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">&quot;continuous_cols=</span><span class="si">{continuous_cols}</span><span class="s2">&quot;</span><span class="p">)</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">cols_and_bins</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="n">list_of_params</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">&quot;cols_and_bins=</span><span class="si">{cols_and_bins}</span><span class="s2">&quot;</span><span class="p">)</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">cols_to_scale</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="n">list_of_params</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">&quot;cols_to_scale=</span><span class="si">{cols_to_scale}</span><span class="s2">&quot;</span><span class="p">)</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">default_embed_dim</span> <span class="o">!=</span> <span class="mi">16</span><span class="p">:</span>
+            <span class="n">list_of_params</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">&quot;default_embed_dim=</span><span class="si">{default_embed_dim}</span><span class="s2">&quot;</span><span class="p">)</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">with_attention</span><span class="p">:</span>
+            <span class="n">list_of_params</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">&quot;with_attention=</span><span class="si">{with_attention}</span><span class="s2">&quot;</span><span class="p">)</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">with_cls_token</span><span class="p">:</span>
+            <span class="n">list_of_params</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">&quot;with_cls_token=</span><span class="si">{with_cls_token}</span><span class="s2">&quot;</span><span class="p">)</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">shared_embed</span><span class="p">:</span>
+            <span class="n">list_of_params</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">&quot;shared_embed=</span><span class="si">{shared_embed}</span><span class="s2">&quot;</span><span class="p">)</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">verbose</span> <span class="o">!=</span> <span class="mi">1</span><span class="p">:</span>
+            <span class="n">list_of_params</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">&quot;verbose=</span><span class="si">{verbose}</span><span class="s2">&quot;</span><span class="p">)</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">scale</span><span class="p">:</span>
+            <span class="n">list_of_params</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">&quot;scale=</span><span class="si">{scale}</span><span class="s2">&quot;</span><span class="p">)</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">already_standard</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="n">list_of_params</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">&quot;already_standard=</span><span class="si">{already_standard}</span><span class="s2">&quot;</span><span class="p">)</span>
+        <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">quant_args</span><span class="p">)</span> <span class="o">&gt;</span> <span class="mi">0</span><span class="p">:</span>
+            <span class="n">list_of_params</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
+                <span class="s2">&quot;, &quot;</span><span class="o">.</span><span class="n">join</span><span class="p">([</span><span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="n">k</span><span class="si">}</span><span class="s2">&quot;</span> <span class="o">+</span> <span class="s2">&quot;=&quot;</span> <span class="o">+</span> <span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="n">v</span><span class="si">}</span><span class="s2">&quot;</span> <span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">quant_args</span><span class="o">.</span><span class="n">items</span><span class="p">()])</span>
+            <span class="p">)</span>
+        <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">scale_args</span><span class="p">)</span> <span class="o">&gt;</span> <span class="mi">0</span><span class="p">:</span>
+            <span class="n">list_of_params</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
+                <span class="s2">&quot;, &quot;</span><span class="o">.</span><span class="n">join</span><span class="p">([</span><span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="n">k</span><span class="si">}</span><span class="s2">&quot;</span> <span class="o">+</span> <span class="s2">&quot;=&quot;</span> <span class="o">+</span> <span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="n">v</span><span class="si">}</span><span class="s2">&quot;</span> <span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">scale_args</span><span class="o">.</span><span class="n">items</span><span class="p">()])</span>
+            <span class="p">)</span>
+        <span class="n">all_params</span> <span class="o">=</span> <span class="s2">&quot;, &quot;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">list_of_params</span><span class="p">)</span>
+        <span class="k">return</span> <span class="sa">f</span><span class="s2">&quot;ChunkTabPreprocessor(</span><span class="si">{</span><span class="n">all_params</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="bp">self</span><span class="o">.</span><span class="vm">__dict__</span><span class="p">)</span><span class="si">}</span><span class="s2">)&quot;</span>
 </code></pre></div></td></tr></table></div>
-                  </details>
+              </details>
 
 
 
@@ -6454,21 +12651,7 @@ <h2 id="pytorch_widedeep.preprocessing.text_preprocessor.ChunkTextPreprocessor"
 
 
 <a href="#pytorch_widedeep.preprocessing.text_preprocessor.ChunkTextPreprocessor" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="nf">ChunkTextPreprocessor</span><span class="p">(</span>
-    <span class="n">text_col</span><span class="p">,</span>
-    <span class="n">n_chunks</span><span class="p">,</span>
-    <span class="n">root_dir</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">max_vocab</span><span class="o">=</span><span class="mi">30000</span><span class="p">,</span>
-    <span class="n">min_freq</span><span class="o">=</span><span class="mi">5</span><span class="p">,</span>
-    <span class="n">maxlen</span><span class="o">=</span><span class="mi">80</span><span class="p">,</span>
-    <span class="n">pad_first</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
-    <span class="n">pad_idx</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span>
-    <span class="n">already_processed</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
-    <span class="n">word_vectors_path</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">n_cpus</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">verbose</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span>
-<span class="p">)</span>
-</code></pre></div>
+
 
     <div class="doc doc-contents first">
             <p class="doc doc-class-bases">
@@ -6479,138 +12662,211 @@ <h2 id="pytorch_widedeep.preprocessing.text_preprocessor.ChunkTextPreprocessor"
 
 
 <p><span class="doc-section-title">Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>text_col</code></b>
-              (<code>str</code>)
-          –
-          <div class="doc-md-description">
-            <p>column in the input dataframe containing either the texts or the
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+          <th>Default</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>text_col</code></td>
+            <td>
+                  <code>str</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>column in the input dataframe containing either the texts or the
 filenames where the text documents are stored</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>n_chunks</code></b>
-              (<code>int</code>)
-          –
-          <div class="doc-md-description">
-            <p>Number of chunks that the text dataset is divided by.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>root_dir</code></b>
-              (<code><span title="typing.Optional">Optional</span>[str]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>If 'text_col' contains the filenames with the text documents, this is
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>n_chunks</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Number of chunks that the text dataset is divided by.</p>
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>root_dir</code></td>
+            <td>
+                  <code><span title="typing.Optional">Optional</span>[str]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>If 'text_col' contains the filenames with the text documents, this is
 the path to the directory where those documents are stored.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>max_vocab</code></b>
-              (<code>int</code>, default:
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>max_vocab</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Maximum number of tokens in the vocabulary</p>
+              </div>
+            </td>
+            <td>
                   <code>30000</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Maximum number of tokens in the vocabulary</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>min_freq</code></b>
-              (<code>int</code>, default:
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>min_freq</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Minimum frequency for a token to be part of the vocabulary</p>
+              </div>
+            </td>
+            <td>
                   <code>5</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Minimum frequency for a token to be part of the vocabulary</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>maxlen</code></b>
-              (<code>int</code>, default:
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>maxlen</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Maximum length of the tokenized sequences</p>
+              </div>
+            </td>
+            <td>
                   <code>80</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Maximum length of the tokenized sequences</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>pad_first</code></b>
-              (<code>bool</code>, default:
-                  <code>True</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Indicates whether the padding index will be added at the beginning or the
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>pad_first</code></td>
+            <td>
+                  <code>bool</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Indicates whether the padding index will be added at the beginning or the
 end of the sequences</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>pad_idx</code></b>
-              (<code>int</code>, default:
+              </div>
+            </td>
+            <td>
+                  <code>True</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>pad_idx</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>padding index. Fastai's Tokenizer leaves 0 for the 'unknown' token.</p>
+              </div>
+            </td>
+            <td>
                   <code>1</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>padding index. Fastai's Tokenizer leaves 0 for the 'unknown' token.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>word_vectors_path</code></b>
-              (<code><span title="typing.Optional">Optional</span>[str]</code>, default:
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>word_vectors_path</code></td>
+            <td>
+                  <code><span title="typing.Optional">Optional</span>[str]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Path to the pretrained word vectors</p>
+              </div>
+            </td>
+            <td>
                   <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Path to the pretrained word vectors</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>n_cpus</code></b>
-              (<code><span title="typing.Optional">Optional</span>[int]</code>, default:
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>n_cpus</code></td>
+            <td>
+                  <code><span title="typing.Optional">Optional</span>[int]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>number of CPUs to used during the tokenization process</p>
+              </div>
+            </td>
+            <td>
                   <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>number of CPUs to used during the tokenization process</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>verbose</code></b>
-              (<code>int</code>, default:
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>verbose</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Enable verbose output.</p>
+              </div>
+            </td>
+            <td>
                   <code>1</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Enable verbose output.</p>
-          </div>
-        </li>
-    </ul>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
 
 <p><span class="doc-section-title">Attributes:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code><span title="pytorch_widedeep.preprocessing.text_preprocessor.ChunkTextPreprocessor.vocab">vocab</span></code></b>
-              (<code><a class="autorefs autorefs-internal" title="pytorch_widedeep.utils.fastai_transforms.Vocab" href="utils/fastai_transforms.html#pytorch_widedeep.utils.fastai_transforms.Vocab">Vocab</a></code>)
-          –
-          <div class="doc-md-description">
-            <p>an instance of <code>pytorch_widedeep.utils.fastai_transforms.ChunkVocab</code></p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code><span title="pytorch_widedeep.preprocessing.text_preprocessor.ChunkTextPreprocessor.embedding_matrix">embedding_matrix</span></code></b>
-              (<code><span title="numpy.ndarray">ndarray</span></code>)
-          –
-          <div class="doc-md-description">
-            <p>Array with the pretrained embeddings if <code>word_vectors_path</code> is not None</p>
-          </div>
-        </li>
-    </ul>
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code><span title="pytorch_widedeep.preprocessing.text_preprocessor.ChunkTextPreprocessor.vocab">vocab</span></code></td>
+            <td>
+                  <code><a class="autorefs autorefs-internal" title="pytorch_widedeep.utils.fastai_transforms.Vocab" href="utils/fastai_transforms.html#pytorch_widedeep.utils.fastai_transforms.Vocab">Vocab</a></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>an instance of <code>pytorch_widedeep.utils.fastai_transforms.ChunkVocab</code></p>
+              </div>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code><span title="pytorch_widedeep.preprocessing.text_preprocessor.ChunkTextPreprocessor.embedding_matrix">embedding_matrix</span></code></td>
+            <td>
+                  <code><span title="numpy.ndarray">ndarray</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Array with the pretrained embeddings if <code>word_vectors_path</code> is not None</p>
+              </div>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
 
 <p><span class="doc-section-title">Examples:</span></p>
@@ -6623,9 +12879,58 @@ <h2 id="pytorch_widedeep.preprocessing.text_preprocessor.ChunkTextPreprocessor"
 <span class="gp">&gt;&gt;&gt; </span><span class="n">processed_chunk</span> <span class="o">=</span> <span class="n">chunk_text_preprocessor</span><span class="o">.</span><span class="n">fit_transform</span><span class="p">(</span><span class="n">chunk_df</span><span class="p">)</span>
 </code></pre></div>
 
-                  <details class="quote">
-                    <summary>Source code in <code>pytorch_widedeep/preprocessing/text_preprocessor.py</code></summary>
-                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">313</span>
+              <details class="quote">
+                <summary>Source code in <code>pytorch_widedeep/preprocessing/text_preprocessor.py</code></summary>
+                <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">264</span>
+<span class="normal">265</span>
+<span class="normal">266</span>
+<span class="normal">267</span>
+<span class="normal">268</span>
+<span class="normal">269</span>
+<span class="normal">270</span>
+<span class="normal">271</span>
+<span class="normal">272</span>
+<span class="normal">273</span>
+<span class="normal">274</span>
+<span class="normal">275</span>
+<span class="normal">276</span>
+<span class="normal">277</span>
+<span class="normal">278</span>
+<span class="normal">279</span>
+<span class="normal">280</span>
+<span class="normal">281</span>
+<span class="normal">282</span>
+<span class="normal">283</span>
+<span class="normal">284</span>
+<span class="normal">285</span>
+<span class="normal">286</span>
+<span class="normal">287</span>
+<span class="normal">288</span>
+<span class="normal">289</span>
+<span class="normal">290</span>
+<span class="normal">291</span>
+<span class="normal">292</span>
+<span class="normal">293</span>
+<span class="normal">294</span>
+<span class="normal">295</span>
+<span class="normal">296</span>
+<span class="normal">297</span>
+<span class="normal">298</span>
+<span class="normal">299</span>
+<span class="normal">300</span>
+<span class="normal">301</span>
+<span class="normal">302</span>
+<span class="normal">303</span>
+<span class="normal">304</span>
+<span class="normal">305</span>
+<span class="normal">306</span>
+<span class="normal">307</span>
+<span class="normal">308</span>
+<span class="normal">309</span>
+<span class="normal">310</span>
+<span class="normal">311</span>
+<span class="normal">312</span>
+<span class="normal">313</span>
 <span class="normal">314</span>
 <span class="normal">315</span>
 <span class="normal">316</span>
@@ -6658,42 +12963,197 @@ <h2 id="pytorch_widedeep.preprocessing.text_preprocessor.ChunkTextPreprocessor"
 <span class="normal">343</span>
 <span class="normal">344</span>
 <span class="normal">345</span>
-<span class="normal">346</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
-    <span class="bp">self</span><span class="p">,</span>
-    <span class="n">text_col</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
-    <span class="n">n_chunks</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span>
-    <span class="n">root_dir</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">max_vocab</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">30000</span><span class="p">,</span>
-    <span class="n">min_freq</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">5</span><span class="p">,</span>
-    <span class="n">maxlen</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">80</span><span class="p">,</span>
-    <span class="n">pad_first</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">,</span>
-    <span class="n">pad_idx</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1</span><span class="p">,</span>
-    <span class="n">already_processed</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
-    <span class="n">word_vectors_path</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">n_cpus</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">verbose</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1</span><span class="p">,</span>
-<span class="p">):</span>
-    <span class="nb">super</span><span class="p">(</span><span class="n">ChunkTextPreprocessor</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span>
-        <span class="n">text_col</span><span class="o">=</span><span class="n">text_col</span><span class="p">,</span>
-        <span class="n">max_vocab</span><span class="o">=</span><span class="n">max_vocab</span><span class="p">,</span>
-        <span class="n">min_freq</span><span class="o">=</span><span class="n">min_freq</span><span class="p">,</span>
-        <span class="n">maxlen</span><span class="o">=</span><span class="n">maxlen</span><span class="p">,</span>
-        <span class="n">pad_first</span><span class="o">=</span><span class="n">pad_first</span><span class="p">,</span>
-        <span class="n">pad_idx</span><span class="o">=</span><span class="n">pad_idx</span><span class="p">,</span>
-        <span class="n">already_processed</span><span class="o">=</span><span class="n">already_processed</span><span class="p">,</span>
-        <span class="n">word_vectors_path</span><span class="o">=</span><span class="n">word_vectors_path</span><span class="p">,</span>
-        <span class="n">n_cpus</span><span class="o">=</span><span class="n">n_cpus</span><span class="p">,</span>
-        <span class="n">verbose</span><span class="o">=</span><span class="n">verbose</span><span class="p">,</span>
-    <span class="p">)</span>
+<span class="normal">346</span>
+<span class="normal">347</span>
+<span class="normal">348</span>
+<span class="normal">349</span>
+<span class="normal">350</span>
+<span class="normal">351</span>
+<span class="normal">352</span>
+<span class="normal">353</span>
+<span class="normal">354</span>
+<span class="normal">355</span>
+<span class="normal">356</span>
+<span class="normal">357</span>
+<span class="normal">358</span>
+<span class="normal">359</span>
+<span class="normal">360</span>
+<span class="normal">361</span>
+<span class="normal">362</span>
+<span class="normal">363</span>
+<span class="normal">364</span>
+<span class="normal">365</span>
+<span class="normal">366</span>
+<span class="normal">367</span>
+<span class="normal">368</span>
+<span class="normal">369</span>
+<span class="normal">370</span>
+<span class="normal">371</span>
+<span class="normal">372</span>
+<span class="normal">373</span>
+<span class="normal">374</span>
+<span class="normal">375</span>
+<span class="normal">376</span>
+<span class="normal">377</span>
+<span class="normal">378</span>
+<span class="normal">379</span>
+<span class="normal">380</span>
+<span class="normal">381</span>
+<span class="normal">382</span>
+<span class="normal">383</span>
+<span class="normal">384</span>
+<span class="normal">385</span>
+<span class="normal">386</span>
+<span class="normal">387</span>
+<span class="normal">388</span>
+<span class="normal">389</span>
+<span class="normal">390</span>
+<span class="normal">391</span>
+<span class="normal">392</span>
+<span class="normal">393</span>
+<span class="normal">394</span>
+<span class="normal">395</span>
+<span class="normal">396</span>
+<span class="normal">397</span>
+<span class="normal">398</span>
+<span class="normal">399</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">class</span> <span class="nc">ChunkTextPreprocessor</span><span class="p">(</span><span class="n">TextPreprocessor</span><span class="p">):</span>
+<span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Preprocessor to prepare the ``deeptext`` input dataset</span>
+
+<span class="sd">    Parameters</span>
+<span class="sd">    ----------</span>
+<span class="sd">    text_col: str</span>
+<span class="sd">        column in the input dataframe containing either the texts or the</span>
+<span class="sd">        filenames where the text documents are stored</span>
+<span class="sd">    n_chunks: int</span>
+<span class="sd">        Number of chunks that the text dataset is divided by.</span>
+<span class="sd">    root_dir: str, Optional, default = None</span>
+<span class="sd">        If &#39;text_col&#39; contains the filenames with the text documents, this is</span>
+<span class="sd">        the path to the directory where those documents are stored.</span>
+<span class="sd">    max_vocab: int, default=30000</span>
+<span class="sd">        Maximum number of tokens in the vocabulary</span>
+<span class="sd">    min_freq: int, default=5</span>
+<span class="sd">        Minimum frequency for a token to be part of the vocabulary</span>
+<span class="sd">    maxlen: int, default=80</span>
+<span class="sd">        Maximum length of the tokenized sequences</span>
+<span class="sd">    pad_first: bool,  default = True</span>
+<span class="sd">        Indicates whether the padding index will be added at the beginning or the</span>
+<span class="sd">        end of the sequences</span>
+<span class="sd">    pad_idx: int, default = 1</span>
+<span class="sd">        padding index. Fastai&#39;s Tokenizer leaves 0 for the &#39;unknown&#39; token.</span>
+<span class="sd">    word_vectors_path: str, Optional</span>
+<span class="sd">        Path to the pretrained word vectors</span>
+<span class="sd">    n_cpus: int, Optional, default = None</span>
+<span class="sd">        number of CPUs to used during the tokenization process</span>
+<span class="sd">    verbose: int, default 1</span>
+<span class="sd">        Enable verbose output.</span>
+
+<span class="sd">    Attributes</span>
+<span class="sd">    ----------</span>
+<span class="sd">    vocab: Vocab</span>
+<span class="sd">        an instance of `pytorch_widedeep.utils.fastai_transforms.ChunkVocab`</span>
+<span class="sd">    embedding_matrix: np.ndarray</span>
+<span class="sd">        Array with the pretrained embeddings if `word_vectors_path` is not None</span>
+
+<span class="sd">    Examples</span>
+<span class="sd">    ---------</span>
+<span class="sd">    &gt;&gt;&gt; import pandas as pd</span>
+<span class="sd">    &gt;&gt;&gt; from pytorch_widedeep.preprocessing import ChunkTextPreprocessor</span>
+<span class="sd">    &gt;&gt;&gt; chunk_df = pd.DataFrame({&#39;text_column&#39;: [&quot;life is like a box of chocolates&quot;,</span>
+<span class="sd">    ... &quot;You never know what you&#39;re gonna get&quot;]})</span>
+<span class="sd">    &gt;&gt;&gt; chunk_text_preprocessor = ChunkTextPreprocessor(text_col=&#39;text_column&#39;, n_chunks=1,</span>
+<span class="sd">    ... max_vocab=25, min_freq=1, maxlen=10, verbose=0, n_cpus=1)</span>
+<span class="sd">    &gt;&gt;&gt; processed_chunk = chunk_text_preprocessor.fit_transform(chunk_df)</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+
+    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span>
+        <span class="n">text_col</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
+        <span class="n">n_chunks</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span>
+        <span class="n">root_dir</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">max_vocab</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">30000</span><span class="p">,</span>
+        <span class="n">min_freq</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">5</span><span class="p">,</span>
+        <span class="n">maxlen</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">80</span><span class="p">,</span>
+        <span class="n">pad_first</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">,</span>
+        <span class="n">pad_idx</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1</span><span class="p">,</span>
+        <span class="n">already_processed</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
+        <span class="n">word_vectors_path</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">n_cpus</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">verbose</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1</span><span class="p">,</span>
+    <span class="p">):</span>
+        <span class="nb">super</span><span class="p">(</span><span class="n">ChunkTextPreprocessor</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span>
+            <span class="n">text_col</span><span class="o">=</span><span class="n">text_col</span><span class="p">,</span>
+            <span class="n">max_vocab</span><span class="o">=</span><span class="n">max_vocab</span><span class="p">,</span>
+            <span class="n">min_freq</span><span class="o">=</span><span class="n">min_freq</span><span class="p">,</span>
+            <span class="n">maxlen</span><span class="o">=</span><span class="n">maxlen</span><span class="p">,</span>
+            <span class="n">pad_first</span><span class="o">=</span><span class="n">pad_first</span><span class="p">,</span>
+            <span class="n">pad_idx</span><span class="o">=</span><span class="n">pad_idx</span><span class="p">,</span>
+            <span class="n">already_processed</span><span class="o">=</span><span class="n">already_processed</span><span class="p">,</span>
+            <span class="n">word_vectors_path</span><span class="o">=</span><span class="n">word_vectors_path</span><span class="p">,</span>
+            <span class="n">n_cpus</span><span class="o">=</span><span class="n">n_cpus</span><span class="p">,</span>
+            <span class="n">verbose</span><span class="o">=</span><span class="n">verbose</span><span class="p">,</span>
+        <span class="p">)</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">n_chunks</span> <span class="o">=</span> <span class="n">n_chunks</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">root_dir</span> <span class="o">=</span> <span class="n">root_dir</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">chunk_counter</span> <span class="o">=</span> <span class="mi">0</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">is_fitted</span> <span class="o">=</span> <span class="kc">False</span>
+
+    <span class="k">def</span> <span class="nf">partial_fit</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="s2">&quot;ChunkTextPreprocessor&quot;</span><span class="p">:</span>
+        <span class="c1"># df is a chunk of the original dataframe</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">chunk_counter</span> <span class="o">+=</span> <span class="mi">1</span>
 
-    <span class="bp">self</span><span class="o">.</span><span class="n">n_chunks</span> <span class="o">=</span> <span class="n">n_chunks</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">root_dir</span> <span class="o">=</span> <span class="n">root_dir</span>
+        <span class="n">texts</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_read_texts</span><span class="p">(</span><span class="n">df</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">root_dir</span><span class="p">)</span>
 
-    <span class="bp">self</span><span class="o">.</span><span class="n">chunk_counter</span> <span class="o">=</span> <span class="mi">0</span>
+        <span class="n">tokens</span> <span class="o">=</span> <span class="n">get_texts</span><span class="p">(</span><span class="n">texts</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">already_processed</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">n_cpus</span><span class="p">)</span>
 
-    <span class="bp">self</span><span class="o">.</span><span class="n">is_fitted</span> <span class="o">=</span> <span class="kc">False</span>
+        <span class="k">if</span> <span class="ow">not</span> <span class="nb">hasattr</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="s2">&quot;vocab&quot;</span><span class="p">):</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">vocab</span> <span class="o">=</span> <span class="n">ChunkVocab</span><span class="p">(</span>
+                <span class="n">max_vocab</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">max_vocab</span><span class="p">,</span>
+                <span class="n">min_freq</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">min_freq</span><span class="p">,</span>
+                <span class="n">pad_idx</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">pad_idx</span><span class="p">,</span>
+                <span class="n">n_chunks</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">n_chunks</span><span class="p">,</span>
+            <span class="p">)</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">vocab</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">tokens</span><span class="p">)</span>
+
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">chunk_counter</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">n_chunks</span><span class="p">:</span>
+            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">verbose</span><span class="p">:</span>
+                <span class="nb">print</span><span class="p">(</span><span class="s2">&quot;The vocabulary contains </span><span class="si">{}</span><span class="s2"> tokens&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">vocab</span><span class="o">.</span><span class="n">stoi</span><span class="p">)))</span>
+            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">word_vectors_path</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">embedding_matrix</span> <span class="o">=</span> <span class="n">build_embeddings_matrix</span><span class="p">(</span>
+                    <span class="bp">self</span><span class="o">.</span><span class="n">vocab</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">word_vectors_path</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">min_freq</span>
+                <span class="p">)</span>
+
+            <span class="bp">self</span><span class="o">.</span><span class="n">is_fitted</span> <span class="o">=</span> <span class="kc">True</span>
+
+        <span class="k">return</span> <span class="bp">self</span>
+
+    <span class="k">def</span> <span class="nf">fit</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="s2">&quot;ChunkTextPreprocessor&quot;</span><span class="p">:</span>
+        <span class="c1"># df is a chunk of the original dataframe</span>
+        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">partial_fit</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="fm">__repr__</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">str</span><span class="p">:</span>
+        <span class="n">list_of_params</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[</span><span class="s2">&quot;text_col=&#39;</span><span class="si">{text_col}</span><span class="s2">&#39;&quot;</span><span class="p">]</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">n_chunks</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="n">list_of_params</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">&quot;n_chunks=</span><span class="si">{n_chunks}</span><span class="s2">&quot;</span><span class="p">)</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">root_dir</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="n">list_of_params</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">&quot;root_dir=</span><span class="si">{root_dir}</span><span class="s2">&quot;</span><span class="p">)</span>
+        <span class="n">list_of_params</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">&quot;max_vocab=</span><span class="si">{max_vocab}</span><span class="s2">&quot;</span><span class="p">)</span>
+        <span class="n">list_of_params</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">&quot;min_freq=</span><span class="si">{min_freq}</span><span class="s2">&quot;</span><span class="p">)</span>
+        <span class="n">list_of_params</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">&quot;maxlen=</span><span class="si">{maxlen}</span><span class="s2">&quot;</span><span class="p">)</span>
+        <span class="n">list_of_params</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">&quot;pad_first=</span><span class="si">{pad_first}</span><span class="s2">&quot;</span><span class="p">)</span>
+        <span class="n">list_of_params</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">&quot;pad_idx=</span><span class="si">{pad_idx}</span><span class="s2">&quot;</span><span class="p">)</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">word_vectors_path</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="n">list_of_params</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">&quot;word_vectors_path=</span><span class="si">{word_vectors_path}</span><span class="s2">&quot;</span><span class="p">)</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">n_cpus</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="n">list_of_params</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">&quot;n_cpus=</span><span class="si">{n_cpus}</span><span class="s2">&quot;</span><span class="p">)</span>
+        <span class="n">list_of_params</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">&quot;verbose=</span><span class="si">{verbose}</span><span class="s2">&quot;</span><span class="p">)</span>
+        <span class="n">all_params</span> <span class="o">=</span> <span class="s2">&quot;, &quot;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">list_of_params</span><span class="p">)</span>
+        <span class="k">return</span> <span class="sa">f</span><span class="s2">&quot;ChunkTextPreprocessor(</span><span class="si">{</span><span class="n">all_params</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="bp">self</span><span class="o">.</span><span class="vm">__dict__</span><span class="p">)</span><span class="si">}</span><span class="s2">)&quot;</span>
 </code></pre></div></td></tr></table></div>
-                  </details>
+              </details>
 
 
 
@@ -6724,18 +13184,7 @@ <h2 id="pytorch_widedeep.preprocessing.hf_preprocessor.ChunkHFPreprocessor" clas
 
 
 <a href="#pytorch_widedeep.preprocessing.hf_preprocessor.ChunkHFPreprocessor" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="nf">ChunkHFPreprocessor</span><span class="p">(</span>
-    <span class="n">model_name</span><span class="p">,</span>
-    <span class="o">*</span><span class="p">,</span>
-    <span class="n">text_col</span><span class="p">,</span>
-    <span class="n">root_dir</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">use_fast_tokenizer</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
-    <span class="n">num_workers</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">preprocessing_rules</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">tokenizer_params</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">encode_params</span><span class="o">=</span><span class="kc">None</span>
-<span class="p">)</span>
-</code></pre></div>
+
 
     <div class="doc doc-contents first">
             <p class="doc doc-class-bases">
@@ -6754,121 +13203,237 @@ <h2 id="pytorch_widedeep.preprocessing.hf_preprocessor.ChunkHFPreprocessor" clas
 
 
 <p><span class="doc-section-title">Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>model_name</code></b>
-              (<code>str</code>)
-          –
-          <div class="doc-md-description">
-            <p>The model name from the transformers library e.g. <em>'bert-base-uncased'</em>.
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+          <th>Default</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>model_name</code></td>
+            <td>
+                  <code>str</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>The model name from the transformers library e.g. <em>'bert-base-uncased'</em>.
 Currently supported models are those from the families: BERT, RoBERTa,
 DistilBERT, ALBERT and ELECTRA.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>text_col</code></b>
-              (<code>str</code>)
-          –
-          <div class="doc-md-description">
-            <p>The column in the input dataframe containing the text data. When using
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>text_col</code></td>
+            <td>
+                  <code>str</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>The column in the input dataframe containing the text data. When using
 the <code>ChunkHFPreprocessor</code> the <code>text_col</code> parameter is mandatory.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>root_dir</code></b>
-              (<code><span title="typing.Optional">Optional</span>[str]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>The root directory where the text files are located. This is only
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>root_dir</code></td>
+            <td>
+                  <code><span title="typing.Optional">Optional</span>[str]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>The root directory where the text files are located. This is only
 needed if the text data is stored in text files. If the text data is
 stored in a column in the input dataframe, this parameter is not
 needed.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>use_fast_tokenizer</code></b>
-              (<code>bool</code>, default:
-                  <code>True</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Whether to use the fast tokenizer from HuggingFace or not</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>num_workers</code></b>
-              (<code><span title="typing.Optional">Optional</span>[int]</code>, default:
+              </div>
+            </td>
+            <td>
                   <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Number of workers to use when preprocessing the text data. If not
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>use_fast_tokenizer</code></td>
+            <td>
+                  <code>bool</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Whether to use the fast tokenizer from HuggingFace or not</p>
+              </div>
+            </td>
+            <td>
+                  <code>True</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>num_workers</code></td>
+            <td>
+                  <code><span title="typing.Optional">Optional</span>[int]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Number of workers to use when preprocessing the text data. If not
 None, and <code>use_fast_tokenizer</code> is False, the text data will be
 preprocessed in parallel using the number of workers specified. If
 <code>use_fast_tokenizer</code> is True, this argument is ignored.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>preprocessing_rules</code></b>
-              (<code><span title="typing.Optional">Optional</span>[<span title="typing.List">List</span>[<span title="typing.Callable">Callable</span>[[str], str]]]</code>, default:
+              </div>
+            </td>
+            <td>
                   <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>A list of functions to be applied to the text data before encoding.
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>preprocessing_rules</code></td>
+            <td>
+                  <code><span title="typing.Optional">Optional</span>[<span title="typing.List">List</span>[<span title="typing.Callable">Callable</span>[[str], str]]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>A list of functions to be applied to the text data before encoding.
 This can be useful to clean the text data before encoding. For
 example, removing html tags, special characters, etc.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>tokenizer_params</code></b>
-              (<code><span title="typing.Optional">Optional</span>[<span title="typing.Dict">Dict</span>[str, <span title="typing.Any">Any</span>]]</code>, default:
+              </div>
+            </td>
+            <td>
                   <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Additional parameters to be passed to the HuggingFace's
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>tokenizer_params</code></td>
+            <td>
+                  <code><span title="typing.Optional">Optional</span>[<span title="typing.Dict">Dict</span>[str, <span title="typing.Any">Any</span>]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Additional parameters to be passed to the HuggingFace's
 <code>PreTrainedTokenizer</code>.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>encode_params</code></b>
-              (<code><span title="typing.Optional">Optional</span>[<span title="typing.Dict">Dict</span>[str, <span title="typing.Any">Any</span>]]</code>, default:
+              </div>
+            </td>
+            <td>
                   <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Additional parameters to be passed to the <code>batch_encode_plus</code> method
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>encode_params</code></td>
+            <td>
+                  <code><span title="typing.Optional">Optional</span>[<span title="typing.Dict">Dict</span>[str, <span title="typing.Any">Any</span>]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Additional parameters to be passed to the <code>batch_encode_plus</code> method
 of the HuggingFace's <code>PreTrainedTokenizer</code>. In the case of the
 <code>ChunkHFPreprocessor</code>, this parameter is not really <code>Optional</code>. It
 must be passed containing at least the 'max_length' encoding
 parameter</p>
-          </div>
-        </li>
-    </ul>
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
 
 <p><span class="doc-section-title">Attributes:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code><span title="pytorch_widedeep.preprocessing.hf_preprocessor.ChunkHFPreprocessor.is_fitted">is_fitted</span></code></b>
-              (<code>bool</code>)
-          –
-          <div class="doc-md-description">
-            <p>Boolean indicating if the preprocessor has been fitted. This is a
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code><span title="pytorch_widedeep.preprocessing.hf_preprocessor.ChunkHFPreprocessor.is_fitted">is_fitted</span></code></td>
+            <td>
+                  <code>bool</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Boolean indicating if the preprocessor has been fitted. This is a
 HuggingFacea tokenizer, so it is always considered fitted and this
 attribute is manually set to True internally. This parameter exists
 for consistency with the rest of the library and because is needed
 for some functionality in the library.</p>
-          </div>
-        </li>
-    </ul>
-
-                  <details class="quote">
-                    <summary>Source code in <code>pytorch_widedeep/preprocessing/hf_preprocessor.py</code></summary>
-                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">427</span>
+              </div>
+            </td>
+          </tr>
+      </tbody>
+    </table>
+
+              <details class="quote">
+                <summary>Source code in <code>pytorch_widedeep/preprocessing/hf_preprocessor.py</code></summary>
+                <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">369</span>
+<span class="normal">370</span>
+<span class="normal">371</span>
+<span class="normal">372</span>
+<span class="normal">373</span>
+<span class="normal">374</span>
+<span class="normal">375</span>
+<span class="normal">376</span>
+<span class="normal">377</span>
+<span class="normal">378</span>
+<span class="normal">379</span>
+<span class="normal">380</span>
+<span class="normal">381</span>
+<span class="normal">382</span>
+<span class="normal">383</span>
+<span class="normal">384</span>
+<span class="normal">385</span>
+<span class="normal">386</span>
+<span class="normal">387</span>
+<span class="normal">388</span>
+<span class="normal">389</span>
+<span class="normal">390</span>
+<span class="normal">391</span>
+<span class="normal">392</span>
+<span class="normal">393</span>
+<span class="normal">394</span>
+<span class="normal">395</span>
+<span class="normal">396</span>
+<span class="normal">397</span>
+<span class="normal">398</span>
+<span class="normal">399</span>
+<span class="normal">400</span>
+<span class="normal">401</span>
+<span class="normal">402</span>
+<span class="normal">403</span>
+<span class="normal">404</span>
+<span class="normal">405</span>
+<span class="normal">406</span>
+<span class="normal">407</span>
+<span class="normal">408</span>
+<span class="normal">409</span>
+<span class="normal">410</span>
+<span class="normal">411</span>
+<span class="normal">412</span>
+<span class="normal">413</span>
+<span class="normal">414</span>
+<span class="normal">415</span>
+<span class="normal">416</span>
+<span class="normal">417</span>
+<span class="normal">418</span>
+<span class="normal">419</span>
+<span class="normal">420</span>
+<span class="normal">421</span>
+<span class="normal">422</span>
+<span class="normal">423</span>
+<span class="normal">424</span>
+<span class="normal">425</span>
+<span class="normal">426</span>
+<span class="normal">427</span>
 <span class="normal">428</span>
 <span class="normal">429</span>
 <span class="normal">430</span>
@@ -6907,48 +13472,122 @@ <h2 id="pytorch_widedeep.preprocessing.hf_preprocessor.ChunkHFPreprocessor" clas
 <span class="normal">463</span>
 <span class="normal">464</span>
 <span class="normal">465</span>
-<span class="normal">466</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
-    <span class="bp">self</span><span class="p">,</span>
-    <span class="n">model_name</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
-    <span class="o">*</span><span class="p">,</span>
-    <span class="n">text_col</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
-    <span class="n">root_dir</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">use_fast_tokenizer</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">,</span>
-    <span class="n">num_workers</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">preprocessing_rules</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="n">Callable</span><span class="p">[[</span><span class="nb">str</span><span class="p">],</span> <span class="nb">str</span><span class="p">]]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">tokenizer_params</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">encode_params</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-<span class="p">):</span>
-    <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span>
-        <span class="n">model_name</span><span class="o">=</span><span class="n">model_name</span><span class="p">,</span>
-        <span class="n">use_fast_tokenizer</span><span class="o">=</span><span class="n">use_fast_tokenizer</span><span class="p">,</span>
-        <span class="n">text_col</span><span class="o">=</span><span class="n">text_col</span><span class="p">,</span>
-        <span class="n">num_workers</span><span class="o">=</span><span class="n">num_workers</span><span class="p">,</span>
-        <span class="n">preprocessing_rules</span><span class="o">=</span><span class="n">preprocessing_rules</span><span class="p">,</span>
-        <span class="n">tokenizer_params</span><span class="o">=</span><span class="n">tokenizer_params</span><span class="p">,</span>
-        <span class="n">encode_params</span><span class="o">=</span><span class="n">encode_params</span><span class="p">,</span>
-    <span class="p">)</span>
+<span class="normal">466</span>
+<span class="normal">467</span>
+<span class="normal">468</span>
+<span class="normal">469</span>
+<span class="normal">470</span>
+<span class="normal">471</span>
+<span class="normal">472</span>
+<span class="normal">473</span>
+<span class="normal">474</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">class</span> <span class="nc">ChunkHFPreprocessor</span><span class="p">(</span><span class="n">HFPreprocessor</span><span class="p">):</span>
+<span class="w">    </span><span class="sd">&quot;&quot;&quot;Text processor to prepare the ``deeptext`` input dataset that is a</span>
+<span class="sd">    wrapper around HuggingFace&#39;s tokenizers.</span>
+
+<span class="sd">    Hugginface Tokenizer&#39;s are already &#39;trained&#39;. Therefore, unlike the</span>
+<span class="sd">    `ChunkTextPreprocessor` this is mostly identical to the `HFPreprocessor`</span>
+<span class="sd">    with the only difference that the class needs a &#39;text_col&#39; parameter to</span>
+<span class="sd">    be passed. Also the parameter `encode_params` is not really optional when</span>
+<span class="sd">    using this class. It must be passed containing at least the</span>
+<span class="sd">    &#39;max_length&#39; encoding parameter. This is because we need to ensure that</span>
+<span class="sd">     all sequences have the same length when encoding in chunks.</span>
 
-    <span class="bp">self</span><span class="o">.</span><span class="n">root_dir</span> <span class="o">=</span> <span class="n">root_dir</span>
+<span class="sd">    Parameters</span>
+<span class="sd">    ----------</span>
+<span class="sd">    model_name: str</span>
+<span class="sd">        The model name from the transformers library e.g. _&#39;bert-base-uncased&#39;_.</span>
+<span class="sd">        Currently supported models are those from the families: BERT, RoBERTa,</span>
+<span class="sd">        DistilBERT, ALBERT and ELECTRA.</span>
+<span class="sd">    text_col: str, default = None</span>
+<span class="sd">        The column in the input dataframe containing the text data. When using</span>
+<span class="sd">        the `ChunkHFPreprocessor` the `text_col` parameter is mandatory.</span>
+<span class="sd">    root_dir: Optional[str], default = None</span>
+<span class="sd">        The root directory where the text files are located. This is only</span>
+<span class="sd">        needed if the text data is stored in text files. If the text data is</span>
+<span class="sd">        stored in a column in the input dataframe, this parameter is not</span>
+<span class="sd">        needed.</span>
+<span class="sd">    use_fast_tokenizer: bool, default = False</span>
+<span class="sd">        Whether to use the fast tokenizer from HuggingFace or not</span>
+<span class="sd">    num_workers: Optional[int], default = None</span>
+<span class="sd">        Number of workers to use when preprocessing the text data. If not</span>
+<span class="sd">        None, and `use_fast_tokenizer` is False, the text data will be</span>
+<span class="sd">        preprocessed in parallel using the number of workers specified. If</span>
+<span class="sd">        `use_fast_tokenizer` is True, this argument is ignored.</span>
+<span class="sd">    preprocessing_rules: Optional[List[Callable[[str], str]]], default = None</span>
+<span class="sd">        A list of functions to be applied to the text data before encoding.</span>
+<span class="sd">        This can be useful to clean the text data before encoding. For</span>
+<span class="sd">        example, removing html tags, special characters, etc.</span>
+<span class="sd">    tokenizer_params: Optional[Dict[str, Any]], default = None</span>
+<span class="sd">        Additional parameters to be passed to the HuggingFace&#39;s</span>
+<span class="sd">        `PreTrainedTokenizer`.</span>
+<span class="sd">    encode_params: Optional[Dict[str, Any]], default = None</span>
+<span class="sd">        Additional parameters to be passed to the `batch_encode_plus` method</span>
+<span class="sd">        of the HuggingFace&#39;s `PreTrainedTokenizer`. In the case of the</span>
+<span class="sd">        `ChunkHFPreprocessor`, this parameter is not really `Optional`. It</span>
+<span class="sd">        must be passed containing at least the &#39;max_length&#39; encoding</span>
+<span class="sd">        parameter</span>
 
-    <span class="c1"># when using in chunks encode_params is not really optional. I will</span>
-    <span class="c1"># review types in due time</span>
-    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">encode_params</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
-        <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
-            <span class="s2">&quot;The &#39;encode_params&#39; dict must be passed to the ChunkHFTokenizer &quot;</span>
-            <span class="s2">&quot;containing at least the &#39;max_length&#39; encoding parameter&quot;</span>
-        <span class="p">)</span>
+<span class="sd">    Attributes</span>
+<span class="sd">    ----------</span>
+<span class="sd">    is_fitted: bool</span>
+<span class="sd">        Boolean indicating if the preprocessor has been fitted. This is a</span>
+<span class="sd">        HuggingFacea tokenizer, so it is always considered fitted and this</span>
+<span class="sd">        attribute is manually set to True internally. This parameter exists</span>
+<span class="sd">        for consistency with the rest of the library and because is needed</span>
+<span class="sd">        for some functionality in the library.</span>
 
-    <span class="k">if</span> <span class="s2">&quot;padding&quot;</span> <span class="ow">not</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">encode_params</span> <span class="ow">or</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">encode_params</span><span class="p">[</span><span class="s2">&quot;padding&quot;</span><span class="p">]:</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">encode_params</span><span class="p">[</span><span class="s2">&quot;padding&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="kc">True</span>
+<span class="sd">    &quot;&quot;&quot;</span>
 
-    <span class="k">if</span> <span class="p">(</span>
-        <span class="s2">&quot;truncation&quot;</span> <span class="ow">not</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">encode_params</span>
-        <span class="ow">or</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">encode_params</span><span class="p">[</span><span class="s2">&quot;truncation&quot;</span><span class="p">]</span>
+    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span>
+        <span class="n">model_name</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
+        <span class="o">*</span><span class="p">,</span>
+        <span class="n">text_col</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
+        <span class="n">root_dir</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">use_fast_tokenizer</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">,</span>
+        <span class="n">num_workers</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">preprocessing_rules</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="n">Callable</span><span class="p">[[</span><span class="nb">str</span><span class="p">],</span> <span class="nb">str</span><span class="p">]]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">tokenizer_params</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">encode_params</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
     <span class="p">):</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">encode_params</span><span class="p">[</span><span class="s2">&quot;truncation&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="kc">True</span>
+        <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span>
+            <span class="n">model_name</span><span class="o">=</span><span class="n">model_name</span><span class="p">,</span>
+            <span class="n">use_fast_tokenizer</span><span class="o">=</span><span class="n">use_fast_tokenizer</span><span class="p">,</span>
+            <span class="n">text_col</span><span class="o">=</span><span class="n">text_col</span><span class="p">,</span>
+            <span class="n">num_workers</span><span class="o">=</span><span class="n">num_workers</span><span class="p">,</span>
+            <span class="n">preprocessing_rules</span><span class="o">=</span><span class="n">preprocessing_rules</span><span class="p">,</span>
+            <span class="n">tokenizer_params</span><span class="o">=</span><span class="n">tokenizer_params</span><span class="p">,</span>
+            <span class="n">encode_params</span><span class="o">=</span><span class="n">encode_params</span><span class="p">,</span>
+        <span class="p">)</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">root_dir</span> <span class="o">=</span> <span class="n">root_dir</span>
+
+        <span class="c1"># when using in chunks encode_params is not really optional. I will</span>
+        <span class="c1"># review types in due time</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">encode_params</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
+                <span class="s2">&quot;The &#39;encode_params&#39; dict must be passed to the ChunkHFTokenizer &quot;</span>
+                <span class="s2">&quot;containing at least the &#39;max_length&#39; encoding parameter&quot;</span>
+            <span class="p">)</span>
+
+        <span class="k">if</span> <span class="s2">&quot;padding&quot;</span> <span class="ow">not</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">encode_params</span> <span class="ow">or</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">encode_params</span><span class="p">[</span><span class="s2">&quot;padding&quot;</span><span class="p">]:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">encode_params</span><span class="p">[</span><span class="s2">&quot;padding&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="kc">True</span>
+
+        <span class="k">if</span> <span class="p">(</span>
+            <span class="s2">&quot;truncation&quot;</span> <span class="ow">not</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">encode_params</span>
+            <span class="ow">or</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">encode_params</span><span class="p">[</span><span class="s2">&quot;truncation&quot;</span><span class="p">]</span>
+        <span class="p">):</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">encode_params</span><span class="p">[</span><span class="s2">&quot;truncation&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="kc">True</span>
+
+    <span class="k">def</span> <span class="fm">__repr__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="k">return</span> <span class="p">(</span>
+            <span class="sa">f</span><span class="s2">&quot;ChunkHFPreprocessor(text_col=</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">text_col</span><span class="si">}</span><span class="s2">, model_name=</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">model_name</span><span class="si">}</span><span class="s2">, &quot;</span>
+            <span class="sa">f</span><span class="s2">&quot;use_fast_tokenizer=</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">use_fast_tokenizer</span><span class="si">}</span><span class="s2">, num_workers=</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">num_workers</span><span class="si">}</span><span class="s2">, &quot;</span>
+            <span class="sa">f</span><span class="s2">&quot;preprocessing_rules=</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">preprocessing_rules</span><span class="si">}</span><span class="s2">, tokenizer_params=</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">tokenizer_params</span><span class="si">}</span><span class="s2">, &quot;</span>
+            <span class="sa">f</span><span class="s2">&quot;encode_params=</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">encode_params</span><span class="si">}</span><span class="s2">, root_dir=</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">root_dir</span><span class="si">}</span><span class="s2">)&quot;</span>
+        <span class="p">)</span>
 </code></pre></div></td></tr></table></div>
-                  </details>
+              </details>
 
 
 
diff --git a/mkdocs/site/pytorch-widedeep/self_supervised_pretraining.html b/mkdocs/site/pytorch-widedeep/self_supervised_pretraining.html
index 63f91bc5..5c447b2f 100644
--- a/mkdocs/site/pytorch-widedeep/self_supervised_pretraining.html
+++ b/mkdocs/site/pytorch-widedeep/self_supervised_pretraining.html
@@ -1396,7 +1396,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/16_Self_Supervised_Pretraning_pt1.ipynb" class="md-nav__link">
+      <a href="../examples/15_Self_Supervised_Pretraning_pt1.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1417,7 +1417,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/16_Self_Supervised_Pretraning_pt2.ipynb" class="md-nav__link">
+      <a href="../examples/15_Self_Supervised_Pretraning_pt2.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1438,7 +1438,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/17_Usign_a_custom_hugging_face_model.ipynb" class="md-nav__link">
+      <a href="../examples/16_Usign_a_custom_hugging_face_model.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1459,7 +1459,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/18_feature_importance_via_attention_weights.ipynb" class="md-nav__link">
+      <a href="../examples/17_feature_importance_via_attention_weights.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1480,7 +1480,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/19_wide_and_deep_for_recsys_pt1.ipynb" class="md-nav__link">
+      <a href="../examples/18_wide_and_deep_for_recsys_pt1.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1501,7 +1501,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/19_wide_and_deep_for_recsys_pt2.ipynb" class="md-nav__link">
+      <a href="../examples/18_wide_and_deep_for_recsys_pt2.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1522,7 +1522,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/20_load_from_folder_functionality.ipynb" class="md-nav__link">
+      <a href="../examples/19_load_from_folder_functionality.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1543,7 +1543,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/21_Using_huggingface_within_widedeep.ipynb" class="md-nav__link">
+      <a href="../examples/20_Using_huggingface_within_widedeep.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1653,18 +1653,7 @@ <h2 id="pytorch_widedeep.self_supervised_training.EncoderDecoderTrainer" class="
 
 
 <a href="#pytorch_widedeep.self_supervised_training.EncoderDecoderTrainer" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="nf">EncoderDecoderTrainer</span><span class="p">(</span>
-    <span class="n">encoder</span><span class="p">,</span>
-    <span class="n">decoder</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">masked_prob</span><span class="o">=</span><span class="mf">0.2</span><span class="p">,</span>
-    <span class="n">optimizer</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">lr_scheduler</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">callbacks</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">verbose</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span>
-    <span class="n">seed</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span>
-    <span class="o">**</span><span class="n">kwargs</span>
-<span class="p">)</span>
-</code></pre></div>
+
 
     <div class="doc doc-contents first">
             <p class="doc doc-class-bases">
@@ -1678,104 +1667,158 @@ <h2 id="pytorch_widedeep.self_supervised_training.EncoderDecoderTrainer" class="
 
 
 <p><span class="doc-section-title">Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>encoder</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.ModelWithoutAttention">ModelWithoutAttention</span></code>)
-          –
-          <div class="doc-md-description">
-            <p>An instance of a <code>TabMlp</code>, <code>TabResNet</code> or <code>TabNet</code> model</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>decoder</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.DecoderWithoutAttention">DecoderWithoutAttention</span>]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>An instance of  a <code>TabMlpDecoder</code>, <code>TabResNetDecoder</code> or
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+          <th>Default</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>encoder</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.ModelWithoutAttention">ModelWithoutAttention</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>An instance of a <code>TabMlp</code>, <code>TabResNet</code> or <code>TabNet</code> model</p>
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>decoder</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.DecoderWithoutAttention">DecoderWithoutAttention</span>]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>An instance of  a <code>TabMlpDecoder</code>, <code>TabResNetDecoder</code> or
 <code>TabNetDecoder</code> model. if <code>None</code> the decoder will be automatically
 built as a '<em>simetric</em>' model to the Encoder</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>masked_prob</code></b>
-              (<code>float</code>, default:
-                  <code>0.2</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Indicates the fraction of elements in the embedding tensor that will
-be masked and hence used for reconstruction</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>optimizer</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Optimizer">Optimizer</span>]</code>, default:
+              </div>
+            </td>
+            <td>
                   <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>An instance of Pytorch's <code>Optimizer</code> object (e.g. <code>torch.optim.Adam
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>masked_prob</code></td>
+            <td>
+                  <code>float</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Indicates the fraction of elements in the embedding tensor that will
+be masked and hence used for reconstruction</p>
+              </div>
+            </td>
+            <td>
+                  <code>0.2</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>optimizer</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Optimizer">Optimizer</span>]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>An instance of Pytorch's <code>Optimizer</code> object (e.g. <code>torch.optim.Adam
 ()</code>). if no optimizer is passed it will default to <code>AdamW</code>.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>lr_scheduler</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.LRScheduler">LRScheduler</span>]</code>, default:
+              </div>
+            </td>
+            <td>
                   <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>An instance of Pytorch's <code>LRScheduler</code> object
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>lr_scheduler</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.LRScheduler">LRScheduler</span>]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>An instance of Pytorch's <code>LRScheduler</code> object
 (e.g <code>torch.optim.lr_scheduler.StepLR(opt, step_size=5)</code>).</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>callbacks</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.callbacks.Callback">Callback</span>]]</code>, default:
+              </div>
+            </td>
+            <td>
                   <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>List with <code>Callback</code> objects. The three callbacks available in
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>callbacks</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.callbacks.Callback">Callback</span>]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>List with <code>Callback</code> objects. The three callbacks available in
 <code>pytorch-widedeep</code> are: <code>LRHistory</code>, <code>ModelCheckpoint</code> and
 <code>EarlyStopping</code>. This can also be a custom callback. See
 <code>pytorch_widedeep.callbacks.Callback</code> or the Examples folder in the
 repo.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>verbose</code></b>
-              (<code>int</code>, default:
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>verbose</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Setting it to 0 will print nothing during training.</p>
+              </div>
+            </td>
+            <td>
                   <code>1</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Setting it to 0 will print nothing during training.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>seed</code></b>
-              (<code>int</code>, default:
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>seed</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Random seed to be used internally for train_test_split</p>
+              </div>
+            </td>
+            <td>
                   <code>1</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Random seed to be used internally for train_test_split</p>
-          </div>
-        </li>
-    </ul>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
 
 <p><span class="doc-section-title">Other Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>**kwargs</code></b>
-          –
-          <div class="doc-md-description">
-            <p>Other infrequently used arguments that can also be passed as kwargs are:</p>
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>**kwargs</code></td>
+            <td>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Other infrequently used arguments that can also be passed as kwargs are:</p>
 <ul>
 <li>
 <p><strong>device</strong>: <code>str</code><br/>
@@ -1792,13 +1835,67 @@ <h2 id="pytorch_widedeep.self_supervised_training.EncoderDecoderTrainer" class="
     learning rate is a bit particular.</p>
 </li>
 </ul>
-          </div>
-        </li>
-    </ul>
-
-                  <details class="quote">
-                    <summary>Source code in <code>pytorch_widedeep/self_supervised_training/encoder_decoder_trainer.py</code></summary>
-                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 80</span>
+              </div>
+            </td>
+          </tr>
+      </tbody>
+    </table>
+
+              <details class="quote">
+                <summary>Source code in <code>pytorch_widedeep/self_supervised_training/encoder_decoder_trainer.py</code></summary>
+                <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 28</span>
+<span class="normal"> 29</span>
+<span class="normal"> 30</span>
+<span class="normal"> 31</span>
+<span class="normal"> 32</span>
+<span class="normal"> 33</span>
+<span class="normal"> 34</span>
+<span class="normal"> 35</span>
+<span class="normal"> 36</span>
+<span class="normal"> 37</span>
+<span class="normal"> 38</span>
+<span class="normal"> 39</span>
+<span class="normal"> 40</span>
+<span class="normal"> 41</span>
+<span class="normal"> 42</span>
+<span class="normal"> 43</span>
+<span class="normal"> 44</span>
+<span class="normal"> 45</span>
+<span class="normal"> 46</span>
+<span class="normal"> 47</span>
+<span class="normal"> 48</span>
+<span class="normal"> 49</span>
+<span class="normal"> 50</span>
+<span class="normal"> 51</span>
+<span class="normal"> 52</span>
+<span class="normal"> 53</span>
+<span class="normal"> 54</span>
+<span class="normal"> 55</span>
+<span class="normal"> 56</span>
+<span class="normal"> 57</span>
+<span class="normal"> 58</span>
+<span class="normal"> 59</span>
+<span class="normal"> 60</span>
+<span class="normal"> 61</span>
+<span class="normal"> 62</span>
+<span class="normal"> 63</span>
+<span class="normal"> 64</span>
+<span class="normal"> 65</span>
+<span class="normal"> 66</span>
+<span class="normal"> 67</span>
+<span class="normal"> 68</span>
+<span class="normal"> 69</span>
+<span class="normal"> 70</span>
+<span class="normal"> 71</span>
+<span class="normal"> 72</span>
+<span class="normal"> 73</span>
+<span class="normal"> 74</span>
+<span class="normal"> 75</span>
+<span class="normal"> 76</span>
+<span class="normal"> 77</span>
+<span class="normal"> 78</span>
+<span class="normal"> 79</span>
+<span class="normal"> 80</span>
 <span class="normal"> 81</span>
 <span class="normal"> 82</span>
 <span class="normal"> 83</span>
@@ -1820,31 +1917,405 @@ <h2 id="pytorch_widedeep.self_supervised_training.EncoderDecoderTrainer" class="
 <span class="normal"> 99</span>
 <span class="normal">100</span>
 <span class="normal">101</span>
-<span class="normal">102</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
-    <span class="bp">self</span><span class="p">,</span>
-    <span class="n">encoder</span><span class="p">:</span> <span class="n">ModelWithoutAttention</span><span class="p">,</span>
-    <span class="n">decoder</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">DecoderWithoutAttention</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">masked_prob</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.2</span><span class="p">,</span>
-    <span class="n">optimizer</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Optimizer</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">lr_scheduler</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">LRScheduler</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">callbacks</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="n">Callback</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">verbose</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1</span><span class="p">,</span>
-    <span class="n">seed</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1</span><span class="p">,</span>
-    <span class="o">**</span><span class="n">kwargs</span><span class="p">,</span>
-<span class="p">):</span>
-    <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span>
-        <span class="n">encoder</span><span class="o">=</span><span class="n">encoder</span><span class="p">,</span>
-        <span class="n">decoder</span><span class="o">=</span><span class="n">decoder</span><span class="p">,</span>
-        <span class="n">masked_prob</span><span class="o">=</span><span class="n">masked_prob</span><span class="p">,</span>
-        <span class="n">optimizer</span><span class="o">=</span><span class="n">optimizer</span><span class="p">,</span>
-        <span class="n">lr_scheduler</span><span class="o">=</span><span class="n">lr_scheduler</span><span class="p">,</span>
-        <span class="n">callbacks</span><span class="o">=</span><span class="n">callbacks</span><span class="p">,</span>
-        <span class="n">verbose</span><span class="o">=</span><span class="n">verbose</span><span class="p">,</span>
-        <span class="n">seed</span><span class="o">=</span><span class="n">seed</span><span class="p">,</span>
+<span class="normal">102</span>
+<span class="normal">103</span>
+<span class="normal">104</span>
+<span class="normal">105</span>
+<span class="normal">106</span>
+<span class="normal">107</span>
+<span class="normal">108</span>
+<span class="normal">109</span>
+<span class="normal">110</span>
+<span class="normal">111</span>
+<span class="normal">112</span>
+<span class="normal">113</span>
+<span class="normal">114</span>
+<span class="normal">115</span>
+<span class="normal">116</span>
+<span class="normal">117</span>
+<span class="normal">118</span>
+<span class="normal">119</span>
+<span class="normal">120</span>
+<span class="normal">121</span>
+<span class="normal">122</span>
+<span class="normal">123</span>
+<span class="normal">124</span>
+<span class="normal">125</span>
+<span class="normal">126</span>
+<span class="normal">127</span>
+<span class="normal">128</span>
+<span class="normal">129</span>
+<span class="normal">130</span>
+<span class="normal">131</span>
+<span class="normal">132</span>
+<span class="normal">133</span>
+<span class="normal">134</span>
+<span class="normal">135</span>
+<span class="normal">136</span>
+<span class="normal">137</span>
+<span class="normal">138</span>
+<span class="normal">139</span>
+<span class="normal">140</span>
+<span class="normal">141</span>
+<span class="normal">142</span>
+<span class="normal">143</span>
+<span class="normal">144</span>
+<span class="normal">145</span>
+<span class="normal">146</span>
+<span class="normal">147</span>
+<span class="normal">148</span>
+<span class="normal">149</span>
+<span class="normal">150</span>
+<span class="normal">151</span>
+<span class="normal">152</span>
+<span class="normal">153</span>
+<span class="normal">154</span>
+<span class="normal">155</span>
+<span class="normal">156</span>
+<span class="normal">157</span>
+<span class="normal">158</span>
+<span class="normal">159</span>
+<span class="normal">160</span>
+<span class="normal">161</span>
+<span class="normal">162</span>
+<span class="normal">163</span>
+<span class="normal">164</span>
+<span class="normal">165</span>
+<span class="normal">166</span>
+<span class="normal">167</span>
+<span class="normal">168</span>
+<span class="normal">169</span>
+<span class="normal">170</span>
+<span class="normal">171</span>
+<span class="normal">172</span>
+<span class="normal">173</span>
+<span class="normal">174</span>
+<span class="normal">175</span>
+<span class="normal">176</span>
+<span class="normal">177</span>
+<span class="normal">178</span>
+<span class="normal">179</span>
+<span class="normal">180</span>
+<span class="normal">181</span>
+<span class="normal">182</span>
+<span class="normal">183</span>
+<span class="normal">184</span>
+<span class="normal">185</span>
+<span class="normal">186</span>
+<span class="normal">187</span>
+<span class="normal">188</span>
+<span class="normal">189</span>
+<span class="normal">190</span>
+<span class="normal">191</span>
+<span class="normal">192</span>
+<span class="normal">193</span>
+<span class="normal">194</span>
+<span class="normal">195</span>
+<span class="normal">196</span>
+<span class="normal">197</span>
+<span class="normal">198</span>
+<span class="normal">199</span>
+<span class="normal">200</span>
+<span class="normal">201</span>
+<span class="normal">202</span>
+<span class="normal">203</span>
+<span class="normal">204</span>
+<span class="normal">205</span>
+<span class="normal">206</span>
+<span class="normal">207</span>
+<span class="normal">208</span>
+<span class="normal">209</span>
+<span class="normal">210</span>
+<span class="normal">211</span>
+<span class="normal">212</span>
+<span class="normal">213</span>
+<span class="normal">214</span>
+<span class="normal">215</span>
+<span class="normal">216</span>
+<span class="normal">217</span>
+<span class="normal">218</span>
+<span class="normal">219</span>
+<span class="normal">220</span>
+<span class="normal">221</span>
+<span class="normal">222</span>
+<span class="normal">223</span>
+<span class="normal">224</span>
+<span class="normal">225</span>
+<span class="normal">226</span>
+<span class="normal">227</span>
+<span class="normal">228</span>
+<span class="normal">229</span>
+<span class="normal">230</span>
+<span class="normal">231</span>
+<span class="normal">232</span>
+<span class="normal">233</span>
+<span class="normal">234</span>
+<span class="normal">235</span>
+<span class="normal">236</span>
+<span class="normal">237</span>
+<span class="normal">238</span>
+<span class="normal">239</span>
+<span class="normal">240</span>
+<span class="normal">241</span>
+<span class="normal">242</span>
+<span class="normal">243</span>
+<span class="normal">244</span>
+<span class="normal">245</span>
+<span class="normal">246</span>
+<span class="normal">247</span>
+<span class="normal">248</span>
+<span class="normal">249</span>
+<span class="normal">250</span>
+<span class="normal">251</span>
+<span class="normal">252</span>
+<span class="normal">253</span>
+<span class="normal">254</span>
+<span class="normal">255</span>
+<span class="normal">256</span>
+<span class="normal">257</span>
+<span class="normal">258</span>
+<span class="normal">259</span>
+<span class="normal">260</span>
+<span class="normal">261</span>
+<span class="normal">262</span>
+<span class="normal">263</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">class</span> <span class="nc">EncoderDecoderTrainer</span><span class="p">(</span><span class="n">BaseEncoderDecoderTrainer</span><span class="p">):</span>
+<span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;This class implements an Encoder-Decoder self-supervised &#39;routine&#39;</span>
+<span class="sd">    inspired by</span>
+<span class="sd">    [TabNet: Attentive Interpretable Tabular Learning](https://arxiv.org/abs/1908.07442).</span>
+<span class="sd">    See Figure 1 above.</span>
+
+<span class="sd">    Parameters</span>
+<span class="sd">    ----------</span>
+<span class="sd">    encoder: ModelWithoutAttention,</span>
+<span class="sd">        An instance of a `TabMlp`, `TabResNet` or `TabNet` model</span>
+<span class="sd">    decoder: Optional[DecoderWithoutAttention] = None,</span>
+<span class="sd">        An instance of  a `TabMlpDecoder`, `TabResNetDecoder` or</span>
+<span class="sd">        `TabNetDecoder` model. if `None` the decoder will be automatically</span>
+<span class="sd">        built as a &#39;_simetric_&#39; model to the Encoder</span>
+<span class="sd">    masked_prob: float = 0.2,</span>
+<span class="sd">        Indicates the fraction of elements in the embedding tensor that will</span>
+<span class="sd">        be masked and hence used for reconstruction</span>
+<span class="sd">    optimizer: Optional[Optimizer] = None,</span>
+<span class="sd">        An instance of Pytorch&#39;s `Optimizer` object (e.g. `torch.optim.Adam</span>
+<span class="sd">        ()`). if no optimizer is passed it will default to `AdamW`.</span>
+<span class="sd">    lr_scheduler: Optional[LRScheduler] = None,</span>
+<span class="sd">        An instance of Pytorch&#39;s `LRScheduler` object</span>
+<span class="sd">        (e.g `torch.optim.lr_scheduler.StepLR(opt, step_size=5)`).</span>
+<span class="sd">    callbacks: Optional[List[Callback]] = None,</span>
+<span class="sd">        List with `Callback` objects. The three callbacks available in</span>
+<span class="sd">        `pytorch-widedeep` are: `LRHistory`, `ModelCheckpoint` and</span>
+<span class="sd">        `EarlyStopping`. This can also be a custom callback. See</span>
+<span class="sd">        `pytorch_widedeep.callbacks.Callback` or the Examples folder in the</span>
+<span class="sd">        repo.</span>
+<span class="sd">    verbose: int, default=1</span>
+<span class="sd">        Setting it to 0 will print nothing during training.</span>
+<span class="sd">    seed: int, default=1</span>
+<span class="sd">        Random seed to be used internally for train_test_split</span>
+
+<span class="sd">    Other Parameters</span>
+<span class="sd">    ----------------</span>
+<span class="sd">    **kwargs: dict</span>
+<span class="sd">        Other infrequently used arguments that can also be passed as kwargs are:</span>
+
+<span class="sd">        - **device**: `str`&lt;br/&gt;</span>
+<span class="sd">            string indicating the device. One of _&#39;cpu&#39;_ or _&#39;gpu&#39;_</span>
+
+<span class="sd">        - **num_workers**: `int`&lt;br/&gt;</span>
+<span class="sd">            number of workers to be used internally by the data loaders</span>
+
+<span class="sd">        - **reducelronplateau_criterion**: `str`</span>
+<span class="sd">            This sets the criterion that will be used by the lr scheduler to</span>
+<span class="sd">            take a step: One of _&#39;loss&#39;_ or _&#39;metric&#39;_. The ReduceLROnPlateau</span>
+<span class="sd">            learning rate is a bit particular.</span>
+
+<span class="sd">    &quot;&quot;&quot;</span>
+
+    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span>
+        <span class="n">encoder</span><span class="p">:</span> <span class="n">ModelWithoutAttention</span><span class="p">,</span>
+        <span class="n">decoder</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">DecoderWithoutAttention</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">masked_prob</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.2</span><span class="p">,</span>
+        <span class="n">optimizer</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Optimizer</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">lr_scheduler</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">LRScheduler</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">callbacks</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="n">Callback</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">verbose</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1</span><span class="p">,</span>
+        <span class="n">seed</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1</span><span class="p">,</span>
         <span class="o">**</span><span class="n">kwargs</span><span class="p">,</span>
-    <span class="p">)</span>
+    <span class="p">):</span>
+        <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span>
+            <span class="n">encoder</span><span class="o">=</span><span class="n">encoder</span><span class="p">,</span>
+            <span class="n">decoder</span><span class="o">=</span><span class="n">decoder</span><span class="p">,</span>
+            <span class="n">masked_prob</span><span class="o">=</span><span class="n">masked_prob</span><span class="p">,</span>
+            <span class="n">optimizer</span><span class="o">=</span><span class="n">optimizer</span><span class="p">,</span>
+            <span class="n">lr_scheduler</span><span class="o">=</span><span class="n">lr_scheduler</span><span class="p">,</span>
+            <span class="n">callbacks</span><span class="o">=</span><span class="n">callbacks</span><span class="p">,</span>
+            <span class="n">verbose</span><span class="o">=</span><span class="n">verbose</span><span class="p">,</span>
+            <span class="n">seed</span><span class="o">=</span><span class="n">seed</span><span class="p">,</span>
+            <span class="o">**</span><span class="n">kwargs</span><span class="p">,</span>
+        <span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">pretrain</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span>
+        <span class="n">X_tab</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">,</span>
+        <span class="n">X_tab_val</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">val_split</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">validation_freq</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1</span><span class="p">,</span>
+        <span class="n">n_epochs</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1</span><span class="p">,</span>
+        <span class="n">batch_size</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">32</span><span class="p">,</span>
+    <span class="p">):</span>
+<span class="w">        </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Pretrain method. Can also be called using `.fit(&lt;same_args&gt;)`</span>
+
+<span class="sd">        Parameters</span>
+<span class="sd">        ----------</span>
+<span class="sd">        X_tab: np.ndarray,</span>
+<span class="sd">            tabular dataset</span>
+<span class="sd">        X_tab_val: np.ndarray, Optional, default = None</span>
+<span class="sd">            validation data</span>
+<span class="sd">        val_split: float, Optional. default=None</span>
+<span class="sd">            An alterative to passing the validation set is to use a train/val</span>
+<span class="sd">            split fraction via `val_split`</span>
+<span class="sd">        validation_freq: int, default=1</span>
+<span class="sd">            epochs validation frequency</span>
+<span class="sd">        n_epochs: int, default=1</span>
+<span class="sd">            number of epochs</span>
+<span class="sd">        batch_size: int, default=32</span>
+<span class="sd">            batch size</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">batch_size</span> <span class="o">=</span> <span class="n">batch_size</span>
+
+        <span class="n">train_set</span><span class="p">,</span> <span class="n">eval_set</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_train_eval_split</span><span class="p">(</span><span class="n">X_tab</span><span class="p">,</span> <span class="n">X_tab_val</span><span class="p">,</span> <span class="n">val_split</span><span class="p">)</span>
+        <span class="n">train_loader</span> <span class="o">=</span> <span class="n">DataLoader</span><span class="p">(</span>
+            <span class="n">dataset</span><span class="o">=</span><span class="n">train_set</span><span class="p">,</span> <span class="n">batch_size</span><span class="o">=</span><span class="n">batch_size</span><span class="p">,</span> <span class="n">num_workers</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">num_workers</span>
+        <span class="p">)</span>
+        <span class="n">train_steps</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">train_loader</span><span class="p">)</span>
+        <span class="k">if</span> <span class="n">eval_set</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="n">eval_loader</span> <span class="o">=</span> <span class="n">DataLoader</span><span class="p">(</span>
+                <span class="n">dataset</span><span class="o">=</span><span class="n">eval_set</span><span class="p">,</span>
+                <span class="n">batch_size</span><span class="o">=</span><span class="n">batch_size</span><span class="p">,</span>
+                <span class="n">num_workers</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">num_workers</span><span class="p">,</span>
+                <span class="n">shuffle</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
+            <span class="p">)</span>
+            <span class="n">eval_steps</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">eval_loader</span><span class="p">)</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">callback_container</span><span class="o">.</span><span class="n">on_train_begin</span><span class="p">(</span>
+            <span class="p">{</span>
+                <span class="s2">&quot;batch_size&quot;</span><span class="p">:</span> <span class="n">batch_size</span><span class="p">,</span>
+                <span class="s2">&quot;train_steps&quot;</span><span class="p">:</span> <span class="n">train_steps</span><span class="p">,</span>
+                <span class="s2">&quot;n_epochs&quot;</span><span class="p">:</span> <span class="n">n_epochs</span><span class="p">,</span>
+            <span class="p">}</span>
+        <span class="p">)</span>
+        <span class="k">for</span> <span class="n">epoch</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">n_epochs</span><span class="p">):</span>
+            <span class="n">epoch_logs</span><span class="p">:</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="p">{}</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">callback_container</span><span class="o">.</span><span class="n">on_epoch_begin</span><span class="p">(</span><span class="n">epoch</span><span class="p">,</span> <span class="n">logs</span><span class="o">=</span><span class="n">epoch_logs</span><span class="p">)</span>
+
+            <span class="bp">self</span><span class="o">.</span><span class="n">train_running_loss</span> <span class="o">=</span> <span class="mf">0.0</span>
+            <span class="k">with</span> <span class="n">trange</span><span class="p">(</span><span class="n">train_steps</span><span class="p">,</span> <span class="n">disable</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">verbose</span> <span class="o">!=</span> <span class="mi">1</span><span class="p">)</span> <span class="k">as</span> <span class="n">t</span><span class="p">:</span>
+                <span class="k">for</span> <span class="n">batch_idx</span><span class="p">,</span> <span class="n">X</span> <span class="ow">in</span> <span class="nb">zip</span><span class="p">(</span><span class="n">t</span><span class="p">,</span> <span class="n">train_loader</span><span class="p">):</span>
+                    <span class="n">t</span><span class="o">.</span><span class="n">set_description</span><span class="p">(</span><span class="s2">&quot;epoch </span><span class="si">%i</span><span class="s2">&quot;</span> <span class="o">%</span> <span class="p">(</span><span class="n">epoch</span> <span class="o">+</span> <span class="mi">1</span><span class="p">))</span>
+                    <span class="n">train_loss</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_train_step</span><span class="p">(</span><span class="n">X</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="n">batch_idx</span><span class="p">)</span>
+                    <span class="bp">self</span><span class="o">.</span><span class="n">callback_container</span><span class="o">.</span><span class="n">on_batch_end</span><span class="p">(</span><span class="n">batch</span><span class="o">=</span><span class="n">batch_idx</span><span class="p">)</span>
+                    <span class="n">print_loss_and_metric</span><span class="p">(</span><span class="n">t</span><span class="p">,</span> <span class="n">train_loss</span><span class="p">)</span>
+
+            <span class="n">epoch_logs</span> <span class="o">=</span> <span class="n">save_epoch_logs</span><span class="p">(</span><span class="n">epoch_logs</span><span class="p">,</span> <span class="n">train_loss</span><span class="p">,</span> <span class="kc">None</span><span class="p">,</span> <span class="s2">&quot;train&quot;</span><span class="p">)</span>
+
+            <span class="n">on_epoch_end_metric</span> <span class="o">=</span> <span class="kc">None</span>
+            <span class="k">if</span> <span class="n">eval_set</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">and</span> <span class="n">epoch</span> <span class="o">%</span> <span class="n">validation_freq</span> <span class="o">==</span> <span class="p">(</span>
+                <span class="n">validation_freq</span> <span class="o">-</span> <span class="mi">1</span>
+            <span class="p">):</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">callback_container</span><span class="o">.</span><span class="n">on_eval_begin</span><span class="p">()</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">valid_running_loss</span> <span class="o">=</span> <span class="mf">0.0</span>
+                <span class="k">with</span> <span class="n">trange</span><span class="p">(</span><span class="n">eval_steps</span><span class="p">,</span> <span class="n">disable</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">verbose</span> <span class="o">!=</span> <span class="mi">1</span><span class="p">)</span> <span class="k">as</span> <span class="n">v</span><span class="p">:</span>
+                    <span class="k">for</span> <span class="n">batch_idx</span><span class="p">,</span> <span class="n">X</span> <span class="ow">in</span> <span class="nb">zip</span><span class="p">(</span><span class="n">v</span><span class="p">,</span> <span class="n">eval_loader</span><span class="p">):</span>
+                        <span class="n">v</span><span class="o">.</span><span class="n">set_description</span><span class="p">(</span><span class="s2">&quot;valid&quot;</span><span class="p">)</span>
+                        <span class="n">val_loss</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_eval_step</span><span class="p">(</span><span class="n">X</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="n">batch_idx</span><span class="p">)</span>
+                        <span class="n">print_loss_and_metric</span><span class="p">(</span><span class="n">v</span><span class="p">,</span> <span class="n">val_loss</span><span class="p">)</span>
+                <span class="n">epoch_logs</span> <span class="o">=</span> <span class="n">save_epoch_logs</span><span class="p">(</span><span class="n">epoch_logs</span><span class="p">,</span> <span class="n">val_loss</span><span class="p">,</span> <span class="kc">None</span><span class="p">,</span> <span class="s2">&quot;val&quot;</span><span class="p">)</span>
+                <span class="n">on_epoch_end_metric</span> <span class="o">=</span> <span class="n">val_loss</span>
+            <span class="k">else</span><span class="p">:</span>
+                <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">reducelronplateau</span><span class="p">:</span>
+                    <span class="k">raise</span> <span class="ne">NotImplementedError</span><span class="p">(</span>
+                        <span class="s2">&quot;ReduceLROnPlateau scheduler can be used only with validation data.&quot;</span>
+                    <span class="p">)</span>
+
+            <span class="bp">self</span><span class="o">.</span><span class="n">callback_container</span><span class="o">.</span><span class="n">on_epoch_end</span><span class="p">(</span><span class="n">epoch</span><span class="p">,</span> <span class="n">epoch_logs</span><span class="p">,</span> <span class="n">on_epoch_end_metric</span><span class="p">)</span>
+
+            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">early_stop</span><span class="p">:</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">callback_container</span><span class="o">.</span><span class="n">on_train_end</span><span class="p">(</span><span class="n">epoch_logs</span><span class="p">)</span>
+                <span class="k">break</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">callback_container</span><span class="o">.</span><span class="n">on_train_end</span><span class="p">(</span><span class="n">epoch_logs</span><span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">_restore_best_weights</span><span class="p">()</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">ed_model</span><span class="o">.</span><span class="n">train</span><span class="p">()</span>
+
+    <span class="k">def</span> <span class="nf">fit</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span>
+        <span class="n">X_tab</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">,</span>
+        <span class="n">X_tab_val</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">val_split</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">validation_freq</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1</span><span class="p">,</span>
+        <span class="n">n_epochs</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1</span><span class="p">,</span>
+        <span class="n">batch_size</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">32</span><span class="p">,</span>
+    <span class="p">):</span>
+        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">pretrain</span><span class="p">(</span>
+            <span class="n">X_tab</span><span class="p">,</span> <span class="n">X_tab_val</span><span class="p">,</span> <span class="n">val_split</span><span class="p">,</span> <span class="n">validation_freq</span><span class="p">,</span> <span class="n">n_epochs</span><span class="p">,</span> <span class="n">batch_size</span>
+        <span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">explain</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">X_tab</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">,</span> <span class="n">save_step_masks</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">):</span>
+        <span class="k">raise</span> <span class="ne">NotImplementedError</span><span class="p">(</span>
+            <span class="s2">&quot;The &#39;explain&#39; is currently not implemented for Self Supervised Pretraining&quot;</span>
+        <span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">_train_step</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">X_tab</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">,</span> <span class="n">batch_idx</span><span class="p">:</span> <span class="nb">int</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">float</span><span class="p">:</span>
+        <span class="n">X</span> <span class="o">=</span> <span class="n">X_tab</span><span class="o">.</span><span class="n">to</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">device</span><span class="p">)</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">optimizer</span><span class="o">.</span><span class="n">zero_grad</span><span class="p">()</span>
+        <span class="n">x_embed</span><span class="p">,</span> <span class="n">x_embed_rec</span><span class="p">,</span> <span class="n">mask</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">ed_model</span><span class="p">(</span><span class="n">X</span><span class="p">)</span>
+        <span class="n">loss</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">loss_fn</span><span class="p">(</span><span class="n">x_embed</span><span class="p">,</span> <span class="n">x_embed_rec</span><span class="p">,</span> <span class="n">mask</span><span class="p">)</span>
+        <span class="n">loss</span><span class="o">.</span><span class="n">backward</span><span class="p">()</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">optimizer</span><span class="o">.</span><span class="n">step</span><span class="p">()</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">train_running_loss</span> <span class="o">+=</span> <span class="n">loss</span><span class="o">.</span><span class="n">item</span><span class="p">()</span>
+        <span class="n">avg_loss</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">train_running_loss</span> <span class="o">/</span> <span class="p">(</span><span class="n">batch_idx</span> <span class="o">+</span> <span class="mi">1</span><span class="p">)</span>
+
+        <span class="k">return</span> <span class="n">avg_loss</span>
+
+    <span class="k">def</span> <span class="nf">_eval_step</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">X_tab</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">,</span> <span class="n">batch_idx</span><span class="p">:</span> <span class="nb">int</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">float</span><span class="p">:</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">ed_model</span><span class="o">.</span><span class="n">eval</span><span class="p">()</span>
+
+        <span class="k">with</span> <span class="n">torch</span><span class="o">.</span><span class="n">no_grad</span><span class="p">():</span>
+            <span class="n">X</span> <span class="o">=</span> <span class="n">X_tab</span><span class="o">.</span><span class="n">to</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">device</span><span class="p">)</span>
+
+            <span class="n">x_embed</span><span class="p">,</span> <span class="n">x_embed_rec</span><span class="p">,</span> <span class="n">mask</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">ed_model</span><span class="p">(</span><span class="n">X</span><span class="p">)</span>
+            <span class="n">loss</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">loss_fn</span><span class="p">(</span><span class="n">x_embed</span><span class="p">,</span> <span class="n">x_embed_rec</span><span class="p">,</span> <span class="n">mask</span><span class="p">)</span>
+
+            <span class="bp">self</span><span class="o">.</span><span class="n">valid_running_loss</span> <span class="o">+=</span> <span class="n">loss</span><span class="o">.</span><span class="n">item</span><span class="p">()</span>
+            <span class="n">avg_loss</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">valid_running_loss</span> <span class="o">/</span> <span class="p">(</span><span class="n">batch_idx</span> <span class="o">+</span> <span class="mi">1</span><span class="p">)</span>
+
+        <span class="k">return</span> <span class="n">avg_loss</span>
+
+    <span class="k">def</span> <span class="nf">_train_eval_split</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span>
+        <span class="n">X</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">,</span>
+        <span class="n">X_tab_val</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">val_split</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+    <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tuple</span><span class="p">[</span><span class="n">TensorDataset</span><span class="p">,</span> <span class="n">Optional</span><span class="p">[</span><span class="n">TensorDataset</span><span class="p">]]:</span>
+        <span class="k">if</span> <span class="n">X_tab_val</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="n">train_set</span> <span class="o">=</span> <span class="n">TensorDataset</span><span class="p">(</span><span class="n">torch</span><span class="o">.</span><span class="n">from_numpy</span><span class="p">(</span><span class="n">X</span><span class="p">))</span>
+            <span class="n">eval_set</span> <span class="o">=</span> <span class="n">TensorDataset</span><span class="p">(</span><span class="n">torch</span><span class="o">.</span><span class="n">from_numpy</span><span class="p">(</span><span class="n">X_tab_val</span><span class="p">))</span>
+        <span class="k">elif</span> <span class="n">val_split</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="n">X_tr</span><span class="p">,</span> <span class="n">X_tab_val</span> <span class="o">=</span> <span class="n">train_test_split</span><span class="p">(</span>  <span class="c1"># type: ignore</span>
+                <span class="n">X</span><span class="p">,</span> <span class="n">test_size</span><span class="o">=</span><span class="n">val_split</span><span class="p">,</span> <span class="n">random_state</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">seed</span>
+            <span class="p">)</span>
+            <span class="n">train_set</span> <span class="o">=</span> <span class="n">TensorDataset</span><span class="p">(</span><span class="n">torch</span><span class="o">.</span><span class="n">from_numpy</span><span class="p">(</span><span class="n">X_tr</span><span class="p">))</span>
+            <span class="n">eval_set</span> <span class="o">=</span> <span class="n">TensorDataset</span><span class="p">(</span><span class="n">torch</span><span class="o">.</span><span class="n">from_numpy</span><span class="p">(</span><span class="n">X_tab_val</span><span class="p">))</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="n">train_set</span> <span class="o">=</span> <span class="n">TensorDataset</span><span class="p">(</span><span class="n">torch</span><span class="o">.</span><span class="n">from_numpy</span><span class="p">(</span><span class="n">X</span><span class="p">))</span>
+            <span class="n">eval_set</span> <span class="o">=</span> <span class="kc">None</span>
+
+        <span class="k">return</span> <span class="n">train_set</span><span class="p">,</span> <span class="n">eval_set</span>
 </code></pre></div></td></tr></table></div>
-                  </details>
+              </details>
 
 
 
@@ -1882,67 +2353,103 @@ <h3 id="pytorch_widedeep.self_supervised_training.EncoderDecoderTrainer.pretrain
 
 
 <p><span class="doc-section-title">Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>X_tab</code></b>
-              (<code><span title="numpy.ndarray">ndarray</span></code>)
-          –
-          <div class="doc-md-description">
-            <p>tabular dataset</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>X_tab_val</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="numpy.ndarray">ndarray</span>]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>validation data</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>val_split</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+          <th>Default</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>X_tab</code></td>
+            <td>
+                  <code><span title="numpy.ndarray">ndarray</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>tabular dataset</p>
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>X_tab_val</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="numpy.ndarray">ndarray</span>]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>validation data</p>
+              </div>
+            </td>
+            <td>
                   <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>An alterative to passing the validation set is to use a train/val
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>val_split</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>An alterative to passing the validation set is to use a train/val
 split fraction via <code>val_split</code></p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>validation_freq</code></b>
-              (<code>int</code>, default:
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>validation_freq</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>epochs validation frequency</p>
+              </div>
+            </td>
+            <td>
                   <code>1</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>epochs validation frequency</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>n_epochs</code></b>
-              (<code>int</code>, default:
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>n_epochs</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>number of epochs</p>
+              </div>
+            </td>
+            <td>
                   <code>1</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>number of epochs</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>batch_size</code></b>
-              (<code>int</code>, default:
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>batch_size</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>batch size</p>
+              </div>
+            </td>
+            <td>
                   <code>32</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>batch size</p>
-          </div>
-        </li>
-    </ul>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
             <details class="quote">
               <summary>Source code in <code>pytorch_widedeep/self_supervised_training/encoder_decoder_trainer.py</code></summary>
@@ -2154,24 +2661,7 @@ <h2 id="pytorch_widedeep.self_supervised_training.ContrastiveDenoisingTrainer" c
 
 
 <a href="#pytorch_widedeep.self_supervised_training.ContrastiveDenoisingTrainer" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="nf">ContrastiveDenoisingTrainer</span><span class="p">(</span>
-    <span class="n">model</span><span class="p">,</span>
-    <span class="n">preprocessor</span><span class="p">,</span>
-    <span class="n">optimizer</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">lr_scheduler</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">callbacks</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">loss_type</span><span class="o">=</span><span class="s2">&quot;both&quot;</span><span class="p">,</span>
-    <span class="n">projection_head1_dims</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">projection_head2_dims</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">projection_heads_activation</span><span class="o">=</span><span class="s2">&quot;relu&quot;</span><span class="p">,</span>
-    <span class="n">cat_mlp_type</span><span class="o">=</span><span class="s2">&quot;multiple&quot;</span><span class="p">,</span>
-    <span class="n">cont_mlp_type</span><span class="o">=</span><span class="s2">&quot;multiple&quot;</span><span class="p">,</span>
-    <span class="n">denoise_mlps_activation</span><span class="o">=</span><span class="s2">&quot;relu&quot;</span><span class="p">,</span>
-    <span class="n">verbose</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span>
-    <span class="n">seed</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span>
-    <span class="o">**</span><span class="n">kwargs</span>
-<span class="p">)</span>
-</code></pre></div>
+
 
     <div class="doc doc-contents first">
             <p class="doc doc-class-bases">
@@ -2185,83 +2675,120 @@ <h2 id="pytorch_widedeep.self_supervised_training.ContrastiveDenoisingTrainer" c
 
 
 <p><span class="doc-section-title">Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>model</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.ModelWithAttention">ModelWithAttention</span></code>)
-          –
-          <div class="doc-md-description">
-            <p>An instance of a <code>TabTransformer</code>, <code>SAINT</code>, <code>FTTransformer</code>,
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+          <th>Default</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>model</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.ModelWithAttention">ModelWithAttention</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>An instance of a <code>TabTransformer</code>, <code>SAINT</code>, <code>FTTransformer</code>,
 <code>TabFastFormer</code>, <code>TabPerceiver</code>, <code>ContextAttentionMLP</code> and
 <code>SelfAttentionMLP</code>.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>preprocessor</code></b>
-              (<code><a class="autorefs autorefs-internal" title="pytorch_widedeep.preprocessing.TabPreprocessor" href="preprocessing.html#pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor">TabPreprocessor</a></code>)
-          –
-          <div class="doc-md-description">
-            <p>A fitted <code>TabPreprocessor</code> object. See
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>preprocessor</code></td>
+            <td>
+                  <code><a class="autorefs autorefs-internal" title="pytorch_widedeep.preprocessing.TabPreprocessor" href="preprocessing.html#pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor">TabPreprocessor</a></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>A fitted <code>TabPreprocessor</code> object. See
 <code>pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor</code></p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>optimizer</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Optimizer">Optimizer</span>]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>An instance of Pytorch's <code>Optimizer</code> object (e.g. <code>torch.optim.Adam
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>optimizer</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Optimizer">Optimizer</span>]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>An instance of Pytorch's <code>Optimizer</code> object (e.g. <code>torch.optim.Adam
 ()</code>). if no optimizer is passed it will default to <code>AdamW</code>.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>lr_scheduler</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.LRScheduler">LRScheduler</span>]</code>, default:
+              </div>
+            </td>
+            <td>
                   <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>An instance of Pytorch's <code>LRScheduler</code> object
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>lr_scheduler</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.LRScheduler">LRScheduler</span>]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>An instance of Pytorch's <code>LRScheduler</code> object
 (e.g <code>torch.optim.lr_scheduler.StepLR(opt, step_size=5)</code>).</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>callbacks</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.callbacks.Callback">Callback</span>]]</code>, default:
+              </div>
+            </td>
+            <td>
                   <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>List with <code>Callback</code> objects. The three callbacks available in
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>callbacks</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.callbacks.Callback">Callback</span>]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>List with <code>Callback</code> objects. The three callbacks available in
 <code>pytorch-widedeep</code> are: <code>LRHistory</code>, <code>ModelCheckpoint</code> and
 <code>EarlyStopping</code>. This can also be a custom callback. See
 <code>pytorch_widedeep.callbacks.Callback</code> or the Examples folder in the
 repo.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>loss_type</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[contrastive, denoising, both]</code>, default:
-                  <code>&#39;both&#39;</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>One of '<em>contrastive</em>', '<em>denoising</em>' or '<em>both</em>'. See <a href="https://arxiv.org/abs/2203.05556">SAINT: Improved
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>loss_type</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[contrastive, denoising, both]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>One of '<em>contrastive</em>', '<em>denoising</em>' or '<em>both</em>'. See <a href="https://arxiv.org/abs/2203.05556">SAINT: Improved
 Neural Networks for Tabular Data via Row Attention and Contrastive
 Pre-Training</a>, their figure (1)
 and their equation (5).</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>projection_head1_dims</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[int]]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>The projection heads are simply MLPs. This parameter is a list
+              </div>
+            </td>
+            <td>
+                  <code>&#39;both&#39;</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>projection_head1_dims</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[int]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>The projection heads are simply MLPs. This parameter is a list
 of integers with the dimensions of the MLP hidden layers. See the
 <a href="https://arxiv.org/abs/2203.05556">paper</a> for details. Note that
 setting up this parameter requires some knowledge of the architecture
@@ -2269,93 +2796,136 @@ <h2 id="pytorch_widedeep.self_supervised_training.ContrastiveDenoisingTrainer" c
 embeddings of dim 32 (i.e. the so called dimension of the model is
 32), then the first dimension of the projection head must be 32 (e.g.
 [32, 16])</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>projection_head2_dims</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[int]]</code>, default:
+              </div>
+            </td>
+            <td>
                   <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Same as '<em>projection_head1_dims</em>' for the second head</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>projection_heads_activation</code></b>
-              (<code>str</code>, default:
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>projection_head2_dims</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[int]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Same as '<em>projection_head1_dims</em>' for the second head</p>
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>projection_heads_activation</code></td>
+            <td>
+                  <code>str</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Activation function for the projection heads</p>
+              </div>
+            </td>
+            <td>
                   <code>&#39;relu&#39;</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Activation function for the projection heads</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>cat_mlp_type</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[single, multiple]</code>, default:
-                  <code>&#39;multiple&#39;</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>If '<em>denoising</em>' loss is used, one can choose two types of 'stacked'
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>cat_mlp_type</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[single, multiple]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>If '<em>denoising</em>' loss is used, one can choose two types of 'stacked'
 MLPs to process the output from the transformer-based encoder that
 receives 'corrupted' (cut-mixed and mixed-up) features. These
 are '<em>single</em>' or '<em>multiple</em>'. The former approach will apply a single
 MLP to all the categorical features while the latter will use one MLP
 per categorical feature</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>cont_mlp_type</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[single, multiple]</code>, default:
+              </div>
+            </td>
+            <td>
                   <code>&#39;multiple&#39;</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Same as 'cat_mlp_type' but for the continuous features</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>denoise_mlps_activation</code></b>
-              (<code>str</code>, default:
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>cont_mlp_type</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[single, multiple]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Same as 'cat_mlp_type' but for the continuous features</p>
+              </div>
+            </td>
+            <td>
+                  <code>&#39;multiple&#39;</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>denoise_mlps_activation</code></td>
+            <td>
+                  <code>str</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>activation function for the so called 'denoising mlps'.</p>
+              </div>
+            </td>
+            <td>
                   <code>&#39;relu&#39;</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>activation function for the so called 'denoising mlps'.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>verbose</code></b>
-              (<code>int</code>, default:
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>verbose</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Setting it to 0 will print nothing during training.</p>
+              </div>
+            </td>
+            <td>
                   <code>1</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Setting it to 0 will print nothing during training.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>seed</code></b>
-              (<code>int</code>, default:
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>seed</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Random seed to be used internally for train_test_split</p>
+              </div>
+            </td>
+            <td>
                   <code>1</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Random seed to be used internally for train_test_split</p>
-          </div>
-        </li>
-    </ul>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
 
 <p><span class="doc-section-title">Other Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>**kwargs</code></b>
-          –
-          <div class="doc-md-description">
-            <p>Other infrequently used arguments that can also be passed as kwargs are:</p>
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>**kwargs</code></td>
+            <td>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Other infrequently used arguments that can also be passed as kwargs are:</p>
 <ul>
 <li>
 <p><strong>device</strong>: <code>str</code><br/>
@@ -2372,13 +2942,94 @@ <h2 id="pytorch_widedeep.self_supervised_training.ContrastiveDenoisingTrainer" c
     learning rate is a bit particular.</p>
 </li>
 </ul>
-          </div>
-        </li>
-    </ul>
-
-                  <details class="quote">
-                    <summary>Source code in <code>pytorch_widedeep/self_supervised_training/contrastive_denoising_trainer.py</code></summary>
-                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">108</span>
+              </div>
+            </td>
+          </tr>
+      </tbody>
+    </table>
+
+              <details class="quote">
+                <summary>Source code in <code>pytorch_widedeep/self_supervised_training/contrastive_denoising_trainer.py</code></summary>
+                <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 29</span>
+<span class="normal"> 30</span>
+<span class="normal"> 31</span>
+<span class="normal"> 32</span>
+<span class="normal"> 33</span>
+<span class="normal"> 34</span>
+<span class="normal"> 35</span>
+<span class="normal"> 36</span>
+<span class="normal"> 37</span>
+<span class="normal"> 38</span>
+<span class="normal"> 39</span>
+<span class="normal"> 40</span>
+<span class="normal"> 41</span>
+<span class="normal"> 42</span>
+<span class="normal"> 43</span>
+<span class="normal"> 44</span>
+<span class="normal"> 45</span>
+<span class="normal"> 46</span>
+<span class="normal"> 47</span>
+<span class="normal"> 48</span>
+<span class="normal"> 49</span>
+<span class="normal"> 50</span>
+<span class="normal"> 51</span>
+<span class="normal"> 52</span>
+<span class="normal"> 53</span>
+<span class="normal"> 54</span>
+<span class="normal"> 55</span>
+<span class="normal"> 56</span>
+<span class="normal"> 57</span>
+<span class="normal"> 58</span>
+<span class="normal"> 59</span>
+<span class="normal"> 60</span>
+<span class="normal"> 61</span>
+<span class="normal"> 62</span>
+<span class="normal"> 63</span>
+<span class="normal"> 64</span>
+<span class="normal"> 65</span>
+<span class="normal"> 66</span>
+<span class="normal"> 67</span>
+<span class="normal"> 68</span>
+<span class="normal"> 69</span>
+<span class="normal"> 70</span>
+<span class="normal"> 71</span>
+<span class="normal"> 72</span>
+<span class="normal"> 73</span>
+<span class="normal"> 74</span>
+<span class="normal"> 75</span>
+<span class="normal"> 76</span>
+<span class="normal"> 77</span>
+<span class="normal"> 78</span>
+<span class="normal"> 79</span>
+<span class="normal"> 80</span>
+<span class="normal"> 81</span>
+<span class="normal"> 82</span>
+<span class="normal"> 83</span>
+<span class="normal"> 84</span>
+<span class="normal"> 85</span>
+<span class="normal"> 86</span>
+<span class="normal"> 87</span>
+<span class="normal"> 88</span>
+<span class="normal"> 89</span>
+<span class="normal"> 90</span>
+<span class="normal"> 91</span>
+<span class="normal"> 92</span>
+<span class="normal"> 93</span>
+<span class="normal"> 94</span>
+<span class="normal"> 95</span>
+<span class="normal"> 96</span>
+<span class="normal"> 97</span>
+<span class="normal"> 98</span>
+<span class="normal"> 99</span>
+<span class="normal">100</span>
+<span class="normal">101</span>
+<span class="normal">102</span>
+<span class="normal">103</span>
+<span class="normal">104</span>
+<span class="normal">105</span>
+<span class="normal">106</span>
+<span class="normal">107</span>
+<span class="normal">108</span>
 <span class="normal">109</span>
 <span class="normal">110</span>
 <span class="normal">111</span>
@@ -2412,43 +3063,450 @@ <h2 id="pytorch_widedeep.self_supervised_training.ContrastiveDenoisingTrainer" c
 <span class="normal">139</span>
 <span class="normal">140</span>
 <span class="normal">141</span>
-<span class="normal">142</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
-    <span class="bp">self</span><span class="p">,</span>
-    <span class="n">model</span><span class="p">:</span> <span class="n">ModelWithAttention</span><span class="p">,</span>
-    <span class="n">preprocessor</span><span class="p">:</span> <span class="n">TabPreprocessor</span><span class="p">,</span>
-    <span class="n">optimizer</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Optimizer</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">lr_scheduler</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">LRScheduler</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">callbacks</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="n">Callback</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">loss_type</span><span class="p">:</span> <span class="n">Literal</span><span class="p">[</span><span class="s2">&quot;contrastive&quot;</span><span class="p">,</span> <span class="s2">&quot;denoising&quot;</span><span class="p">,</span> <span class="s2">&quot;both&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="s2">&quot;both&quot;</span><span class="p">,</span>
-    <span class="n">projection_head1_dims</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">int</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">projection_head2_dims</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">int</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">projection_heads_activation</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;relu&quot;</span><span class="p">,</span>
-    <span class="n">cat_mlp_type</span><span class="p">:</span> <span class="n">Literal</span><span class="p">[</span><span class="s2">&quot;single&quot;</span><span class="p">,</span> <span class="s2">&quot;multiple&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="s2">&quot;multiple&quot;</span><span class="p">,</span>
-    <span class="n">cont_mlp_type</span><span class="p">:</span> <span class="n">Literal</span><span class="p">[</span><span class="s2">&quot;single&quot;</span><span class="p">,</span> <span class="s2">&quot;multiple&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="s2">&quot;multiple&quot;</span><span class="p">,</span>
-    <span class="n">denoise_mlps_activation</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;relu&quot;</span><span class="p">,</span>
-    <span class="n">verbose</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1</span><span class="p">,</span>
-    <span class="n">seed</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1</span><span class="p">,</span>
-    <span class="o">**</span><span class="n">kwargs</span><span class="p">,</span>
-<span class="p">):</span>
-    <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span>
-        <span class="n">model</span><span class="o">=</span><span class="n">model</span><span class="p">,</span>
-        <span class="n">preprocessor</span><span class="o">=</span><span class="n">preprocessor</span><span class="p">,</span>
-        <span class="n">loss_type</span><span class="o">=</span><span class="n">loss_type</span><span class="p">,</span>
-        <span class="n">optimizer</span><span class="o">=</span><span class="n">optimizer</span><span class="p">,</span>
-        <span class="n">lr_scheduler</span><span class="o">=</span><span class="n">lr_scheduler</span><span class="p">,</span>
-        <span class="n">callbacks</span><span class="o">=</span><span class="n">callbacks</span><span class="p">,</span>
-        <span class="n">projection_head1_dims</span><span class="o">=</span><span class="n">projection_head1_dims</span><span class="p">,</span>
-        <span class="n">projection_head2_dims</span><span class="o">=</span><span class="n">projection_head2_dims</span><span class="p">,</span>
-        <span class="n">projection_heads_activation</span><span class="o">=</span><span class="n">projection_heads_activation</span><span class="p">,</span>
-        <span class="n">cat_mlp_type</span><span class="o">=</span><span class="n">cat_mlp_type</span><span class="p">,</span>
-        <span class="n">cont_mlp_type</span><span class="o">=</span><span class="n">cont_mlp_type</span><span class="p">,</span>
-        <span class="n">denoise_mlps_activation</span><span class="o">=</span><span class="n">denoise_mlps_activation</span><span class="p">,</span>
-        <span class="n">verbose</span><span class="o">=</span><span class="n">verbose</span><span class="p">,</span>
-        <span class="n">seed</span><span class="o">=</span><span class="n">seed</span><span class="p">,</span>
+<span class="normal">142</span>
+<span class="normal">143</span>
+<span class="normal">144</span>
+<span class="normal">145</span>
+<span class="normal">146</span>
+<span class="normal">147</span>
+<span class="normal">148</span>
+<span class="normal">149</span>
+<span class="normal">150</span>
+<span class="normal">151</span>
+<span class="normal">152</span>
+<span class="normal">153</span>
+<span class="normal">154</span>
+<span class="normal">155</span>
+<span class="normal">156</span>
+<span class="normal">157</span>
+<span class="normal">158</span>
+<span class="normal">159</span>
+<span class="normal">160</span>
+<span class="normal">161</span>
+<span class="normal">162</span>
+<span class="normal">163</span>
+<span class="normal">164</span>
+<span class="normal">165</span>
+<span class="normal">166</span>
+<span class="normal">167</span>
+<span class="normal">168</span>
+<span class="normal">169</span>
+<span class="normal">170</span>
+<span class="normal">171</span>
+<span class="normal">172</span>
+<span class="normal">173</span>
+<span class="normal">174</span>
+<span class="normal">175</span>
+<span class="normal">176</span>
+<span class="normal">177</span>
+<span class="normal">178</span>
+<span class="normal">179</span>
+<span class="normal">180</span>
+<span class="normal">181</span>
+<span class="normal">182</span>
+<span class="normal">183</span>
+<span class="normal">184</span>
+<span class="normal">185</span>
+<span class="normal">186</span>
+<span class="normal">187</span>
+<span class="normal">188</span>
+<span class="normal">189</span>
+<span class="normal">190</span>
+<span class="normal">191</span>
+<span class="normal">192</span>
+<span class="normal">193</span>
+<span class="normal">194</span>
+<span class="normal">195</span>
+<span class="normal">196</span>
+<span class="normal">197</span>
+<span class="normal">198</span>
+<span class="normal">199</span>
+<span class="normal">200</span>
+<span class="normal">201</span>
+<span class="normal">202</span>
+<span class="normal">203</span>
+<span class="normal">204</span>
+<span class="normal">205</span>
+<span class="normal">206</span>
+<span class="normal">207</span>
+<span class="normal">208</span>
+<span class="normal">209</span>
+<span class="normal">210</span>
+<span class="normal">211</span>
+<span class="normal">212</span>
+<span class="normal">213</span>
+<span class="normal">214</span>
+<span class="normal">215</span>
+<span class="normal">216</span>
+<span class="normal">217</span>
+<span class="normal">218</span>
+<span class="normal">219</span>
+<span class="normal">220</span>
+<span class="normal">221</span>
+<span class="normal">222</span>
+<span class="normal">223</span>
+<span class="normal">224</span>
+<span class="normal">225</span>
+<span class="normal">226</span>
+<span class="normal">227</span>
+<span class="normal">228</span>
+<span class="normal">229</span>
+<span class="normal">230</span>
+<span class="normal">231</span>
+<span class="normal">232</span>
+<span class="normal">233</span>
+<span class="normal">234</span>
+<span class="normal">235</span>
+<span class="normal">236</span>
+<span class="normal">237</span>
+<span class="normal">238</span>
+<span class="normal">239</span>
+<span class="normal">240</span>
+<span class="normal">241</span>
+<span class="normal">242</span>
+<span class="normal">243</span>
+<span class="normal">244</span>
+<span class="normal">245</span>
+<span class="normal">246</span>
+<span class="normal">247</span>
+<span class="normal">248</span>
+<span class="normal">249</span>
+<span class="normal">250</span>
+<span class="normal">251</span>
+<span class="normal">252</span>
+<span class="normal">253</span>
+<span class="normal">254</span>
+<span class="normal">255</span>
+<span class="normal">256</span>
+<span class="normal">257</span>
+<span class="normal">258</span>
+<span class="normal">259</span>
+<span class="normal">260</span>
+<span class="normal">261</span>
+<span class="normal">262</span>
+<span class="normal">263</span>
+<span class="normal">264</span>
+<span class="normal">265</span>
+<span class="normal">266</span>
+<span class="normal">267</span>
+<span class="normal">268</span>
+<span class="normal">269</span>
+<span class="normal">270</span>
+<span class="normal">271</span>
+<span class="normal">272</span>
+<span class="normal">273</span>
+<span class="normal">274</span>
+<span class="normal">275</span>
+<span class="normal">276</span>
+<span class="normal">277</span>
+<span class="normal">278</span>
+<span class="normal">279</span>
+<span class="normal">280</span>
+<span class="normal">281</span>
+<span class="normal">282</span>
+<span class="normal">283</span>
+<span class="normal">284</span>
+<span class="normal">285</span>
+<span class="normal">286</span>
+<span class="normal">287</span>
+<span class="normal">288</span>
+<span class="normal">289</span>
+<span class="normal">290</span>
+<span class="normal">291</span>
+<span class="normal">292</span>
+<span class="normal">293</span>
+<span class="normal">294</span>
+<span class="normal">295</span>
+<span class="normal">296</span>
+<span class="normal">297</span>
+<span class="normal">298</span>
+<span class="normal">299</span>
+<span class="normal">300</span>
+<span class="normal">301</span>
+<span class="normal">302</span>
+<span class="normal">303</span>
+<span class="normal">304</span>
+<span class="normal">305</span>
+<span class="normal">306</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">class</span> <span class="nc">ContrastiveDenoisingTrainer</span><span class="p">(</span><span class="n">BaseContrastiveDenoisingTrainer</span><span class="p">):</span>
+<span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;This class trains a Contrastive, Denoising Self Supervised &#39;routine&#39; that</span>
+<span class="sd">    is based on the one described in</span>
+<span class="sd">    [SAINT: Improved Neural Networks for Tabular Data via Row Attention and</span>
+<span class="sd">    Contrastive Pre-Training](https://arxiv.org/abs/2106.01342), their Figure 1.</span>
+
+<span class="sd">    Parameters</span>
+<span class="sd">    ----------</span>
+<span class="sd">    model: ModelWithAttention,</span>
+<span class="sd">        An instance of a `TabTransformer`, `SAINT`, `FTTransformer`,</span>
+<span class="sd">        `TabFastFormer`, `TabPerceiver`, `ContextAttentionMLP` and</span>
+<span class="sd">        `SelfAttentionMLP`.</span>
+<span class="sd">    preprocessor: `TabPreprocessor`</span>
+<span class="sd">        A fitted `TabPreprocessor` object. See</span>
+<span class="sd">        `pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor`</span>
+<span class="sd">    optimizer: Optional[Optimizer] = None,</span>
+<span class="sd">        An instance of Pytorch&#39;s `Optimizer` object (e.g. `torch.optim.Adam</span>
+<span class="sd">        ()`). if no optimizer is passed it will default to `AdamW`.</span>
+<span class="sd">    lr_scheduler: Optional[LRScheduler] = None,</span>
+<span class="sd">        An instance of Pytorch&#39;s `LRScheduler` object</span>
+<span class="sd">        (e.g `torch.optim.lr_scheduler.StepLR(opt, step_size=5)`).</span>
+<span class="sd">    callbacks: Optional[List[Callback]] = None,</span>
+<span class="sd">        List with `Callback` objects. The three callbacks available in</span>
+<span class="sd">        `pytorch-widedeep` are: `LRHistory`, `ModelCheckpoint` and</span>
+<span class="sd">        `EarlyStopping`. This can also be a custom callback. See</span>
+<span class="sd">        `pytorch_widedeep.callbacks.Callback` or the Examples folder in the</span>
+<span class="sd">        repo.</span>
+<span class="sd">    loss_type: str, default = &quot;both&quot;</span>
+<span class="sd">        One of &#39;_contrastive_&#39;, &#39;_denoising_&#39; or &#39;_both_&#39;. See [SAINT: Improved</span>
+<span class="sd">        Neural Networks for Tabular Data via Row Attention and Contrastive</span>
+<span class="sd">        Pre-Training](https://arxiv.org/abs/2203.05556), their figure (1)</span>
+<span class="sd">        and their equation (5).</span>
+<span class="sd">    projection_head1_dims: list, Optional, default = None</span>
+<span class="sd">        The projection heads are simply MLPs. This parameter is a list</span>
+<span class="sd">        of integers with the dimensions of the MLP hidden layers. See the</span>
+<span class="sd">        [paper](https://arxiv.org/abs/2203.05556) for details. Note that</span>
+<span class="sd">        setting up this parameter requires some knowledge of the architecture</span>
+<span class="sd">        one is using. For example, if we are representing the features with</span>
+<span class="sd">        embeddings of dim 32 (i.e. the so called dimension of the model is</span>
+<span class="sd">        32), then the first dimension of the projection head must be 32 (e.g.</span>
+<span class="sd">        [32, 16])</span>
+<span class="sd">    projection_head2_dims: list, Optional, default = None</span>
+<span class="sd">        Same as &#39;_projection_head1_dims_&#39; for the second head</span>
+<span class="sd">    projection_heads_activation: str, default = &quot;relu&quot;</span>
+<span class="sd">        Activation function for the projection heads</span>
+<span class="sd">    cat_mlp_type: str, default = &quot;multiple&quot;</span>
+<span class="sd">        If &#39;_denoising_&#39; loss is used, one can choose two types of &#39;stacked&#39;</span>
+<span class="sd">        MLPs to process the output from the transformer-based encoder that</span>
+<span class="sd">        receives &#39;corrupted&#39; (cut-mixed and mixed-up) features. These</span>
+<span class="sd">        are &#39;_single_&#39; or &#39;_multiple_&#39;. The former approach will apply a single</span>
+<span class="sd">        MLP to all the categorical features while the latter will use one MLP</span>
+<span class="sd">        per categorical feature</span>
+<span class="sd">    cont_mlp_type: str, default = &quot;multiple&quot;</span>
+<span class="sd">        Same as &#39;cat_mlp_type&#39; but for the continuous features</span>
+<span class="sd">    denoise_mlps_activation: str, default = &quot;relu&quot;</span>
+<span class="sd">        activation function for the so called &#39;denoising mlps&#39;.</span>
+<span class="sd">    verbose: int, default=1</span>
+<span class="sd">        Setting it to 0 will print nothing during training.</span>
+<span class="sd">    seed: int, default=1</span>
+<span class="sd">        Random seed to be used internally for train_test_split</span>
+
+<span class="sd">    Other Parameters</span>
+<span class="sd">    ----------------</span>
+<span class="sd">    **kwargs: dict</span>
+<span class="sd">        Other infrequently used arguments that can also be passed as kwargs are:</span>
+
+<span class="sd">        - **device**: `str`&lt;br/&gt;</span>
+<span class="sd">            string indicating the device. One of _&#39;cpu&#39;_ or _&#39;gpu&#39;_</span>
+
+<span class="sd">        - **num_workers**: `int`&lt;br/&gt;</span>
+<span class="sd">            number of workers to be used internally by the data loaders</span>
+
+<span class="sd">        - **reducelronplateau_criterion**: `str`</span>
+<span class="sd">            This sets the criterion that will be used by the lr scheduler to</span>
+<span class="sd">            take a step: One of _&#39;loss&#39;_ or _&#39;metric&#39;_. The ReduceLROnPlateau</span>
+<span class="sd">            learning rate is a bit particular.</span>
+
+<span class="sd">    &quot;&quot;&quot;</span>
+
+    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span>
+        <span class="n">model</span><span class="p">:</span> <span class="n">ModelWithAttention</span><span class="p">,</span>
+        <span class="n">preprocessor</span><span class="p">:</span> <span class="n">TabPreprocessor</span><span class="p">,</span>
+        <span class="n">optimizer</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Optimizer</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">lr_scheduler</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">LRScheduler</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">callbacks</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="n">Callback</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">loss_type</span><span class="p">:</span> <span class="n">Literal</span><span class="p">[</span><span class="s2">&quot;contrastive&quot;</span><span class="p">,</span> <span class="s2">&quot;denoising&quot;</span><span class="p">,</span> <span class="s2">&quot;both&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="s2">&quot;both&quot;</span><span class="p">,</span>
+        <span class="n">projection_head1_dims</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">int</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">projection_head2_dims</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">int</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">projection_heads_activation</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;relu&quot;</span><span class="p">,</span>
+        <span class="n">cat_mlp_type</span><span class="p">:</span> <span class="n">Literal</span><span class="p">[</span><span class="s2">&quot;single&quot;</span><span class="p">,</span> <span class="s2">&quot;multiple&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="s2">&quot;multiple&quot;</span><span class="p">,</span>
+        <span class="n">cont_mlp_type</span><span class="p">:</span> <span class="n">Literal</span><span class="p">[</span><span class="s2">&quot;single&quot;</span><span class="p">,</span> <span class="s2">&quot;multiple&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="s2">&quot;multiple&quot;</span><span class="p">,</span>
+        <span class="n">denoise_mlps_activation</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;relu&quot;</span><span class="p">,</span>
+        <span class="n">verbose</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1</span><span class="p">,</span>
+        <span class="n">seed</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1</span><span class="p">,</span>
         <span class="o">**</span><span class="n">kwargs</span><span class="p">,</span>
-    <span class="p">)</span>
+    <span class="p">):</span>
+        <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span>
+            <span class="n">model</span><span class="o">=</span><span class="n">model</span><span class="p">,</span>
+            <span class="n">preprocessor</span><span class="o">=</span><span class="n">preprocessor</span><span class="p">,</span>
+            <span class="n">loss_type</span><span class="o">=</span><span class="n">loss_type</span><span class="p">,</span>
+            <span class="n">optimizer</span><span class="o">=</span><span class="n">optimizer</span><span class="p">,</span>
+            <span class="n">lr_scheduler</span><span class="o">=</span><span class="n">lr_scheduler</span><span class="p">,</span>
+            <span class="n">callbacks</span><span class="o">=</span><span class="n">callbacks</span><span class="p">,</span>
+            <span class="n">projection_head1_dims</span><span class="o">=</span><span class="n">projection_head1_dims</span><span class="p">,</span>
+            <span class="n">projection_head2_dims</span><span class="o">=</span><span class="n">projection_head2_dims</span><span class="p">,</span>
+            <span class="n">projection_heads_activation</span><span class="o">=</span><span class="n">projection_heads_activation</span><span class="p">,</span>
+            <span class="n">cat_mlp_type</span><span class="o">=</span><span class="n">cat_mlp_type</span><span class="p">,</span>
+            <span class="n">cont_mlp_type</span><span class="o">=</span><span class="n">cont_mlp_type</span><span class="p">,</span>
+            <span class="n">denoise_mlps_activation</span><span class="o">=</span><span class="n">denoise_mlps_activation</span><span class="p">,</span>
+            <span class="n">verbose</span><span class="o">=</span><span class="n">verbose</span><span class="p">,</span>
+            <span class="n">seed</span><span class="o">=</span><span class="n">seed</span><span class="p">,</span>
+            <span class="o">**</span><span class="n">kwargs</span><span class="p">,</span>
+        <span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">pretrain</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span>
+        <span class="n">X_tab</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">,</span>
+        <span class="n">X_tab_val</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">val_split</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">validation_freq</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1</span><span class="p">,</span>
+        <span class="n">n_epochs</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1</span><span class="p">,</span>
+        <span class="n">batch_size</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">32</span><span class="p">,</span>
+    <span class="p">):</span>
+<span class="w">        </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Pretrain method. Can also be called using `.fit(&lt;same_args&gt;)`</span>
+
+<span class="sd">        Parameters</span>
+<span class="sd">        ----------</span>
+<span class="sd">        X_tab: np.ndarray,</span>
+<span class="sd">            tabular dataset</span>
+<span class="sd">        X_tab_val: np.ndarray, Optional, default = None</span>
+<span class="sd">            validation data. Note that, although it is possible to use</span>
+<span class="sd">            contrastive-denoising training with a validation set, such set</span>
+<span class="sd">            must include feature values that are _all_ seen in the training</span>
+<span class="sd">            set in the case of the categorical columns. This is because the</span>
+<span class="sd">            values of the columns themselves will be used as targets when</span>
+<span class="sd">            computing the loss. Therefore, if a new category is present in</span>
+<span class="sd">            the validation set that was not seen in training this will</span>
+<span class="sd">            effectively be like trying to predict a new, never seen category</span>
+<span class="sd">            (and Pytorch will throw an error)</span>
+<span class="sd">        val_split: float, Optional. default=None</span>
+<span class="sd">            An alterative to passing the validation set is to use a train/val</span>
+<span class="sd">            split fraction via `val_split`</span>
+<span class="sd">        validation_freq: int, default=1</span>
+<span class="sd">            epochs validation frequency</span>
+<span class="sd">        n_epochs: int, default=1</span>
+<span class="sd">            number of epochs</span>
+<span class="sd">        batch_size: int, default=32</span>
+<span class="sd">            batch size</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">batch_size</span> <span class="o">=</span> <span class="n">batch_size</span>
+
+        <span class="n">train_set</span><span class="p">,</span> <span class="n">eval_set</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_train_eval_split</span><span class="p">(</span><span class="n">X_tab</span><span class="p">,</span> <span class="n">X_tab_val</span><span class="p">,</span> <span class="n">val_split</span><span class="p">)</span>
+        <span class="n">train_loader</span> <span class="o">=</span> <span class="n">DataLoader</span><span class="p">(</span>
+            <span class="n">dataset</span><span class="o">=</span><span class="n">train_set</span><span class="p">,</span> <span class="n">batch_size</span><span class="o">=</span><span class="n">batch_size</span><span class="p">,</span> <span class="n">num_workers</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">num_workers</span>
+        <span class="p">)</span>
+        <span class="n">train_steps</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">train_loader</span><span class="p">)</span>
+        <span class="k">if</span> <span class="n">eval_set</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="n">eval_loader</span> <span class="o">=</span> <span class="n">DataLoader</span><span class="p">(</span>
+                <span class="n">dataset</span><span class="o">=</span><span class="n">eval_set</span><span class="p">,</span>
+                <span class="n">batch_size</span><span class="o">=</span><span class="n">batch_size</span><span class="p">,</span>
+                <span class="n">num_workers</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">num_workers</span><span class="p">,</span>
+                <span class="n">shuffle</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
+            <span class="p">)</span>
+            <span class="n">eval_steps</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">eval_loader</span><span class="p">)</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">callback_container</span><span class="o">.</span><span class="n">on_train_begin</span><span class="p">(</span>
+            <span class="p">{</span>
+                <span class="s2">&quot;batch_size&quot;</span><span class="p">:</span> <span class="n">batch_size</span><span class="p">,</span>
+                <span class="s2">&quot;train_steps&quot;</span><span class="p">:</span> <span class="n">train_steps</span><span class="p">,</span>
+                <span class="s2">&quot;n_epochs&quot;</span><span class="p">:</span> <span class="n">n_epochs</span><span class="p">,</span>
+            <span class="p">}</span>
+        <span class="p">)</span>
+        <span class="k">for</span> <span class="n">epoch</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">n_epochs</span><span class="p">):</span>
+            <span class="n">epoch_logs</span><span class="p">:</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="p">{}</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">callback_container</span><span class="o">.</span><span class="n">on_epoch_begin</span><span class="p">(</span><span class="n">epoch</span><span class="p">,</span> <span class="n">logs</span><span class="o">=</span><span class="n">epoch_logs</span><span class="p">)</span>
+
+            <span class="bp">self</span><span class="o">.</span><span class="n">train_running_loss</span> <span class="o">=</span> <span class="mf">0.0</span>
+            <span class="k">with</span> <span class="n">trange</span><span class="p">(</span><span class="n">train_steps</span><span class="p">,</span> <span class="n">disable</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">verbose</span> <span class="o">!=</span> <span class="mi">1</span><span class="p">)</span> <span class="k">as</span> <span class="n">t</span><span class="p">:</span>
+                <span class="k">for</span> <span class="n">batch_idx</span><span class="p">,</span> <span class="n">X</span> <span class="ow">in</span> <span class="nb">zip</span><span class="p">(</span><span class="n">t</span><span class="p">,</span> <span class="n">train_loader</span><span class="p">):</span>
+                    <span class="n">t</span><span class="o">.</span><span class="n">set_description</span><span class="p">(</span><span class="s2">&quot;epoch </span><span class="si">%i</span><span class="s2">&quot;</span> <span class="o">%</span> <span class="p">(</span><span class="n">epoch</span> <span class="o">+</span> <span class="mi">1</span><span class="p">))</span>
+                    <span class="n">train_loss</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_train_step</span><span class="p">(</span><span class="n">X</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="n">batch_idx</span><span class="p">)</span>
+                    <span class="bp">self</span><span class="o">.</span><span class="n">callback_container</span><span class="o">.</span><span class="n">on_batch_end</span><span class="p">(</span><span class="n">batch</span><span class="o">=</span><span class="n">batch_idx</span><span class="p">)</span>
+                    <span class="n">print_loss_and_metric</span><span class="p">(</span><span class="n">t</span><span class="p">,</span> <span class="n">train_loss</span><span class="p">)</span>
+
+            <span class="n">epoch_logs</span> <span class="o">=</span> <span class="n">save_epoch_logs</span><span class="p">(</span><span class="n">epoch_logs</span><span class="p">,</span> <span class="n">train_loss</span><span class="p">,</span> <span class="kc">None</span><span class="p">,</span> <span class="s2">&quot;train&quot;</span><span class="p">)</span>
+
+            <span class="n">on_epoch_end_metric</span> <span class="o">=</span> <span class="kc">None</span>
+            <span class="k">if</span> <span class="n">eval_set</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">and</span> <span class="n">epoch</span> <span class="o">%</span> <span class="n">validation_freq</span> <span class="o">==</span> <span class="p">(</span>
+                <span class="n">validation_freq</span> <span class="o">-</span> <span class="mi">1</span>
+            <span class="p">):</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">callback_container</span><span class="o">.</span><span class="n">on_eval_begin</span><span class="p">()</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">valid_running_loss</span> <span class="o">=</span> <span class="mf">0.0</span>
+                <span class="k">with</span> <span class="n">trange</span><span class="p">(</span><span class="n">eval_steps</span><span class="p">,</span> <span class="n">disable</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">verbose</span> <span class="o">!=</span> <span class="mi">1</span><span class="p">)</span> <span class="k">as</span> <span class="n">v</span><span class="p">:</span>
+                    <span class="k">for</span> <span class="n">batch_idx</span><span class="p">,</span> <span class="n">X</span> <span class="ow">in</span> <span class="nb">zip</span><span class="p">(</span><span class="n">v</span><span class="p">,</span> <span class="n">eval_loader</span><span class="p">):</span>
+                        <span class="n">v</span><span class="o">.</span><span class="n">set_description</span><span class="p">(</span><span class="s2">&quot;valid&quot;</span><span class="p">)</span>
+                        <span class="n">val_loss</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_eval_step</span><span class="p">(</span><span class="n">X</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="n">batch_idx</span><span class="p">)</span>
+                        <span class="n">print_loss_and_metric</span><span class="p">(</span><span class="n">v</span><span class="p">,</span> <span class="n">val_loss</span><span class="p">)</span>
+                <span class="n">epoch_logs</span> <span class="o">=</span> <span class="n">save_epoch_logs</span><span class="p">(</span><span class="n">epoch_logs</span><span class="p">,</span> <span class="n">val_loss</span><span class="p">,</span> <span class="kc">None</span><span class="p">,</span> <span class="s2">&quot;val&quot;</span><span class="p">)</span>
+                <span class="n">on_epoch_end_metric</span> <span class="o">=</span> <span class="n">val_loss</span>
+            <span class="k">else</span><span class="p">:</span>
+                <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">reducelronplateau</span><span class="p">:</span>
+                    <span class="k">raise</span> <span class="ne">NotImplementedError</span><span class="p">(</span>
+                        <span class="s2">&quot;ReduceLROnPlateau scheduler can be used only with validation data.&quot;</span>
+                    <span class="p">)</span>
+
+            <span class="bp">self</span><span class="o">.</span><span class="n">callback_container</span><span class="o">.</span><span class="n">on_epoch_end</span><span class="p">(</span><span class="n">epoch</span><span class="p">,</span> <span class="n">epoch_logs</span><span class="p">,</span> <span class="n">on_epoch_end_metric</span><span class="p">)</span>
+
+            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">early_stop</span><span class="p">:</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">callback_container</span><span class="o">.</span><span class="n">on_train_end</span><span class="p">(</span><span class="n">epoch_logs</span><span class="p">)</span>
+                <span class="k">break</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">callback_container</span><span class="o">.</span><span class="n">on_train_end</span><span class="p">(</span><span class="n">epoch_logs</span><span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">_restore_best_weights</span><span class="p">()</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">cd_model</span><span class="o">.</span><span class="n">train</span><span class="p">()</span>
+
+    <span class="k">def</span> <span class="nf">fit</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span>
+        <span class="n">X_tab</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">,</span>
+        <span class="n">X_tab_val</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">val_split</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">validation_freq</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1</span><span class="p">,</span>
+        <span class="n">n_epochs</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1</span><span class="p">,</span>
+        <span class="n">batch_size</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">32</span><span class="p">,</span>
+    <span class="p">):</span>
+        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">pretrain</span><span class="p">(</span>
+            <span class="n">X_tab</span><span class="p">,</span> <span class="n">X_tab_val</span><span class="p">,</span> <span class="n">val_split</span><span class="p">,</span> <span class="n">validation_freq</span><span class="p">,</span> <span class="n">n_epochs</span><span class="p">,</span> <span class="n">batch_size</span>
+        <span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">_train_step</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">X_tab</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">,</span> <span class="n">batch_idx</span><span class="p">:</span> <span class="nb">int</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">float</span><span class="p">:</span>
+        <span class="n">X</span> <span class="o">=</span> <span class="n">X_tab</span><span class="o">.</span><span class="n">to</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">device</span><span class="p">)</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">optimizer</span><span class="o">.</span><span class="n">zero_grad</span><span class="p">()</span>
+        <span class="n">g_projs</span><span class="p">,</span> <span class="n">cat_x_and_x_</span><span class="p">,</span> <span class="n">cont_x_and_x_</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">cd_model</span><span class="p">(</span><span class="n">X</span><span class="p">)</span>
+        <span class="n">loss</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_compute_loss</span><span class="p">(</span><span class="n">g_projs</span><span class="p">,</span> <span class="n">cat_x_and_x_</span><span class="p">,</span> <span class="n">cont_x_and_x_</span><span class="p">)</span>
+        <span class="n">loss</span><span class="o">.</span><span class="n">backward</span><span class="p">()</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">optimizer</span><span class="o">.</span><span class="n">step</span><span class="p">()</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">train_running_loss</span> <span class="o">+=</span> <span class="n">loss</span><span class="o">.</span><span class="n">item</span><span class="p">()</span>
+        <span class="n">avg_loss</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">train_running_loss</span> <span class="o">/</span> <span class="p">(</span><span class="n">batch_idx</span> <span class="o">+</span> <span class="mi">1</span><span class="p">)</span>
+
+        <span class="k">return</span> <span class="n">avg_loss</span>
+
+    <span class="k">def</span> <span class="nf">_eval_step</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">X_tab</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">,</span> <span class="n">batch_idx</span><span class="p">:</span> <span class="nb">int</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">float</span><span class="p">:</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">cd_model</span><span class="o">.</span><span class="n">eval</span><span class="p">()</span>
+
+        <span class="k">with</span> <span class="n">torch</span><span class="o">.</span><span class="n">no_grad</span><span class="p">():</span>
+            <span class="n">X</span> <span class="o">=</span> <span class="n">X_tab</span><span class="o">.</span><span class="n">to</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">device</span><span class="p">)</span>
+
+            <span class="n">g_projs</span><span class="p">,</span> <span class="n">cat_x_and_x_</span><span class="p">,</span> <span class="n">cont_x_and_x_</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">cd_model</span><span class="p">(</span><span class="n">X</span><span class="p">)</span>
+            <span class="n">loss</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_compute_loss</span><span class="p">(</span><span class="n">g_projs</span><span class="p">,</span> <span class="n">cat_x_and_x_</span><span class="p">,</span> <span class="n">cont_x_and_x_</span><span class="p">)</span>
+
+            <span class="bp">self</span><span class="o">.</span><span class="n">valid_running_loss</span> <span class="o">+=</span> <span class="n">loss</span><span class="o">.</span><span class="n">item</span><span class="p">()</span>
+            <span class="n">avg_loss</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">valid_running_loss</span> <span class="o">/</span> <span class="p">(</span><span class="n">batch_idx</span> <span class="o">+</span> <span class="mi">1</span><span class="p">)</span>
+
+        <span class="k">return</span> <span class="n">avg_loss</span>
+
+    <span class="k">def</span> <span class="nf">_train_eval_split</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span>
+        <span class="n">X</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">,</span>
+        <span class="n">X_tab_val</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">val_split</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+    <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tuple</span><span class="p">[</span><span class="n">TensorDataset</span><span class="p">,</span> <span class="n">Optional</span><span class="p">[</span><span class="n">TensorDataset</span><span class="p">]]:</span>
+        <span class="k">if</span> <span class="n">X_tab_val</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="n">train_set</span> <span class="o">=</span> <span class="n">TensorDataset</span><span class="p">(</span><span class="n">torch</span><span class="o">.</span><span class="n">from_numpy</span><span class="p">(</span><span class="n">X</span><span class="p">))</span>
+            <span class="n">eval_set</span> <span class="o">=</span> <span class="n">TensorDataset</span><span class="p">(</span><span class="n">torch</span><span class="o">.</span><span class="n">from_numpy</span><span class="p">(</span><span class="n">X_tab_val</span><span class="p">))</span>
+        <span class="k">elif</span> <span class="n">val_split</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="n">X_tr</span><span class="p">,</span> <span class="n">X_tab_val</span> <span class="o">=</span> <span class="n">train_test_split</span><span class="p">(</span>  <span class="c1"># type: ignore</span>
+                <span class="n">X</span><span class="p">,</span> <span class="n">test_size</span><span class="o">=</span><span class="n">val_split</span><span class="p">,</span> <span class="n">random_state</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">seed</span>
+            <span class="p">)</span>
+            <span class="n">train_set</span> <span class="o">=</span> <span class="n">TensorDataset</span><span class="p">(</span><span class="n">torch</span><span class="o">.</span><span class="n">from_numpy</span><span class="p">(</span><span class="n">X_tr</span><span class="p">))</span>
+            <span class="n">eval_set</span> <span class="o">=</span> <span class="n">TensorDataset</span><span class="p">(</span><span class="n">torch</span><span class="o">.</span><span class="n">from_numpy</span><span class="p">(</span><span class="n">X_tab_val</span><span class="p">))</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="n">train_set</span> <span class="o">=</span> <span class="n">TensorDataset</span><span class="p">(</span><span class="n">torch</span><span class="o">.</span><span class="n">from_numpy</span><span class="p">(</span><span class="n">X</span><span class="p">))</span>
+            <span class="n">eval_set</span> <span class="o">=</span> <span class="kc">None</span>
+
+        <span class="k">return</span> <span class="n">train_set</span><span class="p">,</span> <span class="n">eval_set</span>
 </code></pre></div></td></tr></table></div>
-                  </details>
+              </details>
 
 
 
@@ -2486,23 +3544,38 @@ <h3 id="pytorch_widedeep.self_supervised_training.ContrastiveDenoisingTrainer.pr
 
 
 <p><span class="doc-section-title">Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>X_tab</code></b>
-              (<code><span title="numpy.ndarray">ndarray</span></code>)
-          –
-          <div class="doc-md-description">
-            <p>tabular dataset</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>X_tab_val</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="numpy.ndarray">ndarray</span>]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>validation data. Note that, although it is possible to use
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+          <th>Default</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>X_tab</code></td>
+            <td>
+                  <code><span title="numpy.ndarray">ndarray</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>tabular dataset</p>
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>X_tab_val</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="numpy.ndarray">ndarray</span>]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>validation data. Note that, although it is possible to use
 contrastive-denoising training with a validation set, such set
 must include feature values that are <em>all</em> seen in the training
 set in the case of the categorical columns. This is because the
@@ -2511,50 +3584,71 @@ <h3 id="pytorch_widedeep.self_supervised_training.ContrastiveDenoisingTrainer.pr
 the validation set that was not seen in training this will
 effectively be like trying to predict a new, never seen category
 (and Pytorch will throw an error)</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>val_split</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
+              </div>
+            </td>
+            <td>
                   <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>An alterative to passing the validation set is to use a train/val
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>val_split</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>An alterative to passing the validation set is to use a train/val
 split fraction via <code>val_split</code></p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>validation_freq</code></b>
-              (<code>int</code>, default:
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>validation_freq</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>epochs validation frequency</p>
+              </div>
+            </td>
+            <td>
                   <code>1</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>epochs validation frequency</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>n_epochs</code></b>
-              (<code>int</code>, default:
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>n_epochs</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>number of epochs</p>
+              </div>
+            </td>
+            <td>
                   <code>1</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>number of epochs</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>batch_size</code></b>
-              (<code>int</code>, default:
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>batch_size</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>batch size</p>
+              </div>
+            </td>
+            <td>
                   <code>32</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>batch size</p>
-          </div>
-        </li>
-    </ul>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
             <details class="quote">
               <summary>Source code in <code>pytorch_widedeep/self_supervised_training/contrastive_denoising_trainer.py</code></summary>
diff --git a/mkdocs/site/pytorch-widedeep/tab2vec.html b/mkdocs/site/pytorch-widedeep/tab2vec.html
index e36e452a..37e611a0 100644
--- a/mkdocs/site/pytorch-widedeep/tab2vec.html
+++ b/mkdocs/site/pytorch-widedeep/tab2vec.html
@@ -1387,7 +1387,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/16_Self_Supervised_Pretraning_pt1.ipynb" class="md-nav__link">
+      <a href="../examples/15_Self_Supervised_Pretraning_pt1.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1408,7 +1408,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/16_Self_Supervised_Pretraning_pt2.ipynb" class="md-nav__link">
+      <a href="../examples/15_Self_Supervised_Pretraning_pt2.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1429,7 +1429,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/17_Usign_a_custom_hugging_face_model.ipynb" class="md-nav__link">
+      <a href="../examples/16_Usign_a_custom_hugging_face_model.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1450,7 +1450,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/18_feature_importance_via_attention_weights.ipynb" class="md-nav__link">
+      <a href="../examples/17_feature_importance_via_attention_weights.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1471,7 +1471,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/19_wide_and_deep_for_recsys_pt1.ipynb" class="md-nav__link">
+      <a href="../examples/18_wide_and_deep_for_recsys_pt1.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1492,7 +1492,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/19_wide_and_deep_for_recsys_pt2.ipynb" class="md-nav__link">
+      <a href="../examples/18_wide_and_deep_for_recsys_pt2.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1513,7 +1513,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/20_load_from_folder_functionality.ipynb" class="md-nav__link">
+      <a href="../examples/19_load_from_folder_functionality.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1534,7 +1534,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/21_Using_huggingface_within_widedeep.ipynb" class="md-nav__link">
+      <a href="../examples/20_Using_huggingface_within_widedeep.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1605,14 +1605,7 @@ <h2 id="pytorch_widedeep.tab2vec.Tab2Vec" class="doc doc-heading">
 
 
 <a href="#pytorch_widedeep.tab2vec.Tab2Vec" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="nf">Tab2Vec</span><span class="p">(</span>
-    <span class="n">tab_preprocessor</span><span class="p">,</span>
-    <span class="n">model</span><span class="p">,</span>
-    <span class="n">device</span><span class="p">,</span>
-    <span class="n">return_dataframe</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
-    <span class="n">verbose</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
-<span class="p">)</span>
-</code></pre></div>
+
 
     <div class="doc doc-contents first">
 
@@ -1629,48 +1622,86 @@ <h2 id="pytorch_widedeep.tab2vec.Tab2Vec" class="doc doc-heading">
 
 
 <p><span class="doc-section-title">Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>model</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Union">Union</span>[<a class="autorefs autorefs-internal" title="pytorch_widedeep.wdtypes.WideDeep" href="model_components.html#pytorch_widedeep.models.wide_deep.WideDeep">WideDeep</a>, <a class="autorefs autorefs-internal" title="pytorch_widedeep.bayesian_models.BayesianWide" href="bayesian_models.html#pytorch_widedeep.bayesian_models.tabular.bayesian_linear.bayesian_wide.BayesianWide">BayesianWide</a>, <a class="autorefs autorefs-internal" title="pytorch_widedeep.bayesian_models.BayesianTabMlp" href="bayesian_models.html#pytorch_widedeep.bayesian_models.tabular.bayesian_mlp.bayesian_tab_mlp.BayesianTabMlp">BayesianTabMlp</a>]</code>)
-          –
-          <div class="doc-md-description">
-            <p><code>WideDeep</code>, <code>BayesianWide</code> or <code>BayesianTabMlp</code> model. Must be trained.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>tab_preprocessor</code></b>
-              (<code><a class="autorefs autorefs-internal" title="pytorch_widedeep.preprocessing.TabPreprocessor" href="preprocessing.html#pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor">TabPreprocessor</a></code>)
-          –
-          <div class="doc-md-description">
-            <p><code>TabPreprocessor</code> object. Must be fitted.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>return_dataframe</code></b>
-              (<code>bool</code>, default:
-                  <code>False</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Boolean indicating of the returned object(s) will be array(s) or
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+          <th>Default</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>model</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Union">Union</span>[<a class="autorefs autorefs-internal" title="pytorch_widedeep.wdtypes.WideDeep" href="model_components.html#pytorch_widedeep.models.wide_deep.WideDeep">WideDeep</a>, <a class="autorefs autorefs-internal" title="pytorch_widedeep.bayesian_models.BayesianWide" href="bayesian_models.html#pytorch_widedeep.bayesian_models.tabular.bayesian_linear.bayesian_wide.BayesianWide">BayesianWide</a>, <a class="autorefs autorefs-internal" title="pytorch_widedeep.bayesian_models.BayesianTabMlp" href="bayesian_models.html#pytorch_widedeep.bayesian_models.tabular.bayesian_mlp.bayesian_tab_mlp.BayesianTabMlp">BayesianTabMlp</a>]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p><code>WideDeep</code>, <code>BayesianWide</code> or <code>BayesianTabMlp</code> model. Must be trained.</p>
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>tab_preprocessor</code></td>
+            <td>
+                  <code><a class="autorefs autorefs-internal" title="pytorch_widedeep.preprocessing.TabPreprocessor" href="preprocessing.html#pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor">TabPreprocessor</a></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p><code>TabPreprocessor</code> object. Must be fitted.</p>
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>return_dataframe</code></td>
+            <td>
+                  <code>bool</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Boolean indicating of the returned object(s) will be array(s) or
 pandas dataframe(s)</p>
-          </div>
-        </li>
-    </ul>
+              </div>
+            </td>
+            <td>
+                  <code>False</code>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
 
 <p><span class="doc-section-title">Attributes:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code><span title="pytorch_widedeep.tab2vec.Tab2Vec.vectorizer">vectorizer</span></code></b>
-              (<code><span title="nn.Module">Module</span></code>)
-          –
-          <div class="doc-md-description">
-            <p>Torch module with the categorical and continuous encoding process</p>
-          </div>
-        </li>
-    </ul>
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code><span title="pytorch_widedeep.tab2vec.Tab2Vec.vectorizer">vectorizer</span></code></td>
+            <td>
+                  <code><span title="nn.Module">Module</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Torch module with the categorical and continuous encoding process</p>
+              </div>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
 
 <p><span class="doc-section-title">Examples:</span></p>
@@ -1714,9 +1745,79 @@ <h2 id="pytorch_widedeep.tab2vec.Tab2Vec" class="doc doc-heading">
 <span class="gp">&gt;&gt;&gt; </span><span class="n">X_vec</span> <span class="o">=</span> <span class="n">t2v</span><span class="o">.</span><span class="n">transform</span><span class="p">(</span><span class="n">df_t2v</span><span class="p">)</span>
 </code></pre></div>
 
-                  <details class="quote">
-                    <summary>Source code in <code>pytorch_widedeep/tab2vec.py</code></summary>
-                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 84</span>
+              <details class="quote">
+                <summary>Source code in <code>pytorch_widedeep/tab2vec.py</code></summary>
+                <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 14</span>
+<span class="normal"> 15</span>
+<span class="normal"> 16</span>
+<span class="normal"> 17</span>
+<span class="normal"> 18</span>
+<span class="normal"> 19</span>
+<span class="normal"> 20</span>
+<span class="normal"> 21</span>
+<span class="normal"> 22</span>
+<span class="normal"> 23</span>
+<span class="normal"> 24</span>
+<span class="normal"> 25</span>
+<span class="normal"> 26</span>
+<span class="normal"> 27</span>
+<span class="normal"> 28</span>
+<span class="normal"> 29</span>
+<span class="normal"> 30</span>
+<span class="normal"> 31</span>
+<span class="normal"> 32</span>
+<span class="normal"> 33</span>
+<span class="normal"> 34</span>
+<span class="normal"> 35</span>
+<span class="normal"> 36</span>
+<span class="normal"> 37</span>
+<span class="normal"> 38</span>
+<span class="normal"> 39</span>
+<span class="normal"> 40</span>
+<span class="normal"> 41</span>
+<span class="normal"> 42</span>
+<span class="normal"> 43</span>
+<span class="normal"> 44</span>
+<span class="normal"> 45</span>
+<span class="normal"> 46</span>
+<span class="normal"> 47</span>
+<span class="normal"> 48</span>
+<span class="normal"> 49</span>
+<span class="normal"> 50</span>
+<span class="normal"> 51</span>
+<span class="normal"> 52</span>
+<span class="normal"> 53</span>
+<span class="normal"> 54</span>
+<span class="normal"> 55</span>
+<span class="normal"> 56</span>
+<span class="normal"> 57</span>
+<span class="normal"> 58</span>
+<span class="normal"> 59</span>
+<span class="normal"> 60</span>
+<span class="normal"> 61</span>
+<span class="normal"> 62</span>
+<span class="normal"> 63</span>
+<span class="normal"> 64</span>
+<span class="normal"> 65</span>
+<span class="normal"> 66</span>
+<span class="normal"> 67</span>
+<span class="normal"> 68</span>
+<span class="normal"> 69</span>
+<span class="normal"> 70</span>
+<span class="normal"> 71</span>
+<span class="normal"> 72</span>
+<span class="normal"> 73</span>
+<span class="normal"> 74</span>
+<span class="normal"> 75</span>
+<span class="normal"> 76</span>
+<span class="normal"> 77</span>
+<span class="normal"> 78</span>
+<span class="normal"> 79</span>
+<span class="normal"> 80</span>
+<span class="normal"> 81</span>
+<span class="normal"> 82</span>
+<span class="normal"> 83</span>
+<span class="normal"> 84</span>
 <span class="normal"> 85</span>
 <span class="normal"> 86</span>
 <span class="normal"> 87</span>
@@ -1739,32 +1840,572 @@ <h2 id="pytorch_widedeep.tab2vec.Tab2Vec" class="doc doc-heading">
 <span class="normal">104</span>
 <span class="normal">105</span>
 <span class="normal">106</span>
-<span class="normal">107</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
-    <span class="bp">self</span><span class="p">,</span>
-    <span class="n">tab_preprocessor</span><span class="p">:</span> <span class="n">TabPreprocessor</span><span class="p">,</span>
-    <span class="n">model</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="n">WideDeep</span><span class="p">,</span> <span class="n">BayesianWide</span><span class="p">,</span> <span class="n">BayesianTabMlp</span><span class="p">],</span>
-    <span class="n">device</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="n">torch</span><span class="o">.</span><span class="n">device</span><span class="p">,</span>
-    <span class="n">return_dataframe</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
-    <span class="n">verbose</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
-<span class="p">):</span>
-    <span class="nb">super</span><span class="p">(</span><span class="n">Tab2Vec</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
-
-    <span class="bp">self</span><span class="o">.</span><span class="n">_check_inputs</span><span class="p">(</span><span class="n">tab_preprocessor</span><span class="p">,</span> <span class="n">model</span><span class="p">,</span> <span class="n">verbose</span><span class="p">)</span>
-
-    <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">device</span><span class="p">,</span> <span class="nb">str</span><span class="p">):</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">device</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">device</span><span class="p">(</span><span class="n">device</span><span class="p">)</span>
-    <span class="k">else</span><span class="p">:</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">device</span> <span class="o">=</span> <span class="n">device</span>
+<span class="normal">107</span>
+<span class="normal">108</span>
+<span class="normal">109</span>
+<span class="normal">110</span>
+<span class="normal">111</span>
+<span class="normal">112</span>
+<span class="normal">113</span>
+<span class="normal">114</span>
+<span class="normal">115</span>
+<span class="normal">116</span>
+<span class="normal">117</span>
+<span class="normal">118</span>
+<span class="normal">119</span>
+<span class="normal">120</span>
+<span class="normal">121</span>
+<span class="normal">122</span>
+<span class="normal">123</span>
+<span class="normal">124</span>
+<span class="normal">125</span>
+<span class="normal">126</span>
+<span class="normal">127</span>
+<span class="normal">128</span>
+<span class="normal">129</span>
+<span class="normal">130</span>
+<span class="normal">131</span>
+<span class="normal">132</span>
+<span class="normal">133</span>
+<span class="normal">134</span>
+<span class="normal">135</span>
+<span class="normal">136</span>
+<span class="normal">137</span>
+<span class="normal">138</span>
+<span class="normal">139</span>
+<span class="normal">140</span>
+<span class="normal">141</span>
+<span class="normal">142</span>
+<span class="normal">143</span>
+<span class="normal">144</span>
+<span class="normal">145</span>
+<span class="normal">146</span>
+<span class="normal">147</span>
+<span class="normal">148</span>
+<span class="normal">149</span>
+<span class="normal">150</span>
+<span class="normal">151</span>
+<span class="normal">152</span>
+<span class="normal">153</span>
+<span class="normal">154</span>
+<span class="normal">155</span>
+<span class="normal">156</span>
+<span class="normal">157</span>
+<span class="normal">158</span>
+<span class="normal">159</span>
+<span class="normal">160</span>
+<span class="normal">161</span>
+<span class="normal">162</span>
+<span class="normal">163</span>
+<span class="normal">164</span>
+<span class="normal">165</span>
+<span class="normal">166</span>
+<span class="normal">167</span>
+<span class="normal">168</span>
+<span class="normal">169</span>
+<span class="normal">170</span>
+<span class="normal">171</span>
+<span class="normal">172</span>
+<span class="normal">173</span>
+<span class="normal">174</span>
+<span class="normal">175</span>
+<span class="normal">176</span>
+<span class="normal">177</span>
+<span class="normal">178</span>
+<span class="normal">179</span>
+<span class="normal">180</span>
+<span class="normal">181</span>
+<span class="normal">182</span>
+<span class="normal">183</span>
+<span class="normal">184</span>
+<span class="normal">185</span>
+<span class="normal">186</span>
+<span class="normal">187</span>
+<span class="normal">188</span>
+<span class="normal">189</span>
+<span class="normal">190</span>
+<span class="normal">191</span>
+<span class="normal">192</span>
+<span class="normal">193</span>
+<span class="normal">194</span>
+<span class="normal">195</span>
+<span class="normal">196</span>
+<span class="normal">197</span>
+<span class="normal">198</span>
+<span class="normal">199</span>
+<span class="normal">200</span>
+<span class="normal">201</span>
+<span class="normal">202</span>
+<span class="normal">203</span>
+<span class="normal">204</span>
+<span class="normal">205</span>
+<span class="normal">206</span>
+<span class="normal">207</span>
+<span class="normal">208</span>
+<span class="normal">209</span>
+<span class="normal">210</span>
+<span class="normal">211</span>
+<span class="normal">212</span>
+<span class="normal">213</span>
+<span class="normal">214</span>
+<span class="normal">215</span>
+<span class="normal">216</span>
+<span class="normal">217</span>
+<span class="normal">218</span>
+<span class="normal">219</span>
+<span class="normal">220</span>
+<span class="normal">221</span>
+<span class="normal">222</span>
+<span class="normal">223</span>
+<span class="normal">224</span>
+<span class="normal">225</span>
+<span class="normal">226</span>
+<span class="normal">227</span>
+<span class="normal">228</span>
+<span class="normal">229</span>
+<span class="normal">230</span>
+<span class="normal">231</span>
+<span class="normal">232</span>
+<span class="normal">233</span>
+<span class="normal">234</span>
+<span class="normal">235</span>
+<span class="normal">236</span>
+<span class="normal">237</span>
+<span class="normal">238</span>
+<span class="normal">239</span>
+<span class="normal">240</span>
+<span class="normal">241</span>
+<span class="normal">242</span>
+<span class="normal">243</span>
+<span class="normal">244</span>
+<span class="normal">245</span>
+<span class="normal">246</span>
+<span class="normal">247</span>
+<span class="normal">248</span>
+<span class="normal">249</span>
+<span class="normal">250</span>
+<span class="normal">251</span>
+<span class="normal">252</span>
+<span class="normal">253</span>
+<span class="normal">254</span>
+<span class="normal">255</span>
+<span class="normal">256</span>
+<span class="normal">257</span>
+<span class="normal">258</span>
+<span class="normal">259</span>
+<span class="normal">260</span>
+<span class="normal">261</span>
+<span class="normal">262</span>
+<span class="normal">263</span>
+<span class="normal">264</span>
+<span class="normal">265</span>
+<span class="normal">266</span>
+<span class="normal">267</span>
+<span class="normal">268</span>
+<span class="normal">269</span>
+<span class="normal">270</span>
+<span class="normal">271</span>
+<span class="normal">272</span>
+<span class="normal">273</span>
+<span class="normal">274</span>
+<span class="normal">275</span>
+<span class="normal">276</span>
+<span class="normal">277</span>
+<span class="normal">278</span>
+<span class="normal">279</span>
+<span class="normal">280</span>
+<span class="normal">281</span>
+<span class="normal">282</span>
+<span class="normal">283</span>
+<span class="normal">284</span>
+<span class="normal">285</span>
+<span class="normal">286</span>
+<span class="normal">287</span>
+<span class="normal">288</span>
+<span class="normal">289</span>
+<span class="normal">290</span>
+<span class="normal">291</span>
+<span class="normal">292</span>
+<span class="normal">293</span>
+<span class="normal">294</span>
+<span class="normal">295</span>
+<span class="normal">296</span>
+<span class="normal">297</span>
+<span class="normal">298</span>
+<span class="normal">299</span>
+<span class="normal">300</span>
+<span class="normal">301</span>
+<span class="normal">302</span>
+<span class="normal">303</span>
+<span class="normal">304</span>
+<span class="normal">305</span>
+<span class="normal">306</span>
+<span class="normal">307</span>
+<span class="normal">308</span>
+<span class="normal">309</span>
+<span class="normal">310</span>
+<span class="normal">311</span>
+<span class="normal">312</span>
+<span class="normal">313</span>
+<span class="normal">314</span>
+<span class="normal">315</span>
+<span class="normal">316</span>
+<span class="normal">317</span>
+<span class="normal">318</span>
+<span class="normal">319</span>
+<span class="normal">320</span>
+<span class="normal">321</span>
+<span class="normal">322</span>
+<span class="normal">323</span>
+<span class="normal">324</span>
+<span class="normal">325</span>
+<span class="normal">326</span>
+<span class="normal">327</span>
+<span class="normal">328</span>
+<span class="normal">329</span>
+<span class="normal">330</span>
+<span class="normal">331</span>
+<span class="normal">332</span>
+<span class="normal">333</span>
+<span class="normal">334</span>
+<span class="normal">335</span>
+<span class="normal">336</span>
+<span class="normal">337</span>
+<span class="normal">338</span>
+<span class="normal">339</span>
+<span class="normal">340</span>
+<span class="normal">341</span>
+<span class="normal">342</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">class</span> <span class="nc">Tab2Vec</span><span class="p">:</span>
+<span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Class to transform an input dataframe into vectorized form.</span>
+
+<span class="sd">    This class will take an input dataframe in the form of the dataframe used</span>
+<span class="sd">    for training, and it will turn it into a vectorised form based on the</span>
+<span class="sd">    processing applied by the model to the categorical and continuous</span>
+<span class="sd">    columns.</span>
+
+<span class="sd">    :information_source: **NOTE**: Currently this class is only implemented</span>
+<span class="sd">     for the deeptabular component. Therefore, if the input dataframe has a</span>
+<span class="sd">     text column or a column with the path to images, these will be ignored.</span>
+<span class="sd">     We will be adding these functionalities in future versions</span>
+
+<span class="sd">    Parameters</span>
+<span class="sd">    ----------</span>
+<span class="sd">    model: `WideDeep`, `BayesianWide` or `BayesianTabMlp`</span>
+<span class="sd">        `WideDeep`, `BayesianWide` or `BayesianTabMlp` model. Must be trained.</span>
+<span class="sd">    tab_preprocessor: `TabPreprocessor`</span>
+<span class="sd">        `TabPreprocessor` object. Must be fitted.</span>
+<span class="sd">    return_dataframe: bool</span>
+<span class="sd">        Boolean indicating of the returned object(s) will be array(s) or</span>
+<span class="sd">        pandas dataframe(s)</span>
+
+<span class="sd">    Attributes</span>
+<span class="sd">    ----------</span>
+<span class="sd">    vectorizer: nn.Module</span>
+<span class="sd">        Torch module with the categorical and continuous encoding process</span>
+
+<span class="sd">    Examples</span>
+<span class="sd">    --------</span>
+<span class="sd">    &gt;&gt;&gt; import string</span>
+<span class="sd">    &gt;&gt;&gt; from random import choices</span>
+<span class="sd">    &gt;&gt;&gt; import numpy as np</span>
+<span class="sd">    &gt;&gt;&gt; import pandas as pd</span>
+<span class="sd">    &gt;&gt;&gt; from pytorch_widedeep import Tab2Vec</span>
+<span class="sd">    &gt;&gt;&gt; from pytorch_widedeep.models import TabMlp, WideDeep</span>
+<span class="sd">    &gt;&gt;&gt; from pytorch_widedeep.preprocessing import TabPreprocessor</span>
+<span class="sd">    &gt;&gt;&gt;</span>
+<span class="sd">    &gt;&gt;&gt; colnames = list(string.ascii_lowercase)[:4]</span>
+<span class="sd">    &gt;&gt;&gt; cat_col1_vals = [&quot;a&quot;, &quot;b&quot;, &quot;c&quot;]</span>
+<span class="sd">    &gt;&gt;&gt; cat_col2_vals = [&quot;d&quot;, &quot;e&quot;, &quot;f&quot;]</span>
+<span class="sd">    &gt;&gt;&gt;</span>
+<span class="sd">    &gt;&gt;&gt; # Create the toy input dataframe and a toy dataframe to be vectorised</span>
+<span class="sd">    &gt;&gt;&gt; cat_inp = [np.array(choices(c, k=5)) for c in [cat_col1_vals, cat_col2_vals]]</span>
+<span class="sd">    &gt;&gt;&gt; cont_inp = [np.round(np.random.rand(5), 2) for _ in range(2)]</span>
+<span class="sd">    &gt;&gt;&gt; df_inp = pd.DataFrame(np.vstack(cat_inp + cont_inp).transpose(), columns=colnames)</span>
+<span class="sd">    &gt;&gt;&gt; cat_t2v = [np.array(choices(c, k=5)) for c in [cat_col1_vals, cat_col2_vals]]</span>
+<span class="sd">    &gt;&gt;&gt; cont_t2v = [np.round(np.random.rand(5), 2) for _ in range(2)]</span>
+<span class="sd">    &gt;&gt;&gt; df_t2v = pd.DataFrame(np.vstack(cat_t2v + cont_t2v).transpose(), columns=colnames)</span>
+<span class="sd">    &gt;&gt;&gt;</span>
+<span class="sd">    &gt;&gt;&gt; # fit the TabPreprocessor</span>
+<span class="sd">    &gt;&gt;&gt; embed_cols = [(&quot;a&quot;, 2), (&quot;b&quot;, 4)]</span>
+<span class="sd">    &gt;&gt;&gt; cont_cols = [&quot;c&quot;, &quot;d&quot;]</span>
+<span class="sd">    &gt;&gt;&gt; tab_preprocessor = TabPreprocessor(cat_embed_cols=embed_cols, continuous_cols=cont_cols)</span>
+<span class="sd">    &gt;&gt;&gt; X_tab = tab_preprocessor.fit_transform(df_inp)</span>
+<span class="sd">    &gt;&gt;&gt;</span>
+<span class="sd">    &gt;&gt;&gt; # define the model (and let&#39;s assume we train it)</span>
+<span class="sd">    &gt;&gt;&gt; tabmlp = TabMlp(</span>
+<span class="sd">    ... column_idx=tab_preprocessor.column_idx,</span>
+<span class="sd">    ... cat_embed_input=tab_preprocessor.cat_embed_input,</span>
+<span class="sd">    ... continuous_cols=tab_preprocessor.continuous_cols,</span>
+<span class="sd">    ... mlp_hidden_dims=[8, 4])</span>
+<span class="sd">    &gt;&gt;&gt; model = WideDeep(deeptabular=tabmlp)</span>
+<span class="sd">    &gt;&gt;&gt; # ...train the model...</span>
+<span class="sd">    &gt;&gt;&gt;</span>
+<span class="sd">    &gt;&gt;&gt; # vectorise the dataframe</span>
+<span class="sd">    &gt;&gt;&gt; t2v = Tab2Vec(tab_preprocessor, model, device=&quot;cpu&quot;)</span>
+<span class="sd">    &gt;&gt;&gt; X_vec = t2v.transform(df_t2v)</span>
+<span class="sd">    &quot;&quot;&quot;</span>
 
-    <span class="bp">self</span><span class="o">.</span><span class="n">tab_preprocessor</span> <span class="o">=</span> <span class="n">tab_preprocessor</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">return_dataframe</span> <span class="o">=</span> <span class="n">return_dataframe</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">verbose</span> <span class="o">=</span> <span class="n">verbose</span>
+    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span>
+        <span class="n">tab_preprocessor</span><span class="p">:</span> <span class="n">TabPreprocessor</span><span class="p">,</span>
+        <span class="n">model</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="n">WideDeep</span><span class="p">,</span> <span class="n">BayesianWide</span><span class="p">,</span> <span class="n">BayesianTabMlp</span><span class="p">],</span>
+        <span class="n">device</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="n">torch</span><span class="o">.</span><span class="n">device</span><span class="p">,</span>
+        <span class="n">return_dataframe</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
+        <span class="n">verbose</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
+    <span class="p">):</span>
+        <span class="nb">super</span><span class="p">(</span><span class="n">Tab2Vec</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
 
-    <span class="bp">self</span><span class="o">.</span><span class="n">vectorizer</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_set_vectorizer</span><span class="p">(</span><span class="n">model</span><span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">_check_inputs</span><span class="p">(</span><span class="n">tab_preprocessor</span><span class="p">,</span> <span class="n">model</span><span class="p">,</span> <span class="n">verbose</span><span class="p">)</span>
 
-    <span class="bp">self</span><span class="o">.</span><span class="n">_set_dim_attributes</span><span class="p">(</span><span class="n">tab_preprocessor</span><span class="p">,</span> <span class="n">model</span><span class="p">)</span>
+        <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">device</span><span class="p">,</span> <span class="nb">str</span><span class="p">):</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">device</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">device</span><span class="p">(</span><span class="n">device</span><span class="p">)</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">device</span> <span class="o">=</span> <span class="n">device</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">tab_preprocessor</span> <span class="o">=</span> <span class="n">tab_preprocessor</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">return_dataframe</span> <span class="o">=</span> <span class="n">return_dataframe</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">verbose</span> <span class="o">=</span> <span class="n">verbose</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">vectorizer</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_set_vectorizer</span><span class="p">(</span><span class="n">model</span><span class="p">)</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">_set_dim_attributes</span><span class="p">(</span><span class="n">tab_preprocessor</span><span class="p">,</span> <span class="n">model</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">fit</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">,</span> <span class="n">target_col</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="s2">&quot;Tab2Vec&quot;</span><span class="p">:</span>
+<span class="w">        </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;This is an empty method i.e. Returns the unchanged object itself. Is</span>
+<span class="sd">        only included for consistency in case `Tab2Vec` is used as part of a</span>
+<span class="sd">        Pipeline</span>
+
+<span class="sd">        Parameters</span>
+<span class="sd">        ----------</span>
+<span class="sd">        df: pd.DataFrame</span>
+<span class="sd">            DataFrame to be vectorised, i.e. the categorical and continuous</span>
+<span class="sd">            columns will be encoded based on the processing applied within</span>
+<span class="sd">            the model</span>
+<span class="sd">        target_col: str, Optional</span>
+<span class="sd">            Column name of the target_col variable. If `None` only the array of</span>
+<span class="sd">            predictors will be returned</span>
+
+<span class="sd">        Returns</span>
+<span class="sd">        -------</span>
+<span class="sd">        Tab2Vec</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+
+        <span class="k">return</span> <span class="bp">self</span>
+
+    <span class="k">def</span> <span class="nf">transform</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span>
+        <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">,</span>
+        <span class="n">target_col</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+    <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Union</span><span class="p">[</span>
+        <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">,</span>
+        <span class="n">Tuple</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">,</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">],</span>
+        <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">,</span>
+        <span class="n">Tuple</span><span class="p">[</span><span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">,</span> <span class="n">pd</span><span class="o">.</span><span class="n">Series</span><span class="p">],</span>
+    <span class="p">]:</span>
+<span class="w">        </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Transforms the input dataframe into vectorized form. If a target</span>
+<span class="sd">        column name is passed the target values will be returned separately</span>
+<span class="sd">        in their corresponding type (np.ndarray or pd.DataFrame)</span>
+
+<span class="sd">        Parameters</span>
+<span class="sd">        ----------</span>
+<span class="sd">        df: pd.DataFrame</span>
+<span class="sd">            DataFrame to be vectorised, i.e. the categorical and continuous</span>
+<span class="sd">            columns will be encoded based on the processing applied within</span>
+<span class="sd">            the model</span>
+<span class="sd">        target_col: str, Optional</span>
+<span class="sd">            Column name of the target_col variable. If `None` only the array of</span>
+<span class="sd">            predictors will be returned</span>
+
+<span class="sd">        Returns</span>
+<span class="sd">        -------</span>
+<span class="sd">        Union[np.ndarray, Tuple[np.ndarray, np.ndarray], pd.DataFrame, Tuple[pd.DataFrame, pd.Series]</span>
+<span class="sd">            Returns eiter a numpy array with the vectorised values, or a Tuple</span>
+<span class="sd">            of numpy arrays with the vectorised values and the target. The</span>
+<span class="sd">            same applies to dataframes in case we choose to set</span>
+<span class="sd">            `return_dataframe = True`</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+
+        <span class="n">X_tab</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">tab_preprocessor</span><span class="o">.</span><span class="n">transform</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
+
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">device</span><span class="o">.</span><span class="n">type</span> <span class="o">==</span> <span class="s2">&quot;mps&quot;</span><span class="p">:</span>
+            <span class="n">X</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">from_numpy</span><span class="p">(</span><span class="n">X_tab</span><span class="o">.</span><span class="n">astype</span><span class="p">(</span><span class="s2">&quot;float32&quot;</span><span class="p">))</span><span class="o">.</span><span class="n">to</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">device</span><span class="p">)</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="n">X</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">from_numpy</span><span class="p">(</span><span class="n">X_tab</span><span class="o">.</span><span class="n">astype</span><span class="p">(</span><span class="s2">&quot;float&quot;</span><span class="p">))</span><span class="o">.</span><span class="n">to</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">device</span><span class="p">)</span>
+
+        <span class="k">with</span> <span class="n">torch</span><span class="o">.</span><span class="n">no_grad</span><span class="p">():</span>
+            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">is_tab_transformer</span><span class="p">:</span>
+                <span class="n">x_vec</span><span class="p">,</span> <span class="n">x_cont_not_embed</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">vectorizer</span><span class="p">(</span><span class="n">X</span><span class="p">)</span>
+            <span class="k">else</span><span class="p">:</span>
+                <span class="n">x_vec</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">vectorizer</span><span class="p">(</span><span class="n">X</span><span class="p">)</span>
+                <span class="n">x_cont_not_embed</span> <span class="o">=</span> <span class="kc">None</span>
+
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">tab_preprocessor</span><span class="o">.</span><span class="n">with_cls_token</span><span class="p">:</span>
+            <span class="n">x_vec</span> <span class="o">=</span> <span class="n">x_vec</span><span class="p">[:,</span> <span class="mi">1</span><span class="p">:,</span> <span class="p">:]</span>
+
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">tab_preprocessor</span><span class="o">.</span><span class="n">with_attention</span><span class="p">:</span>
+            <span class="n">x_vec</span> <span class="o">=</span> <span class="n">einops</span><span class="o">.</span><span class="n">rearrange</span><span class="p">(</span><span class="n">x_vec</span><span class="p">,</span> <span class="s2">&quot;s c e -&gt; s (c e)&quot;</span><span class="p">)</span>
+
+        <span class="k">if</span> <span class="n">x_cont_not_embed</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="n">x_vec</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">cat</span><span class="p">([</span><span class="n">x_vec</span><span class="p">,</span> <span class="n">x_cont_not_embed</span><span class="p">],</span> <span class="mi">1</span><span class="p">)</span><span class="o">.</span><span class="n">detach</span><span class="p">()</span><span class="o">.</span><span class="n">cpu</span><span class="p">()</span><span class="o">.</span><span class="n">numpy</span><span class="p">()</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="n">x_vec</span> <span class="o">=</span> <span class="n">x_vec</span><span class="o">.</span><span class="n">detach</span><span class="p">()</span><span class="o">.</span><span class="n">cpu</span><span class="p">()</span><span class="o">.</span><span class="n">numpy</span><span class="p">()</span>
+
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">return_dataframe</span><span class="p">:</span>
+            <span class="n">new_colnames</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_new_colnames</span><span class="p">()</span>
+            <span class="k">if</span> <span class="n">target_col</span><span class="p">:</span>
+                <span class="k">return</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">(</span><span class="n">data</span><span class="o">=</span><span class="n">x_vec</span><span class="p">,</span> <span class="n">columns</span><span class="o">=</span><span class="n">new_colnames</span><span class="p">),</span> <span class="n">df</span><span class="p">[[</span><span class="n">target_col</span><span class="p">]]</span>
+            <span class="k">else</span><span class="p">:</span>
+                <span class="k">return</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">(</span><span class="n">data</span><span class="o">=</span><span class="n">x_vec</span><span class="p">,</span> <span class="n">columns</span><span class="o">=</span><span class="n">new_colnames</span><span class="p">)</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="k">if</span> <span class="n">target_col</span><span class="p">:</span>
+                <span class="k">return</span> <span class="n">x_vec</span><span class="p">,</span> <span class="n">df</span><span class="p">[</span><span class="n">target_col</span><span class="p">]</span><span class="o">.</span><span class="n">values</span>
+            <span class="k">else</span><span class="p">:</span>
+                <span class="k">return</span> <span class="n">x_vec</span>
+
+    <span class="k">def</span> <span class="nf">fit_transform</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">,</span> <span class="n">target_col</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span>
+    <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Union</span><span class="p">[</span>
+        <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">,</span>
+        <span class="n">Tuple</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">,</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">],</span>
+        <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">,</span>
+        <span class="n">Tuple</span><span class="p">[</span><span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">,</span> <span class="n">pd</span><span class="o">.</span><span class="n">Series</span><span class="p">],</span>
+    <span class="p">]:</span>
+<span class="w">        </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Combines `fit` and `transform`&quot;&quot;&quot;</span>
+        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">df</span><span class="p">,</span> <span class="n">target_col</span><span class="p">)</span><span class="o">.</span><span class="n">transform</span><span class="p">(</span><span class="n">df</span><span class="p">,</span> <span class="n">target_col</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">_new_colnames</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]:</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">tab_preprocessor</span><span class="o">.</span><span class="n">with_attention</span><span class="p">:</span>
+            <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_new_colnames_with_attention</span><span class="p">()</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_new_colnames_without_attention</span><span class="p">()</span>
+
+    <span class="k">def</span> <span class="nf">_new_colnames_with_attention</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span>
+    <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]:</span>
+        <span class="n">cat_cols</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">(</span>
+            <span class="p">[</span>
+                <span class="n">ei</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
+                <span class="k">for</span> <span class="n">ei</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">tab_preprocessor</span><span class="o">.</span><span class="n">cat_embed_input</span>
+                <span class="k">if</span> <span class="n">ei</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">!=</span> <span class="s2">&quot;cls_token&quot;</span>
+            <span class="p">]</span>
+            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">tab_preprocessor</span><span class="o">.</span><span class="n">cat_embed_input</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span>
+            <span class="k">else</span> <span class="p">[]</span>
+        <span class="p">)</span>
+
+        <span class="n">cont_cols</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">(</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">tab_preprocessor</span><span class="o">.</span><span class="n">continuous_cols</span>
+            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">tab_preprocessor</span><span class="o">.</span><span class="n">continuous_cols</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span>
+            <span class="k">else</span> <span class="p">[]</span>
+        <span class="p">)</span>
+
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">are_cont_embed</span><span class="p">:</span>
+            <span class="n">all_embed_cols</span> <span class="o">=</span> <span class="n">cat_cols</span> <span class="o">+</span> <span class="n">cont_cols</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="n">all_embed_cols</span> <span class="o">=</span> <span class="n">cat_cols</span>
+
+        <span class="n">new_colnames</span> <span class="o">=</span> <span class="p">[]</span>
+        <span class="k">for</span> <span class="n">colname</span> <span class="ow">in</span> <span class="n">all_embed_cols</span><span class="p">:</span>
+            <span class="k">assert</span> <span class="bp">self</span><span class="o">.</span><span class="n">input_dim</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span>
+            <span class="n">new_colnames</span><span class="o">.</span><span class="n">extend</span><span class="p">(</span>
+                <span class="p">[</span>
+                    <span class="s2">&quot;_&quot;</span><span class="o">.</span><span class="n">join</span><span class="p">([</span><span class="n">colname</span><span class="p">,</span> <span class="s2">&quot;embed&quot;</span><span class="p">,</span> <span class="nb">str</span><span class="p">(</span><span class="n">i</span> <span class="o">+</span> <span class="mi">1</span><span class="p">)])</span>
+                    <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">input_dim</span><span class="p">)</span>
+                <span class="p">]</span>
+            <span class="p">)</span>
+
+        <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">are_cont_embed</span><span class="p">:</span>
+            <span class="n">new_colnames</span> <span class="o">+=</span> <span class="n">cont_cols</span>
+
+        <span class="k">return</span> <span class="n">new_colnames</span>
+
+    <span class="k">def</span> <span class="nf">_new_colnames_without_attention</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span>
+    <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]:</span>
+        <span class="n">new_colnames</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
+
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">tab_preprocessor</span><span class="o">.</span><span class="n">cat_embed_input</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="k">for</span> <span class="n">colname</span><span class="p">,</span> <span class="n">_</span><span class="p">,</span> <span class="n">embedding_dim</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">tab_preprocessor</span><span class="o">.</span><span class="n">cat_embed_input</span><span class="p">:</span>  <span class="c1"># type: ignore[misc]</span>
+                <span class="n">new_colnames</span><span class="o">.</span><span class="n">extend</span><span class="p">(</span>
+                    <span class="p">[</span>
+                        <span class="s2">&quot;_&quot;</span><span class="o">.</span><span class="n">join</span><span class="p">([</span><span class="n">colname</span><span class="p">,</span> <span class="s2">&quot;embed&quot;</span><span class="p">,</span> <span class="nb">str</span><span class="p">(</span><span class="n">i</span> <span class="o">+</span> <span class="mi">1</span><span class="p">)])</span>
+                        <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">embedding_dim</span><span class="p">)</span>
+                    <span class="p">]</span>
+                <span class="p">)</span>
+
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">tab_preprocessor</span><span class="o">.</span><span class="n">continuous_cols</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">are_cont_embed</span><span class="p">:</span>
+                <span class="k">for</span> <span class="n">colname</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">tab_preprocessor</span><span class="o">.</span><span class="n">continuous_cols</span><span class="p">:</span>
+                    <span class="k">assert</span> <span class="bp">self</span><span class="o">.</span><span class="n">cont_embed_dim</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span>
+                    <span class="n">new_colnames</span><span class="o">.</span><span class="n">extend</span><span class="p">(</span>
+                        <span class="p">[</span>
+                            <span class="s2">&quot;_&quot;</span><span class="o">.</span><span class="n">join</span><span class="p">([</span><span class="n">colname</span><span class="p">,</span> <span class="s2">&quot;embed&quot;</span><span class="p">,</span> <span class="nb">str</span><span class="p">(</span><span class="n">i</span> <span class="o">+</span> <span class="mi">1</span><span class="p">)])</span>
+                            <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">cont_embed_dim</span><span class="p">)</span>
+                        <span class="p">]</span>
+                    <span class="p">)</span>
+            <span class="k">else</span><span class="p">:</span>
+                <span class="n">new_colnames</span> <span class="o">+=</span> <span class="bp">self</span><span class="o">.</span><span class="n">tab_preprocessor</span><span class="o">.</span><span class="n">continuous_cols</span>
+
+        <span class="k">return</span> <span class="n">new_colnames</span>
+
+    <span class="k">def</span> <span class="nf">_check_inputs</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span>
+        <span class="n">tab_preprocessor</span><span class="p">:</span> <span class="n">TabPreprocessor</span><span class="p">,</span>
+        <span class="n">model</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="n">WideDeep</span><span class="p">,</span> <span class="n">BayesianWide</span><span class="p">,</span> <span class="n">BayesianTabMlp</span><span class="p">],</span>
+        <span class="n">verbose</span><span class="p">:</span> <span class="nb">bool</span><span class="p">,</span>
+    <span class="p">)</span> <span class="o">-&gt;</span> <span class="kc">None</span><span class="p">:</span>
+        <span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">model</span><span class="p">,</span> <span class="n">BaseBayesianModel</span><span class="p">):</span>
+            <span class="k">if</span> <span class="n">verbose</span><span class="p">:</span>
+                <span class="k">if</span> <span class="n">model</span><span class="o">.</span><span class="n">deepimage</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">or</span> <span class="n">model</span><span class="o">.</span><span class="n">deeptext</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+                    <span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span>
+                        <span class="s2">&quot;Currently &#39;Tab2Vec&#39; is only implemented for the &#39;deeptabular&#39; component.&quot;</span>
+                    <span class="p">)</span>
+
+            <span class="k">if</span> <span class="n">model</span><span class="o">.</span><span class="n">deeptabular</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+                <span class="k">raise</span> <span class="ne">RuntimeError</span><span class="p">(</span>
+                    <span class="s2">&quot;Currently &#39;Tab2Vec&#39; is only implemented for the &#39;deeptabular&#39; component.&quot;</span>
+                <span class="p">)</span>
+            <span class="k">if</span> <span class="ow">not</span> <span class="n">tab_preprocessor</span><span class="o">.</span><span class="n">is_fitted</span><span class="p">:</span>
+                <span class="k">raise</span> <span class="ne">RuntimeError</span><span class="p">(</span>
+                    <span class="s2">&quot;The &#39;tab_preprocessor&#39; must be fitted before is passed to &#39;Tab2Vec&#39;&quot;</span>
+                <span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">_set_vectorizer</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span> <span class="n">model</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="n">WideDeep</span><span class="p">,</span> <span class="n">BayesianWide</span><span class="p">,</span> <span class="n">BayesianTabMlp</span><span class="p">]</span>
+    <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Callable</span><span class="p">:</span>
+        <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">model</span><span class="p">,</span> <span class="n">BaseBayesianModel</span><span class="p">):</span>
+            <span class="n">vectorizer</span> <span class="o">=</span> <span class="n">model</span><span class="o">.</span><span class="n">_get_embeddings</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">is_tab_transformer</span> <span class="o">=</span> <span class="kc">False</span>
+        <span class="k">elif</span> <span class="n">model</span><span class="o">.</span><span class="n">deeptabular</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="vm">__class__</span><span class="o">.</span><span class="vm">__name__</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="o">==</span> <span class="s2">&quot;tabtransformer&quot;</span><span class="p">:</span>  <span class="c1"># type: ignore</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">is_tab_transformer</span> <span class="o">=</span> <span class="kc">True</span>
+            <span class="n">vectorizer</span> <span class="o">=</span> <span class="n">model</span><span class="o">.</span><span class="n">deeptabular</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">_get_embeddings_tt</span>  <span class="c1"># type: ignore</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">is_tab_transformer</span> <span class="o">=</span> <span class="kc">False</span>
+            <span class="n">vectorizer</span> <span class="o">=</span> <span class="n">model</span><span class="o">.</span><span class="n">deeptabular</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">_get_embeddings</span>  <span class="c1"># type: ignore</span>
+        <span class="k">return</span> <span class="n">vectorizer</span>
+
+    <span class="k">def</span> <span class="nf">_set_dim_attributes</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span>
+        <span class="n">tab_preprocessor</span><span class="p">:</span> <span class="n">TabPreprocessor</span><span class="p">,</span>
+        <span class="n">model</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="n">WideDeep</span><span class="p">,</span> <span class="n">BayesianWide</span><span class="p">,</span> <span class="n">BayesianTabMlp</span><span class="p">],</span>
+    <span class="p">)</span> <span class="o">-&gt;</span> <span class="kc">None</span><span class="p">:</span>
+        <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">model</span><span class="p">,</span> <span class="n">BaseBayesianModel</span><span class="p">):</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">are_cont_embed</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="p">(</span>
+                <span class="n">model</span><span class="o">.</span><span class="n">embed_continuous</span> <span class="k">if</span> <span class="n">model</span><span class="o">.</span><span class="n">embed_continuous</span> <span class="k">else</span> <span class="kc">False</span>
+            <span class="p">)</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">input_dim</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">cont_embed_dim</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="n">model</span><span class="o">.</span><span class="n">cont_embed_dim</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">are_cont_embed</span> <span class="o">=</span> <span class="p">(</span>
+                <span class="n">model</span><span class="o">.</span><span class="n">deeptabular</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">embed_continuous_method</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span>  <span class="c1"># type: ignore</span>
+            <span class="p">)</span>
+            <span class="k">if</span> <span class="n">tab_preprocessor</span><span class="o">.</span><span class="n">with_attention</span><span class="p">:</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">input_dim</span> <span class="o">=</span> <span class="n">model</span><span class="o">.</span><span class="n">deeptabular</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">input_dim</span>  <span class="c1"># type: ignore</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">cont_embed_dim</span> <span class="o">=</span> <span class="kc">None</span>
+            <span class="k">else</span><span class="p">:</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">input_dim</span> <span class="o">=</span> <span class="kc">None</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">cont_embed_dim</span> <span class="o">=</span> <span class="n">model</span><span class="o">.</span><span class="n">deeptabular</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">cont_embed_dim</span>  <span class="c1"># type: ignore</span>
 </code></pre></div></td></tr></table></div>
-                  </details>
+              </details>
 
 
 
@@ -1797,41 +2438,72 @@ <h3 id="pytorch_widedeep.tab2vec.Tab2Vec.fit" class="doc doc-heading">
 
 
 <p><span class="doc-section-title">Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>df</code></b>
-              (<code><span title="pandas.DataFrame">DataFrame</span></code>)
-          –
-          <div class="doc-md-description">
-            <p>DataFrame to be vectorised, i.e. the categorical and continuous
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+          <th>Default</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>df</code></td>
+            <td>
+                  <code><span title="pandas.DataFrame">DataFrame</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>DataFrame to be vectorised, i.e. the categorical and continuous
 columns will be encoded based on the processing applied within
 the model</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>target_col</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Column name of the target_col variable. If <code>None</code> only the array of
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>target_col</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Column name of the target_col variable. If <code>None</code> only the array of
 predictors will be returned</p>
-          </div>
-        </li>
-    </ul>
-
-
-<p><span class="doc-section-title">Returns:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-              <code><a class="autorefs autorefs-internal" title="pytorch_widedeep.tab2vec.Tab2Vec" href="#pytorch_widedeep.tab2vec.Tab2Vec">Tab2Vec</a></code>
-          –
-          <div class="doc-md-description">
-            
-          </div>
-        </li>
-    </ul>
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+      </tbody>
+    </table>
+
+
+    <p><span class="doc-section-title">Returns:</span></p>
+    <table>
+      <thead>
+        <tr>
+          <th>Type</th>
+          <th>Description</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td>
+                  <code><a class="autorefs autorefs-internal" title="pytorch_widedeep.tab2vec.Tab2Vec" href="#pytorch_widedeep.tab2vec.Tab2Vec">Tab2Vec</a></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                
+              </div>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
             <details class="quote">
               <summary>Source code in <code>pytorch_widedeep/tab2vec.py</code></summary>
@@ -1901,44 +2573,75 @@ <h3 id="pytorch_widedeep.tab2vec.Tab2Vec.transform" class="doc doc-heading">
 
 
 <p><span class="doc-section-title">Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>df</code></b>
-              (<code><span title="pandas.DataFrame">DataFrame</span></code>)
-          –
-          <div class="doc-md-description">
-            <p>DataFrame to be vectorised, i.e. the categorical and continuous
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+          <th>Default</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>df</code></td>
+            <td>
+                  <code><span title="pandas.DataFrame">DataFrame</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>DataFrame to be vectorised, i.e. the categorical and continuous
 columns will be encoded based on the processing applied within
 the model</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>target_col</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Column name of the target_col variable. If <code>None</code> only the array of
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>target_col</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Column name of the target_col variable. If <code>None</code> only the array of
 predictors will be returned</p>
-          </div>
-        </li>
-    </ul>
-
-
-<p><span class="doc-section-title">Returns:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-              <code>Union[np.ndarray, Tuple[np.ndarray, np.ndarray], pd.DataFrame, Tuple[pd.DataFrame, pd.Series]</code>
-          –
-          <div class="doc-md-description">
-            <p>Returns eiter a numpy array with the vectorised values, or a Tuple
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+      </tbody>
+    </table>
+
+
+    <p><span class="doc-section-title">Returns:</span></p>
+    <table>
+      <thead>
+        <tr>
+          <th>Type</th>
+          <th>Description</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td>
+                  <code>Union[np.ndarray, Tuple[np.ndarray, np.ndarray], pd.DataFrame, Tuple[pd.DataFrame, pd.Series]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Returns eiter a numpy array with the vectorised values, or a Tuple
 of numpy arrays with the vectorised values and the target. The
 same applies to dataframes in case we choose to set
 <code>return_dataframe = True</code></p>
-          </div>
-        </li>
-    </ul>
+              </div>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
             <details class="quote">
               <summary>Source code in <code>pytorch_widedeep/tab2vec.py</code></summary>
diff --git a/mkdocs/site/pytorch-widedeep/trainer.html b/mkdocs/site/pytorch-widedeep/trainer.html
index e1da16a0..c9ac062d 100644
--- a/mkdocs/site/pytorch-widedeep/trainer.html
+++ b/mkdocs/site/pytorch-widedeep/trainer.html
@@ -1387,7 +1387,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/16_Self_Supervised_Pretraning_pt1.ipynb" class="md-nav__link">
+      <a href="../examples/15_Self_Supervised_Pretraning_pt1.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1408,7 +1408,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/16_Self_Supervised_Pretraning_pt2.ipynb" class="md-nav__link">
+      <a href="../examples/15_Self_Supervised_Pretraning_pt2.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1429,7 +1429,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/17_Usign_a_custom_hugging_face_model.ipynb" class="md-nav__link">
+      <a href="../examples/16_Usign_a_custom_hugging_face_model.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1450,7 +1450,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/18_feature_importance_via_attention_weights.ipynb" class="md-nav__link">
+      <a href="../examples/17_feature_importance_via_attention_weights.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1471,7 +1471,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/19_wide_and_deep_for_recsys_pt1.ipynb" class="md-nav__link">
+      <a href="../examples/18_wide_and_deep_for_recsys_pt1.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1492,7 +1492,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/19_wide_and_deep_for_recsys_pt2.ipynb" class="md-nav__link">
+      <a href="../examples/18_wide_and_deep_for_recsys_pt2.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1513,7 +1513,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/20_load_from_folder_functionality.ipynb" class="md-nav__link">
+      <a href="../examples/19_load_from_folder_functionality.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1534,7 +1534,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../examples/21_Using_huggingface_within_widedeep.ipynb" class="md-nav__link">
+      <a href="../examples/20_Using_huggingface_within_widedeep.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1612,22 +1612,7 @@ <h2 id="pytorch_widedeep.training.Trainer" class="doc doc-heading">
 
 
 <a href="#pytorch_widedeep.training.Trainer" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="nf">Trainer</span><span class="p">(</span>
-    <span class="n">model</span><span class="p">,</span>
-    <span class="n">objective</span><span class="p">,</span>
-    <span class="n">custom_loss_function</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">optimizers</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">lr_schedulers</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">initializers</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">transforms</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">callbacks</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">metrics</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">eval_metrics</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">verbose</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span>
-    <span class="n">seed</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span>
-    <span class="o">**</span><span class="n">kwargs</span>
-<span class="p">)</span>
-</code></pre></div>
+
 
     <div class="doc doc-contents first">
             <p class="doc doc-class-bases">
@@ -1639,21 +1624,38 @@ <h2 id="pytorch_widedeep.training.Trainer" class="doc doc-heading">
 
 
 <p><span class="doc-section-title">Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>model</code></b>
-              (<code><a class="autorefs autorefs-internal" title="pytorch_widedeep.wdtypes.WideDeep" href="model_components.html#pytorch_widedeep.models.wide_deep.WideDeep">WideDeep</a></code>)
-          –
-          <div class="doc-md-description">
-            <p>An object of class <code>WideDeep</code></p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>objective</code></b>
-              (<code>str</code>)
-          –
-          <div class="doc-md-description">
-            <p>Defines the objective, loss or cost function. <br/></p>
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+          <th>Default</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>model</code></td>
+            <td>
+                  <code><a class="autorefs autorefs-internal" title="pytorch_widedeep.wdtypes.WideDeep" href="model_components.html#pytorch_widedeep.models.wide_deep.WideDeep">WideDeep</a></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>An object of class <code>WideDeep</code></p>
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>objective</code></td>
+            <td>
+                  <code>str</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Defines the objective, loss or cost function. <br/></p>
 <p>Param aliases: <code>loss_function</code>, <code>loss_fn</code>, <code>loss</code>,
 <code>cost_function</code>, <code>cost_fn</code>, <code>cost</code>. <br/></p>
 <p>Possible values are:</p>
@@ -1699,31 +1701,39 @@ <h2 id="pytorch_widedeep.training.Trainer" class="doc doc-heading">
 </li>
 </ul>
 <p><strong>NOTE</strong>: For <code>multitarget</code> a custom loss function must be passed</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>custom_loss_function</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="torch.nn.Module">Module</span>]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>It is possible to pass a custom loss function. See for example
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>custom_loss_function</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="torch.nn.Module">Module</span>]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>It is possible to pass a custom loss function. See for example
 <code>pytorch_widedeep.losses.FocalLoss</code> for the required structure of the
 object or the Examples section in this documentation or in the repo.
 Note that if <code>custom_loss_function</code> is not <code>None</code>, <code>objective</code> must
 be <em>'binary'</em>, <em>'multiclass'</em> or <em>'regression'</em>, consistent with the
 loss function</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>optimizers</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="pytorch_widedeep.wdtypes.Optimizer">Optimizer</span>, <span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, <span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="pytorch_widedeep.wdtypes.Optimizer">Optimizer</span>, <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.wdtypes.Optimizer">Optimizer</span>]]]]]</code>, default:
+              </div>
+            </td>
+            <td>
                   <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <ul>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>optimizers</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="pytorch_widedeep.wdtypes.Optimizer">Optimizer</span>, <span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, <span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="pytorch_widedeep.wdtypes.Optimizer">Optimizer</span>, <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.wdtypes.Optimizer">Optimizer</span>]]]]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <ul>
 <li>An instance of Pytorch's <code>Optimizer</code> object
   (e.g. <code>torch.optim.Adam()</code>) or</li>
 <li>a dictionary where there keys are the model components (i.e.
@@ -1736,16 +1746,20 @@ <h2 id="pytorch_widedeep.training.Trainer" class="doc doc-heading">
 </ul>
 <p>if no optimizers are passed it will default to <code>Adam</code> for all
 model components</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>lr_schedulers</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="pytorch_widedeep.wdtypes.LRScheduler">LRScheduler</span>, <span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, <span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="pytorch_widedeep.wdtypes.LRScheduler">LRScheduler</span>, <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.wdtypes.LRScheduler">LRScheduler</span>]]]]]</code>, default:
+              </div>
+            </td>
+            <td>
                   <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <ul>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>lr_schedulers</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="pytorch_widedeep.wdtypes.LRScheduler">LRScheduler</span>, <span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, <span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="pytorch_widedeep.wdtypes.LRScheduler">LRScheduler</span>, <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.wdtypes.LRScheduler">LRScheduler</span>]]]]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <ul>
 <li>An instance of Pytorch's <code>LRScheduler</code> object (e.g
   <code>torch.optim.lr_scheduler.StepLR(opt, step_size=5)</code>) or</li>
 <li>a dictionary where there keys are the model componenst (i.e. <em>'wide'</em>,
@@ -1754,16 +1768,20 @@ <h2 id="pytorch_widedeep.training.Trainer" class="doc doc-heading">
     learning rate schedulers if multiple models are used for the given
     data mode (e.g. two text columns/models for the deeptext component).</li>
 </ul>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>initializers</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="pytorch_widedeep.initializers.Initializer">Initializer</span>, <span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, <span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="pytorch_widedeep.initializers.Initializer">Initializer</span>, <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.initializers.Initializer">Initializer</span>]]]]]</code>, default:
+              </div>
+            </td>
+            <td>
                   <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <ul>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>initializers</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="pytorch_widedeep.initializers.Initializer">Initializer</span>, <span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, <span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="pytorch_widedeep.initializers.Initializer">Initializer</span>, <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.initializers.Initializer">Initializer</span>]]]]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <ul>
 <li>An instance of an <code>Initializer</code> object see <code>pytorch-widedeep.initializers</code> or</li>
 <li>a dictionary where there keys are the model components (i.e. <em>'wide'</em>,
   <em>'deeptabular'</em>, <em>'deeptext'</em>, <em>'deepimage'</em> and/or <em>'deephead'</em>)
@@ -1771,44 +1789,56 @@ <h2 id="pytorch_widedeep.training.Trainer" class="doc doc-heading">
     initializers if multiple models are used for the given data mode (e.g.
     two text columns/models for the deeptext component).</li>
 </ul>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>transforms</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.wdtypes.Transforms">Transforms</span>]]</code>, default:
+              </div>
+            </td>
+            <td>
                   <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>List with <code>torchvision.transforms</code> to be applied to the image
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>transforms</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.wdtypes.Transforms">Transforms</span>]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>List with <code>torchvision.transforms</code> to be applied to the image
 component of the model (i.e. <code>deepimage</code>) See
 <a href="https://pytorch.org/docs/stable/torchvision/transforms.html">torchvision transforms</a>.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>callbacks</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.callbacks.Callback">Callback</span>]]</code>, default:
+              </div>
+            </td>
+            <td>
                   <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>List with <code>Callback</code> objects. The three callbacks available in
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>callbacks</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.callbacks.Callback">Callback</span>]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>List with <code>Callback</code> objects. The three callbacks available in
 <code>pytorch-widedeep</code> are: <code>LRHistory</code>, <code>ModelCheckpoint</code> and
 <code>EarlyStopping</code>. The <code>History</code> and the <code>LRShedulerCallback</code> callbacks
 are used by default. This can also be a custom callback as long as
 the object of type <code>Callback</code>. See
 <code>pytorch_widedeep.callbacks.Callback</code> or the examples folder in the
 repo.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>metrics</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.metrics.Metric">Metric</span>], <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="torchmetrics.Metric">Metric</span>]]]</code>, default:
+              </div>
+            </td>
+            <td>
                   <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <ul>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>metrics</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.metrics.Metric">Metric</span>], <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="torchmetrics.Metric">Metric</span>]]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <ul>
 <li>List of objects of type <code>Metric</code>. Metrics available are:
   <code>Accuracy</code>, <code>Precision</code>, <code>Recall</code>, <code>FBetaScore</code>,
   <code>F1Score</code> and <code>R2Score</code>. This can also be a custom metric as long
@@ -1822,38 +1852,61 @@ <h2 id="pytorch_widedeep.training.Trainer" class="doc doc-heading">
   it is an object of type <code>Metric</code>. See
   <a href="https://torchmetrics.readthedocs.io/en/latest/">the instructions</a>.</li>
 </ul>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>verbose</code></b>
-              (<code>int</code>, default:
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>verbose</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Verbosity level. If set to 0 nothing will be printed during training</p>
+              </div>
+            </td>
+            <td>
                   <code>1</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Verbosity level. If set to 0 nothing will be printed during training</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>seed</code></b>
-              (<code>int</code>, default:
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>seed</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Random seed to be used internally for train/test split</p>
+              </div>
+            </td>
+            <td>
                   <code>1</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Random seed to be used internally for train/test split</p>
-          </div>
-        </li>
-    </ul>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
 
 <p><span class="doc-section-title">Other Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>**kwargs</code></b>
-          –
-          <div class="doc-md-description">
-            <p>Other infrequently used arguments that can also be passed as kwargs are:</p>
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>**kwargs</code></td>
+            <td>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Other infrequently used arguments that can also be passed as kwargs are:</p>
 <ul>
 <li>
 <p><strong>device</strong>: <code>str</code><br/>
@@ -1878,34 +1931,51 @@ <h2 id="pytorch_widedeep.training.Trainer" class="doc doc-heading">
     take a step: One of <em>'loss'</em> or <em>'metric'</em>. The ReduceLROnPlateau
     learning rate is a bit particular.</li>
 </ul>
-          </div>
-        </li>
-    </ul>
+              </div>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
 
 <p><span class="doc-section-title">Attributes:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code><span title="pytorch_widedeep.training.Trainer.cyclic_lr">cyclic_lr</span></code></b>
-              (<code>bool</code>)
-          –
-          <div class="doc-md-description">
-            <p>Attribute that indicates if any of the lr_schedulers is cyclic_lr
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code><span title="pytorch_widedeep.training.Trainer.cyclic_lr">cyclic_lr</span></code></td>
+            <td>
+                  <code>bool</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Attribute that indicates if any of the lr_schedulers is cyclic_lr
 (i.e. <code>CyclicLR</code> or
 <code>OneCycleLR</code>). See <a href="https://pytorch.org/docs/stable/optim.html">Pytorch schedulers</a>.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code><span title="pytorch_widedeep.training.Trainer.feature_importance">feature_importance</span></code></b>
-              (<code>dict</code>)
-          –
-          <div class="doc-md-description">
-            <p>dict where the keys are the column names and the values are the
+              </div>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code><span title="pytorch_widedeep.training.Trainer.feature_importance">feature_importance</span></code></td>
+            <td>
+                  <code>dict</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>dict where the keys are the column names and the values are the
 corresponding feature importances. This attribute will only exist
 if the <code>deeptabular</code> component is a Tabnet model.</p>
-          </div>
-        </li>
-    </ul>
+              </div>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
 
 <p><span class="doc-section-title">Examples:</span></p>
@@ -1952,93 +2022,2309 @@ <h2 id="pytorch_widedeep.training.Trainer" class="doc doc-heading">
 <span class="gp">... </span><span class="n">lr_schedulers</span><span class="o">=</span><span class="n">schedulers</span><span class="p">,</span> <span class="n">callbacks</span><span class="o">=</span><span class="n">callbacks</span><span class="p">,</span> <span class="n">transforms</span><span class="o">=</span><span class="n">transforms</span><span class="p">)</span>
 </code></pre></div>
 
-                  <details class="quote">
-                    <summary>Source code in <code>pytorch_widedeep/training/trainer.py</code></summary>
-                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">230</span>
-<span class="normal">231</span>
-<span class="normal">232</span>
-<span class="normal">233</span>
-<span class="normal">234</span>
-<span class="normal">235</span>
-<span class="normal">236</span>
-<span class="normal">237</span>
-<span class="normal">238</span>
-<span class="normal">239</span>
-<span class="normal">240</span>
-<span class="normal">241</span>
-<span class="normal">242</span>
-<span class="normal">243</span>
-<span class="normal">244</span>
-<span class="normal">245</span>
-<span class="normal">246</span>
-<span class="normal">247</span>
-<span class="normal">248</span>
-<span class="normal">249</span>
-<span class="normal">250</span>
-<span class="normal">251</span>
-<span class="normal">252</span>
-<span class="normal">253</span>
-<span class="normal">254</span>
-<span class="normal">255</span>
-<span class="normal">256</span>
-<span class="normal">257</span>
-<span class="normal">258</span>
-<span class="normal">259</span>
-<span class="normal">260</span>
-<span class="normal">261</span>
-<span class="normal">262</span>
-<span class="normal">263</span>
-<span class="normal">264</span>
-<span class="normal">265</span>
-<span class="normal">266</span>
-<span class="normal">267</span>
-<span class="normal">268</span>
-<span class="normal">269</span>
-<span class="normal">270</span>
-<span class="normal">271</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="nd">@alias</span><span class="p">(</span>  <span class="c1"># noqa: C901</span>
-    <span class="s2">&quot;objective&quot;</span><span class="p">,</span>
-    <span class="p">[</span><span class="s2">&quot;loss_function&quot;</span><span class="p">,</span> <span class="s2">&quot;loss_fn&quot;</span><span class="p">,</span> <span class="s2">&quot;loss&quot;</span><span class="p">,</span> <span class="s2">&quot;cost_function&quot;</span><span class="p">,</span> <span class="s2">&quot;cost_fn&quot;</span><span class="p">,</span> <span class="s2">&quot;cost&quot;</span><span class="p">],</span>
-<span class="p">)</span>
-<span class="nd">@alias</span><span class="p">(</span><span class="s2">&quot;metrics&quot;</span><span class="p">,</span> <span class="p">[</span><span class="s2">&quot;train_metrics&quot;</span><span class="p">])</span>
-<span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
-    <span class="bp">self</span><span class="p">,</span>
-    <span class="n">model</span><span class="p">:</span> <span class="n">WideDeep</span><span class="p">,</span>
-    <span class="n">objective</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
-    <span class="n">custom_loss_function</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">nn</span><span class="o">.</span><span class="n">Module</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">optimizers</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span>
-        <span class="n">Union</span><span class="p">[</span><span class="n">Optimizer</span><span class="p">,</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Union</span><span class="p">[</span><span class="n">Optimizer</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="n">Optimizer</span><span class="p">]]]]</span>
-    <span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">lr_schedulers</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span>
-        <span class="n">Union</span><span class="p">[</span><span class="n">LRScheduler</span><span class="p">,</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Union</span><span class="p">[</span><span class="n">LRScheduler</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="n">LRScheduler</span><span class="p">]]]]</span>
-    <span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">initializers</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span>
-        <span class="n">Union</span><span class="p">[</span><span class="n">Initializer</span><span class="p">,</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Union</span><span class="p">[</span><span class="n">Initializer</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="n">Initializer</span><span class="p">]]]]</span>
-    <span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">transforms</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="n">Transforms</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">callbacks</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="n">Callback</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">metrics</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Union</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="n">Metric</span><span class="p">],</span> <span class="n">List</span><span class="p">[</span><span class="n">TorchMetric</span><span class="p">]]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">eval_metrics</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Union</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="n">Metric</span><span class="p">],</span> <span class="n">List</span><span class="p">[</span><span class="n">TorchMetric</span><span class="p">]]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">verbose</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1</span><span class="p">,</span>
-    <span class="n">seed</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1</span><span class="p">,</span>
-    <span class="o">**</span><span class="n">kwargs</span><span class="p">,</span>
-<span class="p">):</span>
-    <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span>
-        <span class="n">model</span><span class="o">=</span><span class="n">model</span><span class="p">,</span>
-        <span class="n">objective</span><span class="o">=</span><span class="n">objective</span><span class="p">,</span>
-        <span class="n">custom_loss_function</span><span class="o">=</span><span class="n">custom_loss_function</span><span class="p">,</span>
-        <span class="n">optimizers</span><span class="o">=</span><span class="n">optimizers</span><span class="p">,</span>
-        <span class="n">lr_schedulers</span><span class="o">=</span><span class="n">lr_schedulers</span><span class="p">,</span>
-        <span class="n">initializers</span><span class="o">=</span><span class="n">initializers</span><span class="p">,</span>
-        <span class="n">transforms</span><span class="o">=</span><span class="n">transforms</span><span class="p">,</span>
-        <span class="n">callbacks</span><span class="o">=</span><span class="n">callbacks</span><span class="p">,</span>
-        <span class="n">metrics</span><span class="o">=</span><span class="n">metrics</span><span class="p">,</span>
-        <span class="n">eval_metrics</span><span class="o">=</span><span class="n">eval_metrics</span><span class="p">,</span>
-        <span class="n">verbose</span><span class="o">=</span><span class="n">verbose</span><span class="p">,</span>
-        <span class="n">seed</span><span class="o">=</span><span class="n">seed</span><span class="p">,</span>
-        <span class="o">**</span><span class="n">kwargs</span><span class="p">,</span>
+              <details class="quote">
+                <summary>Source code in <code>pytorch_widedeep/training/trainer.py</code></summary>
+                <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">  43</span>
+<span class="normal">  44</span>
+<span class="normal">  45</span>
+<span class="normal">  46</span>
+<span class="normal">  47</span>
+<span class="normal">  48</span>
+<span class="normal">  49</span>
+<span class="normal">  50</span>
+<span class="normal">  51</span>
+<span class="normal">  52</span>
+<span class="normal">  53</span>
+<span class="normal">  54</span>
+<span class="normal">  55</span>
+<span class="normal">  56</span>
+<span class="normal">  57</span>
+<span class="normal">  58</span>
+<span class="normal">  59</span>
+<span class="normal">  60</span>
+<span class="normal">  61</span>
+<span class="normal">  62</span>
+<span class="normal">  63</span>
+<span class="normal">  64</span>
+<span class="normal">  65</span>
+<span class="normal">  66</span>
+<span class="normal">  67</span>
+<span class="normal">  68</span>
+<span class="normal">  69</span>
+<span class="normal">  70</span>
+<span class="normal">  71</span>
+<span class="normal">  72</span>
+<span class="normal">  73</span>
+<span class="normal">  74</span>
+<span class="normal">  75</span>
+<span class="normal">  76</span>
+<span class="normal">  77</span>
+<span class="normal">  78</span>
+<span class="normal">  79</span>
+<span class="normal">  80</span>
+<span class="normal">  81</span>
+<span class="normal">  82</span>
+<span class="normal">  83</span>
+<span class="normal">  84</span>
+<span class="normal">  85</span>
+<span class="normal">  86</span>
+<span class="normal">  87</span>
+<span class="normal">  88</span>
+<span class="normal">  89</span>
+<span class="normal">  90</span>
+<span class="normal">  91</span>
+<span class="normal">  92</span>
+<span class="normal">  93</span>
+<span class="normal">  94</span>
+<span class="normal">  95</span>
+<span class="normal">  96</span>
+<span class="normal">  97</span>
+<span class="normal">  98</span>
+<span class="normal">  99</span>
+<span class="normal"> 100</span>
+<span class="normal"> 101</span>
+<span class="normal"> 102</span>
+<span class="normal"> 103</span>
+<span class="normal"> 104</span>
+<span class="normal"> 105</span>
+<span class="normal"> 106</span>
+<span class="normal"> 107</span>
+<span class="normal"> 108</span>
+<span class="normal"> 109</span>
+<span class="normal"> 110</span>
+<span class="normal"> 111</span>
+<span class="normal"> 112</span>
+<span class="normal"> 113</span>
+<span class="normal"> 114</span>
+<span class="normal"> 115</span>
+<span class="normal"> 116</span>
+<span class="normal"> 117</span>
+<span class="normal"> 118</span>
+<span class="normal"> 119</span>
+<span class="normal"> 120</span>
+<span class="normal"> 121</span>
+<span class="normal"> 122</span>
+<span class="normal"> 123</span>
+<span class="normal"> 124</span>
+<span class="normal"> 125</span>
+<span class="normal"> 126</span>
+<span class="normal"> 127</span>
+<span class="normal"> 128</span>
+<span class="normal"> 129</span>
+<span class="normal"> 130</span>
+<span class="normal"> 131</span>
+<span class="normal"> 132</span>
+<span class="normal"> 133</span>
+<span class="normal"> 134</span>
+<span class="normal"> 135</span>
+<span class="normal"> 136</span>
+<span class="normal"> 137</span>
+<span class="normal"> 138</span>
+<span class="normal"> 139</span>
+<span class="normal"> 140</span>
+<span class="normal"> 141</span>
+<span class="normal"> 142</span>
+<span class="normal"> 143</span>
+<span class="normal"> 144</span>
+<span class="normal"> 145</span>
+<span class="normal"> 146</span>
+<span class="normal"> 147</span>
+<span class="normal"> 148</span>
+<span class="normal"> 149</span>
+<span class="normal"> 150</span>
+<span class="normal"> 151</span>
+<span class="normal"> 152</span>
+<span class="normal"> 153</span>
+<span class="normal"> 154</span>
+<span class="normal"> 155</span>
+<span class="normal"> 156</span>
+<span class="normal"> 157</span>
+<span class="normal"> 158</span>
+<span class="normal"> 159</span>
+<span class="normal"> 160</span>
+<span class="normal"> 161</span>
+<span class="normal"> 162</span>
+<span class="normal"> 163</span>
+<span class="normal"> 164</span>
+<span class="normal"> 165</span>
+<span class="normal"> 166</span>
+<span class="normal"> 167</span>
+<span class="normal"> 168</span>
+<span class="normal"> 169</span>
+<span class="normal"> 170</span>
+<span class="normal"> 171</span>
+<span class="normal"> 172</span>
+<span class="normal"> 173</span>
+<span class="normal"> 174</span>
+<span class="normal"> 175</span>
+<span class="normal"> 176</span>
+<span class="normal"> 177</span>
+<span class="normal"> 178</span>
+<span class="normal"> 179</span>
+<span class="normal"> 180</span>
+<span class="normal"> 181</span>
+<span class="normal"> 182</span>
+<span class="normal"> 183</span>
+<span class="normal"> 184</span>
+<span class="normal"> 185</span>
+<span class="normal"> 186</span>
+<span class="normal"> 187</span>
+<span class="normal"> 188</span>
+<span class="normal"> 189</span>
+<span class="normal"> 190</span>
+<span class="normal"> 191</span>
+<span class="normal"> 192</span>
+<span class="normal"> 193</span>
+<span class="normal"> 194</span>
+<span class="normal"> 195</span>
+<span class="normal"> 196</span>
+<span class="normal"> 197</span>
+<span class="normal"> 198</span>
+<span class="normal"> 199</span>
+<span class="normal"> 200</span>
+<span class="normal"> 201</span>
+<span class="normal"> 202</span>
+<span class="normal"> 203</span>
+<span class="normal"> 204</span>
+<span class="normal"> 205</span>
+<span class="normal"> 206</span>
+<span class="normal"> 207</span>
+<span class="normal"> 208</span>
+<span class="normal"> 209</span>
+<span class="normal"> 210</span>
+<span class="normal"> 211</span>
+<span class="normal"> 212</span>
+<span class="normal"> 213</span>
+<span class="normal"> 214</span>
+<span class="normal"> 215</span>
+<span class="normal"> 216</span>
+<span class="normal"> 217</span>
+<span class="normal"> 218</span>
+<span class="normal"> 219</span>
+<span class="normal"> 220</span>
+<span class="normal"> 221</span>
+<span class="normal"> 222</span>
+<span class="normal"> 223</span>
+<span class="normal"> 224</span>
+<span class="normal"> 225</span>
+<span class="normal"> 226</span>
+<span class="normal"> 227</span>
+<span class="normal"> 228</span>
+<span class="normal"> 229</span>
+<span class="normal"> 230</span>
+<span class="normal"> 231</span>
+<span class="normal"> 232</span>
+<span class="normal"> 233</span>
+<span class="normal"> 234</span>
+<span class="normal"> 235</span>
+<span class="normal"> 236</span>
+<span class="normal"> 237</span>
+<span class="normal"> 238</span>
+<span class="normal"> 239</span>
+<span class="normal"> 240</span>
+<span class="normal"> 241</span>
+<span class="normal"> 242</span>
+<span class="normal"> 243</span>
+<span class="normal"> 244</span>
+<span class="normal"> 245</span>
+<span class="normal"> 246</span>
+<span class="normal"> 247</span>
+<span class="normal"> 248</span>
+<span class="normal"> 249</span>
+<span class="normal"> 250</span>
+<span class="normal"> 251</span>
+<span class="normal"> 252</span>
+<span class="normal"> 253</span>
+<span class="normal"> 254</span>
+<span class="normal"> 255</span>
+<span class="normal"> 256</span>
+<span class="normal"> 257</span>
+<span class="normal"> 258</span>
+<span class="normal"> 259</span>
+<span class="normal"> 260</span>
+<span class="normal"> 261</span>
+<span class="normal"> 262</span>
+<span class="normal"> 263</span>
+<span class="normal"> 264</span>
+<span class="normal"> 265</span>
+<span class="normal"> 266</span>
+<span class="normal"> 267</span>
+<span class="normal"> 268</span>
+<span class="normal"> 269</span>
+<span class="normal"> 270</span>
+<span class="normal"> 271</span>
+<span class="normal"> 272</span>
+<span class="normal"> 273</span>
+<span class="normal"> 274</span>
+<span class="normal"> 275</span>
+<span class="normal"> 276</span>
+<span class="normal"> 277</span>
+<span class="normal"> 278</span>
+<span class="normal"> 279</span>
+<span class="normal"> 280</span>
+<span class="normal"> 281</span>
+<span class="normal"> 282</span>
+<span class="normal"> 283</span>
+<span class="normal"> 284</span>
+<span class="normal"> 285</span>
+<span class="normal"> 286</span>
+<span class="normal"> 287</span>
+<span class="normal"> 288</span>
+<span class="normal"> 289</span>
+<span class="normal"> 290</span>
+<span class="normal"> 291</span>
+<span class="normal"> 292</span>
+<span class="normal"> 293</span>
+<span class="normal"> 294</span>
+<span class="normal"> 295</span>
+<span class="normal"> 296</span>
+<span class="normal"> 297</span>
+<span class="normal"> 298</span>
+<span class="normal"> 299</span>
+<span class="normal"> 300</span>
+<span class="normal"> 301</span>
+<span class="normal"> 302</span>
+<span class="normal"> 303</span>
+<span class="normal"> 304</span>
+<span class="normal"> 305</span>
+<span class="normal"> 306</span>
+<span class="normal"> 307</span>
+<span class="normal"> 308</span>
+<span class="normal"> 309</span>
+<span class="normal"> 310</span>
+<span class="normal"> 311</span>
+<span class="normal"> 312</span>
+<span class="normal"> 313</span>
+<span class="normal"> 314</span>
+<span class="normal"> 315</span>
+<span class="normal"> 316</span>
+<span class="normal"> 317</span>
+<span class="normal"> 318</span>
+<span class="normal"> 319</span>
+<span class="normal"> 320</span>
+<span class="normal"> 321</span>
+<span class="normal"> 322</span>
+<span class="normal"> 323</span>
+<span class="normal"> 324</span>
+<span class="normal"> 325</span>
+<span class="normal"> 326</span>
+<span class="normal"> 327</span>
+<span class="normal"> 328</span>
+<span class="normal"> 329</span>
+<span class="normal"> 330</span>
+<span class="normal"> 331</span>
+<span class="normal"> 332</span>
+<span class="normal"> 333</span>
+<span class="normal"> 334</span>
+<span class="normal"> 335</span>
+<span class="normal"> 336</span>
+<span class="normal"> 337</span>
+<span class="normal"> 338</span>
+<span class="normal"> 339</span>
+<span class="normal"> 340</span>
+<span class="normal"> 341</span>
+<span class="normal"> 342</span>
+<span class="normal"> 343</span>
+<span class="normal"> 344</span>
+<span class="normal"> 345</span>
+<span class="normal"> 346</span>
+<span class="normal"> 347</span>
+<span class="normal"> 348</span>
+<span class="normal"> 349</span>
+<span class="normal"> 350</span>
+<span class="normal"> 351</span>
+<span class="normal"> 352</span>
+<span class="normal"> 353</span>
+<span class="normal"> 354</span>
+<span class="normal"> 355</span>
+<span class="normal"> 356</span>
+<span class="normal"> 357</span>
+<span class="normal"> 358</span>
+<span class="normal"> 359</span>
+<span class="normal"> 360</span>
+<span class="normal"> 361</span>
+<span class="normal"> 362</span>
+<span class="normal"> 363</span>
+<span class="normal"> 364</span>
+<span class="normal"> 365</span>
+<span class="normal"> 366</span>
+<span class="normal"> 367</span>
+<span class="normal"> 368</span>
+<span class="normal"> 369</span>
+<span class="normal"> 370</span>
+<span class="normal"> 371</span>
+<span class="normal"> 372</span>
+<span class="normal"> 373</span>
+<span class="normal"> 374</span>
+<span class="normal"> 375</span>
+<span class="normal"> 376</span>
+<span class="normal"> 377</span>
+<span class="normal"> 378</span>
+<span class="normal"> 379</span>
+<span class="normal"> 380</span>
+<span class="normal"> 381</span>
+<span class="normal"> 382</span>
+<span class="normal"> 383</span>
+<span class="normal"> 384</span>
+<span class="normal"> 385</span>
+<span class="normal"> 386</span>
+<span class="normal"> 387</span>
+<span class="normal"> 388</span>
+<span class="normal"> 389</span>
+<span class="normal"> 390</span>
+<span class="normal"> 391</span>
+<span class="normal"> 392</span>
+<span class="normal"> 393</span>
+<span class="normal"> 394</span>
+<span class="normal"> 395</span>
+<span class="normal"> 396</span>
+<span class="normal"> 397</span>
+<span class="normal"> 398</span>
+<span class="normal"> 399</span>
+<span class="normal"> 400</span>
+<span class="normal"> 401</span>
+<span class="normal"> 402</span>
+<span class="normal"> 403</span>
+<span class="normal"> 404</span>
+<span class="normal"> 405</span>
+<span class="normal"> 406</span>
+<span class="normal"> 407</span>
+<span class="normal"> 408</span>
+<span class="normal"> 409</span>
+<span class="normal"> 410</span>
+<span class="normal"> 411</span>
+<span class="normal"> 412</span>
+<span class="normal"> 413</span>
+<span class="normal"> 414</span>
+<span class="normal"> 415</span>
+<span class="normal"> 416</span>
+<span class="normal"> 417</span>
+<span class="normal"> 418</span>
+<span class="normal"> 419</span>
+<span class="normal"> 420</span>
+<span class="normal"> 421</span>
+<span class="normal"> 422</span>
+<span class="normal"> 423</span>
+<span class="normal"> 424</span>
+<span class="normal"> 425</span>
+<span class="normal"> 426</span>
+<span class="normal"> 427</span>
+<span class="normal"> 428</span>
+<span class="normal"> 429</span>
+<span class="normal"> 430</span>
+<span class="normal"> 431</span>
+<span class="normal"> 432</span>
+<span class="normal"> 433</span>
+<span class="normal"> 434</span>
+<span class="normal"> 435</span>
+<span class="normal"> 436</span>
+<span class="normal"> 437</span>
+<span class="normal"> 438</span>
+<span class="normal"> 439</span>
+<span class="normal"> 440</span>
+<span class="normal"> 441</span>
+<span class="normal"> 442</span>
+<span class="normal"> 443</span>
+<span class="normal"> 444</span>
+<span class="normal"> 445</span>
+<span class="normal"> 446</span>
+<span class="normal"> 447</span>
+<span class="normal"> 448</span>
+<span class="normal"> 449</span>
+<span class="normal"> 450</span>
+<span class="normal"> 451</span>
+<span class="normal"> 452</span>
+<span class="normal"> 453</span>
+<span class="normal"> 454</span>
+<span class="normal"> 455</span>
+<span class="normal"> 456</span>
+<span class="normal"> 457</span>
+<span class="normal"> 458</span>
+<span class="normal"> 459</span>
+<span class="normal"> 460</span>
+<span class="normal"> 461</span>
+<span class="normal"> 462</span>
+<span class="normal"> 463</span>
+<span class="normal"> 464</span>
+<span class="normal"> 465</span>
+<span class="normal"> 466</span>
+<span class="normal"> 467</span>
+<span class="normal"> 468</span>
+<span class="normal"> 469</span>
+<span class="normal"> 470</span>
+<span class="normal"> 471</span>
+<span class="normal"> 472</span>
+<span class="normal"> 473</span>
+<span class="normal"> 474</span>
+<span class="normal"> 475</span>
+<span class="normal"> 476</span>
+<span class="normal"> 477</span>
+<span class="normal"> 478</span>
+<span class="normal"> 479</span>
+<span class="normal"> 480</span>
+<span class="normal"> 481</span>
+<span class="normal"> 482</span>
+<span class="normal"> 483</span>
+<span class="normal"> 484</span>
+<span class="normal"> 485</span>
+<span class="normal"> 486</span>
+<span class="normal"> 487</span>
+<span class="normal"> 488</span>
+<span class="normal"> 489</span>
+<span class="normal"> 490</span>
+<span class="normal"> 491</span>
+<span class="normal"> 492</span>
+<span class="normal"> 493</span>
+<span class="normal"> 494</span>
+<span class="normal"> 495</span>
+<span class="normal"> 496</span>
+<span class="normal"> 497</span>
+<span class="normal"> 498</span>
+<span class="normal"> 499</span>
+<span class="normal"> 500</span>
+<span class="normal"> 501</span>
+<span class="normal"> 502</span>
+<span class="normal"> 503</span>
+<span class="normal"> 504</span>
+<span class="normal"> 505</span>
+<span class="normal"> 506</span>
+<span class="normal"> 507</span>
+<span class="normal"> 508</span>
+<span class="normal"> 509</span>
+<span class="normal"> 510</span>
+<span class="normal"> 511</span>
+<span class="normal"> 512</span>
+<span class="normal"> 513</span>
+<span class="normal"> 514</span>
+<span class="normal"> 515</span>
+<span class="normal"> 516</span>
+<span class="normal"> 517</span>
+<span class="normal"> 518</span>
+<span class="normal"> 519</span>
+<span class="normal"> 520</span>
+<span class="normal"> 521</span>
+<span class="normal"> 522</span>
+<span class="normal"> 523</span>
+<span class="normal"> 524</span>
+<span class="normal"> 525</span>
+<span class="normal"> 526</span>
+<span class="normal"> 527</span>
+<span class="normal"> 528</span>
+<span class="normal"> 529</span>
+<span class="normal"> 530</span>
+<span class="normal"> 531</span>
+<span class="normal"> 532</span>
+<span class="normal"> 533</span>
+<span class="normal"> 534</span>
+<span class="normal"> 535</span>
+<span class="normal"> 536</span>
+<span class="normal"> 537</span>
+<span class="normal"> 538</span>
+<span class="normal"> 539</span>
+<span class="normal"> 540</span>
+<span class="normal"> 541</span>
+<span class="normal"> 542</span>
+<span class="normal"> 543</span>
+<span class="normal"> 544</span>
+<span class="normal"> 545</span>
+<span class="normal"> 546</span>
+<span class="normal"> 547</span>
+<span class="normal"> 548</span>
+<span class="normal"> 549</span>
+<span class="normal"> 550</span>
+<span class="normal"> 551</span>
+<span class="normal"> 552</span>
+<span class="normal"> 553</span>
+<span class="normal"> 554</span>
+<span class="normal"> 555</span>
+<span class="normal"> 556</span>
+<span class="normal"> 557</span>
+<span class="normal"> 558</span>
+<span class="normal"> 559</span>
+<span class="normal"> 560</span>
+<span class="normal"> 561</span>
+<span class="normal"> 562</span>
+<span class="normal"> 563</span>
+<span class="normal"> 564</span>
+<span class="normal"> 565</span>
+<span class="normal"> 566</span>
+<span class="normal"> 567</span>
+<span class="normal"> 568</span>
+<span class="normal"> 569</span>
+<span class="normal"> 570</span>
+<span class="normal"> 571</span>
+<span class="normal"> 572</span>
+<span class="normal"> 573</span>
+<span class="normal"> 574</span>
+<span class="normal"> 575</span>
+<span class="normal"> 576</span>
+<span class="normal"> 577</span>
+<span class="normal"> 578</span>
+<span class="normal"> 579</span>
+<span class="normal"> 580</span>
+<span class="normal"> 581</span>
+<span class="normal"> 582</span>
+<span class="normal"> 583</span>
+<span class="normal"> 584</span>
+<span class="normal"> 585</span>
+<span class="normal"> 586</span>
+<span class="normal"> 587</span>
+<span class="normal"> 588</span>
+<span class="normal"> 589</span>
+<span class="normal"> 590</span>
+<span class="normal"> 591</span>
+<span class="normal"> 592</span>
+<span class="normal"> 593</span>
+<span class="normal"> 594</span>
+<span class="normal"> 595</span>
+<span class="normal"> 596</span>
+<span class="normal"> 597</span>
+<span class="normal"> 598</span>
+<span class="normal"> 599</span>
+<span class="normal"> 600</span>
+<span class="normal"> 601</span>
+<span class="normal"> 602</span>
+<span class="normal"> 603</span>
+<span class="normal"> 604</span>
+<span class="normal"> 605</span>
+<span class="normal"> 606</span>
+<span class="normal"> 607</span>
+<span class="normal"> 608</span>
+<span class="normal"> 609</span>
+<span class="normal"> 610</span>
+<span class="normal"> 611</span>
+<span class="normal"> 612</span>
+<span class="normal"> 613</span>
+<span class="normal"> 614</span>
+<span class="normal"> 615</span>
+<span class="normal"> 616</span>
+<span class="normal"> 617</span>
+<span class="normal"> 618</span>
+<span class="normal"> 619</span>
+<span class="normal"> 620</span>
+<span class="normal"> 621</span>
+<span class="normal"> 622</span>
+<span class="normal"> 623</span>
+<span class="normal"> 624</span>
+<span class="normal"> 625</span>
+<span class="normal"> 626</span>
+<span class="normal"> 627</span>
+<span class="normal"> 628</span>
+<span class="normal"> 629</span>
+<span class="normal"> 630</span>
+<span class="normal"> 631</span>
+<span class="normal"> 632</span>
+<span class="normal"> 633</span>
+<span class="normal"> 634</span>
+<span class="normal"> 635</span>
+<span class="normal"> 636</span>
+<span class="normal"> 637</span>
+<span class="normal"> 638</span>
+<span class="normal"> 639</span>
+<span class="normal"> 640</span>
+<span class="normal"> 641</span>
+<span class="normal"> 642</span>
+<span class="normal"> 643</span>
+<span class="normal"> 644</span>
+<span class="normal"> 645</span>
+<span class="normal"> 646</span>
+<span class="normal"> 647</span>
+<span class="normal"> 648</span>
+<span class="normal"> 649</span>
+<span class="normal"> 650</span>
+<span class="normal"> 651</span>
+<span class="normal"> 652</span>
+<span class="normal"> 653</span>
+<span class="normal"> 654</span>
+<span class="normal"> 655</span>
+<span class="normal"> 656</span>
+<span class="normal"> 657</span>
+<span class="normal"> 658</span>
+<span class="normal"> 659</span>
+<span class="normal"> 660</span>
+<span class="normal"> 661</span>
+<span class="normal"> 662</span>
+<span class="normal"> 663</span>
+<span class="normal"> 664</span>
+<span class="normal"> 665</span>
+<span class="normal"> 666</span>
+<span class="normal"> 667</span>
+<span class="normal"> 668</span>
+<span class="normal"> 669</span>
+<span class="normal"> 670</span>
+<span class="normal"> 671</span>
+<span class="normal"> 672</span>
+<span class="normal"> 673</span>
+<span class="normal"> 674</span>
+<span class="normal"> 675</span>
+<span class="normal"> 676</span>
+<span class="normal"> 677</span>
+<span class="normal"> 678</span>
+<span class="normal"> 679</span>
+<span class="normal"> 680</span>
+<span class="normal"> 681</span>
+<span class="normal"> 682</span>
+<span class="normal"> 683</span>
+<span class="normal"> 684</span>
+<span class="normal"> 685</span>
+<span class="normal"> 686</span>
+<span class="normal"> 687</span>
+<span class="normal"> 688</span>
+<span class="normal"> 689</span>
+<span class="normal"> 690</span>
+<span class="normal"> 691</span>
+<span class="normal"> 692</span>
+<span class="normal"> 693</span>
+<span class="normal"> 694</span>
+<span class="normal"> 695</span>
+<span class="normal"> 696</span>
+<span class="normal"> 697</span>
+<span class="normal"> 698</span>
+<span class="normal"> 699</span>
+<span class="normal"> 700</span>
+<span class="normal"> 701</span>
+<span class="normal"> 702</span>
+<span class="normal"> 703</span>
+<span class="normal"> 704</span>
+<span class="normal"> 705</span>
+<span class="normal"> 706</span>
+<span class="normal"> 707</span>
+<span class="normal"> 708</span>
+<span class="normal"> 709</span>
+<span class="normal"> 710</span>
+<span class="normal"> 711</span>
+<span class="normal"> 712</span>
+<span class="normal"> 713</span>
+<span class="normal"> 714</span>
+<span class="normal"> 715</span>
+<span class="normal"> 716</span>
+<span class="normal"> 717</span>
+<span class="normal"> 718</span>
+<span class="normal"> 719</span>
+<span class="normal"> 720</span>
+<span class="normal"> 721</span>
+<span class="normal"> 722</span>
+<span class="normal"> 723</span>
+<span class="normal"> 724</span>
+<span class="normal"> 725</span>
+<span class="normal"> 726</span>
+<span class="normal"> 727</span>
+<span class="normal"> 728</span>
+<span class="normal"> 729</span>
+<span class="normal"> 730</span>
+<span class="normal"> 731</span>
+<span class="normal"> 732</span>
+<span class="normal"> 733</span>
+<span class="normal"> 734</span>
+<span class="normal"> 735</span>
+<span class="normal"> 736</span>
+<span class="normal"> 737</span>
+<span class="normal"> 738</span>
+<span class="normal"> 739</span>
+<span class="normal"> 740</span>
+<span class="normal"> 741</span>
+<span class="normal"> 742</span>
+<span class="normal"> 743</span>
+<span class="normal"> 744</span>
+<span class="normal"> 745</span>
+<span class="normal"> 746</span>
+<span class="normal"> 747</span>
+<span class="normal"> 748</span>
+<span class="normal"> 749</span>
+<span class="normal"> 750</span>
+<span class="normal"> 751</span>
+<span class="normal"> 752</span>
+<span class="normal"> 753</span>
+<span class="normal"> 754</span>
+<span class="normal"> 755</span>
+<span class="normal"> 756</span>
+<span class="normal"> 757</span>
+<span class="normal"> 758</span>
+<span class="normal"> 759</span>
+<span class="normal"> 760</span>
+<span class="normal"> 761</span>
+<span class="normal"> 762</span>
+<span class="normal"> 763</span>
+<span class="normal"> 764</span>
+<span class="normal"> 765</span>
+<span class="normal"> 766</span>
+<span class="normal"> 767</span>
+<span class="normal"> 768</span>
+<span class="normal"> 769</span>
+<span class="normal"> 770</span>
+<span class="normal"> 771</span>
+<span class="normal"> 772</span>
+<span class="normal"> 773</span>
+<span class="normal"> 774</span>
+<span class="normal"> 775</span>
+<span class="normal"> 776</span>
+<span class="normal"> 777</span>
+<span class="normal"> 778</span>
+<span class="normal"> 779</span>
+<span class="normal"> 780</span>
+<span class="normal"> 781</span>
+<span class="normal"> 782</span>
+<span class="normal"> 783</span>
+<span class="normal"> 784</span>
+<span class="normal"> 785</span>
+<span class="normal"> 786</span>
+<span class="normal"> 787</span>
+<span class="normal"> 788</span>
+<span class="normal"> 789</span>
+<span class="normal"> 790</span>
+<span class="normal"> 791</span>
+<span class="normal"> 792</span>
+<span class="normal"> 793</span>
+<span class="normal"> 794</span>
+<span class="normal"> 795</span>
+<span class="normal"> 796</span>
+<span class="normal"> 797</span>
+<span class="normal"> 798</span>
+<span class="normal"> 799</span>
+<span class="normal"> 800</span>
+<span class="normal"> 801</span>
+<span class="normal"> 802</span>
+<span class="normal"> 803</span>
+<span class="normal"> 804</span>
+<span class="normal"> 805</span>
+<span class="normal"> 806</span>
+<span class="normal"> 807</span>
+<span class="normal"> 808</span>
+<span class="normal"> 809</span>
+<span class="normal"> 810</span>
+<span class="normal"> 811</span>
+<span class="normal"> 812</span>
+<span class="normal"> 813</span>
+<span class="normal"> 814</span>
+<span class="normal"> 815</span>
+<span class="normal"> 816</span>
+<span class="normal"> 817</span>
+<span class="normal"> 818</span>
+<span class="normal"> 819</span>
+<span class="normal"> 820</span>
+<span class="normal"> 821</span>
+<span class="normal"> 822</span>
+<span class="normal"> 823</span>
+<span class="normal"> 824</span>
+<span class="normal"> 825</span>
+<span class="normal"> 826</span>
+<span class="normal"> 827</span>
+<span class="normal"> 828</span>
+<span class="normal"> 829</span>
+<span class="normal"> 830</span>
+<span class="normal"> 831</span>
+<span class="normal"> 832</span>
+<span class="normal"> 833</span>
+<span class="normal"> 834</span>
+<span class="normal"> 835</span>
+<span class="normal"> 836</span>
+<span class="normal"> 837</span>
+<span class="normal"> 838</span>
+<span class="normal"> 839</span>
+<span class="normal"> 840</span>
+<span class="normal"> 841</span>
+<span class="normal"> 842</span>
+<span class="normal"> 843</span>
+<span class="normal"> 844</span>
+<span class="normal"> 845</span>
+<span class="normal"> 846</span>
+<span class="normal"> 847</span>
+<span class="normal"> 848</span>
+<span class="normal"> 849</span>
+<span class="normal"> 850</span>
+<span class="normal"> 851</span>
+<span class="normal"> 852</span>
+<span class="normal"> 853</span>
+<span class="normal"> 854</span>
+<span class="normal"> 855</span>
+<span class="normal"> 856</span>
+<span class="normal"> 857</span>
+<span class="normal"> 858</span>
+<span class="normal"> 859</span>
+<span class="normal"> 860</span>
+<span class="normal"> 861</span>
+<span class="normal"> 862</span>
+<span class="normal"> 863</span>
+<span class="normal"> 864</span>
+<span class="normal"> 865</span>
+<span class="normal"> 866</span>
+<span class="normal"> 867</span>
+<span class="normal"> 868</span>
+<span class="normal"> 869</span>
+<span class="normal"> 870</span>
+<span class="normal"> 871</span>
+<span class="normal"> 872</span>
+<span class="normal"> 873</span>
+<span class="normal"> 874</span>
+<span class="normal"> 875</span>
+<span class="normal"> 876</span>
+<span class="normal"> 877</span>
+<span class="normal"> 878</span>
+<span class="normal"> 879</span>
+<span class="normal"> 880</span>
+<span class="normal"> 881</span>
+<span class="normal"> 882</span>
+<span class="normal"> 883</span>
+<span class="normal"> 884</span>
+<span class="normal"> 885</span>
+<span class="normal"> 886</span>
+<span class="normal"> 887</span>
+<span class="normal"> 888</span>
+<span class="normal"> 889</span>
+<span class="normal"> 890</span>
+<span class="normal"> 891</span>
+<span class="normal"> 892</span>
+<span class="normal"> 893</span>
+<span class="normal"> 894</span>
+<span class="normal"> 895</span>
+<span class="normal"> 896</span>
+<span class="normal"> 897</span>
+<span class="normal"> 898</span>
+<span class="normal"> 899</span>
+<span class="normal"> 900</span>
+<span class="normal"> 901</span>
+<span class="normal"> 902</span>
+<span class="normal"> 903</span>
+<span class="normal"> 904</span>
+<span class="normal"> 905</span>
+<span class="normal"> 906</span>
+<span class="normal"> 907</span>
+<span class="normal"> 908</span>
+<span class="normal"> 909</span>
+<span class="normal"> 910</span>
+<span class="normal"> 911</span>
+<span class="normal"> 912</span>
+<span class="normal"> 913</span>
+<span class="normal"> 914</span>
+<span class="normal"> 915</span>
+<span class="normal"> 916</span>
+<span class="normal"> 917</span>
+<span class="normal"> 918</span>
+<span class="normal"> 919</span>
+<span class="normal"> 920</span>
+<span class="normal"> 921</span>
+<span class="normal"> 922</span>
+<span class="normal"> 923</span>
+<span class="normal"> 924</span>
+<span class="normal"> 925</span>
+<span class="normal"> 926</span>
+<span class="normal"> 927</span>
+<span class="normal"> 928</span>
+<span class="normal"> 929</span>
+<span class="normal"> 930</span>
+<span class="normal"> 931</span>
+<span class="normal"> 932</span>
+<span class="normal"> 933</span>
+<span class="normal"> 934</span>
+<span class="normal"> 935</span>
+<span class="normal"> 936</span>
+<span class="normal"> 937</span>
+<span class="normal"> 938</span>
+<span class="normal"> 939</span>
+<span class="normal"> 940</span>
+<span class="normal"> 941</span>
+<span class="normal"> 942</span>
+<span class="normal"> 943</span>
+<span class="normal"> 944</span>
+<span class="normal"> 945</span>
+<span class="normal"> 946</span>
+<span class="normal"> 947</span>
+<span class="normal"> 948</span>
+<span class="normal"> 949</span>
+<span class="normal"> 950</span>
+<span class="normal"> 951</span>
+<span class="normal"> 952</span>
+<span class="normal"> 953</span>
+<span class="normal"> 954</span>
+<span class="normal"> 955</span>
+<span class="normal"> 956</span>
+<span class="normal"> 957</span>
+<span class="normal"> 958</span>
+<span class="normal"> 959</span>
+<span class="normal"> 960</span>
+<span class="normal"> 961</span>
+<span class="normal"> 962</span>
+<span class="normal"> 963</span>
+<span class="normal"> 964</span>
+<span class="normal"> 965</span>
+<span class="normal"> 966</span>
+<span class="normal"> 967</span>
+<span class="normal"> 968</span>
+<span class="normal"> 969</span>
+<span class="normal"> 970</span>
+<span class="normal"> 971</span>
+<span class="normal"> 972</span>
+<span class="normal"> 973</span>
+<span class="normal"> 974</span>
+<span class="normal"> 975</span>
+<span class="normal"> 976</span>
+<span class="normal"> 977</span>
+<span class="normal"> 978</span>
+<span class="normal"> 979</span>
+<span class="normal"> 980</span>
+<span class="normal"> 981</span>
+<span class="normal"> 982</span>
+<span class="normal"> 983</span>
+<span class="normal"> 984</span>
+<span class="normal"> 985</span>
+<span class="normal"> 986</span>
+<span class="normal"> 987</span>
+<span class="normal"> 988</span>
+<span class="normal"> 989</span>
+<span class="normal"> 990</span>
+<span class="normal"> 991</span>
+<span class="normal"> 992</span>
+<span class="normal"> 993</span>
+<span class="normal"> 994</span>
+<span class="normal"> 995</span>
+<span class="normal"> 996</span>
+<span class="normal"> 997</span>
+<span class="normal"> 998</span>
+<span class="normal"> 999</span>
+<span class="normal">1000</span>
+<span class="normal">1001</span>
+<span class="normal">1002</span>
+<span class="normal">1003</span>
+<span class="normal">1004</span>
+<span class="normal">1005</span>
+<span class="normal">1006</span>
+<span class="normal">1007</span>
+<span class="normal">1008</span>
+<span class="normal">1009</span>
+<span class="normal">1010</span>
+<span class="normal">1011</span>
+<span class="normal">1012</span>
+<span class="normal">1013</span>
+<span class="normal">1014</span>
+<span class="normal">1015</span>
+<span class="normal">1016</span>
+<span class="normal">1017</span>
+<span class="normal">1018</span>
+<span class="normal">1019</span>
+<span class="normal">1020</span>
+<span class="normal">1021</span>
+<span class="normal">1022</span>
+<span class="normal">1023</span>
+<span class="normal">1024</span>
+<span class="normal">1025</span>
+<span class="normal">1026</span>
+<span class="normal">1027</span>
+<span class="normal">1028</span>
+<span class="normal">1029</span>
+<span class="normal">1030</span>
+<span class="normal">1031</span>
+<span class="normal">1032</span>
+<span class="normal">1033</span>
+<span class="normal">1034</span>
+<span class="normal">1035</span>
+<span class="normal">1036</span>
+<span class="normal">1037</span>
+<span class="normal">1038</span>
+<span class="normal">1039</span>
+<span class="normal">1040</span>
+<span class="normal">1041</span>
+<span class="normal">1042</span>
+<span class="normal">1043</span>
+<span class="normal">1044</span>
+<span class="normal">1045</span>
+<span class="normal">1046</span>
+<span class="normal">1047</span>
+<span class="normal">1048</span>
+<span class="normal">1049</span>
+<span class="normal">1050</span>
+<span class="normal">1051</span>
+<span class="normal">1052</span>
+<span class="normal">1053</span>
+<span class="normal">1054</span>
+<span class="normal">1055</span>
+<span class="normal">1056</span>
+<span class="normal">1057</span>
+<span class="normal">1058</span>
+<span class="normal">1059</span>
+<span class="normal">1060</span>
+<span class="normal">1061</span>
+<span class="normal">1062</span>
+<span class="normal">1063</span>
+<span class="normal">1064</span>
+<span class="normal">1065</span>
+<span class="normal">1066</span>
+<span class="normal">1067</span>
+<span class="normal">1068</span>
+<span class="normal">1069</span>
+<span class="normal">1070</span>
+<span class="normal">1071</span>
+<span class="normal">1072</span>
+<span class="normal">1073</span>
+<span class="normal">1074</span>
+<span class="normal">1075</span>
+<span class="normal">1076</span>
+<span class="normal">1077</span>
+<span class="normal">1078</span>
+<span class="normal">1079</span>
+<span class="normal">1080</span>
+<span class="normal">1081</span>
+<span class="normal">1082</span>
+<span class="normal">1083</span>
+<span class="normal">1084</span>
+<span class="normal">1085</span>
+<span class="normal">1086</span>
+<span class="normal">1087</span>
+<span class="normal">1088</span>
+<span class="normal">1089</span>
+<span class="normal">1090</span>
+<span class="normal">1091</span>
+<span class="normal">1092</span>
+<span class="normal">1093</span>
+<span class="normal">1094</span>
+<span class="normal">1095</span>
+<span class="normal">1096</span>
+<span class="normal">1097</span>
+<span class="normal">1098</span>
+<span class="normal">1099</span>
+<span class="normal">1100</span>
+<span class="normal">1101</span>
+<span class="normal">1102</span>
+<span class="normal">1103</span>
+<span class="normal">1104</span>
+<span class="normal">1105</span>
+<span class="normal">1106</span>
+<span class="normal">1107</span>
+<span class="normal">1108</span>
+<span class="normal">1109</span>
+<span class="normal">1110</span>
+<span class="normal">1111</span>
+<span class="normal">1112</span>
+<span class="normal">1113</span>
+<span class="normal">1114</span>
+<span class="normal">1115</span>
+<span class="normal">1116</span>
+<span class="normal">1117</span>
+<span class="normal">1118</span>
+<span class="normal">1119</span>
+<span class="normal">1120</span>
+<span class="normal">1121</span>
+<span class="normal">1122</span>
+<span class="normal">1123</span>
+<span class="normal">1124</span>
+<span class="normal">1125</span>
+<span class="normal">1126</span>
+<span class="normal">1127</span>
+<span class="normal">1128</span>
+<span class="normal">1129</span>
+<span class="normal">1130</span>
+<span class="normal">1131</span>
+<span class="normal">1132</span>
+<span class="normal">1133</span>
+<span class="normal">1134</span>
+<span class="normal">1135</span>
+<span class="normal">1136</span>
+<span class="normal">1137</span>
+<span class="normal">1138</span>
+<span class="normal">1139</span>
+<span class="normal">1140</span>
+<span class="normal">1141</span>
+<span class="normal">1142</span>
+<span class="normal">1143</span>
+<span class="normal">1144</span>
+<span class="normal">1145</span>
+<span class="normal">1146</span>
+<span class="normal">1147</span>
+<span class="normal">1148</span>
+<span class="normal">1149</span>
+<span class="normal">1150</span>
+<span class="normal">1151</span>
+<span class="normal">1152</span>
+<span class="normal">1153</span>
+<span class="normal">1154</span>
+<span class="normal">1155</span>
+<span class="normal">1156</span>
+<span class="normal">1157</span>
+<span class="normal">1158</span>
+<span class="normal">1159</span>
+<span class="normal">1160</span>
+<span class="normal">1161</span>
+<span class="normal">1162</span>
+<span class="normal">1163</span>
+<span class="normal">1164</span>
+<span class="normal">1165</span>
+<span class="normal">1166</span>
+<span class="normal">1167</span>
+<span class="normal">1168</span>
+<span class="normal">1169</span>
+<span class="normal">1170</span>
+<span class="normal">1171</span>
+<span class="normal">1172</span>
+<span class="normal">1173</span>
+<span class="normal">1174</span>
+<span class="normal">1175</span>
+<span class="normal">1176</span>
+<span class="normal">1177</span>
+<span class="normal">1178</span>
+<span class="normal">1179</span>
+<span class="normal">1180</span>
+<span class="normal">1181</span>
+<span class="normal">1182</span>
+<span class="normal">1183</span>
+<span class="normal">1184</span>
+<span class="normal">1185</span>
+<span class="normal">1186</span>
+<span class="normal">1187</span>
+<span class="normal">1188</span>
+<span class="normal">1189</span>
+<span class="normal">1190</span>
+<span class="normal">1191</span>
+<span class="normal">1192</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">class</span> <span class="nc">Trainer</span><span class="p">(</span><span class="n">BaseTrainer</span><span class="p">):</span>
+<span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Class to set the of attributes that will be used during the</span>
+<span class="sd">    training process.</span>
+
+<span class="sd">    Parameters</span>
+<span class="sd">    ----------</span>
+<span class="sd">    model: `WideDeep`</span>
+<span class="sd">        An object of class `WideDeep`</span>
+<span class="sd">    objective: str</span>
+<span class="sd">        Defines the objective, loss or cost function. &lt;br/&gt;</span>
+
+<span class="sd">        Param aliases: `loss_function`, `loss_fn`, `loss`,</span>
+<span class="sd">        `cost_function`, `cost_fn`, `cost`. &lt;br/&gt;</span>
+
+<span class="sd">        Possible values are:</span>
+
+<span class="sd">        - `binary`, aliases: `logistic`, `binary_logloss`, `binary_cross_entropy`</span>
+
+<span class="sd">        - `binary_focal_loss`</span>
+
+<span class="sd">        - `multiclass`, aliases: `multi_logloss`, `cross_entropy`, `categorical_cross_entropy`,</span>
+
+<span class="sd">        - `multiclass_focal_loss`</span>
+
+<span class="sd">        - `regression`, aliases: `mse`, `l2`, `mean_squared_error`</span>
+
+<span class="sd">        - `mean_absolute_error`, aliases: `mae`, `l1`</span>
+
+<span class="sd">        - `mean_squared_log_error`, aliases: `msle`</span>
+
+<span class="sd">        - `root_mean_squared_error`, aliases:  `rmse`</span>
+
+<span class="sd">        - `root_mean_squared_log_error`, aliases: `rmsle`</span>
+
+<span class="sd">        - `zero_inflated_lognormal`, aliases: `ziln`</span>
+
+<span class="sd">        - `quantile`</span>
+
+<span class="sd">        - `tweedie`</span>
+
+<span class="sd">        - `multitarget`, aliases: `multi_target`</span>
+
+<span class="sd">        **NOTE**: For `multitarget` a custom loss function must be passed</span>
+<span class="sd">    custom_loss_function: `nn.Module`. Optional, default = None</span>
+<span class="sd">        It is possible to pass a custom loss function. See for example</span>
+<span class="sd">        `pytorch_widedeep.losses.FocalLoss` for the required structure of the</span>
+<span class="sd">        object or the Examples section in this documentation or in the repo.</span>
+<span class="sd">        Note that if `custom_loss_function` is not `None`, `objective` must</span>
+<span class="sd">        be _&#39;binary&#39;_, _&#39;multiclass&#39;_ or _&#39;regression&#39;_, consistent with the</span>
+<span class="sd">        loss function</span>
+<span class="sd">    optimizers: `Optimizer` or dict. Optional, default=None</span>
+<span class="sd">        - An instance of Pytorch&#39;s `Optimizer` object</span>
+<span class="sd">          (e.g. `torch.optim.Adam()`) or</span>
+<span class="sd">        - a dictionary where there keys are the model components (i.e.</span>
+<span class="sd">          _&#39;wide&#39;_, _&#39;deeptabular&#39;_, _&#39;deeptext&#39;_, _&#39;deepimage&#39;_</span>
+<span class="sd">          and/or _&#39;deephead&#39;_)  and the values are the corresponding</span>
+<span class="sd">          optimizers or list of optimizers if multiple models are used for</span>
+<span class="sd">          the given data mode (e.g. two text columns/models for the deeptext</span>
+<span class="sd">          component). If multiple optimizers are used the</span>
+<span class="sd">          dictionary **MUST** contain an optimizer per model component.</span>
+
+<span class="sd">        if no optimizers are passed it will default to `Adam` for all</span>
+<span class="sd">        model components</span>
+<span class="sd">    lr_schedulers: `LRScheduler` or dict. Optional, default=None</span>
+<span class="sd">        - An instance of Pytorch&#39;s `LRScheduler` object (e.g</span>
+<span class="sd">          `torch.optim.lr_scheduler.StepLR(opt, step_size=5)`) or</span>
+<span class="sd">        - a dictionary where there keys are the model componenst (i.e. _&#39;wide&#39;_,</span>
+<span class="sd">          _&#39;deeptabular&#39;_, _&#39;deeptext&#39;_, _&#39;deepimage&#39;_ and/or _&#39;deephead&#39;_) and the</span>
+<span class="sd">          values are the corresponding learning rate schedulers or list of</span>
+<span class="sd">            learning rate schedulers if multiple models are used for the given</span>
+<span class="sd">            data mode (e.g. two text columns/models for the deeptext component).</span>
+<span class="sd">    initializers: `Initializer` or dict. Optional, default=None</span>
+<span class="sd">        - An instance of an `Initializer` object see `pytorch-widedeep.initializers` or</span>
+<span class="sd">        - a dictionary where there keys are the model components (i.e. _&#39;wide&#39;_,</span>
+<span class="sd">          _&#39;deeptabular&#39;_, _&#39;deeptext&#39;_, _&#39;deepimage&#39;_ and/or _&#39;deephead&#39;_)</span>
+<span class="sd">          and the values are the corresponding initializers or list of</span>
+<span class="sd">            initializers if multiple models are used for the given data mode (e.g.</span>
+<span class="sd">            two text columns/models for the deeptext component).</span>
+<span class="sd">    transforms: List. Optional, default=None</span>
+<span class="sd">        List with `torchvision.transforms` to be applied to the image</span>
+<span class="sd">        component of the model (i.e. `deepimage`) See</span>
+<span class="sd">        [torchvision transforms](https://pytorch.org/docs/stable/torchvision/transforms.html).</span>
+<span class="sd">    callbacks: List. Optional, default=None</span>
+<span class="sd">        List with `Callback` objects. The three callbacks available in</span>
+<span class="sd">        `pytorch-widedeep` are: `LRHistory`, `ModelCheckpoint` and</span>
+<span class="sd">        `EarlyStopping`. The `History` and the `LRShedulerCallback` callbacks</span>
+<span class="sd">        are used by default. This can also be a custom callback as long as</span>
+<span class="sd">        the object of type `Callback`. See</span>
+<span class="sd">        `pytorch_widedeep.callbacks.Callback` or the examples folder in the</span>
+<span class="sd">        repo.</span>
+<span class="sd">    metrics: List. Optional, default=None</span>
+<span class="sd">        - List of objects of type `Metric`. Metrics available are:</span>
+<span class="sd">          `Accuracy`, `Precision`, `Recall`, `FBetaScore`,</span>
+<span class="sd">          `F1Score` and `R2Score`. This can also be a custom metric as long</span>
+<span class="sd">          as it is an object of type `Metric`. See</span>
+<span class="sd">          `pytorch_widedeep.metrics.Metric` or the examples folder in the</span>
+<span class="sd">          repo</span>
+<span class="sd">        - List of objects of type `torchmetrics.Metric`. This can be any</span>
+<span class="sd">          metric from torchmetrics library</span>
+<span class="sd">          [Examples](https://torchmetrics.readthedocs.io/en/latest/).</span>
+<span class="sd">          This can also be a custom metric as long as</span>
+<span class="sd">          it is an object of type `Metric`. See</span>
+<span class="sd">          [the instructions](https://torchmetrics.readthedocs.io/en/latest/).</span>
+<span class="sd">    verbose: int, default=1</span>
+<span class="sd">        Verbosity level. If set to 0 nothing will be printed during training</span>
+<span class="sd">    seed: int, default=1</span>
+<span class="sd">        Random seed to be used internally for train/test split</span>
+
+<span class="sd">    Other Parameters</span>
+<span class="sd">    ----------------</span>
+<span class="sd">    **kwargs: dict</span>
+<span class="sd">        Other infrequently used arguments that can also be passed as kwargs are:</span>
+
+<span class="sd">        - **device**: `str`&lt;br/&gt;</span>
+<span class="sd">            string indicating the device. One of _&#39;cpu&#39;_, _&#39;gpu&#39;_ or &#39;mps&#39; if</span>
+<span class="sd">            run on a Mac with Apple silicon or AMD GPU(s)</span>
+
+<span class="sd">        - **num_workers**: `int`&lt;br/&gt;</span>
+<span class="sd">            number of workers to be used internally by the data loaders</span>
+
+<span class="sd">        - **lambda_sparse**: `float`&lt;br/&gt;</span>
+<span class="sd">            lambda sparse parameter in case the `deeptabular` component is `TabNet`</span>
+
+<span class="sd">        - **class_weight**: `List[float]`&lt;br/&gt;</span>
+<span class="sd">            This is the `weight` or `pos_weight` parameter in</span>
+<span class="sd">            `CrossEntropyLoss` and `BCEWithLogitsLoss`, depending on whether</span>
+<span class="sd">        - **reducelronplateau_criterion**: `str`</span>
+<span class="sd">            This sets the criterion that will be used by the lr scheduler to</span>
+<span class="sd">            take a step: One of _&#39;loss&#39;_ or _&#39;metric&#39;_. The ReduceLROnPlateau</span>
+<span class="sd">            learning rate is a bit particular.</span>
+
+<span class="sd">    Attributes</span>
+<span class="sd">    ----------</span>
+<span class="sd">    cyclic_lr: bool</span>
+<span class="sd">        Attribute that indicates if any of the lr_schedulers is cyclic_lr</span>
+<span class="sd">        (i.e. `CyclicLR` or</span>
+<span class="sd">        `OneCycleLR`). See [Pytorch schedulers](https://pytorch.org/docs/stable/optim.html).</span>
+<span class="sd">    feature_importance: dict</span>
+<span class="sd">        dict where the keys are the column names and the values are the</span>
+<span class="sd">        corresponding feature importances. This attribute will only exist</span>
+<span class="sd">        if the `deeptabular` component is a Tabnet model.</span>
+
+<span class="sd">    Examples</span>
+<span class="sd">    --------</span>
+<span class="sd">    &gt;&gt;&gt; import torch</span>
+<span class="sd">    &gt;&gt;&gt; from torchvision.transforms import ToTensor</span>
+<span class="sd">    &gt;&gt;&gt;</span>
+<span class="sd">    &gt;&gt;&gt; # wide deep imports</span>
+<span class="sd">    &gt;&gt;&gt; from pytorch_widedeep.callbacks import EarlyStopping, LRHistory</span>
+<span class="sd">    &gt;&gt;&gt; from pytorch_widedeep.initializers import KaimingNormal, KaimingUniform, Normal, Uniform</span>
+<span class="sd">    &gt;&gt;&gt; from pytorch_widedeep.models import TabResnet, Vision, BasicRNN, Wide, WideDeep</span>
+<span class="sd">    &gt;&gt;&gt; from pytorch_widedeep import Trainer</span>
+<span class="sd">    &gt;&gt;&gt;</span>
+<span class="sd">    &gt;&gt;&gt; embed_input = [(u, i, j) for u, i, j in zip([&quot;a&quot;, &quot;b&quot;, &quot;c&quot;][:4], [4] * 3, [8] * 3)]</span>
+<span class="sd">    &gt;&gt;&gt; column_idx = {k: v for v, k in enumerate([&quot;a&quot;, &quot;b&quot;, &quot;c&quot;])}</span>
+<span class="sd">    &gt;&gt;&gt; wide = Wide(10, 1)</span>
+<span class="sd">    &gt;&gt;&gt;</span>
+<span class="sd">    &gt;&gt;&gt; # build the model</span>
+<span class="sd">    &gt;&gt;&gt; deeptabular = TabResnet(blocks_dims=[8, 4], column_idx=column_idx, cat_embed_input=embed_input)</span>
+<span class="sd">    &gt;&gt;&gt; deeptext = BasicRNN(vocab_size=10, embed_dim=4, padding_idx=0)</span>
+<span class="sd">    &gt;&gt;&gt; deepimage = Vision()</span>
+<span class="sd">    &gt;&gt;&gt; model = WideDeep(wide=wide, deeptabular=deeptabular, deeptext=deeptext, deepimage=deepimage)</span>
+<span class="sd">    &gt;&gt;&gt;</span>
+<span class="sd">    &gt;&gt;&gt; # set optimizers and schedulers</span>
+<span class="sd">    &gt;&gt;&gt; wide_opt = torch.optim.Adam(model.wide.parameters())</span>
+<span class="sd">    &gt;&gt;&gt; deep_opt = torch.optim.AdamW(model.deeptabular.parameters())</span>
+<span class="sd">    &gt;&gt;&gt; text_opt = torch.optim.Adam(model.deeptext.parameters())</span>
+<span class="sd">    &gt;&gt;&gt; img_opt = torch.optim.AdamW(model.deepimage.parameters())</span>
+<span class="sd">    &gt;&gt;&gt;</span>
+<span class="sd">    &gt;&gt;&gt; wide_sch = torch.optim.lr_scheduler.StepLR(wide_opt, step_size=5)</span>
+<span class="sd">    &gt;&gt;&gt; deep_sch = torch.optim.lr_scheduler.StepLR(deep_opt, step_size=3)</span>
+<span class="sd">    &gt;&gt;&gt; text_sch = torch.optim.lr_scheduler.StepLR(text_opt, step_size=5)</span>
+<span class="sd">    &gt;&gt;&gt; img_sch = torch.optim.lr_scheduler.StepLR(img_opt, step_size=3)</span>
+<span class="sd">    &gt;&gt;&gt;</span>
+<span class="sd">    &gt;&gt;&gt; optimizers = {&quot;wide&quot;: wide_opt, &quot;deeptabular&quot;: deep_opt, &quot;deeptext&quot;: text_opt, &quot;deepimage&quot;: img_opt}</span>
+<span class="sd">    &gt;&gt;&gt; schedulers = {&quot;wide&quot;: wide_sch, &quot;deeptabular&quot;: deep_sch, &quot;deeptext&quot;: text_sch, &quot;deepimage&quot;: img_sch}</span>
+<span class="sd">    &gt;&gt;&gt;</span>
+<span class="sd">    &gt;&gt;&gt; # set initializers and callbacks</span>
+<span class="sd">    &gt;&gt;&gt; initializers = {&quot;wide&quot;: Uniform, &quot;deeptabular&quot;: Normal, &quot;deeptext&quot;: KaimingNormal, &quot;deepimage&quot;: KaimingUniform}</span>
+<span class="sd">    &gt;&gt;&gt; transforms = [ToTensor]</span>
+<span class="sd">    &gt;&gt;&gt; callbacks = [LRHistory(n_epochs=4), EarlyStopping]</span>
+<span class="sd">    &gt;&gt;&gt;</span>
+<span class="sd">    &gt;&gt;&gt; # set the trainer</span>
+<span class="sd">    &gt;&gt;&gt; trainer = Trainer(model, objective=&quot;regression&quot;, initializers=initializers, optimizers=optimizers,</span>
+<span class="sd">    ... lr_schedulers=schedulers, callbacks=callbacks, transforms=transforms)</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+
+    <span class="nd">@alias</span><span class="p">(</span>  <span class="c1"># noqa: C901</span>
+        <span class="s2">&quot;objective&quot;</span><span class="p">,</span>
+        <span class="p">[</span><span class="s2">&quot;loss_function&quot;</span><span class="p">,</span> <span class="s2">&quot;loss_fn&quot;</span><span class="p">,</span> <span class="s2">&quot;loss&quot;</span><span class="p">,</span> <span class="s2">&quot;cost_function&quot;</span><span class="p">,</span> <span class="s2">&quot;cost_fn&quot;</span><span class="p">,</span> <span class="s2">&quot;cost&quot;</span><span class="p">],</span>
     <span class="p">)</span>
+    <span class="nd">@alias</span><span class="p">(</span><span class="s2">&quot;metrics&quot;</span><span class="p">,</span> <span class="p">[</span><span class="s2">&quot;train_metrics&quot;</span><span class="p">])</span>
+    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span>
+        <span class="n">model</span><span class="p">:</span> <span class="n">WideDeep</span><span class="p">,</span>
+        <span class="n">objective</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
+        <span class="n">custom_loss_function</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">nn</span><span class="o">.</span><span class="n">Module</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">optimizers</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span>
+            <span class="n">Union</span><span class="p">[</span><span class="n">Optimizer</span><span class="p">,</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Union</span><span class="p">[</span><span class="n">Optimizer</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="n">Optimizer</span><span class="p">]]]]</span>
+        <span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">lr_schedulers</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span>
+            <span class="n">Union</span><span class="p">[</span><span class="n">LRScheduler</span><span class="p">,</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Union</span><span class="p">[</span><span class="n">LRScheduler</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="n">LRScheduler</span><span class="p">]]]]</span>
+        <span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">initializers</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span>
+            <span class="n">Union</span><span class="p">[</span><span class="n">Initializer</span><span class="p">,</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Union</span><span class="p">[</span><span class="n">Initializer</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="n">Initializer</span><span class="p">]]]]</span>
+        <span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">transforms</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="n">Transforms</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">callbacks</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="n">Callback</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">metrics</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Union</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="n">Metric</span><span class="p">],</span> <span class="n">List</span><span class="p">[</span><span class="n">TorchMetric</span><span class="p">]]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">eval_metrics</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Union</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="n">Metric</span><span class="p">],</span> <span class="n">List</span><span class="p">[</span><span class="n">TorchMetric</span><span class="p">]]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">verbose</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1</span><span class="p">,</span>
+        <span class="n">seed</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1</span><span class="p">,</span>
+        <span class="o">**</span><span class="n">kwargs</span><span class="p">,</span>
+    <span class="p">):</span>
+        <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span>
+            <span class="n">model</span><span class="o">=</span><span class="n">model</span><span class="p">,</span>
+            <span class="n">objective</span><span class="o">=</span><span class="n">objective</span><span class="p">,</span>
+            <span class="n">custom_loss_function</span><span class="o">=</span><span class="n">custom_loss_function</span><span class="p">,</span>
+            <span class="n">optimizers</span><span class="o">=</span><span class="n">optimizers</span><span class="p">,</span>
+            <span class="n">lr_schedulers</span><span class="o">=</span><span class="n">lr_schedulers</span><span class="p">,</span>
+            <span class="n">initializers</span><span class="o">=</span><span class="n">initializers</span><span class="p">,</span>
+            <span class="n">transforms</span><span class="o">=</span><span class="n">transforms</span><span class="p">,</span>
+            <span class="n">callbacks</span><span class="o">=</span><span class="n">callbacks</span><span class="p">,</span>
+            <span class="n">metrics</span><span class="o">=</span><span class="n">metrics</span><span class="p">,</span>
+            <span class="n">eval_metrics</span><span class="o">=</span><span class="n">eval_metrics</span><span class="p">,</span>
+            <span class="n">verbose</span><span class="o">=</span><span class="n">verbose</span><span class="p">,</span>
+            <span class="n">seed</span><span class="o">=</span><span class="n">seed</span><span class="p">,</span>
+            <span class="o">**</span><span class="n">kwargs</span><span class="p">,</span>
+        <span class="p">)</span>
+
+    <span class="nd">@alias</span><span class="p">(</span><span class="s2">&quot;finetune&quot;</span><span class="p">,</span> <span class="p">[</span><span class="s2">&quot;warmup&quot;</span><span class="p">])</span>
+    <span class="nd">@alias</span><span class="p">(</span><span class="s2">&quot;train_dataloader&quot;</span><span class="p">,</span> <span class="p">[</span><span class="s2">&quot;custom_dataloader&quot;</span><span class="p">])</span>
+    <span class="nd">@alias</span><span class="p">(</span><span class="s2">&quot;eval_dataloader&quot;</span><span class="p">,</span> <span class="p">[</span><span class="s2">&quot;custom_eval_dataloader&quot;</span><span class="p">])</span>
+    <span class="k">def</span> <span class="nf">fit</span><span class="p">(</span>  <span class="c1"># noqa: C901</span>
+        <span class="bp">self</span><span class="p">,</span>
+        <span class="n">X_wide</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">X_tab</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Union</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">]]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">X_text</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Union</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">]]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">X_img</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Union</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">]]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">X_train</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Union</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">]]]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">X_val</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Union</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">]]]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">val_split</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">target</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">n_epochs</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1</span><span class="p">,</span>
+        <span class="n">validation_freq</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1</span><span class="p">,</span>
+        <span class="n">batch_size</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">32</span><span class="p">,</span>
+        <span class="n">train_dataloader</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">CustomDataLoader</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">eval_dataloader</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">CustomDataLoader</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">feature_importance_sample_size</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">finetune</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
+        <span class="o">**</span><span class="n">kwargs</span><span class="p">,</span>
+    <span class="p">):</span>
+<span class="w">        </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Fit method.</span>
+
+<span class="sd">        The input datasets can be passed either directly via numpy arrays</span>
+<span class="sd">        (`X_wide`, `X_tab`, `X_text` or `X_img`) or alternatively, in</span>
+<span class="sd">        dictionaries (`X_train` or `X_val`).</span>
+
+<span class="sd">        Parameters</span>
+<span class="sd">        ----------</span>
+<span class="sd">        X_wide: np.ndarray, Optional. default=None</span>
+<span class="sd">            Input for the `wide` model component.</span>
+<span class="sd">            See `pytorch_widedeep.preprocessing.WidePreprocessor`</span>
+<span class="sd">        X_tab: np.ndarray, Optional. default=None</span>
+<span class="sd">            Input for the `deeptabular` model component.</span>
+<span class="sd">            See `pytorch_widedeep.preprocessing.TabPreprocessor`. If multiple</span>
+<span class="sd">            tabular models are used for different columns, this should be a</span>
+<span class="sd">            list of numpy arrays</span>
+<span class="sd">        X_text: Union[np.ndarray, List[np.ndarray]], Optional. default=None</span>
+<span class="sd">            Input for the `deeptext` model component.</span>
+<span class="sd">            See `pytorch_widedeep.preprocessing.TextPreprocessor`.</span>
+<span class="sd">            If multiple text columns/models are used, this should be a list of</span>
+<span class="sd">            numpy arrays</span>
+<span class="sd">        X_img: np.ndarray, Optional. default=None</span>
+<span class="sd">            Input for the `deepimage` model component.</span>
+<span class="sd">            See `pytorch_widedeep.preprocessing.ImagePreprocessor`.</span>
+<span class="sd">            If multiple image columns/models are used, this should be a list of</span>
+<span class="sd">            numpy arrays</span>
+<span class="sd">        X_train: Dict, Optional. default=None</span>
+<span class="sd">            The training dataset can also be passed in a dictionary. Keys are</span>
+<span class="sd">            _&#39;X_wide&#39;_, _&#39;X_tab&#39;_, _&#39;X_text&#39;_, _&#39;X_img&#39;_ and _&#39;target&#39;_. Values</span>
+<span class="sd">            are the corresponding matrices. Note that of multiple text or image</span>
+<span class="sd">            columns/models are used, the corresponding values should be lists</span>
+<span class="sd">            of numpy arrays</span>
+<span class="sd">        X_val: Dict, Optional. default=None</span>
+<span class="sd">            The validation dataset can also be passed in a dictionary. Keys</span>
+<span class="sd">            are _&#39;X_wide&#39;_, _&#39;X_tab&#39;_, _&#39;X_text&#39;_, _&#39;X_img&#39;_ and _&#39;target&#39;_.</span>
+<span class="sd">            Values are the corresponding matrices. Note that of multiple text</span>
+<span class="sd">            or image columns/models are used, the corresponding values should</span>
+<span class="sd">            be lists of numpy arrays</span>
+<span class="sd">        val_split: float, Optional. default=None</span>
+<span class="sd">            train/val split fraction</span>
+<span class="sd">        target: np.ndarray, Optional. default=None</span>
+<span class="sd">            target values</span>
+<span class="sd">        n_epochs: int, default=1</span>
+<span class="sd">            number of epochs</span>
+<span class="sd">        validation_freq: int, default=1</span>
+<span class="sd">            epochs validation frequency</span>
+<span class="sd">        batch_size: int, default=32</span>
+<span class="sd">            batch size</span>
+<span class="sd">        custom_dataloader: `DataLoader`, Optional, default=None</span>
+<span class="sd">            object of class `torch.utils.data.DataLoader`. Available</span>
+<span class="sd">            predefined dataloaders are in `pytorch-widedeep.dataloaders`.If</span>
+<span class="sd">            `None`, a standard torch `DataLoader` is used.</span>
+<span class="sd">        finetune: bool, default=False</span>
+<span class="sd">            fine-tune individual model components. This functionality can also</span>
+<span class="sd">            be used to &#39;warm-up&#39; (and hence the alias `warmup`) individual</span>
+<span class="sd">            components before the joined training starts, and hence its</span>
+<span class="sd">            alias. See the Examples folder in the repo for more details</span>
+
+<span class="sd">            `pytorch_widedeep` implements 3 fine-tune routines.</span>
+
+<span class="sd">            - fine-tune all trainable layers at once. This routine is</span>
+<span class="sd">              inspired by the work of Howard &amp; Sebastian Ruder 2018 in their</span>
+<span class="sd">              [ULMfit paper](https://arxiv.org/abs/1801.06146). Using a</span>
+<span class="sd">              Slanted Triangular learing (see</span>
+<span class="sd">              [Leslie N. Smith paper](https://arxiv.org/pdf/1506.01186.pdf) ) ,</span>
+<span class="sd">              the process is the following: *i*) the learning rate will</span>
+<span class="sd">              gradually increase for 10% of the training steps from max_lr/10</span>
+<span class="sd">              to max_lr. *ii*) It will then gradually decrease to max_lr/10</span>
+<span class="sd">              for the remaining 90% of the steps. The optimizer used in the</span>
+<span class="sd">              process is `Adam`.</span>
+
+<span class="sd">            and two gradual fine-tune routines, where only certain layers are</span>
+<span class="sd">            trained at a time.</span>
+
+<span class="sd">            - The so called `Felbo` gradual fine-tune rourine, based on the the</span>
+<span class="sd">              Felbo et al., 2017 [DeepEmoji paper](https://arxiv.org/abs/1708.00524).</span>
+<span class="sd">            - The `Howard` routine based on the work of Howard &amp; Sebastian Ruder 2018 in their</span>
+<span class="sd">              [ULMfit paper](https://arxiv.org/abs/1801.06146&gt;).</span>
+
+<span class="sd">            For details on how these routines work, please see the Examples</span>
+<span class="sd">            section in this documentation and the Examples folder in the repo. &lt;br/&gt;</span>
+<span class="sd">            Param Alias: `warmup`</span>
+
+<span class="sd">        Other Parameters</span>
+<span class="sd">        ----------------</span>
+<span class="sd">        **kwargs:</span>
+<span class="sd">            Other keyword arguments are:</span>
+
+<span class="sd">            - **DataLoader related parameters**:&lt;br/&gt;</span>
+<span class="sd">                For example,  `sampler`, `batch_sampler`, `collate_fn`, etc.</span>
+<span class="sd">                Please, see the pytorch</span>
+<span class="sd">                [DataLoader docs](https://pytorch.org/docs/stable/data.html#torch.utils.data.DataLoader)</span>
+<span class="sd">                for details.</span>
+
+<span class="sd">            - **Finetune related parameters**:&lt;br/&gt;</span>
+<span class="sd">                see the source code at `pytorch_widedeep._finetune`. Namely, these are:</span>
+
+<span class="sd">                - `finetune_epochs` (`int`):</span>
+<span class="sd">                    number of epochs use for fine tuning</span>
+<span class="sd">                - `finetune_max_lr` (`float`):</span>
+<span class="sd">                   max lr during fine tuning</span>
+<span class="sd">                - `routine` (`str`):</span>
+<span class="sd">                   one of _&#39;howard&#39;_ or _&#39;felbo&#39;_</span>
+<span class="sd">                - `deeptabular_gradual` (`bool`):</span>
+<span class="sd">                   boolean indicating if the `deeptabular` component will be fine tuned gradually</span>
+<span class="sd">                - `deeptabular_layers` (`Optional[Union[List[nn.Module], List[List[nn.Module]]]]`):</span>
+<span class="sd">                   List of pytorch modules indicating the layers of the</span>
+<span class="sd">                   `deeptabular` that will be fine tuned</span>
+<span class="sd">                - `deeptabular_max_lr` (`Union[float, List[float]]`):</span>
+<span class="sd">                   max lr for the `deeptabular` componet during fine tuning</span>
+<span class="sd">                - `deeptext_gradual` (`bool`):</span>
+<span class="sd">                   same as `deeptabular_gradual` but for the `deeptext` component</span>
+<span class="sd">                - `deeptext_layers` (`Optional[Union[List[nn.Module], List[List[nn.Module]]]]`):</span>
+<span class="sd">                   same as `deeptabular_gradual` but for the `deeptext` component.</span>
+<span class="sd">                   If there are multiple text columns/models, this should be a list of lists</span>
+<span class="sd">                - `deeptext_max_lr` (`Union[float, List[float]]`):</span>
+<span class="sd">                   same as `deeptabular_gradual` but for the `deeptext` component</span>
+<span class="sd">                   If there are multiple text columns/models, this should be a list of floats</span>
+<span class="sd">                - `deepimage_gradual` (`bool`):</span>
+<span class="sd">                   same as `deeptext_layers` but for the `deepimage` component</span>
+<span class="sd">                - `deepimage_layers` (`Optional[Union[List[nn.Module], List[List[nn.Module]]]]`):</span>
+<span class="sd">                   same as `deeptext_layers` but for the `deepimage` component</span>
+<span class="sd">                - `deepimage_max_lr` (`Union[float, List[float]]`):</span>
+<span class="sd">                    same as `deeptext_layers` but for the `deepimage` component</span>
+
+<span class="sd">        Examples</span>
+<span class="sd">        --------</span>
+
+<span class="sd">        For a series of comprehensive examples on how to use the `fit` method, please see the</span>
+<span class="sd">        [Examples](https://github.com/jrzaurin/pytorch-widedeep/tree/master/examples)</span>
+<span class="sd">        folder in the repo</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">dataloader_args</span><span class="p">,</span> <span class="n">finetune_args</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_extract_kwargs</span><span class="p">(</span><span class="n">kwargs</span><span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">batch_size</span> <span class="o">=</span> <span class="n">batch_size</span>
+        <span class="n">train_set</span><span class="p">,</span> <span class="n">eval_set</span> <span class="o">=</span> <span class="n">wd_train_val_split</span><span class="p">(</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">seed</span><span class="p">,</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">method</span><span class="p">,</span>  <span class="c1"># type: ignore</span>
+            <span class="n">X_wide</span><span class="p">,</span>
+            <span class="n">X_tab</span><span class="p">,</span>
+            <span class="n">X_text</span><span class="p">,</span>
+            <span class="n">X_img</span><span class="p">,</span>
+            <span class="n">X_train</span><span class="p">,</span>
+            <span class="n">X_val</span><span class="p">,</span>
+            <span class="n">val_split</span><span class="p">,</span>
+            <span class="n">target</span><span class="p">,</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">transforms</span><span class="p">,</span>
+        <span class="p">)</span>
+        <span class="n">train_loader</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_set_dataloader</span><span class="p">(</span>
+            <span class="n">train_dataloader</span><span class="p">,</span> <span class="n">train_set</span><span class="p">,</span> <span class="n">batch_size</span><span class="p">,</span> <span class="n">dataloader_args</span>
+        <span class="p">)</span>
+        <span class="n">eval_loader</span> <span class="o">=</span> <span class="p">(</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">_set_dataloader</span><span class="p">(</span><span class="n">eval_dataloader</span><span class="p">,</span> <span class="n">eval_set</span><span class="p">,</span> <span class="n">batch_size</span><span class="p">,</span> <span class="n">dataloader_args</span><span class="p">)</span>
+            <span class="k">if</span> <span class="n">eval_set</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span>
+            <span class="k">else</span> <span class="kc">None</span>
+        <span class="p">)</span>
+
+        <span class="k">if</span> <span class="n">finetune</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">with_finetuning</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">_do_finetune</span><span class="p">(</span><span class="n">train_loader</span><span class="p">,</span> <span class="o">**</span><span class="n">finetune_args</span><span class="p">)</span>
+            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">verbose</span><span class="p">:</span>
+                <span class="nb">print</span><span class="p">(</span>
+                    <span class="s2">&quot;Fine-tuning (or warmup) of individual components completed. &quot;</span>
+                    <span class="s2">&quot;Training the whole model for </span><span class="si">{}</span><span class="s2"> epochs&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">n_epochs</span><span class="p">)</span>
+                <span class="p">)</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">with_finetuning</span> <span class="o">=</span> <span class="kc">False</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">callback_container</span><span class="o">.</span><span class="n">on_train_begin</span><span class="p">(</span>
+            <span class="p">{</span>
+                <span class="s2">&quot;batch_size&quot;</span><span class="p">:</span> <span class="n">batch_size</span><span class="p">,</span>
+                <span class="s2">&quot;train_steps&quot;</span><span class="p">:</span> <span class="nb">len</span><span class="p">(</span><span class="n">train_loader</span><span class="p">),</span>
+                <span class="s2">&quot;n_epochs&quot;</span><span class="p">:</span> <span class="n">n_epochs</span><span class="p">,</span>
+            <span class="p">}</span>
+        <span class="p">)</span>
+        <span class="k">for</span> <span class="n">epoch</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">n_epochs</span><span class="p">):</span>
+            <span class="n">epoch_logs</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_train_epoch</span><span class="p">(</span><span class="n">train_loader</span><span class="p">,</span> <span class="n">epoch</span><span class="p">)</span>
+            <span class="k">if</span> <span class="n">eval_loader</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">and</span> <span class="n">epoch</span> <span class="o">%</span> <span class="n">validation_freq</span> <span class="o">==</span> <span class="p">(</span>
+                <span class="n">validation_freq</span> <span class="o">-</span> <span class="mi">1</span>
+            <span class="p">):</span>
+                <span class="n">epoch_logs</span><span class="p">,</span> <span class="n">on_epoch_end_metric</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_eval_epoch</span><span class="p">(</span>
+                    <span class="n">eval_loader</span><span class="p">,</span> <span class="n">epoch_logs</span>
+                <span class="p">)</span>
+            <span class="k">else</span><span class="p">:</span>
+                <span class="n">on_epoch_end_metric</span> <span class="o">=</span> <span class="kc">None</span>
+                <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">reducelronplateau</span><span class="p">:</span>
+                    <span class="k">raise</span> <span class="ne">NotImplementedError</span><span class="p">(</span>
+                        <span class="s2">&quot;ReduceLROnPlateau scheduler can be used only with validation data.&quot;</span>
+                    <span class="p">)</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">callback_container</span><span class="o">.</span><span class="n">on_epoch_end</span><span class="p">(</span><span class="n">epoch</span><span class="p">,</span> <span class="n">epoch_logs</span><span class="p">,</span> <span class="n">on_epoch_end_metric</span><span class="p">)</span>
+            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">early_stop</span><span class="p">:</span>
+                <span class="c1"># self.callback_container.on_train_end(epoch_logs)</span>
+                <span class="k">break</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">callback_container</span><span class="o">.</span><span class="n">on_train_end</span><span class="p">(</span><span class="n">epoch_logs</span><span class="p">)</span>
+        <span class="k">if</span> <span class="n">feature_importance_sample_size</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">feature_importance</span> <span class="o">=</span> <span class="n">FeatureImportance</span><span class="p">(</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">device</span><span class="p">,</span> <span class="n">feature_importance_sample_size</span>
+            <span class="p">)</span><span class="o">.</span><span class="n">feature_importance</span><span class="p">(</span><span class="n">train_loader</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">model</span><span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">_restore_best_weights</span><span class="p">()</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">model</span><span class="o">.</span><span class="n">train</span><span class="p">()</span>
+
+    <span class="k">def</span> <span class="nf">predict</span><span class="p">(</span>  <span class="c1"># type: ignore[override, return]</span>
+        <span class="bp">self</span><span class="p">,</span>
+        <span class="n">X_wide</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">X_tab</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Union</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">]]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">X_text</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Union</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">]]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">X_img</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Union</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">]]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">X_test</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Union</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">]]]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">batch_size</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+    <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">:</span>
+<span class="w">        </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Returns the predictions</span>
+
+<span class="sd">        The input datasets can be passed either directly via numpy arrays</span>
+<span class="sd">        (`X_wide`, `X_tab`, `X_text` or `X_img`) or alternatively, in</span>
+<span class="sd">        a dictionary (`X_test`)</span>
+
+
+<span class="sd">        Parameters</span>
+<span class="sd">        ----------</span>
+<span class="sd">        X_wide: np.ndarray, Optional. default=None</span>
+<span class="sd">            Input for the `wide` model component.</span>
+<span class="sd">            See `pytorch_widedeep.preprocessing.WidePreprocessor`</span>
+<span class="sd">        X_tab: np.ndarray or List[np.ndarray], Optional. default=None</span>
+<span class="sd">            Input for the `deeptabular` model component.</span>
+<span class="sd">            See `pytorch_widedeep.preprocessing.TabPreprocessor`</span>
+<span class="sd">        X_text: np.ndarray or List[np.ndarray], Optional. default=None</span>
+<span class="sd">            Input for the `deeptext` model component.</span>
+<span class="sd">            See `pytorch_widedeep.preprocessing.TextPreprocessor`</span>
+<span class="sd">        X_img: np.ndarray or List[np.ndarray], Optional. default=None</span>
+<span class="sd">            Input for the `deepimage` model component.</span>
+<span class="sd">            See `pytorch_widedeep.preprocessing.ImagePreprocessor`</span>
+<span class="sd">        X_test: Dict, Optional. default=None</span>
+<span class="sd">            The test dataset can also be passed in a dictionary. Keys are</span>
+<span class="sd">            `X_wide`, _&#39;X_tab&#39;_, _&#39;X_text&#39;_, _&#39;X_img&#39;_ and _&#39;target&#39;_. Values</span>
+<span class="sd">            are the corresponding matrices.</span>
+<span class="sd">        batch_size: int, default = 256</span>
+<span class="sd">            If a trainer is used to predict after having trained a model, the</span>
+<span class="sd">            `batch_size` needs to be defined as it will not be defined as</span>
+<span class="sd">            the `Trainer` is instantiated</span>
+
+<span class="sd">        Returns</span>
+<span class="sd">        -------</span>
+<span class="sd">        np.ndarray:</span>
+<span class="sd">            array with the predictions</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">preds_l</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_predict</span><span class="p">(</span><span class="n">X_wide</span><span class="p">,</span> <span class="n">X_tab</span><span class="p">,</span> <span class="n">X_text</span><span class="p">,</span> <span class="n">X_img</span><span class="p">,</span> <span class="n">X_test</span><span class="p">,</span> <span class="n">batch_size</span><span class="p">)</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">method</span> <span class="o">==</span> <span class="s2">&quot;regression&quot;</span><span class="p">:</span>
+            <span class="k">return</span> <span class="n">np</span><span class="o">.</span><span class="n">vstack</span><span class="p">(</span><span class="n">preds_l</span><span class="p">)</span><span class="o">.</span><span class="n">squeeze</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">method</span> <span class="o">==</span> <span class="s2">&quot;binary&quot;</span><span class="p">:</span>
+            <span class="n">preds</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">vstack</span><span class="p">(</span><span class="n">preds_l</span><span class="p">)</span><span class="o">.</span><span class="n">squeeze</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span>
+            <span class="k">return</span> <span class="p">(</span><span class="n">preds</span> <span class="o">&gt;</span> <span class="mf">0.5</span><span class="p">)</span><span class="o">.</span><span class="n">astype</span><span class="p">(</span><span class="s2">&quot;int&quot;</span><span class="p">)</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">method</span> <span class="o">==</span> <span class="s2">&quot;qregression&quot;</span><span class="p">:</span>
+            <span class="k">return</span> <span class="n">np</span><span class="o">.</span><span class="n">vstack</span><span class="p">(</span><span class="n">preds_l</span><span class="p">)</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">method</span> <span class="o">==</span> <span class="s2">&quot;multiclass&quot;</span><span class="p">:</span>
+            <span class="n">preds</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">vstack</span><span class="p">(</span><span class="n">preds_l</span><span class="p">)</span>
+            <span class="k">return</span> <span class="n">np</span><span class="o">.</span><span class="n">argmax</span><span class="p">(</span><span class="n">preds</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span>  <span class="c1"># type: ignore[return-value]</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">method</span> <span class="o">==</span> <span class="s2">&quot;multitarget&quot;</span><span class="p">:</span>
+            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">loss_fn</span><span class="o">.</span><span class="vm">__class__</span><span class="o">.</span><span class="vm">__name__</span> <span class="ow">in</span> <span class="p">[</span>
+                <span class="s2">&quot;MultiTargetClassificationLoss&quot;</span><span class="p">,</span>
+                <span class="s2">&quot;MutilTargetRegressionAndClassificationLoss&quot;</span><span class="p">,</span>
+            <span class="p">]:</span>
+                <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
+                    <span class="s2">&quot;MultiTargetClassificationLoss and MutilTargetRegressionAndClassificationLoss &quot;</span>
+                    <span class="s2">&quot;are not supported by predict method. Please use predict_proba method instead.&quot;</span>
+                <span class="p">)</span>
+            <span class="k">return</span> <span class="n">np</span><span class="o">.</span><span class="n">vstack</span><span class="p">(</span><span class="n">preds_l</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">predict_uncertainty</span><span class="p">(</span>  <span class="c1"># type: ignore[return]</span>
+        <span class="bp">self</span><span class="p">,</span>
+        <span class="n">X_wide</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">X_tab</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Union</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">]]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">X_text</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Union</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">]]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">X_img</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Union</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">]]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">X_test</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Union</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">]]]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">batch_size</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">uncertainty_granularity</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1000</span><span class="p">,</span>
+    <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">:</span>
+<span class="w">        </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Returns the predicted ucnertainty of the model for the test dataset</span>
+<span class="sd">        using a Monte Carlo method during which dropout layers are activated</span>
+<span class="sd">        in the evaluation/prediction phase and each sample is predicted N</span>
+<span class="sd">        times (`uncertainty_granularity` times).</span>
+
+<span class="sd">        This is based on</span>
+<span class="sd">        [Dropout as a Bayesian Approximation: Representing</span>
+<span class="sd">        Model Uncertainty in Deep Learning](https://arxiv.org/abs/1506.02142?context=stat).</span>
+
+<span class="sd">        Parameters</span>
+<span class="sd">        ----------</span>
+<span class="sd">        X_wide: np.ndarray, Optional. default=None</span>
+<span class="sd">            Input for the `wide` model component.</span>
+<span class="sd">            See `pytorch_widedeep.preprocessing.WidePreprocessor`</span>
+<span class="sd">        X_tab: np.ndarray or List[np.ndarray], Optional. default=None</span>
+<span class="sd">            Input for the `deeptabular` model component.</span>
+<span class="sd">            See `pytorch_widedeep.preprocessing.TabPreprocessor`</span>
+<span class="sd">        X_text: np.ndarray or List[np.ndarray], Optional. default=None</span>
+<span class="sd">            Input for the `deeptext` model component.</span>
+<span class="sd">            See `pytorch_widedeep.preprocessing.TextPreprocessor`</span>
+<span class="sd">        X_img: np.ndarray or List[np.ndarray], Optional. default=None</span>
+<span class="sd">            Input for the `deepimage` model component.</span>
+<span class="sd">            See `pytorch_widedeep.preprocessing.ImagePreprocessor`</span>
+<span class="sd">        X_test: Dict, Optional. default=None</span>
+<span class="sd">            The test dataset can also be passed in a dictionary. Keys are</span>
+<span class="sd">            _&#39;X_wide&#39;_, _&#39;X_tab&#39;_, _&#39;X_text&#39;_, _&#39;X_img&#39;_ and _&#39;target&#39;_. Values</span>
+<span class="sd">            are the corresponding matrices.</span>
+<span class="sd">        batch_size: int, default = 256</span>
+<span class="sd">            If a trainer is used to predict after having trained a model, the</span>
+<span class="sd">            `batch_size` needs to be defined as it will not be defined as</span>
+<span class="sd">            the `Trainer` is instantiated</span>
+<span class="sd">        uncertainty_granularity: int default = 1000</span>
+<span class="sd">            number of times the model does prediction for each sample</span>
+
+<span class="sd">        Returns</span>
+<span class="sd">        -------</span>
+<span class="sd">        np.ndarray:</span>
+<span class="sd">            - if `method = regression`, it will return an array with `(max, min, mean, stdev)`</span>
+<span class="sd">              values for each sample.</span>
+<span class="sd">            - if `method = binary` it will return an array with</span>
+<span class="sd">              `(mean_cls_0_prob, mean_cls_1_prob, predicted_cls)` for each sample.</span>
+<span class="sd">            - if `method = multiclass` it will return an array with</span>
+<span class="sd">              `(mean_cls_0_prob, mean_cls_1_prob, mean_cls_2_prob, ... , predicted_cls)`</span>
+<span class="sd">              values for each sample.</span>
+
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">preds_l</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_predict</span><span class="p">(</span>
+            <span class="n">X_wide</span><span class="p">,</span>
+            <span class="n">X_tab</span><span class="p">,</span>
+            <span class="n">X_text</span><span class="p">,</span>
+            <span class="n">X_img</span><span class="p">,</span>
+            <span class="n">X_test</span><span class="p">,</span>
+            <span class="n">batch_size</span><span class="p">,</span>
+            <span class="n">uncertainty_granularity</span><span class="p">,</span>
+            <span class="n">uncertainty</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
+        <span class="p">)</span>
+        <span class="n">preds</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">vstack</span><span class="p">(</span><span class="n">preds_l</span><span class="p">)</span>
+        <span class="n">samples_num</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">preds</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">/</span> <span class="n">uncertainty_granularity</span><span class="p">)</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">method</span> <span class="o">==</span> <span class="s2">&quot;regression&quot;</span><span class="p">:</span>
+            <span class="n">preds</span> <span class="o">=</span> <span class="n">preds</span><span class="o">.</span><span class="n">squeeze</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span>
+            <span class="n">preds</span> <span class="o">=</span> <span class="n">preds</span><span class="o">.</span><span class="n">reshape</span><span class="p">((</span><span class="n">uncertainty_granularity</span><span class="p">,</span> <span class="n">samples_num</span><span class="p">))</span>
+            <span class="k">return</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">(</span>
+                <span class="p">(</span>
+                    <span class="n">preds</span><span class="o">.</span><span class="n">max</span><span class="p">(</span><span class="n">axis</span><span class="o">=</span><span class="mi">0</span><span class="p">),</span>
+                    <span class="n">preds</span><span class="o">.</span><span class="n">min</span><span class="p">(</span><span class="n">axis</span><span class="o">=</span><span class="mi">0</span><span class="p">),</span>
+                    <span class="n">preds</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="n">axis</span><span class="o">=</span><span class="mi">0</span><span class="p">),</span>
+                    <span class="n">preds</span><span class="o">.</span><span class="n">std</span><span class="p">(</span><span class="n">axis</span><span class="o">=</span><span class="mi">0</span><span class="p">),</span>
+                <span class="p">)</span>
+            <span class="p">)</span><span class="o">.</span><span class="n">T</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">method</span> <span class="o">==</span> <span class="s2">&quot;qregression&quot;</span><span class="p">:</span>
+            <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
+                <span class="s2">&quot;Currently predict_uncertainty is not supported for qregression method&quot;</span>
+            <span class="p">)</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">method</span> <span class="o">==</span> <span class="s2">&quot;binary&quot;</span><span class="p">:</span>
+            <span class="n">preds</span> <span class="o">=</span> <span class="n">preds</span><span class="o">.</span><span class="n">squeeze</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span>
+            <span class="n">preds</span> <span class="o">=</span> <span class="n">preds</span><span class="o">.</span><span class="n">reshape</span><span class="p">((</span><span class="n">uncertainty_granularity</span><span class="p">,</span> <span class="n">samples_num</span><span class="p">))</span>
+            <span class="n">preds</span> <span class="o">=</span> <span class="n">preds</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="n">axis</span><span class="o">=</span><span class="mi">0</span><span class="p">)</span>
+            <span class="n">probs</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">zeros</span><span class="p">([</span><span class="n">preds</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="mi">3</span><span class="p">])</span>
+            <span class="n">probs</span><span class="p">[:,</span> <span class="mi">0</span><span class="p">]</span> <span class="o">=</span> <span class="mi">1</span> <span class="o">-</span> <span class="n">preds</span>
+            <span class="n">probs</span><span class="p">[:,</span> <span class="mi">1</span><span class="p">]</span> <span class="o">=</span> <span class="n">preds</span>
+            <span class="k">return</span> <span class="n">probs</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">method</span> <span class="o">==</span> <span class="s2">&quot;multiclass&quot;</span><span class="p">:</span>
+            <span class="n">preds</span> <span class="o">=</span> <span class="n">preds</span><span class="o">.</span><span class="n">reshape</span><span class="p">(</span><span class="n">uncertainty_granularity</span><span class="p">,</span> <span class="n">samples_num</span><span class="p">,</span> <span class="n">preds</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">1</span><span class="p">])</span>
+            <span class="n">preds</span> <span class="o">=</span> <span class="n">preds</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="n">axis</span><span class="o">=</span><span class="mi">0</span><span class="p">)</span>
+            <span class="n">preds</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">hstack</span><span class="p">((</span><span class="n">preds</span><span class="p">,</span> <span class="n">np</span><span class="o">.</span><span class="n">vstack</span><span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">argmax</span><span class="p">(</span><span class="n">preds</span><span class="p">,</span> <span class="mi">1</span><span class="p">))))</span>
+            <span class="k">return</span> <span class="n">preds</span>
+
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">method</span> <span class="o">==</span> <span class="s2">&quot;multitarget&quot;</span><span class="p">:</span>
+            <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
+                <span class="s2">&quot;Currently predict_uncertainty is not supported for multitarget method&quot;</span>
+            <span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">predict_proba</span><span class="p">(</span>  <span class="c1"># type: ignore[override, return]  # noqa: C901</span>
+        <span class="bp">self</span><span class="p">,</span>
+        <span class="n">X_wide</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">X_tab</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Union</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">]]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">X_text</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Union</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">]]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">X_img</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Union</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">]]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">X_test</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Union</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">]]]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">batch_size</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+    <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">:</span>
+<span class="w">        </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Returns the predicted probabilities for the test dataset for  binary</span>
+<span class="sd">        and multiclass methods</span>
+
+<span class="sd">        The input datasets can be passed either directly via numpy arrays</span>
+<span class="sd">        (`X_wide`, `X_tab`, `X_text` or `X_img`) or alternatively, in</span>
+<span class="sd">        a dictionary (`X_test`)</span>
+
+<span class="sd">        Parameters</span>
+<span class="sd">        ----------</span>
+<span class="sd">        X_wide: np.ndarray, Optional. default=None</span>
+<span class="sd">            Input for the `wide` model component.</span>
+<span class="sd">            See `pytorch_widedeep.preprocessing.WidePreprocessor`</span>
+<span class="sd">        X_tab: np.ndarray or List[np.ndarray], Optional. default=None</span>
+<span class="sd">            Input for the `deeptabular` model component.</span>
+<span class="sd">            See `pytorch_widedeep.preprocessing.TabPreprocessor`</span>
+<span class="sd">        X_text: np.ndarray or List[np.ndarray], Optional. default=None</span>
+<span class="sd">            Input for the `deeptext` model component.</span>
+<span class="sd">            See `pytorch_widedeep.preprocessing.TextPreprocessor`</span>
+<span class="sd">        X_img: np.ndarray or List[np.ndarray], Optional. default=None</span>
+<span class="sd">            Input for the `deepimage` model component.</span>
+<span class="sd">            See `pytorch_widedeep.preprocessing.ImagePreprocessor`</span>
+<span class="sd">        X_test: Dict, Optional. default=None</span>
+<span class="sd">            The test dataset can also be passed in a dictionary. Keys are</span>
+<span class="sd">            `X_wide`, _&#39;X_tab&#39;_, _&#39;X_text&#39;_, _&#39;X_img&#39;_ and _&#39;target&#39;_. Values</span>
+<span class="sd">            are the corresponding matrices.</span>
+<span class="sd">        batch_size: int, default = 256</span>
+<span class="sd">            If a trainer is used to predict after having trained a model, the</span>
+<span class="sd">            `batch_size` needs to be defined as it will not be defined as</span>
+<span class="sd">            the `Trainer` is instantiated</span>
+
+<span class="sd">        Returns</span>
+<span class="sd">        -------</span>
+<span class="sd">        np.ndarray</span>
+<span class="sd">            array with the probabilities per class</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+
+        <span class="n">preds_l</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_predict</span><span class="p">(</span><span class="n">X_wide</span><span class="p">,</span> <span class="n">X_tab</span><span class="p">,</span> <span class="n">X_text</span><span class="p">,</span> <span class="n">X_img</span><span class="p">,</span> <span class="n">X_test</span><span class="p">,</span> <span class="n">batch_size</span><span class="p">)</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">method</span> <span class="o">==</span> <span class="s2">&quot;binary&quot;</span><span class="p">:</span>
+            <span class="n">preds</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">vstack</span><span class="p">(</span><span class="n">preds_l</span><span class="p">)</span><span class="o">.</span><span class="n">squeeze</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span>
+            <span class="n">probs</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">zeros</span><span class="p">([</span><span class="n">preds</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="mi">2</span><span class="p">])</span>
+            <span class="n">probs</span><span class="p">[:,</span> <span class="mi">0</span><span class="p">]</span> <span class="o">=</span> <span class="mi">1</span> <span class="o">-</span> <span class="n">preds</span>
+            <span class="n">probs</span><span class="p">[:,</span> <span class="mi">1</span><span class="p">]</span> <span class="o">=</span> <span class="n">preds</span>
+            <span class="k">return</span> <span class="n">probs</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">method</span> <span class="o">==</span> <span class="s2">&quot;multiclass&quot;</span><span class="p">:</span>
+            <span class="k">return</span> <span class="n">np</span><span class="o">.</span><span class="n">vstack</span><span class="p">(</span><span class="n">preds_l</span><span class="p">)</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">method</span> <span class="o">==</span> <span class="s2">&quot;multitarget&quot;</span><span class="p">:</span>
+            <span class="k">return</span> <span class="n">np</span><span class="o">.</span><span class="n">vstack</span><span class="p">(</span><span class="n">preds_l</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">explain</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">X_tab</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">,</span> <span class="n">save_step_masks</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">):</span>
+        <span class="c1"># TO DO: Add docs to this, to the feat imp parameter and the all</span>
+        <span class="c1"># related classes</span>
+        <span class="n">explainer</span> <span class="o">=</span> <span class="n">Explainer</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">device</span><span class="p">)</span>
+
+        <span class="n">res</span> <span class="o">=</span> <span class="n">explainer</span><span class="o">.</span><span class="n">explain</span><span class="p">(</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">model</span><span class="p">,</span> <span class="n">X_tab</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">num_workers</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">batch_size</span><span class="p">,</span> <span class="n">save_step_masks</span>
+        <span class="p">)</span>
+
+        <span class="k">return</span> <span class="n">res</span>
+
+    <span class="k">def</span> <span class="nf">save</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span>
+        <span class="n">path</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
+        <span class="n">save_state_dict</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
+        <span class="n">save_optimizer</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
+        <span class="n">model_filename</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;wd_model.pt&quot;</span><span class="p">,</span>
+    <span class="p">):</span>
+<span class="w">        </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Saves the model, training and evaluation history, and the</span>
+<span class="sd">        `feature_importance` attribute (if the `deeptabular` component is a</span>
+<span class="sd">        Tabnet model) to disk</span>
+
+<span class="sd">        The `Trainer` class is built so that it &#39;just&#39; trains a model. With</span>
+<span class="sd">        that in mind, all the torch related parameters (such as optimizers,</span>
+<span class="sd">        learning rate schedulers, initializers, etc) have to be defined</span>
+<span class="sd">        externally and then passed to the `Trainer`. As a result, the</span>
+<span class="sd">        `Trainer` does not generate any attribute or additional data</span>
+<span class="sd">        products that need to be saved other than the `model` object itself,</span>
+<span class="sd">        which can be saved as any other torch model (e.g. `torch.save(model,</span>
+<span class="sd">        path)`).</span>
+
+<span class="sd">        The exception is Tabnet. If the `deeptabular` component is a Tabnet</span>
+<span class="sd">        model, an attribute (a dict) called `feature_importance` will be</span>
+<span class="sd">        created at the end of the training process. Therefore, a `save`</span>
+<span class="sd">        method was created that will save the feature importance dictionary</span>
+<span class="sd">        to a json file and, since we are here, the model weights, training</span>
+<span class="sd">        history and learning rate history.</span>
+
+<span class="sd">        Parameters</span>
+<span class="sd">        ----------</span>
+<span class="sd">        path: str</span>
+<span class="sd">            path to the directory where the model and the feature importance</span>
+<span class="sd">            attribute will be saved.</span>
+<span class="sd">        save_state_dict: bool, default = False</span>
+<span class="sd">            Boolean indicating whether to save directly the model</span>
+<span class="sd">            (and optimizer) or the model&#39;s (and optimizer&#39;s) state</span>
+<span class="sd">            dictionary</span>
+<span class="sd">        save_optimizer: bool, default = False</span>
+<span class="sd">            Boolean indicating whether to save the optimizer</span>
+<span class="sd">        model_filename: str, Optional, default = &quot;wd_model.pt&quot;</span>
+<span class="sd">            filename where the model weights will be store</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">_save_history</span><span class="p">(</span><span class="n">path</span><span class="p">)</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">_save_model_and_optimizer</span><span class="p">(</span>
+            <span class="n">path</span><span class="p">,</span> <span class="n">save_state_dict</span><span class="p">,</span> <span class="n">save_optimizer</span><span class="p">,</span> <span class="n">model_filename</span>
+        <span class="p">)</span>
+
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">model</span><span class="o">.</span><span class="n">is_tabnet</span><span class="p">:</span>
+            <span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">Path</span><span class="p">(</span><span class="n">path</span><span class="p">)</span> <span class="o">/</span> <span class="s2">&quot;feature_importance.json&quot;</span><span class="p">,</span> <span class="s2">&quot;w&quot;</span><span class="p">)</span> <span class="k">as</span> <span class="n">fi</span><span class="p">:</span>
+                <span class="n">json</span><span class="o">.</span><span class="n">dump</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">feature_importance</span><span class="p">,</span> <span class="n">fi</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">_set_dataloader</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span>
+        <span class="n">dataloader</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">CustomDataLoader</span><span class="p">],</span>
+        <span class="n">dataset</span><span class="p">:</span> <span class="n">WideDeepDataset</span><span class="p">,</span>
+        <span class="n">batch_size</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span>
+        <span class="n">dataloader_args</span><span class="p">:</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">],</span>
+    <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">DataLoader</span> <span class="o">|</span> <span class="n">CustomDataLoader</span><span class="p">:</span>
+        <span class="k">if</span> <span class="n">dataloader</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="n">dataloader</span><span class="o">.</span><span class="n">set_dataset</span><span class="p">(</span><span class="n">dataset</span><span class="p">)</span>
+            <span class="k">return</span> <span class="n">dataloader</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="c1"># var name &#39;loader&#39; to avoid reassigment and type errors</span>
+            <span class="n">loader</span> <span class="o">=</span> <span class="n">DataLoader</span><span class="p">(</span>
+                <span class="n">dataset</span><span class="o">=</span><span class="n">dataset</span><span class="p">,</span>
+                <span class="n">batch_size</span><span class="o">=</span><span class="n">batch_size</span><span class="p">,</span>
+                <span class="n">num_workers</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">num_workers</span><span class="p">,</span>
+                <span class="o">**</span><span class="n">dataloader_args</span><span class="p">,</span>
+            <span class="p">)</span>
+        <span class="k">return</span> <span class="n">loader</span>
+
+    <span class="nd">@alias</span><span class="p">(</span><span class="s2">&quot;n_epochs&quot;</span><span class="p">,</span> <span class="p">[</span><span class="s2">&quot;finetune_epochs&quot;</span><span class="p">,</span> <span class="s2">&quot;warmup_epochs&quot;</span><span class="p">])</span>
+    <span class="nd">@alias</span><span class="p">(</span><span class="s2">&quot;max_lr&quot;</span><span class="p">,</span> <span class="p">[</span><span class="s2">&quot;finetune_max_lr&quot;</span><span class="p">,</span> <span class="s2">&quot;warmup_max_lr&quot;</span><span class="p">])</span>
+    <span class="k">def</span> <span class="nf">_do_finetune</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span>
+        <span class="n">loader</span><span class="p">:</span> <span class="n">DataLoader</span><span class="p">,</span>
+        <span class="n">n_epochs</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">5</span><span class="p">,</span>
+        <span class="n">max_lr</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">0.01</span><span class="p">,</span>
+        <span class="n">routine</span><span class="p">:</span> <span class="n">Literal</span><span class="p">[</span><span class="s2">&quot;howard&quot;</span><span class="p">,</span> <span class="s2">&quot;felbo&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="s2">&quot;howard&quot;</span><span class="p">,</span>
+        <span class="n">deeptabular_gradual</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
+        <span class="n">deeptabular_layers</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span>
+            <span class="n">Union</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="n">nn</span><span class="o">.</span><span class="n">Module</span><span class="p">],</span> <span class="n">List</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="n">nn</span><span class="o">.</span><span class="n">Module</span><span class="p">]]]</span>
+        <span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">deeptabular_max_lr</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="nb">float</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="nb">float</span><span class="p">]]</span> <span class="o">=</span> <span class="mf">0.01</span><span class="p">,</span>
+        <span class="n">deeptext_gradual</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
+        <span class="n">deeptext_layers</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Union</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="n">nn</span><span class="o">.</span><span class="n">Module</span><span class="p">],</span> <span class="n">List</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="n">nn</span><span class="o">.</span><span class="n">Module</span><span class="p">]]]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">deeptext_max_lr</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="nb">float</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="nb">float</span><span class="p">]]</span> <span class="o">=</span> <span class="mf">0.01</span><span class="p">,</span>
+        <span class="n">deepimage_gradual</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
+        <span class="n">deepimage_layers</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span>
+            <span class="n">Union</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="n">nn</span><span class="o">.</span><span class="n">Module</span><span class="p">],</span> <span class="n">List</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="n">nn</span><span class="o">.</span><span class="n">Module</span><span class="p">]]]</span>
+        <span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">deepimage_max_lr</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="nb">float</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="nb">float</span><span class="p">]]</span> <span class="o">=</span> <span class="mf">0.01</span><span class="p">,</span>
+    <span class="p">):</span>
+<span class="w">        </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Simple wrap-up to individually fine-tune model components</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">model</span><span class="o">.</span><span class="n">deephead</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
+                <span class="s2">&quot;Currently warming up is only supported without a fully connected &#39;DeepHead&#39;&quot;</span>
+            <span class="p">)</span>
+
+        <span class="n">finetuner</span> <span class="o">=</span> <span class="n">FineTune</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">loss_fn</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">metric</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">method</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">verbose</span><span class="p">)</span>  <span class="c1"># type: ignore[arg-type]</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">model</span><span class="o">.</span><span class="n">wide</span><span class="p">:</span>
+            <span class="n">finetuner</span><span class="o">.</span><span class="n">finetune_all</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">model</span><span class="o">.</span><span class="n">wide</span><span class="p">,</span> <span class="s2">&quot;wide&quot;</span><span class="p">,</span> <span class="n">loader</span><span class="p">,</span> <span class="n">n_epochs</span><span class="p">,</span> <span class="n">max_lr</span><span class="p">)</span>
+
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">model</span><span class="o">.</span><span class="n">deeptabular</span><span class="p">:</span>
+            <span class="k">if</span> <span class="n">deeptabular_gradual</span><span class="p">:</span>
+                <span class="k">assert</span> <span class="p">(</span>
+                    <span class="n">deeptabular_layers</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span>
+                <span class="p">),</span> <span class="s2">&quot;deeptabular_layers must be passed if deeptabular_gradual=True&quot;</span>
+                <span class="n">finetuner</span><span class="o">.</span><span class="n">finetune_gradual</span><span class="p">(</span>
+                    <span class="bp">self</span><span class="o">.</span><span class="n">model</span><span class="o">.</span><span class="n">deeptabular</span><span class="p">,</span>
+                    <span class="s2">&quot;deeptabular&quot;</span><span class="p">,</span>
+                    <span class="n">loader</span><span class="p">,</span>
+                    <span class="n">deeptabular_max_lr</span><span class="p">,</span>
+                    <span class="n">deeptabular_layers</span><span class="p">,</span>
+                    <span class="n">routine</span><span class="p">,</span>
+                <span class="p">)</span>
+            <span class="k">else</span><span class="p">:</span>
+                <span class="n">finetuner</span><span class="o">.</span><span class="n">finetune_all</span><span class="p">(</span>
+                    <span class="bp">self</span><span class="o">.</span><span class="n">model</span><span class="o">.</span><span class="n">deeptabular</span><span class="p">,</span> <span class="s2">&quot;deeptabular&quot;</span><span class="p">,</span> <span class="n">loader</span><span class="p">,</span> <span class="n">n_epochs</span><span class="p">,</span> <span class="n">max_lr</span>
+                <span class="p">)</span>
+
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">model</span><span class="o">.</span><span class="n">deeptext</span><span class="p">:</span>
+            <span class="k">if</span> <span class="n">deeptext_gradual</span><span class="p">:</span>
+                <span class="k">assert</span> <span class="p">(</span>
+                    <span class="n">deeptext_layers</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span>
+                <span class="p">),</span> <span class="s2">&quot;deeptext_layers must be passed if deeptext_gradual=True&quot;</span>
+                <span class="n">finetuner</span><span class="o">.</span><span class="n">finetune_gradual</span><span class="p">(</span>
+                    <span class="bp">self</span><span class="o">.</span><span class="n">model</span><span class="o">.</span><span class="n">deeptext</span><span class="p">,</span>
+                    <span class="s2">&quot;deeptext&quot;</span><span class="p">,</span>
+                    <span class="n">loader</span><span class="p">,</span>
+                    <span class="n">deeptext_max_lr</span><span class="p">,</span>
+                    <span class="n">deeptext_layers</span><span class="p">,</span>
+                    <span class="n">routine</span><span class="p">,</span>
+                <span class="p">)</span>
+            <span class="k">else</span><span class="p">:</span>
+                <span class="n">finetuner</span><span class="o">.</span><span class="n">finetune_all</span><span class="p">(</span>
+                    <span class="bp">self</span><span class="o">.</span><span class="n">model</span><span class="o">.</span><span class="n">deeptext</span><span class="p">,</span> <span class="s2">&quot;deeptext&quot;</span><span class="p">,</span> <span class="n">loader</span><span class="p">,</span> <span class="n">n_epochs</span><span class="p">,</span> <span class="n">max_lr</span>
+                <span class="p">)</span>
+
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">model</span><span class="o">.</span><span class="n">deepimage</span><span class="p">:</span>
+            <span class="k">if</span> <span class="n">deepimage_gradual</span><span class="p">:</span>
+                <span class="k">assert</span> <span class="p">(</span>
+                    <span class="n">deepimage_layers</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span>
+                <span class="p">),</span> <span class="s2">&quot;deepimage_layers must be passed if deepimage_gradual=True&quot;</span>
+                <span class="n">finetuner</span><span class="o">.</span><span class="n">finetune_gradual</span><span class="p">(</span>
+                    <span class="bp">self</span><span class="o">.</span><span class="n">model</span><span class="o">.</span><span class="n">deepimage</span><span class="p">,</span>
+                    <span class="s2">&quot;deepimage&quot;</span><span class="p">,</span>
+                    <span class="n">loader</span><span class="p">,</span>
+                    <span class="n">deepimage_max_lr</span><span class="p">,</span>
+                    <span class="n">deepimage_layers</span><span class="p">,</span>
+                    <span class="n">routine</span><span class="p">,</span>
+                <span class="p">)</span>
+            <span class="k">else</span><span class="p">:</span>
+                <span class="n">finetuner</span><span class="o">.</span><span class="n">finetune_all</span><span class="p">(</span>
+                    <span class="bp">self</span><span class="o">.</span><span class="n">model</span><span class="o">.</span><span class="n">deepimage</span><span class="p">,</span> <span class="s2">&quot;deepimage&quot;</span><span class="p">,</span> <span class="n">loader</span><span class="p">,</span> <span class="n">n_epochs</span><span class="p">,</span> <span class="n">max_lr</span>
+                <span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">_train_epoch</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span>
+        <span class="n">train_loader</span><span class="p">:</span> <span class="n">DataLoader</span><span class="p">,</span>
+        <span class="n">epoch</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span>
+    <span class="p">):</span>
+        <span class="n">epoch_logs</span><span class="p">:</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="p">{}</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">callback_container</span><span class="o">.</span><span class="n">on_epoch_begin</span><span class="p">(</span><span class="n">epoch</span><span class="p">,</span> <span class="n">logs</span><span class="o">=</span><span class="n">epoch_logs</span><span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">train_running_loss</span> <span class="o">=</span> <span class="mf">0.0</span>
+
+        <span class="n">train_steps</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">train_loader</span><span class="p">)</span>
+        <span class="k">with</span> <span class="n">trange</span><span class="p">(</span><span class="n">train_steps</span><span class="p">,</span> <span class="n">disable</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">verbose</span> <span class="o">!=</span> <span class="mi">1</span><span class="p">)</span> <span class="k">as</span> <span class="n">t</span><span class="p">:</span>
+            <span class="k">for</span> <span class="n">batch_idx</span><span class="p">,</span> <span class="p">(</span><span class="n">data</span><span class="p">,</span> <span class="n">targett</span><span class="p">)</span> <span class="ow">in</span> <span class="nb">zip</span><span class="p">(</span><span class="n">t</span><span class="p">,</span> <span class="n">train_loader</span><span class="p">):</span>
+                <span class="n">t</span><span class="o">.</span><span class="n">set_description</span><span class="p">(</span><span class="s2">&quot;epoch </span><span class="si">%i</span><span class="s2">&quot;</span> <span class="o">%</span> <span class="p">(</span><span class="n">epoch</span> <span class="o">+</span> <span class="mi">1</span><span class="p">))</span>
+                <span class="n">train_score</span><span class="p">,</span> <span class="n">train_loss</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_train_step</span><span class="p">(</span><span class="n">data</span><span class="p">,</span> <span class="n">targett</span><span class="p">,</span> <span class="n">batch_idx</span><span class="p">)</span>
+                <span class="n">print_loss_and_metric</span><span class="p">(</span><span class="n">t</span><span class="p">,</span> <span class="n">train_loss</span><span class="p">,</span> <span class="n">train_score</span><span class="p">)</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">callback_container</span><span class="o">.</span><span class="n">on_batch_end</span><span class="p">(</span><span class="n">batch</span><span class="o">=</span><span class="n">batch_idx</span><span class="p">)</span>
+
+        <span class="n">epoch_logs</span> <span class="o">=</span> <span class="n">save_epoch_logs</span><span class="p">(</span><span class="n">epoch_logs</span><span class="p">,</span> <span class="n">train_loss</span><span class="p">,</span> <span class="n">train_score</span><span class="p">,</span> <span class="s2">&quot;train&quot;</span><span class="p">)</span>
+
+        <span class="k">return</span> <span class="n">epoch_logs</span>
+
+    <span class="k">def</span> <span class="nf">_train_step</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span>
+        <span class="n">data</span><span class="p">:</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Union</span><span class="p">[</span><span class="n">Tensor</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="n">Tensor</span><span class="p">]]],</span>
+        <span class="n">target</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">,</span>
+        <span class="n">batch_idx</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span>
+    <span class="p">):</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">model</span><span class="o">.</span><span class="n">train</span><span class="p">()</span>
+
+        <span class="n">X</span><span class="p">:</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Union</span><span class="p">[</span><span class="n">Tensor</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="n">Tensor</span><span class="p">]]]</span> <span class="o">=</span> <span class="p">{}</span>
+        <span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="n">data</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
+            <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">v</span><span class="p">,</span> <span class="nb">list</span><span class="p">):</span>
+                <span class="n">X</span><span class="p">[</span><span class="n">k</span><span class="p">]</span> <span class="o">=</span> <span class="p">[</span><span class="n">to_device</span><span class="p">(</span><span class="n">i</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">device</span><span class="p">)</span> <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="n">v</span><span class="p">]</span>
+            <span class="k">else</span><span class="p">:</span>
+                <span class="n">X</span><span class="p">[</span><span class="n">k</span><span class="p">]</span> <span class="o">=</span> <span class="n">to_device</span><span class="p">(</span><span class="n">v</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">device</span><span class="p">)</span>
+        <span class="n">y</span> <span class="o">=</span> <span class="p">(</span>
+            <span class="n">target</span><span class="o">.</span><span class="n">view</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span><span class="o">.</span><span class="n">float</span><span class="p">()</span>
+            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">method</span> <span class="ow">not</span> <span class="ow">in</span> <span class="p">[</span><span class="s2">&quot;multiclass&quot;</span><span class="p">,</span> <span class="s2">&quot;qregression&quot;</span><span class="p">,</span> <span class="s2">&quot;multitarget&quot;</span><span class="p">]</span>
+            <span class="k">else</span> <span class="n">target</span>
+        <span class="p">)</span>
+        <span class="n">y</span> <span class="o">=</span> <span class="n">to_device</span><span class="p">(</span><span class="n">y</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">device</span><span class="p">)</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">optimizer</span><span class="o">.</span><span class="n">zero_grad</span><span class="p">()</span>
+
+        <span class="n">y_pred</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">model</span><span class="p">(</span><span class="n">X</span><span class="p">)</span>
+
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">model</span><span class="o">.</span><span class="n">is_tabnet</span><span class="p">:</span>
+            <span class="n">loss</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">loss_fn</span><span class="p">(</span><span class="n">y_pred</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="n">y</span><span class="p">)</span> <span class="o">-</span> <span class="bp">self</span><span class="o">.</span><span class="n">lambda_sparse</span> <span class="o">*</span> <span class="n">y_pred</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span>
+            <span class="n">score</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_score</span><span class="p">(</span><span class="n">y_pred</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="n">y</span><span class="p">,</span> <span class="n">is_train</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="n">loss</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">loss_fn</span><span class="p">(</span><span class="n">y_pred</span><span class="p">,</span> <span class="n">y</span><span class="p">)</span>
+            <span class="n">score</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_score</span><span class="p">(</span><span class="n">y_pred</span><span class="p">,</span> <span class="n">y</span><span class="p">,</span> <span class="n">is_train</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
+
+        <span class="n">loss</span><span class="o">.</span><span class="n">backward</span><span class="p">()</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">optimizer</span><span class="o">.</span><span class="n">step</span><span class="p">()</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">train_running_loss</span> <span class="o">+=</span> <span class="n">loss</span><span class="o">.</span><span class="n">item</span><span class="p">()</span>
+        <span class="n">avg_loss</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">train_running_loss</span> <span class="o">/</span> <span class="p">(</span><span class="n">batch_idx</span> <span class="o">+</span> <span class="mi">1</span><span class="p">)</span>
+
+        <span class="k">return</span> <span class="n">score</span><span class="p">,</span> <span class="n">avg_loss</span>
+
+    <span class="k">def</span> <span class="nf">_eval_epoch</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span>
+        <span class="n">eval_loader</span><span class="p">:</span> <span class="n">DataLoader</span><span class="p">,</span>
+        <span class="n">epoch_logs</span><span class="p">:</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">float</span><span class="p">],</span>
+    <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tuple</span><span class="p">[</span><span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">float</span><span class="p">],</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]]:</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">callback_container</span><span class="o">.</span><span class="n">on_eval_begin</span><span class="p">()</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">valid_running_loss</span> <span class="o">=</span> <span class="mf">0.0</span>
+
+        <span class="n">eval_steps</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">eval_loader</span><span class="p">)</span>
+        <span class="k">with</span> <span class="n">trange</span><span class="p">(</span><span class="n">eval_steps</span><span class="p">,</span> <span class="n">disable</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">verbose</span> <span class="o">!=</span> <span class="mi">1</span><span class="p">)</span> <span class="k">as</span> <span class="n">v</span><span class="p">:</span>
+            <span class="k">for</span> <span class="n">i</span><span class="p">,</span> <span class="p">(</span><span class="n">data</span><span class="p">,</span> <span class="n">targett</span><span class="p">)</span> <span class="ow">in</span> <span class="nb">zip</span><span class="p">(</span><span class="n">v</span><span class="p">,</span> <span class="n">eval_loader</span><span class="p">):</span>
+                <span class="n">v</span><span class="o">.</span><span class="n">set_description</span><span class="p">(</span><span class="s2">&quot;valid&quot;</span><span class="p">)</span>
+                <span class="n">val_score</span><span class="p">,</span> <span class="n">val_loss</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_eval_step</span><span class="p">(</span><span class="n">data</span><span class="p">,</span> <span class="n">targett</span><span class="p">,</span> <span class="n">i</span><span class="p">)</span>
+                <span class="n">print_loss_and_metric</span><span class="p">(</span><span class="n">v</span><span class="p">,</span> <span class="n">val_loss</span><span class="p">,</span> <span class="n">val_score</span><span class="p">)</span>
+
+        <span class="n">epoch_logs</span> <span class="o">=</span> <span class="n">save_epoch_logs</span><span class="p">(</span><span class="n">epoch_logs</span><span class="p">,</span> <span class="n">val_loss</span><span class="p">,</span> <span class="n">val_score</span><span class="p">,</span> <span class="s2">&quot;val&quot;</span><span class="p">)</span>
+
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">reducelronplateau</span><span class="p">:</span>
+            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">reducelronplateau_criterion</span> <span class="o">==</span> <span class="s2">&quot;loss&quot;</span><span class="p">:</span>
+                <span class="n">on_epoch_end_metric</span> <span class="o">=</span> <span class="n">val_loss</span>
+            <span class="k">else</span><span class="p">:</span>
+                <span class="n">on_epoch_end_metric</span> <span class="o">=</span> <span class="n">val_score</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">reducelronplateau_criterion</span><span class="p">]</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="n">on_epoch_end_metric</span> <span class="o">=</span> <span class="kc">None</span>
+
+        <span class="k">return</span> <span class="n">epoch_logs</span><span class="p">,</span> <span class="n">on_epoch_end_metric</span>
+
+    <span class="k">def</span> <span class="nf">_eval_step</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span>
+        <span class="n">data</span><span class="p">:</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Union</span><span class="p">[</span><span class="n">Tensor</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="n">Tensor</span><span class="p">]]],</span>
+        <span class="n">target</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">,</span>
+        <span class="n">batch_idx</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span>
+    <span class="p">):</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">model</span><span class="o">.</span><span class="n">eval</span><span class="p">()</span>
+        <span class="k">with</span> <span class="n">torch</span><span class="o">.</span><span class="n">no_grad</span><span class="p">():</span>
+            <span class="n">X</span><span class="p">:</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Union</span><span class="p">[</span><span class="n">Tensor</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="n">Tensor</span><span class="p">]]]</span> <span class="o">=</span> <span class="p">{}</span>
+            <span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="n">data</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
+                <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">v</span><span class="p">,</span> <span class="nb">list</span><span class="p">):</span>
+                    <span class="n">X</span><span class="p">[</span><span class="n">k</span><span class="p">]</span> <span class="o">=</span> <span class="p">[</span><span class="n">to_device</span><span class="p">(</span><span class="n">i</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">device</span><span class="p">)</span> <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="n">v</span><span class="p">]</span>
+                <span class="k">else</span><span class="p">:</span>
+                    <span class="n">X</span><span class="p">[</span><span class="n">k</span><span class="p">]</span> <span class="o">=</span> <span class="n">to_device</span><span class="p">(</span><span class="n">v</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">device</span><span class="p">)</span>
+            <span class="n">y</span> <span class="o">=</span> <span class="p">(</span>
+                <span class="n">target</span><span class="o">.</span><span class="n">view</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span><span class="o">.</span><span class="n">float</span><span class="p">()</span>
+                <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">method</span> <span class="ow">not</span> <span class="ow">in</span> <span class="p">[</span><span class="s2">&quot;multiclass&quot;</span><span class="p">,</span> <span class="s2">&quot;qregression&quot;</span><span class="p">,</span> <span class="s2">&quot;multitarget&quot;</span><span class="p">]</span>
+                <span class="k">else</span> <span class="n">target</span>
+            <span class="p">)</span>
+            <span class="n">y</span> <span class="o">=</span> <span class="n">to_device</span><span class="p">(</span><span class="n">y</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">device</span><span class="p">)</span>
+
+            <span class="n">y_pred</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">model</span><span class="p">(</span><span class="n">X</span><span class="p">)</span>
+            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">model</span><span class="o">.</span><span class="n">is_tabnet</span><span class="p">:</span>
+                <span class="n">loss</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">loss_fn</span><span class="p">(</span><span class="n">y_pred</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="n">y</span><span class="p">)</span> <span class="o">-</span> <span class="bp">self</span><span class="o">.</span><span class="n">lambda_sparse</span> <span class="o">*</span> <span class="n">y_pred</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span>
+                <span class="n">score</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_score</span><span class="p">(</span><span class="n">y_pred</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="n">y</span><span class="p">,</span> <span class="n">is_train</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
+            <span class="k">else</span><span class="p">:</span>
+                <span class="n">score</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_score</span><span class="p">(</span><span class="n">y_pred</span><span class="p">,</span> <span class="n">y</span><span class="p">,</span> <span class="n">is_train</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
+                <span class="n">loss</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">loss_fn</span><span class="p">(</span><span class="n">y_pred</span><span class="p">,</span> <span class="n">y</span><span class="p">)</span>
+
+            <span class="bp">self</span><span class="o">.</span><span class="n">valid_running_loss</span> <span class="o">+=</span> <span class="n">loss</span><span class="o">.</span><span class="n">item</span><span class="p">()</span>
+            <span class="n">avg_loss</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">valid_running_loss</span> <span class="o">/</span> <span class="p">(</span><span class="n">batch_idx</span> <span class="o">+</span> <span class="mi">1</span><span class="p">)</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">model</span><span class="o">.</span><span class="n">train</span><span class="p">()</span>
+        <span class="k">return</span> <span class="n">score</span><span class="p">,</span> <span class="n">avg_loss</span>
+
+    <span class="k">def</span> <span class="nf">_get_score</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span> <span class="n">y_pred</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">,</span> <span class="n">y</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">,</span> <span class="n">is_train</span><span class="p">:</span> <span class="nb">bool</span>
+    <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">float</span><span class="p">]]:</span>
+
+        <span class="n">score</span> <span class="o">=</span> <span class="kc">None</span>
+        <span class="n">metric</span> <span class="o">=</span> <span class="kc">None</span>
+
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">metric</span> <span class="ow">and</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">eval_metric</span><span class="p">:</span>
+            <span class="n">metric</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">metric</span>
+        <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">metric</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">eval_metric</span><span class="p">:</span>
+            <span class="n">metric</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">metric</span> <span class="k">if</span> <span class="n">is_train</span> <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">eval_metric</span>
+        <span class="k">elif</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">metric</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">eval_metric</span><span class="p">:</span>
+            <span class="n">metric</span> <span class="o">=</span> <span class="kc">None</span> <span class="k">if</span> <span class="n">is_train</span> <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">eval_metric</span>
+
+        <span class="k">if</span> <span class="n">metric</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">method</span> <span class="o">==</span> <span class="s2">&quot;regression&quot;</span><span class="p">:</span>
+                <span class="n">score</span> <span class="o">=</span> <span class="n">metric</span><span class="p">(</span><span class="n">y_pred</span><span class="p">,</span> <span class="n">y</span><span class="p">)</span>
+            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">method</span> <span class="o">==</span> <span class="s2">&quot;binary&quot;</span><span class="p">:</span>
+                <span class="n">score</span> <span class="o">=</span> <span class="n">metric</span><span class="p">(</span><span class="n">torch</span><span class="o">.</span><span class="n">sigmoid</span><span class="p">(</span><span class="n">y_pred</span><span class="p">),</span> <span class="n">y</span><span class="p">)</span>
+            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">method</span> <span class="o">==</span> <span class="s2">&quot;qregression&quot;</span><span class="p">:</span>
+                <span class="n">score</span> <span class="o">=</span> <span class="n">metric</span><span class="p">(</span><span class="n">y_pred</span><span class="p">,</span> <span class="n">y</span><span class="p">)</span>
+            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">method</span> <span class="o">==</span> <span class="s2">&quot;multiclass&quot;</span><span class="p">:</span>
+                <span class="n">score</span> <span class="o">=</span> <span class="n">metric</span><span class="p">(</span><span class="n">F</span><span class="o">.</span><span class="n">softmax</span><span class="p">(</span><span class="n">y_pred</span><span class="p">,</span> <span class="n">dim</span><span class="o">=</span><span class="mi">1</span><span class="p">),</span> <span class="n">y</span><span class="p">)</span>
+        <span class="k">return</span> <span class="n">score</span>
+
+    <span class="k">def</span> <span class="nf">_predict</span><span class="p">(</span>  <span class="c1"># type: ignore[override, return]  # noqa: C901</span>
+        <span class="bp">self</span><span class="p">,</span>
+        <span class="n">X_wide</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">X_tab</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Union</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">]]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">X_text</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Union</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">]]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">X_img</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Union</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">]]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">X_test</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Union</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">]]]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">batch_size</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">uncertainty_granularity</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1000</span><span class="p">,</span>
+        <span class="n">uncertainty</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
+    <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">List</span><span class="p">:</span>
+<span class="w">        </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Private method to avoid code repetition in predict and</span>
+<span class="sd">        predict_proba. For parameter information, please, see the .predict()</span>
+<span class="sd">        method documentation</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="k">if</span> <span class="n">X_test</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="n">test_set</span> <span class="o">=</span> <span class="n">WideDeepDataset</span><span class="p">(</span><span class="o">**</span><span class="n">X_test</span><span class="p">)</span>  <span class="c1"># type: ignore[arg-type]</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="n">load_dict</span><span class="p">:</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Union</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">]]]</span> <span class="o">=</span> <span class="p">{}</span>
+            <span class="k">if</span> <span class="n">X_wide</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+                <span class="n">load_dict</span> <span class="o">=</span> <span class="p">{</span><span class="s2">&quot;X_wide&quot;</span><span class="p">:</span> <span class="n">X_wide</span><span class="p">}</span>
+            <span class="k">if</span> <span class="n">X_tab</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+                <span class="n">load_dict</span><span class="o">.</span><span class="n">update</span><span class="p">({</span><span class="s2">&quot;X_tab&quot;</span><span class="p">:</span> <span class="n">X_tab</span><span class="p">})</span>
+            <span class="k">if</span> <span class="n">X_text</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+                <span class="n">load_dict</span><span class="o">.</span><span class="n">update</span><span class="p">({</span><span class="s2">&quot;X_text&quot;</span><span class="p">:</span> <span class="n">X_text</span><span class="p">})</span>
+            <span class="k">if</span> <span class="n">X_img</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+                <span class="n">load_dict</span><span class="o">.</span><span class="n">update</span><span class="p">({</span><span class="s2">&quot;X_img&quot;</span><span class="p">:</span> <span class="n">X_img</span><span class="p">})</span>
+            <span class="n">test_set</span> <span class="o">=</span> <span class="n">WideDeepDataset</span><span class="p">(</span><span class="o">**</span><span class="n">load_dict</span><span class="p">)</span>  <span class="c1"># type: ignore[arg-type]</span>
+
+        <span class="k">if</span> <span class="ow">not</span> <span class="nb">hasattr</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="s2">&quot;batch_size&quot;</span><span class="p">):</span>
+            <span class="k">assert</span> <span class="n">batch_size</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">,</span> <span class="p">(</span>
+                <span class="s2">&quot;&#39;batch_size&#39; has not be previosly set in this Trainer and must be&quot;</span>
+                <span class="s2">&quot; specified via the &#39;batch_size&#39; argument in this predict call&quot;</span>
+            <span class="p">)</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">batch_size</span> <span class="o">=</span> <span class="n">batch_size</span>
+
+        <span class="n">test_loader</span> <span class="o">=</span> <span class="n">DataLoader</span><span class="p">(</span>
+            <span class="n">dataset</span><span class="o">=</span><span class="n">test_set</span><span class="p">,</span>
+            <span class="n">batch_size</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">batch_size</span><span class="p">,</span>
+            <span class="n">num_workers</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">num_workers</span><span class="p">,</span>
+            <span class="n">shuffle</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
+        <span class="p">)</span>
+        <span class="n">test_steps</span> <span class="o">=</span> <span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">test_loader</span><span class="o">.</span><span class="n">dataset</span><span class="p">)</span> <span class="o">//</span> <span class="n">test_loader</span><span class="o">.</span><span class="n">batch_size</span><span class="p">)</span> <span class="o">+</span> <span class="mi">1</span>  <span class="c1"># type: ignore[arg-type]</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">model</span><span class="o">.</span><span class="n">eval</span><span class="p">()</span>
+        <span class="n">preds_l</span> <span class="o">=</span> <span class="p">[]</span>
+
+        <span class="k">if</span> <span class="n">uncertainty</span><span class="p">:</span>
+            <span class="k">for</span> <span class="n">m</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">model</span><span class="o">.</span><span class="n">modules</span><span class="p">():</span>
+                <span class="k">if</span> <span class="n">m</span><span class="o">.</span><span class="vm">__class__</span><span class="o">.</span><span class="vm">__name__</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s2">&quot;Dropout&quot;</span><span class="p">):</span>
+                    <span class="n">m</span><span class="o">.</span><span class="n">train</span><span class="p">()</span>
+            <span class="n">prediction_iters</span> <span class="o">=</span> <span class="n">uncertainty_granularity</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="n">prediction_iters</span> <span class="o">=</span> <span class="mi">1</span>
+
+        <span class="k">with</span> <span class="n">torch</span><span class="o">.</span><span class="n">no_grad</span><span class="p">():</span>
+            <span class="k">with</span> <span class="n">trange</span><span class="p">(</span><span class="n">uncertainty_granularity</span><span class="p">,</span> <span class="n">disable</span><span class="o">=</span><span class="n">uncertainty</span> <span class="ow">is</span> <span class="kc">False</span><span class="p">)</span> <span class="k">as</span> <span class="n">t</span><span class="p">:</span>
+                <span class="k">for</span> <span class="n">_</span><span class="p">,</span> <span class="n">_</span> <span class="ow">in</span> <span class="nb">zip</span><span class="p">(</span><span class="n">t</span><span class="p">,</span> <span class="nb">range</span><span class="p">(</span><span class="n">prediction_iters</span><span class="p">)):</span>
+                    <span class="n">t</span><span class="o">.</span><span class="n">set_description</span><span class="p">(</span><span class="s2">&quot;predict_UncertaintyIter&quot;</span><span class="p">)</span>
+
+                    <span class="k">with</span> <span class="n">trange</span><span class="p">(</span>
+                        <span class="n">test_steps</span><span class="p">,</span> <span class="n">disable</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">verbose</span> <span class="o">!=</span> <span class="mi">1</span> <span class="ow">or</span> <span class="n">uncertainty</span> <span class="ow">is</span> <span class="kc">True</span>
+                    <span class="p">)</span> <span class="k">as</span> <span class="n">tt</span><span class="p">:</span>
+                        <span class="k">for</span> <span class="n">_</span><span class="p">,</span> <span class="n">data</span> <span class="ow">in</span> <span class="nb">zip</span><span class="p">(</span><span class="n">tt</span><span class="p">,</span> <span class="n">test_loader</span><span class="p">):</span>
+                            <span class="n">tt</span><span class="o">.</span><span class="n">set_description</span><span class="p">(</span><span class="s2">&quot;predict&quot;</span><span class="p">)</span>
+                            <span class="n">X</span><span class="p">:</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Union</span><span class="p">[</span><span class="n">Tensor</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="n">Tensor</span><span class="p">]]]</span> <span class="o">=</span> <span class="p">{}</span>
+                            <span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="n">data</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
+                                <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">v</span><span class="p">,</span> <span class="nb">list</span><span class="p">):</span>
+                                    <span class="n">X</span><span class="p">[</span><span class="n">k</span><span class="p">]</span> <span class="o">=</span> <span class="p">[</span><span class="n">to_device</span><span class="p">(</span><span class="n">i</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">device</span><span class="p">)</span> <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="n">v</span><span class="p">]</span>
+                                <span class="k">else</span><span class="p">:</span>
+                                    <span class="n">X</span><span class="p">[</span><span class="n">k</span><span class="p">]</span> <span class="o">=</span> <span class="n">to_device</span><span class="p">(</span><span class="n">v</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">device</span><span class="p">)</span>
+                            <span class="n">preds</span> <span class="o">=</span> <span class="p">(</span>
+                                <span class="bp">self</span><span class="o">.</span><span class="n">model</span><span class="p">(</span><span class="n">X</span><span class="p">)</span>
+                                <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">model</span><span class="o">.</span><span class="n">is_tabnet</span>
+                                <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">model</span><span class="p">(</span><span class="n">X</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span>
+                            <span class="p">)</span>
+                            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">method</span> <span class="o">==</span> <span class="s2">&quot;binary&quot;</span><span class="p">:</span>
+                                <span class="n">preds</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">sigmoid</span><span class="p">(</span><span class="n">preds</span><span class="p">)</span>
+                            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">method</span> <span class="o">==</span> <span class="s2">&quot;multiclass&quot;</span><span class="p">:</span>
+                                <span class="n">preds</span> <span class="o">=</span> <span class="n">F</span><span class="o">.</span><span class="n">softmax</span><span class="p">(</span><span class="n">preds</span><span class="p">,</span> <span class="n">dim</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span>
+                            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">method</span> <span class="o">==</span> <span class="s2">&quot;regression&quot;</span> <span class="ow">and</span> <span class="nb">isinstance</span><span class="p">(</span>
+                                <span class="bp">self</span><span class="o">.</span><span class="n">loss_fn</span><span class="p">,</span> <span class="n">ZILNLoss</span>
+                            <span class="p">):</span>
+                                <span class="n">preds</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_predict_ziln</span><span class="p">(</span><span class="n">preds</span><span class="p">)</span>
+                            <span class="n">preds</span> <span class="o">=</span> <span class="n">preds</span><span class="o">.</span><span class="n">cpu</span><span class="p">()</span><span class="o">.</span><span class="n">data</span><span class="o">.</span><span class="n">numpy</span><span class="p">()</span>
+                            <span class="n">preds_l</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">preds</span><span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">model</span><span class="o">.</span><span class="n">train</span><span class="p">()</span>
+        <span class="k">return</span> <span class="n">preds_l</span>
+
+    <span class="nd">@staticmethod</span>
+    <span class="k">def</span> <span class="nf">_predict_ziln</span><span class="p">(</span><span class="n">preds</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tensor</span><span class="p">:</span>
+        <span class="c1"># Legacy implementation. It will be removed in future versions</span>
+<span class="w">        </span><span class="sd">&quot;&quot;&quot;Calculates predicted mean of zero inflated lognormal logits.</span>
+
+<span class="sd">        Adjusted implementaion of `code</span>
+<span class="sd">        &lt;https://github.com/google/lifetime_value/blob/master/lifetime_value/zero_inflated_lognormal.py&gt;`</span>
+
+<span class="sd">        Arguments:</span>
+<span class="sd">            preds: [batch_size, 3] tensor of logits.</span>
+<span class="sd">        Returns:</span>
+<span class="sd">            ziln_preds: [batch_size, 1] tensor of predicted mean.</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">positive_probs</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">sigmoid</span><span class="p">(</span><span class="n">preds</span><span class="p">[</span><span class="o">...</span><span class="p">,</span> <span class="p">:</span><span class="mi">1</span><span class="p">])</span>
+        <span class="n">loc</span> <span class="o">=</span> <span class="n">preds</span><span class="p">[</span><span class="o">...</span><span class="p">,</span> <span class="mi">1</span><span class="p">:</span><span class="mi">2</span><span class="p">]</span>
+        <span class="n">scale</span> <span class="o">=</span> <span class="n">F</span><span class="o">.</span><span class="n">softplus</span><span class="p">(</span><span class="n">preds</span><span class="p">[</span><span class="o">...</span><span class="p">,</span> <span class="mi">2</span><span class="p">:])</span>
+        <span class="n">ziln_preds</span> <span class="o">=</span> <span class="n">positive_probs</span> <span class="o">*</span> <span class="n">torch</span><span class="o">.</span><span class="n">exp</span><span class="p">(</span><span class="n">loc</span> <span class="o">+</span> <span class="mf">0.5</span> <span class="o">*</span> <span class="n">torch</span><span class="o">.</span><span class="n">square</span><span class="p">(</span><span class="n">scale</span><span class="p">))</span>
+        <span class="k">return</span> <span class="n">ziln_preds</span>
+
+    <span class="nd">@staticmethod</span>
+    <span class="k">def</span> <span class="nf">_extract_kwargs</span><span class="p">(</span><span class="n">kwargs</span><span class="p">):</span>
+        <span class="n">dataloader_params</span> <span class="o">=</span> <span class="p">[</span>
+            <span class="s2">&quot;shuffle&quot;</span><span class="p">,</span>
+            <span class="s2">&quot;sampler&quot;</span><span class="p">,</span>
+            <span class="s2">&quot;batch_sampler&quot;</span><span class="p">,</span>
+            <span class="s2">&quot;num_workers&quot;</span><span class="p">,</span>
+            <span class="s2">&quot;collate_fn&quot;</span><span class="p">,</span>
+            <span class="s2">&quot;pin_memory&quot;</span><span class="p">,</span>
+            <span class="s2">&quot;drop_last&quot;</span><span class="p">,</span>
+            <span class="s2">&quot;timeout&quot;</span><span class="p">,</span>
+            <span class="s2">&quot;worker_init_fn&quot;</span><span class="p">,</span>
+            <span class="s2">&quot;generator&quot;</span><span class="p">,</span>
+            <span class="s2">&quot;prefetch_factor&quot;</span><span class="p">,</span>
+            <span class="s2">&quot;persistent_workers&quot;</span><span class="p">,</span>
+            <span class="s2">&quot;oversample_mul&quot;</span><span class="p">,</span>
+        <span class="p">]</span>
+        <span class="n">finetune_params</span> <span class="o">=</span> <span class="p">[</span>
+            <span class="s2">&quot;n_epochs&quot;</span><span class="p">,</span>
+            <span class="s2">&quot;finetune_epochs&quot;</span><span class="p">,</span>
+            <span class="s2">&quot;warmup_epochs&quot;</span><span class="p">,</span>
+            <span class="s2">&quot;max_lr&quot;</span><span class="p">,</span>
+            <span class="s2">&quot;finetune_max_lr&quot;</span><span class="p">,</span>
+            <span class="s2">&quot;warmup_max_lr&quot;</span><span class="p">,</span>
+            <span class="s2">&quot;routine&quot;</span><span class="p">,</span>
+            <span class="s2">&quot;deeptabular_gradual&quot;</span><span class="p">,</span>
+            <span class="s2">&quot;deeptabular_layers&quot;</span><span class="p">,</span>
+            <span class="s2">&quot;deeptabular_max_lr&quot;</span><span class="p">,</span>
+            <span class="s2">&quot;deeptext_gradual&quot;</span><span class="p">,</span>
+            <span class="s2">&quot;deeptext_layers&quot;</span><span class="p">,</span>
+            <span class="s2">&quot;deeptext_max_lr&quot;</span><span class="p">,</span>
+            <span class="s2">&quot;deepimage_gradual&quot;</span><span class="p">,</span>
+            <span class="s2">&quot;deepimage_layers&quot;</span><span class="p">,</span>
+            <span class="s2">&quot;deepimage_max_lr&quot;</span><span class="p">,</span>
+        <span class="p">]</span>
+
+        <span class="n">dataloader_args</span><span class="p">,</span> <span class="n">finetune_args</span> <span class="o">=</span> <span class="p">{},</span> <span class="p">{}</span>
+        <span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="n">kwargs</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
+            <span class="k">if</span> <span class="n">k</span> <span class="ow">in</span> <span class="n">dataloader_params</span><span class="p">:</span>
+                <span class="n">dataloader_args</span><span class="p">[</span><span class="n">k</span><span class="p">]</span> <span class="o">=</span> <span class="n">v</span>
+            <span class="k">if</span> <span class="n">k</span> <span class="ow">in</span> <span class="n">finetune_params</span><span class="p">:</span>
+                <span class="n">finetune_args</span><span class="p">[</span><span class="n">k</span><span class="p">]</span> <span class="o">=</span> <span class="n">v</span>
+
+        <span class="k">return</span> <span class="n">dataloader_args</span><span class="p">,</span> <span class="n">finetune_args</span>
 </code></pre></div></td></tr></table></div>
-                  </details>
+              </details>
 
 
 
@@ -2089,152 +4375,211 @@ <h3 id="pytorch_widedeep.training.Trainer.fit" class="doc doc-heading">
 
 
 <p><span class="doc-section-title">Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>X_wide</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="numpy.ndarray">ndarray</span>]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Input for the <code>wide</code> model component.
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+          <th>Default</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>X_wide</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="numpy.ndarray">ndarray</span>]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Input for the <code>wide</code> model component.
 See <code>pytorch_widedeep.preprocessing.WidePreprocessor</code></p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>X_tab</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="numpy.ndarray">ndarray</span>, <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="numpy.ndarray">ndarray</span>]]]</code>, default:
+              </div>
+            </td>
+            <td>
                   <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Input for the <code>deeptabular</code> model component.
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>X_tab</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="numpy.ndarray">ndarray</span>, <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="numpy.ndarray">ndarray</span>]]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Input for the <code>deeptabular</code> model component.
 See <code>pytorch_widedeep.preprocessing.TabPreprocessor</code>. If multiple
 tabular models are used for different columns, this should be a
 list of numpy arrays</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>X_text</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="numpy.ndarray">ndarray</span>, <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="numpy.ndarray">ndarray</span>]]]</code>, default:
+              </div>
+            </td>
+            <td>
                   <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Input for the <code>deeptext</code> model component.
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>X_text</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="numpy.ndarray">ndarray</span>, <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="numpy.ndarray">ndarray</span>]]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Input for the <code>deeptext</code> model component.
 See <code>pytorch_widedeep.preprocessing.TextPreprocessor</code>.
 If multiple text columns/models are used, this should be a list of
 numpy arrays</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>X_img</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="numpy.ndarray">ndarray</span>, <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="numpy.ndarray">ndarray</span>]]]</code>, default:
+              </div>
+            </td>
+            <td>
                   <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Input for the <code>deepimage</code> model component.
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>X_img</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="numpy.ndarray">ndarray</span>, <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="numpy.ndarray">ndarray</span>]]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Input for the <code>deepimage</code> model component.
 See <code>pytorch_widedeep.preprocessing.ImagePreprocessor</code>.
 If multiple image columns/models are used, this should be a list of
 numpy arrays</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>X_train</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, <span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="numpy.ndarray">ndarray</span>, <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="numpy.ndarray">ndarray</span>]]]]</code>, default:
+              </div>
+            </td>
+            <td>
                   <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>The training dataset can also be passed in a dictionary. Keys are
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>X_train</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, <span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="numpy.ndarray">ndarray</span>, <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="numpy.ndarray">ndarray</span>]]]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>The training dataset can also be passed in a dictionary. Keys are
 <em>'X_wide'</em>, <em>'X_tab'</em>, <em>'X_text'</em>, <em>'X_img'</em> and <em>'target'</em>. Values
 are the corresponding matrices. Note that of multiple text or image
 columns/models are used, the corresponding values should be lists
 of numpy arrays</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>X_val</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, <span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="numpy.ndarray">ndarray</span>, <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="numpy.ndarray">ndarray</span>]]]]</code>, default:
+              </div>
+            </td>
+            <td>
                   <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>The validation dataset can also be passed in a dictionary. Keys
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>X_val</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, <span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="numpy.ndarray">ndarray</span>, <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="numpy.ndarray">ndarray</span>]]]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>The validation dataset can also be passed in a dictionary. Keys
 are <em>'X_wide'</em>, <em>'X_tab'</em>, <em>'X_text'</em>, <em>'X_img'</em> and <em>'target'</em>.
 Values are the corresponding matrices. Note that of multiple text
 or image columns/models are used, the corresponding values should
 be lists of numpy arrays</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>val_split</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>, default:
+              </div>
+            </td>
+            <td>
                   <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>train/val split fraction</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>target</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="numpy.ndarray">ndarray</span>]</code>, default:
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>val_split</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>train/val split fraction</p>
+              </div>
+            </td>
+            <td>
                   <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>target values</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>n_epochs</code></b>
-              (<code>int</code>, default:
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>target</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="numpy.ndarray">ndarray</span>]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>target values</p>
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>n_epochs</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>number of epochs</p>
+              </div>
+            </td>
+            <td>
                   <code>1</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>number of epochs</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>validation_freq</code></b>
-              (<code>int</code>, default:
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>validation_freq</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>epochs validation frequency</p>
+              </div>
+            </td>
+            <td>
                   <code>1</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>epochs validation frequency</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>batch_size</code></b>
-              (<code>int</code>, default:
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>batch_size</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>batch size</p>
+              </div>
+            </td>
+            <td>
                   <code>32</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>batch size</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>custom_dataloader</code></b>
-          –
-          <div class="doc-md-description">
-            <p>object of class <code>torch.utils.data.DataLoader</code>. Available
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>custom_dataloader</code></td>
+            <td>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>object of class <code>torch.utils.data.DataLoader</code>. Available
 predefined dataloaders are in <code>pytorch-widedeep.dataloaders</code>.If
 <code>None</code>, a standard torch <code>DataLoader</code> is used.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>finetune</code></b>
-              (<code>bool</code>, default:
-                  <code>False</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>fine-tune individual model components. This functionality can also
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>finetune</code></td>
+            <td>
+                  <code>bool</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>fine-tune individual model components. This functionality can also
 be used to 'warm-up' (and hence the alias <code>warmup</code>) individual
 components before the joined training starts, and hence its
 alias. See the Examples folder in the repo for more details</p>
@@ -2262,18 +4607,33 @@ <h3 id="pytorch_widedeep.training.Trainer.fit" class="doc doc-heading">
 <p>For details on how these routines work, please see the Examples
 section in this documentation and the Examples folder in the repo. <br/>
 Param Alias: <code>warmup</code></p>
-          </div>
-        </li>
-    </ul>
+              </div>
+            </td>
+            <td>
+                  <code>False</code>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
 
 <p><span class="doc-section-title">Other Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>**kwargs</code></b>
-          –
-          <div class="doc-md-description">
-            <p>Other keyword arguments are:</p>
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>**kwargs</code></td>
+            <td>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Other keyword arguments are:</p>
 <ul>
 <li>
 <p><strong>DataLoader related parameters</strong>:<br/>
@@ -2316,9 +4676,11 @@ <h3 id="pytorch_widedeep.training.Trainer.fit" class="doc doc-heading">
 </ul>
 </li>
 </ul>
-          </div>
-        </li>
-    </ul>
+              </div>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
 
 <p><span class="doc-section-title">Examples:</span></p>
@@ -2804,88 +5166,133 @@ <h3 id="pytorch_widedeep.training.Trainer.predict" class="doc doc-heading">
 
 
 <p><span class="doc-section-title">Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>X_wide</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="numpy.ndarray">ndarray</span>]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Input for the <code>wide</code> model component.
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+          <th>Default</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>X_wide</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="numpy.ndarray">ndarray</span>]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Input for the <code>wide</code> model component.
 See <code>pytorch_widedeep.preprocessing.WidePreprocessor</code></p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>X_tab</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="numpy.ndarray">ndarray</span>, <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="numpy.ndarray">ndarray</span>]]]</code>, default:
+              </div>
+            </td>
+            <td>
                   <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Input for the <code>deeptabular</code> model component.
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>X_tab</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="numpy.ndarray">ndarray</span>, <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="numpy.ndarray">ndarray</span>]]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Input for the <code>deeptabular</code> model component.
 See <code>pytorch_widedeep.preprocessing.TabPreprocessor</code></p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>X_text</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="numpy.ndarray">ndarray</span>, <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="numpy.ndarray">ndarray</span>]]]</code>, default:
+              </div>
+            </td>
+            <td>
                   <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Input for the <code>deeptext</code> model component.
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>X_text</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="numpy.ndarray">ndarray</span>, <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="numpy.ndarray">ndarray</span>]]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Input for the <code>deeptext</code> model component.
 See <code>pytorch_widedeep.preprocessing.TextPreprocessor</code></p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>X_img</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="numpy.ndarray">ndarray</span>, <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="numpy.ndarray">ndarray</span>]]]</code>, default:
+              </div>
+            </td>
+            <td>
                   <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Input for the <code>deepimage</code> model component.
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>X_img</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="numpy.ndarray">ndarray</span>, <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="numpy.ndarray">ndarray</span>]]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Input for the <code>deepimage</code> model component.
 See <code>pytorch_widedeep.preprocessing.ImagePreprocessor</code></p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>X_test</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, <span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="numpy.ndarray">ndarray</span>, <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="numpy.ndarray">ndarray</span>]]]]</code>, default:
+              </div>
+            </td>
+            <td>
                   <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>The test dataset can also be passed in a dictionary. Keys are
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>X_test</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, <span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="numpy.ndarray">ndarray</span>, <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="numpy.ndarray">ndarray</span>]]]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>The test dataset can also be passed in a dictionary. Keys are
 <code>X_wide</code>, <em>'X_tab'</em>, <em>'X_text'</em>, <em>'X_img'</em> and <em>'target'</em>. Values
 are the corresponding matrices.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>batch_size</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[int]</code>, default:
+              </div>
+            </td>
+            <td>
                   <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>If a trainer is used to predict after having trained a model, the
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>batch_size</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[int]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>If a trainer is used to predict after having trained a model, the
 <code>batch_size</code> needs to be defined as it will not be defined as
 the <code>Trainer</code> is instantiated</p>
-          </div>
-        </li>
-    </ul>
-
-
-<p><span class="doc-section-title">Returns:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-              <code>np.ndarray:</code>
-          –
-          <div class="doc-md-description">
-            <p>array with the predictions</p>
-          </div>
-        </li>
-    </ul>
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+      </tbody>
+    </table>
+
+
+    <p><span class="doc-section-title">Returns:</span></p>
+    <table>
+      <thead>
+        <tr>
+          <th>Type</th>
+          <th>Description</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td>
+                  <code>np.ndarray:</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>array with the predictions</p>
+              </div>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
             <details class="quote">
               <summary>Source code in <code>pytorch_widedeep/training/trainer.py</code></summary>
@@ -3055,95 +5462,142 @@ <h3 id="pytorch_widedeep.training.Trainer.predict_uncertainty" class="doc doc-he
 
 
 <p><span class="doc-section-title">Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>X_wide</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="numpy.ndarray">ndarray</span>]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Input for the <code>wide</code> model component.
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+          <th>Default</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>X_wide</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="numpy.ndarray">ndarray</span>]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Input for the <code>wide</code> model component.
 See <code>pytorch_widedeep.preprocessing.WidePreprocessor</code></p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>X_tab</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="numpy.ndarray">ndarray</span>, <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="numpy.ndarray">ndarray</span>]]]</code>, default:
+              </div>
+            </td>
+            <td>
                   <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Input for the <code>deeptabular</code> model component.
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>X_tab</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="numpy.ndarray">ndarray</span>, <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="numpy.ndarray">ndarray</span>]]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Input for the <code>deeptabular</code> model component.
 See <code>pytorch_widedeep.preprocessing.TabPreprocessor</code></p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>X_text</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="numpy.ndarray">ndarray</span>, <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="numpy.ndarray">ndarray</span>]]]</code>, default:
+              </div>
+            </td>
+            <td>
                   <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Input for the <code>deeptext</code> model component.
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>X_text</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="numpy.ndarray">ndarray</span>, <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="numpy.ndarray">ndarray</span>]]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Input for the <code>deeptext</code> model component.
 See <code>pytorch_widedeep.preprocessing.TextPreprocessor</code></p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>X_img</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="numpy.ndarray">ndarray</span>, <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="numpy.ndarray">ndarray</span>]]]</code>, default:
+              </div>
+            </td>
+            <td>
                   <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Input for the <code>deepimage</code> model component.
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>X_img</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="numpy.ndarray">ndarray</span>, <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="numpy.ndarray">ndarray</span>]]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Input for the <code>deepimage</code> model component.
 See <code>pytorch_widedeep.preprocessing.ImagePreprocessor</code></p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>X_test</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, <span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="numpy.ndarray">ndarray</span>, <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="numpy.ndarray">ndarray</span>]]]]</code>, default:
+              </div>
+            </td>
+            <td>
                   <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>The test dataset can also be passed in a dictionary. Keys are
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>X_test</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, <span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="numpy.ndarray">ndarray</span>, <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="numpy.ndarray">ndarray</span>]]]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>The test dataset can also be passed in a dictionary. Keys are
 <em>'X_wide'</em>, <em>'X_tab'</em>, <em>'X_text'</em>, <em>'X_img'</em> and <em>'target'</em>. Values
 are the corresponding matrices.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>batch_size</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[int]</code>, default:
+              </div>
+            </td>
+            <td>
                   <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>If a trainer is used to predict after having trained a model, the
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>batch_size</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[int]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>If a trainer is used to predict after having trained a model, the
 <code>batch_size</code> needs to be defined as it will not be defined as
 the <code>Trainer</code> is instantiated</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>uncertainty_granularity</code></b>
-              (<code>int</code>, default:
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>uncertainty_granularity</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>number of times the model does prediction for each sample</p>
+              </div>
+            </td>
+            <td>
                   <code>1000</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>number of times the model does prediction for each sample</p>
-          </div>
-        </li>
-    </ul>
-
-
-<p><span class="doc-section-title">Returns:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-              <code>np.ndarray:</code>
-          –
-          <div class="doc-md-description">
-            <ul>
+            </td>
+          </tr>
+      </tbody>
+    </table>
+
+
+    <p><span class="doc-section-title">Returns:</span></p>
+    <table>
+      <thead>
+        <tr>
+          <th>Type</th>
+          <th>Description</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td>
+                  <code>np.ndarray:</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <ul>
 <li>if <code>method = regression</code>, it will return an array with <code>(max, min, mean, stdev)</code>
   values for each sample.</li>
 <li>if <code>method = binary</code> it will return an array with
@@ -3152,9 +5606,11 @@ <h3 id="pytorch_widedeep.training.Trainer.predict_uncertainty" class="doc doc-he
   <code>(mean_cls_0_prob, mean_cls_1_prob, mean_cls_2_prob, ... , predicted_cls)</code>
   values for each sample.</li>
 </ul>
-          </div>
-        </li>
-    </ul>
+              </div>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
             <details class="quote">
               <summary>Source code in <code>pytorch_widedeep/training/trainer.py</code></summary>
@@ -3393,88 +5849,133 @@ <h3 id="pytorch_widedeep.training.Trainer.predict_proba" class="doc doc-heading"
 
 
 <p><span class="doc-section-title">Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>X_wide</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="numpy.ndarray">ndarray</span>]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Input for the <code>wide</code> model component.
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+          <th>Default</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>X_wide</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="numpy.ndarray">ndarray</span>]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Input for the <code>wide</code> model component.
 See <code>pytorch_widedeep.preprocessing.WidePreprocessor</code></p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>X_tab</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="numpy.ndarray">ndarray</span>, <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="numpy.ndarray">ndarray</span>]]]</code>, default:
+              </div>
+            </td>
+            <td>
                   <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Input for the <code>deeptabular</code> model component.
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>X_tab</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="numpy.ndarray">ndarray</span>, <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="numpy.ndarray">ndarray</span>]]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Input for the <code>deeptabular</code> model component.
 See <code>pytorch_widedeep.preprocessing.TabPreprocessor</code></p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>X_text</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="numpy.ndarray">ndarray</span>, <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="numpy.ndarray">ndarray</span>]]]</code>, default:
+              </div>
+            </td>
+            <td>
                   <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Input for the <code>deeptext</code> model component.
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>X_text</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="numpy.ndarray">ndarray</span>, <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="numpy.ndarray">ndarray</span>]]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Input for the <code>deeptext</code> model component.
 See <code>pytorch_widedeep.preprocessing.TextPreprocessor</code></p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>X_img</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="numpy.ndarray">ndarray</span>, <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="numpy.ndarray">ndarray</span>]]]</code>, default:
+              </div>
+            </td>
+            <td>
                   <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Input for the <code>deepimage</code> model component.
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>X_img</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="numpy.ndarray">ndarray</span>, <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="numpy.ndarray">ndarray</span>]]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Input for the <code>deepimage</code> model component.
 See <code>pytorch_widedeep.preprocessing.ImagePreprocessor</code></p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>X_test</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, <span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="numpy.ndarray">ndarray</span>, <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="numpy.ndarray">ndarray</span>]]]]</code>, default:
+              </div>
+            </td>
+            <td>
                   <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>The test dataset can also be passed in a dictionary. Keys are
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>X_test</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, <span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="numpy.ndarray">ndarray</span>, <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="numpy.ndarray">ndarray</span>]]]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>The test dataset can also be passed in a dictionary. Keys are
 <code>X_wide</code>, <em>'X_tab'</em>, <em>'X_text'</em>, <em>'X_img'</em> and <em>'target'</em>. Values
 are the corresponding matrices.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>batch_size</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[int]</code>, default:
+              </div>
+            </td>
+            <td>
                   <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>If a trainer is used to predict after having trained a model, the
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>batch_size</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[int]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>If a trainer is used to predict after having trained a model, the
 <code>batch_size</code> needs to be defined as it will not be defined as
 the <code>Trainer</code> is instantiated</p>
-          </div>
-        </li>
-    </ul>
-
-
-<p><span class="doc-section-title">Returns:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-              <code><span title="numpy.ndarray">ndarray</span></code>
-          –
-          <div class="doc-md-description">
-            <p>array with the probabilities per class</p>
-          </div>
-        </li>
-    </ul>
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+      </tbody>
+    </table>
+
+
+    <p><span class="doc-section-title">Returns:</span></p>
+    <table>
+      <thead>
+        <tr>
+          <th>Type</th>
+          <th>Description</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td>
+                  <code><span title="numpy.ndarray">ndarray</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>array with the probabilities per class</p>
+              </div>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
             <details class="quote">
               <summary>Source code in <code>pytorch_widedeep/training/trainer.py</code></summary>
@@ -3633,49 +6134,77 @@ <h3 id="pytorch_widedeep.training.Trainer.save" class="doc doc-heading">
 
 
 <p><span class="doc-section-title">Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>path</code></b>
-              (<code>str</code>)
-          –
-          <div class="doc-md-description">
-            <p>path to the directory where the model and the feature importance
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+          <th>Default</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>path</code></td>
+            <td>
+                  <code>str</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>path to the directory where the model and the feature importance
 attribute will be saved.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>save_state_dict</code></b>
-              (<code>bool</code>, default:
-                  <code>False</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Boolean indicating whether to save directly the model
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>save_state_dict</code></td>
+            <td>
+                  <code>bool</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Boolean indicating whether to save directly the model
 (and optimizer) or the model's (and optimizer's) state
 dictionary</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>save_optimizer</code></b>
-              (<code>bool</code>, default:
+              </div>
+            </td>
+            <td>
                   <code>False</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Boolean indicating whether to save the optimizer</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>model_filename</code></b>
-              (<code>str</code>, default:
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>save_optimizer</code></td>
+            <td>
+                  <code>bool</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Boolean indicating whether to save the optimizer</p>
+              </div>
+            </td>
+            <td>
+                  <code>False</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>model_filename</code></td>
+            <td>
+                  <code>str</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>filename where the model weights will be store</p>
+              </div>
+            </td>
+            <td>
                   <code>&#39;wd_model.pt&#39;</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>filename where the model weights will be store</p>
-          </div>
-        </li>
-    </ul>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
             <details class="quote">
               <summary>Source code in <code>pytorch_widedeep/training/trainer.py</code></summary>
diff --git a/mkdocs/site/pytorch-widedeep/utils/deeptabular_utils.html b/mkdocs/site/pytorch-widedeep/utils/deeptabular_utils.html
index e14dcbd0..64eb03e9 100644
--- a/mkdocs/site/pytorch-widedeep/utils/deeptabular_utils.html
+++ b/mkdocs/site/pytorch-widedeep/utils/deeptabular_utils.html
@@ -1387,7 +1387,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../../examples/16_Self_Supervised_Pretraning_pt1.ipynb" class="md-nav__link">
+      <a href="../../examples/15_Self_Supervised_Pretraning_pt1.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1408,7 +1408,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../../examples/16_Self_Supervised_Pretraning_pt2.ipynb" class="md-nav__link">
+      <a href="../../examples/15_Self_Supervised_Pretraning_pt2.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1429,7 +1429,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../../examples/17_Usign_a_custom_hugging_face_model.ipynb" class="md-nav__link">
+      <a href="../../examples/16_Usign_a_custom_hugging_face_model.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1450,7 +1450,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../../examples/18_feature_importance_via_attention_weights.ipynb" class="md-nav__link">
+      <a href="../../examples/17_feature_importance_via_attention_weights.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1471,7 +1471,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../../examples/19_wide_and_deep_for_recsys_pt1.ipynb" class="md-nav__link">
+      <a href="../../examples/18_wide_and_deep_for_recsys_pt1.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1492,7 +1492,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../../examples/19_wide_and_deep_for_recsys_pt2.ipynb" class="md-nav__link">
+      <a href="../../examples/18_wide_and_deep_for_recsys_pt2.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1513,7 +1513,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../../examples/20_load_from_folder_functionality.ipynb" class="md-nav__link">
+      <a href="../../examples/19_load_from_folder_functionality.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1534,7 +1534,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../../examples/21_Using_huggingface_within_widedeep.ipynb" class="md-nav__link">
+      <a href="../../examples/20_Using_huggingface_within_widedeep.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1605,12 +1605,7 @@ <h2 id="pytorch_widedeep.utils.deeptabular_utils.LabelEncoder" class="doc doc-he
 
 
 <a href="#pytorch_widedeep.utils.deeptabular_utils.LabelEncoder" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="nf">LabelEncoder</span><span class="p">(</span>
-    <span class="n">columns_to_encode</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">with_attention</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
-    <span class="n">shared_embed</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
-<span class="p">)</span>
-</code></pre></div>
+
 
     <div class="doc doc-contents first">
 
@@ -1622,100 +1617,599 @@ <h2 id="pytorch_widedeep.utils.deeptabular_utils.LabelEncoder" class="doc doc-he
 
 
 <p><span class="doc-section-title">Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>columns_to_encode</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[str]]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>List of strings containing the names of the columns to encode. If
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+          <th>Default</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>columns_to_encode</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[str]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>List of strings containing the names of the columns to encode. If
 <code>None</code> all columns of type <code>object</code> in the dataframe will be label
 encoded.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>with_attention</code></b>
-              (<code>bool</code>, default:
-                  <code>False</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Boolean indicating whether the preprocessed data will be passed to an
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>with_attention</code></td>
+            <td>
+                  <code>bool</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Boolean indicating whether the preprocessed data will be passed to an
 attention-based model. Aliased as <code>for_transformer</code>.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>shared_embed</code></b>
-              (<code>bool</code>, default:
+              </div>
+            </td>
+            <td>
                   <code>False</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Boolean indicating if the embeddings will be "<em>shared</em>" when using
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>shared_embed</code></td>
+            <td>
+                  <code>bool</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Boolean indicating if the embeddings will be "<em>shared</em>" when using
 attention-based models. The idea behind <code>shared_embed</code> is described
 in the Appendix A in the <a href="https://arxiv.org/abs/2012.06678">TabTransformer paper</a>:
 '<em>The goal of having column embedding is to enable the model to
 distinguish the classes in one column from those in the
 other columns</em>'. In other words, the idea is to let the model learn
 which column is embedded at the time. See: <code>pytorch_widedeep.models.transformers._layers.SharedEmbeddings</code>.</p>
-          </div>
-        </li>
-    </ul>
+              </div>
+            </td>
+            <td>
+                  <code>False</code>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
 
 <p><span class="doc-section-title">Attributes:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code><span title="pytorch_widedeep.utils.deeptabular_utils.LabelEncoder.encoding_dict">encoding_dict</span></code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Dict">Dict</span></code>)
-          –
-          <div class="doc-md-description">
-            <p>Dictionary containing the encoding mappings in the format, e.g. : <br/>
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code><span title="pytorch_widedeep.utils.deeptabular_utils.LabelEncoder.encoding_dict">encoding_dict</span></code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Dict">Dict</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Dictionary containing the encoding mappings in the format, e.g. : <br/>
 <code>{'colname1': {'cat1': 1, 'cat2': 2, ...}, 'colname2': {'cat1': 1, 'cat2': 2, ...}, ...}</code></p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code><span title="pytorch_widedeep.utils.deeptabular_utils.LabelEncoder.inverse_encoding_dict">inverse_encoding_dict</span></code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Dict">Dict</span></code>)
-          –
-          <div class="doc-md-description">
-            <p>Dictionary containing the inverse encoding mappings in the format, e.g. : <br/>
+              </div>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code><span title="pytorch_widedeep.utils.deeptabular_utils.LabelEncoder.inverse_encoding_dict">inverse_encoding_dict</span></code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Dict">Dict</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Dictionary containing the inverse encoding mappings in the format, e.g. : <br/>
 <code>{'colname1': {1: 'cat1', 2: 'cat2', ...}, 'colname2': {1: 'cat1', 2: 'cat2', ...}, ...}</code></p>
-          </div>
-        </li>
-    </ul>
+              </div>
+            </td>
+          </tr>
+      </tbody>
+    </table>
+
+              <details class="quote">
+                <summary>Source code in <code>pytorch_widedeep/utils/deeptabular_utils.py</code></summary>
+                <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 14</span>
+<span class="normal"> 15</span>
+<span class="normal"> 16</span>
+<span class="normal"> 17</span>
+<span class="normal"> 18</span>
+<span class="normal"> 19</span>
+<span class="normal"> 20</span>
+<span class="normal"> 21</span>
+<span class="normal"> 22</span>
+<span class="normal"> 23</span>
+<span class="normal"> 24</span>
+<span class="normal"> 25</span>
+<span class="normal"> 26</span>
+<span class="normal"> 27</span>
+<span class="normal"> 28</span>
+<span class="normal"> 29</span>
+<span class="normal"> 30</span>
+<span class="normal"> 31</span>
+<span class="normal"> 32</span>
+<span class="normal"> 33</span>
+<span class="normal"> 34</span>
+<span class="normal"> 35</span>
+<span class="normal"> 36</span>
+<span class="normal"> 37</span>
+<span class="normal"> 38</span>
+<span class="normal"> 39</span>
+<span class="normal"> 40</span>
+<span class="normal"> 41</span>
+<span class="normal"> 42</span>
+<span class="normal"> 43</span>
+<span class="normal"> 44</span>
+<span class="normal"> 45</span>
+<span class="normal"> 46</span>
+<span class="normal"> 47</span>
+<span class="normal"> 48</span>
+<span class="normal"> 49</span>
+<span class="normal"> 50</span>
+<span class="normal"> 51</span>
+<span class="normal"> 52</span>
+<span class="normal"> 53</span>
+<span class="normal"> 54</span>
+<span class="normal"> 55</span>
+<span class="normal"> 56</span>
+<span class="normal"> 57</span>
+<span class="normal"> 58</span>
+<span class="normal"> 59</span>
+<span class="normal"> 60</span>
+<span class="normal"> 61</span>
+<span class="normal"> 62</span>
+<span class="normal"> 63</span>
+<span class="normal"> 64</span>
+<span class="normal"> 65</span>
+<span class="normal"> 66</span>
+<span class="normal"> 67</span>
+<span class="normal"> 68</span>
+<span class="normal"> 69</span>
+<span class="normal"> 70</span>
+<span class="normal"> 71</span>
+<span class="normal"> 72</span>
+<span class="normal"> 73</span>
+<span class="normal"> 74</span>
+<span class="normal"> 75</span>
+<span class="normal"> 76</span>
+<span class="normal"> 77</span>
+<span class="normal"> 78</span>
+<span class="normal"> 79</span>
+<span class="normal"> 80</span>
+<span class="normal"> 81</span>
+<span class="normal"> 82</span>
+<span class="normal"> 83</span>
+<span class="normal"> 84</span>
+<span class="normal"> 85</span>
+<span class="normal"> 86</span>
+<span class="normal"> 87</span>
+<span class="normal"> 88</span>
+<span class="normal"> 89</span>
+<span class="normal"> 90</span>
+<span class="normal"> 91</span>
+<span class="normal"> 92</span>
+<span class="normal"> 93</span>
+<span class="normal"> 94</span>
+<span class="normal"> 95</span>
+<span class="normal"> 96</span>
+<span class="normal"> 97</span>
+<span class="normal"> 98</span>
+<span class="normal"> 99</span>
+<span class="normal">100</span>
+<span class="normal">101</span>
+<span class="normal">102</span>
+<span class="normal">103</span>
+<span class="normal">104</span>
+<span class="normal">105</span>
+<span class="normal">106</span>
+<span class="normal">107</span>
+<span class="normal">108</span>
+<span class="normal">109</span>
+<span class="normal">110</span>
+<span class="normal">111</span>
+<span class="normal">112</span>
+<span class="normal">113</span>
+<span class="normal">114</span>
+<span class="normal">115</span>
+<span class="normal">116</span>
+<span class="normal">117</span>
+<span class="normal">118</span>
+<span class="normal">119</span>
+<span class="normal">120</span>
+<span class="normal">121</span>
+<span class="normal">122</span>
+<span class="normal">123</span>
+<span class="normal">124</span>
+<span class="normal">125</span>
+<span class="normal">126</span>
+<span class="normal">127</span>
+<span class="normal">128</span>
+<span class="normal">129</span>
+<span class="normal">130</span>
+<span class="normal">131</span>
+<span class="normal">132</span>
+<span class="normal">133</span>
+<span class="normal">134</span>
+<span class="normal">135</span>
+<span class="normal">136</span>
+<span class="normal">137</span>
+<span class="normal">138</span>
+<span class="normal">139</span>
+<span class="normal">140</span>
+<span class="normal">141</span>
+<span class="normal">142</span>
+<span class="normal">143</span>
+<span class="normal">144</span>
+<span class="normal">145</span>
+<span class="normal">146</span>
+<span class="normal">147</span>
+<span class="normal">148</span>
+<span class="normal">149</span>
+<span class="normal">150</span>
+<span class="normal">151</span>
+<span class="normal">152</span>
+<span class="normal">153</span>
+<span class="normal">154</span>
+<span class="normal">155</span>
+<span class="normal">156</span>
+<span class="normal">157</span>
+<span class="normal">158</span>
+<span class="normal">159</span>
+<span class="normal">160</span>
+<span class="normal">161</span>
+<span class="normal">162</span>
+<span class="normal">163</span>
+<span class="normal">164</span>
+<span class="normal">165</span>
+<span class="normal">166</span>
+<span class="normal">167</span>
+<span class="normal">168</span>
+<span class="normal">169</span>
+<span class="normal">170</span>
+<span class="normal">171</span>
+<span class="normal">172</span>
+<span class="normal">173</span>
+<span class="normal">174</span>
+<span class="normal">175</span>
+<span class="normal">176</span>
+<span class="normal">177</span>
+<span class="normal">178</span>
+<span class="normal">179</span>
+<span class="normal">180</span>
+<span class="normal">181</span>
+<span class="normal">182</span>
+<span class="normal">183</span>
+<span class="normal">184</span>
+<span class="normal">185</span>
+<span class="normal">186</span>
+<span class="normal">187</span>
+<span class="normal">188</span>
+<span class="normal">189</span>
+<span class="normal">190</span>
+<span class="normal">191</span>
+<span class="normal">192</span>
+<span class="normal">193</span>
+<span class="normal">194</span>
+<span class="normal">195</span>
+<span class="normal">196</span>
+<span class="normal">197</span>
+<span class="normal">198</span>
+<span class="normal">199</span>
+<span class="normal">200</span>
+<span class="normal">201</span>
+<span class="normal">202</span>
+<span class="normal">203</span>
+<span class="normal">204</span>
+<span class="normal">205</span>
+<span class="normal">206</span>
+<span class="normal">207</span>
+<span class="normal">208</span>
+<span class="normal">209</span>
+<span class="normal">210</span>
+<span class="normal">211</span>
+<span class="normal">212</span>
+<span class="normal">213</span>
+<span class="normal">214</span>
+<span class="normal">215</span>
+<span class="normal">216</span>
+<span class="normal">217</span>
+<span class="normal">218</span>
+<span class="normal">219</span>
+<span class="normal">220</span>
+<span class="normal">221</span>
+<span class="normal">222</span>
+<span class="normal">223</span>
+<span class="normal">224</span>
+<span class="normal">225</span>
+<span class="normal">226</span>
+<span class="normal">227</span>
+<span class="normal">228</span>
+<span class="normal">229</span>
+<span class="normal">230</span>
+<span class="normal">231</span>
+<span class="normal">232</span>
+<span class="normal">233</span>
+<span class="normal">234</span>
+<span class="normal">235</span>
+<span class="normal">236</span>
+<span class="normal">237</span>
+<span class="normal">238</span>
+<span class="normal">239</span>
+<span class="normal">240</span>
+<span class="normal">241</span>
+<span class="normal">242</span>
+<span class="normal">243</span>
+<span class="normal">244</span>
+<span class="normal">245</span>
+<span class="normal">246</span>
+<span class="normal">247</span>
+<span class="normal">248</span>
+<span class="normal">249</span>
+<span class="normal">250</span>
+<span class="normal">251</span>
+<span class="normal">252</span>
+<span class="normal">253</span>
+<span class="normal">254</span>
+<span class="normal">255</span>
+<span class="normal">256</span>
+<span class="normal">257</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">class</span> <span class="nc">LabelEncoder</span><span class="p">:</span>
+<span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Label Encode categorical values for multiple columns at once</span>
+
+<span class="sd">    :information_source: **NOTE**:</span>
+<span class="sd">    LabelEncoder reserves 0 for `unseen` new categories. This is convenient</span>
+<span class="sd">    when defining the embedding layers, since we can just set padding idx to 0.</span>
+
+<span class="sd">    Parameters</span>
+<span class="sd">    ----------</span>
+<span class="sd">    columns_to_encode: list, Optional, default = None</span>
+<span class="sd">        List of strings containing the names of the columns to encode. If</span>
+<span class="sd">        `None` all columns of type `object` in the dataframe will be label</span>
+<span class="sd">        encoded.</span>
+<span class="sd">    with_attention: bool, default = False</span>
+<span class="sd">        Boolean indicating whether the preprocessed data will be passed to an</span>
+<span class="sd">        attention-based model. Aliased as `for_transformer`.</span>
+<span class="sd">    shared_embed: bool, default = False</span>
+<span class="sd">        Boolean indicating if the embeddings will be &quot;_shared_&quot; when using</span>
+<span class="sd">        attention-based models. The idea behind `shared_embed` is described</span>
+<span class="sd">        in the Appendix A in the [TabTransformer paper](https://arxiv.org/abs/2012.06678):</span>
+<span class="sd">        &#39;_The goal of having column embedding is to enable the model to</span>
+<span class="sd">        distinguish the classes in one column from those in the</span>
+<span class="sd">        other columns_&#39;. In other words, the idea is to let the model learn</span>
+<span class="sd">        which column is embedded at the time. See: `pytorch_widedeep.models.transformers._layers.SharedEmbeddings`.</span>
+
+<span class="sd">    Attributes</span>
+<span class="sd">    ----------</span>
+<span class="sd">    encoding_dict : Dict</span>
+<span class="sd">        Dictionary containing the encoding mappings in the format, e.g. : &lt;br/&gt;</span>
+<span class="sd">        `{&#39;colname1&#39;: {&#39;cat1&#39;: 1, &#39;cat2&#39;: 2, ...}, &#39;colname2&#39;: {&#39;cat1&#39;: 1, &#39;cat2&#39;: 2, ...}, ...}`</span>
+<span class="sd">    inverse_encoding_dict : Dict</span>
+<span class="sd">        Dictionary containing the inverse encoding mappings in the format, e.g. : &lt;br/&gt;</span>
+<span class="sd">        `{&#39;colname1&#39;: {1: &#39;cat1&#39;, 2: &#39;cat2&#39;, ...}, &#39;colname2&#39;: {1: &#39;cat1&#39;, 2: &#39;cat2&#39;, ...}, ...}`</span>
+
+<span class="sd">    &quot;&quot;&quot;</span>
+
+    <span class="nd">@alias</span><span class="p">(</span><span class="s2">&quot;with_attention&quot;</span><span class="p">,</span> <span class="p">[</span><span class="s2">&quot;for_transformer&quot;</span><span class="p">])</span>
+    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span>
+        <span class="n">columns_to_encode</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">with_attention</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
+        <span class="n">shared_embed</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
+    <span class="p">):</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">columns_to_encode</span> <span class="o">=</span> <span class="n">columns_to_encode</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">shared_embed</span> <span class="o">=</span> <span class="n">shared_embed</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">with_attention</span> <span class="o">=</span> <span class="n">with_attention</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">reset_embed_idx</span> <span class="o">=</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">with_attention</span> <span class="ow">or</span> <span class="bp">self</span><span class="o">.</span><span class="n">shared_embed</span>
+
+    <span class="k">def</span> <span class="nf">partial_fit</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="s2">&quot;LabelEncoder&quot;</span><span class="p">:</span>  <span class="c1"># noqa: C901</span>
+<span class="w">        </span><span class="sd">&quot;&quot;&quot;Main method. Creates encoding attributes.</span>
+
+<span class="sd">        Returns</span>
+<span class="sd">        -------</span>
+<span class="sd">        LabelEncoder</span>
+<span class="sd">            `LabelEncoder` fitted object</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="c1"># here df is a chunk of the data. this is meant to be run when the</span>
+        <span class="c1"># data is large and we pass a chunk at a time. Therefore, we do not</span>
+        <span class="c1"># copy the input chunk as mutating a chunk is ok</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">columns_to_encode</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">columns_to_encode</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="n">df</span><span class="o">.</span><span class="n">select_dtypes</span><span class="p">(</span><span class="n">include</span><span class="o">=</span><span class="p">[</span><span class="s2">&quot;object&quot;</span><span class="p">])</span><span class="o">.</span><span class="n">columns</span><span class="p">)</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="c1"># sanity check to make sure all categorical columns are in an adequate</span>
+            <span class="c1"># format</span>
+            <span class="k">for</span> <span class="n">col</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">columns_to_encode</span><span class="p">:</span>
+                <span class="n">df</span><span class="p">[</span><span class="n">col</span><span class="p">]</span> <span class="o">=</span> <span class="n">df</span><span class="p">[</span><span class="n">col</span><span class="p">]</span><span class="o">.</span><span class="n">astype</span><span class="p">(</span><span class="s2">&quot;O&quot;</span><span class="p">)</span>
+
+        <span class="n">unique_column_vals</span><span class="p">:</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="p">{}</span>
+        <span class="k">for</span> <span class="n">c</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">columns_to_encode</span><span class="p">:</span>
+            <span class="n">unique_column_vals</span><span class="p">[</span><span class="n">c</span><span class="p">]</span> <span class="o">=</span> <span class="n">df</span><span class="p">[</span><span class="n">c</span><span class="p">]</span><span class="o">.</span><span class="n">unique</span><span class="p">()</span><span class="o">.</span><span class="n">tolist</span><span class="p">()</span>
+
+        <span class="k">if</span> <span class="ow">not</span> <span class="nb">hasattr</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="s2">&quot;encoding_dict&quot;</span><span class="p">):</span>
+            <span class="c1"># we run the method &#39;partial_fit&#39; for the 1st time</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">encoding_dict</span><span class="p">:</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">]]</span> <span class="o">=</span> <span class="p">{}</span>
+            <span class="k">if</span> <span class="s2">&quot;cls_token&quot;</span> <span class="ow">in</span> <span class="n">unique_column_vals</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">shared_embed</span><span class="p">:</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">encoding_dict</span><span class="p">[</span><span class="s2">&quot;cls_token&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="p">{</span><span class="s2">&quot;[CLS]&quot;</span><span class="p">:</span> <span class="mi">0</span><span class="p">}</span>
+                <span class="k">del</span> <span class="n">unique_column_vals</span><span class="p">[</span><span class="s2">&quot;cls_token&quot;</span><span class="p">]</span>
+
+            <span class="c1"># leave 0 for padding/&quot;unseen&quot; categories. Also we need an</span>
+            <span class="c1"># attribute to keep track of the encoding in case we use</span>
+            <span class="c1"># attention and we do not re-start the index/counter</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">cum_idx</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1</span>
+            <span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="n">unique_column_vals</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">encoding_dict</span><span class="p">[</span><span class="n">k</span><span class="p">]</span> <span class="o">=</span> <span class="p">{</span><span class="n">o</span><span class="p">:</span> <span class="n">i</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">cum_idx</span> <span class="k">for</span> <span class="n">i</span><span class="p">,</span> <span class="n">o</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="n">v</span><span class="p">)}</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">cum_idx</span> <span class="o">=</span> <span class="mi">1</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">reset_embed_idx</span> <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">cum_idx</span> <span class="o">+</span> <span class="nb">len</span><span class="p">(</span><span class="n">v</span><span class="p">)</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="c1"># the &#39;partial_fit&#39; method has already run.</span>
+            <span class="c1"># &quot;cls_token&quot; will have been added already</span>
+            <span class="k">if</span> <span class="s2">&quot;cls_token&quot;</span> <span class="ow">in</span> <span class="n">unique_column_vals</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">shared_embed</span><span class="p">:</span>
+                <span class="k">del</span> <span class="n">unique_column_vals</span><span class="p">[</span><span class="s2">&quot;cls_token&quot;</span><span class="p">]</span>
+
+            <span class="c1"># Classes in the new df/chunk of the dataset that have not been seen</span>
+            <span class="c1"># before</span>
+            <span class="n">unseen_classes</span><span class="p">:</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="p">{}</span>
+            <span class="k">for</span> <span class="n">c</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">columns_to_encode</span><span class="p">:</span>
+                <span class="n">unseen_classes</span><span class="p">[</span><span class="n">c</span><span class="p">]</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span>
+                    <span class="n">np</span><span class="o">.</span><span class="n">setdiff1d</span><span class="p">(</span>
+                        <span class="n">unique_column_vals</span><span class="p">[</span><span class="n">c</span><span class="p">],</span> <span class="nb">list</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">encoding_dict</span><span class="p">[</span><span class="n">c</span><span class="p">]</span><span class="o">.</span><span class="n">keys</span><span class="p">())</span>
+                    <span class="p">)</span>
+                <span class="p">)</span>
+
+            <span class="c1"># leave 0 for padding/&quot;unseen&quot; categories</span>
+            <span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="n">unique_column_vals</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
+                <span class="c1"># if we use attention we need to start encoding from the</span>
+                <span class="c1"># last &#39;overall&#39; encoding index. Otherwise, we use the max</span>
+                <span class="c1"># encoding index per categorical col</span>
+                <span class="n">_idx</span> <span class="o">=</span> <span class="p">(</span>
+                    <span class="nb">max</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">encoding_dict</span><span class="p">[</span><span class="n">k</span><span class="p">]</span><span class="o">.</span><span class="n">values</span><span class="p">())</span> <span class="o">+</span> <span class="mi">1</span>
+                    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">reset_embed_idx</span>
+                    <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">cum_idx</span>
+                <span class="p">)</span>
+                <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">unseen_classes</span><span class="p">[</span><span class="n">k</span><span class="p">])</span> <span class="o">!=</span> <span class="mi">0</span><span class="p">:</span>
+                    <span class="k">for</span> <span class="n">i</span><span class="p">,</span> <span class="n">o</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="n">unseen_classes</span><span class="p">[</span><span class="n">k</span><span class="p">]):</span>
+                        <span class="k">if</span> <span class="n">o</span> <span class="ow">not</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">encoding_dict</span><span class="p">[</span><span class="n">k</span><span class="p">]:</span>
+                            <span class="bp">self</span><span class="o">.</span><span class="n">encoding_dict</span><span class="p">[</span><span class="n">k</span><span class="p">][</span><span class="n">o</span><span class="p">]</span> <span class="o">=</span> <span class="n">i</span> <span class="o">+</span> <span class="n">_idx</span>
+                    <span class="c1"># if self.reset_embed_idx is True it will be 1 anyway</span>
+                    <span class="bp">self</span><span class="o">.</span><span class="n">cum_idx</span> <span class="o">=</span> <span class="p">(</span>
+                        <span class="mi">1</span>
+                        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">reset_embed_idx</span>
+                        <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">cum_idx</span> <span class="o">+</span> <span class="nb">len</span><span class="p">(</span><span class="n">unseen_classes</span><span class="p">[</span><span class="n">k</span><span class="p">])</span>
+                    <span class="p">)</span>
+
+        <span class="k">return</span> <span class="bp">self</span>
+
+    <span class="k">def</span> <span class="nf">fit</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="s2">&quot;LabelEncoder&quot;</span><span class="p">:</span>
+<span class="w">        </span><span class="sd">&quot;&quot;&quot;Simply runs the `partial_fit` method when the data fits in memory</span>
+
+<span class="sd">        Returns</span>
+<span class="sd">        -------</span>
+<span class="sd">        LabelEncoder</span>
+<span class="sd">            `LabelEncoder` fitted object</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="c1"># this is meant to be run when the data fits in memory and therefore,</span>
+        <span class="c1"># we do not want to mutate the original df, so we copy it</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">partial_fit</span><span class="p">(</span><span class="n">df</span><span class="o">.</span><span class="n">copy</span><span class="p">())</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">inverse_encoding_dict</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">create_inverse_encoding_dict</span><span class="p">()</span>
 
-                  <details class="quote">
-                    <summary>Source code in <code>pytorch_widedeep/utils/deeptabular_utils.py</code></summary>
-                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">50</span>
-<span class="normal">51</span>
-<span class="normal">52</span>
-<span class="normal">53</span>
-<span class="normal">54</span>
-<span class="normal">55</span>
-<span class="normal">56</span>
-<span class="normal">57</span>
-<span class="normal">58</span>
-<span class="normal">59</span>
-<span class="normal">60</span>
-<span class="normal">61</span>
-<span class="normal">62</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="nd">@alias</span><span class="p">(</span><span class="s2">&quot;with_attention&quot;</span><span class="p">,</span> <span class="p">[</span><span class="s2">&quot;for_transformer&quot;</span><span class="p">])</span>
-<span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
-    <span class="bp">self</span><span class="p">,</span>
-    <span class="n">columns_to_encode</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">with_attention</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
-    <span class="n">shared_embed</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
-<span class="p">):</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">columns_to_encode</span> <span class="o">=</span> <span class="n">columns_to_encode</span>
-
-    <span class="bp">self</span><span class="o">.</span><span class="n">shared_embed</span> <span class="o">=</span> <span class="n">shared_embed</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">with_attention</span> <span class="o">=</span> <span class="n">with_attention</span>
-
-    <span class="bp">self</span><span class="o">.</span><span class="n">reset_embed_idx</span> <span class="o">=</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">with_attention</span> <span class="ow">or</span> <span class="bp">self</span><span class="o">.</span><span class="n">shared_embed</span>
+        <span class="k">return</span> <span class="bp">self</span>
+
+    <span class="k">def</span> <span class="nf">transform</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">:</span>
+<span class="w">        </span><span class="sd">&quot;&quot;&quot;Label Encoded the categories in `columns_to_encode`</span>
+
+<span class="sd">        Returns</span>
+<span class="sd">        -------</span>
+<span class="sd">        pd.DataFrame</span>
+<span class="sd">            label-encoded dataframe</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="k">try</span><span class="p">:</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">encoding_dict</span>
+        <span class="k">except</span> <span class="ne">AttributeError</span><span class="p">:</span>
+            <span class="k">raise</span> <span class="n">NotFittedError</span><span class="p">(</span>
+                <span class="s2">&quot;This LabelEncoder instance is not fitted yet. &quot;</span>
+                <span class="s2">&quot;Call &#39;fit&#39; with appropriate arguments before using this LabelEncoder.&quot;</span>
+            <span class="p">)</span>
+
+        <span class="n">df_inp</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
+        <span class="c1"># sanity check to make sure all categorical columns are in an adequate</span>
+        <span class="c1"># format</span>
+        <span class="k">for</span> <span class="n">col</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">columns_to_encode</span><span class="p">:</span>  <span class="c1"># type: ignore</span>
+            <span class="n">df_inp</span><span class="p">[</span><span class="n">col</span><span class="p">]</span> <span class="o">=</span> <span class="n">df_inp</span><span class="p">[</span><span class="n">col</span><span class="p">]</span><span class="o">.</span><span class="n">astype</span><span class="p">(</span><span class="s2">&quot;O&quot;</span><span class="p">)</span>
+
+        <span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">encoding_dict</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
+            <span class="n">df_inp</span><span class="p">[</span><span class="n">k</span><span class="p">]</span> <span class="o">=</span> <span class="n">df_inp</span><span class="p">[</span><span class="n">k</span><span class="p">]</span><span class="o">.</span><span class="n">apply</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">v</span><span class="p">[</span><span class="n">x</span><span class="p">]</span> <span class="k">if</span> <span class="n">x</span> <span class="ow">in</span> <span class="n">v</span><span class="o">.</span><span class="n">keys</span><span class="p">()</span> <span class="k">else</span> <span class="mi">0</span><span class="p">)</span>
+
+        <span class="k">return</span> <span class="n">df_inp</span>
+
+    <span class="k">def</span> <span class="nf">fit_transform</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">:</span>
+<span class="w">        </span><span class="sd">&quot;&quot;&quot;Combines `fit` and `transform`</span>
+
+<span class="sd">        Examples</span>
+<span class="sd">        --------</span>
+
+<span class="sd">        &gt;&gt;&gt; import pandas as pd</span>
+<span class="sd">        &gt;&gt;&gt; from pytorch_widedeep.utils import LabelEncoder</span>
+<span class="sd">        &gt;&gt;&gt; df = pd.DataFrame({&#39;col1&#39;: [1,2,3], &#39;col2&#39;: [&#39;me&#39;, &#39;you&#39;, &#39;him&#39;]})</span>
+<span class="sd">        &gt;&gt;&gt; columns_to_encode = [&#39;col2&#39;]</span>
+<span class="sd">        &gt;&gt;&gt; encoder = LabelEncoder(columns_to_encode)</span>
+<span class="sd">        &gt;&gt;&gt; encoder.fit_transform(df)</span>
+<span class="sd">           col1  col2</span>
+<span class="sd">        0     1     1</span>
+<span class="sd">        1     2     2</span>
+<span class="sd">        2     3     3</span>
+<span class="sd">        &gt;&gt;&gt; encoder.encoding_dict</span>
+<span class="sd">        {&#39;col2&#39;: {&#39;me&#39;: 1, &#39;you&#39;: 2, &#39;him&#39;: 3}}</span>
+
+<span class="sd">        Returns</span>
+<span class="sd">        -------</span>
+<span class="sd">        pd.DataFrame</span>
+<span class="sd">            label-encoded dataframe</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">df</span><span class="p">)</span><span class="o">.</span><span class="n">transform</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">create_inverse_encoding_dict</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">int</span><span class="p">,</span> <span class="nb">str</span><span class="p">]]:</span>
+        <span class="n">inverse_encoding_dict</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">()</span>
+        <span class="k">for</span> <span class="n">c</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">encoding_dict</span><span class="p">:</span>
+            <span class="n">inverse_encoding_dict</span><span class="p">[</span><span class="n">c</span><span class="p">]</span> <span class="o">=</span> <span class="p">{</span><span class="n">v</span><span class="p">:</span> <span class="n">k</span> <span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">encoding_dict</span><span class="p">[</span><span class="n">c</span><span class="p">]</span><span class="o">.</span><span class="n">items</span><span class="p">()}</span>
+            <span class="n">inverse_encoding_dict</span><span class="p">[</span><span class="n">c</span><span class="p">][</span><span class="mi">0</span><span class="p">]</span> <span class="o">=</span> <span class="s2">&quot;unseen&quot;</span>
+        <span class="k">return</span> <span class="n">inverse_encoding_dict</span>
+
+    <span class="k">def</span> <span class="nf">inverse_transform</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">:</span>
+<span class="w">        </span><span class="sd">&quot;&quot;&quot;Returns the original categories</span>
+
+<span class="sd">        Examples</span>
+<span class="sd">        --------</span>
+
+<span class="sd">        &gt;&gt;&gt; import pandas as pd</span>
+<span class="sd">        &gt;&gt;&gt; from pytorch_widedeep.utils import LabelEncoder</span>
+<span class="sd">        &gt;&gt;&gt; df = pd.DataFrame({&#39;col1&#39;: [1,2,3], &#39;col2&#39;: [&#39;me&#39;, &#39;you&#39;, &#39;him&#39;]})</span>
+<span class="sd">        &gt;&gt;&gt; columns_to_encode = [&#39;col2&#39;]</span>
+<span class="sd">        &gt;&gt;&gt; encoder = LabelEncoder(columns_to_encode)</span>
+<span class="sd">        &gt;&gt;&gt; df_enc = encoder.fit_transform(df)</span>
+<span class="sd">        &gt;&gt;&gt; encoder.inverse_transform(df_enc)</span>
+<span class="sd">           col1 col2</span>
+<span class="sd">        0     1   me</span>
+<span class="sd">        1     2  you</span>
+<span class="sd">        2     3  him</span>
+
+<span class="sd">        Returns</span>
+<span class="sd">        -------</span>
+<span class="sd">        pd.DataFrame</span>
+<span class="sd">            DataFrame with original categories</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+
+        <span class="k">if</span> <span class="ow">not</span> <span class="nb">hasattr</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="s2">&quot;inverse_encoding_dict&quot;</span><span class="p">):</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">inverse_encoding_dict</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">create_inverse_encoding_dict</span><span class="p">()</span>
+
+        <span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">inverse_encoding_dict</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
+            <span class="n">df</span><span class="p">[</span><span class="n">k</span><span class="p">]</span> <span class="o">=</span> <span class="n">df</span><span class="p">[</span><span class="n">k</span><span class="p">]</span><span class="o">.</span><span class="n">apply</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">v</span><span class="p">[</span><span class="n">x</span><span class="p">])</span>
+
+        <span class="k">return</span> <span class="n">df</span>
+
+    <span class="k">def</span> <span class="fm">__repr__</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">str</span><span class="p">:</span>
+        <span class="n">list_of_params</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">columns_to_encode</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="n">list_of_params</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">&quot;columns_to_encode=</span><span class="si">{columns_to_encode}</span><span class="s2">&quot;</span><span class="p">)</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">with_attention</span><span class="p">:</span>
+            <span class="n">list_of_params</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">&quot;with_attention=</span><span class="si">{with_attention}</span><span class="s2">&quot;</span><span class="p">)</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">shared_embed</span><span class="p">:</span>
+            <span class="n">list_of_params</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">&quot;shared_embed=</span><span class="si">{shared_embed}</span><span class="s2">&quot;</span><span class="p">)</span>
+        <span class="n">all_params</span> <span class="o">=</span> <span class="s2">&quot;, &quot;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">list_of_params</span><span class="p">)</span>
+        <span class="k">return</span> <span class="sa">f</span><span class="s2">&quot;LabelEncoder(</span><span class="si">{</span><span class="n">all_params</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="bp">self</span><span class="o">.</span><span class="vm">__dict__</span><span class="p">)</span><span class="si">}</span><span class="s2">)&quot;</span>
 </code></pre></div></td></tr></table></div>
-                  </details>
+              </details>
 
 
 
@@ -1745,16 +2239,27 @@ <h3 id="pytorch_widedeep.utils.deeptabular_utils.LabelEncoder.partial_fit" class
         <p>Main method. Creates encoding attributes.</p>
 
 
-<p><span class="doc-section-title">Returns:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-              <code><a class="autorefs autorefs-internal" title="pytorch_widedeep.utils.deeptabular_utils.LabelEncoder" href="#pytorch_widedeep.utils.deeptabular_utils.LabelEncoder">LabelEncoder</a></code>
-          –
-          <div class="doc-md-description">
-            <p><code>LabelEncoder</code> fitted object</p>
-          </div>
-        </li>
-    </ul>
+    <p><span class="doc-section-title">Returns:</span></p>
+    <table>
+      <thead>
+        <tr>
+          <th>Type</th>
+          <th>Description</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td>
+                  <code><a class="autorefs autorefs-internal" title="pytorch_widedeep.utils.deeptabular_utils.LabelEncoder" href="#pytorch_widedeep.utils.deeptabular_utils.LabelEncoder">LabelEncoder</a></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p><code>LabelEncoder</code> fitted object</p>
+              </div>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
             <details class="quote">
               <summary>Source code in <code>pytorch_widedeep/utils/deeptabular_utils.py</code></summary>
@@ -1929,16 +2434,27 @@ <h3 id="pytorch_widedeep.utils.deeptabular_utils.LabelEncoder.fit" class="doc do
         <p>Simply runs the <code>partial_fit</code> method when the data fits in memory</p>
 
 
-<p><span class="doc-section-title">Returns:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-              <code><a class="autorefs autorefs-internal" title="pytorch_widedeep.utils.deeptabular_utils.LabelEncoder" href="#pytorch_widedeep.utils.deeptabular_utils.LabelEncoder">LabelEncoder</a></code>
-          –
-          <div class="doc-md-description">
-            <p><code>LabelEncoder</code> fitted object</p>
-          </div>
-        </li>
-    </ul>
+    <p><span class="doc-section-title">Returns:</span></p>
+    <table>
+      <thead>
+        <tr>
+          <th>Type</th>
+          <th>Description</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td>
+                  <code><a class="autorefs autorefs-internal" title="pytorch_widedeep.utils.deeptabular_utils.LabelEncoder" href="#pytorch_widedeep.utils.deeptabular_utils.LabelEncoder">LabelEncoder</a></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p><code>LabelEncoder</code> fitted object</p>
+              </div>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
             <details class="quote">
               <summary>Source code in <code>pytorch_widedeep/utils/deeptabular_utils.py</code></summary>
@@ -1993,16 +2509,27 @@ <h3 id="pytorch_widedeep.utils.deeptabular_utils.LabelEncoder.transform" class="
         <p>Label Encoded the categories in <code>columns_to_encode</code></p>
 
 
-<p><span class="doc-section-title">Returns:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-              <code><span title="pandas.DataFrame">DataFrame</span></code>
-          –
-          <div class="doc-md-description">
-            <p>label-encoded dataframe</p>
-          </div>
-        </li>
-    </ul>
+    <p><span class="doc-section-title">Returns:</span></p>
+    <table>
+      <thead>
+        <tr>
+          <th>Type</th>
+          <th>Description</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td>
+                  <code><span title="pandas.DataFrame">DataFrame</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>label-encoded dataframe</p>
+              </div>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
             <details class="quote">
               <summary>Source code in <code>pytorch_widedeep/utils/deeptabular_utils.py</code></summary>
@@ -2095,16 +2622,27 @@ <h3 id="pytorch_widedeep.utils.deeptabular_utils.LabelEncoder.fit_transform" cla
 </code></pre></div>
 
 
-<p><span class="doc-section-title">Returns:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-              <code><span title="pandas.DataFrame">DataFrame</span></code>
-          –
-          <div class="doc-md-description">
-            <p>label-encoded dataframe</p>
-          </div>
-        </li>
-    </ul>
+    <p><span class="doc-section-title">Returns:</span></p>
+    <table>
+      <thead>
+        <tr>
+          <th>Type</th>
+          <th>Description</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td>
+                  <code><span title="pandas.DataFrame">DataFrame</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>label-encoded dataframe</p>
+              </div>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
             <details class="quote">
               <summary>Source code in <code>pytorch_widedeep/utils/deeptabular_utils.py</code></summary>
@@ -2194,16 +2732,27 @@ <h3 id="pytorch_widedeep.utils.deeptabular_utils.LabelEncoder.inverse_transform"
 </code></pre></div>
 
 
-<p><span class="doc-section-title">Returns:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-              <code><span title="pandas.DataFrame">DataFrame</span></code>
-          –
-          <div class="doc-md-description">
-            <p>DataFrame with original categories</p>
-          </div>
-        </li>
-    </ul>
+    <p><span class="doc-section-title">Returns:</span></p>
+    <table>
+      <thead>
+        <tr>
+          <th>Type</th>
+          <th>Description</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td>
+                  <code><span title="pandas.DataFrame">DataFrame</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>DataFrame with original categories</p>
+              </div>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
             <details class="quote">
               <summary>Source code in <code>pytorch_widedeep/utils/deeptabular_utils.py</code></summary>
diff --git a/mkdocs/site/pytorch-widedeep/utils/fastai_transforms.html b/mkdocs/site/pytorch-widedeep/utils/fastai_transforms.html
index 920e567d..900ecef5 100644
--- a/mkdocs/site/pytorch-widedeep/utils/fastai_transforms.html
+++ b/mkdocs/site/pytorch-widedeep/utils/fastai_transforms.html
@@ -1396,7 +1396,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../../examples/16_Self_Supervised_Pretraning_pt1.ipynb" class="md-nav__link">
+      <a href="../../examples/15_Self_Supervised_Pretraning_pt1.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1417,7 +1417,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../../examples/16_Self_Supervised_Pretraning_pt2.ipynb" class="md-nav__link">
+      <a href="../../examples/15_Self_Supervised_Pretraning_pt2.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1438,7 +1438,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../../examples/17_Usign_a_custom_hugging_face_model.ipynb" class="md-nav__link">
+      <a href="../../examples/16_Usign_a_custom_hugging_face_model.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1459,7 +1459,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../../examples/18_feature_importance_via_attention_weights.ipynb" class="md-nav__link">
+      <a href="../../examples/17_feature_importance_via_attention_weights.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1480,7 +1480,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../../examples/19_wide_and_deep_for_recsys_pt1.ipynb" class="md-nav__link">
+      <a href="../../examples/18_wide_and_deep_for_recsys_pt1.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1501,7 +1501,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../../examples/19_wide_and_deep_for_recsys_pt2.ipynb" class="md-nav__link">
+      <a href="../../examples/18_wide_and_deep_for_recsys_pt2.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1522,7 +1522,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../../examples/20_load_from_folder_functionality.ipynb" class="md-nav__link">
+      <a href="../../examples/19_load_from_folder_functionality.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1543,7 +1543,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../../examples/21_Using_huggingface_within_widedeep.ipynb" class="md-nav__link">
+      <a href="../../examples/20_Using_huggingface_within_widedeep.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1603,10 +1603,10 @@
 
 
 <h1 id="fastai-transforms">Fastai transforms<a class="headerlink" href="#fastai-transforms" title="Permanent link">&para;</a></h1>
-<p>I directly copied and pasted part of the <code>transforms.py</code> module from
+<p>We directly copied and pasted part of the <code>transforms.py</code> module from
 the <code>fastai</code> library (from an old version). The reason to do such a thing is because
 <code>pytorch_widedeep</code> only needs the <code>Tokenizer</code> and the <code>Vocab</code> classes
-there. This way I avoid extra dependencies. Credit for all the code in the
+there. This way we avoid extra dependencies. Credit for all the code in the
 <code>fastai_transforms</code> module in this <code>pytorch-widedeep</code> package goes to
 Jeremy Howard and the <code>fastai</code> team. I only include the documentation here for
 completion, but I strongly advise the user to read the <code>fastai</code> <a href="https://docs.fast.ai/">documentation</a>.</p>
@@ -1621,15 +1621,7 @@ <h2 id="pytorch_widedeep.utils.fastai_transforms.Tokenizer" class="doc doc-headi
 
 
 <a href="#pytorch_widedeep.utils.fastai_transforms.Tokenizer" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="nf">Tokenizer</span><span class="p">(</span>
-    <span class="n">tok_func</span><span class="o">=</span><span class="n">SpacyTokenizer</span><span class="p">,</span>
-    <span class="n">lang</span><span class="o">=</span><span class="s2">&quot;en&quot;</span><span class="p">,</span>
-    <span class="n">pre_rules</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">post_rules</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">special_cases</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-    <span class="n">n_cpus</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
-<span class="p">)</span>
-</code></pre></div>
+
 
     <div class="doc doc-contents first">
 
@@ -1641,77 +1633,139 @@ <h2 id="pytorch_widedeep.utils.fastai_transforms.Tokenizer" class="doc doc-headi
 
 
 <p><span class="doc-section-title">Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>tok_func</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Callable">Callable</span></code>, default:
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+          <th>Default</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>tok_func</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Callable">Callable</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Tokenizer Object. See <code>pytorch_widedeep.utils.fastai_transforms.SpacyTokenizer</code></p>
+              </div>
+            </td>
+            <td>
                   <code><span title="pytorch_widedeep.utils.fastai_transforms.SpacyTokenizer">SpacyTokenizer</span></code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Tokenizer Object. See <code>pytorch_widedeep.utils.fastai_transforms.SpacyTokenizer</code></p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>lang</code></b>
-              (<code>str</code>, default:
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>lang</code></td>
+            <td>
+                  <code>str</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Text's Language</p>
+              </div>
+            </td>
+            <td>
                   <code>&#39;en&#39;</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Text's Language</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>pre_rules</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.ListRules">ListRules</span>]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Custom type: <code>Collection[Callable[[str], str]]</code>. These are
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>pre_rules</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.ListRules">ListRules</span>]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Custom type: <code>Collection[Callable[[str], str]]</code>. These are
 <code>Callable</code> objects that will be applied to the text (str) directly as
 <code>rule(tok)</code> before being tokenized.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>post_rules</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.ListRules">ListRules</span>]</code>, default:
+              </div>
+            </td>
+            <td>
                   <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Custom type: <code>Collection[Callable[[str], str]]</code>. These are
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>post_rules</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.ListRules">ListRules</span>]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Custom type: <code>Collection[Callable[[str], str]]</code>. These are
 <code>Callable</code> objects that will be applied to the tokens as
 <code>rule(tokens)</code> after the text has been tokenized.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>special_cases</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Collection">Collection</span>[str]]</code>, default:
+              </div>
+            </td>
+            <td>
                   <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>special cases to be added to the tokenizer via <code>Spacy</code>'s
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>special_cases</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Collection">Collection</span>[str]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>special cases to be added to the tokenizer via <code>Spacy</code>'s
 <code>add_special_case</code> method</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>n_cpus</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[int]</code>, default:
+              </div>
+            </td>
+            <td>
                   <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>number of CPUs to used during the tokenization process</p>
-          </div>
-        </li>
-    </ul>
-
-                  <details class="quote">
-                    <summary>Source code in <code>pytorch_widedeep/utils/fastai_transforms.py</code></summary>
-                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">255</span>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>n_cpus</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[int]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>number of CPUs to used during the tokenization process</p>
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+      </tbody>
+    </table>
+
+              <details class="quote">
+                <summary>Source code in <code>pytorch_widedeep/utils/fastai_transforms.py</code></summary>
+                <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">227</span>
+<span class="normal">228</span>
+<span class="normal">229</span>
+<span class="normal">230</span>
+<span class="normal">231</span>
+<span class="normal">232</span>
+<span class="normal">233</span>
+<span class="normal">234</span>
+<span class="normal">235</span>
+<span class="normal">236</span>
+<span class="normal">237</span>
+<span class="normal">238</span>
+<span class="normal">239</span>
+<span class="normal">240</span>
+<span class="normal">241</span>
+<span class="normal">242</span>
+<span class="normal">243</span>
+<span class="normal">244</span>
+<span class="normal">245</span>
+<span class="normal">246</span>
+<span class="normal">247</span>
+<span class="normal">248</span>
+<span class="normal">249</span>
+<span class="normal">250</span>
+<span class="normal">251</span>
+<span class="normal">252</span>
+<span class="normal">253</span>
+<span class="normal">254</span>
+<span class="normal">255</span>
 <span class="normal">256</span>
 <span class="normal">257</span>
 <span class="normal">258</span>
@@ -1726,24 +1780,190 @@ <h2 id="pytorch_widedeep.utils.fastai_transforms.Tokenizer" class="doc doc-headi
 <span class="normal">267</span>
 <span class="normal">268</span>
 <span class="normal">269</span>
-<span class="normal">270</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
-    <span class="bp">self</span><span class="p">,</span>
-    <span class="n">tok_func</span><span class="p">:</span> <span class="n">Callable</span> <span class="o">=</span> <span class="n">SpacyTokenizer</span><span class="p">,</span>
-    <span class="n">lang</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;en&quot;</span><span class="p">,</span>
-    <span class="n">pre_rules</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">ListRules</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">post_rules</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">ListRules</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">special_cases</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Collection</span><span class="p">[</span><span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">n_cpus</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-<span class="p">):</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">tok_func</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">lang</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">special_cases</span> <span class="o">=</span> <span class="n">tok_func</span><span class="p">,</span> <span class="n">lang</span><span class="p">,</span> <span class="n">special_cases</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">pre_rules</span> <span class="o">=</span> <span class="n">ifnone</span><span class="p">(</span><span class="n">pre_rules</span><span class="p">,</span> <span class="n">defaults</span><span class="o">.</span><span class="n">text_pre_rules</span><span class="p">)</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">post_rules</span> <span class="o">=</span> <span class="n">ifnone</span><span class="p">(</span><span class="n">post_rules</span><span class="p">,</span> <span class="n">defaults</span><span class="o">.</span><span class="n">text_post_rules</span><span class="p">)</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">special_cases</span> <span class="o">=</span> <span class="p">(</span>
-        <span class="n">special_cases</span> <span class="k">if</span> <span class="n">special_cases</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="k">else</span> <span class="n">defaults</span><span class="o">.</span><span class="n">text_spec_tok</span>
-    <span class="p">)</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">n_cpus</span> <span class="o">=</span> <span class="n">ifnone</span><span class="p">(</span><span class="n">n_cpus</span><span class="p">,</span> <span class="n">defaults</span><span class="o">.</span><span class="n">cpus</span><span class="p">)</span>
+<span class="normal">270</span>
+<span class="normal">271</span>
+<span class="normal">272</span>
+<span class="normal">273</span>
+<span class="normal">274</span>
+<span class="normal">275</span>
+<span class="normal">276</span>
+<span class="normal">277</span>
+<span class="normal">278</span>
+<span class="normal">279</span>
+<span class="normal">280</span>
+<span class="normal">281</span>
+<span class="normal">282</span>
+<span class="normal">283</span>
+<span class="normal">284</span>
+<span class="normal">285</span>
+<span class="normal">286</span>
+<span class="normal">287</span>
+<span class="normal">288</span>
+<span class="normal">289</span>
+<span class="normal">290</span>
+<span class="normal">291</span>
+<span class="normal">292</span>
+<span class="normal">293</span>
+<span class="normal">294</span>
+<span class="normal">295</span>
+<span class="normal">296</span>
+<span class="normal">297</span>
+<span class="normal">298</span>
+<span class="normal">299</span>
+<span class="normal">300</span>
+<span class="normal">301</span>
+<span class="normal">302</span>
+<span class="normal">303</span>
+<span class="normal">304</span>
+<span class="normal">305</span>
+<span class="normal">306</span>
+<span class="normal">307</span>
+<span class="normal">308</span>
+<span class="normal">309</span>
+<span class="normal">310</span>
+<span class="normal">311</span>
+<span class="normal">312</span>
+<span class="normal">313</span>
+<span class="normal">314</span>
+<span class="normal">315</span>
+<span class="normal">316</span>
+<span class="normal">317</span>
+<span class="normal">318</span>
+<span class="normal">319</span>
+<span class="normal">320</span>
+<span class="normal">321</span>
+<span class="normal">322</span>
+<span class="normal">323</span>
+<span class="normal">324</span>
+<span class="normal">325</span>
+<span class="normal">326</span>
+<span class="normal">327</span>
+<span class="normal">328</span>
+<span class="normal">329</span>
+<span class="normal">330</span>
+<span class="normal">331</span>
+<span class="normal">332</span>
+<span class="normal">333</span>
+<span class="normal">334</span>
+<span class="normal">335</span>
+<span class="normal">336</span>
+<span class="normal">337</span>
+<span class="normal">338</span>
+<span class="normal">339</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">class</span> <span class="nc">Tokenizer</span><span class="p">:</span>
+<span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Class to combine a series of rules and a tokenizer function to tokenize</span>
+<span class="sd">    text with multiprocessing.</span>
+
+<span class="sd">    Setting some of the parameters of this class require perhaps some</span>
+<span class="sd">    familiarity with the source code.</span>
+
+<span class="sd">    Parameters</span>
+<span class="sd">    ----------</span>
+<span class="sd">    tok_func: Callable, default = ``SpacyTokenizer``</span>
+<span class="sd">        Tokenizer Object. See `pytorch_widedeep.utils.fastai_transforms.SpacyTokenizer`</span>
+<span class="sd">    lang: str, default = &quot;en&quot;</span>
+<span class="sd">        Text&#39;s Language</span>
+<span class="sd">    pre_rules: ListRules, Optional, default = None</span>
+<span class="sd">        Custom type: ``Collection[Callable[[str], str]]``. These are</span>
+<span class="sd">        `Callable` objects that will be applied to the text (str) directly as</span>
+<span class="sd">        `rule(tok)` before being tokenized.</span>
+<span class="sd">    post_rules: ListRules, Optional, default = None</span>
+<span class="sd">        Custom type: ``Collection[Callable[[str], str]]``. These are</span>
+<span class="sd">        `Callable` objects that will be applied to the tokens as</span>
+<span class="sd">        `rule(tokens)` after the text has been tokenized.</span>
+<span class="sd">    special_cases: Collection, Optional, default= None</span>
+<span class="sd">        special cases to be added to the tokenizer via ``Spacy``&#39;s</span>
+<span class="sd">        ``add_special_case`` method</span>
+<span class="sd">    n_cpus: int, Optional, default = None</span>
+<span class="sd">        number of CPUs to used during the tokenization process</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+
+    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span>
+        <span class="n">tok_func</span><span class="p">:</span> <span class="n">Callable</span> <span class="o">=</span> <span class="n">SpacyTokenizer</span><span class="p">,</span>
+        <span class="n">lang</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;en&quot;</span><span class="p">,</span>
+        <span class="n">pre_rules</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">ListRules</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">post_rules</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">ListRules</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">special_cases</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Collection</span><span class="p">[</span><span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">n_cpus</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+    <span class="p">):</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">tok_func</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">lang</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">special_cases</span> <span class="o">=</span> <span class="n">tok_func</span><span class="p">,</span> <span class="n">lang</span><span class="p">,</span> <span class="n">special_cases</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">pre_rules</span> <span class="o">=</span> <span class="n">ifnone</span><span class="p">(</span><span class="n">pre_rules</span><span class="p">,</span> <span class="n">defaults</span><span class="o">.</span><span class="n">text_pre_rules</span><span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">post_rules</span> <span class="o">=</span> <span class="n">ifnone</span><span class="p">(</span><span class="n">post_rules</span><span class="p">,</span> <span class="n">defaults</span><span class="o">.</span><span class="n">text_post_rules</span><span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">special_cases</span> <span class="o">=</span> <span class="p">(</span>
+            <span class="n">special_cases</span> <span class="k">if</span> <span class="n">special_cases</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="k">else</span> <span class="n">defaults</span><span class="o">.</span><span class="n">text_spec_tok</span>
+        <span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">n_cpus</span> <span class="o">=</span> <span class="n">ifnone</span><span class="p">(</span><span class="n">n_cpus</span><span class="p">,</span> <span class="n">defaults</span><span class="o">.</span><span class="n">cpus</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="fm">__repr__</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">str</span><span class="p">:</span>
+        <span class="n">res</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">&quot;Tokenizer </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">tok_func</span><span class="o">.</span><span class="vm">__name__</span><span class="si">}</span><span class="s2"> in </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">lang</span><span class="si">}</span><span class="s2"> with the following rules:</span><span class="se">\n</span><span class="s2">&quot;</span>
+        <span class="k">for</span> <span class="n">rule</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">pre_rules</span><span class="p">:</span>
+            <span class="n">res</span> <span class="o">+=</span> <span class="sa">f</span><span class="s2">&quot; - </span><span class="si">{</span><span class="n">rule</span><span class="o">.</span><span class="vm">__name__</span><span class="si">}</span><span class="se">\n</span><span class="s2">&quot;</span>
+        <span class="k">for</span> <span class="n">rule</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">post_rules</span><span class="p">:</span>
+            <span class="n">res</span> <span class="o">+=</span> <span class="sa">f</span><span class="s2">&quot; - </span><span class="si">{</span><span class="n">rule</span><span class="o">.</span><span class="vm">__name__</span><span class="si">}</span><span class="se">\n</span><span class="s2">&quot;</span>
+        <span class="k">return</span> <span class="n">res</span>
+
+    <span class="k">def</span> <span class="nf">process_text</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">t</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">tok</span><span class="p">:</span> <span class="n">BaseTokenizer</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]:</span>
+<span class="w">        </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Process and tokenize one text ``t`` with tokenizer ``tok``.</span>
+
+<span class="sd">        Parameters</span>
+<span class="sd">        ----------</span>
+<span class="sd">        t: str</span>
+<span class="sd">            text to be processed and tokenized</span>
+<span class="sd">        tok: ``BaseTokenizer``</span>
+<span class="sd">            Instance of `BaseTokenizer`. See</span>
+<span class="sd">            `pytorch_widedeep.utils.fastai_transforms.BaseTokenizer`</span>
+
+<span class="sd">        Returns</span>
+<span class="sd">        -------</span>
+<span class="sd">        List[str]</span>
+<span class="sd">            List of tokens</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="k">for</span> <span class="n">rule</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">pre_rules</span><span class="p">:</span>
+            <span class="n">t</span> <span class="o">=</span> <span class="n">rule</span><span class="p">(</span><span class="n">t</span><span class="p">)</span>
+        <span class="n">toks</span> <span class="o">=</span> <span class="n">tok</span><span class="o">.</span><span class="n">tokenizer</span><span class="p">(</span><span class="n">t</span><span class="p">)</span>
+        <span class="k">for</span> <span class="n">rule</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">post_rules</span><span class="p">:</span>
+            <span class="n">toks</span> <span class="o">=</span> <span class="n">rule</span><span class="p">(</span><span class="n">toks</span><span class="p">)</span>
+        <span class="k">return</span> <span class="n">toks</span>
+
+    <span class="k">def</span> <span class="nf">_process_all_1</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">texts</span><span class="p">:</span> <span class="n">Collection</span><span class="p">[</span><span class="nb">str</span><span class="p">])</span> <span class="o">-&gt;</span> <span class="n">List</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]]:</span>
+<span class="w">        </span><span class="sd">&quot;&quot;&quot;Process a list of ``texts`` in one process.&quot;&quot;&quot;</span>
+
+        <span class="n">tok</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">tok_func</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">lang</span><span class="p">)</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">special_cases</span><span class="p">:</span>
+            <span class="n">tok</span><span class="o">.</span><span class="n">add_special_cases</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">special_cases</span><span class="p">)</span>
+        <span class="k">return</span> <span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">process_text</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="n">t</span><span class="p">),</span> <span class="n">tok</span><span class="p">)</span> <span class="k">for</span> <span class="n">t</span> <span class="ow">in</span> <span class="n">texts</span><span class="p">]</span>
+
+    <span class="k">def</span> <span class="nf">process_all</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">texts</span><span class="p">:</span> <span class="n">Collection</span><span class="p">[</span><span class="nb">str</span><span class="p">])</span> <span class="o">-&gt;</span> <span class="n">List</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]]:</span>
+<span class="w">        </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Process a list of texts. Parallel execution of ``process_text``.</span>
+
+<span class="sd">        Examples</span>
+<span class="sd">        --------</span>
+<span class="sd">        &gt;&gt;&gt; from pytorch_widedeep.utils import Tokenizer</span>
+<span class="sd">        &gt;&gt;&gt; texts = [&#39;Machine learning is great&#39;, &#39;but building stuff is even better&#39;]</span>
+<span class="sd">        &gt;&gt;&gt; tok = Tokenizer()</span>
+<span class="sd">        &gt;&gt;&gt; tok.process_all(texts)</span>
+<span class="sd">        [[&#39;xxmaj&#39;, &#39;machine&#39;, &#39;learning&#39;, &#39;is&#39;, &#39;great&#39;], [&#39;but&#39;, &#39;building&#39;, &#39;stuff&#39;, &#39;is&#39;, &#39;even&#39;, &#39;better&#39;]]</span>
+
+<span class="sd">        :information_source: **NOTE**:</span>
+<span class="sd">        Note the token ``TK_MAJ`` (`xxmaj`), used to indicate the</span>
+<span class="sd">        next word begins with a capital in the original text. For more</span>
+<span class="sd">        details of special tokens please see the [``fastai`` docs](https://docs.fast.ai/text.core.html#Tokenizing).</span>
+
+<span class="sd">        Returns</span>
+<span class="sd">        -------</span>
+<span class="sd">        List[List[str]]</span>
+<span class="sd">            List containing lists of tokens. One list per &quot;_document_&quot;</span>
+
+<span class="sd">        &quot;&quot;&quot;</span>
+
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">n_cpus</span> <span class="o">&lt;=</span> <span class="mi">1</span><span class="p">:</span>
+            <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_process_all_1</span><span class="p">(</span><span class="n">texts</span><span class="p">)</span>
+        <span class="k">with</span> <span class="n">ProcessPoolExecutor</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">n_cpus</span><span class="p">)</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
+            <span class="k">return</span> <span class="nb">sum</span><span class="p">(</span>
+                <span class="n">e</span><span class="o">.</span><span class="n">map</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_process_all_1</span><span class="p">,</span> <span class="n">partition_by_cores</span><span class="p">(</span><span class="n">texts</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">n_cpus</span><span class="p">)),</span> <span class="p">[]</span>
+            <span class="p">)</span>
 </code></pre></div></td></tr></table></div>
-                  </details>
+              </details>
 
 
 
@@ -1774,37 +1994,70 @@ <h3 id="pytorch_widedeep.utils.fastai_transforms.Tokenizer.process_text" class="
 
 
 <p><span class="doc-section-title">Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>t</code></b>
-              (<code>str</code>)
-          –
-          <div class="doc-md-description">
-            <p>text to be processed and tokenized</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>tok</code></b>
-              (<code><span title="pytorch_widedeep.utils.fastai_transforms.BaseTokenizer">BaseTokenizer</span></code>)
-          –
-          <div class="doc-md-description">
-            <p>Instance of <code>BaseTokenizer</code>. See
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+          <th>Default</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>t</code></td>
+            <td>
+                  <code>str</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>text to be processed and tokenized</p>
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>tok</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.utils.fastai_transforms.BaseTokenizer">BaseTokenizer</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Instance of <code>BaseTokenizer</code>. See
 <code>pytorch_widedeep.utils.fastai_transforms.BaseTokenizer</code></p>
-          </div>
-        </li>
-    </ul>
-
-
-<p><span class="doc-section-title">Returns:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-              <code><span title="pytorch_widedeep.wdtypes.List">List</span>[str]</code>
-          –
-          <div class="doc-md-description">
-            <p>List of tokens</p>
-          </div>
-        </li>
-    </ul>
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+      </tbody>
+    </table>
+
+
+    <p><span class="doc-section-title">Returns:</span></p>
+    <table>
+      <thead>
+        <tr>
+          <th>Type</th>
+          <th>Description</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.List">List</span>[str]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>List of tokens</p>
+              </div>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
             <details class="quote">
               <summary>Source code in <code>pytorch_widedeep/utils/fastai_transforms.py</code></summary>
@@ -1886,16 +2139,27 @@ <h3 id="pytorch_widedeep.utils.fastai_transforms.Tokenizer.process_all" class="d
 details of special tokens please see the <a href="https://docs.fast.ai/text.core.html#Tokenizing"><code>fastai</code> docs</a>.</p>
 
 
-<p><span class="doc-section-title">Returns:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-              <code><span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[str]]</code>
-          –
-          <div class="doc-md-description">
-            <p>List containing lists of tokens. One list per "<em>document</em>"</p>
-          </div>
-        </li>
-    </ul>
+    <p><span class="doc-section-title">Returns:</span></p>
+    <table>
+      <thead>
+        <tr>
+          <th>Type</th>
+          <th>Description</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[str]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>List containing lists of tokens. One list per "<em>document</em>"</p>
+              </div>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
             <details class="quote">
               <summary>Source code in <code>pytorch_widedeep/utils/fastai_transforms.py</code></summary>
@@ -1979,10 +2243,7 @@ <h2 id="pytorch_widedeep.utils.fastai_transforms.Vocab" class="doc doc-heading">
 
 
 <a href="#pytorch_widedeep.utils.fastai_transforms.Vocab" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="nf">Vocab</span><span class="p">(</span>
-    <span class="n">max_vocab</span><span class="p">,</span> <span class="n">min_freq</span><span class="p">,</span> <span class="n">pad_idx</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">special_cases</span><span class="o">=</span><span class="kc">None</span>
-<span class="p">)</span>
-</code></pre></div>
+
 
     <div class="doc doc-contents first">
 
@@ -1991,63 +2252,128 @@ <h2 id="pytorch_widedeep.utils.fastai_transforms.Vocab" class="doc doc-heading">
 
 
 <p><span class="doc-section-title">Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>max_vocab</code></b>
-              (<code>int</code>)
-          –
-          <div class="doc-md-description">
-            <p>maximum vocabulary size</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>min_freq</code></b>
-              (<code>int</code>)
-          –
-          <div class="doc-md-description">
-            <p>minimum frequency for a token to be considereds</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>pad_idx</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[int]</code>, default:
-                  <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>padding index. If <code>None</code>, Fastai's Tokenizer leaves the 0 index
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+          <th>Default</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>max_vocab</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>maximum vocabulary size</p>
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>min_freq</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>minimum frequency for a token to be considereds</p>
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>pad_idx</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[int]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>padding index. If <code>None</code>, Fastai's Tokenizer leaves the 0 index
 for the unknown token (<em>'xxunk'</em>) and defaults to 1 for the padding
 token (<em>'xxpad'</em>).</p>
-          </div>
-        </li>
-    </ul>
+              </div>
+            </td>
+            <td>
+                  <code>None</code>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
 
 <p><span class="doc-section-title">Attributes:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code><span title="pytorch_widedeep.utils.fastai_transforms.Vocab.itos">itos</span></code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Collection">Collection</span></code>)
-          –
-          <div class="doc-md-description">
-            <p><code>index to str</code>. Collection of strings that are the tokens of the
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code><span title="pytorch_widedeep.utils.fastai_transforms.Vocab.itos">itos</span></code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Collection">Collection</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p><code>index to str</code>. Collection of strings that are the tokens of the
 vocabulary</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code><span title="pytorch_widedeep.utils.fastai_transforms.Vocab.stoi">stoi</span></code></b>
-              (<code><span title="collections.defaultdict">defaultdict</span></code>)
-          –
-          <div class="doc-md-description">
-            <p><code>str to index</code>. Dictionary containing the tokens of the vocabulary and
+              </div>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code><span title="pytorch_widedeep.utils.fastai_transforms.Vocab.stoi">stoi</span></code></td>
+            <td>
+                  <code><span title="collections.defaultdict">defaultdict</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p><code>str to index</code>. Dictionary containing the tokens of the vocabulary and
 their corresponding index</p>
-          </div>
-        </li>
-    </ul>
-
-                  <details class="quote">
-                    <summary>Source code in <code>pytorch_widedeep/utils/fastai_transforms.py</code></summary>
-                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">366</span>
+              </div>
+            </td>
+          </tr>
+      </tbody>
+    </table>
+
+              <details class="quote">
+                <summary>Source code in <code>pytorch_widedeep/utils/fastai_transforms.py</code></summary>
+                <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">342</span>
+<span class="normal">343</span>
+<span class="normal">344</span>
+<span class="normal">345</span>
+<span class="normal">346</span>
+<span class="normal">347</span>
+<span class="normal">348</span>
+<span class="normal">349</span>
+<span class="normal">350</span>
+<span class="normal">351</span>
+<span class="normal">352</span>
+<span class="normal">353</span>
+<span class="normal">354</span>
+<span class="normal">355</span>
+<span class="normal">356</span>
+<span class="normal">357</span>
+<span class="normal">358</span>
+<span class="normal">359</span>
+<span class="normal">360</span>
+<span class="normal">361</span>
+<span class="normal">362</span>
+<span class="normal">363</span>
+<span class="normal">364</span>
+<span class="normal">365</span>
+<span class="normal">366</span>
 <span class="normal">367</span>
 <span class="normal">368</span>
 <span class="normal">369</span>
@@ -2059,21 +2385,295 @@ <h2 id="pytorch_widedeep.utils.fastai_transforms.Vocab" class="doc doc-heading">
 <span class="normal">375</span>
 <span class="normal">376</span>
 <span class="normal">377</span>
-<span class="normal">378</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
-    <span class="bp">self</span><span class="p">,</span>
-    <span class="n">max_vocab</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span>
-    <span class="n">min_freq</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span>
-    <span class="n">pad_idx</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-    <span class="n">special_cases</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Collection</span><span class="p">[</span><span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
-<span class="p">):</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">max_vocab</span> <span class="o">=</span> <span class="n">max_vocab</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">min_freq</span> <span class="o">=</span> <span class="n">min_freq</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">pad_idx</span> <span class="o">=</span> <span class="n">pad_idx</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">special_cases</span> <span class="o">=</span> <span class="p">(</span>
-        <span class="n">special_cases</span> <span class="k">if</span> <span class="n">special_cases</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="k">else</span> <span class="n">defaults</span><span class="o">.</span><span class="n">text_spec_tok</span>
-    <span class="p">)</span>
+<span class="normal">378</span>
+<span class="normal">379</span>
+<span class="normal">380</span>
+<span class="normal">381</span>
+<span class="normal">382</span>
+<span class="normal">383</span>
+<span class="normal">384</span>
+<span class="normal">385</span>
+<span class="normal">386</span>
+<span class="normal">387</span>
+<span class="normal">388</span>
+<span class="normal">389</span>
+<span class="normal">390</span>
+<span class="normal">391</span>
+<span class="normal">392</span>
+<span class="normal">393</span>
+<span class="normal">394</span>
+<span class="normal">395</span>
+<span class="normal">396</span>
+<span class="normal">397</span>
+<span class="normal">398</span>
+<span class="normal">399</span>
+<span class="normal">400</span>
+<span class="normal">401</span>
+<span class="normal">402</span>
+<span class="normal">403</span>
+<span class="normal">404</span>
+<span class="normal">405</span>
+<span class="normal">406</span>
+<span class="normal">407</span>
+<span class="normal">408</span>
+<span class="normal">409</span>
+<span class="normal">410</span>
+<span class="normal">411</span>
+<span class="normal">412</span>
+<span class="normal">413</span>
+<span class="normal">414</span>
+<span class="normal">415</span>
+<span class="normal">416</span>
+<span class="normal">417</span>
+<span class="normal">418</span>
+<span class="normal">419</span>
+<span class="normal">420</span>
+<span class="normal">421</span>
+<span class="normal">422</span>
+<span class="normal">423</span>
+<span class="normal">424</span>
+<span class="normal">425</span>
+<span class="normal">426</span>
+<span class="normal">427</span>
+<span class="normal">428</span>
+<span class="normal">429</span>
+<span class="normal">430</span>
+<span class="normal">431</span>
+<span class="normal">432</span>
+<span class="normal">433</span>
+<span class="normal">434</span>
+<span class="normal">435</span>
+<span class="normal">436</span>
+<span class="normal">437</span>
+<span class="normal">438</span>
+<span class="normal">439</span>
+<span class="normal">440</span>
+<span class="normal">441</span>
+<span class="normal">442</span>
+<span class="normal">443</span>
+<span class="normal">444</span>
+<span class="normal">445</span>
+<span class="normal">446</span>
+<span class="normal">447</span>
+<span class="normal">448</span>
+<span class="normal">449</span>
+<span class="normal">450</span>
+<span class="normal">451</span>
+<span class="normal">452</span>
+<span class="normal">453</span>
+<span class="normal">454</span>
+<span class="normal">455</span>
+<span class="normal">456</span>
+<span class="normal">457</span>
+<span class="normal">458</span>
+<span class="normal">459</span>
+<span class="normal">460</span>
+<span class="normal">461</span>
+<span class="normal">462</span>
+<span class="normal">463</span>
+<span class="normal">464</span>
+<span class="normal">465</span>
+<span class="normal">466</span>
+<span class="normal">467</span>
+<span class="normal">468</span>
+<span class="normal">469</span>
+<span class="normal">470</span>
+<span class="normal">471</span>
+<span class="normal">472</span>
+<span class="normal">473</span>
+<span class="normal">474</span>
+<span class="normal">475</span>
+<span class="normal">476</span>
+<span class="normal">477</span>
+<span class="normal">478</span>
+<span class="normal">479</span>
+<span class="normal">480</span>
+<span class="normal">481</span>
+<span class="normal">482</span>
+<span class="normal">483</span>
+<span class="normal">484</span>
+<span class="normal">485</span>
+<span class="normal">486</span>
+<span class="normal">487</span>
+<span class="normal">488</span>
+<span class="normal">489</span>
+<span class="normal">490</span>
+<span class="normal">491</span>
+<span class="normal">492</span>
+<span class="normal">493</span>
+<span class="normal">494</span>
+<span class="normal">495</span>
+<span class="normal">496</span>
+<span class="normal">497</span>
+<span class="normal">498</span>
+<span class="normal">499</span>
+<span class="normal">500</span>
+<span class="normal">501</span>
+<span class="normal">502</span>
+<span class="normal">503</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">class</span> <span class="nc">Vocab</span><span class="p">:</span>
+<span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Contains the correspondence between numbers and tokens.</span>
+
+<span class="sd">    Parameters</span>
+<span class="sd">    ----------</span>
+<span class="sd">    max_vocab: int</span>
+<span class="sd">        maximum vocabulary size</span>
+<span class="sd">    min_freq: int</span>
+<span class="sd">        minimum frequency for a token to be considereds</span>
+<span class="sd">    pad_idx: int, Optional, default = None</span>
+<span class="sd">        padding index. If `None`, Fastai&#39;s Tokenizer leaves the 0 index</span>
+<span class="sd">        for the unknown token (_&#39;xxunk&#39;_) and defaults to 1 for the padding</span>
+<span class="sd">        token (_&#39;xxpad&#39;_).</span>
+
+<span class="sd">    Attributes</span>
+<span class="sd">    ----------</span>
+<span class="sd">    itos: Collection</span>
+<span class="sd">        `index to str`. Collection of strings that are the tokens of the</span>
+<span class="sd">        vocabulary</span>
+<span class="sd">    stoi: defaultdict</span>
+<span class="sd">        `str to index`. Dictionary containing the tokens of the vocabulary and</span>
+<span class="sd">        their corresponding index</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+
+    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span>
+        <span class="n">max_vocab</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span>
+        <span class="n">min_freq</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span>
+        <span class="n">pad_idx</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">special_cases</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Collection</span><span class="p">[</span><span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+    <span class="p">):</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">max_vocab</span> <span class="o">=</span> <span class="n">max_vocab</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">min_freq</span> <span class="o">=</span> <span class="n">min_freq</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">pad_idx</span> <span class="o">=</span> <span class="n">pad_idx</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">special_cases</span> <span class="o">=</span> <span class="p">(</span>
+            <span class="n">special_cases</span> <span class="k">if</span> <span class="n">special_cases</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="k">else</span> <span class="n">defaults</span><span class="o">.</span><span class="n">text_spec_tok</span>
+        <span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">create</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span>
+        <span class="n">tokens</span><span class="p">:</span> <span class="n">Tokens</span><span class="p">,</span>
+    <span class="p">)</span> <span class="o">-&gt;</span> <span class="s2">&quot;Vocab&quot;</span><span class="p">:</span>
+<span class="w">        </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Create a vocabulary object from a set of tokens.</span>
+
+<span class="sd">        Parameters</span>
+<span class="sd">        ----------</span>
+<span class="sd">        tokens: Tokens</span>
+<span class="sd">            Custom type: ``Collection[Collection[str]]``  see</span>
+<span class="sd">            `pytorch_widedeep.wdtypes`. Collection of collection of</span>
+<span class="sd">            strings (e.g. list of tokenized sentences)</span>
+
+<span class="sd">        Examples</span>
+<span class="sd">        --------</span>
+<span class="sd">        &gt;&gt;&gt; from pytorch_widedeep.utils import Tokenizer, Vocab</span>
+<span class="sd">        &gt;&gt;&gt; texts = [&#39;Machine learning is great&#39;, &#39;but building stuff is even better&#39;]</span>
+<span class="sd">        &gt;&gt;&gt; tokens = Tokenizer().process_all(texts)</span>
+<span class="sd">        &gt;&gt;&gt; vocab = Vocab(max_vocab=18, min_freq=1).create(tokens)</span>
+<span class="sd">        &gt;&gt;&gt; vocab.numericalize([&#39;machine&#39;, &#39;learning&#39;, &#39;is&#39;, &#39;great&#39;])</span>
+<span class="sd">        [10, 11, 9, 12]</span>
+<span class="sd">        &gt;&gt;&gt; vocab.textify([10, 11, 9, 12])</span>
+<span class="sd">        &#39;machine learning is great&#39;</span>
+
+<span class="sd">        :information_source: **NOTE**:</span>
+<span class="sd">        Note the many special tokens that ``fastai``&#39;s&#39; tokenizer adds. These</span>
+<span class="sd">        are particularly useful when building Language models and/or in</span>
+<span class="sd">        classification/Regression tasks. Please see the [``fastai`` docs](https://docs.fast.ai/text.core.html#Tokenizing).</span>
+
+<span class="sd">        Returns</span>
+<span class="sd">        -------</span>
+<span class="sd">        Vocab</span>
+<span class="sd">            An instance of a `Vocab` object</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+
+        <span class="n">freq</span> <span class="o">=</span> <span class="n">Counter</span><span class="p">(</span><span class="n">p</span> <span class="k">for</span> <span class="n">o</span> <span class="ow">in</span> <span class="n">tokens</span> <span class="k">for</span> <span class="n">p</span> <span class="ow">in</span> <span class="n">o</span><span class="p">)</span>
+        <span class="n">itos</span> <span class="o">=</span> <span class="p">[</span><span class="n">o</span> <span class="k">for</span> <span class="n">o</span><span class="p">,</span> <span class="n">c</span> <span class="ow">in</span> <span class="n">freq</span><span class="o">.</span><span class="n">most_common</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">max_vocab</span><span class="p">)</span> <span class="k">if</span> <span class="n">c</span> <span class="o">&gt;=</span> <span class="bp">self</span><span class="o">.</span><span class="n">min_freq</span><span class="p">]</span>
+        <span class="k">for</span> <span class="n">o</span> <span class="ow">in</span> <span class="nb">reversed</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">special_cases</span><span class="p">):</span>
+            <span class="k">if</span> <span class="n">o</span> <span class="ow">in</span> <span class="n">itos</span><span class="p">:</span>
+                <span class="n">itos</span><span class="o">.</span><span class="n">remove</span><span class="p">(</span><span class="n">o</span><span class="p">)</span>
+            <span class="n">itos</span><span class="o">.</span><span class="n">insert</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="n">o</span><span class="p">)</span>
+
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">pad_idx</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">pad_idx</span> <span class="o">!=</span> <span class="mi">1</span><span class="p">:</span>
+            <span class="n">itos</span><span class="o">.</span><span class="n">remove</span><span class="p">(</span><span class="n">PAD</span><span class="p">)</span>
+            <span class="n">itos</span><span class="o">.</span><span class="n">insert</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">pad_idx</span><span class="p">,</span> <span class="n">PAD</span><span class="p">)</span>
+            <span class="c1"># get the new &#39;xxunk&#39; index</span>
+            <span class="n">xxunk_idx</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">where</span><span class="p">([</span><span class="n">el</span> <span class="o">==</span> <span class="s2">&quot;xxunk&quot;</span> <span class="k">for</span> <span class="n">el</span> <span class="ow">in</span> <span class="n">itos</span><span class="p">])[</span><span class="mi">0</span><span class="p">][</span><span class="mi">0</span><span class="p">]</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="n">xxunk_idx</span> <span class="o">=</span> <span class="mi">0</span>
+
+        <span class="n">itos</span> <span class="o">=</span> <span class="n">itos</span><span class="p">[:</span> <span class="bp">self</span><span class="o">.</span><span class="n">max_vocab</span><span class="p">]</span>
+        <span class="k">if</span> <span class="p">(</span>
+            <span class="nb">len</span><span class="p">(</span><span class="n">itos</span><span class="p">)</span> <span class="o">&lt;</span> <span class="bp">self</span><span class="o">.</span><span class="n">max_vocab</span>
+        <span class="p">):</span>  <span class="c1"># Make sure vocab size is a multiple of 8 for fast mixed precision training</span>
+            <span class="k">while</span> <span class="nb">len</span><span class="p">(</span><span class="n">itos</span><span class="p">)</span> <span class="o">%</span> <span class="mi">8</span> <span class="o">!=</span> <span class="mi">0</span><span class="p">:</span>
+                <span class="n">itos</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">&quot;xxfake&quot;</span><span class="p">)</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">itos</span> <span class="o">=</span> <span class="n">itos</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">stoi</span> <span class="o">=</span> <span class="n">defaultdict</span><span class="p">(</span>
+            <span class="k">lambda</span><span class="p">:</span> <span class="n">xxunk_idx</span><span class="p">,</span> <span class="p">{</span><span class="n">v</span><span class="p">:</span> <span class="n">k</span> <span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">itos</span><span class="p">)}</span>
+        <span class="p">)</span>
+
+        <span class="k">return</span> <span class="bp">self</span>
+
+    <span class="k">def</span> <span class="nf">fit</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span>
+        <span class="n">tokens</span><span class="p">:</span> <span class="n">Tokens</span><span class="p">,</span>
+    <span class="p">)</span> <span class="o">-&gt;</span> <span class="s2">&quot;Vocab&quot;</span><span class="p">:</span>
+<span class="w">        </span><span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Calls the `create` method. I simply want to honor fast ai naming, but</span>
+<span class="sd">        for consistency with the rest of the library I am including a fit method</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">create</span><span class="p">(</span><span class="n">tokens</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">numericalize</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">t</span><span class="p">:</span> <span class="n">Collection</span><span class="p">[</span><span class="nb">str</span><span class="p">])</span> <span class="o">-&gt;</span> <span class="n">List</span><span class="p">[</span><span class="nb">int</span><span class="p">]:</span>
+<span class="w">        </span><span class="sd">&quot;&quot;&quot;Convert a list of tokens ``t`` to their ids.</span>
+
+<span class="sd">        Returns</span>
+<span class="sd">        -------</span>
+<span class="sd">        List[int]</span>
+<span class="sd">            List of &#39;_numericalsed_&#39; tokens</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="k">return</span> <span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">stoi</span><span class="p">[</span><span class="n">w</span><span class="p">]</span> <span class="k">for</span> <span class="n">w</span> <span class="ow">in</span> <span class="n">t</span><span class="p">]</span>
+
+    <span class="k">def</span> <span class="nf">transform</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">t</span><span class="p">:</span> <span class="n">Collection</span><span class="p">[</span><span class="nb">str</span><span class="p">])</span> <span class="o">-&gt;</span> <span class="n">List</span><span class="p">[</span><span class="nb">int</span><span class="p">]:</span>
+<span class="w">        </span><span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Calls the `numericalize` method. I simply want to honor fast ai naming,</span>
+<span class="sd">        but for consistency with the rest of the library I am including a</span>
+<span class="sd">        transform method</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">numericalize</span><span class="p">(</span><span class="n">t</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">textify</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">nums</span><span class="p">:</span> <span class="n">Collection</span><span class="p">[</span><span class="nb">int</span><span class="p">],</span> <span class="n">sep</span><span class="o">=</span><span class="s2">&quot; &quot;</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Union</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]]:</span>
+<span class="w">        </span><span class="sd">&quot;&quot;&quot;Convert a list of ``nums`` (or indexes) to their tokens.</span>
+
+<span class="sd">        Returns</span>
+<span class="sd">        -------</span>
+<span class="sd">        List[str]</span>
+<span class="sd">            List of tokens</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="k">return</span> <span class="p">(</span>
+            <span class="n">sep</span><span class="o">.</span><span class="n">join</span><span class="p">([</span><span class="bp">self</span><span class="o">.</span><span class="n">itos</span><span class="p">[</span><span class="n">i</span><span class="p">]</span> <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="n">nums</span><span class="p">])</span>
+            <span class="k">if</span> <span class="n">sep</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span>
+            <span class="k">else</span> <span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">itos</span><span class="p">[</span><span class="n">i</span><span class="p">]</span> <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="n">nums</span><span class="p">]</span>
+        <span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">inverse_transform</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span> <span class="n">nums</span><span class="p">:</span> <span class="n">Collection</span><span class="p">[</span><span class="nb">int</span><span class="p">],</span> <span class="n">sep</span><span class="o">=</span><span class="s2">&quot; &quot;</span>
+    <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Union</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]]:</span>
+<span class="w">        </span><span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Calls the `textify` method. I simply want to honor fast ai naming, but</span>
+<span class="sd">        for consistency with the rest of the library I am including an</span>
+<span class="sd">        inverse_transform method</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="c1"># I simply want to honor fast ai naming, but for consistency with the</span>
+        <span class="c1"># rest of the library I am including an inverse_transform method</span>
+        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">textify</span><span class="p">(</span><span class="n">nums</span><span class="p">,</span> <span class="n">sep</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">__getstate__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="k">return</span> <span class="p">{</span><span class="s2">&quot;itos&quot;</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">itos</span><span class="p">}</span>
+
+    <span class="k">def</span> <span class="nf">__setstate__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">state</span><span class="p">:</span> <span class="nb">dict</span><span class="p">):</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">itos</span> <span class="o">=</span> <span class="n">state</span><span class="p">[</span><span class="s2">&quot;itos&quot;</span><span class="p">]</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">stoi</span> <span class="o">=</span> <span class="n">defaultdict</span><span class="p">(</span><span class="nb">int</span><span class="p">,</span> <span class="p">{</span><span class="n">v</span><span class="p">:</span> <span class="n">k</span> <span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">itos</span><span class="p">)})</span>
 </code></pre></div></td></tr></table></div>
-                  </details>
+              </details>
 
 
 
@@ -2104,18 +2704,34 @@ <h3 id="pytorch_widedeep.utils.fastai_transforms.Vocab.create" class="doc doc-he
 
 
 <p><span class="doc-section-title">Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>tokens</code></b>
-              (<code><span title="pytorch_widedeep.wdtypes.Tokens">Tokens</span></code>)
-          –
-          <div class="doc-md-description">
-            <p>Custom type: <code>Collection[Collection[str]]</code>  see
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+          <th>Default</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>tokens</code></td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Tokens">Tokens</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Custom type: <code>Collection[Collection[str]]</code>  see
 <code>pytorch_widedeep.wdtypes</code>. Collection of collection of
 strings (e.g. list of tokenized sentences)</p>
-          </div>
-        </li>
-    </ul>
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
 
 <p><span class="doc-section-title">Examples:</span></p>
@@ -2134,16 +2750,27 @@ <h3 id="pytorch_widedeep.utils.fastai_transforms.Vocab.create" class="doc doc-he
 classification/Regression tasks. Please see the <a href="https://docs.fast.ai/text.core.html#Tokenizing"><code>fastai</code> docs</a>.</p>
 
 
-<p><span class="doc-section-title">Returns:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-              <code><a class="autorefs autorefs-internal" title="pytorch_widedeep.utils.fastai_transforms.Vocab" href="#pytorch_widedeep.utils.fastai_transforms.Vocab">Vocab</a></code>
-          –
-          <div class="doc-md-description">
-            <p>An instance of a <code>Vocab</code> object</p>
-          </div>
-        </li>
-    </ul>
+    <p><span class="doc-section-title">Returns:</span></p>
+    <table>
+      <thead>
+        <tr>
+          <th>Type</th>
+          <th>Description</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td>
+                  <code><a class="autorefs autorefs-internal" title="pytorch_widedeep.utils.fastai_transforms.Vocab" href="#pytorch_widedeep.utils.fastai_transforms.Vocab">Vocab</a></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>An instance of a <code>Vocab</code> object</p>
+              </div>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
             <details class="quote">
               <summary>Source code in <code>pytorch_widedeep/utils/fastai_transforms.py</code></summary>
@@ -2335,16 +2962,27 @@ <h3 id="pytorch_widedeep.utils.fastai_transforms.Vocab.numericalize" class="doc
         <p>Convert a list of tokens <code>t</code> to their ids.</p>
 
 
-<p><span class="doc-section-title">Returns:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-              <code><span title="pytorch_widedeep.wdtypes.List">List</span>[int]</code>
-          –
-          <div class="doc-md-description">
-            <p>List of '<em>numericalsed</em>' tokens</p>
-          </div>
-        </li>
-    </ul>
+    <p><span class="doc-section-title">Returns:</span></p>
+    <table>
+      <thead>
+        <tr>
+          <th>Type</th>
+          <th>Description</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.List">List</span>[int]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>List of '<em>numericalsed</em>' tokens</p>
+              </div>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
             <details class="quote">
               <summary>Source code in <code>pytorch_widedeep/utils/fastai_transforms.py</code></summary>
@@ -2425,16 +3063,27 @@ <h3 id="pytorch_widedeep.utils.fastai_transforms.Vocab.textify" class="doc doc-h
         <p>Convert a list of <code>nums</code> (or indexes) to their tokens.</p>
 
 
-<p><span class="doc-section-title">Returns:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-              <code><span title="pytorch_widedeep.wdtypes.List">List</span>[str]</code>
-          –
-          <div class="doc-md-description">
-            <p>List of tokens</p>
-          </div>
-        </li>
-    </ul>
+    <p><span class="doc-section-title">Returns:</span></p>
+    <table>
+      <thead>
+        <tr>
+          <th>Type</th>
+          <th>Description</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.List">List</span>[str]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>List of tokens</p>
+              </div>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
             <details class="quote">
               <summary>Source code in <code>pytorch_widedeep/utils/fastai_transforms.py</code></summary>
@@ -2545,7 +3194,8 @@ <h3 id="pytorch_widedeep.utils.fastai_transforms.Vocab.inverse_transform" class=
     <nav>
       
         <a href="mailto:jrzaurin@gmail.com">Javier</a>, 
-        <a href="mailto:mulinka.pavol@gmail.com">Pavol Mulinka</a>
+        <a href="mailto:mulinka.pavol@gmail.com">Pavol Mulinka</a>, 
+        <a href="mailto:not.committed.yet">Not Committed Yet</a>
     </nav>
   </span>
 
diff --git a/mkdocs/site/pytorch-widedeep/utils/fastai_transforms.md b/mkdocs/site/pytorch-widedeep/utils/fastai_transforms.md
index e58f79f9..16a77819 100644
--- a/mkdocs/site/pytorch-widedeep/utils/fastai_transforms.md
+++ b/mkdocs/site/pytorch-widedeep/utils/fastai_transforms.md
@@ -1,9 +1,9 @@
 # Fastai transforms
 
-I directly copied and pasted part of the ``transforms.py`` module from
+We directly copied and pasted part of the ``transforms.py`` module from
 the ``fastai`` library (from an old version). The reason to do such a thing is because
 ``pytorch_widedeep`` only needs the ``Tokenizer`` and the ``Vocab`` classes
-there. This way I avoid extra dependencies. Credit for all the code in the
+there. This way we avoid extra dependencies. Credit for all the code in the
 ``fastai_transforms`` module in this ``pytorch-widedeep`` package goes to
 Jeremy Howard and the `fastai` team. I only include the documentation here for
 completion, but I strongly advise the user to read the ``fastai`` [documentation](https://docs.fast.ai/).
diff --git a/mkdocs/site/pytorch-widedeep/utils/image_utils.html b/mkdocs/site/pytorch-widedeep/utils/image_utils.html
index bf2eb2d1..d4f337c5 100644
--- a/mkdocs/site/pytorch-widedeep/utils/image_utils.html
+++ b/mkdocs/site/pytorch-widedeep/utils/image_utils.html
@@ -1396,7 +1396,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../../examples/16_Self_Supervised_Pretraning_pt1.ipynb" class="md-nav__link">
+      <a href="../../examples/15_Self_Supervised_Pretraning_pt1.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1417,7 +1417,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../../examples/16_Self_Supervised_Pretraning_pt2.ipynb" class="md-nav__link">
+      <a href="../../examples/15_Self_Supervised_Pretraning_pt2.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1438,7 +1438,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../../examples/17_Usign_a_custom_hugging_face_model.ipynb" class="md-nav__link">
+      <a href="../../examples/16_Usign_a_custom_hugging_face_model.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1459,7 +1459,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../../examples/18_feature_importance_via_attention_weights.ipynb" class="md-nav__link">
+      <a href="../../examples/17_feature_importance_via_attention_weights.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1480,7 +1480,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../../examples/19_wide_and_deep_for_recsys_pt1.ipynb" class="md-nav__link">
+      <a href="../../examples/18_wide_and_deep_for_recsys_pt1.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1501,7 +1501,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../../examples/19_wide_and_deep_for_recsys_pt2.ipynb" class="md-nav__link">
+      <a href="../../examples/18_wide_and_deep_for_recsys_pt2.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1522,7 +1522,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../../examples/20_load_from_folder_functionality.ipynb" class="md-nav__link">
+      <a href="../../examples/19_load_from_folder_functionality.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1543,7 +1543,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../../examples/21_Using_huggingface_within_widedeep.ipynb" class="md-nav__link">
+      <a href="../../examples/20_Using_huggingface_within_widedeep.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1619,10 +1619,7 @@ <h2 id="pytorch_widedeep.utils.image_utils.AspectAwarePreprocessor" class="doc d
 
 
 <a href="#pytorch_widedeep.utils.image_utils.AspectAwarePreprocessor" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="nf">AspectAwarePreprocessor</span><span class="p">(</span>
-    <span class="n">width</span><span class="p">,</span> <span class="n">height</span><span class="p">,</span> <span class="n">inter</span><span class="o">=</span><span class="n">cv2</span><span class="o">.</span><span class="n">INTER_AREA</span>
-<span class="p">)</span>
-</code></pre></div>
+
 
     <div class="doc doc-contents first">
 
@@ -1632,44 +1629,192 @@ <h2 id="pytorch_widedeep.utils.image_utils.AspectAwarePreprocessor" class="doc d
 
 
 <p><span class="doc-section-title">Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>width</code></b>
-              (<code>int</code>)
-          –
-          <div class="doc-md-description">
-            <p>output width</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>height</code></b>
-              (<code>int</code>)
-          –
-          <div class="doc-md-description">
-            <p>output height</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>inter</code></b>
-          –
-          <div class="doc-md-description">
-            <p><code>opencv</code> interpolation method. See <code>opencv</code>
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+          <th>Default</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>width</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>output width</p>
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>height</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>output height</p>
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>inter</code></td>
+            <td>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p><code>opencv</code> interpolation method. See <code>opencv</code>
 <code>InterpolationFlags</code>.</p>
-          </div>
-        </li>
-    </ul>
+              </div>
+            </td>
+            <td>
+                  <code><span title="cv2.INTER_AREA">INTER_AREA</span></code>
+            </td>
+          </tr>
+      </tbody>
+    </table>
+
+              <details class="quote">
+                <summary>Source code in <code>pytorch_widedeep/utils/image_utils.py</code></summary>
+                <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 49</span>
+<span class="normal"> 50</span>
+<span class="normal"> 51</span>
+<span class="normal"> 52</span>
+<span class="normal"> 53</span>
+<span class="normal"> 54</span>
+<span class="normal"> 55</span>
+<span class="normal"> 56</span>
+<span class="normal"> 57</span>
+<span class="normal"> 58</span>
+<span class="normal"> 59</span>
+<span class="normal"> 60</span>
+<span class="normal"> 61</span>
+<span class="normal"> 62</span>
+<span class="normal"> 63</span>
+<span class="normal"> 64</span>
+<span class="normal"> 65</span>
+<span class="normal"> 66</span>
+<span class="normal"> 67</span>
+<span class="normal"> 68</span>
+<span class="normal"> 69</span>
+<span class="normal"> 70</span>
+<span class="normal"> 71</span>
+<span class="normal"> 72</span>
+<span class="normal"> 73</span>
+<span class="normal"> 74</span>
+<span class="normal"> 75</span>
+<span class="normal"> 76</span>
+<span class="normal"> 77</span>
+<span class="normal"> 78</span>
+<span class="normal"> 79</span>
+<span class="normal"> 80</span>
+<span class="normal"> 81</span>
+<span class="normal"> 82</span>
+<span class="normal"> 83</span>
+<span class="normal"> 84</span>
+<span class="normal"> 85</span>
+<span class="normal"> 86</span>
+<span class="normal"> 87</span>
+<span class="normal"> 88</span>
+<span class="normal"> 89</span>
+<span class="normal"> 90</span>
+<span class="normal"> 91</span>
+<span class="normal"> 92</span>
+<span class="normal"> 93</span>
+<span class="normal"> 94</span>
+<span class="normal"> 95</span>
+<span class="normal"> 96</span>
+<span class="normal"> 97</span>
+<span class="normal"> 98</span>
+<span class="normal"> 99</span>
+<span class="normal">100</span>
+<span class="normal">101</span>
+<span class="normal">102</span>
+<span class="normal">103</span>
+<span class="normal">104</span>
+<span class="normal">105</span>
+<span class="normal">106</span>
+<span class="normal">107</span>
+<span class="normal">108</span>
+<span class="normal">109</span>
+<span class="normal">110</span>
+<span class="normal">111</span>
+<span class="normal">112</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">class</span> <span class="nc">AspectAwarePreprocessor</span><span class="p">:</span>
+<span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Class to resize an image to a certain width and height taking into account</span>
+<span class="sd">    the image aspect ratio</span>
 
-                  <details class="quote">
-                    <summary>Source code in <code>pytorch_widedeep/utils/image_utils.py</code></summary>
-                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">32</span>
-<span class="normal">33</span>
-<span class="normal">34</span>
-<span class="normal">35</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">width</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span> <span class="n">height</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span> <span class="n">inter</span><span class="o">=</span><span class="n">cv2</span><span class="o">.</span><span class="n">INTER_AREA</span><span class="p">):</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">width</span> <span class="o">=</span> <span class="n">width</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">height</span> <span class="o">=</span> <span class="n">height</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">inter</span> <span class="o">=</span> <span class="n">inter</span>
+<span class="sd">    Parameters</span>
+<span class="sd">    ----------</span>
+<span class="sd">    width: int</span>
+<span class="sd">        output width</span>
+<span class="sd">    height: int</span>
+<span class="sd">        output height</span>
+<span class="sd">    inter: interpolation method,  default = ``cv2.INTER_AREA``</span>
+<span class="sd">        ``opencv`` interpolation method. See ``opencv``</span>
+<span class="sd">        `InterpolationFlags`.</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+
+    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">width</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span> <span class="n">height</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span> <span class="n">inter</span><span class="o">=</span><span class="n">cv2</span><span class="o">.</span><span class="n">INTER_AREA</span><span class="p">):</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">width</span> <span class="o">=</span> <span class="n">width</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">height</span> <span class="o">=</span> <span class="n">height</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">inter</span> <span class="o">=</span> <span class="n">inter</span>
+
+    <span class="k">def</span> <span class="nf">preprocess</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">image</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">:</span>
+<span class="w">        </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Returns the resized input image taking into account the image aspect ratio</span>
+
+<span class="sd">        Parameters</span>
+<span class="sd">        ----------</span>
+<span class="sd">        image: np.ndarray</span>
+<span class="sd">            Input image to be resized</span>
+
+<span class="sd">        Examples</span>
+<span class="sd">        --------</span>
+<span class="sd">        &gt;&gt;&gt; import cv2</span>
+<span class="sd">        &gt;&gt;&gt; from pytorch_widedeep.utils import AspectAwarePreprocessor</span>
+<span class="sd">        &gt;&gt;&gt; img = cv2.imread(&quot;tests/test_data_utils/images/galaxy1.png&quot;)</span>
+<span class="sd">        &gt;&gt;&gt; img.shape</span>
+<span class="sd">        (694, 890, 3)</span>
+<span class="sd">        &gt;&gt;&gt; app = AspectAwarePreprocessor(width=224, height=224)</span>
+<span class="sd">        &gt;&gt;&gt; resized_img = app.preprocess(img)</span>
+<span class="sd">        &gt;&gt;&gt; resized_img.shape</span>
+<span class="sd">        (224, 224, 3)</span>
+
+<span class="sd">        Returns</span>
+<span class="sd">        -------</span>
+<span class="sd">        np.ndarray</span>
+<span class="sd">            Resized image according to its original image aspect ratio</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="p">(</span><span class="n">h</span><span class="p">,</span> <span class="n">w</span><span class="p">)</span> <span class="o">=</span> <span class="n">image</span><span class="o">.</span><span class="n">shape</span><span class="p">[:</span><span class="mi">2</span><span class="p">]</span>
+        <span class="n">dW</span> <span class="o">=</span> <span class="mi">0</span>
+        <span class="n">dH</span> <span class="o">=</span> <span class="mi">0</span>
+
+        <span class="k">if</span> <span class="n">w</span> <span class="o">&lt;</span> <span class="n">h</span><span class="p">:</span>
+            <span class="n">image</span> <span class="o">=</span> <span class="n">resize</span><span class="p">(</span><span class="n">image</span><span class="p">,</span> <span class="n">width</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">width</span><span class="p">,</span> <span class="n">inter</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">inter</span><span class="p">)</span>
+            <span class="n">dH</span> <span class="o">=</span> <span class="nb">int</span><span class="p">((</span><span class="n">image</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">-</span> <span class="bp">self</span><span class="o">.</span><span class="n">height</span><span class="p">)</span> <span class="o">/</span> <span class="mf">2.0</span><span class="p">)</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="n">image</span> <span class="o">=</span> <span class="n">resize</span><span class="p">(</span><span class="n">image</span><span class="p">,</span> <span class="n">height</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">height</span><span class="p">,</span> <span class="n">inter</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">inter</span><span class="p">)</span>
+            <span class="n">dW</span> <span class="o">=</span> <span class="nb">int</span><span class="p">((</span><span class="n">image</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span> <span class="o">-</span> <span class="bp">self</span><span class="o">.</span><span class="n">width</span><span class="p">)</span> <span class="o">/</span> <span class="mf">2.0</span><span class="p">)</span>
+
+        <span class="p">(</span><span class="n">h</span><span class="p">,</span> <span class="n">w</span><span class="p">)</span> <span class="o">=</span> <span class="n">image</span><span class="o">.</span><span class="n">shape</span><span class="p">[:</span><span class="mi">2</span><span class="p">]</span>
+        <span class="n">image</span> <span class="o">=</span> <span class="n">image</span><span class="p">[</span><span class="n">dH</span> <span class="p">:</span> <span class="n">h</span> <span class="o">-</span> <span class="n">dH</span><span class="p">,</span> <span class="n">dW</span> <span class="p">:</span> <span class="n">w</span> <span class="o">-</span> <span class="n">dW</span><span class="p">]</span>
+
+        <span class="n">resized_image</span> <span class="o">=</span> <span class="n">cv2</span><span class="o">.</span><span class="n">resize</span><span class="p">(</span>
+            <span class="n">image</span><span class="p">,</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">width</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">height</span><span class="p">),</span> <span class="n">interpolation</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">inter</span>
+        <span class="p">)</span>
+
+        <span class="k">return</span> <span class="n">resized_image</span>
 </code></pre></div></td></tr></table></div>
-                  </details>
+              </details>
 
 
 
@@ -1700,16 +1845,32 @@ <h3 id="pytorch_widedeep.utils.image_utils.AspectAwarePreprocessor.preprocess" c
 
 
 <p><span class="doc-section-title">Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>image</code></b>
-              (<code><span title="numpy.ndarray">ndarray</span></code>)
-          –
-          <div class="doc-md-description">
-            <p>Input image to be resized</p>
-          </div>
-        </li>
-    </ul>
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+          <th>Default</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>image</code></td>
+            <td>
+                  <code><span title="numpy.ndarray">ndarray</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Input image to be resized</p>
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
 
 <p><span class="doc-section-title">Examples:</span></p>
@@ -1725,63 +1886,74 @@ <h3 id="pytorch_widedeep.utils.image_utils.AspectAwarePreprocessor.preprocess" c
 </code></pre></div>
 
 
-<p><span class="doc-section-title">Returns:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-              <code><span title="numpy.ndarray">ndarray</span></code>
-          –
-          <div class="doc-md-description">
-            <p>Resized image according to its original image aspect ratio</p>
-          </div>
-        </li>
-    </ul>
+    <p><span class="doc-section-title">Returns:</span></p>
+    <table>
+      <thead>
+        <tr>
+          <th>Type</th>
+          <th>Description</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td>
+                  <code><span title="numpy.ndarray">ndarray</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Resized image according to its original image aspect ratio</p>
+              </div>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
             <details class="quote">
               <summary>Source code in <code>pytorch_widedeep/utils/image_utils.py</code></summary>
-              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">37</span>
-<span class="normal">38</span>
-<span class="normal">39</span>
-<span class="normal">40</span>
-<span class="normal">41</span>
-<span class="normal">42</span>
-<span class="normal">43</span>
-<span class="normal">44</span>
-<span class="normal">45</span>
-<span class="normal">46</span>
-<span class="normal">47</span>
-<span class="normal">48</span>
-<span class="normal">49</span>
-<span class="normal">50</span>
-<span class="normal">51</span>
-<span class="normal">52</span>
-<span class="normal">53</span>
-<span class="normal">54</span>
-<span class="normal">55</span>
-<span class="normal">56</span>
-<span class="normal">57</span>
-<span class="normal">58</span>
-<span class="normal">59</span>
-<span class="normal">60</span>
-<span class="normal">61</span>
-<span class="normal">62</span>
-<span class="normal">63</span>
-<span class="normal">64</span>
-<span class="normal">65</span>
-<span class="normal">66</span>
-<span class="normal">67</span>
-<span class="normal">68</span>
-<span class="normal">69</span>
-<span class="normal">70</span>
-<span class="normal">71</span>
-<span class="normal">72</span>
-<span class="normal">73</span>
-<span class="normal">74</span>
-<span class="normal">75</span>
-<span class="normal">76</span>
-<span class="normal">77</span>
-<span class="normal">78</span>
-<span class="normal">79</span>
-<span class="normal">80</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">preprocess</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">image</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">:</span>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 69</span>
+<span class="normal"> 70</span>
+<span class="normal"> 71</span>
+<span class="normal"> 72</span>
+<span class="normal"> 73</span>
+<span class="normal"> 74</span>
+<span class="normal"> 75</span>
+<span class="normal"> 76</span>
+<span class="normal"> 77</span>
+<span class="normal"> 78</span>
+<span class="normal"> 79</span>
+<span class="normal"> 80</span>
+<span class="normal"> 81</span>
+<span class="normal"> 82</span>
+<span class="normal"> 83</span>
+<span class="normal"> 84</span>
+<span class="normal"> 85</span>
+<span class="normal"> 86</span>
+<span class="normal"> 87</span>
+<span class="normal"> 88</span>
+<span class="normal"> 89</span>
+<span class="normal"> 90</span>
+<span class="normal"> 91</span>
+<span class="normal"> 92</span>
+<span class="normal"> 93</span>
+<span class="normal"> 94</span>
+<span class="normal"> 95</span>
+<span class="normal"> 96</span>
+<span class="normal"> 97</span>
+<span class="normal"> 98</span>
+<span class="normal"> 99</span>
+<span class="normal">100</span>
+<span class="normal">101</span>
+<span class="normal">102</span>
+<span class="normal">103</span>
+<span class="normal">104</span>
+<span class="normal">105</span>
+<span class="normal">106</span>
+<span class="normal">107</span>
+<span class="normal">108</span>
+<span class="normal">109</span>
+<span class="normal">110</span>
+<span class="normal">111</span>
+<span class="normal">112</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">preprocess</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">image</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">:</span>
 <span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Returns the resized input image taking into account the image aspect ratio</span>
 
 <span class="sd">    Parameters</span>
@@ -1811,10 +1983,10 @@ <h3 id="pytorch_widedeep.utils.image_utils.AspectAwarePreprocessor.preprocess" c
     <span class="n">dH</span> <span class="o">=</span> <span class="mi">0</span>
 
     <span class="k">if</span> <span class="n">w</span> <span class="o">&lt;</span> <span class="n">h</span><span class="p">:</span>
-        <span class="n">image</span> <span class="o">=</span> <span class="n">imutils</span><span class="o">.</span><span class="n">resize</span><span class="p">(</span><span class="n">image</span><span class="p">,</span> <span class="n">width</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">width</span><span class="p">,</span> <span class="n">inter</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">inter</span><span class="p">)</span>
+        <span class="n">image</span> <span class="o">=</span> <span class="n">resize</span><span class="p">(</span><span class="n">image</span><span class="p">,</span> <span class="n">width</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">width</span><span class="p">,</span> <span class="n">inter</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">inter</span><span class="p">)</span>
         <span class="n">dH</span> <span class="o">=</span> <span class="nb">int</span><span class="p">((</span><span class="n">image</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">-</span> <span class="bp">self</span><span class="o">.</span><span class="n">height</span><span class="p">)</span> <span class="o">/</span> <span class="mf">2.0</span><span class="p">)</span>
     <span class="k">else</span><span class="p">:</span>
-        <span class="n">image</span> <span class="o">=</span> <span class="n">imutils</span><span class="o">.</span><span class="n">resize</span><span class="p">(</span><span class="n">image</span><span class="p">,</span> <span class="n">height</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">height</span><span class="p">,</span> <span class="n">inter</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">inter</span><span class="p">)</span>
+        <span class="n">image</span> <span class="o">=</span> <span class="n">resize</span><span class="p">(</span><span class="n">image</span><span class="p">,</span> <span class="n">height</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">height</span><span class="p">,</span> <span class="n">inter</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">inter</span><span class="p">)</span>
         <span class="n">dW</span> <span class="o">=</span> <span class="nb">int</span><span class="p">((</span><span class="n">image</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span> <span class="o">-</span> <span class="bp">self</span><span class="o">.</span><span class="n">width</span><span class="p">)</span> <span class="o">/</span> <span class="mf">2.0</span><span class="p">)</span>
 
     <span class="p">(</span><span class="n">h</span><span class="p">,</span> <span class="n">w</span><span class="p">)</span> <span class="o">=</span> <span class="n">image</span><span class="o">.</span><span class="n">shape</span><span class="p">[:</span><span class="mi">2</span><span class="p">]</span>
@@ -1848,8 +2020,7 @@ <h2 id="pytorch_widedeep.utils.image_utils.SimplePreprocessor" class="doc doc-he
 
 
 <a href="#pytorch_widedeep.utils.image_utils.SimplePreprocessor" class="headerlink" title="Permanent link">&para;</a></h2>
-<div class="doc-signature highlight"><pre><span></span><code><span class="nf">SimplePreprocessor</span><span class="p">(</span><span class="n">width</span><span class="p">,</span> <span class="n">height</span><span class="p">,</span> <span class="n">inter</span><span class="o">=</span><span class="n">cv2</span><span class="o">.</span><span class="n">INTER_AREA</span><span class="p">)</span>
-</code></pre></div>
+
 
     <div class="doc doc-contents first">
 
@@ -1858,44 +2029,140 @@ <h2 id="pytorch_widedeep.utils.image_utils.SimplePreprocessor" class="doc doc-he
 
 
 <p><span class="doc-section-title">Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>width</code></b>
-              (<code>int</code>)
-          –
-          <div class="doc-md-description">
-            <p>output width</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>height</code></b>
-              (<code>int</code>)
-          –
-          <div class="doc-md-description">
-            <p>output height</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>inter</code></b>
-          –
-          <div class="doc-md-description">
-            <p><code>opencv</code> interpolation method. See <code>opencv</code>
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+          <th>Default</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>width</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>output width</p>
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>height</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>output height</p>
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>inter</code></td>
+            <td>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p><code>opencv</code> interpolation method. See <code>opencv</code>
 <code>InterpolationFlags</code>.</p>
-          </div>
-        </li>
-    </ul>
+              </div>
+            </td>
+            <td>
+                  <code><span title="cv2.INTER_AREA">INTER_AREA</span></code>
+            </td>
+          </tr>
+      </tbody>
+    </table>
+
+              <details class="quote">
+                <summary>Source code in <code>pytorch_widedeep/utils/image_utils.py</code></summary>
+                <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">115</span>
+<span class="normal">116</span>
+<span class="normal">117</span>
+<span class="normal">118</span>
+<span class="normal">119</span>
+<span class="normal">120</span>
+<span class="normal">121</span>
+<span class="normal">122</span>
+<span class="normal">123</span>
+<span class="normal">124</span>
+<span class="normal">125</span>
+<span class="normal">126</span>
+<span class="normal">127</span>
+<span class="normal">128</span>
+<span class="normal">129</span>
+<span class="normal">130</span>
+<span class="normal">131</span>
+<span class="normal">132</span>
+<span class="normal">133</span>
+<span class="normal">134</span>
+<span class="normal">135</span>
+<span class="normal">136</span>
+<span class="normal">137</span>
+<span class="normal">138</span>
+<span class="normal">139</span>
+<span class="normal">140</span>
+<span class="normal">141</span>
+<span class="normal">142</span>
+<span class="normal">143</span>
+<span class="normal">144</span>
+<span class="normal">145</span>
+<span class="normal">146</span>
+<span class="normal">147</span>
+<span class="normal">148</span>
+<span class="normal">149</span>
+<span class="normal">150</span>
+<span class="normal">151</span>
+<span class="normal">152</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">class</span> <span class="nc">SimplePreprocessor</span><span class="p">:</span>
+<span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Class to resize an image to a certain width and height</span>
 
-                  <details class="quote">
-                    <summary>Source code in <code>pytorch_widedeep/utils/image_utils.py</code></summary>
-                    <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 97</span>
-<span class="normal"> 98</span>
-<span class="normal"> 99</span>
-<span class="normal">100</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">width</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span> <span class="n">height</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span> <span class="n">inter</span><span class="o">=</span><span class="n">cv2</span><span class="o">.</span><span class="n">INTER_AREA</span><span class="p">):</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">width</span> <span class="o">=</span> <span class="n">width</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">height</span> <span class="o">=</span> <span class="n">height</span>
-    <span class="bp">self</span><span class="o">.</span><span class="n">inter</span> <span class="o">=</span> <span class="n">inter</span>
+<span class="sd">    Parameters</span>
+<span class="sd">    ----------</span>
+<span class="sd">    width: int</span>
+<span class="sd">        output width</span>
+<span class="sd">    height: int</span>
+<span class="sd">        output height</span>
+<span class="sd">    inter: interpolation method, default = ``cv2.INTER_AREA``</span>
+<span class="sd">        ``opencv`` interpolation method. See ``opencv``</span>
+<span class="sd">        `InterpolationFlags`.</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+
+    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">width</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span> <span class="n">height</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span> <span class="n">inter</span><span class="o">=</span><span class="n">cv2</span><span class="o">.</span><span class="n">INTER_AREA</span><span class="p">):</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">width</span> <span class="o">=</span> <span class="n">width</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">height</span> <span class="o">=</span> <span class="n">height</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">inter</span> <span class="o">=</span> <span class="n">inter</span>
+
+    <span class="k">def</span> <span class="nf">preprocess</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">image</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">:</span>
+<span class="w">        </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Returns the resized input image</span>
+
+<span class="sd">        Parameters</span>
+<span class="sd">        ----------</span>
+<span class="sd">        image: np.ndarray</span>
+<span class="sd">            Input image to be resized</span>
+
+<span class="sd">        Returns</span>
+<span class="sd">        -------</span>
+<span class="sd">        np.ndarray</span>
+<span class="sd">            Resized image</span>
+
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">resized_image</span> <span class="o">=</span> <span class="n">cv2</span><span class="o">.</span><span class="n">resize</span><span class="p">(</span>
+            <span class="n">image</span><span class="p">,</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">width</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">height</span><span class="p">),</span> <span class="n">interpolation</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">inter</span>
+        <span class="p">)</span>
+
+        <span class="k">return</span> <span class="n">resized_image</span>
 </code></pre></div></td></tr></table></div>
-                  </details>
+              </details>
 
 
 
@@ -1926,50 +2193,77 @@ <h3 id="pytorch_widedeep.utils.image_utils.SimplePreprocessor.preprocess" class=
 
 
 <p><span class="doc-section-title">Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>image</code></b>
-              (<code><span title="numpy.ndarray">ndarray</span></code>)
-          –
-          <div class="doc-md-description">
-            <p>Input image to be resized</p>
-          </div>
-        </li>
-    </ul>
-
-
-<p><span class="doc-section-title">Returns:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-              <code><span title="numpy.ndarray">ndarray</span></code>
-          –
-          <div class="doc-md-description">
-            <p>Resized image</p>
-          </div>
-        </li>
-    </ul>
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+          <th>Default</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>image</code></td>
+            <td>
+                  <code><span title="numpy.ndarray">ndarray</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Input image to be resized</p>
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+      </tbody>
+    </table>
+
+
+    <p><span class="doc-section-title">Returns:</span></p>
+    <table>
+      <thead>
+        <tr>
+          <th>Type</th>
+          <th>Description</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td>
+                  <code><span title="numpy.ndarray">ndarray</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Resized image</p>
+              </div>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
             <details class="quote">
               <summary>Source code in <code>pytorch_widedeep/utils/image_utils.py</code></summary>
-              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">102</span>
-<span class="normal">103</span>
-<span class="normal">104</span>
-<span class="normal">105</span>
-<span class="normal">106</span>
-<span class="normal">107</span>
-<span class="normal">108</span>
-<span class="normal">109</span>
-<span class="normal">110</span>
-<span class="normal">111</span>
-<span class="normal">112</span>
-<span class="normal">113</span>
-<span class="normal">114</span>
-<span class="normal">115</span>
-<span class="normal">116</span>
-<span class="normal">117</span>
-<span class="normal">118</span>
-<span class="normal">119</span>
-<span class="normal">120</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">preprocess</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">image</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">:</span>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">134</span>
+<span class="normal">135</span>
+<span class="normal">136</span>
+<span class="normal">137</span>
+<span class="normal">138</span>
+<span class="normal">139</span>
+<span class="normal">140</span>
+<span class="normal">141</span>
+<span class="normal">142</span>
+<span class="normal">143</span>
+<span class="normal">144</span>
+<span class="normal">145</span>
+<span class="normal">146</span>
+<span class="normal">147</span>
+<span class="normal">148</span>
+<span class="normal">149</span>
+<span class="normal">150</span>
+<span class="normal">151</span>
+<span class="normal">152</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">preprocess</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">image</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">:</span>
 <span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Returns the resized input image</span>
 
 <span class="sd">    Parameters</span>
diff --git a/mkdocs/site/pytorch-widedeep/utils/index.html b/mkdocs/site/pytorch-widedeep/utils/index.html
index 20ee1df1..72689bfa 100644
--- a/mkdocs/site/pytorch-widedeep/utils/index.html
+++ b/mkdocs/site/pytorch-widedeep/utils/index.html
@@ -1338,7 +1338,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../../examples/16_Self_Supervised_Pretraning_pt1.ipynb" class="md-nav__link">
+      <a href="../../examples/15_Self_Supervised_Pretraning_pt1.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1359,7 +1359,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../../examples/16_Self_Supervised_Pretraning_pt2.ipynb" class="md-nav__link">
+      <a href="../../examples/15_Self_Supervised_Pretraning_pt2.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1380,7 +1380,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../../examples/17_Usign_a_custom_hugging_face_model.ipynb" class="md-nav__link">
+      <a href="../../examples/16_Usign_a_custom_hugging_face_model.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1401,7 +1401,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../../examples/18_feature_importance_via_attention_weights.ipynb" class="md-nav__link">
+      <a href="../../examples/17_feature_importance_via_attention_weights.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1422,7 +1422,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../../examples/19_wide_and_deep_for_recsys_pt1.ipynb" class="md-nav__link">
+      <a href="../../examples/18_wide_and_deep_for_recsys_pt1.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1443,7 +1443,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../../examples/19_wide_and_deep_for_recsys_pt2.ipynb" class="md-nav__link">
+      <a href="../../examples/18_wide_and_deep_for_recsys_pt2.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1464,7 +1464,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../../examples/20_load_from_folder_functionality.ipynb" class="md-nav__link">
+      <a href="../../examples/19_load_from_folder_functionality.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1485,7 +1485,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../../examples/21_Using_huggingface_within_widedeep.ipynb" class="md-nav__link">
+      <a href="../../examples/20_Using_huggingface_within_widedeep.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
diff --git a/mkdocs/site/pytorch-widedeep/utils/text_utils.html b/mkdocs/site/pytorch-widedeep/utils/text_utils.html
index 99866cc4..e54ba5a9 100644
--- a/mkdocs/site/pytorch-widedeep/utils/text_utils.html
+++ b/mkdocs/site/pytorch-widedeep/utils/text_utils.html
@@ -1414,7 +1414,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../../examples/16_Self_Supervised_Pretraning_pt1.ipynb" class="md-nav__link">
+      <a href="../../examples/15_Self_Supervised_Pretraning_pt1.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1435,7 +1435,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../../examples/16_Self_Supervised_Pretraning_pt2.ipynb" class="md-nav__link">
+      <a href="../../examples/15_Self_Supervised_Pretraning_pt2.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1456,7 +1456,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../../examples/17_Usign_a_custom_hugging_face_model.ipynb" class="md-nav__link">
+      <a href="../../examples/16_Usign_a_custom_hugging_face_model.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1477,7 +1477,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../../examples/18_feature_importance_via_attention_weights.ipynb" class="md-nav__link">
+      <a href="../../examples/17_feature_importance_via_attention_weights.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1498,7 +1498,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../../examples/19_wide_and_deep_for_recsys_pt1.ipynb" class="md-nav__link">
+      <a href="../../examples/18_wide_and_deep_for_recsys_pt1.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1519,7 +1519,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../../examples/19_wide_and_deep_for_recsys_pt2.ipynb" class="md-nav__link">
+      <a href="../../examples/18_wide_and_deep_for_recsys_pt2.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1540,7 +1540,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../../examples/20_load_from_folder_functionality.ipynb" class="md-nav__link">
+      <a href="../../examples/19_load_from_folder_functionality.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1561,7 +1561,7 @@
   
   
     <li class="md-nav__item">
-      <a href="../../examples/21_Using_huggingface_within_widedeep.ipynb" class="md-nav__link">
+      <a href="../../examples/20_Using_huggingface_within_widedeep.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1645,56 +1645,88 @@ <h2 id="pytorch_widedeep.utils.text_utils.simple_preprocess" class="doc doc-head
 
 
 <p><span class="doc-section-title">Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>doc</code></b>
-              (<code>str</code>)
-          –
-          <div class="doc-md-description">
-            <p>Input document.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>lower</code></b>
-              (<code>bool</code>, default:
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+          <th>Default</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>doc</code></td>
+            <td>
+                  <code>str</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Input document.</p>
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>lower</code></td>
+            <td>
+                  <code>bool</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Lower case tokens in the input doc</p>
+              </div>
+            </td>
+            <td>
                   <code>False</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Lower case tokens in the input doc</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>deacc</code></b>
-              (<code>bool</code>, default:
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>deacc</code></td>
+            <td>
+                  <code>bool</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Remove accent marks from tokens using <code>Gensim</code>'s <code>deaccent</code></p>
+              </div>
+            </td>
+            <td>
                   <code>False</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Remove accent marks from tokens using <code>Gensim</code>'s <code>deaccent</code></p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>min_len</code></b>
-              (<code>int</code>, default:
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>min_len</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Minimum length of token (inclusive). Shorter tokens are discarded.</p>
+              </div>
+            </td>
+            <td>
                   <code>2</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Minimum length of token (inclusive). Shorter tokens are discarded.</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>max_len</code></b>
-              (<code>int</code>, default:
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>max_len</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Maximum length of token in result (inclusive). Longer tokens are discarded.</p>
+              </div>
+            </td>
+            <td>
                   <code>15</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Maximum length of token in result (inclusive). Longer tokens are discarded.</p>
-          </div>
-        </li>
-    </ul>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
 
 <p><span class="doc-section-title">Examples:</span></p>
@@ -1704,16 +1736,27 @@ <h2 id="pytorch_widedeep.utils.text_utils.simple_preprocess" class="doc doc-head
 </code></pre></div>
 
 
-<p><span class="doc-section-title">Returns:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-              <code><span title="typing.List">List</span>[str]</code>
-          –
-          <div class="doc-md-description">
-            <p>List with the processed tokens</p>
-          </div>
-        </li>
-    </ul>
+    <p><span class="doc-section-title">Returns:</span></p>
+    <table>
+      <thead>
+        <tr>
+          <th>Type</th>
+          <th>Description</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td>
+                  <code><span title="typing.List">List</span>[str]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>List with the processed tokens</p>
+              </div>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
             <details class="quote">
               <summary>Source code in <code>pytorch_widedeep/utils/text_utils.py</code></summary>
@@ -1824,42 +1867,72 @@ <h2 id="pytorch_widedeep.utils.text_utils.get_texts" class="doc doc-heading">
         <p>Tokenization using <code>Fastai</code>'s <code>Tokenizer</code> because it does a
 series of very convenients things during the tokenization process</p>
 <p>See <code>pytorch_widedeep.utils.fastai_utils.Tokenizer</code></p>
+<p><img alt="ℹ️" class="emojione" src="https://cdnjs.cloudflare.com/ajax/libs/emojione/2.2.7/assets/png/2139.png" title=":information_source:" /> <strong>NOTE</strong>:
+<code>get_texts</code> uses <code>pytorch_widedeep.utils.fastai_transforms.Tokenizer</code>.
+Such tokenizer uses a series of convenient processing steps, including
+the  addition of some special tokens, such as <code>TK_MAJ</code> (<code>xxmaj</code>), used to
+indicate the next word begins with a capital in the original text. For more
+details of special tokens please see the <a href="https://docs.fast.ai/text.core.html#Tokenizing"><code>fastai</code> `docs</a></p>
 
 
 <p><span class="doc-section-title">Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>texts</code></b>
-              (<code><span title="typing.List">List</span>[str]</code>)
-          –
-          <div class="doc-md-description">
-            <p>List of str with the texts (or documents). One str per document</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>already_processed</code></b>
-              (<code><span title="typing.Optional">Optional</span>[bool]</code>, default:
-                  <code>False</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Boolean indicating if the text is already processed and we simply want
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+          <th>Default</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>texts</code></td>
+            <td>
+                  <code><span title="typing.List">List</span>[str]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>List of str with the texts (or documents). One str per document</p>
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>already_processed</code></td>
+            <td>
+                  <code><span title="typing.Optional">Optional</span>[bool]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Boolean indicating if the text is already processed and we simply want
 to tokenize it. This parameter is thought for those cases where the
 input sequences might not be text (but IDs, or anything else) and we
 just want to tokenize it</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>n_cpus</code></b>
-              (<code><span title="typing.Optional">Optional</span>[int]</code>, default:
+              </div>
+            </td>
+            <td>
+                  <code>False</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>n_cpus</code></td>
+            <td>
+                  <code><span title="typing.Optional">Optional</span>[int]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>number of CPUs to used during the tokenization process</p>
+              </div>
+            </td>
+            <td>
                   <code>None</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>number of CPUs to used during the tokenization process</p>
-          </div>
-        </li>
-    </ul>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
 
 <p><span class="doc-section-title">Examples:</span></p>
@@ -1870,58 +1943,27 @@ <h2 id="pytorch_widedeep.utils.text_utils.get_texts" class="doc doc-heading">
 </code></pre></div>
 
 
-<p><span class="doc-section-title">Returns:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-              <code><span title="typing.List">List</span>[<span title="typing.List">List</span>[str]]</code>
-          –
-          <div class="doc-md-description">
-            <p>List of lists, one list per '<em>document</em>' containing its corresponding tokens</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-              <code>information_source: **NOTE**:</code>
-          –
-          <div class="doc-md-description">
-            
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-              <code>`get_texts` uses `pytorch_widedeep.utils.fastai_transforms.Tokenizer`.</code>
-          –
-          <div class="doc-md-description">
-            
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-              <code>Such tokenizer uses a series of convenient processing steps, including</code>
-          –
-          <div class="doc-md-description">
-            
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-              <code>the  addition of some special tokens, such as `TK_MAJ` (`xxmaj`), used to</code>
-          –
-          <div class="doc-md-description">
-            
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-              <code>indicate the next word begins with a capital in the original text. For more</code>
-          –
-          <div class="doc-md-description">
-            
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-              <code>details of special tokens please see the [`fastai` `docs](https://docs.fast.ai/text.core.html#Tokenizing)</code>
-          –
-          <div class="doc-md-description">
-            
-          </div>
-        </li>
-    </ul>
+    <p><span class="doc-section-title">Returns:</span></p>
+    <table>
+      <thead>
+        <tr>
+          <th>Type</th>
+          <th>Description</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td>
+                  <code><span title="typing.List">List</span>[<span title="typing.List">List</span>[str]]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>List of lists, one list per '<em>document</em>' containing its corresponding tokens</p>
+              </div>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
             <details class="quote">
               <summary>Source code in <code>pytorch_widedeep/utils/text_utils.py</code></summary>
@@ -1984,6 +2026,13 @@ <h2 id="pytorch_widedeep.utils.text_utils.get_texts" class="doc doc-heading">
 
 <span class="sd">    See `pytorch_widedeep.utils.fastai_utils.Tokenizer`</span>
 
+<span class="sd">    :information_source: **NOTE**:</span>
+<span class="sd">    `get_texts` uses `pytorch_widedeep.utils.fastai_transforms.Tokenizer`.</span>
+<span class="sd">    Such tokenizer uses a series of convenient processing steps, including</span>
+<span class="sd">    the  addition of some special tokens, such as `TK_MAJ` (`xxmaj`), used to</span>
+<span class="sd">    indicate the next word begins with a capital in the original text. For more</span>
+<span class="sd">    details of special tokens please see the [`fastai` `docs](https://docs.fast.ai/text.core.html#Tokenizing)</span>
+
 <span class="sd">    Parameters</span>
 <span class="sd">    ----------</span>
 <span class="sd">    texts: List</span>
@@ -2007,13 +2056,6 @@ <h2 id="pytorch_widedeep.utils.text_utils.get_texts" class="doc doc-heading">
 <span class="sd">    -------</span>
 <span class="sd">    List[List[str]]</span>
 <span class="sd">        List of lists, one list per &#39;_document_&#39; containing its corresponding tokens</span>
-
-<span class="sd">    :information_source: **NOTE**:</span>
-<span class="sd">    `get_texts` uses `pytorch_widedeep.utils.fastai_transforms.Tokenizer`.</span>
-<span class="sd">    Such tokenizer uses a series of convenient processing steps, including</span>
-<span class="sd">    the  addition of some special tokens, such as `TK_MAJ` (`xxmaj`), used to</span>
-<span class="sd">    indicate the next word begins with a capital in the original text. For more</span>
-<span class="sd">    details of special tokens please see the [`fastai` `docs](https://docs.fast.ai/text.core.html#Tokenizing)</span>
 <span class="sd">    &quot;&quot;&quot;</span>
 
     <span class="n">num_cpus</span> <span class="o">=</span> <span class="n">n_cpus</span> <span class="k">if</span> <span class="n">n_cpus</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="k">else</span> <span class="n">os</span><span class="o">.</span><span class="n">cpu_count</span><span class="p">()</span>
@@ -2048,45 +2090,75 @@ <h2 id="pytorch_widedeep.utils.text_utils.pad_sequences" class="doc doc-heading"
 
 
 <p><span class="doc-section-title">Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>seq</code></b>
-              (<code><span title="typing.List">List</span>[int]</code>)
-          –
-          <div class="doc-md-description">
-            <p>List of int with the <code>numericalised</code> tokens</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>maxlen</code></b>
-              (<code>int</code>)
-          –
-          <div class="doc-md-description">
-            <p>Maximum length of the padded sequences</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>pad_first</code></b>
-              (<code>bool</code>, default:
-                  <code>True</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>Indicates whether the padding index will be added at the beginning or the
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+          <th>Default</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>seq</code></td>
+            <td>
+                  <code><span title="typing.List">List</span>[int]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>List of int with the <code>numericalised</code> tokens</p>
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>maxlen</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Maximum length of the padded sequences</p>
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>pad_first</code></td>
+            <td>
+                  <code>bool</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Indicates whether the padding index will be added at the beginning or the
 end of the sequences</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>pad_idx</code></b>
-              (<code>int</code>, default:
+              </div>
+            </td>
+            <td>
+                  <code>True</code>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>pad_idx</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>padding index. Fastai's Tokenizer leaves 0 for the 'unknown' token.</p>
+              </div>
+            </td>
+            <td>
                   <code>1</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>padding index. Fastai's Tokenizer leaves 0 for the 'unknown' token.</p>
-          </div>
-        </li>
-    </ul>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
 
 <p><span class="doc-section-title">Examples:</span></p>
@@ -2097,16 +2169,27 @@ <h2 id="pytorch_widedeep.utils.text_utils.pad_sequences" class="doc doc-heading"
 </code></pre></div>
 
 
-<p><span class="doc-section-title">Returns:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-              <code><span title="numpy.ndarray">ndarray</span></code>
-          –
-          <div class="doc-md-description">
-            <p>numpy array with the padded sequences</p>
-          </div>
-        </li>
-    </ul>
+    <p><span class="doc-section-title">Returns:</span></p>
+    <table>
+      <thead>
+        <tr>
+          <th>Type</th>
+          <th>Description</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td>
+                  <code><span title="numpy.ndarray">ndarray</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>numpy array with the padded sequences</p>
+              </div>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
             <details class="quote">
               <summary>Source code in <code>pytorch_widedeep/utils/text_utils.py</code></summary>
@@ -2223,54 +2306,97 @@ <h2 id="pytorch_widedeep.utils.text_utils.build_embeddings_matrix" class="doc do
 
 
 <p><span class="doc-section-title">Parameters:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-          <b><code>vocab</code></b>
-              (<code><span title="typing.Union">Union</span>[<a class="autorefs autorefs-internal" title="pytorch_widedeep.utils.fastai_transforms.Vocab" href="fastai_transforms.html#pytorch_widedeep.utils.fastai_transforms.Vocab">Vocab</a>, <span title="pytorch_widedeep.utils.fastai_transforms.ChunkVocab">ChunkVocab</span>]</code>)
-          –
-          <div class="doc-md-description">
-            <p>see <code>pytorch_widedeep.utils.fastai_utils.Vocab</code></p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>word_vectors_path</code></b>
-              (<code>str</code>)
-          –
-          <div class="doc-md-description">
-            <p>path to the pretrained word embeddings</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>min_freq</code></b>
-              (<code>int</code>)
-          –
-          <div class="doc-md-description">
-            <p>minimum frequency required for a word to be in the vocabulary</p>
-          </div>
-        </li>
-        <li class="doc-section-item field-body">
-          <b><code>verbose</code></b>
-              (<code>int</code>, default:
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Type</th>
+          <th>Description</th>
+          <th>Default</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td><code>vocab</code></td>
+            <td>
+                  <code><span title="typing.Union">Union</span>[<a class="autorefs autorefs-internal" title="pytorch_widedeep.utils.fastai_transforms.Vocab" href="fastai_transforms.html#pytorch_widedeep.utils.fastai_transforms.Vocab">Vocab</a>, <span title="pytorch_widedeep.utils.fastai_transforms.ChunkVocab">ChunkVocab</span>]</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>see <code>pytorch_widedeep.utils.fastai_utils.Vocab</code></p>
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>word_vectors_path</code></td>
+            <td>
+                  <code>str</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>path to the pretrained word embeddings</p>
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>min_freq</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>minimum frequency required for a word to be in the vocabulary</p>
+              </div>
+            </td>
+            <td>
+                <em>required</em>
+            </td>
+          </tr>
+          <tr class="doc-section-item">
+            <td><code>verbose</code></td>
+            <td>
+                  <code>int</code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>level of verbosity. Set to 0 for no verbosity</p>
+              </div>
+            </td>
+            <td>
                   <code>1</code>
-)
-          –
-          <div class="doc-md-description">
-            <p>level of verbosity. Set to 0 for no verbosity</p>
-          </div>
-        </li>
-    </ul>
-
-
-<p><span class="doc-section-title">Returns:</span></p>
-    <ul>
-        <li class="doc-section-item field-body">
-              <code><span title="numpy.ndarray">ndarray</span></code>
-          –
-          <div class="doc-md-description">
-            <p>Pretrained word embeddings</p>
-          </div>
-        </li>
-    </ul>
+            </td>
+          </tr>
+      </tbody>
+    </table>
+
+
+    <p><span class="doc-section-title">Returns:</span></p>
+    <table>
+      <thead>
+        <tr>
+          <th>Type</th>
+          <th>Description</th>
+        </tr>
+      </thead>
+      <tbody>
+          <tr class="doc-section-item">
+            <td>
+                  <code><span title="numpy.ndarray">ndarray</span></code>
+            </td>
+            <td>
+              <div class="doc-md-description">
+                <p>Pretrained word embeddings</p>
+              </div>
+            </td>
+          </tr>
+      </tbody>
+    </table>
 
             <details class="quote">
               <summary>Source code in <code>pytorch_widedeep/utils/text_utils.py</code></summary>
diff --git a/mkdocs/site/quick_start.html b/mkdocs/site/quick_start.html
index d23987b2..385f038d 100644
--- a/mkdocs/site/quick_start.html
+++ b/mkdocs/site/quick_start.html
@@ -1343,7 +1343,7 @@
   
   
     <li class="md-nav__item">
-      <a href="examples/16_Self_Supervised_Pretraning_pt1.ipynb" class="md-nav__link">
+      <a href="examples/15_Self_Supervised_Pretraning_pt1.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1364,7 +1364,7 @@
   
   
     <li class="md-nav__item">
-      <a href="examples/16_Self_Supervised_Pretraning_pt2.ipynb" class="md-nav__link">
+      <a href="examples/15_Self_Supervised_Pretraning_pt2.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1385,7 +1385,7 @@
   
   
     <li class="md-nav__item">
-      <a href="examples/17_Usign_a_custom_hugging_face_model.ipynb" class="md-nav__link">
+      <a href="examples/16_Usign_a_custom_hugging_face_model.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1406,7 +1406,7 @@
   
   
     <li class="md-nav__item">
-      <a href="examples/18_feature_importance_via_attention_weights.ipynb" class="md-nav__link">
+      <a href="examples/17_feature_importance_via_attention_weights.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1427,7 +1427,7 @@
   
   
     <li class="md-nav__item">
-      <a href="examples/19_wide_and_deep_for_recsys_pt1.ipynb" class="md-nav__link">
+      <a href="examples/18_wide_and_deep_for_recsys_pt1.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1448,7 +1448,7 @@
   
   
     <li class="md-nav__item">
-      <a href="examples/19_wide_and_deep_for_recsys_pt2.ipynb" class="md-nav__link">
+      <a href="examples/18_wide_and_deep_for_recsys_pt2.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1469,7 +1469,7 @@
   
   
     <li class="md-nav__item">
-      <a href="examples/20_load_from_folder_functionality.ipynb" class="md-nav__link">
+      <a href="examples/19_load_from_folder_functionality.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
@@ -1490,7 +1490,7 @@
   
   
     <li class="md-nav__item">
-      <a href="examples/21_Using_huggingface_within_widedeep.ipynb" class="md-nav__link">
+      <a href="examples/20_Using_huggingface_within_widedeep.html" class="md-nav__link">
         
   
   <span class="md-ellipsis">
diff --git a/mkdocs/site/search/search_index.json b/mkdocs/site/search/search_index.json
index 46740eda..73418793 100644
--- a/mkdocs/site/search/search_index.json
+++ b/mkdocs/site/search/search_index.json
@@ -1 +1 @@
-{"config":{"lang":["en"],"separator":"[\\s\\-]+","pipeline":["stopWordFilter"]},"docs":[{"location":"index.html","title":"Home","text":""},{"location":"index.html#pytorch-widedeep","title":"pytorch-widedeep","text":"<p>A flexible package for multimodal-deep-learning to combine tabular data with text and images using Wide and Deep models in Pytorch</p> <p>Documentation: https://pytorch-widedeep.readthedocs.io</p> <p>Companion posts and tutorials: infinitoml</p> <p>Experiments and comparison with <code>LightGBM</code>: TabularDL vs LightGBM</p> <p>Slack: if you want to contribute or just want to chat with us, join slack</p> <p>The content of this document is organized as follows:</p> <ul> <li>pytorch-widedeep<ul> <li>Introduction</li> <li>Architectures</li> <li>The <code>deeptabular</code> component</li> <li>The <code>rec</code> module</li> <li>Text and Images</li> <li>Acknowledgments</li> <li>License</li> <li>Cite<ul> <li>BibTex</li> <li>APA</li> </ul> </li> </ul> </li> </ul>"},{"location":"index.html#introduction","title":"Introduction","text":"<p><code>pytorch-widedeep</code> is based on Google's Wide and Deep Algorithm, adjusted for multi-modal datasets.</p> <p>In general terms, <code>pytorch-widedeep</code> is a package to use deep learning with tabular data. In particular, is intended to facilitate the combination of text and images with corresponding tabular data using wide and deep models. With that in mind there are a number of architectures that can be implemented with the library. The main components of those architectures are shown in the Figure below:</p> <p> </p> <p>In math terms, and following the notation in the paper, the expression for the architecture without a <code>deephead</code> component can be formulated as:</p> <p> </p> <p>Where \u03c3 is the sigmoid function, 'W' are the weight matrices applied to the wide model and to the final activations of the deep models, 'a' are these final activations, \u03c6(x) are the cross product transformations of the original features 'x', and , and 'b' is the bias term. In case you are wondering what are \"cross product transformations\", here is a quote taken directly from the paper: \"For binary features, a cross-product transformation (e.g., \u201cAND(gender=female, language=en)\u201d) is 1 if and only if the constituent features (\u201cgender=female\u201d and \u201clanguage=en\u201d) are all 1, and 0 otherwise\".</p> <p>It is perfectly possible to use custom models (and not necessarily those in the library) as long as the the custom models have a property called <code>output_dim</code> with the size of the last layer of activations, so that <code>WideDeep</code> can be constructed. Examples on how to use custom components can be found in the Examples folder and the section below.</p>"},{"location":"index.html#architectures","title":"Architectures","text":"<p>The <code>pytorch-widedeep</code> library offers a number of different architectures. In this section we will show some of them in their simplest form (i.e. with default param values in most cases) with their corresponding code snippets. Note that all the snippets below shoud run locally. For a more detailed explanation of the different components and their parameters, please refer to the documentation.</p> <p>For the examples below we will be using a toy dataset generated as follows:</p> <pre><code>import os\nimport random\n\nimport numpy as np\nimport pandas as pd\nfrom PIL import Image\nfrom faker import Faker\n\n\ndef create_and_save_random_image(image_number, size=(32, 32)):\n\n    if not os.path.exists(\"images\"):\n        os.makedirs(\"images\")\n\n    array = np.random.randint(0, 256, (size[0], size[1], 3), dtype=np.uint8)\n\n    image = Image.fromarray(array)\n\n    image_name = f\"image_{image_number}.png\"\n    image.save(os.path.join(\"images\", image_name))\n\n    return image_name\n\n\nfake = Faker()\n\ncities = [\"New York\", \"Los Angeles\", \"Chicago\", \"Houston\"]\nnames = [\"Alice\", \"Bob\", \"Charlie\", \"David\", \"Eva\"]\n\ndata = {\n    \"city\": [random.choice(cities) for _ in range(100)],\n    \"name\": [random.choice(names) for _ in range(100)],\n    \"age\": [random.uniform(18, 70) for _ in range(100)],\n    \"height\": [random.uniform(150, 200) for _ in range(100)],\n    \"sentence\": [fake.sentence() for _ in range(100)],\n    \"other_sentence\": [fake.sentence() for _ in range(100)],\n    \"image_name\": [create_and_save_random_image(i) for i in range(100)],\n    \"target\": [random.choice([0, 1]) for _ in range(100)],\n}\n\ndf = pd.DataFrame(data)\n</code></pre> <p>This will create a 100 rows dataframe and a dir in your local folder, called <code>images</code> with 100 random images (or images with just noise).</p> <p>Perhaps the simplest architecture would be just one component, <code>wide</code>, <code>deeptabular</code>, <code>deeptext</code> or <code>deepimage</code> on their own, which is also possible, but let's start the examples with a standard Wide and Deep architecture. From there, how to build a model comprised only of one component will be straightforward.</p> <p>Note that the examples shown below would be almost identical using any of the models available in the library. For example, <code>TabMlp</code> can be replaced by <code>TabResnet</code>, <code>TabNet</code>, <code>TabTransformer</code>, etc. Similarly, <code>BasicRNN</code> can be replaced by <code>AttentiveRNN</code>, <code>StackedAttentiveRNN</code>, or <code>HFModel</code> with their corresponding parameters and preprocessor in the case of the Hugging Face models.</p> <p>1. Wide and Tabular component (aka deeptabular)</p> <p> </p> <pre><code>from pytorch_widedeep.preprocessing import TabPreprocessor, WidePreprocessor\nfrom pytorch_widedeep.models import Wide, TabMlp, WideDeep\nfrom pytorch_widedeep.training import Trainer\n\n# Wide\nwide_cols = [\"city\"]\ncrossed_cols = [(\"city\", \"name\")]\nwide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols)\nX_wide = wide_preprocessor.fit_transform(df)\nwide = Wide(input_dim=np.unique(X_wide).shape[0])\n\n# Tabular\ntab_preprocessor = TabPreprocessor(\n    embed_cols=[\"city\", \"name\"], continuous_cols=[\"age\", \"height\"]\n)\nX_tab = tab_preprocessor.fit_transform(df)\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=tab_preprocessor.continuous_cols,\n    mlp_hidden_dims=[64, 32],\n)\n\n# WideDeep\nmodel = WideDeep(wide=wide, deeptabular=tab_mlp)\n\n# Train\ntrainer = Trainer(model, objective=\"binary\")\n\ntrainer.fit(\n    X_wide=X_wide,\n    X_tab=X_tab,\n    target=df[\"target\"].values,\n    n_epochs=1,\n    batch_size=32,\n)\n</code></pre> <p>2. Tabular and Text data</p> <p> </p> <pre><code>from pytorch_widedeep.preprocessing import TabPreprocessor, TextPreprocessor\nfrom pytorch_widedeep.models import TabMlp, BasicRNN, WideDeep\nfrom pytorch_widedeep.training import Trainer\n\n# Tabular\ntab_preprocessor = TabPreprocessor(\n    embed_cols=[\"city\", \"name\"], continuous_cols=[\"age\", \"height\"]\n)\nX_tab = tab_preprocessor.fit_transform(df)\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=tab_preprocessor.continuous_cols,\n    mlp_hidden_dims=[64, 32],\n)\n\n# Text\ntext_preprocessor = TextPreprocessor(\n    text_col=\"sentence\", maxlen=20, max_vocab=100, n_cpus=1\n)\nX_text = text_preprocessor.fit_transform(df)\nrnn = BasicRNN(\n    vocab_size=len(text_preprocessor.vocab.itos),\n    embed_dim=16,\n    hidden_dim=8,\n    n_layers=1,\n)\n\n# WideDeep\nmodel = WideDeep(deeptabular=tab_mlp, deeptext=rnn)\n\n# Train\ntrainer = Trainer(model, objective=\"binary\")\n\ntrainer.fit(\n    X_tab=X_tab,\n    X_text=X_text,\n    target=df[\"target\"].values,\n    n_epochs=1,\n    batch_size=32,\n)\n</code></pre> <p>3. Tabular and text with a FC head on top via the <code>head_hidden_dims</code> param   in <code>WideDeep</code></p> <p> </p> <pre><code>from pytorch_widedeep.preprocessing import TabPreprocessor, TextPreprocessor\nfrom pytorch_widedeep.models import TabMlp, BasicRNN, WideDeep\nfrom pytorch_widedeep.training import Trainer\n\n# Tabular\ntab_preprocessor = TabPreprocessor(\n    embed_cols=[\"city\", \"name\"], continuous_cols=[\"age\", \"height\"]\n)\nX_tab = tab_preprocessor.fit_transform(df)\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=tab_preprocessor.continuous_cols,\n    mlp_hidden_dims=[64, 32],\n)\n\n# Text\ntext_preprocessor = TextPreprocessor(\n    text_col=\"sentence\", maxlen=20, max_vocab=100, n_cpus=1\n)\nX_text = text_preprocessor.fit_transform(df)\nrnn = BasicRNN(\n    vocab_size=len(text_preprocessor.vocab.itos),\n    embed_dim=16,\n    hidden_dim=8,\n    n_layers=1,\n)\n\n# WideDeep\nmodel = WideDeep(deeptabular=tab_mlp, deeptext=rnn, head_hidden_dims=[32, 16])\n\n# Train\ntrainer = Trainer(model, objective=\"binary\")\n\ntrainer.fit(\n    X_tab=X_tab,\n    X_text=X_text,\n    target=df[\"target\"].values,\n    n_epochs=1,\n    batch_size=32,\n)\n</code></pre> <p>4. Tabular and multiple text columns that are passed directly to   <code>WideDeep</code></p> <p> </p> <pre><code>from pytorch_widedeep.preprocessing import TabPreprocessor, TextPreprocessor\nfrom pytorch_widedeep.models import TabMlp, BasicRNN, WideDeep\nfrom pytorch_widedeep.training import Trainer\n\n\n# Tabular\ntab_preprocessor = TabPreprocessor(\n    embed_cols=[\"city\", \"name\"], continuous_cols=[\"age\", \"height\"]\n)\nX_tab = tab_preprocessor.fit_transform(df)\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=tab_preprocessor.continuous_cols,\n    mlp_hidden_dims=[64, 32],\n)\n\n# Text\ntext_preprocessor_1 = TextPreprocessor(\n    text_col=\"sentence\", maxlen=20, max_vocab=100, n_cpus=1\n)\nX_text_1 = text_preprocessor_1.fit_transform(df)\ntext_preprocessor_2 = TextPreprocessor(\n    text_col=\"other_sentence\", maxlen=20, max_vocab=100, n_cpus=1\n)\nX_text_2 = text_preprocessor_2.fit_transform(df)\nrnn_1 = BasicRNN(\n    vocab_size=len(text_preprocessor_1.vocab.itos),\n    embed_dim=16,\n    hidden_dim=8,\n    n_layers=1,\n)\nrnn_2 = BasicRNN(\n    vocab_size=len(text_preprocessor_2.vocab.itos),\n    embed_dim=16,\n    hidden_dim=8,\n    n_layers=1,\n)\n\n# WideDeep\nmodel = WideDeep(deeptabular=tab_mlp, deeptext=[rnn_1, rnn_2])\n\n# Train\ntrainer = Trainer(model, objective=\"binary\")\n\ntrainer.fit(\n    X_tab=X_tab,\n    X_text=[X_text_1, X_text_2],\n    target=df[\"target\"].values,\n    n_epochs=1,\n    batch_size=32,\n)\n</code></pre> <p>5. Tabular data and multiple text columns that are fused via a the library's   <code>ModelFuser</code> class</p> <p> </p> <pre><code>from pytorch_widedeep.preprocessing import TabPreprocessor, TextPreprocessor\nfrom pytorch_widedeep.models import TabMlp, BasicRNN, WideDeep, ModelFuser\nfrom pytorch_widedeep import Trainer\n\n# Tabular\ntab_preprocessor = TabPreprocessor(\n    embed_cols=[\"city\", \"name\"], continuous_cols=[\"age\", \"height\"]\n)\nX_tab = tab_preprocessor.fit_transform(df)\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=tab_preprocessor.continuous_cols,\n    mlp_hidden_dims=[64, 32],\n)\n\n# Text\ntext_preprocessor_1 = TextPreprocessor(\n    text_col=\"sentence\", maxlen=20, max_vocab=100, n_cpus=1\n)\nX_text_1 = text_preprocessor_1.fit_transform(df)\ntext_preprocessor_2 = TextPreprocessor(\n    text_col=\"other_sentence\", maxlen=20, max_vocab=100, n_cpus=1\n)\nX_text_2 = text_preprocessor_2.fit_transform(df)\n\nrnn_1 = BasicRNN(\n    vocab_size=len(text_preprocessor_1.vocab.itos),\n    embed_dim=16,\n    hidden_dim=8,\n    n_layers=1,\n)\nrnn_2 = BasicRNN(\n    vocab_size=len(text_preprocessor_2.vocab.itos),\n    embed_dim=16,\n    hidden_dim=8,\n    n_layers=1,\n)\n\nmodels_fuser = ModelFuser(models=[rnn_1, rnn_2], fusion_method=\"mult\")\n\n# WideDeep\nmodel = WideDeep(deeptabular=tab_mlp, deeptext=models_fuser)\n\n# Train\ntrainer = Trainer(model, objective=\"binary\")\n\ntrainer.fit(\n    X_tab=X_tab,\n    X_text=[X_text_1, X_text_2],\n    target=df[\"target\"].values,\n    n_epochs=1,\n    batch_size=32,\n)\n</code></pre> <p>6. Tabular and multiple text columns, with an image column. The text columns   are fused via the library's <code>ModelFuser</code> and then all fused via the   deephead paramenter in <code>WideDeep</code> which is a custom <code>ModelFuser</code> coded by   the user</p> <p>This is perhaps the less elegant solution as it involves a custom component by the user and slicing the 'incoming' tensor. In the future, we will include a <code>TextAndImageModelFuser</code> to make this process more straightforward. Still, is not really complicated and it is a good example of how to use custom components in <code>pytorch-widedeep</code>.</p> <p>Note that the only requirement for the custom component is that it has a property called <code>output_dim</code> that returns the size of the last layer of activations. In other words, it does not need to inherit from <code>BaseWDModelComponent</code>. This base class simply checks the existence of such property and avoids some typing errors internally.</p> <p> </p> <pre><code>import torch\n\nfrom pytorch_widedeep.preprocessing import TabPreprocessor, TextPreprocessor, ImagePreprocessor\nfrom pytorch_widedeep.models import TabMlp, BasicRNN, WideDeep, ModelFuser, Vision\nfrom pytorch_widedeep.models._base_wd_model_component import BaseWDModelComponent\nfrom pytorch_widedeep import Trainer\n\n# Tabular\ntab_preprocessor = TabPreprocessor(\n    embed_cols=[\"city\", \"name\"], continuous_cols=[\"age\", \"height\"]\n)\nX_tab = tab_preprocessor.fit_transform(df)\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=tab_preprocessor.continuous_cols,\n    mlp_hidden_dims=[16, 8],\n)\n\n# Text\ntext_preprocessor_1 = TextPreprocessor(\n    text_col=\"sentence\", maxlen=20, max_vocab=100, n_cpus=1\n)\nX_text_1 = text_preprocessor_1.fit_transform(df)\ntext_preprocessor_2 = TextPreprocessor(\n    text_col=\"other_sentence\", maxlen=20, max_vocab=100, n_cpus=1\n)\nX_text_2 = text_preprocessor_2.fit_transform(df)\nrnn_1 = BasicRNN(\n    vocab_size=len(text_preprocessor_1.vocab.itos),\n    embed_dim=16,\n    hidden_dim=8,\n    n_layers=1,\n)\nrnn_2 = BasicRNN(\n    vocab_size=len(text_preprocessor_2.vocab.itos),\n    embed_dim=16,\n    hidden_dim=8,\n    n_layers=1,\n)\nmodels_fuser = ModelFuser(\n    models=[rnn_1, rnn_2],\n    fusion_method=\"mult\",\n)\n\n# Image\nimage_preprocessor = ImagePreprocessor(img_col=\"image_name\", img_path=\"images\")\nX_img = image_preprocessor.fit_transform(df)\nvision = Vision(pretrained_model_setup=\"resnet18\", head_hidden_dims=[16, 8])\n\n# deephead (custom model fuser)\nclass MyModelFuser(BaseWDModelComponent):\n    \"\"\"\n    Simply a Linear + Relu sequence on top of the text + images followed by a\n    Linear -&gt; Relu -&gt; Linear for the concatenation of tabular slice of the\n    tensor and the output of the text and image sequential model\n    \"\"\"\n    def __init__(\n        self,\n        tab_incoming_dim: int,\n        text_incoming_dim: int,\n        image_incoming_dim: int,\n        output_units: int,\n    ):\n\n        super(MyModelFuser, self).__init__()\n\n        self.tab_incoming_dim = tab_incoming_dim\n        self.text_incoming_dim = text_incoming_dim\n        self.image_incoming_dim = image_incoming_dim\n        self.output_units = output_units\n        self.text_and_image_fuser = torch.nn.Sequential(\n            torch.nn.Linear(text_incoming_dim + image_incoming_dim, output_units),\n            torch.nn.ReLU(),\n        )\n        self.out = torch.nn.Sequential(\n            torch.nn.Linear(output_units + tab_incoming_dim, output_units * 4),\n            torch.nn.ReLU(),\n            torch.nn.Linear(output_units * 4, output_units),\n        )\n\n    def forward(self, X: torch.Tensor) -&gt; torch.Tensor:\n        tab_slice = slice(0, self.tab_incoming_dim)\n        text_slice = slice(\n            self.tab_incoming_dim, self.tab_incoming_dim + self.text_incoming_dim\n        )\n        image_slice = slice(\n            self.tab_incoming_dim + self.text_incoming_dim,\n            self.tab_incoming_dim + self.text_incoming_dim + self.image_incoming_dim,\n        )\n        X_tab = X[:, tab_slice]\n        X_text = X[:, text_slice]\n        X_img = X[:, image_slice]\n        X_text_and_image = self.text_and_image_fuser(torch.cat([X_text, X_img], dim=1))\n        return self.out(torch.cat([X_tab, X_text_and_image], dim=1))\n\n    @property\n    def output_dim(self):\n        return self.output_units\n\ndeephead = MyModelFuser(\n    tab_incoming_dim=tab_mlp.output_dim,\n    text_incoming_dim=models_fuser.output_dim,\n    image_incoming_dim=vision.output_dim,\n    output_units=8,\n)\n\n# WideDeep\nmodel = WideDeep(\n    deeptabular=tab_mlp,\n    deeptext=models_fuser,\n    deepimage=vision,\n    deephead=deephead,\n)\n\n# Train\ntrainer = Trainer(model, objective=\"binary\")\n\ntrainer.fit(\n    X_tab=X_tab,\n    X_text=[X_text_1, X_text_2],\n    X_img=X_img,\n    target=df[\"target\"].values,\n    n_epochs=1,\n    batch_size=32,\n)\n</code></pre> <p>7. A two-tower model</p> <p>This is a popular model in the context of recommendation systems. Let's say we have a tabular dataset formed my triples (user features, item features, target). We can create a two-tower model where the user and item features are passed through two separate models and then \"fused\" via a dot product.</p> <p> </p> <pre><code>import numpy as np\nimport pandas as pd\n\nfrom pytorch_widedeep import Trainer\nfrom pytorch_widedeep.preprocessing import TabPreprocessor\nfrom pytorch_widedeep.models import TabMlp, WideDeep, ModelFuser\n\n# Let's create the interaction dataset\n# user_features dataframe\nnp.random.seed(42)\nuser_ids = np.arange(1, 101)\nages = np.random.randint(18, 60, size=100)\ngenders = np.random.choice([\"male\", \"female\"], size=100)\nlocations = np.random.choice([\"city_a\", \"city_b\", \"city_c\", \"city_d\"], size=100)\nuser_features = pd.DataFrame(\n    {\"id\": user_ids, \"age\": ages, \"gender\": genders, \"location\": locations}\n)\n\n# item_features dataframe\nitem_ids = np.arange(1, 101)\nprices = np.random.uniform(10, 500, size=100).round(2)\ncolors = np.random.choice([\"red\", \"blue\", \"green\", \"black\"], size=100)\ncategories = np.random.choice([\"electronics\", \"clothing\", \"home\", \"toys\"], size=100)\n\nitem_features = pd.DataFrame(\n    {\"id\": item_ids, \"price\": prices, \"color\": colors, \"category\": categories}\n)\n\n# Interactions dataframe\ninteraction_user_ids = np.random.choice(user_ids, size=1000)\ninteraction_item_ids = np.random.choice(item_ids, size=1000)\npurchased = np.random.choice([0, 1], size=1000, p=[0.7, 0.3])\ninteractions = pd.DataFrame(\n    {\n        \"user_id\": interaction_user_ids,\n        \"item_id\": interaction_item_ids,\n        \"purchased\": purchased,\n    }\n)\nuser_item_purchased = interactions.merge(\n    user_features, left_on=\"user_id\", right_on=\"id\"\n).merge(item_features, left_on=\"item_id\", right_on=\"id\")\n\n# Users\ntab_preprocessor_user = TabPreprocessor(\n    cat_embed_cols=[\"gender\", \"location\"],\n    continuous_cols=[\"age\"],\n)\nX_user = tab_preprocessor_user.fit_transform(user_item_purchased)\ntab_mlp_user = TabMlp(\n    column_idx=tab_preprocessor_user.column_idx,\n    cat_embed_input=tab_preprocessor_user.cat_embed_input,\n    continuous_cols=[\"age\"],\n    mlp_hidden_dims=[16, 8],\n    mlp_dropout=[0.2, 0.2],\n)\n\n# Items\ntab_preprocessor_item = TabPreprocessor(\n    cat_embed_cols=[\"color\", \"category\"],\n    continuous_cols=[\"price\"],\n)\nX_item = tab_preprocessor_item.fit_transform(user_item_purchased)\ntab_mlp_item = TabMlp(\n    column_idx=tab_preprocessor_item.column_idx,\n    cat_embed_input=tab_preprocessor_item.cat_embed_input,\n    continuous_cols=[\"price\"],\n    mlp_hidden_dims=[16, 8],\n    mlp_dropout=[0.2, 0.2],\n)\n\ntwo_tower_model = ModelFuser([tab_mlp_user, tab_mlp_item], fusion_method=\"dot\")\n\nmodel = WideDeep(deeptabular=two_tower_model)\n\ntrainer = Trainer(model, objective=\"binary\")\n\ntrainer.fit(\n    X_tab=[X_user, X_item],\n    target=interactions.purchased.values,\n    n_epochs=1,\n    batch_size=32,\n)\n</code></pre> <p>8. Tabular with a multi-target loss</p> <p>This one is \"a bonus\" to illustrate the use of multi-target losses, more than actually a different architecture.</p> <p> </p> <pre><code>from pytorch_widedeep.preprocessing import TabPreprocessor, TextPreprocessor, ImagePreprocessor\nfrom pytorch_widedeep.models import TabMlp, BasicRNN, WideDeep, ModelFuser, Vision\nfrom pytorch_widedeep.losses_multitarget import MultiTargetClassificationLoss\nfrom pytorch_widedeep.models._base_wd_model_component import BaseWDModelComponent\nfrom pytorch_widedeep import Trainer\n\n# let's add a second target to the dataframe\ndf[\"target2\"] = [random.choice([0, 1]) for _ in range(100)]\n\n# Tabular\ntab_preprocessor = TabPreprocessor(\n    embed_cols=[\"city\", \"name\"], continuous_cols=[\"age\", \"height\"]\n)\nX_tab = tab_preprocessor.fit_transform(df)\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=tab_preprocessor.continuous_cols,\n    mlp_hidden_dims=[64, 32],\n)\n\n# 'pred_dim=2' because we have two binary targets. For other types of targets,\n#  please, see the documentation\nmodel = WideDeep(deeptabular=tab_mlp, pred_dim=2).\n\nloss = MultiTargetClassificationLoss(binary_config=[0, 1], reduction=\"mean\")\n\n# When a multi-target loss is used, 'custom_loss_function' must not be None.\n# See the docs\ntrainer = Trainer(model, objective=\"multitarget\", custom_loss_function=loss)\n\ntrainer.fit(\n    X_tab=X_tab,\n    target=df[[\"target\", \"target2\"]].values,\n    n_epochs=1,\n    batch_size=32,\n)\n</code></pre>"},{"location":"index.html#the-deeptabular-component","title":"The <code>deeptabular</code> component","text":"<p>It is important to emphasize again that each individual component, <code>wide</code>, <code>deeptabular</code>, <code>deeptext</code> and <code>deepimage</code>, can be used independently and in isolation. For example, one could use only <code>wide</code>, which is in simply a linear model. In fact, one of the most interesting functionalities in<code>pytorch-widedeep</code> would be the use of the <code>deeptabular</code> component on its own, i.e. what one might normally refer as Deep Learning for Tabular Data. Currently, <code>pytorch-widedeep</code> offers the following different models for that component:</p> <ol> <li>Wide: a simple linear model where the nonlinearities are captured via cross-product transformations, as explained before.</li> <li>TabMlp: a simple MLP that receives embeddings representing the categorical features, concatenated with the continuous features, which can also be embedded.</li> <li>TabResnet: similar to the previous model but the embeddings are passed through a series of ResNet blocks built with dense layers.</li> <li>TabNet: details on TabNet can be found in TabNet: Attentive Interpretable Tabular Learning</li> </ol> <p>Two simpler attention based models that we call:</p> <ol> <li>ContextAttentionMLP: MLP with at attention mechanism \"on top\" that is based on     Hierarchical Attention Networks for Document Classification</li> <li>SelfAttentionMLP: MLP with an attention mechanism that is a simplified     version of a transformer block that we refer as \"query-key self-attention\".</li> </ol> <p>The <code>Tabformer</code> family, i.e. Transformers for Tabular data:</p> <ol> <li>TabTransformer: details on the TabTransformer can be found in TabTransformer: Tabular Data Modeling Using Contextual Embeddings.</li> <li>SAINT: Details on SAINT can be found in SAINT: Improved Neural Networks for Tabular Data via Row Attention and Contrastive Pre-Training.</li> <li>FT-Transformer: details on the FT-Transformer can be found in Revisiting Deep Learning Models for Tabular Data.</li> <li>TabFastFormer: adaptation of the FastFormer for tabular data. Details on the Fasformer can be found in FastFormers: Highly Efficient Transformer Models for Natural Language Understanding</li> <li>TabPerceiver: adaptation of the Perceiver for tabular data. Details on the Perceiver can be found in Perceiver: General Perception with Iterative Attention</li> </ol> <p>And probabilistic DL models for tabular data based on Weight Uncertainty in Neural Networks:</p> <ol> <li>BayesianWide: Probabilistic adaptation of the <code>Wide</code> model.</li> <li>BayesianTabMlp: Probabilistic adaptation of the <code>TabMlp</code> model</li> </ol> <p>Note that while there are scientific publications for the TabTransformer, SAINT and FT-Transformer, the TabFasfFormer and TabPerceiver are our own adaptation of those algorithms for tabular data.</p> <p>In addition, Self-Supervised pre-training can be used for all <code>deeptabular</code> models, with the exception of the <code>TabPerceiver</code>. Self-Supervised pre-training can be used via two methods or routines which we refer as: encoder-decoder method and constrastive-denoising method. Please, see the documentation and the examples for details on this functionality, and all other options in the library.</p>"},{"location":"index.html#the-rec-module","title":"The <code>rec</code> module","text":"<p>This module was introduced as an extension to the existing components in the library, addressing questions and issues related to recommendation systems. While still under active development, it currently includes a select number of powerful recommendation models.</p> <p>It's worth noting that this library already supported the implementation of various recommendation algorithms using existing components. For example, models like Wide and Deep, Two-Tower, or Neural Collaborative Filtering could be constructed using the library's core functionalities.</p> <p>The recommendation algorithms in the <code>rec</code> module are:</p> <ol> <li>DeepFM: A Factorization-Machine based Neural Network for CTR Prediction</li> <li>(Deep) Field Aware Factorization Machine (FFM): a Deep Learning version of the algorithm presented in Field-aware Factorization Machines in a Real-world Online Advertising System</li> <li>xDeepFM: Combining Explicit and Implicit Feature Interactions for Recommender Systems</li> <li>Deep Interest Network for Click-Through Rate Prediction</li> </ol> <p>These can all be used as the <code>deeptabular</code> component in the <code>WideDeep</code> model. See the examples for more details.</p>"},{"location":"index.html#text-and-images","title":"Text and Images","text":"<p>For the text component, <code>deeptext</code>, the library offers the following models:</p> <ol> <li>BasicRNN: a simple RNN 2. AttentiveRNN: a RNN with an attention mechanism based on the Hierarchical Attention Networks for DocumentClassification</li> <li>StackedAttentiveRNN: a stack of AttentiveRNNs</li> <li>HFModel: a wrapper around Hugging Face Transfomer-based models. At the moment only models from the families BERT, RoBERTa, DistilBERT, ALBERT and ELECTRA are supported. This is because this library is designed to address classification and regression tasks and these are the most 'popular' encoder-only models, which have proved to be those that work best for these tasks. If there is demand for other models, they will be included in the future.</li> </ol> <p>For the image component, <code>deepimage</code>, the library supports models from the following families: 'resnet', 'shufflenet', 'resnext', 'wide_resnet', 'regnet', 'densenet', 'mobilenetv3',  'mobilenetv2', 'mnasnet', 'efficientnet' and 'squeezenet'.  These are  offered via <code>torchvision</code> and wrapped up in the <code>Vision</code> class.</p>"},{"location":"index.html#acknowledgments","title":"Acknowledgments","text":"<p>This library takes from a series of other libraries, so I think it is just fair to mention them here in the README (specific mentions are also included in the code).</p> <p>The <code>Callbacks</code> and <code>Initializers</code> structure and code is inspired by the <code>torchsample</code> library, which in itself partially inspired by <code>Keras</code>.</p> <p>The <code>TextProcessor</code> class in this library uses the <code>fastai</code>'s <code>Tokenizer</code> and <code>Vocab</code>. The code at <code>utils.fastai_transforms</code> is a minor adaptation of their code so it functions within this library. To my experience their <code>Tokenizer</code> is the best in class.</p> <p>The <code>ImageProcessor</code> class in this library uses code from the fantastic Deep Learning for Computer Vision (DL4CV) book by Adrian Rosebrock.</p>"},{"location":"index.html#license","title":"License","text":"<p>This work is dual-licensed under Apache 2.0 and MIT (or any later version). You can choose between one of them if you use this work.</p> <p><code>SPDX-License-Identifier: Apache-2.0 AND MIT</code></p>"},{"location":"index.html#cite","title":"Cite","text":""},{"location":"index.html#bibtex","title":"BibTex","text":"<pre><code>@article{Zaurin_pytorch-widedeep_A_flexible_2023,\nauthor = {Zaurin, Javier Rodriguez and Mulinka, Pavol},\ndoi = {10.21105/joss.05027},\njournal = {Journal of Open Source Software},\nmonth = jun,\nnumber = {86},\npages = {5027},\ntitle = {{pytorch-widedeep: A flexible package for multimodal deep learning}},\nurl = {https://joss.theoj.org/papers/10.21105/joss.05027},\nvolume = {8},\nyear = {2023}\n}\n</code></pre>"},{"location":"index.html#apa","title":"APA","text":"<pre><code>Zaurin, J. R., &amp; Mulinka, P. (2023). pytorch-widedeep: A flexible package for\nmultimodal deep learning. Journal of Open Source Software, 8(86), 5027.\nhttps://doi.org/10.21105/joss.05027\n</code></pre>"},{"location":"contributing.html","title":"Contributing","text":"<p>Pytorch-widedeep is being developed and used by many active community members. Your help is very valuable to make it better for everyone.</p> <ul> <li>Check for the Roadmap or Open an issue to report problems or recommend new features and submit a draft pull requests, which will be changed to pull request after intial review</li> <li>Contribute to the tests to make it more reliable.</li> <li>Contribute to the documentation to make it clearer for everyone.</li> <li>Contribute to the examples to share your experience with other users.</li> <li>Join the dicussion on slack</li> </ul>"},{"location":"installation.html","title":"Installation","text":"<p>This section explains how to install <code>pytorch-widedeep</code>.</p> <p>For the latest stable release, execute:</p> <pre><code>pip install pytorch-widedeep\n</code></pre> <p>For the bleeding-edge version, execute:</p> <pre><code>pip install git+https://github.com/jrzaurin/pytorch-widedeep.git\n</code></pre> <p>For developer install</p> <pre><code># Clone the repository\ngit clone https://github.com/jrzaurin/pytorch-widedeep\ncd pytorch-widedeep\n\n# Install in dev mode\npip install -e .\n</code></pre>"},{"location":"installation.html#dependencies","title":"Dependencies","text":"<ul> <li>pandas&gt;=1.3.5</li> <li>numpy&gt;=1.21.6</li> <li>scipy&gt;=1.7.3,&lt;=1.12.0</li> <li>scikit-learn&gt;=1.0.2</li> <li>gensim</li> <li>spacy</li> <li>opencv-contrib-python</li> <li>imutils</li> <li>tqdm</li> <li>torch &gt;= 2.0.0</li> <li>torchvision &gt;= 0.15.0</li> <li>einops</li> <li>wrapt</li> <li>torchmetrics</li> <li>pyarrow</li> <li>fastparquet&gt;=0.8.1</li> <li>transformers</li> <li>sentence-transformers</li> <li>sentencepiece</li> </ul>"},{"location":"quick_start.html","title":"Quick Start","text":"<p>This is an example of a binary classification with the adult census dataset using a combination of a wide and deep model (in this case a so called <code>deeptabular</code> model) with defaults settings.</p> <pre><code>import numpy as np\nimport torch\nfrom sklearn.model_selection import train_test_split\n\nfrom pytorch_widedeep import Trainer\nfrom pytorch_widedeep.preprocessing import WidePreprocessor, TabPreprocessor\nfrom pytorch_widedeep.models import Wide, TabMlp, WideDeep\nfrom pytorch_widedeep.metrics import Accuracy\nfrom pytorch_widedeep.datasets import load_adult\n\n\ndf = load_adult(as_frame=True)\ndf[\"income_label\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int)\ndf.drop(\"income\", axis=1, inplace=True)\ndf_train, df_test = train_test_split(df, test_size=0.2, stratify=df.income_label)\n\n# Define the 'column set up'\nwide_cols = [\n    \"education\",\n    \"relationship\",\n    \"workclass\",\n    \"occupation\",\n    \"native-country\",\n    \"gender\",\n]\ncrossed_cols = [(\"education\", \"occupation\"), (\"native-country\", \"occupation\")]\n\ncat_embed_cols = [\n    \"workclass\",\n    \"education\",\n    \"marital-status\",\n    \"occupation\",\n    \"relationship\",\n    \"race\",\n    \"gender\",\n    \"capital-gain\",\n    \"capital-loss\",\n    \"native-country\",\n]\ncontinuous_cols = [\"age\", \"hours-per-week\"]\ntarget = \"income_label\"\ntarget = df_train[target].values\n\n# prepare the data\nwide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols)\nX_wide = wide_preprocessor.fit_transform(df_train)\n\ntab_preprocessor = TabPreprocessor(\n    cat_embed_cols=cat_embed_cols, continuous_cols=continuous_cols  # type: ignore[arg-type]\n)\nX_tab = tab_preprocessor.fit_transform(df_train)\n\n# build the model\nwide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1)\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=continuous_cols,\n)\nmodel = WideDeep(wide=wide, deeptabular=tab_mlp)\n\n# train and validate\ntrainer = Trainer(model, objective=\"binary\", metrics=[Accuracy])\ntrainer.fit(\n    X_wide=X_wide,\n    X_tab=X_tab,\n    target=target,\n    n_epochs=5,\n    batch_size=256,\n)\n\n# predict on test\nX_wide_te = wide_preprocessor.transform(df_test)\nX_tab_te = tab_preprocessor.transform(df_test)\npreds = trainer.predict(X_wide=X_wide_te, X_tab=X_tab_te)\n\n# Save and load\n\n# Option 1: this will also save training history and lr history if the\n# LRHistory callback is used\ntrainer.save(path=\"model_weights\", save_state_dict=True)\n\n# Option 2: save as any other torch model\ntorch.save(model.state_dict(), \"model_weights/wd_model.pt\")\n\n# From here in advance, Option 1 or 2 are the same. I assume the user has\n# prepared the data and defined the new model components:\n# 1. Build the model\nmodel_new = WideDeep(wide=wide, deeptabular=tab_mlp)\nmodel_new.load_state_dict(torch.load(\"model_weights/wd_model.pt\"))\n\n# 2. Instantiate the trainer\ntrainer_new = Trainer(model_new, objective=\"binary\")\n\n# 3. Either start the fit or directly predict\npreds = trainer_new.predict(X_wide=X_wide, X_tab=X_tab, batch_size=32)\n</code></pre>"},{"location":"examples/01_preprocessors_and_utils.html","title":"01_preprocessors_and_utils","text":"<p>For example</p> In\u00a0[1]: Copied! <pre>import numpy as np\nimport pandas as pd\nimport pytorch_widedeep as wd\n\nfrom pytorch_widedeep.datasets import load_adult\nfrom pytorch_widedeep.preprocessing import WidePreprocessor\n</pre> import numpy as np import pandas as pd import pytorch_widedeep as wd  from pytorch_widedeep.datasets import load_adult from pytorch_widedeep.preprocessing import WidePreprocessor <pre>/Users/javierrodriguezzaurin/.pyenv/versions/3.10.13/envs/widedeep310/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n  from .autonotebook import tqdm as notebook_tqdm\n</pre> In\u00a0[2]: Copied! <pre>df = load_adult(as_frame=True)\ndf.head()\n</pre> df = load_adult(as_frame=True) df.head() Out[2]: age workclass fnlwgt education educational-num marital-status occupation relationship race gender capital-gain capital-loss hours-per-week native-country income 0 25 Private 226802 11th 7 Never-married Machine-op-inspct Own-child Black Male 0 0 40 United-States &lt;=50K 1 38 Private 89814 HS-grad 9 Married-civ-spouse Farming-fishing Husband White Male 0 0 50 United-States &lt;=50K 2 28 Local-gov 336951 Assoc-acdm 12 Married-civ-spouse Protective-serv Husband White Male 0 0 40 United-States &gt;50K 3 44 Private 160323 Some-college 10 Married-civ-spouse Machine-op-inspct Husband Black Male 7688 0 40 United-States &gt;50K 4 18 ? 103497 Some-college 10 Never-married ? Own-child White Female 0 0 30 United-States &lt;=50K In\u00a0[3]: Copied! <pre>wide_cols = [\n    \"education\",\n    \"relationship\",\n    \"workclass\",\n    \"occupation\",\n    \"native-country\",\n    \"gender\",\n]\ncrossed_cols = [(\"education\", \"occupation\"), (\"native-country\", \"occupation\")]\n</pre> wide_cols = [     \"education\",     \"relationship\",     \"workclass\",     \"occupation\",     \"native-country\",     \"gender\", ] crossed_cols = [(\"education\", \"occupation\"), (\"native-country\", \"occupation\")] In\u00a0[4]: Copied! <pre>wide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols)\nX_wide = wide_preprocessor.fit_transform(df)\n# From here on, any new observation can be prepared by simply running `.transform`\n# new_X_wide = wide_preprocessor.transform(new_df)\n</pre> wide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols) X_wide = wide_preprocessor.fit_transform(df) # From here on, any new observation can be prepared by simply running `.transform` # new_X_wide = wide_preprocessor.transform(new_df) In\u00a0[5]: Copied! <pre>X_wide\n</pre> X_wide Out[5]: <pre>array([[  1,  17,  23, ...,  89,  91, 316],\n       [  2,  18,  23, ...,  89,  92, 317],\n       [  3,  18,  24, ...,  89,  93, 318],\n       ...,\n       [  2,  20,  23, ...,  90, 103, 323],\n       [  2,  17,  23, ...,  89, 103, 323],\n       [  2,  21,  29, ...,  90, 115, 324]])</pre> <p>Note that the label encoding starts from <code>1</code>. This is because it is convenient to leave <code>0</code> for padding, i.e. unknown categories. Let's take from example the first entry</p> In\u00a0[6]: Copied! <pre>X_wide[0]\n</pre> X_wide[0] Out[6]: <pre>array([  1,  17,  23,  32,  47,  89,  91, 316])</pre> In\u00a0[7]: Copied! <pre>wide_preprocessor.inverse_transform(X_wide[:1])\n</pre> wide_preprocessor.inverse_transform(X_wide[:1]) Out[7]: education relationship workclass occupation native-country gender education_occupation native-country_occupation 0 11th Own-child Private Machine-op-inspct United-States Male 11th-Machine-op-inspct United-States-Machine-op-inspct <p>As we can see, <code>wide_preprocessor</code> numerically encodes the <code>wide_cols</code> and the <code>crossed_cols</code>, which can be recovered using the method <code>inverse_transform</code>.</p> In\u00a0[8]: Copied! <pre>from pytorch_widedeep.preprocessing import TabPreprocessor\n</pre> from pytorch_widedeep.preprocessing import TabPreprocessor In\u00a0[9]: Copied! <pre># cat_embed_cols = [(column_name, embed_dim), ...]\ncat_embed_cols = [\n    (\"education\", 10),\n    (\"relationship\", 8),\n    (\"workclass\", 10),\n    (\"occupation\", 10),\n    (\"native-country\", 10),\n]\ncontinuous_cols = [\"age\", \"hours-per-week\"]\n</pre> # cat_embed_cols = [(column_name, embed_dim), ...] cat_embed_cols = [     (\"education\", 10),     (\"relationship\", 8),     (\"workclass\", 10),     (\"occupation\", 10),     (\"native-country\", 10), ] continuous_cols = [\"age\", \"hours-per-week\"] In\u00a0[10]: Copied! <pre>tab_preprocessor = TabPreprocessor(\n    cat_embed_cols=cat_embed_cols,\n    continuous_cols=continuous_cols,\n    cols_to_scale=[\"age\"],  # or scale=True or cols_to_scale=continuous_cols\n)\nX_tab = tab_preprocessor.fit_transform(df)\n# From here on, any new observation can be prepared by simply running `.transform`\n# new_X_deep = deep_preprocessor.transform(new_df)\n</pre> tab_preprocessor = TabPreprocessor(     cat_embed_cols=cat_embed_cols,     continuous_cols=continuous_cols,     cols_to_scale=[\"age\"],  # or scale=True or cols_to_scale=continuous_cols ) X_tab = tab_preprocessor.fit_transform(df) # From here on, any new observation can be prepared by simply running `.transform` # new_X_deep = deep_preprocessor.transform(new_df) In\u00a0[11]: Copied! <pre>X_tab\n</pre> X_tab Out[11]: <pre>array([[ 1.00000000e+00,  1.00000000e+00,  1.00000000e+00, ...,\n         1.00000000e+00, -9.95128932e-01,  4.00000000e+01],\n       [ 2.00000000e+00,  2.00000000e+00,  1.00000000e+00, ...,\n         1.00000000e+00, -4.69415091e-02,  5.00000000e+01],\n       [ 3.00000000e+00,  2.00000000e+00,  2.00000000e+00, ...,\n         1.00000000e+00, -7.76316450e-01,  4.00000000e+01],\n       ...,\n       [ 2.00000000e+00,  4.00000000e+00,  1.00000000e+00, ...,\n         1.00000000e+00,  1.41180837e+00,  4.00000000e+01],\n       [ 2.00000000e+00,  1.00000000e+00,  1.00000000e+00, ...,\n         1.00000000e+00, -1.21394141e+00,  2.00000000e+01],\n       [ 2.00000000e+00,  5.00000000e+00,  7.00000000e+00, ...,\n         1.00000000e+00,  9.74183408e-01,  4.00000000e+01]])</pre> <p>Note that the label encoding starts from <code>1</code>. This is because it is convenient to leave <code>0</code> for padding, i.e. unknown categories. Let's take from example the first entry</p> In\u00a0[12]: Copied! <pre>X_tab[0]\n</pre> X_tab[0] Out[12]: <pre>array([ 1.        ,  1.        ,  1.        ,  1.        ,  1.        ,\n       -0.99512893, 40.        ])</pre> In\u00a0[13]: Copied! <pre>tab_preprocessor.inverse_transform(X_tab[:1])\n</pre> tab_preprocessor.inverse_transform(X_tab[:1]) Out[13]: education relationship workclass occupation native-country age hours-per-week 0 11th Own-child Private Machine-op-inspct United-States 25.0 40.0 <p>The <code>TabPreprocessor</code> will have a series of useful attributes that can later be used when instantiating the different Tabular Models, such us for example, the column indexes (used to slice the tensors, internally in the models) or the categorical embeddings set up</p> In\u00a0[14]: Copied! <pre>tab_preprocessor.column_idx\n</pre> tab_preprocessor.column_idx Out[14]: <pre>{'education': 0,\n 'relationship': 1,\n 'workclass': 2,\n 'occupation': 3,\n 'native-country': 4,\n 'age': 5,\n 'hours-per-week': 6}</pre> In\u00a0[15]: Copied! <pre># column name, num unique, embedding dim\ntab_preprocessor.cat_embed_input\n</pre> # column name, num unique, embedding dim tab_preprocessor.cat_embed_input Out[15]: <pre>[('education', 16, 10),\n ('relationship', 6, 8),\n ('workclass', 9, 10),\n ('occupation', 15, 10),\n ('native-country', 42, 10)]</pre> <p>As I mentioned, there is more one can do, such as for example, quantize (or bucketize) the continuous cols. For this we could use the <code>quantization_setup</code> param. This parameter accepts a number of different inputs and uses <code>pd.cut</code> under the hood to quantize the continuous cols. For more info, please, read the docs. Let's use it here to quantize \"age\" and \"hours-per-week\" in 4 and 5 \"buckets\" respectively</p> In\u00a0[16]: Copied! <pre>quantization_setup = {\n    \"age\": 4,\n    \"hours-per-week\": 5,\n}  # you can also pass a list of floats with the boundaries if you wanted\nquant_tab_preprocessor = TabPreprocessor(\n    cat_embed_cols=cat_embed_cols,\n    continuous_cols=continuous_cols,\n    quantization_setup=quantization_setup,\n)\nqX_tab = quant_tab_preprocessor.fit_transform(df)\n</pre> quantization_setup = {     \"age\": 4,     \"hours-per-week\": 5, }  # you can also pass a list of floats with the boundaries if you wanted quant_tab_preprocessor = TabPreprocessor(     cat_embed_cols=cat_embed_cols,     continuous_cols=continuous_cols,     quantization_setup=quantization_setup, ) qX_tab = quant_tab_preprocessor.fit_transform(df) <pre>/Users/javierrodriguezzaurin/Projects/pytorch-widedeep/pytorch_widedeep/preprocessing/tab_preprocessor.py:358: UserWarning: Continuous columns will not be normalised\n  warnings.warn(\"Continuous columns will not be normalised\")\n</pre> In\u00a0[17]: Copied! <pre>qX_tab\n</pre> qX_tab Out[17]: <pre>array([[1, 1, 1, ..., 1, 1, 2],\n       [2, 2, 1, ..., 1, 2, 3],\n       [3, 2, 2, ..., 1, 1, 2],\n       ...,\n       [2, 4, 1, ..., 1, 3, 2],\n       [2, 1, 1, ..., 1, 1, 1],\n       [2, 5, 7, ..., 1, 2, 2]])</pre> <p>Note that the continuous columns that have been bucketised into quantiles are treated as any other categorical column</p> In\u00a0[18]: Copied! <pre>quant_tab_preprocessor.cat_embed_input\n</pre> quant_tab_preprocessor.cat_embed_input Out[18]: <pre>[('education', 16, 10),\n ('relationship', 6, 8),\n ('workclass', 9, 10),\n ('occupation', 15, 10),\n ('native-country', 42, 10),\n ('age', 4, 4),\n ('hours-per-week', 5, 4)]</pre> <p>Where the column 'age' has now 4 categories, which will be encoded using embeddings of 4 dims. Note that, as any other categorical columns, the categorical \"counter\" starts with 1. This is because all incoming values that are lower/higher than the existing lowest/highest value in the train (or already seen) dataset, will be encoded as 0.</p> In\u00a0[19]: Copied! <pre>np.unique(qX_tab[:, quant_tab_preprocessor.column_idx[\"age\"]])\n</pre> np.unique(qX_tab[:, quant_tab_preprocessor.column_idx[\"age\"]]) Out[19]: <pre>array([1, 2, 3, 4])</pre> <p>Finally, if we now wanted to <code>inverse_transform</code> the transformed array into the original dataframe, we could still do it, but the continuous, bucketized columns will be transformed back to the middle of their quantile/bucket range</p> In\u00a0[20]: Copied! <pre>df_decoded = quant_tab_preprocessor.inverse_transform(qX_tab)\n</pre> df_decoded = quant_tab_preprocessor.inverse_transform(qX_tab) <pre>Note that quantized cols will be turned into the mid point of the corresponding bin\n</pre> In\u00a0[21]: Copied! <pre>df.head(2)\n</pre> df.head(2) Out[21]: age workclass fnlwgt education educational-num marital-status occupation relationship race gender capital-gain capital-loss hours-per-week native-country income 0 25 Private 226802 11th 7 Never-married Machine-op-inspct Own-child Black Male 0 0 40 United-States &lt;=50K 1 38 Private 89814 HS-grad 9 Married-civ-spouse Farming-fishing Husband White Male 0 0 50 United-States &lt;=50K In\u00a0[22]: Copied! <pre>df_decoded.head(2)\n</pre> df_decoded.head(2) Out[22]: education relationship workclass occupation native-country age hours-per-week 0 11th Own-child Private Machine-op-inspct United-States 26.0885 30.4 1 HS-grad Husband Private Farming-fishing United-States 44.3750 50.0 <p>there is one final comment to make regarding to the <code>inverse_transform</code> functionality. As we mentioned before, the encoding <code>0</code> is reserved for values that fall outside the range covered by the data we used to run the <code>fit</code> method. For example</p> In\u00a0[23]: Copied! <pre>df.age.min(), df.age.max()\n</pre> df.age.min(), df.age.max() Out[23]: <pre>(17, 90)</pre> <p>All future age values outside that range will be encoded as 0 and decoded as <code>NaN</code></p> In\u00a0[24]: Copied! <pre>tmp_df = df.head(1).copy()\ntmp_df.loc[:, \"age\"] = 5\n</pre> tmp_df = df.head(1).copy() tmp_df.loc[:, \"age\"] = 5 In\u00a0[25]: Copied! <pre>tmp_df\n</pre> tmp_df Out[25]: age workclass fnlwgt education educational-num marital-status occupation relationship race gender capital-gain capital-loss hours-per-week native-country income 0 5 Private 226802 11th 7 Never-married Machine-op-inspct Own-child Black Male 0 0 40 United-States &lt;=50K In\u00a0[26]: Copied! <pre># quant_tab_preprocessor has already been fitted with a data that has an age range between 17 and 90\ntmp_qX_tab = quant_tab_preprocessor.transform(tmp_df)\n</pre> # quant_tab_preprocessor has already been fitted with a data that has an age range between 17 and 90 tmp_qX_tab = quant_tab_preprocessor.transform(tmp_df) In\u00a0[27]: Copied! <pre>tmp_qX_tab\n</pre> tmp_qX_tab Out[27]: <pre>array([[1, 1, 1, 1, 1, 0, 2]])</pre> In\u00a0[28]: Copied! <pre>quant_tab_preprocessor.inverse_transform(tmp_qX_tab)\n</pre> quant_tab_preprocessor.inverse_transform(tmp_qX_tab) <pre>Note that quantized cols will be turned into the mid point of the corresponding bin\n</pre> Out[28]: education relationship workclass occupation native-country age hours-per-week 0 11th Own-child Private Machine-op-inspct United-States NaN 30.4 In\u00a0[29]: Copied! <pre>from pytorch_widedeep.preprocessing import TextPreprocessor\n</pre> from pytorch_widedeep.preprocessing import TextPreprocessor In\u00a0[30]: Copied! <pre># The airbnb dataset, which you could get from here:\n# http://insideairbnb.com/get-the-data.html, is too big to be included in\n# our datasets module (when including images). Therefore, go there,\n# download it, and use the download_images.py script to get the images\n# and the airbnb_data_processing.py to process the data. We'll find\n# better datasets in the future ;). Note that here we are only using a\n# small sample to illustrate the use, so PLEASE ignore the results, just\n# focus on usage\ndf = pd.read_csv(\"../tmp_data/airbnb/airbnb_sample.csv\")\n</pre> # The airbnb dataset, which you could get from here: # http://insideairbnb.com/get-the-data.html, is too big to be included in # our datasets module (when including images). Therefore, go there, # download it, and use the download_images.py script to get the images # and the airbnb_data_processing.py to process the data. We'll find # better datasets in the future ;). Note that here we are only using a # small sample to illustrate the use, so PLEASE ignore the results, just # focus on usage df = pd.read_csv(\"../tmp_data/airbnb/airbnb_sample.csv\") In\u00a0[31]: Copied! <pre>texts = df.description.tolist()\ntexts[:2]\n</pre> texts = df.description.tolist() texts[:2] Out[31]: <pre>[\"My bright double bedroom with a large window has a relaxed feeling! It comfortably fits one or two and is centrally located just two blocks from Finsbury Park. Enjoy great restaurants in the area and easy access to easy transport tubes, trains and buses. Babies and children of all ages are welcome. Hello Everyone, I'm offering my lovely double bedroom in Finsbury Park area (zone 2) for let in a shared apartment.  You will share the apartment with me and it is fully furnished with a self catering kitchen. Two people can easily sleep well as the room has a queen size bed. I also have a travel cot for a baby for guest with small children.  I will require a deposit up front as a security gesture on both our parts and will be given back to you when you return the keys.  I trust anyone who will be responding to this add would treat my home with care and respect .  Best Wishes  Alina Guest will have access to the self catering kitchen and bathroom. There is the flat is equipped wifi internet,\",\n \"Lots of windows and light.  St Luke's Gardens are at the end of the block, and the river not too far the other way. Ten minutes walk if you go slowly. Buses to everywhere round the corner and shops, restaurants, pubs, the cinema and Waitrose . Bright Chelsea Apartment  This is a bright one bedroom ground floor apartment in an interesting listed building. There is one double bedroom and a living room/kitchen The apartment has a full  bathroom and the kitchen is fully equipped. Two wardrobes are available exclusively for guests and bedside tables and two long drawers. This sunny convenient compact flat is just around the corner from the Waitrose supermarket and all sorts of shops, cinemas, restaurants and pubs.  This is a lovely part of London. There is a fun farmers market in the King's Road at the weekend.  Buses to everywhere are just round the corner, and two underground stations are within ten minutes walk. There is a very nice pub round by St. Luke's gardens, 4 mins slow walk, the \"]</pre> In\u00a0[32]: Copied! <pre>text_preprocessor = TextPreprocessor(text_col=\"description\")\nX_text = text_preprocessor.fit_transform(df)\n# From here on, any new observation can be prepared by simply running `.transform`\n# new_X_text = text_preprocessor.transform(new_df)\n</pre> text_preprocessor = TextPreprocessor(text_col=\"description\") X_text = text_preprocessor.fit_transform(df) # From here on, any new observation can be prepared by simply running `.transform` # new_X_text = text_preprocessor.transform(new_df) <pre>The vocabulary contains 2192 tokens\n</pre> In\u00a0[33]: Copied! <pre>print(X_text[0])\n</pre> print(X_text[0]) <pre>[  29   48   37  367  818   17  910   17  177   15  122  349   53  879\n 1174  126  393   40  911    0   23  228   71  819    9   53   55 1380\n  225   11   18  308   18 1564   10  755    0  942  239   53   55    0\n   11   36 1013  277 1974   70   62   15 1475    9  943    5  251    5\n    0    5    0    5  177   53   37   75   11   10  294  726   32    9\n   42    5   25   12   10   22   12  136  100  145]\n</pre> In\u00a0[34]: Copied! <pre>from pytorch_widedeep.preprocessing import ImagePreprocessor\n</pre> from pytorch_widedeep.preprocessing import ImagePreprocessor In\u00a0[35]: Copied! <pre>image_preprocessor = wd.preprocessing.ImagePreprocessor(\n    img_col=\"id\", img_path=\"../tmp_data/airbnb/property_picture/\"\n)\nX_images = image_preprocessor.fit_transform(df)\n# From here on, any new observation can be prepared by simply running `.transform`\n# new_X_images = image_preprocessor.transform(new_df)\n</pre> image_preprocessor = wd.preprocessing.ImagePreprocessor(     img_col=\"id\", img_path=\"../tmp_data/airbnb/property_picture/\" ) X_images = image_preprocessor.fit_transform(df) # From here on, any new observation can be prepared by simply running `.transform` # new_X_images = image_preprocessor.transform(new_df) <pre>Reading Images from ../tmp_data/airbnb/property_picture/\nResizing\n</pre> <pre>100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 1001/1001 [00:01&lt;00:00, 667.89it/s]\n</pre> <pre>Computing normalisation metrics\n</pre> In\u00a0[36]: Copied! <pre>X_images[0].shape\n</pre> X_images[0].shape Out[36]: <pre>(224, 224, 3)</pre>"},{"location":"examples/01_preprocessors_and_utils.html#processors-and-utils","title":"Processors and Utils\u00b6","text":"<p>Description of the main tools and utilities that one needs to prepare the data for a <code>WideDeep</code> model constructor.</p>"},{"location":"examples/01_preprocessors_and_utils.html#the-preprocessing-module","title":"The <code>preprocessing</code>  module\u00b6","text":"<p>There are 4 preprocessors, corresponding to 4 main components of the <code>WideDeep</code> model. These are</p> <ul> <li><code>WidePreprocessor</code></li> <li><code>TabPreprocessor</code></li> <li><code>TextPreprocessor</code></li> <li><code>ImagePreprocessor</code></li> </ul> <p>Behind the scenes, these preprocessors use a series of helper funcions and classes that are in the <code>utils</code> module. If you were interested please go and have a look to the documentation</p>"},{"location":"examples/01_preprocessors_and_utils.html#1-widepreprocessor","title":"1. WidePreprocessor\u00b6","text":"<p>The <code>wide</code> component of the model is a linear model that in principle, could be implemented as a linear layer receiving the result of on one-hot encoding categorical columns. However, this is not memory efficient. Therefore, we implement a liner layer as an Embedding layer plus a bias. I will explain in a bit more detail later.</p> <p>With that in mind, <code>WidePreprocessor</code> simply encodes the categories numerically so that they are the indexes of the lookup table that is an Embedding layer.</p>"},{"location":"examples/01_preprocessors_and_utils.html#2-tabpreprocessor","title":"2. TabPreprocessor\u00b6","text":"<p>The <code>TabPreprocessor</code> has a lot of different functionalities. Let's explore some of them in detail. In its basic use, the <code>TabPreprocessor</code> simply label encodes the categorical columns and normalises the numerical ones (unless otherwised specified).</p>"},{"location":"examples/01_preprocessors_and_utils.html#3-textpreprocessor","title":"3. TextPreprocessor\u00b6","text":"<p>This preprocessor returns the tokenised, padded sequences that will be directly fed to the stack of LSTMs.</p>"},{"location":"examples/01_preprocessors_and_utils.html#4-imagepreprocessor","title":"4. ImagePreprocessor\u00b6","text":"<p><code>ImagePreprocessor</code> simply resizes the images, being aware of the aspect ratio.</p>"},{"location":"examples/02_model_components.html","title":"02_model_components","text":"In\u00a0[1]: Copied! <pre>import torch\nimport pandas as pd\nimport numpy as np\n\nfrom torch import nn\n</pre> import torch import pandas as pd import numpy as np  from torch import nn In\u00a0[2]: Copied! <pre>df = pd.DataFrame({\"color\": [\"r\", \"b\", \"g\"], \"size\": [\"s\", \"n\", \"l\"]})\ndf.head()\n</pre> df = pd.DataFrame({\"color\": [\"r\", \"b\", \"g\"], \"size\": [\"s\", \"n\", \"l\"]}) df.head() Out[2]: color size 0 r s 1 b n 2 g l <p>one hot encoded, the first observation would be</p> In\u00a0[3]: Copied! <pre>obs_0_oh = (np.array([1.0, 0.0, 0.0, 1.0, 0.0, 0.0])).astype(\"float32\")\n</pre> obs_0_oh = (np.array([1.0, 0.0, 0.0, 1.0, 0.0, 0.0])).astype(\"float32\") <p>if we simply numerically encode (label encode or <code>le</code>) the values:</p> In\u00a0[4]: Copied! <pre>obs_0_le = (np.array([0, 3])).astype(\"int64\")\n</pre> obs_0_le = (np.array([0, 3])).astype(\"int64\") <p>Note that in the functioning implementation of the package we start from 1, saving 0 for padding, i.e. unseen values.</p> <p>Now, let's see if the two implementations are equivalent</p> In\u00a0[5]: Copied! <pre># we have 6 different values. Let's assume we are performing a regression, so pred_dim = 1\nlin = nn.Linear(6, 1)\n</pre> # we have 6 different values. Let's assume we are performing a regression, so pred_dim = 1 lin = nn.Linear(6, 1) In\u00a0[6]: Copied! <pre>emb = nn.Embedding(6, 1)\nemb.weight = nn.Parameter(lin.weight.reshape_as(emb.weight))\n</pre> emb = nn.Embedding(6, 1) emb.weight = nn.Parameter(lin.weight.reshape_as(emb.weight)) In\u00a0[7]: Copied! <pre>lin(torch.tensor(obs_0_oh))\n</pre> lin(torch.tensor(obs_0_oh)) Out[7]: <pre>tensor([-0.5181], grad_fn=&lt;ViewBackward0&gt;)</pre> In\u00a0[8]: Copied! <pre>emb(torch.tensor(obs_0_le)).sum() + lin.bias\n</pre> emb(torch.tensor(obs_0_le)).sum() + lin.bias Out[8]: <pre>tensor([-0.5181], grad_fn=&lt;AddBackward0&gt;)</pre> <p>And this is precisely how the linear model <code>Wide</code> is implemented</p> In\u00a0[9]: Copied! <pre>from pytorch_widedeep.models import Wide\n</pre> from pytorch_widedeep.models import Wide <pre>/Users/javierrodriguezzaurin/.pyenv/versions/3.10.13/envs/widedeep310/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n  from .autonotebook import tqdm as notebook_tqdm\n</pre> In\u00a0[10]: Copied! <pre># ?Wide\n</pre> # ?Wide In\u00a0[11]: Copied! <pre>wide = Wide(input_dim=10, pred_dim=1)\nwide\n</pre> wide = Wide(input_dim=10, pred_dim=1) wide Out[11]: <pre>Wide(\n  (wide_linear): Embedding(11, 1, padding_idx=0)\n)</pre> <p>Note that even though the input dim is 10, the Embedding layer has 11 weights. Again, this is because we save <code>0</code> for padding, which is used for unseen values during the encoding process.</p> <p>As I mentioned, <code>deeptabular</code> has enough complexity on its own and it will be described in a separated notebook. Let's then jump to <code>deeptext</code>.</p> In\u00a0[12]: Copied! <pre>from pytorch_widedeep.preprocessing import TabPreprocessor\nfrom pytorch_widedeep.models import TabMlp\n</pre> from pytorch_widedeep.preprocessing import TabPreprocessor from pytorch_widedeep.models import TabMlp In\u00a0[13]: Copied! <pre>data = {\n    \"cat1\": np.random.choice([\"A\", \"B\", \"C\"], size=20),\n    \"cat2\": np.random.choice([\"X\", \"Y\"], size=20),\n    \"cont1\": np.random.rand(20),\n    \"cont2\": np.random.rand(20),\n}\n\ndf = pd.DataFrame(data)\n</pre> data = {     \"cat1\": np.random.choice([\"A\", \"B\", \"C\"], size=20),     \"cat2\": np.random.choice([\"X\", \"Y\"], size=20),     \"cont1\": np.random.rand(20),     \"cont2\": np.random.rand(20), }  df = pd.DataFrame(data) In\u00a0[14]: Copied! <pre>df.head()\n</pre> df.head() Out[14]: cat1 cat2 cont1 cont2 0 A Y 0.789347 0.561789 1 C X 0.050822 0.061538 2 A Y 0.863784 0.241967 3 C X 0.917848 0.644658 4 C Y 0.042328 0.417303 In\u00a0[15]: Copied! <pre># see the docs for details on all params/options\ntab_preprocessor = TabPreprocessor(\n    cat_embed_cols=[\"cat1\", \"cat2\"],\n    continuous_cols=[\"cont1\", \"cont2\"],\n    embedding_rule=\"fastai\",\n)\n</pre> # see the docs for details on all params/options tab_preprocessor = TabPreprocessor(     cat_embed_cols=[\"cat1\", \"cat2\"],     continuous_cols=[\"cont1\", \"cont2\"],     embedding_rule=\"fastai\", ) In\u00a0[16]: Copied! <pre>X_tab = tab_preprocessor.fit_transform(df)\n</pre> X_tab = tab_preprocessor.fit_transform(df) <pre>/Users/javierrodriguezzaurin/Projects/pytorch-widedeep/pytorch_widedeep/preprocessing/tab_preprocessor.py:358: UserWarning: Continuous columns will not be normalised\n  warnings.warn(\"Continuous columns will not be normalised\")\n</pre> In\u00a0[17]: Copied! <pre># toy example just to build a model.\ntabmlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=tab_preprocessor.continuous_cols,\n    embed_continuous_method=\"standard\",\n    cont_embed_dim=4,\n    mlp_hidden_dims=[8, 4],\n    mlp_linear_first=True,\n)\ntabmlp\n</pre> # toy example just to build a model. tabmlp = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     continuous_cols=tab_preprocessor.continuous_cols,     embed_continuous_method=\"standard\",     cont_embed_dim=4,     mlp_hidden_dims=[8, 4],     mlp_linear_first=True, ) tabmlp Out[17]: <pre>TabMlp(\n  (cat_embed): DiffSizeCatEmbeddings(\n    (embed_layers): ModuleDict(\n      (emb_layer_cat1): Embedding(4, 3, padding_idx=0)\n      (emb_layer_cat2): Embedding(3, 2, padding_idx=0)\n    )\n    (embedding_dropout): Dropout(p=0.0, inplace=False)\n  )\n  (cont_norm): Identity()\n  (cont_embed): ContEmbeddings(\n    INFO: [ContLinear = weight(n_cont_cols, embed_dim) + bias(n_cont_cols, embed_dim)]\n    (linear): ContLinear(n_cont_cols=2, embed_dim=4, embed_dropout=0.0)\n    (dropout): Dropout(p=0.0, inplace=False)\n  )\n  (encoder): MLP(\n    (mlp): Sequential(\n      (dense_layer_0): Sequential(\n        (0): Linear(in_features=13, out_features=8, bias=True)\n        (1): ReLU(inplace=True)\n        (2): Dropout(p=0.1, inplace=False)\n      )\n      (dense_layer_1): Sequential(\n        (0): Linear(in_features=8, out_features=4, bias=True)\n        (1): ReLU(inplace=True)\n        (2): Dropout(p=0.1, inplace=False)\n      )\n    )\n  )\n)</pre> <p>Lets describe a bit the model: first we have what we call a <code>DiffSizeCatEmbeddings</code>, where categorical columns with different number of unique categories will be encoded with embeddings of different dimensions. Then the continuous columns will not be normalised (the normalised layer is just the identity) and they will be embedded via a \"standard\" method, using a so-called <code>ContLinear</code> layer. This layer displays some <code>INFO</code> that tells us what it is (<code>ContLinear = weight(n_cont_cols, embed_dim) + bias(n_cont_cols, embed_dim)]</code>). There are two other options available to embed the continuous cols based on the paper On Embeddings for Numerical Features in Tabular Deep Learning. These are <code>PieceWise</code> and <code>Periodic</code> and all available via the <code>embed_continuous_method</code> param, which can adopt values <code>\"standard\", \"piecewise\"</code> and <code>\"periodic\"</code>. The embedded categorical and continuous columns will be then concatenated ($3 + 2 + (4 * 2) = 13$ input dims) and passed to an MLP.</p> In\u00a0[18]: Copied! <pre>from pytorch_widedeep.models import BasicRNN\n</pre> from pytorch_widedeep.models import BasicRNN In\u00a0[19]: Copied! <pre>basic_rnn = BasicRNN(vocab_size=4, hidden_dim=4, n_layers=1, padding_idx=0, embed_dim=4)\n</pre> basic_rnn = BasicRNN(vocab_size=4, hidden_dim=4, n_layers=1, padding_idx=0, embed_dim=4) <pre>/Users/javierrodriguezzaurin/.pyenv/versions/3.10.13/envs/widedeep310/lib/python3.10/site-packages/torch/nn/modules/rnn.py:82: UserWarning: dropout option adds dropout after all but last recurrent layer, so non-zero dropout expects num_layers greater than 1, but got dropout=0.1 and num_layers=1\n  warnings.warn(\"dropout option adds dropout after all but last \"\n</pre> In\u00a0[20]: Copied! <pre>basic_rnn\n</pre> basic_rnn Out[20]: <pre>BasicRNN(\n  (word_embed): Embedding(4, 4, padding_idx=0)\n  (rnn): LSTM(4, 4, batch_first=True, dropout=0.1)\n  (rnn_mlp): Identity()\n)</pre> <p>You could, if you wanted, add a Fully Connected Head (FC-Head) on top of it</p> In\u00a0[21]: Copied! <pre>from pytorch_widedeep.models import Vision\n</pre> from pytorch_widedeep.models import Vision In\u00a0[22]: Copied! <pre>resnet = Vision(pretrained_model_setup=\"resnet18\", n_trainable=0)\n</pre> resnet = Vision(pretrained_model_setup=\"resnet18\", n_trainable=0) In\u00a0[23]: Copied! <pre>resnet\n</pre> resnet Out[23]: <pre>Vision(\n  (features): Sequential(\n    (0): Conv2d(3, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False)\n    (1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n    (2): ReLU(inplace=True)\n    (3): MaxPool2d(kernel_size=3, stride=2, padding=1, dilation=1, ceil_mode=False)\n    (4): Sequential(\n      (0): BasicBlock(\n        (conv1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n        (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (relu): ReLU(inplace=True)\n        (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n        (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n      )\n      (1): BasicBlock(\n        (conv1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n        (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (relu): ReLU(inplace=True)\n        (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n        (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n      )\n    )\n    (5): Sequential(\n      (0): BasicBlock(\n        (conv1): Conv2d(64, 128, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n        (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (relu): ReLU(inplace=True)\n        (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n        (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (downsample): Sequential(\n          (0): Conv2d(64, 128, kernel_size=(1, 1), stride=(2, 2), bias=False)\n          (1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        )\n      )\n      (1): BasicBlock(\n        (conv1): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n        (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (relu): ReLU(inplace=True)\n        (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n        (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n      )\n    )\n    (6): Sequential(\n      (0): BasicBlock(\n        (conv1): Conv2d(128, 256, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n        (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (relu): ReLU(inplace=True)\n        (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n        (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (downsample): Sequential(\n          (0): Conv2d(128, 256, kernel_size=(1, 1), stride=(2, 2), bias=False)\n          (1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        )\n      )\n      (1): BasicBlock(\n        (conv1): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n        (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (relu): ReLU(inplace=True)\n        (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n        (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n      )\n    )\n    (7): Sequential(\n      (0): BasicBlock(\n        (conv1): Conv2d(256, 512, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n        (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (relu): ReLU(inplace=True)\n        (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n        (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (downsample): Sequential(\n          (0): Conv2d(256, 512, kernel_size=(1, 1), stride=(2, 2), bias=False)\n          (1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        )\n      )\n      (1): BasicBlock(\n        (conv1): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n        (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (relu): ReLU(inplace=True)\n        (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n        (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n      )\n    )\n    (8): AdaptiveAvgPool2d(output_size=(1, 1))\n  )\n)</pre>"},{"location":"examples/02_model_components.html#model-components","title":"Model Components\u00b6","text":"<p>The main components of a <code>WideDeep</code> (i.e. Multimodal) model are tabular data, text and images, which are feed into the model via so called <code>wide</code>, <code>deeptabular</code>, <code>deeptext</code> and <code>deepimage</code> model components</p>"},{"location":"examples/02_model_components.html#1-wide","title":"1. <code>wide</code>\u00b6","text":"<p>The <code>wide</code> component is a Linear layer \"plugged\" into the output neuron(s). Here, the non-linearities are captured via crossed columns. Crossed columns are, quoting directly the paper: \"For binary features, a cross-product transformation (e.g., \u201cAND(gender=female, language=en)\u201d) is 1 if and only if the constituent features (\u201cgender=female\u201d and \u201clanguage=en\u201d) are all 1, and 0 otherwise\".</p> <p>The only particularity of our implementation is that we have implemented the linear layer via an Embedding layer plus a bias. While the implementations are equivalent, the latter is faster and far more memory efficient, since we do not need to one hot encode the categorical features.</p> <p>Let's assume we the following dataset:</p>"},{"location":"examples/02_model_components.html#2-deeptabular","title":"2. <code>deeptabular</code>\u00b6","text":"<p>The <code>deeptabular</code> model alone is what normally would be referred as Deep Learning for tabular data. As mentioned a number of times throughout the library, each component can be used independently. Therefore, if you wanted to use any of the models below alone, it is perfectly possible. There are just a couple of simple requirement that will be covered in a later notebook.</p> <p>By the time of writing, there are a number of models available in <code>pytorch-widedeep</code> to do DL for tabular data. These are:</p> <ol> <li><code>TabMlp</code></li> <li><code>ContextAttentionMLP</code></li> <li><code>SelfAttentionMLP</code></li> <li><code>TabResnet</code></li> <li><code>Tabnet</code></li> <li><code>TabTransformer</code></li> <li><code>FT-Tabransformer</code></li> <li><code>SAINT</code></li> <li><code>TabFastFormer</code></li> <li><code>TabPerceiver</code></li> </ol> <p>Let's have a look to one of them. For more information on each of these models, please, have a look to the documentation</p>"},{"location":"examples/02_model_components.html#3-deeptext","title":"3. <code>deeptext</code>\u00b6","text":"<p>At the time of writing, <code>pytorch-widedeep</code> offers three models that can be passed to <code>WideDeep</code> as the <code>deeptext</code> component. These are:</p> <ol> <li>BasicRNN</li> <li>AttentiveRNN</li> <li>StackedAttentiveRNN</li> </ol> <p>For details on each of these models, please, have a look to the documentation of the package.</p> <p>We will soon integrate with Hugginface, but let me insist. It is perfectly possible to use custom models for each component, please, have a look to the corresponding notebook. In general, simply, build them and pass them as the corresponding parameters. Note that the custom models MUST return a last layer of activations (i.e. not the final prediction) so that  these activations are collected by <code>WideDeep</code> and combined accordingly. In  addition, the models MUST also contain an attribute <code>output_dim</code> with the size of these last layers of activations.</p> <p>Let's have a look to the <code>BasicRNN</code> model</p>"},{"location":"examples/02_model_components.html#4-deepimage","title":"4. <code>deepimage</code>\u00b6","text":"<p>At the time of writing <code>pytorch-widedeep</code> is integrated with torchvision via the <code>Vision</code> class. This means that the it is possible to use a variant of the following architectures:</p> <ol> <li>resnet</li> <li>shufflenet</li> <li>resnext</li> <li>wide_resnet</li> <li>regnet</li> <li>densenet</li> <li>mobilenet</li> <li>mnasnet</li> <li>efficientnet</li> <li>squeezenet</li> </ol> <p>The user can choose which layers will be trainable. Alternatively, in none of these architectures is useful, one could use a simple, fully trained CNN (please see the package documentation) or pass a custom model.</p> <p>let's have a look</p>"},{"location":"examples/03_binary_classification_with_defaults.html","title":"03_binary_classification_with_defaults","text":"In\u00a0[1]: Copied! <pre>import numpy as np\nimport pandas as pd\nimport torch\n\nfrom pytorch_widedeep.preprocessing import WidePreprocessor, TabPreprocessor\nfrom pytorch_widedeep.training import Trainer\nfrom pytorch_widedeep.models import Wide, TabMlp, WideDeep\nfrom pytorch_widedeep.metrics import Accuracy, Precision\nfrom pytorch_widedeep.datasets import load_adult\n</pre> import numpy as np import pandas as pd import torch  from pytorch_widedeep.preprocessing import WidePreprocessor, TabPreprocessor from pytorch_widedeep.training import Trainer from pytorch_widedeep.models import Wide, TabMlp, WideDeep from pytorch_widedeep.metrics import Accuracy, Precision from pytorch_widedeep.datasets import load_adult <pre>/Users/javierrodriguezzaurin/.pyenv/versions/3.10.13/envs/widedeep310/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n  from .autonotebook import tqdm as notebook_tqdm\n</pre> In\u00a0[2]: Copied! <pre>df = load_adult(as_frame=True)\ndf.head()\n</pre> df = load_adult(as_frame=True) df.head() Out[2]: age workclass fnlwgt education educational-num marital-status occupation relationship race gender capital-gain capital-loss hours-per-week native-country income 0 25 Private 226802 11th 7 Never-married Machine-op-inspct Own-child Black Male 0 0 40 United-States &lt;=50K 1 38 Private 89814 HS-grad 9 Married-civ-spouse Farming-fishing Husband White Male 0 0 50 United-States &lt;=50K 2 28 Local-gov 336951 Assoc-acdm 12 Married-civ-spouse Protective-serv Husband White Male 0 0 40 United-States &gt;50K 3 44 Private 160323 Some-college 10 Married-civ-spouse Machine-op-inspct Husband Black Male 7688 0 40 United-States &gt;50K 4 18 ? 103497 Some-college 10 Never-married ? Own-child White Female 0 0 30 United-States &lt;=50K In\u00a0[3]: Copied! <pre># For convenience, we'll replace '-' with '_'\ndf.columns = [c.replace(\"-\", \"_\") for c in df.columns]\n# binary target\ndf[\"income_label\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int)\ndf.drop(\"income\", axis=1, inplace=True)\ndf.head()\n</pre> # For convenience, we'll replace '-' with '_' df.columns = [c.replace(\"-\", \"_\") for c in df.columns] # binary target df[\"income_label\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int) df.drop(\"income\", axis=1, inplace=True) df.head() Out[3]: age workclass fnlwgt education educational_num marital_status occupation relationship race gender capital_gain capital_loss hours_per_week native_country income_label 0 25 Private 226802 11th 7 Never-married Machine-op-inspct Own-child Black Male 0 0 40 United-States 0 1 38 Private 89814 HS-grad 9 Married-civ-spouse Farming-fishing Husband White Male 0 0 50 United-States 0 2 28 Local-gov 336951 Assoc-acdm 12 Married-civ-spouse Protective-serv Husband White Male 0 0 40 United-States 1 3 44 Private 160323 Some-college 10 Married-civ-spouse Machine-op-inspct Husband Black Male 7688 0 40 United-States 1 4 18 ? 103497 Some-college 10 Never-married ? Own-child White Female 0 0 30 United-States 0 In\u00a0[4]: Copied! <pre>df.drop([\"fnlwgt\", \"educational_num\"], axis=1, inplace=True)\n</pre> df.drop([\"fnlwgt\", \"educational_num\"], axis=1, inplace=True) In\u00a0[5]: Copied! <pre># Define wide, crossed and deep tabular columns\nwide_cols = [\n    \"workclass\",\n    \"education\",\n    \"marital_status\",\n    \"occupation\",\n    \"relationship\",\n    \"race\",\n    \"gender\",\n    \"native_country\",\n]\ncrossed_cols = [(\"education\", \"occupation\"), (\"native_country\", \"occupation\")]\n</pre> # Define wide, crossed and deep tabular columns wide_cols = [     \"workclass\",     \"education\",     \"marital_status\",     \"occupation\",     \"relationship\",     \"race\",     \"gender\",     \"native_country\", ] crossed_cols = [(\"education\", \"occupation\"), (\"native_country\", \"occupation\")] In\u00a0[6]: Copied! <pre>cat_embed_cols = [\n    \"workclass\",\n    \"education\",\n    \"marital_status\",\n    \"occupation\",\n    \"relationship\",\n    \"race\",\n    \"gender\",\n    \"capital_gain\",\n    \"capital_loss\",\n    \"native_country\",\n]\ncontinuous_cols = [\"age\", \"hours_per_week\"]\n</pre> cat_embed_cols = [     \"workclass\",     \"education\",     \"marital_status\",     \"occupation\",     \"relationship\",     \"race\",     \"gender\",     \"capital_gain\",     \"capital_loss\",     \"native_country\", ] continuous_cols = [\"age\", \"hours_per_week\"] In\u00a0[7]: Copied! <pre># TARGET\ntarget_col = \"income_label\"\ntarget = df[target_col].values\n</pre> # TARGET target_col = \"income_label\" target = df[target_col].values <p>let's see what the preprocessors do</p> In\u00a0[8]: Copied! <pre># wide\nwide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols)\nX_wide = wide_preprocessor.fit_transform(df)\n</pre> # wide wide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols) X_wide = wide_preprocessor.fit_transform(df) In\u00a0[9]: Copied! <pre># # wide_preprocessor has an attribute called encoding_dict with the encoding dictionary\n# wide_preprocessor.encoding_dict\n</pre> # # wide_preprocessor has an attribute called encoding_dict with the encoding dictionary # wide_preprocessor.encoding_dict In\u00a0[10]: Copied! <pre># deeptabular\ntab_preprocessor = TabPreprocessor(\n    embed_cols=cat_embed_cols,\n    continuous_cols=continuous_cols,\n    cols_to_scale=continuous_cols,\n)\nX_tab = tab_preprocessor.fit_transform(df)\n</pre> # deeptabular tab_preprocessor = TabPreprocessor(     embed_cols=cat_embed_cols,     continuous_cols=continuous_cols,     cols_to_scale=continuous_cols, ) X_tab = tab_preprocessor.fit_transform(df) In\u00a0[11]: Copied! <pre># check the docs to understand the useful attributes that the tab_preprocessor has. For example,\n# as well as an encoding dictionary, tab_preprocessor has an attribute called cat_embed_input\n# that specifies the categortical columns that will be represented as embeddings, the number\n# of different categories per feature, and the dimension of the embeddings as defined by some\n# of the internal rules of thumb that the preprocessor has (have a look to the docs)\ntab_preprocessor.cat_embed_input\n</pre> # check the docs to understand the useful attributes that the tab_preprocessor has. For example, # as well as an encoding dictionary, tab_preprocessor has an attribute called cat_embed_input # that specifies the categortical columns that will be represented as embeddings, the number # of different categories per feature, and the dimension of the embeddings as defined by some # of the internal rules of thumb that the preprocessor has (have a look to the docs) tab_preprocessor.cat_embed_input Out[11]: <pre>[('workclass', 9, 5),\n ('education', 16, 8),\n ('marital_status', 7, 5),\n ('occupation', 15, 7),\n ('relationship', 6, 4),\n ('race', 5, 4),\n ('gender', 2, 2),\n ('capital_gain', 123, 24),\n ('capital_loss', 99, 21),\n ('native_country', 42, 13)]</pre> In\u00a0[12]: Copied! <pre>print(X_wide)\nprint(X_wide.shape)\n</pre> print(X_wide) print(X_wide.shape) <pre>[[  1  10  26 ...  61 103 328]\n [  1  11  27 ...  61 104 329]\n [  2  12  27 ...  61 105 330]\n ...\n [  1  11  28 ...  61 115 335]\n [  1  11  26 ...  61 115 335]\n [  7  11  27 ...  61 127 336]]\n(48842, 10)\n</pre> In\u00a0[13]: Copied! <pre>print(X_tab)\nprint(X_tab.shape)\n</pre> print(X_tab) print(X_tab.shape) <pre>[[ 1.          1.          1.         ...  1.         -0.99512893\n  -0.03408696]\n [ 1.          2.          2.         ...  1.         -0.04694151\n   0.77292975]\n [ 2.          3.          2.         ...  1.         -0.77631645\n  -0.03408696]\n ...\n [ 1.          2.          3.         ...  1.          1.41180837\n  -0.03408696]\n [ 1.          2.          1.         ...  1.         -1.21394141\n  -1.64812038]\n [ 7.          2.          2.         ...  1.          0.97418341\n  -0.03408696]]\n(48842, 12)\n</pre> In\u00a0[14]: Copied! <pre>wide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1)\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    cat_embed_dropout=0.1,\n    continuous_cols=continuous_cols,\n    mlp_hidden_dims=[400, 200],\n    mlp_dropout=0.5,\n    mlp_activation=\"leaky_relu\",\n)\n</pre> wide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1) tab_mlp = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     cat_embed_dropout=0.1,     continuous_cols=continuous_cols,     mlp_hidden_dims=[400, 200],     mlp_dropout=0.5,     mlp_activation=\"leaky_relu\", ) <p>Let's first find out how a linear model performs</p> In\u00a0[15]: Copied! <pre>wide\n</pre> wide Out[15]: <pre>Wide(\n  (wide_linear): Embedding(809, 1, padding_idx=0)\n)</pre> <p>Before being passed to the Trainer, the models need to be \"constructed\" with the <code>WideDeep</code> constructor class. For the particular case of the wide/linear model, not much really happens</p> In\u00a0[16]: Copied! <pre>lin_model = WideDeep(wide=wide)\n</pre> lin_model = WideDeep(wide=wide) In\u00a0[17]: Copied! <pre>lin_model\n</pre> lin_model Out[17]: <pre>WideDeep(\n  (wide): Wide(\n    (wide_linear): Embedding(809, 1, padding_idx=0)\n  )\n)</pre> In\u00a0[18]: Copied! <pre>lin_trainer = Trainer(\n    model=lin_model,\n    objective=\"binary\",\n    optimizers=torch.optim.AdamW(lin_model.parameters(), lr=0.01),\n    metrics=[Accuracy, Precision],\n)\n</pre> lin_trainer = Trainer(     model=lin_model,     objective=\"binary\",     optimizers=torch.optim.AdamW(lin_model.parameters(), lr=0.01),     metrics=[Accuracy, Precision], ) In\u00a0[19]: Copied! <pre>lin_trainer.fit(X_wide=X_wide, target=target, n_epochs=4, batch_size=128, val_split=0.2)\n</pre> lin_trainer.fit(X_wide=X_wide, target=target, n_epochs=4, batch_size=128, val_split=0.2) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 306/306 [00:02&lt;00:00, 109.04it/s, loss=0.426, metrics={'acc': 0.7983, 'prec': 0.6152}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 77/77 [00:00&lt;00:00, 102.46it/s, loss=0.366, metrics={'acc': 0.832, 'prec': 0.6916}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 306/306 [00:02&lt;00:00, 130.27it/s, loss=0.364, metrics={'acc': 0.8305, 'prec': 0.6933}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 77/77 [00:00&lt;00:00, 150.46it/s, loss=0.361, metrics={'acc': 0.8357, 'prec': 0.6982}]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 306/306 [00:02&lt;00:00, 133.19it/s, loss=0.359, metrics={'acc': 0.8329, 'prec': 0.6994}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 77/77 [00:00&lt;00:00, 145.75it/s, loss=0.361, metrics={'acc': 0.836, 'prec': 0.7009}]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 306/306 [00:02&lt;00:00, 130.91it/s, loss=0.358, metrics={'acc': 0.8333, 'prec': 0.7005}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 77/77 [00:00&lt;00:00, 155.08it/s, loss=0.361, metrics={'acc': 0.8364, 'prec': 0.702}]\n</pre> <p>Bear in mind that <code>wide</code> is a linear model where the non-linearities are captured via the crossed columns. For the crossed-columns to be effective one needs proper business knowledge. There is no magic formula to produce them</p> <p>Let's have a look to the tabular model by itself</p> In\u00a0[20]: Copied! <pre>tab_model = WideDeep(deeptabular=tab_mlp)\n</pre> tab_model = WideDeep(deeptabular=tab_mlp) In\u00a0[21]: Copied! <pre>tab_model\n</pre> tab_model Out[21]: <pre>WideDeep(\n  (deeptabular): Sequential(\n    (0): TabMlp(\n      (cat_embed): DiffSizeCatEmbeddings(\n        (embed_layers): ModuleDict(\n          (emb_layer_workclass): Embedding(10, 5, padding_idx=0)\n          (emb_layer_education): Embedding(17, 8, padding_idx=0)\n          (emb_layer_marital_status): Embedding(8, 5, padding_idx=0)\n          (emb_layer_occupation): Embedding(16, 7, padding_idx=0)\n          (emb_layer_relationship): Embedding(7, 4, padding_idx=0)\n          (emb_layer_race): Embedding(6, 4, padding_idx=0)\n          (emb_layer_gender): Embedding(3, 2, padding_idx=0)\n          (emb_layer_capital_gain): Embedding(124, 24, padding_idx=0)\n          (emb_layer_capital_loss): Embedding(100, 21, padding_idx=0)\n          (emb_layer_native_country): Embedding(43, 13, padding_idx=0)\n        )\n        (embedding_dropout): Dropout(p=0.1, inplace=False)\n      )\n      (cont_norm): Identity()\n      (encoder): MLP(\n        (mlp): Sequential(\n          (dense_layer_0): Sequential(\n            (0): Linear(in_features=95, out_features=400, bias=True)\n            (1): LeakyReLU(negative_slope=0.01, inplace=True)\n            (2): Dropout(p=0.5, inplace=False)\n          )\n          (dense_layer_1): Sequential(\n            (0): Linear(in_features=400, out_features=200, bias=True)\n            (1): LeakyReLU(negative_slope=0.01, inplace=True)\n            (2): Dropout(p=0.5, inplace=False)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=200, out_features=1, bias=True)\n  )\n)</pre> <p>You can see how the <code>WideDeep</code> class has added a final prediction layer that collects the activations from the last layer of the model and plugs them into the output neuron. If this was a multiclass classification problem, the prediction dimension (i.e. the size of that final layer) needs to be specified via the <code>pred_dim</code> when instantiating the <code>WideDeep</code> class, as we will see later</p> In\u00a0[22]: Copied! <pre>tab_trainer = Trainer(\n    model=tab_model,\n    objective=\"binary\",\n    optimizers=torch.optim.AdamW(tab_model.parameters(), lr=0.001),\n    metrics=[Accuracy, Precision],\n)\n</pre> tab_trainer = Trainer(     model=tab_model,     objective=\"binary\",     optimizers=torch.optim.AdamW(tab_model.parameters(), lr=0.001),     metrics=[Accuracy, Precision], ) In\u00a0[23]: Copied! <pre>tab_trainer.fit(X_tab=X_tab, target=target, n_epochs=4, batch_size=128, val_split=0.2)\n</pre> tab_trainer.fit(X_tab=X_tab, target=target, n_epochs=4, batch_size=128, val_split=0.2) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 306/306 [00:03&lt;00:00, 97.00it/s, loss=0.37, metrics={'acc': 0.8267, 'prec': 0.7037}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 77/77 [00:00&lt;00:00, 134.91it/s, loss=0.313, metrics={'acc': 0.8588, 'prec': 0.7577}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 306/306 [00:03&lt;00:00, 86.86it/s, loss=0.319, metrics={'acc': 0.8514, 'prec': 0.761}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 77/77 [00:01&lt;00:00, 73.13it/s, loss=0.296, metrics={'acc': 0.8675, 'prec': 0.7685}]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 306/306 [00:03&lt;00:00, 79.07it/s, loss=0.305, metrics={'acc': 0.8574, 'prec': 0.7646}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 77/77 [00:00&lt;00:00, 130.11it/s, loss=0.289, metrics={'acc': 0.8696, 'prec': 0.7765}]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 306/306 [00:03&lt;00:00, 87.39it/s, loss=0.296, metrics={'acc': 0.8622, 'prec': 0.7769}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 77/77 [00:00&lt;00:00, 90.63it/s, loss=0.285, metrics={'acc': 0.8697, 'prec': 0.7741}]\n</pre> <p>The best result I ever obtained with <code>LightGBM</code> on this dataset is 0.8782...so we are pretty close.</p> <p>Let's combine the <code>wide</code> and <code>tab_mlp</code> components see if it helps</p> In\u00a0[24]: Copied! <pre>wide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1)\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    cat_embed_dropout=0.1,\n    continuous_cols=continuous_cols,\n    mlp_hidden_dims=[400, 200],\n    mlp_dropout=0.5,\n    mlp_activation=\"leaky_relu\",\n)\nwd_model = WideDeep(wide=wide, deeptabular=tab_mlp)\n</pre> wide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1) tab_mlp = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     cat_embed_dropout=0.1,     continuous_cols=continuous_cols,     mlp_hidden_dims=[400, 200],     mlp_dropout=0.5,     mlp_activation=\"leaky_relu\", ) wd_model = WideDeep(wide=wide, deeptabular=tab_mlp) In\u00a0[25]: Copied! <pre>wd_trainer = Trainer(\n    model=wd_model,\n    objective=\"binary\",\n    optimizers=torch.optim.AdamW(wd_model.parameters(), lr=0.001),\n    metrics=[Accuracy, Precision],\n)\n</pre> wd_trainer = Trainer(     model=wd_model,     objective=\"binary\",     optimizers=torch.optim.AdamW(wd_model.parameters(), lr=0.001),     metrics=[Accuracy, Precision], ) In\u00a0[26]: Copied! <pre>wd_trainer.fit(\n    X_wide=X_wide, X_tab=X_tab, target=target, n_epochs=4, batch_size=128, val_split=0.2\n)\n</pre> wd_trainer.fit(     X_wide=X_wide, X_tab=X_tab, target=target, n_epochs=4, batch_size=128, val_split=0.2 ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 306/306 [00:03&lt;00:00, 77.48it/s, loss=0.418, metrics={'acc': 0.8047, 'prec': 0.6154}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 77/77 [00:00&lt;00:00, 110.51it/s, loss=0.321, metrics={'acc': 0.8521, 'prec': 0.7059}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 306/306 [00:03&lt;00:00, 82.70it/s, loss=0.333, metrics={'acc': 0.8428, 'prec': 0.7141}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 77/77 [00:00&lt;00:00, 112.52it/s, loss=0.299, metrics={'acc': 0.866, 'prec': 0.7447}]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 306/306 [00:04&lt;00:00, 74.34it/s, loss=0.312, metrics={'acc': 0.8533, 'prec': 0.7404}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 77/77 [00:00&lt;00:00, 89.86it/s, loss=0.29, metrics={'acc': 0.8683, 'prec': 0.7496}]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 306/306 [00:04&lt;00:00, 65.32it/s, loss=0.301, metrics={'acc': 0.8591, 'prec': 0.7542}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 77/77 [00:00&lt;00:00, 86.81it/s, loss=0.286, metrics={'acc': 0.8712, 'prec': 0.7552}]\n</pre> <p>For this particular case, the combination of both did not lead to better results that using just the tab_mlp model, when using only 4 epochs.</p> <p>Note that we have use a <code>TabMlp</code> model, but we could use any other model in the library using the same syntax</p> In\u00a0[27]: Copied! <pre>from pytorch_widedeep.models import TabTransformer\n</pre> from pytorch_widedeep.models import TabTransformer <p>The parameters for the <code>TabTransformer</code> are this</p> <pre><code>column_idx: Dict[str, int],\ncat_embed_input: Optional[List[Tuple[str, int]]] = None,\ncat_embed_dropout: Optional[float] = None,\nuse_cat_bias: Optional[bool] = None,\ncat_embed_activation: Optional[str] = None,\nshared_embed: Optional[bool] = None,\nadd_shared_embed: Optional[bool] = None,\nfrac_shared_embed: Optional[float] = None,\ncontinuous_cols: Optional[List[str]] = None,\ncont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\nembed_continuous: Optional[bool] = None,\nembed_continuous_method: Optional[Literal[\"standard\", \"piecewise\", \"periodic\"]] = None,\ncont_embed_dropout: Optional[float] = None,\ncont_embed_activation: Optional[str] = None,\nquantization_setup: Optional[Dict[str, List[float]]] = None,\nn_frequencies: Optional[int] = None,\nsigma: Optional[float] = None,\nshare_last_layer: Optional[bool] = None,\nfull_embed_dropout: Optional[bool] = None,\ninput_dim: int = 32,\nn_heads: int = 8,\nuse_qkv_bias: bool = False,\nn_blocks: int = 4,\nattn_dropout: float = 0.2,\nff_dropout: float = 0.1,\nff_factor: int = 4,\ntransformer_activation: str = \"gelu\",\nuse_linear_attention: bool = False,\nuse_flash_attention: bool = False,\nmlp_hidden_dims: Optional[List[int]] = None,\nmlp_activation: str = \"relu\",\nmlp_dropout: float = 0.1,\nmlp_batchnorm: bool = False,\nmlp_batchnorm_last: bool = False,\nmlp_linear_first: bool = True,\n</code></pre> <p>Please, see the documentation for details on each one of them, for now let's see how one could use a <code>TabTransformer</code> model in a few lines of code</p> In\u00a0[28]: Copied! <pre>tab_transformer = TabTransformer(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    cat_embed_dropout=0.1,\n    continuous_cols=continuous_cols,\n    embed_continuous_method=\"standard\",\n    cont_norm_layer=\"layernorm\",\n    cont_embed_dropout=0.2,\n    cont_embed_activation=\"leaky_relu\",\n    n_heads=4,\n    ff_dropout=0.2,\n    mlp_dropout=0.5,\n    mlp_activation=\"leaky_relu\",\n    mlp_linear_first=\"True\",\n)\n</pre> tab_transformer = TabTransformer(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     cat_embed_dropout=0.1,     continuous_cols=continuous_cols,     embed_continuous_method=\"standard\",     cont_norm_layer=\"layernorm\",     cont_embed_dropout=0.2,     cont_embed_activation=\"leaky_relu\",     n_heads=4,     ff_dropout=0.2,     mlp_dropout=0.5,     mlp_activation=\"leaky_relu\",     mlp_linear_first=\"True\", ) In\u00a0[29]: Copied! <pre>tab_model = WideDeep(deeptabular=tab_transformer)\n</pre> tab_model = WideDeep(deeptabular=tab_transformer) In\u00a0[30]: Copied! <pre>tab_model\n</pre> tab_model Out[30]: <pre>WideDeep(\n  (deeptabular): Sequential(\n    (0): TabTransformer(\n      (cat_embed): SameSizeCatEmbeddings(\n        (embed): Embedding(325, 32, padding_idx=0)\n        (dropout): Dropout(p=0.1, inplace=False)\n      )\n      (cont_norm): LayerNorm((2,), eps=1e-05, elementwise_affine=True)\n      (cont_embed): ContEmbeddings(\n        INFO: [ContLinear = weight(n_cont_cols, embed_dim) + bias(n_cont_cols, embed_dim)]\n        (linear): ContLinear(n_cont_cols=2, embed_dim=32, embed_dropout=0.2)\n        (activation_fn): LeakyReLU(negative_slope=0.01, inplace=True)\n        (dropout): Dropout(p=0.2, inplace=False)\n      )\n      (encoder): Sequential(\n        (transformer_block0): TransformerEncoder(\n          (attn): MultiHeadedAttention(\n            (dropout): Dropout(p=0.2, inplace=False)\n            (q_proj): Linear(in_features=32, out_features=32, bias=False)\n            (kv_proj): Linear(in_features=32, out_features=64, bias=False)\n            (out_proj): Linear(in_features=32, out_features=32, bias=False)\n          )\n          (ff): FeedForward(\n            (w_1): Linear(in_features=32, out_features=128, bias=True)\n            (w_2): Linear(in_features=128, out_features=32, bias=True)\n            (dropout): Dropout(p=0.2, inplace=False)\n            (activation): GELU(approximate='none')\n          )\n          (attn_addnorm): AddNorm(\n            (dropout): Dropout(p=0.2, inplace=False)\n            (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n          )\n          (ff_addnorm): AddNorm(\n            (dropout): Dropout(p=0.2, inplace=False)\n            (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n          )\n        )\n        (transformer_block1): TransformerEncoder(\n          (attn): MultiHeadedAttention(\n            (dropout): Dropout(p=0.2, inplace=False)\n            (q_proj): Linear(in_features=32, out_features=32, bias=False)\n            (kv_proj): Linear(in_features=32, out_features=64, bias=False)\n            (out_proj): Linear(in_features=32, out_features=32, bias=False)\n          )\n          (ff): FeedForward(\n            (w_1): Linear(in_features=32, out_features=128, bias=True)\n            (w_2): Linear(in_features=128, out_features=32, bias=True)\n            (dropout): Dropout(p=0.2, inplace=False)\n            (activation): GELU(approximate='none')\n          )\n          (attn_addnorm): AddNorm(\n            (dropout): Dropout(p=0.2, inplace=False)\n            (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n          )\n          (ff_addnorm): AddNorm(\n            (dropout): Dropout(p=0.2, inplace=False)\n            (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n          )\n        )\n        (transformer_block2): TransformerEncoder(\n          (attn): MultiHeadedAttention(\n            (dropout): Dropout(p=0.2, inplace=False)\n            (q_proj): Linear(in_features=32, out_features=32, bias=False)\n            (kv_proj): Linear(in_features=32, out_features=64, bias=False)\n            (out_proj): Linear(in_features=32, out_features=32, bias=False)\n          )\n          (ff): FeedForward(\n            (w_1): Linear(in_features=32, out_features=128, bias=True)\n            (w_2): Linear(in_features=128, out_features=32, bias=True)\n            (dropout): Dropout(p=0.2, inplace=False)\n            (activation): GELU(approximate='none')\n          )\n          (attn_addnorm): AddNorm(\n            (dropout): Dropout(p=0.2, inplace=False)\n            (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n          )\n          (ff_addnorm): AddNorm(\n            (dropout): Dropout(p=0.2, inplace=False)\n            (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n          )\n        )\n        (transformer_block3): TransformerEncoder(\n          (attn): MultiHeadedAttention(\n            (dropout): Dropout(p=0.2, inplace=False)\n            (q_proj): Linear(in_features=32, out_features=32, bias=False)\n            (kv_proj): Linear(in_features=32, out_features=64, bias=False)\n            (out_proj): Linear(in_features=32, out_features=32, bias=False)\n          )\n          (ff): FeedForward(\n            (w_1): Linear(in_features=32, out_features=128, bias=True)\n            (w_2): Linear(in_features=128, out_features=32, bias=True)\n            (dropout): Dropout(p=0.2, inplace=False)\n            (activation): GELU(approximate='none')\n          )\n          (attn_addnorm): AddNorm(\n            (dropout): Dropout(p=0.2, inplace=False)\n            (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n          )\n          (ff_addnorm): AddNorm(\n            (dropout): Dropout(p=0.2, inplace=False)\n            (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=384, out_features=1, bias=True)\n  )\n)</pre> In\u00a0[31]: Copied! <pre>tab_trainer = Trainer(\n    model=tab_model,\n    objective=\"binary\",\n    optimizers=torch.optim.AdamW(tab_model.parameters(), lr=0.001),\n    metrics=[Accuracy, Precision],\n)\n</pre> tab_trainer = Trainer(     model=tab_model,     objective=\"binary\",     optimizers=torch.optim.AdamW(tab_model.parameters(), lr=0.001),     metrics=[Accuracy, Precision], ) In\u00a0[32]: Copied! <pre>tab_trainer.fit(X_tab=X_tab, target=target, n_epochs=1, batch_size=128, val_split=0.2)\n</pre> tab_trainer.fit(X_tab=X_tab, target=target, n_epochs=1, batch_size=128, val_split=0.2) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 306/306 [00:11&lt;00:00, 27.57it/s, loss=0.359, metrics={'acc': 0.8334, 'prec': 0.7082}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 77/77 [00:01&lt;00:00, 57.89it/s, loss=0.33, metrics={'acc': 0.8536, 'prec': 0.7152}]\n</pre>"},{"location":"examples/03_binary_classification_with_defaults.html#simple-binary-classification-with-defaults","title":"Simple Binary Classification with defaults\u00b6","text":"<p>In this notebook we will train a Wide and Deep model and simply a \"Deep\" model using the well known adult dataset</p>"},{"location":"examples/03_binary_classification_with_defaults.html#preparing-the-data","title":"Preparing the data\u00b6","text":""},{"location":"examples/03_binary_classification_with_defaults.html#defining-the-model","title":"Defining the model\u00b6","text":""},{"location":"examples/04_regression_with_images_and_text.html","title":"04_regression_with_images_and_text","text":"In\u00a0[1]: Copied! <pre>import numpy as np\nimport pandas as pd\nimport os\nimport torch\nfrom torchvision.transforms import ToTensor, Normalize\n\nfrom pytorch_widedeep import Trainer\nfrom pytorch_widedeep.preprocessing import (\n    WidePreprocessor,\n    TabPreprocessor,\n    TextPreprocessor,\n    ImagePreprocessor,\n)\nfrom pytorch_widedeep.models import (\n    Wide,\n    TabMlp,\n    Vision,\n    BasicRNN,\n    WideDeep,\n)\nfrom pytorch_widedeep.losses import RMSELoss\nfrom pytorch_widedeep.initializers import *\nfrom pytorch_widedeep.callbacks import *\n</pre> import numpy as np import pandas as pd import os import torch from torchvision.transforms import ToTensor, Normalize  from pytorch_widedeep import Trainer from pytorch_widedeep.preprocessing import (     WidePreprocessor,     TabPreprocessor,     TextPreprocessor,     ImagePreprocessor, ) from pytorch_widedeep.models import (     Wide,     TabMlp,     Vision,     BasicRNN,     WideDeep, ) from pytorch_widedeep.losses import RMSELoss from pytorch_widedeep.initializers import * from pytorch_widedeep.callbacks import * <pre>/Users/javierrodriguezzaurin/.pyenv/versions/3.10.13/envs/widedeep310/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n  from .autonotebook import tqdm as notebook_tqdm\n</pre> In\u00a0[2]: Copied! <pre>df = pd.read_csv(\"../tmp_data/airbnb/airbnb_sample.csv\")\ndf.head()\n</pre> df = pd.read_csv(\"../tmp_data/airbnb/airbnb_sample.csv\") df.head() Out[2]: id host_id description host_listings_count host_identity_verified neighbourhood_cleansed latitude longitude is_location_exact property_type ... amenity_wide_entrance amenity_wide_entrance_for_guests amenity_wide_entryway amenity_wide_hallways amenity_wifi amenity_window_guards amenity_wine_cooler security_deposit extra_people yield 0 13913.jpg 54730 My bright double bedroom with a large window h... 4.0 f Islington 51.56802 -0.11121 t apartment ... 1 0 0 0 1 0 0 100.0 15.0 12.00 1 15400.jpg 60302 Lots of windows and light.  St Luke's Gardens ... 1.0 t Kensington and Chelsea 51.48796 -0.16898 t apartment ... 0 0 0 0 1 0 0 150.0 0.0 109.50 2 17402.jpg 67564 Open from June 2018 after a 3-year break, we a... 19.0 t Westminster 51.52098 -0.14002 t apartment ... 0 0 0 0 1 0 0 350.0 10.0 149.65 3 24328.jpg 41759 Artist house, bright high ceiling rooms, priva... 2.0 t Wandsworth 51.47298 -0.16376 t other ... 0 0 0 0 1 0 0 250.0 0.0 215.60 4 25023.jpg 102813 Large, all comforts, 2-bed flat; first floor; ... 1.0 f Wandsworth 51.44687 -0.21874 t apartment ... 0 0 0 0 1 0 0 250.0 11.0 79.35 <p>5 rows \u00d7 223 columns</p> In\u00a0[3]: Copied! <pre># There are a number of columns that are already binary. Therefore, no need to one hot encode them\ncrossed_cols = [(\"property_type\", \"room_type\")]\nalready_dummies = [c for c in df.columns if \"amenity\" in c] + [\"has_house_rules\"]\nwide_cols = [\n    \"is_location_exact\",\n    \"property_type\",\n    \"room_type\",\n    \"host_gender\",\n    \"instant_bookable\",\n] + already_dummies\n\ncat_embed_cols = [(c, 16) for c in df.columns if \"catg\" in c] + [\n    (\"neighbourhood_cleansed\", 64),\n    (\"cancellation_policy\", 16),\n]\ncontinuous_cols = [\"latitude\", \"longitude\", \"security_deposit\", \"extra_people\"]\n\n# text and image colnames\ntext_col = \"description\"\nimg_col = \"id\"\n\n# path to pretrained word embeddings and the images\nword_vectors_path = \"../tmp_data/glove.6B/glove.6B.100d.txt\"\nimg_path = \"../tmp_data/airbnb/property_picture\"\n\n# target\ntarget_col = \"yield\"\n</pre> # There are a number of columns that are already binary. Therefore, no need to one hot encode them crossed_cols = [(\"property_type\", \"room_type\")] already_dummies = [c for c in df.columns if \"amenity\" in c] + [\"has_house_rules\"] wide_cols = [     \"is_location_exact\",     \"property_type\",     \"room_type\",     \"host_gender\",     \"instant_bookable\", ] + already_dummies  cat_embed_cols = [(c, 16) for c in df.columns if \"catg\" in c] + [     (\"neighbourhood_cleansed\", 64),     (\"cancellation_policy\", 16), ] continuous_cols = [\"latitude\", \"longitude\", \"security_deposit\", \"extra_people\"]  # text and image colnames text_col = \"description\" img_col = \"id\"  # path to pretrained word embeddings and the images word_vectors_path = \"../tmp_data/glove.6B/glove.6B.100d.txt\" img_path = \"../tmp_data/airbnb/property_picture\"  # target target_col = \"yield\" In\u00a0[4]: Copied! <pre>target = df[target_col].values\n</pre> target = df[target_col].values In\u00a0[5]: Copied! <pre>wide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols)\nX_wide = wide_preprocessor.fit_transform(df)\n</pre> wide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols) X_wide = wide_preprocessor.fit_transform(df) In\u00a0[6]: Copied! <pre>tab_preprocessor = TabPreprocessor(\n    cat_embed_cols=cat_embed_cols,\n    continuous_cols=continuous_cols,\n)\nX_tab = tab_preprocessor.fit_transform(df)\n</pre> tab_preprocessor = TabPreprocessor(     cat_embed_cols=cat_embed_cols,     continuous_cols=continuous_cols, ) X_tab = tab_preprocessor.fit_transform(df) <pre>/Users/javierrodriguezzaurin/Projects/pytorch-widedeep/pytorch_widedeep/preprocessing/tab_preprocessor.py:358: UserWarning: Continuous columns will not be normalised\n  warnings.warn(\"Continuous columns will not be normalised\")\n</pre> In\u00a0[7]: Copied! <pre>text_preprocessor = TextPreprocessor(\n    word_vectors_path=word_vectors_path, text_col=text_col\n)\nX_text = text_preprocessor.fit_transform(df)\n</pre> text_preprocessor = TextPreprocessor(     word_vectors_path=word_vectors_path, text_col=text_col ) X_text = text_preprocessor.fit_transform(df) <pre>The vocabulary contains 2192 tokens\nIndexing word vectors...\nLoaded 400000 word vectors\nPreparing embeddings matrix...\n2175 words in the vocabulary had ../tmp_data/glove.6B/glove.6B.100d.txt vectors and appear more than 5 times\n</pre> In\u00a0[8]: Copied! <pre>image_processor = ImagePreprocessor(img_col=img_col, img_path=img_path)\nX_images = image_processor.fit_transform(df)\n</pre> image_processor = ImagePreprocessor(img_col=img_col, img_path=img_path) X_images = image_processor.fit_transform(df) <pre>Reading Images from ../tmp_data/airbnb/property_picture\nResizing\n</pre> <pre>100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 1001/1001 [00:01&lt;00:00, 638.00it/s]\n</pre> <pre>Computing normalisation metrics\n</pre> In\u00a0[9]: Copied! <pre># Linear model\nwide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1)\n\n# DeepDense: 2 Dense layers\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=continuous_cols,\n    mlp_hidden_dims=[128, 64],\n    mlp_dropout=0.1,\n)\n\n# DeepText: a stack of 2 LSTMs\nbasic_rnn = BasicRNN(\n    vocab_size=len(text_preprocessor.vocab.itos),\n    embed_matrix=text_preprocessor.embedding_matrix,\n    n_layers=2,\n    hidden_dim=64,\n    rnn_dropout=0.5,\n)\n\n# Pretrained Resnet 18\nresnet = Vision(pretrained_model_setup=\"resnet18\", n_trainable=4)\n</pre> # Linear model wide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1)  # DeepDense: 2 Dense layers tab_mlp = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     continuous_cols=continuous_cols,     mlp_hidden_dims=[128, 64],     mlp_dropout=0.1, )  # DeepText: a stack of 2 LSTMs basic_rnn = BasicRNN(     vocab_size=len(text_preprocessor.vocab.itos),     embed_matrix=text_preprocessor.embedding_matrix,     n_layers=2,     hidden_dim=64,     rnn_dropout=0.5, )  # Pretrained Resnet 18 resnet = Vision(pretrained_model_setup=\"resnet18\", n_trainable=4) <p>Combine them all with the \"collector\" class <code>WideDeep</code></p> In\u00a0[10]: Copied! <pre>model = WideDeep(\n    wide=wide,\n    deeptabular=tab_mlp,\n    deeptext=basic_rnn,\n    deepimage=resnet,\n    head_hidden_dims=[256, 128],\n)\n</pre> model = WideDeep(     wide=wide,     deeptabular=tab_mlp,     deeptext=basic_rnn,     deepimage=resnet,     head_hidden_dims=[256, 128], ) In\u00a0[11]: Copied! <pre>trainer = Trainer(model, objective=\"rmse\")\n</pre> trainer = Trainer(model, objective=\"rmse\") In\u00a0[12]: Copied! <pre>trainer.fit(\n    X_wide=X_wide,\n    X_tab=X_tab,\n    X_text=X_text,\n    X_img=X_images,\n    target=target,\n    n_epochs=1,\n    batch_size=32,\n    val_split=0.2,\n)\n</pre> trainer.fit(     X_wide=X_wide,     X_tab=X_tab,     X_text=X_text,     X_img=X_images,     target=target,     n_epochs=1,     batch_size=32,     val_split=0.2, ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 25/25 [00:19&lt;00:00,  1.28it/s, loss=115]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 7/7 [00:04&lt;00:00,  1.62it/s, loss=94.1]\n</pre> <p>Both, the Text and Image components allow FC-heads on their own (have a look to the documentation).</p> <p>Now let's go \"kaggle crazy\". Let's use different optimizers, initializers and schedulers for different components. Moreover, let's use a different learning rate for different parameter groups, for the <code>deeptabular</code> component</p> In\u00a0[13]: Copied! <pre>deep_params = []\nfor childname, child in model.named_children():\n    if childname == \"deeptabular\":\n        for n, p in child.named_parameters():\n            if \"embed_layer\" in n:\n                deep_params.append({\"params\": p, \"lr\": 1e-4})\n            else:\n                deep_params.append({\"params\": p, \"lr\": 1e-3})\n</pre> deep_params = [] for childname, child in model.named_children():     if childname == \"deeptabular\":         for n, p in child.named_parameters():             if \"embed_layer\" in n:                 deep_params.append({\"params\": p, \"lr\": 1e-4})             else:                 deep_params.append({\"params\": p, \"lr\": 1e-3}) In\u00a0[14]: Copied! <pre>wide_opt = torch.optim.Adam(model.wide.parameters(), lr=0.03)\ndeep_opt = torch.optim.Adam(deep_params)\ntext_opt = torch.optim.AdamW(model.deeptext.parameters())\nimg_opt = torch.optim.AdamW(model.deepimage.parameters())\nhead_opt = torch.optim.Adam(model.deephead.parameters())\n</pre> wide_opt = torch.optim.Adam(model.wide.parameters(), lr=0.03) deep_opt = torch.optim.Adam(deep_params) text_opt = torch.optim.AdamW(model.deeptext.parameters()) img_opt = torch.optim.AdamW(model.deepimage.parameters()) head_opt = torch.optim.Adam(model.deephead.parameters()) In\u00a0[15]: Copied! <pre>wide_sch = torch.optim.lr_scheduler.StepLR(wide_opt, step_size=5)\ndeep_sch = torch.optim.lr_scheduler.MultiStepLR(deep_opt, milestones=[3, 8])\ntext_sch = torch.optim.lr_scheduler.StepLR(text_opt, step_size=5)\nimg_sch = torch.optim.lr_scheduler.MultiStepLR(deep_opt, milestones=[3, 8])\nhead_sch = torch.optim.lr_scheduler.StepLR(head_opt, step_size=5)\n</pre> wide_sch = torch.optim.lr_scheduler.StepLR(wide_opt, step_size=5) deep_sch = torch.optim.lr_scheduler.MultiStepLR(deep_opt, milestones=[3, 8]) text_sch = torch.optim.lr_scheduler.StepLR(text_opt, step_size=5) img_sch = torch.optim.lr_scheduler.MultiStepLR(deep_opt, milestones=[3, 8]) head_sch = torch.optim.lr_scheduler.StepLR(head_opt, step_size=5) In\u00a0[16]: Copied! <pre># remember, one optimizer per model components, for lr_schedures and initializers is not neccesary\noptimizers = {\n    \"wide\": wide_opt,\n    \"deeptabular\": deep_opt,\n    \"deeptext\": text_opt,\n    \"deepimage\": img_opt,\n    \"deephead\": head_opt,\n}\nschedulers = {\n    \"wide\": wide_sch,\n    \"deeptabular\": deep_sch,\n    \"deeptext\": text_sch,\n    \"deepimage\": img_sch,\n    \"deephead\": head_sch,\n}\n\n# Now...we have used pretrained word embeddings, so you do not want to\n# initialise these  embeddings. However you might still want to initialise the\n# other layers in the DeepText component. No probs, you can do that with the\n# parameter pattern and your knowledge on regular  expressions. Here we are\n# telling to the KaimingNormal initializer to NOT touch the  parameters whose\n# name contains the string word_embed.\ninitializers = {\n    \"wide\": KaimingNormal,\n    \"deeptabular\": KaimingNormal,\n    \"deeptext\": KaimingNormal(pattern=r\"^(?!.*word_embed).*$\"),\n    \"deepimage\": KaimingNormal,\n}\n\nmean = [0.406, 0.456, 0.485]  # BGR\nstd = [0.225, 0.224, 0.229]  # BGR\ntransforms = [ToTensor, Normalize(mean=mean, std=std)]\ncallbacks = [\n    LRHistory(n_epochs=10),\n    EarlyStopping,\n    ModelCheckpoint(filepath=\"model_weights/wd_out\"),\n]\n</pre> # remember, one optimizer per model components, for lr_schedures and initializers is not neccesary optimizers = {     \"wide\": wide_opt,     \"deeptabular\": deep_opt,     \"deeptext\": text_opt,     \"deepimage\": img_opt,     \"deephead\": head_opt, } schedulers = {     \"wide\": wide_sch,     \"deeptabular\": deep_sch,     \"deeptext\": text_sch,     \"deepimage\": img_sch,     \"deephead\": head_sch, }  # Now...we have used pretrained word embeddings, so you do not want to # initialise these  embeddings. However you might still want to initialise the # other layers in the DeepText component. No probs, you can do that with the # parameter pattern and your knowledge on regular  expressions. Here we are # telling to the KaimingNormal initializer to NOT touch the  parameters whose # name contains the string word_embed. initializers = {     \"wide\": KaimingNormal,     \"deeptabular\": KaimingNormal,     \"deeptext\": KaimingNormal(pattern=r\"^(?!.*word_embed).*$\"),     \"deepimage\": KaimingNormal, }  mean = [0.406, 0.456, 0.485]  # BGR std = [0.225, 0.224, 0.229]  # BGR transforms = [ToTensor, Normalize(mean=mean, std=std)] callbacks = [     LRHistory(n_epochs=10),     EarlyStopping,     ModelCheckpoint(filepath=\"model_weights/wd_out\"), ] In\u00a0[17]: Copied! <pre>trainer = Trainer(\n    model,\n    objective=\"rmse\",\n    initializers=initializers,\n    optimizers=optimizers,\n    lr_schedulers=schedulers,\n    callbacks=callbacks,\n    transforms=transforms,\n)\n</pre> trainer = Trainer(     model,     objective=\"rmse\",     initializers=initializers,     optimizers=optimizers,     lr_schedulers=schedulers,     callbacks=callbacks,     transforms=transforms, ) <pre>/Users/javierrodriguezzaurin/Projects/pytorch-widedeep/pytorch_widedeep/initializers.py:34: UserWarning: No initializer found for deephead\n  warnings.warn(\n</pre> In\u00a0[18]: Copied! <pre>trainer.fit(\n    X_wide=X_wide,\n    X_tab=X_tab,\n    X_text=X_text,\n    X_img=X_images,\n    target=target,\n    n_epochs=1,\n    batch_size=32,\n    val_split=0.2,\n)\n</pre> trainer.fit(     X_wide=X_wide,     X_tab=X_tab,     X_text=X_text,     X_img=X_images,     target=target,     n_epochs=1,     batch_size=32,     val_split=0.2, ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 25/25 [00:19&lt;00:00,  1.25it/s, loss=101]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 7/7 [00:04&lt;00:00,  1.62it/s, loss=90.6]\n</pre> <pre>Model weights after training corresponds to the those of the final epoch which might not be the best performing weights. Use the 'ModelCheckpoint' Callback to restore the best epoch weights.\n</pre> <p>we have only run one epoch, but let's check that the LRHistory callback records the lr values for each group</p> In\u00a0[19]: Copied! <pre>trainer.lr_history\n</pre> trainer.lr_history Out[19]: <pre>{'lr_wide_0': [0.03, 0.03],\n 'lr_deeptabular_0': [0.0001, 0.0001],\n 'lr_deeptabular_1': [0.0001, 0.0001],\n 'lr_deeptabular_2': [0.0001, 0.0001],\n 'lr_deeptabular_3': [0.0001, 0.0001],\n 'lr_deeptabular_4': [0.0001, 0.0001],\n 'lr_deeptabular_5': [0.0001, 0.0001],\n 'lr_deeptabular_6': [0.0001, 0.0001],\n 'lr_deeptabular_7': [0.0001, 0.0001],\n 'lr_deeptabular_8': [0.0001, 0.0001],\n 'lr_deeptabular_9': [0.001, 0.001],\n 'lr_deeptabular_10': [0.001, 0.001],\n 'lr_deeptabular_11': [0.001, 0.001],\n 'lr_deeptabular_12': [0.001, 0.001],\n 'lr_deeptext_0': [0.001, 0.001],\n 'lr_deepimage_0': [0.001, 0.001],\n 'lr_deephead_0': [0.001, 0.001]}</pre>"},{"location":"examples/04_regression_with_images_and_text.html#regression-with-images-and-text","title":"Regression with Images and Text\u00b6","text":"<p>In this notebook we will go through a series of examples on how to combine all Wide &amp; Deep components.</p> <p>To that aim I will use the Airbnb listings dataset for London, which you can download from here. I use this dataset simply because it contains tabular data, images and text.</p> <p>I have taken a sample of 1000 listings to keep the data tractable in this notebook. Also, I have preprocessed the data and prepared it for this exercise. All preprocessing steps can be found in the notebook <code>airbnb_data_preprocessing.ipynb</code> in this <code>examples</code> folder.</p>"},{"location":"examples/04_regression_with_images_and_text.html#regression-with-the-defaults","title":"Regression with the defaults\u00b6","text":"<p>The set up</p>"},{"location":"examples/04_regression_with_images_and_text.html#prepare-the-data","title":"Prepare the data\u00b6","text":"<p>I will focus here on how to prepare the data and run the model. Check notebooks 1 and 2 to see what's going on behind the scences</p> <p>Preparing the data is rather simple</p>"},{"location":"examples/04_regression_with_images_and_text.html#build-the-model-components","title":"Build the model components\u00b6","text":""},{"location":"examples/04_regression_with_images_and_text.html#build-the-trainer-and-fit","title":"Build the trainer and fit\u00b6","text":""},{"location":"examples/05_save_and_load_model_and_artifacts.html","title":"05_save_and_load_model_and_artifacts","text":"In\u00a0[1]: Copied! <pre>import pickle\nimport numpy as np\nimport pandas as pd\nimport torch\nimport shutil\n\nfrom pytorch_widedeep.preprocessing import TabPreprocessor\nfrom pytorch_widedeep.training import Trainer\nfrom pytorch_widedeep.callbacks import EarlyStopping, ModelCheckpoint, LRHistory\nfrom pytorch_widedeep.models import TabMlp, WideDeep\nfrom pytorch_widedeep.metrics import Accuracy\nfrom pytorch_widedeep.datasets import load_adult\nfrom sklearn.model_selection import train_test_split\n</pre> import pickle import numpy as np import pandas as pd import torch import shutil  from pytorch_widedeep.preprocessing import TabPreprocessor from pytorch_widedeep.training import Trainer from pytorch_widedeep.callbacks import EarlyStopping, ModelCheckpoint, LRHistory from pytorch_widedeep.models import TabMlp, WideDeep from pytorch_widedeep.metrics import Accuracy from pytorch_widedeep.datasets import load_adult from sklearn.model_selection import train_test_split <pre>/Users/javierrodriguezzaurin/.pyenv/versions/3.10.13/envs/widedeep310/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n  from .autonotebook import tqdm as notebook_tqdm\n</pre> In\u00a0[2]: Copied! <pre>df = load_adult(as_frame=True)\ndf.head()\n</pre> df = load_adult(as_frame=True) df.head() Out[2]: age workclass fnlwgt education educational-num marital-status occupation relationship race gender capital-gain capital-loss hours-per-week native-country income 0 25 Private 226802 11th 7 Never-married Machine-op-inspct Own-child Black Male 0 0 40 United-States &lt;=50K 1 38 Private 89814 HS-grad 9 Married-civ-spouse Farming-fishing Husband White Male 0 0 50 United-States &lt;=50K 2 28 Local-gov 336951 Assoc-acdm 12 Married-civ-spouse Protective-serv Husband White Male 0 0 40 United-States &gt;50K 3 44 Private 160323 Some-college 10 Married-civ-spouse Machine-op-inspct Husband Black Male 7688 0 40 United-States &gt;50K 4 18 ? 103497 Some-college 10 Never-married ? Own-child White Female 0 0 30 United-States &lt;=50K In\u00a0[3]: Copied! <pre># For convenience, we'll replace '-' with '_'\ndf.columns = [c.replace(\"-\", \"_\") for c in df.columns]\n# binary target\ndf[\"target\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int)\ndf.drop(\"income\", axis=1, inplace=True)\ndf.head()\n</pre> # For convenience, we'll replace '-' with '_' df.columns = [c.replace(\"-\", \"_\") for c in df.columns] # binary target df[\"target\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int) df.drop(\"income\", axis=1, inplace=True) df.head() Out[3]: age workclass fnlwgt education educational_num marital_status occupation relationship race gender capital_gain capital_loss hours_per_week native_country target 0 25 Private 226802 11th 7 Never-married Machine-op-inspct Own-child Black Male 0 0 40 United-States 0 1 38 Private 89814 HS-grad 9 Married-civ-spouse Farming-fishing Husband White Male 0 0 50 United-States 0 2 28 Local-gov 336951 Assoc-acdm 12 Married-civ-spouse Protective-serv Husband White Male 0 0 40 United-States 1 3 44 Private 160323 Some-college 10 Married-civ-spouse Machine-op-inspct Husband Black Male 7688 0 40 United-States 1 4 18 ? 103497 Some-college 10 Never-married ? Own-child White Female 0 0 30 United-States 0 In\u00a0[4]: Copied! <pre>train, valid = train_test_split(df, test_size=0.2, stratify=df.target)\n# the test data will be used lately as if it was \"fresh\", new data coming after some time...\nvalid, test = train_test_split(valid, test_size=0.5, stratify=valid.target)\n</pre> train, valid = train_test_split(df, test_size=0.2, stratify=df.target) # the test data will be used lately as if it was \"fresh\", new data coming after some time... valid, test = train_test_split(valid, test_size=0.5, stratify=valid.target) In\u00a0[5]: Copied! <pre>print(f\"train shape: {train.shape}\")\nprint(f\"valid shape: {valid.shape}\")\nprint(f\"test shape: {test.shape}\")\n</pre> print(f\"train shape: {train.shape}\") print(f\"valid shape: {valid.shape}\") print(f\"test shape: {test.shape}\") <pre>train shape: (39073, 15)\nvalid shape: (4884, 15)\ntest shape: (4885, 15)\n</pre> In\u00a0[6]: Copied! <pre>cat_embed_cols = [\n    \"workclass\",\n    \"education\",\n    \"marital_status\",\n    \"occupation\",\n    \"relationship\",\n    \"race\",\n    \"gender\",\n    \"capital_gain\",\n    \"capital_loss\",\n    \"native_country\",\n]\ncontinuous_cols = [\"age\", \"hours_per_week\"]\n</pre> cat_embed_cols = [     \"workclass\",     \"education\",     \"marital_status\",     \"occupation\",     \"relationship\",     \"race\",     \"gender\",     \"capital_gain\",     \"capital_loss\",     \"native_country\", ] continuous_cols = [\"age\", \"hours_per_week\"] In\u00a0[7]: Copied! <pre>tab_preprocessor = TabPreprocessor(\n    embed_cols=cat_embed_cols,\n    continuous_cols=continuous_cols,\n)\nX_tab_train = tab_preprocessor.fit_transform(train)\ny_train = train.target.values\nX_tab_valid = tab_preprocessor.transform(valid)\ny_valid = valid.target.values\n</pre> tab_preprocessor = TabPreprocessor(     embed_cols=cat_embed_cols,     continuous_cols=continuous_cols, ) X_tab_train = tab_preprocessor.fit_transform(train) y_train = train.target.values X_tab_valid = tab_preprocessor.transform(valid) y_valid = valid.target.values <pre>/Users/javierrodriguezzaurin/Projects/pytorch-widedeep/pytorch_widedeep/preprocessing/tab_preprocessor.py:358: UserWarning: Continuous columns will not be normalised\n  warnings.warn(\"Continuous columns will not be normalised\")\n</pre> In\u00a0[8]: Copied! <pre>tab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    cat_embed_dropout=0.1,\n    continuous_cols=continuous_cols,\n    cont_norm_layer=\"layernorm\",\n    embed_continuous_method=\"standard\",\n    cont_embed_dim=8,\n    mlp_hidden_dims=[64, 32],\n    mlp_dropout=0.2,\n    mlp_activation=\"leaky_relu\",\n)\nmodel = WideDeep(deeptabular=tab_mlp)\n</pre> tab_mlp = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     cat_embed_dropout=0.1,     continuous_cols=continuous_cols,     cont_norm_layer=\"layernorm\",     embed_continuous_method=\"standard\",     cont_embed_dim=8,     mlp_hidden_dims=[64, 32],     mlp_dropout=0.2,     mlp_activation=\"leaky_relu\", ) model = WideDeep(deeptabular=tab_mlp) In\u00a0[9]: Copied! <pre>model\n</pre> model Out[9]: <pre>WideDeep(\n  (deeptabular): Sequential(\n    (0): TabMlp(\n      (cat_embed): DiffSizeCatEmbeddings(\n        (embed_layers): ModuleDict(\n          (emb_layer_workclass): Embedding(10, 5, padding_idx=0)\n          (emb_layer_education): Embedding(17, 8, padding_idx=0)\n          (emb_layer_marital_status): Embedding(8, 5, padding_idx=0)\n          (emb_layer_occupation): Embedding(16, 7, padding_idx=0)\n          (emb_layer_relationship): Embedding(7, 4, padding_idx=0)\n          (emb_layer_race): Embedding(6, 4, padding_idx=0)\n          (emb_layer_gender): Embedding(3, 2, padding_idx=0)\n          (emb_layer_capital_gain): Embedding(122, 23, padding_idx=0)\n          (emb_layer_capital_loss): Embedding(97, 21, padding_idx=0)\n          (emb_layer_native_country): Embedding(43, 13, padding_idx=0)\n        )\n        (embedding_dropout): Dropout(p=0.1, inplace=False)\n      )\n      (cont_norm): LayerNorm((2,), eps=1e-05, elementwise_affine=True)\n      (cont_embed): ContEmbeddings(\n        INFO: [ContLinear = weight(n_cont_cols, embed_dim) + bias(n_cont_cols, embed_dim)]\n        (linear): ContLinear(n_cont_cols=2, embed_dim=8, embed_dropout=0.0)\n        (dropout): Dropout(p=0.0, inplace=False)\n      )\n      (encoder): MLP(\n        (mlp): Sequential(\n          (dense_layer_0): Sequential(\n            (0): Linear(in_features=108, out_features=64, bias=True)\n            (1): LeakyReLU(negative_slope=0.01, inplace=True)\n            (2): Dropout(p=0.2, inplace=False)\n          )\n          (dense_layer_1): Sequential(\n            (0): Linear(in_features=64, out_features=32, bias=True)\n            (1): LeakyReLU(negative_slope=0.01, inplace=True)\n            (2): Dropout(p=0.2, inplace=False)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=32, out_features=1, bias=True)\n  )\n)</pre> In\u00a0[10]: Copied! <pre>early_stopping = EarlyStopping()\nmodel_checkpoint = ModelCheckpoint(\n    filepath=\"tmp_dir/adult_tabmlp_model\",\n    save_best_only=True,\n    verbose=1,\n    max_save=1,\n)\n\ntrainer = Trainer(\n    model,\n    objective=\"binary\",\n    callbacks=[early_stopping, model_checkpoint],\n    metrics=[Accuracy],\n)\n\ntrainer.fit(\n    X_train={\"X_tab\": X_tab_train, \"target\": y_train},\n    X_val={\"X_tab\": X_tab_valid, \"target\": y_valid},\n    n_epochs=4,\n    batch_size=256,\n)\n</pre> early_stopping = EarlyStopping() model_checkpoint = ModelCheckpoint(     filepath=\"tmp_dir/adult_tabmlp_model\",     save_best_only=True,     verbose=1,     max_save=1, )  trainer = Trainer(     model,     objective=\"binary\",     callbacks=[early_stopping, model_checkpoint],     metrics=[Accuracy], )  trainer.fit(     X_train={\"X_tab\": X_tab_train, \"target\": y_train},     X_val={\"X_tab\": X_tab_valid, \"target\": y_valid},     n_epochs=4,     batch_size=256, ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:02&lt;00:00, 76.25it/s, loss=0.452, metrics={'acc': 0.7867}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 20/20 [00:00&lt;00:00, 125.36it/s, loss=0.335, metrics={'acc': 0.8532}]\n</pre> <pre>\nEpoch 1: val_loss improved from inf to 0.33532 Saving model to tmp_dir/adult_tabmlp_model_1.p\n</pre> <pre>epoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 76.98it/s, loss=0.355, metrics={'acc': 0.8401}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 20/20 [00:00&lt;00:00, 106.51it/s, loss=0.303, metrics={'acc': 0.8665}]\n</pre> <pre>\nEpoch 2: val_loss improved from 0.33532 to 0.30273 Saving model to tmp_dir/adult_tabmlp_model_2.p\n</pre> <pre>epoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 82.71it/s, loss=0.332, metrics={'acc': 0.849}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 20/20 [00:00&lt;00:00, 107.80it/s, loss=0.288, metrics={'acc': 0.8757}]\n</pre> <pre>\nEpoch 3: val_loss improved from 0.30273 to 0.28791 Saving model to tmp_dir/adult_tabmlp_model_3.p\n</pre> <pre>epoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 79.02it/s, loss=0.32, metrics={'acc': 0.8541}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 20/20 [00:00&lt;00:00, 127.07it/s, loss=0.282, metrics={'acc': 0.8763}]</pre> <pre>\nEpoch 4: val_loss improved from 0.28791 to 0.28238 Saving model to tmp_dir/adult_tabmlp_model_4.p\nModel weights restored to best epoch: 4\n</pre> <pre>\n</pre> In\u00a0[11]: Copied! <pre>torch.save(model, \"tmp_dir/model_saved_option_1.pt\")\n</pre> torch.save(model, \"tmp_dir/model_saved_option_1.pt\") In\u00a0[12]: Copied! <pre>torch.save(model.state_dict(), \"tmp_dir/model_state_dict_saved_option_1.pt\")\n</pre> torch.save(model.state_dict(), \"tmp_dir/model_state_dict_saved_option_1.pt\") In\u00a0[13]: Copied! <pre>trainer.save(path=\"tmp_dir/\", model_filename=\"model_saved_option_2.pt\")\n</pre> trainer.save(path=\"tmp_dir/\", model_filename=\"model_saved_option_2.pt\") <p>or the state dict</p> In\u00a0[14]: Copied! <pre>trainer.save(\n    path=\"tmp_dir/\",\n    model_filename=\"model_state_dict_saved_option_2.pt\",\n    save_state_dict=True,\n)\n</pre> trainer.save(     path=\"tmp_dir/\",     model_filename=\"model_state_dict_saved_option_2.pt\",     save_state_dict=True, ) In\u00a0[15]: Copied! <pre>%%bash\n\nls tmp_dir/\n</pre> %%bash  ls tmp_dir/ <pre>adult_tabmlp_model_4.p\nhistory\nmodel_saved_option_1.pt\nmodel_saved_option_2.pt\nmodel_state_dict_saved_option_1.pt\nmodel_state_dict_saved_option_2.pt\n</pre> In\u00a0[16]: Copied! <pre>%%bash\n\nls tmp_dir/history/\n</pre> %%bash  ls tmp_dir/history/ <pre>train_eval_history.json\n</pre> <p>Note that since we have used the <code>ModelCheckpoint</code> Callback, <code>adult_tabmlp_model_2.p</code> is the model state dict of the model at epoch 2, i.e. same as <code>model_state_dict_saved_option_1.p</code> or <code>model_state_dict_saved_option_2.p</code>.</p> In\u00a0[17]: Copied! <pre>with open(\"tmp_dir/tab_preproc.pkl\", \"wb\") as dp:\n    pickle.dump(tab_preprocessor, dp)\n</pre> with open(\"tmp_dir/tab_preproc.pkl\", \"wb\") as dp:     pickle.dump(tab_preprocessor, dp) In\u00a0[18]: Copied! <pre>with open(\"tmp_dir/eary_stop.pkl\", \"wb\") as es:\n    pickle.dump(early_stopping, es)\n</pre> with open(\"tmp_dir/eary_stop.pkl\", \"wb\") as es:     pickle.dump(early_stopping, es) In\u00a0[19]: Copied! <pre>%%bash\n\nls tmp_dir/\n</pre> %%bash  ls tmp_dir/ <pre>adult_tabmlp_model_4.p\neary_stop.pkl\nhistory\nmodel_saved_option_1.pt\nmodel_saved_option_2.pt\nmodel_state_dict_saved_option_1.pt\nmodel_state_dict_saved_option_2.pt\ntab_preproc.pkl\n</pre> <p>And that is pretty much all you need to resume training or directly predict, let's see</p> In\u00a0[20]: Copied! <pre>test.head()\n</pre> test.head() Out[20]: age workclass fnlwgt education educational_num marital_status occupation relationship race gender capital_gain capital_loss hours_per_week native_country target 10103 43 Private 198282 HS-grad 9 Married-civ-spouse Craft-repair Husband White Male 0 0 40 United-States 1 31799 20 Private 228686 11th 7 Married-civ-spouse Other-service Husband White Male 0 0 40 United-States 0 19971 26 Private 291968 HS-grad 9 Married-civ-spouse Transport-moving Husband White Male 0 0 44 United-States 0 3039 48 Private 175958 Bachelors 13 Divorced Prof-specialty Not-in-family White Male 0 0 30 United-States 0 20725 18 Private 232024 11th 7 Never-married Machine-op-inspct Own-child White Male 0 0 55 United-States 0 In\u00a0[21]: Copied! <pre>with open(\"tmp_dir/tab_preproc.pkl\", \"rb\") as tp:\n    tab_preprocessor_new = pickle.load(tp)\n</pre> with open(\"tmp_dir/tab_preproc.pkl\", \"rb\") as tp:     tab_preprocessor_new = pickle.load(tp) In\u00a0[22]: Copied! <pre>X_test_tab = tab_preprocessor_new.transform(test)\ny_test = test.target\n</pre> X_test_tab = tab_preprocessor_new.transform(test) y_test = test.target In\u00a0[23]: Copied! <pre>tab_mlp_new = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    cat_embed_dropout=0.1,\n    continuous_cols=continuous_cols,\n    cont_norm_layer=\"layernorm\",\n    embed_continuous_method=\"standard\",\n    cont_embed_dim=8,\n    mlp_hidden_dims=[64, 32],\n    mlp_dropout=0.2,\n    mlp_activation=\"leaky_relu\",\n)\nnew_model = WideDeep(deeptabular=tab_mlp)\n</pre> tab_mlp_new = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     cat_embed_dropout=0.1,     continuous_cols=continuous_cols,     cont_norm_layer=\"layernorm\",     embed_continuous_method=\"standard\",     cont_embed_dim=8,     mlp_hidden_dims=[64, 32],     mlp_dropout=0.2,     mlp_activation=\"leaky_relu\", ) new_model = WideDeep(deeptabular=tab_mlp) In\u00a0[24]: Copied! <pre>new_model.load_state_dict(torch.load(\"tmp_dir/model_state_dict_saved_option_2.pt\"))\n</pre> new_model.load_state_dict(torch.load(\"tmp_dir/model_state_dict_saved_option_2.pt\")) Out[24]: <pre>&lt;All keys matched successfully&gt;</pre> In\u00a0[25]: Copied! <pre>trainer = Trainer(\n    model,\n    objective=\"binary\",\n)\n</pre> trainer = Trainer(     model,     objective=\"binary\", ) In\u00a0[26]: Copied! <pre>preds = trainer.predict(X_tab=X_test_tab, batch_size=32)\n</pre> preds = trainer.predict(X_tab=X_test_tab, batch_size=32) <pre>predict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:00&lt;00:00, 309.83it/s]\n</pre> In\u00a0[27]: Copied! <pre>from sklearn.metrics import accuracy_score\n</pre> from sklearn.metrics import accuracy_score In\u00a0[28]: Copied! <pre>accuracy_score(y_test, preds)\n</pre> accuracy_score(y_test, preds) Out[28]: <pre>0.8595701125895598</pre> In\u00a0[29]: Copied! <pre>shutil.rmtree(\"tmp_dir/\")\n</pre> shutil.rmtree(\"tmp_dir/\")"},{"location":"examples/05_save_and_load_model_and_artifacts.html#save-and-load-model-and-artifacts","title":"Save and load model and artifacts\u00b6","text":"<p>In this notebook I will show the different options to save and load a model, as well as some additional objects produced during training.</p> <p>On a given day, you train a model...</p>"},{"location":"examples/05_save_and_load_model_and_artifacts.html#save-model-option-1","title":"Save model: option 1\u00b6","text":"<p>save (and load) a model as you woud do with any other torch model</p>"},{"location":"examples/05_save_and_load_model_and_artifacts.html#save-model-option-2","title":"Save model: option 2\u00b6","text":"<p>use the <code>trainer</code>. The <code>trainer</code> will also save the training history and the learning rate history (if learning rate schedulers are used)</p>"},{"location":"examples/05_save_and_load_model_and_artifacts.html#save-preprocessors-and-callbacks","title":"Save preprocessors and callbacks\u00b6","text":"<p>...just pickle them</p>"},{"location":"examples/05_save_and_load_model_and_artifacts.html#run-new-experiment-prepare-new-dataset-load-model-and-predict","title":"Run New experiment: prepare new dataset, load model, and predict\u00b6","text":""},{"location":"examples/06_finetune_and_warmup.html","title":"06_finetune_and_warmup","text":"In\u00a0[1]: Copied! <pre>import numpy as np\nimport pandas as pd\nimport torch\n\nfrom pytorch_widedeep import Trainer\nfrom pytorch_widedeep.preprocessing import WidePreprocessor, TabPreprocessor\nfrom pytorch_widedeep.models import Wide, TabMlp, TabResnet, WideDeep\nfrom pytorch_widedeep.metrics import Accuracy\nfrom pytorch_widedeep.datasets import load_adult\n</pre> import numpy as np import pandas as pd import torch  from pytorch_widedeep import Trainer from pytorch_widedeep.preprocessing import WidePreprocessor, TabPreprocessor from pytorch_widedeep.models import Wide, TabMlp, TabResnet, WideDeep from pytorch_widedeep.metrics import Accuracy from pytorch_widedeep.datasets import load_adult <pre>/Users/javierrodriguezzaurin/.pyenv/versions/3.10.13/envs/widedeep310/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n  from .autonotebook import tqdm as notebook_tqdm\n</pre> In\u00a0[2]: Copied! <pre>df = load_adult(as_frame=True)\n# For convenience, we'll replace '-' with '_'\ndf.columns = [c.replace(\"-\", \"_\") for c in df.columns]\n# binary target\ndf[\"income_label\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int)\ndf.drop(\"income\", axis=1, inplace=True)\ndf.head()\n</pre> df = load_adult(as_frame=True) # For convenience, we'll replace '-' with '_' df.columns = [c.replace(\"-\", \"_\") for c in df.columns] # binary target df[\"income_label\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int) df.drop(\"income\", axis=1, inplace=True) df.head() Out[2]: age workclass fnlwgt education educational_num marital_status occupation relationship race gender capital_gain capital_loss hours_per_week native_country income_label 0 25 Private 226802 11th 7 Never-married Machine-op-inspct Own-child Black Male 0 0 40 United-States 0 1 38 Private 89814 HS-grad 9 Married-civ-spouse Farming-fishing Husband White Male 0 0 50 United-States 0 2 28 Local-gov 336951 Assoc-acdm 12 Married-civ-spouse Protective-serv Husband White Male 0 0 40 United-States 1 3 44 Private 160323 Some-college 10 Married-civ-spouse Machine-op-inspct Husband Black Male 7688 0 40 United-States 1 4 18 ? 103497 Some-college 10 Never-married ? Own-child White Female 0 0 30 United-States 0 In\u00a0[3]: Copied! <pre># Define wide, crossed and deep tabular columns\nwide_cols = [\n    \"workclass\",\n    \"education\",\n    \"marital_status\",\n    \"occupation\",\n    \"relationship\",\n    \"race\",\n    \"gender\",\n    \"native_country\",\n]\ncrossed_cols = [(\"education\", \"occupation\"), (\"native_country\", \"occupation\")]\ncat_embed_cols = [\n    \"workclass\",\n    \"education\",\n    \"marital_status\",\n    \"occupation\",\n    \"relationship\",\n    \"race\",\n    \"gender\",\n    \"capital_gain\",\n    \"capital_loss\",\n    \"native_country\",\n]\ncontinuous_cols = [\"age\", \"hours_per_week\"]\ntarget_col = \"income_label\"\ntarget = df[target_col].values\n</pre> # Define wide, crossed and deep tabular columns wide_cols = [     \"workclass\",     \"education\",     \"marital_status\",     \"occupation\",     \"relationship\",     \"race\",     \"gender\",     \"native_country\", ] crossed_cols = [(\"education\", \"occupation\"), (\"native_country\", \"occupation\")] cat_embed_cols = [     \"workclass\",     \"education\",     \"marital_status\",     \"occupation\",     \"relationship\",     \"race\",     \"gender\",     \"capital_gain\",     \"capital_loss\",     \"native_country\", ] continuous_cols = [\"age\", \"hours_per_week\"] target_col = \"income_label\" target = df[target_col].values In\u00a0[4]: Copied! <pre># TARGET\ntarget = df[target_col].values\n\n# WIDE\nwide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols)\nX_wide = wide_preprocessor.fit_transform(df)\n\n# DEEP\ntab_preprocessor = TabPreprocessor(\n    cat_embed_cols=cat_embed_cols, continuous_cols=continuous_cols\n)\nX_tab = tab_preprocessor.fit_transform(df)\n</pre> # TARGET target = df[target_col].values  # WIDE wide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols) X_wide = wide_preprocessor.fit_transform(df)  # DEEP tab_preprocessor = TabPreprocessor(     cat_embed_cols=cat_embed_cols, continuous_cols=continuous_cols ) X_tab = tab_preprocessor.fit_transform(df) <pre>/Users/javierrodriguezzaurin/Projects/pytorch-widedeep/pytorch_widedeep/preprocessing/tab_preprocessor.py:358: UserWarning: Continuous columns will not be normalised\n  warnings.warn(\"Continuous columns will not be normalised\")\n</pre> In\u00a0[5]: Copied! <pre>wide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1)\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    cat_embed_dropout=0.1,\n    continuous_cols=continuous_cols,\n    embed_continuous_method=\"standard\",\n    cont_embed_dim=8,\n    mlp_hidden_dims=[64, 32],\n    mlp_dropout=0.2,\n    mlp_activation=\"leaky_relu\",\n)\nmodel = WideDeep(wide=wide, deeptabular=tab_mlp)\n</pre> wide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1) tab_mlp = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     cat_embed_dropout=0.1,     continuous_cols=continuous_cols,     embed_continuous_method=\"standard\",     cont_embed_dim=8,     mlp_hidden_dims=[64, 32],     mlp_dropout=0.2,     mlp_activation=\"leaky_relu\", ) model = WideDeep(wide=wide, deeptabular=tab_mlp) In\u00a0[6]: Copied! <pre>trainer = Trainer(\n    model,\n    objective=\"binary\",\n    optimizers=torch.optim.Adam(model.parameters(), lr=0.01),\n    metrics=[Accuracy],\n)\n</pre> trainer = Trainer(     model,     objective=\"binary\",     optimizers=torch.optim.Adam(model.parameters(), lr=0.01),     metrics=[Accuracy], ) In\u00a0[7]: Copied! <pre>trainer.fit(\n    X_wide=X_wide, X_tab=X_tab, target=target, n_epochs=2, val_split=0.2, batch_size=256\n)\n</pre> trainer.fit(     X_wide=X_wide, X_tab=X_tab, target=target, n_epochs=2, val_split=0.2, batch_size=256 ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:02&lt;00:00, 74.26it/s, loss=0.399, metrics={'acc': 0.8163}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 39/39 [00:00&lt;00:00, 91.03it/s, loss=0.296, metrics={'acc': 0.8677}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 81.31it/s, loss=0.3, metrics={'acc': 0.8614}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 39/39 [00:00&lt;00:00, 106.45it/s, loss=0.285, metrics={'acc': 0.8721}]\n</pre> In\u00a0[8]: Copied! <pre>trainer.save(path=\"models_dir/\", save_state_dict=True, model_filename=\"model_1.pt\")\n</pre> trainer.save(path=\"models_dir/\", save_state_dict=True, model_filename=\"model_1.pt\") <p>Now time goes by...and we want to fine-tune the model to another, new dataset (for example, a dataset that is identical to the one you used to train the previous model but for another country).</p> <p>Here I will use the same dataset just for illustration purposes, but the flow would be identical to that new dataset</p> In\u00a0[9]: Copied! <pre>wide_1 = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1)\ntab_mlp_1 = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    cat_embed_dropout=0.1,\n    continuous_cols=continuous_cols,\n    embed_continuous_method=\"standard\",\n    cont_embed_dim=8,\n    mlp_hidden_dims=[64, 32],\n    mlp_dropout=0.2,\n    mlp_activation=\"leaky_relu\",\n)\nmodel_1 = WideDeep(wide=wide, deeptabular=tab_mlp)\n</pre> wide_1 = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1) tab_mlp_1 = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     cat_embed_dropout=0.1,     continuous_cols=continuous_cols,     embed_continuous_method=\"standard\",     cont_embed_dim=8,     mlp_hidden_dims=[64, 32],     mlp_dropout=0.2,     mlp_activation=\"leaky_relu\", ) model_1 = WideDeep(wide=wide, deeptabular=tab_mlp) In\u00a0[10]: Copied! <pre>model_1.load_state_dict(torch.load(\"models_dir/model_1.pt\"))\n</pre> model_1.load_state_dict(torch.load(\"models_dir/model_1.pt\")) Out[10]: <pre>&lt;All keys matched successfully&gt;</pre> In\u00a0[11]: Copied! <pre>trainer_1 = Trainer(model_1, objective=\"binary\", metrics=[Accuracy])\n</pre> trainer_1 = Trainer(model_1, objective=\"binary\", metrics=[Accuracy]) In\u00a0[12]: Copied! <pre>trainer_1.fit(\n    X_wide=X_wide,\n    X_tab=X_tab,\n    target=target,\n    n_epochs=2,\n    batch_size=256,\n    finetune=True,\n    finetune_epochs=2,\n)\n</pre> trainer_1.fit(     X_wide=X_wide,     X_tab=X_tab,     target=target,     n_epochs=2,     batch_size=256,     finetune=True,     finetune_epochs=2, ) <pre>Training wide for 2 epochs\n</pre> <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 191/191 [00:01&lt;00:00, 97.37it/s, loss=0.39, metrics={'acc': 0.8152}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 191/191 [00:01&lt;00:00, 104.04it/s, loss=0.359, metrics={'acc': 0.824}]\n</pre> <pre>Training deeptabular for 2 epochs\n</pre> <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 191/191 [00:02&lt;00:00, 83.83it/s, loss=0.297, metrics={'acc': 0.8365}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 191/191 [00:02&lt;00:00, 82.78it/s, loss=0.283, metrics={'acc': 0.8445}]\n</pre> <pre>Fine-tuning (or warmup) of individual components completed. Training the whole model for 2 epochs\n</pre> <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 191/191 [00:02&lt;00:00, 72.84it/s, loss=0.281, metrics={'acc': 0.8716}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 191/191 [00:02&lt;00:00, 77.46it/s, loss=0.273, metrics={'acc': 0.8744}]\n</pre> <p>Note that, as I describe above, in scenario 2, we can just use this to warm up models before they joined training begins:</p> In\u00a0[13]: Copied! <pre>wide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1)\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    cat_embed_dropout=0.1,\n    continuous_cols=continuous_cols,\n    embed_continuous_method=\"standard\",\n    cont_embed_dim=8,\n    mlp_hidden_dims=[64, 32],\n    mlp_dropout=0.2,\n    mlp_activation=\"leaky_relu\",\n)\nmodel = WideDeep(wide=wide, deeptabular=tab_mlp)\n</pre> wide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1) tab_mlp = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     cat_embed_dropout=0.1,     continuous_cols=continuous_cols,     embed_continuous_method=\"standard\",     cont_embed_dim=8,     mlp_hidden_dims=[64, 32],     mlp_dropout=0.2,     mlp_activation=\"leaky_relu\", ) model = WideDeep(wide=wide, deeptabular=tab_mlp) In\u00a0[14]: Copied! <pre>trainer_2 = Trainer(model, objective=\"binary\", metrics=[Accuracy])\n</pre> trainer_2 = Trainer(model, objective=\"binary\", metrics=[Accuracy]) In\u00a0[15]: Copied! <pre>trainer_2.fit(\n    X_wide=X_wide,\n    X_tab=X_tab,\n    target=target,\n    val_split=0.1,\n    warmup=True,\n    warmup_epochs=2,\n    n_epochs=2,\n    batch_size=256,\n)\n</pre> trainer_2.fit(     X_wide=X_wide,     X_tab=X_tab,     target=target,     val_split=0.1,     warmup=True,     warmup_epochs=2,     n_epochs=2,     batch_size=256, ) <pre>Training wide for 2 epochs\n</pre> <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:01&lt;00:00, 102.49it/s, loss=0.52, metrics={'acc': 0.7519}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:01&lt;00:00, 98.15it/s, loss=0.381, metrics={'acc': 0.7891}]\n</pre> <pre>Training deeptabular for 2 epochs\n</pre> <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:02&lt;00:00, 82.97it/s, loss=0.356, metrics={'acc': 0.8043}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:02&lt;00:00, 80.27it/s, loss=0.295, metrics={'acc': 0.8195}]\n</pre> <pre>Fine-tuning (or warmup) of individual components completed. Training the whole model for 2 epochs\n</pre> <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:02&lt;00:00, 77.27it/s, loss=0.291, metrics={'acc': 0.8667}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 20/20 [00:00&lt;00:00, 89.57it/s, loss=0.289, metrics={'acc': 0.8665}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:02&lt;00:00, 72.69it/s, loss=0.283, metrics={'acc': 0.8693}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 20/20 [00:00&lt;00:00, 91.81it/s, loss=0.284, metrics={'acc': 0.869}]\n</pre> <p>We need to explicitly indicate</p> <ol> <li><p>That we want fine-tune</p> </li> <li><p>The components that we want to individually fine-tune</p> </li> <li><p>In case of gradual fine-tuning, the routine (\"felbo\" or \"howard\")</p> </li> <li><p>The layers we want to fine-tune.</p> </li> </ol> <p>For example</p> In\u00a0[16]: Copied! <pre>wide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1)\ntab_resnet = TabResnet(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    cat_embed_dropout=0.1,\n    continuous_cols=continuous_cols,\n    blocks_dims=[200, 200, 200],\n)\nmodel = WideDeep(wide=wide, deeptabular=tab_resnet)\n</pre> wide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1) tab_resnet = TabResnet(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     cat_embed_dropout=0.1,     continuous_cols=continuous_cols,     blocks_dims=[200, 200, 200], ) model = WideDeep(wide=wide, deeptabular=tab_resnet) In\u00a0[17]: Copied! <pre>model\n</pre> model Out[17]: <pre>WideDeep(\n  (wide): Wide(\n    (wide_linear): Embedding(809, 1, padding_idx=0)\n  )\n  (deeptabular): Sequential(\n    (0): TabResnet(\n      (cat_embed): DiffSizeCatEmbeddings(\n        (embed_layers): ModuleDict(\n          (emb_layer_workclass): Embedding(10, 5, padding_idx=0)\n          (emb_layer_education): Embedding(17, 8, padding_idx=0)\n          (emb_layer_marital_status): Embedding(8, 5, padding_idx=0)\n          (emb_layer_occupation): Embedding(16, 7, padding_idx=0)\n          (emb_layer_relationship): Embedding(7, 4, padding_idx=0)\n          (emb_layer_race): Embedding(6, 4, padding_idx=0)\n          (emb_layer_gender): Embedding(3, 2, padding_idx=0)\n          (emb_layer_capital_gain): Embedding(124, 24, padding_idx=0)\n          (emb_layer_capital_loss): Embedding(100, 21, padding_idx=0)\n          (emb_layer_native_country): Embedding(43, 13, padding_idx=0)\n        )\n        (embedding_dropout): Dropout(p=0.1, inplace=False)\n      )\n      (cont_norm): Identity()\n      (encoder): DenseResnet(\n        (dense_resnet): Sequential(\n          (lin_inp): Linear(in_features=95, out_features=200, bias=False)\n          (bn_inp): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n          (block_0): BasicBlock(\n            (lin1): Linear(in_features=200, out_features=200, bias=False)\n            (bn1): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n            (leaky_relu): LeakyReLU(negative_slope=0.01, inplace=True)\n            (dp): Dropout(p=0.1, inplace=False)\n            (lin2): Linear(in_features=200, out_features=200, bias=False)\n            (bn2): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n          )\n          (block_1): BasicBlock(\n            (lin1): Linear(in_features=200, out_features=200, bias=False)\n            (bn1): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n            (leaky_relu): LeakyReLU(negative_slope=0.01, inplace=True)\n            (dp): Dropout(p=0.1, inplace=False)\n            (lin2): Linear(in_features=200, out_features=200, bias=False)\n            (bn2): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=200, out_features=1, bias=True)\n  )\n)</pre> <p>let's first train as usual</p> In\u00a0[18]: Copied! <pre>trainer_3 = Trainer(model, objective=\"binary\", metrics=[Accuracy])\n</pre> trainer_3 = Trainer(model, objective=\"binary\", metrics=[Accuracy]) In\u00a0[19]: Copied! <pre>trainer_3.fit(\n    X_wide=X_wide, X_tab=X_tab, target=target, val_split=0.1, n_epochs=2, batch_size=256\n)\n</pre> trainer_3.fit(     X_wide=X_wide, X_tab=X_tab, target=target, val_split=0.1, n_epochs=2, batch_size=256 ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:03&lt;00:00, 54.23it/s, loss=0.382, metrics={'acc': 0.8239}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 20/20 [00:00&lt;00:00, 84.72it/s, loss=0.331, metrics={'acc': 0.8526}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:03&lt;00:00, 54.35it/s, loss=0.33, metrics={'acc': 0.8465}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 20/20 [00:00&lt;00:00, 68.15it/s, loss=0.312, metrics={'acc': 0.8604}]\n</pre> In\u00a0[20]: Copied! <pre>trainer_3.save(path=\"models_dir\", save_state_dict=True, model_filename=\"model_3.pt\")\n</pre> trainer_3.save(path=\"models_dir\", save_state_dict=True, model_filename=\"model_3.pt\") <p>Now we are going to fine-tune the model components, and in the case of the <code>deeptabular</code> component, we will fine-tune the resnet-blocks and the linear layer but NOT the embeddings.</p> <p>For this, we need to access the model component's children: <code>deeptabular</code> $\\rightarrow$ <code>tab_resnet</code> $\\rightarrow$ <code>dense_resnet</code> $\\rightarrow$ <code>blocks</code></p> In\u00a0[21]: Copied! <pre>wide_3 = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1)\ntab_resnet_3 = TabResnet(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    cat_embed_dropout=0.1,\n    continuous_cols=continuous_cols,\n    blocks_dims=[200, 200, 200],\n)\nmodel_3 = WideDeep(wide=wide, deeptabular=tab_resnet)\n</pre> wide_3 = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1) tab_resnet_3 = TabResnet(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     cat_embed_dropout=0.1,     continuous_cols=continuous_cols,     blocks_dims=[200, 200, 200], ) model_3 = WideDeep(wide=wide, deeptabular=tab_resnet) In\u00a0[22]: Copied! <pre>model_3.load_state_dict(torch.load(\"models_dir/model_3.pt\"))\n</pre> model_3.load_state_dict(torch.load(\"models_dir/model_3.pt\")) Out[22]: <pre>&lt;All keys matched successfully&gt;</pre> In\u00a0[23]: Copied! <pre>model_3\n</pre> model_3 Out[23]: <pre>WideDeep(\n  (wide): Wide(\n    (wide_linear): Embedding(809, 1, padding_idx=0)\n  )\n  (deeptabular): Sequential(\n    (0): TabResnet(\n      (cat_embed): DiffSizeCatEmbeddings(\n        (embed_layers): ModuleDict(\n          (emb_layer_workclass): Embedding(10, 5, padding_idx=0)\n          (emb_layer_education): Embedding(17, 8, padding_idx=0)\n          (emb_layer_marital_status): Embedding(8, 5, padding_idx=0)\n          (emb_layer_occupation): Embedding(16, 7, padding_idx=0)\n          (emb_layer_relationship): Embedding(7, 4, padding_idx=0)\n          (emb_layer_race): Embedding(6, 4, padding_idx=0)\n          (emb_layer_gender): Embedding(3, 2, padding_idx=0)\n          (emb_layer_capital_gain): Embedding(124, 24, padding_idx=0)\n          (emb_layer_capital_loss): Embedding(100, 21, padding_idx=0)\n          (emb_layer_native_country): Embedding(43, 13, padding_idx=0)\n        )\n        (embedding_dropout): Dropout(p=0.1, inplace=False)\n      )\n      (cont_norm): Identity()\n      (encoder): DenseResnet(\n        (dense_resnet): Sequential(\n          (lin_inp): Linear(in_features=95, out_features=200, bias=False)\n          (bn_inp): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n          (block_0): BasicBlock(\n            (lin1): Linear(in_features=200, out_features=200, bias=False)\n            (bn1): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n            (leaky_relu): LeakyReLU(negative_slope=0.01, inplace=True)\n            (dp): Dropout(p=0.1, inplace=False)\n            (lin2): Linear(in_features=200, out_features=200, bias=False)\n            (bn2): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n          )\n          (block_1): BasicBlock(\n            (lin1): Linear(in_features=200, out_features=200, bias=False)\n            (bn1): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n            (leaky_relu): LeakyReLU(negative_slope=0.01, inplace=True)\n            (dp): Dropout(p=0.1, inplace=False)\n            (lin2): Linear(in_features=200, out_features=200, bias=False)\n            (bn2): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=200, out_features=1, bias=True)\n  )\n)</pre> In\u00a0[24]: Copied! <pre>tab_lin_layer = list(model_3.deeptabular.children())[1]\n</pre> tab_lin_layer = list(model_3.deeptabular.children())[1] In\u00a0[25]: Copied! <pre>tab_lin_layer\n</pre> tab_lin_layer Out[25]: <pre>Linear(in_features=200, out_features=1, bias=True)</pre> In\u00a0[26]: Copied! <pre>tab_deep_layers = []\nfor n1, c1 in model_3.deeptabular.named_children():\n    if (\n        n1 == \"0\"\n    ):  # 0 is the model component and 1 is always the prediction layer added by the `WideDeep` class\n        for n2, c2 in c1.named_children():\n            if n2 == \"encoder\":  # TabResnet\n                for _, c3 in c2.named_children():\n                    for n4, c4 in c3.named_children():  # dense_resnet\n                        if \"block\" in n4:\n                            tab_deep_layers.append((n4, c4))\n</pre> tab_deep_layers = [] for n1, c1 in model_3.deeptabular.named_children():     if (         n1 == \"0\"     ):  # 0 is the model component and 1 is always the prediction layer added by the `WideDeep` class         for n2, c2 in c1.named_children():             if n2 == \"encoder\":  # TabResnet                 for _, c3 in c2.named_children():                     for n4, c4 in c3.named_children():  # dense_resnet                         if \"block\" in n4:                             tab_deep_layers.append((n4, c4)) In\u00a0[27]: Copied! <pre>tab_deep_layers\n</pre> tab_deep_layers Out[27]: <pre>[('block_0',\n  BasicBlock(\n    (lin1): Linear(in_features=200, out_features=200, bias=False)\n    (bn1): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n    (leaky_relu): LeakyReLU(negative_slope=0.01, inplace=True)\n    (dp): Dropout(p=0.1, inplace=False)\n    (lin2): Linear(in_features=200, out_features=200, bias=False)\n    (bn2): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n  )),\n ('block_1',\n  BasicBlock(\n    (lin1): Linear(in_features=200, out_features=200, bias=False)\n    (bn1): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n    (leaky_relu): LeakyReLU(negative_slope=0.01, inplace=True)\n    (dp): Dropout(p=0.1, inplace=False)\n    (lin2): Linear(in_features=200, out_features=200, bias=False)\n    (bn2): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n  ))]</pre> <p>Now remember, we need to pass ONLY LAYERS (before I included the name for clarity) the layers in WARM UP ORDER, therefore:</p> In\u00a0[28]: Copied! <pre>tab_deep_layers = [el[1] for el in tab_deep_layers][::-1]\n</pre> tab_deep_layers = [el[1] for el in tab_deep_layers][::-1] In\u00a0[29]: Copied! <pre>tab_layers = [tab_lin_layer] + tab_deep_layers[::-1]\n</pre> tab_layers = [tab_lin_layer] + tab_deep_layers[::-1] In\u00a0[30]: Copied! <pre>tab_layers\n</pre> tab_layers Out[30]: <pre>[Linear(in_features=200, out_features=1, bias=True),\n BasicBlock(\n   (lin1): Linear(in_features=200, out_features=200, bias=False)\n   (bn1): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n   (leaky_relu): LeakyReLU(negative_slope=0.01, inplace=True)\n   (dp): Dropout(p=0.1, inplace=False)\n   (lin2): Linear(in_features=200, out_features=200, bias=False)\n   (bn2): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n ),\n BasicBlock(\n   (lin1): Linear(in_features=200, out_features=200, bias=False)\n   (bn1): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n   (leaky_relu): LeakyReLU(negative_slope=0.01, inplace=True)\n   (dp): Dropout(p=0.1, inplace=False)\n   (lin2): Linear(in_features=200, out_features=200, bias=False)\n   (bn2): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n )]</pre> <p>And now simply</p> In\u00a0[31]: Copied! <pre>trainer_4 = Trainer(model_3, objective=\"binary\", metrics=[Accuracy])\n</pre> trainer_4 = Trainer(model_3, objective=\"binary\", metrics=[Accuracy]) In\u00a0[32]: Copied! <pre>trainer_4.fit(\n    X_wide=X_wide,\n    X_tab=X_tab,\n    target=target,\n    val_split=0.1,\n    finetune=True,\n    finetune_epochs=2,\n    deeptabular_gradual=True,\n    deeptabular_layers=tab_layers,\n    deeptabular_max_lr=0.01,\n    n_epochs=2,\n    batch_size=256,\n)\n</pre> trainer_4.fit(     X_wide=X_wide,     X_tab=X_tab,     target=target,     val_split=0.1,     finetune=True,     finetune_epochs=2,     deeptabular_gradual=True,     deeptabular_layers=tab_layers,     deeptabular_max_lr=0.01,     n_epochs=2,     batch_size=256, ) <pre>Training wide for 2 epochs\n</pre> <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:01&lt;00:00, 95.17it/s, loss=0.504, metrics={'acc': 0.7523}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:01&lt;00:00, 99.83it/s, loss=0.384, metrics={'acc': 0.789}]\n</pre> <pre>Training deeptabular, layer 1 of 3\n</pre> <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:02&lt;00:00, 72.31it/s, loss=0.317, metrics={'acc': 0.8098}]\n</pre> <pre>Training deeptabular, layer 2 of 3\n</pre> <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:02&lt;00:00, 65.97it/s, loss=0.312, metrics={'acc': 0.8214}]\n</pre> <pre>Training deeptabular, layer 3 of 3\n</pre> <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:02&lt;00:00, 63.92it/s, loss=0.306, metrics={'acc': 0.8284}]\n</pre> <pre>Fine-tuning (or warmup) of individual components completed. Training the whole model for 2 epochs\n</pre> <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:03&lt;00:00, 57.26it/s, loss=0.292, metrics={'acc': 0.8664}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 20/20 [00:00&lt;00:00, 84.56it/s, loss=0.292, metrics={'acc': 0.8696}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:03&lt;00:00, 53.61it/s, loss=0.282, metrics={'acc': 0.8693}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 20/20 [00:00&lt;00:00, 80.59it/s, loss=0.289, metrics={'acc': 0.8719}]\n</pre> <p>Finally, there is one more use case I would like to consider. The case where we train only one component and we just want to fine-tune and stop the training afterwards, since there is no joined training. This is a simple as</p> In\u00a0[33]: Copied! <pre>tab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    cat_embed_dropout=0.1,\n    continuous_cols=continuous_cols,\n    mlp_hidden_dims=[64, 32],\n    mlp_dropout=0.2,\n    mlp_activation=\"leaky_relu\",\n)\nmodel = WideDeep(deeptabular=tab_mlp)\n</pre> tab_mlp = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     cat_embed_dropout=0.1,     continuous_cols=continuous_cols,     mlp_hidden_dims=[64, 32],     mlp_dropout=0.2,     mlp_activation=\"leaky_relu\", ) model = WideDeep(deeptabular=tab_mlp) In\u00a0[34]: Copied! <pre>trainer_5 = Trainer(\n    model,\n    objective=\"binary\",\n    optimizers=torch.optim.Adam(model.parameters(), lr=0.01),\n    metrics=[Accuracy],\n)\n</pre> trainer_5 = Trainer(     model,     objective=\"binary\",     optimizers=torch.optim.Adam(model.parameters(), lr=0.01),     metrics=[Accuracy], ) In\u00a0[35]: Copied! <pre>trainer_5.fit(\n    X_wide=X_wide, X_tab=X_tab, target=target, val_split=0.1, n_epochs=1, batch_size=256\n)\n</pre> trainer_5.fit(     X_wide=X_wide, X_tab=X_tab, target=target, val_split=0.1, n_epochs=1, batch_size=256 ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:02&lt;00:00, 73.69it/s, loss=0.365, metrics={'acc': 0.8331}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 20/20 [00:00&lt;00:00, 92.56it/s, loss=0.299, metrics={'acc': 0.8673}]\n</pre> In\u00a0[36]: Copied! <pre>trainer_5.save(path=\"models_dir\", save_state_dict=True, model_filename=\"model_5.pt\")\n</pre> trainer_5.save(path=\"models_dir\", save_state_dict=True, model_filename=\"model_5.pt\") In\u00a0[37]: Copied! <pre>tab_mlp_5 = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    cat_embed_dropout=0.1,\n    continuous_cols=continuous_cols,\n    mlp_hidden_dims=[64, 32],\n    mlp_dropout=0.2,\n    mlp_activation=\"leaky_relu\",\n)\nmodel_5 = WideDeep(deeptabular=tab_mlp_5)\n</pre> tab_mlp_5 = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     cat_embed_dropout=0.1,     continuous_cols=continuous_cols,     mlp_hidden_dims=[64, 32],     mlp_dropout=0.2,     mlp_activation=\"leaky_relu\", ) model_5 = WideDeep(deeptabular=tab_mlp_5) In\u00a0[38]: Copied! <pre>model_5.load_state_dict(torch.load(\"models_dir/model_5.pt\"))\n</pre> model_5.load_state_dict(torch.load(\"models_dir/model_5.pt\")) Out[38]: <pre>&lt;All keys matched successfully&gt;</pre> <p>...times go by...</p> In\u00a0[39]: Copied! <pre>trainer_6 = Trainer(\n    model_5,\n    objective=\"binary\",\n    optimizers=torch.optim.Adam(model.parameters(), lr=0.01),\n    metrics=[Accuracy],\n)\n</pre> trainer_6 = Trainer(     model_5,     objective=\"binary\",     optimizers=torch.optim.Adam(model.parameters(), lr=0.01),     metrics=[Accuracy], ) In\u00a0[40]: Copied! <pre>trainer_6.fit(\n    X_wide=X_wide,\n    X_tab=X_tab,\n    target=target,\n    val_split=0.1,\n    finetune=True,\n    finetune_epochs=2,\n    finetune_max_lr=0.01,\n    stop_after_finetuning=True,\n    batch_size=256,\n)\n</pre> trainer_6.fit(     X_wide=X_wide,     X_tab=X_tab,     target=target,     val_split=0.1,     finetune=True,     finetune_epochs=2,     finetune_max_lr=0.01,     stop_after_finetuning=True,     batch_size=256, ) <pre>Training deeptabular for 2 epochs\n</pre> <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:02&lt;00:00, 73.86it/s, loss=0.298, metrics={'acc': 0.8652}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:02&lt;00:00, 75.45it/s, loss=0.286, metrics={'acc': 0.8669}]\n</pre> <pre>Fine-tuning (or warmup) of individual components completed. Training the whole model for 1 epochs\n</pre> <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:02&lt;00:00, 76.29it/s, loss=0.282, metrics={'acc': 0.8698}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 20/20 [00:00&lt;00:00, 84.93it/s, loss=0.281, metrics={'acc': 0.8749}]\n</pre> In\u00a0[42]: Copied! <pre>import shutil\n\nshutil.rmtree(\"models_dir/\")\nshutil.rmtree(\"model_weights/\")\n</pre> import shutil  shutil.rmtree(\"models_dir/\") shutil.rmtree(\"model_weights/\") In\u00a0[\u00a0]: Copied! <pre>\n</pre>"},{"location":"examples/06_finetune_and_warmup.html#the-finetunewarm-up-option","title":"The FineTune/Warm Up option\u00b6","text":"<p>Let's place ourselves in two possible scenarios.</p> <ol> <li><p>Let's assume we have run a model and we want to just transfer the learnings (you know...transfer-learning) to another dataset, or simply we have received new data and we do not want to start the training of each component from scratch. Simply, we want to load the pre-trained weights and fine-tune.</p> </li> <li><p>We just want to \"warm up\" individual model components individually before the joined training begins.</p> </li> </ol> <p>This can be done with the <code>finetune</code> set of parameters. There are 3 fine-tuning routines:</p> <ol> <li>Fine-tune all trainable layers at once with a triangular one-cycle learning rate (referred as slanted triangular learning rates in Howard &amp; Ruder 2018)</li> <li>Gradual fine-tuning inspired by the work of Felbo et al., 2017</li> <li>Gradual fine-tuning based on the work of Howard &amp; Ruder 2018</li> </ol> <p>Currently fine-tunning is only supported without a fully connected head, i.e. if <code>deephead=None</code>. In addition, <code>Felbo</code> and <code>Howard</code> routines only applied, of course, to the <code>deeptabular</code>, <code>deeptext</code> and <code>deepimage</code> models. The <code>wide</code> component can also be fine-tuned, but only in an \"all at once\" mode.</p>"},{"location":"examples/06_finetune_and_warmup.html#fine-tune-or-warm-up-all-at-once","title":"Fine-tune or warm-up all at once\u00b6","text":"<p>Here, the model components will be trained for <code>finetune_epochs</code> using a triangular one-cycle learning rate (slanted triangular learning rate) ranging from <code>finetune_max_lr/10</code> to <code>finetune_max_lr</code> (default is 0.01). 10% of the training steps are used to increase the learning rate which then decreases for the remaining 90%.</p> <p>Here all trainable layers are fine-tuned.</p> <p>Let's have a look to one example.</p>"},{"location":"examples/06_finetune_and_warmup.html#fine-tune-gradually-the-felbo-and-the-howard-routines","title":"Fine-tune Gradually: The \"felbo\"  and the \"howard\" routines\u00b6","text":"<p>The Felbo routine can be illustrated as follows:</p> <p> </p> <p>Figure 1. The figure can be described as follows: fine-tune (or train) the last layer for one epoch using a one cycle triangular learning rate. Then fine-tune the next deeper layer for one epoch, with a learning rate that is a factor of 2.5 lower than the previous learning rate (the 2.5 factor is fixed) while freezing the already warmed up layer(s). Repeat untill all individual layers are warmed. Then warm one last epoch with all warmed layers trainable. The vanishing color gradient in the figure attempts to illustrate the decreasing learning rate.</p> <p>Note that this is not identical to the Fine-Tunning routine described in Felbo et al, 2017, this is why I used the word 'inspired'.</p> <p>The Howard routine can be illustrated as follows:</p> <p> </p> <p>Figure 2. The figure can be described as follows: fine-tune (or train) the last layer for one epoch using a one cycle triangular learning rate. Then fine-tune the next deeper layer for one epoch, with a learning rate that is a factor of 2.5 lower than the previous learning rate (the 2.5 factor is fixed) while keeping the already warmed up layer(s) trainable. Repeat. The vanishing color gradient in the figure attempts to illustrate the decreasing learning rate.</p> <p>Note that I write \"fine-tune (or train) the last layer for one epoch [...]\". However, in practice the user will have to specify the order of the layers to be fine-tuned. This is another reason why I wrote that the fine-tune routines I have implemented are inspired by the work of Felbo and Howard and not identical to their implemenations.</p> <p>The <code>felbo</code> and <code>howard</code> routines can be accessed with via the <code>fine-tune</code> parameters.</p>"},{"location":"examples/07_custom_components.html","title":"07_custom_components","text":"In\u00a0[1]: Copied! <pre>import numpy as np\nimport pandas as pd\nimport os\nimport torch\n\nfrom torch import Tensor\nfrom pytorch_widedeep import Trainer\nfrom pytorch_widedeep.preprocessing import (\n    WidePreprocessor,\n    TabPreprocessor,\n    TextPreprocessor,\n    ImagePreprocessor,\n)\nfrom pytorch_widedeep.models import (\n    Wide,\n    TabMlp,\n    Vision,\n    BasicRNN,\n    WideDeep,\n)\nfrom pytorch_widedeep.losses import RMSELoss\nfrom pytorch_widedeep.initializers import *\nfrom pytorch_widedeep.callbacks import *\nfrom pytorch_widedeep.datasets import load_adult\n</pre> import numpy as np import pandas as pd import os import torch  from torch import Tensor from pytorch_widedeep import Trainer from pytorch_widedeep.preprocessing import (     WidePreprocessor,     TabPreprocessor,     TextPreprocessor,     ImagePreprocessor, ) from pytorch_widedeep.models import (     Wide,     TabMlp,     Vision,     BasicRNN,     WideDeep, ) from pytorch_widedeep.losses import RMSELoss from pytorch_widedeep.initializers import * from pytorch_widedeep.callbacks import * from pytorch_widedeep.datasets import load_adult <pre>/Users/javierrodriguezzaurin/.pyenv/versions/3.10.13/envs/widedeep310/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n  from .autonotebook import tqdm as notebook_tqdm\n</pre> In\u00a0[2]: Copied! <pre>df = pd.read_csv(\"../tmp_data/airbnb/airbnb_sample.csv\")\ndf.head()\n</pre> df = pd.read_csv(\"../tmp_data/airbnb/airbnb_sample.csv\") df.head() Out[2]: id host_id description host_listings_count host_identity_verified neighbourhood_cleansed latitude longitude is_location_exact property_type ... amenity_wide_entrance amenity_wide_entrance_for_guests amenity_wide_entryway amenity_wide_hallways amenity_wifi amenity_window_guards amenity_wine_cooler security_deposit extra_people yield 0 13913.jpg 54730 My bright double bedroom with a large window h... 4.0 f Islington 51.56802 -0.11121 t apartment ... 1 0 0 0 1 0 0 100.0 15.0 12.00 1 15400.jpg 60302 Lots of windows and light.  St Luke's Gardens ... 1.0 t Kensington and Chelsea 51.48796 -0.16898 t apartment ... 0 0 0 0 1 0 0 150.0 0.0 109.50 2 17402.jpg 67564 Open from June 2018 after a 3-year break, we a... 19.0 t Westminster 51.52098 -0.14002 t apartment ... 0 0 0 0 1 0 0 350.0 10.0 149.65 3 24328.jpg 41759 Artist house, bright high ceiling rooms, priva... 2.0 t Wandsworth 51.47298 -0.16376 t other ... 0 0 0 0 1 0 0 250.0 0.0 215.60 4 25023.jpg 102813 Large, all comforts, 2-bed flat; first floor; ... 1.0 f Wandsworth 51.44687 -0.21874 t apartment ... 0 0 0 0 1 0 0 250.0 11.0 79.35 <p>5 rows \u00d7 223 columns</p> In\u00a0[3]: Copied! <pre># There are a number of columns that are already binary. Therefore, no need to one hot encode them\ncrossed_cols = [(\"property_type\", \"room_type\")]\nalready_dummies = [c for c in df.columns if \"amenity\" in c] + [\"has_house_rules\"]\nwide_cols = [\n    \"is_location_exact\",\n    \"property_type\",\n    \"room_type\",\n    \"host_gender\",\n    \"instant_bookable\",\n] + already_dummies\n\ncat_embed_cols = [(c, 16) for c in df.columns if \"catg\" in c] + [\n    (\"neighbourhood_cleansed\", 64),\n    (\"cancellation_policy\", 16),\n]\ncontinuous_cols = [\"latitude\", \"longitude\", \"security_deposit\", \"extra_people\"]\n# it does not make sense to standarised Latitude and Longitude\nalready_standard = [\"latitude\", \"longitude\"]\n\n# text and image colnames\ntext_col = \"description\"\nimg_col = \"id\"\n\n# path to pretrained word embeddings and the images\nword_vectors_path = \"../tmp_data/glove.6B/glove.6B.100d.txt\"\nimg_path = \"../tmp_data/airbnb/property_picture\"\n\n# target\ntarget_col = \"yield\"\n</pre> # There are a number of columns that are already binary. Therefore, no need to one hot encode them crossed_cols = [(\"property_type\", \"room_type\")] already_dummies = [c for c in df.columns if \"amenity\" in c] + [\"has_house_rules\"] wide_cols = [     \"is_location_exact\",     \"property_type\",     \"room_type\",     \"host_gender\",     \"instant_bookable\", ] + already_dummies  cat_embed_cols = [(c, 16) for c in df.columns if \"catg\" in c] + [     (\"neighbourhood_cleansed\", 64),     (\"cancellation_policy\", 16), ] continuous_cols = [\"latitude\", \"longitude\", \"security_deposit\", \"extra_people\"] # it does not make sense to standarised Latitude and Longitude already_standard = [\"latitude\", \"longitude\"]  # text and image colnames text_col = \"description\" img_col = \"id\"  # path to pretrained word embeddings and the images word_vectors_path = \"../tmp_data/glove.6B/glove.6B.100d.txt\" img_path = \"../tmp_data/airbnb/property_picture\"  # target target_col = \"yield\" In\u00a0[4]: Copied! <pre>target = df[target_col].values\n</pre> target = df[target_col].values In\u00a0[5]: Copied! <pre>wide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols)\nX_wide = wide_preprocessor.fit_transform(df)\n\ntab_preprocessor = TabPreprocessor(\n    embed_cols=cat_embed_cols, continuous_cols=continuous_cols\n)\nX_tab = tab_preprocessor.fit_transform(df)\n\ntext_preprocessor = TextPreprocessor(\n    word_vectors_path=word_vectors_path, text_col=text_col\n)\nX_text = text_preprocessor.fit_transform(df)\n\nimage_processor = ImagePreprocessor(img_col=img_col, img_path=img_path)\nX_images = image_processor.fit_transform(df)\n</pre> wide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols) X_wide = wide_preprocessor.fit_transform(df)  tab_preprocessor = TabPreprocessor(     embed_cols=cat_embed_cols, continuous_cols=continuous_cols ) X_tab = tab_preprocessor.fit_transform(df)  text_preprocessor = TextPreprocessor(     word_vectors_path=word_vectors_path, text_col=text_col ) X_text = text_preprocessor.fit_transform(df)  image_processor = ImagePreprocessor(img_col=img_col, img_path=img_path) X_images = image_processor.fit_transform(df) <pre>/Users/javierrodriguezzaurin/Projects/pytorch-widedeep/pytorch_widedeep/preprocessing/tab_preprocessor.py:358: UserWarning: Continuous columns will not be normalised\n  warnings.warn(\"Continuous columns will not be normalised\")\n</pre> <pre>The vocabulary contains 2192 tokens\nIndexing word vectors...\nLoaded 400000 word vectors\nPreparing embeddings matrix...\n2175 words in the vocabulary had ../tmp_data/glove.6B/glove.6B.100d.txt vectors and appear more than 5 times\nReading Images from ../tmp_data/airbnb/property_picture\nResizing\n</pre> <pre>100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 1001/1001 [00:02&lt;00:00, 497.80it/s]\n</pre> <pre>Computing normalisation metrics\n</pre> <p>Now we are ready to build a wide and deep model. Three of the four components we will use are included in this package, and they will be combined with a custom <code>deeptext</code> component. Then the fit process will run with a custom loss function.</p> <p>Let's have a look</p> In\u00a0[6]: Copied! <pre># Linear model\nwide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1)\n\n# DeepDense: 2 Dense layers\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    cat_embed_dropout=0.1,\n    continuous_cols=continuous_cols,\n    mlp_hidden_dims=[128, 64],\n    mlp_dropout=0.1,\n)\n\n# Pretrained Resnet 18\nresnet = Vision(pretrained_model_name=\"resnet18\", n_trainable=0)\n</pre> # Linear model wide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1)  # DeepDense: 2 Dense layers tab_mlp = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     cat_embed_dropout=0.1,     continuous_cols=continuous_cols,     mlp_hidden_dims=[128, 64],     mlp_dropout=0.1, )  # Pretrained Resnet 18 resnet = Vision(pretrained_model_name=\"resnet18\", n_trainable=0) In\u00a0[7]: Copied! <pre>class MyDeepText(nn.Module):\n    def __init__(self, vocab_size, padding_idx=1, embed_dim=100, hidden_dim=64):\n        super(MyDeepText, self).__init__()\n\n        # word/token embeddings\n        self.word_embed = nn.Embedding(vocab_size, embed_dim, padding_idx=padding_idx)\n\n        # stack of RNNs\n        self.rnn = nn.GRU(\n            embed_dim,\n            hidden_dim,\n            num_layers=2,\n            bidirectional=True,\n            batch_first=True,\n        )\n\n        # Remember, this MUST be defined. If not WideDeep will through an error\n        self.output_dim = hidden_dim * 2\n\n    def forward(self, X):\n        embed = self.word_embed(X.long())\n        o, h = self.rnn(embed)\n        return torch.cat((h[-2], h[-1]), dim=1)\n</pre> class MyDeepText(nn.Module):     def __init__(self, vocab_size, padding_idx=1, embed_dim=100, hidden_dim=64):         super(MyDeepText, self).__init__()          # word/token embeddings         self.word_embed = nn.Embedding(vocab_size, embed_dim, padding_idx=padding_idx)          # stack of RNNs         self.rnn = nn.GRU(             embed_dim,             hidden_dim,             num_layers=2,             bidirectional=True,             batch_first=True,         )          # Remember, this MUST be defined. If not WideDeep will through an error         self.output_dim = hidden_dim * 2      def forward(self, X):         embed = self.word_embed(X.long())         o, h = self.rnn(embed)         return torch.cat((h[-2], h[-1]), dim=1) In\u00a0[8]: Copied! <pre>mydeeptext = MyDeepText(vocab_size=len(text_preprocessor.vocab.itos))\n</pre> mydeeptext = MyDeepText(vocab_size=len(text_preprocessor.vocab.itos)) In\u00a0[9]: Copied! <pre>model = WideDeep(wide=wide, deeptabular=tab_mlp, deeptext=mydeeptext, deepimage=resnet)\n</pre> model = WideDeep(wide=wide, deeptabular=tab_mlp, deeptext=mydeeptext, deepimage=resnet) In\u00a0[10]: Copied! <pre>class RMSELoss(nn.Module):\n    def __init__(self):\n        \"\"\"root mean squared error\"\"\"\n        super().__init__()\n        self.mse = nn.MSELoss()\n\n    def forward(self, input: Tensor, target: Tensor) -&gt; Tensor:\n        return torch.sqrt(self.mse(input, target))\n</pre> class RMSELoss(nn.Module):     def __init__(self):         \"\"\"root mean squared error\"\"\"         super().__init__()         self.mse = nn.MSELoss()      def forward(self, input: Tensor, target: Tensor) -&gt; Tensor:         return torch.sqrt(self.mse(input, target)) <p>and now we just instantiate the <code>Trainer</code> as usual. Needless to say, but this runs with 1000 random observations, so loss and metric values are meaningless. This is just an example</p> In\u00a0[11]: Copied! <pre>trainer = Trainer(model, objective=\"regression\", custom_loss_function=RMSELoss())\n</pre> trainer = Trainer(model, objective=\"regression\", custom_loss_function=RMSELoss()) In\u00a0[12]: Copied! <pre>trainer.fit(\n    X_wide=X_wide,\n    X_tab=X_tab,\n    X_text=X_text,\n    X_img=X_images,\n    target=target,\n    n_epochs=1,\n    batch_size=32,\n    val_split=0.2,\n)\n</pre> trainer.fit(     X_wide=X_wide,     X_tab=X_tab,     X_text=X_text,     X_img=X_images,     target=target,     n_epochs=1,     batch_size=32,     val_split=0.2, ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 25/25 [00:23&lt;00:00,  1.07it/s, loss=126]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 7/7 [00:05&lt;00:00,  1.24it/s, loss=97.4]\n</pre> <p>In addition to model components and loss functions, we can also use custom callbacks or custom metrics. The former need to be of type <code>Callback</code> and the latter need to be of type <code>Metric</code>. See:</p> <pre>pytorch-widedeep.callbacks\n</pre> <p>and</p> <pre>pytorch-widedeep.metrics\n</pre> <p>For this example let me use the adult dataset. Again, we first prepare the data as usual</p> In\u00a0[13]: Copied! <pre>df = load_adult(as_frame=True)\ndf.head()\n</pre> df = load_adult(as_frame=True) df.head() Out[13]: age workclass fnlwgt education educational-num marital-status occupation relationship race gender capital-gain capital-loss hours-per-week native-country income 0 25 Private 226802 11th 7 Never-married Machine-op-inspct Own-child Black Male 0 0 40 United-States &lt;=50K 1 38 Private 89814 HS-grad 9 Married-civ-spouse Farming-fishing Husband White Male 0 0 50 United-States &lt;=50K 2 28 Local-gov 336951 Assoc-acdm 12 Married-civ-spouse Protective-serv Husband White Male 0 0 40 United-States &gt;50K 3 44 Private 160323 Some-college 10 Married-civ-spouse Machine-op-inspct Husband Black Male 7688 0 40 United-States &gt;50K 4 18 ? 103497 Some-college 10 Never-married ? Own-child White Female 0 0 30 United-States &lt;=50K In\u00a0[14]: Copied! <pre># For convenience, we'll replace '-' with '_'\ndf.columns = [c.replace(\"-\", \"_\") for c in df.columns]\n# binary target\ndf[\"income_label\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int)\ndf.drop(\"income\", axis=1, inplace=True)\ndf.head()\n</pre> # For convenience, we'll replace '-' with '_' df.columns = [c.replace(\"-\", \"_\") for c in df.columns] # binary target df[\"income_label\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int) df.drop(\"income\", axis=1, inplace=True) df.head() Out[14]: age workclass fnlwgt education educational_num marital_status occupation relationship race gender capital_gain capital_loss hours_per_week native_country income_label 0 25 Private 226802 11th 7 Never-married Machine-op-inspct Own-child Black Male 0 0 40 United-States 0 1 38 Private 89814 HS-grad 9 Married-civ-spouse Farming-fishing Husband White Male 0 0 50 United-States 0 2 28 Local-gov 336951 Assoc-acdm 12 Married-civ-spouse Protective-serv Husband White Male 0 0 40 United-States 1 3 44 Private 160323 Some-college 10 Married-civ-spouse Machine-op-inspct Husband Black Male 7688 0 40 United-States 1 4 18 ? 103497 Some-college 10 Never-married ? Own-child White Female 0 0 30 United-States 0 In\u00a0[15]: Copied! <pre># Define wide, crossed and deep tabular columns\nwide_cols = [\n    \"workclass\",\n    \"education\",\n    \"marital_status\",\n    \"occupation\",\n    \"relationship\",\n    \"race\",\n    \"gender\",\n    \"native_country\",\n]\ncrossed_cols = [(\"education\", \"occupation\"), (\"native_country\", \"occupation\")]\ncat_embed_cols = [\n    \"workclass\",\n    \"education\",\n    \"marital_status\",\n    \"occupation\",\n    \"relationship\",\n    \"race\",\n    \"gender\",\n    \"capital_gain\",\n    \"capital_loss\",\n    \"native_country\",\n]\ncontinuous_cols = [\"age\", \"hours_per_week\"]\ntarget_col = \"income_label\"\ntarget = df[target_col].values\n</pre> # Define wide, crossed and deep tabular columns wide_cols = [     \"workclass\",     \"education\",     \"marital_status\",     \"occupation\",     \"relationship\",     \"race\",     \"gender\",     \"native_country\", ] crossed_cols = [(\"education\", \"occupation\"), (\"native_country\", \"occupation\")] cat_embed_cols = [     \"workclass\",     \"education\",     \"marital_status\",     \"occupation\",     \"relationship\",     \"race\",     \"gender\",     \"capital_gain\",     \"capital_loss\",     \"native_country\", ] continuous_cols = [\"age\", \"hours_per_week\"] target_col = \"income_label\" target = df[target_col].values In\u00a0[16]: Copied! <pre># wide\nwide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols)\nX_wide = wide_preprocessor.fit_transform(df)\n\n# deeptabular\ntab_preprocessor = TabPreprocessor(\n    embed_cols=cat_embed_cols, continuous_cols=continuous_cols\n)\nX_tab = tab_preprocessor.fit_transform(df)\n</pre> # wide wide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols) X_wide = wide_preprocessor.fit_transform(df)  # deeptabular tab_preprocessor = TabPreprocessor(     embed_cols=cat_embed_cols, continuous_cols=continuous_cols ) X_tab = tab_preprocessor.fit_transform(df) <pre>/Users/javierrodriguezzaurin/Projects/pytorch-widedeep/pytorch_widedeep/preprocessing/tab_preprocessor.py:358: UserWarning: Continuous columns will not be normalised\n  warnings.warn(\"Continuous columns will not be normalised\")\n</pre> In\u00a0[17]: Copied! <pre>wide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1)\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=continuous_cols,\n    mlp_hidden_dims=[128, 64],\n    mlp_dropout=0.2,\n    mlp_activation=\"leaky_relu\",\n)\nmodel = WideDeep(wide=wide, deeptabular=tab_mlp)\n</pre> wide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1) tab_mlp = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     continuous_cols=continuous_cols,     mlp_hidden_dims=[128, 64],     mlp_dropout=0.2,     mlp_activation=\"leaky_relu\", ) model = WideDeep(wide=wide, deeptabular=tab_mlp) In\u00a0[18]: Copied! <pre>from pytorch_widedeep.metrics import Metric\n</pre> from pytorch_widedeep.metrics import Metric In\u00a0[19]: Copied! <pre>class Accuracy(Metric):\n    def __init__(self, top_k: int = 1):\n        super(Accuracy, self).__init__()\n\n        self.top_k = top_k\n        self.correct_count = 0\n        self.total_count = 0\n\n        # \u00a0metric name needs to be defined\n        self._name = \"acc\"\n\n    def reset(self):\n        self.correct_count = 0\n        self.total_count = 0\n\n    def __call__(self, y_pred: Tensor, y_true: Tensor) -&gt; np.ndarray:\n        num_classes = y_pred.size(1)\n\n        if num_classes == 1:\n            y_pred = y_pred.round()\n            y_true = y_true\n        elif num_classes &gt; 1:\n            y_pred = y_pred.topk(self.top_k, 1)[1]\n            y_true = y_true.view(-1, 1).expand_as(y_pred)\n\n        self.correct_count += y_pred.eq(y_true).sum().item()\n        self.total_count += len(y_pred)\n        accuracy = float(self.correct_count) / float(self.total_count)\n        return np.array(accuracy)\n</pre> class Accuracy(Metric):     def __init__(self, top_k: int = 1):         super(Accuracy, self).__init__()          self.top_k = top_k         self.correct_count = 0         self.total_count = 0          # \u00a0metric name needs to be defined         self._name = \"acc\"      def reset(self):         self.correct_count = 0         self.total_count = 0      def __call__(self, y_pred: Tensor, y_true: Tensor) -&gt; np.ndarray:         num_classes = y_pred.size(1)          if num_classes == 1:             y_pred = y_pred.round()             y_true = y_true         elif num_classes &gt; 1:             y_pred = y_pred.topk(self.top_k, 1)[1]             y_true = y_true.view(-1, 1).expand_as(y_pred)          self.correct_count += y_pred.eq(y_true).sum().item()         self.total_count += len(y_pred)         accuracy = float(self.correct_count) / float(self.total_count)         return np.array(accuracy) In\u00a0[20]: Copied! <pre># have a look to the class\nfrom pytorch_widedeep.callbacks import Callback\n</pre> # have a look to the class from pytorch_widedeep.callbacks import Callback In\u00a0[21]: Copied! <pre>class SillyCallback(Callback):\n    def on_train_begin(self, logs=None):\n        # recordings will be the trainer object attributes\n        self.trainer.silly_callback = {}\n\n        self.trainer.silly_callback[\"beginning\"] = []\n        self.trainer.silly_callback[\"end\"] = []\n\n    def on_epoch_begin(self, epoch, logs=None):\n        self.trainer.silly_callback[\"beginning\"].append(epoch + 1)\n\n    def on_epoch_end(self, epoch, logs=None, metric=None):\n        self.trainer.silly_callback[\"end\"].append(epoch + 1)\n</pre> class SillyCallback(Callback):     def on_train_begin(self, logs=None):         # recordings will be the trainer object attributes         self.trainer.silly_callback = {}          self.trainer.silly_callback[\"beginning\"] = []         self.trainer.silly_callback[\"end\"] = []      def on_epoch_begin(self, epoch, logs=None):         self.trainer.silly_callback[\"beginning\"].append(epoch + 1)      def on_epoch_end(self, epoch, logs=None, metric=None):         self.trainer.silly_callback[\"end\"].append(epoch + 1) <p>and now, as usual:</p> In\u00a0[22]: Copied! <pre>trainer = Trainer(\n    model, objective=\"binary\", metrics=[Accuracy], callbacks=[SillyCallback]\n)\n</pre> trainer = Trainer(     model, objective=\"binary\", metrics=[Accuracy], callbacks=[SillyCallback] ) In\u00a0[23]: Copied! <pre>trainer.fit(\n    X_wide=X_wide, X_tab=X_tab, target=target, n_epochs=5, batch_size=64, val_split=0.2\n)\n</pre> trainer.fit(     X_wide=X_wide, X_tab=X_tab, target=target, n_epochs=5, batch_size=64, val_split=0.2 ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 611/611 [00:06&lt;00:00, 94.39it/s, loss=0.411, metrics={'acc': 0.814}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 121.91it/s, loss=0.327, metrics={'acc': 0.8449}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 611/611 [00:07&lt;00:00, 85.39it/s, loss=0.324, metrics={'acc': 0.8495}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 88.68it/s, loss=0.298, metrics={'acc': 0.8612}]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 611/611 [00:08&lt;00:00, 74.35it/s, loss=0.302, metrics={'acc': 0.8593}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 100.51it/s, loss=0.29, metrics={'acc': 0.8665}]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 611/611 [00:08&lt;00:00, 73.83it/s, loss=0.292, metrics={'acc': 0.8637}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 105.98it/s, loss=0.286, metrics={'acc': 0.8695}]\nepoch 5: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 611/611 [00:08&lt;00:00, 72.15it/s, loss=0.286, metrics={'acc': 0.866}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 92.27it/s, loss=0.284, metrics={'acc': 0.8698}]\n</pre> In\u00a0[24]: Copied! <pre>trainer.silly_callback\n</pre> trainer.silly_callback Out[24]: <pre>{'beginning': [1, 2, 3, 4, 5], 'end': [1, 2, 3, 4, 5]}</pre>"},{"location":"examples/07_custom_components.html#custom-components","title":"Custom components\u00b6","text":"<p>As I mentioned earlier in the example notebooks, and also in the <code>README</code>, it is possible to customise almost every component in <code>pytorch-widedeep</code>.</p> <p>Let's now go through a couple of simple examples to illustrate how that could be done.</p> <p>First let's load and process the data \"as usual\", let's start with a regression and the airbnb dataset.</p>"},{"location":"examples/07_custom_components.html#custom-deeptext","title":"Custom <code>deeptext</code>\u00b6","text":"<p>Standard Pytorch model</p>"},{"location":"examples/07_custom_components.html#custom-loss-function","title":"Custom loss function\u00b6","text":"<p>Loss functions must simply inherit pytorch's <code>nn.Module</code>. For example, let's say we want to use <code>RMSE</code> (note that this is already available in the package, but I will pass it here as a custom loss for illustration purposes)</p>"},{"location":"examples/07_custom_components.html#custom-metric","title":"Custom metric\u00b6","text":"<p>Let's say we want to use our own accuracy metric (again, this is already available in the package, but I will pass it here as a custom loss for illustration purposes).</p> <p>This could be done as:</p>"},{"location":"examples/07_custom_components.html#custom-callback","title":"Custom Callback\u00b6","text":"<p>Let's code a callback that records the current epoch at the beginning and the end of each epoch (silly, but you know, this is just an example)</p>"},{"location":"examples/08_custom_dataLoader_imbalanced_dataset.html","title":"08_custom_dataLoader_imbalanced_dataset","text":"<ul> <li>In this notebook we will use the higly imbalanced Protein Homology Dataset from KDD cup 2004</li> </ul> <pre><code>* The first element of each line is a BLOCK ID that denotes to which native sequence this example belongs. There is a unique BLOCK ID for each native sequence. BLOCK IDs are integers running from 1 to 303 (one for each native sequence, i.e. for each query). BLOCK IDs were assigned before the blocks were split into the train and test sets, so they do not run consecutively in either file.\n* The second element of each line is an EXAMPLE ID that uniquely describes the example. You will need this EXAMPLE ID and the BLOCK ID when you submit results.\n* The third element is the class of the example. Proteins that are homologous to the native sequence are denoted by 1, non-homologous proteins (i.e. decoys) by 0. Test examples have a \"?\" in this position.\n* All following elements are feature values. There are 74 feature values in each line. The features describe the match (e.g. the score of a sequence alignment) between the native protein sequence and the sequence that is tested for homology.\n</code></pre> In\u00a0[1]: Copied! <pre>import numpy as np\nimport pandas as pd\nimport torch\nfrom torch.optim import SGD, lr_scheduler\n\nfrom pytorch_widedeep import Trainer\nfrom pytorch_widedeep.preprocessing import TabPreprocessor\nfrom pytorch_widedeep.models import TabMlp, WideDeep\nfrom pytorch_widedeep.dataloaders import DataLoaderImbalanced, DataLoaderDefault\nfrom torchmetrics import F1Score as F1_torchmetrics\nfrom torchmetrics import Accuracy as Accuracy_torchmetrics\nfrom torchmetrics import Precision as Precision_torchmetrics\nfrom torchmetrics import Recall as Recall_torchmetrics\nfrom pytorch_widedeep.metrics import Accuracy, Recall, Precision, F1Score, R2Score\nfrom pytorch_widedeep.initializers import XavierNormal\nfrom pytorch_widedeep.datasets import load_bio_kdd04\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import classification_report\n\nimport time\nimport datetime\n\nimport warnings\n\nwarnings.filterwarnings(\"ignore\", category=DeprecationWarning)\n\n# increase displayed columns in jupyter notebook\npd.set_option(\"display.max_columns\", 200)\npd.set_option(\"display.max_rows\", 300)\n</pre> import numpy as np import pandas as pd import torch from torch.optim import SGD, lr_scheduler  from pytorch_widedeep import Trainer from pytorch_widedeep.preprocessing import TabPreprocessor from pytorch_widedeep.models import TabMlp, WideDeep from pytorch_widedeep.dataloaders import DataLoaderImbalanced, DataLoaderDefault from torchmetrics import F1Score as F1_torchmetrics from torchmetrics import Accuracy as Accuracy_torchmetrics from torchmetrics import Precision as Precision_torchmetrics from torchmetrics import Recall as Recall_torchmetrics from pytorch_widedeep.metrics import Accuracy, Recall, Precision, F1Score, R2Score from pytorch_widedeep.initializers import XavierNormal from pytorch_widedeep.datasets import load_bio_kdd04  from sklearn.model_selection import train_test_split from sklearn.metrics import classification_report  import time import datetime  import warnings  warnings.filterwarnings(\"ignore\", category=DeprecationWarning)  # increase displayed columns in jupyter notebook pd.set_option(\"display.max_columns\", 200) pd.set_option(\"display.max_rows\", 300) <pre>/Users/javierrodriguezzaurin/.pyenv/versions/3.10.13/envs/widedeep310/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n  from .autonotebook import tqdm as notebook_tqdm\n</pre> In\u00a0[2]: Copied! <pre>df = load_bio_kdd04(as_frame=True)\n# drop columns we won't need in this example\ndf.drop(columns=[\"EXAMPLE_ID\", \"BLOCK_ID\"], inplace=True)\n\ndf_train, df_valid = train_test_split(\n    df, test_size=0.2, stratify=df[\"target\"], random_state=1\n)\ndf_valid, df_test = train_test_split(\n    df_valid, test_size=0.5, stratify=df_valid[\"target\"], random_state=1\n)\n\ncontinuous_cols = df.drop(columns=[\"target\"]).columns.values.tolist()\n</pre> df = load_bio_kdd04(as_frame=True) # drop columns we won't need in this example df.drop(columns=[\"EXAMPLE_ID\", \"BLOCK_ID\"], inplace=True)  df_train, df_valid = train_test_split(     df, test_size=0.2, stratify=df[\"target\"], random_state=1 ) df_valid, df_test = train_test_split(     df_valid, test_size=0.5, stratify=df_valid[\"target\"], random_state=1 )  continuous_cols = df.drop(columns=[\"target\"]).columns.values.tolist() In\u00a0[3]: Copied! <pre># deeptabular\ntab_preprocessor = TabPreprocessor(continuous_cols=continuous_cols, scale=True)\nX_tab_train = tab_preprocessor.fit_transform(df_train)\nX_tab_valid = tab_preprocessor.transform(df_valid)\nX_tab_test = tab_preprocessor.transform(df_test)\n\n# target\ny_train = df_train[\"target\"].values\ny_valid = df_valid[\"target\"].values\ny_test = df_test[\"target\"].values\n</pre> # deeptabular tab_preprocessor = TabPreprocessor(continuous_cols=continuous_cols, scale=True) X_tab_train = tab_preprocessor.fit_transform(df_train) X_tab_valid = tab_preprocessor.transform(df_valid) X_tab_test = tab_preprocessor.transform(df_test)  # target y_train = df_train[\"target\"].values y_valid = df_valid[\"target\"].values y_test = df_test[\"target\"].values In\u00a0[4]: Copied! <pre># Define the model\ninput_layer = len(tab_preprocessor.continuous_cols)\noutput_layer = 1\nhidden_layers = np.linspace(\n    input_layer * 2, output_layer, 5, endpoint=False, dtype=int\n).tolist()\n\ndeeptabular = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    continuous_cols=tab_preprocessor.continuous_cols,\n    mlp_hidden_dims=hidden_layers,\n)\nmodel = WideDeep(deeptabular=deeptabular)\nmodel\n</pre> # Define the model input_layer = len(tab_preprocessor.continuous_cols) output_layer = 1 hidden_layers = np.linspace(     input_layer * 2, output_layer, 5, endpoint=False, dtype=int ).tolist()  deeptabular = TabMlp(     column_idx=tab_preprocessor.column_idx,     continuous_cols=tab_preprocessor.continuous_cols,     mlp_hidden_dims=hidden_layers, ) model = WideDeep(deeptabular=deeptabular) model Out[4]: <pre>WideDeep(\n  (deeptabular): Sequential(\n    (0): TabMlp(\n      (cont_norm): Identity()\n      (encoder): MLP(\n        (mlp): Sequential(\n          (dense_layer_0): Sequential(\n            (0): Linear(in_features=74, out_features=148, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_1): Sequential(\n            (0): Linear(in_features=148, out_features=118, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_2): Sequential(\n            (0): Linear(in_features=118, out_features=89, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_3): Sequential(\n            (0): Linear(in_features=89, out_features=59, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_4): Sequential(\n            (0): Linear(in_features=59, out_features=30, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=30, out_features=1, bias=True)\n  )\n)</pre> In\u00a0[5]: Copied! <pre># Metrics from pytorch-widedeep\naccuracy = Accuracy(top_k=2)\nprecision = Precision(average=False)\n\n# # Metrics from torchmetrics\n# accuracy = Accuracy_torchmetrics(average=None, num_classes=1)\n# precision = Precision_torchmetrics(average=\"micro\", num_classes=1)\n\n# Optimizers\ndeep_opt = SGD(model.deeptabular.parameters(), lr=0.1)\n\n# LR Scheduler\ndeep_sch = lr_scheduler.StepLR(deep_opt, step_size=3)\n\ntrainer = Trainer(\n    model,\n    objective=\"binary\",\n    lr_schedulers={\"deeptabular\": deep_sch},\n    initializers={\"deeptabular\": XavierNormal},\n    optimizers={\"deeptabular\": deep_opt},\n    metrics=[accuracy, precision],\n    verbose=1,\n)\n</pre> # Metrics from pytorch-widedeep accuracy = Accuracy(top_k=2) precision = Precision(average=False)  # # Metrics from torchmetrics # accuracy = Accuracy_torchmetrics(average=None, num_classes=1) # precision = Precision_torchmetrics(average=\"micro\", num_classes=1)  # Optimizers deep_opt = SGD(model.deeptabular.parameters(), lr=0.1)  # LR Scheduler deep_sch = lr_scheduler.StepLR(deep_opt, step_size=3)  trainer = Trainer(     model,     objective=\"binary\",     lr_schedulers={\"deeptabular\": deep_sch},     initializers={\"deeptabular\": XavierNormal},     optimizers={\"deeptabular\": deep_opt},     metrics=[accuracy, precision],     verbose=1, ) In\u00a0[6]: Copied! <pre>start = time.time()\ntrainer.fit(\n    X_train={\"X_tab\": X_tab_train, \"target\": y_train},\n    X_val={\"X_tab\": X_tab_valid, \"target\": y_valid},\n    n_epochs=1,\n    batch_size=32,\n    custom_dataloader=DataLoaderImbalanced,\n    oversample_mul=5,\n)\nprint(\n    \"Training time[s]: {}\".format(\n        datetime.timedelta(seconds=round(time.time() - start))\n    )\n)\n\npd.DataFrame(trainer.history)\n\ndf_pred = trainer.predict(X_tab=X_tab_test)\nprint(classification_report(df_test[\"target\"].to_list(), df_pred))\nprint(\"Actual predicted values:\\n{}\".format(np.unique(df_pred, return_counts=True)))\n</pre> start = time.time() trainer.fit(     X_train={\"X_tab\": X_tab_train, \"target\": y_train},     X_val={\"X_tab\": X_tab_valid, \"target\": y_valid},     n_epochs=1,     batch_size=32,     custom_dataloader=DataLoaderImbalanced,     oversample_mul=5, ) print(     \"Training time[s]: {}\".format(         datetime.timedelta(seconds=round(time.time() - start))     ) )  pd.DataFrame(trainer.history)  df_pred = trainer.predict(X_tab=X_tab_test) print(classification_report(df_test[\"target\"].to_list(), df_pred)) print(\"Actual predicted values:\\n{}\".format(np.unique(df_pred, return_counts=True))) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 325/325 [00:02&lt;00:00, 153.99it/s, loss=0.163, metrics={'acc': 0.9363, 'prec': [0.9358]}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 456/456 [00:02&lt;00:00, 205.93it/s, loss=0.1, metrics={'acc': 0.9501, 'prec': [0.1447]}]\n</pre> <pre>Training time[s]: 0:00:04\n</pre> <pre>predict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 456/456 [00:01&lt;00:00, 368.16it/s]\n</pre> <pre>              precision    recall  f1-score   support\n\n           0       1.00      0.95      0.97     14446\n           1       0.15      0.95      0.25       130\n\n    accuracy                           0.95     14576\n   macro avg       0.57      0.95      0.61     14576\nweighted avg       0.99      0.95      0.97     14576\n\nActual predicted values:\n(array([0, 1]), array([13736,   840]))\n</pre>"},{"location":"examples/08_custom_dataLoader_imbalanced_dataset.html#custom-dataloader-for-imbalanced-dataset","title":"Custom DataLoader for Imbalanced dataset\u00b6","text":""},{"location":"examples/08_custom_dataLoader_imbalanced_dataset.html#initial-imports","title":"Initial imports\u00b6","text":""},{"location":"examples/09_extracting_embeddings.html","title":"09_extracting_embeddings","text":"In\u00a0[1]: Copied! <pre>import numpy as np\nimport pandas as pd\nimport torch\n\nfrom pytorch_widedeep.preprocessing import TabPreprocessor\nfrom pytorch_widedeep.training import Trainer\nfrom pytorch_widedeep.models import FTTransformer, WideDeep\nfrom pytorch_widedeep.metrics import Accuracy\nfrom pytorch_widedeep import Tab2Vec\nfrom pytorch_widedeep.datasets import load_adult\n</pre> import numpy as np import pandas as pd import torch  from pytorch_widedeep.preprocessing import TabPreprocessor from pytorch_widedeep.training import Trainer from pytorch_widedeep.models import FTTransformer, WideDeep from pytorch_widedeep.metrics import Accuracy from pytorch_widedeep import Tab2Vec from pytorch_widedeep.datasets import load_adult In\u00a0[2]: Copied! <pre>df = load_adult(as_frame=True)\ndf.head()\n</pre> df = load_adult(as_frame=True) df.head() Out[2]: age workclass fnlwgt education educational-num marital-status occupation relationship race gender capital-gain capital-loss hours-per-week native-country income 0 25 Private 226802 11th 7 Never-married Machine-op-inspct Own-child Black Male 0 0 40 United-States &lt;=50K 1 38 Private 89814 HS-grad 9 Married-civ-spouse Farming-fishing Husband White Male 0 0 50 United-States &lt;=50K 2 28 Local-gov 336951 Assoc-acdm 12 Married-civ-spouse Protective-serv Husband White Male 0 0 40 United-States &gt;50K 3 44 Private 160323 Some-college 10 Married-civ-spouse Machine-op-inspct Husband Black Male 7688 0 40 United-States &gt;50K 4 18 ? 103497 Some-college 10 Never-married ? Own-child White Female 0 0 30 United-States &lt;=50K In\u00a0[3]: Copied! <pre># For convenience, we'll replace '-' with '_'\ndf.columns = [c.replace(\"-\", \"_\") for c in df.columns]\n# binary target\ndf[\"target\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int)\ndf.drop([\"income\", \"educational_num\"], axis=1, inplace=True)\n\ndf.head()\n</pre> # For convenience, we'll replace '-' with '_' df.columns = [c.replace(\"-\", \"_\") for c in df.columns] # binary target df[\"target\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int) df.drop([\"income\", \"educational_num\"], axis=1, inplace=True)  df.head() Out[3]: age workclass fnlwgt education marital_status occupation relationship race gender capital_gain capital_loss hours_per_week native_country target 0 25 Private 226802 11th Never-married Machine-op-inspct Own-child Black Male 0 0 40 United-States 0 1 38 Private 89814 HS-grad Married-civ-spouse Farming-fishing Husband White Male 0 0 50 United-States 0 2 28 Local-gov 336951 Assoc-acdm Married-civ-spouse Protective-serv Husband White Male 0 0 40 United-States 1 3 44 Private 160323 Some-college Married-civ-spouse Machine-op-inspct Husband Black Male 7688 0 40 United-States 1 4 18 ? 103497 Some-college Never-married ? Own-child White Female 0 0 30 United-States 0 In\u00a0[4]: Copied! <pre>cat_cols, cont_cols = [], []\nfor col in df.columns:\n    # 50 is just a random number I choose here for this example\n    if df[col].dtype == \"O\" or df[col].nunique() &lt; 50 and col != \"target\":\n        cat_cols.append(col)\n    elif col != \"target\":\n        cont_cols.append(col)\ntarget_col = \"target\"\n</pre> cat_cols, cont_cols = [], [] for col in df.columns:     # 50 is just a random number I choose here for this example     if df[col].dtype == \"O\" or df[col].nunique() &lt; 50 and col != \"target\":         cat_cols.append(col)     elif col != \"target\":         cont_cols.append(col) target_col = \"target\" In\u00a0[5]: Copied! <pre>target = df[target_col].values\n\ntab_preprocessor = TabPreprocessor(\n    embed_cols=cat_cols, continuous_cols=cont_cols, for_transformer=True\n)\nX_tab = tab_preprocessor.fit_transform(df)\n</pre> target = df[target_col].values  tab_preprocessor = TabPreprocessor(     embed_cols=cat_cols, continuous_cols=cont_cols, for_transformer=True ) X_tab = tab_preprocessor.fit_transform(df) <pre>/Users/javierrodriguezzaurin/Projects/pytorch-widedeep/pytorch_widedeep/preprocessing/tab_preprocessor.py:358: UserWarning: Continuous columns will not be normalised\n  warnings.warn(\"Continuous columns will not be normalised\")\n</pre> In\u00a0[6]: Copied! <pre>ft_transformer = FTTransformer(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=tab_preprocessor.continuous_cols,\n    embed_continuous_method=\"standard\",\n    n_blocks=2,\n    n_heads=4,\n    input_dim=16,\n)\n</pre> ft_transformer = FTTransformer(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     continuous_cols=tab_preprocessor.continuous_cols,     embed_continuous_method=\"standard\",     n_blocks=2,     n_heads=4,     input_dim=16, ) <pre>/Users/javierrodriguezzaurin/Projects/pytorch-widedeep/pytorch_widedeep/utils/general_utils.py:12: DeprecationWarning: The 'embed_continuous' parameter is deprecated and will be removed in the next release. Please use 'embed_continuous_method' instead See the documentation for more details.\n  return func(*args, **kwargs)\n</pre> In\u00a0[7]: Copied! <pre>model = WideDeep(deeptabular=ft_transformer)\ntrainer = Trainer(model, objective=\"binary\", metrics=[Accuracy])\ntrainer.fit(X_tab=X_tab, target=target, n_epochs=1, batch_size=256, val_split=0.2)\n</pre> model = WideDeep(deeptabular=ft_transformer) trainer = Trainer(model, objective=\"binary\", metrics=[Accuracy]) trainer.fit(X_tab=X_tab, target=target, n_epochs=1, batch_size=256, val_split=0.2) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:03&lt;00:00, 41.47it/s, loss=221, metrics={'acc': 0.686}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 39/39 [00:00&lt;00:00, 86.24it/s, loss=9.28, metrics={'acc': 0.76}]\n</pre> In\u00a0[8]: Copied! <pre>t2v = Tab2Vec(model=model, tab_preprocessor=tab_preprocessor)\n</pre> t2v = Tab2Vec(model=model, tab_preprocessor=tab_preprocessor) In\u00a0[9]: Copied! <pre># assuming is a test set with target col\nX_vec, y = t2v.transform(df.sample(100), target_col=\"target\")\n</pre> # assuming is a test set with target col X_vec, y = t2v.transform(df.sample(100), target_col=\"target\") In\u00a0[10]: Copied! <pre># X vec is the dataframe turned into the embeddings\nX_vec.shape\n</pre> # X vec is the dataframe turned into the embeddings X_vec.shape Out[10]: <pre>(100, 208)</pre> <p><code>208 = input_dim (16) * n_cols (13)</code></p> In\u00a0[11]: Copied! <pre># ...or if we don't have target col\nX_vec = t2v.transform(df.sample(100))\n</pre> # ...or if we don't have target col X_vec = t2v.transform(df.sample(100))"},{"location":"examples/09_extracting_embeddings.html#extracting-embeddings","title":"Extracting embeddings\u00b6","text":"<p>This notebook is a simple guide to extracting learned feature embeddings using Tab2Vec</p>"},{"location":"examples/10_3rd_party_integration-RayTune_WnB.html","title":"10_3rd_party_integration-RayTune_WnB","text":"In\u00a0[11]: Copied! <pre>from typing import Optional, Dict\nimport os\n\nimport numpy as np\nimport pandas as pd\nimport torch\nimport wandb\nfrom torch.optim import SGD, lr_scheduler\n\nfrom pytorch_widedeep import Trainer\nfrom pytorch_widedeep.preprocessing import TabPreprocessor\nfrom pytorch_widedeep.models import TabMlp, WideDeep\nfrom torchmetrics import F1Score as F1_torchmetrics\nfrom torchmetrics import Accuracy as Accuracy_torchmetrics\nfrom torchmetrics import Precision as Precision_torchmetrics\nfrom torchmetrics import Recall as Recall_torchmetrics\nfrom pytorch_widedeep.metrics import Accuracy, Recall, Precision, F1Score, R2Score\nfrom pytorch_widedeep.initializers import XavierNormal\nfrom pytorch_widedeep.callbacks import (\n    EarlyStopping,\n    ModelCheckpoint,\n    Callback,\n)\nfrom pytorch_widedeep.datasets import load_bio_kdd04\n\nfrom sklearn.model_selection import train_test_split\nimport warnings\n\nwarnings.filterwarnings(\"ignore\", category=DeprecationWarning)\n\nfrom ray import tune\nfrom ray.tune.schedulers import AsyncHyperBandScheduler\nfrom ray.tune import JupyterNotebookReporter\nfrom ray.air.integrations.wandb import WandbLoggerCallback\n\n# from ray.tune.integration.wandb import wandb_mixin\n\nimport tracemalloc\n\ntracemalloc.start()\n\n# increase displayed columns in jupyter notebook\npd.set_option(\"display.max_columns\", 200)\npd.set_option(\"display.max_rows\", 300)\n</pre> from typing import Optional, Dict import os  import numpy as np import pandas as pd import torch import wandb from torch.optim import SGD, lr_scheduler  from pytorch_widedeep import Trainer from pytorch_widedeep.preprocessing import TabPreprocessor from pytorch_widedeep.models import TabMlp, WideDeep from torchmetrics import F1Score as F1_torchmetrics from torchmetrics import Accuracy as Accuracy_torchmetrics from torchmetrics import Precision as Precision_torchmetrics from torchmetrics import Recall as Recall_torchmetrics from pytorch_widedeep.metrics import Accuracy, Recall, Precision, F1Score, R2Score from pytorch_widedeep.initializers import XavierNormal from pytorch_widedeep.callbacks import (     EarlyStopping,     ModelCheckpoint,     Callback, ) from pytorch_widedeep.datasets import load_bio_kdd04  from sklearn.model_selection import train_test_split import warnings  warnings.filterwarnings(\"ignore\", category=DeprecationWarning)  from ray import tune from ray.tune.schedulers import AsyncHyperBandScheduler from ray.tune import JupyterNotebookReporter from ray.air.integrations.wandb import WandbLoggerCallback  # from ray.tune.integration.wandb import wandb_mixin  import tracemalloc  tracemalloc.start()  # increase displayed columns in jupyter notebook pd.set_option(\"display.max_columns\", 200) pd.set_option(\"display.max_rows\", 300) In\u00a0[12]: Copied! <pre>class RayTuneReporter(Callback):\n    r\"\"\"Callback that allows reporting history and lr_history values to RayTune\n    during Hyperparameter tuning\n\n    Callbacks are passed as input parameters to the ``Trainer`` class. See\n    :class:`pytorch_widedeep.trainer.Trainer`\n\n    For examples see the examples folder at:\n\n        .. code-block:: bash\n\n            /examples/12_HyperParameter_tuning_w_RayTune.ipynb\n    \"\"\"\n\n    def on_epoch_end(\n        self, epoch: int, logs: Optional[Dict] = None, metric: Optional[float] = None\n    ):\n        report_dict = {}\n        for k, v in self.trainer.history.items():\n            report_dict.update({k: v[-1]})\n        if hasattr(self.trainer, \"lr_history\"):\n            for k, v in self.trainer.lr_history.items():\n                report_dict.update({k: v[-1]})\n        tune.report(report_dict)\n\n\nclass WnBReportBest(Callback):\n    r\"\"\"Callback that allows reporting best performance of a run to WnB\n    during Hyperparameter tuning. It is an adjusted pytorch_widedeep.callbacks.ModelCheckpoint\n    with added WnB and removed checkpoint saving.\n\n    Callbacks are passed as input parameters to the ``Trainer`` class.\n\n    Parameters\n    ----------\n    wb: obj\n        Weights&amp;Biases API interface to report single best result usable for\n        comparisson of multiple paramater combinations by, for example,\n        `parallel coordinates\n        &lt;https://docs.wandb.ai/ref/app/features/panels/parallel-coordinates&gt;`_.\n        E.g W&amp;B summary report `wandb.run.summary[\"best\"]`.\n    monitor: str, default=\"loss\"\n        quantity to monitor. Typically `'val_loss'` or metric name\n        (e.g. `'val_acc'`)\n    mode: str, default=\"auto\"\n        If ``save_best_only=True``, the decision to overwrite the current save\n        file is made based on either the maximization or the minimization of\n        the monitored quantity. For `'acc'`, this should be `'max'`, for\n        `'loss'` this should be `'min'`, etc. In `'auto'` mode, the\n        direction is automatically inferred from the name of the monitored\n        quantity.\n\n    \"\"\"\n\n    def __init__(\n        self,\n        wb: object,\n        monitor: str = \"val_loss\",\n        mode: str = \"auto\",\n    ):\n        super(WnBReportBest, self).__init__()\n\n        self.monitor = monitor\n        self.mode = mode\n        self.wb = wb\n\n        if self.mode not in [\"auto\", \"min\", \"max\"]:\n            warnings.warn(\n                \"WnBReportBest mode %s is unknown, \"\n                \"fallback to auto mode.\" % (self.mode),\n                RuntimeWarning,\n            )\n            self.mode = \"auto\"\n        if self.mode == \"min\":\n            self.monitor_op = np.less\n            self.best = np.Inf\n        elif self.mode == \"max\":\n            self.monitor_op = np.greater  # type: ignore[assignment]\n            self.best = -np.Inf\n        else:\n            if self._is_metric(self.monitor):\n                self.monitor_op = np.greater  # type: ignore[assignment]\n                self.best = -np.Inf\n            else:\n                self.monitor_op = np.less\n                self.best = np.Inf\n\n    def on_epoch_end(  # noqa: C901\n        self, epoch: int, logs: Optional[Dict] = None, metric: Optional[float] = None\n    ):\n        logs = logs or {}\n        current = logs.get(self.monitor)\n        if current is not None:\n            if self.monitor_op(current, self.best):\n                self.wb.run.summary[\"best\"] = current  # type: ignore[attr-defined]\n                self.best = current\n                self.best_epoch = epoch\n\n    @staticmethod\n    def _is_metric(monitor: str):\n        \"copied from pytorch_widedeep.callbacks\"\n        if any([s in monitor for s in [\"acc\", \"prec\", \"rec\", \"fscore\", \"f1\", \"f2\"]]):\n            return True\n        else:\n            return False\n</pre> class RayTuneReporter(Callback):     r\"\"\"Callback that allows reporting history and lr_history values to RayTune     during Hyperparameter tuning      Callbacks are passed as input parameters to the ``Trainer`` class. See     :class:`pytorch_widedeep.trainer.Trainer`      For examples see the examples folder at:          .. code-block:: bash              /examples/12_HyperParameter_tuning_w_RayTune.ipynb     \"\"\"      def on_epoch_end(         self, epoch: int, logs: Optional[Dict] = None, metric: Optional[float] = None     ):         report_dict = {}         for k, v in self.trainer.history.items():             report_dict.update({k: v[-1]})         if hasattr(self.trainer, \"lr_history\"):             for k, v in self.trainer.lr_history.items():                 report_dict.update({k: v[-1]})         tune.report(report_dict)   class WnBReportBest(Callback):     r\"\"\"Callback that allows reporting best performance of a run to WnB     during Hyperparameter tuning. It is an adjusted pytorch_widedeep.callbacks.ModelCheckpoint     with added WnB and removed checkpoint saving.      Callbacks are passed as input parameters to the ``Trainer`` class.      Parameters     ----------     wb: obj         Weights&amp;Biases API interface to report single best result usable for         comparisson of multiple paramater combinations by, for example,         `parallel coordinates         `_.         E.g W&amp;B summary report `wandb.run.summary[\"best\"]`.     monitor: str, default=\"loss\"         quantity to monitor. Typically `'val_loss'` or metric name         (e.g. `'val_acc'`)     mode: str, default=\"auto\"         If ``save_best_only=True``, the decision to overwrite the current save         file is made based on either the maximization or the minimization of         the monitored quantity. For `'acc'`, this should be `'max'`, for         `'loss'` this should be `'min'`, etc. In `'auto'` mode, the         direction is automatically inferred from the name of the monitored         quantity.      \"\"\"      def __init__(         self,         wb: object,         monitor: str = \"val_loss\",         mode: str = \"auto\",     ):         super(WnBReportBest, self).__init__()          self.monitor = monitor         self.mode = mode         self.wb = wb          if self.mode not in [\"auto\", \"min\", \"max\"]:             warnings.warn(                 \"WnBReportBest mode %s is unknown, \"                 \"fallback to auto mode.\" % (self.mode),                 RuntimeWarning,             )             self.mode = \"auto\"         if self.mode == \"min\":             self.monitor_op = np.less             self.best = np.Inf         elif self.mode == \"max\":             self.monitor_op = np.greater  # type: ignore[assignment]             self.best = -np.Inf         else:             if self._is_metric(self.monitor):                 self.monitor_op = np.greater  # type: ignore[assignment]                 self.best = -np.Inf             else:                 self.monitor_op = np.less                 self.best = np.Inf      def on_epoch_end(  # noqa: C901         self, epoch: int, logs: Optional[Dict] = None, metric: Optional[float] = None     ):         logs = logs or {}         current = logs.get(self.monitor)         if current is not None:             if self.monitor_op(current, self.best):                 self.wb.run.summary[\"best\"] = current  # type: ignore[attr-defined]                 self.best = current                 self.best_epoch = epoch      @staticmethod     def _is_metric(monitor: str):         \"copied from pytorch_widedeep.callbacks\"         if any([s in monitor for s in [\"acc\", \"prec\", \"rec\", \"fscore\", \"f1\", \"f2\"]]):             return True         else:             return False In\u00a0[13]: Copied! <pre>df = load_bio_kdd04(as_frame=True)\ndf.head()\n</pre> df = load_bio_kdd04(as_frame=True) df.head() Out[13]: EXAMPLE_ID BLOCK_ID target 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 0 279 261532 0 52.0 32.69 0.30 2.5 20.0 1256.8 -0.89 0.33 11.0 -55.0 267.2 0.52 0.05 -2.36 49.6 252.0 0.43 1.16 -2.06 -33.0 -123.2 1.60 -0.49 -6.06 65.0 296.1 -0.28 -0.26 -3.83 -22.6 -170.0 3.06 -1.05 -3.29 22.9 286.3 0.12 2.58 4.08 -33.0 -178.9 1.88 0.53 -7.0 -44.0 1987.0 -5.41 0.95 -4.0 -57.0 722.9 -3.26 -0.55 -7.5 125.5 1547.2 -0.36 1.12 9.0 -37.0 72.5 0.47 0.74 -11.0 -8.0 1595.1 -1.64 2.83 -2.0 -50.0 445.2 -0.35 0.26 0.76 1 279 261533 0 58.0 33.33 0.00 16.5 9.5 608.1 0.50 0.07 20.5 -52.5 521.6 -1.08 0.58 -0.02 -3.2 103.6 -0.95 0.23 -2.87 -25.9 -52.2 -0.21 0.87 -1.81 10.4 62.0 -0.28 -0.04 1.48 -17.6 -198.3 3.43 2.84 5.87 -16.9 72.6 -0.31 2.79 2.71 -33.5 -11.6 -1.11 4.01 5.0 -57.0 666.3 1.13 4.38 5.0 -64.0 39.3 1.07 -0.16 32.5 100.0 1893.7 -2.80 -0.22 2.5 -28.5 45.0 0.58 0.41 -19.0 -6.0 762.9 0.29 0.82 -3.0 -35.0 140.3 1.16 0.39 0.73 2 279 261534 0 77.0 27.27 -0.91 6.0 58.5 1623.6 -1.40 0.02 -6.5 -48.0 621.0 -1.20 0.14 -0.20 73.6 609.1 -0.44 -0.58 -0.04 -23.0 -27.4 -0.72 -1.04 -1.09 91.1 635.6 -0.88 0.24 0.59 -18.7 -7.2 -0.60 -2.82 -0.71 52.4 504.1 0.89 -0.67 -9.30 -20.8 -25.7 -0.77 -0.85 0.0 -20.0 2259.0 -0.94 1.15 -4.0 -44.0 -22.7 0.94 -0.98 -19.0 105.0 1267.9 1.03 1.27 11.0 -39.5 82.3 0.47 -0.19 -10.0 7.0 1491.8 0.32 -1.29 0.0 -34.0 658.2 -0.76 0.26 0.24 3 279 261535 0 41.0 27.91 -0.35 3.0 46.0 1921.6 -1.36 -0.47 -32.0 -51.5 560.9 -0.29 -0.10 -1.11 124.3 791.6 0.00 0.39 -1.85 -21.7 -44.9 -0.21 0.02 0.89 133.9 797.8 -0.08 1.06 -0.26 -16.4 -74.1 0.97 -0.80 -0.41 66.9 955.3 -1.90 1.28 -6.65 -28.1 47.5 -1.91 1.42 1.0 -30.0 1846.7 0.76 1.10 -4.0 -52.0 -53.9 1.71 -0.22 -12.0 97.5 1969.8 -1.70 0.16 -1.0 -32.5 255.9 -0.46 1.57 10.0 6.0 2047.7 -0.98 1.53 0.0 -49.0 554.2 -0.83 0.39 0.73 4 279 261536 0 50.0 28.00 -1.32 -9.0 12.0 464.8 0.88 0.19 8.0 -51.5 98.1 1.09 -0.33 -2.16 -3.9 102.7 0.39 -1.22 -3.39 -15.2 -42.2 -1.18 -1.11 -3.55 8.9 141.3 -0.16 -0.43 -4.15 -12.9 -13.4 -1.32 -0.98 -3.69 8.8 136.1 -0.30 4.13 1.89 -13.0 -18.7 -1.37 -0.93 0.0 -1.0 810.1 -2.29 6.72 1.0 -23.0 -29.7 0.58 -1.10 -18.5 33.5 206.8 1.84 -0.13 4.0 -29.0 30.1 0.80 -0.24 5.0 -14.0 479.5 0.68 -0.59 2.0 -36.0 -6.9 2.02 0.14 -0.23 In\u00a0[14]: Copied! <pre># imbalance of the classes\ndf[\"target\"].value_counts()\n</pre> # imbalance of the classes df[\"target\"].value_counts() Out[14]: <pre>target\n0    144455\n1      1296\nName: count, dtype: int64</pre> In\u00a0[15]: Copied! <pre># drop columns we won't need in this example\ndf.drop(columns=[\"EXAMPLE_ID\", \"BLOCK_ID\"], inplace=True)\n</pre> # drop columns we won't need in this example df.drop(columns=[\"EXAMPLE_ID\", \"BLOCK_ID\"], inplace=True) In\u00a0[16]: Copied! <pre>df_train, df_valid = train_test_split(\n    df, test_size=0.2, stratify=df[\"target\"], random_state=1\n)\ndf_valid, df_test = train_test_split(\n    df_valid, test_size=0.5, stratify=df_valid[\"target\"], random_state=1\n)\n</pre> df_train, df_valid = train_test_split(     df, test_size=0.2, stratify=df[\"target\"], random_state=1 ) df_valid, df_test = train_test_split(     df_valid, test_size=0.5, stratify=df_valid[\"target\"], random_state=1 ) In\u00a0[17]: Copied! <pre>continuous_cols = df.drop(columns=[\"target\"]).columns.values.tolist()\n</pre> continuous_cols = df.drop(columns=[\"target\"]).columns.values.tolist() In\u00a0[18]: Copied! <pre># deeptabular\ntab_preprocessor = TabPreprocessor(continuous_cols=continuous_cols, scale=True)\nX_tab_train = tab_preprocessor.fit_transform(df_train)\nX_tab_valid = tab_preprocessor.transform(df_valid)\nX_tab_test = tab_preprocessor.transform(df_test)\n\n# target\ny_train = df_train[\"target\"].values\ny_valid = df_valid[\"target\"].values\ny_test = df_test[\"target\"].values\n</pre> # deeptabular tab_preprocessor = TabPreprocessor(continuous_cols=continuous_cols, scale=True) X_tab_train = tab_preprocessor.fit_transform(df_train) X_tab_valid = tab_preprocessor.transform(df_valid) X_tab_test = tab_preprocessor.transform(df_test)  # target y_train = df_train[\"target\"].values y_valid = df_valid[\"target\"].values y_test = df_test[\"target\"].values In\u00a0[19]: Copied! <pre>input_layer = len(tab_preprocessor.continuous_cols)\noutput_layer = 1\nhidden_layers = np.linspace(\n    input_layer * 2, output_layer, 5, endpoint=False, dtype=int\n).tolist()\n</pre> input_layer = len(tab_preprocessor.continuous_cols) output_layer = 1 hidden_layers = np.linspace(     input_layer * 2, output_layer, 5, endpoint=False, dtype=int ).tolist() In\u00a0[20]: Copied! <pre>deeptabular = TabMlp(\n    mlp_hidden_dims=hidden_layers,\n    column_idx=tab_preprocessor.column_idx,\n    continuous_cols=tab_preprocessor.continuous_cols,\n)\nmodel = WideDeep(deeptabular=deeptabular)\nmodel\n</pre> deeptabular = TabMlp(     mlp_hidden_dims=hidden_layers,     column_idx=tab_preprocessor.column_idx,     continuous_cols=tab_preprocessor.continuous_cols, ) model = WideDeep(deeptabular=deeptabular) model Out[20]: <pre>WideDeep(\n  (deeptabular): Sequential(\n    (0): TabMlp(\n      (cont_norm): Identity()\n      (encoder): MLP(\n        (mlp): Sequential(\n          (dense_layer_0): Sequential(\n            (0): Linear(in_features=74, out_features=148, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_1): Sequential(\n            (0): Linear(in_features=148, out_features=118, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_2): Sequential(\n            (0): Linear(in_features=118, out_features=89, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_3): Sequential(\n            (0): Linear(in_features=89, out_features=59, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_4): Sequential(\n            (0): Linear(in_features=59, out_features=30, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=30, out_features=1, bias=True)\n  )\n)</pre> In\u00a0[24]: Copied! <pre># Metrics from torchmetrics\naccuracy = Accuracy_torchmetrics(average=None, num_classes=1, task=\"binary\")\nprecision = Precision_torchmetrics(average=\"micro\", num_classes=1, task=\"binary\")\nf1 = F1_torchmetrics(average=None, num_classes=1, task=\"binary\")\nrecall = Recall_torchmetrics(average=None, num_classes=1, task=\"binary\")\n</pre> # Metrics from torchmetrics accuracy = Accuracy_torchmetrics(average=None, num_classes=1, task=\"binary\") precision = Precision_torchmetrics(average=\"micro\", num_classes=1, task=\"binary\") f1 = F1_torchmetrics(average=None, num_classes=1, task=\"binary\") recall = Recall_torchmetrics(average=None, num_classes=1, task=\"binary\") <p>Note:</p> <p>Following cells includes usage of both <code>RayTuneReporter</code> and <code>WnBReportBest</code> callbacks. In case you want to use just <code>RayTuneReporter</code>, remove following:</p> <ul> <li>wandb from config</li> <li><code>WandbLoggerCallback</code></li> <li><code>WnBReportBest</code></li> <li><code>@wandb_mixin</code> decorator</li> </ul> <p>We do not see strong reason to use WnB without RayTune for a single paramater combination run, but it is possible:</p> <ul> <li>option01: define paramaters in config only for a single value <code>tune.grid_search([1000])</code> (single value RayTune run)</li> <li>option02: define WnB callback that reports currnet validation/training loss, metrics, etc. at the end of batch, ie. do not report to WnB at <code>epoch_end</code> as in <code>WnBReportBest</code> but at the <code>on_batch_end</code>, see <code>pytorch_widedeep.callbacks.Callback</code></li> </ul> In\u00a0[26]: Copied! <pre>config = {\n    \"batch_size\": tune.grid_search([1000, 5000]),\n    \"wandb\": {\n        \"project\": \"test\",\n        # \"api_key_file\": os.getcwd() + \"/wandb_api.key\",\n        \"api_key\": \"WNB_API_KEY\",\n    },\n}\n\n# Optimizers\ndeep_opt = SGD(model.deeptabular.parameters(), lr=0.1)\n# LR Scheduler\ndeep_sch = lr_scheduler.StepLR(deep_opt, step_size=3)\n\n\n@wandb_mixin\ndef training_function(config, X_train, X_val):\n    early_stopping = EarlyStopping()\n    model_checkpoint = ModelCheckpoint(save_best_only=True)\n    # Hyperparameters\n    batch_size = config[\"batch_size\"]\n    trainer = Trainer(\n        model,\n        objective=\"binary_focal_loss\",\n        callbacks=[\n            RayTuneReporter,\n            WnBReportBest(wb=wandb),\n            early_stopping,\n            model_checkpoint,\n        ],\n        lr_schedulers={\"deeptabular\": deep_sch},\n        initializers={\"deeptabular\": XavierNormal},\n        optimizers={\"deeptabular\": deep_opt},\n        metrics=[accuracy, precision, recall, f1],\n        verbose=0,\n    )\n\n    trainer.fit(X_train=X_train, X_val=X_val, n_epochs=5, batch_size=batch_size)\n\n\nX_train = {\"X_tab\": X_tab_train, \"target\": y_train}\nX_val = {\"X_tab\": X_tab_valid, \"target\": y_valid}\n\nasha_scheduler = AsyncHyperBandScheduler(\n    time_attr=\"training_iteration\",\n    metric=\"_metric/val_loss\",\n    mode=\"min\",\n    max_t=100,\n    grace_period=10,\n    reduction_factor=3,\n    brackets=1,\n)\n\nanalysis = tune.run(\n    tune.with_parameters(training_function, X_train=X_train, X_val=X_val),\n    resources_per_trial={\"cpu\": 1, \"gpu\": 0},\n    progress_reporter=JupyterNotebookReporter(overwrite=True),\n    scheduler=asha_scheduler,\n    config=config,\n    callbacks=[\n        WandbLoggerCallback(\n            project=config[\"wandb\"][\"project\"],\n            # api_key_file=config[\"wandb\"][\"api_key_file\"],\n            api_key=config[\"wandb\"][\"api_key\"],\n            log_config=True,\n        )\n    ],\n)\n</pre> config = {     \"batch_size\": tune.grid_search([1000, 5000]),     \"wandb\": {         \"project\": \"test\",         # \"api_key_file\": os.getcwd() + \"/wandb_api.key\",         \"api_key\": \"WNB_API_KEY\",     }, }  # Optimizers deep_opt = SGD(model.deeptabular.parameters(), lr=0.1) # LR Scheduler deep_sch = lr_scheduler.StepLR(deep_opt, step_size=3)   @wandb_mixin def training_function(config, X_train, X_val):     early_stopping = EarlyStopping()     model_checkpoint = ModelCheckpoint(save_best_only=True)     # Hyperparameters     batch_size = config[\"batch_size\"]     trainer = Trainer(         model,         objective=\"binary_focal_loss\",         callbacks=[             RayTuneReporter,             WnBReportBest(wb=wandb),             early_stopping,             model_checkpoint,         ],         lr_schedulers={\"deeptabular\": deep_sch},         initializers={\"deeptabular\": XavierNormal},         optimizers={\"deeptabular\": deep_opt},         metrics=[accuracy, precision, recall, f1],         verbose=0,     )      trainer.fit(X_train=X_train, X_val=X_val, n_epochs=5, batch_size=batch_size)   X_train = {\"X_tab\": X_tab_train, \"target\": y_train} X_val = {\"X_tab\": X_tab_valid, \"target\": y_valid}  asha_scheduler = AsyncHyperBandScheduler(     time_attr=\"training_iteration\",     metric=\"_metric/val_loss\",     mode=\"min\",     max_t=100,     grace_period=10,     reduction_factor=3,     brackets=1, )  analysis = tune.run(     tune.with_parameters(training_function, X_train=X_train, X_val=X_val),     resources_per_trial={\"cpu\": 1, \"gpu\": 0},     progress_reporter=JupyterNotebookReporter(overwrite=True),     scheduler=asha_scheduler,     config=config,     callbacks=[         WandbLoggerCallback(             project=config[\"wandb\"][\"project\"],             # api_key_file=config[\"wandb\"][\"api_key_file\"],             api_key=config[\"wandb\"][\"api_key\"],             log_config=True,         )     ], ) <pre>/Users/javierrodriguezzaurin/.pyenv/versions/3.10.13/lib/python3.10/tempfile.py:860: ResourceWarning: Implicitly cleaning up &lt;TemporaryDirectory '/var/folders/_2/lrjn1qn54c758tdtktr1bvkc0000gn/T/tmp60pfyl1kwandb'&gt;\n  _warnings.warn(warn_message, ResourceWarning)\n/Users/javierrodriguezzaurin/.pyenv/versions/3.10.13/lib/python3.10/tempfile.py:860: ResourceWarning: Implicitly cleaning up &lt;TemporaryDirectory '/var/folders/_2/lrjn1qn54c758tdtktr1bvkc0000gn/T/tmpnjv2rg1wwandb-artifacts'&gt;\n  _warnings.warn(warn_message, ResourceWarning)\n/Users/javierrodriguezzaurin/.pyenv/versions/3.10.13/lib/python3.10/tempfile.py:860: ResourceWarning: Implicitly cleaning up &lt;TemporaryDirectory '/var/folders/_2/lrjn1qn54c758tdtktr1bvkc0000gn/T/tmpgebu5k1kwandb-media'&gt;\n  _warnings.warn(warn_message, ResourceWarning)\n/Users/javierrodriguezzaurin/.pyenv/versions/3.10.13/lib/python3.10/tempfile.py:860: ResourceWarning: Implicitly cleaning up &lt;TemporaryDirectory '/var/folders/_2/lrjn1qn54c758tdtktr1bvkc0000gn/T/tmpxy9y2yriwandb-media'&gt;\n  _warnings.warn(warn_message, ResourceWarning)\n</pre> In\u00a0[14]: Copied! <pre>analysis.results\n</pre> analysis.results Out[14]: <pre>{'fc9a8_00000': {'_metric': {'train_loss': 0.006297602537127896,\n   'train_Accuracy': 0.9925042986869812,\n   'train_Precision': 0.9939393997192383,\n   'train_Recall': 0.15814851224422455,\n   'train_F1Score': 0.2728785574436188,\n   'val_loss': 0.005045663565397263,\n   'val_Accuracy': 0.9946483969688416,\n   'val_Precision': 1.0,\n   'val_Recall': 0.39534884691238403,\n   'val_F1Score': 0.5666667222976685},\n  'time_this_iter_s': 2.388202428817749,\n  'done': True,\n  'timesteps_total': None,\n  'episodes_total': None,\n  'training_iteration': 5,\n  'trial_id': 'fc9a8_00000',\n  'experiment_id': 'baad1d4f3d924b48b9ece1b9f26c80cc',\n  'date': '2022-07-31_14-06-51',\n  'timestamp': 1659276411,\n  'time_total_s': 12.656474113464355,\n  'pid': 1813,\n  'hostname': 'jupyter-5uperpalo',\n  'node_ip': '10.32.44.172',\n  'config': {'batch_size': 1000},\n  'time_since_restore': 12.656474113464355,\n  'timesteps_since_restore': 0,\n  'iterations_since_restore': 5,\n  'warmup_time': 0.8006253242492676,\n  'experiment_tag': '0_batch_size=1000'},\n 'fc9a8_00001': {'_metric': {'train_loss': 0.02519632239515583,\n   'train_Accuracy': 0.9910891652107239,\n   'train_Precision': 0.25,\n   'train_Recall': 0.0009643201483413577,\n   'train_F1Score': 0.0019212296465411782,\n   'val_loss': 0.02578434906899929,\n   'val_Accuracy': 0.9911492466926575,\n   'val_Precision': 0.0,\n   'val_Recall': 0.0,\n   'val_F1Score': 0.0},\n  'time_this_iter_s': 4.113586902618408,\n  'done': True,\n  'timesteps_total': None,\n  'episodes_total': None,\n  'training_iteration': 5,\n  'trial_id': 'fc9a8_00001',\n  'experiment_id': 'f2e54a6a5780429fbf0db0746853347e',\n  'date': '2022-07-31_14-06-56',\n  'timestamp': 1659276416,\n  'time_total_s': 12.926990509033203,\n  'pid': 1962,\n  'hostname': 'jupyter-5uperpalo',\n  'node_ip': '10.32.44.172',\n  'config': {'batch_size': 5000},\n  'time_since_restore': 12.926990509033203,\n  'timesteps_since_restore': 0,\n  'iterations_since_restore': 5,\n  'warmup_time': 0.9253025054931641,\n  'experiment_tag': '1_batch_size=5000'}}</pre> <p>Using Weights and Biases logging you can create parallel coordinates graphs that map parametr combinations to the best(lowest) loss achieved during the training of the networks</p> <p></p> <p>local visualization of raytune reults using tensorboard</p> In\u00a0[23]: Copied! <pre>%load_ext tensorboard\n%tensorboard --logdir ~/ray_results\n</pre> %load_ext tensorboard %tensorboard --logdir ~/ray_results"},{"location":"examples/10_3rd_party_integration-RayTune_WnB.html#3rd-party-integration-raytune-weights-biases","title":"3rd party integration - RayTune, Weights &amp; Biases\u00b6","text":"<p>This notebook provides guideline for integration of external library functions in the model training process through <code>Callback</code> objects, a popular concept of using objects as arguments for other objects.</p> <p>[DISCLAIMER]</p> <p>We show integration of RayTune (a hyperparameter tuning framework) and Weights &amp; Biases (ML projects experiment tracking and versioning solution) in the <code>pytorch_widedeep</code> model training process. We did not include <code>RayTuneReporter</code> and <code>WnBReportBest</code> in the library code to minimize the dependencies on other libraries that are not directly included in the model design and training.</p>"},{"location":"examples/10_3rd_party_integration-RayTune_WnB.html#initial-imports","title":"Initial imports\u00b6","text":""},{"location":"examples/10_3rd_party_integration-RayTune_WnB.html#preparing-the-data","title":"Preparing the data\u00b6","text":""},{"location":"examples/10_3rd_party_integration-RayTune_WnB.html#define-the-model","title":"Define the model\u00b6","text":""},{"location":"examples/11_auc_multiclass.html","title":"11_auc_multiclass","text":"In\u00a0[1]: Copied! <pre>import numpy as np\nimport pandas as pd\nfrom torch.optim import SGD, lr_scheduler\n\nfrom pytorch_widedeep import Trainer\nfrom pytorch_widedeep.preprocessing import TabPreprocessor\nfrom pytorch_widedeep.models import TabMlp, WideDeep\nfrom torchmetrics import AUROC\nfrom pytorch_widedeep.initializers import XavierNormal\nfrom pytorch_widedeep.datasets import load_ecoli\nfrom pytorch_widedeep.utils import LabelEncoder\n\nfrom sklearn.model_selection import train_test_split\n\n# increase displayed columns in jupyter notebook\npd.set_option(\"display.max_columns\", 200)\npd.set_option(\"display.max_rows\", 300)\n</pre> import numpy as np import pandas as pd from torch.optim import SGD, lr_scheduler  from pytorch_widedeep import Trainer from pytorch_widedeep.preprocessing import TabPreprocessor from pytorch_widedeep.models import TabMlp, WideDeep from torchmetrics import AUROC from pytorch_widedeep.initializers import XavierNormal from pytorch_widedeep.datasets import load_ecoli from pytorch_widedeep.utils import LabelEncoder  from sklearn.model_selection import train_test_split  # increase displayed columns in jupyter notebook pd.set_option(\"display.max_columns\", 200) pd.set_option(\"display.max_rows\", 300) In\u00a0[2]: Copied! <pre>df = load_ecoli(as_frame=True)\ndf.head()\n</pre> df = load_ecoli(as_frame=True) df.head() Out[2]: SequenceName mcg gvh lip chg aac alm1 alm2 class 0 AAT_ECOLI 0.49 0.29 0.48 0.5 0.56 0.24 0.35 cp 1 ACEA_ECOLI 0.07 0.40 0.48 0.5 0.54 0.35 0.44 cp 2 ACEK_ECOLI 0.56 0.40 0.48 0.5 0.49 0.37 0.46 cp 3 ACKA_ECOLI 0.59 0.49 0.48 0.5 0.52 0.45 0.36 cp 4 ADI_ECOLI 0.23 0.32 0.48 0.5 0.55 0.25 0.35 cp In\u00a0[3]: Copied! <pre># imbalance of the classes\ndf[\"class\"].value_counts()\n</pre> # imbalance of the classes df[\"class\"].value_counts() Out[3]: <pre>class\ncp     143\nim      77\npp      52\nimU     35\nom      20\nomL      5\nimS      2\nimL      2\nName: count, dtype: int64</pre> In\u00a0[4]: Copied! <pre>df = df.loc[~df[\"class\"].isin([\"omL\", \"imS\", \"imL\"])]\ndf.reset_index(inplace=True, drop=True)\n</pre> df = df.loc[~df[\"class\"].isin([\"omL\", \"imS\", \"imL\"])] df.reset_index(inplace=True, drop=True) In\u00a0[5]: Copied! <pre>encoder = LabelEncoder([\"class\"])\ndf_enc = encoder.fit_transform(df)\ndf_enc[\"class\"] = df_enc[\"class\"] - 1\n</pre> encoder = LabelEncoder([\"class\"]) df_enc = encoder.fit_transform(df) df_enc[\"class\"] = df_enc[\"class\"] - 1 In\u00a0[6]: Copied! <pre># drop columns we won't need in this example\ndf_enc = df_enc.drop(columns=[\"SequenceName\"])\n</pre> # drop columns we won't need in this example df_enc = df_enc.drop(columns=[\"SequenceName\"]) In\u00a0[7]: Copied! <pre>df_train, df_valid = train_test_split(\n    df_enc, test_size=0.2, stratify=df_enc[\"class\"], random_state=1\n)\ndf_valid, df_test = train_test_split(\n    df_valid, test_size=0.5, stratify=df_valid[\"class\"], random_state=1\n)\n</pre> df_train, df_valid = train_test_split(     df_enc, test_size=0.2, stratify=df_enc[\"class\"], random_state=1 ) df_valid, df_test = train_test_split(     df_valid, test_size=0.5, stratify=df_valid[\"class\"], random_state=1 ) In\u00a0[8]: Copied! <pre>continuous_cols = df_enc.drop(columns=[\"class\"]).columns.values.tolist()\n</pre> continuous_cols = df_enc.drop(columns=[\"class\"]).columns.values.tolist() In\u00a0[9]: Copied! <pre># deeptabular\ntab_preprocessor = TabPreprocessor(continuous_cols=continuous_cols, scale=True)\nX_tab_train = tab_preprocessor.fit_transform(df_train)\nX_tab_valid = tab_preprocessor.transform(df_valid)\nX_tab_test = tab_preprocessor.transform(df_test)\n\n# target\ny_train = df_train[\"class\"].values\ny_valid = df_valid[\"class\"].values\ny_test = df_test[\"class\"].values\n\nX_train = {\"X_tab\": X_tab_train, \"target\": y_train}\nX_val = {\"X_tab\": X_tab_valid, \"target\": y_valid}\n</pre> # deeptabular tab_preprocessor = TabPreprocessor(continuous_cols=continuous_cols, scale=True) X_tab_train = tab_preprocessor.fit_transform(df_train) X_tab_valid = tab_preprocessor.transform(df_valid) X_tab_test = tab_preprocessor.transform(df_test)  # target y_train = df_train[\"class\"].values y_valid = df_valid[\"class\"].values y_test = df_test[\"class\"].values  X_train = {\"X_tab\": X_tab_train, \"target\": y_train} X_val = {\"X_tab\": X_tab_valid, \"target\": y_valid} <pre>/Users/javierrodriguezzaurin/Projects/pytorch-widedeep/pytorch_widedeep/preprocessing/tab_preprocessor.py:295: DeprecationWarning: 'scale' and 'already_standard' will be deprecated in the next release. Please use 'cols_to_scale' instead\n  self._check_inputs(cat_embed_cols)\n</pre> In\u00a0[10]: Copied! <pre>deeptabular = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    continuous_cols=tab_preprocessor.continuous_cols,\n)\nmodel = WideDeep(deeptabular=deeptabular, pred_dim=df_enc[\"class\"].nunique())\nmodel\n</pre> deeptabular = TabMlp(     column_idx=tab_preprocessor.column_idx,     continuous_cols=tab_preprocessor.continuous_cols, ) model = WideDeep(deeptabular=deeptabular, pred_dim=df_enc[\"class\"].nunique()) model Out[10]: <pre>WideDeep(\n  (deeptabular): Sequential(\n    (0): TabMlp(\n      (cont_norm): Identity()\n      (encoder): MLP(\n        (mlp): Sequential(\n          (dense_layer_0): Sequential(\n            (0): Linear(in_features=7, out_features=200, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_1): Sequential(\n            (0): Linear(in_features=200, out_features=100, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=100, out_features=5, bias=True)\n  )\n)</pre> In\u00a0[11]: Copied! <pre>auroc = AUROC(num_classes=df_enc[\"class\"].nunique(), task=\"multiclass\")\n</pre> auroc = AUROC(num_classes=df_enc[\"class\"].nunique(), task=\"multiclass\") In\u00a0[12]: Copied! <pre># Optimizers\ndeep_opt = SGD(model.deeptabular.parameters(), lr=0.1)\n# LR Scheduler\ndeep_sch = lr_scheduler.StepLR(deep_opt, step_size=3)\n# Hyperparameters\ntrainer = Trainer(\n    model,\n    objective=\"multiclass_focal_loss\",\n    lr_schedulers={\"deeptabular\": deep_sch},\n    initializers={\"deeptabular\": XavierNormal},\n    optimizers={\"deeptabular\": deep_opt},\n    metrics=[auroc],\n)\n\ntrainer.fit(X_train=X_train, X_val=X_val, n_epochs=5, batch_size=50)\n</pre> # Optimizers deep_opt = SGD(model.deeptabular.parameters(), lr=0.1) # LR Scheduler deep_sch = lr_scheduler.StepLR(deep_opt, step_size=3) # Hyperparameters trainer = Trainer(     model,     objective=\"multiclass_focal_loss\",     lr_schedulers={\"deeptabular\": deep_sch},     initializers={\"deeptabular\": XavierNormal},     optimizers={\"deeptabular\": deep_opt},     metrics=[auroc], )  trainer.fit(X_train=X_train, X_val=X_val, n_epochs=5, batch_size=50) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 6/6 [00:00&lt;00:00, 54.59it/s, loss=0.109, metrics={'MulticlassAUROC': 0.314}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 1/1 [00:00&lt;00:00, 98.35it/s, loss=0.105, metrics={'MulticlassAUROC': 0.2558}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 6/6 [00:00&lt;00:00, 91.55it/s, loss=0.105, metrics={'MulticlassAUROC': 0.3546}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 1/1 [00:00&lt;00:00, 111.68it/s, loss=0.101, metrics={'MulticlassAUROC': 0.2737}]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 6/6 [00:00&lt;00:00, 62.55it/s, loss=0.1, metrics={'MulticlassAUROC': 0.3795}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 1/1 [00:00&lt;00:00, 108.51it/s, loss=0.0966, metrics={'MulticlassAUROC': 0.3053}]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 6/6 [00:00&lt;00:00, 99.35it/s, loss=0.0965, metrics={'MulticlassAUROC': 0.3809}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 1/1 [00:00&lt;00:00, 117.73it/s, loss=0.0962, metrics={'MulticlassAUROC': 0.3089}]\nepoch 5: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 6/6 [00:00&lt;00:00, 110.56it/s, loss=0.0967, metrics={'MulticlassAUROC': 0.3509}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 1/1 [00:00&lt;00:00, 127.35it/s, loss=0.0958, metrics={'MulticlassAUROC': 0.3089}]\n</pre> In\u00a0[\u00a0]: Copied! <pre>\n</pre>"},{"location":"examples/11_auc_multiclass.html#auc-multiclass-computation","title":"AUC multiclass computation\u00b6","text":""},{"location":"examples/11_auc_multiclass.html#initial-imports","title":"Initial imports\u00b6","text":""},{"location":"examples/11_auc_multiclass.html#preparing-the-data","title":"Preparing the data\u00b6","text":""},{"location":"examples/11_auc_multiclass.html#define-the-model","title":"Define the model\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html","title":"12_ZILNLoss_origkeras_vs_pytorch_widedeep","text":"In\u00a0[1]: Copied! <pre># @title Copyright 2019 The Lifetime Value Authors.\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     https://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n# ============================================================================\n</pre> # @title Copyright 2019 The Lifetime Value Authors. # Licensed under the Apache License, Version 2.0 (the \"License\"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # #     https://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an \"AS IS\" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ Run in Google Colab View source on GitHub In\u00a0[3]: Copied! <pre>import os\n\nimport numpy as np\nimport pandas as pd\nfrom scipy import stats\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport tensorflow as tf\nimport tensorflow_probability as tfp\nfrom typing import Sequence\n\n# install and import ltv\n!pip install -q git+https://github.com/google/lifetime_value\nimport lifetime_value as ltv\n</pre> import os  import numpy as np import pandas as pd from scipy import stats import matplotlib.pyplot as plt import seaborn as sns import tensorflow as tf import tensorflow_probability as tfp from typing import Sequence  # install and import ltv !pip install -q git+https://github.com/google/lifetime_value import lifetime_value as ltv In\u00a0[\u00a0]: Copied! <pre>tfd = tfp.distributions\n%config InlineBackend.figure_format='retina'\nsns.set_style(\"whitegrid\")\n</pre> tfd = tfp.distributions %config InlineBackend.figure_format='retina' sns.set_style(\"whitegrid\") In\u00a0[\u00a0]: Copied! <pre>MODEL = \"dnn\"\nLOSS = \"ziln\"  # @param { isTemplate: true, type: 'string'} ['mse', 'ziln']\nLEARNING_RATE = 0.001  # @param { isTemplate: true}\nVERSION = 0  # @param { isTemplate: true, type: 'integer'}\nOUTPUT_CSV_FOLDER = \"/tmp/lifetime-value/kdd_cup_98/result\"  # @param { isTemplate: true, type: 'string'}\n</pre> MODEL = \"dnn\" LOSS = \"ziln\"  # @param { isTemplate: true, type: 'string'} ['mse', 'ziln'] LEARNING_RATE = 0.001  # @param { isTemplate: true} VERSION = 0  # @param { isTemplate: true, type: 'integer'} OUTPUT_CSV_FOLDER = \"/tmp/lifetime-value/kdd_cup_98/result\"  # @param { isTemplate: true, type: 'string'} <p>Download kdd_cup_98 data to /tmp/lifetime-value/kdd_cup_98</p> In\u00a0[\u00a0]: Copied! <pre>%%bash\nmkdir -p /tmp/lifetime-value/kdd_cup_98\nwget https://kdd.ics.uci.edu/databases/kddcup98/epsilon_mirror/cup98lrn.zip -P /tmp/lifetime-value/kdd_cup_98/\nwget https://kdd.ics.uci.edu/databases/kddcup98/epsilon_mirror/cup98val.zip -P /tmp/lifetime-value/kdd_cup_98/\nwget https://kdd.ics.uci.edu/databases/kddcup98/epsilon_mirror/valtargt.txt -P /tmp/lifetime-value/kdd_cup_98/\ncd /tmp/lifetime-value/kdd_cup_98/\nunzip cup98lrn.zip\nunzip cup98val.zip\n</pre> %%bash mkdir -p /tmp/lifetime-value/kdd_cup_98 wget https://kdd.ics.uci.edu/databases/kddcup98/epsilon_mirror/cup98lrn.zip -P /tmp/lifetime-value/kdd_cup_98/ wget https://kdd.ics.uci.edu/databases/kddcup98/epsilon_mirror/cup98val.zip -P /tmp/lifetime-value/kdd_cup_98/ wget https://kdd.ics.uci.edu/databases/kddcup98/epsilon_mirror/valtargt.txt -P /tmp/lifetime-value/kdd_cup_98/ cd /tmp/lifetime-value/kdd_cup_98/ unzip cup98lrn.zip unzip cup98val.zip In\u00a0[\u00a0]: Copied! <pre>df_train = pd.read_csv(\"/tmp/lifetime-value/kdd_cup_98/cup98LRN.txt\")\nnum_train = df_train.shape[0]\ndf_eval = pd.read_csv(\"/tmp/lifetime-value/kdd_cup_98/cup98VAL.txt\")\ndf_eval_target = pd.read_csv(\"/tmp/lifetime-value/kdd_cup_98/valtargt.txt\")\ndf_eval = df_eval.merge(df_eval_target, on=\"CONTROLN\")\n</pre> df_train = pd.read_csv(\"/tmp/lifetime-value/kdd_cup_98/cup98LRN.txt\") num_train = df_train.shape[0] df_eval = pd.read_csv(\"/tmp/lifetime-value/kdd_cup_98/cup98VAL.txt\") df_eval_target = pd.read_csv(\"/tmp/lifetime-value/kdd_cup_98/valtargt.txt\") df_eval = df_eval.merge(df_eval_target, on=\"CONTROLN\") In\u00a0[\u00a0]: Copied! <pre>df = pd.concat([df_train, df_eval], axis=0, sort=True)\n</pre> df = pd.concat([df_train, df_eval], axis=0, sort=True) In\u00a0[\u00a0]: Copied! <pre>y = df[\"TARGET_D\"][:num_train]\n</pre> y = df[\"TARGET_D\"][:num_train] In\u00a0[\u00a0]: Copied! <pre>def plot_hist_log_scale(y):\n    max_val = y.max() + 1.0\n    ax = pd.Series(y).hist(\n        figsize=(8, 5), bins=10 ** np.linspace(0.0, np.log10(max_val), 20)\n    )\n\n    plt.xlabel(\"Donation ($)\")\n    plt.ylabel(\"Count\")\n    # plt.title('Histogram of LTV')\n    plt.xticks(rotation=\"horizontal\")\n    plt.legend(loc=\"upper left\")\n    ax.set_xscale(\"log\")\n    ax.grid(False)\n    # Hide the right and top spines\n    ax.spines[\"right\"].set_visible(False)\n    ax.spines[\"top\"].set_visible(False)\n    # Only show ticks on the left and bottom spines\n    ax.yaxis.set_ticks_position(\"left\")\n    ax.xaxis.set_ticks_position(\"bottom\")\n    plt.show()\n\n    fig = ax.get_figure()\n    output_file = tf.io.gfile.GFile(\n        \"/tmp/lifetime-value/kdd_cup_98/histogram_kdd98_log_scale.pdf\", \"wb\"\n    )\n    fig.savefig(output_file, bbox_inches=\"tight\", format=\"pdf\")\n</pre> def plot_hist_log_scale(y):     max_val = y.max() + 1.0     ax = pd.Series(y).hist(         figsize=(8, 5), bins=10 ** np.linspace(0.0, np.log10(max_val), 20)     )      plt.xlabel(\"Donation ($)\")     plt.ylabel(\"Count\")     # plt.title('Histogram of LTV')     plt.xticks(rotation=\"horizontal\")     plt.legend(loc=\"upper left\")     ax.set_xscale(\"log\")     ax.grid(False)     # Hide the right and top spines     ax.spines[\"right\"].set_visible(False)     ax.spines[\"top\"].set_visible(False)     # Only show ticks on the left and bottom spines     ax.yaxis.set_ticks_position(\"left\")     ax.xaxis.set_ticks_position(\"bottom\")     plt.show()      fig = ax.get_figure()     output_file = tf.io.gfile.GFile(         \"/tmp/lifetime-value/kdd_cup_98/histogram_kdd98_log_scale.pdf\", \"wb\"     )     fig.savefig(output_file, bbox_inches=\"tight\", format=\"pdf\") In\u00a0[\u00a0]: Copied! <pre>plot_hist_log_scale(y[y &gt; 0])\n</pre> plot_hist_log_scale(y[y &gt; 0]) In\u00a0[\u00a0]: Copied! <pre>VOCAB_FEATURES = [\n    \"ODATEDW\",  # date of donor's first gift (YYMM)\n    \"OSOURCE\",  # donor acquisition mailing list\n    \"TCODE\",  # donor title code\n    \"STATE\",\n    \"ZIP\",\n    \"DOMAIN\",  # urbanicity level and socio-economic status of the neighborhood\n    \"CLUSTER\",  # socio-economic status\n    \"GENDER\",\n    \"MAXADATE\",  # date of the most recent promotion received\n    \"MINRDATE\",\n    \"LASTDATE\",\n    \"FISTDATE\",\n    \"RFA_2A\",\n]\n</pre> VOCAB_FEATURES = [     \"ODATEDW\",  # date of donor's first gift (YYMM)     \"OSOURCE\",  # donor acquisition mailing list     \"TCODE\",  # donor title code     \"STATE\",     \"ZIP\",     \"DOMAIN\",  # urbanicity level and socio-economic status of the neighborhood     \"CLUSTER\",  # socio-economic status     \"GENDER\",     \"MAXADATE\",  # date of the most recent promotion received     \"MINRDATE\",     \"LASTDATE\",     \"FISTDATE\",     \"RFA_2A\", ] In\u00a0[\u00a0]: Copied! <pre>df[\"ODATEDW\"] = df[\"ODATEDW\"].astype(\"str\")\ndf[\"TCODE\"] = df[\"TCODE\"].apply(lambda x: \"{:03d}\".format(x // 1000 if x &gt; 1000 else x))\ndf[\"ZIP\"] = df[\"ZIP\"].str.slice(0, 5)\ndf[\"MAXADATE\"] = df[\"MAXADATE\"].astype(\"str\")\ndf[\"MINRDATE\"] = df[\"MINRDATE\"].astype(\"str\")\ndf[\"LASTDATE\"] = df[\"LASTDATE\"].astype(\"str\")\ndf[\"FISTDATE\"] = df[\"FISTDATE\"].astype(\"str\")\n</pre> df[\"ODATEDW\"] = df[\"ODATEDW\"].astype(\"str\") df[\"TCODE\"] = df[\"TCODE\"].apply(lambda x: \"{:03d}\".format(x // 1000 if x &gt; 1000 else x)) df[\"ZIP\"] = df[\"ZIP\"].str.slice(0, 5) df[\"MAXADATE\"] = df[\"MAXADATE\"].astype(\"str\") df[\"MINRDATE\"] = df[\"MINRDATE\"].astype(\"str\") df[\"LASTDATE\"] = df[\"LASTDATE\"].astype(\"str\") df[\"FISTDATE\"] = df[\"FISTDATE\"].astype(\"str\") In\u00a0[\u00a0]: Copied! <pre>def label_encoding(y, frequency_threshold=100):\n    value_counts = pd.value_counts(y)\n    categories = value_counts[value_counts &gt;= frequency_threshold].index.to_numpy()\n    # 0 indicates the unknown category.\n    return pd.Categorical(y, categories=categories).codes + 1\n</pre> def label_encoding(y, frequency_threshold=100):     value_counts = pd.value_counts(y)     categories = value_counts[value_counts &gt;= frequency_threshold].index.to_numpy()     # 0 indicates the unknown category.     return pd.Categorical(y, categories=categories).codes + 1 In\u00a0[\u00a0]: Copied! <pre>for key in VOCAB_FEATURES:\n    df[key] = label_encoding(df[key])\n</pre> for key in VOCAB_FEATURES:     df[key] = label_encoding(df[key]) In\u00a0[\u00a0]: Copied! <pre>MAIL_ORDER_RESPONSES = [\n    \"MBCRAFT\",\n    \"MBGARDEN\",\n    \"MBBOOKS\",\n    \"MBCOLECT\",\n    \"MAGFAML\",\n    \"MAGFEM\",\n    \"MAGMALE\",\n    \"PUBGARDN\",\n    \"PUBCULIN\",\n    \"PUBHLTH\",\n    \"PUBDOITY\",\n    \"PUBNEWFN\",\n    \"PUBPHOTO\",\n    \"PUBOPP\",\n    \"RFA_2F\",\n]\n</pre> MAIL_ORDER_RESPONSES = [     \"MBCRAFT\",     \"MBGARDEN\",     \"MBBOOKS\",     \"MBCOLECT\",     \"MAGFAML\",     \"MAGFEM\",     \"MAGMALE\",     \"PUBGARDN\",     \"PUBCULIN\",     \"PUBHLTH\",     \"PUBDOITY\",     \"PUBNEWFN\",     \"PUBPHOTO\",     \"PUBOPP\",     \"RFA_2F\", ] In\u00a0[\u00a0]: Copied! <pre>INDICATOR_FEATURES = [\n    \"AGE\",  # age decile, 0 indicates unknown\n    \"NUMCHLD\",\n    \"INCOME\",\n    \"WEALTH1\",\n    \"HIT\",\n] + MAIL_ORDER_RESPONSES\n</pre> INDICATOR_FEATURES = [     \"AGE\",  # age decile, 0 indicates unknown     \"NUMCHLD\",     \"INCOME\",     \"WEALTH1\",     \"HIT\", ] + MAIL_ORDER_RESPONSES In\u00a0[\u00a0]: Copied! <pre>df[\"AGE\"] = pd.qcut(df[\"AGE\"].values, 10).codes + 1\ndf[\"NUMCHLD\"] = df[\"NUMCHLD\"].apply(lambda x: 0 if np.isnan(x) else int(x))\ndf[\"INCOME\"] = df[\"INCOME\"].apply(lambda x: 0 if np.isnan(x) else int(x))\ndf[\"WEALTH1\"] = df[\"WEALTH1\"].apply(lambda x: 0 if np.isnan(x) else int(x) + 1)\ndf[\"HIT\"] = pd.qcut(df[\"HIT\"].values, q=50, duplicates=\"drop\").codes\n\nfor col in MAIL_ORDER_RESPONSES:\n    df[col] = pd.qcut(df[col].values, q=20, duplicates=\"drop\").codes + 1\n</pre> df[\"AGE\"] = pd.qcut(df[\"AGE\"].values, 10).codes + 1 df[\"NUMCHLD\"] = df[\"NUMCHLD\"].apply(lambda x: 0 if np.isnan(x) else int(x)) df[\"INCOME\"] = df[\"INCOME\"].apply(lambda x: 0 if np.isnan(x) else int(x)) df[\"WEALTH1\"] = df[\"WEALTH1\"].apply(lambda x: 0 if np.isnan(x) else int(x) + 1) df[\"HIT\"] = pd.qcut(df[\"HIT\"].values, q=50, duplicates=\"drop\").codes  for col in MAIL_ORDER_RESPONSES:     df[col] = pd.qcut(df[col].values, q=20, duplicates=\"drop\").codes + 1 In\u00a0[\u00a0]: Copied! <pre>NUMERIC_FEATURES = [\n    # binary\n    \"MAILCODE\",  # bad address\n    \"NOEXCH\",  # do not exchange\n    \"RECINHSE\",  # donor has given to PVA's in house program\n    \"RECP3\",  # donor has given to PVA's P3 program\n    \"RECPGVG\",  # planned giving record\n    \"RECSWEEP\",  # sweepstakes record\n    \"HOMEOWNR\",  # home owner\n    \"CHILD03\",\n    \"CHILD07\",\n    \"CHILD12\",\n    \"CHILD18\",\n    # continuous\n    \"CARDPROM\",\n    \"NUMPROM\",\n    \"CARDPM12\",\n    \"NUMPRM12\",\n    \"RAMNTALL\",\n    \"NGIFTALL\",\n    \"MINRAMNT\",\n    \"MAXRAMNT\",\n    \"LASTGIFT\",\n    \"AVGGIFT\",\n]\n</pre> NUMERIC_FEATURES = [     # binary     \"MAILCODE\",  # bad address     \"NOEXCH\",  # do not exchange     \"RECINHSE\",  # donor has given to PVA's in house program     \"RECP3\",  # donor has given to PVA's P3 program     \"RECPGVG\",  # planned giving record     \"RECSWEEP\",  # sweepstakes record     \"HOMEOWNR\",  # home owner     \"CHILD03\",     \"CHILD07\",     \"CHILD12\",     \"CHILD18\",     # continuous     \"CARDPROM\",     \"NUMPROM\",     \"CARDPM12\",     \"NUMPRM12\",     \"RAMNTALL\",     \"NGIFTALL\",     \"MINRAMNT\",     \"MAXRAMNT\",     \"LASTGIFT\",     \"AVGGIFT\", ] In\u00a0[\u00a0]: Copied! <pre>df[\"MAILCODE\"] = (df[\"MAILCODE\"] == \"B\").astype(\"float32\")\ndf[\"PVASTATE\"] = df[\"PVASTATE\"].isin([\"P\", \"E\"]).astype(\"float32\")\ndf[\"NOEXCH\"] = df[\"NOEXCH\"].isin([\"X\", \"1\"]).astype(\"float32\")\ndf[\"RECINHSE\"] = (df[\"RECINHSE\"] == \"X\").astype(\"float32\")\ndf[\"RECP3\"] = (df[\"RECP3\"] == \"X\").astype(\"float32\")\ndf[\"RECPGVG\"] = (df[\"RECPGVG\"] == \"X\").astype(\"float32\")\ndf[\"RECSWEEP\"] = (df[\"RECSWEEP\"] == \"X\").astype(\"float32\")\ndf[\"HOMEOWNR\"] = (df[\"HOMEOWNR\"] == \"H\").astype(\"float32\")\ndf[\"CHILD03\"] = df[\"CHILD03\"].isin([\"M\", \"F\", \"B\"]).astype(\"float32\")\ndf[\"CHILD07\"] = df[\"CHILD07\"].isin([\"M\", \"F\", \"B\"]).astype(\"float32\")\ndf[\"CHILD12\"] = df[\"CHILD12\"].isin([\"M\", \"F\", \"B\"]).astype(\"float32\")\ndf[\"CHILD18\"] = df[\"CHILD18\"].isin([\"M\", \"F\", \"B\"]).astype(\"float32\")\n\ndf[\"CARDPROM\"] = df[\"CARDPROM\"] / 100\ndf[\"NUMPROM\"] = df[\"NUMPROM\"] / 100\ndf[\"CARDPM12\"] = df[\"CARDPM12\"] / 100\ndf[\"NUMPRM12\"] = df[\"NUMPRM12\"] / 100\ndf[\"RAMNTALL\"] = np.log1p(df[\"RAMNTALL\"])\ndf[\"NGIFTALL\"] = np.log1p(df[\"NGIFTALL\"])\ndf[\"MINRAMNT\"] = np.log1p(df[\"MINRAMNT\"])\ndf[\"MAXRAMNT\"] = np.log1p(df[\"MAXRAMNT\"])\ndf[\"LASTGIFT\"] = np.log1p(df[\"LASTGIFT\"])\ndf[\"AVGGIFT\"] = np.log1p(df[\"AVGGIFT\"])\n</pre> df[\"MAILCODE\"] = (df[\"MAILCODE\"] == \"B\").astype(\"float32\") df[\"PVASTATE\"] = df[\"PVASTATE\"].isin([\"P\", \"E\"]).astype(\"float32\") df[\"NOEXCH\"] = df[\"NOEXCH\"].isin([\"X\", \"1\"]).astype(\"float32\") df[\"RECINHSE\"] = (df[\"RECINHSE\"] == \"X\").astype(\"float32\") df[\"RECP3\"] = (df[\"RECP3\"] == \"X\").astype(\"float32\") df[\"RECPGVG\"] = (df[\"RECPGVG\"] == \"X\").astype(\"float32\") df[\"RECSWEEP\"] = (df[\"RECSWEEP\"] == \"X\").astype(\"float32\") df[\"HOMEOWNR\"] = (df[\"HOMEOWNR\"] == \"H\").astype(\"float32\") df[\"CHILD03\"] = df[\"CHILD03\"].isin([\"M\", \"F\", \"B\"]).astype(\"float32\") df[\"CHILD07\"] = df[\"CHILD07\"].isin([\"M\", \"F\", \"B\"]).astype(\"float32\") df[\"CHILD12\"] = df[\"CHILD12\"].isin([\"M\", \"F\", \"B\"]).astype(\"float32\") df[\"CHILD18\"] = df[\"CHILD18\"].isin([\"M\", \"F\", \"B\"]).astype(\"float32\")  df[\"CARDPROM\"] = df[\"CARDPROM\"] / 100 df[\"NUMPROM\"] = df[\"NUMPROM\"] / 100 df[\"CARDPM12\"] = df[\"CARDPM12\"] / 100 df[\"NUMPRM12\"] = df[\"NUMPRM12\"] / 100 df[\"RAMNTALL\"] = np.log1p(df[\"RAMNTALL\"]) df[\"NGIFTALL\"] = np.log1p(df[\"NGIFTALL\"]) df[\"MINRAMNT\"] = np.log1p(df[\"MINRAMNT\"]) df[\"MAXRAMNT\"] = np.log1p(df[\"MAXRAMNT\"]) df[\"LASTGIFT\"] = np.log1p(df[\"LASTGIFT\"]) df[\"AVGGIFT\"] = np.log1p(df[\"AVGGIFT\"]) In\u00a0[\u00a0]: Copied! <pre>CATEGORICAL_FEATURES = VOCAB_FEATURES + INDICATOR_FEATURES\nALL_FEATURES = CATEGORICAL_FEATURES + NUMERIC_FEATURES\n</pre> CATEGORICAL_FEATURES = VOCAB_FEATURES + INDICATOR_FEATURES ALL_FEATURES = CATEGORICAL_FEATURES + NUMERIC_FEATURES In\u00a0[\u00a0]: Copied! <pre>def dnn_split(df):\n    df_train = df.iloc[:num_train]\n    df_eval = df.iloc[num_train:]\n\n    def feature_dict(df):\n        features = {k: v.values for k, v in dict(df[CATEGORICAL_FEATURES]).items()}\n        features[\"numeric\"] = df[NUMERIC_FEATURES].astype(\"float32\").values\n        return features\n\n    x_train, y_train = (\n        feature_dict(df_train),\n        df_train[\"TARGET_D\"].astype(\"float32\").values,\n    )\n    x_eval, y_eval = feature_dict(df_eval), df_eval[\"TARGET_D\"].astype(\"float32\").values\n\n    return x_train, x_eval, y_train, y_eval\n</pre> def dnn_split(df):     df_train = df.iloc[:num_train]     df_eval = df.iloc[num_train:]      def feature_dict(df):         features = {k: v.values for k, v in dict(df[CATEGORICAL_FEATURES]).items()}         features[\"numeric\"] = df[NUMERIC_FEATURES].astype(\"float32\").values         return features      x_train, y_train = (         feature_dict(df_train),         df_train[\"TARGET_D\"].astype(\"float32\").values,     )     x_eval, y_eval = feature_dict(df_eval), df_eval[\"TARGET_D\"].astype(\"float32\").values      return x_train, x_eval, y_train, y_eval In\u00a0[\u00a0]: Copied! <pre>def embedding_dim(x):\n    return int(x**0.25) + 1\n\n\ndef embedding_layer(vocab_size):\n    return tf.keras.Sequential(\n        [\n            tf.keras.layers.Embedding(\n                input_dim=vocab_size,\n                output_dim=embedding_dim(vocab_size),\n                input_length=1,\n            ),\n            tf.keras.layers.Flatten(),\n        ]\n    )\n\n\ndef dnn_model(output_units):\n    numeric_input = tf.keras.layers.Input(\n        shape=(len(NUMERIC_FEATURES),), name=\"numeric\"\n    )\n\n    embedding_inputs = [\n        tf.keras.layers.Input(shape=(1,), name=key, dtype=np.int64)\n        for key in CATEGORICAL_FEATURES\n    ]\n\n    embedding_outputs = [\n        embedding_layer(vocab_size=df[key].max() + 1)(input)\n        for key, input in zip(CATEGORICAL_FEATURES, embedding_inputs)\n    ]\n\n    deep_input = tf.keras.layers.concatenate([numeric_input] + embedding_outputs)\n    deep_model = tf.keras.Sequential(\n        [\n            tf.keras.layers.Dense(128, activation=\"relu\"),\n            tf.keras.layers.Dense(128, activation=\"relu\"),\n            tf.keras.layers.Dense(64, activation=\"relu\"),\n            tf.keras.layers.Dense(64, activation=\"relu\"),\n            tf.keras.layers.Dense(units=output_units),\n        ]\n    )\n    return tf.keras.Model(\n        inputs=[numeric_input] + embedding_inputs, outputs=deep_model(deep_input)\n    )\n</pre> def embedding_dim(x):     return int(x**0.25) + 1   def embedding_layer(vocab_size):     return tf.keras.Sequential(         [             tf.keras.layers.Embedding(                 input_dim=vocab_size,                 output_dim=embedding_dim(vocab_size),                 input_length=1,             ),             tf.keras.layers.Flatten(),         ]     )   def dnn_model(output_units):     numeric_input = tf.keras.layers.Input(         shape=(len(NUMERIC_FEATURES),), name=\"numeric\"     )      embedding_inputs = [         tf.keras.layers.Input(shape=(1,), name=key, dtype=np.int64)         for key in CATEGORICAL_FEATURES     ]      embedding_outputs = [         embedding_layer(vocab_size=df[key].max() + 1)(input)         for key, input in zip(CATEGORICAL_FEATURES, embedding_inputs)     ]      deep_input = tf.keras.layers.concatenate([numeric_input] + embedding_outputs)     deep_model = tf.keras.Sequential(         [             tf.keras.layers.Dense(128, activation=\"relu\"),             tf.keras.layers.Dense(128, activation=\"relu\"),             tf.keras.layers.Dense(64, activation=\"relu\"),             tf.keras.layers.Dense(64, activation=\"relu\"),             tf.keras.layers.Dense(units=output_units),         ]     )     return tf.keras.Model(         inputs=[numeric_input] + embedding_inputs, outputs=deep_model(deep_input)     ) In\u00a0[\u00a0]: Copied! <pre>if LOSS == \"mse\":\n    loss = tf.keras.losses.MeanSquaredError()\n    output_units = 1\n\nif LOSS == \"ziln\":\n    loss = ltv.zero_inflated_lognormal_loss\n    output_units = 3\n</pre> if LOSS == \"mse\":     loss = tf.keras.losses.MeanSquaredError()     output_units = 1  if LOSS == \"ziln\":     loss = ltv.zero_inflated_lognormal_loss     output_units = 3 In\u00a0[\u00a0]: Copied! <pre>x_train, x_eval, y_train, y_eval = dnn_split(df)\nmodel = dnn_model(output_units)\n</pre> x_train, x_eval, y_train, y_eval = dnn_split(df) model = dnn_model(output_units) In\u00a0[\u00a0]: Copied! <pre>model.compile(optimizer=tf.keras.optimizers.Nadam(lr=LEARNING_RATE), loss=loss)\n</pre> model.compile(optimizer=tf.keras.optimizers.Nadam(lr=LEARNING_RATE), loss=loss) In\u00a0[\u00a0]: Copied! <pre>callbacks = [\n    tf.keras.callbacks.ReduceLROnPlateau(monitor=\"val_loss\", min_lr=1e-6),\n    tf.keras.callbacks.EarlyStopping(monitor=\"val_loss\", patience=10),\n]\n</pre> callbacks = [     tf.keras.callbacks.ReduceLROnPlateau(monitor=\"val_loss\", min_lr=1e-6),     tf.keras.callbacks.EarlyStopping(monitor=\"val_loss\", patience=10), ] In\u00a0[\u00a0]: Copied! <pre>history = model.fit(\n    x=x_train,\n    y=y_train,\n    batch_size=2048,\n    epochs=200,\n    verbose=2,\n    callbacks=callbacks,\n    validation_data=(x_eval, y_eval),\n).history\n</pre> history = model.fit(     x=x_train,     y=y_train,     batch_size=2048,     epochs=200,     verbose=2,     callbacks=callbacks,     validation_data=(x_eval, y_eval), ).history In\u00a0[\u00a0]: Copied! <pre>pd.DataFrame(history)[[\"loss\", \"val_loss\"]].plot();\n</pre> pd.DataFrame(history)[[\"loss\", \"val_loss\"]].plot(); In\u00a0[\u00a0]: Copied! <pre>if LOSS == \"mse\":\n    y_pred = model.predict(x=x_eval, batch_size=1024).flatten()\n\nif LOSS == \"ziln\":\n    logits = model.predict(x=x_eval, batch_size=1024)\n    y_pred = ltv.zero_inflated_lognormal_pred(logits).numpy().flatten()\n</pre> if LOSS == \"mse\":     y_pred = model.predict(x=x_eval, batch_size=1024).flatten()  if LOSS == \"ziln\":     logits = model.predict(x=x_eval, batch_size=1024)     y_pred = ltv.zero_inflated_lognormal_pred(logits).numpy().flatten() In\u00a0[\u00a0]: Copied! <pre>from pytorch_widedeep.preprocessing import TabPreprocessor\nfrom pytorch_widedeep.training import Trainer\nfrom pytorch_widedeep.models import TabMlp, WideDeep\nfrom torch.optim.lr_scheduler import ReduceLROnPlateau\nfrom pytorch_widedeep.callbacks import EarlyStopping\nfrom torch.optim import NAdam\n</pre> from pytorch_widedeep.preprocessing import TabPreprocessor from pytorch_widedeep.training import Trainer from pytorch_widedeep.models import TabMlp, WideDeep from torch.optim.lr_scheduler import ReduceLROnPlateau from pytorch_widedeep.callbacks import EarlyStopping from torch.optim import NAdam In\u00a0[\u00a0]: Copied! <pre># CATEGORICAL_FEATURES\nNUMERICAL_FEATURES = [\"num\" + str(i) for i in range(21)]\nx_train_pyt_num = pd.DataFrame(columns=NUMERICAL_FEATURES, data=x_train[\"numeric\"])\nx_train_pyt_cat = pd.DataFrame(\n    {key: value for key, value in x_train.items() if key not in [\"numeric\"]}\n)\n\nx_eval_pyt_num = pd.DataFrame(columns=NUMERICAL_FEATURES, data=x_eval[\"numeric\"])\nx_eval_pyt_cat = pd.DataFrame(\n    {key: value for key, value in x_eval.items() if key not in [\"numeric\"]}\n)\n</pre> # CATEGORICAL_FEATURES NUMERICAL_FEATURES = [\"num\" + str(i) for i in range(21)] x_train_pyt_num = pd.DataFrame(columns=NUMERICAL_FEATURES, data=x_train[\"numeric\"]) x_train_pyt_cat = pd.DataFrame(     {key: value for key, value in x_train.items() if key not in [\"numeric\"]} )  x_eval_pyt_num = pd.DataFrame(columns=NUMERICAL_FEATURES, data=x_eval[\"numeric\"]) x_eval_pyt_cat = pd.DataFrame(     {key: value for key, value in x_eval.items() if key not in [\"numeric\"]} ) In\u00a0[\u00a0]: Copied! <pre>x_train_pyt = pd.concat([x_train_pyt_num, x_train_pyt_cat], axis=1)\nx_eval_pyt = pd.concat([x_eval_pyt_num, x_eval_pyt_cat], axis=1)\n</pre> x_train_pyt = pd.concat([x_train_pyt_num, x_train_pyt_cat], axis=1) x_eval_pyt = pd.concat([x_eval_pyt_num, x_eval_pyt_cat], axis=1) In\u00a0[\u00a0]: Copied! <pre>embed_input = [\n    (u, int(x_train_pyt[u].nunique() ** 0.25) + 1) for u in CATEGORICAL_FEATURES\n]\n</pre> embed_input = [     (u, int(x_train_pyt[u].nunique() ** 0.25) + 1) for u in CATEGORICAL_FEATURES ] In\u00a0[\u00a0]: Copied! <pre># deeptabular\ntab_preprocessor = TabPreprocessor(\n    embed_cols=embed_input,\n    continuous_cols=NUMERICAL_FEATURES,\n    shared_embed=False,\n    scale=False,\n)\nX_tab_train = tab_preprocessor.fit_transform(x_train_pyt)\nX_tab_valid = tab_preprocessor.transform(x_eval_pyt)\nX_tab_test = tab_preprocessor.transform(x_eval_pyt)\n\n# target\ny_train = y_train\ny_valid = y_eval\ny_test = y_train\n\nX_train = {\"X_tab\": X_tab_train, \"target\": y_train}\nX_val = {\"X_tab\": X_tab_valid, \"target\": y_valid}\nX_test = {\"X_tab\": X_tab_test}\n\ndeeptabular = TabMlp(\n    mlp_hidden_dims=[128, 128, 64, 64],\n    column_idx=tab_preprocessor.column_idx,\n    embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=tab_preprocessor.continuous_cols,\n)\n\nmodel = WideDeep(deeptabular=deeptabular, pred_dim=3)\n\ndeep_opt = NAdam(model.deeptabular.parameters(), lr=LEARNING_RATE)\ncallbacks = [EarlyStopping()]\ndeep_sch = ReduceLROnPlateau(deep_opt, min_lr=1e-6)\n\nobjective = \"ziln\"\n\ntrainer = Trainer(\n    model,\n    callbacks=callbacks,\n    lr_schedulers={\"deeptabular\": deep_sch},\n    objective=objective,\n    optimizers={\"deeptabular\": deep_opt},\n)\n\ntrainer.fit(\n    X_train=X_train,\n    X_val=X_val,\n    n_epochs=200,\n    batch_size=2048,\n)\n\ny_pred_pytorch = trainer.predict(X_test=X_test)\n</pre> # deeptabular tab_preprocessor = TabPreprocessor(     embed_cols=embed_input,     continuous_cols=NUMERICAL_FEATURES,     shared_embed=False,     scale=False, ) X_tab_train = tab_preprocessor.fit_transform(x_train_pyt) X_tab_valid = tab_preprocessor.transform(x_eval_pyt) X_tab_test = tab_preprocessor.transform(x_eval_pyt)  # target y_train = y_train y_valid = y_eval y_test = y_train  X_train = {\"X_tab\": X_tab_train, \"target\": y_train} X_val = {\"X_tab\": X_tab_valid, \"target\": y_valid} X_test = {\"X_tab\": X_tab_test}  deeptabular = TabMlp(     mlp_hidden_dims=[128, 128, 64, 64],     column_idx=tab_preprocessor.column_idx,     embed_input=tab_preprocessor.cat_embed_input,     continuous_cols=tab_preprocessor.continuous_cols, )  model = WideDeep(deeptabular=deeptabular, pred_dim=3)  deep_opt = NAdam(model.deeptabular.parameters(), lr=LEARNING_RATE) callbacks = [EarlyStopping()] deep_sch = ReduceLROnPlateau(deep_opt, min_lr=1e-6)  objective = \"ziln\"  trainer = Trainer(     model,     callbacks=callbacks,     lr_schedulers={\"deeptabular\": deep_sch},     objective=objective,     optimizers={\"deeptabular\": deep_opt}, )  trainer.fit(     X_train=X_train,     X_val=X_val,     n_epochs=200,     batch_size=2048, )  y_pred_pytorch = trainer.predict(X_test=X_test) In\u00a0[\u00a0]: Copied! <pre>pd.DataFrame(trainer.history)[[\"train_loss\", \"val_loss\"]].plot();\n</pre> pd.DataFrame(trainer.history)[[\"train_loss\", \"val_loss\"]].plot(); In\u00a0[\u00a0]: Copied! <pre>from sklearn.metrics import mean_squared_error\n\nmean_squared_error(y_pred, y_pred_pytorch)\n</pre> from sklearn.metrics import mean_squared_error  mean_squared_error(y_pred, y_pred_pytorch) In\u00a0[\u00a0]: Copied! <pre>unit_costs = [0.4, 0.5, 0.6, 0.68, 0.7, 0.8, 0.9, 1.0]\n</pre> unit_costs = [0.4, 0.5, 0.6, 0.68, 0.7, 0.8, 0.9, 1.0] In\u00a0[\u00a0]: Copied! <pre>num_mailed = [np.sum(y_pred &gt; v) for v in unit_costs]\nnum_mailed\n</pre> num_mailed = [np.sum(y_pred &gt; v) for v in unit_costs] num_mailed In\u00a0[\u00a0]: Copied! <pre>baseline_total_profit = np.sum(y_eval - 0.68)\nbaseline_total_profit\n</pre> baseline_total_profit = np.sum(y_eval - 0.68) baseline_total_profit In\u00a0[\u00a0]: Copied! <pre>total_profits = [np.sum(y_eval[y_pred &gt; v] - v) for v in unit_costs]\ntotal_profits\n</pre> total_profits = [np.sum(y_eval[y_pred &gt; v] - v) for v in unit_costs] total_profits In\u00a0[\u00a0]: Copied! <pre>gain = pd.DataFrame(\n    {\n        \"lorenz\": ltv.cumulative_true(y_eval, y_eval),\n        \"baseline\": ltv.cumulative_true(y_eval, x_eval[\"numeric\"][:, 19]),\n        \"model\": ltv.cumulative_true(y_eval, y_pred),\n    }\n)\n</pre> gain = pd.DataFrame(     {         \"lorenz\": ltv.cumulative_true(y_eval, y_eval),         \"baseline\": ltv.cumulative_true(y_eval, x_eval[\"numeric\"][:, 19]),         \"model\": ltv.cumulative_true(y_eval, y_pred),     } ) In\u00a0[\u00a0]: Copied! <pre>num_customers = np.float32(gain.shape[0])\ngain[\"cumulative_customer\"] = (np.arange(num_customers) + 1.0) / num_customers\n</pre> num_customers = np.float32(gain.shape[0]) gain[\"cumulative_customer\"] = (np.arange(num_customers) + 1.0) / num_customers In\u00a0[\u00a0]: Copied! <pre>ax = gain[\n    [\n        \"cumulative_customer\",\n        \"lorenz\",\n        \"baseline\",\n        \"model\",\n    ]\n].plot(x=\"cumulative_customer\", figsize=(8, 5), legend=True)\n\nax.legend([\"Groundtruth\", \"Baseline\", \"Model\"], loc=\"lower right\")\n\nax.set_xlabel(\"Cumulative Fraction of Customers\")\nax.set_xticks(np.arange(0, 1.1, 0.1))\nax.set_xlim((0, 1.0))\n\nax.set_ylabel(\"Cumulative Fraction of Total Lifetime Value\")\nax.set_yticks(np.arange(0, 1.1, 0.1))\nax.set_ylim((0, 1.05))\nax.set_title(\"Gain Chart\");\n</pre> ax = gain[     [         \"cumulative_customer\",         \"lorenz\",         \"baseline\",         \"model\",     ] ].plot(x=\"cumulative_customer\", figsize=(8, 5), legend=True)  ax.legend([\"Groundtruth\", \"Baseline\", \"Model\"], loc=\"lower right\")  ax.set_xlabel(\"Cumulative Fraction of Customers\") ax.set_xticks(np.arange(0, 1.1, 0.1)) ax.set_xlim((0, 1.0))  ax.set_ylabel(\"Cumulative Fraction of Total Lifetime Value\") ax.set_yticks(np.arange(0, 1.1, 0.1)) ax.set_ylim((0, 1.05)) ax.set_title(\"Gain Chart\"); In\u00a0[\u00a0]: Copied! <pre>gini = ltv.gini_from_gain(gain[[\"lorenz\", \"baseline\", \"model\"]])\ngini\n</pre> gini = ltv.gini_from_gain(gain[[\"lorenz\", \"baseline\", \"model\"]]) gini In\u00a0[\u00a0]: Copied! <pre>df_decile = ltv.decile_stats(y_eval, y_pred)\ndf_decile\n</pre> df_decile = ltv.decile_stats(y_eval, y_pred) df_decile In\u00a0[\u00a0]: Copied! <pre>ax = df_decile[[\"label_mean\", \"pred_mean\"]].plot.bar(rot=0)\n\nax.set_title(\"Decile Chart\")\nax.set_xlabel(\"Prediction bucket\")\nax.set_ylabel(\"Average bucket value\")\nax.legend([\"Label\", \"Prediction\"], loc=\"upper left\");\n</pre> ax = df_decile[[\"label_mean\", \"pred_mean\"]].plot.bar(rot=0)  ax.set_title(\"Decile Chart\") ax.set_xlabel(\"Prediction bucket\") ax.set_ylabel(\"Average bucket value\") ax.legend([\"Label\", \"Prediction\"], loc=\"upper left\"); In\u00a0[\u00a0]: Copied! <pre>def spearmanr(x1: Sequence[float], x2: Sequence[float]) -&gt; float:\n    \"\"\"Calculates spearmanr rank correlation coefficient.\n\n    See https://docs.scipy.org/doc/scipy/reference/stats.html.\n\n    Args:\n      x1: 1D array_like.\n      x2: 1D array_like.\n\n    Returns:\n      correlation: float.\n    \"\"\"\n    return stats.spearmanr(x1, x2, nan_policy=\"raise\")[0]\n\n\nspearman_corr = spearmanr(y_eval, y_pred)\nspearman_corr\n</pre> def spearmanr(x1: Sequence[float], x2: Sequence[float]) -&gt; float:     \"\"\"Calculates spearmanr rank correlation coefficient.      See https://docs.scipy.org/doc/scipy/reference/stats.html.      Args:       x1: 1D array_like.       x2: 1D array_like.      Returns:       correlation: float.     \"\"\"     return stats.spearmanr(x1, x2, nan_policy=\"raise\")[0]   spearman_corr = spearmanr(y_eval, y_pred) spearman_corr In\u00a0[\u00a0]: Copied! <pre>df_metrics = pd.DataFrame(\n    {\n        \"model\": MODEL,\n        \"loss_function\": LOSS,\n        \"train_loss\": history[\"loss\"][-1],\n        \"eval_loss\": history[\"val_loss\"][-1],\n        \"label_positive\": np.mean(y_eval &gt; 0),\n        \"label_mean\": y_eval.mean(),\n        \"pred_mean\": y_pred.mean(),\n        \"decile_mape\": df_decile[\"decile_mape\"].mean(),\n        \"baseline_gini\": gini[\"normalized\"][1],\n        \"gini\": gini[\"normalized\"][2],\n        \"spearman_corr\": spearman_corr,\n    },\n    index=[VERSION],\n)\n</pre> df_metrics = pd.DataFrame(     {         \"model\": MODEL,         \"loss_function\": LOSS,         \"train_loss\": history[\"loss\"][-1],         \"eval_loss\": history[\"val_loss\"][-1],         \"label_positive\": np.mean(y_eval &gt; 0),         \"label_mean\": y_eval.mean(),         \"pred_mean\": y_pred.mean(),         \"decile_mape\": df_decile[\"decile_mape\"].mean(),         \"baseline_gini\": gini[\"normalized\"][1],         \"gini\": gini[\"normalized\"][2],         \"spearman_corr\": spearman_corr,     },     index=[VERSION], ) In\u00a0[\u00a0]: Copied! <pre>for unit_cost, total_profit in zip(unit_costs, total_profits):\n    df_metrics[\"total_profit_{:02d}\".format(int(unit_cost * 100))] = total_profit\n</pre> for unit_cost, total_profit in zip(unit_costs, total_profits):     df_metrics[\"total_profit_{:02d}\".format(int(unit_cost * 100))] = total_profit In\u00a0[\u00a0]: Copied! <pre>df_metrics.T\n</pre> df_metrics.T In\u00a0[\u00a0]: Copied! <pre>output_path = OUTPUT_CSV_FOLDER\n</pre> output_path = OUTPUT_CSV_FOLDER In\u00a0[\u00a0]: Copied! <pre>if not os.path.isdir(output_path):\n    os.makedirs(output_path)\n</pre> if not os.path.isdir(output_path):     os.makedirs(output_path) In\u00a0[\u00a0]: Copied! <pre>output_file = os.path.join(\n    output_path, \"{}_regression_{}_{}.csv\".format(MODEL, LOSS, VERSION)\n)\n</pre> output_file = os.path.join(     output_path, \"{}_regression_{}_{}.csv\".format(MODEL, LOSS, VERSION) ) In\u00a0[\u00a0]: Copied! <pre>df_metrics.to_csv(output_file, index=False)\n</pre> df_metrics.to_csv(output_file, index=False)"},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#zilnloss","title":"ZILNLoss\u00b6","text":"<p>[DISCLAIMER]</p> <p>Purpose of this notebook is to check if ZILNloss implemented originaly Keras give same results in pytorch-widedeep implemenatation</p>"},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#kdd-cup-98-ltv-prediction","title":"KDD Cup 98 LTV Prediction\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#configs","title":"Configs\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#load-data","title":"Load data\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#label-distribution","title":"Label distribution\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#preprocess-features","title":"Preprocess features\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#vocab","title":"Vocab\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#indicator","title":"Indicator\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#numeric","title":"Numeric\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#all","title":"All\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#traineval-split","title":"Train/eval split\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#model","title":"Model\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#loss","title":"Loss\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#train","title":"Train\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#eval","title":"Eval\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#pytorch-widedeep-approach","title":"Pytorch-widedeep approach\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#appendix","title":"Appendix\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#total-profit","title":"Total Profit\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#gini-coefficient","title":"Gini Coefficient\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#calibration","title":"Calibration\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#rank-correlation","title":"Rank Correlation\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#all-metrics-together","title":"All metrics together\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#save","title":"Save\u00b6","text":""},{"location":"examples/13_model_uncertainty_prediction.html","title":"13_model_uncertainty_prediction","text":"<ul> <li>In this notebook we will use the higly imbalanced Protein Homology Dataset from KDD cup 2004</li> </ul> <pre><code>* The first element of each line is a BLOCK ID that denotes to which native sequence this example belongs. There is a unique BLOCK ID for each native sequence. BLOCK IDs are integers running from 1 to 303 (one for each native sequence, i.e. for each query). BLOCK IDs were assigned before the blocks were split into the train and test sets, so they do not run consecutively in either file.\n* The second element of each line is an EXAMPLE ID that uniquely describes the example. You will need this EXAMPLE ID and the BLOCK ID when you submit results.\n* The third element is the class of the example. Proteins that are homologous to the native sequence are denoted by 1, non-homologous proteins (i.e. decoys) by 0. Test examples have a \"?\" in this position.\n* All following elements are feature values. There are 74 feature values in each line. The features describe the match (e.g. the score of a sequence alignment) between the native protein sequence and the sequence that is tested for homology.\n</code></pre> In\u00a0[1]: Copied! <pre>import numpy as np\nimport pandas as pd\nimport torch\nfrom torch.optim import SGD, lr_scheduler\n\nfrom pytorch_widedeep import Trainer\nfrom pytorch_widedeep.preprocessing import TabPreprocessor\nfrom pytorch_widedeep.models import TabMlp, WideDeep\nfrom pytorch_widedeep.dataloaders import DataLoaderImbalanced, DataLoaderDefault\nfrom torchmetrics import F1Score as F1_torchmetrics\nfrom torchmetrics import Accuracy as Accuracy_torchmetrics\nfrom torchmetrics import Precision as Precision_torchmetrics\nfrom torchmetrics import Recall as Recall_torchmetrics\nfrom pytorch_widedeep.metrics import Accuracy, Recall, Precision, F1Score, R2Score\nfrom pytorch_widedeep.initializers import XavierNormal\nfrom pytorch_widedeep.datasets import load_bio_kdd04\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import classification_report\n\nimport time\nimport datetime\n\nimport warnings\n\nwarnings.filterwarnings(\"ignore\", category=DeprecationWarning)\n\n# increase displayed columns in jupyter notebook\npd.set_option(\"display.max_columns\", 200)\npd.set_option(\"display.max_rows\", 300)\n</pre> import numpy as np import pandas as pd import torch from torch.optim import SGD, lr_scheduler  from pytorch_widedeep import Trainer from pytorch_widedeep.preprocessing import TabPreprocessor from pytorch_widedeep.models import TabMlp, WideDeep from pytorch_widedeep.dataloaders import DataLoaderImbalanced, DataLoaderDefault from torchmetrics import F1Score as F1_torchmetrics from torchmetrics import Accuracy as Accuracy_torchmetrics from torchmetrics import Precision as Precision_torchmetrics from torchmetrics import Recall as Recall_torchmetrics from pytorch_widedeep.metrics import Accuracy, Recall, Precision, F1Score, R2Score from pytorch_widedeep.initializers import XavierNormal from pytorch_widedeep.datasets import load_bio_kdd04  from sklearn.model_selection import train_test_split from sklearn.metrics import classification_report  import time import datetime  import warnings  warnings.filterwarnings(\"ignore\", category=DeprecationWarning)  # increase displayed columns in jupyter notebook pd.set_option(\"display.max_columns\", 200) pd.set_option(\"display.max_rows\", 300) In\u00a0[2]: Copied! <pre>df = load_bio_kdd04(as_frame=True)\ndf.head()\n</pre> df = load_bio_kdd04(as_frame=True) df.head() Out[2]: EXAMPLE_ID BLOCK_ID target 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 0 279 261532 0 52.0 32.69 0.30 2.5 20.0 1256.8 -0.89 0.33 11.0 -55.0 267.2 0.52 0.05 -2.36 49.6 252.0 0.43 1.16 -2.06 -33.0 -123.2 1.60 -0.49 -6.06 65.0 296.1 -0.28 -0.26 -3.83 -22.6 -170.0 3.06 -1.05 -3.29 22.9 286.3 0.12 2.58 4.08 -33.0 -178.9 1.88 0.53 -7.0 -44.0 1987.0 -5.41 0.95 -4.0 -57.0 722.9 -3.26 -0.55 -7.5 125.5 1547.2 -0.36 1.12 9.0 -37.0 72.5 0.47 0.74 -11.0 -8.0 1595.1 -1.64 2.83 -2.0 -50.0 445.2 -0.35 0.26 0.76 1 279 261533 0 58.0 33.33 0.00 16.5 9.5 608.1 0.50 0.07 20.5 -52.5 521.6 -1.08 0.58 -0.02 -3.2 103.6 -0.95 0.23 -2.87 -25.9 -52.2 -0.21 0.87 -1.81 10.4 62.0 -0.28 -0.04 1.48 -17.6 -198.3 3.43 2.84 5.87 -16.9 72.6 -0.31 2.79 2.71 -33.5 -11.6 -1.11 4.01 5.0 -57.0 666.3 1.13 4.38 5.0 -64.0 39.3 1.07 -0.16 32.5 100.0 1893.7 -2.80 -0.22 2.5 -28.5 45.0 0.58 0.41 -19.0 -6.0 762.9 0.29 0.82 -3.0 -35.0 140.3 1.16 0.39 0.73 2 279 261534 0 77.0 27.27 -0.91 6.0 58.5 1623.6 -1.40 0.02 -6.5 -48.0 621.0 -1.20 0.14 -0.20 73.6 609.1 -0.44 -0.58 -0.04 -23.0 -27.4 -0.72 -1.04 -1.09 91.1 635.6 -0.88 0.24 0.59 -18.7 -7.2 -0.60 -2.82 -0.71 52.4 504.1 0.89 -0.67 -9.30 -20.8 -25.7 -0.77 -0.85 0.0 -20.0 2259.0 -0.94 1.15 -4.0 -44.0 -22.7 0.94 -0.98 -19.0 105.0 1267.9 1.03 1.27 11.0 -39.5 82.3 0.47 -0.19 -10.0 7.0 1491.8 0.32 -1.29 0.0 -34.0 658.2 -0.76 0.26 0.24 3 279 261535 0 41.0 27.91 -0.35 3.0 46.0 1921.6 -1.36 -0.47 -32.0 -51.5 560.9 -0.29 -0.10 -1.11 124.3 791.6 0.00 0.39 -1.85 -21.7 -44.9 -0.21 0.02 0.89 133.9 797.8 -0.08 1.06 -0.26 -16.4 -74.1 0.97 -0.80 -0.41 66.9 955.3 -1.90 1.28 -6.65 -28.1 47.5 -1.91 1.42 1.0 -30.0 1846.7 0.76 1.10 -4.0 -52.0 -53.9 1.71 -0.22 -12.0 97.5 1969.8 -1.70 0.16 -1.0 -32.5 255.9 -0.46 1.57 10.0 6.0 2047.7 -0.98 1.53 0.0 -49.0 554.2 -0.83 0.39 0.73 4 279 261536 0 50.0 28.00 -1.32 -9.0 12.0 464.8 0.88 0.19 8.0 -51.5 98.1 1.09 -0.33 -2.16 -3.9 102.7 0.39 -1.22 -3.39 -15.2 -42.2 -1.18 -1.11 -3.55 8.9 141.3 -0.16 -0.43 -4.15 -12.9 -13.4 -1.32 -0.98 -3.69 8.8 136.1 -0.30 4.13 1.89 -13.0 -18.7 -1.37 -0.93 0.0 -1.0 810.1 -2.29 6.72 1.0 -23.0 -29.7 0.58 -1.10 -18.5 33.5 206.8 1.84 -0.13 4.0 -29.0 30.1 0.80 -0.24 5.0 -14.0 479.5 0.68 -0.59 2.0 -36.0 -6.9 2.02 0.14 -0.23 In\u00a0[3]: Copied! <pre># imbalance of the classes\ndf[\"target\"].value_counts()\n</pre> # imbalance of the classes df[\"target\"].value_counts() Out[3]: <pre>target\n0    144455\n1      1296\nName: count, dtype: int64</pre> In\u00a0[4]: Copied! <pre># drop columns we won't need in this example\ndf.drop(columns=[\"EXAMPLE_ID\", \"BLOCK_ID\"], inplace=True)\n</pre> # drop columns we won't need in this example df.drop(columns=[\"EXAMPLE_ID\", \"BLOCK_ID\"], inplace=True) In\u00a0[5]: Copied! <pre>df_train, df_valid = train_test_split(\n    df, test_size=0.2, stratify=df[\"target\"], random_state=1\n)\ndf_valid, df_test = train_test_split(\n    df_valid, test_size=0.5, stratify=df_valid[\"target\"], random_state=1\n)\n</pre> df_train, df_valid = train_test_split(     df, test_size=0.2, stratify=df[\"target\"], random_state=1 ) df_valid, df_test = train_test_split(     df_valid, test_size=0.5, stratify=df_valid[\"target\"], random_state=1 ) In\u00a0[6]: Copied! <pre>continuous_cols = df.drop(columns=[\"target\"]).columns.values.tolist()\n</pre> continuous_cols = df.drop(columns=[\"target\"]).columns.values.tolist() In\u00a0[7]: Copied! <pre># deeptabular\ntab_preprocessor = TabPreprocessor(continuous_cols=continuous_cols, scale=True)\nX_tab_train = tab_preprocessor.fit_transform(df_train)\nX_tab_valid = tab_preprocessor.transform(df_valid)\nX_tab_test = tab_preprocessor.transform(df_test)\n\n# target\ny_train = df_train[\"target\"].values\ny_valid = df_valid[\"target\"].values\ny_test = df_test[\"target\"].values\n</pre> # deeptabular tab_preprocessor = TabPreprocessor(continuous_cols=continuous_cols, scale=True) X_tab_train = tab_preprocessor.fit_transform(df_train) X_tab_valid = tab_preprocessor.transform(df_valid) X_tab_test = tab_preprocessor.transform(df_test)  # target y_train = df_train[\"target\"].values y_valid = df_valid[\"target\"].values y_test = df_test[\"target\"].values In\u00a0[8]: Copied! <pre>input_layer = len(tab_preprocessor.continuous_cols)\noutput_layer = 1\nhidden_layers = np.linspace(\n    input_layer * 2, output_layer, 5, endpoint=False, dtype=int\n).tolist()\n</pre> input_layer = len(tab_preprocessor.continuous_cols) output_layer = 1 hidden_layers = np.linspace(     input_layer * 2, output_layer, 5, endpoint=False, dtype=int ).tolist() In\u00a0[9]: Copied! <pre>deeptabular = TabMlp(\n    mlp_hidden_dims=hidden_layers,\n    column_idx=tab_preprocessor.column_idx,\n    continuous_cols=tab_preprocessor.continuous_cols,\n)\nmodel = WideDeep(deeptabular=deeptabular, pred_dim=1)\nmodel\n</pre> deeptabular = TabMlp(     mlp_hidden_dims=hidden_layers,     column_idx=tab_preprocessor.column_idx,     continuous_cols=tab_preprocessor.continuous_cols, ) model = WideDeep(deeptabular=deeptabular, pred_dim=1) model Out[9]: <pre>WideDeep(\n  (deeptabular): Sequential(\n    (0): TabMlp(\n      (cont_norm): Identity()\n      (encoder): MLP(\n        (mlp): Sequential(\n          (dense_layer_0): Sequential(\n            (0): Linear(in_features=74, out_features=148, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_1): Sequential(\n            (0): Linear(in_features=148, out_features=118, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_2): Sequential(\n            (0): Linear(in_features=118, out_features=89, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_3): Sequential(\n            (0): Linear(in_features=89, out_features=59, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_4): Sequential(\n            (0): Linear(in_features=59, out_features=30, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=30, out_features=1, bias=True)\n  )\n)</pre> In\u00a0[10]: Copied! <pre># # Metrics from torchmetrics\n# accuracy = Accuracy_torchmetrics(average=None, num_classes=1)\n# precision = Precision_torchmetrics(average=\"micro\", num_classes=1)\n# f1 = F1_torchmetrics(average=None, num_classes=1)\n# recall = Recall_torchmetrics(average=None, num_classes=1)\n</pre> # # Metrics from torchmetrics # accuracy = Accuracy_torchmetrics(average=None, num_classes=1) # precision = Precision_torchmetrics(average=\"micro\", num_classes=1) # f1 = F1_torchmetrics(average=None, num_classes=1) # recall = Recall_torchmetrics(average=None, num_classes=1) In\u00a0[11]: Copied! <pre># Metrics from pytorch-widedeep\naccuracy = Accuracy(top_k=2)\nprecision = Precision(average=False)\nrecall = Recall(average=True)\nf1 = F1Score(average=False)\n</pre> # Metrics from pytorch-widedeep accuracy = Accuracy(top_k=2) precision = Precision(average=False) recall = Recall(average=True) f1 = F1Score(average=False) In\u00a0[12]: Copied! <pre># Optimizers\ndeep_opt = SGD(model.deeptabular.parameters(), lr=0.1)\n# LR Scheduler\ndeep_sch = lr_scheduler.StepLR(deep_opt, step_size=3)\n\ntrainer = Trainer(\n    model,\n    objective=\"binary\",\n    lr_schedulers={\"deeptabular\": deep_sch},\n    initializers={\"deeptabular\": XavierNormal},\n    optimizers={\"deeptabular\": deep_opt},\n    metrics=[accuracy, precision, recall, f1],\n    verbose=1,\n)\n</pre> # Optimizers deep_opt = SGD(model.deeptabular.parameters(), lr=0.1) # LR Scheduler deep_sch = lr_scheduler.StepLR(deep_opt, step_size=3)  trainer = Trainer(     model,     objective=\"binary\",     lr_schedulers={\"deeptabular\": deep_sch},     initializers={\"deeptabular\": XavierNormal},     optimizers={\"deeptabular\": deep_opt},     metrics=[accuracy, precision, recall, f1],     verbose=1, ) In\u00a0[13]: Copied! <pre>start = time.time()\ntrainer.fit(\n    X_train={\"X_tab\": X_tab_train, \"target\": y_train},\n    X_val={\"X_tab\": X_tab_valid, \"target\": y_valid},\n    n_epochs=3,\n    batch_size=50,\n    custom_dataloader=DataLoaderImbalanced,\n    oversample_mul=5,\n)\nprint(\n    \"Training time[s]: {}\".format(\n        datetime.timedelta(seconds=round(time.time() - start))\n    )\n)\n</pre> start = time.time() trainer.fit(     X_train={\"X_tab\": X_tab_train, \"target\": y_train},     X_val={\"X_tab\": X_tab_valid, \"target\": y_valid},     n_epochs=3,     batch_size=50,     custom_dataloader=DataLoaderImbalanced,     oversample_mul=5, ) print(     \"Training time[s]: {}\".format(         datetime.timedelta(seconds=round(time.time() - start))     ) ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 208/208 [00:01&lt;00:00, 130.75it/s, loss=0.187, metrics={'acc': 0.9214, 'prec': [0.9149], 'rec': 0.9318, 'f1': [0.9233]}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 292/292 [00:01&lt;00:00, 173.71it/s, loss=0.106, metrics={'acc': 0.9499, 'prec': [0.1435], 'rec': 0.938, 'f1': [0.249]}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 208/208 [00:01&lt;00:00, 139.68it/s, loss=0.109, metrics={'acc': 0.9559, 'prec': [0.9537], 'rec': 0.9572, 'f1': [0.9554]}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 292/292 [00:01&lt;00:00, 177.53it/s, loss=0.0888, metrics={'acc': 0.9602, 'prec': [0.1755], 'rec': 0.9457, 'f1': [0.2961]}]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 208/208 [00:01&lt;00:00, 141.63it/s, loss=0.08, metrics={'acc': 0.9706, 'prec': [0.9648], 'rec': 0.9766, 'f1': [0.9707]}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 292/292 [00:01&lt;00:00, 165.36it/s, loss=0.0969, metrics={'acc': 0.9564, 'prec': [0.1636], 'rec': 0.9535, 'f1': [0.2792]}]</pre> <pre>Training time[s]: 0:00:10\n</pre> <pre>\n</pre> In\u00a0[14]: Copied! <pre>pd.DataFrame(trainer.history)\n</pre> pd.DataFrame(trainer.history) Out[14]: train_loss train_acc train_prec train_rec train_f1 val_loss val_acc val_prec val_rec val_f1 0 0.186707 0.921408 [0.9149412512779236] 0.931801 [0.9232940673828125] 0.106023 0.949914 [0.14353498816490173] 0.937984 [0.24897116422653198] 1 0.109498 0.955931 [0.9536514282226562] 0.957193 [0.9554190039634705] 0.088787 0.960206 [0.17553956806659698] 0.945736 [0.29611650109291077] 2 0.079979 0.970588 [0.9648183584213257] 0.976582 [0.9706646203994751] 0.096858 0.956432 [0.1635638326406479] 0.953488 [0.279228150844574] In\u00a0[15]: Copied! <pre>df_pred = trainer.predict(X_tab=X_tab_test)\nprint(classification_report(df_test[\"target\"].to_list(), df_pred))\nprint(\"Actual predicted values:\\n{}\".format(np.unique(df_pred, return_counts=True)))\n</pre> df_pred = trainer.predict(X_tab=X_tab_test) print(classification_report(df_test[\"target\"].to_list(), df_pred)) print(\"Actual predicted values:\\n{}\".format(np.unique(df_pred, return_counts=True))) <pre>predict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 292/292 [00:00&lt;00:00, 346.55it/s]\n</pre> <pre>              precision    recall  f1-score   support\n\n           0       1.00      0.96      0.98     14446\n           1       0.17      0.95      0.29       130\n\n    accuracy                           0.96     14576\n   macro avg       0.58      0.95      0.63     14576\nweighted avg       0.99      0.96      0.97     14576\n\nActual predicted values:\n(array([0, 1]), array([13845,   731]))\n</pre> In\u00a0[16]: Copied! <pre>df_pred_unc = trainer.predict_uncertainty(X_tab=X_tab_test, uncertainty_granularity=10)\nprint(classification_report(df_test[\"target\"].to_list(), df_pred))\nprint(\n    \"Actual predicted values:\\n{}\".format(\n        np.unique(df_pred_unc[:, -1], return_counts=True)\n    )\n)\n</pre> df_pred_unc = trainer.predict_uncertainty(X_tab=X_tab_test, uncertainty_granularity=10) print(classification_report(df_test[\"target\"].to_list(), df_pred)) print(     \"Actual predicted values:\\n{}\".format(         np.unique(df_pred_unc[:, -1], return_counts=True)     ) ) <pre>predict_UncertaintyIter: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 10/10 [00:03&lt;00:00,  3.25it/s]</pre> <pre>              precision    recall  f1-score   support\n\n           0       1.00      0.96      0.98     14446\n           1       0.17      0.95      0.29       130\n\n    accuracy                           0.96     14576\n   macro avg       0.58      0.95      0.63     14576\nweighted avg       0.99      0.96      0.97     14576\n\nActual predicted values:\n(array([0.]), array([14576]))\n</pre> <pre>\n</pre> In\u00a0[17]: Copied! <pre>df_pred_unc\n</pre> df_pred_unc Out[17]: <pre>array([[9.98401165e-01, 1.59881881e-03, 0.00000000e+00],\n       [9.99941409e-01, 5.85634953e-05, 0.00000000e+00],\n       [9.97351170e-01, 2.64881272e-03, 0.00000000e+00],\n       ...,\n       [9.99494374e-01, 5.05603210e-04, 0.00000000e+00],\n       [9.99981642e-01, 1.83574630e-05, 0.00000000e+00],\n       [9.99996483e-01, 3.52600046e-06, 0.00000000e+00]])</pre>"},{"location":"examples/13_model_uncertainty_prediction.html#model-uncertainty-prediction","title":"Model Uncertainty prediction\u00b6","text":"<p>Note:</p> <p>This notebook extends the \"Custom DataLoader for Imbalanced dataset\" notebook</p>"},{"location":"examples/13_model_uncertainty_prediction.html#initial-imports","title":"Initial imports\u00b6","text":""},{"location":"examples/13_model_uncertainty_prediction.html#preparing-the-data","title":"Preparing the data\u00b6","text":""},{"location":"examples/13_model_uncertainty_prediction.html#define-the-model","title":"Define the model\u00b6","text":""},{"location":"examples/13_model_uncertainty_prediction.html#normal-prediction","title":"\"Normal\" prediction\u00b6","text":""},{"location":"examples/13_model_uncertainty_prediction.html#prediction-using-uncertainty","title":"Prediction using uncertainty\u00b6","text":""},{"location":"examples/14_bayesian_models.html","title":"14_bayesian_models","text":"In\u00a0[1]: Copied! <pre>import numpy as np\nimport torch\nimport pandas as pd\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import accuracy_score\n\nfrom pytorch_widedeep.metrics import Accuracy\nfrom pytorch_widedeep.datasets import load_adult\nfrom pytorch_widedeep.callbacks import EarlyStopping, ModelCheckpoint\nfrom pytorch_widedeep.preprocessing import TabPreprocessor, WidePreprocessor\nfrom pytorch_widedeep.bayesian_models import BayesianWide, BayesianTabMlp\nfrom pytorch_widedeep.training.bayesian_trainer import BayesianTrainer\n</pre> import numpy as np import torch import pandas as pd  from sklearn.model_selection import train_test_split from sklearn.metrics import accuracy_score  from pytorch_widedeep.metrics import Accuracy from pytorch_widedeep.datasets import load_adult from pytorch_widedeep.callbacks import EarlyStopping, ModelCheckpoint from pytorch_widedeep.preprocessing import TabPreprocessor, WidePreprocessor from pytorch_widedeep.bayesian_models import BayesianWide, BayesianTabMlp from pytorch_widedeep.training.bayesian_trainer import BayesianTrainer <p>The first few things to do we know them very well, like with any other model described in any of the other notebooks</p> In\u00a0[2]: Copied! <pre>df = load_adult(as_frame=True)\ndf.columns = [c.replace(\"-\", \"_\") for c in df.columns]\ndf[\"age_buckets\"] = pd.cut(\n    df.age, bins=[16, 25, 30, 35, 40, 45, 50, 55, 60, 91], labels=np.arange(9)\n)\ndf[\"income_label\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int)\ndf.drop(\"income\", axis=1, inplace=True)\ndf.head()\n</pre> df = load_adult(as_frame=True) df.columns = [c.replace(\"-\", \"_\") for c in df.columns] df[\"age_buckets\"] = pd.cut(     df.age, bins=[16, 25, 30, 35, 40, 45, 50, 55, 60, 91], labels=np.arange(9) ) df[\"income_label\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int) df.drop(\"income\", axis=1, inplace=True) df.head() Out[2]: age workclass fnlwgt education educational_num marital_status occupation relationship race gender capital_gain capital_loss hours_per_week native_country age_buckets income_label 0 25 Private 226802 11th 7 Never-married Machine-op-inspct Own-child Black Male 0 0 40 United-States 0 0 1 38 Private 89814 HS-grad 9 Married-civ-spouse Farming-fishing Husband White Male 0 0 50 United-States 3 0 2 28 Local-gov 336951 Assoc-acdm 12 Married-civ-spouse Protective-serv Husband White Male 0 0 40 United-States 1 1 3 44 Private 160323 Some-college 10 Married-civ-spouse Machine-op-inspct Husband Black Male 7688 0 40 United-States 4 1 4 18 ? 103497 Some-college 10 Never-married ? Own-child White Female 0 0 30 United-States 0 0 In\u00a0[3]: Copied! <pre>train, test = train_test_split(df, test_size=0.2, stratify=df.income_label)\n</pre> train, test = train_test_split(df, test_size=0.2, stratify=df.income_label) In\u00a0[4]: Copied! <pre>wide_cols = [\n    \"age_buckets\",\n    \"education\",\n    \"relationship\",\n    \"workclass\",\n    \"occupation\",\n    \"native_country\",\n    \"gender\",\n]\ncrossed_cols = [(\"education\", \"occupation\"), (\"native_country\", \"occupation\")]\n\ncat_embed_cols = [\n    \"workclass\",\n    \"education\",\n    \"marital_status\",\n    \"occupation\",\n    \"relationship\",\n    \"race\",\n    \"gender\",\n    \"capital_gain\",\n    \"capital_loss\",\n    \"native_country\",\n]\ncontinuous_cols = [\"age\", \"hours_per_week\"]\n\ntarget = train[\"income_label\"].values\n</pre> wide_cols = [     \"age_buckets\",     \"education\",     \"relationship\",     \"workclass\",     \"occupation\",     \"native_country\",     \"gender\", ] crossed_cols = [(\"education\", \"occupation\"), (\"native_country\", \"occupation\")]  cat_embed_cols = [     \"workclass\",     \"education\",     \"marital_status\",     \"occupation\",     \"relationship\",     \"race\",     \"gender\",     \"capital_gain\",     \"capital_loss\",     \"native_country\", ] continuous_cols = [\"age\", \"hours_per_week\"]  target = train[\"income_label\"].values In\u00a0[5]: Copied! <pre>wide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols)\nX_tab = wide_preprocessor.fit_transform(train)\n</pre> wide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols) X_tab = wide_preprocessor.fit_transform(train) In\u00a0[6]: Copied! <pre>model = BayesianWide(\n    input_dim=np.unique(X_tab).shape[0],\n    prior_sigma_1=1.0,\n    prior_sigma_2=0.002,\n    prior_pi=0.8,\n    posterior_mu_init=0,\n    posterior_rho_init=-7.0,\n    pred_dim=1,  # here the models are NOT passed to a WideDeep constructor class so the output dim MUST be specified\n)\n</pre> model = BayesianWide(     input_dim=np.unique(X_tab).shape[0],     prior_sigma_1=1.0,     prior_sigma_2=0.002,     prior_pi=0.8,     posterior_mu_init=0,     posterior_rho_init=-7.0,     pred_dim=1,  # here the models are NOT passed to a WideDeep constructor class so the output dim MUST be specified ) In\u00a0[7]: Copied! <pre>trainer = BayesianTrainer(\n    model,\n    objective=\"binary\",\n    optimizer=torch.optim.Adam(model.parameters(), lr=0.01),\n    metrics=[Accuracy],\n)\n</pre> trainer = BayesianTrainer(     model,     objective=\"binary\",     optimizer=torch.optim.Adam(model.parameters(), lr=0.01),     metrics=[Accuracy], ) In\u00a0[8]: Copied! <pre>trainer.fit(\n    X_tab=X_tab,\n    target=target,\n    val_split=0.2,\n    n_epochs=2,\n    batch_size=256,\n)\n</pre> trainer.fit(     X_tab=X_tab,     target=target,     val_split=0.2,     n_epochs=2,     batch_size=256, ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 123/123 [00:00&lt;00:00, 124.32it/s, loss=163, metrics={'acc': 0.7813}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 31/31 [00:00&lt;00:00, 238.67it/s, loss=141, metrics={'acc': 0.8219}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 123/123 [00:00&lt;00:00, 132.81it/s, loss=140, metrics={'acc': 0.8285}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 31/31 [00:00&lt;00:00, 190.16it/s, loss=140, metrics={'acc': 0.8298}]\n</pre> In\u00a0[9]: Copied! <pre>tab_preprocessor = TabPreprocessor(\n    cat_embed_cols=cat_embed_cols, continuous_cols=continuous_cols\n)\nX_tab = tab_preprocessor.fit_transform(train)\n</pre> tab_preprocessor = TabPreprocessor(     cat_embed_cols=cat_embed_cols, continuous_cols=continuous_cols ) X_tab = tab_preprocessor.fit_transform(train) <pre>/Users/javierrodriguezzaurin/Projects/pytorch-widedeep/pytorch_widedeep/preprocessing/tab_preprocessor.py:358: UserWarning: Continuous columns will not be normalised\n  warnings.warn(\"Continuous columns will not be normalised\")\n</pre> In\u00a0[10]: Copied! <pre>model = BayesianTabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=continuous_cols,\n    #     embed_continuous_method = \"standard\",\n    #     cont_embed_activation=\"leaky_relu\",\n    #     cont_embed_dim = 8,\n    mlp_hidden_dims=[128, 64],\n    prior_sigma_1=1.0,\n    prior_sigma_2=0.002,\n    prior_pi=0.8,\n    posterior_mu_init=0,\n    posterior_rho_init=-7.0,\n    pred_dim=1,\n)\n</pre> model = BayesianTabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     continuous_cols=continuous_cols,     #     embed_continuous_method = \"standard\",     #     cont_embed_activation=\"leaky_relu\",     #     cont_embed_dim = 8,     mlp_hidden_dims=[128, 64],     prior_sigma_1=1.0,     prior_sigma_2=0.002,     prior_pi=0.8,     posterior_mu_init=0,     posterior_rho_init=-7.0,     pred_dim=1, ) In\u00a0[11]: Copied! <pre>trainer = BayesianTrainer(\n    model,\n    objective=\"binary\",\n    optimizer=torch.optim.Adam(model.parameters(), lr=0.01),\n    metrics=[Accuracy],\n)\n</pre> trainer = BayesianTrainer(     model,     objective=\"binary\",     optimizer=torch.optim.Adam(model.parameters(), lr=0.01),     metrics=[Accuracy], ) In\u00a0[12]: Copied! <pre>trainer.fit(\n    X_tab=X_tab,\n    target=target,\n    val_split=0.2,\n    n_epochs=2,\n    batch_size=256,\n)\n</pre> trainer.fit(     X_tab=X_tab,     target=target,     val_split=0.2,     n_epochs=2,     batch_size=256, ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 123/123 [00:04&lt;00:00, 28.74it/s, loss=2e+3, metrics={'acc': 0.8007}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 31/31 [00:00&lt;00:00, 136.89it/s, loss=1.75e+3, metrics={'acc': 0.8418}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 123/123 [00:04&lt;00:00, 29.41it/s, loss=1.73e+3, metrics={'acc': 0.8596}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 31/31 [00:00&lt;00:00, 143.87it/s, loss=1.71e+3, metrics={'acc': 0.8569}]\n</pre> <p>These models are powerful beyond the success metrics because they give us a sense of uncertainty as we predict. Let's have a look</p> In\u00a0[13]: Copied! <pre>X_tab_test = tab_preprocessor.transform(test)\n</pre> X_tab_test = tab_preprocessor.transform(test) In\u00a0[14]: Copied! <pre>preds = trainer.predict(X_tab_test, return_samples=True, n_samples=5)\n</pre> preds = trainer.predict(X_tab_test, return_samples=True, n_samples=5) <pre>predict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 39/39 [00:01&lt;00:00, 33.92it/s]\n</pre> In\u00a0[15]: Copied! <pre>preds.shape\n</pre> preds.shape Out[15]: <pre>(5, 9769)</pre> <p>as we can see the prediction have shape <code>(5, 9769)</code>, one set of predictions each time we have internally run predict (i.e. sample the network and predict, defined by the parameter <code>n_samples</code>). This gives us an idea of how certain the model is about a certain prediction.</p> <p>Similarly, we could obtain the probabilities</p> In\u00a0[16]: Copied! <pre>probs = trainer.predict_proba(X_tab_test, return_samples=True, n_samples=5)\n</pre> probs = trainer.predict_proba(X_tab_test, return_samples=True, n_samples=5) <pre>predict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 39/39 [00:01&lt;00:00, 32.79it/s]\n</pre> In\u00a0[17]: Copied! <pre>probs.shape\n</pre> probs.shape Out[17]: <pre>(5, 9769, 2)</pre> <p>And we could see how the model performs each time we sampled the network</p> In\u00a0[18]: Copied! <pre>for p in preds:\n    print(accuracy_score(p, test[\"income_label\"].values))\n</pre> for p in preds:     print(accuracy_score(p, test[\"income_label\"].values)) <pre>0.8559729757395844\n0.8564847988535162\n0.8567918927218753\n0.8562800696079435\n0.8558706111167981\n</pre> In\u00a0[\u00a0]: Copied! <pre>\n</pre>"},{"location":"examples/14_bayesian_models.html#the-bayesian-models","title":"The Bayesian Models\u00b6","text":"<p>Perhaps one of the most interesting functionality in the library is the access to full Bayesian models in almost exactly the same way one would use any of the other models in the library.</p> <p>Note however that the Bayesian models are ONLY available for tabular data and, at the moment, we do not support combining them to form a Wide and Deep model.</p> <p>The implementation in this library is based on the publication: Weight Uncertainty in Neural Networks, by Blundell et al., 2015. Code-wise, our implementation is inspired by a number of source:</p> <ol> <li>https://joshfeldman.net/WeightUncertainty/</li> <li>https://www.nitarshan.com/bayes-by-backprop/</li> <li>https://github.com/piEsposito/blitz-bayesian-deep-learning</li> <li>https://github.com/zackchase/mxnet-the-straight-dope/tree/master/chapter18_variational-methods-and-uncertainty</li> </ol> <p>The two Bayesian models available in the library are:</p> <ol> <li>BayesianWide: this is a linear model where the non-linearities are captured via crossed-columns</li> <li>BayesianMLP: this is a standard MLP that receives categorical embeddings and continuous cols (embedded or not) which are the passed through a series of dense layers. All parameters in the model are probabilistic.</li> </ol>"},{"location":"examples/14_bayesian_models.html#1-bayesianwide","title":"1. <code>BayesianWide</code>\u00b6","text":""},{"location":"examples/14_bayesian_models.html#2-bayesiantabmlp","title":"2. <code>BayesianTabMlp</code>\u00b6","text":""},{"location":"examples/15_Self_Supervised_Pretraning_pt1.html","title":"15 Self Supervised Pretraning pt1","text":"In\u00a0[1]: Copied! <pre>import torch\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.model_selection import train_test_split\n\nfrom pytorch_widedeep import Trainer\nfrom pytorch_widedeep.models import TabMlp, WideDeep\nfrom pytorch_widedeep.metrics import Accuracy\nfrom pytorch_widedeep.datasets import load_adult\nfrom pytorch_widedeep.preprocessing import TabPreprocessor\nfrom pytorch_widedeep.self_supervised_training import EncoderDecoderTrainer\n</pre> import torch from sklearn.metrics import accuracy_score from sklearn.model_selection import train_test_split  from pytorch_widedeep import Trainer from pytorch_widedeep.models import TabMlp, WideDeep from pytorch_widedeep.metrics import Accuracy from pytorch_widedeep.datasets import load_adult from pytorch_widedeep.preprocessing import TabPreprocessor from pytorch_widedeep.self_supervised_training import EncoderDecoderTrainer In\u00a0[2]: Copied! <pre>df = load_adult(as_frame=True)\ndf.columns = [c.replace(\"-\", \"_\") for c in df.columns]\ndf[\"income_label\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int)\ndf.drop(\"income\", axis=1, inplace=True)\n</pre> df = load_adult(as_frame=True) df.columns = [c.replace(\"-\", \"_\") for c in df.columns] df[\"income_label\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int) df.drop(\"income\", axis=1, inplace=True) In\u00a0[3]: Copied! <pre># one could chose to use a validation set for early stopping, hyperparam\n# optimization, etc. This is just an example, so we simply use train/test\n# split\ndf_tr, df_te = train_test_split(df, test_size=0.2, stratify=df.income_label)\n</pre> # one could chose to use a validation set for early stopping, hyperparam # optimization, etc. This is just an example, so we simply use train/test # split df_tr, df_te = train_test_split(df, test_size=0.2, stratify=df.income_label) In\u00a0[4]: Copied! <pre>df_tr.head(2)\n</pre> df_tr.head(2) Out[4]: age workclass fnlwgt education educational_num marital_status occupation relationship race gender capital_gain capital_loss hours_per_week native_country income_label 9042 26 Local-gov 250551 HS-grad 9 Married-civ-spouse Craft-repair Own-child Black Male 0 0 40 United-States 0 25322 50 Private 34832 Bachelors 13 Married-civ-spouse Tech-support Husband White Male 15024 0 40 United-States 1 In\u00a0[5]: Copied! <pre># As always, we need to define which cols will be represented as embeddings\n# and which one will be continuous features\ncat_embed_cols = [\n    \"workclass\",\n    \"education\",\n    \"marital_status\",\n    \"occupation\",\n    \"relationship\",\n    \"race\",\n    \"gender\",\n    \"capital_gain\",\n    \"capital_loss\",\n    \"native_country\",\n]\ncontinuous_cols = [\"age\", \"hours_per_week\"]\ntarget_col = \"income_label\"\n</pre> # As always, we need to define which cols will be represented as embeddings # and which one will be continuous features cat_embed_cols = [     \"workclass\",     \"education\",     \"marital_status\",     \"occupation\",     \"relationship\",     \"race\",     \"gender\",     \"capital_gain\",     \"capital_loss\",     \"native_country\", ] continuous_cols = [\"age\", \"hours_per_week\"] target_col = \"income_label\" In\u00a0[6]: Copied! <pre># We prepare the data to be passed to the model\ntab_preprocessor = TabPreprocessor(\n    cat_embed_cols=cat_embed_cols, continuous_cols=continuous_cols\n)\nX_tab = tab_preprocessor.fit_transform(df_tr)\ntarget = df_tr[target_col].values\n</pre> # We prepare the data to be passed to the model tab_preprocessor = TabPreprocessor(     cat_embed_cols=cat_embed_cols, continuous_cols=continuous_cols ) X_tab = tab_preprocessor.fit_transform(df_tr) target = df_tr[target_col].values <pre>/Users/javierrodriguezzaurin/Projects/pytorch-widedeep/pytorch_widedeep/preprocessing/tab_preprocessor.py:358: UserWarning: Continuous columns will not be normalised\n  warnings.warn(\"Continuous columns will not be normalised\")\n</pre> In\u00a0[7]: Copied! <pre>X_tab[:5]\n</pre> X_tab[:5] Out[7]: <pre>array([[ 1,  1,  1,  1,  1,  1,  1,  1,  1,  1, 26, 40],\n       [ 2,  2,  1,  2,  2,  2,  1,  2,  1,  1, 50, 40],\n       [ 2,  1,  1,  3,  2,  2,  1,  1,  2,  1, 39, 46],\n       [ 2,  3,  2,  4,  1,  2,  2,  1,  1,  1, 17, 10],\n       [ 3,  4,  2,  1,  1,  2,  1,  1,  1,  1, 32, 20]])</pre> In\u00a0[8]: Copied! <pre># We define a model that will act as the encoder in the encoder/decoder\n# architecture. This could be any of: TabMlp, TabResnet or TabNet\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=tab_preprocessor.continuous_cols,\n)\n</pre> # We define a model that will act as the encoder in the encoder/decoder # architecture. This could be any of: TabMlp, TabResnet or TabNet tab_mlp = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     continuous_cols=tab_preprocessor.continuous_cols, ) In\u00a0[9]: Copied! <pre>tab_mlp\n</pre> tab_mlp Out[9]: <pre>TabMlp(\n  (cat_embed): DiffSizeCatEmbeddings(\n    (embed_layers): ModuleDict(\n      (emb_layer_workclass): Embedding(10, 5, padding_idx=0)\n      (emb_layer_education): Embedding(17, 8, padding_idx=0)\n      (emb_layer_marital_status): Embedding(8, 5, padding_idx=0)\n      (emb_layer_occupation): Embedding(16, 7, padding_idx=0)\n      (emb_layer_relationship): Embedding(7, 4, padding_idx=0)\n      (emb_layer_race): Embedding(6, 4, padding_idx=0)\n      (emb_layer_gender): Embedding(3, 2, padding_idx=0)\n      (emb_layer_capital_gain): Embedding(124, 24, padding_idx=0)\n      (emb_layer_capital_loss): Embedding(98, 21, padding_idx=0)\n      (emb_layer_native_country): Embedding(42, 13, padding_idx=0)\n    )\n    (embedding_dropout): Dropout(p=0.0, inplace=False)\n  )\n  (cont_norm): Identity()\n  (encoder): MLP(\n    (mlp): Sequential(\n      (dense_layer_0): Sequential(\n        (0): Linear(in_features=95, out_features=200, bias=True)\n        (1): ReLU(inplace=True)\n        (2): Dropout(p=0.1, inplace=False)\n      )\n      (dense_layer_1): Sequential(\n        (0): Linear(in_features=200, out_features=100, bias=True)\n        (1): ReLU(inplace=True)\n        (2): Dropout(p=0.1, inplace=False)\n      )\n    )\n  )\n)</pre> In\u00a0[10]: Copied! <pre># If we do not pass a custom decoder, which is perfectly possible via the\n# decoder param,  the EncoderDecoderTrainer will automatically build a\n# decoder which will be the 'mirror' image of the encoder\nencoder_decoder_trainer = EncoderDecoderTrainer(encoder=tab_mlp)\n</pre> # If we do not pass a custom decoder, which is perfectly possible via the # decoder param,  the EncoderDecoderTrainer will automatically build a # decoder which will be the 'mirror' image of the encoder encoder_decoder_trainer = EncoderDecoderTrainer(encoder=tab_mlp) In\u00a0[11]: Copied! <pre># let's have a look to the encoder_decoder_model (aka ed_model)\nencoder_decoder_trainer.ed_model\n</pre> # let's have a look to the encoder_decoder_model (aka ed_model) encoder_decoder_trainer.ed_model Out[11]: <pre>EncoderDecoderModel(\n  (encoder): TabMlp(\n    (cat_embed): DiffSizeCatEmbeddings(\n      (embed_layers): ModuleDict(\n        (emb_layer_workclass): Embedding(10, 5, padding_idx=0)\n        (emb_layer_education): Embedding(17, 8, padding_idx=0)\n        (emb_layer_marital_status): Embedding(8, 5, padding_idx=0)\n        (emb_layer_occupation): Embedding(16, 7, padding_idx=0)\n        (emb_layer_relationship): Embedding(7, 4, padding_idx=0)\n        (emb_layer_race): Embedding(6, 4, padding_idx=0)\n        (emb_layer_gender): Embedding(3, 2, padding_idx=0)\n        (emb_layer_capital_gain): Embedding(124, 24, padding_idx=0)\n        (emb_layer_capital_loss): Embedding(98, 21, padding_idx=0)\n        (emb_layer_native_country): Embedding(42, 13, padding_idx=0)\n      )\n      (embedding_dropout): Dropout(p=0.0, inplace=False)\n    )\n    (cont_norm): Identity()\n    (encoder): MLP(\n      (mlp): Sequential(\n        (dense_layer_0): Sequential(\n          (0): Linear(in_features=95, out_features=200, bias=True)\n          (1): ReLU(inplace=True)\n          (2): Dropout(p=0.1, inplace=False)\n        )\n        (dense_layer_1): Sequential(\n          (0): Linear(in_features=200, out_features=100, bias=True)\n          (1): ReLU(inplace=True)\n          (2): Dropout(p=0.1, inplace=False)\n        )\n      )\n    )\n  )\n  (decoder): TabMlpDecoder(\n    (decoder): MLP(\n      (mlp): Sequential(\n        (dense_layer_0): Sequential(\n          (0): Linear(in_features=100, out_features=200, bias=True)\n          (1): ReLU(inplace=True)\n          (2): Dropout(p=0.1, inplace=False)\n        )\n        (dense_layer_1): Sequential(\n          (0): Linear(in_features=200, out_features=95, bias=True)\n          (1): ReLU(inplace=True)\n          (2): Dropout(p=0.1, inplace=False)\n        )\n      )\n    )\n  )\n  (masker): RandomObfuscator()\n)</pre> <p>Ignoring the <code>masker</code>, which just...well...masks, the <code>ed_model</code> consists of:</p> <ol> <li>An encoder model that is a <code>TabMlp</code> model that is in itself comprised by an Embedding layer (or rather a collection of them, referred as <code>cat_and_cont_embed</code>) and an encoder (a simple MLP, referred as <code>encoder</code>)</li> <li>A decoder which is just an \"inverted\" MLP (referred as <code>decoder</code>)</li> </ol> In\u00a0[12]: Copied! <pre># And we just...pretrain\nencoder_decoder_trainer.pretrain(X_tab, n_epochs=5, batch_size=256)\n</pre> # And we just...pretrain encoder_decoder_trainer.pretrain(X_tab, n_epochs=5, batch_size=256) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 82.90it/s, loss=4.07]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 89.87it/s, loss=3.09]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 92.86it/s, loss=2.53]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 91.24it/s, loss=2.09]\nepoch 5: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 91.38it/s, loss=1.78]\n</pre> <p>At this point we have two options, we could either save the model for later use or we could continue to supervised training. The latter is rather simple, after running:</p> <pre>encoder_decoder_trainer.pretrain(X_tab, n_epochs=5, batch_size=256)\n</pre> <p>you just have to</p> <pre>model = WideDeep(deeptabular=tab_mlp)\ntrainer = Trainer(model=model, objective=\"binary\", metrics=[Accuracy])\n\ntrainer.fit(X_tab=X_tab, target=target, n_epochs=5, batch_size=256)\n\n# And, you know...we get a test metric\nX_tab_te = tab_preprocessor.transform(df_te)\ntarget_te = df_te[target_col].values\n\npreds = trainer.predict(X_tab=X_tab_te)\ntest_acc = accuracy_score(target_te, preds)\n</pre> <p>Let's say that in any case, we are 'decent' scientists/people and we want to save the model:</p> In\u00a0[13]: Copied! <pre>encoder_decoder_trainer.save(\n    path=\"pretrained_weights\", model_filename=\"encoder_decoder_model.pt\"\n)\n</pre> encoder_decoder_trainer.save(     path=\"pretrained_weights\", model_filename=\"encoder_decoder_model.pt\" ) <p>some time has passed...</p> In\u00a0[14]: Copied! <pre>encoder_decoder_model = torch.load(\"pretrained_weights/encoder_decoder_model.pt\")\n</pre> encoder_decoder_model = torch.load(\"pretrained_weights/encoder_decoder_model.pt\") <p>Now, AND THIS IS IMPORTANT We have loaded the encoder AND the decoder. To proceed to the supervised training we ONLY need the encoder</p> In\u00a0[15]: Copied! <pre>pretrained_encoder = encoder_decoder_model.encoder\n</pre> pretrained_encoder = encoder_decoder_model.encoder In\u00a0[16]: Copied! <pre>pretrained_encoder\n</pre> pretrained_encoder Out[16]: <pre>TabMlp(\n  (cat_embed): DiffSizeCatEmbeddings(\n    (embed_layers): ModuleDict(\n      (emb_layer_workclass): Embedding(10, 5, padding_idx=0)\n      (emb_layer_education): Embedding(17, 8, padding_idx=0)\n      (emb_layer_marital_status): Embedding(8, 5, padding_idx=0)\n      (emb_layer_occupation): Embedding(16, 7, padding_idx=0)\n      (emb_layer_relationship): Embedding(7, 4, padding_idx=0)\n      (emb_layer_race): Embedding(6, 4, padding_idx=0)\n      (emb_layer_gender): Embedding(3, 2, padding_idx=0)\n      (emb_layer_capital_gain): Embedding(124, 24, padding_idx=0)\n      (emb_layer_capital_loss): Embedding(98, 21, padding_idx=0)\n      (emb_layer_native_country): Embedding(42, 13, padding_idx=0)\n    )\n    (embedding_dropout): Dropout(p=0.0, inplace=False)\n  )\n  (cont_norm): Identity()\n  (encoder): MLP(\n    (mlp): Sequential(\n      (dense_layer_0): Sequential(\n        (0): Linear(in_features=95, out_features=200, bias=True)\n        (1): ReLU(inplace=True)\n        (2): Dropout(p=0.1, inplace=False)\n      )\n      (dense_layer_1): Sequential(\n        (0): Linear(in_features=200, out_features=100, bias=True)\n        (1): ReLU(inplace=True)\n        (2): Dropout(p=0.1, inplace=False)\n      )\n    )\n  )\n)</pre> In\u00a0[17]: Copied! <pre># and as always, ANY supervised model in this library has to go throuth the WideDeep class:\nmodel = WideDeep(deeptabular=pretrained_encoder)\ntrainer = Trainer(model=model, objective=\"binary\", metrics=[Accuracy])\n\ntrainer.fit(X_tab=X_tab, target=target, n_epochs=5, batch_size=256)\n\nX_tab_te = tab_preprocessor.transform(df_te)\ntarget_te = df_te[target_col].values\n\npreds = trainer.predict(X_tab=X_tab_te)\ntest_acc = accuracy_score(target_te, preds)\nprint(test_acc)\n</pre> # and as always, ANY supervised model in this library has to go throuth the WideDeep class: model = WideDeep(deeptabular=pretrained_encoder) trainer = Trainer(model=model, objective=\"binary\", metrics=[Accuracy])  trainer.fit(X_tab=X_tab, target=target, n_epochs=5, batch_size=256)  X_tab_te = tab_preprocessor.transform(df_te) target_te = df_te[target_col].values  preds = trainer.predict(X_tab=X_tab_te) test_acc = accuracy_score(target_te, preds) print(test_acc) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 88.04it/s, loss=0.374, metrics={'acc': 0.8253}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 85.63it/s, loss=0.324, metrics={'acc': 0.8491}]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 87.56it/s, loss=0.301, metrics={'acc': 0.8608}]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:02&lt;00:00, 73.38it/s, loss=0.29, metrics={'acc': 0.8655}]\nepoch 5: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 78.68it/s, loss=0.284, metrics={'acc': 0.8686}]\npredict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 39/39 [00:00&lt;00:00, 173.02it/s]\n</pre> <pre>0.8730678677449074\n</pre> <p>As we mentioned before, we can also use a <code>TabResNet</code> or <code>TabNet</code> model and a custom decoder. Let's have a look:</p> In\u00a0[18]: Copied! <pre>from pytorch_widedeep.models import TabResnet as TabResnetEncoder, TabResnetDecoder\n</pre> from pytorch_widedeep.models import TabResnet as TabResnetEncoder, TabResnetDecoder In\u00a0[19]: Copied! <pre>resnet_encoder = TabResnetEncoder(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=continuous_cols,\n    blocks_dims=[200, 100, 100],\n)\n</pre> resnet_encoder = TabResnetEncoder(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     continuous_cols=continuous_cols,     blocks_dims=[200, 100, 100], ) <p>let's have a look to the model</p> In\u00a0[20]: Copied! <pre>resnet_encoder\n</pre> resnet_encoder Out[20]: <pre>TabResnet(\n  (cat_embed): DiffSizeCatEmbeddings(\n    (embed_layers): ModuleDict(\n      (emb_layer_workclass): Embedding(10, 5, padding_idx=0)\n      (emb_layer_education): Embedding(17, 8, padding_idx=0)\n      (emb_layer_marital_status): Embedding(8, 5, padding_idx=0)\n      (emb_layer_occupation): Embedding(16, 7, padding_idx=0)\n      (emb_layer_relationship): Embedding(7, 4, padding_idx=0)\n      (emb_layer_race): Embedding(6, 4, padding_idx=0)\n      (emb_layer_gender): Embedding(3, 2, padding_idx=0)\n      (emb_layer_capital_gain): Embedding(124, 24, padding_idx=0)\n      (emb_layer_capital_loss): Embedding(98, 21, padding_idx=0)\n      (emb_layer_native_country): Embedding(42, 13, padding_idx=0)\n    )\n    (embedding_dropout): Dropout(p=0.0, inplace=False)\n  )\n  (cont_norm): Identity()\n  (encoder): DenseResnet(\n    (dense_resnet): Sequential(\n      (lin_inp): Linear(in_features=95, out_features=200, bias=False)\n      (bn_inp): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n      (block_0): BasicBlock(\n        (resize): Sequential(\n          (0): Linear(in_features=200, out_features=100, bias=False)\n          (1): BatchNorm1d(100, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        )\n        (lin1): Linear(in_features=200, out_features=100, bias=False)\n        (bn1): BatchNorm1d(100, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (leaky_relu): LeakyReLU(negative_slope=0.01, inplace=True)\n        (dp): Dropout(p=0.1, inplace=False)\n        (lin2): Linear(in_features=100, out_features=100, bias=False)\n        (bn2): BatchNorm1d(100, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n      )\n      (block_1): BasicBlock(\n        (lin1): Linear(in_features=100, out_features=100, bias=False)\n        (bn1): BatchNorm1d(100, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (leaky_relu): LeakyReLU(negative_slope=0.01, inplace=True)\n        (dp): Dropout(p=0.1, inplace=False)\n        (lin2): Linear(in_features=100, out_features=100, bias=False)\n        (bn2): BatchNorm1d(100, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n      )\n    )\n  )\n)</pre> <p>As we can see, the tensor we are trying to reconstruct, the embeddings, is of size <code>94</code> (this number is stored in the attribute: <code>esnet_encoder.cat_and_cont_embed.output_dim</code>), with that information we could build or own decoder as:</p> In\u00a0[21]: Copied! <pre># for all possible params see the docs\nresnet_decoder = TabResnetDecoder(\n    embed_dim=resnet_encoder.cat_out_dim + resnet_encoder.cont_out_dim,\n    blocks_dims=[100, 100, 200],\n)\n</pre> # for all possible params see the docs resnet_decoder = TabResnetDecoder(     embed_dim=resnet_encoder.cat_out_dim + resnet_encoder.cont_out_dim,     blocks_dims=[100, 100, 200], ) In\u00a0[22]: Copied! <pre>resnet_decoder\n</pre> resnet_decoder Out[22]: <pre>TabResnetDecoder(\n  (decoder): DenseResnet(\n    (dense_resnet): Sequential(\n      (block_0): BasicBlock(\n        (lin1): Linear(in_features=100, out_features=100, bias=False)\n        (bn1): BatchNorm1d(100, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (leaky_relu): LeakyReLU(negative_slope=0.01, inplace=True)\n        (dp): Dropout(p=0.1, inplace=False)\n        (lin2): Linear(in_features=100, out_features=100, bias=False)\n        (bn2): BatchNorm1d(100, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n      )\n      (block_1): BasicBlock(\n        (resize): Sequential(\n          (0): Linear(in_features=100, out_features=200, bias=False)\n          (1): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        )\n        (lin1): Linear(in_features=100, out_features=200, bias=False)\n        (bn1): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (leaky_relu): LeakyReLU(negative_slope=0.01, inplace=True)\n        (dp): Dropout(p=0.1, inplace=False)\n        (lin2): Linear(in_features=200, out_features=200, bias=False)\n        (bn2): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n      )\n    )\n  )\n  (reconstruction_layer): Linear(in_features=200, out_features=95, bias=False)\n)</pre> <p>and now:</p> In\u00a0[23]: Copied! <pre>ec_trainer = EncoderDecoderTrainer(\n    encoder=resnet_encoder,\n    decoder=resnet_decoder,\n    masked_prob=0.2,\n)\nec_trainer.pretrain(X_tab, n_epochs=5, batch_size=256)\n</pre> ec_trainer = EncoderDecoderTrainer(     encoder=resnet_encoder,     decoder=resnet_decoder,     masked_prob=0.2, ) ec_trainer.pretrain(X_tab, n_epochs=5, batch_size=256) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:03&lt;00:00, 46.89it/s, loss=1.52]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:03&lt;00:00, 46.78it/s, loss=0.81]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:03&lt;00:00, 39.82it/s, loss=0.56]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:03&lt;00:00, 46.73it/s, loss=0.417]\nepoch 5: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:03&lt;00:00, 46.24it/s, loss=0.329]\n</pre> In\u00a0[24]: Copied! <pre># and as always, ANY supervised model in this library has to go throuth the WideDeep class:\nmodel = WideDeep(deeptabular=resnet_encoder)\ntrainer = Trainer(model=model, objective=\"binary\", metrics=[Accuracy])\n\ntrainer.fit(X_tab=X_tab, target=target, n_epochs=5, batch_size=256)\n\nX_tab_te = tab_preprocessor.transform(df_te)\ntarget_te = df_te[target_col].values\n\npreds = trainer.predict(X_tab=X_tab_te)\ntest_acc = accuracy_score(target_te, preds)\nprint(test_acc)\n</pre> # and as always, ANY supervised model in this library has to go throuth the WideDeep class: model = WideDeep(deeptabular=resnet_encoder) trainer = Trainer(model=model, objective=\"binary\", metrics=[Accuracy])  trainer.fit(X_tab=X_tab, target=target, n_epochs=5, batch_size=256)  X_tab_te = tab_preprocessor.transform(df_te) target_te = df_te[target_col].values  preds = trainer.predict(X_tab=X_tab_te) test_acc = accuracy_score(target_te, preds) print(test_acc) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:02&lt;00:00, 58.63it/s, loss=0.335, metrics={'acc': 0.8442}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:02&lt;00:00, 58.02it/s, loss=0.296, metrics={'acc': 0.864}]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:02&lt;00:00, 55.91it/s, loss=0.283, metrics={'acc': 0.8687}]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:02&lt;00:00, 55.00it/s, loss=0.276, metrics={'acc': 0.871}]\nepoch 5: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:02&lt;00:00, 51.95it/s, loss=0.272, metrics={'acc': 0.8732}]\npredict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 39/39 [00:00&lt;00:00, 120.15it/s]\n</pre> <pre>0.8725560446309756\n</pre> In\u00a0[\u00a0]: Copied! <pre>\n</pre>"},{"location":"examples/15_Self_Supervised_Pretraning_pt1.html#self-supervised-pretraining-for-tabular-data","title":"Self Supervised Pretraining for Tabular Data\u00b6","text":"<p>We have implemented two Self Supervised Pre-training routines that allow the user to pre-train all tabular models in the library with the exception of the TabPerceiver (which is a special monster).</p> <p>The two routines implemented are illustrated in the figures below. The 1st is from TabNet: Attentive Interpretable Tabular Learning and is designed for models that do not use transformer-based architectures, while the second is from SAINT: Improved Neural Networks for Tabular Data via Row Attention and Contrastive Pre-Training, and is designed for models that use transformer-based architectures.</p> <p> </p> <p>Fig 1. Figure 2 in their paper. I have included de original caption in case is useful, althought the Figure itself is pretty self explanatory</p> <p> </p> <p>Fig 2. Figure 1 in their paper. Here the caption is necessary \ud83d\ude0f</p> <p>It is beyond the scope of this notebook to explain in detail those implementations. Therefore, we strongly recommend the user to go and read the papers if this functionality is of interest to her/him.</p> <p>One thing is worth noticing however. As seen in Fig 1(the TabNet paper's Fig 2) the masking of the input features happens in the feature space. However, the implementation in this library is inspired by that at the dreamquark-ai repo, which is in itself inspired by the original implementation (by the way, at this point I will write it once again. All TabNet related things in this library are inspired when not directly based in the code in that repo, therefore, ALL CREDIT TO THE GUYS AT dreamquark-ai).</p> <p>In that implementation the masking happens in the embedding space, and currently does not mask the entire embedding (i.e. categorical feature). We decided to release as it is in this version and we will implement the exact same process described in the paper in future releases.</p> <p>Having said all of the above let's see how to use self supervision for tabular data with <code>pytorch-widedeep</code>. We will concentrate in this notebook on the 1st of the two approaches (the 'TabNet approach'). For details on the second approach please see <code>16_Self_Supervised_Pretraning_pt2</code>.</p>"},{"location":"examples/15_Self_Supervised_Pretraning_pt1.html#self-supervision-for-non-transformer-based-models","title":"Self Supervision for non-transformer-based models..\u00b6","text":"<p>...or in general, for models where the embeddigns can have all different dimensions. In this library, these are: <code>TabMlp</code>, <code>TabResNet</code> and <code>TabNet</code></p> <p>As shown in Figure, this is an encoder-encoder approach where we learn to predict values in the incoming data that have been masked. However, as I mentioned before, our implementation is a bit different, and the masking occurs in th embedding space.</p> <p>Nonetheless, the code below illustrates how to use this encoder-decoder approach with <code>pytorch-widedeep</code></p>"},{"location":"examples/15_Self_Supervised_Pretraning_pt2.html","title":"15 Self Supervised Pretraning pt2","text":"In\u00a0[1]: Copied! <pre>import torch\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.model_selection import train_test_split\n\nfrom pytorch_widedeep import Trainer\nfrom pytorch_widedeep.models import WideDeep, FTTransformer\nfrom pytorch_widedeep.metrics import Accuracy\nfrom pytorch_widedeep.datasets import load_adult\nfrom pytorch_widedeep.preprocessing import TabPreprocessor\nfrom pytorch_widedeep.self_supervised_training import (\n    ContrastiveDenoisingTrainer,\n)\n</pre> import torch from sklearn.metrics import accuracy_score from sklearn.model_selection import train_test_split  from pytorch_widedeep import Trainer from pytorch_widedeep.models import WideDeep, FTTransformer from pytorch_widedeep.metrics import Accuracy from pytorch_widedeep.datasets import load_adult from pytorch_widedeep.preprocessing import TabPreprocessor from pytorch_widedeep.self_supervised_training import (     ContrastiveDenoisingTrainer, ) In\u00a0[2]: Copied! <pre>df = load_adult(as_frame=True)\ndf.columns = [c.replace(\"-\", \"_\") for c in df.columns]\ndf[\"income_label\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int)\ndf.drop(\"income\", axis=1, inplace=True)\n\n# one could chose to use a validation set for early stopping, hyperparam\n# optimization, etc. This is just an example, so we simply use train/test\n# split\ndf_tr, df_te = train_test_split(df, test_size=0.2, stratify=df.income_label)\n\ncat_embed_cols = [\n    \"workclass\",\n    \"education\",\n    \"marital_status\",\n    \"occupation\",\n    \"relationship\",\n    \"race\",\n    \"gender\",\n    \"capital_gain\",\n    \"capital_loss\",\n    \"native_country\",\n]\ncontinuous_cols = [\"age\", \"hours_per_week\"]\ntarget_col = \"income_label\"\n\ntab_preprocessor = TabPreprocessor(\n    cat_embed_cols=cat_embed_cols,\n    continuous_cols=continuous_cols,\n    with_attention=True,\n    with_cls_token=True,  # this is optional\n)\nX_tab = tab_preprocessor.fit_transform(df_tr)\ntarget = df_tr[target_col].values\n</pre> df = load_adult(as_frame=True) df.columns = [c.replace(\"-\", \"_\") for c in df.columns] df[\"income_label\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int) df.drop(\"income\", axis=1, inplace=True)  # one could chose to use a validation set for early stopping, hyperparam # optimization, etc. This is just an example, so we simply use train/test # split df_tr, df_te = train_test_split(df, test_size=0.2, stratify=df.income_label)  cat_embed_cols = [     \"workclass\",     \"education\",     \"marital_status\",     \"occupation\",     \"relationship\",     \"race\",     \"gender\",     \"capital_gain\",     \"capital_loss\",     \"native_country\", ] continuous_cols = [\"age\", \"hours_per_week\"] target_col = \"income_label\"  tab_preprocessor = TabPreprocessor(     cat_embed_cols=cat_embed_cols,     continuous_cols=continuous_cols,     with_attention=True,     with_cls_token=True,  # this is optional ) X_tab = tab_preprocessor.fit_transform(df_tr) target = df_tr[target_col].values <pre>/Users/javierrodriguezzaurin/Projects/pytorch-widedeep/pytorch_widedeep/preprocessing/tab_preprocessor.py:358: UserWarning: Continuous columns will not be normalised\n  warnings.warn(\"Continuous columns will not be normalised\")\n</pre> In\u00a0[3]: Copied! <pre>ft_transformer = FTTransformer(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=tab_preprocessor.continuous_cols,\n    embed_continuous_method=\"standard\",\n    input_dim=32,\n    kv_compression_factor=0.5,\n    n_blocks=3,\n    n_heads=4,\n)\n</pre> ft_transformer = FTTransformer(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     continuous_cols=tab_preprocessor.continuous_cols,     embed_continuous_method=\"standard\",     input_dim=32,     kv_compression_factor=0.5,     n_blocks=3,     n_heads=4, ) In\u00a0[4]: Copied! <pre># for a full list of the params for the the ContrastiveDenoisingTrainer (which are many) please see the docs.\n# Note that using these params involves some knowledge of the routine and the architecture of the model used\ncontrastive_denoising_trainer = ContrastiveDenoisingTrainer(\n    model=ft_transformer,\n    preprocessor=tab_preprocessor,\n)\ncontrastive_denoising_trainer.pretrain(X_tab, n_epochs=5, batch_size=256)\n</pre> # for a full list of the params for the the ContrastiveDenoisingTrainer (which are many) please see the docs. # Note that using these params involves some knowledge of the routine and the architecture of the model used contrastive_denoising_trainer = ContrastiveDenoisingTrainer(     model=ft_transformer,     preprocessor=tab_preprocessor, ) contrastive_denoising_trainer.pretrain(X_tab, n_epochs=5, batch_size=256) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:13&lt;00:00, 11.73it/s, loss=579]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:12&lt;00:00, 12.56it/s, loss=143]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:12&lt;00:00, 12.49it/s, loss=141]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:11&lt;00:00, 12.77it/s, loss=138]\nepoch 5: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:11&lt;00:00, 13.29it/s, loss=137]\n</pre> In\u00a0[5]: Copied! <pre>contrastive_denoising_trainer.save(\n    path=\"pretrained_weights\", model_filename=\"contrastive_denoising_model.pt\"\n)\n</pre> contrastive_denoising_trainer.save(     path=\"pretrained_weights\", model_filename=\"contrastive_denoising_model.pt\" ) <p>some time has passed</p> In\u00a0[6]: Copied! <pre># some time has passed, we load the model with torch as usual:\ncontrastive_denoising_model = torch.load(\n    \"pretrained_weights/contrastive_denoising_model.pt\"\n)\n</pre> # some time has passed, we load the model with torch as usual: contrastive_denoising_model = torch.load(     \"pretrained_weights/contrastive_denoising_model.pt\" ) <p>NOW, AND THIS IS IMPORTANT! We have loaded the entire contrastive, denoising model. To proceed to the supervised training we ONLY need the attention-based model, which is the 'model' attribute of the trainer, let's have a look</p> In\u00a0[7]: Copied! <pre>contrastive_denoising_model.model\n</pre> contrastive_denoising_model.model Out[7]: <pre>FTTransformer(\n  (cat_embed): SameSizeCatEmbeddings(\n    (embed): Embedding(323, 32, padding_idx=0)\n    (dropout): Dropout(p=0.0, inplace=False)\n  )\n  (cont_norm): Identity()\n  (cont_embed): ContEmbeddings(\n    INFO: [ContLinear = weight(n_cont_cols, embed_dim) + bias(n_cont_cols, embed_dim)]\n    (linear): ContLinear(n_cont_cols=2, embed_dim=32, embed_dropout=0.0)\n    (dropout): Dropout(p=0.0, inplace=False)\n  )\n  (encoder): Sequential(\n    (fttransformer_block0): FTTransformerEncoder(\n      (attn): LinearAttentionLinformer(\n        (dropout): Dropout(p=0.2, inplace=False)\n        (qkv_proj): Linear(in_features=32, out_features=96, bias=False)\n        (out_proj): Linear(in_features=32, out_features=32, bias=False)\n      )\n      (ff): FeedForward(\n        (w_1): Linear(in_features=32, out_features=84, bias=True)\n        (w_2): Linear(in_features=42, out_features=32, bias=True)\n        (dropout): Dropout(p=0.1, inplace=False)\n        (activation): REGLU()\n      )\n      (attn_normadd): NormAdd(\n        (dropout): Dropout(p=0.2, inplace=False)\n        (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n      )\n      (ff_normadd): NormAdd(\n        (dropout): Dropout(p=0.1, inplace=False)\n        (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n      )\n    )\n    (fttransformer_block1): FTTransformerEncoder(\n      (attn): LinearAttentionLinformer(\n        (dropout): Dropout(p=0.2, inplace=False)\n        (qkv_proj): Linear(in_features=32, out_features=96, bias=False)\n        (out_proj): Linear(in_features=32, out_features=32, bias=False)\n      )\n      (ff): FeedForward(\n        (w_1): Linear(in_features=32, out_features=84, bias=True)\n        (w_2): Linear(in_features=42, out_features=32, bias=True)\n        (dropout): Dropout(p=0.1, inplace=False)\n        (activation): REGLU()\n      )\n      (attn_normadd): NormAdd(\n        (dropout): Dropout(p=0.2, inplace=False)\n        (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n      )\n      (ff_normadd): NormAdd(\n        (dropout): Dropout(p=0.1, inplace=False)\n        (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n      )\n    )\n    (fttransformer_block2): FTTransformerEncoder(\n      (attn): LinearAttentionLinformer(\n        (dropout): Dropout(p=0.2, inplace=False)\n        (qkv_proj): Linear(in_features=32, out_features=96, bias=False)\n        (out_proj): Linear(in_features=32, out_features=32, bias=False)\n      )\n      (ff): FeedForward(\n        (w_1): Linear(in_features=32, out_features=84, bias=True)\n        (w_2): Linear(in_features=42, out_features=32, bias=True)\n        (dropout): Dropout(p=0.1, inplace=False)\n        (activation): REGLU()\n      )\n      (attn_normadd): NormAdd(\n        (dropout): Dropout(p=0.2, inplace=False)\n        (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n      )\n      (ff_normadd): NormAdd(\n        (dropout): Dropout(p=0.1, inplace=False)\n        (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n      )\n    )\n  )\n)</pre> In\u00a0[8]: Copied! <pre>pretrained_model = contrastive_denoising_model.model\n</pre> pretrained_model = contrastive_denoising_model.model In\u00a0[9]: Copied! <pre># and as always, ANY supervised model in this library has to go throuth the WideDeep class:\nmodel = WideDeep(deeptabular=pretrained_model)\ntrainer = Trainer(model=model, objective=\"binary\", metrics=[Accuracy])\n\ntrainer.fit(X_tab=X_tab, target=target, n_epochs=5, batch_size=256)\n\n# And, you know...we get a test metric\nX_tab_te = tab_preprocessor.transform(df_te)\ntarget_te = df_te[target_col].values\n\npreds = trainer.predict(X_tab=X_tab_te)\ntest_acc = accuracy_score(target_te, preds)\nprint(test_acc)\n</pre> # and as always, ANY supervised model in this library has to go throuth the WideDeep class: model = WideDeep(deeptabular=pretrained_model) trainer = Trainer(model=model, objective=\"binary\", metrics=[Accuracy])  trainer.fit(X_tab=X_tab, target=target, n_epochs=5, batch_size=256)  # And, you know...we get a test metric X_tab_te = tab_preprocessor.transform(df_te) target_te = df_te[target_col].values  preds = trainer.predict(X_tab=X_tab_te) test_acc = accuracy_score(target_te, preds) print(test_acc) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:05&lt;00:00, 27.19it/s, loss=0.383, metrics={'acc': 0.8176}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:05&lt;00:00, 26.08it/s, loss=0.325, metrics={'acc': 0.8502}]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:05&lt;00:00, 26.56it/s, loss=0.306, metrics={'acc': 0.8601}]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:05&lt;00:00, 27.41it/s, loss=0.295, metrics={'acc': 0.8641}]\nepoch 5: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:06&lt;00:00, 24.70it/s, loss=0.289, metrics={'acc': 0.8656}]\npredict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 39/39 [00:00&lt;00:00, 97.26it/s]</pre> <pre>0.8695874705701709\n</pre> <pre>\n</pre> In\u00a0[\u00a0]: Copied! <pre>\n</pre>"},{"location":"examples/15_Self_Supervised_Pretraning_pt2.html#self-supervised-pretraining-for-tabular-data","title":"Self Supervised Pretraining for Tabular Data\u00b6","text":"<p>We have implemented two Self Supervised Pre-training routines that allow the user to pre-train all tabular models in the library with the exception of the TabPerceiver (which is a special monster).</p> <p>The two routines implemented are illustrated in the figures below. The 1st is from TabNet: Attentive Interpretable Tabular Learning and is designed for models that do not use transformer-based architectures, while the second is from SAINT: Improved Neural Networks for Tabular Data via Row Attention and Contrastive Pre-Training, and is designed for models that use transformer-based architectures.</p> <p> </p> <p>Fig 1. Figure 2 in their paper. I have included de original caption in case is useful, althought the Figure itself is pretty self explanatory</p> <p> </p> <p>Fig 2. Figure 1 in their paper. Here the caption is necessary \ud83d\ude0f</p> <p>It is beyond the scope of this notebook to explain in detail those implementations. Therefore, we strongly recommend the user to go and read the papers if this functionality is of interest to her/him.</p> <p>One thing is worth noticing however. As seen in Fig 1(the TabNet paper's Fig 2) the masking of the input features happens in the feature space. However, the implementation in this library is inspired by that at the dreamquark-ai repo, which is in itself inspired by the original implementation (by the way, at this point I will write it once again. All TabNet related things in this library are inspired when not directly based in the code in that repo, therefore, ALL CREDIT TO THE GUYS AT dreamquark-ai).</p> <p>In that implementation the masking happens in the embedding space, and currently does not mask the entire embedding (i.e. categorical feature). We decided to release as it is in this version and we will implement the exact same process described in the paper in future releases.</p> <p>Having said all of the above let's see how to use self supervision for tabular data with <code>pytorch-widedeep</code>. We will concentrate in this notebook on the 2nd of the two approaches (the 'SAINT approach'). For details on the 1st approach (the 'TabNet' approach) please see <code>16_Self_Supervised_Pretraning_pt1</code>.</p>"},{"location":"examples/15_Self_Supervised_Pretraning_pt2.html#self-supervision-transformer-based-models","title":"Self Supervision transformer-based models..\u00b6","text":"<p>...or in general, for models where the embeddigns have all the same dimensions. In this library, these are:</p> <ul> <li>TabTransformer</li> <li>FTTransformer</li> <li>SAINT</li> <li>TabFastFormer</li> </ul> <p>Note that there is one additional Transformer-based model, the <code>TabPerceiver</code>, however this is a \"particular\" model and at the moment we do not support self supervision for it, but it will come.</p> <p>Let see at one example using the <code>FTTransformer</code>.</p>"},{"location":"examples/16_Usign_a_custom_hugging_face_model.html","title":"16 Usign a custom hugging face model","text":"In\u00a0[1]: Copied! <pre>import numpy as np\nimport torch\nimport lightgbm as lgb\nfrom lightgbm import Dataset as lgbDataset\nfrom scipy.sparse import hstack, csr_matrix\nfrom sklearn.metrics import (\n    f1_score,\n    recall_score,\n    accuracy_score,\n    precision_score,\n    confusion_matrix,\n)\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.feature_extraction.text import TfidfVectorizer\n\nfrom torch import Tensor, nn\nfrom transformers import DistilBertModel, DistilBertTokenizer\nfrom pytorch_widedeep import Trainer\nfrom pytorch_widedeep.models import TabMlp, BasicRNN, WideDeep\nfrom pytorch_widedeep.metrics import F1Score, Accuracy\nfrom pytorch_widedeep.utils import Tokenizer, LabelEncoder\nfrom pytorch_widedeep.preprocessing import TextPreprocessor, TabPreprocessor\nfrom pytorch_widedeep.datasets import load_womens_ecommerce\nfrom pytorch_widedeep.utils.fastai_transforms import (\n    fix_html,\n    spec_add_spaces,\n    rm_useless_spaces,\n)\n</pre> import numpy as np import torch import lightgbm as lgb from lightgbm import Dataset as lgbDataset from scipy.sparse import hstack, csr_matrix from sklearn.metrics import (     f1_score,     recall_score,     accuracy_score,     precision_score,     confusion_matrix, ) from sklearn.model_selection import train_test_split from sklearn.feature_extraction.text import TfidfVectorizer  from torch import Tensor, nn from transformers import DistilBertModel, DistilBertTokenizer from pytorch_widedeep import Trainer from pytorch_widedeep.models import TabMlp, BasicRNN, WideDeep from pytorch_widedeep.metrics import F1Score, Accuracy from pytorch_widedeep.utils import Tokenizer, LabelEncoder from pytorch_widedeep.preprocessing import TextPreprocessor, TabPreprocessor from pytorch_widedeep.datasets import load_womens_ecommerce from pytorch_widedeep.utils.fastai_transforms import (     fix_html,     spec_add_spaces,     rm_useless_spaces, ) <p>Let's load the data and have a look:</p> In\u00a0[2]: Copied! <pre>df = load_womens_ecommerce(as_frame=True)\n\ndf.columns = [c.replace(\" \", \"_\").lower() for c in df.columns]\n\n# classes from [0,num_class)\ndf[\"rating\"] = (df[\"rating\"] - 1).astype(\"int64\")\n\n# group reviews with 1 and 2 scores into one class\ndf.loc[df.rating == 0, \"rating\"] = 1\n\n# and back again to [0,num_class)\ndf[\"rating\"] = (df[\"rating\"] - 1).astype(\"int64\")\n\n# drop short reviews\ndf = df[~df.review_text.isna()]\ndf[\"review_length\"] = df.review_text.apply(lambda x: len(x.split(\" \")))\ndf = df[df.review_length &gt;= 5]\ndf = df.drop(\"review_length\", axis=1).reset_index(drop=True)\n</pre> df = load_womens_ecommerce(as_frame=True)  df.columns = [c.replace(\" \", \"_\").lower() for c in df.columns]  # classes from [0,num_class) df[\"rating\"] = (df[\"rating\"] - 1).astype(\"int64\")  # group reviews with 1 and 2 scores into one class df.loc[df.rating == 0, \"rating\"] = 1  # and back again to [0,num_class) df[\"rating\"] = (df[\"rating\"] - 1).astype(\"int64\")  # drop short reviews df = df[~df.review_text.isna()] df[\"review_length\"] = df.review_text.apply(lambda x: len(x.split(\" \"))) df = df[df.review_length &gt;= 5] df = df.drop(\"review_length\", axis=1).reset_index(drop=True) In\u00a0[3]: Copied! <pre>df.head()\n</pre> df.head() Out[3]: clothing_id age title review_text rating recommended_ind positive_feedback_count division_name department_name class_name 0 767 33 None Absolutely wonderful - silky and sexy and comf... 2 1 0 Initmates Intimate Intimates 1 1080 34 None Love this dress!  it's sooo pretty.  i happene... 3 1 4 General Dresses Dresses 2 1077 60 Some major design flaws I had such high hopes for this dress and reall... 1 0 0 General Dresses Dresses 3 1049 50 My favorite buy! I love, love, love this jumpsuit. it's fun, fl... 3 1 0 General Petite Bottoms Pants 4 847 47 Flattering shirt This shirt is very flattering to all due to th... 3 1 6 General Tops Blouses <p>So, we will use the <code>review_text</code> column to predict the <code>rating</code>. Later on, we will try to combine it with some other columns (like <code>division_name</code> and <code>age</code>) see if these help.</p> <p>Let's first have a look to the distribution of ratings</p> In\u00a0[4]: Copied! <pre>df.rating.value_counts()\n</pre> df.rating.value_counts() Out[4]: <pre>rating\n3    12515\n2     4904\n1     2820\n0     2369\nName: count, dtype: int64</pre> <p>This shows that we could have perhaps grouped rating scores of 1, 2 and 3 into 1...but anyway, let's just move on with those 4 classes.</p> <p>We are not going to carry any hyperparameter optimization here, so, we will only need a train and a test set (i.e.  no need of a validation set for the example in this notebook)</p> In\u00a0[5]: Copied! <pre>train, test = train_test_split(df, train_size=0.8, random_state=1, stratify=df.rating)\n</pre> train, test = train_test_split(df, train_size=0.8, random_state=1, stratify=df.rating) <p>Let's see what we have to beat. What metrics would we obtain if we always predict the most common rating (3)?</p> In\u00a0[6]: Copied! <pre>most_common_pred = [train.rating.value_counts().index[0]] * len(test)\n\nmost_common_acc = accuracy_score(test.rating, most_common_pred)\nmost_common_f1 = f1_score(test.rating, most_common_pred, average=\"weighted\")\n</pre> most_common_pred = [train.rating.value_counts().index[0]] * len(test)  most_common_acc = accuracy_score(test.rating, most_common_pred) most_common_f1 = f1_score(test.rating, most_common_pred, average=\"weighted\") In\u00a0[7]: Copied! <pre>print(f\"Accuracy: {most_common_acc}. F1 Score: {most_common_f1}\")\n</pre> print(f\"Accuracy: {most_common_acc}. F1 Score: {most_common_f1}\") <pre>Accuracy: 0.553516143299425. F1 Score: 0.3944344218301668\n</pre> <p>ok, these are our \"baseline\" metrics.</p> <p>Let's start by using simply tf-idf + lightGBM</p> In\u00a0[8]: Copied! <pre># ?Tokenizer\n</pre> # ?Tokenizer In\u00a0[9]: Copied! <pre># this Tokenizer is part of our utils module but of course, any valid tokenizer can be used here.\n\n# When using notebooks there seems to be an issue related with multiprocessing (and sometimes tqdm)\n# that can only be solved by using only one CPU\ntok = Tokenizer(n_cpus=1)\ntok_reviews_tr = tok.process_all(train.review_text.tolist())\ntok_reviews_te = tok.process_all(test.review_text.tolist())\n</pre> # this Tokenizer is part of our utils module but of course, any valid tokenizer can be used here.  # When using notebooks there seems to be an issue related with multiprocessing (and sometimes tqdm) # that can only be solved by using only one CPU tok = Tokenizer(n_cpus=1) tok_reviews_tr = tok.process_all(train.review_text.tolist()) tok_reviews_te = tok.process_all(test.review_text.tolist()) In\u00a0[10]: Copied! <pre>vectorizer = TfidfVectorizer(\n    max_features=5000, preprocessor=lambda x: x, tokenizer=lambda x: x, min_df=5\n)\n\nX_text_tr = vectorizer.fit_transform(tok_reviews_tr)\nX_text_te = vectorizer.transform(tok_reviews_te)\n</pre> vectorizer = TfidfVectorizer(     max_features=5000, preprocessor=lambda x: x, tokenizer=lambda x: x, min_df=5 )  X_text_tr = vectorizer.fit_transform(tok_reviews_tr) X_text_te = vectorizer.transform(tok_reviews_te) <pre>/Users/javierrodriguezzaurin/.pyenv/versions/3.10.13/envs/widedeep310/lib/python3.10/site-packages/sklearn/feature_extraction/text.py:525: UserWarning: The parameter 'token_pattern' will not be used since 'tokenizer' is not None'\n  warnings.warn(\n</pre> In\u00a0[11]: Copied! <pre>X_text_tr\n</pre> X_text_tr Out[11]: <pre>&lt;18086x4566 sparse matrix of type '&lt;class 'numpy.float64'&gt;'\n\twith 884074 stored elements in Compressed Sparse Row format&gt;</pre> <p>We now move our matrices to lightGBM <code>Dataset</code> format</p> In\u00a0[12]: Copied! <pre>lgbtrain_text = lgbDataset(\n    X_text_tr,\n    train.rating.values,\n    free_raw_data=False,\n)\n\nlgbtest_text = lgbDataset(\n    X_text_te,\n    test.rating.values,\n    reference=lgbtrain_text,\n    free_raw_data=False,\n)\n</pre> lgbtrain_text = lgbDataset(     X_text_tr,     train.rating.values,     free_raw_data=False, )  lgbtest_text = lgbDataset(     X_text_te,     test.rating.values,     reference=lgbtrain_text,     free_raw_data=False, ) <p>and off we go. By the way, I think as we run the next cell, we should appreciate how fast lightGBM runs. Yes, the input is a sparse matrix, but still, trains on 18086x4566 in a matter of secs</p> In\u00a0[\u00a0]: Copied! <pre>lgb_text_model = lgb.train(\n    {\"objective\": \"multiclass\", \"num_classes\": 4},\n    lgbtrain_text,\n    valid_sets=[lgbtest_text, lgbtrain_text],\n    valid_names=[\"test\", \"train\"],\n)\n</pre> lgb_text_model = lgb.train(     {\"objective\": \"multiclass\", \"num_classes\": 4},     lgbtrain_text,     valid_sets=[lgbtest_text, lgbtrain_text],     valid_names=[\"test\", \"train\"], ) In\u00a0[14]: Copied! <pre>preds_text = lgb_text_model.predict(X_text_te)\npred_text_class = np.argmax(preds_text, 1)\n</pre> preds_text = lgb_text_model.predict(X_text_te) pred_text_class = np.argmax(preds_text, 1) In\u00a0[15]: Copied! <pre>acc_text = accuracy_score(lgbtest_text.label, pred_text_class)\nf1_text = f1_score(lgbtest_text.label, pred_text_class, average=\"weighted\")\ncm_text = confusion_matrix(lgbtest_text.label, pred_text_class)\n</pre> acc_text = accuracy_score(lgbtest_text.label, pred_text_class) f1_text = f1_score(lgbtest_text.label, pred_text_class, average=\"weighted\") cm_text = confusion_matrix(lgbtest_text.label, pred_text_class) In\u00a0[16]: Copied! <pre>print(f\"LightGBM Accuracy: {acc_text}. LightGBM F1 Score: {f1_text}\")\n</pre> print(f\"LightGBM Accuracy: {acc_text}. LightGBM F1 Score: {f1_text}\") <pre>LightGBM Accuracy: 0.6444051304732419. LightGBM F1 Score: 0.617154488246181\n</pre> In\u00a0[17]: Copied! <pre>print(f\"LightGBM Confusion Matrix: \\n {cm_text}\")\n</pre> print(f\"LightGBM Confusion Matrix: \\n {cm_text}\") <pre>LightGBM Confusion Matrix: \n [[ 199  135   61   79]\n [ 123  169  149  123]\n [  30   94  279  578]\n [  16   30  190 2267]]\n</pre> <p>Ok, so, with no hyperparameter optimization lightGBM gets an accuracy of 0.64 and a F1 score of 0.62. This is significantly better than predicting always the most popular.</p> <p>Let's see if in this implementation, some additional features, like <code>age</code> or <code>class_name</code> are of any help</p> In\u00a0[18]: Copied! <pre>tab_cols = [\n    \"age\",\n    \"division_name\",\n    \"department_name\",\n    \"class_name\",\n]\n\nfor tab_df in [train, test]:\n    for c in [\"division_name\", \"department_name\", \"class_name\"]:\n        tab_df[c] = tab_df[c].str.lower()\n        tab_df[c].fillna(\"missing\", inplace=True)\n</pre> tab_cols = [     \"age\",     \"division_name\",     \"department_name\",     \"class_name\", ]  for tab_df in [train, test]:     for c in [\"division_name\", \"department_name\", \"class_name\"]:         tab_df[c] = tab_df[c].str.lower()         tab_df[c].fillna(\"missing\", inplace=True) In\u00a0[19]: Copied! <pre># This is our LabelEncoder. A class that is designed to work with the models in this library but\n# can be used for general purposes\nle = LabelEncoder(columns_to_encode=[\"division_name\", \"department_name\", \"class_name\"])\ntrain_tab_le = le.fit_transform(train)\ntest_tab_le = le.transform(test)\n</pre> # This is our LabelEncoder. A class that is designed to work with the models in this library but # can be used for general purposes le = LabelEncoder(columns_to_encode=[\"division_name\", \"department_name\", \"class_name\"]) train_tab_le = le.fit_transform(train) test_tab_le = le.transform(test) In\u00a0[20]: Copied! <pre>train_tab_le.head()\n</pre> train_tab_le.head() Out[20]: clothing_id age title review_text rating recommended_ind positive_feedback_count division_name department_name class_name 4541 836 35 None Bought this on sale in my reg size- 10. im 5'9... 2 1 2 1 1 1 18573 1022 25 Look like \"mom jeans\" Maybe i just have the wrong body type for thes... 1 0 0 2 2 2 1058 815 39 Ig brought me here Love the way this top layers under my jackets ... 2 1 0 1 1 1 12132 984 47 Runs small especially the arms I love this jacket. it's the prettiest and mos... 3 1 0 1 3 3 20756 1051 42 True red, true beauty. These pants are gorgeous--the fabric has a sat... 3 1 0 2 2 4 <p>let's for example have a look to the encodings for the categorical feature <code>class_name</code></p> In\u00a0[21]: Copied! <pre>le.encoding_dict[\"class_name\"]\n</pre> le.encoding_dict[\"class_name\"] Out[21]: <pre>{'blouses': 1,\n 'jeans': 2,\n 'jackets': 3,\n 'pants': 4,\n 'knits': 5,\n 'dresses': 6,\n 'skirts': 7,\n 'sweaters': 8,\n 'fine gauge': 9,\n 'legwear': 10,\n 'lounge': 11,\n 'shorts': 12,\n 'outerwear': 13,\n 'intimates': 14,\n 'swim': 15,\n 'trend': 16,\n 'sleep': 17,\n 'layering': 18,\n 'missing': 19,\n 'casual bottoms': 20,\n 'chemises': 21}</pre> In\u00a0[22]: Copied! <pre># tabular training and test sets\nX_tab_tr = csr_matrix(train_tab_le[tab_cols].values)\nX_tab_te = csr_matrix(test_tab_le[tab_cols].values)\n\n# text + tabular training and test sets\nX_tab_text_tr = hstack((X_tab_tr, X_text_tr))\nX_tab_text_te = hstack((X_tab_te, X_text_te))\n</pre> # tabular training and test sets X_tab_tr = csr_matrix(train_tab_le[tab_cols].values) X_tab_te = csr_matrix(test_tab_le[tab_cols].values)  # text + tabular training and test sets X_tab_text_tr = hstack((X_tab_tr, X_text_tr)) X_tab_text_te = hstack((X_tab_te, X_text_te)) In\u00a0[23]: Copied! <pre>X_tab_tr\n</pre> X_tab_tr Out[23]: <pre>&lt;18086x4 sparse matrix of type '&lt;class 'numpy.int64'&gt;'\n\twith 72344 stored elements in Compressed Sparse Row format&gt;</pre> In\u00a0[24]: Copied! <pre>X_tab_text_tr\n</pre> X_tab_text_tr Out[24]: <pre>&lt;18086x4570 sparse matrix of type '&lt;class 'numpy.float64'&gt;'\n\twith 956418 stored elements in Compressed Sparse Row format&gt;</pre> In\u00a0[25]: Copied! <pre>lgbtrain_tab_text = lgbDataset(\n    X_tab_text_tr,\n    train.rating.values,\n    categorical_feature=[0, 1, 2, 3],\n    free_raw_data=False,\n)\n\nlgbtest_tab_text = lgbDataset(\n    X_tab_text_te,\n    test.rating.values,\n    reference=lgbtrain_tab_text,\n    free_raw_data=False,\n)\n</pre> lgbtrain_tab_text = lgbDataset(     X_tab_text_tr,     train.rating.values,     categorical_feature=[0, 1, 2, 3],     free_raw_data=False, )  lgbtest_tab_text = lgbDataset(     X_tab_text_te,     test.rating.values,     reference=lgbtrain_tab_text,     free_raw_data=False, ) In\u00a0[26]: Copied! <pre>lgb_tab_text_model = lgb.train(\n    {\"objective\": \"multiclass\", \"num_classes\": 4},\n    lgbtrain_tab_text,\n    valid_sets=[lgbtrain_tab_text, lgbtest_tab_text],\n    valid_names=[\"test\", \"train\"],\n    verbose_eval=False,\n)\n</pre> lgb_tab_text_model = lgb.train(     {\"objective\": \"multiclass\", \"num_classes\": 4},     lgbtrain_tab_text,     valid_sets=[lgbtrain_tab_text, lgbtest_tab_text],     valid_names=[\"test\", \"train\"],     verbose_eval=False, ) <pre>/opt/conda/envs/wd38/lib/python3.8/site-packages/lightgbm/basic.py:2065: UserWarning: Using categorical_feature in Dataset.\n  _log_warning('Using categorical_feature in Dataset.')\n/opt/conda/envs/wd38/lib/python3.8/site-packages/lightgbm/basic.py:2068: UserWarning: categorical_feature in Dataset is overridden.\nNew categorical_feature is [0, 1, 2, 3]\n  _log_warning('categorical_feature in Dataset is overridden.\\n'\n/opt/conda/envs/wd38/lib/python3.8/site-packages/lightgbm/engine.py:239: UserWarning: 'verbose_eval' argument is deprecated and will be removed in a future release of LightGBM. Pass 'log_evaluation()' callback via 'callbacks' argument instead.\n  _log_warning(\"'verbose_eval' argument is deprecated and will be removed in a future release of LightGBM. \"\n</pre> <pre>[LightGBM] [Warning] Auto-choosing col-wise multi-threading, the overhead of testing was 0.138280 seconds.\nYou can set `force_col_wise=true` to remove the overhead.\n[LightGBM] [Info] Total Bins 143432\n[LightGBM] [Info] Number of data points in the train set: 18086, number of used features: 2289\n[LightGBM] [Info] Start training from score -2.255919\n[LightGBM] [Info] Start training from score -2.081545\n[LightGBM] [Info] Start training from score -1.528281\n[LightGBM] [Info] Start training from score -0.591354\n</pre> <pre>/opt/conda/envs/wd38/lib/python3.8/site-packages/lightgbm/basic.py:1780: UserWarning: Overriding the parameters from Reference Dataset.\n  _log_warning('Overriding the parameters from Reference Dataset.')\n/opt/conda/envs/wd38/lib/python3.8/site-packages/lightgbm/basic.py:1513: UserWarning: categorical_column in param dict is overridden.\n  _log_warning(f'{cat_alias} in param dict is overridden.')\n</pre> In\u00a0[27]: Copied! <pre>preds_tab_text = lgb_tab_text_model.predict(X_tab_text_te)\npreds_tab_text_class = np.argmax(preds_tab_text, 1)\n\nacc_tab_text = accuracy_score(lgbtest_tab_text.label, preds_tab_text_class)\nf1_tab_text = f1_score(lgbtest_tab_text.label, preds_tab_text_class, average=\"weighted\")\ncm_tab_text = confusion_matrix(lgbtest_tab_text.label, preds_tab_text_class)\n</pre> preds_tab_text = lgb_tab_text_model.predict(X_tab_text_te) preds_tab_text_class = np.argmax(preds_tab_text, 1)  acc_tab_text = accuracy_score(lgbtest_tab_text.label, preds_tab_text_class) f1_tab_text = f1_score(lgbtest_tab_text.label, preds_tab_text_class, average=\"weighted\") cm_tab_text = confusion_matrix(lgbtest_tab_text.label, preds_tab_text_class) In\u00a0[28]: Copied! <pre>print(\n    f\"LightGBM text + tabular Accuracy: {acc_tab_text}. LightGBM text + tabular F1 Score: {f1_tab_text}\"\n)\n</pre> print(     f\"LightGBM text + tabular Accuracy: {acc_tab_text}. LightGBM text + tabular F1 Score: {f1_tab_text}\" ) <pre>LightGBM text + tabular Accuracy: 0.6382131800088456. LightGBM text + tabular F1 Score: 0.6080251307242649\n</pre> In\u00a0[29]: Copied! <pre>print(f\"LightGBM text + tabular Confusion Matrix:\\n {cm_tab_text}\")\n</pre> print(f\"LightGBM text + tabular Confusion Matrix:\\n {cm_tab_text}\") <pre>LightGBM text + tabular Confusion Matrix:\n [[ 193  123   68   90]\n [ 123  146  157  138]\n [  37   90  272  582]\n [  16   37  175 2275]]\n</pre> <p>So, in this set up, the addition tabular columns do not help performance.</p> In\u00a0[30]: Copied! <pre>text_preprocessor = TextPreprocessor(\n    text_col=\"review_text\", max_vocab=5000, min_freq=5, maxlen=90, n_cpus=1\n)\n\nwd_X_text_tr = text_preprocessor.fit_transform(train)\nwd_X_text_te = text_preprocessor.transform(test)\n</pre> text_preprocessor = TextPreprocessor(     text_col=\"review_text\", max_vocab=5000, min_freq=5, maxlen=90, n_cpus=1 )  wd_X_text_tr = text_preprocessor.fit_transform(train) wd_X_text_te = text_preprocessor.transform(test) <pre>The vocabulary contains 4328 tokens\n</pre> In\u00a0[31]: Copied! <pre>basic_rnn = BasicRNN(\n    vocab_size=len(text_preprocessor.vocab.itos),\n    embed_dim=300,\n    hidden_dim=64,\n    n_layers=3,\n    rnn_dropout=0.2,\n    head_hidden_dims=[32],\n)\n\n\nwd_text_model = WideDeep(deeptext=basic_rnn, pred_dim=4)\n</pre> basic_rnn = BasicRNN(     vocab_size=len(text_preprocessor.vocab.itos),     embed_dim=300,     hidden_dim=64,     n_layers=3,     rnn_dropout=0.2,     head_hidden_dims=[32], )   wd_text_model = WideDeep(deeptext=basic_rnn, pred_dim=4) In\u00a0[32]: Copied! <pre>wd_text_model\n</pre> wd_text_model Out[32]: <pre>WideDeep(\n  (deeptext): Sequential(\n    (0): BasicRNN(\n      (word_embed): Embedding(4328, 300, padding_idx=1)\n      (rnn): LSTM(300, 64, num_layers=3, batch_first=True, dropout=0.2)\n      (rnn_mlp): MLP(\n        (mlp): Sequential(\n          (dense_layer_0): Sequential(\n            (0): Linear(in_features=64, out_features=32, bias=True)\n            (1): ReLU(inplace=True)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=32, out_features=4, bias=True)\n  )\n)</pre> In\u00a0[33]: Copied! <pre>text_trainer = Trainer(\n    wd_text_model,\n    objective=\"multiclass\",\n    metrics=[Accuracy, F1Score(average=True)],\n    num_workers=0,  # As in the case of the tokenizer, in notebook I need to set this to 0 for the Trainer to work\n)\n</pre> text_trainer = Trainer(     wd_text_model,     objective=\"multiclass\",     metrics=[Accuracy, F1Score(average=True)],     num_workers=0,  # As in the case of the tokenizer, in notebook I need to set this to 0 for the Trainer to work ) In\u00a0[34]: Copied! <pre>text_trainer.fit(\n    X_text=wd_X_text_tr,\n    target=train.rating.values,\n    n_epochs=5,\n    batch_size=256,\n)\n</pre> text_trainer.fit(     X_text=wd_X_text_tr,     target=train.rating.values,     n_epochs=5,     batch_size=256, ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 71/71 [00:01&lt;00:00, 52.39it/s, loss=1.16, metrics={'acc': 0.5349, 'f1': 0.2011}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 71/71 [00:01&lt;00:00, 70.35it/s, loss=0.964, metrics={'acc': 0.5827, 'f1': 0.3005}]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 71/71 [00:01&lt;00:00, 70.33it/s, loss=0.845, metrics={'acc': 0.6252, 'f1': 0.4133}]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 71/71 [00:01&lt;00:00, 69.99it/s, loss=0.765, metrics={'acc': 0.6575, 'f1': 0.4875}]\nepoch 5: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 71/71 [00:01&lt;00:00, 69.55it/s, loss=0.709, metrics={'acc': 0.6879, 'f1': 0.5423}]\n</pre> In\u00a0[35]: Copied! <pre>wd_pred_text = text_trainer.predict_proba(X_text=wd_X_text_te)\nwd_pred_text_class = np.argmax(wd_pred_text, 1)\n</pre> wd_pred_text = text_trainer.predict_proba(X_text=wd_X_text_te) wd_pred_text_class = np.argmax(wd_pred_text, 1) <pre>predict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 18/18 [00:00&lt;00:00, 211.51it/s]\n</pre> In\u00a0[36]: Copied! <pre>wd_acc_text = accuracy_score(test.rating, wd_pred_text_class)\nwd_f1_text = f1_score(test.rating, wd_pred_text_class, average=\"weighted\")\nwd_cm_text = confusion_matrix(test.rating, wd_pred_text_class)\n</pre> wd_acc_text = accuracy_score(test.rating, wd_pred_text_class) wd_f1_text = f1_score(test.rating, wd_pred_text_class, average=\"weighted\") wd_cm_text = confusion_matrix(test.rating, wd_pred_text_class) In\u00a0[37]: Copied! <pre>print(f\"Basic RNN Accuracy: {wd_acc_text}. Basic RNN F1 Score: {wd_f1_text}\")\n</pre> print(f\"Basic RNN Accuracy: {wd_acc_text}. Basic RNN F1 Score: {wd_f1_text}\") <pre>Basic RNN Accuracy: 0.6076957098628926. Basic RNN F1 Score: 0.6017335854471788\n</pre> In\u00a0[38]: Copied! <pre>print(f\"Basic RNN Confusion Matrix:\\n {wd_cm_text}\")\n</pre> print(f\"Basic RNN Confusion Matrix:\\n {wd_cm_text}\") <pre>Basic RNN Confusion Matrix:\n [[ 327   76   62    9]\n [ 285  115  117   47]\n [ 131  122  315  413]\n [  42   69  401 1991]]\n</pre> <p>The performance is very similar to that of using simply tf-idf and lightgbm. Let see if adding tabular features helps when using <code>pytorch-widedeep</code></p> In\u00a0[39]: Copied! <pre># ?TabPreprocessor\n</pre> # ?TabPreprocessor In\u00a0[40]: Copied! <pre>tab_preprocessor = TabPreprocessor(cat_embed_cols=tab_cols)\n\nwd_X_tab_tr = tab_preprocessor.fit_transform(train)\nwd_X_tab_te = tab_preprocessor.transform(test)\n</pre> tab_preprocessor = TabPreprocessor(cat_embed_cols=tab_cols)  wd_X_tab_tr = tab_preprocessor.fit_transform(train) wd_X_tab_te = tab_preprocessor.transform(test) In\u00a0[41]: Copied! <pre># ?TabMlp\n</pre> # ?TabMlp In\u00a0[42]: Copied! <pre>tab_model = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    mlp_hidden_dims=[100, 50],\n)\n</pre> tab_model = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     mlp_hidden_dims=[100, 50], ) In\u00a0[43]: Copied! <pre>tab_model\n</pre> tab_model Out[43]: <pre>TabMlp(\n  (cat_and_cont_embed): DiffSizeCatAndContEmbeddings(\n    (cat_embed): DiffSizeCatEmbeddings(\n      (embed_layers): ModuleDict(\n        (emb_layer_age): Embedding(78, 18, padding_idx=0)\n        (emb_layer_division_name): Embedding(5, 3, padding_idx=0)\n        (emb_layer_department_name): Embedding(8, 5, padding_idx=0)\n        (emb_layer_class_name): Embedding(22, 9, padding_idx=0)\n      )\n      (embedding_dropout): Dropout(p=0.1, inplace=False)\n    )\n  )\n  (encoder): MLP(\n    (mlp): Sequential(\n      (dense_layer_0): Sequential(\n        (0): Dropout(p=0.1, inplace=False)\n        (1): Linear(in_features=35, out_features=100, bias=True)\n        (2): ReLU(inplace=True)\n      )\n      (dense_layer_1): Sequential(\n        (0): Dropout(p=0.1, inplace=False)\n        (1): Linear(in_features=100, out_features=50, bias=True)\n        (2): ReLU(inplace=True)\n      )\n    )\n  )\n)</pre> In\u00a0[44]: Copied! <pre>text_model = BasicRNN(\n    vocab_size=len(text_preprocessor.vocab.itos),\n    embed_dim=300,\n    hidden_dim=64,\n    n_layers=3,\n    rnn_dropout=0.2,\n    head_hidden_dims=[32],\n)\n</pre> text_model = BasicRNN(     vocab_size=len(text_preprocessor.vocab.itos),     embed_dim=300,     hidden_dim=64,     n_layers=3,     rnn_dropout=0.2,     head_hidden_dims=[32], ) In\u00a0[45]: Copied! <pre>wd_tab_and_text_model = WideDeep(deeptabular=tab_model, deeptext=text_model, pred_dim=4)\n</pre> wd_tab_and_text_model = WideDeep(deeptabular=tab_model, deeptext=text_model, pred_dim=4) In\u00a0[46]: Copied! <pre>wd_tab_and_text_model\n</pre> wd_tab_and_text_model Out[46]: <pre>WideDeep(\n  (deeptabular): Sequential(\n    (0): TabMlp(\n      (cat_and_cont_embed): DiffSizeCatAndContEmbeddings(\n        (cat_embed): DiffSizeCatEmbeddings(\n          (embed_layers): ModuleDict(\n            (emb_layer_age): Embedding(78, 18, padding_idx=0)\n            (emb_layer_division_name): Embedding(5, 3, padding_idx=0)\n            (emb_layer_department_name): Embedding(8, 5, padding_idx=0)\n            (emb_layer_class_name): Embedding(22, 9, padding_idx=0)\n          )\n          (embedding_dropout): Dropout(p=0.1, inplace=False)\n        )\n      )\n      (encoder): MLP(\n        (mlp): Sequential(\n          (dense_layer_0): Sequential(\n            (0): Dropout(p=0.1, inplace=False)\n            (1): Linear(in_features=35, out_features=100, bias=True)\n            (2): ReLU(inplace=True)\n          )\n          (dense_layer_1): Sequential(\n            (0): Dropout(p=0.1, inplace=False)\n            (1): Linear(in_features=100, out_features=50, bias=True)\n            (2): ReLU(inplace=True)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=50, out_features=4, bias=True)\n  )\n  (deeptext): Sequential(\n    (0): BasicRNN(\n      (word_embed): Embedding(4328, 300, padding_idx=1)\n      (rnn): LSTM(300, 64, num_layers=3, batch_first=True, dropout=0.2)\n      (rnn_mlp): MLP(\n        (mlp): Sequential(\n          (dense_layer_0): Sequential(\n            (0): Linear(in_features=64, out_features=32, bias=True)\n            (1): ReLU(inplace=True)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=32, out_features=4, bias=True)\n  )\n)</pre> In\u00a0[47]: Copied! <pre>tab_and_text_trainer = Trainer(\n    wd_tab_and_text_model,\n    objective=\"multiclass\",\n    metrics=[Accuracy, F1Score(average=True)],\n    num_workers=0,  # As in the case of the tokenizer, in notebook I need to set this to 0 for the Trainer to work\n)\n</pre> tab_and_text_trainer = Trainer(     wd_tab_and_text_model,     objective=\"multiclass\",     metrics=[Accuracy, F1Score(average=True)],     num_workers=0,  # As in the case of the tokenizer, in notebook I need to set this to 0 for the Trainer to work ) In\u00a0[48]: Copied! <pre>tab_and_text_trainer.fit(\n    X_tab=wd_X_tab_tr,\n    X_text=wd_X_text_tr,\n    target=train.rating.values,\n    n_epochs=5,\n    batch_size=256,\n)\n</pre> tab_and_text_trainer.fit(     X_tab=wd_X_tab_tr,     X_text=wd_X_text_tr,     target=train.rating.values,     n_epochs=5,     batch_size=256, ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 71/71 [00:01&lt;00:00, 52.04it/s, loss=1.13, metrics={'acc': 0.538, 'f1': 0.1911}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 71/71 [00:01&lt;00:00, 52.28it/s, loss=0.936, metrics={'acc': 0.5887, 'f1': 0.3507}]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 71/71 [00:01&lt;00:00, 52.26it/s, loss=0.825, metrics={'acc': 0.6394, 'f1': 0.4545}]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 71/71 [00:01&lt;00:00, 51.33it/s, loss=0.757, metrics={'acc': 0.6696, 'f1': 0.5214}]\nepoch 5: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 71/71 [00:01&lt;00:00, 50.39it/s, loss=0.702, metrics={'acc': 0.6963, 'f1': 0.5654}]\n</pre> In\u00a0[49]: Copied! <pre>wd_pred_tab_and_text = tab_and_text_trainer.predict_proba(\n    X_tab=wd_X_tab_te, X_text=wd_X_text_te\n)\nwd_pred_tab_and_text_class = np.argmax(wd_pred_tab_and_text, 1)\n</pre> wd_pred_tab_and_text = tab_and_text_trainer.predict_proba(     X_tab=wd_X_tab_te, X_text=wd_X_text_te ) wd_pred_tab_and_text_class = np.argmax(wd_pred_tab_and_text, 1) <pre>predict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 18/18 [00:00&lt;00:00, 136.94it/s]\n</pre> In\u00a0[50]: Copied! <pre>wd_acc_tab_and_text = accuracy_score(test.rating, wd_pred_tab_and_text_class)\nwd_f1_tab_and_text = f1_score(\n    test.rating, wd_pred_tab_and_text_class, average=\"weighted\"\n)\nwd_cm_tab_and_text = confusion_matrix(test.rating, wd_pred_tab_and_text_class)\n</pre> wd_acc_tab_and_text = accuracy_score(test.rating, wd_pred_tab_and_text_class) wd_f1_tab_and_text = f1_score(     test.rating, wd_pred_tab_and_text_class, average=\"weighted\" ) wd_cm_tab_and_text = confusion_matrix(test.rating, wd_pred_tab_and_text_class) In\u00a0[51]: Copied! <pre>print(\n    f\"Basic RNN + Tabular  Accuracy: {wd_acc_tab_and_text}. Basic RNN + TabularF1 Score: {wd_f1_tab_and_text}\"\n)\nprint(f\"Basic RNN + Tabular  Confusion Matrix:\\n {wd_cm_tab_and_text}\")\n</pre> print(     f\"Basic RNN + Tabular  Accuracy: {wd_acc_tab_and_text}. Basic RNN + TabularF1 Score: {wd_f1_tab_and_text}\" ) print(f\"Basic RNN + Tabular  Confusion Matrix:\\n {wd_cm_tab_and_text}\") <pre>Basic RNN + Tabular  Accuracy: 0.6333480760725343. Basic RNN + TabularF1 Score: 0.6332310089593208\nBasic RNN + Tabular  Confusion Matrix:\n [[ 267  132   65   10]\n [ 198  168  159   39]\n [  57  113  410  401]\n [  12   58  414 2019]]\n</pre> <p>We are going to \"manually\" code the Tokenizer and the model and see how they can be used as part of the process along with the <code>pytorch-widedeep</code> library.</p> <p>Tokenizer:</p> In\u00a0[52]: Copied! <pre>class BertTokenizer(object):\n    def __init__(\n        self,\n        pretrained_tokenizer=\"distilbert-base-uncased\",\n        do_lower_case=True,\n        max_length=90,\n    ):\n        super(BertTokenizer, self).__init__()\n        self.pretrained_tokenizer = pretrained_tokenizer\n        self.do_lower_case = do_lower_case\n        self.max_length = max_length\n\n    def fit(self, texts):\n        self.tokenizer = DistilBertTokenizer.from_pretrained(\n            self.pretrained_tokenizer, do_lower_case=self.do_lower_case\n        )\n\n        return self\n\n    def transform(self, texts):\n        input_ids = []\n        for text in texts:\n            encoded_sent = self.tokenizer.encode_plus(\n                text=self._pre_rules(text),\n                add_special_tokens=True,\n                max_length=self.max_length,\n                padding=\"max_length\",\n                truncation=True,\n            )\n\n            input_ids.append(encoded_sent.get(\"input_ids\"))\n        return np.stack(input_ids)\n\n    def fit_transform(self, texts):\n        return self.fit(texts).transform(texts)\n\n    @staticmethod\n    def _pre_rules(text):\n        return fix_html(rm_useless_spaces(spec_add_spaces(text)))\n</pre> class BertTokenizer(object):     def __init__(         self,         pretrained_tokenizer=\"distilbert-base-uncased\",         do_lower_case=True,         max_length=90,     ):         super(BertTokenizer, self).__init__()         self.pretrained_tokenizer = pretrained_tokenizer         self.do_lower_case = do_lower_case         self.max_length = max_length      def fit(self, texts):         self.tokenizer = DistilBertTokenizer.from_pretrained(             self.pretrained_tokenizer, do_lower_case=self.do_lower_case         )          return self      def transform(self, texts):         input_ids = []         for text in texts:             encoded_sent = self.tokenizer.encode_plus(                 text=self._pre_rules(text),                 add_special_tokens=True,                 max_length=self.max_length,                 padding=\"max_length\",                 truncation=True,             )              input_ids.append(encoded_sent.get(\"input_ids\"))         return np.stack(input_ids)      def fit_transform(self, texts):         return self.fit(texts).transform(texts)      @staticmethod     def _pre_rules(text):         return fix_html(rm_useless_spaces(spec_add_spaces(text))) <p>Model:</p> In\u00a0[53]: Copied! <pre>class BertModel(nn.Module):\n    def __init__(\n        self,\n        model_name: str = \"distilbert-base-uncased\",\n        freeze_bert: bool = False,\n    ):\n        super(BertModel, self).__init__()\n\n        self.bert = DistilBertModel.from_pretrained(\n            model_name,\n        )\n\n        if freeze_bert:\n            for param in self.bert.parameters():\n                param.requires_grad = False\n\n    def forward(self, X_inp: Tensor) -&gt; Tensor:\n        attn_mask = (X_inp != 0).type(torch.int8)\n        outputs = self.bert(input_ids=X_inp, attention_mask=attn_mask)\n        return outputs[0][:, 0, :]\n\n    @property\n    def output_dim(self) -&gt; int:\n        # This is THE ONLY requirement for any model to work with pytorch-widedeep. Must\n        # have a 'output_dim' property so the WideDeep class knows the incoming dims\n        # from the custom model. in this case, I hardcoded it\n        return 768\n</pre> class BertModel(nn.Module):     def __init__(         self,         model_name: str = \"distilbert-base-uncased\",         freeze_bert: bool = False,     ):         super(BertModel, self).__init__()          self.bert = DistilBertModel.from_pretrained(             model_name,         )          if freeze_bert:             for param in self.bert.parameters():                 param.requires_grad = False      def forward(self, X_inp: Tensor) -&gt; Tensor:         attn_mask = (X_inp != 0).type(torch.int8)         outputs = self.bert(input_ids=X_inp, attention_mask=attn_mask)         return outputs[0][:, 0, :]      @property     def output_dim(self) -&gt; int:         # This is THE ONLY requirement for any model to work with pytorch-widedeep. Must         # have a 'output_dim' property so the WideDeep class knows the incoming dims         # from the custom model. in this case, I hardcoded it         return 768 In\u00a0[54]: Copied! <pre>bert_tokenizer = BertTokenizer()\nX_bert_tr = bert_tokenizer.fit_transform(train[\"review_text\"].tolist())\nX_bert_te = bert_tokenizer.transform(test[\"review_text\"].tolist())\n</pre> bert_tokenizer = BertTokenizer() X_bert_tr = bert_tokenizer.fit_transform(train[\"review_text\"].tolist()) X_bert_te = bert_tokenizer.transform(test[\"review_text\"].tolist()) <p>As I mentioned a number of times in the documentation and examples, <code>pytorch-widedeep</code> is designed for flexibility. For any of the data modes (tabular, text and images) there are available components/models in the library. However, the user can choose to use any model they want with the only requirement that such model must have a <code>output_dim</code> property.</p> <p>With that in mind, the <code>BertModel</code> class defined above can be used by <code>pytorch-widedeep</code> as any other of the internal components. In other words, simply...pass it to the <code>WideDeep</code> class. In this case we are going to add a FC-head as part of the classifier.</p> In\u00a0[55]: Copied! <pre>bert_model = BertModel(freeze_bert=True)\nwd_bert_model = WideDeep(\n    deeptext=bert_model,\n    head_hidden_dims=[256, 128, 64],\n    pred_dim=4,\n)\n</pre> bert_model = BertModel(freeze_bert=True) wd_bert_model = WideDeep(     deeptext=bert_model,     head_hidden_dims=[256, 128, 64],     pred_dim=4, ) <pre>Some weights of the model checkpoint at distilbert-base-uncased were not used when initializing DistilBertModel: ['vocab_projector.bias', 'vocab_layer_norm.bias', 'vocab_transform.weight', 'vocab_transform.bias', 'vocab_projector.weight', 'vocab_layer_norm.weight']\n- This IS expected if you are initializing DistilBertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n- This IS NOT expected if you are initializing DistilBertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n</pre> In\u00a0[56]: Copied! <pre>wd_bert_model\n</pre> wd_bert_model Out[56]: <pre>WideDeep(\n  (deeptext): BertModel(\n    (bert): DistilBertModel(\n      (embeddings): Embeddings(\n        (word_embeddings): Embedding(30522, 768, padding_idx=0)\n        (position_embeddings): Embedding(512, 768)\n        (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n        (dropout): Dropout(p=0.1, inplace=False)\n      )\n      (transformer): Transformer(\n        (layer): ModuleList(\n          (0): TransformerBlock(\n            (attention): MultiHeadSelfAttention(\n              (dropout): Dropout(p=0.1, inplace=False)\n              (q_lin): Linear(in_features=768, out_features=768, bias=True)\n              (k_lin): Linear(in_features=768, out_features=768, bias=True)\n              (v_lin): Linear(in_features=768, out_features=768, bias=True)\n              (out_lin): Linear(in_features=768, out_features=768, bias=True)\n            )\n            (sa_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n            (ffn): FFN(\n              (dropout): Dropout(p=0.1, inplace=False)\n              (lin1): Linear(in_features=768, out_features=3072, bias=True)\n              (lin2): Linear(in_features=3072, out_features=768, bias=True)\n              (activation): GELUActivation()\n            )\n            (output_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n          )\n          (1): TransformerBlock(\n            (attention): MultiHeadSelfAttention(\n              (dropout): Dropout(p=0.1, inplace=False)\n              (q_lin): Linear(in_features=768, out_features=768, bias=True)\n              (k_lin): Linear(in_features=768, out_features=768, bias=True)\n              (v_lin): Linear(in_features=768, out_features=768, bias=True)\n              (out_lin): Linear(in_features=768, out_features=768, bias=True)\n            )\n            (sa_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n            (ffn): FFN(\n              (dropout): Dropout(p=0.1, inplace=False)\n              (lin1): Linear(in_features=768, out_features=3072, bias=True)\n              (lin2): Linear(in_features=3072, out_features=768, bias=True)\n              (activation): GELUActivation()\n            )\n            (output_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n          )\n          (2): TransformerBlock(\n            (attention): MultiHeadSelfAttention(\n              (dropout): Dropout(p=0.1, inplace=False)\n              (q_lin): Linear(in_features=768, out_features=768, bias=True)\n              (k_lin): Linear(in_features=768, out_features=768, bias=True)\n              (v_lin): Linear(in_features=768, out_features=768, bias=True)\n              (out_lin): Linear(in_features=768, out_features=768, bias=True)\n            )\n            (sa_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n            (ffn): FFN(\n              (dropout): Dropout(p=0.1, inplace=False)\n              (lin1): Linear(in_features=768, out_features=3072, bias=True)\n              (lin2): Linear(in_features=3072, out_features=768, bias=True)\n              (activation): GELUActivation()\n            )\n            (output_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n          )\n          (3): TransformerBlock(\n            (attention): MultiHeadSelfAttention(\n              (dropout): Dropout(p=0.1, inplace=False)\n              (q_lin): Linear(in_features=768, out_features=768, bias=True)\n              (k_lin): Linear(in_features=768, out_features=768, bias=True)\n              (v_lin): Linear(in_features=768, out_features=768, bias=True)\n              (out_lin): Linear(in_features=768, out_features=768, bias=True)\n            )\n            (sa_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n            (ffn): FFN(\n              (dropout): Dropout(p=0.1, inplace=False)\n              (lin1): Linear(in_features=768, out_features=3072, bias=True)\n              (lin2): Linear(in_features=3072, out_features=768, bias=True)\n              (activation): GELUActivation()\n            )\n            (output_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n          )\n          (4): TransformerBlock(\n            (attention): MultiHeadSelfAttention(\n              (dropout): Dropout(p=0.1, inplace=False)\n              (q_lin): Linear(in_features=768, out_features=768, bias=True)\n              (k_lin): Linear(in_features=768, out_features=768, bias=True)\n              (v_lin): Linear(in_features=768, out_features=768, bias=True)\n              (out_lin): Linear(in_features=768, out_features=768, bias=True)\n            )\n            (sa_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n            (ffn): FFN(\n              (dropout): Dropout(p=0.1, inplace=False)\n              (lin1): Linear(in_features=768, out_features=3072, bias=True)\n              (lin2): Linear(in_features=3072, out_features=768, bias=True)\n              (activation): GELUActivation()\n            )\n            (output_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n          )\n          (5): TransformerBlock(\n            (attention): MultiHeadSelfAttention(\n              (dropout): Dropout(p=0.1, inplace=False)\n              (q_lin): Linear(in_features=768, out_features=768, bias=True)\n              (k_lin): Linear(in_features=768, out_features=768, bias=True)\n              (v_lin): Linear(in_features=768, out_features=768, bias=True)\n              (out_lin): Linear(in_features=768, out_features=768, bias=True)\n            )\n            (sa_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n            (ffn): FFN(\n              (dropout): Dropout(p=0.1, inplace=False)\n              (lin1): Linear(in_features=768, out_features=3072, bias=True)\n              (lin2): Linear(in_features=3072, out_features=768, bias=True)\n              (activation): GELUActivation()\n            )\n            (output_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n          )\n        )\n      )\n    )\n  )\n  (deephead): Sequential(\n    (0): MLP(\n      (mlp): Sequential(\n        (dense_layer_0): Sequential(\n          (0): Linear(in_features=768, out_features=256, bias=True)\n          (1): ReLU(inplace=True)\n          (2): Dropout(p=0.1, inplace=False)\n        )\n        (dense_layer_1): Sequential(\n          (0): Linear(in_features=256, out_features=128, bias=True)\n          (1): ReLU(inplace=True)\n          (2): Dropout(p=0.1, inplace=False)\n        )\n        (dense_layer_2): Sequential(\n          (0): Linear(in_features=128, out_features=64, bias=True)\n          (1): ReLU(inplace=True)\n          (2): Dropout(p=0.1, inplace=False)\n        )\n      )\n    )\n    (1): Linear(in_features=64, out_features=4, bias=True)\n  )\n)</pre> In\u00a0[57]: Copied! <pre>wd_bert_trainer = Trainer(\n    wd_bert_model,\n    objective=\"multiclass\",\n    metrics=[Accuracy, F1Score(average=True)],\n    num_workers=0,  # As in the case of the tokenizer, in notebook I need to set this to 0 for the Trainer to work\n)\n\nwd_bert_trainer.fit(\n    X_text=X_bert_tr,\n    target=train.rating.values,\n    n_epochs=3,\n    batch_size=64,\n)\n</pre> wd_bert_trainer = Trainer(     wd_bert_model,     objective=\"multiclass\",     metrics=[Accuracy, F1Score(average=True)],     num_workers=0,  # As in the case of the tokenizer, in notebook I need to set this to 0 for the Trainer to work )  wd_bert_trainer.fit(     X_text=X_bert_tr,     target=train.rating.values,     n_epochs=3,     batch_size=64, ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 283/283 [00:14&lt;00:00, 19.68it/s, loss=0.968, metrics={'acc': 0.5879, 'f1': 0.3591}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 283/283 [00:14&lt;00:00, 19.63it/s, loss=0.884, metrics={'acc': 0.6178, 'f1': 0.4399}]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 283/283 [00:14&lt;00:00, 19.55it/s, loss=0.87, metrics={'acc': 0.6234, 'f1': 0.4527}]\n</pre> In\u00a0[58]: Copied! <pre>wd_bert_pred_text = wd_bert_trainer.predict_proba(X_text=X_bert_te)\nwd_bert_pred_text_class = np.argmax(wd_bert_pred_text, 1)\n</pre> wd_bert_pred_text = wd_bert_trainer.predict_proba(X_text=X_bert_te) wd_bert_pred_text_class = np.argmax(wd_bert_pred_text, 1) <pre>predict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 71/71 [00:03&lt;00:00, 21.97it/s]\n</pre> In\u00a0[59]: Copied! <pre>wd_bert_acc = accuracy_score(test.rating, wd_bert_pred_text_class)\nwd_bert_f1 = f1_score(test.rating, wd_bert_pred_text_class, average=\"weighted\")\nwd_bert_cm = confusion_matrix(test.rating, wd_bert_pred_text_class)\n</pre> wd_bert_acc = accuracy_score(test.rating, wd_bert_pred_text_class) wd_bert_f1 = f1_score(test.rating, wd_bert_pred_text_class, average=\"weighted\") wd_bert_cm = confusion_matrix(test.rating, wd_bert_pred_text_class) In\u00a0[60]: Copied! <pre>print(f\"Distilbert Accuracy: {wd_bert_acc}. Distilbert F1 Score: {wd_bert_f1}\")\nprint(f\"Distilbert Confusion Matrix:\\n {wd_bert_cm}\")\n</pre> print(f\"Distilbert Accuracy: {wd_bert_acc}. Distilbert F1 Score: {wd_bert_f1}\") print(f\"Distilbert Confusion Matrix:\\n {wd_bert_cm}\") <pre>Distilbert Accuracy: 0.6326846528084918. Distilbert F1 Score: 0.5796652991272998\nDistilbert Confusion Matrix:\n [[ 287   75   22   90]\n [ 197  136   62  169]\n [  68  119  123  671]\n [  40   64   84 2315]]\n</pre> <p>Now, adding a tabular model follows the exact same process as the one described in section 2.</p> In\u00a0[61]: Copied! <pre>tab_model = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    mlp_hidden_dims=[100, 50],\n)\n</pre> tab_model = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     mlp_hidden_dims=[100, 50], ) In\u00a0[62]: Copied! <pre>wd_tab_bert_model = WideDeep(\n    deeptabular=tab_model,\n    deeptext=bert_model,\n    head_hidden_dims=[256, 128, 64],\n    pred_dim=4,\n)\n</pre> wd_tab_bert_model = WideDeep(     deeptabular=tab_model,     deeptext=bert_model,     head_hidden_dims=[256, 128, 64],     pred_dim=4, ) In\u00a0[63]: Copied! <pre>wd_tab_bert_trainer = Trainer(\n    wd_tab_bert_model,\n    objective=\"multiclass\",\n    metrics=[Accuracy, F1Score(average=True)],\n    num_workers=0,  # As in the case of the tokenizer, in notebook I need to set this to 0 for the Trainer to work\n)\n</pre> wd_tab_bert_trainer = Trainer(     wd_tab_bert_model,     objective=\"multiclass\",     metrics=[Accuracy, F1Score(average=True)],     num_workers=0,  # As in the case of the tokenizer, in notebook I need to set this to 0 for the Trainer to work ) In\u00a0[64]: Copied! <pre>wd_tab_bert_trainer.fit(\n    X_tab=wd_X_tab_tr,\n    X_text=X_bert_tr,\n    target=train.rating.values,\n    n_epochs=3,\n    batch_size=64,\n)\n</pre> wd_tab_bert_trainer.fit(     X_tab=wd_X_tab_tr,     X_text=X_bert_tr,     target=train.rating.values,     n_epochs=3,     batch_size=64, ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 283/283 [00:15&lt;00:00, 18.15it/s, loss=0.974, metrics={'acc': 0.5838, 'f1': 0.3404}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 283/283 [00:15&lt;00:00, 18.38it/s, loss=0.885, metrics={'acc': 0.618, 'f1': 0.4378}]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 283/283 [00:15&lt;00:00, 18.40it/s, loss=0.868, metrics={'acc': 0.6252, 'f1': 0.4575}]\n</pre> In\u00a0[65]: Copied! <pre>wd_tab_bert_pred_text = wd_tab_bert_trainer.predict_proba(\n    X_tab=wd_X_tab_te, X_text=X_bert_te\n)\nwd_tab_bert_pred_text_class = np.argmax(wd_tab_bert_pred_text, 1)\n</pre> wd_tab_bert_pred_text = wd_tab_bert_trainer.predict_proba(     X_tab=wd_X_tab_te, X_text=X_bert_te ) wd_tab_bert_pred_text_class = np.argmax(wd_tab_bert_pred_text, 1) <pre>predict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 71/71 [00:03&lt;00:00, 21.32it/s]\n</pre> In\u00a0[66]: Copied! <pre>wd_tab_bert_acc = accuracy_score(test.rating, wd_tab_bert_pred_text_class)\nwd_tab_bert_f1 = f1_score(test.rating, wd_tab_bert_pred_text_class, average=\"weighted\")\nwd_tab_bert_cm = confusion_matrix(test.rating, wd_tab_bert_pred_text_class)\n</pre> wd_tab_bert_acc = accuracy_score(test.rating, wd_tab_bert_pred_text_class) wd_tab_bert_f1 = f1_score(test.rating, wd_tab_bert_pred_text_class, average=\"weighted\") wd_tab_bert_cm = confusion_matrix(test.rating, wd_tab_bert_pred_text_class) In\u00a0[67]: Copied! <pre>print(\n    f\"Distilbert + Tabular Accuracy: {wd_tab_bert_acc}. Distilbert+ Tabular F1 Score: {wd_tab_bert_f1}\"\n)\nprint(f\"Distilbert + Tabular Confusion Matrix:\\n {wd_tab_bert_cm}\")\n</pre> print(     f\"Distilbert + Tabular Accuracy: {wd_tab_bert_acc}. Distilbert+ Tabular F1 Score: {wd_tab_bert_f1}\" ) print(f\"Distilbert + Tabular Confusion Matrix:\\n {wd_tab_bert_cm}\") <pre>Distilbert + Tabular Accuracy: 0.6242812914639541. Distilbert+ Tabular F1 Score: 0.5508351761564895\nDistilbert + Tabular Confusion Matrix:\n [[ 297   56   11  110]\n [ 229   91   38  206]\n [  86   90   71  734]\n [  49   48   42 2364]]\n</pre>"},{"location":"examples/16_Usign_a_custom_hugging_face_model.html#using-a-hugginface-model","title":"Using a Hugginface model\u00b6","text":"<p>In this notebook we will show how to use an \"external\" Hugginface model along with any other model in the libray. In particular we will show how to combine it with a tabular DL model.</p> <p>Since we are here, we will also compare the performance of a few models on a text classification problem.</p> <p>The notebook will go as follows:</p> <ol> <li>Text classification using tf-idf + LightGBM</li> <li>Text classification using a basic RNN</li> <li>Text classification using Distilbert</li> </ol> <p>In all 3 cases we will add some tabular features to see if these help.</p> <p>In general, I would not pay much attention to the results since I have placed no effort in getting the best possible results (i.e. no hyperparameter optimization or trying different architectures, for example).</p> <p>Let's go</p>"},{"location":"examples/16_Usign_a_custom_hugging_face_model.html#1-text-classification-using-tf-idf-lightgbm","title":"1. Text classification using tf-idf + LightGBM\u00b6","text":""},{"location":"examples/16_Usign_a_custom_hugging_face_model.html#2-text-classification-using-pytorch-widedeeps-built-in-models-a-basic-rnn","title":"2. Text classification using pytorch-widedeep's built-in models (a basic RNN)\u00b6","text":"<p>Moving on now to fully using <code>pytorch-widedeep</code> in this dataset, let's have a look on how one could use a simple RNN to predict the ratings with the library.</p>"},{"location":"examples/16_Usign_a_custom_hugging_face_model.html#3-text-classification-using-a-hugginface-model-as-a-custom-model-in-pytorch-widedeeps","title":"3. Text classification using a Hugginface model as a custom model in pytorch-widedeep's\u00b6","text":""},{"location":"examples/17_feature_importance_via_attention_weights.html","title":"17 feature importance via attention weights","text":"In\u00a0[1]: Copied! <pre>import torch\n\nimport numpy as np\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import accuracy_score\n\n\nfrom pytorch_widedeep import Trainer\nfrom pytorch_widedeep.models import TabTransformer, ContextAttentionMLP, WideDeep\nfrom pytorch_widedeep.callbacks import EarlyStopping\nfrom pytorch_widedeep.metrics import Accuracy\nfrom pytorch_widedeep.datasets import load_adult\nfrom pytorch_widedeep.preprocessing import TabPreprocessor\n</pre> import torch  import numpy as np from sklearn.model_selection import train_test_split from sklearn.metrics import accuracy_score   from pytorch_widedeep import Trainer from pytorch_widedeep.models import TabTransformer, ContextAttentionMLP, WideDeep from pytorch_widedeep.callbacks import EarlyStopping from pytorch_widedeep.metrics import Accuracy from pytorch_widedeep.datasets import load_adult from pytorch_widedeep.preprocessing import TabPreprocessor In\u00a0[2]: Copied! <pre># use_cuda = torch.cuda.is_available()\ndf = load_adult(as_frame=True)\ndf.columns = [c.replace(\"-\", \"_\") for c in df.columns]\ndf[\"income_label\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int)\ndf.drop([\"income\", \"fnlwgt\", \"educational_num\"], axis=1, inplace=True)\ntarget_colname = \"income_label\"\n</pre> # use_cuda = torch.cuda.is_available() df = load_adult(as_frame=True) df.columns = [c.replace(\"-\", \"_\") for c in df.columns] df[\"income_label\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int) df.drop([\"income\", \"fnlwgt\", \"educational_num\"], axis=1, inplace=True) target_colname = \"income_label\" In\u00a0[3]: Copied! <pre>df.head()\n</pre> df.head() Out[3]: age workclass education marital_status occupation relationship race gender capital_gain capital_loss hours_per_week native_country income_label 0 25 Private 11th Never-married Machine-op-inspct Own-child Black Male 0 0 40 United-States 0 1 38 Private HS-grad Married-civ-spouse Farming-fishing Husband White Male 0 0 50 United-States 0 2 28 Local-gov Assoc-acdm Married-civ-spouse Protective-serv Husband White Male 0 0 40 United-States 1 3 44 Private Some-college Married-civ-spouse Machine-op-inspct Husband Black Male 7688 0 40 United-States 1 4 18 ? Some-college Never-married ? Own-child White Female 0 0 30 United-States 0 In\u00a0[4]: Copied! <pre>cat_embed_cols = []\nfor col in df.columns:\n    if df[col].dtype == \"O\" or df[col].nunique() &lt; 200 and col != target_colname:\n        cat_embed_cols.append(col)\n</pre> cat_embed_cols = [] for col in df.columns:     if df[col].dtype == \"O\" or df[col].nunique() &lt; 200 and col != target_colname:         cat_embed_cols.append(col) In\u00a0[5]: Copied! <pre># all cols will be categorical\nassert len(cat_embed_cols) == df.shape[1] - 1\n</pre> # all cols will be categorical assert len(cat_embed_cols) == df.shape[1] - 1 In\u00a0[6]: Copied! <pre>train, test = train_test_split(\n    df, test_size=0.1, random_state=1, stratify=df[[target_colname]]\n)\n</pre> train, test = train_test_split(     df, test_size=0.1, random_state=1, stratify=df[[target_colname]] ) In\u00a0[7]: Copied! <pre>tab_preprocessor = TabPreprocessor(cat_embed_cols=cat_embed_cols, with_attention=True)\n</pre> tab_preprocessor = TabPreprocessor(cat_embed_cols=cat_embed_cols, with_attention=True) In\u00a0[8]: Copied! <pre>X_tab_train = tab_preprocessor.fit_transform(train)\nX_tab_test = tab_preprocessor.transform(test)\ntarget = train[target_colname].values\n</pre> X_tab_train = tab_preprocessor.fit_transform(train) X_tab_test = tab_preprocessor.transform(test) target = train[target_colname].values In\u00a0[9]: Copied! <pre>tab_transformer = TabTransformer(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    embed_continuous_method=\"standard\",\n    input_dim=8,\n    n_heads=2,\n    n_blocks=1,\n    attn_dropout=0.1,\n    transformer_activation=\"relu\",\n)\n</pre> tab_transformer = TabTransformer(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     embed_continuous_method=\"standard\",     input_dim=8,     n_heads=2,     n_blocks=1,     attn_dropout=0.1,     transformer_activation=\"relu\", ) In\u00a0[10]: Copied! <pre>model = WideDeep(deeptabular=tab_transformer)\n</pre> model = WideDeep(deeptabular=tab_transformer) In\u00a0[11]: Copied! <pre>optimizer = torch.optim.Adam(model.parameters(), lr=0.01, weight_decay=0.0)\n</pre> optimizer = torch.optim.Adam(model.parameters(), lr=0.01, weight_decay=0.0) In\u00a0[12]: Copied! <pre>lr_scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(\n    optimizer,\n    threshold=0.001,\n    threshold_mode=\"abs\",\n    patience=10,\n)\n</pre> lr_scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(     optimizer,     threshold=0.001,     threshold_mode=\"abs\",     patience=10, ) In\u00a0[13]: Copied! <pre>early_stopping = EarlyStopping(\n    min_delta=0.001, patience=30, restore_best_weights=True, verbose=True\n)\n</pre> early_stopping = EarlyStopping(     min_delta=0.001, patience=30, restore_best_weights=True, verbose=True ) In\u00a0[14]: Copied! <pre>trainer = Trainer(\n    model,\n    objective=\"binary\",\n    optimizers=optimizer,\n    lr_schedulers=lr_scheduler,\n    reducelronplateau_criterion=\"loss\",\n    callbacks=[early_stopping],\n    metrics=[Accuracy],\n)\n</pre> trainer = Trainer(     model,     objective=\"binary\",     optimizers=optimizer,     lr_schedulers=lr_scheduler,     reducelronplateau_criterion=\"loss\",     callbacks=[early_stopping],     metrics=[Accuracy], ) <p>The feature importances will be computed after training, using a sample of the training dataset of size <code>feature_importance_sample_size</code></p> In\u00a0[15]: Copied! <pre>trainer.fit(\n    X_tab=X_tab_train,\n    target=target,\n    val_split=0.2,\n    n_epochs=100,\n    batch_size=128,\n    validation_freq=1,\n    feature_importance_sample_size=1000,\n)\n</pre> trainer.fit(     X_tab=X_tab_train,     target=target,     val_split=0.2,     n_epochs=100,     batch_size=128,     validation_freq=1,     feature_importance_sample_size=1000, ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 81.80it/s, loss=0.334, metrics={'acc': 0.847}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 111.34it/s, loss=0.294, metrics={'acc': 0.8669}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 83.02it/s, loss=0.293, metrics={'acc': 0.8656}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 124.03it/s, loss=0.283, metrics={'acc': 0.8678}]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 87.69it/s, loss=0.282, metrics={'acc': 0.8703}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 118.22it/s, loss=0.279, metrics={'acc': 0.8717}]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 79.24it/s, loss=0.277, metrics={'acc': 0.8718}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 115.29it/s, loss=0.277, metrics={'acc': 0.8731}]\nepoch 5: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 83.76it/s, loss=0.275, metrics={'acc': 0.8727}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 120.80it/s, loss=0.276, metrics={'acc': 0.8727}]\nepoch 6: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 82.78it/s, loss=0.273, metrics={'acc': 0.873}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 100.43it/s, loss=0.276, metrics={'acc': 0.871}]\nepoch 7: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 76.35it/s, loss=0.271, metrics={'acc': 0.8742}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 116.14it/s, loss=0.275, metrics={'acc': 0.8726}]\nepoch 8: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 74.29it/s, loss=0.271, metrics={'acc': 0.875}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 117.46it/s, loss=0.276, metrics={'acc': 0.8718}]\nepoch 9: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 77.10it/s, loss=0.27, metrics={'acc': 0.8761}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 105.49it/s, loss=0.275, metrics={'acc': 0.8728}]\nepoch 10: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 70.40it/s, loss=0.269, metrics={'acc': 0.8747}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 105.47it/s, loss=0.275, metrics={'acc': 0.8726}]\nepoch 11: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 72.83it/s, loss=0.268, metrics={'acc': 0.8742}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 105.03it/s, loss=0.274, metrics={'acc': 0.873}]\nepoch 12: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 71.86it/s, loss=0.267, metrics={'acc': 0.8743}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 106.61it/s, loss=0.274, metrics={'acc': 0.8734}]\nepoch 13: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 72.39it/s, loss=0.267, metrics={'acc': 0.876}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 108.05it/s, loss=0.275, metrics={'acc': 0.8717}]\nepoch 14: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 73.36it/s, loss=0.265, metrics={'acc': 0.8767}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 109.60it/s, loss=0.276, metrics={'acc': 0.8747}]\nepoch 15: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 73.34it/s, loss=0.264, metrics={'acc': 0.876}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 117.55it/s, loss=0.276, metrics={'acc': 0.8706}]\nepoch 16: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 75.35it/s, loss=0.264, metrics={'acc': 0.8777}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 122.08it/s, loss=0.275, metrics={'acc': 0.8753}]\nepoch 17: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 75.96it/s, loss=0.263, metrics={'acc': 0.877}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 120.83it/s, loss=0.277, metrics={'acc': 0.8739}]\nepoch 18: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 75.38it/s, loss=0.263, metrics={'acc': 0.8779}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 118.18it/s, loss=0.278, metrics={'acc': 0.8714}]\nepoch 19: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 75.61it/s, loss=0.261, metrics={'acc': 0.8784}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 119.19it/s, loss=0.278, metrics={'acc': 0.8712}]\nepoch 20: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 72.43it/s, loss=0.261, metrics={'acc': 0.8791}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 118.87it/s, loss=0.28, metrics={'acc': 0.873}]\nepoch 21: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 73.97it/s, loss=0.26, metrics={'acc': 0.8787}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 107.50it/s, loss=0.279, metrics={'acc': 0.8732}]\nepoch 22: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 71.76it/s, loss=0.253, metrics={'acc': 0.8816}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 108.11it/s, loss=0.279, metrics={'acc': 0.8707}]\nepoch 23: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 71.92it/s, loss=0.252, metrics={'acc': 0.8828}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 100.14it/s, loss=0.28, metrics={'acc': 0.8711}]\nepoch 24: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 75.04it/s, loss=0.252, metrics={'acc': 0.8829}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 98.36it/s, loss=0.28, metrics={'acc': 0.8708}]\nepoch 25: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 76.31it/s, loss=0.251, metrics={'acc': 0.883}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 122.97it/s, loss=0.281, metrics={'acc': 0.8709}]\nepoch 26: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 76.63it/s, loss=0.25, metrics={'acc': 0.8834}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 123.07it/s, loss=0.281, metrics={'acc': 0.8698}]\nepoch 27: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 78.37it/s, loss=0.251, metrics={'acc': 0.884}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 114.75it/s, loss=0.281, metrics={'acc': 0.87}]\nepoch 28: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 77.19it/s, loss=0.25, metrics={'acc': 0.883}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 117.99it/s, loss=0.282, metrics={'acc': 0.8699}]\nepoch 29: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 76.19it/s, loss=0.25, metrics={'acc': 0.8829}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 111.11it/s, loss=0.282, metrics={'acc': 0.8695}]\nepoch 30: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 77.57it/s, loss=0.249, metrics={'acc': 0.8839}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 115.64it/s, loss=0.283, metrics={'acc': 0.8689}]\nepoch 31: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 79.55it/s, loss=0.249, metrics={'acc': 0.8846}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 125.10it/s, loss=0.283, metrics={'acc': 0.869}]\nepoch 32: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 82.56it/s, loss=0.248, metrics={'acc': 0.8841}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 118.45it/s, loss=0.284, metrics={'acc': 0.8687}]\nepoch 33: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 81.06it/s, loss=0.248, metrics={'acc': 0.8848}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 129.26it/s, loss=0.284, metrics={'acc': 0.8689}]\nepoch 34: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 80.53it/s, loss=0.248, metrics={'acc': 0.8854}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 120.61it/s, loss=0.283, metrics={'acc': 0.869}]\nepoch 35: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 80.78it/s, loss=0.248, metrics={'acc': 0.8853}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 127.31it/s, loss=0.283, metrics={'acc': 0.8694}]\nepoch 36: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 82.51it/s, loss=0.248, metrics={'acc': 0.8863}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 125.94it/s, loss=0.283, metrics={'acc': 0.8693}]\nepoch 37: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 81.35it/s, loss=0.247, metrics={'acc': 0.8844}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 126.77it/s, loss=0.283, metrics={'acc': 0.8692}]\nepoch 38: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 80.62it/s, loss=0.248, metrics={'acc': 0.8837}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 116.62it/s, loss=0.283, metrics={'acc': 0.8692}]\nepoch 39: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 79.82it/s, loss=0.248, metrics={'acc': 0.8842}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 123.64it/s, loss=0.283, metrics={'acc': 0.8695}]\nepoch 40: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 76.86it/s, loss=0.247, metrics={'acc': 0.8855}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 120.34it/s, loss=0.283, metrics={'acc': 0.8692}]\n</pre> <pre>Best Epoch: 10. Best val_loss: 0.27451\nRestoring model weights from the end of the best epoch\n</pre> In\u00a0[16]: Copied! <pre>trainer.feature_importance\n</pre> trainer.feature_importance Out[16]: <pre>{'age': 0.09718182,\n 'workclass': 0.090637445,\n 'education': 0.08910798,\n 'marital_status': 0.08971319,\n 'occupation': 0.12546304,\n 'relationship': 0.086381145,\n 'race': 0.050686445,\n 'gender': 0.05116429,\n 'capital_gain': 0.08165918,\n 'capital_loss': 0.07702667,\n 'hours_per_week': 0.08205996,\n 'native_country': 0.07891885}</pre> In\u00a0[17]: Copied! <pre>preds = trainer.predict(X_tab=X_tab_test)\n</pre> preds = trainer.predict(X_tab=X_tab_test) <pre>predict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 39/39 [00:00&lt;00:00, 199.63it/s]\n</pre> In\u00a0[18]: Copied! <pre>accuracy_score(preds, test.income_label)\n</pre> accuracy_score(preds, test.income_label) Out[18]: <pre>0.8685772773797339</pre> In\u00a0[19]: Copied! <pre>test.reset_index(drop=True, inplace=True)\n</pre> test.reset_index(drop=True, inplace=True) In\u00a0[20]: Copied! <pre>test[test.income_label == 0].head(1)\n</pre> test[test.income_label == 0].head(1) Out[20]: age workclass education marital_status occupation relationship race gender capital_gain capital_loss hours_per_week native_country income_label 0 26 Private Some-college Never-married Exec-managerial Not-in-family White Male 0 0 60 United-States 0 In\u00a0[21]: Copied! <pre>test[test.income_label == 1].head(1)\n</pre> test[test.income_label == 1].head(1) Out[21]: age workclass education marital_status occupation relationship race gender capital_gain capital_loss hours_per_week native_country income_label 3 36 Local-gov Doctorate Married-civ-spouse Prof-specialty Husband White Male 0 1887 50 United-States 1 <p>To get the feature importance of a test dataset, simply use the <code>explain</code> method</p> In\u00a0[22]: Copied! <pre>feat_imp_per_sample = trainer.explain(X_tab_test, save_step_masks=False)\n</pre> feat_imp_per_sample = trainer.explain(X_tab_test, save_step_masks=False) In\u00a0[23]: Copied! <pre>list(test.iloc[0].index[np.argsort(-feat_imp_per_sample[0])])\n</pre> list(test.iloc[0].index[np.argsort(-feat_imp_per_sample[0])]) Out[23]: <pre>['hours_per_week',\n 'education',\n 'relationship',\n 'occupation',\n 'workclass',\n 'capital_gain',\n 'native_country',\n 'marital_status',\n 'capital_loss',\n 'age',\n 'race',\n 'gender']</pre> In\u00a0[24]: Copied! <pre>list(test.iloc[3].index[np.argsort(-feat_imp_per_sample[3])])\n</pre> list(test.iloc[3].index[np.argsort(-feat_imp_per_sample[3])]) Out[24]: <pre>['age',\n 'capital_loss',\n 'hours_per_week',\n 'marital_status',\n 'native_country',\n 'relationship',\n 'race',\n 'education',\n 'occupation',\n 'capital_gain',\n 'gender',\n 'workclass']</pre> <p>We could do the same with the <code>ContextAttentionMLP</code></p> In\u00a0[25]: Copied! <pre>context_attn_mlp = ContextAttentionMLP(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    cat_embed_dropout=0.0,\n    input_dim=16,\n    attn_dropout=0.1,\n    attn_activation=\"relu\",\n)\n</pre> context_attn_mlp = ContextAttentionMLP(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     cat_embed_dropout=0.0,     input_dim=16,     attn_dropout=0.1,     attn_activation=\"relu\", ) In\u00a0[26]: Copied! <pre>mlp_model = WideDeep(deeptabular=context_attn_mlp)\n</pre> mlp_model = WideDeep(deeptabular=context_attn_mlp) In\u00a0[27]: Copied! <pre>mlp_optimizer = torch.optim.Adam(mlp_model.parameters(), lr=0.01, weight_decay=0.0)\n</pre> mlp_optimizer = torch.optim.Adam(mlp_model.parameters(), lr=0.01, weight_decay=0.0) In\u00a0[28]: Copied! <pre>mlp_lr_scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(\n    mlp_optimizer,\n    threshold=0.001,\n    threshold_mode=\"abs\",\n    patience=10,\n)\n</pre> mlp_lr_scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(     mlp_optimizer,     threshold=0.001,     threshold_mode=\"abs\",     patience=10, ) In\u00a0[29]: Copied! <pre>mlp_early_stopping = EarlyStopping(\n    min_delta=0.001, patience=30, restore_best_weights=True, verbose=True\n)\n</pre> mlp_early_stopping = EarlyStopping(     min_delta=0.001, patience=30, restore_best_weights=True, verbose=True ) In\u00a0[30]: Copied! <pre>mlp_trainer = Trainer(\n    mlp_model,\n    objective=\"binary\",\n    optimizers=mlp_optimizer,\n    lr_schedulers=mlp_lr_scheduler,\n    reducelronplateau_criterion=\"loss\",\n    callbacks=[mlp_early_stopping],\n    metrics=[Accuracy],\n)\n</pre> mlp_trainer = Trainer(     mlp_model,     objective=\"binary\",     optimizers=mlp_optimizer,     lr_schedulers=mlp_lr_scheduler,     reducelronplateau_criterion=\"loss\",     callbacks=[mlp_early_stopping],     metrics=[Accuracy], ) In\u00a0[31]: Copied! <pre>mlp_trainer.fit(\n    X_tab=X_tab_train,\n    target=target,\n    val_split=0.2,\n    n_epochs=100,\n    batch_size=128,\n    validation_freq=1,\n    feature_importance_sample_size=1000,\n)\n</pre> mlp_trainer.fit(     X_tab=X_tab_train,     target=target,     val_split=0.2,     n_epochs=100,     batch_size=128,     validation_freq=1,     feature_importance_sample_size=1000, ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 73.11it/s, loss=0.405, metrics={'acc': 0.8094}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 119.26it/s, loss=0.309, metrics={'acc': 0.8583}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 71.70it/s, loss=0.332, metrics={'acc': 0.8447}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 116.36it/s, loss=0.293, metrics={'acc': 0.8646}]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 78.42it/s, loss=0.319, metrics={'acc': 0.8505}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 118.05it/s, loss=0.293, metrics={'acc': 0.8654}]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 79.00it/s, loss=0.312, metrics={'acc': 0.8554}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 116.49it/s, loss=0.291, metrics={'acc': 0.8661}]\nepoch 5: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 79.50it/s, loss=0.308, metrics={'acc': 0.8583}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 120.48it/s, loss=0.287, metrics={'acc': 0.8669}]\nepoch 6: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 80.84it/s, loss=0.303, metrics={'acc': 0.8605}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 128.70it/s, loss=0.288, metrics={'acc': 0.8673}]\nepoch 7: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 79.93it/s, loss=0.301, metrics={'acc': 0.8597}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 121.47it/s, loss=0.298, metrics={'acc': 0.8628}]\nepoch 8: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 80.56it/s, loss=0.3, metrics={'acc': 0.8592}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 119.84it/s, loss=0.281, metrics={'acc': 0.8718}]\nepoch 9: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 79.17it/s, loss=0.298, metrics={'acc': 0.8619}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 126.32it/s, loss=0.28, metrics={'acc': 0.8716}]\nepoch 10: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 82.13it/s, loss=0.297, metrics={'acc': 0.8615}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 120.50it/s, loss=0.281, metrics={'acc': 0.8718}]\nepoch 11: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 82.54it/s, loss=0.293, metrics={'acc': 0.8641}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 122.57it/s, loss=0.284, metrics={'acc': 0.867}]\nepoch 12: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 80.92it/s, loss=0.293, metrics={'acc': 0.863}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 126.42it/s, loss=0.282, metrics={'acc': 0.8701}]\nepoch 13: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 79.61it/s, loss=0.293, metrics={'acc': 0.8635}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 117.56it/s, loss=0.276, metrics={'acc': 0.8719}]\nepoch 14: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 74.92it/s, loss=0.29, metrics={'acc': 0.8633}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 112.06it/s, loss=0.286, metrics={'acc': 0.8669}]\nepoch 15: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 76.83it/s, loss=0.291, metrics={'acc': 0.865}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 112.88it/s, loss=0.282, metrics={'acc': 0.8677}]\nepoch 16: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 76.74it/s, loss=0.29, metrics={'acc': 0.8653}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 120.85it/s, loss=0.285, metrics={'acc': 0.8672}]\nepoch 17: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 75.99it/s, loss=0.29, metrics={'acc': 0.865}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 113.53it/s, loss=0.282, metrics={'acc': 0.8681}]\nepoch 18: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 71.22it/s, loss=0.288, metrics={'acc': 0.8651}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 112.89it/s, loss=0.288, metrics={'acc': 0.8676}]\nepoch 19: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 75.85it/s, loss=0.29, metrics={'acc': 0.8661}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 114.26it/s, loss=0.284, metrics={'acc': 0.8662}]\nepoch 20: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 75.80it/s, loss=0.289, metrics={'acc': 0.8661}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 119.44it/s, loss=0.281, metrics={'acc': 0.8703}]\nepoch 21: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 75.72it/s, loss=0.29, metrics={'acc': 0.8661}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 113.04it/s, loss=0.285, metrics={'acc': 0.8648}]\nepoch 22: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 74.86it/s, loss=0.289, metrics={'acc': 0.8656}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 111.75it/s, loss=0.282, metrics={'acc': 0.8666}]\nepoch 23: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 76.68it/s, loss=0.289, metrics={'acc': 0.8668}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 115.86it/s, loss=0.282, metrics={'acc': 0.8724}]\nepoch 24: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 75.93it/s, loss=0.288, metrics={'acc': 0.8653}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 114.69it/s, loss=0.285, metrics={'acc': 0.8656}]\nepoch 25: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 79.99it/s, loss=0.284, metrics={'acc': 0.8671}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 118.27it/s, loss=0.277, metrics={'acc': 0.8707}]\nepoch 26: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 79.86it/s, loss=0.282, metrics={'acc': 0.8686}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 123.94it/s, loss=0.276, metrics={'acc': 0.8712}]\nepoch 27: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 77.86it/s, loss=0.283, metrics={'acc': 0.8691}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 114.11it/s, loss=0.277, metrics={'acc': 0.8716}]\nepoch 28: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 77.43it/s, loss=0.281, metrics={'acc': 0.8696}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 114.64it/s, loss=0.277, metrics={'acc': 0.8712}]\nepoch 29: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 77.39it/s, loss=0.281, metrics={'acc': 0.8696}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 115.83it/s, loss=0.277, metrics={'acc': 0.872}]\nepoch 30: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 74.93it/s, loss=0.28, metrics={'acc': 0.8706}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 113.07it/s, loss=0.275, metrics={'acc': 0.8714}]\nepoch 31: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 77.35it/s, loss=0.281, metrics={'acc': 0.8697}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 112.68it/s, loss=0.276, metrics={'acc': 0.872}]\nepoch 32: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 76.72it/s, loss=0.28, metrics={'acc': 0.8693}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 122.50it/s, loss=0.276, metrics={'acc': 0.8709}]\nepoch 33: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 76.68it/s, loss=0.28, metrics={'acc': 0.8716}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 110.07it/s, loss=0.277, metrics={'acc': 0.8709}]\nepoch 34: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 73.58it/s, loss=0.279, metrics={'acc': 0.8704}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 111.07it/s, loss=0.274, metrics={'acc': 0.8719}]\nepoch 35: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 74.03it/s, loss=0.28, metrics={'acc': 0.8687}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 110.15it/s, loss=0.276, metrics={'acc': 0.871}]\nepoch 36: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 73.11it/s, loss=0.279, metrics={'acc': 0.8706}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 111.10it/s, loss=0.278, metrics={'acc': 0.8705}]\nepoch 37: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 74.30it/s, loss=0.279, metrics={'acc': 0.869}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 110.75it/s, loss=0.279, metrics={'acc': 0.8702}]\nepoch 38: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 72.34it/s, loss=0.28, metrics={'acc': 0.8691}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 117.79it/s, loss=0.277, metrics={'acc': 0.8698}]\nepoch 39: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 75.49it/s, loss=0.279, metrics={'acc': 0.8694}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 115.25it/s, loss=0.279, metrics={'acc': 0.87}]\nepoch 40: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 75.51it/s, loss=0.28, metrics={'acc': 0.8694}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 110.90it/s, loss=0.277, metrics={'acc': 0.8694}]\nepoch 41: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 75.35it/s, loss=0.278, metrics={'acc': 0.8716}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 115.74it/s, loss=0.28, metrics={'acc': 0.8675}]\nepoch 42: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 76.35it/s, loss=0.279, metrics={'acc': 0.8695}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 122.76it/s, loss=0.277, metrics={'acc': 0.8699}]\nepoch 43: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:04&lt;00:00, 66.14it/s, loss=0.279, metrics={'acc': 0.8681}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 106.20it/s, loss=0.277, metrics={'acc': 0.8714}]\nepoch 44: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 73.37it/s, loss=0.279, metrics={'acc': 0.8704}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 115.18it/s, loss=0.277, metrics={'acc': 0.8716}]\nepoch 45: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 76.23it/s, loss=0.278, metrics={'acc': 0.8702}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 124.83it/s, loss=0.278, metrics={'acc': 0.8707}]\nepoch 46: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 77.12it/s, loss=0.278, metrics={'acc': 0.8704}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 126.62it/s, loss=0.279, metrics={'acc': 0.8693}]\nepoch 47: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 78.55it/s, loss=0.276, metrics={'acc': 0.8713}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 117.99it/s, loss=0.279, metrics={'acc': 0.8691}]\nepoch 48: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 77.25it/s, loss=0.278, metrics={'acc': 0.8719}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 124.52it/s, loss=0.278, metrics={'acc': 0.8695}]\nepoch 49: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 78.35it/s, loss=0.277, metrics={'acc': 0.8721}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 119.82it/s, loss=0.279, metrics={'acc': 0.8691}]\nepoch 50: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 77.15it/s, loss=0.277, metrics={'acc': 0.8717}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 122.62it/s, loss=0.278, metrics={'acc': 0.8699}]\nepoch 51: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 78.55it/s, loss=0.277, metrics={'acc': 0.8713}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 117.63it/s, loss=0.278, metrics={'acc': 0.87}]\nepoch 52: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 77.04it/s, loss=0.276, metrics={'acc': 0.8721}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 126.39it/s, loss=0.278, metrics={'acc': 0.8697}]\nepoch 53: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 78.15it/s, loss=0.277, metrics={'acc': 0.8721}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 127.56it/s, loss=0.278, metrics={'acc': 0.8699}]\nepoch 54: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 77.41it/s, loss=0.277, metrics={'acc': 0.8711}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 118.95it/s, loss=0.278, metrics={'acc': 0.8698}]\nepoch 55: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 76.35it/s, loss=0.277, metrics={'acc': 0.8718}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 126.90it/s, loss=0.278, metrics={'acc': 0.8699}]\nepoch 56: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 77.83it/s, loss=0.277, metrics={'acc': 0.8707}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 115.13it/s, loss=0.279, metrics={'acc': 0.8691}]\nepoch 57: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 75.20it/s, loss=0.277, metrics={'acc': 0.8722}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 123.16it/s, loss=0.279, metrics={'acc': 0.8691}]\nepoch 58: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 73.33it/s, loss=0.276, metrics={'acc': 0.871}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 123.37it/s, loss=0.278, metrics={'acc': 0.8691}]\nepoch 59: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 78.41it/s, loss=0.277, metrics={'acc': 0.8714}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 125.17it/s, loss=0.278, metrics={'acc': 0.8695}]\nepoch 60: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 79.26it/s, loss=0.276, metrics={'acc': 0.8721}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 120.60it/s, loss=0.278, metrics={'acc': 0.869}]\nepoch 61: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 75.88it/s, loss=0.278, metrics={'acc': 0.8703}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 124.47it/s, loss=0.278, metrics={'acc': 0.8692}]\nepoch 62: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 77.13it/s, loss=0.276, metrics={'acc': 0.8711}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 125.80it/s, loss=0.278, metrics={'acc': 0.8691}]\nepoch 63: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 77.20it/s, loss=0.277, metrics={'acc': 0.8715}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 118.50it/s, loss=0.278, metrics={'acc': 0.8695}]\nepoch 64: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 78.11it/s, loss=0.276, metrics={'acc': 0.8719}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 114.52it/s, loss=0.278, metrics={'acc': 0.869}]\n</pre> <pre>Best Epoch: 34. Best val_loss: 0.27449\nRestoring model weights from the end of the best epoch\n</pre> In\u00a0[32]: Copied! <pre>mlp_trainer.feature_importance\n</pre> mlp_trainer.feature_importance Out[32]: <pre>{'age': 0.116632804,\n 'workclass': 0.050255153,\n 'education': 0.094621316,\n 'marital_status': 0.12328919,\n 'occupation': 0.107893184,\n 'relationship': 0.11747801,\n 'race': 0.054717205,\n 'gender': 0.07514235,\n 'capital_gain': 0.059732802,\n 'capital_loss': 0.06738944,\n 'hours_per_week': 0.0610674,\n 'native_country': 0.07178114}</pre> In\u00a0[33]: Copied! <pre>mlp_preds = mlp_trainer.predict(X_tab=X_tab_test)\n</pre> mlp_preds = mlp_trainer.predict(X_tab=X_tab_test) <pre>predict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 39/39 [00:00&lt;00:00, 212.38it/s]\n</pre> In\u00a0[34]: Copied! <pre>accuracy_score(mlp_preds, test.income_label)\n</pre> accuracy_score(mlp_preds, test.income_label) Out[34]: <pre>0.8726714431934494</pre>"},{"location":"examples/17_feature_importance_via_attention_weights.html#feature-importance-via-the-attention-weights","title":"Feature Importance via the attention weights\u00b6","text":"<p>I will start by saying that I consider this feature of the library purely experimental. First of all I think there are multiple ways one could address finding the features importances for these models. However, and more importantly, one has to bear in mind that even tree-based algorithms on the same dataset produce different feature importances. This is more \"dramatic\" if one uses different techniques, such as shap or feature permutation (see for example this and references therein). All this to say that, sometimes, feature importance is just a measure contained within the experiment run, and for the model used.</p> <p>With that in mind, each instantiation of a deep tabular model, that has millions of trainable parameters, will potentially produce a different set of feature importances, even if the model has the same architecture. Moreover, this effect will become more apparent if the dataset is relatively easy and there are dependent/related columns so that one could get to the same success metric with different parameters.</p> <p>In summary, feature importances are implemented in this librray for all attention-based models for tabular data, with the exception of the <code>TabPerceiver</code>. However this functionality has to be used and interpreted with care and consider of value within the 'universe' (or context) of the model with which these features were produced.</p> <p>Nonetheless, let's have a look to how one would access to the feature importances when using this library.</p>"},{"location":"examples/18_wide_and_deep_for_recsys_pt1.html","title":"18 wide and deep for recsys pt1","text":"<p>The goal of this, and the companion (part 2) notebooks is to illustrate how one could use this library in the context of recommendation systems. In particular, this notebook and the scripts at the <code>wide_deep_for_recsys</code> dir are a response to this issue. Therefore, we will use the Kaggle notebook referred in that issue here.</p> <p>In order to keep the length of the notebook tractable, we will split this exercise in 2. In this first notebook we will prepare the data in almost the exact same way as it is done in the Kaggle notebook and also show how one could use <code>pytorch-widedeep</code> to build a model almost identical to the one in that notebook.</p> <p>In a second notebook, we will show how one could use this library to implement other models, still following the same problem formulation.</p> In\u00a0[1]: Copied! <pre>from pathlib import Path\nimport warnings\n\nimport pandas as pd\nfrom sklearn.model_selection import train_test_split\n\nfrom pytorch_widedeep.datasets import load_movielens100k\n</pre> from pathlib import Path import warnings  import pandas as pd from sklearn.model_selection import train_test_split  from pytorch_widedeep.datasets import load_movielens100k In\u00a0[2]: Copied! <pre>warnings.filterwarnings(\"ignore\")\n</pre> warnings.filterwarnings(\"ignore\") In\u00a0[3]: Copied! <pre>save_path = Path(\"prepared_data\")\nif not save_path.exists():\n    save_path.mkdir(parents=True, exist_ok=True)\n</pre> save_path = Path(\"prepared_data\") if not save_path.exists():     save_path.mkdir(parents=True, exist_ok=True) In\u00a0[4]: Copied! <pre>data, users, items = load_movielens100k(as_frame=True)\n</pre> data, users, items = load_movielens100k(as_frame=True) In\u00a0[5]: Copied! <pre># Alternatively, as specified in the docs: 'The last 19 fields are the genres' so:\n# list_of_genres = items.columns.tolist()[-19:]\nlist_of_genres = [\n    \"unknown\",\n    \"Action\",\n    \"Adventure\",\n    \"Animation\",\n    \"Children's\",\n    \"Comedy\",\n    \"Crime\",\n    \"Documentary\",\n    \"Drama\",\n    \"Fantasy\",\n    \"Film-Noir\",\n    \"Horror\",\n    \"Musical\",\n    \"Mystery\",\n    \"Romance\",\n    \"Sci-Fi\",\n    \"Thriller\",\n    \"War\",\n    \"Western\",\n]\n</pre> # Alternatively, as specified in the docs: 'The last 19 fields are the genres' so: # list_of_genres = items.columns.tolist()[-19:] list_of_genres = [     \"unknown\",     \"Action\",     \"Adventure\",     \"Animation\",     \"Children's\",     \"Comedy\",     \"Crime\",     \"Documentary\",     \"Drama\",     \"Fantasy\",     \"Film-Noir\",     \"Horror\",     \"Musical\",     \"Mystery\",     \"Romance\",     \"Sci-Fi\",     \"Thriller\",     \"War\",     \"Western\", ] <p>Let's first start by loading the interactions, user and item data</p> In\u00a0[6]: Copied! <pre>data.head()\n</pre> data.head() Out[6]: user_id movie_id rating timestamp 0 196 242 3 881250949 1 186 302 3 891717742 2 22 377 1 878887116 3 244 51 2 880606923 4 166 346 1 886397596 In\u00a0[7]: Copied! <pre>users.head()\n</pre> users.head() Out[7]: user_id age gender occupation zip_code 0 1 24 M technician 85711 1 2 53 F other 94043 2 3 23 M writer 32067 3 4 24 M technician 43537 4 5 33 F other 15213 In\u00a0[8]: Copied! <pre>items.head()\n</pre> items.head() Out[8]: movie_id movie_title release_date video_release_date IMDb_URL unknown Action Adventure Animation Children's ... Fantasy Film-Noir Horror Musical Mystery Romance Sci-Fi Thriller War Western 0 1 Toy Story (1995) 01-Jan-1995 NaN http://us.imdb.com/M/title-exact?Toy%20Story%2... 0 0 0 1 1 ... 0 0 0 0 0 0 0 0 0 0 1 2 GoldenEye (1995) 01-Jan-1995 NaN http://us.imdb.com/M/title-exact?GoldenEye%20(... 0 1 1 0 0 ... 0 0 0 0 0 0 0 1 0 0 2 3 Four Rooms (1995) 01-Jan-1995 NaN http://us.imdb.com/M/title-exact?Four%20Rooms%... 0 0 0 0 0 ... 0 0 0 0 0 0 0 1 0 0 3 4 Get Shorty (1995) 01-Jan-1995 NaN http://us.imdb.com/M/title-exact?Get%20Shorty%... 0 1 0 0 0 ... 0 0 0 0 0 0 0 0 0 0 4 5 Copycat (1995) 01-Jan-1995 NaN http://us.imdb.com/M/title-exact?Copycat%20(1995) 0 0 0 0 0 ... 0 0 0 0 0 0 0 1 0 0 <p>5 rows \u00d7 24 columns</p> In\u00a0[9]: Copied! <pre># adding a column with the number of movies watched per user\ndataset = data.sort_values([\"user_id\", \"timestamp\"]).reset_index(drop=True)\ndataset[\"one\"] = 1\ndataset[\"num_watched\"] = dataset.groupby(\"user_id\")[\"one\"].cumsum()\ndataset.drop(\"one\", axis=1, inplace=True)\ndataset.head()\n</pre> # adding a column with the number of movies watched per user dataset = data.sort_values([\"user_id\", \"timestamp\"]).reset_index(drop=True) dataset[\"one\"] = 1 dataset[\"num_watched\"] = dataset.groupby(\"user_id\")[\"one\"].cumsum() dataset.drop(\"one\", axis=1, inplace=True) dataset.head() Out[9]: user_id movie_id rating timestamp num_watched 0 1 168 5 874965478 1 1 1 172 5 874965478 2 2 1 165 5 874965518 3 3 1 156 4 874965556 4 4 1 196 5 874965677 5 In\u00a0[10]: Copied! <pre># adding a column with the mean rating at a point in time per user\ndataset[\"mean_rate\"] = (\n    dataset.groupby(\"user_id\")[\"rating\"].cumsum() / dataset[\"num_watched\"]\n)\ndataset.head()\n</pre> # adding a column with the mean rating at a point in time per user dataset[\"mean_rate\"] = (     dataset.groupby(\"user_id\")[\"rating\"].cumsum() / dataset[\"num_watched\"] ) dataset.head() Out[10]: user_id movie_id rating timestamp num_watched mean_rate 0 1 168 5 874965478 1 5.00 1 1 172 5 874965478 2 5.00 2 1 165 5 874965518 3 5.00 3 1 156 4 874965556 4 4.75 4 1 196 5 874965677 5 4.80 In\u00a0[11]: Copied! <pre>dataset[\"target\"] = dataset.groupby(\"user_id\")[\"movie_id\"].shift(-1)\n</pre> dataset[\"target\"] = dataset.groupby(\"user_id\")[\"movie_id\"].shift(-1) <p>Following the same processing used by the author in the before-mentioned Kaggle notebook, we build sequences of previous movies watched</p> In\u00a0[12]: Copied! <pre># Here the author builds the sequences\ndataset[\"prev_movies\"] = dataset[\"movie_id\"].apply(lambda x: str(x))\ndataset[\"prev_movies\"] = (\n    dataset.groupby(\"user_id\")[\"prev_movies\"]\n    .apply(lambda x: (x + \" \").cumsum().str.strip())\n    .reset_index(drop=True)\n)\ndataset[\"prev_movies\"] = dataset[\"prev_movies\"].apply(lambda x: x.split())\ndataset.head()\n</pre> # Here the author builds the sequences dataset[\"prev_movies\"] = dataset[\"movie_id\"].apply(lambda x: str(x)) dataset[\"prev_movies\"] = (     dataset.groupby(\"user_id\")[\"prev_movies\"]     .apply(lambda x: (x + \" \").cumsum().str.strip())     .reset_index(drop=True) ) dataset[\"prev_movies\"] = dataset[\"prev_movies\"].apply(lambda x: x.split()) dataset.head() Out[12]: user_id movie_id rating timestamp num_watched mean_rate target prev_movies 0 1 168 5 874965478 1 5.00 172.0 [168] 1 1 172 5 874965478 2 5.00 165.0 [168, 172] 2 1 165 5 874965518 3 5.00 156.0 [168, 172, 165] 3 1 156 4 874965556 4 4.75 196.0 [168, 172, 165, 156] 4 1 196 5 874965677 5 4.80 166.0 [168, 172, 165, 156, 196] <p>And now we add a <code>genre_rate</code> as the mean of all movies rated for a given genre per user</p> In\u00a0[13]: Copied! <pre>dataset = dataset.merge(items[[\"movie_id\"] + list_of_genres], on=\"movie_id\", how=\"left\")\nfor genre in list_of_genres:\n    dataset[f\"{genre}_rate\"] = dataset[genre] * dataset[\"rating\"]\n    dataset[genre] = dataset.groupby(\"user_id\")[genre].cumsum()\n    dataset[f\"{genre}_rate\"] = (\n        dataset.groupby(\"user_id\")[f\"{genre}_rate\"].cumsum() / dataset[genre]\n    )\ndataset[list_of_genres] = dataset[list_of_genres].apply(\n    lambda x: x / dataset[\"num_watched\"]\n)\ndataset.head()\n</pre> dataset = dataset.merge(items[[\"movie_id\"] + list_of_genres], on=\"movie_id\", how=\"left\") for genre in list_of_genres:     dataset[f\"{genre}_rate\"] = dataset[genre] * dataset[\"rating\"]     dataset[genre] = dataset.groupby(\"user_id\")[genre].cumsum()     dataset[f\"{genre}_rate\"] = (         dataset.groupby(\"user_id\")[f\"{genre}_rate\"].cumsum() / dataset[genre]     ) dataset[list_of_genres] = dataset[list_of_genres].apply(     lambda x: x / dataset[\"num_watched\"] ) dataset.head() Out[13]: user_id movie_id rating timestamp num_watched mean_rate target prev_movies unknown Action ... Fantasy_rate Film-Noir_rate Horror_rate Musical_rate Mystery_rate Romance_rate Sci-Fi_rate Thriller_rate War_rate Western_rate 0 1 168 5 874965478 1 5.00 172.0 [168] 0.0 0.000000 ... NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN 1 1 172 5 874965478 2 5.00 165.0 [168, 172] 0.0 0.500000 ... NaN NaN NaN NaN NaN 5.0 5.0 NaN 5.0 NaN 2 1 165 5 874965518 3 5.00 156.0 [168, 172, 165] 0.0 0.333333 ... NaN NaN NaN NaN NaN 5.0 5.0 NaN 5.0 NaN 3 1 156 4 874965556 4 4.75 196.0 [168, 172, 165, 156] 0.0 0.250000 ... NaN NaN NaN NaN NaN 5.0 5.0 4.0 5.0 NaN 4 1 196 5 874965677 5 4.80 166.0 [168, 172, 165, 156, 196] 0.0 0.200000 ... NaN NaN NaN NaN NaN 5.0 5.0 4.0 5.0 NaN <p>5 rows \u00d7 46 columns</p> <p>Adding user features</p> In\u00a0[14]: Copied! <pre>dataset = dataset.merge(users, on=\"user_id\", how=\"left\")\ndataset.head()\n</pre> dataset = dataset.merge(users, on=\"user_id\", how=\"left\") dataset.head() Out[14]: user_id movie_id rating timestamp num_watched mean_rate target prev_movies unknown Action ... Mystery_rate Romance_rate Sci-Fi_rate Thriller_rate War_rate Western_rate age gender occupation zip_code 0 1 168 5 874965478 1 5.00 172.0 [168] 0.0 0.000000 ... NaN NaN NaN NaN NaN NaN 24 M technician 85711 1 1 172 5 874965478 2 5.00 165.0 [168, 172] 0.0 0.500000 ... NaN 5.0 5.0 NaN 5.0 NaN 24 M technician 85711 2 1 165 5 874965518 3 5.00 156.0 [168, 172, 165] 0.0 0.333333 ... NaN 5.0 5.0 NaN 5.0 NaN 24 M technician 85711 3 1 156 4 874965556 4 4.75 196.0 [168, 172, 165, 156] 0.0 0.250000 ... NaN 5.0 5.0 4.0 5.0 NaN 24 M technician 85711 4 1 196 5 874965677 5 4.80 166.0 [168, 172, 165, 156, 196] 0.0 0.200000 ... NaN 5.0 5.0 4.0 5.0 NaN 24 M technician 85711 <p>5 rows \u00d7 50 columns</p> <p>Again, we use the same settings as those in the Kaggle notebook, but <code>COLD_START_TRESH</code> is pretty aggressive</p> In\u00a0[15]: Copied! <pre>COLD_START_TRESH = 5\n\nfiltred_data = dataset[\n    (dataset[\"num_watched\"] &gt;= COLD_START_TRESH) &amp; ~(dataset[\"target\"].isna())\n].sort_values(\"timestamp\")\ntrain_data, _test_data = train_test_split(filtred_data, test_size=0.2, shuffle=False)\nvalid_data, test_data = train_test_split(_test_data, test_size=0.5, shuffle=False)\n</pre> COLD_START_TRESH = 5  filtred_data = dataset[     (dataset[\"num_watched\"] &gt;= COLD_START_TRESH) &amp; ~(dataset[\"target\"].isna()) ].sort_values(\"timestamp\") train_data, _test_data = train_test_split(filtred_data, test_size=0.2, shuffle=False) valid_data, test_data = train_test_split(_test_data, test_size=0.5, shuffle=False) In\u00a0[16]: Copied! <pre>cols_to_drop = [\n    # \"rating\",\n    \"timestamp\",\n    \"num_watched\",\n]\n\ndf_train = train_data.drop(cols_to_drop, axis=1)\ndf_valid = valid_data.drop(cols_to_drop, axis=1)\ndf_test = test_data.drop(cols_to_drop, axis=1)\n\ndf_train.to_pickle(save_path / \"df_train.pkl\")\ndf_valid.to_pickle(save_path / \"df_valid.pkl\")\ndf_test.to_pickle(save_path / \"df_test.pkl\")\n</pre> cols_to_drop = [     # \"rating\",     \"timestamp\",     \"num_watched\", ]  df_train = train_data.drop(cols_to_drop, axis=1) df_valid = valid_data.drop(cols_to_drop, axis=1) df_test = test_data.drop(cols_to_drop, axis=1)  df_train.to_pickle(save_path / \"df_train.pkl\") df_valid.to_pickle(save_path / \"df_valid.pkl\") df_test.to_pickle(save_path / \"df_test.pkl\") <p>Let's now build a model that is nearly identical to the one use in the Kaggle notebook</p> In\u00a0[17]: Copied! <pre>import numpy as np\nimport torch\nfrom torch import nn\nfrom scipy.sparse import coo_matrix\n\nfrom pytorch_widedeep import Trainer\nfrom pytorch_widedeep.models import TabMlp, BasicRNN, WideDeep\nfrom pytorch_widedeep.preprocessing import TabPreprocessor\n</pre> import numpy as np import torch from torch import nn from scipy.sparse import coo_matrix  from pytorch_widedeep import Trainer from pytorch_widedeep.models import TabMlp, BasicRNN, WideDeep from pytorch_widedeep.preprocessing import TabPreprocessor In\u00a0[18]: Copied! <pre>device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n\nsave_path = Path(\"prepared_data\")\n\nPAD_IDX = 0\n</pre> device = \"cuda\" if torch.cuda.is_available() else \"cpu\"  save_path = Path(\"prepared_data\")  PAD_IDX = 0 <p>Let's use some of the functions the author of the kaggle's notebook uses to prepare the data</p> In\u00a0[19]: Copied! <pre>def get_coo_indexes(lil):\n    rows = []\n    cols = []\n    for i, el in enumerate(lil):\n        if type(el) != list:\n            el = [el]\n        for j in el:\n            rows.append(i)\n            cols.append(j)\n    return rows, cols\n\n\ndef get_sparse_features(series, shape):\n    coo_indexes = get_coo_indexes(series.tolist())\n    sparse_df = coo_matrix(\n        (np.ones(len(coo_indexes[0])), (coo_indexes[0], coo_indexes[1])), shape=shape\n    )\n    return sparse_df\n\n\ndef sparse_to_idx(data, pad_idx=-1):\n    indexes = data.nonzero()\n    indexes_df = pd.DataFrame()\n    indexes_df[\"rows\"] = indexes[0]\n    indexes_df[\"cols\"] = indexes[1]\n    mdf = indexes_df.groupby(\"rows\").apply(lambda x: x[\"cols\"].tolist())\n    max_len = mdf.apply(lambda x: len(x)).max()\n    return mdf.apply(lambda x: pd.Series(x + [pad_idx] * (max_len - len(x)))).values\n</pre> def get_coo_indexes(lil):     rows = []     cols = []     for i, el in enumerate(lil):         if type(el) != list:             el = [el]         for j in el:             rows.append(i)             cols.append(j)     return rows, cols   def get_sparse_features(series, shape):     coo_indexes = get_coo_indexes(series.tolist())     sparse_df = coo_matrix(         (np.ones(len(coo_indexes[0])), (coo_indexes[0], coo_indexes[1])), shape=shape     )     return sparse_df   def sparse_to_idx(data, pad_idx=-1):     indexes = data.nonzero()     indexes_df = pd.DataFrame()     indexes_df[\"rows\"] = indexes[0]     indexes_df[\"cols\"] = indexes[1]     mdf = indexes_df.groupby(\"rows\").apply(lambda x: x[\"cols\"].tolist())     max_len = mdf.apply(lambda x: len(x)).max()     return mdf.apply(lambda x: pd.Series(x + [pad_idx] * (max_len - len(x)))).values <p>For the time being, we will not use a validation set for hyperparameter optimization, and we will simply concatenate the validation and the test set in one test set. I simply splitted the data into train/valid/test in case the reader wants to actually do hyperparameter optimization (and because I know in the future I will).</p> <p>There is also another caveat worth mentioning, related to the indexing of the movies. To build the matrices of movies watched, we use the entire dataset. A more realistic (and correct) approach would be to use ONLY the movies that appear in the training set and consider <code>unknown</code> or <code>unseen</code> those in the testing set that have not been seen during training. Nonetheless, this will not affect the purposes of this notebook, which is to illustrate how one could use <code>pytorch-widedeep</code> to build a recommendation algorithm. However, if one wanted to explore the performance of different algorithms in a \"proper\" way, these \"details\" need to be accounted for.</p> In\u00a0[20]: Copied! <pre>df_test = pd.concat([df_valid, df_test], ignore_index=True)\n</pre> df_test = pd.concat([df_valid, df_test], ignore_index=True) In\u00a0[21]: Copied! <pre>id_cols = [\"user_id\", \"movie_id\"]\nmax_movie_index = max(df_train.movie_id.max(), df_test.movie_id.max())\n</pre> id_cols = [\"user_id\", \"movie_id\"] max_movie_index = max(df_train.movie_id.max(), df_test.movie_id.max()) In\u00a0[22]: Copied! <pre>X_train = df_train.drop(id_cols + [\"rating\", \"prev_movies\", \"target\"], axis=1)\ny_train = np.array(df_train.target.values, dtype=\"int64\")\ntrain_movies_watched = get_sparse_features(\n    df_train[\"prev_movies\"], (len(df_train), max_movie_index + 1)\n)\n\nX_test = df_test.drop(id_cols + [\"rating\", \"prev_movies\", \"target\"], axis=1)\ny_test = np.array(df_test.target.values, dtype=\"int64\")\ntest_movies_watched = get_sparse_features(\n    df_test[\"prev_movies\"], (len(df_test), max_movie_index + 1)\n)\n</pre> X_train = df_train.drop(id_cols + [\"rating\", \"prev_movies\", \"target\"], axis=1) y_train = np.array(df_train.target.values, dtype=\"int64\") train_movies_watched = get_sparse_features(     df_train[\"prev_movies\"], (len(df_train), max_movie_index + 1) )  X_test = df_test.drop(id_cols + [\"rating\", \"prev_movies\", \"target\"], axis=1) y_test = np.array(df_test.target.values, dtype=\"int64\") test_movies_watched = get_sparse_features(     df_test[\"prev_movies\"], (len(df_test), max_movie_index + 1) ) <p>let's have a look to the information in each dataset</p> In\u00a0[23]: Copied! <pre>X_train.head()\n</pre> X_train.head() Out[23]: mean_rate unknown Action Adventure Animation Children's Comedy Crime Documentary Drama ... Mystery_rate Romance_rate Sci-Fi_rate Thriller_rate War_rate Western_rate age gender occupation zip_code 25423 4.000000 0.0 0.400000 0.200000 0.0 0.0 0.400000 0.0 0.0 0.200000 ... NaN 4.0 4.0 4.000000 4.0 NaN 21 M student 48823 25425 4.000000 0.0 0.285714 0.142857 0.0 0.0 0.428571 0.0 0.0 0.285714 ... NaN 4.0 4.0 4.000000 4.0 NaN 21 M student 48823 25424 4.000000 0.0 0.333333 0.166667 0.0 0.0 0.333333 0.0 0.0 0.333333 ... NaN 4.0 4.0 4.000000 4.0 NaN 21 M student 48823 25426 3.875000 0.0 0.250000 0.125000 0.0 0.0 0.375000 0.0 0.0 0.250000 ... NaN 4.0 4.0 3.666667 4.0 NaN 21 M student 48823 25427 3.888889 0.0 0.222222 0.111111 0.0 0.0 0.333333 0.0 0.0 0.333333 ... NaN 4.0 4.0 3.666667 4.0 NaN 21 M student 48823 <p>5 rows \u00d7 43 columns</p> In\u00a0[24]: Copied! <pre>y_train\n</pre> y_train Out[24]: <pre>array([772, 288, 108, ..., 183, 432, 509])</pre> In\u00a0[25]: Copied! <pre>train_movies_watched\n</pre> train_movies_watched Out[25]: <pre>&lt;76228x1683 sparse matrix of type '&lt;class 'numpy.float64'&gt;'\n\twith 7957390 stored elements in COOrdinate format&gt;</pre> In\u00a0[26]: Copied! <pre>sorted(df_train.prev_movies.tolist()[0])\n</pre> sorted(df_train.prev_movies.tolist()[0]) Out[26]: <pre>['173', '185', '255', '286', '298']</pre> In\u00a0[27]: Copied! <pre>np.where(train_movies_watched.todense()[0])\n</pre> np.where(train_movies_watched.todense()[0]) Out[27]: <pre>(array([0, 0, 0, 0, 0]), array([173, 185, 255, 286, 298]))</pre> <p>And from now on is when the specifics related to this library start to appear. The only component that is going to be a bit different is the so-called tabular component, referred as <code>continuous</code> in the notebook.</p> <p>In the case of <code>pytorch-widedeep</code> we have the <code>TabPreprocessor</code> that allows for a lot of flexibility as to how we would like to process the tabular component of this Wide and Deep model. In other words, here our tabular component is a bit more elaborated than that in the notebook, just a bit...</p> In\u00a0[28]: Copied! <pre>cat_cols = [\"gender\", \"occupation\", \"zip_code\"]\ncont_cols = [c for c in X_train if c not in cat_cols]\ntab_preprocessor = TabPreprocessor(\n    cat_embed_cols=cat_cols,\n    continuous_cols=cont_cols,\n)\n</pre> cat_cols = [\"gender\", \"occupation\", \"zip_code\"] cont_cols = [c for c in X_train if c not in cat_cols] tab_preprocessor = TabPreprocessor(     cat_embed_cols=cat_cols,     continuous_cols=cont_cols, ) In\u00a0[29]: Copied! <pre>X_train_tab = tab_preprocessor.fit_transform(X_train.fillna(0))\nX_test_tab = tab_preprocessor.transform(X_test.fillna(0))\n</pre> X_train_tab = tab_preprocessor.fit_transform(X_train.fillna(0)) X_test_tab = tab_preprocessor.transform(X_test.fillna(0)) <p>Now, in the notebook, the author moves the sparse matrices to sparse tensors and then turns them into dense tensors. In reality, this is not neccessary, one could feed sparse tensors to <code>nn.Linear</code> layers in pytorch. Nonetheless, this is not the most efficient implementation and is the reason why in our library the wide, linear component is implemented as an embedding layer.</p> <p>Nonetheless, to reproduce the notebook the best we can and because currently the <code>Wide</code> model in <code>pytorch-widedeep</code> is not designed to receive sparse tensors (we might consider implementing this functionality), we will turn the sparse COO matrices into dense arrays. We will then code a fairly simple, custom <code>Wide</code> component.</p> In\u00a0[30]: Copied! <pre>X_train_wide = np.array(train_movies_watched.todense())\nX_test_wide = np.array(test_movies_watched.todense())\n</pre> X_train_wide = np.array(train_movies_watched.todense()) X_test_wide = np.array(test_movies_watched.todense()) <p>Finally, the author of the notebook uses a simple <code>Embedding</code> layer to encode the sequences of movies watched, the <code>prev_movies</code> columns. In my opinion, there is an element of information redundancy here. This is because the wide and text components have implicitely the same information, but in different form. Moreover, both of the models used for these two components ignore the sequential element in the data. Nonetheless, we want to reproduce the Kaggle notebook as close as possible, AND as one can explore later (by simply performing simple ablation studies), the wide component seems to carry most of the predictive power.</p> In\u00a0[31]: Copied! <pre>X_train_text = sparse_to_idx(train_movies_watched, pad_idx=PAD_IDX)\nX_test_text = sparse_to_idx(test_movies_watched, pad_idx=PAD_IDX)\n</pre> X_train_text = sparse_to_idx(train_movies_watched, pad_idx=PAD_IDX) X_test_text = sparse_to_idx(test_movies_watched, pad_idx=PAD_IDX) <p>Let's now build the models</p> In\u00a0[32]: Copied! <pre>class Wide(nn.Module):\n    def __init__(self, input_dim: int, pred_dim: int):\n        super().__init__()\n\n        self.input_dim = input_dim\n        self.pred_dim = pred_dim\n\n        # When I coded the library I never though that someone would want to code\n        # their own wide component. However, if you do, the wide component must have\n        # a 'wide_linear' attribute. In other words, the linear layer must be\n        # called 'wide_linear'\n        self.wide_linear = nn.Linear(input_dim, pred_dim)\n\n    def forward(self, X):\n        out = self.wide_linear(X.type(torch.float32))\n        return out\n\n\nwide = Wide(X_train_wide.shape[1], max_movie_index + 1)\n</pre> class Wide(nn.Module):     def __init__(self, input_dim: int, pred_dim: int):         super().__init__()          self.input_dim = input_dim         self.pred_dim = pred_dim          # When I coded the library I never though that someone would want to code         # their own wide component. However, if you do, the wide component must have         # a 'wide_linear' attribute. In other words, the linear layer must be         # called 'wide_linear'         self.wide_linear = nn.Linear(input_dim, pred_dim)      def forward(self, X):         out = self.wide_linear(X.type(torch.float32))         return out   wide = Wide(X_train_wide.shape[1], max_movie_index + 1) In\u00a0[33]: Copied! <pre>wide\n</pre> wide Out[33]: <pre>Wide(\n  (wide_linear): Linear(in_features=1683, out_features=1683, bias=True)\n)</pre> In\u00a0[34]: Copied! <pre>class SimpleEmbed(nn.Module):\n    def __init__(self, vocab_size: int, embed_dim: int, pad_idx: int):\n        super().__init__()\n\n        self.vocab_size = vocab_size\n        self.embed_dim = embed_dim\n        self.pad_idx = pad_idx\n\n        # The sequences of movies watched are simply embedded in the Kaggle\n        # notebook. No RNN, Transformer or any model is used\n        self.embed = nn.Embedding(vocab_size, embed_dim, padding_idx=pad_idx)\n\n    def forward(self, X):\n        embed = self.embed(X)\n        embed_mean = torch.mean(embed, dim=1)\n        return embed_mean\n\n    @property\n    def output_dim(self) -&gt; int:\n        # All deep components in a custom 'pytorch-widedeep' model must have\n        # an output_dim property\n        return self.embed_dim\n\n\n#  In the notebook the author uses simply embeddings\nsimple_embed = SimpleEmbed(max_movie_index + 1, 16, 0)\n</pre> class SimpleEmbed(nn.Module):     def __init__(self, vocab_size: int, embed_dim: int, pad_idx: int):         super().__init__()          self.vocab_size = vocab_size         self.embed_dim = embed_dim         self.pad_idx = pad_idx          # The sequences of movies watched are simply embedded in the Kaggle         # notebook. No RNN, Transformer or any model is used         self.embed = nn.Embedding(vocab_size, embed_dim, padding_idx=pad_idx)      def forward(self, X):         embed = self.embed(X)         embed_mean = torch.mean(embed, dim=1)         return embed_mean      @property     def output_dim(self) -&gt; int:         # All deep components in a custom 'pytorch-widedeep' model must have         # an output_dim property         return self.embed_dim   #  In the notebook the author uses simply embeddings simple_embed = SimpleEmbed(max_movie_index + 1, 16, 0) In\u00a0[35]: Copied! <pre>simple_embed\n</pre> simple_embed Out[35]: <pre>SimpleEmbed(\n  (embed): Embedding(1683, 16, padding_idx=0)\n)</pre> <p>Maybe one would like to use an RNN to account for the sequence nature of the problem. If that was the case it would be as easy as:</p> In\u00a0[36]: Copied! <pre>basic_rnn = BasicRNN(\n    vocab_size=max_movie_index + 1,\n    embed_dim=16,\n    hidden_dim=32,\n    n_layers=2,\n    rnn_type=\"gru\",\n)\n</pre> basic_rnn = BasicRNN(     vocab_size=max_movie_index + 1,     embed_dim=16,     hidden_dim=32,     n_layers=2,     rnn_type=\"gru\", ) <p>And finally, the tabular component, which is the notebook is simply a stak of linear + Rely layers. In our case we have an embedding layer before the linear layers to encode categorial and numerical cols</p> In\u00a0[37]: Copied! <pre>tab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=tab_preprocessor.continuous_cols,\n    cont_norm_layer=None,\n    mlp_hidden_dims=[1024, 512, 256],\n    mlp_activation=\"relu\",\n)\n</pre> tab_mlp = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     continuous_cols=tab_preprocessor.continuous_cols,     cont_norm_layer=None,     mlp_hidden_dims=[1024, 512, 256],     mlp_activation=\"relu\", ) In\u00a0[38]: Copied! <pre>tab_mlp\n</pre> tab_mlp Out[38]: <pre>TabMlp(\n  (cat_embed): DiffSizeCatEmbeddings(\n    (embed_layers): ModuleDict(\n      (emb_layer_gender): Embedding(3, 2, padding_idx=0)\n      (emb_layer_occupation): Embedding(22, 9, padding_idx=0)\n      (emb_layer_zip_code): Embedding(648, 60, padding_idx=0)\n    )\n    (embedding_dropout): Dropout(p=0.0, inplace=False)\n  )\n  (cont_norm): Identity()\n  (encoder): MLP(\n    (mlp): Sequential(\n      (dense_layer_0): Sequential(\n        (0): Linear(in_features=111, out_features=1024, bias=True)\n        (1): ReLU(inplace=True)\n        (2): Dropout(p=0.1, inplace=False)\n      )\n      (dense_layer_1): Sequential(\n        (0): Linear(in_features=1024, out_features=512, bias=True)\n        (1): ReLU(inplace=True)\n        (2): Dropout(p=0.1, inplace=False)\n      )\n      (dense_layer_2): Sequential(\n        (0): Linear(in_features=512, out_features=256, bias=True)\n        (1): ReLU(inplace=True)\n        (2): Dropout(p=0.1, inplace=False)\n      )\n    )\n  )\n)</pre> <p>Finally, we simply wrap up all models with the <code>WideDeep</code> 'collector' class and we are ready to train.</p> In\u00a0[39]: Copied! <pre>wide_deep_model = WideDeep(\n    wide=wide, deeptabular=tab_mlp, deeptext=simple_embed, pred_dim=max_movie_index + 1\n)\n</pre> wide_deep_model = WideDeep(     wide=wide, deeptabular=tab_mlp, deeptext=simple_embed, pred_dim=max_movie_index + 1 ) In\u00a0[40]: Copied! <pre>wide_deep_model\n</pre> wide_deep_model Out[40]: <pre>WideDeep(\n  (wide): Wide(\n    (wide_linear): Linear(in_features=1683, out_features=1683, bias=True)\n  )\n  (deeptabular): Sequential(\n    (0): TabMlp(\n      (cat_embed): DiffSizeCatEmbeddings(\n        (embed_layers): ModuleDict(\n          (emb_layer_gender): Embedding(3, 2, padding_idx=0)\n          (emb_layer_occupation): Embedding(22, 9, padding_idx=0)\n          (emb_layer_zip_code): Embedding(648, 60, padding_idx=0)\n        )\n        (embedding_dropout): Dropout(p=0.0, inplace=False)\n      )\n      (cont_norm): Identity()\n      (encoder): MLP(\n        (mlp): Sequential(\n          (dense_layer_0): Sequential(\n            (0): Linear(in_features=111, out_features=1024, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_1): Sequential(\n            (0): Linear(in_features=1024, out_features=512, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_2): Sequential(\n            (0): Linear(in_features=512, out_features=256, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=256, out_features=1683, bias=True)\n  )\n  (deeptext): Sequential(\n    (0): SimpleEmbed(\n      (embed): Embedding(1683, 16, padding_idx=0)\n    )\n    (1): Linear(in_features=16, out_features=1683, bias=True)\n  )\n)</pre> <p>Note that the main difference between this wide and deep model and the Wide and Deep model in the Kaggle notebook is that in that notebook, the author concatenates the embedings and the tabular features, then passes this concatenation through a stack of linear + Relu layers with a final output dim of 256. Then concatenates this output with the binary features and connects this concatenation with the final linear layer (so the final weights are of dim (batch_size, 256 + 1683)). Our implementation follows the notation of the original paper and instead of concatenating the tabular, text and wide components and then connect them to the output neurons, we first compute their output, and then add it (see here: https://arxiv.org/pdf/1606.07792.pdf, their Eq 3). Note that this is effectively the same, with the caveat that while in one case one initialises a big weight matrix \"at once\", in our implementation we initialise different matrices for different components. Anyway, let's give it a go.</p> In\u00a0[41]: Copied! <pre>trainer = Trainer(\n    model=wide_deep_model,\n    objective=\"multiclass\",\n    custom_loss_function=nn.CrossEntropyLoss(ignore_index=PAD_IDX),\n    optimizers=torch.optim.Adam(wide_deep_model.parameters(), lr=1e-3),\n)\n</pre> trainer = Trainer(     model=wide_deep_model,     objective=\"multiclass\",     custom_loss_function=nn.CrossEntropyLoss(ignore_index=PAD_IDX),     optimizers=torch.optim.Adam(wide_deep_model.parameters(), lr=1e-3), ) In\u00a0[42]: Copied! <pre>trainer.fit(\n    X_train={\n        \"X_wide\": X_train_wide,\n        \"X_tab\": X_train_tab,\n        \"X_text\": X_train_text,\n        \"target\": y_train,\n    },\n    X_val={\n        \"X_wide\": X_test_wide,\n        \"X_tab\": X_test_tab,\n        \"X_text\": X_test_text,\n        \"target\": y_test,\n    },\n    n_epochs=5,\n    batch_size=512,\n    shuffle=False,\n)\n</pre> trainer.fit(     X_train={         \"X_wide\": X_train_wide,         \"X_tab\": X_train_tab,         \"X_text\": X_train_text,         \"target\": y_train,     },     X_val={         \"X_wide\": X_test_wide,         \"X_tab\": X_test_tab,         \"X_text\": X_test_text,         \"target\": y_test,     },     n_epochs=5,     batch_size=512,     shuffle=False, ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 149/149 [00:19&lt;00:00,  7.66it/s, loss=6.66]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 38/38 [00:02&lt;00:00, 18.75it/s, loss=6.6]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 149/149 [00:21&lt;00:00,  6.95it/s, loss=5.97]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 38/38 [00:01&lt;00:00, 21.03it/s, loss=6.52]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 149/149 [00:19&lt;00:00,  7.51it/s, loss=5.65]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 38/38 [00:01&lt;00:00, 20.16it/s, loss=6.53]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 149/149 [00:23&lt;00:00,  6.29it/s, loss=5.41]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 38/38 [00:02&lt;00:00, 13.97it/s, loss=6.57]\nepoch 5: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 149/149 [00:19&lt;00:00,  7.58it/s, loss=5.2]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 38/38 [00:02&lt;00:00, 18.82it/s, loss=6.63]\n</pre> <p>Now one could continue to the 'compare' metrics section of the Kaggle notebook. However, for the purposes of illustrating how one could use <code>pytorch-widedeep</code> to build recommendation algorithms we consider this notebook completed and move onto part 2</p>"},{"location":"examples/18_wide_and_deep_for_recsys_pt1.html#problem-formulation","title":"Problem formulation\u00b6","text":"<p>In this particular exercise the problem is formulated as predicting the next movie that will be watched (in consequence the last interactions will be discarded)</p>"},{"location":"examples/18_wide_and_deep_for_recsys_pt2.html","title":"18 wide and deep for recsys pt2","text":"<p>This is the second of the two notebooks where we aim to illustrate how one could use this library to build recommendation algorithms using the example in this Kaggle notebook as guidance. In the previous notebook we used <code>pytorch-widedeep</code> to build a model that replicated almost exactly that in the notebook. In this, shorter notebook we will show how one could use the library to explore other models, following the same problem formulation, this is: given a state of a user at a certain point in time having watched a series of movies, our goal is to predict which movie the user will watch next.</p> <p>Assuming that one has read (and run) the previous notebook, the required data will be stored in a local dir called <code>prepared_data</code>, so let's read it:</p> In\u00a0[1]: Copied! <pre>from pathlib import Path\n\nimport numpy as np\nimport torch\nimport pandas as pd\nfrom torch import nn\n\nfrom pytorch_widedeep import Trainer\nfrom pytorch_widedeep.utils import pad_sequences\nfrom pytorch_widedeep.models import TabMlp, WideDeep, Transformer\nfrom pytorch_widedeep.preprocessing import TabPreprocessor\n</pre> from pathlib import Path  import numpy as np import torch import pandas as pd from torch import nn  from pytorch_widedeep import Trainer from pytorch_widedeep.utils import pad_sequences from pytorch_widedeep.models import TabMlp, WideDeep, Transformer from pytorch_widedeep.preprocessing import TabPreprocessor In\u00a0[2]: Copied! <pre>save_path = Path(\"prepared_data\")\n\nPAD_IDX = 0\n\nid_cols = [\"user_id\", \"movie_id\"]\n\ndf_train = pd.read_pickle(save_path / \"df_train.pkl\")\ndf_valid = pd.read_pickle(save_path / \"df_valid.pkl\")\ndf_test = pd.read_pickle(save_path / \"df_test.pkl\")\n</pre> save_path = Path(\"prepared_data\")  PAD_IDX = 0  id_cols = [\"user_id\", \"movie_id\"]  df_train = pd.read_pickle(save_path / \"df_train.pkl\") df_valid = pd.read_pickle(save_path / \"df_valid.pkl\") df_test = pd.read_pickle(save_path / \"df_test.pkl\") <p>...remember that in the previous notebook we explained that we are not  going to use a validation set here (in a real-world example, or simply a more realistic example, one should always use it).</p> In\u00a0[3]: Copied! <pre>df_test = pd.concat([df_valid, df_test], ignore_index=True)\n</pre> df_test = pd.concat([df_valid, df_test], ignore_index=True) <p>Also remember that, in the previous notebook we discussed that the <code>'maxlen'</code> and <code>'max_movie_index'</code> parameters should be computed using only the train set. In particular, to properly do the tokenization, one would have to use ONLY train tokens and add a token for new 'unknown'/'unseen' movies in the test set. This can also be done with this library or manually, so I will leave it to the reader to implement that tokenzation appraoch.</p> In\u00a0[4]: Copied! <pre>maxlen = max(\n    df_train.prev_movies.apply(lambda x: len(x)).max(),\n    df_test.prev_movies.apply(lambda x: len(x)).max(),\n)\n\nmax_movie_index = max(df_train.movie_id.max(), df_test.movie_id.max())\n</pre> maxlen = max(     df_train.prev_movies.apply(lambda x: len(x)).max(),     df_test.prev_movies.apply(lambda x: len(x)).max(), )  max_movie_index = max(df_train.movie_id.max(), df_test.movie_id.max()) <p>From now one things are pretty simple, moreover bearing in mind that in this example we are not going to use a wide component since, in pple, one would believe that the information in that component is also 'carried' by the movie sequences (However in the previous notebook, if one performs ablation studies, these suggest that most of the prediction power comes from the linear, wide model).</p> <p>In the example here we are going to explore one (of many) possibilities. We are simply going to encode the triplet <code>(user, item, rating)</code> and use it as a <code>deeptabular</code> component and the sequences of previously watched movies as the <code>deeptext</code> component. For the <code>deeptext</code> component we are going to use a basic encoder-only transformer model.</p> <p>Let's start with the tabular data preparation</p> In\u00a0[5]: Copied! <pre>df_train_user_item = df_train[[\"user_id\", \"movie_id\", \"rating\"]]\ntrain_movies_sequences = df_train.prev_movies.apply(\n    lambda x: [int(el) for el in x]\n).to_list()\ny_train = df_train.target.values.astype(int)\n\ndf_test_user_item = df_train[[\"user_id\", \"movie_id\", \"rating\"]]\ntest_movies_sequences = df_test.prev_movies.apply(\n    lambda x: [int(el) for el in x]\n).to_list()\ny_test = df_test.target.values.astype(int)\n\ntab_preprocessor = tab_preprocessor = TabPreprocessor(\n    cat_embed_cols=[\"user_id\", \"movie_id\", \"rating\"],\n)\nX_train_tab = tab_preprocessor.fit_transform(df_train_user_item)\nX_test_tab = tab_preprocessor.transform(df_test_user_item)\n</pre> df_train_user_item = df_train[[\"user_id\", \"movie_id\", \"rating\"]] train_movies_sequences = df_train.prev_movies.apply(     lambda x: [int(el) for el in x] ).to_list() y_train = df_train.target.values.astype(int)  df_test_user_item = df_train[[\"user_id\", \"movie_id\", \"rating\"]] test_movies_sequences = df_test.prev_movies.apply(     lambda x: [int(el) for el in x] ).to_list() y_test = df_test.target.values.astype(int)  tab_preprocessor = tab_preprocessor = TabPreprocessor(     cat_embed_cols=[\"user_id\", \"movie_id\", \"rating\"], ) X_train_tab = tab_preprocessor.fit_transform(df_train_user_item) X_test_tab = tab_preprocessor.transform(df_test_user_item) <p>And not the text component, simply padding the sequences:</p> In\u00a0[6]: Copied! <pre>X_train_text = np.array(\n    [\n        pad_sequences(\n            s,\n            maxlen=maxlen,\n            pad_first=False,\n            pad_idx=PAD_IDX,\n        )\n        for s in train_movies_sequences\n    ]\n)\nX_test_text = np.array(\n    [\n        pad_sequences(\n            s,\n            maxlen=maxlen,\n            pad_first=False,\n            pad_idx=0,\n        )\n        for s in test_movies_sequences\n    ]\n)\n</pre> X_train_text = np.array(     [         pad_sequences(             s,             maxlen=maxlen,             pad_first=False,             pad_idx=PAD_IDX,         )         for s in train_movies_sequences     ] ) X_test_text = np.array(     [         pad_sequences(             s,             maxlen=maxlen,             pad_first=False,             pad_idx=0,         )         for s in test_movies_sequences     ] ) <p>We now define the model components and the wide and deep model.</p> In\u00a0[7]: Copied! <pre>tab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    mlp_hidden_dims=[1024, 512, 256],\n    mlp_activation=\"relu\",\n)\n\n# plenty of options here, see the docs\ntransformer = Transformer(\n    vocab_size=max_movie_index + 1,\n    embed_dim=32,\n    n_heads=2,\n    n_blocks=2,\n    seq_length=maxlen,\n)\n\nwide_deep_model = WideDeep(\n    deeptabular=tab_mlp, deeptext=transformer, pred_dim=max_movie_index + 1\n)\n</pre> tab_mlp = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     mlp_hidden_dims=[1024, 512, 256],     mlp_activation=\"relu\", )  # plenty of options here, see the docs transformer = Transformer(     vocab_size=max_movie_index + 1,     embed_dim=32,     n_heads=2,     n_blocks=2,     seq_length=maxlen, )  wide_deep_model = WideDeep(     deeptabular=tab_mlp, deeptext=transformer, pred_dim=max_movie_index + 1 ) In\u00a0[8]: Copied! <pre>wide_deep_model\n</pre> wide_deep_model Out[8]: <pre>WideDeep(\n  (deeptabular): Sequential(\n    (0): TabMlp(\n      (cat_embed): DiffSizeCatEmbeddings(\n        (embed_layers): ModuleDict(\n          (emb_layer_user_id): Embedding(749, 65, padding_idx=0)\n          (emb_layer_movie_id): Embedding(1612, 100, padding_idx=0)\n          (emb_layer_rating): Embedding(6, 4, padding_idx=0)\n        )\n        (embedding_dropout): Dropout(p=0.0, inplace=False)\n      )\n      (encoder): MLP(\n        (mlp): Sequential(\n          (dense_layer_0): Sequential(\n            (0): Linear(in_features=169, out_features=1024, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_1): Sequential(\n            (0): Linear(in_features=1024, out_features=512, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_2): Sequential(\n            (0): Linear(in_features=512, out_features=256, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=256, out_features=1683, bias=True)\n  )\n  (deeptext): Sequential(\n    (0): Transformer(\n      (embedding): Embedding(1683, 32, padding_idx=0)\n      (pos_encoder): PositionalEncoding(\n        (dropout): Dropout(p=0.1, inplace=False)\n      )\n      (encoder): Sequential(\n        (transformer_block0): TransformerEncoder(\n          (attn): MultiHeadedAttention(\n            (dropout): Dropout(p=0.1, inplace=False)\n            (q_proj): Linear(in_features=32, out_features=32, bias=False)\n            (kv_proj): Linear(in_features=32, out_features=64, bias=False)\n            (out_proj): Linear(in_features=32, out_features=32, bias=False)\n          )\n          (ff): FeedForward(\n            (w_1): Linear(in_features=32, out_features=128, bias=True)\n            (w_2): Linear(in_features=128, out_features=32, bias=True)\n            (dropout): Dropout(p=0.1, inplace=False)\n            (activation): GELU(approximate='none')\n          )\n          (attn_addnorm): AddNorm(\n            (dropout): Dropout(p=0.1, inplace=False)\n            (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n          )\n          (ff_addnorm): AddNorm(\n            (dropout): Dropout(p=0.1, inplace=False)\n            (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n          )\n        )\n        (transformer_block1): TransformerEncoder(\n          (attn): MultiHeadedAttention(\n            (dropout): Dropout(p=0.1, inplace=False)\n            (q_proj): Linear(in_features=32, out_features=32, bias=False)\n            (kv_proj): Linear(in_features=32, out_features=64, bias=False)\n            (out_proj): Linear(in_features=32, out_features=32, bias=False)\n          )\n          (ff): FeedForward(\n            (w_1): Linear(in_features=32, out_features=128, bias=True)\n            (w_2): Linear(in_features=128, out_features=32, bias=True)\n            (dropout): Dropout(p=0.1, inplace=False)\n            (activation): GELU(approximate='none')\n          )\n          (attn_addnorm): AddNorm(\n            (dropout): Dropout(p=0.1, inplace=False)\n            (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n          )\n          (ff_addnorm): AddNorm(\n            (dropout): Dropout(p=0.1, inplace=False)\n            (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=23552, out_features=1683, bias=True)\n  )\n)</pre> <p>And as in the previous notebook, let's train (you will need a GPU for this)</p> In\u00a0[\u00a0]: Copied! <pre>trainer = Trainer(\n    model=wide_deep_model,\n    objective=\"multiclass\",\n    custom_loss_function=nn.CrossEntropyLoss(ignore_index=PAD_IDX),\n    optimizers=torch.optim.Adam(wide_deep_model.parameters(), lr=1e-3),\n)\n\ntrainer.fit(\n    X_train={\n        \"X_tab\": X_train_tab,\n        \"X_text\": X_train_text,\n        \"target\": y_train,\n    },\n    X_val={\n        \"X_tab\": X_test_tab,\n        \"X_text\": X_test_text,\n        \"target\": y_test,\n    },\n    n_epochs=10,\n    batch_size=521,\n    shuffle=False,\n)\n</pre> trainer = Trainer(     model=wide_deep_model,     objective=\"multiclass\",     custom_loss_function=nn.CrossEntropyLoss(ignore_index=PAD_IDX),     optimizers=torch.optim.Adam(wide_deep_model.parameters(), lr=1e-3), )  trainer.fit(     X_train={         \"X_tab\": X_train_tab,         \"X_text\": X_train_text,         \"target\": y_train,     },     X_val={         \"X_tab\": X_test_tab,         \"X_text\": X_test_text,         \"target\": y_test,     },     n_epochs=10,     batch_size=521,     shuffle=False, ) <pre>epoch 1:   0%|                                                                                                         | 0/147 [00:34&lt;?, ?it/s]\n</pre> In\u00a0[\u00a0]: Copied! <pre>\n</pre>"},{"location":"examples/19_load_from_folder_functionality.html","title":"19 load from folder functionality","text":"<p>In this notebook I want to illustrate how one can use our <code>[...]FromFolder</code> functionalities along with the <code>[...]ChunkPreProcessors</code> in those cases where the dataset is too bit to fit in memory.</p> <p>These functionalities in the library have been designed for the following scenarop</p> In\u00a0[1]: Copied! <pre>import numpy as np\nimport torch\nimport pandas as pd\nfrom torch.utils.data import DataLoader\n\nfrom pytorch_widedeep.models import TabMlp, Vision, BasicRNN, WideDeep\nfrom pytorch_widedeep.training import TrainerFromFolder\nfrom pytorch_widedeep.callbacks import EarlyStopping, ModelCheckpoint\nfrom pytorch_widedeep.preprocessing import (\n    TabPreprocessor,\n    TextPreprocessor,\n    ImagePreprocessor,\n    ChunkTabPreprocessor,\n    ChunkTextPreprocessor,\n)\nfrom pytorch_widedeep.load_from_folder import (\n    TabFromFolder,\n    TextFromFolder,\n    ImageFromFolder,\n    WideDeepDatasetFromFolder,\n)\n</pre> import numpy as np import torch import pandas as pd from torch.utils.data import DataLoader  from pytorch_widedeep.models import TabMlp, Vision, BasicRNN, WideDeep from pytorch_widedeep.training import TrainerFromFolder from pytorch_widedeep.callbacks import EarlyStopping, ModelCheckpoint from pytorch_widedeep.preprocessing import (     TabPreprocessor,     TextPreprocessor,     ImagePreprocessor,     ChunkTabPreprocessor,     ChunkTextPreprocessor, ) from pytorch_widedeep.load_from_folder import (     TabFromFolder,     TextFromFolder,     ImageFromFolder,     WideDeepDatasetFromFolder, ) In\u00a0[2]: Copied! <pre># in my case, I place the data in a folder I call tmp_data, let's see how it looks\nairbnb_data = pd.read_csv(\"../tmp_data/airbnb/airbnb_sample.csv\")\nairbnb_data.sample(5)\n</pre> # in my case, I place the data in a folder I call tmp_data, let's see how it looks airbnb_data = pd.read_csv(\"../tmp_data/airbnb/airbnb_sample.csv\") airbnb_data.sample(5) Out[2]: id host_id description host_listings_count host_identity_verified neighbourhood_cleansed latitude longitude is_location_exact property_type ... amenity_wide_entrance amenity_wide_entrance_for_guests amenity_wide_entryway amenity_wide_hallways amenity_wifi amenity_window_guards amenity_wine_cooler security_deposit extra_people yield 39 53242.jpg 247650 A lovely big bright bedroom in a 2 bedroom fla... 2.0 t Lambeth 51.47075 -0.12913 t apartment ... 0 0 0 0 1 0 0 250.0 5.0 9.75 214 236716.jpg 1241070 We offer a warm welcome in our quiet double ro... 1.0 t Hackney 51.56593 -0.07482 t other ... 0 0 0 0 1 0 0 200.0 10.0 76.50 400 346523.jpg 1756532 Available for you to rent is a cozy studio in ... 2.0 t Kensington and Chelsea 51.48311 -0.18428 t other ... 0 0 0 0 1 0 0 0.0 50.0 180.90 512 389627.jpg 1949299 This gorgeous studio flat is situated in the v... 1.0 t Westminster 51.51838 -0.14238 f apartment ... 0 0 0 0 1 0 0 250.0 25.0 276.90 504 388767.jpg 1945165 If you want to experience London at it's best ... 2.0 f Camden 51.54293 -0.14073 t apartment ... 0 0 0 0 1 0 0 150.0 10.0 591.10 <p>5 rows \u00d7 223 columns</p> In\u00a0[3]: Copied! <pre># for example\nfrom IPython.display import Image\n\npil_img = Image(filename=\"../tmp_data/airbnb/property_picture/272908.jpg\")\ndisplay(pil_img)\n</pre> # for example from IPython.display import Image  pil_img = Image(filename=\"../tmp_data/airbnb/property_picture/272908.jpg\") display(pil_img) In\u00a0[4]: Copied! <pre># And the description for the property that that picture belongs to is:\nairbnb_data[airbnb_data.id == \"272908.jpg\"].description.tolist()\n</pre> # And the description for the property that that picture belongs to is: airbnb_data[airbnb_data.id == \"272908.jpg\"].description.tolist() Out[4]: <pre>[\"Bright, sunny beautiful room that will give you the perfect base to explore all of London. Come and explore one of London's best neighbourhoods - Herne Hill! As mentioned in (Website hidden by Airbnb)   (Website hidden by Airbnb)  WiFi availability with a fully stocked and clean uplifting home. Lovely sunny, airy and big double bedroom on a leafy south-London street.    Note: This room comes with a reserved Off-Street parking spot! The room is on the first floor and boasts an enormous Super King bed, gorgeous wooden floors, tall ceilings and large windows which let in the sunshine almost all day. (Yoga May or meditation cushion available on request) The flat is bright and airy and big! So lots of space for all.  Location wise you are only 10 minutes walk to either Herne Hill or West Dulwich stations, both of which will take you to Victoria and the city within minutes. You can also hop on a bus right outside the flat that will take you to Brixton tube station within 8 minutes where you \"]</pre> <p>Ok, so we have tabular data where one column is <code>description</code> and another <code>id</code>, points towards the images stored in disk. Now, remember the following, because this will appear a few times in the notebook: our \"reference dataset\" is the tabular data.</p> <p>Therefore, since I want to illustrate a \"semi-realistic\" case, if we need to split the data into training, validation and test datasets, these datasets needs to be separetely stored in disk. In my case I have done this and in the <code>tmp_data/airbnb</code> dir I have the following:</p> <pre>../tmp_data/airbnb\n\u251c\u2500\u2500 airbnb_sample.csv\n\u251c\u2500\u2500 airbnb_sample_eval.csv\n\u251c\u2500\u2500 airbnb_sample_test.csv\n\u251c\u2500\u2500 airbnb_sample_train.csv\n\u2514\u2500\u2500 property_picture\n</pre> <p>Where <code>airbnb_sample.csv</code> is the full sample (1001 rows) and the <code>train</code>, <code>eval</code> and <code>test</code> set is the corresponding splits. In a realistic example, the full sample would be the 'gigantic' dataset and the rest the corresponding splits. One has to do this 'offline', prior to start the coding.</p> <p>Also, one thing that one needs to know is the number of total observations/rows, as well as the splits. In our case the train size is 800, and the eval and test sizes are 100 and 101 respectively.</p> <p>With all that info, let's star</p> In\u00a0[5]: Copied! <pre># path to the tabular data and the splits\ndata_path = \"../tmp_data/airbnb/\"\ntrain_fname = \"airbnb_sample_train.csv\"\neval_fname = \"airbnb_sample_eval.csv\"\ntest_fname = \"airbnb_sample_test.csv\"\n\n# split sizes\ntrain_size = 800\neval_size = 100\ntest_size = 101\n\n# number of chunks for the Chunk Preprocessors\nchunksize = 100\nn_chunks = int(np.ceil(train_size / chunksize))\n\n# path to the image dataset and name of the image col\nimg_path = \"../tmp_data/airbnb/property_picture/\"\nimg_col = \"id\"\n\n# name of the text col\ntext_col = \"description\"\n\n# mane of the target\ntarget_col = \"yield\"\n\n# definition of the categorical and continuous cols for the TabPreprocessor\ncat_embed_cols = [\n    \"host_listings_count\",\n    \"neighbourhood_cleansed\",\n    \"is_location_exact\",\n    \"property_type\",\n    \"room_type\",\n    \"accommodates\",\n    \"bathrooms\",\n    \"bedrooms\",\n    \"beds\",\n    \"guests_included\",\n    \"minimum_nights\",\n    \"instant_bookable\",\n    \"cancellation_policy\",\n    \"has_house_rules\",\n    \"host_gender\",\n    \"accommodates_catg\",\n    \"guests_included_catg\",\n    \"minimum_nights_catg\",\n    \"host_listings_count_catg\",\n    \"bathrooms_catg\",\n    \"bedrooms_catg\",\n    \"beds_catg\",\n    \"security_deposit\",\n    \"extra_people\",\n]\ncont_cols = [\"latitude\", \"longitude\"]\n</pre> # path to the tabular data and the splits data_path = \"../tmp_data/airbnb/\" train_fname = \"airbnb_sample_train.csv\" eval_fname = \"airbnb_sample_eval.csv\" test_fname = \"airbnb_sample_test.csv\"  # split sizes train_size = 800 eval_size = 100 test_size = 101  # number of chunks for the Chunk Preprocessors chunksize = 100 n_chunks = int(np.ceil(train_size / chunksize))  # path to the image dataset and name of the image col img_path = \"../tmp_data/airbnb/property_picture/\" img_col = \"id\"  # name of the text col text_col = \"description\"  # mane of the target target_col = \"yield\"  # definition of the categorical and continuous cols for the TabPreprocessor cat_embed_cols = [     \"host_listings_count\",     \"neighbourhood_cleansed\",     \"is_location_exact\",     \"property_type\",     \"room_type\",     \"accommodates\",     \"bathrooms\",     \"bedrooms\",     \"beds\",     \"guests_included\",     \"minimum_nights\",     \"instant_bookable\",     \"cancellation_policy\",     \"has_house_rules\",     \"host_gender\",     \"accommodates_catg\",     \"guests_included_catg\",     \"minimum_nights_catg\",     \"host_listings_count_catg\",     \"bathrooms_catg\",     \"bedrooms_catg\",     \"beds_catg\",     \"security_deposit\",     \"extra_people\", ] cont_cols = [\"latitude\", \"longitude\"] In\u00a0[6]: Copied! <pre>tab_preprocessor = TabPreprocessor(\n    embed_cols=cat_embed_cols,\n    continuous_cols=cont_cols,\n    default_embed_dim=8,\n    verbose=0,\n)\n\ntext_preprocessor = TextPreprocessor(\n    text_col=text_col,\n    n_cpus=1,\n)\n\nimg_preprocessor = ImagePreprocessor(\n    img_col=img_col,\n    img_path=img_path,\n)\n</pre> tab_preprocessor = TabPreprocessor(     embed_cols=cat_embed_cols,     continuous_cols=cont_cols,     default_embed_dim=8,     verbose=0, )  text_preprocessor = TextPreprocessor(     text_col=text_col,     n_cpus=1, )  img_preprocessor = ImagePreprocessor(     img_col=img_col,     img_path=img_path, ) In\u00a0[7]: Copied! <pre>tab_preprocessor.fit(airbnb_data)\ntext_preprocessor.fit(airbnb_data)\nimg_preprocessor.fit(airbnb_data)\n</pre> tab_preprocessor.fit(airbnb_data) text_preprocessor.fit(airbnb_data) img_preprocessor.fit(airbnb_data) <pre>The vocabulary contains 2192 tokens\n</pre> Out[7]: <pre>ImagePreprocessor(img_col=id, img_path=../tmp_data/airbnb/property_picture/, width=224, height=224, verbose=1)</pre> In\u00a0[8]: Copied! <pre>chunk_tab_preprocessor = ChunkTabPreprocessor(\n    embed_cols=cat_embed_cols,\n    continuous_cols=cont_cols,\n    n_chunks=n_chunks,\n    default_embed_dim=8,\n    verbose=0,\n)\n\nchunk_text_preprocessor = ChunkTextPreprocessor(\n    n_chunks=n_chunks,\n    text_col=text_col,\n    n_cpus=1,\n    verbose=0,\n)\n\nfor i, chunk in enumerate(\n    pd.read_csv(\"/\".join([data_path, train_fname]), chunksize=chunksize)\n):\n    print(f\"chunk in loop: {i + 1}\")\n    chunk_tab_preprocessor.fit(chunk)\n    chunk_text_preprocessor.fit(chunk)\n</pre> chunk_tab_preprocessor = ChunkTabPreprocessor(     embed_cols=cat_embed_cols,     continuous_cols=cont_cols,     n_chunks=n_chunks,     default_embed_dim=8,     verbose=0, )  chunk_text_preprocessor = ChunkTextPreprocessor(     n_chunks=n_chunks,     text_col=text_col,     n_cpus=1,     verbose=0, )  for i, chunk in enumerate(     pd.read_csv(\"/\".join([data_path, train_fname]), chunksize=chunksize) ):     print(f\"chunk in loop: {i + 1}\")     chunk_tab_preprocessor.fit(chunk)     chunk_text_preprocessor.fit(chunk) <pre>chunk in loop: 1\nchunk in loop: 2\nchunk in loop: 3\nchunk in loop: 4\nchunk in loop: 5\nchunk in loop: 6\nchunk in loop: 7\nchunk in loop: 8\n</pre> In\u00a0[9]: Copied! <pre>train_tab_folder = TabFromFolder(\n    fname=train_fname,\n    directory=data_path,\n    target_col=target_col,\n    preprocessor=tab_preprocessor,\n    text_col=text_col,\n    img_col=img_col,\n)\n\n# Note how we can use the `train_tab_folder` as reference so we don't have to\n# define all parameters again\neval_tab_folder = TabFromFolder(fname=eval_fname, reference=train_tab_folder)\n\n# Note that for the test set we can ignore the target as no metric will be\n# computed by the `predict` method\ntest_tab_folder = TabFromFolder(\n    fname=test_fname, reference=train_tab_folder, ignore_target=True\n)\n</pre> train_tab_folder = TabFromFolder(     fname=train_fname,     directory=data_path,     target_col=target_col,     preprocessor=tab_preprocessor,     text_col=text_col,     img_col=img_col, )  # Note how we can use the `train_tab_folder` as reference so we don't have to # define all parameters again eval_tab_folder = TabFromFolder(fname=eval_fname, reference=train_tab_folder)  # Note that for the test set we can ignore the target as no metric will be # computed by the `predict` method test_tab_folder = TabFromFolder(     fname=test_fname, reference=train_tab_folder, ignore_target=True ) In\u00a0[10]: Copied! <pre># for the text and image datasets we do not need to specify eval or test loaders\ntext_folder = TextFromFolder(preprocessor=text_preprocessor)\nimg_folder = ImageFromFolder(preprocessor=img_preprocessor)\n</pre> # for the text and image datasets we do not need to specify eval or test loaders text_folder = TextFromFolder(preprocessor=text_preprocessor) img_folder = ImageFromFolder(preprocessor=img_preprocessor) In\u00a0[11]: Copied! <pre>train_dataset_folder = WideDeepDatasetFromFolder(\n    n_samples=train_size,\n    tab_from_folder=train_tab_folder,\n    text_from_folder=text_folder,\n    img_from_folder=img_folder,\n)\n\n# Note that the eval and test loaders only need their corresponding\n# `TabFromFolder` classes. The rest of the parameters can be defined\n# via a `reference` `TabFromFolder` class\neval_dataset_folder = WideDeepDatasetFromFolder(\n    n_samples=eval_size,\n    tab_from_folder=eval_tab_folder,\n    reference=train_dataset_folder,\n)\n\ntest_dataset_folder = WideDeepDatasetFromFolder(\n    n_samples=test_size,\n    tab_from_folder=test_tab_folder,\n    reference=train_dataset_folder,\n)\n</pre> train_dataset_folder = WideDeepDatasetFromFolder(     n_samples=train_size,     tab_from_folder=train_tab_folder,     text_from_folder=text_folder,     img_from_folder=img_folder, )  # Note that the eval and test loaders only need their corresponding # `TabFromFolder` classes. The rest of the parameters can be defined # via a `reference` `TabFromFolder` class eval_dataset_folder = WideDeepDatasetFromFolder(     n_samples=eval_size,     tab_from_folder=eval_tab_folder,     reference=train_dataset_folder, )  test_dataset_folder = WideDeepDatasetFromFolder(     n_samples=test_size,     tab_from_folder=test_tab_folder,     reference=train_dataset_folder, ) In\u00a0[12]: Copied! <pre>train_loader = DataLoader(train_dataset_folder, batch_size=16, num_workers=1)\neval_loader = DataLoader(eval_dataset_folder, batch_size=16, num_workers=1)\ntest_loader = DataLoader(test_dataset_folder, batch_size=16, num_workers=1)\n</pre> train_loader = DataLoader(train_dataset_folder, batch_size=16, num_workers=1) eval_loader = DataLoader(eval_dataset_folder, batch_size=16, num_workers=1) test_loader = DataLoader(test_dataset_folder, batch_size=16, num_workers=1) <p>And from here on is business as usual:</p> In\u00a0[13]: Copied! <pre># for example\nbasic_rnn = BasicRNN(\n    vocab_size=len(text_preprocessor.vocab.itos),\n    embed_dim=32,\n    hidden_dim=64,\n    n_layers=2,\n)\n\ndeepimage = Vision()\n\ndeepdense = TabMlp(\n    mlp_hidden_dims=[32, 16],\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=cont_cols,\n)\n\nmodel = WideDeep(\n    deeptabular=deepdense,\n    deeptext=basic_rnn,\n    deepimage=deepimage,\n)\n\nmodel\n</pre> # for example basic_rnn = BasicRNN(     vocab_size=len(text_preprocessor.vocab.itos),     embed_dim=32,     hidden_dim=64,     n_layers=2, )  deepimage = Vision()  deepdense = TabMlp(     mlp_hidden_dims=[32, 16],     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     continuous_cols=cont_cols, )  model = WideDeep(     deeptabular=deepdense,     deeptext=basic_rnn,     deepimage=deepimage, )  model Out[13]: <pre>WideDeep(\n  (deeptabular): Sequential(\n    (0): TabMlp(\n      (cat_embed): DiffSizeCatEmbeddings(\n        (embed_layers): ModuleDict(\n          (emb_layer_host_listings_count): Embedding(28, 10, padding_idx=0)\n          (emb_layer_neighbourhood_cleansed): Embedding(33, 11, padding_idx=0)\n          (emb_layer_is_location_exact): Embedding(3, 2, padding_idx=0)\n          (emb_layer_property_type): Embedding(4, 3, padding_idx=0)\n          (emb_layer_room_type): Embedding(4, 3, padding_idx=0)\n          (emb_layer_accommodates): Embedding(14, 7, padding_idx=0)\n          (emb_layer_bathrooms): Embedding(11, 6, padding_idx=0)\n          (emb_layer_bedrooms): Embedding(7, 4, padding_idx=0)\n          (emb_layer_beds): Embedding(11, 6, padding_idx=0)\n          (emb_layer_guests_included): Embedding(11, 6, padding_idx=0)\n          (emb_layer_minimum_nights): Embedding(25, 9, padding_idx=0)\n          (emb_layer_instant_bookable): Embedding(3, 2, padding_idx=0)\n          (emb_layer_cancellation_policy): Embedding(6, 4, padding_idx=0)\n          (emb_layer_has_house_rules): Embedding(3, 2, padding_idx=0)\n          (emb_layer_host_gender): Embedding(4, 3, padding_idx=0)\n          (emb_layer_accommodates_catg): Embedding(4, 3, padding_idx=0)\n          (emb_layer_guests_included_catg): Embedding(4, 3, padding_idx=0)\n          (emb_layer_minimum_nights_catg): Embedding(4, 3, padding_idx=0)\n          (emb_layer_host_listings_count_catg): Embedding(5, 3, padding_idx=0)\n          (emb_layer_bathrooms_catg): Embedding(4, 3, padding_idx=0)\n          (emb_layer_bedrooms_catg): Embedding(5, 3, padding_idx=0)\n          (emb_layer_beds_catg): Embedding(5, 3, padding_idx=0)\n          (emb_layer_security_deposit): Embedding(53, 15, padding_idx=0)\n          (emb_layer_extra_people): Embedding(39, 12, padding_idx=0)\n        )\n        (embedding_dropout): Dropout(p=0.0, inplace=False)\n      )\n      (cont_norm): Identity()\n      (encoder): MLP(\n        (mlp): Sequential(\n          (dense_layer_0): Sequential(\n            (0): Linear(in_features=128, out_features=32, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_1): Sequential(\n            (0): Linear(in_features=32, out_features=16, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=16, out_features=1, bias=True)\n  )\n  (deeptext): Sequential(\n    (0): BasicRNN(\n      (word_embed): Embedding(2192, 32, padding_idx=1)\n      (rnn): LSTM(32, 64, num_layers=2, batch_first=True, dropout=0.1)\n      (rnn_mlp): Identity()\n    )\n    (1): Linear(in_features=64, out_features=1, bias=True)\n  )\n  (deepimage): Sequential(\n    (0): Vision(\n      (features): Sequential(\n        (conv_layer_0): Sequential(\n          (0): Conv2d(3, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False)\n          (1): BatchNorm2d(64, eps=1e-05, momentum=0.01, affine=True, track_running_stats=True)\n          (2): LeakyReLU(negative_slope=0.1, inplace=True)\n          (maxpool): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n        )\n        (conv_layer_1): Sequential(\n          (0): Conv2d(64, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n          (1): BatchNorm2d(128, eps=1e-05, momentum=0.01, affine=True, track_running_stats=True)\n          (2): LeakyReLU(negative_slope=0.1, inplace=True)\n        )\n        (conv_layer_2): Sequential(\n          (0): Conv2d(128, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n          (1): BatchNorm2d(256, eps=1e-05, momentum=0.01, affine=True, track_running_stats=True)\n          (2): LeakyReLU(negative_slope=0.1, inplace=True)\n        )\n        (conv_layer_3): Sequential(\n          (0): Conv2d(256, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n          (1): BatchNorm2d(512, eps=1e-05, momentum=0.01, affine=True, track_running_stats=True)\n          (2): LeakyReLU(negative_slope=0.1, inplace=True)\n          (adaptiveavgpool): AdaptiveAvgPool2d(output_size=(1, 1))\n        )\n      )\n    )\n    (1): Linear(in_features=512, out_features=1, bias=True)\n  )\n)</pre> In\u00a0[14]: Copied! <pre>trainer = TrainerFromFolder(\n    model,\n    objective=\"regression\",\n)\n\ntrainer.fit(\n    train_loader=train_loader,\n    eval_loader=eval_loader,\n)\n</pre> trainer = TrainerFromFolder(     model,     objective=\"regression\", )  trainer.fit(     train_loader=train_loader,     eval_loader=eval_loader, ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 50/50 [03:41&lt;00:00,  4.42s/it, loss=1.64e+4]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 7/7 [00:23&lt;00:00,  3.30s/it, loss=6.27e+3]\n</pre> In\u00a0[15]: Copied! <pre>preds = trainer.predict(test_loader=test_loader)\n</pre> preds = trainer.predict(test_loader=test_loader) <pre>predict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 7/7 [00:22&lt;00:00,  3.26s/it]\n</pre> <p>Note that in the case of predict you could also choose to do this</p> In\u00a0[16]: Copied! <pre>df_test = pd.read_csv(\"/\".join([data_path, test_fname]))\n</pre> df_test = pd.read_csv(\"/\".join([data_path, test_fname])) In\u00a0[17]: Copied! <pre># if the images for the test set fit in memory\nX_tab_test = chunk_tab_preprocessor.transform(df_test)\nX_text_test = chunk_text_preprocessor.transform(df_test)\nX_img_test = img_preprocessor.transform(df_test)\n</pre> # if the images for the test set fit in memory X_tab_test = chunk_tab_preprocessor.transform(df_test) X_text_test = chunk_text_preprocessor.transform(df_test) X_img_test = img_preprocessor.transform(df_test) <pre>Reading Images from ../tmp_data/airbnb/property_picture/\nResizing\n</pre> <pre>100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 101/101 [00:00&lt;00:00, 708.23it/s]</pre> <pre>Computing normalisation metrics\n</pre> <pre>\n</pre> In\u00a0[18]: Copied! <pre>preds = trainer.predict(\n    X_tab=X_tab_test, X_text=X_text_test, X_img=X_img_test, batch_size=32\n)\n</pre> preds = trainer.predict(     X_tab=X_tab_test, X_text=X_text_test, X_img=X_img_test, batch_size=32 ) <pre>predict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 4/4 [00:03&lt;00:00,  1.14it/s]\n</pre>"},{"location":"examples/19_load_from_folder_functionality.html#scenario","title":"Scenario\u00b6","text":"<p>We have a tabular dataset combined with images and text and either some, or all these datasets do not fit in memory. Note that the tabular dataset MUST ALWAYS be present as it is considered the rerefence. This is, if we have an image dataset, the tabular dataset must contain a column that points to the image file names as stored in disk. Similarly, if we have a text dataset, then the tabular dataset must contain a column with the texts themselves or a the file names of the text files as stored in disk.</p> <p>If you only have text and/or images and not a tabular component, I would suggest using other libraries (such as hugginface probably).</p> <p>Within this scenario, they are two possible scenarios that we will cover here:</p> <ol> <li><p>The tabular data itsel fits in memory and is only the images that do not: in this case you could use the 'standard' <code>Preprocessors</code> in the library and off you go, move directly to the <code>[...]FromFolder</code> functionalities</p> </li> <li><p>The tabular data is also very large and does not fit in memory, so we have to process it in chuncks. For this second case I have created the so called <code>Chunk[...]Preprocessor</code> (Wide, Tab and Text).</p> </li> </ol> <p>Note that at the moment ONLY csv format is allowed for the tabular file. More formats will be supported in the future.</p> <p>Let's see a complete example to illustrate how each of these cases would be addressed with the new functionalities in the library. For this example we will use a sample of the airbnb dataset</p> <p>The airbnb dataset, which you could get from here, is too big to be included in our datasets module (when including images). Therefore, what I did was, go there, download it, and use the download_images.py script to get the images and the airbnb_data_processing.py to process the data. I did this ages ago and I believe the format of the dataset might be different now. Nonetheless, I will show samples of the dataset as we go through so you can extrapolate the content of this notebook to your particular problem.</p> <p>In the future we will find better datasets\ud83d\ude42. Finally, note that here we are only using a small sample to illustrate the use, so PLEASE ignore the results, just focus on usage.</p>"},{"location":"examples/19_load_from_folder_functionality.html#setting-variables-and-constants","title":"Setting variables and constants\u00b6","text":""},{"location":"examples/19_load_from_folder_functionality.html#step-1-the-preprocessors","title":"Step 1: the preprocessors\u00b6","text":""},{"location":"examples/19_load_from_folder_functionality.html#scenario-1-only-the-images-do-not-fit-in-disk","title":"Scenario 1: only the images do not fit in disk\u00b6","text":"<p>In this case we can prepare the data in the 'standard' way</p>"},{"location":"examples/19_load_from_folder_functionality.html#scenario-2-the-tabular-data-is-also-huge","title":"Scenario 2: the tabular data is also huge\u00b6","text":"<p>Then we need to prepare it in chunks. Note that, unfortunately, the tabular and text preprocessors need to see the whole dataset once. This is because to process tabular or text data we need to encode values. For those encodings to be consistent they need to have seen the whole dataset. Alternatively, one could code a solution with some streaming encoder for both datasets. However, such implementation is not trivial for this library (and in general). I also don't think that having to see the whole data once is such a big limitation. Let's see how is done.</p> <p>Note that I have not mentioned the image dataset. This is because the processing of the image dataset does not require any form of encoding and in consequence can be done 'on the fly'. Therefore, no <code>ChunkImgPreprocessor</code> processor is needed.</p>"},{"location":"examples/19_load_from_folder_functionality.html#step-2-the-fromfolder-classes","title":"Step 2: the <code>[...]FromFolder</code> classes\u00b6","text":"<p>Once we have the preprocessors, we need to instantiate the classes that will enable us to load the data from their respective folders. From now on I am going to proceed with the <code>chunk_tab_preprocessor</code>, <code>chunk_text_preprocessor</code> and <code>img_preprocessor</code>, but the code would be identical if instead of the first two preprocessors we decided to use the <code>tab_preprocessor</code> and <code>text_preprocessor</code>.</p> <p>Once more, our reference datasets are the tabular datasets, which we have splitted in train, eval and test prior to start the coding. Therefore, we will eventually need a loader for each split</p>"},{"location":"examples/19_load_from_folder_functionality.html#step-3-pytorch-datasets-and-dataloaders","title":"Step 3: pytorch datasets and dataloaders\u00b6","text":"<p>From here in advance, is all very 'standard' if you are familiar with pytorch. One needs to define a class that inherits from the <code>Dataset</code> class in pytorch. Then this will be passed to a <code>DataLoader</code> class and we are ready to train. Our <code>Dataset</code> child class is <code>WideDeepDatasetFromFolder</code>. This class will use the tabular dataset and the corresponding text and image columns to load the adequate data in the batches</p> <p>Let's do it</p>"},{"location":"examples/19_load_from_folder_functionality.html#step-4-define-the-model","title":"Step 4: define the model\u00b6","text":""},{"location":"examples/19_load_from_folder_functionality.html#step-5-fit-and-predict","title":"Step 5: fit and predict\u00b6","text":""},{"location":"examples/20_Using_huggingface_within_widedeep.html","title":"20 Using huggingface within widedeep","text":"<p>In this notebook we will show how to use Hugginface's tokenizers and models as they are integrated within the library. In notebook number 17 you can find examples on how to code your own, custom, Hugginface (hereafter HF) model and use it in combination of any other model in the library</p> In\u00a0[1]: Copied! <pre>import numpy as np\nimport pandas as pd\nfrom sklearn.metrics import f1_score, accuracy_score\nfrom sklearn.model_selection import train_test_split\n\nfrom pytorch_widedeep import Trainer\nfrom pytorch_widedeep.models import HFModel, WideDeep\nfrom pytorch_widedeep.metrics import F1Score, Accuracy\nfrom pytorch_widedeep.datasets import load_womens_ecommerce\nfrom pytorch_widedeep.preprocessing import HFPreprocessor\n</pre> import numpy as np import pandas as pd from sklearn.metrics import f1_score, accuracy_score from sklearn.model_selection import train_test_split  from pytorch_widedeep import Trainer from pytorch_widedeep.models import HFModel, WideDeep from pytorch_widedeep.metrics import F1Score, Accuracy from pytorch_widedeep.datasets import load_womens_ecommerce from pytorch_widedeep.preprocessing import HFPreprocessor <pre>/Users/javierrodriguezzaurin/.pyenv/versions/3.10.13/envs/widedeep310/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n  from .autonotebook import tqdm as notebook_tqdm\n</pre> In\u00a0[2]: Copied! <pre>df: pd.DataFrame = load_womens_ecommerce(as_frame=True)  # type: ignore\n</pre> df: pd.DataFrame = load_womens_ecommerce(as_frame=True)  # type: ignore In\u00a0[3]: Copied! <pre>df.shape\n</pre> df.shape Out[3]: <pre>(23486, 10)</pre> In\u00a0[4]: Copied! <pre>df.sample(3)\n</pre> df.sample(3) Out[4]: Clothing ID Age Title Review Text Rating Recommended IND Positive Feedback Count Division Name Department Name Class Name 7004 862 43 Cute and feminine Loved this sweater wrap and bought it in both ... 5 1 2 General Tops Knits 12508 975 66 Love it The linen fabric is elegantly thin feels and l... 5 1 3 General Jackets Jackets 10288 950 41 Perfect for fall This sweater is just as pictured. the fit is t... 5 1 0 General Tops Sweaters In\u00a0[5]: Copied! <pre># Let's do some mild preprocessing\ndf.columns = [c.replace(\" \", \"_\").lower() for c in df.columns]\n\n# classes from [0,num_class)\ndf[\"rating\"] = (df[\"rating\"] - 1).astype(\"int64\")\n\n# group reviews with 1 and 2 scores into one class\ndf.loc[df.rating == 0, \"rating\"] = 1\n\n# and back again to [0,num_class)\ndf[\"rating\"] = (df[\"rating\"] - 1).astype(\"int64\")\n</pre> # Let's do some mild preprocessing df.columns = [c.replace(\" \", \"_\").lower() for c in df.columns]  # classes from [0,num_class) df[\"rating\"] = (df[\"rating\"] - 1).astype(\"int64\")  # group reviews with 1 and 2 scores into one class df.loc[df.rating == 0, \"rating\"] = 1  # and back again to [0,num_class) df[\"rating\"] = (df[\"rating\"] - 1).astype(\"int64\") In\u00a0[6]: Copied! <pre># drop short reviews\ndf = df[~df.review_text.isna()]\ndf[\"review_length\"] = df.review_text.apply(lambda x: len(x.split(\" \")))\ndf = df[df.review_length &gt;= 5]\ndf = df.drop(\"review_length\", axis=1).reset_index(drop=True)\n</pre> # drop short reviews df = df[~df.review_text.isna()] df[\"review_length\"] = df.review_text.apply(lambda x: len(x.split(\" \"))) df = df[df.review_length &gt;= 5] df = df.drop(\"review_length\", axis=1).reset_index(drop=True) In\u00a0[7]: Copied! <pre>df.shape\n</pre> df.shape Out[7]: <pre>(22608, 10)</pre> In\u00a0[8]: Copied! <pre># if you run this on a CPU, you might want to subsample the dataset. With that in mind I am simply going to stratify-sample to the minimum category occurrence and then sample at random\n# If you run this on a GPU you can comment out the following two cells\ndf.rating.value_counts()\n</pre> # if you run this on a CPU, you might want to subsample the dataset. With that in mind I am simply going to stratify-sample to the minimum category occurrence and then sample at random # If you run this on a GPU you can comment out the following two cells df.rating.value_counts() Out[8]: <pre>rating\n3    12515\n2     4904\n1     2820\n0     2369\nName: count, dtype: int64</pre> In\u00a0[9]: Copied! <pre>df = (\n    df.groupby(\"rating\", group_keys=False)\n    .apply(lambda x: x.sample(min(len(x), 2369)))\n    .sample(1000)\n)\n</pre> df = (     df.groupby(\"rating\", group_keys=False)     .apply(lambda x: x.sample(min(len(x), 2369)))     .sample(1000) ) <pre>/var/folders/_2/lrjn1qn54c758tdtktr1bvkc0000gn/T/ipykernel_5886/895673206.py:3: DeprecationWarning: DataFrameGroupBy.apply operated on the grouping columns. This behavior is deprecated, and in a future version of pandas the grouping columns will be excluded from the operation. Either pass `include_groups=False` to exclude the groupings or explicitly select the grouping columns after groupby to silence this warning.\n  .apply(lambda x: x.sample(min(len(x), 2369)))\n</pre> In\u00a0[10]: Copied! <pre>train, test = train_test_split(df, train_size=0.8, random_state=1, stratify=df.rating)\n\n# possible model names currently supported in the library\nmodel_names = [\n    \"distilbert-base-uncased\",\n    \"bert-base-uncased\",\n    \"FacebookAI/roberta-base\",\n    \"albert-base-v2\",\n    \"google/electra-base-discriminator\",\n]\n\n# Let's choose one. The syntax is the same for all the models\nmodel_name = \"distilbert-base-uncased\"\n</pre> train, test = train_test_split(df, train_size=0.8, random_state=1, stratify=df.rating)  # possible model names currently supported in the library model_names = [     \"distilbert-base-uncased\",     \"bert-base-uncased\",     \"FacebookAI/roberta-base\",     \"albert-base-v2\",     \"google/electra-base-discriminator\", ]  # Let's choose one. The syntax is the same for all the models model_name = \"distilbert-base-uncased\" <p>Now we can use the <code>HFPreprocessor</code> class. As most things in this library, the integration with HF has been coded aiming for a flexible use. With this in mind, there are two ways one can use a <code>HFPreprocessor</code> class.</p> <ol> <li>Passing a <code>text_col</code> and <code>encode_params</code> as the class is instantiated and then using the <code>fit</code> and <code>transform</code> as with any other preprocessor in the library</li> <li>Without passing  <code>text_col</code> and <code>encode_params</code> as the class is instantiated and using the <code>encode</code> method of the <code>HFPreprocessor</code> which is simply a wrapper around the encode method of HF's tokenizers</li> </ol> <p>Let's have a look</p> In\u00a0[11]: Copied! <pre>tokenizer1 = HFPreprocessor(\n    model_name=model_name,\n    text_col=\"review_text\",\n    num_workers=1,\n    encode_params={\n        \"max_length\": 90,\n        \"padding\": \"max_length\",\n        \"truncation\": True,\n        \"add_special_tokens\": True,\n    },\n)\n\nX_text_tr1 = tokenizer1.fit_transform(train)\nX_text_te1 = tokenizer1.transform(test)\n</pre> tokenizer1 = HFPreprocessor(     model_name=model_name,     text_col=\"review_text\",     num_workers=1,     encode_params={         \"max_length\": 90,         \"padding\": \"max_length\",         \"truncation\": True,         \"add_special_tokens\": True,     }, )  X_text_tr1 = tokenizer1.fit_transform(train) X_text_te1 = tokenizer1.transform(test) In\u00a0[12]: Copied! <pre>tokenizer2 = HFPreprocessor(\n    model_name=model_name,\n    num_workers=1,\n)\n\nX_text_tr2 = tokenizer2.encode(\n    train.review_text.tolist(),\n    max_length=90,\n    padding=\"max_length\",\n    truncation=True,\n    add_special_tokens=True,\n)\nX_text_te2 = tokenizer2.encode(\n    test.review_text.tolist(),\n    max_length=90,\n    padding=\"max_length\",\n    truncation=True,\n    add_special_tokens=True,\n)\n</pre> tokenizer2 = HFPreprocessor(     model_name=model_name,     num_workers=1, )  X_text_tr2 = tokenizer2.encode(     train.review_text.tolist(),     max_length=90,     padding=\"max_length\",     truncation=True,     add_special_tokens=True, ) X_text_te2 = tokenizer2.encode(     test.review_text.tolist(),     max_length=90,     padding=\"max_length\",     truncation=True,     add_special_tokens=True, ) In\u00a0[13]: Copied! <pre>all(X_text_tr1[0] == X_text_tr2[0])\n</pre> all(X_text_tr1[0] == X_text_tr2[0]) Out[13]: <pre>True</pre> In\u00a0[14]: Copied! <pre># Now we define a model which is as easy as:\n# Note that this will instantiation will lead to NO parameter trainable in the HF model.\n# If you want to fine-tune the HF model, you can set the trainable parameters via the 'trainable_parameters' argument.\n# Alternatively, you can use a head (MLP) via the 'head'-related arguments (see the docs for more details)\nhf_model = HFModel(model_name=model_name)\n</pre> # Now we define a model which is as easy as: # Note that this will instantiation will lead to NO parameter trainable in the HF model. # If you want to fine-tune the HF model, you can set the trainable parameters via the 'trainable_parameters' argument. # Alternatively, you can use a head (MLP) via the 'head'-related arguments (see the docs for more details) hf_model = HFModel(model_name=model_name) In\u00a0[15]: Copied! <pre># And from here on is the same as any other WideDeep model\nmodel = WideDeep(\n    deeptext=hf_model,\n    pred_dim=4,\n)\n\ntrainer = Trainer(\n    model,\n    objective=\"multiclass\",\n    metrics=[Accuracy(), F1Score(average=True)],\n)\n\ntrainer.fit(\n    X_text=X_text_tr2,\n    target=train.rating.values,\n    n_epochs=1,\n    batch_size=64,\n)\n# If you run this on a CPU and you sampled the data, the metrics will not be better than a random guess. Remember, this is just a demo\n</pre> # And from here on is the same as any other WideDeep model model = WideDeep(     deeptext=hf_model,     pred_dim=4, )  trainer = Trainer(     model,     objective=\"multiclass\",     metrics=[Accuracy(), F1Score(average=True)], )  trainer.fit(     X_text=X_text_tr2,     target=train.rating.values,     n_epochs=1,     batch_size=64, ) # If you run this on a CPU and you sampled the data, the metrics will not be better than a random guess. Remember, this is just a demo <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 13/13 [02:06&lt;00:00,  9.75s/it, loss=3.2, metrics={'acc': 0.235, 'f1': 0.2336}]  \n</pre> In\u00a0[17]: Copied! <pre>preds_text = trainer.predict_proba(X_text=X_text_te2)\npred_text_class = np.argmax(preds_text, 1)\n\nacc_text = accuracy_score(test.rating, pred_text_class)\nf1_text = f1_score(test.rating, pred_text_class, average=\"weighted\")\nprint(f\"Accuracy: {acc_text:.4f}\")\nprint(f\"F1: {f1_text:.4f}\")\n</pre> preds_text = trainer.predict_proba(X_text=X_text_te2) pred_text_class = np.argmax(preds_text, 1)  acc_text = accuracy_score(test.rating, pred_text_class) f1_text = f1_score(test.rating, pred_text_class, average=\"weighted\") print(f\"Accuracy: {acc_text:.4f}\") print(f\"F1: {f1_text:.4f}\") <pre>predict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 4/4 [00:05&lt;00:00,  1.43s/it]</pre> <pre>Accuracy: 0.2500\nF1: 0.1000\n</pre> <pre>\n</pre>"},{"location":"pytorch-widedeep/bayesian_models.html","title":"The <code>bayesian models</code> module","text":"<p>This module contains the two Bayesian Models available in this library, namely the bayesian version of the <code>Wide</code> and <code>TabMlp</code> models, referred as <code>BayesianWide</code> and <code>BayesianTabMlp</code>. These models are very useful in scenarios where getting a measure of uncertainty is important.</p> <p>The models in this module are based on the publication: Weight Uncertainty in Neural Networks.</p>"},{"location":"pytorch-widedeep/bayesian_models.html#pytorch_widedeep.bayesian_models.tabular.bayesian_linear.bayesian_wide.BayesianWide","title":"BayesianWide","text":"<pre><code>BayesianWide(\n    input_dim,\n    pred_dim=1,\n    prior_sigma_1=1.0,\n    prior_sigma_2=0.002,\n    prior_pi=0.8,\n    posterior_mu_init=0.0,\n    posterior_rho_init=-7.0,\n)\n</code></pre> <p>               Bases: <code>BaseBayesianModel</code></p> <p>Defines a <code>Wide</code> model. This is a linear model where the non-linearlities are captured via crossed-columns</p> <p>Parameters:</p> <ul> <li> <code>input_dim</code>               (<code>int</code>)           \u2013            <p>size of the Embedding layer. <code>input_dim</code> is the summation of all the individual values for all the features that go through the wide component. For example, if the wide component receives 2 features with 5 individual values each, <code>input_dim = 10</code></p> </li> <li> <code>pred_dim</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>size of the ouput tensor containing the predictions</p> </li> <li> <code>prior_sigma_1</code>               (<code>float</code>, default:                   <code>1.0</code> )           \u2013            <p>The prior weight distribution is a scaled mixture of two Gaussian densities:</p> \\[    \\begin{aligned}    P(\\mathbf{w}) = \\prod_{i=j} \\pi N (\\mathbf{w}_j | 0, \\sigma_{1}^{2}) + (1 - \\pi) N (\\mathbf{w}_j | 0, \\sigma_{2}^{2})    \\end{aligned} \\] <p><code>prior_sigma_1</code> is the prior of the sigma parameter for the first of the two Gaussians that will be mixed to produce the prior weight distribution.</p> </li> <li> <code>prior_sigma_2</code>               (<code>float</code>, default:                   <code>0.002</code> )           \u2013            <p>Prior of the sigma parameter for the second of the two Gaussian distributions that will be mixed to produce the prior weight distribution</p> </li> <li> <code>prior_pi</code>               (<code>float</code>, default:                   <code>0.8</code> )           \u2013            <p>Scaling factor that will be used to mix the Gaussians to produce the prior weight distribution</p> </li> <li> <code>posterior_mu_init</code>               (<code>float</code>, default:                   <code>0.0</code> )           \u2013            <p>The posterior sample of the weights is defined as:</p> \\[    \\begin{aligned}    \\mathbf{w} &amp;= \\mu + log(1 + exp(\\rho))    \\end{aligned} \\] <p>where:</p> \\[    \\begin{aligned}    \\mathcal{N}(x\\vert \\mu, \\sigma) &amp;= \\frac{1}{\\sqrt{2\\pi}\\sigma}e^{-\\frac{(x-\\mu)^2}{2\\sigma^2}}\\\\    \\log{\\mathcal{N}(x\\vert \\mu, \\sigma)} &amp;= -\\log{\\sqrt{2\\pi}} -\\log{\\sigma} -\\frac{(x-\\mu)^2}{2\\sigma^2}\\\\    \\end{aligned} \\] <p>\\(\\mu\\) is initialised using a normal distributtion with mean <code>posterior_mu_init</code> and std equal to 0.1.</p> </li> <li> <code>posterior_rho_init</code>               (<code>float</code>, default:                   <code>-7.0</code> )           \u2013            <p>As in the case of \\(\\mu\\), \\(\\rho\\) is initialised using a normal distributtion with mean <code>posterior_rho_init</code> and std equal to 0.1.</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>bayesian_wide_linear</code>               (<code>Module</code>)           \u2013            <p>the linear layer that comprises the wide branch of the model</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.bayesian_models import BayesianWide\n&gt;&gt;&gt; X = torch.empty(4, 4).random_(6)\n&gt;&gt;&gt; wide = BayesianWide(input_dim=int(X.max().item()), pred_dim=1)\n&gt;&gt;&gt; out = wide(X)\n</code></pre> Source code in <code>pytorch_widedeep/bayesian_models/tabular/bayesian_linear/bayesian_wide.py</code> <pre><code>def __init__(\n    self,\n    input_dim: int,\n    pred_dim: int = 1,\n    prior_sigma_1: float = 1.0,\n    prior_sigma_2: float = 0.002,\n    prior_pi: float = 0.8,\n    posterior_mu_init: float = 0.0,\n    posterior_rho_init: float = -7.0,\n):\n    super(BayesianWide, self).__init__()\n    #  Embeddings: val + 1 because 0 is reserved for padding/unseen cateogories.\n    self.bayesian_wide_linear = bnn.BayesianEmbedding(\n        n_embed=input_dim + 1,\n        embed_dim=pred_dim,\n        padding_idx=0,\n        prior_sigma_1=prior_sigma_1,\n        prior_sigma_2=prior_sigma_2,\n        prior_pi=prior_pi,\n        posterior_mu_init=posterior_mu_init,\n        posterior_rho_init=posterior_rho_init,\n    )\n    self.bias = nn.Parameter(torch.zeros(pred_dim))\n</code></pre>"},{"location":"pytorch-widedeep/bayesian_models.html#pytorch_widedeep.bayesian_models.tabular.bayesian_mlp.bayesian_tab_mlp.BayesianTabMlp","title":"BayesianTabMlp","text":"<pre><code>BayesianTabMlp(\n    column_idx,\n    *,\n    cat_embed_input=None,\n    cat_embed_activation=None,\n    continuous_cols=None,\n    embed_continuous=None,\n    cont_embed_dim=None,\n    cont_embed_dropout=None,\n    cont_embed_activation=None,\n    use_cont_bias=None,\n    cont_norm_layer=None,\n    mlp_hidden_dims=[200, 100],\n    mlp_activation=\"leaky_relu\",\n    prior_sigma_1=1,\n    prior_sigma_2=0.002,\n    prior_pi=0.8,\n    posterior_mu_init=0.0,\n    posterior_rho_init=-7.0,\n    pred_dim=1\n)\n</code></pre> <p>               Bases: <code>BaseBayesianModel</code></p> <p>Defines a <code>BayesianTabMlp</code> model.</p> <p>This class combines embedding representations of the categorical features with numerical (aka continuous) features, embedded or not. These are then passed through a series of probabilistic dense layers (i.e. a MLP).</p> <p>Parameters:</p> <ul> <li> <code>column_idx</code>               (<code>Dict[str, int]</code>)           \u2013            <p>Dict containing the index of the columns that will be passed through the <code>TabMlp</code> model. Required to slice the tensors. e.g. {'education': 0, 'relationship': 1, 'workclass': 2, ...}</p> </li> <li> <code>cat_embed_input</code>               (<code>Optional[List[Tuple[str, int, int]]]</code>, default:                   <code>None</code> )           \u2013            <p>List of Tuples with the column name, number of unique values and embedding dimension. e.g. [(education, 11, 32), ...]</p> </li> <li> <code>cat_embed_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the categorical embeddings, if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> </li> <li> <code>continuous_cols</code>               (<code>Optional[List[str]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the name of the numeric (aka continuous) columns</p> </li> <li> <code>cont_norm_layer</code>               (<code>Optional[Literal[batchnorm, layernorm]]</code>, default:                   <code>None</code> )           \u2013            <p>Type of normalization layer applied to the continuous features. Options are: 'layernorm', 'batchnorm' or None.</p> </li> <li> <code>embed_continuous</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating if the continuous columns will be embedded (i.e. passed each through a linear layer with or without activation)</p> </li> <li> <code>cont_embed_dim</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>Size of the continuous embeddings</p> </li> <li> <code>cont_embed_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Dropout for the continuous embeddings</p> </li> <li> <code>use_cont_bias</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating if bias will be used for the continuous embeddings</p> </li> <li> <code>cont_embed_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the continuous embeddings if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> </li> <li> <code>mlp_hidden_dims</code>               (<code>List[int]</code>, default:                   <code>[200, 100]</code> )           \u2013            <p>List with the number of neurons per dense layer in the mlp.</p> </li> <li> <code>mlp_activation</code>               (<code>str</code>, default:                   <code>'leaky_relu'</code> )           \u2013            <p>Activation function for the dense layers of the MLP. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> </li> <li> <code>prior_sigma_1</code>               (<code>float</code>, default:                   <code>1</code> )           \u2013            <p>The prior weight distribution is a scaled mixture of two Gaussian densities:</p> \\[    \\begin{aligned}    P(\\mathbf{w}) = \\prod_{i=j} \\pi N (\\mathbf{w}_j | 0, \\sigma_{1}^{2}) + (1 - \\pi) N (\\mathbf{w}_j | 0, \\sigma_{2}^{2})    \\end{aligned} \\] <p><code>prior_sigma_1</code> is the prior of the sigma parameter for the first of the two Gaussians that will be mixed to produce the prior weight distribution.</p> </li> <li> <code>prior_sigma_2</code>               (<code>float</code>, default:                   <code>0.002</code> )           \u2013            <p>Prior of the sigma parameter for the second of the two Gaussian distributions that will be mixed to produce the prior weight distribution for each Bayesian linear and embedding layer</p> </li> <li> <code>prior_pi</code>               (<code>float</code>, default:                   <code>0.8</code> )           \u2013            <p>Scaling factor that will be used to mix the Gaussians to produce the prior weight distribution ffor each Bayesian linear and embedding layer</p> </li> <li> <code>posterior_mu_init</code>               (<code>float</code>, default:                   <code>0.0</code> )           \u2013            <p>The posterior sample of the weights is defined as:</p> <p>$$    \\begin{aligned}    \\mathbf{w} &amp;= \\mu + log(1 + exp(\\rho))    \\end{aligned} $$ where:</p> \\[    \\begin{aligned}    \\mathcal{N}(x\\vert \\mu, \\sigma) &amp;= \\frac{1}{\\sqrt{2\\pi}\\sigma}e^{-\\frac{(x-\\mu)^2}{2\\sigma^2}}\\\\    \\log{\\mathcal{N}(x\\vert \\mu, \\sigma)} &amp;= -\\log{\\sqrt{2\\pi}} -\\log{\\sigma} -\\frac{(x-\\mu)^2}{2\\sigma^2}\\\\    \\end{aligned} \\] <p>\\(\\mu\\) is initialised using a normal distributtion with mean <code>posterior_mu_init</code> and std equal to 0.1.</p> </li> <li> <code>posterior_rho_init</code>               (<code>float</code>, default:                   <code>-7.0</code> )           \u2013            <p>As in the case of \\(\\mu\\), \\(\\rho\\) is initialised using a normal distributtion with mean <code>posterior_rho_init</code> and std equal to 0.1.</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>bayesian_cat_and_cont_embed</code>               (<code>Module</code>)           \u2013            <p>This is the module that processes the categorical and continuous columns</p> </li> <li> <code>bayesian_tab_mlp</code>               (<code>Sequential</code>)           \u2013            <p>mlp model that will receive the concatenation of the embeddings and the continuous columns</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.bayesian_models import BayesianTabMlp\n&gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n&gt;&gt;&gt; colnames = ['a', 'b', 'c', 'd', 'e']\n&gt;&gt;&gt; cat_embed_input = [(u,i,j) for u,i,j in zip(colnames[:4], [4]*4, [8]*4)]\n&gt;&gt;&gt; column_idx = {k:v for v,k in enumerate(colnames)}\n&gt;&gt;&gt; model = BayesianTabMlp(mlp_hidden_dims=[8,4], column_idx=column_idx, cat_embed_input=cat_embed_input,\n... continuous_cols = ['e'])\n&gt;&gt;&gt; out = model(X_tab)\n</code></pre> Source code in <code>pytorch_widedeep/bayesian_models/tabular/bayesian_mlp/bayesian_tab_mlp.py</code> <pre><code>def __init__(\n    self,\n    column_idx: Dict[str, int],\n    *,\n    cat_embed_input: Optional[List[Tuple[str, int, int]]] = None,\n    cat_embed_activation: Optional[str] = None,\n    continuous_cols: Optional[List[str]] = None,\n    embed_continuous: Optional[bool] = None,\n    cont_embed_dim: Optional[int] = None,\n    cont_embed_dropout: Optional[float] = None,\n    cont_embed_activation: Optional[str] = None,\n    use_cont_bias: Optional[bool] = None,\n    cont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\n    mlp_hidden_dims: List[int] = [200, 100],\n    mlp_activation: str = \"leaky_relu\",\n    prior_sigma_1: float = 1,\n    prior_sigma_2: float = 0.002,\n    prior_pi: float = 0.8,\n    posterior_mu_init: float = 0.0,\n    posterior_rho_init: float = -7.0,\n    pred_dim=1,  # Bayesian models will require their own trainer and need the output layer\n):\n    super(BayesianTabMlp, self).__init__()\n\n    self.column_idx = column_idx\n    self.cat_embed_input = cat_embed_input\n    self.cat_embed_activation = cat_embed_activation\n\n    self.continuous_cols = continuous_cols\n    self.cont_norm_layer = cont_norm_layer\n    self.embed_continuous = embed_continuous\n    self.cont_embed_dim = cont_embed_dim\n    self.cont_embed_dropout = cont_embed_dropout\n    self.use_cont_bias = use_cont_bias\n    self.cont_embed_activation = cont_embed_activation\n\n    self.mlp_hidden_dims = mlp_hidden_dims\n    self.mlp_activation = mlp_activation\n\n    self.prior_sigma_1 = prior_sigma_1\n    self.prior_sigma_2 = prior_sigma_2\n    self.prior_pi = prior_pi\n    self.posterior_mu_init = posterior_mu_init\n    self.posterior_rho_init = posterior_rho_init\n\n    self.pred_dim = pred_dim\n\n    allowed_activations = [\"relu\", \"leaky_relu\", \"tanh\", \"gelu\"]\n    if self.mlp_activation not in allowed_activations:\n        raise ValueError(\n            \"Currently, only the following activation functions are supported \"\n            \"for the Bayesian MLP's dense layers: {}. Got '{}' instead\".format(\n                \", \".join(allowed_activations),\n                self.mlp_activation,\n            )\n        )\n\n    # Categorical\n    if self.cat_embed_input is not None:\n        self.cat_embed = BayesianDiffSizeCatEmbeddings(\n            column_idx=self.column_idx,\n            embed_input=self.cat_embed_input,\n            prior_sigma_1=self.prior_sigma_1,\n            prior_sigma_2=self.prior_sigma_2,\n            prior_pi=self.prior_pi,\n            posterior_mu_init=self.posterior_mu_init,\n            posterior_rho_init=self.posterior_rho_init,\n            activation_fn=self.cat_embed_activation,\n        )\n        self.cat_out_dim = int(np.sum([embed[2] for embed in self.cat_embed_input]))\n    else:\n        self.cat_out_dim = 0\n\n    # Continuous\n    if self.continuous_cols is not None:\n        self.cont_idx = [self.column_idx[col] for col in self.continuous_cols]\n        if cont_norm_layer == \"layernorm\":\n            self.cont_norm: NormLayers = nn.LayerNorm(len(self.continuous_cols))\n        elif cont_norm_layer == \"batchnorm\":\n            self.cont_norm = nn.BatchNorm1d(len(self.continuous_cols))\n        else:\n            self.cont_norm = nn.Identity()\n        if self.embed_continuous:\n            assert self.cont_embed_dim is not None, (\n                \"If 'embed_continuous' is True, 'cont_embed_dim' must be \"\n                \"provided\"\n            )\n            self.cont_embed = BayesianContEmbeddings(\n                n_cont_cols=len(self.continuous_cols),\n                embed_dim=self.cont_embed_dim,\n                prior_sigma_1=self.prior_sigma_1,\n                prior_sigma_2=self.prior_sigma_2,\n                prior_pi=self.prior_pi,\n                posterior_mu_init=self.posterior_mu_init,\n                posterior_rho_init=self.posterior_rho_init,\n                use_bias=(\n                    False if self.use_cont_bias is None else self.use_cont_bias\n                ),\n                activation_fn=self.cont_embed_activation,\n            )\n            self.cont_out_dim = len(self.continuous_cols) * self.cont_embed_dim\n        else:\n            self.cont_out_dim = len(self.continuous_cols)\n    else:\n        self.cont_out_dim = 0\n\n    self.output_dim = self.cat_out_dim + self.cont_out_dim\n\n    mlp_hidden_dims = [self.output_dim] + mlp_hidden_dims + [pred_dim]\n    self.bayesian_tab_mlp = BayesianMLP(\n        mlp_hidden_dims,\n        mlp_activation,\n        True,  # use_bias\n        prior_sigma_1,\n        prior_sigma_2,\n        prior_pi,\n        posterior_mu_init,\n        posterior_rho_init,\n    )\n</code></pre>"},{"location":"pytorch-widedeep/bayesian_trainer.html","title":"Training Deep Learning Probabilistic Models","text":""},{"location":"pytorch-widedeep/bayesian_trainer.html#pytorch_widedeep.training.BayesianTrainer","title":"BayesianTrainer","text":"<pre><code>BayesianTrainer(\n    model,\n    objective,\n    custom_loss_function=None,\n    optimizer=None,\n    lr_scheduler=None,\n    callbacks=None,\n    metrics=None,\n    verbose=1,\n    seed=1,\n    **kwargs\n)\n</code></pre> <p>               Bases: <code>BaseBayesianTrainer</code></p> <p>Class to set the of attributes that will be used during the training process.</p> <p>Both the Bayesian models and the Trainer in this repo are based on the paper: Weight Uncertainty in Neural Networks.</p> <p>Parameters:</p> <ul> <li> <code>model</code>               (<code>BaseBayesianModel</code>)           \u2013            <p>An object of class <code>BaseBayesianModel</code>. See the <code>Model Components</code> section here in the docs.</p> </li> <li> <code>objective</code>               (<code>str</code>)           \u2013            <p>Defines the objective, loss or cost function. Param aliases: <code>loss_function</code>, <code>loss_fn</code>, <code>loss</code>, <code>cost_function</code>, <code>cost_fn</code>, <code>cost</code> Possible values are: 'binary', 'multiclass', 'regression'</p> </li> <li> <code>custom_loss_function</code>               (<code>Optional[Module]</code>, default:                   <code>None</code> )           \u2013            <p>If none of the loss functions available suits the user, it is possible to pass a custom loss function. See for example <code>pytorch_widedeep.losses.FocalLoss</code> for the required structure of the object or the Examples folder in the repo.</p> </li> <li> <code>optimizer</code>               (<code>Optional[Optimizer]</code>, default:                   <code>None</code> )           \u2013            <p>An instance of Pytorch's <code>Optimizer</code> object(e.g. <code>torch.optim.Adam ()</code>). if no optimizer is passed it will default to <code>AdamW</code>.</p> </li> <li> <code>lr_scheduler</code>               (<code>Optional[LRScheduler]</code>, default:                   <code>None</code> )           \u2013            <p>An instance of Pytorch's <code>LRScheduler</code> object (e.g <code>torch.optim.lr_scheduler.StepLR(opt, step_size=5)</code>).</p> </li> <li> <code>callbacks</code>               (<code>Optional[List[Callback]]</code>, default:                   <code>None</code> )           \u2013            <p>List with <code>Callback</code> objects. The three callbacks available in <code>pytorch-widedeep</code> are: <code>LRHistory</code>, <code>ModelCheckpoint</code> and <code>EarlyStopping</code>. This can also be a custom callback. See <code>pytorch_widedeep.callbacks.Callback</code> or the Examples folder in the repo.</p> </li> <li> <code>metrics</code>               (<code>Optional[Union[List[Metric], List[Metric]]]</code>, default:                   <code>None</code> )           \u2013            <ul> <li>List of objects of type <code>Metric</code>. Metrics available are:   <code>Accuracy</code>, <code>Precision</code>, <code>Recall</code>, <code>FBetaScore</code>,   <code>F1Score</code> and <code>R2Score</code>. This can also be a custom metric as   long as it is an object of type <code>Metric</code>. See   <code>pytorch_widedeep.metrics.Metric</code> or the Examples folder in the repo</li> <li>List of objects of type <code>torchmetrics.Metric</code>. This can be any   metric from torchmetrics library Examples   classification-metrics&gt;<code>_. It can also be a torchmetric custom metric as   long as it is an object of type</code>Metric<code>.   See</code>the instructions</li> </ul> </li> <li> <code>verbose</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>Setting it to 0 will print nothing during training.</p> </li> <li> <code>seed</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>Random seed to be used internally for train_test_split</p> </li> </ul> <p>Other Parameters:</p> <ul> <li> <code>**kwargs</code>           \u2013            <p>Other infrequently used arguments that can also be passed as kwargs are:</p> <ul> <li> <p>device: <code>str</code>     string indicating the device. One of 'cpu', 'gpu' or 'mps' if     run on a Mac with Apple silicon or AMD GPU(s)</p> </li> <li> <p>num_workers: <code>int</code>     number of workers to be used internally by the data loaders</p> </li> <li> <p>class_weight: <code>List[float]</code>     This is the <code>weight</code> or <code>pos_weight</code> parameter in     <code>CrossEntropyLoss</code> and <code>BCEWithLogitsLoss</code>, depending on whether</p> </li> <li> <p>reducelronplateau_criterion: <code>str</code>     This sets the criterion that will be used by the lr scheduler to     take a step: One of 'loss' or 'metric'. The ReduceLROnPlateau     learning rate is a bit particular.</p> </li> </ul> </li> </ul> <p>Attributes:</p> <ul> <li> <code>cyclic_lr</code>               (<code>bool</code>)           \u2013            <p>Attribute that indicates if  the lr_scheduler is cyclic_lr (i.e. <code>CyclicLR</code> or <code>OneCycleLR</code>). See <code>Pytorch schedulers &lt;https://pytorch.org/docs/stable/optim.html&gt;</code>_.</p> </li> </ul> Source code in <code>pytorch_widedeep/training/bayesian_trainer.py</code> <pre><code>@alias(  # noqa: C901\n    \"objective\",\n    [\"loss_function\", \"loss_fn\", \"loss\", \"cost_function\", \"cost_fn\", \"cost\"],\n)\ndef __init__(\n    self,\n    model: BaseBayesianModel,\n    objective: str,\n    custom_loss_function: Optional[Module] = None,\n    optimizer: Optional[Optimizer] = None,\n    lr_scheduler: Optional[LRScheduler] = None,\n    callbacks: Optional[List[Callback]] = None,\n    metrics: Optional[Union[List[Metric], List[TorchMetric]]] = None,\n    verbose: int = 1,\n    seed: int = 1,\n    **kwargs,\n):\n    super().__init__(\n        model=model,\n        objective=objective,\n        custom_loss_function=custom_loss_function,\n        optimizer=optimizer,\n        lr_scheduler=lr_scheduler,\n        callbacks=callbacks,\n        metrics=metrics,\n        verbose=verbose,\n        seed=seed,\n        **kwargs,\n    )\n</code></pre>"},{"location":"pytorch-widedeep/bayesian_trainer.html#pytorch_widedeep.training.BayesianTrainer.fit","title":"fit","text":"<pre><code>fit(\n    X_tab,\n    target,\n    X_tab_val=None,\n    target_val=None,\n    val_split=None,\n    n_epochs=1,\n    validation_freq=1,\n    batch_size=32,\n    n_train_samples=2,\n    n_val_samples=2,\n)\n</code></pre> <p>Fit method.</p> <p>Parameters:</p> <ul> <li> <code>X_tab</code>               (<code>ndarray</code>)           \u2013            <p>tabular dataset</p> </li> <li> <code>target</code>               (<code>ndarray</code>)           \u2013            <p>target values</p> </li> <li> <code>X_tab_val</code>               (<code>Optional[ndarray]</code>, default:                   <code>None</code> )           \u2013            <p>validation data</p> </li> <li> <code>target_val</code>               (<code>Optional[ndarray]</code>, default:                   <code>None</code> )           \u2013            <p>validation target values</p> </li> <li> <code>val_split</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>An alterative to passing the validation set is to use a train/val split fraction via <code>val_split</code></p> </li> <li> <code>n_epochs</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>number of epochs</p> </li> <li> <code>validation_freq</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>epochs validation frequency</p> </li> <li> <code>batch_size</code>               (<code>int</code>, default:                   <code>32</code> )           \u2013            <p>batch size</p> </li> <li> <code>n_train_samples</code>               (<code>int</code>, default:                   <code>2</code> )           \u2013            <p>number of samples to average over during the training process. See Weight Uncertainty in Neural Networks for details.</p> </li> <li> <code>n_val_samples</code>               (<code>int</code>, default:                   <code>2</code> )           \u2013            <p>number of samples to average over during the validation process. See Weight Uncertainty in Neural Networks for details.</p> </li> </ul> Source code in <code>pytorch_widedeep/training/bayesian_trainer.py</code> <pre><code>def fit(  # noqa: C901\n    self,\n    X_tab: np.ndarray,\n    target: np.ndarray,\n    X_tab_val: Optional[np.ndarray] = None,\n    target_val: Optional[np.ndarray] = None,\n    val_split: Optional[float] = None,\n    n_epochs: int = 1,\n    validation_freq: int = 1,\n    batch_size: int = 32,\n    n_train_samples: int = 2,\n    n_val_samples: int = 2,\n):\n    r\"\"\"Fit method.\n\n    Parameters\n    ----------\n    X_tab: np.ndarray,\n        tabular dataset\n    target: np.ndarray\n        target values\n    X_tab_val: np.ndarray, Optional, default = None\n        validation data\n    target_val: np.ndarray, Optional, default = None\n        validation target values\n    val_split: float, Optional. default=None\n        An alterative to passing the validation set is to use a train/val\n        split fraction via `val_split`\n    n_epochs: int, default=1\n        number of epochs\n    validation_freq: int, default=1\n        epochs validation frequency\n    batch_size: int, default=32\n        batch size\n    n_train_samples: int, default=2\n        number of samples to average over during the training process.\n        See [Weight Uncertainty in Neural Networks](https://arxiv.org/pdf/1505.05424.pdf) for details.\n    n_val_samples: int, default=2\n        number of samples to average over during the validation process.\n        See [Weight Uncertainty in Neural Networks](https://arxiv.org/pdf/1505.05424.pdf) for details.\n    \"\"\"\n\n    self.batch_size = batch_size\n\n    train_set, eval_set = tabular_train_val_split(\n        self.seed, self.objective, X_tab, target, X_tab_val, target_val, val_split\n    )\n    train_loader = DataLoader(\n        dataset=train_set, batch_size=batch_size, num_workers=self.num_workers\n    )\n    train_steps = len(train_loader)\n\n    if eval_set is not None:\n        eval_loader = DataLoader(\n            dataset=eval_set,\n            batch_size=batch_size,\n            num_workers=self.num_workers,\n            shuffle=False,\n        )\n        eval_steps = len(eval_loader)\n\n    self.callback_container.on_train_begin(\n        {\n            \"batch_size\": batch_size,\n            \"train_steps\": train_steps,\n            \"n_epochs\": n_epochs,\n        }\n    )\n    for epoch in range(n_epochs):\n        epoch_logs: Dict[str, float] = {}\n        self.callback_container.on_epoch_begin(epoch, logs=epoch_logs)\n\n        self.train_running_loss = 0.0\n        with trange(train_steps, disable=self.verbose != 1) as t:\n            for batch_idx, (X, y) in zip(t, train_loader):\n                t.set_description(\"epoch %i\" % (epoch + 1))\n                train_score, train_loss = self._train_step(\n                    X, y, n_train_samples, train_steps, batch_idx\n                )\n                print_loss_and_metric(t, train_loss, train_score)\n                self.callback_container.on_batch_end(batch=batch_idx)\n        epoch_logs = save_epoch_logs(epoch_logs, train_loss, train_score, \"train\")\n\n        on_epoch_end_metric = None\n        if eval_set is not None and epoch % validation_freq == (\n            validation_freq - 1\n        ):\n            self.callback_container.on_eval_begin()\n            self.valid_running_loss = 0.0\n            with trange(eval_steps, disable=self.verbose != 1) as v:\n                for i, (X, y) in zip(v, eval_loader):\n                    v.set_description(\"valid\")\n                    val_score, val_loss = self._eval_step(\n                        X, y, n_val_samples, train_steps, i\n                    )\n                    print_loss_and_metric(v, val_loss, val_score)\n            epoch_logs = save_epoch_logs(epoch_logs, val_loss, val_score, \"val\")\n\n            if self.reducelronplateau:\n                if self.reducelronplateau_criterion == \"loss\":\n                    on_epoch_end_metric = val_loss\n                else:\n                    on_epoch_end_metric = val_score[\n                        self.reducelronplateau_criterion\n                    ]\n\n        self.callback_container.on_epoch_end(epoch, epoch_logs, on_epoch_end_metric)\n\n        if self.early_stop:\n            self.callback_container.on_train_end(epoch_logs)\n            break\n\n    self.callback_container.on_train_end(epoch_logs)\n    self._restore_best_weights()\n    self.model.train()\n</code></pre>"},{"location":"pytorch-widedeep/bayesian_trainer.html#pytorch_widedeep.training.BayesianTrainer.predict","title":"predict","text":"<pre><code>predict(\n    X_tab, n_samples=5, return_samples=False, batch_size=256\n)\n</code></pre> <p>Returns the predictions</p> <p>Parameters:</p> <ul> <li> <code>X_tab</code>               (<code>ndarray</code>)           \u2013            <p>tabular dataset</p> </li> <li> <code>n_samples</code>               (<code>int</code>, default:                   <code>5</code> )           \u2013            <p>number of samples that will be either returned or averaged to produce an overal prediction</p> </li> <li> <code>return_samples</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether the n samples will be averaged or directly returned</p> </li> <li> <code>batch_size</code>               (<code>int</code>, default:                   <code>256</code> )           \u2013            <p>batch size</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>np.ndarray:</code>           \u2013            <p>array with the predictions</p> </li> </ul> Source code in <code>pytorch_widedeep/training/bayesian_trainer.py</code> <pre><code>def predict(  # type: ignore[return]\n    self,\n    X_tab: np.ndarray,\n    n_samples: int = 5,\n    return_samples: bool = False,\n    batch_size: int = 256,\n) -&gt; np.ndarray:\n    r\"\"\"Returns the predictions\n\n    Parameters\n    ----------\n    X_tab: np.ndarray,\n        tabular dataset\n    n_samples: int, default=5\n        number of samples that will be either returned or averaged to\n        produce an overal prediction\n    return_samples: bool, default = False\n        Boolean indicating whether the n samples will be averaged or directly returned\n    batch_size: int, default = 256\n        batch size\n\n    Returns\n    -------\n    np.ndarray:\n        array with the predictions\n    \"\"\"\n\n    preds_l = self._predict(X_tab, n_samples, return_samples, batch_size)\n    preds = np.hstack(preds_l) if return_samples else np.vstack(preds_l)\n    axis = 2 if return_samples else 1\n\n    if self.objective == \"regression\":\n        return preds.squeeze(axis)\n    if self.objective == \"binary\":\n        return (preds.squeeze(axis) &gt; 0.5).astype(\"int\")\n    if self.objective == \"multiclass\":\n        return np.argmax(preds, axis)\n</code></pre>"},{"location":"pytorch-widedeep/bayesian_trainer.html#pytorch_widedeep.training.BayesianTrainer.predict_proba","title":"predict_proba","text":"<pre><code>predict_proba(\n    X_tab, n_samples=5, return_samples=False, batch_size=256\n)\n</code></pre> <p>Returns the predicted probabilities</p> <p>Parameters:</p> <ul> <li> <code>X_tab</code>               (<code>ndarray</code>)           \u2013            <p>tabular dataset</p> </li> <li> <code>n_samples</code>               (<code>int</code>, default:                   <code>5</code> )           \u2013            <p>number of samples that will be either returned or averaged to produce an overal prediction</p> </li> <li> <code>return_samples</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether the n samples will be averaged or directly returned</p> </li> <li> <code>batch_size</code>               (<code>int</code>, default:                   <code>256</code> )           \u2013            <p>batch size</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>ndarray</code>           \u2013            <p>array with the probabilities per class</p> </li> </ul> Source code in <code>pytorch_widedeep/training/bayesian_trainer.py</code> <pre><code>def predict_proba(  # type: ignore[return]\n    self,\n    X_tab: np.ndarray,\n    n_samples: int = 5,\n    return_samples: bool = False,\n    batch_size: int = 256,\n) -&gt; np.ndarray:\n    r\"\"\"Returns the predicted probabilities\n\n    Parameters\n    ----------\n    X_tab: np.ndarray,\n        tabular dataset\n    n_samples: int, default=5\n        number of samples that will be either returned or averaged to\n        produce an overal prediction\n    return_samples: bool, default = False\n        Boolean indicating whether the n samples will be averaged or directly returned\n    batch_size: int, default = 256\n        batch size\n\n    Returns\n    -------\n    np.ndarray\n        array with the probabilities per class\n    \"\"\"\n    preds_l = self._predict(X_tab, n_samples, return_samples, batch_size)\n    preds = np.hstack(preds_l) if return_samples else np.vstack(preds_l)\n\n    if self.objective == \"binary\":\n        if return_samples:\n            preds = preds.squeeze(2)\n            probs = np.zeros([n_samples, preds.shape[1], 2])\n            for i in range(n_samples):\n                probs[i, :, 0] = 1 - preds[i]\n                probs[i, :, 1] = preds[i]\n        else:\n            preds = preds.squeeze(1)\n            probs = np.zeros([preds.shape[0], 2])\n            probs[:, 0] = 1 - preds\n            probs[:, 1] = preds\n        return probs\n    if self.objective == \"multiclass\":\n        return preds\n</code></pre>"},{"location":"pytorch-widedeep/bayesian_trainer.html#pytorch_widedeep.training.BayesianTrainer.save","title":"save","text":"<pre><code>save(\n    path,\n    save_state_dict=False,\n    model_filename=\"bayesian_model.pt\",\n)\n</code></pre> <p>Saves the model, training and evaluation history to disk</p> <p>The <code>Trainer</code> class is built so that it 'just' trains a model. With that in mind, all the torch related parameters (such as optimizers or learning rate schedulers) have to be defined externally and then passed to the <code>Trainer</code>. As a result, the <code>Trainer</code> does not generate any attribute or additional data products that need to be saved other than the <code>model</code> object itself, which can be saved as any other torch model (e.g. <code>torch.save(model, path)</code>).</p> <p>Parameters:</p> <ul> <li> <code>path</code>               (<code>str</code>)           \u2013            <p>path to the directory where the model and the feature importance attribute will be saved.</p> </li> <li> <code>save_state_dict</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether to save directly the model or the model's state dictionary</p> </li> <li> <code>model_filename</code>               (<code>str</code>, default:                   <code>'bayesian_model.pt'</code> )           \u2013            <p>filename where the model weights will be store</p> </li> </ul> Source code in <code>pytorch_widedeep/training/bayesian_trainer.py</code> <pre><code>def save(\n    self,\n    path: str,\n    save_state_dict: bool = False,\n    model_filename: str = \"bayesian_model.pt\",\n):\n    r\"\"\"Saves the model, training and evaluation history to disk\n\n    The `Trainer` class is built so that it 'just' trains a model. With\n    that in mind, all the torch related parameters (such as optimizers or\n    learning rate schedulers) have to be defined externally and then\n    passed to the `Trainer`. As a result, the `Trainer` does not\n    generate any attribute or additional data products that need to be\n    saved other than the `model` object itself, which can be saved as\n    any other torch model (e.g. `torch.save(model, path)`).\n\n    Parameters\n    ----------\n    path: str\n        path to the directory where the model and the feature importance\n        attribute will be saved.\n    save_state_dict: bool, default = False\n        Boolean indicating whether to save directly the model or the\n        model's state dictionary\n    model_filename: str, Optional, default = \"wd_model.pt\"\n        filename where the model weights will be store\n    \"\"\"\n\n    save_dir = Path(path)\n    history_dir = save_dir / \"history\"\n    history_dir.mkdir(exist_ok=True, parents=True)\n\n    # the trainer is run with the History Callback by default\n    with open(history_dir / \"train_eval_history.json\", \"w\") as teh:\n        json.dump(self.history, teh)  # type: ignore[attr-defined]\n\n    has_lr_history = any(\n        [clbk.__class__.__name__ == \"LRHistory\" for clbk in self.callbacks]\n    )\n    if self.lr_scheduler is not None and has_lr_history:\n        with open(history_dir / \"lr_history.json\", \"w\") as lrh:\n            json.dump(self.lr_history, lrh)  # type: ignore[attr-defined]\n\n    model_path = save_dir / model_filename\n    if save_state_dict:\n        torch.save(self.model.state_dict(), model_path)\n    else:\n        torch.save(self.model, model_path)\n</code></pre>"},{"location":"pytorch-widedeep/callbacks.html","title":"Callbacks","text":"<p>Here are the 4 callbacks available to the user in <code>pytorch-widedepp</code>: <code>LRHistory</code>, <code>ModelCheckpoint</code>, <code>EarlyStopping</code> and <code>RayTuneReporter</code>.</p> <p> NOTE: other callbacks , like <code>History</code>, run always  by default. In particular, the <code>History</code> callback saves the metrics in the  <code>history</code> attribute of the <code>Trainer</code>.</p>"},{"location":"pytorch-widedeep/callbacks.html#pytorch_widedeep.callbacks.LRHistory","title":"LRHistory","text":"<pre><code>LRHistory(n_epochs)\n</code></pre> <p>               Bases: <code>Callback</code></p> <p>Saves the learning rates during training in the <code>lr_history</code> attribute of the <code>Trainer</code>.</p> <p>Callbacks are passed as input parameters to the <code>Trainer</code> class. See <code>pytorch_widedeep.trainer.Trainer</code></p> <p>Parameters:</p> <ul> <li> <code>n_epochs</code>               (<code>int</code>)           \u2013            <p>number of training epochs</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; from pytorch_widedeep.callbacks import LRHistory\n&gt;&gt;&gt; from pytorch_widedeep.models import TabMlp, Wide, WideDeep\n&gt;&gt;&gt; from pytorch_widedeep.training import Trainer\n&gt;&gt;&gt;\n&gt;&gt;&gt; embed_input = [(u, i, j) for u, i, j in zip([\"a\", \"b\", \"c\"][:4], [4] * 3, [8] * 3)]\n&gt;&gt;&gt; column_idx = {k: v for v, k in enumerate([\"a\", \"b\", \"c\"])}\n&gt;&gt;&gt; wide = Wide(10, 1)\n&gt;&gt;&gt; deep = TabMlp(mlp_hidden_dims=[8, 4], column_idx=column_idx, cat_embed_input=embed_input)\n&gt;&gt;&gt; model = WideDeep(wide, deep)\n&gt;&gt;&gt; trainer = Trainer(model, objective=\"regression\", callbacks=[LRHistory(n_epochs=10)])\n</code></pre> Source code in <code>pytorch_widedeep/callbacks.py</code> <pre><code>def __init__(self, n_epochs: int):\n    super(LRHistory, self).__init__()\n    self.n_epochs = n_epochs\n</code></pre>"},{"location":"pytorch-widedeep/callbacks.html#pytorch_widedeep.callbacks.ModelCheckpoint","title":"ModelCheckpoint","text":"<pre><code>ModelCheckpoint(\n    filepath=None,\n    monitor=\"val_loss\",\n    min_delta=0.0,\n    verbose=0,\n    save_best_only=False,\n    mode=\"auto\",\n    period=1,\n    max_save=-1,\n)\n</code></pre> <p>               Bases: <code>Callback</code></p> <p>Saves the model after every epoch.</p> <p>This class is almost identical to the corresponding keras class. Therefore, credit to the Keras Team.</p> <p>Callbacks are passed as input parameters to the <code>Trainer</code> class. See <code>pytorch_widedeep.trainer.Trainer</code></p> <p>Parameters:</p> <ul> <li> <code>filepath</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Full path to save the output weights. It must contain only the root of the filenames. Epoch number and <code>.pt</code> extension (for pytorch) will be added. e.g. <code>filepath=\"path/to/output_weights/weights_out\"</code> And the saved files in that directory will be named: 'weights_out_1.pt', 'weights_out_2.pt', .... If set to <code>None</code> the class just report best metric and best_epoch.</p> </li> <li> <code>monitor</code>               (<code>str</code>, default:                   <code>'val_loss'</code> )           \u2013            <p>quantity to monitor. Typically 'val_loss' or metric name (e.g. 'val_acc')</p> </li> <li> <code>min_delta</code>               (<code>float</code>, default:                   <code>0.0</code> )           \u2013            <p>minimum change in the monitored quantity to qualify as an improvement, i.e. an absolute change of less than min_delta, will count as no improvement.</p> </li> <li> <code>verbose</code>               (<code>int</code>, default:                   <code>0</code> )           \u2013            <p>verbosity mode</p> </li> <li> <code>save_best_only</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>the latest best model according to the quantity monitored will not be overwritten.</p> </li> <li> <code>mode</code>               (<code>str</code>, default:                   <code>'auto'</code> )           \u2013            <p>If <code>save_best_only=True</code>, the decision to overwrite the current save file is made based on either the maximization or the minimization of the monitored quantity. For 'acc', this should be 'max', for 'loss' this should be 'min', etc. In 'auto' mode, the direction is automatically inferred from the name of the monitored quantity.</p> </li> <li> <code>period</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>Interval (number of epochs) between checkpoints.</p> </li> <li> <code>max_save</code>               (<code>int</code>, default:                   <code>-1</code> )           \u2013            <p>Maximum number of outputs to save. If -1 will save all outputs</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>best</code>               (<code>float</code>)           \u2013            <p>best metric</p> </li> <li> <code>best_epoch</code>               (<code>int</code>)           \u2013            <p>best epoch</p> </li> <li> <code>best_state_dict</code>               (<code>dict</code>)           \u2013            <p>best model state dictionary. To restore model to its best state use <code>Trainer.model.load_state_dict (model_checkpoint.best_state_dict)</code> where <code>model_checkpoint</code> is an instance of the class <code>ModelCheckpoint</code>. See the Examples folder in the repo or the Examples section in this documentation for details</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; from pytorch_widedeep.callbacks import ModelCheckpoint\n&gt;&gt;&gt; from pytorch_widedeep.models import TabMlp, Wide, WideDeep\n&gt;&gt;&gt; from pytorch_widedeep.training import Trainer\n&gt;&gt;&gt;\n&gt;&gt;&gt; embed_input = [(u, i, j) for u, i, j in zip([\"a\", \"b\", \"c\"][:4], [4] * 3, [8] * 3)]\n&gt;&gt;&gt; column_idx = {k: v for v, k in enumerate([\"a\", \"b\", \"c\"])}\n&gt;&gt;&gt; wide = Wide(10, 1)\n&gt;&gt;&gt; deep = TabMlp(mlp_hidden_dims=[8, 4], column_idx=column_idx, cat_embed_input=embed_input)\n&gt;&gt;&gt; model = WideDeep(wide, deep)\n&gt;&gt;&gt; trainer = Trainer(model, objective=\"regression\", callbacks=[ModelCheckpoint(filepath='checkpoints/weights_out')])\n</code></pre> Source code in <code>pytorch_widedeep/callbacks.py</code> <pre><code>def __init__(\n    self,\n    filepath: Optional[str] = None,\n    monitor: str = \"val_loss\",\n    min_delta: float = 0.0,\n    verbose: int = 0,\n    save_best_only: bool = False,\n    mode: str = \"auto\",\n    period: int = 1,\n    max_save: int = -1,\n):\n    super(ModelCheckpoint, self).__init__()\n\n    self.filepath = filepath\n    self.monitor = monitor\n    self.min_delta = min_delta\n    self.verbose = verbose\n    self.save_best_only = save_best_only\n    self.mode = mode\n    self.period = period\n    self.max_save = max_save\n\n    self.epochs_since_last_save = 0\n\n    if self.filepath:\n        if len(self.filepath.split(\"/\")[:-1]) == 0:\n            raise ValueError(\n                \"'filepath' must be the full path to save the output weights,\"\n                \" including the root of the filenames. e.g. 'checkpoints/weights_out'\"\n            )\n\n        root_dir = (\"/\").join(self.filepath.split(\"/\")[:-1])\n        if not os.path.exists(root_dir):\n            os.makedirs(root_dir)\n\n    if self.max_save &gt; 0:\n        self.old_files: List[str] = []\n\n    if self.mode not in [\"auto\", \"min\", \"max\"]:\n        warnings.warn(\n            \"ModelCheckpoint mode %s is unknown, \"\n            \"fallback to auto mode.\" % (self.mode),\n            RuntimeWarning,\n        )\n        self.mode = \"auto\"\n    if self.mode == \"min\":\n        self.monitor_op = np.less\n        self.best = np.Inf\n    elif self.mode == \"max\":\n        self.monitor_op = np.greater  # type: ignore[assignment]\n        self.best = -np.Inf\n    else:\n        if _is_metric(self.monitor):\n            self.monitor_op = np.greater  # type: ignore[assignment]\n            self.best = -np.Inf\n        else:\n            self.monitor_op = np.less\n            self.best = np.Inf\n\n    if self.monitor_op == np.greater:\n        self.min_delta *= 1\n    else:\n        self.min_delta *= -1\n</code></pre>"},{"location":"pytorch-widedeep/callbacks.html#pytorch_widedeep.callbacks.EarlyStopping","title":"EarlyStopping","text":"<pre><code>EarlyStopping(\n    monitor=\"val_loss\",\n    min_delta=0.0,\n    patience=10,\n    verbose=0,\n    mode=\"auto\",\n    baseline=None,\n    restore_best_weights=False,\n)\n</code></pre> <p>               Bases: <code>Callback</code></p> <p>Stop training when a monitored quantity has stopped improving.</p> <p>This class is almost identical to the corresponding keras class. Therefore, credit to the Keras Team.</p> <p>Callbacks are passed as input parameters to the <code>Trainer</code> class. See <code>pytorch_widedeep.trainer.Trainer</code></p> <p>Parameters:</p> <ul> <li> <code>monitor</code>               (<code>str</code>, default:                   <code>'val_loss'</code> )           \u2013            <p>Quantity to monitor. Typically 'val_loss' or metric name (e.g. 'val_acc')</p> </li> <li> <code>min_delta</code>               (<code>float</code>, default:                   <code>0.0</code> )           \u2013            <p>minimum change in the monitored quantity to qualify as an improvement, i.e. an absolute change of less than min_delta, will count as no improvement.</p> </li> <li> <code>patience</code>               (<code>int</code>, default:                   <code>10</code> )           \u2013            <p>Number of epochs that produced the monitored quantity with no improvement after which training will be stopped.</p> </li> <li> <code>verbose</code>               (<code>int</code>, default:                   <code>0</code> )           \u2013            <p>verbosity mode.</p> </li> <li> <code>mode</code>               (<code>str</code>, default:                   <code>'auto'</code> )           \u2013            <p>one of {'auto', 'min', 'max'}. In 'min' mode, training will stop when the quantity monitored has stopped decreasing; in 'max' mode it will stop when the quantity monitored has stopped increasing; in 'auto' mode, the direction is automatically inferred from the name of the monitored quantity.</p> </li> <li> <code>baseline</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Baseline value for the monitored quantity to reach. Training will stop if the model does not show improvement over the baseline.</p> </li> <li> <code>restore_best_weights</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Whether to restore model weights from the epoch with the best value of the monitored quantity. If <code>False</code>, the model weights obtained at the last step of training are used.</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>best</code>               (<code>float</code>)           \u2013            <p>best metric</p> </li> <li> <code>stopped_epoch</code>               (<code>int</code>)           \u2013            <p>epoch when the training stopped</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; from pytorch_widedeep.callbacks import EarlyStopping\n&gt;&gt;&gt; from pytorch_widedeep.models import TabMlp, Wide, WideDeep\n&gt;&gt;&gt; from pytorch_widedeep.training import Trainer\n&gt;&gt;&gt;\n&gt;&gt;&gt; embed_input = [(u, i, j) for u, i, j in zip([\"a\", \"b\", \"c\"][:4], [4] * 3, [8] * 3)]\n&gt;&gt;&gt; column_idx = {k: v for v, k in enumerate([\"a\", \"b\", \"c\"])}\n&gt;&gt;&gt; wide = Wide(10, 1)\n&gt;&gt;&gt; deep = TabMlp(mlp_hidden_dims=[8, 4], column_idx=column_idx, cat_embed_input=embed_input)\n&gt;&gt;&gt; model = WideDeep(wide, deep)\n&gt;&gt;&gt; trainer = Trainer(model, objective=\"regression\", callbacks=[EarlyStopping(patience=10)])\n</code></pre> Source code in <code>pytorch_widedeep/callbacks.py</code> <pre><code>def __init__(\n    self,\n    monitor: str = \"val_loss\",\n    min_delta: float = 0.0,\n    patience: int = 10,\n    verbose: int = 0,\n    mode: str = \"auto\",\n    baseline: Optional[float] = None,\n    restore_best_weights: bool = False,\n):\n    super(EarlyStopping, self).__init__()\n\n    self.monitor = monitor\n    self.min_delta = min_delta\n    self.patience = patience\n    self.verbose = verbose\n    self.mode = mode\n    self.baseline = baseline\n    self.restore_best_weights = restore_best_weights\n\n    self.wait = 0\n    self.stopped_epoch = 0\n    self.state_dict = None\n\n    if self.mode not in [\"auto\", \"min\", \"max\"]:\n        warnings.warn(\n            \"EarlyStopping mode %s is unknown, \"\n            \"fallback to auto mode.\" % self.mode,\n            RuntimeWarning,\n        )\n        self.mode = \"auto\"\n\n    if self.mode == \"min\":\n        self.monitor_op = np.less\n    elif self.mode == \"max\":\n        self.monitor_op = np.greater  # type: ignore[assignment]\n    else:\n        if _is_metric(self.monitor):\n            self.monitor_op = np.greater  # type: ignore[assignment]\n        else:\n            self.monitor_op = np.less\n\n    if self.monitor_op == np.greater:\n        self.min_delta *= 1\n    else:\n        self.min_delta *= -1\n</code></pre>"},{"location":"pytorch-widedeep/dataloaders.html","title":"Dataloaders","text":"<p> NOTE: This module should contain custom dataloaders  that the user might want to implement. At the moment <code>pytorch-widedeep</code>  offers one custom dataloader, <code>DataLoaderImbalanced</code>.</p>"},{"location":"pytorch-widedeep/dataloaders.html#pytorch_widedeep.dataloaders.DataLoaderImbalanced","title":"DataLoaderImbalanced","text":"<pre><code>DataLoaderImbalanced(dataset=None, *args, **kwargs)\n</code></pre> <p>               Bases: <code>CustomDataLoader</code></p> <p>Class to load and shuffle batches with adjusted weights for imbalanced datasets. If the classes do not begin from 0 remapping is necessary. See here.</p> <p>Parameters:</p> <ul> <li> <code>dataset</code>               (<code>Optional[WideDeepDataset]</code>, default:                   <code>None</code> )           \u2013            <p>see <code>pytorch_widedeep.training._wd_dataset</code></p> </li> </ul> <p>Other Parameters:</p> <ul> <li> <code>*args</code>           \u2013            <p>Positional arguments to be passed to the parent CustomDataLoader.</p> </li> <li> <code>**kwargs</code>           \u2013            <p>This can include any parameter that can be passed to the 'standard' pytorch DataLoader and that is not already explicitely passed to the class. In addition, the dictionary can also include the extra parameter <code>oversample_mul</code> which will multiply the number of samples of the minority class to be sampled by the <code>WeightedRandomSampler</code>.</p> <p>In other words, the <code>num_samples</code> param in <code>WeightedRandomSampler</code> will be defined as:</p> \\[ minority \\space class \\space count \\times number \\space of \\space classes \\times oversample\\_mul \\] </li> </ul> Source code in <code>pytorch_widedeep/dataloaders.py</code> <pre><code>def __init__(self, dataset: Optional[WideDeepDataset] = None, *args, **kwargs):\n    self.args = args\n    self.kwargs = kwargs\n\n    if dataset is not None:\n        self._setup_sampler(dataset)\n        super().__init__(dataset, *args, sampler=self.sampler, **kwargs)\n    else:\n        super().__init__()\n</code></pre>"},{"location":"pytorch-widedeep/load_from_folder.html","title":"The <code>load_from_folder</code> module","text":"<p>The <code>load_from_folder</code> module contains the classes that are necessary to load data from disk and these are inspired by the <code>ImageFolder</code> class in the <code>torchvision</code> library. This module is designed with one specific case in mind. Such case is the following: given a multi-modal dataset with tabular data, images and text, the images do not fit in memory, and therefore, they have to be loaded from disk. However, as any other functionality in this library, there is some flexibility and some additional cases can also be addressed using this module.</p> <p>For this module to be used, the datasets must be prepared in a certain way:</p> <ol> <li> <p>the tabular data must contain a column with the images names as stored in disk, including the extension (<code>.jpg</code>, <code>.png</code>, etc...).</p> </li> <li> <p>Regarding to the text dataset, the tabular data can contain a column with the texts themselves or the names of the files containing the texts as stored in disk.</p> </li> </ol> <p>The tabular data might or might not fit in disk itself. If it does not, please see the <code>ChunkPreprocessor</code> utilities at the[<code>preprocessing</code>] (preprocessing.md) module and the examples folder in the repo, which illustrate such case. Finally note that only <code>csv</code> format is currently supported in that case(more formats coming soon).</p>"},{"location":"pytorch-widedeep/load_from_folder.html#pytorch_widedeep.load_from_folder.tabular.tabular_from_folder.TabFromFolder","title":"TabFromFolder","text":"<pre><code>TabFromFolder(\n    fname,\n    directory=None,\n    target_col=None,\n    preprocessor=None,\n    text_col=None,\n    img_col=None,\n    ignore_target=False,\n    reference=None,\n    verbose=1,\n)\n</code></pre> <p>This class is used to load tabular data from disk. The current constrains are:</p> <ol> <li>The only file format supported right now is csv</li> <li>The csv file must contain headers</li> </ol> <p>For examples, please, see the examples folder in the repo.</p> <p>Parameters:</p> <ul> <li> <code>fname</code>               (<code>str</code>)           \u2013            <p>the name of the csv file</p> </li> <li> <code>directory</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>the path to the directory where the csv file is located. If None, a <code>TabFromFolder</code> reference object must be provided</p> </li> <li> <code>target_col</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>the name of the target column. If None, a <code>TabFromFolder</code> reference object must be provided</p> </li> <li> <code>preprocessor</code>               (<code>Optional[TabularPreprocessor]</code>, default:                   <code>None</code> )           \u2013            <p>a fitted <code>TabularPreprocessor</code> object. If None, a <code>TabFromFolder</code> reference object must be provided</p> </li> <li> <code>text_col</code>               (<code>Optional[Union[str, List[str]]]</code>, default:                   <code>None</code> )           \u2013            <p>the name of the column with the texts themselves or the names of the files that contain the text dataset. If None, either there is no text column or a <code>TabFromFolder</code> reference object must be provided</p> </li> <li> <code>img_col</code>               (<code>Optional[Union[str, List[str]]]</code>, default:                   <code>None</code> )           \u2013            <p>the name of the column with the the names of the images. If None, either there is no image column or a <code>TabFromFolder</code> reference object must be provided</p> </li> <li> <code>ignore_target</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>whether to ignore the target column. This is normally set to True when this class is used for a test dataset.</p> </li> <li> <code>reference</code>               (<code>Optional[Any]</code>, default:                   <code>None</code> )           \u2013            <p>a reference <code>TabFromFolder</code> object. If provided, the <code>TabFromFolder</code> object will be created using the attributes of the reference object. This is useful to instantiate a <code>TabFromFolder</code> object for evaluation or test purposes</p> </li> <li> <code>verbose</code>               (<code>Optional[int]</code>, default:                   <code>1</code> )           \u2013            <p>verbosity. If 0, no output will be printed during the process.</p> </li> </ul> Source code in <code>pytorch_widedeep/load_from_folder/tabular/tabular_from_folder.py</code> <pre><code>def __init__(\n    self,\n    fname: str,\n    directory: Optional[str] = None,\n    target_col: Optional[str] = None,\n    preprocessor: Optional[TabularPreprocessor] = None,\n    text_col: Optional[Union[str, List[str]]] = None,\n    img_col: Optional[Union[str, List[str]]] = None,\n    ignore_target: bool = False,\n    reference: Optional[Any] = None,  # is Type[\"TabFromFolder\"],\n    verbose: Optional[int] = 1,\n):\n    self.fname = fname\n    self.ignore_target = ignore_target\n    self.verbose = verbose\n\n    if reference is not None:\n        (\n            self.directory,\n            self.target_col,\n            self.preprocessor,\n            self.text_col,\n            self.img_col,\n        ) = self._set_from_reference(reference, preprocessor)\n    else:\n        assert (\n            directory is not None\n            and (target_col is not None and not ignore_target)\n            and preprocessor is not None\n        ), (\n            \"if no reference is provided, 'directory', 'target_col' and 'preprocessor' \"\n            \"must be provided\"\n        )\n\n        self.directory = directory\n        self.target_col = target_col\n        self.preprocessor = preprocessor\n        self.text_col = text_col\n        self.img_col = img_col\n\n    assert (\n        self.preprocessor.is_fitted\n    ), \"The preprocessor must be fitted before passing it to this class\"\n</code></pre>"},{"location":"pytorch-widedeep/load_from_folder.html#pytorch_widedeep.load_from_folder.tabular.tabular_from_folder.WideFromFolder","title":"WideFromFolder","text":"<pre><code>WideFromFolder(\n    fname,\n    directory=None,\n    target_col=None,\n    preprocessor=None,\n    text_col=None,\n    img_col=None,\n    ignore_target=False,\n    reference=None,\n    verbose=1,\n)\n</code></pre> <p>               Bases: <code>TabFromFolder</code></p> <p>This class is mostly identical to <code>TabFromFolder</code> but exists because we want to separate the treatment of the wide and the deep tabular components</p> <p>Parameters:</p> <ul> <li> <code>fname</code>               (<code>str</code>)           \u2013            <p>the name of the csv file</p> </li> <li> <code>directory</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>the path to the directory where the csv file is located. If None, a <code>WideFromFolder</code> reference object must be provided</p> </li> <li> <code>target_col</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>the name of the target column. If None, a <code>WideFromFolder</code> reference object must be provided</p> </li> <li> <code>preprocessor</code>               (<code>Optional[TabularPreprocessor]</code>, default:                   <code>None</code> )           \u2013            <p>a fitted <code>TabularPreprocessor</code> object. If None, a <code>WideFromFolder</code> reference object must be provided</p> </li> <li> <code>text_col</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>the name of the column with the texts themselves or the names of the files that contain the text dataset. If None, either there is no text column or a <code>WideFromFolder</code> reference object must be provided=</p> </li> <li> <code>img_col</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>the name of the column with the the names of the images. If None, either there is no image column or a <code>WideFromFolder</code> reference object must be provided</p> </li> <li> <code>ignore_target</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>whether to ignore the target column. This is normally used when this class is used for a test dataset.</p> </li> <li> <code>reference</code>               (<code>Optional[Any]</code>, default:                   <code>None</code> )           \u2013            <p>a reference <code>WideFromFolder</code> object. If provided, the <code>WideFromFolder</code> object will be created using the attributes of the reference object. This is useful to instantiate a <code>WideFromFolder</code> object for evaluation or test purposes</p> </li> <li> <code>verbose</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>verbosity. If 0, no output will be printed during the process.</p> </li> </ul> Source code in <code>pytorch_widedeep/load_from_folder/tabular/tabular_from_folder.py</code> <pre><code>def __init__(\n    self,\n    fname: str,\n    directory: Optional[str] = None,\n    target_col: Optional[str] = None,\n    preprocessor: Optional[TabularPreprocessor] = None,\n    text_col: Optional[str] = None,\n    img_col: Optional[str] = None,\n    ignore_target: bool = False,\n    reference: Optional[Any] = None,  # is Type[\"WideFromFolder\"],\n    verbose: int = 1,\n):\n    super(WideFromFolder, self).__init__(\n        fname=fname,\n        directory=directory,\n        target_col=target_col,\n        preprocessor=preprocessor,\n        text_col=text_col,\n        img_col=img_col,\n        reference=reference,\n        ignore_target=ignore_target,\n        verbose=verbose,\n    )\n</code></pre>"},{"location":"pytorch-widedeep/load_from_folder.html#pytorch_widedeep.load_from_folder.text.text_from_folder.TextFromFolder","title":"TextFromFolder","text":"<pre><code>TextFromFolder(preprocessor)\n</code></pre> <p>This class is used to load the text dataset (i.e. the text files) from a folder, or to retrieve the text given a texts column specified within the preprocessor object.</p> <p>For examples, please, see the examples folder in the repo.</p> <p>Parameters:</p> <ul> <li> <code>preprocessor</code>               (<code>Union[TextPreprocessor, ChunkTextPreprocessor, HFPreprocessor, ChunkHFPreprocessor, List[TextPreprocessor], List[ChunkTextPreprocessor], List[HFPreprocessor], List[ChunkHFPreprocessor]]</code>)           \u2013            <p>The preprocessor used to process the text. It must be fitted before using this class</p> </li> </ul> Source code in <code>pytorch_widedeep/load_from_folder/text/text_from_folder.py</code> <pre><code>def __init__(\n    self,\n    preprocessor: Union[\n        TextPreprocessor,\n        ChunkTextPreprocessor,\n        HFPreprocessor,\n        ChunkHFPreprocessor,\n        List[TextPreprocessor],\n        List[ChunkTextPreprocessor],\n        List[HFPreprocessor],\n        List[ChunkHFPreprocessor],\n    ],\n):\n    if isinstance(preprocessor, list):\n        for p in preprocessor:\n            assert (\n                p.is_fitted\n            ), \"All preprocessors must be fitted before using this class\"\n    else:\n        assert (\n            preprocessor.is_fitted\n        ), \"The preprocessor must be fitted before using this class\"\n\n    self.preprocessor = preprocessor\n</code></pre>"},{"location":"pytorch-widedeep/load_from_folder.html#pytorch_widedeep.load_from_folder.image.image_from_folder.ImageFromFolder","title":"ImageFromFolder","text":"<pre><code>ImageFromFolder(\n    directory=None,\n    preprocessor=None,\n    loader=default_loader,\n    extensions=None,\n    transforms=None,\n)\n</code></pre> <p>This class is used to load the image dataset from disk. It is inspired by the <code>ImageFolder</code> class at the <code>torchvision</code> library. Here, we have simply adapted to work within the context of a Wide and Deep multi-modal model.</p> <p>For examples, please, see the examples folder in the repo.</p> <p>Parameters:</p> <ul> <li> <code>directory</code>               (<code>Optional[Union[str, List[str]]]</code>, default:                   <code>None</code> )           \u2013            <p>the path to the directory where the images are located. If None, a preprocessor must be provided.</p> </li> <li> <code>preprocessor</code>               (<code>Optional[Union[ImagePreprocessor, List[ImagePreprocessor]]]</code>, default:                   <code>None</code> )           \u2013            <p>a fitted <code>ImagePreprocessor</code> object.</p> </li> <li> <code>loader</code>               (<code>Callable[[str], Any]</code>, default:                   <code>default_loader</code> )           \u2013            <p>a function to load a sample given its path.</p> </li> <li> <code>extensions</code>               (<code>Optional[Tuple[str, ...]]</code>, default:                   <code>None</code> )           \u2013            <p>a tuple with the allowed extensions. If None, IMG_EXTENSIONS will be used where IMG_EXTENSIONS =\".jpg\", \".jpeg\", \".png\", \".ppm\", \".bmp\", \".pgm\", \".tif\", \".tiff\", \".webp\"</p> </li> <li> <code>transforms</code>               (<code>Optional[Any]</code>, default:                   <code>None</code> )           \u2013            <p>a <code>torchvision.transforms</code> object. If None, this class will simply return an array representation of the PIL Image</p> </li> </ul> Source code in <code>pytorch_widedeep/load_from_folder/image/image_from_folder.py</code> <pre><code>def __init__(\n    self,\n    directory: Optional[Union[str, List[str]]] = None,\n    preprocessor: Optional[\n        Union[ImagePreprocessor, List[ImagePreprocessor]]\n    ] = None,\n    loader: Callable[[str], Any] = default_loader,\n    extensions: Optional[Tuple[str, ...]] = None,\n    transforms: Optional[Any] = None,\n) -&gt; None:\n    assert (\n        directory is not None or preprocessor is not None\n    ), \"Either a directory or an instance of ImagePreprocessor(s) must be provided\"\n\n    if directory is not None and preprocessor is not None:  # pragma: no cover\n        error_msg = (\n            \"If both 'directory' and 'preprocessor' are provided, the 'img_path' \"\n            \"attribute of the 'preprocessor' must be the same as the 'directory'\"\n        )\n        if isinstance(directory, list):\n            assert isinstance(preprocessor, list)\n            assert len(directory) == len(preprocessor)\n            for d, p in zip(directory, preprocessor):\n                assert d == p.img_path, error_msg\n        else:\n            assert isinstance(preprocessor, ImagePreprocessor)\n            assert directory == preprocessor.img_path, error_msg\n\n    if directory is not None:\n        self.directory = directory\n    else:\n        assert (\n            preprocessor is not None\n        ), \"Either a directory or an instance of ImagePreprocessor must be provided\"\n        if isinstance(preprocessor, list):\n            self.directory = [p.img_path for p in preprocessor]\n        else:\n            self.directory = preprocessor.img_path\n\n    self.preprocessor = preprocessor\n    self.loader = loader\n    self.extensions = extensions if extensions is not None else IMG_EXTENSIONS\n    self.transforms = transforms\n    if self.transforms:\n        self.transforms_names = [\n            tr.__class__.__name__ for tr in self.transforms.transforms\n        ]\n    else:\n        self.transforms_names = []\n\n        self.transpose = True\n</code></pre>"},{"location":"pytorch-widedeep/load_from_folder.html#pytorch_widedeep.load_from_folder.wd_dataset_from_folder.WideDeepDatasetFromFolder","title":"WideDeepDatasetFromFolder","text":"<pre><code>WideDeepDatasetFromFolder(\n    n_samples,\n    tab_from_folder=None,\n    wide_from_folder=None,\n    text_from_folder=None,\n    img_from_folder=None,\n    reference=None,\n)\n</code></pre> <p>               Bases: <code>Dataset</code></p> <p>This class is the Dataset counterpart of the <code>WideDeepDataset</code> class.</p> <p>Given a reference tabular dataset, with columns that indicate the path to the images and to the text files or the texts themselves, it will use the <code>[...]FromFolder</code> classes to load the data consistently from disk per batch.</p> <p>For examples, please, see the examples folder in the repo.</p> <p>Parameters:</p> <ul> <li> <code>n_samples</code>               (<code>int</code>)           \u2013            <p>Number of samples in the dataset</p> </li> <li> <code>tab_from_folder</code>               (<code>Optional[TabFromFolder]</code>, default:                   <code>None</code> )           \u2013            <p>Instance of the <code>TabFromFolder</code> class</p> </li> <li> <code>wide_from_folder</code>               (<code>Optional[WideFromFolder]</code>, default:                   <code>None</code> )           \u2013            <p>Instance of the <code>WideFromFolder</code> class</p> </li> <li> <code>text_from_folder</code>               (<code>Optional[TextFromFolder]</code>, default:                   <code>None</code> )           \u2013            <p>Instance of the <code>TextFromFolder</code> class</p> </li> <li> <code>img_from_folder</code>               (<code>Optional[ImageFromFolder]</code>, default:                   <code>None</code> )           \u2013            <p>Instance of the <code>ImageFromFolder</code> class</p> </li> <li> <code>reference</code>               (<code>Optional[Any]</code>, default:                   <code>None</code> )           \u2013            <p>If not None, the 'text_from_folder' and 'img_from_folder' objects will be retrieved from the reference class. This is useful when we want to use a <code>WideDeepDatasetFromFolder</code> class used for a train dataset as a reference for the validation and test datasets. In this case, the <code>text_from_folder</code> and <code>img_from_folder</code> objects will be the same for all three datasets, so there is no need to create a new instance for each dataset.</p> </li> </ul> Source code in <code>pytorch_widedeep/load_from_folder/wd_dataset_from_folder.py</code> <pre><code>def __init__(\n    self,\n    n_samples: int,\n    tab_from_folder: Optional[TabFromFolder] = None,\n    wide_from_folder: Optional[WideFromFolder] = None,\n    text_from_folder: Optional[TextFromFolder] = None,\n    img_from_folder: Optional[ImageFromFolder] = None,\n    reference: Optional[Any] = None,  # is Type[\"WideDeepDatasetFromFolder\"],\n):\n    super(WideDeepDatasetFromFolder, self).__init__()\n\n    if tab_from_folder is None and wide_from_folder is None:\n        raise ValueError(\n            \"Either 'tab_from_folder' or 'wide_from_folder' must be not None\"\n        )\n\n    if reference is not None:\n        assert (\n            img_from_folder is None and text_from_folder is None\n        ), \"If reference is not None, 'img_from_folder' and 'text_from_folder' left as None\"\n        self.text_from_folder, self.img_from_folder = self._get_from_reference(\n            reference\n        )\n    else:\n        assert (\n            text_from_folder is not None and img_from_folder is not None\n        ), \"If reference is None, 'img_from_folder' and 'text_from_folder' must be not None\"\n        self.text_from_folder = text_from_folder\n        self.img_from_folder = img_from_folder\n\n    self.n_samples = n_samples\n    self.tab_from_folder = tab_from_folder\n    self.wide_from_folder = wide_from_folder\n</code></pre>"},{"location":"pytorch-widedeep/losses.html","title":"Losses","text":"<p><code>pytorch-widedeep</code> accepts a number of losses and objectives that can be passed to the <code>Trainer</code> class via the parameter <code>objective</code> (see <code>pytorch-widedeep.training.Trainer</code>). For most cases the loss function that <code>pytorch-widedeep</code> will use internally is already implemented in Pytorch.</p> <p>In addition, <code>pytorch-widedeep</code> implements a series of  \"custom\" loss functions. These are described below for completion since, as mentioned before, they are used internally by the <code>Trainer</code>. Of course, onen could always use them on their own and can be imported as:</p> <p><code>from pytorch_widedeep.losses import FocalLoss</code></p> <p> NOTE:  Losses in this module expect the predictions  and ground truth to have the same dimensions for regression and binary  classification problems \\((N_{samples}, 1)\\). In the case of multiclass  classification problems the ground truth is expected to be a 1D tensor with  the corresponding classes. See Examples below</p>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.MSELoss","title":"MSELoss","text":"<pre><code>MSELoss()\n</code></pre> <p>               Bases: <code>Module</code></p> <p>Mean square error loss</p> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def __init__(self):\n    super().__init__()\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.MSELoss.forward","title":"forward","text":"<pre><code>forward(input, target)\n</code></pre> <p>Parameters:</p> <ul> <li> <code>input</code>               (<code>Tensor</code>)           \u2013            <p>Input tensor with predictions</p> </li> <li> <code>target</code>               (<code>Tensor</code>)           \u2013            <p>Target tensor with the actual values</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.losses import MSELoss\n&gt;&gt;&gt;\n&gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n&gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n&gt;&gt;&gt; loss = MSELoss()(input, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(\n    self,\n    input: Tensor,\n    target: Tensor,\n) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    input: Tensor\n        Input tensor with predictions\n    target: Tensor\n        Target tensor with the actual values\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.losses import MSELoss\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n    &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n    &gt;&gt;&gt; loss = MSELoss()(input, target)\n    \"\"\"\n    loss = (input - target) ** 2\n    return torch.mean(loss)\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.MSLELoss","title":"MSLELoss","text":"<pre><code>MSLELoss()\n</code></pre> <p>               Bases: <code>Module</code></p> <p>Mean square log error loss</p> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def __init__(self):\n    super().__init__()\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.MSLELoss.forward","title":"forward","text":"<pre><code>forward(input, target)\n</code></pre> <p>Parameters:</p> <ul> <li> <code>input</code>               (<code>Tensor</code>)           \u2013            <p>Input tensor with predictions (not probabilities)</p> </li> <li> <code>target</code>               (<code>Tensor</code>)           \u2013            <p>Target tensor with the actual classes</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.losses import MSLELoss\n&gt;&gt;&gt;\n&gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n&gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n&gt;&gt;&gt; loss = MSLELoss()(input, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(\n    self,\n    input: Tensor,\n    target: Tensor,\n) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    input: Tensor\n        Input tensor with predictions (not probabilities)\n    target: Tensor\n        Target tensor with the actual classes\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.losses import MSLELoss\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n    &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n    &gt;&gt;&gt; loss = MSLELoss()(input, target)\n    \"\"\"\n    assert (\n        input.min() &gt;= 0\n    ), \"\"\"All input values must be &gt;=0, if your model is predicting\n        values &lt;0 try to enforce positive values by activation function\n        on last layer with `trainer.enforce_positive_output=True`\"\"\"\n    assert target.min() &gt;= 0, \"All target values must be &gt;=0\"\n\n    loss = (torch.log(input + 1) - torch.log(target + 1)) ** 2\n    return torch.mean(loss)\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.RMSELoss","title":"RMSELoss","text":"<pre><code>RMSELoss()\n</code></pre> <p>               Bases: <code>Module</code></p> <p>Root mean square error loss</p> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def __init__(self):\n    super().__init__()\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.RMSELoss.forward","title":"forward","text":"<pre><code>forward(input, target)\n</code></pre> <p>Parameters:</p> <ul> <li> <code>input</code>               (<code>Tensor</code>)           \u2013            <p>Input tensor with predictions (not probabilities)</p> </li> <li> <code>target</code>               (<code>Tensor</code>)           \u2013            <p>Target tensor with the actual classes</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.losses import RMSELoss\n&gt;&gt;&gt;\n&gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n&gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n&gt;&gt;&gt; loss = RMSELoss()(input, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(self, input: Tensor, target: Tensor) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    input: Tensor\n        Input tensor with predictions (not probabilities)\n    target: Tensor\n        Target tensor with the actual classes\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.losses import RMSELoss\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n    &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n    &gt;&gt;&gt; loss = RMSELoss()(input, target)\n    \"\"\"\n    loss = (input - target) ** 2\n    return torch.sqrt(torch.mean(loss))\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.RMSLELoss","title":"RMSLELoss","text":"<pre><code>RMSLELoss()\n</code></pre> <p>               Bases: <code>Module</code></p> <p>Root mean square log error loss</p> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def __init__(self):\n    super().__init__()\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.RMSLELoss.forward","title":"forward","text":"<pre><code>forward(input, target)\n</code></pre> <p>Parameters:</p> <ul> <li> <code>input</code>               (<code>Tensor</code>)           \u2013            <p>Input tensor with predictions (not probabilities)</p> </li> <li> <code>target</code>               (<code>Tensor</code>)           \u2013            <p>Target tensor with the actual classes</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.losses import RMSLELoss\n&gt;&gt;&gt;\n&gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n&gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n&gt;&gt;&gt; loss = RMSLELoss()(input, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(self, input: Tensor, target: Tensor) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    input: Tensor\n        Input tensor with predictions (not probabilities)\n    target: Tensor\n        Target tensor with the actual classes\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.losses import RMSLELoss\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n    &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n    &gt;&gt;&gt; loss = RMSLELoss()(input, target)\n    \"\"\"\n    assert (\n        input.min() &gt;= 0\n    ), \"\"\"All input values must be &gt;=0, if your model is predicting\n        values &lt;0 try to enforce positive values by activation function\n        on last layer with `trainer.enforce_positive_output=True`\"\"\"\n    assert target.min() &gt;= 0, \"All target values must be &gt;=0\"\n\n    loss = (torch.log(input + 1) - torch.log(target + 1)) ** 2\n    return torch.sqrt(torch.mean(loss))\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.QuantileLoss","title":"QuantileLoss","text":"<pre><code>QuantileLoss(\n    quantiles=[0.02, 0.1, 0.25, 0.5, 0.75, 0.9, 0.98]\n)\n</code></pre> <p>               Bases: <code>Module</code></p> <p>Quantile loss defined as:</p> \\[ Loss = max(q \\times (y-y_{pred}), (1-q) \\times (y_{pred}-y)) \\] <p>All credits go to the implementation at pytorch-forecasting.</p> <p>Parameters:</p> <ul> <li> <code>quantiles</code>               (<code>List[float]</code>, default:                   <code>[0.02, 0.1, 0.25, 0.5, 0.75, 0.9, 0.98]</code> )           \u2013            <p>List of quantiles</p> </li> </ul> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def __init__(\n    self,\n    quantiles: List[float] = [0.02, 0.1, 0.25, 0.5, 0.75, 0.9, 0.98],\n):\n    super().__init__()\n    self.quantiles = quantiles\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.QuantileLoss.forward","title":"forward","text":"<pre><code>forward(input, target)\n</code></pre> <p>Parameters:</p> <ul> <li> <code>input</code>               (<code>Tensor</code>)           \u2013            <p>Input tensor with predictions</p> </li> <li> <code>target</code>               (<code>Tensor</code>)           \u2013            <p>Target tensor with the actual values</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt;\n&gt;&gt;&gt; from pytorch_widedeep.losses import QuantileLoss\n&gt;&gt;&gt;\n&gt;&gt;&gt; # REGRESSION\n&gt;&gt;&gt; target = torch.tensor([[0.6, 1.5]]).view(-1, 1)\n&gt;&gt;&gt; input = torch.tensor([[.1, .2,], [.4, .5]])\n&gt;&gt;&gt; qloss = QuantileLoss([0.25, 0.75])\n&gt;&gt;&gt; loss = qloss(input, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(self, input: Tensor, target: Tensor) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    input: Tensor\n        Input tensor with predictions\n    target: Tensor\n        Target tensor with the actual values\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; from pytorch_widedeep.losses import QuantileLoss\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; # REGRESSION\n    &gt;&gt;&gt; target = torch.tensor([[0.6, 1.5]]).view(-1, 1)\n    &gt;&gt;&gt; input = torch.tensor([[.1, .2,], [.4, .5]])\n    &gt;&gt;&gt; qloss = QuantileLoss([0.25, 0.75])\n    &gt;&gt;&gt; loss = qloss(input, target)\n    \"\"\"\n\n    assert input.shape == torch.Size([target.shape[0], len(self.quantiles)]), (\n        \"The input and target have inconsistent shape. The dimension of the prediction \"\n        \"of the model that is using QuantileLoss must be equal to number of quantiles, \"\n        f\"i.e. {len(self.quantiles)}.\"\n    )\n    target = target.view(-1, 1).float()\n    losses = []\n    for i, q in enumerate(self.quantiles):\n        errors = target - input[..., i]\n        losses.append(torch.max((q - 1) * errors, q * errors).unsqueeze(-1))\n\n    loss = torch.cat(losses, dim=2)\n\n    return torch.mean(loss)\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.FocalLoss","title":"FocalLoss","text":"<pre><code>FocalLoss(alpha=0.25, gamma=1.0)\n</code></pre> <p>               Bases: <code>Module</code></p> <p>Implementation of the Focal loss for both binary and multiclass classification:</p> \\[ FL(p_t) = \\alpha (1 - p_t)^{\\gamma} log(p_t) \\] <p>where, for a case of a binary classification problem</p> \\[ \\begin{equation} p_t= \\begin{cases}p, &amp; \\text{if $y=1$}.\\\\1-p, &amp; \\text{otherwise}. \\end{cases} \\end{equation} \\] <p>Parameters:</p> <ul> <li> <code>alpha</code>               (<code>float</code>, default:                   <code>0.25</code> )           \u2013            <p>Focal Loss <code>alpha</code> parameter</p> </li> <li> <code>gamma</code>               (<code>float</code>, default:                   <code>1.0</code> )           \u2013            <p>Focal Loss <code>gamma</code> parameter</p> </li> </ul> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def __init__(self, alpha: float = 0.25, gamma: float = 1.0):\n    super().__init__()\n    self.alpha = alpha\n    self.gamma = gamma\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.FocalLoss.forward","title":"forward","text":"<pre><code>forward(input, target)\n</code></pre> <p>Parameters:</p> <ul> <li> <code>input</code>               (<code>Tensor</code>)           \u2013            <p>Input tensor with predictions (not probabilities)</p> </li> <li> <code>target</code>               (<code>Tensor</code>)           \u2013            <p>Target tensor with the actual classes</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt;\n&gt;&gt;&gt; from pytorch_widedeep.losses import FocalLoss\n&gt;&gt;&gt;\n&gt;&gt;&gt; # BINARY\n&gt;&gt;&gt; target = torch.tensor([0, 1, 0, 1]).view(-1, 1)\n&gt;&gt;&gt; input = torch.tensor([[0.6, 0.7, 0.3, 0.8]]).t()\n&gt;&gt;&gt; loss = FocalLoss()(input, target)\n&gt;&gt;&gt;\n&gt;&gt;&gt; # MULTICLASS\n&gt;&gt;&gt; target = torch.tensor([1, 0, 2]).view(-1, 1)\n&gt;&gt;&gt; input = torch.tensor([[0.2, 0.5, 0.3], [0.8, 0.1, 0.1], [0.7, 0.2, 0.1]])\n&gt;&gt;&gt; loss = FocalLoss()(input, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(self, input: Tensor, target: Tensor) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    input: Tensor\n        Input tensor with predictions (not probabilities)\n    target: Tensor\n        Target tensor with the actual classes\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; from pytorch_widedeep.losses import FocalLoss\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; # BINARY\n    &gt;&gt;&gt; target = torch.tensor([0, 1, 0, 1]).view(-1, 1)\n    &gt;&gt;&gt; input = torch.tensor([[0.6, 0.7, 0.3, 0.8]]).t()\n    &gt;&gt;&gt; loss = FocalLoss()(input, target)\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; # MULTICLASS\n    &gt;&gt;&gt; target = torch.tensor([1, 0, 2]).view(-1, 1)\n    &gt;&gt;&gt; input = torch.tensor([[0.2, 0.5, 0.3], [0.8, 0.1, 0.1], [0.7, 0.2, 0.1]])\n    &gt;&gt;&gt; loss = FocalLoss()(input, target)\n    \"\"\"\n    input_prob = torch.sigmoid(input)\n    if input.size(1) == 1:\n        input_prob = torch.cat([1 - input_prob, input_prob], axis=1)  # type: ignore\n        num_class = 2\n    else:\n        num_class = input_prob.size(1)\n    binary_target = torch.eye(num_class)[target.squeeze().cpu().long()]\n    binary_target = binary_target.to(input.device)\n    binary_target = binary_target.contiguous()\n    weight = self._get_weight(input_prob, binary_target)\n\n    return F.binary_cross_entropy(\n        input_prob, binary_target, weight, reduction=\"mean\"\n    )\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.BayesianSELoss","title":"BayesianSELoss","text":"<pre><code>BayesianSELoss()\n</code></pre> <p>               Bases: <code>Module</code></p> <p>Squared Loss (log Gaussian) for the case of a regression as specified in the original publication Weight Uncertainty in Neural Networks.</p> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def __init__(self):\n    super().__init__()\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.BayesianSELoss.forward","title":"forward","text":"<pre><code>forward(input, target)\n</code></pre> <p>Parameters:</p> <ul> <li> <code>input</code>               (<code>Tensor</code>)           \u2013            <p>Input tensor with predictions (not probabilities)</p> </li> <li> <code>target</code>               (<code>Tensor</code>)           \u2013            <p>Target tensor with the actual classes</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.losses import BayesianSELoss\n&gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n&gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n&gt;&gt;&gt; loss = BayesianSELoss()(input, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(self, input: Tensor, target: Tensor) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    input: Tensor\n        Input tensor with predictions (not probabilities)\n    target: Tensor\n        Target tensor with the actual classes\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.losses import BayesianSELoss\n    &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n    &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n    &gt;&gt;&gt; loss = BayesianSELoss()(input, target)\n    \"\"\"\n    return (0.5 * (input - target) ** 2).sum()\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.TweedieLoss","title":"TweedieLoss","text":"<pre><code>TweedieLoss()\n</code></pre> <p>               Bases: <code>Module</code></p> <p>Tweedie loss for extremely unbalanced zero-inflated data</p> <p>All credits go to Wenbo Shi. See this post and the original publication for details.</p> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def __init__(self):\n    super().__init__()\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.TweedieLoss.forward","title":"forward","text":"<pre><code>forward(input, target, p=1.5)\n</code></pre> <p>Parameters:</p> <ul> <li> <code>input</code>               (<code>Tensor</code>)           \u2013            <p>Input tensor with predictions</p> </li> <li> <code>target</code>               (<code>Tensor</code>)           \u2013            <p>Target tensor with the actual values</p> </li> <li> <code>p</code>               (<code>float</code>, default:                   <code>1.5</code> )           \u2013            <p>the power to be used to compute the loss. See the original publication for details</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.losses import TweedieLoss\n&gt;&gt;&gt;\n&gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n&gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n&gt;&gt;&gt; loss = TweedieLoss()(input, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(\n    self,\n    input: Tensor,\n    target: Tensor,\n    p: float = 1.5,\n) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    input: Tensor\n        Input tensor with predictions\n    target: Tensor\n        Target tensor with the actual values\n    p: float, default = 1.5\n        the power to be used to compute the loss. See the original\n        publication for details\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.losses import TweedieLoss\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n    &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n    &gt;&gt;&gt; loss = TweedieLoss()(input, target)\n    \"\"\"\n\n    assert (\n        input.min() &gt; 0\n    ), \"\"\"All input values must be &gt;=0, if your model is predicting\n        values &lt;0 try to enforce positive values by activation function\n        on last layer with `trainer.enforce_positive_output=True`\"\"\"\n    assert target.min() &gt;= 0, \"All target values must be &gt;=0\"\n    loss = -target * torch.pow(input, 1 - p) / (1 - p) + torch.pow(input, 2 - p) / (\n        2 - p\n    )\n    return torch.mean(loss)\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.ZILNLoss","title":"ZILNLoss","text":"<pre><code>ZILNLoss()\n</code></pre> <p>               Bases: <code>Module</code></p> <p>Adjusted implementation of the Zero Inflated LogNormal Loss</p> <p>See A Deep Probabilistic Model for Customer Lifetime Value Prediction and the corresponding code.</p> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def __init__(self):\n    super().__init__()\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.ZILNLoss.forward","title":"forward","text":"<pre><code>forward(input, target)\n</code></pre> <p>Parameters:</p> <ul> <li> <code>input</code>               (<code>Tensor</code>)           \u2013            <p>Input tensor with predictions with spape (N,3), where N is the batch size</p> </li> <li> <code>target</code>               (<code>Tensor</code>)           \u2013            <p>Target tensor with the actual target values</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.losses import ZILNLoss\n&gt;&gt;&gt;\n&gt;&gt;&gt; target = torch.tensor([[0., 1.5]]).view(-1, 1)\n&gt;&gt;&gt; input = torch.tensor([[.1, .2, .3], [.4, .5, .6]])\n&gt;&gt;&gt; loss = ZILNLoss()(input, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(self, input: Tensor, target: Tensor) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    input: Tensor\n        Input tensor with predictions with spape (N,3), where N is the batch size\n    target: Tensor\n        Target tensor with the actual target values\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.losses import ZILNLoss\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; target = torch.tensor([[0., 1.5]]).view(-1, 1)\n    &gt;&gt;&gt; input = torch.tensor([[.1, .2, .3], [.4, .5, .6]])\n    &gt;&gt;&gt; loss = ZILNLoss()(input, target)\n    \"\"\"\n    positive = target &gt; 0\n    positive = positive.float()\n\n    assert input.shape == torch.Size([target.shape[0], 3]), (\n        \"Wrong shape of the 'input' tensor. The pred_dim of the \"\n        \"model that is using ZILNLoss must be equal to 3.\"\n    )\n\n    positive_input = input[..., :1]\n\n    classification_loss = F.binary_cross_entropy_with_logits(\n        positive_input, positive, reduction=\"none\"\n    ).flatten()\n\n    loc = input[..., 1:2]\n\n    # when using max the two input tensors (input and other) have to be of\n    # the same type\n    max_input = F.softplus(input[..., 2:])\n    max_other = self.get_eps(max_input)\n    scale = torch.max(max_input, max_other)\n    safe_labels = positive * target + (1 - positive) * torch.ones_like(target)\n\n    regression_loss = -torch.mean(\n        positive\n        * torch.distributions.log_normal.LogNormal(loc=loc, scale=scale).log_prob(\n            safe_labels\n        ),\n        dim=-1,\n    )\n\n    return torch.mean(classification_loss + regression_loss)\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.L1Loss","title":"L1Loss","text":"<pre><code>L1Loss()\n</code></pre> <p>               Bases: <code>Module</code></p> <p>L1 loss</p> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def __init__(self):\n    super().__init__()\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.L1Loss.forward","title":"forward","text":"<pre><code>forward(input, target)\n</code></pre> <p>Parameters:</p> <ul> <li> <code>input</code>               (<code>Tensor</code>)           \u2013            <p>Input tensor with predictions</p> </li> <li> <code>target</code>               (<code>Tensor</code>)           \u2013            <p>Target tensor with the actual values</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt;\n&gt;&gt;&gt; from pytorch_widedeep.losses import L1Loss\n&gt;&gt;&gt;\n&gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n&gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n&gt;&gt;&gt; loss = L1Loss()(input, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(self, input: Tensor, target: Tensor) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    input: Tensor\n        Input tensor with predictions\n    target: Tensor\n        Target tensor with the actual values\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; from pytorch_widedeep.losses import L1Loss\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n    &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n    &gt;&gt;&gt; loss = L1Loss()(input, target)\n    \"\"\"\n    loss = F.l1_loss(input, target, reduction=\"none\")\n    return torch.mean(loss)\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.FocalR_L1Loss","title":"FocalR_L1Loss","text":"<pre><code>FocalR_L1Loss(beta=0.2, gamma=1.0, activation_fn='sigmoid')\n</code></pre> <p>               Bases: <code>Module</code></p> <p>Focal-R L1 loss</p> <p>Based on Delving into Deep Imbalanced Regression.</p> <p>Parameters:</p> <ul> <li> <code>beta</code>               (<code>float</code>, default:                   <code>0.2</code> )           \u2013            <p>Focal Loss <code>beta</code> parameter in their implementation</p> </li> <li> <code>gamma</code>               (<code>float</code>, default:                   <code>1.0</code> )           \u2013            <p>Focal Loss <code>gamma</code> parameter</p> </li> <li> <code>activation_fn</code>               (<code>Literal[sigmoid, tanh]</code>, default:                   <code>'sigmoid'</code> )           \u2013            <p>Activation function to be used during the computation of the loss. Possible values are 'sigmoid' and 'tanh'. See the original publication for details.</p> </li> </ul> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def __init__(\n    self,\n    beta: float = 0.2,\n    gamma: float = 1.0,\n    activation_fn: Literal[\"sigmoid\", \"tanh\"] = \"sigmoid\",\n):\n    super().__init__()\n    self.beta = beta\n    self.gamma = gamma\n    self.activation_fn = activation_fn\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.FocalR_L1Loss.forward","title":"forward","text":"<pre><code>forward(input, target)\n</code></pre> <p>Parameters:</p> <ul> <li> <code>input</code>               (<code>Tensor</code>)           \u2013            <p>Input tensor with predictions (not probabilities)</p> </li> <li> <code>target</code>               (<code>Tensor</code>)           \u2013            <p>Target tensor with the actual classes</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt;\n&gt;&gt;&gt; from pytorch_widedeep.losses import FocalR_L1Loss\n&gt;&gt;&gt;\n&gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n&gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n&gt;&gt;&gt; loss = FocalR_L1Loss()(input, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(\n    self,\n    input: Tensor,\n    target: Tensor,\n) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    input: Tensor\n        Input tensor with predictions (not probabilities)\n    target: Tensor\n        Target tensor with the actual classes\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; from pytorch_widedeep.losses import FocalR_L1Loss\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n    &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n    &gt;&gt;&gt; loss = FocalR_L1Loss()(input, target)\n    \"\"\"\n    loss = F.l1_loss(input, target, reduction=\"none\")\n    if self.activation_fn == \"tanh\":\n        loss *= (torch.tanh(self.beta * torch.abs(input - target))) ** self.gamma\n    elif self.activation_fn == \"sigmoid\":\n        loss *= (\n            2 * torch.sigmoid(self.beta * torch.abs(input - target)) - 1\n        ) ** self.gamma\n    else:\n        ValueError(\n            \"Incorrect activation function value - must be in ['sigmoid', 'tanh']\"\n        )\n    return torch.mean(loss)\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.FocalR_MSELoss","title":"FocalR_MSELoss","text":"<pre><code>FocalR_MSELoss(\n    beta=0.2, gamma=1.0, activation_fn=\"sigmoid\"\n)\n</code></pre> <p>               Bases: <code>Module</code></p> <p>Focal-R MSE loss</p> <p>Based on Delving into Deep Imbalanced Regression.</p> <p>Parameters:</p> <ul> <li> <code>beta</code>               (<code>float</code>, default:                   <code>0.2</code> )           \u2013            <p>Focal Loss <code>beta</code> parameter in their implementation</p> </li> <li> <code>gamma</code>               (<code>float</code>, default:                   <code>1.0</code> )           \u2013            <p>Focal Loss <code>gamma</code> parameter</p> </li> <li> <code>activation_fn</code>               (<code>Literal[sigmoid, tanh]</code>, default:                   <code>'sigmoid'</code> )           \u2013            <p>Activation function to be used during the computation of the loss. Possible values are 'sigmoid' and 'tanh'. See the original publication for details.</p> </li> </ul> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def __init__(\n    self,\n    beta: float = 0.2,\n    gamma: float = 1.0,\n    activation_fn: Literal[\"sigmoid\", \"tanh\"] = \"sigmoid\",\n):\n    super().__init__()\n    self.beta = beta\n    self.gamma = gamma\n    self.activation_fn = activation_fn\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.FocalR_MSELoss.forward","title":"forward","text":"<pre><code>forward(input, target)\n</code></pre> <p>Parameters:</p> <ul> <li> <code>input</code>               (<code>Tensor</code>)           \u2013            <p>Input tensor with predictions (not probabilities)</p> </li> <li> <code>target</code>               (<code>Tensor</code>)           \u2013            <p>Target tensor with the actual classes</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt;\n&gt;&gt;&gt; from pytorch_widedeep.losses import FocalR_MSELoss\n&gt;&gt;&gt;\n&gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n&gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n&gt;&gt;&gt; loss = FocalR_MSELoss()(input, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(\n    self,\n    input: Tensor,\n    target: Tensor,\n) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    input: Tensor\n        Input tensor with predictions (not probabilities)\n    target: Tensor\n        Target tensor with the actual classes\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; from pytorch_widedeep.losses import FocalR_MSELoss\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n    &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n    &gt;&gt;&gt; loss = FocalR_MSELoss()(input, target)\n    \"\"\"\n    loss = (input - target) ** 2\n    if self.activation_fn == \"tanh\":\n        loss *= (torch.tanh(self.beta * torch.abs(input - target))) ** self.gamma\n    elif self.activation_fn == \"sigmoid\":\n        loss *= (\n            2 * torch.sigmoid(self.beta * torch.abs((input - target) ** 2)) - 1\n        ) ** self.gamma\n    else:\n        ValueError(\n            \"Incorrect activation function value - must be in ['sigmoid', 'tanh']\"\n        )\n    return torch.mean(loss)\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.FocalR_RMSELoss","title":"FocalR_RMSELoss","text":"<pre><code>FocalR_RMSELoss(\n    beta=0.2, gamma=1.0, activation_fn=\"sigmoid\"\n)\n</code></pre> <p>               Bases: <code>Module</code></p> <p>Focal-R RMSE loss</p> <p>Based on Delving into Deep Imbalanced Regression.</p> <p>Parameters:</p> <ul> <li> <code>beta</code>               (<code>float</code>, default:                   <code>0.2</code> )           \u2013            <p>Focal Loss <code>beta</code> parameter in their implementation</p> </li> <li> <code>gamma</code>               (<code>float</code>, default:                   <code>1.0</code> )           \u2013            <p>Focal Loss <code>gamma</code> parameter</p> </li> <li> <code>activation_fn</code>               (<code>Literal[sigmoid, tanh]</code>, default:                   <code>'sigmoid'</code> )           \u2013            <p>Activation function to be used during the computation of the loss. Possible values are 'sigmoid' and 'tanh'. See the original publication for details.</p> </li> </ul> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def __init__(\n    self,\n    beta: float = 0.2,\n    gamma: float = 1.0,\n    activation_fn: Literal[\"sigmoid\", \"tanh\"] = \"sigmoid\",\n):\n    super().__init__()\n    self.beta = beta\n    self.gamma = gamma\n    self.activation_fn = activation_fn\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.FocalR_RMSELoss.forward","title":"forward","text":"<pre><code>forward(input, target)\n</code></pre> <p>Parameters:</p> <ul> <li> <code>input</code>               (<code>Tensor</code>)           \u2013            <p>Input tensor with predictions (not probabilities)</p> </li> <li> <code>target</code>               (<code>Tensor</code>)           \u2013            <p>Target tensor with the actual classes</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt;\n&gt;&gt;&gt; from pytorch_widedeep.losses import FocalR_RMSELoss\n&gt;&gt;&gt;\n&gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n&gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n&gt;&gt;&gt; loss = FocalR_RMSELoss()(input, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(\n    self,\n    input: Tensor,\n    target: Tensor,\n) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    input: Tensor\n        Input tensor with predictions (not probabilities)\n    target: Tensor\n        Target tensor with the actual classes\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; from pytorch_widedeep.losses import FocalR_RMSELoss\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n    &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n    &gt;&gt;&gt; loss = FocalR_RMSELoss()(input, target)\n    \"\"\"\n    loss = (input - target) ** 2\n    if self.activation_fn == \"tanh\":\n        loss *= (torch.tanh(self.beta * torch.abs(input - target))) ** self.gamma\n    elif self.activation_fn == \"sigmoid\":\n        loss *= (\n            2 * torch.sigmoid(self.beta * torch.abs((input - target) ** 2)) - 1\n        ) ** self.gamma\n    else:\n        ValueError(\n            \"Incorrect activation function value - must be in ['sigmoid', 'tanh']\"\n        )\n    return torch.sqrt(torch.mean(loss))\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.HuberLoss","title":"HuberLoss","text":"<pre><code>HuberLoss(beta=0.2)\n</code></pre> <p>               Bases: <code>Module</code></p> <p>Hubbler Loss</p> <p>Based on Delving into Deep Imbalanced Regression.</p> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def __init__(self, beta: float = 0.2):\n    super().__init__()\n    self.beta = beta\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.HuberLoss.forward","title":"forward","text":"<pre><code>forward(input, target)\n</code></pre> <p>Parameters:</p> <ul> <li> <code>input</code>               (<code>Tensor</code>)           \u2013            <p>Input tensor with predictions (not probabilities)</p> </li> <li> <code>target</code>               (<code>Tensor</code>)           \u2013            <p>Target tensor with the actual classes</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt;\n&gt;&gt;&gt; from pytorch_widedeep.losses import HuberLoss\n&gt;&gt;&gt;\n&gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n&gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n&gt;&gt;&gt; loss = HuberLoss()(input, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(\n    self,\n    input: Tensor,\n    target: Tensor,\n) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    input: Tensor\n        Input tensor with predictions (not probabilities)\n    target: Tensor\n        Target tensor with the actual classes\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; from pytorch_widedeep.losses import HuberLoss\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n    &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n    &gt;&gt;&gt; loss = HuberLoss()(input, target)\n    \"\"\"\n    l1_loss = torch.abs(input - target)\n    cond = l1_loss &lt; self.beta\n    loss = torch.where(\n        cond, 0.5 * l1_loss**2 / self.beta, l1_loss - 0.5 * self.beta\n    )\n    return torch.mean(loss)\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.InfoNCELoss","title":"InfoNCELoss","text":"<pre><code>InfoNCELoss(temperature=0.1, reduction='mean')\n</code></pre> <p>               Bases: <code>Module</code></p> <p>InfoNCE Loss. Loss applied during the Contrastive Denoising Self Supervised Pre-training routine available in this library</p> <p> NOTE: This loss is in principle not exposed to  the user, as it is used internally in the library, but it is included  here for completion.</p> <p>See SAINT: Improved Neural Networks for Tabular Data via Row Attention and Contrastive Pre-Training and references therein</p> <p>Partially inspired by the code in this repo</p> <p>Parameters:</p> <ul> <li> <code>temperature</code>               (<code>float</code>, default:                   <code>0.1</code> )           \u2013            <p>The logits are divided by the temperature before computing the loss value</p> </li> <li> <code>reduction</code>               (<code>str</code>, default:                   <code>'mean'</code> )           \u2013            <p>Loss reduction method</p> </li> </ul> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def __init__(self, temperature: float = 0.1, reduction: str = \"mean\"):\n    super(InfoNCELoss, self).__init__()\n\n    self.temperature = temperature\n    self.reduction = reduction\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.InfoNCELoss.forward","title":"forward","text":"<pre><code>forward(g_projs)\n</code></pre> <p>Parameters:</p> <ul> <li> <code>g_projs</code>               (<code>Tuple[Tensor, Tensor]</code>)           \u2013            <p>Tuple with the two tensors corresponding to the output of the two projection heads, as described 'SAINT: Improved Neural Networks for Tabular Data via Row Attention and Contrastive Pre-Training'.</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.losses import InfoNCELoss\n&gt;&gt;&gt; g_projs = (torch.rand(3, 5, 16), torch.rand(3, 5, 16))\n&gt;&gt;&gt; loss = InfoNCELoss()\n&gt;&gt;&gt; res = loss(g_projs)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(self, g_projs: Tuple[Tensor, Tensor]) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    g_projs: Tuple\n        Tuple with the two tensors corresponding to the output of the two\n        projection heads, as described 'SAINT: Improved Neural Networks\n        for Tabular Data via Row Attention and Contrastive Pre-Training'.\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.losses import InfoNCELoss\n    &gt;&gt;&gt; g_projs = (torch.rand(3, 5, 16), torch.rand(3, 5, 16))\n    &gt;&gt;&gt; loss = InfoNCELoss()\n    &gt;&gt;&gt; res = loss(g_projs)\n    \"\"\"\n    z, z_ = g_projs[0], g_projs[1]\n\n    norm_z = F.normalize(z, dim=-1).flatten(1)\n    norm_z_ = F.normalize(z_, dim=-1).flatten(1)\n\n    logits = (norm_z @ norm_z_.t()) / self.temperature\n    logits_ = (norm_z_ @ norm_z.t()) / self.temperature\n\n    # the target/labels are the entries on the diagonal\n    target = torch.arange(len(norm_z), device=norm_z.device)\n\n    loss = F.cross_entropy(logits, target, reduction=self.reduction)\n    loss_ = F.cross_entropy(logits_, target, reduction=self.reduction)\n\n    return (loss + loss_) / 2.0\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.DenoisingLoss","title":"DenoisingLoss","text":"<pre><code>DenoisingLoss(\n    lambda_cat=1.0, lambda_cont=1.0, reduction=\"mean\"\n)\n</code></pre> <p>               Bases: <code>Module</code></p> <p>Denoising Loss. Loss applied during the Contrastive Denoising Self Supervised Pre-training routine available in this library</p> <p> NOTE: This loss is in principle not exposed to  the user, as it is used internally in the library, but it is included  here for completion.</p> <p>See SAINT: Improved Neural Networks for Tabular Data via Row Attention and Contrastive Pre-Training and references therein</p> <p>Parameters:</p> <ul> <li> <code>lambda_cat</code>               (<code>float</code>, default:                   <code>1.0</code> )           \u2013            <p>Multiplicative factor that will be applied to loss associated to the categorical features</p> </li> <li> <code>lambda_cont</code>               (<code>float</code>, default:                   <code>1.0</code> )           \u2013            <p>Multiplicative factor that will be applied to loss associated to the continuous features</p> </li> <li> <code>reduction</code>               (<code>str</code>, default:                   <code>'mean'</code> )           \u2013            <p>Loss reduction method</p> </li> </ul> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def __init__(\n    self, lambda_cat: float = 1.0, lambda_cont: float = 1.0, reduction: str = \"mean\"\n):\n    super(DenoisingLoss, self).__init__()\n\n    self.lambda_cat = lambda_cat\n    self.lambda_cont = lambda_cont\n    self.reduction = reduction\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.DenoisingLoss.forward","title":"forward","text":"<pre><code>forward(x_cat_and_cat_, x_cont_and_cont_)\n</code></pre> <p>Parameters:</p> <ul> <li> <code>x_cat_and_cat_</code>               (<code>Optional[Union[List[Tuple[Tensor, Tensor]], Tuple[Tensor, Tensor]]]</code>)           \u2013            <p>Tuple of tensors containing the raw input features and their encodings, referred in the SAINT paper as \\(x\\) and \\(x''\\) respectively. If one denoising MLP is used per categorical feature <code>x_cat_and_cat_</code> will be a list of tuples, one per categorical feature</p> </li> <li> <code>x_cont_and_cont_</code>               (<code>Optional[Union[List[Tuple[Tensor, Tensor]], Tuple[Tensor, Tensor]]]</code>)           \u2013            <p>same as <code>x_cat_and_cat_</code> but for continuous columns</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.losses import DenoisingLoss\n&gt;&gt;&gt; x_cat_and_cat_ = (torch.empty(3).random_(3).long(), torch.randn(3, 3))\n&gt;&gt;&gt; x_cont_and_cont_ = (torch.randn(3, 1), torch.randn(3, 1))\n&gt;&gt;&gt; loss = DenoisingLoss()\n&gt;&gt;&gt; res = loss(x_cat_and_cat_, x_cont_and_cont_)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(\n    self,\n    x_cat_and_cat_: Optional[\n        Union[List[Tuple[Tensor, Tensor]], Tuple[Tensor, Tensor]]\n    ],\n    x_cont_and_cont_: Optional[\n        Union[List[Tuple[Tensor, Tensor]], Tuple[Tensor, Tensor]]\n    ],\n) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    x_cat_and_cat_: tuple of Tensors or lists of tuples\n        Tuple of tensors containing the raw input features and their\n        encodings, referred in the SAINT paper as $x$ and $x''$\n        respectively. If one denoising MLP is used per categorical\n        feature `x_cat_and_cat_` will be a list of tuples, one per\n        categorical feature\n    x_cont_and_cont_: tuple of Tensors or lists of tuples\n        same as `x_cat_and_cat_` but for continuous columns\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.losses import DenoisingLoss\n    &gt;&gt;&gt; x_cat_and_cat_ = (torch.empty(3).random_(3).long(), torch.randn(3, 3))\n    &gt;&gt;&gt; x_cont_and_cont_ = (torch.randn(3, 1), torch.randn(3, 1))\n    &gt;&gt;&gt; loss = DenoisingLoss()\n    &gt;&gt;&gt; res = loss(x_cat_and_cat_, x_cont_and_cont_)\n    \"\"\"\n\n    loss_cat = (\n        self._compute_cat_loss(x_cat_and_cat_)\n        if x_cat_and_cat_ is not None\n        else torch.tensor(0.0)\n    )\n    loss_cont = (\n        self._compute_cont_loss(x_cont_and_cont_)\n        if x_cont_and_cont_ is not None\n        else torch.tensor(0.0)\n    )\n\n    return self.lambda_cat * loss_cat + self.lambda_cont * loss_cont\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.EncoderDecoderLoss","title":"EncoderDecoderLoss","text":"<pre><code>EncoderDecoderLoss(eps=1e-09)\n</code></pre> <p>               Bases: <code>Module</code></p> <p>'Standard' Encoder Decoder Loss. Loss applied during the Endoder-Decoder  Self-Supervised Pre-Training routine available in this library</p> <p> NOTE: This loss is in principle not exposed to  the user, as it is used internally in the library, but it is included  here for completion.</p> <p>The implementation of this lost is based on that at the tabnet repo, which is in itself an adaptation of that in the original paper TabNet: Attentive Interpretable Tabular Learning.</p> <p>Parameters:</p> <ul> <li> <code>eps</code>               (<code>float</code>, default:                   <code>1e-09</code> )           \u2013            <p>Simply a small number to avoid dividing by zero</p> </li> </ul> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def __init__(self, eps: float = 1e-9):\n    super(EncoderDecoderLoss, self).__init__()\n    self.eps = eps\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.EncoderDecoderLoss.forward","title":"forward","text":"<pre><code>forward(x_true, x_pred, mask)\n</code></pre> <p>Parameters:</p> <ul> <li> <code>x_true</code>               (<code>Tensor</code>)           \u2013            <p>Embeddings of the input data</p> </li> <li> <code>x_pred</code>               (<code>Tensor</code>)           \u2013            <p>Reconstructed embeddings</p> </li> <li> <code>mask</code>               (<code>Tensor</code>)           \u2013            <p>Mask with 1s indicated that the reconstruction, and therefore the loss, is based on those features.</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.losses import EncoderDecoderLoss\n&gt;&gt;&gt; x_true = torch.rand(3, 3)\n&gt;&gt;&gt; x_pred = torch.rand(3, 3)\n&gt;&gt;&gt; mask = torch.empty(3, 3).random_(2)\n&gt;&gt;&gt; loss = EncoderDecoderLoss()\n&gt;&gt;&gt; res = loss(x_true, x_pred, mask)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(self, x_true: Tensor, x_pred: Tensor, mask: Tensor) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    x_true: Tensor\n        Embeddings of the input data\n    x_pred: Tensor\n        Reconstructed embeddings\n    mask: Tensor\n        Mask with 1s indicated that the reconstruction, and therefore the\n        loss, is based on those features.\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.losses import EncoderDecoderLoss\n    &gt;&gt;&gt; x_true = torch.rand(3, 3)\n    &gt;&gt;&gt; x_pred = torch.rand(3, 3)\n    &gt;&gt;&gt; mask = torch.empty(3, 3).random_(2)\n    &gt;&gt;&gt; loss = EncoderDecoderLoss()\n    &gt;&gt;&gt; res = loss(x_true, x_pred, mask)\n    \"\"\"\n\n    errors = x_pred - x_true\n\n    reconstruction_errors = torch.mul(errors, mask) ** 2\n\n    x_true_means = torch.mean(x_true, dim=0)\n    x_true_means[x_true_means == 0] = 1\n\n    x_true_stds = torch.std(x_true, dim=0) ** 2\n    x_true_stds[x_true_stds == 0] = x_true_means[x_true_stds == 0]\n\n    features_loss = torch.matmul(reconstruction_errors, 1 / x_true_stds)\n    nb_reconstructed_variables = torch.sum(mask, dim=1)\n    features_loss_norm = features_loss / (nb_reconstructed_variables + self.eps)\n\n    loss = torch.mean(features_loss_norm)\n\n    return loss\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses_multitarget.MultiTargetRegressionLoss","title":"MultiTargetRegressionLoss","text":"<pre><code>MultiTargetRegressionLoss(weights=None, reduction='mean')\n</code></pre> <p>               Bases: <code>Module</code></p> <p>This class is a wrapper around the Pytorch MSELoss. It allows for multi-target regression problems. The user can provide a list of weights to apply to each target. The loss can be either the sum or the mean of the individual losses</p> <p>Parameters:</p> <ul> <li> <code>weights</code>               (<code>Optional[List[float]]</code>, default:                   <code>None</code> )           \u2013            <p>List of weights to apply to the loss associated to each target. The length of the list must match the number of targets. Alias: 'target_weights'</p> </li> <li> <code>reduction</code>               (<code>Literal[mean, sum]</code>, default:                   <code>'mean'</code> )           \u2013            <p>Specifies the reduction to apply to the loss associated to each target: 'mean' | 'sum'. Note that this is NOT the same as the reduction in the MSELoss. This reduction is applied after the loss for each target has been computed. Alias: 'target_reduction'</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.losses_multitarget import MultiTargetRegressionLoss\n&gt;&gt;&gt; input = torch.randn(3, 2)\n&gt;&gt;&gt; target = torch.randn(3, 2)\n&gt;&gt;&gt; loss = MultiTargetRegressionLoss(weights=[0.5, 0.5], reduction=\"mean\")\n&gt;&gt;&gt; output = loss(input, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses_multitarget.py</code> <pre><code>@alias(\"reduction\", [\"target_reduction\"])\n@alias(\"weights\", [\"target_weights\"])\ndef __init__(\n    self,\n    weights: Optional[List[float]] = None,\n    reduction: Literal[\"mean\", \"sum\"] = \"mean\",\n):\n    super(MultiTargetRegressionLoss, self).__init__()\n\n    self.weights = weights\n    self.reduction = reduction\n\n    if self.reduction not in [\"mean\", \"sum\"]:\n        raise ValueError(\"reduction must be either 'mean' or 'sum'\")\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses_multitarget.MultiTargetClassificationLoss","title":"MultiTargetClassificationLoss","text":"<pre><code>MultiTargetClassificationLoss(\n    binary_config=None,\n    multiclass_config=None,\n    weights=None,\n    reduction=\"mean\",\n    binary_trick=False,\n)\n</code></pre> <p>               Bases: <code>Module</code></p> <p>This class is a wrapper around the Pytorch binary_cross_entropy_with_logits and cross_entropy losses. It allows for multi-target classification problems. The user can provide a list of weights to apply to each target. The loss can be either the sum or the mean of the individual losses</p> <p>Parameters:</p> <ul> <li> <code>binary_config</code>               (<code>Optional[List[Union[int, Tuple[int, float]]]]</code>, default:                   <code>None</code> )           \u2013            <p>List of integers with the index of the target for binary classification or tuples with two elements: the index of the targets or binary classification and the positive weight for binary classification</p> </li> <li> <code>multiclass_config</code>               (<code>Optional[List[Union[Tuple[int, int], Tuple[int, int, List[float]]]]]</code>, default:                   <code>None</code> )           \u2013            <p>List of tuples with two or three elements: the index of the target and the number of classes for multiclass classification, or a tuple with the index of the target, the number of classes and a list of weights to apply to each class (i.e. the 'weight' parameter in the cross_entropy loss)</p> </li> <li> <code>weights</code>               (<code>Optional[List[float]]</code>, default:                   <code>None</code> )           \u2013            <p>List of weights to apply to the loss associated to each target. The length of the list must match the number of targets. Alias: 'target_weights'</p> </li> <li> <code>reduction</code>               (<code>Literal[mean, sum]</code>, default:                   <code>'mean'</code> )           \u2013            <p>Specifies the reduction to apply to the loss associated to each target: 'mean' | 'sum'. Note that this is NOT the same as the reduction in the cross_entropy loss or the binary_cross_entropy_with_logits. This reduction is applied after the loss for each target has been computed. Alias: 'target_reduction'</p> </li> <li> <code>binary_trick</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>If True, each target will be considered independently and the loss will be computed as binary_cross_entropy_with_logits. This is a faster implementation. Note that the 'weights' parameter is not compatible with binary_trick=True. Also note that if binary_trick=True, the 'binary_config' must be a list of integers and the 'multiclass_config' must be a list of tuples with two integers: the index of the target and the number of classes. Finally, if binary_trick=True, the binary targets must be the first targets in the target tensor.</p> <p> NOTE: When using the binary_trick, the binary targets are   considered as 2 classes. Therefore, the pred_dim parametere of the   WideDeep class should be adjusted accordingly (adding 2 to per   binary target). For example, in a problem with a binary target and   a 4 class multiclassification target, the pred_dim should be 6.</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.losses_multitarget import MultiTargetClassificationLoss\n&gt;&gt;&gt; input = torch.randn(5, 4)\n&gt;&gt;&gt; input_binary_trick = torch.randn(5, 5)\n&gt;&gt;&gt; target = torch.stack([torch.tensor([0, 1, 0, 1, 1]), torch.tensor([0, 1, 2, 0, 2])], 1)\n&gt;&gt;&gt; loss_1 = MultiTargetClassificationLoss(binary_config=[0], multiclass_config=[(1, 3)], reduction=\"mean\")\n&gt;&gt;&gt; output_1 = loss_1(input, target)\n&gt;&gt;&gt; loss_2 = MultiTargetClassificationLoss(binary_config=[(0, 0.5)], multiclass_config=[(1, 3, [1., 2., 3.])],\n... reduction=\"sum\", weights=[0.5, 0.5])\n&gt;&gt;&gt; output_2 = loss_2(input, target)\n&gt;&gt;&gt; loss_3 = MultiTargetClassificationLoss(binary_config=[0], multiclass_config=[(1, 3)], binary_trick=True)\n&gt;&gt;&gt; output_3 = loss_3(input_binary_trick, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses_multitarget.py</code> <pre><code>@alias(\"reduction\", [\"target_reduction\"])\n@alias(\"weights\", [\"target_weights\"])\ndef __init__(  # noqa: C901\n    self,\n    binary_config: Optional[List[Union[int, Tuple[int, float]]]] = None,\n    multiclass_config: Optional[\n        List[Union[Tuple[int, int], Tuple[int, int, List[float]]]]\n    ] = None,\n    weights: Optional[List[float]] = None,\n    reduction: Literal[\"mean\", \"sum\"] = \"mean\",\n    binary_trick: bool = False,\n):\n    super(MultiTargetClassificationLoss, self).__init__()\n\n    if reduction not in [\"mean\", \"sum\"]:\n        raise ValueError(\"reduction must be either 'mean' or 'sum'\")\n\n    self.binary_config = binary_config\n    self.multiclass_config = multiclass_config\n    self.weights = weights\n    self.reduction = reduction\n    self.binary_trick = binary_trick\n\n    if self.weights is not None:\n        if len(self.weights) != (\n            len(self.binary_config) if self.binary_config is not None else 0\n        ) + (\n            len(self.multiclass_config) if self.multiclass_config is not None else 0\n        ):\n            raise ValueError(\n                \"The number of weights must match the number of binary and multiclass targets\"\n            )\n\n    if self.binary_trick:\n        self._check_inputs_with_binary_trick()\n        self._binary_config: List[int] = binary_config  # type: ignore[assignment]\n        self._multiclass_config: List[Tuple[int, int]] = self.multiclass_config  # type: ignore[assignment]\n    else:\n        self.binary_config_with_pos_weights = (\n            (self._set_binary_config_without_binary_trick())\n            if self.binary_config is not None\n            else None\n        )\n        self.multiclass_config_with_weights = (\n            (self._set_multiclass_config_without_binary_trick())\n            if self.multiclass_config is not None\n            else None\n        )\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses_multitarget.MutilTargetRegressionAndClassificationLoss","title":"MutilTargetRegressionAndClassificationLoss","text":"<pre><code>MutilTargetRegressionAndClassificationLoss(\n    regression_config=[],\n    binary_config=None,\n    multiclass_config=None,\n    weights=None,\n    reduction=\"mean\",\n    binary_trick=False,\n)\n</code></pre> <p>               Bases: <code>Module</code></p> <p>This class is a wrapper around the MultiTargetRegressionLoss and the MultiTargetClassificationLoss. It allows for multi-target regression and classification problems. The user can provide a list of weights to apply to each target. The loss can be either the sum or the mean of the individual losses</p> <p>Parameters:</p> <ul> <li> <code>regression_config</code>               (<code>List[int]</code>, default:                   <code>[]</code> )           \u2013            <p>List of integers with the indices of the regression targets</p> </li> <li> <code>binary_config</code>               (<code>Optional[List[Union[int, Tuple[int, float]]]]</code>, default:                   <code>None</code> )           \u2013            <p>List of integers with the index of the target for binary classification or tuples with two elements: the index of the targets or binary classification and the positive weight for binary classification</p> </li> <li> <code>multiclass_config</code>               (<code>Optional[List[Union[Tuple[int, int], Tuple[int, int, List[float]]]]]</code>, default:                   <code>None</code> )           \u2013            <p>List of tuples with two or three elements: the index of the target and the number of classes for multiclass classification, or a tuple with the index of the target, the number of classes and a list of weights to apply to each class (i.e. the 'weight' parameter in the cross_entropy loss)</p> </li> <li> <code>weights</code>               (<code>Optional[List[float]]</code>, default:                   <code>None</code> )           \u2013            <p>List of weights to apply to the loss associated to each target. The length of the list must match the number of targets. Alias: 'target_weights'</p> </li> <li> <code>reduction</code>               (<code>Literal[mean, sum]</code>, default:                   <code>'mean'</code> )           \u2013            <p>Specifies the reduction to apply to the output: 'mean' | 'sum'. Note that this is NOT the same as the reduction in the cross_entropy loss, the binary_cross_entropy_with_logits or the MSELoss. This reduction is applied after each target has been computed. Alias: 'target_reduction'</p> </li> <li> <code>binary_trick</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>If True, each target will be considered independently and the loss will be computed as binary_cross_entropy_with_logits. This is a faster implementation. Note that the 'weights' parameter is not compatible with binary_trick=True. Also note that if binary_trick=True, the 'binary_config' must be a list of integers and the 'multiclass_config' must be a list of tuples with two integers: the index of the target and the number of classes. Finally, if binary_trick=True, the binary targets must be the first targets in the target tensor.</p> <p> NOTE: When using the binary_trick, the binary targets are   considered as 2 classes. Therefore, the pred_dim parametere of the   WideDeep class should be adjusted accordingly (adding 2 to per   binary target). For example, in a problem with a binary target and   a 4 class multiclassification target, the pred_dim should be 6.</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.losses_multitarget import MutilTargetRegressionAndClassificationLoss\n&gt;&gt;&gt; input = torch.randn(5, 5)\n&gt;&gt;&gt; target = torch.stack([torch.randn(5), torch.tensor([0, 1, 0, 1, 1]), torch.tensor([0, 1, 2, 0, 2])], 1)\n&gt;&gt;&gt; loss = MutilTargetRegressionAndClassificationLoss(regression_config=[0], binary_config=[2],\n... multiclass_config=[(2, 3)], reduction=\"mean\")\n&gt;&gt;&gt; output = loss(input, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses_multitarget.py</code> <pre><code>@alias(\"reduction\", [\"target_reduction\"])\n@alias(\"weights\", [\"target_weights\"])\ndef __init__(  # noqa: C901\n    self,\n    regression_config: List[int] = [],\n    binary_config: Optional[List[Union[int, Tuple[int, float]]]] = None,\n    multiclass_config: Optional[\n        List[Union[Tuple[int, int], Tuple[int, int, List[float]]]]\n    ] = None,\n    weights: Optional[List[float]] = None,\n    reduction: Literal[\"mean\", \"sum\"] = \"mean\",\n    binary_trick: bool = False,\n):\n\n    super(MutilTargetRegressionAndClassificationLoss, self).__init__()\n\n    self.regression_config = regression_config\n\n    assert binary_config is not None or multiclass_config is not None, (\n        \"Either binary_config or multiclass_config must be provided. \"\n        \"Otherwise, use the MultiTargetRegressionLoss\"\n    )\n\n    if binary_trick:\n        self._check_inputs_with_binary_trick(\n            regression_config, binary_config, multiclass_config\n        )\n\n    if weights is not None:\n        if len(weights) != (\n            len(regression_config)\n            + (len(binary_config) if binary_config is not None else 0)\n            + (len(multiclass_config) if multiclass_config is not None else 0)\n        ):\n            raise ValueError(\n                \"The number of weights must match the number of regression, binary and multiclass targets\"\n            )\n\n        self.weights_regression = self._prepare_weights_for_regression_targets(\n            weights, regression_config\n        )\n        self.weights_binary = self._prepare_weights_per_binary_targets(\n            weights, binary_config\n        )\n        self.weights_multiclass = self._prepare_weights_per_multiclass_targets(\n            weights, multiclass_config\n        )\n        self.weights = weights\n    else:\n        self.weights_regression = None\n        self.weights_binary = None\n        self.weights_multiclass = None\n\n    self.multi_target_regression_loss = MultiTargetRegressionLoss(\n        weights=self.weights_regression, reduction=reduction\n    )\n\n    self.multi_target_classification_loss = MultiTargetClassificationLoss(\n        binary_config=binary_config,\n        multiclass_config=multiclass_config,\n        weights=(\n            self.weights_binary + self.weights_multiclass\n            if self.weights_binary is not None\n            and self.weights_multiclass is not None\n            else (\n                self.weights_binary\n                if self.weights_binary is not None\n                else self.weights_multiclass\n            )\n        ),\n        reduction=reduction,\n        binary_trick=binary_trick,\n    )\n</code></pre>"},{"location":"pytorch-widedeep/metrics.html","title":"Metrics","text":"<p> NOTE: metrics in this module expect the predictions  and ground truth to have the same dimensions for regression and binary  classification problems: \\((N_{samples}, 1)\\). In the case of multiclass  classification problems the ground truth is expected to be a 1D tensor with  the corresponding classes. See Examples below</p> <p>We have added the possibility of using the metrics available at the torchmetrics library. Note that this library is still in its early versions and therefore this option should be used with caution. To use <code>torchmetrics</code> simply import them and use them as any of the <code>pytorch-widedeep</code> metrics described below.</p> <pre><code>from torchmetrics import Accuracy, Precision\n\naccuracy = Accuracy(average=None, num_classes=2)\nprecision = Precision(average='micro', num_classes=2)\n\ntrainer = Trainer(model, objective=\"binary\", metrics=[accuracy, precision])\n</code></pre> <p>A functioning example for <code>pytorch-widedeep</code> using <code>torchmetrics</code> can be found in the Examples folder</p> <p> NOTE: the forward method for all metrics in this  module takes two tensors, <code>y_pred</code> and <code>y_true</code> (in that order). Therefore,  we do not include the method in the documentation.</p>"},{"location":"pytorch-widedeep/metrics.html#pytorch_widedeep.metrics.Accuracy","title":"Accuracy","text":"<pre><code>Accuracy(top_k=1)\n</code></pre> <p>               Bases: <code>Metric</code></p> <p>Class to calculate the accuracy for both binary and categorical problems</p> <p>Parameters:</p> <ul> <li> <code>top_k</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>Accuracy will be computed using the top k most likely classes in multiclass problems</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt;\n&gt;&gt;&gt; from pytorch_widedeep.metrics import Accuracy\n&gt;&gt;&gt;\n&gt;&gt;&gt; acc = Accuracy()\n&gt;&gt;&gt; y_true = torch.tensor([0, 1, 0, 1]).view(-1, 1)\n&gt;&gt;&gt; y_pred = torch.tensor([[0.3, 0.2, 0.6, 0.7]]).view(-1, 1)\n&gt;&gt;&gt; acc(y_pred, y_true)\narray(0.5)\n&gt;&gt;&gt;\n&gt;&gt;&gt; acc = Accuracy(top_k=2)\n&gt;&gt;&gt; y_true = torch.tensor([0, 1, 2])\n&gt;&gt;&gt; y_pred = torch.tensor([[0.3, 0.5, 0.2], [0.1, 0.1, 0.8], [0.1, 0.5, 0.4]])\n&gt;&gt;&gt; acc(y_pred, y_true)\narray(0.66666667)\n</code></pre> Source code in <code>pytorch_widedeep/metrics.py</code> <pre><code>def __init__(self, top_k: int = 1):\n    super(Accuracy, self).__init__()\n\n    self.top_k = top_k\n    self.correct_count = 0\n    self.total_count = 0\n    self._name = \"acc\"\n</code></pre>"},{"location":"pytorch-widedeep/metrics.html#pytorch_widedeep.metrics.Accuracy.reset","title":"reset","text":"<pre><code>reset()\n</code></pre> <p>resets counters to 0</p> Source code in <code>pytorch_widedeep/metrics.py</code> <pre><code>def reset(self):\n    \"\"\"\n    resets counters to 0\n    \"\"\"\n    self.correct_count = 0\n    self.total_count = 0\n</code></pre>"},{"location":"pytorch-widedeep/metrics.html#pytorch_widedeep.metrics.Precision","title":"Precision","text":"<pre><code>Precision(average=True)\n</code></pre> <p>               Bases: <code>Metric</code></p> <p>Class to calculate the precision for both binary and categorical problems</p> <p>Parameters:</p> <ul> <li> <code>average</code>               (<code>bool</code>, default:                   <code>True</code> )           \u2013            <p>This applies only to multiclass problems. if <code>True</code> calculate precision for each label, and finds their unweighted mean.</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt;\n&gt;&gt;&gt; from pytorch_widedeep.metrics import Precision\n&gt;&gt;&gt;\n&gt;&gt;&gt; prec = Precision()\n&gt;&gt;&gt; y_true = torch.tensor([0, 1, 0, 1]).view(-1, 1)\n&gt;&gt;&gt; y_pred = torch.tensor([[0.3, 0.2, 0.6, 0.7]]).view(-1, 1)\n&gt;&gt;&gt; prec(y_pred, y_true)\narray(0.5)\n&gt;&gt;&gt;\n&gt;&gt;&gt; prec = Precision(average=True)\n&gt;&gt;&gt; y_true = torch.tensor([0, 1, 2])\n&gt;&gt;&gt; y_pred = torch.tensor([[0.7, 0.1, 0.2], [0.1, 0.1, 0.8], [0.1, 0.5, 0.4]])\n&gt;&gt;&gt; prec(y_pred, y_true)\narray(0.33333334)\n</code></pre> Source code in <code>pytorch_widedeep/metrics.py</code> <pre><code>def __init__(self, average: bool = True):\n    super(Precision, self).__init__()\n\n    self.average = average\n    self.true_positives = 0\n    self.all_positives = 0\n    self.eps = 1e-20\n    self._name = \"prec\"\n</code></pre>"},{"location":"pytorch-widedeep/metrics.html#pytorch_widedeep.metrics.Precision.reset","title":"reset","text":"<pre><code>reset()\n</code></pre> <p>resets counters to 0</p> Source code in <code>pytorch_widedeep/metrics.py</code> <pre><code>def reset(self):\n    \"\"\"\n    resets counters to 0\n    \"\"\"\n    self.true_positives = 0\n    self.all_positives = 0\n</code></pre>"},{"location":"pytorch-widedeep/metrics.html#pytorch_widedeep.metrics.Recall","title":"Recall","text":"<pre><code>Recall(average=True)\n</code></pre> <p>               Bases: <code>Metric</code></p> <p>Class to calculate the recall for both binary and categorical problems</p> <p>Parameters:</p> <ul> <li> <code>average</code>               (<code>bool</code>, default:                   <code>True</code> )           \u2013            <p>This applies only to multiclass problems. if <code>True</code> calculate recall for each label, and finds their unweighted mean.</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt;\n&gt;&gt;&gt; from pytorch_widedeep.metrics import Recall\n&gt;&gt;&gt;\n&gt;&gt;&gt; rec = Recall()\n&gt;&gt;&gt; y_true = torch.tensor([0, 1, 0, 1]).view(-1, 1)\n&gt;&gt;&gt; y_pred = torch.tensor([[0.3, 0.2, 0.6, 0.7]]).view(-1, 1)\n&gt;&gt;&gt; rec(y_pred, y_true)\narray(0.5)\n&gt;&gt;&gt;\n&gt;&gt;&gt; rec = Recall(average=True)\n&gt;&gt;&gt; y_true = torch.tensor([0, 1, 2])\n&gt;&gt;&gt; y_pred = torch.tensor([[0.7, 0.1, 0.2], [0.1, 0.1, 0.8], [0.1, 0.5, 0.4]])\n&gt;&gt;&gt; rec(y_pred, y_true)\narray(0.33333334)\n</code></pre> Source code in <code>pytorch_widedeep/metrics.py</code> <pre><code>def __init__(self, average: bool = True):\n    super(Recall, self).__init__()\n\n    self.average = average\n    self.true_positives = 0\n    self.actual_positives = 0\n    self.eps = 1e-20\n    self._name = \"rec\"\n</code></pre>"},{"location":"pytorch-widedeep/metrics.html#pytorch_widedeep.metrics.Recall.reset","title":"reset","text":"<pre><code>reset()\n</code></pre> <p>resets counters to 0</p> Source code in <code>pytorch_widedeep/metrics.py</code> <pre><code>def reset(self):\n    \"\"\"\n    resets counters to 0\n    \"\"\"\n    self.true_positives = 0\n    self.actual_positives = 0\n</code></pre>"},{"location":"pytorch-widedeep/metrics.html#pytorch_widedeep.metrics.FBetaScore","title":"FBetaScore","text":"<pre><code>FBetaScore(beta, average=True)\n</code></pre> <p>               Bases: <code>Metric</code></p> <p>Class to calculate the fbeta score for both binary and categorical problems</p> \\[ F_{\\beta} = ((1 + {\\beta}^2) * \\frac{(precision * recall)}{({\\beta}^2 * precision + recall)} \\] <p>Parameters:</p> <ul> <li> <code>beta</code>               (<code>int</code>)           \u2013            <p>Coefficient to control the balance between precision and recall</p> </li> <li> <code>average</code>               (<code>bool</code>, default:                   <code>True</code> )           \u2013            <p>This applies only to multiclass problems. if <code>True</code> calculate fbeta for each label, and find their unweighted mean.</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt;\n&gt;&gt;&gt; from pytorch_widedeep.metrics import FBetaScore\n&gt;&gt;&gt;\n&gt;&gt;&gt; fbeta = FBetaScore(beta=2)\n&gt;&gt;&gt; y_true = torch.tensor([0, 1, 0, 1]).view(-1, 1)\n&gt;&gt;&gt; y_pred = torch.tensor([[0.3, 0.2, 0.6, 0.7]]).view(-1, 1)\n&gt;&gt;&gt; fbeta(y_pred, y_true)\narray(0.5)\n&gt;&gt;&gt;\n&gt;&gt;&gt; fbeta = FBetaScore(beta=2)\n&gt;&gt;&gt; y_true = torch.tensor([0, 1, 2])\n&gt;&gt;&gt; y_pred = torch.tensor([[0.7, 0.1, 0.2], [0.1, 0.1, 0.8], [0.1, 0.5, 0.4]])\n&gt;&gt;&gt; fbeta(y_pred, y_true)\narray(0.33333334)\n</code></pre> Source code in <code>pytorch_widedeep/metrics.py</code> <pre><code>def __init__(self, beta: int, average: bool = True):\n    super(FBetaScore, self).__init__()\n\n    self.beta = beta\n    self.average = average\n    self.precision = Precision(average=False)\n    self.recall = Recall(average=False)\n    self.eps = 1e-20\n    self._name = \"\".join([\"f\", str(self.beta)])\n</code></pre>"},{"location":"pytorch-widedeep/metrics.html#pytorch_widedeep.metrics.FBetaScore.reset","title":"reset","text":"<pre><code>reset()\n</code></pre> <p>resets precision and recall</p> Source code in <code>pytorch_widedeep/metrics.py</code> <pre><code>def reset(self):\n    \"\"\"\n    resets precision and recall\n    \"\"\"\n    self.precision.reset()\n    self.recall.reset()\n</code></pre>"},{"location":"pytorch-widedeep/metrics.html#pytorch_widedeep.metrics.F1Score","title":"F1Score","text":"<pre><code>F1Score(average=True)\n</code></pre> <p>               Bases: <code>Metric</code></p> <p>Class to calculate the f1 score for both binary and categorical problems</p> <p>Parameters:</p> <ul> <li> <code>average</code>               (<code>bool</code>, default:                   <code>True</code> )           \u2013            <p>This applies only to multiclass problems. if <code>True</code> calculate f1 for each label, and find their unweighted mean.</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt;\n&gt;&gt;&gt; from pytorch_widedeep.metrics import F1Score\n&gt;&gt;&gt;\n&gt;&gt;&gt; f1 = F1Score()\n&gt;&gt;&gt; y_true = torch.tensor([0, 1, 0, 1]).view(-1, 1)\n&gt;&gt;&gt; y_pred = torch.tensor([[0.3, 0.2, 0.6, 0.7]]).view(-1, 1)\n&gt;&gt;&gt; f1(y_pred, y_true)\narray(0.5)\n&gt;&gt;&gt;\n&gt;&gt;&gt; f1 = F1Score()\n&gt;&gt;&gt; y_true = torch.tensor([0, 1, 2])\n&gt;&gt;&gt; y_pred = torch.tensor([[0.7, 0.1, 0.2], [0.1, 0.1, 0.8], [0.1, 0.5, 0.4]])\n&gt;&gt;&gt; f1(y_pred, y_true)\narray(0.33333334)\n</code></pre> Source code in <code>pytorch_widedeep/metrics.py</code> <pre><code>def __init__(self, average: bool = True):\n    super(F1Score, self).__init__()\n\n    self.average = average\n    self.f1 = FBetaScore(beta=1, average=self.average)\n    self._name = self.f1._name\n</code></pre>"},{"location":"pytorch-widedeep/metrics.html#pytorch_widedeep.metrics.F1Score.reset","title":"reset","text":"<pre><code>reset()\n</code></pre> <p>resets counters to 0</p> Source code in <code>pytorch_widedeep/metrics.py</code> <pre><code>def reset(self):\n    \"\"\"\n    resets counters to 0\n    \"\"\"\n    self.f1.reset()\n</code></pre>"},{"location":"pytorch-widedeep/metrics.html#pytorch_widedeep.metrics.R2Score","title":"R2Score","text":"<pre><code>R2Score()\n</code></pre> <p>               Bases: <code>Metric</code></p> <p>Calculates R-Squared, the coefficient of determination:</p> \\[ R^2 = 1 - \\frac{\\sum_{j=1}^n(y_j - \\hat{y_j})^2}{\\sum_{j=1}^n(y_j - \\bar{y})^2} \\] <p>where \\(\\hat{y_j}\\) is the ground truth, \\(y_j\\) is the predicted value and \\(\\bar{y}\\) is the mean of the ground truth.</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt;\n&gt;&gt;&gt; from pytorch_widedeep.metrics import R2Score\n&gt;&gt;&gt;\n&gt;&gt;&gt; r2 = R2Score()\n&gt;&gt;&gt; y_true = torch.tensor([3, -0.5, 2, 7]).view(-1, 1)\n&gt;&gt;&gt; y_pred = torch.tensor([2.5, 0.0, 2, 8]).view(-1, 1)\n&gt;&gt;&gt; r2(y_pred, y_true)\narray(0.94860814)\n</code></pre> Source code in <code>pytorch_widedeep/metrics.py</code> <pre><code>def __init__(self):\n    self.numerator = 0\n    self.denominator = 0\n    self.num_examples = 0\n    self.y_true_sum = 0\n\n    self._name = \"r2\"\n</code></pre>"},{"location":"pytorch-widedeep/metrics.html#pytorch_widedeep.metrics.R2Score.reset","title":"reset","text":"<pre><code>reset()\n</code></pre> <p>resets counters to 0</p> Source code in <code>pytorch_widedeep/metrics.py</code> <pre><code>def reset(self):\n    \"\"\"\n    resets counters to 0\n    \"\"\"\n    self.numerator = 0\n    self.denominator = 0\n    self.num_examples = 0\n    self.y_true_sum = 0\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html","title":"The <code>models</code> module","text":"<p>This module contains the models that can be used as the four main components that will comprise a Wide and Deep model (<code>wide</code>, <code>deeptabular</code>, <code>deeptext</code>, <code>deepimage</code>), as well as the <code>WideDeep</code> \"constructor\" class. Note that each of the four components can be used independently. It also contains all the documentation for the models that can be used for self-supervised pre-training with tabular data.</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.linear.wide.Wide","title":"Wide","text":"<pre><code>Wide(input_dim, pred_dim=1)\n</code></pre> <p>               Bases: <code>Module</code></p> <p>Defines a <code>Wide</code> (linear) model where the non-linearities are captured via the so-called crossed-columns. This can be used as the <code>wide</code> component of a Wide &amp; Deep model.</p> <p>Parameters:</p> <ul> <li> <code>input_dim</code>               (<code>int</code>)           \u2013            <p>size of the Linear layer (implemented via an Embedding layer). <code>input_dim</code> is the summation of all the individual values for all the features that go through the wide model. For example, if the wide model receives 2 features with 5 individual values each, <code>input_dim = 10</code></p> </li> <li> <code>pred_dim</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>size of the ouput tensor containing the predictions. Note that unlike all the other models, the wide model is connected directly to the output neuron(s) when used to build a Wide and Deep model. Therefore, it requires the <code>pred_dim</code> parameter.</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>wide_linear</code>               (<code>Module</code>)           \u2013            <p>the linear layer that comprises the wide branch of the model</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import Wide\n&gt;&gt;&gt; X = torch.empty(4, 4).random_(20)\n&gt;&gt;&gt; wide = Wide(input_dim=int(X.max().item()), pred_dim=1)\n&gt;&gt;&gt; out = wide(X)\n</code></pre> Source code in <code>pytorch_widedeep/models/tabular/linear/wide.py</code> <pre><code>@alias(\"pred_dim\", [\"pred_size\", \"num_class\"])\ndef __init__(self, input_dim: int, pred_dim: int = 1):\n    super(Wide, self).__init__()\n\n    self.input_dim = input_dim\n    self.pred_dim = pred_dim\n\n    # Embeddings: val + 1 because 0 is reserved for padding/unseen cateogories.\n    self.wide_linear = nn.Embedding(input_dim + 1, pred_dim, padding_idx=0)\n    # (Sum(Embedding) + bias) is equivalent to (OneHotVector + Linear)\n    self.bias = nn.Parameter(torch.zeros(pred_dim))\n    self._reset_parameters()\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.linear.wide.Wide.forward","title":"forward","text":"<pre><code>forward(X)\n</code></pre> <p>Forward pass. Simply connecting the Embedding layer with the ouput neuron(s)</p> Source code in <code>pytorch_widedeep/models/tabular/linear/wide.py</code> <pre><code>def forward(self, X: Tensor) -&gt; Tensor:\n    r\"\"\"Forward pass. Simply connecting the Embedding layer with the ouput\n    neuron(s)\"\"\"\n    out = self.wide_linear(X.long()).sum(dim=1) + self.bias\n    return out\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.mlp.tab_mlp.TabMlp","title":"TabMlp","text":"<pre><code>TabMlp(\n    column_idx,\n    *,\n    cat_embed_input=None,\n    cat_embed_dropout=None,\n    use_cat_bias=None,\n    cat_embed_activation=None,\n    continuous_cols=None,\n    cont_norm_layer=None,\n    embed_continuous=None,\n    embed_continuous_method=None,\n    cont_embed_dim=None,\n    cont_embed_dropout=None,\n    cont_embed_activation=None,\n    quantization_setup=None,\n    n_frequencies=None,\n    sigma=None,\n    share_last_layer=None,\n    full_embed_dropout=None,\n    mlp_hidden_dims=[200, 100],\n    mlp_activation=\"relu\",\n    mlp_dropout=0.1,\n    mlp_batchnorm=False,\n    mlp_batchnorm_last=False,\n    mlp_linear_first=True\n)\n</code></pre> <p>               Bases: <code>BaseTabularModelWithoutAttention</code></p> <p>Defines a <code>TabMlp</code> model that can be used as the <code>deeptabular</code> component of a Wide &amp; Deep model or independently by itself.</p> <p>This class combines embedding representations of the categorical features with numerical (aka continuous) features, embedded or not. These are then passed through a series of dense layers (i.e. a MLP).</p> <p>Most of the parameters for this class are <code>Optional</code> since the use of categorical or continuous is in fact optional (i.e. one can use categorical features only, continuous features only or both).</p> <p>Parameters:</p> <ul> <li> <code>column_idx</code>               (<code>Dict[str, int]</code>)           \u2013            <p>Dict containing the index of the columns that will be passed through the <code>TabMlp</code> model. Required to slice the tensors. e.g. {'education': 0, 'relationship': 1, 'workclass': 2, ...}.</p> </li> <li> <code>cat_embed_input</code>               (<code>Optional[List[Tuple[str, int, int]]]</code>, default:                   <code>None</code> )           \u2013            <p>List of Tuples with the column name, number of unique values and embedding dimension. e.g. [(education, 11, 32), ...]</p> </li> <li> <code>cat_embed_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Categorical embeddings dropout. If <code>None</code>, it will default to 0.</p> </li> <li> <code>use_cat_bias</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating if bias will be used for the categorical embeddings. If <code>None</code>, it will default to 'False'.</p> </li> <li> <code>cat_embed_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the categorical embeddings, if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> </li> <li> <code>continuous_cols</code>               (<code>Optional[List[str]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the name of the numeric (aka continuous) columns</p> </li> <li> <code>cont_norm_layer</code>               (<code>Optional[Literal[batchnorm, layernorm]]</code>, default:                   <code>None</code> )           \u2013            <p>Type of normalization layer applied to the continuous features. Options are: 'layernorm' and 'batchnorm'. if <code>None</code>, no normalization layer will be used.</p> </li> <li> <code>embed_continuous</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating if the continuous columns will be embedded using one of the available methods: 'standard', 'periodic' or 'piecewise'. If <code>None</code>, it will default to 'False'. NOTE: This parameter is deprecated and it  will be removed in future releases. Please, use the  <code>embed_continuous_method</code> parameter instead.</p> </li> <li> <code>embed_continuous_method</code>               (<code>Optional[Literal[standard, piecewise, periodic]]</code>, default:                   <code>None</code> )           \u2013            <p>Method to use to embed the continuous features. Options are: 'standard', 'periodic' or 'piecewise'. The 'standard' embedding method is based on the FT-Transformer implementation presented in the paper: Revisiting Deep Learning Models for Tabular Data. The 'periodic' and_'piecewise'_ methods were presented in the paper: On Embeddings for Numerical Features in Tabular Deep Learning. Please, read the papers for details.</p> </li> <li> <code>cont_embed_dim</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>Size of the continuous embeddings. If the continuous columns are embedded, <code>cont_embed_dim</code> must be passed.</p> </li> <li> <code>cont_embed_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p> </li> <li> <code>cont_embed_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the continuous embeddings if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported. If <code>None</code>, no activation function will be applied.</p> </li> <li> <code>quantization_setup</code>               (<code>Optional[Dict[str, List[float]]]</code>, default:                   <code>None</code> )           \u2013            <p>This parameter is used when the 'piecewise' method is used to embed the continuous cols. It is a dict where keys are the name of the continuous columns and values are lists with the boundaries for the quantization of the continuous_cols. See the examples for details. If If the 'piecewise' method is used, this parameter is required.</p> </li> <li> <code>n_frequencies</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>This is the so called 'k' in their paper On Embeddings for Numerical Features in Tabular Deep Learning, and is the number of 'frequencies' that will be used to represent each continuous column. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>sigma</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>This is the sigma parameter in the paper mentioned when describing the previous parameters and it is used to initialise the 'frequency weights'. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>share_last_layer</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>This parameter is not present in the before mentioned paper but it is implemented in the official repo. If <code>True</code> the linear layer that turns the frequencies into embeddings will be shared across the continuous columns. If <code>False</code> a different linear layer will be used for each continuous column. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>full_embed_dropout</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>If <code>True</code>, the full embedding corresponding to a column will be masked out/dropout. If <code>None</code>, it will default to <code>False</code>.</p> </li> <li> <code>mlp_hidden_dims</code>               (<code>List[int]</code>, default:                   <code>[200, 100]</code> )           \u2013            <p>List with the number of neurons per dense layer in the mlp.</p> </li> <li> <code>mlp_activation</code>               (<code>str</code>, default:                   <code>'relu'</code> )           \u2013            <p>Activation function for the dense layers of the MLP. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> </li> <li> <code>mlp_dropout</code>               (<code>Union[float, List[float]]</code>, default:                   <code>0.1</code> )           \u2013            <p>float or List of floats with the dropout between the dense layers. e.g: [0.5,0.5]</p> </li> <li> <code>mlp_batchnorm</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether or not batch normalization will be applied to the dense layers</p> </li> <li> <code>mlp_batchnorm_last</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether or not batch normalization will be applied to the last of the dense layers</p> </li> <li> <code>mlp_linear_first</code>               (<code>bool</code>, default:                   <code>True</code> )           \u2013            <p>Boolean indicating the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code></p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>encoder</code>               (<code>Module</code>)           \u2013            <p>mlp model that will receive the concatenation of the embeddings and the continuous columns</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import TabMlp\n&gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n&gt;&gt;&gt; colnames = [\"a\", \"b\", \"c\", \"d\", \"e\"]\n&gt;&gt;&gt; cat_embed_input = [(u, i, j) for u, i, j in zip(colnames[:4], [4] * 4, [8] * 4)]\n&gt;&gt;&gt; column_idx = {k: v for v, k in enumerate(colnames)}\n&gt;&gt;&gt; model = TabMlp(mlp_hidden_dims=[8, 4], column_idx=column_idx, cat_embed_input=cat_embed_input,\n... continuous_cols=[\"e\"])\n&gt;&gt;&gt; out = model(X_tab)\n</code></pre> Source code in <code>pytorch_widedeep/models/tabular/mlp/tab_mlp.py</code> <pre><code>def __init__(\n    self,\n    column_idx: Dict[str, int],\n    *,\n    cat_embed_input: Optional[List[Tuple[str, int, int]]] = None,\n    cat_embed_dropout: Optional[float] = None,\n    use_cat_bias: Optional[bool] = None,\n    cat_embed_activation: Optional[str] = None,\n    continuous_cols: Optional[List[str]] = None,\n    cont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\n    embed_continuous: Optional[bool] = None,\n    embed_continuous_method: Optional[\n        Literal[\"standard\", \"piecewise\", \"periodic\"]\n    ] = None,\n    cont_embed_dim: Optional[int] = None,\n    cont_embed_dropout: Optional[float] = None,\n    cont_embed_activation: Optional[str] = None,\n    quantization_setup: Optional[Dict[str, List[float]]] = None,\n    n_frequencies: Optional[int] = None,\n    sigma: Optional[float] = None,\n    share_last_layer: Optional[bool] = None,\n    full_embed_dropout: Optional[bool] = None,\n    mlp_hidden_dims: List[int] = [200, 100],\n    mlp_activation: str = \"relu\",\n    mlp_dropout: Union[float, List[float]] = 0.1,\n    mlp_batchnorm: bool = False,\n    mlp_batchnorm_last: bool = False,\n    mlp_linear_first: bool = True,\n):\n    super(TabMlp, self).__init__(\n        column_idx=column_idx,\n        cat_embed_input=cat_embed_input,\n        cat_embed_dropout=cat_embed_dropout,\n        use_cat_bias=use_cat_bias,\n        cat_embed_activation=cat_embed_activation,\n        continuous_cols=continuous_cols,\n        cont_norm_layer=cont_norm_layer,\n        embed_continuous=embed_continuous,\n        embed_continuous_method=embed_continuous_method,\n        cont_embed_dim=cont_embed_dim,\n        cont_embed_dropout=cont_embed_dropout,\n        cont_embed_activation=cont_embed_activation,\n        quantization_setup=quantization_setup,\n        n_frequencies=n_frequencies,\n        sigma=sigma,\n        share_last_layer=share_last_layer,\n        full_embed_dropout=full_embed_dropout,\n    )\n\n    self.mlp_hidden_dims = mlp_hidden_dims\n    self.mlp_activation = mlp_activation\n    self.mlp_dropout = mlp_dropout\n    self.mlp_batchnorm = mlp_batchnorm\n    self.mlp_batchnorm_last = mlp_batchnorm_last\n    self.mlp_linear_first = mlp_linear_first\n\n    # Embeddings are instantiated at the base model\n    # Mlp\n    mlp_input_dim = self.cat_out_dim + self.cont_out_dim\n    mlp_hidden_dims = [mlp_input_dim] + mlp_hidden_dims\n    self.encoder = MLP(\n        mlp_hidden_dims,\n        mlp_activation,\n        mlp_dropout,\n        mlp_batchnorm,\n        mlp_batchnorm_last,\n        mlp_linear_first,\n    )\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.mlp.tab_mlp.TabMlp.output_dim","title":"output_dim  <code>property</code>","text":"<pre><code>output_dim\n</code></pre> <p>The output dimension of the model. This is a required property neccesary to build the <code>WideDeep</code> class</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.mlp.tab_mlp.TabMlpDecoder","title":"TabMlpDecoder","text":"<pre><code>TabMlpDecoder(\n    embed_dim,\n    mlp_hidden_dims=[100, 200],\n    mlp_activation=\"relu\",\n    mlp_dropout=0.1,\n    mlp_batchnorm=False,\n    mlp_batchnorm_last=False,\n    mlp_linear_first=True,\n)\n</code></pre> <p>               Bases: <code>Module</code></p> <p>Companion decoder model for the <code>TabMlp</code> model (which can be considered an encoder itself).</p> <p>This class is designed to be used with the <code>EncoderDecoderTrainer</code> when using self-supervised pre-training (see the corresponding section in the docs). The <code>TabMlpDecoder</code> will receive the output from the MLP and 'reconstruct' the embeddings.</p> <p>Parameters:</p> <ul> <li> <code>embed_dim</code>               (<code>int</code>)           \u2013            <p>Size of the embeddings tensor that needs to be reconstructed.</p> </li> <li> <code>mlp_hidden_dims</code>               (<code>List[int]</code>, default:                   <code>[100, 200]</code> )           \u2013            <p>List with the number of neurons per dense layer in the mlp.</p> </li> <li> <code>mlp_activation</code>               (<code>str</code>, default:                   <code>'relu'</code> )           \u2013            <p>Activation function for the dense layers of the MLP. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> </li> <li> <code>mlp_dropout</code>               (<code>Union[float, List[float]]</code>, default:                   <code>0.1</code> )           \u2013            <p>float or List of floats with the dropout between the dense layers. e.g: [0.5,0.5]</p> </li> <li> <code>mlp_batchnorm</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether or not batch normalization will be applied to the dense layers</p> </li> <li> <code>mlp_batchnorm_last</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether or not batch normalization will be applied to the last of the dense layers</p> </li> <li> <code>mlp_linear_first</code>               (<code>bool</code>, default:                   <code>True</code> )           \u2013            <p>Boolean indicating the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code></p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>decoder</code>               (<code>Module</code>)           \u2013            <p>mlp model that will receive the output of the encoder</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import TabMlpDecoder\n&gt;&gt;&gt; x_inp = torch.rand(3, 8)\n&gt;&gt;&gt; decoder = TabMlpDecoder(embed_dim=32, mlp_hidden_dims=[8,16])\n&gt;&gt;&gt; res = decoder(x_inp)\n&gt;&gt;&gt; res.shape\ntorch.Size([3, 32])\n</code></pre> Source code in <code>pytorch_widedeep/models/tabular/mlp/tab_mlp.py</code> <pre><code>def __init__(\n    self,\n    embed_dim: int,\n    mlp_hidden_dims: List[int] = [100, 200],\n    mlp_activation: str = \"relu\",\n    mlp_dropout: Union[float, List[float]] = 0.1,\n    mlp_batchnorm: bool = False,\n    mlp_batchnorm_last: bool = False,\n    mlp_linear_first: bool = True,\n):\n    super(TabMlpDecoder, self).__init__()\n\n    self.embed_dim = embed_dim\n\n    self.mlp_hidden_dims = mlp_hidden_dims\n    self.mlp_activation = mlp_activation\n    self.mlp_dropout = mlp_dropout\n    self.mlp_batchnorm = mlp_batchnorm\n    self.mlp_batchnorm_last = mlp_batchnorm_last\n    self.mlp_linear_first = mlp_linear_first\n\n    self.decoder = MLP(\n        mlp_hidden_dims + [self.embed_dim],\n        mlp_activation,\n        mlp_dropout,\n        mlp_batchnorm,\n        mlp_batchnorm_last,\n        mlp_linear_first,\n    )\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.resnet.tab_resnet.TabResnet","title":"TabResnet","text":"<pre><code>TabResnet(\n    column_idx,\n    *,\n    cat_embed_input=None,\n    cat_embed_dropout=None,\n    use_cat_bias=None,\n    cat_embed_activation=None,\n    continuous_cols=None,\n    cont_norm_layer=None,\n    embed_continuous=None,\n    embed_continuous_method=None,\n    cont_embed_dim=None,\n    cont_embed_dropout=None,\n    cont_embed_activation=None,\n    quantization_setup=None,\n    n_frequencies=None,\n    sigma=None,\n    share_last_layer=None,\n    full_embed_dropout=None,\n    blocks_dims=[200, 100, 100],\n    blocks_dropout=0.1,\n    simplify_blocks=False,\n    mlp_hidden_dims=None,\n    mlp_activation=None,\n    mlp_dropout=None,\n    mlp_batchnorm=None,\n    mlp_batchnorm_last=None,\n    mlp_linear_first=None\n)\n</code></pre> <p>               Bases: <code>BaseTabularModelWithoutAttention</code></p> <p>Defines a <code>TabResnet</code> model that can be used as the <code>deeptabular</code> component of a Wide &amp; Deep model or independently by itself.</p> <p>This class combines embedding representations of the categorical features with numerical (aka continuous) features, embedded or not. These are then passed through a series of Resnet blocks. See <code>pytorch_widedeep.models.tab_resnet._layers</code> for details on the structure of each block.</p> <p>Most of the parameters for this class are <code>Optional</code> since the use of categorical or continuous is in fact optional (i.e. one can use categorical features only, continuous features only or both).</p> <p>Parameters:</p> <ul> <li> <code>column_idx</code>               (<code>Dict[str, int]</code>)           \u2013            <p>Dict containing the index of the columns that will be passed through the <code>TabMlp</code> model. Required to slice the tensors. e.g. {'education': 0, 'relationship': 1, 'workclass': 2, ...}.</p> </li> <li> <code>cat_embed_input</code>               (<code>Optional[List[Tuple[str, int, int]]]</code>, default:                   <code>None</code> )           \u2013            <p>List of Tuples with the column name, number of unique values and embedding dimension. e.g. [(education, 11, 32), ...]</p> </li> <li> <code>cat_embed_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Categorical embeddings dropout. If <code>None</code>, it will default to 0.</p> </li> <li> <code>use_cat_bias</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating if bias will be used for the categorical embeddings. If <code>None</code>, it will default to 'False'.</p> </li> <li> <code>cat_embed_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the categorical embeddings, if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> </li> <li> <code>continuous_cols</code>               (<code>Optional[List[str]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the name of the numeric (aka continuous) columns</p> </li> <li> <code>cont_norm_layer</code>               (<code>Optional[Literal[batchnorm, layernorm]]</code>, default:                   <code>None</code> )           \u2013            <p>Type of normalization layer applied to the continuous features. Options are: 'layernorm' and 'batchnorm'. if <code>None</code>, no normalization layer will be used.</p> </li> <li> <code>embed_continuous</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating if the continuous columns will be embedded using one of the available methods: 'standard', 'periodic' or 'piecewise'. If <code>None</code>, it will default to 'False'. NOTE: This parameter is deprecated and it  will be removed in future releases. Please, use the  <code>embed_continuous_method</code> parameter instead.</p> </li> <li> <code>embed_continuous_method</code>               (<code>Optional[Literal[standard, piecewise, periodic]]</code>, default:                   <code>None</code> )           \u2013            <p>Method to use to embed the continuous features. Options are: 'standard', 'periodic' or 'piecewise'. The 'standard' embedding method is based on the FT-Transformer implementation presented in the paper: Revisiting Deep Learning Models for Tabular Data. The 'periodic' and_'piecewise'_ methods were presented in the paper: On Embeddings for Numerical Features in Tabular Deep Learning. Please, read the papers for details.</p> </li> <li> <code>cont_embed_dim</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>Size of the continuous embeddings. If the continuous columns are embedded, <code>cont_embed_dim</code> must be passed.</p> </li> <li> <code>cont_embed_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p> </li> <li> <code>cont_embed_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the continuous embeddings if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported. If <code>None</code>, no activation function will be applied.</p> </li> <li> <code>quantization_setup</code>               (<code>Optional[Dict[str, List[float]]]</code>, default:                   <code>None</code> )           \u2013            <p>This parameter is used when the 'piecewise' method is used to embed the continuous cols. It is a dict where keys are the name of the continuous columns and values are lists with the boundaries for the quantization of the continuous_cols. See the examples for details. If If the 'piecewise' method is used, this parameter is required.</p> </li> <li> <code>n_frequencies</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>This is the so called 'k' in their paper On Embeddings for Numerical Features in Tabular Deep Learning, and is the number of 'frequencies' that will be used to represent each continuous column. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>sigma</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>This is the sigma parameter in the paper mentioned when describing the previous parameters and it is used to initialise the 'frequency weights'. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>share_last_layer</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>This parameter is not present in the before mentioned paper but it is implemented in the official repo. If <code>True</code> the linear layer that turns the frequencies into embeddings will be shared across the continuous columns. If <code>False</code> a different linear layer will be used for each continuous column. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>full_embed_dropout</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>If <code>True</code>, the full embedding corresponding to a column will be masked out/dropout. If <code>None</code>, it will default to <code>False</code>.</p> </li> <li> <code>blocks_dims</code>               (<code>List[int]</code>, default:                   <code>[200, 100, 100]</code> )           \u2013            <p>List of integers that define the input and output units of each block. For example: [200, 100, 100] will generate 2 blocks. The first will receive a tensor of size 200 and output a tensor of size 100, and the second will receive a tensor of size 100 and output a tensor of size 100. See <code>pytorch_widedeep.models.tab_resnet._layers</code> for details on the structure of each block.</p> </li> <li> <code>blocks_dropout</code>               (<code>float</code>, default:                   <code>0.1</code> )           \u2013            <p>Block's internal dropout.</p> </li> <li> <code>simplify_blocks</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating if the simplest possible residual blocks (<code>X -&gt; [ [LIN, BN, ACT]  + X ]</code>) will be used instead of a standard one (<code>X -&gt; [ [LIN1, BN1, ACT1] -&gt; [LIN2, BN2]  + X ]</code>).</p> </li> <li> <code>mlp_hidden_dims</code>               (<code>Optional[List[int]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the number of neurons per dense layer in the MLP. e.g: [64, 32]. If <code>None</code> the  output of the Resnet Blocks will be connected directly to the output neuron(s).</p> </li> <li> <code>mlp_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the dense layers of the MLP. Currently 'tanh', 'relu', 'leaky'_relu' and _'gelu' are supported. If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to 'relu'.</p> </li> <li> <code>mlp_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>float with the dropout between the dense layers of the MLP. If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to 0.0.</p> </li> <li> <code>mlp_batchnorm</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating whether or not batch normalization will be applied to the dense layers If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to False.</p> </li> <li> <code>mlp_batchnorm_last</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating whether or not batch normalization will be applied to the last of the dense layers If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to False.</p> </li> <li> <code>mlp_linear_first</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code> If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to <code>True</code>.</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>encoder</code>               (<code>Module</code>)           \u2013            <p>deep dense Resnet model that will receive the concatenation of the embeddings and the continuous columns</p> </li> <li> <code>mlp</code>               (<code>Module</code>)           \u2013            <p>if <code>mlp_hidden_dims</code> is <code>True</code>, this attribute will be an mlp model that will receive the results of the concatenation of the embeddings and the continuous columns -- if present --.</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import TabResnet\n&gt;&gt;&gt; X_deep = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n&gt;&gt;&gt; colnames = ['a', 'b', 'c', 'd', 'e']\n&gt;&gt;&gt; cat_embed_input = [(u,i,j) for u,i,j in zip(colnames[:4], [4]*4, [8]*4)]\n&gt;&gt;&gt; column_idx = {k:v for v,k in enumerate(colnames)}\n&gt;&gt;&gt; model = TabResnet(blocks_dims=[16,4], column_idx=column_idx, cat_embed_input=cat_embed_input,\n... continuous_cols = ['e'])\n&gt;&gt;&gt; out = model(X_deep)\n</code></pre> Source code in <code>pytorch_widedeep/models/tabular/resnet/tab_resnet.py</code> <pre><code>def __init__(\n    self,\n    column_idx: Dict[str, int],\n    *,\n    cat_embed_input: Optional[List[Tuple[str, int, int]]] = None,\n    cat_embed_dropout: Optional[float] = None,\n    use_cat_bias: Optional[bool] = None,\n    cat_embed_activation: Optional[str] = None,\n    continuous_cols: Optional[List[str]] = None,\n    cont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\n    embed_continuous: Optional[bool] = None,\n    embed_continuous_method: Optional[\n        Literal[\"standard\", \"piecewise\", \"periodic\"]\n    ] = None,\n    cont_embed_dim: Optional[int] = None,\n    cont_embed_dropout: Optional[float] = None,\n    cont_embed_activation: Optional[str] = None,\n    quantization_setup: Optional[Dict[str, List[float]]] = None,\n    n_frequencies: Optional[int] = None,\n    sigma: Optional[float] = None,\n    share_last_layer: Optional[bool] = None,\n    full_embed_dropout: Optional[bool] = None,\n    blocks_dims: List[int] = [200, 100, 100],\n    blocks_dropout: float = 0.1,\n    simplify_blocks: bool = False,\n    mlp_hidden_dims: Optional[List[int]] = None,\n    mlp_activation: Optional[str] = None,\n    mlp_dropout: Optional[float] = None,\n    mlp_batchnorm: Optional[bool] = None,\n    mlp_batchnorm_last: Optional[bool] = None,\n    mlp_linear_first: Optional[bool] = None,\n):\n    super(TabResnet, self).__init__(\n        column_idx=column_idx,\n        cat_embed_input=cat_embed_input,\n        cat_embed_dropout=cat_embed_dropout,\n        use_cat_bias=use_cat_bias,\n        cat_embed_activation=cat_embed_activation,\n        continuous_cols=continuous_cols,\n        cont_norm_layer=cont_norm_layer,\n        embed_continuous=embed_continuous,\n        embed_continuous_method=embed_continuous_method,\n        cont_embed_dim=cont_embed_dim,\n        cont_embed_dropout=cont_embed_dropout,\n        cont_embed_activation=cont_embed_activation,\n        quantization_setup=quantization_setup,\n        n_frequencies=n_frequencies,\n        sigma=sigma,\n        share_last_layer=share_last_layer,\n        full_embed_dropout=full_embed_dropout,\n    )\n\n    if len(blocks_dims) &lt; 2:\n        raise ValueError(\n            \"'blocks' must contain at least two elements, e.g. [256, 128]\"\n        )\n\n    self.blocks_dims = blocks_dims\n    self.blocks_dropout = blocks_dropout\n    self.simplify_blocks = simplify_blocks\n\n    self.mlp_hidden_dims = mlp_hidden_dims\n    self.mlp_activation = mlp_activation\n    self.mlp_dropout = mlp_dropout\n    self.mlp_batchnorm = mlp_batchnorm\n    self.mlp_batchnorm_last = mlp_batchnorm_last\n    self.mlp_linear_first = mlp_linear_first\n\n    # Embeddings are instantiated at the base model\n\n    # Resnet\n    dense_resnet_input_dim = self.cat_out_dim + self.cont_out_dim\n    self.encoder = DenseResnet(\n        dense_resnet_input_dim, blocks_dims, blocks_dropout, self.simplify_blocks\n    )\n\n    # Mlp: adding an MLP on top of the Resnet blocks is optional and\n    # therefore all related params are optional\n    if self.mlp_hidden_dims is not None:\n        self.mlp = MLP(\n            d_hidden=[self.blocks_dims[-1]] + self.mlp_hidden_dims,\n            activation=(\n                \"relu\" if self.mlp_activation is None else self.mlp_activation\n            ),\n            dropout=0.0 if self.mlp_dropout is None else self.mlp_dropout,\n            batchnorm=False if self.mlp_batchnorm is None else self.mlp_batchnorm,\n            batchnorm_last=(\n                False\n                if self.mlp_batchnorm_last is None\n                else self.mlp_batchnorm_last\n            ),\n            linear_first=(\n                True if self.mlp_linear_first is None else self.mlp_linear_first\n            ),\n        )\n    else:\n        self.mlp = None\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.resnet.tab_resnet.TabResnet.output_dim","title":"output_dim  <code>property</code>","text":"<pre><code>output_dim\n</code></pre> <p>The output dimension of the model. This is a required property neccesary to build the <code>WideDeep</code> class</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.resnet.tab_resnet.TabResnetDecoder","title":"TabResnetDecoder","text":"<pre><code>TabResnetDecoder(\n    embed_dim,\n    blocks_dims=[100, 100, 200],\n    blocks_dropout=0.1,\n    simplify_blocks=False,\n    mlp_hidden_dims=None,\n    mlp_activation=None,\n    mlp_dropout=None,\n    mlp_batchnorm=None,\n    mlp_batchnorm_last=None,\n    mlp_linear_first=None,\n)\n</code></pre> <p>               Bases: <code>Module</code></p> <p>Companion decoder model for the <code>TabResnet</code> model (which can be considered an encoder itself)</p> <p>This class is designed to be used with the <code>EncoderDecoderTrainer</code> when using self-supervised pre-training (see the corresponding section in the docs). This class will receive the output from the ResNet blocks or the MLP(if present) and 'reconstruct' the embeddings.</p> <p>Parameters:</p> <ul> <li> <code>embed_dim</code>               (<code>int</code>)           \u2013            <p>Size of the embeddings tensor to be reconstructed.</p> </li> <li> <code>blocks_dims</code>               (<code>List[int]</code>, default:                   <code>[100, 100, 200]</code> )           \u2013            <p>List of integers that define the input and output units of each block. For example: [200, 100, 100] will generate 2 blocks. The first will receive a tensor of size 200 and output a tensor of size 100, and the second will receive a tensor of size 100 and output a tensor of size 100. See <code>pytorch_widedeep.models.tab_resnet._layers</code> for details on the structure of each block.</p> </li> <li> <code>blocks_dropout</code>               (<code>float</code>, default:                   <code>0.1</code> )           \u2013            <p>Block's internal dropout.</p> </li> <li> <code>simplify_blocks</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating if the simplest possible residual blocks (<code>X -&gt; [ [LIN, BN, ACT]  + X ]</code>) will be used instead of a standard one (<code>X -&gt; [ [LIN1, BN1, ACT1] -&gt; [LIN2, BN2]  + X ]</code>).</p> </li> <li> <code>mlp_hidden_dims</code>               (<code>Optional[List[int]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the number of neurons per dense layer in the MLP. e.g: [64, 32]. If <code>None</code> the  output of the Resnet Blocks will be connected directly to the output neuron(s).</p> </li> <li> <code>mlp_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the dense layers of the MLP. Currently 'tanh', 'relu', 'leaky'_relu' and _'gelu' are supported. If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to 'relu'.</p> </li> <li> <code>mlp_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>float with the dropout between the dense layers of the MLP. If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to 0.0.</p> </li> <li> <code>mlp_batchnorm</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating whether or not batch normalization will be applied to the dense layers If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to False.</p> </li> <li> <code>mlp_batchnorm_last</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating whether or not batch normalization will be applied to the last of the dense layers If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to False.</p> </li> <li> <code>mlp_linear_first</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code> If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to <code>True</code>.</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>decoder</code>               (<code>Module</code>)           \u2013            <p>deep dense Resnet model that will receive the output of the encoder IF <code>mlp_hidden_dims</code> is None</p> </li> <li> <code>mlp</code>               (<code>Module</code>)           \u2013            <p>if <code>mlp_hidden_dims</code> is not None, the overall decoder will consist in an MLP that will receive the output of the encoder followed by the deep dense Resnet.</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import TabResnetDecoder\n&gt;&gt;&gt; x_inp = torch.rand(3, 8)\n&gt;&gt;&gt; decoder = TabResnetDecoder(embed_dim=32, blocks_dims=[8, 16, 16])\n&gt;&gt;&gt; res = decoder(x_inp)\n&gt;&gt;&gt; res.shape\ntorch.Size([3, 32])\n</code></pre> Source code in <code>pytorch_widedeep/models/tabular/resnet/tab_resnet.py</code> <pre><code>def __init__(\n    self,\n    embed_dim: int,\n    blocks_dims: List[int] = [100, 100, 200],\n    blocks_dropout: float = 0.1,\n    simplify_blocks: bool = False,\n    mlp_hidden_dims: Optional[List[int]] = None,\n    mlp_activation: Optional[str] = None,\n    mlp_dropout: Optional[float] = None,\n    mlp_batchnorm: Optional[bool] = None,\n    mlp_batchnorm_last: Optional[bool] = None,\n    mlp_linear_first: Optional[bool] = None,\n):\n    super(TabResnetDecoder, self).__init__()\n\n    if len(blocks_dims) &lt; 2:\n        raise ValueError(\n            \"'blocks' must contain at least two elements, e.g. [256, 128]\"\n        )\n\n    self.embed_dim = embed_dim\n\n    self.blocks_dims = blocks_dims\n    self.blocks_dropout = blocks_dropout\n    self.simplify_blocks = simplify_blocks\n\n    self.mlp_hidden_dims = mlp_hidden_dims\n    self.mlp_activation = mlp_activation\n    self.mlp_dropout = mlp_dropout\n    self.mlp_batchnorm = mlp_batchnorm\n    self.mlp_batchnorm_last = mlp_batchnorm_last\n    self.mlp_linear_first = mlp_linear_first\n\n    if self.mlp_hidden_dims is not None:\n        self.mlp = MLP(\n            d_hidden=[self.mlp_first_hidden_dim] + self.mlp_hidden_dims,\n            activation=(\n                \"relu\" if self.mlp_activation is None else self.mlp_activation\n            ),\n            dropout=0.0 if self.mlp_dropout is None else self.mlp_dropout,\n            batchnorm=False if self.mlp_batchnorm is None else self.mlp_batchnorm,\n            batchnorm_last=(\n                False\n                if self.mlp_batchnorm_last is None\n                else self.mlp_batchnorm_last\n            ),\n            linear_first=(\n                True if self.mlp_linear_first is None else self.mlp_linear_first\n            ),\n        )\n        self.decoder = DenseResnet(\n            self.mlp_hidden_dims[-1],\n            blocks_dims,\n            blocks_dropout,\n            self.simplify_blocks,\n        )\n    else:\n        self.mlp = None\n        self.decoder = DenseResnet(\n            blocks_dims[0], blocks_dims, blocks_dropout, self.simplify_blocks\n        )\n\n    self.reconstruction_layer = nn.Linear(blocks_dims[-1], embed_dim, bias=False)\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.tabnet.tab_net.TabNet","title":"TabNet","text":"<pre><code>TabNet(\n    column_idx,\n    *,\n    cat_embed_input=None,\n    cat_embed_dropout=None,\n    use_cat_bias=None,\n    cat_embed_activation=None,\n    continuous_cols=None,\n    cont_norm_layer=None,\n    embed_continuous=None,\n    embed_continuous_method=None,\n    cont_embed_dim=None,\n    cont_embed_dropout=None,\n    cont_embed_activation=None,\n    quantization_setup=None,\n    n_frequencies=None,\n    sigma=None,\n    share_last_layer=None,\n    full_embed_dropout=None,\n    n_steps=3,\n    step_dim=8,\n    attn_dim=8,\n    dropout=0.0,\n    n_glu_step_dependent=2,\n    n_glu_shared=2,\n    ghost_bn=True,\n    virtual_batch_size=128,\n    momentum=0.02,\n    gamma=1.3,\n    epsilon=1e-15,\n    mask_type=\"sparsemax\"\n)\n</code></pre> <p>               Bases: <code>BaseTabularModelWithoutAttention</code></p> <p>Defines a TabNet model that can be used as the <code>deeptabular</code> component of a Wide &amp; Deep model or independently by itself.</p> <p>The implementation in this library is fully based on that here by the dreamquark-ai team, simply adapted so that it can work within the <code>WideDeep</code> frame. Therefore, ALL CREDIT TO THE DREAMQUARK-AI TEAM.</p> <p>Parameters:</p> <ul> <li> <code>column_idx</code>               (<code>Dict[str, int]</code>)           \u2013            <p>Dict containing the index of the columns that will be passed through the <code>TabMlp</code> model. Required to slice the tensors. e.g. {'education': 0, 'relationship': 1, 'workclass': 2, ...}.</p> </li> <li> <code>cat_embed_input</code>               (<code>Optional[List[Tuple[str, int, int]]]</code>, default:                   <code>None</code> )           \u2013            <p>List of Tuples with the column name, number of unique values and embedding dimension. e.g. [(education, 11, 32), ...]</p> </li> <li> <code>cat_embed_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Categorical embeddings dropout. If <code>None</code>, it will default to 0.</p> </li> <li> <code>use_cat_bias</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating if bias will be used for the categorical embeddings. If <code>None</code>, it will default to 'False'.</p> </li> <li> <code>cat_embed_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the categorical embeddings, if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> </li> <li> <code>continuous_cols</code>               (<code>Optional[List[str]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the name of the numeric (aka continuous) columns</p> </li> <li> <code>cont_norm_layer</code>               (<code>Optional[Literal[batchnorm, layernorm]]</code>, default:                   <code>None</code> )           \u2013            <p>Type of normalization layer applied to the continuous features. Options are: 'layernorm' and 'batchnorm'. if <code>None</code>, no normalization layer will be used.</p> </li> <li> <code>embed_continuous</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating if the continuous columns will be embedded using one of the available methods: 'standard', 'periodic' or 'piecewise'. If <code>None</code>, it will default to 'False'. NOTE: This parameter is deprecated and it  will be removed in future releases. Please, use the  <code>embed_continuous_method</code> parameter instead.</p> </li> <li> <code>embed_continuous_method</code>               (<code>Optional[Literal[standard, piecewise, periodic]]</code>, default:                   <code>None</code> )           \u2013            <p>Method to use to embed the continuous features. Options are: 'standard', 'periodic' or 'piecewise'. The 'standard' embedding method is based on the FT-Transformer implementation presented in the paper: Revisiting Deep Learning Models for Tabular Data. The 'periodic' and_'piecewise'_ methods were presented in the paper: On Embeddings for Numerical Features in Tabular Deep Learning. Please, read the papers for details.</p> </li> <li> <code>cont_embed_dim</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>Size of the continuous embeddings. If the continuous columns are embedded, <code>cont_embed_dim</code> must be passed.</p> </li> <li> <code>cont_embed_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p> </li> <li> <code>cont_embed_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the continuous embeddings if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported. If <code>None</code>, no activation function will be applied.</p> </li> <li> <code>quantization_setup</code>               (<code>Optional[Dict[str, List[float]]]</code>, default:                   <code>None</code> )           \u2013            <p>This parameter is used when the 'piecewise' method is used to embed the continuous cols. It is a dict where keys are the name of the continuous columns and values are lists with the boundaries for the quantization of the continuous_cols. See the examples for details. If If the 'piecewise' method is used, this parameter is required.</p> </li> <li> <code>n_frequencies</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>This is the so called 'k' in their paper On Embeddings for Numerical Features in Tabular Deep Learning, and is the number of 'frequencies' that will be used to represent each continuous column. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>sigma</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>This is the sigma parameter in the paper mentioned when describing the previous parameters and it is used to initialise the 'frequency weights'. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>share_last_layer</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>This parameter is not present in the before mentioned paper but it is implemented in the official repo. If <code>True</code> the linear layer that turns the frequencies into embeddings will be shared across the continuous columns. If <code>False</code> a different linear layer will be used for each continuous column. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>full_embed_dropout</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>If <code>True</code>, the full embedding corresponding to a column will be masked out/dropout. If <code>None</code>, it will default to <code>False</code>.</p> </li> <li> <code>n_steps</code>               (<code>int</code>, default:                   <code>3</code> )           \u2013            <p>number of decision steps. For a better understanding of the function of <code>n_steps</code> and the upcoming parameters, please see the paper.</p> </li> <li> <code>step_dim</code>               (<code>int</code>, default:                   <code>8</code> )           \u2013            <p>Step's output dimension. This is the output dimension that <code>WideDeep</code> will collect and connect to the output neuron(s).</p> </li> <li> <code>attn_dim</code>               (<code>int</code>, default:                   <code>8</code> )           \u2013            <p>Attention dimension</p> </li> <li> <code>dropout</code>               (<code>float</code>, default:                   <code>0.0</code> )           \u2013            <p>GLU block's internal dropout</p> </li> <li> <code>n_glu_step_dependent</code>               (<code>int</code>, default:                   <code>2</code> )           \u2013            <p>number of GLU Blocks (<code>[FC -&gt; BN -&gt; GLU]</code>) that are step dependent</p> </li> <li> <code>n_glu_shared</code>               (<code>int</code>, default:                   <code>2</code> )           \u2013            <p>number of GLU Blocks (<code>[FC -&gt; BN -&gt; GLU]</code>) that will be shared across decision steps</p> </li> <li> <code>ghost_bn</code>               (<code>bool</code>, default:                   <code>True</code> )           \u2013            <p>Boolean indicating if Ghost Batch Normalization will be used.</p> </li> <li> <code>virtual_batch_size</code>               (<code>int</code>, default:                   <code>128</code> )           \u2013            <p>Batch size when using Ghost Batch Normalization</p> </li> <li> <code>momentum</code>               (<code>float</code>, default:                   <code>0.02</code> )           \u2013            <p>Ghost Batch Normalization's momentum. The dreamquark-ai advises for very low values. However high values are used in the original publication. During our tests higher values lead to better results</p> </li> <li> <code>gamma</code>               (<code>float</code>, default:                   <code>1.3</code> )           \u2013            <p>Relaxation parameter in the paper. When gamma = 1, a feature is enforced to be used only at one decision step. As gamma increases, more flexibility is provided to use a feature at multiple decision steps</p> </li> <li> <code>epsilon</code>               (<code>float</code>, default:                   <code>1e-15</code> )           \u2013            <p>Float to avoid log(0). Always keep low</p> </li> <li> <code>mask_type</code>               (<code>str</code>, default:                   <code>'sparsemax'</code> )           \u2013            <p>Mask function to use. Either 'sparsemax' or 'entmax'</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>encoder</code>               (<code>Module</code>)           \u2013            <p>the TabNet encoder. For details see the original publication.</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n&gt;&gt;&gt; colnames = [\"a\", \"b\", \"c\", \"d\", \"e\"]\n&gt;&gt;&gt; cat_embed_input = [(u, i, j) for u, i, j in zip(colnames[:4], [4] * 4, [8] * 4)]\n&gt;&gt;&gt; column_idx = {k: v for v, k in enumerate(colnames)}\n&gt;&gt;&gt; model = TabNet(column_idx=column_idx, cat_embed_input=cat_embed_input, continuous_cols=[\"e\"])\n&gt;&gt;&gt; out = model(X_tab)\n</code></pre> Source code in <code>pytorch_widedeep/models/tabular/tabnet/tab_net.py</code> <pre><code>def __init__(\n    self,\n    column_idx: Dict[str, int],\n    *,\n    cat_embed_input: Optional[List[Tuple[str, int, int]]] = None,\n    cat_embed_dropout: Optional[float] = None,\n    use_cat_bias: Optional[bool] = None,\n    cat_embed_activation: Optional[str] = None,\n    continuous_cols: Optional[List[str]] = None,\n    cont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\n    embed_continuous: Optional[bool] = None,\n    embed_continuous_method: Optional[\n        Literal[\"standard\", \"piecewise\", \"periodic\"]\n    ] = None,\n    cont_embed_dim: Optional[int] = None,\n    cont_embed_dropout: Optional[float] = None,\n    cont_embed_activation: Optional[str] = None,\n    quantization_setup: Optional[Dict[str, List[float]]] = None,\n    n_frequencies: Optional[int] = None,\n    sigma: Optional[float] = None,\n    share_last_layer: Optional[bool] = None,\n    full_embed_dropout: Optional[bool] = None,\n    n_steps: int = 3,\n    step_dim: int = 8,\n    attn_dim: int = 8,\n    dropout: float = 0.0,\n    n_glu_step_dependent: int = 2,\n    n_glu_shared: int = 2,\n    ghost_bn: bool = True,\n    virtual_batch_size: int = 128,\n    momentum: float = 0.02,\n    gamma: float = 1.3,\n    epsilon: float = 1e-15,\n    mask_type: str = \"sparsemax\",\n):\n    super(TabNet, self).__init__(\n        column_idx=column_idx,\n        cat_embed_input=cat_embed_input,\n        cat_embed_dropout=cat_embed_dropout,\n        use_cat_bias=use_cat_bias,\n        cat_embed_activation=cat_embed_activation,\n        continuous_cols=continuous_cols,\n        cont_norm_layer=cont_norm_layer,\n        embed_continuous=embed_continuous,\n        embed_continuous_method=embed_continuous_method,\n        cont_embed_dim=cont_embed_dim,\n        cont_embed_dropout=cont_embed_dropout,\n        cont_embed_activation=cont_embed_activation,\n        quantization_setup=quantization_setup,\n        n_frequencies=n_frequencies,\n        sigma=sigma,\n        share_last_layer=share_last_layer,\n        full_embed_dropout=full_embed_dropout,\n    )\n\n    self.n_steps = n_steps\n    self.step_dim = step_dim\n    self.attn_dim = attn_dim\n    self.dropout = dropout\n    self.n_glu_step_dependent = n_glu_step_dependent\n    self.n_glu_shared = n_glu_shared\n    self.ghost_bn = ghost_bn\n    self.virtual_batch_size = virtual_batch_size\n    self.momentum = momentum\n    self.gamma = gamma\n    self.epsilon = epsilon\n    self.mask_type = mask_type\n\n    # Embeddings are instantiated at the base model\n    self.embed_out_dim = self.cat_out_dim + self.cont_out_dim\n\n    # TabNet\n    self.encoder = TabNetEncoder(\n        self.embed_out_dim,\n        n_steps,\n        step_dim,\n        attn_dim,\n        dropout,\n        n_glu_step_dependent,\n        n_glu_shared,\n        ghost_bn,\n        virtual_batch_size,\n        momentum,\n        gamma,\n        epsilon,\n        mask_type,\n    )\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.tabnet.tab_net.TabNet.output_dim","title":"output_dim  <code>property</code>","text":"<pre><code>output_dim\n</code></pre> <p>The output dimension of the model. This is a required property neccesary to build the <code>WideDeep</code> class</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.tabnet.tab_net.TabNetDecoder","title":"TabNetDecoder","text":"<pre><code>TabNetDecoder(\n    embed_dim,\n    n_steps=3,\n    step_dim=8,\n    dropout=0.0,\n    n_glu_step_dependent=2,\n    n_glu_shared=2,\n    ghost_bn=True,\n    virtual_batch_size=128,\n    momentum=0.02,\n)\n</code></pre> <p>               Bases: <code>Module</code></p> <p>Companion decoder model for the <code>TabNet</code> model (which can be considered an encoder itself)</p> <p>This class is designed to be used with the <code>EncoderDecoderTrainer</code> when using self-supervised pre-training (see the corresponding section in the docs). This class will receive the output from the <code>TabNet</code> encoder (i.e. the output from the so called 'steps') and 'reconstruct' the embeddings.</p> <p>Parameters:</p> <ul> <li> <code>embed_dim</code>               (<code>int</code>)           \u2013            <p>Size of the embeddings tensor to be reconstructed.</p> </li> <li> <code>n_steps</code>               (<code>int</code>, default:                   <code>3</code> )           \u2013            <p>number of decision steps. For a better understanding of the function of <code>n_steps</code> and the upcoming parameters, please see the paper.</p> </li> <li> <code>step_dim</code>               (<code>int</code>, default:                   <code>8</code> )           \u2013            <p>Step's output dimension. This is the output dimension that <code>WideDeep</code> will collect and connect to the output neuron(s).</p> </li> <li> <code>dropout</code>               (<code>float</code>, default:                   <code>0.0</code> )           \u2013            <p>GLU block's internal dropout</p> </li> <li> <code>n_glu_step_dependent</code>               (<code>int</code>, default:                   <code>2</code> )           \u2013            <p>number of GLU Blocks (<code>[FC -&gt; BN -&gt; GLU]</code>) that are step dependent</p> </li> <li> <code>n_glu_shared</code>               (<code>int</code>, default:                   <code>2</code> )           \u2013            <p>number of GLU Blocks (<code>[FC -&gt; BN -&gt; GLU]</code>) that will be shared across decision steps</p> </li> <li> <code>ghost_bn</code>               (<code>bool</code>, default:                   <code>True</code> )           \u2013            <p>Boolean indicating if Ghost Batch Normalization will be used.</p> </li> <li> <code>virtual_batch_size</code>               (<code>int</code>, default:                   <code>128</code> )           \u2013            <p>Batch size when using Ghost Batch Normalization</p> </li> <li> <code>momentum</code>               (<code>float</code>, default:                   <code>0.02</code> )           \u2013            <p>Ghost Batch Normalization's momentum. The dreamquark-ai advises for very low values. However high values are used in the original publication. During our tests higher values lead to better results</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>decoder</code>               (<code>Module</code>)           \u2013            <p>decoder that will receive the output from the encoder's steps and will reconstruct the embeddings</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import TabNetDecoder\n&gt;&gt;&gt; x_inp = [torch.rand(3, 8), torch.rand(3, 8), torch.rand(3, 8)]\n&gt;&gt;&gt; decoder = TabNetDecoder(embed_dim=32, ghost_bn=False)\n&gt;&gt;&gt; res = decoder(x_inp)\n&gt;&gt;&gt; res.shape\ntorch.Size([3, 32])\n</code></pre> Source code in <code>pytorch_widedeep/models/tabular/tabnet/tab_net.py</code> <pre><code>def __init__(\n    self,\n    embed_dim: int,\n    n_steps: int = 3,\n    step_dim: int = 8,\n    dropout: float = 0.0,\n    n_glu_step_dependent: int = 2,\n    n_glu_shared: int = 2,\n    ghost_bn: bool = True,\n    virtual_batch_size: int = 128,\n    momentum: float = 0.02,\n):\n    super(TabNetDecoder, self).__init__()\n\n    self.n_steps = n_steps\n    self.step_dim = step_dim\n    self.dropout = dropout\n    self.n_glu_step_dependent = n_glu_step_dependent\n    self.n_glu_shared = n_glu_shared\n    self.ghost_bn = ghost_bn\n    self.virtual_batch_size = virtual_batch_size\n    self.momentum = momentum\n\n    shared_layers = nn.ModuleList()\n    for i in range(n_glu_shared):\n        if i == 0:\n            shared_layers.append(nn.Linear(step_dim, 2 * step_dim, bias=False))\n        else:\n            shared_layers.append(nn.Linear(step_dim, 2 * step_dim, bias=False))\n\n    self.decoder = nn.ModuleList()\n    for step in range(n_steps):\n        transformer = FeatTransformer(\n            step_dim,\n            step_dim,\n            dropout,\n            shared_layers,\n            n_glu_step_dependent,\n            ghost_bn,\n            virtual_batch_size,\n            momentum=momentum,\n        )\n        self.decoder.append(transformer)\n\n    self.reconstruction_layer = nn.Linear(step_dim, embed_dim, bias=False)\n    initialize_non_glu(self.reconstruction_layer, step_dim, embed_dim)\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.mlp.context_attention_mlp.ContextAttentionMLP","title":"ContextAttentionMLP","text":"<pre><code>ContextAttentionMLP(\n    column_idx,\n    *,\n    cat_embed_input=None,\n    cat_embed_dropout=None,\n    use_cat_bias=None,\n    cat_embed_activation=None,\n    shared_embed=None,\n    add_shared_embed=None,\n    frac_shared_embed=None,\n    continuous_cols=None,\n    cont_norm_layer=None,\n    embed_continuous_method=\"standard\",\n    cont_embed_dropout=None,\n    cont_embed_activation=None,\n    quantization_setup=None,\n    n_frequencies=None,\n    sigma=None,\n    share_last_layer=None,\n    full_embed_dropout=None,\n    input_dim=32,\n    attn_dropout=0.2,\n    with_addnorm=False,\n    attn_activation=\"leaky_relu\",\n    n_blocks=3\n)\n</code></pre> <p>               Bases: <code>BaseTabularModelWithAttention</code></p> <p>Defines a <code>ContextAttentionMLP</code> model that can be used as the <code>deeptabular</code> component of a Wide &amp; Deep model or independently by itself.</p> <p>This class combines embedding representations of the categorical features with numerical (aka continuous) features that are also embedded. These are then passed through a series of attention blocks. Each attention block is comprised by a <code>ContextAttentionEncoder</code>. Such encoder is in part inspired by the attention mechanism described in Hierarchical Attention Networks for Document Classification. See <code>pytorch_widedeep.models.tabular.mlp._attention_layers</code> for details.</p> <p>Most of the parameters for this class are <code>Optional</code> since the use of categorical or continuous is in fact optional (i.e. one can use categorical features only, continuous features only or both).</p> <p>Parameters:</p> <ul> <li> <code>column_idx</code>               (<code>Dict[str, int]</code>)           \u2013            <p>Dict containing the index of the columns that will be passed through the <code>TabMlp</code> model. Required to slice the tensors. e.g. {'education': 0, 'relationship': 1, 'workclass': 2, ...}.</p> </li> <li> <code>cat_embed_input</code>               (<code>Optional[List[Tuple[str, int]]]</code>, default:                   <code>None</code> )           \u2013            <p>List of Tuples with the column name and number of unique values and embedding dimension. e.g. [(education, 11), ...]</p> </li> <li> <code>cat_embed_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Categorical embeddings dropout. If <code>None</code>, it will default to 0.</p> </li> <li> <code>use_cat_bias</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating if bias will be used for the categorical embeddings. If <code>None</code>, it will default to 'False'.</p> </li> <li> <code>cat_embed_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the categorical embeddings, if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> </li> <li> <code>shared_embed</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating if the embeddings will be \"shared\". The idea behind <code>shared_embed</code> is described in the Appendix A in the TabTransformer paper: 'The goal of having column embedding is to enable the model to distinguish the classes in one column from those in the other columns'. In other words, the idea is to let the model learn which column is embedded at the time. See: <code>pytorch_widedeep.models.transformers._layers.SharedEmbeddings</code>.</p> </li> <li> <code>add_shared_embed</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>The two embedding sharing strategies are: 1) add the shared embeddings to the column embeddings or 2) to replace the first <code>frac_shared_embed</code> with the shared embeddings. See <code>pytorch_widedeep.models.embeddings_layers.SharedEmbeddings</code> If 'None' is passed, it will default to 'False'.</p> </li> <li> <code>frac_shared_embed</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>The fraction of embeddings that will be shared (if <code>add_shared_embed = False</code>) by all the different categories for one particular column. If 'None' is passed, it will default to 0.0.</p> </li> <li> <code>continuous_cols</code>               (<code>Optional[List[str]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the name of the numeric (aka continuous) columns</p> </li> <li> <code>cont_norm_layer</code>               (<code>Optional[Literal[batchnorm, layernorm]]</code>, default:                   <code>None</code> )           \u2013            <p>Type of normalization layer applied to the continuous features. Options are: 'layernorm' and 'batchnorm'. if <code>None</code>, no normalization layer will be used.</p> </li> <li> <code>embed_continuous_method</code>               (<code>Optional[Literal[standard, piecewise, periodic]]</code>, default:                   <code>'standard'</code> )           \u2013            <p>Method to use to embed the continuous features. Options are: 'standard', 'periodic' or 'piecewise'. The 'standard' embedding method is based on the FT-Transformer implementation presented in the paper: Revisiting Deep Learning Models for Tabular Data. The 'periodic' and_'piecewise'_ methods were presented in the paper: On Embeddings for Numerical Features in Tabular Deep Learning. Please, read the papers for details.</p> </li> <li> <code>cont_embed_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p> </li> <li> <code>cont_embed_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the continuous embeddings if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported. If <code>None</code>, no activation function will be applied.</p> </li> <li> <code>quantization_setup</code>               (<code>Optional[Dict[str, List[float]]]</code>, default:                   <code>None</code> )           \u2013            <p>This parameter is used when the 'piecewise' method is used to embed the continuous cols. It is a dict where keys are the name of the continuous columns and values are lists with the boundaries for the quantization of the continuous_cols. See the examples for details. If If the 'piecewise' method is used, this parameter is required.</p> </li> <li> <code>n_frequencies</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>This is the so called 'k' in their paper On Embeddings for Numerical Features in Tabular Deep Learning, and is the number of 'frequencies' that will be used to represent each continuous column. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>sigma</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>This is the sigma parameter in the paper mentioned when describing the previous parameters and it is used to initialise the 'frequency weights'. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>share_last_layer</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>This parameter is not present in the before mentioned paper but it is implemented in the official repo. If <code>True</code> the linear layer that turns the frequencies into embeddings will be shared across the continuous columns. If <code>False</code> a different linear layer will be used for each continuous column. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>full_embed_dropout</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>If <code>True</code>, the full embedding corresponding to a column will be masked out/dropout. If <code>None</code>, it will default to <code>False</code>.</p> </li> <li> <code>input_dim</code>               (<code>int</code>, default:                   <code>32</code> )           \u2013            <p>The so-called dimension of the model. Is the number of embeddings used to encode the categorical and/or continuous columns</p> </li> <li> <code>attn_dropout</code>               (<code>float</code>, default:                   <code>0.2</code> )           \u2013            <p>Dropout for each attention block</p> </li> <li> <code>with_addnorm</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating if residual connections will be used in the attention blocks</p> </li> <li> <code>attn_activation</code>               (<code>str</code>, default:                   <code>'leaky_relu'</code> )           \u2013            <p>String indicating the activation function to be applied to the dense layer in each attention encoder. 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported.</p> </li> <li> <code>n_blocks</code>               (<code>int</code>, default:                   <code>3</code> )           \u2013            <p>Number of attention blocks</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>encoder</code>               (<code>Module</code>)           \u2013            <p>Sequence of attention encoders.</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import ContextAttentionMLP\n&gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n&gt;&gt;&gt; colnames = ['a', 'b', 'c', 'd', 'e']\n&gt;&gt;&gt; cat_embed_input = [(u,i,j) for u,i,j in zip(colnames[:4], [4]*4, [8]*4)]\n&gt;&gt;&gt; column_idx = {k:v for v,k in enumerate(colnames)}\n&gt;&gt;&gt; model = ContextAttentionMLP(column_idx=column_idx, cat_embed_input=cat_embed_input, continuous_cols = ['e'])\n&gt;&gt;&gt; out = model(X_tab)\n</code></pre> Source code in <code>pytorch_widedeep/models/tabular/mlp/context_attention_mlp.py</code> <pre><code>def __init__(\n    self,\n    column_idx: Dict[str, int],\n    *,\n    cat_embed_input: Optional[List[Tuple[str, int]]] = None,\n    cat_embed_dropout: Optional[float] = None,\n    use_cat_bias: Optional[bool] = None,\n    cat_embed_activation: Optional[str] = None,\n    shared_embed: Optional[bool] = None,\n    add_shared_embed: Optional[bool] = None,\n    frac_shared_embed: Optional[float] = None,\n    continuous_cols: Optional[List[str]] = None,\n    cont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\n    embed_continuous_method: Optional[\n        Literal[\"standard\", \"piecewise\", \"periodic\"]\n    ] = \"standard\",\n    cont_embed_dropout: Optional[float] = None,\n    cont_embed_activation: Optional[str] = None,\n    quantization_setup: Optional[Dict[str, List[float]]] = None,\n    n_frequencies: Optional[int] = None,\n    sigma: Optional[float] = None,\n    share_last_layer: Optional[bool] = None,\n    full_embed_dropout: Optional[bool] = None,\n    input_dim: int = 32,\n    attn_dropout: float = 0.2,\n    with_addnorm: bool = False,\n    attn_activation: str = \"leaky_relu\",\n    n_blocks: int = 3,\n):\n    super(ContextAttentionMLP, self).__init__(\n        column_idx=column_idx,\n        cat_embed_input=cat_embed_input,\n        cat_embed_dropout=cat_embed_dropout,\n        use_cat_bias=use_cat_bias,\n        cat_embed_activation=cat_embed_activation,\n        shared_embed=shared_embed,\n        add_shared_embed=add_shared_embed,\n        frac_shared_embed=frac_shared_embed,\n        continuous_cols=continuous_cols,\n        cont_norm_layer=cont_norm_layer,\n        embed_continuous_method=embed_continuous_method,\n        cont_embed_dropout=cont_embed_dropout,\n        cont_embed_activation=cont_embed_activation,\n        input_dim=input_dim,\n        quantization_setup=quantization_setup,\n        n_frequencies=n_frequencies,\n        sigma=sigma,\n        share_last_layer=share_last_layer,\n        full_embed_dropout=full_embed_dropout,\n    )\n\n    self.attn_dropout = attn_dropout\n    self.with_addnorm = with_addnorm\n    self.attn_activation = attn_activation\n    self.n_blocks = n_blocks\n\n    self.with_cls_token = \"cls_token\" in column_idx\n    self.n_cat = len(cat_embed_input) if cat_embed_input is not None else 0\n    self.n_cont = len(continuous_cols) if continuous_cols is not None else 0\n\n    # Embeddings are instantiated at the base model\n    # Attention Blocks\n    self.encoder = nn.Sequential()\n    for i in range(n_blocks):\n        self.encoder.add_module(\n            \"attention_block\" + str(i),\n            ContextAttentionEncoder(\n                input_dim,\n                attn_dropout,\n                with_addnorm,\n                attn_activation,\n            ),\n        )\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.mlp.context_attention_mlp.ContextAttentionMLP.output_dim","title":"output_dim  <code>property</code>","text":"<pre><code>output_dim\n</code></pre> <p>The output dimension of the model. This is a required property neccesary to build the <code>WideDeep</code> class</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.mlp.context_attention_mlp.ContextAttentionMLP.attention_weights","title":"attention_weights  <code>property</code>","text":"<pre><code>attention_weights\n</code></pre> <p>List with the attention weights per block</p> <p>The shape of the attention weights is \\((N, F)\\), where \\(N\\) is the batch size and \\(F\\) is the number of features/columns in the dataset</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.mlp.self_attention_mlp.SelfAttentionMLP","title":"SelfAttentionMLP","text":"<pre><code>SelfAttentionMLP(\n    column_idx,\n    *,\n    cat_embed_input=None,\n    cat_embed_dropout=None,\n    use_cat_bias=None,\n    cat_embed_activation=None,\n    shared_embed=None,\n    add_shared_embed=None,\n    frac_shared_embed=None,\n    continuous_cols=None,\n    cont_norm_layer=None,\n    embed_continuous_method=\"standard\",\n    cont_embed_dropout=None,\n    cont_embed_activation=None,\n    quantization_setup=None,\n    n_frequencies=None,\n    sigma=None,\n    share_last_layer=None,\n    full_embed_dropout=None,\n    input_dim=32,\n    attn_dropout=0.2,\n    n_heads=8,\n    use_bias=False,\n    with_addnorm=False,\n    attn_activation=\"leaky_relu\",\n    n_blocks=3\n)\n</code></pre> <p>               Bases: <code>BaseTabularModelWithAttention</code></p> <p>Defines a <code>SelfAttentionMLP</code> model that can be used as the deeptabular component of a Wide &amp; Deep model or independently by itself.</p> <p>This class combines embedding representations of the categorical features with numerical (aka continuous) features that are also embedded. These are then passed through a series of attention blocks. Each attention block is comprised by what we would refer as a simplified <code>SelfAttentionEncoder</code>. See <code>pytorch_widedeep.models.tabular.mlp._attention_layers</code> for details. The reason to use a simplified version of self attention is because we observed that the 'standard' attention mechanism used in the TabTransformer has a notable tendency to overfit.</p> <p>In more detail, this model only uses Q and K (and not V). If we think about it as in terms of text (and intuitively), the Softmax(QK^T) is the attention mechanism that tells us how much, at each position in the input sentence, each word is represented or 'expressed'. We refer to that as 'attention weights'. These attention weighst are normally multiplied by a Value matrix to further strength the focus on the words that each word should be attending to (again, intuitively).</p> <p>In this implementation we skip this last multiplication and instead we multiply the attention weights directly by the input tensor. This is a simplification that we expect is beneficial in terms of avoiding overfitting for tabular data.</p> <p>Most of the parameters for this class are <code>Optional</code> since the use of categorical or continuous is in fact optional (i.e. one can use categorical features only, continuous features only or both).</p> <p>Parameters:</p> <ul> <li> <code>column_idx</code>               (<code>Dict[str, int]</code>)           \u2013            <p>Dict containing the index of the columns that will be passed through the <code>TabMlp</code> model. Required to slice the tensors. e.g. {'education': 0, 'relationship': 1, 'workclass': 2, ...}.</p> </li> <li> <code>cat_embed_input</code>               (<code>Optional[List[Tuple[str, int]]]</code>, default:                   <code>None</code> )           \u2013            <p>List of Tuples with the column name and number of unique values and embedding dimension. e.g. [(education, 11), ...]</p> </li> <li> <code>cat_embed_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Categorical embeddings dropout. If <code>None</code>, it will default to 0.</p> </li> <li> <code>use_cat_bias</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating if bias will be used for the categorical embeddings. If <code>None</code>, it will default to 'False'.</p> </li> <li> <code>cat_embed_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the categorical embeddings, if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> </li> <li> <code>shared_embed</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating if the embeddings will be \"shared\". The idea behind <code>shared_embed</code> is described in the Appendix A in the TabTransformer paper: 'The goal of having column embedding is to enable the model to distinguish the classes in one column from those in the other columns'. In other words, the idea is to let the model learn which column is embedded at the time. See: <code>pytorch_widedeep.models.transformers._layers.SharedEmbeddings</code>.</p> </li> <li> <code>add_shared_embed</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>The two embedding sharing strategies are: 1) add the shared embeddings to the column embeddings or 2) to replace the first <code>frac_shared_embed</code> with the shared embeddings. See <code>pytorch_widedeep.models.embeddings_layers.SharedEmbeddings</code> If 'None' is passed, it will default to 'False'.</p> </li> <li> <code>frac_shared_embed</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>The fraction of embeddings that will be shared (if <code>add_shared_embed = False</code>) by all the different categories for one particular column. If 'None' is passed, it will default to 0.0.</p> </li> <li> <code>continuous_cols</code>               (<code>Optional[List[str]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the name of the numeric (aka continuous) columns</p> </li> <li> <code>cont_norm_layer</code>               (<code>Optional[Literal[batchnorm, layernorm]]</code>, default:                   <code>None</code> )           \u2013            <p>Type of normalization layer applied to the continuous features. Options are: 'layernorm' and 'batchnorm'. if <code>None</code>, no normalization layer will be used.</p> </li> <li> <code>embed_continuous_method</code>               (<code>Optional[Literal[standard, piecewise, periodic]]</code>, default:                   <code>'standard'</code> )           \u2013            <p>Method to use to embed the continuous features. Options are: 'standard', 'periodic' or 'piecewise'. The 'standard' embedding method is based on the FT-Transformer implementation presented in the paper: Revisiting Deep Learning Models for Tabular Data. The 'periodic' and_'piecewise'_ methods were presented in the paper: On Embeddings for Numerical Features in Tabular Deep Learning. Please, read the papers for details.</p> </li> <li> <code>cont_embed_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p> </li> <li> <code>cont_embed_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the continuous embeddings if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported. If <code>None</code>, no activation function will be applied.</p> </li> <li> <code>quantization_setup</code>               (<code>Optional[Dict[str, List[float]]]</code>, default:                   <code>None</code> )           \u2013            <p>This parameter is used when the 'piecewise' method is used to embed the continuous cols. It is a dict where keys are the name of the continuous columns and values are lists with the boundaries for the quantization of the continuous_cols. See the examples for details. If If the 'piecewise' method is used, this parameter is required.</p> </li> <li> <code>n_frequencies</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>This is the so called 'k' in their paper On Embeddings for Numerical Features in Tabular Deep Learning, and is the number of 'frequencies' that will be used to represent each continuous column. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>sigma</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>This is the sigma parameter in the paper mentioned when describing the previous parameters and it is used to initialise the 'frequency weights'. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>share_last_layer</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>This parameter is not present in the before mentioned paper but it is implemented in the official repo. If <code>True</code> the linear layer that turns the frequencies into embeddings will be shared across the continuous columns. If <code>False</code> a different linear layer will be used for each continuous column. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>full_embed_dropout</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>If <code>True</code>, the full embedding corresponding to a column will be masked out/dropout. If <code>None</code>, it will default to <code>False</code>.</p> </li> <li> <code>input_dim</code>               (<code>int</code>, default:                   <code>32</code> )           \u2013            <p>The so-called dimension of the model. Is the number of embeddings used to encode the categorical and/or continuous columns</p> </li> <li> <code>attn_dropout</code>               (<code>float</code>, default:                   <code>0.2</code> )           \u2013            <p>Dropout for each attention block</p> </li> <li> <code>n_heads</code>               (<code>int</code>, default:                   <code>8</code> )           \u2013            <p>Number of attention heads per attention block.</p> </li> <li> <code>use_bias</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether or not to use bias in the Q, K projection layers.</p> </li> <li> <code>with_addnorm</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating if residual connections will be used in the attention blocks</p> </li> <li> <code>attn_activation</code>               (<code>str</code>, default:                   <code>'leaky_relu'</code> )           \u2013            <p>String indicating the activation function to be applied to the dense layer in each attention encoder. 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported.</p> </li> <li> <code>n_blocks</code>               (<code>int</code>, default:                   <code>3</code> )           \u2013            <p>Number of attention blocks</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>cat_and_cont_embed</code>               (<code>Module</code>)           \u2013            <p>This is the module that processes the categorical and continuous columns</p> </li> <li> <code>encoder</code>               (<code>Module</code>)           \u2013            <p>Sequence of attention encoders.</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import SelfAttentionMLP\n&gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n&gt;&gt;&gt; colnames = ['a', 'b', 'c', 'd', 'e']\n&gt;&gt;&gt; cat_embed_input = [(u,i,j) for u,i,j in zip(colnames[:4], [4]*4, [8]*4)]\n&gt;&gt;&gt; column_idx = {k:v for v,k in enumerate(colnames)}\n&gt;&gt;&gt; model = SelfAttentionMLP(column_idx=column_idx, cat_embed_input=cat_embed_input, continuous_cols = ['e'])\n&gt;&gt;&gt; out = model(X_tab)\n</code></pre> Source code in <code>pytorch_widedeep/models/tabular/mlp/self_attention_mlp.py</code> <pre><code>def __init__(\n    self,\n    column_idx: Dict[str, int],\n    *,\n    cat_embed_input: Optional[List[Tuple[str, int]]] = None,\n    cat_embed_dropout: Optional[float] = None,\n    use_cat_bias: Optional[bool] = None,\n    cat_embed_activation: Optional[str] = None,\n    shared_embed: Optional[bool] = None,\n    add_shared_embed: Optional[bool] = None,\n    frac_shared_embed: Optional[float] = None,\n    continuous_cols: Optional[List[str]] = None,\n    cont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\n    embed_continuous_method: Optional[\n        Literal[\"standard\", \"piecewise\", \"periodic\"]\n    ] = \"standard\",\n    cont_embed_dropout: Optional[float] = None,\n    cont_embed_activation: Optional[str] = None,\n    quantization_setup: Optional[Dict[str, List[float]]] = None,\n    n_frequencies: Optional[int] = None,\n    sigma: Optional[float] = None,\n    share_last_layer: Optional[bool] = None,\n    full_embed_dropout: Optional[bool] = None,\n    input_dim: int = 32,\n    attn_dropout: float = 0.2,\n    n_heads: int = 8,\n    use_bias: bool = False,\n    with_addnorm: bool = False,\n    attn_activation: str = \"leaky_relu\",\n    n_blocks: int = 3,\n):\n    super(SelfAttentionMLP, self).__init__(\n        column_idx=column_idx,\n        input_dim=input_dim,\n        cat_embed_input=cat_embed_input,\n        cat_embed_dropout=cat_embed_dropout,\n        use_cat_bias=use_cat_bias,\n        cat_embed_activation=cat_embed_activation,\n        shared_embed=shared_embed,\n        add_shared_embed=add_shared_embed,\n        frac_shared_embed=frac_shared_embed,\n        continuous_cols=continuous_cols,\n        cont_norm_layer=cont_norm_layer,\n        embed_continuous_method=embed_continuous_method,\n        cont_embed_dropout=cont_embed_dropout,\n        cont_embed_activation=cont_embed_activation,\n        quantization_setup=quantization_setup,\n        n_frequencies=n_frequencies,\n        sigma=sigma,\n        share_last_layer=share_last_layer,\n        full_embed_dropout=full_embed_dropout,\n    )\n\n    self.attn_dropout = attn_dropout\n    self.n_heads = n_heads\n    self.use_bias = use_bias\n    self.with_addnorm = with_addnorm\n    self.attn_activation = attn_activation\n    self.n_blocks = n_blocks\n\n    self.with_cls_token = \"cls_token\" in column_idx\n    self.n_cat = len(cat_embed_input) if cat_embed_input is not None else 0\n    self.n_cont = len(continuous_cols) if continuous_cols is not None else 0\n\n    # Embeddings are instantiated at the base model\n    # Attention Blocks\n    self.encoder = nn.Sequential()\n    for i in range(n_blocks):\n        self.encoder.add_module(\n            \"attention_block\" + str(i),\n            SelfAttentionEncoder(\n                input_dim,\n                attn_dropout,\n                use_bias,\n                n_heads,\n                with_addnorm,\n                attn_activation,\n            ),\n        )\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.mlp.self_attention_mlp.SelfAttentionMLP.output_dim","title":"output_dim  <code>property</code>","text":"<pre><code>output_dim\n</code></pre> <p>The output dimension of the model. This is a required property neccesary to build the WideDeep class</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.mlp.self_attention_mlp.SelfAttentionMLP.attention_weights","title":"attention_weights  <code>property</code>","text":"<pre><code>attention_weights\n</code></pre> <p>List with the attention weights per block</p> <p>The shape of the attention weights is \\((N, H, F, F)\\), where \\(N\\) is the batch size, \\(H\\) is the number of attention heads and \\(F\\) is the number of features/columns in the dataset</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.transformers.tab_transformer.TabTransformer","title":"TabTransformer","text":"<pre><code>TabTransformer(\n    column_idx,\n    *,\n    cat_embed_input=None,\n    cat_embed_dropout=None,\n    use_cat_bias=None,\n    cat_embed_activation=None,\n    shared_embed=None,\n    add_shared_embed=None,\n    frac_shared_embed=None,\n    continuous_cols=None,\n    cont_norm_layer=None,\n    embed_continuous=None,\n    embed_continuous_method=None,\n    cont_embed_dropout=None,\n    cont_embed_activation=None,\n    quantization_setup=None,\n    n_frequencies=None,\n    sigma=None,\n    share_last_layer=None,\n    full_embed_dropout=None,\n    input_dim=32,\n    n_heads=8,\n    use_qkv_bias=False,\n    n_blocks=4,\n    attn_dropout=0.2,\n    ff_dropout=0.1,\n    ff_factor=4,\n    transformer_activation=\"gelu\",\n    use_linear_attention=False,\n    use_flash_attention=False,\n    mlp_hidden_dims=None,\n    mlp_activation=\"relu\",\n    mlp_dropout=0.1,\n    mlp_batchnorm=False,\n    mlp_batchnorm_last=False,\n    mlp_linear_first=True\n)\n</code></pre> <p>               Bases: <code>BaseTabularModelWithAttention</code></p> <p>Defines our adptation of the TabTransformer model that can be used as the <code>deeptabular</code> component of a Wide &amp; Deep model or independently by itself.</p> <p>Most of the parameters for this class are <code>Optional</code> since the use of categorical or continuous is in fact optional (i.e. one can use categorical features only, continuous features only or both).</p> <p> NOTE: This is an enhanced adaptation of the model described in the paper. It can be considered as the flagship of our transformer family of models for tabular data and offers mutiple, additional features relative to the original publication(and some other models in the library)</p> <p>Parameters:</p> <ul> <li> <code>column_idx</code>               (<code>Dict[str, int]</code>)           \u2013            <p>Dict containing the index of the columns that will be passed through the <code>TabMlp</code> model. Required to slice the tensors. e.g. {'education': 0, 'relationship': 1, 'workclass': 2, ...}.</p> </li> <li> <code>cat_embed_input</code>               (<code>Optional[List[Tuple[str, int]]]</code>, default:                   <code>None</code> )           \u2013            <p>List of Tuples with the column name and number of unique values and embedding dimension. e.g. [(education, 11), ...]</p> </li> <li> <code>cat_embed_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Categorical embeddings dropout. If <code>None</code>, it will default to 0.</p> </li> <li> <code>use_cat_bias</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating if bias will be used for the categorical embeddings. If <code>None</code>, it will default to 'False'.</p> </li> <li> <code>cat_embed_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the categorical embeddings, if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> </li> <li> <code>shared_embed</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating if the embeddings will be \"shared\". The idea behind <code>shared_embed</code> is described in the Appendix A in the TabTransformer paper: 'The goal of having column embedding is to enable the model to distinguish the classes in one column from those in the other columns'. In other words, the idea is to let the model learn which column is embedded at the time. See: <code>pytorch_widedeep.models.transformers._layers.SharedEmbeddings</code>.</p> </li> <li> <code>add_shared_embed</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>The two embedding sharing strategies are: 1) add the shared embeddings to the column embeddings or 2) to replace the first <code>frac_shared_embed</code> with the shared embeddings. See <code>pytorch_widedeep.models.embeddings_layers.SharedEmbeddings</code> If 'None' is passed, it will default to 'False'.</p> </li> <li> <code>frac_shared_embed</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>The fraction of embeddings that will be shared (if <code>add_shared_embed = False</code>) by all the different categories for one particular column. If 'None' is passed, it will default to 0.0.</p> </li> <li> <code>continuous_cols</code>               (<code>Optional[List[str]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the name of the numeric (aka continuous) columns</p> </li> <li> <code>cont_norm_layer</code>               (<code>Optional[Literal[batchnorm, layernorm]]</code>, default:                   <code>None</code> )           \u2013            <p>Type of normalization layer applied to the continuous features. Options are: 'layernorm' and 'batchnorm'. if <code>None</code>, no normalization layer will be used.</p> </li> <li> <code>embed_continuous_method</code>               (<code>Optional[Literal[standard, piecewise, periodic]]</code>, default:                   <code>None</code> )           \u2013            <p>Method to use to embed the continuous features. Options are: 'standard', 'periodic' or 'piecewise'. The 'standard' embedding method is based on the FT-Transformer implementation presented in the paper: Revisiting Deep Learning Models for Tabular Data. The 'periodic' and_'piecewise'_ methods were presented in the paper: On Embeddings for Numerical Features in Tabular Deep Learning. Please, read the papers for details.</p> </li> <li> <code>cont_embed_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p> </li> <li> <code>cont_embed_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the continuous embeddings if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported. If <code>None</code>, no activation function will be applied.</p> </li> <li> <code>quantization_setup</code>               (<code>Optional[Dict[str, List[float]]]</code>, default:                   <code>None</code> )           \u2013            <p>This parameter is used when the 'piecewise' method is used to embed the continuous cols. It is a dict where keys are the name of the continuous columns and values are lists with the boundaries for the quantization of the continuous_cols. See the examples for details. If If the 'piecewise' method is used, this parameter is required.</p> </li> <li> <code>n_frequencies</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>This is the so called 'k' in their paper On Embeddings for Numerical Features in Tabular Deep Learning, and is the number of 'frequencies' that will be used to represent each continuous column. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>sigma</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>This is the sigma parameter in the paper mentioned when describing the previous parameters and it is used to initialise the 'frequency weights'. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>share_last_layer</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>This parameter is not present in the before mentioned paper but it is implemented in the official repo. If <code>True</code> the linear layer that turns the frequencies into embeddings will be shared across the continuous columns. If <code>False</code> a different linear layer will be used for each continuous column. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>full_embed_dropout</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>If <code>True</code>, the full embedding corresponding to a column will be masked out/dropout. If <code>None</code>, it will default to <code>False</code>.</p> </li> <li> <code>input_dim</code>               (<code>int</code>, default:                   <code>32</code> )           \u2013            <p>The so-called dimension of the model. Is the number of embeddings used to encode the categorical and/or continuous columns</p> </li> <li> <code>n_heads</code>               (<code>int</code>, default:                   <code>8</code> )           \u2013            <p>Number of attention heads per Transformer block</p> </li> <li> <code>use_qkv_bias</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether or not to use bias in the Q, K, and V projection layers.</p> </li> <li> <code>n_blocks</code>               (<code>int</code>, default:                   <code>4</code> )           \u2013            <p>Number of Transformer blocks</p> </li> <li> <code>attn_dropout</code>               (<code>float</code>, default:                   <code>0.2</code> )           \u2013            <p>Dropout that will be applied to the Multi-Head Attention layers</p> </li> <li> <code>ff_dropout</code>               (<code>float</code>, default:                   <code>0.1</code> )           \u2013            <p>Dropout that will be applied to the FeedForward network</p> </li> <li> <code>ff_factor</code>               (<code>int</code>, default:                   <code>4</code> )           \u2013            <p>Multiplicative factor applied to the first layer of the FF network in each Transformer block, This is normally set to 4.</p> </li> <li> <code>transformer_activation</code>               (<code>str</code>, default:                   <code>'gelu'</code> )           \u2013            <p>Transformer Encoder activation function. 'tanh', 'relu', 'leaky_relu', 'gelu', 'geglu' and 'reglu' are supported</p> </li> <li> <code>use_linear_attention</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating if Linear Attention (from Transformers are RNNs: Fast Autoregressive Transformers with Linear Attention) will be used. The inclusing of this mode of attention is inspired by this post, where the Uber team finds that this attention mechanism leads to the best results for their tabular data.</p> </li> <li> <code>use_flash_attention</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating if Flash Attention will be used. </p> </li> <li> <code>mlp_hidden_dims</code>               (<code>Optional[List[int]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the number of neurons per dense layer in the MLP. e.g: [64, 32]. If not provided no MLP on top of the final Transformer block will be used.</p> </li> <li> <code>mlp_activation</code>               (<code>str</code>, default:                   <code>'relu'</code> )           \u2013            <p>Activation function for the dense layers of the MLP. Currently 'tanh', 'relu', 'leaky'_relu' and _'gelu' are supported. If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to 'relu'.</p> </li> <li> <code>mlp_dropout</code>               (<code>float</code>, default:                   <code>0.1</code> )           \u2013            <p>float with the dropout between the dense layers of the MLP. If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to 0.0.</p> </li> <li> <code>mlp_batchnorm</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether or not batch normalization will be applied to the dense layers If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to False.</p> </li> <li> <code>mlp_batchnorm_last</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether or not batch normalization will be applied to the last of the dense layers If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to False.</p> </li> <li> <code>mlp_linear_first</code>               (<code>bool</code>, default:                   <code>True</code> )           \u2013            <p>Boolean indicating the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code> If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to <code>True</code>.</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>encoder</code>               (<code>Module</code>)           \u2013            <p>Sequence of Transformer blocks</p> </li> <li> <code>mlp</code>               (<code>Module</code>)           \u2013            <p>MLP component in the model</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import TabTransformer\n&gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n&gt;&gt;&gt; colnames = ['a', 'b', 'c', 'd', 'e']\n&gt;&gt;&gt; cat_embed_input = [(u,i) for u,i in zip(colnames[:4], [4]*4)]\n&gt;&gt;&gt; continuous_cols = ['e']\n&gt;&gt;&gt; column_idx = {k:v for v,k in enumerate(colnames)}\n&gt;&gt;&gt; model = TabTransformer(column_idx=column_idx, cat_embed_input=cat_embed_input, continuous_cols=continuous_cols)\n&gt;&gt;&gt; out = model(X_tab)\n</code></pre> Source code in <code>pytorch_widedeep/models/tabular/transformers/tab_transformer.py</code> <pre><code>def __init__(\n    self,\n    column_idx: Dict[str, int],\n    *,\n    cat_embed_input: Optional[List[Tuple[str, int]]] = None,\n    cat_embed_dropout: Optional[float] = None,\n    use_cat_bias: Optional[bool] = None,\n    cat_embed_activation: Optional[str] = None,\n    shared_embed: Optional[bool] = None,\n    add_shared_embed: Optional[bool] = None,\n    frac_shared_embed: Optional[float] = None,\n    continuous_cols: Optional[List[str]] = None,\n    cont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\n    embed_continuous: Optional[bool] = None,\n    embed_continuous_method: Optional[\n        Literal[\"standard\", \"piecewise\", \"periodic\"]\n    ] = None,\n    cont_embed_dropout: Optional[float] = None,\n    cont_embed_activation: Optional[str] = None,\n    quantization_setup: Optional[Dict[str, List[float]]] = None,\n    n_frequencies: Optional[int] = None,\n    sigma: Optional[float] = None,\n    share_last_layer: Optional[bool] = None,\n    full_embed_dropout: Optional[bool] = None,\n    input_dim: int = 32,\n    n_heads: int = 8,\n    use_qkv_bias: bool = False,\n    n_blocks: int = 4,\n    attn_dropout: float = 0.2,\n    ff_dropout: float = 0.1,\n    ff_factor: int = 4,\n    transformer_activation: str = \"gelu\",\n    use_linear_attention: bool = False,\n    use_flash_attention: bool = False,\n    mlp_hidden_dims: Optional[List[int]] = None,\n    mlp_activation: str = \"relu\",\n    mlp_dropout: float = 0.1,\n    mlp_batchnorm: bool = False,\n    mlp_batchnorm_last: bool = False,\n    mlp_linear_first: bool = True,\n):\n    super(TabTransformer, self).__init__(\n        column_idx=column_idx,\n        cat_embed_input=cat_embed_input,\n        cat_embed_dropout=cat_embed_dropout,\n        use_cat_bias=use_cat_bias,\n        cat_embed_activation=cat_embed_activation,\n        shared_embed=shared_embed,\n        add_shared_embed=add_shared_embed,\n        frac_shared_embed=frac_shared_embed,\n        continuous_cols=continuous_cols,\n        cont_norm_layer=cont_norm_layer,\n        # UGLY hack to be able to use the base class __init__ method\n        embed_continuous_method=(\n            \"standard\"\n            if embed_continuous_method is None\n            else embed_continuous_method\n        ),\n        cont_embed_dropout=cont_embed_dropout,\n        cont_embed_activation=cont_embed_activation,\n        quantization_setup=quantization_setup,\n        n_frequencies=n_frequencies,\n        sigma=sigma,\n        share_last_layer=share_last_layer,\n        input_dim=input_dim,\n        full_embed_dropout=full_embed_dropout,\n    )\n\n    # we overwrite the embed_continuous_method parameter that was set\n    # to 'standard' if embed_continuous_method is None. In other words,\n    # this will already have the right value if the user provided a value\n    # for embed_continuous_method, otherwise will be \"standard\" when it\n    # should be None\n    self.embed_continuous_method = embed_continuous_method\n\n    if embed_continuous is not None:\n        self.embed_continuous = embed_continuous\n        if embed_continuous and embed_continuous_method is None:\n            raise ValueError(\n                \"If 'embed_continuous' is True, 'embed_continuous_method' must be \"\n                \"one of 'standard', 'piecewise' or 'periodic'.\"\n            )\n    elif self.embed_continuous_method is not None:\n        self.embed_continuous = True\n    else:\n        self.embed_continuous = False\n\n    self.n_heads = n_heads\n    self.use_qkv_bias = use_qkv_bias\n    self.n_blocks = n_blocks\n    self.attn_dropout = attn_dropout\n    self.ff_dropout = ff_dropout\n    self.transformer_activation = transformer_activation\n    self.use_linear_attention = use_linear_attention\n    self.use_flash_attention = use_flash_attention\n    self.ff_factor = ff_factor\n\n    self.mlp_hidden_dims = mlp_hidden_dims\n    self.mlp_activation = mlp_activation\n    self.mlp_dropout = mlp_dropout\n    self.mlp_batchnorm = mlp_batchnorm\n    self.mlp_batchnorm_last = mlp_batchnorm_last\n    self.mlp_linear_first = mlp_linear_first\n\n    self.with_cls_token = \"cls_token\" in column_idx\n    self.n_cat = len(cat_embed_input) if cat_embed_input is not None else 0\n    self.n_cont = len(continuous_cols) if continuous_cols is not None else 0\n\n    if self.n_cont and not self.n_cat and not self.embed_continuous:\n        raise ValueError(\n            \"If only continuous features are used 'embed_continuous' must be set to 'True'\"\n        )\n\n    # Embeddings are instantiated at the base model\n    # Transformer blocks\n    self.encoder = nn.Sequential()\n    for i in range(n_blocks):\n        self.encoder.add_module(\n            \"transformer_block\" + str(i),\n            TransformerEncoder(\n                input_dim,\n                n_heads,\n                use_qkv_bias,\n                attn_dropout,\n                ff_dropout,\n                ff_factor,\n                transformer_activation,\n                use_linear_attention,\n                use_flash_attention,\n            ),\n        )\n\n    self.mlp_first_hidden_dim = self._mlp_first_hidden_dim()\n\n    if self.mlp_hidden_dims is not None:\n        self.mlp = MLP(\n            d_hidden=[self.mlp_first_hidden_dim] + self.mlp_hidden_dims,\n            activation=(\n                \"relu\" if self.mlp_activation is None else self.mlp_activation\n            ),\n            dropout=0.0 if self.mlp_dropout is None else self.mlp_dropout,\n            batchnorm=False if self.mlp_batchnorm is None else self.mlp_batchnorm,\n            batchnorm_last=(\n                False\n                if self.mlp_batchnorm_last is None\n                else self.mlp_batchnorm_last\n            ),\n            linear_first=(\n                False if self.mlp_linear_first is None else self.mlp_linear_first\n            ),\n        )\n    else:\n        self.mlp = None\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.transformers.tab_transformer.TabTransformer.output_dim","title":"output_dim  <code>property</code>","text":"<pre><code>output_dim\n</code></pre> <p>The output dimension of the model. This is a required property neccesary to build the <code>WideDeep</code> class</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.transformers.tab_transformer.TabTransformer.attention_weights","title":"attention_weights  <code>property</code>","text":"<pre><code>attention_weights\n</code></pre> <p>List with the attention weights per block</p> <p>The shape of the attention weights is \\((N, H, F, F)\\), where \\(N\\) is the batch size, \\(H\\) is the number of attention heads and \\(F\\) is the number of features/columns in the dataset</p> <p> NOTE: if flash attention or linear attention are used, no attention weights are saved during the training process and calling this property will throw a ValueError</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.transformers.saint.SAINT","title":"SAINT","text":"<pre><code>SAINT(\n    column_idx,\n    *,\n    cat_embed_input=None,\n    cat_embed_dropout=None,\n    use_cat_bias=None,\n    cat_embed_activation=None,\n    shared_embed=None,\n    add_shared_embed=None,\n    frac_shared_embed=None,\n    continuous_cols=None,\n    cont_norm_layer=None,\n    embed_continuous_method=\"standard\",\n    cont_embed_dropout=None,\n    cont_embed_activation=None,\n    quantization_setup=None,\n    n_frequencies=None,\n    sigma=None,\n    share_last_layer=None,\n    full_embed_dropout=None,\n    input_dim=32,\n    use_qkv_bias=False,\n    n_heads=8,\n    n_blocks=2,\n    attn_dropout=0.1,\n    ff_dropout=0.2,\n    ff_factor=4,\n    transformer_activation=\"gelu\",\n    mlp_hidden_dims=None,\n    mlp_activation=None,\n    mlp_dropout=None,\n    mlp_batchnorm=None,\n    mlp_batchnorm_last=None,\n    mlp_linear_first=None\n)\n</code></pre> <p>               Bases: <code>BaseTabularModelWithAttention</code></p> <p>Defines a SAINT model that can be used as the <code>deeptabular</code> component of a Wide &amp; Deep model or independently by itself.</p> <p>Most of the parameters for this class are <code>Optional</code> since the use of categorical or continuous is in fact optional (i.e. one can use categorical features only, continuous features only or both).</p> <p> NOTE: This is an slightly modified and enhanced  version of the model described in the paper,</p> <p>Parameters:</p> <ul> <li> <code>column_idx</code>               (<code>Dict[str, int]</code>)           \u2013            <p>Dict containing the index of the columns that will be passed through the <code>TabMlp</code> model. Required to slice the tensors. e.g. {'education': 0, 'relationship': 1, 'workclass': 2, ...}.</p> </li> <li> <code>cat_embed_input</code>               (<code>Optional[List[Tuple[str, int]]]</code>, default:                   <code>None</code> )           \u2013            <p>List of Tuples with the column name and number of unique values and embedding dimension. e.g. [(education, 11), ...]</p> </li> <li> <code>cat_embed_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Categorical embeddings dropout. If <code>None</code>, it will default to 0.</p> </li> <li> <code>use_cat_bias</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating if bias will be used for the categorical embeddings. If <code>None</code>, it will default to 'False'.</p> </li> <li> <code>cat_embed_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the categorical embeddings, if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> </li> <li> <code>shared_embed</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating if the embeddings will be \"shared\". The idea behind <code>shared_embed</code> is described in the Appendix A in the TabTransformer paper: 'The goal of having column embedding is to enable the model to distinguish the classes in one column from those in the other columns'. In other words, the idea is to let the model learn which column is embedded at the time. See: <code>pytorch_widedeep.models.transformers._layers.SharedEmbeddings</code>.</p> </li> <li> <code>add_shared_embed</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>The two embedding sharing strategies are: 1) add the shared embeddings to the column embeddings or 2) to replace the first <code>frac_shared_embed</code> with the shared embeddings. See <code>pytorch_widedeep.models.embeddings_layers.SharedEmbeddings</code> If 'None' is passed, it will default to 'False'.</p> </li> <li> <code>frac_shared_embed</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>The fraction of embeddings that will be shared (if <code>add_shared_embed = False</code>) by all the different categories for one particular column. If 'None' is passed, it will default to 0.0.</p> </li> <li> <code>continuous_cols</code>               (<code>Optional[List[str]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the name of the numeric (aka continuous) columns</p> </li> <li> <code>cont_norm_layer</code>               (<code>Optional[Literal[batchnorm, layernorm]]</code>, default:                   <code>None</code> )           \u2013            <p>Type of normalization layer applied to the continuous features. Options are: 'layernorm' and 'batchnorm'. if <code>None</code>, no normalization layer will be used.</p> </li> <li> <code>embed_continuous_method</code>               (<code>Optional[Literal[standard, piecewise, periodic]]</code>, default:                   <code>'standard'</code> )           \u2013            <p>Method to use to embed the continuous features. Options are: 'standard', 'periodic' or 'piecewise'. The 'standard' embedding method is based on the FT-Transformer implementation presented in the paper: Revisiting Deep Learning Models for Tabular Data. The 'periodic' and_'piecewise'_ methods were presented in the paper: On Embeddings for Numerical Features in Tabular Deep Learning. Please, read the papers for details.</p> </li> <li> <code>cont_embed_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p> </li> <li> <code>cont_embed_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the continuous embeddings if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported. If <code>None</code>, no activation function will be applied.</p> </li> <li> <code>quantization_setup</code>               (<code>Optional[Dict[str, List[float]]]</code>, default:                   <code>None</code> )           \u2013            <p>This parameter is used when the 'piecewise' method is used to embed the continuous cols. It is a dict where keys are the name of the continuous columns and values are lists with the boundaries for the quantization of the continuous_cols. See the examples for details. If If the 'piecewise' method is used, this parameter is required.</p> </li> <li> <code>n_frequencies</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>This is the so called 'k' in their paper On Embeddings for Numerical Features in Tabular Deep Learning, and is the number of 'frequencies' that will be used to represent each continuous column. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>sigma</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>This is the sigma parameter in the paper mentioned when describing the previous parameters and it is used to initialise the 'frequency weights'. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>share_last_layer</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>This parameter is not present in the before mentioned paper but it is implemented in the official repo. If <code>True</code> the linear layer that turns the frequencies into embeddings will be shared across the continuous columns. If <code>False</code> a different linear layer will be used for each continuous column. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>full_embed_dropout</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>If <code>True</code>, the full embedding corresponding to a column will be masked out/dropout. If <code>None</code>, it will default to <code>False</code>.</p> </li> <li> <code>input_dim</code>               (<code>int</code>, default:                   <code>32</code> )           \u2013            <p>The so-called dimension of the model. Is the number of embeddings used to encode the categorical and/or continuous columns</p> </li> <li> <code>n_heads</code>               (<code>int</code>, default:                   <code>8</code> )           \u2013            <p>Number of attention heads per Transformer block</p> </li> <li> <code>use_qkv_bias</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether or not to use bias in the Q, K, and V projection layers</p> </li> <li> <code>n_blocks</code>               (<code>int</code>, default:                   <code>2</code> )           \u2013            <p>Number of SAINT-Transformer blocks.</p> </li> <li> <code>attn_dropout</code>               (<code>float</code>, default:                   <code>0.1</code> )           \u2013            <p>Dropout that will be applied to the Multi-Head Attention column and row layers</p> </li> <li> <code>ff_dropout</code>               (<code>float</code>, default:                   <code>0.2</code> )           \u2013            <p>Dropout that will be applied to the FeedForward network</p> </li> <li> <code>ff_factor</code>               (<code>int</code>, default:                   <code>4</code> )           \u2013            <p>Multiplicative factor applied to the first layer of the FF network in each Transformer block, This is normally set to 4.</p> </li> <li> <code>transformer_activation</code>               (<code>str</code>, default:                   <code>'gelu'</code> )           \u2013            <p>Transformer Encoder activation function. 'tanh', 'relu', 'leaky_relu', 'gelu', 'geglu' and 'reglu' are supported</p> </li> <li> <code>mlp_hidden_dims</code>               (<code>Optional[List[int]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the number of neurons per dense layer in the MLP. e.g: [64, 32]. If not provided no MLP on top of the final Transformer block will be used.</p> </li> <li> <code>mlp_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the dense layers of the MLP. Currently 'tanh', 'relu', 'leaky'_relu' and _'gelu' are supported. If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to 'relu'.</p> </li> <li> <code>mlp_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>float with the dropout between the dense layers of the MLP. If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to 0.0.</p> </li> <li> <code>mlp_batchnorm</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating whether or not batch normalization will be applied to the dense layers If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to False.</p> </li> <li> <code>mlp_batchnorm_last</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating whether or not batch normalization will be applied to the last of the dense layers If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to False.</p> </li> <li> <code>mlp_linear_first</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code> If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to <code>True</code>.</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>encoder</code>               (<code>Module</code>)           \u2013            <p>Sequence of SAINT-Transformer blocks</p> </li> <li> <code>mlp</code>               (<code>Module</code>)           \u2013            <p>MLP component in the model</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import SAINT\n&gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n&gt;&gt;&gt; colnames = ['a', 'b', 'c', 'd', 'e']\n&gt;&gt;&gt; cat_embed_input = [(u,i) for u,i in zip(colnames[:4], [4]*4)]\n&gt;&gt;&gt; continuous_cols = ['e']\n&gt;&gt;&gt; column_idx = {k:v for v,k in enumerate(colnames)}\n&gt;&gt;&gt; model = SAINT(column_idx=column_idx, cat_embed_input=cat_embed_input, continuous_cols=continuous_cols)\n&gt;&gt;&gt; out = model(X_tab)\n</code></pre> Source code in <code>pytorch_widedeep/models/tabular/transformers/saint.py</code> <pre><code>def __init__(\n    self,\n    column_idx: Dict[str, int],\n    *,\n    cat_embed_input: Optional[List[Tuple[str, int]]] = None,\n    cat_embed_dropout: Optional[float] = None,\n    use_cat_bias: Optional[bool] = None,\n    cat_embed_activation: Optional[str] = None,\n    shared_embed: Optional[bool] = None,\n    add_shared_embed: Optional[bool] = None,\n    frac_shared_embed: Optional[float] = None,\n    continuous_cols: Optional[List[str]] = None,\n    cont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\n    embed_continuous_method: Optional[\n        Literal[\"standard\", \"piecewise\", \"periodic\"]\n    ] = \"standard\",\n    cont_embed_dropout: Optional[float] = None,\n    cont_embed_activation: Optional[str] = None,\n    quantization_setup: Optional[Dict[str, List[float]]] = None,\n    n_frequencies: Optional[int] = None,\n    sigma: Optional[float] = None,\n    share_last_layer: Optional[bool] = None,\n    full_embed_dropout: Optional[bool] = None,\n    input_dim: int = 32,\n    use_qkv_bias: bool = False,\n    n_heads: int = 8,\n    n_blocks: int = 2,\n    attn_dropout: float = 0.1,\n    ff_dropout: float = 0.2,\n    ff_factor: int = 4,\n    transformer_activation: str = \"gelu\",\n    mlp_hidden_dims: Optional[List[int]] = None,\n    mlp_activation: Optional[str] = None,\n    mlp_dropout: Optional[float] = None,\n    mlp_batchnorm: Optional[bool] = None,\n    mlp_batchnorm_last: Optional[bool] = None,\n    mlp_linear_first: Optional[bool] = None,\n):\n    super(SAINT, self).__init__(\n        column_idx=column_idx,\n        cat_embed_input=cat_embed_input,\n        cat_embed_dropout=cat_embed_dropout,\n        use_cat_bias=use_cat_bias,\n        cat_embed_activation=cat_embed_activation,\n        shared_embed=shared_embed,\n        add_shared_embed=add_shared_embed,\n        frac_shared_embed=frac_shared_embed,\n        continuous_cols=continuous_cols,\n        cont_norm_layer=cont_norm_layer,\n        embed_continuous_method=embed_continuous_method,\n        cont_embed_dropout=cont_embed_dropout,\n        cont_embed_activation=cont_embed_activation,\n        input_dim=input_dim,\n        quantization_setup=quantization_setup,\n        n_frequencies=n_frequencies,\n        sigma=sigma,\n        share_last_layer=share_last_layer,\n        full_embed_dropout=full_embed_dropout,\n    )\n\n    self.use_qkv_bias = use_qkv_bias\n    self.n_heads = n_heads\n    self.n_blocks = n_blocks\n    self.attn_dropout = attn_dropout\n    self.ff_dropout = ff_dropout\n    self.ff_factor = ff_factor\n    self.transformer_activation = transformer_activation\n\n    self.mlp_hidden_dims = mlp_hidden_dims\n    self.mlp_activation = mlp_activation\n    self.mlp_dropout = mlp_dropout\n    self.mlp_batchnorm = mlp_batchnorm\n    self.mlp_batchnorm_last = mlp_batchnorm_last\n    self.mlp_linear_first = mlp_linear_first\n\n    self.with_cls_token = \"cls_token\" in column_idx\n    self.n_cat = len(cat_embed_input) if cat_embed_input is not None else 0\n    self.n_cont = len(continuous_cols) if continuous_cols is not None else 0\n    self.n_feats = self.n_cat + self.n_cont\n\n    # Embeddings are instantiated at the base model\n    # Transformer blocks\n    self.encoder = nn.Sequential()\n    for i in range(n_blocks):\n        self.encoder.add_module(\n            \"saint_block\" + str(i),\n            SaintEncoder(\n                input_dim,\n                n_heads,\n                use_qkv_bias,\n                attn_dropout,\n                ff_dropout,\n                ff_factor,\n                transformer_activation,\n                self.n_feats,\n            ),\n        )\n\n    self.mlp_first_hidden_dim = (\n        self.input_dim if self.with_cls_token else (self.n_feats * self.input_dim)\n    )\n\n    # Mlp: adding an MLP on top of the Resnet blocks is optional and\n    # therefore all related params are optional\n    if self.mlp_hidden_dims is not None:\n        self.mlp = MLP(\n            d_hidden=[self.mlp_first_hidden_dim] + self.mlp_hidden_dims,\n            activation=(\n                \"relu\" if self.mlp_activation is None else self.mlp_activation\n            ),\n            dropout=0.0 if self.mlp_dropout is None else self.mlp_dropout,\n            batchnorm=False if self.mlp_batchnorm is None else self.mlp_batchnorm,\n            batchnorm_last=(\n                False\n                if self.mlp_batchnorm_last is None\n                else self.mlp_batchnorm_last\n            ),\n            linear_first=(\n                False if self.mlp_linear_first is None else self.mlp_linear_first\n            ),\n        )\n    else:\n        self.mlp = None\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.transformers.saint.SAINT.output_dim","title":"output_dim  <code>property</code>","text":"<pre><code>output_dim\n</code></pre> <p>The output dimension of the model. This is a required property neccesary to build the <code>WideDeep</code> class</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.transformers.saint.SAINT.attention_weights","title":"attention_weights  <code>property</code>","text":"<pre><code>attention_weights\n</code></pre> <p>List with the attention weights. Each element of the list is a tuple where the first and the second elements are the column and row attention weights respectively</p> <p>The shape of the attention weights is:</p> <ul> <li> <p>column attention: \\((N, H, F, F)\\)</p> </li> <li> <p>row attention: \\((1, H, N, N)\\)</p> </li> </ul> <p>where \\(N\\) is the batch size, \\(H\\) is the number of heads and \\(F\\) is the number of features/columns in the dataset</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.transformers.ft_transformer.FTTransformer","title":"FTTransformer","text":"<pre><code>FTTransformer(\n    column_idx,\n    *,\n    cat_embed_input=None,\n    cat_embed_dropout=None,\n    use_cat_bias=None,\n    cat_embed_activation=None,\n    shared_embed=None,\n    add_shared_embed=None,\n    frac_shared_embed=None,\n    continuous_cols=None,\n    cont_norm_layer=None,\n    embed_continuous_method=\"standard\",\n    cont_embed_dropout=None,\n    cont_embed_activation=None,\n    quantization_setup=None,\n    n_frequencies=None,\n    sigma=None,\n    share_last_layer=None,\n    full_embed_dropout=None,\n    input_dim=64,\n    kv_compression_factor=0.5,\n    kv_sharing=False,\n    use_qkv_bias=False,\n    n_heads=8,\n    n_blocks=4,\n    attn_dropout=0.2,\n    ff_dropout=0.1,\n    ff_factor=1.33,\n    transformer_activation=\"reglu\",\n    mlp_hidden_dims=None,\n    mlp_activation=None,\n    mlp_dropout=None,\n    mlp_batchnorm=None,\n    mlp_batchnorm_last=None,\n    mlp_linear_first=None\n)\n</code></pre> <p>               Bases: <code>BaseTabularModelWithAttention</code></p> <p>Defines a FTTransformer model that can be used as the <code>deeptabular</code> component of a Wide &amp; Deep model or independently by itself.</p> <p>Most of the parameters for this class are <code>Optional</code> since the use of categorical or continuous is in fact optional (i.e. one can use categorical features only, continuous features only or both).</p> <p>Parameters:</p> <ul> <li> <code>column_idx</code>               (<code>Dict[str, int]</code>)           \u2013            <p>Dict containing the index of the columns that will be passed through the <code>TabMlp</code> model. Required to slice the tensors. e.g. {'education': 0, 'relationship': 1, 'workclass': 2, ...}.</p> </li> <li> <code>cat_embed_input</code>               (<code>Optional[List[Tuple[str, int]]]</code>, default:                   <code>None</code> )           \u2013            <p>List of Tuples with the column name and number of unique values and embedding dimension. e.g. [(education, 11), ...]</p> </li> <li> <code>cat_embed_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Categorical embeddings dropout. If <code>None</code>, it will default to 0.</p> </li> <li> <code>use_cat_bias</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating if bias will be used for the categorical embeddings. If <code>None</code>, it will default to 'False'.</p> </li> <li> <code>cat_embed_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the categorical embeddings, if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> </li> <li> <code>shared_embed</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating if the embeddings will be \"shared\". The idea behind <code>shared_embed</code> is described in the Appendix A in the TabTransformer paper: 'The goal of having column embedding is to enable the model to distinguish the classes in one column from those in the other columns'. In other words, the idea is to let the model learn which column is embedded at the time. See: <code>pytorch_widedeep.models.transformers._layers.SharedEmbeddings</code>.</p> </li> <li> <code>add_shared_embed</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>The two embedding sharing strategies are: 1) add the shared embeddings to the column embeddings or 2) to replace the first <code>frac_shared_embed</code> with the shared embeddings. See <code>pytorch_widedeep.models.embeddings_layers.SharedEmbeddings</code> If 'None' is passed, it will default to 'False'.</p> </li> <li> <code>frac_shared_embed</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>The fraction of embeddings that will be shared (if <code>add_shared_embed = False</code>) by all the different categories for one particular column. If 'None' is passed, it will default to 0.0.</p> </li> <li> <code>continuous_cols</code>               (<code>Optional[List[str]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the name of the numeric (aka continuous) columns</p> </li> <li> <code>cont_norm_layer</code>               (<code>Optional[Literal[batchnorm, layernorm]]</code>, default:                   <code>None</code> )           \u2013            <p>Type of normalization layer applied to the continuous features. Options are: 'layernorm' and 'batchnorm'. if <code>None</code>, no normalization layer will be used.</p> </li> <li> <code>embed_continuous_method</code>               (<code>Optional[Literal[standard, piecewise, periodic]]</code>, default:                   <code>'standard'</code> )           \u2013            <p>Method to use to embed the continuous features. Options are: 'standard', 'periodic' or 'piecewise'. The 'standard' embedding method is based on the FT-Transformer implementation presented in the paper: Revisiting Deep Learning Models for Tabular Data. The 'periodic' and_'piecewise'_ methods were presented in the paper: On Embeddings for Numerical Features in Tabular Deep Learning. Please, read the papers for details.</p> </li> <li> <code>cont_embed_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p> </li> <li> <code>cont_embed_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the continuous embeddings if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported. If <code>None</code>, no activation function will be applied.</p> </li> <li> <code>quantization_setup</code>               (<code>Optional[Dict[str, List[float]]]</code>, default:                   <code>None</code> )           \u2013            <p>This parameter is used when the 'piecewise' method is used to embed the continuous cols. It is a dict where keys are the name of the continuous columns and values are lists with the boundaries for the quantization of the continuous_cols. See the examples for details. If If the 'piecewise' method is used, this parameter is required.</p> </li> <li> <code>n_frequencies</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>This is the so called 'k' in their paper On Embeddings for Numerical Features in Tabular Deep Learning, and is the number of 'frequencies' that will be used to represent each continuous column. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>sigma</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>This is the sigma parameter in the paper mentioned when describing the previous parameters and it is used to initialise the 'frequency weights'. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>share_last_layer</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>This parameter is not present in the before mentioned paper but it is implemented in the official repo. If <code>True</code> the linear layer that turns the frequencies into embeddings will be shared across the continuous columns. If <code>False</code> a different linear layer will be used for each continuous column. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>full_embed_dropout</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>If <code>True</code>, the full embedding corresponding to a column will be masked out/dropout. If <code>None</code>, it will default to <code>False</code>.</p> </li> <li> <code>input_dim</code>               (<code>int</code>, default:                   <code>64</code> )           \u2013            <p>The so-called dimension of the model. Is the number of embeddings used to encode the categorical and/or continuous columns.</p> </li> <li> <code>kv_compression_factor</code>               (<code>float</code>, default:                   <code>0.5</code> )           \u2013            <p>By default, the FTTransformer uses Linear Attention (See Linformer: Self-Attention with Linear Complexity ). The compression factor that will be used to reduce the input sequence length. If we denote the resulting sequence length as \\(k = int(kv_{compression \\space factor} \\times s)\\) where \\(s\\) is the input sequence length.</p> </li> <li> <code>kv_sharing</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating if the \\(E\\) and \\(F\\) projection matrices will share weights.  See Linformer: Self-Attention with Linear Complexity for details</p> </li> <li> <code>n_heads</code>               (<code>int</code>, default:                   <code>8</code> )           \u2013            <p>Number of attention heads per FTTransformer block</p> </li> <li> <code>use_qkv_bias</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether or not to use bias in the Q, K, and V projection layers</p> </li> <li> <code>n_blocks</code>               (<code>int</code>, default:                   <code>4</code> )           \u2013            <p>Number of FTTransformer blocks</p> </li> <li> <code>attn_dropout</code>               (<code>float</code>, default:                   <code>0.2</code> )           \u2013            <p>Dropout that will be applied to the Linear-Attention layers</p> </li> <li> <code>ff_dropout</code>               (<code>float</code>, default:                   <code>0.1</code> )           \u2013            <p>Dropout that will be applied to the FeedForward network</p> </li> <li> <code>ff_factor</code>               (<code>float</code>, default:                   <code>1.33</code> )           \u2013            <p>Multiplicative factor applied to the first layer of the FF network in each Transformer block, This is normally set to 4, but they use 4/3 in the paper.</p> </li> <li> <code>transformer_activation</code>               (<code>str</code>, default:                   <code>'reglu'</code> )           \u2013            <p>Transformer Encoder activation function. 'tanh', 'relu', 'leaky_relu', 'gelu', 'geglu' and 'reglu' are supported</p> </li> <li> <code>mlp_hidden_dims</code>               (<code>Optional[List[int]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the number of neurons per dense layer in the MLP. e.g: [64, 32]. If not provided no MLP on top of the final FTTransformer block will be used.</p> </li> <li> <code>mlp_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the dense layers of the MLP. Currently 'tanh', 'relu', 'leaky'_relu' and _'gelu' are supported. If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to 'relu'.</p> </li> <li> <code>mlp_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>float with the dropout between the dense layers of the MLP. If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to 0.0.</p> </li> <li> <code>mlp_batchnorm</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating whether or not batch normalization will be applied to the dense layers If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to False.</p> </li> <li> <code>mlp_batchnorm_last</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating whether or not batch normalization will be applied to the last of the dense layers If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to False.</p> </li> <li> <code>mlp_linear_first</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code> If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to <code>True</code>.</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>encoder</code>               (<code>Module</code>)           \u2013            <p>Sequence of FTTransformer blocks</p> </li> <li> <code>mlp</code>               (<code>Module</code>)           \u2013            <p>MLP component in the model</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import FTTransformer\n&gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n&gt;&gt;&gt; colnames = ['a', 'b', 'c', 'd', 'e']\n&gt;&gt;&gt; cat_embed_input = [(u,i) for u,i in zip(colnames[:4], [4]*4)]\n&gt;&gt;&gt; continuous_cols = ['e']\n&gt;&gt;&gt; column_idx = {k:v for v,k in enumerate(colnames)}\n&gt;&gt;&gt; model = FTTransformer(column_idx=column_idx, cat_embed_input=cat_embed_input, continuous_cols=continuous_cols)\n&gt;&gt;&gt; out = model(X_tab)\n</code></pre> Source code in <code>pytorch_widedeep/models/tabular/transformers/ft_transformer.py</code> <pre><code>def __init__(\n    self,\n    column_idx: Dict[str, int],\n    *,\n    cat_embed_input: Optional[List[Tuple[str, int]]] = None,\n    cat_embed_dropout: Optional[float] = None,\n    use_cat_bias: Optional[bool] = None,\n    cat_embed_activation: Optional[str] = None,\n    shared_embed: Optional[bool] = None,\n    add_shared_embed: Optional[bool] = None,\n    frac_shared_embed: Optional[float] = None,\n    continuous_cols: Optional[List[str]] = None,\n    cont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\n    embed_continuous_method: Optional[\n        Literal[\"standard\", \"piecewise\", \"periodic\"]\n    ] = \"standard\",\n    cont_embed_dropout: Optional[float] = None,\n    cont_embed_activation: Optional[str] = None,\n    quantization_setup: Optional[Dict[str, List[float]]] = None,\n    n_frequencies: Optional[int] = None,\n    sigma: Optional[float] = None,\n    share_last_layer: Optional[bool] = None,\n    full_embed_dropout: Optional[bool] = None,\n    input_dim: int = 64,\n    kv_compression_factor: float = 0.5,\n    kv_sharing: bool = False,\n    use_qkv_bias: bool = False,\n    n_heads: int = 8,\n    n_blocks: int = 4,\n    attn_dropout: float = 0.2,\n    ff_dropout: float = 0.1,\n    ff_factor: float = 1.33,\n    transformer_activation: str = \"reglu\",\n    mlp_hidden_dims: Optional[List[int]] = None,\n    mlp_activation: Optional[str] = None,\n    mlp_dropout: Optional[float] = None,\n    mlp_batchnorm: Optional[bool] = None,\n    mlp_batchnorm_last: Optional[bool] = None,\n    mlp_linear_first: Optional[bool] = None,\n):\n    super(FTTransformer, self).__init__(\n        column_idx=column_idx,\n        cat_embed_input=cat_embed_input,\n        cat_embed_dropout=cat_embed_dropout,\n        use_cat_bias=use_cat_bias,\n        cat_embed_activation=cat_embed_activation,\n        shared_embed=shared_embed,\n        add_shared_embed=add_shared_embed,\n        frac_shared_embed=frac_shared_embed,\n        continuous_cols=continuous_cols,\n        cont_norm_layer=cont_norm_layer,\n        embed_continuous_method=embed_continuous_method,\n        cont_embed_dropout=cont_embed_dropout,\n        cont_embed_activation=cont_embed_activation,\n        input_dim=input_dim,\n        quantization_setup=quantization_setup,\n        n_frequencies=n_frequencies,\n        sigma=sigma,\n        share_last_layer=share_last_layer,\n        full_embed_dropout=full_embed_dropout,\n    )\n\n    self.kv_compression_factor = kv_compression_factor\n    self.kv_sharing = kv_sharing\n    self.use_qkv_bias = use_qkv_bias\n    self.n_heads = n_heads\n    self.n_blocks = n_blocks\n    self.attn_dropout = attn_dropout\n    self.ff_dropout = ff_dropout\n    self.ff_factor = ff_factor\n    self.transformer_activation = transformer_activation\n\n    self.mlp_hidden_dims = mlp_hidden_dims\n    self.mlp_activation = mlp_activation\n    self.mlp_dropout = mlp_dropout\n    self.mlp_batchnorm = mlp_batchnorm\n    self.mlp_batchnorm_last = mlp_batchnorm_last\n    self.mlp_linear_first = mlp_linear_first\n\n    self.with_cls_token = \"cls_token\" in column_idx\n    self.n_cat = len(cat_embed_input) if cat_embed_input is not None else 0\n    self.n_cont = len(continuous_cols) if continuous_cols is not None else 0\n    self.n_feats = self.n_cat + self.n_cont\n\n    # Embeddings are instantiated at the base model\n    # Transformer blocks\n    is_first = True\n    self.encoder = nn.Sequential()\n    for i in range(n_blocks):\n        self.encoder.add_module(\n            \"fttransformer_block\" + str(i),\n            FTTransformerEncoder(\n                input_dim,\n                self.n_feats,\n                n_heads,\n                use_qkv_bias,\n                attn_dropout,\n                ff_dropout,\n                ff_factor,\n                kv_compression_factor,\n                kv_sharing,\n                transformer_activation,\n                is_first,\n            ),\n        )\n        is_first = False\n\n    self.mlp_first_hidden_dim = (\n        self.input_dim if self.with_cls_token else (self.n_feats * self.input_dim)\n    )\n\n    # Mlp: adding an MLP on top of the Resnet blocks is optional and\n    # therefore all related params are optional\n    if self.mlp_hidden_dims is not None:\n        self.mlp = MLP(\n            d_hidden=[self.mlp_first_hidden_dim] + self.mlp_hidden_dims,\n            activation=(\n                \"relu\" if self.mlp_activation is None else self.mlp_activation\n            ),\n            dropout=0.0 if self.mlp_dropout is None else self.mlp_dropout,\n            batchnorm=False if self.mlp_batchnorm is None else self.mlp_batchnorm,\n            batchnorm_last=(\n                False\n                if self.mlp_batchnorm_last is None\n                else self.mlp_batchnorm_last\n            ),\n            linear_first=(\n                False if self.mlp_linear_first is None else self.mlp_linear_first\n            ),\n        )\n    else:\n        self.mlp = None\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.transformers.ft_transformer.FTTransformer.output_dim","title":"output_dim  <code>property</code>","text":"<pre><code>output_dim\n</code></pre> <p>The output dimension of the model. This is a required property neccesary to build the <code>WideDeep</code> class</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.transformers.ft_transformer.FTTransformer.attention_weights","title":"attention_weights  <code>property</code>","text":"<pre><code>attention_weights\n</code></pre> <p>List with the attention weights per block</p> <p>The shape of the attention weights is: \\((N, H, F, k)\\), where \\(N\\) is the batch size, \\(H\\) is the number of attention heads, \\(F\\) is the number of features/columns and \\(k\\) is the reduced sequence length or dimension, i.e. \\(k = int(kv_{compression \\space factor} \\times s)\\)</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.transformers.tab_perceiver.TabPerceiver","title":"TabPerceiver","text":"<pre><code>TabPerceiver(\n    column_idx,\n    *,\n    cat_embed_input=None,\n    cat_embed_dropout=None,\n    use_cat_bias=None,\n    cat_embed_activation=None,\n    shared_embed=None,\n    add_shared_embed=None,\n    frac_shared_embed=None,\n    continuous_cols=None,\n    cont_norm_layer=None,\n    embed_continuous_method=\"standard\",\n    cont_embed_dropout=None,\n    cont_embed_activation=None,\n    quantization_setup=None,\n    n_frequencies=None,\n    sigma=None,\n    share_last_layer=None,\n    full_embed_dropout=None,\n    input_dim=32,\n    n_cross_attns=1,\n    n_cross_attn_heads=4,\n    n_latents=16,\n    latent_dim=128,\n    n_latent_heads=4,\n    n_latent_blocks=4,\n    n_perceiver_blocks=4,\n    share_weights=False,\n    attn_dropout=0.1,\n    ff_dropout=0.1,\n    ff_factor=4,\n    transformer_activation=\"geglu\",\n    mlp_hidden_dims=None,\n    mlp_activation=None,\n    mlp_dropout=None,\n    mlp_batchnorm=None,\n    mlp_batchnorm_last=None,\n    mlp_linear_first=None\n)\n</code></pre> <p>               Bases: <code>BaseTabularModelWithAttention</code></p> <p>Defines an adaptation of a Perceiver  that can be used as the <code>deeptabular</code> component of a Wide &amp; Deep model  or independently by itself.</p> <p>Most of the parameters for this class are <code>Optional</code> since the use of categorical or continuous is in fact optional (i.e. one can use categorical features only, continuous features only or both).</p> <p> NOTE: while there are scientific publications for  the <code>TabTransformer</code>, <code>SAINT</code> and <code>FTTransformer</code>, the <code>TabPerceiver</code>  and the <code>TabFastFormer</code> are our own adaptations of the  Perceiver and the  FastFormer for tabular data.</p> <p>Parameters:</p> <ul> <li> <code>column_idx</code>               (<code>Dict[str, int]</code>)           \u2013            <p>Dict containing the index of the columns that will be passed through the <code>TabMlp</code> model. Required to slice the tensors. e.g. {'education': 0, 'relationship': 1, 'workclass': 2, ...}.</p> </li> <li> <code>cat_embed_input</code>               (<code>Optional[List[Tuple[str, int]]]</code>, default:                   <code>None</code> )           \u2013            <p>List of Tuples with the column name and number of unique values and embedding dimension. e.g. [(education, 11), ...]</p> </li> <li> <code>cat_embed_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Categorical embeddings dropout. If <code>None</code>, it will default to 0.</p> </li> <li> <code>use_cat_bias</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating if bias will be used for the categorical embeddings. If <code>None</code>, it will default to 'False'.</p> </li> <li> <code>cat_embed_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the categorical embeddings, if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> </li> <li> <code>shared_embed</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating if the embeddings will be \"shared\". The idea behind <code>shared_embed</code> is described in the Appendix A in the TabTransformer paper: 'The goal of having column embedding is to enable the model to distinguish the classes in one column from those in the other columns'. In other words, the idea is to let the model learn which column is embedded at the time. See: <code>pytorch_widedeep.models.transformers._layers.SharedEmbeddings</code>.</p> </li> <li> <code>add_shared_embed</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>The two embedding sharing strategies are: 1) add the shared embeddings to the column embeddings or 2) to replace the first <code>frac_shared_embed</code> with the shared embeddings. See <code>pytorch_widedeep.models.embeddings_layers.SharedEmbeddings</code> If 'None' is passed, it will default to 'False'.</p> </li> <li> <code>frac_shared_embed</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>The fraction of embeddings that will be shared (if <code>add_shared_embed = False</code>) by all the different categories for one particular column. If 'None' is passed, it will default to 0.0.</p> </li> <li> <code>continuous_cols</code>               (<code>Optional[List[str]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the name of the numeric (aka continuous) columns</p> </li> <li> <code>cont_norm_layer</code>               (<code>Optional[Literal[batchnorm, layernorm]]</code>, default:                   <code>None</code> )           \u2013            <p>Type of normalization layer applied to the continuous features. Options are: 'layernorm' and 'batchnorm'. if <code>None</code>, no normalization layer will be used.</p> </li> <li> <code>embed_continuous_method</code>               (<code>Optional[Literal[standard, piecewise, periodic]]</code>, default:                   <code>'standard'</code> )           \u2013            <p>Method to use to embed the continuous features. Options are: 'standard', 'periodic' or 'piecewise'. The 'standard' embedding method is based on the FT-Transformer implementation presented in the paper: Revisiting Deep Learning Models for Tabular Data. The 'periodic' and_'piecewise'_ methods were presented in the paper: On Embeddings for Numerical Features in Tabular Deep Learning. Please, read the papers for details.</p> </li> <li> <code>cont_embed_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p> </li> <li> <code>cont_embed_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the continuous embeddings if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported. If <code>None</code>, no activation function will be applied.</p> </li> <li> <code>quantization_setup</code>               (<code>Optional[Dict[str, List[float]]]</code>, default:                   <code>None</code> )           \u2013            <p>This parameter is used when the 'piecewise' method is used to embed the continuous cols. It is a dict where keys are the name of the continuous columns and values are lists with the boundaries for the quantization of the continuous_cols. See the examples for details. If If the 'piecewise' method is used, this parameter is required.</p> </li> <li> <code>n_frequencies</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>This is the so called 'k' in their paper On Embeddings for Numerical Features in Tabular Deep Learning, and is the number of 'frequencies' that will be used to represent each continuous column. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>sigma</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>This is the sigma parameter in the paper mentioned when describing the previous parameters and it is used to initialise the 'frequency weights'. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>share_last_layer</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>This parameter is not present in the before mentioned paper but it is implemented in the official repo. If <code>True</code> the linear layer that turns the frequencies into embeddings will be shared across the continuous columns. If <code>False</code> a different linear layer will be used for each continuous column. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>full_embed_dropout</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>If <code>True</code>, the full embedding corresponding to a column will be masked out/dropout. If <code>None</code>, it will default to <code>False</code>.</p> </li> <li> <code>input_dim</code>               (<code>int</code>, default:                   <code>32</code> )           \u2013            <p>The so-called dimension of the model. Is the number of embeddings used to encode the categorical and/or continuous columns.</p> </li> <li> <code>n_cross_attns</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>Number of times each perceiver block will cross attend to the input data (i.e. number of cross attention components per perceiver block). This should normally be 1. However, in the paper they describe some architectures (normally computer vision-related problems) where the Perceiver attends multiple times to the input array. Therefore, maybe multiple cross attention to the input array is also useful in some cases for tabular data  .</p> </li> <li> <code>n_cross_attn_heads</code>               (<code>int</code>, default:                   <code>4</code> )           \u2013            <p>Number of attention heads for the cross attention component</p> </li> <li> <code>n_latents</code>               (<code>int</code>, default:                   <code>16</code> )           \u2013            <p>Number of latents. This is the \\(N\\) parameter in the paper. As indicated in the paper, this number should be significantly lower than \\(M\\) (the number of columns in the dataset). Setting \\(N\\) closer to \\(M\\) defies the main purpose of the Perceiver, which is to overcome the transformer quadratic bottleneck</p> </li> <li> <code>latent_dim</code>               (<code>int</code>, default:                   <code>128</code> )           \u2013            <p>Latent dimension.</p> </li> <li> <code>n_latent_heads</code>               (<code>int</code>, default:                   <code>4</code> )           \u2013            <p>Number of attention heads per Latent Transformer</p> </li> <li> <code>n_latent_blocks</code>               (<code>int</code>, default:                   <code>4</code> )           \u2013            <p>Number of transformer encoder blocks (normalised MHA + normalised FF) per Latent Transformer</p> </li> <li> <code>n_perceiver_blocks</code>               (<code>int</code>, default:                   <code>4</code> )           \u2013            <p>Number of Perceiver blocks defined as [Cross Attention + Latent Transformer]</p> </li> <li> <code>share_weights</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating if the weights will be shared between Perceiver blocks</p> </li> <li> <code>attn_dropout</code>               (<code>float</code>, default:                   <code>0.1</code> )           \u2013            <p>Dropout that will be applied to the Multi-Head Attention layers</p> </li> <li> <code>ff_dropout</code>               (<code>float</code>, default:                   <code>0.1</code> )           \u2013            <p>Dropout that will be applied to the FeedForward network</p> </li> <li> <code>ff_factor</code>               (<code>int</code>, default:                   <code>4</code> )           \u2013            <p>Multiplicative factor applied to the first layer of the FF network in each Transformer block, This is normally set to 4.</p> </li> <li> <code>transformer_activation</code>               (<code>str</code>, default:                   <code>'geglu'</code> )           \u2013            <p>Transformer Encoder activation function. 'tanh', 'relu', 'leaky_relu', 'gelu', 'geglu' and 'reglu' are supported</p> </li> <li> <code>mlp_hidden_dims</code>               (<code>Optional[List[int]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the number of neurons per dense layer in the MLP. e.g: [64, 32]. If not provided no MLP on top of the final Transformer block will be used.</p> </li> <li> <code>mlp_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the dense layers of the MLP. Currently 'tanh', 'relu', 'leaky'_relu' and _'gelu' are supported. If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to 'relu'.</p> </li> <li> <code>mlp_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>float with the dropout between the dense layers of the MLP. If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to 0.0.</p> </li> <li> <code>mlp_batchnorm</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating whether or not batch normalization will be applied to the dense layers If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to False.</p> </li> <li> <code>mlp_batchnorm_last</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating whether or not batch normalization will be applied to the last of the dense layers If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to False.</p> </li> <li> <code>mlp_linear_first</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code> If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to <code>True</code>.</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>encoder</code>               (<code>ModuleDict</code>)           \u2013            <p>ModuleDict with the Perceiver blocks</p> </li> <li> <code>latents</code>               (<code>Parameter</code>)           \u2013            <p>Latents that will be used for prediction</p> </li> <li> <code>mlp</code>               (<code>Module</code>)           \u2013            <p>MLP component in the model</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import TabPerceiver\n&gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n&gt;&gt;&gt; colnames = ['a', 'b', 'c', 'd', 'e']\n&gt;&gt;&gt; cat_embed_input = [(u,i) for u,i in zip(colnames[:4], [4]*4)]\n&gt;&gt;&gt; continuous_cols = ['e']\n&gt;&gt;&gt; column_idx = {k:v for v,k in enumerate(colnames)}\n&gt;&gt;&gt; model = TabPerceiver(column_idx=column_idx, cat_embed_input=cat_embed_input,\n... continuous_cols=continuous_cols, n_latents=2, latent_dim=16,\n... n_perceiver_blocks=2)\n&gt;&gt;&gt; out = model(X_tab)\n</code></pre> Source code in <code>pytorch_widedeep/models/tabular/transformers/tab_perceiver.py</code> <pre><code>def __init__(\n    self,\n    column_idx: Dict[str, int],\n    *,\n    cat_embed_input: Optional[List[Tuple[str, int]]] = None,\n    cat_embed_dropout: Optional[float] = None,\n    use_cat_bias: Optional[bool] = None,\n    cat_embed_activation: Optional[str] = None,\n    shared_embed: Optional[bool] = None,\n    add_shared_embed: Optional[bool] = None,\n    frac_shared_embed: Optional[float] = None,\n    continuous_cols: Optional[List[str]] = None,\n    cont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\n    embed_continuous_method: Optional[\n        Literal[\"standard\", \"piecewise\", \"periodic\"]\n    ] = \"standard\",\n    cont_embed_dropout: Optional[float] = None,\n    cont_embed_activation: Optional[str] = None,\n    quantization_setup: Optional[Dict[str, List[float]]] = None,\n    n_frequencies: Optional[int] = None,\n    sigma: Optional[float] = None,\n    share_last_layer: Optional[bool] = None,\n    full_embed_dropout: Optional[bool] = None,\n    input_dim: int = 32,\n    n_cross_attns: int = 1,\n    n_cross_attn_heads: int = 4,\n    n_latents: int = 16,\n    latent_dim: int = 128,\n    n_latent_heads: int = 4,\n    n_latent_blocks: int = 4,\n    n_perceiver_blocks: int = 4,\n    share_weights: bool = False,\n    attn_dropout: float = 0.1,\n    ff_dropout: float = 0.1,\n    ff_factor: int = 4,\n    transformer_activation: str = \"geglu\",\n    mlp_hidden_dims: Optional[List[int]] = None,\n    mlp_activation: Optional[str] = None,\n    mlp_dropout: Optional[float] = None,\n    mlp_batchnorm: Optional[bool] = None,\n    mlp_batchnorm_last: Optional[bool] = None,\n    mlp_linear_first: Optional[bool] = None,\n):\n    super(TabPerceiver, self).__init__(\n        column_idx=column_idx,\n        cat_embed_input=cat_embed_input,\n        cat_embed_dropout=cat_embed_dropout,\n        use_cat_bias=use_cat_bias,\n        cat_embed_activation=cat_embed_activation,\n        shared_embed=shared_embed,\n        add_shared_embed=add_shared_embed,\n        frac_shared_embed=frac_shared_embed,\n        continuous_cols=continuous_cols,\n        cont_norm_layer=cont_norm_layer,\n        embed_continuous_method=embed_continuous_method,\n        cont_embed_dropout=cont_embed_dropout,\n        cont_embed_activation=cont_embed_activation,\n        input_dim=input_dim,\n        quantization_setup=quantization_setup,\n        n_frequencies=n_frequencies,\n        sigma=sigma,\n        share_last_layer=share_last_layer,\n        full_embed_dropout=full_embed_dropout,\n    )\n\n    self.n_cross_attns = n_cross_attns\n    self.n_cross_attn_heads = n_cross_attn_heads\n    self.n_latents = n_latents\n    self.latent_dim = latent_dim\n    self.n_latent_heads = n_latent_heads\n    self.n_latent_blocks = n_latent_blocks\n    self.n_perceiver_blocks = n_perceiver_blocks\n    self.share_weights = share_weights\n    self.attn_dropout = attn_dropout\n    self.ff_dropout = ff_dropout\n    self.ff_factor = ff_factor\n    self.transformer_activation = transformer_activation\n\n    self.mlp_hidden_dims = mlp_hidden_dims\n    self.mlp_activation = mlp_activation\n    self.mlp_dropout = mlp_dropout\n    self.mlp_batchnorm = mlp_batchnorm\n    self.mlp_batchnorm_last = mlp_batchnorm_last\n    self.mlp_linear_first = mlp_linear_first\n\n    # Embeddings are instantiated at the base model\n    # Transformer blocks\n    self.latents = nn.init.trunc_normal_(\n        nn.Parameter(torch.empty(n_latents, latent_dim))\n    )\n\n    self.encoder = nn.ModuleDict()\n    first_perceiver_block = self._build_perceiver_block()\n    self.encoder[\"perceiver_block0\"] = first_perceiver_block\n\n    if share_weights:\n        for n in range(1, n_perceiver_blocks):\n            self.encoder[\"perceiver_block\" + str(n)] = first_perceiver_block\n    else:\n        for n in range(1, n_perceiver_blocks):\n            self.encoder[\"perceiver_block\" + str(n)] = self._build_perceiver_block()\n\n    self.mlp_first_hidden_dim = self.latent_dim\n\n    # Mlp\n    if self.mlp_hidden_dims is not None:\n        self.mlp = MLP(\n            d_hidden=[self.mlp_first_hidden_dim] + self.mlp_hidden_dims,\n            activation=(\n                \"relu\" if self.mlp_activation is None else self.mlp_activation\n            ),\n            dropout=0.0 if self.mlp_dropout is None else self.mlp_dropout,\n            batchnorm=False if self.mlp_batchnorm is None else self.mlp_batchnorm,\n            batchnorm_last=(\n                False\n                if self.mlp_batchnorm_last is None\n                else self.mlp_batchnorm_last\n            ),\n            linear_first=(\n                False if self.mlp_linear_first is None else self.mlp_linear_first\n            ),\n        )\n    else:\n        self.mlp = None\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.transformers.tab_perceiver.TabPerceiver.output_dim","title":"output_dim  <code>property</code>","text":"<pre><code>output_dim\n</code></pre> <p>The output dimension of the model. This is a required property neccesary to build the <code>WideDeep</code> class</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.transformers.tab_perceiver.TabPerceiver.attention_weights","title":"attention_weights  <code>property</code>","text":"<pre><code>attention_weights\n</code></pre> <p>List with the attention weights. If the weights are not shared between perceiver blocks each element of the list will be a list itself containing the Cross Attention and Latent Transformer attention weights respectively</p> <p>The shape of the attention weights is:</p> <ul> <li> <p>Cross Attention: \\((N, C, L, F)\\)</p> </li> <li> <p>Latent Attention: \\((N, T, L, L)\\)</p> </li> </ul> <p>WHere \\(N\\) is the batch size, \\(C\\) is the number of Cross Attention heads, \\(L\\) is the number of Latents, \\(F\\) is the number of features/columns in the dataset and \\(T\\) is the number of Latent Attention heads</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.transformers.tab_fastformer.TabFastFormer","title":"TabFastFormer","text":"<pre><code>TabFastFormer(\n    column_idx,\n    *,\n    cat_embed_input=None,\n    cat_embed_dropout=None,\n    use_cat_bias=None,\n    cat_embed_activation=None,\n    shared_embed=None,\n    add_shared_embed=None,\n    frac_shared_embed=None,\n    continuous_cols=None,\n    cont_norm_layer=None,\n    embed_continuous_method=\"standard\",\n    cont_embed_dropout=None,\n    cont_embed_activation=None,\n    quantization_setup=None,\n    n_frequencies=None,\n    sigma=None,\n    share_last_layer=None,\n    full_embed_dropout=None,\n    input_dim=32,\n    n_heads=8,\n    use_bias=False,\n    n_blocks=4,\n    attn_dropout=0.1,\n    ff_dropout=0.2,\n    ff_factor=4,\n    share_qv_weights=False,\n    share_weights=False,\n    transformer_activation=\"relu\",\n    mlp_hidden_dims=None,\n    mlp_activation=None,\n    mlp_dropout=None,\n    mlp_batchnorm=None,\n    mlp_batchnorm_last=None,\n    mlp_linear_first=None\n)\n</code></pre> <p>               Bases: <code>BaseTabularModelWithAttention</code></p> <p>Defines an adaptation of a FastFormer that can be used as the <code>deeptabular</code> component of a Wide &amp; Deep model or independently by itself.</p> <p>Most of the parameters for this class are <code>Optional</code> since the use of categorical or continuous is in fact optional (i.e. one can use categorical features only, continuous features only or both).</p> <p> NOTE: while there are scientific publications for  the <code>TabTransformer</code>, <code>SAINT</code> and <code>FTTransformer</code>, the <code>TabPerceiver</code>  and the <code>TabFastFormer</code> are our own adaptations of the  Perceiver and the  FastFormer for tabular data.</p> <p>Parameters:</p> <ul> <li> <code>column_idx</code>               (<code>Dict[str, int]</code>)           \u2013            <p>Dict containing the index of the columns that will be passed through the <code>TabMlp</code> model. Required to slice the tensors. e.g. {'education': 0, 'relationship': 1, 'workclass': 2, ...}.</p> </li> <li> <code>cat_embed_input</code>               (<code>Optional[List[Tuple[str, int]]]</code>, default:                   <code>None</code> )           \u2013            <p>List of Tuples with the column name and number of unique values and embedding dimension. e.g. [(education, 11), ...]</p> </li> <li> <code>cat_embed_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Categorical embeddings dropout. If <code>None</code>, it will default to 0.</p> </li> <li> <code>use_cat_bias</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating if bias will be used for the categorical embeddings. If <code>None</code>, it will default to 'False'.</p> </li> <li> <code>cat_embed_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the categorical embeddings, if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> </li> <li> <code>shared_embed</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating if the embeddings will be \"shared\". The idea behind <code>shared_embed</code> is described in the Appendix A in the TabTransformer paper: 'The goal of having column embedding is to enable the model to distinguish the classes in one column from those in the other columns'. In other words, the idea is to let the model learn which column is embedded at the time. See: <code>pytorch_widedeep.models.transformers._layers.SharedEmbeddings</code>.</p> </li> <li> <code>add_shared_embed</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>The two embedding sharing strategies are: 1) add the shared embeddings to the column embeddings or 2) to replace the first <code>frac_shared_embed</code> with the shared embeddings. See <code>pytorch_widedeep.models.embeddings_layers.SharedEmbeddings</code> If 'None' is passed, it will default to 'False'.</p> </li> <li> <code>frac_shared_embed</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>The fraction of embeddings that will be shared (if <code>add_shared_embed = False</code>) by all the different categories for one particular column. If 'None' is passed, it will default to 0.0.</p> </li> <li> <code>continuous_cols</code>               (<code>Optional[List[str]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the name of the numeric (aka continuous) columns</p> </li> <li> <code>cont_norm_layer</code>               (<code>Optional[Literal[batchnorm, layernorm]]</code>, default:                   <code>None</code> )           \u2013            <p>Type of normalization layer applied to the continuous features. Options are: 'layernorm' and 'batchnorm'. if <code>None</code>, no normalization layer will be used.</p> </li> <li> <code>embed_continuous_method</code>               (<code>Optional[Literal[standard, piecewise, periodic]]</code>, default:                   <code>'standard'</code> )           \u2013            <p>Method to use to embed the continuous features. Options are: 'standard', 'periodic' or 'piecewise'. The 'standard' embedding method is based on the FT-Transformer implementation presented in the paper: Revisiting Deep Learning Models for Tabular Data. The 'periodic' and_'piecewise'_ methods were presented in the paper: On Embeddings for Numerical Features in Tabular Deep Learning. Please, read the papers for details.</p> </li> <li> <code>cont_embed_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p> </li> <li> <code>cont_embed_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the continuous embeddings if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported. If <code>None</code>, no activation function will be applied.</p> </li> <li> <code>quantization_setup</code>               (<code>Optional[Dict[str, List[float]]]</code>, default:                   <code>None</code> )           \u2013            <p>This parameter is used when the 'piecewise' method is used to embed the continuous cols. It is a dict where keys are the name of the continuous columns and values are lists with the boundaries for the quantization of the continuous_cols. See the examples for details. If If the 'piecewise' method is used, this parameter is required.</p> </li> <li> <code>n_frequencies</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>This is the so called 'k' in their paper On Embeddings for Numerical Features in Tabular Deep Learning, and is the number of 'frequencies' that will be used to represent each continuous column. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>sigma</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>This is the sigma parameter in the paper mentioned when describing the previous parameters and it is used to initialise the 'frequency weights'. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>share_last_layer</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>This parameter is not present in the before mentioned paper but it is implemented in the official repo. If <code>True</code> the linear layer that turns the frequencies into embeddings will be shared across the continuous columns. If <code>False</code> a different linear layer will be used for each continuous column. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>full_embed_dropout</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>If <code>True</code>, the full embedding corresponding to a column will be masked out/dropout. If <code>None</code>, it will default to <code>False</code>.</p> </li> <li> <code>input_dim</code>               (<code>int</code>, default:                   <code>32</code> )           \u2013            <p>The so-called dimension of the model. Is the number of embeddings used to encode the categorical and/or continuous columns</p> </li> <li> <code>n_heads</code>               (<code>int</code>, default:                   <code>8</code> )           \u2013            <p>Number of attention heads per FastFormer block</p> </li> <li> <code>use_bias</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether or not to use bias in the Q, K, and V projection layers</p> </li> <li> <code>n_blocks</code>               (<code>int</code>, default:                   <code>4</code> )           \u2013            <p>Number of FastFormer blocks</p> </li> <li> <code>attn_dropout</code>               (<code>float</code>, default:                   <code>0.1</code> )           \u2013            <p>Dropout that will be applied to the Additive Attention layers</p> </li> <li> <code>ff_dropout</code>               (<code>float</code>, default:                   <code>0.2</code> )           \u2013            <p>Dropout that will be applied to the FeedForward network</p> </li> <li> <code>ff_factor</code>               (<code>int</code>, default:                   <code>4</code> )           \u2013            <p>Multiplicative factor applied to the first layer of the FF network in each Transformer block, This is normally set to 4.</p> </li> <li> <code>share_qv_weights</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Following the paper, this is a boolean indicating if the Value (\\(V\\)) and the Query (\\(Q\\)) transformation parameters will be shared.</p> </li> <li> <code>share_weights</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>In addition to sharing the \\(V\\) and \\(Q\\) transformation parameters, the parameters across different Fastformer layers can also be shared. Please, see <code>pytorch_widedeep/models/tabular/transformers/tab_fastformer.py</code> for details</p> </li> <li> <code>transformer_activation</code>               (<code>str</code>, default:                   <code>'relu'</code> )           \u2013            <p>Transformer Encoder activation function. 'tanh', 'relu', 'leaky_relu', 'gelu', 'geglu' and 'reglu' are supported</p> </li> <li> <code>mlp_hidden_dims</code>               (<code>Optional[List[int]]</code>, default:                   <code>None</code> )           \u2013            <p>MLP hidden dimensions. If not provided no MLP on top of the final FTTransformer block will be used</p> </li> <li> <code>mlp_hidden_dims</code>               (<code>Optional[List[int]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the number of neurons per dense layer in the MLP. e.g: [64, 32]. If not provided no MLP on top of the final Transformer block will be used.</p> </li> <li> <code>mlp_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the dense layers of the MLP. Currently 'tanh', 'relu', 'leaky'_relu' and _'gelu' are supported. If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to 'relu'.</p> </li> <li> <code>mlp_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>float with the dropout between the dense layers of the MLP. If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to 0.0.</p> </li> <li> <code>mlp_batchnorm</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating whether or not batch normalization will be applied to the dense layers If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to False.</p> </li> <li> <code>mlp_batchnorm_last</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating whether or not batch normalization will be applied to the last of the dense layers If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to False.</p> </li> <li> <code>mlp_linear_first</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code> If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to <code>True</code>.</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>encoder</code>               (<code>Module</code>)           \u2013            <p>Sequence of FasFormer blocks.</p> </li> <li> <code>mlp</code>               (<code>Module</code>)           \u2013            <p>MLP component in the model</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import TabFastFormer\n&gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n&gt;&gt;&gt; colnames = ['a', 'b', 'c', 'd', 'e']\n&gt;&gt;&gt; cat_embed_input = [(u,i) for u,i in zip(colnames[:4], [4]*4)]\n&gt;&gt;&gt; continuous_cols = ['e']\n&gt;&gt;&gt; column_idx = {k:v for v,k in enumerate(colnames)}\n&gt;&gt;&gt; model = TabFastFormer(column_idx=column_idx, cat_embed_input=cat_embed_input, continuous_cols=continuous_cols)\n&gt;&gt;&gt; out = model(X_tab)\n</code></pre> Source code in <code>pytorch_widedeep/models/tabular/transformers/tab_fastformer.py</code> <pre><code>def __init__(\n    self,\n    column_idx: Dict[str, int],\n    *,\n    cat_embed_input: Optional[List[Tuple[str, int]]] = None,\n    cat_embed_dropout: Optional[float] = None,\n    use_cat_bias: Optional[bool] = None,\n    cat_embed_activation: Optional[str] = None,\n    shared_embed: Optional[bool] = None,\n    add_shared_embed: Optional[bool] = None,\n    frac_shared_embed: Optional[float] = None,\n    continuous_cols: Optional[List[str]] = None,\n    cont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\n    embed_continuous_method: Optional[\n        Literal[\"standard\", \"piecewise\", \"periodic\"]\n    ] = \"standard\",\n    cont_embed_dropout: Optional[float] = None,\n    cont_embed_activation: Optional[str] = None,\n    quantization_setup: Optional[Dict[str, List[float]]] = None,\n    n_frequencies: Optional[int] = None,\n    sigma: Optional[float] = None,\n    share_last_layer: Optional[bool] = None,\n    full_embed_dropout: Optional[bool] = None,\n    input_dim: int = 32,\n    n_heads: int = 8,\n    use_bias: bool = False,\n    n_blocks: int = 4,\n    attn_dropout: float = 0.1,\n    ff_dropout: float = 0.2,\n    ff_factor: int = 4,\n    share_qv_weights: bool = False,\n    share_weights: bool = False,\n    transformer_activation: str = \"relu\",\n    mlp_hidden_dims: Optional[List[int]] = None,\n    mlp_activation: Optional[str] = None,\n    mlp_dropout: Optional[float] = None,\n    mlp_batchnorm: Optional[bool] = None,\n    mlp_batchnorm_last: Optional[bool] = None,\n    mlp_linear_first: Optional[bool] = None,\n):\n    super(TabFastFormer, self).__init__(\n        column_idx=column_idx,\n        cat_embed_input=cat_embed_input,\n        cat_embed_dropout=cat_embed_dropout,\n        use_cat_bias=use_cat_bias,\n        cat_embed_activation=cat_embed_activation,\n        shared_embed=shared_embed,\n        add_shared_embed=add_shared_embed,\n        frac_shared_embed=frac_shared_embed,\n        continuous_cols=continuous_cols,\n        cont_norm_layer=cont_norm_layer,\n        embed_continuous_method=embed_continuous_method,\n        cont_embed_dropout=cont_embed_dropout,\n        cont_embed_activation=cont_embed_activation,\n        input_dim=input_dim,\n        quantization_setup=quantization_setup,\n        n_frequencies=n_frequencies,\n        sigma=sigma,\n        share_last_layer=share_last_layer,\n        full_embed_dropout=full_embed_dropout,\n    )\n\n    self.n_heads = n_heads\n    self.use_bias = use_bias\n    self.n_blocks = n_blocks\n    self.attn_dropout = attn_dropout\n    self.ff_dropout = ff_dropout\n    self.ff_factor = ff_factor\n    self.share_qv_weights = share_qv_weights\n    self.share_weights = share_weights\n    self.transformer_activation = transformer_activation\n\n    self.mlp_hidden_dims = mlp_hidden_dims\n    self.mlp_activation = mlp_activation\n    self.mlp_dropout = mlp_dropout\n    self.mlp_batchnorm = mlp_batchnorm\n    self.mlp_batchnorm_last = mlp_batchnorm_last\n    self.mlp_linear_first = mlp_linear_first\n\n    self.with_cls_token = \"cls_token\" in column_idx\n    self.n_cat = len(cat_embed_input) if cat_embed_input is not None else 0\n    self.n_cont = len(continuous_cols) if continuous_cols is not None else 0\n    self.n_feats = self.n_cat + self.n_cont\n\n    # Embeddings are instantiated at the base model\n    # Transformer blocks\n    self.encoder = nn.Sequential()\n    first_fastformer_block = FastFormerEncoder(\n        input_dim,\n        n_heads,\n        use_bias,\n        attn_dropout,\n        ff_dropout,\n        ff_factor,\n        share_qv_weights,\n        transformer_activation,\n    )\n    self.encoder.add_module(\"fastformer_block0\", first_fastformer_block)\n    for i in range(1, n_blocks):\n        if share_weights:\n            self.encoder.add_module(\n                \"fastformer_block\" + str(i), first_fastformer_block\n            )\n        else:\n            self.encoder.add_module(\n                \"fastformer_block\" + str(i),\n                FastFormerEncoder(\n                    input_dim,\n                    n_heads,\n                    use_bias,\n                    attn_dropout,\n                    ff_dropout,\n                    ff_factor,\n                    share_qv_weights,\n                    transformer_activation,\n                ),\n            )\n\n    self.mlp_first_hidden_dim = (\n        self.input_dim if self.with_cls_token else (self.n_feats * self.input_dim)\n    )\n\n    # Mlp: adding an MLP on top of the Resnet blocks is optional and\n    # therefore all related params are optional\n    if self.mlp_hidden_dims is not None:\n        self.mlp = MLP(\n            d_hidden=[self.mlp_first_hidden_dim] + self.mlp_hidden_dims,\n            activation=(\n                \"relu\" if self.mlp_activation is None else self.mlp_activation\n            ),\n            dropout=0.0 if self.mlp_dropout is None else self.mlp_dropout,\n            batchnorm=False if self.mlp_batchnorm is None else self.mlp_batchnorm,\n            batchnorm_last=(\n                False\n                if self.mlp_batchnorm_last is None\n                else self.mlp_batchnorm_last\n            ),\n            linear_first=(\n                False if self.mlp_linear_first is None else self.mlp_linear_first\n            ),\n        )\n    else:\n        self.mlp = None\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.transformers.tab_fastformer.TabFastFormer.output_dim","title":"output_dim  <code>property</code>","text":"<pre><code>output_dim\n</code></pre> <p>The output dimension of the model. This is a required property neccesary to build the <code>WideDeep</code> class</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.transformers.tab_fastformer.TabFastFormer.attention_weights","title":"attention_weights  <code>property</code>","text":"<pre><code>attention_weights\n</code></pre> <p>List with the attention weights. Each element of the list is a tuple where the first and second elements are the \\(\\alpha\\) and \\(\\beta\\) attention weights in the paper.</p> <p>The shape of the attention weights is \\((N, H, F)\\) where \\(N\\) is the batch size, \\(H\\) is the number of attention heads and \\(F\\) is the number of features/columns in the dataset</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.text.rnns.basic_rnn.BasicRNN","title":"BasicRNN","text":"<pre><code>BasicRNN(\n    vocab_size,\n    embed_dim=None,\n    embed_matrix=None,\n    embed_trainable=True,\n    rnn_type=\"lstm\",\n    hidden_dim=64,\n    n_layers=3,\n    rnn_dropout=0.0,\n    bidirectional=False,\n    use_hidden_state=True,\n    padding_idx=1,\n    head_hidden_dims=None,\n    head_activation=\"relu\",\n    head_dropout=None,\n    head_batchnorm=False,\n    head_batchnorm_last=False,\n    head_linear_first=False,\n)\n</code></pre> <p>               Bases: <code>BaseWDModelComponent</code></p> <p>Standard text classifier/regressor comprised by a stack of RNNs (LSTMs or GRUs) that can be used as the <code>deeptext</code> component of a Wide &amp; Deep model or independently by itself.</p> <p>In addition, there is the option to add a Fully Connected (FC) set of dense layers on top of the stack of RNNs</p> <p>Parameters:</p> <ul> <li> <code>vocab_size</code>               (<code>int</code>)           \u2013            <p>Number of words in the vocabulary</p> </li> <li> <code>embed_dim</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>Dimension of the word embeddings if non-pretained word vectors are used</p> </li> <li> <code>embed_matrix</code>               (<code>Optional[ndarray]</code>, default:                   <code>None</code> )           \u2013            <p>Pretrained word embeddings</p> </li> <li> <code>embed_trainable</code>               (<code>bool</code>, default:                   <code>True</code> )           \u2013            <p>Boolean indicating if the pretrained embeddings are trainable</p> </li> <li> <code>rnn_type</code>               (<code>Literal[lstm, gru]</code>, default:                   <code>'lstm'</code> )           \u2013            <p>String indicating the type of RNN to use. One of 'lstm' or 'gru'</p> </li> <li> <code>hidden_dim</code>               (<code>int</code>, default:                   <code>64</code> )           \u2013            <p>Hidden dim of the RNN</p> </li> <li> <code>n_layers</code>               (<code>int</code>, default:                   <code>3</code> )           \u2013            <p>Number of recurrent layers</p> </li> <li> <code>rnn_dropout</code>               (<code>float</code>, default:                   <code>0.0</code> )           \u2013            <p>Dropout for each RNN layer except the last layer</p> </li> <li> <code>bidirectional</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether the staked RNNs are bidirectional</p> </li> <li> <code>use_hidden_state</code>               (<code>bool</code>, default:                   <code>True</code> )           \u2013            <p>Boolean indicating whether to use the final hidden state or the RNN's output as predicting features. Typically the former is used.</p> </li> <li> <code>padding_idx</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>index of the padding token in the padded-tokenised sequences. The <code>TextPreprocessor</code> class within this library uses fastai's tokenizer where the token index 0 is reserved for the 'unknown' word token. Therefore, the default value is set to 1.</p> </li> <li> <code>head_hidden_dims</code>               (<code>Optional[List[int]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the sizes of the dense layers in the head e.g: [128, 64]</p> </li> <li> <code>head_activation</code>               (<code>str</code>, default:                   <code>'relu'</code> )           \u2013            <p>Activation function for the dense layers in the head. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> </li> <li> <code>head_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Dropout of the dense layers in the head</p> </li> <li> <code>head_batchnorm</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether or not to include batch normalization in the dense layers that form the 'rnn_mlp'</p> </li> <li> <code>head_batchnorm_last</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether or not to apply batch normalization to the last of the dense layers in the head</p> </li> <li> <code>head_linear_first</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code></p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>word_embed</code>               (<code>Module</code>)           \u2013            <p>word embedding matrix</p> </li> <li> <code>rnn</code>               (<code>Module</code>)           \u2013            <p>Stack of RNNs</p> </li> <li> <code>rnn_mlp</code>               (<code>Module</code>)           \u2013            <p>Stack of dense layers on top of the RNN. This will only exists if <code>head_layers_dim</code> is not None</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import BasicRNN\n&gt;&gt;&gt; X_text = torch.cat((torch.zeros([5,1]), torch.empty(5, 4).random_(1,4)), axis=1)\n&gt;&gt;&gt; model = BasicRNN(vocab_size=4, hidden_dim=4, n_layers=2, padding_idx=0, embed_dim=4)\n&gt;&gt;&gt; out = model(X_text)\n</code></pre> Source code in <code>pytorch_widedeep/models/text/rnns/basic_rnn.py</code> <pre><code>def __init__(\n    self,\n    vocab_size: int,\n    embed_dim: Optional[int] = None,\n    embed_matrix: Optional[np.ndarray] = None,\n    embed_trainable: bool = True,\n    rnn_type: Literal[\"lstm\", \"gru\"] = \"lstm\",\n    hidden_dim: int = 64,\n    n_layers: int = 3,\n    rnn_dropout: float = 0.0,\n    bidirectional: bool = False,\n    use_hidden_state: bool = True,\n    padding_idx: int = 1,\n    head_hidden_dims: Optional[List[int]] = None,\n    head_activation: str = \"relu\",\n    head_dropout: Optional[float] = None,\n    head_batchnorm: bool = False,\n    head_batchnorm_last: bool = False,\n    head_linear_first: bool = False,\n):\n    super(BasicRNN, self).__init__()\n\n    if embed_dim is None and embed_matrix is None:\n        raise ValueError(\n            \"If no 'embed_matrix' is passed, the embedding dimension must\"\n            \"be specified with 'embed_dim'\"\n        )\n\n    if rnn_type.lower() not in [\"lstm\", \"gru\"]:\n        raise ValueError(\n            f\"'rnn_type' must be 'lstm' or 'gru', got {rnn_type} instead\"\n        )\n\n    if (\n        embed_dim is not None\n        and embed_matrix is not None\n        and not embed_dim == embed_matrix.shape[1]\n    ):\n        warnings.warn(\n            \"the input embedding dimension {} and the dimension of the \"\n            \"pretrained embeddings {} do not match. The pretrained embeddings \"\n            \"dimension ({}) will be used\".format(\n                embed_dim, embed_matrix.shape[1], embed_matrix.shape[1]\n            ),\n            UserWarning,\n        )\n\n    self.vocab_size = vocab_size\n    self.embed_trainable = embed_trainable\n    self.embed_dim = embed_dim\n\n    self.rnn_type = rnn_type\n    self.hidden_dim = hidden_dim\n    self.n_layers = n_layers\n    self.rnn_dropout = rnn_dropout\n    self.bidirectional = bidirectional\n    self.use_hidden_state = use_hidden_state\n    self.padding_idx = padding_idx\n\n    self.head_hidden_dims = head_hidden_dims\n    self.head_activation = head_activation\n    self.head_dropout = head_dropout\n    self.head_batchnorm = head_batchnorm\n    self.head_batchnorm_last = head_batchnorm_last\n    self.head_linear_first = head_linear_first\n\n    # Embeddings\n    if embed_matrix is not None:\n        self.word_embed, self.embed_dim = self._set_embeddings(embed_matrix)\n    else:\n        assert self.embed_dim is not None\n        self.word_embed = nn.Embedding(\n            self.vocab_size, self.embed_dim, padding_idx=self.padding_idx\n        )\n\n    # RNN\n    rnn_params = {\n        \"input_size\": self.embed_dim,\n        \"hidden_size\": hidden_dim,\n        \"num_layers\": n_layers,\n        \"bidirectional\": bidirectional,\n        \"dropout\": rnn_dropout,\n        \"batch_first\": True,\n    }\n    if self.rnn_type.lower() == \"lstm\":\n        self.rnn: Union[nn.LSTM, nn.GRU] = nn.LSTM(**rnn_params)\n    elif self.rnn_type.lower() == \"gru\":\n        self.rnn = nn.GRU(**rnn_params)\n    else:\n        raise ValueError(\n            f\"'rnn_type' must be 'lstm' or 'gru', got {self.rnn_type} instead\"\n        )\n\n    self.rnn_output_dim = hidden_dim * 2 if bidirectional else hidden_dim\n\n    # FC-Head (Mlp)\n    if self.head_hidden_dims is not None:\n        head_hidden_dims = [self.rnn_output_dim] + self.head_hidden_dims\n        self.rnn_mlp: Union[MLP, nn.Identity] = MLP(\n            head_hidden_dims,\n            head_activation,\n            head_dropout,\n            head_batchnorm,\n            head_batchnorm_last,\n            head_linear_first,\n        )\n    else:\n        # simple hack to add readability in the forward pass\n        self.rnn_mlp = nn.Identity()\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.text.rnns.basic_rnn.BasicRNN.output_dim","title":"output_dim  <code>property</code>","text":"<pre><code>output_dim\n</code></pre> <p>The output dimension of the model. This is a required property neccesary to build the <code>WideDeep</code> class</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.text.rnns.attentive_rnn.AttentiveRNN","title":"AttentiveRNN","text":"<pre><code>AttentiveRNN(\n    vocab_size,\n    embed_dim=None,\n    embed_matrix=None,\n    embed_trainable=True,\n    rnn_type=\"lstm\",\n    hidden_dim=64,\n    n_layers=3,\n    rnn_dropout=0.1,\n    bidirectional=False,\n    use_hidden_state=True,\n    padding_idx=1,\n    attn_concatenate=True,\n    attn_dropout=0.1,\n    head_hidden_dims=None,\n    head_activation=\"relu\",\n    head_dropout=None,\n    head_batchnorm=False,\n    head_batchnorm_last=False,\n    head_linear_first=False,\n)\n</code></pre> <p>               Bases: <code>BasicRNN</code></p> <p>Text classifier/regressor comprised by a stack of RNNs (LSTMs or GRUs) plus an attention layer. This model can be used as the <code>deeptext</code> component of a Wide &amp; Deep model or independently by itself.</p> <p>In addition, there is the option to add a Fully Connected (FC) set of dense layers on top of attention layer</p> <p>Parameters:</p> <ul> <li> <code>vocab_size</code>               (<code>int</code>)           \u2013            <p>Number of words in the vocabulary</p> </li> <li> <code>embed_dim</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>Dimension of the word embeddings if non-pretained word vectors are used</p> </li> <li> <code>embed_matrix</code>               (<code>Optional[ndarray]</code>, default:                   <code>None</code> )           \u2013            <p>Pretrained word embeddings</p> </li> <li> <code>embed_trainable</code>               (<code>bool</code>, default:                   <code>True</code> )           \u2013            <p>Boolean indicating if the pretrained embeddings are trainable</p> </li> <li> <code>rnn_type</code>               (<code>Literal[lstm, gru]</code>, default:                   <code>'lstm'</code> )           \u2013            <p>String indicating the type of RNN to use. One of 'lstm' or 'gru'</p> </li> <li> <code>hidden_dim</code>               (<code>int</code>, default:                   <code>64</code> )           \u2013            <p>Hidden dim of the RNN</p> </li> <li> <code>n_layers</code>               (<code>int</code>, default:                   <code>3</code> )           \u2013            <p>Number of recurrent layers</p> </li> <li> <code>rnn_dropout</code>               (<code>float</code>, default:                   <code>0.1</code> )           \u2013            <p>Dropout for each RNN layer except the last layer</p> </li> <li> <code>bidirectional</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether the staked RNNs are bidirectional</p> </li> <li> <code>use_hidden_state</code>               (<code>bool</code>, default:                   <code>True</code> )           \u2013            <p>Boolean indicating whether to use the final hidden state or the RNN's output as predicting features. Typically the former is used.</p> </li> <li> <code>padding_idx</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>index of the padding token in the padded-tokenised sequences. The <code>TextPreprocessor</code> class within this library uses fastai's tokenizer where the token index 0 is reserved for the 'unknown' word token. Therefore, the default value is set to 1.</p> </li> <li> <code>attn_concatenate</code>               (<code>bool</code>, default:                   <code>True</code> )           \u2013            <p>Boolean indicating if the input to the attention mechanism will be the output of the RNN or the output of the RNN concatenated with the last hidden state.</p> </li> <li> <code>attn_dropout</code>               (<code>float</code>, default:                   <code>0.1</code> )           \u2013            <p>Internal dropout for the attention mechanism</p> </li> <li> <code>head_hidden_dims</code>               (<code>Optional[List[int]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the sizes of the dense layers in the head e.g: [128, 64]</p> </li> <li> <code>head_activation</code>               (<code>str</code>, default:                   <code>'relu'</code> )           \u2013            <p>Activation function for the dense layers in the head. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> </li> <li> <code>head_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Dropout of the dense layers in the head</p> </li> <li> <code>head_batchnorm</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether or not to include batch normalization in the dense layers that form the 'rnn_mlp'</p> </li> <li> <code>head_batchnorm_last</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether or not to apply batch normalization to the last of the dense layers in the head</p> </li> <li> <code>head_linear_first</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code></p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>word_embed</code>               (<code>Module</code>)           \u2013            <p>word embedding matrix</p> </li> <li> <code>rnn</code>               (<code>Module</code>)           \u2013            <p>Stack of RNNs</p> </li> <li> <code>rnn_mlp</code>               (<code>Module</code>)           \u2013            <p>Stack of dense layers on top of the RNN. This will only exists if <code>head_layers_dim</code> is not <code>None</code></p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import AttentiveRNN\n&gt;&gt;&gt; X_text = torch.cat((torch.zeros([5,1]), torch.empty(5, 4).random_(1,4)), axis=1)\n&gt;&gt;&gt; model = AttentiveRNN(vocab_size=4, hidden_dim=4, n_layers=2, padding_idx=0, embed_dim=4)\n&gt;&gt;&gt; out = model(X_text)\n</code></pre> Source code in <code>pytorch_widedeep/models/text/rnns/attentive_rnn.py</code> <pre><code>def __init__(\n    self,\n    vocab_size: int,\n    embed_dim: Optional[int] = None,\n    embed_matrix: Optional[np.ndarray] = None,\n    embed_trainable: bool = True,\n    rnn_type: Literal[\"lstm\", \"gru\"] = \"lstm\",\n    hidden_dim: int = 64,\n    n_layers: int = 3,\n    rnn_dropout: float = 0.1,\n    bidirectional: bool = False,\n    use_hidden_state: bool = True,\n    padding_idx: int = 1,\n    attn_concatenate: bool = True,\n    attn_dropout: float = 0.1,\n    head_hidden_dims: Optional[List[int]] = None,\n    head_activation: str = \"relu\",\n    head_dropout: Optional[float] = None,\n    head_batchnorm: bool = False,\n    head_batchnorm_last: bool = False,\n    head_linear_first: bool = False,\n):\n    super(AttentiveRNN, self).__init__(\n        vocab_size=vocab_size,\n        embed_dim=embed_dim,\n        embed_matrix=embed_matrix,\n        embed_trainable=embed_trainable,\n        rnn_type=rnn_type,\n        hidden_dim=hidden_dim,\n        n_layers=n_layers,\n        rnn_dropout=rnn_dropout,\n        bidirectional=bidirectional,\n        use_hidden_state=use_hidden_state,\n        padding_idx=padding_idx,\n        head_hidden_dims=head_hidden_dims,\n        head_activation=head_activation,\n        head_dropout=head_dropout,\n        head_batchnorm=head_batchnorm,\n        head_batchnorm_last=head_batchnorm_last,\n        head_linear_first=head_linear_first,\n    )\n\n    # Embeddings and RNN defined in the BasicRNN inherited class\n\n    # Attention\n    self.attn_concatenate = attn_concatenate\n    self.attn_dropout = attn_dropout\n\n    if bidirectional and attn_concatenate:\n        self.rnn_output_dim = hidden_dim * 4\n    elif bidirectional or attn_concatenate:\n        self.rnn_output_dim = hidden_dim * 2\n    else:\n        self.rnn_output_dim = hidden_dim\n    self.attn = ContextAttention(\n        self.rnn_output_dim, attn_dropout, sum_along_seq=True\n    )\n\n    # FC-Head (Mlp)\n    if self.head_hidden_dims is not None:\n        head_hidden_dims = [self.rnn_output_dim] + self.head_hidden_dims\n        self.rnn_mlp = MLP(\n            head_hidden_dims,\n            head_activation,\n            head_dropout,\n            head_batchnorm,\n            head_batchnorm_last,\n            head_linear_first,\n        )\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.text.rnns.attentive_rnn.AttentiveRNN.attention_weights","title":"attention_weights  <code>property</code>","text":"<pre><code>attention_weights\n</code></pre> <p>List with the attention weights</p> <p>The shape of the attention weights is \\((N, S)\\), where \\(N\\) is the batch size and \\(S\\) is the length of the sequence</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.text.rnns.stacked_attentive_rnn.StackedAttentiveRNN","title":"StackedAttentiveRNN","text":"<pre><code>StackedAttentiveRNN(\n    vocab_size,\n    embed_dim=None,\n    embed_matrix=None,\n    embed_trainable=True,\n    rnn_type=\"lstm\",\n    hidden_dim=64,\n    bidirectional=False,\n    padding_idx=1,\n    n_blocks=3,\n    attn_concatenate=False,\n    attn_dropout=0.1,\n    with_addnorm=False,\n    head_hidden_dims=None,\n    head_activation=\"relu\",\n    head_dropout=None,\n    head_batchnorm=False,\n    head_batchnorm_last=False,\n    head_linear_first=False,\n)\n</code></pre> <p>               Bases: <code>BaseWDModelComponent</code></p> <p>Text classifier/regressor comprised by a stack of blocks: <code>[RNN + Attention]</code>. This can be used as the <code>deeptext</code> component of a Wide &amp; Deep model or independently by itself.</p> <p>In addition, there is the option to add a Fully Connected (FC) set of dense layers on top of the attentiob blocks</p> <p>Parameters:</p> <ul> <li> <code>vocab_size</code>               (<code>int</code>)           \u2013            <p>Number of words in the vocabulary</p> </li> <li> <code>embed_dim</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>Dimension of the word embeddings if non-pretained word vectors are used</p> </li> <li> <code>embed_matrix</code>               (<code>Optional[ndarray]</code>, default:                   <code>None</code> )           \u2013            <p>Pretrained word embeddings</p> </li> <li> <code>embed_trainable</code>               (<code>bool</code>, default:                   <code>True</code> )           \u2013            <p>Boolean indicating if the pretrained embeddings are trainable</p> </li> <li> <code>rnn_type</code>               (<code>Literal[lstm, gru]</code>, default:                   <code>'lstm'</code> )           \u2013            <p>String indicating the type of RNN to use. One of 'lstm' or 'gru'</p> </li> <li> <code>hidden_dim</code>               (<code>int</code>, default:                   <code>64</code> )           \u2013            <p>Hidden dim of the RNN</p> </li> <li> <code>bidirectional</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether the staked RNNs are bidirectional</p> </li> <li> <code>padding_idx</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>index of the padding token in the padded-tokenised sequences. The <code>TextPreprocessor</code> class within this library uses fastai's tokenizer where the token index 0 is reserved for the 'unknown' word token. Therefore, the default value is set to 1.</p> </li> <li> <code>n_blocks</code>               (<code>int</code>, default:                   <code>3</code> )           \u2013            <p>Number of attention blocks. Each block is comprised by an RNN and a Context Attention Encoder</p> </li> <li> <code>attn_concatenate</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating if the input to the attention mechanism will be the output of the RNN or the output of the RNN concatenated with the last hidden state or simply</p> </li> <li> <code>attn_dropout</code>               (<code>float</code>, default:                   <code>0.1</code> )           \u2013            <p>Internal dropout for the attention mechanism</p> </li> <li> <code>with_addnorm</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating if the output of each block will be added to the input and normalised</p> </li> <li> <code>head_hidden_dims</code>               (<code>Optional[List[int]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the sizes of the dense layers in the head e.g: [128, 64]</p> </li> <li> <code>head_activation</code>               (<code>str</code>, default:                   <code>'relu'</code> )           \u2013            <p>Activation function for the dense layers in the head. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> </li> <li> <code>head_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Dropout of the dense layers in the head</p> </li> <li> <code>head_batchnorm</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether or not to include batch normalization in the dense layers that form the 'rnn_mlp'</p> </li> <li> <code>head_batchnorm_last</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether or not to apply batch normalization to the last of the dense layers in the head</p> </li> <li> <code>head_linear_first</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code></p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>word_embed</code>               (<code>Module</code>)           \u2013            <p>word embedding matrix</p> </li> <li> <code>rnn</code>               (<code>Module</code>)           \u2013            <p>Stack of RNNs</p> </li> <li> <code>rnn_mlp</code>               (<code>Module</code>)           \u2013            <p>Stack of dense layers on top of the RNN. This will only exists if <code>head_layers_dim</code> is not <code>None</code></p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import StackedAttentiveRNN\n&gt;&gt;&gt; X_text = torch.cat((torch.zeros([5,1]), torch.empty(5, 4).random_(1,4)), axis=1)\n&gt;&gt;&gt; model = StackedAttentiveRNN(vocab_size=4, hidden_dim=4, padding_idx=0, embed_dim=4)\n&gt;&gt;&gt; out = model(X_text)\n</code></pre> Source code in <code>pytorch_widedeep/models/text/rnns/stacked_attentive_rnn.py</code> <pre><code>def __init__(\n    self,\n    vocab_size: int,\n    embed_dim: Optional[int] = None,\n    embed_matrix: Optional[np.ndarray] = None,\n    embed_trainable: bool = True,\n    rnn_type: Literal[\"lstm\", \"gru\"] = \"lstm\",\n    hidden_dim: int = 64,\n    bidirectional: bool = False,\n    padding_idx: int = 1,\n    n_blocks: int = 3,\n    attn_concatenate: bool = False,\n    attn_dropout: float = 0.1,\n    with_addnorm: bool = False,\n    head_hidden_dims: Optional[List[int]] = None,\n    head_activation: str = \"relu\",\n    head_dropout: Optional[float] = None,\n    head_batchnorm: bool = False,\n    head_batchnorm_last: bool = False,\n    head_linear_first: bool = False,\n):\n    super(StackedAttentiveRNN, self).__init__()\n\n    if (\n        embed_dim is not None\n        and embed_matrix is not None\n        and not embed_dim == embed_matrix.shape[1]\n    ):\n        warnings.warn(\n            \"the input embedding dimension {} and the dimension of the \"\n            \"pretrained embeddings {} do not match. The pretrained embeddings \"\n            \"dimension ({}) will be used\".format(\n                embed_dim, embed_matrix.shape[1], embed_matrix.shape[1]\n            ),\n            UserWarning,\n        )\n\n    if rnn_type.lower() not in [\"lstm\", \"gru\"]:\n        raise ValueError(\n            f\"'rnn_type' must be 'lstm' or 'gru', got {rnn_type} instead\"\n        )\n\n    self.vocab_size = vocab_size\n    self.embed_trainable = embed_trainable\n    self.embed_dim = embed_dim\n\n    self.rnn_type = rnn_type\n    self.hidden_dim = hidden_dim\n    self.bidirectional = bidirectional\n    self.padding_idx = padding_idx\n\n    self.n_blocks = n_blocks\n    self.attn_concatenate = attn_concatenate\n    self.attn_dropout = attn_dropout\n    self.with_addnorm = with_addnorm\n\n    self.head_hidden_dims = head_hidden_dims\n    self.head_activation = head_activation\n    self.head_dropout = head_dropout\n    self.head_batchnorm = head_batchnorm\n    self.head_batchnorm_last = head_batchnorm_last\n    self.head_linear_first = head_linear_first\n\n    # Embeddings\n    self.word_embed, self.embed_dim = self._set_embeddings(embed_matrix)\n\n    # Linear Projection: if embed_dim is different that the input of the\n    # attention blocks we add a linear projection\n    if bidirectional and attn_concatenate:\n        self.rnn_output_dim = hidden_dim * 4\n    elif bidirectional or attn_concatenate:\n        self.rnn_output_dim = hidden_dim * 2\n    else:\n        self.rnn_output_dim = hidden_dim\n\n    if self.rnn_output_dim != self.embed_dim:\n        self.embed_proj: Union[nn.Linear, nn.Identity] = nn.Linear(\n            self.embed_dim, self.rnn_output_dim\n        )\n    else:\n        self.embed_proj = nn.Identity()\n\n    # RNN\n    rnn_params = {\n        \"input_size\": self.rnn_output_dim,\n        \"hidden_size\": hidden_dim,\n        \"bidirectional\": bidirectional,\n        \"batch_first\": True,\n    }\n    if self.rnn_type.lower() == \"lstm\":\n        self.rnn: Union[nn.LSTM, nn.GRU] = nn.LSTM(**rnn_params)\n    elif self.rnn_type.lower() == \"gru\":\n        self.rnn = nn.GRU(**rnn_params)\n\n    # FC-Head (Mlp)\n    self.attention_blks = nn.ModuleList()\n    for i in range(n_blocks):\n        self.attention_blks.append(\n            ContextAttentionEncoder(\n                self.rnn,\n                self.rnn_output_dim,\n                attn_dropout,\n                attn_concatenate,\n                with_addnorm=with_addnorm if i != n_blocks - 1 else False,\n                sum_along_seq=i == n_blocks - 1,\n            )\n        )\n\n    # Mlp\n    if self.head_hidden_dims is not None:\n        head_hidden_dims = [self.rnn_output_dim] + self.head_hidden_dims\n        self.rnn_mlp: Union[MLP, nn.Identity] = MLP(\n            head_hidden_dims,\n            head_activation,\n            head_dropout,\n            head_batchnorm,\n            head_batchnorm_last,\n            head_linear_first,\n        )\n    else:\n        # simple hack to add readability in the forward pass\n        self.rnn_mlp = nn.Identity()\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.text.rnns.stacked_attentive_rnn.StackedAttentiveRNN.output_dim","title":"output_dim  <code>property</code>","text":"<pre><code>output_dim\n</code></pre> <p>The output dimension of the model. This is a required property neccesary to build the <code>WideDeep</code> class</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.text.rnns.stacked_attentive_rnn.StackedAttentiveRNN.attention_weights","title":"attention_weights  <code>property</code>","text":"<pre><code>attention_weights\n</code></pre> <p>List with the attention weights per block</p> <p>The shape of the attention weights is \\((N, S)\\) Where \\(N\\) is the batch size and \\(S\\) is the length of the sequence</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.text.huggingface_transformers.hf_model.HFModel","title":"HFModel","text":"<pre><code>HFModel(\n    model_name,\n    use_cls_token=True,\n    trainable_parameters=None,\n    head_hidden_dims=None,\n    head_activation=\"relu\",\n    head_dropout=None,\n    head_batchnorm=False,\n    head_batchnorm_last=False,\n    head_linear_first=False,\n    verbose=False,\n    **kwargs\n)\n</code></pre> <p>               Bases: <code>BaseWDModelComponent</code></p> <p>This class is a wrapper around the Hugging Face transformers library. It can be used as the text component of a Wide &amp; Deep model or independently by itself.</p> <p>At the moment only models from the families BERT, RoBERTa, DistilBERT, ALBERT and ELECTRA are supported. This is because this library is designed to address classification and regression tasks and these are the most 'popular' encoder-only models, which have proved to be those that work best for these tasks.</p> <p>Parameters:</p> <ul> <li> <code>model_name</code>               (<code>str</code>)           \u2013            <p>The model name from the transformers library e.g. 'bert-base-uncased'. Currently supported models are those from the families: BERT, RoBERTa, DistilBERT, ALBERT and ELECTRA.</p> </li> <li> <code>use_cls_token</code>               (<code>bool</code>, default:                   <code>True</code> )           \u2013            <p>Boolean indicating whether to use the [CLS] token or the mean of the sequence of hidden states as the sentence embedding</p> </li> <li> <code>trainable_parameters</code>               (<code>Optional[List[str]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the names of the model parameters that will be trained. If None, none of the parameters will be trainable</p> </li> <li> <code>head_hidden_dims</code>               (<code>Optional[List[int]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the sizes of the dense layers in the head e.g: [128, 64]</p> </li> <li> <code>head_activation</code>               (<code>str</code>, default:                   <code>'relu'</code> )           \u2013            <p>Activation function for the dense layers in the head. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> </li> <li> <code>head_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Dropout of the dense layers in the head</p> </li> <li> <code>head_batchnorm</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether or not to include batch normalization in the dense layers that form the head</p> </li> <li> <code>head_batchnorm_last</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether or not to apply batch normalization to the last of the dense layers in the head</p> </li> <li> <code>head_linear_first</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code></p> </li> <li> <code>verbose</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>If True, it will print information about the model</p> </li> <li> <code>**kwargs</code>           \u2013            <p>Additional kwargs to be passed to the model</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>head</code>               (<code>Module</code>)           \u2013            <p>Stack of dense layers on top of the transformer. This will only exists if <code>head_layers_dim</code> is not None</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import HFModel\n&gt;&gt;&gt; X_text = torch.cat((torch.zeros([5,1]), torch.empty(5, 4).random_(1,4)), axis=1).long()\n&gt;&gt;&gt; model = HFModel(model_name='bert-base-uncased')\n&gt;&gt;&gt; out = model(X_text)\n</code></pre> Source code in <code>pytorch_widedeep/models/text/huggingface_transformers/hf_model.py</code> <pre><code>@alias(\"use_cls_token\", [\"use_special_token\"])\ndef __init__(\n    self,\n    model_name: str,\n    use_cls_token: bool = True,\n    trainable_parameters: Optional[List[str]] = None,\n    head_hidden_dims: Optional[List[int]] = None,\n    head_activation: str = \"relu\",\n    head_dropout: Optional[float] = None,\n    head_batchnorm: bool = False,\n    head_batchnorm_last: bool = False,\n    head_linear_first: bool = False,\n    verbose: bool = False,\n    **kwargs,\n):\n    super().__init__()\n\n    # TO DO: add warning regarging ELECTRA as ELECTRA does not have a cls\n    # token.  Research what happens with ELECTRA\n    self.model_name = model_name\n    self.use_cls_token = use_cls_token\n    self.trainable_parameters = trainable_parameters\n    self.head_hidden_dims = head_hidden_dims\n    self.head_activation = head_activation\n    self.head_dropout = head_dropout\n    self.head_batchnorm = head_batchnorm\n    self.head_batchnorm_last = head_batchnorm_last\n    self.head_linear_first = head_linear_first\n    self.verbose = verbose\n    self.kwargs = kwargs\n\n    if self.verbose and self.use_cls_token:\n        warnings.warn(\n            \"The model will use the [CLS] token. Make sure the tokenizer \"\n            \"was run with add_special_tokens=True\",\n            UserWarning,\n        )\n\n    self.model_class = get_model_class(model_name)\n\n    self.config, self.model = get_config_and_model(self.model_name)\n\n    self.output_attention_weights = kwargs.get(\"output_attentions\", False)\n\n    if self.trainable_parameters is not None:\n        for n, p in self.model.named_parameters():\n            p.requires_grad = any([tl in n for tl in self.trainable_parameters])\n\n    # FC-Head (Mlp). Note that the FC head will always be trainable\n    if self.head_hidden_dims is not None:\n        head_hidden_dims = [self.config.hidden_size] + self.head_hidden_dims\n        self.head = MLP(\n            head_hidden_dims,\n            head_activation,\n            head_dropout,\n            head_batchnorm,\n            head_batchnorm_last,\n            head_linear_first,\n        )\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.text.huggingface_transformers.hf_model.HFModel.attention_weight","title":"attention_weight  <code>property</code>","text":"<pre><code>attention_weight\n</code></pre> <p>Returns the attention weights if the model was created with the output_attention_weights=True argument. If not, it will raise an AttributeError.</p> <p>The shape of the attention weights is \\((N, H, F, F)\\), where \\(N\\) is the batch size, \\(H\\) is the number of attention heads and \\(F\\) is the sequence length.</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.image.vision.Vision","title":"Vision","text":"<pre><code>Vision(\n    pretrained_model_setup=None,\n    n_trainable=None,\n    trainable_params=None,\n    channel_sizes=[64, 128, 256, 512],\n    kernel_sizes=[7, 3, 3, 3],\n    strides=[2, 1, 1, 1],\n    head_hidden_dims=None,\n    head_activation=\"relu\",\n    head_dropout=0.1,\n    head_batchnorm=False,\n    head_batchnorm_last=False,\n    head_linear_first=False,\n)\n</code></pre> <p>               Bases: <code>BaseWDModelComponent</code></p> <p>Defines a standard image classifier/regressor using a pretrained network or a sequence of convolution layers that can be used as the <code>deepimage</code> component of a Wide &amp; Deep model or independently by itself.</p> <p> NOTE: this class represents the integration  between <code>pytorch-widedeep</code> and <code>torchvision</code>. New architectures will be  available as they are added to <code>torchvision</code>. In a distant future we aim  to bring transformer-based architectures as well. However, simple  CNN-based architectures (and even MLP-based) seem to produce SoTA  results. For the time being, we describe below the options available  through this class</p> <p>Parameters:</p> <ul> <li> <code>pretrained_model_setup</code>               (<code>Union[str, Dict[str, Union[str, WeightsEnum]]]</code>, default:                   <code>None</code> )           \u2013            <p>Name of the pretrained model. Should be a variant of the following architectures: 'resnet', 'shufflenet', 'resnext', 'wide_resnet', 'regnet', 'densenet', 'mobilenetv3', 'mobilenetv2', 'mnasnet', 'efficientnet' and 'squeezenet'. if <code>pretrained_model_setup = None</code> a basic, fully trainable CNN will be used. Alternatively, since Torchvision 0.13 one can use pretrained models with different weigths. Therefore, <code>pretrained_model_setup</code> can also be dictionary with the name of the model and the weights (e.g. <code>{'resnet50': ResNet50_Weights.DEFAULT}</code> or <code>{'resnet50': \"IMAGENET1K_V2\"}</code>).  Aliased as <code>pretrained_model_name</code>.</p> </li> <li> <code>n_trainable</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>Number of trainable layers starting from the layer closer to the output neuron(s). Note that this number DOES NOT take into account the so-called 'head' which is ALWAYS trainable. If <code>trainable_params</code> is not None this parameter will be ignored</p> </li> <li> <code>trainable_params</code>               (<code>Optional[List[str]]</code>, default:                   <code>None</code> )           \u2013            <p>List of strings containing the names (or substring within the name) of the parameters that will be trained. For example, if we use a 'resnet18' pretrained model and we set <code>trainable_params = ['layer4']</code> only the parameters of 'layer4' of the network (and the head, as mentioned before) will be trained. Note that setting this or the previous parameter involves some knowledge of the architecture used.</p> </li> <li> <code>channel_sizes</code>               (<code>List[int]</code>, default:                   <code>[64, 128, 256, 512]</code> )           \u2013            <p>List of integers with the channel sizes of a CNN in case we choose not to use a pretrained model</p> </li> <li> <code>kernel_sizes</code>               (<code>Union[int, List[int]]</code>, default:                   <code>[7, 3, 3, 3]</code> )           \u2013            <p>List of integers with the kernel sizes of a CNN in case we choose not to use a pretrained model. Must be of length equal to <code>len(channel_sizes) - 1</code>.</p> </li> <li> <code>strides</code>               (<code>Union[int, List[int]]</code>, default:                   <code>[2, 1, 1, 1]</code> )           \u2013            <p>List of integers with the stride sizes of a CNN in case we choose not to use a pretrained model. Must be of length equal to <code>len(channel_sizes) - 1</code>.</p> </li> <li> <code>head_hidden_dims</code>               (<code>Optional[List[int]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the number of neurons per dense layer in the head. e.g: [64,32]</p> </li> <li> <code>head_activation</code>               (<code>str</code>, default:                   <code>'relu'</code> )           \u2013            <p>Activation function for the dense layers in the head. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> </li> <li> <code>head_dropout</code>               (<code>Union[float, List[float]]</code>, default:                   <code>0.1</code> )           \u2013            <p>float indicating the dropout between the dense layers.</p> </li> <li> <code>head_batchnorm</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether or not batch normalization will be applied to the dense layers</p> </li> <li> <code>head_batchnorm_last</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether or not batch normalization will be applied to the last of the dense layers</p> </li> <li> <code>head_linear_first</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code></p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>features</code>               (<code>Module</code>)           \u2013            <p>The pretrained model or Standard CNN plus the optional head</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import Vision\n&gt;&gt;&gt; X_img = torch.rand((2,3,224,224))\n&gt;&gt;&gt; model = Vision(channel_sizes=[64, 128], kernel_sizes = [3, 3], strides=[1, 1], head_hidden_dims=[32, 8])\n&gt;&gt;&gt; out = model(X_img)\n</code></pre> Source code in <code>pytorch_widedeep/models/image/vision.py</code> <pre><code>@alias(\"pretrained_model_setup\", [\"pretrained_model_name\"])\ndef __init__(\n    self,\n    pretrained_model_setup: Union[str, Dict[str, Union[str, WeightsEnum]]] = None,\n    n_trainable: Optional[int] = None,\n    trainable_params: Optional[List[str]] = None,\n    channel_sizes: List[int] = [64, 128, 256, 512],\n    kernel_sizes: Union[int, List[int]] = [7, 3, 3, 3],\n    strides: Union[int, List[int]] = [2, 1, 1, 1],\n    head_hidden_dims: Optional[List[int]] = None,\n    head_activation: str = \"relu\",\n    head_dropout: Union[float, List[float]] = 0.1,\n    head_batchnorm: bool = False,\n    head_batchnorm_last: bool = False,\n    head_linear_first: bool = False,\n):\n    super(Vision, self).__init__()\n\n    self._check_pretrained_model_setup(\n        pretrained_model_setup, n_trainable, trainable_params\n    )\n\n    self.pretrained_model_setup = pretrained_model_setup\n    self.n_trainable = n_trainable\n    self.trainable_params = trainable_params\n    self.channel_sizes = channel_sizes\n    self.kernel_sizes = kernel_sizes\n    self.strides = strides\n    self.head_hidden_dims = head_hidden_dims\n    self.head_activation = head_activation\n    self.head_dropout = head_dropout\n    self.head_batchnorm = head_batchnorm\n    self.head_batchnorm_last = head_batchnorm_last\n    self.head_linear_first = head_linear_first\n\n    self.features, self.backbone_output_dim = self._get_features()\n\n    if pretrained_model_setup is not None:\n        self._freeze(self.features)\n\n    if self.head_hidden_dims is not None:\n        head_hidden_dims = [self.backbone_output_dim] + self.head_hidden_dims\n        self.vision_mlp = MLP(\n            head_hidden_dims,\n            self.head_activation,\n            self.head_dropout,\n            self.head_batchnorm,\n            self.head_batchnorm_last,\n            self.head_linear_first,\n        )\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.image.vision.Vision.output_dim","title":"output_dim  <code>property</code>","text":"<pre><code>output_dim\n</code></pre> <p>The output dimension of the model. This is a required property neccesary to build the <code>WideDeep</code> class</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.model_fusion.ModelFuser","title":"ModelFuser","text":"<pre><code>ModelFuser(\n    models,\n    *,\n    fusion_method,\n    projection_method=None,\n    custom_head=None,\n    head_hidden_dims=None,\n    head_activation=None,\n    head_dropout=None,\n    head_batchnorm=None,\n    head_batchnorm_last=None,\n    head_linear_first=None\n)\n</code></pre> <p>               Bases: <code>BaseWDModelComponent</code></p> <p>This class is a wrapper around a list of models that are associated to the different text and/or image columns (and datasets) The class is designed to 'fuse' the models using a variety of methods.</p> <p>Parameters:</p> <ul> <li> <code>models</code>               (<code>List[BaseWDModelComponent]</code>)           \u2013            <p>List of models whose outputs will be fused</p> </li> <li> <code>fusion_method</code>               (<code>Union[Literal[concatenate, mean, max, sum, mult, dot, head], List[Literal[concatenate, mean, max, sum, mult, head]]]</code>)           \u2013            <p>Method to fuse the output of the models. It can be one of ['concatenate', 'mean', 'max', 'sum', 'mult', 'dot', 'head'] or a list of those, but 'dot'. If a list is provided the output of the models will be fused using all the methods in the list and the final output will be the concatenation of the outputs of each method</p> </li> <li> <code>projection_method</code>               (<code>Optional[Literal[min, max, mean]]</code>, default:                   <code>None</code> )           \u2013            <p>If the fusion_method is not 'concatenate', this parameter will determine how to project the output of the models to a common dimension. It can be one of ['min', 'max', 'mean']. Default is None</p> </li> <li> <code>custom_head</code>               (<code>Optional[Union[BaseWDModelComponent, Module]]</code>, default:                   <code>None</code> )           \u2013            <p>Custom head to be used to fuse the output of the models. If provided, this will take precedence over head_hidden_dims. Also, if provided, 'projection_method' will be ignored.</p> </li> <li> <code>head_hidden_dims</code>               (<code>Optional[List[int]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the number of neurons per layer in the custom head. If custom_head is provided, this parameter will be ignored</p> </li> <li> <code>head_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function to be used in the custom head. Default is None</p> </li> <li> <code>head_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Dropout to be used in the custom head. Default is None</p> </li> <li> <code>head_batchnorm</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Whether to use batchnorm in the custom head. Default is None</p> </li> <li> <code>head_batchnorm_last</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Whether or not batch normalization will be applied to the last of the dense layers</p> </li> <li> <code>head_linear_first</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code></p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>head</code>               (<code>Module or BaseWDModelComponent</code>)           \u2013            <p>Custom head to be used to fuse the output of the models. If custom_head is provided, this will take precedence over head_hidden_dims</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; from pytorch_widedeep.preprocessing import TextPreprocessor\n&gt;&gt;&gt; from pytorch_widedeep.models import BasicRNN, ModelFuser\n&gt;&gt;&gt; import torch\n&gt;&gt;&gt; import pandas as pd\n&gt;&gt;&gt;\n&gt;&gt;&gt; df = pd.DataFrame({'text_col1': ['hello world', 'this is a test'],\n... 'text_col2': ['goodbye world', 'this is another test']})\n&gt;&gt;&gt; text_preprocessor_1 = TextPreprocessor(\n...     text_col=\"text_col1\",\n...     max_vocab=10,\n...     min_freq=1,\n...     maxlen=5,\n...     n_cpus=1,\n...     verbose=0)\n&gt;&gt;&gt; text_preprocessor_2 = TextPreprocessor(\n...     text_col=\"text_col2\",\n...     max_vocab=10,\n...     min_freq=1,\n...     maxlen=5,\n...     n_cpus=1,\n...     verbose=0)\n&gt;&gt;&gt; X_text1 = text_preprocessor_1.fit_transform(df)\n&gt;&gt;&gt; X_text2 = text_preprocessor_2.fit_transform(df)\n&gt;&gt;&gt; X_text1_tnsr = torch.from_numpy(X_text1)\n&gt;&gt;&gt; X_text2_tnsr = torch.from_numpy(X_text2)\n&gt;&gt;&gt; rnn1 = BasicRNN(\n...     vocab_size=len(text_preprocessor_1.vocab.itos),\n...     embed_dim=4,\n...     hidden_dim=4,\n...     n_layers=1,\n...     bidirectional=False)\n&gt;&gt;&gt; rnn2 = BasicRNN(\n...     vocab_size=len(text_preprocessor_2.vocab.itos),\n...     embed_dim=4,\n...     hidden_dim=4,\n...     n_layers=1,\n...     bidirectional=False)\n&gt;&gt;&gt; fused_model = ModelFuser(models=[rnn1, rnn2], fusion_method='concatenate')\n&gt;&gt;&gt; out = fused_model([X_text1_tnsr, X_text2_tnsr])\n</code></pre> Source code in <code>pytorch_widedeep/models/model_fusion.py</code> <pre><code>def __init__(\n    self,\n    models: List[BaseWDModelComponent],\n    *,\n    fusion_method: Union[\n        Literal[\n            \"concatenate\",\n            \"mean\",\n            \"max\",\n            \"sum\",\n            \"mult\",\n            \"dot\",\n            \"head\",\n        ],\n        List[Literal[\"concatenate\", \"mean\", \"max\", \"sum\", \"mult\", \"head\"]],\n    ],\n    projection_method: Optional[Literal[\"min\", \"max\", \"mean\"]] = None,\n    custom_head: Optional[Union[BaseWDModelComponent, nn.Module]] = None,\n    head_hidden_dims: Optional[List[int]] = None,\n    head_activation: Optional[str] = None,\n    head_dropout: Optional[float] = None,\n    head_batchnorm: Optional[bool] = None,\n    head_batchnorm_last: Optional[bool] = None,\n    head_linear_first: Optional[bool] = None,\n) -&gt; None:\n    super(ModelFuser, self).__init__()\n\n    self.models = nn.ModuleList(models)\n    self.fusion_method = fusion_method\n    self.projection_method = projection_method\n\n    self.all_output_dim_equal = all(\n        model.output_dim == self.models[0].output_dim for model in self.models\n    )\n\n    self.check_input_parameters()\n\n    if self.fusion_method == \"head\":\n        assert (\n            head_hidden_dims is not None or custom_head is not None\n        ), \"When using 'head' as fusion_method, either head_hidden_dims or custom_head must be provided\"\n        if custom_head is not None:\n            # custom_head takes precedence over head_hidden_dims (in case\n            # both are provided)\n            assert hasattr(\n                custom_head, \"output_dim\"\n            ), \"custom_head must have an 'output_dim' property\"\n            self.head: Union[BaseWDModelComponent, nn.Module] = custom_head\n        else:\n            assert head_hidden_dims is not None\n            self.head_hidden_dims = head_hidden_dims\n            self.head_activation = head_activation\n            self.head_dropout = head_dropout\n            self.head_batchnorm = head_batchnorm\n            self.head_batchnorm_last = head_batchnorm_last\n            self.head_linear_first = head_linear_first\n\n            self.head = MLP(\n                d_hidden=[sum([model.output_dim for model in self.models])]\n                + self.head_hidden_dims,\n                activation=(\n                    \"relu\" if self.head_activation is None else self.head_activation\n                ),\n                dropout=0.0 if self.head_dropout is None else self.head_dropout,\n                batchnorm=(\n                    False if self.head_batchnorm is None else self.head_batchnorm\n                ),\n                batchnorm_last=(\n                    False\n                    if self.head_batchnorm_last is None\n                    else self.head_batchnorm_last\n                ),\n                linear_first=(\n                    True\n                    if self.head_linear_first is None\n                    else self.head_linear_first\n                ),\n            )\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.model_fusion.ModelFuser.output_dim","title":"output_dim  <code>property</code>","text":"<pre><code>output_dim\n</code></pre> <p>Returns the output dimension of the model.</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.model_fusion.ModelFuser.project","title":"project","text":"<pre><code>project(X)\n</code></pre> <p>Projects the output of the models to a common dimension.</p> Source code in <code>pytorch_widedeep/models/model_fusion.py</code> <pre><code>def project(self, X: List[Tensor]) -&gt; List[Tensor]:\n    r\"\"\"Projects the output of the models to a common dimension.\"\"\"\n\n    if self.all_output_dim_equal and self.projection_method is None:\n        return X\n\n    output_dims = [model.output_dim for model in self.models]\n\n    if self.projection_method == \"min\":\n        proj_dim = min(output_dims)\n        idx = output_dims.index(proj_dim)\n    elif self.projection_method == \"max\":\n        proj_dim = max(output_dims)\n        idx = output_dims.index(proj_dim)\n    elif self.projection_method == \"mean\":\n        proj_dim = int(sum(output_dims) / len(output_dims))\n        idx = None\n    else:\n        raise ValueError(\"projection_method must be one of ['min', 'max', 'mean']\")\n\n    x_proj: List[Tensor] = []\n    for i, x in enumerate(X):\n        if i == idx:\n            x_proj.append(x)\n        else:\n            x_proj.append(\n                nn.Linear(output_dims[i], proj_dim, bias=False, device=x.device)(x)\n            )\n\n    return x_proj\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.wide_deep.WideDeep","title":"WideDeep","text":"<pre><code>WideDeep(\n    wide=None,\n    deeptabular=None,\n    deeptext=None,\n    deepimage=None,\n    deephead=None,\n    head_hidden_dims=None,\n    head_activation=\"relu\",\n    head_dropout=0.1,\n    head_batchnorm=False,\n    head_batchnorm_last=False,\n    head_linear_first=True,\n    enforce_positive=False,\n    enforce_positive_activation=\"softplus\",\n    pred_dim=1,\n)\n</code></pre> <p>               Bases: <code>Module</code></p> <p>Main collector class that combines all <code>wide</code>, <code>deeptabular</code> <code>deeptext</code> and <code>deepimage</code> models.</p> <p>Note that all models described so far in this library must be passed to the <code>WideDeep</code> class once constructed. This is because the models output the last layer before the prediction layer. Such prediction layer is added by the <code>WideDeep</code> class as it collects the components for every data mode.</p> <p>There are two options to combine these models that correspond to the two main architectures that <code>pytorch-widedeep</code> can build.</p> <ul> <li> <p>Directly connecting the output of the model components to an ouput neuron(s).</p> </li> <li> <p>Adding a <code>Fully-Connected Head</code> (FC-Head) on top of the deep models.   This FC-Head will combine the output form the <code>deeptabular</code>, <code>deeptext</code> and   <code>deepimage</code> and will be then connected to the output neuron(s).</p> </li> </ul> <p>Parameters:</p> <ul> <li> <code>wide</code>               (<code>Optional[Module]</code>, default:                   <code>None</code> )           \u2013            <p><code>Wide</code> model. This is a linear model where the non-linearities are captured via crossed-columns.</p> </li> <li> <code>deeptabular</code>               (<code>Optional[Union[BaseWDModelComponent, List[BaseWDModelComponent]]]</code>, default:                   <code>None</code> )           \u2013            <p>Currently this library implements a number of possible architectures for the <code>deeptabular</code> component. See the documenation of the package. Note that <code>deeptabular</code> can be a list of models. This is useful when using multiple tabular inputs (e.g. for example in the context of a two-tower model for recommendation systems)</p> </li> <li> <code>deeptext</code>               (<code>Optional[Union[BaseWDModelComponent, List[BaseWDModelComponent]]]</code>, default:                   <code>None</code> )           \u2013            <p>Currently this library implements a number of possible architectures for the <code>deeptext</code> component. See the documenation of the package. Note that <code>deeptext</code> can be a list of models. This is useful when using multiple text inputs.</p> </li> <li> <code>deepimage</code>               (<code>Optional[Union[BaseWDModelComponent, List[BaseWDModelComponent]]]</code>, default:                   <code>None</code> )           \u2013            <p>Currently this library uses <code>torchvision</code> and implements a number of possible architectures for the <code>deepimage</code> component. See the documenation of the package. Note that <code>deepimage</code> can be a list of models. This is useful when using multiple image inputs.</p> </li> <li> <code>deephead</code>               (<code>Optional[BaseWDModelComponent]</code>, default:                   <code>None</code> )           \u2013            <p>Alternatively, the user can pass a custom model that will receive the output of the deep component. If <code>deephead</code> is not None all the previous fc-head parameters will be ignored</p> </li> <li> <code>head_hidden_dims</code>               (<code>Optional[List[int]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the sizes of the dense layers in the head e.g: [128, 64]</p> </li> <li> <code>head_activation</code>               (<code>str</code>, default:                   <code>'relu'</code> )           \u2013            <p>Activation function for the dense layers in the head. Currently <code>'tanh'</code>, <code>'relu'</code>, <code>'leaky_relu'</code> and <code>'gelu'</code> are supported</p> </li> <li> <code>head_dropout</code>               (<code>float</code>, default:                   <code>0.1</code> )           \u2013            <p>Dropout of the dense layers in the head</p> </li> <li> <code>head_batchnorm</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether or not to include batch normalization in the dense layers that form the <code>'rnn_mlp'</code></p> </li> <li> <code>head_batchnorm_last</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether or not to apply batch normalization to the last of the dense layers in the head</p> </li> <li> <code>head_linear_first</code>               (<code>bool</code>, default:                   <code>True</code> )           \u2013            <p>Boolean indicating whether the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code></p> </li> <li> <code>enforce_positive</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating if the output from the final layer must be positive. This is important if you are using loss functions with non-negative input restrictions, e.g. RMSLE, or if you know your predictions are bounded in between 0 and inf</p> </li> <li> <code>enforce_positive_activation</code>               (<code>str</code>, default:                   <code>'softplus'</code> )           \u2013            <p>Activation function to enforce that the final layer has a positive output. <code>'softplus'</code> or <code>'relu'</code> are supported.</p> </li> <li> <code>pred_dim</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>Size of the final wide and deep output layer containing the predictions. <code>1</code> for regression and binary classification or number of classes for multiclass classification.</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; from pytorch_widedeep.models import TabResnet, Vision, BasicRNN, Wide, WideDeep\n&gt;&gt;&gt; embed_input = [(u, i, j) for u, i, j in zip([\"a\", \"b\", \"c\"][:4], [4] * 3, [8] * 3)]\n&gt;&gt;&gt; column_idx = {k: v for v, k in enumerate([\"a\", \"b\", \"c\"])}\n&gt;&gt;&gt; wide = Wide(10, 1)\n&gt;&gt;&gt; deeptabular = TabResnet(blocks_dims=[8, 4], column_idx=column_idx, cat_embed_input=embed_input)\n&gt;&gt;&gt; deeptext = BasicRNN(vocab_size=10, embed_dim=4, padding_idx=0)\n&gt;&gt;&gt; deepimage = Vision()\n&gt;&gt;&gt; model = WideDeep(wide=wide, deeptabular=deeptabular, deeptext=deeptext, deepimage=deepimage)\n</code></pre> <p> NOTE: It is possible to use custom components to  build Wide &amp; Deep models. Simply, build them and pass them as the  corresponding parameters. Note that the custom models MUST return a last  layer of activations(i.e. not the final prediction) so that  these  activations are collected by <code>WideDeep</code> and combined accordingly. In  addition, the models MUST also contain an attribute <code>output_dim</code> with  the size of these last layers of activations. See for example  <code>pytorch_widedeep.models.tab_mlp.TabMlp</code></p> Source code in <code>pytorch_widedeep/models/wide_deep.py</code> <pre><code>@alias(  # noqa: C901\n    \"pred_dim\",\n    [\"num_class\", \"pred_size\"],\n)\ndef __init__(\n    self,\n    wide: Optional[nn.Module] = None,\n    deeptabular: Optional[\n        Union[BaseWDModelComponent, List[BaseWDModelComponent]]\n    ] = None,\n    deeptext: Optional[\n        Union[BaseWDModelComponent, List[BaseWDModelComponent]]\n    ] = None,\n    deepimage: Optional[\n        Union[BaseWDModelComponent, List[BaseWDModelComponent]]\n    ] = None,\n    deephead: Optional[BaseWDModelComponent] = None,\n    head_hidden_dims: Optional[List[int]] = None,\n    head_activation: str = \"relu\",\n    head_dropout: float = 0.1,\n    head_batchnorm: bool = False,\n    head_batchnorm_last: bool = False,\n    head_linear_first: bool = True,\n    enforce_positive: bool = False,\n    enforce_positive_activation: str = \"softplus\",\n    pred_dim: int = 1,\n):\n    super(WideDeep, self).__init__()\n\n    self._check_inputs(\n        wide,\n        deeptabular,\n        deeptext,\n        deepimage,\n        deephead,\n        head_hidden_dims,\n        pred_dim,\n    )\n\n    # this attribute will be eventually over-written by the Trainer's\n    # device. Acts here as a 'placeholder'.\n    self.wd_device: Optional[str] = None\n\n    # required as attribute just in case we pass a deephead\n    self.pred_dim = pred_dim\n\n    self.enforce_positive = enforce_positive\n\n    # better to set this attribute already here\n    if isinstance(deeptabular, list):\n        self.is_tabnet = False\n    else:\n        self.is_tabnet = deeptabular.__class__.__name__ == \"TabNet\"\n\n    # The main 5 components of the wide and deep assemble: wide,\n    # deeptabular, deeptext, deepimage and deephead\n    self.with_deephead = deephead is not None or head_hidden_dims is not None\n    if deephead is None and head_hidden_dims is not None:\n        self.deephead = self._build_deephead(\n            deeptabular,\n            deeptext,\n            deepimage,\n            head_hidden_dims,\n            head_activation,\n            head_dropout,\n            head_batchnorm,\n            head_batchnorm_last,\n            head_linear_first,\n        )\n    elif deephead is not None:\n        self.deephead = nn.Sequential(\n            deephead, nn.Linear(deephead.output_dim, self.pred_dim)\n        )\n    else:\n        # for consistency with other components we default to None\n        self.deephead = None\n\n    self.wide = wide\n    self.deeptabular, self.deeptext, self.deepimage = self._set_model_components(\n        deeptabular, deeptext, deepimage\n    )\n\n    if self.enforce_positive:\n        self.enf_pos = get_activation_fn(enforce_positive_activation)\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html","title":"The <code>preprocessing</code> module","text":"<p>This module contains the classes that are used to prepare the data before being passed to the models. There is one Preprocessor per data mode or model component (<code>wide</code>, <code>deeptabular</code>, <code>deepimage</code> and <code>deeptext</code>) with the exception of the <code>deeptext</code> component. In this case, two processors are available: one for the case when no Hugging Face model is used (<code>TextPreprocessor</code>) and another one when a Hugging Face model is used (<code>HFPreprocessor</code>).</p>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.wide_preprocessor.WidePreprocessor","title":"WidePreprocessor","text":"<pre><code>WidePreprocessor(wide_cols, crossed_cols=None)\n</code></pre> <p>               Bases: <code>BasePreprocessor</code></p> <p>Preprocessor to prepare the wide input dataset</p> <p>This Preprocessor prepares the data for the wide, linear component. This linear model is implemented via an Embedding layer that is connected to the output neuron. <code>WidePreprocessor</code> numerically encodes all the unique values of all categorical columns <code>wide_cols + crossed_cols</code>. See the Example below.</p> <p>Parameters:</p> <ul> <li> <code>wide_cols</code>               (<code>List[str]</code>)           \u2013            <p>List of strings with the name of the columns that will label encoded and passed through the <code>wide</code> component</p> </li> <li> <code>crossed_cols</code>               (<code>Optional[List[Tuple[str, str]]]</code>, default:                   <code>None</code> )           \u2013            <p>List of Tuples with the name of the columns that will be <code>'crossed'</code> and then label encoded. e.g. [('education', 'occupation'), ...]. For binary features, a cross-product transformation is 1 if and only if the constituent features are all 1, and 0 otherwise.</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>wide_crossed_cols</code>               (<code>List</code>)           \u2013            <p>List with the names of all columns that will be label encoded</p> </li> <li> <code>encoding_dict</code>               (<code>Dict</code>)           \u2013            <p>Dictionary where the keys are the result of pasting <code>colname + '_' + column value</code> and the values are the corresponding mapped integer.</p> </li> <li> <code>inverse_encoding_dict</code>               (<code>Dict</code>)           \u2013            <p>the inverse encoding dictionary</p> </li> <li> <code>wide_dim</code>               (<code>int</code>)           \u2013            <p>Dimension of the wide model (i.e. dim of the linear layer)</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import pandas as pd\n&gt;&gt;&gt; from pytorch_widedeep.preprocessing import WidePreprocessor\n&gt;&gt;&gt; df = pd.DataFrame({'color': ['r', 'b', 'g'], 'size': ['s', 'n', 'l']})\n&gt;&gt;&gt; wide_cols = ['color']\n&gt;&gt;&gt; crossed_cols = [('color', 'size')]\n&gt;&gt;&gt; wide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols)\n&gt;&gt;&gt; X_wide = wide_preprocessor.fit_transform(df)\n&gt;&gt;&gt; X_wide\narray([[1, 4],\n       [2, 5],\n       [3, 6]])\n&gt;&gt;&gt; wide_preprocessor.encoding_dict\n{'color_r': 1, 'color_b': 2, 'color_g': 3, 'color_size_r-s': 4, 'color_size_b-n': 5, 'color_size_g-l': 6}\n&gt;&gt;&gt; wide_preprocessor.inverse_transform(X_wide)\n  color color_size\n0     r        r-s\n1     b        b-n\n2     g        g-l\n</code></pre> Source code in <code>pytorch_widedeep/preprocessing/wide_preprocessor.py</code> <pre><code>def __init__(\n    self, wide_cols: List[str], crossed_cols: Optional[List[Tuple[str, str]]] = None\n):\n    super(WidePreprocessor, self).__init__()\n\n    self.wide_cols = wide_cols\n    self.crossed_cols = crossed_cols\n\n    self.is_fitted = False\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.wide_preprocessor.WidePreprocessor.fit","title":"fit","text":"<pre><code>fit(df)\n</code></pre> <p>Fits the Preprocessor and creates required attributes</p> <p>Parameters:</p> <ul> <li> <code>df</code>               (<code>DataFrame</code>)           \u2013            <p>Input pandas dataframe</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>WidePreprocessor</code>           \u2013            <p><code>WidePreprocessor</code> fitted object</p> </li> </ul> Source code in <code>pytorch_widedeep/preprocessing/wide_preprocessor.py</code> <pre><code>def fit(self, df: pd.DataFrame) -&gt; \"WidePreprocessor\":\n    r\"\"\"Fits the Preprocessor and creates required attributes\n\n    Parameters\n    ----------\n    df: pd.DataFrame\n        Input pandas dataframe\n\n    Returns\n    -------\n    WidePreprocessor\n        `WidePreprocessor` fitted object\n    \"\"\"\n    df_wide = self._prepare_wide(df)\n    self.wide_crossed_cols = df_wide.columns.tolist()\n    glob_feature_list = self._make_global_feature_list(\n        df_wide[self.wide_crossed_cols]\n    )\n    # leave 0 for padding/\"unseen\" categories\n    self.encoding_dict = {v: i + 1 for i, v in enumerate(glob_feature_list)}\n    self.wide_dim = len(self.encoding_dict)\n    self.inverse_encoding_dict = {k: v for v, k in self.encoding_dict.items()}\n    self.inverse_encoding_dict[0] = \"unseen\"\n\n    self.is_fitted = True\n\n    return self\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.wide_preprocessor.WidePreprocessor.transform","title":"transform","text":"<pre><code>transform(df)\n</code></pre> <p>Parameters:</p> <ul> <li> <code>df</code>               (<code>DataFrame</code>)           \u2013            <p>Input pandas dataframe</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>ndarray</code>           \u2013            <p>transformed input dataframe</p> </li> </ul> Source code in <code>pytorch_widedeep/preprocessing/wide_preprocessor.py</code> <pre><code>def transform(self, df: pd.DataFrame) -&gt; np.ndarray:\n    r\"\"\"\n    Parameters\n    ----------\n    df: pd.DataFrame\n        Input pandas dataframe\n\n    Returns\n    -------\n    np.ndarray\n        transformed input dataframe\n    \"\"\"\n    check_is_fitted(self, attributes=[\"encoding_dict\"])\n    df_wide = self._prepare_wide(df)\n    encoded = np.zeros([len(df_wide), len(self.wide_crossed_cols)])\n    for col_i, col in enumerate(self.wide_crossed_cols):\n        encoded[:, col_i] = df_wide[col].apply(\n            lambda x: (\n                self.encoding_dict[col + \"_\" + str(x)]\n                if col + \"_\" + str(x) in self.encoding_dict\n                else 0\n            )\n        )\n    return encoded.astype(\"int64\")\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.wide_preprocessor.WidePreprocessor.inverse_transform","title":"inverse_transform","text":"<pre><code>inverse_transform(encoded)\n</code></pre> <p>Takes as input the output from the <code>transform</code> method and it will return the original values.</p> <p>Parameters:</p> <ul> <li> <code>encoded</code>               (<code>ndarray</code>)           \u2013            <p>numpy array with the encoded values that are the output from the <code>transform</code> method</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>DataFrame</code>           \u2013            <p>Pandas dataframe with the original values</p> </li> </ul> Source code in <code>pytorch_widedeep/preprocessing/wide_preprocessor.py</code> <pre><code>def inverse_transform(self, encoded: np.ndarray) -&gt; pd.DataFrame:\n    r\"\"\"Takes as input the output from the `transform` method and it will\n    return the original values.\n\n    Parameters\n    ----------\n    encoded: np.ndarray\n        numpy array with the encoded values that are the output from the\n        `transform` method\n\n    Returns\n    -------\n    pd.DataFrame\n        Pandas dataframe with the original values\n    \"\"\"\n    decoded = pd.DataFrame(encoded, columns=self.wide_crossed_cols)\n\n    if pd.__version__ &gt;= \"2.1.0\":\n        decoded = decoded.map(lambda x: self.inverse_encoding_dict[x])\n    else:\n        decoded = decoded.applymap(lambda x: self.inverse_encoding_dict[x])\n\n    for col in decoded.columns:\n        rm_str = \"\".join([col, \"_\"])\n        decoded[col] = decoded[col].apply(lambda x: x.replace(rm_str, \"\"))\n    return decoded\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.wide_preprocessor.WidePreprocessor.fit_transform","title":"fit_transform","text":"<pre><code>fit_transform(df)\n</code></pre> <p>Combines <code>fit</code> and <code>transform</code></p> <p>Parameters:</p> <ul> <li> <code>df</code>               (<code>DataFrame</code>)           \u2013            <p>Input pandas dataframe</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>ndarray</code>           \u2013            <p>transformed input dataframe</p> </li> </ul> Source code in <code>pytorch_widedeep/preprocessing/wide_preprocessor.py</code> <pre><code>def fit_transform(self, df: pd.DataFrame) -&gt; np.ndarray:\n    \"\"\"Combines `fit` and `transform`\n\n    Parameters\n    ----------\n    df: pd.DataFrame\n        Input pandas dataframe\n\n    Returns\n    -------\n    np.ndarray\n        transformed input dataframe\n    \"\"\"\n    return self.fit(df).transform(df)\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor","title":"TabPreprocessor","text":"<pre><code>TabPreprocessor(\n    cat_embed_cols=None,\n    continuous_cols=None,\n    quantization_setup=None,\n    cols_to_scale=None,\n    auto_embed_dim=True,\n    embedding_rule=\"fastai_new\",\n    default_embed_dim=16,\n    with_attention=False,\n    with_cls_token=False,\n    shared_embed=False,\n    verbose=1,\n    *,\n    scale=False,\n    already_standard=None,\n    **kwargs\n)\n</code></pre> <p>               Bases: <code>BasePreprocessor</code></p> <p>Preprocessor to prepare the <code>deeptabular</code> component input dataset</p> <p>Parameters:</p> <ul> <li> <code>cat_embed_cols</code>               (<code>Optional[Union[List[str], List[Tuple[str, int]]]]</code>, default:                   <code>None</code> )           \u2013            <p>List containing the name of the categorical columns that will be represented by embeddings (e.g. ['education', 'relationship', ...]) or a Tuple with the name and the embedding dimension (e.g.: [ ('education',32), ('relationship',16), ...])</p> </li> <li> <code>continuous_cols</code>               (<code>Optional[List[str]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the name of the continuous cols</p> </li> <li> <code>quantization_setup</code>               (<code>Optional[Union[int, Dict[str, Union[int, List[float]]]]]</code>, default:                   <code>None</code> )           \u2013            <p>Continuous columns can be turned into categorical via <code>pd.cut</code>. If <code>quantization_setup</code> is an <code>int</code>, all continuous columns will be quantized using this value as the number of bins. Alternatively, a dictionary where the keys are the column names to quantize and the values are the either integers indicating the number of bins or a list of scalars indicating the bin edges can also be used.</p> </li> <li> <code>cols_to_scale</code>               (<code>Optional[Union[List[str], str]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the names of the columns that will be standarised via sklearn's <code>StandardScaler</code>. It can also be the string <code>'all'</code> in which case all the continuous cols will be scaled.</p> </li> <li> <code>auto_embed_dim</code>               (<code>bool</code>, default:                   <code>True</code> )           \u2013            <p>Boolean indicating whether the embedding dimensions will be automatically defined via rule of thumb. See <code>embedding_rule</code> below.</p> </li> <li> <code>embedding_rule</code>               (<code>Literal[google, fastai_old, fastai_new]</code>, default:                   <code>'fastai_new'</code> )           \u2013            <p>If <code>auto_embed_dim=True</code>, this is the choice of embedding rule of thumb. Choices are:</p> <ul> <li> <p>fastai_new: \\(min(600, round(1.6 \\times n_{cat}^{0.56}))\\)</p> </li> <li> <p>fastai_old: \\(min(50, (n_{cat}//{2})+1)\\)</p> </li> <li> <p>google: \\(min(600, round(n_{cat}^{0.24}))\\)</p> </li> </ul> </li> <li> <code>default_embed_dim</code>               (<code>int</code>, default:                   <code>16</code> )           \u2013            <p>Dimension for the embeddings if the embedding dimension is not provided in the <code>cat_embed_cols</code> parameter and <code>auto_embed_dim</code> is set to <code>False</code>.</p> </li> <li> <code>with_attention</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether the preprocessed data will be passed to an attention-based model (more precisely a model where all embeddings must have the same dimensions). If <code>True</code>, the param <code>cat_embed_cols</code> must just be a list containing just the categorical column names: e.g. ['education', 'relationship', ...]. This is because they will all be  encoded using embeddings of the same dim, which will be specified  later when the model is defined.  Param alias:  <code>for_transformer</code></p> </li> <li> <code>with_cls_token</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating if a <code>'[CLS]'</code> token will be added to the dataset when using attention-based models. The final hidden state corresponding to this token is used as the aggregated representation for classification and regression tasks. If not, the categorical and/or continuous embeddings will be concatenated before being passed to the final MLP (if present).</p> </li> <li> <code>shared_embed</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating if the embeddings will be \"shared\" when using attention-based models. The idea behind <code>shared_embed</code> is described in the Appendix A in the TabTransformer paper: 'The goal of having column embedding is to enable the model to distinguish the classes in one column from those in the other columns'. In other words, the idea is to let the model learn which column is embedded at the time. See: <code>pytorch_widedeep.models.transformers._layers.SharedEmbeddings</code>.</p> </li> <li> <code>verbose</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            </li> <li> <code>scale</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p> note: this arg will be removed in upcoming  releases. Please use <code>cols_to_scale</code> instead.  Bool indicating  whether or not to scale/standarise continuous cols. It is important  to emphasize that all the DL models for tabular data in the library  also include the possibility of normalising the input continuous  features via a <code>BatchNorm</code> or a <code>LayerNorm</code>.  Param alias:  <code>scale_cont_cols</code>.</p> </li> <li> <code>already_standard</code>               (<code>Optional[List[str]]</code>, default:                   <code>None</code> )           \u2013            <p> note: this arg will be removed in upcoming  releases. Please use <code>cols_to_scale</code> instead.  List with the  name of the continuous cols that do not need to be  scaled/standarised.</p> </li> </ul> <p>Other Parameters:</p> <ul> <li> <code>**kwargs</code>           \u2013            <p><code>pd.cut</code> and <code>StandardScaler</code> related args</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>embed_dim</code>               (<code>Dict</code>)           \u2013            <p>Dictionary where keys are the embed cols and values are the embedding dimensions. If <code>with_attention</code> is set to <code>True</code> this attribute is not generated during the <code>fit</code> process</p> </li> <li> <code>label_encoder</code>               (<code>LabelEncoder</code>)           \u2013            <p>see <code>pytorch_widedeep.utils.dense_utils.LabelEncder</code></p> </li> <li> <code>cat_embed_input</code>               (<code>List</code>)           \u2013            <p>List of Tuples with the column name, number of individual values for that column and, If <code>with_attention</code> is set to <code>False</code>, the corresponding embeddings dim, e.g. [('education', 16, 10), ('relationship', 6, 8), ...].</p> </li> <li> <code>standardize_cols</code>               (<code>List</code>)           \u2013            <p>List of the columns that will be standarized</p> </li> <li> <code>scaler</code>               (<code>StandardScaler</code>)           \u2013            <p>an instance of <code>sklearn.preprocessing.StandardScaler</code></p> </li> <li> <code>column_idx</code>               (<code>Dict</code>)           \u2013            <p>Dictionary where keys are column names and values are column indexes. This is neccesary to slice tensors</p> </li> <li> <code>quantizer</code>               (<code>Quantizer</code>)           \u2013            <p>an instance of <code>Quantizer</code></p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import pandas as pd\n&gt;&gt;&gt; import numpy as np\n&gt;&gt;&gt; from pytorch_widedeep.preprocessing import TabPreprocessor\n&gt;&gt;&gt; df = pd.DataFrame({'color': ['r', 'b', 'g'], 'size': ['s', 'n', 'l'], 'age': [25, 40, 55]})\n&gt;&gt;&gt; cat_embed_cols = [('color',5), ('size',5)]\n&gt;&gt;&gt; cont_cols = ['age']\n&gt;&gt;&gt; deep_preprocessor = TabPreprocessor(cat_embed_cols=cat_embed_cols, continuous_cols=cont_cols)\n&gt;&gt;&gt; X_tab = deep_preprocessor.fit_transform(df)\n&gt;&gt;&gt; deep_preprocessor.cat_embed_cols\n[('color', 5), ('size', 5)]\n&gt;&gt;&gt; deep_preprocessor.column_idx\n{'color': 0, 'size': 1, 'age': 2}\n&gt;&gt;&gt; cont_df = pd.DataFrame({\"col1\": np.random.rand(10), \"col2\": np.random.rand(10) + 1})\n&gt;&gt;&gt; cont_cols = [\"col1\", \"col2\"]\n&gt;&gt;&gt; tab_preprocessor = TabPreprocessor(continuous_cols=cont_cols, quantization_setup=3)\n&gt;&gt;&gt; ft_cont_df = tab_preprocessor.fit_transform(cont_df)\n&gt;&gt;&gt; # or...\n&gt;&gt;&gt; quantization_setup = {'col1': [0., 0.4, 1.], 'col2': [1., 1.4, 2.]}\n&gt;&gt;&gt; tab_preprocessor2 = TabPreprocessor(continuous_cols=cont_cols, quantization_setup=quantization_setup)\n&gt;&gt;&gt; ft_cont_df2 = tab_preprocessor2.fit_transform(cont_df)\n</code></pre> Source code in <code>pytorch_widedeep/preprocessing/tab_preprocessor.py</code> <pre><code>@alias(\"with_attention\", [\"for_transformer\", \"for_matrix_factorization\", \"for_mf\"])\n@alias(\"cat_embed_cols\", [\"embed_cols\"])\n@alias(\"scale\", [\"scale_cont_cols\"])\n@alias(\"quantization_setup\", [\"cols_and_bins\"])\ndef __init__(\n    self,\n    cat_embed_cols: Optional[Union[List[str], List[Tuple[str, int]]]] = None,\n    continuous_cols: Optional[List[str]] = None,\n    quantization_setup: Optional[\n        Union[int, Dict[str, Union[int, List[float]]]]\n    ] = None,\n    cols_to_scale: Optional[Union[List[str], str]] = None,\n    auto_embed_dim: bool = True,\n    embedding_rule: Literal[\"google\", \"fastai_old\", \"fastai_new\"] = \"fastai_new\",\n    default_embed_dim: int = 16,\n    with_attention: bool = False,\n    with_cls_token: bool = False,\n    shared_embed: bool = False,\n    verbose: int = 1,\n    *,\n    scale: bool = False,\n    already_standard: Optional[List[str]] = None,\n    **kwargs,\n):\n    super(TabPreprocessor, self).__init__()\n\n    self.continuous_cols = continuous_cols\n    self.quantization_setup = quantization_setup\n    self.cols_to_scale = cols_to_scale\n    self.scale = scale\n    self.already_standard = already_standard\n    self.auto_embed_dim = auto_embed_dim\n    self.embedding_rule = embedding_rule\n    self.default_embed_dim = default_embed_dim\n    self.with_attention = with_attention\n    self.with_cls_token = with_cls_token\n    self.shared_embed = shared_embed\n    self.verbose = verbose\n\n    self.quant_args = {\n        k: v for k, v in kwargs.items() if k in pd.cut.__code__.co_varnames\n    }\n    self.scale_args = {\n        k: v for k, v in kwargs.items() if k in StandardScaler().get_params()\n    }\n\n    self._check_inputs(cat_embed_cols)\n\n    if with_cls_token:\n        self.cat_embed_cols = (\n            [\"cls_token\"] + cat_embed_cols  # type: ignore[operator]\n            if cat_embed_cols is not None\n            else [\"cls_token\"]\n        )\n    else:\n        self.cat_embed_cols = cat_embed_cols  # type: ignore[assignment]\n\n    self.is_fitted = False\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor.fit","title":"fit","text":"<pre><code>fit(df)\n</code></pre> <p>Fits the Preprocessor and creates required attributes</p> <p>Parameters:</p> <ul> <li> <code>df</code>               (<code>DataFrame</code>)           \u2013            <p>Input pandas dataframe</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>TabPreprocessor</code>           \u2013            <p><code>TabPreprocessor</code> fitted object</p> </li> </ul> Source code in <code>pytorch_widedeep/preprocessing/tab_preprocessor.py</code> <pre><code>def fit(self, df: pd.DataFrame) -&gt; BasePreprocessor:  # noqa: C901\n    \"\"\"Fits the Preprocessor and creates required attributes\n\n    Parameters\n    ----------\n    df: pd.DataFrame\n        Input pandas dataframe\n\n    Returns\n    -------\n    TabPreprocessor\n        `TabPreprocessor` fitted object\n    \"\"\"\n\n    df_adj = self._insert_cls_token(df) if self.with_cls_token else df.copy()\n\n    self.column_idx: Dict[str, int] = {}\n\n    # Categorical embeddings logic\n    if self.cat_embed_cols is not None or self.quantization_setup is not None:\n        self.cat_embed_input: List[Union[Tuple[str, int], Tuple[str, int, int]]] = (\n            []\n        )\n\n    if self.cat_embed_cols is not None:\n        df_cat, cat_embed_dim = self._prepare_categorical(df_adj)\n\n        self.label_encoder = LabelEncoder(\n            columns_to_encode=df_cat.columns.tolist(),\n            shared_embed=self.shared_embed,\n            with_attention=self.with_attention,\n        )\n        self.label_encoder.fit(df_cat)\n\n        for k, v in self.label_encoder.encoding_dict.items():\n            if self.with_attention:\n                self.cat_embed_input.append((k, len(v)))\n            else:\n                self.cat_embed_input.append((k, len(v), cat_embed_dim[k]))\n\n        self.column_idx.update({k: v for v, k in enumerate(df_cat.columns)})\n\n    # Continuous columns logic\n    if self.continuous_cols is not None:\n        df_cont, cont_embed_dim = self._prepare_continuous(df_adj)\n\n        # Standardization logic\n        if self.standardize_cols is not None:\n            self.scaler = StandardScaler(**self.scale_args).fit(\n                df_cont[self.standardize_cols].values\n            )\n        elif self.verbose:\n            warnings.warn(\"Continuous columns will not be normalised\")\n\n        # Quantization logic\n        if self.cols_and_bins is not None:\n            # we do not run 'Quantizer.fit' here since in the wild case\n            # someone wants standardization and quantization for the same\n            # columns, the Quantizer will run on the scaled data\n            self.quantizer = Quantizer(self.cols_and_bins, **self.quant_args)\n\n            if self.with_attention:\n                for col, n_cat, _ in cont_embed_dim:\n                    self.cat_embed_input.append((col, n_cat))\n            else:\n                self.cat_embed_input.extend(cont_embed_dim)\n\n        self.column_idx.update(\n            {k: v + len(self.column_idx) for v, k in enumerate(df_cont)}\n        )\n\n    self.is_fitted = True\n\n    return self\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor.transform","title":"transform","text":"<pre><code>transform(df)\n</code></pre> <p>Returns the processed <code>dataframe</code> as a np.ndarray</p> <p>Parameters:</p> <ul> <li> <code>df</code>               (<code>DataFrame</code>)           \u2013            <p>Input pandas dataframe</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>ndarray</code>           \u2013            <p>transformed input dataframe</p> </li> </ul> Source code in <code>pytorch_widedeep/preprocessing/tab_preprocessor.py</code> <pre><code>def transform(self, df: pd.DataFrame) -&gt; np.ndarray:  # noqa: C901\n    \"\"\"Returns the processed `dataframe` as a np.ndarray\n\n    Parameters\n    ----------\n    df: pd.DataFrame\n        Input pandas dataframe\n\n    Returns\n    -------\n    np.ndarray\n        transformed input dataframe\n    \"\"\"\n    check_is_fitted(self, condition=self.is_fitted)\n\n    df_adj = self._insert_cls_token(df) if self.with_cls_token else df.copy()\n\n    if self.cat_embed_cols is not None:\n        df_cat = df_adj[self.cat_cols]\n        df_cat = self.label_encoder.transform(df_cat)\n    if self.continuous_cols is not None:\n        df_cont = df_adj[self.continuous_cols]\n        # Standardization logic\n        if self.standardize_cols:\n            df_cont[self.standardize_cols] = self.scaler.transform(\n                df_cont[self.standardize_cols].values\n            )\n        # Quantization logic\n        if self.cols_and_bins is not None:\n            # Adjustment so I don't have to override the method\n            # in 'ChunkTabPreprocessor'\n            if self.quantizer.is_fitted:\n                df_cont = self.quantizer.transform(df_cont)\n            else:\n                df_cont = self.quantizer.fit_transform(df_cont)\n    try:\n        df_deep = pd.concat([df_cat, df_cont], axis=1)\n    except NameError:\n        try:\n            df_deep = df_cat.copy()\n        except NameError:\n            df_deep = df_cont.copy()\n\n    return df_deep.values\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor.inverse_transform","title":"inverse_transform","text":"<pre><code>inverse_transform(encoded)\n</code></pre> <p>Takes as input the output from the <code>transform</code> method and it will return the original values.</p> <p>Parameters:</p> <ul> <li> <code>encoded</code>               (<code>ndarray</code>)           \u2013            <p>array with the output of the <code>transform</code> method</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>DataFrame</code>           \u2013            <p>Pandas dataframe with the original values</p> </li> </ul> Source code in <code>pytorch_widedeep/preprocessing/tab_preprocessor.py</code> <pre><code>def inverse_transform(self, encoded: np.ndarray) -&gt; pd.DataFrame:  # noqa: C901\n    r\"\"\"Takes as input the output from the `transform` method and it will\n    return the original values.\n\n    Parameters\n    ----------\n    encoded: np.ndarray\n        array with the output of the `transform` method\n\n    Returns\n    -------\n    pd.DataFrame\n        Pandas dataframe with the original values\n    \"\"\"\n    decoded = pd.DataFrame(encoded, columns=list(self.column_idx.keys()))\n    # embeddings back to original category\n    if self.cat_embed_cols is not None:\n        decoded = self.label_encoder.inverse_transform(decoded)\n    if self.continuous_cols is not None:\n        # quantized cols to the mid point\n        if self.cols_and_bins is not None:\n            if self.verbose:\n                print(\n                    \"Note that quantized cols will be turned into the mid point of \"\n                    \"the corresponding bin\"\n                )\n            for k, v in self.quantizer.inversed_bins.items():\n                decoded[k] = decoded[k].map(v)\n        # continuous_cols back to non-standarised\n        try:\n            decoded[self.standardize_cols] = self.scaler.inverse_transform(\n                decoded[self.standardize_cols]\n            )\n        except Exception:  # KeyError:\n            pass\n\n    if \"cls_token\" in decoded.columns:\n        decoded.drop(\"cls_token\", axis=1, inplace=True)\n\n    return decoded\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor.fit_transform","title":"fit_transform","text":"<pre><code>fit_transform(df)\n</code></pre> <p>Combines <code>fit</code> and <code>transform</code></p> <p>Parameters:</p> <ul> <li> <code>df</code>               (<code>DataFrame</code>)           \u2013            <p>Input pandas dataframe</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>ndarray</code>           \u2013            <p>transformed input dataframe</p> </li> </ul> Source code in <code>pytorch_widedeep/preprocessing/tab_preprocessor.py</code> <pre><code>def fit_transform(self, df: pd.DataFrame) -&gt; np.ndarray:\n    \"\"\"Combines `fit` and `transform`\n\n    Parameters\n    ----------\n    df: pd.DataFrame\n        Input pandas dataframe\n\n    Returns\n    -------\n    np.ndarray\n        transformed input dataframe\n    \"\"\"\n    return self.fit(df).transform(df)\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.tab_preprocessor.Quantizer","title":"Quantizer","text":"<pre><code>Quantizer(quantization_setup, **kwargs)\n</code></pre> <p>Helper class to perform the quantization of continuous columns. It is included in this docs for completion, since depending on the value of the parameter <code>'quantization_setup'</code> of the <code>TabPreprocessor</code> class, that class might have an attribute of type <code>Quantizer</code>. However, this class is designed to always run internally within the <code>TabPreprocessor</code> class.</p> <p>Parameters:</p> <ul> <li> <code>quantization_setup</code>               (<code>Dict[str, Union[int, List[float]]]</code>)           \u2013            <p>Dictionary where the keys are the column names to quantize and the values are the either integers indicating the number of bins or a list of scalars indicating the bin edges.</p> </li> </ul> Source code in <code>pytorch_widedeep/preprocessing/tab_preprocessor.py</code> <pre><code>def __init__(\n    self,\n    quantization_setup: Dict[str, Union[int, List[float]]],\n    **kwargs,\n):\n    self.quantization_setup = quantization_setup\n    self.quant_args = kwargs\n\n    self.is_fitted = False\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor","title":"TextPreprocessor","text":"<pre><code>TextPreprocessor(\n    text_col,\n    max_vocab=30000,\n    min_freq=5,\n    maxlen=80,\n    pad_first=True,\n    pad_idx=1,\n    already_processed=False,\n    word_vectors_path=None,\n    n_cpus=None,\n    verbose=1,\n)\n</code></pre> <p>               Bases: <code>BasePreprocessor</code></p> <p>Preprocessor to prepare the <code>deeptext</code> input dataset</p> <p>Parameters:</p> <ul> <li> <code>text_col</code>               (<code>str</code>)           \u2013            <p>column in the input dataframe containing the texts</p> </li> <li> <code>max_vocab</code>               (<code>int</code>, default:                   <code>30000</code> )           \u2013            <p>Maximum number of tokens in the vocabulary</p> </li> <li> <code>min_freq</code>               (<code>int</code>, default:                   <code>5</code> )           \u2013            <p>Minimum frequency for a token to be part of the vocabulary</p> </li> <li> <code>maxlen</code>               (<code>int</code>, default:                   <code>80</code> )           \u2013            <p>Maximum length of the tokenized sequences</p> </li> <li> <code>pad_first</code>               (<code>bool</code>, default:                   <code>True</code> )           \u2013            <p>Indicates whether the padding index will be added at the beginning or the end of the sequences</p> </li> <li> <code>pad_idx</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>padding index. Fastai's Tokenizer leaves 0 for the 'unknown' token.</p> </li> <li> <code>already_processed</code>               (<code>Optional[bool]</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating if the sequence of elements is already processed or prepared. If this is the case, this Preprocessor will simply tokenize and pad the sequence. </p> <pre><code>Param aliases: `not_text`. &lt;br/&gt;\n</code></pre> <p>This parameter is thought for those cases where the input sequences are already fully processed or are directly not text (e.g. IDs)</p> </li> <li> <code>word_vectors_path</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Path to the pretrained word vectors</p> </li> <li> <code>n_cpus</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>number of CPUs to used during the tokenization process</p> </li> <li> <code>verbose</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>Enable verbose output.</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>vocab</code>               (<code>Vocab</code>)           \u2013            <p>an instance of <code>pytorch_widedeep.utils.fastai_transforms.Vocab</code></p> </li> <li> <code>embedding_matrix</code>               (<code>ndarray</code>)           \u2013            <p>Array with the pretrained embeddings</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import pandas as pd\n&gt;&gt;&gt; from pytorch_widedeep.preprocessing import TextPreprocessor\n&gt;&gt;&gt; df_train = pd.DataFrame({'text_column': [\"life is like a box of chocolates\",\n... \"You never know what you're gonna get\"]})\n&gt;&gt;&gt; text_preprocessor = TextPreprocessor(text_col='text_column', max_vocab=25, min_freq=1, maxlen=10)\n&gt;&gt;&gt; text_preprocessor.fit_transform(df_train)\nThe vocabulary contains 24 tokens\narray([[ 1,  1,  1,  1, 10, 11, 12, 13, 14, 15],\n       [ 5,  9, 16, 17, 18,  9, 19, 20, 21, 22]], dtype=int32)\n&gt;&gt;&gt; df_te = pd.DataFrame({'text_column': ['you never know what is in the box']})\n&gt;&gt;&gt; text_preprocessor.transform(df_te)\narray([[ 1,  1,  9, 16, 17, 18, 11,  0,  0, 13]], dtype=int32)\n</code></pre> Source code in <code>pytorch_widedeep/preprocessing/text_preprocessor.py</code> <pre><code>@alias(\"already_processed\", [\"not_text\"])\ndef __init__(\n    self,\n    text_col: str,\n    max_vocab: int = 30000,\n    min_freq: int = 5,\n    maxlen: int = 80,\n    pad_first: bool = True,\n    pad_idx: int = 1,\n    already_processed: Optional[bool] = False,\n    word_vectors_path: Optional[str] = None,\n    n_cpus: Optional[int] = None,\n    verbose: int = 1,\n):\n    super(TextPreprocessor, self).__init__()\n\n    self.text_col = text_col\n    self.max_vocab = max_vocab\n    self.min_freq = min_freq\n    self.maxlen = maxlen\n    self.pad_first = pad_first\n    self.pad_idx = pad_idx\n    self.already_processed = already_processed\n    self.word_vectors_path = word_vectors_path\n    self.verbose = verbose\n    self.n_cpus = n_cpus if n_cpus is not None else os.cpu_count()\n\n    self.is_fitted = False\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor.fit","title":"fit","text":"<pre><code>fit(df)\n</code></pre> <p>Builds the vocabulary</p> <p>Parameters:</p> <ul> <li> <code>df</code>               (<code>DataFrame</code>)           \u2013            <p>Input pandas dataframe</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>TextPreprocessor</code>           \u2013            <p><code>TextPreprocessor</code> fitted object</p> </li> </ul> Source code in <code>pytorch_widedeep/preprocessing/text_preprocessor.py</code> <pre><code>def fit(self, df: pd.DataFrame) -&gt; BasePreprocessor:\n    \"\"\"Builds the vocabulary\n\n    Parameters\n    ----------\n    df: pd.DataFrame\n        Input pandas dataframe\n\n    Returns\n    -------\n    TextPreprocessor\n        `TextPreprocessor` fitted object\n    \"\"\"\n    texts = self._read_texts(df)\n\n    tokens = get_texts(texts, self.already_processed, self.n_cpus)\n\n    self.vocab: TVocab = Vocab(\n        max_vocab=self.max_vocab,\n        min_freq=self.min_freq,\n        pad_idx=self.pad_idx,\n    ).fit(\n        tokens,\n    )\n\n    if self.verbose:\n        print(\"The vocabulary contains {} tokens\".format(len(self.vocab.stoi)))\n    if self.word_vectors_path is not None:\n        self.embedding_matrix = build_embeddings_matrix(\n            self.vocab, self.word_vectors_path, self.min_freq\n        )\n\n    self.is_fitted = True\n\n    return self\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor.transform","title":"transform","text":"<pre><code>transform(df)\n</code></pre> <p>Returns the padded, 'numericalised' sequences</p> <p>Parameters:</p> <ul> <li> <code>df</code>               (<code>DataFrame</code>)           \u2013            <p>Input pandas dataframe</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>ndarray</code>           \u2013            <p>Padded, 'numericalised' sequences</p> </li> </ul> Source code in <code>pytorch_widedeep/preprocessing/text_preprocessor.py</code> <pre><code>def transform(self, df: pd.DataFrame) -&gt; np.ndarray:\n    \"\"\"Returns the padded, _'numericalised'_ sequences\n\n    Parameters\n    ----------\n    df: pd.DataFrame\n        Input pandas dataframe\n\n    Returns\n    -------\n    np.ndarray\n        Padded, _'numericalised'_ sequences\n    \"\"\"\n    check_is_fitted(self, attributes=[\"vocab\"])\n    texts = self._read_texts(df)\n    tokens = get_texts(texts, self.already_processed, self.n_cpus)\n    return self._pad_sequences(tokens)\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor.transform_sample","title":"transform_sample","text":"<pre><code>transform_sample(text)\n</code></pre> <p>Returns the padded, 'numericalised' sequence</p> <p>Parameters:</p> <ul> <li> <code>text</code>               (<code>str</code>)           \u2013            <p>text to be tokenized and padded</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>ndarray</code>           \u2013            <p>Padded, 'numericalised' sequence</p> </li> </ul> Source code in <code>pytorch_widedeep/preprocessing/text_preprocessor.py</code> <pre><code>def transform_sample(self, text: str) -&gt; np.ndarray:\n    \"\"\"Returns the padded, _'numericalised'_ sequence\n\n    Parameters\n    ----------\n    text: str\n        text to be tokenized and padded\n\n    Returns\n    -------\n    np.ndarray\n        Padded, _'numericalised'_ sequence\n    \"\"\"\n    check_is_fitted(self, attributes=[\"vocab\"])\n    tokens = get_texts([text], self.already_processed, self.n_cpus)\n    return self._pad_sequences(tokens)[0]\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor.fit_transform","title":"fit_transform","text":"<pre><code>fit_transform(df)\n</code></pre> <p>Combines <code>fit</code> and <code>transform</code></p> <p>Parameters:</p> <ul> <li> <code>df</code>               (<code>DataFrame</code>)           \u2013            <p>Input pandas dataframe</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>ndarray</code>           \u2013            <p>Padded, 'numericalised' sequences</p> </li> </ul> Source code in <code>pytorch_widedeep/preprocessing/text_preprocessor.py</code> <pre><code>def fit_transform(self, df: pd.DataFrame) -&gt; np.ndarray:\n    \"\"\"Combines `fit` and `transform`\n\n    Parameters\n    ----------\n    df: pd.DataFrame\n        Input pandas dataframe\n\n    Returns\n    -------\n    np.ndarray\n        Padded, _'numericalised'_ sequences\n    \"\"\"\n    return self.fit(df).transform(df)\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor.inverse_transform","title":"inverse_transform","text":"<pre><code>inverse_transform(padded_seq)\n</code></pre> <p>Returns the original text plus the added 'special' tokens</p> <p>Parameters:</p> <ul> <li> <code>padded_seq</code>               (<code>ndarray</code>)           \u2013            <p>array with the output of the <code>transform</code> method</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>DataFrame</code>           \u2013            <p>Pandas dataframe with the original text plus the added 'special' tokens</p> </li> </ul> Source code in <code>pytorch_widedeep/preprocessing/text_preprocessor.py</code> <pre><code>def inverse_transform(self, padded_seq: np.ndarray) -&gt; pd.DataFrame:\n    \"\"\"Returns the original text plus the added 'special' tokens\n\n    Parameters\n    ----------\n    padded_seq: np.ndarray\n        array with the output of the `transform` method\n\n    Returns\n    -------\n    pd.DataFrame\n        Pandas dataframe with the original text plus the added 'special' tokens\n    \"\"\"\n    texts = [self.vocab.inverse_transform(num) for num in padded_seq]\n    return pd.DataFrame({self.text_col: texts})\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor","title":"HFPreprocessor","text":"<pre><code>HFPreprocessor(\n    model_name,\n    *,\n    use_fast_tokenizer=False,\n    text_col=None,\n    root_dir=None,\n    num_workers=None,\n    preprocessing_rules=None,\n    tokenizer_params=None,\n    encode_params=None,\n    **kwargs\n)\n</code></pre> <p>               Bases: <code>BasePreprocessor</code></p> <p>Text processor to prepare the <code>deeptext</code> input dataset that is a wrapper around HuggingFace's tokenizers.</p> <p>Following the main phylosophy of the <code>pytorch-widedeep</code> library, this class is designed to be as flexible as possible. Therefore, it is coded so that the user can use it as one would use any HuggingFace tokenizers, or following the API call 'protocol' of the rest of the library.</p> <p>Parameters:</p> <ul> <li> <code>model_name</code>               (<code>str</code>)           \u2013            <p>The model name from the transformers library e.g. 'bert-base-uncased'. Currently supported models are those from the families: BERT, RoBERTa, DistilBERT, ALBERT and ELECTRA.</p> </li> <li> <code>use_fast_tokenizer</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Whether to use the fast tokenizer from HuggingFace or not</p> </li> <li> <code>text_col</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>The column in the input dataframe containing the text data. If this tokenizer is used via the <code>fit</code> and <code>transform</code> methods, this argument is mandatory. If the tokenizer is used via the <code>encode</code> method, this argument is not needed since the input text is passed directly to the <code>encode</code> method.</p> </li> <li> <code>num_workers</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>Number of workers to use when preprocessing the text data. If not None, and <code>use_fast_tokenizer</code> is False, the text data will be preprocessed in parallel using the number of workers specified. If <code>use_fast_tokenizer</code> is True, this argument is ignored.</p> </li> <li> <code>preprocessing_rules</code>               (<code>Optional[List[Callable[[str], str]]]</code>, default:                   <code>None</code> )           \u2013            <p>A list of functions to be applied to the text data before encoding. This can be useful to clean the text data before encoding. For example, removing html tags, special characters, etc.</p> </li> <li> <code>tokenizer_params</code>               (<code>Optional[Dict[str, Any]]</code>, default:                   <code>None</code> )           \u2013            <p>Additional parameters to be passed to the HuggingFace's <code>PreTrainedTokenizer</code>. Parameters to the <code>PreTrainedTokenizer</code> can also be passed via the <code>**kwargs</code> argument</p> </li> <li> <code>encode_params</code>               (<code>Optional[Dict[str, Any]]</code>, default:                   <code>None</code> )           \u2013            <p>Additional parameters to be passed to the <code>batch_encode_plus</code> method of the HuggingFace's <code>PreTrainedTokenizer</code>. If the <code>fit</code> and <code>transform</code> methods are used, the <code>encode_params</code> dict parameter is mandatory. If the <code>encode</code> method is used, this parameter is not needed since the input text is passed directly to the <code>encode</code> method.</p> </li> <li> <code>**kwargs</code>           \u2013            <p>Additional kwargs to be passed to the model, in particular to the <code>PreTrainedTokenizer</code> class.</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>is_fitted</code>               (<code>bool</code>)           \u2013            <p>Boolean indicating if the preprocessor has been fitted. This is a HuggingFacea tokenizer, so it is always considered fitted and this attribute is manually set to True internally. This parameter exists for consistency with the rest of the library and because is needed for some functionality in the library.</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import pandas as pd\n&gt;&gt;&gt; from pytorch_widedeep.preprocessing import HFPreprocessor\n&gt;&gt;&gt; df = pd.DataFrame({\"text\": [\"this is the first text\", \"this is the second text\"]})\n&gt;&gt;&gt; hf_processor_1 = HFPreprocessor(model_name=\"bert-base-uncased\", text_col=\"text\")\n&gt;&gt;&gt; X_text_1 = hf_processor_1.fit_transform(df)\n&gt;&gt;&gt; texts = [\"this is a new text\", \"this is another text\"]\n&gt;&gt;&gt; hf_processor_2 = HFPreprocessor(model_name=\"bert-base-uncased\")\n&gt;&gt;&gt; X_text_2 = hf_processor_2.encode(texts, max_length=10, padding=\"max_length\", truncation=True)\n</code></pre> Source code in <code>pytorch_widedeep/preprocessing/hf_preprocessor.py</code> <pre><code>def __init__(\n    self,\n    model_name: str,\n    *,\n    use_fast_tokenizer: bool = False,\n    text_col: Optional[str] = None,\n    root_dir: Optional[str] = None,\n    num_workers: Optional[int] = None,\n    preprocessing_rules: Optional[List[Callable[[str], str]]] = None,\n    tokenizer_params: Optional[Dict[str, Any]] = None,\n    encode_params: Optional[Dict[str, Any]] = None,\n    **kwargs,\n):\n    self.model_name = model_name\n    self.use_fast_tokenizer = use_fast_tokenizer\n    self.text_col = text_col\n    self.root_dir = root_dir\n    self.num_workers = num_workers\n    self.preprocessing_rules = preprocessing_rules\n    self.tokenizer_params = tokenizer_params if tokenizer_params is not None else {}\n    self.encode_params = encode_params if encode_params is not None else {}\n\n    self._multiprocessing = (\n        num_workers is not None and num_workers &gt; 1 and not use_fast_tokenizer\n    )\n\n    if kwargs:\n        self.tokenizer_params.update(kwargs)\n\n    self.tokenizer = get_tokenizer(\n        model_name=self.model_name,\n        use_fast_tokenizer=self.use_fast_tokenizer,\n        **self.tokenizer_params,\n    )\n\n    # A HuggingFace tokenizer is already trained, since we need this\n    # attribute elsewhere in the library, we simply set it to True\n    self.is_fitted = True\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.encode","title":"encode","text":"<pre><code>encode(texts, **kwargs)\n</code></pre> <p>Encodes a list of texts. The method is a wrapper around the <code>batch_encode_plus</code> method of the HuggingFace's tokenizer.</p> <p>if 'use_fast_tokenizer' is True, the method will use the <code>batch_encode_plus</code></p> <p>Parameters:</p> <ul> <li> <code>texts</code>               (<code>List[str]</code>)           \u2013            <p>List of texts to be encoded</p> </li> <li> <code>**kwargs</code>           \u2013            <p>Additional parameters to be passed to the <code>batch_encode_plus</code> method of the HuggingFace's tokenizer. If the 'encode_params' dict was passed when instantiating the class, that dictionaly will be updated with the kwargs passed here.</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>array</code>           \u2013            <p>The encoded texts</p> </li> </ul> Source code in <code>pytorch_widedeep/preprocessing/hf_preprocessor.py</code> <pre><code>def encode(self, texts: List[str], **kwargs) -&gt; npt.NDArray[np.int64]:\n    \"\"\"\n    Encodes a list of texts. The method is a wrapper around the\n    `batch_encode_plus` method of the HuggingFace's tokenizer.\n\n    if 'use_fast_tokenizer' is True, the method will use the `batch_encode_plus`\n\n    Parameters\n    ----------\n    texts: List[str]\n        List of texts to be encoded\n    **kwargs\n        Additional parameters to be passed to the `batch_encode_plus` method\n        of the HuggingFace's tokenizer. If the 'encode_params' dict was passed\n        when instantiating the class, that dictionaly will be updated with\n        the kwargs passed here.\n\n    Returns\n    -------\n    np.array\n        The encoded texts\n    \"\"\"\n    if kwargs:\n        self.encode_params.update(kwargs)\n\n    if self.preprocessing_rules:\n        if self._multiprocessing:\n            texts = self._process_text_parallel(texts)\n        else:\n            texts = [self._preprocess_text(text) for text in texts]\n\n    if self._multiprocessing:\n        input_ids = self._encode_paralell(texts, **self.encode_params)\n    else:\n        encoded_texts = self.tokenizer.batch_encode_plus(\n            texts,\n            **self.encode_params,\n        )\n        input_ids = encoded_texts.get(\"input_ids\")\n\n    self.is_fitted = True\n\n    try:\n        output = np.array(input_ids)\n    except ValueError:\n        warnings.warn(\n            \"Padding and Truncating parameters were not passed and all input arrays \"\n            \"do not have the same shape. Padding to the longest sequence. \"\n            \"Padding will be done with the index of the pad token for the model\",\n            UserWarning,\n        )\n        max_len = max([len(ids) for ids in input_ids])\n        output = np.array(\n            [\n                np.pad(ids, (self.tokenizer.pad_token_id, max_len - len(ids)))\n                for ids in input_ids\n            ]\n        )\n\n    return output\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.decode","title":"decode","text":"<pre><code>decode(input_ids, skip_special_tokens)\n</code></pre> <p>Decodes a list of input_ids. The method is a wrapper around the <code>convert_ids_to_tokens</code> and <code>convert_tokens_to_string</code> methods of the HuggingFace's tokenizer.</p> <p>Parameters:</p> <ul> <li> <code>input_ids</code>               (<code>NDArray[int64]</code>)           \u2013            <p>The input_ids to be decoded</p> </li> <li> <code>skip_special_tokens</code>               (<code>bool</code>)           \u2013            <p>Whether to skip the special tokens or not</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>List[str]</code>           \u2013            <p>The decoded texts</p> </li> </ul> Source code in <code>pytorch_widedeep/preprocessing/hf_preprocessor.py</code> <pre><code>def decode(\n    self, input_ids: npt.NDArray[np.int64], skip_special_tokens: bool\n) -&gt; List[str]:\n    \"\"\"\n    Decodes a list of input_ids. The method is a wrapper around the\n    `convert_ids_to_tokens` and `convert_tokens_to_string` methods of the\n    HuggingFace's tokenizer.\n\n    Parameters\n    ----------\n    input_ids: npt.NDArray[np.int64]\n        The input_ids to be decoded\n    skip_special_tokens: bool\n        Whether to skip the special tokens or not\n\n    Returns\n    -------\n    List[str]\n        The decoded texts\n    \"\"\"\n    texts = [\n        self.tokenizer.convert_tokens_to_string(\n            self.tokenizer.convert_ids_to_tokens(input_ids[i], skip_special_tokens)\n        )\n        for i in range(input_ids.shape[0])\n    ]\n    return texts\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.fit","title":"fit","text":"<pre><code>fit(df)\n</code></pre> <p>This method is included for consistency with the rest of the library in general and with the <code>BasePreprocessor</code> in particular. HuggingFace's tokenizers and models are already trained. Therefore, the 'fit' method here does nothing other than checking that the 'text_col' parameter is not <code>None</code>.</p> <p>Parameters:</p> <ul> <li> <code>df</code>               (<code>DataFrame</code>)           \u2013            <p>The dataframe containing the text data in the column specified by the 'text_col' parameter</p> </li> </ul> Source code in <code>pytorch_widedeep/preprocessing/hf_preprocessor.py</code> <pre><code>def fit(self, df: pd.DataFrame) -&gt; \"HFPreprocessor\":\n    \"\"\"\n    This method is included for consistency with the rest of the library\n    in general and with the `BasePreprocessor` in particular. HuggingFace's\n    tokenizers and models are already trained. Therefore, the 'fit' method\n    here does nothing other than checking that the 'text_col' parameter is\n    not `None`.\n\n    Parameters\n    ----------\n    df: pd.DataFrame\n        The dataframe containing the text data in the column specified by\n        the 'text_col' parameter\n    \"\"\"\n    if self.text_col is None:\n        raise ValueError(\n            \"'text_col' is None. Please specify the column name containing the text data\"\n            \" if you want to use the 'fit' method\"\n        )\n    return self\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.transform","title":"transform","text":"<pre><code>transform(df)\n</code></pre> <p>Encodes the text data in the input dataframe. This method simply calls the <code>encode</code> method under the hood. Similar to the <code>fit</code> method, this method is included for consistency with the rest of the library in general and with the <code>BasePreprocessor</code> in particular.</p> <p>Parameters:</p> <ul> <li> <code>df</code>               (<code>DataFrame</code>)           \u2013            <p>The dataframe containing the text data in the column specified by the 'text_col' parameter</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>array</code>           \u2013            <p>The encoded texts</p> </li> </ul> Source code in <code>pytorch_widedeep/preprocessing/hf_preprocessor.py</code> <pre><code>def transform(self, df: pd.DataFrame) -&gt; npt.NDArray[np.int64]:\n    \"\"\"\n    Encodes the text data in the input dataframe. This method simply\n    calls the `encode` method under the hood. Similar to the `fit` method,\n    this method is included for consistency with the rest of the library\n    in general and with the `BasePreprocessor` in particular.\n\n    Parameters\n    ----------\n    df: pd.DataFrame\n        The dataframe containing the text data in the column specified by\n        the 'text_col' parameter\n\n    Returns\n    -------\n    np.array\n        The encoded texts\n    \"\"\"\n    if self.text_col is None:\n        raise ValueError(\n            \"'text_col' is None. Please specify the column name containing the text data\"\n            \" if you want to use the 'fit' method\"\n        )\n\n    texts = self._read_texts(df, self.root_dir)\n\n    return self.encode(texts)\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.transform_sample","title":"transform_sample","text":"<pre><code>transform_sample(text)\n</code></pre> <p>Encodes a single text sample.</p> <p>Parameters:</p> <ul> <li> <code>text</code>               (<code>str</code>)           \u2013            <p>The text sample to be encoded</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>array</code>           \u2013            <p>The encoded text</p> </li> </ul> Source code in <code>pytorch_widedeep/preprocessing/hf_preprocessor.py</code> <pre><code>def transform_sample(self, text: str) -&gt; npt.NDArray[np.int64]:\n    \"\"\"\n    Encodes a single text sample.\n\n    Parameters\n    ----------\n    text: str\n        The text sample to be encoded\n\n    Returns\n    -------\n    np.array\n        The encoded text\n    \"\"\"\n\n    if not self.is_fitted:\n        raise ValueError(\n            \"The `encode` (or `fit`) method must be called before calling `transform_sample`\"\n        )\n    return self.encode([text])[0]\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.fit_transform","title":"fit_transform","text":"<pre><code>fit_transform(df)\n</code></pre> <p>Encodes the text data in the input dataframe.</p> <p>Parameters:</p> <ul> <li> <code>df</code>               (<code>DataFrame</code>)           \u2013            <p>The dataframe containing the text data in the column specified by the 'text_col' parameter</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>array</code>           \u2013            <p>The encoded texts</p> </li> </ul> Source code in <code>pytorch_widedeep/preprocessing/hf_preprocessor.py</code> <pre><code>def fit_transform(self, df: pd.DataFrame) -&gt; npt.NDArray[np.int64]:\n    \"\"\"\n    Encodes the text data in the input dataframe.\n\n    Parameters\n    ----------\n    df: pd.DataFrame\n        The dataframe containing the text data in the column specified by\n        the 'text_col' parameter\n\n    Returns\n    -------\n    np.array\n        The encoded texts\n    \"\"\"\n    return self.fit(df).transform(df)\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.inverse_transform","title":"inverse_transform","text":"<pre><code>inverse_transform(input_ids, skip_special_tokens)\n</code></pre> <p>Decodes a list of input_ids. The method simply calls the <code>decode</code> method under the hood.</p> <p>Parameters:</p> <ul> <li> <code>input_ids</code>               (<code>NDArray[int64]</code>)           \u2013            <p>The input_ids to be decoded</p> </li> <li> <code>skip_special_tokens</code>               (<code>bool</code>)           \u2013            <p>Whether to skip the special tokens or not</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>List[str]</code>           \u2013            <p>The decoded texts</p> </li> </ul> Source code in <code>pytorch_widedeep/preprocessing/hf_preprocessor.py</code> <pre><code>def inverse_transform(\n    self, input_ids: npt.NDArray[np.int64], skip_special_tokens: bool\n) -&gt; List[str]:\n    \"\"\"\n    Decodes a list of input_ids. The method simply calls the `decode` method\n    under the hood.\n\n    Parameters\n    ----------\n    input_ids: npt.NDArray[np.int64]\n        The input_ids to be decoded\n    skip_special_tokens: bool\n        Whether to skip the special tokens or not\n\n    Returns\n    -------\n    List[str]\n        The decoded texts\n    \"\"\"\n    return self.decode(input_ids, skip_special_tokens)\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.image_preprocessor.ImagePreprocessor","title":"ImagePreprocessor","text":"<pre><code>ImagePreprocessor(\n    img_col, img_path, width=224, height=224, verbose=1\n)\n</code></pre> <p>               Bases: <code>BasePreprocessor</code></p> <p>Preprocessor to prepare the <code>deepimage</code> input dataset.</p> <p>The Preprocessing consists simply on resizing according to their aspect ratio</p> <p>Parameters:</p> <ul> <li> <code>img_col</code>               (<code>str</code>)           \u2013            <p>name of the column with the images filenames</p> </li> <li> <code>img_path</code>               (<code>str</code>)           \u2013            <p>path to the dicrectory where the images are stored</p> </li> <li> <code>width</code>               (<code>int</code>, default:                   <code>224</code> )           \u2013            <p>width of the resulting processed image.</p> </li> <li> <code>height</code>               (<code>int</code>, default:                   <code>224</code> )           \u2013            <p>width of the resulting processed image.</p> </li> <li> <code>verbose</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>Enable verbose output.</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>aap</code>               (<code>AspectAwarePreprocessor</code>)           \u2013            <p>an instance of <code>pytorch_widedeep.utils.image_utils.AspectAwarePreprocessor</code></p> </li> <li> <code>spp</code>               (<code>SimplePreprocessor</code>)           \u2013            <p>an instance of <code>pytorch_widedeep.utils.image_utils.SimplePreprocessor</code></p> </li> <li> <code>normalise_metrics</code>               (<code>Dict</code>)           \u2013            <p>Dict containing the normalisation metrics of the image dataset, i.e. mean and std for the R, G and B channels</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import pandas as pd\n&gt;&gt;&gt;\n&gt;&gt;&gt; from pytorch_widedeep.preprocessing import ImagePreprocessor\n&gt;&gt;&gt;\n&gt;&gt;&gt; path_to_image1 = 'tests/test_data_utils/images/galaxy1.png'\n&gt;&gt;&gt; path_to_image2 = 'tests/test_data_utils/images/galaxy2.png'\n&gt;&gt;&gt;\n&gt;&gt;&gt; df_train = pd.DataFrame({'images_column': [path_to_image1]})\n&gt;&gt;&gt; df_test = pd.DataFrame({'images_column': [path_to_image2]})\n&gt;&gt;&gt; img_preprocessor = ImagePreprocessor(img_col='images_column', img_path='.', verbose=0)\n&gt;&gt;&gt; resized_images = img_preprocessor.fit_transform(df_train)\n&gt;&gt;&gt; new_resized_images = img_preprocessor.transform(df_train)\n</code></pre> <p> NOTE: Normalising metrics will only be computed when the <code>fit_transform</code> method is run. Running <code>transform</code> only will not change the computed metrics and running <code>fit</code> only simply instantiates the resizing functions.</p> Source code in <code>pytorch_widedeep/preprocessing/image_preprocessor.py</code> <pre><code>def __init__(\n    self,\n    img_col: str,\n    img_path: str,\n    width: int = 224,\n    height: int = 224,\n    verbose: int = 1,\n):\n    super(ImagePreprocessor, self).__init__()\n\n    self.img_col = img_col\n    self.img_path = img_path\n    self.width = width\n    self.height = height\n    self.verbose = verbose\n\n    self.aap = AspectAwarePreprocessor(self.width, self.height)\n    self.spp = SimplePreprocessor(self.width, self.height)\n\n    self.compute_normalising_computed = False\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.image_preprocessor.ImagePreprocessor.transform","title":"transform","text":"<pre><code>transform(df)\n</code></pre> <p>Resizes the images to the input height and width.</p> <p>Parameters:</p> <ul> <li> <code>df</code>               (<code>DataFrame</code>)           \u2013            <p>Input pandas dataframe with the <code>img_col</code></p> </li> </ul> <p>Returns:</p> <ul> <li> <code>ndarray</code>           \u2013            <p>Resized images to the input height and width</p> </li> </ul> Source code in <code>pytorch_widedeep/preprocessing/image_preprocessor.py</code> <pre><code>def transform(self, df: pd.DataFrame) -&gt; np.ndarray:\n    \"\"\"Resizes the images to the input height and width.\n\n\n    Parameters\n    ----------\n    df: pd.DataFrame\n        Input pandas dataframe with the `img_col`\n\n    Returns\n    -------\n    np.ndarray\n        Resized images to the input height and width\n    \"\"\"\n    image_list = df[self.img_col].tolist()\n    if self.verbose:\n        print(\"Reading Images from {}\".format(self.img_path))\n    imgs = [cv2.imread(\"/\".join([self.img_path, img])) for img in image_list]\n\n    # finding images with different height and width\n    aspect = [(im.shape[0], im.shape[1]) for im in imgs]\n    aspect_r = [a[0] / a[1] for a in aspect]\n    diff_idx = [i for i, r in enumerate(aspect_r) if r != 1.0]\n\n    if self.verbose:\n        print(\"Resizing\")\n    resized_imgs = []\n    for i, img in tqdm(enumerate(imgs), total=len(imgs), disable=self.verbose != 1):\n        if i in diff_idx:\n            resized_imgs.append(self.aap.preprocess(img))\n        else:\n            # if aspect ratio is 1:1, no need for AspectAwarePreprocessor\n            resized_imgs.append(self.spp.preprocess(img))\n\n    if not self.compute_normalising_computed:\n        if self.verbose:\n            print(\"Computing normalisation metrics\")\n        # mean and std deviation will only be computed when the fit method\n        # is called\n        mean_R, mean_G, mean_B = [], [], []\n        std_R, std_G, std_B = [], [], []\n        for rsz_img in resized_imgs:\n            (mean_b, mean_g, mean_r), (std_b, std_g, std_r) = cv2.meanStdDev(\n                rsz_img\n            )\n            mean_R.append(mean_r)\n            mean_G.append(mean_g)\n            mean_B.append(mean_b)\n            std_R.append(std_r)\n            std_G.append(std_g)\n            std_B.append(std_b)\n        self.normalise_metrics = dict(\n            mean={\n                \"R\": np.mean(mean_R) / 255.0,\n                \"G\": np.mean(mean_G) / 255.0,\n                \"B\": np.mean(mean_B) / 255.0,\n            },\n            std={\n                \"R\": np.mean(std_R) / 255.0,\n                \"G\": np.mean(std_G) / 255.0,\n                \"B\": np.mean(std_B) / 255.0,\n            },\n        )\n        self.compute_normalising_computed = True\n    return np.asarray(resized_imgs)\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.image_preprocessor.ImagePreprocessor.fit_transform","title":"fit_transform","text":"<pre><code>fit_transform(df)\n</code></pre> <p>Combines <code>fit</code> and <code>transform</code></p> <p>Parameters:</p> <ul> <li> <code>df</code>               (<code>DataFrame</code>)           \u2013            <p>Input pandas dataframe</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>ndarray</code>           \u2013            <p>Resized images to the input height and width</p> </li> </ul> Source code in <code>pytorch_widedeep/preprocessing/image_preprocessor.py</code> <pre><code>def fit_transform(self, df: pd.DataFrame) -&gt; np.ndarray:\n    \"\"\"Combines `fit` and `transform`\n\n    Parameters\n    ----------\n    df: pd.DataFrame\n        Input pandas dataframe\n\n    Returns\n    -------\n    np.ndarray\n        Resized images to the input height and width\n    \"\"\"\n    return self.fit(df).transform(df)\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.din_preprocessor.DINPreprocessor","title":"DINPreprocessor","text":"<pre><code>DINPreprocessor(\n    *,\n    user_id_col,\n    item_embed_col,\n    target_col,\n    max_seq_length,\n    action_col=None,\n    other_seq_embed_cols=None,\n    cat_embed_cols=None,\n    continuous_cols=None,\n    quantization_setup=None,\n    cols_to_scale=None,\n    auto_embed_dim=True,\n    embedding_rule=\"fastai_new\",\n    default_embed_dim=16,\n    verbose=1,\n    scale=False,\n    already_standard=None,\n    **kwargs\n)\n</code></pre> <p>               Bases: <code>BasePreprocessor</code></p> <p>Preprocessor for Deep Interest Network (DIN) models.</p> <p>This preprocessor handles the preparation of data for DIN models, including sequence building, label encoding, and handling of various types of input columns (categorical, continuous, and sequential).</p> <p>user_id_col : str     Name of the column containing user IDs. item_embed_col : Union[str, Tuple[str, int]]     Name of the column containing item IDs to be embedded, or a tuple of     (column_name, embedding_dim). target_col : str     Name of the column containing the target variable. max_seq_length : int     Maximum length of sequences to be created. action_col : Optional[str], default=None     Name of the column containing user actions (if applicable). other_seq_embed_cols : Optional[List[str] | List[Tuple[str, int]]], default=None     List of other columns to be treated as sequences. cat_embed_cols : Optional[Union[List[str], List[Tuple[str, int]]]], default=None     List of categorical columns to be represented by embeddings. continuous_cols: List, default = None     List with the name of the continuous cols. quantization_setup: int or Dict[str, Union[int, List[float]]], default=None     Continuous columns can be turned into categorical via <code>pd.cut</code>. cols_to_scale: List or str, default = None,     List with the names of the columns that will be standardized via     sklearn's <code>StandardScaler</code>. auto_embed_dim: bool, default = True     Boolean indicating whether the embedding dimensions will be     automatically defined via rule of thumb. embedding_rule: str, default = 'fastai_new'     Rule of thumb for embedding size. default_embed_dim: int, default=16     Default dimension for the embeddings. verbose : int, default=1     Verbosity level. scale: bool, default = False     Boolean indicating whether or not to scale/standardize continuous cols. already_standard: List, default = None     List with the name of the continuous cols that do not need to be     scaled/standardized. **kwargs :     Additional keyword arguments to be passed to the TabPreprocessor.</p> <p>is_fitted : bool     Whether the preprocessor has been fitted. has_standard_tab_data : bool     Whether the data includes standard tabular data. tab_preprocessor : TabPreprocessor     Preprocessor for standard tabular data. din_columns_idx : Dict[str, int]     Dictionary mapping column names to their indices in the processed data. item_le : LabelEncoder     Label encoder for item IDs. n_items : int     Number of unique items. user_behaviour_config : Tuple[List[str], int, int]     Configuration for user behavior sequences. action_le : LabelEncoder     Label encoder for action column (if applicable). n_actions : int     Number of unique actions (if applicable). action_seq_config : Tuple[List[str], int]     Configuration for action sequences (if applicable). other_seq_le : LabelEncoder     Label encoder for other sequence columns (if applicable). n_other_seq_cols : Dict[str, int]     Number of unique values in each other sequence column. other_seq_config : List[Tuple[List[str], int, int]]     Configuration for other sequence columns.</p> <p>import pandas as pd from pytorch_widedeep.preprocessing import DINPreprocessor data = { ...    'user_id': [1, 1, 1, 2, 2, 2, 3, 3, 3], ...    'item_id': [101, 102, 103, 101, 103, 104, 102, 103, 104], ...    'timestamp': [1, 2, 3, 1, 2, 3, 1, 2, 3], ...    'category': ['A', 'B', 'A', 'B', 'A', 'C', 'B', 'A', 'C'], ...    'price': [10.5, 15.0, 12.0, 10.5, 12.0, 20.0, 15.0, 12.0, 20.0], ...    'rating': [0, 1, 0, 1, 0, 1, 0, 1, 0] ... } df = pd.DataFrame(data) din_preprocessor = DINPreprocessor( ...     user_id_col='user_id', ...     item_embed_col='item_id', ...     target_col='rating', ...     max_seq_length=2, ...     action_col='rating', ...     other_seq_embed_cols=['category'], ...     cat_embed_cols=['user_id'], ...     continuous_cols=['price'], ...     cols_to_scale=['price'] ... ) X, y = din_preprocessor.fit_transform(df)</p> Source code in <code>pytorch_widedeep/preprocessing/din_preprocessor.py</code> <pre><code>@alias(\"item_embed_col\", [\"item_id_col\"])\ndef __init__(\n    self,\n    *,\n    user_id_col: str,\n    item_embed_col: Union[str, Tuple[str, int]],\n    target_col: str,\n    max_seq_length: int,\n    action_col: Optional[str] = None,\n    other_seq_embed_cols: Optional[Union[List[str], List[Tuple[str, int]]]] = None,\n    cat_embed_cols: Optional[Union[List[str], List[Tuple[str, int]]]] = None,\n    continuous_cols: Optional[List[str]] = None,\n    quantization_setup: Optional[\n        Union[int, Dict[str, Union[int, List[float]]]]\n    ] = None,\n    cols_to_scale: Optional[Union[List[str], str]] = None,\n    auto_embed_dim: bool = True,\n    embedding_rule: Literal[\"google\", \"fastai_old\", \"fastai_new\"] = \"fastai_new\",\n    default_embed_dim: int = 16,\n    verbose: int = 1,\n    scale: bool = False,\n    already_standard: Optional[List[str]] = None,\n    **kwargs,\n):\n    self.user_id_col = user_id_col\n    self.item_embed_col = item_embed_col\n    self.max_seq_length = max_seq_length\n    self.target_col = target_col if target_col is not None else \"target\"\n    self.action_col = action_col\n    self.other_seq_embed_cols = other_seq_embed_cols\n    self.cat_embed_cols = cat_embed_cols\n    self.continuous_cols = continuous_cols\n    self.quantization_setup = quantization_setup\n    self.cols_to_scale = cols_to_scale\n    self.auto_embed_dim = auto_embed_dim\n    self.embedding_rule = embedding_rule\n    self.default_embed_dim = default_embed_dim\n    self.verbose = verbose\n    self.scale = scale\n    self.already_standard = already_standard\n    self.kwargs = kwargs\n\n    self.has_standard_tab_data = bool(self.cat_embed_cols or self.continuous_cols)\n    if self.has_standard_tab_data:\n        self.tab_preprocessor = TabPreprocessor(\n            cat_embed_cols=self.cat_embed_cols,\n            continuous_cols=self.continuous_cols,\n            quantization_setup=self.quantization_setup,\n            cols_to_scale=self.cols_to_scale,\n            auto_embed_dim=self.auto_embed_dim,\n            embedding_rule=self.embedding_rule,\n            default_embed_dim=self.default_embed_dim,\n            verbose=self.verbose,\n            scale=self.scale,\n            already_standard=self.already_standard,\n            **self.kwargs,\n        )\n\n    self.is_fitted = False\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#chunked-versions","title":"Chunked versions","text":"<p>Chunked versions of the preprocessors are also available. These are useful when the data is too big to fit in memory. See also the <code>load_from_folder</code> module in the library and the corresponding section here in the documentation.</p> <p>Note that there is not a <code>ChunkImagePreprocessor</code>. This is because the processing of the images will occur inside the <code>ImageFromFolder</code> class in the <code>load_from_folder</code> module.</p>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.wide_preprocessor.ChunkWidePreprocessor","title":"ChunkWidePreprocessor","text":"<pre><code>ChunkWidePreprocessor(\n    wide_cols, n_chunks, crossed_cols=None\n)\n</code></pre> <p>               Bases: <code>WidePreprocessor</code></p> <p>Preprocessor to prepare the wide input dataset</p> <p>This Preprocessor prepares the data for the wide, linear component. This linear model is implemented via an Embedding layer that is connected to the output neuron. <code>ChunkWidePreprocessor</code> numerically encodes all the unique values of all categorical columns <code>wide_cols + crossed_cols</code>. See the Example below.</p> <p>Parameters:</p> <ul> <li> <code>wide_cols</code>               (<code>List[str]</code>)           \u2013            <p>List of strings with the name of the columns that will label encoded and passed through the <code>wide</code> component</p> </li> <li> <code>crossed_cols</code>               (<code>Optional[List[Tuple[str, str]]]</code>, default:                   <code>None</code> )           \u2013            <p>List of Tuples with the name of the columns that will be <code>'crossed'</code> and then label encoded. e.g. [('education', 'occupation'), ...]. For binary features, a cross-product transformation is 1 if and only if the constituent features are all 1, and 0 otherwise.</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>wide_crossed_cols</code>               (<code>List</code>)           \u2013            <p>List with the names of all columns that will be label encoded</p> </li> <li> <code>encoding_dict</code>               (<code>Dict</code>)           \u2013            <p>Dictionary where the keys are the result of pasting <code>colname + '_' + column value</code> and the values are the corresponding mapped integer.</p> </li> <li> <code>inverse_encoding_dict</code>               (<code>Dict</code>)           \u2013            <p>the inverse encoding dictionary</p> </li> <li> <code>wide_dim</code>               (<code>int</code>)           \u2013            <p>Dimension of the wide model (i.e. dim of the linear layer)</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import pandas as pd\n&gt;&gt;&gt; from pytorch_widedeep.preprocessing import ChunkWidePreprocessor\n&gt;&gt;&gt; chunk = pd.DataFrame({'color': ['r', 'b', 'g'], 'size': ['s', 'n', 'l']})\n&gt;&gt;&gt; wide_cols = ['color']\n&gt;&gt;&gt; crossed_cols = [('color', 'size')]\n&gt;&gt;&gt; chunk_wide_preprocessor = ChunkWidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols,\n... n_chunks=1)\n&gt;&gt;&gt; X_wide = chunk_wide_preprocessor.fit_transform(chunk)\n</code></pre> Source code in <code>pytorch_widedeep/preprocessing/wide_preprocessor.py</code> <pre><code>def __init__(\n    self,\n    wide_cols: List[str],\n    n_chunks: int,\n    crossed_cols: Optional[List[Tuple[str, str]]] = None,\n):\n    super(ChunkWidePreprocessor, self).__init__(wide_cols, crossed_cols)\n\n    self.n_chunks = n_chunks\n\n    self.chunk_counter = 0\n\n    self.is_fitted = False\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.wide_preprocessor.ChunkWidePreprocessor.partial_fit","title":"partial_fit","text":"<pre><code>partial_fit(chunk)\n</code></pre> <p>Fits the Preprocessor and creates required attributes</p> <p>Parameters:</p> <ul> <li> <code>chunk</code>               (<code>DataFrame</code>)           \u2013            <p>Input pandas dataframe</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>ChunkWidePreprocessor</code>           \u2013            <p><code>ChunkWidePreprocessor</code> fitted object</p> </li> </ul> Source code in <code>pytorch_widedeep/preprocessing/wide_preprocessor.py</code> <pre><code>def partial_fit(self, chunk: pd.DataFrame) -&gt; \"ChunkWidePreprocessor\":\n    r\"\"\"Fits the Preprocessor and creates required attributes\n\n    Parameters\n    ----------\n    chunk: pd.DataFrame\n        Input pandas dataframe\n\n    Returns\n    -------\n    ChunkWidePreprocessor\n        `ChunkWidePreprocessor` fitted object\n    \"\"\"\n    df_wide = self._prepare_wide(chunk)\n    self.wide_crossed_cols = df_wide.columns.tolist()\n\n    if self.chunk_counter == 0:\n        self.glob_feature_set = set(\n            self._make_global_feature_list(df_wide[self.wide_crossed_cols])\n        )\n    else:\n        self.glob_feature_set.update(\n            self._make_global_feature_list(df_wide[self.wide_crossed_cols])\n        )\n\n    self.chunk_counter += 1\n\n    if self.chunk_counter == self.n_chunks:\n        self.encoding_dict = {v: i + 1 for i, v in enumerate(self.glob_feature_set)}\n        self.wide_dim = len(self.encoding_dict)\n        self.inverse_encoding_dict = {k: v for v, k in self.encoding_dict.items()}\n        self.inverse_encoding_dict[0] = \"unseen\"\n\n        self.is_fitted = True\n\n    return self\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.wide_preprocessor.ChunkWidePreprocessor.fit","title":"fit","text":"<pre><code>fit(df)\n</code></pre> <p>Runs <code>partial_fit</code>. This is just to override the fit method in the base class. This class is not designed or thought to run fit</p> Source code in <code>pytorch_widedeep/preprocessing/wide_preprocessor.py</code> <pre><code>def fit(self, df: pd.DataFrame) -&gt; \"ChunkWidePreprocessor\":\n    \"\"\"\n    Runs `partial_fit`. This is just to override the fit method in the base\n    class. This class is not designed or thought to run fit\n    \"\"\"\n    return self.partial_fit(df)\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.tab_preprocessor.ChunkTabPreprocessor","title":"ChunkTabPreprocessor","text":"<pre><code>ChunkTabPreprocessor(\n    n_chunks,\n    cat_embed_cols=None,\n    continuous_cols=None,\n    cols_and_bins=None,\n    cols_to_scale=None,\n    default_embed_dim=16,\n    with_attention=False,\n    with_cls_token=False,\n    shared_embed=False,\n    verbose=1,\n    *,\n    scale=False,\n    already_standard=None,\n    **kwargs\n)\n</code></pre> <p>               Bases: <code>TabPreprocessor</code></p> <p>Preprocessor to prepare the <code>deeptabular</code> component input dataset</p> <p>Parameters:</p> <ul> <li> <code>n_chunks</code>               (<code>int</code>)           \u2013            <p>Number of chunks that the tabular dataset is divided by.</p> </li> <li> <code>cat_embed_cols</code>               (<code>Optional[Union[List[str], List[Tuple[str, int]]]]</code>, default:                   <code>None</code> )           \u2013            <p>List containing the name of the categorical columns that will be represented by embeddings (e.g. ['education', 'relationship', ...]) or a Tuple with the name and the embedding dimension (e.g.: [ ('education',32), ('relationship',16), ...])</p> </li> <li> <code>continuous_cols</code>               (<code>Optional[List[str]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the name of the continuous cols</p> </li> <li> <code>cols_and_bins</code>               (<code>Optional[Dict[str, List[float]]]</code>, default:                   <code>None</code> )           \u2013            <p>Continuous columns can be turned into categorical via <code>pd.cut</code>. 'cols_and_bins' is dictionary where the keys are the column names to quantize and the values are a list of scalars indicating the bin edges.</p> </li> <li> <code>cols_to_scale</code>               (<code>Optional[Union[List[str], str]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the names of the columns that will be standarised via sklearn's <code>StandardScaler</code></p> </li> <li> <code>default_embed_dim</code>               (<code>int</code>, default:                   <code>16</code> )           \u2013            <p>Dimension for the embeddings if the embed_dim is not provided in the <code>cat_embed_cols</code> parameter and <code>auto_embed_dim</code> is set to <code>False</code>.</p> </li> <li> <code>with_attention</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether the preprocessed data will be passed to an attention-based model (more precisely a model where all embeddings must have the same dimensions). If <code>True</code>, the param <code>cat_embed_cols</code> must just be a list containing just the categorical column names: e.g. ['education', 'relationship', ...]. This is because they will all be  encoded using embeddings of the same dim, which will be specified  later when the model is defined.  Param alias:  <code>for_transformer</code></p> </li> <li> <code>with_cls_token</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating if a <code>'[CLS]'</code> token will be added to the dataset when using attention-based models. The final hidden state corresponding to this token is used as the aggregated representation for classification and regression tasks. If not, the categorical (and continuous embeddings if present) will be concatenated before being passed to the final MLP (if present).</p> </li> <li> <code>shared_embed</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating if the embeddings will be \"shared\" when using attention-based models. The idea behind <code>shared_embed</code> is described in the Appendix A in the TabTransformer paper: 'The goal of having column embedding is to enable the model to distinguish the classes in one column from those in the other columns'. In other words, the idea is to let the model learn which column is embedded at the time. See: <code>pytorch_widedeep.models.transformers._layers.SharedEmbeddings</code>.</p> </li> <li> <code>verbose</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            </li> <li> <code>scale</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p> note: this arg will be removed in upcoming  releases. Please use <code>cols_to_scale</code> instead.  Bool indicating  whether or not to scale/standarise continuous cols. It is important  to emphasize that all the DL models for tabular data in the library  also include the possibility of normalising the input continuous  features via a <code>BatchNorm</code> or a <code>LayerNorm</code>.  Param alias:  <code>scale_cont_cols</code>.</p> </li> <li> <code>already_standard</code>               (<code>Optional[List[str]]</code>, default:                   <code>None</code> )           \u2013            <p> note: this arg will be removed in upcoming  releases. Please use <code>cols_to_scale</code> instead.  List with the  name of the continuous cols that do not need to be  scaled/standarised.</p> </li> </ul> <p>Other Parameters:</p> <ul> <li> <code>**kwargs</code>           \u2013            <p><code>pd.cut</code> and <code>StandardScaler</code> related args</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>embed_dim</code>               (<code>Dict</code>)           \u2013            <p>Dictionary where keys are the embed cols and values are the embedding dimensions. If <code>with_attention</code> is set to <code>True</code> this attribute is not generated during the <code>fit</code> process</p> </li> <li> <code>label_encoder</code>               (<code>LabelEncoder</code>)           \u2013            <p>see <code>pytorch_widedeep.utils.dense_utils.LabelEncder</code></p> </li> <li> <code>cat_embed_input</code>               (<code>List</code>)           \u2013            <p>List of Tuples with the column name, number of individual values for that column and, If <code>with_attention</code> is set to <code>False</code>, the corresponding embeddings dim, e.g. [('education', 16, 10), ('relationship', 6, 8), ...].</p> </li> <li> <code>standardize_cols</code>               (<code>List</code>)           \u2013            <p>List of the columns that will be standarized</p> </li> <li> <code>scaler</code>               (<code>StandardScaler</code>)           \u2013            <p>an instance of <code>sklearn.preprocessing.StandardScaler</code> if 'cols_to_scale' is not None or 'scale' is 'True'</p> </li> <li> <code>column_idx</code>               (<code>Dict</code>)           \u2013            <p>Dictionary where keys are column names and values are column indexes. This is neccesary to slice tensors</p> </li> <li> <code>quantizer</code>               (<code>Quantizer</code>)           \u2013            <p>an instance of <code>Quantizer</code></p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import pandas as pd\n&gt;&gt;&gt; import numpy as np\n&gt;&gt;&gt; from pytorch_widedeep.preprocessing import ChunkTabPreprocessor\n&gt;&gt;&gt; np.random.seed(42)\n&gt;&gt;&gt; chunk_df = pd.DataFrame({'cat_col': np.random.choice(['A', 'B', 'C'], size=8),\n... 'cont_col': np.random.uniform(1, 100, size=8)})\n&gt;&gt;&gt; cat_embed_cols = [('cat_col',4)]\n&gt;&gt;&gt; cont_cols = ['cont_col']\n&gt;&gt;&gt; tab_preprocessor = ChunkTabPreprocessor(\n... n_chunks=1, cat_embed_cols=cat_embed_cols, continuous_cols=cont_cols\n... )\n&gt;&gt;&gt; X_tab = tab_preprocessor.fit_transform(chunk_df)\n&gt;&gt;&gt; tab_preprocessor.cat_embed_cols\n[('cat_col', 4)]\n&gt;&gt;&gt; tab_preprocessor.column_idx\n{'cat_col': 0, 'cont_col': 1}\n</code></pre> Source code in <code>pytorch_widedeep/preprocessing/tab_preprocessor.py</code> <pre><code>@alias(\"with_attention\", [\"for_transformer\"])\n@alias(\"cat_embed_cols\", [\"embed_cols\"])\n@alias(\"scale\", [\"scale_cont_cols\"])\n@alias(\"cols_and_bins\", [\"quantization_setup\"])\ndef __init__(\n    self,\n    n_chunks: int,\n    cat_embed_cols: Optional[Union[List[str], List[Tuple[str, int]]]] = None,\n    continuous_cols: Optional[List[str]] = None,\n    cols_and_bins: Optional[Dict[str, List[float]]] = None,\n    cols_to_scale: Optional[Union[List[str], str]] = None,\n    default_embed_dim: int = 16,\n    with_attention: bool = False,\n    with_cls_token: bool = False,\n    shared_embed: bool = False,\n    verbose: int = 1,\n    *,\n    scale: bool = False,\n    already_standard: Optional[List[str]] = None,\n    **kwargs,\n):\n    super(ChunkTabPreprocessor, self).__init__(\n        cat_embed_cols=cat_embed_cols,\n        continuous_cols=continuous_cols,\n        quantization_setup=None,\n        cols_to_scale=cols_to_scale,\n        auto_embed_dim=False,\n        embedding_rule=\"google\",  # does not matter, irrelevant\n        default_embed_dim=default_embed_dim,\n        with_attention=with_attention,\n        with_cls_token=with_cls_token,\n        shared_embed=shared_embed,\n        verbose=verbose,\n        scale=scale,\n        already_standard=already_standard,\n        **kwargs,\n    )\n\n    self.n_chunks = n_chunks\n    self.chunk_counter = 0\n\n    self.cols_and_bins = cols_and_bins  # type: ignore[assignment]\n    if self.cols_and_bins is not None:\n        self.quantizer = Quantizer(self.cols_and_bins, **self.quant_args)\n\n    self.embed_prepared = False\n    self.continuous_prepared = False\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.text_preprocessor.ChunkTextPreprocessor","title":"ChunkTextPreprocessor","text":"<pre><code>ChunkTextPreprocessor(\n    text_col,\n    n_chunks,\n    root_dir=None,\n    max_vocab=30000,\n    min_freq=5,\n    maxlen=80,\n    pad_first=True,\n    pad_idx=1,\n    already_processed=False,\n    word_vectors_path=None,\n    n_cpus=None,\n    verbose=1,\n)\n</code></pre> <p>               Bases: <code>TextPreprocessor</code></p> <p>Preprocessor to prepare the <code>deeptext</code> input dataset</p> <p>Parameters:</p> <ul> <li> <code>text_col</code>               (<code>str</code>)           \u2013            <p>column in the input dataframe containing either the texts or the filenames where the text documents are stored</p> </li> <li> <code>n_chunks</code>               (<code>int</code>)           \u2013            <p>Number of chunks that the text dataset is divided by.</p> </li> <li> <code>root_dir</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>If 'text_col' contains the filenames with the text documents, this is the path to the directory where those documents are stored.</p> </li> <li> <code>max_vocab</code>               (<code>int</code>, default:                   <code>30000</code> )           \u2013            <p>Maximum number of tokens in the vocabulary</p> </li> <li> <code>min_freq</code>               (<code>int</code>, default:                   <code>5</code> )           \u2013            <p>Minimum frequency for a token to be part of the vocabulary</p> </li> <li> <code>maxlen</code>               (<code>int</code>, default:                   <code>80</code> )           \u2013            <p>Maximum length of the tokenized sequences</p> </li> <li> <code>pad_first</code>               (<code>bool</code>, default:                   <code>True</code> )           \u2013            <p>Indicates whether the padding index will be added at the beginning or the end of the sequences</p> </li> <li> <code>pad_idx</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>padding index. Fastai's Tokenizer leaves 0 for the 'unknown' token.</p> </li> <li> <code>word_vectors_path</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Path to the pretrained word vectors</p> </li> <li> <code>n_cpus</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>number of CPUs to used during the tokenization process</p> </li> <li> <code>verbose</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>Enable verbose output.</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>vocab</code>               (<code>Vocab</code>)           \u2013            <p>an instance of <code>pytorch_widedeep.utils.fastai_transforms.ChunkVocab</code></p> </li> <li> <code>embedding_matrix</code>               (<code>ndarray</code>)           \u2013            <p>Array with the pretrained embeddings if <code>word_vectors_path</code> is not None</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import pandas as pd\n&gt;&gt;&gt; from pytorch_widedeep.preprocessing import ChunkTextPreprocessor\n&gt;&gt;&gt; chunk_df = pd.DataFrame({'text_column': [\"life is like a box of chocolates\",\n... \"You never know what you're gonna get\"]})\n&gt;&gt;&gt; chunk_text_preprocessor = ChunkTextPreprocessor(text_col='text_column', n_chunks=1,\n... max_vocab=25, min_freq=1, maxlen=10, verbose=0, n_cpus=1)\n&gt;&gt;&gt; processed_chunk = chunk_text_preprocessor.fit_transform(chunk_df)\n</code></pre> Source code in <code>pytorch_widedeep/preprocessing/text_preprocessor.py</code> <pre><code>def __init__(\n    self,\n    text_col: str,\n    n_chunks: int,\n    root_dir: Optional[str] = None,\n    max_vocab: int = 30000,\n    min_freq: int = 5,\n    maxlen: int = 80,\n    pad_first: bool = True,\n    pad_idx: int = 1,\n    already_processed: Optional[bool] = False,\n    word_vectors_path: Optional[str] = None,\n    n_cpus: Optional[int] = None,\n    verbose: int = 1,\n):\n    super(ChunkTextPreprocessor, self).__init__(\n        text_col=text_col,\n        max_vocab=max_vocab,\n        min_freq=min_freq,\n        maxlen=maxlen,\n        pad_first=pad_first,\n        pad_idx=pad_idx,\n        already_processed=already_processed,\n        word_vectors_path=word_vectors_path,\n        n_cpus=n_cpus,\n        verbose=verbose,\n    )\n\n    self.n_chunks = n_chunks\n    self.root_dir = root_dir\n\n    self.chunk_counter = 0\n\n    self.is_fitted = False\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.hf_preprocessor.ChunkHFPreprocessor","title":"ChunkHFPreprocessor","text":"<pre><code>ChunkHFPreprocessor(\n    model_name,\n    *,\n    text_col,\n    root_dir=None,\n    use_fast_tokenizer=True,\n    num_workers=None,\n    preprocessing_rules=None,\n    tokenizer_params=None,\n    encode_params=None\n)\n</code></pre> <p>               Bases: <code>HFPreprocessor</code></p> <p>Text processor to prepare the <code>deeptext</code> input dataset that is a wrapper around HuggingFace's tokenizers.</p> <p>Hugginface Tokenizer's are already 'trained'. Therefore, unlike the <code>ChunkTextPreprocessor</code> this is mostly identical to the <code>HFPreprocessor</code> with the only difference that the class needs a 'text_col' parameter to be passed. Also the parameter <code>encode_params</code> is not really optional when using this class. It must be passed containing at least the 'max_length' encoding parameter. This is because we need to ensure that  all sequences have the same length when encoding in chunks.</p> <p>Parameters:</p> <ul> <li> <code>model_name</code>               (<code>str</code>)           \u2013            <p>The model name from the transformers library e.g. 'bert-base-uncased'. Currently supported models are those from the families: BERT, RoBERTa, DistilBERT, ALBERT and ELECTRA.</p> </li> <li> <code>text_col</code>               (<code>str</code>)           \u2013            <p>The column in the input dataframe containing the text data. When using the <code>ChunkHFPreprocessor</code> the <code>text_col</code> parameter is mandatory.</p> </li> <li> <code>root_dir</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>The root directory where the text files are located. This is only needed if the text data is stored in text files. If the text data is stored in a column in the input dataframe, this parameter is not needed.</p> </li> <li> <code>use_fast_tokenizer</code>               (<code>bool</code>, default:                   <code>True</code> )           \u2013            <p>Whether to use the fast tokenizer from HuggingFace or not</p> </li> <li> <code>num_workers</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>Number of workers to use when preprocessing the text data. If not None, and <code>use_fast_tokenizer</code> is False, the text data will be preprocessed in parallel using the number of workers specified. If <code>use_fast_tokenizer</code> is True, this argument is ignored.</p> </li> <li> <code>preprocessing_rules</code>               (<code>Optional[List[Callable[[str], str]]]</code>, default:                   <code>None</code> )           \u2013            <p>A list of functions to be applied to the text data before encoding. This can be useful to clean the text data before encoding. For example, removing html tags, special characters, etc.</p> </li> <li> <code>tokenizer_params</code>               (<code>Optional[Dict[str, Any]]</code>, default:                   <code>None</code> )           \u2013            <p>Additional parameters to be passed to the HuggingFace's <code>PreTrainedTokenizer</code>.</p> </li> <li> <code>encode_params</code>               (<code>Optional[Dict[str, Any]]</code>, default:                   <code>None</code> )           \u2013            <p>Additional parameters to be passed to the <code>batch_encode_plus</code> method of the HuggingFace's <code>PreTrainedTokenizer</code>. In the case of the <code>ChunkHFPreprocessor</code>, this parameter is not really <code>Optional</code>. It must be passed containing at least the 'max_length' encoding parameter</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>is_fitted</code>               (<code>bool</code>)           \u2013            <p>Boolean indicating if the preprocessor has been fitted. This is a HuggingFacea tokenizer, so it is always considered fitted and this attribute is manually set to True internally. This parameter exists for consistency with the rest of the library and because is needed for some functionality in the library.</p> </li> </ul> Source code in <code>pytorch_widedeep/preprocessing/hf_preprocessor.py</code> <pre><code>def __init__(\n    self,\n    model_name: str,\n    *,\n    text_col: str,\n    root_dir: Optional[str] = None,\n    use_fast_tokenizer: bool = True,\n    num_workers: Optional[int] = None,\n    preprocessing_rules: Optional[List[Callable[[str], str]]] = None,\n    tokenizer_params: Optional[Dict[str, Any]] = None,\n    encode_params: Optional[Dict[str, Any]] = None,\n):\n    super().__init__(\n        model_name=model_name,\n        use_fast_tokenizer=use_fast_tokenizer,\n        text_col=text_col,\n        num_workers=num_workers,\n        preprocessing_rules=preprocessing_rules,\n        tokenizer_params=tokenizer_params,\n        encode_params=encode_params,\n    )\n\n    self.root_dir = root_dir\n\n    # when using in chunks encode_params is not really optional. I will\n    # review types in due time\n    if self.encode_params is None:\n        raise ValueError(\n            \"The 'encode_params' dict must be passed to the ChunkHFTokenizer \"\n            \"containing at least the 'max_length' encoding parameter\"\n        )\n\n    if \"padding\" not in self.encode_params or not self.encode_params[\"padding\"]:\n        self.encode_params[\"padding\"] = True\n\n    if (\n        \"truncation\" not in self.encode_params\n        or not self.encode_params[\"truncation\"]\n    ):\n        self.encode_params[\"truncation\"] = True\n</code></pre>"},{"location":"pytorch-widedeep/self_supervised_pretraining.html","title":"Self Supervised Pre-training for tabular data","text":"<p>In this library we have implemented two methods or routines that allow the user to use self-suerpvised pre-training for all tabular models in the library with the exception of the <code>TabPerceiver</code> (this is a particular model and self-supervised pre-training requires some adjustments that will be implemented in future versions). Please see the examples folder in the repo or the examples section in the docs for details on how to use self-supervised pre-training with this library.</p> <p>The two routines implemented are illustrated in the figures below. The first is from TabNet: Attentive Interpretable Tabular Learning. It is a 'standard' encoder-decoder architecture and and is designed here for models that do not use transformer-based architectures (or when the embeddings can all have different dimensions). The second is from SAINT: Improved Neural Networks for Tabular Data via Row Attention and Contrastive Pre-Training, it is based on Contrastive and Denoising learning and is designed for models that use transformer-based architectures (or when the embeddings all need to have the same dimension):</p> <p> </p> <p>Figure 1. Figure 2 in their paper. The caption of the original paper is included in case it is useful.</p> <p> </p> <p>Figure 2. Figure 1 in their paper. The caption of the original paper is included in case it is useful.</p> <p>Note that the self-supervised pre-trainers described below focus, of course, on the self-supervised pre-training phase, i.e. the left side in Figure 1 and the upper part in Figure 2. When combined with the <code>Trainer</code> described earlier in the documenation, one can reproduce the full process illustrated in the figures above.</p> <p>Also Note that it is beyond the scope of this docs to explain in detail these routines. In addition, to fully utilise the self-supervised trainers implemented in this library a minimum understanding of the processes as described in the papers is required. Therefore, we strongly encourage the users to have a look to the papers.</p>"},{"location":"pytorch-widedeep/self_supervised_pretraining.html#pytorch_widedeep.self_supervised_training.EncoderDecoderTrainer","title":"EncoderDecoderTrainer","text":"<pre><code>EncoderDecoderTrainer(\n    encoder,\n    decoder=None,\n    masked_prob=0.2,\n    optimizer=None,\n    lr_scheduler=None,\n    callbacks=None,\n    verbose=1,\n    seed=1,\n    **kwargs\n)\n</code></pre> <p>               Bases: <code>BaseEncoderDecoderTrainer</code></p> <p>This class implements an Encoder-Decoder self-supervised 'routine' inspired by TabNet: Attentive Interpretable Tabular Learning. See Figure 1 above.</p> <p>Parameters:</p> <ul> <li> <code>encoder</code>               (<code>ModelWithoutAttention</code>)           \u2013            <p>An instance of a <code>TabMlp</code>, <code>TabResNet</code> or <code>TabNet</code> model</p> </li> <li> <code>decoder</code>               (<code>Optional[DecoderWithoutAttention]</code>, default:                   <code>None</code> )           \u2013            <p>An instance of  a <code>TabMlpDecoder</code>, <code>TabResNetDecoder</code> or <code>TabNetDecoder</code> model. if <code>None</code> the decoder will be automatically built as a 'simetric' model to the Encoder</p> </li> <li> <code>masked_prob</code>               (<code>float</code>, default:                   <code>0.2</code> )           \u2013            <p>Indicates the fraction of elements in the embedding tensor that will be masked and hence used for reconstruction</p> </li> <li> <code>optimizer</code>               (<code>Optional[Optimizer]</code>, default:                   <code>None</code> )           \u2013            <p>An instance of Pytorch's <code>Optimizer</code> object (e.g. <code>torch.optim.Adam ()</code>). if no optimizer is passed it will default to <code>AdamW</code>.</p> </li> <li> <code>lr_scheduler</code>               (<code>Optional[LRScheduler]</code>, default:                   <code>None</code> )           \u2013            <p>An instance of Pytorch's <code>LRScheduler</code> object (e.g <code>torch.optim.lr_scheduler.StepLR(opt, step_size=5)</code>).</p> </li> <li> <code>callbacks</code>               (<code>Optional[List[Callback]]</code>, default:                   <code>None</code> )           \u2013            <p>List with <code>Callback</code> objects. The three callbacks available in <code>pytorch-widedeep</code> are: <code>LRHistory</code>, <code>ModelCheckpoint</code> and <code>EarlyStopping</code>. This can also be a custom callback. See <code>pytorch_widedeep.callbacks.Callback</code> or the Examples folder in the repo.</p> </li> <li> <code>verbose</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>Setting it to 0 will print nothing during training.</p> </li> <li> <code>seed</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>Random seed to be used internally for train_test_split</p> </li> </ul> <p>Other Parameters:</p> <ul> <li> <code>**kwargs</code>           \u2013            <p>Other infrequently used arguments that can also be passed as kwargs are:</p> <ul> <li> <p>device: <code>str</code>     string indicating the device. One of 'cpu' or 'gpu'</p> </li> <li> <p>num_workers: <code>int</code>     number of workers to be used internally by the data loaders</p> </li> <li> <p>reducelronplateau_criterion: <code>str</code>     This sets the criterion that will be used by the lr scheduler to     take a step: One of 'loss' or 'metric'. The ReduceLROnPlateau     learning rate is a bit particular.</p> </li> </ul> </li> </ul> Source code in <code>pytorch_widedeep/self_supervised_training/encoder_decoder_trainer.py</code> <pre><code>def __init__(\n    self,\n    encoder: ModelWithoutAttention,\n    decoder: Optional[DecoderWithoutAttention] = None,\n    masked_prob: float = 0.2,\n    optimizer: Optional[Optimizer] = None,\n    lr_scheduler: Optional[LRScheduler] = None,\n    callbacks: Optional[List[Callback]] = None,\n    verbose: int = 1,\n    seed: int = 1,\n    **kwargs,\n):\n    super().__init__(\n        encoder=encoder,\n        decoder=decoder,\n        masked_prob=masked_prob,\n        optimizer=optimizer,\n        lr_scheduler=lr_scheduler,\n        callbacks=callbacks,\n        verbose=verbose,\n        seed=seed,\n        **kwargs,\n    )\n</code></pre>"},{"location":"pytorch-widedeep/self_supervised_pretraining.html#pytorch_widedeep.self_supervised_training.EncoderDecoderTrainer.pretrain","title":"pretrain","text":"<pre><code>pretrain(\n    X_tab,\n    X_tab_val=None,\n    val_split=None,\n    validation_freq=1,\n    n_epochs=1,\n    batch_size=32,\n)\n</code></pre> <p>Pretrain method. Can also be called using <code>.fit(&lt;same_args&gt;)</code></p> <p>Parameters:</p> <ul> <li> <code>X_tab</code>               (<code>ndarray</code>)           \u2013            <p>tabular dataset</p> </li> <li> <code>X_tab_val</code>               (<code>Optional[ndarray]</code>, default:                   <code>None</code> )           \u2013            <p>validation data</p> </li> <li> <code>val_split</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>An alterative to passing the validation set is to use a train/val split fraction via <code>val_split</code></p> </li> <li> <code>validation_freq</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>epochs validation frequency</p> </li> <li> <code>n_epochs</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>number of epochs</p> </li> <li> <code>batch_size</code>               (<code>int</code>, default:                   <code>32</code> )           \u2013            <p>batch size</p> </li> </ul> Source code in <code>pytorch_widedeep/self_supervised_training/encoder_decoder_trainer.py</code> <pre><code>def pretrain(\n    self,\n    X_tab: np.ndarray,\n    X_tab_val: Optional[np.ndarray] = None,\n    val_split: Optional[float] = None,\n    validation_freq: int = 1,\n    n_epochs: int = 1,\n    batch_size: int = 32,\n):\n    r\"\"\"Pretrain method. Can also be called using `.fit(&lt;same_args&gt;)`\n\n    Parameters\n    ----------\n    X_tab: np.ndarray,\n        tabular dataset\n    X_tab_val: np.ndarray, Optional, default = None\n        validation data\n    val_split: float, Optional. default=None\n        An alterative to passing the validation set is to use a train/val\n        split fraction via `val_split`\n    validation_freq: int, default=1\n        epochs validation frequency\n    n_epochs: int, default=1\n        number of epochs\n    batch_size: int, default=32\n        batch size\n    \"\"\"\n\n    self.batch_size = batch_size\n\n    train_set, eval_set = self._train_eval_split(X_tab, X_tab_val, val_split)\n    train_loader = DataLoader(\n        dataset=train_set, batch_size=batch_size, num_workers=self.num_workers\n    )\n    train_steps = len(train_loader)\n    if eval_set is not None:\n        eval_loader = DataLoader(\n            dataset=eval_set,\n            batch_size=batch_size,\n            num_workers=self.num_workers,\n            shuffle=False,\n        )\n        eval_steps = len(eval_loader)\n\n    self.callback_container.on_train_begin(\n        {\n            \"batch_size\": batch_size,\n            \"train_steps\": train_steps,\n            \"n_epochs\": n_epochs,\n        }\n    )\n    for epoch in range(n_epochs):\n        epoch_logs: Dict[str, float] = {}\n        self.callback_container.on_epoch_begin(epoch, logs=epoch_logs)\n\n        self.train_running_loss = 0.0\n        with trange(train_steps, disable=self.verbose != 1) as t:\n            for batch_idx, X in zip(t, train_loader):\n                t.set_description(\"epoch %i\" % (epoch + 1))\n                train_loss = self._train_step(X[0], batch_idx)\n                self.callback_container.on_batch_end(batch=batch_idx)\n                print_loss_and_metric(t, train_loss)\n\n        epoch_logs = save_epoch_logs(epoch_logs, train_loss, None, \"train\")\n\n        on_epoch_end_metric = None\n        if eval_set is not None and epoch % validation_freq == (\n            validation_freq - 1\n        ):\n            self.callback_container.on_eval_begin()\n            self.valid_running_loss = 0.0\n            with trange(eval_steps, disable=self.verbose != 1) as v:\n                for batch_idx, X in zip(v, eval_loader):\n                    v.set_description(\"valid\")\n                    val_loss = self._eval_step(X[0], batch_idx)\n                    print_loss_and_metric(v, val_loss)\n            epoch_logs = save_epoch_logs(epoch_logs, val_loss, None, \"val\")\n            on_epoch_end_metric = val_loss\n        else:\n            if self.reducelronplateau:\n                raise NotImplementedError(\n                    \"ReduceLROnPlateau scheduler can be used only with validation data.\"\n                )\n\n        self.callback_container.on_epoch_end(epoch, epoch_logs, on_epoch_end_metric)\n\n        if self.early_stop:\n            self.callback_container.on_train_end(epoch_logs)\n            break\n\n    self.callback_container.on_train_end(epoch_logs)\n    self._restore_best_weights()\n    self.ed_model.train()\n</code></pre>"},{"location":"pytorch-widedeep/self_supervised_pretraining.html#pytorch_widedeep.self_supervised_training.ContrastiveDenoisingTrainer","title":"ContrastiveDenoisingTrainer","text":"<pre><code>ContrastiveDenoisingTrainer(\n    model,\n    preprocessor,\n    optimizer=None,\n    lr_scheduler=None,\n    callbacks=None,\n    loss_type=\"both\",\n    projection_head1_dims=None,\n    projection_head2_dims=None,\n    projection_heads_activation=\"relu\",\n    cat_mlp_type=\"multiple\",\n    cont_mlp_type=\"multiple\",\n    denoise_mlps_activation=\"relu\",\n    verbose=1,\n    seed=1,\n    **kwargs\n)\n</code></pre> <p>               Bases: <code>BaseContrastiveDenoisingTrainer</code></p> <p>This class trains a Contrastive, Denoising Self Supervised 'routine' that is based on the one described in SAINT: Improved Neural Networks for Tabular Data via Row Attention and Contrastive Pre-Training, their Figure 1.</p> <p>Parameters:</p> <ul> <li> <code>model</code>               (<code>ModelWithAttention</code>)           \u2013            <p>An instance of a <code>TabTransformer</code>, <code>SAINT</code>, <code>FTTransformer</code>, <code>TabFastFormer</code>, <code>TabPerceiver</code>, <code>ContextAttentionMLP</code> and <code>SelfAttentionMLP</code>.</p> </li> <li> <code>preprocessor</code>               (<code>TabPreprocessor</code>)           \u2013            <p>A fitted <code>TabPreprocessor</code> object. See <code>pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor</code></p> </li> <li> <code>optimizer</code>               (<code>Optional[Optimizer]</code>, default:                   <code>None</code> )           \u2013            <p>An instance of Pytorch's <code>Optimizer</code> object (e.g. <code>torch.optim.Adam ()</code>). if no optimizer is passed it will default to <code>AdamW</code>.</p> </li> <li> <code>lr_scheduler</code>               (<code>Optional[LRScheduler]</code>, default:                   <code>None</code> )           \u2013            <p>An instance of Pytorch's <code>LRScheduler</code> object (e.g <code>torch.optim.lr_scheduler.StepLR(opt, step_size=5)</code>).</p> </li> <li> <code>callbacks</code>               (<code>Optional[List[Callback]]</code>, default:                   <code>None</code> )           \u2013            <p>List with <code>Callback</code> objects. The three callbacks available in <code>pytorch-widedeep</code> are: <code>LRHistory</code>, <code>ModelCheckpoint</code> and <code>EarlyStopping</code>. This can also be a custom callback. See <code>pytorch_widedeep.callbacks.Callback</code> or the Examples folder in the repo.</p> </li> <li> <code>loss_type</code>               (<code>Literal[contrastive, denoising, both]</code>, default:                   <code>'both'</code> )           \u2013            <p>One of 'contrastive', 'denoising' or 'both'. See SAINT: Improved Neural Networks for Tabular Data via Row Attention and Contrastive Pre-Training, their figure (1) and their equation (5).</p> </li> <li> <code>projection_head1_dims</code>               (<code>Optional[List[int]]</code>, default:                   <code>None</code> )           \u2013            <p>The projection heads are simply MLPs. This parameter is a list of integers with the dimensions of the MLP hidden layers. See the paper for details. Note that setting up this parameter requires some knowledge of the architecture one is using. For example, if we are representing the features with embeddings of dim 32 (i.e. the so called dimension of the model is 32), then the first dimension of the projection head must be 32 (e.g. [32, 16])</p> </li> <li> <code>projection_head2_dims</code>               (<code>Optional[List[int]]</code>, default:                   <code>None</code> )           \u2013            <p>Same as 'projection_head1_dims' for the second head</p> </li> <li> <code>projection_heads_activation</code>               (<code>str</code>, default:                   <code>'relu'</code> )           \u2013            <p>Activation function for the projection heads</p> </li> <li> <code>cat_mlp_type</code>               (<code>Literal[single, multiple]</code>, default:                   <code>'multiple'</code> )           \u2013            <p>If 'denoising' loss is used, one can choose two types of 'stacked' MLPs to process the output from the transformer-based encoder that receives 'corrupted' (cut-mixed and mixed-up) features. These are 'single' or 'multiple'. The former approach will apply a single MLP to all the categorical features while the latter will use one MLP per categorical feature</p> </li> <li> <code>cont_mlp_type</code>               (<code>Literal[single, multiple]</code>, default:                   <code>'multiple'</code> )           \u2013            <p>Same as 'cat_mlp_type' but for the continuous features</p> </li> <li> <code>denoise_mlps_activation</code>               (<code>str</code>, default:                   <code>'relu'</code> )           \u2013            <p>activation function for the so called 'denoising mlps'.</p> </li> <li> <code>verbose</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>Setting it to 0 will print nothing during training.</p> </li> <li> <code>seed</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>Random seed to be used internally for train_test_split</p> </li> </ul> <p>Other Parameters:</p> <ul> <li> <code>**kwargs</code>           \u2013            <p>Other infrequently used arguments that can also be passed as kwargs are:</p> <ul> <li> <p>device: <code>str</code>     string indicating the device. One of 'cpu' or 'gpu'</p> </li> <li> <p>num_workers: <code>int</code>     number of workers to be used internally by the data loaders</p> </li> <li> <p>reducelronplateau_criterion: <code>str</code>     This sets the criterion that will be used by the lr scheduler to     take a step: One of 'loss' or 'metric'. The ReduceLROnPlateau     learning rate is a bit particular.</p> </li> </ul> </li> </ul> Source code in <code>pytorch_widedeep/self_supervised_training/contrastive_denoising_trainer.py</code> <pre><code>def __init__(\n    self,\n    model: ModelWithAttention,\n    preprocessor: TabPreprocessor,\n    optimizer: Optional[Optimizer] = None,\n    lr_scheduler: Optional[LRScheduler] = None,\n    callbacks: Optional[List[Callback]] = None,\n    loss_type: Literal[\"contrastive\", \"denoising\", \"both\"] = \"both\",\n    projection_head1_dims: Optional[List[int]] = None,\n    projection_head2_dims: Optional[List[int]] = None,\n    projection_heads_activation: str = \"relu\",\n    cat_mlp_type: Literal[\"single\", \"multiple\"] = \"multiple\",\n    cont_mlp_type: Literal[\"single\", \"multiple\"] = \"multiple\",\n    denoise_mlps_activation: str = \"relu\",\n    verbose: int = 1,\n    seed: int = 1,\n    **kwargs,\n):\n    super().__init__(\n        model=model,\n        preprocessor=preprocessor,\n        loss_type=loss_type,\n        optimizer=optimizer,\n        lr_scheduler=lr_scheduler,\n        callbacks=callbacks,\n        projection_head1_dims=projection_head1_dims,\n        projection_head2_dims=projection_head2_dims,\n        projection_heads_activation=projection_heads_activation,\n        cat_mlp_type=cat_mlp_type,\n        cont_mlp_type=cont_mlp_type,\n        denoise_mlps_activation=denoise_mlps_activation,\n        verbose=verbose,\n        seed=seed,\n        **kwargs,\n    )\n</code></pre>"},{"location":"pytorch-widedeep/self_supervised_pretraining.html#pytorch_widedeep.self_supervised_training.ContrastiveDenoisingTrainer.pretrain","title":"pretrain","text":"<pre><code>pretrain(\n    X_tab,\n    X_tab_val=None,\n    val_split=None,\n    validation_freq=1,\n    n_epochs=1,\n    batch_size=32,\n)\n</code></pre> <p>Pretrain method. Can also be called using <code>.fit(&lt;same_args&gt;)</code></p> <p>Parameters:</p> <ul> <li> <code>X_tab</code>               (<code>ndarray</code>)           \u2013            <p>tabular dataset</p> </li> <li> <code>X_tab_val</code>               (<code>Optional[ndarray]</code>, default:                   <code>None</code> )           \u2013            <p>validation data. Note that, although it is possible to use contrastive-denoising training with a validation set, such set must include feature values that are all seen in the training set in the case of the categorical columns. This is because the values of the columns themselves will be used as targets when computing the loss. Therefore, if a new category is present in the validation set that was not seen in training this will effectively be like trying to predict a new, never seen category (and Pytorch will throw an error)</p> </li> <li> <code>val_split</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>An alterative to passing the validation set is to use a train/val split fraction via <code>val_split</code></p> </li> <li> <code>validation_freq</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>epochs validation frequency</p> </li> <li> <code>n_epochs</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>number of epochs</p> </li> <li> <code>batch_size</code>               (<code>int</code>, default:                   <code>32</code> )           \u2013            <p>batch size</p> </li> </ul> Source code in <code>pytorch_widedeep/self_supervised_training/contrastive_denoising_trainer.py</code> <pre><code>def pretrain(\n    self,\n    X_tab: np.ndarray,\n    X_tab_val: Optional[np.ndarray] = None,\n    val_split: Optional[float] = None,\n    validation_freq: int = 1,\n    n_epochs: int = 1,\n    batch_size: int = 32,\n):\n    r\"\"\"Pretrain method. Can also be called using `.fit(&lt;same_args&gt;)`\n\n    Parameters\n    ----------\n    X_tab: np.ndarray,\n        tabular dataset\n    X_tab_val: np.ndarray, Optional, default = None\n        validation data. Note that, although it is possible to use\n        contrastive-denoising training with a validation set, such set\n        must include feature values that are _all_ seen in the training\n        set in the case of the categorical columns. This is because the\n        values of the columns themselves will be used as targets when\n        computing the loss. Therefore, if a new category is present in\n        the validation set that was not seen in training this will\n        effectively be like trying to predict a new, never seen category\n        (and Pytorch will throw an error)\n    val_split: float, Optional. default=None\n        An alterative to passing the validation set is to use a train/val\n        split fraction via `val_split`\n    validation_freq: int, default=1\n        epochs validation frequency\n    n_epochs: int, default=1\n        number of epochs\n    batch_size: int, default=32\n        batch size\n    \"\"\"\n\n    self.batch_size = batch_size\n\n    train_set, eval_set = self._train_eval_split(X_tab, X_tab_val, val_split)\n    train_loader = DataLoader(\n        dataset=train_set, batch_size=batch_size, num_workers=self.num_workers\n    )\n    train_steps = len(train_loader)\n    if eval_set is not None:\n        eval_loader = DataLoader(\n            dataset=eval_set,\n            batch_size=batch_size,\n            num_workers=self.num_workers,\n            shuffle=False,\n        )\n        eval_steps = len(eval_loader)\n\n    self.callback_container.on_train_begin(\n        {\n            \"batch_size\": batch_size,\n            \"train_steps\": train_steps,\n            \"n_epochs\": n_epochs,\n        }\n    )\n    for epoch in range(n_epochs):\n        epoch_logs: Dict[str, float] = {}\n        self.callback_container.on_epoch_begin(epoch, logs=epoch_logs)\n\n        self.train_running_loss = 0.0\n        with trange(train_steps, disable=self.verbose != 1) as t:\n            for batch_idx, X in zip(t, train_loader):\n                t.set_description(\"epoch %i\" % (epoch + 1))\n                train_loss = self._train_step(X[0], batch_idx)\n                self.callback_container.on_batch_end(batch=batch_idx)\n                print_loss_and_metric(t, train_loss)\n\n        epoch_logs = save_epoch_logs(epoch_logs, train_loss, None, \"train\")\n\n        on_epoch_end_metric = None\n        if eval_set is not None and epoch % validation_freq == (\n            validation_freq - 1\n        ):\n            self.callback_container.on_eval_begin()\n            self.valid_running_loss = 0.0\n            with trange(eval_steps, disable=self.verbose != 1) as v:\n                for batch_idx, X in zip(v, eval_loader):\n                    v.set_description(\"valid\")\n                    val_loss = self._eval_step(X[0], batch_idx)\n                    print_loss_and_metric(v, val_loss)\n            epoch_logs = save_epoch_logs(epoch_logs, val_loss, None, \"val\")\n            on_epoch_end_metric = val_loss\n        else:\n            if self.reducelronplateau:\n                raise NotImplementedError(\n                    \"ReduceLROnPlateau scheduler can be used only with validation data.\"\n                )\n\n        self.callback_container.on_epoch_end(epoch, epoch_logs, on_epoch_end_metric)\n\n        if self.early_stop:\n            self.callback_container.on_train_end(epoch_logs)\n            break\n\n    self.callback_container.on_train_end(epoch_logs)\n    self._restore_best_weights()\n    self.cd_model.train()\n</code></pre>"},{"location":"pytorch-widedeep/tab2vec.html","title":"Tab2Vec","text":""},{"location":"pytorch-widedeep/tab2vec.html#pytorch_widedeep.tab2vec.Tab2Vec","title":"Tab2Vec","text":"<pre><code>Tab2Vec(\n    tab_preprocessor,\n    model,\n    device,\n    return_dataframe=False,\n    verbose=False,\n)\n</code></pre> <p>Class to transform an input dataframe into vectorized form.</p> <p>This class will take an input dataframe in the form of the dataframe used for training, and it will turn it into a vectorised form based on the processing applied by the model to the categorical and continuous columns.</p> <p> NOTE: Currently this class is only implemented  for the deeptabular component. Therefore, if the input dataframe has a  text column or a column with the path to images, these will be ignored.  We will be adding these functionalities in future versions</p> <p>Parameters:</p> <ul> <li> <code>model</code>               (<code>Union[WideDeep, BayesianWide, BayesianTabMlp]</code>)           \u2013            <p><code>WideDeep</code>, <code>BayesianWide</code> or <code>BayesianTabMlp</code> model. Must be trained.</p> </li> <li> <code>tab_preprocessor</code>               (<code>TabPreprocessor</code>)           \u2013            <p><code>TabPreprocessor</code> object. Must be fitted.</p> </li> <li> <code>return_dataframe</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating of the returned object(s) will be array(s) or pandas dataframe(s)</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>vectorizer</code>               (<code>Module</code>)           \u2013            <p>Torch module with the categorical and continuous encoding process</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import string\n&gt;&gt;&gt; from random import choices\n&gt;&gt;&gt; import numpy as np\n&gt;&gt;&gt; import pandas as pd\n&gt;&gt;&gt; from pytorch_widedeep import Tab2Vec\n&gt;&gt;&gt; from pytorch_widedeep.models import TabMlp, WideDeep\n&gt;&gt;&gt; from pytorch_widedeep.preprocessing import TabPreprocessor\n&gt;&gt;&gt;\n&gt;&gt;&gt; colnames = list(string.ascii_lowercase)[:4]\n&gt;&gt;&gt; cat_col1_vals = [\"a\", \"b\", \"c\"]\n&gt;&gt;&gt; cat_col2_vals = [\"d\", \"e\", \"f\"]\n&gt;&gt;&gt;\n&gt;&gt;&gt; # Create the toy input dataframe and a toy dataframe to be vectorised\n&gt;&gt;&gt; cat_inp = [np.array(choices(c, k=5)) for c in [cat_col1_vals, cat_col2_vals]]\n&gt;&gt;&gt; cont_inp = [np.round(np.random.rand(5), 2) for _ in range(2)]\n&gt;&gt;&gt; df_inp = pd.DataFrame(np.vstack(cat_inp + cont_inp).transpose(), columns=colnames)\n&gt;&gt;&gt; cat_t2v = [np.array(choices(c, k=5)) for c in [cat_col1_vals, cat_col2_vals]]\n&gt;&gt;&gt; cont_t2v = [np.round(np.random.rand(5), 2) for _ in range(2)]\n&gt;&gt;&gt; df_t2v = pd.DataFrame(np.vstack(cat_t2v + cont_t2v).transpose(), columns=colnames)\n&gt;&gt;&gt;\n&gt;&gt;&gt; # fit the TabPreprocessor\n&gt;&gt;&gt; embed_cols = [(\"a\", 2), (\"b\", 4)]\n&gt;&gt;&gt; cont_cols = [\"c\", \"d\"]\n&gt;&gt;&gt; tab_preprocessor = TabPreprocessor(cat_embed_cols=embed_cols, continuous_cols=cont_cols)\n&gt;&gt;&gt; X_tab = tab_preprocessor.fit_transform(df_inp)\n&gt;&gt;&gt;\n&gt;&gt;&gt; # define the model (and let's assume we train it)\n&gt;&gt;&gt; tabmlp = TabMlp(\n... column_idx=tab_preprocessor.column_idx,\n... cat_embed_input=tab_preprocessor.cat_embed_input,\n... continuous_cols=tab_preprocessor.continuous_cols,\n... mlp_hidden_dims=[8, 4])\n&gt;&gt;&gt; model = WideDeep(deeptabular=tabmlp)\n&gt;&gt;&gt; # ...train the model...\n&gt;&gt;&gt;\n&gt;&gt;&gt; # vectorise the dataframe\n&gt;&gt;&gt; t2v = Tab2Vec(tab_preprocessor, model, device=\"cpu\")\n&gt;&gt;&gt; X_vec = t2v.transform(df_t2v)\n</code></pre> Source code in <code>pytorch_widedeep/tab2vec.py</code> <pre><code>def __init__(\n    self,\n    tab_preprocessor: TabPreprocessor,\n    model: Union[WideDeep, BayesianWide, BayesianTabMlp],\n    device: str | torch.device,\n    return_dataframe: bool = False,\n    verbose: bool = False,\n):\n    super(Tab2Vec, self).__init__()\n\n    self._check_inputs(tab_preprocessor, model, verbose)\n\n    if isinstance(device, str):\n        self.device = torch.device(device)\n    else:\n        self.device = device\n\n    self.tab_preprocessor = tab_preprocessor\n    self.return_dataframe = return_dataframe\n    self.verbose = verbose\n\n    self.vectorizer = self._set_vectorizer(model)\n\n    self._set_dim_attributes(tab_preprocessor, model)\n</code></pre>"},{"location":"pytorch-widedeep/tab2vec.html#pytorch_widedeep.tab2vec.Tab2Vec.fit","title":"fit","text":"<pre><code>fit(df, target_col=None)\n</code></pre> <p>This is an empty method i.e. Returns the unchanged object itself. Is only included for consistency in case <code>Tab2Vec</code> is used as part of a Pipeline</p> <p>Parameters:</p> <ul> <li> <code>df</code>               (<code>DataFrame</code>)           \u2013            <p>DataFrame to be vectorised, i.e. the categorical and continuous columns will be encoded based on the processing applied within the model</p> </li> <li> <code>target_col</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Column name of the target_col variable. If <code>None</code> only the array of predictors will be returned</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>Tab2Vec</code>           \u2013            </li> </ul> Source code in <code>pytorch_widedeep/tab2vec.py</code> <pre><code>def fit(self, df: pd.DataFrame, target_col: Optional[str] = None) -&gt; \"Tab2Vec\":\n    r\"\"\"This is an empty method i.e. Returns the unchanged object itself. Is\n    only included for consistency in case `Tab2Vec` is used as part of a\n    Pipeline\n\n    Parameters\n    ----------\n    df: pd.DataFrame\n        DataFrame to be vectorised, i.e. the categorical and continuous\n        columns will be encoded based on the processing applied within\n        the model\n    target_col: str, Optional\n        Column name of the target_col variable. If `None` only the array of\n        predictors will be returned\n\n    Returns\n    -------\n    Tab2Vec\n    \"\"\"\n\n    return self\n</code></pre>"},{"location":"pytorch-widedeep/tab2vec.html#pytorch_widedeep.tab2vec.Tab2Vec.transform","title":"transform","text":"<pre><code>transform(df, target_col=None)\n</code></pre> <p>Transforms the input dataframe into vectorized form. If a target column name is passed the target values will be returned separately in their corresponding type (np.ndarray or pd.DataFrame)</p> <p>Parameters:</p> <ul> <li> <code>df</code>               (<code>DataFrame</code>)           \u2013            <p>DataFrame to be vectorised, i.e. the categorical and continuous columns will be encoded based on the processing applied within the model</p> </li> <li> <code>target_col</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Column name of the target_col variable. If <code>None</code> only the array of predictors will be returned</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>Union[np.ndarray, Tuple[np.ndarray, np.ndarray], pd.DataFrame, Tuple[pd.DataFrame, pd.Series]</code>           \u2013            <p>Returns eiter a numpy array with the vectorised values, or a Tuple of numpy arrays with the vectorised values and the target. The same applies to dataframes in case we choose to set <code>return_dataframe = True</code></p> </li> </ul> Source code in <code>pytorch_widedeep/tab2vec.py</code> <pre><code>def transform(\n    self,\n    df: pd.DataFrame,\n    target_col: Optional[str] = None,\n) -&gt; Union[\n    np.ndarray,\n    Tuple[np.ndarray, np.ndarray],\n    pd.DataFrame,\n    Tuple[pd.DataFrame, pd.Series],\n]:\n    r\"\"\"Transforms the input dataframe into vectorized form. If a target\n    column name is passed the target values will be returned separately\n    in their corresponding type (np.ndarray or pd.DataFrame)\n\n    Parameters\n    ----------\n    df: pd.DataFrame\n        DataFrame to be vectorised, i.e. the categorical and continuous\n        columns will be encoded based on the processing applied within\n        the model\n    target_col: str, Optional\n        Column name of the target_col variable. If `None` only the array of\n        predictors will be returned\n\n    Returns\n    -------\n    Union[np.ndarray, Tuple[np.ndarray, np.ndarray], pd.DataFrame, Tuple[pd.DataFrame, pd.Series]\n        Returns eiter a numpy array with the vectorised values, or a Tuple\n        of numpy arrays with the vectorised values and the target. The\n        same applies to dataframes in case we choose to set\n        `return_dataframe = True`\n    \"\"\"\n\n    X_tab = self.tab_preprocessor.transform(df)\n\n    if self.device.type == \"mps\":\n        X = torch.from_numpy(X_tab.astype(\"float32\")).to(self.device)\n    else:\n        X = torch.from_numpy(X_tab.astype(\"float\")).to(self.device)\n\n    with torch.no_grad():\n        if self.is_tab_transformer:\n            x_vec, x_cont_not_embed = self.vectorizer(X)\n        else:\n            x_vec = self.vectorizer(X)\n            x_cont_not_embed = None\n\n    if self.tab_preprocessor.with_cls_token:\n        x_vec = x_vec[:, 1:, :]\n\n    if self.tab_preprocessor.with_attention:\n        x_vec = einops.rearrange(x_vec, \"s c e -&gt; s (c e)\")\n\n    if x_cont_not_embed is not None:\n        x_vec = torch.cat([x_vec, x_cont_not_embed], 1).detach().cpu().numpy()\n    else:\n        x_vec = x_vec.detach().cpu().numpy()\n\n    if self.return_dataframe:\n        new_colnames = self._new_colnames()\n        if target_col:\n            return pd.DataFrame(data=x_vec, columns=new_colnames), df[[target_col]]\n        else:\n            return pd.DataFrame(data=x_vec, columns=new_colnames)\n    else:\n        if target_col:\n            return x_vec, df[target_col].values\n        else:\n            return x_vec\n</code></pre>"},{"location":"pytorch-widedeep/tab2vec.html#pytorch_widedeep.tab2vec.Tab2Vec.fit_transform","title":"fit_transform","text":"<pre><code>fit_transform(df, target_col=None)\n</code></pre> <p>Combines <code>fit</code> and <code>transform</code></p> Source code in <code>pytorch_widedeep/tab2vec.py</code> <pre><code>def fit_transform(\n    self, df: pd.DataFrame, target_col: Optional[str] = None\n) -&gt; Union[\n    np.ndarray,\n    Tuple[np.ndarray, np.ndarray],\n    pd.DataFrame,\n    Tuple[pd.DataFrame, pd.Series],\n]:\n    r\"\"\"Combines `fit` and `transform`\"\"\"\n    return self.fit(df, target_col).transform(df, target_col)\n</code></pre>"},{"location":"pytorch-widedeep/the_rec_module.html","title":"The <code>rec</code> module","text":"<p>This module contains models are that specifically designed for recommendation systems. While the rest of the models can be accessed from the <code>pytorch_widedeep.models</code> module, models in this module need to be specifically imported from the <code>rec</code> module, e.g.:</p> <pre><code>from pytorch_widedeep.models.rec import DeepFactorizationMachine\n</code></pre> <p>The list of models here is not meant to be exhaustive, but it includes some common architectures such as factorization machines, field aware factorization machines or extreme factorization machines. More models will be added in the future.</p>"},{"location":"pytorch-widedeep/the_rec_module.html#pytorch_widedeep.models.rec.deepfm.DeepFactorizationMachine","title":"DeepFactorizationMachine","text":"<pre><code>DeepFactorizationMachine(\n    *,\n    column_idx,\n    num_factors,\n    reduce_sum=True,\n    cat_embed_input=None,\n    cat_embed_dropout=None,\n    use_cat_bias=None,\n    cat_embed_activation=None,\n    continuous_cols=None,\n    cont_norm_layer=None,\n    embed_continuous=None,\n    embed_continuous_method=\"piecewise\",\n    cont_embed_dropout=None,\n    cont_embed_activation=None,\n    quantization_setup=None,\n    n_frequencies=None,\n    sigma=None,\n    share_last_layer=None,\n    full_embed_dropout=None,\n    mlp_hidden_dims=None,\n    mlp_activation=None,\n    mlp_dropout=None,\n    mlp_batchnorm=None,\n    mlp_batchnorm_last=None,\n    mlp_linear_first=None\n)\n</code></pre> <p>               Bases: <code>BaseTabularModelWithAttention</code></p> <p>Deep Factorization Machine (DeepFM) for recommendation systems, which is an adaptation of 'Factorization Machines' by Steffen Rendle. Presented in 'DeepFM: A Factorization-Machine based Neural Network for CTR Prediction' by Huifeng Guo, Ruiming Tang, Yunming Yey, Zhenguo Li, Xiuqiang He. 2017.</p> <p>The implementation in this library takes advantage of all the functionalities available to encode categorical and continuous features. The model can be used with only the factorization machine</p> <p>Note that this class implements only the 'Deep' component of the model described in the paper. The linear component is not implemented 'internally' and, if one wants to include it, it can be easily added using the 'wide' (aka linear) component available in this library. See the examples in the examples folder.</p> <p>Parameters:</p> <ul> <li> <code>column_idx</code>               (<code>Dict[str, int]</code>)           \u2013            <p>Dictionary mapping column names to their corresponding index.</p> </li> <li> <code>num_factors</code>               (<code>int</code>)           \u2013            <p>Number of factors for the factorization machine.</p> </li> <li> <code>reduce_sum</code>               (<code>bool</code>, default:                   <code>True</code> )           \u2013            <p>Whether to reduce the sum in the factorization machine output.</p> </li> <li> <code>cat_embed_input</code>               (<code>Optional[List[Tuple[str, int]]]</code>, default:                   <code>None</code> )           \u2013            <p>List of tuples with categorical column names and number of unique values.</p> </li> <li> <code>cat_embed_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Categorical embeddings dropout. If <code>None</code>, it will default to 0.</p> </li> <li> <code>use_cat_bias</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating if bias will be used for the categorical embeddings. If <code>None</code>, it will default to 'False'.</p> </li> <li> <code>cat_embed_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the categorical embeddings, if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> </li> <li> <code>continuous_cols</code>               (<code>Optional[List[str]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the name of the numeric (aka continuous) columns</p> </li> <li> <code>cont_norm_layer</code>               (<code>Optional[Literal['batchnorm', 'layernorm']]</code>, default:                   <code>None</code> )           \u2013            <p>Type of normalization layer applied to the continuous features. Options are: 'layernorm' and 'batchnorm'. if <code>None</code>, no normalization layer will be used.</p> </li> <li> <code>embed_continuous_method</code>               (<code>Optional[Literal['piecewise', 'periodic']]</code>, default:                   <code>\"piecewise\"</code> )           \u2013            <p>Method to use to embed the continuous features. Options are: 'standard', 'periodic' or 'piecewise'. The 'standard' embedding method is based on the FT-Transformer implementation presented in the paper: Revisiting Deep Learning Models for Tabular Data. The 'periodic' and_'piecewise'_ methods were presented in the paper: On Embeddings for Numerical Features in Tabular Deep Learning. Please, read the papers for details.</p> </li> <li> <code>cont_embed_dim</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>Size of the continuous embeddings. If the continuous columns are embedded, <code>cont_embed_dim</code> must be passed.</p> </li> <li> <code>cont_embed_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p> </li> <li> <code>cont_embed_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the continuous embeddings if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported. If <code>None</code>, no activation function will be applied.</p> </li> <li> <code>quantization_setup</code>               (<code>Optional[Dict[str, List[float]]]</code>, default:                   <code>None</code> )           \u2013            <p>This parameter is used when the 'piecewise' method is used to embed the continuous cols. It is a dict where keys are the name of the continuous columns and values are lists with the boundaries for the quantization of the continuous_cols. See the examples for details. If If the 'piecewise' method is used, this parameter is required.</p> </li> <li> <code>n_frequencies</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>This is the so called 'k' in their paper On Embeddings for Numerical Features in Tabular Deep Learning, and is the number of 'frequencies' that will be used to represent each continuous column. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>sigma</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>This is the sigma parameter in the paper mentioned when describing the previous parameters and it is used to initialise the 'frequency weights'. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>share_last_layer</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>This parameter is not present in the before mentioned paper but it is implemented in the official repo. If <code>True</code> the linear layer that turns the frequencies into embeddings will be shared across the continuous columns. If <code>False</code> a different linear layer will be used for each continuous column. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>full_embed_dropout</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>If <code>True</code>, the full embedding corresponding to a column will be masked out/dropout. If <code>None</code>, it will default to <code>False</code>.</p> </li> <li> <code>mlp_hidden_dims</code>               (<code>Optional[List[int]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the number of neurons per dense layer in the mlp.</p> </li> <li> <code>mlp_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the dense layers of the MLP. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> </li> <li> <code>mlp_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>float or List of floats with the dropout between the dense layers. e.g: [0.5,0.5]</p> </li> <li> <code>mlp_batchnorm</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating whether or not batch normalization will be applied to the dense layers</p> </li> <li> <code>mlp_batchnorm_last</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating whether or not batch normalization will be applied to the last of the dense layers</p> </li> <li> <code>mlp_linear_first</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code></p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>mlp</code>               (<code>MLP</code>)           \u2013            <p>MLP component of the model if the mlp_hidden_dims parameter is not None. If None, the model will only return the output of the factorization machine.</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; from typing import Dict, List, Tuple\n&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from torch import Tensor\n&gt;&gt;&gt; from pytorch_widedeep.models.rec import DeepFactorizationMachine\n&gt;&gt;&gt; X = torch.randint(0, 10, (16, 2))\n&gt;&gt;&gt; column_idx: Dict[str, int] = {\"col1\": 0, \"col2\": 1}\n&gt;&gt;&gt; cat_embed_input: List[Tuple[str, int]] = [(\"col1\", 10), (\"col2\", 10)]\n&gt;&gt;&gt; fm = DeepFactorizationMachine(\n...     column_idx=column_idx,\n...     num_factors=8,\n...     cat_embed_input=cat_embed_input,\n...     mlp_hidden_dims=[16, 8]\n... )\n&gt;&gt;&gt; out = fm(X)\n</code></pre> Source code in <code>pytorch_widedeep/models/rec/deepfm.py</code> <pre><code>def __init__(\n    self,\n    *,\n    column_idx: Dict[str, int],\n    num_factors: int,\n    reduce_sum: bool = True,\n    cat_embed_input: Optional[List[Tuple[str, int]]] = None,\n    cat_embed_dropout: Optional[float] = None,\n    use_cat_bias: Optional[bool] = None,\n    cat_embed_activation: Optional[str] = None,\n    continuous_cols: Optional[List[str]] = None,\n    cont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\n    embed_continuous: Optional[bool] = None,\n    embed_continuous_method: Optional[\n        Literal[\"piecewise\", \"periodic\"]\n    ] = \"piecewise\",\n    cont_embed_dropout: Optional[float] = None,\n    cont_embed_activation: Optional[str] = None,\n    quantization_setup: Optional[Dict[str, List[float]]] = None,\n    n_frequencies: Optional[int] = None,\n    sigma: Optional[float] = None,\n    share_last_layer: Optional[bool] = None,\n    full_embed_dropout: Optional[bool] = None,\n    mlp_hidden_dims: Optional[List[int]] = None,\n    mlp_activation: Optional[str] = None,\n    mlp_dropout: Optional[float] = None,\n    mlp_batchnorm: Optional[bool] = None,\n    mlp_batchnorm_last: Optional[bool] = None,\n    mlp_linear_first: Optional[bool] = None,\n):\n    super(DeepFactorizationMachine, self).__init__(\n        column_idx=column_idx,\n        input_dim=num_factors,\n        cat_embed_input=cat_embed_input,\n        cat_embed_dropout=cat_embed_dropout,\n        use_cat_bias=use_cat_bias,\n        cat_embed_activation=cat_embed_activation,\n        shared_embed=False,\n        add_shared_embed=None,\n        frac_shared_embed=None,\n        continuous_cols=continuous_cols,\n        cont_norm_layer=cont_norm_layer,\n        embed_continuous_method=embed_continuous_method,\n        cont_embed_dropout=cont_embed_dropout,\n        cont_embed_activation=cont_embed_activation,\n        quantization_setup=quantization_setup,\n        n_frequencies=n_frequencies,\n        sigma=sigma,\n        share_last_layer=share_last_layer,\n        full_embed_dropout=full_embed_dropout,\n    )\n\n    self.reduce_sum = reduce_sum\n\n    self.mlp_hidden_dims = mlp_hidden_dims\n    self.mlp_activation = mlp_activation\n    self.mlp_dropout = mlp_dropout\n    self.mlp_batchnorm = mlp_batchnorm\n    self.mlp_batchnorm_last = mlp_batchnorm_last\n    self.mlp_linear_first = mlp_linear_first\n\n    if self.mlp_hidden_dims is not None:\n\n        if self.mlp_hidden_dims[-1] != self.input_dim:\n            d_hidden = (\n                [self.input_dim * len(self.column_idx)]\n                + self.mlp_hidden_dims\n                + [self.input_dim]\n            )\n        else:\n            d_hidden = [\n                self.input_dim * len(self.column_idx)\n            ] + self.mlp_hidden_dims\n\n        self.mlp = MLP(\n            d_hidden=d_hidden,\n            activation=(\n                \"relu\" if self.mlp_activation is None else self.mlp_activation\n            ),\n            dropout=0.0 if self.mlp_dropout is None else self.mlp_dropout,\n            batchnorm=False if self.mlp_batchnorm is None else self.mlp_batchnorm,\n            batchnorm_last=(\n                False\n                if self.mlp_batchnorm_last is None\n                else self.mlp_batchnorm_last\n            ),\n            linear_first=(\n                False if self.mlp_linear_first is None else self.mlp_linear_first\n            ),\n        )\n    else:\n        self.mlp = None\n</code></pre>"},{"location":"pytorch-widedeep/the_rec_module.html#pytorch_widedeep.models.rec.deepffm.DeepFieldAwareFactorizationMachine","title":"DeepFieldAwareFactorizationMachine","text":"<pre><code>DeepFieldAwareFactorizationMachine(\n    *,\n    column_idx,\n    num_factors,\n    cat_embed_input,\n    reduce_sum=True,\n    cat_embed_dropout=None,\n    use_cat_bias=None,\n    cat_embed_activation=None,\n    mlp_hidden_dims=None,\n    mlp_activation=None,\n    mlp_dropout=None,\n    mlp_batchnorm=None,\n    mlp_batchnorm_last=None,\n    mlp_linear_first=None\n)\n</code></pre> <p>               Bases: <code>BaseTabularModelWithAttention</code></p> <p>Deep Field Aware Factorization Machine (DeepFFM) for recommendation systems. Adaptation of the paper 'Field-aware Factorization Machines in a Real-world Online Advertising System', Juan et al. 2017.</p> <p>This class implements only the 'Deep' component of the model described in the paper. The linear component is not implemented 'internally' and, if one wants to include it, it can be easily added using the 'wide'/linear component in this library. See the examples in the examples folder.</p> <p>Note that in this case, only categorical features are accepted. This is because the embeddings of each feature will be learned using all other features. Therefore these embeddings have to be all of the same nature. This does not occur if we mix categorical and continuous features.</p> <p>Parameters:</p> <ul> <li> <code>column_idx</code>               (<code>Dict[str, int]</code>)           \u2013            <p>Dictionary mapping column names to their corresponding index.</p> </li> <li> <code>num_factors</code>               (<code>int</code>)           \u2013            <p>Number of factors for the factorization machine.</p> </li> <li> <code>reduce_sum</code>               (<code>bool</code>, default:                   <code>True</code> )           \u2013            <p>Whether to reduce the sum in the factorization machine output.</p> </li> <li> <code>cat_embed_input</code>               (<code>Optional[List[Tuple[str, int]]]</code>, default:                   <code>None</code> )           \u2013            <p>List of tuples with categorical column names and number of unique values.</p> </li> <li> <code>cat_embed_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Categorical embeddings dropout. If <code>None</code>, it will default to 0.</p> </li> <li> <code>use_cat_bias</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating if bias will be used for the categorical embeddings. If <code>None</code>, it will default to 'False'.</p> </li> <li> <code>cat_embed_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the categorical embeddings, if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> </li> <li> <code>mlp_hidden_dims</code>               (<code>Optional[List[int]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the number of neurons per dense layer in the mlp.</p> </li> <li> <code>mlp_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the dense layers of the MLP. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> </li> <li> <code>mlp_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>float or List of floats with the dropout between the dense layers. e.g: [0.5,0.5]</p> </li> <li> <code>mlp_batchnorm</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating whether or not batch normalization will be applied to the dense layers</p> </li> <li> <code>mlp_batchnorm_last</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating whether or not batch normalization will be applied to the last of the dense layers</p> </li> <li> <code>mlp_linear_first</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code></p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>n_features</code>               (<code>int</code>)           \u2013            <p>Number of unique features/columns</p> </li> <li> <code>n_tokens</code>               (<code>int</code>)           \u2013            <p>Number of unique values (tokens) in the full dataset (corpus)</p> </li> <li> <code>encoders</code>               (<code>ModuleList</code>)           \u2013            <p>List of <code>BaseTabularModelWithAttention</code> instances. One per categorical column</p> </li> <li> <code>mlp</code>               (<code>Module</code>)           \u2013            <p>Multi-layer perceptron. If <code>None</code> the output will be the output of the factorization machine (i.e. the sum of the interactions)</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from torch import Tensor\n&gt;&gt;&gt; from typing import Dict, List, Tuple\n&gt;&gt;&gt; from pytorch_widedeep.models.rec import DeepFieldAwareFactorizationMachine\n&gt;&gt;&gt; X = torch.randint(0, 10, (16, 2))\n&gt;&gt;&gt; column_idx: Dict[str, int] = {\"col1\": 0, \"col2\": 1}\n&gt;&gt;&gt; cat_embed_input: List[Tuple[str, int]] = [(\"col1\", 10), (\"col2\", 10)]\n&gt;&gt;&gt; ffm = DeepFieldAwareFactorizationMachine(\n...     column_idx=column_idx,\n...     num_factors=4,\n...     cat_embed_input=cat_embed_input,\n...     mlp_hidden_dims=[16, 8]\n... )\n&gt;&gt;&gt; output = ffm(X)\n</code></pre> Source code in <code>pytorch_widedeep/models/rec/deepffm.py</code> <pre><code>def __init__(\n    self,\n    *,\n    column_idx: Dict[str, int],\n    num_factors: int,\n    cat_embed_input: List[Tuple[str, int]],\n    reduce_sum: bool = True,\n    cat_embed_dropout: Optional[float] = None,\n    use_cat_bias: Optional[bool] = None,\n    cat_embed_activation: Optional[str] = None,\n    mlp_hidden_dims: Optional[List[int]] = None,\n    mlp_activation: Optional[str] = None,\n    mlp_dropout: Optional[float] = None,\n    mlp_batchnorm: Optional[bool] = None,\n    mlp_batchnorm_last: Optional[bool] = None,\n    mlp_linear_first: Optional[bool] = None,\n):\n    super(DeepFieldAwareFactorizationMachine, self).__init__(\n        column_idx=column_idx,\n        input_dim=num_factors,\n        cat_embed_input=cat_embed_input,\n        cat_embed_dropout=cat_embed_dropout,\n        use_cat_bias=use_cat_bias,\n        cat_embed_activation=cat_embed_activation,\n        shared_embed=False,\n        add_shared_embed=None,\n        frac_shared_embed=None,\n        continuous_cols=None,\n        cont_norm_layer=None,\n        embed_continuous_method=None,\n        cont_embed_dropout=None,\n        cont_embed_activation=None,\n        quantization_setup=None,\n        n_frequencies=None,\n        sigma=None,\n        share_last_layer=None,\n        full_embed_dropout=None,\n    )\n\n    self.reduce_sum = reduce_sum\n\n    self.mlp_hidden_dims = mlp_hidden_dims\n    self.mlp_activation = mlp_activation\n    self.mlp_dropout = mlp_dropout\n    self.mlp_batchnorm = mlp_batchnorm\n    self.mlp_batchnorm_last = mlp_batchnorm_last\n    self.mlp_linear_first = mlp_linear_first\n\n    self.n_features = len(self.column_idx)\n    self.n_tokens = sum([ei[1] for ei in cat_embed_input])\n\n    self.encoders = nn.ModuleList(\n        [\n            BaseTabularModelWithAttention(**config)\n            for config in self._get_encoder_configs()\n        ]\n    )\n\n    if self.mlp_hidden_dims is not None:\n        d_hidden = [\n            self.n_features * (self.n_features - 1) // 2 * num_factors\n        ] + self.mlp_hidden_dims\n        self.mlp = MLP(\n            d_hidden=d_hidden,\n            activation=(\n                \"relu\" if self.mlp_activation is None else self.mlp_activation\n            ),\n            dropout=0.0 if self.mlp_dropout is None else self.mlp_dropout,\n            batchnorm=False if self.mlp_batchnorm is None else self.mlp_batchnorm,\n            batchnorm_last=(\n                False\n                if self.mlp_batchnorm_last is None\n                else self.mlp_batchnorm_last\n            ),\n            linear_first=(\n                False if self.mlp_linear_first is None else self.mlp_linear_first\n            ),\n        )\n    else:\n        self.mlp = None\n</code></pre>"},{"location":"pytorch-widedeep/the_rec_module.html#pytorch_widedeep.models.rec.xdeepfm.ExtremeDeepFactorizationMachine","title":"ExtremeDeepFactorizationMachine","text":"<pre><code>ExtremeDeepFactorizationMachine(\n    *,\n    column_idx,\n    input_dim,\n    reduce_sum=True,\n    cin_layer_dims,\n    cat_embed_input,\n    cat_embed_dropout=None,\n    use_cat_bias=None,\n    cat_embed_activation=None,\n    continuous_cols=None,\n    cont_norm_layer=None,\n    embed_continuous_method=\"piecewise\",\n    cont_embed_dropout=None,\n    cont_embed_activation=None,\n    quantization_setup=None,\n    n_frequencies=None,\n    sigma=None,\n    share_last_layer=None,\n    full_embed_dropout=None,\n    mlp_hidden_dims=None,\n    mlp_activation=None,\n    mlp_dropout=None,\n    mlp_batchnorm=None,\n    mlp_batchnorm_last=None,\n    mlp_linear_first=None\n)\n</code></pre> <p>               Bases: <code>BaseTabularModelWithAttention</code></p> <p>Adaptation of 'xDeepFM implementation: xDeepFM: Combining Explicit and Implicit Feature Interactions for Recommender Systems' by Jianxun Lian, Xiaohuan Zhou, Fuzheng Zhang, Zhongxia Chen, Xing Xie, Guangzhong Sun and Enhong Chen, 2018</p> <p>The implementation in this library takes advantage of all the functionalities available to encode categorical and continuous features. The model can be used with only the factorization machine</p> <p>Note that this class implements only the 'Deep' component of the model described in the paper. The linear component is not implemented 'internally' and, if one wants to include it, it can be easily added using the 'wide'/linear component in this library. See the examples in the examples folder.</p> <p>Parameters:</p> <ul> <li> <code>column_idx</code>               (<code>Dict[str, int]</code>)           \u2013            <p>Dictionary mapping column names to their corresponding index.</p> </li> <li> <code>input_dim</code>               (<code>int</code>)           \u2013            <p>Embedding input dimensions</p> </li> <li> <code>reduce_sum</code>               (<code>bool</code>, default:                   <code>True</code> )           \u2013            <p>Whether to reduce the sum in the factorization machine output.</p> </li> <li> <code>cin_layer_dims</code>               (<code>List[int]</code>)           \u2013            <p>List with the number of units per CIN layer. e.g: [128, 64]</p> </li> <li> <code>cat_embed_input</code>               (<code>Optional[List[Tuple[str, int]]]</code>, default:                   <code>None</code> )           \u2013            <p>List of tuples with categorical column names and number of unique values.</p> </li> <li> <code>cat_embed_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Categorical embeddings dropout. If <code>None</code>, it will default to 0.</p> </li> <li> <code>use_cat_bias</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating if bias will be used for the categorical embeddings. If <code>None</code>, it will default to 'False'.</p> </li> <li> <code>cat_embed_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the categorical embeddings, if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> </li> <li> <code>continuous_cols</code>               (<code>Optional[List[str]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the name of the numeric (aka continuous) columns</p> </li> <li> <code>cont_norm_layer</code>               (<code>Optional[Literal['batchnorm', 'layernorm']]</code>, default:                   <code>None</code> )           \u2013            <p>Type of normalization layer applied to the continuous features. Options are: 'layernorm' and 'batchnorm'. if <code>None</code>, no normalization layer will be used.</p> </li> <li> <code>embed_continuous_method</code>               (<code>Optional[Literal['piecewise', 'periodic']]</code>, default:                   <code>\"piecewise\"</code> )           \u2013            <p>Method to use to embed the continuous features. Options are: 'standard', 'periodic' or 'piecewise'. The 'standard' embedding method is based on the FT-Transformer implementation presented in the paper: Revisiting Deep Learning Models for Tabular Data. The 'periodic' and_'piecewise'_ methods were presented in the paper: On Embeddings for Numerical Features in Tabular Deep Learning. Please, read the papers for details.</p> </li> <li> <code>cont_embed_dim</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>Size of the continuous embeddings. If the continuous columns are embedded, <code>cont_embed_dim</code> must be passed.</p> </li> <li> <code>cont_embed_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p> </li> <li> <code>cont_embed_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the continuous embeddings if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported. If <code>None</code>, no activation function will be applied.</p> </li> <li> <code>quantization_setup</code>               (<code>Optional[Dict[str, List[float]]]</code>, default:                   <code>None</code> )           \u2013            <p>This parameter is used when the 'piecewise' method is used to embed the continuous cols. It is a dict where keys are the name of the continuous columns and values are lists with the boundaries for the quantization of the continuous_cols. See the examples for details. If If the 'piecewise' method is used, this parameter is required.</p> </li> <li> <code>n_frequencies</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>This is the so called 'k' in their paper On Embeddings for Numerical Features in Tabular Deep Learning, and is the number of 'frequencies' that will be used to represent each continuous column. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>sigma</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>This is the sigma parameter in the paper mentioned when describing the previous parameters and it is used to initialise the 'frequency weights'. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>share_last_layer</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>This parameter is not present in the before mentioned paper but it is implemented in the official repo. If <code>True</code> the linear layer that turns the frequencies into embeddings will be shared across the continuous columns. If <code>False</code> a different linear layer will be used for each continuous column. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>full_embed_dropout</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>If <code>True</code>, the full embedding corresponding to a column will be masked out/dropout. If <code>None</code>, it will default to <code>False</code>.</p> </li> <li> <code>mlp_hidden_dims</code>               (<code>Optional[List[int]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the number of neurons per dense layer in the mlp.</p> </li> <li> <code>mlp_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the dense layers of the MLP. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> </li> <li> <code>mlp_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>float or List of floats with the dropout between the dense layers. e.g: [0.5,0.5]</p> </li> <li> <code>mlp_batchnorm</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating whether or not batch normalization will be applied to the dense layers</p> </li> <li> <code>mlp_batchnorm_last</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating whether or not batch normalization will be applied to the last of the dense layers</p> </li> <li> <code>mlp_linear_first</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code></p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>n_features</code>               (<code>int</code>)           \u2013            <p>Number of unique features/columns</p> </li> <li> <code>cin</code>               (<code>CompressedInteractionNetwork</code>)           \u2013            <p>Instance of the <code>CompressedInteractionNetwork</code> class</p> </li> <li> <code>mlp</code>               (<code>MLP</code>)           \u2013            <p>Instance of the <code>MLP</code> class if <code>mlp_hidden_dims</code> is not None. If None, the model will return directly the output of the <code>CIN</code></p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models.rec import ExtremeDeepFactorizationMachine\n&gt;&gt;&gt; X_tab = torch.randint(0, 10, (16, 2))\n&gt;&gt;&gt; column_idx = {\"col1\": 0, \"col2\": 1}\n&gt;&gt;&gt; cat_embed_input = [(\"col1\", 10), (\"col2\", 10)]\n&gt;&gt;&gt; xdeepfm = ExtremeDeepFactorizationMachine(\n...     column_idx=column_idx,\n...     input_dim=4,\n...     cin_layer_dims=[8, 16],\n...     cat_embed_input=cat_embed_input,\n...     mlp_hidden_dims=[16, 8]\n... )\n&gt;&gt;&gt; output = xdeepfm(X_tab)\n</code></pre> Source code in <code>pytorch_widedeep/models/rec/xdeepfm.py</code> <pre><code>def __init__(\n    self,\n    *,\n    column_idx: Dict[str, int],\n    input_dim: int,\n    reduce_sum: bool = True,\n    cin_layer_dims: List[int],\n    cat_embed_input: List[Tuple[str, int]],\n    cat_embed_dropout: Optional[float] = None,\n    use_cat_bias: Optional[bool] = None,\n    cat_embed_activation: Optional[str] = None,\n    continuous_cols: Optional[List[str]] = None,\n    cont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\n    embed_continuous_method: Optional[\n        Literal[\"piecewise\", \"periodic\"]\n    ] = \"piecewise\",\n    cont_embed_dropout: Optional[float] = None,\n    cont_embed_activation: Optional[str] = None,\n    quantization_setup: Optional[Dict[str, List[float]]] = None,\n    n_frequencies: Optional[int] = None,\n    sigma: Optional[float] = None,\n    share_last_layer: Optional[bool] = None,\n    full_embed_dropout: Optional[bool] = None,\n    mlp_hidden_dims: Optional[List[int]] = None,\n    mlp_activation: Optional[str] = None,\n    mlp_dropout: Optional[float] = None,\n    mlp_batchnorm: Optional[bool] = None,\n    mlp_batchnorm_last: Optional[bool] = None,\n    mlp_linear_first: Optional[bool] = None,\n):\n    super(ExtremeDeepFactorizationMachine, self).__init__(\n        column_idx=column_idx,\n        input_dim=input_dim,\n        cat_embed_input=cat_embed_input,\n        cat_embed_dropout=cat_embed_dropout,\n        use_cat_bias=use_cat_bias,\n        cat_embed_activation=cat_embed_activation,\n        shared_embed=False,\n        add_shared_embed=None,\n        frac_shared_embed=None,\n        continuous_cols=continuous_cols,\n        cont_norm_layer=cont_norm_layer,\n        embed_continuous_method=embed_continuous_method,\n        cont_embed_dropout=cont_embed_dropout,\n        cont_embed_activation=cont_embed_activation,\n        quantization_setup=quantization_setup,\n        n_frequencies=n_frequencies,\n        sigma=sigma,\n        share_last_layer=share_last_layer,\n        full_embed_dropout=full_embed_dropout,\n    )\n\n    self.mlp_hidden_dims = mlp_hidden_dims\n    self.mlp_activation = mlp_activation\n    self.mlp_dropout = mlp_dropout\n    self.mlp_batchnorm = mlp_batchnorm\n    self.mlp_batchnorm_last = mlp_batchnorm_last\n    self.mlp_linear_first = mlp_linear_first\n\n    self.reduce_sum = reduce_sum\n    self.cin_layer_dims = cin_layer_dims\n\n    self.n_features = len(self.column_idx)\n\n    self.cin = CompressedInteractionNetwork(\n        num_cols=self.n_features, cin_layer_dims=self.cin_layer_dims\n    )\n\n    if self.mlp_hidden_dims is not None:\n        if (\n            self.mlp_hidden_dims[-1] != sum(self.cin_layer_dims)\n            and not self.reduce_sum\n        ):\n            d_hidden = (\n                [sum(self.cin_layer_dims)]\n                + self.mlp_hidden_dims\n                + [sum(self.cin_layer_dims)]\n            )\n        else:\n            d_hidden = [sum(self.cin_layer_dims)] + self.mlp_hidden_dims\n\n        self.mlp = MLP(\n            d_hidden=d_hidden,\n            activation=(\n                \"relu\" if self.mlp_activation is None else self.mlp_activation\n            ),\n            dropout=0.0 if self.mlp_dropout is None else self.mlp_dropout,\n            batchnorm=False if self.mlp_batchnorm is None else self.mlp_batchnorm,\n            batchnorm_last=(\n                False\n                if self.mlp_batchnorm_last is None\n                else self.mlp_batchnorm_last\n            ),\n            linear_first=(\n                False if self.mlp_linear_first is None else self.mlp_linear_first\n            ),\n        )\n    else:\n        self.mlp = None\n</code></pre>"},{"location":"pytorch-widedeep/the_rec_module.html#pytorch_widedeep.models.rec.din.DeepInterestNetwork","title":"DeepInterestNetwork","text":"<pre><code>DeepInterestNetwork(\n    *,\n    column_idx,\n    user_behavior_confiq,\n    target_item_col=\"target_item\",\n    action_seq_config=None,\n    other_seq_cols_confiq=None,\n    attention_unit_activation=\"prelu\",\n    cat_embed_input=None,\n    cat_embed_dropout=None,\n    use_cat_bias=None,\n    cat_embed_activation=None,\n    continuous_cols=None,\n    cont_norm_layer=None,\n    embed_continuous=None,\n    embed_continuous_method=None,\n    cont_embed_dim=None,\n    cont_embed_dropout=None,\n    cont_embed_activation=None,\n    quantization_setup=None,\n    n_frequencies=None,\n    sigma=None,\n    share_last_layer=None,\n    full_embed_dropout=None,\n    mlp_hidden_dims=None,\n    mlp_activation=None,\n    mlp_dropout=None,\n    mlp_batchnorm=None,\n    mlp_batchnorm_last=None,\n    mlp_linear_first=None\n)\n</code></pre> <p>               Bases: <code>BaseWDModelComponent</code></p> <p>Adaptation of the Deep Interest Network (DIN) for recommendation systems as described in the paper: 'Deep Interest Network for Click-Through Rate Prediction' by Guorui Zhou et al. 2018.</p> <p>Note that all the categorical- and continuous-related parameters refer to the categorical and continuous columns that are not part of the sequential columns and will be treated as standard tabular data.</p> <p>This model requires some specific data preparation that allows for quite a lot of flexibility. Therefore, I have included a preprocessor (<code>DINPreprocessor</code>) in the preprocessing module that will take care of the data preparation.</p> <p>Parameters:</p> <ul> <li> <code>column_idx</code>               (<code>Dict[str, int]</code>)           \u2013            <p>Dictionary mapping column names to their corresponding index.</p> </li> <li> <code>target_item_col</code>               (<code>str</code>, default:                   <code>'target_item'</code> )           \u2013            <p>Name of the target item column.</p> </li> <li> <code>user_behavior_confiq</code>               (<code>Tuple[List[str], int, int]</code>)           \u2013            <p>Configuration for user behavior sequence columns. Tuple containing: - List of column names that correspond to the user behavior sequence - Number of unique feature values (n_tokens) - Embedding dimension Example: <code>([\"item_1\", \"item_2\", \"item_3\"], 5, 8)</code></p> </li> <li> <code>action_seq_config</code>               (<code>Optional[Tuple[List[str], int]]</code>, default:                   <code>None</code> )           \u2013            <p>Configuration for a so-called action sequence columns (for example a rating, or purchased/not-purchased, etc). Tuple containing: - List of column names - Number of unique feature values (n_tokens) This action will always be learned as a 1d embedding and will be combined with the user behaviour. For example, imagine that the action is purchased/not-purchased. then per item in the user behaviour sequence there will be a binary action to learn 0/1. Such action will be represented by a float number that will multiply the corresponding item embedding in the user behaviour sequence. Example: <code>([\"rating_1\", \"rating_2\", \"rating_3\"], 5)</code> Internally, the embedding dimension will be set to 1</p> </li> <li> <code>other_seq_cols_confiq</code>               (<code>Optional[List[Tuple[List[str], int, int]]]</code>, default:                   <code>None</code> )           \u2013            <p>Configuration for other sequential columns. List of tuples containing: - List of column names that correspond to the sequential column - Number of unique feature values (n_tokens) - Embedding dimension Example: <code>[([\"seq1_col1\", \"seq1_col2\"], 5, 8), ([\"seq2_col1\", \"seq2_col2\"], 5, 8)]</code></p> </li> <li> <code>attention_unit_activation</code>               (<code>Literal['prelu', 'dice']</code>, default:                   <code>\"prelu\"</code> )           \u2013            <p>Activation function to use in the attention unit.</p> </li> <li> <code>cat_embed_input</code>               (<code>Optional[List[Tuple[str, int, int]]]</code>, default:                   <code>None</code> )           \u2013            <p>Configuration for other columns. List of tuples containing: - Column name - Number of unique feature values (n_tokens) - Embedding dimension</p> <p>Note: From here in advance the remaining parameters are related to the categorical and continuous columns that are not part of the sequential columns and will be treated as standard tabular data.</p> </li> <li> <code>cat_embed_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Categorical embeddings dropout. If <code>None</code>, it will default to 0.</p> </li> <li> <code>use_cat_bias</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating if bias will be used for the categorical embeddings. If <code>None</code>, it will default to 'False'.</p> </li> <li> <code>cat_embed_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the categorical embeddings, if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> </li> <li> <code>continuous_cols</code>               (<code>Optional[List[str]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the name of the numeric (aka continuous) columns</p> </li> <li> <code>cont_norm_layer</code>               (<code>Optional[Literal['batchnorm', 'layernorm']]</code>, default:                   <code>None</code> )           \u2013            <p>Type of normalization layer applied to the continuous features. Options are: 'layernorm' and 'batchnorm'. if <code>None</code>, no normalization layer will be used.</p> </li> <li> <code>embed_continuous</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating if the continuous columns will be embedded using one of the available methods: 'standard', 'periodic' or 'piecewise'. If <code>None</code>, it will default to 'False'. NOTE: This parameter is deprecated and it  will be removed in future releases. Please, use the  <code>embed_continuous_method</code> parameter instead.</p> </li> <li> <code>embed_continuous_method</code>               (<code>Optional[Literal['piecewise', 'periodic']]</code>, default:                   <code>\"piecewise\"</code> )           \u2013            <p>Method to use to embed the continuous features. Options are: 'standard', 'periodic' or 'piecewise'. The 'standard' embedding method is based on the FT-Transformer implementation presented in the paper: Revisiting Deep Learning Models for Tabular Data. The 'periodic' and_'piecewise'_ methods were presented in the paper: On Embeddings for Numerical Features in Tabular Deep Learning. Please, read the papers for details.</p> </li> <li> <code>cont_embed_dim</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>Size of the continuous embeddings. If the continuous columns are embedded, <code>cont_embed_dim</code> must be passed.</p> </li> <li> <code>cont_embed_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p> </li> <li> <code>cont_embed_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the continuous embeddings if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported. If <code>None</code>, no activation function will be applied.</p> </li> <li> <code>quantization_setup</code>               (<code>Optional[Dict[str, List[float]]]</code>, default:                   <code>None</code> )           \u2013            <p>This parameter is used when the 'piecewise' method is used to embed the continuous cols. It is a dict where keys are the name of the continuous columns and values are lists with the boundaries for the quantization of the continuous_cols. See the examples for details. If If the 'piecewise' method is used, this parameter is required.</p> </li> <li> <code>n_frequencies</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>This is the so called 'k' in their paper On Embeddings for Numerical Features in Tabular Deep Learning, and is the number of 'frequencies' that will be used to represent each continuous column. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>sigma</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>This is the sigma parameter in the paper mentioned when describing the previous parameters and it is used to initialise the 'frequency weights'. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>share_last_layer</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>This parameter is not present in the before mentioned paper but it is implemented in the official repo. If <code>True</code> the linear layer that turns the frequencies into embeddings will be shared across the continuous columns. If <code>False</code> a different linear layer will be used for each continuous column. If the 'periodic' method is used, this parameter is required.</p> </li> <li> <code>full_embed_dropout</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>If <code>True</code>, the full embedding corresponding to a column will be masked out/dropout. If <code>None</code>, it will default to <code>False</code>.</p> </li> <li> <code>mlp_hidden_dims</code>               (<code>Optional[List[int]]</code>, default:                   <code>None</code> )           \u2013            <p>List with the number of neurons per dense layer in the mlp.</p> </li> <li> <code>mlp_activation</code>               (<code>Optional[str]</code>, default:                   <code>None</code> )           \u2013            <p>Activation function for the dense layers of the MLP. Currently 'tanh', 'relu', 'leaky_relu', 'gelu' and 'preglu' are   supported</p> </li> <li> <code>mlp_dropout</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>float or List of floats with the dropout between the dense layers. e.g: [0.5,0.5]</p> </li> <li> <code>mlp_batchnorm</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating whether or not batch normalization will be applied to the dense layers</p> </li> <li> <code>mlp_batchnorm_last</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating whether or not batch normalization will be applied to the last of the dense layers</p> </li> <li> <code>mlp_linear_first</code>               (<code>Optional[bool]</code>, default:                   <code>None</code> )           \u2013            <p>Boolean indicating the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code></p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>user_behavior_indexes</code>               (<code>List[int]</code>)           \u2013            <p>List with the indexes of the user behavior columns</p> </li> <li> <code>user_behavior_embed</code>               (<code>BaseTabularModelWithAttention</code>)           \u2013            <p>Embedding layer for the user</p> </li> <li> <code>action_seq_indexes</code>               (<code>List[int]</code>)           \u2013            <p>List with the indexes of the rating sequence columns if the action_seq_config parameter is not None</p> </li> <li> <code>action_embed</code>               (<code>BaseTabularModelWithAttention</code>)           \u2013            <p>Embedding layer for the rating sequence columns if the action_seq_config parameter is not None</p> </li> <li> <code>other_seq_cols_indexes</code>               (<code>Dict[str, List[int]]</code>)           \u2013            <p>Dictionary with the indexes of the other sequential columns if the other_seq_cols_confiq parameter is not None</p> </li> <li> <code>other_seq_cols_embed</code>               (<code>ModuleDict</code>)           \u2013            <p>Dictionary with the embedding layers for the other sequential columns if the other_seq_cols_confiq parameter is not None</p> </li> <li> <code>other_cols_idx</code>               (<code>List[int]</code>)           \u2013            <p>List with the indexes of the other columns if the other_cols_config parameter is not None</p> </li> <li> <code>other_col_embed</code>               (<code>BaseTabularModel</code>)           \u2013            <p>Embedding layer for the other columns if the other_cols_config parameter is not None</p> </li> <li> <code>mlp</code>               (<code>Optional[MLP]</code>)           \u2013            <p>MLP component of the model. If None, no MLP will be used. This should almost always be not None.</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; import numpy as np\n&gt;&gt;&gt; from torch import Tensor\n&gt;&gt;&gt; from typing import Dict, List, Tuple\n&gt;&gt;&gt; from pytorch_widedeep.models.rec import DeepInterestNetwork\n&gt;&gt;&gt; np_seed = np.random.seed(42)\n&gt;&gt;&gt; torch_seed = torch.manual_seed(42)\n&gt;&gt;&gt; num_users = 10\n&gt;&gt;&gt; num_items = 5\n&gt;&gt;&gt; num_contexts = 3\n&gt;&gt;&gt; seq_length = 3\n&gt;&gt;&gt; num_samples = 10\n&gt;&gt;&gt; user_ids = np.random.randint(0, num_users, num_samples)\n&gt;&gt;&gt; target_item_ids = np.random.randint(0, num_items, num_samples)\n&gt;&gt;&gt; context_ids = np.random.randint(0, num_contexts, num_samples)\n&gt;&gt;&gt; user_behavior = np.array(\n...     [\n...         np.random.choice(num_items, seq_length, replace=False)\n...         for _ in range(num_samples)\n...     ]\n... )\n&gt;&gt;&gt; X_arr = np.column_stack((user_ids, target_item_ids, context_ids, user_behavior))\n&gt;&gt;&gt; X = torch.tensor(X_arr, dtype=torch.long)\n&gt;&gt;&gt; column_idx: Dict[str, int] = {\n...     \"user_id\": 0,\n...     \"target_item\": 1,\n...     \"context\": 2,\n...     \"item_1\": 3,\n...     \"item_2\": 4,\n...     \"item_3\": 5,\n... }\n&gt;&gt;&gt; user_behavior_config: Tuple[List[str], int, int] = (\n...     [\"item_1\", \"item_2\", \"item_3\"],\n...     num_items,\n...     8,\n... )\n&gt;&gt;&gt; cat_embed_input: List[Tuple[str, int, int]] = [\n...     (\"user_id\", num_users, 8),\n...     (\"context\", num_contexts, 4),\n... ]\n&gt;&gt;&gt; model = DeepInterestNetwork(\n...     column_idx=column_idx,\n...     target_item_col=\"target_item\",\n...     user_behavior_confiq=user_behavior_config,\n...     cat_embed_input=cat_embed_input,\n...     mlp_hidden_dims=[16, 8],\n... )\n&gt;&gt;&gt; output = model(X)\n</code></pre> Source code in <code>pytorch_widedeep/models/rec/din.py</code> <pre><code>def __init__(\n    self,\n    *,\n    column_idx: Dict[str, int],\n    user_behavior_confiq: Tuple[List[str], int, int],\n    target_item_col: str = \"target_item\",\n    action_seq_config: Optional[Tuple[List[str], int]] = None,\n    other_seq_cols_confiq: Optional[List[Tuple[List[str], int, int]]] = None,\n    attention_unit_activation: Literal[\"prelu\", \"dice\"] = \"prelu\",\n    cat_embed_input: Optional[List[Tuple[str, int, int]]] = None,\n    cat_embed_dropout: Optional[float] = None,\n    use_cat_bias: Optional[bool] = None,\n    cat_embed_activation: Optional[str] = None,\n    continuous_cols: Optional[List[str]] = None,\n    cont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\n    embed_continuous: Optional[bool] = None,\n    embed_continuous_method: Optional[\n        Literal[\"standard\", \"piecewise\", \"periodic\"]\n    ] = None,\n    cont_embed_dim: Optional[int] = None,\n    cont_embed_dropout: Optional[float] = None,\n    cont_embed_activation: Optional[str] = None,\n    quantization_setup: Optional[Dict[str, List[float]]] = None,\n    n_frequencies: Optional[int] = None,\n    sigma: Optional[float] = None,\n    share_last_layer: Optional[bool] = None,\n    full_embed_dropout: Optional[bool] = None,\n    mlp_hidden_dims: Optional[List[int]] = None,\n    mlp_activation: Optional[str] = None,\n    mlp_dropout: Optional[float] = None,\n    mlp_batchnorm: Optional[bool] = None,\n    mlp_batchnorm_last: Optional[bool] = None,\n    mlp_linear_first: Optional[bool] = None,\n):\n    super(DeepInterestNetwork, self).__init__()\n\n    self.column_idx = {\n        k: v for k, v in sorted(column_idx.items(), key=lambda x: x[1])\n    }\n\n    self.column_idx = column_idx\n    self.target_item_col = target_item_col\n    self.user_behavior_confiq = user_behavior_confiq\n    self.action_seq_config = (\n        (action_seq_config[0], action_seq_config[1], 1)\n        if action_seq_config is not None\n        else None\n    )\n    self.other_seq_cols_confiq = other_seq_cols_confiq\n    self.cat_embed_input = cat_embed_input\n    self.attention_unit_activation = attention_unit_activation\n\n    self.cat_embed_dropout = cat_embed_dropout\n    self.use_cat_bias = use_cat_bias\n    self.cat_embed_activation = cat_embed_activation\n    self.continuous_cols = continuous_cols\n    self.cont_norm_layer = cont_norm_layer\n    self.embed_continuous = embed_continuous\n    self.embed_continuous_method = embed_continuous_method\n    self.cont_embed_dim = cont_embed_dim\n    self.cont_embed_dropout = cont_embed_dropout\n    self.cont_embed_activation = cont_embed_activation\n    self.quantization_setup = quantization_setup\n    self.n_frequencies = n_frequencies\n    self.sigma = sigma\n    self.share_last_layer = share_last_layer\n    self.full_embed_dropout = full_embed_dropout\n\n    self.mlp_hidden_dims = mlp_hidden_dims\n    self.mlp_activation = mlp_activation\n    self.mlp_dropout = mlp_dropout\n    self.mlp_batchnorm = mlp_batchnorm\n    self.mlp_batchnorm_last = mlp_batchnorm_last\n    self.mlp_linear_first = mlp_linear_first\n\n    self.target_item_idx = self.column_idx[target_item_col]\n\n    self.user_behavior_indexes, self.user_behavior_embed = (\n        self.set_user_behavior_indexes_and_embed(user_behavior_confiq)\n    )\n    self.user_behavior_dim = user_behavior_confiq[2]\n\n    if self.action_seq_config is not None:\n        self.action_seq_indexes, self.action_embed = (\n            self._set_rating_indexes_and_embed(self.action_seq_config)\n        )\n    else:\n        self.action_embed = None\n\n    if self.other_seq_cols_confiq is not None:\n        (\n            self.other_seq_cols_indexes,\n            self.other_seq_cols_embed,\n            self.other_seq_dim,\n        ) = self._set_other_seq_cols_indexes_embed_and_dim(\n            self.other_seq_cols_confiq\n        )\n    else:\n        self.other_seq_cols_embed = None\n        self.other_seq_dim = 0\n\n    if self.cat_embed_input is not None or self.continuous_cols is not None:\n        self.other_cols_idx, self.other_col_embed, self.other_cols_dim = (\n            self._set_other_cols_idx_embed_and_dim(\n                self.cat_embed_input, self.continuous_cols\n            )\n        )\n    else:\n        self.other_col_embed = None\n        self.other_cols_dim = 0\n\n    self.attention = ActivationUnit(\n        user_behavior_confiq[2], attention_unit_activation\n    )\n\n    if self.mlp_hidden_dims is not None:\n        mlp_input_dim = (\n            self.user_behavior_dim * 2 + self.other_seq_dim + self.other_cols_dim\n        )\n        self.mlp = MLP(\n            d_hidden=[mlp_input_dim] + self.mlp_hidden_dims,\n            activation=(\n                \"relu\" if self.mlp_activation is None else self.mlp_activation\n            ),\n            dropout=0.0 if self.mlp_dropout is None else self.mlp_dropout,\n            batchnorm=False if self.mlp_batchnorm is None else self.mlp_batchnorm,\n            batchnorm_last=(\n                False\n                if self.mlp_batchnorm_last is None\n                else self.mlp_batchnorm_last\n            ),\n            linear_first=(\n                False if self.mlp_linear_first is None else self.mlp_linear_first\n            ),\n        )\n    else:\n        self.mlp = None\n</code></pre>"},{"location":"pytorch-widedeep/the_rec_module.html#pytorch_widedeep.models.rec.basic_transformer.Transformer","title":"Transformer","text":"<pre><code>Transformer(\n    vocab_size,\n    seq_length,\n    input_dim,\n    n_heads,\n    n_blocks,\n    attn_dropout=0.1,\n    ff_dropout=0.1,\n    ff_factor=4,\n    activation=\"gelu\",\n    use_linear_attention=False,\n    use_flash_attention=False,\n    padding_idx=0,\n    with_cls_token=False,\n    *,\n    with_pos_encoding=True,\n    pos_encoding_dropout=0.1,\n    pos_encoder=None\n)\n</code></pre> <p>               Bases: <code>Module</code></p> <p>Basic Encoder-Only Transformer Model for sequence classification/regression. As all other models in the library this model can be used as the <code>deeptext</code> component of a Wide &amp; Deep model or independently by itself.</p> <p> NOTE: This model is introduced in the context of recommendation systems and thought for sequences of any nature (e.g. items). It can, of course, still be used for text.</p> <p>Parameters:</p> <ul> <li> <code>vocab_size</code>               (<code>int</code>)           \u2013            <p>Number of words in the vocabulary</p> </li> <li> <code>input_dim</code>               (<code>int</code>)           \u2013            <p>Dimension of the token embeddings</p> <p>Param aliases: <code>embed_dim</code>, <code>d_model</code>. </p> </li> <li> <code>seq_length</code>               (<code>int</code>)           \u2013            <p>Input sequence length</p> </li> <li> <code>n_heads</code>               (<code>int</code>)           \u2013            <p>Number of attention heads per Transformer block</p> </li> <li> <code>n_blocks</code>               (<code>int</code>)           \u2013            <p>Number of Transformer blocks</p> </li> <li> <code>attn_dropout</code>               (<code>float</code>, default:                   <code>0.1</code> )           \u2013            <p>Dropout that will be applied to the Multi-Head Attention layers</p> </li> <li> <code>ff_dropout</code>               (<code>float</code>, default:                   <code>0.1</code> )           \u2013            <p>Dropout that will be applied to the FeedForward network</p> </li> <li> <code>ff_factor</code>               (<code>int</code>, default:                   <code>4</code> )           \u2013            <p>Multiplicative factor applied to the first layer of the FF network in each Transformer block, This is normally set to 4.</p> </li> <li> <code>activation</code>               (<code>str</code>, default:                   <code>'gelu'</code> )           \u2013            <p>Transformer Encoder activation function. 'tanh', 'relu', 'leaky_relu', 'gelu', 'geglu' and 'reglu' are supported</p> </li> <li> <code>padding_idx</code>               (<code>int</code>, default:                   <code>0</code> )           \u2013            <p>index of the padding token in the padded-tokenised sequences.</p> </li> <li> <code>with_cls_token</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating if a <code>'[CLS]'</code> token is included in the tokenized sequences. If present, the final hidden state corresponding to this token is used as the aggregated representation for classification and regression tasks. NOTE: if included in the tokenized sequences it must be inserted as the first token in the sequences.</p> </li> <li> <code>with_pos_encoding</code>               (<code>bool</code>, default:                   <code>True</code> )           \u2013            <p>Boolean indicating if positional encoding will be used</p> </li> <li> <code>pos_encoding_dropout</code>               (<code>float</code>, default:                   <code>0.1</code> )           \u2013            <p>Positional encoding dropout</p> </li> <li> <code>pos_encoder</code>               (<code>Optional[Module]</code>, default:                   <code>None</code> )           \u2013            <p>This model uses by default a standard positional encoding approach. However, any custom positional encoder can also be used and pass to the Transformer model via the 'pos_encoder' parameter</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>embedding</code>               (<code>Module</code>)           \u2013            <p>Standard token embedding layer</p> </li> <li> <code>pos_encoder</code>               (<code>Module</code>)           \u2013            <p>Positional Encoder</p> </li> <li> <code>encoder</code>               (<code>Module</code>)           \u2013            <p>Sequence of Transformer blocks</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models.rec import Transformer\n&gt;&gt;&gt; X = torch.cat((torch.zeros([5,1]), torch.empty(5, 4).random_(1,4)), axis=1)\n&gt;&gt;&gt; model = Transformer(vocab_size=4, seq_length=5, input_dim=8, n_heads=1, n_blocks=1)\n&gt;&gt;&gt; out = model(X)\n</code></pre> Source code in <code>pytorch_widedeep/models/rec/basic_transformer.py</code> <pre><code>@alias(\"input_dim\", [\"embed_dim\", \"d_model\"])\n@alias(\"seq_length\", [\"max_length\", \"maxlen\"])\ndef __init__(\n    self,\n    vocab_size: int,\n    seq_length: int,\n    input_dim: int,\n    n_heads: int,\n    n_blocks: int,\n    attn_dropout: float = 0.1,\n    ff_dropout: float = 0.1,\n    ff_factor: int = 4,\n    activation: str = \"gelu\",\n    use_linear_attention: bool = False,\n    use_flash_attention: bool = False,\n    padding_idx: int = 0,\n    with_cls_token: bool = False,\n    *,  # from here on pos encoding args\n    with_pos_encoding: bool = True,\n    pos_encoding_dropout: float = 0.1,\n    pos_encoder: Optional[nn.Module] = None,\n):\n    super().__init__()\n\n    self.input_dim = input_dim\n    self.seq_length = seq_length\n    self.n_heads = n_heads\n    self.n_blocks = n_blocks\n    self.attn_dropout = attn_dropout\n    self.ff_dropout = ff_dropout\n    self.ff_factor = ff_factor\n    self.activation = activation\n    self.use_linear_attention = use_linear_attention\n    self.use_flash_attention = use_flash_attention\n    self.padding_idx = padding_idx\n    self.with_cls_token = with_cls_token\n    self.with_pos_encoding = with_pos_encoding\n    self.pos_encoding_dropout = pos_encoding_dropout\n\n    self.embedding = nn.Embedding(\n        vocab_size, input_dim, padding_idx=self.padding_idx\n    )\n\n    if with_pos_encoding:\n        if pos_encoder is not None:\n            self.pos_encoder: Union[nn.Module, nn.Identity, PositionalEncoding] = (\n                pos_encoder\n            )\n        else:\n            self.pos_encoder = PositionalEncoding(\n                input_dim, pos_encoding_dropout, seq_length\n            )\n    else:\n        self.pos_encoder = nn.Identity()\n\n    self.encoder = nn.Sequential()\n    for i in range(n_blocks):\n        self.encoder.add_module(\n            \"transformer_block\" + str(i),\n            TransformerEncoder(\n                input_dim,\n                n_heads,\n                False,  # use_qkv_bias\n                attn_dropout,\n                ff_dropout,\n                ff_factor,\n                activation,\n                use_linear_attention,\n                use_flash_attention,\n            ),\n        )\n</code></pre>"},{"location":"pytorch-widedeep/trainer.html","title":"Training multimodal Deep Learning Models","text":"<p>Here is the documentation for the <code>Trainer</code> class, that will do all the heavy lifting.</p> <p>Trainer is also available from <code>pytorch-widedeep</code> directly, for example, one could do:</p> <pre><code>    from pytorch-widedeep.training import Trainer\n</code></pre> <p>or also:</p> <pre><code>    from pytorch-widedeep import Trainer\n</code></pre>"},{"location":"pytorch-widedeep/trainer.html#pytorch_widedeep.training.Trainer","title":"Trainer","text":"<pre><code>Trainer(\n    model,\n    objective,\n    custom_loss_function=None,\n    optimizers=None,\n    lr_schedulers=None,\n    initializers=None,\n    transforms=None,\n    callbacks=None,\n    metrics=None,\n    eval_metrics=None,\n    verbose=1,\n    seed=1,\n    **kwargs\n)\n</code></pre> <p>               Bases: <code>BaseTrainer</code></p> <p>Class to set the of attributes that will be used during the training process.</p> <p>Parameters:</p> <ul> <li> <code>model</code>               (<code>WideDeep</code>)           \u2013            <p>An object of class <code>WideDeep</code></p> </li> <li> <code>objective</code>               (<code>str</code>)           \u2013            <p>Defines the objective, loss or cost function. </p> <p>Param aliases: <code>loss_function</code>, <code>loss_fn</code>, <code>loss</code>, <code>cost_function</code>, <code>cost_fn</code>, <code>cost</code>. </p> <p>Possible values are:</p> <ul> <li> <p><code>binary</code>, aliases: <code>logistic</code>, <code>binary_logloss</code>, <code>binary_cross_entropy</code></p> </li> <li> <p><code>binary_focal_loss</code></p> </li> <li> <p><code>multiclass</code>, aliases: <code>multi_logloss</code>, <code>cross_entropy</code>, <code>categorical_cross_entropy</code>,</p> </li> <li> <p><code>multiclass_focal_loss</code></p> </li> <li> <p><code>regression</code>, aliases: <code>mse</code>, <code>l2</code>, <code>mean_squared_error</code></p> </li> <li> <p><code>mean_absolute_error</code>, aliases: <code>mae</code>, <code>l1</code></p> </li> <li> <p><code>mean_squared_log_error</code>, aliases: <code>msle</code></p> </li> <li> <p><code>root_mean_squared_error</code>, aliases:  <code>rmse</code></p> </li> <li> <p><code>root_mean_squared_log_error</code>, aliases: <code>rmsle</code></p> </li> <li> <p><code>zero_inflated_lognormal</code>, aliases: <code>ziln</code></p> </li> <li> <p><code>quantile</code></p> </li> <li> <p><code>tweedie</code></p> </li> <li> <p><code>multitarget</code>, aliases: <code>multi_target</code></p> </li> </ul> <p>NOTE: For <code>multitarget</code> a custom loss function must be passed</p> </li> <li> <code>custom_loss_function</code>               (<code>Optional[Module]</code>, default:                   <code>None</code> )           \u2013            <p>It is possible to pass a custom loss function. See for example <code>pytorch_widedeep.losses.FocalLoss</code> for the required structure of the object or the Examples section in this documentation or in the repo. Note that if <code>custom_loss_function</code> is not <code>None</code>, <code>objective</code> must be 'binary', 'multiclass' or 'regression', consistent with the loss function</p> </li> <li> <code>optimizers</code>               (<code>Optional[Union[Optimizer, Dict[str, Union[Optimizer, List[Optimizer]]]]]</code>, default:                   <code>None</code> )           \u2013            <ul> <li>An instance of Pytorch's <code>Optimizer</code> object   (e.g. <code>torch.optim.Adam()</code>) or</li> <li>a dictionary where there keys are the model components (i.e.   'wide', 'deeptabular', 'deeptext', 'deepimage'   and/or 'deephead')  and the values are the corresponding   optimizers or list of optimizers if multiple models are used for   the given data mode (e.g. two text columns/models for the deeptext   component). If multiple optimizers are used the   dictionary MUST contain an optimizer per model component.</li> </ul> <p>if no optimizers are passed it will default to <code>Adam</code> for all model components</p> </li> <li> <code>lr_schedulers</code>               (<code>Optional[Union[LRScheduler, Dict[str, Union[LRScheduler, List[LRScheduler]]]]]</code>, default:                   <code>None</code> )           \u2013            <ul> <li>An instance of Pytorch's <code>LRScheduler</code> object (e.g   <code>torch.optim.lr_scheduler.StepLR(opt, step_size=5)</code>) or</li> <li>a dictionary where there keys are the model componenst (i.e. 'wide',   'deeptabular', 'deeptext', 'deepimage' and/or 'deephead') and the   values are the corresponding learning rate schedulers or list of     learning rate schedulers if multiple models are used for the given     data mode (e.g. two text columns/models for the deeptext component).</li> </ul> </li> <li> <code>initializers</code>               (<code>Optional[Union[Initializer, Dict[str, Union[Initializer, List[Initializer]]]]]</code>, default:                   <code>None</code> )           \u2013            <ul> <li>An instance of an <code>Initializer</code> object see <code>pytorch-widedeep.initializers</code> or</li> <li>a dictionary where there keys are the model components (i.e. 'wide',   'deeptabular', 'deeptext', 'deepimage' and/or 'deephead')   and the values are the corresponding initializers or list of     initializers if multiple models are used for the given data mode (e.g.     two text columns/models for the deeptext component).</li> </ul> </li> <li> <code>transforms</code>               (<code>Optional[List[Transforms]]</code>, default:                   <code>None</code> )           \u2013            <p>List with <code>torchvision.transforms</code> to be applied to the image component of the model (i.e. <code>deepimage</code>) See torchvision transforms.</p> </li> <li> <code>callbacks</code>               (<code>Optional[List[Callback]]</code>, default:                   <code>None</code> )           \u2013            <p>List with <code>Callback</code> objects. The three callbacks available in <code>pytorch-widedeep</code> are: <code>LRHistory</code>, <code>ModelCheckpoint</code> and <code>EarlyStopping</code>. The <code>History</code> and the <code>LRShedulerCallback</code> callbacks are used by default. This can also be a custom callback as long as the object of type <code>Callback</code>. See <code>pytorch_widedeep.callbacks.Callback</code> or the examples folder in the repo.</p> </li> <li> <code>metrics</code>               (<code>Optional[Union[List[Metric], List[Metric]]]</code>, default:                   <code>None</code> )           \u2013            <ul> <li>List of objects of type <code>Metric</code>. Metrics available are:   <code>Accuracy</code>, <code>Precision</code>, <code>Recall</code>, <code>FBetaScore</code>,   <code>F1Score</code> and <code>R2Score</code>. This can also be a custom metric as long   as it is an object of type <code>Metric</code>. See   <code>pytorch_widedeep.metrics.Metric</code> or the examples folder in the   repo</li> <li>List of objects of type <code>torchmetrics.Metric</code>. This can be any   metric from torchmetrics library   Examples.   This can also be a custom metric as long as   it is an object of type <code>Metric</code>. See   the instructions.</li> </ul> </li> <li> <code>verbose</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>Verbosity level. If set to 0 nothing will be printed during training</p> </li> <li> <code>seed</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>Random seed to be used internally for train/test split</p> </li> </ul> <p>Other Parameters:</p> <ul> <li> <code>**kwargs</code>           \u2013            <p>Other infrequently used arguments that can also be passed as kwargs are:</p> <ul> <li> <p>device: <code>str</code>     string indicating the device. One of 'cpu', 'gpu' or 'mps' if     run on a Mac with Apple silicon or AMD GPU(s)</p> </li> <li> <p>num_workers: <code>int</code>     number of workers to be used internally by the data loaders</p> </li> <li> <p>lambda_sparse: <code>float</code>     lambda sparse parameter in case the <code>deeptabular</code> component is <code>TabNet</code></p> </li> <li> <p>class_weight: <code>List[float]</code>     This is the <code>weight</code> or <code>pos_weight</code> parameter in     <code>CrossEntropyLoss</code> and <code>BCEWithLogitsLoss</code>, depending on whether</p> </li> <li>reducelronplateau_criterion: <code>str</code>     This sets the criterion that will be used by the lr scheduler to     take a step: One of 'loss' or 'metric'. The ReduceLROnPlateau     learning rate is a bit particular.</li> </ul> </li> </ul> <p>Attributes:</p> <ul> <li> <code>cyclic_lr</code>               (<code>bool</code>)           \u2013            <p>Attribute that indicates if any of the lr_schedulers is cyclic_lr (i.e. <code>CyclicLR</code> or <code>OneCycleLR</code>). See Pytorch schedulers.</p> </li> <li> <code>feature_importance</code>               (<code>dict</code>)           \u2013            <p>dict where the keys are the column names and the values are the corresponding feature importances. This attribute will only exist if the <code>deeptabular</code> component is a Tabnet model.</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from torchvision.transforms import ToTensor\n&gt;&gt;&gt;\n&gt;&gt;&gt; # wide deep imports\n&gt;&gt;&gt; from pytorch_widedeep.callbacks import EarlyStopping, LRHistory\n&gt;&gt;&gt; from pytorch_widedeep.initializers import KaimingNormal, KaimingUniform, Normal, Uniform\n&gt;&gt;&gt; from pytorch_widedeep.models import TabResnet, Vision, BasicRNN, Wide, WideDeep\n&gt;&gt;&gt; from pytorch_widedeep import Trainer\n&gt;&gt;&gt;\n&gt;&gt;&gt; embed_input = [(u, i, j) for u, i, j in zip([\"a\", \"b\", \"c\"][:4], [4] * 3, [8] * 3)]\n&gt;&gt;&gt; column_idx = {k: v for v, k in enumerate([\"a\", \"b\", \"c\"])}\n&gt;&gt;&gt; wide = Wide(10, 1)\n&gt;&gt;&gt;\n&gt;&gt;&gt; # build the model\n&gt;&gt;&gt; deeptabular = TabResnet(blocks_dims=[8, 4], column_idx=column_idx, cat_embed_input=embed_input)\n&gt;&gt;&gt; deeptext = BasicRNN(vocab_size=10, embed_dim=4, padding_idx=0)\n&gt;&gt;&gt; deepimage = Vision()\n&gt;&gt;&gt; model = WideDeep(wide=wide, deeptabular=deeptabular, deeptext=deeptext, deepimage=deepimage)\n&gt;&gt;&gt;\n&gt;&gt;&gt; # set optimizers and schedulers\n&gt;&gt;&gt; wide_opt = torch.optim.Adam(model.wide.parameters())\n&gt;&gt;&gt; deep_opt = torch.optim.AdamW(model.deeptabular.parameters())\n&gt;&gt;&gt; text_opt = torch.optim.Adam(model.deeptext.parameters())\n&gt;&gt;&gt; img_opt = torch.optim.AdamW(model.deepimage.parameters())\n&gt;&gt;&gt;\n&gt;&gt;&gt; wide_sch = torch.optim.lr_scheduler.StepLR(wide_opt, step_size=5)\n&gt;&gt;&gt; deep_sch = torch.optim.lr_scheduler.StepLR(deep_opt, step_size=3)\n&gt;&gt;&gt; text_sch = torch.optim.lr_scheduler.StepLR(text_opt, step_size=5)\n&gt;&gt;&gt; img_sch = torch.optim.lr_scheduler.StepLR(img_opt, step_size=3)\n&gt;&gt;&gt;\n&gt;&gt;&gt; optimizers = {\"wide\": wide_opt, \"deeptabular\": deep_opt, \"deeptext\": text_opt, \"deepimage\": img_opt}\n&gt;&gt;&gt; schedulers = {\"wide\": wide_sch, \"deeptabular\": deep_sch, \"deeptext\": text_sch, \"deepimage\": img_sch}\n&gt;&gt;&gt;\n&gt;&gt;&gt; # set initializers and callbacks\n&gt;&gt;&gt; initializers = {\"wide\": Uniform, \"deeptabular\": Normal, \"deeptext\": KaimingNormal, \"deepimage\": KaimingUniform}\n&gt;&gt;&gt; transforms = [ToTensor]\n&gt;&gt;&gt; callbacks = [LRHistory(n_epochs=4), EarlyStopping]\n&gt;&gt;&gt;\n&gt;&gt;&gt; # set the trainer\n&gt;&gt;&gt; trainer = Trainer(model, objective=\"regression\", initializers=initializers, optimizers=optimizers,\n... lr_schedulers=schedulers, callbacks=callbacks, transforms=transforms)\n</code></pre> Source code in <code>pytorch_widedeep/training/trainer.py</code> <pre><code>@alias(  # noqa: C901\n    \"objective\",\n    [\"loss_function\", \"loss_fn\", \"loss\", \"cost_function\", \"cost_fn\", \"cost\"],\n)\n@alias(\"metrics\", [\"train_metrics\"])\ndef __init__(\n    self,\n    model: WideDeep,\n    objective: str,\n    custom_loss_function: Optional[nn.Module] = None,\n    optimizers: Optional[\n        Union[Optimizer, Dict[str, Union[Optimizer, List[Optimizer]]]]\n    ] = None,\n    lr_schedulers: Optional[\n        Union[LRScheduler, Dict[str, Union[LRScheduler, List[LRScheduler]]]]\n    ] = None,\n    initializers: Optional[\n        Union[Initializer, Dict[str, Union[Initializer, List[Initializer]]]]\n    ] = None,\n    transforms: Optional[List[Transforms]] = None,\n    callbacks: Optional[List[Callback]] = None,\n    metrics: Optional[Union[List[Metric], List[TorchMetric]]] = None,\n    eval_metrics: Optional[Union[List[Metric], List[TorchMetric]]] = None,\n    verbose: int = 1,\n    seed: int = 1,\n    **kwargs,\n):\n    super().__init__(\n        model=model,\n        objective=objective,\n        custom_loss_function=custom_loss_function,\n        optimizers=optimizers,\n        lr_schedulers=lr_schedulers,\n        initializers=initializers,\n        transforms=transforms,\n        callbacks=callbacks,\n        metrics=metrics,\n        eval_metrics=eval_metrics,\n        verbose=verbose,\n        seed=seed,\n        **kwargs,\n    )\n</code></pre>"},{"location":"pytorch-widedeep/trainer.html#pytorch_widedeep.training.Trainer.fit","title":"fit","text":"<pre><code>fit(\n    X_wide=None,\n    X_tab=None,\n    X_text=None,\n    X_img=None,\n    X_train=None,\n    X_val=None,\n    val_split=None,\n    target=None,\n    n_epochs=1,\n    validation_freq=1,\n    batch_size=32,\n    train_dataloader=None,\n    eval_dataloader=None,\n    feature_importance_sample_size=None,\n    finetune=False,\n    **kwargs\n)\n</code></pre> <p>Fit method.</p> <p>The input datasets can be passed either directly via numpy arrays (<code>X_wide</code>, <code>X_tab</code>, <code>X_text</code> or <code>X_img</code>) or alternatively, in dictionaries (<code>X_train</code> or <code>X_val</code>).</p> <p>Parameters:</p> <ul> <li> <code>X_wide</code>               (<code>Optional[ndarray]</code>, default:                   <code>None</code> )           \u2013            <p>Input for the <code>wide</code> model component. See <code>pytorch_widedeep.preprocessing.WidePreprocessor</code></p> </li> <li> <code>X_tab</code>               (<code>Optional[Union[ndarray, List[ndarray]]]</code>, default:                   <code>None</code> )           \u2013            <p>Input for the <code>deeptabular</code> model component. See <code>pytorch_widedeep.preprocessing.TabPreprocessor</code>. If multiple tabular models are used for different columns, this should be a list of numpy arrays</p> </li> <li> <code>X_text</code>               (<code>Optional[Union[ndarray, List[ndarray]]]</code>, default:                   <code>None</code> )           \u2013            <p>Input for the <code>deeptext</code> model component. See <code>pytorch_widedeep.preprocessing.TextPreprocessor</code>. If multiple text columns/models are used, this should be a list of numpy arrays</p> </li> <li> <code>X_img</code>               (<code>Optional[Union[ndarray, List[ndarray]]]</code>, default:                   <code>None</code> )           \u2013            <p>Input for the <code>deepimage</code> model component. See <code>pytorch_widedeep.preprocessing.ImagePreprocessor</code>. If multiple image columns/models are used, this should be a list of numpy arrays</p> </li> <li> <code>X_train</code>               (<code>Optional[Dict[str, Union[ndarray, List[ndarray]]]]</code>, default:                   <code>None</code> )           \u2013            <p>The training dataset can also be passed in a dictionary. Keys are 'X_wide', 'X_tab', 'X_text', 'X_img' and 'target'. Values are the corresponding matrices. Note that of multiple text or image columns/models are used, the corresponding values should be lists of numpy arrays</p> </li> <li> <code>X_val</code>               (<code>Optional[Dict[str, Union[ndarray, List[ndarray]]]]</code>, default:                   <code>None</code> )           \u2013            <p>The validation dataset can also be passed in a dictionary. Keys are 'X_wide', 'X_tab', 'X_text', 'X_img' and 'target'. Values are the corresponding matrices. Note that of multiple text or image columns/models are used, the corresponding values should be lists of numpy arrays</p> </li> <li> <code>val_split</code>               (<code>Optional[float]</code>, default:                   <code>None</code> )           \u2013            <p>train/val split fraction</p> </li> <li> <code>target</code>               (<code>Optional[ndarray]</code>, default:                   <code>None</code> )           \u2013            <p>target values</p> </li> <li> <code>n_epochs</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>number of epochs</p> </li> <li> <code>validation_freq</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>epochs validation frequency</p> </li> <li> <code>batch_size</code>               (<code>int</code>, default:                   <code>32</code> )           \u2013            <p>batch size</p> </li> <li> <code>custom_dataloader</code>           \u2013            <p>object of class <code>torch.utils.data.DataLoader</code>. Available predefined dataloaders are in <code>pytorch-widedeep.dataloaders</code>.If <code>None</code>, a standard torch <code>DataLoader</code> is used.</p> </li> <li> <code>finetune</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>fine-tune individual model components. This functionality can also be used to 'warm-up' (and hence the alias <code>warmup</code>) individual components before the joined training starts, and hence its alias. See the Examples folder in the repo for more details</p> <p><code>pytorch_widedeep</code> implements 3 fine-tune routines.</p> <ul> <li>fine-tune all trainable layers at once. This routine is   inspired by the work of Howard &amp; Sebastian Ruder 2018 in their   ULMfit paper. Using a   Slanted Triangular learing (see   Leslie N. Smith paper ) ,   the process is the following: i) the learning rate will   gradually increase for 10% of the training steps from max_lr/10   to max_lr. ii) It will then gradually decrease to max_lr/10   for the remaining 90% of the steps. The optimizer used in the   process is <code>Adam</code>.</li> </ul> <p>and two gradual fine-tune routines, where only certain layers are trained at a time.</p> <ul> <li>The so called <code>Felbo</code> gradual fine-tune rourine, based on the the   Felbo et al., 2017 DeepEmoji paper.</li> <li>The <code>Howard</code> routine based on the work of Howard &amp; Sebastian Ruder 2018 in their   ULMfit paper.</li> </ul> <p>For details on how these routines work, please see the Examples section in this documentation and the Examples folder in the repo.  Param Alias: <code>warmup</code></p> </li> </ul> <p>Other Parameters:</p> <ul> <li> <code>**kwargs</code>           \u2013            <p>Other keyword arguments are:</p> <ul> <li> <p>DataLoader related parameters:     For example,  <code>sampler</code>, <code>batch_sampler</code>, <code>collate_fn</code>, etc.     Please, see the pytorch     DataLoader docs     for details.</p> </li> <li> <p>Finetune related parameters:     see the source code at <code>pytorch_widedeep._finetune</code>. Namely, these are:</p> <ul> <li><code>finetune_epochs</code> (<code>int</code>):     number of epochs use for fine tuning</li> <li><code>finetune_max_lr</code> (<code>float</code>):    max lr during fine tuning</li> <li><code>routine</code> (<code>str</code>):    one of 'howard' or 'felbo'</li> <li><code>deeptabular_gradual</code> (<code>bool</code>):    boolean indicating if the <code>deeptabular</code> component will be fine tuned gradually</li> <li><code>deeptabular_layers</code> (<code>Optional[Union[List[nn.Module], List[List[nn.Module]]]]</code>):    List of pytorch modules indicating the layers of the    <code>deeptabular</code> that will be fine tuned</li> <li><code>deeptabular_max_lr</code> (<code>Union[float, List[float]]</code>):    max lr for the <code>deeptabular</code> componet during fine tuning</li> <li><code>deeptext_gradual</code> (<code>bool</code>):    same as <code>deeptabular_gradual</code> but for the <code>deeptext</code> component</li> <li><code>deeptext_layers</code> (<code>Optional[Union[List[nn.Module], List[List[nn.Module]]]]</code>):    same as <code>deeptabular_gradual</code> but for the <code>deeptext</code> component.    If there are multiple text columns/models, this should be a list of lists</li> <li><code>deeptext_max_lr</code> (<code>Union[float, List[float]]</code>):    same as <code>deeptabular_gradual</code> but for the <code>deeptext</code> component    If there are multiple text columns/models, this should be a list of floats</li> <li><code>deepimage_gradual</code> (<code>bool</code>):    same as <code>deeptext_layers</code> but for the <code>deepimage</code> component</li> <li><code>deepimage_layers</code> (<code>Optional[Union[List[nn.Module], List[List[nn.Module]]]]</code>):    same as <code>deeptext_layers</code> but for the <code>deepimage</code> component</li> <li><code>deepimage_max_lr</code> (<code>Union[float, List[float]]</code>):     same as <code>deeptext_layers</code> but for the <code>deepimage</code> component</li> </ul> </li> </ul> </li> </ul> <p>Examples:</p> <p>For a series of comprehensive examples on how to use the <code>fit</code> method, please see the Examples folder in the repo</p> Source code in <code>pytorch_widedeep/training/trainer.py</code> <pre><code>@alias(\"finetune\", [\"warmup\"])\n@alias(\"train_dataloader\", [\"custom_dataloader\"])\n@alias(\"eval_dataloader\", [\"custom_eval_dataloader\"])\ndef fit(  # noqa: C901\n    self,\n    X_wide: Optional[np.ndarray] = None,\n    X_tab: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n    X_text: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n    X_img: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n    X_train: Optional[Dict[str, Union[np.ndarray, List[np.ndarray]]]] = None,\n    X_val: Optional[Dict[str, Union[np.ndarray, List[np.ndarray]]]] = None,\n    val_split: Optional[float] = None,\n    target: Optional[np.ndarray] = None,\n    n_epochs: int = 1,\n    validation_freq: int = 1,\n    batch_size: int = 32,\n    train_dataloader: Optional[CustomDataLoader] = None,\n    eval_dataloader: Optional[CustomDataLoader] = None,\n    feature_importance_sample_size: Optional[int] = None,\n    finetune: bool = False,\n    **kwargs,\n):\n    r\"\"\"Fit method.\n\n    The input datasets can be passed either directly via numpy arrays\n    (`X_wide`, `X_tab`, `X_text` or `X_img`) or alternatively, in\n    dictionaries (`X_train` or `X_val`).\n\n    Parameters\n    ----------\n    X_wide: np.ndarray, Optional. default=None\n        Input for the `wide` model component.\n        See `pytorch_widedeep.preprocessing.WidePreprocessor`\n    X_tab: np.ndarray, Optional. default=None\n        Input for the `deeptabular` model component.\n        See `pytorch_widedeep.preprocessing.TabPreprocessor`. If multiple\n        tabular models are used for different columns, this should be a\n        list of numpy arrays\n    X_text: Union[np.ndarray, List[np.ndarray]], Optional. default=None\n        Input for the `deeptext` model component.\n        See `pytorch_widedeep.preprocessing.TextPreprocessor`.\n        If multiple text columns/models are used, this should be a list of\n        numpy arrays\n    X_img: np.ndarray, Optional. default=None\n        Input for the `deepimage` model component.\n        See `pytorch_widedeep.preprocessing.ImagePreprocessor`.\n        If multiple image columns/models are used, this should be a list of\n        numpy arrays\n    X_train: Dict, Optional. default=None\n        The training dataset can also be passed in a dictionary. Keys are\n        _'X_wide'_, _'X_tab'_, _'X_text'_, _'X_img'_ and _'target'_. Values\n        are the corresponding matrices. Note that of multiple text or image\n        columns/models are used, the corresponding values should be lists\n        of numpy arrays\n    X_val: Dict, Optional. default=None\n        The validation dataset can also be passed in a dictionary. Keys\n        are _'X_wide'_, _'X_tab'_, _'X_text'_, _'X_img'_ and _'target'_.\n        Values are the corresponding matrices. Note that of multiple text\n        or image columns/models are used, the corresponding values should\n        be lists of numpy arrays\n    val_split: float, Optional. default=None\n        train/val split fraction\n    target: np.ndarray, Optional. default=None\n        target values\n    n_epochs: int, default=1\n        number of epochs\n    validation_freq: int, default=1\n        epochs validation frequency\n    batch_size: int, default=32\n        batch size\n    custom_dataloader: `DataLoader`, Optional, default=None\n        object of class `torch.utils.data.DataLoader`. Available\n        predefined dataloaders are in `pytorch-widedeep.dataloaders`.If\n        `None`, a standard torch `DataLoader` is used.\n    finetune: bool, default=False\n        fine-tune individual model components. This functionality can also\n        be used to 'warm-up' (and hence the alias `warmup`) individual\n        components before the joined training starts, and hence its\n        alias. See the Examples folder in the repo for more details\n\n        `pytorch_widedeep` implements 3 fine-tune routines.\n\n        - fine-tune all trainable layers at once. This routine is\n          inspired by the work of Howard &amp; Sebastian Ruder 2018 in their\n          [ULMfit paper](https://arxiv.org/abs/1801.06146). Using a\n          Slanted Triangular learing (see\n          [Leslie N. Smith paper](https://arxiv.org/pdf/1506.01186.pdf) ) ,\n          the process is the following: *i*) the learning rate will\n          gradually increase for 10% of the training steps from max_lr/10\n          to max_lr. *ii*) It will then gradually decrease to max_lr/10\n          for the remaining 90% of the steps. The optimizer used in the\n          process is `Adam`.\n\n        and two gradual fine-tune routines, where only certain layers are\n        trained at a time.\n\n        - The so called `Felbo` gradual fine-tune rourine, based on the the\n          Felbo et al., 2017 [DeepEmoji paper](https://arxiv.org/abs/1708.00524).\n        - The `Howard` routine based on the work of Howard &amp; Sebastian Ruder 2018 in their\n          [ULMfit paper](https://arxiv.org/abs/1801.06146&gt;).\n\n        For details on how these routines work, please see the Examples\n        section in this documentation and the Examples folder in the repo. &lt;br/&gt;\n        Param Alias: `warmup`\n\n    Other Parameters\n    ----------------\n    **kwargs:\n        Other keyword arguments are:\n\n        - **DataLoader related parameters**:&lt;br/&gt;\n            For example,  `sampler`, `batch_sampler`, `collate_fn`, etc.\n            Please, see the pytorch\n            [DataLoader docs](https://pytorch.org/docs/stable/data.html#torch.utils.data.DataLoader)\n            for details.\n\n        - **Finetune related parameters**:&lt;br/&gt;\n            see the source code at `pytorch_widedeep._finetune`. Namely, these are:\n\n            - `finetune_epochs` (`int`):\n                number of epochs use for fine tuning\n            - `finetune_max_lr` (`float`):\n               max lr during fine tuning\n            - `routine` (`str`):\n               one of _'howard'_ or _'felbo'_\n            - `deeptabular_gradual` (`bool`):\n               boolean indicating if the `deeptabular` component will be fine tuned gradually\n            - `deeptabular_layers` (`Optional[Union[List[nn.Module], List[List[nn.Module]]]]`):\n               List of pytorch modules indicating the layers of the\n               `deeptabular` that will be fine tuned\n            - `deeptabular_max_lr` (`Union[float, List[float]]`):\n               max lr for the `deeptabular` componet during fine tuning\n            - `deeptext_gradual` (`bool`):\n               same as `deeptabular_gradual` but for the `deeptext` component\n            - `deeptext_layers` (`Optional[Union[List[nn.Module], List[List[nn.Module]]]]`):\n               same as `deeptabular_gradual` but for the `deeptext` component.\n               If there are multiple text columns/models, this should be a list of lists\n            - `deeptext_max_lr` (`Union[float, List[float]]`):\n               same as `deeptabular_gradual` but for the `deeptext` component\n               If there are multiple text columns/models, this should be a list of floats\n            - `deepimage_gradual` (`bool`):\n               same as `deeptext_layers` but for the `deepimage` component\n            - `deepimage_layers` (`Optional[Union[List[nn.Module], List[List[nn.Module]]]]`):\n               same as `deeptext_layers` but for the `deepimage` component\n            - `deepimage_max_lr` (`Union[float, List[float]]`):\n                same as `deeptext_layers` but for the `deepimage` component\n\n    Examples\n    --------\n\n    For a series of comprehensive examples on how to use the `fit` method, please see the\n    [Examples](https://github.com/jrzaurin/pytorch-widedeep/tree/master/examples)\n    folder in the repo\n    \"\"\"\n    dataloader_args, finetune_args = self._extract_kwargs(kwargs)\n    self.batch_size = batch_size\n    train_set, eval_set = wd_train_val_split(\n        self.seed,\n        self.method,  # type: ignore\n        X_wide,\n        X_tab,\n        X_text,\n        X_img,\n        X_train,\n        X_val,\n        val_split,\n        target,\n        self.transforms,\n    )\n    train_loader = self._set_dataloader(\n        train_dataloader, train_set, batch_size, dataloader_args\n    )\n    eval_loader = (\n        self._set_dataloader(eval_dataloader, eval_set, batch_size, dataloader_args)\n        if eval_set is not None\n        else None\n    )\n\n    if finetune:\n        self.with_finetuning: bool = True\n        self._do_finetune(train_loader, **finetune_args)\n        if self.verbose:\n            print(\n                \"Fine-tuning (or warmup) of individual components completed. \"\n                \"Training the whole model for {} epochs\".format(n_epochs)\n            )\n    else:\n        self.with_finetuning = False\n\n    self.callback_container.on_train_begin(\n        {\n            \"batch_size\": batch_size,\n            \"train_steps\": len(train_loader),\n            \"n_epochs\": n_epochs,\n        }\n    )\n    for epoch in range(n_epochs):\n        epoch_logs = self._train_epoch(train_loader, epoch)\n        if eval_loader is not None and epoch % validation_freq == (\n            validation_freq - 1\n        ):\n            epoch_logs, on_epoch_end_metric = self._eval_epoch(\n                eval_loader, epoch_logs\n            )\n        else:\n            on_epoch_end_metric = None\n            if self.reducelronplateau:\n                raise NotImplementedError(\n                    \"ReduceLROnPlateau scheduler can be used only with validation data.\"\n                )\n        self.callback_container.on_epoch_end(epoch, epoch_logs, on_epoch_end_metric)\n        if self.early_stop:\n            # self.callback_container.on_train_end(epoch_logs)\n            break\n\n    self.callback_container.on_train_end(epoch_logs)\n    if feature_importance_sample_size is not None:\n        self.feature_importance = FeatureImportance(\n            self.device, feature_importance_sample_size\n        ).feature_importance(train_loader, self.model)\n    self._restore_best_weights()\n    self.model.train()\n</code></pre>"},{"location":"pytorch-widedeep/trainer.html#pytorch_widedeep.training.Trainer.predict","title":"predict","text":"<pre><code>predict(\n    X_wide=None,\n    X_tab=None,\n    X_text=None,\n    X_img=None,\n    X_test=None,\n    batch_size=None,\n)\n</code></pre> <p>Returns the predictions</p> <p>The input datasets can be passed either directly via numpy arrays (<code>X_wide</code>, <code>X_tab</code>, <code>X_text</code> or <code>X_img</code>) or alternatively, in a dictionary (<code>X_test</code>)</p> <p>Parameters:</p> <ul> <li> <code>X_wide</code>               (<code>Optional[ndarray]</code>, default:                   <code>None</code> )           \u2013            <p>Input for the <code>wide</code> model component. See <code>pytorch_widedeep.preprocessing.WidePreprocessor</code></p> </li> <li> <code>X_tab</code>               (<code>Optional[Union[ndarray, List[ndarray]]]</code>, default:                   <code>None</code> )           \u2013            <p>Input for the <code>deeptabular</code> model component. See <code>pytorch_widedeep.preprocessing.TabPreprocessor</code></p> </li> <li> <code>X_text</code>               (<code>Optional[Union[ndarray, List[ndarray]]]</code>, default:                   <code>None</code> )           \u2013            <p>Input for the <code>deeptext</code> model component. See <code>pytorch_widedeep.preprocessing.TextPreprocessor</code></p> </li> <li> <code>X_img</code>               (<code>Optional[Union[ndarray, List[ndarray]]]</code>, default:                   <code>None</code> )           \u2013            <p>Input for the <code>deepimage</code> model component. See <code>pytorch_widedeep.preprocessing.ImagePreprocessor</code></p> </li> <li> <code>X_test</code>               (<code>Optional[Dict[str, Union[ndarray, List[ndarray]]]]</code>, default:                   <code>None</code> )           \u2013            <p>The test dataset can also be passed in a dictionary. Keys are <code>X_wide</code>, 'X_tab', 'X_text', 'X_img' and 'target'. Values are the corresponding matrices.</p> </li> <li> <code>batch_size</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>If a trainer is used to predict after having trained a model, the <code>batch_size</code> needs to be defined as it will not be defined as the <code>Trainer</code> is instantiated</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>np.ndarray:</code>           \u2013            <p>array with the predictions</p> </li> </ul> Source code in <code>pytorch_widedeep/training/trainer.py</code> <pre><code>def predict(  # type: ignore[override, return]\n    self,\n    X_wide: Optional[np.ndarray] = None,\n    X_tab: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n    X_text: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n    X_img: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n    X_test: Optional[Dict[str, Union[np.ndarray, List[np.ndarray]]]] = None,\n    batch_size: Optional[int] = None,\n) -&gt; np.ndarray:\n    r\"\"\"Returns the predictions\n\n    The input datasets can be passed either directly via numpy arrays\n    (`X_wide`, `X_tab`, `X_text` or `X_img`) or alternatively, in\n    a dictionary (`X_test`)\n\n\n    Parameters\n    ----------\n    X_wide: np.ndarray, Optional. default=None\n        Input for the `wide` model component.\n        See `pytorch_widedeep.preprocessing.WidePreprocessor`\n    X_tab: np.ndarray or List[np.ndarray], Optional. default=None\n        Input for the `deeptabular` model component.\n        See `pytorch_widedeep.preprocessing.TabPreprocessor`\n    X_text: np.ndarray or List[np.ndarray], Optional. default=None\n        Input for the `deeptext` model component.\n        See `pytorch_widedeep.preprocessing.TextPreprocessor`\n    X_img: np.ndarray or List[np.ndarray], Optional. default=None\n        Input for the `deepimage` model component.\n        See `pytorch_widedeep.preprocessing.ImagePreprocessor`\n    X_test: Dict, Optional. default=None\n        The test dataset can also be passed in a dictionary. Keys are\n        `X_wide`, _'X_tab'_, _'X_text'_, _'X_img'_ and _'target'_. Values\n        are the corresponding matrices.\n    batch_size: int, default = 256\n        If a trainer is used to predict after having trained a model, the\n        `batch_size` needs to be defined as it will not be defined as\n        the `Trainer` is instantiated\n\n    Returns\n    -------\n    np.ndarray:\n        array with the predictions\n    \"\"\"\n    preds_l = self._predict(X_wide, X_tab, X_text, X_img, X_test, batch_size)\n    if self.method == \"regression\":\n        return np.vstack(preds_l).squeeze(1)\n    if self.method == \"binary\":\n        preds = np.vstack(preds_l).squeeze(1)\n        return (preds &gt; 0.5).astype(\"int\")\n    if self.method == \"qregression\":\n        return np.vstack(preds_l)\n    if self.method == \"multiclass\":\n        preds = np.vstack(preds_l)\n        return np.argmax(preds, 1)  # type: ignore[return-value]\n    if self.method == \"multitarget\":\n        if self.loss_fn.__class__.__name__ in [\n            \"MultiTargetClassificationLoss\",\n            \"MutilTargetRegressionAndClassificationLoss\",\n        ]:\n            raise ValueError(\n                \"MultiTargetClassificationLoss and MutilTargetRegressionAndClassificationLoss \"\n                \"are not supported by predict method. Please use predict_proba method instead.\"\n            )\n        return np.vstack(preds_l)\n</code></pre>"},{"location":"pytorch-widedeep/trainer.html#pytorch_widedeep.training.Trainer.predict_uncertainty","title":"predict_uncertainty","text":"<pre><code>predict_uncertainty(\n    X_wide=None,\n    X_tab=None,\n    X_text=None,\n    X_img=None,\n    X_test=None,\n    batch_size=None,\n    uncertainty_granularity=1000,\n)\n</code></pre> <p>Returns the predicted ucnertainty of the model for the test dataset using a Monte Carlo method during which dropout layers are activated in the evaluation/prediction phase and each sample is predicted N times (<code>uncertainty_granularity</code> times).</p> <p>This is based on Dropout as a Bayesian Approximation: Representing Model Uncertainty in Deep Learning.</p> <p>Parameters:</p> <ul> <li> <code>X_wide</code>               (<code>Optional[ndarray]</code>, default:                   <code>None</code> )           \u2013            <p>Input for the <code>wide</code> model component. See <code>pytorch_widedeep.preprocessing.WidePreprocessor</code></p> </li> <li> <code>X_tab</code>               (<code>Optional[Union[ndarray, List[ndarray]]]</code>, default:                   <code>None</code> )           \u2013            <p>Input for the <code>deeptabular</code> model component. See <code>pytorch_widedeep.preprocessing.TabPreprocessor</code></p> </li> <li> <code>X_text</code>               (<code>Optional[Union[ndarray, List[ndarray]]]</code>, default:                   <code>None</code> )           \u2013            <p>Input for the <code>deeptext</code> model component. See <code>pytorch_widedeep.preprocessing.TextPreprocessor</code></p> </li> <li> <code>X_img</code>               (<code>Optional[Union[ndarray, List[ndarray]]]</code>, default:                   <code>None</code> )           \u2013            <p>Input for the <code>deepimage</code> model component. See <code>pytorch_widedeep.preprocessing.ImagePreprocessor</code></p> </li> <li> <code>X_test</code>               (<code>Optional[Dict[str, Union[ndarray, List[ndarray]]]]</code>, default:                   <code>None</code> )           \u2013            <p>The test dataset can also be passed in a dictionary. Keys are 'X_wide', 'X_tab', 'X_text', 'X_img' and 'target'. Values are the corresponding matrices.</p> </li> <li> <code>batch_size</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>If a trainer is used to predict after having trained a model, the <code>batch_size</code> needs to be defined as it will not be defined as the <code>Trainer</code> is instantiated</p> </li> <li> <code>uncertainty_granularity</code>               (<code>int</code>, default:                   <code>1000</code> )           \u2013            <p>number of times the model does prediction for each sample</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>np.ndarray:</code>           \u2013            <ul> <li>if <code>method = regression</code>, it will return an array with <code>(max, min, mean, stdev)</code>   values for each sample.</li> <li>if <code>method = binary</code> it will return an array with   <code>(mean_cls_0_prob, mean_cls_1_prob, predicted_cls)</code> for each sample.</li> <li>if <code>method = multiclass</code> it will return an array with   <code>(mean_cls_0_prob, mean_cls_1_prob, mean_cls_2_prob, ... , predicted_cls)</code>   values for each sample.</li> </ul> </li> </ul> Source code in <code>pytorch_widedeep/training/trainer.py</code> <pre><code>def predict_uncertainty(  # type: ignore[return]\n    self,\n    X_wide: Optional[np.ndarray] = None,\n    X_tab: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n    X_text: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n    X_img: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n    X_test: Optional[Dict[str, Union[np.ndarray, List[np.ndarray]]]] = None,\n    batch_size: Optional[int] = None,\n    uncertainty_granularity: int = 1000,\n) -&gt; np.ndarray:\n    r\"\"\"Returns the predicted ucnertainty of the model for the test dataset\n    using a Monte Carlo method during which dropout layers are activated\n    in the evaluation/prediction phase and each sample is predicted N\n    times (`uncertainty_granularity` times).\n\n    This is based on\n    [Dropout as a Bayesian Approximation: Representing\n    Model Uncertainty in Deep Learning](https://arxiv.org/abs/1506.02142?context=stat).\n\n    Parameters\n    ----------\n    X_wide: np.ndarray, Optional. default=None\n        Input for the `wide` model component.\n        See `pytorch_widedeep.preprocessing.WidePreprocessor`\n    X_tab: np.ndarray or List[np.ndarray], Optional. default=None\n        Input for the `deeptabular` model component.\n        See `pytorch_widedeep.preprocessing.TabPreprocessor`\n    X_text: np.ndarray or List[np.ndarray], Optional. default=None\n        Input for the `deeptext` model component.\n        See `pytorch_widedeep.preprocessing.TextPreprocessor`\n    X_img: np.ndarray or List[np.ndarray], Optional. default=None\n        Input for the `deepimage` model component.\n        See `pytorch_widedeep.preprocessing.ImagePreprocessor`\n    X_test: Dict, Optional. default=None\n        The test dataset can also be passed in a dictionary. Keys are\n        _'X_wide'_, _'X_tab'_, _'X_text'_, _'X_img'_ and _'target'_. Values\n        are the corresponding matrices.\n    batch_size: int, default = 256\n        If a trainer is used to predict after having trained a model, the\n        `batch_size` needs to be defined as it will not be defined as\n        the `Trainer` is instantiated\n    uncertainty_granularity: int default = 1000\n        number of times the model does prediction for each sample\n\n    Returns\n    -------\n    np.ndarray:\n        - if `method = regression`, it will return an array with `(max, min, mean, stdev)`\n          values for each sample.\n        - if `method = binary` it will return an array with\n          `(mean_cls_0_prob, mean_cls_1_prob, predicted_cls)` for each sample.\n        - if `method = multiclass` it will return an array with\n          `(mean_cls_0_prob, mean_cls_1_prob, mean_cls_2_prob, ... , predicted_cls)`\n          values for each sample.\n\n    \"\"\"\n    preds_l = self._predict(\n        X_wide,\n        X_tab,\n        X_text,\n        X_img,\n        X_test,\n        batch_size,\n        uncertainty_granularity,\n        uncertainty=True,\n    )\n    preds = np.vstack(preds_l)\n    samples_num = int(preds.shape[0] / uncertainty_granularity)\n    if self.method == \"regression\":\n        preds = preds.squeeze(1)\n        preds = preds.reshape((uncertainty_granularity, samples_num))\n        return np.array(\n            (\n                preds.max(axis=0),\n                preds.min(axis=0),\n                preds.mean(axis=0),\n                preds.std(axis=0),\n            )\n        ).T\n    if self.method == \"qregression\":\n        raise ValueError(\n            \"Currently predict_uncertainty is not supported for qregression method\"\n        )\n    if self.method == \"binary\":\n        preds = preds.squeeze(1)\n        preds = preds.reshape((uncertainty_granularity, samples_num))\n        preds = preds.mean(axis=0)\n        probs = np.zeros([preds.shape[0], 3])\n        probs[:, 0] = 1 - preds\n        probs[:, 1] = preds\n        return probs\n    if self.method == \"multiclass\":\n        preds = preds.reshape(uncertainty_granularity, samples_num, preds.shape[1])\n        preds = preds.mean(axis=0)\n        preds = np.hstack((preds, np.vstack(np.argmax(preds, 1))))\n        return preds\n\n    if self.method == \"multitarget\":\n        raise ValueError(\n            \"Currently predict_uncertainty is not supported for multitarget method\"\n        )\n</code></pre>"},{"location":"pytorch-widedeep/trainer.html#pytorch_widedeep.training.Trainer.predict_proba","title":"predict_proba","text":"<pre><code>predict_proba(\n    X_wide=None,\n    X_tab=None,\n    X_text=None,\n    X_img=None,\n    X_test=None,\n    batch_size=None,\n)\n</code></pre> <p>Returns the predicted probabilities for the test dataset for  binary and multiclass methods</p> <p>The input datasets can be passed either directly via numpy arrays (<code>X_wide</code>, <code>X_tab</code>, <code>X_text</code> or <code>X_img</code>) or alternatively, in a dictionary (<code>X_test</code>)</p> <p>Parameters:</p> <ul> <li> <code>X_wide</code>               (<code>Optional[ndarray]</code>, default:                   <code>None</code> )           \u2013            <p>Input for the <code>wide</code> model component. See <code>pytorch_widedeep.preprocessing.WidePreprocessor</code></p> </li> <li> <code>X_tab</code>               (<code>Optional[Union[ndarray, List[ndarray]]]</code>, default:                   <code>None</code> )           \u2013            <p>Input for the <code>deeptabular</code> model component. See <code>pytorch_widedeep.preprocessing.TabPreprocessor</code></p> </li> <li> <code>X_text</code>               (<code>Optional[Union[ndarray, List[ndarray]]]</code>, default:                   <code>None</code> )           \u2013            <p>Input for the <code>deeptext</code> model component. See <code>pytorch_widedeep.preprocessing.TextPreprocessor</code></p> </li> <li> <code>X_img</code>               (<code>Optional[Union[ndarray, List[ndarray]]]</code>, default:                   <code>None</code> )           \u2013            <p>Input for the <code>deepimage</code> model component. See <code>pytorch_widedeep.preprocessing.ImagePreprocessor</code></p> </li> <li> <code>X_test</code>               (<code>Optional[Dict[str, Union[ndarray, List[ndarray]]]]</code>, default:                   <code>None</code> )           \u2013            <p>The test dataset can also be passed in a dictionary. Keys are <code>X_wide</code>, 'X_tab', 'X_text', 'X_img' and 'target'. Values are the corresponding matrices.</p> </li> <li> <code>batch_size</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>If a trainer is used to predict after having trained a model, the <code>batch_size</code> needs to be defined as it will not be defined as the <code>Trainer</code> is instantiated</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>ndarray</code>           \u2013            <p>array with the probabilities per class</p> </li> </ul> Source code in <code>pytorch_widedeep/training/trainer.py</code> <pre><code>def predict_proba(  # type: ignore[override, return]  # noqa: C901\n    self,\n    X_wide: Optional[np.ndarray] = None,\n    X_tab: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n    X_text: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n    X_img: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n    X_test: Optional[Dict[str, Union[np.ndarray, List[np.ndarray]]]] = None,\n    batch_size: Optional[int] = None,\n) -&gt; np.ndarray:\n    r\"\"\"Returns the predicted probabilities for the test dataset for  binary\n    and multiclass methods\n\n    The input datasets can be passed either directly via numpy arrays\n    (`X_wide`, `X_tab`, `X_text` or `X_img`) or alternatively, in\n    a dictionary (`X_test`)\n\n    Parameters\n    ----------\n    X_wide: np.ndarray, Optional. default=None\n        Input for the `wide` model component.\n        See `pytorch_widedeep.preprocessing.WidePreprocessor`\n    X_tab: np.ndarray or List[np.ndarray], Optional. default=None\n        Input for the `deeptabular` model component.\n        See `pytorch_widedeep.preprocessing.TabPreprocessor`\n    X_text: np.ndarray or List[np.ndarray], Optional. default=None\n        Input for the `deeptext` model component.\n        See `pytorch_widedeep.preprocessing.TextPreprocessor`\n    X_img: np.ndarray or List[np.ndarray], Optional. default=None\n        Input for the `deepimage` model component.\n        See `pytorch_widedeep.preprocessing.ImagePreprocessor`\n    X_test: Dict, Optional. default=None\n        The test dataset can also be passed in a dictionary. Keys are\n        `X_wide`, _'X_tab'_, _'X_text'_, _'X_img'_ and _'target'_. Values\n        are the corresponding matrices.\n    batch_size: int, default = 256\n        If a trainer is used to predict after having trained a model, the\n        `batch_size` needs to be defined as it will not be defined as\n        the `Trainer` is instantiated\n\n    Returns\n    -------\n    np.ndarray\n        array with the probabilities per class\n    \"\"\"\n\n    preds_l = self._predict(X_wide, X_tab, X_text, X_img, X_test, batch_size)\n    if self.method == \"binary\":\n        preds = np.vstack(preds_l).squeeze(1)\n        probs = np.zeros([preds.shape[0], 2])\n        probs[:, 0] = 1 - preds\n        probs[:, 1] = preds\n        return probs\n    if self.method == \"multiclass\":\n        return np.vstack(preds_l)\n    if self.method == \"multitarget\":\n        return np.vstack(preds_l)\n</code></pre>"},{"location":"pytorch-widedeep/trainer.html#pytorch_widedeep.training.Trainer.save","title":"save","text":"<pre><code>save(\n    path,\n    save_state_dict=False,\n    save_optimizer=False,\n    model_filename=\"wd_model.pt\",\n)\n</code></pre> <p>Saves the model, training and evaluation history, and the <code>feature_importance</code> attribute (if the <code>deeptabular</code> component is a Tabnet model) to disk</p> <p>The <code>Trainer</code> class is built so that it 'just' trains a model. With that in mind, all the torch related parameters (such as optimizers, learning rate schedulers, initializers, etc) have to be defined externally and then passed to the <code>Trainer</code>. As a result, the <code>Trainer</code> does not generate any attribute or additional data products that need to be saved other than the <code>model</code> object itself, which can be saved as any other torch model (e.g. <code>torch.save(model, path)</code>).</p> <p>The exception is Tabnet. If the <code>deeptabular</code> component is a Tabnet model, an attribute (a dict) called <code>feature_importance</code> will be created at the end of the training process. Therefore, a <code>save</code> method was created that will save the feature importance dictionary to a json file and, since we are here, the model weights, training history and learning rate history.</p> <p>Parameters:</p> <ul> <li> <code>path</code>               (<code>str</code>)           \u2013            <p>path to the directory where the model and the feature importance attribute will be saved.</p> </li> <li> <code>save_state_dict</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether to save directly the model (and optimizer) or the model's (and optimizer's) state dictionary</p> </li> <li> <code>save_optimizer</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether to save the optimizer</p> </li> <li> <code>model_filename</code>               (<code>str</code>, default:                   <code>'wd_model.pt'</code> )           \u2013            <p>filename where the model weights will be store</p> </li> </ul> Source code in <code>pytorch_widedeep/training/trainer.py</code> <pre><code>def save(\n    self,\n    path: str,\n    save_state_dict: bool = False,\n    save_optimizer: bool = False,\n    model_filename: str = \"wd_model.pt\",\n):\n    r\"\"\"Saves the model, training and evaluation history, and the\n    `feature_importance` attribute (if the `deeptabular` component is a\n    Tabnet model) to disk\n\n    The `Trainer` class is built so that it 'just' trains a model. With\n    that in mind, all the torch related parameters (such as optimizers,\n    learning rate schedulers, initializers, etc) have to be defined\n    externally and then passed to the `Trainer`. As a result, the\n    `Trainer` does not generate any attribute or additional data\n    products that need to be saved other than the `model` object itself,\n    which can be saved as any other torch model (e.g. `torch.save(model,\n    path)`).\n\n    The exception is Tabnet. If the `deeptabular` component is a Tabnet\n    model, an attribute (a dict) called `feature_importance` will be\n    created at the end of the training process. Therefore, a `save`\n    method was created that will save the feature importance dictionary\n    to a json file and, since we are here, the model weights, training\n    history and learning rate history.\n\n    Parameters\n    ----------\n    path: str\n        path to the directory where the model and the feature importance\n        attribute will be saved.\n    save_state_dict: bool, default = False\n        Boolean indicating whether to save directly the model\n        (and optimizer) or the model's (and optimizer's) state\n        dictionary\n    save_optimizer: bool, default = False\n        Boolean indicating whether to save the optimizer\n    model_filename: str, Optional, default = \"wd_model.pt\"\n        filename where the model weights will be store\n    \"\"\"\n\n    self._save_history(path)\n\n    self._save_model_and_optimizer(\n        path, save_state_dict, save_optimizer, model_filename\n    )\n\n    if self.model.is_tabnet:\n        with open(Path(path) / \"feature_importance.json\", \"w\") as fi:\n            json.dump(self.feature_importance, fi)\n</code></pre>"},{"location":"pytorch-widedeep/utils/index.html","title":"The <code>utils</code> module","text":"<p>These are a series of utilities that might be useful for a number of preprocessing tasks, even not directly related to <code>pytorch-widedeep</code>. All the classes and functions discussed here are available directly from the <code>utils</code> module. For example, the <code>LabelEncoder</code> within the <code>deeptabular_utils</code> submodule can be imported as:</p> <pre><code>from pytorch_widedeep.utils import LabelEncoder\n</code></pre> <p>These are classes and functions that are internally used in the library. We include them here in case the user finds them useful for other purposes.</p>"},{"location":"pytorch-widedeep/utils/deeptabular_utils.html","title":"deeptabular utils","text":""},{"location":"pytorch-widedeep/utils/deeptabular_utils.html#pytorch_widedeep.utils.deeptabular_utils.LabelEncoder","title":"LabelEncoder","text":"<pre><code>LabelEncoder(\n    columns_to_encode=None,\n    with_attention=False,\n    shared_embed=False,\n)\n</code></pre> <p>Label Encode categorical values for multiple columns at once</p> <p> NOTE: LabelEncoder reserves 0 for <code>unseen</code> new categories. This is convenient when defining the embedding layers, since we can just set padding idx to 0.</p> <p>Parameters:</p> <ul> <li> <code>columns_to_encode</code>               (<code>Optional[List[str]]</code>, default:                   <code>None</code> )           \u2013            <p>List of strings containing the names of the columns to encode. If <code>None</code> all columns of type <code>object</code> in the dataframe will be label encoded.</p> </li> <li> <code>with_attention</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating whether the preprocessed data will be passed to an attention-based model. Aliased as <code>for_transformer</code>.</p> </li> <li> <code>shared_embed</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating if the embeddings will be \"shared\" when using attention-based models. The idea behind <code>shared_embed</code> is described in the Appendix A in the TabTransformer paper: 'The goal of having column embedding is to enable the model to distinguish the classes in one column from those in the other columns'. In other words, the idea is to let the model learn which column is embedded at the time. See: <code>pytorch_widedeep.models.transformers._layers.SharedEmbeddings</code>.</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>encoding_dict</code>               (<code>Dict</code>)           \u2013            <p>Dictionary containing the encoding mappings in the format, e.g. :  <code>{'colname1': {'cat1': 1, 'cat2': 2, ...}, 'colname2': {'cat1': 1, 'cat2': 2, ...}, ...}</code></p> </li> <li> <code>inverse_encoding_dict</code>               (<code>Dict</code>)           \u2013            <p>Dictionary containing the inverse encoding mappings in the format, e.g. :  <code>{'colname1': {1: 'cat1', 2: 'cat2', ...}, 'colname2': {1: 'cat1', 2: 'cat2', ...}, ...}</code></p> </li> </ul> Source code in <code>pytorch_widedeep/utils/deeptabular_utils.py</code> <pre><code>@alias(\"with_attention\", [\"for_transformer\"])\ndef __init__(\n    self,\n    columns_to_encode: Optional[List[str]] = None,\n    with_attention: bool = False,\n    shared_embed: bool = False,\n):\n    self.columns_to_encode = columns_to_encode\n\n    self.shared_embed = shared_embed\n    self.with_attention = with_attention\n\n    self.reset_embed_idx = not self.with_attention or self.shared_embed\n</code></pre>"},{"location":"pytorch-widedeep/utils/deeptabular_utils.html#pytorch_widedeep.utils.deeptabular_utils.LabelEncoder.partial_fit","title":"partial_fit","text":"<pre><code>partial_fit(df)\n</code></pre> <p>Main method. Creates encoding attributes.</p> <p>Returns:</p> <ul> <li> <code>LabelEncoder</code>           \u2013            <p><code>LabelEncoder</code> fitted object</p> </li> </ul> Source code in <code>pytorch_widedeep/utils/deeptabular_utils.py</code> <pre><code>def partial_fit(self, df: pd.DataFrame) -&gt; \"LabelEncoder\":  # noqa: C901\n    \"\"\"Main method. Creates encoding attributes.\n\n    Returns\n    -------\n    LabelEncoder\n        `LabelEncoder` fitted object\n    \"\"\"\n    # here df is a chunk of the data. this is meant to be run when the\n    # data is large and we pass a chunk at a time. Therefore, we do not\n    # copy the input chunk as mutating a chunk is ok\n    if self.columns_to_encode is None:\n        self.columns_to_encode = list(df.select_dtypes(include=[\"object\"]).columns)\n    else:\n        # sanity check to make sure all categorical columns are in an adequate\n        # format\n        for col in self.columns_to_encode:\n            df[col] = df[col].astype(\"O\")\n\n    unique_column_vals: Dict[str, List[str]] = {}\n    for c in self.columns_to_encode:\n        unique_column_vals[c] = df[c].unique().tolist()\n\n    if not hasattr(self, \"encoding_dict\"):\n        # we run the method 'partial_fit' for the 1st time\n        self.encoding_dict: Dict[str, Dict[str, int]] = {}\n        if \"cls_token\" in unique_column_vals and self.shared_embed:\n            self.encoding_dict[\"cls_token\"] = {\"[CLS]\": 0}\n            del unique_column_vals[\"cls_token\"]\n\n        # leave 0 for padding/\"unseen\" categories. Also we need an\n        # attribute to keep track of the encoding in case we use\n        # attention and we do not re-start the index/counter\n        self.cum_idx: int = 1\n        for k, v in unique_column_vals.items():\n            self.encoding_dict[k] = {o: i + self.cum_idx for i, o in enumerate(v)}\n            self.cum_idx = 1 if self.reset_embed_idx else self.cum_idx + len(v)\n    else:\n        # the 'partial_fit' method has already run.\n        # \"cls_token\" will have been added already\n        if \"cls_token\" in unique_column_vals and self.shared_embed:\n            del unique_column_vals[\"cls_token\"]\n\n        # Classes in the new df/chunk of the dataset that have not been seen\n        # before\n        unseen_classes: Dict[str, List[str]] = {}\n        for c in self.columns_to_encode:\n            unseen_classes[c] = list(\n                np.setdiff1d(\n                    unique_column_vals[c], list(self.encoding_dict[c].keys())\n                )\n            )\n\n        # leave 0 for padding/\"unseen\" categories\n        for k, v in unique_column_vals.items():\n            # if we use attention we need to start encoding from the\n            # last 'overall' encoding index. Otherwise, we use the max\n            # encoding index per categorical col\n            _idx = (\n                max(self.encoding_dict[k].values()) + 1\n                if self.reset_embed_idx\n                else self.cum_idx\n            )\n            if len(unseen_classes[k]) != 0:\n                for i, o in enumerate(unseen_classes[k]):\n                    if o not in self.encoding_dict[k]:\n                        self.encoding_dict[k][o] = i + _idx\n                # if self.reset_embed_idx is True it will be 1 anyway\n                self.cum_idx = (\n                    1\n                    if self.reset_embed_idx\n                    else self.cum_idx + len(unseen_classes[k])\n                )\n\n    return self\n</code></pre>"},{"location":"pytorch-widedeep/utils/deeptabular_utils.html#pytorch_widedeep.utils.deeptabular_utils.LabelEncoder.fit","title":"fit","text":"<pre><code>fit(df)\n</code></pre> <p>Simply runs the <code>partial_fit</code> method when the data fits in memory</p> <p>Returns:</p> <ul> <li> <code>LabelEncoder</code>           \u2013            <p><code>LabelEncoder</code> fitted object</p> </li> </ul> Source code in <code>pytorch_widedeep/utils/deeptabular_utils.py</code> <pre><code>def fit(self, df: pd.DataFrame) -&gt; \"LabelEncoder\":\n    \"\"\"Simply runs the `partial_fit` method when the data fits in memory\n\n    Returns\n    -------\n    LabelEncoder\n        `LabelEncoder` fitted object\n    \"\"\"\n    # this is meant to be run when the data fits in memory and therefore,\n    # we do not want to mutate the original df, so we copy it\n    self.partial_fit(df.copy())\n\n    self.inverse_encoding_dict = self.create_inverse_encoding_dict()\n\n    return self\n</code></pre>"},{"location":"pytorch-widedeep/utils/deeptabular_utils.html#pytorch_widedeep.utils.deeptabular_utils.LabelEncoder.transform","title":"transform","text":"<pre><code>transform(df)\n</code></pre> <p>Label Encoded the categories in <code>columns_to_encode</code></p> <p>Returns:</p> <ul> <li> <code>DataFrame</code>           \u2013            <p>label-encoded dataframe</p> </li> </ul> Source code in <code>pytorch_widedeep/utils/deeptabular_utils.py</code> <pre><code>def transform(self, df: pd.DataFrame) -&gt; pd.DataFrame:\n    \"\"\"Label Encoded the categories in `columns_to_encode`\n\n    Returns\n    -------\n    pd.DataFrame\n        label-encoded dataframe\n    \"\"\"\n    try:\n        self.encoding_dict\n    except AttributeError:\n        raise NotFittedError(\n            \"This LabelEncoder instance is not fitted yet. \"\n            \"Call 'fit' with appropriate arguments before using this LabelEncoder.\"\n        )\n\n    df_inp = df.copy()\n    # sanity check to make sure all categorical columns are in an adequate\n    # format\n    for col in self.columns_to_encode:  # type: ignore\n        df_inp[col] = df_inp[col].astype(\"O\")\n\n    for k, v in self.encoding_dict.items():\n        df_inp[k] = df_inp[k].apply(lambda x: v[x] if x in v.keys() else 0)\n\n    return df_inp\n</code></pre>"},{"location":"pytorch-widedeep/utils/deeptabular_utils.html#pytorch_widedeep.utils.deeptabular_utils.LabelEncoder.fit_transform","title":"fit_transform","text":"<pre><code>fit_transform(df)\n</code></pre> <p>Combines <code>fit</code> and <code>transform</code></p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import pandas as pd\n&gt;&gt;&gt; from pytorch_widedeep.utils import LabelEncoder\n&gt;&gt;&gt; df = pd.DataFrame({'col1': [1,2,3], 'col2': ['me', 'you', 'him']})\n&gt;&gt;&gt; columns_to_encode = ['col2']\n&gt;&gt;&gt; encoder = LabelEncoder(columns_to_encode)\n&gt;&gt;&gt; encoder.fit_transform(df)\n   col1  col2\n0     1     1\n1     2     2\n2     3     3\n&gt;&gt;&gt; encoder.encoding_dict\n{'col2': {'me': 1, 'you': 2, 'him': 3}}\n</code></pre> <p>Returns:</p> <ul> <li> <code>DataFrame</code>           \u2013            <p>label-encoded dataframe</p> </li> </ul> Source code in <code>pytorch_widedeep/utils/deeptabular_utils.py</code> <pre><code>def fit_transform(self, df: pd.DataFrame) -&gt; pd.DataFrame:\n    \"\"\"Combines `fit` and `transform`\n\n    Examples\n    --------\n\n    &gt;&gt;&gt; import pandas as pd\n    &gt;&gt;&gt; from pytorch_widedeep.utils import LabelEncoder\n    &gt;&gt;&gt; df = pd.DataFrame({'col1': [1,2,3], 'col2': ['me', 'you', 'him']})\n    &gt;&gt;&gt; columns_to_encode = ['col2']\n    &gt;&gt;&gt; encoder = LabelEncoder(columns_to_encode)\n    &gt;&gt;&gt; encoder.fit_transform(df)\n       col1  col2\n    0     1     1\n    1     2     2\n    2     3     3\n    &gt;&gt;&gt; encoder.encoding_dict\n    {'col2': {'me': 1, 'you': 2, 'him': 3}}\n\n    Returns\n    -------\n    pd.DataFrame\n        label-encoded dataframe\n    \"\"\"\n    return self.fit(df).transform(df)\n</code></pre>"},{"location":"pytorch-widedeep/utils/deeptabular_utils.html#pytorch_widedeep.utils.deeptabular_utils.LabelEncoder.inverse_transform","title":"inverse_transform","text":"<pre><code>inverse_transform(df)\n</code></pre> <p>Returns the original categories</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import pandas as pd\n&gt;&gt;&gt; from pytorch_widedeep.utils import LabelEncoder\n&gt;&gt;&gt; df = pd.DataFrame({'col1': [1,2,3], 'col2': ['me', 'you', 'him']})\n&gt;&gt;&gt; columns_to_encode = ['col2']\n&gt;&gt;&gt; encoder = LabelEncoder(columns_to_encode)\n&gt;&gt;&gt; df_enc = encoder.fit_transform(df)\n&gt;&gt;&gt; encoder.inverse_transform(df_enc)\n   col1 col2\n0     1   me\n1     2  you\n2     3  him\n</code></pre> <p>Returns:</p> <ul> <li> <code>DataFrame</code>           \u2013            <p>DataFrame with original categories</p> </li> </ul> Source code in <code>pytorch_widedeep/utils/deeptabular_utils.py</code> <pre><code>def inverse_transform(self, df: pd.DataFrame) -&gt; pd.DataFrame:\n    \"\"\"Returns the original categories\n\n    Examples\n    --------\n\n    &gt;&gt;&gt; import pandas as pd\n    &gt;&gt;&gt; from pytorch_widedeep.utils import LabelEncoder\n    &gt;&gt;&gt; df = pd.DataFrame({'col1': [1,2,3], 'col2': ['me', 'you', 'him']})\n    &gt;&gt;&gt; columns_to_encode = ['col2']\n    &gt;&gt;&gt; encoder = LabelEncoder(columns_to_encode)\n    &gt;&gt;&gt; df_enc = encoder.fit_transform(df)\n    &gt;&gt;&gt; encoder.inverse_transform(df_enc)\n       col1 col2\n    0     1   me\n    1     2  you\n    2     3  him\n\n    Returns\n    -------\n    pd.DataFrame\n        DataFrame with original categories\n    \"\"\"\n\n    if not hasattr(self, \"inverse_encoding_dict\"):\n        self.inverse_encoding_dict = self.create_inverse_encoding_dict()\n\n    for k, v in self.inverse_encoding_dict.items():\n        df[k] = df[k].apply(lambda x: v[x])\n\n    return df\n</code></pre>"},{"location":"pytorch-widedeep/utils/fastai_transforms.html","title":"Fastai transforms","text":"<p>I directly copied and pasted part of the <code>transforms.py</code> module from the <code>fastai</code> library (from an old version). The reason to do such a thing is because <code>pytorch_widedeep</code> only needs the <code>Tokenizer</code> and the <code>Vocab</code> classes there. This way I avoid extra dependencies. Credit for all the code in the <code>fastai_transforms</code> module in this <code>pytorch-widedeep</code> package goes to Jeremy Howard and the <code>fastai</code> team. I only include the documentation here for completion, but I strongly advise the user to read the <code>fastai</code> documentation.</p>"},{"location":"pytorch-widedeep/utils/fastai_transforms.html#pytorch_widedeep.utils.fastai_transforms.Tokenizer","title":"Tokenizer","text":"<pre><code>Tokenizer(\n    tok_func=SpacyTokenizer,\n    lang=\"en\",\n    pre_rules=None,\n    post_rules=None,\n    special_cases=None,\n    n_cpus=None,\n)\n</code></pre> <p>Class to combine a series of rules and a tokenizer function to tokenize text with multiprocessing.</p> <p>Setting some of the parameters of this class require perhaps some familiarity with the source code.</p> <p>Parameters:</p> <ul> <li> <code>tok_func</code>               (<code>Callable</code>, default:                   <code>SpacyTokenizer</code> )           \u2013            <p>Tokenizer Object. See <code>pytorch_widedeep.utils.fastai_transforms.SpacyTokenizer</code></p> </li> <li> <code>lang</code>               (<code>str</code>, default:                   <code>'en'</code> )           \u2013            <p>Text's Language</p> </li> <li> <code>pre_rules</code>               (<code>Optional[ListRules]</code>, default:                   <code>None</code> )           \u2013            <p>Custom type: <code>Collection[Callable[[str], str]]</code>. These are <code>Callable</code> objects that will be applied to the text (str) directly as <code>rule(tok)</code> before being tokenized.</p> </li> <li> <code>post_rules</code>               (<code>Optional[ListRules]</code>, default:                   <code>None</code> )           \u2013            <p>Custom type: <code>Collection[Callable[[str], str]]</code>. These are <code>Callable</code> objects that will be applied to the tokens as <code>rule(tokens)</code> after the text has been tokenized.</p> </li> <li> <code>special_cases</code>               (<code>Optional[Collection[str]]</code>, default:                   <code>None</code> )           \u2013            <p>special cases to be added to the tokenizer via <code>Spacy</code>'s <code>add_special_case</code> method</p> </li> <li> <code>n_cpus</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>number of CPUs to used during the tokenization process</p> </li> </ul> Source code in <code>pytorch_widedeep/utils/fastai_transforms.py</code> <pre><code>def __init__(\n    self,\n    tok_func: Callable = SpacyTokenizer,\n    lang: str = \"en\",\n    pre_rules: Optional[ListRules] = None,\n    post_rules: Optional[ListRules] = None,\n    special_cases: Optional[Collection[str]] = None,\n    n_cpus: Optional[int] = None,\n):\n    self.tok_func, self.lang, self.special_cases = tok_func, lang, special_cases\n    self.pre_rules = ifnone(pre_rules, defaults.text_pre_rules)\n    self.post_rules = ifnone(post_rules, defaults.text_post_rules)\n    self.special_cases = (\n        special_cases if special_cases is not None else defaults.text_spec_tok\n    )\n    self.n_cpus = ifnone(n_cpus, defaults.cpus)\n</code></pre>"},{"location":"pytorch-widedeep/utils/fastai_transforms.html#pytorch_widedeep.utils.fastai_transforms.Tokenizer.process_text","title":"process_text","text":"<pre><code>process_text(t, tok)\n</code></pre> <p>Process and tokenize one text <code>t</code> with tokenizer <code>tok</code>.</p> <p>Parameters:</p> <ul> <li> <code>t</code>               (<code>str</code>)           \u2013            <p>text to be processed and tokenized</p> </li> <li> <code>tok</code>               (<code>BaseTokenizer</code>)           \u2013            <p>Instance of <code>BaseTokenizer</code>. See <code>pytorch_widedeep.utils.fastai_transforms.BaseTokenizer</code></p> </li> </ul> <p>Returns:</p> <ul> <li> <code>List[str]</code>           \u2013            <p>List of tokens</p> </li> </ul> Source code in <code>pytorch_widedeep/utils/fastai_transforms.py</code> <pre><code>def process_text(self, t: str, tok: BaseTokenizer) -&gt; List[str]:\n    r\"\"\"Process and tokenize one text ``t`` with tokenizer ``tok``.\n\n    Parameters\n    ----------\n    t: str\n        text to be processed and tokenized\n    tok: ``BaseTokenizer``\n        Instance of `BaseTokenizer`. See\n        `pytorch_widedeep.utils.fastai_transforms.BaseTokenizer`\n\n    Returns\n    -------\n    List[str]\n        List of tokens\n    \"\"\"\n    for rule in self.pre_rules:\n        t = rule(t)\n    toks = tok.tokenizer(t)\n    for rule in self.post_rules:\n        toks = rule(toks)\n    return toks\n</code></pre>"},{"location":"pytorch-widedeep/utils/fastai_transforms.html#pytorch_widedeep.utils.fastai_transforms.Tokenizer.process_all","title":"process_all","text":"<pre><code>process_all(texts)\n</code></pre> <p>Process a list of texts. Parallel execution of <code>process_text</code>.</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; from pytorch_widedeep.utils import Tokenizer\n&gt;&gt;&gt; texts = ['Machine learning is great', 'but building stuff is even better']\n&gt;&gt;&gt; tok = Tokenizer()\n&gt;&gt;&gt; tok.process_all(texts)\n[['xxmaj', 'machine', 'learning', 'is', 'great'], ['but', 'building', 'stuff', 'is', 'even', 'better']]\n</code></pre> <p> NOTE: Note the token <code>TK_MAJ</code> (<code>xxmaj</code>), used to indicate the next word begins with a capital in the original text. For more details of special tokens please see the <code>fastai</code> docs.</p> <p>Returns:</p> <ul> <li> <code>List[List[str]]</code>           \u2013            <p>List containing lists of tokens. One list per \"document\"</p> </li> </ul> Source code in <code>pytorch_widedeep/utils/fastai_transforms.py</code> <pre><code>def process_all(self, texts: Collection[str]) -&gt; List[List[str]]:\n    r\"\"\"Process a list of texts. Parallel execution of ``process_text``.\n\n    Examples\n    --------\n    &gt;&gt;&gt; from pytorch_widedeep.utils import Tokenizer\n    &gt;&gt;&gt; texts = ['Machine learning is great', 'but building stuff is even better']\n    &gt;&gt;&gt; tok = Tokenizer()\n    &gt;&gt;&gt; tok.process_all(texts)\n    [['xxmaj', 'machine', 'learning', 'is', 'great'], ['but', 'building', 'stuff', 'is', 'even', 'better']]\n\n    :information_source: **NOTE**:\n    Note the token ``TK_MAJ`` (`xxmaj`), used to indicate the\n    next word begins with a capital in the original text. For more\n    details of special tokens please see the [``fastai`` docs](https://docs.fast.ai/text.core.html#Tokenizing).\n\n    Returns\n    -------\n    List[List[str]]\n        List containing lists of tokens. One list per \"_document_\"\n\n    \"\"\"\n\n    if self.n_cpus &lt;= 1:\n        return self._process_all_1(texts)\n    with ProcessPoolExecutor(self.n_cpus) as e:\n        return sum(\n            e.map(self._process_all_1, partition_by_cores(texts, self.n_cpus)), []\n        )\n</code></pre>"},{"location":"pytorch-widedeep/utils/fastai_transforms.html#pytorch_widedeep.utils.fastai_transforms.Vocab","title":"Vocab","text":"<pre><code>Vocab(\n    max_vocab, min_freq, pad_idx=None, special_cases=None\n)\n</code></pre> <p>Contains the correspondence between numbers and tokens.</p> <p>Parameters:</p> <ul> <li> <code>max_vocab</code>               (<code>int</code>)           \u2013            <p>maximum vocabulary size</p> </li> <li> <code>min_freq</code>               (<code>int</code>)           \u2013            <p>minimum frequency for a token to be considereds</p> </li> <li> <code>pad_idx</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>padding index. If <code>None</code>, Fastai's Tokenizer leaves the 0 index for the unknown token ('xxunk') and defaults to 1 for the padding token ('xxpad').</p> </li> </ul> <p>Attributes:</p> <ul> <li> <code>itos</code>               (<code>Collection</code>)           \u2013            <p><code>index to str</code>. Collection of strings that are the tokens of the vocabulary</p> </li> <li> <code>stoi</code>               (<code>defaultdict</code>)           \u2013            <p><code>str to index</code>. Dictionary containing the tokens of the vocabulary and their corresponding index</p> </li> </ul> Source code in <code>pytorch_widedeep/utils/fastai_transforms.py</code> <pre><code>def __init__(\n    self,\n    max_vocab: int,\n    min_freq: int,\n    pad_idx: Optional[int] = None,\n    special_cases: Optional[Collection[str]] = None,\n):\n    self.max_vocab = max_vocab\n    self.min_freq = min_freq\n    self.pad_idx = pad_idx\n    self.special_cases = (\n        special_cases if special_cases is not None else defaults.text_spec_tok\n    )\n</code></pre>"},{"location":"pytorch-widedeep/utils/fastai_transforms.html#pytorch_widedeep.utils.fastai_transforms.Vocab.create","title":"create","text":"<pre><code>create(tokens)\n</code></pre> <p>Create a vocabulary object from a set of tokens.</p> <p>Parameters:</p> <ul> <li> <code>tokens</code>               (<code>Tokens</code>)           \u2013            <p>Custom type: <code>Collection[Collection[str]]</code>  see <code>pytorch_widedeep.wdtypes</code>. Collection of collection of strings (e.g. list of tokenized sentences)</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; from pytorch_widedeep.utils import Tokenizer, Vocab\n&gt;&gt;&gt; texts = ['Machine learning is great', 'but building stuff is even better']\n&gt;&gt;&gt; tokens = Tokenizer().process_all(texts)\n&gt;&gt;&gt; vocab = Vocab(max_vocab=18, min_freq=1).create(tokens)\n&gt;&gt;&gt; vocab.numericalize(['machine', 'learning', 'is', 'great'])\n[10, 11, 9, 12]\n&gt;&gt;&gt; vocab.textify([10, 11, 9, 12])\n'machine learning is great'\n</code></pre> <p> NOTE: Note the many special tokens that <code>fastai</code>'s' tokenizer adds. These are particularly useful when building Language models and/or in classification/Regression tasks. Please see the <code>fastai</code> docs.</p> <p>Returns:</p> <ul> <li> <code>Vocab</code>           \u2013            <p>An instance of a <code>Vocab</code> object</p> </li> </ul> Source code in <code>pytorch_widedeep/utils/fastai_transforms.py</code> <pre><code>def create(\n    self,\n    tokens: Tokens,\n) -&gt; \"Vocab\":\n    r\"\"\"Create a vocabulary object from a set of tokens.\n\n    Parameters\n    ----------\n    tokens: Tokens\n        Custom type: ``Collection[Collection[str]]``  see\n        `pytorch_widedeep.wdtypes`. Collection of collection of\n        strings (e.g. list of tokenized sentences)\n\n    Examples\n    --------\n    &gt;&gt;&gt; from pytorch_widedeep.utils import Tokenizer, Vocab\n    &gt;&gt;&gt; texts = ['Machine learning is great', 'but building stuff is even better']\n    &gt;&gt;&gt; tokens = Tokenizer().process_all(texts)\n    &gt;&gt;&gt; vocab = Vocab(max_vocab=18, min_freq=1).create(tokens)\n    &gt;&gt;&gt; vocab.numericalize(['machine', 'learning', 'is', 'great'])\n    [10, 11, 9, 12]\n    &gt;&gt;&gt; vocab.textify([10, 11, 9, 12])\n    'machine learning is great'\n\n    :information_source: **NOTE**:\n    Note the many special tokens that ``fastai``'s' tokenizer adds. These\n    are particularly useful when building Language models and/or in\n    classification/Regression tasks. Please see the [``fastai`` docs](https://docs.fast.ai/text.core.html#Tokenizing).\n\n    Returns\n    -------\n    Vocab\n        An instance of a `Vocab` object\n    \"\"\"\n\n    freq = Counter(p for o in tokens for p in o)\n    itos = [o for o, c in freq.most_common(self.max_vocab) if c &gt;= self.min_freq]\n    for o in reversed(self.special_cases):\n        if o in itos:\n            itos.remove(o)\n        itos.insert(0, o)\n\n    if self.pad_idx is not None and self.pad_idx != 1:\n        itos.remove(PAD)\n        itos.insert(self.pad_idx, PAD)\n        # get the new 'xxunk' index\n        xxunk_idx = np.where([el == \"xxunk\" for el in itos])[0][0]\n    else:\n        xxunk_idx = 0\n\n    itos = itos[: self.max_vocab]\n    if (\n        len(itos) &lt; self.max_vocab\n    ):  # Make sure vocab size is a multiple of 8 for fast mixed precision training\n        while len(itos) % 8 != 0:\n            itos.append(\"xxfake\")\n\n    self.itos = itos\n    self.stoi = defaultdict(\n        lambda: xxunk_idx, {v: k for k, v in enumerate(self.itos)}\n    )\n\n    return self\n</code></pre>"},{"location":"pytorch-widedeep/utils/fastai_transforms.html#pytorch_widedeep.utils.fastai_transforms.Vocab.fit","title":"fit","text":"<pre><code>fit(tokens)\n</code></pre> <p>Calls the <code>create</code> method. I simply want to honor fast ai naming, but for consistency with the rest of the library I am including a fit method</p> Source code in <code>pytorch_widedeep/utils/fastai_transforms.py</code> <pre><code>def fit(\n    self,\n    tokens: Tokens,\n) -&gt; \"Vocab\":\n    \"\"\"\n    Calls the `create` method. I simply want to honor fast ai naming, but\n    for consistency with the rest of the library I am including a fit method\n    \"\"\"\n    return self.create(tokens)\n</code></pre>"},{"location":"pytorch-widedeep/utils/fastai_transforms.html#pytorch_widedeep.utils.fastai_transforms.Vocab.numericalize","title":"numericalize","text":"<pre><code>numericalize(t)\n</code></pre> <p>Convert a list of tokens <code>t</code> to their ids.</p> <p>Returns:</p> <ul> <li> <code>List[int]</code>           \u2013            <p>List of 'numericalsed' tokens</p> </li> </ul> Source code in <code>pytorch_widedeep/utils/fastai_transforms.py</code> <pre><code>def numericalize(self, t: Collection[str]) -&gt; List[int]:\n    \"\"\"Convert a list of tokens ``t`` to their ids.\n\n    Returns\n    -------\n    List[int]\n        List of '_numericalsed_' tokens\n    \"\"\"\n    return [self.stoi[w] for w in t]\n</code></pre>"},{"location":"pytorch-widedeep/utils/fastai_transforms.html#pytorch_widedeep.utils.fastai_transforms.Vocab.transform","title":"transform","text":"<pre><code>transform(t)\n</code></pre> <p>Calls the <code>numericalize</code> method. I simply want to honor fast ai naming, but for consistency with the rest of the library I am including a transform method</p> Source code in <code>pytorch_widedeep/utils/fastai_transforms.py</code> <pre><code>def transform(self, t: Collection[str]) -&gt; List[int]:\n    \"\"\"\n    Calls the `numericalize` method. I simply want to honor fast ai naming,\n    but for consistency with the rest of the library I am including a\n    transform method\n    \"\"\"\n    return self.numericalize(t)\n</code></pre>"},{"location":"pytorch-widedeep/utils/fastai_transforms.html#pytorch_widedeep.utils.fastai_transforms.Vocab.textify","title":"textify","text":"<pre><code>textify(nums, sep=' ')\n</code></pre> <p>Convert a list of <code>nums</code> (or indexes) to their tokens.</p> <p>Returns:</p> <ul> <li> <code>List[str]</code>           \u2013            <p>List of tokens</p> </li> </ul> Source code in <code>pytorch_widedeep/utils/fastai_transforms.py</code> <pre><code>def textify(self, nums: Collection[int], sep=\" \") -&gt; Union[str, List[str]]:\n    \"\"\"Convert a list of ``nums`` (or indexes) to their tokens.\n\n    Returns\n    -------\n    List[str]\n        List of tokens\n    \"\"\"\n    return (\n        sep.join([self.itos[i] for i in nums])\n        if sep is not None\n        else [self.itos[i] for i in nums]\n    )\n</code></pre>"},{"location":"pytorch-widedeep/utils/fastai_transforms.html#pytorch_widedeep.utils.fastai_transforms.Vocab.inverse_transform","title":"inverse_transform","text":"<pre><code>inverse_transform(nums, sep=' ')\n</code></pre> <p>Calls the <code>textify</code> method. I simply want to honor fast ai naming, but for consistency with the rest of the library I am including an inverse_transform method</p> Source code in <code>pytorch_widedeep/utils/fastai_transforms.py</code> <pre><code>def inverse_transform(\n    self, nums: Collection[int], sep=\" \"\n) -&gt; Union[str, List[str]]:\n    \"\"\"\n    Calls the `textify` method. I simply want to honor fast ai naming, but\n    for consistency with the rest of the library I am including an\n    inverse_transform method\n    \"\"\"\n    # I simply want to honor fast ai naming, but for consistency with the\n    # rest of the library I am including an inverse_transform method\n    return self.textify(nums, sep)\n</code></pre>"},{"location":"pytorch-widedeep/utils/image_utils.html","title":"Image utils","text":"<p>SimplePreprocessor and AspectAwarePreprocessor are directly taked from the great series of Books `Deep Learning for Computer Vision by Adrian. Therefore, all credit for the code in the <code>image_utils</code> module goes to Adrian Rosebrock.</p>"},{"location":"pytorch-widedeep/utils/image_utils.html#pytorch_widedeep.utils.image_utils.AspectAwarePreprocessor","title":"AspectAwarePreprocessor","text":"<pre><code>AspectAwarePreprocessor(\n    width, height, inter=cv2.INTER_AREA\n)\n</code></pre> <p>Class to resize an image to a certain width and height taking into account the image aspect ratio</p> <p>Parameters:</p> <ul> <li> <code>width</code>               (<code>int</code>)           \u2013            <p>output width</p> </li> <li> <code>height</code>               (<code>int</code>)           \u2013            <p>output height</p> </li> <li> <code>inter</code>           \u2013            <p><code>opencv</code> interpolation method. See <code>opencv</code> <code>InterpolationFlags</code>.</p> </li> </ul> Source code in <code>pytorch_widedeep/utils/image_utils.py</code> <pre><code>def __init__(self, width: int, height: int, inter=cv2.INTER_AREA):\n    self.width = width\n    self.height = height\n    self.inter = inter\n</code></pre>"},{"location":"pytorch-widedeep/utils/image_utils.html#pytorch_widedeep.utils.image_utils.AspectAwarePreprocessor.preprocess","title":"preprocess","text":"<pre><code>preprocess(image)\n</code></pre> <p>Returns the resized input image taking into account the image aspect ratio</p> <p>Parameters:</p> <ul> <li> <code>image</code>               (<code>ndarray</code>)           \u2013            <p>Input image to be resized</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import cv2\n&gt;&gt;&gt; from pytorch_widedeep.utils import AspectAwarePreprocessor\n&gt;&gt;&gt; img = cv2.imread(\"tests/test_data_utils/images/galaxy1.png\")\n&gt;&gt;&gt; img.shape\n(694, 890, 3)\n&gt;&gt;&gt; app = AspectAwarePreprocessor(width=224, height=224)\n&gt;&gt;&gt; resized_img = app.preprocess(img)\n&gt;&gt;&gt; resized_img.shape\n(224, 224, 3)\n</code></pre> <p>Returns:</p> <ul> <li> <code>ndarray</code>           \u2013            <p>Resized image according to its original image aspect ratio</p> </li> </ul> Source code in <code>pytorch_widedeep/utils/image_utils.py</code> <pre><code>def preprocess(self, image: np.ndarray) -&gt; np.ndarray:\n    r\"\"\"Returns the resized input image taking into account the image aspect ratio\n\n    Parameters\n    ----------\n    image: np.ndarray\n        Input image to be resized\n\n    Examples\n    --------\n    &gt;&gt;&gt; import cv2\n    &gt;&gt;&gt; from pytorch_widedeep.utils import AspectAwarePreprocessor\n    &gt;&gt;&gt; img = cv2.imread(\"tests/test_data_utils/images/galaxy1.png\")\n    &gt;&gt;&gt; img.shape\n    (694, 890, 3)\n    &gt;&gt;&gt; app = AspectAwarePreprocessor(width=224, height=224)\n    &gt;&gt;&gt; resized_img = app.preprocess(img)\n    &gt;&gt;&gt; resized_img.shape\n    (224, 224, 3)\n\n    Returns\n    -------\n    np.ndarray\n        Resized image according to its original image aspect ratio\n    \"\"\"\n    (h, w) = image.shape[:2]\n    dW = 0\n    dH = 0\n\n    if w &lt; h:\n        image = imutils.resize(image, width=self.width, inter=self.inter)\n        dH = int((image.shape[0] - self.height) / 2.0)\n    else:\n        image = imutils.resize(image, height=self.height, inter=self.inter)\n        dW = int((image.shape[1] - self.width) / 2.0)\n\n    (h, w) = image.shape[:2]\n    image = image[dH : h - dH, dW : w - dW]\n\n    resized_image = cv2.resize(\n        image, (self.width, self.height), interpolation=self.inter\n    )\n\n    return resized_image\n</code></pre>"},{"location":"pytorch-widedeep/utils/image_utils.html#pytorch_widedeep.utils.image_utils.SimplePreprocessor","title":"SimplePreprocessor","text":"<pre><code>SimplePreprocessor(width, height, inter=cv2.INTER_AREA)\n</code></pre> <p>Class to resize an image to a certain width and height</p> <p>Parameters:</p> <ul> <li> <code>width</code>               (<code>int</code>)           \u2013            <p>output width</p> </li> <li> <code>height</code>               (<code>int</code>)           \u2013            <p>output height</p> </li> <li> <code>inter</code>           \u2013            <p><code>opencv</code> interpolation method. See <code>opencv</code> <code>InterpolationFlags</code>.</p> </li> </ul> Source code in <code>pytorch_widedeep/utils/image_utils.py</code> <pre><code>def __init__(self, width: int, height: int, inter=cv2.INTER_AREA):\n    self.width = width\n    self.height = height\n    self.inter = inter\n</code></pre>"},{"location":"pytorch-widedeep/utils/image_utils.html#pytorch_widedeep.utils.image_utils.SimplePreprocessor.preprocess","title":"preprocess","text":"<pre><code>preprocess(image)\n</code></pre> <p>Returns the resized input image</p> <p>Parameters:</p> <ul> <li> <code>image</code>               (<code>ndarray</code>)           \u2013            <p>Input image to be resized</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>ndarray</code>           \u2013            <p>Resized image</p> </li> </ul> Source code in <code>pytorch_widedeep/utils/image_utils.py</code> <pre><code>def preprocess(self, image: np.ndarray) -&gt; np.ndarray:\n    r\"\"\"Returns the resized input image\n\n    Parameters\n    ----------\n    image: np.ndarray\n        Input image to be resized\n\n    Returns\n    -------\n    np.ndarray\n        Resized image\n\n    \"\"\"\n    resized_image = cv2.resize(\n        image, (self.width, self.height), interpolation=self.inter\n    )\n\n    return resized_image\n</code></pre>"},{"location":"pytorch-widedeep/utils/text_utils.html","title":"Text utils","text":"<p>Collection of helper function that facilitate processing text.</p>"},{"location":"pytorch-widedeep/utils/text_utils.html#pytorch_widedeep.utils.text_utils.simple_preprocess","title":"simple_preprocess","text":"<pre><code>simple_preprocess(\n    doc, lower=False, deacc=False, min_len=2, max_len=15\n)\n</code></pre> <p>This is <code>Gensim</code>'s <code>simple_preprocess</code> with a <code>lower</code> param to indicate wether or not to lower case all the token in the doc</p> <p>For more information see: <code>Gensim</code> utils module</p> <p>Parameters:</p> <ul> <li> <code>doc</code>               (<code>str</code>)           \u2013            <p>Input document.</p> </li> <li> <code>lower</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Lower case tokens in the input doc</p> </li> <li> <code>deacc</code>               (<code>bool</code>, default:                   <code>False</code> )           \u2013            <p>Remove accent marks from tokens using <code>Gensim</code>'s <code>deaccent</code></p> </li> <li> <code>min_len</code>               (<code>int</code>, default:                   <code>2</code> )           \u2013            <p>Minimum length of token (inclusive). Shorter tokens are discarded.</p> </li> <li> <code>max_len</code>               (<code>int</code>, default:                   <code>15</code> )           \u2013            <p>Maximum length of token in result (inclusive). Longer tokens are discarded.</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; from pytorch_widedeep.utils import simple_preprocess\n&gt;&gt;&gt; simple_preprocess('Machine learning is great')\n['Machine', 'learning', 'is', 'great']\n</code></pre> <p>Returns:</p> <ul> <li> <code>List[str]</code>           \u2013            <p>List with the processed tokens</p> </li> </ul> Source code in <code>pytorch_widedeep/utils/text_utils.py</code> <pre><code>def simple_preprocess(\n    doc: str,\n    lower: bool = False,\n    deacc: bool = False,\n    min_len: int = 2,\n    max_len: int = 15,\n) -&gt; List[str]:\n    r\"\"\"\n    This is `Gensim`'s `simple_preprocess` with a `lower` param to\n    indicate wether or not to lower case all the token in the doc\n\n    For more information see: `Gensim` [utils module](https://radimrehurek.com/gensim/utils.html)\n\n    Parameters\n    ----------\n    doc: str\n        Input document.\n    lower: bool, default = False\n        Lower case tokens in the input doc\n    deacc: bool, default = False\n        Remove accent marks from tokens using `Gensim`'s `deaccent`\n    min_len: int, default = 2\n        Minimum length of token (inclusive). Shorter tokens are discarded.\n    max_len: int, default = 15\n        Maximum length of token in result (inclusive). Longer tokens are discarded.\n\n    Examples\n    --------\n    &gt;&gt;&gt; from pytorch_widedeep.utils import simple_preprocess\n    &gt;&gt;&gt; simple_preprocess('Machine learning is great')\n    ['Machine', 'learning', 'is', 'great']\n\n    Returns\n    -------\n    List[str]\n        List with the processed tokens\n    \"\"\"\n    tokens = [\n        token\n        for token in tokenize(doc, lower=lower, deacc=deacc, errors=\"ignore\")\n        if min_len &lt;= len(token) &lt;= max_len and not token.startswith(\"_\")\n    ]\n    return tokens\n</code></pre>"},{"location":"pytorch-widedeep/utils/text_utils.html#pytorch_widedeep.utils.text_utils.get_texts","title":"get_texts","text":"<pre><code>get_texts(texts, already_processed=False, n_cpus=None)\n</code></pre> <p>Tokenization using <code>Fastai</code>'s <code>Tokenizer</code> because it does a series of very convenients things during the tokenization process</p> <p>See <code>pytorch_widedeep.utils.fastai_utils.Tokenizer</code></p> <p>Parameters:</p> <ul> <li> <code>texts</code>               (<code>List[str]</code>)           \u2013            <p>List of str with the texts (or documents). One str per document</p> </li> <li> <code>already_processed</code>               (<code>Optional[bool]</code>, default:                   <code>False</code> )           \u2013            <p>Boolean indicating if the text is already processed and we simply want to tokenize it. This parameter is thought for those cases where the input sequences might not be text (but IDs, or anything else) and we just want to tokenize it</p> </li> <li> <code>n_cpus</code>               (<code>Optional[int]</code>, default:                   <code>None</code> )           \u2013            <p>number of CPUs to used during the tokenization process</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; from pytorch_widedeep.utils import get_texts\n&gt;&gt;&gt; texts = ['Machine learning is great', 'but building stuff is even better']\n&gt;&gt;&gt; get_texts(texts)\n[['xxmaj', 'machine', 'learning', 'is', 'great'], ['but', 'building', 'stuff', 'is', 'even', 'better']]\n</code></pre> <p>Returns:</p> <ul> <li> <code>List[List[str]]</code>           \u2013            <p>List of lists, one list per 'document' containing its corresponding tokens</p> </li> <li> <code>information_source: **NOTE**:</code>           \u2013            </li> <li> <code>`get_texts` uses `pytorch_widedeep.utils.fastai_transforms.Tokenizer`.</code>           \u2013            </li> <li> <code>Such tokenizer uses a series of convenient processing steps, including</code>           \u2013            </li> <li> <code>the  addition of some special tokens, such as `TK_MAJ` (`xxmaj`), used to</code>           \u2013            </li> <li> <code>indicate the next word begins with a capital in the original text. For more</code>           \u2013            </li> <li> <code>details of special tokens please see the [`fastai` `docs](https://docs.fast.ai/text.core.html#Tokenizing)</code>           \u2013            </li> </ul> Source code in <code>pytorch_widedeep/utils/text_utils.py</code> <pre><code>def get_texts(\n    texts: List[str],\n    already_processed: Optional[bool] = False,\n    n_cpus: Optional[int] = None,\n) -&gt; List[List[str]]:\n    r\"\"\"Tokenization using `Fastai`'s `Tokenizer` because it does a\n    series of very convenients things during the tokenization process\n\n    See `pytorch_widedeep.utils.fastai_utils.Tokenizer`\n\n    Parameters\n    ----------\n    texts: List\n        List of str with the texts (or documents). One str per document\n    already_processed: bool, Optional, default = False\n        Boolean indicating if the text is already processed and we simply want\n        to tokenize it. This parameter is thought for those cases where the\n        input sequences might not be text (but IDs, or anything else) and we\n        just want to tokenize it\n    n_cpus: int, Optional, default = None\n        number of CPUs to used during the tokenization process\n\n    Examples\n    --------\n    &gt;&gt;&gt; from pytorch_widedeep.utils import get_texts\n    &gt;&gt;&gt; texts = ['Machine learning is great', 'but building stuff is even better']\n    &gt;&gt;&gt; get_texts(texts)\n    [['xxmaj', 'machine', 'learning', 'is', 'great'], ['but', 'building', 'stuff', 'is', 'even', 'better']]\n\n    Returns\n    -------\n    List[List[str]]\n        List of lists, one list per '_document_' containing its corresponding tokens\n\n    :information_source: **NOTE**:\n    `get_texts` uses `pytorch_widedeep.utils.fastai_transforms.Tokenizer`.\n    Such tokenizer uses a series of convenient processing steps, including\n    the  addition of some special tokens, such as `TK_MAJ` (`xxmaj`), used to\n    indicate the next word begins with a capital in the original text. For more\n    details of special tokens please see the [`fastai` `docs](https://docs.fast.ai/text.core.html#Tokenizing)\n    \"\"\"\n\n    num_cpus = n_cpus if n_cpus is not None else os.cpu_count()\n\n    if not already_processed:\n        processed_texts = [\" \".join(simple_preprocess(t)) for t in texts]\n    else:\n        processed_texts = texts\n    tok = Tokenizer(n_cpus=num_cpus).process_all(processed_texts)\n    return tok\n</code></pre>"},{"location":"pytorch-widedeep/utils/text_utils.html#pytorch_widedeep.utils.text_utils.pad_sequences","title":"pad_sequences","text":"<pre><code>pad_sequences(seq, maxlen, pad_first=True, pad_idx=1)\n</code></pre> <p>Given a List of tokenized and <code>numericalised</code> sequences it will return padded sequences according to the input parameters.</p> <p>Parameters:</p> <ul> <li> <code>seq</code>               (<code>List[int]</code>)           \u2013            <p>List of int with the <code>numericalised</code> tokens</p> </li> <li> <code>maxlen</code>               (<code>int</code>)           \u2013            <p>Maximum length of the padded sequences</p> </li> <li> <code>pad_first</code>               (<code>bool</code>, default:                   <code>True</code> )           \u2013            <p>Indicates whether the padding index will be added at the beginning or the end of the sequences</p> </li> <li> <code>pad_idx</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>padding index. Fastai's Tokenizer leaves 0 for the 'unknown' token.</p> </li> </ul> <p>Examples:</p> <pre><code>&gt;&gt;&gt; from pytorch_widedeep.utils import pad_sequences\n&gt;&gt;&gt; seq = [1,2,3]\n&gt;&gt;&gt; pad_sequences(seq, maxlen=5, pad_idx=0)\narray([0, 0, 1, 2, 3], dtype=int32)\n</code></pre> <p>Returns:</p> <ul> <li> <code>ndarray</code>           \u2013            <p>numpy array with the padded sequences</p> </li> </ul> Source code in <code>pytorch_widedeep/utils/text_utils.py</code> <pre><code>def pad_sequences(\n    seq: List[int], maxlen: int, pad_first: bool = True, pad_idx: int = 1\n) -&gt; np.ndarray:\n    r\"\"\"\n    Given a List of tokenized and `numericalised` sequences it will return\n    padded sequences according to the input parameters.\n\n    Parameters\n    ----------\n    seq: List\n        List of int with the `numericalised` tokens\n    maxlen: int\n        Maximum length of the padded sequences\n    pad_first: bool,  default = True\n        Indicates whether the padding index will be added at the beginning or the\n        end of the sequences\n    pad_idx: int, default = 1\n        padding index. Fastai's Tokenizer leaves 0 for the 'unknown' token.\n\n    Examples\n    --------\n    &gt;&gt;&gt; from pytorch_widedeep.utils import pad_sequences\n    &gt;&gt;&gt; seq = [1,2,3]\n    &gt;&gt;&gt; pad_sequences(seq, maxlen=5, pad_idx=0)\n    array([0, 0, 1, 2, 3], dtype=int32)\n\n    Returns\n    -------\n    np.ndarray\n        numpy array with the padded sequences\n    \"\"\"\n    if len(seq) == 0:\n        return np.zeros(maxlen, dtype=\"int32\") + pad_idx\n    elif len(seq) &gt;= maxlen:\n        res = np.array(seq[-maxlen:]).astype(\"int32\")\n        return res\n    else:\n        res = np.zeros(maxlen, dtype=\"int32\") + pad_idx\n        if pad_first:\n            res[-len(seq) :] = seq\n        else:\n            res[: len(seq) :] = seq\n        return res\n</code></pre>"},{"location":"pytorch-widedeep/utils/text_utils.html#pytorch_widedeep.utils.text_utils.build_embeddings_matrix","title":"build_embeddings_matrix","text":"<pre><code>build_embeddings_matrix(\n    vocab, word_vectors_path, min_freq, verbose=1\n)\n</code></pre> <p>Build the embedding matrix using pretrained word vectors.</p> <p>Returns pretrained word embeddings. If a word in our vocabulary is not among the pretrained embeddings it will be assigned the mean pretrained word-embeddings vector</p> <p>Parameters:</p> <ul> <li> <code>vocab</code>               (<code>Union[Vocab, ChunkVocab]</code>)           \u2013            <p>see <code>pytorch_widedeep.utils.fastai_utils.Vocab</code></p> </li> <li> <code>word_vectors_path</code>               (<code>str</code>)           \u2013            <p>path to the pretrained word embeddings</p> </li> <li> <code>min_freq</code>               (<code>int</code>)           \u2013            <p>minimum frequency required for a word to be in the vocabulary</p> </li> <li> <code>verbose</code>               (<code>int</code>, default:                   <code>1</code> )           \u2013            <p>level of verbosity. Set to 0 for no verbosity</p> </li> </ul> <p>Returns:</p> <ul> <li> <code>ndarray</code>           \u2013            <p>Pretrained word embeddings</p> </li> </ul> Source code in <code>pytorch_widedeep/utils/text_utils.py</code> <pre><code>def build_embeddings_matrix(\n    vocab: Union[Vocab, ChunkVocab],\n    word_vectors_path: str,\n    min_freq: int,\n    verbose: int = 1,\n) -&gt; np.ndarray:  # pragma: no cover\n    r\"\"\"Build the embedding matrix using pretrained word vectors.\n\n    Returns pretrained word embeddings. If a word in our vocabulary is not\n    among the pretrained embeddings it will be assigned the mean pretrained\n    word-embeddings vector\n\n    Parameters\n    ----------\n    vocab: Vocab\n        see `pytorch_widedeep.utils.fastai_utils.Vocab`\n    word_vectors_path: str\n        path to the pretrained word embeddings\n    min_freq: int\n        minimum frequency required for a word to be in the vocabulary\n    verbose: int,  default=1\n        level of verbosity. Set to 0 for no verbosity\n\n    Returns\n    -------\n    np.ndarray\n        Pretrained word embeddings\n    \"\"\"\n    if not os.path.isfile(word_vectors_path):\n        raise FileNotFoundError(\"{} not found\".format(word_vectors_path))\n    if verbose:\n        print(\"Indexing word vectors...\")\n\n    embeddings_index = {}\n    f = open(word_vectors_path)\n    for line in f:\n        values = line.split()\n        word = values[0]\n        coefs = np.asarray(values[1:], dtype=\"float32\")\n        embeddings_index[word] = coefs\n    f.close()\n\n    if verbose:\n        print(\"Loaded {} word vectors\".format(len(embeddings_index)))\n        print(\"Preparing embeddings matrix...\")\n\n    mean_word_vector = np.mean(list(embeddings_index.values()), axis=0)  # type: ignore[arg-type]\n    embedding_dim = len(list(embeddings_index.values())[0])\n    num_words = len(vocab.itos)\n    embedding_matrix = np.zeros((num_words, embedding_dim))\n    found_words = 0\n    for i, word in enumerate(vocab.itos):\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[i] = embedding_vector\n            found_words += 1\n        else:\n            embedding_matrix[i] = mean_word_vector\n\n    if verbose:\n        print(\n            \"{} words in the vocabulary had {} vectors and appear more than {} times\".format(\n                found_words, word_vectors_path, min_freq\n            )\n        )\n\n    return embedding_matrix.astype(\"float32\")\n</code></pre>"}]}
\ No newline at end of file
+{"config":{"lang":["en"],"separator":"[\\s\\-]+","pipeline":["stopWordFilter"]},"docs":[{"location":"index.html","title":"Home","text":""},{"location":"index.html#pytorch-widedeep","title":"pytorch-widedeep","text":"<p>A flexible package for multimodal-deep-learning to combine tabular data with text and images using Wide and Deep models in Pytorch</p> <p>Documentation: https://pytorch-widedeep.readthedocs.io</p> <p>Companion posts and tutorials: infinitoml</p> <p>Experiments and comparison with <code>LightGBM</code>: TabularDL vs LightGBM</p> <p>Slack: if you want to contribute or just want to chat with us, join slack</p> <p>The content of this document is organized as follows:</p> <ul> <li>pytorch-widedeep<ul> <li>Introduction</li> <li>Architectures</li> <li>The <code>deeptabular</code> component</li> <li>The <code>rec</code> module</li> <li>Text and Images</li> <li>Acknowledgments</li> <li>License</li> <li>Cite<ul> <li>BibTex</li> <li>APA</li> </ul> </li> </ul> </li> </ul>"},{"location":"index.html#introduction","title":"Introduction","text":"<p><code>pytorch-widedeep</code> is based on Google's Wide and Deep Algorithm, adjusted for multi-modal datasets.</p> <p>In general terms, <code>pytorch-widedeep</code> is a package to use deep learning with tabular data. In particular, is intended to facilitate the combination of text and images with corresponding tabular data using wide and deep models. With that in mind there are a number of architectures that can be implemented with the library. The main components of those architectures are shown in the Figure below:</p> <p> </p> <p>In math terms, and following the notation in the paper, the expression for the architecture without a <code>deephead</code> component can be formulated as:</p> <p> </p> <p>Where \u03c3 is the sigmoid function, 'W' are the weight matrices applied to the wide model and to the final activations of the deep models, 'a' are these final activations, \u03c6(x) are the cross product transformations of the original features 'x', and , and 'b' is the bias term. In case you are wondering what are \"cross product transformations\", here is a quote taken directly from the paper: \"For binary features, a cross-product transformation (e.g., \u201cAND(gender=female, language=en)\u201d) is 1 if and only if the constituent features (\u201cgender=female\u201d and \u201clanguage=en\u201d) are all 1, and 0 otherwise\".</p> <p>It is perfectly possible to use custom models (and not necessarily those in the library) as long as the the custom models have a property called <code>output_dim</code> with the size of the last layer of activations, so that <code>WideDeep</code> can be constructed. Examples on how to use custom components can be found in the Examples folder and the section below.</p>"},{"location":"index.html#architectures","title":"Architectures","text":"<p>The <code>pytorch-widedeep</code> library offers a number of different architectures. In this section we will show some of them in their simplest form (i.e. with default param values in most cases) with their corresponding code snippets. Note that all the snippets below shoud run locally. For a more detailed explanation of the different components and their parameters, please refer to the documentation.</p> <p>For the examples below we will be using a toy dataset generated as follows:</p> <pre><code>import os\nimport random\n\nimport numpy as np\nimport pandas as pd\nfrom PIL import Image\nfrom faker import Faker\n\n\ndef create_and_save_random_image(image_number, size=(32, 32)):\n\n    if not os.path.exists(\"images\"):\n        os.makedirs(\"images\")\n\n    array = np.random.randint(0, 256, (size[0], size[1], 3), dtype=np.uint8)\n\n    image = Image.fromarray(array)\n\n    image_name = f\"image_{image_number}.png\"\n    image.save(os.path.join(\"images\", image_name))\n\n    return image_name\n\n\nfake = Faker()\n\ncities = [\"New York\", \"Los Angeles\", \"Chicago\", \"Houston\"]\nnames = [\"Alice\", \"Bob\", \"Charlie\", \"David\", \"Eva\"]\n\ndata = {\n    \"city\": [random.choice(cities) for _ in range(100)],\n    \"name\": [random.choice(names) for _ in range(100)],\n    \"age\": [random.uniform(18, 70) for _ in range(100)],\n    \"height\": [random.uniform(150, 200) for _ in range(100)],\n    \"sentence\": [fake.sentence() for _ in range(100)],\n    \"other_sentence\": [fake.sentence() for _ in range(100)],\n    \"image_name\": [create_and_save_random_image(i) for i in range(100)],\n    \"target\": [random.choice([0, 1]) for _ in range(100)],\n}\n\ndf = pd.DataFrame(data)\n</code></pre> <p>This will create a 100 rows dataframe and a dir in your local folder, called <code>images</code> with 100 random images (or images with just noise).</p> <p>Perhaps the simplest architecture would be just one component, <code>wide</code>, <code>deeptabular</code>, <code>deeptext</code> or <code>deepimage</code> on their own, which is also possible, but let's start the examples with a standard Wide and Deep architecture. From there, how to build a model comprised only of one component will be straightforward.</p> <p>Note that the examples shown below would be almost identical using any of the models available in the library. For example, <code>TabMlp</code> can be replaced by <code>TabResnet</code>, <code>TabNet</code>, <code>TabTransformer</code>, etc. Similarly, <code>BasicRNN</code> can be replaced by <code>AttentiveRNN</code>, <code>StackedAttentiveRNN</code>, or <code>HFModel</code> with their corresponding parameters and preprocessor in the case of the Hugging Face models.</p> <p>1. Wide and Tabular component (aka deeptabular)</p> <p> </p> <pre><code>from pytorch_widedeep.preprocessing import TabPreprocessor, WidePreprocessor\nfrom pytorch_widedeep.models import Wide, TabMlp, WideDeep\nfrom pytorch_widedeep.training import Trainer\n\n# Wide\nwide_cols = [\"city\"]\ncrossed_cols = [(\"city\", \"name\")]\nwide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols)\nX_wide = wide_preprocessor.fit_transform(df)\nwide = Wide(input_dim=np.unique(X_wide).shape[0])\n\n# Tabular\ntab_preprocessor = TabPreprocessor(\n    embed_cols=[\"city\", \"name\"], continuous_cols=[\"age\", \"height\"]\n)\nX_tab = tab_preprocessor.fit_transform(df)\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=tab_preprocessor.continuous_cols,\n    mlp_hidden_dims=[64, 32],\n)\n\n# WideDeep\nmodel = WideDeep(wide=wide, deeptabular=tab_mlp)\n\n# Train\ntrainer = Trainer(model, objective=\"binary\")\n\ntrainer.fit(\n    X_wide=X_wide,\n    X_tab=X_tab,\n    target=df[\"target\"].values,\n    n_epochs=1,\n    batch_size=32,\n)\n</code></pre> <p>2. Tabular and Text data</p> <p> </p> <pre><code>from pytorch_widedeep.preprocessing import TabPreprocessor, TextPreprocessor\nfrom pytorch_widedeep.models import TabMlp, BasicRNN, WideDeep\nfrom pytorch_widedeep.training import Trainer\n\n# Tabular\ntab_preprocessor = TabPreprocessor(\n    embed_cols=[\"city\", \"name\"], continuous_cols=[\"age\", \"height\"]\n)\nX_tab = tab_preprocessor.fit_transform(df)\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=tab_preprocessor.continuous_cols,\n    mlp_hidden_dims=[64, 32],\n)\n\n# Text\ntext_preprocessor = TextPreprocessor(\n    text_col=\"sentence\", maxlen=20, max_vocab=100, n_cpus=1\n)\nX_text = text_preprocessor.fit_transform(df)\nrnn = BasicRNN(\n    vocab_size=len(text_preprocessor.vocab.itos),\n    embed_dim=16,\n    hidden_dim=8,\n    n_layers=1,\n)\n\n# WideDeep\nmodel = WideDeep(deeptabular=tab_mlp, deeptext=rnn)\n\n# Train\ntrainer = Trainer(model, objective=\"binary\")\n\ntrainer.fit(\n    X_tab=X_tab,\n    X_text=X_text,\n    target=df[\"target\"].values,\n    n_epochs=1,\n    batch_size=32,\n)\n</code></pre> <p>3. Tabular and text with a FC head on top via the <code>head_hidden_dims</code> param   in <code>WideDeep</code></p> <p> </p> <pre><code>from pytorch_widedeep.preprocessing import TabPreprocessor, TextPreprocessor\nfrom pytorch_widedeep.models import TabMlp, BasicRNN, WideDeep\nfrom pytorch_widedeep.training import Trainer\n\n# Tabular\ntab_preprocessor = TabPreprocessor(\n    embed_cols=[\"city\", \"name\"], continuous_cols=[\"age\", \"height\"]\n)\nX_tab = tab_preprocessor.fit_transform(df)\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=tab_preprocessor.continuous_cols,\n    mlp_hidden_dims=[64, 32],\n)\n\n# Text\ntext_preprocessor = TextPreprocessor(\n    text_col=\"sentence\", maxlen=20, max_vocab=100, n_cpus=1\n)\nX_text = text_preprocessor.fit_transform(df)\nrnn = BasicRNN(\n    vocab_size=len(text_preprocessor.vocab.itos),\n    embed_dim=16,\n    hidden_dim=8,\n    n_layers=1,\n)\n\n# WideDeep\nmodel = WideDeep(deeptabular=tab_mlp, deeptext=rnn, head_hidden_dims=[32, 16])\n\n# Train\ntrainer = Trainer(model, objective=\"binary\")\n\ntrainer.fit(\n    X_tab=X_tab,\n    X_text=X_text,\n    target=df[\"target\"].values,\n    n_epochs=1,\n    batch_size=32,\n)\n</code></pre> <p>4. Tabular and multiple text columns that are passed directly to   <code>WideDeep</code></p> <p> </p> <pre><code>from pytorch_widedeep.preprocessing import TabPreprocessor, TextPreprocessor\nfrom pytorch_widedeep.models import TabMlp, BasicRNN, WideDeep\nfrom pytorch_widedeep.training import Trainer\n\n\n# Tabular\ntab_preprocessor = TabPreprocessor(\n    embed_cols=[\"city\", \"name\"], continuous_cols=[\"age\", \"height\"]\n)\nX_tab = tab_preprocessor.fit_transform(df)\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=tab_preprocessor.continuous_cols,\n    mlp_hidden_dims=[64, 32],\n)\n\n# Text\ntext_preprocessor_1 = TextPreprocessor(\n    text_col=\"sentence\", maxlen=20, max_vocab=100, n_cpus=1\n)\nX_text_1 = text_preprocessor_1.fit_transform(df)\ntext_preprocessor_2 = TextPreprocessor(\n    text_col=\"other_sentence\", maxlen=20, max_vocab=100, n_cpus=1\n)\nX_text_2 = text_preprocessor_2.fit_transform(df)\nrnn_1 = BasicRNN(\n    vocab_size=len(text_preprocessor_1.vocab.itos),\n    embed_dim=16,\n    hidden_dim=8,\n    n_layers=1,\n)\nrnn_2 = BasicRNN(\n    vocab_size=len(text_preprocessor_2.vocab.itos),\n    embed_dim=16,\n    hidden_dim=8,\n    n_layers=1,\n)\n\n# WideDeep\nmodel = WideDeep(deeptabular=tab_mlp, deeptext=[rnn_1, rnn_2])\n\n# Train\ntrainer = Trainer(model, objective=\"binary\")\n\ntrainer.fit(\n    X_tab=X_tab,\n    X_text=[X_text_1, X_text_2],\n    target=df[\"target\"].values,\n    n_epochs=1,\n    batch_size=32,\n)\n</code></pre> <p>5. Tabular data and multiple text columns that are fused via a the library's   <code>ModelFuser</code> class</p> <p> </p> <pre><code>from pytorch_widedeep.preprocessing import TabPreprocessor, TextPreprocessor\nfrom pytorch_widedeep.models import TabMlp, BasicRNN, WideDeep, ModelFuser\nfrom pytorch_widedeep import Trainer\n\n# Tabular\ntab_preprocessor = TabPreprocessor(\n    embed_cols=[\"city\", \"name\"], continuous_cols=[\"age\", \"height\"]\n)\nX_tab = tab_preprocessor.fit_transform(df)\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=tab_preprocessor.continuous_cols,\n    mlp_hidden_dims=[64, 32],\n)\n\n# Text\ntext_preprocessor_1 = TextPreprocessor(\n    text_col=\"sentence\", maxlen=20, max_vocab=100, n_cpus=1\n)\nX_text_1 = text_preprocessor_1.fit_transform(df)\ntext_preprocessor_2 = TextPreprocessor(\n    text_col=\"other_sentence\", maxlen=20, max_vocab=100, n_cpus=1\n)\nX_text_2 = text_preprocessor_2.fit_transform(df)\n\nrnn_1 = BasicRNN(\n    vocab_size=len(text_preprocessor_1.vocab.itos),\n    embed_dim=16,\n    hidden_dim=8,\n    n_layers=1,\n)\nrnn_2 = BasicRNN(\n    vocab_size=len(text_preprocessor_2.vocab.itos),\n    embed_dim=16,\n    hidden_dim=8,\n    n_layers=1,\n)\n\nmodels_fuser = ModelFuser(models=[rnn_1, rnn_2], fusion_method=\"mult\")\n\n# WideDeep\nmodel = WideDeep(deeptabular=tab_mlp, deeptext=models_fuser)\n\n# Train\ntrainer = Trainer(model, objective=\"binary\")\n\ntrainer.fit(\n    X_tab=X_tab,\n    X_text=[X_text_1, X_text_2],\n    target=df[\"target\"].values,\n    n_epochs=1,\n    batch_size=32,\n)\n</code></pre> <p>6. Tabular and multiple text columns, with an image column. The text columns   are fused via the library's <code>ModelFuser</code> and then all fused via the   deephead paramenter in <code>WideDeep</code> which is a custom <code>ModelFuser</code> coded by   the user</p> <p>This is perhaps the less elegant solution as it involves a custom component by the user and slicing the 'incoming' tensor. In the future, we will include a <code>TextAndImageModelFuser</code> to make this process more straightforward. Still, is not really complicated and it is a good example of how to use custom components in <code>pytorch-widedeep</code>.</p> <p>Note that the only requirement for the custom component is that it has a property called <code>output_dim</code> that returns the size of the last layer of activations. In other words, it does not need to inherit from <code>BaseWDModelComponent</code>. This base class simply checks the existence of such property and avoids some typing errors internally.</p> <p> </p> <pre><code>import torch\n\nfrom pytorch_widedeep.preprocessing import TabPreprocessor, TextPreprocessor, ImagePreprocessor\nfrom pytorch_widedeep.models import TabMlp, BasicRNN, WideDeep, ModelFuser, Vision\nfrom pytorch_widedeep.models._base_wd_model_component import BaseWDModelComponent\nfrom pytorch_widedeep import Trainer\n\n# Tabular\ntab_preprocessor = TabPreprocessor(\n    embed_cols=[\"city\", \"name\"], continuous_cols=[\"age\", \"height\"]\n)\nX_tab = tab_preprocessor.fit_transform(df)\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=tab_preprocessor.continuous_cols,\n    mlp_hidden_dims=[16, 8],\n)\n\n# Text\ntext_preprocessor_1 = TextPreprocessor(\n    text_col=\"sentence\", maxlen=20, max_vocab=100, n_cpus=1\n)\nX_text_1 = text_preprocessor_1.fit_transform(df)\ntext_preprocessor_2 = TextPreprocessor(\n    text_col=\"other_sentence\", maxlen=20, max_vocab=100, n_cpus=1\n)\nX_text_2 = text_preprocessor_2.fit_transform(df)\nrnn_1 = BasicRNN(\n    vocab_size=len(text_preprocessor_1.vocab.itos),\n    embed_dim=16,\n    hidden_dim=8,\n    n_layers=1,\n)\nrnn_2 = BasicRNN(\n    vocab_size=len(text_preprocessor_2.vocab.itos),\n    embed_dim=16,\n    hidden_dim=8,\n    n_layers=1,\n)\nmodels_fuser = ModelFuser(\n    models=[rnn_1, rnn_2],\n    fusion_method=\"mult\",\n)\n\n# Image\nimage_preprocessor = ImagePreprocessor(img_col=\"image_name\", img_path=\"images\")\nX_img = image_preprocessor.fit_transform(df)\nvision = Vision(pretrained_model_setup=\"resnet18\", head_hidden_dims=[16, 8])\n\n# deephead (custom model fuser)\nclass MyModelFuser(BaseWDModelComponent):\n    \"\"\"\n    Simply a Linear + Relu sequence on top of the text + images followed by a\n    Linear -&gt; Relu -&gt; Linear for the concatenation of tabular slice of the\n    tensor and the output of the text and image sequential model\n    \"\"\"\n    def __init__(\n        self,\n        tab_incoming_dim: int,\n        text_incoming_dim: int,\n        image_incoming_dim: int,\n        output_units: int,\n    ):\n\n        super(MyModelFuser, self).__init__()\n\n        self.tab_incoming_dim = tab_incoming_dim\n        self.text_incoming_dim = text_incoming_dim\n        self.image_incoming_dim = image_incoming_dim\n        self.output_units = output_units\n        self.text_and_image_fuser = torch.nn.Sequential(\n            torch.nn.Linear(text_incoming_dim + image_incoming_dim, output_units),\n            torch.nn.ReLU(),\n        )\n        self.out = torch.nn.Sequential(\n            torch.nn.Linear(output_units + tab_incoming_dim, output_units * 4),\n            torch.nn.ReLU(),\n            torch.nn.Linear(output_units * 4, output_units),\n        )\n\n    def forward(self, X: torch.Tensor) -&gt; torch.Tensor:\n        tab_slice = slice(0, self.tab_incoming_dim)\n        text_slice = slice(\n            self.tab_incoming_dim, self.tab_incoming_dim + self.text_incoming_dim\n        )\n        image_slice = slice(\n            self.tab_incoming_dim + self.text_incoming_dim,\n            self.tab_incoming_dim + self.text_incoming_dim + self.image_incoming_dim,\n        )\n        X_tab = X[:, tab_slice]\n        X_text = X[:, text_slice]\n        X_img = X[:, image_slice]\n        X_text_and_image = self.text_and_image_fuser(torch.cat([X_text, X_img], dim=1))\n        return self.out(torch.cat([X_tab, X_text_and_image], dim=1))\n\n    @property\n    def output_dim(self):\n        return self.output_units\n\ndeephead = MyModelFuser(\n    tab_incoming_dim=tab_mlp.output_dim,\n    text_incoming_dim=models_fuser.output_dim,\n    image_incoming_dim=vision.output_dim,\n    output_units=8,\n)\n\n# WideDeep\nmodel = WideDeep(\n    deeptabular=tab_mlp,\n    deeptext=models_fuser,\n    deepimage=vision,\n    deephead=deephead,\n)\n\n# Train\ntrainer = Trainer(model, objective=\"binary\")\n\ntrainer.fit(\n    X_tab=X_tab,\n    X_text=[X_text_1, X_text_2],\n    X_img=X_img,\n    target=df[\"target\"].values,\n    n_epochs=1,\n    batch_size=32,\n)\n</code></pre> <p>7. A two-tower model</p> <p>This is a popular model in the context of recommendation systems. Let's say we have a tabular dataset formed my triples (user features, item features, target). We can create a two-tower model where the user and item features are passed through two separate models and then \"fused\" via a dot product.</p> <p> </p> <pre><code>import numpy as np\nimport pandas as pd\n\nfrom pytorch_widedeep import Trainer\nfrom pytorch_widedeep.preprocessing import TabPreprocessor\nfrom pytorch_widedeep.models import TabMlp, WideDeep, ModelFuser\n\n# Let's create the interaction dataset\n# user_features dataframe\nnp.random.seed(42)\nuser_ids = np.arange(1, 101)\nages = np.random.randint(18, 60, size=100)\ngenders = np.random.choice([\"male\", \"female\"], size=100)\nlocations = np.random.choice([\"city_a\", \"city_b\", \"city_c\", \"city_d\"], size=100)\nuser_features = pd.DataFrame(\n    {\"id\": user_ids, \"age\": ages, \"gender\": genders, \"location\": locations}\n)\n\n# item_features dataframe\nitem_ids = np.arange(1, 101)\nprices = np.random.uniform(10, 500, size=100).round(2)\ncolors = np.random.choice([\"red\", \"blue\", \"green\", \"black\"], size=100)\ncategories = np.random.choice([\"electronics\", \"clothing\", \"home\", \"toys\"], size=100)\n\nitem_features = pd.DataFrame(\n    {\"id\": item_ids, \"price\": prices, \"color\": colors, \"category\": categories}\n)\n\n# Interactions dataframe\ninteraction_user_ids = np.random.choice(user_ids, size=1000)\ninteraction_item_ids = np.random.choice(item_ids, size=1000)\npurchased = np.random.choice([0, 1], size=1000, p=[0.7, 0.3])\ninteractions = pd.DataFrame(\n    {\n        \"user_id\": interaction_user_ids,\n        \"item_id\": interaction_item_ids,\n        \"purchased\": purchased,\n    }\n)\nuser_item_purchased = interactions.merge(\n    user_features, left_on=\"user_id\", right_on=\"id\"\n).merge(item_features, left_on=\"item_id\", right_on=\"id\")\n\n# Users\ntab_preprocessor_user = TabPreprocessor(\n    cat_embed_cols=[\"gender\", \"location\"],\n    continuous_cols=[\"age\"],\n)\nX_user = tab_preprocessor_user.fit_transform(user_item_purchased)\ntab_mlp_user = TabMlp(\n    column_idx=tab_preprocessor_user.column_idx,\n    cat_embed_input=tab_preprocessor_user.cat_embed_input,\n    continuous_cols=[\"age\"],\n    mlp_hidden_dims=[16, 8],\n    mlp_dropout=[0.2, 0.2],\n)\n\n# Items\ntab_preprocessor_item = TabPreprocessor(\n    cat_embed_cols=[\"color\", \"category\"],\n    continuous_cols=[\"price\"],\n)\nX_item = tab_preprocessor_item.fit_transform(user_item_purchased)\ntab_mlp_item = TabMlp(\n    column_idx=tab_preprocessor_item.column_idx,\n    cat_embed_input=tab_preprocessor_item.cat_embed_input,\n    continuous_cols=[\"price\"],\n    mlp_hidden_dims=[16, 8],\n    mlp_dropout=[0.2, 0.2],\n)\n\ntwo_tower_model = ModelFuser([tab_mlp_user, tab_mlp_item], fusion_method=\"dot\")\n\nmodel = WideDeep(deeptabular=two_tower_model)\n\ntrainer = Trainer(model, objective=\"binary\")\n\ntrainer.fit(\n    X_tab=[X_user, X_item],\n    target=interactions.purchased.values,\n    n_epochs=1,\n    batch_size=32,\n)\n</code></pre> <p>8. Tabular with a multi-target loss</p> <p>This one is \"a bonus\" to illustrate the use of multi-target losses, more than actually a different architecture.</p> <p> </p> <pre><code>from pytorch_widedeep.preprocessing import TabPreprocessor, TextPreprocessor, ImagePreprocessor\nfrom pytorch_widedeep.models import TabMlp, BasicRNN, WideDeep, ModelFuser, Vision\nfrom pytorch_widedeep.losses_multitarget import MultiTargetClassificationLoss\nfrom pytorch_widedeep.models._base_wd_model_component import BaseWDModelComponent\nfrom pytorch_widedeep import Trainer\n\n# let's add a second target to the dataframe\ndf[\"target2\"] = [random.choice([0, 1]) for _ in range(100)]\n\n# Tabular\ntab_preprocessor = TabPreprocessor(\n    embed_cols=[\"city\", \"name\"], continuous_cols=[\"age\", \"height\"]\n)\nX_tab = tab_preprocessor.fit_transform(df)\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=tab_preprocessor.continuous_cols,\n    mlp_hidden_dims=[64, 32],\n)\n\n# 'pred_dim=2' because we have two binary targets. For other types of targets,\n#  please, see the documentation\nmodel = WideDeep(deeptabular=tab_mlp, pred_dim=2).\n\nloss = MultiTargetClassificationLoss(binary_config=[0, 1], reduction=\"mean\")\n\n# When a multi-target loss is used, 'custom_loss_function' must not be None.\n# See the docs\ntrainer = Trainer(model, objective=\"multitarget\", custom_loss_function=loss)\n\ntrainer.fit(\n    X_tab=X_tab,\n    target=df[[\"target\", \"target2\"]].values,\n    n_epochs=1,\n    batch_size=32,\n)\n</code></pre>"},{"location":"index.html#the-deeptabular-component","title":"The <code>deeptabular</code> component","text":"<p>It is important to emphasize again that each individual component, <code>wide</code>, <code>deeptabular</code>, <code>deeptext</code> and <code>deepimage</code>, can be used independently and in isolation. For example, one could use only <code>wide</code>, which is in simply a linear model. In fact, one of the most interesting functionalities in<code>pytorch-widedeep</code> would be the use of the <code>deeptabular</code> component on its own, i.e. what one might normally refer as Deep Learning for Tabular Data. Currently, <code>pytorch-widedeep</code> offers the following different models for that component:</p> <ol> <li>Wide: a simple linear model where the nonlinearities are captured via cross-product transformations, as explained before.</li> <li>TabMlp: a simple MLP that receives embeddings representing the categorical features, concatenated with the continuous features, which can also be embedded.</li> <li>TabResnet: similar to the previous model but the embeddings are passed through a series of ResNet blocks built with dense layers.</li> <li>TabNet: details on TabNet can be found in TabNet: Attentive Interpretable Tabular Learning</li> </ol> <p>Two simpler attention based models that we call:</p> <ol> <li>ContextAttentionMLP: MLP with at attention mechanism \"on top\" that is based on     Hierarchical Attention Networks for Document Classification</li> <li>SelfAttentionMLP: MLP with an attention mechanism that is a simplified     version of a transformer block that we refer as \"query-key self-attention\".</li> </ol> <p>The <code>Tabformer</code> family, i.e. Transformers for Tabular data:</p> <ol> <li>TabTransformer: details on the TabTransformer can be found in TabTransformer: Tabular Data Modeling Using Contextual Embeddings.</li> <li>SAINT: Details on SAINT can be found in SAINT: Improved Neural Networks for Tabular Data via Row Attention and Contrastive Pre-Training.</li> <li>FT-Transformer: details on the FT-Transformer can be found in Revisiting Deep Learning Models for Tabular Data.</li> <li>TabFastFormer: adaptation of the FastFormer for tabular data. Details on the Fasformer can be found in FastFormers: Highly Efficient Transformer Models for Natural Language Understanding</li> <li>TabPerceiver: adaptation of the Perceiver for tabular data. Details on the Perceiver can be found in Perceiver: General Perception with Iterative Attention</li> </ol> <p>And probabilistic DL models for tabular data based on Weight Uncertainty in Neural Networks:</p> <ol> <li>BayesianWide: Probabilistic adaptation of the <code>Wide</code> model.</li> <li>BayesianTabMlp: Probabilistic adaptation of the <code>TabMlp</code> model</li> </ol> <p>Note that while there are scientific publications for the TabTransformer, SAINT and FT-Transformer, the TabFasfFormer and TabPerceiver are our own adaptation of those algorithms for tabular data.</p> <p>In addition, Self-Supervised pre-training can be used for all <code>deeptabular</code> models, with the exception of the <code>TabPerceiver</code>. Self-Supervised pre-training can be used via two methods or routines which we refer as: encoder-decoder method and constrastive-denoising method. Please, see the documentation and the examples for details on this functionality, and all other options in the library.</p>"},{"location":"index.html#the-rec-module","title":"The <code>rec</code> module","text":"<p>This module was introduced as an extension to the existing components in the library, addressing questions and issues related to recommendation systems. While still under active development, it currently includes a select number of powerful recommendation models.</p> <p>It's worth noting that this library already supported the implementation of various recommendation algorithms using existing components. For example, models like Wide and Deep, Two-Tower, or Neural Collaborative Filtering could be constructed using the library's core functionalities.</p> <p>The recommendation algorithms in the <code>rec</code> module are:</p> <ol> <li>DeepFM: A Factorization-Machine based Neural Network for CTR Prediction</li> <li>(Deep) Field Aware Factorization Machine (FFM): a Deep Learning version of the algorithm presented in Field-aware Factorization Machines in a Real-world Online Advertising System</li> <li>xDeepFM: Combining Explicit and Implicit Feature Interactions for Recommender Systems</li> <li>Deep Interest Network for Click-Through Rate Prediction</li> </ol> <p>These can all be used as the <code>deeptabular</code> component in the <code>WideDeep</code> model. See the examples for more details.</p>"},{"location":"index.html#text-and-images","title":"Text and Images","text":"<p>For the text component, <code>deeptext</code>, the library offers the following models:</p> <ol> <li>BasicRNN: a simple RNN 2. AttentiveRNN: a RNN with an attention mechanism based on the Hierarchical Attention Networks for DocumentClassification</li> <li>StackedAttentiveRNN: a stack of AttentiveRNNs</li> <li>HFModel: a wrapper around Hugging Face Transfomer-based models. At the moment only models from the families BERT, RoBERTa, DistilBERT, ALBERT and ELECTRA are supported. This is because this library is designed to address classification and regression tasks and these are the most 'popular' encoder-only models, which have proved to be those that work best for these tasks. If there is demand for other models, they will be included in the future.</li> </ol> <p>For the image component, <code>deepimage</code>, the library supports models from the following families: 'resnet', 'shufflenet', 'resnext', 'wide_resnet', 'regnet', 'densenet', 'mobilenetv3',  'mobilenetv2', 'mnasnet', 'efficientnet' and 'squeezenet'.  These are  offered via <code>torchvision</code> and wrapped up in the <code>Vision</code> class.</p>"},{"location":"index.html#acknowledgments","title":"Acknowledgments","text":"<p>This library takes from a series of other libraries, so I think it is just fair to mention them here in the README (specific mentions are also included in the code).</p> <p>The <code>Callbacks</code> and <code>Initializers</code> structure and code is inspired by the <code>torchsample</code> library, which in itself partially inspired by <code>Keras</code>.</p> <p>The <code>TextProcessor</code> class in this library uses the <code>fastai</code>'s <code>Tokenizer</code> and <code>Vocab</code>. The code at <code>utils.fastai_transforms</code> is a minor adaptation of their code so it functions within this library. To my experience their <code>Tokenizer</code> is the best in class.</p> <p>The <code>ImageProcessor</code> class in this library uses code from the fantastic Deep Learning for Computer Vision (DL4CV) book by Adrian Rosebrock.</p>"},{"location":"index.html#license","title":"License","text":"<p>This work is dual-licensed under Apache 2.0 and MIT (or any later version). You can choose between one of them if you use this work.</p> <p><code>SPDX-License-Identifier: Apache-2.0 AND MIT</code></p>"},{"location":"index.html#cite","title":"Cite","text":""},{"location":"index.html#bibtex","title":"BibTex","text":"<pre><code>@article{Zaurin_pytorch-widedeep_A_flexible_2023,\nauthor = {Zaurin, Javier Rodriguez and Mulinka, Pavol},\ndoi = {10.21105/joss.05027},\njournal = {Journal of Open Source Software},\nmonth = jun,\nnumber = {86},\npages = {5027},\ntitle = {{pytorch-widedeep: A flexible package for multimodal deep learning}},\nurl = {https://joss.theoj.org/papers/10.21105/joss.05027},\nvolume = {8},\nyear = {2023}\n}\n</code></pre>"},{"location":"index.html#apa","title":"APA","text":"<pre><code>Zaurin, J. R., &amp; Mulinka, P. (2023). pytorch-widedeep: A flexible package for\nmultimodal deep learning. Journal of Open Source Software, 8(86), 5027.\nhttps://doi.org/10.21105/joss.05027\n</code></pre>"},{"location":"contributing.html","title":"Contributing","text":"<p>Pytorch-widedeep is being developed and used by many active community members. Your help is very valuable to make it better for everyone.</p> <ul> <li>Check for the Roadmap or Open an issue to report problems or recommend new features and submit a draft pull requests, which will be changed to pull request after intial review</li> <li>Contribute to the tests to make it more reliable.</li> <li>Contribute to the documentation to make it clearer for everyone.</li> <li>Contribute to the examples to share your experience with other users.</li> <li>Join the dicussion on slack</li> </ul>"},{"location":"installation.html","title":"Installation","text":"<p>This section explains how to install <code>pytorch-widedeep</code>.</p> <p>For the latest stable release, execute:</p> <pre><code>pip install pytorch-widedeep\n</code></pre> <p>For the bleeding-edge version, execute:</p> <pre><code>pip install git+https://github.com/jrzaurin/pytorch-widedeep.git\n</code></pre> <p>For developer install</p> <pre><code># Clone the repository\ngit clone https://github.com/jrzaurin/pytorch-widedeep\ncd pytorch-widedeep\n\n# Install in dev mode\npip install -e .\n</code></pre>"},{"location":"installation.html#dependencies","title":"Dependencies","text":"<ul> <li>pandas&gt;=1.3.5</li> <li>numpy&gt;=1.21.6</li> <li>scipy&gt;=1.7.3,&lt;=1.12.0</li> <li>scikit-learn&gt;=1.0.2</li> <li>gensim</li> <li>spacy</li> <li>opencv-contrib-python</li> <li>imutils</li> <li>tqdm</li> <li>torch &gt;= 2.0.0</li> <li>torchvision &gt;= 0.15.0</li> <li>einops</li> <li>wrapt</li> <li>torchmetrics</li> <li>pyarrow</li> <li>fastparquet&gt;=0.8.1</li> <li>transformers</li> <li>sentence-transformers</li> <li>sentencepiece</li> </ul>"},{"location":"quick_start.html","title":"Quick Start","text":"<p>This is an example of a binary classification with the adult census dataset using a combination of a wide and deep model (in this case a so called <code>deeptabular</code> model) with defaults settings.</p> <pre><code>import numpy as np\nimport torch\nfrom sklearn.model_selection import train_test_split\n\nfrom pytorch_widedeep import Trainer\nfrom pytorch_widedeep.preprocessing import WidePreprocessor, TabPreprocessor\nfrom pytorch_widedeep.models import Wide, TabMlp, WideDeep\nfrom pytorch_widedeep.metrics import Accuracy\nfrom pytorch_widedeep.datasets import load_adult\n\n\ndf = load_adult(as_frame=True)\ndf[\"income_label\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int)\ndf.drop(\"income\", axis=1, inplace=True)\ndf_train, df_test = train_test_split(df, test_size=0.2, stratify=df.income_label)\n\n# Define the 'column set up'\nwide_cols = [\n    \"education\",\n    \"relationship\",\n    \"workclass\",\n    \"occupation\",\n    \"native-country\",\n    \"gender\",\n]\ncrossed_cols = [(\"education\", \"occupation\"), (\"native-country\", \"occupation\")]\n\ncat_embed_cols = [\n    \"workclass\",\n    \"education\",\n    \"marital-status\",\n    \"occupation\",\n    \"relationship\",\n    \"race\",\n    \"gender\",\n    \"capital-gain\",\n    \"capital-loss\",\n    \"native-country\",\n]\ncontinuous_cols = [\"age\", \"hours-per-week\"]\ntarget = \"income_label\"\ntarget = df_train[target].values\n\n# prepare the data\nwide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols)\nX_wide = wide_preprocessor.fit_transform(df_train)\n\ntab_preprocessor = TabPreprocessor(\n    cat_embed_cols=cat_embed_cols, continuous_cols=continuous_cols  # type: ignore[arg-type]\n)\nX_tab = tab_preprocessor.fit_transform(df_train)\n\n# build the model\nwide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1)\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=continuous_cols,\n)\nmodel = WideDeep(wide=wide, deeptabular=tab_mlp)\n\n# train and validate\ntrainer = Trainer(model, objective=\"binary\", metrics=[Accuracy])\ntrainer.fit(\n    X_wide=X_wide,\n    X_tab=X_tab,\n    target=target,\n    n_epochs=5,\n    batch_size=256,\n)\n\n# predict on test\nX_wide_te = wide_preprocessor.transform(df_test)\nX_tab_te = tab_preprocessor.transform(df_test)\npreds = trainer.predict(X_wide=X_wide_te, X_tab=X_tab_te)\n\n# Save and load\n\n# Option 1: this will also save training history and lr history if the\n# LRHistory callback is used\ntrainer.save(path=\"model_weights\", save_state_dict=True)\n\n# Option 2: save as any other torch model\ntorch.save(model.state_dict(), \"model_weights/wd_model.pt\")\n\n# From here in advance, Option 1 or 2 are the same. I assume the user has\n# prepared the data and defined the new model components:\n# 1. Build the model\nmodel_new = WideDeep(wide=wide, deeptabular=tab_mlp)\nmodel_new.load_state_dict(torch.load(\"model_weights/wd_model.pt\"))\n\n# 2. Instantiate the trainer\ntrainer_new = Trainer(model_new, objective=\"binary\")\n\n# 3. Either start the fit or directly predict\npreds = trainer_new.predict(X_wide=X_wide, X_tab=X_tab, batch_size=32)\n</code></pre>"},{"location":"examples/01_preprocessors_and_utils.html","title":"01_preprocessors_and_utils","text":"<p>For example</p> In\u00a0[1]: Copied! <pre>import numpy as np\nimport pandas as pd\nimport pytorch_widedeep as wd\n\nfrom pytorch_widedeep.datasets import load_adult\nfrom pytorch_widedeep.preprocessing import WidePreprocessor\n</pre> import numpy as np import pandas as pd import pytorch_widedeep as wd  from pytorch_widedeep.datasets import load_adult from pytorch_widedeep.preprocessing import WidePreprocessor <pre>/Users/javierrodriguezzaurin/.pyenv/versions/3.10.13/envs/widedeep310/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n  from .autonotebook import tqdm as notebook_tqdm\n</pre> In\u00a0[2]: Copied! <pre>df = load_adult(as_frame=True)\ndf.head()\n</pre> df = load_adult(as_frame=True) df.head() Out[2]: age workclass fnlwgt education educational-num marital-status occupation relationship race gender capital-gain capital-loss hours-per-week native-country income 0 25 Private 226802 11th 7 Never-married Machine-op-inspct Own-child Black Male 0 0 40 United-States &lt;=50K 1 38 Private 89814 HS-grad 9 Married-civ-spouse Farming-fishing Husband White Male 0 0 50 United-States &lt;=50K 2 28 Local-gov 336951 Assoc-acdm 12 Married-civ-spouse Protective-serv Husband White Male 0 0 40 United-States &gt;50K 3 44 Private 160323 Some-college 10 Married-civ-spouse Machine-op-inspct Husband Black Male 7688 0 40 United-States &gt;50K 4 18 ? 103497 Some-college 10 Never-married ? Own-child White Female 0 0 30 United-States &lt;=50K In\u00a0[3]: Copied! <pre>wide_cols = [\n    \"education\",\n    \"relationship\",\n    \"workclass\",\n    \"occupation\",\n    \"native-country\",\n    \"gender\",\n]\ncrossed_cols = [(\"education\", \"occupation\"), (\"native-country\", \"occupation\")]\n</pre> wide_cols = [     \"education\",     \"relationship\",     \"workclass\",     \"occupation\",     \"native-country\",     \"gender\", ] crossed_cols = [(\"education\", \"occupation\"), (\"native-country\", \"occupation\")] In\u00a0[4]: Copied! <pre>wide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols)\nX_wide = wide_preprocessor.fit_transform(df)\n# From here on, any new observation can be prepared by simply running `.transform`\n# new_X_wide = wide_preprocessor.transform(new_df)\n</pre> wide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols) X_wide = wide_preprocessor.fit_transform(df) # From here on, any new observation can be prepared by simply running `.transform` # new_X_wide = wide_preprocessor.transform(new_df) In\u00a0[5]: Copied! <pre>X_wide\n</pre> X_wide Out[5]: <pre>array([[  1,  17,  23, ...,  89,  91, 316],\n       [  2,  18,  23, ...,  89,  92, 317],\n       [  3,  18,  24, ...,  89,  93, 318],\n       ...,\n       [  2,  20,  23, ...,  90, 103, 323],\n       [  2,  17,  23, ...,  89, 103, 323],\n       [  2,  21,  29, ...,  90, 115, 324]])</pre> <p>Note that the label encoding starts from <code>1</code>. This is because it is convenient to leave <code>0</code> for padding, i.e. unknown categories. Let's take from example the first entry</p> In\u00a0[6]: Copied! <pre>X_wide[0]\n</pre> X_wide[0] Out[6]: <pre>array([  1,  17,  23,  32,  47,  89,  91, 316])</pre> In\u00a0[7]: Copied! <pre>wide_preprocessor.inverse_transform(X_wide[:1])\n</pre> wide_preprocessor.inverse_transform(X_wide[:1]) Out[7]: education relationship workclass occupation native-country gender education_occupation native-country_occupation 0 11th Own-child Private Machine-op-inspct United-States Male 11th-Machine-op-inspct United-States-Machine-op-inspct <p>As we can see, <code>wide_preprocessor</code> numerically encodes the <code>wide_cols</code> and the <code>crossed_cols</code>, which can be recovered using the method <code>inverse_transform</code>.</p> In\u00a0[8]: Copied! <pre>from pytorch_widedeep.preprocessing import TabPreprocessor\n</pre> from pytorch_widedeep.preprocessing import TabPreprocessor In\u00a0[9]: Copied! <pre># cat_embed_cols = [(column_name, embed_dim), ...]\ncat_embed_cols = [\n    (\"education\", 10),\n    (\"relationship\", 8),\n    (\"workclass\", 10),\n    (\"occupation\", 10),\n    (\"native-country\", 10),\n]\ncontinuous_cols = [\"age\", \"hours-per-week\"]\n</pre> # cat_embed_cols = [(column_name, embed_dim), ...] cat_embed_cols = [     (\"education\", 10),     (\"relationship\", 8),     (\"workclass\", 10),     (\"occupation\", 10),     (\"native-country\", 10), ] continuous_cols = [\"age\", \"hours-per-week\"] In\u00a0[10]: Copied! <pre>tab_preprocessor = TabPreprocessor(\n    cat_embed_cols=cat_embed_cols,\n    continuous_cols=continuous_cols,\n    cols_to_scale=[\"age\"],  # or scale=True or cols_to_scale=continuous_cols\n)\nX_tab = tab_preprocessor.fit_transform(df)\n# From here on, any new observation can be prepared by simply running `.transform`\n# new_X_deep = deep_preprocessor.transform(new_df)\n</pre> tab_preprocessor = TabPreprocessor(     cat_embed_cols=cat_embed_cols,     continuous_cols=continuous_cols,     cols_to_scale=[\"age\"],  # or scale=True or cols_to_scale=continuous_cols ) X_tab = tab_preprocessor.fit_transform(df) # From here on, any new observation can be prepared by simply running `.transform` # new_X_deep = deep_preprocessor.transform(new_df) In\u00a0[11]: Copied! <pre>X_tab\n</pre> X_tab Out[11]: <pre>array([[ 1.00000000e+00,  1.00000000e+00,  1.00000000e+00, ...,\n         1.00000000e+00, -9.95128932e-01,  4.00000000e+01],\n       [ 2.00000000e+00,  2.00000000e+00,  1.00000000e+00, ...,\n         1.00000000e+00, -4.69415091e-02,  5.00000000e+01],\n       [ 3.00000000e+00,  2.00000000e+00,  2.00000000e+00, ...,\n         1.00000000e+00, -7.76316450e-01,  4.00000000e+01],\n       ...,\n       [ 2.00000000e+00,  4.00000000e+00,  1.00000000e+00, ...,\n         1.00000000e+00,  1.41180837e+00,  4.00000000e+01],\n       [ 2.00000000e+00,  1.00000000e+00,  1.00000000e+00, ...,\n         1.00000000e+00, -1.21394141e+00,  2.00000000e+01],\n       [ 2.00000000e+00,  5.00000000e+00,  7.00000000e+00, ...,\n         1.00000000e+00,  9.74183408e-01,  4.00000000e+01]])</pre> <p>Note that the label encoding starts from <code>1</code>. This is because it is convenient to leave <code>0</code> for padding, i.e. unknown categories. Let's take from example the first entry</p> In\u00a0[12]: Copied! <pre>X_tab[0]\n</pre> X_tab[0] Out[12]: <pre>array([ 1.        ,  1.        ,  1.        ,  1.        ,  1.        ,\n       -0.99512893, 40.        ])</pre> In\u00a0[13]: Copied! <pre>tab_preprocessor.inverse_transform(X_tab[:1])\n</pre> tab_preprocessor.inverse_transform(X_tab[:1]) Out[13]: education relationship workclass occupation native-country age hours-per-week 0 11th Own-child Private Machine-op-inspct United-States 25.0 40.0 <p>The <code>TabPreprocessor</code> will have a series of useful attributes that can later be used when instantiating the different Tabular Models, such us for example, the column indexes (used to slice the tensors, internally in the models) or the categorical embeddings set up</p> In\u00a0[14]: Copied! <pre>tab_preprocessor.column_idx\n</pre> tab_preprocessor.column_idx Out[14]: <pre>{'education': 0,\n 'relationship': 1,\n 'workclass': 2,\n 'occupation': 3,\n 'native-country': 4,\n 'age': 5,\n 'hours-per-week': 6}</pre> In\u00a0[15]: Copied! <pre># column name, num unique, embedding dim\ntab_preprocessor.cat_embed_input\n</pre> # column name, num unique, embedding dim tab_preprocessor.cat_embed_input Out[15]: <pre>[('education', 16, 10),\n ('relationship', 6, 8),\n ('workclass', 9, 10),\n ('occupation', 15, 10),\n ('native-country', 42, 10)]</pre> <p>As I mentioned, there is more one can do, such as for example, quantize (or bucketize) the continuous cols. For this we could use the <code>quantization_setup</code> param. This parameter accepts a number of different inputs and uses <code>pd.cut</code> under the hood to quantize the continuous cols. For more info, please, read the docs. Let's use it here to quantize \"age\" and \"hours-per-week\" in 4 and 5 \"buckets\" respectively</p> In\u00a0[16]: Copied! <pre>quantization_setup = {\n    \"age\": 4,\n    \"hours-per-week\": 5,\n}  # you can also pass a list of floats with the boundaries if you wanted\nquant_tab_preprocessor = TabPreprocessor(\n    cat_embed_cols=cat_embed_cols,\n    continuous_cols=continuous_cols,\n    quantization_setup=quantization_setup,\n)\nqX_tab = quant_tab_preprocessor.fit_transform(df)\n</pre> quantization_setup = {     \"age\": 4,     \"hours-per-week\": 5, }  # you can also pass a list of floats with the boundaries if you wanted quant_tab_preprocessor = TabPreprocessor(     cat_embed_cols=cat_embed_cols,     continuous_cols=continuous_cols,     quantization_setup=quantization_setup, ) qX_tab = quant_tab_preprocessor.fit_transform(df) <pre>/Users/javierrodriguezzaurin/Projects/pytorch-widedeep/pytorch_widedeep/preprocessing/tab_preprocessor.py:358: UserWarning: Continuous columns will not be normalised\n  warnings.warn(\"Continuous columns will not be normalised\")\n</pre> In\u00a0[17]: Copied! <pre>qX_tab\n</pre> qX_tab Out[17]: <pre>array([[1, 1, 1, ..., 1, 1, 2],\n       [2, 2, 1, ..., 1, 2, 3],\n       [3, 2, 2, ..., 1, 1, 2],\n       ...,\n       [2, 4, 1, ..., 1, 3, 2],\n       [2, 1, 1, ..., 1, 1, 1],\n       [2, 5, 7, ..., 1, 2, 2]])</pre> <p>Note that the continuous columns that have been bucketised into quantiles are treated as any other categorical column</p> In\u00a0[18]: Copied! <pre>quant_tab_preprocessor.cat_embed_input\n</pre> quant_tab_preprocessor.cat_embed_input Out[18]: <pre>[('education', 16, 10),\n ('relationship', 6, 8),\n ('workclass', 9, 10),\n ('occupation', 15, 10),\n ('native-country', 42, 10),\n ('age', 4, 4),\n ('hours-per-week', 5, 4)]</pre> <p>Where the column 'age' has now 4 categories, which will be encoded using embeddings of 4 dims. Note that, as any other categorical columns, the categorical \"counter\" starts with 1. This is because all incoming values that are lower/higher than the existing lowest/highest value in the train (or already seen) dataset, will be encoded as 0.</p> In\u00a0[19]: Copied! <pre>np.unique(qX_tab[:, quant_tab_preprocessor.column_idx[\"age\"]])\n</pre> np.unique(qX_tab[:, quant_tab_preprocessor.column_idx[\"age\"]]) Out[19]: <pre>array([1, 2, 3, 4])</pre> <p>Finally, if we now wanted to <code>inverse_transform</code> the transformed array into the original dataframe, we could still do it, but the continuous, bucketized columns will be transformed back to the middle of their quantile/bucket range</p> In\u00a0[20]: Copied! <pre>df_decoded = quant_tab_preprocessor.inverse_transform(qX_tab)\n</pre> df_decoded = quant_tab_preprocessor.inverse_transform(qX_tab) <pre>Note that quantized cols will be turned into the mid point of the corresponding bin\n</pre> In\u00a0[21]: Copied! <pre>df.head(2)\n</pre> df.head(2) Out[21]: age workclass fnlwgt education educational-num marital-status occupation relationship race gender capital-gain capital-loss hours-per-week native-country income 0 25 Private 226802 11th 7 Never-married Machine-op-inspct Own-child Black Male 0 0 40 United-States &lt;=50K 1 38 Private 89814 HS-grad 9 Married-civ-spouse Farming-fishing Husband White Male 0 0 50 United-States &lt;=50K In\u00a0[22]: Copied! <pre>df_decoded.head(2)\n</pre> df_decoded.head(2) Out[22]: education relationship workclass occupation native-country age hours-per-week 0 11th Own-child Private Machine-op-inspct United-States 26.0885 30.4 1 HS-grad Husband Private Farming-fishing United-States 44.3750 50.0 <p>there is one final comment to make regarding to the <code>inverse_transform</code> functionality. As we mentioned before, the encoding <code>0</code> is reserved for values that fall outside the range covered by the data we used to run the <code>fit</code> method. For example</p> In\u00a0[23]: Copied! <pre>df.age.min(), df.age.max()\n</pre> df.age.min(), df.age.max() Out[23]: <pre>(17, 90)</pre> <p>All future age values outside that range will be encoded as 0 and decoded as <code>NaN</code></p> In\u00a0[24]: Copied! <pre>tmp_df = df.head(1).copy()\ntmp_df.loc[:, \"age\"] = 5\n</pre> tmp_df = df.head(1).copy() tmp_df.loc[:, \"age\"] = 5 In\u00a0[25]: Copied! <pre>tmp_df\n</pre> tmp_df Out[25]: age workclass fnlwgt education educational-num marital-status occupation relationship race gender capital-gain capital-loss hours-per-week native-country income 0 5 Private 226802 11th 7 Never-married Machine-op-inspct Own-child Black Male 0 0 40 United-States &lt;=50K In\u00a0[26]: Copied! <pre># quant_tab_preprocessor has already been fitted with a data that has an age range between 17 and 90\ntmp_qX_tab = quant_tab_preprocessor.transform(tmp_df)\n</pre> # quant_tab_preprocessor has already been fitted with a data that has an age range between 17 and 90 tmp_qX_tab = quant_tab_preprocessor.transform(tmp_df) In\u00a0[27]: Copied! <pre>tmp_qX_tab\n</pre> tmp_qX_tab Out[27]: <pre>array([[1, 1, 1, 1, 1, 0, 2]])</pre> In\u00a0[28]: Copied! <pre>quant_tab_preprocessor.inverse_transform(tmp_qX_tab)\n</pre> quant_tab_preprocessor.inverse_transform(tmp_qX_tab) <pre>Note that quantized cols will be turned into the mid point of the corresponding bin\n</pre> Out[28]: education relationship workclass occupation native-country age hours-per-week 0 11th Own-child Private Machine-op-inspct United-States NaN 30.4 In\u00a0[29]: Copied! <pre>from pytorch_widedeep.preprocessing import TextPreprocessor\n</pre> from pytorch_widedeep.preprocessing import TextPreprocessor In\u00a0[30]: Copied! <pre># The airbnb dataset, which you could get from here:\n# http://insideairbnb.com/get-the-data.html, is too big to be included in\n# our datasets module (when including images). Therefore, go there,\n# download it, and use the download_images.py script to get the images\n# and the airbnb_data_processing.py to process the data. We'll find\n# better datasets in the future ;). Note that here we are only using a\n# small sample to illustrate the use, so PLEASE ignore the results, just\n# focus on usage\ndf = pd.read_csv(\"../tmp_data/airbnb/airbnb_sample.csv\")\n</pre> # The airbnb dataset, which you could get from here: # http://insideairbnb.com/get-the-data.html, is too big to be included in # our datasets module (when including images). Therefore, go there, # download it, and use the download_images.py script to get the images # and the airbnb_data_processing.py to process the data. We'll find # better datasets in the future ;). Note that here we are only using a # small sample to illustrate the use, so PLEASE ignore the results, just # focus on usage df = pd.read_csv(\"../tmp_data/airbnb/airbnb_sample.csv\") In\u00a0[31]: Copied! <pre>texts = df.description.tolist()\ntexts[:2]\n</pre> texts = df.description.tolist() texts[:2] Out[31]: <pre>[\"My bright double bedroom with a large window has a relaxed feeling! It comfortably fits one or two and is centrally located just two blocks from Finsbury Park. Enjoy great restaurants in the area and easy access to easy transport tubes, trains and buses. Babies and children of all ages are welcome. Hello Everyone, I'm offering my lovely double bedroom in Finsbury Park area (zone 2) for let in a shared apartment.  You will share the apartment with me and it is fully furnished with a self catering kitchen. Two people can easily sleep well as the room has a queen size bed. I also have a travel cot for a baby for guest with small children.  I will require a deposit up front as a security gesture on both our parts and will be given back to you when you return the keys.  I trust anyone who will be responding to this add would treat my home with care and respect .  Best Wishes  Alina Guest will have access to the self catering kitchen and bathroom. There is the flat is equipped wifi internet,\",\n \"Lots of windows and light.  St Luke's Gardens are at the end of the block, and the river not too far the other way. Ten minutes walk if you go slowly. Buses to everywhere round the corner and shops, restaurants, pubs, the cinema and Waitrose . Bright Chelsea Apartment  This is a bright one bedroom ground floor apartment in an interesting listed building. There is one double bedroom and a living room/kitchen The apartment has a full  bathroom and the kitchen is fully equipped. Two wardrobes are available exclusively for guests and bedside tables and two long drawers. This sunny convenient compact flat is just around the corner from the Waitrose supermarket and all sorts of shops, cinemas, restaurants and pubs.  This is a lovely part of London. There is a fun farmers market in the King's Road at the weekend.  Buses to everywhere are just round the corner, and two underground stations are within ten minutes walk. There is a very nice pub round by St. Luke's gardens, 4 mins slow walk, the \"]</pre> In\u00a0[32]: Copied! <pre>text_preprocessor = TextPreprocessor(text_col=\"description\")\nX_text = text_preprocessor.fit_transform(df)\n# From here on, any new observation can be prepared by simply running `.transform`\n# new_X_text = text_preprocessor.transform(new_df)\n</pre> text_preprocessor = TextPreprocessor(text_col=\"description\") X_text = text_preprocessor.fit_transform(df) # From here on, any new observation can be prepared by simply running `.transform` # new_X_text = text_preprocessor.transform(new_df) <pre>The vocabulary contains 2192 tokens\n</pre> In\u00a0[33]: Copied! <pre>print(X_text[0])\n</pre> print(X_text[0]) <pre>[  29   48   37  367  818   17  910   17  177   15  122  349   53  879\n 1174  126  393   40  911    0   23  228   71  819    9   53   55 1380\n  225   11   18  308   18 1564   10  755    0  942  239   53   55    0\n   11   36 1013  277 1974   70   62   15 1475    9  943    5  251    5\n    0    5    0    5  177   53   37   75   11   10  294  726   32    9\n   42    5   25   12   10   22   12  136  100  145]\n</pre> In\u00a0[34]: Copied! <pre>from pytorch_widedeep.preprocessing import ImagePreprocessor\n</pre> from pytorch_widedeep.preprocessing import ImagePreprocessor In\u00a0[35]: Copied! <pre>image_preprocessor = wd.preprocessing.ImagePreprocessor(\n    img_col=\"id\", img_path=\"../tmp_data/airbnb/property_picture/\"\n)\nX_images = image_preprocessor.fit_transform(df)\n# From here on, any new observation can be prepared by simply running `.transform`\n# new_X_images = image_preprocessor.transform(new_df)\n</pre> image_preprocessor = wd.preprocessing.ImagePreprocessor(     img_col=\"id\", img_path=\"../tmp_data/airbnb/property_picture/\" ) X_images = image_preprocessor.fit_transform(df) # From here on, any new observation can be prepared by simply running `.transform` # new_X_images = image_preprocessor.transform(new_df) <pre>Reading Images from ../tmp_data/airbnb/property_picture/\nResizing\n</pre> <pre>100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 1001/1001 [00:01&lt;00:00, 667.89it/s]\n</pre> <pre>Computing normalisation metrics\n</pre> In\u00a0[36]: Copied! <pre>X_images[0].shape\n</pre> X_images[0].shape Out[36]: <pre>(224, 224, 3)</pre>"},{"location":"examples/01_preprocessors_and_utils.html#processors-and-utils","title":"Processors and Utils\u00b6","text":"<p>Description of the main tools and utilities that one needs to prepare the data for a <code>WideDeep</code> model constructor.</p>"},{"location":"examples/01_preprocessors_and_utils.html#the-preprocessing-module","title":"The <code>preprocessing</code>  module\u00b6","text":"<p>There are 4 preprocessors, corresponding to 4 main components of the <code>WideDeep</code> model. These are</p> <ul> <li><code>WidePreprocessor</code></li> <li><code>TabPreprocessor</code></li> <li><code>TextPreprocessor</code></li> <li><code>ImagePreprocessor</code></li> </ul> <p>Behind the scenes, these preprocessors use a series of helper funcions and classes that are in the <code>utils</code> module. If you were interested please go and have a look to the documentation</p>"},{"location":"examples/01_preprocessors_and_utils.html#1-widepreprocessor","title":"1. WidePreprocessor\u00b6","text":"<p>The <code>wide</code> component of the model is a linear model that in principle, could be implemented as a linear layer receiving the result of on one-hot encoding categorical columns. However, this is not memory efficient. Therefore, we implement a liner layer as an Embedding layer plus a bias. I will explain in a bit more detail later.</p> <p>With that in mind, <code>WidePreprocessor</code> simply encodes the categories numerically so that they are the indexes of the lookup table that is an Embedding layer.</p>"},{"location":"examples/01_preprocessors_and_utils.html#2-tabpreprocessor","title":"2. TabPreprocessor\u00b6","text":"<p>The <code>TabPreprocessor</code> has a lot of different functionalities. Let's explore some of them in detail. In its basic use, the <code>TabPreprocessor</code> simply label encodes the categorical columns and normalises the numerical ones (unless otherwised specified).</p>"},{"location":"examples/01_preprocessors_and_utils.html#3-textpreprocessor","title":"3. TextPreprocessor\u00b6","text":"<p>This preprocessor returns the tokenised, padded sequences that will be directly fed to the stack of LSTMs.</p>"},{"location":"examples/01_preprocessors_and_utils.html#4-imagepreprocessor","title":"4. ImagePreprocessor\u00b6","text":"<p><code>ImagePreprocessor</code> simply resizes the images, being aware of the aspect ratio.</p>"},{"location":"examples/02_model_components.html","title":"02_model_components","text":"In\u00a0[1]: Copied! <pre>import torch\nimport pandas as pd\nimport numpy as np\n\nfrom torch import nn\n</pre> import torch import pandas as pd import numpy as np  from torch import nn In\u00a0[2]: Copied! <pre>df = pd.DataFrame({\"color\": [\"r\", \"b\", \"g\"], \"size\": [\"s\", \"n\", \"l\"]})\ndf.head()\n</pre> df = pd.DataFrame({\"color\": [\"r\", \"b\", \"g\"], \"size\": [\"s\", \"n\", \"l\"]}) df.head() Out[2]: color size 0 r s 1 b n 2 g l <p>one hot encoded, the first observation would be</p> In\u00a0[3]: Copied! <pre>obs_0_oh = (np.array([1.0, 0.0, 0.0, 1.0, 0.0, 0.0])).astype(\"float32\")\n</pre> obs_0_oh = (np.array([1.0, 0.0, 0.0, 1.0, 0.0, 0.0])).astype(\"float32\") <p>if we simply numerically encode (label encode or <code>le</code>) the values:</p> In\u00a0[4]: Copied! <pre>obs_0_le = (np.array([0, 3])).astype(\"int64\")\n</pre> obs_0_le = (np.array([0, 3])).astype(\"int64\") <p>Note that in the functioning implementation of the package we start from 1, saving 0 for padding, i.e. unseen values.</p> <p>Now, let's see if the two implementations are equivalent</p> In\u00a0[5]: Copied! <pre># we have 6 different values. Let's assume we are performing a regression, so pred_dim = 1\nlin = nn.Linear(6, 1)\n</pre> # we have 6 different values. Let's assume we are performing a regression, so pred_dim = 1 lin = nn.Linear(6, 1) In\u00a0[6]: Copied! <pre>emb = nn.Embedding(6, 1)\nemb.weight = nn.Parameter(lin.weight.reshape_as(emb.weight))\n</pre> emb = nn.Embedding(6, 1) emb.weight = nn.Parameter(lin.weight.reshape_as(emb.weight)) In\u00a0[7]: Copied! <pre>lin(torch.tensor(obs_0_oh))\n</pre> lin(torch.tensor(obs_0_oh)) Out[7]: <pre>tensor([-0.5181], grad_fn=&lt;ViewBackward0&gt;)</pre> In\u00a0[8]: Copied! <pre>emb(torch.tensor(obs_0_le)).sum() + lin.bias\n</pre> emb(torch.tensor(obs_0_le)).sum() + lin.bias Out[8]: <pre>tensor([-0.5181], grad_fn=&lt;AddBackward0&gt;)</pre> <p>And this is precisely how the linear model <code>Wide</code> is implemented</p> In\u00a0[9]: Copied! <pre>from pytorch_widedeep.models import Wide\n</pre> from pytorch_widedeep.models import Wide <pre>/Users/javierrodriguezzaurin/.pyenv/versions/3.10.13/envs/widedeep310/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n  from .autonotebook import tqdm as notebook_tqdm\n</pre> In\u00a0[10]: Copied! <pre># ?Wide\n</pre> # ?Wide In\u00a0[11]: Copied! <pre>wide = Wide(input_dim=10, pred_dim=1)\nwide\n</pre> wide = Wide(input_dim=10, pred_dim=1) wide Out[11]: <pre>Wide(\n  (wide_linear): Embedding(11, 1, padding_idx=0)\n)</pre> <p>Note that even though the input dim is 10, the Embedding layer has 11 weights. Again, this is because we save <code>0</code> for padding, which is used for unseen values during the encoding process.</p> <p>As I mentioned, <code>deeptabular</code> has enough complexity on its own and it will be described in a separated notebook. Let's then jump to <code>deeptext</code>.</p> In\u00a0[12]: Copied! <pre>from pytorch_widedeep.preprocessing import TabPreprocessor\nfrom pytorch_widedeep.models import TabMlp\n</pre> from pytorch_widedeep.preprocessing import TabPreprocessor from pytorch_widedeep.models import TabMlp In\u00a0[13]: Copied! <pre>data = {\n    \"cat1\": np.random.choice([\"A\", \"B\", \"C\"], size=20),\n    \"cat2\": np.random.choice([\"X\", \"Y\"], size=20),\n    \"cont1\": np.random.rand(20),\n    \"cont2\": np.random.rand(20),\n}\n\ndf = pd.DataFrame(data)\n</pre> data = {     \"cat1\": np.random.choice([\"A\", \"B\", \"C\"], size=20),     \"cat2\": np.random.choice([\"X\", \"Y\"], size=20),     \"cont1\": np.random.rand(20),     \"cont2\": np.random.rand(20), }  df = pd.DataFrame(data) In\u00a0[14]: Copied! <pre>df.head()\n</pre> df.head() Out[14]: cat1 cat2 cont1 cont2 0 A Y 0.789347 0.561789 1 C X 0.050822 0.061538 2 A Y 0.863784 0.241967 3 C X 0.917848 0.644658 4 C Y 0.042328 0.417303 In\u00a0[15]: Copied! <pre># see the docs for details on all params/options\ntab_preprocessor = TabPreprocessor(\n    cat_embed_cols=[\"cat1\", \"cat2\"],\n    continuous_cols=[\"cont1\", \"cont2\"],\n    embedding_rule=\"fastai\",\n)\n</pre> # see the docs for details on all params/options tab_preprocessor = TabPreprocessor(     cat_embed_cols=[\"cat1\", \"cat2\"],     continuous_cols=[\"cont1\", \"cont2\"],     embedding_rule=\"fastai\", ) In\u00a0[16]: Copied! <pre>X_tab = tab_preprocessor.fit_transform(df)\n</pre> X_tab = tab_preprocessor.fit_transform(df) <pre>/Users/javierrodriguezzaurin/Projects/pytorch-widedeep/pytorch_widedeep/preprocessing/tab_preprocessor.py:358: UserWarning: Continuous columns will not be normalised\n  warnings.warn(\"Continuous columns will not be normalised\")\n</pre> In\u00a0[17]: Copied! <pre># toy example just to build a model.\ntabmlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=tab_preprocessor.continuous_cols,\n    embed_continuous_method=\"standard\",\n    cont_embed_dim=4,\n    mlp_hidden_dims=[8, 4],\n    mlp_linear_first=True,\n)\ntabmlp\n</pre> # toy example just to build a model. tabmlp = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     continuous_cols=tab_preprocessor.continuous_cols,     embed_continuous_method=\"standard\",     cont_embed_dim=4,     mlp_hidden_dims=[8, 4],     mlp_linear_first=True, ) tabmlp Out[17]: <pre>TabMlp(\n  (cat_embed): DiffSizeCatEmbeddings(\n    (embed_layers): ModuleDict(\n      (emb_layer_cat1): Embedding(4, 3, padding_idx=0)\n      (emb_layer_cat2): Embedding(3, 2, padding_idx=0)\n    )\n    (embedding_dropout): Dropout(p=0.0, inplace=False)\n  )\n  (cont_norm): Identity()\n  (cont_embed): ContEmbeddings(\n    INFO: [ContLinear = weight(n_cont_cols, embed_dim) + bias(n_cont_cols, embed_dim)]\n    (linear): ContLinear(n_cont_cols=2, embed_dim=4, embed_dropout=0.0)\n    (dropout): Dropout(p=0.0, inplace=False)\n  )\n  (encoder): MLP(\n    (mlp): Sequential(\n      (dense_layer_0): Sequential(\n        (0): Linear(in_features=13, out_features=8, bias=True)\n        (1): ReLU(inplace=True)\n        (2): Dropout(p=0.1, inplace=False)\n      )\n      (dense_layer_1): Sequential(\n        (0): Linear(in_features=8, out_features=4, bias=True)\n        (1): ReLU(inplace=True)\n        (2): Dropout(p=0.1, inplace=False)\n      )\n    )\n  )\n)</pre> <p>Lets describe a bit the model: first we have what we call a <code>DiffSizeCatEmbeddings</code>, where categorical columns with different number of unique categories will be encoded with embeddings of different dimensions. Then the continuous columns will not be normalised (the normalised layer is just the identity) and they will be embedded via a \"standard\" method, using a so-called <code>ContLinear</code> layer. This layer displays some <code>INFO</code> that tells us what it is (<code>ContLinear = weight(n_cont_cols, embed_dim) + bias(n_cont_cols, embed_dim)]</code>). There are two other options available to embed the continuous cols based on the paper On Embeddings for Numerical Features in Tabular Deep Learning. These are <code>PieceWise</code> and <code>Periodic</code> and all available via the <code>embed_continuous_method</code> param, which can adopt values <code>\"standard\", \"piecewise\"</code> and <code>\"periodic\"</code>. The embedded categorical and continuous columns will be then concatenated ($3 + 2 + (4 * 2) = 13$ input dims) and passed to an MLP.</p> In\u00a0[18]: Copied! <pre>from pytorch_widedeep.models import BasicRNN\n</pre> from pytorch_widedeep.models import BasicRNN In\u00a0[19]: Copied! <pre>basic_rnn = BasicRNN(vocab_size=4, hidden_dim=4, n_layers=1, padding_idx=0, embed_dim=4)\n</pre> basic_rnn = BasicRNN(vocab_size=4, hidden_dim=4, n_layers=1, padding_idx=0, embed_dim=4) <pre>/Users/javierrodriguezzaurin/.pyenv/versions/3.10.13/envs/widedeep310/lib/python3.10/site-packages/torch/nn/modules/rnn.py:82: UserWarning: dropout option adds dropout after all but last recurrent layer, so non-zero dropout expects num_layers greater than 1, but got dropout=0.1 and num_layers=1\n  warnings.warn(\"dropout option adds dropout after all but last \"\n</pre> In\u00a0[20]: Copied! <pre>basic_rnn\n</pre> basic_rnn Out[20]: <pre>BasicRNN(\n  (word_embed): Embedding(4, 4, padding_idx=0)\n  (rnn): LSTM(4, 4, batch_first=True, dropout=0.1)\n  (rnn_mlp): Identity()\n)</pre> <p>You could, if you wanted, add a Fully Connected Head (FC-Head) on top of it</p> In\u00a0[21]: Copied! <pre>from pytorch_widedeep.models import Vision\n</pre> from pytorch_widedeep.models import Vision In\u00a0[22]: Copied! <pre>resnet = Vision(pretrained_model_setup=\"resnet18\", n_trainable=0)\n</pre> resnet = Vision(pretrained_model_setup=\"resnet18\", n_trainable=0) In\u00a0[23]: Copied! <pre>resnet\n</pre> resnet Out[23]: <pre>Vision(\n  (features): Sequential(\n    (0): Conv2d(3, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False)\n    (1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n    (2): ReLU(inplace=True)\n    (3): MaxPool2d(kernel_size=3, stride=2, padding=1, dilation=1, ceil_mode=False)\n    (4): Sequential(\n      (0): BasicBlock(\n        (conv1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n        (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (relu): ReLU(inplace=True)\n        (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n        (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n      )\n      (1): BasicBlock(\n        (conv1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n        (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (relu): ReLU(inplace=True)\n        (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n        (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n      )\n    )\n    (5): Sequential(\n      (0): BasicBlock(\n        (conv1): Conv2d(64, 128, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n        (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (relu): ReLU(inplace=True)\n        (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n        (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (downsample): Sequential(\n          (0): Conv2d(64, 128, kernel_size=(1, 1), stride=(2, 2), bias=False)\n          (1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        )\n      )\n      (1): BasicBlock(\n        (conv1): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n        (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (relu): ReLU(inplace=True)\n        (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n        (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n      )\n    )\n    (6): Sequential(\n      (0): BasicBlock(\n        (conv1): Conv2d(128, 256, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n        (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (relu): ReLU(inplace=True)\n        (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n        (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (downsample): Sequential(\n          (0): Conv2d(128, 256, kernel_size=(1, 1), stride=(2, 2), bias=False)\n          (1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        )\n      )\n      (1): BasicBlock(\n        (conv1): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n        (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (relu): ReLU(inplace=True)\n        (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n        (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n      )\n    )\n    (7): Sequential(\n      (0): BasicBlock(\n        (conv1): Conv2d(256, 512, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n        (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (relu): ReLU(inplace=True)\n        (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n        (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (downsample): Sequential(\n          (0): Conv2d(256, 512, kernel_size=(1, 1), stride=(2, 2), bias=False)\n          (1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        )\n      )\n      (1): BasicBlock(\n        (conv1): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n        (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (relu): ReLU(inplace=True)\n        (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n        (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n      )\n    )\n    (8): AdaptiveAvgPool2d(output_size=(1, 1))\n  )\n)</pre>"},{"location":"examples/02_model_components.html#model-components","title":"Model Components\u00b6","text":"<p>The main components of a <code>WideDeep</code> (i.e. Multimodal) model are tabular data, text and images, which are feed into the model via so called <code>wide</code>, <code>deeptabular</code>, <code>deeptext</code> and <code>deepimage</code> model components</p>"},{"location":"examples/02_model_components.html#1-wide","title":"1. <code>wide</code>\u00b6","text":"<p>The <code>wide</code> component is a Linear layer \"plugged\" into the output neuron(s). Here, the non-linearities are captured via crossed columns. Crossed columns are, quoting directly the paper: \"For binary features, a cross-product transformation (e.g., \u201cAND(gender=female, language=en)\u201d) is 1 if and only if the constituent features (\u201cgender=female\u201d and \u201clanguage=en\u201d) are all 1, and 0 otherwise\".</p> <p>The only particularity of our implementation is that we have implemented the linear layer via an Embedding layer plus a bias. While the implementations are equivalent, the latter is faster and far more memory efficient, since we do not need to one hot encode the categorical features.</p> <p>Let's assume we the following dataset:</p>"},{"location":"examples/02_model_components.html#2-deeptabular","title":"2. <code>deeptabular</code>\u00b6","text":"<p>The <code>deeptabular</code> model alone is what normally would be referred as Deep Learning for tabular data. As mentioned a number of times throughout the library, each component can be used independently. Therefore, if you wanted to use any of the models below alone, it is perfectly possible. There are just a couple of simple requirement that will be covered in a later notebook.</p> <p>By the time of writing, there are a number of models available in <code>pytorch-widedeep</code> to do DL for tabular data. These are:</p> <ol> <li><code>TabMlp</code></li> <li><code>ContextAttentionMLP</code></li> <li><code>SelfAttentionMLP</code></li> <li><code>TabResnet</code></li> <li><code>Tabnet</code></li> <li><code>TabTransformer</code></li> <li><code>FT-Tabransformer</code></li> <li><code>SAINT</code></li> <li><code>TabFastFormer</code></li> <li><code>TabPerceiver</code></li> </ol> <p>Let's have a look to one of them. For more information on each of these models, please, have a look to the documentation</p>"},{"location":"examples/02_model_components.html#3-deeptext","title":"3. <code>deeptext</code>\u00b6","text":"<p>At the time of writing, <code>pytorch-widedeep</code> offers three models that can be passed to <code>WideDeep</code> as the <code>deeptext</code> component. These are:</p> <ol> <li>BasicRNN</li> <li>AttentiveRNN</li> <li>StackedAttentiveRNN</li> </ol> <p>For details on each of these models, please, have a look to the documentation of the package.</p> <p>We will soon integrate with Hugginface, but let me insist. It is perfectly possible to use custom models for each component, please, have a look to the corresponding notebook. In general, simply, build them and pass them as the corresponding parameters. Note that the custom models MUST return a last layer of activations (i.e. not the final prediction) so that  these activations are collected by <code>WideDeep</code> and combined accordingly. In  addition, the models MUST also contain an attribute <code>output_dim</code> with the size of these last layers of activations.</p> <p>Let's have a look to the <code>BasicRNN</code> model</p>"},{"location":"examples/02_model_components.html#4-deepimage","title":"4. <code>deepimage</code>\u00b6","text":"<p>At the time of writing <code>pytorch-widedeep</code> is integrated with torchvision via the <code>Vision</code> class. This means that the it is possible to use a variant of the following architectures:</p> <ol> <li>resnet</li> <li>shufflenet</li> <li>resnext</li> <li>wide_resnet</li> <li>regnet</li> <li>densenet</li> <li>mobilenet</li> <li>mnasnet</li> <li>efficientnet</li> <li>squeezenet</li> </ol> <p>The user can choose which layers will be trainable. Alternatively, in none of these architectures is useful, one could use a simple, fully trained CNN (please see the package documentation) or pass a custom model.</p> <p>let's have a look</p>"},{"location":"examples/03_binary_classification_with_defaults.html","title":"03_binary_classification_with_defaults","text":"In\u00a0[1]: Copied! <pre>import numpy as np\nimport pandas as pd\nimport torch\n\nfrom pytorch_widedeep.preprocessing import WidePreprocessor, TabPreprocessor\nfrom pytorch_widedeep.training import Trainer\nfrom pytorch_widedeep.models import Wide, TabMlp, WideDeep\nfrom pytorch_widedeep.metrics import Accuracy, Precision\nfrom pytorch_widedeep.datasets import load_adult\n</pre> import numpy as np import pandas as pd import torch  from pytorch_widedeep.preprocessing import WidePreprocessor, TabPreprocessor from pytorch_widedeep.training import Trainer from pytorch_widedeep.models import Wide, TabMlp, WideDeep from pytorch_widedeep.metrics import Accuracy, Precision from pytorch_widedeep.datasets import load_adult <pre>/Users/javierrodriguezzaurin/.pyenv/versions/3.10.13/envs/widedeep310/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n  from .autonotebook import tqdm as notebook_tqdm\n</pre> In\u00a0[2]: Copied! <pre>df = load_adult(as_frame=True)\ndf.head()\n</pre> df = load_adult(as_frame=True) df.head() Out[2]: age workclass fnlwgt education educational-num marital-status occupation relationship race gender capital-gain capital-loss hours-per-week native-country income 0 25 Private 226802 11th 7 Never-married Machine-op-inspct Own-child Black Male 0 0 40 United-States &lt;=50K 1 38 Private 89814 HS-grad 9 Married-civ-spouse Farming-fishing Husband White Male 0 0 50 United-States &lt;=50K 2 28 Local-gov 336951 Assoc-acdm 12 Married-civ-spouse Protective-serv Husband White Male 0 0 40 United-States &gt;50K 3 44 Private 160323 Some-college 10 Married-civ-spouse Machine-op-inspct Husband Black Male 7688 0 40 United-States &gt;50K 4 18 ? 103497 Some-college 10 Never-married ? Own-child White Female 0 0 30 United-States &lt;=50K In\u00a0[3]: Copied! <pre># For convenience, we'll replace '-' with '_'\ndf.columns = [c.replace(\"-\", \"_\") for c in df.columns]\n# binary target\ndf[\"income_label\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int)\ndf.drop(\"income\", axis=1, inplace=True)\ndf.head()\n</pre> # For convenience, we'll replace '-' with '_' df.columns = [c.replace(\"-\", \"_\") for c in df.columns] # binary target df[\"income_label\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int) df.drop(\"income\", axis=1, inplace=True) df.head() Out[3]: age workclass fnlwgt education educational_num marital_status occupation relationship race gender capital_gain capital_loss hours_per_week native_country income_label 0 25 Private 226802 11th 7 Never-married Machine-op-inspct Own-child Black Male 0 0 40 United-States 0 1 38 Private 89814 HS-grad 9 Married-civ-spouse Farming-fishing Husband White Male 0 0 50 United-States 0 2 28 Local-gov 336951 Assoc-acdm 12 Married-civ-spouse Protective-serv Husband White Male 0 0 40 United-States 1 3 44 Private 160323 Some-college 10 Married-civ-spouse Machine-op-inspct Husband Black Male 7688 0 40 United-States 1 4 18 ? 103497 Some-college 10 Never-married ? Own-child White Female 0 0 30 United-States 0 In\u00a0[4]: Copied! <pre>df.drop([\"fnlwgt\", \"educational_num\"], axis=1, inplace=True)\n</pre> df.drop([\"fnlwgt\", \"educational_num\"], axis=1, inplace=True) In\u00a0[5]: Copied! <pre># Define wide, crossed and deep tabular columns\nwide_cols = [\n    \"workclass\",\n    \"education\",\n    \"marital_status\",\n    \"occupation\",\n    \"relationship\",\n    \"race\",\n    \"gender\",\n    \"native_country\",\n]\ncrossed_cols = [(\"education\", \"occupation\"), (\"native_country\", \"occupation\")]\n</pre> # Define wide, crossed and deep tabular columns wide_cols = [     \"workclass\",     \"education\",     \"marital_status\",     \"occupation\",     \"relationship\",     \"race\",     \"gender\",     \"native_country\", ] crossed_cols = [(\"education\", \"occupation\"), (\"native_country\", \"occupation\")] In\u00a0[6]: Copied! <pre>cat_embed_cols = [\n    \"workclass\",\n    \"education\",\n    \"marital_status\",\n    \"occupation\",\n    \"relationship\",\n    \"race\",\n    \"gender\",\n    \"capital_gain\",\n    \"capital_loss\",\n    \"native_country\",\n]\ncontinuous_cols = [\"age\", \"hours_per_week\"]\n</pre> cat_embed_cols = [     \"workclass\",     \"education\",     \"marital_status\",     \"occupation\",     \"relationship\",     \"race\",     \"gender\",     \"capital_gain\",     \"capital_loss\",     \"native_country\", ] continuous_cols = [\"age\", \"hours_per_week\"] In\u00a0[7]: Copied! <pre># TARGET\ntarget_col = \"income_label\"\ntarget = df[target_col].values\n</pre> # TARGET target_col = \"income_label\" target = df[target_col].values <p>let's see what the preprocessors do</p> In\u00a0[8]: Copied! <pre># wide\nwide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols)\nX_wide = wide_preprocessor.fit_transform(df)\n</pre> # wide wide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols) X_wide = wide_preprocessor.fit_transform(df) In\u00a0[9]: Copied! <pre># # wide_preprocessor has an attribute called encoding_dict with the encoding dictionary\n# wide_preprocessor.encoding_dict\n</pre> # # wide_preprocessor has an attribute called encoding_dict with the encoding dictionary # wide_preprocessor.encoding_dict In\u00a0[10]: Copied! <pre># deeptabular\ntab_preprocessor = TabPreprocessor(\n    embed_cols=cat_embed_cols,\n    continuous_cols=continuous_cols,\n    cols_to_scale=continuous_cols,\n)\nX_tab = tab_preprocessor.fit_transform(df)\n</pre> # deeptabular tab_preprocessor = TabPreprocessor(     embed_cols=cat_embed_cols,     continuous_cols=continuous_cols,     cols_to_scale=continuous_cols, ) X_tab = tab_preprocessor.fit_transform(df) In\u00a0[11]: Copied! <pre># check the docs to understand the useful attributes that the tab_preprocessor has. For example,\n# as well as an encoding dictionary, tab_preprocessor has an attribute called cat_embed_input\n# that specifies the categortical columns that will be represented as embeddings, the number\n# of different categories per feature, and the dimension of the embeddings as defined by some\n# of the internal rules of thumb that the preprocessor has (have a look to the docs)\ntab_preprocessor.cat_embed_input\n</pre> # check the docs to understand the useful attributes that the tab_preprocessor has. For example, # as well as an encoding dictionary, tab_preprocessor has an attribute called cat_embed_input # that specifies the categortical columns that will be represented as embeddings, the number # of different categories per feature, and the dimension of the embeddings as defined by some # of the internal rules of thumb that the preprocessor has (have a look to the docs) tab_preprocessor.cat_embed_input Out[11]: <pre>[('workclass', 9, 5),\n ('education', 16, 8),\n ('marital_status', 7, 5),\n ('occupation', 15, 7),\n ('relationship', 6, 4),\n ('race', 5, 4),\n ('gender', 2, 2),\n ('capital_gain', 123, 24),\n ('capital_loss', 99, 21),\n ('native_country', 42, 13)]</pre> In\u00a0[12]: Copied! <pre>print(X_wide)\nprint(X_wide.shape)\n</pre> print(X_wide) print(X_wide.shape) <pre>[[  1  10  26 ...  61 103 328]\n [  1  11  27 ...  61 104 329]\n [  2  12  27 ...  61 105 330]\n ...\n [  1  11  28 ...  61 115 335]\n [  1  11  26 ...  61 115 335]\n [  7  11  27 ...  61 127 336]]\n(48842, 10)\n</pre> In\u00a0[13]: Copied! <pre>print(X_tab)\nprint(X_tab.shape)\n</pre> print(X_tab) print(X_tab.shape) <pre>[[ 1.          1.          1.         ...  1.         -0.99512893\n  -0.03408696]\n [ 1.          2.          2.         ...  1.         -0.04694151\n   0.77292975]\n [ 2.          3.          2.         ...  1.         -0.77631645\n  -0.03408696]\n ...\n [ 1.          2.          3.         ...  1.          1.41180837\n  -0.03408696]\n [ 1.          2.          1.         ...  1.         -1.21394141\n  -1.64812038]\n [ 7.          2.          2.         ...  1.          0.97418341\n  -0.03408696]]\n(48842, 12)\n</pre> In\u00a0[14]: Copied! <pre>wide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1)\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    cat_embed_dropout=0.1,\n    continuous_cols=continuous_cols,\n    mlp_hidden_dims=[400, 200],\n    mlp_dropout=0.5,\n    mlp_activation=\"leaky_relu\",\n)\n</pre> wide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1) tab_mlp = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     cat_embed_dropout=0.1,     continuous_cols=continuous_cols,     mlp_hidden_dims=[400, 200],     mlp_dropout=0.5,     mlp_activation=\"leaky_relu\", ) <p>Let's first find out how a linear model performs</p> In\u00a0[15]: Copied! <pre>wide\n</pre> wide Out[15]: <pre>Wide(\n  (wide_linear): Embedding(809, 1, padding_idx=0)\n)</pre> <p>Before being passed to the Trainer, the models need to be \"constructed\" with the <code>WideDeep</code> constructor class. For the particular case of the wide/linear model, not much really happens</p> In\u00a0[16]: Copied! <pre>lin_model = WideDeep(wide=wide)\n</pre> lin_model = WideDeep(wide=wide) In\u00a0[17]: Copied! <pre>lin_model\n</pre> lin_model Out[17]: <pre>WideDeep(\n  (wide): Wide(\n    (wide_linear): Embedding(809, 1, padding_idx=0)\n  )\n)</pre> In\u00a0[18]: Copied! <pre>lin_trainer = Trainer(\n    model=lin_model,\n    objective=\"binary\",\n    optimizers=torch.optim.AdamW(lin_model.parameters(), lr=0.01),\n    metrics=[Accuracy, Precision],\n)\n</pre> lin_trainer = Trainer(     model=lin_model,     objective=\"binary\",     optimizers=torch.optim.AdamW(lin_model.parameters(), lr=0.01),     metrics=[Accuracy, Precision], ) In\u00a0[19]: Copied! <pre>lin_trainer.fit(X_wide=X_wide, target=target, n_epochs=4, batch_size=128, val_split=0.2)\n</pre> lin_trainer.fit(X_wide=X_wide, target=target, n_epochs=4, batch_size=128, val_split=0.2) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 306/306 [00:02&lt;00:00, 109.04it/s, loss=0.426, metrics={'acc': 0.7983, 'prec': 0.6152}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 77/77 [00:00&lt;00:00, 102.46it/s, loss=0.366, metrics={'acc': 0.832, 'prec': 0.6916}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 306/306 [00:02&lt;00:00, 130.27it/s, loss=0.364, metrics={'acc': 0.8305, 'prec': 0.6933}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 77/77 [00:00&lt;00:00, 150.46it/s, loss=0.361, metrics={'acc': 0.8357, 'prec': 0.6982}]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 306/306 [00:02&lt;00:00, 133.19it/s, loss=0.359, metrics={'acc': 0.8329, 'prec': 0.6994}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 77/77 [00:00&lt;00:00, 145.75it/s, loss=0.361, metrics={'acc': 0.836, 'prec': 0.7009}]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 306/306 [00:02&lt;00:00, 130.91it/s, loss=0.358, metrics={'acc': 0.8333, 'prec': 0.7005}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 77/77 [00:00&lt;00:00, 155.08it/s, loss=0.361, metrics={'acc': 0.8364, 'prec': 0.702}]\n</pre> <p>Bear in mind that <code>wide</code> is a linear model where the non-linearities are captured via the crossed columns. For the crossed-columns to be effective one needs proper business knowledge. There is no magic formula to produce them</p> <p>Let's have a look to the tabular model by itself</p> In\u00a0[20]: Copied! <pre>tab_model = WideDeep(deeptabular=tab_mlp)\n</pre> tab_model = WideDeep(deeptabular=tab_mlp) In\u00a0[21]: Copied! <pre>tab_model\n</pre> tab_model Out[21]: <pre>WideDeep(\n  (deeptabular): Sequential(\n    (0): TabMlp(\n      (cat_embed): DiffSizeCatEmbeddings(\n        (embed_layers): ModuleDict(\n          (emb_layer_workclass): Embedding(10, 5, padding_idx=0)\n          (emb_layer_education): Embedding(17, 8, padding_idx=0)\n          (emb_layer_marital_status): Embedding(8, 5, padding_idx=0)\n          (emb_layer_occupation): Embedding(16, 7, padding_idx=0)\n          (emb_layer_relationship): Embedding(7, 4, padding_idx=0)\n          (emb_layer_race): Embedding(6, 4, padding_idx=0)\n          (emb_layer_gender): Embedding(3, 2, padding_idx=0)\n          (emb_layer_capital_gain): Embedding(124, 24, padding_idx=0)\n          (emb_layer_capital_loss): Embedding(100, 21, padding_idx=0)\n          (emb_layer_native_country): Embedding(43, 13, padding_idx=0)\n        )\n        (embedding_dropout): Dropout(p=0.1, inplace=False)\n      )\n      (cont_norm): Identity()\n      (encoder): MLP(\n        (mlp): Sequential(\n          (dense_layer_0): Sequential(\n            (0): Linear(in_features=95, out_features=400, bias=True)\n            (1): LeakyReLU(negative_slope=0.01, inplace=True)\n            (2): Dropout(p=0.5, inplace=False)\n          )\n          (dense_layer_1): Sequential(\n            (0): Linear(in_features=400, out_features=200, bias=True)\n            (1): LeakyReLU(negative_slope=0.01, inplace=True)\n            (2): Dropout(p=0.5, inplace=False)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=200, out_features=1, bias=True)\n  )\n)</pre> <p>You can see how the <code>WideDeep</code> class has added a final prediction layer that collects the activations from the last layer of the model and plugs them into the output neuron. If this was a multiclass classification problem, the prediction dimension (i.e. the size of that final layer) needs to be specified via the <code>pred_dim</code> when instantiating the <code>WideDeep</code> class, as we will see later</p> In\u00a0[22]: Copied! <pre>tab_trainer = Trainer(\n    model=tab_model,\n    objective=\"binary\",\n    optimizers=torch.optim.AdamW(tab_model.parameters(), lr=0.001),\n    metrics=[Accuracy, Precision],\n)\n</pre> tab_trainer = Trainer(     model=tab_model,     objective=\"binary\",     optimizers=torch.optim.AdamW(tab_model.parameters(), lr=0.001),     metrics=[Accuracy, Precision], ) In\u00a0[23]: Copied! <pre>tab_trainer.fit(X_tab=X_tab, target=target, n_epochs=4, batch_size=128, val_split=0.2)\n</pre> tab_trainer.fit(X_tab=X_tab, target=target, n_epochs=4, batch_size=128, val_split=0.2) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 306/306 [00:03&lt;00:00, 97.00it/s, loss=0.37, metrics={'acc': 0.8267, 'prec': 0.7037}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 77/77 [00:00&lt;00:00, 134.91it/s, loss=0.313, metrics={'acc': 0.8588, 'prec': 0.7577}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 306/306 [00:03&lt;00:00, 86.86it/s, loss=0.319, metrics={'acc': 0.8514, 'prec': 0.761}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 77/77 [00:01&lt;00:00, 73.13it/s, loss=0.296, metrics={'acc': 0.8675, 'prec': 0.7685}]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 306/306 [00:03&lt;00:00, 79.07it/s, loss=0.305, metrics={'acc': 0.8574, 'prec': 0.7646}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 77/77 [00:00&lt;00:00, 130.11it/s, loss=0.289, metrics={'acc': 0.8696, 'prec': 0.7765}]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 306/306 [00:03&lt;00:00, 87.39it/s, loss=0.296, metrics={'acc': 0.8622, 'prec': 0.7769}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 77/77 [00:00&lt;00:00, 90.63it/s, loss=0.285, metrics={'acc': 0.8697, 'prec': 0.7741}]\n</pre> <p>The best result I ever obtained with <code>LightGBM</code> on this dataset is 0.8782...so we are pretty close.</p> <p>Let's combine the <code>wide</code> and <code>tab_mlp</code> components see if it helps</p> In\u00a0[24]: Copied! <pre>wide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1)\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    cat_embed_dropout=0.1,\n    continuous_cols=continuous_cols,\n    mlp_hidden_dims=[400, 200],\n    mlp_dropout=0.5,\n    mlp_activation=\"leaky_relu\",\n)\nwd_model = WideDeep(wide=wide, deeptabular=tab_mlp)\n</pre> wide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1) tab_mlp = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     cat_embed_dropout=0.1,     continuous_cols=continuous_cols,     mlp_hidden_dims=[400, 200],     mlp_dropout=0.5,     mlp_activation=\"leaky_relu\", ) wd_model = WideDeep(wide=wide, deeptabular=tab_mlp) In\u00a0[25]: Copied! <pre>wd_trainer = Trainer(\n    model=wd_model,\n    objective=\"binary\",\n    optimizers=torch.optim.AdamW(wd_model.parameters(), lr=0.001),\n    metrics=[Accuracy, Precision],\n)\n</pre> wd_trainer = Trainer(     model=wd_model,     objective=\"binary\",     optimizers=torch.optim.AdamW(wd_model.parameters(), lr=0.001),     metrics=[Accuracy, Precision], ) In\u00a0[26]: Copied! <pre>wd_trainer.fit(\n    X_wide=X_wide, X_tab=X_tab, target=target, n_epochs=4, batch_size=128, val_split=0.2\n)\n</pre> wd_trainer.fit(     X_wide=X_wide, X_tab=X_tab, target=target, n_epochs=4, batch_size=128, val_split=0.2 ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 306/306 [00:03&lt;00:00, 77.48it/s, loss=0.418, metrics={'acc': 0.8047, 'prec': 0.6154}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 77/77 [00:00&lt;00:00, 110.51it/s, loss=0.321, metrics={'acc': 0.8521, 'prec': 0.7059}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 306/306 [00:03&lt;00:00, 82.70it/s, loss=0.333, metrics={'acc': 0.8428, 'prec': 0.7141}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 77/77 [00:00&lt;00:00, 112.52it/s, loss=0.299, metrics={'acc': 0.866, 'prec': 0.7447}]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 306/306 [00:04&lt;00:00, 74.34it/s, loss=0.312, metrics={'acc': 0.8533, 'prec': 0.7404}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 77/77 [00:00&lt;00:00, 89.86it/s, loss=0.29, metrics={'acc': 0.8683, 'prec': 0.7496}]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 306/306 [00:04&lt;00:00, 65.32it/s, loss=0.301, metrics={'acc': 0.8591, 'prec': 0.7542}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 77/77 [00:00&lt;00:00, 86.81it/s, loss=0.286, metrics={'acc': 0.8712, 'prec': 0.7552}]\n</pre> <p>For this particular case, the combination of both did not lead to better results that using just the tab_mlp model, when using only 4 epochs.</p> <p>Note that we have use a <code>TabMlp</code> model, but we could use any other model in the library using the same syntax</p> In\u00a0[27]: Copied! <pre>from pytorch_widedeep.models import TabTransformer\n</pre> from pytorch_widedeep.models import TabTransformer <p>The parameters for the <code>TabTransformer</code> are this</p> <pre><code>column_idx: Dict[str, int],\ncat_embed_input: Optional[List[Tuple[str, int]]] = None,\ncat_embed_dropout: Optional[float] = None,\nuse_cat_bias: Optional[bool] = None,\ncat_embed_activation: Optional[str] = None,\nshared_embed: Optional[bool] = None,\nadd_shared_embed: Optional[bool] = None,\nfrac_shared_embed: Optional[float] = None,\ncontinuous_cols: Optional[List[str]] = None,\ncont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\nembed_continuous: Optional[bool] = None,\nembed_continuous_method: Optional[Literal[\"standard\", \"piecewise\", \"periodic\"]] = None,\ncont_embed_dropout: Optional[float] = None,\ncont_embed_activation: Optional[str] = None,\nquantization_setup: Optional[Dict[str, List[float]]] = None,\nn_frequencies: Optional[int] = None,\nsigma: Optional[float] = None,\nshare_last_layer: Optional[bool] = None,\nfull_embed_dropout: Optional[bool] = None,\ninput_dim: int = 32,\nn_heads: int = 8,\nuse_qkv_bias: bool = False,\nn_blocks: int = 4,\nattn_dropout: float = 0.2,\nff_dropout: float = 0.1,\nff_factor: int = 4,\ntransformer_activation: str = \"gelu\",\nuse_linear_attention: bool = False,\nuse_flash_attention: bool = False,\nmlp_hidden_dims: Optional[List[int]] = None,\nmlp_activation: str = \"relu\",\nmlp_dropout: float = 0.1,\nmlp_batchnorm: bool = False,\nmlp_batchnorm_last: bool = False,\nmlp_linear_first: bool = True,\n</code></pre> <p>Please, see the documentation for details on each one of them, for now let's see how one could use a <code>TabTransformer</code> model in a few lines of code</p> In\u00a0[28]: Copied! <pre>tab_transformer = TabTransformer(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    cat_embed_dropout=0.1,\n    continuous_cols=continuous_cols,\n    embed_continuous_method=\"standard\",\n    cont_norm_layer=\"layernorm\",\n    cont_embed_dropout=0.2,\n    cont_embed_activation=\"leaky_relu\",\n    n_heads=4,\n    ff_dropout=0.2,\n    mlp_dropout=0.5,\n    mlp_activation=\"leaky_relu\",\n    mlp_linear_first=\"True\",\n)\n</pre> tab_transformer = TabTransformer(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     cat_embed_dropout=0.1,     continuous_cols=continuous_cols,     embed_continuous_method=\"standard\",     cont_norm_layer=\"layernorm\",     cont_embed_dropout=0.2,     cont_embed_activation=\"leaky_relu\",     n_heads=4,     ff_dropout=0.2,     mlp_dropout=0.5,     mlp_activation=\"leaky_relu\",     mlp_linear_first=\"True\", ) In\u00a0[29]: Copied! <pre>tab_model = WideDeep(deeptabular=tab_transformer)\n</pre> tab_model = WideDeep(deeptabular=tab_transformer) In\u00a0[30]: Copied! <pre>tab_model\n</pre> tab_model Out[30]: <pre>WideDeep(\n  (deeptabular): Sequential(\n    (0): TabTransformer(\n      (cat_embed): SameSizeCatEmbeddings(\n        (embed): Embedding(325, 32, padding_idx=0)\n        (dropout): Dropout(p=0.1, inplace=False)\n      )\n      (cont_norm): LayerNorm((2,), eps=1e-05, elementwise_affine=True)\n      (cont_embed): ContEmbeddings(\n        INFO: [ContLinear = weight(n_cont_cols, embed_dim) + bias(n_cont_cols, embed_dim)]\n        (linear): ContLinear(n_cont_cols=2, embed_dim=32, embed_dropout=0.2)\n        (activation_fn): LeakyReLU(negative_slope=0.01, inplace=True)\n        (dropout): Dropout(p=0.2, inplace=False)\n      )\n      (encoder): Sequential(\n        (transformer_block0): TransformerEncoder(\n          (attn): MultiHeadedAttention(\n            (dropout): Dropout(p=0.2, inplace=False)\n            (q_proj): Linear(in_features=32, out_features=32, bias=False)\n            (kv_proj): Linear(in_features=32, out_features=64, bias=False)\n            (out_proj): Linear(in_features=32, out_features=32, bias=False)\n          )\n          (ff): FeedForward(\n            (w_1): Linear(in_features=32, out_features=128, bias=True)\n            (w_2): Linear(in_features=128, out_features=32, bias=True)\n            (dropout): Dropout(p=0.2, inplace=False)\n            (activation): GELU(approximate='none')\n          )\n          (attn_addnorm): AddNorm(\n            (dropout): Dropout(p=0.2, inplace=False)\n            (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n          )\n          (ff_addnorm): AddNorm(\n            (dropout): Dropout(p=0.2, inplace=False)\n            (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n          )\n        )\n        (transformer_block1): TransformerEncoder(\n          (attn): MultiHeadedAttention(\n            (dropout): Dropout(p=0.2, inplace=False)\n            (q_proj): Linear(in_features=32, out_features=32, bias=False)\n            (kv_proj): Linear(in_features=32, out_features=64, bias=False)\n            (out_proj): Linear(in_features=32, out_features=32, bias=False)\n          )\n          (ff): FeedForward(\n            (w_1): Linear(in_features=32, out_features=128, bias=True)\n            (w_2): Linear(in_features=128, out_features=32, bias=True)\n            (dropout): Dropout(p=0.2, inplace=False)\n            (activation): GELU(approximate='none')\n          )\n          (attn_addnorm): AddNorm(\n            (dropout): Dropout(p=0.2, inplace=False)\n            (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n          )\n          (ff_addnorm): AddNorm(\n            (dropout): Dropout(p=0.2, inplace=False)\n            (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n          )\n        )\n        (transformer_block2): TransformerEncoder(\n          (attn): MultiHeadedAttention(\n            (dropout): Dropout(p=0.2, inplace=False)\n            (q_proj): Linear(in_features=32, out_features=32, bias=False)\n            (kv_proj): Linear(in_features=32, out_features=64, bias=False)\n            (out_proj): Linear(in_features=32, out_features=32, bias=False)\n          )\n          (ff): FeedForward(\n            (w_1): Linear(in_features=32, out_features=128, bias=True)\n            (w_2): Linear(in_features=128, out_features=32, bias=True)\n            (dropout): Dropout(p=0.2, inplace=False)\n            (activation): GELU(approximate='none')\n          )\n          (attn_addnorm): AddNorm(\n            (dropout): Dropout(p=0.2, inplace=False)\n            (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n          )\n          (ff_addnorm): AddNorm(\n            (dropout): Dropout(p=0.2, inplace=False)\n            (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n          )\n        )\n        (transformer_block3): TransformerEncoder(\n          (attn): MultiHeadedAttention(\n            (dropout): Dropout(p=0.2, inplace=False)\n            (q_proj): Linear(in_features=32, out_features=32, bias=False)\n            (kv_proj): Linear(in_features=32, out_features=64, bias=False)\n            (out_proj): Linear(in_features=32, out_features=32, bias=False)\n          )\n          (ff): FeedForward(\n            (w_1): Linear(in_features=32, out_features=128, bias=True)\n            (w_2): Linear(in_features=128, out_features=32, bias=True)\n            (dropout): Dropout(p=0.2, inplace=False)\n            (activation): GELU(approximate='none')\n          )\n          (attn_addnorm): AddNorm(\n            (dropout): Dropout(p=0.2, inplace=False)\n            (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n          )\n          (ff_addnorm): AddNorm(\n            (dropout): Dropout(p=0.2, inplace=False)\n            (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=384, out_features=1, bias=True)\n  )\n)</pre> In\u00a0[31]: Copied! <pre>tab_trainer = Trainer(\n    model=tab_model,\n    objective=\"binary\",\n    optimizers=torch.optim.AdamW(tab_model.parameters(), lr=0.001),\n    metrics=[Accuracy, Precision],\n)\n</pre> tab_trainer = Trainer(     model=tab_model,     objective=\"binary\",     optimizers=torch.optim.AdamW(tab_model.parameters(), lr=0.001),     metrics=[Accuracy, Precision], ) In\u00a0[32]: Copied! <pre>tab_trainer.fit(X_tab=X_tab, target=target, n_epochs=1, batch_size=128, val_split=0.2)\n</pre> tab_trainer.fit(X_tab=X_tab, target=target, n_epochs=1, batch_size=128, val_split=0.2) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 306/306 [00:11&lt;00:00, 27.57it/s, loss=0.359, metrics={'acc': 0.8334, 'prec': 0.7082}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 77/77 [00:01&lt;00:00, 57.89it/s, loss=0.33, metrics={'acc': 0.8536, 'prec': 0.7152}]\n</pre>"},{"location":"examples/03_binary_classification_with_defaults.html#simple-binary-classification-with-defaults","title":"Simple Binary Classification with defaults\u00b6","text":"<p>In this notebook we will train a Wide and Deep model and simply a \"Deep\" model using the well known adult dataset</p>"},{"location":"examples/03_binary_classification_with_defaults.html#preparing-the-data","title":"Preparing the data\u00b6","text":""},{"location":"examples/03_binary_classification_with_defaults.html#defining-the-model","title":"Defining the model\u00b6","text":""},{"location":"examples/04_regression_with_images_and_text.html","title":"04_regression_with_images_and_text","text":"In\u00a0[1]: Copied! <pre>import numpy as np\nimport pandas as pd\nimport os\nimport torch\nfrom torchvision.transforms import ToTensor, Normalize\n\nfrom pytorch_widedeep import Trainer\nfrom pytorch_widedeep.preprocessing import (\n    WidePreprocessor,\n    TabPreprocessor,\n    TextPreprocessor,\n    ImagePreprocessor,\n)\nfrom pytorch_widedeep.models import (\n    Wide,\n    TabMlp,\n    Vision,\n    BasicRNN,\n    WideDeep,\n)\nfrom pytorch_widedeep.losses import RMSELoss\nfrom pytorch_widedeep.initializers import *\nfrom pytorch_widedeep.callbacks import *\n</pre> import numpy as np import pandas as pd import os import torch from torchvision.transforms import ToTensor, Normalize  from pytorch_widedeep import Trainer from pytorch_widedeep.preprocessing import (     WidePreprocessor,     TabPreprocessor,     TextPreprocessor,     ImagePreprocessor, ) from pytorch_widedeep.models import (     Wide,     TabMlp,     Vision,     BasicRNN,     WideDeep, ) from pytorch_widedeep.losses import RMSELoss from pytorch_widedeep.initializers import * from pytorch_widedeep.callbacks import * <pre>/Users/javierrodriguezzaurin/.pyenv/versions/3.10.13/envs/widedeep310/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n  from .autonotebook import tqdm as notebook_tqdm\n</pre> In\u00a0[2]: Copied! <pre>df = pd.read_csv(\"../tmp_data/airbnb/airbnb_sample.csv\")\ndf.head()\n</pre> df = pd.read_csv(\"../tmp_data/airbnb/airbnb_sample.csv\") df.head() Out[2]: id host_id description host_listings_count host_identity_verified neighbourhood_cleansed latitude longitude is_location_exact property_type ... amenity_wide_entrance amenity_wide_entrance_for_guests amenity_wide_entryway amenity_wide_hallways amenity_wifi amenity_window_guards amenity_wine_cooler security_deposit extra_people yield 0 13913.jpg 54730 My bright double bedroom with a large window h... 4.0 f Islington 51.56802 -0.11121 t apartment ... 1 0 0 0 1 0 0 100.0 15.0 12.00 1 15400.jpg 60302 Lots of windows and light.  St Luke's Gardens ... 1.0 t Kensington and Chelsea 51.48796 -0.16898 t apartment ... 0 0 0 0 1 0 0 150.0 0.0 109.50 2 17402.jpg 67564 Open from June 2018 after a 3-year break, we a... 19.0 t Westminster 51.52098 -0.14002 t apartment ... 0 0 0 0 1 0 0 350.0 10.0 149.65 3 24328.jpg 41759 Artist house, bright high ceiling rooms, priva... 2.0 t Wandsworth 51.47298 -0.16376 t other ... 0 0 0 0 1 0 0 250.0 0.0 215.60 4 25023.jpg 102813 Large, all comforts, 2-bed flat; first floor; ... 1.0 f Wandsworth 51.44687 -0.21874 t apartment ... 0 0 0 0 1 0 0 250.0 11.0 79.35 <p>5 rows \u00d7 223 columns</p> In\u00a0[3]: Copied! <pre># There are a number of columns that are already binary. Therefore, no need to one hot encode them\ncrossed_cols = [(\"property_type\", \"room_type\")]\nalready_dummies = [c for c in df.columns if \"amenity\" in c] + [\"has_house_rules\"]\nwide_cols = [\n    \"is_location_exact\",\n    \"property_type\",\n    \"room_type\",\n    \"host_gender\",\n    \"instant_bookable\",\n] + already_dummies\n\ncat_embed_cols = [(c, 16) for c in df.columns if \"catg\" in c] + [\n    (\"neighbourhood_cleansed\", 64),\n    (\"cancellation_policy\", 16),\n]\ncontinuous_cols = [\"latitude\", \"longitude\", \"security_deposit\", \"extra_people\"]\n\n# text and image colnames\ntext_col = \"description\"\nimg_col = \"id\"\n\n# path to pretrained word embeddings and the images\nword_vectors_path = \"../tmp_data/glove.6B/glove.6B.100d.txt\"\nimg_path = \"../tmp_data/airbnb/property_picture\"\n\n# target\ntarget_col = \"yield\"\n</pre> # There are a number of columns that are already binary. Therefore, no need to one hot encode them crossed_cols = [(\"property_type\", \"room_type\")] already_dummies = [c for c in df.columns if \"amenity\" in c] + [\"has_house_rules\"] wide_cols = [     \"is_location_exact\",     \"property_type\",     \"room_type\",     \"host_gender\",     \"instant_bookable\", ] + already_dummies  cat_embed_cols = [(c, 16) for c in df.columns if \"catg\" in c] + [     (\"neighbourhood_cleansed\", 64),     (\"cancellation_policy\", 16), ] continuous_cols = [\"latitude\", \"longitude\", \"security_deposit\", \"extra_people\"]  # text and image colnames text_col = \"description\" img_col = \"id\"  # path to pretrained word embeddings and the images word_vectors_path = \"../tmp_data/glove.6B/glove.6B.100d.txt\" img_path = \"../tmp_data/airbnb/property_picture\"  # target target_col = \"yield\" In\u00a0[4]: Copied! <pre>target = df[target_col].values\n</pre> target = df[target_col].values In\u00a0[5]: Copied! <pre>wide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols)\nX_wide = wide_preprocessor.fit_transform(df)\n</pre> wide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols) X_wide = wide_preprocessor.fit_transform(df) In\u00a0[6]: Copied! <pre>tab_preprocessor = TabPreprocessor(\n    cat_embed_cols=cat_embed_cols,\n    continuous_cols=continuous_cols,\n)\nX_tab = tab_preprocessor.fit_transform(df)\n</pre> tab_preprocessor = TabPreprocessor(     cat_embed_cols=cat_embed_cols,     continuous_cols=continuous_cols, ) X_tab = tab_preprocessor.fit_transform(df) <pre>/Users/javierrodriguezzaurin/Projects/pytorch-widedeep/pytorch_widedeep/preprocessing/tab_preprocessor.py:358: UserWarning: Continuous columns will not be normalised\n  warnings.warn(\"Continuous columns will not be normalised\")\n</pre> In\u00a0[7]: Copied! <pre>text_preprocessor = TextPreprocessor(\n    word_vectors_path=word_vectors_path, text_col=text_col\n)\nX_text = text_preprocessor.fit_transform(df)\n</pre> text_preprocessor = TextPreprocessor(     word_vectors_path=word_vectors_path, text_col=text_col ) X_text = text_preprocessor.fit_transform(df) <pre>The vocabulary contains 2192 tokens\nIndexing word vectors...\nLoaded 400000 word vectors\nPreparing embeddings matrix...\n2175 words in the vocabulary had ../tmp_data/glove.6B/glove.6B.100d.txt vectors and appear more than 5 times\n</pre> In\u00a0[8]: Copied! <pre>image_processor = ImagePreprocessor(img_col=img_col, img_path=img_path)\nX_images = image_processor.fit_transform(df)\n</pre> image_processor = ImagePreprocessor(img_col=img_col, img_path=img_path) X_images = image_processor.fit_transform(df) <pre>Reading Images from ../tmp_data/airbnb/property_picture\nResizing\n</pre> <pre>100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 1001/1001 [00:01&lt;00:00, 638.00it/s]\n</pre> <pre>Computing normalisation metrics\n</pre> In\u00a0[9]: Copied! <pre># Linear model\nwide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1)\n\n# DeepDense: 2 Dense layers\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=continuous_cols,\n    mlp_hidden_dims=[128, 64],\n    mlp_dropout=0.1,\n)\n\n# DeepText: a stack of 2 LSTMs\nbasic_rnn = BasicRNN(\n    vocab_size=len(text_preprocessor.vocab.itos),\n    embed_matrix=text_preprocessor.embedding_matrix,\n    n_layers=2,\n    hidden_dim=64,\n    rnn_dropout=0.5,\n)\n\n# Pretrained Resnet 18\nresnet = Vision(pretrained_model_setup=\"resnet18\", n_trainable=4)\n</pre> # Linear model wide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1)  # DeepDense: 2 Dense layers tab_mlp = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     continuous_cols=continuous_cols,     mlp_hidden_dims=[128, 64],     mlp_dropout=0.1, )  # DeepText: a stack of 2 LSTMs basic_rnn = BasicRNN(     vocab_size=len(text_preprocessor.vocab.itos),     embed_matrix=text_preprocessor.embedding_matrix,     n_layers=2,     hidden_dim=64,     rnn_dropout=0.5, )  # Pretrained Resnet 18 resnet = Vision(pretrained_model_setup=\"resnet18\", n_trainable=4) <p>Combine them all with the \"collector\" class <code>WideDeep</code></p> In\u00a0[10]: Copied! <pre>model = WideDeep(\n    wide=wide,\n    deeptabular=tab_mlp,\n    deeptext=basic_rnn,\n    deepimage=resnet,\n    head_hidden_dims=[256, 128],\n)\n</pre> model = WideDeep(     wide=wide,     deeptabular=tab_mlp,     deeptext=basic_rnn,     deepimage=resnet,     head_hidden_dims=[256, 128], ) In\u00a0[11]: Copied! <pre>trainer = Trainer(model, objective=\"rmse\")\n</pre> trainer = Trainer(model, objective=\"rmse\") In\u00a0[12]: Copied! <pre>trainer.fit(\n    X_wide=X_wide,\n    X_tab=X_tab,\n    X_text=X_text,\n    X_img=X_images,\n    target=target,\n    n_epochs=1,\n    batch_size=32,\n    val_split=0.2,\n)\n</pre> trainer.fit(     X_wide=X_wide,     X_tab=X_tab,     X_text=X_text,     X_img=X_images,     target=target,     n_epochs=1,     batch_size=32,     val_split=0.2, ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 25/25 [00:19&lt;00:00,  1.28it/s, loss=115]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 7/7 [00:04&lt;00:00,  1.62it/s, loss=94.1]\n</pre> <p>Both, the Text and Image components allow FC-heads on their own (have a look to the documentation).</p> <p>Now let's go \"kaggle crazy\". Let's use different optimizers, initializers and schedulers for different components. Moreover, let's use a different learning rate for different parameter groups, for the <code>deeptabular</code> component</p> In\u00a0[13]: Copied! <pre>deep_params = []\nfor childname, child in model.named_children():\n    if childname == \"deeptabular\":\n        for n, p in child.named_parameters():\n            if \"embed_layer\" in n:\n                deep_params.append({\"params\": p, \"lr\": 1e-4})\n            else:\n                deep_params.append({\"params\": p, \"lr\": 1e-3})\n</pre> deep_params = [] for childname, child in model.named_children():     if childname == \"deeptabular\":         for n, p in child.named_parameters():             if \"embed_layer\" in n:                 deep_params.append({\"params\": p, \"lr\": 1e-4})             else:                 deep_params.append({\"params\": p, \"lr\": 1e-3}) In\u00a0[14]: Copied! <pre>wide_opt = torch.optim.Adam(model.wide.parameters(), lr=0.03)\ndeep_opt = torch.optim.Adam(deep_params)\ntext_opt = torch.optim.AdamW(model.deeptext.parameters())\nimg_opt = torch.optim.AdamW(model.deepimage.parameters())\nhead_opt = torch.optim.Adam(model.deephead.parameters())\n</pre> wide_opt = torch.optim.Adam(model.wide.parameters(), lr=0.03) deep_opt = torch.optim.Adam(deep_params) text_opt = torch.optim.AdamW(model.deeptext.parameters()) img_opt = torch.optim.AdamW(model.deepimage.parameters()) head_opt = torch.optim.Adam(model.deephead.parameters()) In\u00a0[15]: Copied! <pre>wide_sch = torch.optim.lr_scheduler.StepLR(wide_opt, step_size=5)\ndeep_sch = torch.optim.lr_scheduler.MultiStepLR(deep_opt, milestones=[3, 8])\ntext_sch = torch.optim.lr_scheduler.StepLR(text_opt, step_size=5)\nimg_sch = torch.optim.lr_scheduler.MultiStepLR(deep_opt, milestones=[3, 8])\nhead_sch = torch.optim.lr_scheduler.StepLR(head_opt, step_size=5)\n</pre> wide_sch = torch.optim.lr_scheduler.StepLR(wide_opt, step_size=5) deep_sch = torch.optim.lr_scheduler.MultiStepLR(deep_opt, milestones=[3, 8]) text_sch = torch.optim.lr_scheduler.StepLR(text_opt, step_size=5) img_sch = torch.optim.lr_scheduler.MultiStepLR(deep_opt, milestones=[3, 8]) head_sch = torch.optim.lr_scheduler.StepLR(head_opt, step_size=5) In\u00a0[16]: Copied! <pre># remember, one optimizer per model components, for lr_schedures and initializers is not neccesary\noptimizers = {\n    \"wide\": wide_opt,\n    \"deeptabular\": deep_opt,\n    \"deeptext\": text_opt,\n    \"deepimage\": img_opt,\n    \"deephead\": head_opt,\n}\nschedulers = {\n    \"wide\": wide_sch,\n    \"deeptabular\": deep_sch,\n    \"deeptext\": text_sch,\n    \"deepimage\": img_sch,\n    \"deephead\": head_sch,\n}\n\n# Now...we have used pretrained word embeddings, so you do not want to\n# initialise these  embeddings. However you might still want to initialise the\n# other layers in the DeepText component. No probs, you can do that with the\n# parameter pattern and your knowledge on regular  expressions. Here we are\n# telling to the KaimingNormal initializer to NOT touch the  parameters whose\n# name contains the string word_embed.\ninitializers = {\n    \"wide\": KaimingNormal,\n    \"deeptabular\": KaimingNormal,\n    \"deeptext\": KaimingNormal(pattern=r\"^(?!.*word_embed).*$\"),\n    \"deepimage\": KaimingNormal,\n}\n\nmean = [0.406, 0.456, 0.485]  # BGR\nstd = [0.225, 0.224, 0.229]  # BGR\ntransforms = [ToTensor, Normalize(mean=mean, std=std)]\ncallbacks = [\n    LRHistory(n_epochs=10),\n    EarlyStopping,\n    ModelCheckpoint(filepath=\"model_weights/wd_out\"),\n]\n</pre> # remember, one optimizer per model components, for lr_schedures and initializers is not neccesary optimizers = {     \"wide\": wide_opt,     \"deeptabular\": deep_opt,     \"deeptext\": text_opt,     \"deepimage\": img_opt,     \"deephead\": head_opt, } schedulers = {     \"wide\": wide_sch,     \"deeptabular\": deep_sch,     \"deeptext\": text_sch,     \"deepimage\": img_sch,     \"deephead\": head_sch, }  # Now...we have used pretrained word embeddings, so you do not want to # initialise these  embeddings. However you might still want to initialise the # other layers in the DeepText component. No probs, you can do that with the # parameter pattern and your knowledge on regular  expressions. Here we are # telling to the KaimingNormal initializer to NOT touch the  parameters whose # name contains the string word_embed. initializers = {     \"wide\": KaimingNormal,     \"deeptabular\": KaimingNormal,     \"deeptext\": KaimingNormal(pattern=r\"^(?!.*word_embed).*$\"),     \"deepimage\": KaimingNormal, }  mean = [0.406, 0.456, 0.485]  # BGR std = [0.225, 0.224, 0.229]  # BGR transforms = [ToTensor, Normalize(mean=mean, std=std)] callbacks = [     LRHistory(n_epochs=10),     EarlyStopping,     ModelCheckpoint(filepath=\"model_weights/wd_out\"), ] In\u00a0[17]: Copied! <pre>trainer = Trainer(\n    model,\n    objective=\"rmse\",\n    initializers=initializers,\n    optimizers=optimizers,\n    lr_schedulers=schedulers,\n    callbacks=callbacks,\n    transforms=transforms,\n)\n</pre> trainer = Trainer(     model,     objective=\"rmse\",     initializers=initializers,     optimizers=optimizers,     lr_schedulers=schedulers,     callbacks=callbacks,     transforms=transforms, ) <pre>/Users/javierrodriguezzaurin/Projects/pytorch-widedeep/pytorch_widedeep/initializers.py:34: UserWarning: No initializer found for deephead\n  warnings.warn(\n</pre> In\u00a0[18]: Copied! <pre>trainer.fit(\n    X_wide=X_wide,\n    X_tab=X_tab,\n    X_text=X_text,\n    X_img=X_images,\n    target=target,\n    n_epochs=1,\n    batch_size=32,\n    val_split=0.2,\n)\n</pre> trainer.fit(     X_wide=X_wide,     X_tab=X_tab,     X_text=X_text,     X_img=X_images,     target=target,     n_epochs=1,     batch_size=32,     val_split=0.2, ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 25/25 [00:19&lt;00:00,  1.25it/s, loss=101]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 7/7 [00:04&lt;00:00,  1.62it/s, loss=90.6]\n</pre> <pre>Model weights after training corresponds to the those of the final epoch which might not be the best performing weights. Use the 'ModelCheckpoint' Callback to restore the best epoch weights.\n</pre> <p>we have only run one epoch, but let's check that the LRHistory callback records the lr values for each group</p> In\u00a0[19]: Copied! <pre>trainer.lr_history\n</pre> trainer.lr_history Out[19]: <pre>{'lr_wide_0': [0.03, 0.03],\n 'lr_deeptabular_0': [0.0001, 0.0001],\n 'lr_deeptabular_1': [0.0001, 0.0001],\n 'lr_deeptabular_2': [0.0001, 0.0001],\n 'lr_deeptabular_3': [0.0001, 0.0001],\n 'lr_deeptabular_4': [0.0001, 0.0001],\n 'lr_deeptabular_5': [0.0001, 0.0001],\n 'lr_deeptabular_6': [0.0001, 0.0001],\n 'lr_deeptabular_7': [0.0001, 0.0001],\n 'lr_deeptabular_8': [0.0001, 0.0001],\n 'lr_deeptabular_9': [0.001, 0.001],\n 'lr_deeptabular_10': [0.001, 0.001],\n 'lr_deeptabular_11': [0.001, 0.001],\n 'lr_deeptabular_12': [0.001, 0.001],\n 'lr_deeptext_0': [0.001, 0.001],\n 'lr_deepimage_0': [0.001, 0.001],\n 'lr_deephead_0': [0.001, 0.001]}</pre>"},{"location":"examples/04_regression_with_images_and_text.html#regression-with-images-and-text","title":"Regression with Images and Text\u00b6","text":"<p>In this notebook we will go through a series of examples on how to combine all Wide &amp; Deep components.</p> <p>To that aim I will use the Airbnb listings dataset for London, which you can download from here. I use this dataset simply because it contains tabular data, images and text.</p> <p>I have taken a sample of 1000 listings to keep the data tractable in this notebook. Also, I have preprocessed the data and prepared it for this exercise. All preprocessing steps can be found in the notebook <code>airbnb_data_preprocessing.ipynb</code> in this <code>examples</code> folder.</p>"},{"location":"examples/04_regression_with_images_and_text.html#regression-with-the-defaults","title":"Regression with the defaults\u00b6","text":"<p>The set up</p>"},{"location":"examples/04_regression_with_images_and_text.html#prepare-the-data","title":"Prepare the data\u00b6","text":"<p>I will focus here on how to prepare the data and run the model. Check notebooks 1 and 2 to see what's going on behind the scences</p> <p>Preparing the data is rather simple</p>"},{"location":"examples/04_regression_with_images_and_text.html#build-the-model-components","title":"Build the model components\u00b6","text":""},{"location":"examples/04_regression_with_images_and_text.html#build-the-trainer-and-fit","title":"Build the trainer and fit\u00b6","text":""},{"location":"examples/05_save_and_load_model_and_artifacts.html","title":"05_save_and_load_model_and_artifacts","text":"In\u00a0[1]: Copied! <pre>import pickle\nimport numpy as np\nimport pandas as pd\nimport torch\nimport shutil\n\nfrom pytorch_widedeep.preprocessing import TabPreprocessor\nfrom pytorch_widedeep.training import Trainer\nfrom pytorch_widedeep.callbacks import EarlyStopping, ModelCheckpoint, LRHistory\nfrom pytorch_widedeep.models import TabMlp, WideDeep\nfrom pytorch_widedeep.metrics import Accuracy\nfrom pytorch_widedeep.datasets import load_adult\nfrom sklearn.model_selection import train_test_split\n</pre> import pickle import numpy as np import pandas as pd import torch import shutil  from pytorch_widedeep.preprocessing import TabPreprocessor from pytorch_widedeep.training import Trainer from pytorch_widedeep.callbacks import EarlyStopping, ModelCheckpoint, LRHistory from pytorch_widedeep.models import TabMlp, WideDeep from pytorch_widedeep.metrics import Accuracy from pytorch_widedeep.datasets import load_adult from sklearn.model_selection import train_test_split <pre>/Users/javierrodriguezzaurin/.pyenv/versions/3.10.13/envs/widedeep310/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n  from .autonotebook import tqdm as notebook_tqdm\n</pre> In\u00a0[2]: Copied! <pre>df = load_adult(as_frame=True)\ndf.head()\n</pre> df = load_adult(as_frame=True) df.head() Out[2]: age workclass fnlwgt education educational-num marital-status occupation relationship race gender capital-gain capital-loss hours-per-week native-country income 0 25 Private 226802 11th 7 Never-married Machine-op-inspct Own-child Black Male 0 0 40 United-States &lt;=50K 1 38 Private 89814 HS-grad 9 Married-civ-spouse Farming-fishing Husband White Male 0 0 50 United-States &lt;=50K 2 28 Local-gov 336951 Assoc-acdm 12 Married-civ-spouse Protective-serv Husband White Male 0 0 40 United-States &gt;50K 3 44 Private 160323 Some-college 10 Married-civ-spouse Machine-op-inspct Husband Black Male 7688 0 40 United-States &gt;50K 4 18 ? 103497 Some-college 10 Never-married ? Own-child White Female 0 0 30 United-States &lt;=50K In\u00a0[3]: Copied! <pre># For convenience, we'll replace '-' with '_'\ndf.columns = [c.replace(\"-\", \"_\") for c in df.columns]\n# binary target\ndf[\"target\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int)\ndf.drop(\"income\", axis=1, inplace=True)\ndf.head()\n</pre> # For convenience, we'll replace '-' with '_' df.columns = [c.replace(\"-\", \"_\") for c in df.columns] # binary target df[\"target\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int) df.drop(\"income\", axis=1, inplace=True) df.head() Out[3]: age workclass fnlwgt education educational_num marital_status occupation relationship race gender capital_gain capital_loss hours_per_week native_country target 0 25 Private 226802 11th 7 Never-married Machine-op-inspct Own-child Black Male 0 0 40 United-States 0 1 38 Private 89814 HS-grad 9 Married-civ-spouse Farming-fishing Husband White Male 0 0 50 United-States 0 2 28 Local-gov 336951 Assoc-acdm 12 Married-civ-spouse Protective-serv Husband White Male 0 0 40 United-States 1 3 44 Private 160323 Some-college 10 Married-civ-spouse Machine-op-inspct Husband Black Male 7688 0 40 United-States 1 4 18 ? 103497 Some-college 10 Never-married ? Own-child White Female 0 0 30 United-States 0 In\u00a0[4]: Copied! <pre>train, valid = train_test_split(df, test_size=0.2, stratify=df.target)\n# the test data will be used lately as if it was \"fresh\", new data coming after some time...\nvalid, test = train_test_split(valid, test_size=0.5, stratify=valid.target)\n</pre> train, valid = train_test_split(df, test_size=0.2, stratify=df.target) # the test data will be used lately as if it was \"fresh\", new data coming after some time... valid, test = train_test_split(valid, test_size=0.5, stratify=valid.target) In\u00a0[5]: Copied! <pre>print(f\"train shape: {train.shape}\")\nprint(f\"valid shape: {valid.shape}\")\nprint(f\"test shape: {test.shape}\")\n</pre> print(f\"train shape: {train.shape}\") print(f\"valid shape: {valid.shape}\") print(f\"test shape: {test.shape}\") <pre>train shape: (39073, 15)\nvalid shape: (4884, 15)\ntest shape: (4885, 15)\n</pre> In\u00a0[6]: Copied! <pre>cat_embed_cols = [\n    \"workclass\",\n    \"education\",\n    \"marital_status\",\n    \"occupation\",\n    \"relationship\",\n    \"race\",\n    \"gender\",\n    \"capital_gain\",\n    \"capital_loss\",\n    \"native_country\",\n]\ncontinuous_cols = [\"age\", \"hours_per_week\"]\n</pre> cat_embed_cols = [     \"workclass\",     \"education\",     \"marital_status\",     \"occupation\",     \"relationship\",     \"race\",     \"gender\",     \"capital_gain\",     \"capital_loss\",     \"native_country\", ] continuous_cols = [\"age\", \"hours_per_week\"] In\u00a0[7]: Copied! <pre>tab_preprocessor = TabPreprocessor(\n    embed_cols=cat_embed_cols,\n    continuous_cols=continuous_cols,\n)\nX_tab_train = tab_preprocessor.fit_transform(train)\ny_train = train.target.values\nX_tab_valid = tab_preprocessor.transform(valid)\ny_valid = valid.target.values\n</pre> tab_preprocessor = TabPreprocessor(     embed_cols=cat_embed_cols,     continuous_cols=continuous_cols, ) X_tab_train = tab_preprocessor.fit_transform(train) y_train = train.target.values X_tab_valid = tab_preprocessor.transform(valid) y_valid = valid.target.values <pre>/Users/javierrodriguezzaurin/Projects/pytorch-widedeep/pytorch_widedeep/preprocessing/tab_preprocessor.py:358: UserWarning: Continuous columns will not be normalised\n  warnings.warn(\"Continuous columns will not be normalised\")\n</pre> In\u00a0[8]: Copied! <pre>tab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    cat_embed_dropout=0.1,\n    continuous_cols=continuous_cols,\n    cont_norm_layer=\"layernorm\",\n    embed_continuous_method=\"standard\",\n    cont_embed_dim=8,\n    mlp_hidden_dims=[64, 32],\n    mlp_dropout=0.2,\n    mlp_activation=\"leaky_relu\",\n)\nmodel = WideDeep(deeptabular=tab_mlp)\n</pre> tab_mlp = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     cat_embed_dropout=0.1,     continuous_cols=continuous_cols,     cont_norm_layer=\"layernorm\",     embed_continuous_method=\"standard\",     cont_embed_dim=8,     mlp_hidden_dims=[64, 32],     mlp_dropout=0.2,     mlp_activation=\"leaky_relu\", ) model = WideDeep(deeptabular=tab_mlp) In\u00a0[9]: Copied! <pre>model\n</pre> model Out[9]: <pre>WideDeep(\n  (deeptabular): Sequential(\n    (0): TabMlp(\n      (cat_embed): DiffSizeCatEmbeddings(\n        (embed_layers): ModuleDict(\n          (emb_layer_workclass): Embedding(10, 5, padding_idx=0)\n          (emb_layer_education): Embedding(17, 8, padding_idx=0)\n          (emb_layer_marital_status): Embedding(8, 5, padding_idx=0)\n          (emb_layer_occupation): Embedding(16, 7, padding_idx=0)\n          (emb_layer_relationship): Embedding(7, 4, padding_idx=0)\n          (emb_layer_race): Embedding(6, 4, padding_idx=0)\n          (emb_layer_gender): Embedding(3, 2, padding_idx=0)\n          (emb_layer_capital_gain): Embedding(122, 23, padding_idx=0)\n          (emb_layer_capital_loss): Embedding(97, 21, padding_idx=0)\n          (emb_layer_native_country): Embedding(43, 13, padding_idx=0)\n        )\n        (embedding_dropout): Dropout(p=0.1, inplace=False)\n      )\n      (cont_norm): LayerNorm((2,), eps=1e-05, elementwise_affine=True)\n      (cont_embed): ContEmbeddings(\n        INFO: [ContLinear = weight(n_cont_cols, embed_dim) + bias(n_cont_cols, embed_dim)]\n        (linear): ContLinear(n_cont_cols=2, embed_dim=8, embed_dropout=0.0)\n        (dropout): Dropout(p=0.0, inplace=False)\n      )\n      (encoder): MLP(\n        (mlp): Sequential(\n          (dense_layer_0): Sequential(\n            (0): Linear(in_features=108, out_features=64, bias=True)\n            (1): LeakyReLU(negative_slope=0.01, inplace=True)\n            (2): Dropout(p=0.2, inplace=False)\n          )\n          (dense_layer_1): Sequential(\n            (0): Linear(in_features=64, out_features=32, bias=True)\n            (1): LeakyReLU(negative_slope=0.01, inplace=True)\n            (2): Dropout(p=0.2, inplace=False)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=32, out_features=1, bias=True)\n  )\n)</pre> In\u00a0[10]: Copied! <pre>early_stopping = EarlyStopping()\nmodel_checkpoint = ModelCheckpoint(\n    filepath=\"tmp_dir/adult_tabmlp_model\",\n    save_best_only=True,\n    verbose=1,\n    max_save=1,\n)\n\ntrainer = Trainer(\n    model,\n    objective=\"binary\",\n    callbacks=[early_stopping, model_checkpoint],\n    metrics=[Accuracy],\n)\n\ntrainer.fit(\n    X_train={\"X_tab\": X_tab_train, \"target\": y_train},\n    X_val={\"X_tab\": X_tab_valid, \"target\": y_valid},\n    n_epochs=4,\n    batch_size=256,\n)\n</pre> early_stopping = EarlyStopping() model_checkpoint = ModelCheckpoint(     filepath=\"tmp_dir/adult_tabmlp_model\",     save_best_only=True,     verbose=1,     max_save=1, )  trainer = Trainer(     model,     objective=\"binary\",     callbacks=[early_stopping, model_checkpoint],     metrics=[Accuracy], )  trainer.fit(     X_train={\"X_tab\": X_tab_train, \"target\": y_train},     X_val={\"X_tab\": X_tab_valid, \"target\": y_valid},     n_epochs=4,     batch_size=256, ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:02&lt;00:00, 76.25it/s, loss=0.452, metrics={'acc': 0.7867}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 20/20 [00:00&lt;00:00, 125.36it/s, loss=0.335, metrics={'acc': 0.8532}]\n</pre> <pre>\nEpoch 1: val_loss improved from inf to 0.33532 Saving model to tmp_dir/adult_tabmlp_model_1.p\n</pre> <pre>epoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 76.98it/s, loss=0.355, metrics={'acc': 0.8401}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 20/20 [00:00&lt;00:00, 106.51it/s, loss=0.303, metrics={'acc': 0.8665}]\n</pre> <pre>\nEpoch 2: val_loss improved from 0.33532 to 0.30273 Saving model to tmp_dir/adult_tabmlp_model_2.p\n</pre> <pre>epoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 82.71it/s, loss=0.332, metrics={'acc': 0.849}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 20/20 [00:00&lt;00:00, 107.80it/s, loss=0.288, metrics={'acc': 0.8757}]\n</pre> <pre>\nEpoch 3: val_loss improved from 0.30273 to 0.28791 Saving model to tmp_dir/adult_tabmlp_model_3.p\n</pre> <pre>epoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 79.02it/s, loss=0.32, metrics={'acc': 0.8541}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 20/20 [00:00&lt;00:00, 127.07it/s, loss=0.282, metrics={'acc': 0.8763}]</pre> <pre>\nEpoch 4: val_loss improved from 0.28791 to 0.28238 Saving model to tmp_dir/adult_tabmlp_model_4.p\nModel weights restored to best epoch: 4\n</pre> <pre>\n</pre> In\u00a0[11]: Copied! <pre>torch.save(model, \"tmp_dir/model_saved_option_1.pt\")\n</pre> torch.save(model, \"tmp_dir/model_saved_option_1.pt\") In\u00a0[12]: Copied! <pre>torch.save(model.state_dict(), \"tmp_dir/model_state_dict_saved_option_1.pt\")\n</pre> torch.save(model.state_dict(), \"tmp_dir/model_state_dict_saved_option_1.pt\") In\u00a0[13]: Copied! <pre>trainer.save(path=\"tmp_dir/\", model_filename=\"model_saved_option_2.pt\")\n</pre> trainer.save(path=\"tmp_dir/\", model_filename=\"model_saved_option_2.pt\") <p>or the state dict</p> In\u00a0[14]: Copied! <pre>trainer.save(\n    path=\"tmp_dir/\",\n    model_filename=\"model_state_dict_saved_option_2.pt\",\n    save_state_dict=True,\n)\n</pre> trainer.save(     path=\"tmp_dir/\",     model_filename=\"model_state_dict_saved_option_2.pt\",     save_state_dict=True, ) In\u00a0[15]: Copied! <pre>%%bash\n\nls tmp_dir/\n</pre> %%bash  ls tmp_dir/ <pre>adult_tabmlp_model_4.p\nhistory\nmodel_saved_option_1.pt\nmodel_saved_option_2.pt\nmodel_state_dict_saved_option_1.pt\nmodel_state_dict_saved_option_2.pt\n</pre> In\u00a0[16]: Copied! <pre>%%bash\n\nls tmp_dir/history/\n</pre> %%bash  ls tmp_dir/history/ <pre>train_eval_history.json\n</pre> <p>Note that since we have used the <code>ModelCheckpoint</code> Callback, <code>adult_tabmlp_model_2.p</code> is the model state dict of the model at epoch 2, i.e. same as <code>model_state_dict_saved_option_1.p</code> or <code>model_state_dict_saved_option_2.p</code>.</p> In\u00a0[17]: Copied! <pre>with open(\"tmp_dir/tab_preproc.pkl\", \"wb\") as dp:\n    pickle.dump(tab_preprocessor, dp)\n</pre> with open(\"tmp_dir/tab_preproc.pkl\", \"wb\") as dp:     pickle.dump(tab_preprocessor, dp) In\u00a0[18]: Copied! <pre>with open(\"tmp_dir/eary_stop.pkl\", \"wb\") as es:\n    pickle.dump(early_stopping, es)\n</pre> with open(\"tmp_dir/eary_stop.pkl\", \"wb\") as es:     pickle.dump(early_stopping, es) In\u00a0[19]: Copied! <pre>%%bash\n\nls tmp_dir/\n</pre> %%bash  ls tmp_dir/ <pre>adult_tabmlp_model_4.p\neary_stop.pkl\nhistory\nmodel_saved_option_1.pt\nmodel_saved_option_2.pt\nmodel_state_dict_saved_option_1.pt\nmodel_state_dict_saved_option_2.pt\ntab_preproc.pkl\n</pre> <p>And that is pretty much all you need to resume training or directly predict, let's see</p> In\u00a0[20]: Copied! <pre>test.head()\n</pre> test.head() Out[20]: age workclass fnlwgt education educational_num marital_status occupation relationship race gender capital_gain capital_loss hours_per_week native_country target 10103 43 Private 198282 HS-grad 9 Married-civ-spouse Craft-repair Husband White Male 0 0 40 United-States 1 31799 20 Private 228686 11th 7 Married-civ-spouse Other-service Husband White Male 0 0 40 United-States 0 19971 26 Private 291968 HS-grad 9 Married-civ-spouse Transport-moving Husband White Male 0 0 44 United-States 0 3039 48 Private 175958 Bachelors 13 Divorced Prof-specialty Not-in-family White Male 0 0 30 United-States 0 20725 18 Private 232024 11th 7 Never-married Machine-op-inspct Own-child White Male 0 0 55 United-States 0 In\u00a0[21]: Copied! <pre>with open(\"tmp_dir/tab_preproc.pkl\", \"rb\") as tp:\n    tab_preprocessor_new = pickle.load(tp)\n</pre> with open(\"tmp_dir/tab_preproc.pkl\", \"rb\") as tp:     tab_preprocessor_new = pickle.load(tp) In\u00a0[22]: Copied! <pre>X_test_tab = tab_preprocessor_new.transform(test)\ny_test = test.target\n</pre> X_test_tab = tab_preprocessor_new.transform(test) y_test = test.target In\u00a0[23]: Copied! <pre>tab_mlp_new = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    cat_embed_dropout=0.1,\n    continuous_cols=continuous_cols,\n    cont_norm_layer=\"layernorm\",\n    embed_continuous_method=\"standard\",\n    cont_embed_dim=8,\n    mlp_hidden_dims=[64, 32],\n    mlp_dropout=0.2,\n    mlp_activation=\"leaky_relu\",\n)\nnew_model = WideDeep(deeptabular=tab_mlp)\n</pre> tab_mlp_new = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     cat_embed_dropout=0.1,     continuous_cols=continuous_cols,     cont_norm_layer=\"layernorm\",     embed_continuous_method=\"standard\",     cont_embed_dim=8,     mlp_hidden_dims=[64, 32],     mlp_dropout=0.2,     mlp_activation=\"leaky_relu\", ) new_model = WideDeep(deeptabular=tab_mlp) In\u00a0[24]: Copied! <pre>new_model.load_state_dict(torch.load(\"tmp_dir/model_state_dict_saved_option_2.pt\"))\n</pre> new_model.load_state_dict(torch.load(\"tmp_dir/model_state_dict_saved_option_2.pt\")) Out[24]: <pre>&lt;All keys matched successfully&gt;</pre> In\u00a0[25]: Copied! <pre>trainer = Trainer(\n    model,\n    objective=\"binary\",\n)\n</pre> trainer = Trainer(     model,     objective=\"binary\", ) In\u00a0[26]: Copied! <pre>preds = trainer.predict(X_tab=X_test_tab, batch_size=32)\n</pre> preds = trainer.predict(X_tab=X_test_tab, batch_size=32) <pre>predict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:00&lt;00:00, 309.83it/s]\n</pre> In\u00a0[27]: Copied! <pre>from sklearn.metrics import accuracy_score\n</pre> from sklearn.metrics import accuracy_score In\u00a0[28]: Copied! <pre>accuracy_score(y_test, preds)\n</pre> accuracy_score(y_test, preds) Out[28]: <pre>0.8595701125895598</pre> In\u00a0[29]: Copied! <pre>shutil.rmtree(\"tmp_dir/\")\n</pre> shutil.rmtree(\"tmp_dir/\")"},{"location":"examples/05_save_and_load_model_and_artifacts.html#save-and-load-model-and-artifacts","title":"Save and load model and artifacts\u00b6","text":"<p>In this notebook I will show the different options to save and load a model, as well as some additional objects produced during training.</p> <p>On a given day, you train a model...</p>"},{"location":"examples/05_save_and_load_model_and_artifacts.html#save-model-option-1","title":"Save model: option 1\u00b6","text":"<p>save (and load) a model as you woud do with any other torch model</p>"},{"location":"examples/05_save_and_load_model_and_artifacts.html#save-model-option-2","title":"Save model: option 2\u00b6","text":"<p>use the <code>trainer</code>. The <code>trainer</code> will also save the training history and the learning rate history (if learning rate schedulers are used)</p>"},{"location":"examples/05_save_and_load_model_and_artifacts.html#save-preprocessors-and-callbacks","title":"Save preprocessors and callbacks\u00b6","text":"<p>...just pickle them</p>"},{"location":"examples/05_save_and_load_model_and_artifacts.html#run-new-experiment-prepare-new-dataset-load-model-and-predict","title":"Run New experiment: prepare new dataset, load model, and predict\u00b6","text":""},{"location":"examples/06_finetune_and_warmup.html","title":"06_finetune_and_warmup","text":"In\u00a0[1]: Copied! <pre>import numpy as np\nimport pandas as pd\nimport torch\n\nfrom pytorch_widedeep import Trainer\nfrom pytorch_widedeep.preprocessing import WidePreprocessor, TabPreprocessor\nfrom pytorch_widedeep.models import Wide, TabMlp, TabResnet, WideDeep\nfrom pytorch_widedeep.metrics import Accuracy\nfrom pytorch_widedeep.datasets import load_adult\n</pre> import numpy as np import pandas as pd import torch  from pytorch_widedeep import Trainer from pytorch_widedeep.preprocessing import WidePreprocessor, TabPreprocessor from pytorch_widedeep.models import Wide, TabMlp, TabResnet, WideDeep from pytorch_widedeep.metrics import Accuracy from pytorch_widedeep.datasets import load_adult <pre>/Users/javierrodriguezzaurin/.pyenv/versions/3.10.13/envs/widedeep310/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n  from .autonotebook import tqdm as notebook_tqdm\n</pre> In\u00a0[2]: Copied! <pre>df = load_adult(as_frame=True)\n# For convenience, we'll replace '-' with '_'\ndf.columns = [c.replace(\"-\", \"_\") for c in df.columns]\n# binary target\ndf[\"income_label\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int)\ndf.drop(\"income\", axis=1, inplace=True)\ndf.head()\n</pre> df = load_adult(as_frame=True) # For convenience, we'll replace '-' with '_' df.columns = [c.replace(\"-\", \"_\") for c in df.columns] # binary target df[\"income_label\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int) df.drop(\"income\", axis=1, inplace=True) df.head() Out[2]: age workclass fnlwgt education educational_num marital_status occupation relationship race gender capital_gain capital_loss hours_per_week native_country income_label 0 25 Private 226802 11th 7 Never-married Machine-op-inspct Own-child Black Male 0 0 40 United-States 0 1 38 Private 89814 HS-grad 9 Married-civ-spouse Farming-fishing Husband White Male 0 0 50 United-States 0 2 28 Local-gov 336951 Assoc-acdm 12 Married-civ-spouse Protective-serv Husband White Male 0 0 40 United-States 1 3 44 Private 160323 Some-college 10 Married-civ-spouse Machine-op-inspct Husband Black Male 7688 0 40 United-States 1 4 18 ? 103497 Some-college 10 Never-married ? Own-child White Female 0 0 30 United-States 0 In\u00a0[3]: Copied! <pre># Define wide, crossed and deep tabular columns\nwide_cols = [\n    \"workclass\",\n    \"education\",\n    \"marital_status\",\n    \"occupation\",\n    \"relationship\",\n    \"race\",\n    \"gender\",\n    \"native_country\",\n]\ncrossed_cols = [(\"education\", \"occupation\"), (\"native_country\", \"occupation\")]\ncat_embed_cols = [\n    \"workclass\",\n    \"education\",\n    \"marital_status\",\n    \"occupation\",\n    \"relationship\",\n    \"race\",\n    \"gender\",\n    \"capital_gain\",\n    \"capital_loss\",\n    \"native_country\",\n]\ncontinuous_cols = [\"age\", \"hours_per_week\"]\ntarget_col = \"income_label\"\ntarget = df[target_col].values\n</pre> # Define wide, crossed and deep tabular columns wide_cols = [     \"workclass\",     \"education\",     \"marital_status\",     \"occupation\",     \"relationship\",     \"race\",     \"gender\",     \"native_country\", ] crossed_cols = [(\"education\", \"occupation\"), (\"native_country\", \"occupation\")] cat_embed_cols = [     \"workclass\",     \"education\",     \"marital_status\",     \"occupation\",     \"relationship\",     \"race\",     \"gender\",     \"capital_gain\",     \"capital_loss\",     \"native_country\", ] continuous_cols = [\"age\", \"hours_per_week\"] target_col = \"income_label\" target = df[target_col].values In\u00a0[4]: Copied! <pre># TARGET\ntarget = df[target_col].values\n\n# WIDE\nwide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols)\nX_wide = wide_preprocessor.fit_transform(df)\n\n# DEEP\ntab_preprocessor = TabPreprocessor(\n    cat_embed_cols=cat_embed_cols, continuous_cols=continuous_cols\n)\nX_tab = tab_preprocessor.fit_transform(df)\n</pre> # TARGET target = df[target_col].values  # WIDE wide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols) X_wide = wide_preprocessor.fit_transform(df)  # DEEP tab_preprocessor = TabPreprocessor(     cat_embed_cols=cat_embed_cols, continuous_cols=continuous_cols ) X_tab = tab_preprocessor.fit_transform(df) <pre>/Users/javierrodriguezzaurin/Projects/pytorch-widedeep/pytorch_widedeep/preprocessing/tab_preprocessor.py:358: UserWarning: Continuous columns will not be normalised\n  warnings.warn(\"Continuous columns will not be normalised\")\n</pre> In\u00a0[5]: Copied! <pre>wide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1)\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    cat_embed_dropout=0.1,\n    continuous_cols=continuous_cols,\n    embed_continuous_method=\"standard\",\n    cont_embed_dim=8,\n    mlp_hidden_dims=[64, 32],\n    mlp_dropout=0.2,\n    mlp_activation=\"leaky_relu\",\n)\nmodel = WideDeep(wide=wide, deeptabular=tab_mlp)\n</pre> wide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1) tab_mlp = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     cat_embed_dropout=0.1,     continuous_cols=continuous_cols,     embed_continuous_method=\"standard\",     cont_embed_dim=8,     mlp_hidden_dims=[64, 32],     mlp_dropout=0.2,     mlp_activation=\"leaky_relu\", ) model = WideDeep(wide=wide, deeptabular=tab_mlp) In\u00a0[6]: Copied! <pre>trainer = Trainer(\n    model,\n    objective=\"binary\",\n    optimizers=torch.optim.Adam(model.parameters(), lr=0.01),\n    metrics=[Accuracy],\n)\n</pre> trainer = Trainer(     model,     objective=\"binary\",     optimizers=torch.optim.Adam(model.parameters(), lr=0.01),     metrics=[Accuracy], ) In\u00a0[7]: Copied! <pre>trainer.fit(\n    X_wide=X_wide, X_tab=X_tab, target=target, n_epochs=2, val_split=0.2, batch_size=256\n)\n</pre> trainer.fit(     X_wide=X_wide, X_tab=X_tab, target=target, n_epochs=2, val_split=0.2, batch_size=256 ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:02&lt;00:00, 74.26it/s, loss=0.399, metrics={'acc': 0.8163}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 39/39 [00:00&lt;00:00, 91.03it/s, loss=0.296, metrics={'acc': 0.8677}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 81.31it/s, loss=0.3, metrics={'acc': 0.8614}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 39/39 [00:00&lt;00:00, 106.45it/s, loss=0.285, metrics={'acc': 0.8721}]\n</pre> In\u00a0[8]: Copied! <pre>trainer.save(path=\"models_dir/\", save_state_dict=True, model_filename=\"model_1.pt\")\n</pre> trainer.save(path=\"models_dir/\", save_state_dict=True, model_filename=\"model_1.pt\") <p>Now time goes by...and we want to fine-tune the model to another, new dataset (for example, a dataset that is identical to the one you used to train the previous model but for another country).</p> <p>Here I will use the same dataset just for illustration purposes, but the flow would be identical to that new dataset</p> In\u00a0[9]: Copied! <pre>wide_1 = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1)\ntab_mlp_1 = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    cat_embed_dropout=0.1,\n    continuous_cols=continuous_cols,\n    embed_continuous_method=\"standard\",\n    cont_embed_dim=8,\n    mlp_hidden_dims=[64, 32],\n    mlp_dropout=0.2,\n    mlp_activation=\"leaky_relu\",\n)\nmodel_1 = WideDeep(wide=wide, deeptabular=tab_mlp)\n</pre> wide_1 = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1) tab_mlp_1 = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     cat_embed_dropout=0.1,     continuous_cols=continuous_cols,     embed_continuous_method=\"standard\",     cont_embed_dim=8,     mlp_hidden_dims=[64, 32],     mlp_dropout=0.2,     mlp_activation=\"leaky_relu\", ) model_1 = WideDeep(wide=wide, deeptabular=tab_mlp) In\u00a0[10]: Copied! <pre>model_1.load_state_dict(torch.load(\"models_dir/model_1.pt\"))\n</pre> model_1.load_state_dict(torch.load(\"models_dir/model_1.pt\")) Out[10]: <pre>&lt;All keys matched successfully&gt;</pre> In\u00a0[11]: Copied! <pre>trainer_1 = Trainer(model_1, objective=\"binary\", metrics=[Accuracy])\n</pre> trainer_1 = Trainer(model_1, objective=\"binary\", metrics=[Accuracy]) In\u00a0[12]: Copied! <pre>trainer_1.fit(\n    X_wide=X_wide,\n    X_tab=X_tab,\n    target=target,\n    n_epochs=2,\n    batch_size=256,\n    finetune=True,\n    finetune_epochs=2,\n)\n</pre> trainer_1.fit(     X_wide=X_wide,     X_tab=X_tab,     target=target,     n_epochs=2,     batch_size=256,     finetune=True,     finetune_epochs=2, ) <pre>Training wide for 2 epochs\n</pre> <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 191/191 [00:01&lt;00:00, 97.37it/s, loss=0.39, metrics={'acc': 0.8152}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 191/191 [00:01&lt;00:00, 104.04it/s, loss=0.359, metrics={'acc': 0.824}]\n</pre> <pre>Training deeptabular for 2 epochs\n</pre> <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 191/191 [00:02&lt;00:00, 83.83it/s, loss=0.297, metrics={'acc': 0.8365}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 191/191 [00:02&lt;00:00, 82.78it/s, loss=0.283, metrics={'acc': 0.8445}]\n</pre> <pre>Fine-tuning (or warmup) of individual components completed. Training the whole model for 2 epochs\n</pre> <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 191/191 [00:02&lt;00:00, 72.84it/s, loss=0.281, metrics={'acc': 0.8716}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 191/191 [00:02&lt;00:00, 77.46it/s, loss=0.273, metrics={'acc': 0.8744}]\n</pre> <p>Note that, as I describe above, in scenario 2, we can just use this to warm up models before they joined training begins:</p> In\u00a0[13]: Copied! <pre>wide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1)\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    cat_embed_dropout=0.1,\n    continuous_cols=continuous_cols,\n    embed_continuous_method=\"standard\",\n    cont_embed_dim=8,\n    mlp_hidden_dims=[64, 32],\n    mlp_dropout=0.2,\n    mlp_activation=\"leaky_relu\",\n)\nmodel = WideDeep(wide=wide, deeptabular=tab_mlp)\n</pre> wide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1) tab_mlp = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     cat_embed_dropout=0.1,     continuous_cols=continuous_cols,     embed_continuous_method=\"standard\",     cont_embed_dim=8,     mlp_hidden_dims=[64, 32],     mlp_dropout=0.2,     mlp_activation=\"leaky_relu\", ) model = WideDeep(wide=wide, deeptabular=tab_mlp) In\u00a0[14]: Copied! <pre>trainer_2 = Trainer(model, objective=\"binary\", metrics=[Accuracy])\n</pre> trainer_2 = Trainer(model, objective=\"binary\", metrics=[Accuracy]) In\u00a0[15]: Copied! <pre>trainer_2.fit(\n    X_wide=X_wide,\n    X_tab=X_tab,\n    target=target,\n    val_split=0.1,\n    warmup=True,\n    warmup_epochs=2,\n    n_epochs=2,\n    batch_size=256,\n)\n</pre> trainer_2.fit(     X_wide=X_wide,     X_tab=X_tab,     target=target,     val_split=0.1,     warmup=True,     warmup_epochs=2,     n_epochs=2,     batch_size=256, ) <pre>Training wide for 2 epochs\n</pre> <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:01&lt;00:00, 102.49it/s, loss=0.52, metrics={'acc': 0.7519}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:01&lt;00:00, 98.15it/s, loss=0.381, metrics={'acc': 0.7891}]\n</pre> <pre>Training deeptabular for 2 epochs\n</pre> <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:02&lt;00:00, 82.97it/s, loss=0.356, metrics={'acc': 0.8043}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:02&lt;00:00, 80.27it/s, loss=0.295, metrics={'acc': 0.8195}]\n</pre> <pre>Fine-tuning (or warmup) of individual components completed. Training the whole model for 2 epochs\n</pre> <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:02&lt;00:00, 77.27it/s, loss=0.291, metrics={'acc': 0.8667}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 20/20 [00:00&lt;00:00, 89.57it/s, loss=0.289, metrics={'acc': 0.8665}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:02&lt;00:00, 72.69it/s, loss=0.283, metrics={'acc': 0.8693}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 20/20 [00:00&lt;00:00, 91.81it/s, loss=0.284, metrics={'acc': 0.869}]\n</pre> <p>We need to explicitly indicate</p> <ol> <li><p>That we want fine-tune</p> </li> <li><p>The components that we want to individually fine-tune</p> </li> <li><p>In case of gradual fine-tuning, the routine (\"felbo\" or \"howard\")</p> </li> <li><p>The layers we want to fine-tune.</p> </li> </ol> <p>For example</p> In\u00a0[16]: Copied! <pre>wide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1)\ntab_resnet = TabResnet(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    cat_embed_dropout=0.1,\n    continuous_cols=continuous_cols,\n    blocks_dims=[200, 200, 200],\n)\nmodel = WideDeep(wide=wide, deeptabular=tab_resnet)\n</pre> wide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1) tab_resnet = TabResnet(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     cat_embed_dropout=0.1,     continuous_cols=continuous_cols,     blocks_dims=[200, 200, 200], ) model = WideDeep(wide=wide, deeptabular=tab_resnet) In\u00a0[17]: Copied! <pre>model\n</pre> model Out[17]: <pre>WideDeep(\n  (wide): Wide(\n    (wide_linear): Embedding(809, 1, padding_idx=0)\n  )\n  (deeptabular): Sequential(\n    (0): TabResnet(\n      (cat_embed): DiffSizeCatEmbeddings(\n        (embed_layers): ModuleDict(\n          (emb_layer_workclass): Embedding(10, 5, padding_idx=0)\n          (emb_layer_education): Embedding(17, 8, padding_idx=0)\n          (emb_layer_marital_status): Embedding(8, 5, padding_idx=0)\n          (emb_layer_occupation): Embedding(16, 7, padding_idx=0)\n          (emb_layer_relationship): Embedding(7, 4, padding_idx=0)\n          (emb_layer_race): Embedding(6, 4, padding_idx=0)\n          (emb_layer_gender): Embedding(3, 2, padding_idx=0)\n          (emb_layer_capital_gain): Embedding(124, 24, padding_idx=0)\n          (emb_layer_capital_loss): Embedding(100, 21, padding_idx=0)\n          (emb_layer_native_country): Embedding(43, 13, padding_idx=0)\n        )\n        (embedding_dropout): Dropout(p=0.1, inplace=False)\n      )\n      (cont_norm): Identity()\n      (encoder): DenseResnet(\n        (dense_resnet): Sequential(\n          (lin_inp): Linear(in_features=95, out_features=200, bias=False)\n          (bn_inp): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n          (block_0): BasicBlock(\n            (lin1): Linear(in_features=200, out_features=200, bias=False)\n            (bn1): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n            (leaky_relu): LeakyReLU(negative_slope=0.01, inplace=True)\n            (dp): Dropout(p=0.1, inplace=False)\n            (lin2): Linear(in_features=200, out_features=200, bias=False)\n            (bn2): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n          )\n          (block_1): BasicBlock(\n            (lin1): Linear(in_features=200, out_features=200, bias=False)\n            (bn1): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n            (leaky_relu): LeakyReLU(negative_slope=0.01, inplace=True)\n            (dp): Dropout(p=0.1, inplace=False)\n            (lin2): Linear(in_features=200, out_features=200, bias=False)\n            (bn2): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=200, out_features=1, bias=True)\n  )\n)</pre> <p>let's first train as usual</p> In\u00a0[18]: Copied! <pre>trainer_3 = Trainer(model, objective=\"binary\", metrics=[Accuracy])\n</pre> trainer_3 = Trainer(model, objective=\"binary\", metrics=[Accuracy]) In\u00a0[19]: Copied! <pre>trainer_3.fit(\n    X_wide=X_wide, X_tab=X_tab, target=target, val_split=0.1, n_epochs=2, batch_size=256\n)\n</pre> trainer_3.fit(     X_wide=X_wide, X_tab=X_tab, target=target, val_split=0.1, n_epochs=2, batch_size=256 ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:03&lt;00:00, 54.23it/s, loss=0.382, metrics={'acc': 0.8239}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 20/20 [00:00&lt;00:00, 84.72it/s, loss=0.331, metrics={'acc': 0.8526}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:03&lt;00:00, 54.35it/s, loss=0.33, metrics={'acc': 0.8465}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 20/20 [00:00&lt;00:00, 68.15it/s, loss=0.312, metrics={'acc': 0.8604}]\n</pre> In\u00a0[20]: Copied! <pre>trainer_3.save(path=\"models_dir\", save_state_dict=True, model_filename=\"model_3.pt\")\n</pre> trainer_3.save(path=\"models_dir\", save_state_dict=True, model_filename=\"model_3.pt\") <p>Now we are going to fine-tune the model components, and in the case of the <code>deeptabular</code> component, we will fine-tune the resnet-blocks and the linear layer but NOT the embeddings.</p> <p>For this, we need to access the model component's children: <code>deeptabular</code> $\\rightarrow$ <code>tab_resnet</code> $\\rightarrow$ <code>dense_resnet</code> $\\rightarrow$ <code>blocks</code></p> In\u00a0[21]: Copied! <pre>wide_3 = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1)\ntab_resnet_3 = TabResnet(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    cat_embed_dropout=0.1,\n    continuous_cols=continuous_cols,\n    blocks_dims=[200, 200, 200],\n)\nmodel_3 = WideDeep(wide=wide, deeptabular=tab_resnet)\n</pre> wide_3 = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1) tab_resnet_3 = TabResnet(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     cat_embed_dropout=0.1,     continuous_cols=continuous_cols,     blocks_dims=[200, 200, 200], ) model_3 = WideDeep(wide=wide, deeptabular=tab_resnet) In\u00a0[22]: Copied! <pre>model_3.load_state_dict(torch.load(\"models_dir/model_3.pt\"))\n</pre> model_3.load_state_dict(torch.load(\"models_dir/model_3.pt\")) Out[22]: <pre>&lt;All keys matched successfully&gt;</pre> In\u00a0[23]: Copied! <pre>model_3\n</pre> model_3 Out[23]: <pre>WideDeep(\n  (wide): Wide(\n    (wide_linear): Embedding(809, 1, padding_idx=0)\n  )\n  (deeptabular): Sequential(\n    (0): TabResnet(\n      (cat_embed): DiffSizeCatEmbeddings(\n        (embed_layers): ModuleDict(\n          (emb_layer_workclass): Embedding(10, 5, padding_idx=0)\n          (emb_layer_education): Embedding(17, 8, padding_idx=0)\n          (emb_layer_marital_status): Embedding(8, 5, padding_idx=0)\n          (emb_layer_occupation): Embedding(16, 7, padding_idx=0)\n          (emb_layer_relationship): Embedding(7, 4, padding_idx=0)\n          (emb_layer_race): Embedding(6, 4, padding_idx=0)\n          (emb_layer_gender): Embedding(3, 2, padding_idx=0)\n          (emb_layer_capital_gain): Embedding(124, 24, padding_idx=0)\n          (emb_layer_capital_loss): Embedding(100, 21, padding_idx=0)\n          (emb_layer_native_country): Embedding(43, 13, padding_idx=0)\n        )\n        (embedding_dropout): Dropout(p=0.1, inplace=False)\n      )\n      (cont_norm): Identity()\n      (encoder): DenseResnet(\n        (dense_resnet): Sequential(\n          (lin_inp): Linear(in_features=95, out_features=200, bias=False)\n          (bn_inp): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n          (block_0): BasicBlock(\n            (lin1): Linear(in_features=200, out_features=200, bias=False)\n            (bn1): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n            (leaky_relu): LeakyReLU(negative_slope=0.01, inplace=True)\n            (dp): Dropout(p=0.1, inplace=False)\n            (lin2): Linear(in_features=200, out_features=200, bias=False)\n            (bn2): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n          )\n          (block_1): BasicBlock(\n            (lin1): Linear(in_features=200, out_features=200, bias=False)\n            (bn1): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n            (leaky_relu): LeakyReLU(negative_slope=0.01, inplace=True)\n            (dp): Dropout(p=0.1, inplace=False)\n            (lin2): Linear(in_features=200, out_features=200, bias=False)\n            (bn2): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=200, out_features=1, bias=True)\n  )\n)</pre> In\u00a0[24]: Copied! <pre>tab_lin_layer = list(model_3.deeptabular.children())[1]\n</pre> tab_lin_layer = list(model_3.deeptabular.children())[1] In\u00a0[25]: Copied! <pre>tab_lin_layer\n</pre> tab_lin_layer Out[25]: <pre>Linear(in_features=200, out_features=1, bias=True)</pre> In\u00a0[26]: Copied! <pre>tab_deep_layers = []\nfor n1, c1 in model_3.deeptabular.named_children():\n    if (\n        n1 == \"0\"\n    ):  # 0 is the model component and 1 is always the prediction layer added by the `WideDeep` class\n        for n2, c2 in c1.named_children():\n            if n2 == \"encoder\":  # TabResnet\n                for _, c3 in c2.named_children():\n                    for n4, c4 in c3.named_children():  # dense_resnet\n                        if \"block\" in n4:\n                            tab_deep_layers.append((n4, c4))\n</pre> tab_deep_layers = [] for n1, c1 in model_3.deeptabular.named_children():     if (         n1 == \"0\"     ):  # 0 is the model component and 1 is always the prediction layer added by the `WideDeep` class         for n2, c2 in c1.named_children():             if n2 == \"encoder\":  # TabResnet                 for _, c3 in c2.named_children():                     for n4, c4 in c3.named_children():  # dense_resnet                         if \"block\" in n4:                             tab_deep_layers.append((n4, c4)) In\u00a0[27]: Copied! <pre>tab_deep_layers\n</pre> tab_deep_layers Out[27]: <pre>[('block_0',\n  BasicBlock(\n    (lin1): Linear(in_features=200, out_features=200, bias=False)\n    (bn1): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n    (leaky_relu): LeakyReLU(negative_slope=0.01, inplace=True)\n    (dp): Dropout(p=0.1, inplace=False)\n    (lin2): Linear(in_features=200, out_features=200, bias=False)\n    (bn2): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n  )),\n ('block_1',\n  BasicBlock(\n    (lin1): Linear(in_features=200, out_features=200, bias=False)\n    (bn1): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n    (leaky_relu): LeakyReLU(negative_slope=0.01, inplace=True)\n    (dp): Dropout(p=0.1, inplace=False)\n    (lin2): Linear(in_features=200, out_features=200, bias=False)\n    (bn2): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n  ))]</pre> <p>Now remember, we need to pass ONLY LAYERS (before I included the name for clarity) the layers in WARM UP ORDER, therefore:</p> In\u00a0[28]: Copied! <pre>tab_deep_layers = [el[1] for el in tab_deep_layers][::-1]\n</pre> tab_deep_layers = [el[1] for el in tab_deep_layers][::-1] In\u00a0[29]: Copied! <pre>tab_layers = [tab_lin_layer] + tab_deep_layers[::-1]\n</pre> tab_layers = [tab_lin_layer] + tab_deep_layers[::-1] In\u00a0[30]: Copied! <pre>tab_layers\n</pre> tab_layers Out[30]: <pre>[Linear(in_features=200, out_features=1, bias=True),\n BasicBlock(\n   (lin1): Linear(in_features=200, out_features=200, bias=False)\n   (bn1): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n   (leaky_relu): LeakyReLU(negative_slope=0.01, inplace=True)\n   (dp): Dropout(p=0.1, inplace=False)\n   (lin2): Linear(in_features=200, out_features=200, bias=False)\n   (bn2): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n ),\n BasicBlock(\n   (lin1): Linear(in_features=200, out_features=200, bias=False)\n   (bn1): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n   (leaky_relu): LeakyReLU(negative_slope=0.01, inplace=True)\n   (dp): Dropout(p=0.1, inplace=False)\n   (lin2): Linear(in_features=200, out_features=200, bias=False)\n   (bn2): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n )]</pre> <p>And now simply</p> In\u00a0[31]: Copied! <pre>trainer_4 = Trainer(model_3, objective=\"binary\", metrics=[Accuracy])\n</pre> trainer_4 = Trainer(model_3, objective=\"binary\", metrics=[Accuracy]) In\u00a0[32]: Copied! <pre>trainer_4.fit(\n    X_wide=X_wide,\n    X_tab=X_tab,\n    target=target,\n    val_split=0.1,\n    finetune=True,\n    finetune_epochs=2,\n    deeptabular_gradual=True,\n    deeptabular_layers=tab_layers,\n    deeptabular_max_lr=0.01,\n    n_epochs=2,\n    batch_size=256,\n)\n</pre> trainer_4.fit(     X_wide=X_wide,     X_tab=X_tab,     target=target,     val_split=0.1,     finetune=True,     finetune_epochs=2,     deeptabular_gradual=True,     deeptabular_layers=tab_layers,     deeptabular_max_lr=0.01,     n_epochs=2,     batch_size=256, ) <pre>Training wide for 2 epochs\n</pre> <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:01&lt;00:00, 95.17it/s, loss=0.504, metrics={'acc': 0.7523}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:01&lt;00:00, 99.83it/s, loss=0.384, metrics={'acc': 0.789}]\n</pre> <pre>Training deeptabular, layer 1 of 3\n</pre> <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:02&lt;00:00, 72.31it/s, loss=0.317, metrics={'acc': 0.8098}]\n</pre> <pre>Training deeptabular, layer 2 of 3\n</pre> <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:02&lt;00:00, 65.97it/s, loss=0.312, metrics={'acc': 0.8214}]\n</pre> <pre>Training deeptabular, layer 3 of 3\n</pre> <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:02&lt;00:00, 63.92it/s, loss=0.306, metrics={'acc': 0.8284}]\n</pre> <pre>Fine-tuning (or warmup) of individual components completed. Training the whole model for 2 epochs\n</pre> <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:03&lt;00:00, 57.26it/s, loss=0.292, metrics={'acc': 0.8664}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 20/20 [00:00&lt;00:00, 84.56it/s, loss=0.292, metrics={'acc': 0.8696}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:03&lt;00:00, 53.61it/s, loss=0.282, metrics={'acc': 0.8693}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 20/20 [00:00&lt;00:00, 80.59it/s, loss=0.289, metrics={'acc': 0.8719}]\n</pre> <p>Finally, there is one more use case I would like to consider. The case where we train only one component and we just want to fine-tune and stop the training afterwards, since there is no joined training. This is a simple as</p> In\u00a0[33]: Copied! <pre>tab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    cat_embed_dropout=0.1,\n    continuous_cols=continuous_cols,\n    mlp_hidden_dims=[64, 32],\n    mlp_dropout=0.2,\n    mlp_activation=\"leaky_relu\",\n)\nmodel = WideDeep(deeptabular=tab_mlp)\n</pre> tab_mlp = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     cat_embed_dropout=0.1,     continuous_cols=continuous_cols,     mlp_hidden_dims=[64, 32],     mlp_dropout=0.2,     mlp_activation=\"leaky_relu\", ) model = WideDeep(deeptabular=tab_mlp) In\u00a0[34]: Copied! <pre>trainer_5 = Trainer(\n    model,\n    objective=\"binary\",\n    optimizers=torch.optim.Adam(model.parameters(), lr=0.01),\n    metrics=[Accuracy],\n)\n</pre> trainer_5 = Trainer(     model,     objective=\"binary\",     optimizers=torch.optim.Adam(model.parameters(), lr=0.01),     metrics=[Accuracy], ) In\u00a0[35]: Copied! <pre>trainer_5.fit(\n    X_wide=X_wide, X_tab=X_tab, target=target, val_split=0.1, n_epochs=1, batch_size=256\n)\n</pre> trainer_5.fit(     X_wide=X_wide, X_tab=X_tab, target=target, val_split=0.1, n_epochs=1, batch_size=256 ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:02&lt;00:00, 73.69it/s, loss=0.365, metrics={'acc': 0.8331}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 20/20 [00:00&lt;00:00, 92.56it/s, loss=0.299, metrics={'acc': 0.8673}]\n</pre> In\u00a0[36]: Copied! <pre>trainer_5.save(path=\"models_dir\", save_state_dict=True, model_filename=\"model_5.pt\")\n</pre> trainer_5.save(path=\"models_dir\", save_state_dict=True, model_filename=\"model_5.pt\") In\u00a0[37]: Copied! <pre>tab_mlp_5 = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    cat_embed_dropout=0.1,\n    continuous_cols=continuous_cols,\n    mlp_hidden_dims=[64, 32],\n    mlp_dropout=0.2,\n    mlp_activation=\"leaky_relu\",\n)\nmodel_5 = WideDeep(deeptabular=tab_mlp_5)\n</pre> tab_mlp_5 = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     cat_embed_dropout=0.1,     continuous_cols=continuous_cols,     mlp_hidden_dims=[64, 32],     mlp_dropout=0.2,     mlp_activation=\"leaky_relu\", ) model_5 = WideDeep(deeptabular=tab_mlp_5) In\u00a0[38]: Copied! <pre>model_5.load_state_dict(torch.load(\"models_dir/model_5.pt\"))\n</pre> model_5.load_state_dict(torch.load(\"models_dir/model_5.pt\")) Out[38]: <pre>&lt;All keys matched successfully&gt;</pre> <p>...times go by...</p> In\u00a0[39]: Copied! <pre>trainer_6 = Trainer(\n    model_5,\n    objective=\"binary\",\n    optimizers=torch.optim.Adam(model.parameters(), lr=0.01),\n    metrics=[Accuracy],\n)\n</pre> trainer_6 = Trainer(     model_5,     objective=\"binary\",     optimizers=torch.optim.Adam(model.parameters(), lr=0.01),     metrics=[Accuracy], ) In\u00a0[40]: Copied! <pre>trainer_6.fit(\n    X_wide=X_wide,\n    X_tab=X_tab,\n    target=target,\n    val_split=0.1,\n    finetune=True,\n    finetune_epochs=2,\n    finetune_max_lr=0.01,\n    stop_after_finetuning=True,\n    batch_size=256,\n)\n</pre> trainer_6.fit(     X_wide=X_wide,     X_tab=X_tab,     target=target,     val_split=0.1,     finetune=True,     finetune_epochs=2,     finetune_max_lr=0.01,     stop_after_finetuning=True,     batch_size=256, ) <pre>Training deeptabular for 2 epochs\n</pre> <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:02&lt;00:00, 73.86it/s, loss=0.298, metrics={'acc': 0.8652}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:02&lt;00:00, 75.45it/s, loss=0.286, metrics={'acc': 0.8669}]\n</pre> <pre>Fine-tuning (or warmup) of individual components completed. Training the whole model for 1 epochs\n</pre> <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:02&lt;00:00, 76.29it/s, loss=0.282, metrics={'acc': 0.8698}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 20/20 [00:00&lt;00:00, 84.93it/s, loss=0.281, metrics={'acc': 0.8749}]\n</pre> In\u00a0[42]: Copied! <pre>import shutil\n\nshutil.rmtree(\"models_dir/\")\nshutil.rmtree(\"model_weights/\")\n</pre> import shutil  shutil.rmtree(\"models_dir/\") shutil.rmtree(\"model_weights/\") In\u00a0[\u00a0]: Copied! <pre>\n</pre>"},{"location":"examples/06_finetune_and_warmup.html#the-finetunewarm-up-option","title":"The FineTune/Warm Up option\u00b6","text":"<p>Let's place ourselves in two possible scenarios.</p> <ol> <li><p>Let's assume we have run a model and we want to just transfer the learnings (you know...transfer-learning) to another dataset, or simply we have received new data and we do not want to start the training of each component from scratch. Simply, we want to load the pre-trained weights and fine-tune.</p> </li> <li><p>We just want to \"warm up\" individual model components individually before the joined training begins.</p> </li> </ol> <p>This can be done with the <code>finetune</code> set of parameters. There are 3 fine-tuning routines:</p> <ol> <li>Fine-tune all trainable layers at once with a triangular one-cycle learning rate (referred as slanted triangular learning rates in Howard &amp; Ruder 2018)</li> <li>Gradual fine-tuning inspired by the work of Felbo et al., 2017</li> <li>Gradual fine-tuning based on the work of Howard &amp; Ruder 2018</li> </ol> <p>Currently fine-tunning is only supported without a fully connected head, i.e. if <code>deephead=None</code>. In addition, <code>Felbo</code> and <code>Howard</code> routines only applied, of course, to the <code>deeptabular</code>, <code>deeptext</code> and <code>deepimage</code> models. The <code>wide</code> component can also be fine-tuned, but only in an \"all at once\" mode.</p>"},{"location":"examples/06_finetune_and_warmup.html#fine-tune-or-warm-up-all-at-once","title":"Fine-tune or warm-up all at once\u00b6","text":"<p>Here, the model components will be trained for <code>finetune_epochs</code> using a triangular one-cycle learning rate (slanted triangular learning rate) ranging from <code>finetune_max_lr/10</code> to <code>finetune_max_lr</code> (default is 0.01). 10% of the training steps are used to increase the learning rate which then decreases for the remaining 90%.</p> <p>Here all trainable layers are fine-tuned.</p> <p>Let's have a look to one example.</p>"},{"location":"examples/06_finetune_and_warmup.html#fine-tune-gradually-the-felbo-and-the-howard-routines","title":"Fine-tune Gradually: The \"felbo\"  and the \"howard\" routines\u00b6","text":"<p>The Felbo routine can be illustrated as follows:</p> <p> </p> <p>Figure 1. The figure can be described as follows: fine-tune (or train) the last layer for one epoch using a one cycle triangular learning rate. Then fine-tune the next deeper layer for one epoch, with a learning rate that is a factor of 2.5 lower than the previous learning rate (the 2.5 factor is fixed) while freezing the already warmed up layer(s). Repeat untill all individual layers are warmed. Then warm one last epoch with all warmed layers trainable. The vanishing color gradient in the figure attempts to illustrate the decreasing learning rate.</p> <p>Note that this is not identical to the Fine-Tunning routine described in Felbo et al, 2017, this is why I used the word 'inspired'.</p> <p>The Howard routine can be illustrated as follows:</p> <p> </p> <p>Figure 2. The figure can be described as follows: fine-tune (or train) the last layer for one epoch using a one cycle triangular learning rate. Then fine-tune the next deeper layer for one epoch, with a learning rate that is a factor of 2.5 lower than the previous learning rate (the 2.5 factor is fixed) while keeping the already warmed up layer(s) trainable. Repeat. The vanishing color gradient in the figure attempts to illustrate the decreasing learning rate.</p> <p>Note that I write \"fine-tune (or train) the last layer for one epoch [...]\". However, in practice the user will have to specify the order of the layers to be fine-tuned. This is another reason why I wrote that the fine-tune routines I have implemented are inspired by the work of Felbo and Howard and not identical to their implemenations.</p> <p>The <code>felbo</code> and <code>howard</code> routines can be accessed with via the <code>fine-tune</code> parameters.</p>"},{"location":"examples/07_custom_components.html","title":"07_custom_components","text":"In\u00a0[1]: Copied! <pre>import numpy as np\nimport pandas as pd\nimport os\nimport torch\n\nfrom torch import Tensor\nfrom pytorch_widedeep import Trainer\nfrom pytorch_widedeep.preprocessing import (\n    WidePreprocessor,\n    TabPreprocessor,\n    TextPreprocessor,\n    ImagePreprocessor,\n)\nfrom pytorch_widedeep.models import (\n    Wide,\n    TabMlp,\n    Vision,\n    BasicRNN,\n    WideDeep,\n)\nfrom pytorch_widedeep.losses import RMSELoss\nfrom pytorch_widedeep.initializers import *\nfrom pytorch_widedeep.callbacks import *\nfrom pytorch_widedeep.datasets import load_adult\n</pre> import numpy as np import pandas as pd import os import torch  from torch import Tensor from pytorch_widedeep import Trainer from pytorch_widedeep.preprocessing import (     WidePreprocessor,     TabPreprocessor,     TextPreprocessor,     ImagePreprocessor, ) from pytorch_widedeep.models import (     Wide,     TabMlp,     Vision,     BasicRNN,     WideDeep, ) from pytorch_widedeep.losses import RMSELoss from pytorch_widedeep.initializers import * from pytorch_widedeep.callbacks import * from pytorch_widedeep.datasets import load_adult <pre>/Users/javierrodriguezzaurin/.pyenv/versions/3.10.13/envs/widedeep310/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n  from .autonotebook import tqdm as notebook_tqdm\n</pre> In\u00a0[2]: Copied! <pre>df = pd.read_csv(\"../tmp_data/airbnb/airbnb_sample.csv\")\ndf.head()\n</pre> df = pd.read_csv(\"../tmp_data/airbnb/airbnb_sample.csv\") df.head() Out[2]: id host_id description host_listings_count host_identity_verified neighbourhood_cleansed latitude longitude is_location_exact property_type ... amenity_wide_entrance amenity_wide_entrance_for_guests amenity_wide_entryway amenity_wide_hallways amenity_wifi amenity_window_guards amenity_wine_cooler security_deposit extra_people yield 0 13913.jpg 54730 My bright double bedroom with a large window h... 4.0 f Islington 51.56802 -0.11121 t apartment ... 1 0 0 0 1 0 0 100.0 15.0 12.00 1 15400.jpg 60302 Lots of windows and light.  St Luke's Gardens ... 1.0 t Kensington and Chelsea 51.48796 -0.16898 t apartment ... 0 0 0 0 1 0 0 150.0 0.0 109.50 2 17402.jpg 67564 Open from June 2018 after a 3-year break, we a... 19.0 t Westminster 51.52098 -0.14002 t apartment ... 0 0 0 0 1 0 0 350.0 10.0 149.65 3 24328.jpg 41759 Artist house, bright high ceiling rooms, priva... 2.0 t Wandsworth 51.47298 -0.16376 t other ... 0 0 0 0 1 0 0 250.0 0.0 215.60 4 25023.jpg 102813 Large, all comforts, 2-bed flat; first floor; ... 1.0 f Wandsworth 51.44687 -0.21874 t apartment ... 0 0 0 0 1 0 0 250.0 11.0 79.35 <p>5 rows \u00d7 223 columns</p> In\u00a0[3]: Copied! <pre># There are a number of columns that are already binary. Therefore, no need to one hot encode them\ncrossed_cols = [(\"property_type\", \"room_type\")]\nalready_dummies = [c for c in df.columns if \"amenity\" in c] + [\"has_house_rules\"]\nwide_cols = [\n    \"is_location_exact\",\n    \"property_type\",\n    \"room_type\",\n    \"host_gender\",\n    \"instant_bookable\",\n] + already_dummies\n\ncat_embed_cols = [(c, 16) for c in df.columns if \"catg\" in c] + [\n    (\"neighbourhood_cleansed\", 64),\n    (\"cancellation_policy\", 16),\n]\ncontinuous_cols = [\"latitude\", \"longitude\", \"security_deposit\", \"extra_people\"]\n# it does not make sense to standarised Latitude and Longitude\nalready_standard = [\"latitude\", \"longitude\"]\n\n# text and image colnames\ntext_col = \"description\"\nimg_col = \"id\"\n\n# path to pretrained word embeddings and the images\nword_vectors_path = \"../tmp_data/glove.6B/glove.6B.100d.txt\"\nimg_path = \"../tmp_data/airbnb/property_picture\"\n\n# target\ntarget_col = \"yield\"\n</pre> # There are a number of columns that are already binary. Therefore, no need to one hot encode them crossed_cols = [(\"property_type\", \"room_type\")] already_dummies = [c for c in df.columns if \"amenity\" in c] + [\"has_house_rules\"] wide_cols = [     \"is_location_exact\",     \"property_type\",     \"room_type\",     \"host_gender\",     \"instant_bookable\", ] + already_dummies  cat_embed_cols = [(c, 16) for c in df.columns if \"catg\" in c] + [     (\"neighbourhood_cleansed\", 64),     (\"cancellation_policy\", 16), ] continuous_cols = [\"latitude\", \"longitude\", \"security_deposit\", \"extra_people\"] # it does not make sense to standarised Latitude and Longitude already_standard = [\"latitude\", \"longitude\"]  # text and image colnames text_col = \"description\" img_col = \"id\"  # path to pretrained word embeddings and the images word_vectors_path = \"../tmp_data/glove.6B/glove.6B.100d.txt\" img_path = \"../tmp_data/airbnb/property_picture\"  # target target_col = \"yield\" In\u00a0[4]: Copied! <pre>target = df[target_col].values\n</pre> target = df[target_col].values In\u00a0[5]: Copied! <pre>wide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols)\nX_wide = wide_preprocessor.fit_transform(df)\n\ntab_preprocessor = TabPreprocessor(\n    embed_cols=cat_embed_cols, continuous_cols=continuous_cols\n)\nX_tab = tab_preprocessor.fit_transform(df)\n\ntext_preprocessor = TextPreprocessor(\n    word_vectors_path=word_vectors_path, text_col=text_col\n)\nX_text = text_preprocessor.fit_transform(df)\n\nimage_processor = ImagePreprocessor(img_col=img_col, img_path=img_path)\nX_images = image_processor.fit_transform(df)\n</pre> wide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols) X_wide = wide_preprocessor.fit_transform(df)  tab_preprocessor = TabPreprocessor(     embed_cols=cat_embed_cols, continuous_cols=continuous_cols ) X_tab = tab_preprocessor.fit_transform(df)  text_preprocessor = TextPreprocessor(     word_vectors_path=word_vectors_path, text_col=text_col ) X_text = text_preprocessor.fit_transform(df)  image_processor = ImagePreprocessor(img_col=img_col, img_path=img_path) X_images = image_processor.fit_transform(df) <pre>/Users/javierrodriguezzaurin/Projects/pytorch-widedeep/pytorch_widedeep/preprocessing/tab_preprocessor.py:358: UserWarning: Continuous columns will not be normalised\n  warnings.warn(\"Continuous columns will not be normalised\")\n</pre> <pre>The vocabulary contains 2192 tokens\nIndexing word vectors...\nLoaded 400000 word vectors\nPreparing embeddings matrix...\n2175 words in the vocabulary had ../tmp_data/glove.6B/glove.6B.100d.txt vectors and appear more than 5 times\nReading Images from ../tmp_data/airbnb/property_picture\nResizing\n</pre> <pre>100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 1001/1001 [00:02&lt;00:00, 497.80it/s]\n</pre> <pre>Computing normalisation metrics\n</pre> <p>Now we are ready to build a wide and deep model. Three of the four components we will use are included in this package, and they will be combined with a custom <code>deeptext</code> component. Then the fit process will run with a custom loss function.</p> <p>Let's have a look</p> In\u00a0[6]: Copied! <pre># Linear model\nwide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1)\n\n# DeepDense: 2 Dense layers\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    cat_embed_dropout=0.1,\n    continuous_cols=continuous_cols,\n    mlp_hidden_dims=[128, 64],\n    mlp_dropout=0.1,\n)\n\n# Pretrained Resnet 18\nresnet = Vision(pretrained_model_name=\"resnet18\", n_trainable=0)\n</pre> # Linear model wide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1)  # DeepDense: 2 Dense layers tab_mlp = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     cat_embed_dropout=0.1,     continuous_cols=continuous_cols,     mlp_hidden_dims=[128, 64],     mlp_dropout=0.1, )  # Pretrained Resnet 18 resnet = Vision(pretrained_model_name=\"resnet18\", n_trainable=0) In\u00a0[7]: Copied! <pre>class MyDeepText(nn.Module):\n    def __init__(self, vocab_size, padding_idx=1, embed_dim=100, hidden_dim=64):\n        super(MyDeepText, self).__init__()\n\n        # word/token embeddings\n        self.word_embed = nn.Embedding(vocab_size, embed_dim, padding_idx=padding_idx)\n\n        # stack of RNNs\n        self.rnn = nn.GRU(\n            embed_dim,\n            hidden_dim,\n            num_layers=2,\n            bidirectional=True,\n            batch_first=True,\n        )\n\n        # Remember, this MUST be defined. If not WideDeep will through an error\n        self.output_dim = hidden_dim * 2\n\n    def forward(self, X):\n        embed = self.word_embed(X.long())\n        o, h = self.rnn(embed)\n        return torch.cat((h[-2], h[-1]), dim=1)\n</pre> class MyDeepText(nn.Module):     def __init__(self, vocab_size, padding_idx=1, embed_dim=100, hidden_dim=64):         super(MyDeepText, self).__init__()          # word/token embeddings         self.word_embed = nn.Embedding(vocab_size, embed_dim, padding_idx=padding_idx)          # stack of RNNs         self.rnn = nn.GRU(             embed_dim,             hidden_dim,             num_layers=2,             bidirectional=True,             batch_first=True,         )          # Remember, this MUST be defined. If not WideDeep will through an error         self.output_dim = hidden_dim * 2      def forward(self, X):         embed = self.word_embed(X.long())         o, h = self.rnn(embed)         return torch.cat((h[-2], h[-1]), dim=1) In\u00a0[8]: Copied! <pre>mydeeptext = MyDeepText(vocab_size=len(text_preprocessor.vocab.itos))\n</pre> mydeeptext = MyDeepText(vocab_size=len(text_preprocessor.vocab.itos)) In\u00a0[9]: Copied! <pre>model = WideDeep(wide=wide, deeptabular=tab_mlp, deeptext=mydeeptext, deepimage=resnet)\n</pre> model = WideDeep(wide=wide, deeptabular=tab_mlp, deeptext=mydeeptext, deepimage=resnet) In\u00a0[10]: Copied! <pre>class RMSELoss(nn.Module):\n    def __init__(self):\n        \"\"\"root mean squared error\"\"\"\n        super().__init__()\n        self.mse = nn.MSELoss()\n\n    def forward(self, input: Tensor, target: Tensor) -&gt; Tensor:\n        return torch.sqrt(self.mse(input, target))\n</pre> class RMSELoss(nn.Module):     def __init__(self):         \"\"\"root mean squared error\"\"\"         super().__init__()         self.mse = nn.MSELoss()      def forward(self, input: Tensor, target: Tensor) -&gt; Tensor:         return torch.sqrt(self.mse(input, target)) <p>and now we just instantiate the <code>Trainer</code> as usual. Needless to say, but this runs with 1000 random observations, so loss and metric values are meaningless. This is just an example</p> In\u00a0[11]: Copied! <pre>trainer = Trainer(model, objective=\"regression\", custom_loss_function=RMSELoss())\n</pre> trainer = Trainer(model, objective=\"regression\", custom_loss_function=RMSELoss()) In\u00a0[12]: Copied! <pre>trainer.fit(\n    X_wide=X_wide,\n    X_tab=X_tab,\n    X_text=X_text,\n    X_img=X_images,\n    target=target,\n    n_epochs=1,\n    batch_size=32,\n    val_split=0.2,\n)\n</pre> trainer.fit(     X_wide=X_wide,     X_tab=X_tab,     X_text=X_text,     X_img=X_images,     target=target,     n_epochs=1,     batch_size=32,     val_split=0.2, ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 25/25 [00:23&lt;00:00,  1.07it/s, loss=126]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 7/7 [00:05&lt;00:00,  1.24it/s, loss=97.4]\n</pre> <p>In addition to model components and loss functions, we can also use custom callbacks or custom metrics. The former need to be of type <code>Callback</code> and the latter need to be of type <code>Metric</code>. See:</p> <pre>pytorch-widedeep.callbacks\n</pre> <p>and</p> <pre>pytorch-widedeep.metrics\n</pre> <p>For this example let me use the adult dataset. Again, we first prepare the data as usual</p> In\u00a0[13]: Copied! <pre>df = load_adult(as_frame=True)\ndf.head()\n</pre> df = load_adult(as_frame=True) df.head() Out[13]: age workclass fnlwgt education educational-num marital-status occupation relationship race gender capital-gain capital-loss hours-per-week native-country income 0 25 Private 226802 11th 7 Never-married Machine-op-inspct Own-child Black Male 0 0 40 United-States &lt;=50K 1 38 Private 89814 HS-grad 9 Married-civ-spouse Farming-fishing Husband White Male 0 0 50 United-States &lt;=50K 2 28 Local-gov 336951 Assoc-acdm 12 Married-civ-spouse Protective-serv Husband White Male 0 0 40 United-States &gt;50K 3 44 Private 160323 Some-college 10 Married-civ-spouse Machine-op-inspct Husband Black Male 7688 0 40 United-States &gt;50K 4 18 ? 103497 Some-college 10 Never-married ? Own-child White Female 0 0 30 United-States &lt;=50K In\u00a0[14]: Copied! <pre># For convenience, we'll replace '-' with '_'\ndf.columns = [c.replace(\"-\", \"_\") for c in df.columns]\n# binary target\ndf[\"income_label\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int)\ndf.drop(\"income\", axis=1, inplace=True)\ndf.head()\n</pre> # For convenience, we'll replace '-' with '_' df.columns = [c.replace(\"-\", \"_\") for c in df.columns] # binary target df[\"income_label\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int) df.drop(\"income\", axis=1, inplace=True) df.head() Out[14]: age workclass fnlwgt education educational_num marital_status occupation relationship race gender capital_gain capital_loss hours_per_week native_country income_label 0 25 Private 226802 11th 7 Never-married Machine-op-inspct Own-child Black Male 0 0 40 United-States 0 1 38 Private 89814 HS-grad 9 Married-civ-spouse Farming-fishing Husband White Male 0 0 50 United-States 0 2 28 Local-gov 336951 Assoc-acdm 12 Married-civ-spouse Protective-serv Husband White Male 0 0 40 United-States 1 3 44 Private 160323 Some-college 10 Married-civ-spouse Machine-op-inspct Husband Black Male 7688 0 40 United-States 1 4 18 ? 103497 Some-college 10 Never-married ? Own-child White Female 0 0 30 United-States 0 In\u00a0[15]: Copied! <pre># Define wide, crossed and deep tabular columns\nwide_cols = [\n    \"workclass\",\n    \"education\",\n    \"marital_status\",\n    \"occupation\",\n    \"relationship\",\n    \"race\",\n    \"gender\",\n    \"native_country\",\n]\ncrossed_cols = [(\"education\", \"occupation\"), (\"native_country\", \"occupation\")]\ncat_embed_cols = [\n    \"workclass\",\n    \"education\",\n    \"marital_status\",\n    \"occupation\",\n    \"relationship\",\n    \"race\",\n    \"gender\",\n    \"capital_gain\",\n    \"capital_loss\",\n    \"native_country\",\n]\ncontinuous_cols = [\"age\", \"hours_per_week\"]\ntarget_col = \"income_label\"\ntarget = df[target_col].values\n</pre> # Define wide, crossed and deep tabular columns wide_cols = [     \"workclass\",     \"education\",     \"marital_status\",     \"occupation\",     \"relationship\",     \"race\",     \"gender\",     \"native_country\", ] crossed_cols = [(\"education\", \"occupation\"), (\"native_country\", \"occupation\")] cat_embed_cols = [     \"workclass\",     \"education\",     \"marital_status\",     \"occupation\",     \"relationship\",     \"race\",     \"gender\",     \"capital_gain\",     \"capital_loss\",     \"native_country\", ] continuous_cols = [\"age\", \"hours_per_week\"] target_col = \"income_label\" target = df[target_col].values In\u00a0[16]: Copied! <pre># wide\nwide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols)\nX_wide = wide_preprocessor.fit_transform(df)\n\n# deeptabular\ntab_preprocessor = TabPreprocessor(\n    embed_cols=cat_embed_cols, continuous_cols=continuous_cols\n)\nX_tab = tab_preprocessor.fit_transform(df)\n</pre> # wide wide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols) X_wide = wide_preprocessor.fit_transform(df)  # deeptabular tab_preprocessor = TabPreprocessor(     embed_cols=cat_embed_cols, continuous_cols=continuous_cols ) X_tab = tab_preprocessor.fit_transform(df) <pre>/Users/javierrodriguezzaurin/Projects/pytorch-widedeep/pytorch_widedeep/preprocessing/tab_preprocessor.py:358: UserWarning: Continuous columns will not be normalised\n  warnings.warn(\"Continuous columns will not be normalised\")\n</pre> In\u00a0[17]: Copied! <pre>wide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1)\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=continuous_cols,\n    mlp_hidden_dims=[128, 64],\n    mlp_dropout=0.2,\n    mlp_activation=\"leaky_relu\",\n)\nmodel = WideDeep(wide=wide, deeptabular=tab_mlp)\n</pre> wide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1) tab_mlp = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     continuous_cols=continuous_cols,     mlp_hidden_dims=[128, 64],     mlp_dropout=0.2,     mlp_activation=\"leaky_relu\", ) model = WideDeep(wide=wide, deeptabular=tab_mlp) In\u00a0[18]: Copied! <pre>from pytorch_widedeep.metrics import Metric\n</pre> from pytorch_widedeep.metrics import Metric In\u00a0[19]: Copied! <pre>class Accuracy(Metric):\n    def __init__(self, top_k: int = 1):\n        super(Accuracy, self).__init__()\n\n        self.top_k = top_k\n        self.correct_count = 0\n        self.total_count = 0\n\n        # \u00a0metric name needs to be defined\n        self._name = \"acc\"\n\n    def reset(self):\n        self.correct_count = 0\n        self.total_count = 0\n\n    def __call__(self, y_pred: Tensor, y_true: Tensor) -&gt; np.ndarray:\n        num_classes = y_pred.size(1)\n\n        if num_classes == 1:\n            y_pred = y_pred.round()\n            y_true = y_true\n        elif num_classes &gt; 1:\n            y_pred = y_pred.topk(self.top_k, 1)[1]\n            y_true = y_true.view(-1, 1).expand_as(y_pred)\n\n        self.correct_count += y_pred.eq(y_true).sum().item()\n        self.total_count += len(y_pred)\n        accuracy = float(self.correct_count) / float(self.total_count)\n        return np.array(accuracy)\n</pre> class Accuracy(Metric):     def __init__(self, top_k: int = 1):         super(Accuracy, self).__init__()          self.top_k = top_k         self.correct_count = 0         self.total_count = 0          # \u00a0metric name needs to be defined         self._name = \"acc\"      def reset(self):         self.correct_count = 0         self.total_count = 0      def __call__(self, y_pred: Tensor, y_true: Tensor) -&gt; np.ndarray:         num_classes = y_pred.size(1)          if num_classes == 1:             y_pred = y_pred.round()             y_true = y_true         elif num_classes &gt; 1:             y_pred = y_pred.topk(self.top_k, 1)[1]             y_true = y_true.view(-1, 1).expand_as(y_pred)          self.correct_count += y_pred.eq(y_true).sum().item()         self.total_count += len(y_pred)         accuracy = float(self.correct_count) / float(self.total_count)         return np.array(accuracy) In\u00a0[20]: Copied! <pre># have a look to the class\nfrom pytorch_widedeep.callbacks import Callback\n</pre> # have a look to the class from pytorch_widedeep.callbacks import Callback In\u00a0[21]: Copied! <pre>class SillyCallback(Callback):\n    def on_train_begin(self, logs=None):\n        # recordings will be the trainer object attributes\n        self.trainer.silly_callback = {}\n\n        self.trainer.silly_callback[\"beginning\"] = []\n        self.trainer.silly_callback[\"end\"] = []\n\n    def on_epoch_begin(self, epoch, logs=None):\n        self.trainer.silly_callback[\"beginning\"].append(epoch + 1)\n\n    def on_epoch_end(self, epoch, logs=None, metric=None):\n        self.trainer.silly_callback[\"end\"].append(epoch + 1)\n</pre> class SillyCallback(Callback):     def on_train_begin(self, logs=None):         # recordings will be the trainer object attributes         self.trainer.silly_callback = {}          self.trainer.silly_callback[\"beginning\"] = []         self.trainer.silly_callback[\"end\"] = []      def on_epoch_begin(self, epoch, logs=None):         self.trainer.silly_callback[\"beginning\"].append(epoch + 1)      def on_epoch_end(self, epoch, logs=None, metric=None):         self.trainer.silly_callback[\"end\"].append(epoch + 1) <p>and now, as usual:</p> In\u00a0[22]: Copied! <pre>trainer = Trainer(\n    model, objective=\"binary\", metrics=[Accuracy], callbacks=[SillyCallback]\n)\n</pre> trainer = Trainer(     model, objective=\"binary\", metrics=[Accuracy], callbacks=[SillyCallback] ) In\u00a0[23]: Copied! <pre>trainer.fit(\n    X_wide=X_wide, X_tab=X_tab, target=target, n_epochs=5, batch_size=64, val_split=0.2\n)\n</pre> trainer.fit(     X_wide=X_wide, X_tab=X_tab, target=target, n_epochs=5, batch_size=64, val_split=0.2 ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 611/611 [00:06&lt;00:00, 94.39it/s, loss=0.411, metrics={'acc': 0.814}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 121.91it/s, loss=0.327, metrics={'acc': 0.8449}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 611/611 [00:07&lt;00:00, 85.39it/s, loss=0.324, metrics={'acc': 0.8495}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 88.68it/s, loss=0.298, metrics={'acc': 0.8612}]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 611/611 [00:08&lt;00:00, 74.35it/s, loss=0.302, metrics={'acc': 0.8593}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 100.51it/s, loss=0.29, metrics={'acc': 0.8665}]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 611/611 [00:08&lt;00:00, 73.83it/s, loss=0.292, metrics={'acc': 0.8637}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 105.98it/s, loss=0.286, metrics={'acc': 0.8695}]\nepoch 5: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 611/611 [00:08&lt;00:00, 72.15it/s, loss=0.286, metrics={'acc': 0.866}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 92.27it/s, loss=0.284, metrics={'acc': 0.8698}]\n</pre> In\u00a0[24]: Copied! <pre>trainer.silly_callback\n</pre> trainer.silly_callback Out[24]: <pre>{'beginning': [1, 2, 3, 4, 5], 'end': [1, 2, 3, 4, 5]}</pre>"},{"location":"examples/07_custom_components.html#custom-components","title":"Custom components\u00b6","text":"<p>As I mentioned earlier in the example notebooks, and also in the <code>README</code>, it is possible to customise almost every component in <code>pytorch-widedeep</code>.</p> <p>Let's now go through a couple of simple examples to illustrate how that could be done.</p> <p>First let's load and process the data \"as usual\", let's start with a regression and the airbnb dataset.</p>"},{"location":"examples/07_custom_components.html#custom-deeptext","title":"Custom <code>deeptext</code>\u00b6","text":"<p>Standard Pytorch model</p>"},{"location":"examples/07_custom_components.html#custom-loss-function","title":"Custom loss function\u00b6","text":"<p>Loss functions must simply inherit pytorch's <code>nn.Module</code>. For example, let's say we want to use <code>RMSE</code> (note that this is already available in the package, but I will pass it here as a custom loss for illustration purposes)</p>"},{"location":"examples/07_custom_components.html#custom-metric","title":"Custom metric\u00b6","text":"<p>Let's say we want to use our own accuracy metric (again, this is already available in the package, but I will pass it here as a custom loss for illustration purposes).</p> <p>This could be done as:</p>"},{"location":"examples/07_custom_components.html#custom-callback","title":"Custom Callback\u00b6","text":"<p>Let's code a callback that records the current epoch at the beginning and the end of each epoch (silly, but you know, this is just an example)</p>"},{"location":"examples/08_custom_dataLoader_imbalanced_dataset.html","title":"08_custom_dataLoader_imbalanced_dataset","text":"<ul> <li>In this notebook we will use the higly imbalanced Protein Homology Dataset from KDD cup 2004</li> </ul> <pre><code>* The first element of each line is a BLOCK ID that denotes to which native sequence this example belongs. There is a unique BLOCK ID for each native sequence. BLOCK IDs are integers running from 1 to 303 (one for each native sequence, i.e. for each query). BLOCK IDs were assigned before the blocks were split into the train and test sets, so they do not run consecutively in either file.\n* The second element of each line is an EXAMPLE ID that uniquely describes the example. You will need this EXAMPLE ID and the BLOCK ID when you submit results.\n* The third element is the class of the example. Proteins that are homologous to the native sequence are denoted by 1, non-homologous proteins (i.e. decoys) by 0. Test examples have a \"?\" in this position.\n* All following elements are feature values. There are 74 feature values in each line. The features describe the match (e.g. the score of a sequence alignment) between the native protein sequence and the sequence that is tested for homology.\n</code></pre> In\u00a0[1]: Copied! <pre>import numpy as np\nimport pandas as pd\nimport torch\nfrom torch.optim import SGD, lr_scheduler\n\nfrom pytorch_widedeep import Trainer\nfrom pytorch_widedeep.preprocessing import TabPreprocessor\nfrom pytorch_widedeep.models import TabMlp, WideDeep\nfrom pytorch_widedeep.dataloaders import DataLoaderImbalanced, DataLoaderDefault\nfrom torchmetrics import F1Score as F1_torchmetrics\nfrom torchmetrics import Accuracy as Accuracy_torchmetrics\nfrom torchmetrics import Precision as Precision_torchmetrics\nfrom torchmetrics import Recall as Recall_torchmetrics\nfrom pytorch_widedeep.metrics import Accuracy, Recall, Precision, F1Score, R2Score\nfrom pytorch_widedeep.initializers import XavierNormal\nfrom pytorch_widedeep.datasets import load_bio_kdd04\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import classification_report\n\nimport time\nimport datetime\n\nimport warnings\n\nwarnings.filterwarnings(\"ignore\", category=DeprecationWarning)\n\n# increase displayed columns in jupyter notebook\npd.set_option(\"display.max_columns\", 200)\npd.set_option(\"display.max_rows\", 300)\n</pre> import numpy as np import pandas as pd import torch from torch.optim import SGD, lr_scheduler  from pytorch_widedeep import Trainer from pytorch_widedeep.preprocessing import TabPreprocessor from pytorch_widedeep.models import TabMlp, WideDeep from pytorch_widedeep.dataloaders import DataLoaderImbalanced, DataLoaderDefault from torchmetrics import F1Score as F1_torchmetrics from torchmetrics import Accuracy as Accuracy_torchmetrics from torchmetrics import Precision as Precision_torchmetrics from torchmetrics import Recall as Recall_torchmetrics from pytorch_widedeep.metrics import Accuracy, Recall, Precision, F1Score, R2Score from pytorch_widedeep.initializers import XavierNormal from pytorch_widedeep.datasets import load_bio_kdd04  from sklearn.model_selection import train_test_split from sklearn.metrics import classification_report  import time import datetime  import warnings  warnings.filterwarnings(\"ignore\", category=DeprecationWarning)  # increase displayed columns in jupyter notebook pd.set_option(\"display.max_columns\", 200) pd.set_option(\"display.max_rows\", 300) <pre>/Users/javierrodriguezzaurin/.pyenv/versions/3.10.13/envs/widedeep310/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n  from .autonotebook import tqdm as notebook_tqdm\n</pre> In\u00a0[2]: Copied! <pre>df = load_bio_kdd04(as_frame=True)\n# drop columns we won't need in this example\ndf.drop(columns=[\"EXAMPLE_ID\", \"BLOCK_ID\"], inplace=True)\n\ndf_train, df_valid = train_test_split(\n    df, test_size=0.2, stratify=df[\"target\"], random_state=1\n)\ndf_valid, df_test = train_test_split(\n    df_valid, test_size=0.5, stratify=df_valid[\"target\"], random_state=1\n)\n\ncontinuous_cols = df.drop(columns=[\"target\"]).columns.values.tolist()\n</pre> df = load_bio_kdd04(as_frame=True) # drop columns we won't need in this example df.drop(columns=[\"EXAMPLE_ID\", \"BLOCK_ID\"], inplace=True)  df_train, df_valid = train_test_split(     df, test_size=0.2, stratify=df[\"target\"], random_state=1 ) df_valid, df_test = train_test_split(     df_valid, test_size=0.5, stratify=df_valid[\"target\"], random_state=1 )  continuous_cols = df.drop(columns=[\"target\"]).columns.values.tolist() In\u00a0[3]: Copied! <pre># deeptabular\ntab_preprocessor = TabPreprocessor(continuous_cols=continuous_cols, scale=True)\nX_tab_train = tab_preprocessor.fit_transform(df_train)\nX_tab_valid = tab_preprocessor.transform(df_valid)\nX_tab_test = tab_preprocessor.transform(df_test)\n\n# target\ny_train = df_train[\"target\"].values\ny_valid = df_valid[\"target\"].values\ny_test = df_test[\"target\"].values\n</pre> # deeptabular tab_preprocessor = TabPreprocessor(continuous_cols=continuous_cols, scale=True) X_tab_train = tab_preprocessor.fit_transform(df_train) X_tab_valid = tab_preprocessor.transform(df_valid) X_tab_test = tab_preprocessor.transform(df_test)  # target y_train = df_train[\"target\"].values y_valid = df_valid[\"target\"].values y_test = df_test[\"target\"].values In\u00a0[4]: Copied! <pre># Define the model\ninput_layer = len(tab_preprocessor.continuous_cols)\noutput_layer = 1\nhidden_layers = np.linspace(\n    input_layer * 2, output_layer, 5, endpoint=False, dtype=int\n).tolist()\n\ndeeptabular = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    continuous_cols=tab_preprocessor.continuous_cols,\n    mlp_hidden_dims=hidden_layers,\n)\nmodel = WideDeep(deeptabular=deeptabular)\nmodel\n</pre> # Define the model input_layer = len(tab_preprocessor.continuous_cols) output_layer = 1 hidden_layers = np.linspace(     input_layer * 2, output_layer, 5, endpoint=False, dtype=int ).tolist()  deeptabular = TabMlp(     column_idx=tab_preprocessor.column_idx,     continuous_cols=tab_preprocessor.continuous_cols,     mlp_hidden_dims=hidden_layers, ) model = WideDeep(deeptabular=deeptabular) model Out[4]: <pre>WideDeep(\n  (deeptabular): Sequential(\n    (0): TabMlp(\n      (cont_norm): Identity()\n      (encoder): MLP(\n        (mlp): Sequential(\n          (dense_layer_0): Sequential(\n            (0): Linear(in_features=74, out_features=148, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_1): Sequential(\n            (0): Linear(in_features=148, out_features=118, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_2): Sequential(\n            (0): Linear(in_features=118, out_features=89, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_3): Sequential(\n            (0): Linear(in_features=89, out_features=59, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_4): Sequential(\n            (0): Linear(in_features=59, out_features=30, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=30, out_features=1, bias=True)\n  )\n)</pre> In\u00a0[5]: Copied! <pre># Metrics from pytorch-widedeep\naccuracy = Accuracy(top_k=2)\nprecision = Precision(average=False)\n\n# # Metrics from torchmetrics\n# accuracy = Accuracy_torchmetrics(average=None, num_classes=1)\n# precision = Precision_torchmetrics(average=\"micro\", num_classes=1)\n\n# Optimizers\ndeep_opt = SGD(model.deeptabular.parameters(), lr=0.1)\n\n# LR Scheduler\ndeep_sch = lr_scheduler.StepLR(deep_opt, step_size=3)\n\ntrainer = Trainer(\n    model,\n    objective=\"binary\",\n    lr_schedulers={\"deeptabular\": deep_sch},\n    initializers={\"deeptabular\": XavierNormal},\n    optimizers={\"deeptabular\": deep_opt},\n    metrics=[accuracy, precision],\n    verbose=1,\n)\n</pre> # Metrics from pytorch-widedeep accuracy = Accuracy(top_k=2) precision = Precision(average=False)  # # Metrics from torchmetrics # accuracy = Accuracy_torchmetrics(average=None, num_classes=1) # precision = Precision_torchmetrics(average=\"micro\", num_classes=1)  # Optimizers deep_opt = SGD(model.deeptabular.parameters(), lr=0.1)  # LR Scheduler deep_sch = lr_scheduler.StepLR(deep_opt, step_size=3)  trainer = Trainer(     model,     objective=\"binary\",     lr_schedulers={\"deeptabular\": deep_sch},     initializers={\"deeptabular\": XavierNormal},     optimizers={\"deeptabular\": deep_opt},     metrics=[accuracy, precision],     verbose=1, ) In\u00a0[6]: Copied! <pre>start = time.time()\ntrainer.fit(\n    X_train={\"X_tab\": X_tab_train, \"target\": y_train},\n    X_val={\"X_tab\": X_tab_valid, \"target\": y_valid},\n    n_epochs=1,\n    batch_size=32,\n    custom_dataloader=DataLoaderImbalanced,\n    oversample_mul=5,\n)\nprint(\n    \"Training time[s]: {}\".format(\n        datetime.timedelta(seconds=round(time.time() - start))\n    )\n)\n\npd.DataFrame(trainer.history)\n\ndf_pred = trainer.predict(X_tab=X_tab_test)\nprint(classification_report(df_test[\"target\"].to_list(), df_pred))\nprint(\"Actual predicted values:\\n{}\".format(np.unique(df_pred, return_counts=True)))\n</pre> start = time.time() trainer.fit(     X_train={\"X_tab\": X_tab_train, \"target\": y_train},     X_val={\"X_tab\": X_tab_valid, \"target\": y_valid},     n_epochs=1,     batch_size=32,     custom_dataloader=DataLoaderImbalanced,     oversample_mul=5, ) print(     \"Training time[s]: {}\".format(         datetime.timedelta(seconds=round(time.time() - start))     ) )  pd.DataFrame(trainer.history)  df_pred = trainer.predict(X_tab=X_tab_test) print(classification_report(df_test[\"target\"].to_list(), df_pred)) print(\"Actual predicted values:\\n{}\".format(np.unique(df_pred, return_counts=True))) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 325/325 [00:02&lt;00:00, 153.99it/s, loss=0.163, metrics={'acc': 0.9363, 'prec': [0.9358]}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 456/456 [00:02&lt;00:00, 205.93it/s, loss=0.1, metrics={'acc': 0.9501, 'prec': [0.1447]}]\n</pre> <pre>Training time[s]: 0:00:04\n</pre> <pre>predict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 456/456 [00:01&lt;00:00, 368.16it/s]\n</pre> <pre>              precision    recall  f1-score   support\n\n           0       1.00      0.95      0.97     14446\n           1       0.15      0.95      0.25       130\n\n    accuracy                           0.95     14576\n   macro avg       0.57      0.95      0.61     14576\nweighted avg       0.99      0.95      0.97     14576\n\nActual predicted values:\n(array([0, 1]), array([13736,   840]))\n</pre>"},{"location":"examples/08_custom_dataLoader_imbalanced_dataset.html#custom-dataloader-for-imbalanced-dataset","title":"Custom DataLoader for Imbalanced dataset\u00b6","text":""},{"location":"examples/08_custom_dataLoader_imbalanced_dataset.html#initial-imports","title":"Initial imports\u00b6","text":""},{"location":"examples/09_extracting_embeddings.html","title":"09_extracting_embeddings","text":"In\u00a0[1]: Copied! <pre>import numpy as np\nimport pandas as pd\nimport torch\n\nfrom pytorch_widedeep.preprocessing import TabPreprocessor\nfrom pytorch_widedeep.training import Trainer\nfrom pytorch_widedeep.models import FTTransformer, WideDeep\nfrom pytorch_widedeep.metrics import Accuracy\nfrom pytorch_widedeep import Tab2Vec\nfrom pytorch_widedeep.datasets import load_adult\n</pre> import numpy as np import pandas as pd import torch  from pytorch_widedeep.preprocessing import TabPreprocessor from pytorch_widedeep.training import Trainer from pytorch_widedeep.models import FTTransformer, WideDeep from pytorch_widedeep.metrics import Accuracy from pytorch_widedeep import Tab2Vec from pytorch_widedeep.datasets import load_adult In\u00a0[2]: Copied! <pre>df = load_adult(as_frame=True)\ndf.head()\n</pre> df = load_adult(as_frame=True) df.head() Out[2]: age workclass fnlwgt education educational-num marital-status occupation relationship race gender capital-gain capital-loss hours-per-week native-country income 0 25 Private 226802 11th 7 Never-married Machine-op-inspct Own-child Black Male 0 0 40 United-States &lt;=50K 1 38 Private 89814 HS-grad 9 Married-civ-spouse Farming-fishing Husband White Male 0 0 50 United-States &lt;=50K 2 28 Local-gov 336951 Assoc-acdm 12 Married-civ-spouse Protective-serv Husband White Male 0 0 40 United-States &gt;50K 3 44 Private 160323 Some-college 10 Married-civ-spouse Machine-op-inspct Husband Black Male 7688 0 40 United-States &gt;50K 4 18 ? 103497 Some-college 10 Never-married ? Own-child White Female 0 0 30 United-States &lt;=50K In\u00a0[3]: Copied! <pre># For convenience, we'll replace '-' with '_'\ndf.columns = [c.replace(\"-\", \"_\") for c in df.columns]\n# binary target\ndf[\"target\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int)\ndf.drop([\"income\", \"educational_num\"], axis=1, inplace=True)\n\ndf.head()\n</pre> # For convenience, we'll replace '-' with '_' df.columns = [c.replace(\"-\", \"_\") for c in df.columns] # binary target df[\"target\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int) df.drop([\"income\", \"educational_num\"], axis=1, inplace=True)  df.head() Out[3]: age workclass fnlwgt education marital_status occupation relationship race gender capital_gain capital_loss hours_per_week native_country target 0 25 Private 226802 11th Never-married Machine-op-inspct Own-child Black Male 0 0 40 United-States 0 1 38 Private 89814 HS-grad Married-civ-spouse Farming-fishing Husband White Male 0 0 50 United-States 0 2 28 Local-gov 336951 Assoc-acdm Married-civ-spouse Protective-serv Husband White Male 0 0 40 United-States 1 3 44 Private 160323 Some-college Married-civ-spouse Machine-op-inspct Husband Black Male 7688 0 40 United-States 1 4 18 ? 103497 Some-college Never-married ? Own-child White Female 0 0 30 United-States 0 In\u00a0[4]: Copied! <pre>cat_cols, cont_cols = [], []\nfor col in df.columns:\n    # 50 is just a random number I choose here for this example\n    if df[col].dtype == \"O\" or df[col].nunique() &lt; 50 and col != \"target\":\n        cat_cols.append(col)\n    elif col != \"target\":\n        cont_cols.append(col)\ntarget_col = \"target\"\n</pre> cat_cols, cont_cols = [], [] for col in df.columns:     # 50 is just a random number I choose here for this example     if df[col].dtype == \"O\" or df[col].nunique() &lt; 50 and col != \"target\":         cat_cols.append(col)     elif col != \"target\":         cont_cols.append(col) target_col = \"target\" In\u00a0[5]: Copied! <pre>target = df[target_col].values\n\ntab_preprocessor = TabPreprocessor(\n    embed_cols=cat_cols, continuous_cols=cont_cols, for_transformer=True\n)\nX_tab = tab_preprocessor.fit_transform(df)\n</pre> target = df[target_col].values  tab_preprocessor = TabPreprocessor(     embed_cols=cat_cols, continuous_cols=cont_cols, for_transformer=True ) X_tab = tab_preprocessor.fit_transform(df) <pre>/Users/javierrodriguezzaurin/Projects/pytorch-widedeep/pytorch_widedeep/preprocessing/tab_preprocessor.py:358: UserWarning: Continuous columns will not be normalised\n  warnings.warn(\"Continuous columns will not be normalised\")\n</pre> In\u00a0[6]: Copied! <pre>ft_transformer = FTTransformer(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=tab_preprocessor.continuous_cols,\n    embed_continuous_method=\"standard\",\n    n_blocks=2,\n    n_heads=4,\n    input_dim=16,\n)\n</pre> ft_transformer = FTTransformer(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     continuous_cols=tab_preprocessor.continuous_cols,     embed_continuous_method=\"standard\",     n_blocks=2,     n_heads=4,     input_dim=16, ) <pre>/Users/javierrodriguezzaurin/Projects/pytorch-widedeep/pytorch_widedeep/utils/general_utils.py:12: DeprecationWarning: The 'embed_continuous' parameter is deprecated and will be removed in the next release. Please use 'embed_continuous_method' instead See the documentation for more details.\n  return func(*args, **kwargs)\n</pre> In\u00a0[7]: Copied! <pre>model = WideDeep(deeptabular=ft_transformer)\ntrainer = Trainer(model, objective=\"binary\", metrics=[Accuracy])\ntrainer.fit(X_tab=X_tab, target=target, n_epochs=1, batch_size=256, val_split=0.2)\n</pre> model = WideDeep(deeptabular=ft_transformer) trainer = Trainer(model, objective=\"binary\", metrics=[Accuracy]) trainer.fit(X_tab=X_tab, target=target, n_epochs=1, batch_size=256, val_split=0.2) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:03&lt;00:00, 41.47it/s, loss=221, metrics={'acc': 0.686}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 39/39 [00:00&lt;00:00, 86.24it/s, loss=9.28, metrics={'acc': 0.76}]\n</pre> In\u00a0[8]: Copied! <pre>t2v = Tab2Vec(model=model, tab_preprocessor=tab_preprocessor)\n</pre> t2v = Tab2Vec(model=model, tab_preprocessor=tab_preprocessor) In\u00a0[9]: Copied! <pre># assuming is a test set with target col\nX_vec, y = t2v.transform(df.sample(100), target_col=\"target\")\n</pre> # assuming is a test set with target col X_vec, y = t2v.transform(df.sample(100), target_col=\"target\") In\u00a0[10]: Copied! <pre># X vec is the dataframe turned into the embeddings\nX_vec.shape\n</pre> # X vec is the dataframe turned into the embeddings X_vec.shape Out[10]: <pre>(100, 208)</pre> <p><code>208 = input_dim (16) * n_cols (13)</code></p> In\u00a0[11]: Copied! <pre># ...or if we don't have target col\nX_vec = t2v.transform(df.sample(100))\n</pre> # ...or if we don't have target col X_vec = t2v.transform(df.sample(100))"},{"location":"examples/09_extracting_embeddings.html#extracting-embeddings","title":"Extracting embeddings\u00b6","text":"<p>This notebook is a simple guide to extracting learned feature embeddings using Tab2Vec</p>"},{"location":"examples/10_3rd_party_integration-RayTune_WnB.html","title":"10_3rd_party_integration-RayTune_WnB","text":"In\u00a0[11]: Copied! <pre>from typing import Optional, Dict\nimport os\n\nimport numpy as np\nimport pandas as pd\nimport torch\nimport wandb\nfrom torch.optim import SGD, lr_scheduler\n\nfrom pytorch_widedeep import Trainer\nfrom pytorch_widedeep.preprocessing import TabPreprocessor\nfrom pytorch_widedeep.models import TabMlp, WideDeep\nfrom torchmetrics import F1Score as F1_torchmetrics\nfrom torchmetrics import Accuracy as Accuracy_torchmetrics\nfrom torchmetrics import Precision as Precision_torchmetrics\nfrom torchmetrics import Recall as Recall_torchmetrics\nfrom pytorch_widedeep.metrics import Accuracy, Recall, Precision, F1Score, R2Score\nfrom pytorch_widedeep.initializers import XavierNormal\nfrom pytorch_widedeep.callbacks import (\n    EarlyStopping,\n    ModelCheckpoint,\n    Callback,\n)\nfrom pytorch_widedeep.datasets import load_bio_kdd04\n\nfrom sklearn.model_selection import train_test_split\nimport warnings\n\nwarnings.filterwarnings(\"ignore\", category=DeprecationWarning)\n\nfrom ray import tune\nfrom ray.tune.schedulers import AsyncHyperBandScheduler\nfrom ray.tune import JupyterNotebookReporter\nfrom ray.air.integrations.wandb import WandbLoggerCallback\n\n# from ray.tune.integration.wandb import wandb_mixin\n\nimport tracemalloc\n\ntracemalloc.start()\n\n# increase displayed columns in jupyter notebook\npd.set_option(\"display.max_columns\", 200)\npd.set_option(\"display.max_rows\", 300)\n</pre> from typing import Optional, Dict import os  import numpy as np import pandas as pd import torch import wandb from torch.optim import SGD, lr_scheduler  from pytorch_widedeep import Trainer from pytorch_widedeep.preprocessing import TabPreprocessor from pytorch_widedeep.models import TabMlp, WideDeep from torchmetrics import F1Score as F1_torchmetrics from torchmetrics import Accuracy as Accuracy_torchmetrics from torchmetrics import Precision as Precision_torchmetrics from torchmetrics import Recall as Recall_torchmetrics from pytorch_widedeep.metrics import Accuracy, Recall, Precision, F1Score, R2Score from pytorch_widedeep.initializers import XavierNormal from pytorch_widedeep.callbacks import (     EarlyStopping,     ModelCheckpoint,     Callback, ) from pytorch_widedeep.datasets import load_bio_kdd04  from sklearn.model_selection import train_test_split import warnings  warnings.filterwarnings(\"ignore\", category=DeprecationWarning)  from ray import tune from ray.tune.schedulers import AsyncHyperBandScheduler from ray.tune import JupyterNotebookReporter from ray.air.integrations.wandb import WandbLoggerCallback  # from ray.tune.integration.wandb import wandb_mixin  import tracemalloc  tracemalloc.start()  # increase displayed columns in jupyter notebook pd.set_option(\"display.max_columns\", 200) pd.set_option(\"display.max_rows\", 300) In\u00a0[12]: Copied! <pre>class RayTuneReporter(Callback):\n    r\"\"\"Callback that allows reporting history and lr_history values to RayTune\n    during Hyperparameter tuning\n\n    Callbacks are passed as input parameters to the ``Trainer`` class. See\n    :class:`pytorch_widedeep.trainer.Trainer`\n\n    For examples see the examples folder at:\n\n        .. code-block:: bash\n\n            /examples/12_HyperParameter_tuning_w_RayTune.ipynb\n    \"\"\"\n\n    def on_epoch_end(\n        self, epoch: int, logs: Optional[Dict] = None, metric: Optional[float] = None\n    ):\n        report_dict = {}\n        for k, v in self.trainer.history.items():\n            report_dict.update({k: v[-1]})\n        if hasattr(self.trainer, \"lr_history\"):\n            for k, v in self.trainer.lr_history.items():\n                report_dict.update({k: v[-1]})\n        tune.report(report_dict)\n\n\nclass WnBReportBest(Callback):\n    r\"\"\"Callback that allows reporting best performance of a run to WnB\n    during Hyperparameter tuning. It is an adjusted pytorch_widedeep.callbacks.ModelCheckpoint\n    with added WnB and removed checkpoint saving.\n\n    Callbacks are passed as input parameters to the ``Trainer`` class.\n\n    Parameters\n    ----------\n    wb: obj\n        Weights&amp;Biases API interface to report single best result usable for\n        comparisson of multiple paramater combinations by, for example,\n        `parallel coordinates\n        &lt;https://docs.wandb.ai/ref/app/features/panels/parallel-coordinates&gt;`_.\n        E.g W&amp;B summary report `wandb.run.summary[\"best\"]`.\n    monitor: str, default=\"loss\"\n        quantity to monitor. Typically `'val_loss'` or metric name\n        (e.g. `'val_acc'`)\n    mode: str, default=\"auto\"\n        If ``save_best_only=True``, the decision to overwrite the current save\n        file is made based on either the maximization or the minimization of\n        the monitored quantity. For `'acc'`, this should be `'max'`, for\n        `'loss'` this should be `'min'`, etc. In `'auto'` mode, the\n        direction is automatically inferred from the name of the monitored\n        quantity.\n\n    \"\"\"\n\n    def __init__(\n        self,\n        wb: object,\n        monitor: str = \"val_loss\",\n        mode: str = \"auto\",\n    ):\n        super(WnBReportBest, self).__init__()\n\n        self.monitor = monitor\n        self.mode = mode\n        self.wb = wb\n\n        if self.mode not in [\"auto\", \"min\", \"max\"]:\n            warnings.warn(\n                \"WnBReportBest mode %s is unknown, \"\n                \"fallback to auto mode.\" % (self.mode),\n                RuntimeWarning,\n            )\n            self.mode = \"auto\"\n        if self.mode == \"min\":\n            self.monitor_op = np.less\n            self.best = np.Inf\n        elif self.mode == \"max\":\n            self.monitor_op = np.greater  # type: ignore[assignment]\n            self.best = -np.Inf\n        else:\n            if self._is_metric(self.monitor):\n                self.monitor_op = np.greater  # type: ignore[assignment]\n                self.best = -np.Inf\n            else:\n                self.monitor_op = np.less\n                self.best = np.Inf\n\n    def on_epoch_end(  # noqa: C901\n        self, epoch: int, logs: Optional[Dict] = None, metric: Optional[float] = None\n    ):\n        logs = logs or {}\n        current = logs.get(self.monitor)\n        if current is not None:\n            if self.monitor_op(current, self.best):\n                self.wb.run.summary[\"best\"] = current  # type: ignore[attr-defined]\n                self.best = current\n                self.best_epoch = epoch\n\n    @staticmethod\n    def _is_metric(monitor: str):\n        \"copied from pytorch_widedeep.callbacks\"\n        if any([s in monitor for s in [\"acc\", \"prec\", \"rec\", \"fscore\", \"f1\", \"f2\"]]):\n            return True\n        else:\n            return False\n</pre> class RayTuneReporter(Callback):     r\"\"\"Callback that allows reporting history and lr_history values to RayTune     during Hyperparameter tuning      Callbacks are passed as input parameters to the ``Trainer`` class. See     :class:`pytorch_widedeep.trainer.Trainer`      For examples see the examples folder at:          .. code-block:: bash              /examples/12_HyperParameter_tuning_w_RayTune.ipynb     \"\"\"      def on_epoch_end(         self, epoch: int, logs: Optional[Dict] = None, metric: Optional[float] = None     ):         report_dict = {}         for k, v in self.trainer.history.items():             report_dict.update({k: v[-1]})         if hasattr(self.trainer, \"lr_history\"):             for k, v in self.trainer.lr_history.items():                 report_dict.update({k: v[-1]})         tune.report(report_dict)   class WnBReportBest(Callback):     r\"\"\"Callback that allows reporting best performance of a run to WnB     during Hyperparameter tuning. It is an adjusted pytorch_widedeep.callbacks.ModelCheckpoint     with added WnB and removed checkpoint saving.      Callbacks are passed as input parameters to the ``Trainer`` class.      Parameters     ----------     wb: obj         Weights&amp;Biases API interface to report single best result usable for         comparisson of multiple paramater combinations by, for example,         `parallel coordinates         `_.         E.g W&amp;B summary report `wandb.run.summary[\"best\"]`.     monitor: str, default=\"loss\"         quantity to monitor. Typically `'val_loss'` or metric name         (e.g. `'val_acc'`)     mode: str, default=\"auto\"         If ``save_best_only=True``, the decision to overwrite the current save         file is made based on either the maximization or the minimization of         the monitored quantity. For `'acc'`, this should be `'max'`, for         `'loss'` this should be `'min'`, etc. In `'auto'` mode, the         direction is automatically inferred from the name of the monitored         quantity.      \"\"\"      def __init__(         self,         wb: object,         monitor: str = \"val_loss\",         mode: str = \"auto\",     ):         super(WnBReportBest, self).__init__()          self.monitor = monitor         self.mode = mode         self.wb = wb          if self.mode not in [\"auto\", \"min\", \"max\"]:             warnings.warn(                 \"WnBReportBest mode %s is unknown, \"                 \"fallback to auto mode.\" % (self.mode),                 RuntimeWarning,             )             self.mode = \"auto\"         if self.mode == \"min\":             self.monitor_op = np.less             self.best = np.Inf         elif self.mode == \"max\":             self.monitor_op = np.greater  # type: ignore[assignment]             self.best = -np.Inf         else:             if self._is_metric(self.monitor):                 self.monitor_op = np.greater  # type: ignore[assignment]                 self.best = -np.Inf             else:                 self.monitor_op = np.less                 self.best = np.Inf      def on_epoch_end(  # noqa: C901         self, epoch: int, logs: Optional[Dict] = None, metric: Optional[float] = None     ):         logs = logs or {}         current = logs.get(self.monitor)         if current is not None:             if self.monitor_op(current, self.best):                 self.wb.run.summary[\"best\"] = current  # type: ignore[attr-defined]                 self.best = current                 self.best_epoch = epoch      @staticmethod     def _is_metric(monitor: str):         \"copied from pytorch_widedeep.callbacks\"         if any([s in monitor for s in [\"acc\", \"prec\", \"rec\", \"fscore\", \"f1\", \"f2\"]]):             return True         else:             return False In\u00a0[13]: Copied! <pre>df = load_bio_kdd04(as_frame=True)\ndf.head()\n</pre> df = load_bio_kdd04(as_frame=True) df.head() Out[13]: EXAMPLE_ID BLOCK_ID target 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 0 279 261532 0 52.0 32.69 0.30 2.5 20.0 1256.8 -0.89 0.33 11.0 -55.0 267.2 0.52 0.05 -2.36 49.6 252.0 0.43 1.16 -2.06 -33.0 -123.2 1.60 -0.49 -6.06 65.0 296.1 -0.28 -0.26 -3.83 -22.6 -170.0 3.06 -1.05 -3.29 22.9 286.3 0.12 2.58 4.08 -33.0 -178.9 1.88 0.53 -7.0 -44.0 1987.0 -5.41 0.95 -4.0 -57.0 722.9 -3.26 -0.55 -7.5 125.5 1547.2 -0.36 1.12 9.0 -37.0 72.5 0.47 0.74 -11.0 -8.0 1595.1 -1.64 2.83 -2.0 -50.0 445.2 -0.35 0.26 0.76 1 279 261533 0 58.0 33.33 0.00 16.5 9.5 608.1 0.50 0.07 20.5 -52.5 521.6 -1.08 0.58 -0.02 -3.2 103.6 -0.95 0.23 -2.87 -25.9 -52.2 -0.21 0.87 -1.81 10.4 62.0 -0.28 -0.04 1.48 -17.6 -198.3 3.43 2.84 5.87 -16.9 72.6 -0.31 2.79 2.71 -33.5 -11.6 -1.11 4.01 5.0 -57.0 666.3 1.13 4.38 5.0 -64.0 39.3 1.07 -0.16 32.5 100.0 1893.7 -2.80 -0.22 2.5 -28.5 45.0 0.58 0.41 -19.0 -6.0 762.9 0.29 0.82 -3.0 -35.0 140.3 1.16 0.39 0.73 2 279 261534 0 77.0 27.27 -0.91 6.0 58.5 1623.6 -1.40 0.02 -6.5 -48.0 621.0 -1.20 0.14 -0.20 73.6 609.1 -0.44 -0.58 -0.04 -23.0 -27.4 -0.72 -1.04 -1.09 91.1 635.6 -0.88 0.24 0.59 -18.7 -7.2 -0.60 -2.82 -0.71 52.4 504.1 0.89 -0.67 -9.30 -20.8 -25.7 -0.77 -0.85 0.0 -20.0 2259.0 -0.94 1.15 -4.0 -44.0 -22.7 0.94 -0.98 -19.0 105.0 1267.9 1.03 1.27 11.0 -39.5 82.3 0.47 -0.19 -10.0 7.0 1491.8 0.32 -1.29 0.0 -34.0 658.2 -0.76 0.26 0.24 3 279 261535 0 41.0 27.91 -0.35 3.0 46.0 1921.6 -1.36 -0.47 -32.0 -51.5 560.9 -0.29 -0.10 -1.11 124.3 791.6 0.00 0.39 -1.85 -21.7 -44.9 -0.21 0.02 0.89 133.9 797.8 -0.08 1.06 -0.26 -16.4 -74.1 0.97 -0.80 -0.41 66.9 955.3 -1.90 1.28 -6.65 -28.1 47.5 -1.91 1.42 1.0 -30.0 1846.7 0.76 1.10 -4.0 -52.0 -53.9 1.71 -0.22 -12.0 97.5 1969.8 -1.70 0.16 -1.0 -32.5 255.9 -0.46 1.57 10.0 6.0 2047.7 -0.98 1.53 0.0 -49.0 554.2 -0.83 0.39 0.73 4 279 261536 0 50.0 28.00 -1.32 -9.0 12.0 464.8 0.88 0.19 8.0 -51.5 98.1 1.09 -0.33 -2.16 -3.9 102.7 0.39 -1.22 -3.39 -15.2 -42.2 -1.18 -1.11 -3.55 8.9 141.3 -0.16 -0.43 -4.15 -12.9 -13.4 -1.32 -0.98 -3.69 8.8 136.1 -0.30 4.13 1.89 -13.0 -18.7 -1.37 -0.93 0.0 -1.0 810.1 -2.29 6.72 1.0 -23.0 -29.7 0.58 -1.10 -18.5 33.5 206.8 1.84 -0.13 4.0 -29.0 30.1 0.80 -0.24 5.0 -14.0 479.5 0.68 -0.59 2.0 -36.0 -6.9 2.02 0.14 -0.23 In\u00a0[14]: Copied! <pre># imbalance of the classes\ndf[\"target\"].value_counts()\n</pre> # imbalance of the classes df[\"target\"].value_counts() Out[14]: <pre>target\n0    144455\n1      1296\nName: count, dtype: int64</pre> In\u00a0[15]: Copied! <pre># drop columns we won't need in this example\ndf.drop(columns=[\"EXAMPLE_ID\", \"BLOCK_ID\"], inplace=True)\n</pre> # drop columns we won't need in this example df.drop(columns=[\"EXAMPLE_ID\", \"BLOCK_ID\"], inplace=True) In\u00a0[16]: Copied! <pre>df_train, df_valid = train_test_split(\n    df, test_size=0.2, stratify=df[\"target\"], random_state=1\n)\ndf_valid, df_test = train_test_split(\n    df_valid, test_size=0.5, stratify=df_valid[\"target\"], random_state=1\n)\n</pre> df_train, df_valid = train_test_split(     df, test_size=0.2, stratify=df[\"target\"], random_state=1 ) df_valid, df_test = train_test_split(     df_valid, test_size=0.5, stratify=df_valid[\"target\"], random_state=1 ) In\u00a0[17]: Copied! <pre>continuous_cols = df.drop(columns=[\"target\"]).columns.values.tolist()\n</pre> continuous_cols = df.drop(columns=[\"target\"]).columns.values.tolist() In\u00a0[18]: Copied! <pre># deeptabular\ntab_preprocessor = TabPreprocessor(continuous_cols=continuous_cols, scale=True)\nX_tab_train = tab_preprocessor.fit_transform(df_train)\nX_tab_valid = tab_preprocessor.transform(df_valid)\nX_tab_test = tab_preprocessor.transform(df_test)\n\n# target\ny_train = df_train[\"target\"].values\ny_valid = df_valid[\"target\"].values\ny_test = df_test[\"target\"].values\n</pre> # deeptabular tab_preprocessor = TabPreprocessor(continuous_cols=continuous_cols, scale=True) X_tab_train = tab_preprocessor.fit_transform(df_train) X_tab_valid = tab_preprocessor.transform(df_valid) X_tab_test = tab_preprocessor.transform(df_test)  # target y_train = df_train[\"target\"].values y_valid = df_valid[\"target\"].values y_test = df_test[\"target\"].values In\u00a0[19]: Copied! <pre>input_layer = len(tab_preprocessor.continuous_cols)\noutput_layer = 1\nhidden_layers = np.linspace(\n    input_layer * 2, output_layer, 5, endpoint=False, dtype=int\n).tolist()\n</pre> input_layer = len(tab_preprocessor.continuous_cols) output_layer = 1 hidden_layers = np.linspace(     input_layer * 2, output_layer, 5, endpoint=False, dtype=int ).tolist() In\u00a0[20]: Copied! <pre>deeptabular = TabMlp(\n    mlp_hidden_dims=hidden_layers,\n    column_idx=tab_preprocessor.column_idx,\n    continuous_cols=tab_preprocessor.continuous_cols,\n)\nmodel = WideDeep(deeptabular=deeptabular)\nmodel\n</pre> deeptabular = TabMlp(     mlp_hidden_dims=hidden_layers,     column_idx=tab_preprocessor.column_idx,     continuous_cols=tab_preprocessor.continuous_cols, ) model = WideDeep(deeptabular=deeptabular) model Out[20]: <pre>WideDeep(\n  (deeptabular): Sequential(\n    (0): TabMlp(\n      (cont_norm): Identity()\n      (encoder): MLP(\n        (mlp): Sequential(\n          (dense_layer_0): Sequential(\n            (0): Linear(in_features=74, out_features=148, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_1): Sequential(\n            (0): Linear(in_features=148, out_features=118, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_2): Sequential(\n            (0): Linear(in_features=118, out_features=89, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_3): Sequential(\n            (0): Linear(in_features=89, out_features=59, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_4): Sequential(\n            (0): Linear(in_features=59, out_features=30, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=30, out_features=1, bias=True)\n  )\n)</pre> In\u00a0[24]: Copied! <pre># Metrics from torchmetrics\naccuracy = Accuracy_torchmetrics(average=None, num_classes=1, task=\"binary\")\nprecision = Precision_torchmetrics(average=\"micro\", num_classes=1, task=\"binary\")\nf1 = F1_torchmetrics(average=None, num_classes=1, task=\"binary\")\nrecall = Recall_torchmetrics(average=None, num_classes=1, task=\"binary\")\n</pre> # Metrics from torchmetrics accuracy = Accuracy_torchmetrics(average=None, num_classes=1, task=\"binary\") precision = Precision_torchmetrics(average=\"micro\", num_classes=1, task=\"binary\") f1 = F1_torchmetrics(average=None, num_classes=1, task=\"binary\") recall = Recall_torchmetrics(average=None, num_classes=1, task=\"binary\") <p>Note:</p> <p>Following cells includes usage of both <code>RayTuneReporter</code> and <code>WnBReportBest</code> callbacks. In case you want to use just <code>RayTuneReporter</code>, remove following:</p> <ul> <li>wandb from config</li> <li><code>WandbLoggerCallback</code></li> <li><code>WnBReportBest</code></li> <li><code>@wandb_mixin</code> decorator</li> </ul> <p>We do not see strong reason to use WnB without RayTune for a single paramater combination run, but it is possible:</p> <ul> <li>option01: define paramaters in config only for a single value <code>tune.grid_search([1000])</code> (single value RayTune run)</li> <li>option02: define WnB callback that reports currnet validation/training loss, metrics, etc. at the end of batch, ie. do not report to WnB at <code>epoch_end</code> as in <code>WnBReportBest</code> but at the <code>on_batch_end</code>, see <code>pytorch_widedeep.callbacks.Callback</code></li> </ul> In\u00a0[26]: Copied! <pre>config = {\n    \"batch_size\": tune.grid_search([1000, 5000]),\n    \"wandb\": {\n        \"project\": \"test\",\n        # \"api_key_file\": os.getcwd() + \"/wandb_api.key\",\n        \"api_key\": \"WNB_API_KEY\",\n    },\n}\n\n# Optimizers\ndeep_opt = SGD(model.deeptabular.parameters(), lr=0.1)\n# LR Scheduler\ndeep_sch = lr_scheduler.StepLR(deep_opt, step_size=3)\n\n\n@wandb_mixin\ndef training_function(config, X_train, X_val):\n    early_stopping = EarlyStopping()\n    model_checkpoint = ModelCheckpoint(save_best_only=True)\n    # Hyperparameters\n    batch_size = config[\"batch_size\"]\n    trainer = Trainer(\n        model,\n        objective=\"binary_focal_loss\",\n        callbacks=[\n            RayTuneReporter,\n            WnBReportBest(wb=wandb),\n            early_stopping,\n            model_checkpoint,\n        ],\n        lr_schedulers={\"deeptabular\": deep_sch},\n        initializers={\"deeptabular\": XavierNormal},\n        optimizers={\"deeptabular\": deep_opt},\n        metrics=[accuracy, precision, recall, f1],\n        verbose=0,\n    )\n\n    trainer.fit(X_train=X_train, X_val=X_val, n_epochs=5, batch_size=batch_size)\n\n\nX_train = {\"X_tab\": X_tab_train, \"target\": y_train}\nX_val = {\"X_tab\": X_tab_valid, \"target\": y_valid}\n\nasha_scheduler = AsyncHyperBandScheduler(\n    time_attr=\"training_iteration\",\n    metric=\"_metric/val_loss\",\n    mode=\"min\",\n    max_t=100,\n    grace_period=10,\n    reduction_factor=3,\n    brackets=1,\n)\n\nanalysis = tune.run(\n    tune.with_parameters(training_function, X_train=X_train, X_val=X_val),\n    resources_per_trial={\"cpu\": 1, \"gpu\": 0},\n    progress_reporter=JupyterNotebookReporter(overwrite=True),\n    scheduler=asha_scheduler,\n    config=config,\n    callbacks=[\n        WandbLoggerCallback(\n            project=config[\"wandb\"][\"project\"],\n            # api_key_file=config[\"wandb\"][\"api_key_file\"],\n            api_key=config[\"wandb\"][\"api_key\"],\n            log_config=True,\n        )\n    ],\n)\n</pre> config = {     \"batch_size\": tune.grid_search([1000, 5000]),     \"wandb\": {         \"project\": \"test\",         # \"api_key_file\": os.getcwd() + \"/wandb_api.key\",         \"api_key\": \"WNB_API_KEY\",     }, }  # Optimizers deep_opt = SGD(model.deeptabular.parameters(), lr=0.1) # LR Scheduler deep_sch = lr_scheduler.StepLR(deep_opt, step_size=3)   @wandb_mixin def training_function(config, X_train, X_val):     early_stopping = EarlyStopping()     model_checkpoint = ModelCheckpoint(save_best_only=True)     # Hyperparameters     batch_size = config[\"batch_size\"]     trainer = Trainer(         model,         objective=\"binary_focal_loss\",         callbacks=[             RayTuneReporter,             WnBReportBest(wb=wandb),             early_stopping,             model_checkpoint,         ],         lr_schedulers={\"deeptabular\": deep_sch},         initializers={\"deeptabular\": XavierNormal},         optimizers={\"deeptabular\": deep_opt},         metrics=[accuracy, precision, recall, f1],         verbose=0,     )      trainer.fit(X_train=X_train, X_val=X_val, n_epochs=5, batch_size=batch_size)   X_train = {\"X_tab\": X_tab_train, \"target\": y_train} X_val = {\"X_tab\": X_tab_valid, \"target\": y_valid}  asha_scheduler = AsyncHyperBandScheduler(     time_attr=\"training_iteration\",     metric=\"_metric/val_loss\",     mode=\"min\",     max_t=100,     grace_period=10,     reduction_factor=3,     brackets=1, )  analysis = tune.run(     tune.with_parameters(training_function, X_train=X_train, X_val=X_val),     resources_per_trial={\"cpu\": 1, \"gpu\": 0},     progress_reporter=JupyterNotebookReporter(overwrite=True),     scheduler=asha_scheduler,     config=config,     callbacks=[         WandbLoggerCallback(             project=config[\"wandb\"][\"project\"],             # api_key_file=config[\"wandb\"][\"api_key_file\"],             api_key=config[\"wandb\"][\"api_key\"],             log_config=True,         )     ], ) <pre>/Users/javierrodriguezzaurin/.pyenv/versions/3.10.13/lib/python3.10/tempfile.py:860: ResourceWarning: Implicitly cleaning up &lt;TemporaryDirectory '/var/folders/_2/lrjn1qn54c758tdtktr1bvkc0000gn/T/tmp60pfyl1kwandb'&gt;\n  _warnings.warn(warn_message, ResourceWarning)\n/Users/javierrodriguezzaurin/.pyenv/versions/3.10.13/lib/python3.10/tempfile.py:860: ResourceWarning: Implicitly cleaning up &lt;TemporaryDirectory '/var/folders/_2/lrjn1qn54c758tdtktr1bvkc0000gn/T/tmpnjv2rg1wwandb-artifacts'&gt;\n  _warnings.warn(warn_message, ResourceWarning)\n/Users/javierrodriguezzaurin/.pyenv/versions/3.10.13/lib/python3.10/tempfile.py:860: ResourceWarning: Implicitly cleaning up &lt;TemporaryDirectory '/var/folders/_2/lrjn1qn54c758tdtktr1bvkc0000gn/T/tmpgebu5k1kwandb-media'&gt;\n  _warnings.warn(warn_message, ResourceWarning)\n/Users/javierrodriguezzaurin/.pyenv/versions/3.10.13/lib/python3.10/tempfile.py:860: ResourceWarning: Implicitly cleaning up &lt;TemporaryDirectory '/var/folders/_2/lrjn1qn54c758tdtktr1bvkc0000gn/T/tmpxy9y2yriwandb-media'&gt;\n  _warnings.warn(warn_message, ResourceWarning)\n</pre> In\u00a0[14]: Copied! <pre>analysis.results\n</pre> analysis.results Out[14]: <pre>{'fc9a8_00000': {'_metric': {'train_loss': 0.006297602537127896,\n   'train_Accuracy': 0.9925042986869812,\n   'train_Precision': 0.9939393997192383,\n   'train_Recall': 0.15814851224422455,\n   'train_F1Score': 0.2728785574436188,\n   'val_loss': 0.005045663565397263,\n   'val_Accuracy': 0.9946483969688416,\n   'val_Precision': 1.0,\n   'val_Recall': 0.39534884691238403,\n   'val_F1Score': 0.5666667222976685},\n  'time_this_iter_s': 2.388202428817749,\n  'done': True,\n  'timesteps_total': None,\n  'episodes_total': None,\n  'training_iteration': 5,\n  'trial_id': 'fc9a8_00000',\n  'experiment_id': 'baad1d4f3d924b48b9ece1b9f26c80cc',\n  'date': '2022-07-31_14-06-51',\n  'timestamp': 1659276411,\n  'time_total_s': 12.656474113464355,\n  'pid': 1813,\n  'hostname': 'jupyter-5uperpalo',\n  'node_ip': '10.32.44.172',\n  'config': {'batch_size': 1000},\n  'time_since_restore': 12.656474113464355,\n  'timesteps_since_restore': 0,\n  'iterations_since_restore': 5,\n  'warmup_time': 0.8006253242492676,\n  'experiment_tag': '0_batch_size=1000'},\n 'fc9a8_00001': {'_metric': {'train_loss': 0.02519632239515583,\n   'train_Accuracy': 0.9910891652107239,\n   'train_Precision': 0.25,\n   'train_Recall': 0.0009643201483413577,\n   'train_F1Score': 0.0019212296465411782,\n   'val_loss': 0.02578434906899929,\n   'val_Accuracy': 0.9911492466926575,\n   'val_Precision': 0.0,\n   'val_Recall': 0.0,\n   'val_F1Score': 0.0},\n  'time_this_iter_s': 4.113586902618408,\n  'done': True,\n  'timesteps_total': None,\n  'episodes_total': None,\n  'training_iteration': 5,\n  'trial_id': 'fc9a8_00001',\n  'experiment_id': 'f2e54a6a5780429fbf0db0746853347e',\n  'date': '2022-07-31_14-06-56',\n  'timestamp': 1659276416,\n  'time_total_s': 12.926990509033203,\n  'pid': 1962,\n  'hostname': 'jupyter-5uperpalo',\n  'node_ip': '10.32.44.172',\n  'config': {'batch_size': 5000},\n  'time_since_restore': 12.926990509033203,\n  'timesteps_since_restore': 0,\n  'iterations_since_restore': 5,\n  'warmup_time': 0.9253025054931641,\n  'experiment_tag': '1_batch_size=5000'}}</pre> <p>Using Weights and Biases logging you can create parallel coordinates graphs that map parametr combinations to the best(lowest) loss achieved during the training of the networks</p> <p></p> <p>local visualization of raytune reults using tensorboard</p> In\u00a0[23]: Copied! <pre>%load_ext tensorboard\n%tensorboard --logdir ~/ray_results\n</pre> %load_ext tensorboard %tensorboard --logdir ~/ray_results"},{"location":"examples/10_3rd_party_integration-RayTune_WnB.html#3rd-party-integration-raytune-weights-biases","title":"3rd party integration - RayTune, Weights &amp; Biases\u00b6","text":"<p>This notebook provides guideline for integration of external library functions in the model training process through <code>Callback</code> objects, a popular concept of using objects as arguments for other objects.</p> <p>[DISCLAIMER]</p> <p>We show integration of RayTune (a hyperparameter tuning framework) and Weights &amp; Biases (ML projects experiment tracking and versioning solution) in the <code>pytorch_widedeep</code> model training process. We did not include <code>RayTuneReporter</code> and <code>WnBReportBest</code> in the library code to minimize the dependencies on other libraries that are not directly included in the model design and training.</p>"},{"location":"examples/10_3rd_party_integration-RayTune_WnB.html#initial-imports","title":"Initial imports\u00b6","text":""},{"location":"examples/10_3rd_party_integration-RayTune_WnB.html#preparing-the-data","title":"Preparing the data\u00b6","text":""},{"location":"examples/10_3rd_party_integration-RayTune_WnB.html#define-the-model","title":"Define the model\u00b6","text":""},{"location":"examples/11_auc_multiclass.html","title":"11_auc_multiclass","text":"In\u00a0[1]: Copied! <pre>import numpy as np\nimport pandas as pd\nfrom torch.optim import SGD, lr_scheduler\n\nfrom pytorch_widedeep import Trainer\nfrom pytorch_widedeep.preprocessing import TabPreprocessor\nfrom pytorch_widedeep.models import TabMlp, WideDeep\nfrom torchmetrics import AUROC\nfrom pytorch_widedeep.initializers import XavierNormal\nfrom pytorch_widedeep.datasets import load_ecoli\nfrom pytorch_widedeep.utils import LabelEncoder\n\nfrom sklearn.model_selection import train_test_split\n\n# increase displayed columns in jupyter notebook\npd.set_option(\"display.max_columns\", 200)\npd.set_option(\"display.max_rows\", 300)\n</pre> import numpy as np import pandas as pd from torch.optim import SGD, lr_scheduler  from pytorch_widedeep import Trainer from pytorch_widedeep.preprocessing import TabPreprocessor from pytorch_widedeep.models import TabMlp, WideDeep from torchmetrics import AUROC from pytorch_widedeep.initializers import XavierNormal from pytorch_widedeep.datasets import load_ecoli from pytorch_widedeep.utils import LabelEncoder  from sklearn.model_selection import train_test_split  # increase displayed columns in jupyter notebook pd.set_option(\"display.max_columns\", 200) pd.set_option(\"display.max_rows\", 300) In\u00a0[2]: Copied! <pre>df = load_ecoli(as_frame=True)\ndf.head()\n</pre> df = load_ecoli(as_frame=True) df.head() Out[2]: SequenceName mcg gvh lip chg aac alm1 alm2 class 0 AAT_ECOLI 0.49 0.29 0.48 0.5 0.56 0.24 0.35 cp 1 ACEA_ECOLI 0.07 0.40 0.48 0.5 0.54 0.35 0.44 cp 2 ACEK_ECOLI 0.56 0.40 0.48 0.5 0.49 0.37 0.46 cp 3 ACKA_ECOLI 0.59 0.49 0.48 0.5 0.52 0.45 0.36 cp 4 ADI_ECOLI 0.23 0.32 0.48 0.5 0.55 0.25 0.35 cp In\u00a0[3]: Copied! <pre># imbalance of the classes\ndf[\"class\"].value_counts()\n</pre> # imbalance of the classes df[\"class\"].value_counts() Out[3]: <pre>class\ncp     143\nim      77\npp      52\nimU     35\nom      20\nomL      5\nimS      2\nimL      2\nName: count, dtype: int64</pre> In\u00a0[4]: Copied! <pre>df = df.loc[~df[\"class\"].isin([\"omL\", \"imS\", \"imL\"])]\ndf.reset_index(inplace=True, drop=True)\n</pre> df = df.loc[~df[\"class\"].isin([\"omL\", \"imS\", \"imL\"])] df.reset_index(inplace=True, drop=True) In\u00a0[5]: Copied! <pre>encoder = LabelEncoder([\"class\"])\ndf_enc = encoder.fit_transform(df)\ndf_enc[\"class\"] = df_enc[\"class\"] - 1\n</pre> encoder = LabelEncoder([\"class\"]) df_enc = encoder.fit_transform(df) df_enc[\"class\"] = df_enc[\"class\"] - 1 In\u00a0[6]: Copied! <pre># drop columns we won't need in this example\ndf_enc = df_enc.drop(columns=[\"SequenceName\"])\n</pre> # drop columns we won't need in this example df_enc = df_enc.drop(columns=[\"SequenceName\"]) In\u00a0[7]: Copied! <pre>df_train, df_valid = train_test_split(\n    df_enc, test_size=0.2, stratify=df_enc[\"class\"], random_state=1\n)\ndf_valid, df_test = train_test_split(\n    df_valid, test_size=0.5, stratify=df_valid[\"class\"], random_state=1\n)\n</pre> df_train, df_valid = train_test_split(     df_enc, test_size=0.2, stratify=df_enc[\"class\"], random_state=1 ) df_valid, df_test = train_test_split(     df_valid, test_size=0.5, stratify=df_valid[\"class\"], random_state=1 ) In\u00a0[8]: Copied! <pre>continuous_cols = df_enc.drop(columns=[\"class\"]).columns.values.tolist()\n</pre> continuous_cols = df_enc.drop(columns=[\"class\"]).columns.values.tolist() In\u00a0[9]: Copied! <pre># deeptabular\ntab_preprocessor = TabPreprocessor(continuous_cols=continuous_cols, scale=True)\nX_tab_train = tab_preprocessor.fit_transform(df_train)\nX_tab_valid = tab_preprocessor.transform(df_valid)\nX_tab_test = tab_preprocessor.transform(df_test)\n\n# target\ny_train = df_train[\"class\"].values\ny_valid = df_valid[\"class\"].values\ny_test = df_test[\"class\"].values\n\nX_train = {\"X_tab\": X_tab_train, \"target\": y_train}\nX_val = {\"X_tab\": X_tab_valid, \"target\": y_valid}\n</pre> # deeptabular tab_preprocessor = TabPreprocessor(continuous_cols=continuous_cols, scale=True) X_tab_train = tab_preprocessor.fit_transform(df_train) X_tab_valid = tab_preprocessor.transform(df_valid) X_tab_test = tab_preprocessor.transform(df_test)  # target y_train = df_train[\"class\"].values y_valid = df_valid[\"class\"].values y_test = df_test[\"class\"].values  X_train = {\"X_tab\": X_tab_train, \"target\": y_train} X_val = {\"X_tab\": X_tab_valid, \"target\": y_valid} <pre>/Users/javierrodriguezzaurin/Projects/pytorch-widedeep/pytorch_widedeep/preprocessing/tab_preprocessor.py:295: DeprecationWarning: 'scale' and 'already_standard' will be deprecated in the next release. Please use 'cols_to_scale' instead\n  self._check_inputs(cat_embed_cols)\n</pre> In\u00a0[10]: Copied! <pre>deeptabular = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    continuous_cols=tab_preprocessor.continuous_cols,\n)\nmodel = WideDeep(deeptabular=deeptabular, pred_dim=df_enc[\"class\"].nunique())\nmodel\n</pre> deeptabular = TabMlp(     column_idx=tab_preprocessor.column_idx,     continuous_cols=tab_preprocessor.continuous_cols, ) model = WideDeep(deeptabular=deeptabular, pred_dim=df_enc[\"class\"].nunique()) model Out[10]: <pre>WideDeep(\n  (deeptabular): Sequential(\n    (0): TabMlp(\n      (cont_norm): Identity()\n      (encoder): MLP(\n        (mlp): Sequential(\n          (dense_layer_0): Sequential(\n            (0): Linear(in_features=7, out_features=200, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_1): Sequential(\n            (0): Linear(in_features=200, out_features=100, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=100, out_features=5, bias=True)\n  )\n)</pre> In\u00a0[11]: Copied! <pre>auroc = AUROC(num_classes=df_enc[\"class\"].nunique(), task=\"multiclass\")\n</pre> auroc = AUROC(num_classes=df_enc[\"class\"].nunique(), task=\"multiclass\") In\u00a0[12]: Copied! <pre># Optimizers\ndeep_opt = SGD(model.deeptabular.parameters(), lr=0.1)\n# LR Scheduler\ndeep_sch = lr_scheduler.StepLR(deep_opt, step_size=3)\n# Hyperparameters\ntrainer = Trainer(\n    model,\n    objective=\"multiclass_focal_loss\",\n    lr_schedulers={\"deeptabular\": deep_sch},\n    initializers={\"deeptabular\": XavierNormal},\n    optimizers={\"deeptabular\": deep_opt},\n    metrics=[auroc],\n)\n\ntrainer.fit(X_train=X_train, X_val=X_val, n_epochs=5, batch_size=50)\n</pre> # Optimizers deep_opt = SGD(model.deeptabular.parameters(), lr=0.1) # LR Scheduler deep_sch = lr_scheduler.StepLR(deep_opt, step_size=3) # Hyperparameters trainer = Trainer(     model,     objective=\"multiclass_focal_loss\",     lr_schedulers={\"deeptabular\": deep_sch},     initializers={\"deeptabular\": XavierNormal},     optimizers={\"deeptabular\": deep_opt},     metrics=[auroc], )  trainer.fit(X_train=X_train, X_val=X_val, n_epochs=5, batch_size=50) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 6/6 [00:00&lt;00:00, 54.59it/s, loss=0.109, metrics={'MulticlassAUROC': 0.314}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 1/1 [00:00&lt;00:00, 98.35it/s, loss=0.105, metrics={'MulticlassAUROC': 0.2558}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 6/6 [00:00&lt;00:00, 91.55it/s, loss=0.105, metrics={'MulticlassAUROC': 0.3546}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 1/1 [00:00&lt;00:00, 111.68it/s, loss=0.101, metrics={'MulticlassAUROC': 0.2737}]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 6/6 [00:00&lt;00:00, 62.55it/s, loss=0.1, metrics={'MulticlassAUROC': 0.3795}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 1/1 [00:00&lt;00:00, 108.51it/s, loss=0.0966, metrics={'MulticlassAUROC': 0.3053}]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 6/6 [00:00&lt;00:00, 99.35it/s, loss=0.0965, metrics={'MulticlassAUROC': 0.3809}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 1/1 [00:00&lt;00:00, 117.73it/s, loss=0.0962, metrics={'MulticlassAUROC': 0.3089}]\nepoch 5: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 6/6 [00:00&lt;00:00, 110.56it/s, loss=0.0967, metrics={'MulticlassAUROC': 0.3509}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 1/1 [00:00&lt;00:00, 127.35it/s, loss=0.0958, metrics={'MulticlassAUROC': 0.3089}]\n</pre> In\u00a0[\u00a0]: Copied! <pre>\n</pre>"},{"location":"examples/11_auc_multiclass.html#auc-multiclass-computation","title":"AUC multiclass computation\u00b6","text":""},{"location":"examples/11_auc_multiclass.html#initial-imports","title":"Initial imports\u00b6","text":""},{"location":"examples/11_auc_multiclass.html#preparing-the-data","title":"Preparing the data\u00b6","text":""},{"location":"examples/11_auc_multiclass.html#define-the-model","title":"Define the model\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html","title":"12_ZILNLoss_origkeras_vs_pytorch_widedeep","text":"In\u00a0[1]: Copied! <pre># @title Copyright 2019 The Lifetime Value Authors.\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     https://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n# ============================================================================\n</pre> # @title Copyright 2019 The Lifetime Value Authors. # Licensed under the Apache License, Version 2.0 (the \"License\"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # #     https://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an \"AS IS\" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ Run in Google Colab View source on GitHub In\u00a0[3]: Copied! <pre>import os\n\nimport numpy as np\nimport pandas as pd\nfrom scipy import stats\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport tensorflow as tf\nimport tensorflow_probability as tfp\nfrom typing import Sequence\n\n# install and import ltv\n!pip install -q git+https://github.com/google/lifetime_value\nimport lifetime_value as ltv\n</pre> import os  import numpy as np import pandas as pd from scipy import stats import matplotlib.pyplot as plt import seaborn as sns import tensorflow as tf import tensorflow_probability as tfp from typing import Sequence  # install and import ltv !pip install -q git+https://github.com/google/lifetime_value import lifetime_value as ltv In\u00a0[\u00a0]: Copied! <pre>tfd = tfp.distributions\n%config InlineBackend.figure_format='retina'\nsns.set_style(\"whitegrid\")\n</pre> tfd = tfp.distributions %config InlineBackend.figure_format='retina' sns.set_style(\"whitegrid\") In\u00a0[\u00a0]: Copied! <pre>MODEL = \"dnn\"\nLOSS = \"ziln\"  # @param { isTemplate: true, type: 'string'} ['mse', 'ziln']\nLEARNING_RATE = 0.001  # @param { isTemplate: true}\nVERSION = 0  # @param { isTemplate: true, type: 'integer'}\nOUTPUT_CSV_FOLDER = \"/tmp/lifetime-value/kdd_cup_98/result\"  # @param { isTemplate: true, type: 'string'}\n</pre> MODEL = \"dnn\" LOSS = \"ziln\"  # @param { isTemplate: true, type: 'string'} ['mse', 'ziln'] LEARNING_RATE = 0.001  # @param { isTemplate: true} VERSION = 0  # @param { isTemplate: true, type: 'integer'} OUTPUT_CSV_FOLDER = \"/tmp/lifetime-value/kdd_cup_98/result\"  # @param { isTemplate: true, type: 'string'} <p>Download kdd_cup_98 data to /tmp/lifetime-value/kdd_cup_98</p> In\u00a0[\u00a0]: Copied! <pre>%%bash\nmkdir -p /tmp/lifetime-value/kdd_cup_98\nwget https://kdd.ics.uci.edu/databases/kddcup98/epsilon_mirror/cup98lrn.zip -P /tmp/lifetime-value/kdd_cup_98/\nwget https://kdd.ics.uci.edu/databases/kddcup98/epsilon_mirror/cup98val.zip -P /tmp/lifetime-value/kdd_cup_98/\nwget https://kdd.ics.uci.edu/databases/kddcup98/epsilon_mirror/valtargt.txt -P /tmp/lifetime-value/kdd_cup_98/\ncd /tmp/lifetime-value/kdd_cup_98/\nunzip cup98lrn.zip\nunzip cup98val.zip\n</pre> %%bash mkdir -p /tmp/lifetime-value/kdd_cup_98 wget https://kdd.ics.uci.edu/databases/kddcup98/epsilon_mirror/cup98lrn.zip -P /tmp/lifetime-value/kdd_cup_98/ wget https://kdd.ics.uci.edu/databases/kddcup98/epsilon_mirror/cup98val.zip -P /tmp/lifetime-value/kdd_cup_98/ wget https://kdd.ics.uci.edu/databases/kddcup98/epsilon_mirror/valtargt.txt -P /tmp/lifetime-value/kdd_cup_98/ cd /tmp/lifetime-value/kdd_cup_98/ unzip cup98lrn.zip unzip cup98val.zip In\u00a0[\u00a0]: Copied! <pre>df_train = pd.read_csv(\"/tmp/lifetime-value/kdd_cup_98/cup98LRN.txt\")\nnum_train = df_train.shape[0]\ndf_eval = pd.read_csv(\"/tmp/lifetime-value/kdd_cup_98/cup98VAL.txt\")\ndf_eval_target = pd.read_csv(\"/tmp/lifetime-value/kdd_cup_98/valtargt.txt\")\ndf_eval = df_eval.merge(df_eval_target, on=\"CONTROLN\")\n</pre> df_train = pd.read_csv(\"/tmp/lifetime-value/kdd_cup_98/cup98LRN.txt\") num_train = df_train.shape[0] df_eval = pd.read_csv(\"/tmp/lifetime-value/kdd_cup_98/cup98VAL.txt\") df_eval_target = pd.read_csv(\"/tmp/lifetime-value/kdd_cup_98/valtargt.txt\") df_eval = df_eval.merge(df_eval_target, on=\"CONTROLN\") In\u00a0[\u00a0]: Copied! <pre>df = pd.concat([df_train, df_eval], axis=0, sort=True)\n</pre> df = pd.concat([df_train, df_eval], axis=0, sort=True) In\u00a0[\u00a0]: Copied! <pre>y = df[\"TARGET_D\"][:num_train]\n</pre> y = df[\"TARGET_D\"][:num_train] In\u00a0[\u00a0]: Copied! <pre>def plot_hist_log_scale(y):\n    max_val = y.max() + 1.0\n    ax = pd.Series(y).hist(\n        figsize=(8, 5), bins=10 ** np.linspace(0.0, np.log10(max_val), 20)\n    )\n\n    plt.xlabel(\"Donation ($)\")\n    plt.ylabel(\"Count\")\n    # plt.title('Histogram of LTV')\n    plt.xticks(rotation=\"horizontal\")\n    plt.legend(loc=\"upper left\")\n    ax.set_xscale(\"log\")\n    ax.grid(False)\n    # Hide the right and top spines\n    ax.spines[\"right\"].set_visible(False)\n    ax.spines[\"top\"].set_visible(False)\n    # Only show ticks on the left and bottom spines\n    ax.yaxis.set_ticks_position(\"left\")\n    ax.xaxis.set_ticks_position(\"bottom\")\n    plt.show()\n\n    fig = ax.get_figure()\n    output_file = tf.io.gfile.GFile(\n        \"/tmp/lifetime-value/kdd_cup_98/histogram_kdd98_log_scale.pdf\", \"wb\"\n    )\n    fig.savefig(output_file, bbox_inches=\"tight\", format=\"pdf\")\n</pre> def plot_hist_log_scale(y):     max_val = y.max() + 1.0     ax = pd.Series(y).hist(         figsize=(8, 5), bins=10 ** np.linspace(0.0, np.log10(max_val), 20)     )      plt.xlabel(\"Donation ($)\")     plt.ylabel(\"Count\")     # plt.title('Histogram of LTV')     plt.xticks(rotation=\"horizontal\")     plt.legend(loc=\"upper left\")     ax.set_xscale(\"log\")     ax.grid(False)     # Hide the right and top spines     ax.spines[\"right\"].set_visible(False)     ax.spines[\"top\"].set_visible(False)     # Only show ticks on the left and bottom spines     ax.yaxis.set_ticks_position(\"left\")     ax.xaxis.set_ticks_position(\"bottom\")     plt.show()      fig = ax.get_figure()     output_file = tf.io.gfile.GFile(         \"/tmp/lifetime-value/kdd_cup_98/histogram_kdd98_log_scale.pdf\", \"wb\"     )     fig.savefig(output_file, bbox_inches=\"tight\", format=\"pdf\") In\u00a0[\u00a0]: Copied! <pre>plot_hist_log_scale(y[y &gt; 0])\n</pre> plot_hist_log_scale(y[y &gt; 0]) In\u00a0[\u00a0]: Copied! <pre>VOCAB_FEATURES = [\n    \"ODATEDW\",  # date of donor's first gift (YYMM)\n    \"OSOURCE\",  # donor acquisition mailing list\n    \"TCODE\",  # donor title code\n    \"STATE\",\n    \"ZIP\",\n    \"DOMAIN\",  # urbanicity level and socio-economic status of the neighborhood\n    \"CLUSTER\",  # socio-economic status\n    \"GENDER\",\n    \"MAXADATE\",  # date of the most recent promotion received\n    \"MINRDATE\",\n    \"LASTDATE\",\n    \"FISTDATE\",\n    \"RFA_2A\",\n]\n</pre> VOCAB_FEATURES = [     \"ODATEDW\",  # date of donor's first gift (YYMM)     \"OSOURCE\",  # donor acquisition mailing list     \"TCODE\",  # donor title code     \"STATE\",     \"ZIP\",     \"DOMAIN\",  # urbanicity level and socio-economic status of the neighborhood     \"CLUSTER\",  # socio-economic status     \"GENDER\",     \"MAXADATE\",  # date of the most recent promotion received     \"MINRDATE\",     \"LASTDATE\",     \"FISTDATE\",     \"RFA_2A\", ] In\u00a0[\u00a0]: Copied! <pre>df[\"ODATEDW\"] = df[\"ODATEDW\"].astype(\"str\")\ndf[\"TCODE\"] = df[\"TCODE\"].apply(lambda x: \"{:03d}\".format(x // 1000 if x &gt; 1000 else x))\ndf[\"ZIP\"] = df[\"ZIP\"].str.slice(0, 5)\ndf[\"MAXADATE\"] = df[\"MAXADATE\"].astype(\"str\")\ndf[\"MINRDATE\"] = df[\"MINRDATE\"].astype(\"str\")\ndf[\"LASTDATE\"] = df[\"LASTDATE\"].astype(\"str\")\ndf[\"FISTDATE\"] = df[\"FISTDATE\"].astype(\"str\")\n</pre> df[\"ODATEDW\"] = df[\"ODATEDW\"].astype(\"str\") df[\"TCODE\"] = df[\"TCODE\"].apply(lambda x: \"{:03d}\".format(x // 1000 if x &gt; 1000 else x)) df[\"ZIP\"] = df[\"ZIP\"].str.slice(0, 5) df[\"MAXADATE\"] = df[\"MAXADATE\"].astype(\"str\") df[\"MINRDATE\"] = df[\"MINRDATE\"].astype(\"str\") df[\"LASTDATE\"] = df[\"LASTDATE\"].astype(\"str\") df[\"FISTDATE\"] = df[\"FISTDATE\"].astype(\"str\") In\u00a0[\u00a0]: Copied! <pre>def label_encoding(y, frequency_threshold=100):\n    value_counts = pd.value_counts(y)\n    categories = value_counts[value_counts &gt;= frequency_threshold].index.to_numpy()\n    # 0 indicates the unknown category.\n    return pd.Categorical(y, categories=categories).codes + 1\n</pre> def label_encoding(y, frequency_threshold=100):     value_counts = pd.value_counts(y)     categories = value_counts[value_counts &gt;= frequency_threshold].index.to_numpy()     # 0 indicates the unknown category.     return pd.Categorical(y, categories=categories).codes + 1 In\u00a0[\u00a0]: Copied! <pre>for key in VOCAB_FEATURES:\n    df[key] = label_encoding(df[key])\n</pre> for key in VOCAB_FEATURES:     df[key] = label_encoding(df[key]) In\u00a0[\u00a0]: Copied! <pre>MAIL_ORDER_RESPONSES = [\n    \"MBCRAFT\",\n    \"MBGARDEN\",\n    \"MBBOOKS\",\n    \"MBCOLECT\",\n    \"MAGFAML\",\n    \"MAGFEM\",\n    \"MAGMALE\",\n    \"PUBGARDN\",\n    \"PUBCULIN\",\n    \"PUBHLTH\",\n    \"PUBDOITY\",\n    \"PUBNEWFN\",\n    \"PUBPHOTO\",\n    \"PUBOPP\",\n    \"RFA_2F\",\n]\n</pre> MAIL_ORDER_RESPONSES = [     \"MBCRAFT\",     \"MBGARDEN\",     \"MBBOOKS\",     \"MBCOLECT\",     \"MAGFAML\",     \"MAGFEM\",     \"MAGMALE\",     \"PUBGARDN\",     \"PUBCULIN\",     \"PUBHLTH\",     \"PUBDOITY\",     \"PUBNEWFN\",     \"PUBPHOTO\",     \"PUBOPP\",     \"RFA_2F\", ] In\u00a0[\u00a0]: Copied! <pre>INDICATOR_FEATURES = [\n    \"AGE\",  # age decile, 0 indicates unknown\n    \"NUMCHLD\",\n    \"INCOME\",\n    \"WEALTH1\",\n    \"HIT\",\n] + MAIL_ORDER_RESPONSES\n</pre> INDICATOR_FEATURES = [     \"AGE\",  # age decile, 0 indicates unknown     \"NUMCHLD\",     \"INCOME\",     \"WEALTH1\",     \"HIT\", ] + MAIL_ORDER_RESPONSES In\u00a0[\u00a0]: Copied! <pre>df[\"AGE\"] = pd.qcut(df[\"AGE\"].values, 10).codes + 1\ndf[\"NUMCHLD\"] = df[\"NUMCHLD\"].apply(lambda x: 0 if np.isnan(x) else int(x))\ndf[\"INCOME\"] = df[\"INCOME\"].apply(lambda x: 0 if np.isnan(x) else int(x))\ndf[\"WEALTH1\"] = df[\"WEALTH1\"].apply(lambda x: 0 if np.isnan(x) else int(x) + 1)\ndf[\"HIT\"] = pd.qcut(df[\"HIT\"].values, q=50, duplicates=\"drop\").codes\n\nfor col in MAIL_ORDER_RESPONSES:\n    df[col] = pd.qcut(df[col].values, q=20, duplicates=\"drop\").codes + 1\n</pre> df[\"AGE\"] = pd.qcut(df[\"AGE\"].values, 10).codes + 1 df[\"NUMCHLD\"] = df[\"NUMCHLD\"].apply(lambda x: 0 if np.isnan(x) else int(x)) df[\"INCOME\"] = df[\"INCOME\"].apply(lambda x: 0 if np.isnan(x) else int(x)) df[\"WEALTH1\"] = df[\"WEALTH1\"].apply(lambda x: 0 if np.isnan(x) else int(x) + 1) df[\"HIT\"] = pd.qcut(df[\"HIT\"].values, q=50, duplicates=\"drop\").codes  for col in MAIL_ORDER_RESPONSES:     df[col] = pd.qcut(df[col].values, q=20, duplicates=\"drop\").codes + 1 In\u00a0[\u00a0]: Copied! <pre>NUMERIC_FEATURES = [\n    # binary\n    \"MAILCODE\",  # bad address\n    \"NOEXCH\",  # do not exchange\n    \"RECINHSE\",  # donor has given to PVA's in house program\n    \"RECP3\",  # donor has given to PVA's P3 program\n    \"RECPGVG\",  # planned giving record\n    \"RECSWEEP\",  # sweepstakes record\n    \"HOMEOWNR\",  # home owner\n    \"CHILD03\",\n    \"CHILD07\",\n    \"CHILD12\",\n    \"CHILD18\",\n    # continuous\n    \"CARDPROM\",\n    \"NUMPROM\",\n    \"CARDPM12\",\n    \"NUMPRM12\",\n    \"RAMNTALL\",\n    \"NGIFTALL\",\n    \"MINRAMNT\",\n    \"MAXRAMNT\",\n    \"LASTGIFT\",\n    \"AVGGIFT\",\n]\n</pre> NUMERIC_FEATURES = [     # binary     \"MAILCODE\",  # bad address     \"NOEXCH\",  # do not exchange     \"RECINHSE\",  # donor has given to PVA's in house program     \"RECP3\",  # donor has given to PVA's P3 program     \"RECPGVG\",  # planned giving record     \"RECSWEEP\",  # sweepstakes record     \"HOMEOWNR\",  # home owner     \"CHILD03\",     \"CHILD07\",     \"CHILD12\",     \"CHILD18\",     # continuous     \"CARDPROM\",     \"NUMPROM\",     \"CARDPM12\",     \"NUMPRM12\",     \"RAMNTALL\",     \"NGIFTALL\",     \"MINRAMNT\",     \"MAXRAMNT\",     \"LASTGIFT\",     \"AVGGIFT\", ] In\u00a0[\u00a0]: Copied! <pre>df[\"MAILCODE\"] = (df[\"MAILCODE\"] == \"B\").astype(\"float32\")\ndf[\"PVASTATE\"] = df[\"PVASTATE\"].isin([\"P\", \"E\"]).astype(\"float32\")\ndf[\"NOEXCH\"] = df[\"NOEXCH\"].isin([\"X\", \"1\"]).astype(\"float32\")\ndf[\"RECINHSE\"] = (df[\"RECINHSE\"] == \"X\").astype(\"float32\")\ndf[\"RECP3\"] = (df[\"RECP3\"] == \"X\").astype(\"float32\")\ndf[\"RECPGVG\"] = (df[\"RECPGVG\"] == \"X\").astype(\"float32\")\ndf[\"RECSWEEP\"] = (df[\"RECSWEEP\"] == \"X\").astype(\"float32\")\ndf[\"HOMEOWNR\"] = (df[\"HOMEOWNR\"] == \"H\").astype(\"float32\")\ndf[\"CHILD03\"] = df[\"CHILD03\"].isin([\"M\", \"F\", \"B\"]).astype(\"float32\")\ndf[\"CHILD07\"] = df[\"CHILD07\"].isin([\"M\", \"F\", \"B\"]).astype(\"float32\")\ndf[\"CHILD12\"] = df[\"CHILD12\"].isin([\"M\", \"F\", \"B\"]).astype(\"float32\")\ndf[\"CHILD18\"] = df[\"CHILD18\"].isin([\"M\", \"F\", \"B\"]).astype(\"float32\")\n\ndf[\"CARDPROM\"] = df[\"CARDPROM\"] / 100\ndf[\"NUMPROM\"] = df[\"NUMPROM\"] / 100\ndf[\"CARDPM12\"] = df[\"CARDPM12\"] / 100\ndf[\"NUMPRM12\"] = df[\"NUMPRM12\"] / 100\ndf[\"RAMNTALL\"] = np.log1p(df[\"RAMNTALL\"])\ndf[\"NGIFTALL\"] = np.log1p(df[\"NGIFTALL\"])\ndf[\"MINRAMNT\"] = np.log1p(df[\"MINRAMNT\"])\ndf[\"MAXRAMNT\"] = np.log1p(df[\"MAXRAMNT\"])\ndf[\"LASTGIFT\"] = np.log1p(df[\"LASTGIFT\"])\ndf[\"AVGGIFT\"] = np.log1p(df[\"AVGGIFT\"])\n</pre> df[\"MAILCODE\"] = (df[\"MAILCODE\"] == \"B\").astype(\"float32\") df[\"PVASTATE\"] = df[\"PVASTATE\"].isin([\"P\", \"E\"]).astype(\"float32\") df[\"NOEXCH\"] = df[\"NOEXCH\"].isin([\"X\", \"1\"]).astype(\"float32\") df[\"RECINHSE\"] = (df[\"RECINHSE\"] == \"X\").astype(\"float32\") df[\"RECP3\"] = (df[\"RECP3\"] == \"X\").astype(\"float32\") df[\"RECPGVG\"] = (df[\"RECPGVG\"] == \"X\").astype(\"float32\") df[\"RECSWEEP\"] = (df[\"RECSWEEP\"] == \"X\").astype(\"float32\") df[\"HOMEOWNR\"] = (df[\"HOMEOWNR\"] == \"H\").astype(\"float32\") df[\"CHILD03\"] = df[\"CHILD03\"].isin([\"M\", \"F\", \"B\"]).astype(\"float32\") df[\"CHILD07\"] = df[\"CHILD07\"].isin([\"M\", \"F\", \"B\"]).astype(\"float32\") df[\"CHILD12\"] = df[\"CHILD12\"].isin([\"M\", \"F\", \"B\"]).astype(\"float32\") df[\"CHILD18\"] = df[\"CHILD18\"].isin([\"M\", \"F\", \"B\"]).astype(\"float32\")  df[\"CARDPROM\"] = df[\"CARDPROM\"] / 100 df[\"NUMPROM\"] = df[\"NUMPROM\"] / 100 df[\"CARDPM12\"] = df[\"CARDPM12\"] / 100 df[\"NUMPRM12\"] = df[\"NUMPRM12\"] / 100 df[\"RAMNTALL\"] = np.log1p(df[\"RAMNTALL\"]) df[\"NGIFTALL\"] = np.log1p(df[\"NGIFTALL\"]) df[\"MINRAMNT\"] = np.log1p(df[\"MINRAMNT\"]) df[\"MAXRAMNT\"] = np.log1p(df[\"MAXRAMNT\"]) df[\"LASTGIFT\"] = np.log1p(df[\"LASTGIFT\"]) df[\"AVGGIFT\"] = np.log1p(df[\"AVGGIFT\"]) In\u00a0[\u00a0]: Copied! <pre>CATEGORICAL_FEATURES = VOCAB_FEATURES + INDICATOR_FEATURES\nALL_FEATURES = CATEGORICAL_FEATURES + NUMERIC_FEATURES\n</pre> CATEGORICAL_FEATURES = VOCAB_FEATURES + INDICATOR_FEATURES ALL_FEATURES = CATEGORICAL_FEATURES + NUMERIC_FEATURES In\u00a0[\u00a0]: Copied! <pre>def dnn_split(df):\n    df_train = df.iloc[:num_train]\n    df_eval = df.iloc[num_train:]\n\n    def feature_dict(df):\n        features = {k: v.values for k, v in dict(df[CATEGORICAL_FEATURES]).items()}\n        features[\"numeric\"] = df[NUMERIC_FEATURES].astype(\"float32\").values\n        return features\n\n    x_train, y_train = (\n        feature_dict(df_train),\n        df_train[\"TARGET_D\"].astype(\"float32\").values,\n    )\n    x_eval, y_eval = feature_dict(df_eval), df_eval[\"TARGET_D\"].astype(\"float32\").values\n\n    return x_train, x_eval, y_train, y_eval\n</pre> def dnn_split(df):     df_train = df.iloc[:num_train]     df_eval = df.iloc[num_train:]      def feature_dict(df):         features = {k: v.values for k, v in dict(df[CATEGORICAL_FEATURES]).items()}         features[\"numeric\"] = df[NUMERIC_FEATURES].astype(\"float32\").values         return features      x_train, y_train = (         feature_dict(df_train),         df_train[\"TARGET_D\"].astype(\"float32\").values,     )     x_eval, y_eval = feature_dict(df_eval), df_eval[\"TARGET_D\"].astype(\"float32\").values      return x_train, x_eval, y_train, y_eval In\u00a0[\u00a0]: Copied! <pre>def embedding_dim(x):\n    return int(x**0.25) + 1\n\n\ndef embedding_layer(vocab_size):\n    return tf.keras.Sequential(\n        [\n            tf.keras.layers.Embedding(\n                input_dim=vocab_size,\n                output_dim=embedding_dim(vocab_size),\n                input_length=1,\n            ),\n            tf.keras.layers.Flatten(),\n        ]\n    )\n\n\ndef dnn_model(output_units):\n    numeric_input = tf.keras.layers.Input(\n        shape=(len(NUMERIC_FEATURES),), name=\"numeric\"\n    )\n\n    embedding_inputs = [\n        tf.keras.layers.Input(shape=(1,), name=key, dtype=np.int64)\n        for key in CATEGORICAL_FEATURES\n    ]\n\n    embedding_outputs = [\n        embedding_layer(vocab_size=df[key].max() + 1)(input)\n        for key, input in zip(CATEGORICAL_FEATURES, embedding_inputs)\n    ]\n\n    deep_input = tf.keras.layers.concatenate([numeric_input] + embedding_outputs)\n    deep_model = tf.keras.Sequential(\n        [\n            tf.keras.layers.Dense(128, activation=\"relu\"),\n            tf.keras.layers.Dense(128, activation=\"relu\"),\n            tf.keras.layers.Dense(64, activation=\"relu\"),\n            tf.keras.layers.Dense(64, activation=\"relu\"),\n            tf.keras.layers.Dense(units=output_units),\n        ]\n    )\n    return tf.keras.Model(\n        inputs=[numeric_input] + embedding_inputs, outputs=deep_model(deep_input)\n    )\n</pre> def embedding_dim(x):     return int(x**0.25) + 1   def embedding_layer(vocab_size):     return tf.keras.Sequential(         [             tf.keras.layers.Embedding(                 input_dim=vocab_size,                 output_dim=embedding_dim(vocab_size),                 input_length=1,             ),             tf.keras.layers.Flatten(),         ]     )   def dnn_model(output_units):     numeric_input = tf.keras.layers.Input(         shape=(len(NUMERIC_FEATURES),), name=\"numeric\"     )      embedding_inputs = [         tf.keras.layers.Input(shape=(1,), name=key, dtype=np.int64)         for key in CATEGORICAL_FEATURES     ]      embedding_outputs = [         embedding_layer(vocab_size=df[key].max() + 1)(input)         for key, input in zip(CATEGORICAL_FEATURES, embedding_inputs)     ]      deep_input = tf.keras.layers.concatenate([numeric_input] + embedding_outputs)     deep_model = tf.keras.Sequential(         [             tf.keras.layers.Dense(128, activation=\"relu\"),             tf.keras.layers.Dense(128, activation=\"relu\"),             tf.keras.layers.Dense(64, activation=\"relu\"),             tf.keras.layers.Dense(64, activation=\"relu\"),             tf.keras.layers.Dense(units=output_units),         ]     )     return tf.keras.Model(         inputs=[numeric_input] + embedding_inputs, outputs=deep_model(deep_input)     ) In\u00a0[\u00a0]: Copied! <pre>if LOSS == \"mse\":\n    loss = tf.keras.losses.MeanSquaredError()\n    output_units = 1\n\nif LOSS == \"ziln\":\n    loss = ltv.zero_inflated_lognormal_loss\n    output_units = 3\n</pre> if LOSS == \"mse\":     loss = tf.keras.losses.MeanSquaredError()     output_units = 1  if LOSS == \"ziln\":     loss = ltv.zero_inflated_lognormal_loss     output_units = 3 In\u00a0[\u00a0]: Copied! <pre>x_train, x_eval, y_train, y_eval = dnn_split(df)\nmodel = dnn_model(output_units)\n</pre> x_train, x_eval, y_train, y_eval = dnn_split(df) model = dnn_model(output_units) In\u00a0[\u00a0]: Copied! <pre>model.compile(optimizer=tf.keras.optimizers.Nadam(lr=LEARNING_RATE), loss=loss)\n</pre> model.compile(optimizer=tf.keras.optimizers.Nadam(lr=LEARNING_RATE), loss=loss) In\u00a0[\u00a0]: Copied! <pre>callbacks = [\n    tf.keras.callbacks.ReduceLROnPlateau(monitor=\"val_loss\", min_lr=1e-6),\n    tf.keras.callbacks.EarlyStopping(monitor=\"val_loss\", patience=10),\n]\n</pre> callbacks = [     tf.keras.callbacks.ReduceLROnPlateau(monitor=\"val_loss\", min_lr=1e-6),     tf.keras.callbacks.EarlyStopping(monitor=\"val_loss\", patience=10), ] In\u00a0[\u00a0]: Copied! <pre>history = model.fit(\n    x=x_train,\n    y=y_train,\n    batch_size=2048,\n    epochs=200,\n    verbose=2,\n    callbacks=callbacks,\n    validation_data=(x_eval, y_eval),\n).history\n</pre> history = model.fit(     x=x_train,     y=y_train,     batch_size=2048,     epochs=200,     verbose=2,     callbacks=callbacks,     validation_data=(x_eval, y_eval), ).history In\u00a0[\u00a0]: Copied! <pre>pd.DataFrame(history)[[\"loss\", \"val_loss\"]].plot();\n</pre> pd.DataFrame(history)[[\"loss\", \"val_loss\"]].plot(); In\u00a0[\u00a0]: Copied! <pre>if LOSS == \"mse\":\n    y_pred = model.predict(x=x_eval, batch_size=1024).flatten()\n\nif LOSS == \"ziln\":\n    logits = model.predict(x=x_eval, batch_size=1024)\n    y_pred = ltv.zero_inflated_lognormal_pred(logits).numpy().flatten()\n</pre> if LOSS == \"mse\":     y_pred = model.predict(x=x_eval, batch_size=1024).flatten()  if LOSS == \"ziln\":     logits = model.predict(x=x_eval, batch_size=1024)     y_pred = ltv.zero_inflated_lognormal_pred(logits).numpy().flatten() In\u00a0[\u00a0]: Copied! <pre>from pytorch_widedeep.preprocessing import TabPreprocessor\nfrom pytorch_widedeep.training import Trainer\nfrom pytorch_widedeep.models import TabMlp, WideDeep\nfrom torch.optim.lr_scheduler import ReduceLROnPlateau\nfrom pytorch_widedeep.callbacks import EarlyStopping\nfrom torch.optim import NAdam\n</pre> from pytorch_widedeep.preprocessing import TabPreprocessor from pytorch_widedeep.training import Trainer from pytorch_widedeep.models import TabMlp, WideDeep from torch.optim.lr_scheduler import ReduceLROnPlateau from pytorch_widedeep.callbacks import EarlyStopping from torch.optim import NAdam In\u00a0[\u00a0]: Copied! <pre># CATEGORICAL_FEATURES\nNUMERICAL_FEATURES = [\"num\" + str(i) for i in range(21)]\nx_train_pyt_num = pd.DataFrame(columns=NUMERICAL_FEATURES, data=x_train[\"numeric\"])\nx_train_pyt_cat = pd.DataFrame(\n    {key: value for key, value in x_train.items() if key not in [\"numeric\"]}\n)\n\nx_eval_pyt_num = pd.DataFrame(columns=NUMERICAL_FEATURES, data=x_eval[\"numeric\"])\nx_eval_pyt_cat = pd.DataFrame(\n    {key: value for key, value in x_eval.items() if key not in [\"numeric\"]}\n)\n</pre> # CATEGORICAL_FEATURES NUMERICAL_FEATURES = [\"num\" + str(i) for i in range(21)] x_train_pyt_num = pd.DataFrame(columns=NUMERICAL_FEATURES, data=x_train[\"numeric\"]) x_train_pyt_cat = pd.DataFrame(     {key: value for key, value in x_train.items() if key not in [\"numeric\"]} )  x_eval_pyt_num = pd.DataFrame(columns=NUMERICAL_FEATURES, data=x_eval[\"numeric\"]) x_eval_pyt_cat = pd.DataFrame(     {key: value for key, value in x_eval.items() if key not in [\"numeric\"]} ) In\u00a0[\u00a0]: Copied! <pre>x_train_pyt = pd.concat([x_train_pyt_num, x_train_pyt_cat], axis=1)\nx_eval_pyt = pd.concat([x_eval_pyt_num, x_eval_pyt_cat], axis=1)\n</pre> x_train_pyt = pd.concat([x_train_pyt_num, x_train_pyt_cat], axis=1) x_eval_pyt = pd.concat([x_eval_pyt_num, x_eval_pyt_cat], axis=1) In\u00a0[\u00a0]: Copied! <pre>embed_input = [\n    (u, int(x_train_pyt[u].nunique() ** 0.25) + 1) for u in CATEGORICAL_FEATURES\n]\n</pre> embed_input = [     (u, int(x_train_pyt[u].nunique() ** 0.25) + 1) for u in CATEGORICAL_FEATURES ] In\u00a0[\u00a0]: Copied! <pre># deeptabular\ntab_preprocessor = TabPreprocessor(\n    embed_cols=embed_input,\n    continuous_cols=NUMERICAL_FEATURES,\n    shared_embed=False,\n    scale=False,\n)\nX_tab_train = tab_preprocessor.fit_transform(x_train_pyt)\nX_tab_valid = tab_preprocessor.transform(x_eval_pyt)\nX_tab_test = tab_preprocessor.transform(x_eval_pyt)\n\n# target\ny_train = y_train\ny_valid = y_eval\ny_test = y_train\n\nX_train = {\"X_tab\": X_tab_train, \"target\": y_train}\nX_val = {\"X_tab\": X_tab_valid, \"target\": y_valid}\nX_test = {\"X_tab\": X_tab_test}\n\ndeeptabular = TabMlp(\n    mlp_hidden_dims=[128, 128, 64, 64],\n    column_idx=tab_preprocessor.column_idx,\n    embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=tab_preprocessor.continuous_cols,\n)\n\nmodel = WideDeep(deeptabular=deeptabular, pred_dim=3)\n\ndeep_opt = NAdam(model.deeptabular.parameters(), lr=LEARNING_RATE)\ncallbacks = [EarlyStopping()]\ndeep_sch = ReduceLROnPlateau(deep_opt, min_lr=1e-6)\n\nobjective = \"ziln\"\n\ntrainer = Trainer(\n    model,\n    callbacks=callbacks,\n    lr_schedulers={\"deeptabular\": deep_sch},\n    objective=objective,\n    optimizers={\"deeptabular\": deep_opt},\n)\n\ntrainer.fit(\n    X_train=X_train,\n    X_val=X_val,\n    n_epochs=200,\n    batch_size=2048,\n)\n\ny_pred_pytorch = trainer.predict(X_test=X_test)\n</pre> # deeptabular tab_preprocessor = TabPreprocessor(     embed_cols=embed_input,     continuous_cols=NUMERICAL_FEATURES,     shared_embed=False,     scale=False, ) X_tab_train = tab_preprocessor.fit_transform(x_train_pyt) X_tab_valid = tab_preprocessor.transform(x_eval_pyt) X_tab_test = tab_preprocessor.transform(x_eval_pyt)  # target y_train = y_train y_valid = y_eval y_test = y_train  X_train = {\"X_tab\": X_tab_train, \"target\": y_train} X_val = {\"X_tab\": X_tab_valid, \"target\": y_valid} X_test = {\"X_tab\": X_tab_test}  deeptabular = TabMlp(     mlp_hidden_dims=[128, 128, 64, 64],     column_idx=tab_preprocessor.column_idx,     embed_input=tab_preprocessor.cat_embed_input,     continuous_cols=tab_preprocessor.continuous_cols, )  model = WideDeep(deeptabular=deeptabular, pred_dim=3)  deep_opt = NAdam(model.deeptabular.parameters(), lr=LEARNING_RATE) callbacks = [EarlyStopping()] deep_sch = ReduceLROnPlateau(deep_opt, min_lr=1e-6)  objective = \"ziln\"  trainer = Trainer(     model,     callbacks=callbacks,     lr_schedulers={\"deeptabular\": deep_sch},     objective=objective,     optimizers={\"deeptabular\": deep_opt}, )  trainer.fit(     X_train=X_train,     X_val=X_val,     n_epochs=200,     batch_size=2048, )  y_pred_pytorch = trainer.predict(X_test=X_test) In\u00a0[\u00a0]: Copied! <pre>pd.DataFrame(trainer.history)[[\"train_loss\", \"val_loss\"]].plot();\n</pre> pd.DataFrame(trainer.history)[[\"train_loss\", \"val_loss\"]].plot(); In\u00a0[\u00a0]: Copied! <pre>from sklearn.metrics import mean_squared_error\n\nmean_squared_error(y_pred, y_pred_pytorch)\n</pre> from sklearn.metrics import mean_squared_error  mean_squared_error(y_pred, y_pred_pytorch) In\u00a0[\u00a0]: Copied! <pre>unit_costs = [0.4, 0.5, 0.6, 0.68, 0.7, 0.8, 0.9, 1.0]\n</pre> unit_costs = [0.4, 0.5, 0.6, 0.68, 0.7, 0.8, 0.9, 1.0] In\u00a0[\u00a0]: Copied! <pre>num_mailed = [np.sum(y_pred &gt; v) for v in unit_costs]\nnum_mailed\n</pre> num_mailed = [np.sum(y_pred &gt; v) for v in unit_costs] num_mailed In\u00a0[\u00a0]: Copied! <pre>baseline_total_profit = np.sum(y_eval - 0.68)\nbaseline_total_profit\n</pre> baseline_total_profit = np.sum(y_eval - 0.68) baseline_total_profit In\u00a0[\u00a0]: Copied! <pre>total_profits = [np.sum(y_eval[y_pred &gt; v] - v) for v in unit_costs]\ntotal_profits\n</pre> total_profits = [np.sum(y_eval[y_pred &gt; v] - v) for v in unit_costs] total_profits In\u00a0[\u00a0]: Copied! <pre>gain = pd.DataFrame(\n    {\n        \"lorenz\": ltv.cumulative_true(y_eval, y_eval),\n        \"baseline\": ltv.cumulative_true(y_eval, x_eval[\"numeric\"][:, 19]),\n        \"model\": ltv.cumulative_true(y_eval, y_pred),\n    }\n)\n</pre> gain = pd.DataFrame(     {         \"lorenz\": ltv.cumulative_true(y_eval, y_eval),         \"baseline\": ltv.cumulative_true(y_eval, x_eval[\"numeric\"][:, 19]),         \"model\": ltv.cumulative_true(y_eval, y_pred),     } ) In\u00a0[\u00a0]: Copied! <pre>num_customers = np.float32(gain.shape[0])\ngain[\"cumulative_customer\"] = (np.arange(num_customers) + 1.0) / num_customers\n</pre> num_customers = np.float32(gain.shape[0]) gain[\"cumulative_customer\"] = (np.arange(num_customers) + 1.0) / num_customers In\u00a0[\u00a0]: Copied! <pre>ax = gain[\n    [\n        \"cumulative_customer\",\n        \"lorenz\",\n        \"baseline\",\n        \"model\",\n    ]\n].plot(x=\"cumulative_customer\", figsize=(8, 5), legend=True)\n\nax.legend([\"Groundtruth\", \"Baseline\", \"Model\"], loc=\"lower right\")\n\nax.set_xlabel(\"Cumulative Fraction of Customers\")\nax.set_xticks(np.arange(0, 1.1, 0.1))\nax.set_xlim((0, 1.0))\n\nax.set_ylabel(\"Cumulative Fraction of Total Lifetime Value\")\nax.set_yticks(np.arange(0, 1.1, 0.1))\nax.set_ylim((0, 1.05))\nax.set_title(\"Gain Chart\");\n</pre> ax = gain[     [         \"cumulative_customer\",         \"lorenz\",         \"baseline\",         \"model\",     ] ].plot(x=\"cumulative_customer\", figsize=(8, 5), legend=True)  ax.legend([\"Groundtruth\", \"Baseline\", \"Model\"], loc=\"lower right\")  ax.set_xlabel(\"Cumulative Fraction of Customers\") ax.set_xticks(np.arange(0, 1.1, 0.1)) ax.set_xlim((0, 1.0))  ax.set_ylabel(\"Cumulative Fraction of Total Lifetime Value\") ax.set_yticks(np.arange(0, 1.1, 0.1)) ax.set_ylim((0, 1.05)) ax.set_title(\"Gain Chart\"); In\u00a0[\u00a0]: Copied! <pre>gini = ltv.gini_from_gain(gain[[\"lorenz\", \"baseline\", \"model\"]])\ngini\n</pre> gini = ltv.gini_from_gain(gain[[\"lorenz\", \"baseline\", \"model\"]]) gini In\u00a0[\u00a0]: Copied! <pre>df_decile = ltv.decile_stats(y_eval, y_pred)\ndf_decile\n</pre> df_decile = ltv.decile_stats(y_eval, y_pred) df_decile In\u00a0[\u00a0]: Copied! <pre>ax = df_decile[[\"label_mean\", \"pred_mean\"]].plot.bar(rot=0)\n\nax.set_title(\"Decile Chart\")\nax.set_xlabel(\"Prediction bucket\")\nax.set_ylabel(\"Average bucket value\")\nax.legend([\"Label\", \"Prediction\"], loc=\"upper left\");\n</pre> ax = df_decile[[\"label_mean\", \"pred_mean\"]].plot.bar(rot=0)  ax.set_title(\"Decile Chart\") ax.set_xlabel(\"Prediction bucket\") ax.set_ylabel(\"Average bucket value\") ax.legend([\"Label\", \"Prediction\"], loc=\"upper left\"); In\u00a0[\u00a0]: Copied! <pre>def spearmanr(x1: Sequence[float], x2: Sequence[float]) -&gt; float:\n    \"\"\"Calculates spearmanr rank correlation coefficient.\n\n    See https://docs.scipy.org/doc/scipy/reference/stats.html.\n\n    Args:\n      x1: 1D array_like.\n      x2: 1D array_like.\n\n    Returns:\n      correlation: float.\n    \"\"\"\n    return stats.spearmanr(x1, x2, nan_policy=\"raise\")[0]\n\n\nspearman_corr = spearmanr(y_eval, y_pred)\nspearman_corr\n</pre> def spearmanr(x1: Sequence[float], x2: Sequence[float]) -&gt; float:     \"\"\"Calculates spearmanr rank correlation coefficient.      See https://docs.scipy.org/doc/scipy/reference/stats.html.      Args:       x1: 1D array_like.       x2: 1D array_like.      Returns:       correlation: float.     \"\"\"     return stats.spearmanr(x1, x2, nan_policy=\"raise\")[0]   spearman_corr = spearmanr(y_eval, y_pred) spearman_corr In\u00a0[\u00a0]: Copied! <pre>df_metrics = pd.DataFrame(\n    {\n        \"model\": MODEL,\n        \"loss_function\": LOSS,\n        \"train_loss\": history[\"loss\"][-1],\n        \"eval_loss\": history[\"val_loss\"][-1],\n        \"label_positive\": np.mean(y_eval &gt; 0),\n        \"label_mean\": y_eval.mean(),\n        \"pred_mean\": y_pred.mean(),\n        \"decile_mape\": df_decile[\"decile_mape\"].mean(),\n        \"baseline_gini\": gini[\"normalized\"][1],\n        \"gini\": gini[\"normalized\"][2],\n        \"spearman_corr\": spearman_corr,\n    },\n    index=[VERSION],\n)\n</pre> df_metrics = pd.DataFrame(     {         \"model\": MODEL,         \"loss_function\": LOSS,         \"train_loss\": history[\"loss\"][-1],         \"eval_loss\": history[\"val_loss\"][-1],         \"label_positive\": np.mean(y_eval &gt; 0),         \"label_mean\": y_eval.mean(),         \"pred_mean\": y_pred.mean(),         \"decile_mape\": df_decile[\"decile_mape\"].mean(),         \"baseline_gini\": gini[\"normalized\"][1],         \"gini\": gini[\"normalized\"][2],         \"spearman_corr\": spearman_corr,     },     index=[VERSION], ) In\u00a0[\u00a0]: Copied! <pre>for unit_cost, total_profit in zip(unit_costs, total_profits):\n    df_metrics[\"total_profit_{:02d}\".format(int(unit_cost * 100))] = total_profit\n</pre> for unit_cost, total_profit in zip(unit_costs, total_profits):     df_metrics[\"total_profit_{:02d}\".format(int(unit_cost * 100))] = total_profit In\u00a0[\u00a0]: Copied! <pre>df_metrics.T\n</pre> df_metrics.T In\u00a0[\u00a0]: Copied! <pre>output_path = OUTPUT_CSV_FOLDER\n</pre> output_path = OUTPUT_CSV_FOLDER In\u00a0[\u00a0]: Copied! <pre>if not os.path.isdir(output_path):\n    os.makedirs(output_path)\n</pre> if not os.path.isdir(output_path):     os.makedirs(output_path) In\u00a0[\u00a0]: Copied! <pre>output_file = os.path.join(\n    output_path, \"{}_regression_{}_{}.csv\".format(MODEL, LOSS, VERSION)\n)\n</pre> output_file = os.path.join(     output_path, \"{}_regression_{}_{}.csv\".format(MODEL, LOSS, VERSION) ) In\u00a0[\u00a0]: Copied! <pre>df_metrics.to_csv(output_file, index=False)\n</pre> df_metrics.to_csv(output_file, index=False)"},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#zilnloss","title":"ZILNLoss\u00b6","text":"<p>[DISCLAIMER]</p> <p>Purpose of this notebook is to check if ZILNloss implemented originaly Keras give same results in pytorch-widedeep implemenatation</p>"},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#kdd-cup-98-ltv-prediction","title":"KDD Cup 98 LTV Prediction\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#configs","title":"Configs\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#load-data","title":"Load data\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#label-distribution","title":"Label distribution\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#preprocess-features","title":"Preprocess features\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#vocab","title":"Vocab\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#indicator","title":"Indicator\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#numeric","title":"Numeric\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#all","title":"All\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#traineval-split","title":"Train/eval split\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#model","title":"Model\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#loss","title":"Loss\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#train","title":"Train\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#eval","title":"Eval\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#pytorch-widedeep-approach","title":"Pytorch-widedeep approach\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#appendix","title":"Appendix\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#total-profit","title":"Total Profit\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#gini-coefficient","title":"Gini Coefficient\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#calibration","title":"Calibration\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#rank-correlation","title":"Rank Correlation\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#all-metrics-together","title":"All metrics together\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#save","title":"Save\u00b6","text":""},{"location":"examples/13_model_uncertainty_prediction.html","title":"13_model_uncertainty_prediction","text":"<ul> <li>In this notebook we will use the higly imbalanced Protein Homology Dataset from KDD cup 2004</li> </ul> <pre><code>* The first element of each line is a BLOCK ID that denotes to which native sequence this example belongs. There is a unique BLOCK ID for each native sequence. BLOCK IDs are integers running from 1 to 303 (one for each native sequence, i.e. for each query). BLOCK IDs were assigned before the blocks were split into the train and test sets, so they do not run consecutively in either file.\n* The second element of each line is an EXAMPLE ID that uniquely describes the example. You will need this EXAMPLE ID and the BLOCK ID when you submit results.\n* The third element is the class of the example. Proteins that are homologous to the native sequence are denoted by 1, non-homologous proteins (i.e. decoys) by 0. Test examples have a \"?\" in this position.\n* All following elements are feature values. There are 74 feature values in each line. The features describe the match (e.g. the score of a sequence alignment) between the native protein sequence and the sequence that is tested for homology.\n</code></pre> In\u00a0[1]: Copied! <pre>import numpy as np\nimport pandas as pd\nimport torch\nfrom torch.optim import SGD, lr_scheduler\n\nfrom pytorch_widedeep import Trainer\nfrom pytorch_widedeep.preprocessing import TabPreprocessor\nfrom pytorch_widedeep.models import TabMlp, WideDeep\nfrom pytorch_widedeep.dataloaders import DataLoaderImbalanced, DataLoaderDefault\nfrom torchmetrics import F1Score as F1_torchmetrics\nfrom torchmetrics import Accuracy as Accuracy_torchmetrics\nfrom torchmetrics import Precision as Precision_torchmetrics\nfrom torchmetrics import Recall as Recall_torchmetrics\nfrom pytorch_widedeep.metrics import Accuracy, Recall, Precision, F1Score, R2Score\nfrom pytorch_widedeep.initializers import XavierNormal\nfrom pytorch_widedeep.datasets import load_bio_kdd04\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import classification_report\n\nimport time\nimport datetime\n\nimport warnings\n\nwarnings.filterwarnings(\"ignore\", category=DeprecationWarning)\n\n# increase displayed columns in jupyter notebook\npd.set_option(\"display.max_columns\", 200)\npd.set_option(\"display.max_rows\", 300)\n</pre> import numpy as np import pandas as pd import torch from torch.optim import SGD, lr_scheduler  from pytorch_widedeep import Trainer from pytorch_widedeep.preprocessing import TabPreprocessor from pytorch_widedeep.models import TabMlp, WideDeep from pytorch_widedeep.dataloaders import DataLoaderImbalanced, DataLoaderDefault from torchmetrics import F1Score as F1_torchmetrics from torchmetrics import Accuracy as Accuracy_torchmetrics from torchmetrics import Precision as Precision_torchmetrics from torchmetrics import Recall as Recall_torchmetrics from pytorch_widedeep.metrics import Accuracy, Recall, Precision, F1Score, R2Score from pytorch_widedeep.initializers import XavierNormal from pytorch_widedeep.datasets import load_bio_kdd04  from sklearn.model_selection import train_test_split from sklearn.metrics import classification_report  import time import datetime  import warnings  warnings.filterwarnings(\"ignore\", category=DeprecationWarning)  # increase displayed columns in jupyter notebook pd.set_option(\"display.max_columns\", 200) pd.set_option(\"display.max_rows\", 300) In\u00a0[2]: Copied! <pre>df = load_bio_kdd04(as_frame=True)\ndf.head()\n</pre> df = load_bio_kdd04(as_frame=True) df.head() Out[2]: EXAMPLE_ID BLOCK_ID target 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 0 279 261532 0 52.0 32.69 0.30 2.5 20.0 1256.8 -0.89 0.33 11.0 -55.0 267.2 0.52 0.05 -2.36 49.6 252.0 0.43 1.16 -2.06 -33.0 -123.2 1.60 -0.49 -6.06 65.0 296.1 -0.28 -0.26 -3.83 -22.6 -170.0 3.06 -1.05 -3.29 22.9 286.3 0.12 2.58 4.08 -33.0 -178.9 1.88 0.53 -7.0 -44.0 1987.0 -5.41 0.95 -4.0 -57.0 722.9 -3.26 -0.55 -7.5 125.5 1547.2 -0.36 1.12 9.0 -37.0 72.5 0.47 0.74 -11.0 -8.0 1595.1 -1.64 2.83 -2.0 -50.0 445.2 -0.35 0.26 0.76 1 279 261533 0 58.0 33.33 0.00 16.5 9.5 608.1 0.50 0.07 20.5 -52.5 521.6 -1.08 0.58 -0.02 -3.2 103.6 -0.95 0.23 -2.87 -25.9 -52.2 -0.21 0.87 -1.81 10.4 62.0 -0.28 -0.04 1.48 -17.6 -198.3 3.43 2.84 5.87 -16.9 72.6 -0.31 2.79 2.71 -33.5 -11.6 -1.11 4.01 5.0 -57.0 666.3 1.13 4.38 5.0 -64.0 39.3 1.07 -0.16 32.5 100.0 1893.7 -2.80 -0.22 2.5 -28.5 45.0 0.58 0.41 -19.0 -6.0 762.9 0.29 0.82 -3.0 -35.0 140.3 1.16 0.39 0.73 2 279 261534 0 77.0 27.27 -0.91 6.0 58.5 1623.6 -1.40 0.02 -6.5 -48.0 621.0 -1.20 0.14 -0.20 73.6 609.1 -0.44 -0.58 -0.04 -23.0 -27.4 -0.72 -1.04 -1.09 91.1 635.6 -0.88 0.24 0.59 -18.7 -7.2 -0.60 -2.82 -0.71 52.4 504.1 0.89 -0.67 -9.30 -20.8 -25.7 -0.77 -0.85 0.0 -20.0 2259.0 -0.94 1.15 -4.0 -44.0 -22.7 0.94 -0.98 -19.0 105.0 1267.9 1.03 1.27 11.0 -39.5 82.3 0.47 -0.19 -10.0 7.0 1491.8 0.32 -1.29 0.0 -34.0 658.2 -0.76 0.26 0.24 3 279 261535 0 41.0 27.91 -0.35 3.0 46.0 1921.6 -1.36 -0.47 -32.0 -51.5 560.9 -0.29 -0.10 -1.11 124.3 791.6 0.00 0.39 -1.85 -21.7 -44.9 -0.21 0.02 0.89 133.9 797.8 -0.08 1.06 -0.26 -16.4 -74.1 0.97 -0.80 -0.41 66.9 955.3 -1.90 1.28 -6.65 -28.1 47.5 -1.91 1.42 1.0 -30.0 1846.7 0.76 1.10 -4.0 -52.0 -53.9 1.71 -0.22 -12.0 97.5 1969.8 -1.70 0.16 -1.0 -32.5 255.9 -0.46 1.57 10.0 6.0 2047.7 -0.98 1.53 0.0 -49.0 554.2 -0.83 0.39 0.73 4 279 261536 0 50.0 28.00 -1.32 -9.0 12.0 464.8 0.88 0.19 8.0 -51.5 98.1 1.09 -0.33 -2.16 -3.9 102.7 0.39 -1.22 -3.39 -15.2 -42.2 -1.18 -1.11 -3.55 8.9 141.3 -0.16 -0.43 -4.15 -12.9 -13.4 -1.32 -0.98 -3.69 8.8 136.1 -0.30 4.13 1.89 -13.0 -18.7 -1.37 -0.93 0.0 -1.0 810.1 -2.29 6.72 1.0 -23.0 -29.7 0.58 -1.10 -18.5 33.5 206.8 1.84 -0.13 4.0 -29.0 30.1 0.80 -0.24 5.0 -14.0 479.5 0.68 -0.59 2.0 -36.0 -6.9 2.02 0.14 -0.23 In\u00a0[3]: Copied! <pre># imbalance of the classes\ndf[\"target\"].value_counts()\n</pre> # imbalance of the classes df[\"target\"].value_counts() Out[3]: <pre>target\n0    144455\n1      1296\nName: count, dtype: int64</pre> In\u00a0[4]: Copied! <pre># drop columns we won't need in this example\ndf.drop(columns=[\"EXAMPLE_ID\", \"BLOCK_ID\"], inplace=True)\n</pre> # drop columns we won't need in this example df.drop(columns=[\"EXAMPLE_ID\", \"BLOCK_ID\"], inplace=True) In\u00a0[5]: Copied! <pre>df_train, df_valid = train_test_split(\n    df, test_size=0.2, stratify=df[\"target\"], random_state=1\n)\ndf_valid, df_test = train_test_split(\n    df_valid, test_size=0.5, stratify=df_valid[\"target\"], random_state=1\n)\n</pre> df_train, df_valid = train_test_split(     df, test_size=0.2, stratify=df[\"target\"], random_state=1 ) df_valid, df_test = train_test_split(     df_valid, test_size=0.5, stratify=df_valid[\"target\"], random_state=1 ) In\u00a0[6]: Copied! <pre>continuous_cols = df.drop(columns=[\"target\"]).columns.values.tolist()\n</pre> continuous_cols = df.drop(columns=[\"target\"]).columns.values.tolist() In\u00a0[7]: Copied! <pre># deeptabular\ntab_preprocessor = TabPreprocessor(continuous_cols=continuous_cols, scale=True)\nX_tab_train = tab_preprocessor.fit_transform(df_train)\nX_tab_valid = tab_preprocessor.transform(df_valid)\nX_tab_test = tab_preprocessor.transform(df_test)\n\n# target\ny_train = df_train[\"target\"].values\ny_valid = df_valid[\"target\"].values\ny_test = df_test[\"target\"].values\n</pre> # deeptabular tab_preprocessor = TabPreprocessor(continuous_cols=continuous_cols, scale=True) X_tab_train = tab_preprocessor.fit_transform(df_train) X_tab_valid = tab_preprocessor.transform(df_valid) X_tab_test = tab_preprocessor.transform(df_test)  # target y_train = df_train[\"target\"].values y_valid = df_valid[\"target\"].values y_test = df_test[\"target\"].values In\u00a0[8]: Copied! <pre>input_layer = len(tab_preprocessor.continuous_cols)\noutput_layer = 1\nhidden_layers = np.linspace(\n    input_layer * 2, output_layer, 5, endpoint=False, dtype=int\n).tolist()\n</pre> input_layer = len(tab_preprocessor.continuous_cols) output_layer = 1 hidden_layers = np.linspace(     input_layer * 2, output_layer, 5, endpoint=False, dtype=int ).tolist() In\u00a0[9]: Copied! <pre>deeptabular = TabMlp(\n    mlp_hidden_dims=hidden_layers,\n    column_idx=tab_preprocessor.column_idx,\n    continuous_cols=tab_preprocessor.continuous_cols,\n)\nmodel = WideDeep(deeptabular=deeptabular, pred_dim=1)\nmodel\n</pre> deeptabular = TabMlp(     mlp_hidden_dims=hidden_layers,     column_idx=tab_preprocessor.column_idx,     continuous_cols=tab_preprocessor.continuous_cols, ) model = WideDeep(deeptabular=deeptabular, pred_dim=1) model Out[9]: <pre>WideDeep(\n  (deeptabular): Sequential(\n    (0): TabMlp(\n      (cont_norm): Identity()\n      (encoder): MLP(\n        (mlp): Sequential(\n          (dense_layer_0): Sequential(\n            (0): Linear(in_features=74, out_features=148, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_1): Sequential(\n            (0): Linear(in_features=148, out_features=118, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_2): Sequential(\n            (0): Linear(in_features=118, out_features=89, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_3): Sequential(\n            (0): Linear(in_features=89, out_features=59, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_4): Sequential(\n            (0): Linear(in_features=59, out_features=30, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=30, out_features=1, bias=True)\n  )\n)</pre> In\u00a0[10]: Copied! <pre># # Metrics from torchmetrics\n# accuracy = Accuracy_torchmetrics(average=None, num_classes=1)\n# precision = Precision_torchmetrics(average=\"micro\", num_classes=1)\n# f1 = F1_torchmetrics(average=None, num_classes=1)\n# recall = Recall_torchmetrics(average=None, num_classes=1)\n</pre> # # Metrics from torchmetrics # accuracy = Accuracy_torchmetrics(average=None, num_classes=1) # precision = Precision_torchmetrics(average=\"micro\", num_classes=1) # f1 = F1_torchmetrics(average=None, num_classes=1) # recall = Recall_torchmetrics(average=None, num_classes=1) In\u00a0[11]: Copied! <pre># Metrics from pytorch-widedeep\naccuracy = Accuracy(top_k=2)\nprecision = Precision(average=False)\nrecall = Recall(average=True)\nf1 = F1Score(average=False)\n</pre> # Metrics from pytorch-widedeep accuracy = Accuracy(top_k=2) precision = Precision(average=False) recall = Recall(average=True) f1 = F1Score(average=False) In\u00a0[12]: Copied! <pre># Optimizers\ndeep_opt = SGD(model.deeptabular.parameters(), lr=0.1)\n# LR Scheduler\ndeep_sch = lr_scheduler.StepLR(deep_opt, step_size=3)\n\ntrainer = Trainer(\n    model,\n    objective=\"binary\",\n    lr_schedulers={\"deeptabular\": deep_sch},\n    initializers={\"deeptabular\": XavierNormal},\n    optimizers={\"deeptabular\": deep_opt},\n    metrics=[accuracy, precision, recall, f1],\n    verbose=1,\n)\n</pre> # Optimizers deep_opt = SGD(model.deeptabular.parameters(), lr=0.1) # LR Scheduler deep_sch = lr_scheduler.StepLR(deep_opt, step_size=3)  trainer = Trainer(     model,     objective=\"binary\",     lr_schedulers={\"deeptabular\": deep_sch},     initializers={\"deeptabular\": XavierNormal},     optimizers={\"deeptabular\": deep_opt},     metrics=[accuracy, precision, recall, f1],     verbose=1, ) In\u00a0[13]: Copied! <pre>start = time.time()\ntrainer.fit(\n    X_train={\"X_tab\": X_tab_train, \"target\": y_train},\n    X_val={\"X_tab\": X_tab_valid, \"target\": y_valid},\n    n_epochs=3,\n    batch_size=50,\n    custom_dataloader=DataLoaderImbalanced,\n    oversample_mul=5,\n)\nprint(\n    \"Training time[s]: {}\".format(\n        datetime.timedelta(seconds=round(time.time() - start))\n    )\n)\n</pre> start = time.time() trainer.fit(     X_train={\"X_tab\": X_tab_train, \"target\": y_train},     X_val={\"X_tab\": X_tab_valid, \"target\": y_valid},     n_epochs=3,     batch_size=50,     custom_dataloader=DataLoaderImbalanced,     oversample_mul=5, ) print(     \"Training time[s]: {}\".format(         datetime.timedelta(seconds=round(time.time() - start))     ) ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 208/208 [00:01&lt;00:00, 130.75it/s, loss=0.187, metrics={'acc': 0.9214, 'prec': [0.9149], 'rec': 0.9318, 'f1': [0.9233]}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 292/292 [00:01&lt;00:00, 173.71it/s, loss=0.106, metrics={'acc': 0.9499, 'prec': [0.1435], 'rec': 0.938, 'f1': [0.249]}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 208/208 [00:01&lt;00:00, 139.68it/s, loss=0.109, metrics={'acc': 0.9559, 'prec': [0.9537], 'rec': 0.9572, 'f1': [0.9554]}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 292/292 [00:01&lt;00:00, 177.53it/s, loss=0.0888, metrics={'acc': 0.9602, 'prec': [0.1755], 'rec': 0.9457, 'f1': [0.2961]}]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 208/208 [00:01&lt;00:00, 141.63it/s, loss=0.08, metrics={'acc': 0.9706, 'prec': [0.9648], 'rec': 0.9766, 'f1': [0.9707]}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 292/292 [00:01&lt;00:00, 165.36it/s, loss=0.0969, metrics={'acc': 0.9564, 'prec': [0.1636], 'rec': 0.9535, 'f1': [0.2792]}]</pre> <pre>Training time[s]: 0:00:10\n</pre> <pre>\n</pre> In\u00a0[14]: Copied! <pre>pd.DataFrame(trainer.history)\n</pre> pd.DataFrame(trainer.history) Out[14]: train_loss train_acc train_prec train_rec train_f1 val_loss val_acc val_prec val_rec val_f1 0 0.186707 0.921408 [0.9149412512779236] 0.931801 [0.9232940673828125] 0.106023 0.949914 [0.14353498816490173] 0.937984 [0.24897116422653198] 1 0.109498 0.955931 [0.9536514282226562] 0.957193 [0.9554190039634705] 0.088787 0.960206 [0.17553956806659698] 0.945736 [0.29611650109291077] 2 0.079979 0.970588 [0.9648183584213257] 0.976582 [0.9706646203994751] 0.096858 0.956432 [0.1635638326406479] 0.953488 [0.279228150844574] In\u00a0[15]: Copied! <pre>df_pred = trainer.predict(X_tab=X_tab_test)\nprint(classification_report(df_test[\"target\"].to_list(), df_pred))\nprint(\"Actual predicted values:\\n{}\".format(np.unique(df_pred, return_counts=True)))\n</pre> df_pred = trainer.predict(X_tab=X_tab_test) print(classification_report(df_test[\"target\"].to_list(), df_pred)) print(\"Actual predicted values:\\n{}\".format(np.unique(df_pred, return_counts=True))) <pre>predict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 292/292 [00:00&lt;00:00, 346.55it/s]\n</pre> <pre>              precision    recall  f1-score   support\n\n           0       1.00      0.96      0.98     14446\n           1       0.17      0.95      0.29       130\n\n    accuracy                           0.96     14576\n   macro avg       0.58      0.95      0.63     14576\nweighted avg       0.99      0.96      0.97     14576\n\nActual predicted values:\n(array([0, 1]), array([13845,   731]))\n</pre> In\u00a0[16]: Copied! <pre>df_pred_unc = trainer.predict_uncertainty(X_tab=X_tab_test, uncertainty_granularity=10)\nprint(classification_report(df_test[\"target\"].to_list(), df_pred))\nprint(\n    \"Actual predicted values:\\n{}\".format(\n        np.unique(df_pred_unc[:, -1], return_counts=True)\n    )\n)\n</pre> df_pred_unc = trainer.predict_uncertainty(X_tab=X_tab_test, uncertainty_granularity=10) print(classification_report(df_test[\"target\"].to_list(), df_pred)) print(     \"Actual predicted values:\\n{}\".format(         np.unique(df_pred_unc[:, -1], return_counts=True)     ) ) <pre>predict_UncertaintyIter: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 10/10 [00:03&lt;00:00,  3.25it/s]</pre> <pre>              precision    recall  f1-score   support\n\n           0       1.00      0.96      0.98     14446\n           1       0.17      0.95      0.29       130\n\n    accuracy                           0.96     14576\n   macro avg       0.58      0.95      0.63     14576\nweighted avg       0.99      0.96      0.97     14576\n\nActual predicted values:\n(array([0.]), array([14576]))\n</pre> <pre>\n</pre> In\u00a0[17]: Copied! <pre>df_pred_unc\n</pre> df_pred_unc Out[17]: <pre>array([[9.98401165e-01, 1.59881881e-03, 0.00000000e+00],\n       [9.99941409e-01, 5.85634953e-05, 0.00000000e+00],\n       [9.97351170e-01, 2.64881272e-03, 0.00000000e+00],\n       ...,\n       [9.99494374e-01, 5.05603210e-04, 0.00000000e+00],\n       [9.99981642e-01, 1.83574630e-05, 0.00000000e+00],\n       [9.99996483e-01, 3.52600046e-06, 0.00000000e+00]])</pre>"},{"location":"examples/13_model_uncertainty_prediction.html#model-uncertainty-prediction","title":"Model Uncertainty prediction\u00b6","text":"<p>Note:</p> <p>This notebook extends the \"Custom DataLoader for Imbalanced dataset\" notebook</p>"},{"location":"examples/13_model_uncertainty_prediction.html#initial-imports","title":"Initial imports\u00b6","text":""},{"location":"examples/13_model_uncertainty_prediction.html#preparing-the-data","title":"Preparing the data\u00b6","text":""},{"location":"examples/13_model_uncertainty_prediction.html#define-the-model","title":"Define the model\u00b6","text":""},{"location":"examples/13_model_uncertainty_prediction.html#normal-prediction","title":"\"Normal\" prediction\u00b6","text":""},{"location":"examples/13_model_uncertainty_prediction.html#prediction-using-uncertainty","title":"Prediction using uncertainty\u00b6","text":""},{"location":"examples/14_bayesian_models.html","title":"14_bayesian_models","text":"In\u00a0[1]: Copied! <pre>import numpy as np\nimport torch\nimport pandas as pd\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import accuracy_score\n\nfrom pytorch_widedeep.metrics import Accuracy\nfrom pytorch_widedeep.datasets import load_adult\nfrom pytorch_widedeep.callbacks import EarlyStopping, ModelCheckpoint\nfrom pytorch_widedeep.preprocessing import TabPreprocessor, WidePreprocessor\nfrom pytorch_widedeep.bayesian_models import BayesianWide, BayesianTabMlp\nfrom pytorch_widedeep.training.bayesian_trainer import BayesianTrainer\n</pre> import numpy as np import torch import pandas as pd  from sklearn.model_selection import train_test_split from sklearn.metrics import accuracy_score  from pytorch_widedeep.metrics import Accuracy from pytorch_widedeep.datasets import load_adult from pytorch_widedeep.callbacks import EarlyStopping, ModelCheckpoint from pytorch_widedeep.preprocessing import TabPreprocessor, WidePreprocessor from pytorch_widedeep.bayesian_models import BayesianWide, BayesianTabMlp from pytorch_widedeep.training.bayesian_trainer import BayesianTrainer <p>The first few things to do we know them very well, like with any other model described in any of the other notebooks</p> In\u00a0[2]: Copied! <pre>df = load_adult(as_frame=True)\ndf.columns = [c.replace(\"-\", \"_\") for c in df.columns]\ndf[\"age_buckets\"] = pd.cut(\n    df.age, bins=[16, 25, 30, 35, 40, 45, 50, 55, 60, 91], labels=np.arange(9)\n)\ndf[\"income_label\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int)\ndf.drop(\"income\", axis=1, inplace=True)\ndf.head()\n</pre> df = load_adult(as_frame=True) df.columns = [c.replace(\"-\", \"_\") for c in df.columns] df[\"age_buckets\"] = pd.cut(     df.age, bins=[16, 25, 30, 35, 40, 45, 50, 55, 60, 91], labels=np.arange(9) ) df[\"income_label\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int) df.drop(\"income\", axis=1, inplace=True) df.head() Out[2]: age workclass fnlwgt education educational_num marital_status occupation relationship race gender capital_gain capital_loss hours_per_week native_country age_buckets income_label 0 25 Private 226802 11th 7 Never-married Machine-op-inspct Own-child Black Male 0 0 40 United-States 0 0 1 38 Private 89814 HS-grad 9 Married-civ-spouse Farming-fishing Husband White Male 0 0 50 United-States 3 0 2 28 Local-gov 336951 Assoc-acdm 12 Married-civ-spouse Protective-serv Husband White Male 0 0 40 United-States 1 1 3 44 Private 160323 Some-college 10 Married-civ-spouse Machine-op-inspct Husband Black Male 7688 0 40 United-States 4 1 4 18 ? 103497 Some-college 10 Never-married ? Own-child White Female 0 0 30 United-States 0 0 In\u00a0[3]: Copied! <pre>train, test = train_test_split(df, test_size=0.2, stratify=df.income_label)\n</pre> train, test = train_test_split(df, test_size=0.2, stratify=df.income_label) In\u00a0[4]: Copied! <pre>wide_cols = [\n    \"age_buckets\",\n    \"education\",\n    \"relationship\",\n    \"workclass\",\n    \"occupation\",\n    \"native_country\",\n    \"gender\",\n]\ncrossed_cols = [(\"education\", \"occupation\"), (\"native_country\", \"occupation\")]\n\ncat_embed_cols = [\n    \"workclass\",\n    \"education\",\n    \"marital_status\",\n    \"occupation\",\n    \"relationship\",\n    \"race\",\n    \"gender\",\n    \"capital_gain\",\n    \"capital_loss\",\n    \"native_country\",\n]\ncontinuous_cols = [\"age\", \"hours_per_week\"]\n\ntarget = train[\"income_label\"].values\n</pre> wide_cols = [     \"age_buckets\",     \"education\",     \"relationship\",     \"workclass\",     \"occupation\",     \"native_country\",     \"gender\", ] crossed_cols = [(\"education\", \"occupation\"), (\"native_country\", \"occupation\")]  cat_embed_cols = [     \"workclass\",     \"education\",     \"marital_status\",     \"occupation\",     \"relationship\",     \"race\",     \"gender\",     \"capital_gain\",     \"capital_loss\",     \"native_country\", ] continuous_cols = [\"age\", \"hours_per_week\"]  target = train[\"income_label\"].values In\u00a0[5]: Copied! <pre>wide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols)\nX_tab = wide_preprocessor.fit_transform(train)\n</pre> wide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols) X_tab = wide_preprocessor.fit_transform(train) In\u00a0[6]: Copied! <pre>model = BayesianWide(\n    input_dim=np.unique(X_tab).shape[0],\n    prior_sigma_1=1.0,\n    prior_sigma_2=0.002,\n    prior_pi=0.8,\n    posterior_mu_init=0,\n    posterior_rho_init=-7.0,\n    pred_dim=1,  # here the models are NOT passed to a WideDeep constructor class so the output dim MUST be specified\n)\n</pre> model = BayesianWide(     input_dim=np.unique(X_tab).shape[0],     prior_sigma_1=1.0,     prior_sigma_2=0.002,     prior_pi=0.8,     posterior_mu_init=0,     posterior_rho_init=-7.0,     pred_dim=1,  # here the models are NOT passed to a WideDeep constructor class so the output dim MUST be specified ) In\u00a0[7]: Copied! <pre>trainer = BayesianTrainer(\n    model,\n    objective=\"binary\",\n    optimizer=torch.optim.Adam(model.parameters(), lr=0.01),\n    metrics=[Accuracy],\n)\n</pre> trainer = BayesianTrainer(     model,     objective=\"binary\",     optimizer=torch.optim.Adam(model.parameters(), lr=0.01),     metrics=[Accuracy], ) In\u00a0[8]: Copied! <pre>trainer.fit(\n    X_tab=X_tab,\n    target=target,\n    val_split=0.2,\n    n_epochs=2,\n    batch_size=256,\n)\n</pre> trainer.fit(     X_tab=X_tab,     target=target,     val_split=0.2,     n_epochs=2,     batch_size=256, ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 123/123 [00:00&lt;00:00, 124.32it/s, loss=163, metrics={'acc': 0.7813}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 31/31 [00:00&lt;00:00, 238.67it/s, loss=141, metrics={'acc': 0.8219}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 123/123 [00:00&lt;00:00, 132.81it/s, loss=140, metrics={'acc': 0.8285}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 31/31 [00:00&lt;00:00, 190.16it/s, loss=140, metrics={'acc': 0.8298}]\n</pre> In\u00a0[9]: Copied! <pre>tab_preprocessor = TabPreprocessor(\n    cat_embed_cols=cat_embed_cols, continuous_cols=continuous_cols\n)\nX_tab = tab_preprocessor.fit_transform(train)\n</pre> tab_preprocessor = TabPreprocessor(     cat_embed_cols=cat_embed_cols, continuous_cols=continuous_cols ) X_tab = tab_preprocessor.fit_transform(train) <pre>/Users/javierrodriguezzaurin/Projects/pytorch-widedeep/pytorch_widedeep/preprocessing/tab_preprocessor.py:358: UserWarning: Continuous columns will not be normalised\n  warnings.warn(\"Continuous columns will not be normalised\")\n</pre> In\u00a0[10]: Copied! <pre>model = BayesianTabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=continuous_cols,\n    #     embed_continuous_method = \"standard\",\n    #     cont_embed_activation=\"leaky_relu\",\n    #     cont_embed_dim = 8,\n    mlp_hidden_dims=[128, 64],\n    prior_sigma_1=1.0,\n    prior_sigma_2=0.002,\n    prior_pi=0.8,\n    posterior_mu_init=0,\n    posterior_rho_init=-7.0,\n    pred_dim=1,\n)\n</pre> model = BayesianTabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     continuous_cols=continuous_cols,     #     embed_continuous_method = \"standard\",     #     cont_embed_activation=\"leaky_relu\",     #     cont_embed_dim = 8,     mlp_hidden_dims=[128, 64],     prior_sigma_1=1.0,     prior_sigma_2=0.002,     prior_pi=0.8,     posterior_mu_init=0,     posterior_rho_init=-7.0,     pred_dim=1, ) In\u00a0[11]: Copied! <pre>trainer = BayesianTrainer(\n    model,\n    objective=\"binary\",\n    optimizer=torch.optim.Adam(model.parameters(), lr=0.01),\n    metrics=[Accuracy],\n)\n</pre> trainer = BayesianTrainer(     model,     objective=\"binary\",     optimizer=torch.optim.Adam(model.parameters(), lr=0.01),     metrics=[Accuracy], ) In\u00a0[12]: Copied! <pre>trainer.fit(\n    X_tab=X_tab,\n    target=target,\n    val_split=0.2,\n    n_epochs=2,\n    batch_size=256,\n)\n</pre> trainer.fit(     X_tab=X_tab,     target=target,     val_split=0.2,     n_epochs=2,     batch_size=256, ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 123/123 [00:04&lt;00:00, 28.74it/s, loss=2e+3, metrics={'acc': 0.8007}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 31/31 [00:00&lt;00:00, 136.89it/s, loss=1.75e+3, metrics={'acc': 0.8418}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 123/123 [00:04&lt;00:00, 29.41it/s, loss=1.73e+3, metrics={'acc': 0.8596}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 31/31 [00:00&lt;00:00, 143.87it/s, loss=1.71e+3, metrics={'acc': 0.8569}]\n</pre> <p>These models are powerful beyond the success metrics because they give us a sense of uncertainty as we predict. Let's have a look</p> In\u00a0[13]: Copied! <pre>X_tab_test = tab_preprocessor.transform(test)\n</pre> X_tab_test = tab_preprocessor.transform(test) In\u00a0[14]: Copied! <pre>preds = trainer.predict(X_tab_test, return_samples=True, n_samples=5)\n</pre> preds = trainer.predict(X_tab_test, return_samples=True, n_samples=5) <pre>predict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 39/39 [00:01&lt;00:00, 33.92it/s]\n</pre> In\u00a0[15]: Copied! <pre>preds.shape\n</pre> preds.shape Out[15]: <pre>(5, 9769)</pre> <p>as we can see the prediction have shape <code>(5, 9769)</code>, one set of predictions each time we have internally run predict (i.e. sample the network and predict, defined by the parameter <code>n_samples</code>). This gives us an idea of how certain the model is about a certain prediction.</p> <p>Similarly, we could obtain the probabilities</p> In\u00a0[16]: Copied! <pre>probs = trainer.predict_proba(X_tab_test, return_samples=True, n_samples=5)\n</pre> probs = trainer.predict_proba(X_tab_test, return_samples=True, n_samples=5) <pre>predict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 39/39 [00:01&lt;00:00, 32.79it/s]\n</pre> In\u00a0[17]: Copied! <pre>probs.shape\n</pre> probs.shape Out[17]: <pre>(5, 9769, 2)</pre> <p>And we could see how the model performs each time we sampled the network</p> In\u00a0[18]: Copied! <pre>for p in preds:\n    print(accuracy_score(p, test[\"income_label\"].values))\n</pre> for p in preds:     print(accuracy_score(p, test[\"income_label\"].values)) <pre>0.8559729757395844\n0.8564847988535162\n0.8567918927218753\n0.8562800696079435\n0.8558706111167981\n</pre> In\u00a0[\u00a0]: Copied! <pre>\n</pre>"},{"location":"examples/14_bayesian_models.html#the-bayesian-models","title":"The Bayesian Models\u00b6","text":"<p>Perhaps one of the most interesting functionality in the library is the access to full Bayesian models in almost exactly the same way one would use any of the other models in the library.</p> <p>Note however that the Bayesian models are ONLY available for tabular data and, at the moment, we do not support combining them to form a Wide and Deep model.</p> <p>The implementation in this library is based on the publication: Weight Uncertainty in Neural Networks, by Blundell et al., 2015. Code-wise, our implementation is inspired by a number of source:</p> <ol> <li>https://joshfeldman.net/WeightUncertainty/</li> <li>https://www.nitarshan.com/bayes-by-backprop/</li> <li>https://github.com/piEsposito/blitz-bayesian-deep-learning</li> <li>https://github.com/zackchase/mxnet-the-straight-dope/tree/master/chapter18_variational-methods-and-uncertainty</li> </ol> <p>The two Bayesian models available in the library are:</p> <ol> <li>BayesianWide: this is a linear model where the non-linearities are captured via crossed-columns</li> <li>BayesianMLP: this is a standard MLP that receives categorical embeddings and continuous cols (embedded or not) which are the passed through a series of dense layers. All parameters in the model are probabilistic.</li> </ol>"},{"location":"examples/14_bayesian_models.html#1-bayesianwide","title":"1. <code>BayesianWide</code>\u00b6","text":""},{"location":"examples/14_bayesian_models.html#2-bayesiantabmlp","title":"2. <code>BayesianTabMlp</code>\u00b6","text":""},{"location":"examples/15_Self_Supervised_Pretraning_pt1.html","title":"15_Self-Supervised Pre-Training pt 1","text":"In\u00a0[1]: Copied! <pre>import torch\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.model_selection import train_test_split\n\nfrom pytorch_widedeep import Trainer\nfrom pytorch_widedeep.models import TabMlp, WideDeep\nfrom pytorch_widedeep.metrics import Accuracy\nfrom pytorch_widedeep.datasets import load_adult\nfrom pytorch_widedeep.preprocessing import TabPreprocessor\nfrom pytorch_widedeep.self_supervised_training import EncoderDecoderTrainer\n</pre> import torch from sklearn.metrics import accuracy_score from sklearn.model_selection import train_test_split  from pytorch_widedeep import Trainer from pytorch_widedeep.models import TabMlp, WideDeep from pytorch_widedeep.metrics import Accuracy from pytorch_widedeep.datasets import load_adult from pytorch_widedeep.preprocessing import TabPreprocessor from pytorch_widedeep.self_supervised_training import EncoderDecoderTrainer In\u00a0[2]: Copied! <pre>df = load_adult(as_frame=True)\ndf.columns = [c.replace(\"-\", \"_\") for c in df.columns]\ndf[\"income_label\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int)\ndf.drop(\"income\", axis=1, inplace=True)\n</pre> df = load_adult(as_frame=True) df.columns = [c.replace(\"-\", \"_\") for c in df.columns] df[\"income_label\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int) df.drop(\"income\", axis=1, inplace=True) In\u00a0[3]: Copied! <pre># one could chose to use a validation set for early stopping, hyperparam\n# optimization, etc. This is just an example, so we simply use train/test\n# split\ndf_tr, df_te = train_test_split(df, test_size=0.2, stratify=df.income_label)\n</pre> # one could chose to use a validation set for early stopping, hyperparam # optimization, etc. This is just an example, so we simply use train/test # split df_tr, df_te = train_test_split(df, test_size=0.2, stratify=df.income_label) In\u00a0[4]: Copied! <pre>df_tr.head(2)\n</pre> df_tr.head(2) Out[4]: age workclass fnlwgt education educational_num marital_status occupation relationship race gender capital_gain capital_loss hours_per_week native_country income_label 9042 26 Local-gov 250551 HS-grad 9 Married-civ-spouse Craft-repair Own-child Black Male 0 0 40 United-States 0 25322 50 Private 34832 Bachelors 13 Married-civ-spouse Tech-support Husband White Male 15024 0 40 United-States 1 In\u00a0[5]: Copied! <pre># As always, we need to define which cols will be represented as embeddings\n# and which one will be continuous features\ncat_embed_cols = [\n    \"workclass\",\n    \"education\",\n    \"marital_status\",\n    \"occupation\",\n    \"relationship\",\n    \"race\",\n    \"gender\",\n    \"capital_gain\",\n    \"capital_loss\",\n    \"native_country\",\n]\ncontinuous_cols = [\"age\", \"hours_per_week\"]\ntarget_col = \"income_label\"\n</pre> # As always, we need to define which cols will be represented as embeddings # and which one will be continuous features cat_embed_cols = [     \"workclass\",     \"education\",     \"marital_status\",     \"occupation\",     \"relationship\",     \"race\",     \"gender\",     \"capital_gain\",     \"capital_loss\",     \"native_country\", ] continuous_cols = [\"age\", \"hours_per_week\"] target_col = \"income_label\" In\u00a0[6]: Copied! <pre># We prepare the data to be passed to the model\ntab_preprocessor = TabPreprocessor(\n    cat_embed_cols=cat_embed_cols, continuous_cols=continuous_cols\n)\nX_tab = tab_preprocessor.fit_transform(df_tr)\ntarget = df_tr[target_col].values\n</pre> # We prepare the data to be passed to the model tab_preprocessor = TabPreprocessor(     cat_embed_cols=cat_embed_cols, continuous_cols=continuous_cols ) X_tab = tab_preprocessor.fit_transform(df_tr) target = df_tr[target_col].values <pre>/Users/javierrodriguezzaurin/Projects/pytorch-widedeep/pytorch_widedeep/preprocessing/tab_preprocessor.py:358: UserWarning: Continuous columns will not be normalised\n  warnings.warn(\"Continuous columns will not be normalised\")\n</pre> In\u00a0[7]: Copied! <pre>X_tab[:5]\n</pre> X_tab[:5] Out[7]: <pre>array([[ 1,  1,  1,  1,  1,  1,  1,  1,  1,  1, 26, 40],\n       [ 2,  2,  1,  2,  2,  2,  1,  2,  1,  1, 50, 40],\n       [ 2,  1,  1,  3,  2,  2,  1,  1,  2,  1, 39, 46],\n       [ 2,  3,  2,  4,  1,  2,  2,  1,  1,  1, 17, 10],\n       [ 3,  4,  2,  1,  1,  2,  1,  1,  1,  1, 32, 20]])</pre> In\u00a0[8]: Copied! <pre># We define a model that will act as the encoder in the encoder/decoder\n# architecture. This could be any of: TabMlp, TabResnet or TabNet\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=tab_preprocessor.continuous_cols,\n)\n</pre> # We define a model that will act as the encoder in the encoder/decoder # architecture. This could be any of: TabMlp, TabResnet or TabNet tab_mlp = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     continuous_cols=tab_preprocessor.continuous_cols, ) In\u00a0[9]: Copied! <pre>tab_mlp\n</pre> tab_mlp Out[9]: <pre>TabMlp(\n  (cat_embed): DiffSizeCatEmbeddings(\n    (embed_layers): ModuleDict(\n      (emb_layer_workclass): Embedding(10, 5, padding_idx=0)\n      (emb_layer_education): Embedding(17, 8, padding_idx=0)\n      (emb_layer_marital_status): Embedding(8, 5, padding_idx=0)\n      (emb_layer_occupation): Embedding(16, 7, padding_idx=0)\n      (emb_layer_relationship): Embedding(7, 4, padding_idx=0)\n      (emb_layer_race): Embedding(6, 4, padding_idx=0)\n      (emb_layer_gender): Embedding(3, 2, padding_idx=0)\n      (emb_layer_capital_gain): Embedding(124, 24, padding_idx=0)\n      (emb_layer_capital_loss): Embedding(98, 21, padding_idx=0)\n      (emb_layer_native_country): Embedding(42, 13, padding_idx=0)\n    )\n    (embedding_dropout): Dropout(p=0.0, inplace=False)\n  )\n  (cont_norm): Identity()\n  (encoder): MLP(\n    (mlp): Sequential(\n      (dense_layer_0): Sequential(\n        (0): Linear(in_features=95, out_features=200, bias=True)\n        (1): ReLU(inplace=True)\n        (2): Dropout(p=0.1, inplace=False)\n      )\n      (dense_layer_1): Sequential(\n        (0): Linear(in_features=200, out_features=100, bias=True)\n        (1): ReLU(inplace=True)\n        (2): Dropout(p=0.1, inplace=False)\n      )\n    )\n  )\n)</pre> In\u00a0[10]: Copied! <pre># If we do not pass a custom decoder, which is perfectly possible via the\n# decoder param,  the EncoderDecoderTrainer will automatically build a\n# decoder which will be the 'mirror' image of the encoder\nencoder_decoder_trainer = EncoderDecoderTrainer(encoder=tab_mlp)\n</pre> # If we do not pass a custom decoder, which is perfectly possible via the # decoder param,  the EncoderDecoderTrainer will automatically build a # decoder which will be the 'mirror' image of the encoder encoder_decoder_trainer = EncoderDecoderTrainer(encoder=tab_mlp) In\u00a0[11]: Copied! <pre># let's have a look to the encoder_decoder_model (aka ed_model)\nencoder_decoder_trainer.ed_model\n</pre> # let's have a look to the encoder_decoder_model (aka ed_model) encoder_decoder_trainer.ed_model Out[11]: <pre>EncoderDecoderModel(\n  (encoder): TabMlp(\n    (cat_embed): DiffSizeCatEmbeddings(\n      (embed_layers): ModuleDict(\n        (emb_layer_workclass): Embedding(10, 5, padding_idx=0)\n        (emb_layer_education): Embedding(17, 8, padding_idx=0)\n        (emb_layer_marital_status): Embedding(8, 5, padding_idx=0)\n        (emb_layer_occupation): Embedding(16, 7, padding_idx=0)\n        (emb_layer_relationship): Embedding(7, 4, padding_idx=0)\n        (emb_layer_race): Embedding(6, 4, padding_idx=0)\n        (emb_layer_gender): Embedding(3, 2, padding_idx=0)\n        (emb_layer_capital_gain): Embedding(124, 24, padding_idx=0)\n        (emb_layer_capital_loss): Embedding(98, 21, padding_idx=0)\n        (emb_layer_native_country): Embedding(42, 13, padding_idx=0)\n      )\n      (embedding_dropout): Dropout(p=0.0, inplace=False)\n    )\n    (cont_norm): Identity()\n    (encoder): MLP(\n      (mlp): Sequential(\n        (dense_layer_0): Sequential(\n          (0): Linear(in_features=95, out_features=200, bias=True)\n          (1): ReLU(inplace=True)\n          (2): Dropout(p=0.1, inplace=False)\n        )\n        (dense_layer_1): Sequential(\n          (0): Linear(in_features=200, out_features=100, bias=True)\n          (1): ReLU(inplace=True)\n          (2): Dropout(p=0.1, inplace=False)\n        )\n      )\n    )\n  )\n  (decoder): TabMlpDecoder(\n    (decoder): MLP(\n      (mlp): Sequential(\n        (dense_layer_0): Sequential(\n          (0): Linear(in_features=100, out_features=200, bias=True)\n          (1): ReLU(inplace=True)\n          (2): Dropout(p=0.1, inplace=False)\n        )\n        (dense_layer_1): Sequential(\n          (0): Linear(in_features=200, out_features=95, bias=True)\n          (1): ReLU(inplace=True)\n          (2): Dropout(p=0.1, inplace=False)\n        )\n      )\n    )\n  )\n  (masker): RandomObfuscator()\n)</pre> <p>Ignoring the <code>masker</code>, which just...well...masks, the <code>ed_model</code> consists of:</p> <ol> <li>An encoder model that is a <code>TabMlp</code> model that is in itself comprised by an Embedding layer (or rather a collection of them, referred as <code>cat_and_cont_embed</code>) and an encoder (a simple MLP, referred as <code>encoder</code>)</li> <li>A decoder which is just an \"inverted\" MLP (referred as <code>decoder</code>)</li> </ol> In\u00a0[12]: Copied! <pre># And we just...pretrain\nencoder_decoder_trainer.pretrain(X_tab, n_epochs=5, batch_size=256)\n</pre> # And we just...pretrain encoder_decoder_trainer.pretrain(X_tab, n_epochs=5, batch_size=256) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 82.90it/s, loss=4.07]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 89.87it/s, loss=3.09]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 92.86it/s, loss=2.53]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 91.24it/s, loss=2.09]\nepoch 5: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 91.38it/s, loss=1.78]\n</pre> <p>At this point we have two options, we could either save the model for later use or we could continue to supervised training. The latter is rather simple, after running:</p> <pre>encoder_decoder_trainer.pretrain(X_tab, n_epochs=5, batch_size=256)\n</pre> <p>you just have to</p> <pre>model = WideDeep(deeptabular=tab_mlp)\ntrainer = Trainer(model=model, objective=\"binary\", metrics=[Accuracy])\n\ntrainer.fit(X_tab=X_tab, target=target, n_epochs=5, batch_size=256)\n\n# And, you know...we get a test metric\nX_tab_te = tab_preprocessor.transform(df_te)\ntarget_te = df_te[target_col].values\n\npreds = trainer.predict(X_tab=X_tab_te)\ntest_acc = accuracy_score(target_te, preds)\n</pre> <p>Let's say that in any case, we are 'decent' scientists/people and we want to save the model:</p> In\u00a0[13]: Copied! <pre>encoder_decoder_trainer.save(\n    path=\"pretrained_weights\", model_filename=\"encoder_decoder_model.pt\"\n)\n</pre> encoder_decoder_trainer.save(     path=\"pretrained_weights\", model_filename=\"encoder_decoder_model.pt\" ) <p>some time has passed...</p> In\u00a0[14]: Copied! <pre>encoder_decoder_model = torch.load(\"pretrained_weights/encoder_decoder_model.pt\")\n</pre> encoder_decoder_model = torch.load(\"pretrained_weights/encoder_decoder_model.pt\") <p>Now, AND THIS IS IMPORTANT We have loaded the encoder AND the decoder. To proceed to the supervised training we ONLY need the encoder</p> In\u00a0[15]: Copied! <pre>pretrained_encoder = encoder_decoder_model.encoder\n</pre> pretrained_encoder = encoder_decoder_model.encoder In\u00a0[16]: Copied! <pre>pretrained_encoder\n</pre> pretrained_encoder Out[16]: <pre>TabMlp(\n  (cat_embed): DiffSizeCatEmbeddings(\n    (embed_layers): ModuleDict(\n      (emb_layer_workclass): Embedding(10, 5, padding_idx=0)\n      (emb_layer_education): Embedding(17, 8, padding_idx=0)\n      (emb_layer_marital_status): Embedding(8, 5, padding_idx=0)\n      (emb_layer_occupation): Embedding(16, 7, padding_idx=0)\n      (emb_layer_relationship): Embedding(7, 4, padding_idx=0)\n      (emb_layer_race): Embedding(6, 4, padding_idx=0)\n      (emb_layer_gender): Embedding(3, 2, padding_idx=0)\n      (emb_layer_capital_gain): Embedding(124, 24, padding_idx=0)\n      (emb_layer_capital_loss): Embedding(98, 21, padding_idx=0)\n      (emb_layer_native_country): Embedding(42, 13, padding_idx=0)\n    )\n    (embedding_dropout): Dropout(p=0.0, inplace=False)\n  )\n  (cont_norm): Identity()\n  (encoder): MLP(\n    (mlp): Sequential(\n      (dense_layer_0): Sequential(\n        (0): Linear(in_features=95, out_features=200, bias=True)\n        (1): ReLU(inplace=True)\n        (2): Dropout(p=0.1, inplace=False)\n      )\n      (dense_layer_1): Sequential(\n        (0): Linear(in_features=200, out_features=100, bias=True)\n        (1): ReLU(inplace=True)\n        (2): Dropout(p=0.1, inplace=False)\n      )\n    )\n  )\n)</pre> In\u00a0[17]: Copied! <pre># and as always, ANY supervised model in this library has to go throuth the WideDeep class:\nmodel = WideDeep(deeptabular=pretrained_encoder)\ntrainer = Trainer(model=model, objective=\"binary\", metrics=[Accuracy])\n\ntrainer.fit(X_tab=X_tab, target=target, n_epochs=5, batch_size=256)\n\nX_tab_te = tab_preprocessor.transform(df_te)\ntarget_te = df_te[target_col].values\n\npreds = trainer.predict(X_tab=X_tab_te)\ntest_acc = accuracy_score(target_te, preds)\nprint(test_acc)\n</pre> # and as always, ANY supervised model in this library has to go throuth the WideDeep class: model = WideDeep(deeptabular=pretrained_encoder) trainer = Trainer(model=model, objective=\"binary\", metrics=[Accuracy])  trainer.fit(X_tab=X_tab, target=target, n_epochs=5, batch_size=256)  X_tab_te = tab_preprocessor.transform(df_te) target_te = df_te[target_col].values  preds = trainer.predict(X_tab=X_tab_te) test_acc = accuracy_score(target_te, preds) print(test_acc) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 88.04it/s, loss=0.374, metrics={'acc': 0.8253}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 85.63it/s, loss=0.324, metrics={'acc': 0.8491}]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 87.56it/s, loss=0.301, metrics={'acc': 0.8608}]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:02&lt;00:00, 73.38it/s, loss=0.29, metrics={'acc': 0.8655}]\nepoch 5: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 78.68it/s, loss=0.284, metrics={'acc': 0.8686}]\npredict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 39/39 [00:00&lt;00:00, 173.02it/s]\n</pre> <pre>0.8730678677449074\n</pre> <p>As we mentioned before, we can also use a <code>TabResNet</code> or <code>TabNet</code> model and a custom decoder. Let's have a look:</p> In\u00a0[18]: Copied! <pre>from pytorch_widedeep.models import TabResnet as TabResnetEncoder, TabResnetDecoder\n</pre> from pytorch_widedeep.models import TabResnet as TabResnetEncoder, TabResnetDecoder In\u00a0[19]: Copied! <pre>resnet_encoder = TabResnetEncoder(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=continuous_cols,\n    blocks_dims=[200, 100, 100],\n)\n</pre> resnet_encoder = TabResnetEncoder(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     continuous_cols=continuous_cols,     blocks_dims=[200, 100, 100], ) <p>let's have a look to the model</p> In\u00a0[20]: Copied! <pre>resnet_encoder\n</pre> resnet_encoder Out[20]: <pre>TabResnet(\n  (cat_embed): DiffSizeCatEmbeddings(\n    (embed_layers): ModuleDict(\n      (emb_layer_workclass): Embedding(10, 5, padding_idx=0)\n      (emb_layer_education): Embedding(17, 8, padding_idx=0)\n      (emb_layer_marital_status): Embedding(8, 5, padding_idx=0)\n      (emb_layer_occupation): Embedding(16, 7, padding_idx=0)\n      (emb_layer_relationship): Embedding(7, 4, padding_idx=0)\n      (emb_layer_race): Embedding(6, 4, padding_idx=0)\n      (emb_layer_gender): Embedding(3, 2, padding_idx=0)\n      (emb_layer_capital_gain): Embedding(124, 24, padding_idx=0)\n      (emb_layer_capital_loss): Embedding(98, 21, padding_idx=0)\n      (emb_layer_native_country): Embedding(42, 13, padding_idx=0)\n    )\n    (embedding_dropout): Dropout(p=0.0, inplace=False)\n  )\n  (cont_norm): Identity()\n  (encoder): DenseResnet(\n    (dense_resnet): Sequential(\n      (lin_inp): Linear(in_features=95, out_features=200, bias=False)\n      (bn_inp): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n      (block_0): BasicBlock(\n        (resize): Sequential(\n          (0): Linear(in_features=200, out_features=100, bias=False)\n          (1): BatchNorm1d(100, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        )\n        (lin1): Linear(in_features=200, out_features=100, bias=False)\n        (bn1): BatchNorm1d(100, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (leaky_relu): LeakyReLU(negative_slope=0.01, inplace=True)\n        (dp): Dropout(p=0.1, inplace=False)\n        (lin2): Linear(in_features=100, out_features=100, bias=False)\n        (bn2): BatchNorm1d(100, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n      )\n      (block_1): BasicBlock(\n        (lin1): Linear(in_features=100, out_features=100, bias=False)\n        (bn1): BatchNorm1d(100, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (leaky_relu): LeakyReLU(negative_slope=0.01, inplace=True)\n        (dp): Dropout(p=0.1, inplace=False)\n        (lin2): Linear(in_features=100, out_features=100, bias=False)\n        (bn2): BatchNorm1d(100, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n      )\n    )\n  )\n)</pre> <p>As we can see, the tensor we are trying to reconstruct, the embeddings, is of size <code>94</code> (this number is stored in the attribute: <code>esnet_encoder.cat_and_cont_embed.output_dim</code>), with that information we could build or own decoder as:</p> In\u00a0[21]: Copied! <pre># for all possible params see the docs\nresnet_decoder = TabResnetDecoder(\n    embed_dim=resnet_encoder.cat_out_dim + resnet_encoder.cont_out_dim,\n    blocks_dims=[100, 100, 200],\n)\n</pre> # for all possible params see the docs resnet_decoder = TabResnetDecoder(     embed_dim=resnet_encoder.cat_out_dim + resnet_encoder.cont_out_dim,     blocks_dims=[100, 100, 200], ) In\u00a0[22]: Copied! <pre>resnet_decoder\n</pre> resnet_decoder Out[22]: <pre>TabResnetDecoder(\n  (decoder): DenseResnet(\n    (dense_resnet): Sequential(\n      (block_0): BasicBlock(\n        (lin1): Linear(in_features=100, out_features=100, bias=False)\n        (bn1): BatchNorm1d(100, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (leaky_relu): LeakyReLU(negative_slope=0.01, inplace=True)\n        (dp): Dropout(p=0.1, inplace=False)\n        (lin2): Linear(in_features=100, out_features=100, bias=False)\n        (bn2): BatchNorm1d(100, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n      )\n      (block_1): BasicBlock(\n        (resize): Sequential(\n          (0): Linear(in_features=100, out_features=200, bias=False)\n          (1): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        )\n        (lin1): Linear(in_features=100, out_features=200, bias=False)\n        (bn1): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (leaky_relu): LeakyReLU(negative_slope=0.01, inplace=True)\n        (dp): Dropout(p=0.1, inplace=False)\n        (lin2): Linear(in_features=200, out_features=200, bias=False)\n        (bn2): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n      )\n    )\n  )\n  (reconstruction_layer): Linear(in_features=200, out_features=95, bias=False)\n)</pre> <p>and now:</p> In\u00a0[23]: Copied! <pre>ec_trainer = EncoderDecoderTrainer(\n    encoder=resnet_encoder,\n    decoder=resnet_decoder,\n    masked_prob=0.2,\n)\nec_trainer.pretrain(X_tab, n_epochs=5, batch_size=256)\n</pre> ec_trainer = EncoderDecoderTrainer(     encoder=resnet_encoder,     decoder=resnet_decoder,     masked_prob=0.2, ) ec_trainer.pretrain(X_tab, n_epochs=5, batch_size=256) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:03&lt;00:00, 46.89it/s, loss=1.52]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:03&lt;00:00, 46.78it/s, loss=0.81]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:03&lt;00:00, 39.82it/s, loss=0.56]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:03&lt;00:00, 46.73it/s, loss=0.417]\nepoch 5: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:03&lt;00:00, 46.24it/s, loss=0.329]\n</pre> In\u00a0[24]: Copied! <pre># and as always, ANY supervised model in this library has to go throuth the WideDeep class:\nmodel = WideDeep(deeptabular=resnet_encoder)\ntrainer = Trainer(model=model, objective=\"binary\", metrics=[Accuracy])\n\ntrainer.fit(X_tab=X_tab, target=target, n_epochs=5, batch_size=256)\n\nX_tab_te = tab_preprocessor.transform(df_te)\ntarget_te = df_te[target_col].values\n\npreds = trainer.predict(X_tab=X_tab_te)\ntest_acc = accuracy_score(target_te, preds)\nprint(test_acc)\n</pre> # and as always, ANY supervised model in this library has to go throuth the WideDeep class: model = WideDeep(deeptabular=resnet_encoder) trainer = Trainer(model=model, objective=\"binary\", metrics=[Accuracy])  trainer.fit(X_tab=X_tab, target=target, n_epochs=5, batch_size=256)  X_tab_te = tab_preprocessor.transform(df_te) target_te = df_te[target_col].values  preds = trainer.predict(X_tab=X_tab_te) test_acc = accuracy_score(target_te, preds) print(test_acc) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:02&lt;00:00, 58.63it/s, loss=0.335, metrics={'acc': 0.8442}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:02&lt;00:00, 58.02it/s, loss=0.296, metrics={'acc': 0.864}]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:02&lt;00:00, 55.91it/s, loss=0.283, metrics={'acc': 0.8687}]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:02&lt;00:00, 55.00it/s, loss=0.276, metrics={'acc': 0.871}]\nepoch 5: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:02&lt;00:00, 51.95it/s, loss=0.272, metrics={'acc': 0.8732}]\npredict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 39/39 [00:00&lt;00:00, 120.15it/s]\n</pre> <pre>0.8725560446309756\n</pre> In\u00a0[\u00a0]: Copied! <pre>\n</pre>"},{"location":"examples/15_Self_Supervised_Pretraning_pt1.html#self-supervised-pretraining-for-tabular-data","title":"Self Supervised Pretraining for Tabular Data\u00b6","text":"<p>We have implemented two Self Supervised Pre-training routines that allow the user to pre-train all tabular models in the library with the exception of the TabPerceiver (which is a special monster).</p> <p>The two routines implemented are illustrated in the figures below. The 1st is from TabNet: Attentive Interpretable Tabular Learning and is designed for models that do not use transformer-based architectures, while the second is from SAINT: Improved Neural Networks for Tabular Data via Row Attention and Contrastive Pre-Training, and is designed for models that use transformer-based architectures.</p> <p> </p> <p>Fig 1. Figure 2 in their paper. I have included de original caption in case is useful, althought the Figure itself is pretty self explanatory</p> <p> </p> <p>Fig 2. Figure 1 in their paper. Here the caption is necessary \ud83d\ude0f</p> <p>It is beyond the scope of this notebook to explain in detail those implementations. Therefore, we strongly recommend the user to go and read the papers if this functionality is of interest to her/him.</p> <p>One thing is worth noticing however. As seen in Fig 1(the TabNet paper's Fig 2) the masking of the input features happens in the feature space. However, the implementation in this library is inspired by that at the dreamquark-ai repo, which is in itself inspired by the original implementation (by the way, at this point I will write it once again. All TabNet related things in this library are inspired when not directly based in the code in that repo, therefore, ALL CREDIT TO THE GUYS AT dreamquark-ai).</p> <p>In that implementation the masking happens in the embedding space, and currently does not mask the entire embedding (i.e. categorical feature). We decided to release as it is in this version and we will implement the exact same process described in the paper in future releases.</p> <p>Having said all of the above let's see how to use self supervision for tabular data with <code>pytorch-widedeep</code>. We will concentrate in this notebook on the 1st of the two approaches (the 'TabNet approach'). For details on the second approach please see <code>16_Self_Supervised_Pretraning_pt2</code>.</p>"},{"location":"examples/15_Self_Supervised_Pretraning_pt1.html#self-supervision-for-non-transformer-based-models","title":"Self Supervision for non-transformer-based models..\u00b6","text":"<p>...or in general, for models where the embeddigns can have all different dimensions. In this library, these are: <code>TabMlp</code>, <code>TabResNet</code> and <code>TabNet</code></p> <p>As shown in Figure, this is an encoder-encoder approach where we learn to predict values in the incoming data that have been masked. However, as I mentioned before, our implementation is a bit different, and the masking occurs in th embedding space.</p> <p>Nonetheless, the code below illustrates how to use this encoder-decoder approach with <code>pytorch-widedeep</code></p>"},{"location":"examples/15_Self_Supervised_Pretraning_pt2.html","title":"15_Self-Supervised Pre-Training pt 2","text":"In\u00a0[1]: Copied! <pre>import torch\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.model_selection import train_test_split\n\nfrom pytorch_widedeep import Trainer\nfrom pytorch_widedeep.models import WideDeep, FTTransformer\nfrom pytorch_widedeep.metrics import Accuracy\nfrom pytorch_widedeep.datasets import load_adult\nfrom pytorch_widedeep.preprocessing import TabPreprocessor\nfrom pytorch_widedeep.self_supervised_training import (\n    ContrastiveDenoisingTrainer,\n)\n</pre> import torch from sklearn.metrics import accuracy_score from sklearn.model_selection import train_test_split  from pytorch_widedeep import Trainer from pytorch_widedeep.models import WideDeep, FTTransformer from pytorch_widedeep.metrics import Accuracy from pytorch_widedeep.datasets import load_adult from pytorch_widedeep.preprocessing import TabPreprocessor from pytorch_widedeep.self_supervised_training import (     ContrastiveDenoisingTrainer, ) In\u00a0[2]: Copied! <pre>df = load_adult(as_frame=True)\ndf.columns = [c.replace(\"-\", \"_\") for c in df.columns]\ndf[\"income_label\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int)\ndf.drop(\"income\", axis=1, inplace=True)\n\n# one could chose to use a validation set for early stopping, hyperparam\n# optimization, etc. This is just an example, so we simply use train/test\n# split\ndf_tr, df_te = train_test_split(df, test_size=0.2, stratify=df.income_label)\n\ncat_embed_cols = [\n    \"workclass\",\n    \"education\",\n    \"marital_status\",\n    \"occupation\",\n    \"relationship\",\n    \"race\",\n    \"gender\",\n    \"capital_gain\",\n    \"capital_loss\",\n    \"native_country\",\n]\ncontinuous_cols = [\"age\", \"hours_per_week\"]\ntarget_col = \"income_label\"\n\ntab_preprocessor = TabPreprocessor(\n    cat_embed_cols=cat_embed_cols,\n    continuous_cols=continuous_cols,\n    with_attention=True,\n    with_cls_token=True,  # this is optional\n)\nX_tab = tab_preprocessor.fit_transform(df_tr)\ntarget = df_tr[target_col].values\n</pre> df = load_adult(as_frame=True) df.columns = [c.replace(\"-\", \"_\") for c in df.columns] df[\"income_label\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int) df.drop(\"income\", axis=1, inplace=True)  # one could chose to use a validation set for early stopping, hyperparam # optimization, etc. This is just an example, so we simply use train/test # split df_tr, df_te = train_test_split(df, test_size=0.2, stratify=df.income_label)  cat_embed_cols = [     \"workclass\",     \"education\",     \"marital_status\",     \"occupation\",     \"relationship\",     \"race\",     \"gender\",     \"capital_gain\",     \"capital_loss\",     \"native_country\", ] continuous_cols = [\"age\", \"hours_per_week\"] target_col = \"income_label\"  tab_preprocessor = TabPreprocessor(     cat_embed_cols=cat_embed_cols,     continuous_cols=continuous_cols,     with_attention=True,     with_cls_token=True,  # this is optional ) X_tab = tab_preprocessor.fit_transform(df_tr) target = df_tr[target_col].values <pre>/Users/javierrodriguezzaurin/Projects/pytorch-widedeep/pytorch_widedeep/preprocessing/tab_preprocessor.py:358: UserWarning: Continuous columns will not be normalised\n  warnings.warn(\"Continuous columns will not be normalised\")\n</pre> In\u00a0[3]: Copied! <pre>ft_transformer = FTTransformer(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=tab_preprocessor.continuous_cols,\n    embed_continuous_method=\"standard\",\n    input_dim=32,\n    kv_compression_factor=0.5,\n    n_blocks=3,\n    n_heads=4,\n)\n</pre> ft_transformer = FTTransformer(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     continuous_cols=tab_preprocessor.continuous_cols,     embed_continuous_method=\"standard\",     input_dim=32,     kv_compression_factor=0.5,     n_blocks=3,     n_heads=4, ) In\u00a0[4]: Copied! <pre># for a full list of the params for the the ContrastiveDenoisingTrainer (which are many) please see the docs.\n# Note that using these params involves some knowledge of the routine and the architecture of the model used\ncontrastive_denoising_trainer = ContrastiveDenoisingTrainer(\n    model=ft_transformer,\n    preprocessor=tab_preprocessor,\n)\ncontrastive_denoising_trainer.pretrain(X_tab, n_epochs=5, batch_size=256)\n</pre> # for a full list of the params for the the ContrastiveDenoisingTrainer (which are many) please see the docs. # Note that using these params involves some knowledge of the routine and the architecture of the model used contrastive_denoising_trainer = ContrastiveDenoisingTrainer(     model=ft_transformer,     preprocessor=tab_preprocessor, ) contrastive_denoising_trainer.pretrain(X_tab, n_epochs=5, batch_size=256) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:13&lt;00:00, 11.73it/s, loss=579]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:12&lt;00:00, 12.56it/s, loss=143]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:12&lt;00:00, 12.49it/s, loss=141]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:11&lt;00:00, 12.77it/s, loss=138]\nepoch 5: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:11&lt;00:00, 13.29it/s, loss=137]\n</pre> In\u00a0[5]: Copied! <pre>contrastive_denoising_trainer.save(\n    path=\"pretrained_weights\", model_filename=\"contrastive_denoising_model.pt\"\n)\n</pre> contrastive_denoising_trainer.save(     path=\"pretrained_weights\", model_filename=\"contrastive_denoising_model.pt\" ) <p>some time has passed</p> In\u00a0[6]: Copied! <pre># some time has passed, we load the model with torch as usual:\ncontrastive_denoising_model = torch.load(\n    \"pretrained_weights/contrastive_denoising_model.pt\"\n)\n</pre> # some time has passed, we load the model with torch as usual: contrastive_denoising_model = torch.load(     \"pretrained_weights/contrastive_denoising_model.pt\" ) <p>NOW, AND THIS IS IMPORTANT! We have loaded the entire contrastive, denoising model. To proceed to the supervised training we ONLY need the attention-based model, which is the 'model' attribute of the trainer, let's have a look</p> In\u00a0[7]: Copied! <pre>contrastive_denoising_model.model\n</pre> contrastive_denoising_model.model Out[7]: <pre>FTTransformer(\n  (cat_embed): SameSizeCatEmbeddings(\n    (embed): Embedding(323, 32, padding_idx=0)\n    (dropout): Dropout(p=0.0, inplace=False)\n  )\n  (cont_norm): Identity()\n  (cont_embed): ContEmbeddings(\n    INFO: [ContLinear = weight(n_cont_cols, embed_dim) + bias(n_cont_cols, embed_dim)]\n    (linear): ContLinear(n_cont_cols=2, embed_dim=32, embed_dropout=0.0)\n    (dropout): Dropout(p=0.0, inplace=False)\n  )\n  (encoder): Sequential(\n    (fttransformer_block0): FTTransformerEncoder(\n      (attn): LinearAttentionLinformer(\n        (dropout): Dropout(p=0.2, inplace=False)\n        (qkv_proj): Linear(in_features=32, out_features=96, bias=False)\n        (out_proj): Linear(in_features=32, out_features=32, bias=False)\n      )\n      (ff): FeedForward(\n        (w_1): Linear(in_features=32, out_features=84, bias=True)\n        (w_2): Linear(in_features=42, out_features=32, bias=True)\n        (dropout): Dropout(p=0.1, inplace=False)\n        (activation): REGLU()\n      )\n      (attn_normadd): NormAdd(\n        (dropout): Dropout(p=0.2, inplace=False)\n        (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n      )\n      (ff_normadd): NormAdd(\n        (dropout): Dropout(p=0.1, inplace=False)\n        (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n      )\n    )\n    (fttransformer_block1): FTTransformerEncoder(\n      (attn): LinearAttentionLinformer(\n        (dropout): Dropout(p=0.2, inplace=False)\n        (qkv_proj): Linear(in_features=32, out_features=96, bias=False)\n        (out_proj): Linear(in_features=32, out_features=32, bias=False)\n      )\n      (ff): FeedForward(\n        (w_1): Linear(in_features=32, out_features=84, bias=True)\n        (w_2): Linear(in_features=42, out_features=32, bias=True)\n        (dropout): Dropout(p=0.1, inplace=False)\n        (activation): REGLU()\n      )\n      (attn_normadd): NormAdd(\n        (dropout): Dropout(p=0.2, inplace=False)\n        (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n      )\n      (ff_normadd): NormAdd(\n        (dropout): Dropout(p=0.1, inplace=False)\n        (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n      )\n    )\n    (fttransformer_block2): FTTransformerEncoder(\n      (attn): LinearAttentionLinformer(\n        (dropout): Dropout(p=0.2, inplace=False)\n        (qkv_proj): Linear(in_features=32, out_features=96, bias=False)\n        (out_proj): Linear(in_features=32, out_features=32, bias=False)\n      )\n      (ff): FeedForward(\n        (w_1): Linear(in_features=32, out_features=84, bias=True)\n        (w_2): Linear(in_features=42, out_features=32, bias=True)\n        (dropout): Dropout(p=0.1, inplace=False)\n        (activation): REGLU()\n      )\n      (attn_normadd): NormAdd(\n        (dropout): Dropout(p=0.2, inplace=False)\n        (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n      )\n      (ff_normadd): NormAdd(\n        (dropout): Dropout(p=0.1, inplace=False)\n        (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n      )\n    )\n  )\n)</pre> In\u00a0[8]: Copied! <pre>pretrained_model = contrastive_denoising_model.model\n</pre> pretrained_model = contrastive_denoising_model.model In\u00a0[9]: Copied! <pre># and as always, ANY supervised model in this library has to go throuth the WideDeep class:\nmodel = WideDeep(deeptabular=pretrained_model)\ntrainer = Trainer(model=model, objective=\"binary\", metrics=[Accuracy])\n\ntrainer.fit(X_tab=X_tab, target=target, n_epochs=5, batch_size=256)\n\n# And, you know...we get a test metric\nX_tab_te = tab_preprocessor.transform(df_te)\ntarget_te = df_te[target_col].values\n\npreds = trainer.predict(X_tab=X_tab_te)\ntest_acc = accuracy_score(target_te, preds)\nprint(test_acc)\n</pre> # and as always, ANY supervised model in this library has to go throuth the WideDeep class: model = WideDeep(deeptabular=pretrained_model) trainer = Trainer(model=model, objective=\"binary\", metrics=[Accuracy])  trainer.fit(X_tab=X_tab, target=target, n_epochs=5, batch_size=256)  # And, you know...we get a test metric X_tab_te = tab_preprocessor.transform(df_te) target_te = df_te[target_col].values  preds = trainer.predict(X_tab=X_tab_te) test_acc = accuracy_score(target_te, preds) print(test_acc) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:05&lt;00:00, 27.19it/s, loss=0.383, metrics={'acc': 0.8176}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:05&lt;00:00, 26.08it/s, loss=0.325, metrics={'acc': 0.8502}]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:05&lt;00:00, 26.56it/s, loss=0.306, metrics={'acc': 0.8601}]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:05&lt;00:00, 27.41it/s, loss=0.295, metrics={'acc': 0.8641}]\nepoch 5: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:06&lt;00:00, 24.70it/s, loss=0.289, metrics={'acc': 0.8656}]\npredict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 39/39 [00:00&lt;00:00, 97.26it/s]</pre> <pre>0.8695874705701709\n</pre> <pre>\n</pre> In\u00a0[\u00a0]: Copied! <pre>\n</pre>"},{"location":"examples/15_Self_Supervised_Pretraning_pt2.html#self-supervised-pretraining-for-tabular-data","title":"Self Supervised Pretraining for Tabular Data\u00b6","text":"<p>We have implemented two Self Supervised Pre-training routines that allow the user to pre-train all tabular models in the library with the exception of the TabPerceiver (which is a special monster).</p> <p>The two routines implemented are illustrated in the figures below. The 1st is from TabNet: Attentive Interpretable Tabular Learning and is designed for models that do not use transformer-based architectures, while the second is from SAINT: Improved Neural Networks for Tabular Data via Row Attention and Contrastive Pre-Training, and is designed for models that use transformer-based architectures.</p> <p> </p> <p>Fig 1. Figure 2 in their paper. I have included de original caption in case is useful, althought the Figure itself is pretty self explanatory</p> <p> </p> <p>Fig 2. Figure 1 in their paper. Here the caption is necessary \ud83d\ude0f</p> <p>It is beyond the scope of this notebook to explain in detail those implementations. Therefore, we strongly recommend the user to go and read the papers if this functionality is of interest to her/him.</p> <p>One thing is worth noticing however. As seen in Fig 1(the TabNet paper's Fig 2) the masking of the input features happens in the feature space. However, the implementation in this library is inspired by that at the dreamquark-ai repo, which is in itself inspired by the original implementation (by the way, at this point I will write it once again. All TabNet related things in this library are inspired when not directly based in the code in that repo, therefore, ALL CREDIT TO THE GUYS AT dreamquark-ai).</p> <p>In that implementation the masking happens in the embedding space, and currently does not mask the entire embedding (i.e. categorical feature). We decided to release as it is in this version and we will implement the exact same process described in the paper in future releases.</p> <p>Having said all of the above let's see how to use self supervision for tabular data with <code>pytorch-widedeep</code>. We will concentrate in this notebook on the 2nd of the two approaches (the 'SAINT approach'). For details on the 1st approach (the 'TabNet' approach) please see <code>16_Self_Supervised_Pretraning_pt1</code>.</p>"},{"location":"examples/15_Self_Supervised_Pretraning_pt2.html#self-supervision-transformer-based-models","title":"Self Supervision transformer-based models..\u00b6","text":"<p>...or in general, for models where the embeddigns have all the same dimensions. In this library, these are:</p> <ul> <li>TabTransformer</li> <li>FTTransformer</li> <li>SAINT</li> <li>TabFastFormer</li> </ul> <p>Note that there is one additional Transformer-based model, the <code>TabPerceiver</code>, however this is a \"particular\" model and at the moment we do not support self supervision for it, but it will come.</p> <p>Let see at one example using the <code>FTTransformer</code>.</p>"},{"location":"examples/16_Usign_a_custom_hugging_face_model.html","title":"16_Usign-a-custom-hugging-face-model","text":"In\u00a0[1]: Copied! <pre>import numpy as np\nimport torch\nimport lightgbm as lgb\nfrom lightgbm import Dataset as lgbDataset\nfrom scipy.sparse import hstack, csr_matrix\nfrom sklearn.metrics import (\n    f1_score,\n    recall_score,\n    accuracy_score,\n    precision_score,\n    confusion_matrix,\n)\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.feature_extraction.text import TfidfVectorizer\n\nfrom torch import Tensor, nn\nfrom transformers import DistilBertModel, DistilBertTokenizer\nfrom pytorch_widedeep import Trainer\nfrom pytorch_widedeep.models import TabMlp, BasicRNN, WideDeep\nfrom pytorch_widedeep.metrics import F1Score, Accuracy\nfrom pytorch_widedeep.utils import Tokenizer, LabelEncoder\nfrom pytorch_widedeep.preprocessing import TextPreprocessor, TabPreprocessor\nfrom pytorch_widedeep.datasets import load_womens_ecommerce\nfrom pytorch_widedeep.utils.fastai_transforms import (\n    fix_html,\n    spec_add_spaces,\n    rm_useless_spaces,\n)\n</pre> import numpy as np import torch import lightgbm as lgb from lightgbm import Dataset as lgbDataset from scipy.sparse import hstack, csr_matrix from sklearn.metrics import (     f1_score,     recall_score,     accuracy_score,     precision_score,     confusion_matrix, ) from sklearn.model_selection import train_test_split from sklearn.feature_extraction.text import TfidfVectorizer  from torch import Tensor, nn from transformers import DistilBertModel, DistilBertTokenizer from pytorch_widedeep import Trainer from pytorch_widedeep.models import TabMlp, BasicRNN, WideDeep from pytorch_widedeep.metrics import F1Score, Accuracy from pytorch_widedeep.utils import Tokenizer, LabelEncoder from pytorch_widedeep.preprocessing import TextPreprocessor, TabPreprocessor from pytorch_widedeep.datasets import load_womens_ecommerce from pytorch_widedeep.utils.fastai_transforms import (     fix_html,     spec_add_spaces,     rm_useless_spaces, ) <p>Let's load the data and have a look:</p> In\u00a0[2]: Copied! <pre>df = load_womens_ecommerce(as_frame=True)\n\ndf.columns = [c.replace(\" \", \"_\").lower() for c in df.columns]\n\n# classes from [0,num_class)\ndf[\"rating\"] = (df[\"rating\"] - 1).astype(\"int64\")\n\n# group reviews with 1 and 2 scores into one class\ndf.loc[df.rating == 0, \"rating\"] = 1\n\n# and back again to [0,num_class)\ndf[\"rating\"] = (df[\"rating\"] - 1).astype(\"int64\")\n\n# drop short reviews\ndf = df[~df.review_text.isna()]\ndf[\"review_length\"] = df.review_text.apply(lambda x: len(x.split(\" \")))\ndf = df[df.review_length &gt;= 5]\ndf = df.drop(\"review_length\", axis=1).reset_index(drop=True)\n</pre> df = load_womens_ecommerce(as_frame=True)  df.columns = [c.replace(\" \", \"_\").lower() for c in df.columns]  # classes from [0,num_class) df[\"rating\"] = (df[\"rating\"] - 1).astype(\"int64\")  # group reviews with 1 and 2 scores into one class df.loc[df.rating == 0, \"rating\"] = 1  # and back again to [0,num_class) df[\"rating\"] = (df[\"rating\"] - 1).astype(\"int64\")  # drop short reviews df = df[~df.review_text.isna()] df[\"review_length\"] = df.review_text.apply(lambda x: len(x.split(\" \"))) df = df[df.review_length &gt;= 5] df = df.drop(\"review_length\", axis=1).reset_index(drop=True) In\u00a0[3]: Copied! <pre>df.head()\n</pre> df.head() Out[3]: clothing_id age title review_text rating recommended_ind positive_feedback_count division_name department_name class_name 0 767 33 None Absolutely wonderful - silky and sexy and comf... 2 1 0 Initmates Intimate Intimates 1 1080 34 None Love this dress!  it's sooo pretty.  i happene... 3 1 4 General Dresses Dresses 2 1077 60 Some major design flaws I had such high hopes for this dress and reall... 1 0 0 General Dresses Dresses 3 1049 50 My favorite buy! I love, love, love this jumpsuit. it's fun, fl... 3 1 0 General Petite Bottoms Pants 4 847 47 Flattering shirt This shirt is very flattering to all due to th... 3 1 6 General Tops Blouses <p>So, we will use the <code>review_text</code> column to predict the <code>rating</code>. Later on, we will try to combine it with some other columns (like <code>division_name</code> and <code>age</code>) see if these help.</p> <p>Let's first have a look to the distribution of ratings</p> In\u00a0[4]: Copied! <pre>df.rating.value_counts()\n</pre> df.rating.value_counts() Out[4]: <pre>rating\n3    12515\n2     4904\n1     2820\n0     2369\nName: count, dtype: int64</pre> <p>This shows that we could have perhaps grouped rating scores of 1, 2 and 3 into 1...but anyway, let's just move on with those 4 classes.</p> <p>We are not going to carry any hyperparameter optimization here, so, we will only need a train and a test set (i.e.  no need of a validation set for the example in this notebook)</p> In\u00a0[5]: Copied! <pre>train, test = train_test_split(df, train_size=0.8, random_state=1, stratify=df.rating)\n</pre> train, test = train_test_split(df, train_size=0.8, random_state=1, stratify=df.rating) <p>Let's see what we have to beat. What metrics would we obtain if we always predict the most common rating (3)?</p> In\u00a0[6]: Copied! <pre>most_common_pred = [train.rating.value_counts().index[0]] * len(test)\n\nmost_common_acc = accuracy_score(test.rating, most_common_pred)\nmost_common_f1 = f1_score(test.rating, most_common_pred, average=\"weighted\")\n</pre> most_common_pred = [train.rating.value_counts().index[0]] * len(test)  most_common_acc = accuracy_score(test.rating, most_common_pred) most_common_f1 = f1_score(test.rating, most_common_pred, average=\"weighted\") In\u00a0[7]: Copied! <pre>print(f\"Accuracy: {most_common_acc}. F1 Score: {most_common_f1}\")\n</pre> print(f\"Accuracy: {most_common_acc}. F1 Score: {most_common_f1}\") <pre>Accuracy: 0.553516143299425. F1 Score: 0.3944344218301668\n</pre> <p>ok, these are our \"baseline\" metrics.</p> <p>Let's start by using simply tf-idf + lightGBM</p> In\u00a0[8]: Copied! <pre># ?Tokenizer\n</pre> # ?Tokenizer In\u00a0[9]: Copied! <pre># this Tokenizer is part of our utils module but of course, any valid tokenizer can be used here.\n\n# When using notebooks there seems to be an issue related with multiprocessing (and sometimes tqdm)\n# that can only be solved by using only one CPU\ntok = Tokenizer(n_cpus=1)\ntok_reviews_tr = tok.process_all(train.review_text.tolist())\ntok_reviews_te = tok.process_all(test.review_text.tolist())\n</pre> # this Tokenizer is part of our utils module but of course, any valid tokenizer can be used here.  # When using notebooks there seems to be an issue related with multiprocessing (and sometimes tqdm) # that can only be solved by using only one CPU tok = Tokenizer(n_cpus=1) tok_reviews_tr = tok.process_all(train.review_text.tolist()) tok_reviews_te = tok.process_all(test.review_text.tolist()) In\u00a0[10]: Copied! <pre>vectorizer = TfidfVectorizer(\n    max_features=5000, preprocessor=lambda x: x, tokenizer=lambda x: x, min_df=5\n)\n\nX_text_tr = vectorizer.fit_transform(tok_reviews_tr)\nX_text_te = vectorizer.transform(tok_reviews_te)\n</pre> vectorizer = TfidfVectorizer(     max_features=5000, preprocessor=lambda x: x, tokenizer=lambda x: x, min_df=5 )  X_text_tr = vectorizer.fit_transform(tok_reviews_tr) X_text_te = vectorizer.transform(tok_reviews_te) <pre>/Users/javierrodriguezzaurin/.pyenv/versions/3.10.13/envs/widedeep310/lib/python3.10/site-packages/sklearn/feature_extraction/text.py:525: UserWarning: The parameter 'token_pattern' will not be used since 'tokenizer' is not None'\n  warnings.warn(\n</pre> In\u00a0[11]: Copied! <pre>X_text_tr\n</pre> X_text_tr Out[11]: <pre>&lt;18086x4566 sparse matrix of type '&lt;class 'numpy.float64'&gt;'\n\twith 884074 stored elements in Compressed Sparse Row format&gt;</pre> <p>We now move our matrices to lightGBM <code>Dataset</code> format</p> In\u00a0[12]: Copied! <pre>lgbtrain_text = lgbDataset(\n    X_text_tr,\n    train.rating.values,\n    free_raw_data=False,\n)\n\nlgbtest_text = lgbDataset(\n    X_text_te,\n    test.rating.values,\n    reference=lgbtrain_text,\n    free_raw_data=False,\n)\n</pre> lgbtrain_text = lgbDataset(     X_text_tr,     train.rating.values,     free_raw_data=False, )  lgbtest_text = lgbDataset(     X_text_te,     test.rating.values,     reference=lgbtrain_text,     free_raw_data=False, ) <p>and off we go. By the way, I think as we run the next cell, we should appreciate how fast lightGBM runs. Yes, the input is a sparse matrix, but still, trains on 18086x4566 in a matter of secs</p> In\u00a0[\u00a0]: Copied! <pre>lgb_text_model = lgb.train(\n    {\"objective\": \"multiclass\", \"num_classes\": 4},\n    lgbtrain_text,\n    valid_sets=[lgbtest_text, lgbtrain_text],\n    valid_names=[\"test\", \"train\"],\n)\n</pre> lgb_text_model = lgb.train(     {\"objective\": \"multiclass\", \"num_classes\": 4},     lgbtrain_text,     valid_sets=[lgbtest_text, lgbtrain_text],     valid_names=[\"test\", \"train\"], ) In\u00a0[14]: Copied! <pre>preds_text = lgb_text_model.predict(X_text_te)\npred_text_class = np.argmax(preds_text, 1)\n</pre> preds_text = lgb_text_model.predict(X_text_te) pred_text_class = np.argmax(preds_text, 1) In\u00a0[15]: Copied! <pre>acc_text = accuracy_score(lgbtest_text.label, pred_text_class)\nf1_text = f1_score(lgbtest_text.label, pred_text_class, average=\"weighted\")\ncm_text = confusion_matrix(lgbtest_text.label, pred_text_class)\n</pre> acc_text = accuracy_score(lgbtest_text.label, pred_text_class) f1_text = f1_score(lgbtest_text.label, pred_text_class, average=\"weighted\") cm_text = confusion_matrix(lgbtest_text.label, pred_text_class) In\u00a0[16]: Copied! <pre>print(f\"LightGBM Accuracy: {acc_text}. LightGBM F1 Score: {f1_text}\")\n</pre> print(f\"LightGBM Accuracy: {acc_text}. LightGBM F1 Score: {f1_text}\") <pre>LightGBM Accuracy: 0.6444051304732419. LightGBM F1 Score: 0.617154488246181\n</pre> In\u00a0[17]: Copied! <pre>print(f\"LightGBM Confusion Matrix: \\n {cm_text}\")\n</pre> print(f\"LightGBM Confusion Matrix: \\n {cm_text}\") <pre>LightGBM Confusion Matrix: \n [[ 199  135   61   79]\n [ 123  169  149  123]\n [  30   94  279  578]\n [  16   30  190 2267]]\n</pre> <p>Ok, so, with no hyperparameter optimization lightGBM gets an accuracy of 0.64 and a F1 score of 0.62. This is significantly better than predicting always the most popular.</p> <p>Let's see if in this implementation, some additional features, like <code>age</code> or <code>class_name</code> are of any help</p> In\u00a0[18]: Copied! <pre>tab_cols = [\n    \"age\",\n    \"division_name\",\n    \"department_name\",\n    \"class_name\",\n]\n\nfor tab_df in [train, test]:\n    for c in [\"division_name\", \"department_name\", \"class_name\"]:\n        tab_df[c] = tab_df[c].str.lower()\n        tab_df[c].fillna(\"missing\", inplace=True)\n</pre> tab_cols = [     \"age\",     \"division_name\",     \"department_name\",     \"class_name\", ]  for tab_df in [train, test]:     for c in [\"division_name\", \"department_name\", \"class_name\"]:         tab_df[c] = tab_df[c].str.lower()         tab_df[c].fillna(\"missing\", inplace=True) In\u00a0[19]: Copied! <pre># This is our LabelEncoder. A class that is designed to work with the models in this library but\n# can be used for general purposes\nle = LabelEncoder(columns_to_encode=[\"division_name\", \"department_name\", \"class_name\"])\ntrain_tab_le = le.fit_transform(train)\ntest_tab_le = le.transform(test)\n</pre> # This is our LabelEncoder. A class that is designed to work with the models in this library but # can be used for general purposes le = LabelEncoder(columns_to_encode=[\"division_name\", \"department_name\", \"class_name\"]) train_tab_le = le.fit_transform(train) test_tab_le = le.transform(test) In\u00a0[20]: Copied! <pre>train_tab_le.head()\n</pre> train_tab_le.head() Out[20]: clothing_id age title review_text rating recommended_ind positive_feedback_count division_name department_name class_name 4541 836 35 None Bought this on sale in my reg size- 10. im 5'9... 2 1 2 1 1 1 18573 1022 25 Look like \"mom jeans\" Maybe i just have the wrong body type for thes... 1 0 0 2 2 2 1058 815 39 Ig brought me here Love the way this top layers under my jackets ... 2 1 0 1 1 1 12132 984 47 Runs small especially the arms I love this jacket. it's the prettiest and mos... 3 1 0 1 3 3 20756 1051 42 True red, true beauty. These pants are gorgeous--the fabric has a sat... 3 1 0 2 2 4 <p>let's for example have a look to the encodings for the categorical feature <code>class_name</code></p> In\u00a0[21]: Copied! <pre>le.encoding_dict[\"class_name\"]\n</pre> le.encoding_dict[\"class_name\"] Out[21]: <pre>{'blouses': 1,\n 'jeans': 2,\n 'jackets': 3,\n 'pants': 4,\n 'knits': 5,\n 'dresses': 6,\n 'skirts': 7,\n 'sweaters': 8,\n 'fine gauge': 9,\n 'legwear': 10,\n 'lounge': 11,\n 'shorts': 12,\n 'outerwear': 13,\n 'intimates': 14,\n 'swim': 15,\n 'trend': 16,\n 'sleep': 17,\n 'layering': 18,\n 'missing': 19,\n 'casual bottoms': 20,\n 'chemises': 21}</pre> In\u00a0[22]: Copied! <pre># tabular training and test sets\nX_tab_tr = csr_matrix(train_tab_le[tab_cols].values)\nX_tab_te = csr_matrix(test_tab_le[tab_cols].values)\n\n# text + tabular training and test sets\nX_tab_text_tr = hstack((X_tab_tr, X_text_tr))\nX_tab_text_te = hstack((X_tab_te, X_text_te))\n</pre> # tabular training and test sets X_tab_tr = csr_matrix(train_tab_le[tab_cols].values) X_tab_te = csr_matrix(test_tab_le[tab_cols].values)  # text + tabular training and test sets X_tab_text_tr = hstack((X_tab_tr, X_text_tr)) X_tab_text_te = hstack((X_tab_te, X_text_te)) In\u00a0[23]: Copied! <pre>X_tab_tr\n</pre> X_tab_tr Out[23]: <pre>&lt;18086x4 sparse matrix of type '&lt;class 'numpy.int64'&gt;'\n\twith 72344 stored elements in Compressed Sparse Row format&gt;</pre> In\u00a0[24]: Copied! <pre>X_tab_text_tr\n</pre> X_tab_text_tr Out[24]: <pre>&lt;18086x4570 sparse matrix of type '&lt;class 'numpy.float64'&gt;'\n\twith 956418 stored elements in Compressed Sparse Row format&gt;</pre> In\u00a0[25]: Copied! <pre>lgbtrain_tab_text = lgbDataset(\n    X_tab_text_tr,\n    train.rating.values,\n    categorical_feature=[0, 1, 2, 3],\n    free_raw_data=False,\n)\n\nlgbtest_tab_text = lgbDataset(\n    X_tab_text_te,\n    test.rating.values,\n    reference=lgbtrain_tab_text,\n    free_raw_data=False,\n)\n</pre> lgbtrain_tab_text = lgbDataset(     X_tab_text_tr,     train.rating.values,     categorical_feature=[0, 1, 2, 3],     free_raw_data=False, )  lgbtest_tab_text = lgbDataset(     X_tab_text_te,     test.rating.values,     reference=lgbtrain_tab_text,     free_raw_data=False, ) In\u00a0[26]: Copied! <pre>lgb_tab_text_model = lgb.train(\n    {\"objective\": \"multiclass\", \"num_classes\": 4},\n    lgbtrain_tab_text,\n    valid_sets=[lgbtrain_tab_text, lgbtest_tab_text],\n    valid_names=[\"test\", \"train\"],\n    verbose_eval=False,\n)\n</pre> lgb_tab_text_model = lgb.train(     {\"objective\": \"multiclass\", \"num_classes\": 4},     lgbtrain_tab_text,     valid_sets=[lgbtrain_tab_text, lgbtest_tab_text],     valid_names=[\"test\", \"train\"],     verbose_eval=False, ) <pre>/opt/conda/envs/wd38/lib/python3.8/site-packages/lightgbm/basic.py:2065: UserWarning: Using categorical_feature in Dataset.\n  _log_warning('Using categorical_feature in Dataset.')\n/opt/conda/envs/wd38/lib/python3.8/site-packages/lightgbm/basic.py:2068: UserWarning: categorical_feature in Dataset is overridden.\nNew categorical_feature is [0, 1, 2, 3]\n  _log_warning('categorical_feature in Dataset is overridden.\\n'\n/opt/conda/envs/wd38/lib/python3.8/site-packages/lightgbm/engine.py:239: UserWarning: 'verbose_eval' argument is deprecated and will be removed in a future release of LightGBM. Pass 'log_evaluation()' callback via 'callbacks' argument instead.\n  _log_warning(\"'verbose_eval' argument is deprecated and will be removed in a future release of LightGBM. \"\n</pre> <pre>[LightGBM] [Warning] Auto-choosing col-wise multi-threading, the overhead of testing was 0.138280 seconds.\nYou can set `force_col_wise=true` to remove the overhead.\n[LightGBM] [Info] Total Bins 143432\n[LightGBM] [Info] Number of data points in the train set: 18086, number of used features: 2289\n[LightGBM] [Info] Start training from score -2.255919\n[LightGBM] [Info] Start training from score -2.081545\n[LightGBM] [Info] Start training from score -1.528281\n[LightGBM] [Info] Start training from score -0.591354\n</pre> <pre>/opt/conda/envs/wd38/lib/python3.8/site-packages/lightgbm/basic.py:1780: UserWarning: Overriding the parameters from Reference Dataset.\n  _log_warning('Overriding the parameters from Reference Dataset.')\n/opt/conda/envs/wd38/lib/python3.8/site-packages/lightgbm/basic.py:1513: UserWarning: categorical_column in param dict is overridden.\n  _log_warning(f'{cat_alias} in param dict is overridden.')\n</pre> In\u00a0[27]: Copied! <pre>preds_tab_text = lgb_tab_text_model.predict(X_tab_text_te)\npreds_tab_text_class = np.argmax(preds_tab_text, 1)\n\nacc_tab_text = accuracy_score(lgbtest_tab_text.label, preds_tab_text_class)\nf1_tab_text = f1_score(lgbtest_tab_text.label, preds_tab_text_class, average=\"weighted\")\ncm_tab_text = confusion_matrix(lgbtest_tab_text.label, preds_tab_text_class)\n</pre> preds_tab_text = lgb_tab_text_model.predict(X_tab_text_te) preds_tab_text_class = np.argmax(preds_tab_text, 1)  acc_tab_text = accuracy_score(lgbtest_tab_text.label, preds_tab_text_class) f1_tab_text = f1_score(lgbtest_tab_text.label, preds_tab_text_class, average=\"weighted\") cm_tab_text = confusion_matrix(lgbtest_tab_text.label, preds_tab_text_class) In\u00a0[28]: Copied! <pre>print(\n    f\"LightGBM text + tabular Accuracy: {acc_tab_text}. LightGBM text + tabular F1 Score: {f1_tab_text}\"\n)\n</pre> print(     f\"LightGBM text + tabular Accuracy: {acc_tab_text}. LightGBM text + tabular F1 Score: {f1_tab_text}\" ) <pre>LightGBM text + tabular Accuracy: 0.6382131800088456. LightGBM text + tabular F1 Score: 0.6080251307242649\n</pre> In\u00a0[29]: Copied! <pre>print(f\"LightGBM text + tabular Confusion Matrix:\\n {cm_tab_text}\")\n</pre> print(f\"LightGBM text + tabular Confusion Matrix:\\n {cm_tab_text}\") <pre>LightGBM text + tabular Confusion Matrix:\n [[ 193  123   68   90]\n [ 123  146  157  138]\n [  37   90  272  582]\n [  16   37  175 2275]]\n</pre> <p>So, in this set up, the addition tabular columns do not help performance.</p> In\u00a0[30]: Copied! <pre>text_preprocessor = TextPreprocessor(\n    text_col=\"review_text\", max_vocab=5000, min_freq=5, maxlen=90, n_cpus=1\n)\n\nwd_X_text_tr = text_preprocessor.fit_transform(train)\nwd_X_text_te = text_preprocessor.transform(test)\n</pre> text_preprocessor = TextPreprocessor(     text_col=\"review_text\", max_vocab=5000, min_freq=5, maxlen=90, n_cpus=1 )  wd_X_text_tr = text_preprocessor.fit_transform(train) wd_X_text_te = text_preprocessor.transform(test) <pre>The vocabulary contains 4328 tokens\n</pre> In\u00a0[31]: Copied! <pre>basic_rnn = BasicRNN(\n    vocab_size=len(text_preprocessor.vocab.itos),\n    embed_dim=300,\n    hidden_dim=64,\n    n_layers=3,\n    rnn_dropout=0.2,\n    head_hidden_dims=[32],\n)\n\n\nwd_text_model = WideDeep(deeptext=basic_rnn, pred_dim=4)\n</pre> basic_rnn = BasicRNN(     vocab_size=len(text_preprocessor.vocab.itos),     embed_dim=300,     hidden_dim=64,     n_layers=3,     rnn_dropout=0.2,     head_hidden_dims=[32], )   wd_text_model = WideDeep(deeptext=basic_rnn, pred_dim=4) In\u00a0[32]: Copied! <pre>wd_text_model\n</pre> wd_text_model Out[32]: <pre>WideDeep(\n  (deeptext): Sequential(\n    (0): BasicRNN(\n      (word_embed): Embedding(4328, 300, padding_idx=1)\n      (rnn): LSTM(300, 64, num_layers=3, batch_first=True, dropout=0.2)\n      (rnn_mlp): MLP(\n        (mlp): Sequential(\n          (dense_layer_0): Sequential(\n            (0): Linear(in_features=64, out_features=32, bias=True)\n            (1): ReLU(inplace=True)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=32, out_features=4, bias=True)\n  )\n)</pre> In\u00a0[33]: Copied! <pre>text_trainer = Trainer(\n    wd_text_model,\n    objective=\"multiclass\",\n    metrics=[Accuracy, F1Score(average=True)],\n    num_workers=0,  # As in the case of the tokenizer, in notebook I need to set this to 0 for the Trainer to work\n)\n</pre> text_trainer = Trainer(     wd_text_model,     objective=\"multiclass\",     metrics=[Accuracy, F1Score(average=True)],     num_workers=0,  # As in the case of the tokenizer, in notebook I need to set this to 0 for the Trainer to work ) In\u00a0[34]: Copied! <pre>text_trainer.fit(\n    X_text=wd_X_text_tr,\n    target=train.rating.values,\n    n_epochs=5,\n    batch_size=256,\n)\n</pre> text_trainer.fit(     X_text=wd_X_text_tr,     target=train.rating.values,     n_epochs=5,     batch_size=256, ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 71/71 [00:01&lt;00:00, 52.39it/s, loss=1.16, metrics={'acc': 0.5349, 'f1': 0.2011}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 71/71 [00:01&lt;00:00, 70.35it/s, loss=0.964, metrics={'acc': 0.5827, 'f1': 0.3005}]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 71/71 [00:01&lt;00:00, 70.33it/s, loss=0.845, metrics={'acc': 0.6252, 'f1': 0.4133}]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 71/71 [00:01&lt;00:00, 69.99it/s, loss=0.765, metrics={'acc': 0.6575, 'f1': 0.4875}]\nepoch 5: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 71/71 [00:01&lt;00:00, 69.55it/s, loss=0.709, metrics={'acc': 0.6879, 'f1': 0.5423}]\n</pre> In\u00a0[35]: Copied! <pre>wd_pred_text = text_trainer.predict_proba(X_text=wd_X_text_te)\nwd_pred_text_class = np.argmax(wd_pred_text, 1)\n</pre> wd_pred_text = text_trainer.predict_proba(X_text=wd_X_text_te) wd_pred_text_class = np.argmax(wd_pred_text, 1) <pre>predict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 18/18 [00:00&lt;00:00, 211.51it/s]\n</pre> In\u00a0[36]: Copied! <pre>wd_acc_text = accuracy_score(test.rating, wd_pred_text_class)\nwd_f1_text = f1_score(test.rating, wd_pred_text_class, average=\"weighted\")\nwd_cm_text = confusion_matrix(test.rating, wd_pred_text_class)\n</pre> wd_acc_text = accuracy_score(test.rating, wd_pred_text_class) wd_f1_text = f1_score(test.rating, wd_pred_text_class, average=\"weighted\") wd_cm_text = confusion_matrix(test.rating, wd_pred_text_class) In\u00a0[37]: Copied! <pre>print(f\"Basic RNN Accuracy: {wd_acc_text}. Basic RNN F1 Score: {wd_f1_text}\")\n</pre> print(f\"Basic RNN Accuracy: {wd_acc_text}. Basic RNN F1 Score: {wd_f1_text}\") <pre>Basic RNN Accuracy: 0.6076957098628926. Basic RNN F1 Score: 0.6017335854471788\n</pre> In\u00a0[38]: Copied! <pre>print(f\"Basic RNN Confusion Matrix:\\n {wd_cm_text}\")\n</pre> print(f\"Basic RNN Confusion Matrix:\\n {wd_cm_text}\") <pre>Basic RNN Confusion Matrix:\n [[ 327   76   62    9]\n [ 285  115  117   47]\n [ 131  122  315  413]\n [  42   69  401 1991]]\n</pre> <p>The performance is very similar to that of using simply tf-idf and lightgbm. Let see if adding tabular features helps when using <code>pytorch-widedeep</code></p> In\u00a0[39]: Copied! <pre># ?TabPreprocessor\n</pre> # ?TabPreprocessor In\u00a0[40]: Copied! <pre>tab_preprocessor = TabPreprocessor(cat_embed_cols=tab_cols)\n\nwd_X_tab_tr = tab_preprocessor.fit_transform(train)\nwd_X_tab_te = tab_preprocessor.transform(test)\n</pre> tab_preprocessor = TabPreprocessor(cat_embed_cols=tab_cols)  wd_X_tab_tr = tab_preprocessor.fit_transform(train) wd_X_tab_te = tab_preprocessor.transform(test) In\u00a0[41]: Copied! <pre># ?TabMlp\n</pre> # ?TabMlp In\u00a0[42]: Copied! <pre>tab_model = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    mlp_hidden_dims=[100, 50],\n)\n</pre> tab_model = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     mlp_hidden_dims=[100, 50], ) In\u00a0[43]: Copied! <pre>tab_model\n</pre> tab_model Out[43]: <pre>TabMlp(\n  (cat_and_cont_embed): DiffSizeCatAndContEmbeddings(\n    (cat_embed): DiffSizeCatEmbeddings(\n      (embed_layers): ModuleDict(\n        (emb_layer_age): Embedding(78, 18, padding_idx=0)\n        (emb_layer_division_name): Embedding(5, 3, padding_idx=0)\n        (emb_layer_department_name): Embedding(8, 5, padding_idx=0)\n        (emb_layer_class_name): Embedding(22, 9, padding_idx=0)\n      )\n      (embedding_dropout): Dropout(p=0.1, inplace=False)\n    )\n  )\n  (encoder): MLP(\n    (mlp): Sequential(\n      (dense_layer_0): Sequential(\n        (0): Dropout(p=0.1, inplace=False)\n        (1): Linear(in_features=35, out_features=100, bias=True)\n        (2): ReLU(inplace=True)\n      )\n      (dense_layer_1): Sequential(\n        (0): Dropout(p=0.1, inplace=False)\n        (1): Linear(in_features=100, out_features=50, bias=True)\n        (2): ReLU(inplace=True)\n      )\n    )\n  )\n)</pre> In\u00a0[44]: Copied! <pre>text_model = BasicRNN(\n    vocab_size=len(text_preprocessor.vocab.itos),\n    embed_dim=300,\n    hidden_dim=64,\n    n_layers=3,\n    rnn_dropout=0.2,\n    head_hidden_dims=[32],\n)\n</pre> text_model = BasicRNN(     vocab_size=len(text_preprocessor.vocab.itos),     embed_dim=300,     hidden_dim=64,     n_layers=3,     rnn_dropout=0.2,     head_hidden_dims=[32], ) In\u00a0[45]: Copied! <pre>wd_tab_and_text_model = WideDeep(deeptabular=tab_model, deeptext=text_model, pred_dim=4)\n</pre> wd_tab_and_text_model = WideDeep(deeptabular=tab_model, deeptext=text_model, pred_dim=4) In\u00a0[46]: Copied! <pre>wd_tab_and_text_model\n</pre> wd_tab_and_text_model Out[46]: <pre>WideDeep(\n  (deeptabular): Sequential(\n    (0): TabMlp(\n      (cat_and_cont_embed): DiffSizeCatAndContEmbeddings(\n        (cat_embed): DiffSizeCatEmbeddings(\n          (embed_layers): ModuleDict(\n            (emb_layer_age): Embedding(78, 18, padding_idx=0)\n            (emb_layer_division_name): Embedding(5, 3, padding_idx=0)\n            (emb_layer_department_name): Embedding(8, 5, padding_idx=0)\n            (emb_layer_class_name): Embedding(22, 9, padding_idx=0)\n          )\n          (embedding_dropout): Dropout(p=0.1, inplace=False)\n        )\n      )\n      (encoder): MLP(\n        (mlp): Sequential(\n          (dense_layer_0): Sequential(\n            (0): Dropout(p=0.1, inplace=False)\n            (1): Linear(in_features=35, out_features=100, bias=True)\n            (2): ReLU(inplace=True)\n          )\n          (dense_layer_1): Sequential(\n            (0): Dropout(p=0.1, inplace=False)\n            (1): Linear(in_features=100, out_features=50, bias=True)\n            (2): ReLU(inplace=True)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=50, out_features=4, bias=True)\n  )\n  (deeptext): Sequential(\n    (0): BasicRNN(\n      (word_embed): Embedding(4328, 300, padding_idx=1)\n      (rnn): LSTM(300, 64, num_layers=3, batch_first=True, dropout=0.2)\n      (rnn_mlp): MLP(\n        (mlp): Sequential(\n          (dense_layer_0): Sequential(\n            (0): Linear(in_features=64, out_features=32, bias=True)\n            (1): ReLU(inplace=True)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=32, out_features=4, bias=True)\n  )\n)</pre> In\u00a0[47]: Copied! <pre>tab_and_text_trainer = Trainer(\n    wd_tab_and_text_model,\n    objective=\"multiclass\",\n    metrics=[Accuracy, F1Score(average=True)],\n    num_workers=0,  # As in the case of the tokenizer, in notebook I need to set this to 0 for the Trainer to work\n)\n</pre> tab_and_text_trainer = Trainer(     wd_tab_and_text_model,     objective=\"multiclass\",     metrics=[Accuracy, F1Score(average=True)],     num_workers=0,  # As in the case of the tokenizer, in notebook I need to set this to 0 for the Trainer to work ) In\u00a0[48]: Copied! <pre>tab_and_text_trainer.fit(\n    X_tab=wd_X_tab_tr,\n    X_text=wd_X_text_tr,\n    target=train.rating.values,\n    n_epochs=5,\n    batch_size=256,\n)\n</pre> tab_and_text_trainer.fit(     X_tab=wd_X_tab_tr,     X_text=wd_X_text_tr,     target=train.rating.values,     n_epochs=5,     batch_size=256, ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 71/71 [00:01&lt;00:00, 52.04it/s, loss=1.13, metrics={'acc': 0.538, 'f1': 0.1911}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 71/71 [00:01&lt;00:00, 52.28it/s, loss=0.936, metrics={'acc': 0.5887, 'f1': 0.3507}]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 71/71 [00:01&lt;00:00, 52.26it/s, loss=0.825, metrics={'acc': 0.6394, 'f1': 0.4545}]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 71/71 [00:01&lt;00:00, 51.33it/s, loss=0.757, metrics={'acc': 0.6696, 'f1': 0.5214}]\nepoch 5: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 71/71 [00:01&lt;00:00, 50.39it/s, loss=0.702, metrics={'acc': 0.6963, 'f1': 0.5654}]\n</pre> In\u00a0[49]: Copied! <pre>wd_pred_tab_and_text = tab_and_text_trainer.predict_proba(\n    X_tab=wd_X_tab_te, X_text=wd_X_text_te\n)\nwd_pred_tab_and_text_class = np.argmax(wd_pred_tab_and_text, 1)\n</pre> wd_pred_tab_and_text = tab_and_text_trainer.predict_proba(     X_tab=wd_X_tab_te, X_text=wd_X_text_te ) wd_pred_tab_and_text_class = np.argmax(wd_pred_tab_and_text, 1) <pre>predict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 18/18 [00:00&lt;00:00, 136.94it/s]\n</pre> In\u00a0[50]: Copied! <pre>wd_acc_tab_and_text = accuracy_score(test.rating, wd_pred_tab_and_text_class)\nwd_f1_tab_and_text = f1_score(\n    test.rating, wd_pred_tab_and_text_class, average=\"weighted\"\n)\nwd_cm_tab_and_text = confusion_matrix(test.rating, wd_pred_tab_and_text_class)\n</pre> wd_acc_tab_and_text = accuracy_score(test.rating, wd_pred_tab_and_text_class) wd_f1_tab_and_text = f1_score(     test.rating, wd_pred_tab_and_text_class, average=\"weighted\" ) wd_cm_tab_and_text = confusion_matrix(test.rating, wd_pred_tab_and_text_class) In\u00a0[51]: Copied! <pre>print(\n    f\"Basic RNN + Tabular  Accuracy: {wd_acc_tab_and_text}. Basic RNN + TabularF1 Score: {wd_f1_tab_and_text}\"\n)\nprint(f\"Basic RNN + Tabular  Confusion Matrix:\\n {wd_cm_tab_and_text}\")\n</pre> print(     f\"Basic RNN + Tabular  Accuracy: {wd_acc_tab_and_text}. Basic RNN + TabularF1 Score: {wd_f1_tab_and_text}\" ) print(f\"Basic RNN + Tabular  Confusion Matrix:\\n {wd_cm_tab_and_text}\") <pre>Basic RNN + Tabular  Accuracy: 0.6333480760725343. Basic RNN + TabularF1 Score: 0.6332310089593208\nBasic RNN + Tabular  Confusion Matrix:\n [[ 267  132   65   10]\n [ 198  168  159   39]\n [  57  113  410  401]\n [  12   58  414 2019]]\n</pre> <p>We are going to \"manually\" code the Tokenizer and the model and see how they can be used as part of the process along with the <code>pytorch-widedeep</code> library.</p> <p>Tokenizer:</p> In\u00a0[52]: Copied! <pre>class BertTokenizer(object):\n    def __init__(\n        self,\n        pretrained_tokenizer=\"distilbert-base-uncased\",\n        do_lower_case=True,\n        max_length=90,\n    ):\n        super(BertTokenizer, self).__init__()\n        self.pretrained_tokenizer = pretrained_tokenizer\n        self.do_lower_case = do_lower_case\n        self.max_length = max_length\n\n    def fit(self, texts):\n        self.tokenizer = DistilBertTokenizer.from_pretrained(\n            self.pretrained_tokenizer, do_lower_case=self.do_lower_case\n        )\n\n        return self\n\n    def transform(self, texts):\n        input_ids = []\n        for text in texts:\n            encoded_sent = self.tokenizer.encode_plus(\n                text=self._pre_rules(text),\n                add_special_tokens=True,\n                max_length=self.max_length,\n                padding=\"max_length\",\n                truncation=True,\n            )\n\n            input_ids.append(encoded_sent.get(\"input_ids\"))\n        return np.stack(input_ids)\n\n    def fit_transform(self, texts):\n        return self.fit(texts).transform(texts)\n\n    @staticmethod\n    def _pre_rules(text):\n        return fix_html(rm_useless_spaces(spec_add_spaces(text)))\n</pre> class BertTokenizer(object):     def __init__(         self,         pretrained_tokenizer=\"distilbert-base-uncased\",         do_lower_case=True,         max_length=90,     ):         super(BertTokenizer, self).__init__()         self.pretrained_tokenizer = pretrained_tokenizer         self.do_lower_case = do_lower_case         self.max_length = max_length      def fit(self, texts):         self.tokenizer = DistilBertTokenizer.from_pretrained(             self.pretrained_tokenizer, do_lower_case=self.do_lower_case         )          return self      def transform(self, texts):         input_ids = []         for text in texts:             encoded_sent = self.tokenizer.encode_plus(                 text=self._pre_rules(text),                 add_special_tokens=True,                 max_length=self.max_length,                 padding=\"max_length\",                 truncation=True,             )              input_ids.append(encoded_sent.get(\"input_ids\"))         return np.stack(input_ids)      def fit_transform(self, texts):         return self.fit(texts).transform(texts)      @staticmethod     def _pre_rules(text):         return fix_html(rm_useless_spaces(spec_add_spaces(text))) <p>Model:</p> In\u00a0[53]: Copied! <pre>class BertModel(nn.Module):\n    def __init__(\n        self,\n        model_name: str = \"distilbert-base-uncased\",\n        freeze_bert: bool = False,\n    ):\n        super(BertModel, self).__init__()\n\n        self.bert = DistilBertModel.from_pretrained(\n            model_name,\n        )\n\n        if freeze_bert:\n            for param in self.bert.parameters():\n                param.requires_grad = False\n\n    def forward(self, X_inp: Tensor) -&gt; Tensor:\n        attn_mask = (X_inp != 0).type(torch.int8)\n        outputs = self.bert(input_ids=X_inp, attention_mask=attn_mask)\n        return outputs[0][:, 0, :]\n\n    @property\n    def output_dim(self) -&gt; int:\n        # This is THE ONLY requirement for any model to work with pytorch-widedeep. Must\n        # have a 'output_dim' property so the WideDeep class knows the incoming dims\n        # from the custom model. in this case, I hardcoded it\n        return 768\n</pre> class BertModel(nn.Module):     def __init__(         self,         model_name: str = \"distilbert-base-uncased\",         freeze_bert: bool = False,     ):         super(BertModel, self).__init__()          self.bert = DistilBertModel.from_pretrained(             model_name,         )          if freeze_bert:             for param in self.bert.parameters():                 param.requires_grad = False      def forward(self, X_inp: Tensor) -&gt; Tensor:         attn_mask = (X_inp != 0).type(torch.int8)         outputs = self.bert(input_ids=X_inp, attention_mask=attn_mask)         return outputs[0][:, 0, :]      @property     def output_dim(self) -&gt; int:         # This is THE ONLY requirement for any model to work with pytorch-widedeep. Must         # have a 'output_dim' property so the WideDeep class knows the incoming dims         # from the custom model. in this case, I hardcoded it         return 768 In\u00a0[54]: Copied! <pre>bert_tokenizer = BertTokenizer()\nX_bert_tr = bert_tokenizer.fit_transform(train[\"review_text\"].tolist())\nX_bert_te = bert_tokenizer.transform(test[\"review_text\"].tolist())\n</pre> bert_tokenizer = BertTokenizer() X_bert_tr = bert_tokenizer.fit_transform(train[\"review_text\"].tolist()) X_bert_te = bert_tokenizer.transform(test[\"review_text\"].tolist()) <p>As I mentioned a number of times in the documentation and examples, <code>pytorch-widedeep</code> is designed for flexibility. For any of the data modes (tabular, text and images) there are available components/models in the library. However, the user can choose to use any model they want with the only requirement that such model must have a <code>output_dim</code> property.</p> <p>With that in mind, the <code>BertModel</code> class defined above can be used by <code>pytorch-widedeep</code> as any other of the internal components. In other words, simply...pass it to the <code>WideDeep</code> class. In this case we are going to add a FC-head as part of the classifier.</p> In\u00a0[55]: Copied! <pre>bert_model = BertModel(freeze_bert=True)\nwd_bert_model = WideDeep(\n    deeptext=bert_model,\n    head_hidden_dims=[256, 128, 64],\n    pred_dim=4,\n)\n</pre> bert_model = BertModel(freeze_bert=True) wd_bert_model = WideDeep(     deeptext=bert_model,     head_hidden_dims=[256, 128, 64],     pred_dim=4, ) <pre>Some weights of the model checkpoint at distilbert-base-uncased were not used when initializing DistilBertModel: ['vocab_projector.bias', 'vocab_layer_norm.bias', 'vocab_transform.weight', 'vocab_transform.bias', 'vocab_projector.weight', 'vocab_layer_norm.weight']\n- This IS expected if you are initializing DistilBertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n- This IS NOT expected if you are initializing DistilBertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n</pre> In\u00a0[56]: Copied! <pre>wd_bert_model\n</pre> wd_bert_model Out[56]: <pre>WideDeep(\n  (deeptext): BertModel(\n    (bert): DistilBertModel(\n      (embeddings): Embeddings(\n        (word_embeddings): Embedding(30522, 768, padding_idx=0)\n        (position_embeddings): Embedding(512, 768)\n        (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n        (dropout): Dropout(p=0.1, inplace=False)\n      )\n      (transformer): Transformer(\n        (layer): ModuleList(\n          (0): TransformerBlock(\n            (attention): MultiHeadSelfAttention(\n              (dropout): Dropout(p=0.1, inplace=False)\n              (q_lin): Linear(in_features=768, out_features=768, bias=True)\n              (k_lin): Linear(in_features=768, out_features=768, bias=True)\n              (v_lin): Linear(in_features=768, out_features=768, bias=True)\n              (out_lin): Linear(in_features=768, out_features=768, bias=True)\n            )\n            (sa_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n            (ffn): FFN(\n              (dropout): Dropout(p=0.1, inplace=False)\n              (lin1): Linear(in_features=768, out_features=3072, bias=True)\n              (lin2): Linear(in_features=3072, out_features=768, bias=True)\n              (activation): GELUActivation()\n            )\n            (output_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n          )\n          (1): TransformerBlock(\n            (attention): MultiHeadSelfAttention(\n              (dropout): Dropout(p=0.1, inplace=False)\n              (q_lin): Linear(in_features=768, out_features=768, bias=True)\n              (k_lin): Linear(in_features=768, out_features=768, bias=True)\n              (v_lin): Linear(in_features=768, out_features=768, bias=True)\n              (out_lin): Linear(in_features=768, out_features=768, bias=True)\n            )\n            (sa_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n            (ffn): FFN(\n              (dropout): Dropout(p=0.1, inplace=False)\n              (lin1): Linear(in_features=768, out_features=3072, bias=True)\n              (lin2): Linear(in_features=3072, out_features=768, bias=True)\n              (activation): GELUActivation()\n            )\n            (output_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n          )\n          (2): TransformerBlock(\n            (attention): MultiHeadSelfAttention(\n              (dropout): Dropout(p=0.1, inplace=False)\n              (q_lin): Linear(in_features=768, out_features=768, bias=True)\n              (k_lin): Linear(in_features=768, out_features=768, bias=True)\n              (v_lin): Linear(in_features=768, out_features=768, bias=True)\n              (out_lin): Linear(in_features=768, out_features=768, bias=True)\n            )\n            (sa_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n            (ffn): FFN(\n              (dropout): Dropout(p=0.1, inplace=False)\n              (lin1): Linear(in_features=768, out_features=3072, bias=True)\n              (lin2): Linear(in_features=3072, out_features=768, bias=True)\n              (activation): GELUActivation()\n            )\n            (output_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n          )\n          (3): TransformerBlock(\n            (attention): MultiHeadSelfAttention(\n              (dropout): Dropout(p=0.1, inplace=False)\n              (q_lin): Linear(in_features=768, out_features=768, bias=True)\n              (k_lin): Linear(in_features=768, out_features=768, bias=True)\n              (v_lin): Linear(in_features=768, out_features=768, bias=True)\n              (out_lin): Linear(in_features=768, out_features=768, bias=True)\n            )\n            (sa_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n            (ffn): FFN(\n              (dropout): Dropout(p=0.1, inplace=False)\n              (lin1): Linear(in_features=768, out_features=3072, bias=True)\n              (lin2): Linear(in_features=3072, out_features=768, bias=True)\n              (activation): GELUActivation()\n            )\n            (output_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n          )\n          (4): TransformerBlock(\n            (attention): MultiHeadSelfAttention(\n              (dropout): Dropout(p=0.1, inplace=False)\n              (q_lin): Linear(in_features=768, out_features=768, bias=True)\n              (k_lin): Linear(in_features=768, out_features=768, bias=True)\n              (v_lin): Linear(in_features=768, out_features=768, bias=True)\n              (out_lin): Linear(in_features=768, out_features=768, bias=True)\n            )\n            (sa_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n            (ffn): FFN(\n              (dropout): Dropout(p=0.1, inplace=False)\n              (lin1): Linear(in_features=768, out_features=3072, bias=True)\n              (lin2): Linear(in_features=3072, out_features=768, bias=True)\n              (activation): GELUActivation()\n            )\n            (output_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n          )\n          (5): TransformerBlock(\n            (attention): MultiHeadSelfAttention(\n              (dropout): Dropout(p=0.1, inplace=False)\n              (q_lin): Linear(in_features=768, out_features=768, bias=True)\n              (k_lin): Linear(in_features=768, out_features=768, bias=True)\n              (v_lin): Linear(in_features=768, out_features=768, bias=True)\n              (out_lin): Linear(in_features=768, out_features=768, bias=True)\n            )\n            (sa_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n            (ffn): FFN(\n              (dropout): Dropout(p=0.1, inplace=False)\n              (lin1): Linear(in_features=768, out_features=3072, bias=True)\n              (lin2): Linear(in_features=3072, out_features=768, bias=True)\n              (activation): GELUActivation()\n            )\n            (output_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n          )\n        )\n      )\n    )\n  )\n  (deephead): Sequential(\n    (0): MLP(\n      (mlp): Sequential(\n        (dense_layer_0): Sequential(\n          (0): Linear(in_features=768, out_features=256, bias=True)\n          (1): ReLU(inplace=True)\n          (2): Dropout(p=0.1, inplace=False)\n        )\n        (dense_layer_1): Sequential(\n          (0): Linear(in_features=256, out_features=128, bias=True)\n          (1): ReLU(inplace=True)\n          (2): Dropout(p=0.1, inplace=False)\n        )\n        (dense_layer_2): Sequential(\n          (0): Linear(in_features=128, out_features=64, bias=True)\n          (1): ReLU(inplace=True)\n          (2): Dropout(p=0.1, inplace=False)\n        )\n      )\n    )\n    (1): Linear(in_features=64, out_features=4, bias=True)\n  )\n)</pre> In\u00a0[57]: Copied! <pre>wd_bert_trainer = Trainer(\n    wd_bert_model,\n    objective=\"multiclass\",\n    metrics=[Accuracy, F1Score(average=True)],\n    num_workers=0,  # As in the case of the tokenizer, in notebook I need to set this to 0 for the Trainer to work\n)\n\nwd_bert_trainer.fit(\n    X_text=X_bert_tr,\n    target=train.rating.values,\n    n_epochs=3,\n    batch_size=64,\n)\n</pre> wd_bert_trainer = Trainer(     wd_bert_model,     objective=\"multiclass\",     metrics=[Accuracy, F1Score(average=True)],     num_workers=0,  # As in the case of the tokenizer, in notebook I need to set this to 0 for the Trainer to work )  wd_bert_trainer.fit(     X_text=X_bert_tr,     target=train.rating.values,     n_epochs=3,     batch_size=64, ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 283/283 [00:14&lt;00:00, 19.68it/s, loss=0.968, metrics={'acc': 0.5879, 'f1': 0.3591}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 283/283 [00:14&lt;00:00, 19.63it/s, loss=0.884, metrics={'acc': 0.6178, 'f1': 0.4399}]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 283/283 [00:14&lt;00:00, 19.55it/s, loss=0.87, metrics={'acc': 0.6234, 'f1': 0.4527}]\n</pre> In\u00a0[58]: Copied! <pre>wd_bert_pred_text = wd_bert_trainer.predict_proba(X_text=X_bert_te)\nwd_bert_pred_text_class = np.argmax(wd_bert_pred_text, 1)\n</pre> wd_bert_pred_text = wd_bert_trainer.predict_proba(X_text=X_bert_te) wd_bert_pred_text_class = np.argmax(wd_bert_pred_text, 1) <pre>predict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 71/71 [00:03&lt;00:00, 21.97it/s]\n</pre> In\u00a0[59]: Copied! <pre>wd_bert_acc = accuracy_score(test.rating, wd_bert_pred_text_class)\nwd_bert_f1 = f1_score(test.rating, wd_bert_pred_text_class, average=\"weighted\")\nwd_bert_cm = confusion_matrix(test.rating, wd_bert_pred_text_class)\n</pre> wd_bert_acc = accuracy_score(test.rating, wd_bert_pred_text_class) wd_bert_f1 = f1_score(test.rating, wd_bert_pred_text_class, average=\"weighted\") wd_bert_cm = confusion_matrix(test.rating, wd_bert_pred_text_class) In\u00a0[60]: Copied! <pre>print(f\"Distilbert Accuracy: {wd_bert_acc}. Distilbert F1 Score: {wd_bert_f1}\")\nprint(f\"Distilbert Confusion Matrix:\\n {wd_bert_cm}\")\n</pre> print(f\"Distilbert Accuracy: {wd_bert_acc}. Distilbert F1 Score: {wd_bert_f1}\") print(f\"Distilbert Confusion Matrix:\\n {wd_bert_cm}\") <pre>Distilbert Accuracy: 0.6326846528084918. Distilbert F1 Score: 0.5796652991272998\nDistilbert Confusion Matrix:\n [[ 287   75   22   90]\n [ 197  136   62  169]\n [  68  119  123  671]\n [  40   64   84 2315]]\n</pre> <p>Now, adding a tabular model follows the exact same process as the one described in section 2.</p> In\u00a0[61]: Copied! <pre>tab_model = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    mlp_hidden_dims=[100, 50],\n)\n</pre> tab_model = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     mlp_hidden_dims=[100, 50], ) In\u00a0[62]: Copied! <pre>wd_tab_bert_model = WideDeep(\n    deeptabular=tab_model,\n    deeptext=bert_model,\n    head_hidden_dims=[256, 128, 64],\n    pred_dim=4,\n)\n</pre> wd_tab_bert_model = WideDeep(     deeptabular=tab_model,     deeptext=bert_model,     head_hidden_dims=[256, 128, 64],     pred_dim=4, ) In\u00a0[63]: Copied! <pre>wd_tab_bert_trainer = Trainer(\n    wd_tab_bert_model,\n    objective=\"multiclass\",\n    metrics=[Accuracy, F1Score(average=True)],\n    num_workers=0,  # As in the case of the tokenizer, in notebook I need to set this to 0 for the Trainer to work\n)\n</pre> wd_tab_bert_trainer = Trainer(     wd_tab_bert_model,     objective=\"multiclass\",     metrics=[Accuracy, F1Score(average=True)],     num_workers=0,  # As in the case of the tokenizer, in notebook I need to set this to 0 for the Trainer to work ) In\u00a0[64]: Copied! <pre>wd_tab_bert_trainer.fit(\n    X_tab=wd_X_tab_tr,\n    X_text=X_bert_tr,\n    target=train.rating.values,\n    n_epochs=3,\n    batch_size=64,\n)\n</pre> wd_tab_bert_trainer.fit(     X_tab=wd_X_tab_tr,     X_text=X_bert_tr,     target=train.rating.values,     n_epochs=3,     batch_size=64, ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 283/283 [00:15&lt;00:00, 18.15it/s, loss=0.974, metrics={'acc': 0.5838, 'f1': 0.3404}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 283/283 [00:15&lt;00:00, 18.38it/s, loss=0.885, metrics={'acc': 0.618, 'f1': 0.4378}]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 283/283 [00:15&lt;00:00, 18.40it/s, loss=0.868, metrics={'acc': 0.6252, 'f1': 0.4575}]\n</pre> In\u00a0[65]: Copied! <pre>wd_tab_bert_pred_text = wd_tab_bert_trainer.predict_proba(\n    X_tab=wd_X_tab_te, X_text=X_bert_te\n)\nwd_tab_bert_pred_text_class = np.argmax(wd_tab_bert_pred_text, 1)\n</pre> wd_tab_bert_pred_text = wd_tab_bert_trainer.predict_proba(     X_tab=wd_X_tab_te, X_text=X_bert_te ) wd_tab_bert_pred_text_class = np.argmax(wd_tab_bert_pred_text, 1) <pre>predict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 71/71 [00:03&lt;00:00, 21.32it/s]\n</pre> In\u00a0[66]: Copied! <pre>wd_tab_bert_acc = accuracy_score(test.rating, wd_tab_bert_pred_text_class)\nwd_tab_bert_f1 = f1_score(test.rating, wd_tab_bert_pred_text_class, average=\"weighted\")\nwd_tab_bert_cm = confusion_matrix(test.rating, wd_tab_bert_pred_text_class)\n</pre> wd_tab_bert_acc = accuracy_score(test.rating, wd_tab_bert_pred_text_class) wd_tab_bert_f1 = f1_score(test.rating, wd_tab_bert_pred_text_class, average=\"weighted\") wd_tab_bert_cm = confusion_matrix(test.rating, wd_tab_bert_pred_text_class) In\u00a0[67]: Copied! <pre>print(\n    f\"Distilbert + Tabular Accuracy: {wd_tab_bert_acc}. Distilbert+ Tabular F1 Score: {wd_tab_bert_f1}\"\n)\nprint(f\"Distilbert + Tabular Confusion Matrix:\\n {wd_tab_bert_cm}\")\n</pre> print(     f\"Distilbert + Tabular Accuracy: {wd_tab_bert_acc}. Distilbert+ Tabular F1 Score: {wd_tab_bert_f1}\" ) print(f\"Distilbert + Tabular Confusion Matrix:\\n {wd_tab_bert_cm}\") <pre>Distilbert + Tabular Accuracy: 0.6242812914639541. Distilbert+ Tabular F1 Score: 0.5508351761564895\nDistilbert + Tabular Confusion Matrix:\n [[ 297   56   11  110]\n [ 229   91   38  206]\n [  86   90   71  734]\n [  49   48   42 2364]]\n</pre>"},{"location":"examples/16_Usign_a_custom_hugging_face_model.html#using-a-hugginface-model","title":"Using a Hugginface model\u00b6","text":"<p>In this notebook we will show how to use an \"external\" Hugginface model along with any other model in the libray. In particular we will show how to combine it with a tabular DL model.</p> <p>Since we are here, we will also compare the performance of a few models on a text classification problem.</p> <p>The notebook will go as follows:</p> <ol> <li>Text classification using tf-idf + LightGBM</li> <li>Text classification using a basic RNN</li> <li>Text classification using Distilbert</li> </ol> <p>In all 3 cases we will add some tabular features to see if these help.</p> <p>In general, I would not pay much attention to the results since I have placed no effort in getting the best possible results (i.e. no hyperparameter optimization or trying different architectures, for example).</p> <p>Let's go</p>"},{"location":"examples/16_Usign_a_custom_hugging_face_model.html#1-text-classification-using-tf-idf-lightgbm","title":"1. Text classification using tf-idf + LightGBM\u00b6","text":""},{"location":"examples/16_Usign_a_custom_hugging_face_model.html#2-text-classification-using-pytorch-widedeeps-built-in-models-a-basic-rnn","title":"2. Text classification using pytorch-widedeep's built-in models (a basic RNN)\u00b6","text":"<p>Moving on now to fully using <code>pytorch-widedeep</code> in this dataset, let's have a look on how one could use a simple RNN to predict the ratings with the library.</p>"},{"location":"examples/16_Usign_a_custom_hugging_face_model.html#3-text-classification-using-a-hugginface-model-as-a-custom-model-in-pytorch-widedeeps","title":"3. Text classification using a Hugginface model as a custom model in pytorch-widedeep's\u00b6","text":""},{"location":"examples/17_feature_importance_via_attention_weights.html","title":"17_feature_importance_via_attention_weights","text":"In\u00a0[1]: Copied! <pre>import torch\n\nimport numpy as np\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import accuracy_score\n\n\nfrom pytorch_widedeep import Trainer\nfrom pytorch_widedeep.models import TabTransformer, ContextAttentionMLP, WideDeep\nfrom pytorch_widedeep.callbacks import EarlyStopping\nfrom pytorch_widedeep.metrics import Accuracy\nfrom pytorch_widedeep.datasets import load_adult\nfrom pytorch_widedeep.preprocessing import TabPreprocessor\n</pre> import torch  import numpy as np from sklearn.model_selection import train_test_split from sklearn.metrics import accuracy_score   from pytorch_widedeep import Trainer from pytorch_widedeep.models import TabTransformer, ContextAttentionMLP, WideDeep from pytorch_widedeep.callbacks import EarlyStopping from pytorch_widedeep.metrics import Accuracy from pytorch_widedeep.datasets import load_adult from pytorch_widedeep.preprocessing import TabPreprocessor In\u00a0[2]: Copied! <pre># use_cuda = torch.cuda.is_available()\ndf = load_adult(as_frame=True)\ndf.columns = [c.replace(\"-\", \"_\") for c in df.columns]\ndf[\"income_label\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int)\ndf.drop([\"income\", \"fnlwgt\", \"educational_num\"], axis=1, inplace=True)\ntarget_colname = \"income_label\"\n</pre> # use_cuda = torch.cuda.is_available() df = load_adult(as_frame=True) df.columns = [c.replace(\"-\", \"_\") for c in df.columns] df[\"income_label\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int) df.drop([\"income\", \"fnlwgt\", \"educational_num\"], axis=1, inplace=True) target_colname = \"income_label\" In\u00a0[3]: Copied! <pre>df.head()\n</pre> df.head() Out[3]: age workclass education marital_status occupation relationship race gender capital_gain capital_loss hours_per_week native_country income_label 0 25 Private 11th Never-married Machine-op-inspct Own-child Black Male 0 0 40 United-States 0 1 38 Private HS-grad Married-civ-spouse Farming-fishing Husband White Male 0 0 50 United-States 0 2 28 Local-gov Assoc-acdm Married-civ-spouse Protective-serv Husband White Male 0 0 40 United-States 1 3 44 Private Some-college Married-civ-spouse Machine-op-inspct Husband Black Male 7688 0 40 United-States 1 4 18 ? Some-college Never-married ? Own-child White Female 0 0 30 United-States 0 In\u00a0[4]: Copied! <pre>cat_embed_cols = []\nfor col in df.columns:\n    if df[col].dtype == \"O\" or df[col].nunique() &lt; 200 and col != target_colname:\n        cat_embed_cols.append(col)\n</pre> cat_embed_cols = [] for col in df.columns:     if df[col].dtype == \"O\" or df[col].nunique() &lt; 200 and col != target_colname:         cat_embed_cols.append(col) In\u00a0[5]: Copied! <pre># all cols will be categorical\nassert len(cat_embed_cols) == df.shape[1] - 1\n</pre> # all cols will be categorical assert len(cat_embed_cols) == df.shape[1] - 1 In\u00a0[6]: Copied! <pre>train, test = train_test_split(\n    df, test_size=0.1, random_state=1, stratify=df[[target_colname]]\n)\n</pre> train, test = train_test_split(     df, test_size=0.1, random_state=1, stratify=df[[target_colname]] ) In\u00a0[7]: Copied! <pre>tab_preprocessor = TabPreprocessor(cat_embed_cols=cat_embed_cols, with_attention=True)\n</pre> tab_preprocessor = TabPreprocessor(cat_embed_cols=cat_embed_cols, with_attention=True) In\u00a0[8]: Copied! <pre>X_tab_train = tab_preprocessor.fit_transform(train)\nX_tab_test = tab_preprocessor.transform(test)\ntarget = train[target_colname].values\n</pre> X_tab_train = tab_preprocessor.fit_transform(train) X_tab_test = tab_preprocessor.transform(test) target = train[target_colname].values In\u00a0[9]: Copied! <pre>tab_transformer = TabTransformer(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    embed_continuous_method=\"standard\",\n    input_dim=8,\n    n_heads=2,\n    n_blocks=1,\n    attn_dropout=0.1,\n    transformer_activation=\"relu\",\n)\n</pre> tab_transformer = TabTransformer(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     embed_continuous_method=\"standard\",     input_dim=8,     n_heads=2,     n_blocks=1,     attn_dropout=0.1,     transformer_activation=\"relu\", ) In\u00a0[10]: Copied! <pre>model = WideDeep(deeptabular=tab_transformer)\n</pre> model = WideDeep(deeptabular=tab_transformer) In\u00a0[11]: Copied! <pre>optimizer = torch.optim.Adam(model.parameters(), lr=0.01, weight_decay=0.0)\n</pre> optimizer = torch.optim.Adam(model.parameters(), lr=0.01, weight_decay=0.0) In\u00a0[12]: Copied! <pre>lr_scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(\n    optimizer,\n    threshold=0.001,\n    threshold_mode=\"abs\",\n    patience=10,\n)\n</pre> lr_scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(     optimizer,     threshold=0.001,     threshold_mode=\"abs\",     patience=10, ) In\u00a0[13]: Copied! <pre>early_stopping = EarlyStopping(\n    min_delta=0.001, patience=30, restore_best_weights=True, verbose=True\n)\n</pre> early_stopping = EarlyStopping(     min_delta=0.001, patience=30, restore_best_weights=True, verbose=True ) In\u00a0[14]: Copied! <pre>trainer = Trainer(\n    model,\n    objective=\"binary\",\n    optimizers=optimizer,\n    lr_schedulers=lr_scheduler,\n    reducelronplateau_criterion=\"loss\",\n    callbacks=[early_stopping],\n    metrics=[Accuracy],\n)\n</pre> trainer = Trainer(     model,     objective=\"binary\",     optimizers=optimizer,     lr_schedulers=lr_scheduler,     reducelronplateau_criterion=\"loss\",     callbacks=[early_stopping],     metrics=[Accuracy], ) <p>The feature importances will be computed after training, using a sample of the training dataset of size <code>feature_importance_sample_size</code></p> In\u00a0[15]: Copied! <pre>trainer.fit(\n    X_tab=X_tab_train,\n    target=target,\n    val_split=0.2,\n    n_epochs=100,\n    batch_size=128,\n    validation_freq=1,\n    feature_importance_sample_size=1000,\n)\n</pre> trainer.fit(     X_tab=X_tab_train,     target=target,     val_split=0.2,     n_epochs=100,     batch_size=128,     validation_freq=1,     feature_importance_sample_size=1000, ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 81.80it/s, loss=0.334, metrics={'acc': 0.847}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 111.34it/s, loss=0.294, metrics={'acc': 0.8669}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 83.02it/s, loss=0.293, metrics={'acc': 0.8656}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 124.03it/s, loss=0.283, metrics={'acc': 0.8678}]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 87.69it/s, loss=0.282, metrics={'acc': 0.8703}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 118.22it/s, loss=0.279, metrics={'acc': 0.8717}]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 79.24it/s, loss=0.277, metrics={'acc': 0.8718}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 115.29it/s, loss=0.277, metrics={'acc': 0.8731}]\nepoch 5: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 83.76it/s, loss=0.275, metrics={'acc': 0.8727}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 120.80it/s, loss=0.276, metrics={'acc': 0.8727}]\nepoch 6: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 82.78it/s, loss=0.273, metrics={'acc': 0.873}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 100.43it/s, loss=0.276, metrics={'acc': 0.871}]\nepoch 7: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 76.35it/s, loss=0.271, metrics={'acc': 0.8742}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 116.14it/s, loss=0.275, metrics={'acc': 0.8726}]\nepoch 8: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 74.29it/s, loss=0.271, metrics={'acc': 0.875}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 117.46it/s, loss=0.276, metrics={'acc': 0.8718}]\nepoch 9: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 77.10it/s, loss=0.27, metrics={'acc': 0.8761}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 105.49it/s, loss=0.275, metrics={'acc': 0.8728}]\nepoch 10: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 70.40it/s, loss=0.269, metrics={'acc': 0.8747}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 105.47it/s, loss=0.275, metrics={'acc': 0.8726}]\nepoch 11: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 72.83it/s, loss=0.268, metrics={'acc': 0.8742}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 105.03it/s, loss=0.274, metrics={'acc': 0.873}]\nepoch 12: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 71.86it/s, loss=0.267, metrics={'acc': 0.8743}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 106.61it/s, loss=0.274, metrics={'acc': 0.8734}]\nepoch 13: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 72.39it/s, loss=0.267, metrics={'acc': 0.876}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 108.05it/s, loss=0.275, metrics={'acc': 0.8717}]\nepoch 14: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 73.36it/s, loss=0.265, metrics={'acc': 0.8767}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 109.60it/s, loss=0.276, metrics={'acc': 0.8747}]\nepoch 15: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 73.34it/s, loss=0.264, metrics={'acc': 0.876}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 117.55it/s, loss=0.276, metrics={'acc': 0.8706}]\nepoch 16: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 75.35it/s, loss=0.264, metrics={'acc': 0.8777}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 122.08it/s, loss=0.275, metrics={'acc': 0.8753}]\nepoch 17: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 75.96it/s, loss=0.263, metrics={'acc': 0.877}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 120.83it/s, loss=0.277, metrics={'acc': 0.8739}]\nepoch 18: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 75.38it/s, loss=0.263, metrics={'acc': 0.8779}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 118.18it/s, loss=0.278, metrics={'acc': 0.8714}]\nepoch 19: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 75.61it/s, loss=0.261, metrics={'acc': 0.8784}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 119.19it/s, loss=0.278, metrics={'acc': 0.8712}]\nepoch 20: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 72.43it/s, loss=0.261, metrics={'acc': 0.8791}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 118.87it/s, loss=0.28, metrics={'acc': 0.873}]\nepoch 21: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 73.97it/s, loss=0.26, metrics={'acc': 0.8787}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 107.50it/s, loss=0.279, metrics={'acc': 0.8732}]\nepoch 22: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 71.76it/s, loss=0.253, metrics={'acc': 0.8816}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 108.11it/s, loss=0.279, metrics={'acc': 0.8707}]\nepoch 23: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 71.92it/s, loss=0.252, metrics={'acc': 0.8828}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 100.14it/s, loss=0.28, metrics={'acc': 0.8711}]\nepoch 24: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 75.04it/s, loss=0.252, metrics={'acc': 0.8829}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 98.36it/s, loss=0.28, metrics={'acc': 0.8708}]\nepoch 25: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 76.31it/s, loss=0.251, metrics={'acc': 0.883}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 122.97it/s, loss=0.281, metrics={'acc': 0.8709}]\nepoch 26: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 76.63it/s, loss=0.25, metrics={'acc': 0.8834}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 123.07it/s, loss=0.281, metrics={'acc': 0.8698}]\nepoch 27: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 78.37it/s, loss=0.251, metrics={'acc': 0.884}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 114.75it/s, loss=0.281, metrics={'acc': 0.87}]\nepoch 28: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 77.19it/s, loss=0.25, metrics={'acc': 0.883}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 117.99it/s, loss=0.282, metrics={'acc': 0.8699}]\nepoch 29: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 76.19it/s, loss=0.25, metrics={'acc': 0.8829}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 111.11it/s, loss=0.282, metrics={'acc': 0.8695}]\nepoch 30: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 77.57it/s, loss=0.249, metrics={'acc': 0.8839}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 115.64it/s, loss=0.283, metrics={'acc': 0.8689}]\nepoch 31: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 79.55it/s, loss=0.249, metrics={'acc': 0.8846}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 125.10it/s, loss=0.283, metrics={'acc': 0.869}]\nepoch 32: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 82.56it/s, loss=0.248, metrics={'acc': 0.8841}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 118.45it/s, loss=0.284, metrics={'acc': 0.8687}]\nepoch 33: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 81.06it/s, loss=0.248, metrics={'acc': 0.8848}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 129.26it/s, loss=0.284, metrics={'acc': 0.8689}]\nepoch 34: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 80.53it/s, loss=0.248, metrics={'acc': 0.8854}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 120.61it/s, loss=0.283, metrics={'acc': 0.869}]\nepoch 35: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 80.78it/s, loss=0.248, metrics={'acc': 0.8853}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 127.31it/s, loss=0.283, metrics={'acc': 0.8694}]\nepoch 36: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 82.51it/s, loss=0.248, metrics={'acc': 0.8863}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 125.94it/s, loss=0.283, metrics={'acc': 0.8693}]\nepoch 37: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 81.35it/s, loss=0.247, metrics={'acc': 0.8844}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 126.77it/s, loss=0.283, metrics={'acc': 0.8692}]\nepoch 38: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 80.62it/s, loss=0.248, metrics={'acc': 0.8837}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 116.62it/s, loss=0.283, metrics={'acc': 0.8692}]\nepoch 39: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 79.82it/s, loss=0.248, metrics={'acc': 0.8842}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 123.64it/s, loss=0.283, metrics={'acc': 0.8695}]\nepoch 40: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 76.86it/s, loss=0.247, metrics={'acc': 0.8855}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 120.34it/s, loss=0.283, metrics={'acc': 0.8692}]\n</pre> <pre>Best Epoch: 10. Best val_loss: 0.27451\nRestoring model weights from the end of the best epoch\n</pre> In\u00a0[16]: Copied! <pre>trainer.feature_importance\n</pre> trainer.feature_importance Out[16]: <pre>{'age': 0.09718182,\n 'workclass': 0.090637445,\n 'education': 0.08910798,\n 'marital_status': 0.08971319,\n 'occupation': 0.12546304,\n 'relationship': 0.086381145,\n 'race': 0.050686445,\n 'gender': 0.05116429,\n 'capital_gain': 0.08165918,\n 'capital_loss': 0.07702667,\n 'hours_per_week': 0.08205996,\n 'native_country': 0.07891885}</pre> In\u00a0[17]: Copied! <pre>preds = trainer.predict(X_tab=X_tab_test)\n</pre> preds = trainer.predict(X_tab=X_tab_test) <pre>predict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 39/39 [00:00&lt;00:00, 199.63it/s]\n</pre> In\u00a0[18]: Copied! <pre>accuracy_score(preds, test.income_label)\n</pre> accuracy_score(preds, test.income_label) Out[18]: <pre>0.8685772773797339</pre> In\u00a0[19]: Copied! <pre>test.reset_index(drop=True, inplace=True)\n</pre> test.reset_index(drop=True, inplace=True) In\u00a0[20]: Copied! <pre>test[test.income_label == 0].head(1)\n</pre> test[test.income_label == 0].head(1) Out[20]: age workclass education marital_status occupation relationship race gender capital_gain capital_loss hours_per_week native_country income_label 0 26 Private Some-college Never-married Exec-managerial Not-in-family White Male 0 0 60 United-States 0 In\u00a0[21]: Copied! <pre>test[test.income_label == 1].head(1)\n</pre> test[test.income_label == 1].head(1) Out[21]: age workclass education marital_status occupation relationship race gender capital_gain capital_loss hours_per_week native_country income_label 3 36 Local-gov Doctorate Married-civ-spouse Prof-specialty Husband White Male 0 1887 50 United-States 1 <p>To get the feature importance of a test dataset, simply use the <code>explain</code> method</p> In\u00a0[22]: Copied! <pre>feat_imp_per_sample = trainer.explain(X_tab_test, save_step_masks=False)\n</pre> feat_imp_per_sample = trainer.explain(X_tab_test, save_step_masks=False) In\u00a0[23]: Copied! <pre>list(test.iloc[0].index[np.argsort(-feat_imp_per_sample[0])])\n</pre> list(test.iloc[0].index[np.argsort(-feat_imp_per_sample[0])]) Out[23]: <pre>['hours_per_week',\n 'education',\n 'relationship',\n 'occupation',\n 'workclass',\n 'capital_gain',\n 'native_country',\n 'marital_status',\n 'capital_loss',\n 'age',\n 'race',\n 'gender']</pre> In\u00a0[24]: Copied! <pre>list(test.iloc[3].index[np.argsort(-feat_imp_per_sample[3])])\n</pre> list(test.iloc[3].index[np.argsort(-feat_imp_per_sample[3])]) Out[24]: <pre>['age',\n 'capital_loss',\n 'hours_per_week',\n 'marital_status',\n 'native_country',\n 'relationship',\n 'race',\n 'education',\n 'occupation',\n 'capital_gain',\n 'gender',\n 'workclass']</pre> <p>We could do the same with the <code>ContextAttentionMLP</code></p> In\u00a0[25]: Copied! <pre>context_attn_mlp = ContextAttentionMLP(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    cat_embed_dropout=0.0,\n    input_dim=16,\n    attn_dropout=0.1,\n    attn_activation=\"relu\",\n)\n</pre> context_attn_mlp = ContextAttentionMLP(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     cat_embed_dropout=0.0,     input_dim=16,     attn_dropout=0.1,     attn_activation=\"relu\", ) In\u00a0[26]: Copied! <pre>mlp_model = WideDeep(deeptabular=context_attn_mlp)\n</pre> mlp_model = WideDeep(deeptabular=context_attn_mlp) In\u00a0[27]: Copied! <pre>mlp_optimizer = torch.optim.Adam(mlp_model.parameters(), lr=0.01, weight_decay=0.0)\n</pre> mlp_optimizer = torch.optim.Adam(mlp_model.parameters(), lr=0.01, weight_decay=0.0) In\u00a0[28]: Copied! <pre>mlp_lr_scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(\n    mlp_optimizer,\n    threshold=0.001,\n    threshold_mode=\"abs\",\n    patience=10,\n)\n</pre> mlp_lr_scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(     mlp_optimizer,     threshold=0.001,     threshold_mode=\"abs\",     patience=10, ) In\u00a0[29]: Copied! <pre>mlp_early_stopping = EarlyStopping(\n    min_delta=0.001, patience=30, restore_best_weights=True, verbose=True\n)\n</pre> mlp_early_stopping = EarlyStopping(     min_delta=0.001, patience=30, restore_best_weights=True, verbose=True ) In\u00a0[30]: Copied! <pre>mlp_trainer = Trainer(\n    mlp_model,\n    objective=\"binary\",\n    optimizers=mlp_optimizer,\n    lr_schedulers=mlp_lr_scheduler,\n    reducelronplateau_criterion=\"loss\",\n    callbacks=[mlp_early_stopping],\n    metrics=[Accuracy],\n)\n</pre> mlp_trainer = Trainer(     mlp_model,     objective=\"binary\",     optimizers=mlp_optimizer,     lr_schedulers=mlp_lr_scheduler,     reducelronplateau_criterion=\"loss\",     callbacks=[mlp_early_stopping],     metrics=[Accuracy], ) In\u00a0[31]: Copied! <pre>mlp_trainer.fit(\n    X_tab=X_tab_train,\n    target=target,\n    val_split=0.2,\n    n_epochs=100,\n    batch_size=128,\n    validation_freq=1,\n    feature_importance_sample_size=1000,\n)\n</pre> mlp_trainer.fit(     X_tab=X_tab_train,     target=target,     val_split=0.2,     n_epochs=100,     batch_size=128,     validation_freq=1,     feature_importance_sample_size=1000, ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 73.11it/s, loss=0.405, metrics={'acc': 0.8094}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 119.26it/s, loss=0.309, metrics={'acc': 0.8583}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 71.70it/s, loss=0.332, metrics={'acc': 0.8447}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 116.36it/s, loss=0.293, metrics={'acc': 0.8646}]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 78.42it/s, loss=0.319, metrics={'acc': 0.8505}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 118.05it/s, loss=0.293, metrics={'acc': 0.8654}]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 79.00it/s, loss=0.312, metrics={'acc': 0.8554}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 116.49it/s, loss=0.291, metrics={'acc': 0.8661}]\nepoch 5: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 79.50it/s, loss=0.308, metrics={'acc': 0.8583}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 120.48it/s, loss=0.287, metrics={'acc': 0.8669}]\nepoch 6: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 80.84it/s, loss=0.303, metrics={'acc': 0.8605}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 128.70it/s, loss=0.288, metrics={'acc': 0.8673}]\nepoch 7: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 79.93it/s, loss=0.301, metrics={'acc': 0.8597}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 121.47it/s, loss=0.298, metrics={'acc': 0.8628}]\nepoch 8: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 80.56it/s, loss=0.3, metrics={'acc': 0.8592}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 119.84it/s, loss=0.281, metrics={'acc': 0.8718}]\nepoch 9: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 79.17it/s, loss=0.298, metrics={'acc': 0.8619}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 126.32it/s, loss=0.28, metrics={'acc': 0.8716}]\nepoch 10: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 82.13it/s, loss=0.297, metrics={'acc': 0.8615}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 120.50it/s, loss=0.281, metrics={'acc': 0.8718}]\nepoch 11: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 82.54it/s, loss=0.293, metrics={'acc': 0.8641}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 122.57it/s, loss=0.284, metrics={'acc': 0.867}]\nepoch 12: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 80.92it/s, loss=0.293, metrics={'acc': 0.863}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 126.42it/s, loss=0.282, metrics={'acc': 0.8701}]\nepoch 13: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 79.61it/s, loss=0.293, metrics={'acc': 0.8635}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 117.56it/s, loss=0.276, metrics={'acc': 0.8719}]\nepoch 14: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 74.92it/s, loss=0.29, metrics={'acc': 0.8633}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 112.06it/s, loss=0.286, metrics={'acc': 0.8669}]\nepoch 15: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 76.83it/s, loss=0.291, metrics={'acc': 0.865}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 112.88it/s, loss=0.282, metrics={'acc': 0.8677}]\nepoch 16: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 76.74it/s, loss=0.29, metrics={'acc': 0.8653}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 120.85it/s, loss=0.285, metrics={'acc': 0.8672}]\nepoch 17: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 75.99it/s, loss=0.29, metrics={'acc': 0.865}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 113.53it/s, loss=0.282, metrics={'acc': 0.8681}]\nepoch 18: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 71.22it/s, loss=0.288, metrics={'acc': 0.8651}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 112.89it/s, loss=0.288, metrics={'acc': 0.8676}]\nepoch 19: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 75.85it/s, loss=0.29, metrics={'acc': 0.8661}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 114.26it/s, loss=0.284, metrics={'acc': 0.8662}]\nepoch 20: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 75.80it/s, loss=0.289, metrics={'acc': 0.8661}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 119.44it/s, loss=0.281, metrics={'acc': 0.8703}]\nepoch 21: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 75.72it/s, loss=0.29, metrics={'acc': 0.8661}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 113.04it/s, loss=0.285, metrics={'acc': 0.8648}]\nepoch 22: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 74.86it/s, loss=0.289, metrics={'acc': 0.8656}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 111.75it/s, loss=0.282, metrics={'acc': 0.8666}]\nepoch 23: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 76.68it/s, loss=0.289, metrics={'acc': 0.8668}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 115.86it/s, loss=0.282, metrics={'acc': 0.8724}]\nepoch 24: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 75.93it/s, loss=0.288, metrics={'acc': 0.8653}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 114.69it/s, loss=0.285, metrics={'acc': 0.8656}]\nepoch 25: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 79.99it/s, loss=0.284, metrics={'acc': 0.8671}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 118.27it/s, loss=0.277, metrics={'acc': 0.8707}]\nepoch 26: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 79.86it/s, loss=0.282, metrics={'acc': 0.8686}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 123.94it/s, loss=0.276, metrics={'acc': 0.8712}]\nepoch 27: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 77.86it/s, loss=0.283, metrics={'acc': 0.8691}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 114.11it/s, loss=0.277, metrics={'acc': 0.8716}]\nepoch 28: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 77.43it/s, loss=0.281, metrics={'acc': 0.8696}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 114.64it/s, loss=0.277, metrics={'acc': 0.8712}]\nepoch 29: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 77.39it/s, loss=0.281, metrics={'acc': 0.8696}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 115.83it/s, loss=0.277, metrics={'acc': 0.872}]\nepoch 30: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 74.93it/s, loss=0.28, metrics={'acc': 0.8706}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 113.07it/s, loss=0.275, metrics={'acc': 0.8714}]\nepoch 31: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 77.35it/s, loss=0.281, metrics={'acc': 0.8697}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 112.68it/s, loss=0.276, metrics={'acc': 0.872}]\nepoch 32: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 76.72it/s, loss=0.28, metrics={'acc': 0.8693}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 122.50it/s, loss=0.276, metrics={'acc': 0.8709}]\nepoch 33: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 76.68it/s, loss=0.28, metrics={'acc': 0.8716}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 110.07it/s, loss=0.277, metrics={'acc': 0.8709}]\nepoch 34: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 73.58it/s, loss=0.279, metrics={'acc': 0.8704}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 111.07it/s, loss=0.274, metrics={'acc': 0.8719}]\nepoch 35: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 74.03it/s, loss=0.28, metrics={'acc': 0.8687}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 110.15it/s, loss=0.276, metrics={'acc': 0.871}]\nepoch 36: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 73.11it/s, loss=0.279, metrics={'acc': 0.8706}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 111.10it/s, loss=0.278, metrics={'acc': 0.8705}]\nepoch 37: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 74.30it/s, loss=0.279, metrics={'acc': 0.869}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 110.75it/s, loss=0.279, metrics={'acc': 0.8702}]\nepoch 38: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 72.34it/s, loss=0.28, metrics={'acc': 0.8691}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 117.79it/s, loss=0.277, metrics={'acc': 0.8698}]\nepoch 39: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 75.49it/s, loss=0.279, metrics={'acc': 0.8694}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 115.25it/s, loss=0.279, metrics={'acc': 0.87}]\nepoch 40: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 75.51it/s, loss=0.28, metrics={'acc': 0.8694}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 110.90it/s, loss=0.277, metrics={'acc': 0.8694}]\nepoch 41: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 75.35it/s, loss=0.278, metrics={'acc': 0.8716}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 115.74it/s, loss=0.28, metrics={'acc': 0.8675}]\nepoch 42: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 76.35it/s, loss=0.279, metrics={'acc': 0.8695}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 122.76it/s, loss=0.277, metrics={'acc': 0.8699}]\nepoch 43: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:04&lt;00:00, 66.14it/s, loss=0.279, metrics={'acc': 0.8681}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 106.20it/s, loss=0.277, metrics={'acc': 0.8714}]\nepoch 44: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 73.37it/s, loss=0.279, metrics={'acc': 0.8704}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 115.18it/s, loss=0.277, metrics={'acc': 0.8716}]\nepoch 45: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 76.23it/s, loss=0.278, metrics={'acc': 0.8702}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 124.83it/s, loss=0.278, metrics={'acc': 0.8707}]\nepoch 46: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 77.12it/s, loss=0.278, metrics={'acc': 0.8704}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 126.62it/s, loss=0.279, metrics={'acc': 0.8693}]\nepoch 47: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 78.55it/s, loss=0.276, metrics={'acc': 0.8713}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 117.99it/s, loss=0.279, metrics={'acc': 0.8691}]\nepoch 48: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 77.25it/s, loss=0.278, metrics={'acc': 0.8719}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 124.52it/s, loss=0.278, metrics={'acc': 0.8695}]\nepoch 49: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 78.35it/s, loss=0.277, metrics={'acc': 0.8721}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 119.82it/s, loss=0.279, metrics={'acc': 0.8691}]\nepoch 50: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 77.15it/s, loss=0.277, metrics={'acc': 0.8717}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 122.62it/s, loss=0.278, metrics={'acc': 0.8699}]\nepoch 51: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 78.55it/s, loss=0.277, metrics={'acc': 0.8713}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 117.63it/s, loss=0.278, metrics={'acc': 0.87}]\nepoch 52: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 77.04it/s, loss=0.276, metrics={'acc': 0.8721}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 126.39it/s, loss=0.278, metrics={'acc': 0.8697}]\nepoch 53: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 78.15it/s, loss=0.277, metrics={'acc': 0.8721}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 127.56it/s, loss=0.278, metrics={'acc': 0.8699}]\nepoch 54: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 77.41it/s, loss=0.277, metrics={'acc': 0.8711}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 118.95it/s, loss=0.278, metrics={'acc': 0.8698}]\nepoch 55: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 76.35it/s, loss=0.277, metrics={'acc': 0.8718}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 126.90it/s, loss=0.278, metrics={'acc': 0.8699}]\nepoch 56: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 77.83it/s, loss=0.277, metrics={'acc': 0.8707}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 115.13it/s, loss=0.279, metrics={'acc': 0.8691}]\nepoch 57: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 75.20it/s, loss=0.277, metrics={'acc': 0.8722}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 123.16it/s, loss=0.279, metrics={'acc': 0.8691}]\nepoch 58: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 73.33it/s, loss=0.276, metrics={'acc': 0.871}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 123.37it/s, loss=0.278, metrics={'acc': 0.8691}]\nepoch 59: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 78.41it/s, loss=0.277, metrics={'acc': 0.8714}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 125.17it/s, loss=0.278, metrics={'acc': 0.8695}]\nepoch 60: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 79.26it/s, loss=0.276, metrics={'acc': 0.8721}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 120.60it/s, loss=0.278, metrics={'acc': 0.869}]\nepoch 61: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 75.88it/s, loss=0.278, metrics={'acc': 0.8703}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 124.47it/s, loss=0.278, metrics={'acc': 0.8692}]\nepoch 62: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 77.13it/s, loss=0.276, metrics={'acc': 0.8711}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 125.80it/s, loss=0.278, metrics={'acc': 0.8691}]\nepoch 63: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 77.20it/s, loss=0.277, metrics={'acc': 0.8715}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 118.50it/s, loss=0.278, metrics={'acc': 0.8695}]\nepoch 64: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 78.11it/s, loss=0.276, metrics={'acc': 0.8719}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 114.52it/s, loss=0.278, metrics={'acc': 0.869}]\n</pre> <pre>Best Epoch: 34. Best val_loss: 0.27449\nRestoring model weights from the end of the best epoch\n</pre> In\u00a0[32]: Copied! <pre>mlp_trainer.feature_importance\n</pre> mlp_trainer.feature_importance Out[32]: <pre>{'age': 0.116632804,\n 'workclass': 0.050255153,\n 'education': 0.094621316,\n 'marital_status': 0.12328919,\n 'occupation': 0.107893184,\n 'relationship': 0.11747801,\n 'race': 0.054717205,\n 'gender': 0.07514235,\n 'capital_gain': 0.059732802,\n 'capital_loss': 0.06738944,\n 'hours_per_week': 0.0610674,\n 'native_country': 0.07178114}</pre> In\u00a0[33]: Copied! <pre>mlp_preds = mlp_trainer.predict(X_tab=X_tab_test)\n</pre> mlp_preds = mlp_trainer.predict(X_tab=X_tab_test) <pre>predict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 39/39 [00:00&lt;00:00, 212.38it/s]\n</pre> In\u00a0[34]: Copied! <pre>accuracy_score(mlp_preds, test.income_label)\n</pre> accuracy_score(mlp_preds, test.income_label) Out[34]: <pre>0.8726714431934494</pre>"},{"location":"examples/17_feature_importance_via_attention_weights.html#feature-importance-via-the-attention-weights","title":"Feature Importance via the attention weights\u00b6","text":"<p>I will start by saying that I consider this feature of the library purely experimental. First of all I think there are multiple ways one could address finding the features importances for these models. However, and more importantly, one has to bear in mind that even tree-based algorithms on the same dataset produce different feature importances. This is more \"dramatic\" if one uses different techniques, such as shap or feature permutation (see for example this and references therein). All this to say that, sometimes, feature importance is just a measure contained within the experiment run, and for the model used.</p> <p>With that in mind, each instantiation of a deep tabular model, that has millions of trainable parameters, will potentially produce a different set of feature importances, even if the model has the same architecture. Moreover, this effect will become more apparent if the dataset is relatively easy and there are dependent/related columns so that one could get to the same success metric with different parameters.</p> <p>In summary, feature importances are implemented in this librray for all attention-based models for tabular data, with the exception of the <code>TabPerceiver</code>. However this functionality has to be used and interpreted with care and consider of value within the 'universe' (or context) of the model with which these features were produced.</p> <p>Nonetheless, let's have a look to how one would access to the feature importances when using this library.</p>"},{"location":"examples/18_wide_and_deep_for_recsys_pt1.html","title":"18_wide_and_deep_for_recsys_pt1","text":"<p>The goal of this, and the companion (part 2) notebooks is to illustrate how one could use this library in the context of recommendation systems. In particular, this notebook and the scripts at the <code>wide_deep_for_recsys</code> dir are a response to this issue. Therefore, we will use the Kaggle notebook referred in that issue here.</p> <p>In order to keep the length of the notebook tractable, we will split this exercise in 2. In this first notebook we will prepare the data in almost the exact same way as it is done in the Kaggle notebook and also show how one could use <code>pytorch-widedeep</code> to build a model almost identical to the one in that notebook.</p> <p>In a second notebook, we will show how one could use this library to implement other models, still following the same problem formulation.</p> In\u00a0[1]: Copied! <pre>from pathlib import Path\nimport warnings\n\nimport pandas as pd\nfrom sklearn.model_selection import train_test_split\n\nfrom pytorch_widedeep.datasets import load_movielens100k\n</pre> from pathlib import Path import warnings  import pandas as pd from sklearn.model_selection import train_test_split  from pytorch_widedeep.datasets import load_movielens100k In\u00a0[2]: Copied! <pre>warnings.filterwarnings(\"ignore\")\n</pre> warnings.filterwarnings(\"ignore\") In\u00a0[3]: Copied! <pre>save_path = Path(\"prepared_data\")\nif not save_path.exists():\n    save_path.mkdir(parents=True, exist_ok=True)\n</pre> save_path = Path(\"prepared_data\") if not save_path.exists():     save_path.mkdir(parents=True, exist_ok=True) In\u00a0[4]: Copied! <pre>data, users, items = load_movielens100k(as_frame=True)\n</pre> data, users, items = load_movielens100k(as_frame=True) In\u00a0[5]: Copied! <pre># Alternatively, as specified in the docs: 'The last 19 fields are the genres' so:\n# list_of_genres = items.columns.tolist()[-19:]\nlist_of_genres = [\n    \"unknown\",\n    \"Action\",\n    \"Adventure\",\n    \"Animation\",\n    \"Children's\",\n    \"Comedy\",\n    \"Crime\",\n    \"Documentary\",\n    \"Drama\",\n    \"Fantasy\",\n    \"Film-Noir\",\n    \"Horror\",\n    \"Musical\",\n    \"Mystery\",\n    \"Romance\",\n    \"Sci-Fi\",\n    \"Thriller\",\n    \"War\",\n    \"Western\",\n]\n</pre> # Alternatively, as specified in the docs: 'The last 19 fields are the genres' so: # list_of_genres = items.columns.tolist()[-19:] list_of_genres = [     \"unknown\",     \"Action\",     \"Adventure\",     \"Animation\",     \"Children's\",     \"Comedy\",     \"Crime\",     \"Documentary\",     \"Drama\",     \"Fantasy\",     \"Film-Noir\",     \"Horror\",     \"Musical\",     \"Mystery\",     \"Romance\",     \"Sci-Fi\",     \"Thriller\",     \"War\",     \"Western\", ] <p>Let's first start by loading the interactions, user and item data</p> In\u00a0[6]: Copied! <pre>data.head()\n</pre> data.head() Out[6]: user_id movie_id rating timestamp 0 196 242 3 881250949 1 186 302 3 891717742 2 22 377 1 878887116 3 244 51 2 880606923 4 166 346 1 886397596 In\u00a0[7]: Copied! <pre>users.head()\n</pre> users.head() Out[7]: user_id age gender occupation zip_code 0 1 24 M technician 85711 1 2 53 F other 94043 2 3 23 M writer 32067 3 4 24 M technician 43537 4 5 33 F other 15213 In\u00a0[8]: Copied! <pre>items.head()\n</pre> items.head() Out[8]: movie_id movie_title release_date video_release_date IMDb_URL unknown Action Adventure Animation Children's ... Fantasy Film-Noir Horror Musical Mystery Romance Sci-Fi Thriller War Western 0 1 Toy Story (1995) 01-Jan-1995 NaN http://us.imdb.com/M/title-exact?Toy%20Story%2... 0 0 0 1 1 ... 0 0 0 0 0 0 0 0 0 0 1 2 GoldenEye (1995) 01-Jan-1995 NaN http://us.imdb.com/M/title-exact?GoldenEye%20(... 0 1 1 0 0 ... 0 0 0 0 0 0 0 1 0 0 2 3 Four Rooms (1995) 01-Jan-1995 NaN http://us.imdb.com/M/title-exact?Four%20Rooms%... 0 0 0 0 0 ... 0 0 0 0 0 0 0 1 0 0 3 4 Get Shorty (1995) 01-Jan-1995 NaN http://us.imdb.com/M/title-exact?Get%20Shorty%... 0 1 0 0 0 ... 0 0 0 0 0 0 0 0 0 0 4 5 Copycat (1995) 01-Jan-1995 NaN http://us.imdb.com/M/title-exact?Copycat%20(1995) 0 0 0 0 0 ... 0 0 0 0 0 0 0 1 0 0 <p>5 rows \u00d7 24 columns</p> In\u00a0[9]: Copied! <pre># adding a column with the number of movies watched per user\ndataset = data.sort_values([\"user_id\", \"timestamp\"]).reset_index(drop=True)\ndataset[\"one\"] = 1\ndataset[\"num_watched\"] = dataset.groupby(\"user_id\")[\"one\"].cumsum()\ndataset.drop(\"one\", axis=1, inplace=True)\ndataset.head()\n</pre> # adding a column with the number of movies watched per user dataset = data.sort_values([\"user_id\", \"timestamp\"]).reset_index(drop=True) dataset[\"one\"] = 1 dataset[\"num_watched\"] = dataset.groupby(\"user_id\")[\"one\"].cumsum() dataset.drop(\"one\", axis=1, inplace=True) dataset.head() Out[9]: user_id movie_id rating timestamp num_watched 0 1 168 5 874965478 1 1 1 172 5 874965478 2 2 1 165 5 874965518 3 3 1 156 4 874965556 4 4 1 196 5 874965677 5 In\u00a0[10]: Copied! <pre># adding a column with the mean rating at a point in time per user\ndataset[\"mean_rate\"] = (\n    dataset.groupby(\"user_id\")[\"rating\"].cumsum() / dataset[\"num_watched\"]\n)\ndataset.head()\n</pre> # adding a column with the mean rating at a point in time per user dataset[\"mean_rate\"] = (     dataset.groupby(\"user_id\")[\"rating\"].cumsum() / dataset[\"num_watched\"] ) dataset.head() Out[10]: user_id movie_id rating timestamp num_watched mean_rate 0 1 168 5 874965478 1 5.00 1 1 172 5 874965478 2 5.00 2 1 165 5 874965518 3 5.00 3 1 156 4 874965556 4 4.75 4 1 196 5 874965677 5 4.80 In\u00a0[11]: Copied! <pre>dataset[\"target\"] = dataset.groupby(\"user_id\")[\"movie_id\"].shift(-1)\n</pre> dataset[\"target\"] = dataset.groupby(\"user_id\")[\"movie_id\"].shift(-1) <p>Following the same processing used by the author in the before-mentioned Kaggle notebook, we build sequences of previous movies watched</p> In\u00a0[12]: Copied! <pre># Here the author builds the sequences\ndataset[\"prev_movies\"] = dataset[\"movie_id\"].apply(lambda x: str(x))\ndataset[\"prev_movies\"] = (\n    dataset.groupby(\"user_id\")[\"prev_movies\"]\n    .apply(lambda x: (x + \" \").cumsum().str.strip())\n    .reset_index(drop=True)\n)\ndataset[\"prev_movies\"] = dataset[\"prev_movies\"].apply(lambda x: x.split())\ndataset.head()\n</pre> # Here the author builds the sequences dataset[\"prev_movies\"] = dataset[\"movie_id\"].apply(lambda x: str(x)) dataset[\"prev_movies\"] = (     dataset.groupby(\"user_id\")[\"prev_movies\"]     .apply(lambda x: (x + \" \").cumsum().str.strip())     .reset_index(drop=True) ) dataset[\"prev_movies\"] = dataset[\"prev_movies\"].apply(lambda x: x.split()) dataset.head() Out[12]: user_id movie_id rating timestamp num_watched mean_rate target prev_movies 0 1 168 5 874965478 1 5.00 172.0 [168] 1 1 172 5 874965478 2 5.00 165.0 [168, 172] 2 1 165 5 874965518 3 5.00 156.0 [168, 172, 165] 3 1 156 4 874965556 4 4.75 196.0 [168, 172, 165, 156] 4 1 196 5 874965677 5 4.80 166.0 [168, 172, 165, 156, 196] <p>And now we add a <code>genre_rate</code> as the mean of all movies rated for a given genre per user</p> In\u00a0[13]: Copied! <pre>dataset = dataset.merge(items[[\"movie_id\"] + list_of_genres], on=\"movie_id\", how=\"left\")\nfor genre in list_of_genres:\n    dataset[f\"{genre}_rate\"] = dataset[genre] * dataset[\"rating\"]\n    dataset[genre] = dataset.groupby(\"user_id\")[genre].cumsum()\n    dataset[f\"{genre}_rate\"] = (\n        dataset.groupby(\"user_id\")[f\"{genre}_rate\"].cumsum() / dataset[genre]\n    )\ndataset[list_of_genres] = dataset[list_of_genres].apply(\n    lambda x: x / dataset[\"num_watched\"]\n)\ndataset.head()\n</pre> dataset = dataset.merge(items[[\"movie_id\"] + list_of_genres], on=\"movie_id\", how=\"left\") for genre in list_of_genres:     dataset[f\"{genre}_rate\"] = dataset[genre] * dataset[\"rating\"]     dataset[genre] = dataset.groupby(\"user_id\")[genre].cumsum()     dataset[f\"{genre}_rate\"] = (         dataset.groupby(\"user_id\")[f\"{genre}_rate\"].cumsum() / dataset[genre]     ) dataset[list_of_genres] = dataset[list_of_genres].apply(     lambda x: x / dataset[\"num_watched\"] ) dataset.head() Out[13]: user_id movie_id rating timestamp num_watched mean_rate target prev_movies unknown Action ... Fantasy_rate Film-Noir_rate Horror_rate Musical_rate Mystery_rate Romance_rate Sci-Fi_rate Thriller_rate War_rate Western_rate 0 1 168 5 874965478 1 5.00 172.0 [168] 0.0 0.000000 ... NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN 1 1 172 5 874965478 2 5.00 165.0 [168, 172] 0.0 0.500000 ... NaN NaN NaN NaN NaN 5.0 5.0 NaN 5.0 NaN 2 1 165 5 874965518 3 5.00 156.0 [168, 172, 165] 0.0 0.333333 ... NaN NaN NaN NaN NaN 5.0 5.0 NaN 5.0 NaN 3 1 156 4 874965556 4 4.75 196.0 [168, 172, 165, 156] 0.0 0.250000 ... NaN NaN NaN NaN NaN 5.0 5.0 4.0 5.0 NaN 4 1 196 5 874965677 5 4.80 166.0 [168, 172, 165, 156, 196] 0.0 0.200000 ... NaN NaN NaN NaN NaN 5.0 5.0 4.0 5.0 NaN <p>5 rows \u00d7 46 columns</p> <p>Adding user features</p> In\u00a0[14]: Copied! <pre>dataset = dataset.merge(users, on=\"user_id\", how=\"left\")\ndataset.head()\n</pre> dataset = dataset.merge(users, on=\"user_id\", how=\"left\") dataset.head() Out[14]: user_id movie_id rating timestamp num_watched mean_rate target prev_movies unknown Action ... Mystery_rate Romance_rate Sci-Fi_rate Thriller_rate War_rate Western_rate age gender occupation zip_code 0 1 168 5 874965478 1 5.00 172.0 [168] 0.0 0.000000 ... NaN NaN NaN NaN NaN NaN 24 M technician 85711 1 1 172 5 874965478 2 5.00 165.0 [168, 172] 0.0 0.500000 ... NaN 5.0 5.0 NaN 5.0 NaN 24 M technician 85711 2 1 165 5 874965518 3 5.00 156.0 [168, 172, 165] 0.0 0.333333 ... NaN 5.0 5.0 NaN 5.0 NaN 24 M technician 85711 3 1 156 4 874965556 4 4.75 196.0 [168, 172, 165, 156] 0.0 0.250000 ... NaN 5.0 5.0 4.0 5.0 NaN 24 M technician 85711 4 1 196 5 874965677 5 4.80 166.0 [168, 172, 165, 156, 196] 0.0 0.200000 ... NaN 5.0 5.0 4.0 5.0 NaN 24 M technician 85711 <p>5 rows \u00d7 50 columns</p> <p>Again, we use the same settings as those in the Kaggle notebook, but <code>COLD_START_TRESH</code> is pretty aggressive</p> In\u00a0[15]: Copied! <pre>COLD_START_TRESH = 5\n\nfiltred_data = dataset[\n    (dataset[\"num_watched\"] &gt;= COLD_START_TRESH) &amp; ~(dataset[\"target\"].isna())\n].sort_values(\"timestamp\")\ntrain_data, _test_data = train_test_split(filtred_data, test_size=0.2, shuffle=False)\nvalid_data, test_data = train_test_split(_test_data, test_size=0.5, shuffle=False)\n</pre> COLD_START_TRESH = 5  filtred_data = dataset[     (dataset[\"num_watched\"] &gt;= COLD_START_TRESH) &amp; ~(dataset[\"target\"].isna()) ].sort_values(\"timestamp\") train_data, _test_data = train_test_split(filtred_data, test_size=0.2, shuffle=False) valid_data, test_data = train_test_split(_test_data, test_size=0.5, shuffle=False) In\u00a0[16]: Copied! <pre>cols_to_drop = [\n    # \"rating\",\n    \"timestamp\",\n    \"num_watched\",\n]\n\ndf_train = train_data.drop(cols_to_drop, axis=1)\ndf_valid = valid_data.drop(cols_to_drop, axis=1)\ndf_test = test_data.drop(cols_to_drop, axis=1)\n\ndf_train.to_pickle(save_path / \"df_train.pkl\")\ndf_valid.to_pickle(save_path / \"df_valid.pkl\")\ndf_test.to_pickle(save_path / \"df_test.pkl\")\n</pre> cols_to_drop = [     # \"rating\",     \"timestamp\",     \"num_watched\", ]  df_train = train_data.drop(cols_to_drop, axis=1) df_valid = valid_data.drop(cols_to_drop, axis=1) df_test = test_data.drop(cols_to_drop, axis=1)  df_train.to_pickle(save_path / \"df_train.pkl\") df_valid.to_pickle(save_path / \"df_valid.pkl\") df_test.to_pickle(save_path / \"df_test.pkl\") <p>Let's now build a model that is nearly identical to the one use in the Kaggle notebook</p> In\u00a0[17]: Copied! <pre>import numpy as np\nimport torch\nfrom torch import nn\nfrom scipy.sparse import coo_matrix\n\nfrom pytorch_widedeep import Trainer\nfrom pytorch_widedeep.models import TabMlp, BasicRNN, WideDeep\nfrom pytorch_widedeep.preprocessing import TabPreprocessor\n</pre> import numpy as np import torch from torch import nn from scipy.sparse import coo_matrix  from pytorch_widedeep import Trainer from pytorch_widedeep.models import TabMlp, BasicRNN, WideDeep from pytorch_widedeep.preprocessing import TabPreprocessor In\u00a0[18]: Copied! <pre>device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n\nsave_path = Path(\"prepared_data\")\n\nPAD_IDX = 0\n</pre> device = \"cuda\" if torch.cuda.is_available() else \"cpu\"  save_path = Path(\"prepared_data\")  PAD_IDX = 0 <p>Let's use some of the functions the author of the kaggle's notebook uses to prepare the data</p> In\u00a0[19]: Copied! <pre>def get_coo_indexes(lil):\n    rows = []\n    cols = []\n    for i, el in enumerate(lil):\n        if type(el) != list:\n            el = [el]\n        for j in el:\n            rows.append(i)\n            cols.append(j)\n    return rows, cols\n\n\ndef get_sparse_features(series, shape):\n    coo_indexes = get_coo_indexes(series.tolist())\n    sparse_df = coo_matrix(\n        (np.ones(len(coo_indexes[0])), (coo_indexes[0], coo_indexes[1])), shape=shape\n    )\n    return sparse_df\n\n\ndef sparse_to_idx(data, pad_idx=-1):\n    indexes = data.nonzero()\n    indexes_df = pd.DataFrame()\n    indexes_df[\"rows\"] = indexes[0]\n    indexes_df[\"cols\"] = indexes[1]\n    mdf = indexes_df.groupby(\"rows\").apply(lambda x: x[\"cols\"].tolist())\n    max_len = mdf.apply(lambda x: len(x)).max()\n    return mdf.apply(lambda x: pd.Series(x + [pad_idx] * (max_len - len(x)))).values\n</pre> def get_coo_indexes(lil):     rows = []     cols = []     for i, el in enumerate(lil):         if type(el) != list:             el = [el]         for j in el:             rows.append(i)             cols.append(j)     return rows, cols   def get_sparse_features(series, shape):     coo_indexes = get_coo_indexes(series.tolist())     sparse_df = coo_matrix(         (np.ones(len(coo_indexes[0])), (coo_indexes[0], coo_indexes[1])), shape=shape     )     return sparse_df   def sparse_to_idx(data, pad_idx=-1):     indexes = data.nonzero()     indexes_df = pd.DataFrame()     indexes_df[\"rows\"] = indexes[0]     indexes_df[\"cols\"] = indexes[1]     mdf = indexes_df.groupby(\"rows\").apply(lambda x: x[\"cols\"].tolist())     max_len = mdf.apply(lambda x: len(x)).max()     return mdf.apply(lambda x: pd.Series(x + [pad_idx] * (max_len - len(x)))).values <p>For the time being, we will not use a validation set for hyperparameter optimization, and we will simply concatenate the validation and the test set in one test set. I simply splitted the data into train/valid/test in case the reader wants to actually do hyperparameter optimization (and because I know in the future I will).</p> <p>There is also another caveat worth mentioning, related to the indexing of the movies. To build the matrices of movies watched, we use the entire dataset. A more realistic (and correct) approach would be to use ONLY the movies that appear in the training set and consider <code>unknown</code> or <code>unseen</code> those in the testing set that have not been seen during training. Nonetheless, this will not affect the purposes of this notebook, which is to illustrate how one could use <code>pytorch-widedeep</code> to build a recommendation algorithm. However, if one wanted to explore the performance of different algorithms in a \"proper\" way, these \"details\" need to be accounted for.</p> In\u00a0[20]: Copied! <pre>df_test = pd.concat([df_valid, df_test], ignore_index=True)\n</pre> df_test = pd.concat([df_valid, df_test], ignore_index=True) In\u00a0[21]: Copied! <pre>id_cols = [\"user_id\", \"movie_id\"]\nmax_movie_index = max(df_train.movie_id.max(), df_test.movie_id.max())\n</pre> id_cols = [\"user_id\", \"movie_id\"] max_movie_index = max(df_train.movie_id.max(), df_test.movie_id.max()) In\u00a0[22]: Copied! <pre>X_train = df_train.drop(id_cols + [\"rating\", \"prev_movies\", \"target\"], axis=1)\ny_train = np.array(df_train.target.values, dtype=\"int64\")\ntrain_movies_watched = get_sparse_features(\n    df_train[\"prev_movies\"], (len(df_train), max_movie_index + 1)\n)\n\nX_test = df_test.drop(id_cols + [\"rating\", \"prev_movies\", \"target\"], axis=1)\ny_test = np.array(df_test.target.values, dtype=\"int64\")\ntest_movies_watched = get_sparse_features(\n    df_test[\"prev_movies\"], (len(df_test), max_movie_index + 1)\n)\n</pre> X_train = df_train.drop(id_cols + [\"rating\", \"prev_movies\", \"target\"], axis=1) y_train = np.array(df_train.target.values, dtype=\"int64\") train_movies_watched = get_sparse_features(     df_train[\"prev_movies\"], (len(df_train), max_movie_index + 1) )  X_test = df_test.drop(id_cols + [\"rating\", \"prev_movies\", \"target\"], axis=1) y_test = np.array(df_test.target.values, dtype=\"int64\") test_movies_watched = get_sparse_features(     df_test[\"prev_movies\"], (len(df_test), max_movie_index + 1) ) <p>let's have a look to the information in each dataset</p> In\u00a0[23]: Copied! <pre>X_train.head()\n</pre> X_train.head() Out[23]: mean_rate unknown Action Adventure Animation Children's Comedy Crime Documentary Drama ... Mystery_rate Romance_rate Sci-Fi_rate Thriller_rate War_rate Western_rate age gender occupation zip_code 25423 4.000000 0.0 0.400000 0.200000 0.0 0.0 0.400000 0.0 0.0 0.200000 ... NaN 4.0 4.0 4.000000 4.0 NaN 21 M student 48823 25425 4.000000 0.0 0.285714 0.142857 0.0 0.0 0.428571 0.0 0.0 0.285714 ... NaN 4.0 4.0 4.000000 4.0 NaN 21 M student 48823 25424 4.000000 0.0 0.333333 0.166667 0.0 0.0 0.333333 0.0 0.0 0.333333 ... NaN 4.0 4.0 4.000000 4.0 NaN 21 M student 48823 25426 3.875000 0.0 0.250000 0.125000 0.0 0.0 0.375000 0.0 0.0 0.250000 ... NaN 4.0 4.0 3.666667 4.0 NaN 21 M student 48823 25427 3.888889 0.0 0.222222 0.111111 0.0 0.0 0.333333 0.0 0.0 0.333333 ... NaN 4.0 4.0 3.666667 4.0 NaN 21 M student 48823 <p>5 rows \u00d7 43 columns</p> In\u00a0[24]: Copied! <pre>y_train\n</pre> y_train Out[24]: <pre>array([772, 288, 108, ..., 183, 432, 509])</pre> In\u00a0[25]: Copied! <pre>train_movies_watched\n</pre> train_movies_watched Out[25]: <pre>&lt;76228x1683 sparse matrix of type '&lt;class 'numpy.float64'&gt;'\n\twith 7957390 stored elements in COOrdinate format&gt;</pre> In\u00a0[26]: Copied! <pre>sorted(df_train.prev_movies.tolist()[0])\n</pre> sorted(df_train.prev_movies.tolist()[0]) Out[26]: <pre>['173', '185', '255', '286', '298']</pre> In\u00a0[27]: Copied! <pre>np.where(train_movies_watched.todense()[0])\n</pre> np.where(train_movies_watched.todense()[0]) Out[27]: <pre>(array([0, 0, 0, 0, 0]), array([173, 185, 255, 286, 298]))</pre> <p>And from now on is when the specifics related to this library start to appear. The only component that is going to be a bit different is the so-called tabular component, referred as <code>continuous</code> in the notebook.</p> <p>In the case of <code>pytorch-widedeep</code> we have the <code>TabPreprocessor</code> that allows for a lot of flexibility as to how we would like to process the tabular component of this Wide and Deep model. In other words, here our tabular component is a bit more elaborated than that in the notebook, just a bit...</p> In\u00a0[28]: Copied! <pre>cat_cols = [\"gender\", \"occupation\", \"zip_code\"]\ncont_cols = [c for c in X_train if c not in cat_cols]\ntab_preprocessor = TabPreprocessor(\n    cat_embed_cols=cat_cols,\n    continuous_cols=cont_cols,\n)\n</pre> cat_cols = [\"gender\", \"occupation\", \"zip_code\"] cont_cols = [c for c in X_train if c not in cat_cols] tab_preprocessor = TabPreprocessor(     cat_embed_cols=cat_cols,     continuous_cols=cont_cols, ) In\u00a0[29]: Copied! <pre>X_train_tab = tab_preprocessor.fit_transform(X_train.fillna(0))\nX_test_tab = tab_preprocessor.transform(X_test.fillna(0))\n</pre> X_train_tab = tab_preprocessor.fit_transform(X_train.fillna(0)) X_test_tab = tab_preprocessor.transform(X_test.fillna(0)) <p>Now, in the notebook, the author moves the sparse matrices to sparse tensors and then turns them into dense tensors. In reality, this is not neccessary, one could feed sparse tensors to <code>nn.Linear</code> layers in pytorch. Nonetheless, this is not the most efficient implementation and is the reason why in our library the wide, linear component is implemented as an embedding layer.</p> <p>Nonetheless, to reproduce the notebook the best we can and because currently the <code>Wide</code> model in <code>pytorch-widedeep</code> is not designed to receive sparse tensors (we might consider implementing this functionality), we will turn the sparse COO matrices into dense arrays. We will then code a fairly simple, custom <code>Wide</code> component.</p> In\u00a0[30]: Copied! <pre>X_train_wide = np.array(train_movies_watched.todense())\nX_test_wide = np.array(test_movies_watched.todense())\n</pre> X_train_wide = np.array(train_movies_watched.todense()) X_test_wide = np.array(test_movies_watched.todense()) <p>Finally, the author of the notebook uses a simple <code>Embedding</code> layer to encode the sequences of movies watched, the <code>prev_movies</code> columns. In my opinion, there is an element of information redundancy here. This is because the wide and text components have implicitely the same information, but in different form. Moreover, both of the models used for these two components ignore the sequential element in the data. Nonetheless, we want to reproduce the Kaggle notebook as close as possible, AND as one can explore later (by simply performing simple ablation studies), the wide component seems to carry most of the predictive power.</p> In\u00a0[31]: Copied! <pre>X_train_text = sparse_to_idx(train_movies_watched, pad_idx=PAD_IDX)\nX_test_text = sparse_to_idx(test_movies_watched, pad_idx=PAD_IDX)\n</pre> X_train_text = sparse_to_idx(train_movies_watched, pad_idx=PAD_IDX) X_test_text = sparse_to_idx(test_movies_watched, pad_idx=PAD_IDX) <p>Let's now build the models</p> In\u00a0[32]: Copied! <pre>class Wide(nn.Module):\n    def __init__(self, input_dim: int, pred_dim: int):\n        super().__init__()\n\n        self.input_dim = input_dim\n        self.pred_dim = pred_dim\n\n        # When I coded the library I never though that someone would want to code\n        # their own wide component. However, if you do, the wide component must have\n        # a 'wide_linear' attribute. In other words, the linear layer must be\n        # called 'wide_linear'\n        self.wide_linear = nn.Linear(input_dim, pred_dim)\n\n    def forward(self, X):\n        out = self.wide_linear(X.type(torch.float32))\n        return out\n\n\nwide = Wide(X_train_wide.shape[1], max_movie_index + 1)\n</pre> class Wide(nn.Module):     def __init__(self, input_dim: int, pred_dim: int):         super().__init__()          self.input_dim = input_dim         self.pred_dim = pred_dim          # When I coded the library I never though that someone would want to code         # their own wide component. However, if you do, the wide component must have         # a 'wide_linear' attribute. In other words, the linear layer must be         # called 'wide_linear'         self.wide_linear = nn.Linear(input_dim, pred_dim)      def forward(self, X):         out = self.wide_linear(X.type(torch.float32))         return out   wide = Wide(X_train_wide.shape[1], max_movie_index + 1) In\u00a0[33]: Copied! <pre>wide\n</pre> wide Out[33]: <pre>Wide(\n  (wide_linear): Linear(in_features=1683, out_features=1683, bias=True)\n)</pre> In\u00a0[34]: Copied! <pre>class SimpleEmbed(nn.Module):\n    def __init__(self, vocab_size: int, embed_dim: int, pad_idx: int):\n        super().__init__()\n\n        self.vocab_size = vocab_size\n        self.embed_dim = embed_dim\n        self.pad_idx = pad_idx\n\n        # The sequences of movies watched are simply embedded in the Kaggle\n        # notebook. No RNN, Transformer or any model is used\n        self.embed = nn.Embedding(vocab_size, embed_dim, padding_idx=pad_idx)\n\n    def forward(self, X):\n        embed = self.embed(X)\n        embed_mean = torch.mean(embed, dim=1)\n        return embed_mean\n\n    @property\n    def output_dim(self) -&gt; int:\n        # All deep components in a custom 'pytorch-widedeep' model must have\n        # an output_dim property\n        return self.embed_dim\n\n\n#  In the notebook the author uses simply embeddings\nsimple_embed = SimpleEmbed(max_movie_index + 1, 16, 0)\n</pre> class SimpleEmbed(nn.Module):     def __init__(self, vocab_size: int, embed_dim: int, pad_idx: int):         super().__init__()          self.vocab_size = vocab_size         self.embed_dim = embed_dim         self.pad_idx = pad_idx          # The sequences of movies watched are simply embedded in the Kaggle         # notebook. No RNN, Transformer or any model is used         self.embed = nn.Embedding(vocab_size, embed_dim, padding_idx=pad_idx)      def forward(self, X):         embed = self.embed(X)         embed_mean = torch.mean(embed, dim=1)         return embed_mean      @property     def output_dim(self) -&gt; int:         # All deep components in a custom 'pytorch-widedeep' model must have         # an output_dim property         return self.embed_dim   #  In the notebook the author uses simply embeddings simple_embed = SimpleEmbed(max_movie_index + 1, 16, 0) In\u00a0[35]: Copied! <pre>simple_embed\n</pre> simple_embed Out[35]: <pre>SimpleEmbed(\n  (embed): Embedding(1683, 16, padding_idx=0)\n)</pre> <p>Maybe one would like to use an RNN to account for the sequence nature of the problem. If that was the case it would be as easy as:</p> In\u00a0[36]: Copied! <pre>basic_rnn = BasicRNN(\n    vocab_size=max_movie_index + 1,\n    embed_dim=16,\n    hidden_dim=32,\n    n_layers=2,\n    rnn_type=\"gru\",\n)\n</pre> basic_rnn = BasicRNN(     vocab_size=max_movie_index + 1,     embed_dim=16,     hidden_dim=32,     n_layers=2,     rnn_type=\"gru\", ) <p>And finally, the tabular component, which is the notebook is simply a stak of linear + Rely layers. In our case we have an embedding layer before the linear layers to encode categorial and numerical cols</p> In\u00a0[37]: Copied! <pre>tab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=tab_preprocessor.continuous_cols,\n    cont_norm_layer=None,\n    mlp_hidden_dims=[1024, 512, 256],\n    mlp_activation=\"relu\",\n)\n</pre> tab_mlp = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     continuous_cols=tab_preprocessor.continuous_cols,     cont_norm_layer=None,     mlp_hidden_dims=[1024, 512, 256],     mlp_activation=\"relu\", ) In\u00a0[38]: Copied! <pre>tab_mlp\n</pre> tab_mlp Out[38]: <pre>TabMlp(\n  (cat_embed): DiffSizeCatEmbeddings(\n    (embed_layers): ModuleDict(\n      (emb_layer_gender): Embedding(3, 2, padding_idx=0)\n      (emb_layer_occupation): Embedding(22, 9, padding_idx=0)\n      (emb_layer_zip_code): Embedding(648, 60, padding_idx=0)\n    )\n    (embedding_dropout): Dropout(p=0.0, inplace=False)\n  )\n  (cont_norm): Identity()\n  (encoder): MLP(\n    (mlp): Sequential(\n      (dense_layer_0): Sequential(\n        (0): Linear(in_features=111, out_features=1024, bias=True)\n        (1): ReLU(inplace=True)\n        (2): Dropout(p=0.1, inplace=False)\n      )\n      (dense_layer_1): Sequential(\n        (0): Linear(in_features=1024, out_features=512, bias=True)\n        (1): ReLU(inplace=True)\n        (2): Dropout(p=0.1, inplace=False)\n      )\n      (dense_layer_2): Sequential(\n        (0): Linear(in_features=512, out_features=256, bias=True)\n        (1): ReLU(inplace=True)\n        (2): Dropout(p=0.1, inplace=False)\n      )\n    )\n  )\n)</pre> <p>Finally, we simply wrap up all models with the <code>WideDeep</code> 'collector' class and we are ready to train.</p> In\u00a0[39]: Copied! <pre>wide_deep_model = WideDeep(\n    wide=wide, deeptabular=tab_mlp, deeptext=simple_embed, pred_dim=max_movie_index + 1\n)\n</pre> wide_deep_model = WideDeep(     wide=wide, deeptabular=tab_mlp, deeptext=simple_embed, pred_dim=max_movie_index + 1 ) In\u00a0[40]: Copied! <pre>wide_deep_model\n</pre> wide_deep_model Out[40]: <pre>WideDeep(\n  (wide): Wide(\n    (wide_linear): Linear(in_features=1683, out_features=1683, bias=True)\n  )\n  (deeptabular): Sequential(\n    (0): TabMlp(\n      (cat_embed): DiffSizeCatEmbeddings(\n        (embed_layers): ModuleDict(\n          (emb_layer_gender): Embedding(3, 2, padding_idx=0)\n          (emb_layer_occupation): Embedding(22, 9, padding_idx=0)\n          (emb_layer_zip_code): Embedding(648, 60, padding_idx=0)\n        )\n        (embedding_dropout): Dropout(p=0.0, inplace=False)\n      )\n      (cont_norm): Identity()\n      (encoder): MLP(\n        (mlp): Sequential(\n          (dense_layer_0): Sequential(\n            (0): Linear(in_features=111, out_features=1024, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_1): Sequential(\n            (0): Linear(in_features=1024, out_features=512, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_2): Sequential(\n            (0): Linear(in_features=512, out_features=256, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=256, out_features=1683, bias=True)\n  )\n  (deeptext): Sequential(\n    (0): SimpleEmbed(\n      (embed): Embedding(1683, 16, padding_idx=0)\n    )\n    (1): Linear(in_features=16, out_features=1683, bias=True)\n  )\n)</pre> <p>Note that the main difference between this wide and deep model and the Wide and Deep model in the Kaggle notebook is that in that notebook, the author concatenates the embedings and the tabular features, then passes this concatenation through a stack of linear + Relu layers with a final output dim of 256. Then concatenates this output with the binary features and connects this concatenation with the final linear layer (so the final weights are of dim (batch_size, 256 + 1683)). Our implementation follows the notation of the original paper and instead of concatenating the tabular, text and wide components and then connect them to the output neurons, we first compute their output, and then add it (see here: https://arxiv.org/pdf/1606.07792.pdf, their Eq 3). Note that this is effectively the same, with the caveat that while in one case one initialises a big weight matrix \"at once\", in our implementation we initialise different matrices for different components. Anyway, let's give it a go.</p> In\u00a0[41]: Copied! <pre>trainer = Trainer(\n    model=wide_deep_model,\n    objective=\"multiclass\",\n    custom_loss_function=nn.CrossEntropyLoss(ignore_index=PAD_IDX),\n    optimizers=torch.optim.Adam(wide_deep_model.parameters(), lr=1e-3),\n)\n</pre> trainer = Trainer(     model=wide_deep_model,     objective=\"multiclass\",     custom_loss_function=nn.CrossEntropyLoss(ignore_index=PAD_IDX),     optimizers=torch.optim.Adam(wide_deep_model.parameters(), lr=1e-3), ) In\u00a0[42]: Copied! <pre>trainer.fit(\n    X_train={\n        \"X_wide\": X_train_wide,\n        \"X_tab\": X_train_tab,\n        \"X_text\": X_train_text,\n        \"target\": y_train,\n    },\n    X_val={\n        \"X_wide\": X_test_wide,\n        \"X_tab\": X_test_tab,\n        \"X_text\": X_test_text,\n        \"target\": y_test,\n    },\n    n_epochs=5,\n    batch_size=512,\n    shuffle=False,\n)\n</pre> trainer.fit(     X_train={         \"X_wide\": X_train_wide,         \"X_tab\": X_train_tab,         \"X_text\": X_train_text,         \"target\": y_train,     },     X_val={         \"X_wide\": X_test_wide,         \"X_tab\": X_test_tab,         \"X_text\": X_test_text,         \"target\": y_test,     },     n_epochs=5,     batch_size=512,     shuffle=False, ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 149/149 [00:19&lt;00:00,  7.66it/s, loss=6.66]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 38/38 [00:02&lt;00:00, 18.75it/s, loss=6.6]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 149/149 [00:21&lt;00:00,  6.95it/s, loss=5.97]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 38/38 [00:01&lt;00:00, 21.03it/s, loss=6.52]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 149/149 [00:19&lt;00:00,  7.51it/s, loss=5.65]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 38/38 [00:01&lt;00:00, 20.16it/s, loss=6.53]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 149/149 [00:23&lt;00:00,  6.29it/s, loss=5.41]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 38/38 [00:02&lt;00:00, 13.97it/s, loss=6.57]\nepoch 5: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 149/149 [00:19&lt;00:00,  7.58it/s, loss=5.2]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 38/38 [00:02&lt;00:00, 18.82it/s, loss=6.63]\n</pre> <p>Now one could continue to the 'compare' metrics section of the Kaggle notebook. However, for the purposes of illustrating how one could use <code>pytorch-widedeep</code> to build recommendation algorithms we consider this notebook completed and move onto part 2</p>"},{"location":"examples/18_wide_and_deep_for_recsys_pt1.html#problem-formulation","title":"Problem formulation\u00b6","text":"<p>In this particular exercise the problem is formulated as predicting the next movie that will be watched (in consequence the last interactions will be discarded)</p>"},{"location":"examples/18_wide_and_deep_for_recsys_pt2.html","title":"18_wide_and_deep_for_recsys_pt2","text":"<p>This is the second of the two notebooks where we aim to illustrate how one could use this library to build recommendation algorithms using the example in this Kaggle notebook as guidance. In the previous notebook we used <code>pytorch-widedeep</code> to build a model that replicated almost exactly that in the notebook. In this, shorter notebook we will show how one could use the library to explore other models, following the same problem formulation, this is: given a state of a user at a certain point in time having watched a series of movies, our goal is to predict which movie the user will watch next.</p> <p>Assuming that one has read (and run) the previous notebook, the required data will be stored in a local dir called <code>prepared_data</code>, so let's read it:</p> In\u00a0[1]: Copied! <pre>from pathlib import Path\n\nimport numpy as np\nimport torch\nimport pandas as pd\nfrom torch import nn\n\nfrom pytorch_widedeep import Trainer\nfrom pytorch_widedeep.utils import pad_sequences\nfrom pytorch_widedeep.models import TabMlp, WideDeep, Transformer\nfrom pytorch_widedeep.preprocessing import TabPreprocessor\n</pre> from pathlib import Path  import numpy as np import torch import pandas as pd from torch import nn  from pytorch_widedeep import Trainer from pytorch_widedeep.utils import pad_sequences from pytorch_widedeep.models import TabMlp, WideDeep, Transformer from pytorch_widedeep.preprocessing import TabPreprocessor In\u00a0[2]: Copied! <pre>save_path = Path(\"prepared_data\")\n\nPAD_IDX = 0\n\nid_cols = [\"user_id\", \"movie_id\"]\n\ndf_train = pd.read_pickle(save_path / \"df_train.pkl\")\ndf_valid = pd.read_pickle(save_path / \"df_valid.pkl\")\ndf_test = pd.read_pickle(save_path / \"df_test.pkl\")\n</pre> save_path = Path(\"prepared_data\")  PAD_IDX = 0  id_cols = [\"user_id\", \"movie_id\"]  df_train = pd.read_pickle(save_path / \"df_train.pkl\") df_valid = pd.read_pickle(save_path / \"df_valid.pkl\") df_test = pd.read_pickle(save_path / \"df_test.pkl\") <p>...remember that in the previous notebook we explained that we are not  going to use a validation set here (in a real-world example, or simply a more realistic example, one should always use it).</p> In\u00a0[3]: Copied! <pre>df_test = pd.concat([df_valid, df_test], ignore_index=True)\n</pre> df_test = pd.concat([df_valid, df_test], ignore_index=True) <p>Also remember that, in the previous notebook we discussed that the <code>'maxlen'</code> and <code>'max_movie_index'</code> parameters should be computed using only the train set. In particular, to properly do the tokenization, one would have to use ONLY train tokens and add a token for new 'unknown'/'unseen' movies in the test set. This can also be done with this library or manually, so I will leave it to the reader to implement that tokenzation appraoch.</p> In\u00a0[4]: Copied! <pre>maxlen = max(\n    df_train.prev_movies.apply(lambda x: len(x)).max(),\n    df_test.prev_movies.apply(lambda x: len(x)).max(),\n)\n\nmax_movie_index = max(df_train.movie_id.max(), df_test.movie_id.max())\n</pre> maxlen = max(     df_train.prev_movies.apply(lambda x: len(x)).max(),     df_test.prev_movies.apply(lambda x: len(x)).max(), )  max_movie_index = max(df_train.movie_id.max(), df_test.movie_id.max()) <p>From now one things are pretty simple, moreover bearing in mind that in this example we are not going to use a wide component since, in pple, one would believe that the information in that component is also 'carried' by the movie sequences (However in the previous notebook, if one performs ablation studies, these suggest that most of the prediction power comes from the linear, wide model).</p> <p>In the example here we are going to explore one (of many) possibilities. We are simply going to encode the triplet <code>(user, item, rating)</code> and use it as a <code>deeptabular</code> component and the sequences of previously watched movies as the <code>deeptext</code> component. For the <code>deeptext</code> component we are going to use a basic encoder-only transformer model.</p> <p>Let's start with the tabular data preparation</p> In\u00a0[5]: Copied! <pre>df_train_user_item = df_train[[\"user_id\", \"movie_id\", \"rating\"]]\ntrain_movies_sequences = df_train.prev_movies.apply(\n    lambda x: [int(el) for el in x]\n).to_list()\ny_train = df_train.target.values.astype(int)\n\ndf_test_user_item = df_train[[\"user_id\", \"movie_id\", \"rating\"]]\ntest_movies_sequences = df_test.prev_movies.apply(\n    lambda x: [int(el) for el in x]\n).to_list()\ny_test = df_test.target.values.astype(int)\n\ntab_preprocessor = tab_preprocessor = TabPreprocessor(\n    cat_embed_cols=[\"user_id\", \"movie_id\", \"rating\"],\n)\nX_train_tab = tab_preprocessor.fit_transform(df_train_user_item)\nX_test_tab = tab_preprocessor.transform(df_test_user_item)\n</pre> df_train_user_item = df_train[[\"user_id\", \"movie_id\", \"rating\"]] train_movies_sequences = df_train.prev_movies.apply(     lambda x: [int(el) for el in x] ).to_list() y_train = df_train.target.values.astype(int)  df_test_user_item = df_train[[\"user_id\", \"movie_id\", \"rating\"]] test_movies_sequences = df_test.prev_movies.apply(     lambda x: [int(el) for el in x] ).to_list() y_test = df_test.target.values.astype(int)  tab_preprocessor = tab_preprocessor = TabPreprocessor(     cat_embed_cols=[\"user_id\", \"movie_id\", \"rating\"], ) X_train_tab = tab_preprocessor.fit_transform(df_train_user_item) X_test_tab = tab_preprocessor.transform(df_test_user_item) <p>And not the text component, simply padding the sequences:</p> In\u00a0[6]: Copied! <pre>X_train_text = np.array(\n    [\n        pad_sequences(\n            s,\n            maxlen=maxlen,\n            pad_first=False,\n            pad_idx=PAD_IDX,\n        )\n        for s in train_movies_sequences\n    ]\n)\nX_test_text = np.array(\n    [\n        pad_sequences(\n            s,\n            maxlen=maxlen,\n            pad_first=False,\n            pad_idx=0,\n        )\n        for s in test_movies_sequences\n    ]\n)\n</pre> X_train_text = np.array(     [         pad_sequences(             s,             maxlen=maxlen,             pad_first=False,             pad_idx=PAD_IDX,         )         for s in train_movies_sequences     ] ) X_test_text = np.array(     [         pad_sequences(             s,             maxlen=maxlen,             pad_first=False,             pad_idx=0,         )         for s in test_movies_sequences     ] ) <p>We now define the model components and the wide and deep model.</p> In\u00a0[7]: Copied! <pre>tab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    mlp_hidden_dims=[1024, 512, 256],\n    mlp_activation=\"relu\",\n)\n\n# plenty of options here, see the docs\ntransformer = Transformer(\n    vocab_size=max_movie_index + 1,\n    embed_dim=32,\n    n_heads=2,\n    n_blocks=2,\n    seq_length=maxlen,\n)\n\nwide_deep_model = WideDeep(\n    deeptabular=tab_mlp, deeptext=transformer, pred_dim=max_movie_index + 1\n)\n</pre> tab_mlp = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     mlp_hidden_dims=[1024, 512, 256],     mlp_activation=\"relu\", )  # plenty of options here, see the docs transformer = Transformer(     vocab_size=max_movie_index + 1,     embed_dim=32,     n_heads=2,     n_blocks=2,     seq_length=maxlen, )  wide_deep_model = WideDeep(     deeptabular=tab_mlp, deeptext=transformer, pred_dim=max_movie_index + 1 ) In\u00a0[8]: Copied! <pre>wide_deep_model\n</pre> wide_deep_model Out[8]: <pre>WideDeep(\n  (deeptabular): Sequential(\n    (0): TabMlp(\n      (cat_embed): DiffSizeCatEmbeddings(\n        (embed_layers): ModuleDict(\n          (emb_layer_user_id): Embedding(749, 65, padding_idx=0)\n          (emb_layer_movie_id): Embedding(1612, 100, padding_idx=0)\n          (emb_layer_rating): Embedding(6, 4, padding_idx=0)\n        )\n        (embedding_dropout): Dropout(p=0.0, inplace=False)\n      )\n      (encoder): MLP(\n        (mlp): Sequential(\n          (dense_layer_0): Sequential(\n            (0): Linear(in_features=169, out_features=1024, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_1): Sequential(\n            (0): Linear(in_features=1024, out_features=512, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_2): Sequential(\n            (0): Linear(in_features=512, out_features=256, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=256, out_features=1683, bias=True)\n  )\n  (deeptext): Sequential(\n    (0): Transformer(\n      (embedding): Embedding(1683, 32, padding_idx=0)\n      (pos_encoder): PositionalEncoding(\n        (dropout): Dropout(p=0.1, inplace=False)\n      )\n      (encoder): Sequential(\n        (transformer_block0): TransformerEncoder(\n          (attn): MultiHeadedAttention(\n            (dropout): Dropout(p=0.1, inplace=False)\n            (q_proj): Linear(in_features=32, out_features=32, bias=False)\n            (kv_proj): Linear(in_features=32, out_features=64, bias=False)\n            (out_proj): Linear(in_features=32, out_features=32, bias=False)\n          )\n          (ff): FeedForward(\n            (w_1): Linear(in_features=32, out_features=128, bias=True)\n            (w_2): Linear(in_features=128, out_features=32, bias=True)\n            (dropout): Dropout(p=0.1, inplace=False)\n            (activation): GELU(approximate='none')\n          )\n          (attn_addnorm): AddNorm(\n            (dropout): Dropout(p=0.1, inplace=False)\n            (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n          )\n          (ff_addnorm): AddNorm(\n            (dropout): Dropout(p=0.1, inplace=False)\n            (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n          )\n        )\n        (transformer_block1): TransformerEncoder(\n          (attn): MultiHeadedAttention(\n            (dropout): Dropout(p=0.1, inplace=False)\n            (q_proj): Linear(in_features=32, out_features=32, bias=False)\n            (kv_proj): Linear(in_features=32, out_features=64, bias=False)\n            (out_proj): Linear(in_features=32, out_features=32, bias=False)\n          )\n          (ff): FeedForward(\n            (w_1): Linear(in_features=32, out_features=128, bias=True)\n            (w_2): Linear(in_features=128, out_features=32, bias=True)\n            (dropout): Dropout(p=0.1, inplace=False)\n            (activation): GELU(approximate='none')\n          )\n          (attn_addnorm): AddNorm(\n            (dropout): Dropout(p=0.1, inplace=False)\n            (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n          )\n          (ff_addnorm): AddNorm(\n            (dropout): Dropout(p=0.1, inplace=False)\n            (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=23552, out_features=1683, bias=True)\n  )\n)</pre> <p>And as in the previous notebook, let's train (you will need a GPU for this)</p> In\u00a0[\u00a0]: Copied! <pre>trainer = Trainer(\n    model=wide_deep_model,\n    objective=\"multiclass\",\n    custom_loss_function=nn.CrossEntropyLoss(ignore_index=PAD_IDX),\n    optimizers=torch.optim.Adam(wide_deep_model.parameters(), lr=1e-3),\n)\n\ntrainer.fit(\n    X_train={\n        \"X_tab\": X_train_tab,\n        \"X_text\": X_train_text,\n        \"target\": y_train,\n    },\n    X_val={\n        \"X_tab\": X_test_tab,\n        \"X_text\": X_test_text,\n        \"target\": y_test,\n    },\n    n_epochs=10,\n    batch_size=521,\n    shuffle=False,\n)\n</pre> trainer = Trainer(     model=wide_deep_model,     objective=\"multiclass\",     custom_loss_function=nn.CrossEntropyLoss(ignore_index=PAD_IDX),     optimizers=torch.optim.Adam(wide_deep_model.parameters(), lr=1e-3), )  trainer.fit(     X_train={         \"X_tab\": X_train_tab,         \"X_text\": X_train_text,         \"target\": y_train,     },     X_val={         \"X_tab\": X_test_tab,         \"X_text\": X_test_text,         \"target\": y_test,     },     n_epochs=10,     batch_size=521,     shuffle=False, ) <pre>epoch 1:   0%|                                                                                                         | 0/147 [00:34&lt;?, ?it/s]\n</pre> In\u00a0[\u00a0]: Copied! <pre>\n</pre>"},{"location":"examples/19_load_from_folder_functionality.html","title":"19_load_from_folder_functionality","text":"<p>In this notebook I want to illustrate how one can use our <code>[...]FromFolder</code> functionalities along with the <code>[...]ChunkPreProcessors</code> in those cases where the dataset is too bit to fit in memory.</p> <p>These functionalities in the library have been designed for the following scenarop</p> In\u00a0[1]: Copied! <pre>import numpy as np\nimport torch\nimport pandas as pd\nfrom torch.utils.data import DataLoader\n\nfrom pytorch_widedeep.models import TabMlp, Vision, BasicRNN, WideDeep\nfrom pytorch_widedeep.training import TrainerFromFolder\nfrom pytorch_widedeep.callbacks import EarlyStopping, ModelCheckpoint\nfrom pytorch_widedeep.preprocessing import (\n    TabPreprocessor,\n    TextPreprocessor,\n    ImagePreprocessor,\n    ChunkTabPreprocessor,\n    ChunkTextPreprocessor,\n)\nfrom pytorch_widedeep.load_from_folder import (\n    TabFromFolder,\n    TextFromFolder,\n    ImageFromFolder,\n    WideDeepDatasetFromFolder,\n)\n</pre> import numpy as np import torch import pandas as pd from torch.utils.data import DataLoader  from pytorch_widedeep.models import TabMlp, Vision, BasicRNN, WideDeep from pytorch_widedeep.training import TrainerFromFolder from pytorch_widedeep.callbacks import EarlyStopping, ModelCheckpoint from pytorch_widedeep.preprocessing import (     TabPreprocessor,     TextPreprocessor,     ImagePreprocessor,     ChunkTabPreprocessor,     ChunkTextPreprocessor, ) from pytorch_widedeep.load_from_folder import (     TabFromFolder,     TextFromFolder,     ImageFromFolder,     WideDeepDatasetFromFolder, ) In\u00a0[2]: Copied! <pre># in my case, I place the data in a folder I call tmp_data, let's see how it looks\nairbnb_data = pd.read_csv(\"../tmp_data/airbnb/airbnb_sample.csv\")\nairbnb_data.sample(5)\n</pre> # in my case, I place the data in a folder I call tmp_data, let's see how it looks airbnb_data = pd.read_csv(\"../tmp_data/airbnb/airbnb_sample.csv\") airbnb_data.sample(5) Out[2]: id host_id description host_listings_count host_identity_verified neighbourhood_cleansed latitude longitude is_location_exact property_type ... amenity_wide_entrance amenity_wide_entrance_for_guests amenity_wide_entryway amenity_wide_hallways amenity_wifi amenity_window_guards amenity_wine_cooler security_deposit extra_people yield 39 53242.jpg 247650 A lovely big bright bedroom in a 2 bedroom fla... 2.0 t Lambeth 51.47075 -0.12913 t apartment ... 0 0 0 0 1 0 0 250.0 5.0 9.75 214 236716.jpg 1241070 We offer a warm welcome in our quiet double ro... 1.0 t Hackney 51.56593 -0.07482 t other ... 0 0 0 0 1 0 0 200.0 10.0 76.50 400 346523.jpg 1756532 Available for you to rent is a cozy studio in ... 2.0 t Kensington and Chelsea 51.48311 -0.18428 t other ... 0 0 0 0 1 0 0 0.0 50.0 180.90 512 389627.jpg 1949299 This gorgeous studio flat is situated in the v... 1.0 t Westminster 51.51838 -0.14238 f apartment ... 0 0 0 0 1 0 0 250.0 25.0 276.90 504 388767.jpg 1945165 If you want to experience London at it's best ... 2.0 f Camden 51.54293 -0.14073 t apartment ... 0 0 0 0 1 0 0 150.0 10.0 591.10 <p>5 rows \u00d7 223 columns</p> In\u00a0[3]: Copied! <pre># for example\nfrom IPython.display import Image\n\npil_img = Image(filename=\"../tmp_data/airbnb/property_picture/272908.jpg\")\ndisplay(pil_img)\n</pre> # for example from IPython.display import Image  pil_img = Image(filename=\"../tmp_data/airbnb/property_picture/272908.jpg\") display(pil_img) In\u00a0[4]: Copied! <pre># And the description for the property that that picture belongs to is:\nairbnb_data[airbnb_data.id == \"272908.jpg\"].description.tolist()\n</pre> # And the description for the property that that picture belongs to is: airbnb_data[airbnb_data.id == \"272908.jpg\"].description.tolist() Out[4]: <pre>[\"Bright, sunny beautiful room that will give you the perfect base to explore all of London. Come and explore one of London's best neighbourhoods - Herne Hill! As mentioned in (Website hidden by Airbnb)   (Website hidden by Airbnb)  WiFi availability with a fully stocked and clean uplifting home. Lovely sunny, airy and big double bedroom on a leafy south-London street.    Note: This room comes with a reserved Off-Street parking spot! The room is on the first floor and boasts an enormous Super King bed, gorgeous wooden floors, tall ceilings and large windows which let in the sunshine almost all day. (Yoga May or meditation cushion available on request) The flat is bright and airy and big! So lots of space for all.  Location wise you are only 10 minutes walk to either Herne Hill or West Dulwich stations, both of which will take you to Victoria and the city within minutes. You can also hop on a bus right outside the flat that will take you to Brixton tube station within 8 minutes where you \"]</pre> <p>Ok, so we have tabular data where one column is <code>description</code> and another <code>id</code>, points towards the images stored in disk. Now, remember the following, because this will appear a few times in the notebook: our \"reference dataset\" is the tabular data.</p> <p>Therefore, since I want to illustrate a \"semi-realistic\" case, if we need to split the data into training, validation and test datasets, these datasets needs to be separetely stored in disk. In my case I have done this and in the <code>tmp_data/airbnb</code> dir I have the following:</p> <pre>../tmp_data/airbnb\n\u251c\u2500\u2500 airbnb_sample.csv\n\u251c\u2500\u2500 airbnb_sample_eval.csv\n\u251c\u2500\u2500 airbnb_sample_test.csv\n\u251c\u2500\u2500 airbnb_sample_train.csv\n\u2514\u2500\u2500 property_picture\n</pre> <p>Where <code>airbnb_sample.csv</code> is the full sample (1001 rows) and the <code>train</code>, <code>eval</code> and <code>test</code> set is the corresponding splits. In a realistic example, the full sample would be the 'gigantic' dataset and the rest the corresponding splits. One has to do this 'offline', prior to start the coding.</p> <p>Also, one thing that one needs to know is the number of total observations/rows, as well as the splits. In our case the train size is 800, and the eval and test sizes are 100 and 101 respectively.</p> <p>With all that info, let's star</p> In\u00a0[5]: Copied! <pre># path to the tabular data and the splits\ndata_path = \"../tmp_data/airbnb/\"\ntrain_fname = \"airbnb_sample_train.csv\"\neval_fname = \"airbnb_sample_eval.csv\"\ntest_fname = \"airbnb_sample_test.csv\"\n\n# split sizes\ntrain_size = 800\neval_size = 100\ntest_size = 101\n\n# number of chunks for the Chunk Preprocessors\nchunksize = 100\nn_chunks = int(np.ceil(train_size / chunksize))\n\n# path to the image dataset and name of the image col\nimg_path = \"../tmp_data/airbnb/property_picture/\"\nimg_col = \"id\"\n\n# name of the text col\ntext_col = \"description\"\n\n# mane of the target\ntarget_col = \"yield\"\n\n# definition of the categorical and continuous cols for the TabPreprocessor\ncat_embed_cols = [\n    \"host_listings_count\",\n    \"neighbourhood_cleansed\",\n    \"is_location_exact\",\n    \"property_type\",\n    \"room_type\",\n    \"accommodates\",\n    \"bathrooms\",\n    \"bedrooms\",\n    \"beds\",\n    \"guests_included\",\n    \"minimum_nights\",\n    \"instant_bookable\",\n    \"cancellation_policy\",\n    \"has_house_rules\",\n    \"host_gender\",\n    \"accommodates_catg\",\n    \"guests_included_catg\",\n    \"minimum_nights_catg\",\n    \"host_listings_count_catg\",\n    \"bathrooms_catg\",\n    \"bedrooms_catg\",\n    \"beds_catg\",\n    \"security_deposit\",\n    \"extra_people\",\n]\ncont_cols = [\"latitude\", \"longitude\"]\n</pre> # path to the tabular data and the splits data_path = \"../tmp_data/airbnb/\" train_fname = \"airbnb_sample_train.csv\" eval_fname = \"airbnb_sample_eval.csv\" test_fname = \"airbnb_sample_test.csv\"  # split sizes train_size = 800 eval_size = 100 test_size = 101  # number of chunks for the Chunk Preprocessors chunksize = 100 n_chunks = int(np.ceil(train_size / chunksize))  # path to the image dataset and name of the image col img_path = \"../tmp_data/airbnb/property_picture/\" img_col = \"id\"  # name of the text col text_col = \"description\"  # mane of the target target_col = \"yield\"  # definition of the categorical and continuous cols for the TabPreprocessor cat_embed_cols = [     \"host_listings_count\",     \"neighbourhood_cleansed\",     \"is_location_exact\",     \"property_type\",     \"room_type\",     \"accommodates\",     \"bathrooms\",     \"bedrooms\",     \"beds\",     \"guests_included\",     \"minimum_nights\",     \"instant_bookable\",     \"cancellation_policy\",     \"has_house_rules\",     \"host_gender\",     \"accommodates_catg\",     \"guests_included_catg\",     \"minimum_nights_catg\",     \"host_listings_count_catg\",     \"bathrooms_catg\",     \"bedrooms_catg\",     \"beds_catg\",     \"security_deposit\",     \"extra_people\", ] cont_cols = [\"latitude\", \"longitude\"] In\u00a0[6]: Copied! <pre>tab_preprocessor = TabPreprocessor(\n    embed_cols=cat_embed_cols,\n    continuous_cols=cont_cols,\n    default_embed_dim=8,\n    verbose=0,\n)\n\ntext_preprocessor = TextPreprocessor(\n    text_col=text_col,\n    n_cpus=1,\n)\n\nimg_preprocessor = ImagePreprocessor(\n    img_col=img_col,\n    img_path=img_path,\n)\n</pre> tab_preprocessor = TabPreprocessor(     embed_cols=cat_embed_cols,     continuous_cols=cont_cols,     default_embed_dim=8,     verbose=0, )  text_preprocessor = TextPreprocessor(     text_col=text_col,     n_cpus=1, )  img_preprocessor = ImagePreprocessor(     img_col=img_col,     img_path=img_path, ) In\u00a0[7]: Copied! <pre>tab_preprocessor.fit(airbnb_data)\ntext_preprocessor.fit(airbnb_data)\nimg_preprocessor.fit(airbnb_data)\n</pre> tab_preprocessor.fit(airbnb_data) text_preprocessor.fit(airbnb_data) img_preprocessor.fit(airbnb_data) <pre>The vocabulary contains 2192 tokens\n</pre> Out[7]: <pre>ImagePreprocessor(img_col=id, img_path=../tmp_data/airbnb/property_picture/, width=224, height=224, verbose=1)</pre> In\u00a0[8]: Copied! <pre>chunk_tab_preprocessor = ChunkTabPreprocessor(\n    embed_cols=cat_embed_cols,\n    continuous_cols=cont_cols,\n    n_chunks=n_chunks,\n    default_embed_dim=8,\n    verbose=0,\n)\n\nchunk_text_preprocessor = ChunkTextPreprocessor(\n    n_chunks=n_chunks,\n    text_col=text_col,\n    n_cpus=1,\n    verbose=0,\n)\n\nfor i, chunk in enumerate(\n    pd.read_csv(\"/\".join([data_path, train_fname]), chunksize=chunksize)\n):\n    print(f\"chunk in loop: {i + 1}\")\n    chunk_tab_preprocessor.fit(chunk)\n    chunk_text_preprocessor.fit(chunk)\n</pre> chunk_tab_preprocessor = ChunkTabPreprocessor(     embed_cols=cat_embed_cols,     continuous_cols=cont_cols,     n_chunks=n_chunks,     default_embed_dim=8,     verbose=0, )  chunk_text_preprocessor = ChunkTextPreprocessor(     n_chunks=n_chunks,     text_col=text_col,     n_cpus=1,     verbose=0, )  for i, chunk in enumerate(     pd.read_csv(\"/\".join([data_path, train_fname]), chunksize=chunksize) ):     print(f\"chunk in loop: {i + 1}\")     chunk_tab_preprocessor.fit(chunk)     chunk_text_preprocessor.fit(chunk) <pre>chunk in loop: 1\nchunk in loop: 2\nchunk in loop: 3\nchunk in loop: 4\nchunk in loop: 5\nchunk in loop: 6\nchunk in loop: 7\nchunk in loop: 8\n</pre> In\u00a0[9]: Copied! <pre>train_tab_folder = TabFromFolder(\n    fname=train_fname,\n    directory=data_path,\n    target_col=target_col,\n    preprocessor=tab_preprocessor,\n    text_col=text_col,\n    img_col=img_col,\n)\n\n# Note how we can use the `train_tab_folder` as reference so we don't have to\n# define all parameters again\neval_tab_folder = TabFromFolder(fname=eval_fname, reference=train_tab_folder)\n\n# Note that for the test set we can ignore the target as no metric will be\n# computed by the `predict` method\ntest_tab_folder = TabFromFolder(\n    fname=test_fname, reference=train_tab_folder, ignore_target=True\n)\n</pre> train_tab_folder = TabFromFolder(     fname=train_fname,     directory=data_path,     target_col=target_col,     preprocessor=tab_preprocessor,     text_col=text_col,     img_col=img_col, )  # Note how we can use the `train_tab_folder` as reference so we don't have to # define all parameters again eval_tab_folder = TabFromFolder(fname=eval_fname, reference=train_tab_folder)  # Note that for the test set we can ignore the target as no metric will be # computed by the `predict` method test_tab_folder = TabFromFolder(     fname=test_fname, reference=train_tab_folder, ignore_target=True ) In\u00a0[10]: Copied! <pre># for the text and image datasets we do not need to specify eval or test loaders\ntext_folder = TextFromFolder(preprocessor=text_preprocessor)\nimg_folder = ImageFromFolder(preprocessor=img_preprocessor)\n</pre> # for the text and image datasets we do not need to specify eval or test loaders text_folder = TextFromFolder(preprocessor=text_preprocessor) img_folder = ImageFromFolder(preprocessor=img_preprocessor) In\u00a0[11]: Copied! <pre>train_dataset_folder = WideDeepDatasetFromFolder(\n    n_samples=train_size,\n    tab_from_folder=train_tab_folder,\n    text_from_folder=text_folder,\n    img_from_folder=img_folder,\n)\n\n# Note that the eval and test loaders only need their corresponding\n# `TabFromFolder` classes. The rest of the parameters can be defined\n# via a `reference` `TabFromFolder` class\neval_dataset_folder = WideDeepDatasetFromFolder(\n    n_samples=eval_size,\n    tab_from_folder=eval_tab_folder,\n    reference=train_dataset_folder,\n)\n\ntest_dataset_folder = WideDeepDatasetFromFolder(\n    n_samples=test_size,\n    tab_from_folder=test_tab_folder,\n    reference=train_dataset_folder,\n)\n</pre> train_dataset_folder = WideDeepDatasetFromFolder(     n_samples=train_size,     tab_from_folder=train_tab_folder,     text_from_folder=text_folder,     img_from_folder=img_folder, )  # Note that the eval and test loaders only need their corresponding # `TabFromFolder` classes. The rest of the parameters can be defined # via a `reference` `TabFromFolder` class eval_dataset_folder = WideDeepDatasetFromFolder(     n_samples=eval_size,     tab_from_folder=eval_tab_folder,     reference=train_dataset_folder, )  test_dataset_folder = WideDeepDatasetFromFolder(     n_samples=test_size,     tab_from_folder=test_tab_folder,     reference=train_dataset_folder, ) In\u00a0[12]: Copied! <pre>train_loader = DataLoader(train_dataset_folder, batch_size=16, num_workers=1)\neval_loader = DataLoader(eval_dataset_folder, batch_size=16, num_workers=1)\ntest_loader = DataLoader(test_dataset_folder, batch_size=16, num_workers=1)\n</pre> train_loader = DataLoader(train_dataset_folder, batch_size=16, num_workers=1) eval_loader = DataLoader(eval_dataset_folder, batch_size=16, num_workers=1) test_loader = DataLoader(test_dataset_folder, batch_size=16, num_workers=1) <p>And from here on is business as usual:</p> In\u00a0[13]: Copied! <pre># for example\nbasic_rnn = BasicRNN(\n    vocab_size=len(text_preprocessor.vocab.itos),\n    embed_dim=32,\n    hidden_dim=64,\n    n_layers=2,\n)\n\ndeepimage = Vision()\n\ndeepdense = TabMlp(\n    mlp_hidden_dims=[32, 16],\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=cont_cols,\n)\n\nmodel = WideDeep(\n    deeptabular=deepdense,\n    deeptext=basic_rnn,\n    deepimage=deepimage,\n)\n\nmodel\n</pre> # for example basic_rnn = BasicRNN(     vocab_size=len(text_preprocessor.vocab.itos),     embed_dim=32,     hidden_dim=64,     n_layers=2, )  deepimage = Vision()  deepdense = TabMlp(     mlp_hidden_dims=[32, 16],     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     continuous_cols=cont_cols, )  model = WideDeep(     deeptabular=deepdense,     deeptext=basic_rnn,     deepimage=deepimage, )  model Out[13]: <pre>WideDeep(\n  (deeptabular): Sequential(\n    (0): TabMlp(\n      (cat_embed): DiffSizeCatEmbeddings(\n        (embed_layers): ModuleDict(\n          (emb_layer_host_listings_count): Embedding(28, 10, padding_idx=0)\n          (emb_layer_neighbourhood_cleansed): Embedding(33, 11, padding_idx=0)\n          (emb_layer_is_location_exact): Embedding(3, 2, padding_idx=0)\n          (emb_layer_property_type): Embedding(4, 3, padding_idx=0)\n          (emb_layer_room_type): Embedding(4, 3, padding_idx=0)\n          (emb_layer_accommodates): Embedding(14, 7, padding_idx=0)\n          (emb_layer_bathrooms): Embedding(11, 6, padding_idx=0)\n          (emb_layer_bedrooms): Embedding(7, 4, padding_idx=0)\n          (emb_layer_beds): Embedding(11, 6, padding_idx=0)\n          (emb_layer_guests_included): Embedding(11, 6, padding_idx=0)\n          (emb_layer_minimum_nights): Embedding(25, 9, padding_idx=0)\n          (emb_layer_instant_bookable): Embedding(3, 2, padding_idx=0)\n          (emb_layer_cancellation_policy): Embedding(6, 4, padding_idx=0)\n          (emb_layer_has_house_rules): Embedding(3, 2, padding_idx=0)\n          (emb_layer_host_gender): Embedding(4, 3, padding_idx=0)\n          (emb_layer_accommodates_catg): Embedding(4, 3, padding_idx=0)\n          (emb_layer_guests_included_catg): Embedding(4, 3, padding_idx=0)\n          (emb_layer_minimum_nights_catg): Embedding(4, 3, padding_idx=0)\n          (emb_layer_host_listings_count_catg): Embedding(5, 3, padding_idx=0)\n          (emb_layer_bathrooms_catg): Embedding(4, 3, padding_idx=0)\n          (emb_layer_bedrooms_catg): Embedding(5, 3, padding_idx=0)\n          (emb_layer_beds_catg): Embedding(5, 3, padding_idx=0)\n          (emb_layer_security_deposit): Embedding(53, 15, padding_idx=0)\n          (emb_layer_extra_people): Embedding(39, 12, padding_idx=0)\n        )\n        (embedding_dropout): Dropout(p=0.0, inplace=False)\n      )\n      (cont_norm): Identity()\n      (encoder): MLP(\n        (mlp): Sequential(\n          (dense_layer_0): Sequential(\n            (0): Linear(in_features=128, out_features=32, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_1): Sequential(\n            (0): Linear(in_features=32, out_features=16, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=16, out_features=1, bias=True)\n  )\n  (deeptext): Sequential(\n    (0): BasicRNN(\n      (word_embed): Embedding(2192, 32, padding_idx=1)\n      (rnn): LSTM(32, 64, num_layers=2, batch_first=True, dropout=0.1)\n      (rnn_mlp): Identity()\n    )\n    (1): Linear(in_features=64, out_features=1, bias=True)\n  )\n  (deepimage): Sequential(\n    (0): Vision(\n      (features): Sequential(\n        (conv_layer_0): Sequential(\n          (0): Conv2d(3, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False)\n          (1): BatchNorm2d(64, eps=1e-05, momentum=0.01, affine=True, track_running_stats=True)\n          (2): LeakyReLU(negative_slope=0.1, inplace=True)\n          (maxpool): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n        )\n        (conv_layer_1): Sequential(\n          (0): Conv2d(64, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n          (1): BatchNorm2d(128, eps=1e-05, momentum=0.01, affine=True, track_running_stats=True)\n          (2): LeakyReLU(negative_slope=0.1, inplace=True)\n        )\n        (conv_layer_2): Sequential(\n          (0): Conv2d(128, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n          (1): BatchNorm2d(256, eps=1e-05, momentum=0.01, affine=True, track_running_stats=True)\n          (2): LeakyReLU(negative_slope=0.1, inplace=True)\n        )\n        (conv_layer_3): Sequential(\n          (0): Conv2d(256, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n          (1): BatchNorm2d(512, eps=1e-05, momentum=0.01, affine=True, track_running_stats=True)\n          (2): LeakyReLU(negative_slope=0.1, inplace=True)\n          (adaptiveavgpool): AdaptiveAvgPool2d(output_size=(1, 1))\n        )\n      )\n    )\n    (1): Linear(in_features=512, out_features=1, bias=True)\n  )\n)</pre> In\u00a0[14]: Copied! <pre>trainer = TrainerFromFolder(\n    model,\n    objective=\"regression\",\n)\n\ntrainer.fit(\n    train_loader=train_loader,\n    eval_loader=eval_loader,\n)\n</pre> trainer = TrainerFromFolder(     model,     objective=\"regression\", )  trainer.fit(     train_loader=train_loader,     eval_loader=eval_loader, ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 50/50 [03:41&lt;00:00,  4.42s/it, loss=1.64e+4]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 7/7 [00:23&lt;00:00,  3.30s/it, loss=6.27e+3]\n</pre> In\u00a0[15]: Copied! <pre>preds = trainer.predict(test_loader=test_loader)\n</pre> preds = trainer.predict(test_loader=test_loader) <pre>predict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 7/7 [00:22&lt;00:00,  3.26s/it]\n</pre> <p>Note that in the case of predict you could also choose to do this</p> In\u00a0[16]: Copied! <pre>df_test = pd.read_csv(\"/\".join([data_path, test_fname]))\n</pre> df_test = pd.read_csv(\"/\".join([data_path, test_fname])) In\u00a0[17]: Copied! <pre># if the images for the test set fit in memory\nX_tab_test = chunk_tab_preprocessor.transform(df_test)\nX_text_test = chunk_text_preprocessor.transform(df_test)\nX_img_test = img_preprocessor.transform(df_test)\n</pre> # if the images for the test set fit in memory X_tab_test = chunk_tab_preprocessor.transform(df_test) X_text_test = chunk_text_preprocessor.transform(df_test) X_img_test = img_preprocessor.transform(df_test) <pre>Reading Images from ../tmp_data/airbnb/property_picture/\nResizing\n</pre> <pre>100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 101/101 [00:00&lt;00:00, 708.23it/s]</pre> <pre>Computing normalisation metrics\n</pre> <pre>\n</pre> In\u00a0[18]: Copied! <pre>preds = trainer.predict(\n    X_tab=X_tab_test, X_text=X_text_test, X_img=X_img_test, batch_size=32\n)\n</pre> preds = trainer.predict(     X_tab=X_tab_test, X_text=X_text_test, X_img=X_img_test, batch_size=32 ) <pre>predict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 4/4 [00:03&lt;00:00,  1.14it/s]\n</pre>"},{"location":"examples/19_load_from_folder_functionality.html#scenario","title":"Scenario\u00b6","text":"<p>We have a tabular dataset combined with images and text and either some, or all these datasets do not fit in memory. Note that the tabular dataset MUST ALWAYS be present as it is considered the rerefence. This is, if we have an image dataset, the tabular dataset must contain a column that points to the image file names as stored in disk. Similarly, if we have a text dataset, then the tabular dataset must contain a column with the texts themselves or a the file names of the text files as stored in disk.</p> <p>If you only have text and/or images and not a tabular component, I would suggest using other libraries (such as hugginface probably).</p> <p>Within this scenario, they are two possible scenarios that we will cover here:</p> <ol> <li><p>The tabular data itsel fits in memory and is only the images that do not: in this case you could use the 'standard' <code>Preprocessors</code> in the library and off you go, move directly to the <code>[...]FromFolder</code> functionalities</p> </li> <li><p>The tabular data is also very large and does not fit in memory, so we have to process it in chuncks. For this second case I have created the so called <code>Chunk[...]Preprocessor</code> (Wide, Tab and Text).</p> </li> </ol> <p>Note that at the moment ONLY csv format is allowed for the tabular file. More formats will be supported in the future.</p> <p>Let's see a complete example to illustrate how each of these cases would be addressed with the new functionalities in the library. For this example we will use a sample of the airbnb dataset</p> <p>The airbnb dataset, which you could get from here, is too big to be included in our datasets module (when including images). Therefore, what I did was, go there, download it, and use the download_images.py script to get the images and the airbnb_data_processing.py to process the data. I did this ages ago and I believe the format of the dataset might be different now. Nonetheless, I will show samples of the dataset as we go through so you can extrapolate the content of this notebook to your particular problem.</p> <p>In the future we will find better datasets\ud83d\ude42. Finally, note that here we are only using a small sample to illustrate the use, so PLEASE ignore the results, just focus on usage.</p>"},{"location":"examples/19_load_from_folder_functionality.html#setting-variables-and-constants","title":"Setting variables and constants\u00b6","text":""},{"location":"examples/19_load_from_folder_functionality.html#step-1-the-preprocessors","title":"Step 1: the preprocessors\u00b6","text":""},{"location":"examples/19_load_from_folder_functionality.html#scenario-1-only-the-images-do-not-fit-in-disk","title":"Scenario 1: only the images do not fit in disk\u00b6","text":"<p>In this case we can prepare the data in the 'standard' way</p>"},{"location":"examples/19_load_from_folder_functionality.html#scenario-2-the-tabular-data-is-also-huge","title":"Scenario 2: the tabular data is also huge\u00b6","text":"<p>Then we need to prepare it in chunks. Note that, unfortunately, the tabular and text preprocessors need to see the whole dataset once. This is because to process tabular or text data we need to encode values. For those encodings to be consistent they need to have seen the whole dataset. Alternatively, one could code a solution with some streaming encoder for both datasets. However, such implementation is not trivial for this library (and in general). I also don't think that having to see the whole data once is such a big limitation. Let's see how is done.</p> <p>Note that I have not mentioned the image dataset. This is because the processing of the image dataset does not require any form of encoding and in consequence can be done 'on the fly'. Therefore, no <code>ChunkImgPreprocessor</code> processor is needed.</p>"},{"location":"examples/19_load_from_folder_functionality.html#step-2-the-fromfolder-classes","title":"Step 2: the <code>[...]FromFolder</code> classes\u00b6","text":"<p>Once we have the preprocessors, we need to instantiate the classes that will enable us to load the data from their respective folders. From now on I am going to proceed with the <code>chunk_tab_preprocessor</code>, <code>chunk_text_preprocessor</code> and <code>img_preprocessor</code>, but the code would be identical if instead of the first two preprocessors we decided to use the <code>tab_preprocessor</code> and <code>text_preprocessor</code>.</p> <p>Once more, our reference datasets are the tabular datasets, which we have splitted in train, eval and test prior to start the coding. Therefore, we will eventually need a loader for each split</p>"},{"location":"examples/19_load_from_folder_functionality.html#step-3-pytorch-datasets-and-dataloaders","title":"Step 3: pytorch datasets and dataloaders\u00b6","text":"<p>From here in advance, is all very 'standard' if you are familiar with pytorch. One needs to define a class that inherits from the <code>Dataset</code> class in pytorch. Then this will be passed to a <code>DataLoader</code> class and we are ready to train. Our <code>Dataset</code> child class is <code>WideDeepDatasetFromFolder</code>. This class will use the tabular dataset and the corresponding text and image columns to load the adequate data in the batches</p> <p>Let's do it</p>"},{"location":"examples/19_load_from_folder_functionality.html#step-4-define-the-model","title":"Step 4: define the model\u00b6","text":""},{"location":"examples/19_load_from_folder_functionality.html#step-5-fit-and-predict","title":"Step 5: fit and predict\u00b6","text":""},{"location":"examples/20_Using_huggingface_within_widedeep.html","title":"20-Using-huggingface-within-widedeep","text":"<p>In this notebook we will show how to use Hugginface's tokenizers and models as they are integrated within the library. In notebook number 17 you can find examples on how to code your own, custom, Hugginface (hereafter HF) model and use it in combination of any other model in the library</p> In\u00a0[1]: Copied! <pre>import numpy as np\nimport pandas as pd\nfrom sklearn.metrics import f1_score, accuracy_score\nfrom sklearn.model_selection import train_test_split\n\nfrom pytorch_widedeep import Trainer\nfrom pytorch_widedeep.models import HFModel, WideDeep\nfrom pytorch_widedeep.metrics import F1Score, Accuracy\nfrom pytorch_widedeep.datasets import load_womens_ecommerce\nfrom pytorch_widedeep.preprocessing import HFPreprocessor\n</pre> import numpy as np import pandas as pd from sklearn.metrics import f1_score, accuracy_score from sklearn.model_selection import train_test_split  from pytorch_widedeep import Trainer from pytorch_widedeep.models import HFModel, WideDeep from pytorch_widedeep.metrics import F1Score, Accuracy from pytorch_widedeep.datasets import load_womens_ecommerce from pytorch_widedeep.preprocessing import HFPreprocessor <pre>/Users/javierrodriguezzaurin/.pyenv/versions/3.10.13/envs/widedeep310/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n  from .autonotebook import tqdm as notebook_tqdm\n</pre> In\u00a0[2]: Copied! <pre>df: pd.DataFrame = load_womens_ecommerce(as_frame=True)  # type: ignore\n</pre> df: pd.DataFrame = load_womens_ecommerce(as_frame=True)  # type: ignore In\u00a0[3]: Copied! <pre>df.shape\n</pre> df.shape Out[3]: <pre>(23486, 10)</pre> In\u00a0[4]: Copied! <pre>df.sample(3)\n</pre> df.sample(3) Out[4]: Clothing ID Age Title Review Text Rating Recommended IND Positive Feedback Count Division Name Department Name Class Name 7004 862 43 Cute and feminine Loved this sweater wrap and bought it in both ... 5 1 2 General Tops Knits 12508 975 66 Love it The linen fabric is elegantly thin feels and l... 5 1 3 General Jackets Jackets 10288 950 41 Perfect for fall This sweater is just as pictured. the fit is t... 5 1 0 General Tops Sweaters In\u00a0[5]: Copied! <pre># Let's do some mild preprocessing\ndf.columns = [c.replace(\" \", \"_\").lower() for c in df.columns]\n\n# classes from [0,num_class)\ndf[\"rating\"] = (df[\"rating\"] - 1).astype(\"int64\")\n\n# group reviews with 1 and 2 scores into one class\ndf.loc[df.rating == 0, \"rating\"] = 1\n\n# and back again to [0,num_class)\ndf[\"rating\"] = (df[\"rating\"] - 1).astype(\"int64\")\n</pre> # Let's do some mild preprocessing df.columns = [c.replace(\" \", \"_\").lower() for c in df.columns]  # classes from [0,num_class) df[\"rating\"] = (df[\"rating\"] - 1).astype(\"int64\")  # group reviews with 1 and 2 scores into one class df.loc[df.rating == 0, \"rating\"] = 1  # and back again to [0,num_class) df[\"rating\"] = (df[\"rating\"] - 1).astype(\"int64\") In\u00a0[6]: Copied! <pre># drop short reviews\ndf = df[~df.review_text.isna()]\ndf[\"review_length\"] = df.review_text.apply(lambda x: len(x.split(\" \")))\ndf = df[df.review_length &gt;= 5]\ndf = df.drop(\"review_length\", axis=1).reset_index(drop=True)\n</pre> # drop short reviews df = df[~df.review_text.isna()] df[\"review_length\"] = df.review_text.apply(lambda x: len(x.split(\" \"))) df = df[df.review_length &gt;= 5] df = df.drop(\"review_length\", axis=1).reset_index(drop=True) In\u00a0[7]: Copied! <pre>df.shape\n</pre> df.shape Out[7]: <pre>(22608, 10)</pre> In\u00a0[8]: Copied! <pre># if you run this on a CPU, you might want to subsample the dataset. With that in mind I am simply going to stratify-sample to the minimum category occurrence and then sample at random\n# If you run this on a GPU you can comment out the following two cells\ndf.rating.value_counts()\n</pre> # if you run this on a CPU, you might want to subsample the dataset. With that in mind I am simply going to stratify-sample to the minimum category occurrence and then sample at random # If you run this on a GPU you can comment out the following two cells df.rating.value_counts() Out[8]: <pre>rating\n3    12515\n2     4904\n1     2820\n0     2369\nName: count, dtype: int64</pre> In\u00a0[9]: Copied! <pre>df = (\n    df.groupby(\"rating\", group_keys=False)\n    .apply(lambda x: x.sample(min(len(x), 2369)))\n    .sample(1000)\n)\n</pre> df = (     df.groupby(\"rating\", group_keys=False)     .apply(lambda x: x.sample(min(len(x), 2369)))     .sample(1000) ) <pre>/var/folders/_2/lrjn1qn54c758tdtktr1bvkc0000gn/T/ipykernel_5886/895673206.py:3: DeprecationWarning: DataFrameGroupBy.apply operated on the grouping columns. This behavior is deprecated, and in a future version of pandas the grouping columns will be excluded from the operation. Either pass `include_groups=False` to exclude the groupings or explicitly select the grouping columns after groupby to silence this warning.\n  .apply(lambda x: x.sample(min(len(x), 2369)))\n</pre> In\u00a0[10]: Copied! <pre>train, test = train_test_split(df, train_size=0.8, random_state=1, stratify=df.rating)\n\n# possible model names currently supported in the library\nmodel_names = [\n    \"distilbert-base-uncased\",\n    \"bert-base-uncased\",\n    \"FacebookAI/roberta-base\",\n    \"albert-base-v2\",\n    \"google/electra-base-discriminator\",\n]\n\n# Let's choose one. The syntax is the same for all the models\nmodel_name = \"distilbert-base-uncased\"\n</pre> train, test = train_test_split(df, train_size=0.8, random_state=1, stratify=df.rating)  # possible model names currently supported in the library model_names = [     \"distilbert-base-uncased\",     \"bert-base-uncased\",     \"FacebookAI/roberta-base\",     \"albert-base-v2\",     \"google/electra-base-discriminator\", ]  # Let's choose one. The syntax is the same for all the models model_name = \"distilbert-base-uncased\" <p>Now we can use the <code>HFPreprocessor</code> class. As most things in this library, the integration with HF has been coded aiming for a flexible use. With this in mind, there are two ways one can use a <code>HFPreprocessor</code> class.</p> <ol> <li>Passing a <code>text_col</code> and <code>encode_params</code> as the class is instantiated and then using the <code>fit</code> and <code>transform</code> as with any other preprocessor in the library</li> <li>Without passing  <code>text_col</code> and <code>encode_params</code> as the class is instantiated and using the <code>encode</code> method of the <code>HFPreprocessor</code> which is simply a wrapper around the encode method of HF's tokenizers</li> </ol> <p>Let's have a look</p> In\u00a0[11]: Copied! <pre>tokenizer1 = HFPreprocessor(\n    model_name=model_name,\n    text_col=\"review_text\",\n    num_workers=1,\n    encode_params={\n        \"max_length\": 90,\n        \"padding\": \"max_length\",\n        \"truncation\": True,\n        \"add_special_tokens\": True,\n    },\n)\n\nX_text_tr1 = tokenizer1.fit_transform(train)\nX_text_te1 = tokenizer1.transform(test)\n</pre> tokenizer1 = HFPreprocessor(     model_name=model_name,     text_col=\"review_text\",     num_workers=1,     encode_params={         \"max_length\": 90,         \"padding\": \"max_length\",         \"truncation\": True,         \"add_special_tokens\": True,     }, )  X_text_tr1 = tokenizer1.fit_transform(train) X_text_te1 = tokenizer1.transform(test) In\u00a0[12]: Copied! <pre>tokenizer2 = HFPreprocessor(\n    model_name=model_name,\n    num_workers=1,\n)\n\nX_text_tr2 = tokenizer2.encode(\n    train.review_text.tolist(),\n    max_length=90,\n    padding=\"max_length\",\n    truncation=True,\n    add_special_tokens=True,\n)\nX_text_te2 = tokenizer2.encode(\n    test.review_text.tolist(),\n    max_length=90,\n    padding=\"max_length\",\n    truncation=True,\n    add_special_tokens=True,\n)\n</pre> tokenizer2 = HFPreprocessor(     model_name=model_name,     num_workers=1, )  X_text_tr2 = tokenizer2.encode(     train.review_text.tolist(),     max_length=90,     padding=\"max_length\",     truncation=True,     add_special_tokens=True, ) X_text_te2 = tokenizer2.encode(     test.review_text.tolist(),     max_length=90,     padding=\"max_length\",     truncation=True,     add_special_tokens=True, ) In\u00a0[13]: Copied! <pre>all(X_text_tr1[0] == X_text_tr2[0])\n</pre> all(X_text_tr1[0] == X_text_tr2[0]) Out[13]: <pre>True</pre> In\u00a0[14]: Copied! <pre># Now we define a model which is as easy as:\n# Note that this will instantiation will lead to NO parameter trainable in the HF model.\n# If you want to fine-tune the HF model, you can set the trainable parameters via the 'trainable_parameters' argument.\n# Alternatively, you can use a head (MLP) via the 'head'-related arguments (see the docs for more details)\nhf_model = HFModel(model_name=model_name)\n</pre> # Now we define a model which is as easy as: # Note that this will instantiation will lead to NO parameter trainable in the HF model. # If you want to fine-tune the HF model, you can set the trainable parameters via the 'trainable_parameters' argument. # Alternatively, you can use a head (MLP) via the 'head'-related arguments (see the docs for more details) hf_model = HFModel(model_name=model_name) In\u00a0[15]: Copied! <pre># And from here on is the same as any other WideDeep model\nmodel = WideDeep(\n    deeptext=hf_model,\n    pred_dim=4,\n)\n\ntrainer = Trainer(\n    model,\n    objective=\"multiclass\",\n    metrics=[Accuracy(), F1Score(average=True)],\n)\n\ntrainer.fit(\n    X_text=X_text_tr2,\n    target=train.rating.values,\n    n_epochs=1,\n    batch_size=64,\n)\n# If you run this on a CPU and you sampled the data, the metrics will not be better than a random guess. Remember, this is just a demo\n</pre> # And from here on is the same as any other WideDeep model model = WideDeep(     deeptext=hf_model,     pred_dim=4, )  trainer = Trainer(     model,     objective=\"multiclass\",     metrics=[Accuracy(), F1Score(average=True)], )  trainer.fit(     X_text=X_text_tr2,     target=train.rating.values,     n_epochs=1,     batch_size=64, ) # If you run this on a CPU and you sampled the data, the metrics will not be better than a random guess. Remember, this is just a demo <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 13/13 [02:06&lt;00:00,  9.75s/it, loss=3.2, metrics={'acc': 0.235, 'f1': 0.2336}]  \n</pre> In\u00a0[17]: Copied! <pre>preds_text = trainer.predict_proba(X_text=X_text_te2)\npred_text_class = np.argmax(preds_text, 1)\n\nacc_text = accuracy_score(test.rating, pred_text_class)\nf1_text = f1_score(test.rating, pred_text_class, average=\"weighted\")\nprint(f\"Accuracy: {acc_text:.4f}\")\nprint(f\"F1: {f1_text:.4f}\")\n</pre> preds_text = trainer.predict_proba(X_text=X_text_te2) pred_text_class = np.argmax(preds_text, 1)  acc_text = accuracy_score(test.rating, pred_text_class) f1_text = f1_score(test.rating, pred_text_class, average=\"weighted\") print(f\"Accuracy: {acc_text:.4f}\") print(f\"F1: {f1_text:.4f}\") <pre>predict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 4/4 [00:05&lt;00:00,  1.43s/it]</pre> <pre>Accuracy: 0.2500\nF1: 0.1000\n</pre> <pre>\n</pre>"},{"location":"pytorch-widedeep/bayesian_models.html","title":"The <code>bayesian models</code> module","text":"<p>This module contains the two Bayesian Models available in this library, namely the bayesian version of the <code>Wide</code> and <code>TabMlp</code> models, referred as <code>BayesianWide</code> and <code>BayesianTabMlp</code>. These models are very useful in scenarios where getting a measure of uncertainty is important.</p> <p>The models in this module are based on the publication: Weight Uncertainty in Neural Networks.</p>"},{"location":"pytorch-widedeep/bayesian_models.html#pytorch_widedeep.bayesian_models.tabular.bayesian_linear.bayesian_wide.BayesianWide","title":"BayesianWide","text":"<p>               Bases: <code>BaseBayesianModel</code></p> <p>Defines a <code>Wide</code> model. This is a linear model where the non-linearlities are captured via crossed-columns</p> <p>Parameters:</p> Name Type Description Default <code>input_dim</code> <code>int</code> <p>size of the Embedding layer. <code>input_dim</code> is the summation of all the individual values for all the features that go through the wide component. For example, if the wide component receives 2 features with 5 individual values each, <code>input_dim = 10</code></p> required <code>pred_dim</code> <code>int</code> <p>size of the ouput tensor containing the predictions</p> <code>1</code> <code>prior_sigma_1</code> <code>float</code> <p>The prior weight distribution is a scaled mixture of two Gaussian densities:</p> \\[    \\begin{aligned}    P(\\mathbf{w}) = \\prod_{i=j} \\pi N (\\mathbf{w}_j | 0, \\sigma_{1}^{2}) + (1 - \\pi) N (\\mathbf{w}_j | 0, \\sigma_{2}^{2})    \\end{aligned} \\] <p><code>prior_sigma_1</code> is the prior of the sigma parameter for the first of the two Gaussians that will be mixed to produce the prior weight distribution.</p> <code>1.0</code> <code>prior_sigma_2</code> <code>float</code> <p>Prior of the sigma parameter for the second of the two Gaussian distributions that will be mixed to produce the prior weight distribution</p> <code>0.002</code> <code>prior_pi</code> <code>float</code> <p>Scaling factor that will be used to mix the Gaussians to produce the prior weight distribution</p> <code>0.8</code> <code>posterior_mu_init</code> <code>float</code> <p>The posterior sample of the weights is defined as:</p> \\[    \\begin{aligned}    \\mathbf{w} &amp;= \\mu + log(1 + exp(\\rho))    \\end{aligned} \\] <p>where:</p> \\[    \\begin{aligned}    \\mathcal{N}(x\\vert \\mu, \\sigma) &amp;= \\frac{1}{\\sqrt{2\\pi}\\sigma}e^{-\\frac{(x-\\mu)^2}{2\\sigma^2}}\\\\    \\log{\\mathcal{N}(x\\vert \\mu, \\sigma)} &amp;= -\\log{\\sqrt{2\\pi}} -\\log{\\sigma} -\\frac{(x-\\mu)^2}{2\\sigma^2}\\\\    \\end{aligned} \\] <p>\\(\\mu\\) is initialised using a normal distributtion with mean <code>posterior_mu_init</code> and std equal to 0.1.</p> <code>0.0</code> <code>posterior_rho_init</code> <code>float</code> <p>As in the case of \\(\\mu\\), \\(\\rho\\) is initialised using a normal distributtion with mean <code>posterior_rho_init</code> and std equal to 0.1.</p> <code>-7.0</code> <p>Attributes:</p> Name Type Description <code>bayesian_wide_linear</code> <code>Module</code> <p>the linear layer that comprises the wide branch of the model</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.bayesian_models import BayesianWide\n&gt;&gt;&gt; X = torch.empty(4, 4).random_(6)\n&gt;&gt;&gt; wide = BayesianWide(input_dim=int(X.max().item()), pred_dim=1)\n&gt;&gt;&gt; out = wide(X)\n</code></pre> Source code in <code>pytorch_widedeep/bayesian_models/tabular/bayesian_linear/bayesian_wide.py</code> <pre><code>class BayesianWide(BaseBayesianModel):\n    r\"\"\"Defines a `Wide` model. This is a linear model where the\n    non-linearlities are captured via crossed-columns\n\n    Parameters\n    ----------\n    input_dim: int\n        size of the Embedding layer. `input_dim` is the summation of all the\n        individual values for all the features that go through the wide\n        component. For example, if the wide component receives 2 features with\n        5 individual values each, `input_dim = 10`\n    pred_dim: int\n        size of the ouput tensor containing the predictions\n    prior_sigma_1: float, default = 1.0\n        The prior weight distribution is a scaled mixture of two Gaussian\n        densities:\n\n        $$\n           \\begin{aligned}\n           P(\\mathbf{w}) = \\prod_{i=j} \\pi N (\\mathbf{w}_j | 0, \\sigma_{1}^{2}) + (1 - \\pi) N (\\mathbf{w}_j | 0, \\sigma_{2}^{2})\n           \\end{aligned}\n        $$\n\n        `prior_sigma_1` is the prior of the sigma parameter for the first of the two\n        Gaussians that will be mixed to produce the prior weight\n        distribution.\n    prior_sigma_2: float, default = 0.002\n        Prior of the sigma parameter for the second of the two Gaussian\n        distributions that will be mixed to produce the prior weight\n        distribution\n    prior_pi: float, default = 0.8\n        Scaling factor that will be used to mix the Gaussians to produce the\n        prior weight distribution\n    posterior_mu_init: float = 0.0\n        The posterior sample of the weights is defined as:\n\n        $$\n           \\begin{aligned}\n           \\mathbf{w} &amp;= \\mu + log(1 + exp(\\rho))\n           \\end{aligned}\n        $$\n\n        where:\n\n        $$\n           \\begin{aligned}\n           \\mathcal{N}(x\\vert \\mu, \\sigma) &amp;= \\frac{1}{\\sqrt{2\\pi}\\sigma}e^{-\\frac{(x-\\mu)^2}{2\\sigma^2}}\\\\\n           \\log{\\mathcal{N}(x\\vert \\mu, \\sigma)} &amp;= -\\log{\\sqrt{2\\pi}} -\\log{\\sigma} -\\frac{(x-\\mu)^2}{2\\sigma^2}\\\\\n           \\end{aligned}\n        $$\n\n        $\\mu$ is initialised using a normal distributtion with mean\n        `posterior_mu_init` and std equal to 0.1.\n    posterior_rho_init: float = -7.0\n        As in the case of $\\mu$, $\\rho$ is initialised using a\n        normal distributtion with mean `posterior_rho_init` and std equal to\n        0.1.\n\n    Attributes\n    -----------\n    bayesian_wide_linear: nn.Module\n        the linear layer that comprises the wide branch of the model\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.bayesian_models import BayesianWide\n    &gt;&gt;&gt; X = torch.empty(4, 4).random_(6)\n    &gt;&gt;&gt; wide = BayesianWide(input_dim=int(X.max().item()), pred_dim=1)\n    &gt;&gt;&gt; out = wide(X)\n    \"\"\"\n\n    def __init__(\n        self,\n        input_dim: int,\n        pred_dim: int = 1,\n        prior_sigma_1: float = 1.0,\n        prior_sigma_2: float = 0.002,\n        prior_pi: float = 0.8,\n        posterior_mu_init: float = 0.0,\n        posterior_rho_init: float = -7.0,\n    ):\n        super(BayesianWide, self).__init__()\n        #  Embeddings: val + 1 because 0 is reserved for padding/unseen cateogories.\n        self.bayesian_wide_linear = bnn.BayesianEmbedding(\n            n_embed=input_dim + 1,\n            embed_dim=pred_dim,\n            padding_idx=0,\n            prior_sigma_1=prior_sigma_1,\n            prior_sigma_2=prior_sigma_2,\n            prior_pi=prior_pi,\n            posterior_mu_init=posterior_mu_init,\n            posterior_rho_init=posterior_rho_init,\n        )\n        self.bias = nn.Parameter(torch.zeros(pred_dim))\n\n    def forward(self, X: Tensor) -&gt; Tensor:\n        out = self.bayesian_wide_linear(X.long()).sum(dim=1) + self.bias\n        return out\n</code></pre>"},{"location":"pytorch-widedeep/bayesian_models.html#pytorch_widedeep.bayesian_models.tabular.bayesian_mlp.bayesian_tab_mlp.BayesianTabMlp","title":"BayesianTabMlp","text":"<p>               Bases: <code>BaseBayesianModel</code></p> <p>Defines a <code>BayesianTabMlp</code> model.</p> <p>This class combines embedding representations of the categorical features with numerical (aka continuous) features, embedded or not. These are then passed through a series of probabilistic dense layers (i.e. a MLP).</p> <p>Parameters:</p> Name Type Description Default <code>column_idx</code> <code>Dict[str, int]</code> <p>Dict containing the index of the columns that will be passed through the <code>TabMlp</code> model. Required to slice the tensors. e.g. {'education': 0, 'relationship': 1, 'workclass': 2, ...}</p> required <code>cat_embed_input</code> <code>Optional[List[Tuple[str, int, int]]]</code> <p>List of Tuples with the column name, number of unique values and embedding dimension. e.g. [(education, 11, 32), ...]</p> <code>None</code> <code>cat_embed_activation</code> <code>Optional[str]</code> <p>Activation function for the categorical embeddings, if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> <code>None</code> <code>continuous_cols</code> <code>Optional[List[str]]</code> <p>List with the name of the numeric (aka continuous) columns</p> <code>None</code> <code>cont_norm_layer</code> <code>Optional[Literal[batchnorm, layernorm]]</code> <p>Type of normalization layer applied to the continuous features. Options are: 'layernorm', 'batchnorm' or None.</p> <code>None</code> <code>embed_continuous</code> <code>Optional[bool]</code> <p>Boolean indicating if the continuous columns will be embedded (i.e. passed each through a linear layer with or without activation)</p> <code>None</code> <code>cont_embed_dim</code> <code>Optional[int]</code> <p>Size of the continuous embeddings</p> <code>None</code> <code>cont_embed_dropout</code> <code>Optional[float]</code> <p>Dropout for the continuous embeddings</p> <code>None</code> <code>use_cont_bias</code> <code>Optional[bool]</code> <p>Boolean indicating if bias will be used for the continuous embeddings</p> <code>None</code> <code>cont_embed_activation</code> <code>Optional[str]</code> <p>Activation function for the continuous embeddings if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> <code>None</code> <code>mlp_hidden_dims</code> <code>List[int]</code> <p>List with the number of neurons per dense layer in the mlp.</p> <code>[200, 100]</code> <code>mlp_activation</code> <code>str</code> <p>Activation function for the dense layers of the MLP. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> <code>'leaky_relu'</code> <code>prior_sigma_1</code> <code>float</code> <p>The prior weight distribution is a scaled mixture of two Gaussian densities:</p> \\[    \\begin{aligned}    P(\\mathbf{w}) = \\prod_{i=j} \\pi N (\\mathbf{w}_j | 0, \\sigma_{1}^{2}) + (1 - \\pi) N (\\mathbf{w}_j | 0, \\sigma_{2}^{2})    \\end{aligned} \\] <p><code>prior_sigma_1</code> is the prior of the sigma parameter for the first of the two Gaussians that will be mixed to produce the prior weight distribution.</p> <code>1</code> <code>prior_sigma_2</code> <code>float</code> <p>Prior of the sigma parameter for the second of the two Gaussian distributions that will be mixed to produce the prior weight distribution for each Bayesian linear and embedding layer</p> <code>0.002</code> <code>prior_pi</code> <code>float</code> <p>Scaling factor that will be used to mix the Gaussians to produce the prior weight distribution ffor each Bayesian linear and embedding layer</p> <code>0.8</code> <code>posterior_mu_init</code> <code>float</code> <p>The posterior sample of the weights is defined as:</p> <p>$$    \\begin{aligned}    \\mathbf{w} &amp;= \\mu + log(1 + exp(\\rho))    \\end{aligned} $$ where:</p> \\[    \\begin{aligned}    \\mathcal{N}(x\\vert \\mu, \\sigma) &amp;= \\frac{1}{\\sqrt{2\\pi}\\sigma}e^{-\\frac{(x-\\mu)^2}{2\\sigma^2}}\\\\    \\log{\\mathcal{N}(x\\vert \\mu, \\sigma)} &amp;= -\\log{\\sqrt{2\\pi}} -\\log{\\sigma} -\\frac{(x-\\mu)^2}{2\\sigma^2}\\\\    \\end{aligned} \\] <p>\\(\\mu\\) is initialised using a normal distributtion with mean <code>posterior_mu_init</code> and std equal to 0.1.</p> <code>0.0</code> <code>posterior_rho_init</code> <code>float</code> <p>As in the case of \\(\\mu\\), \\(\\rho\\) is initialised using a normal distributtion with mean <code>posterior_rho_init</code> and std equal to 0.1.</p> <code>-7.0</code> <p>Attributes:</p> Name Type Description <code>bayesian_cat_and_cont_embed</code> <code>Module</code> <p>This is the module that processes the categorical and continuous columns</p> <code>bayesian_tab_mlp</code> <code>Sequential</code> <p>mlp model that will receive the concatenation of the embeddings and the continuous columns</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.bayesian_models import BayesianTabMlp\n&gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n&gt;&gt;&gt; colnames = ['a', 'b', 'c', 'd', 'e']\n&gt;&gt;&gt; cat_embed_input = [(u,i,j) for u,i,j in zip(colnames[:4], [4]*4, [8]*4)]\n&gt;&gt;&gt; column_idx = {k:v for v,k in enumerate(colnames)}\n&gt;&gt;&gt; model = BayesianTabMlp(mlp_hidden_dims=[8,4], column_idx=column_idx, cat_embed_input=cat_embed_input,\n... continuous_cols = ['e'])\n&gt;&gt;&gt; out = model(X_tab)\n</code></pre> Source code in <code>pytorch_widedeep/bayesian_models/tabular/bayesian_mlp/bayesian_tab_mlp.py</code> <pre><code>class BayesianTabMlp(BaseBayesianModel):\n    r\"\"\"Defines a `BayesianTabMlp` model.\n\n    This class combines embedding representations of the categorical features\n    with numerical (aka continuous) features, embedded or not. These are then\n    passed through a series of probabilistic dense layers (i.e. a MLP).\n\n    Parameters\n    ----------\n    column_idx: Dict\n        Dict containing the index of the columns that will be passed through\n        the `TabMlp` model. Required to slice the tensors. e.g. _{'education':\n        0, 'relationship': 1, 'workclass': 2, ...}_\n    cat_embed_input: List, Optional, default = None\n        List of Tuples with the column name, number of unique values and\n        embedding dimension. e.g. _[(education, 11, 32), ...]_\n    cat_embed_activation: Optional, str, default = None,\n        Activation function for the categorical embeddings, if any. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported\n    continuous_cols: List, Optional, default = None\n        List with the name of the numeric (aka continuous) columns\n    cont_norm_layer: str, default =  \"batchnorm\"\n        Type of normalization layer applied to the continuous features. Options\n        are: 'layernorm', 'batchnorm' or None.\n    embed_continuous: bool, default = False,\n        Boolean indicating if the continuous columns will be embedded\n        (i.e. passed each through a linear layer with or without activation)\n    cont_embed_dim: int, default = 32,\n        Size of the continuous embeddings\n    cont_embed_dropout: float, default = 0.1,\n        Dropout for the continuous embeddings\n    use_cont_bias: bool, default = True,\n        Boolean indicating if bias will be used for the continuous embeddings\n    cont_embed_activation: Optional, str, default = None,\n        Activation function for the continuous embeddings if any. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported\n    mlp_hidden_dims: List, default = [200, 100]\n        List with the number of neurons per dense layer in the mlp.\n    mlp_activation: str, default = \"relu\"\n        Activation function for the dense layers of the MLP. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported\n    prior_sigma_1: float, default = 1.0\n        The prior weight distribution is a scaled mixture of two Gaussian\n        densities:\n\n        $$\n           \\begin{aligned}\n           P(\\mathbf{w}) = \\prod_{i=j} \\pi N (\\mathbf{w}_j | 0, \\sigma_{1}^{2}) + (1 - \\pi) N (\\mathbf{w}_j | 0, \\sigma_{2}^{2})\n           \\end{aligned}\n        $$\n\n        `prior_sigma_1` is the prior of the sigma parameter for the first of the two\n        Gaussians that will be mixed to produce the prior weight\n        distribution.\n    prior_sigma_2: float, default = 0.002\n        Prior of the sigma parameter for the second of the two Gaussian\n        distributions that will be mixed to produce the prior weight\n        distribution for each Bayesian linear and embedding layer\n    prior_pi: float, default = 0.8\n        Scaling factor that will be used to mix the Gaussians to produce the\n        prior weight distribution ffor each Bayesian linear and embedding\n        layer\n    posterior_mu_init: float = 0.0\n        The posterior sample of the weights is defined as:\n\n        $$\n           \\begin{aligned}\n           \\mathbf{w} &amp;= \\mu + log(1 + exp(\\rho))\n           \\end{aligned}\n        $$\n        where:\n\n        $$\n           \\begin{aligned}\n           \\mathcal{N}(x\\vert \\mu, \\sigma) &amp;= \\frac{1}{\\sqrt{2\\pi}\\sigma}e^{-\\frac{(x-\\mu)^2}{2\\sigma^2}}\\\\\n           \\log{\\mathcal{N}(x\\vert \\mu, \\sigma)} &amp;= -\\log{\\sqrt{2\\pi}} -\\log{\\sigma} -\\frac{(x-\\mu)^2}{2\\sigma^2}\\\\\n           \\end{aligned}\n        $$\n\n        $\\mu$ is initialised using a normal distributtion with mean\n        `posterior_mu_init` and std equal to 0.1.\n    posterior_rho_init: float = -7.0\n        As in the case of $\\mu$, $\\rho$ is initialised using a\n        normal distributtion with mean `posterior_rho_init` and std equal to\n        0.1.\n\n    Attributes\n    ----------\n    bayesian_cat_and_cont_embed: nn.Module\n        This is the module that processes the categorical and continuous columns\n    bayesian_tab_mlp: nn.Sequential\n        mlp model that will receive the concatenation of the embeddings and\n        the continuous columns\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.bayesian_models import BayesianTabMlp\n    &gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n    &gt;&gt;&gt; colnames = ['a', 'b', 'c', 'd', 'e']\n    &gt;&gt;&gt; cat_embed_input = [(u,i,j) for u,i,j in zip(colnames[:4], [4]*4, [8]*4)]\n    &gt;&gt;&gt; column_idx = {k:v for v,k in enumerate(colnames)}\n    &gt;&gt;&gt; model = BayesianTabMlp(mlp_hidden_dims=[8,4], column_idx=column_idx, cat_embed_input=cat_embed_input,\n    ... continuous_cols = ['e'])\n    &gt;&gt;&gt; out = model(X_tab)\n    \"\"\"\n\n    def __init__(\n        self,\n        column_idx: Dict[str, int],\n        *,\n        cat_embed_input: Optional[List[Tuple[str, int, int]]] = None,\n        cat_embed_activation: Optional[str] = None,\n        continuous_cols: Optional[List[str]] = None,\n        embed_continuous: Optional[bool] = None,\n        cont_embed_dim: Optional[int] = None,\n        cont_embed_dropout: Optional[float] = None,\n        cont_embed_activation: Optional[str] = None,\n        use_cont_bias: Optional[bool] = None,\n        cont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\n        mlp_hidden_dims: List[int] = [200, 100],\n        mlp_activation: str = \"leaky_relu\",\n        prior_sigma_1: float = 1,\n        prior_sigma_2: float = 0.002,\n        prior_pi: float = 0.8,\n        posterior_mu_init: float = 0.0,\n        posterior_rho_init: float = -7.0,\n        pred_dim=1,  # Bayesian models will require their own trainer and need the output layer\n    ):\n        super(BayesianTabMlp, self).__init__()\n\n        self.column_idx = column_idx\n        self.cat_embed_input = cat_embed_input\n        self.cat_embed_activation = cat_embed_activation\n\n        self.continuous_cols = continuous_cols\n        self.cont_norm_layer = cont_norm_layer\n        self.embed_continuous = embed_continuous\n        self.cont_embed_dim = cont_embed_dim\n        self.cont_embed_dropout = cont_embed_dropout\n        self.use_cont_bias = use_cont_bias\n        self.cont_embed_activation = cont_embed_activation\n\n        self.mlp_hidden_dims = mlp_hidden_dims\n        self.mlp_activation = mlp_activation\n\n        self.prior_sigma_1 = prior_sigma_1\n        self.prior_sigma_2 = prior_sigma_2\n        self.prior_pi = prior_pi\n        self.posterior_mu_init = posterior_mu_init\n        self.posterior_rho_init = posterior_rho_init\n\n        self.pred_dim = pred_dim\n\n        allowed_activations = [\"relu\", \"leaky_relu\", \"tanh\", \"gelu\"]\n        if self.mlp_activation not in allowed_activations:\n            raise ValueError(\n                \"Currently, only the following activation functions are supported \"\n                \"for the Bayesian MLP's dense layers: {}. Got '{}' instead\".format(\n                    \", \".join(allowed_activations),\n                    self.mlp_activation,\n                )\n            )\n\n        # Categorical\n        if self.cat_embed_input is not None:\n            self.cat_embed = BayesianDiffSizeCatEmbeddings(\n                column_idx=self.column_idx,\n                embed_input=self.cat_embed_input,\n                prior_sigma_1=self.prior_sigma_1,\n                prior_sigma_2=self.prior_sigma_2,\n                prior_pi=self.prior_pi,\n                posterior_mu_init=self.posterior_mu_init,\n                posterior_rho_init=self.posterior_rho_init,\n                activation_fn=self.cat_embed_activation,\n            )\n            self.cat_out_dim = int(np.sum([embed[2] for embed in self.cat_embed_input]))\n        else:\n            self.cat_out_dim = 0\n\n        # Continuous\n        if self.continuous_cols is not None:\n            self.cont_idx = [self.column_idx[col] for col in self.continuous_cols]\n            if cont_norm_layer == \"layernorm\":\n                self.cont_norm: NormLayers = nn.LayerNorm(len(self.continuous_cols))\n            elif cont_norm_layer == \"batchnorm\":\n                self.cont_norm = nn.BatchNorm1d(len(self.continuous_cols))\n            else:\n                self.cont_norm = nn.Identity()\n            if self.embed_continuous:\n                assert self.cont_embed_dim is not None, (\n                    \"If 'embed_continuous' is True, 'cont_embed_dim' must be \"\n                    \"provided\"\n                )\n                self.cont_embed = BayesianContEmbeddings(\n                    n_cont_cols=len(self.continuous_cols),\n                    embed_dim=self.cont_embed_dim,\n                    prior_sigma_1=self.prior_sigma_1,\n                    prior_sigma_2=self.prior_sigma_2,\n                    prior_pi=self.prior_pi,\n                    posterior_mu_init=self.posterior_mu_init,\n                    posterior_rho_init=self.posterior_rho_init,\n                    use_bias=(\n                        False if self.use_cont_bias is None else self.use_cont_bias\n                    ),\n                    activation_fn=self.cont_embed_activation,\n                )\n                self.cont_out_dim = len(self.continuous_cols) * self.cont_embed_dim\n            else:\n                self.cont_out_dim = len(self.continuous_cols)\n        else:\n            self.cont_out_dim = 0\n\n        self.output_dim = self.cat_out_dim + self.cont_out_dim\n\n        mlp_hidden_dims = [self.output_dim] + mlp_hidden_dims + [pred_dim]\n        self.bayesian_tab_mlp = BayesianMLP(\n            mlp_hidden_dims,\n            mlp_activation,\n            True,  # use_bias\n            prior_sigma_1,\n            prior_sigma_2,\n            prior_pi,\n            posterior_mu_init,\n            posterior_rho_init,\n        )\n\n    def forward(self, X: Tensor) -&gt; Tensor:\n        x = self._get_embeddings(X)\n        x = self.bayesian_tab_mlp(x)\n        return x\n\n    def _get_embeddings(self, X: Tensor) -&gt; Tensor:\n        tensors_to_concat: List[Tensor] = []\n        if self.cat_embed_input is not None:\n            x_cat = self.cat_embed(X)\n            tensors_to_concat.append(x_cat)\n\n        if self.continuous_cols is not None:\n            x_cont = self.cont_norm((X[:, self.cont_idx].float()))\n            if self.embed_continuous:\n                x_cont = self.cont_embed(x_cont)\n                x_cont = einops.rearrange(x_cont, \"b s d -&gt; b (s d)\")\n            tensors_to_concat.append(x_cont)\n\n        x = torch.cat(tensors_to_concat, 1)\n\n        return x\n</code></pre>"},{"location":"pytorch-widedeep/bayesian_trainer.html","title":"Training Deep Learning Probabilistic Models","text":""},{"location":"pytorch-widedeep/bayesian_trainer.html#pytorch_widedeep.training.BayesianTrainer","title":"BayesianTrainer","text":"<p>               Bases: <code>BaseBayesianTrainer</code></p> <p>Class to set the of attributes that will be used during the training process.</p> <p>Both the Bayesian models and the Trainer in this repo are based on the paper: Weight Uncertainty in Neural Networks.</p> <p>Parameters:</p> Name Type Description Default <code>model</code> <code>BaseBayesianModel</code> <p>An object of class <code>BaseBayesianModel</code>. See the <code>Model Components</code> section here in the docs.</p> required <code>objective</code> <code>str</code> <p>Defines the objective, loss or cost function. Param aliases: <code>loss_function</code>, <code>loss_fn</code>, <code>loss</code>, <code>cost_function</code>, <code>cost_fn</code>, <code>cost</code> Possible values are: 'binary', 'multiclass', 'regression'</p> required <code>custom_loss_function</code> <code>Optional[Module]</code> <p>If none of the loss functions available suits the user, it is possible to pass a custom loss function. See for example <code>pytorch_widedeep.losses.FocalLoss</code> for the required structure of the object or the Examples folder in the repo.</p> <code>None</code> <code>optimizer</code> <code>Optional[Optimizer]</code> <p>An instance of Pytorch's <code>Optimizer</code> object(e.g. <code>torch.optim.Adam ()</code>). if no optimizer is passed it will default to <code>AdamW</code>.</p> <code>None</code> <code>lr_scheduler</code> <code>Optional[LRScheduler]</code> <p>An instance of Pytorch's <code>LRScheduler</code> object (e.g <code>torch.optim.lr_scheduler.StepLR(opt, step_size=5)</code>).</p> <code>None</code> <code>callbacks</code> <code>Optional[List[Callback]]</code> <p>List with <code>Callback</code> objects. The three callbacks available in <code>pytorch-widedeep</code> are: <code>LRHistory</code>, <code>ModelCheckpoint</code> and <code>EarlyStopping</code>. This can also be a custom callback. See <code>pytorch_widedeep.callbacks.Callback</code> or the Examples folder in the repo.</p> <code>None</code> <code>metrics</code> <code>Optional[Union[List[Metric], List[Metric]]]</code> <ul> <li>List of objects of type <code>Metric</code>. Metrics available are:   <code>Accuracy</code>, <code>Precision</code>, <code>Recall</code>, <code>FBetaScore</code>,   <code>F1Score</code> and <code>R2Score</code>. This can also be a custom metric as   long as it is an object of type <code>Metric</code>. See   <code>pytorch_widedeep.metrics.Metric</code> or the Examples folder in the repo</li> <li>List of objects of type <code>torchmetrics.Metric</code>. This can be any   metric from torchmetrics library Examples   classification-metrics&gt;<code>_. It can also be a torchmetric custom metric as   long as it is an object of type</code>Metric<code>.   See</code>the instructions</li> </ul> <code>None</code> <code>verbose</code> <code>int</code> <p>Setting it to 0 will print nothing during training.</p> <code>1</code> <code>seed</code> <code>int</code> <p>Random seed to be used internally for train_test_split</p> <code>1</code> <p>Other Parameters:</p> Name Type Description <code>**kwargs</code> <p>Other infrequently used arguments that can also be passed as kwargs are:</p> <ul> <li> <p>device: <code>str</code>     string indicating the device. One of 'cpu', 'gpu' or 'mps' if     run on a Mac with Apple silicon or AMD GPU(s)</p> </li> <li> <p>num_workers: <code>int</code>     number of workers to be used internally by the data loaders</p> </li> <li> <p>class_weight: <code>List[float]</code>     This is the <code>weight</code> or <code>pos_weight</code> parameter in     <code>CrossEntropyLoss</code> and <code>BCEWithLogitsLoss</code>, depending on whether</p> </li> <li> <p>reducelronplateau_criterion: <code>str</code>     This sets the criterion that will be used by the lr scheduler to     take a step: One of 'loss' or 'metric'. The ReduceLROnPlateau     learning rate is a bit particular.</p> </li> </ul> <p>Attributes:</p> Name Type Description <code>cyclic_lr</code> <code>bool</code> <p>Attribute that indicates if  the lr_scheduler is cyclic_lr (i.e. <code>CyclicLR</code> or <code>OneCycleLR</code>). See <code>Pytorch schedulers &lt;https://pytorch.org/docs/stable/optim.html&gt;</code>_.</p> Source code in <code>pytorch_widedeep/training/bayesian_trainer.py</code> <pre><code>class BayesianTrainer(BaseBayesianTrainer):\n    r\"\"\"Class to set the of attributes that will be used during the\n    training process.\n\n    Both the Bayesian models and the Trainer in this repo are based on the paper:\n    [Weight Uncertainty in Neural Networks](https://arxiv.org/pdf/1505.05424.pdf).\n\n    Parameters\n    ----------\n    model: `BaseBayesianModel`\n        An object of class `BaseBayesianModel`. See the `Model Components`\n        section here in the docs.\n    objective: str\n        Defines the objective, loss or cost function.&lt;br/&gt;\n        Param aliases: `loss_function`, `loss_fn`, `loss`,\n        `cost_function`, `cost_fn`, `cost`&lt;br/&gt;\n        Possible values are: _'binary'_, _'multiclass'_, _'regression'_\n    custom_loss_function: `nn.Module`, optional, default = None\n        If none of the loss functions available suits the user, it is possible\n        to pass a custom loss function. See for example\n        `pytorch_widedeep.losses.FocalLoss` for the required structure of the\n        object or the Examples folder in the repo.\n    optimizer: `Optimzer`, optional, default= None\n        An instance of Pytorch's `Optimizer` object(e.g. `torch.optim.Adam\n        ()`). if no optimizer is passed it will default to `AdamW`.\n    lr_scheduler: `LRScheduler`, optional, default=None\n        An instance of Pytorch's `LRScheduler` object\n        (e.g `torch.optim.lr_scheduler.StepLR(opt, step_size=5)`).\n    callbacks: List, optional, default=None\n        List with `Callback` objects. The three callbacks available in\n        `pytorch-widedeep` are: `LRHistory`, `ModelCheckpoint` and\n        `EarlyStopping`. This can also be a custom callback. See\n        `pytorch_widedeep.callbacks.Callback` or the Examples folder in the\n        repo.\n    metrics: List, optional, default=None\n        - List of objects of type `Metric`. Metrics available are:\n          `Accuracy`, `Precision`, `Recall`, `FBetaScore`,\n          `F1Score` and `R2Score`. This can also be a custom metric as\n          long as it is an object of type `Metric`. See\n          `pytorch_widedeep.metrics.Metric` or the Examples folder in the repo\n        - List of objects of type `torchmetrics.Metric`. This can be any\n          metric from torchmetrics library [Examples](https://lightning.ai/docs/torchmetrics)\n          classification-metrics&gt;`_. It can also be a torchmetric custom metric as\n          long as it is an object of type `Metric`.\n          See `the [instructions]((https://lightning.ai/docs/torchmetrics))\n    verbose: int, default=1\n        Setting it to 0 will print nothing during training.\n    seed: int, default=1\n        Random seed to be used internally for train_test_split\n\n    Other Parameters\n    ----------------\n    **kwargs: dict\n        Other infrequently used arguments that can also be passed as kwargs are:\n\n        - **device**: `str`&lt;br/&gt;\n            string indicating the device. One of _'cpu'_, _'gpu'_ or 'mps' if\n            run on a Mac with Apple silicon or AMD GPU(s)\n\n        - **num_workers**: `int`&lt;br/&gt;\n            number of workers to be used internally by the data loaders\n\n        - **class_weight**: `List[float]`&lt;br/&gt;\n            This is the `weight` or `pos_weight` parameter in\n            `CrossEntropyLoss` and `BCEWithLogitsLoss`, depending on whether\n\n        - **reducelronplateau_criterion**: `str`\n            This sets the criterion that will be used by the lr scheduler to\n            take a step: One of _'loss'_ or _'metric'_. The ReduceLROnPlateau\n            learning rate is a bit particular.\n\n    Attributes\n    ----------\n    cyclic_lr: bool\n        Attribute that indicates if  the lr_scheduler is cyclic_lr\n        (i.e. `CyclicLR` or `OneCycleLR`). See `Pytorch schedulers\n        &lt;https://pytorch.org/docs/stable/optim.html&gt;`_.\n    \"\"\"\n\n    @alias(  # noqa: C901\n        \"objective\",\n        [\"loss_function\", \"loss_fn\", \"loss\", \"cost_function\", \"cost_fn\", \"cost\"],\n    )\n    def __init__(\n        self,\n        model: BaseBayesianModel,\n        objective: str,\n        custom_loss_function: Optional[Module] = None,\n        optimizer: Optional[Optimizer] = None,\n        lr_scheduler: Optional[LRScheduler] = None,\n        callbacks: Optional[List[Callback]] = None,\n        metrics: Optional[Union[List[Metric], List[TorchMetric]]] = None,\n        verbose: int = 1,\n        seed: int = 1,\n        **kwargs,\n    ):\n        super().__init__(\n            model=model,\n            objective=objective,\n            custom_loss_function=custom_loss_function,\n            optimizer=optimizer,\n            lr_scheduler=lr_scheduler,\n            callbacks=callbacks,\n            metrics=metrics,\n            verbose=verbose,\n            seed=seed,\n            **kwargs,\n        )\n\n    def fit(  # noqa: C901\n        self,\n        X_tab: np.ndarray,\n        target: np.ndarray,\n        X_tab_val: Optional[np.ndarray] = None,\n        target_val: Optional[np.ndarray] = None,\n        val_split: Optional[float] = None,\n        n_epochs: int = 1,\n        validation_freq: int = 1,\n        batch_size: int = 32,\n        n_train_samples: int = 2,\n        n_val_samples: int = 2,\n    ):\n        r\"\"\"Fit method.\n\n        Parameters\n        ----------\n        X_tab: np.ndarray,\n            tabular dataset\n        target: np.ndarray\n            target values\n        X_tab_val: np.ndarray, Optional, default = None\n            validation data\n        target_val: np.ndarray, Optional, default = None\n            validation target values\n        val_split: float, Optional. default=None\n            An alterative to passing the validation set is to use a train/val\n            split fraction via `val_split`\n        n_epochs: int, default=1\n            number of epochs\n        validation_freq: int, default=1\n            epochs validation frequency\n        batch_size: int, default=32\n            batch size\n        n_train_samples: int, default=2\n            number of samples to average over during the training process.\n            See [Weight Uncertainty in Neural Networks](https://arxiv.org/pdf/1505.05424.pdf) for details.\n        n_val_samples: int, default=2\n            number of samples to average over during the validation process.\n            See [Weight Uncertainty in Neural Networks](https://arxiv.org/pdf/1505.05424.pdf) for details.\n        \"\"\"\n\n        self.batch_size = batch_size\n\n        train_set, eval_set = tabular_train_val_split(\n            self.seed, self.objective, X_tab, target, X_tab_val, target_val, val_split\n        )\n        train_loader = DataLoader(\n            dataset=train_set, batch_size=batch_size, num_workers=self.num_workers\n        )\n        train_steps = len(train_loader)\n\n        if eval_set is not None:\n            eval_loader = DataLoader(\n                dataset=eval_set,\n                batch_size=batch_size,\n                num_workers=self.num_workers,\n                shuffle=False,\n            )\n            eval_steps = len(eval_loader)\n\n        self.callback_container.on_train_begin(\n            {\n                \"batch_size\": batch_size,\n                \"train_steps\": train_steps,\n                \"n_epochs\": n_epochs,\n            }\n        )\n        for epoch in range(n_epochs):\n            epoch_logs: Dict[str, float] = {}\n            self.callback_container.on_epoch_begin(epoch, logs=epoch_logs)\n\n            self.train_running_loss = 0.0\n            with trange(train_steps, disable=self.verbose != 1) as t:\n                for batch_idx, (X, y) in zip(t, train_loader):\n                    t.set_description(\"epoch %i\" % (epoch + 1))\n                    train_score, train_loss = self._train_step(\n                        X, y, n_train_samples, train_steps, batch_idx\n                    )\n                    print_loss_and_metric(t, train_loss, train_score)\n                    self.callback_container.on_batch_end(batch=batch_idx)\n            epoch_logs = save_epoch_logs(epoch_logs, train_loss, train_score, \"train\")\n\n            on_epoch_end_metric = None\n            if eval_set is not None and epoch % validation_freq == (\n                validation_freq - 1\n            ):\n                self.callback_container.on_eval_begin()\n                self.valid_running_loss = 0.0\n                with trange(eval_steps, disable=self.verbose != 1) as v:\n                    for i, (X, y) in zip(v, eval_loader):\n                        v.set_description(\"valid\")\n                        val_score, val_loss = self._eval_step(\n                            X, y, n_val_samples, train_steps, i\n                        )\n                        print_loss_and_metric(v, val_loss, val_score)\n                epoch_logs = save_epoch_logs(epoch_logs, val_loss, val_score, \"val\")\n\n                if self.reducelronplateau:\n                    if self.reducelronplateau_criterion == \"loss\":\n                        on_epoch_end_metric = val_loss\n                    else:\n                        on_epoch_end_metric = val_score[\n                            self.reducelronplateau_criterion\n                        ]\n\n            self.callback_container.on_epoch_end(epoch, epoch_logs, on_epoch_end_metric)\n\n            if self.early_stop:\n                self.callback_container.on_train_end(epoch_logs)\n                break\n\n        self.callback_container.on_train_end(epoch_logs)\n        self._restore_best_weights()\n        self.model.train()\n\n    def predict(  # type: ignore[return]\n        self,\n        X_tab: np.ndarray,\n        n_samples: int = 5,\n        return_samples: bool = False,\n        batch_size: int = 256,\n    ) -&gt; np.ndarray:\n        r\"\"\"Returns the predictions\n\n        Parameters\n        ----------\n        X_tab: np.ndarray,\n            tabular dataset\n        n_samples: int, default=5\n            number of samples that will be either returned or averaged to\n            produce an overal prediction\n        return_samples: bool, default = False\n            Boolean indicating whether the n samples will be averaged or directly returned\n        batch_size: int, default = 256\n            batch size\n\n        Returns\n        -------\n        np.ndarray:\n            array with the predictions\n        \"\"\"\n\n        preds_l = self._predict(X_tab, n_samples, return_samples, batch_size)\n        preds = np.hstack(preds_l) if return_samples else np.vstack(preds_l)\n        axis = 2 if return_samples else 1\n\n        if self.objective == \"regression\":\n            return preds.squeeze(axis)\n        if self.objective == \"binary\":\n            return (preds.squeeze(axis) &gt; 0.5).astype(\"int\")\n        if self.objective == \"multiclass\":\n            return np.argmax(preds, axis)\n\n    def predict_proba(  # type: ignore[return]\n        self,\n        X_tab: np.ndarray,\n        n_samples: int = 5,\n        return_samples: bool = False,\n        batch_size: int = 256,\n    ) -&gt; np.ndarray:\n        r\"\"\"Returns the predicted probabilities\n\n        Parameters\n        ----------\n        X_tab: np.ndarray,\n            tabular dataset\n        n_samples: int, default=5\n            number of samples that will be either returned or averaged to\n            produce an overal prediction\n        return_samples: bool, default = False\n            Boolean indicating whether the n samples will be averaged or directly returned\n        batch_size: int, default = 256\n            batch size\n\n        Returns\n        -------\n        np.ndarray\n            array with the probabilities per class\n        \"\"\"\n        preds_l = self._predict(X_tab, n_samples, return_samples, batch_size)\n        preds = np.hstack(preds_l) if return_samples else np.vstack(preds_l)\n\n        if self.objective == \"binary\":\n            if return_samples:\n                preds = preds.squeeze(2)\n                probs = np.zeros([n_samples, preds.shape[1], 2])\n                for i in range(n_samples):\n                    probs[i, :, 0] = 1 - preds[i]\n                    probs[i, :, 1] = preds[i]\n            else:\n                preds = preds.squeeze(1)\n                probs = np.zeros([preds.shape[0], 2])\n                probs[:, 0] = 1 - preds\n                probs[:, 1] = preds\n            return probs\n        if self.objective == \"multiclass\":\n            return preds\n\n    def save(\n        self,\n        path: str,\n        save_state_dict: bool = False,\n        model_filename: str = \"bayesian_model.pt\",\n    ):\n        r\"\"\"Saves the model, training and evaluation history to disk\n\n        The `Trainer` class is built so that it 'just' trains a model. With\n        that in mind, all the torch related parameters (such as optimizers or\n        learning rate schedulers) have to be defined externally and then\n        passed to the `Trainer`. As a result, the `Trainer` does not\n        generate any attribute or additional data products that need to be\n        saved other than the `model` object itself, which can be saved as\n        any other torch model (e.g. `torch.save(model, path)`).\n\n        Parameters\n        ----------\n        path: str\n            path to the directory where the model and the feature importance\n            attribute will be saved.\n        save_state_dict: bool, default = False\n            Boolean indicating whether to save directly the model or the\n            model's state dictionary\n        model_filename: str, Optional, default = \"wd_model.pt\"\n            filename where the model weights will be store\n        \"\"\"\n\n        save_dir = Path(path)\n        history_dir = save_dir / \"history\"\n        history_dir.mkdir(exist_ok=True, parents=True)\n\n        # the trainer is run with the History Callback by default\n        with open(history_dir / \"train_eval_history.json\", \"w\") as teh:\n            json.dump(self.history, teh)  # type: ignore[attr-defined]\n\n        has_lr_history = any(\n            [clbk.__class__.__name__ == \"LRHistory\" for clbk in self.callbacks]\n        )\n        if self.lr_scheduler is not None and has_lr_history:\n            with open(history_dir / \"lr_history.json\", \"w\") as lrh:\n                json.dump(self.lr_history, lrh)  # type: ignore[attr-defined]\n\n        model_path = save_dir / model_filename\n        if save_state_dict:\n            torch.save(self.model.state_dict(), model_path)\n        else:\n            torch.save(self.model, model_path)\n\n    def _train_step(\n        self,\n        X_tab: Tensor,\n        target: Tensor,\n        n_samples: int,\n        n_batches: int,\n        batch_idx: int,\n    ):\n        self.model.train()\n\n        X = to_device(X_tab, self.device)\n        y = target.view(-1, 1).float() if self.objective != \"multiclass\" else target\n        y = to_device(y, self.device)\n\n        self.optimizer.zero_grad()\n        y_pred, loss = self.model.sample_elbo(X, y, self.loss_fn, n_samples, n_batches)  # type: ignore[arg-type]\n\n        y_pred = y_pred.mean(dim=0)\n        score = self._get_score(y_pred, y)\n\n        loss.backward()\n        self.optimizer.step()\n\n        self.train_running_loss += loss.item()\n        avg_loss = self.train_running_loss / (batch_idx + 1)\n\n        return score, avg_loss\n\n    def _eval_step(\n        self,\n        X_tab: Tensor,\n        target: Tensor,\n        n_samples: int,\n        n_batches: int,\n        batch_idx: int,\n    ):\n        self.model.eval()\n        with torch.no_grad():\n            X = to_device(X_tab, self.device)\n            y = target.view(-1, 1).float() if self.objective != \"multiclass\" else target\n            y = to_device(y, self.device)\n\n            y_pred, loss = self.model.sample_elbo(\n                X,  # type: ignore[arg-type]\n                y,\n                self.loss_fn,\n                n_samples,\n                n_batches,\n            )\n            y_pred = y_pred.mean(dim=0)\n            score = self._get_score(y_pred, y)\n\n            self.valid_running_loss += loss.item()\n            avg_loss = self.valid_running_loss / (batch_idx + 1)\n\n        return score, avg_loss\n\n    def _get_score(self, y_pred, y):\n        if self.metric is not None:\n            if self.objective == \"regression\":\n                score = self.metric(y_pred, y)\n            if self.objective == \"binary\":\n                score = self.metric(torch.sigmoid(y_pred), y)\n            if self.objective == \"multiclass\":\n                score = self.metric(F.softmax(y_pred, dim=1), y)\n            return score\n        else:\n            return None\n\n    def _predict(  # noqa: C901\n        self,\n        X_tab: np.ndarray,\n        n_samples: int = 5,\n        return_samples: bool = False,\n        batch_size: int = 256,\n    ) -&gt; List:\n        self.batch_size = batch_size\n\n        test_set = TensorDataset(torch.from_numpy(X_tab))\n        test_loader = DataLoader(\n            dataset=test_set,\n            batch_size=self.batch_size,\n            num_workers=self.num_workers,\n            shuffle=False,\n        )\n        test_steps = (len(test_loader.dataset) // test_loader.batch_size) + 1  # type: ignore[arg-type]\n\n        preds_l = []\n        with torch.no_grad():\n            with trange(test_steps, disable=self.verbose != 1) as tt:\n                for _, Xl in zip(tt, test_loader):\n                    tt.set_description(\"predict\")\n\n                    try:\n                        X = Xl[0].to(self.device)\n                    except TypeError:\n                        X = Xl[0].to(self.device, dtype=torch.float32)\n\n                    if return_samples:\n                        preds = torch.stack([self.model(X) for _ in range(n_samples)])\n                    else:\n                        self.model.eval()\n                        preds = self.model(X)\n\n                    if self.objective == \"binary\":\n                        preds = torch.sigmoid(preds)\n                    if self.objective == \"multiclass\":\n                        preds = (\n                            F.softmax(preds, dim=2)\n                            if return_samples\n                            else F.softmax(preds, dim=1)\n                        )\n\n                    preds_cpu = preds.cpu().data.numpy()\n                    preds_l.append(preds_cpu)\n\n        self.model.train()\n\n        return preds_l\n</code></pre>"},{"location":"pytorch-widedeep/bayesian_trainer.html#pytorch_widedeep.training.BayesianTrainer.fit","title":"fit","text":"<pre><code>fit(\n    X_tab,\n    target,\n    X_tab_val=None,\n    target_val=None,\n    val_split=None,\n    n_epochs=1,\n    validation_freq=1,\n    batch_size=32,\n    n_train_samples=2,\n    n_val_samples=2,\n)\n</code></pre> <p>Fit method.</p> <p>Parameters:</p> Name Type Description Default <code>X_tab</code> <code>ndarray</code> <p>tabular dataset</p> required <code>target</code> <code>ndarray</code> <p>target values</p> required <code>X_tab_val</code> <code>Optional[ndarray]</code> <p>validation data</p> <code>None</code> <code>target_val</code> <code>Optional[ndarray]</code> <p>validation target values</p> <code>None</code> <code>val_split</code> <code>Optional[float]</code> <p>An alterative to passing the validation set is to use a train/val split fraction via <code>val_split</code></p> <code>None</code> <code>n_epochs</code> <code>int</code> <p>number of epochs</p> <code>1</code> <code>validation_freq</code> <code>int</code> <p>epochs validation frequency</p> <code>1</code> <code>batch_size</code> <code>int</code> <p>batch size</p> <code>32</code> <code>n_train_samples</code> <code>int</code> <p>number of samples to average over during the training process. See Weight Uncertainty in Neural Networks for details.</p> <code>2</code> <code>n_val_samples</code> <code>int</code> <p>number of samples to average over during the validation process. See Weight Uncertainty in Neural Networks for details.</p> <code>2</code> Source code in <code>pytorch_widedeep/training/bayesian_trainer.py</code> <pre><code>def fit(  # noqa: C901\n    self,\n    X_tab: np.ndarray,\n    target: np.ndarray,\n    X_tab_val: Optional[np.ndarray] = None,\n    target_val: Optional[np.ndarray] = None,\n    val_split: Optional[float] = None,\n    n_epochs: int = 1,\n    validation_freq: int = 1,\n    batch_size: int = 32,\n    n_train_samples: int = 2,\n    n_val_samples: int = 2,\n):\n    r\"\"\"Fit method.\n\n    Parameters\n    ----------\n    X_tab: np.ndarray,\n        tabular dataset\n    target: np.ndarray\n        target values\n    X_tab_val: np.ndarray, Optional, default = None\n        validation data\n    target_val: np.ndarray, Optional, default = None\n        validation target values\n    val_split: float, Optional. default=None\n        An alterative to passing the validation set is to use a train/val\n        split fraction via `val_split`\n    n_epochs: int, default=1\n        number of epochs\n    validation_freq: int, default=1\n        epochs validation frequency\n    batch_size: int, default=32\n        batch size\n    n_train_samples: int, default=2\n        number of samples to average over during the training process.\n        See [Weight Uncertainty in Neural Networks](https://arxiv.org/pdf/1505.05424.pdf) for details.\n    n_val_samples: int, default=2\n        number of samples to average over during the validation process.\n        See [Weight Uncertainty in Neural Networks](https://arxiv.org/pdf/1505.05424.pdf) for details.\n    \"\"\"\n\n    self.batch_size = batch_size\n\n    train_set, eval_set = tabular_train_val_split(\n        self.seed, self.objective, X_tab, target, X_tab_val, target_val, val_split\n    )\n    train_loader = DataLoader(\n        dataset=train_set, batch_size=batch_size, num_workers=self.num_workers\n    )\n    train_steps = len(train_loader)\n\n    if eval_set is not None:\n        eval_loader = DataLoader(\n            dataset=eval_set,\n            batch_size=batch_size,\n            num_workers=self.num_workers,\n            shuffle=False,\n        )\n        eval_steps = len(eval_loader)\n\n    self.callback_container.on_train_begin(\n        {\n            \"batch_size\": batch_size,\n            \"train_steps\": train_steps,\n            \"n_epochs\": n_epochs,\n        }\n    )\n    for epoch in range(n_epochs):\n        epoch_logs: Dict[str, float] = {}\n        self.callback_container.on_epoch_begin(epoch, logs=epoch_logs)\n\n        self.train_running_loss = 0.0\n        with trange(train_steps, disable=self.verbose != 1) as t:\n            for batch_idx, (X, y) in zip(t, train_loader):\n                t.set_description(\"epoch %i\" % (epoch + 1))\n                train_score, train_loss = self._train_step(\n                    X, y, n_train_samples, train_steps, batch_idx\n                )\n                print_loss_and_metric(t, train_loss, train_score)\n                self.callback_container.on_batch_end(batch=batch_idx)\n        epoch_logs = save_epoch_logs(epoch_logs, train_loss, train_score, \"train\")\n\n        on_epoch_end_metric = None\n        if eval_set is not None and epoch % validation_freq == (\n            validation_freq - 1\n        ):\n            self.callback_container.on_eval_begin()\n            self.valid_running_loss = 0.0\n            with trange(eval_steps, disable=self.verbose != 1) as v:\n                for i, (X, y) in zip(v, eval_loader):\n                    v.set_description(\"valid\")\n                    val_score, val_loss = self._eval_step(\n                        X, y, n_val_samples, train_steps, i\n                    )\n                    print_loss_and_metric(v, val_loss, val_score)\n            epoch_logs = save_epoch_logs(epoch_logs, val_loss, val_score, \"val\")\n\n            if self.reducelronplateau:\n                if self.reducelronplateau_criterion == \"loss\":\n                    on_epoch_end_metric = val_loss\n                else:\n                    on_epoch_end_metric = val_score[\n                        self.reducelronplateau_criterion\n                    ]\n\n        self.callback_container.on_epoch_end(epoch, epoch_logs, on_epoch_end_metric)\n\n        if self.early_stop:\n            self.callback_container.on_train_end(epoch_logs)\n            break\n\n    self.callback_container.on_train_end(epoch_logs)\n    self._restore_best_weights()\n    self.model.train()\n</code></pre>"},{"location":"pytorch-widedeep/bayesian_trainer.html#pytorch_widedeep.training.BayesianTrainer.predict","title":"predict","text":"<pre><code>predict(\n    X_tab, n_samples=5, return_samples=False, batch_size=256\n)\n</code></pre> <p>Returns the predictions</p> <p>Parameters:</p> Name Type Description Default <code>X_tab</code> <code>ndarray</code> <p>tabular dataset</p> required <code>n_samples</code> <code>int</code> <p>number of samples that will be either returned or averaged to produce an overal prediction</p> <code>5</code> <code>return_samples</code> <code>bool</code> <p>Boolean indicating whether the n samples will be averaged or directly returned</p> <code>False</code> <code>batch_size</code> <code>int</code> <p>batch size</p> <code>256</code> <p>Returns:</p> Type Description <code>np.ndarray:</code> <p>array with the predictions</p> Source code in <code>pytorch_widedeep/training/bayesian_trainer.py</code> <pre><code>def predict(  # type: ignore[return]\n    self,\n    X_tab: np.ndarray,\n    n_samples: int = 5,\n    return_samples: bool = False,\n    batch_size: int = 256,\n) -&gt; np.ndarray:\n    r\"\"\"Returns the predictions\n\n    Parameters\n    ----------\n    X_tab: np.ndarray,\n        tabular dataset\n    n_samples: int, default=5\n        number of samples that will be either returned or averaged to\n        produce an overal prediction\n    return_samples: bool, default = False\n        Boolean indicating whether the n samples will be averaged or directly returned\n    batch_size: int, default = 256\n        batch size\n\n    Returns\n    -------\n    np.ndarray:\n        array with the predictions\n    \"\"\"\n\n    preds_l = self._predict(X_tab, n_samples, return_samples, batch_size)\n    preds = np.hstack(preds_l) if return_samples else np.vstack(preds_l)\n    axis = 2 if return_samples else 1\n\n    if self.objective == \"regression\":\n        return preds.squeeze(axis)\n    if self.objective == \"binary\":\n        return (preds.squeeze(axis) &gt; 0.5).astype(\"int\")\n    if self.objective == \"multiclass\":\n        return np.argmax(preds, axis)\n</code></pre>"},{"location":"pytorch-widedeep/bayesian_trainer.html#pytorch_widedeep.training.BayesianTrainer.predict_proba","title":"predict_proba","text":"<pre><code>predict_proba(\n    X_tab, n_samples=5, return_samples=False, batch_size=256\n)\n</code></pre> <p>Returns the predicted probabilities</p> <p>Parameters:</p> Name Type Description Default <code>X_tab</code> <code>ndarray</code> <p>tabular dataset</p> required <code>n_samples</code> <code>int</code> <p>number of samples that will be either returned or averaged to produce an overal prediction</p> <code>5</code> <code>return_samples</code> <code>bool</code> <p>Boolean indicating whether the n samples will be averaged or directly returned</p> <code>False</code> <code>batch_size</code> <code>int</code> <p>batch size</p> <code>256</code> <p>Returns:</p> Type Description <code>ndarray</code> <p>array with the probabilities per class</p> Source code in <code>pytorch_widedeep/training/bayesian_trainer.py</code> <pre><code>def predict_proba(  # type: ignore[return]\n    self,\n    X_tab: np.ndarray,\n    n_samples: int = 5,\n    return_samples: bool = False,\n    batch_size: int = 256,\n) -&gt; np.ndarray:\n    r\"\"\"Returns the predicted probabilities\n\n    Parameters\n    ----------\n    X_tab: np.ndarray,\n        tabular dataset\n    n_samples: int, default=5\n        number of samples that will be either returned or averaged to\n        produce an overal prediction\n    return_samples: bool, default = False\n        Boolean indicating whether the n samples will be averaged or directly returned\n    batch_size: int, default = 256\n        batch size\n\n    Returns\n    -------\n    np.ndarray\n        array with the probabilities per class\n    \"\"\"\n    preds_l = self._predict(X_tab, n_samples, return_samples, batch_size)\n    preds = np.hstack(preds_l) if return_samples else np.vstack(preds_l)\n\n    if self.objective == \"binary\":\n        if return_samples:\n            preds = preds.squeeze(2)\n            probs = np.zeros([n_samples, preds.shape[1], 2])\n            for i in range(n_samples):\n                probs[i, :, 0] = 1 - preds[i]\n                probs[i, :, 1] = preds[i]\n        else:\n            preds = preds.squeeze(1)\n            probs = np.zeros([preds.shape[0], 2])\n            probs[:, 0] = 1 - preds\n            probs[:, 1] = preds\n        return probs\n    if self.objective == \"multiclass\":\n        return preds\n</code></pre>"},{"location":"pytorch-widedeep/bayesian_trainer.html#pytorch_widedeep.training.BayesianTrainer.save","title":"save","text":"<pre><code>save(\n    path,\n    save_state_dict=False,\n    model_filename=\"bayesian_model.pt\",\n)\n</code></pre> <p>Saves the model, training and evaluation history to disk</p> <p>The <code>Trainer</code> class is built so that it 'just' trains a model. With that in mind, all the torch related parameters (such as optimizers or learning rate schedulers) have to be defined externally and then passed to the <code>Trainer</code>. As a result, the <code>Trainer</code> does not generate any attribute or additional data products that need to be saved other than the <code>model</code> object itself, which can be saved as any other torch model (e.g. <code>torch.save(model, path)</code>).</p> <p>Parameters:</p> Name Type Description Default <code>path</code> <code>str</code> <p>path to the directory where the model and the feature importance attribute will be saved.</p> required <code>save_state_dict</code> <code>bool</code> <p>Boolean indicating whether to save directly the model or the model's state dictionary</p> <code>False</code> <code>model_filename</code> <code>str</code> <p>filename where the model weights will be store</p> <code>'bayesian_model.pt'</code> Source code in <code>pytorch_widedeep/training/bayesian_trainer.py</code> <pre><code>def save(\n    self,\n    path: str,\n    save_state_dict: bool = False,\n    model_filename: str = \"bayesian_model.pt\",\n):\n    r\"\"\"Saves the model, training and evaluation history to disk\n\n    The `Trainer` class is built so that it 'just' trains a model. With\n    that in mind, all the torch related parameters (such as optimizers or\n    learning rate schedulers) have to be defined externally and then\n    passed to the `Trainer`. As a result, the `Trainer` does not\n    generate any attribute or additional data products that need to be\n    saved other than the `model` object itself, which can be saved as\n    any other torch model (e.g. `torch.save(model, path)`).\n\n    Parameters\n    ----------\n    path: str\n        path to the directory where the model and the feature importance\n        attribute will be saved.\n    save_state_dict: bool, default = False\n        Boolean indicating whether to save directly the model or the\n        model's state dictionary\n    model_filename: str, Optional, default = \"wd_model.pt\"\n        filename where the model weights will be store\n    \"\"\"\n\n    save_dir = Path(path)\n    history_dir = save_dir / \"history\"\n    history_dir.mkdir(exist_ok=True, parents=True)\n\n    # the trainer is run with the History Callback by default\n    with open(history_dir / \"train_eval_history.json\", \"w\") as teh:\n        json.dump(self.history, teh)  # type: ignore[attr-defined]\n\n    has_lr_history = any(\n        [clbk.__class__.__name__ == \"LRHistory\" for clbk in self.callbacks]\n    )\n    if self.lr_scheduler is not None and has_lr_history:\n        with open(history_dir / \"lr_history.json\", \"w\") as lrh:\n            json.dump(self.lr_history, lrh)  # type: ignore[attr-defined]\n\n    model_path = save_dir / model_filename\n    if save_state_dict:\n        torch.save(self.model.state_dict(), model_path)\n    else:\n        torch.save(self.model, model_path)\n</code></pre>"},{"location":"pytorch-widedeep/callbacks.html","title":"Callbacks","text":"<p>Here are the 4 callbacks available to the user in <code>pytorch-widedepp</code>: <code>LRHistory</code>, <code>ModelCheckpoint</code>, <code>EarlyStopping</code> and <code>RayTuneReporter</code>.</p> <p> NOTE: other callbacks , like <code>History</code>, run always  by default. In particular, the <code>History</code> callback saves the metrics in the  <code>history</code> attribute of the <code>Trainer</code>.</p>"},{"location":"pytorch-widedeep/callbacks.html#pytorch_widedeep.callbacks.LRHistory","title":"LRHistory","text":"<p>               Bases: <code>Callback</code></p> <p>Saves the learning rates during training in the <code>lr_history</code> attribute of the <code>Trainer</code>.</p> <p>Callbacks are passed as input parameters to the <code>Trainer</code> class. See <code>pytorch_widedeep.trainer.Trainer</code></p> <p>Parameters:</p> Name Type Description Default <code>n_epochs</code> <code>int</code> <p>number of training epochs</p> required <p>Examples:</p> <pre><code>&gt;&gt;&gt; from pytorch_widedeep.callbacks import LRHistory\n&gt;&gt;&gt; from pytorch_widedeep.models import TabMlp, Wide, WideDeep\n&gt;&gt;&gt; from pytorch_widedeep.training import Trainer\n&gt;&gt;&gt;\n&gt;&gt;&gt; embed_input = [(u, i, j) for u, i, j in zip([\"a\", \"b\", \"c\"][:4], [4] * 3, [8] * 3)]\n&gt;&gt;&gt; column_idx = {k: v for v, k in enumerate([\"a\", \"b\", \"c\"])}\n&gt;&gt;&gt; wide = Wide(10, 1)\n&gt;&gt;&gt; deep = TabMlp(mlp_hidden_dims=[8, 4], column_idx=column_idx, cat_embed_input=embed_input)\n&gt;&gt;&gt; model = WideDeep(wide, deep)\n&gt;&gt;&gt; trainer = Trainer(model, objective=\"regression\", callbacks=[LRHistory(n_epochs=10)])\n</code></pre> Source code in <code>pytorch_widedeep/callbacks.py</code> <pre><code>class LRHistory(Callback):\n    r\"\"\"Saves the learning rates during training in the `lr_history` attribute\n    of the `Trainer`.\n\n    Callbacks are passed as input parameters to the `Trainer` class. See\n    `pytorch_widedeep.trainer.Trainer`\n\n    Parameters\n    ----------\n    n_epochs: int\n        number of training epochs\n\n    Examples\n    --------\n    &gt;&gt;&gt; from pytorch_widedeep.callbacks import LRHistory\n    &gt;&gt;&gt; from pytorch_widedeep.models import TabMlp, Wide, WideDeep\n    &gt;&gt;&gt; from pytorch_widedeep.training import Trainer\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; embed_input = [(u, i, j) for u, i, j in zip([\"a\", \"b\", \"c\"][:4], [4] * 3, [8] * 3)]\n    &gt;&gt;&gt; column_idx = {k: v for v, k in enumerate([\"a\", \"b\", \"c\"])}\n    &gt;&gt;&gt; wide = Wide(10, 1)\n    &gt;&gt;&gt; deep = TabMlp(mlp_hidden_dims=[8, 4], column_idx=column_idx, cat_embed_input=embed_input)\n    &gt;&gt;&gt; model = WideDeep(wide, deep)\n    &gt;&gt;&gt; trainer = Trainer(model, objective=\"regression\", callbacks=[LRHistory(n_epochs=10)])\n    \"\"\"\n\n    def __init__(self, n_epochs: int):\n        super(LRHistory, self).__init__()\n        self.n_epochs = n_epochs\n\n    def on_epoch_begin(self, epoch: int, logs: Optional[Dict] = None):\n        if epoch == 0 and self.trainer.lr_scheduler is not None:\n            self.trainer.lr_history = {}\n            if self._multiple_scheduler(self.trainer.lr_scheduler):\n                self._save_group_lr_mulitple_scheduler(step_location=\"on_epoch_begin\")\n            else:\n                self._save_group_lr(self.trainer.optimizer)\n\n    def on_batch_end(self, batch: int, logs: Optional[Dict] = None):\n        if self.trainer.lr_scheduler is not None:\n            if self._multiple_scheduler(self.trainer.lr_scheduler):\n                self._save_group_lr_mulitple_scheduler(step_location=\"on_batch_end\")\n            elif self.trainer.cyclic_lr:\n                self._save_group_lr(self.trainer.optimizer)\n\n    def on_epoch_end(\n        self, epoch: int, logs: Optional[Dict] = None, metric: Optional[float] = None\n    ):\n        if epoch != (self.n_epochs - 1) and self.trainer.lr_scheduler is not None:\n            if self._multiple_scheduler(self.trainer.lr_scheduler):\n                self._save_group_lr_mulitple_scheduler(step_location=\"on_epoch_end\")\n            elif not self.trainer.cyclic_lr:\n                self._save_group_lr(self.trainer.optimizer)\n\n    def _save_group_lr_mulitple_scheduler(self, step_location: str):\n        for model_name, optimizer in self.trainer.optimizer._optimizers.items():\n            if isinstance(optimizer, list):\n                # then, if it has schedulers, we assume it has to have the\n                # same number of schedulers as optimizers\n                for i, opt in enumerate(optimizer):\n                    if (\n                        step_location == \"on_epoch_begin\"\n                        or (\n                            step_location == \"on_batch_end\"\n                            and self._has_cyclic_scheduler(model_name)\n                        )\n                        or (\n                            step_location == \"on_epoch_end\"\n                            and not self._has_cyclic_scheduler(model_name)\n                        )\n                    ):\n                        self._save_group_lr(opt, model_name, \"_\".join([\"opt\", str(i)]))\n                else:\n                    # do nothing\n                    pass\n            else:\n                if (\n                    step_location == \"on_epoch_begin\"\n                    or (\n                        step_location == \"on_batch_end\"\n                        and self._has_cyclic_scheduler(model_name)\n                    )\n                    or (\n                        step_location == \"on_epoch_end\"\n                        and not self._has_cyclic_scheduler(model_name)\n                    )\n                ):\n                    self._save_group_lr(optimizer, model_name)\n\n    def _save_group_lr(\n        self,\n        opt: Optimizer,\n        suffix: Optional[str] = None,\n        model_name: Optional[str] = None,\n    ):\n        suffix = suffix or \"\"\n        model_name = model_name or \"\"\n        for group_idx, group in enumerate(opt.param_groups):\n            group_name = (\"_\").join(\n                [x for x in [\"lr\", model_name, suffix, str(group_idx)] if x]\n            )\n            self.trainer.lr_history.setdefault(group_name, []).append(group[\"lr\"])\n\n    @staticmethod\n    def _multiple_scheduler(scheduler: LRScheduler) -&gt; bool:\n        return scheduler.__class__.__name__ == \"MultipleLRScheduler\"\n\n    def _has_cyclic_scheduler(self, model_name: str):\n        if model_name in self.trainer.lr_scheduler._schedulers:\n            if isinstance(self.trainer.lr_scheduler._schedulers[model_name], list):\n                return any(\n                    [\n                        self._is_cyclic(s)\n                        for s in self.trainer.lr_scheduler._schedulers[model_name]\n                    ]\n                )\n            else:\n                return self._is_cyclic(\n                    self.trainer.lr_scheduler._schedulers[model_name]\n                )\n\n    @staticmethod\n    def _is_cyclic(scheduler: LRScheduler) -&gt; bool:\n        return \"cycl\" in scheduler.__class__.__name__.lower()\n</code></pre>"},{"location":"pytorch-widedeep/callbacks.html#pytorch_widedeep.callbacks.ModelCheckpoint","title":"ModelCheckpoint","text":"<p>               Bases: <code>Callback</code></p> <p>Saves the model after every epoch.</p> <p>This class is almost identical to the corresponding keras class. Therefore, credit to the Keras Team.</p> <p>Callbacks are passed as input parameters to the <code>Trainer</code> class. See <code>pytorch_widedeep.trainer.Trainer</code></p> <p>Parameters:</p> Name Type Description Default <code>filepath</code> <code>Optional[str]</code> <p>Full path to save the output weights. It must contain only the root of the filenames. Epoch number and <code>.pt</code> extension (for pytorch) will be added. e.g. <code>filepath=\"path/to/output_weights/weights_out\"</code> And the saved files in that directory will be named: 'weights_out_1.pt', 'weights_out_2.pt', .... If set to <code>None</code> the class just report best metric and best_epoch.</p> <code>None</code> <code>monitor</code> <code>str</code> <p>quantity to monitor. Typically 'val_loss' or metric name (e.g. 'val_acc')</p> <code>'val_loss'</code> <code>min_delta</code> <code>float</code> <p>minimum change in the monitored quantity to qualify as an improvement, i.e. an absolute change of less than min_delta, will count as no improvement.</p> <code>0.0</code> <code>verbose</code> <code>int</code> <p>verbosity mode</p> <code>0</code> <code>save_best_only</code> <code>bool</code> <p>the latest best model according to the quantity monitored will not be overwritten.</p> <code>False</code> <code>mode</code> <code>str</code> <p>If <code>save_best_only=True</code>, the decision to overwrite the current save file is made based on either the maximization or the minimization of the monitored quantity. For 'acc', this should be 'max', for 'loss' this should be 'min', etc. In 'auto' mode, the direction is automatically inferred from the name of the monitored quantity.</p> <code>'auto'</code> <code>period</code> <code>int</code> <p>Interval (number of epochs) between checkpoints.</p> <code>1</code> <code>max_save</code> <code>int</code> <p>Maximum number of outputs to save. If -1 will save all outputs</p> <code>-1</code> <p>Attributes:</p> Name Type Description <code>best</code> <code>float</code> <p>best metric</p> <code>best_epoch</code> <code>int</code> <p>best epoch</p> <code>best_state_dict</code> <code>dict</code> <p>best model state dictionary. To restore model to its best state use <code>Trainer.model.load_state_dict (model_checkpoint.best_state_dict)</code> where <code>model_checkpoint</code> is an instance of the class <code>ModelCheckpoint</code>. See the Examples folder in the repo or the Examples section in this documentation for details</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; from pytorch_widedeep.callbacks import ModelCheckpoint\n&gt;&gt;&gt; from pytorch_widedeep.models import TabMlp, Wide, WideDeep\n&gt;&gt;&gt; from pytorch_widedeep.training import Trainer\n&gt;&gt;&gt;\n&gt;&gt;&gt; embed_input = [(u, i, j) for u, i, j in zip([\"a\", \"b\", \"c\"][:4], [4] * 3, [8] * 3)]\n&gt;&gt;&gt; column_idx = {k: v for v, k in enumerate([\"a\", \"b\", \"c\"])}\n&gt;&gt;&gt; wide = Wide(10, 1)\n&gt;&gt;&gt; deep = TabMlp(mlp_hidden_dims=[8, 4], column_idx=column_idx, cat_embed_input=embed_input)\n&gt;&gt;&gt; model = WideDeep(wide, deep)\n&gt;&gt;&gt; trainer = Trainer(model, objective=\"regression\", callbacks=[ModelCheckpoint(filepath='checkpoints/weights_out')])\n</code></pre> Source code in <code>pytorch_widedeep/callbacks.py</code> <pre><code>class ModelCheckpoint(Callback):\n    r\"\"\"Saves the model after every epoch.\n\n    This class is almost identical to the corresponding keras class.\n    Therefore, **credit** to the Keras Team.\n\n    Callbacks are passed as input parameters to the `Trainer` class. See\n    `pytorch_widedeep.trainer.Trainer`\n\n    Parameters\n    ----------\n    filepath: str, default=None\n        Full path to save the output weights. It must contain only the root of\n        the filenames. Epoch number and `.pt` extension (for pytorch) will be\n        added. e.g. `filepath=\"path/to/output_weights/weights_out\"` And the\n        saved files in that directory will be named:\n        _'weights_out_1.pt', 'weights_out_2.pt', ..._. If set to `None` the\n        class just report best metric and best_epoch.\n    monitor: str, default=\"loss\"\n        quantity to monitor. Typically _'val_loss'_ or metric name\n        (e.g. _'val_acc'_)\n    min_delta: float, default=0.\n        minimum change in the monitored quantity to qualify as an\n        improvement, i.e. an absolute change of less than min_delta, will\n        count as no improvement.\n    verbose:int, default=0\n        verbosity mode\n    save_best_only: bool, default=False,\n        the latest best model according to the quantity monitored will not be\n        overwritten.\n    mode: str, default=\"auto\"\n        If `save_best_only=True`, the decision to overwrite the current save\n        file is made based on either the maximization or the minimization of\n        the monitored quantity. For _'acc'_, this should be _'max'_, for\n        _'loss'_ this should be _'min'_, etc. In '_auto'_ mode, the\n        direction is automatically inferred from the name of the monitored\n        quantity.\n    period: int, default=1\n        Interval (number of epochs) between checkpoints.\n    max_save: int, default=-1\n        Maximum number of outputs to save. If -1 will save all outputs\n\n    Attributes\n    ----------\n    best: float\n        best metric\n    best_epoch: int\n        best epoch\n    best_state_dict: dict\n        best model state dictionary.&lt;br/&gt;\n        To restore model to its best state use `Trainer.model.load_state_dict\n        (model_checkpoint.best_state_dict)` where `model_checkpoint` is an\n        instance of the class `ModelCheckpoint`. See the Examples folder in\n        the repo or the Examples section in this documentation for details\n\n    Examples\n    --------\n    &gt;&gt;&gt; from pytorch_widedeep.callbacks import ModelCheckpoint\n    &gt;&gt;&gt; from pytorch_widedeep.models import TabMlp, Wide, WideDeep\n    &gt;&gt;&gt; from pytorch_widedeep.training import Trainer\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; embed_input = [(u, i, j) for u, i, j in zip([\"a\", \"b\", \"c\"][:4], [4] * 3, [8] * 3)]\n    &gt;&gt;&gt; column_idx = {k: v for v, k in enumerate([\"a\", \"b\", \"c\"])}\n    &gt;&gt;&gt; wide = Wide(10, 1)\n    &gt;&gt;&gt; deep = TabMlp(mlp_hidden_dims=[8, 4], column_idx=column_idx, cat_embed_input=embed_input)\n    &gt;&gt;&gt; model = WideDeep(wide, deep)\n    &gt;&gt;&gt; trainer = Trainer(model, objective=\"regression\", callbacks=[ModelCheckpoint(filepath='checkpoints/weights_out')])\n    \"\"\"\n\n    def __init__(\n        self,\n        filepath: Optional[str] = None,\n        monitor: str = \"val_loss\",\n        min_delta: float = 0.0,\n        verbose: int = 0,\n        save_best_only: bool = False,\n        mode: str = \"auto\",\n        period: int = 1,\n        max_save: int = -1,\n    ):\n        super(ModelCheckpoint, self).__init__()\n\n        self.filepath = filepath\n        self.monitor = monitor\n        self.min_delta = min_delta\n        self.verbose = verbose\n        self.save_best_only = save_best_only\n        self.mode = mode\n        self.period = period\n        self.max_save = max_save\n\n        self.epochs_since_last_save = 0\n\n        if self.filepath:\n            if len(self.filepath.split(\"/\")[:-1]) == 0:\n                raise ValueError(\n                    \"'filepath' must be the full path to save the output weights,\"\n                    \" including the root of the filenames. e.g. 'checkpoints/weights_out'\"\n                )\n\n            root_dir = (\"/\").join(self.filepath.split(\"/\")[:-1])\n            if not os.path.exists(root_dir):\n                os.makedirs(root_dir)\n\n        if self.max_save &gt; 0:\n            self.old_files: List[str] = []\n\n        if self.mode not in [\"auto\", \"min\", \"max\"]:\n            warnings.warn(\n                \"ModelCheckpoint mode %s is unknown, \"\n                \"fallback to auto mode.\" % (self.mode),\n                RuntimeWarning,\n            )\n            self.mode = \"auto\"\n        if self.mode == \"min\":\n            self.monitor_op = np.less\n            self.best = np.Inf\n        elif self.mode == \"max\":\n            self.monitor_op = np.greater  # type: ignore[assignment]\n            self.best = -np.Inf\n        else:\n            if _is_metric(self.monitor):\n                self.monitor_op = np.greater  # type: ignore[assignment]\n                self.best = -np.Inf\n            else:\n                self.monitor_op = np.less\n                self.best = np.Inf\n\n        if self.monitor_op == np.greater:\n            self.min_delta *= 1\n        else:\n            self.min_delta *= -1\n\n    def on_epoch_end(  # noqa: C901\n        self, epoch: int, logs: Optional[Dict] = None, metric: Optional[float] = None\n    ):\n        logs = logs or {}\n        self.epochs_since_last_save += 1\n        if self.epochs_since_last_save &gt;= self.period:\n            self.epochs_since_last_save = 0\n            if self.filepath:\n                filepath = \"{}_{}.p\".format(self.filepath, epoch + 1)\n            if self.save_best_only:\n                current = logs.get(self.monitor)\n                if current is None:\n                    warnings.warn(\n                        \"Can save best model only with %s available, \"\n                        \"skipping.\" % (self.monitor),\n                        RuntimeWarning,\n                    )\n                else:\n                    if self.monitor_op(current - self.min_delta, self.best):\n                        if self.verbose &gt; 0:\n                            if self.filepath:\n                                print(\n                                    f\"\\nEpoch {epoch + 1}: {self.monitor} improved from {self.best:.5f} to {current:.5f} \"\n                                    f\"Saving model to {filepath}\"\n                                )\n                            else:\n                                print(\n                                    f\"\\nEpoch {epoch + 1}: {self.monitor} improved from {self.best:.5f} to {current:.5f} \"\n                                )\n                        self.best = current\n                        self.best_epoch = epoch\n                        self.best_state_dict = copy.deepcopy(self.model.state_dict())\n                        if self.filepath:\n                            torch.save(self.best_state_dict, filepath)\n                            if self.max_save &gt; 0:\n                                if len(self.old_files) == self.max_save:\n                                    try:\n                                        os.remove(self.old_files[0])\n                                    except FileNotFoundError:\n                                        pass\n                                    self.old_files = self.old_files[1:]\n                                self.old_files.append(filepath)\n                    else:\n                        if self.verbose &gt; 0:\n                            print(\n                                f\"\\nEpoch {epoch + 1}: {self.monitor} did not improve from {self.best:.5f} \"\n                                f\" considering a 'min_delta' improvement of {self.min_delta:.5f}\"\n                            )\n            if not self.save_best_only and self.filepath:\n                if self.verbose &gt; 0:\n                    print(\"\\nEpoch %05d: saving model to %s\" % (epoch + 1, filepath))\n                torch.save(self.model.state_dict(), filepath)\n                if self.max_save &gt; 0:\n                    if len(self.old_files) == self.max_save:\n                        try:\n                            os.remove(self.old_files[0])\n                        except FileNotFoundError:\n                            pass\n                        self.old_files = self.old_files[1:]\n                    self.old_files.append(filepath)\n\n    def __getstate__(self):\n        d = self.__dict__\n        self_dict = {k: d[k] for k in d if k not in [\"trainer\", \"model\"]}\n        return self_dict\n\n    def __setstate__(self, state):\n        self.__dict__ = state\n</code></pre>"},{"location":"pytorch-widedeep/callbacks.html#pytorch_widedeep.callbacks.EarlyStopping","title":"EarlyStopping","text":"<p>               Bases: <code>Callback</code></p> <p>Stop training when a monitored quantity has stopped improving.</p> <p>This class is almost identical to the corresponding keras class. Therefore, credit to the Keras Team.</p> <p>Callbacks are passed as input parameters to the <code>Trainer</code> class. See <code>pytorch_widedeep.trainer.Trainer</code></p> <p>Parameters:</p> Name Type Description Default <code>monitor</code> <code>str</code> <p>Quantity to monitor. Typically 'val_loss' or metric name (e.g. 'val_acc')</p> <code>'val_loss'</code> <code>min_delta</code> <code>float</code> <p>minimum change in the monitored quantity to qualify as an improvement, i.e. an absolute change of less than min_delta, will count as no improvement.</p> <code>0.0</code> <code>patience</code> <code>int</code> <p>Number of epochs that produced the monitored quantity with no improvement after which training will be stopped.</p> <code>10</code> <code>verbose</code> <code>int</code> <p>verbosity mode.</p> <code>0</code> <code>mode</code> <code>str</code> <p>one of {'auto', 'min', 'max'}. In 'min' mode, training will stop when the quantity monitored has stopped decreasing; in 'max' mode it will stop when the quantity monitored has stopped increasing; in 'auto' mode, the direction is automatically inferred from the name of the monitored quantity.</p> <code>'auto'</code> <code>baseline</code> <code>Optional[float]</code> <p>Baseline value for the monitored quantity to reach. Training will stop if the model does not show improvement over the baseline.</p> <code>None</code> <code>restore_best_weights</code> <code>bool</code> <p>Whether to restore model weights from the epoch with the best value of the monitored quantity. If <code>False</code>, the model weights obtained at the last step of training are used.</p> <code>False</code> <p>Attributes:</p> Name Type Description <code>best</code> <code>float</code> <p>best metric</p> <code>stopped_epoch</code> <code>int</code> <p>epoch when the training stopped</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; from pytorch_widedeep.callbacks import EarlyStopping\n&gt;&gt;&gt; from pytorch_widedeep.models import TabMlp, Wide, WideDeep\n&gt;&gt;&gt; from pytorch_widedeep.training import Trainer\n&gt;&gt;&gt;\n&gt;&gt;&gt; embed_input = [(u, i, j) for u, i, j in zip([\"a\", \"b\", \"c\"][:4], [4] * 3, [8] * 3)]\n&gt;&gt;&gt; column_idx = {k: v for v, k in enumerate([\"a\", \"b\", \"c\"])}\n&gt;&gt;&gt; wide = Wide(10, 1)\n&gt;&gt;&gt; deep = TabMlp(mlp_hidden_dims=[8, 4], column_idx=column_idx, cat_embed_input=embed_input)\n&gt;&gt;&gt; model = WideDeep(wide, deep)\n&gt;&gt;&gt; trainer = Trainer(model, objective=\"regression\", callbacks=[EarlyStopping(patience=10)])\n</code></pre> Source code in <code>pytorch_widedeep/callbacks.py</code> <pre><code>class EarlyStopping(Callback):\n    r\"\"\"Stop training when a monitored quantity has stopped improving.\n\n    This class is almost identical to the corresponding keras class.\n    Therefore, **credit** to the Keras Team.\n\n    Callbacks are passed as input parameters to the `Trainer` class. See\n    `pytorch_widedeep.trainer.Trainer`\n\n    Parameters\n    -----------\n    monitor: str, default='val_loss'.\n        Quantity to monitor. Typically _'val_loss'_ or metric name\n        (e.g. _'val_acc'_)\n    min_delta: float, default=0.\n        minimum change in the monitored quantity to qualify as an\n        improvement, i.e. an absolute change of less than min_delta, will\n        count as no improvement.\n    patience: int, default=10.\n        Number of epochs that produced the monitored quantity with no\n        improvement after which training will be stopped.\n    verbose: int.\n        verbosity mode.\n    mode: str, default='auto'\n        one of _{'auto', 'min', 'max'}_. In _'min'_ mode, training will\n        stop when the quantity monitored has stopped decreasing; in _'max'_\n        mode it will stop when the quantity monitored has stopped increasing;\n        in _'auto'_ mode, the direction is automatically inferred from the\n        name of the monitored quantity.\n    baseline: float, Optional. default=None.\n        Baseline value for the monitored quantity to reach. Training will\n        stop if the model does not show improvement over the baseline.\n    restore_best_weights: bool, default=None\n        Whether to restore model weights from the epoch with the best\n        value of the monitored quantity. If `False`, the model weights\n        obtained at the last step of training are used.\n\n    Attributes\n    ----------\n    best: float\n        best metric\n    stopped_epoch: int\n        epoch when the training stopped\n\n    Examples\n    --------\n    &gt;&gt;&gt; from pytorch_widedeep.callbacks import EarlyStopping\n    &gt;&gt;&gt; from pytorch_widedeep.models import TabMlp, Wide, WideDeep\n    &gt;&gt;&gt; from pytorch_widedeep.training import Trainer\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; embed_input = [(u, i, j) for u, i, j in zip([\"a\", \"b\", \"c\"][:4], [4] * 3, [8] * 3)]\n    &gt;&gt;&gt; column_idx = {k: v for v, k in enumerate([\"a\", \"b\", \"c\"])}\n    &gt;&gt;&gt; wide = Wide(10, 1)\n    &gt;&gt;&gt; deep = TabMlp(mlp_hidden_dims=[8, 4], column_idx=column_idx, cat_embed_input=embed_input)\n    &gt;&gt;&gt; model = WideDeep(wide, deep)\n    &gt;&gt;&gt; trainer = Trainer(model, objective=\"regression\", callbacks=[EarlyStopping(patience=10)])\n    \"\"\"\n\n    def __init__(\n        self,\n        monitor: str = \"val_loss\",\n        min_delta: float = 0.0,\n        patience: int = 10,\n        verbose: int = 0,\n        mode: str = \"auto\",\n        baseline: Optional[float] = None,\n        restore_best_weights: bool = False,\n    ):\n        super(EarlyStopping, self).__init__()\n\n        self.monitor = monitor\n        self.min_delta = min_delta\n        self.patience = patience\n        self.verbose = verbose\n        self.mode = mode\n        self.baseline = baseline\n        self.restore_best_weights = restore_best_weights\n\n        self.wait = 0\n        self.stopped_epoch = 0\n        self.state_dict = None\n\n        if self.mode not in [\"auto\", \"min\", \"max\"]:\n            warnings.warn(\n                \"EarlyStopping mode %s is unknown, \"\n                \"fallback to auto mode.\" % self.mode,\n                RuntimeWarning,\n            )\n            self.mode = \"auto\"\n\n        if self.mode == \"min\":\n            self.monitor_op = np.less\n        elif self.mode == \"max\":\n            self.monitor_op = np.greater  # type: ignore[assignment]\n        else:\n            if _is_metric(self.monitor):\n                self.monitor_op = np.greater  # type: ignore[assignment]\n            else:\n                self.monitor_op = np.less\n\n        if self.monitor_op == np.greater:\n            self.min_delta *= 1\n        else:\n            self.min_delta *= -1\n\n    def on_train_begin(self, logs: Optional[Dict] = None):\n        # Allow instances to be re-used\n        self.wait = 0\n        self.stopped_epoch = 0\n        if self.baseline is not None:\n            self.best = self.baseline\n        else:\n            self.best = np.Inf if self.monitor_op == np.less else -np.Inf\n\n    def on_epoch_end(\n        self, epoch: int, logs: Optional[Dict] = None, metric: Optional[float] = None\n    ):\n        current = self.get_monitor_value(logs)\n        if current is None:\n            return\n\n        if self.monitor_op(current - self.min_delta, self.best):\n            self.best = current\n            self.wait = 0\n            self.best_epoch = epoch\n            if self.restore_best_weights:\n                self.state_dict = copy.deepcopy(self.model.state_dict())\n        else:\n            self.wait += 1\n            if self.wait &gt;= self.patience:\n                self.stopped_epoch = epoch\n                self.trainer.early_stop = True\n\n    def on_train_end(self, logs: Optional[Dict] = None):\n        if self.stopped_epoch &gt; 0 and self.verbose &gt; 0:\n            print(\n                f\"Best Epoch: {self.best_epoch + 1}. Best {self.monitor}: {self.best:.5f}\"\n            )\n        if self.restore_best_weights and self.state_dict is not None:\n            if self.verbose &gt; 0:\n                print(\"Restoring model weights from the end of the best epoch\")\n            self.model.load_state_dict(self.state_dict)\n\n    def get_monitor_value(self, logs):\n        monitor_value = logs.get(self.monitor)\n        if monitor_value is None:\n            warnings.warn(\n                \"Early stopping conditioned on metric `%s` \"\n                \"which is not available. Available metrics are: %s\"\n                % (self.monitor, \",\".join(list(logs.keys()))),\n                RuntimeWarning,\n            )\n        return monitor_value\n\n    def __getstate__(self):\n        d = self.__dict__\n        self_dict = {k: d[k] for k in d if k not in [\"trainer\", \"model\"]}\n        return self_dict\n\n    def __setstate__(self, state):\n        self.__dict__ = state\n</code></pre>"},{"location":"pytorch-widedeep/dataloaders.html","title":"Dataloaders","text":"<p> NOTE: This module should contain custom dataloaders  that the user might want to implement. At the moment <code>pytorch-widedeep</code>  offers one custom dataloader, <code>DataLoaderImbalanced</code>.</p>"},{"location":"pytorch-widedeep/dataloaders.html#pytorch_widedeep.dataloaders.DataLoaderImbalanced","title":"DataLoaderImbalanced","text":"<p>               Bases: <code>CustomDataLoader</code></p> <p>Class to load and shuffle batches with adjusted weights for imbalanced datasets. If the classes do not begin from 0 remapping is necessary. See here.</p> <p>Parameters:</p> Name Type Description Default <code>dataset</code> <code>Optional[WideDeepDataset]</code> <p>see <code>pytorch_widedeep.training._wd_dataset</code></p> <code>None</code> <p>Other Parameters:</p> Name Type Description <code>*args</code> <p>Positional arguments to be passed to the parent CustomDataLoader.</p> <code>**kwargs</code> <p>This can include any parameter that can be passed to the 'standard' pytorch DataLoader and that is not already explicitely passed to the class. In addition, the dictionary can also include the extra parameter <code>oversample_mul</code> which will multiply the number of samples of the minority class to be sampled by the <code>WeightedRandomSampler</code>.</p> <p>In other words, the <code>num_samples</code> param in <code>WeightedRandomSampler</code> will be defined as:</p> \\[ minority \\space class \\space count \\times number \\space of \\space classes \\times oversample\\_mul \\] Source code in <code>pytorch_widedeep/dataloaders.py</code> <pre><code>class DataLoaderImbalanced(CustomDataLoader):\n    r\"\"\"Class to load and shuffle batches with adjusted weights for imbalanced\n    datasets. If the classes do not begin from 0 remapping is necessary. See\n    [here](https://towardsdatascience.com/pytorch-tabular-multiclass-classification-9f8211a123ab).\n\n    Parameters\n    ----------\n    dataset: `WideDeepDataset`\n        see `pytorch_widedeep.training._wd_dataset`\n\n    Other Parameters\n    ----------------\n    *args: Any\n        Positional arguments to be passed to the parent CustomDataLoader.\n    **kwargs: Dict\n        This can include any parameter that can be passed to the _'standard'_\n        pytorch\n        [DataLoader](https://pytorch.org/docs/stable/data.html#torch.utils.data.DataLoader)\n        and that is not already explicitely passed to the class. In addition,\n        the dictionary can also include the extra parameter `oversample_mul` which\n        will multiply the number of samples of the minority class to be sampled by\n        the [`WeightedRandomSampler`](https://pytorch.org/docs/stable/data.html#torch.utils.data.WeightedRandomSampler).\n\n        In other words, the `num_samples` param in `WeightedRandomSampler` will be defined as:\n\n        $$\n        minority \\space class \\space count \\times number \\space of \\space classes \\times oversample\\_mul\n        $$\n    \"\"\"\n\n    def __init__(self, dataset: Optional[WideDeepDataset] = None, *args, **kwargs):\n        self.args = args\n        self.kwargs = kwargs\n\n        if dataset is not None:\n            self._setup_sampler(dataset)\n            super().__init__(dataset, *args, sampler=self.sampler, **kwargs)\n        else:\n            super().__init__()\n\n    def set_dataset(self, dataset: WideDeepDataset):\n        sampler = self._setup_sampler(dataset)\n        # update the kwargs with the new sampler\n        self.kwargs[\"sampler\"] = sampler\n        super().set_dataset(dataset)\n\n    def _setup_sampler(self, dataset: WideDeepDataset) -&gt; WeightedRandomSampler:\n        assert dataset.Y is not None, (\n            \"The 'dataset' instance of WideDeepDataset must contain a \"\n            \"target array 'Y'\"\n        )\n\n        oversample_mul = self.kwargs.pop(\"oversample_mul\", 1)\n        weights, minor_cls_cnt, num_clss = get_class_weights(dataset)\n        num_samples = int(minor_cls_cnt * num_clss * oversample_mul)\n        samples_weight = list(np.array([weights[i] for i in dataset.Y]))\n        sampler = WeightedRandomSampler(samples_weight, num_samples, replacement=True)\n        return sampler\n</code></pre>"},{"location":"pytorch-widedeep/load_from_folder.html","title":"The <code>load_from_folder</code> module","text":"<p>The <code>load_from_folder</code> module contains the classes that are necessary to load data from disk and these are inspired by the <code>ImageFolder</code> class in the <code>torchvision</code> library. This module is designed with one specific case in mind. Such case is the following: given a multi-modal dataset with tabular data, images and text, the images do not fit in memory, and therefore, they have to be loaded from disk. However, as any other functionality in this library, there is some flexibility and some additional cases can also be addressed using this module.</p> <p>For this module to be used, the datasets must be prepared in a certain way:</p> <ol> <li> <p>the tabular data must contain a column with the images names as stored in disk, including the extension (<code>.jpg</code>, <code>.png</code>, etc...).</p> </li> <li> <p>Regarding to the text dataset, the tabular data can contain a column with the texts themselves or the names of the files containing the texts as stored in disk.</p> </li> </ol> <p>The tabular data might or might not fit in disk itself. If it does not, please see the <code>ChunkPreprocessor</code> utilities at the <code>preprocessing</code> module and the examples folder in the repo, which illustrate such case. Finally note that only <code>csv</code> format is currently supported in that case (more formats might come soon).</p>"},{"location":"pytorch-widedeep/load_from_folder.html#pytorch_widedeep.load_from_folder.tabular.tabular_from_folder.TabFromFolder","title":"TabFromFolder","text":"<p>This class is used to load tabular data from disk. The current constrains are:</p> <ol> <li>The only file format supported right now is csv</li> <li>The csv file must contain headers</li> </ol> <p>For examples, please, see the examples folder in the repo.</p> <p>Parameters:</p> Name Type Description Default <code>fname</code> <code>str</code> <p>the name of the csv file</p> required <code>directory</code> <code>Optional[str]</code> <p>the path to the directory where the csv file is located. If None, a <code>TabFromFolder</code> reference object must be provided</p> <code>None</code> <code>target_col</code> <code>Optional[str]</code> <p>the name of the target column. If None, a <code>TabFromFolder</code> reference object must be provided</p> <code>None</code> <code>preprocessor</code> <code>Optional[TabularPreprocessor]</code> <p>a fitted <code>TabularPreprocessor</code> object. If None, a <code>TabFromFolder</code> reference object must be provided</p> <code>None</code> <code>text_col</code> <code>Optional[Union[str, List[str]]]</code> <p>the name of the column with the texts themselves or the names of the files that contain the text dataset. If None, either there is no text column or a <code>TabFromFolder</code> reference object must be provided</p> <code>None</code> <code>img_col</code> <code>Optional[Union[str, List[str]]]</code> <p>the name of the column with the the names of the images. If None, either there is no image column or a <code>TabFromFolder</code> reference object must be provided</p> <code>None</code> <code>ignore_target</code> <code>bool</code> <p>whether to ignore the target column. This is normally set to True when this class is used for a test dataset.</p> <code>False</code> <code>reference</code> <code>Optional[Any]</code> <p>a reference <code>TabFromFolder</code> object. If provided, the <code>TabFromFolder</code> object will be created using the attributes of the reference object. This is useful to instantiate a <code>TabFromFolder</code> object for evaluation or test purposes</p> <code>None</code> <code>verbose</code> <code>Optional[int]</code> <p>verbosity. If 0, no output will be printed during the process.</p> <code>1</code> Source code in <code>pytorch_widedeep/load_from_folder/tabular/tabular_from_folder.py</code> <pre><code>class TabFromFolder:\n    \"\"\"\n    This class is used to load tabular data from disk. The current constrains are:\n\n    1. The only file format supported right now is csv\n    2. The csv file must contain headers\n\n    For examples, please, see the examples folder in the repo.\n\n    Parameters\n    ----------\n    fname: str\n        the name of the csv file\n    directory: str, Optional, default = None\n        the path to the directory where the csv file is located. If None,\n        a `TabFromFolder` reference object must be provided\n    target_col: str, Optional, default = None\n        the name of the target column. If None, a `TabFromFolder` reference\n        object must be provided\n    preprocessor: `TabularPreprocessor`, Optional, default = None\n        a fitted `TabularPreprocessor` object. If None, a `TabFromFolder`\n        reference object must be provided\n    text_col: str, Optional, default = None\n        the name of the column with the texts themselves or the names of the\n        files that contain the text dataset. If None, either there is no text\n        column or a `TabFromFolder` reference object must be provided\n    img_col: str, Optional, default = None\n        the name of the column with the the names of the images. If None,\n        either there is no image column or a `TabFromFolder` reference object\n        must be provided\n    ignore_target: bool, default = False\n        whether to ignore the target column. This is normally set to True when\n        this class is used for a test dataset.\n    reference: `TabFromFolder`, Optional, default = None\n        a reference `TabFromFolder` object. If provided, the `TabFromFolder`\n        object will be created using the attributes of the reference object.\n        This is useful to instantiate a `TabFromFolder` object for evaluation\n        or test purposes\n    verbose: int, default = 1\n        verbosity. If 0, no output will be printed during the process.\n    \"\"\"\n\n    def __init__(\n        self,\n        fname: str,\n        directory: Optional[str] = None,\n        target_col: Optional[str] = None,\n        preprocessor: Optional[TabularPreprocessor] = None,\n        text_col: Optional[Union[str, List[str]]] = None,\n        img_col: Optional[Union[str, List[str]]] = None,\n        ignore_target: bool = False,\n        reference: Optional[Any] = None,  # is Type[\"TabFromFolder\"],\n        verbose: Optional[int] = 1,\n    ):\n        self.fname = fname\n        self.ignore_target = ignore_target\n        self.verbose = verbose\n\n        if reference is not None:\n            (\n                self.directory,\n                self.target_col,\n                self.preprocessor,\n                self.text_col,\n                self.img_col,\n            ) = self._set_from_reference(reference, preprocessor)\n        else:\n            assert (\n                directory is not None\n                and (target_col is not None and not ignore_target)\n                and preprocessor is not None\n            ), (\n                \"if no reference is provided, 'directory', 'target_col' and 'preprocessor' \"\n                \"must be provided\"\n            )\n\n            self.directory = directory\n            self.target_col = target_col\n            self.preprocessor = preprocessor\n            self.text_col = text_col\n            self.img_col = img_col\n\n        assert (\n            self.preprocessor.is_fitted\n        ), \"The preprocessor must be fitted before passing it to this class\"\n\n    def get_item(self, idx: int) -&gt; Tuple[  # noqa: C901\n        np.ndarray,\n        Optional[Union[str, List[str]]],\n        Optional[Union[str, List[str]]],\n        Optional[Union[int, float]],\n    ]:\n        \"\"\"\n        This method is used to retrieve a sample from the csv file\n\n        Parameters\n        ----------\n        idx: int\n            the index of the sample to retrieve\n\n        Returns\n        -------\n        Tuple\n            a tuple with the processed tabular data, the text data and/or the\n            image data, and the target variable\n        \"\"\"\n\n        path = os.path.join(self.directory, self.fname)\n\n        try:\n            if not hasattr(self, \"colnames\"):\n                self.colnames = pd.read_csv(path, nrows=0).columns.tolist()\n\n            # TO DO: we need to look into this as the treatment is different\n            # whether the csv contains headers or not. For the time being we\n            # will require that the csv file has headers\n\n            _sample = pd.read_csv(\n                path, skiprows=lambda x: x != idx + 1, header=None\n            ).values\n            sample = pd.DataFrame(_sample, columns=self.colnames)\n        except Exception:\n            raise ValueError(\"Currently only csv format is supported.\")\n\n        text_fnames_or_text: Optional[Union[str, List[str]]] = None\n        if self.text_col is not None:\n            if isinstance(self.text_col, list):\n                text_fnames_or_text = [\n                    sample[col].to_list()[0] for col in self.text_col\n                ]\n            else:\n                text_fnames_or_text = sample[self.text_col].to_list()[0]\n\n        img_fname: Optional[Union[str, List[str]]] = None\n        if self.img_col is not None:\n            if isinstance(self.img_col, list):\n                img_fname = [sample[col].to_list()[0] for col in self.img_col]\n            else:\n                img_fname = sample[self.img_col].to_list()[0]\n\n        processed_sample = self.preprocessor.transform_sample(sample)\n\n        if not self.ignore_target:\n            target = sample[self.target_col].to_list()[0]\n        else:\n            target = None\n\n        return processed_sample, text_fnames_or_text, img_fname, target\n\n    def _set_from_reference(\n        self,\n        reference: Any,  # is Type[\"TabFromFolder\"],\n        preprocessor: Optional[TabularPreprocessor],\n    ) -&gt; Tuple[\n        str,\n        str,\n        TabularPreprocessor,\n        Optional[Union[str, List[str]]],\n        Optional[Union[str, List[str]]],\n    ]:\n        (\n            directory,\n            target_col,\n            _preprocessor,\n            text_col,\n            img_col,\n        ) = self._get_from_reference(reference)\n\n        if preprocessor is not None:\n            preprocessor = preprocessor\n            if self.verbose:\n                UserWarning(\n                    \"The preprocessor from the reference object is overwritten \"\n                    \"by the provided preprocessor\"\n                )\n        else:\n            preprocessor = _preprocessor\n\n        return directory, target_col, preprocessor, text_col, img_col\n\n    @staticmethod\n    def _get_from_reference(\n        reference: Any,  # is Type[\"TabFromFolder\"],\n    ) -&gt; Tuple[str, str, TabularPreprocessor, Optional[str], Optional[str]]:\n        return (\n            reference.directory,\n            reference.target_col,\n            reference.preprocessor,\n            reference.text_col,\n            reference.img_col,\n        )\n\n    def __repr__(self) -&gt; str:  # noqa: C901\n        list_of_params: List[str] = []\n        if self.fname is not None:\n            list_of_params.append(\"fname={fname}\")\n        if self.directory is not None:\n            list_of_params.append(\"directory={directory}\")\n        if self.target_col is not None:\n            list_of_params.append(\"target_col={target_col}\")\n        if self.preprocessor is not None:\n            list_of_params.append(\n                f\"preprocessor={self.preprocessor.__class__.__name__}\"\n            )\n        if self.text_col is not None:\n            if isinstance(self.text_col, list):\n                list_of_params.append(\n                    f\"text_col={[text_col for text_col in self.text_col]}\"\n                )\n            else:\n                list_of_params.append(\"text_col={text_col}\")\n        if self.img_col is not None:\n            if isinstance(self.img_col, list):\n                list_of_params.append(\n                    f\"img_col={[img_col for img_col in self.img_col]}\"\n                )\n            else:\n                list_of_params.append(\"img_col={img_col}\")\n        if self.ignore_target is not None:\n            list_of_params.append(\"ignore_target={ignore_target}\")\n        if self.verbose is not None:\n            list_of_params.append(\"verbose={verbose}\")\n        all_params = \", \".join(list_of_params)\n        return f\"{self.__class__.__name__}({all_params.format(**self.__dict__)})\"\n</code></pre>"},{"location":"pytorch-widedeep/load_from_folder.html#pytorch_widedeep.load_from_folder.tabular.tabular_from_folder.TabFromFolder.get_item","title":"get_item","text":"<pre><code>get_item(idx)\n</code></pre> <p>This method is used to retrieve a sample from the csv file</p> <p>Parameters:</p> Name Type Description Default <code>idx</code> <code>int</code> <p>the index of the sample to retrieve</p> required <p>Returns:</p> Type Description <code>Tuple</code> <p>a tuple with the processed tabular data, the text data and/or the image data, and the target variable</p> Source code in <code>pytorch_widedeep/load_from_folder/tabular/tabular_from_folder.py</code> <pre><code>def get_item(self, idx: int) -&gt; Tuple[  # noqa: C901\n    np.ndarray,\n    Optional[Union[str, List[str]]],\n    Optional[Union[str, List[str]]],\n    Optional[Union[int, float]],\n]:\n    \"\"\"\n    This method is used to retrieve a sample from the csv file\n\n    Parameters\n    ----------\n    idx: int\n        the index of the sample to retrieve\n\n    Returns\n    -------\n    Tuple\n        a tuple with the processed tabular data, the text data and/or the\n        image data, and the target variable\n    \"\"\"\n\n    path = os.path.join(self.directory, self.fname)\n\n    try:\n        if not hasattr(self, \"colnames\"):\n            self.colnames = pd.read_csv(path, nrows=0).columns.tolist()\n\n        # TO DO: we need to look into this as the treatment is different\n        # whether the csv contains headers or not. For the time being we\n        # will require that the csv file has headers\n\n        _sample = pd.read_csv(\n            path, skiprows=lambda x: x != idx + 1, header=None\n        ).values\n        sample = pd.DataFrame(_sample, columns=self.colnames)\n    except Exception:\n        raise ValueError(\"Currently only csv format is supported.\")\n\n    text_fnames_or_text: Optional[Union[str, List[str]]] = None\n    if self.text_col is not None:\n        if isinstance(self.text_col, list):\n            text_fnames_or_text = [\n                sample[col].to_list()[0] for col in self.text_col\n            ]\n        else:\n            text_fnames_or_text = sample[self.text_col].to_list()[0]\n\n    img_fname: Optional[Union[str, List[str]]] = None\n    if self.img_col is not None:\n        if isinstance(self.img_col, list):\n            img_fname = [sample[col].to_list()[0] for col in self.img_col]\n        else:\n            img_fname = sample[self.img_col].to_list()[0]\n\n    processed_sample = self.preprocessor.transform_sample(sample)\n\n    if not self.ignore_target:\n        target = sample[self.target_col].to_list()[0]\n    else:\n        target = None\n\n    return processed_sample, text_fnames_or_text, img_fname, target\n</code></pre>"},{"location":"pytorch-widedeep/load_from_folder.html#pytorch_widedeep.load_from_folder.tabular.tabular_from_folder.WideFromFolder","title":"WideFromFolder","text":"<p>               Bases: <code>TabFromFolder</code></p> <p>This class is mostly identical to <code>TabFromFolder</code> but exists because we want to separate the treatment of the wide and the deep tabular components</p> <p>Parameters:</p> Name Type Description Default <code>fname</code> <code>str</code> <p>the name of the csv file</p> required <code>directory</code> <code>Optional[str]</code> <p>the path to the directory where the csv file is located. If None, a <code>WideFromFolder</code> reference object must be provided</p> <code>None</code> <code>target_col</code> <code>Optional[str]</code> <p>the name of the target column. If None, a <code>WideFromFolder</code> reference object must be provided</p> <code>None</code> <code>preprocessor</code> <code>Optional[TabularPreprocessor]</code> <p>a fitted <code>TabularPreprocessor</code> object. If None, a <code>WideFromFolder</code> reference object must be provided</p> <code>None</code> <code>text_col</code> <code>Optional[str]</code> <p>the name of the column with the texts themselves or the names of the files that contain the text dataset. If None, either there is no text column or a <code>WideFromFolder</code> reference object must be provided=</p> <code>None</code> <code>img_col</code> <code>Optional[str]</code> <p>the name of the column with the the names of the images. If None, either there is no image column or a <code>WideFromFolder</code> reference object must be provided</p> <code>None</code> <code>ignore_target</code> <code>bool</code> <p>whether to ignore the target column. This is normally used when this class is used for a test dataset.</p> <code>False</code> <code>reference</code> <code>Optional[Any]</code> <p>a reference <code>WideFromFolder</code> object. If provided, the <code>WideFromFolder</code> object will be created using the attributes of the reference object. This is useful to instantiate a <code>WideFromFolder</code> object for evaluation or test purposes</p> <code>None</code> <code>verbose</code> <code>int</code> <p>verbosity. If 0, no output will be printed during the process.</p> <code>1</code> Source code in <code>pytorch_widedeep/load_from_folder/tabular/tabular_from_folder.py</code> <pre><code>class WideFromFolder(TabFromFolder):\n    \"\"\"\n    This class is mostly identical to `TabFromFolder` but exists because we\n    want to separate the treatment of the wide and the deep tabular\n    components\n\n    Parameters\n    ----------\n    fname: str\n        the name of the csv file\n    directory: str, Optional, default = None\n        the path to the directory where the csv file is located. If None,\n        a `WideFromFolder` reference object must be provided\n    target_col: str, Optional, default = None\n        the name of the target column. If None, a `WideFromFolder` reference\n        object must be provided\n    preprocessor: `TabularPreprocessor`, Optional, default = None\n        a fitted `TabularPreprocessor` object. If None, a `WideFromFolder`\n        reference object must be provided\n    text_col: str, Optional, default = None\n        the name of the column with the texts themselves or the names of the\n        files that contain the text dataset. If None, either there is no text\n        column or a `WideFromFolder` reference object must be provided=\n    img_col: str, Optional, default = None\n        the name of the column with the the names of the images. If None,\n        either there is no image column or a `WideFromFolder` reference object\n        must be provided\n    ignore_target: bool, default = False\n        whether to ignore the target column. This is normally used when this\n        class is used for a test dataset.\n    reference: `WideFromFolder`, Optional, default = None\n        a reference `WideFromFolder` object. If provided, the `WideFromFolder`\n        object will be created using the attributes of the reference object.\n        This is useful to instantiate a `WideFromFolder` object for evaluation\n        or test purposes\n    verbose: int, default = 1\n        verbosity. If 0, no output will be printed during the process.\n    \"\"\"\n\n    def __init__(\n        self,\n        fname: str,\n        directory: Optional[str] = None,\n        target_col: Optional[str] = None,\n        preprocessor: Optional[TabularPreprocessor] = None,\n        text_col: Optional[str] = None,\n        img_col: Optional[str] = None,\n        ignore_target: bool = False,\n        reference: Optional[Any] = None,  # is Type[\"WideFromFolder\"],\n        verbose: int = 1,\n    ):\n        super(WideFromFolder, self).__init__(\n            fname=fname,\n            directory=directory,\n            target_col=target_col,\n            preprocessor=preprocessor,\n            text_col=text_col,\n            img_col=img_col,\n            reference=reference,\n            ignore_target=ignore_target,\n            verbose=verbose,\n        )\n</code></pre>"},{"location":"pytorch-widedeep/load_from_folder.html#pytorch_widedeep.load_from_folder.text.text_from_folder.TextFromFolder","title":"TextFromFolder","text":"<p>This class is used to load the text dataset (i.e. the text files) from a folder, or to retrieve the text given a texts column specified within the preprocessor object.</p> <p>For examples, please, see the examples folder in the repo.</p> <p>Parameters:</p> Name Type Description Default <code>preprocessor</code> <code>Union[TextPreprocessor, ChunkTextPreprocessor, HFPreprocessor, ChunkHFPreprocessor, List[TextPreprocessor], List[ChunkTextPreprocessor], List[HFPreprocessor], List[ChunkHFPreprocessor]]</code> <p>The preprocessor used to process the text. It must be fitted before using this class</p> required Source code in <code>pytorch_widedeep/load_from_folder/text/text_from_folder.py</code> <pre><code>class TextFromFolder:\n    \"\"\"\n    This class is used to load the text dataset (i.e. the text files) from a\n    folder, or to retrieve the text given a texts column specified within the\n    preprocessor object.\n\n    For examples, please, see the examples folder in the repo.\n\n    Parameters\n    ----------\n    preprocessor:\n        The preprocessor used to process the text. It must be fitted before using\n        this class\n    \"\"\"\n\n    def __init__(\n        self,\n        preprocessor: Union[\n            TextPreprocessor,\n            ChunkTextPreprocessor,\n            HFPreprocessor,\n            ChunkHFPreprocessor,\n            List[TextPreprocessor],\n            List[ChunkTextPreprocessor],\n            List[HFPreprocessor],\n            List[ChunkHFPreprocessor],\n        ],\n    ):\n        if isinstance(preprocessor, list):\n            for p in preprocessor:\n                assert (\n                    p.is_fitted\n                ), \"All preprocessors must be fitted before using this class\"\n        else:\n            assert (\n                preprocessor.is_fitted\n            ), \"The preprocessor must be fitted before using this class\"\n\n        self.preprocessor = preprocessor\n\n    def get_item(\n        self, text: Union[str, List[str]]\n    ) -&gt; Union[np.ndarray, List[np.ndarray]]:\n        \"\"\"\n        Given a text or a list of texts corresponding to different text\n        columns, this method will return the processed text or a list of\n        processed texts\n\n        Parameters\n        ----------\n        text: Union[str, List[str]]\n            The text or list of texts to be processed\n\n        Returns\n        -------\n        Union[np.ndarray, List[np.ndarray]]\n            The processed text or a list of processed texts\n        \"\"\"\n        if isinstance(self.preprocessor, list):\n            assert isinstance(text, list)\n            processed_sample: Union[np.ndarray, List[np.ndarray]] = [\n                self._preprocess_one_sample(t, self.preprocessor[i])\n                for i, t in enumerate(text)\n            ]\n        else:\n            assert isinstance(text, str)\n            processed_sample = self._preprocess_one_sample(text, self.preprocessor)\n\n        return processed_sample\n\n    def _preprocess_one_sample(\n        self,\n        text: str,\n        preprocessor: Union[\n            TextPreprocessor,\n            ChunkTextPreprocessor,\n            HFPreprocessor,\n            ChunkHFPreprocessor,\n        ],\n    ) -&gt; np.ndarray:\n        if (\n            isinstance(preprocessor, ChunkTextPreprocessor)\n            and preprocessor.root_dir is not None\n        ):\n            path = os.path.join(preprocessor.root_dir, text)\n\n            with open(path, \"r\") as f:\n                sample = f.read().replace(\"\\n\", \"\")\n        else:\n            sample = text\n\n        processed_sample = preprocessor.transform_sample(sample)\n\n        return processed_sample\n\n    def __repr__(self):\n        if isinstance(self.preprocessor, list):\n            return f\"{self.__class__.__name__}({[p.__class__.__name__ for p in self.preprocessor]})\"\n        else:\n            return f\"{self.__class__.__name__}({self.preprocessor.__class__.__name__})\"\n</code></pre>"},{"location":"pytorch-widedeep/load_from_folder.html#pytorch_widedeep.load_from_folder.text.text_from_folder.TextFromFolder.get_item","title":"get_item","text":"<pre><code>get_item(text)\n</code></pre> <p>Given a text or a list of texts corresponding to different text columns, this method will return the processed text or a list of processed texts</p> <p>Parameters:</p> Name Type Description Default <code>text</code> <code>Union[str, List[str]]</code> <p>The text or list of texts to be processed</p> required <p>Returns:</p> Type Description <code>Union[ndarray, List[ndarray]]</code> <p>The processed text or a list of processed texts</p> Source code in <code>pytorch_widedeep/load_from_folder/text/text_from_folder.py</code> <pre><code>def get_item(\n    self, text: Union[str, List[str]]\n) -&gt; Union[np.ndarray, List[np.ndarray]]:\n    \"\"\"\n    Given a text or a list of texts corresponding to different text\n    columns, this method will return the processed text or a list of\n    processed texts\n\n    Parameters\n    ----------\n    text: Union[str, List[str]]\n        The text or list of texts to be processed\n\n    Returns\n    -------\n    Union[np.ndarray, List[np.ndarray]]\n        The processed text or a list of processed texts\n    \"\"\"\n    if isinstance(self.preprocessor, list):\n        assert isinstance(text, list)\n        processed_sample: Union[np.ndarray, List[np.ndarray]] = [\n            self._preprocess_one_sample(t, self.preprocessor[i])\n            for i, t in enumerate(text)\n        ]\n    else:\n        assert isinstance(text, str)\n        processed_sample = self._preprocess_one_sample(text, self.preprocessor)\n\n    return processed_sample\n</code></pre>"},{"location":"pytorch-widedeep/load_from_folder.html#pytorch_widedeep.load_from_folder.image.image_from_folder.ImageFromFolder","title":"ImageFromFolder","text":"<p>This class is used to load the image dataset from disk. It is inspired by the <code>ImageFolder</code> class at the <code>torchvision</code> library. Here, we have simply adapted to work within the context of a Wide and Deep multi-modal model.</p> <p>For examples, please, see the examples folder in the repo.</p> <p>Parameters:</p> Name Type Description Default <code>directory</code> <code>Optional[Union[str, List[str]]]</code> <p>the path to the directory where the images are located. If None, a preprocessor must be provided.</p> <code>None</code> <code>preprocessor</code> <code>Optional[Union[ImagePreprocessor, List[ImagePreprocessor]]]</code> <p>a fitted <code>ImagePreprocessor</code> object.</p> <code>None</code> <code>loader</code> <code>Callable[[str], Any]</code> <p>a function to load a sample given its path.</p> <code>default_loader</code> <code>extensions</code> <code>Optional[Tuple[str, ...]]</code> <p>a tuple with the allowed extensions. If None, IMG_EXTENSIONS will be used where IMG_EXTENSIONS =\".jpg\", \".jpeg\", \".png\", \".ppm\", \".bmp\", \".pgm\", \".tif\", \".tiff\", \".webp\"</p> <code>None</code> <code>transforms</code> <code>Optional[Any]</code> <p>a <code>torchvision.transforms</code> object. If None, this class will simply return an array representation of the PIL Image</p> <code>None</code> Source code in <code>pytorch_widedeep/load_from_folder/image/image_from_folder.py</code> <pre><code>class ImageFromFolder:\n    \"\"\"\n    This class is used to load the image dataset from disk. It is inspired by\n    the `ImageFolder` class at the `torchvision` library. Here, we have\n    simply adapted to work within the context of a Wide and Deep multi-modal\n    model.\n\n    For examples, please, see the examples folder in the repo.\n\n    Parameters\n    ----------\n    directory: str, Optional, default = None\n        the path to the directory where the images are located. If None, a\n        preprocessor must be provided.\n    preprocessor: `ImagePreprocessor`, Optional, default = None\n        a fitted `ImagePreprocessor` object.\n    loader: Callable[[str], Any], Optional, default = default_loader\n        a function to load a sample given its path.\n    extensions: Tuple[str, ...], Optional, default = IMG_EXTENSIONS\n        a tuple with the allowed extensions. If None, IMG_EXTENSIONS will be\n        used where IMG_EXTENSIONS\n        =\".jpg\", \".jpeg\", \".png\", \".ppm\", \".bmp\", \".pgm\", \".tif\", \".tiff\", \".webp\"\n    transforms: Optional[Any], default = None\n        a `torchvision.transforms` object. If None, this class will simply\n        return an array representation of the PIL Image\n    \"\"\"\n\n    def __init__(\n        self,\n        directory: Optional[Union[str, List[str]]] = None,\n        preprocessor: Optional[\n            Union[ImagePreprocessor, List[ImagePreprocessor]]\n        ] = None,\n        loader: Callable[[str], Any] = default_loader,\n        extensions: Optional[Tuple[str, ...]] = None,\n        transforms: Optional[Any] = None,\n    ) -&gt; None:\n        assert (\n            directory is not None or preprocessor is not None\n        ), \"Either a directory or an instance of ImagePreprocessor(s) must be provided\"\n\n        if directory is not None and preprocessor is not None:  # pragma: no cover\n            error_msg = (\n                \"If both 'directory' and 'preprocessor' are provided, the 'img_path' \"\n                \"attribute of the 'preprocessor' must be the same as the 'directory'\"\n            )\n            if isinstance(directory, list):\n                assert isinstance(preprocessor, list)\n                assert len(directory) == len(preprocessor)\n                for d, p in zip(directory, preprocessor):\n                    assert d == p.img_path, error_msg\n            else:\n                assert isinstance(preprocessor, ImagePreprocessor)\n                assert directory == preprocessor.img_path, error_msg\n\n        if directory is not None:\n            self.directory = directory\n        else:\n            assert (\n                preprocessor is not None\n            ), \"Either a directory or an instance of ImagePreprocessor must be provided\"\n            if isinstance(preprocessor, list):\n                self.directory = [p.img_path for p in preprocessor]\n            else:\n                self.directory = preprocessor.img_path\n\n        self.preprocessor = preprocessor\n        self.loader = loader\n        self.extensions = extensions if extensions is not None else IMG_EXTENSIONS\n        self.transforms = transforms\n        if self.transforms:\n            self.transforms_names = [\n                tr.__class__.__name__ for tr in self.transforms.transforms\n            ]\n        else:\n            self.transforms_names = []\n\n            self.transpose = True\n\n    def get_item(\n        self, fname: Union[str, List[str]]\n    ) -&gt; Union[np.ndarray, List[np.ndarray]]:\n        \"\"\"\n        This method is used to load the image dataset(s) from disk.\n\n        Parameters\n        ----------\n        fname: Union[str, List[str]]\n            the name of the image file(s) to load. If a list is provided, the\n            method will return a list of numpy arrays. Each element in the list\n            corresponds to the image in the different image columns.\n\n        Returns\n        -------\n        Union[np.ndarray, List[np.ndarray]]\n            a numpy array or a list of numpy arrays representing the image(s)\n        \"\"\"\n\n        if isinstance(fname, list):\n            if not isinstance(self.directory, list):\n                _directory = [self.directory] * len(fname)\n            else:\n                _directory = self.directory\n            if self.preprocessor is not None:\n                assert isinstance(self.preprocessor, list)\n                processed_sample: Union[np.ndarray, List[np.ndarray]] = [\n                    self._preprocess_one_sample(f, d, p)\n                    for f, d, p in zip(fname, _directory, self.preprocessor)\n                ]\n            else:\n                processed_sample = [\n                    self._preprocess_one_sample(f, d) for f, d in zip(fname, _directory)\n                ]\n        else:\n            assert isinstance(self.directory, str)\n            if self.preprocessor is not None:\n                assert isinstance(self.preprocessor, ImagePreprocessor)\n                processed_sample = self._preprocess_one_sample(\n                    fname, self.directory, self.preprocessor\n                )\n            else:\n                processed_sample = self._preprocess_one_sample(fname, self.directory)\n\n        return processed_sample\n\n    def _preprocess_one_sample(\n        self,\n        fname: str,\n        directory: str,\n        preprocessor: Optional[ImagePreprocessor] = None,\n    ) -&gt; np.ndarray:\n        assert has_file_allowed_extension(fname, self.extensions)\n\n        path = os.path.join(directory, fname)\n        sample = self.loader(path)\n\n        assert isinstance(sample, (Image.Image, np.ndarray)), (  # pragma: no cover\n            \"The loader must return an instance of PIL.Image or np.ndarray, \"\n            f\"got {type(sample)} instead\"\n        )\n\n        if preprocessor is not None:\n            if not isinstance(sample, np.ndarray):\n                processed_sample = preprocessor.transform_sample(np.asarray(sample))\n            else:\n                processed_sample = preprocessor.transform_sample(sample)\n        else:\n            processed_sample = sample  # type: ignore\n\n        prepared_sample = self._prepare_sample(processed_sample)\n\n        return prepared_sample\n\n    def _prepare_sample(  # noqa: C901\n        self, processed_sample: Union[np.ndarray, Image.Image]\n    ) -&gt; np.ndarray:\n        # if an image dataset is used, make sure is in the right format to\n        # be ingested by the conv layers\n\n        if isinstance(processed_sample, Image.Image):\n            processed_sample = np.asarray(processed_sample)\n\n        # if int must be uint8\n        if \"int\" in str(processed_sample.dtype) and \"uint8\" != str(\n            processed_sample.dtype\n        ):\n            processed_sample = processed_sample.astype(\"uint8\")\n\n        # if float must be float32\n        if \"float\" in str(processed_sample.dtype) and \"float32\" != str(\n            processed_sample.dtype\n        ):\n            processed_sample = processed_sample.astype(\"float32\")\n\n        # if there are no transforms, or these do not include ToTensor()\n        # (weird or unexpected case, not sure is even possible) then we need\n        # to  replicate what ToTensor() does -&gt; transpose axis and normalize if\n        # necessary\n        if not self.transforms or \"ToTensor\" not in self.transforms_names:\n            if processed_sample.ndim == 2:\n                processed_sample = processed_sample[:, :, None]\n\n            processed_sample = processed_sample.transpose(2, 0, 1)\n\n            if \"int\" in str(processed_sample.dtype):\n                processed_sample = (processed_sample / processed_sample.max()).astype(\n                    \"float32\"\n                )\n        elif \"ToTensor\" in self.transforms_names:\n            # if ToTensor() is included, simply apply transforms\n            assert self.transforms_names[0] == \"ToTensor\", (\n                \"If ToTensor() is included in the transforms, it must be the \"\n                \"first transform in the list\"\n            )\n            processed_sample = self.transforms(processed_sample)\n        else:\n            # else apply transforms on the result of calling torch.tensor on\n            # processed_sample after all the previous manipulation\n            processed_sample = self.transforms(torch.tensor(processed_sample))\n\n        return processed_sample  # type: ignore\n\n    def __repr__(self) -&gt; str:\n        list_of_params: List[str] = []\n        if self.directory is not None:\n            list_of_params.append(\"directory={directory}\")\n        if self.preprocessor is not None:\n            list_of_params.append(\n                f\"preprocessor={self.preprocessor.__class__.__name__}\"\n            )\n        if self.loader is not None:\n            list_of_params.append(f\"loader={self.loader.__name__}\")\n        if self.extensions is not None:\n            list_of_params.append(\"extensions={extensions}\")\n        if self.transforms is not None:\n            list_of_params.append(f\"transforms={self.transforms_names}\")\n        all_params = \", \".join(list_of_params)\n        return f\"ImageFromFolder({all_params.format(**self.__dict__)})\"\n</code></pre>"},{"location":"pytorch-widedeep/load_from_folder.html#pytorch_widedeep.load_from_folder.image.image_from_folder.ImageFromFolder.get_item","title":"get_item","text":"<pre><code>get_item(fname)\n</code></pre> <p>This method is used to load the image dataset(s) from disk.</p> <p>Parameters:</p> Name Type Description Default <code>fname</code> <code>Union[str, List[str]]</code> <p>the name of the image file(s) to load. If a list is provided, the method will return a list of numpy arrays. Each element in the list corresponds to the image in the different image columns.</p> required <p>Returns:</p> Type Description <code>Union[ndarray, List[ndarray]]</code> <p>a numpy array or a list of numpy arrays representing the image(s)</p> Source code in <code>pytorch_widedeep/load_from_folder/image/image_from_folder.py</code> <pre><code>def get_item(\n    self, fname: Union[str, List[str]]\n) -&gt; Union[np.ndarray, List[np.ndarray]]:\n    \"\"\"\n    This method is used to load the image dataset(s) from disk.\n\n    Parameters\n    ----------\n    fname: Union[str, List[str]]\n        the name of the image file(s) to load. If a list is provided, the\n        method will return a list of numpy arrays. Each element in the list\n        corresponds to the image in the different image columns.\n\n    Returns\n    -------\n    Union[np.ndarray, List[np.ndarray]]\n        a numpy array or a list of numpy arrays representing the image(s)\n    \"\"\"\n\n    if isinstance(fname, list):\n        if not isinstance(self.directory, list):\n            _directory = [self.directory] * len(fname)\n        else:\n            _directory = self.directory\n        if self.preprocessor is not None:\n            assert isinstance(self.preprocessor, list)\n            processed_sample: Union[np.ndarray, List[np.ndarray]] = [\n                self._preprocess_one_sample(f, d, p)\n                for f, d, p in zip(fname, _directory, self.preprocessor)\n            ]\n        else:\n            processed_sample = [\n                self._preprocess_one_sample(f, d) for f, d in zip(fname, _directory)\n            ]\n    else:\n        assert isinstance(self.directory, str)\n        if self.preprocessor is not None:\n            assert isinstance(self.preprocessor, ImagePreprocessor)\n            processed_sample = self._preprocess_one_sample(\n                fname, self.directory, self.preprocessor\n            )\n        else:\n            processed_sample = self._preprocess_one_sample(fname, self.directory)\n\n    return processed_sample\n</code></pre>"},{"location":"pytorch-widedeep/load_from_folder.html#pytorch_widedeep.load_from_folder.wd_dataset_from_folder.WideDeepDatasetFromFolder","title":"WideDeepDatasetFromFolder","text":"<p>               Bases: <code>Dataset</code></p> <p>This class is the Dataset counterpart of the <code>WideDeepDataset</code> class.</p> <p>Given a reference tabular dataset, with columns that indicate the path to the images and to the text files or the texts themselves, it will use the <code>[...]FromFolder</code> classes to load the data consistently from disk per batch.</p> <p>For examples, please, see the examples folder in the repo.</p> <p>Parameters:</p> Name Type Description Default <code>n_samples</code> <code>int</code> <p>Number of samples in the dataset</p> required <code>tab_from_folder</code> <code>Optional[TabFromFolder]</code> <p>Instance of the <code>TabFromFolder</code> class</p> <code>None</code> <code>wide_from_folder</code> <code>Optional[WideFromFolder]</code> <p>Instance of the <code>WideFromFolder</code> class</p> <code>None</code> <code>text_from_folder</code> <code>Optional[TextFromFolder]</code> <p>Instance of the <code>TextFromFolder</code> class</p> <code>None</code> <code>img_from_folder</code> <code>Optional[ImageFromFolder]</code> <p>Instance of the <code>ImageFromFolder</code> class</p> <code>None</code> <code>reference</code> <code>Optional[Any]</code> <p>If <code>reference</code> not None, the <code>text_from_folder</code> and <code>img_from_folder</code> objects will be retrieved from the reference class. This is useful when we want to use a <code>WideDeepDatasetFromFolder</code> class used for a train dataset as a reference for the validation and test datasets. In this case, the <code>text_from_folder</code> and <code>img_from_folder</code> objects will be the same for all three datasets, so there is no need to create a new instance for each dataset.</p> <code>None</code> Source code in <code>pytorch_widedeep/load_from_folder/wd_dataset_from_folder.py</code> <pre><code>class WideDeepDatasetFromFolder(Dataset):\n    \"\"\"\n    This class is the Dataset counterpart of the `WideDeepDataset` class.\n\n    Given a reference tabular dataset, with columns that indicate the path to\n    the images and to the text files or the texts themselves, it will use the\n    `[...]FromFolder` classes to load the data consistently from disk per batch.\n\n    For examples, please, see the examples folder in the repo.\n\n    Parameters\n    ----------\n    n_samples: int\n        Number of samples in the dataset\n    tab_from_folder: TabFromFolder\n        Instance of the `TabFromFolder` class\n    wide_from_folder: Optional[WideFromFolder], default = None\n        Instance of the `WideFromFolder` class\n    text_from_folder: Optional[TextFromFolder], default = None\n        Instance of the `TextFromFolder` class\n    img_from_folder: Optional[ImageFromFolder], default = None\n        Instance of the `ImageFromFolder` class\n    reference: Type[\"WideDeepDatasetFromFolder\"], default = None\n        If `reference` not None, the `text_from_folder` and `img_from_folder`\n        objects will be retrieved from the reference class. This is useful\n        when we want to use a `WideDeepDatasetFromFolder` class used for a\n        train dataset as a reference for the validation and test datasets. In\n        this case, the `text_from_folder` and `img_from_folder` objects will\n        be the same for all three datasets, so there is no need to create a\n        new instance for each dataset.\n    \"\"\"\n\n    def __init__(\n        self,\n        n_samples: int,\n        tab_from_folder: Optional[TabFromFolder] = None,\n        wide_from_folder: Optional[WideFromFolder] = None,\n        text_from_folder: Optional[TextFromFolder] = None,\n        img_from_folder: Optional[ImageFromFolder] = None,\n        reference: Optional[Any] = None,  # is Type[\"WideDeepDatasetFromFolder\"],\n    ):\n        super(WideDeepDatasetFromFolder, self).__init__()\n\n        if tab_from_folder is None and wide_from_folder is None:\n            raise ValueError(\n                \"Either 'tab_from_folder' or 'wide_from_folder' must be not None\"\n            )\n\n        if reference is not None:\n            assert (\n                img_from_folder is None and text_from_folder is None\n            ), \"If reference is not None, 'img_from_folder' and 'text_from_folder' left as None\"\n            self.text_from_folder, self.img_from_folder = self._get_from_reference(\n                reference\n            )\n        else:\n            assert (\n                text_from_folder is not None and img_from_folder is not None\n            ), \"If reference is None, 'img_from_folder' and 'text_from_folder' must be not None\"\n            self.text_from_folder = text_from_folder\n            self.img_from_folder = img_from_folder\n\n        self.n_samples = n_samples\n        self.tab_from_folder = tab_from_folder\n        self.wide_from_folder = wide_from_folder\n\n    def __getitem__(self, idx: int):  # noqa: C901\n        x = (\n            Bunch()\n        )  # for consistency with WideDeepDataset, but this is just a Dict[str, Any]\n\n        if self.tab_from_folder is not None:\n            X_tab, text_fname_or_text, img_fname, y = self.tab_from_folder.get_item(\n                idx=idx\n            )\n            x.deeptabular = X_tab\n\n        if self.wide_from_folder is not None:\n            if self.tab_from_folder is None:\n                (\n                    X_wide,\n                    text_fname_or_text,\n                    img_fname,\n                    y,\n                ) = self.wide_from_folder.get_item(idx=idx)\n            else:\n                X_wide, _, _, _ = self.wide_from_folder.get_item(idx=idx)\n            x.wide = X_wide\n\n        if text_fname_or_text is not None:\n            # These assertions should never be raised, but just in case...\n            assert (\n                self.text_from_folder is not None\n            ), \"text_fname_or_text is not None but self.text_from_folder is None\"\n            X_text = self.text_from_folder.get_item(text_fname_or_text)\n            x.deeptext = X_text\n\n        if img_fname is not None:\n            assert (\n                self.img_from_folder is not None\n            ), \"img_fname is not None but self.img_from_folder is None\"\n            X_img = self.img_from_folder.get_item(img_fname)\n            x.deepimage = X_img\n\n        # We are aware that returning sometimes X and sometimes X, y is not\n        # the best practice, but is the easiest way at this stage\n        if y is not None:\n            return x, y\n        else:\n            return x\n\n    def __len__(self):\n        return self.n_samples\n\n    @staticmethod\n    def _get_from_reference(\n        reference: Type[\"WideDeepDatasetFromFolder\"],\n    ) -&gt; Tuple[Optional[TextFromFolder], Optional[ImageFromFolder]]:\n        return reference.text_from_folder, reference.img_from_folder\n\n    def __repr__(self) -&gt; str:\n        list_of_params: List[str] = []\n        list_of_params.append(\"n_samples={n_samples}\")\n        if self.tab_from_folder is not None:\n            list_of_params.append(\n                f\"tab_from_folder={self.tab_from_folder.__class__.__name__}\"\n            )\n        if self.wide_from_folder is not None:\n            list_of_params.append(\n                f\"wide_from_folder={self.wide_from_folder.__class__.__name__}\"\n            )\n        if self.text_from_folder is not None:\n            list_of_params.append(\n                f\"text_from_folder={self.text_from_folder.__class__.__name__}\"\n            )\n        if self.img_from_folder is not None:\n            list_of_params.append(\n                f\"img_from_folder={self.img_from_folder.__class__.__name__}\"\n            )\n        all_params = \", \".join(list_of_params)\n        return f\"WideDeepDatasetFromFolder({all_params.format(**self.__dict__)})\"\n</code></pre>"},{"location":"pytorch-widedeep/losses.html","title":"Losses","text":"<p><code>pytorch-widedeep</code> accepts a number of losses and objectives that can be passed to the <code>Trainer</code> class via the parameter <code>objective</code> (see <code>pytorch-widedeep.training.Trainer</code>). For most cases the loss function that <code>pytorch-widedeep</code> will use internally is already implemented in Pytorch.</p> <p>In addition, <code>pytorch-widedeep</code> implements a series of  \"custom\" loss functions. These are described below for completion since, as mentioned before, they are used internally by the <code>Trainer</code>. Of course, onen could always use them on their own and can be imported as:</p> <p><code>from pytorch_widedeep.losses import FocalLoss</code></p> <p> NOTE:  Losses in this module expect the predictions  and ground truth to have the same dimensions for regression and binary  classification problems \\((N_{samples}, 1)\\). In the case of multiclass  classification problems the ground truth is expected to be a 1D tensor with  the corresponding classes. See Examples below</p>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.MSELoss","title":"MSELoss","text":"<p>               Bases: <code>Module</code></p> <p>Mean square error loss</p> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>class MSELoss(nn.Module):\n    r\"\"\"Mean square error loss\"\"\"\n\n    # legacy code from when we used to support FDS-LDS and this class could\n    # taked the corresponding params. At this stage probably you want to use\n    # torch.nn.MSELoss\n    def __init__(self):\n        super().__init__()\n\n    def forward(\n        self,\n        input: Tensor,\n        target: Tensor,\n    ) -&gt; Tensor:\n        r\"\"\"\n        Parameters\n        ----------\n        input: Tensor\n            Input tensor with predictions\n        target: Tensor\n            Target tensor with the actual values\n\n        Examples\n        --------\n        &gt;&gt;&gt; import torch\n        &gt;&gt;&gt; from pytorch_widedeep.losses import MSELoss\n        &gt;&gt;&gt;\n        &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n        &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n        &gt;&gt;&gt; loss = MSELoss()(input, target)\n        \"\"\"\n        loss = (input - target) ** 2\n        return torch.mean(loss)\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.MSELoss.forward","title":"forward","text":"<pre><code>forward(input, target)\n</code></pre> <p>Parameters:</p> Name Type Description Default <code>input</code> <code>Tensor</code> <p>Input tensor with predictions</p> required <code>target</code> <code>Tensor</code> <p>Target tensor with the actual values</p> required <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.losses import MSELoss\n&gt;&gt;&gt;\n&gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n&gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n&gt;&gt;&gt; loss = MSELoss()(input, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(\n    self,\n    input: Tensor,\n    target: Tensor,\n) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    input: Tensor\n        Input tensor with predictions\n    target: Tensor\n        Target tensor with the actual values\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.losses import MSELoss\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n    &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n    &gt;&gt;&gt; loss = MSELoss()(input, target)\n    \"\"\"\n    loss = (input - target) ** 2\n    return torch.mean(loss)\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.MSLELoss","title":"MSLELoss","text":"<p>               Bases: <code>Module</code></p> <p>Mean square log error loss</p> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>class MSLELoss(nn.Module):\n    r\"\"\"Mean square log error loss\"\"\"\n\n    def __init__(self):\n        super().__init__()\n\n    def forward(\n        self,\n        input: Tensor,\n        target: Tensor,\n    ) -&gt; Tensor:\n        r\"\"\"\n        Parameters\n        ----------\n        input: Tensor\n            Input tensor with predictions (not probabilities)\n        target: Tensor\n            Target tensor with the actual classes\n\n        Examples\n        --------\n        &gt;&gt;&gt; import torch\n        &gt;&gt;&gt; from pytorch_widedeep.losses import MSLELoss\n        &gt;&gt;&gt;\n        &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n        &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n        &gt;&gt;&gt; loss = MSLELoss()(input, target)\n        \"\"\"\n        assert (\n            input.min() &gt;= 0\n        ), \"\"\"All input values must be &gt;=0, if your model is predicting\n            values &lt;0 try to enforce positive values by activation function\n            on last layer with `trainer.enforce_positive_output=True`\"\"\"\n        assert target.min() &gt;= 0, \"All target values must be &gt;=0\"\n\n        loss = (torch.log(input + 1) - torch.log(target + 1)) ** 2\n        return torch.mean(loss)\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.MSLELoss.forward","title":"forward","text":"<pre><code>forward(input, target)\n</code></pre> <p>Parameters:</p> Name Type Description Default <code>input</code> <code>Tensor</code> <p>Input tensor with predictions (not probabilities)</p> required <code>target</code> <code>Tensor</code> <p>Target tensor with the actual classes</p> required <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.losses import MSLELoss\n&gt;&gt;&gt;\n&gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n&gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n&gt;&gt;&gt; loss = MSLELoss()(input, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(\n    self,\n    input: Tensor,\n    target: Tensor,\n) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    input: Tensor\n        Input tensor with predictions (not probabilities)\n    target: Tensor\n        Target tensor with the actual classes\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.losses import MSLELoss\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n    &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n    &gt;&gt;&gt; loss = MSLELoss()(input, target)\n    \"\"\"\n    assert (\n        input.min() &gt;= 0\n    ), \"\"\"All input values must be &gt;=0, if your model is predicting\n        values &lt;0 try to enforce positive values by activation function\n        on last layer with `trainer.enforce_positive_output=True`\"\"\"\n    assert target.min() &gt;= 0, \"All target values must be &gt;=0\"\n\n    loss = (torch.log(input + 1) - torch.log(target + 1)) ** 2\n    return torch.mean(loss)\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.RMSELoss","title":"RMSELoss","text":"<p>               Bases: <code>Module</code></p> <p>Root mean square error loss</p> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>class RMSELoss(nn.Module):\n    r\"\"\"Root mean square error loss\"\"\"\n\n    # legacy code from when we used to support FDS-LDS and this class could\n    # taked the corresponding params. At this stage probably you want to use\n    # torch.sqrt(nn.MSELoss)\n    def __init__(self):\n        super().__init__()\n\n    def forward(self, input: Tensor, target: Tensor) -&gt; Tensor:\n        r\"\"\"\n        Parameters\n        ----------\n        input: Tensor\n            Input tensor with predictions (not probabilities)\n        target: Tensor\n            Target tensor with the actual classes\n\n        Examples\n        --------\n        &gt;&gt;&gt; import torch\n        &gt;&gt;&gt; from pytorch_widedeep.losses import RMSELoss\n        &gt;&gt;&gt;\n        &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n        &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n        &gt;&gt;&gt; loss = RMSELoss()(input, target)\n        \"\"\"\n        loss = (input - target) ** 2\n        return torch.sqrt(torch.mean(loss))\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.RMSELoss.forward","title":"forward","text":"<pre><code>forward(input, target)\n</code></pre> <p>Parameters:</p> Name Type Description Default <code>input</code> <code>Tensor</code> <p>Input tensor with predictions (not probabilities)</p> required <code>target</code> <code>Tensor</code> <p>Target tensor with the actual classes</p> required <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.losses import RMSELoss\n&gt;&gt;&gt;\n&gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n&gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n&gt;&gt;&gt; loss = RMSELoss()(input, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(self, input: Tensor, target: Tensor) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    input: Tensor\n        Input tensor with predictions (not probabilities)\n    target: Tensor\n        Target tensor with the actual classes\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.losses import RMSELoss\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n    &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n    &gt;&gt;&gt; loss = RMSELoss()(input, target)\n    \"\"\"\n    loss = (input - target) ** 2\n    return torch.sqrt(torch.mean(loss))\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.RMSLELoss","title":"RMSLELoss","text":"<p>               Bases: <code>Module</code></p> <p>Root mean square log error loss</p> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>class RMSLELoss(nn.Module):\n    r\"\"\"Root mean square log error loss\"\"\"\n\n    def __init__(self):\n        super().__init__()\n\n    def forward(self, input: Tensor, target: Tensor) -&gt; Tensor:\n        r\"\"\"\n        Parameters\n        ----------\n        input: Tensor\n            Input tensor with predictions (not probabilities)\n        target: Tensor\n            Target tensor with the actual classes\n\n        Examples\n        --------\n        &gt;&gt;&gt; import torch\n        &gt;&gt;&gt; from pytorch_widedeep.losses import RMSLELoss\n        &gt;&gt;&gt;\n        &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n        &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n        &gt;&gt;&gt; loss = RMSLELoss()(input, target)\n        \"\"\"\n        assert (\n            input.min() &gt;= 0\n        ), \"\"\"All input values must be &gt;=0, if your model is predicting\n            values &lt;0 try to enforce positive values by activation function\n            on last layer with `trainer.enforce_positive_output=True`\"\"\"\n        assert target.min() &gt;= 0, \"All target values must be &gt;=0\"\n\n        loss = (torch.log(input + 1) - torch.log(target + 1)) ** 2\n        return torch.sqrt(torch.mean(loss))\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.RMSLELoss.forward","title":"forward","text":"<pre><code>forward(input, target)\n</code></pre> <p>Parameters:</p> Name Type Description Default <code>input</code> <code>Tensor</code> <p>Input tensor with predictions (not probabilities)</p> required <code>target</code> <code>Tensor</code> <p>Target tensor with the actual classes</p> required <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.losses import RMSLELoss\n&gt;&gt;&gt;\n&gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n&gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n&gt;&gt;&gt; loss = RMSLELoss()(input, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(self, input: Tensor, target: Tensor) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    input: Tensor\n        Input tensor with predictions (not probabilities)\n    target: Tensor\n        Target tensor with the actual classes\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.losses import RMSLELoss\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n    &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n    &gt;&gt;&gt; loss = RMSLELoss()(input, target)\n    \"\"\"\n    assert (\n        input.min() &gt;= 0\n    ), \"\"\"All input values must be &gt;=0, if your model is predicting\n        values &lt;0 try to enforce positive values by activation function\n        on last layer with `trainer.enforce_positive_output=True`\"\"\"\n    assert target.min() &gt;= 0, \"All target values must be &gt;=0\"\n\n    loss = (torch.log(input + 1) - torch.log(target + 1)) ** 2\n    return torch.sqrt(torch.mean(loss))\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.QuantileLoss","title":"QuantileLoss","text":"<p>               Bases: <code>Module</code></p> <p>Quantile loss defined as:</p> \\[ Loss = max(q \\times (y-y_{pred}), (1-q) \\times (y_{pred}-y)) \\] <p>All credits go to the implementation at pytorch-forecasting.</p> <p>Parameters:</p> Name Type Description Default <code>quantiles</code> <code>List[float]</code> <p>List of quantiles</p> <code>[0.02, 0.1, 0.25, 0.5, 0.75, 0.9, 0.98]</code> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>class QuantileLoss(nn.Module):\n    r\"\"\"Quantile loss defined as:\n\n    $$\n    Loss = max(q \\times (y-y_{pred}), (1-q) \\times (y_{pred}-y))\n    $$\n\n    All credits go to the implementation at\n    [pytorch-forecasting](https://pytorch-forecasting.readthedocs.io/en/latest/_modules/pytorch_forecasting/metrics.html#QuantileLoss).\n\n    Parameters\n    ----------\n    quantiles: List, default = [0.02, 0.1, 0.25, 0.5, 0.75, 0.9, 0.98]\n        List of quantiles\n    \"\"\"\n\n    def __init__(\n        self,\n        quantiles: List[float] = [0.02, 0.1, 0.25, 0.5, 0.75, 0.9, 0.98],\n    ):\n        super().__init__()\n        self.quantiles = quantiles\n\n    def forward(self, input: Tensor, target: Tensor) -&gt; Tensor:\n        r\"\"\"\n        Parameters\n        ----------\n        input: Tensor\n            Input tensor with predictions\n        target: Tensor\n            Target tensor with the actual values\n\n        Examples\n        --------\n        &gt;&gt;&gt; import torch\n        &gt;&gt;&gt;\n        &gt;&gt;&gt; from pytorch_widedeep.losses import QuantileLoss\n        &gt;&gt;&gt;\n        &gt;&gt;&gt; # REGRESSION\n        &gt;&gt;&gt; target = torch.tensor([[0.6, 1.5]]).view(-1, 1)\n        &gt;&gt;&gt; input = torch.tensor([[.1, .2,], [.4, .5]])\n        &gt;&gt;&gt; qloss = QuantileLoss([0.25, 0.75])\n        &gt;&gt;&gt; loss = qloss(input, target)\n        \"\"\"\n\n        assert input.shape == torch.Size([target.shape[0], len(self.quantiles)]), (\n            \"The input and target have inconsistent shape. The dimension of the prediction \"\n            \"of the model that is using QuantileLoss must be equal to number of quantiles, \"\n            f\"i.e. {len(self.quantiles)}.\"\n        )\n        target = target.view(-1, 1).float()\n        losses = []\n        for i, q in enumerate(self.quantiles):\n            errors = target - input[..., i]\n            losses.append(torch.max((q - 1) * errors, q * errors).unsqueeze(-1))\n\n        loss = torch.cat(losses, dim=2)\n\n        return torch.mean(loss)\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.QuantileLoss.forward","title":"forward","text":"<pre><code>forward(input, target)\n</code></pre> <p>Parameters:</p> Name Type Description Default <code>input</code> <code>Tensor</code> <p>Input tensor with predictions</p> required <code>target</code> <code>Tensor</code> <p>Target tensor with the actual values</p> required <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt;\n&gt;&gt;&gt; from pytorch_widedeep.losses import QuantileLoss\n&gt;&gt;&gt;\n&gt;&gt;&gt; # REGRESSION\n&gt;&gt;&gt; target = torch.tensor([[0.6, 1.5]]).view(-1, 1)\n&gt;&gt;&gt; input = torch.tensor([[.1, .2,], [.4, .5]])\n&gt;&gt;&gt; qloss = QuantileLoss([0.25, 0.75])\n&gt;&gt;&gt; loss = qloss(input, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(self, input: Tensor, target: Tensor) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    input: Tensor\n        Input tensor with predictions\n    target: Tensor\n        Target tensor with the actual values\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; from pytorch_widedeep.losses import QuantileLoss\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; # REGRESSION\n    &gt;&gt;&gt; target = torch.tensor([[0.6, 1.5]]).view(-1, 1)\n    &gt;&gt;&gt; input = torch.tensor([[.1, .2,], [.4, .5]])\n    &gt;&gt;&gt; qloss = QuantileLoss([0.25, 0.75])\n    &gt;&gt;&gt; loss = qloss(input, target)\n    \"\"\"\n\n    assert input.shape == torch.Size([target.shape[0], len(self.quantiles)]), (\n        \"The input and target have inconsistent shape. The dimension of the prediction \"\n        \"of the model that is using QuantileLoss must be equal to number of quantiles, \"\n        f\"i.e. {len(self.quantiles)}.\"\n    )\n    target = target.view(-1, 1).float()\n    losses = []\n    for i, q in enumerate(self.quantiles):\n        errors = target - input[..., i]\n        losses.append(torch.max((q - 1) * errors, q * errors).unsqueeze(-1))\n\n    loss = torch.cat(losses, dim=2)\n\n    return torch.mean(loss)\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.FocalLoss","title":"FocalLoss","text":"<p>               Bases: <code>Module</code></p> <p>Implementation of the Focal loss for both binary and multiclass classification:</p> \\[ FL(p_t) = \\alpha (1 - p_t)^{\\gamma} log(p_t) \\] <p>where, for a case of a binary classification problem</p> \\[ \\begin{equation} p_t= \\begin{cases}p, &amp; \\text{if $y=1$}.\\\\1-p, &amp; \\text{otherwise}. \\end{cases} \\end{equation} \\] <p>Parameters:</p> Name Type Description Default <code>alpha</code> <code>float</code> <p>Focal Loss <code>alpha</code> parameter</p> <code>0.25</code> <code>gamma</code> <code>float</code> <p>Focal Loss <code>gamma</code> parameter</p> <code>1.0</code> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>class FocalLoss(nn.Module):\n    r\"\"\"Implementation of the [Focal loss](https://arxiv.org/pdf/1708.02002.pdf)\n    for both binary and multiclass classification:\n\n    $$\n    FL(p_t) = \\alpha (1 - p_t)^{\\gamma} log(p_t)\n    $$\n\n    where, for a case of a binary classification problem\n\n    $$\n    \\begin{equation} p_t= \\begin{cases}p, &amp; \\text{if $y=1$}.\\\\1-p, &amp; \\text{otherwise}. \\end{cases} \\end{equation}\n    $$\n\n    Parameters\n    ----------\n    alpha: float\n        Focal Loss `alpha` parameter\n    gamma: float\n        Focal Loss `gamma` parameter\n    \"\"\"\n\n    def __init__(self, alpha: float = 0.25, gamma: float = 1.0):\n        super().__init__()\n        self.alpha = alpha\n        self.gamma = gamma\n\n    def _get_weight(self, p: Tensor, t: Tensor) -&gt; Tensor:\n        pt = p * t + (1 - p) * (1 - t)  # type: ignore\n        w = self.alpha * t + (1 - self.alpha) * (1 - t)  # type: ignore\n        return (w * (1 - pt).pow(self.gamma)).detach()  # type: ignore\n\n    def forward(self, input: Tensor, target: Tensor) -&gt; Tensor:\n        r\"\"\"\n        Parameters\n        ----------\n        input: Tensor\n            Input tensor with predictions (not probabilities)\n        target: Tensor\n            Target tensor with the actual classes\n\n        Examples\n        --------\n        &gt;&gt;&gt; import torch\n        &gt;&gt;&gt;\n        &gt;&gt;&gt; from pytorch_widedeep.losses import FocalLoss\n        &gt;&gt;&gt;\n        &gt;&gt;&gt; # BINARY\n        &gt;&gt;&gt; target = torch.tensor([0, 1, 0, 1]).view(-1, 1)\n        &gt;&gt;&gt; input = torch.tensor([[0.6, 0.7, 0.3, 0.8]]).t()\n        &gt;&gt;&gt; loss = FocalLoss()(input, target)\n        &gt;&gt;&gt;\n        &gt;&gt;&gt; # MULTICLASS\n        &gt;&gt;&gt; target = torch.tensor([1, 0, 2]).view(-1, 1)\n        &gt;&gt;&gt; input = torch.tensor([[0.2, 0.5, 0.3], [0.8, 0.1, 0.1], [0.7, 0.2, 0.1]])\n        &gt;&gt;&gt; loss = FocalLoss()(input, target)\n        \"\"\"\n        input_prob = torch.sigmoid(input)\n        if input.size(1) == 1:\n            input_prob = torch.cat([1 - input_prob, input_prob], axis=1)  # type: ignore\n            num_class = 2\n        else:\n            num_class = input_prob.size(1)\n        binary_target = torch.eye(num_class)[target.squeeze().cpu().long()]\n        binary_target = binary_target.to(input.device)\n        binary_target = binary_target.contiguous()\n        weight = self._get_weight(input_prob, binary_target)\n\n        return F.binary_cross_entropy(\n            input_prob, binary_target, weight, reduction=\"mean\"\n        )\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.FocalLoss.forward","title":"forward","text":"<pre><code>forward(input, target)\n</code></pre> <p>Parameters:</p> Name Type Description Default <code>input</code> <code>Tensor</code> <p>Input tensor with predictions (not probabilities)</p> required <code>target</code> <code>Tensor</code> <p>Target tensor with the actual classes</p> required <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt;\n&gt;&gt;&gt; from pytorch_widedeep.losses import FocalLoss\n&gt;&gt;&gt;\n&gt;&gt;&gt; # BINARY\n&gt;&gt;&gt; target = torch.tensor([0, 1, 0, 1]).view(-1, 1)\n&gt;&gt;&gt; input = torch.tensor([[0.6, 0.7, 0.3, 0.8]]).t()\n&gt;&gt;&gt; loss = FocalLoss()(input, target)\n&gt;&gt;&gt;\n&gt;&gt;&gt; # MULTICLASS\n&gt;&gt;&gt; target = torch.tensor([1, 0, 2]).view(-1, 1)\n&gt;&gt;&gt; input = torch.tensor([[0.2, 0.5, 0.3], [0.8, 0.1, 0.1], [0.7, 0.2, 0.1]])\n&gt;&gt;&gt; loss = FocalLoss()(input, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(self, input: Tensor, target: Tensor) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    input: Tensor\n        Input tensor with predictions (not probabilities)\n    target: Tensor\n        Target tensor with the actual classes\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; from pytorch_widedeep.losses import FocalLoss\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; # BINARY\n    &gt;&gt;&gt; target = torch.tensor([0, 1, 0, 1]).view(-1, 1)\n    &gt;&gt;&gt; input = torch.tensor([[0.6, 0.7, 0.3, 0.8]]).t()\n    &gt;&gt;&gt; loss = FocalLoss()(input, target)\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; # MULTICLASS\n    &gt;&gt;&gt; target = torch.tensor([1, 0, 2]).view(-1, 1)\n    &gt;&gt;&gt; input = torch.tensor([[0.2, 0.5, 0.3], [0.8, 0.1, 0.1], [0.7, 0.2, 0.1]])\n    &gt;&gt;&gt; loss = FocalLoss()(input, target)\n    \"\"\"\n    input_prob = torch.sigmoid(input)\n    if input.size(1) == 1:\n        input_prob = torch.cat([1 - input_prob, input_prob], axis=1)  # type: ignore\n        num_class = 2\n    else:\n        num_class = input_prob.size(1)\n    binary_target = torch.eye(num_class)[target.squeeze().cpu().long()]\n    binary_target = binary_target.to(input.device)\n    binary_target = binary_target.contiguous()\n    weight = self._get_weight(input_prob, binary_target)\n\n    return F.binary_cross_entropy(\n        input_prob, binary_target, weight, reduction=\"mean\"\n    )\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.BayesianSELoss","title":"BayesianSELoss","text":"<p>               Bases: <code>Module</code></p> <p>Squared Loss (log Gaussian) for the case of a regression as specified in the original publication Weight Uncertainty in Neural Networks.</p> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>class BayesianSELoss(nn.Module):\n    r\"\"\"Squared Loss (log Gaussian) for the case of a regression as specified in\n    the original publication\n    [Weight Uncertainty in Neural Networks](https://arxiv.org/abs/1505.05424).\n    \"\"\"\n\n    def __init__(self):\n        super().__init__()\n\n    def forward(self, input: Tensor, target: Tensor) -&gt; Tensor:\n        r\"\"\"\n        Parameters\n        ----------\n        input: Tensor\n            Input tensor with predictions (not probabilities)\n        target: Tensor\n            Target tensor with the actual classes\n\n        Examples\n        --------\n        &gt;&gt;&gt; import torch\n        &gt;&gt;&gt; from pytorch_widedeep.losses import BayesianSELoss\n        &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n        &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n        &gt;&gt;&gt; loss = BayesianSELoss()(input, target)\n        \"\"\"\n        return (0.5 * (input - target) ** 2).sum()\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.BayesianSELoss.forward","title":"forward","text":"<pre><code>forward(input, target)\n</code></pre> <p>Parameters:</p> Name Type Description Default <code>input</code> <code>Tensor</code> <p>Input tensor with predictions (not probabilities)</p> required <code>target</code> <code>Tensor</code> <p>Target tensor with the actual classes</p> required <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.losses import BayesianSELoss\n&gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n&gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n&gt;&gt;&gt; loss = BayesianSELoss()(input, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(self, input: Tensor, target: Tensor) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    input: Tensor\n        Input tensor with predictions (not probabilities)\n    target: Tensor\n        Target tensor with the actual classes\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.losses import BayesianSELoss\n    &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n    &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n    &gt;&gt;&gt; loss = BayesianSELoss()(input, target)\n    \"\"\"\n    return (0.5 * (input - target) ** 2).sum()\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.TweedieLoss","title":"TweedieLoss","text":"<p>               Bases: <code>Module</code></p> <p>Tweedie loss for extremely unbalanced zero-inflated data</p> <p>All credits go to Wenbo Shi. See this post and the original publication for details.</p> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>class TweedieLoss(nn.Module):\n    \"\"\"\n    Tweedie loss for extremely unbalanced zero-inflated data\n\n    All credits go to Wenbo Shi. See\n    [this post](https://towardsdatascience.com/tweedie-loss-function-for-right-skewed-data-2c5ca470678f)\n    and the [original publication](https://arxiv.org/abs/1811.10192) for details.\n    \"\"\"\n\n    def __init__(self):\n        super().__init__()\n\n    def forward(\n        self,\n        input: Tensor,\n        target: Tensor,\n        p: float = 1.5,\n    ) -&gt; Tensor:\n        r\"\"\"\n        Parameters\n        ----------\n        input: Tensor\n            Input tensor with predictions\n        target: Tensor\n            Target tensor with the actual values\n        p: float, default = 1.5\n            the power to be used to compute the loss. See the original\n            publication for details\n\n        Examples\n        --------\n        &gt;&gt;&gt; import torch\n        &gt;&gt;&gt; from pytorch_widedeep.losses import TweedieLoss\n        &gt;&gt;&gt;\n        &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n        &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n        &gt;&gt;&gt; loss = TweedieLoss()(input, target)\n        \"\"\"\n\n        assert (\n            input.min() &gt; 0\n        ), \"\"\"All input values must be &gt;=0, if your model is predicting\n            values &lt;0 try to enforce positive values by activation function\n            on last layer with `trainer.enforce_positive_output=True`\"\"\"\n        assert target.min() &gt;= 0, \"All target values must be &gt;=0\"\n        loss = -target * torch.pow(input, 1 - p) / (1 - p) + torch.pow(input, 2 - p) / (\n            2 - p\n        )\n        return torch.mean(loss)\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.TweedieLoss.forward","title":"forward","text":"<pre><code>forward(input, target, p=1.5)\n</code></pre> <p>Parameters:</p> Name Type Description Default <code>input</code> <code>Tensor</code> <p>Input tensor with predictions</p> required <code>target</code> <code>Tensor</code> <p>Target tensor with the actual values</p> required <code>p</code> <code>float</code> <p>the power to be used to compute the loss. See the original publication for details</p> <code>1.5</code> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.losses import TweedieLoss\n&gt;&gt;&gt;\n&gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n&gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n&gt;&gt;&gt; loss = TweedieLoss()(input, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(\n    self,\n    input: Tensor,\n    target: Tensor,\n    p: float = 1.5,\n) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    input: Tensor\n        Input tensor with predictions\n    target: Tensor\n        Target tensor with the actual values\n    p: float, default = 1.5\n        the power to be used to compute the loss. See the original\n        publication for details\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.losses import TweedieLoss\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n    &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n    &gt;&gt;&gt; loss = TweedieLoss()(input, target)\n    \"\"\"\n\n    assert (\n        input.min() &gt; 0\n    ), \"\"\"All input values must be &gt;=0, if your model is predicting\n        values &lt;0 try to enforce positive values by activation function\n        on last layer with `trainer.enforce_positive_output=True`\"\"\"\n    assert target.min() &gt;= 0, \"All target values must be &gt;=0\"\n    loss = -target * torch.pow(input, 1 - p) / (1 - p) + torch.pow(input, 2 - p) / (\n        2 - p\n    )\n    return torch.mean(loss)\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.ZILNLoss","title":"ZILNLoss","text":"<p>               Bases: <code>Module</code></p> <p>Adjusted implementation of the Zero Inflated LogNormal Loss</p> <p>See A Deep Probabilistic Model for Customer Lifetime Value Prediction and the corresponding code.</p> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>class ZILNLoss(nn.Module):\n    r\"\"\"Adjusted implementation of the Zero Inflated LogNormal Loss\n\n    See [A Deep Probabilistic Model for Customer Lifetime Value Prediction](https://arxiv.org/pdf/1912.07753.pdf)\n    and the corresponding\n    [code](https://github.com/google/lifetime_value/blob/master/lifetime_value/zero_inflated_lognormal.py).\n    \"\"\"\n\n    def __init__(self):\n        super().__init__()\n\n    def forward(self, input: Tensor, target: Tensor) -&gt; Tensor:\n        r\"\"\"\n        Parameters\n        ----------\n        input: Tensor\n            Input tensor with predictions with spape (N,3), where N is the batch size\n        target: Tensor\n            Target tensor with the actual target values\n\n        Examples\n        --------\n        &gt;&gt;&gt; import torch\n        &gt;&gt;&gt; from pytorch_widedeep.losses import ZILNLoss\n        &gt;&gt;&gt;\n        &gt;&gt;&gt; target = torch.tensor([[0., 1.5]]).view(-1, 1)\n        &gt;&gt;&gt; input = torch.tensor([[.1, .2, .3], [.4, .5, .6]])\n        &gt;&gt;&gt; loss = ZILNLoss()(input, target)\n        \"\"\"\n        positive = target &gt; 0\n        positive = positive.float()\n\n        assert input.shape == torch.Size([target.shape[0], 3]), (\n            \"Wrong shape of the 'input' tensor. The pred_dim of the \"\n            \"model that is using ZILNLoss must be equal to 3.\"\n        )\n\n        positive_input = input[..., :1]\n\n        classification_loss = F.binary_cross_entropy_with_logits(\n            positive_input, positive, reduction=\"none\"\n        ).flatten()\n\n        loc = input[..., 1:2]\n\n        # when using max the two input tensors (input and other) have to be of\n        # the same type\n        max_input = F.softplus(input[..., 2:])\n        max_other = self.get_eps(max_input)\n        scale = torch.max(max_input, max_other)\n        safe_labels = positive * target + (1 - positive) * torch.ones_like(target)\n\n        regression_loss = -torch.mean(\n            positive\n            * torch.distributions.log_normal.LogNormal(loc=loc, scale=scale).log_prob(\n                safe_labels\n            ),\n            dim=-1,\n        )\n\n        return torch.mean(classification_loss + regression_loss)\n\n    @staticmethod\n    def get_eps(max_input: Tensor) -&gt; Tensor:\n        if max_input.device.type == \"mps\":\n            # For MPS, use float32 and then convert to the input type\n            eps = torch.finfo(torch.float32).eps\n            max_other = (\n                torch.sqrt(torch.tensor([eps], device=\"cpu\"))\n                .to(max_input.device)\n                .to(max_input.dtype)\n            )\n        else:\n            # For other devices, use the original approach\n            eps = torch.finfo(torch.double).eps\n            max_other = (\n                torch.sqrt(torch.tensor([eps])).to(max_input.device).to(max_input.dtype)\n            )\n\n        return max_other\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.ZILNLoss.forward","title":"forward","text":"<pre><code>forward(input, target)\n</code></pre> <p>Parameters:</p> Name Type Description Default <code>input</code> <code>Tensor</code> <p>Input tensor with predictions with spape (N,3), where N is the batch size</p> required <code>target</code> <code>Tensor</code> <p>Target tensor with the actual target values</p> required <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.losses import ZILNLoss\n&gt;&gt;&gt;\n&gt;&gt;&gt; target = torch.tensor([[0., 1.5]]).view(-1, 1)\n&gt;&gt;&gt; input = torch.tensor([[.1, .2, .3], [.4, .5, .6]])\n&gt;&gt;&gt; loss = ZILNLoss()(input, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(self, input: Tensor, target: Tensor) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    input: Tensor\n        Input tensor with predictions with spape (N,3), where N is the batch size\n    target: Tensor\n        Target tensor with the actual target values\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.losses import ZILNLoss\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; target = torch.tensor([[0., 1.5]]).view(-1, 1)\n    &gt;&gt;&gt; input = torch.tensor([[.1, .2, .3], [.4, .5, .6]])\n    &gt;&gt;&gt; loss = ZILNLoss()(input, target)\n    \"\"\"\n    positive = target &gt; 0\n    positive = positive.float()\n\n    assert input.shape == torch.Size([target.shape[0], 3]), (\n        \"Wrong shape of the 'input' tensor. The pred_dim of the \"\n        \"model that is using ZILNLoss must be equal to 3.\"\n    )\n\n    positive_input = input[..., :1]\n\n    classification_loss = F.binary_cross_entropy_with_logits(\n        positive_input, positive, reduction=\"none\"\n    ).flatten()\n\n    loc = input[..., 1:2]\n\n    # when using max the two input tensors (input and other) have to be of\n    # the same type\n    max_input = F.softplus(input[..., 2:])\n    max_other = self.get_eps(max_input)\n    scale = torch.max(max_input, max_other)\n    safe_labels = positive * target + (1 - positive) * torch.ones_like(target)\n\n    regression_loss = -torch.mean(\n        positive\n        * torch.distributions.log_normal.LogNormal(loc=loc, scale=scale).log_prob(\n            safe_labels\n        ),\n        dim=-1,\n    )\n\n    return torch.mean(classification_loss + regression_loss)\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.L1Loss","title":"L1Loss","text":"<p>               Bases: <code>Module</code></p> <p>L1 loss</p> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>class L1Loss(nn.Module):\n    r\"\"\"L1 loss\"\"\"\n\n    # legacy code from when we used to support FDS-LDS and this class could\n    # taked the corresponding params. At this stage probably you want to use\n    # torch.nn.L1Loss\n    def __init__(self):\n        super().__init__()\n\n    def forward(self, input: Tensor, target: Tensor) -&gt; Tensor:\n        r\"\"\"\n        Parameters\n        ----------\n        input: Tensor\n            Input tensor with predictions\n        target: Tensor\n            Target tensor with the actual values\n\n        Examples\n        --------\n        &gt;&gt;&gt; import torch\n        &gt;&gt;&gt;\n        &gt;&gt;&gt; from pytorch_widedeep.losses import L1Loss\n        &gt;&gt;&gt;\n        &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n        &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n        &gt;&gt;&gt; loss = L1Loss()(input, target)\n        \"\"\"\n        loss = F.l1_loss(input, target, reduction=\"none\")\n        return torch.mean(loss)\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.L1Loss.forward","title":"forward","text":"<pre><code>forward(input, target)\n</code></pre> <p>Parameters:</p> Name Type Description Default <code>input</code> <code>Tensor</code> <p>Input tensor with predictions</p> required <code>target</code> <code>Tensor</code> <p>Target tensor with the actual values</p> required <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt;\n&gt;&gt;&gt; from pytorch_widedeep.losses import L1Loss\n&gt;&gt;&gt;\n&gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n&gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n&gt;&gt;&gt; loss = L1Loss()(input, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(self, input: Tensor, target: Tensor) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    input: Tensor\n        Input tensor with predictions\n    target: Tensor\n        Target tensor with the actual values\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; from pytorch_widedeep.losses import L1Loss\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n    &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n    &gt;&gt;&gt; loss = L1Loss()(input, target)\n    \"\"\"\n    loss = F.l1_loss(input, target, reduction=\"none\")\n    return torch.mean(loss)\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.FocalR_L1Loss","title":"FocalR_L1Loss","text":"<p>               Bases: <code>Module</code></p> <p>Focal-R L1 loss</p> <p>Based on Delving into Deep Imbalanced Regression.</p> <p>Parameters:</p> Name Type Description Default <code>beta</code> <code>float</code> <p>Focal Loss <code>beta</code> parameter in their implementation</p> <code>0.2</code> <code>gamma</code> <code>float</code> <p>Focal Loss <code>gamma</code> parameter</p> <code>1.0</code> <code>activation_fn</code> <code>Literal[sigmoid, tanh]</code> <p>Activation function to be used during the computation of the loss. Possible values are 'sigmoid' and 'tanh'. See the original publication for details.</p> <code>'sigmoid'</code> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>class FocalR_L1Loss(nn.Module):\n    r\"\"\"Focal-R L1 loss\n\n    Based on [Delving into Deep Imbalanced Regression](https://arxiv.org/abs/2102.09554).\n\n    Parameters\n    ----------\n    beta: float\n        Focal Loss `beta` parameter in their implementation\n    gamma: float\n        Focal Loss `gamma` parameter\n    activation_fn: str, default = \"sigmoid\"\n        Activation function to be used during the computation of the loss.\n        Possible values are _'sigmoid'_ and _'tanh'_. See the original\n        publication for details.\n    \"\"\"\n\n    def __init__(\n        self,\n        beta: float = 0.2,\n        gamma: float = 1.0,\n        activation_fn: Literal[\"sigmoid\", \"tanh\"] = \"sigmoid\",\n    ):\n        super().__init__()\n        self.beta = beta\n        self.gamma = gamma\n        self.activation_fn = activation_fn\n\n    def forward(\n        self,\n        input: Tensor,\n        target: Tensor,\n    ) -&gt; Tensor:\n        r\"\"\"\n        Parameters\n        ----------\n        input: Tensor\n            Input tensor with predictions (not probabilities)\n        target: Tensor\n            Target tensor with the actual classes\n\n        Examples\n        --------\n        &gt;&gt;&gt; import torch\n        &gt;&gt;&gt;\n        &gt;&gt;&gt; from pytorch_widedeep.losses import FocalR_L1Loss\n        &gt;&gt;&gt;\n        &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n        &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n        &gt;&gt;&gt; loss = FocalR_L1Loss()(input, target)\n        \"\"\"\n        loss = F.l1_loss(input, target, reduction=\"none\")\n        if self.activation_fn == \"tanh\":\n            loss *= (torch.tanh(self.beta * torch.abs(input - target))) ** self.gamma\n        elif self.activation_fn == \"sigmoid\":\n            loss *= (\n                2 * torch.sigmoid(self.beta * torch.abs(input - target)) - 1\n            ) ** self.gamma\n        else:\n            ValueError(\n                \"Incorrect activation function value - must be in ['sigmoid', 'tanh']\"\n            )\n        return torch.mean(loss)\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.FocalR_L1Loss.forward","title":"forward","text":"<pre><code>forward(input, target)\n</code></pre> <p>Parameters:</p> Name Type Description Default <code>input</code> <code>Tensor</code> <p>Input tensor with predictions (not probabilities)</p> required <code>target</code> <code>Tensor</code> <p>Target tensor with the actual classes</p> required <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt;\n&gt;&gt;&gt; from pytorch_widedeep.losses import FocalR_L1Loss\n&gt;&gt;&gt;\n&gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n&gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n&gt;&gt;&gt; loss = FocalR_L1Loss()(input, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(\n    self,\n    input: Tensor,\n    target: Tensor,\n) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    input: Tensor\n        Input tensor with predictions (not probabilities)\n    target: Tensor\n        Target tensor with the actual classes\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; from pytorch_widedeep.losses import FocalR_L1Loss\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n    &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n    &gt;&gt;&gt; loss = FocalR_L1Loss()(input, target)\n    \"\"\"\n    loss = F.l1_loss(input, target, reduction=\"none\")\n    if self.activation_fn == \"tanh\":\n        loss *= (torch.tanh(self.beta * torch.abs(input - target))) ** self.gamma\n    elif self.activation_fn == \"sigmoid\":\n        loss *= (\n            2 * torch.sigmoid(self.beta * torch.abs(input - target)) - 1\n        ) ** self.gamma\n    else:\n        ValueError(\n            \"Incorrect activation function value - must be in ['sigmoid', 'tanh']\"\n        )\n    return torch.mean(loss)\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.FocalR_MSELoss","title":"FocalR_MSELoss","text":"<p>               Bases: <code>Module</code></p> <p>Focal-R MSE loss</p> <p>Based on Delving into Deep Imbalanced Regression.</p> <p>Parameters:</p> Name Type Description Default <code>beta</code> <code>float</code> <p>Focal Loss <code>beta</code> parameter in their implementation</p> <code>0.2</code> <code>gamma</code> <code>float</code> <p>Focal Loss <code>gamma</code> parameter</p> <code>1.0</code> <code>activation_fn</code> <code>Literal[sigmoid, tanh]</code> <p>Activation function to be used during the computation of the loss. Possible values are 'sigmoid' and 'tanh'. See the original publication for details.</p> <code>'sigmoid'</code> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>class FocalR_MSELoss(nn.Module):\n    r\"\"\"Focal-R MSE loss\n\n    Based on [Delving into Deep Imbalanced Regression](https://arxiv.org/abs/2102.09554).\n\n    Parameters\n    ----------\n    beta: float\n        Focal Loss `beta` parameter in their implementation\n    gamma: float\n        Focal Loss `gamma` parameter\n    activation_fn: str, default = \"sigmoid\"\n        Activation function to be used during the computation of the loss.\n        Possible values are _'sigmoid'_ and _'tanh'_. See the original\n        publication for details.\n    \"\"\"\n\n    def __init__(\n        self,\n        beta: float = 0.2,\n        gamma: float = 1.0,\n        activation_fn: Literal[\"sigmoid\", \"tanh\"] = \"sigmoid\",\n    ):\n        super().__init__()\n        self.beta = beta\n        self.gamma = gamma\n        self.activation_fn = activation_fn\n\n    def forward(\n        self,\n        input: Tensor,\n        target: Tensor,\n    ) -&gt; Tensor:\n        r\"\"\"\n        Parameters\n        ----------\n        input: Tensor\n            Input tensor with predictions (not probabilities)\n        target: Tensor\n            Target tensor with the actual classes\n\n        Examples\n        --------\n        &gt;&gt;&gt; import torch\n        &gt;&gt;&gt;\n        &gt;&gt;&gt; from pytorch_widedeep.losses import FocalR_MSELoss\n        &gt;&gt;&gt;\n        &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n        &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n        &gt;&gt;&gt; loss = FocalR_MSELoss()(input, target)\n        \"\"\"\n        loss = (input - target) ** 2\n        if self.activation_fn == \"tanh\":\n            loss *= (torch.tanh(self.beta * torch.abs(input - target))) ** self.gamma\n        elif self.activation_fn == \"sigmoid\":\n            loss *= (\n                2 * torch.sigmoid(self.beta * torch.abs((input - target) ** 2)) - 1\n            ) ** self.gamma\n        else:\n            ValueError(\n                \"Incorrect activation function value - must be in ['sigmoid', 'tanh']\"\n            )\n        return torch.mean(loss)\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.FocalR_MSELoss.forward","title":"forward","text":"<pre><code>forward(input, target)\n</code></pre> <p>Parameters:</p> Name Type Description Default <code>input</code> <code>Tensor</code> <p>Input tensor with predictions (not probabilities)</p> required <code>target</code> <code>Tensor</code> <p>Target tensor with the actual classes</p> required <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt;\n&gt;&gt;&gt; from pytorch_widedeep.losses import FocalR_MSELoss\n&gt;&gt;&gt;\n&gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n&gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n&gt;&gt;&gt; loss = FocalR_MSELoss()(input, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(\n    self,\n    input: Tensor,\n    target: Tensor,\n) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    input: Tensor\n        Input tensor with predictions (not probabilities)\n    target: Tensor\n        Target tensor with the actual classes\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; from pytorch_widedeep.losses import FocalR_MSELoss\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n    &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n    &gt;&gt;&gt; loss = FocalR_MSELoss()(input, target)\n    \"\"\"\n    loss = (input - target) ** 2\n    if self.activation_fn == \"tanh\":\n        loss *= (torch.tanh(self.beta * torch.abs(input - target))) ** self.gamma\n    elif self.activation_fn == \"sigmoid\":\n        loss *= (\n            2 * torch.sigmoid(self.beta * torch.abs((input - target) ** 2)) - 1\n        ) ** self.gamma\n    else:\n        ValueError(\n            \"Incorrect activation function value - must be in ['sigmoid', 'tanh']\"\n        )\n    return torch.mean(loss)\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.FocalR_RMSELoss","title":"FocalR_RMSELoss","text":"<p>               Bases: <code>Module</code></p> <p>Focal-R RMSE loss</p> <p>Based on Delving into Deep Imbalanced Regression.</p> <p>Parameters:</p> Name Type Description Default <code>beta</code> <code>float</code> <p>Focal Loss <code>beta</code> parameter in their implementation</p> <code>0.2</code> <code>gamma</code> <code>float</code> <p>Focal Loss <code>gamma</code> parameter</p> <code>1.0</code> <code>activation_fn</code> <code>Literal[sigmoid, tanh]</code> <p>Activation function to be used during the computation of the loss. Possible values are 'sigmoid' and 'tanh'. See the original publication for details.</p> <code>'sigmoid'</code> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>class FocalR_RMSELoss(nn.Module):\n    r\"\"\"Focal-R RMSE loss\n\n    Based on [Delving into Deep Imbalanced Regression](https://arxiv.org/abs/2102.09554).\n\n    Parameters\n    ----------\n    beta: float\n        Focal Loss `beta` parameter in their implementation\n    gamma: float\n        Focal Loss `gamma` parameter\n    activation_fn: str, default = \"sigmoid\"\n        Activation function to be used during the computation of the loss.\n        Possible values are _'sigmoid'_ and _'tanh'_. See the original\n        publication for details.\n    \"\"\"\n\n    def __init__(\n        self,\n        beta: float = 0.2,\n        gamma: float = 1.0,\n        activation_fn: Literal[\"sigmoid\", \"tanh\"] = \"sigmoid\",\n    ):\n        super().__init__()\n        self.beta = beta\n        self.gamma = gamma\n        self.activation_fn = activation_fn\n\n    def forward(\n        self,\n        input: Tensor,\n        target: Tensor,\n    ) -&gt; Tensor:\n        r\"\"\"\n        Parameters\n        ----------\n        input: Tensor\n            Input tensor with predictions (not probabilities)\n        target: Tensor\n            Target tensor with the actual classes\n\n        Examples\n        --------\n        &gt;&gt;&gt; import torch\n        &gt;&gt;&gt;\n        &gt;&gt;&gt; from pytorch_widedeep.losses import FocalR_RMSELoss\n        &gt;&gt;&gt;\n        &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n        &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n        &gt;&gt;&gt; loss = FocalR_RMSELoss()(input, target)\n        \"\"\"\n        loss = (input - target) ** 2\n        if self.activation_fn == \"tanh\":\n            loss *= (torch.tanh(self.beta * torch.abs(input - target))) ** self.gamma\n        elif self.activation_fn == \"sigmoid\":\n            loss *= (\n                2 * torch.sigmoid(self.beta * torch.abs((input - target) ** 2)) - 1\n            ) ** self.gamma\n        else:\n            ValueError(\n                \"Incorrect activation function value - must be in ['sigmoid', 'tanh']\"\n            )\n        return torch.sqrt(torch.mean(loss))\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.FocalR_RMSELoss.forward","title":"forward","text":"<pre><code>forward(input, target)\n</code></pre> <p>Parameters:</p> Name Type Description Default <code>input</code> <code>Tensor</code> <p>Input tensor with predictions (not probabilities)</p> required <code>target</code> <code>Tensor</code> <p>Target tensor with the actual classes</p> required <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt;\n&gt;&gt;&gt; from pytorch_widedeep.losses import FocalR_RMSELoss\n&gt;&gt;&gt;\n&gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n&gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n&gt;&gt;&gt; loss = FocalR_RMSELoss()(input, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(\n    self,\n    input: Tensor,\n    target: Tensor,\n) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    input: Tensor\n        Input tensor with predictions (not probabilities)\n    target: Tensor\n        Target tensor with the actual classes\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; from pytorch_widedeep.losses import FocalR_RMSELoss\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n    &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n    &gt;&gt;&gt; loss = FocalR_RMSELoss()(input, target)\n    \"\"\"\n    loss = (input - target) ** 2\n    if self.activation_fn == \"tanh\":\n        loss *= (torch.tanh(self.beta * torch.abs(input - target))) ** self.gamma\n    elif self.activation_fn == \"sigmoid\":\n        loss *= (\n            2 * torch.sigmoid(self.beta * torch.abs((input - target) ** 2)) - 1\n        ) ** self.gamma\n    else:\n        ValueError(\n            \"Incorrect activation function value - must be in ['sigmoid', 'tanh']\"\n        )\n    return torch.sqrt(torch.mean(loss))\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.HuberLoss","title":"HuberLoss","text":"<p>               Bases: <code>Module</code></p> <p>Hubbler Loss</p> <p>Based on Delving into Deep Imbalanced Regression.</p> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>class HuberLoss(nn.Module):\n    r\"\"\"Hubbler Loss\n\n    Based on [Delving into Deep Imbalanced Regression](https://arxiv.org/abs/2102.09554).\n    \"\"\"\n\n    def __init__(self, beta: float = 0.2):\n        super().__init__()\n        self.beta = beta\n\n    def forward(\n        self,\n        input: Tensor,\n        target: Tensor,\n    ) -&gt; Tensor:\n        r\"\"\"\n        Parameters\n        ----------\n        input: Tensor\n            Input tensor with predictions (not probabilities)\n        target: Tensor\n            Target tensor with the actual classes\n\n        Examples\n        --------\n        &gt;&gt;&gt; import torch\n        &gt;&gt;&gt;\n        &gt;&gt;&gt; from pytorch_widedeep.losses import HuberLoss\n        &gt;&gt;&gt;\n        &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n        &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n        &gt;&gt;&gt; loss = HuberLoss()(input, target)\n        \"\"\"\n        l1_loss = torch.abs(input - target)\n        cond = l1_loss &lt; self.beta\n        loss = torch.where(\n            cond, 0.5 * l1_loss**2 / self.beta, l1_loss - 0.5 * self.beta\n        )\n        return torch.mean(loss)\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.HuberLoss.forward","title":"forward","text":"<pre><code>forward(input, target)\n</code></pre> <p>Parameters:</p> Name Type Description Default <code>input</code> <code>Tensor</code> <p>Input tensor with predictions (not probabilities)</p> required <code>target</code> <code>Tensor</code> <p>Target tensor with the actual classes</p> required <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt;\n&gt;&gt;&gt; from pytorch_widedeep.losses import HuberLoss\n&gt;&gt;&gt;\n&gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n&gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n&gt;&gt;&gt; loss = HuberLoss()(input, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(\n    self,\n    input: Tensor,\n    target: Tensor,\n) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    input: Tensor\n        Input tensor with predictions (not probabilities)\n    target: Tensor\n        Target tensor with the actual classes\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; from pytorch_widedeep.losses import HuberLoss\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n    &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n    &gt;&gt;&gt; loss = HuberLoss()(input, target)\n    \"\"\"\n    l1_loss = torch.abs(input - target)\n    cond = l1_loss &lt; self.beta\n    loss = torch.where(\n        cond, 0.5 * l1_loss**2 / self.beta, l1_loss - 0.5 * self.beta\n    )\n    return torch.mean(loss)\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.InfoNCELoss","title":"InfoNCELoss","text":"<p>               Bases: <code>Module</code></p> <p>InfoNCE Loss. Loss applied during the Contrastive Denoising Self Supervised Pre-training routine available in this library</p> <p> NOTE: This loss is in principle not exposed to  the user, as it is used internally in the library, but it is included  here for completion.</p> <p>See SAINT: Improved Neural Networks for Tabular Data via Row Attention and Contrastive Pre-Training and references therein</p> <p>Partially inspired by the code in this repo</p> <p>Parameters:</p> Name Type Description Default <code>temperature</code> <code>float</code> <p>The logits are divided by the temperature before computing the loss value</p> <code>0.1</code> <code>reduction</code> <code>str</code> <p>Loss reduction method</p> <code>'mean'</code> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>class InfoNCELoss(nn.Module):\n    r\"\"\"InfoNCE Loss. Loss applied during the Contrastive Denoising Self\n    Supervised Pre-training routine available in this library\n\n    :information_source: **NOTE**: This loss is in principle not exposed to\n     the user, as it is used internally in the library, but it is included\n     here for completion.\n\n    See [SAINT: Improved Neural Networks for Tabular Data via Row Attention\n    and Contrastive Pre-Training](https://arxiv.org/abs/2106.01342) and\n    references therein\n\n    Partially inspired by the code in this [repo](https://github.com/RElbers/info-nce-pytorch)\n\n    Parameters\n    ----------\n    temperature: float, default = 0.1\n        The logits are divided by the temperature before computing the loss value\n    reduction: str, default = \"mean\"\n        Loss reduction method\n    \"\"\"\n\n    def __init__(self, temperature: float = 0.1, reduction: str = \"mean\"):\n        super(InfoNCELoss, self).__init__()\n\n        self.temperature = temperature\n        self.reduction = reduction\n\n    def forward(self, g_projs: Tuple[Tensor, Tensor]) -&gt; Tensor:\n        r\"\"\"\n        Parameters\n        ----------\n        g_projs: Tuple\n            Tuple with the two tensors corresponding to the output of the two\n            projection heads, as described 'SAINT: Improved Neural Networks\n            for Tabular Data via Row Attention and Contrastive Pre-Training'.\n\n        Examples\n        --------\n        &gt;&gt;&gt; import torch\n        &gt;&gt;&gt; from pytorch_widedeep.losses import InfoNCELoss\n        &gt;&gt;&gt; g_projs = (torch.rand(3, 5, 16), torch.rand(3, 5, 16))\n        &gt;&gt;&gt; loss = InfoNCELoss()\n        &gt;&gt;&gt; res = loss(g_projs)\n        \"\"\"\n        z, z_ = g_projs[0], g_projs[1]\n\n        norm_z = F.normalize(z, dim=-1).flatten(1)\n        norm_z_ = F.normalize(z_, dim=-1).flatten(1)\n\n        logits = (norm_z @ norm_z_.t()) / self.temperature\n        logits_ = (norm_z_ @ norm_z.t()) / self.temperature\n\n        # the target/labels are the entries on the diagonal\n        target = torch.arange(len(norm_z), device=norm_z.device)\n\n        loss = F.cross_entropy(logits, target, reduction=self.reduction)\n        loss_ = F.cross_entropy(logits_, target, reduction=self.reduction)\n\n        return (loss + loss_) / 2.0\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.InfoNCELoss.forward","title":"forward","text":"<pre><code>forward(g_projs)\n</code></pre> <p>Parameters:</p> Name Type Description Default <code>g_projs</code> <code>Tuple[Tensor, Tensor]</code> <p>Tuple with the two tensors corresponding to the output of the two projection heads, as described 'SAINT: Improved Neural Networks for Tabular Data via Row Attention and Contrastive Pre-Training'.</p> required <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.losses import InfoNCELoss\n&gt;&gt;&gt; g_projs = (torch.rand(3, 5, 16), torch.rand(3, 5, 16))\n&gt;&gt;&gt; loss = InfoNCELoss()\n&gt;&gt;&gt; res = loss(g_projs)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(self, g_projs: Tuple[Tensor, Tensor]) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    g_projs: Tuple\n        Tuple with the two tensors corresponding to the output of the two\n        projection heads, as described 'SAINT: Improved Neural Networks\n        for Tabular Data via Row Attention and Contrastive Pre-Training'.\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.losses import InfoNCELoss\n    &gt;&gt;&gt; g_projs = (torch.rand(3, 5, 16), torch.rand(3, 5, 16))\n    &gt;&gt;&gt; loss = InfoNCELoss()\n    &gt;&gt;&gt; res = loss(g_projs)\n    \"\"\"\n    z, z_ = g_projs[0], g_projs[1]\n\n    norm_z = F.normalize(z, dim=-1).flatten(1)\n    norm_z_ = F.normalize(z_, dim=-1).flatten(1)\n\n    logits = (norm_z @ norm_z_.t()) / self.temperature\n    logits_ = (norm_z_ @ norm_z.t()) / self.temperature\n\n    # the target/labels are the entries on the diagonal\n    target = torch.arange(len(norm_z), device=norm_z.device)\n\n    loss = F.cross_entropy(logits, target, reduction=self.reduction)\n    loss_ = F.cross_entropy(logits_, target, reduction=self.reduction)\n\n    return (loss + loss_) / 2.0\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.DenoisingLoss","title":"DenoisingLoss","text":"<p>               Bases: <code>Module</code></p> <p>Denoising Loss. Loss applied during the Contrastive Denoising Self Supervised Pre-training routine available in this library</p> <p> NOTE: This loss is in principle not exposed to  the user, as it is used internally in the library, but it is included  here for completion.</p> <p>See SAINT: Improved Neural Networks for Tabular Data via Row Attention and Contrastive Pre-Training and references therein</p> <p>Parameters:</p> Name Type Description Default <code>lambda_cat</code> <code>float</code> <p>Multiplicative factor that will be applied to loss associated to the categorical features</p> <code>1.0</code> <code>lambda_cont</code> <code>float</code> <p>Multiplicative factor that will be applied to loss associated to the continuous features</p> <code>1.0</code> <code>reduction</code> <code>str</code> <p>Loss reduction method</p> <code>'mean'</code> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>class DenoisingLoss(nn.Module):\n    r\"\"\"Denoising Loss. Loss applied during the Contrastive Denoising Self\n    Supervised Pre-training routine available in this library\n\n    :information_source: **NOTE**: This loss is in principle not exposed to\n     the user, as it is used internally in the library, but it is included\n     here for completion.\n\n    See [SAINT: Improved Neural Networks for Tabular Data via Row Attention\n    and Contrastive Pre-Training](https://arxiv.org/abs/2106.01342) and\n    references therein\n\n    Parameters\n    ----------\n    lambda_cat: float, default = 1.\n        Multiplicative factor that will be applied to loss associated to the\n        categorical features\n    lambda_cont: float, default = 1.\n        Multiplicative factor that will be applied to loss associated to the\n        continuous features\n    reduction: str, default = \"mean\"\n        Loss reduction method\n    \"\"\"\n\n    def __init__(\n        self, lambda_cat: float = 1.0, lambda_cont: float = 1.0, reduction: str = \"mean\"\n    ):\n        super(DenoisingLoss, self).__init__()\n\n        self.lambda_cat = lambda_cat\n        self.lambda_cont = lambda_cont\n        self.reduction = reduction\n\n    def forward(\n        self,\n        x_cat_and_cat_: Optional[\n            Union[List[Tuple[Tensor, Tensor]], Tuple[Tensor, Tensor]]\n        ],\n        x_cont_and_cont_: Optional[\n            Union[List[Tuple[Tensor, Tensor]], Tuple[Tensor, Tensor]]\n        ],\n    ) -&gt; Tensor:\n        r\"\"\"\n        Parameters\n        ----------\n        x_cat_and_cat_: tuple of Tensors or lists of tuples\n            Tuple of tensors containing the raw input features and their\n            encodings, referred in the SAINT paper as $x$ and $x''$\n            respectively. If one denoising MLP is used per categorical\n            feature `x_cat_and_cat_` will be a list of tuples, one per\n            categorical feature\n        x_cont_and_cont_: tuple of Tensors or lists of tuples\n            same as `x_cat_and_cat_` but for continuous columns\n\n        Examples\n        --------\n        &gt;&gt;&gt; import torch\n        &gt;&gt;&gt; from pytorch_widedeep.losses import DenoisingLoss\n        &gt;&gt;&gt; x_cat_and_cat_ = (torch.empty(3).random_(3).long(), torch.randn(3, 3))\n        &gt;&gt;&gt; x_cont_and_cont_ = (torch.randn(3, 1), torch.randn(3, 1))\n        &gt;&gt;&gt; loss = DenoisingLoss()\n        &gt;&gt;&gt; res = loss(x_cat_and_cat_, x_cont_and_cont_)\n        \"\"\"\n\n        loss_cat = (\n            self._compute_cat_loss(x_cat_and_cat_)\n            if x_cat_and_cat_ is not None\n            else torch.tensor(0.0)\n        )\n        loss_cont = (\n            self._compute_cont_loss(x_cont_and_cont_)\n            if x_cont_and_cont_ is not None\n            else torch.tensor(0.0)\n        )\n\n        return self.lambda_cat * loss_cat + self.lambda_cont * loss_cont\n\n    def _compute_cat_loss(\n        self, x_cat_and_cat_: Union[List[Tuple[Tensor, Tensor]], Tuple[Tensor, Tensor]]\n    ) -&gt; Tensor:\n        loss_cat = torch.tensor(0.0, device=self._get_device(x_cat_and_cat_))\n        if isinstance(x_cat_and_cat_, list):\n            for x, x_ in x_cat_and_cat_:\n                loss_cat += F.cross_entropy(x_, x, reduction=self.reduction)\n        elif isinstance(x_cat_and_cat_, tuple):\n            x, x_ = x_cat_and_cat_\n            loss_cat += F.cross_entropy(x_, x, reduction=self.reduction)\n\n        return loss_cat\n\n    def _compute_cont_loss(self, x_cont_and_cont_) -&gt; Tensor:\n        loss_cont = torch.tensor(0.0, device=self._get_device(x_cont_and_cont_))\n        if isinstance(x_cont_and_cont_, list):\n            for x, x_ in x_cont_and_cont_:\n                loss_cont += F.mse_loss(x_, x, reduction=self.reduction)\n        elif isinstance(x_cont_and_cont_, tuple):\n            x, x_ = x_cont_and_cont_\n            loss_cont += F.mse_loss(x_, x, reduction=self.reduction)\n\n        return loss_cont\n\n    @staticmethod\n    def _get_device(\n        x_and_x_: Union[List[Tuple[Tensor, Tensor]], Tuple[Tensor, Tensor]]\n    ):\n        if isinstance(x_and_x_, tuple):\n            device = x_and_x_[0].device\n        elif isinstance(x_and_x_, list):\n            device = x_and_x_[0][0].device\n        return device\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.DenoisingLoss.forward","title":"forward","text":"<pre><code>forward(x_cat_and_cat_, x_cont_and_cont_)\n</code></pre> <p>Parameters:</p> Name Type Description Default <code>x_cat_and_cat_</code> <code>Optional[Union[List[Tuple[Tensor, Tensor]], Tuple[Tensor, Tensor]]]</code> <p>Tuple of tensors containing the raw input features and their encodings, referred in the SAINT paper as \\(x\\) and \\(x''\\) respectively. If one denoising MLP is used per categorical feature <code>x_cat_and_cat_</code> will be a list of tuples, one per categorical feature</p> required <code>x_cont_and_cont_</code> <code>Optional[Union[List[Tuple[Tensor, Tensor]], Tuple[Tensor, Tensor]]]</code> <p>same as <code>x_cat_and_cat_</code> but for continuous columns</p> required <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.losses import DenoisingLoss\n&gt;&gt;&gt; x_cat_and_cat_ = (torch.empty(3).random_(3).long(), torch.randn(3, 3))\n&gt;&gt;&gt; x_cont_and_cont_ = (torch.randn(3, 1), torch.randn(3, 1))\n&gt;&gt;&gt; loss = DenoisingLoss()\n&gt;&gt;&gt; res = loss(x_cat_and_cat_, x_cont_and_cont_)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(\n    self,\n    x_cat_and_cat_: Optional[\n        Union[List[Tuple[Tensor, Tensor]], Tuple[Tensor, Tensor]]\n    ],\n    x_cont_and_cont_: Optional[\n        Union[List[Tuple[Tensor, Tensor]], Tuple[Tensor, Tensor]]\n    ],\n) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    x_cat_and_cat_: tuple of Tensors or lists of tuples\n        Tuple of tensors containing the raw input features and their\n        encodings, referred in the SAINT paper as $x$ and $x''$\n        respectively. If one denoising MLP is used per categorical\n        feature `x_cat_and_cat_` will be a list of tuples, one per\n        categorical feature\n    x_cont_and_cont_: tuple of Tensors or lists of tuples\n        same as `x_cat_and_cat_` but for continuous columns\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.losses import DenoisingLoss\n    &gt;&gt;&gt; x_cat_and_cat_ = (torch.empty(3).random_(3).long(), torch.randn(3, 3))\n    &gt;&gt;&gt; x_cont_and_cont_ = (torch.randn(3, 1), torch.randn(3, 1))\n    &gt;&gt;&gt; loss = DenoisingLoss()\n    &gt;&gt;&gt; res = loss(x_cat_and_cat_, x_cont_and_cont_)\n    \"\"\"\n\n    loss_cat = (\n        self._compute_cat_loss(x_cat_and_cat_)\n        if x_cat_and_cat_ is not None\n        else torch.tensor(0.0)\n    )\n    loss_cont = (\n        self._compute_cont_loss(x_cont_and_cont_)\n        if x_cont_and_cont_ is not None\n        else torch.tensor(0.0)\n    )\n\n    return self.lambda_cat * loss_cat + self.lambda_cont * loss_cont\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.EncoderDecoderLoss","title":"EncoderDecoderLoss","text":"<p>               Bases: <code>Module</code></p> <p>'Standard' Encoder Decoder Loss. Loss applied during the Endoder-Decoder  Self-Supervised Pre-Training routine available in this library</p> <p> NOTE: This loss is in principle not exposed to  the user, as it is used internally in the library, but it is included  here for completion.</p> <p>The implementation of this lost is based on that at the tabnet repo, which is in itself an adaptation of that in the original paper TabNet: Attentive Interpretable Tabular Learning.</p> <p>Parameters:</p> Name Type Description Default <code>eps</code> <code>float</code> <p>Simply a small number to avoid dividing by zero</p> <code>1e-09</code> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>class EncoderDecoderLoss(nn.Module):\n    r\"\"\"'_Standard_' Encoder Decoder Loss. Loss applied during the Endoder-Decoder\n     Self-Supervised Pre-Training routine available in this library\n\n    :information_source: **NOTE**: This loss is in principle not exposed to\n     the user, as it is used internally in the library, but it is included\n     here for completion.\n\n    The implementation of this lost is based on that at the\n    [tabnet repo](https://github.com/dreamquark-ai/tabnet), which is in itself an\n    adaptation of that in the original paper [TabNet: Attentive\n    Interpretable Tabular Learning](https://arxiv.org/abs/1908.07442).\n\n    Parameters\n    ----------\n    eps: float\n        Simply a small number to avoid dividing by zero\n    \"\"\"\n\n    def __init__(self, eps: float = 1e-9):\n        super(EncoderDecoderLoss, self).__init__()\n        self.eps = eps\n\n    def forward(self, x_true: Tensor, x_pred: Tensor, mask: Tensor) -&gt; Tensor:\n        r\"\"\"\n        Parameters\n        ----------\n        x_true: Tensor\n            Embeddings of the input data\n        x_pred: Tensor\n            Reconstructed embeddings\n        mask: Tensor\n            Mask with 1s indicated that the reconstruction, and therefore the\n            loss, is based on those features.\n\n        Examples\n        --------\n        &gt;&gt;&gt; import torch\n        &gt;&gt;&gt; from pytorch_widedeep.losses import EncoderDecoderLoss\n        &gt;&gt;&gt; x_true = torch.rand(3, 3)\n        &gt;&gt;&gt; x_pred = torch.rand(3, 3)\n        &gt;&gt;&gt; mask = torch.empty(3, 3).random_(2)\n        &gt;&gt;&gt; loss = EncoderDecoderLoss()\n        &gt;&gt;&gt; res = loss(x_true, x_pred, mask)\n        \"\"\"\n\n        errors = x_pred - x_true\n\n        reconstruction_errors = torch.mul(errors, mask) ** 2\n\n        x_true_means = torch.mean(x_true, dim=0)\n        x_true_means[x_true_means == 0] = 1\n\n        x_true_stds = torch.std(x_true, dim=0) ** 2\n        x_true_stds[x_true_stds == 0] = x_true_means[x_true_stds == 0]\n\n        features_loss = torch.matmul(reconstruction_errors, 1 / x_true_stds)\n        nb_reconstructed_variables = torch.sum(mask, dim=1)\n        features_loss_norm = features_loss / (nb_reconstructed_variables + self.eps)\n\n        loss = torch.mean(features_loss_norm)\n\n        return loss\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.EncoderDecoderLoss.forward","title":"forward","text":"<pre><code>forward(x_true, x_pred, mask)\n</code></pre> <p>Parameters:</p> Name Type Description Default <code>x_true</code> <code>Tensor</code> <p>Embeddings of the input data</p> required <code>x_pred</code> <code>Tensor</code> <p>Reconstructed embeddings</p> required <code>mask</code> <code>Tensor</code> <p>Mask with 1s indicated that the reconstruction, and therefore the loss, is based on those features.</p> required <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.losses import EncoderDecoderLoss\n&gt;&gt;&gt; x_true = torch.rand(3, 3)\n&gt;&gt;&gt; x_pred = torch.rand(3, 3)\n&gt;&gt;&gt; mask = torch.empty(3, 3).random_(2)\n&gt;&gt;&gt; loss = EncoderDecoderLoss()\n&gt;&gt;&gt; res = loss(x_true, x_pred, mask)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(self, x_true: Tensor, x_pred: Tensor, mask: Tensor) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    x_true: Tensor\n        Embeddings of the input data\n    x_pred: Tensor\n        Reconstructed embeddings\n    mask: Tensor\n        Mask with 1s indicated that the reconstruction, and therefore the\n        loss, is based on those features.\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.losses import EncoderDecoderLoss\n    &gt;&gt;&gt; x_true = torch.rand(3, 3)\n    &gt;&gt;&gt; x_pred = torch.rand(3, 3)\n    &gt;&gt;&gt; mask = torch.empty(3, 3).random_(2)\n    &gt;&gt;&gt; loss = EncoderDecoderLoss()\n    &gt;&gt;&gt; res = loss(x_true, x_pred, mask)\n    \"\"\"\n\n    errors = x_pred - x_true\n\n    reconstruction_errors = torch.mul(errors, mask) ** 2\n\n    x_true_means = torch.mean(x_true, dim=0)\n    x_true_means[x_true_means == 0] = 1\n\n    x_true_stds = torch.std(x_true, dim=0) ** 2\n    x_true_stds[x_true_stds == 0] = x_true_means[x_true_stds == 0]\n\n    features_loss = torch.matmul(reconstruction_errors, 1 / x_true_stds)\n    nb_reconstructed_variables = torch.sum(mask, dim=1)\n    features_loss_norm = features_loss / (nb_reconstructed_variables + self.eps)\n\n    loss = torch.mean(features_loss_norm)\n\n    return loss\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses_multitarget.MultiTargetRegressionLoss","title":"MultiTargetRegressionLoss","text":"<p>               Bases: <code>Module</code></p> <p>This class is a wrapper around the Pytorch MSELoss. It allows for multi-target regression problems. The user can provide a list of weights to apply to each target. The loss can be either the sum or the mean of the individual losses</p> <p>Parameters:</p> Name Type Description Default <code>weights</code> <code>Optional[List[float]]</code> <p>List of weights to apply to the loss associated to each target. The length of the list must match the number of targets. Alias: 'target_weights'</p> <code>None</code> <code>reduction</code> <code>Literal[mean, sum]</code> <p>Specifies the reduction to apply to the loss associated to each target: 'mean' | 'sum'. Note that this is NOT the same as the reduction in the MSELoss. This reduction is applied after the loss for each target has been computed. Alias: 'target_reduction'</p> <code>'mean'</code> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.losses_multitarget import MultiTargetRegressionLoss\n&gt;&gt;&gt; input = torch.randn(3, 2)\n&gt;&gt;&gt; target = torch.randn(3, 2)\n&gt;&gt;&gt; loss = MultiTargetRegressionLoss(weights=[0.5, 0.5], reduction=\"mean\")\n&gt;&gt;&gt; output = loss(input, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses_multitarget.py</code> <pre><code>class MultiTargetRegressionLoss(nn.Module):\n    \"\"\"\n    This class is a wrapper around the Pytorch MSELoss. It allows for multi-target\n    regression problems. The user can provide a list of weights to apply to each\n    target. The loss can be either the sum or the mean of the individual losses\n\n    Parameters\n    ----------\n    weights: Optional[List[float], default = None]\n        List of weights to apply to the loss associated to each target. The\n        length of the list must match the number of targets.\n        Alias: 'target_weights'\n    reduction: Literal[\"mean\", \"sum\"], default = \"mean\n        Specifies the reduction to apply to the loss associated to each\n        target: 'mean' | 'sum'. Note that this is NOT the same as the\n        reduction in the MSELoss. This reduction is applied after the loss\n        for each target has been computed. Alias: 'target_reduction'\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.losses_multitarget import MultiTargetRegressionLoss\n    &gt;&gt;&gt; input = torch.randn(3, 2)\n    &gt;&gt;&gt; target = torch.randn(3, 2)\n    &gt;&gt;&gt; loss = MultiTargetRegressionLoss(weights=[0.5, 0.5], reduction=\"mean\")\n    &gt;&gt;&gt; output = loss(input, target)\n    \"\"\"\n\n    @alias(\"reduction\", [\"target_reduction\"])\n    @alias(\"weights\", [\"target_weights\"])\n    def __init__(\n        self,\n        weights: Optional[List[float]] = None,\n        reduction: Literal[\"mean\", \"sum\"] = \"mean\",\n    ):\n        super(MultiTargetRegressionLoss, self).__init__()\n\n        self.weights = weights\n        self.reduction = reduction\n\n        if self.reduction not in [\"mean\", \"sum\"]:\n            raise ValueError(\"reduction must be either 'mean' or 'sum'\")\n\n    def forward(self, input: Tensor, target: Tensor) -&gt; Tensor:\n\n        assert input.size() == target.size()\n\n        if self.weights is not None:\n\n            assert len(self.weights) == input.size(1), (\n                \"The number of weights must match the number of targets. \"\n                f\"Got {len(self.weights)} weights and {input.size(1)} targets\"\n            )\n\n            loss = F.mse_loss(input, target, reduction=\"none\") * torch.tensor(\n                self.weights\n            ).to(input.device)\n        else:\n            loss = F.mse_loss(input, target, reduction=\"none\")\n\n        return loss.mean() if self.reduction == \"mean\" else loss.sum()\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses_multitarget.MultiTargetClassificationLoss","title":"MultiTargetClassificationLoss","text":"<p>               Bases: <code>Module</code></p> <p>This class is a wrapper around the Pytorch binary_cross_entropy_with_logits and cross_entropy losses. It allows for multi-target classification problems. The user can provide a list of weights to apply to each target. The loss can be either the sum or the mean of the individual losses</p> <p>Parameters:</p> Name Type Description Default <code>binary_config</code> <code>Optional[List[Union[int, Tuple[int, float]]]]</code> <p>List of integers with the index of the target for binary classification or tuples with two elements: the index of the targets or binary classification and the positive weight for binary classification</p> <code>None</code> <code>multiclass_config</code> <code>Optional[List[Union[Tuple[int, int], Tuple[int, int, List[float]]]]]</code> <p>List of tuples with two or three elements: the index of the target and the number of classes for multiclass classification, or a tuple with the index of the target, the number of classes and a list of weights to apply to each class (i.e. the 'weight' parameter in the cross_entropy loss)</p> <code>None</code> <code>weights</code> <code>Optional[List[float]]</code> <p>List of weights to apply to the loss associated to each target. The length of the list must match the number of targets. Alias: 'target_weights'</p> <code>None</code> <code>reduction</code> <code>Literal[mean, sum]</code> <p>Specifies the reduction to apply to the loss associated to each target: 'mean' | 'sum'. Note that this is NOT the same as the reduction in the cross_entropy loss or the binary_cross_entropy_with_logits. This reduction is applied after the loss for each target has been computed. Alias: 'target_reduction'</p> <code>'mean'</code> <code>binary_trick</code> <code>bool</code> <p>If True, each target will be considered independently and the loss will be computed as binary_cross_entropy_with_logits. This is a faster implementation. Note that the 'weights' parameter is not compatible with binary_trick=True. Also note that if binary_trick=True, the 'binary_config' must be a list of integers and the 'multiclass_config' must be a list of tuples with two integers: the index of the target and the number of classes. Finally, if binary_trick=True, the binary targets must be the first targets in the target tensor.</p> <p> NOTE: When using the binary_trick, the binary targets are   considered as 2 classes. Therefore, the pred_dim parametere of the   WideDeep class should be adjusted accordingly (adding 2 to per   binary target). For example, in a problem with a binary target and   a 4 class multiclassification target, the pred_dim should be 6.</p> <code>False</code> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.losses_multitarget import MultiTargetClassificationLoss\n&gt;&gt;&gt; input = torch.randn(5, 4)\n&gt;&gt;&gt; input_binary_trick = torch.randn(5, 5)\n&gt;&gt;&gt; target = torch.stack([torch.tensor([0, 1, 0, 1, 1]), torch.tensor([0, 1, 2, 0, 2])], 1)\n&gt;&gt;&gt; loss_1 = MultiTargetClassificationLoss(binary_config=[0], multiclass_config=[(1, 3)], reduction=\"mean\")\n&gt;&gt;&gt; output_1 = loss_1(input, target)\n&gt;&gt;&gt; loss_2 = MultiTargetClassificationLoss(binary_config=[(0, 0.5)], multiclass_config=[(1, 3, [1., 2., 3.])],\n... reduction=\"sum\", weights=[0.5, 0.5])\n&gt;&gt;&gt; output_2 = loss_2(input, target)\n&gt;&gt;&gt; loss_3 = MultiTargetClassificationLoss(binary_config=[0], multiclass_config=[(1, 3)], binary_trick=True)\n&gt;&gt;&gt; output_3 = loss_3(input_binary_trick, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses_multitarget.py</code> <pre><code>class MultiTargetClassificationLoss(nn.Module):\n    \"\"\"\n    This class is a wrapper around the Pytorch binary_cross_entropy_with_logits and\n    cross_entropy losses. It allows for multi-target classification problems. The\n    user can provide a list of weights to apply to each target. The loss can be\n    either the sum or the mean of the individual losses\n\n    Parameters\n    ----------\n    binary_config: Optional[List[int | Tuple[int, float]]], default = None\n        List of integers with the index of the target for binary\n        classification or tuples with two elements: the index of the targets\n        or binary classification and the positive weight for binary\n        classification\n    multiclass_config: Optional[Tuple[int, int] | Tuple[int, int, List[float]]], default = None\n        List of tuples with two or three elements: the index of the target and the\n        number of classes for multiclass classification, or a tuple with the index of\n        the target, the number of classes and a list of weights to apply to each class\n        (i.e. the 'weight' parameter in the cross_entropy loss)\n    weights: Optional[List[float], default = None]\n        List of weights to apply to the loss associated to each target. The\n        length of the list must match the number of targets.\n        Alias: 'target_weights'\n    reduction: Literal[\"mean\", \"sum\"], default = \"sum\n        Specifies the reduction to apply to the loss associated to each\n        target: 'mean' | 'sum'. Note that this is NOT the same as the\n        reduction in the cross_entropy loss or the\n        binary_cross_entropy_with_logits. This reduction is applied after the\n        loss for each target has been computed. Alias: 'target_reduction'\n    binary_trick: bool, default = False\n        If True, each target will be considered independently and the loss\n        will be computed as binary_cross_entropy_with_logits. This is a\n        faster implementation. Note that the 'weights' parameter is not\n        compatible with binary_trick=True. Also note that if\n        binary_trick=True, the 'binary_config' must be a list of integers and\n        the 'multiclass_config' must be a list of tuples with two integers:\n        the index of the target and the number of classes. Finally, if\n        binary_trick=True, the binary targets must be the first targets in\n        the target tensor.\n\n        :information_source: **NOTE**: When using the binary_trick, the binary targets are\n          considered as 2 classes. Therefore, the pred_dim parametere of the\n          WideDeep class should be adjusted accordingly (adding 2 to per\n          binary target). For example, in a problem with a binary target and\n          a 4 class multiclassification target, the pred_dim should be 6.\n\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.losses_multitarget import MultiTargetClassificationLoss\n    &gt;&gt;&gt; input = torch.randn(5, 4)\n    &gt;&gt;&gt; input_binary_trick = torch.randn(5, 5)\n    &gt;&gt;&gt; target = torch.stack([torch.tensor([0, 1, 0, 1, 1]), torch.tensor([0, 1, 2, 0, 2])], 1)\n    &gt;&gt;&gt; loss_1 = MultiTargetClassificationLoss(binary_config=[0], multiclass_config=[(1, 3)], reduction=\"mean\")\n    &gt;&gt;&gt; output_1 = loss_1(input, target)\n    &gt;&gt;&gt; loss_2 = MultiTargetClassificationLoss(binary_config=[(0, 0.5)], multiclass_config=[(1, 3, [1., 2., 3.])],\n    ... reduction=\"sum\", weights=[0.5, 0.5])\n    &gt;&gt;&gt; output_2 = loss_2(input, target)\n    &gt;&gt;&gt; loss_3 = MultiTargetClassificationLoss(binary_config=[0], multiclass_config=[(1, 3)], binary_trick=True)\n    &gt;&gt;&gt; output_3 = loss_3(input_binary_trick, target)\n    \"\"\"\n\n    @alias(\"reduction\", [\"target_reduction\"])\n    @alias(\"weights\", [\"target_weights\"])\n    def __init__(  # noqa: C901\n        self,\n        binary_config: Optional[List[Union[int, Tuple[int, float]]]] = None,\n        multiclass_config: Optional[\n            List[Union[Tuple[int, int], Tuple[int, int, List[float]]]]\n        ] = None,\n        weights: Optional[List[float]] = None,\n        reduction: Literal[\"mean\", \"sum\"] = \"mean\",\n        binary_trick: bool = False,\n    ):\n        super(MultiTargetClassificationLoss, self).__init__()\n\n        if reduction not in [\"mean\", \"sum\"]:\n            raise ValueError(\"reduction must be either 'mean' or 'sum'\")\n\n        self.binary_config = binary_config\n        self.multiclass_config = multiclass_config\n        self.weights = weights\n        self.reduction = reduction\n        self.binary_trick = binary_trick\n\n        if self.weights is not None:\n            if len(self.weights) != (\n                len(self.binary_config) if self.binary_config is not None else 0\n            ) + (\n                len(self.multiclass_config) if self.multiclass_config is not None else 0\n            ):\n                raise ValueError(\n                    \"The number of weights must match the number of binary and multiclass targets\"\n                )\n\n        if self.binary_trick:\n            self._check_inputs_with_binary_trick()\n            self._binary_config: List[int] = binary_config  # type: ignore[assignment]\n            self._multiclass_config: List[Tuple[int, int]] = self.multiclass_config  # type: ignore[assignment]\n        else:\n            self.binary_config_with_pos_weights = (\n                (self._set_binary_config_without_binary_trick())\n                if self.binary_config is not None\n                else None\n            )\n            self.multiclass_config_with_weights = (\n                (self._set_multiclass_config_without_binary_trick())\n                if self.multiclass_config is not None\n                else None\n            )\n\n    def forward(self, input: Tensor, target: Tensor) -&gt; Tensor:\n        if self.binary_trick:\n            return self._forward_binary_trick(input, target)\n        else:\n            return self._forward_without_binary_trick(input, target)\n\n    def _forward_binary_trick(self, input: Tensor, target: Tensor) -&gt; Tensor:\n        binary_target_tensors: List[Tensor] = []\n        if self._binary_config:\n            for idx in self._binary_config:\n                binary_target_tensors.append(\n                    torch.eye(2)[target[:, idx].long()].to(input.device)\n                )\n        if self._multiclass_config:\n            for idx, n_classes in self._multiclass_config:\n                binary_target_tensors.append(\n                    torch.eye(n_classes)[target[:, idx].long()].to(input.device)\n                )\n        binary_target = torch.cat(binary_target_tensors, 1)\n        return F.binary_cross_entropy_with_logits(input, binary_target)\n\n    def _forward_without_binary_trick(self, input: Tensor, target: Tensor) -&gt; Tensor:\n        losses: List[Tensor] = []\n        if self.binary_config_with_pos_weights:\n            for idx, bpos_weight in self.binary_config_with_pos_weights:\n                _loss = F.binary_cross_entropy_with_logits(\n                    input[:, idx],\n                    target[:, idx].float(),\n                    pos_weight=(\n                        torch.tensor(bpos_weight).to(input.device)\n                        if bpos_weight is not None\n                        else None\n                    ),\n                )\n                losses.append(_loss)\n        if self.multiclass_config_with_weights:\n            for idx, n_classes, mpos_weight in self.multiclass_config_with_weights:\n                _loss = F.cross_entropy(\n                    input[:, idx : idx + n_classes],\n                    target[:, idx].long(),\n                    weight=(\n                        torch.tensor(mpos_weight).to(input.device)\n                        if mpos_weight is not None\n                        else None\n                    ),\n                )\n                losses.append(_loss)\n\n            if self.weights is not None:\n                losses = [l * w for l, w in zip(losses, self.weights)]  # noqa: E741\n\n        return (\n            torch.stack(losses).sum()\n            if self.reduction == \"sum\"\n            else torch.stack(losses).mean()\n        )\n\n    def _check_inputs_with_binary_trick(self):\n        if self.binary_config is not None:\n            if any(isinstance(bc, tuple) for bc in self.binary_config):\n                raise ValueError(\n                    \"binary_trick=True is only compatible with binary_config as a list of integers\"\n                )\n\n        if self.multiclass_config is not None:\n            if not all(len(mc) == 2 for mc in self.multiclass_config):\n                raise ValueError(\n                    \"binary_trick=True is only compatible with multiclass_config as a list of \"\n                    \"tuples with two integers: the index of the target and the number of classes\"\n                )\n\n        if self.binary_config is not None and self.multiclass_config is not None:\n            last_binary_idx = (\n                self.binary_config[-1][0]\n                if isinstance(self.binary_config[-1], tuple)\n                else self.binary_config[-1]\n            )\n            if last_binary_idx &gt;= self.multiclass_config[0][0]:\n                raise ValueError(\n                    \"When using binary_trick=True, the binary targets must be the first targets\"\n                    \" in the target tensor\"\n                )\n\n    def _set_binary_config_without_binary_trick(\n        self,\n    ) -&gt; List[Tuple[int, Optional[float]]]:\n        binary_config_with_pos_weights: List[Tuple[int, Optional[float]]] = []\n        for bc in self.binary_config:\n            if isinstance(bc, tuple):\n                binary_config_with_pos_weights.append(bc)\n            else:\n                binary_config_with_pos_weights.append((bc, None))\n        return binary_config_with_pos_weights\n\n    def _set_multiclass_config_without_binary_trick(\n        self,\n    ) -&gt; List[Tuple[int, int, Optional[List[float]]]]:\n        multiclass_config_with_weights: List[Tuple[int, int, Optional[List[float]]]] = (\n            []\n        )\n        for mc in self.multiclass_config:\n            if len(mc) == 3:\n                multiclass_config_with_weights.append(mc)  # type: ignore[arg-type]\n            else:\n                multiclass_config_with_weights.append((mc[0], mc[1], None))\n        return multiclass_config_with_weights\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses_multitarget.MutilTargetRegressionAndClassificationLoss","title":"MutilTargetRegressionAndClassificationLoss","text":"<p>               Bases: <code>Module</code></p> <p>This class is a wrapper around the MultiTargetRegressionLoss and the MultiTargetClassificationLoss. It allows for multi-target regression and classification problems. The user can provide a list of weights to apply to each target. The loss can be either the sum or the mean of the individual losses</p> <p>Parameters:</p> Name Type Description Default <code>regression_config</code> <code>List[int]</code> <p>List of integers with the indices of the regression targets</p> <code>[]</code> <code>binary_config</code> <code>Optional[List[Union[int, Tuple[int, float]]]]</code> <p>List of integers with the index of the target for binary classification or tuples with two elements: the index of the targets or binary classification and the positive weight for binary classification</p> <code>None</code> <code>multiclass_config</code> <code>Optional[List[Union[Tuple[int, int], Tuple[int, int, List[float]]]]]</code> <p>List of tuples with two or three elements: the index of the target and the number of classes for multiclass classification, or a tuple with the index of the target, the number of classes and a list of weights to apply to each class (i.e. the 'weight' parameter in the cross_entropy loss)</p> <code>None</code> <code>weights</code> <code>Optional[List[float]]</code> <p>List of weights to apply to the loss associated to each target. The length of the list must match the number of targets. Alias: 'target_weights'</p> <code>None</code> <code>reduction</code> <code>Literal[mean, sum]</code> <p>Specifies the reduction to apply to the output: 'mean' | 'sum'. Note that this is NOT the same as the reduction in the cross_entropy loss, the binary_cross_entropy_with_logits or the MSELoss. This reduction is applied after each target has been computed. Alias: 'target_reduction'</p> <code>'mean'</code> <code>binary_trick</code> <code>bool</code> <p>If True, each target will be considered independently and the loss will be computed as binary_cross_entropy_with_logits. This is a faster implementation. Note that the 'weights' parameter is not compatible with binary_trick=True. Also note that if binary_trick=True, the 'binary_config' must be a list of integers and the 'multiclass_config' must be a list of tuples with two integers: the index of the target and the number of classes. Finally, if binary_trick=True, the binary targets must be the first targets in the target tensor.</p> <p> NOTE: When using the binary_trick, the binary targets are   considered as 2 classes. Therefore, the pred_dim parametere of the   WideDeep class should be adjusted accordingly (adding 2 to per   binary target). For example, in a problem with a binary target and   a 4 class multiclassification target, the pred_dim should be 6.</p> <code>False</code> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.losses_multitarget import MutilTargetRegressionAndClassificationLoss\n&gt;&gt;&gt; input = torch.randn(5, 5)\n&gt;&gt;&gt; target = torch.stack([torch.randn(5), torch.tensor([0, 1, 0, 1, 1]), torch.tensor([0, 1, 2, 0, 2])], 1)\n&gt;&gt;&gt; loss = MutilTargetRegressionAndClassificationLoss(regression_config=[0], binary_config=[2],\n... multiclass_config=[(2, 3)], reduction=\"mean\")\n&gt;&gt;&gt; output = loss(input, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses_multitarget.py</code> <pre><code>class MutilTargetRegressionAndClassificationLoss(nn.Module):\n    \"\"\"\n    This class is a wrapper around the MultiTargetRegressionLoss and the\n    MultiTargetClassificationLoss. It allows for multi-target regression and\n    classification problems. The user can provide a list of weights to apply to\n    each target. The loss can be either the sum or the mean of the individual losses\n\n    Parameters\n    ----------\n    regression_config: List[int], default = []\n        List of integers with the indices of the regression targets\n    binary_config: Optional[List[int | Tuple[int, float]]], default = None\n        List of integers with the index of the target for binary\n        classification or tuples with two elements: the index of the targets\n        or binary classification and the positive weight for binary\n        classification\n    multiclass_config: Optional[Tuple[int, int] | Tuple[int, int, List[float]]], default = None\n        List of tuples with two or three elements: the index of the target and the\n        number of classes for multiclass classification, or a tuple with the index of\n        the target, the number of classes and a list of weights to apply to each class\n        (i.e. the 'weight' parameter in the cross_entropy loss)\n    weights: Optional[List[float], default = None]\n        List of weights to apply to the loss associated to each target. The\n        length of the list must match the number of targets.\n        Alias: 'target_weights'\n    reduction: Literal[\"mean\", \"sum\"], default = \"sum\n        Specifies the reduction to apply to the output: 'mean' | 'sum'. Note\n        that this is NOT the same as the reduction in the cross_entropy loss,\n        the binary_cross_entropy_with_logits or the MSELoss. This reduction\n        is applied after each target has been computed. Alias: 'target_reduction'\n    binary_trick: bool, default = False\n        If True, each target will be considered independently and the loss\n        will be computed as binary_cross_entropy_with_logits. This is a\n        faster implementation. Note that the 'weights' parameter is not\n        compatible with binary_trick=True. Also note that if\n        binary_trick=True, the 'binary_config' must be a list of integers and\n        the 'multiclass_config' must be a list of tuples with two integers:\n        the index of the target and the number of classes. Finally, if\n        binary_trick=True, the binary targets must be the first targets in\n        the target tensor.\n\n        :information_source: **NOTE**: When using the binary_trick, the binary targets are\n          considered as 2 classes. Therefore, the pred_dim parametere of the\n          WideDeep class should be adjusted accordingly (adding 2 to per\n          binary target). For example, in a problem with a binary target and\n          a 4 class multiclassification target, the pred_dim should be 6.\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.losses_multitarget import MutilTargetRegressionAndClassificationLoss\n    &gt;&gt;&gt; input = torch.randn(5, 5)\n    &gt;&gt;&gt; target = torch.stack([torch.randn(5), torch.tensor([0, 1, 0, 1, 1]), torch.tensor([0, 1, 2, 0, 2])], 1)\n    &gt;&gt;&gt; loss = MutilTargetRegressionAndClassificationLoss(regression_config=[0], binary_config=[2],\n    ... multiclass_config=[(2, 3)], reduction=\"mean\")\n    &gt;&gt;&gt; output = loss(input, target)\n    \"\"\"\n\n    @alias(\"reduction\", [\"target_reduction\"])\n    @alias(\"weights\", [\"target_weights\"])\n    def __init__(  # noqa: C901\n        self,\n        regression_config: List[int] = [],\n        binary_config: Optional[List[Union[int, Tuple[int, float]]]] = None,\n        multiclass_config: Optional[\n            List[Union[Tuple[int, int], Tuple[int, int, List[float]]]]\n        ] = None,\n        weights: Optional[List[float]] = None,\n        reduction: Literal[\"mean\", \"sum\"] = \"mean\",\n        binary_trick: bool = False,\n    ):\n\n        super(MutilTargetRegressionAndClassificationLoss, self).__init__()\n\n        self.regression_config = regression_config\n\n        assert binary_config is not None or multiclass_config is not None, (\n            \"Either binary_config or multiclass_config must be provided. \"\n            \"Otherwise, use the MultiTargetRegressionLoss\"\n        )\n\n        if binary_trick:\n            self._check_inputs_with_binary_trick(\n                regression_config, binary_config, multiclass_config\n            )\n\n        if weights is not None:\n            if len(weights) != (\n                len(regression_config)\n                + (len(binary_config) if binary_config is not None else 0)\n                + (len(multiclass_config) if multiclass_config is not None else 0)\n            ):\n                raise ValueError(\n                    \"The number of weights must match the number of regression, binary and multiclass targets\"\n                )\n\n            self.weights_regression = self._prepare_weights_for_regression_targets(\n                weights, regression_config\n            )\n            self.weights_binary = self._prepare_weights_per_binary_targets(\n                weights, binary_config\n            )\n            self.weights_multiclass = self._prepare_weights_per_multiclass_targets(\n                weights, multiclass_config\n            )\n            self.weights = weights\n        else:\n            self.weights_regression = None\n            self.weights_binary = None\n            self.weights_multiclass = None\n\n        self.multi_target_regression_loss = MultiTargetRegressionLoss(\n            weights=self.weights_regression, reduction=reduction\n        )\n\n        self.multi_target_classification_loss = MultiTargetClassificationLoss(\n            binary_config=binary_config,\n            multiclass_config=multiclass_config,\n            weights=(\n                self.weights_binary + self.weights_multiclass\n                if self.weights_binary is not None\n                and self.weights_multiclass is not None\n                else (\n                    self.weights_binary\n                    if self.weights_binary is not None\n                    else self.weights_multiclass\n                )\n            ),\n            reduction=reduction,\n            binary_trick=binary_trick,\n        )\n\n    def forward(self, input: Tensor, target: Tensor) -&gt; Tensor:\n\n        regression_loss = self.multi_target_regression_loss(\n            input[:, self.regression_config],\n            target[:, self.regression_config],\n        )\n\n        if self.multi_target_classification_loss.binary_trick:\n            classification_loss = self.multi_target_classification_loss(\n                input[:, len(self.regression_config) :], target\n            )\n        else:\n            classification_loss = self.multi_target_classification_loss(input, target)\n\n        return regression_loss + classification_loss\n\n    def _check_inputs_with_binary_trick(\n        self,\n        regression_config: List[int],\n        binary_config: Optional[List[Union[int, Tuple[int, float]]]],\n        multiclass_config: Optional[\n            List[Union[Tuple[int, int], Tuple[int, int, List[float]]]]\n        ],\n    ) -&gt; None:\n\n        error_msg = \"When using binary_trick=True, the targets order must be: regression, binary and multiclass\"\n\n        first_regression_idx = regression_config[0]\n        last_regression_idx = regression_config[-1]\n        if first_regression_idx != 0:\n            raise ValueError(error_msg)\n\n        if binary_config is not None and multiclass_config is not None:\n            first_binary_idx = (\n                binary_config[0][0]\n                if isinstance(binary_config[0], tuple)\n                else binary_config[0]\n            )\n            last_binary_idx = (\n                binary_config[-1][0]\n                if isinstance(binary_config[-1], tuple)\n                else binary_config[-1]\n            )\n            first_multiclass_idx = multiclass_config[0][0]\n\n            if (first_binary_idx != last_regression_idx + 1) or (\n                last_binary_idx &gt;= first_multiclass_idx\n            ):\n                raise ValueError(error_msg)\n        elif binary_config is not None:\n            first_binary_idx = (\n                binary_config[0][0]\n                if isinstance(binary_config[0], tuple)\n                else binary_config[0]\n            )\n            if first_binary_idx != last_regression_idx + 1:\n                raise ValueError(error_msg)\n        elif multiclass_config is not None:\n            first_multiclass_idx = multiclass_config[0][0]\n            if first_multiclass_idx != last_regression_idx + 1:\n                raise ValueError(error_msg)\n        else:\n            raise ValueError(\n                \"Either binary_config or multiclass_config must be provided. \"\n                \"Otherwise, use the MultiTargetRegressionLoss\"\n            )\n\n    def _prepare_weights_for_regression_targets(\n        self,\n        weights: List[float],\n        regression_config: List[int],\n    ) -&gt; List[float]:\n\n        weights_regression = [\n            w for idx, w in enumerate(weights) if idx in regression_config\n        ]\n\n        return weights_regression\n\n    def _prepare_weights_per_binary_targets(\n        self,\n        weights: List[float],\n        binary_config: Optional[List[Union[int, Tuple[int, float]]]],\n    ) -&gt; Optional[List[float]]:\n\n        if binary_config is not None:\n            binary_idx: List[int] = []\n            for bc in binary_config:\n                if isinstance(bc, tuple):\n                    binary_idx.append(bc[0])\n                else:\n                    binary_idx.append(bc)\n            weights_binary = [w for idx, w in enumerate(weights) if idx in binary_idx]\n        else:\n            weights_binary = None\n\n        return weights_binary\n\n    def _prepare_weights_per_multiclass_targets(\n        self,\n        weights: List[float],\n        multiclass_config: Optional[\n            List[Union[Tuple[int, int], Tuple[int, int, List[float]]]]\n        ],\n    ) -&gt; Optional[List[float]]:\n\n        if multiclass_config is not None:\n            multiclass_idx: List[int] = [mc[0] for mc in multiclass_config]\n            weights_multiclass = [\n                w for idx, w in enumerate(weights) if idx in multiclass_idx\n            ]\n        else:\n            weights_multiclass = None\n\n        return weights_multiclass\n</code></pre>"},{"location":"pytorch-widedeep/metrics.html","title":"Metrics","text":"<p> NOTE: metrics in this module expect the predictions  and ground truth to have the same dimensions for regression and binary  classification problems: \\((N_{samples}, 1)\\). In the case of multiclass  classification problems the ground truth is expected to be a 1D tensor with  the corresponding classes. See Examples below</p> <p>We have added the possibility of using the metrics available at the torchmetrics library. Note that this library is still in its early versions and therefore this option should be used with caution. To use <code>torchmetrics</code> simply import them and use them as any of the <code>pytorch-widedeep</code> metrics described below.</p> <pre><code>from torchmetrics import Accuracy, Precision\n\naccuracy = Accuracy(average=None, num_classes=2)\nprecision = Precision(average='micro', num_classes=2)\n\ntrainer = Trainer(model, objective=\"binary\", metrics=[accuracy, precision])\n</code></pre> <p>A functioning example for <code>pytorch-widedeep</code> using <code>torchmetrics</code> can be found in the Examples folder</p> <p> NOTE: the forward method for all metrics in this  module takes two tensors, <code>y_pred</code> and <code>y_true</code> (in that order). Therefore,  we do not include the method in the documentation.</p>"},{"location":"pytorch-widedeep/metrics.html#pytorch_widedeep.metrics.Accuracy","title":"Accuracy","text":"<p>               Bases: <code>Metric</code></p> <p>Class to calculate the accuracy for both binary and categorical problems</p> <p>Parameters:</p> Name Type Description Default <code>top_k</code> <code>int</code> <p>Accuracy will be computed using the top k most likely classes in multiclass problems</p> <code>1</code> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt;\n&gt;&gt;&gt; from pytorch_widedeep.metrics import Accuracy\n&gt;&gt;&gt;\n&gt;&gt;&gt; acc = Accuracy()\n&gt;&gt;&gt; y_true = torch.tensor([0, 1, 0, 1]).view(-1, 1)\n&gt;&gt;&gt; y_pred = torch.tensor([[0.3, 0.2, 0.6, 0.7]]).view(-1, 1)\n&gt;&gt;&gt; acc(y_pred, y_true)\narray(0.5)\n&gt;&gt;&gt;\n&gt;&gt;&gt; acc = Accuracy(top_k=2)\n&gt;&gt;&gt; y_true = torch.tensor([0, 1, 2])\n&gt;&gt;&gt; y_pred = torch.tensor([[0.3, 0.5, 0.2], [0.1, 0.1, 0.8], [0.1, 0.5, 0.4]])\n&gt;&gt;&gt; acc(y_pred, y_true)\narray(0.66666667)\n</code></pre> Source code in <code>pytorch_widedeep/metrics.py</code> <pre><code>class Accuracy(Metric):\n    r\"\"\"Class to calculate the accuracy for both binary and categorical problems\n\n    Parameters\n    ----------\n    top_k: int, default = 1\n        Accuracy will be computed using the top k most likely classes in\n        multiclass problems\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; from pytorch_widedeep.metrics import Accuracy\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; acc = Accuracy()\n    &gt;&gt;&gt; y_true = torch.tensor([0, 1, 0, 1]).view(-1, 1)\n    &gt;&gt;&gt; y_pred = torch.tensor([[0.3, 0.2, 0.6, 0.7]]).view(-1, 1)\n    &gt;&gt;&gt; acc(y_pred, y_true)\n    array(0.5)\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; acc = Accuracy(top_k=2)\n    &gt;&gt;&gt; y_true = torch.tensor([0, 1, 2])\n    &gt;&gt;&gt; y_pred = torch.tensor([[0.3, 0.5, 0.2], [0.1, 0.1, 0.8], [0.1, 0.5, 0.4]])\n    &gt;&gt;&gt; acc(y_pred, y_true)\n    array(0.66666667)\n    \"\"\"\n\n    def __init__(self, top_k: int = 1):\n        super(Accuracy, self).__init__()\n\n        self.top_k = top_k\n        self.correct_count = 0\n        self.total_count = 0\n        self._name = \"acc\"\n\n    def reset(self):\n        \"\"\"\n        resets counters to 0\n        \"\"\"\n        self.correct_count = 0\n        self.total_count = 0\n\n    def __call__(self, y_pred: Tensor, y_true: Tensor) -&gt; np.ndarray:\n        num_classes = y_pred.size(1)\n\n        if num_classes == 1:\n            y_pred = y_pred.round()\n            y_true = y_true\n        elif num_classes &gt; 1:\n            y_pred = y_pred.topk(self.top_k, 1)[1]\n            y_true = y_true.view(-1, 1).expand_as(y_pred)\n\n        self.correct_count += y_pred.eq(y_true).sum().item()  # type: ignore[assignment]\n        self.total_count += len(y_pred)\n        accuracy = float(self.correct_count) / float(self.total_count)\n        return np.array(accuracy)\n</code></pre>"},{"location":"pytorch-widedeep/metrics.html#pytorch_widedeep.metrics.Accuracy.reset","title":"reset","text":"<pre><code>reset()\n</code></pre> <p>resets counters to 0</p> Source code in <code>pytorch_widedeep/metrics.py</code> <pre><code>def reset(self):\n    \"\"\"\n    resets counters to 0\n    \"\"\"\n    self.correct_count = 0\n    self.total_count = 0\n</code></pre>"},{"location":"pytorch-widedeep/metrics.html#pytorch_widedeep.metrics.Precision","title":"Precision","text":"<p>               Bases: <code>Metric</code></p> <p>Class to calculate the precision for both binary and categorical problems</p> <p>Parameters:</p> Name Type Description Default <code>average</code> <code>bool</code> <p>This applies only to multiclass problems. if <code>True</code> calculate precision for each label, and finds their unweighted mean.</p> <code>True</code> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt;\n&gt;&gt;&gt; from pytorch_widedeep.metrics import Precision\n&gt;&gt;&gt;\n&gt;&gt;&gt; prec = Precision()\n&gt;&gt;&gt; y_true = torch.tensor([0, 1, 0, 1]).view(-1, 1)\n&gt;&gt;&gt; y_pred = torch.tensor([[0.3, 0.2, 0.6, 0.7]]).view(-1, 1)\n&gt;&gt;&gt; prec(y_pred, y_true)\narray(0.5)\n&gt;&gt;&gt;\n&gt;&gt;&gt; prec = Precision(average=True)\n&gt;&gt;&gt; y_true = torch.tensor([0, 1, 2])\n&gt;&gt;&gt; y_pred = torch.tensor([[0.7, 0.1, 0.2], [0.1, 0.1, 0.8], [0.1, 0.5, 0.4]])\n&gt;&gt;&gt; prec(y_pred, y_true)\narray(0.33333334)\n</code></pre> Source code in <code>pytorch_widedeep/metrics.py</code> <pre><code>class Precision(Metric):\n    r\"\"\"Class to calculate the precision for both binary and categorical problems\n\n    Parameters\n    ----------\n    average: bool, default = True\n        This applies only to multiclass problems. if ``True`` calculate\n        precision for each label, and finds their unweighted mean.\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; from pytorch_widedeep.metrics import Precision\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; prec = Precision()\n    &gt;&gt;&gt; y_true = torch.tensor([0, 1, 0, 1]).view(-1, 1)\n    &gt;&gt;&gt; y_pred = torch.tensor([[0.3, 0.2, 0.6, 0.7]]).view(-1, 1)\n    &gt;&gt;&gt; prec(y_pred, y_true)\n    array(0.5)\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; prec = Precision(average=True)\n    &gt;&gt;&gt; y_true = torch.tensor([0, 1, 2])\n    &gt;&gt;&gt; y_pred = torch.tensor([[0.7, 0.1, 0.2], [0.1, 0.1, 0.8], [0.1, 0.5, 0.4]])\n    &gt;&gt;&gt; prec(y_pred, y_true)\n    array(0.33333334)\n    \"\"\"\n\n    def __init__(self, average: bool = True):\n        super(Precision, self).__init__()\n\n        self.average = average\n        self.true_positives = 0\n        self.all_positives = 0\n        self.eps = 1e-20\n        self._name = \"prec\"\n\n    def reset(self):\n        \"\"\"\n        resets counters to 0\n        \"\"\"\n        self.true_positives = 0\n        self.all_positives = 0\n\n    def __call__(self, y_pred: Tensor, y_true: Tensor) -&gt; np.ndarray:\n        num_class = y_pred.size(1)\n\n        if num_class == 1:\n            y_pred = y_pred.round()\n            y_true = y_true\n        elif num_class &gt; 1:\n            y_true = torch.eye(num_class)[y_true.squeeze().cpu().long()]\n            y_pred = y_pred.topk(1, 1)[1].view(-1)\n            y_pred = torch.eye(num_class)[y_pred.cpu().long()]\n\n        self.true_positives += (y_true * y_pred).sum(dim=0)  # type:ignore\n        self.all_positives += y_pred.sum(dim=0)  # type:ignore\n\n        precision = self.true_positives / (self.all_positives + self.eps)\n\n        if self.average:\n            return np.array(precision.mean().item())  # type:ignore\n        else:\n            return precision.detach().cpu().numpy()  # type: ignore[attr-defined]\n</code></pre>"},{"location":"pytorch-widedeep/metrics.html#pytorch_widedeep.metrics.Precision.reset","title":"reset","text":"<pre><code>reset()\n</code></pre> <p>resets counters to 0</p> Source code in <code>pytorch_widedeep/metrics.py</code> <pre><code>def reset(self):\n    \"\"\"\n    resets counters to 0\n    \"\"\"\n    self.true_positives = 0\n    self.all_positives = 0\n</code></pre>"},{"location":"pytorch-widedeep/metrics.html#pytorch_widedeep.metrics.Recall","title":"Recall","text":"<p>               Bases: <code>Metric</code></p> <p>Class to calculate the recall for both binary and categorical problems</p> <p>Parameters:</p> Name Type Description Default <code>average</code> <code>bool</code> <p>This applies only to multiclass problems. if <code>True</code> calculate recall for each label, and finds their unweighted mean.</p> <code>True</code> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt;\n&gt;&gt;&gt; from pytorch_widedeep.metrics import Recall\n&gt;&gt;&gt;\n&gt;&gt;&gt; rec = Recall()\n&gt;&gt;&gt; y_true = torch.tensor([0, 1, 0, 1]).view(-1, 1)\n&gt;&gt;&gt; y_pred = torch.tensor([[0.3, 0.2, 0.6, 0.7]]).view(-1, 1)\n&gt;&gt;&gt; rec(y_pred, y_true)\narray(0.5)\n&gt;&gt;&gt;\n&gt;&gt;&gt; rec = Recall(average=True)\n&gt;&gt;&gt; y_true = torch.tensor([0, 1, 2])\n&gt;&gt;&gt; y_pred = torch.tensor([[0.7, 0.1, 0.2], [0.1, 0.1, 0.8], [0.1, 0.5, 0.4]])\n&gt;&gt;&gt; rec(y_pred, y_true)\narray(0.33333334)\n</code></pre> Source code in <code>pytorch_widedeep/metrics.py</code> <pre><code>class Recall(Metric):\n    r\"\"\"Class to calculate the recall for both binary and categorical problems\n\n    Parameters\n    ----------\n    average: bool, default = True\n        This applies only to multiclass problems. if ``True`` calculate recall\n        for each label, and finds their unweighted mean.\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; from pytorch_widedeep.metrics import Recall\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; rec = Recall()\n    &gt;&gt;&gt; y_true = torch.tensor([0, 1, 0, 1]).view(-1, 1)\n    &gt;&gt;&gt; y_pred = torch.tensor([[0.3, 0.2, 0.6, 0.7]]).view(-1, 1)\n    &gt;&gt;&gt; rec(y_pred, y_true)\n    array(0.5)\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; rec = Recall(average=True)\n    &gt;&gt;&gt; y_true = torch.tensor([0, 1, 2])\n    &gt;&gt;&gt; y_pred = torch.tensor([[0.7, 0.1, 0.2], [0.1, 0.1, 0.8], [0.1, 0.5, 0.4]])\n    &gt;&gt;&gt; rec(y_pred, y_true)\n    array(0.33333334)\n    \"\"\"\n\n    def __init__(self, average: bool = True):\n        super(Recall, self).__init__()\n\n        self.average = average\n        self.true_positives = 0\n        self.actual_positives = 0\n        self.eps = 1e-20\n        self._name = \"rec\"\n\n    def reset(self):\n        \"\"\"\n        resets counters to 0\n        \"\"\"\n        self.true_positives = 0\n        self.actual_positives = 0\n\n    def __call__(self, y_pred: Tensor, y_true: Tensor) -&gt; np.ndarray:\n        num_class = y_pred.size(1)\n\n        if num_class == 1:\n            y_pred = y_pred.round()\n            y_true = y_true\n        elif num_class &gt; 1:\n            y_true = torch.eye(num_class)[y_true.squeeze().cpu().long()]\n            y_pred = y_pred.topk(1, 1)[1].view(-1)\n            y_pred = torch.eye(num_class)[y_pred.cpu().long()]\n\n        self.true_positives += (y_true * y_pred).sum(dim=0)  # type: ignore\n        self.actual_positives += y_true.sum(dim=0)  # type: ignore\n\n        recall = self.true_positives / (self.actual_positives + self.eps)\n\n        if self.average:\n            return np.array(recall.mean().item())  # type:ignore\n        else:\n            return recall.detach().cpu().numpy()  # type: ignore[attr-defined]\n</code></pre>"},{"location":"pytorch-widedeep/metrics.html#pytorch_widedeep.metrics.Recall.reset","title":"reset","text":"<pre><code>reset()\n</code></pre> <p>resets counters to 0</p> Source code in <code>pytorch_widedeep/metrics.py</code> <pre><code>def reset(self):\n    \"\"\"\n    resets counters to 0\n    \"\"\"\n    self.true_positives = 0\n    self.actual_positives = 0\n</code></pre>"},{"location":"pytorch-widedeep/metrics.html#pytorch_widedeep.metrics.FBetaScore","title":"FBetaScore","text":"<p>               Bases: <code>Metric</code></p> <p>Class to calculate the fbeta score for both binary and categorical problems</p> \\[ F_{\\beta} = ((1 + {\\beta}^2) * \\frac{(precision * recall)}{({\\beta}^2 * precision + recall)} \\] <p>Parameters:</p> Name Type Description Default <code>beta</code> <code>int</code> <p>Coefficient to control the balance between precision and recall</p> required <code>average</code> <code>bool</code> <p>This applies only to multiclass problems. if <code>True</code> calculate fbeta for each label, and find their unweighted mean.</p> <code>True</code> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt;\n&gt;&gt;&gt; from pytorch_widedeep.metrics import FBetaScore\n&gt;&gt;&gt;\n&gt;&gt;&gt; fbeta = FBetaScore(beta=2)\n&gt;&gt;&gt; y_true = torch.tensor([0, 1, 0, 1]).view(-1, 1)\n&gt;&gt;&gt; y_pred = torch.tensor([[0.3, 0.2, 0.6, 0.7]]).view(-1, 1)\n&gt;&gt;&gt; fbeta(y_pred, y_true)\narray(0.5)\n&gt;&gt;&gt;\n&gt;&gt;&gt; fbeta = FBetaScore(beta=2)\n&gt;&gt;&gt; y_true = torch.tensor([0, 1, 2])\n&gt;&gt;&gt; y_pred = torch.tensor([[0.7, 0.1, 0.2], [0.1, 0.1, 0.8], [0.1, 0.5, 0.4]])\n&gt;&gt;&gt; fbeta(y_pred, y_true)\narray(0.33333334)\n</code></pre> Source code in <code>pytorch_widedeep/metrics.py</code> <pre><code>class FBetaScore(Metric):\n    r\"\"\"Class to calculate the fbeta score for both binary and categorical problems\n\n    $$\n    F_{\\beta} = ((1 + {\\beta}^2) * \\frac{(precision * recall)}{({\\beta}^2 * precision + recall)}\n    $$\n\n    Parameters\n    ----------\n    beta: int\n        Coefficient to control the balance between precision and recall\n    average: bool, default = True\n        This applies only to multiclass problems. if ``True`` calculate fbeta\n        for each label, and find their unweighted mean.\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; from pytorch_widedeep.metrics import FBetaScore\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; fbeta = FBetaScore(beta=2)\n    &gt;&gt;&gt; y_true = torch.tensor([0, 1, 0, 1]).view(-1, 1)\n    &gt;&gt;&gt; y_pred = torch.tensor([[0.3, 0.2, 0.6, 0.7]]).view(-1, 1)\n    &gt;&gt;&gt; fbeta(y_pred, y_true)\n    array(0.5)\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; fbeta = FBetaScore(beta=2)\n    &gt;&gt;&gt; y_true = torch.tensor([0, 1, 2])\n    &gt;&gt;&gt; y_pred = torch.tensor([[0.7, 0.1, 0.2], [0.1, 0.1, 0.8], [0.1, 0.5, 0.4]])\n    &gt;&gt;&gt; fbeta(y_pred, y_true)\n    array(0.33333334)\n    \"\"\"\n\n    def __init__(self, beta: int, average: bool = True):\n        super(FBetaScore, self).__init__()\n\n        self.beta = beta\n        self.average = average\n        self.precision = Precision(average=False)\n        self.recall = Recall(average=False)\n        self.eps = 1e-20\n        self._name = \"\".join([\"f\", str(self.beta)])\n\n    def reset(self):\n        \"\"\"\n        resets precision and recall\n        \"\"\"\n        self.precision.reset()\n        self.recall.reset()\n\n    def __call__(self, y_pred: Tensor, y_true: Tensor) -&gt; np.ndarray:\n        prec = self.precision(y_pred, y_true)\n        rec = self.recall(y_pred, y_true)\n        beta2 = self.beta**2\n\n        fbeta = ((1 + beta2) * prec * rec) / (beta2 * prec + rec + self.eps)\n\n        if self.average:\n            return np.array(fbeta.mean().item())  # type: ignore[attr-defined]\n        else:\n            return fbeta\n</code></pre>"},{"location":"pytorch-widedeep/metrics.html#pytorch_widedeep.metrics.FBetaScore.reset","title":"reset","text":"<pre><code>reset()\n</code></pre> <p>resets precision and recall</p> Source code in <code>pytorch_widedeep/metrics.py</code> <pre><code>def reset(self):\n    \"\"\"\n    resets precision and recall\n    \"\"\"\n    self.precision.reset()\n    self.recall.reset()\n</code></pre>"},{"location":"pytorch-widedeep/metrics.html#pytorch_widedeep.metrics.F1Score","title":"F1Score","text":"<p>               Bases: <code>Metric</code></p> <p>Class to calculate the f1 score for both binary and categorical problems</p> <p>Parameters:</p> Name Type Description Default <code>average</code> <code>bool</code> <p>This applies only to multiclass problems. if <code>True</code> calculate f1 for each label, and find their unweighted mean.</p> <code>True</code> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt;\n&gt;&gt;&gt; from pytorch_widedeep.metrics import F1Score\n&gt;&gt;&gt;\n&gt;&gt;&gt; f1 = F1Score()\n&gt;&gt;&gt; y_true = torch.tensor([0, 1, 0, 1]).view(-1, 1)\n&gt;&gt;&gt; y_pred = torch.tensor([[0.3, 0.2, 0.6, 0.7]]).view(-1, 1)\n&gt;&gt;&gt; f1(y_pred, y_true)\narray(0.5)\n&gt;&gt;&gt;\n&gt;&gt;&gt; f1 = F1Score()\n&gt;&gt;&gt; y_true = torch.tensor([0, 1, 2])\n&gt;&gt;&gt; y_pred = torch.tensor([[0.7, 0.1, 0.2], [0.1, 0.1, 0.8], [0.1, 0.5, 0.4]])\n&gt;&gt;&gt; f1(y_pred, y_true)\narray(0.33333334)\n</code></pre> Source code in <code>pytorch_widedeep/metrics.py</code> <pre><code>class F1Score(Metric):\n    r\"\"\"Class to calculate the f1 score for both binary and categorical problems\n\n    Parameters\n    ----------\n    average: bool, default = True\n        This applies only to multiclass problems. if ``True`` calculate f1 for\n        each label, and find their unweighted mean.\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; from pytorch_widedeep.metrics import F1Score\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; f1 = F1Score()\n    &gt;&gt;&gt; y_true = torch.tensor([0, 1, 0, 1]).view(-1, 1)\n    &gt;&gt;&gt; y_pred = torch.tensor([[0.3, 0.2, 0.6, 0.7]]).view(-1, 1)\n    &gt;&gt;&gt; f1(y_pred, y_true)\n    array(0.5)\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; f1 = F1Score()\n    &gt;&gt;&gt; y_true = torch.tensor([0, 1, 2])\n    &gt;&gt;&gt; y_pred = torch.tensor([[0.7, 0.1, 0.2], [0.1, 0.1, 0.8], [0.1, 0.5, 0.4]])\n    &gt;&gt;&gt; f1(y_pred, y_true)\n    array(0.33333334)\n    \"\"\"\n\n    def __init__(self, average: bool = True):\n        super(F1Score, self).__init__()\n\n        self.average = average\n        self.f1 = FBetaScore(beta=1, average=self.average)\n        self._name = self.f1._name\n\n    def reset(self):\n        \"\"\"\n        resets counters to 0\n        \"\"\"\n        self.f1.reset()\n\n    def __call__(self, y_pred: Tensor, y_true: Tensor) -&gt; np.ndarray:\n        return self.f1(y_pred, y_true)\n</code></pre>"},{"location":"pytorch-widedeep/metrics.html#pytorch_widedeep.metrics.F1Score.reset","title":"reset","text":"<pre><code>reset()\n</code></pre> <p>resets counters to 0</p> Source code in <code>pytorch_widedeep/metrics.py</code> <pre><code>def reset(self):\n    \"\"\"\n    resets counters to 0\n    \"\"\"\n    self.f1.reset()\n</code></pre>"},{"location":"pytorch-widedeep/metrics.html#pytorch_widedeep.metrics.R2Score","title":"R2Score","text":"<p>               Bases: <code>Metric</code></p> <p>Calculates R-Squared, the coefficient of determination:</p> \\[ R^2 = 1 - \\frac{\\sum_{j=1}^n(y_j - \\hat{y_j})^2}{\\sum_{j=1}^n(y_j - \\bar{y})^2} \\] <p>where \\(\\hat{y_j}\\) is the ground truth, \\(y_j\\) is the predicted value and \\(\\bar{y}\\) is the mean of the ground truth.</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt;\n&gt;&gt;&gt; from pytorch_widedeep.metrics import R2Score\n&gt;&gt;&gt;\n&gt;&gt;&gt; r2 = R2Score()\n&gt;&gt;&gt; y_true = torch.tensor([3, -0.5, 2, 7]).view(-1, 1)\n&gt;&gt;&gt; y_pred = torch.tensor([2.5, 0.0, 2, 8]).view(-1, 1)\n&gt;&gt;&gt; r2(y_pred, y_true)\narray(0.94860814)\n</code></pre> Source code in <code>pytorch_widedeep/metrics.py</code> <pre><code>class R2Score(Metric):\n    r\"\"\"\n    Calculates R-Squared, the\n    [coefficient of determination](https://en.wikipedia.org/wiki/Coefficient_of_determination&gt;):\n\n    $$\n    R^2 = 1 - \\frac{\\sum_{j=1}^n(y_j - \\hat{y_j})^2}{\\sum_{j=1}^n(y_j - \\bar{y})^2}\n    $$\n\n    where $\\hat{y_j}$ is the ground truth, $y_j$ is the predicted value and\n    $\\bar{y}$ is the mean of the ground truth.\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; from pytorch_widedeep.metrics import R2Score\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; r2 = R2Score()\n    &gt;&gt;&gt; y_true = torch.tensor([3, -0.5, 2, 7]).view(-1, 1)\n    &gt;&gt;&gt; y_pred = torch.tensor([2.5, 0.0, 2, 8]).view(-1, 1)\n    &gt;&gt;&gt; r2(y_pred, y_true)\n    array(0.94860814)\n    \"\"\"\n\n    def __init__(self):\n        self.numerator = 0\n        self.denominator = 0\n        self.num_examples = 0\n        self.y_true_sum = 0\n\n        self._name = \"r2\"\n\n    def reset(self):\n        \"\"\"\n        resets counters to 0\n        \"\"\"\n        self.numerator = 0\n        self.denominator = 0\n        self.num_examples = 0\n        self.y_true_sum = 0\n\n    def __call__(self, y_pred: Tensor, y_true: Tensor) -&gt; np.ndarray:\n        self.numerator += ((y_pred - y_true) ** 2).sum().item()\n\n        self.num_examples += y_true.shape[0]\n        self.y_true_sum += y_true.sum().item()\n        y_true_avg = self.y_true_sum / self.num_examples\n        self.denominator += ((y_true - y_true_avg) ** 2).sum().item()\n        return np.array((1 - (self.numerator / self.denominator)))\n</code></pre>"},{"location":"pytorch-widedeep/metrics.html#pytorch_widedeep.metrics.R2Score.reset","title":"reset","text":"<pre><code>reset()\n</code></pre> <p>resets counters to 0</p> Source code in <code>pytorch_widedeep/metrics.py</code> <pre><code>def reset(self):\n    \"\"\"\n    resets counters to 0\n    \"\"\"\n    self.numerator = 0\n    self.denominator = 0\n    self.num_examples = 0\n    self.y_true_sum = 0\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html","title":"The <code>models</code> module","text":"<p>This module contains the models that can be used as the four main components that will comprise a Wide and Deep model (<code>wide</code>, <code>deeptabular</code>, <code>deeptext</code>, <code>deepimage</code>), as well as the <code>WideDeep</code> \"constructor\" class. Note that each of the four components can be used independently. It also contains all the documentation for the models that can be used for self-supervised pre-training with tabular data.</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.linear.wide.Wide","title":"Wide","text":"<p>               Bases: <code>Module</code></p> <p>Defines a <code>Wide</code> (linear) model where the non-linearities are captured via the so-called crossed-columns. This can be used as the <code>wide</code> component of a Wide &amp; Deep model.</p> <p>Parameters:</p> Name Type Description Default <code>input_dim</code> <code>int</code> <p>size of the Linear layer (implemented via an Embedding layer). <code>input_dim</code> is the summation of all the individual values for all the features that go through the wide model. For example, if the wide model receives 2 features with 5 individual values each, <code>input_dim = 10</code></p> required <code>pred_dim</code> <code>int</code> <p>size of the ouput tensor containing the predictions. Note that unlike all the other models, the wide model is connected directly to the output neuron(s) when used to build a Wide and Deep model. Therefore, it requires the <code>pred_dim</code> parameter.</p> <code>1</code> <p>Attributes:</p> Name Type Description <code>wide_linear</code> <code>Module</code> <p>the linear layer that comprises the wide branch of the model</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import Wide\n&gt;&gt;&gt; X = torch.empty(4, 4).random_(20)\n&gt;&gt;&gt; wide = Wide(input_dim=int(X.max().item()), pred_dim=1)\n&gt;&gt;&gt; out = wide(X)\n</code></pre> Source code in <code>pytorch_widedeep/models/tabular/linear/wide.py</code> <pre><code>class Wide(nn.Module):\n    r\"\"\"Defines a `Wide` (linear) model where the non-linearities are\n    captured via the so-called crossed-columns. This can be used as the\n    `wide` component of a Wide &amp; Deep model.\n\n    Parameters\n    -----------\n    input_dim: int\n        size of the Linear layer (implemented via an Embedding layer).\n        `input_dim` is the summation of all the individual values for all the\n        features that go through the wide model. For example, if the wide\n        model receives 2 features with 5 individual values each, `input_dim =\n        10`\n    pred_dim: int, default = 1\n        size of the ouput tensor containing the predictions. Note that unlike\n        all the other models, the wide model is connected directly to the\n        output neuron(s) when used to build a Wide and Deep model. Therefore,\n        it requires the `pred_dim` parameter.\n\n    Attributes\n    -----------\n    wide_linear: nn.Module\n        the linear layer that comprises the wide branch of the model\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.models import Wide\n    &gt;&gt;&gt; X = torch.empty(4, 4).random_(20)\n    &gt;&gt;&gt; wide = Wide(input_dim=int(X.max().item()), pred_dim=1)\n    &gt;&gt;&gt; out = wide(X)\n    \"\"\"\n\n    @alias(\"pred_dim\", [\"pred_size\", \"num_class\"])\n    def __init__(self, input_dim: int, pred_dim: int = 1):\n        super(Wide, self).__init__()\n\n        self.input_dim = input_dim\n        self.pred_dim = pred_dim\n\n        # Embeddings: val + 1 because 0 is reserved for padding/unseen cateogories.\n        self.wide_linear = nn.Embedding(input_dim + 1, pred_dim, padding_idx=0)\n        # (Sum(Embedding) + bias) is equivalent to (OneHotVector + Linear)\n        self.bias = nn.Parameter(torch.zeros(pred_dim))\n        self._reset_parameters()\n\n    def _reset_parameters(self) -&gt; None:\n        r\"\"\"initialize Embedding and bias like nn.Linear. See [original\n        implementation](https://pytorch.org/docs/stable/_modules/torch/nn/modules/linear.html#Linear).\n        \"\"\"\n        nn.init.kaiming_uniform_(self.wide_linear.weight, a=math.sqrt(5))\n        fan_in, _ = nn.init._calculate_fan_in_and_fan_out(self.wide_linear.weight)\n        bound = 1 / math.sqrt(fan_in)\n        nn.init.uniform_(self.bias, -bound, bound)\n\n    def forward(self, X: Tensor) -&gt; Tensor:\n        r\"\"\"Forward pass. Simply connecting the Embedding layer with the ouput\n        neuron(s)\"\"\"\n        out = self.wide_linear(X.long()).sum(dim=1) + self.bias\n        return out\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.linear.wide.Wide.forward","title":"forward","text":"<pre><code>forward(X)\n</code></pre> <p>Forward pass. Simply connecting the Embedding layer with the ouput neuron(s)</p> Source code in <code>pytorch_widedeep/models/tabular/linear/wide.py</code> <pre><code>def forward(self, X: Tensor) -&gt; Tensor:\n    r\"\"\"Forward pass. Simply connecting the Embedding layer with the ouput\n    neuron(s)\"\"\"\n    out = self.wide_linear(X.long()).sum(dim=1) + self.bias\n    return out\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.mlp.tab_mlp.TabMlp","title":"TabMlp","text":"<p>               Bases: <code>BaseTabularModelWithoutAttention</code></p> <p>Defines a <code>TabMlp</code> model that can be used as the <code>deeptabular</code> component of a Wide &amp; Deep model or independently by itself.</p> <p>This class combines embedding representations of the categorical features with numerical (aka continuous) features, embedded or not. These are then passed through a series of dense layers (i.e. a MLP).</p> <p>Most of the parameters for this class are <code>Optional</code> since the use of categorical or continuous is in fact optional (i.e. one can use categorical features only, continuous features only or both).</p> <p>Parameters:</p> Name Type Description Default <code>column_idx</code> <code>Dict[str, int]</code> <p>Dict containing the index of the columns that will be passed through the <code>TabMlp</code> model. Required to slice the tensors. e.g. {'education': 0, 'relationship': 1, 'workclass': 2, ...}.</p> required <code>cat_embed_input</code> <code>Optional[List[Tuple[str, int, int]]]</code> <p>List of Tuples with the column name, number of unique values and embedding dimension. e.g. [(education, 11, 32), ...]</p> <code>None</code> <code>cat_embed_dropout</code> <code>Optional[float]</code> <p>Categorical embeddings dropout. If <code>None</code>, it will default to 0.</p> <code>None</code> <code>use_cat_bias</code> <code>Optional[bool]</code> <p>Boolean indicating if bias will be used for the categorical embeddings. If <code>None</code>, it will default to 'False'.</p> <code>None</code> <code>cat_embed_activation</code> <code>Optional[str]</code> <p>Activation function for the categorical embeddings, if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> <code>None</code> <code>continuous_cols</code> <code>Optional[List[str]]</code> <p>List with the name of the numeric (aka continuous) columns</p> <code>None</code> <code>cont_norm_layer</code> <code>Optional[Literal[batchnorm, layernorm]]</code> <p>Type of normalization layer applied to the continuous features. Options are: 'layernorm' and 'batchnorm'. if <code>None</code>, no normalization layer will be used.</p> <code>None</code> <code>embed_continuous</code> <code>Optional[bool]</code> <p>Boolean indicating if the continuous columns will be embedded using one of the available methods: 'standard', 'periodic' or 'piecewise'. If <code>None</code>, it will default to 'False'. NOTE: This parameter is deprecated and it  will be removed in future releases. Please, use the  <code>embed_continuous_method</code> parameter instead.</p> <code>None</code> <code>embed_continuous_method</code> <code>Optional[Literal[standard, piecewise, periodic]]</code> <p>Method to use to embed the continuous features. Options are: 'standard', 'periodic' or 'piecewise'. The 'standard' embedding method is based on the FT-Transformer implementation presented in the paper: Revisiting Deep Learning Models for Tabular Data. The 'periodic' and_'piecewise'_ methods were presented in the paper: On Embeddings for Numerical Features in Tabular Deep Learning. Please, read the papers for details.</p> <code>None</code> <code>cont_embed_dim</code> <code>Optional[int]</code> <p>Size of the continuous embeddings. If the continuous columns are embedded, <code>cont_embed_dim</code> must be passed.</p> <code>None</code> <code>cont_embed_dropout</code> <code>Optional[float]</code> <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p> <code>None</code> <code>cont_embed_activation</code> <code>Optional[str]</code> <p>Activation function for the continuous embeddings if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported. If <code>None</code>, no activation function will be applied.</p> <code>None</code> <code>quantization_setup</code> <code>Optional[Dict[str, List[float]]]</code> <p>This parameter is used when the 'piecewise' method is used to embed the continuous cols. It is a dict where keys are the name of the continuous columns and values are lists with the boundaries for the quantization of the continuous_cols. See the examples for details. If If the 'piecewise' method is used, this parameter is required.</p> <code>None</code> <code>n_frequencies</code> <code>Optional[int]</code> <p>This is the so called 'k' in their paper On Embeddings for Numerical Features in Tabular Deep Learning, and is the number of 'frequencies' that will be used to represent each continuous column. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>sigma</code> <code>Optional[float]</code> <p>This is the sigma parameter in the paper mentioned when describing the previous parameters and it is used to initialise the 'frequency weights'. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>share_last_layer</code> <code>Optional[bool]</code> <p>This parameter is not present in the before mentioned paper but it is implemented in the official repo. If <code>True</code> the linear layer that turns the frequencies into embeddings will be shared across the continuous columns. If <code>False</code> a different linear layer will be used for each continuous column. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>full_embed_dropout</code> <code>Optional[bool]</code> <p>If <code>True</code>, the full embedding corresponding to a column will be masked out/dropout. If <code>None</code>, it will default to <code>False</code>.</p> <code>None</code> <code>mlp_hidden_dims</code> <code>List[int]</code> <p>List with the number of neurons per dense layer in the mlp.</p> <code>[200, 100]</code> <code>mlp_activation</code> <code>str</code> <p>Activation function for the dense layers of the MLP. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> <code>'relu'</code> <code>mlp_dropout</code> <code>Union[float, List[float]]</code> <p>float or List of floats with the dropout between the dense layers. e.g: [0.5,0.5]</p> <code>0.1</code> <code>mlp_batchnorm</code> <code>bool</code> <p>Boolean indicating whether or not batch normalization will be applied to the dense layers</p> <code>False</code> <code>mlp_batchnorm_last</code> <code>bool</code> <p>Boolean indicating whether or not batch normalization will be applied to the last of the dense layers</p> <code>False</code> <code>mlp_linear_first</code> <code>bool</code> <p>Boolean indicating the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code></p> <code>True</code> <p>Attributes:</p> Name Type Description <code>encoder</code> <code>Module</code> <p>mlp model that will receive the concatenation of the embeddings and the continuous columns</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import TabMlp\n&gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n&gt;&gt;&gt; colnames = [\"a\", \"b\", \"c\", \"d\", \"e\"]\n&gt;&gt;&gt; cat_embed_input = [(u, i, j) for u, i, j in zip(colnames[:4], [4] * 4, [8] * 4)]\n&gt;&gt;&gt; column_idx = {k: v for v, k in enumerate(colnames)}\n&gt;&gt;&gt; model = TabMlp(mlp_hidden_dims=[8, 4], column_idx=column_idx, cat_embed_input=cat_embed_input,\n... continuous_cols=[\"e\"])\n&gt;&gt;&gt; out = model(X_tab)\n</code></pre> Source code in <code>pytorch_widedeep/models/tabular/mlp/tab_mlp.py</code> <pre><code>class TabMlp(BaseTabularModelWithoutAttention):\n    r\"\"\"Defines a `TabMlp` model that can be used as the `deeptabular`\n    component of a Wide &amp; Deep model or independently by itself.\n\n    This class combines embedding representations of the categorical features\n    with numerical (aka continuous) features, embedded or not. These are then\n    passed through a series of dense layers (i.e. a MLP).\n\n    Most of the parameters for this class are `Optional` since the use of\n    categorical or continuous is in fact optional (i.e. one can use\n    categorical features only, continuous features only or both).\n\n    Parameters\n    ----------\n    column_idx: Dict\n        Dict containing the index of the columns that will be passed through\n        the `TabMlp` model. Required to slice the tensors. e.g. _{'education':\n        0, 'relationship': 1, 'workclass': 2, ...}_.\n    cat_embed_input: List, Optional, default = None\n        List of Tuples with the column name, number of unique values and\n        embedding dimension. e.g. _[(education, 11, 32), ...]_\n    cat_embed_dropout: float, Optional, default = None\n        Categorical embeddings dropout. If `None`, it will default\n        to 0.\n    use_cat_bias: bool, Optional, default = None,\n        Boolean indicating if bias will be used for the categorical embeddings.\n        If `None`, it will default to 'False'.\n    cat_embed_activation: Optional, str, default = None,\n        Activation function for the categorical embeddings, if any. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported\n    continuous_cols: List, Optional, default = None\n        List with the name of the numeric (aka continuous) columns\n    cont_norm_layer: str, Optional, default =  None\n        Type of normalization layer applied to the continuous features.\n        Options are: _'layernorm'_ and _'batchnorm'_. if `None`, no\n        normalization layer will be used.\n    embed_continuous: bool, Optional, default = None,\n        Boolean indicating if the continuous columns will be embedded using\n        one of the available methods: _'standard'_, _'periodic'_\n        or _'piecewise'_. If `None`, it will default to 'False'.&lt;br/&gt;\n        :information_source: **NOTE**: This parameter is deprecated and it\n         will be removed in future releases. Please, use the\n         `embed_continuous_method` parameter instead.\n    embed_continuous_method: Optional, str, default = None,\n        Method to use to embed the continuous features. Options are:\n        _'standard'_, _'periodic'_ or _'piecewise'_. The _'standard'_\n        embedding method is based on the FT-Transformer implementation\n        presented in the paper: [Revisiting Deep Learning Models for\n        Tabular Data](https://arxiv.org/abs/2106.11959v5). The _'periodic'_\n        and_'piecewise'_ methods were presented in the paper: [On Embeddings for\n        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556).\n        Please, read the papers for details.\n    cont_embed_dim: int, Optional, default = None,\n        Size of the continuous embeddings. If the continuous columns are\n        embedded, `cont_embed_dim` must be passed.\n    cont_embed_dropout: float, Optional, default = None,\n        Dropout for the continuous embeddings. If `None`, it will default to 0.0\n    cont_embed_activation: Optional, str, default = None,\n        Activation function for the continuous embeddings if any. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported.\n        If `None`, no activation function will be applied.\n    quantization_setup: Dict[str, List[float]], Optional, default = None,\n        This parameter is used when the _'piecewise'_ method is used to embed\n        the continuous cols. It is a dict where keys are the name of the continuous\n        columns and values are lists with the boundaries for the quantization\n        of the continuous_cols. See the examples for details. If\n        If the _'piecewise'_ method is used, this parameter is required.\n    n_frequencies: int, Optional, default = None,\n        This is the so called _'k'_ in their paper [On Embeddings for\n        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556),\n        and is the number of 'frequencies' that will be used to represent each\n        continuous column. See their Eq 2 in the paper for details. If\n        the _'periodic'_ method is used, this parameter is required.\n    sigma: float, Optional, default = None,\n        This is the sigma parameter in the paper mentioned when describing the\n        previous parameters and it is used to initialise the 'frequency\n        weights'. See their Eq 2 in the paper for details. If\n        the _'periodic'_ method is used, this parameter is required.\n    share_last_layer: bool, Optional, default = None,\n        This parameter is not present in the before mentioned paper but it is implemented in\n        the [official repo](https://github.com/yandex-research/rtdl-num-embeddings/tree/main).\n        If `True` the linear layer that turns the frequencies into embeddings\n        will be shared across the continuous columns. If `False` a different\n        linear layer will be used for each continuous column.\n        If the _'periodic'_ method is used, this parameter is required.\n    full_embed_dropout: bool, Optional, default = None,\n        If `True`, the full embedding corresponding to a column will be masked\n        out/dropout. If `None`, it will default to `False`.\n    mlp_hidden_dims: List, default = [200, 100]\n        List with the number of neurons per dense layer in the mlp.\n    mlp_activation: str, default = \"relu\"\n        Activation function for the dense layers of the MLP. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported\n    mlp_dropout: float or List, default = 0.1\n        float or List of floats with the dropout between the dense layers.\n        e.g: _[0.5,0.5]_\n    mlp_batchnorm: bool, default = False\n        Boolean indicating whether or not batch normalization will be applied\n        to the dense layers\n    mlp_batchnorm_last: bool, default = False\n        Boolean indicating whether or not batch normalization will be applied\n        to the last of the dense layers\n    mlp_linear_first: bool, default = False\n        Boolean indicating the order of the operations in the dense\n        layer. If `True: [LIN -&gt; ACT -&gt; BN -&gt; DP]`. If `False: [BN -&gt; DP -&gt;\n        LIN -&gt; ACT]`\n\n    Attributes\n    ----------\n    encoder: nn.Module\n        mlp model that will receive the concatenation of the embeddings and\n        the continuous columns\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.models import TabMlp\n    &gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n    &gt;&gt;&gt; colnames = [\"a\", \"b\", \"c\", \"d\", \"e\"]\n    &gt;&gt;&gt; cat_embed_input = [(u, i, j) for u, i, j in zip(colnames[:4], [4] * 4, [8] * 4)]\n    &gt;&gt;&gt; column_idx = {k: v for v, k in enumerate(colnames)}\n    &gt;&gt;&gt; model = TabMlp(mlp_hidden_dims=[8, 4], column_idx=column_idx, cat_embed_input=cat_embed_input,\n    ... continuous_cols=[\"e\"])\n    &gt;&gt;&gt; out = model(X_tab)\n    \"\"\"\n\n    def __init__(\n        self,\n        column_idx: Dict[str, int],\n        *,\n        cat_embed_input: Optional[List[Tuple[str, int, int]]] = None,\n        cat_embed_dropout: Optional[float] = None,\n        use_cat_bias: Optional[bool] = None,\n        cat_embed_activation: Optional[str] = None,\n        continuous_cols: Optional[List[str]] = None,\n        cont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\n        embed_continuous: Optional[bool] = None,\n        embed_continuous_method: Optional[\n            Literal[\"standard\", \"piecewise\", \"periodic\"]\n        ] = None,\n        cont_embed_dim: Optional[int] = None,\n        cont_embed_dropout: Optional[float] = None,\n        cont_embed_activation: Optional[str] = None,\n        quantization_setup: Optional[Dict[str, List[float]]] = None,\n        n_frequencies: Optional[int] = None,\n        sigma: Optional[float] = None,\n        share_last_layer: Optional[bool] = None,\n        full_embed_dropout: Optional[bool] = None,\n        mlp_hidden_dims: List[int] = [200, 100],\n        mlp_activation: str = \"relu\",\n        mlp_dropout: Union[float, List[float]] = 0.1,\n        mlp_batchnorm: bool = False,\n        mlp_batchnorm_last: bool = False,\n        mlp_linear_first: bool = True,\n    ):\n        super(TabMlp, self).__init__(\n            column_idx=column_idx,\n            cat_embed_input=cat_embed_input,\n            cat_embed_dropout=cat_embed_dropout,\n            use_cat_bias=use_cat_bias,\n            cat_embed_activation=cat_embed_activation,\n            continuous_cols=continuous_cols,\n            cont_norm_layer=cont_norm_layer,\n            embed_continuous=embed_continuous,\n            embed_continuous_method=embed_continuous_method,\n            cont_embed_dim=cont_embed_dim,\n            cont_embed_dropout=cont_embed_dropout,\n            cont_embed_activation=cont_embed_activation,\n            quantization_setup=quantization_setup,\n            n_frequencies=n_frequencies,\n            sigma=sigma,\n            share_last_layer=share_last_layer,\n            full_embed_dropout=full_embed_dropout,\n        )\n\n        self.mlp_hidden_dims = mlp_hidden_dims\n        self.mlp_activation = mlp_activation\n        self.mlp_dropout = mlp_dropout\n        self.mlp_batchnorm = mlp_batchnorm\n        self.mlp_batchnorm_last = mlp_batchnorm_last\n        self.mlp_linear_first = mlp_linear_first\n\n        # Embeddings are instantiated at the base model\n        # Mlp\n        mlp_input_dim = self.cat_out_dim + self.cont_out_dim\n        mlp_hidden_dims = [mlp_input_dim] + mlp_hidden_dims\n        self.encoder = MLP(\n            mlp_hidden_dims,\n            mlp_activation,\n            mlp_dropout,\n            mlp_batchnorm,\n            mlp_batchnorm_last,\n            mlp_linear_first,\n        )\n\n    def forward(self, X: Tensor) -&gt; Tensor:\n        x = self._get_embeddings(X)\n        return self.encoder(x)\n\n    @property\n    def output_dim(self) -&gt; int:\n        r\"\"\"The output dimension of the model. This is a required property\n        neccesary to build the `WideDeep` class\"\"\"\n        return self.mlp_hidden_dims[-1]\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.mlp.tab_mlp.TabMlp.output_dim","title":"output_dim  <code>property</code>","text":"<pre><code>output_dim\n</code></pre> <p>The output dimension of the model. This is a required property neccesary to build the <code>WideDeep</code> class</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.mlp.tab_mlp.TabMlpDecoder","title":"TabMlpDecoder","text":"<p>               Bases: <code>Module</code></p> <p>Companion decoder model for the <code>TabMlp</code> model (which can be considered an encoder itself).</p> <p>This class is designed to be used with the <code>EncoderDecoderTrainer</code> when using self-supervised pre-training (see the corresponding section in the docs). The <code>TabMlpDecoder</code> will receive the output from the MLP and 'reconstruct' the embeddings.</p> <p>Parameters:</p> Name Type Description Default <code>embed_dim</code> <code>int</code> <p>Size of the embeddings tensor that needs to be reconstructed.</p> required <code>mlp_hidden_dims</code> <code>List[int]</code> <p>List with the number of neurons per dense layer in the mlp.</p> <code>[100, 200]</code> <code>mlp_activation</code> <code>str</code> <p>Activation function for the dense layers of the MLP. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> <code>'relu'</code> <code>mlp_dropout</code> <code>Union[float, List[float]]</code> <p>float or List of floats with the dropout between the dense layers. e.g: [0.5,0.5]</p> <code>0.1</code> <code>mlp_batchnorm</code> <code>bool</code> <p>Boolean indicating whether or not batch normalization will be applied to the dense layers</p> <code>False</code> <code>mlp_batchnorm_last</code> <code>bool</code> <p>Boolean indicating whether or not batch normalization will be applied to the last of the dense layers</p> <code>False</code> <code>mlp_linear_first</code> <code>bool</code> <p>Boolean indicating the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code></p> <code>True</code> <p>Attributes:</p> Name Type Description <code>decoder</code> <code>Module</code> <p>mlp model that will receive the output of the encoder</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import TabMlpDecoder\n&gt;&gt;&gt; x_inp = torch.rand(3, 8)\n&gt;&gt;&gt; decoder = TabMlpDecoder(embed_dim=32, mlp_hidden_dims=[8,16])\n&gt;&gt;&gt; res = decoder(x_inp)\n&gt;&gt;&gt; res.shape\ntorch.Size([3, 32])\n</code></pre> Source code in <code>pytorch_widedeep/models/tabular/mlp/tab_mlp.py</code> <pre><code>class TabMlpDecoder(nn.Module):\n    r\"\"\"Companion decoder model for the `TabMlp` model (which can be considered\n    an encoder itself).\n\n    This class is designed to be used with the `EncoderDecoderTrainer` when\n    using self-supervised pre-training (see the corresponding section in the\n    docs). The `TabMlpDecoder` will receive the output from the MLP\n    and '_reconstruct_' the embeddings.\n\n    Parameters\n    ----------\n    embed_dim: int\n        Size of the embeddings tensor that needs to be reconstructed.\n    mlp_hidden_dims: List, default = [200, 100]\n        List with the number of neurons per dense layer in the mlp.\n    mlp_activation: str, default = \"relu\"\n        Activation function for the dense layers of the MLP. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported\n    mlp_dropout: float or List, default = 0.1\n        float or List of floats with the dropout between the dense layers.\n        e.g: _[0.5,0.5]_\n    mlp_batchnorm: bool, default = False\n        Boolean indicating whether or not batch normalization will be applied\n        to the dense layers\n    mlp_batchnorm_last: bool, default = False\n        Boolean indicating whether or not batch normalization will be applied\n        to the last of the dense layers\n    mlp_linear_first: bool, default = False\n        Boolean indicating the order of the operations in the dense\n        layer. If `True: [LIN -&gt; ACT -&gt; BN -&gt; DP]`. If `False: [BN -&gt; DP -&gt;\n        LIN -&gt; ACT]`\n\n    Attributes\n    ----------\n    decoder: nn.Module\n        mlp model that will receive the output of the encoder\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.models import TabMlpDecoder\n    &gt;&gt;&gt; x_inp = torch.rand(3, 8)\n    &gt;&gt;&gt; decoder = TabMlpDecoder(embed_dim=32, mlp_hidden_dims=[8,16])\n    &gt;&gt;&gt; res = decoder(x_inp)\n    &gt;&gt;&gt; res.shape\n    torch.Size([3, 32])\n    \"\"\"\n\n    def __init__(\n        self,\n        embed_dim: int,\n        mlp_hidden_dims: List[int] = [100, 200],\n        mlp_activation: str = \"relu\",\n        mlp_dropout: Union[float, List[float]] = 0.1,\n        mlp_batchnorm: bool = False,\n        mlp_batchnorm_last: bool = False,\n        mlp_linear_first: bool = True,\n    ):\n        super(TabMlpDecoder, self).__init__()\n\n        self.embed_dim = embed_dim\n\n        self.mlp_hidden_dims = mlp_hidden_dims\n        self.mlp_activation = mlp_activation\n        self.mlp_dropout = mlp_dropout\n        self.mlp_batchnorm = mlp_batchnorm\n        self.mlp_batchnorm_last = mlp_batchnorm_last\n        self.mlp_linear_first = mlp_linear_first\n\n        self.decoder = MLP(\n            mlp_hidden_dims + [self.embed_dim],\n            mlp_activation,\n            mlp_dropout,\n            mlp_batchnorm,\n            mlp_batchnorm_last,\n            mlp_linear_first,\n        )\n\n    def forward(self, X: Tensor) -&gt; Tensor:\n        return self.decoder(X)\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.resnet.tab_resnet.TabResnet","title":"TabResnet","text":"<p>               Bases: <code>BaseTabularModelWithoutAttention</code></p> <p>Defines a <code>TabResnet</code> model that can be used as the <code>deeptabular</code> component of a Wide &amp; Deep model or independently by itself.</p> <p>This class combines embedding representations of the categorical features with numerical (aka continuous) features, embedded or not. These are then passed through a series of Resnet blocks. See <code>pytorch_widedeep.models.tab_resnet._layers</code> for details on the structure of each block.</p> <p>Most of the parameters for this class are <code>Optional</code> since the use of categorical or continuous is in fact optional (i.e. one can use categorical features only, continuous features only or both).</p> <p>Parameters:</p> Name Type Description Default <code>column_idx</code> <code>Dict[str, int]</code> <p>Dict containing the index of the columns that will be passed through the <code>TabMlp</code> model. Required to slice the tensors. e.g. {'education': 0, 'relationship': 1, 'workclass': 2, ...}.</p> required <code>cat_embed_input</code> <code>Optional[List[Tuple[str, int, int]]]</code> <p>List of Tuples with the column name, number of unique values and embedding dimension. e.g. [(education, 11, 32), ...]</p> <code>None</code> <code>cat_embed_dropout</code> <code>Optional[float]</code> <p>Categorical embeddings dropout. If <code>None</code>, it will default to 0.</p> <code>None</code> <code>use_cat_bias</code> <code>Optional[bool]</code> <p>Boolean indicating if bias will be used for the categorical embeddings. If <code>None</code>, it will default to 'False'.</p> <code>None</code> <code>cat_embed_activation</code> <code>Optional[str]</code> <p>Activation function for the categorical embeddings, if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> <code>None</code> <code>continuous_cols</code> <code>Optional[List[str]]</code> <p>List with the name of the numeric (aka continuous) columns</p> <code>None</code> <code>cont_norm_layer</code> <code>Optional[Literal[batchnorm, layernorm]]</code> <p>Type of normalization layer applied to the continuous features. Options are: 'layernorm' and 'batchnorm'. if <code>None</code>, no normalization layer will be used.</p> <code>None</code> <code>embed_continuous</code> <code>Optional[bool]</code> <p>Boolean indicating if the continuous columns will be embedded using one of the available methods: 'standard', 'periodic' or 'piecewise'. If <code>None</code>, it will default to 'False'. NOTE: This parameter is deprecated and it  will be removed in future releases. Please, use the  <code>embed_continuous_method</code> parameter instead.</p> <code>None</code> <code>embed_continuous_method</code> <code>Optional[Literal[standard, piecewise, periodic]]</code> <p>Method to use to embed the continuous features. Options are: 'standard', 'periodic' or 'piecewise'. The 'standard' embedding method is based on the FT-Transformer implementation presented in the paper: Revisiting Deep Learning Models for Tabular Data. The 'periodic' and_'piecewise'_ methods were presented in the paper: On Embeddings for Numerical Features in Tabular Deep Learning. Please, read the papers for details.</p> <code>None</code> <code>cont_embed_dim</code> <code>Optional[int]</code> <p>Size of the continuous embeddings. If the continuous columns are embedded, <code>cont_embed_dim</code> must be passed.</p> <code>None</code> <code>cont_embed_dropout</code> <code>Optional[float]</code> <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p> <code>None</code> <code>cont_embed_activation</code> <code>Optional[str]</code> <p>Activation function for the continuous embeddings if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported. If <code>None</code>, no activation function will be applied.</p> <code>None</code> <code>quantization_setup</code> <code>Optional[Dict[str, List[float]]]</code> <p>This parameter is used when the 'piecewise' method is used to embed the continuous cols. It is a dict where keys are the name of the continuous columns and values are lists with the boundaries for the quantization of the continuous_cols. See the examples for details. If If the 'piecewise' method is used, this parameter is required.</p> <code>None</code> <code>n_frequencies</code> <code>Optional[int]</code> <p>This is the so called 'k' in their paper On Embeddings for Numerical Features in Tabular Deep Learning, and is the number of 'frequencies' that will be used to represent each continuous column. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>sigma</code> <code>Optional[float]</code> <p>This is the sigma parameter in the paper mentioned when describing the previous parameters and it is used to initialise the 'frequency weights'. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>share_last_layer</code> <code>Optional[bool]</code> <p>This parameter is not present in the before mentioned paper but it is implemented in the official repo. If <code>True</code> the linear layer that turns the frequencies into embeddings will be shared across the continuous columns. If <code>False</code> a different linear layer will be used for each continuous column. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>full_embed_dropout</code> <code>Optional[bool]</code> <p>If <code>True</code>, the full embedding corresponding to a column will be masked out/dropout. If <code>None</code>, it will default to <code>False</code>.</p> <code>None</code> <code>blocks_dims</code> <code>List[int]</code> <p>List of integers that define the input and output units of each block. For example: [200, 100, 100] will generate 2 blocks. The first will receive a tensor of size 200 and output a tensor of size 100, and the second will receive a tensor of size 100 and output a tensor of size 100. See <code>pytorch_widedeep.models.tab_resnet._layers</code> for details on the structure of each block.</p> <code>[200, 100, 100]</code> <code>blocks_dropout</code> <code>float</code> <p>Block's internal dropout.</p> <code>0.1</code> <code>simplify_blocks</code> <code>bool</code> <p>Boolean indicating if the simplest possible residual blocks (<code>X -&gt; [ [LIN, BN, ACT]  + X ]</code>) will be used instead of a standard one (<code>X -&gt; [ [LIN1, BN1, ACT1] -&gt; [LIN2, BN2]  + X ]</code>).</p> <code>False</code> <code>mlp_hidden_dims</code> <code>Optional[List[int]]</code> <p>List with the number of neurons per dense layer in the MLP. e.g: [64, 32]. If <code>None</code> the  output of the Resnet Blocks will be connected directly to the output neuron(s).</p> <code>None</code> <code>mlp_activation</code> <code>Optional[str]</code> <p>Activation function for the dense layers of the MLP. Currently 'tanh', 'relu', 'leaky'_relu' and _'gelu' are supported. If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to 'relu'.</p> <code>None</code> <code>mlp_dropout</code> <code>Optional[float]</code> <p>float with the dropout between the dense layers of the MLP. If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to 0.0.</p> <code>None</code> <code>mlp_batchnorm</code> <code>Optional[bool]</code> <p>Boolean indicating whether or not batch normalization will be applied to the dense layers If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to False.</p> <code>None</code> <code>mlp_batchnorm_last</code> <code>Optional[bool]</code> <p>Boolean indicating whether or not batch normalization will be applied to the last of the dense layers If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to False.</p> <code>None</code> <code>mlp_linear_first</code> <code>Optional[bool]</code> <p>Boolean indicating the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code> If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to <code>True</code>.</p> <code>None</code> <p>Attributes:</p> Name Type Description <code>encoder</code> <code>Module</code> <p>deep dense Resnet model that will receive the concatenation of the embeddings and the continuous columns</p> <code>mlp</code> <code>Module</code> <p>if <code>mlp_hidden_dims</code> is <code>True</code>, this attribute will be an mlp model that will receive the results of the concatenation of the embeddings and the continuous columns -- if present --.</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import TabResnet\n&gt;&gt;&gt; X_deep = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n&gt;&gt;&gt; colnames = ['a', 'b', 'c', 'd', 'e']\n&gt;&gt;&gt; cat_embed_input = [(u,i,j) for u,i,j in zip(colnames[:4], [4]*4, [8]*4)]\n&gt;&gt;&gt; column_idx = {k:v for v,k in enumerate(colnames)}\n&gt;&gt;&gt; model = TabResnet(blocks_dims=[16,4], column_idx=column_idx, cat_embed_input=cat_embed_input,\n... continuous_cols = ['e'])\n&gt;&gt;&gt; out = model(X_deep)\n</code></pre> Source code in <code>pytorch_widedeep/models/tabular/resnet/tab_resnet.py</code> <pre><code>class TabResnet(BaseTabularModelWithoutAttention):\n    r\"\"\"Defines a `TabResnet` model that can be used as the `deeptabular`\n    component of a Wide &amp; Deep model or independently by itself.\n\n    This class combines embedding representations of the categorical features\n    with numerical (aka continuous) features, embedded or not. These are then\n    passed through a series of Resnet blocks. See\n    `pytorch_widedeep.models.tab_resnet._layers` for details on the\n    structure of each block.\n\n    Most of the parameters for this class are `Optional` since the use of\n    categorical or continuous is in fact optional (i.e. one can use\n    categorical features only, continuous features only or both).\n\n    Parameters\n    ----------\n    column_idx: Dict\n        Dict containing the index of the columns that will be passed through\n        the `TabMlp` model. Required to slice the tensors. e.g. _{'education':\n        0, 'relationship': 1, 'workclass': 2, ...}_.\n    cat_embed_input: List, Optional, default = None\n        List of Tuples with the column name, number of unique values and\n        embedding dimension. e.g. _[(education, 11, 32), ...]_\n    cat_embed_dropout: float, Optional, default = None\n        Categorical embeddings dropout. If `None`, it will default\n        to 0.\n    use_cat_bias: bool, Optional, default = None,\n        Boolean indicating if bias will be used for the categorical embeddings.\n        If `None`, it will default to 'False'.\n    cat_embed_activation: Optional, str, default = None,\n        Activation function for the categorical embeddings, if any. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported\n    continuous_cols: List, Optional, default = None\n        List with the name of the numeric (aka continuous) columns\n    cont_norm_layer: str, Optional, default =  None\n        Type of normalization layer applied to the continuous features.\n        Options are: _'layernorm'_ and _'batchnorm'_. if `None`, no\n        normalization layer will be used.\n    embed_continuous: bool, Optional, default = None,\n        Boolean indicating if the continuous columns will be embedded using\n        one of the available methods: _'standard'_, _'periodic'_\n        or _'piecewise'_. If `None`, it will default to 'False'.&lt;br/&gt;\n        :information_source: **NOTE**: This parameter is deprecated and it\n         will be removed in future releases. Please, use the\n         `embed_continuous_method` parameter instead.\n    embed_continuous_method: Optional, str, default = None,\n        Method to use to embed the continuous features. Options are:\n        _'standard'_, _'periodic'_ or _'piecewise'_. The _'standard'_\n        embedding method is based on the FT-Transformer implementation\n        presented in the paper: [Revisiting Deep Learning Models for\n        Tabular Data](https://arxiv.org/abs/2106.11959v5). The _'periodic'_\n        and_'piecewise'_ methods were presented in the paper: [On Embeddings for\n        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556).\n        Please, read the papers for details.\n    cont_embed_dim: int, Optional, default = None,\n        Size of the continuous embeddings. If the continuous columns are\n        embedded, `cont_embed_dim` must be passed.\n    cont_embed_dropout: float, Optional, default = None,\n        Dropout for the continuous embeddings. If `None`, it will default to 0.0\n    cont_embed_activation: Optional, str, default = None,\n        Activation function for the continuous embeddings if any. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported.\n        If `None`, no activation function will be applied.\n    quantization_setup: Dict[str, List[float]], Optional, default = None,\n        This parameter is used when the _'piecewise'_ method is used to embed\n        the continuous cols. It is a dict where keys are the name of the continuous\n        columns and values are lists with the boundaries for the quantization\n        of the continuous_cols. See the examples for details. If\n        If the _'piecewise'_ method is used, this parameter is required.\n    n_frequencies: int, Optional, default = None,\n        This is the so called _'k'_ in their paper [On Embeddings for\n        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556),\n        and is the number of 'frequencies' that will be used to represent each\n        continuous column. See their Eq 2 in the paper for details. If\n        the _'periodic'_ method is used, this parameter is required.\n    sigma: float, Optional, default = None,\n        This is the sigma parameter in the paper mentioned when describing the\n        previous parameters and it is used to initialise the 'frequency\n        weights'. See their Eq 2 in the paper for details. If\n        the _'periodic'_ method is used, this parameter is required.\n    share_last_layer: bool, Optional, default = None,\n        This parameter is not present in the before mentioned paper but it is implemented in\n        the [official repo](https://github.com/yandex-research/rtdl-num-embeddings/tree/main).\n        If `True` the linear layer that turns the frequencies into embeddings\n        will be shared across the continuous columns. If `False` a different\n        linear layer will be used for each continuous column.\n        If the _'periodic'_ method is used, this parameter is required.\n    full_embed_dropout: bool, Optional, default = None,\n        If `True`, the full embedding corresponding to a column will be masked\n        out/dropout. If `None`, it will default to `False`.\n    blocks_dims: List, default = [200, 100, 100]\n        List of integers that define the input and output units of each block.\n        For example: _[200, 100, 100]_ will generate 2 blocks. The first will\n        receive a tensor of size 200 and output a tensor of size 100, and the\n        second will receive a tensor of size 100 and output a tensor of size\n        100. See `pytorch_widedeep.models.tab_resnet._layers` for\n        details on the structure of each block.\n    blocks_dropout: float, default =  0.1\n        Block's internal dropout.\n    simplify_blocks: bool, default = False,\n        Boolean indicating if the simplest possible residual blocks (`X -&gt; [\n        [LIN, BN, ACT]  + X ]`) will be used instead of a standard one\n        (`X -&gt; [ [LIN1, BN1, ACT1] -&gt; [LIN2, BN2]  + X ]`).\n    mlp_hidden_dims: List, Optional, default = None\n        List with the number of neurons per dense layer in the MLP. e.g:\n        _[64, 32]_. If `None` the  output of the Resnet Blocks will be\n        connected directly to the output neuron(s).\n    mlp_activation: str, Optional, default = None\n        Activation function for the dense layers of the MLP. Currently\n        _'tanh'_, _'relu'_, _'leaky'_relu' and _'gelu'_ are supported.\n        If 'mlp_hidden_dims' is not `None` and this parameter is `None`, it\n        will default to _'relu'_.\n    mlp_dropout: float, Optional, default = None\n        float with the dropout between the dense layers of the MLP.\n        If 'mlp_hidden_dims' is not `None` and this parameter is `None`, it\n        will default to 0.0.\n    mlp_batchnorm: bool, Optional, default = None\n        Boolean indicating whether or not batch normalization will be applied\n        to the dense layers\n        If 'mlp_hidden_dims' is not `None` and this parameter is `None`, it\n        will default to False.\n    mlp_batchnorm_last: bool, Optional, default = None\n        Boolean indicating whether or not batch normalization will be applied\n        to the last of the dense layers\n        If 'mlp_hidden_dims' is not `None` and this parameter is `None`, it\n        will default to False.\n    mlp_linear_first: bool, Optional, default = None\n        Boolean indicating the order of the operations in the dense\n        layer. If `True: [LIN -&gt; ACT -&gt; BN -&gt; DP]`. If `False: [BN -&gt; DP -&gt;\n        LIN -&gt; ACT]`\n        If 'mlp_hidden_dims' is not `None` and this parameter is `None`, it\n        will default to `True`.\n\n    Attributes\n    ----------\n    encoder: nn.Module\n        deep dense Resnet model that will receive the concatenation of the\n        embeddings and the continuous columns\n    mlp: nn.Module\n        if `mlp_hidden_dims` is `True`, this attribute will be an mlp\n        model that will receive the results of the concatenation of the\n        embeddings and the continuous columns -- if present --.\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.models import TabResnet\n    &gt;&gt;&gt; X_deep = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n    &gt;&gt;&gt; colnames = ['a', 'b', 'c', 'd', 'e']\n    &gt;&gt;&gt; cat_embed_input = [(u,i,j) for u,i,j in zip(colnames[:4], [4]*4, [8]*4)]\n    &gt;&gt;&gt; column_idx = {k:v for v,k in enumerate(colnames)}\n    &gt;&gt;&gt; model = TabResnet(blocks_dims=[16,4], column_idx=column_idx, cat_embed_input=cat_embed_input,\n    ... continuous_cols = ['e'])\n    &gt;&gt;&gt; out = model(X_deep)\n    \"\"\"\n\n    def __init__(\n        self,\n        column_idx: Dict[str, int],\n        *,\n        cat_embed_input: Optional[List[Tuple[str, int, int]]] = None,\n        cat_embed_dropout: Optional[float] = None,\n        use_cat_bias: Optional[bool] = None,\n        cat_embed_activation: Optional[str] = None,\n        continuous_cols: Optional[List[str]] = None,\n        cont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\n        embed_continuous: Optional[bool] = None,\n        embed_continuous_method: Optional[\n            Literal[\"standard\", \"piecewise\", \"periodic\"]\n        ] = None,\n        cont_embed_dim: Optional[int] = None,\n        cont_embed_dropout: Optional[float] = None,\n        cont_embed_activation: Optional[str] = None,\n        quantization_setup: Optional[Dict[str, List[float]]] = None,\n        n_frequencies: Optional[int] = None,\n        sigma: Optional[float] = None,\n        share_last_layer: Optional[bool] = None,\n        full_embed_dropout: Optional[bool] = None,\n        blocks_dims: List[int] = [200, 100, 100],\n        blocks_dropout: float = 0.1,\n        simplify_blocks: bool = False,\n        mlp_hidden_dims: Optional[List[int]] = None,\n        mlp_activation: Optional[str] = None,\n        mlp_dropout: Optional[float] = None,\n        mlp_batchnorm: Optional[bool] = None,\n        mlp_batchnorm_last: Optional[bool] = None,\n        mlp_linear_first: Optional[bool] = None,\n    ):\n        super(TabResnet, self).__init__(\n            column_idx=column_idx,\n            cat_embed_input=cat_embed_input,\n            cat_embed_dropout=cat_embed_dropout,\n            use_cat_bias=use_cat_bias,\n            cat_embed_activation=cat_embed_activation,\n            continuous_cols=continuous_cols,\n            cont_norm_layer=cont_norm_layer,\n            embed_continuous=embed_continuous,\n            embed_continuous_method=embed_continuous_method,\n            cont_embed_dim=cont_embed_dim,\n            cont_embed_dropout=cont_embed_dropout,\n            cont_embed_activation=cont_embed_activation,\n            quantization_setup=quantization_setup,\n            n_frequencies=n_frequencies,\n            sigma=sigma,\n            share_last_layer=share_last_layer,\n            full_embed_dropout=full_embed_dropout,\n        )\n\n        if len(blocks_dims) &lt; 2:\n            raise ValueError(\n                \"'blocks' must contain at least two elements, e.g. [256, 128]\"\n            )\n\n        self.blocks_dims = blocks_dims\n        self.blocks_dropout = blocks_dropout\n        self.simplify_blocks = simplify_blocks\n\n        self.mlp_hidden_dims = mlp_hidden_dims\n        self.mlp_activation = mlp_activation\n        self.mlp_dropout = mlp_dropout\n        self.mlp_batchnorm = mlp_batchnorm\n        self.mlp_batchnorm_last = mlp_batchnorm_last\n        self.mlp_linear_first = mlp_linear_first\n\n        # Embeddings are instantiated at the base model\n\n        # Resnet\n        dense_resnet_input_dim = self.cat_out_dim + self.cont_out_dim\n        self.encoder = DenseResnet(\n            dense_resnet_input_dim, blocks_dims, blocks_dropout, self.simplify_blocks\n        )\n\n        # Mlp: adding an MLP on top of the Resnet blocks is optional and\n        # therefore all related params are optional\n        if self.mlp_hidden_dims is not None:\n            self.mlp = MLP(\n                d_hidden=[self.blocks_dims[-1]] + self.mlp_hidden_dims,\n                activation=(\n                    \"relu\" if self.mlp_activation is None else self.mlp_activation\n                ),\n                dropout=0.0 if self.mlp_dropout is None else self.mlp_dropout,\n                batchnorm=False if self.mlp_batchnorm is None else self.mlp_batchnorm,\n                batchnorm_last=(\n                    False\n                    if self.mlp_batchnorm_last is None\n                    else self.mlp_batchnorm_last\n                ),\n                linear_first=(\n                    True if self.mlp_linear_first is None else self.mlp_linear_first\n                ),\n            )\n        else:\n            self.mlp = None\n\n    def forward(self, X: Tensor) -&gt; Tensor:\n        x = self._get_embeddings(X)\n        x = self.encoder(x)\n        if self.mlp is not None:\n            x = self.mlp(x)\n        return x\n\n    @property\n    def output_dim(self) -&gt; int:\n        r\"\"\"The output dimension of the model. This is a required property\n        neccesary to build the `WideDeep` class\n        \"\"\"\n        return (\n            self.mlp_hidden_dims[-1]\n            if self.mlp_hidden_dims is not None\n            else self.blocks_dims[-1]\n        )\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.resnet.tab_resnet.TabResnet.output_dim","title":"output_dim  <code>property</code>","text":"<pre><code>output_dim\n</code></pre> <p>The output dimension of the model. This is a required property neccesary to build the <code>WideDeep</code> class</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.resnet.tab_resnet.TabResnetDecoder","title":"TabResnetDecoder","text":"<p>               Bases: <code>Module</code></p> <p>Companion decoder model for the <code>TabResnet</code> model (which can be considered an encoder itself)</p> <p>This class is designed to be used with the <code>EncoderDecoderTrainer</code> when using self-supervised pre-training (see the corresponding section in the docs). This class will receive the output from the ResNet blocks or the MLP(if present) and 'reconstruct' the embeddings.</p> <p>Parameters:</p> Name Type Description Default <code>embed_dim</code> <code>int</code> <p>Size of the embeddings tensor to be reconstructed.</p> required <code>blocks_dims</code> <code>List[int]</code> <p>List of integers that define the input and output units of each block. For example: [200, 100, 100] will generate 2 blocks. The first will receive a tensor of size 200 and output a tensor of size 100, and the second will receive a tensor of size 100 and output a tensor of size 100. See <code>pytorch_widedeep.models.tab_resnet._layers</code> for details on the structure of each block.</p> <code>[100, 100, 200]</code> <code>blocks_dropout</code> <code>float</code> <p>Block's internal dropout.</p> <code>0.1</code> <code>simplify_blocks</code> <code>bool</code> <p>Boolean indicating if the simplest possible residual blocks (<code>X -&gt; [ [LIN, BN, ACT]  + X ]</code>) will be used instead of a standard one (<code>X -&gt; [ [LIN1, BN1, ACT1] -&gt; [LIN2, BN2]  + X ]</code>).</p> <code>False</code> <code>mlp_hidden_dims</code> <code>Optional[List[int]]</code> <p>List with the number of neurons per dense layer in the MLP. e.g: [64, 32]. If <code>None</code> the  output of the Resnet Blocks will be connected directly to the output neuron(s).</p> <code>None</code> <code>mlp_activation</code> <code>Optional[str]</code> <p>Activation function for the dense layers of the MLP. Currently 'tanh', 'relu', 'leaky'_relu' and _'gelu' are supported. If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to 'relu'.</p> <code>None</code> <code>mlp_dropout</code> <code>Optional[float]</code> <p>float with the dropout between the dense layers of the MLP. If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to 0.0.</p> <code>None</code> <code>mlp_batchnorm</code> <code>Optional[bool]</code> <p>Boolean indicating whether or not batch normalization will be applied to the dense layers If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to False.</p> <code>None</code> <code>mlp_batchnorm_last</code> <code>Optional[bool]</code> <p>Boolean indicating whether or not batch normalization will be applied to the last of the dense layers If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to False.</p> <code>None</code> <code>mlp_linear_first</code> <code>Optional[bool]</code> <p>Boolean indicating the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code> If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to <code>True</code>.</p> <code>None</code> <p>Attributes:</p> Name Type Description <code>decoder</code> <code>Module</code> <p>deep dense Resnet model that will receive the output of the encoder IF <code>mlp_hidden_dims</code> is None</p> <code>mlp</code> <code>Module</code> <p>if <code>mlp_hidden_dims</code> is not None, the overall decoder will consist in an MLP that will receive the output of the encoder followed by the deep dense Resnet.</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import TabResnetDecoder\n&gt;&gt;&gt; x_inp = torch.rand(3, 8)\n&gt;&gt;&gt; decoder = TabResnetDecoder(embed_dim=32, blocks_dims=[8, 16, 16])\n&gt;&gt;&gt; res = decoder(x_inp)\n&gt;&gt;&gt; res.shape\ntorch.Size([3, 32])\n</code></pre> Source code in <code>pytorch_widedeep/models/tabular/resnet/tab_resnet.py</code> <pre><code>class TabResnetDecoder(nn.Module):\n    r\"\"\"Companion decoder model for the `TabResnet` model (which can be\n    considered an encoder itself)\n\n    This class is designed to be used with the `EncoderDecoderTrainer` when\n    using self-supervised pre-training (see the corresponding section in the\n    docs). This class will receive the output from the ResNet blocks or the\n    MLP(if present) and '_reconstruct_' the embeddings.\n\n    Parameters\n    ----------\n    embed_dim: int\n        Size of the embeddings tensor to be reconstructed.\n    blocks_dims: List, default = [200, 100, 100]\n        List of integers that define the input and output units of each block.\n        For example: _[200, 100, 100]_ will generate 2 blocks. The first will\n        receive a tensor of size 200 and output a tensor of size 100, and the\n        second will receive a tensor of size 100 and output a tensor of size\n        100. See `pytorch_widedeep.models.tab_resnet._layers` for\n        details on the structure of each block.\n    blocks_dropout: float, default =  0.1\n        Block's internal dropout.\n    simplify_blocks: bool, default = False,\n        Boolean indicating if the simplest possible residual blocks (`X -&gt; [\n        [LIN, BN, ACT]  + X ]`) will be used instead of a standard one\n        (`X -&gt; [ [LIN1, BN1, ACT1] -&gt; [LIN2, BN2]  + X ]`).\n    mlp_hidden_dims: List, Optional, default = None\n        List with the number of neurons per dense layer in the MLP. e.g:\n        _[64, 32]_. If `None` the  output of the Resnet Blocks will be\n        connected directly to the output neuron(s).\n    mlp_activation: str, Optional, default = None\n        Activation function for the dense layers of the MLP. Currently\n        _'tanh'_, _'relu'_, _'leaky'_relu' and _'gelu'_ are supported.\n        If 'mlp_hidden_dims' is not `None` and this parameter is `None`, it\n        will default to _'relu'_.\n    mlp_dropout: float, Optional, default = None\n        float with the dropout between the dense layers of the MLP.\n        If 'mlp_hidden_dims' is not `None` and this parameter is `None`, it\n        will default to 0.0.\n    mlp_batchnorm: bool, Optional, default = None\n        Boolean indicating whether or not batch normalization will be applied\n        to the dense layers\n        If 'mlp_hidden_dims' is not `None` and this parameter is `None`, it\n        will default to False.\n    mlp_batchnorm_last: bool, Optional, default = None\n        Boolean indicating whether or not batch normalization will be applied\n        to the last of the dense layers\n        If 'mlp_hidden_dims' is not `None` and this parameter is `None`, it\n        will default to False.\n    mlp_linear_first: bool, Optional, default = None\n        Boolean indicating the order of the operations in the dense\n        layer. If `True: [LIN -&gt; ACT -&gt; BN -&gt; DP]`. If `False: [BN -&gt; DP -&gt;\n        LIN -&gt; ACT]`\n        If 'mlp_hidden_dims' is not `None` and this parameter is `None`, it\n        will default to `True`.\n\n    Attributes\n    ----------\n    decoder: nn.Module\n        deep dense Resnet model that will receive the output of the encoder IF\n        `mlp_hidden_dims` is None\n    mlp: nn.Module\n        if `mlp_hidden_dims` is not None, the overall decoder will consist\n        in an MLP that will receive the output of the encoder followed by the\n        deep dense Resnet.\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.models import TabResnetDecoder\n    &gt;&gt;&gt; x_inp = torch.rand(3, 8)\n    &gt;&gt;&gt; decoder = TabResnetDecoder(embed_dim=32, blocks_dims=[8, 16, 16])\n    &gt;&gt;&gt; res = decoder(x_inp)\n    &gt;&gt;&gt; res.shape\n    torch.Size([3, 32])\n    \"\"\"\n\n    def __init__(\n        self,\n        embed_dim: int,\n        blocks_dims: List[int] = [100, 100, 200],\n        blocks_dropout: float = 0.1,\n        simplify_blocks: bool = False,\n        mlp_hidden_dims: Optional[List[int]] = None,\n        mlp_activation: Optional[str] = None,\n        mlp_dropout: Optional[float] = None,\n        mlp_batchnorm: Optional[bool] = None,\n        mlp_batchnorm_last: Optional[bool] = None,\n        mlp_linear_first: Optional[bool] = None,\n    ):\n        super(TabResnetDecoder, self).__init__()\n\n        if len(blocks_dims) &lt; 2:\n            raise ValueError(\n                \"'blocks' must contain at least two elements, e.g. [256, 128]\"\n            )\n\n        self.embed_dim = embed_dim\n\n        self.blocks_dims = blocks_dims\n        self.blocks_dropout = blocks_dropout\n        self.simplify_blocks = simplify_blocks\n\n        self.mlp_hidden_dims = mlp_hidden_dims\n        self.mlp_activation = mlp_activation\n        self.mlp_dropout = mlp_dropout\n        self.mlp_batchnorm = mlp_batchnorm\n        self.mlp_batchnorm_last = mlp_batchnorm_last\n        self.mlp_linear_first = mlp_linear_first\n\n        if self.mlp_hidden_dims is not None:\n            self.mlp = MLP(\n                d_hidden=[self.mlp_first_hidden_dim] + self.mlp_hidden_dims,\n                activation=(\n                    \"relu\" if self.mlp_activation is None else self.mlp_activation\n                ),\n                dropout=0.0 if self.mlp_dropout is None else self.mlp_dropout,\n                batchnorm=False if self.mlp_batchnorm is None else self.mlp_batchnorm,\n                batchnorm_last=(\n                    False\n                    if self.mlp_batchnorm_last is None\n                    else self.mlp_batchnorm_last\n                ),\n                linear_first=(\n                    True if self.mlp_linear_first is None else self.mlp_linear_first\n                ),\n            )\n            self.decoder = DenseResnet(\n                self.mlp_hidden_dims[-1],\n                blocks_dims,\n                blocks_dropout,\n                self.simplify_blocks,\n            )\n        else:\n            self.mlp = None\n            self.decoder = DenseResnet(\n                blocks_dims[0], blocks_dims, blocks_dropout, self.simplify_blocks\n            )\n\n        self.reconstruction_layer = nn.Linear(blocks_dims[-1], embed_dim, bias=False)\n\n    def forward(self, X: Tensor) -&gt; Tensor:\n        x = self.mlp(X) if self.mlp is not None else X\n        return self.reconstruction_layer(self.decoder(x))\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.tabnet.tab_net.TabNet","title":"TabNet","text":"<p>               Bases: <code>BaseTabularModelWithoutAttention</code></p> <p>Defines a TabNet model that can be used as the <code>deeptabular</code> component of a Wide &amp; Deep model or independently by itself.</p> <p>The implementation in this library is fully based on that here by the dreamquark-ai team, simply adapted so that it can work within the <code>WideDeep</code> frame. Therefore, ALL CREDIT TO THE DREAMQUARK-AI TEAM.</p> <p>Parameters:</p> Name Type Description Default <code>column_idx</code> <code>Dict[str, int]</code> <p>Dict containing the index of the columns that will be passed through the <code>TabMlp</code> model. Required to slice the tensors. e.g. {'education': 0, 'relationship': 1, 'workclass': 2, ...}.</p> required <code>cat_embed_input</code> <code>Optional[List[Tuple[str, int, int]]]</code> <p>List of Tuples with the column name, number of unique values and embedding dimension. e.g. [(education, 11, 32), ...]</p> <code>None</code> <code>cat_embed_dropout</code> <code>Optional[float]</code> <p>Categorical embeddings dropout. If <code>None</code>, it will default to 0.</p> <code>None</code> <code>use_cat_bias</code> <code>Optional[bool]</code> <p>Boolean indicating if bias will be used for the categorical embeddings. If <code>None</code>, it will default to 'False'.</p> <code>None</code> <code>cat_embed_activation</code> <code>Optional[str]</code> <p>Activation function for the categorical embeddings, if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> <code>None</code> <code>continuous_cols</code> <code>Optional[List[str]]</code> <p>List with the name of the numeric (aka continuous) columns</p> <code>None</code> <code>cont_norm_layer</code> <code>Optional[Literal[batchnorm, layernorm]]</code> <p>Type of normalization layer applied to the continuous features. Options are: 'layernorm' and 'batchnorm'. if <code>None</code>, no normalization layer will be used.</p> <code>None</code> <code>embed_continuous</code> <code>Optional[bool]</code> <p>Boolean indicating if the continuous columns will be embedded using one of the available methods: 'standard', 'periodic' or 'piecewise'. If <code>None</code>, it will default to 'False'. NOTE: This parameter is deprecated and it  will be removed in future releases. Please, use the  <code>embed_continuous_method</code> parameter instead.</p> <code>None</code> <code>embed_continuous_method</code> <code>Optional[Literal[standard, piecewise, periodic]]</code> <p>Method to use to embed the continuous features. Options are: 'standard', 'periodic' or 'piecewise'. The 'standard' embedding method is based on the FT-Transformer implementation presented in the paper: Revisiting Deep Learning Models for Tabular Data. The 'periodic' and_'piecewise'_ methods were presented in the paper: On Embeddings for Numerical Features in Tabular Deep Learning. Please, read the papers for details.</p> <code>None</code> <code>cont_embed_dim</code> <code>Optional[int]</code> <p>Size of the continuous embeddings. If the continuous columns are embedded, <code>cont_embed_dim</code> must be passed.</p> <code>None</code> <code>cont_embed_dropout</code> <code>Optional[float]</code> <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p> <code>None</code> <code>cont_embed_activation</code> <code>Optional[str]</code> <p>Activation function for the continuous embeddings if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported. If <code>None</code>, no activation function will be applied.</p> <code>None</code> <code>quantization_setup</code> <code>Optional[Dict[str, List[float]]]</code> <p>This parameter is used when the 'piecewise' method is used to embed the continuous cols. It is a dict where keys are the name of the continuous columns and values are lists with the boundaries for the quantization of the continuous_cols. See the examples for details. If If the 'piecewise' method is used, this parameter is required.</p> <code>None</code> <code>n_frequencies</code> <code>Optional[int]</code> <p>This is the so called 'k' in their paper On Embeddings for Numerical Features in Tabular Deep Learning, and is the number of 'frequencies' that will be used to represent each continuous column. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>sigma</code> <code>Optional[float]</code> <p>This is the sigma parameter in the paper mentioned when describing the previous parameters and it is used to initialise the 'frequency weights'. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>share_last_layer</code> <code>Optional[bool]</code> <p>This parameter is not present in the before mentioned paper but it is implemented in the official repo. If <code>True</code> the linear layer that turns the frequencies into embeddings will be shared across the continuous columns. If <code>False</code> a different linear layer will be used for each continuous column. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>full_embed_dropout</code> <code>Optional[bool]</code> <p>If <code>True</code>, the full embedding corresponding to a column will be masked out/dropout. If <code>None</code>, it will default to <code>False</code>.</p> <code>None</code> <code>n_steps</code> <code>int</code> <p>number of decision steps. For a better understanding of the function of <code>n_steps</code> and the upcoming parameters, please see the paper.</p> <code>3</code> <code>step_dim</code> <code>int</code> <p>Step's output dimension. This is the output dimension that <code>WideDeep</code> will collect and connect to the output neuron(s).</p> <code>8</code> <code>attn_dim</code> <code>int</code> <p>Attention dimension</p> <code>8</code> <code>dropout</code> <code>float</code> <p>GLU block's internal dropout</p> <code>0.0</code> <code>n_glu_step_dependent</code> <code>int</code> <p>number of GLU Blocks (<code>[FC -&gt; BN -&gt; GLU]</code>) that are step dependent</p> <code>2</code> <code>n_glu_shared</code> <code>int</code> <p>number of GLU Blocks (<code>[FC -&gt; BN -&gt; GLU]</code>) that will be shared across decision steps</p> <code>2</code> <code>ghost_bn</code> <code>bool</code> <p>Boolean indicating if Ghost Batch Normalization will be used.</p> <code>True</code> <code>virtual_batch_size</code> <code>int</code> <p>Batch size when using Ghost Batch Normalization</p> <code>128</code> <code>momentum</code> <code>float</code> <p>Ghost Batch Normalization's momentum. The dreamquark-ai advises for very low values. However high values are used in the original publication. During our tests higher values lead to better results</p> <code>0.02</code> <code>gamma</code> <code>float</code> <p>Relaxation parameter in the paper. When gamma = 1, a feature is enforced to be used only at one decision step. As gamma increases, more flexibility is provided to use a feature at multiple decision steps</p> <code>1.3</code> <code>epsilon</code> <code>float</code> <p>Float to avoid log(0). Always keep low</p> <code>1e-15</code> <code>mask_type</code> <code>str</code> <p>Mask function to use. Either 'sparsemax' or 'entmax'</p> <code>'sparsemax'</code> <p>Attributes:</p> Name Type Description <code>encoder</code> <code>Module</code> <p>the TabNet encoder. For details see the original publication.</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n&gt;&gt;&gt; colnames = [\"a\", \"b\", \"c\", \"d\", \"e\"]\n&gt;&gt;&gt; cat_embed_input = [(u, i, j) for u, i, j in zip(colnames[:4], [4] * 4, [8] * 4)]\n&gt;&gt;&gt; column_idx = {k: v for v, k in enumerate(colnames)}\n&gt;&gt;&gt; model = TabNet(column_idx=column_idx, cat_embed_input=cat_embed_input, continuous_cols=[\"e\"])\n&gt;&gt;&gt; out = model(X_tab)\n</code></pre> Source code in <code>pytorch_widedeep/models/tabular/tabnet/tab_net.py</code> <pre><code>class TabNet(BaseTabularModelWithoutAttention):\n    r\"\"\"Defines a [TabNet model](https://arxiv.org/abs/1908.07442) that\n    can be used as the `deeptabular` component of a Wide &amp; Deep model or\n    independently by itself.\n\n    The implementation in this library is fully based on that\n    [here](https://github.com/dreamquark-ai/tabnet) by the dreamquark-ai team,\n    simply adapted so that it can work within the `WideDeep` frame.\n    Therefore, **ALL CREDIT TO THE DREAMQUARK-AI TEAM**.\n\n    Parameters\n    ----------\n    column_idx: Dict\n        Dict containing the index of the columns that will be passed through\n        the `TabMlp` model. Required to slice the tensors. e.g. _{'education':\n        0, 'relationship': 1, 'workclass': 2, ...}_.\n    cat_embed_input: List, Optional, default = None\n        List of Tuples with the column name, number of unique values and\n        embedding dimension. e.g. _[(education, 11, 32), ...]_\n    cat_embed_dropout: float, Optional, default = None\n        Categorical embeddings dropout. If `None`, it will default\n        to 0.\n    use_cat_bias: bool, Optional, default = None,\n        Boolean indicating if bias will be used for the categorical embeddings.\n        If `None`, it will default to 'False'.\n    cat_embed_activation: Optional, str, default = None,\n        Activation function for the categorical embeddings, if any. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported\n    continuous_cols: List, Optional, default = None\n        List with the name of the numeric (aka continuous) columns\n    cont_norm_layer: str, Optional, default =  None\n        Type of normalization layer applied to the continuous features.\n        Options are: _'layernorm'_ and _'batchnorm'_. if `None`, no\n        normalization layer will be used.\n    embed_continuous: bool, Optional, default = None,\n        Boolean indicating if the continuous columns will be embedded using\n        one of the available methods: _'standard'_, _'periodic'_\n        or _'piecewise'_. If `None`, it will default to 'False'.&lt;br/&gt;\n        :information_source: **NOTE**: This parameter is deprecated and it\n         will be removed in future releases. Please, use the\n         `embed_continuous_method` parameter instead.\n    embed_continuous_method: Optional, str, default = None,\n        Method to use to embed the continuous features. Options are:\n        _'standard'_, _'periodic'_ or _'piecewise'_. The _'standard'_\n        embedding method is based on the FT-Transformer implementation\n        presented in the paper: [Revisiting Deep Learning Models for\n        Tabular Data](https://arxiv.org/abs/2106.11959v5). The _'periodic'_\n        and_'piecewise'_ methods were presented in the paper: [On Embeddings for\n        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556).\n        Please, read the papers for details.\n    cont_embed_dim: int, Optional, default = None,\n        Size of the continuous embeddings. If the continuous columns are\n        embedded, `cont_embed_dim` must be passed.\n    cont_embed_dropout: float, Optional, default = None,\n        Dropout for the continuous embeddings. If `None`, it will default to 0.0\n    cont_embed_activation: Optional, str, default = None,\n        Activation function for the continuous embeddings if any. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported.\n        If `None`, no activation function will be applied.\n    quantization_setup: Dict[str, List[float]], Optional, default = None,\n        This parameter is used when the _'piecewise'_ method is used to embed\n        the continuous cols. It is a dict where keys are the name of the continuous\n        columns and values are lists with the boundaries for the quantization\n        of the continuous_cols. See the examples for details. If\n        If the _'piecewise'_ method is used, this parameter is required.\n    n_frequencies: int, Optional, default = None,\n        This is the so called _'k'_ in their paper [On Embeddings for\n        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556),\n        and is the number of 'frequencies' that will be used to represent each\n        continuous column. See their Eq 2 in the paper for details. If\n        the _'periodic'_ method is used, this parameter is required.\n    sigma: float, Optional, default = None,\n        This is the sigma parameter in the paper mentioned when describing the\n        previous parameters and it is used to initialise the 'frequency\n        weights'. See their Eq 2 in the paper for details. If\n        the _'periodic'_ method is used, this parameter is required.\n    share_last_layer: bool, Optional, default = None,\n        This parameter is not present in the before mentioned paper but it is implemented in\n        the [official repo](https://github.com/yandex-research/rtdl-num-embeddings/tree/main).\n        If `True` the linear layer that turns the frequencies into embeddings\n        will be shared across the continuous columns. If `False` a different\n        linear layer will be used for each continuous column.\n        If the _'periodic'_ method is used, this parameter is required.\n    full_embed_dropout: bool, Optional, default = None,\n        If `True`, the full embedding corresponding to a column will be masked\n        out/dropout. If `None`, it will default to `False`.\n    n_steps: int, default = 3\n        number of decision steps. For a better understanding of the function\n        of `n_steps` and the upcoming parameters, please see the\n        [paper](https://arxiv.org/abs/1908.07442).\n    step_dim: int, default = 8\n        Step's output dimension. This is the output dimension that\n        `WideDeep` will collect and connect to the output neuron(s).\n    attn_dim: int, default = 8\n        Attention dimension\n    dropout: float, default = 0.0\n        GLU block's internal dropout\n    n_glu_step_dependent: int, default = 2\n        number of GLU Blocks (`[FC -&gt; BN -&gt; GLU]`) that are step dependent\n    n_glu_shared: int, default = 2\n        number of GLU Blocks (`[FC -&gt; BN -&gt; GLU]`) that will be shared\n        across decision steps\n    ghost_bn: bool, default=True\n        Boolean indicating if [Ghost Batch Normalization](https://arxiv.org/abs/1705.08741)\n        will be used.\n    virtual_batch_size: int, default = 128\n        Batch size when using Ghost Batch Normalization\n    momentum: float, default = 0.02\n        Ghost Batch Normalization's momentum. The dreamquark-ai advises for\n        very low values. However high values are used in the original\n        publication. During our tests higher values lead to better results\n    gamma: float, default = 1.3\n        Relaxation parameter in the paper. When gamma = 1, a feature is\n        enforced to be used only at one decision step. As gamma increases,\n        more flexibility is provided to use a feature at multiple decision\n        steps\n    epsilon: float, default = 1e-15\n        Float to avoid log(0). Always keep low\n    mask_type: str, default = \"sparsemax\"\n        Mask function to use. Either _'sparsemax'_ or _'entmax'_\n\n    Attributes\n    ----------\n    encoder: nn.Module\n        the TabNet encoder. For details see the [original publication](https://arxiv.org/abs/1908.07442).\n\n    Examples\n    --------\n    &gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n    &gt;&gt;&gt; colnames = [\"a\", \"b\", \"c\", \"d\", \"e\"]\n    &gt;&gt;&gt; cat_embed_input = [(u, i, j) for u, i, j in zip(colnames[:4], [4] * 4, [8] * 4)]\n    &gt;&gt;&gt; column_idx = {k: v for v, k in enumerate(colnames)}\n    &gt;&gt;&gt; model = TabNet(column_idx=column_idx, cat_embed_input=cat_embed_input, continuous_cols=[\"e\"])\n    &gt;&gt;&gt; out = model(X_tab)\n    \"\"\"\n\n    def __init__(\n        self,\n        column_idx: Dict[str, int],\n        *,\n        cat_embed_input: Optional[List[Tuple[str, int, int]]] = None,\n        cat_embed_dropout: Optional[float] = None,\n        use_cat_bias: Optional[bool] = None,\n        cat_embed_activation: Optional[str] = None,\n        continuous_cols: Optional[List[str]] = None,\n        cont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\n        embed_continuous: Optional[bool] = None,\n        embed_continuous_method: Optional[\n            Literal[\"standard\", \"piecewise\", \"periodic\"]\n        ] = None,\n        cont_embed_dim: Optional[int] = None,\n        cont_embed_dropout: Optional[float] = None,\n        cont_embed_activation: Optional[str] = None,\n        quantization_setup: Optional[Dict[str, List[float]]] = None,\n        n_frequencies: Optional[int] = None,\n        sigma: Optional[float] = None,\n        share_last_layer: Optional[bool] = None,\n        full_embed_dropout: Optional[bool] = None,\n        n_steps: int = 3,\n        step_dim: int = 8,\n        attn_dim: int = 8,\n        dropout: float = 0.0,\n        n_glu_step_dependent: int = 2,\n        n_glu_shared: int = 2,\n        ghost_bn: bool = True,\n        virtual_batch_size: int = 128,\n        momentum: float = 0.02,\n        gamma: float = 1.3,\n        epsilon: float = 1e-15,\n        mask_type: str = \"sparsemax\",\n    ):\n        super(TabNet, self).__init__(\n            column_idx=column_idx,\n            cat_embed_input=cat_embed_input,\n            cat_embed_dropout=cat_embed_dropout,\n            use_cat_bias=use_cat_bias,\n            cat_embed_activation=cat_embed_activation,\n            continuous_cols=continuous_cols,\n            cont_norm_layer=cont_norm_layer,\n            embed_continuous=embed_continuous,\n            embed_continuous_method=embed_continuous_method,\n            cont_embed_dim=cont_embed_dim,\n            cont_embed_dropout=cont_embed_dropout,\n            cont_embed_activation=cont_embed_activation,\n            quantization_setup=quantization_setup,\n            n_frequencies=n_frequencies,\n            sigma=sigma,\n            share_last_layer=share_last_layer,\n            full_embed_dropout=full_embed_dropout,\n        )\n\n        self.n_steps = n_steps\n        self.step_dim = step_dim\n        self.attn_dim = attn_dim\n        self.dropout = dropout\n        self.n_glu_step_dependent = n_glu_step_dependent\n        self.n_glu_shared = n_glu_shared\n        self.ghost_bn = ghost_bn\n        self.virtual_batch_size = virtual_batch_size\n        self.momentum = momentum\n        self.gamma = gamma\n        self.epsilon = epsilon\n        self.mask_type = mask_type\n\n        # Embeddings are instantiated at the base model\n        self.embed_out_dim = self.cat_out_dim + self.cont_out_dim\n\n        # TabNet\n        self.encoder = TabNetEncoder(\n            self.embed_out_dim,\n            n_steps,\n            step_dim,\n            attn_dim,\n            dropout,\n            n_glu_step_dependent,\n            n_glu_shared,\n            ghost_bn,\n            virtual_batch_size,\n            momentum,\n            gamma,\n            epsilon,\n            mask_type,\n        )\n\n    def forward(\n        self, X: Tensor, prior: Optional[Tensor] = None\n    ) -&gt; Tuple[Tensor, Tensor]:\n        x = self._get_embeddings(X)\n        steps_output, M_loss = self.encoder(x, prior)\n        res = torch.sum(torch.stack(steps_output, dim=0), dim=0)\n        return (res, M_loss)\n\n    def forward_masks(self, X: Tensor) -&gt; Tuple[Tensor, Dict[int, Tensor]]:\n        x = self._get_embeddings(X)\n        return self.encoder.forward_masks(x)\n\n    @property\n    def output_dim(self) -&gt; int:\n        r\"\"\"The output dimension of the model. This is a required property\n        neccesary to build the `WideDeep` class\n        \"\"\"\n        return self.step_dim\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.tabnet.tab_net.TabNet.output_dim","title":"output_dim  <code>property</code>","text":"<pre><code>output_dim\n</code></pre> <p>The output dimension of the model. This is a required property neccesary to build the <code>WideDeep</code> class</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.tabnet.tab_net.TabNetDecoder","title":"TabNetDecoder","text":"<p>               Bases: <code>Module</code></p> <p>Companion decoder model for the <code>TabNet</code> model (which can be considered an encoder itself)</p> <p>This class is designed to be used with the <code>EncoderDecoderTrainer</code> when using self-supervised pre-training (see the corresponding section in the docs). This class will receive the output from the <code>TabNet</code> encoder (i.e. the output from the so called 'steps') and 'reconstruct' the embeddings.</p> <p>Parameters:</p> Name Type Description Default <code>embed_dim</code> <code>int</code> <p>Size of the embeddings tensor to be reconstructed.</p> required <code>n_steps</code> <code>int</code> <p>number of decision steps. For a better understanding of the function of <code>n_steps</code> and the upcoming parameters, please see the paper.</p> <code>3</code> <code>step_dim</code> <code>int</code> <p>Step's output dimension. This is the output dimension that <code>WideDeep</code> will collect and connect to the output neuron(s).</p> <code>8</code> <code>dropout</code> <code>float</code> <p>GLU block's internal dropout</p> <code>0.0</code> <code>n_glu_step_dependent</code> <code>int</code> <p>number of GLU Blocks (<code>[FC -&gt; BN -&gt; GLU]</code>) that are step dependent</p> <code>2</code> <code>n_glu_shared</code> <code>int</code> <p>number of GLU Blocks (<code>[FC -&gt; BN -&gt; GLU]</code>) that will be shared across decision steps</p> <code>2</code> <code>ghost_bn</code> <code>bool</code> <p>Boolean indicating if Ghost Batch Normalization will be used.</p> <code>True</code> <code>virtual_batch_size</code> <code>int</code> <p>Batch size when using Ghost Batch Normalization</p> <code>128</code> <code>momentum</code> <code>float</code> <p>Ghost Batch Normalization's momentum. The dreamquark-ai advises for very low values. However high values are used in the original publication. During our tests higher values lead to better results</p> <code>0.02</code> <p>Attributes:</p> Name Type Description <code>decoder</code> <code>Module</code> <p>decoder that will receive the output from the encoder's steps and will reconstruct the embeddings</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import TabNetDecoder\n&gt;&gt;&gt; x_inp = [torch.rand(3, 8), torch.rand(3, 8), torch.rand(3, 8)]\n&gt;&gt;&gt; decoder = TabNetDecoder(embed_dim=32, ghost_bn=False)\n&gt;&gt;&gt; res = decoder(x_inp)\n&gt;&gt;&gt; res.shape\ntorch.Size([3, 32])\n</code></pre> Source code in <code>pytorch_widedeep/models/tabular/tabnet/tab_net.py</code> <pre><code>class TabNetDecoder(nn.Module):\n    r\"\"\"Companion decoder model for the `TabNet` model (which can be\n    considered an encoder itself)\n\n    This class is designed to be used with the `EncoderDecoderTrainer` when\n    using self-supervised pre-training (see the corresponding section in the\n    docs). This class will receive the output from the `TabNet` encoder\n    (i.e. the output from the so called 'steps') and '_reconstruct_' the\n    embeddings.\n\n    Parameters\n    ----------\n    embed_dim: int\n        Size of the embeddings tensor to be reconstructed.\n    n_steps: int, default = 3\n        number of decision steps. For a better understanding of the function\n        of `n_steps` and the upcoming parameters, please see the\n        [paper](https://arxiv.org/abs/1908.07442).\n    step_dim: int, default = 8\n        Step's output dimension. This is the output dimension that\n        `WideDeep` will collect and connect to the output neuron(s).\n    dropout: float, default = 0.0\n        GLU block's internal dropout\n    n_glu_step_dependent: int, default = 2\n        number of GLU Blocks (`[FC -&gt; BN -&gt; GLU]`) that are step dependent\n    n_glu_shared: int, default = 2\n        number of GLU Blocks (`[FC -&gt; BN -&gt; GLU]`) that will be shared\n        across decision steps\n    ghost_bn: bool, default=True\n        Boolean indicating if [Ghost Batch Normalization](https://arxiv.org/abs/1705.08741)\n        will be used.\n    virtual_batch_size: int, default = 128\n        Batch size when using Ghost Batch Normalization\n    momentum: float, default = 0.02\n        Ghost Batch Normalization's momentum. The dreamquark-ai advises for\n        very low values. However high values are used in the original\n        publication. During our tests higher values lead to better results\n\n    Attributes\n    ----------\n    decoder: nn.Module\n        decoder that will receive the output from the encoder's steps and will\n        reconstruct the embeddings\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.models import TabNetDecoder\n    &gt;&gt;&gt; x_inp = [torch.rand(3, 8), torch.rand(3, 8), torch.rand(3, 8)]\n    &gt;&gt;&gt; decoder = TabNetDecoder(embed_dim=32, ghost_bn=False)\n    &gt;&gt;&gt; res = decoder(x_inp)\n    &gt;&gt;&gt; res.shape\n    torch.Size([3, 32])\n    \"\"\"\n\n    def __init__(\n        self,\n        embed_dim: int,\n        n_steps: int = 3,\n        step_dim: int = 8,\n        dropout: float = 0.0,\n        n_glu_step_dependent: int = 2,\n        n_glu_shared: int = 2,\n        ghost_bn: bool = True,\n        virtual_batch_size: int = 128,\n        momentum: float = 0.02,\n    ):\n        super(TabNetDecoder, self).__init__()\n\n        self.n_steps = n_steps\n        self.step_dim = step_dim\n        self.dropout = dropout\n        self.n_glu_step_dependent = n_glu_step_dependent\n        self.n_glu_shared = n_glu_shared\n        self.ghost_bn = ghost_bn\n        self.virtual_batch_size = virtual_batch_size\n        self.momentum = momentum\n\n        shared_layers = nn.ModuleList()\n        for i in range(n_glu_shared):\n            if i == 0:\n                shared_layers.append(nn.Linear(step_dim, 2 * step_dim, bias=False))\n            else:\n                shared_layers.append(nn.Linear(step_dim, 2 * step_dim, bias=False))\n\n        self.decoder = nn.ModuleList()\n        for step in range(n_steps):\n            transformer = FeatTransformer(\n                step_dim,\n                step_dim,\n                dropout,\n                shared_layers,\n                n_glu_step_dependent,\n                ghost_bn,\n                virtual_batch_size,\n                momentum=momentum,\n            )\n            self.decoder.append(transformer)\n\n        self.reconstruction_layer = nn.Linear(step_dim, embed_dim, bias=False)\n        initialize_non_glu(self.reconstruction_layer, step_dim, embed_dim)\n\n    def forward(self, X: List[Tensor]) -&gt; Tensor:\n        out = torch.tensor(0.0)\n        for i, x in enumerate(X):\n            x = self.decoder[i](x)\n            out = torch.add(out, x)\n        out = self.reconstruction_layer(out)\n        return out\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.mlp.context_attention_mlp.ContextAttentionMLP","title":"ContextAttentionMLP","text":"<p>               Bases: <code>BaseTabularModelWithAttention</code></p> <p>Defines a <code>ContextAttentionMLP</code> model that can be used as the <code>deeptabular</code> component of a Wide &amp; Deep model or independently by itself.</p> <p>This class combines embedding representations of the categorical features with numerical (aka continuous) features that are also embedded. These are then passed through a series of attention blocks. Each attention block is comprised by a <code>ContextAttentionEncoder</code>. Such encoder is in part inspired by the attention mechanism described in Hierarchical Attention Networks for Document Classification. See <code>pytorch_widedeep.models.tabular.mlp._attention_layers</code> for details.</p> <p>Most of the parameters for this class are <code>Optional</code> since the use of categorical or continuous is in fact optional (i.e. one can use categorical features only, continuous features only or both).</p> <p>Parameters:</p> Name Type Description Default <code>column_idx</code> <code>Dict[str, int]</code> <p>Dict containing the index of the columns that will be passed through the <code>TabMlp</code> model. Required to slice the tensors. e.g. {'education': 0, 'relationship': 1, 'workclass': 2, ...}.</p> required <code>cat_embed_input</code> <code>Optional[List[Tuple[str, int]]]</code> <p>List of Tuples with the column name and number of unique values and embedding dimension. e.g. [(education, 11), ...]</p> <code>None</code> <code>cat_embed_dropout</code> <code>Optional[float]</code> <p>Categorical embeddings dropout. If <code>None</code>, it will default to 0.</p> <code>None</code> <code>use_cat_bias</code> <code>Optional[bool]</code> <p>Boolean indicating if bias will be used for the categorical embeddings. If <code>None</code>, it will default to 'False'.</p> <code>None</code> <code>cat_embed_activation</code> <code>Optional[str]</code> <p>Activation function for the categorical embeddings, if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> <code>None</code> <code>shared_embed</code> <code>Optional[bool]</code> <p>Boolean indicating if the embeddings will be \"shared\". The idea behind <code>shared_embed</code> is described in the Appendix A in the TabTransformer paper: 'The goal of having column embedding is to enable the model to distinguish the classes in one column from those in the other columns'. In other words, the idea is to let the model learn which column is embedded at the time. See: <code>pytorch_widedeep.models.transformers._layers.SharedEmbeddings</code>.</p> <code>None</code> <code>add_shared_embed</code> <code>Optional[bool]</code> <p>The two embedding sharing strategies are: 1) add the shared embeddings to the column embeddings or 2) to replace the first <code>frac_shared_embed</code> with the shared embeddings. See <code>pytorch_widedeep.models.embeddings_layers.SharedEmbeddings</code> If 'None' is passed, it will default to 'False'.</p> <code>None</code> <code>frac_shared_embed</code> <code>Optional[float]</code> <p>The fraction of embeddings that will be shared (if <code>add_shared_embed = False</code>) by all the different categories for one particular column. If 'None' is passed, it will default to 0.0.</p> <code>None</code> <code>continuous_cols</code> <code>Optional[List[str]]</code> <p>List with the name of the numeric (aka continuous) columns</p> <code>None</code> <code>cont_norm_layer</code> <code>Optional[Literal[batchnorm, layernorm]]</code> <p>Type of normalization layer applied to the continuous features. Options are: 'layernorm' and 'batchnorm'. if <code>None</code>, no normalization layer will be used.</p> <code>None</code> <code>embed_continuous_method</code> <code>Optional[Literal[standard, piecewise, periodic]]</code> <p>Method to use to embed the continuous features. Options are: 'standard', 'periodic' or 'piecewise'. The 'standard' embedding method is based on the FT-Transformer implementation presented in the paper: Revisiting Deep Learning Models for Tabular Data. The 'periodic' and_'piecewise'_ methods were presented in the paper: On Embeddings for Numerical Features in Tabular Deep Learning. Please, read the papers for details.</p> <code>'standard'</code> <code>cont_embed_dropout</code> <code>Optional[float]</code> <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p> <code>None</code> <code>cont_embed_activation</code> <code>Optional[str]</code> <p>Activation function for the continuous embeddings if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported. If <code>None</code>, no activation function will be applied.</p> <code>None</code> <code>quantization_setup</code> <code>Optional[Dict[str, List[float]]]</code> <p>This parameter is used when the 'piecewise' method is used to embed the continuous cols. It is a dict where keys are the name of the continuous columns and values are lists with the boundaries for the quantization of the continuous_cols. See the examples for details. If If the 'piecewise' method is used, this parameter is required.</p> <code>None</code> <code>n_frequencies</code> <code>Optional[int]</code> <p>This is the so called 'k' in their paper On Embeddings for Numerical Features in Tabular Deep Learning, and is the number of 'frequencies' that will be used to represent each continuous column. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>sigma</code> <code>Optional[float]</code> <p>This is the sigma parameter in the paper mentioned when describing the previous parameters and it is used to initialise the 'frequency weights'. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>share_last_layer</code> <code>Optional[bool]</code> <p>This parameter is not present in the before mentioned paper but it is implemented in the official repo. If <code>True</code> the linear layer that turns the frequencies into embeddings will be shared across the continuous columns. If <code>False</code> a different linear layer will be used for each continuous column. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>full_embed_dropout</code> <code>Optional[bool]</code> <p>If <code>True</code>, the full embedding corresponding to a column will be masked out/dropout. If <code>None</code>, it will default to <code>False</code>.</p> <code>None</code> <code>input_dim</code> <code>int</code> <p>The so-called dimension of the model. Is the number of embeddings used to encode the categorical and/or continuous columns</p> <code>32</code> <code>attn_dropout</code> <code>float</code> <p>Dropout for each attention block</p> <code>0.2</code> <code>with_addnorm</code> <code>bool</code> <p>Boolean indicating if residual connections will be used in the attention blocks</p> <code>False</code> <code>attn_activation</code> <code>str</code> <p>String indicating the activation function to be applied to the dense layer in each attention encoder. 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported.</p> <code>'leaky_relu'</code> <code>n_blocks</code> <code>int</code> <p>Number of attention blocks</p> <code>3</code> <p>Attributes:</p> Name Type Description <code>encoder</code> <code>Module</code> <p>Sequence of attention encoders.</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import ContextAttentionMLP\n&gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n&gt;&gt;&gt; colnames = ['a', 'b', 'c', 'd', 'e']\n&gt;&gt;&gt; cat_embed_input = [(u,i,j) for u,i,j in zip(colnames[:4], [4]*4, [8]*4)]\n&gt;&gt;&gt; column_idx = {k:v for v,k in enumerate(colnames)}\n&gt;&gt;&gt; model = ContextAttentionMLP(column_idx=column_idx, cat_embed_input=cat_embed_input, continuous_cols = ['e'])\n&gt;&gt;&gt; out = model(X_tab)\n</code></pre> Source code in <code>pytorch_widedeep/models/tabular/mlp/context_attention_mlp.py</code> <pre><code>class ContextAttentionMLP(BaseTabularModelWithAttention):\n    r\"\"\"Defines a `ContextAttentionMLP` model that can be used as the\n    `deeptabular` component of a Wide &amp; Deep model or independently by\n    itself.\n\n    This class combines embedding representations of the categorical features\n    with numerical (aka continuous) features that are also embedded. These\n    are then passed through a series of attention blocks. Each attention\n    block is comprised by a `ContextAttentionEncoder`. Such encoder is in\n    part inspired by the attention mechanism described in\n    [Hierarchical Attention Networks for Document\n    Classification](https://www.cs.cmu.edu/~./hovy/papers/16HLT-hierarchical-attention-networks.pdf).\n    See `pytorch_widedeep.models.tabular.mlp._attention_layers` for details.\n\n    Most of the parameters for this class are `Optional` since the use of\n    categorical or continuous is in fact optional (i.e. one can use\n    categorical features only, continuous features only or both).\n\n    Parameters\n    ----------\n    column_idx: Dict\n        Dict containing the index of the columns that will be passed through\n        the `TabMlp` model. Required to slice the tensors. e.g. _{'education':\n        0, 'relationship': 1, 'workclass': 2, ...}_.\n    cat_embed_input: List, Optional, default = None\n        List of Tuples with the column name and number of unique values and\n        embedding dimension. e.g. _[(education, 11), ...]_\n    cat_embed_dropout: float, Optional, default = None\n        Categorical embeddings dropout. If `None`, it will default\n        to 0.\n    use_cat_bias: bool, Optional, default = None,\n        Boolean indicating if bias will be used for the categorical embeddings.\n        If `None`, it will default to 'False'.\n    cat_embed_activation: Optional, str, default = None,\n        Activation function for the categorical embeddings, if any. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported\n    shared_embed: bool, Optional, default = None\n        Boolean indicating if the embeddings will be \"shared\". The idea behind `shared_embed` is\n        described in the Appendix A in the [TabTransformer paper](https://arxiv.org/abs/2012.06678):\n        _'The goal of having column embedding is to enable the model to\n        distinguish the classes in one column from those in the other\n        columns'_. In other words, the idea is to let the model learn which\n        column is embedded at the time. See: `pytorch_widedeep.models.transformers._layers.SharedEmbeddings`.\n    add_shared_embed: bool, Optional, default = None\n        The two embedding sharing strategies are: 1) add the shared embeddings\n        to the column embeddings or 2) to replace the first\n        `frac_shared_embed` with the shared embeddings.\n        See `pytorch_widedeep.models.embeddings_layers.SharedEmbeddings`\n        If 'None' is passed, it will default to 'False'.\n    frac_shared_embed: float, Optional, default = None\n        The fraction of embeddings that will be shared (if `add_shared_embed\n        = False`) by all the different categories for one particular\n        column. If 'None' is passed, it will default to 0.0.\n    continuous_cols: List, Optional, default = None\n        List with the name of the numeric (aka continuous) columns\n    cont_norm_layer: str, Optional, default =  None\n        Type of normalization layer applied to the continuous features.\n        Options are: _'layernorm'_ and _'batchnorm'_. if `None`, no\n        normalization layer will be used.\n    embed_continuous_method: Optional, str, default = None,\n        Method to use to embed the continuous features. Options are:\n        _'standard'_, _'periodic'_ or _'piecewise'_. The _'standard'_\n        embedding method is based on the FT-Transformer implementation\n        presented in the paper: [Revisiting Deep Learning Models for\n        Tabular Data](https://arxiv.org/abs/2106.11959v5). The _'periodic'_\n        and_'piecewise'_ methods were presented in the paper: [On Embeddings for\n        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556).\n        Please, read the papers for details.\n    cont_embed_dropout: float, Optional, default = None,\n        Dropout for the continuous embeddings. If `None`, it will default to 0.0\n    cont_embed_activation: Optional, str, default = None,\n        Activation function for the continuous embeddings if any. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported.\n        If `None`, no activation function will be applied.\n    quantization_setup: Dict[str, List[float]], Optional, default = None,\n        This parameter is used when the _'piecewise'_ method is used to embed\n        the continuous cols. It is a dict where keys are the name of the continuous\n        columns and values are lists with the boundaries for the quantization\n        of the continuous_cols. See the examples for details. If\n        If the _'piecewise'_ method is used, this parameter is required.\n    n_frequencies: int, Optional, default = None,\n        This is the so called _'k'_ in their paper [On Embeddings for\n        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556),\n        and is the number of 'frequencies' that will be used to represent each\n        continuous column. See their Eq 2 in the paper for details. If\n        the _'periodic'_ method is used, this parameter is required.\n    sigma: float, Optional, default = None,\n        This is the sigma parameter in the paper mentioned when describing the\n        previous parameters and it is used to initialise the 'frequency\n        weights'. See their Eq 2 in the paper for details. If\n        the _'periodic'_ method is used, this parameter is required.\n    share_last_layer: bool, Optional, default = None,\n        This parameter is not present in the before mentioned paper but it is implemented in\n        the [official repo](https://github.com/yandex-research/rtdl-num-embeddings/tree/main).\n        If `True` the linear layer that turns the frequencies into embeddings\n        will be shared across the continuous columns. If `False` a different\n        linear layer will be used for each continuous column.\n        If the _'periodic'_ method is used, this parameter is required.\n    full_embed_dropout: bool, Optional, default = None,\n        If `True`, the full embedding corresponding to a column will be masked\n        out/dropout. If `None`, it will default to `False`.\n    input_dim: int, default = 32\n        The so-called *dimension of the model*. Is the number of embeddings\n        used to encode the categorical and/or continuous columns\n    attn_dropout: float, default = 0.2\n        Dropout for each attention block\n    with_addnorm: bool = False,\n        Boolean indicating if residual connections will be used in the\n        attention blocks\n    attn_activation: str, default = \"leaky_relu\"\n        String indicating the activation function to be applied to the dense\n        layer in each attention encoder. _'tanh'_, _'relu'_, _'leaky_relu'_\n        and _'gelu'_ are supported.\n    n_blocks: int, default = 3\n        Number of attention blocks\n\n    Attributes\n    ----------\n    encoder: nn.Module\n        Sequence of attention encoders.\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.models import ContextAttentionMLP\n    &gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n    &gt;&gt;&gt; colnames = ['a', 'b', 'c', 'd', 'e']\n    &gt;&gt;&gt; cat_embed_input = [(u,i,j) for u,i,j in zip(colnames[:4], [4]*4, [8]*4)]\n    &gt;&gt;&gt; column_idx = {k:v for v,k in enumerate(colnames)}\n    &gt;&gt;&gt; model = ContextAttentionMLP(column_idx=column_idx, cat_embed_input=cat_embed_input, continuous_cols = ['e'])\n    &gt;&gt;&gt; out = model(X_tab)\n    \"\"\"\n\n    def __init__(\n        self,\n        column_idx: Dict[str, int],\n        *,\n        cat_embed_input: Optional[List[Tuple[str, int]]] = None,\n        cat_embed_dropout: Optional[float] = None,\n        use_cat_bias: Optional[bool] = None,\n        cat_embed_activation: Optional[str] = None,\n        shared_embed: Optional[bool] = None,\n        add_shared_embed: Optional[bool] = None,\n        frac_shared_embed: Optional[float] = None,\n        continuous_cols: Optional[List[str]] = None,\n        cont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\n        embed_continuous_method: Optional[\n            Literal[\"standard\", \"piecewise\", \"periodic\"]\n        ] = \"standard\",\n        cont_embed_dropout: Optional[float] = None,\n        cont_embed_activation: Optional[str] = None,\n        quantization_setup: Optional[Dict[str, List[float]]] = None,\n        n_frequencies: Optional[int] = None,\n        sigma: Optional[float] = None,\n        share_last_layer: Optional[bool] = None,\n        full_embed_dropout: Optional[bool] = None,\n        input_dim: int = 32,\n        attn_dropout: float = 0.2,\n        with_addnorm: bool = False,\n        attn_activation: str = \"leaky_relu\",\n        n_blocks: int = 3,\n    ):\n        super(ContextAttentionMLP, self).__init__(\n            column_idx=column_idx,\n            cat_embed_input=cat_embed_input,\n            cat_embed_dropout=cat_embed_dropout,\n            use_cat_bias=use_cat_bias,\n            cat_embed_activation=cat_embed_activation,\n            shared_embed=shared_embed,\n            add_shared_embed=add_shared_embed,\n            frac_shared_embed=frac_shared_embed,\n            continuous_cols=continuous_cols,\n            cont_norm_layer=cont_norm_layer,\n            embed_continuous_method=embed_continuous_method,\n            cont_embed_dropout=cont_embed_dropout,\n            cont_embed_activation=cont_embed_activation,\n            input_dim=input_dim,\n            quantization_setup=quantization_setup,\n            n_frequencies=n_frequencies,\n            sigma=sigma,\n            share_last_layer=share_last_layer,\n            full_embed_dropout=full_embed_dropout,\n        )\n\n        self.attn_dropout = attn_dropout\n        self.with_addnorm = with_addnorm\n        self.attn_activation = attn_activation\n        self.n_blocks = n_blocks\n\n        self.with_cls_token = \"cls_token\" in column_idx\n        self.n_cat = len(cat_embed_input) if cat_embed_input is not None else 0\n        self.n_cont = len(continuous_cols) if continuous_cols is not None else 0\n\n        # Embeddings are instantiated at the base model\n        # Attention Blocks\n        self.encoder = nn.Sequential()\n        for i in range(n_blocks):\n            self.encoder.add_module(\n                \"attention_block\" + str(i),\n                ContextAttentionEncoder(\n                    input_dim,\n                    attn_dropout,\n                    with_addnorm,\n                    attn_activation,\n                ),\n            )\n\n    def forward(self, X: Tensor) -&gt; Tensor:\n        x = self._get_embeddings(X)\n        x = self.encoder(x)\n        if self.with_cls_token:\n            out = x[:, 0, :]\n        else:\n            out = x.flatten(1)\n        return out\n\n    @property\n    def output_dim(self) -&gt; int:\n        r\"\"\"The output dimension of the model. This is a required property\n        neccesary to build the `WideDeep` class\n        \"\"\"\n        return (\n            self.input_dim\n            if self.with_cls_token\n            else ((self.n_cat + self.n_cont) * self.input_dim)\n        )\n\n    @property\n    def attention_weights(self) -&gt; List[Tensor]:\n        r\"\"\"List with the attention weights per block\n\n        The shape of the attention weights is $(N, F)$, where $N$ is the batch\n        size and $F$ is the number of features/columns in the dataset\n        \"\"\"\n        return [blk.attn.attn_weights for blk in self.encoder]\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.mlp.context_attention_mlp.ContextAttentionMLP.output_dim","title":"output_dim  <code>property</code>","text":"<pre><code>output_dim\n</code></pre> <p>The output dimension of the model. This is a required property neccesary to build the <code>WideDeep</code> class</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.mlp.context_attention_mlp.ContextAttentionMLP.attention_weights","title":"attention_weights  <code>property</code>","text":"<pre><code>attention_weights\n</code></pre> <p>List with the attention weights per block</p> <p>The shape of the attention weights is \\((N, F)\\), where \\(N\\) is the batch size and \\(F\\) is the number of features/columns in the dataset</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.mlp.self_attention_mlp.SelfAttentionMLP","title":"SelfAttentionMLP","text":"<p>               Bases: <code>BaseTabularModelWithAttention</code></p> <p>Defines a <code>SelfAttentionMLP</code> model that can be used as the deeptabular component of a Wide &amp; Deep model or independently by itself.</p> <p>This class combines embedding representations of the categorical features with numerical (aka continuous) features that are also embedded. These are then passed through a series of attention blocks. Each attention block is comprised by what we would refer as a simplified <code>SelfAttentionEncoder</code>. See <code>pytorch_widedeep.models.tabular.mlp._attention_layers</code> for details. The reason to use a simplified version of self attention is because we observed that the 'standard' attention mechanism used in the TabTransformer has a notable tendency to overfit.</p> <p>In more detail, this model only uses Q and K (and not V). If we think about it as in terms of text (and intuitively), the Softmax(QK^T) is the attention mechanism that tells us how much, at each position in the input sentence, each word is represented or 'expressed'. We refer to that as 'attention weights'. These attention weighst are normally multiplied by a Value matrix to further strength the focus on the words that each word should be attending to (again, intuitively).</p> <p>In this implementation we skip this last multiplication and instead we multiply the attention weights directly by the input tensor. This is a simplification that we expect is beneficial in terms of avoiding overfitting for tabular data.</p> <p>Most of the parameters for this class are <code>Optional</code> since the use of categorical or continuous is in fact optional (i.e. one can use categorical features only, continuous features only or both).</p> <p>Parameters:</p> Name Type Description Default <code>column_idx</code> <code>Dict[str, int]</code> <p>Dict containing the index of the columns that will be passed through the <code>TabMlp</code> model. Required to slice the tensors. e.g. {'education': 0, 'relationship': 1, 'workclass': 2, ...}.</p> required <code>cat_embed_input</code> <code>Optional[List[Tuple[str, int]]]</code> <p>List of Tuples with the column name and number of unique values and embedding dimension. e.g. [(education, 11), ...]</p> <code>None</code> <code>cat_embed_dropout</code> <code>Optional[float]</code> <p>Categorical embeddings dropout. If <code>None</code>, it will default to 0.</p> <code>None</code> <code>use_cat_bias</code> <code>Optional[bool]</code> <p>Boolean indicating if bias will be used for the categorical embeddings. If <code>None</code>, it will default to 'False'.</p> <code>None</code> <code>cat_embed_activation</code> <code>Optional[str]</code> <p>Activation function for the categorical embeddings, if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> <code>None</code> <code>shared_embed</code> <code>Optional[bool]</code> <p>Boolean indicating if the embeddings will be \"shared\". The idea behind <code>shared_embed</code> is described in the Appendix A in the TabTransformer paper: 'The goal of having column embedding is to enable the model to distinguish the classes in one column from those in the other columns'. In other words, the idea is to let the model learn which column is embedded at the time. See: <code>pytorch_widedeep.models.transformers._layers.SharedEmbeddings</code>.</p> <code>None</code> <code>add_shared_embed</code> <code>Optional[bool]</code> <p>The two embedding sharing strategies are: 1) add the shared embeddings to the column embeddings or 2) to replace the first <code>frac_shared_embed</code> with the shared embeddings. See <code>pytorch_widedeep.models.embeddings_layers.SharedEmbeddings</code> If 'None' is passed, it will default to 'False'.</p> <code>None</code> <code>frac_shared_embed</code> <code>Optional[float]</code> <p>The fraction of embeddings that will be shared (if <code>add_shared_embed = False</code>) by all the different categories for one particular column. If 'None' is passed, it will default to 0.0.</p> <code>None</code> <code>continuous_cols</code> <code>Optional[List[str]]</code> <p>List with the name of the numeric (aka continuous) columns</p> <code>None</code> <code>cont_norm_layer</code> <code>Optional[Literal[batchnorm, layernorm]]</code> <p>Type of normalization layer applied to the continuous features. Options are: 'layernorm' and 'batchnorm'. if <code>None</code>, no normalization layer will be used.</p> <code>None</code> <code>embed_continuous_method</code> <code>Optional[Literal[standard, piecewise, periodic]]</code> <p>Method to use to embed the continuous features. Options are: 'standard', 'periodic' or 'piecewise'. The 'standard' embedding method is based on the FT-Transformer implementation presented in the paper: Revisiting Deep Learning Models for Tabular Data. The 'periodic' and_'piecewise'_ methods were presented in the paper: On Embeddings for Numerical Features in Tabular Deep Learning. Please, read the papers for details.</p> <code>'standard'</code> <code>cont_embed_dropout</code> <code>Optional[float]</code> <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p> <code>None</code> <code>cont_embed_activation</code> <code>Optional[str]</code> <p>Activation function for the continuous embeddings if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported. If <code>None</code>, no activation function will be applied.</p> <code>None</code> <code>quantization_setup</code> <code>Optional[Dict[str, List[float]]]</code> <p>This parameter is used when the 'piecewise' method is used to embed the continuous cols. It is a dict where keys are the name of the continuous columns and values are lists with the boundaries for the quantization of the continuous_cols. See the examples for details. If If the 'piecewise' method is used, this parameter is required.</p> <code>None</code> <code>n_frequencies</code> <code>Optional[int]</code> <p>This is the so called 'k' in their paper On Embeddings for Numerical Features in Tabular Deep Learning, and is the number of 'frequencies' that will be used to represent each continuous column. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>sigma</code> <code>Optional[float]</code> <p>This is the sigma parameter in the paper mentioned when describing the previous parameters and it is used to initialise the 'frequency weights'. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>share_last_layer</code> <code>Optional[bool]</code> <p>This parameter is not present in the before mentioned paper but it is implemented in the official repo. If <code>True</code> the linear layer that turns the frequencies into embeddings will be shared across the continuous columns. If <code>False</code> a different linear layer will be used for each continuous column. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>full_embed_dropout</code> <code>Optional[bool]</code> <p>If <code>True</code>, the full embedding corresponding to a column will be masked out/dropout. If <code>None</code>, it will default to <code>False</code>.</p> <code>None</code> <code>input_dim</code> <code>int</code> <p>The so-called dimension of the model. Is the number of embeddings used to encode the categorical and/or continuous columns</p> <code>32</code> <code>attn_dropout</code> <code>float</code> <p>Dropout for each attention block</p> <code>0.2</code> <code>n_heads</code> <code>int</code> <p>Number of attention heads per attention block.</p> <code>8</code> <code>use_bias</code> <code>bool</code> <p>Boolean indicating whether or not to use bias in the Q, K projection layers.</p> <code>False</code> <code>with_addnorm</code> <code>bool</code> <p>Boolean indicating if residual connections will be used in the attention blocks</p> <code>False</code> <code>attn_activation</code> <code>str</code> <p>String indicating the activation function to be applied to the dense layer in each attention encoder. 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported.</p> <code>'leaky_relu'</code> <code>n_blocks</code> <code>int</code> <p>Number of attention blocks</p> <code>3</code> <p>Attributes:</p> Name Type Description <code>cat_and_cont_embed</code> <code>Module</code> <p>This is the module that processes the categorical and continuous columns</p> <code>encoder</code> <code>Module</code> <p>Sequence of attention encoders.</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import SelfAttentionMLP\n&gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n&gt;&gt;&gt; colnames = ['a', 'b', 'c', 'd', 'e']\n&gt;&gt;&gt; cat_embed_input = [(u,i,j) for u,i,j in zip(colnames[:4], [4]*4, [8]*4)]\n&gt;&gt;&gt; column_idx = {k:v for v,k in enumerate(colnames)}\n&gt;&gt;&gt; model = SelfAttentionMLP(column_idx=column_idx, cat_embed_input=cat_embed_input, continuous_cols = ['e'])\n&gt;&gt;&gt; out = model(X_tab)\n</code></pre> Source code in <code>pytorch_widedeep/models/tabular/mlp/self_attention_mlp.py</code> <pre><code>class SelfAttentionMLP(BaseTabularModelWithAttention):\n    r\"\"\"Defines a `SelfAttentionMLP` model that can be used as the\n    deeptabular component of a Wide &amp; Deep model or independently by\n    itself.\n\n    This class combines embedding representations of the categorical features\n    with numerical (aka continuous) features that are also embedded. These\n    are then passed through a series of attention blocks. Each attention\n    block is comprised by what we would refer as a simplified\n    `SelfAttentionEncoder`. See\n    `pytorch_widedeep.models.tabular.mlp._attention_layers` for details. The\n    reason to use a simplified version of self attention is because we\n    observed that the '_standard_' attention mechanism used in the\n    TabTransformer has a notable tendency to overfit.\n\n    In more detail, this model only uses Q and K (and not V). If we think\n    about it as in terms of text (and intuitively), the Softmax(QK^T) is the\n    attention mechanism that tells us how much, at each position in the input\n    sentence, each word is represented or 'expressed'. We refer to that\n    as 'attention weights'. These attention weighst are normally multiplied\n    by a Value matrix to further strength the focus on the words that each\n    word should be attending to (again, intuitively).\n\n    In this implementation we skip this last multiplication and instead we\n    multiply the attention weights directly by the input tensor. This is a\n    simplification that we expect is beneficial in terms of avoiding\n    overfitting for tabular data.\n\n    Most of the parameters for this class are `Optional` since the use of\n    categorical or continuous is in fact optional (i.e. one can use\n    categorical features only, continuous features only or both).\n\n    Parameters\n    ----------\n    column_idx: Dict\n        Dict containing the index of the columns that will be passed through\n        the `TabMlp` model. Required to slice the tensors. e.g. _{'education':\n        0, 'relationship': 1, 'workclass': 2, ...}_.\n    cat_embed_input: List, Optional, default = None\n        List of Tuples with the column name and number of unique values and\n        embedding dimension. e.g. _[(education, 11), ...]_\n    cat_embed_dropout: float, Optional, default = None\n        Categorical embeddings dropout. If `None`, it will default\n        to 0.\n    use_cat_bias: bool, Optional, default = None,\n        Boolean indicating if bias will be used for the categorical embeddings.\n        If `None`, it will default to 'False'.\n    cat_embed_activation: Optional, str, default = None,\n        Activation function for the categorical embeddings, if any. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported\n    shared_embed: bool, Optional, default = None\n        Boolean indicating if the embeddings will be \"shared\". The idea behind `shared_embed` is\n        described in the Appendix A in the [TabTransformer paper](https://arxiv.org/abs/2012.06678):\n        _'The goal of having column embedding is to enable the model to\n        distinguish the classes in one column from those in the other\n        columns'_. In other words, the idea is to let the model learn which\n        column is embedded at the time. See: `pytorch_widedeep.models.transformers._layers.SharedEmbeddings`.\n    add_shared_embed: bool, Optional, default = None\n        The two embedding sharing strategies are: 1) add the shared embeddings\n        to the column embeddings or 2) to replace the first\n        `frac_shared_embed` with the shared embeddings.\n        See `pytorch_widedeep.models.embeddings_layers.SharedEmbeddings`\n        If 'None' is passed, it will default to 'False'.\n    frac_shared_embed: float, Optional, default = None\n        The fraction of embeddings that will be shared (if `add_shared_embed\n        = False`) by all the different categories for one particular\n        column. If 'None' is passed, it will default to 0.0.\n    continuous_cols: List, Optional, default = None\n        List with the name of the numeric (aka continuous) columns\n    cont_norm_layer: str, Optional, default =  None\n        Type of normalization layer applied to the continuous features.\n        Options are: _'layernorm'_ and _'batchnorm'_. if `None`, no\n        normalization layer will be used.\n    embed_continuous_method: Optional, str, default = None,\n        Method to use to embed the continuous features. Options are:\n        _'standard'_, _'periodic'_ or _'piecewise'_. The _'standard'_\n        embedding method is based on the FT-Transformer implementation\n        presented in the paper: [Revisiting Deep Learning Models for\n        Tabular Data](https://arxiv.org/abs/2106.11959v5). The _'periodic'_\n        and_'piecewise'_ methods were presented in the paper: [On Embeddings for\n        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556).\n        Please, read the papers for details.\n    cont_embed_dropout: float, Optional, default = None,\n        Dropout for the continuous embeddings. If `None`, it will default to 0.0\n    cont_embed_activation: Optional, str, default = None,\n        Activation function for the continuous embeddings if any. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported.\n        If `None`, no activation function will be applied.\n    quantization_setup: Dict[str, List[float]], Optional, default = None,\n        This parameter is used when the _'piecewise'_ method is used to embed\n        the continuous cols. It is a dict where keys are the name of the continuous\n        columns and values are lists with the boundaries for the quantization\n        of the continuous_cols. See the examples for details. If\n        If the _'piecewise'_ method is used, this parameter is required.\n    n_frequencies: int, Optional, default = None,\n        This is the so called _'k'_ in their paper [On Embeddings for\n        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556),\n        and is the number of 'frequencies' that will be used to represent each\n        continuous column. See their Eq 2 in the paper for details. If\n        the _'periodic'_ method is used, this parameter is required.\n    sigma: float, Optional, default = None,\n        This is the sigma parameter in the paper mentioned when describing the\n        previous parameters and it is used to initialise the 'frequency\n        weights'. See their Eq 2 in the paper for details. If\n        the _'periodic'_ method is used, this parameter is required.\n    share_last_layer: bool, Optional, default = None,\n        This parameter is not present in the before mentioned paper but it is implemented in\n        the [official repo](https://github.com/yandex-research/rtdl-num-embeddings/tree/main).\n        If `True` the linear layer that turns the frequencies into embeddings\n        will be shared across the continuous columns. If `False` a different\n        linear layer will be used for each continuous column.\n        If the _'periodic'_ method is used, this parameter is required.\n    full_embed_dropout: bool, Optional, default = None,\n        If `True`, the full embedding corresponding to a column will be masked\n        out/dropout. If `None`, it will default to `False`.\n    input_dim: int, default = 32\n        The so-called *dimension of the model*. Is the number of\n        embeddings used to encode the categorical and/or continuous columns\n    attn_dropout: float, default = 0.2\n        Dropout for each attention block\n    n_heads: int, default = 8\n        Number of attention heads per attention block.\n    use_bias: bool, default = False\n        Boolean indicating whether or not to use bias in the Q, K projection\n        layers.\n    with_addnorm: bool = False,\n        Boolean indicating if residual connections will be used in the attention blocks\n    attn_activation: str, default = \"leaky_relu\"\n        String indicating the activation function to be applied to the dense\n        layer in each attention encoder. _'tanh'_, _'relu'_, _'leaky_relu'_\n        and _'gelu'_ are supported.\n    n_blocks: int, default = 3\n        Number of attention blocks\n\n    Attributes\n    ----------\n    cat_and_cont_embed: nn.Module\n        This is the module that processes the categorical and continuous columns\n    encoder: nn.Module\n        Sequence of attention encoders.\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.models import SelfAttentionMLP\n    &gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n    &gt;&gt;&gt; colnames = ['a', 'b', 'c', 'd', 'e']\n    &gt;&gt;&gt; cat_embed_input = [(u,i,j) for u,i,j in zip(colnames[:4], [4]*4, [8]*4)]\n    &gt;&gt;&gt; column_idx = {k:v for v,k in enumerate(colnames)}\n    &gt;&gt;&gt; model = SelfAttentionMLP(column_idx=column_idx, cat_embed_input=cat_embed_input, continuous_cols = ['e'])\n    &gt;&gt;&gt; out = model(X_tab)\n    \"\"\"\n\n    def __init__(\n        self,\n        column_idx: Dict[str, int],\n        *,\n        cat_embed_input: Optional[List[Tuple[str, int]]] = None,\n        cat_embed_dropout: Optional[float] = None,\n        use_cat_bias: Optional[bool] = None,\n        cat_embed_activation: Optional[str] = None,\n        shared_embed: Optional[bool] = None,\n        add_shared_embed: Optional[bool] = None,\n        frac_shared_embed: Optional[float] = None,\n        continuous_cols: Optional[List[str]] = None,\n        cont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\n        embed_continuous_method: Optional[\n            Literal[\"standard\", \"piecewise\", \"periodic\"]\n        ] = \"standard\",\n        cont_embed_dropout: Optional[float] = None,\n        cont_embed_activation: Optional[str] = None,\n        quantization_setup: Optional[Dict[str, List[float]]] = None,\n        n_frequencies: Optional[int] = None,\n        sigma: Optional[float] = None,\n        share_last_layer: Optional[bool] = None,\n        full_embed_dropout: Optional[bool] = None,\n        input_dim: int = 32,\n        attn_dropout: float = 0.2,\n        n_heads: int = 8,\n        use_bias: bool = False,\n        with_addnorm: bool = False,\n        attn_activation: str = \"leaky_relu\",\n        n_blocks: int = 3,\n    ):\n        super(SelfAttentionMLP, self).__init__(\n            column_idx=column_idx,\n            input_dim=input_dim,\n            cat_embed_input=cat_embed_input,\n            cat_embed_dropout=cat_embed_dropout,\n            use_cat_bias=use_cat_bias,\n            cat_embed_activation=cat_embed_activation,\n            shared_embed=shared_embed,\n            add_shared_embed=add_shared_embed,\n            frac_shared_embed=frac_shared_embed,\n            continuous_cols=continuous_cols,\n            cont_norm_layer=cont_norm_layer,\n            embed_continuous_method=embed_continuous_method,\n            cont_embed_dropout=cont_embed_dropout,\n            cont_embed_activation=cont_embed_activation,\n            quantization_setup=quantization_setup,\n            n_frequencies=n_frequencies,\n            sigma=sigma,\n            share_last_layer=share_last_layer,\n            full_embed_dropout=full_embed_dropout,\n        )\n\n        self.attn_dropout = attn_dropout\n        self.n_heads = n_heads\n        self.use_bias = use_bias\n        self.with_addnorm = with_addnorm\n        self.attn_activation = attn_activation\n        self.n_blocks = n_blocks\n\n        self.with_cls_token = \"cls_token\" in column_idx\n        self.n_cat = len(cat_embed_input) if cat_embed_input is not None else 0\n        self.n_cont = len(continuous_cols) if continuous_cols is not None else 0\n\n        # Embeddings are instantiated at the base model\n        # Attention Blocks\n        self.encoder = nn.Sequential()\n        for i in range(n_blocks):\n            self.encoder.add_module(\n                \"attention_block\" + str(i),\n                SelfAttentionEncoder(\n                    input_dim,\n                    attn_dropout,\n                    use_bias,\n                    n_heads,\n                    with_addnorm,\n                    attn_activation,\n                ),\n            )\n\n    def forward(self, X: Tensor) -&gt; Tensor:\n        x = self._get_embeddings(X)\n        x = self.encoder(x)\n        if self.with_cls_token:\n            out = x[:, 0, :]\n        else:\n            out = x.flatten(1)\n        return out\n\n    @property\n    def output_dim(self) -&gt; int:\n        r\"\"\"The output dimension of the model. This is a required property\n        neccesary to build the WideDeep class\n        \"\"\"\n        return (\n            self.input_dim\n            if self.with_cls_token\n            else ((self.n_cat + self.n_cont) * self.input_dim)\n        )\n\n    @property\n    def attention_weights(self) -&gt; List[Tensor]:\n        r\"\"\"List with the attention weights per block\n\n        The shape of the attention weights is $(N, H, F, F)$, where $N$ is the\n        batch size, $H$ is the number of attention heads and $F$ is the\n        number of features/columns in the dataset\n        \"\"\"\n        return [blk.attn.attn_weights for blk in self.encoder]\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.mlp.self_attention_mlp.SelfAttentionMLP.output_dim","title":"output_dim  <code>property</code>","text":"<pre><code>output_dim\n</code></pre> <p>The output dimension of the model. This is a required property neccesary to build the WideDeep class</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.mlp.self_attention_mlp.SelfAttentionMLP.attention_weights","title":"attention_weights  <code>property</code>","text":"<pre><code>attention_weights\n</code></pre> <p>List with the attention weights per block</p> <p>The shape of the attention weights is \\((N, H, F, F)\\), where \\(N\\) is the batch size, \\(H\\) is the number of attention heads and \\(F\\) is the number of features/columns in the dataset</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.transformers.tab_transformer.TabTransformer","title":"TabTransformer","text":"<p>               Bases: <code>BaseTabularModelWithAttention</code></p> <p>Defines our adptation of the TabTransformer model that can be used as the <code>deeptabular</code> component of a Wide &amp; Deep model or independently by itself.</p> <p>Most of the parameters for this class are <code>Optional</code> since the use of categorical or continuous is in fact optional (i.e. one can use categorical features only, continuous features only or both).</p> <p> NOTE: This is an enhanced adaptation of the model described in the paper. It can be considered as the flagship of our transformer family of models for tabular data and offers mutiple, additional features relative to the original publication(and some other models in the library)</p> <p>Parameters:</p> Name Type Description Default <code>column_idx</code> <code>Dict[str, int]</code> <p>Dict containing the index of the columns that will be passed through the <code>TabMlp</code> model. Required to slice the tensors. e.g. {'education': 0, 'relationship': 1, 'workclass': 2, ...}.</p> required <code>cat_embed_input</code> <code>Optional[List[Tuple[str, int]]]</code> <p>List of Tuples with the column name and number of unique values and embedding dimension. e.g. [(education, 11), ...]</p> <code>None</code> <code>cat_embed_dropout</code> <code>Optional[float]</code> <p>Categorical embeddings dropout. If <code>None</code>, it will default to 0.</p> <code>None</code> <code>use_cat_bias</code> <code>Optional[bool]</code> <p>Boolean indicating if bias will be used for the categorical embeddings. If <code>None</code>, it will default to 'False'.</p> <code>None</code> <code>cat_embed_activation</code> <code>Optional[str]</code> <p>Activation function for the categorical embeddings, if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> <code>None</code> <code>shared_embed</code> <code>Optional[bool]</code> <p>Boolean indicating if the embeddings will be \"shared\". The idea behind <code>shared_embed</code> is described in the Appendix A in the TabTransformer paper: 'The goal of having column embedding is to enable the model to distinguish the classes in one column from those in the other columns'. In other words, the idea is to let the model learn which column is embedded at the time. See: <code>pytorch_widedeep.models.transformers._layers.SharedEmbeddings</code>.</p> <code>None</code> <code>add_shared_embed</code> <code>Optional[bool]</code> <p>The two embedding sharing strategies are: 1) add the shared embeddings to the column embeddings or 2) to replace the first <code>frac_shared_embed</code> with the shared embeddings. See <code>pytorch_widedeep.models.embeddings_layers.SharedEmbeddings</code> If 'None' is passed, it will default to 'False'.</p> <code>None</code> <code>frac_shared_embed</code> <code>Optional[float]</code> <p>The fraction of embeddings that will be shared (if <code>add_shared_embed = False</code>) by all the different categories for one particular column. If 'None' is passed, it will default to 0.0.</p> <code>None</code> <code>continuous_cols</code> <code>Optional[List[str]]</code> <p>List with the name of the numeric (aka continuous) columns</p> <code>None</code> <code>cont_norm_layer</code> <code>Optional[Literal[batchnorm, layernorm]]</code> <p>Type of normalization layer applied to the continuous features. Options are: 'layernorm' and 'batchnorm'. if <code>None</code>, no normalization layer will be used.</p> <code>None</code> <code>embed_continuous_method</code> <code>Optional[Literal[standard, piecewise, periodic]]</code> <p>Method to use to embed the continuous features. Options are: 'standard', 'periodic' or 'piecewise'. The 'standard' embedding method is based on the FT-Transformer implementation presented in the paper: Revisiting Deep Learning Models for Tabular Data. The 'periodic' and_'piecewise'_ methods were presented in the paper: On Embeddings for Numerical Features in Tabular Deep Learning. Please, read the papers for details.</p> <code>None</code> <code>cont_embed_dropout</code> <code>Optional[float]</code> <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p> <code>None</code> <code>cont_embed_activation</code> <code>Optional[str]</code> <p>Activation function for the continuous embeddings if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported. If <code>None</code>, no activation function will be applied.</p> <code>None</code> <code>quantization_setup</code> <code>Optional[Dict[str, List[float]]]</code> <p>This parameter is used when the 'piecewise' method is used to embed the continuous cols. It is a dict where keys are the name of the continuous columns and values are lists with the boundaries for the quantization of the continuous_cols. See the examples for details. If If the 'piecewise' method is used, this parameter is required.</p> <code>None</code> <code>n_frequencies</code> <code>Optional[int]</code> <p>This is the so called 'k' in their paper On Embeddings for Numerical Features in Tabular Deep Learning, and is the number of 'frequencies' that will be used to represent each continuous column. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>sigma</code> <code>Optional[float]</code> <p>This is the sigma parameter in the paper mentioned when describing the previous parameters and it is used to initialise the 'frequency weights'. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>share_last_layer</code> <code>Optional[bool]</code> <p>This parameter is not present in the before mentioned paper but it is implemented in the official repo. If <code>True</code> the linear layer that turns the frequencies into embeddings will be shared across the continuous columns. If <code>False</code> a different linear layer will be used for each continuous column. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>full_embed_dropout</code> <code>Optional[bool]</code> <p>If <code>True</code>, the full embedding corresponding to a column will be masked out/dropout. If <code>None</code>, it will default to <code>False</code>.</p> <code>None</code> <code>input_dim</code> <code>int</code> <p>The so-called dimension of the model. Is the number of embeddings used to encode the categorical and/or continuous columns</p> <code>32</code> <code>n_heads</code> <code>int</code> <p>Number of attention heads per Transformer block</p> <code>8</code> <code>use_qkv_bias</code> <code>bool</code> <p>Boolean indicating whether or not to use bias in the Q, K, and V projection layers.</p> <code>False</code> <code>n_blocks</code> <code>int</code> <p>Number of Transformer blocks</p> <code>4</code> <code>attn_dropout</code> <code>float</code> <p>Dropout that will be applied to the Multi-Head Attention layers</p> <code>0.2</code> <code>ff_dropout</code> <code>float</code> <p>Dropout that will be applied to the FeedForward network</p> <code>0.1</code> <code>ff_factor</code> <code>int</code> <p>Multiplicative factor applied to the first layer of the FF network in each Transformer block, This is normally set to 4.</p> <code>4</code> <code>transformer_activation</code> <code>str</code> <p>Transformer Encoder activation function. 'tanh', 'relu', 'leaky_relu', 'gelu', 'geglu' and 'reglu' are supported</p> <code>'gelu'</code> <code>use_linear_attention</code> <code>bool</code> <p>Boolean indicating if Linear Attention (from Transformers are RNNs: Fast Autoregressive Transformers with Linear Attention) will be used. The inclusing of this mode of attention is inspired by this post, where the Uber team finds that this attention mechanism leads to the best results for their tabular data.</p> <code>False</code> <code>use_flash_attention</code> <code>bool</code> <p>Boolean indicating if Flash Attention will be used. </p> <code>False</code> <code>mlp_hidden_dims</code> <code>Optional[List[int]]</code> <p>List with the number of neurons per dense layer in the MLP. e.g: [64, 32]. If not provided no MLP on top of the final Transformer block will be used.</p> <code>None</code> <code>mlp_activation</code> <code>str</code> <p>Activation function for the dense layers of the MLP. Currently 'tanh', 'relu', 'leaky'_relu' and _'gelu' are supported. If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to 'relu'.</p> <code>'relu'</code> <code>mlp_dropout</code> <code>float</code> <p>float with the dropout between the dense layers of the MLP. If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to 0.0.</p> <code>0.1</code> <code>mlp_batchnorm</code> <code>bool</code> <p>Boolean indicating whether or not batch normalization will be applied to the dense layers If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to False.</p> <code>False</code> <code>mlp_batchnorm_last</code> <code>bool</code> <p>Boolean indicating whether or not batch normalization will be applied to the last of the dense layers If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to False.</p> <code>False</code> <code>mlp_linear_first</code> <code>bool</code> <p>Boolean indicating the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code> If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to <code>True</code>.</p> <code>True</code> <p>Attributes:</p> Name Type Description <code>encoder</code> <code>Module</code> <p>Sequence of Transformer blocks</p> <code>mlp</code> <code>Module</code> <p>MLP component in the model</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import TabTransformer\n&gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n&gt;&gt;&gt; colnames = ['a', 'b', 'c', 'd', 'e']\n&gt;&gt;&gt; cat_embed_input = [(u,i) for u,i in zip(colnames[:4], [4]*4)]\n&gt;&gt;&gt; continuous_cols = ['e']\n&gt;&gt;&gt; column_idx = {k:v for v,k in enumerate(colnames)}\n&gt;&gt;&gt; model = TabTransformer(column_idx=column_idx, cat_embed_input=cat_embed_input, continuous_cols=continuous_cols)\n&gt;&gt;&gt; out = model(X_tab)\n</code></pre> Source code in <code>pytorch_widedeep/models/tabular/transformers/tab_transformer.py</code> <pre><code>class TabTransformer(BaseTabularModelWithAttention):\n    r\"\"\"Defines our adptation of the\n    [TabTransformer model](https://arxiv.org/abs/2012.06678)\n    that can be used as the `deeptabular` component of a\n    Wide &amp; Deep model or independently by itself.\n\n    Most of the parameters for this class are `Optional` since the use of\n    categorical or continuous is in fact optional (i.e. one can use\n    categorical features only, continuous features only or both).\n\n    :information_source: **NOTE**:\n    This is an enhanced adaptation of the model described in the paper. It can\n    be considered as the flagship of our transformer family of models for\n    tabular data and offers mutiple, additional features relative to the\n    original publication(and some other models in the library)\n\n    Parameters\n    ----------\n    column_idx: Dict\n        Dict containing the index of the columns that will be passed through\n        the `TabMlp` model. Required to slice the tensors. e.g. _{'education':\n        0, 'relationship': 1, 'workclass': 2, ...}_.\n    cat_embed_input: List, Optional, default = None\n        List of Tuples with the column name and number of unique values and\n        embedding dimension. e.g. _[(education, 11), ...]_\n    cat_embed_dropout: float, Optional, default = None\n        Categorical embeddings dropout. If `None`, it will default\n        to 0.\n    use_cat_bias: bool, Optional, default = None,\n        Boolean indicating if bias will be used for the categorical embeddings.\n        If `None`, it will default to 'False'.\n    cat_embed_activation: Optional, str, default = None,\n        Activation function for the categorical embeddings, if any. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported\n    shared_embed: bool, Optional, default = None\n        Boolean indicating if the embeddings will be \"shared\". The idea behind `shared_embed` is\n        described in the Appendix A in the [TabTransformer paper](https://arxiv.org/abs/2012.06678):\n        _'The goal of having column embedding is to enable the model to\n        distinguish the classes in one column from those in the other\n        columns'_. In other words, the idea is to let the model learn which\n        column is embedded at the time. See: `pytorch_widedeep.models.transformers._layers.SharedEmbeddings`.\n    add_shared_embed: bool, Optional, default = None\n        The two embedding sharing strategies are: 1) add the shared embeddings\n        to the column embeddings or 2) to replace the first\n        `frac_shared_embed` with the shared embeddings.\n        See `pytorch_widedeep.models.embeddings_layers.SharedEmbeddings`\n        If 'None' is passed, it will default to 'False'.\n    frac_shared_embed: float, Optional, default = None\n        The fraction of embeddings that will be shared (if `add_shared_embed\n        = False`) by all the different categories for one particular\n        column. If 'None' is passed, it will default to 0.0.\n    continuous_cols: List, Optional, default = None\n        List with the name of the numeric (aka continuous) columns\n    cont_norm_layer: str, Optional, default =  None\n        Type of normalization layer applied to the continuous features.\n        Options are: _'layernorm'_ and _'batchnorm'_. if `None`, no\n        normalization layer will be used.\n    embed_continuous_method: Optional, str, default = None,\n        Method to use to embed the continuous features. Options are:\n        _'standard'_, _'periodic'_ or _'piecewise'_. The _'standard'_\n        embedding method is based on the FT-Transformer implementation\n        presented in the paper: [Revisiting Deep Learning Models for\n        Tabular Data](https://arxiv.org/abs/2106.11959v5). The _'periodic'_\n        and_'piecewise'_ methods were presented in the paper: [On Embeddings for\n        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556).\n        Please, read the papers for details.\n    cont_embed_dropout: float, Optional, default = None,\n        Dropout for the continuous embeddings. If `None`, it will default to 0.0\n    cont_embed_activation: Optional, str, default = None,\n        Activation function for the continuous embeddings if any. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported.\n        If `None`, no activation function will be applied.\n    quantization_setup: Dict[str, List[float]], Optional, default = None,\n        This parameter is used when the _'piecewise'_ method is used to embed\n        the continuous cols. It is a dict where keys are the name of the continuous\n        columns and values are lists with the boundaries for the quantization\n        of the continuous_cols. See the examples for details. If\n        If the _'piecewise'_ method is used, this parameter is required.\n    n_frequencies: int, Optional, default = None,\n        This is the so called _'k'_ in their paper [On Embeddings for\n        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556),\n        and is the number of 'frequencies' that will be used to represent each\n        continuous column. See their Eq 2 in the paper for details. If\n        the _'periodic'_ method is used, this parameter is required.\n    sigma: float, Optional, default = None,\n        This is the sigma parameter in the paper mentioned when describing the\n        previous parameters and it is used to initialise the 'frequency\n        weights'. See their Eq 2 in the paper for details. If\n        the _'periodic'_ method is used, this parameter is required.\n    share_last_layer: bool, Optional, default = None,\n        This parameter is not present in the before mentioned paper but it is implemented in\n        the [official repo](https://github.com/yandex-research/rtdl-num-embeddings/tree/main).\n        If `True` the linear layer that turns the frequencies into embeddings\n        will be shared across the continuous columns. If `False` a different\n        linear layer will be used for each continuous column.\n        If the _'periodic'_ method is used, this parameter is required.\n    full_embed_dropout: bool, Optional, default = None,\n        If `True`, the full embedding corresponding to a column will be masked\n        out/dropout. If `None`, it will default to `False`.\n    input_dim: int, default = 32\n        The so-called *dimension of the model*. Is the number of\n        embeddings used to encode the categorical and/or continuous columns\n    n_heads: int, default = 8\n        Number of attention heads per Transformer block\n    use_qkv_bias: bool, default = False\n        Boolean indicating whether or not to use bias in the Q, K, and V\n        projection layers.\n    n_blocks: int, default = 4\n        Number of Transformer blocks\n    attn_dropout: float, default = 0.2\n        Dropout that will be applied to the Multi-Head Attention layers\n    ff_dropout: float, default = 0.1\n        Dropout that will be applied to the FeedForward network\n    ff_factor: float, default = 4\n        Multiplicative factor applied to the first layer of the FF network in\n        each Transformer block, This is normally set to 4.\n    transformer_activation: str, default = \"gelu\"\n        Transformer Encoder activation function. _'tanh'_, _'relu'_,\n        _'leaky_relu'_, _'gelu'_, _'geglu'_ and _'reglu'_ are supported\n    use_linear_attention: Boolean, default = False,\n        Boolean indicating if Linear Attention (from [Transformers are RNNs:\n        Fast Autoregressive Transformers with Linear Attention](https://arxiv.org/abs/2006.16236))\n        will be used. The inclusing of this mode of attention is inspired by\n        [this post](https://www.uber.com/en-GB/blog/deepeta-how-uber-predicts-arrival-times/),\n        where the Uber team finds that this attention mechanism leads to the\n        best results for their tabular data.\n    use_flash_attention: Boolean, default = False,\n        Boolean indicating if\n        [Flash Attention](https://pytorch.org/docs/stable/generated/torch.nn.functional.scaled_dot_product_attention.html)\n        will be used. &lt;br/&gt;\n    mlp_hidden_dims: List, Optional, default = None\n        List with the number of neurons per dense layer in the MLP. e.g:\n        _[64, 32]_. If not provided no MLP on top of the final\n        Transformer block will be used.\n    mlp_activation: str, Optional, default = None\n        Activation function for the dense layers of the MLP. Currently\n        _'tanh'_, _'relu'_, _'leaky'_relu' and _'gelu'_ are supported.\n        If 'mlp_hidden_dims' is not `None` and this parameter is `None`, it\n        will default to _'relu'_.\n    mlp_dropout: float, Optional, default = None\n        float with the dropout between the dense layers of the MLP.\n        If 'mlp_hidden_dims' is not `None` and this parameter is `None`, it\n        will default to 0.0.\n    mlp_batchnorm: bool, Optional, default = None\n        Boolean indicating whether or not batch normalization will be applied\n        to the dense layers\n        If 'mlp_hidden_dims' is not `None` and this parameter is `None`, it\n        will default to False.\n    mlp_batchnorm_last: bool, Optional, default = None\n        Boolean indicating whether or not batch normalization will be applied\n        to the last of the dense layers\n        If 'mlp_hidden_dims' is not `None` and this parameter is `None`, it\n        will default to False.\n    mlp_linear_first: bool, Optional, default = None\n        Boolean indicating the order of the operations in the dense\n        layer. If `True: [LIN -&gt; ACT -&gt; BN -&gt; DP]`. If `False: [BN -&gt; DP -&gt;\n        LIN -&gt; ACT]`\n        If 'mlp_hidden_dims' is not `None` and this parameter is `None`, it\n        will default to `True`.\n\n    Attributes\n    ----------\n    encoder: nn.Module\n        Sequence of Transformer blocks\n    mlp: nn.Module\n        MLP component in the model\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.models import TabTransformer\n    &gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n    &gt;&gt;&gt; colnames = ['a', 'b', 'c', 'd', 'e']\n    &gt;&gt;&gt; cat_embed_input = [(u,i) for u,i in zip(colnames[:4], [4]*4)]\n    &gt;&gt;&gt; continuous_cols = ['e']\n    &gt;&gt;&gt; column_idx = {k:v for v,k in enumerate(colnames)}\n    &gt;&gt;&gt; model = TabTransformer(column_idx=column_idx, cat_embed_input=cat_embed_input, continuous_cols=continuous_cols)\n    &gt;&gt;&gt; out = model(X_tab)\n    \"\"\"\n\n    def __init__(\n        self,\n        column_idx: Dict[str, int],\n        *,\n        cat_embed_input: Optional[List[Tuple[str, int]]] = None,\n        cat_embed_dropout: Optional[float] = None,\n        use_cat_bias: Optional[bool] = None,\n        cat_embed_activation: Optional[str] = None,\n        shared_embed: Optional[bool] = None,\n        add_shared_embed: Optional[bool] = None,\n        frac_shared_embed: Optional[float] = None,\n        continuous_cols: Optional[List[str]] = None,\n        cont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\n        embed_continuous: Optional[bool] = None,\n        embed_continuous_method: Optional[\n            Literal[\"standard\", \"piecewise\", \"periodic\"]\n        ] = None,\n        cont_embed_dropout: Optional[float] = None,\n        cont_embed_activation: Optional[str] = None,\n        quantization_setup: Optional[Dict[str, List[float]]] = None,\n        n_frequencies: Optional[int] = None,\n        sigma: Optional[float] = None,\n        share_last_layer: Optional[bool] = None,\n        full_embed_dropout: Optional[bool] = None,\n        input_dim: int = 32,\n        n_heads: int = 8,\n        use_qkv_bias: bool = False,\n        n_blocks: int = 4,\n        attn_dropout: float = 0.2,\n        ff_dropout: float = 0.1,\n        ff_factor: int = 4,\n        transformer_activation: str = \"gelu\",\n        use_linear_attention: bool = False,\n        use_flash_attention: bool = False,\n        mlp_hidden_dims: Optional[List[int]] = None,\n        mlp_activation: str = \"relu\",\n        mlp_dropout: float = 0.1,\n        mlp_batchnorm: bool = False,\n        mlp_batchnorm_last: bool = False,\n        mlp_linear_first: bool = True,\n    ):\n        super(TabTransformer, self).__init__(\n            column_idx=column_idx,\n            cat_embed_input=cat_embed_input,\n            cat_embed_dropout=cat_embed_dropout,\n            use_cat_bias=use_cat_bias,\n            cat_embed_activation=cat_embed_activation,\n            shared_embed=shared_embed,\n            add_shared_embed=add_shared_embed,\n            frac_shared_embed=frac_shared_embed,\n            continuous_cols=continuous_cols,\n            cont_norm_layer=cont_norm_layer,\n            # UGLY hack to be able to use the base class __init__ method\n            embed_continuous_method=(\n                \"standard\"\n                if embed_continuous_method is None\n                else embed_continuous_method\n            ),\n            cont_embed_dropout=cont_embed_dropout,\n            cont_embed_activation=cont_embed_activation,\n            quantization_setup=quantization_setup,\n            n_frequencies=n_frequencies,\n            sigma=sigma,\n            share_last_layer=share_last_layer,\n            input_dim=input_dim,\n            full_embed_dropout=full_embed_dropout,\n        )\n\n        # we overwrite the embed_continuous_method parameter that was set\n        # to 'standard' if embed_continuous_method is None. In other words,\n        # this will already have the right value if the user provided a value\n        # for embed_continuous_method, otherwise will be \"standard\" when it\n        # should be None\n        self.embed_continuous_method = embed_continuous_method\n\n        if embed_continuous is not None:\n            self.embed_continuous = embed_continuous\n            if embed_continuous and embed_continuous_method is None:\n                raise ValueError(\n                    \"If 'embed_continuous' is True, 'embed_continuous_method' must be \"\n                    \"one of 'standard', 'piecewise' or 'periodic'.\"\n                )\n        elif self.embed_continuous_method is not None:\n            self.embed_continuous = True\n        else:\n            self.embed_continuous = False\n\n        self.n_heads = n_heads\n        self.use_qkv_bias = use_qkv_bias\n        self.n_blocks = n_blocks\n        self.attn_dropout = attn_dropout\n        self.ff_dropout = ff_dropout\n        self.transformer_activation = transformer_activation\n        self.use_linear_attention = use_linear_attention\n        self.use_flash_attention = use_flash_attention\n        self.ff_factor = ff_factor\n\n        self.mlp_hidden_dims = mlp_hidden_dims\n        self.mlp_activation = mlp_activation\n        self.mlp_dropout = mlp_dropout\n        self.mlp_batchnorm = mlp_batchnorm\n        self.mlp_batchnorm_last = mlp_batchnorm_last\n        self.mlp_linear_first = mlp_linear_first\n\n        self.with_cls_token = \"cls_token\" in column_idx\n        self.n_cat = len(cat_embed_input) if cat_embed_input is not None else 0\n        self.n_cont = len(continuous_cols) if continuous_cols is not None else 0\n\n        if self.n_cont and not self.n_cat and not self.embed_continuous:\n            raise ValueError(\n                \"If only continuous features are used 'embed_continuous' must be set to 'True'\"\n            )\n\n        # Embeddings are instantiated at the base model\n        # Transformer blocks\n        self.encoder = nn.Sequential()\n        for i in range(n_blocks):\n            self.encoder.add_module(\n                \"transformer_block\" + str(i),\n                TransformerEncoder(\n                    input_dim,\n                    n_heads,\n                    use_qkv_bias,\n                    attn_dropout,\n                    ff_dropout,\n                    ff_factor,\n                    transformer_activation,\n                    use_linear_attention,\n                    use_flash_attention,\n                ),\n            )\n\n        self.mlp_first_hidden_dim = self._mlp_first_hidden_dim()\n\n        if self.mlp_hidden_dims is not None:\n            self.mlp = MLP(\n                d_hidden=[self.mlp_first_hidden_dim] + self.mlp_hidden_dims,\n                activation=(\n                    \"relu\" if self.mlp_activation is None else self.mlp_activation\n                ),\n                dropout=0.0 if self.mlp_dropout is None else self.mlp_dropout,\n                batchnorm=False if self.mlp_batchnorm is None else self.mlp_batchnorm,\n                batchnorm_last=(\n                    False\n                    if self.mlp_batchnorm_last is None\n                    else self.mlp_batchnorm_last\n                ),\n                linear_first=(\n                    False if self.mlp_linear_first is None else self.mlp_linear_first\n                ),\n            )\n        else:\n            self.mlp = None\n\n    def forward(self, X: Tensor) -&gt; Tensor:\n        x_embed, x_cont_not_embed = self._get_embeddings_tt(X)\n        x = self.encoder(x_embed)\n\n        if self.with_cls_token:\n            x = x[:, 0, :]\n        else:\n            x = x.flatten(1)\n\n        if x_cont_not_embed is not None:\n            x = torch.cat([x, x_cont_not_embed], 1)\n\n        if self.mlp is not None:\n            x = self.mlp(x)\n        return x\n\n    def _mlp_first_hidden_dim(self) -&gt; int:\n        if self.with_cls_token:\n            if self.embed_continuous:\n                attn_output_dim = self.input_dim\n            else:\n                attn_output_dim = self.input_dim + self.n_cont\n        elif self.embed_continuous:\n            attn_output_dim = (self.n_cat + self.n_cont) * self.input_dim\n        else:\n            attn_output_dim = self.n_cat * self.input_dim + self.n_cont\n\n        return attn_output_dim\n\n    @property\n    def output_dim(self) -&gt; int:\n        r\"\"\"The output dimension of the model. This is a required property\n        neccesary to build the `WideDeep` class\n        \"\"\"\n        return (\n            self.mlp_hidden_dims[-1]\n            if self.mlp_hidden_dims is not None\n            else self.mlp_first_hidden_dim\n        )\n\n    @property\n    def attention_weights(self) -&gt; List[Tensor]:\n        r\"\"\"List with the attention weights per block\n\n        The shape of the attention weights is $(N, H, F, F)$, where $N$ is the\n        batch size, $H$ is the number of attention heads and $F$ is the\n        number of features/columns in the dataset\n\n        :information_source: **NOTE**:\n        if flash attention or linear attention\n        are used, no attention weights are saved during the training process\n        and calling this property will throw a ValueError\n        \"\"\"\n        if self.use_flash_attention or self.use_linear_attention:\n            raise ValueError(\n                \"Extraction of the attention weights is not supported for \"\n                \"linear or flash attention\"\n            )\n\n        return [blk.attn.attn_weights for blk in self.encoder]\n\n    def _get_embeddings_tt(self, X: Tensor) -&gt; Tuple[Tensor, Optional[Tensor]]:  # type: ignore[override]\n        if self.n_cont and self.embed_continuous and self.n_cat:\n            # there are continuous and categorical features and the continuous\n            # features are embedded\n            x_embed = self._get_embeddings(X)\n            x_cont_not_embed = None\n        elif self.n_cont and self.embed_continuous:\n            # there are only continuous features and they are embedded\n            x_embed = self.cont_norm((X[:, self.cont_idx].float()))\n            x_embed = self.cont_embed(x_embed)\n            x_cont_not_embed = None\n        elif self.n_cont and self.n_cat:\n            # there are continuous and categorical features but the continuous\n            # features are not embedded\n            x_embed = self.cat_embed(X)\n            x_cont_not_embed = self.cont_norm((X[:, self.cont_idx].float()))\n        elif self.n_cat:\n            # there are only categorical features\n            x_embed = self.cat_embed(X)\n            x_cont_not_embed = None\n        else:\n            raise ValueError(\n                \"Using only continuous features that are not embedded is \"\n                \"is not supported in this implementation. Please set 'embed_continuous_method' \"\n                \"to one of ['standard', 'piecewise', 'periodic']\"\n            )\n        return x_embed, x_cont_not_embed\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.transformers.tab_transformer.TabTransformer.output_dim","title":"output_dim  <code>property</code>","text":"<pre><code>output_dim\n</code></pre> <p>The output dimension of the model. This is a required property neccesary to build the <code>WideDeep</code> class</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.transformers.tab_transformer.TabTransformer.attention_weights","title":"attention_weights  <code>property</code>","text":"<pre><code>attention_weights\n</code></pre> <p>List with the attention weights per block</p> <p>The shape of the attention weights is \\((N, H, F, F)\\), where \\(N\\) is the batch size, \\(H\\) is the number of attention heads and \\(F\\) is the number of features/columns in the dataset</p> <p> NOTE: if flash attention or linear attention are used, no attention weights are saved during the training process and calling this property will throw a ValueError</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.transformers.saint.SAINT","title":"SAINT","text":"<p>               Bases: <code>BaseTabularModelWithAttention</code></p> <p>Defines a SAINT model that can be used as the <code>deeptabular</code> component of a Wide &amp; Deep model or independently by itself.</p> <p>Most of the parameters for this class are <code>Optional</code> since the use of categorical or continuous is in fact optional (i.e. one can use categorical features only, continuous features only or both).</p> <p> NOTE: This is an slightly modified and enhanced  version of the model described in the paper,</p> <p>Parameters:</p> Name Type Description Default <code>column_idx</code> <code>Dict[str, int]</code> <p>Dict containing the index of the columns that will be passed through the <code>TabMlp</code> model. Required to slice the tensors. e.g. {'education': 0, 'relationship': 1, 'workclass': 2, ...}.</p> required <code>cat_embed_input</code> <code>Optional[List[Tuple[str, int]]]</code> <p>List of Tuples with the column name and number of unique values and embedding dimension. e.g. [(education, 11), ...]</p> <code>None</code> <code>cat_embed_dropout</code> <code>Optional[float]</code> <p>Categorical embeddings dropout. If <code>None</code>, it will default to 0.</p> <code>None</code> <code>use_cat_bias</code> <code>Optional[bool]</code> <p>Boolean indicating if bias will be used for the categorical embeddings. If <code>None</code>, it will default to 'False'.</p> <code>None</code> <code>cat_embed_activation</code> <code>Optional[str]</code> <p>Activation function for the categorical embeddings, if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> <code>None</code> <code>shared_embed</code> <code>Optional[bool]</code> <p>Boolean indicating if the embeddings will be \"shared\". The idea behind <code>shared_embed</code> is described in the Appendix A in the TabTransformer paper: 'The goal of having column embedding is to enable the model to distinguish the classes in one column from those in the other columns'. In other words, the idea is to let the model learn which column is embedded at the time. See: <code>pytorch_widedeep.models.transformers._layers.SharedEmbeddings</code>.</p> <code>None</code> <code>add_shared_embed</code> <code>Optional[bool]</code> <p>The two embedding sharing strategies are: 1) add the shared embeddings to the column embeddings or 2) to replace the first <code>frac_shared_embed</code> with the shared embeddings. See <code>pytorch_widedeep.models.embeddings_layers.SharedEmbeddings</code> If 'None' is passed, it will default to 'False'.</p> <code>None</code> <code>frac_shared_embed</code> <code>Optional[float]</code> <p>The fraction of embeddings that will be shared (if <code>add_shared_embed = False</code>) by all the different categories for one particular column. If 'None' is passed, it will default to 0.0.</p> <code>None</code> <code>continuous_cols</code> <code>Optional[List[str]]</code> <p>List with the name of the numeric (aka continuous) columns</p> <code>None</code> <code>cont_norm_layer</code> <code>Optional[Literal[batchnorm, layernorm]]</code> <p>Type of normalization layer applied to the continuous features. Options are: 'layernorm' and 'batchnorm'. if <code>None</code>, no normalization layer will be used.</p> <code>None</code> <code>embed_continuous_method</code> <code>Optional[Literal[standard, piecewise, periodic]]</code> <p>Method to use to embed the continuous features. Options are: 'standard', 'periodic' or 'piecewise'. The 'standard' embedding method is based on the FT-Transformer implementation presented in the paper: Revisiting Deep Learning Models for Tabular Data. The 'periodic' and_'piecewise'_ methods were presented in the paper: On Embeddings for Numerical Features in Tabular Deep Learning. Please, read the papers for details.</p> <code>'standard'</code> <code>cont_embed_dropout</code> <code>Optional[float]</code> <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p> <code>None</code> <code>cont_embed_activation</code> <code>Optional[str]</code> <p>Activation function for the continuous embeddings if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported. If <code>None</code>, no activation function will be applied.</p> <code>None</code> <code>quantization_setup</code> <code>Optional[Dict[str, List[float]]]</code> <p>This parameter is used when the 'piecewise' method is used to embed the continuous cols. It is a dict where keys are the name of the continuous columns and values are lists with the boundaries for the quantization of the continuous_cols. See the examples for details. If If the 'piecewise' method is used, this parameter is required.</p> <code>None</code> <code>n_frequencies</code> <code>Optional[int]</code> <p>This is the so called 'k' in their paper On Embeddings for Numerical Features in Tabular Deep Learning, and is the number of 'frequencies' that will be used to represent each continuous column. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>sigma</code> <code>Optional[float]</code> <p>This is the sigma parameter in the paper mentioned when describing the previous parameters and it is used to initialise the 'frequency weights'. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>share_last_layer</code> <code>Optional[bool]</code> <p>This parameter is not present in the before mentioned paper but it is implemented in the official repo. If <code>True</code> the linear layer that turns the frequencies into embeddings will be shared across the continuous columns. If <code>False</code> a different linear layer will be used for each continuous column. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>full_embed_dropout</code> <code>Optional[bool]</code> <p>If <code>True</code>, the full embedding corresponding to a column will be masked out/dropout. If <code>None</code>, it will default to <code>False</code>.</p> <code>None</code> <code>input_dim</code> <code>int</code> <p>The so-called dimension of the model. Is the number of embeddings used to encode the categorical and/or continuous columns</p> <code>32</code> <code>n_heads</code> <code>int</code> <p>Number of attention heads per Transformer block</p> <code>8</code> <code>use_qkv_bias</code> <code>bool</code> <p>Boolean indicating whether or not to use bias in the Q, K, and V projection layers</p> <code>False</code> <code>n_blocks</code> <code>int</code> <p>Number of SAINT-Transformer blocks.</p> <code>2</code> <code>attn_dropout</code> <code>float</code> <p>Dropout that will be applied to the Multi-Head Attention column and row layers</p> <code>0.1</code> <code>ff_dropout</code> <code>float</code> <p>Dropout that will be applied to the FeedForward network</p> <code>0.2</code> <code>ff_factor</code> <code>int</code> <p>Multiplicative factor applied to the first layer of the FF network in each Transformer block, This is normally set to 4.</p> <code>4</code> <code>transformer_activation</code> <code>str</code> <p>Transformer Encoder activation function. 'tanh', 'relu', 'leaky_relu', 'gelu', 'geglu' and 'reglu' are supported</p> <code>'gelu'</code> <code>mlp_hidden_dims</code> <code>Optional[List[int]]</code> <p>List with the number of neurons per dense layer in the MLP. e.g: [64, 32]. If not provided no MLP on top of the final Transformer block will be used.</p> <code>None</code> <code>mlp_activation</code> <code>Optional[str]</code> <p>Activation function for the dense layers of the MLP. Currently 'tanh', 'relu', 'leaky'_relu' and _'gelu' are supported. If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to 'relu'.</p> <code>None</code> <code>mlp_dropout</code> <code>Optional[float]</code> <p>float with the dropout between the dense layers of the MLP. If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to 0.0.</p> <code>None</code> <code>mlp_batchnorm</code> <code>Optional[bool]</code> <p>Boolean indicating whether or not batch normalization will be applied to the dense layers If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to False.</p> <code>None</code> <code>mlp_batchnorm_last</code> <code>Optional[bool]</code> <p>Boolean indicating whether or not batch normalization will be applied to the last of the dense layers If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to False.</p> <code>None</code> <code>mlp_linear_first</code> <code>Optional[bool]</code> <p>Boolean indicating the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code> If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to <code>True</code>.</p> <code>None</code> <p>Attributes:</p> Name Type Description <code>encoder</code> <code>Module</code> <p>Sequence of SAINT-Transformer blocks</p> <code>mlp</code> <code>Module</code> <p>MLP component in the model</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import SAINT\n&gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n&gt;&gt;&gt; colnames = ['a', 'b', 'c', 'd', 'e']\n&gt;&gt;&gt; cat_embed_input = [(u,i) for u,i in zip(colnames[:4], [4]*4)]\n&gt;&gt;&gt; continuous_cols = ['e']\n&gt;&gt;&gt; column_idx = {k:v for v,k in enumerate(colnames)}\n&gt;&gt;&gt; model = SAINT(column_idx=column_idx, cat_embed_input=cat_embed_input, continuous_cols=continuous_cols)\n&gt;&gt;&gt; out = model(X_tab)\n</code></pre> Source code in <code>pytorch_widedeep/models/tabular/transformers/saint.py</code> <pre><code>class SAINT(BaseTabularModelWithAttention):\n    r\"\"\"Defines a [SAINT model](https://arxiv.org/abs/2106.01342) that\n    can be used as the `deeptabular` component of a Wide &amp; Deep model or\n    independently by itself.\n\n    Most of the parameters for this class are `Optional` since the use of\n    categorical or continuous is in fact optional (i.e. one can use\n    categorical features only, continuous features only or both).\n\n    :information_source: **NOTE**: This is an slightly modified and enhanced\n     version of the model described in the paper,\n\n    Parameters\n    ----------\n    column_idx: Dict\n        Dict containing the index of the columns that will be passed through\n        the `TabMlp` model. Required to slice the tensors. e.g. _{'education':\n        0, 'relationship': 1, 'workclass': 2, ...}_.\n    cat_embed_input: List, Optional, default = None\n        List of Tuples with the column name and number of unique values and\n        embedding dimension. e.g. _[(education, 11), ...]_\n    cat_embed_dropout: float, Optional, default = None\n        Categorical embeddings dropout. If `None`, it will default\n        to 0.\n    use_cat_bias: bool, Optional, default = None,\n        Boolean indicating if bias will be used for the categorical embeddings.\n        If `None`, it will default to 'False'.\n    cat_embed_activation: Optional, str, default = None,\n        Activation function for the categorical embeddings, if any. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported\n    shared_embed: bool, Optional, default = None\n        Boolean indicating if the embeddings will be \"shared\". The idea behind `shared_embed` is\n        described in the Appendix A in the [TabTransformer paper](https://arxiv.org/abs/2012.06678):\n        _'The goal of having column embedding is to enable the model to\n        distinguish the classes in one column from those in the other\n        columns'_. In other words, the idea is to let the model learn which\n        column is embedded at the time. See: `pytorch_widedeep.models.transformers._layers.SharedEmbeddings`.\n    add_shared_embed: bool, Optional, default = None\n        The two embedding sharing strategies are: 1) add the shared embeddings\n        to the column embeddings or 2) to replace the first\n        `frac_shared_embed` with the shared embeddings.\n        See `pytorch_widedeep.models.embeddings_layers.SharedEmbeddings`\n        If 'None' is passed, it will default to 'False'.\n    frac_shared_embed: float, Optional, default = None\n        The fraction of embeddings that will be shared (if `add_shared_embed\n        = False`) by all the different categories for one particular\n        column. If 'None' is passed, it will default to 0.0.\n    continuous_cols: List, Optional, default = None\n        List with the name of the numeric (aka continuous) columns\n    cont_norm_layer: str, Optional, default =  None\n        Type of normalization layer applied to the continuous features.\n        Options are: _'layernorm'_ and _'batchnorm'_. if `None`, no\n        normalization layer will be used.\n    embed_continuous_method: Optional, str, default = None,\n        Method to use to embed the continuous features. Options are:\n        _'standard'_, _'periodic'_ or _'piecewise'_. The _'standard'_\n        embedding method is based on the FT-Transformer implementation\n        presented in the paper: [Revisiting Deep Learning Models for\n        Tabular Data](https://arxiv.org/abs/2106.11959v5). The _'periodic'_\n        and_'piecewise'_ methods were presented in the paper: [On Embeddings for\n        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556).\n        Please, read the papers for details.\n    cont_embed_dropout: float, Optional, default = None,\n        Dropout for the continuous embeddings. If `None`, it will default to 0.0\n    cont_embed_activation: Optional, str, default = None,\n        Activation function for the continuous embeddings if any. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported.\n        If `None`, no activation function will be applied.\n    quantization_setup: Dict[str, List[float]], Optional, default = None,\n        This parameter is used when the _'piecewise'_ method is used to embed\n        the continuous cols. It is a dict where keys are the name of the continuous\n        columns and values are lists with the boundaries for the quantization\n        of the continuous_cols. See the examples for details. If\n        If the _'piecewise'_ method is used, this parameter is required.\n    n_frequencies: int, Optional, default = None,\n        This is the so called _'k'_ in their paper [On Embeddings for\n        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556),\n        and is the number of 'frequencies' that will be used to represent each\n        continuous column. See their Eq 2 in the paper for details. If\n        the _'periodic'_ method is used, this parameter is required.\n    sigma: float, Optional, default = None,\n        This is the sigma parameter in the paper mentioned when describing the\n        previous parameters and it is used to initialise the 'frequency\n        weights'. See their Eq 2 in the paper for details. If\n        the _'periodic'_ method is used, this parameter is required.\n    share_last_layer: bool, Optional, default = None,\n        This parameter is not present in the before mentioned paper but it is implemented in\n        the [official repo](https://github.com/yandex-research/rtdl-num-embeddings/tree/main).\n        If `True` the linear layer that turns the frequencies into embeddings\n        will be shared across the continuous columns. If `False` a different\n        linear layer will be used for each continuous column.\n        If the _'periodic'_ method is used, this parameter is required.\n    full_embed_dropout: bool, Optional, default = None,\n        If `True`, the full embedding corresponding to a column will be masked\n        out/dropout. If `None`, it will default to `False`.\n    input_dim: int, default = 32\n        The so-called *dimension of the model*. Is the number of\n        embeddings used to encode the categorical and/or continuous columns\n    n_heads: int, default = 8\n        Number of attention heads per Transformer block\n    use_qkv_bias: bool, default = False\n        Boolean indicating whether or not to use bias in the Q, K, and V\n        projection layers\n    n_blocks: int, default = 2\n        Number of SAINT-Transformer blocks.\n    attn_dropout: float, default = 0.2\n        Dropout that will be applied to the Multi-Head Attention column and\n        row layers\n    ff_dropout: float, default = 0.1\n        Dropout that will be applied to the FeedForward network\n    ff_factor: float, default = 4\n        Multiplicative factor applied to the first layer of the FF network in\n        each Transformer block, This is normally set to 4.\n    transformer_activation: str, default = \"gelu\"\n        Transformer Encoder activation function. _'tanh'_, _'relu'_,\n        _'leaky_relu'_, _'gelu'_, _'geglu'_ and _'reglu'_ are supported\n    mlp_hidden_dims: List, Optional, default = None\n        List with the number of neurons per dense layer in the MLP. e.g:\n        _[64, 32]_. If not provided no MLP on top of the final\n        Transformer block will be used.\n    mlp_activation: str, Optional, default = None\n        Activation function for the dense layers of the MLP. Currently\n        _'tanh'_, _'relu'_, _'leaky'_relu' and _'gelu'_ are supported.\n        If 'mlp_hidden_dims' is not `None` and this parameter is `None`, it\n        will default to _'relu'_.\n    mlp_dropout: float, Optional, default = None\n        float with the dropout between the dense layers of the MLP.\n        If 'mlp_hidden_dims' is not `None` and this parameter is `None`, it\n        will default to 0.0.\n    mlp_batchnorm: bool, Optional, default = None\n        Boolean indicating whether or not batch normalization will be applied\n        to the dense layers\n        If 'mlp_hidden_dims' is not `None` and this parameter is `None`, it\n        will default to False.\n    mlp_batchnorm_last: bool, Optional, default = None\n        Boolean indicating whether or not batch normalization will be applied\n        to the last of the dense layers\n        If 'mlp_hidden_dims' is not `None` and this parameter is `None`, it\n        will default to False.\n    mlp_linear_first: bool, Optional, default = None\n        Boolean indicating the order of the operations in the dense\n        layer. If `True: [LIN -&gt; ACT -&gt; BN -&gt; DP]`. If `False: [BN -&gt; DP -&gt;\n        LIN -&gt; ACT]`\n        If 'mlp_hidden_dims' is not `None` and this parameter is `None`, it\n        will default to `True`.\n\n    Attributes\n    ----------\n    encoder: nn.Module\n        Sequence of SAINT-Transformer blocks\n    mlp: nn.Module\n        MLP component in the model\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.models import SAINT\n    &gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n    &gt;&gt;&gt; colnames = ['a', 'b', 'c', 'd', 'e']\n    &gt;&gt;&gt; cat_embed_input = [(u,i) for u,i in zip(colnames[:4], [4]*4)]\n    &gt;&gt;&gt; continuous_cols = ['e']\n    &gt;&gt;&gt; column_idx = {k:v for v,k in enumerate(colnames)}\n    &gt;&gt;&gt; model = SAINT(column_idx=column_idx, cat_embed_input=cat_embed_input, continuous_cols=continuous_cols)\n    &gt;&gt;&gt; out = model(X_tab)\n    \"\"\"\n\n    def __init__(\n        self,\n        column_idx: Dict[str, int],\n        *,\n        cat_embed_input: Optional[List[Tuple[str, int]]] = None,\n        cat_embed_dropout: Optional[float] = None,\n        use_cat_bias: Optional[bool] = None,\n        cat_embed_activation: Optional[str] = None,\n        shared_embed: Optional[bool] = None,\n        add_shared_embed: Optional[bool] = None,\n        frac_shared_embed: Optional[float] = None,\n        continuous_cols: Optional[List[str]] = None,\n        cont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\n        embed_continuous_method: Optional[\n            Literal[\"standard\", \"piecewise\", \"periodic\"]\n        ] = \"standard\",\n        cont_embed_dropout: Optional[float] = None,\n        cont_embed_activation: Optional[str] = None,\n        quantization_setup: Optional[Dict[str, List[float]]] = None,\n        n_frequencies: Optional[int] = None,\n        sigma: Optional[float] = None,\n        share_last_layer: Optional[bool] = None,\n        full_embed_dropout: Optional[bool] = None,\n        input_dim: int = 32,\n        use_qkv_bias: bool = False,\n        n_heads: int = 8,\n        n_blocks: int = 2,\n        attn_dropout: float = 0.1,\n        ff_dropout: float = 0.2,\n        ff_factor: int = 4,\n        transformer_activation: str = \"gelu\",\n        mlp_hidden_dims: Optional[List[int]] = None,\n        mlp_activation: Optional[str] = None,\n        mlp_dropout: Optional[float] = None,\n        mlp_batchnorm: Optional[bool] = None,\n        mlp_batchnorm_last: Optional[bool] = None,\n        mlp_linear_first: Optional[bool] = None,\n    ):\n        super(SAINT, self).__init__(\n            column_idx=column_idx,\n            cat_embed_input=cat_embed_input,\n            cat_embed_dropout=cat_embed_dropout,\n            use_cat_bias=use_cat_bias,\n            cat_embed_activation=cat_embed_activation,\n            shared_embed=shared_embed,\n            add_shared_embed=add_shared_embed,\n            frac_shared_embed=frac_shared_embed,\n            continuous_cols=continuous_cols,\n            cont_norm_layer=cont_norm_layer,\n            embed_continuous_method=embed_continuous_method,\n            cont_embed_dropout=cont_embed_dropout,\n            cont_embed_activation=cont_embed_activation,\n            input_dim=input_dim,\n            quantization_setup=quantization_setup,\n            n_frequencies=n_frequencies,\n            sigma=sigma,\n            share_last_layer=share_last_layer,\n            full_embed_dropout=full_embed_dropout,\n        )\n\n        self.use_qkv_bias = use_qkv_bias\n        self.n_heads = n_heads\n        self.n_blocks = n_blocks\n        self.attn_dropout = attn_dropout\n        self.ff_dropout = ff_dropout\n        self.ff_factor = ff_factor\n        self.transformer_activation = transformer_activation\n\n        self.mlp_hidden_dims = mlp_hidden_dims\n        self.mlp_activation = mlp_activation\n        self.mlp_dropout = mlp_dropout\n        self.mlp_batchnorm = mlp_batchnorm\n        self.mlp_batchnorm_last = mlp_batchnorm_last\n        self.mlp_linear_first = mlp_linear_first\n\n        self.with_cls_token = \"cls_token\" in column_idx\n        self.n_cat = len(cat_embed_input) if cat_embed_input is not None else 0\n        self.n_cont = len(continuous_cols) if continuous_cols is not None else 0\n        self.n_feats = self.n_cat + self.n_cont\n\n        # Embeddings are instantiated at the base model\n        # Transformer blocks\n        self.encoder = nn.Sequential()\n        for i in range(n_blocks):\n            self.encoder.add_module(\n                \"saint_block\" + str(i),\n                SaintEncoder(\n                    input_dim,\n                    n_heads,\n                    use_qkv_bias,\n                    attn_dropout,\n                    ff_dropout,\n                    ff_factor,\n                    transformer_activation,\n                    self.n_feats,\n                ),\n            )\n\n        self.mlp_first_hidden_dim = (\n            self.input_dim if self.with_cls_token else (self.n_feats * self.input_dim)\n        )\n\n        # Mlp: adding an MLP on top of the Resnet blocks is optional and\n        # therefore all related params are optional\n        if self.mlp_hidden_dims is not None:\n            self.mlp = MLP(\n                d_hidden=[self.mlp_first_hidden_dim] + self.mlp_hidden_dims,\n                activation=(\n                    \"relu\" if self.mlp_activation is None else self.mlp_activation\n                ),\n                dropout=0.0 if self.mlp_dropout is None else self.mlp_dropout,\n                batchnorm=False if self.mlp_batchnorm is None else self.mlp_batchnorm,\n                batchnorm_last=(\n                    False\n                    if self.mlp_batchnorm_last is None\n                    else self.mlp_batchnorm_last\n                ),\n                linear_first=(\n                    False if self.mlp_linear_first is None else self.mlp_linear_first\n                ),\n            )\n        else:\n            self.mlp = None\n\n    def forward(self, X: Tensor) -&gt; Tensor:\n        x = self._get_embeddings(X)\n        x = self.encoder(x)\n        if self.with_cls_token:\n            x = x[:, 0, :]\n        else:\n            x = x.flatten(1)\n        if self.mlp is not None:\n            x = self.mlp(x)\n        return x\n\n    @property\n    def output_dim(self) -&gt; int:\n        r\"\"\"The output dimension of the model. This is a required property\n        neccesary to build the `WideDeep` class\n        \"\"\"\n        return (\n            self.mlp_hidden_dims[-1]\n            if self.mlp_hidden_dims is not None\n            else self.mlp_first_hidden_dim\n        )\n\n    @property\n    def attention_weights(self) -&gt; List[Tuple[Tensor, Tensor]]:\n        r\"\"\"List with the attention weights. Each element of the list is a tuple\n        where the first and the second elements are the column and row\n        attention weights respectively\n\n        The shape of the attention weights is:\n\n        - column attention: $(N, H, F, F)$\n\n        - row attention: $(1, H, N, N)$\n\n        where $N$ is the batch size, $H$ is the number of heads and $F$ is the\n        number of features/columns in the dataset\n        \"\"\"\n        attention_weights = []\n        for blk in self.encoder:\n            attention_weights.append(\n                (blk.col_attn.attn_weights, blk.row_attn.attn_weights)\n            )\n        return attention_weights\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.transformers.saint.SAINT.output_dim","title":"output_dim  <code>property</code>","text":"<pre><code>output_dim\n</code></pre> <p>The output dimension of the model. This is a required property neccesary to build the <code>WideDeep</code> class</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.transformers.saint.SAINT.attention_weights","title":"attention_weights  <code>property</code>","text":"<pre><code>attention_weights\n</code></pre> <p>List with the attention weights. Each element of the list is a tuple where the first and the second elements are the column and row attention weights respectively</p> <p>The shape of the attention weights is:</p> <ul> <li> <p>column attention: \\((N, H, F, F)\\)</p> </li> <li> <p>row attention: \\((1, H, N, N)\\)</p> </li> </ul> <p>where \\(N\\) is the batch size, \\(H\\) is the number of heads and \\(F\\) is the number of features/columns in the dataset</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.transformers.ft_transformer.FTTransformer","title":"FTTransformer","text":"<p>               Bases: <code>BaseTabularModelWithAttention</code></p> <p>Defines a FTTransformer model that can be used as the <code>deeptabular</code> component of a Wide &amp; Deep model or independently by itself.</p> <p>Most of the parameters for this class are <code>Optional</code> since the use of categorical or continuous is in fact optional (i.e. one can use categorical features only, continuous features only or both).</p> <p>Parameters:</p> Name Type Description Default <code>column_idx</code> <code>Dict[str, int]</code> <p>Dict containing the index of the columns that will be passed through the <code>TabMlp</code> model. Required to slice the tensors. e.g. {'education': 0, 'relationship': 1, 'workclass': 2, ...}.</p> required <code>cat_embed_input</code> <code>Optional[List[Tuple[str, int]]]</code> <p>List of Tuples with the column name and number of unique values and embedding dimension. e.g. [(education, 11), ...]</p> <code>None</code> <code>cat_embed_dropout</code> <code>Optional[float]</code> <p>Categorical embeddings dropout. If <code>None</code>, it will default to 0.</p> <code>None</code> <code>use_cat_bias</code> <code>Optional[bool]</code> <p>Boolean indicating if bias will be used for the categorical embeddings. If <code>None</code>, it will default to 'False'.</p> <code>None</code> <code>cat_embed_activation</code> <code>Optional[str]</code> <p>Activation function for the categorical embeddings, if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> <code>None</code> <code>shared_embed</code> <code>Optional[bool]</code> <p>Boolean indicating if the embeddings will be \"shared\". The idea behind <code>shared_embed</code> is described in the Appendix A in the TabTransformer paper: 'The goal of having column embedding is to enable the model to distinguish the classes in one column from those in the other columns'. In other words, the idea is to let the model learn which column is embedded at the time. See: <code>pytorch_widedeep.models.transformers._layers.SharedEmbeddings</code>.</p> <code>None</code> <code>add_shared_embed</code> <code>Optional[bool]</code> <p>The two embedding sharing strategies are: 1) add the shared embeddings to the column embeddings or 2) to replace the first <code>frac_shared_embed</code> with the shared embeddings. See <code>pytorch_widedeep.models.embeddings_layers.SharedEmbeddings</code> If 'None' is passed, it will default to 'False'.</p> <code>None</code> <code>frac_shared_embed</code> <code>Optional[float]</code> <p>The fraction of embeddings that will be shared (if <code>add_shared_embed = False</code>) by all the different categories for one particular column. If 'None' is passed, it will default to 0.0.</p> <code>None</code> <code>continuous_cols</code> <code>Optional[List[str]]</code> <p>List with the name of the numeric (aka continuous) columns</p> <code>None</code> <code>cont_norm_layer</code> <code>Optional[Literal[batchnorm, layernorm]]</code> <p>Type of normalization layer applied to the continuous features. Options are: 'layernorm' and 'batchnorm'. if <code>None</code>, no normalization layer will be used.</p> <code>None</code> <code>embed_continuous_method</code> <code>Optional[Literal[standard, piecewise, periodic]]</code> <p>Method to use to embed the continuous features. Options are: 'standard', 'periodic' or 'piecewise'. The 'standard' embedding method is based on the FT-Transformer implementation presented in the paper: Revisiting Deep Learning Models for Tabular Data. The 'periodic' and_'piecewise'_ methods were presented in the paper: On Embeddings for Numerical Features in Tabular Deep Learning. Please, read the papers for details.</p> <code>'standard'</code> <code>cont_embed_dropout</code> <code>Optional[float]</code> <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p> <code>None</code> <code>cont_embed_activation</code> <code>Optional[str]</code> <p>Activation function for the continuous embeddings if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported. If <code>None</code>, no activation function will be applied.</p> <code>None</code> <code>quantization_setup</code> <code>Optional[Dict[str, List[float]]]</code> <p>This parameter is used when the 'piecewise' method is used to embed the continuous cols. It is a dict where keys are the name of the continuous columns and values are lists with the boundaries for the quantization of the continuous_cols. See the examples for details. If If the 'piecewise' method is used, this parameter is required.</p> <code>None</code> <code>n_frequencies</code> <code>Optional[int]</code> <p>This is the so called 'k' in their paper On Embeddings for Numerical Features in Tabular Deep Learning, and is the number of 'frequencies' that will be used to represent each continuous column. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>sigma</code> <code>Optional[float]</code> <p>This is the sigma parameter in the paper mentioned when describing the previous parameters and it is used to initialise the 'frequency weights'. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>share_last_layer</code> <code>Optional[bool]</code> <p>This parameter is not present in the before mentioned paper but it is implemented in the official repo. If <code>True</code> the linear layer that turns the frequencies into embeddings will be shared across the continuous columns. If <code>False</code> a different linear layer will be used for each continuous column. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>full_embed_dropout</code> <code>Optional[bool]</code> <p>If <code>True</code>, the full embedding corresponding to a column will be masked out/dropout. If <code>None</code>, it will default to <code>False</code>.</p> <code>None</code> <code>input_dim</code> <code>int</code> <p>The so-called dimension of the model. Is the number of embeddings used to encode the categorical and/or continuous columns.</p> <code>64</code> <code>kv_compression_factor</code> <code>float</code> <p>By default, the FTTransformer uses Linear Attention (See Linformer: Self-Attention with Linear Complexity ). The compression factor that will be used to reduce the input sequence length. If we denote the resulting sequence length as \\(k = int(kv_{compression \\space factor} \\times s)\\) where \\(s\\) is the input sequence length.</p> <code>0.5</code> <code>kv_sharing</code> <code>bool</code> <p>Boolean indicating if the \\(E\\) and \\(F\\) projection matrices will share weights.  See Linformer: Self-Attention with Linear Complexity for details</p> <code>False</code> <code>n_heads</code> <code>int</code> <p>Number of attention heads per FTTransformer block</p> <code>8</code> <code>use_qkv_bias</code> <code>bool</code> <p>Boolean indicating whether or not to use bias in the Q, K, and V projection layers</p> <code>False</code> <code>n_blocks</code> <code>int</code> <p>Number of FTTransformer blocks</p> <code>4</code> <code>attn_dropout</code> <code>float</code> <p>Dropout that will be applied to the Linear-Attention layers</p> <code>0.2</code> <code>ff_dropout</code> <code>float</code> <p>Dropout that will be applied to the FeedForward network</p> <code>0.1</code> <code>ff_factor</code> <code>float</code> <p>Multiplicative factor applied to the first layer of the FF network in each Transformer block, This is normally set to 4, but they use 4/3 in the paper.</p> <code>1.33</code> <code>transformer_activation</code> <code>str</code> <p>Transformer Encoder activation function. 'tanh', 'relu', 'leaky_relu', 'gelu', 'geglu' and 'reglu' are supported</p> <code>'reglu'</code> <code>mlp_hidden_dims</code> <code>Optional[List[int]]</code> <p>List with the number of neurons per dense layer in the MLP. e.g: [64, 32]. If not provided no MLP on top of the final FTTransformer block will be used.</p> <code>None</code> <code>mlp_activation</code> <code>Optional[str]</code> <p>Activation function for the dense layers of the MLP. Currently 'tanh', 'relu', 'leaky'_relu' and _'gelu' are supported. If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to 'relu'.</p> <code>None</code> <code>mlp_dropout</code> <code>Optional[float]</code> <p>float with the dropout between the dense layers of the MLP. If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to 0.0.</p> <code>None</code> <code>mlp_batchnorm</code> <code>Optional[bool]</code> <p>Boolean indicating whether or not batch normalization will be applied to the dense layers If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to False.</p> <code>None</code> <code>mlp_batchnorm_last</code> <code>Optional[bool]</code> <p>Boolean indicating whether or not batch normalization will be applied to the last of the dense layers If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to False.</p> <code>None</code> <code>mlp_linear_first</code> <code>Optional[bool]</code> <p>Boolean indicating the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code> If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to <code>True</code>.</p> <code>None</code> <p>Attributes:</p> Name Type Description <code>encoder</code> <code>Module</code> <p>Sequence of FTTransformer blocks</p> <code>mlp</code> <code>Module</code> <p>MLP component in the model</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import FTTransformer\n&gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n&gt;&gt;&gt; colnames = ['a', 'b', 'c', 'd', 'e']\n&gt;&gt;&gt; cat_embed_input = [(u,i) for u,i in zip(colnames[:4], [4]*4)]\n&gt;&gt;&gt; continuous_cols = ['e']\n&gt;&gt;&gt; column_idx = {k:v for v,k in enumerate(colnames)}\n&gt;&gt;&gt; model = FTTransformer(column_idx=column_idx, cat_embed_input=cat_embed_input, continuous_cols=continuous_cols)\n&gt;&gt;&gt; out = model(X_tab)\n</code></pre> Source code in <code>pytorch_widedeep/models/tabular/transformers/ft_transformer.py</code> <pre><code>class FTTransformer(BaseTabularModelWithAttention):\n    r\"\"\"Defines a [FTTransformer model](https://arxiv.org/abs/2106.11959) that\n    can be used as the `deeptabular` component of a Wide &amp; Deep model or\n    independently by itself.\n\n    Most of the parameters for this class are `Optional` since the use of\n    categorical or continuous is in fact optional (i.e. one can use\n    categorical features only, continuous features only or both).\n\n    Parameters\n    ----------\n    column_idx: Dict\n        Dict containing the index of the columns that will be passed through\n        the `TabMlp` model. Required to slice the tensors. e.g. _{'education':\n        0, 'relationship': 1, 'workclass': 2, ...}_.\n    cat_embed_input: List, Optional, default = None\n        List of Tuples with the column name and number of unique values and\n        embedding dimension. e.g. _[(education, 11), ...]_\n    cat_embed_dropout: float, Optional, default = None\n        Categorical embeddings dropout. If `None`, it will default\n        to 0.\n    use_cat_bias: bool, Optional, default = None,\n        Boolean indicating if bias will be used for the categorical embeddings.\n        If `None`, it will default to 'False'.\n    cat_embed_activation: Optional, str, default = None,\n        Activation function for the categorical embeddings, if any. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported\n    shared_embed: bool, Optional, default = None\n        Boolean indicating if the embeddings will be \"shared\". The idea behind `shared_embed` is\n        described in the Appendix A in the [TabTransformer paper](https://arxiv.org/abs/2012.06678):\n        _'The goal of having column embedding is to enable the model to\n        distinguish the classes in one column from those in the other\n        columns'_. In other words, the idea is to let the model learn which\n        column is embedded at the time. See: `pytorch_widedeep.models.transformers._layers.SharedEmbeddings`.\n    add_shared_embed: bool, Optional, default = None\n        The two embedding sharing strategies are: 1) add the shared embeddings\n        to the column embeddings or 2) to replace the first\n        `frac_shared_embed` with the shared embeddings.\n        See `pytorch_widedeep.models.embeddings_layers.SharedEmbeddings`\n        If 'None' is passed, it will default to 'False'.\n    frac_shared_embed: float, Optional, default = None\n        The fraction of embeddings that will be shared (if `add_shared_embed\n        = False`) by all the different categories for one particular\n        column. If 'None' is passed, it will default to 0.0.\n    continuous_cols: List, Optional, default = None\n        List with the name of the numeric (aka continuous) columns\n    cont_norm_layer: str, Optional, default =  None\n        Type of normalization layer applied to the continuous features.\n        Options are: _'layernorm'_ and _'batchnorm'_. if `None`, no\n        normalization layer will be used.\n    embed_continuous_method: Optional, str, default = None,\n        Method to use to embed the continuous features. Options are:\n        _'standard'_, _'periodic'_ or _'piecewise'_. The _'standard'_\n        embedding method is based on the FT-Transformer implementation\n        presented in the paper: [Revisiting Deep Learning Models for\n        Tabular Data](https://arxiv.org/abs/2106.11959v5). The _'periodic'_\n        and_'piecewise'_ methods were presented in the paper: [On Embeddings for\n        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556).\n        Please, read the papers for details.\n    cont_embed_dropout: float, Optional, default = None,\n        Dropout for the continuous embeddings. If `None`, it will default to 0.0\n    cont_embed_activation: Optional, str, default = None,\n        Activation function for the continuous embeddings if any. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported.\n        If `None`, no activation function will be applied.\n    quantization_setup: Dict[str, List[float]], Optional, default = None,\n        This parameter is used when the _'piecewise'_ method is used to embed\n        the continuous cols. It is a dict where keys are the name of the continuous\n        columns and values are lists with the boundaries for the quantization\n        of the continuous_cols. See the examples for details. If\n        If the _'piecewise'_ method is used, this parameter is required.\n    n_frequencies: int, Optional, default = None,\n        This is the so called _'k'_ in their paper [On Embeddings for\n        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556),\n        and is the number of 'frequencies' that will be used to represent each\n        continuous column. See their Eq 2 in the paper for details. If\n        the _'periodic'_ method is used, this parameter is required.\n    sigma: float, Optional, default = None,\n        This is the sigma parameter in the paper mentioned when describing the\n        previous parameters and it is used to initialise the 'frequency\n        weights'. See their Eq 2 in the paper for details. If\n        the _'periodic'_ method is used, this parameter is required.\n    share_last_layer: bool, Optional, default = None,\n        This parameter is not present in the before mentioned paper but it is implemented in\n        the [official repo](https://github.com/yandex-research/rtdl-num-embeddings/tree/main).\n        If `True` the linear layer that turns the frequencies into embeddings\n        will be shared across the continuous columns. If `False` a different\n        linear layer will be used for each continuous column.\n        If the _'periodic'_ method is used, this parameter is required.\n    full_embed_dropout: bool, Optional, default = None,\n        If `True`, the full embedding corresponding to a column will be masked\n        out/dropout. If `None`, it will default to `False`.\n    input_dim: int, default = 64\n        The so-called *dimension of the model*. Is the number of embeddings used to encode\n        the categorical and/or continuous columns.\n    kv_compression_factor: int, default = 0.5\n        By default, the FTTransformer uses Linear Attention\n        (See [Linformer: Self-Attention with Linear Complexity](https://arxiv.org/abs/2006.04768&gt;) ).\n        The compression factor that will be used to reduce the input sequence\n        length. If we denote the resulting sequence length as\n        $k = int(kv_{compression \\space factor} \\times s)$\n        where $s$ is the input sequence length.\n    kv_sharing: bool, default = False\n        Boolean indicating if the $E$ and $F$ projection matrices\n        will share weights.  See [Linformer: Self-Attention with Linear\n        Complexity](https://arxiv.org/abs/2006.04768) for details\n    n_heads: int, default = 8\n        Number of attention heads per FTTransformer block\n    use_qkv_bias: bool, default = False\n        Boolean indicating whether or not to use bias in the Q, K, and V\n        projection layers\n    n_blocks: int, default = 4\n        Number of FTTransformer blocks\n    attn_dropout: float, default = 0.2\n        Dropout that will be applied to the Linear-Attention layers\n    ff_dropout: float, default = 0.1\n        Dropout that will be applied to the FeedForward network\n    ff_factor: float, default = 4 / 3\n        Multiplicative factor applied to the first layer of the FF network in\n        each Transformer block, This is normally set to 4, but they use 4/3\n        in the paper.\n    transformer_activation: str, default = \"gelu\"\n        Transformer Encoder activation function. _'tanh'_, _'relu'_,\n        _'leaky_relu'_, _'gelu'_, _'geglu'_ and _'reglu'_ are supported\n    mlp_hidden_dims: List, Optional, default = None\n        List with the number of neurons per dense layer in the MLP. e.g:\n        _[64, 32]_. If not provided no MLP on top of the final\n        FTTransformer block will be used.\n    mlp_activation: str, Optional, default = None\n        Activation function for the dense layers of the MLP. Currently\n        _'tanh'_, _'relu'_, _'leaky'_relu' and _'gelu'_ are supported.\n        If 'mlp_hidden_dims' is not `None` and this parameter is `None`, it\n        will default to _'relu'_.\n    mlp_dropout: float, Optional, default = None\n        float with the dropout between the dense layers of the MLP.\n        If 'mlp_hidden_dims' is not `None` and this parameter is `None`, it\n        will default to 0.0.\n    mlp_batchnorm: bool, Optional, default = None\n        Boolean indicating whether or not batch normalization will be applied\n        to the dense layers\n        If 'mlp_hidden_dims' is not `None` and this parameter is `None`, it\n        will default to False.\n    mlp_batchnorm_last: bool, Optional, default = None\n        Boolean indicating whether or not batch normalization will be applied\n        to the last of the dense layers\n        If 'mlp_hidden_dims' is not `None` and this parameter is `None`, it\n        will default to False.\n    mlp_linear_first: bool, Optional, default = None\n        Boolean indicating the order of the operations in the dense\n        layer. If `True: [LIN -&gt; ACT -&gt; BN -&gt; DP]`. If `False: [BN -&gt; DP -&gt;\n        LIN -&gt; ACT]`\n        If 'mlp_hidden_dims' is not `None` and this parameter is `None`, it\n        will default to `True`.\n\n    Attributes\n    ----------\n    encoder: nn.Module\n        Sequence of FTTransformer blocks\n    mlp: nn.Module\n        MLP component in the model\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.models import FTTransformer\n    &gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n    &gt;&gt;&gt; colnames = ['a', 'b', 'c', 'd', 'e']\n    &gt;&gt;&gt; cat_embed_input = [(u,i) for u,i in zip(colnames[:4], [4]*4)]\n    &gt;&gt;&gt; continuous_cols = ['e']\n    &gt;&gt;&gt; column_idx = {k:v for v,k in enumerate(colnames)}\n    &gt;&gt;&gt; model = FTTransformer(column_idx=column_idx, cat_embed_input=cat_embed_input, continuous_cols=continuous_cols)\n    &gt;&gt;&gt; out = model(X_tab)\n    \"\"\"\n\n    def __init__(\n        self,\n        column_idx: Dict[str, int],\n        *,\n        cat_embed_input: Optional[List[Tuple[str, int]]] = None,\n        cat_embed_dropout: Optional[float] = None,\n        use_cat_bias: Optional[bool] = None,\n        cat_embed_activation: Optional[str] = None,\n        shared_embed: Optional[bool] = None,\n        add_shared_embed: Optional[bool] = None,\n        frac_shared_embed: Optional[float] = None,\n        continuous_cols: Optional[List[str]] = None,\n        cont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\n        embed_continuous_method: Optional[\n            Literal[\"standard\", \"piecewise\", \"periodic\"]\n        ] = \"standard\",\n        cont_embed_dropout: Optional[float] = None,\n        cont_embed_activation: Optional[str] = None,\n        quantization_setup: Optional[Dict[str, List[float]]] = None,\n        n_frequencies: Optional[int] = None,\n        sigma: Optional[float] = None,\n        share_last_layer: Optional[bool] = None,\n        full_embed_dropout: Optional[bool] = None,\n        input_dim: int = 64,\n        kv_compression_factor: float = 0.5,\n        kv_sharing: bool = False,\n        use_qkv_bias: bool = False,\n        n_heads: int = 8,\n        n_blocks: int = 4,\n        attn_dropout: float = 0.2,\n        ff_dropout: float = 0.1,\n        ff_factor: float = 1.33,\n        transformer_activation: str = \"reglu\",\n        mlp_hidden_dims: Optional[List[int]] = None,\n        mlp_activation: Optional[str] = None,\n        mlp_dropout: Optional[float] = None,\n        mlp_batchnorm: Optional[bool] = None,\n        mlp_batchnorm_last: Optional[bool] = None,\n        mlp_linear_first: Optional[bool] = None,\n    ):\n        super(FTTransformer, self).__init__(\n            column_idx=column_idx,\n            cat_embed_input=cat_embed_input,\n            cat_embed_dropout=cat_embed_dropout,\n            use_cat_bias=use_cat_bias,\n            cat_embed_activation=cat_embed_activation,\n            shared_embed=shared_embed,\n            add_shared_embed=add_shared_embed,\n            frac_shared_embed=frac_shared_embed,\n            continuous_cols=continuous_cols,\n            cont_norm_layer=cont_norm_layer,\n            embed_continuous_method=embed_continuous_method,\n            cont_embed_dropout=cont_embed_dropout,\n            cont_embed_activation=cont_embed_activation,\n            input_dim=input_dim,\n            quantization_setup=quantization_setup,\n            n_frequencies=n_frequencies,\n            sigma=sigma,\n            share_last_layer=share_last_layer,\n            full_embed_dropout=full_embed_dropout,\n        )\n\n        self.kv_compression_factor = kv_compression_factor\n        self.kv_sharing = kv_sharing\n        self.use_qkv_bias = use_qkv_bias\n        self.n_heads = n_heads\n        self.n_blocks = n_blocks\n        self.attn_dropout = attn_dropout\n        self.ff_dropout = ff_dropout\n        self.ff_factor = ff_factor\n        self.transformer_activation = transformer_activation\n\n        self.mlp_hidden_dims = mlp_hidden_dims\n        self.mlp_activation = mlp_activation\n        self.mlp_dropout = mlp_dropout\n        self.mlp_batchnorm = mlp_batchnorm\n        self.mlp_batchnorm_last = mlp_batchnorm_last\n        self.mlp_linear_first = mlp_linear_first\n\n        self.with_cls_token = \"cls_token\" in column_idx\n        self.n_cat = len(cat_embed_input) if cat_embed_input is not None else 0\n        self.n_cont = len(continuous_cols) if continuous_cols is not None else 0\n        self.n_feats = self.n_cat + self.n_cont\n\n        # Embeddings are instantiated at the base model\n        # Transformer blocks\n        is_first = True\n        self.encoder = nn.Sequential()\n        for i in range(n_blocks):\n            self.encoder.add_module(\n                \"fttransformer_block\" + str(i),\n                FTTransformerEncoder(\n                    input_dim,\n                    self.n_feats,\n                    n_heads,\n                    use_qkv_bias,\n                    attn_dropout,\n                    ff_dropout,\n                    ff_factor,\n                    kv_compression_factor,\n                    kv_sharing,\n                    transformer_activation,\n                    is_first,\n                ),\n            )\n            is_first = False\n\n        self.mlp_first_hidden_dim = (\n            self.input_dim if self.with_cls_token else (self.n_feats * self.input_dim)\n        )\n\n        # Mlp: adding an MLP on top of the Resnet blocks is optional and\n        # therefore all related params are optional\n        if self.mlp_hidden_dims is not None:\n            self.mlp = MLP(\n                d_hidden=[self.mlp_first_hidden_dim] + self.mlp_hidden_dims,\n                activation=(\n                    \"relu\" if self.mlp_activation is None else self.mlp_activation\n                ),\n                dropout=0.0 if self.mlp_dropout is None else self.mlp_dropout,\n                batchnorm=False if self.mlp_batchnorm is None else self.mlp_batchnorm,\n                batchnorm_last=(\n                    False\n                    if self.mlp_batchnorm_last is None\n                    else self.mlp_batchnorm_last\n                ),\n                linear_first=(\n                    False if self.mlp_linear_first is None else self.mlp_linear_first\n                ),\n            )\n        else:\n            self.mlp = None\n\n    def forward(self, X: Tensor) -&gt; Tensor:\n        x = self._get_embeddings(X)\n        x = self.encoder(x)\n        if self.with_cls_token:\n            x = x[:, 0, :]\n        else:\n            x = x.flatten(1)\n        if self.mlp is not None:\n            x = self.mlp(x)\n        return x\n\n    @property\n    def output_dim(self) -&gt; int:\n        r\"\"\"The output dimension of the model. This is a required property\n        neccesary to build the `WideDeep` class\n        \"\"\"\n        return (\n            self.mlp_hidden_dims[-1]\n            if self.mlp_hidden_dims is not None\n            else self.mlp_first_hidden_dim\n        )\n\n    @property\n    def attention_weights(self) -&gt; List[Tensor]:\n        r\"\"\"List with the attention weights per block\n\n        The shape of the attention weights is: $(N, H, F, k)$, where $N$ is\n        the batch size, $H$ is the number of attention heads, $F$ is the\n        number of features/columns and $k$ is the reduced sequence length or\n        dimension, i.e. $k = int(kv_{compression \\space factor} \\times s)$\n        \"\"\"\n        return [blk.attn.attn_weights for blk in self.encoder]\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.transformers.ft_transformer.FTTransformer.output_dim","title":"output_dim  <code>property</code>","text":"<pre><code>output_dim\n</code></pre> <p>The output dimension of the model. This is a required property neccesary to build the <code>WideDeep</code> class</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.transformers.ft_transformer.FTTransformer.attention_weights","title":"attention_weights  <code>property</code>","text":"<pre><code>attention_weights\n</code></pre> <p>List with the attention weights per block</p> <p>The shape of the attention weights is: \\((N, H, F, k)\\), where \\(N\\) is the batch size, \\(H\\) is the number of attention heads, \\(F\\) is the number of features/columns and \\(k\\) is the reduced sequence length or dimension, i.e. \\(k = int(kv_{compression \\space factor} \\times s)\\)</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.transformers.tab_perceiver.TabPerceiver","title":"TabPerceiver","text":"<p>               Bases: <code>BaseTabularModelWithAttention</code></p> <p>Defines an adaptation of a Perceiver  that can be used as the <code>deeptabular</code> component of a Wide &amp; Deep model  or independently by itself.</p> <p>Most of the parameters for this class are <code>Optional</code> since the use of categorical or continuous is in fact optional (i.e. one can use categorical features only, continuous features only or both).</p> <p> NOTE: while there are scientific publications for  the <code>TabTransformer</code>, <code>SAINT</code> and <code>FTTransformer</code>, the <code>TabPerceiver</code>  and the <code>TabFastFormer</code> are our own adaptations of the  Perceiver and the  FastFormer for tabular data.</p> <p>Parameters:</p> Name Type Description Default <code>column_idx</code> <code>Dict[str, int]</code> <p>Dict containing the index of the columns that will be passed through the <code>TabMlp</code> model. Required to slice the tensors. e.g. {'education': 0, 'relationship': 1, 'workclass': 2, ...}.</p> required <code>cat_embed_input</code> <code>Optional[List[Tuple[str, int]]]</code> <p>List of Tuples with the column name and number of unique values and embedding dimension. e.g. [(education, 11), ...]</p> <code>None</code> <code>cat_embed_dropout</code> <code>Optional[float]</code> <p>Categorical embeddings dropout. If <code>None</code>, it will default to 0.</p> <code>None</code> <code>use_cat_bias</code> <code>Optional[bool]</code> <p>Boolean indicating if bias will be used for the categorical embeddings. If <code>None</code>, it will default to 'False'.</p> <code>None</code> <code>cat_embed_activation</code> <code>Optional[str]</code> <p>Activation function for the categorical embeddings, if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> <code>None</code> <code>shared_embed</code> <code>Optional[bool]</code> <p>Boolean indicating if the embeddings will be \"shared\". The idea behind <code>shared_embed</code> is described in the Appendix A in the TabTransformer paper: 'The goal of having column embedding is to enable the model to distinguish the classes in one column from those in the other columns'. In other words, the idea is to let the model learn which column is embedded at the time. See: <code>pytorch_widedeep.models.transformers._layers.SharedEmbeddings</code>.</p> <code>None</code> <code>add_shared_embed</code> <code>Optional[bool]</code> <p>The two embedding sharing strategies are: 1) add the shared embeddings to the column embeddings or 2) to replace the first <code>frac_shared_embed</code> with the shared embeddings. See <code>pytorch_widedeep.models.embeddings_layers.SharedEmbeddings</code> If 'None' is passed, it will default to 'False'.</p> <code>None</code> <code>frac_shared_embed</code> <code>Optional[float]</code> <p>The fraction of embeddings that will be shared (if <code>add_shared_embed = False</code>) by all the different categories for one particular column. If 'None' is passed, it will default to 0.0.</p> <code>None</code> <code>continuous_cols</code> <code>Optional[List[str]]</code> <p>List with the name of the numeric (aka continuous) columns</p> <code>None</code> <code>cont_norm_layer</code> <code>Optional[Literal[batchnorm, layernorm]]</code> <p>Type of normalization layer applied to the continuous features. Options are: 'layernorm' and 'batchnorm'. if <code>None</code>, no normalization layer will be used.</p> <code>None</code> <code>embed_continuous_method</code> <code>Optional[Literal[standard, piecewise, periodic]]</code> <p>Method to use to embed the continuous features. Options are: 'standard', 'periodic' or 'piecewise'. The 'standard' embedding method is based on the FT-Transformer implementation presented in the paper: Revisiting Deep Learning Models for Tabular Data. The 'periodic' and_'piecewise'_ methods were presented in the paper: On Embeddings for Numerical Features in Tabular Deep Learning. Please, read the papers for details.</p> <code>'standard'</code> <code>cont_embed_dropout</code> <code>Optional[float]</code> <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p> <code>None</code> <code>cont_embed_activation</code> <code>Optional[str]</code> <p>Activation function for the continuous embeddings if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported. If <code>None</code>, no activation function will be applied.</p> <code>None</code> <code>quantization_setup</code> <code>Optional[Dict[str, List[float]]]</code> <p>This parameter is used when the 'piecewise' method is used to embed the continuous cols. It is a dict where keys are the name of the continuous columns and values are lists with the boundaries for the quantization of the continuous_cols. See the examples for details. If If the 'piecewise' method is used, this parameter is required.</p> <code>None</code> <code>n_frequencies</code> <code>Optional[int]</code> <p>This is the so called 'k' in their paper On Embeddings for Numerical Features in Tabular Deep Learning, and is the number of 'frequencies' that will be used to represent each continuous column. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>sigma</code> <code>Optional[float]</code> <p>This is the sigma parameter in the paper mentioned when describing the previous parameters and it is used to initialise the 'frequency weights'. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>share_last_layer</code> <code>Optional[bool]</code> <p>This parameter is not present in the before mentioned paper but it is implemented in the official repo. If <code>True</code> the linear layer that turns the frequencies into embeddings will be shared across the continuous columns. If <code>False</code> a different linear layer will be used for each continuous column. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>full_embed_dropout</code> <code>Optional[bool]</code> <p>If <code>True</code>, the full embedding corresponding to a column will be masked out/dropout. If <code>None</code>, it will default to <code>False</code>.</p> <code>None</code> <code>input_dim</code> <code>int</code> <p>The so-called dimension of the model. Is the number of embeddings used to encode the categorical and/or continuous columns.</p> <code>32</code> <code>n_cross_attns</code> <code>int</code> <p>Number of times each perceiver block will cross attend to the input data (i.e. number of cross attention components per perceiver block). This should normally be 1. However, in the paper they describe some architectures (normally computer vision-related problems) where the Perceiver attends multiple times to the input array. Therefore, maybe multiple cross attention to the input array is also useful in some cases for tabular data  .</p> <code>1</code> <code>n_cross_attn_heads</code> <code>int</code> <p>Number of attention heads for the cross attention component</p> <code>4</code> <code>n_latents</code> <code>int</code> <p>Number of latents. This is the \\(N\\) parameter in the paper. As indicated in the paper, this number should be significantly lower than \\(M\\) (the number of columns in the dataset). Setting \\(N\\) closer to \\(M\\) defies the main purpose of the Perceiver, which is to overcome the transformer quadratic bottleneck</p> <code>16</code> <code>latent_dim</code> <code>int</code> <p>Latent dimension.</p> <code>128</code> <code>n_latent_heads</code> <code>int</code> <p>Number of attention heads per Latent Transformer</p> <code>4</code> <code>n_latent_blocks</code> <code>int</code> <p>Number of transformer encoder blocks (normalised MHA + normalised FF) per Latent Transformer</p> <code>4</code> <code>n_perceiver_blocks</code> <code>int</code> <p>Number of Perceiver blocks defined as [Cross Attention + Latent Transformer]</p> <code>4</code> <code>share_weights</code> <code>bool</code> <p>Boolean indicating if the weights will be shared between Perceiver blocks</p> <code>False</code> <code>attn_dropout</code> <code>float</code> <p>Dropout that will be applied to the Multi-Head Attention layers</p> <code>0.1</code> <code>ff_dropout</code> <code>float</code> <p>Dropout that will be applied to the FeedForward network</p> <code>0.1</code> <code>ff_factor</code> <code>int</code> <p>Multiplicative factor applied to the first layer of the FF network in each Transformer block, This is normally set to 4.</p> <code>4</code> <code>transformer_activation</code> <code>str</code> <p>Transformer Encoder activation function. 'tanh', 'relu', 'leaky_relu', 'gelu', 'geglu' and 'reglu' are supported</p> <code>'geglu'</code> <code>mlp_hidden_dims</code> <code>Optional[List[int]]</code> <p>List with the number of neurons per dense layer in the MLP. e.g: [64, 32]. If not provided no MLP on top of the final Transformer block will be used.</p> <code>None</code> <code>mlp_activation</code> <code>Optional[str]</code> <p>Activation function for the dense layers of the MLP. Currently 'tanh', 'relu', 'leaky'_relu' and _'gelu' are supported. If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to 'relu'.</p> <code>None</code> <code>mlp_dropout</code> <code>Optional[float]</code> <p>float with the dropout between the dense layers of the MLP. If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to 0.0.</p> <code>None</code> <code>mlp_batchnorm</code> <code>Optional[bool]</code> <p>Boolean indicating whether or not batch normalization will be applied to the dense layers If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to False.</p> <code>None</code> <code>mlp_batchnorm_last</code> <code>Optional[bool]</code> <p>Boolean indicating whether or not batch normalization will be applied to the last of the dense layers If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to False.</p> <code>None</code> <code>mlp_linear_first</code> <code>Optional[bool]</code> <p>Boolean indicating the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code> If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to <code>True</code>.</p> <code>None</code> <p>Attributes:</p> Name Type Description <code>encoder</code> <code>ModuleDict</code> <p>ModuleDict with the Perceiver blocks</p> <code>latents</code> <code>Parameter</code> <p>Latents that will be used for prediction</p> <code>mlp</code> <code>Module</code> <p>MLP component in the model</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import TabPerceiver\n&gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n&gt;&gt;&gt; colnames = ['a', 'b', 'c', 'd', 'e']\n&gt;&gt;&gt; cat_embed_input = [(u,i) for u,i in zip(colnames[:4], [4]*4)]\n&gt;&gt;&gt; continuous_cols = ['e']\n&gt;&gt;&gt; column_idx = {k:v for v,k in enumerate(colnames)}\n&gt;&gt;&gt; model = TabPerceiver(column_idx=column_idx, cat_embed_input=cat_embed_input,\n... continuous_cols=continuous_cols, n_latents=2, latent_dim=16,\n... n_perceiver_blocks=2)\n&gt;&gt;&gt; out = model(X_tab)\n</code></pre> Source code in <code>pytorch_widedeep/models/tabular/transformers/tab_perceiver.py</code> <pre><code>class TabPerceiver(BaseTabularModelWithAttention):\n    r\"\"\"Defines an adaptation of a [Perceiver](https://arxiv.org/abs/2103.03206)\n     that can be used as the `deeptabular` component of a Wide &amp; Deep model\n     or independently by itself.\n\n    Most of the parameters for this class are `Optional` since the use of\n    categorical or continuous is in fact optional (i.e. one can use\n    categorical features only, continuous features only or both).\n\n    :information_source: **NOTE**: while there are scientific publications for\n     the `TabTransformer`, `SAINT` and `FTTransformer`, the `TabPerceiver`\n     and the `TabFastFormer` are our own adaptations of the\n     [Perceiver](https://arxiv.org/abs/2103.03206) and the\n     [FastFormer](https://arxiv.org/abs/2108.09084) for tabular data.\n\n    Parameters\n    ----------\n    column_idx: Dict\n        Dict containing the index of the columns that will be passed through\n        the `TabMlp` model. Required to slice the tensors. e.g. _{'education':\n        0, 'relationship': 1, 'workclass': 2, ...}_.\n    cat_embed_input: List, Optional, default = None\n        List of Tuples with the column name and number of unique values and\n        embedding dimension. e.g. _[(education, 11), ...]_\n    cat_embed_dropout: float, Optional, default = None\n        Categorical embeddings dropout. If `None`, it will default\n        to 0.\n    use_cat_bias: bool, Optional, default = None,\n        Boolean indicating if bias will be used for the categorical embeddings.\n        If `None`, it will default to 'False'.\n    cat_embed_activation: Optional, str, default = None,\n        Activation function for the categorical embeddings, if any. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported\n    shared_embed: bool, Optional, default = None\n        Boolean indicating if the embeddings will be \"shared\". The idea behind `shared_embed` is\n        described in the Appendix A in the [TabTransformer paper](https://arxiv.org/abs/2012.06678):\n        _'The goal of having column embedding is to enable the model to\n        distinguish the classes in one column from those in the other\n        columns'_. In other words, the idea is to let the model learn which\n        column is embedded at the time. See: `pytorch_widedeep.models.transformers._layers.SharedEmbeddings`.\n    add_shared_embed: bool, Optional, default = None\n        The two embedding sharing strategies are: 1) add the shared embeddings\n        to the column embeddings or 2) to replace the first\n        `frac_shared_embed` with the shared embeddings.\n        See `pytorch_widedeep.models.embeddings_layers.SharedEmbeddings`\n        If 'None' is passed, it will default to 'False'.\n    frac_shared_embed: float, Optional, default = None\n        The fraction of embeddings that will be shared (if `add_shared_embed\n        = False`) by all the different categories for one particular\n        column. If 'None' is passed, it will default to 0.0.\n    continuous_cols: List, Optional, default = None\n        List with the name of the numeric (aka continuous) columns\n    cont_norm_layer: str, Optional, default =  None\n        Type of normalization layer applied to the continuous features.\n        Options are: _'layernorm'_ and _'batchnorm'_. if `None`, no\n        normalization layer will be used.\n    embed_continuous_method: Optional, str, default = None,\n        Method to use to embed the continuous features. Options are:\n        _'standard'_, _'periodic'_ or _'piecewise'_. The _'standard'_\n        embedding method is based on the FT-Transformer implementation\n        presented in the paper: [Revisiting Deep Learning Models for\n        Tabular Data](https://arxiv.org/abs/2106.11959v5). The _'periodic'_\n        and_'piecewise'_ methods were presented in the paper: [On Embeddings for\n        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556).\n        Please, read the papers for details.\n    cont_embed_dropout: float, Optional, default = None,\n        Dropout for the continuous embeddings. If `None`, it will default to 0.0\n    cont_embed_activation: Optional, str, default = None,\n        Activation function for the continuous embeddings if any. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported.\n        If `None`, no activation function will be applied.\n    quantization_setup: Dict[str, List[float]], Optional, default = None,\n        This parameter is used when the _'piecewise'_ method is used to embed\n        the continuous cols. It is a dict where keys are the name of the continuous\n        columns and values are lists with the boundaries for the quantization\n        of the continuous_cols. See the examples for details. If\n        If the _'piecewise'_ method is used, this parameter is required.\n    n_frequencies: int, Optional, default = None,\n        This is the so called _'k'_ in their paper [On Embeddings for\n        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556),\n        and is the number of 'frequencies' that will be used to represent each\n        continuous column. See their Eq 2 in the paper for details. If\n        the _'periodic'_ method is used, this parameter is required.\n    sigma: float, Optional, default = None,\n        This is the sigma parameter in the paper mentioned when describing the\n        previous parameters and it is used to initialise the 'frequency\n        weights'. See their Eq 2 in the paper for details. If\n        the _'periodic'_ method is used, this parameter is required.\n    share_last_layer: bool, Optional, default = None,\n        This parameter is not present in the before mentioned paper but it is implemented in\n        the [official repo](https://github.com/yandex-research/rtdl-num-embeddings/tree/main).\n        If `True` the linear layer that turns the frequencies into embeddings\n        will be shared across the continuous columns. If `False` a different\n        linear layer will be used for each continuous column.\n        If the _'periodic'_ method is used, this parameter is required.\n    full_embed_dropout: bool, Optional, default = None,\n        If `True`, the full embedding corresponding to a column will be masked\n        out/dropout. If `None`, it will default to `False`.\n    input_dim: int, default = 32\n        The so-called *dimension of the model*. Is the number of embeddings\n        used to encode the categorical and/or continuous columns.\n    n_cross_attns: int, default = 1\n        Number of times each perceiver block will cross attend to the input\n        data (i.e. number of cross attention components per perceiver block).\n        This should normally be 1. However, in the paper they describe some\n        architectures (normally computer vision-related problems) where the\n        Perceiver attends multiple times to the input array. Therefore, maybe\n        multiple cross attention to the input array is also useful in some\n        cases for tabular data :shrug: .\n    n_cross_attn_heads: int, default = 4\n        Number of attention heads for the cross attention component\n    n_latents: int, default = 16\n        Number of latents. This is the $N$ parameter in the paper. As\n        indicated in the paper, this number should be significantly lower\n        than $M$ (the number of columns in the dataset). Setting $N$ closer\n        to $M$ defies the main purpose of the Perceiver, which is to overcome\n        the transformer quadratic bottleneck\n    latent_dim: int, default = 128\n        Latent dimension.\n    n_latent_heads: int, default = 4\n        Number of attention heads per Latent Transformer\n    n_latent_blocks: int, default = 4\n        Number of transformer encoder blocks (normalised MHA + normalised FF)\n        per Latent Transformer\n    n_perceiver_blocks: int, default = 4\n        Number of Perceiver blocks defined as [Cross Attention + Latent\n        Transformer]\n    share_weights: Boolean, default = False\n        Boolean indicating if the weights will be shared between Perceiver\n        blocks\n    attn_dropout: float, default = 0.2\n        Dropout that will be applied to the Multi-Head Attention layers\n    ff_dropout: float, default = 0.1\n        Dropout that will be applied to the FeedForward network\n    ff_factor: float, default = 4\n        Multiplicative factor applied to the first layer of the FF network in\n        each Transformer block, This is normally set to 4.\n    transformer_activation: str, default = \"gelu\"\n        Transformer Encoder activation function. _'tanh'_, _'relu'_,\n        _'leaky_relu'_, _'gelu'_, _'geglu'_ and _'reglu'_ are supported\n    mlp_hidden_dims: List, Optional, default = None\n        List with the number of neurons per dense layer in the MLP. e.g:\n        _[64, 32]_. If not provided no MLP on top of the final\n        Transformer block will be used.\n    mlp_activation: str, Optional, default = None\n        Activation function for the dense layers of the MLP. Currently\n        _'tanh'_, _'relu'_, _'leaky'_relu' and _'gelu'_ are supported.\n        If 'mlp_hidden_dims' is not `None` and this parameter is `None`, it\n        will default to _'relu'_.\n    mlp_dropout: float, Optional, default = None\n        float with the dropout between the dense layers of the MLP.\n        If 'mlp_hidden_dims' is not `None` and this parameter is `None`, it\n        will default to 0.0.\n    mlp_batchnorm: bool, Optional, default = None\n        Boolean indicating whether or not batch normalization will be applied\n        to the dense layers\n        If 'mlp_hidden_dims' is not `None` and this parameter is `None`, it\n        will default to False.\n    mlp_batchnorm_last: bool, Optional, default = None\n        Boolean indicating whether or not batch normalization will be applied\n        to the last of the dense layers\n        If 'mlp_hidden_dims' is not `None` and this parameter is `None`, it\n        will default to False.\n    mlp_linear_first: bool, Optional, default = None\n        Boolean indicating the order of the operations in the dense\n        layer. If `True: [LIN -&gt; ACT -&gt; BN -&gt; DP]`. If `False: [BN -&gt; DP -&gt;\n        LIN -&gt; ACT]`\n        If 'mlp_hidden_dims' is not `None` and this parameter is `None`, it\n        will default to `True`.\n\n    Attributes\n    ----------\n    encoder: nn.ModuleDict\n        ModuleDict with the Perceiver blocks\n    latents: nn.Parameter\n        Latents that will be used for prediction\n    mlp: nn.Module\n        MLP component in the model\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.models import TabPerceiver\n    &gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n    &gt;&gt;&gt; colnames = ['a', 'b', 'c', 'd', 'e']\n    &gt;&gt;&gt; cat_embed_input = [(u,i) for u,i in zip(colnames[:4], [4]*4)]\n    &gt;&gt;&gt; continuous_cols = ['e']\n    &gt;&gt;&gt; column_idx = {k:v for v,k in enumerate(colnames)}\n    &gt;&gt;&gt; model = TabPerceiver(column_idx=column_idx, cat_embed_input=cat_embed_input,\n    ... continuous_cols=continuous_cols, n_latents=2, latent_dim=16,\n    ... n_perceiver_blocks=2)\n    &gt;&gt;&gt; out = model(X_tab)\n    \"\"\"\n\n    def __init__(\n        self,\n        column_idx: Dict[str, int],\n        *,\n        cat_embed_input: Optional[List[Tuple[str, int]]] = None,\n        cat_embed_dropout: Optional[float] = None,\n        use_cat_bias: Optional[bool] = None,\n        cat_embed_activation: Optional[str] = None,\n        shared_embed: Optional[bool] = None,\n        add_shared_embed: Optional[bool] = None,\n        frac_shared_embed: Optional[float] = None,\n        continuous_cols: Optional[List[str]] = None,\n        cont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\n        embed_continuous_method: Optional[\n            Literal[\"standard\", \"piecewise\", \"periodic\"]\n        ] = \"standard\",\n        cont_embed_dropout: Optional[float] = None,\n        cont_embed_activation: Optional[str] = None,\n        quantization_setup: Optional[Dict[str, List[float]]] = None,\n        n_frequencies: Optional[int] = None,\n        sigma: Optional[float] = None,\n        share_last_layer: Optional[bool] = None,\n        full_embed_dropout: Optional[bool] = None,\n        input_dim: int = 32,\n        n_cross_attns: int = 1,\n        n_cross_attn_heads: int = 4,\n        n_latents: int = 16,\n        latent_dim: int = 128,\n        n_latent_heads: int = 4,\n        n_latent_blocks: int = 4,\n        n_perceiver_blocks: int = 4,\n        share_weights: bool = False,\n        attn_dropout: float = 0.1,\n        ff_dropout: float = 0.1,\n        ff_factor: int = 4,\n        transformer_activation: str = \"geglu\",\n        mlp_hidden_dims: Optional[List[int]] = None,\n        mlp_activation: Optional[str] = None,\n        mlp_dropout: Optional[float] = None,\n        mlp_batchnorm: Optional[bool] = None,\n        mlp_batchnorm_last: Optional[bool] = None,\n        mlp_linear_first: Optional[bool] = None,\n    ):\n        super(TabPerceiver, self).__init__(\n            column_idx=column_idx,\n            cat_embed_input=cat_embed_input,\n            cat_embed_dropout=cat_embed_dropout,\n            use_cat_bias=use_cat_bias,\n            cat_embed_activation=cat_embed_activation,\n            shared_embed=shared_embed,\n            add_shared_embed=add_shared_embed,\n            frac_shared_embed=frac_shared_embed,\n            continuous_cols=continuous_cols,\n            cont_norm_layer=cont_norm_layer,\n            embed_continuous_method=embed_continuous_method,\n            cont_embed_dropout=cont_embed_dropout,\n            cont_embed_activation=cont_embed_activation,\n            input_dim=input_dim,\n            quantization_setup=quantization_setup,\n            n_frequencies=n_frequencies,\n            sigma=sigma,\n            share_last_layer=share_last_layer,\n            full_embed_dropout=full_embed_dropout,\n        )\n\n        self.n_cross_attns = n_cross_attns\n        self.n_cross_attn_heads = n_cross_attn_heads\n        self.n_latents = n_latents\n        self.latent_dim = latent_dim\n        self.n_latent_heads = n_latent_heads\n        self.n_latent_blocks = n_latent_blocks\n        self.n_perceiver_blocks = n_perceiver_blocks\n        self.share_weights = share_weights\n        self.attn_dropout = attn_dropout\n        self.ff_dropout = ff_dropout\n        self.ff_factor = ff_factor\n        self.transformer_activation = transformer_activation\n\n        self.mlp_hidden_dims = mlp_hidden_dims\n        self.mlp_activation = mlp_activation\n        self.mlp_dropout = mlp_dropout\n        self.mlp_batchnorm = mlp_batchnorm\n        self.mlp_batchnorm_last = mlp_batchnorm_last\n        self.mlp_linear_first = mlp_linear_first\n\n        # Embeddings are instantiated at the base model\n        # Transformer blocks\n        self.latents = nn.init.trunc_normal_(\n            nn.Parameter(torch.empty(n_latents, latent_dim))\n        )\n\n        self.encoder = nn.ModuleDict()\n        first_perceiver_block = self._build_perceiver_block()\n        self.encoder[\"perceiver_block0\"] = first_perceiver_block\n\n        if share_weights:\n            for n in range(1, n_perceiver_blocks):\n                self.encoder[\"perceiver_block\" + str(n)] = first_perceiver_block\n        else:\n            for n in range(1, n_perceiver_blocks):\n                self.encoder[\"perceiver_block\" + str(n)] = self._build_perceiver_block()\n\n        self.mlp_first_hidden_dim = self.latent_dim\n\n        # Mlp\n        if self.mlp_hidden_dims is not None:\n            self.mlp = MLP(\n                d_hidden=[self.mlp_first_hidden_dim] + self.mlp_hidden_dims,\n                activation=(\n                    \"relu\" if self.mlp_activation is None else self.mlp_activation\n                ),\n                dropout=0.0 if self.mlp_dropout is None else self.mlp_dropout,\n                batchnorm=False if self.mlp_batchnorm is None else self.mlp_batchnorm,\n                batchnorm_last=(\n                    False\n                    if self.mlp_batchnorm_last is None\n                    else self.mlp_batchnorm_last\n                ),\n                linear_first=(\n                    False if self.mlp_linear_first is None else self.mlp_linear_first\n                ),\n            )\n        else:\n            self.mlp = None\n\n    def forward(self, X: Tensor) -&gt; Tensor:\n        x_emb = self._get_embeddings(X)\n\n        x = einops.repeat(self.latents, \"n d -&gt; b n d\", b=X.shape[0])\n\n        for n in range(self.n_perceiver_blocks):\n            cross_attns = self.encoder[\"perceiver_block\" + str(n)][\"cross_attns\"]\n            latent_transformer = self.encoder[\"perceiver_block\" + str(n)][\n                \"latent_transformer\"\n            ]\n            for cross_attn in cross_attns:\n                x = cross_attn(x, x_emb)\n            x = latent_transformer(x)\n\n        # average along the latent index axis\n        x = x.mean(dim=1)\n\n        if self.mlp is not None:\n            x = self.mlp(x)\n\n        return x\n\n    @property\n    def output_dim(self) -&gt; int:\n        r\"\"\"The output dimension of the model. This is a required property\n        neccesary to build the `WideDeep` class\n        \"\"\"\n        return (\n            self.mlp_hidden_dims[-1]\n            if self.mlp_hidden_dims is not None\n            else self.mlp_first_hidden_dim\n        )\n\n    @property\n    def attention_weights(self) -&gt; List:\n        r\"\"\"List with the attention weights. If the weights are not shared\n        between perceiver blocks each element of the list will be a list\n        itself containing the Cross Attention and Latent Transformer\n        attention weights respectively\n\n        The shape of the attention weights is:\n\n        - Cross Attention: $(N, C, L, F)$\n\n        - Latent Attention: $(N, T, L, L)$\n\n        WHere $N$ is the batch size, $C$ is the number of Cross Attention\n        heads, $L$ is the number of Latents, $F$ is the number of\n        features/columns in the dataset and $T$ is the number of Latent\n        Attention heads\n        \"\"\"\n        if self.share_weights:\n            cross_attns = self.encoder[\"perceiver_block0\"][\"cross_attns\"]\n            latent_transformer = self.encoder[\"perceiver_block0\"][\"latent_transformer\"]\n            attention_weights = self._extract_attn_weights(\n                cross_attns, latent_transformer\n            )\n        else:\n            attention_weights = []\n            for n in range(self.n_perceiver_blocks):\n                cross_attns = self.encoder[\"perceiver_block\" + str(n)][\"cross_attns\"]\n                latent_transformer = self.encoder[\"perceiver_block\" + str(n)][\n                    \"latent_transformer\"\n                ]\n                attention_weights.append(\n                    self._extract_attn_weights(cross_attns, latent_transformer)\n                )\n        return attention_weights\n\n    def _build_perceiver_block(self) -&gt; nn.ModuleDict:\n        perceiver_block = nn.ModuleDict()\n\n        # Cross Attention\n        cross_attns = nn.ModuleList()\n        for _ in range(self.n_cross_attns):\n            cross_attns.append(\n                PerceiverEncoder(\n                    self.input_dim,\n                    self.n_cross_attn_heads,\n                    False,  # use_bias\n                    self.attn_dropout,\n                    self.ff_dropout,\n                    self.ff_factor,\n                    self.transformer_activation,\n                    self.latent_dim,  # q_dim,\n                ),\n            )\n        perceiver_block[\"cross_attns\"] = cross_attns\n\n        # Latent Transformer\n        latent_transformer = nn.Sequential()\n        for i in range(self.n_latent_blocks):\n            latent_transformer.add_module(\n                \"latent_block\" + str(i),\n                PerceiverEncoder(\n                    self.latent_dim,  # input_dim\n                    self.n_latent_heads,\n                    False,  # use_bias\n                    self.attn_dropout,\n                    self.ff_dropout,\n                    self.ff_factor,\n                    self.transformer_activation,\n                ),\n            )\n        perceiver_block[\"latent_transformer\"] = latent_transformer\n\n        return perceiver_block\n\n    @staticmethod\n    def _extract_attn_weights(cross_attns, latent_transformer) -&gt; List:\n        attention_weights = []\n        for cross_attn in cross_attns:\n            attention_weights.append(cross_attn.attn.attn_weights)\n        for latent_block in latent_transformer:\n            attention_weights.append(latent_block.attn.attn_weights)\n        return attention_weights\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.transformers.tab_perceiver.TabPerceiver.output_dim","title":"output_dim  <code>property</code>","text":"<pre><code>output_dim\n</code></pre> <p>The output dimension of the model. This is a required property neccesary to build the <code>WideDeep</code> class</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.transformers.tab_perceiver.TabPerceiver.attention_weights","title":"attention_weights  <code>property</code>","text":"<pre><code>attention_weights\n</code></pre> <p>List with the attention weights. If the weights are not shared between perceiver blocks each element of the list will be a list itself containing the Cross Attention and Latent Transformer attention weights respectively</p> <p>The shape of the attention weights is:</p> <ul> <li> <p>Cross Attention: \\((N, C, L, F)\\)</p> </li> <li> <p>Latent Attention: \\((N, T, L, L)\\)</p> </li> </ul> <p>WHere \\(N\\) is the batch size, \\(C\\) is the number of Cross Attention heads, \\(L\\) is the number of Latents, \\(F\\) is the number of features/columns in the dataset and \\(T\\) is the number of Latent Attention heads</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.transformers.tab_fastformer.TabFastFormer","title":"TabFastFormer","text":"<p>               Bases: <code>BaseTabularModelWithAttention</code></p> <p>Defines an adaptation of a FastFormer that can be used as the <code>deeptabular</code> component of a Wide &amp; Deep model or independently by itself.</p> <p>Most of the parameters for this class are <code>Optional</code> since the use of categorical or continuous is in fact optional (i.e. one can use categorical features only, continuous features only or both).</p> <p> NOTE: while there are scientific publications for  the <code>TabTransformer</code>, <code>SAINT</code> and <code>FTTransformer</code>, the <code>TabPerceiver</code>  and the <code>TabFastFormer</code> are our own adaptations of the  Perceiver and the  FastFormer for tabular data.</p> <p>Parameters:</p> Name Type Description Default <code>column_idx</code> <code>Dict[str, int]</code> <p>Dict containing the index of the columns that will be passed through the <code>TabMlp</code> model. Required to slice the tensors. e.g. {'education': 0, 'relationship': 1, 'workclass': 2, ...}.</p> required <code>cat_embed_input</code> <code>Optional[List[Tuple[str, int]]]</code> <p>List of Tuples with the column name and number of unique values and embedding dimension. e.g. [(education, 11), ...]</p> <code>None</code> <code>cat_embed_dropout</code> <code>Optional[float]</code> <p>Categorical embeddings dropout. If <code>None</code>, it will default to 0.</p> <code>None</code> <code>use_cat_bias</code> <code>Optional[bool]</code> <p>Boolean indicating if bias will be used for the categorical embeddings. If <code>None</code>, it will default to 'False'.</p> <code>None</code> <code>cat_embed_activation</code> <code>Optional[str]</code> <p>Activation function for the categorical embeddings, if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> <code>None</code> <code>shared_embed</code> <code>Optional[bool]</code> <p>Boolean indicating if the embeddings will be \"shared\". The idea behind <code>shared_embed</code> is described in the Appendix A in the TabTransformer paper: 'The goal of having column embedding is to enable the model to distinguish the classes in one column from those in the other columns'. In other words, the idea is to let the model learn which column is embedded at the time. See: <code>pytorch_widedeep.models.transformers._layers.SharedEmbeddings</code>.</p> <code>None</code> <code>add_shared_embed</code> <code>Optional[bool]</code> <p>The two embedding sharing strategies are: 1) add the shared embeddings to the column embeddings or 2) to replace the first <code>frac_shared_embed</code> with the shared embeddings. See <code>pytorch_widedeep.models.embeddings_layers.SharedEmbeddings</code> If 'None' is passed, it will default to 'False'.</p> <code>None</code> <code>frac_shared_embed</code> <code>Optional[float]</code> <p>The fraction of embeddings that will be shared (if <code>add_shared_embed = False</code>) by all the different categories for one particular column. If 'None' is passed, it will default to 0.0.</p> <code>None</code> <code>continuous_cols</code> <code>Optional[List[str]]</code> <p>List with the name of the numeric (aka continuous) columns</p> <code>None</code> <code>cont_norm_layer</code> <code>Optional[Literal[batchnorm, layernorm]]</code> <p>Type of normalization layer applied to the continuous features. Options are: 'layernorm' and 'batchnorm'. if <code>None</code>, no normalization layer will be used.</p> <code>None</code> <code>embed_continuous_method</code> <code>Optional[Literal[standard, piecewise, periodic]]</code> <p>Method to use to embed the continuous features. Options are: 'standard', 'periodic' or 'piecewise'. The 'standard' embedding method is based on the FT-Transformer implementation presented in the paper: Revisiting Deep Learning Models for Tabular Data. The 'periodic' and_'piecewise'_ methods were presented in the paper: On Embeddings for Numerical Features in Tabular Deep Learning. Please, read the papers for details.</p> <code>'standard'</code> <code>cont_embed_dropout</code> <code>Optional[float]</code> <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p> <code>None</code> <code>cont_embed_activation</code> <code>Optional[str]</code> <p>Activation function for the continuous embeddings if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported. If <code>None</code>, no activation function will be applied.</p> <code>None</code> <code>quantization_setup</code> <code>Optional[Dict[str, List[float]]]</code> <p>This parameter is used when the 'piecewise' method is used to embed the continuous cols. It is a dict where keys are the name of the continuous columns and values are lists with the boundaries for the quantization of the continuous_cols. See the examples for details. If If the 'piecewise' method is used, this parameter is required.</p> <code>None</code> <code>n_frequencies</code> <code>Optional[int]</code> <p>This is the so called 'k' in their paper On Embeddings for Numerical Features in Tabular Deep Learning, and is the number of 'frequencies' that will be used to represent each continuous column. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>sigma</code> <code>Optional[float]</code> <p>This is the sigma parameter in the paper mentioned when describing the previous parameters and it is used to initialise the 'frequency weights'. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>share_last_layer</code> <code>Optional[bool]</code> <p>This parameter is not present in the before mentioned paper but it is implemented in the official repo. If <code>True</code> the linear layer that turns the frequencies into embeddings will be shared across the continuous columns. If <code>False</code> a different linear layer will be used for each continuous column. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>full_embed_dropout</code> <code>Optional[bool]</code> <p>If <code>True</code>, the full embedding corresponding to a column will be masked out/dropout. If <code>None</code>, it will default to <code>False</code>.</p> <code>None</code> <code>input_dim</code> <code>int</code> <p>The so-called dimension of the model. Is the number of embeddings used to encode the categorical and/or continuous columns</p> <code>32</code> <code>n_heads</code> <code>int</code> <p>Number of attention heads per FastFormer block</p> <code>8</code> <code>use_bias</code> <code>bool</code> <p>Boolean indicating whether or not to use bias in the Q, K, and V projection layers</p> <code>False</code> <code>n_blocks</code> <code>int</code> <p>Number of FastFormer blocks</p> <code>4</code> <code>attn_dropout</code> <code>float</code> <p>Dropout that will be applied to the Additive Attention layers</p> <code>0.1</code> <code>ff_dropout</code> <code>float</code> <p>Dropout that will be applied to the FeedForward network</p> <code>0.2</code> <code>ff_factor</code> <code>int</code> <p>Multiplicative factor applied to the first layer of the FF network in each Transformer block, This is normally set to 4.</p> <code>4</code> <code>share_qv_weights</code> <code>bool</code> <p>Following the paper, this is a boolean indicating if the Value (\\(V\\)) and the Query (\\(Q\\)) transformation parameters will be shared.</p> <code>False</code> <code>share_weights</code> <code>bool</code> <p>In addition to sharing the \\(V\\) and \\(Q\\) transformation parameters, the parameters across different Fastformer layers can also be shared. Please, see <code>pytorch_widedeep/models/tabular/transformers/tab_fastformer.py</code> for details</p> <code>False</code> <code>transformer_activation</code> <code>str</code> <p>Transformer Encoder activation function. 'tanh', 'relu', 'leaky_relu', 'gelu', 'geglu' and 'reglu' are supported</p> <code>'relu'</code> <code>mlp_hidden_dims</code> <code>Optional[List[int]]</code> <p>MLP hidden dimensions. If not provided no MLP on top of the final FTTransformer block will be used</p> <code>None</code> <code>mlp_hidden_dims</code> <code>Optional[List[int]]</code> <p>List with the number of neurons per dense layer in the MLP. e.g: [64, 32]. If not provided no MLP on top of the final Transformer block will be used.</p> <code>None</code> <code>mlp_activation</code> <code>Optional[str]</code> <p>Activation function for the dense layers of the MLP. Currently 'tanh', 'relu', 'leaky'_relu' and _'gelu' are supported. If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to 'relu'.</p> <code>None</code> <code>mlp_dropout</code> <code>Optional[float]</code> <p>float with the dropout between the dense layers of the MLP. If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to 0.0.</p> <code>None</code> <code>mlp_batchnorm</code> <code>Optional[bool]</code> <p>Boolean indicating whether or not batch normalization will be applied to the dense layers If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to False.</p> <code>None</code> <code>mlp_batchnorm_last</code> <code>Optional[bool]</code> <p>Boolean indicating whether or not batch normalization will be applied to the last of the dense layers If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to False.</p> <code>None</code> <code>mlp_linear_first</code> <code>Optional[bool]</code> <p>Boolean indicating the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code> If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to <code>True</code>.</p> <code>None</code> <p>Attributes:</p> Name Type Description <code>encoder</code> <code>Module</code> <p>Sequence of FasFormer blocks.</p> <code>mlp</code> <code>Module</code> <p>MLP component in the model</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import TabFastFormer\n&gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n&gt;&gt;&gt; colnames = ['a', 'b', 'c', 'd', 'e']\n&gt;&gt;&gt; cat_embed_input = [(u,i) for u,i in zip(colnames[:4], [4]*4)]\n&gt;&gt;&gt; continuous_cols = ['e']\n&gt;&gt;&gt; column_idx = {k:v for v,k in enumerate(colnames)}\n&gt;&gt;&gt; model = TabFastFormer(column_idx=column_idx, cat_embed_input=cat_embed_input, continuous_cols=continuous_cols)\n&gt;&gt;&gt; out = model(X_tab)\n</code></pre> Source code in <code>pytorch_widedeep/models/tabular/transformers/tab_fastformer.py</code> <pre><code>class TabFastFormer(BaseTabularModelWithAttention):\n    r\"\"\"Defines an adaptation of a [FastFormer](https://arxiv.org/abs/2108.09084)\n    that can be used as the `deeptabular` component of a Wide &amp; Deep model\n    or independently by itself.\n\n    Most of the parameters for this class are `Optional` since the use of\n    categorical or continuous is in fact optional (i.e. one can use\n    categorical features only, continuous features only or both).\n\n    :information_source: **NOTE**: while there are scientific publications for\n     the `TabTransformer`, `SAINT` and `FTTransformer`, the `TabPerceiver`\n     and the `TabFastFormer` are our own adaptations of the\n     [Perceiver](https://arxiv.org/abs/2103.03206) and the\n     [FastFormer](https://arxiv.org/abs/2108.09084) for tabular data.\n\n    Parameters\n    ----------\n    column_idx: Dict\n        Dict containing the index of the columns that will be passed through\n        the `TabMlp` model. Required to slice the tensors. e.g. _{'education':\n        0, 'relationship': 1, 'workclass': 2, ...}_.\n    cat_embed_input: List, Optional, default = None\n        List of Tuples with the column name and number of unique values and\n        embedding dimension. e.g. _[(education, 11), ...]_\n    cat_embed_dropout: float, Optional, default = None\n        Categorical embeddings dropout. If `None`, it will default\n        to 0.\n    use_cat_bias: bool, Optional, default = None,\n        Boolean indicating if bias will be used for the categorical embeddings.\n        If `None`, it will default to 'False'.\n    cat_embed_activation: Optional, str, default = None,\n        Activation function for the categorical embeddings, if any. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported\n    shared_embed: bool, Optional, default = None\n        Boolean indicating if the embeddings will be \"shared\". The idea behind `shared_embed` is\n        described in the Appendix A in the [TabTransformer paper](https://arxiv.org/abs/2012.06678):\n        _'The goal of having column embedding is to enable the model to\n        distinguish the classes in one column from those in the other\n        columns'_. In other words, the idea is to let the model learn which\n        column is embedded at the time. See: `pytorch_widedeep.models.transformers._layers.SharedEmbeddings`.\n    add_shared_embed: bool, Optional, default = None\n        The two embedding sharing strategies are: 1) add the shared embeddings\n        to the column embeddings or 2) to replace the first\n        `frac_shared_embed` with the shared embeddings.\n        See `pytorch_widedeep.models.embeddings_layers.SharedEmbeddings`\n        If 'None' is passed, it will default to 'False'.\n    frac_shared_embed: float, Optional, default = None\n        The fraction of embeddings that will be shared (if `add_shared_embed\n        = False`) by all the different categories for one particular\n        column. If 'None' is passed, it will default to 0.0.\n    continuous_cols: List, Optional, default = None\n        List with the name of the numeric (aka continuous) columns\n    cont_norm_layer: str, Optional, default =  None\n        Type of normalization layer applied to the continuous features.\n        Options are: _'layernorm'_ and _'batchnorm'_. if `None`, no\n        normalization layer will be used.\n    embed_continuous_method: Optional, str, default = None,\n        Method to use to embed the continuous features. Options are:\n        _'standard'_, _'periodic'_ or _'piecewise'_. The _'standard'_\n        embedding method is based on the FT-Transformer implementation\n        presented in the paper: [Revisiting Deep Learning Models for\n        Tabular Data](https://arxiv.org/abs/2106.11959v5). The _'periodic'_\n        and_'piecewise'_ methods were presented in the paper: [On Embeddings for\n        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556).\n        Please, read the papers for details.\n    cont_embed_dropout: float, Optional, default = None,\n        Dropout for the continuous embeddings. If `None`, it will default to 0.0\n    cont_embed_activation: Optional, str, default = None,\n        Activation function for the continuous embeddings if any. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported.\n        If `None`, no activation function will be applied.\n    quantization_setup: Dict[str, List[float]], Optional, default = None,\n        This parameter is used when the _'piecewise'_ method is used to embed\n        the continuous cols. It is a dict where keys are the name of the continuous\n        columns and values are lists with the boundaries for the quantization\n        of the continuous_cols. See the examples for details. If\n        If the _'piecewise'_ method is used, this parameter is required.\n    n_frequencies: int, Optional, default = None,\n        This is the so called _'k'_ in their paper [On Embeddings for\n        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556),\n        and is the number of 'frequencies' that will be used to represent each\n        continuous column. See their Eq 2 in the paper for details. If\n        the _'periodic'_ method is used, this parameter is required.\n    sigma: float, Optional, default = None,\n        This is the sigma parameter in the paper mentioned when describing the\n        previous parameters and it is used to initialise the 'frequency\n        weights'. See their Eq 2 in the paper for details. If\n        the _'periodic'_ method is used, this parameter is required.\n    share_last_layer: bool, Optional, default = None,\n        This parameter is not present in the before mentioned paper but it is implemented in\n        the [official repo](https://github.com/yandex-research/rtdl-num-embeddings/tree/main).\n        If `True` the linear layer that turns the frequencies into embeddings\n        will be shared across the continuous columns. If `False` a different\n        linear layer will be used for each continuous column.\n        If the _'periodic'_ method is used, this parameter is required.\n    full_embed_dropout: bool, Optional, default = None,\n        If `True`, the full embedding corresponding to a column will be masked\n        out/dropout. If `None`, it will default to `False`.\n    input_dim: int, default = 32\n        The so-called *dimension of the model*. Is the number of\n        embeddings used to encode the categorical and/or continuous columns\n    n_heads: int, default = 8\n        Number of attention heads per FastFormer block\n    use_bias: bool, default = False\n        Boolean indicating whether or not to use bias in the Q, K, and V\n        projection layers\n    n_blocks: int, default = 4\n        Number of FastFormer blocks\n    attn_dropout: float, default = 0.2\n        Dropout that will be applied to the Additive Attention layers\n    ff_dropout: float, default = 0.1\n        Dropout that will be applied to the FeedForward network\n    ff_factor: float, default = 4\n        Multiplicative factor applied to the first layer of the FF network in\n        each Transformer block, This is normally set to 4.\n    share_qv_weights: bool, default = False\n        Following the paper, this is a boolean indicating if the Value ($V$) and\n        the Query ($Q$) transformation parameters will be shared.\n    share_weights: bool, default = False\n        In addition to sharing the $V$ and $Q$ transformation parameters, the\n        parameters across different Fastformer layers can also be shared.\n        Please, see\n        `pytorch_widedeep/models/tabular/transformers/tab_fastformer.py` for\n        details\n    transformer_activation: str, default = \"gelu\"\n        Transformer Encoder activation function. _'tanh'_, _'relu'_,\n        _'leaky_relu'_, _'gelu'_, _'geglu'_ and _'reglu'_ are supported\n    mlp_hidden_dims: List, Optional, default = None\n        MLP hidden dimensions. If not provided no MLP on top of the final\n        FTTransformer block will be used\n    mlp_hidden_dims: List, Optional, default = None\n        List with the number of neurons per dense layer in the MLP. e.g:\n        _[64, 32]_. If not provided no MLP on top of the final\n        Transformer block will be used.\n    mlp_activation: str, Optional, default = None\n        Activation function for the dense layers of the MLP. Currently\n        _'tanh'_, _'relu'_, _'leaky'_relu' and _'gelu'_ are supported.\n        If 'mlp_hidden_dims' is not `None` and this parameter is `None`, it\n        will default to _'relu'_.\n    mlp_dropout: float, Optional, default = None\n        float with the dropout between the dense layers of the MLP.\n        If 'mlp_hidden_dims' is not `None` and this parameter is `None`, it\n        will default to 0.0.\n    mlp_batchnorm: bool, Optional, default = None\n        Boolean indicating whether or not batch normalization will be applied\n        to the dense layers\n        If 'mlp_hidden_dims' is not `None` and this parameter is `None`, it\n        will default to False.\n    mlp_batchnorm_last: bool, Optional, default = None\n        Boolean indicating whether or not batch normalization will be applied\n        to the last of the dense layers\n        If 'mlp_hidden_dims' is not `None` and this parameter is `None`, it\n        will default to False.\n    mlp_linear_first: bool, Optional, default = None\n        Boolean indicating the order of the operations in the dense\n        layer. If `True: [LIN -&gt; ACT -&gt; BN -&gt; DP]`. If `False: [BN -&gt; DP -&gt;\n        LIN -&gt; ACT]`\n        If 'mlp_hidden_dims' is not `None` and this parameter is `None`, it\n        will default to `True`.\n\n    Attributes\n    ----------\n    encoder: nn.Module\n        Sequence of FasFormer blocks.\n    mlp: nn.Module\n        MLP component in the model\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.models import TabFastFormer\n    &gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n    &gt;&gt;&gt; colnames = ['a', 'b', 'c', 'd', 'e']\n    &gt;&gt;&gt; cat_embed_input = [(u,i) for u,i in zip(colnames[:4], [4]*4)]\n    &gt;&gt;&gt; continuous_cols = ['e']\n    &gt;&gt;&gt; column_idx = {k:v for v,k in enumerate(colnames)}\n    &gt;&gt;&gt; model = TabFastFormer(column_idx=column_idx, cat_embed_input=cat_embed_input, continuous_cols=continuous_cols)\n    &gt;&gt;&gt; out = model(X_tab)\n    \"\"\"\n\n    def __init__(\n        self,\n        column_idx: Dict[str, int],\n        *,\n        cat_embed_input: Optional[List[Tuple[str, int]]] = None,\n        cat_embed_dropout: Optional[float] = None,\n        use_cat_bias: Optional[bool] = None,\n        cat_embed_activation: Optional[str] = None,\n        shared_embed: Optional[bool] = None,\n        add_shared_embed: Optional[bool] = None,\n        frac_shared_embed: Optional[float] = None,\n        continuous_cols: Optional[List[str]] = None,\n        cont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\n        embed_continuous_method: Optional[\n            Literal[\"standard\", \"piecewise\", \"periodic\"]\n        ] = \"standard\",\n        cont_embed_dropout: Optional[float] = None,\n        cont_embed_activation: Optional[str] = None,\n        quantization_setup: Optional[Dict[str, List[float]]] = None,\n        n_frequencies: Optional[int] = None,\n        sigma: Optional[float] = None,\n        share_last_layer: Optional[bool] = None,\n        full_embed_dropout: Optional[bool] = None,\n        input_dim: int = 32,\n        n_heads: int = 8,\n        use_bias: bool = False,\n        n_blocks: int = 4,\n        attn_dropout: float = 0.1,\n        ff_dropout: float = 0.2,\n        ff_factor: int = 4,\n        share_qv_weights: bool = False,\n        share_weights: bool = False,\n        transformer_activation: str = \"relu\",\n        mlp_hidden_dims: Optional[List[int]] = None,\n        mlp_activation: Optional[str] = None,\n        mlp_dropout: Optional[float] = None,\n        mlp_batchnorm: Optional[bool] = None,\n        mlp_batchnorm_last: Optional[bool] = None,\n        mlp_linear_first: Optional[bool] = None,\n    ):\n        super(TabFastFormer, self).__init__(\n            column_idx=column_idx,\n            cat_embed_input=cat_embed_input,\n            cat_embed_dropout=cat_embed_dropout,\n            use_cat_bias=use_cat_bias,\n            cat_embed_activation=cat_embed_activation,\n            shared_embed=shared_embed,\n            add_shared_embed=add_shared_embed,\n            frac_shared_embed=frac_shared_embed,\n            continuous_cols=continuous_cols,\n            cont_norm_layer=cont_norm_layer,\n            embed_continuous_method=embed_continuous_method,\n            cont_embed_dropout=cont_embed_dropout,\n            cont_embed_activation=cont_embed_activation,\n            input_dim=input_dim,\n            quantization_setup=quantization_setup,\n            n_frequencies=n_frequencies,\n            sigma=sigma,\n            share_last_layer=share_last_layer,\n            full_embed_dropout=full_embed_dropout,\n        )\n\n        self.n_heads = n_heads\n        self.use_bias = use_bias\n        self.n_blocks = n_blocks\n        self.attn_dropout = attn_dropout\n        self.ff_dropout = ff_dropout\n        self.ff_factor = ff_factor\n        self.share_qv_weights = share_qv_weights\n        self.share_weights = share_weights\n        self.transformer_activation = transformer_activation\n\n        self.mlp_hidden_dims = mlp_hidden_dims\n        self.mlp_activation = mlp_activation\n        self.mlp_dropout = mlp_dropout\n        self.mlp_batchnorm = mlp_batchnorm\n        self.mlp_batchnorm_last = mlp_batchnorm_last\n        self.mlp_linear_first = mlp_linear_first\n\n        self.with_cls_token = \"cls_token\" in column_idx\n        self.n_cat = len(cat_embed_input) if cat_embed_input is not None else 0\n        self.n_cont = len(continuous_cols) if continuous_cols is not None else 0\n        self.n_feats = self.n_cat + self.n_cont\n\n        # Embeddings are instantiated at the base model\n        # Transformer blocks\n        self.encoder = nn.Sequential()\n        first_fastformer_block = FastFormerEncoder(\n            input_dim,\n            n_heads,\n            use_bias,\n            attn_dropout,\n            ff_dropout,\n            ff_factor,\n            share_qv_weights,\n            transformer_activation,\n        )\n        self.encoder.add_module(\"fastformer_block0\", first_fastformer_block)\n        for i in range(1, n_blocks):\n            if share_weights:\n                self.encoder.add_module(\n                    \"fastformer_block\" + str(i), first_fastformer_block\n                )\n            else:\n                self.encoder.add_module(\n                    \"fastformer_block\" + str(i),\n                    FastFormerEncoder(\n                        input_dim,\n                        n_heads,\n                        use_bias,\n                        attn_dropout,\n                        ff_dropout,\n                        ff_factor,\n                        share_qv_weights,\n                        transformer_activation,\n                    ),\n                )\n\n        self.mlp_first_hidden_dim = (\n            self.input_dim if self.with_cls_token else (self.n_feats * self.input_dim)\n        )\n\n        # Mlp: adding an MLP on top of the Resnet blocks is optional and\n        # therefore all related params are optional\n        if self.mlp_hidden_dims is not None:\n            self.mlp = MLP(\n                d_hidden=[self.mlp_first_hidden_dim] + self.mlp_hidden_dims,\n                activation=(\n                    \"relu\" if self.mlp_activation is None else self.mlp_activation\n                ),\n                dropout=0.0 if self.mlp_dropout is None else self.mlp_dropout,\n                batchnorm=False if self.mlp_batchnorm is None else self.mlp_batchnorm,\n                batchnorm_last=(\n                    False\n                    if self.mlp_batchnorm_last is None\n                    else self.mlp_batchnorm_last\n                ),\n                linear_first=(\n                    False if self.mlp_linear_first is None else self.mlp_linear_first\n                ),\n            )\n        else:\n            self.mlp = None\n\n    def forward(self, X: Tensor) -&gt; Tensor:\n        x = self._get_embeddings(X)\n        x = self.encoder(x)\n        if self.with_cls_token:\n            x = x[:, 0, :]\n        else:\n            x = x.flatten(1)\n        if self.mlp is not None:\n            x = self.mlp(x)\n        return x\n\n    @property\n    def output_dim(self) -&gt; int:\n        r\"\"\"The output dimension of the model. This is a required property\n        neccesary to build the `WideDeep` class\n        \"\"\"\n        return (\n            self.mlp_hidden_dims[-1]\n            if self.mlp_hidden_dims is not None\n            else self.mlp_first_hidden_dim\n        )\n\n    @property\n    def attention_weights(self) -&gt; List[Tuple[Tensor, Tensor]]:\n        r\"\"\"List with the attention weights. Each element of the list is a\n        tuple where the first and second elements are the $\\alpha$\n        and $\\beta$ attention weights in the paper.\n\n        The shape of the attention weights is $(N, H, F)$ where $N$ is the\n        batch size, $H$ is the number of attention heads and $F$ is the\n        number of features/columns in the dataset\n        \"\"\"\n        if self.share_weights:\n            attention_weights = [self.encoder[0].attn.attn_weight]\n        else:\n            attention_weights = [blk.attn.attn_weights for blk in self.encoder]\n        return attention_weights\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.transformers.tab_fastformer.TabFastFormer.output_dim","title":"output_dim  <code>property</code>","text":"<pre><code>output_dim\n</code></pre> <p>The output dimension of the model. This is a required property neccesary to build the <code>WideDeep</code> class</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.transformers.tab_fastformer.TabFastFormer.attention_weights","title":"attention_weights  <code>property</code>","text":"<pre><code>attention_weights\n</code></pre> <p>List with the attention weights. Each element of the list is a tuple where the first and second elements are the \\(\\alpha\\) and \\(\\beta\\) attention weights in the paper.</p> <p>The shape of the attention weights is \\((N, H, F)\\) where \\(N\\) is the batch size, \\(H\\) is the number of attention heads and \\(F\\) is the number of features/columns in the dataset</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.text.rnns.basic_rnn.BasicRNN","title":"BasicRNN","text":"<p>               Bases: <code>BaseWDModelComponent</code></p> <p>Standard text classifier/regressor comprised by a stack of RNNs (LSTMs or GRUs) that can be used as the <code>deeptext</code> component of a Wide &amp; Deep model or independently by itself.</p> <p>In addition, there is the option to add a Fully Connected (FC) set of dense layers on top of the stack of RNNs</p> <p>Parameters:</p> Name Type Description Default <code>vocab_size</code> <code>int</code> <p>Number of words in the vocabulary</p> required <code>embed_dim</code> <code>Optional[int]</code> <p>Dimension of the word embeddings if non-pretained word vectors are used</p> <code>None</code> <code>embed_matrix</code> <code>Optional[ndarray]</code> <p>Pretrained word embeddings</p> <code>None</code> <code>embed_trainable</code> <code>bool</code> <p>Boolean indicating if the pretrained embeddings are trainable</p> <code>True</code> <code>rnn_type</code> <code>Literal[lstm, gru]</code> <p>String indicating the type of RNN to use. One of 'lstm' or 'gru'</p> <code>'lstm'</code> <code>hidden_dim</code> <code>int</code> <p>Hidden dim of the RNN</p> <code>64</code> <code>n_layers</code> <code>int</code> <p>Number of recurrent layers</p> <code>3</code> <code>rnn_dropout</code> <code>float</code> <p>Dropout for each RNN layer except the last layer</p> <code>0.0</code> <code>bidirectional</code> <code>bool</code> <p>Boolean indicating whether the staked RNNs are bidirectional</p> <code>False</code> <code>use_hidden_state</code> <code>bool</code> <p>Boolean indicating whether to use the final hidden state or the RNN's output as predicting features. Typically the former is used.</p> <code>True</code> <code>padding_idx</code> <code>int</code> <p>index of the padding token in the padded-tokenised sequences. The <code>TextPreprocessor</code> class within this library uses fastai's tokenizer where the token index 0 is reserved for the 'unknown' word token. Therefore, the default value is set to 1.</p> <code>1</code> <code>head_hidden_dims</code> <code>Optional[List[int]]</code> <p>List with the sizes of the dense layers in the head e.g: [128, 64]</p> <code>None</code> <code>head_activation</code> <code>str</code> <p>Activation function for the dense layers in the head. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> <code>'relu'</code> <code>head_dropout</code> <code>Optional[float]</code> <p>Dropout of the dense layers in the head</p> <code>None</code> <code>head_batchnorm</code> <code>bool</code> <p>Boolean indicating whether or not to include batch normalization in the dense layers that form the 'rnn_mlp'</p> <code>False</code> <code>head_batchnorm_last</code> <code>bool</code> <p>Boolean indicating whether or not to apply batch normalization to the last of the dense layers in the head</p> <code>False</code> <code>head_linear_first</code> <code>bool</code> <p>Boolean indicating whether the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code></p> <code>False</code> <p>Attributes:</p> Name Type Description <code>word_embed</code> <code>Module</code> <p>word embedding matrix</p> <code>rnn</code> <code>Module</code> <p>Stack of RNNs</p> <code>rnn_mlp</code> <code>Module</code> <p>Stack of dense layers on top of the RNN. This will only exists if <code>head_layers_dim</code> is not None</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import BasicRNN\n&gt;&gt;&gt; X_text = torch.cat((torch.zeros([5,1]), torch.empty(5, 4).random_(1,4)), axis=1)\n&gt;&gt;&gt; model = BasicRNN(vocab_size=4, hidden_dim=4, n_layers=2, padding_idx=0, embed_dim=4)\n&gt;&gt;&gt; out = model(X_text)\n</code></pre> Source code in <code>pytorch_widedeep/models/text/rnns/basic_rnn.py</code> <pre><code>class BasicRNN(BaseWDModelComponent):\n    r\"\"\"Standard text classifier/regressor comprised by a stack of RNNs\n    (LSTMs or GRUs) that can be used as the `deeptext` component of a Wide &amp;\n    Deep model or independently by itself.\n\n    In addition, there is the option to add a Fully Connected (FC) set of\n    dense layers on top of the stack of RNNs\n\n    Parameters\n    ----------\n    vocab_size: int\n        Number of words in the vocabulary\n    embed_dim: int, Optional, default = None\n        Dimension of the word embeddings if non-pretained word vectors are\n        used\n    embed_matrix: np.ndarray, Optional, default = None\n        Pretrained word embeddings\n    embed_trainable: bool, default = True\n        Boolean indicating if the pretrained embeddings are trainable\n    rnn_type: str, default = 'lstm'\n        String indicating the type of RNN to use. One of _'lstm'_ or _'gru'_\n    hidden_dim: int, default = 64\n        Hidden dim of the RNN\n    n_layers: int, default = 3\n        Number of recurrent layers\n    rnn_dropout: float, default = 0.1\n        Dropout for each RNN layer except the last layer\n    bidirectional: bool, default = True\n        Boolean indicating whether the staked RNNs are bidirectional\n    use_hidden_state: str, default = True\n        Boolean indicating whether to use the final hidden state or the RNN's\n        output as predicting features. Typically the former is used.\n    padding_idx: int, default = 1\n        index of the padding token in the padded-tokenised sequences. The\n        `TextPreprocessor` class within this library uses fastai's tokenizer\n        where the token index 0 is reserved for the _'unknown'_ word token.\n        Therefore, the default value is set to 1.\n    head_hidden_dims: List, Optional, default = None\n        List with the sizes of the dense layers in the head e.g: _[128, 64]_\n    head_activation: str, default = \"relu\"\n        Activation function for the dense layers in the head. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported\n    head_dropout: float, Optional, default = None\n        Dropout of the dense layers in the head\n    head_batchnorm: bool, default = False\n        Boolean indicating whether or not to include batch normalization in\n        the dense layers that form the _'rnn_mlp'_\n    head_batchnorm_last: bool, default = False\n        Boolean indicating whether or not to apply batch normalization to the\n        last of the dense layers in the head\n    head_linear_first: bool, default = False\n        Boolean indicating whether the order of the operations in the dense\n        layer. If `True: [LIN -&gt; ACT -&gt; BN -&gt; DP]`. If `False: [BN -&gt; DP -&gt;\n        LIN -&gt; ACT]`\n\n    Attributes\n    ----------\n    word_embed: nn.Module\n        word embedding matrix\n    rnn: nn.Module\n        Stack of RNNs\n    rnn_mlp: nn.Module\n        Stack of dense layers on top of the RNN. This will only exists if\n        `head_layers_dim` is not None\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.models import BasicRNN\n    &gt;&gt;&gt; X_text = torch.cat((torch.zeros([5,1]), torch.empty(5, 4).random_(1,4)), axis=1)\n    &gt;&gt;&gt; model = BasicRNN(vocab_size=4, hidden_dim=4, n_layers=2, padding_idx=0, embed_dim=4)\n    &gt;&gt;&gt; out = model(X_text)\n    \"\"\"\n\n    def __init__(\n        self,\n        vocab_size: int,\n        embed_dim: Optional[int] = None,\n        embed_matrix: Optional[np.ndarray] = None,\n        embed_trainable: bool = True,\n        rnn_type: Literal[\"lstm\", \"gru\"] = \"lstm\",\n        hidden_dim: int = 64,\n        n_layers: int = 3,\n        rnn_dropout: float = 0.0,\n        bidirectional: bool = False,\n        use_hidden_state: bool = True,\n        padding_idx: int = 1,\n        head_hidden_dims: Optional[List[int]] = None,\n        head_activation: str = \"relu\",\n        head_dropout: Optional[float] = None,\n        head_batchnorm: bool = False,\n        head_batchnorm_last: bool = False,\n        head_linear_first: bool = False,\n    ):\n        super(BasicRNN, self).__init__()\n\n        if embed_dim is None and embed_matrix is None:\n            raise ValueError(\n                \"If no 'embed_matrix' is passed, the embedding dimension must\"\n                \"be specified with 'embed_dim'\"\n            )\n\n        if rnn_type.lower() not in [\"lstm\", \"gru\"]:\n            raise ValueError(\n                f\"'rnn_type' must be 'lstm' or 'gru', got {rnn_type} instead\"\n            )\n\n        if (\n            embed_dim is not None\n            and embed_matrix is not None\n            and not embed_dim == embed_matrix.shape[1]\n        ):\n            warnings.warn(\n                \"the input embedding dimension {} and the dimension of the \"\n                \"pretrained embeddings {} do not match. The pretrained embeddings \"\n                \"dimension ({}) will be used\".format(\n                    embed_dim, embed_matrix.shape[1], embed_matrix.shape[1]\n                ),\n                UserWarning,\n            )\n\n        self.vocab_size = vocab_size\n        self.embed_trainable = embed_trainable\n        self.embed_dim = embed_dim\n\n        self.rnn_type = rnn_type\n        self.hidden_dim = hidden_dim\n        self.n_layers = n_layers\n        self.rnn_dropout = rnn_dropout\n        self.bidirectional = bidirectional\n        self.use_hidden_state = use_hidden_state\n        self.padding_idx = padding_idx\n\n        self.head_hidden_dims = head_hidden_dims\n        self.head_activation = head_activation\n        self.head_dropout = head_dropout\n        self.head_batchnorm = head_batchnorm\n        self.head_batchnorm_last = head_batchnorm_last\n        self.head_linear_first = head_linear_first\n\n        # Embeddings\n        if embed_matrix is not None:\n            self.word_embed, self.embed_dim = self._set_embeddings(embed_matrix)\n        else:\n            assert self.embed_dim is not None\n            self.word_embed = nn.Embedding(\n                self.vocab_size, self.embed_dim, padding_idx=self.padding_idx\n            )\n\n        # RNN\n        rnn_params = {\n            \"input_size\": self.embed_dim,\n            \"hidden_size\": hidden_dim,\n            \"num_layers\": n_layers,\n            \"bidirectional\": bidirectional,\n            \"dropout\": rnn_dropout,\n            \"batch_first\": True,\n        }\n        if self.rnn_type.lower() == \"lstm\":\n            self.rnn: Union[nn.LSTM, nn.GRU] = nn.LSTM(**rnn_params)\n        elif self.rnn_type.lower() == \"gru\":\n            self.rnn = nn.GRU(**rnn_params)\n        else:\n            raise ValueError(\n                f\"'rnn_type' must be 'lstm' or 'gru', got {self.rnn_type} instead\"\n            )\n\n        self.rnn_output_dim = hidden_dim * 2 if bidirectional else hidden_dim\n\n        # FC-Head (Mlp)\n        if self.head_hidden_dims is not None:\n            head_hidden_dims = [self.rnn_output_dim] + self.head_hidden_dims\n            self.rnn_mlp: Union[MLP, nn.Identity] = MLP(\n                head_hidden_dims,\n                head_activation,\n                head_dropout,\n                head_batchnorm,\n                head_batchnorm_last,\n                head_linear_first,\n            )\n        else:\n            # simple hack to add readability in the forward pass\n            self.rnn_mlp = nn.Identity()\n\n    def forward(self, X: Tensor) -&gt; Tensor:\n        embed = self.word_embed(X.long())\n\n        if self.rnn_type.lower() == \"lstm\":\n            o, (h, c) = self.rnn(embed)\n        elif self.rnn_type.lower() == \"gru\":\n            o, h = self.rnn(embed)\n        else:\n            raise ValueError(\n                f\"'rnn_type' must be 'lstm' or 'gru', got {self.rnn_type} instead\"\n            )\n\n        processed_outputs = self._process_rnn_outputs(o, h)\n\n        return self.rnn_mlp(processed_outputs)\n\n    @property\n    def output_dim(self) -&gt; int:\n        r\"\"\"The output dimension of the model. This is a required property\n        neccesary to build the `WideDeep` class\n        \"\"\"\n        return (\n            self.head_hidden_dims[-1]\n            if self.head_hidden_dims is not None\n            else self.rnn_output_dim\n        )\n\n    def _set_embeddings(\n        self, embed_matrix: Optional[np.ndarray] = None\n    ) -&gt; Tuple[nn.Module, int]:\n        if embed_matrix is not None:\n            assert (\n                embed_matrix.dtype == \"float32\"\n            ), \"'embed_matrix' must be of dtype 'float32', got dtype '{}'\".format(\n                str(embed_matrix.dtype)\n            )\n            word_embed = nn.Embedding(\n                self.vocab_size, embed_matrix.shape[1], padding_idx=self.padding_idx\n            )\n            if self.embed_trainable:\n                word_embed.weight = nn.Parameter(\n                    torch.tensor(embed_matrix), requires_grad=True\n                )\n            else:\n                word_embed.weight = nn.Parameter(\n                    torch.tensor(embed_matrix), requires_grad=False\n                )\n            embed_dim = embed_matrix.shape[1]\n        else:\n            assert self.embed_dim is not None\n            word_embed = nn.Embedding(\n                self.vocab_size, self.embed_dim, padding_idx=self.padding_idx\n            )\n            embed_dim = self.embed_dim\n\n        return word_embed, embed_dim\n\n    def _process_rnn_outputs(self, output: Tensor, hidden: Tensor) -&gt; Tensor:\n        output = output.permute(1, 0, 2)\n        if self.bidirectional:\n            processed_outputs = (\n                torch.cat((hidden[-2], hidden[-1]), dim=1)\n                if self.use_hidden_state\n                else output[-1]\n            )\n        else:\n            processed_outputs = hidden[-1] if self.use_hidden_state else output[-1]\n\n        return processed_outputs\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.text.rnns.basic_rnn.BasicRNN.output_dim","title":"output_dim  <code>property</code>","text":"<pre><code>output_dim\n</code></pre> <p>The output dimension of the model. This is a required property neccesary to build the <code>WideDeep</code> class</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.text.rnns.attentive_rnn.AttentiveRNN","title":"AttentiveRNN","text":"<p>               Bases: <code>BasicRNN</code></p> <p>Text classifier/regressor comprised by a stack of RNNs (LSTMs or GRUs) plus an attention layer. This model can be used as the <code>deeptext</code> component of a Wide &amp; Deep model or independently by itself.</p> <p>In addition, there is the option to add a Fully Connected (FC) set of dense layers on top of attention layer</p> <p>Parameters:</p> Name Type Description Default <code>vocab_size</code> <code>int</code> <p>Number of words in the vocabulary</p> required <code>embed_dim</code> <code>Optional[int]</code> <p>Dimension of the word embeddings if non-pretained word vectors are used</p> <code>None</code> <code>embed_matrix</code> <code>Optional[ndarray]</code> <p>Pretrained word embeddings</p> <code>None</code> <code>embed_trainable</code> <code>bool</code> <p>Boolean indicating if the pretrained embeddings are trainable</p> <code>True</code> <code>rnn_type</code> <code>Literal[lstm, gru]</code> <p>String indicating the type of RNN to use. One of 'lstm' or 'gru'</p> <code>'lstm'</code> <code>hidden_dim</code> <code>int</code> <p>Hidden dim of the RNN</p> <code>64</code> <code>n_layers</code> <code>int</code> <p>Number of recurrent layers</p> <code>3</code> <code>rnn_dropout</code> <code>float</code> <p>Dropout for each RNN layer except the last layer</p> <code>0.1</code> <code>bidirectional</code> <code>bool</code> <p>Boolean indicating whether the staked RNNs are bidirectional</p> <code>False</code> <code>use_hidden_state</code> <code>bool</code> <p>Boolean indicating whether to use the final hidden state or the RNN's output as predicting features. Typically the former is used.</p> <code>True</code> <code>padding_idx</code> <code>int</code> <p>index of the padding token in the padded-tokenised sequences. The <code>TextPreprocessor</code> class within this library uses fastai's tokenizer where the token index 0 is reserved for the 'unknown' word token. Therefore, the default value is set to 1.</p> <code>1</code> <code>attn_concatenate</code> <code>bool</code> <p>Boolean indicating if the input to the attention mechanism will be the output of the RNN or the output of the RNN concatenated with the last hidden state.</p> <code>True</code> <code>attn_dropout</code> <code>float</code> <p>Internal dropout for the attention mechanism</p> <code>0.1</code> <code>head_hidden_dims</code> <code>Optional[List[int]]</code> <p>List with the sizes of the dense layers in the head e.g: [128, 64]</p> <code>None</code> <code>head_activation</code> <code>str</code> <p>Activation function for the dense layers in the head. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> <code>'relu'</code> <code>head_dropout</code> <code>Optional[float]</code> <p>Dropout of the dense layers in the head</p> <code>None</code> <code>head_batchnorm</code> <code>bool</code> <p>Boolean indicating whether or not to include batch normalization in the dense layers that form the 'rnn_mlp'</p> <code>False</code> <code>head_batchnorm_last</code> <code>bool</code> <p>Boolean indicating whether or not to apply batch normalization to the last of the dense layers in the head</p> <code>False</code> <code>head_linear_first</code> <code>bool</code> <p>Boolean indicating whether the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code></p> <code>False</code> <p>Attributes:</p> Name Type Description <code>word_embed</code> <code>Module</code> <p>word embedding matrix</p> <code>rnn</code> <code>Module</code> <p>Stack of RNNs</p> <code>rnn_mlp</code> <code>Module</code> <p>Stack of dense layers on top of the RNN. This will only exists if <code>head_layers_dim</code> is not <code>None</code></p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import AttentiveRNN\n&gt;&gt;&gt; X_text = torch.cat((torch.zeros([5,1]), torch.empty(5, 4).random_(1,4)), axis=1)\n&gt;&gt;&gt; model = AttentiveRNN(vocab_size=4, hidden_dim=4, n_layers=2, padding_idx=0, embed_dim=4)\n&gt;&gt;&gt; out = model(X_text)\n</code></pre> Source code in <code>pytorch_widedeep/models/text/rnns/attentive_rnn.py</code> <pre><code>class AttentiveRNN(BasicRNN):\n    r\"\"\"Text classifier/regressor comprised by a stack of RNNs\n    (LSTMs or GRUs) plus an attention layer. This model can be used as the\n    `deeptext` component of a Wide &amp; Deep model or independently by\n    itself.\n\n    In addition, there is the option to add a Fully Connected (FC) set of dense\n    layers on top of attention layer\n\n    Parameters\n    ----------\n    vocab_size: int\n        Number of words in the vocabulary\n    embed_dim: int, Optional, default = None\n        Dimension of the word embeddings if non-pretained word vectors are\n        used\n    embed_matrix: np.ndarray, Optional, default = None\n        Pretrained word embeddings\n    embed_trainable: bool, default = True\n        Boolean indicating if the pretrained embeddings are trainable\n    rnn_type: str, default = 'lstm'\n        String indicating the type of RNN to use. One of _'lstm'_ or _'gru'_\n    hidden_dim: int, default = 64\n        Hidden dim of the RNN\n    n_layers: int, default = 3\n        Number of recurrent layers\n    rnn_dropout: float, default = 0.1\n        Dropout for each RNN layer except the last layer\n    bidirectional: bool, default = True\n        Boolean indicating whether the staked RNNs are bidirectional\n    use_hidden_state: str, default = True\n        Boolean indicating whether to use the final hidden state or the RNN's\n        output as predicting features. Typically the former is used.\n    padding_idx: int, default = 1\n        index of the padding token in the padded-tokenised sequences. The\n        `TextPreprocessor` class within this library uses fastai's\n        tokenizer where the token index 0 is reserved for the _'unknown'_\n        word token. Therefore, the default value is set to 1.\n    attn_concatenate: bool, default = True\n        Boolean indicating if the input to the attention mechanism will be the\n        output of the RNN or the output of the RNN concatenated with the last\n        hidden state.\n    attn_dropout: float, default = 0.1\n        Internal dropout for the attention mechanism\n    head_hidden_dims: List, Optional, default = None\n        List with the sizes of the dense layers in the head e.g: _[128, 64]_\n    head_activation: str, default = \"relu\"\n        Activation function for the dense layers in the head. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported\n    head_dropout: float, Optional, default = None\n        Dropout of the dense layers in the head\n    head_batchnorm: bool, default = False\n        Boolean indicating whether or not to include batch normalization in\n        the dense layers that form the _'rnn_mlp'_\n    head_batchnorm_last: bool, default = False\n        Boolean indicating whether or not to apply batch normalization to the\n        last of the dense layers in the head\n    head_linear_first: bool, default = False\n        Boolean indicating whether the order of the operations in the dense\n        layer. If `True: [LIN -&gt; ACT -&gt; BN -&gt; DP]`. If `False: [BN -&gt; DP -&gt;\n        LIN -&gt; ACT]`\n\n    Attributes\n    ----------\n    word_embed: nn.Module\n        word embedding matrix\n    rnn: nn.Module\n        Stack of RNNs\n    rnn_mlp: nn.Module\n        Stack of dense layers on top of the RNN. This will only exists if\n        `head_layers_dim` is not `None`\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.models import AttentiveRNN\n    &gt;&gt;&gt; X_text = torch.cat((torch.zeros([5,1]), torch.empty(5, 4).random_(1,4)), axis=1)\n    &gt;&gt;&gt; model = AttentiveRNN(vocab_size=4, hidden_dim=4, n_layers=2, padding_idx=0, embed_dim=4)\n    &gt;&gt;&gt; out = model(X_text)\n    \"\"\"\n\n    def __init__(\n        self,\n        vocab_size: int,\n        embed_dim: Optional[int] = None,\n        embed_matrix: Optional[np.ndarray] = None,\n        embed_trainable: bool = True,\n        rnn_type: Literal[\"lstm\", \"gru\"] = \"lstm\",\n        hidden_dim: int = 64,\n        n_layers: int = 3,\n        rnn_dropout: float = 0.1,\n        bidirectional: bool = False,\n        use_hidden_state: bool = True,\n        padding_idx: int = 1,\n        attn_concatenate: bool = True,\n        attn_dropout: float = 0.1,\n        head_hidden_dims: Optional[List[int]] = None,\n        head_activation: str = \"relu\",\n        head_dropout: Optional[float] = None,\n        head_batchnorm: bool = False,\n        head_batchnorm_last: bool = False,\n        head_linear_first: bool = False,\n    ):\n        super(AttentiveRNN, self).__init__(\n            vocab_size=vocab_size,\n            embed_dim=embed_dim,\n            embed_matrix=embed_matrix,\n            embed_trainable=embed_trainable,\n            rnn_type=rnn_type,\n            hidden_dim=hidden_dim,\n            n_layers=n_layers,\n            rnn_dropout=rnn_dropout,\n            bidirectional=bidirectional,\n            use_hidden_state=use_hidden_state,\n            padding_idx=padding_idx,\n            head_hidden_dims=head_hidden_dims,\n            head_activation=head_activation,\n            head_dropout=head_dropout,\n            head_batchnorm=head_batchnorm,\n            head_batchnorm_last=head_batchnorm_last,\n            head_linear_first=head_linear_first,\n        )\n\n        # Embeddings and RNN defined in the BasicRNN inherited class\n\n        # Attention\n        self.attn_concatenate = attn_concatenate\n        self.attn_dropout = attn_dropout\n\n        if bidirectional and attn_concatenate:\n            self.rnn_output_dim = hidden_dim * 4\n        elif bidirectional or attn_concatenate:\n            self.rnn_output_dim = hidden_dim * 2\n        else:\n            self.rnn_output_dim = hidden_dim\n        self.attn = ContextAttention(\n            self.rnn_output_dim, attn_dropout, sum_along_seq=True\n        )\n\n        # FC-Head (Mlp)\n        if self.head_hidden_dims is not None:\n            head_hidden_dims = [self.rnn_output_dim] + self.head_hidden_dims\n            self.rnn_mlp = MLP(\n                head_hidden_dims,\n                head_activation,\n                head_dropout,\n                head_batchnorm,\n                head_batchnorm_last,\n                head_linear_first,\n            )\n\n    def _process_rnn_outputs(self, output: Tensor, hidden: Tensor) -&gt; Tensor:\n        if self.attn_concatenate:\n            if self.bidirectional:\n                bi_hidden = torch.cat((hidden[-2], hidden[-1]), dim=1)\n                attn_inp = torch.cat(\n                    [output, bi_hidden.unsqueeze(1).expand_as(output)], dim=2\n                )\n            else:\n                attn_inp = torch.cat(\n                    [output, hidden[-1].unsqueeze(1).expand_as(output)], dim=2\n                )\n        else:\n            attn_inp = output\n\n        return self.attn(attn_inp)\n\n    @property\n    def attention_weights(self) -&gt; List:\n        r\"\"\"List with the attention weights\n\n        The shape of the attention weights is $(N, S)$, where $N$ is the batch\n        size and $S$ is the length of the sequence\n        \"\"\"\n        return self.attn.attn_weights\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.text.rnns.attentive_rnn.AttentiveRNN.attention_weights","title":"attention_weights  <code>property</code>","text":"<pre><code>attention_weights\n</code></pre> <p>List with the attention weights</p> <p>The shape of the attention weights is \\((N, S)\\), where \\(N\\) is the batch size and \\(S\\) is the length of the sequence</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.text.rnns.stacked_attentive_rnn.StackedAttentiveRNN","title":"StackedAttentiveRNN","text":"<p>               Bases: <code>BaseWDModelComponent</code></p> <p>Text classifier/regressor comprised by a stack of blocks: <code>[RNN + Attention]</code>. This can be used as the <code>deeptext</code> component of a Wide &amp; Deep model or independently by itself.</p> <p>In addition, there is the option to add a Fully Connected (FC) set of dense layers on top of the attentiob blocks</p> <p>Parameters:</p> Name Type Description Default <code>vocab_size</code> <code>int</code> <p>Number of words in the vocabulary</p> required <code>embed_dim</code> <code>Optional[int]</code> <p>Dimension of the word embeddings if non-pretained word vectors are used</p> <code>None</code> <code>embed_matrix</code> <code>Optional[ndarray]</code> <p>Pretrained word embeddings</p> <code>None</code> <code>embed_trainable</code> <code>bool</code> <p>Boolean indicating if the pretrained embeddings are trainable</p> <code>True</code> <code>rnn_type</code> <code>Literal[lstm, gru]</code> <p>String indicating the type of RNN to use. One of 'lstm' or 'gru'</p> <code>'lstm'</code> <code>hidden_dim</code> <code>int</code> <p>Hidden dim of the RNN</p> <code>64</code> <code>bidirectional</code> <code>bool</code> <p>Boolean indicating whether the staked RNNs are bidirectional</p> <code>False</code> <code>padding_idx</code> <code>int</code> <p>index of the padding token in the padded-tokenised sequences. The <code>TextPreprocessor</code> class within this library uses fastai's tokenizer where the token index 0 is reserved for the 'unknown' word token. Therefore, the default value is set to 1.</p> <code>1</code> <code>n_blocks</code> <code>int</code> <p>Number of attention blocks. Each block is comprised by an RNN and a Context Attention Encoder</p> <code>3</code> <code>attn_concatenate</code> <code>bool</code> <p>Boolean indicating if the input to the attention mechanism will be the output of the RNN or the output of the RNN concatenated with the last hidden state or simply</p> <code>False</code> <code>attn_dropout</code> <code>float</code> <p>Internal dropout for the attention mechanism</p> <code>0.1</code> <code>with_addnorm</code> <code>bool</code> <p>Boolean indicating if the output of each block will be added to the input and normalised</p> <code>False</code> <code>head_hidden_dims</code> <code>Optional[List[int]]</code> <p>List with the sizes of the dense layers in the head e.g: [128, 64]</p> <code>None</code> <code>head_activation</code> <code>str</code> <p>Activation function for the dense layers in the head. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> <code>'relu'</code> <code>head_dropout</code> <code>Optional[float]</code> <p>Dropout of the dense layers in the head</p> <code>None</code> <code>head_batchnorm</code> <code>bool</code> <p>Boolean indicating whether or not to include batch normalization in the dense layers that form the 'rnn_mlp'</p> <code>False</code> <code>head_batchnorm_last</code> <code>bool</code> <p>Boolean indicating whether or not to apply batch normalization to the last of the dense layers in the head</p> <code>False</code> <code>head_linear_first</code> <code>bool</code> <p>Boolean indicating whether the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code></p> <code>False</code> <p>Attributes:</p> Name Type Description <code>word_embed</code> <code>Module</code> <p>word embedding matrix</p> <code>rnn</code> <code>Module</code> <p>Stack of RNNs</p> <code>rnn_mlp</code> <code>Module</code> <p>Stack of dense layers on top of the RNN. This will only exists if <code>head_layers_dim</code> is not <code>None</code></p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import StackedAttentiveRNN\n&gt;&gt;&gt; X_text = torch.cat((torch.zeros([5,1]), torch.empty(5, 4).random_(1,4)), axis=1)\n&gt;&gt;&gt; model = StackedAttentiveRNN(vocab_size=4, hidden_dim=4, padding_idx=0, embed_dim=4)\n&gt;&gt;&gt; out = model(X_text)\n</code></pre> Source code in <code>pytorch_widedeep/models/text/rnns/stacked_attentive_rnn.py</code> <pre><code>class StackedAttentiveRNN(BaseWDModelComponent):\n    r\"\"\"Text classifier/regressor comprised by a stack of blocks:\n    `[RNN + Attention]`. This can be used as the `deeptext` component of a\n    Wide &amp; Deep model or independently by itself.\n\n    In addition, there is the option to add a Fully Connected (FC) set of\n    dense layers on top of the attentiob blocks\n\n    Parameters\n    ----------\n    vocab_size: int\n        Number of words in the vocabulary\n    embed_dim: int, Optional, default = None\n        Dimension of the word embeddings if non-pretained word vectors are\n        used\n    embed_matrix: np.ndarray, Optional, default = None\n        Pretrained word embeddings\n    embed_trainable: bool, default = True\n        Boolean indicating if the pretrained embeddings are trainable\n    rnn_type: str, default = 'lstm'\n        String indicating the type of RNN to use. One of 'lstm' or 'gru'\n    hidden_dim: int, default = 64\n        Hidden dim of the RNN\n    bidirectional: bool, default = True\n        Boolean indicating whether the staked RNNs are bidirectional\n    padding_idx: int, default = 1\n        index of the padding token in the padded-tokenised sequences. The\n        `TextPreprocessor` class within this library uses fastai's\n        tokenizer where the token index 0 is reserved for the _'unknown'_\n        word token. Therefore, the default value is set to 1.\n    n_blocks: int, default = 3\n        Number of attention blocks. Each block is comprised by an RNN and a\n        Context Attention Encoder\n    attn_concatenate: bool, default = True\n        Boolean indicating if the input to the attention mechanism will be the\n        output of the RNN or the output of the RNN concatenated with the last\n        hidden state or simply\n    attn_dropout: float, default = 0.1\n        Internal dropout for the attention mechanism\n    with_addnorm: bool, default = False\n        Boolean indicating if the output of each block will be added to the\n        input and normalised\n    head_hidden_dims: List, Optional, default = None\n        List with the sizes of the dense layers in the head e.g: [128, 64]\n    head_activation: str, default = \"relu\"\n        Activation function for the dense layers in the head. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported\n    head_dropout: float, Optional, default = None\n        Dropout of the dense layers in the head\n    head_batchnorm: bool, default = False\n        Boolean indicating whether or not to include batch normalization in\n        the dense layers that form the _'rnn_mlp'_\n    head_batchnorm_last: bool, default = False\n        Boolean indicating whether or not to apply batch normalization to the\n        last of the dense layers in the head\n    head_linear_first: bool, default = False\n        Boolean indicating whether the order of the operations in the dense\n        layer. If `True: [LIN -&gt; ACT -&gt; BN -&gt; DP]`. If `False: [BN -&gt; DP -&gt;\n        LIN -&gt; ACT]`\n\n    Attributes\n    ----------\n    word_embed: nn.Module\n        word embedding matrix\n    rnn: nn.Module\n        Stack of RNNs\n    rnn_mlp: nn.Module\n        Stack of dense layers on top of the RNN. This will only exists if\n        `head_layers_dim` is not `None`\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.models import StackedAttentiveRNN\n    &gt;&gt;&gt; X_text = torch.cat((torch.zeros([5,1]), torch.empty(5, 4).random_(1,4)), axis=1)\n    &gt;&gt;&gt; model = StackedAttentiveRNN(vocab_size=4, hidden_dim=4, padding_idx=0, embed_dim=4)\n    &gt;&gt;&gt; out = model(X_text)\n    \"\"\"\n\n    def __init__(\n        self,\n        vocab_size: int,\n        embed_dim: Optional[int] = None,\n        embed_matrix: Optional[np.ndarray] = None,\n        embed_trainable: bool = True,\n        rnn_type: Literal[\"lstm\", \"gru\"] = \"lstm\",\n        hidden_dim: int = 64,\n        bidirectional: bool = False,\n        padding_idx: int = 1,\n        n_blocks: int = 3,\n        attn_concatenate: bool = False,\n        attn_dropout: float = 0.1,\n        with_addnorm: bool = False,\n        head_hidden_dims: Optional[List[int]] = None,\n        head_activation: str = \"relu\",\n        head_dropout: Optional[float] = None,\n        head_batchnorm: bool = False,\n        head_batchnorm_last: bool = False,\n        head_linear_first: bool = False,\n    ):\n        super(StackedAttentiveRNN, self).__init__()\n\n        if (\n            embed_dim is not None\n            and embed_matrix is not None\n            and not embed_dim == embed_matrix.shape[1]\n        ):\n            warnings.warn(\n                \"the input embedding dimension {} and the dimension of the \"\n                \"pretrained embeddings {} do not match. The pretrained embeddings \"\n                \"dimension ({}) will be used\".format(\n                    embed_dim, embed_matrix.shape[1], embed_matrix.shape[1]\n                ),\n                UserWarning,\n            )\n\n        if rnn_type.lower() not in [\"lstm\", \"gru\"]:\n            raise ValueError(\n                f\"'rnn_type' must be 'lstm' or 'gru', got {rnn_type} instead\"\n            )\n\n        self.vocab_size = vocab_size\n        self.embed_trainable = embed_trainable\n        self.embed_dim = embed_dim\n\n        self.rnn_type = rnn_type\n        self.hidden_dim = hidden_dim\n        self.bidirectional = bidirectional\n        self.padding_idx = padding_idx\n\n        self.n_blocks = n_blocks\n        self.attn_concatenate = attn_concatenate\n        self.attn_dropout = attn_dropout\n        self.with_addnorm = with_addnorm\n\n        self.head_hidden_dims = head_hidden_dims\n        self.head_activation = head_activation\n        self.head_dropout = head_dropout\n        self.head_batchnorm = head_batchnorm\n        self.head_batchnorm_last = head_batchnorm_last\n        self.head_linear_first = head_linear_first\n\n        # Embeddings\n        self.word_embed, self.embed_dim = self._set_embeddings(embed_matrix)\n\n        # Linear Projection: if embed_dim is different that the input of the\n        # attention blocks we add a linear projection\n        if bidirectional and attn_concatenate:\n            self.rnn_output_dim = hidden_dim * 4\n        elif bidirectional or attn_concatenate:\n            self.rnn_output_dim = hidden_dim * 2\n        else:\n            self.rnn_output_dim = hidden_dim\n\n        if self.rnn_output_dim != self.embed_dim:\n            self.embed_proj: Union[nn.Linear, nn.Identity] = nn.Linear(\n                self.embed_dim, self.rnn_output_dim\n            )\n        else:\n            self.embed_proj = nn.Identity()\n\n        # RNN\n        rnn_params = {\n            \"input_size\": self.rnn_output_dim,\n            \"hidden_size\": hidden_dim,\n            \"bidirectional\": bidirectional,\n            \"batch_first\": True,\n        }\n        if self.rnn_type.lower() == \"lstm\":\n            self.rnn: Union[nn.LSTM, nn.GRU] = nn.LSTM(**rnn_params)\n        elif self.rnn_type.lower() == \"gru\":\n            self.rnn = nn.GRU(**rnn_params)\n\n        # FC-Head (Mlp)\n        self.attention_blks = nn.ModuleList()\n        for i in range(n_blocks):\n            self.attention_blks.append(\n                ContextAttentionEncoder(\n                    self.rnn,\n                    self.rnn_output_dim,\n                    attn_dropout,\n                    attn_concatenate,\n                    with_addnorm=with_addnorm if i != n_blocks - 1 else False,\n                    sum_along_seq=i == n_blocks - 1,\n                )\n            )\n\n        # Mlp\n        if self.head_hidden_dims is not None:\n            head_hidden_dims = [self.rnn_output_dim] + self.head_hidden_dims\n            self.rnn_mlp: Union[MLP, nn.Identity] = MLP(\n                head_hidden_dims,\n                head_activation,\n                head_dropout,\n                head_batchnorm,\n                head_batchnorm_last,\n                head_linear_first,\n            )\n        else:\n            # simple hack to add readability in the forward pass\n            self.rnn_mlp = nn.Identity()\n\n    def forward(self, X: Tensor) -&gt; Tensor:  # type: ignore\n        x = self.embed_proj(self.word_embed(X.long()))\n\n        h = nn.init.zeros_(\n            torch.Tensor(2 if self.bidirectional else 1, X.shape[0], self.hidden_dim)\n        ).to(x.device)\n        if self.rnn_type == \"lstm\":\n            c = nn.init.zeros_(\n                torch.Tensor(\n                    2 if self.bidirectional else 1, X.shape[0], self.hidden_dim\n                )\n            ).to(x.device)\n        else:\n            c = None\n\n        for blk in self.attention_blks:\n            x, h, c = blk(x, h, c)\n\n        return self.rnn_mlp(x)\n\n    @property\n    def output_dim(self) -&gt; int:\n        r\"\"\"The output dimension of the model. This is a required property\n        neccesary to build the `WideDeep` class\n        \"\"\"\n        return (\n            self.head_hidden_dims[-1]\n            if self.head_hidden_dims is not None\n            else self.rnn_output_dim\n        )\n\n    @property\n    def attention_weights(self) -&gt; List:\n        r\"\"\"List with the attention weights per block\n\n        The shape of the attention weights is $(N, S)$ Where $N$ is the batch\n        size and $S$ is the length of the sequence\n        \"\"\"\n        return [blk.attn.attn_weights for blk in self.attention_blks]\n\n    def _set_embeddings(\n        self, embed_matrix: Union[Any, np.ndarray]\n    ) -&gt; Tuple[nn.Module, int]:\n        if isinstance(embed_matrix, np.ndarray):\n            assert (\n                embed_matrix.dtype == \"float32\"\n            ), \"'embed_matrix' must be of dtype 'float32', got dtype '{}'\".format(\n                str(embed_matrix.dtype)\n            )\n            word_embed = nn.Embedding(\n                self.vocab_size, embed_matrix.shape[1], padding_idx=self.padding_idx\n            )\n            if self.embed_trainable:\n                word_embed.weight = nn.Parameter(\n                    torch.tensor(embed_matrix), requires_grad=True\n                )\n            else:\n                word_embed.weight = nn.Parameter(\n                    torch.tensor(embed_matrix), requires_grad=False\n                )\n            embed_dim = embed_matrix.shape[1]\n        else:\n            assert self.embed_dim is not None\n            word_embed = nn.Embedding(\n                self.vocab_size, self.embed_dim, padding_idx=self.padding_idx\n            )\n            embed_dim = self.embed_dim\n\n        return word_embed, embed_dim\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.text.rnns.stacked_attentive_rnn.StackedAttentiveRNN.output_dim","title":"output_dim  <code>property</code>","text":"<pre><code>output_dim\n</code></pre> <p>The output dimension of the model. This is a required property neccesary to build the <code>WideDeep</code> class</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.text.rnns.stacked_attentive_rnn.StackedAttentiveRNN.attention_weights","title":"attention_weights  <code>property</code>","text":"<pre><code>attention_weights\n</code></pre> <p>List with the attention weights per block</p> <p>The shape of the attention weights is \\((N, S)\\) Where \\(N\\) is the batch size and \\(S\\) is the length of the sequence</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.text.huggingface_transformers.hf_model.HFModel","title":"HFModel","text":"<p>               Bases: <code>BaseWDModelComponent</code></p> <p>This class is a wrapper around the Hugging Face transformers library. It can be used as the text component of a Wide &amp; Deep model or independently by itself.</p> <p>At the moment only models from the families BERT, RoBERTa, DistilBERT, ALBERT and ELECTRA are supported. This is because this library is designed to address classification and regression tasks and these are the most 'popular' encoder-only models, which have proved to be those that work best for these tasks.</p> <p>Parameters:</p> Name Type Description Default <code>model_name</code> <code>str</code> <p>The model name from the transformers library e.g. 'bert-base-uncased'. Currently supported models are those from the families: BERT, RoBERTa, DistilBERT, ALBERT and ELECTRA.</p> required <code>use_cls_token</code> <code>bool</code> <p>Boolean indicating whether to use the [CLS] token or the mean of the sequence of hidden states as the sentence embedding</p> <code>True</code> <code>trainable_parameters</code> <code>Optional[List[str]]</code> <p>List with the names of the model parameters that will be trained. If None, none of the parameters will be trainable</p> <code>None</code> <code>head_hidden_dims</code> <code>Optional[List[int]]</code> <p>List with the sizes of the dense layers in the head e.g: [128, 64]</p> <code>None</code> <code>head_activation</code> <code>str</code> <p>Activation function for the dense layers in the head. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> <code>'relu'</code> <code>head_dropout</code> <code>Optional[float]</code> <p>Dropout of the dense layers in the head</p> <code>None</code> <code>head_batchnorm</code> <code>bool</code> <p>Boolean indicating whether or not to include batch normalization in the dense layers that form the head</p> <code>False</code> <code>head_batchnorm_last</code> <code>bool</code> <p>Boolean indicating whether or not to apply batch normalization to the last of the dense layers in the head</p> <code>False</code> <code>head_linear_first</code> <code>bool</code> <p>Boolean indicating whether the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code></p> <code>False</code> <code>verbose</code> <code>bool</code> <p>If True, it will print information about the model</p> <code>False</code> <code>**kwargs</code> <p>Additional kwargs to be passed to the model</p> <code>{}</code> <p>Attributes:</p> Name Type Description <code>head</code> <code>Module</code> <p>Stack of dense layers on top of the transformer. This will only exists if <code>head_layers_dim</code> is not None</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import HFModel\n&gt;&gt;&gt; X_text = torch.cat((torch.zeros([5,1]), torch.empty(5, 4).random_(1,4)), axis=1).long()\n&gt;&gt;&gt; model = HFModel(model_name='bert-base-uncased')\n&gt;&gt;&gt; out = model(X_text)\n</code></pre> Source code in <code>pytorch_widedeep/models/text/huggingface_transformers/hf_model.py</code> <pre><code>class HFModel(BaseWDModelComponent):\n    \"\"\"This class is a wrapper around the Hugging Face transformers library. It\n    can be used as the text component of a Wide &amp; Deep model or independently\n    by itself.\n\n    At the moment only models from the families BERT, RoBERTa, DistilBERT,\n    ALBERT and ELECTRA are supported. This is because this library is\n    designed to address classification and regression tasks and these are the\n    most 'popular' encoder-only models, which have proved to be those that\n    work best for these tasks.\n\n    Parameters\n    ----------\n    model_name: str\n        The model name from the transformers library e.g. 'bert-base-uncased'.\n        Currently supported models are those from the families: BERT, RoBERTa,\n        DistilBERT, ALBERT and ELECTRA.\n    use_cls_token: bool, default = True\n        Boolean indicating whether to use the [CLS] token or the mean of the\n        sequence of hidden states as the sentence embedding\n    trainable_parameters: List, Optional, default = None\n        List with the names of the model parameters that will be trained. If\n        None, none of the parameters will be trainable\n    head_hidden_dims: List, Optional, default = None\n        List with the sizes of the dense layers in the head e.g: _[128, 64]_\n    head_activation: str, default = \"relu\"\n        Activation function for the dense layers in the head. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported\n    head_dropout: float, Optional, default = None\n        Dropout of the dense layers in the head\n    head_batchnorm: bool, default = False\n        Boolean indicating whether or not to include batch normalization in the\n        dense layers that form the head\n    head_batchnorm_last: bool, default = False\n        Boolean indicating whether or not to apply batch normalization to the\n        last of the dense layers in the head\n    head_linear_first: bool, default = False\n        Boolean indicating whether the order of the operations in the dense\n        layer. If `True: [LIN -&gt; ACT -&gt; BN -&gt; DP]`. If `False: [BN -&gt; DP -&gt;\n        LIN -&gt; ACT]`\n    verbose: bool, default = False\n        If True, it will print information about the model\n    **kwargs\n        Additional kwargs to be passed to the model\n\n    Attributes\n    ----------\n    head: nn.Module\n        Stack of dense layers on top of the transformer. This will only exists\n        if `head_layers_dim` is not None\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.models import HFModel\n    &gt;&gt;&gt; X_text = torch.cat((torch.zeros([5,1]), torch.empty(5, 4).random_(1,4)), axis=1).long()\n    &gt;&gt;&gt; model = HFModel(model_name='bert-base-uncased')\n    &gt;&gt;&gt; out = model(X_text)\n    \"\"\"\n\n    @alias(\"use_cls_token\", [\"use_special_token\"])\n    def __init__(\n        self,\n        model_name: str,\n        use_cls_token: bool = True,\n        trainable_parameters: Optional[List[str]] = None,\n        head_hidden_dims: Optional[List[int]] = None,\n        head_activation: str = \"relu\",\n        head_dropout: Optional[float] = None,\n        head_batchnorm: bool = False,\n        head_batchnorm_last: bool = False,\n        head_linear_first: bool = False,\n        verbose: bool = False,\n        **kwargs,\n    ):\n        super().__init__()\n\n        # TO DO: add warning regarging ELECTRA as ELECTRA does not have a cls\n        # token.  Research what happens with ELECTRA\n        self.model_name = model_name\n        self.use_cls_token = use_cls_token\n        self.trainable_parameters = trainable_parameters\n        self.head_hidden_dims = head_hidden_dims\n        self.head_activation = head_activation\n        self.head_dropout = head_dropout\n        self.head_batchnorm = head_batchnorm\n        self.head_batchnorm_last = head_batchnorm_last\n        self.head_linear_first = head_linear_first\n        self.verbose = verbose\n        self.kwargs = kwargs\n\n        if self.verbose and self.use_cls_token:\n            warnings.warn(\n                \"The model will use the [CLS] token. Make sure the tokenizer \"\n                \"was run with add_special_tokens=True\",\n                UserWarning,\n            )\n\n        self.model_class = get_model_class(model_name)\n\n        self.config, self.model = get_config_and_model(self.model_name)\n\n        self.output_attention_weights = kwargs.get(\"output_attentions\", False)\n\n        if self.trainable_parameters is not None:\n            for n, p in self.model.named_parameters():\n                p.requires_grad = any([tl in n for tl in self.trainable_parameters])\n\n        # FC-Head (Mlp). Note that the FC head will always be trainable\n        if self.head_hidden_dims is not None:\n            head_hidden_dims = [self.config.hidden_size] + self.head_hidden_dims\n            self.head = MLP(\n                head_hidden_dims,\n                head_activation,\n                head_dropout,\n                head_batchnorm,\n                head_batchnorm_last,\n                head_linear_first,\n            )\n\n    def forward(self, X: Tensor) -&gt; Tensor:\n\n        # this is inefficient since the attention mask is returned by the\n        # tokenizer, but all models in this library use a forward pass that\n        # takes ONLY an input tensor. A fix will be addressed in a future\n        attn_mask = (X != 0).type(torch.int8)\n\n        output = self.model(input_ids=X, attention_mask=attn_mask, **self.kwargs)\n\n        if self.output_attention_weights:\n            # TO CONSIDER: attention weights as a returned object and not an\n            # attribute\n            self.attn_weights = output[\"attentions\"]\n\n        if self.use_cls_token:\n            output = output[0][:, 0, :]\n        else:\n            # Here one can choose to flatten, but unless the sequence length\n            # is very small, flatten will result in a very large output\n            # tensor.\n            output = output[0].mean(dim=1)\n\n        if self.head_hidden_dims is not None:\n            output = self.head(output)\n\n        return output\n\n    @property\n    def output_dim(self) -&gt; int:\n        return (\n            self.head_hidden_dims[-1]\n            if self.head_hidden_dims is not None\n            else self.config.hidden_size\n        )\n\n    @property\n    def attention_weight(self) -&gt; Tensor:\n        r\"\"\"Returns the attention weights if the model was created with the\n        output_attention_weights=True argument. If not, it will raise an\n        AttributeError.\n\n        The shape of the attention weights is $(N, H, F, F)$, where $N$ is the\n        batch size, $H$ is the number of attention heads and $F$ is the\n        sequence length.\n        \"\"\"\n        if not self.output_attention_weights:\n            raise AttributeError(\n                \"The output_attention_weights attribute was not set to True when creating the model object \"\n                \"Please pass an output_attention_weights=True argument when creating the HFModel object\"\n            )\n        return self.attn_weights\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.text.huggingface_transformers.hf_model.HFModel.attention_weight","title":"attention_weight  <code>property</code>","text":"<pre><code>attention_weight\n</code></pre> <p>Returns the attention weights if the model was created with the output_attention_weights=True argument. If not, it will raise an AttributeError.</p> <p>The shape of the attention weights is \\((N, H, F, F)\\), where \\(N\\) is the batch size, \\(H\\) is the number of attention heads and \\(F\\) is the sequence length.</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.image.vision.Vision","title":"Vision","text":"<p>               Bases: <code>BaseWDModelComponent</code></p> <p>Defines a standard image classifier/regressor using a pretrained network or a sequence of convolution layers that can be used as the <code>deepimage</code> component of a Wide &amp; Deep model or independently by itself.</p> <p> NOTE: this class represents the integration  between <code>pytorch-widedeep</code> and <code>torchvision</code>. New architectures will be  available as they are added to <code>torchvision</code>. In a distant future we aim  to bring transformer-based architectures as well. However, simple  CNN-based architectures (and even MLP-based) seem to produce SoTA  results. For the time being, we describe below the options available  through this class</p> <p>Parameters:</p> Name Type Description Default <code>pretrained_model_setup</code> <code>Union[str, Dict[str, Union[str, WeightsEnum]]]</code> <p>Name of the pretrained model. Should be a variant of the following architectures: 'resnet', 'shufflenet', 'resnext', 'wide_resnet', 'regnet', 'densenet', 'mobilenetv3', 'mobilenetv2', 'mnasnet', 'efficientnet' and 'squeezenet'. if <code>pretrained_model_setup = None</code> a basic, fully trainable CNN will be used. Alternatively, since Torchvision 0.13 one can use pretrained models with different weigths. Therefore, <code>pretrained_model_setup</code> can also be dictionary with the name of the model and the weights (e.g. <code>{'resnet50': ResNet50_Weights.DEFAULT}</code> or <code>{'resnet50': \"IMAGENET1K_V2\"}</code>).  Aliased as <code>pretrained_model_name</code>.</p> <code>None</code> <code>n_trainable</code> <code>Optional[int]</code> <p>Number of trainable layers starting from the layer closer to the output neuron(s). Note that this number DOES NOT take into account the so-called 'head' which is ALWAYS trainable. If <code>trainable_params</code> is not None this parameter will be ignored</p> <code>None</code> <code>trainable_params</code> <code>Optional[List[str]]</code> <p>List of strings containing the names (or substring within the name) of the parameters that will be trained. For example, if we use a 'resnet18' pretrained model and we set <code>trainable_params = ['layer4']</code> only the parameters of 'layer4' of the network (and the head, as mentioned before) will be trained. Note that setting this or the previous parameter involves some knowledge of the architecture used.</p> <code>None</code> <code>channel_sizes</code> <code>List[int]</code> <p>List of integers with the channel sizes of a CNN in case we choose not to use a pretrained model</p> <code>[64, 128, 256, 512]</code> <code>kernel_sizes</code> <code>Union[int, List[int]]</code> <p>List of integers with the kernel sizes of a CNN in case we choose not to use a pretrained model. Must be of length equal to <code>len(channel_sizes) - 1</code>.</p> <code>[7, 3, 3, 3]</code> <code>strides</code> <code>Union[int, List[int]]</code> <p>List of integers with the stride sizes of a CNN in case we choose not to use a pretrained model. Must be of length equal to <code>len(channel_sizes) - 1</code>.</p> <code>[2, 1, 1, 1]</code> <code>head_hidden_dims</code> <code>Optional[List[int]]</code> <p>List with the number of neurons per dense layer in the head. e.g: [64,32]</p> <code>None</code> <code>head_activation</code> <code>str</code> <p>Activation function for the dense layers in the head. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> <code>'relu'</code> <code>head_dropout</code> <code>Union[float, List[float]]</code> <p>float indicating the dropout between the dense layers.</p> <code>0.1</code> <code>head_batchnorm</code> <code>bool</code> <p>Boolean indicating whether or not batch normalization will be applied to the dense layers</p> <code>False</code> <code>head_batchnorm_last</code> <code>bool</code> <p>Boolean indicating whether or not batch normalization will be applied to the last of the dense layers</p> <code>False</code> <code>head_linear_first</code> <code>bool</code> <p>Boolean indicating the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code></p> <code>False</code> <p>Attributes:</p> Name Type Description <code>features</code> <code>Module</code> <p>The pretrained model or Standard CNN plus the optional head</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import Vision\n&gt;&gt;&gt; X_img = torch.rand((2,3,224,224))\n&gt;&gt;&gt; model = Vision(channel_sizes=[64, 128], kernel_sizes = [3, 3], strides=[1, 1], head_hidden_dims=[32, 8])\n&gt;&gt;&gt; out = model(X_img)\n</code></pre> Source code in <code>pytorch_widedeep/models/image/vision.py</code> <pre><code>class Vision(BaseWDModelComponent):\n    r\"\"\"Defines a standard image classifier/regressor using a pretrained\n    network or a sequence of convolution layers that can be used as the\n    `deepimage` component of a Wide &amp; Deep model or independently by\n    itself.\n\n    :information_source: **NOTE**: this class represents the integration\n     between `pytorch-widedeep` and `torchvision`. New architectures will be\n     available as they are added to `torchvision`. In a distant future we aim\n     to bring transformer-based architectures as well. However, simple\n     CNN-based architectures (and even MLP-based) seem to produce SoTA\n     results. For the time being, we describe below the options available\n     through this class\n\n    Parameters\n    ----------\n    pretrained_model_setup: Optional, str or dict, default = None\n        Name of the pretrained model. Should be a variant of the following\n        architectures: _'resnet'_, _'shufflenet'_, _'resnext'_,\n        _'wide_resnet'_, _'regnet'_, _'densenet'_, _'mobilenetv3'_,\n        _'mobilenetv2'_, _'mnasnet'_, _'efficientnet'_ and _'squeezenet'_. if\n        `pretrained_model_setup = None` a basic, fully trainable CNN will be\n        used. Alternatively, since Torchvision 0.13 one can use pretrained\n        models with different weigths. Therefore, `pretrained_model_setup` can\n        also be dictionary with the name of the model and the weights (e.g.\n        `{'resnet50': ResNet50_Weights.DEFAULT}` or\n        `{'resnet50': \"IMAGENET1K_V2\"}`). &lt;br/&gt; Aliased as `pretrained_model_name`.\n    n_trainable: Optional, int, default = None\n        Number of trainable layers starting from the layer closer to the\n        output neuron(s). Note that this number DOES NOT take into account\n        the so-called _'head'_ which is ALWAYS trainable. If\n        `trainable_params` is not None this parameter will be ignored\n    trainable_params: Optional, list, default = None\n        List of strings containing the names (or substring within the name) of\n        the parameters that will be trained. For example, if we use a\n        _'resnet18'_ pretrained model and we set `trainable_params =\n        ['layer4']` only the parameters of _'layer4'_ of the network\n        (and the head, as mentioned before) will be trained. Note that\n        setting this or the previous parameter involves some knowledge of\n        the architecture used.\n    channel_sizes: list, default = [64, 128, 256, 512]\n        List of integers with the channel sizes of a CNN in case we choose not\n        to use a pretrained model\n    kernel_sizes: list or int, default = 3\n        List of integers with the kernel sizes of a CNN in case we choose not\n        to use a pretrained model. Must be of length equal to `len(channel_sizes) - 1`.\n    strides: list or int, default = 1\n        List of integers with the stride sizes of a CNN in case we choose not\n        to use a pretrained model. Must be of length equal to `len(channel_sizes) - 1`.\n    head_hidden_dims: Optional, list, default = None\n        List with the number of neurons per dense layer in the head. e.g: _[64,32]_\n    head_activation: str, default = \"relu\"\n        Activation function for the dense layers in the head. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported\n    head_dropout: float, default = 0.1\n        float indicating the dropout between the dense layers.\n    head_batchnorm: bool, default = False\n        Boolean indicating whether or not batch normalization will be applied\n        to the dense layers\n    head_batchnorm_last: bool, default = False\n        Boolean indicating whether or not batch normalization will be applied\n        to the last of the dense layers\n    head_linear_first: bool, default = False\n        Boolean indicating the order of the operations in the dense\n        layer. If `True: [LIN -&gt; ACT -&gt; BN -&gt; DP]`. If `False: [BN -&gt; DP -&gt;\n        LIN -&gt; ACT]`\n\n    Attributes\n    ----------\n    features: nn.Module\n        The pretrained model or Standard CNN plus the optional head\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.models import Vision\n    &gt;&gt;&gt; X_img = torch.rand((2,3,224,224))\n    &gt;&gt;&gt; model = Vision(channel_sizes=[64, 128], kernel_sizes = [3, 3], strides=[1, 1], head_hidden_dims=[32, 8])\n    &gt;&gt;&gt; out = model(X_img)\n    \"\"\"\n\n    @alias(\"pretrained_model_setup\", [\"pretrained_model_name\"])\n    def __init__(\n        self,\n        pretrained_model_setup: Union[str, Dict[str, Union[str, WeightsEnum]]] = None,\n        n_trainable: Optional[int] = None,\n        trainable_params: Optional[List[str]] = None,\n        channel_sizes: List[int] = [64, 128, 256, 512],\n        kernel_sizes: Union[int, List[int]] = [7, 3, 3, 3],\n        strides: Union[int, List[int]] = [2, 1, 1, 1],\n        head_hidden_dims: Optional[List[int]] = None,\n        head_activation: str = \"relu\",\n        head_dropout: Union[float, List[float]] = 0.1,\n        head_batchnorm: bool = False,\n        head_batchnorm_last: bool = False,\n        head_linear_first: bool = False,\n    ):\n        super(Vision, self).__init__()\n\n        self._check_pretrained_model_setup(\n            pretrained_model_setup, n_trainable, trainable_params\n        )\n\n        self.pretrained_model_setup = pretrained_model_setup\n        self.n_trainable = n_trainable\n        self.trainable_params = trainable_params\n        self.channel_sizes = channel_sizes\n        self.kernel_sizes = kernel_sizes\n        self.strides = strides\n        self.head_hidden_dims = head_hidden_dims\n        self.head_activation = head_activation\n        self.head_dropout = head_dropout\n        self.head_batchnorm = head_batchnorm\n        self.head_batchnorm_last = head_batchnorm_last\n        self.head_linear_first = head_linear_first\n\n        self.features, self.backbone_output_dim = self._get_features()\n\n        if pretrained_model_setup is not None:\n            self._freeze(self.features)\n\n        if self.head_hidden_dims is not None:\n            head_hidden_dims = [self.backbone_output_dim] + self.head_hidden_dims\n            self.vision_mlp = MLP(\n                head_hidden_dims,\n                self.head_activation,\n                self.head_dropout,\n                self.head_batchnorm,\n                self.head_batchnorm_last,\n                self.head_linear_first,\n            )\n\n    def forward(self, X: Tensor) -&gt; Tensor:\n        x = self.features(X)\n\n        if len(x.shape) &gt; 2:\n            if x.shape[2] &gt; 1:\n                x = nn.functional.adaptive_avg_pool2d(x, (1, 1))\n            x = torch.flatten(x, 1)\n\n        if self.head_hidden_dims is not None:\n            x = self.vision_mlp(x)\n\n        return x\n\n    @property\n    def output_dim(self) -&gt; int:\n        r\"\"\"The output dimension of the model. This is a required property\n        neccesary to build the `WideDeep` class\n        \"\"\"\n        return (\n            self.head_hidden_dims[-1]\n            if self.head_hidden_dims is not None\n            else self.backbone_output_dim\n        )\n\n    def _get_features(self) -&gt; Tuple[nn.Module, int]:\n        if self.pretrained_model_setup is not None:\n            if isinstance(self.pretrained_model_setup, str):\n                if self.pretrained_model_setup in allowed_pretrained_models.keys():\n                    model = allowed_pretrained_models[self.pretrained_model_setup]\n                    pretrained_model = torchvision.models.__dict__[model](\n                        weights=torchvision.models.get_model_weights(model).DEFAULT\n                    )\n                    warnings.warn(\n                        f\"{self.pretrained_model_setup} defaulting to {model}\",\n                        UserWarning,\n                    )\n                else:\n                    pretrained_model = torchvision.models.__dict__[\n                        self.pretrained_model_setup\n                    ](weights=\"IMAGENET1K_V1\")\n\n            elif isinstance(self.pretrained_model_setup, Dict):\n                model_name = next(iter(self.pretrained_model_setup))\n                model_weights = self.pretrained_model_setup[model_name]\n\n                if model_name in allowed_pretrained_models.keys():\n                    model_name = allowed_pretrained_models[model_name]\n\n                pretrained_model = torchvision.models.__dict__[model_name](\n                    weights=model_weights\n                )\n            output_dim: int = self.get_backbone_output_dim(pretrained_model)\n            features = nn.Sequential(*(list(pretrained_model.children())[:-1]))\n\n        else:\n            features = self._basic_cnn()\n            output_dim = self.channel_sizes[-1]\n\n        return features, output_dim\n\n    def _basic_cnn(self):\n        channel_sizes = [3] + self.channel_sizes\n        kernel_sizes = (\n            [self.kernel_sizes] * len(self.channel_sizes)\n            if isinstance(self.kernel_sizes, int)\n            else self.kernel_sizes\n        )\n        strides = (\n            [self.strides] * len(self.channel_sizes)\n            if isinstance(self.strides, int)\n            else self.strides\n        )\n\n        BasicCNN = nn.Sequential()\n        for i in range(1, len(channel_sizes)):\n            BasicCNN.add_module(\n                \"conv_layer_{}\".format(i - 1),\n                conv_layer(\n                    channel_sizes[i - 1],\n                    channel_sizes[i],\n                    kernel_sizes[i - 1],\n                    strides[i - 1],\n                    maxpool=i == 1,\n                    adaptiveavgpool=i == len(channel_sizes) - 1,\n                ),\n            )\n        return BasicCNN\n\n    def _freeze(self, features):\n        if self.trainable_params is not None:\n            for name, param in features.named_parameters():\n                for tl in self.trainable_params:\n                    param.requires_grad = tl in name\n        elif self.n_trainable is not None:\n            for i, (name, param) in enumerate(\n                reversed(list(features.named_parameters()))\n            ):\n                param.requires_grad = i &lt; self.n_trainable\n        else:\n            warnings.warn(\n                \"Both 'trainable_params' and 'n_trainable' are 'None' and the entire network will be trained\",\n                UserWarning,\n            )\n\n    @staticmethod\n    def get_backbone_output_dim(features):\n        try:\n            return features.fc.in_features\n        except AttributeError:\n            try:\n                features.classifier.__dict__[\"_modules\"][\"0\"].in_features\n            except AttributeError:\n                try:\n                    return features.classifier.__dict__[\"_modules\"][\"1\"].in_features\n                except AttributeError:\n                    return features.classifier.__dict__[\"_modules\"][\"1\"].in_channels\n\n    @staticmethod\n    def _check_pretrained_model_setup(\n        pretrained_model_setup, n_trainable, trainable_params\n    ):\n        if pretrained_model_setup is not None:\n            if isinstance(pretrained_model_setup, str):\n                pretrained_model_name = pretrained_model_setup\n            elif isinstance(pretrained_model_setup, Dict):\n                pretrained_model_name = list(pretrained_model_setup.keys())[0]\n        else:\n            pretrained_model_name = None\n\n        if pretrained_model_name is not None:\n            valid_pretrained_model_name = any(\n                [name in pretrained_model_name for name in allowed_pretrained_models]\n            )\n            if not valid_pretrained_model_name:\n                raise ValueError(\n                    f\"{pretrained_model_setup} is not among the allowed pretrained models.\"\n                    f\" These are {allowed_pretrained_models.keys()}. Please choose a variant of these architectures\"\n                )\n            if n_trainable is not None and trainable_params is not None:\n                raise UserWarning(\n                    \"Both 'n_trainable' and 'trainable_params' are not None. 'trainable_params' will be used\"\n                )\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.image.vision.Vision.output_dim","title":"output_dim  <code>property</code>","text":"<pre><code>output_dim\n</code></pre> <p>The output dimension of the model. This is a required property neccesary to build the <code>WideDeep</code> class</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.model_fusion.ModelFuser","title":"ModelFuser","text":"<p>               Bases: <code>BaseWDModelComponent</code></p> <p>This class is a wrapper around a list of models that are associated to the different text and/or image columns (and datasets) The class is designed to 'fuse' the models using a variety of methods.</p> <p>Parameters:</p> Name Type Description Default <code>models</code> <code>List[BaseWDModelComponent]</code> <p>List of models whose outputs will be fused</p> required <code>fusion_method</code> <code>Union[Literal[concatenate, mean, max, sum, mult, dot, head], List[Literal[concatenate, mean, max, sum, mult, head]]]</code> <p>Method to fuse the output of the models. It can be one of ['concatenate', 'mean', 'max', 'sum', 'mult', 'dot', 'head'] or a list of those, but 'dot'. If a list is provided the output of the models will be fused using all the methods in the list and the final output will be the concatenation of the outputs of each method</p> required <code>projection_method</code> <code>Optional[Literal[min, max, mean]]</code> <p>If the fusion_method is not 'concatenate', this parameter will determine how to project the output of the models to a common dimension. It can be one of ['min', 'max', 'mean']. Default is None</p> <code>None</code> <code>custom_head</code> <code>Optional[Union[BaseWDModelComponent, Module]]</code> <p>Custom head to be used to fuse the output of the models. If provided, this will take precedence over head_hidden_dims. Also, if provided, 'projection_method' will be ignored.</p> <code>None</code> <code>head_hidden_dims</code> <code>Optional[List[int]]</code> <p>List with the number of neurons per layer in the custom head. If custom_head is provided, this parameter will be ignored</p> <code>None</code> <code>head_activation</code> <code>Optional[str]</code> <p>Activation function to be used in the custom head. Default is None</p> <code>None</code> <code>head_dropout</code> <code>Optional[float]</code> <p>Dropout to be used in the custom head. Default is None</p> <code>None</code> <code>head_batchnorm</code> <code>Optional[bool]</code> <p>Whether to use batchnorm in the custom head. Default is None</p> <code>None</code> <code>head_batchnorm_last</code> <code>Optional[bool]</code> <p>Whether or not batch normalization will be applied to the last of the dense layers</p> <code>None</code> <code>head_linear_first</code> <code>Optional[bool]</code> <p>Boolean indicating the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code></p> <code>None</code> <p>Attributes:</p> Name Type Description <code>head</code> <code>Module or BaseWDModelComponent</code> <p>Custom head to be used to fuse the output of the models. If custom_head is provided, this will take precedence over head_hidden_dims</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; from pytorch_widedeep.preprocessing import TextPreprocessor\n&gt;&gt;&gt; from pytorch_widedeep.models import BasicRNN, ModelFuser\n&gt;&gt;&gt; import torch\n&gt;&gt;&gt; import pandas as pd\n&gt;&gt;&gt;\n&gt;&gt;&gt; df = pd.DataFrame({'text_col1': ['hello world', 'this is a test'],\n... 'text_col2': ['goodbye world', 'this is another test']})\n&gt;&gt;&gt; text_preprocessor_1 = TextPreprocessor(\n...     text_col=\"text_col1\",\n...     max_vocab=10,\n...     min_freq=1,\n...     maxlen=5,\n...     n_cpus=1,\n...     verbose=0)\n&gt;&gt;&gt; text_preprocessor_2 = TextPreprocessor(\n...     text_col=\"text_col2\",\n...     max_vocab=10,\n...     min_freq=1,\n...     maxlen=5,\n...     n_cpus=1,\n...     verbose=0)\n&gt;&gt;&gt; X_text1 = text_preprocessor_1.fit_transform(df)\n&gt;&gt;&gt; X_text2 = text_preprocessor_2.fit_transform(df)\n&gt;&gt;&gt; X_text1_tnsr = torch.from_numpy(X_text1)\n&gt;&gt;&gt; X_text2_tnsr = torch.from_numpy(X_text2)\n&gt;&gt;&gt; rnn1 = BasicRNN(\n...     vocab_size=len(text_preprocessor_1.vocab.itos),\n...     embed_dim=4,\n...     hidden_dim=4,\n...     n_layers=1,\n...     bidirectional=False)\n&gt;&gt;&gt; rnn2 = BasicRNN(\n...     vocab_size=len(text_preprocessor_2.vocab.itos),\n...     embed_dim=4,\n...     hidden_dim=4,\n...     n_layers=1,\n...     bidirectional=False)\n&gt;&gt;&gt; fused_model = ModelFuser(models=[rnn1, rnn2], fusion_method='concatenate')\n&gt;&gt;&gt; out = fused_model([X_text1_tnsr, X_text2_tnsr])\n</code></pre> Source code in <code>pytorch_widedeep/models/model_fusion.py</code> <pre><code>class ModelFuser(BaseWDModelComponent):\n    \"\"\"\n    This class is a wrapper around a list of models that are associated to the\n    different text and/or image columns (and datasets) The class is designed\n    to 'fuse' the models using a variety of methods.\n\n    Parameters\n    ----------\n    models: List[BaseWDModelComponent]\n        List of models whose outputs will be fused\n    fusion_method: Union[str, List[str]]\n        Method to fuse the output of the models. It can be one of\n        ['concatenate', 'mean', 'max', 'sum', 'mult', 'dot', 'head'] or a\n        list of those, but 'dot'. If a list is provided the output of the\n        models will be fused using all the methods in the list and the final\n        output will be the concatenation of the outputs of each method\n    projection_method: Optional[str]\n        If the fusion_method is not 'concatenate', this parameter will\n        determine how to project the output of the models to a common\n        dimension. It can be one of ['min', 'max', 'mean']. Default is None\n    custom_head: Optional[BaseWDModelComponent | nn.Module]\n        Custom head to be used to fuse the output of the models. If provided,\n        this will take precedence over head_hidden_dims. Also, if\n        provided, 'projection_method' will be ignored.\n    head_hidden_dims: Optional[List[int]]\n        List with the number of neurons per layer in the custom head. If\n        custom_head is provided, this parameter will be ignored\n    head_activation: Optional[str]\n        Activation function to be used in the custom head. Default is None\n    head_dropout: Optional[float]\n        Dropout to be used in the custom head. Default is None\n    head_batchnorm: Optional[bool]\n        Whether to use batchnorm in the custom head. Default is None\n    head_batchnorm_last: Optional[bool]\n        Whether or not batch normalization will be applied to the last of the\n        dense layers\n    head_linear_first: Optional[bool]\n        Boolean indicating the order of the operations in the dense\n        layer. If `True: [LIN -&gt; ACT -&gt; BN -&gt; DP]`. If `False: [BN -&gt; DP -&gt;\n        LIN -&gt; ACT]`\n\n    Attributes\n    ----------\n    head: nn.Module or BaseWDModelComponent\n        Custom head to be used to fuse the output of the models. If\n        custom_head is provided, this will take precedence over\n        head_hidden_dims\n\n    Examples\n    --------\n    &gt;&gt;&gt; from pytorch_widedeep.preprocessing import TextPreprocessor\n    &gt;&gt;&gt; from pytorch_widedeep.models import BasicRNN, ModelFuser\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; import pandas as pd\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; df = pd.DataFrame({'text_col1': ['hello world', 'this is a test'],\n    ... 'text_col2': ['goodbye world', 'this is another test']})\n    &gt;&gt;&gt; text_preprocessor_1 = TextPreprocessor(\n    ...     text_col=\"text_col1\",\n    ...     max_vocab=10,\n    ...     min_freq=1,\n    ...     maxlen=5,\n    ...     n_cpus=1,\n    ...     verbose=0)\n    &gt;&gt;&gt; text_preprocessor_2 = TextPreprocessor(\n    ...     text_col=\"text_col2\",\n    ...     max_vocab=10,\n    ...     min_freq=1,\n    ...     maxlen=5,\n    ...     n_cpus=1,\n    ...     verbose=0)\n    &gt;&gt;&gt; X_text1 = text_preprocessor_1.fit_transform(df)\n    &gt;&gt;&gt; X_text2 = text_preprocessor_2.fit_transform(df)\n    &gt;&gt;&gt; X_text1_tnsr = torch.from_numpy(X_text1)\n    &gt;&gt;&gt; X_text2_tnsr = torch.from_numpy(X_text2)\n    &gt;&gt;&gt; rnn1 = BasicRNN(\n    ...     vocab_size=len(text_preprocessor_1.vocab.itos),\n    ...     embed_dim=4,\n    ...     hidden_dim=4,\n    ...     n_layers=1,\n    ...     bidirectional=False)\n    &gt;&gt;&gt; rnn2 = BasicRNN(\n    ...     vocab_size=len(text_preprocessor_2.vocab.itos),\n    ...     embed_dim=4,\n    ...     hidden_dim=4,\n    ...     n_layers=1,\n    ...     bidirectional=False)\n    &gt;&gt;&gt; fused_model = ModelFuser(models=[rnn1, rnn2], fusion_method='concatenate')\n    &gt;&gt;&gt; out = fused_model([X_text1_tnsr, X_text2_tnsr])\n    \"\"\"\n\n    def __init__(\n        self,\n        models: List[BaseWDModelComponent],\n        *,\n        fusion_method: Union[\n            Literal[\n                \"concatenate\",\n                \"mean\",\n                \"max\",\n                \"sum\",\n                \"mult\",\n                \"dot\",\n                \"head\",\n            ],\n            List[Literal[\"concatenate\", \"mean\", \"max\", \"sum\", \"mult\", \"head\"]],\n        ],\n        projection_method: Optional[Literal[\"min\", \"max\", \"mean\"]] = None,\n        custom_head: Optional[Union[BaseWDModelComponent, nn.Module]] = None,\n        head_hidden_dims: Optional[List[int]] = None,\n        head_activation: Optional[str] = None,\n        head_dropout: Optional[float] = None,\n        head_batchnorm: Optional[bool] = None,\n        head_batchnorm_last: Optional[bool] = None,\n        head_linear_first: Optional[bool] = None,\n    ) -&gt; None:\n        super(ModelFuser, self).__init__()\n\n        self.models = nn.ModuleList(models)\n        self.fusion_method = fusion_method\n        self.projection_method = projection_method\n\n        self.all_output_dim_equal = all(\n            model.output_dim == self.models[0].output_dim for model in self.models\n        )\n\n        self.check_input_parameters()\n\n        if self.fusion_method == \"head\":\n            assert (\n                head_hidden_dims is not None or custom_head is not None\n            ), \"When using 'head' as fusion_method, either head_hidden_dims or custom_head must be provided\"\n            if custom_head is not None:\n                # custom_head takes precedence over head_hidden_dims (in case\n                # both are provided)\n                assert hasattr(\n                    custom_head, \"output_dim\"\n                ), \"custom_head must have an 'output_dim' property\"\n                self.head: Union[BaseWDModelComponent, nn.Module] = custom_head\n            else:\n                assert head_hidden_dims is not None\n                self.head_hidden_dims = head_hidden_dims\n                self.head_activation = head_activation\n                self.head_dropout = head_dropout\n                self.head_batchnorm = head_batchnorm\n                self.head_batchnorm_last = head_batchnorm_last\n                self.head_linear_first = head_linear_first\n\n                self.head = MLP(\n                    d_hidden=[sum([model.output_dim for model in self.models])]\n                    + self.head_hidden_dims,\n                    activation=(\n                        \"relu\" if self.head_activation is None else self.head_activation\n                    ),\n                    dropout=0.0 if self.head_dropout is None else self.head_dropout,\n                    batchnorm=(\n                        False if self.head_batchnorm is None else self.head_batchnorm\n                    ),\n                    batchnorm_last=(\n                        False\n                        if self.head_batchnorm_last is None\n                        else self.head_batchnorm_last\n                    ),\n                    linear_first=(\n                        True\n                        if self.head_linear_first is None\n                        else self.head_linear_first\n                    ),\n                )\n\n    def forward(self, X: List[Tensor]) -&gt; Tensor:  # noqa: C901\n        if self.fusion_method == \"head\":\n            return self._head_fusion(X)\n        elif self.fusion_method == \"dot\":\n            return self._dot_fusion(X)\n        else:\n            return self._other_fusions(X)\n\n    def _head_fusion(self, X: List[Tensor]) -&gt; Tensor:\n        return self.head(torch.cat([model(x) for model, x in zip(self.models, X)], -1))\n\n    def _dot_fusion(self, X: List[Tensor]) -&gt; Tensor:\n        assert (\n            len(X) == 2\n        ), \"When using 'dot' as fusion_method, only two models can be fused\"\n        outputs = [model(x) for model, x in zip(self.models, X)]\n        return torch.bmm(outputs[1].unsqueeze(1), outputs[0].unsqueeze(2)).view(-1, 1)\n\n    def _other_fusions(self, X: List[Tensor]) -&gt; Tensor:\n        fusion_methods = (\n            [self.fusion_method]\n            if isinstance(self.fusion_method, str)\n            else self.fusion_method\n        )\n        fused_outputs = [self._apply_fusion_method(fm, X) for fm in fusion_methods]  # type: ignore[attr-defined]\n        return (\n            fused_outputs[0] if len(fused_outputs) == 1 else torch.cat(fused_outputs, 1)\n        )\n\n    def _apply_fusion_method(self, fusion_method: str, X: List[Tensor]) -&gt; Tensor:\n        if fusion_method == \"concatenate\":\n            return torch.cat([model(x) for model, x in zip(self.models, X)], 1)\n\n        model_outputs = [model(x) for model, x in zip(self.models, X)]\n        projections = self._project(model_outputs)\n        stacked = torch.stack(projections, -1)\n\n        if fusion_method == \"mean\":\n            return torch.mean(stacked, -1)\n        elif fusion_method == \"max\":\n            return torch.max(stacked, -1)[0]\n        elif fusion_method == \"min\":\n            return torch.min(stacked, -1)[0]\n        elif fusion_method == \"sum\":\n            return torch.sum(stacked, -1)\n        elif fusion_method == \"mult\":\n            return torch.prod(stacked, -1)\n        else:\n            raise ValueError(f\"Unsupported fusion method: {fusion_method}\")\n\n    def _project(self, X: List[Tensor]) -&gt; List[Tensor]:\n        r\"\"\"Projects the output of the models to a common dimension.\"\"\"\n\n        if self.all_output_dim_equal and self.projection_method is None:\n            return X\n\n        output_dims = [model.output_dim for model in self.models]\n\n        if self.projection_method == \"min\":\n            proj_dim = min(output_dims)\n            idx = output_dims.index(proj_dim)\n        elif self.projection_method == \"max\":\n            proj_dim = max(output_dims)\n            idx = output_dims.index(proj_dim)\n        elif self.projection_method == \"mean\":\n            proj_dim = int(sum(output_dims) / len(output_dims))\n            idx = None\n        else:\n            raise ValueError(\"projection_method must be one of ['min', 'max', 'mean']\")\n\n        x_proj: List[Tensor] = []\n        for i, x in enumerate(X):\n            if i == idx:\n                x_proj.append(x)\n            else:\n                x_proj.append(\n                    nn.Linear(output_dims[i], proj_dim, bias=False, device=x.device)(x)\n                )\n\n        return x_proj\n\n    @property\n    def output_dim(self) -&gt; int:\n        r\"\"\"Returns the output dimension of the model.\"\"\"\n        if self.fusion_method == \"head\":\n            output_dim = (\n                self.head_hidden_dims[-1]\n                if hasattr(self, \"head_hidden_dims\")\n                else self.head.output_dim\n            )\n        elif self.fusion_method == \"dot\":\n            output_dim = 1\n        else:\n            output_dim = 0\n            if isinstance(self.fusion_method, str):\n                fusion_methods = [self.fusion_method]\n            else:\n                fusion_methods = self.fusion_method  # type: ignore\n            for fm in fusion_methods:\n                if fm == \"concatenate\":\n                    output_dim += sum([model.output_dim for model in self.models])\n                elif self.projection_method == \"mean\":\n                    output_dim += int(\n                        sum([model.output_dim for model in self.models])\n                        / len(self.models)\n                    )\n                elif self.projection_method == \"min\":\n                    output_dim += min([model.output_dim for model in self.models])\n                elif self.projection_method == \"max\":\n                    output_dim += max([model.output_dim for model in self.models])\n                elif self.all_output_dim_equal:\n                    output_dim += self.models[0].output_dim\n                else:\n                    raise ValueError(\n                        \"projection_method must be one of ['min', 'max', 'mean']\"\n                    )\n\n        return output_dim\n\n    def check_input_parameters(self):\n        self._validate_tabnet()\n        self._validate_fusion_methods()\n        self._validate_projection_requirements()\n        self._validate_head_dot_exclusivity()\n\n    def _validate_tabnet(self):\n        if any(isinstance(model, TabNet) for model in self.models):\n            raise ValueError(\n                \"TabNet is not supported in ModelFuser. \"\n                \"Please, use another model for tabular data\"\n            )\n\n    def _validate_fusion_methods(self):\n        valid_methods = [\n            \"concatenate\",\n            \"min\",\n            \"max\",\n            \"mean\",\n            \"sum\",\n            \"mult\",\n            \"dot\",\n            \"head\",\n        ]\n\n        if isinstance(self.fusion_method, str):\n            if self.fusion_method not in valid_methods:\n                raise ValueError(\n                    \"fusion_method must be one of ['concatenate', 'mean', 'max', 'sum', 'mult', 'dot', 'head'] \"\n                    \"or a list of any those but 'dot'\"\n                )\n        else:\n            if not all(fm in valid_methods for fm in self.fusion_method):\n                raise ValueError(\n                    \"fusion_method must be one of ['concatenate', 'mean', 'max', 'sum', 'mult', 'dot', 'head'] \"\n                    \"or a list of those but 'dot'\"\n                )\n\n    def _validate_projection_requirements(self):\n        needs_projection = not self.all_output_dim_equal\n        has_size_dependent_method = False\n\n        if isinstance(self.fusion_method, str):\n            has_size_dependent_method = self.fusion_method in [\"min\", \"max\", \"mean\"]\n        else:\n            has_size_dependent_method = all(\n                fm in [\"min\", \"max\", \"mean\"] for fm in self.fusion_method\n            )\n\n        if has_size_dependent_method and needs_projection:\n            if self.projection_method is None:\n                raise ValueError(\n                    \"If 'fusion_method' is not 'concatenate' or 'head', \"\n                    \"and the output dimensions of the models are not equal, \"\n                    \"'projection_method' must be provided\"\n                )\n            elif self.projection_method not in [\"min\", \"max\", \"mean\"]:\n                raise ValueError(\n                    \"projection_method must be one of ['min', 'max', 'mean']\"\n                )\n\n    def _validate_head_dot_exclusivity(self):\n        if isinstance(self.fusion_method, list):\n            if any(method in [\"head\", \"dot\"] for method in self.fusion_method):\n                raise ValueError(\n                    \"When using 'head' or 'dot' as fusion_method, no other method should be provided\"\n                )\n\n    def __repr__(self):\n        if self.projection_method is not None:\n            proj = f\"{self.projection_method}\"\n        else:\n            proj = \"\"\n\n        return f\"Fusion method: {self.fusion_method}. Projection method: {proj}\\nFused Models:\\n{self.models}\"\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.model_fusion.ModelFuser.output_dim","title":"output_dim  <code>property</code>","text":"<pre><code>output_dim\n</code></pre> <p>Returns the output dimension of the model.</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.wide_deep.WideDeep","title":"WideDeep","text":"<p>               Bases: <code>Module</code></p> <p>Main collector class that combines all <code>wide</code>, <code>deeptabular</code> <code>deeptext</code> and <code>deepimage</code> models.</p> <p>Note that all models described so far in this library must be passed to the <code>WideDeep</code> class once constructed. This is because the models output the last layer before the prediction layer. Such prediction layer is added by the <code>WideDeep</code> class as it collects the components for every data mode.</p> <p>There are two options to combine these models that correspond to the two main architectures that <code>pytorch-widedeep</code> can build.</p> <ul> <li> <p>Directly connecting the output of the model components to an ouput neuron(s).</p> </li> <li> <p>Adding a <code>Fully-Connected Head</code> (FC-Head) on top of the deep models.   This FC-Head will combine the output form the <code>deeptabular</code>, <code>deeptext</code> and   <code>deepimage</code> and will be then connected to the output neuron(s).</p> </li> </ul> <p>Parameters:</p> Name Type Description Default <code>wide</code> <code>Optional[Module]</code> <p><code>Wide</code> model. This is a linear model where the non-linearities are captured via crossed-columns.</p> <code>None</code> <code>deeptabular</code> <code>Optional[Union[BaseWDModelComponent, List[BaseWDModelComponent]]]</code> <p>Currently this library implements a number of possible architectures for the <code>deeptabular</code> component. See the documenation of the package. Note that <code>deeptabular</code> can be a list of models. This is useful when using multiple tabular inputs (e.g. for example in the context of a two-tower model for recommendation systems)</p> <code>None</code> <code>deeptext</code> <code>Optional[Union[BaseWDModelComponent, List[BaseWDModelComponent]]]</code> <p>Currently this library implements a number of possible architectures for the <code>deeptext</code> component. See the documenation of the package. Note that <code>deeptext</code> can be a list of models. This is useful when using multiple text inputs.</p> <code>None</code> <code>deepimage</code> <code>Optional[Union[BaseWDModelComponent, List[BaseWDModelComponent]]]</code> <p>Currently this library uses <code>torchvision</code> and implements a number of possible architectures for the <code>deepimage</code> component. See the documenation of the package. Note that <code>deepimage</code> can be a list of models. This is useful when using multiple image inputs.</p> <code>None</code> <code>deephead</code> <code>Optional[BaseWDModelComponent]</code> <p>Alternatively, the user can pass a custom model that will receive the output of the deep component. If <code>deephead</code> is not None all the previous fc-head parameters will be ignored</p> <code>None</code> <code>head_hidden_dims</code> <code>Optional[List[int]]</code> <p>List with the sizes of the dense layers in the head e.g: [128, 64]</p> <code>None</code> <code>head_activation</code> <code>str</code> <p>Activation function for the dense layers in the head. Currently <code>'tanh'</code>, <code>'relu'</code>, <code>'leaky_relu'</code> and <code>'gelu'</code> are supported</p> <code>'relu'</code> <code>head_dropout</code> <code>float</code> <p>Dropout of the dense layers in the head</p> <code>0.1</code> <code>head_batchnorm</code> <code>bool</code> <p>Boolean indicating whether or not to include batch normalization in the dense layers that form the <code>'rnn_mlp'</code></p> <code>False</code> <code>head_batchnorm_last</code> <code>bool</code> <p>Boolean indicating whether or not to apply batch normalization to the last of the dense layers in the head</p> <code>False</code> <code>head_linear_first</code> <code>bool</code> <p>Boolean indicating whether the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code></p> <code>True</code> <code>enforce_positive</code> <code>bool</code> <p>Boolean indicating if the output from the final layer must be positive. This is important if you are using loss functions with non-negative input restrictions, e.g. RMSLE, or if you know your predictions are bounded in between 0 and inf</p> <code>False</code> <code>enforce_positive_activation</code> <code>str</code> <p>Activation function to enforce that the final layer has a positive output. <code>'softplus'</code> or <code>'relu'</code> are supported.</p> <code>'softplus'</code> <code>pred_dim</code> <code>int</code> <p>Size of the final wide and deep output layer containing the predictions. <code>1</code> for regression and binary classification or number of classes for multiclass classification.</p> <code>1</code> <p>Examples:</p> <pre><code>&gt;&gt;&gt; from pytorch_widedeep.models import TabResnet, Vision, BasicRNN, Wide, WideDeep\n&gt;&gt;&gt; embed_input = [(u, i, j) for u, i, j in zip([\"a\", \"b\", \"c\"][:4], [4] * 3, [8] * 3)]\n&gt;&gt;&gt; column_idx = {k: v for v, k in enumerate([\"a\", \"b\", \"c\"])}\n&gt;&gt;&gt; wide = Wide(10, 1)\n&gt;&gt;&gt; deeptabular = TabResnet(blocks_dims=[8, 4], column_idx=column_idx, cat_embed_input=embed_input)\n&gt;&gt;&gt; deeptext = BasicRNN(vocab_size=10, embed_dim=4, padding_idx=0)\n&gt;&gt;&gt; deepimage = Vision()\n&gt;&gt;&gt; model = WideDeep(wide=wide, deeptabular=deeptabular, deeptext=deeptext, deepimage=deepimage)\n</code></pre> <p> NOTE: It is possible to use custom components to  build Wide &amp; Deep models. Simply, build them and pass them as the  corresponding parameters. Note that the custom models MUST return a last  layer of activations(i.e. not the final prediction) so that  these  activations are collected by <code>WideDeep</code> and combined accordingly. In  addition, the models MUST also contain an attribute <code>output_dim</code> with  the size of these last layers of activations. See for example  <code>pytorch_widedeep.models.tab_mlp.TabMlp</code></p> Source code in <code>pytorch_widedeep/models/wide_deep.py</code> <pre><code>class WideDeep(nn.Module):\n    r\"\"\"Main collector class that combines all `wide`, `deeptabular`\n    `deeptext` and `deepimage` models.\n\n    Note that all models described so far in this library must be passed to\n    the `WideDeep` class once constructed. This is because the models output\n    the last layer before the prediction layer. Such prediction layer is\n    added by the `WideDeep` class as it collects the components for every\n    data mode.\n\n    There are two options to combine these models that correspond to the\n    two main architectures that `pytorch-widedeep` can build.\n\n    - Directly connecting the output of the model components to an ouput neuron(s).\n\n    - Adding a `Fully-Connected Head` (FC-Head) on top of the deep models.\n      This FC-Head will combine the output form the `deeptabular`, `deeptext` and\n      `deepimage` and will be then connected to the output neuron(s).\n\n    Parameters\n    ----------\n    wide: nn.Module, Optional, default = None\n        `Wide` model. This is a linear model where the non-linearities are\n        captured via crossed-columns.\n    deeptabular: BaseWDModelComponent, Optional, default = None\n        Currently this library implements a number of possible architectures\n        for the `deeptabular` component. See the documenation of the\n        package. Note that `deeptabular` can be a list of models. This is\n        useful when using multiple tabular inputs (e.g. for example in the\n        context of a two-tower model for recommendation systems)\n    deeptext: BaseWDModelComponent | List[BaseWDModelComponent], Optional, default = None\n        Currently this library implements a number of possible architectures\n        for the `deeptext` component. See the documenation of the\n        package. Note that `deeptext` can be a list of models. This is useful\n        when using multiple text inputs.\n    deepimage: BaseWDModelComponent | List[BaseWDModelComponent], Optional, default = None\n        Currently this library uses `torchvision` and implements a number of\n        possible architectures for the `deepimage` component. See the\n        documenation of the package. Note that `deepimage` can be a list of\n        models. This is useful when using multiple image inputs.\n    deephead: BaseWDModelComponent, Optional, default = None\n        Alternatively, the user can pass a custom model that will receive the\n        output of the deep component. If `deephead` is not None all the\n        previous fc-head parameters will be ignored\n    head_hidden_dims: List, Optional, default = None\n        List with the sizes of the dense layers in the head e.g: [128, 64]\n    head_activation: str, default = \"relu\"\n        Activation function for the dense layers in the head. Currently\n        `'tanh'`, `'relu'`, `'leaky_relu'` and `'gelu'` are supported\n    head_dropout: float, Optional, default = None\n        Dropout of the dense layers in the head\n    head_batchnorm: bool, default = False\n        Boolean indicating whether or not to include batch normalization in\n        the dense layers that form the `'rnn_mlp'`\n    head_batchnorm_last: bool, default = False\n        Boolean indicating whether or not to apply batch normalization to the\n        last of the dense layers in the head\n    head_linear_first: bool, default = False\n        Boolean indicating whether the order of the operations in the dense\n        layer. If `True: [LIN -&gt; ACT -&gt; BN -&gt; DP]`. If `False: [BN -&gt; DP -&gt;\n        LIN -&gt; ACT]`\n    enforce_positive: bool, default = False\n        Boolean indicating if the output from the final layer must be\n        positive. This is important if you are using loss functions with\n        non-negative input restrictions, e.g. RMSLE, or if you know your\n        predictions are bounded in between 0 and inf\n    enforce_positive_activation: str, default = \"softplus\"\n        Activation function to enforce that the final layer has a positive\n        output. `'softplus'` or `'relu'` are supported.\n    pred_dim: int, default = 1\n        Size of the final wide and deep output layer containing the\n        predictions. `1` for regression and binary classification or number\n        of classes for multiclass classification.\n\n\n    Examples\n    --------\n\n    &gt;&gt;&gt; from pytorch_widedeep.models import TabResnet, Vision, BasicRNN, Wide, WideDeep\n    &gt;&gt;&gt; embed_input = [(u, i, j) for u, i, j in zip([\"a\", \"b\", \"c\"][:4], [4] * 3, [8] * 3)]\n    &gt;&gt;&gt; column_idx = {k: v for v, k in enumerate([\"a\", \"b\", \"c\"])}\n    &gt;&gt;&gt; wide = Wide(10, 1)\n    &gt;&gt;&gt; deeptabular = TabResnet(blocks_dims=[8, 4], column_idx=column_idx, cat_embed_input=embed_input)\n    &gt;&gt;&gt; deeptext = BasicRNN(vocab_size=10, embed_dim=4, padding_idx=0)\n    &gt;&gt;&gt; deepimage = Vision()\n    &gt;&gt;&gt; model = WideDeep(wide=wide, deeptabular=deeptabular, deeptext=deeptext, deepimage=deepimage)\n\n\n    :information_source: **NOTE**: It is possible to use custom components to\n     build Wide &amp; Deep models. Simply, build them and pass them as the\n     corresponding parameters. Note that the custom models MUST return a last\n     layer of activations(i.e. not the final prediction) so that  these\n     activations are collected by `WideDeep` and combined accordingly. In\n     addition, the models MUST also contain an attribute `output_dim` with\n     the size of these last layers of activations. See for example\n     `pytorch_widedeep.models.tab_mlp.TabMlp`\n    \"\"\"\n\n    @alias(  # noqa: C901\n        \"pred_dim\",\n        [\"num_class\", \"pred_size\"],\n    )\n    def __init__(\n        self,\n        wide: Optional[nn.Module] = None,\n        deeptabular: Optional[\n            Union[BaseWDModelComponent, List[BaseWDModelComponent]]\n        ] = None,\n        deeptext: Optional[\n            Union[BaseWDModelComponent, List[BaseWDModelComponent]]\n        ] = None,\n        deepimage: Optional[\n            Union[BaseWDModelComponent, List[BaseWDModelComponent]]\n        ] = None,\n        deephead: Optional[BaseWDModelComponent] = None,\n        head_hidden_dims: Optional[List[int]] = None,\n        head_activation: str = \"relu\",\n        head_dropout: float = 0.1,\n        head_batchnorm: bool = False,\n        head_batchnorm_last: bool = False,\n        head_linear_first: bool = True,\n        enforce_positive: bool = False,\n        enforce_positive_activation: str = \"softplus\",\n        pred_dim: int = 1,\n    ):\n        super(WideDeep, self).__init__()\n\n        self._check_inputs(\n            wide,\n            deeptabular,\n            deeptext,\n            deepimage,\n            deephead,\n            head_hidden_dims,\n            pred_dim,\n        )\n\n        # this attribute will be eventually over-written by the Trainer's\n        # device. Acts here as a 'placeholder'.\n        self.wd_device: Optional[str] = None\n\n        # required as attribute just in case we pass a deephead\n        self.pred_dim = pred_dim\n\n        self.enforce_positive = enforce_positive\n\n        # better to set this attribute already here\n        if isinstance(deeptabular, list):\n            self.is_tabnet = False\n        else:\n            self.is_tabnet = deeptabular.__class__.__name__ == \"TabNet\"\n\n        # The main 5 components of the wide and deep assemble: wide,\n        # deeptabular, deeptext, deepimage and deephead\n        self.with_deephead = deephead is not None or head_hidden_dims is not None\n        if deephead is None and head_hidden_dims is not None:\n            self.deephead = self._build_deephead(\n                deeptabular,\n                deeptext,\n                deepimage,\n                head_hidden_dims,\n                head_activation,\n                head_dropout,\n                head_batchnorm,\n                head_batchnorm_last,\n                head_linear_first,\n            )\n        elif deephead is not None:\n            self.deephead = nn.Sequential(\n                deephead, nn.Linear(deephead.output_dim, self.pred_dim)\n            )\n        else:\n            # for consistency with other components we default to None\n            self.deephead = None\n\n        self.wide = wide\n        self.deeptabular, self.deeptext, self.deepimage = self._set_model_components(\n            deeptabular, deeptext, deepimage\n        )\n\n        if self.enforce_positive:\n            self.enf_pos = get_activation_fn(enforce_positive_activation)\n\n    def forward(\n        self,\n        X: Dict[str, Union[Tensor, List[Tensor]]],\n        y: Optional[Tensor] = None,\n    ) -&gt; Union[Tensor, Tuple[Tensor, Tensor]]:\n\n        wide_out = self._forward_wide(X)\n        if self.with_deephead:\n            deep = self._forward_deephead(X, wide_out)\n        else:\n            deep = self._forward_deep(X, wide_out)\n\n        if self.enforce_positive:\n            return self.enf_pos(deep)\n        else:\n            return deep\n\n    def _build_deephead(\n        self,\n        deeptabular: Optional[Union[BaseWDModelComponent, List[BaseWDModelComponent]]],\n        deeptext: Optional[Union[BaseWDModelComponent, List[BaseWDModelComponent]]],\n        deepimage: Optional[Union[BaseWDModelComponent, List[BaseWDModelComponent]]],\n        head_hidden_dims: List[int],\n        head_activation: str,\n        head_dropout: float,\n        head_batchnorm: bool,\n        head_batchnorm_last: bool,\n        head_linear_first: bool,\n    ) -&gt; nn.Sequential:\n        deep_dim = 0\n        if deeptabular is not None:\n            if isinstance(deeptabular, list):\n                for dt in deeptabular:\n                    deep_dim += dt.output_dim\n            else:\n                deep_dim += deeptabular.output_dim\n        if deeptext is not None:\n            if isinstance(deeptext, list):\n                for dt in deeptext:\n                    deep_dim += dt.output_dim\n            else:\n                deep_dim += deeptext.output_dim\n        if deepimage is not None:\n            if isinstance(deepimage, list):\n                for di in deepimage:\n                    deep_dim += di.output_dim\n            else:\n                deep_dim += deepimage.output_dim\n\n        head_hidden_dims = [deep_dim] + head_hidden_dims\n        deephead = nn.Sequential(\n            MLP(\n                head_hidden_dims,\n                head_activation,\n                head_dropout,\n                head_batchnorm,\n                head_batchnorm_last,\n                head_linear_first,\n            ),\n            nn.Linear(head_hidden_dims[-1], self.pred_dim),\n        )\n\n        return deephead\n\n    def _set_model_components(  # noqa: C901\n        self,\n        deeptabular: Optional[Union[BaseWDModelComponent, List[BaseWDModelComponent]]],\n        deeptext: Optional[Union[BaseWDModelComponent, List[BaseWDModelComponent]]],\n        deepimage: Optional[Union[BaseWDModelComponent, List[BaseWDModelComponent]]],\n    ) -&gt; Tuple[\n        Optional[Union[nn.ModuleList, WDModel]],\n        Optional[Union[nn.ModuleList, WDModel]],\n        Optional[Union[nn.ModuleList, WDModel]],\n    ]:\n        if deeptabular is not None:\n            deeptabular_ = self._set_model_component(deeptabular, is_deeptabular=True)\n        else:\n            deeptabular_ = None\n\n        if deeptext is not None:\n            deeptext_ = self._set_model_component(deeptext)\n        else:\n            deeptext_ = None\n\n        if deepimage is not None:\n            deepimage_ = self._set_model_component(deepimage)\n        else:\n            deepimage_ = None\n\n        return deeptabular_, deeptext_, deepimage_\n\n    def _forward_wide(self, X: Dict[str, Union[Tensor, List[Tensor]]]) -&gt; Tensor:\n        if self.wide is not None:\n            out = self.wide(X[\"wide\"])\n        else:\n            first_model_mode = list(X.keys())[0]\n            if isinstance(X[first_model_mode], list):\n                batch_size = X[first_model_mode][0].size(0)\n            else:\n                batch_size = X[first_model_mode].size(0)  # type: ignore[union-attr]\n            out = torch.zeros(batch_size, self.pred_dim).to(self.wd_device)\n\n        return out\n\n    def _forward_deep(\n        self, X: Dict[str, Union[Tensor, List[Tensor]]], wide_out: Tensor\n    ) -&gt; Union[Tensor, Tuple[Tensor, Tensor]]:\n        if self.deeptabular is not None:\n            if self.is_tabnet:\n                tab_out, M_loss = self.deeptabular(X[\"deeptabular\"])\n                wide_out.add_(tab_out)\n            else:\n                wide_out = self._forward_component(\n                    X, self.deeptabular, \"deeptabular\", wide_out\n                )\n\n        if self.deeptext is not None:\n            wide_out = self._forward_component(X, self.deeptext, \"deeptext\", wide_out)\n\n        if self.deepimage is not None:\n            wide_out = self._forward_component(X, self.deepimage, \"deepimage\", wide_out)\n\n        if self.is_tabnet:\n            res: Union[Tensor, Tuple[Tensor, Tensor]] = (wide_out, M_loss)\n        else:\n            res = wide_out\n\n        return res\n\n    def _forward_deephead(\n        self, X: Dict[str, Union[Tensor, List[Tensor]]], wide_out: Tensor\n    ) -&gt; Union[Tensor, Tuple[Tensor, Tensor]]:\n        deepside = torch.FloatTensor().to(self.wd_device)\n\n        if self.deeptabular is not None:\n            if self.is_tabnet:\n                deepside, M_loss = self.deeptabular(X[\"deeptabular\"])\n            else:\n                deepside = self._forward_component_with_head(\n                    X, self.deeptabular, \"deeptabular\", deepside\n                )\n\n        if self.deeptext is not None:\n            deepside = self._forward_component_with_head(\n                X, self.deeptext, \"deeptext\", deepside\n            )\n\n        if self.deepimage is not None:\n            deepside = self._forward_component_with_head(\n                X, self.deepimage, \"deepimage\", deepside\n            )\n\n        # assertion to avoid type issues\n        assert self.deephead is not None\n        deepside_out = self.deephead(deepside)\n\n        if self.is_tabnet:\n            res: Union[Tensor, Tuple[Tensor, Tensor]] = (\n                wide_out.add_(deepside_out),\n                M_loss,\n            )\n        else:\n            res = wide_out.add_(deepside_out)\n\n        return res\n\n    def _forward_component(\n        self,\n        X: Dict[str, Union[Tensor, List[Tensor]]],\n        component: Union[nn.ModuleList, WDModel],\n        component_type: Literal[\"deeptabular\", \"deeptext\", \"deepimage\"],\n        wide_out: Tensor,\n    ) -&gt; Tensor:\n        if isinstance(component, nn.ModuleList):\n            component_out = torch.add(  # type: ignore[call-overload]\n                *[cp(X[component_type][i]) for i, cp in enumerate(component)]\n            )\n        else:\n            component_out = component(X[component_type])\n\n        return wide_out.add_(component_out)\n\n    def _forward_component_with_head(\n        self,\n        X: Dict[str, Union[Tensor, List[Tensor]]],\n        component: Union[nn.ModuleList, WDModel],\n        component_type: Literal[\"deeptabular\", \"deeptext\", \"deepimage\"],\n        deepside: Tensor,\n    ) -&gt; Tensor:\n        if isinstance(component, nn.ModuleList):\n            component_out = torch.cat(  # type: ignore[call-overload]\n                [cp(X[component_type][i]) for i, cp in enumerate(component)], axis=1\n            )\n        else:\n            component_out = component(X[component_type])\n\n        return torch.cat([deepside, component_out], axis=1)  # type: ignore[call-overload]\n\n    def _set_model_component(\n        self,\n        component: Union[BaseWDModelComponent, List[BaseWDModelComponent]],\n        is_deeptabular: bool = False,\n    ) -&gt; Union[nn.ModuleList, WDModel]:\n        if isinstance(component, list):\n            component_: Optional[Union[nn.ModuleList, WDModel]] = nn.ModuleList()\n            for cp in component:\n                if self.with_deephead or cp.output_dim == 1:\n                    component_.append(cp)\n                else:\n                    component_.append(\n                        nn.Sequential(cp, nn.Linear(cp.output_dim, self.pred_dim))\n                    )\n        elif self.with_deephead or component.output_dim == 1:\n            component_ = component\n        elif is_deeptabular and self.is_tabnet:\n            component_ = nn.Sequential(\n                component, TabNetPredLayer(component.output_dim, self.pred_dim)\n            )\n        else:\n            component_ = nn.Sequential(\n                component, nn.Linear(component.output_dim, self.pred_dim)\n            )\n\n        return component_\n\n    @staticmethod  # noqa: C901\n    def _check_inputs(  # noqa: C901\n        wide,\n        deeptabular,\n        deeptext,\n        deepimage,\n        deephead,\n        head_hidden_dims,\n        pred_dim,\n    ):\n        if wide is not None:\n            assert wide.wide_linear.weight.size(1) == pred_dim, (\n                \"the 'pred_dim' of the wide component ({}) must be equal to the 'pred_dim' \"\n                \"of the deep component and the overall model itself ({})\".format(\n                    wide.wide_linear.weight.size(1), pred_dim\n                )\n            )\n\n        if deeptabular is not None:\n            err_msg = (\n                \"deeptabular model must have an 'output_dim' attribute or property.\"\n            )\n            if isinstance(deeptabular, list):\n                all_have_output_dim = all(\n                    hasattr(dt, \"output_dim\") for dt in deeptabular\n                )\n                if not all_have_output_dim:\n                    raise AttributeError(err_msg)\n            else:\n                if not hasattr(deeptabular, \"output_dim\"):\n                    raise AttributeError(err_msg)\n                # the following assertion is thought for those cases where we\n                # use fusion with 'dot product' so that the output_dim will\n                # be 1 and the pred_dim is not 1\n                if deeptabular.output_dim == 1:\n                    assert pred_dim == 1, \"If 'output_dim' is 1, 'pred_dim' must be 1\"\n\n        if deeptabular is not None:\n            is_tabnet = False\n            if isinstance(deeptabular, list):\n                is_any_tabnet = any(\n                    dt.__class__.__name__ == \"TabNet\" for dt in deeptabular\n                )\n                if is_any_tabnet:\n                    raise ValueError(\n                        \"Currently TabNet is not supported as a component of a multiple \"\n                        \"tabular component model.\"\n                    )\n            else:\n                is_tabnet = deeptabular.__class__.__name__ == \"TabNet\"\n            has_wide_text_or_image = (\n                wide is not None or deeptext is not None or deepimage is not None\n            )\n            if is_tabnet and has_wide_text_or_image:\n                warnings.warn(\n                    \"'WideDeep' is a model comprised by multiple components and the 'deeptabular'\"\n                    \" component is 'TabNet'. We recommend using 'TabNet' in isolation.\"\n                    \" The reasons are: i)'TabNet' uses sparse regularization which partially losses\"\n                    \" its purpose when used in combination with other components.\"\n                    \" If you still want to use a multiple component model with 'TabNet',\"\n                    \" consider setting 'lambda_sparse' to 0 during training. ii) The feature\"\n                    \" importances will be computed only for TabNet but the model will comprise multiple\"\n                    \" components. Therefore, such importances will partially lose their 'meaning'.\",\n                    UserWarning,\n                )\n\n        if deeptext is not None:\n            err_msg = \"deeptext model must have an 'output_dim' attribute or property.\"\n            if isinstance(deeptext, list):\n                all_have_output_dim = all(hasattr(dt, \"output_dim\") for dt in deeptext)\n                if not all_have_output_dim:\n                    raise AttributeError(err_msg)\n            else:\n                if not hasattr(deeptext, \"output_dim\"):\n                    raise AttributeError(err_msg)\n                if deeptext.output_dim == 1:\n                    assert pred_dim == 1, \"If 'output_dim' is 1, 'pred_dim' must be 1\"\n\n        if deepimage is not None:\n            err_msg = \"deepimage model must have an 'output_dim' attribute or property.\"\n            if isinstance(deepimage, list):\n                all_have_output_dim = all(hasattr(di, \"output_dim\") for di in deepimage)\n                if not all_have_output_dim:\n                    raise AttributeError(err_msg)\n            else:\n                if not hasattr(deepimage, \"output_dim\"):\n                    raise AttributeError(err_msg)\n                if deepimage.output_dim == 1:\n                    assert pred_dim == 1, \"If 'output_dim' is 1, 'pred_dim' must be 1\"\n\n        if deephead is not None and head_hidden_dims is not None:\n            raise ValueError(\n                \"both 'deephead' and 'head_hidden_dims' are not None. Use one of the other, but not both\"\n            )\n        if (\n            head_hidden_dims is not None\n            and not deeptabular\n            and not deeptext\n            and not deepimage\n        ):\n            raise ValueError(\n                \"if 'head_hidden_dims' is not None, at least one deep component must be used\"\n            )\n\n        if deephead is not None:\n            if not hasattr(deephead, \"output_dim\"):\n                raise AttributeError(\n                    \"As any other custom model passed to 'WideDeep', 'deephead' must have an \"\n                    \"'output_dim' attribute or property. \"\n                )\n            deephead_inp_feat = next(deephead.parameters()).size(1)\n            output_dim = 0\n            if deeptabular is not None:\n                if isinstance(deeptabular, list):\n                    for dt in deeptabular:\n                        output_dim += dt.output_dim\n                else:\n                    output_dim += deeptabular.output_dim\n            if deeptext is not None:\n                if isinstance(deeptext, list):\n                    for dt in deeptext:\n                        output_dim += dt.output_dim\n                else:\n                    output_dim += deeptext.output_dim\n            if deepimage is not None:\n                if isinstance(deepimage, list):\n                    for di in deepimage:\n                        output_dim += di.output_dim\n                else:\n                    output_dim += deepimage.output_dim\n            if deephead_inp_feat != output_dim:\n                warnings.warn(\n                    \"A custom 'deephead' is used and it seems that the input features \"\n                    \"do not match the output of the deep components\",\n                    UserWarning,\n                )\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html","title":"The <code>preprocessing</code> module","text":"<p>This module contains the classes that are used to prepare the data before being passed to the models. There is one Preprocessor per data mode or model component (<code>wide</code>, <code>deeptabular</code>, <code>deepimage</code> and <code>deeptext</code>) with the exception of the <code>deeptext</code> component. In this case, two processors are available: one for the case when no Hugging Face model is used (<code>TextPreprocessor</code>) and another one when a Hugging Face model is used (<code>HFPreprocessor</code>).</p>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.wide_preprocessor.WidePreprocessor","title":"WidePreprocessor","text":"<p>               Bases: <code>BasePreprocessor</code></p> <p>Preprocessor to prepare the wide input dataset</p> <p>This Preprocessor prepares the data for the wide, linear component. This linear model is implemented via an Embedding layer that is connected to the output neuron. <code>WidePreprocessor</code> numerically encodes all the unique values of all categorical columns <code>wide_cols + crossed_cols</code>. See the Example below.</p> <p>Parameters:</p> Name Type Description Default <code>wide_cols</code> <code>List[str]</code> <p>List of strings with the name of the columns that will label encoded and passed through the <code>wide</code> component</p> required <code>crossed_cols</code> <code>Optional[List[Tuple[str, str]]]</code> <p>List of Tuples with the name of the columns that will be <code>'crossed'</code> and then label encoded. e.g. [('education', 'occupation'), ...]. For binary features, a cross-product transformation is 1 if and only if the constituent features are all 1, and 0 otherwise.</p> <code>None</code> <p>Attributes:</p> Name Type Description <code>wide_crossed_cols</code> <code>List</code> <p>List with the names of all columns that will be label encoded</p> <code>encoding_dict</code> <code>Dict</code> <p>Dictionary where the keys are the result of pasting <code>colname + '_' + column value</code> and the values are the corresponding mapped integer.</p> <code>inverse_encoding_dict</code> <code>Dict</code> <p>the inverse encoding dictionary</p> <code>wide_dim</code> <code>int</code> <p>Dimension of the wide model (i.e. dim of the linear layer)</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import pandas as pd\n&gt;&gt;&gt; from pytorch_widedeep.preprocessing import WidePreprocessor\n&gt;&gt;&gt; df = pd.DataFrame({'color': ['r', 'b', 'g'], 'size': ['s', 'n', 'l']})\n&gt;&gt;&gt; wide_cols = ['color']\n&gt;&gt;&gt; crossed_cols = [('color', 'size')]\n&gt;&gt;&gt; wide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols)\n&gt;&gt;&gt; X_wide = wide_preprocessor.fit_transform(df)\n&gt;&gt;&gt; X_wide\narray([[1, 4],\n       [2, 5],\n       [3, 6]])\n&gt;&gt;&gt; wide_preprocessor.encoding_dict\n{'color_r': 1, 'color_b': 2, 'color_g': 3, 'color_size_r-s': 4, 'color_size_b-n': 5, 'color_size_g-l': 6}\n&gt;&gt;&gt; wide_preprocessor.inverse_transform(X_wide)\n  color color_size\n0     r        r-s\n1     b        b-n\n2     g        g-l\n</code></pre> Source code in <code>pytorch_widedeep/preprocessing/wide_preprocessor.py</code> <pre><code>class WidePreprocessor(BasePreprocessor):\n    r\"\"\"Preprocessor to prepare the wide input dataset\n\n    This Preprocessor prepares the data for the wide, linear component.\n    This linear model is implemented via an Embedding layer that is\n    connected to the output neuron. `WidePreprocessor` numerically\n    encodes all the unique values of all categorical columns `wide_cols +\n    crossed_cols`. See the Example below.\n\n    Parameters\n    ----------\n    wide_cols: List\n        List of strings with the name of the columns that will label\n        encoded and passed through the `wide` component\n    crossed_cols: List, default = None\n        List of Tuples with the name of the columns that will be `'crossed'`\n        and then label encoded. e.g. _[('education', 'occupation'), ...]_. For\n        binary features, a cross-product transformation is 1 if and only if\n        the constituent features are all 1, and 0 otherwise.\n\n    Attributes\n    ----------\n    wide_crossed_cols: List\n        List with the names of all columns that will be label encoded\n    encoding_dict: Dict\n        Dictionary where the keys are the result of pasting `colname + '_' +\n        column value` and the values are the corresponding mapped integer.\n    inverse_encoding_dict: Dict\n        the inverse encoding dictionary\n    wide_dim: int\n        Dimension of the wide model (i.e. dim of the linear layer)\n\n    Examples\n    --------\n    &gt;&gt;&gt; import pandas as pd\n    &gt;&gt;&gt; from pytorch_widedeep.preprocessing import WidePreprocessor\n    &gt;&gt;&gt; df = pd.DataFrame({'color': ['r', 'b', 'g'], 'size': ['s', 'n', 'l']})\n    &gt;&gt;&gt; wide_cols = ['color']\n    &gt;&gt;&gt; crossed_cols = [('color', 'size')]\n    &gt;&gt;&gt; wide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols)\n    &gt;&gt;&gt; X_wide = wide_preprocessor.fit_transform(df)\n    &gt;&gt;&gt; X_wide\n    array([[1, 4],\n           [2, 5],\n           [3, 6]])\n    &gt;&gt;&gt; wide_preprocessor.encoding_dict\n    {'color_r': 1, 'color_b': 2, 'color_g': 3, 'color_size_r-s': 4, 'color_size_b-n': 5, 'color_size_g-l': 6}\n    &gt;&gt;&gt; wide_preprocessor.inverse_transform(X_wide)\n      color color_size\n    0     r        r-s\n    1     b        b-n\n    2     g        g-l\n    \"\"\"\n\n    def __init__(\n        self, wide_cols: List[str], crossed_cols: Optional[List[Tuple[str, str]]] = None\n    ):\n        super(WidePreprocessor, self).__init__()\n\n        self.wide_cols = wide_cols\n        self.crossed_cols = crossed_cols\n\n        self.is_fitted = False\n\n    def fit(self, df: pd.DataFrame) -&gt; \"WidePreprocessor\":\n        r\"\"\"Fits the Preprocessor and creates required attributes\n\n        Parameters\n        ----------\n        df: pd.DataFrame\n            Input pandas dataframe\n\n        Returns\n        -------\n        WidePreprocessor\n            `WidePreprocessor` fitted object\n        \"\"\"\n        df_wide = self._prepare_wide(df)\n        self.wide_crossed_cols = df_wide.columns.tolist()\n        glob_feature_list = self._make_global_feature_list(\n            df_wide[self.wide_crossed_cols]\n        )\n        # leave 0 for padding/\"unseen\" categories\n        self.encoding_dict = {v: i + 1 for i, v in enumerate(glob_feature_list)}\n        self.wide_dim = len(self.encoding_dict)\n        self.inverse_encoding_dict = {k: v for v, k in self.encoding_dict.items()}\n        self.inverse_encoding_dict[0] = \"unseen\"\n\n        self.is_fitted = True\n\n        return self\n\n    def transform(self, df: pd.DataFrame) -&gt; np.ndarray:\n        r\"\"\"\n        Parameters\n        ----------\n        df: pd.DataFrame\n            Input pandas dataframe\n\n        Returns\n        -------\n        np.ndarray\n            transformed input dataframe\n        \"\"\"\n        check_is_fitted(self, attributes=[\"encoding_dict\"])\n        df_wide = self._prepare_wide(df)\n        encoded = np.zeros([len(df_wide), len(self.wide_crossed_cols)])\n        for col_i, col in enumerate(self.wide_crossed_cols):\n            encoded[:, col_i] = df_wide[col].apply(\n                lambda x: (\n                    self.encoding_dict[col + \"_\" + str(x)]\n                    if col + \"_\" + str(x) in self.encoding_dict\n                    else 0\n                )\n            )\n        return encoded.astype(\"int64\")\n\n    def transform_sample(self, df: pd.DataFrame) -&gt; np.ndarray:\n        return self.transform(df)[0]\n\n    def inverse_transform(self, encoded: np.ndarray) -&gt; pd.DataFrame:\n        r\"\"\"Takes as input the output from the `transform` method and it will\n        return the original values.\n\n        Parameters\n        ----------\n        encoded: np.ndarray\n            numpy array with the encoded values that are the output from the\n            `transform` method\n\n        Returns\n        -------\n        pd.DataFrame\n            Pandas dataframe with the original values\n        \"\"\"\n        decoded = pd.DataFrame(encoded, columns=self.wide_crossed_cols)\n\n        if pd.__version__ &gt;= \"2.1.0\":\n            decoded = decoded.map(lambda x: self.inverse_encoding_dict[x])\n        else:\n            decoded = decoded.applymap(lambda x: self.inverse_encoding_dict[x])\n\n        for col in decoded.columns:\n            rm_str = \"\".join([col, \"_\"])\n            decoded[col] = decoded[col].apply(lambda x: x.replace(rm_str, \"\"))\n        return decoded\n\n    def fit_transform(self, df: pd.DataFrame) -&gt; np.ndarray:\n        \"\"\"Combines `fit` and `transform`\n\n        Parameters\n        ----------\n        df: pd.DataFrame\n            Input pandas dataframe\n\n        Returns\n        -------\n        np.ndarray\n            transformed input dataframe\n        \"\"\"\n        return self.fit(df).transform(df)\n\n    def _make_global_feature_list(self, df: pd.DataFrame) -&gt; List:\n        glob_feature_list = []\n        for column in df.columns:\n            glob_feature_list += self._make_column_feature_list(df[column])\n        return glob_feature_list\n\n    def _make_column_feature_list(self, s: pd.Series) -&gt; List:\n        return [s.name + \"_\" + str(x) for x in s.unique()]\n\n    def _cross_cols(self, df: pd.DataFrame):\n        df_cc = df.copy()\n        crossed_colnames = []\n        for cols in self.crossed_cols:\n            for c in cols:\n                df_cc[c] = df_cc[c].astype(\"str\")\n            colname = \"_\".join(cols)\n            df_cc[colname] = df_cc[list(cols)].apply(lambda x: \"-\".join(x), axis=1)\n            crossed_colnames.append(colname)\n        return df_cc[crossed_colnames]\n\n    def _prepare_wide(self, df: pd.DataFrame):\n        if self.crossed_cols is not None:\n            df_cc = self._cross_cols(df)\n            return pd.concat([df[self.wide_cols], df_cc], axis=1)\n        else:\n            return df.copy()[self.wide_cols]\n\n    def __repr__(self) -&gt; str:\n        list_of_params: List[str] = [\"wide_cols={wide_cols}\"]\n        if self.crossed_cols is not None:\n            list_of_params.append(\"crossed_cols={crossed_cols}\")\n        all_params = \", \".join(list_of_params)\n        return f\"WidePreprocessor({all_params.format(**self.__dict__)})\"\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.wide_preprocessor.WidePreprocessor.fit","title":"fit","text":"<pre><code>fit(df)\n</code></pre> <p>Fits the Preprocessor and creates required attributes</p> <p>Parameters:</p> Name Type Description Default <code>df</code> <code>DataFrame</code> <p>Input pandas dataframe</p> required <p>Returns:</p> Type Description <code>WidePreprocessor</code> <p><code>WidePreprocessor</code> fitted object</p> Source code in <code>pytorch_widedeep/preprocessing/wide_preprocessor.py</code> <pre><code>def fit(self, df: pd.DataFrame) -&gt; \"WidePreprocessor\":\n    r\"\"\"Fits the Preprocessor and creates required attributes\n\n    Parameters\n    ----------\n    df: pd.DataFrame\n        Input pandas dataframe\n\n    Returns\n    -------\n    WidePreprocessor\n        `WidePreprocessor` fitted object\n    \"\"\"\n    df_wide = self._prepare_wide(df)\n    self.wide_crossed_cols = df_wide.columns.tolist()\n    glob_feature_list = self._make_global_feature_list(\n        df_wide[self.wide_crossed_cols]\n    )\n    # leave 0 for padding/\"unseen\" categories\n    self.encoding_dict = {v: i + 1 for i, v in enumerate(glob_feature_list)}\n    self.wide_dim = len(self.encoding_dict)\n    self.inverse_encoding_dict = {k: v for v, k in self.encoding_dict.items()}\n    self.inverse_encoding_dict[0] = \"unseen\"\n\n    self.is_fitted = True\n\n    return self\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.wide_preprocessor.WidePreprocessor.transform","title":"transform","text":"<pre><code>transform(df)\n</code></pre> <p>Parameters:</p> Name Type Description Default <code>df</code> <code>DataFrame</code> <p>Input pandas dataframe</p> required <p>Returns:</p> Type Description <code>ndarray</code> <p>transformed input dataframe</p> Source code in <code>pytorch_widedeep/preprocessing/wide_preprocessor.py</code> <pre><code>def transform(self, df: pd.DataFrame) -&gt; np.ndarray:\n    r\"\"\"\n    Parameters\n    ----------\n    df: pd.DataFrame\n        Input pandas dataframe\n\n    Returns\n    -------\n    np.ndarray\n        transformed input dataframe\n    \"\"\"\n    check_is_fitted(self, attributes=[\"encoding_dict\"])\n    df_wide = self._prepare_wide(df)\n    encoded = np.zeros([len(df_wide), len(self.wide_crossed_cols)])\n    for col_i, col in enumerate(self.wide_crossed_cols):\n        encoded[:, col_i] = df_wide[col].apply(\n            lambda x: (\n                self.encoding_dict[col + \"_\" + str(x)]\n                if col + \"_\" + str(x) in self.encoding_dict\n                else 0\n            )\n        )\n    return encoded.astype(\"int64\")\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.wide_preprocessor.WidePreprocessor.inverse_transform","title":"inverse_transform","text":"<pre><code>inverse_transform(encoded)\n</code></pre> <p>Takes as input the output from the <code>transform</code> method and it will return the original values.</p> <p>Parameters:</p> Name Type Description Default <code>encoded</code> <code>ndarray</code> <p>numpy array with the encoded values that are the output from the <code>transform</code> method</p> required <p>Returns:</p> Type Description <code>DataFrame</code> <p>Pandas dataframe with the original values</p> Source code in <code>pytorch_widedeep/preprocessing/wide_preprocessor.py</code> <pre><code>def inverse_transform(self, encoded: np.ndarray) -&gt; pd.DataFrame:\n    r\"\"\"Takes as input the output from the `transform` method and it will\n    return the original values.\n\n    Parameters\n    ----------\n    encoded: np.ndarray\n        numpy array with the encoded values that are the output from the\n        `transform` method\n\n    Returns\n    -------\n    pd.DataFrame\n        Pandas dataframe with the original values\n    \"\"\"\n    decoded = pd.DataFrame(encoded, columns=self.wide_crossed_cols)\n\n    if pd.__version__ &gt;= \"2.1.0\":\n        decoded = decoded.map(lambda x: self.inverse_encoding_dict[x])\n    else:\n        decoded = decoded.applymap(lambda x: self.inverse_encoding_dict[x])\n\n    for col in decoded.columns:\n        rm_str = \"\".join([col, \"_\"])\n        decoded[col] = decoded[col].apply(lambda x: x.replace(rm_str, \"\"))\n    return decoded\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.wide_preprocessor.WidePreprocessor.fit_transform","title":"fit_transform","text":"<pre><code>fit_transform(df)\n</code></pre> <p>Combines <code>fit</code> and <code>transform</code></p> <p>Parameters:</p> Name Type Description Default <code>df</code> <code>DataFrame</code> <p>Input pandas dataframe</p> required <p>Returns:</p> Type Description <code>ndarray</code> <p>transformed input dataframe</p> Source code in <code>pytorch_widedeep/preprocessing/wide_preprocessor.py</code> <pre><code>def fit_transform(self, df: pd.DataFrame) -&gt; np.ndarray:\n    \"\"\"Combines `fit` and `transform`\n\n    Parameters\n    ----------\n    df: pd.DataFrame\n        Input pandas dataframe\n\n    Returns\n    -------\n    np.ndarray\n        transformed input dataframe\n    \"\"\"\n    return self.fit(df).transform(df)\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor","title":"TabPreprocessor","text":"<p>               Bases: <code>BasePreprocessor</code></p> <p>Preprocessor to prepare the <code>deeptabular</code> component input dataset</p> <p>Parameters:</p> Name Type Description Default <code>cat_embed_cols</code> <code>Optional[Union[List[str], List[Tuple[str, int]]]]</code> <p>List containing the name of the categorical columns that will be represented by embeddings (e.g. ['education', 'relationship', ...]) or a Tuple with the name and the embedding dimension (e.g.: [ ('education',32), ('relationship',16), ...]). Param alias: <code>embed_cols</code></p> <code>None</code> <code>continuous_cols</code> <code>Optional[List[str]]</code> <p>List with the name of the continuous cols</p> <code>None</code> <code>quantization_setup</code> <code>Optional[Union[int, Dict[str, Union[int, List[float]]]]]</code> <p>Continuous columns can be turned into categorical via <code>pd.cut</code>. If <code>quantization_setup</code> is an <code>int</code>, all continuous columns will be quantized using this value as the number of bins. Alternatively, a dictionary where the keys are the column names to quantize and the values are the either integers indicating the number of bins or a list of scalars indicating the bin edges can also be used. Param alias: <code>cols_and_bins</code></p> <code>None</code> <code>cols_to_scale</code> <code>Optional[Union[List[str], str]]</code> <p>List with the names of the columns that will be standarised via sklearn's <code>StandardScaler</code>. It can also be the string <code>'all'</code> in which case all the continuous cols will be scaled.</p> <code>None</code> <code>auto_embed_dim</code> <code>bool</code> <p>Boolean indicating whether the embedding dimensions will be automatically defined via rule of thumb. See <code>embedding_rule</code> below.</p> <code>True</code> <code>embedding_rule</code> <code>Literal[google, fastai_old, fastai_new]</code> <p>If <code>auto_embed_dim=True</code>, this is the choice of embedding rule of thumb. Choices are:</p> <ul> <li> <p>fastai_new: \\(min(600, round(1.6 \\times n_{cat}^{0.56}))\\)</p> </li> <li> <p>fastai_old: \\(min(50, (n_{cat}//{2})+1)\\)</p> </li> <li> <p>google: \\(min(600, round(n_{cat}^{0.24}))\\)</p> </li> </ul> <code>'fastai_new'</code> <code>default_embed_dim</code> <code>int</code> <p>Dimension for the embeddings if the embedding dimension is not provided in the <code>cat_embed_cols</code> parameter and <code>auto_embed_dim</code> is set to <code>False</code>.</p> <code>16</code> <code>with_attention</code> <code>bool</code> <p>Boolean indicating whether the preprocessed data will be passed to an attention-based model (more precisely a model where all embeddings must have the same dimensions). If <code>True</code>, the param <code>cat_embed_cols</code> must just be a list containing just the categorical column names: e.g. ['education', 'relationship', ...]. This is because they will all be  encoded using embeddings of the same dim, which will be specified  later when the model is defined.  Param alias:  <code>for_transformer</code>, <code>for_matrix_factorization</code>, <code>for_mf</code></p> <code>False</code> <code>with_cls_token</code> <code>bool</code> <p>Boolean indicating if a <code>'[CLS]'</code> token will be added to the dataset when using attention-based models. The final hidden state corresponding to this token is used as the aggregated representation for classification and regression tasks. If not, the categorical and/or continuous embeddings will be concatenated before being passed to the final MLP (if present).</p> <code>False</code> <code>shared_embed</code> <code>bool</code> <p>Boolean indicating if the embeddings will be \"shared\" when using attention-based models. The idea behind <code>shared_embed</code> is described in the Appendix A in the TabTransformer paper: 'The goal of having column embedding is to enable the model to distinguish the classes in one column from those in the other columns'. In other words, the idea is to let the model learn which column is embedded at the time. See: <code>pytorch_widedeep.models.transformers._layers.SharedEmbeddings</code>.</p> <code>False</code> <code>verbose</code> <code>int</code> <code>1</code> <code>scale</code> <code>bool</code> <p> note: this arg will be removed in upcoming  releases. Please use <code>cols_to_scale</code> instead.  Bool indicating  whether or not to scale/standarise continuous cols. It is important  to emphasize that all the DL models for tabular data in the library  also include the possibility of normalising the input continuous  features via a <code>BatchNorm</code> or a <code>LayerNorm</code>.  Param alias:  <code>scale_cont_cols</code>.</p> <code>False</code> <code>already_standard</code> <code>Optional[List[str]]</code> <p> note: this arg will be removed in upcoming  releases. Please use <code>cols_to_scale</code> instead.  List with the  name of the continuous cols that do not need to be  scaled/standarised.</p> <code>None</code> <p>Other Parameters:</p> Name Type Description <code>**kwargs</code> <p><code>pd.cut</code> and <code>StandardScaler</code> related args</p> <p>Attributes:</p> Name Type Description <code>cat_cols</code> <code>List[str]</code> <p>List containing the names of the categorical columns after processing.</p> <code>cols_and_bins</code> <code>Optional[Dict[str, Union[int, List[float]]]]</code> <p>Dictionary containing the quantization setup after processing if quantization is requested. This is derived from the quantization_setup parameter.</p> <code>quant_args</code> <code>Dict</code> <p>Dictionary containing arguments passed to pandas.cut() function for quantization.</p> <code>scale_args</code> <code>Dict</code> <p>Dictionary containing arguments passed to StandardScaler.</p> <code>label_encoder</code> <code>LabelEncoder</code> <p>Instance of LabelEncoder used to encode categorical variables. See <code>pytorch_widedeep.utils.dense_utils.LabelEncoder</code>.</p> <code>cat_embed_input</code> <code>List</code> <p>List of tuples with the column name, number of individual values for that column and, if <code>with_attention</code> is set to <code>False</code>, the corresponding embeddings dim, e.g. [('education', 16, 10), ('relationship', 6, 8), ...].</p> <code>standardize_cols</code> <code>List</code> <p>List of the columns that will be standardized.</p> <code>scaler</code> <code>StandardScaler</code> <p>An instance of <code>sklearn.preprocessing.StandardScaler</code> if standardization is requested.</p> <code>column_idx</code> <code>Dict</code> <p>Dictionary where keys are column names and values are column indexes. This is necessary to slice tensors.</p> <code>quantizer</code> <code>Quantizer</code> <p>An instance of <code>Quantizer</code> if quantization is requested.</p> <code>is_fitted</code> <code>bool</code> <p>Boolean indicating if the preprocessor has been fitted.</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import pandas as pd\n&gt;&gt;&gt; import numpy as np\n&gt;&gt;&gt; from pytorch_widedeep.preprocessing import TabPreprocessor\n&gt;&gt;&gt; df = pd.DataFrame({'color': ['r', 'b', 'g'], 'size': ['s', 'n', 'l'], 'age': [25, 40, 55]})\n&gt;&gt;&gt; cat_embed_cols = [('color',5), ('size',5)]\n&gt;&gt;&gt; cont_cols = ['age']\n&gt;&gt;&gt; deep_preprocessor = TabPreprocessor(cat_embed_cols=cat_embed_cols, continuous_cols=cont_cols)\n&gt;&gt;&gt; X_tab = deep_preprocessor.fit_transform(df)\n&gt;&gt;&gt; deep_preprocessor.cat_embed_cols\n[('color', 5), ('size', 5)]\n&gt;&gt;&gt; deep_preprocessor.column_idx\n{'color': 0, 'size': 1, 'age': 2}\n&gt;&gt;&gt; cont_df = pd.DataFrame({\"col1\": np.random.rand(10), \"col2\": np.random.rand(10) + 1})\n&gt;&gt;&gt; cont_cols = [\"col1\", \"col2\"]\n&gt;&gt;&gt; tab_preprocessor = TabPreprocessor(continuous_cols=cont_cols, quantization_setup=3)\n&gt;&gt;&gt; ft_cont_df = tab_preprocessor.fit_transform(cont_df)\n&gt;&gt;&gt; # or...\n&gt;&gt;&gt; quantization_setup = {'col1': [0., 0.4, 1.], 'col2': [1., 1.4, 2.]}\n&gt;&gt;&gt; tab_preprocessor2 = TabPreprocessor(continuous_cols=cont_cols, quantization_setup=quantization_setup)\n&gt;&gt;&gt; ft_cont_df2 = tab_preprocessor2.fit_transform(cont_df)\n</code></pre> Source code in <code>pytorch_widedeep/preprocessing/tab_preprocessor.py</code> <pre><code>class TabPreprocessor(BasePreprocessor):\n    r\"\"\"Preprocessor to prepare the `deeptabular` component input dataset\n\n    Parameters\n    ----------\n    cat_embed_cols: List, default = None\n        List containing the name of the categorical columns that will be\n        represented by embeddings (e.g. _['education', 'relationship', ...]_) or\n        a Tuple with the name and the embedding dimension (e.g.: _[\n        ('education',32), ('relationship',16), ...]_).&lt;br/&gt; Param alias: `embed_cols`\n    continuous_cols: List, default = None\n        List with the name of the continuous cols\n    quantization_setup: int or Dict, default = None\n        Continuous columns can be turned into categorical via `pd.cut`. If\n        `quantization_setup` is an `int`, all continuous columns will be\n        quantized using this value as the number of bins. Alternatively, a\n        dictionary where the keys are the column names to quantize and the\n        values are the either integers indicating the number of bins or a\n        list of scalars indicating the bin edges can also be used.&lt;br/&gt;\n        Param alias: `cols_and_bins`\n    cols_to_scale: List or str, default = None,\n        List with the names of the columns that will be standarised via\n        sklearn's `StandardScaler`. It can also be the string `'all'` in\n        which case all the continuous cols will be scaled.\n    auto_embed_dim: bool, default = True\n        Boolean indicating whether the embedding dimensions will be\n        automatically defined via rule of thumb. See `embedding_rule`\n        below.\n    embedding_rule: str, default = 'fastai_new'\n        If `auto_embed_dim=True`, this is the choice of embedding rule of\n        thumb. Choices are:\n\n        - _fastai_new_: $min(600, round(1.6 \\times n_{cat}^{0.56}))$\n\n        - _fastai_old_: $min(50, (n_{cat}//{2})+1)$\n\n        - _google_: $min(600, round(n_{cat}^{0.24}))$\n    default_embed_dim: int, default=16\n        Dimension for the embeddings if the embedding dimension is not\n        provided in the `cat_embed_cols` parameter and `auto_embed_dim` is\n        set to `False`.\n    with_attention: bool, default = False\n        Boolean indicating whether the preprocessed data will be passed to an\n        attention-based model (more precisely a model where all embeddings\n        must have the same dimensions). If `True`, the param `cat_embed_cols`\n        must just be a list containing just the categorical column names:\n        e.g.\n        _['education', 'relationship', ...]_. This is because they will all be\n         encoded using embeddings of the same dim, which will be specified\n         later when the model is defined. &lt;br/&gt; Param alias:\n         `for_transformer`, `for_matrix_factorization`, `for_mf`\n    with_cls_token: bool, default = False\n        Boolean indicating if a `'[CLS]'` token will be added to the dataset\n        when using attention-based models. The final hidden state\n        corresponding to this token is used as the aggregated representation\n        for classification and regression tasks. If not, the categorical\n        and/or continuous embeddings will be concatenated before being passed\n        to the final MLP (if present).\n    shared_embed: bool, default = False\n        Boolean indicating if the embeddings will be \"shared\" when using\n        attention-based models. The idea behind `shared_embed` is\n        described in the Appendix A in the [TabTransformer paper](https://arxiv.org/abs/2012.06678):\n        _'The goal of having column embedding is to enable the model to\n        distinguish the classes in one column from those in the other\n        columns'_. In other words, the idea is to let the model learn which\n        column is embedded at the time. See: `pytorch_widedeep.models.transformers._layers.SharedEmbeddings`.\n    verbose: int, default = 1\n    scale: bool, default = False\n        :information_source: **note**: this arg will be removed in upcoming\n         releases. Please use `cols_to_scale` instead. &lt;br/&gt; Bool indicating\n         whether or not to scale/standarise continuous cols. It is important\n         to emphasize that all the DL models for tabular data in the library\n         also include the possibility of normalising the input continuous\n         features via a `BatchNorm` or a `LayerNorm`. &lt;br/&gt; Param alias:\n         `scale_cont_cols`.\n    already_standard: List, default = None\n        :information_source: **note**: this arg will be removed in upcoming\n         releases. Please use `cols_to_scale` instead. &lt;br/&gt; List with the\n         name of the continuous cols that do not need to be\n         scaled/standarised.\n\n    Other Parameters\n    ----------------\n    **kwargs: dict\n        `pd.cut` and `StandardScaler` related args\n\n    Attributes\n    ----------\n    cat_cols: List[str]\n        List containing the names of the categorical columns after processing.\n    cols_and_bins: Optional[Dict[str, Union[int, List[float]]]]\n        Dictionary containing the quantization setup after processing if\n        quantization is requested. This is derived from the\n        quantization_setup parameter.\n    quant_args: Dict\n        Dictionary containing arguments passed to pandas.cut() function for quantization.\n    scale_args: Dict\n        Dictionary containing arguments passed to StandardScaler.\n    label_encoder: LabelEncoder\n        Instance of LabelEncoder used to encode categorical variables.\n        See `pytorch_widedeep.utils.dense_utils.LabelEncoder`.\n    cat_embed_input: List\n        List of tuples with the column name, number of individual values for\n        that column and, if `with_attention` is set to `False`, the\n        corresponding embeddings dim, e.g. _[('education', 16, 10),\n        ('relationship', 6, 8), ...]_.\n    standardize_cols: List\n        List of the columns that will be standardized.\n    scaler: StandardScaler\n        An instance of `sklearn.preprocessing.StandardScaler` if standardization\n        is requested.\n    column_idx: Dict\n        Dictionary where keys are column names and values are column indexes.\n        This is necessary to slice tensors.\n    quantizer: Quantizer\n        An instance of `Quantizer` if quantization is requested.\n    is_fitted: bool\n        Boolean indicating if the preprocessor has been fitted.\n\n    Examples\n    --------\n    &gt;&gt;&gt; import pandas as pd\n    &gt;&gt;&gt; import numpy as np\n    &gt;&gt;&gt; from pytorch_widedeep.preprocessing import TabPreprocessor\n    &gt;&gt;&gt; df = pd.DataFrame({'color': ['r', 'b', 'g'], 'size': ['s', 'n', 'l'], 'age': [25, 40, 55]})\n    &gt;&gt;&gt; cat_embed_cols = [('color',5), ('size',5)]\n    &gt;&gt;&gt; cont_cols = ['age']\n    &gt;&gt;&gt; deep_preprocessor = TabPreprocessor(cat_embed_cols=cat_embed_cols, continuous_cols=cont_cols)\n    &gt;&gt;&gt; X_tab = deep_preprocessor.fit_transform(df)\n    &gt;&gt;&gt; deep_preprocessor.cat_embed_cols\n    [('color', 5), ('size', 5)]\n    &gt;&gt;&gt; deep_preprocessor.column_idx\n    {'color': 0, 'size': 1, 'age': 2}\n    &gt;&gt;&gt; cont_df = pd.DataFrame({\"col1\": np.random.rand(10), \"col2\": np.random.rand(10) + 1})\n    &gt;&gt;&gt; cont_cols = [\"col1\", \"col2\"]\n    &gt;&gt;&gt; tab_preprocessor = TabPreprocessor(continuous_cols=cont_cols, quantization_setup=3)\n    &gt;&gt;&gt; ft_cont_df = tab_preprocessor.fit_transform(cont_df)\n    &gt;&gt;&gt; # or...\n    &gt;&gt;&gt; quantization_setup = {'col1': [0., 0.4, 1.], 'col2': [1., 1.4, 2.]}\n    &gt;&gt;&gt; tab_preprocessor2 = TabPreprocessor(continuous_cols=cont_cols, quantization_setup=quantization_setup)\n    &gt;&gt;&gt; ft_cont_df2 = tab_preprocessor2.fit_transform(cont_df)\n    \"\"\"\n\n    @alias(\"with_attention\", [\"for_transformer\", \"for_matrix_factorization\", \"for_mf\"])\n    @alias(\"cat_embed_cols\", [\"embed_cols\"])\n    @alias(\"scale\", [\"scale_cont_cols\"])\n    @alias(\"quantization_setup\", [\"cols_and_bins\"])\n    def __init__(\n        self,\n        cat_embed_cols: Optional[Union[List[str], List[Tuple[str, int]]]] = None,\n        continuous_cols: Optional[List[str]] = None,\n        quantization_setup: Optional[\n            Union[int, Dict[str, Union[int, List[float]]]]\n        ] = None,\n        cols_to_scale: Optional[Union[List[str], str]] = None,\n        auto_embed_dim: bool = True,\n        embedding_rule: Literal[\"google\", \"fastai_old\", \"fastai_new\"] = \"fastai_new\",\n        default_embed_dim: int = 16,\n        with_attention: bool = False,\n        with_cls_token: bool = False,\n        shared_embed: bool = False,\n        verbose: int = 1,\n        *,\n        scale: bool = False,\n        already_standard: Optional[List[str]] = None,\n        **kwargs,\n    ):\n        super(TabPreprocessor, self).__init__()\n\n        self.continuous_cols = continuous_cols\n        self.quantization_setup = quantization_setup\n        self.cols_to_scale = cols_to_scale\n        self.scale = scale\n        self.already_standard = already_standard\n        self.auto_embed_dim = auto_embed_dim\n        self.embedding_rule = embedding_rule\n        self.default_embed_dim = default_embed_dim\n        self.with_attention = with_attention\n        self.with_cls_token = with_cls_token\n        self.shared_embed = shared_embed\n        self.verbose = verbose\n\n        self.quant_args = {\n            k: v for k, v in kwargs.items() if k in pd.cut.__code__.co_varnames\n        }\n        self.scale_args = {\n            k: v for k, v in kwargs.items() if k in StandardScaler().get_params()\n        }\n\n        self._check_inputs(cat_embed_cols)\n\n        if with_cls_token:\n            self.cat_embed_cols = (\n                [\"cls_token\"] + cat_embed_cols  # type: ignore[operator]\n                if cat_embed_cols is not None\n                else [\"cls_token\"]\n            )\n        else:\n            self.cat_embed_cols = cat_embed_cols  # type: ignore[assignment]\n\n        self.is_fitted = False\n\n    def fit(self, df: pd.DataFrame) -&gt; BasePreprocessor:  # noqa: C901\n        \"\"\"Fits the Preprocessor and creates required attributes\n\n        Parameters\n        ----------\n        df: pd.DataFrame\n            Input pandas dataframe\n\n        Returns\n        -------\n        TabPreprocessor\n            `TabPreprocessor` fitted object\n        \"\"\"\n\n        df_adj = self._insert_cls_token(df) if self.with_cls_token else df.copy()\n\n        self.column_idx: Dict[str, int] = {}\n\n        # Categorical embeddings logic\n        if self.cat_embed_cols is not None or self.quantization_setup is not None:\n            self.cat_embed_input: List[Union[Tuple[str, int], Tuple[str, int, int]]] = (\n                []\n            )\n\n        if self.cat_embed_cols is not None:\n            df_cat, cat_embed_dim = self._prepare_categorical(df_adj)\n\n            self.label_encoder = LabelEncoder(\n                columns_to_encode=df_cat.columns.tolist(),\n                shared_embed=self.shared_embed,\n                with_attention=self.with_attention,\n            )\n            self.label_encoder.fit(df_cat)\n\n            for k, v in self.label_encoder.encoding_dict.items():\n                if self.with_attention:\n                    self.cat_embed_input.append((k, len(v)))\n                else:\n                    self.cat_embed_input.append((k, len(v), cat_embed_dim[k]))\n\n            self.column_idx.update({k: v for v, k in enumerate(df_cat.columns)})\n\n        # Continuous columns logic\n        if self.continuous_cols is not None:\n            df_cont, cont_embed_dim = self._prepare_continuous(df_adj)\n\n            # Standardization logic\n            if self.standardize_cols is not None:\n                self.scaler = StandardScaler(**self.scale_args).fit(\n                    df_cont[self.standardize_cols].values\n                )\n            elif self.verbose:\n                warnings.warn(\"Continuous columns will not be normalised\")\n\n            # Quantization logic\n            if self.cols_and_bins is not None:\n                # we do not run 'Quantizer.fit' here since in the wild case\n                # someone wants standardization and quantization for the same\n                # columns, the Quantizer will run on the scaled data\n                self.quantizer = Quantizer(self.cols_and_bins, **self.quant_args)\n\n                if self.with_attention:\n                    for col, n_cat, _ in cont_embed_dim:\n                        self.cat_embed_input.append((col, n_cat))\n                else:\n                    self.cat_embed_input.extend(cont_embed_dim)\n\n            self.column_idx.update(\n                {k: v + len(self.column_idx) for v, k in enumerate(df_cont)}\n            )\n\n        self.is_fitted = True\n\n        return self\n\n    def transform(self, df: pd.DataFrame) -&gt; np.ndarray:  # noqa: C901\n        \"\"\"Returns the processed `dataframe` as a np.ndarray\n\n        Parameters\n        ----------\n        df: pd.DataFrame\n            Input pandas dataframe\n\n        Returns\n        -------\n        np.ndarray\n            transformed input dataframe\n        \"\"\"\n        check_is_fitted(self, condition=self.is_fitted)\n\n        df_adj = self._insert_cls_token(df) if self.with_cls_token else df.copy()\n\n        if self.cat_embed_cols is not None:\n            df_cat = df_adj[self.cat_cols]\n            df_cat = self.label_encoder.transform(df_cat)\n        if self.continuous_cols is not None:\n            df_cont = df_adj[self.continuous_cols]\n            # Standardization logic\n            if self.standardize_cols:\n                df_cont[self.standardize_cols] = self.scaler.transform(\n                    df_cont[self.standardize_cols].values\n                )\n            # Quantization logic\n            if self.cols_and_bins is not None:\n                # Adjustment so I don't have to override the method\n                # in 'ChunkTabPreprocessor'\n                if self.quantizer.is_fitted:\n                    df_cont = self.quantizer.transform(df_cont)\n                else:\n                    df_cont = self.quantizer.fit_transform(df_cont)\n        try:\n            df_deep = pd.concat([df_cat, df_cont], axis=1)\n        except NameError:\n            try:\n                df_deep = df_cat.copy()\n            except NameError:\n                df_deep = df_cont.copy()\n\n        return df_deep.values\n\n    def transform_sample(self, df: pd.DataFrame) -&gt; np.ndarray:\n        return self.transform(df).astype(\"float\")[0]\n\n    def inverse_transform(self, encoded: np.ndarray) -&gt; pd.DataFrame:  # noqa: C901\n        r\"\"\"Takes as input the output from the `transform` method and it will\n        return the original values.\n\n        Parameters\n        ----------\n        encoded: np.ndarray\n            array with the output of the `transform` method\n\n        Returns\n        -------\n        pd.DataFrame\n            Pandas dataframe with the original values\n        \"\"\"\n        decoded = pd.DataFrame(encoded, columns=list(self.column_idx.keys()))\n        # embeddings back to original category\n        if self.cat_embed_cols is not None:\n            decoded = self.label_encoder.inverse_transform(decoded)\n        if self.continuous_cols is not None:\n            # quantized cols to the mid point\n            if self.cols_and_bins is not None:\n                if self.verbose:\n                    print(\n                        \"Note that quantized cols will be turned into the mid point of \"\n                        \"the corresponding bin\"\n                    )\n                for k, v in self.quantizer.inversed_bins.items():\n                    decoded[k] = decoded[k].map(v)\n            # continuous_cols back to non-standarised\n            try:\n                decoded[self.standardize_cols] = self.scaler.inverse_transform(\n                    decoded[self.standardize_cols]\n                )\n            except Exception:  # KeyError:\n                pass\n\n        if \"cls_token\" in decoded.columns:\n            decoded.drop(\"cls_token\", axis=1, inplace=True)\n\n        return decoded\n\n    def fit_transform(self, df: pd.DataFrame) -&gt; np.ndarray:\n        \"\"\"Combines `fit` and `transform`\n\n        Parameters\n        ----------\n        df: pd.DataFrame\n            Input pandas dataframe\n\n        Returns\n        -------\n        np.ndarray\n            transformed input dataframe\n        \"\"\"\n        return self.fit(df).transform(df)\n\n    def _insert_cls_token(self, df: pd.DataFrame) -&gt; pd.DataFrame:\n        df_cls = df.copy()\n        df_cls.insert(loc=0, column=\"cls_token\", value=\"[CLS]\")\n        return df_cls\n\n    def _prepare_categorical(\n        self, df: pd.DataFrame\n    ) -&gt; Tuple[pd.DataFrame, Dict[str, int]]:\n        if isinstance(self.cat_embed_cols[0], tuple):\n            self.cat_cols: List[str] = [emb[0] for emb in self.cat_embed_cols]\n            cat_embed_dim: Dict[str, int] = dict(self.cat_embed_cols)  # type: ignore\n        else:\n            self.cat_cols = self.cat_embed_cols  # type: ignore[assignment]\n            if self.auto_embed_dim:\n                assert isinstance(self.cat_embed_cols[0], str), (\n                    \"If 'auto_embed_dim' is 'True' and 'with_attention' is 'False', \"\n                    \"'cat_embed_cols' must be a list of strings with the columns to \"\n                    \"be encoded as embeddings.\"\n                )\n                n_cats: Dict[str, int] = {col: df[col].nunique() for col in self.cat_embed_cols}  # type: ignore[misc]\n                cat_embed_dim = {\n                    col: embed_sz_rule(n_cat, self.embedding_rule)  # type: ignore[misc]\n                    for col, n_cat in n_cats.items()\n                }\n            else:\n                cat_embed_dim = {\n                    e: self.default_embed_dim for e in self.cat_embed_cols  # type: ignore[misc]\n                }  # type: ignore\n        return df[self.cat_cols], cat_embed_dim\n\n    def _prepare_continuous(\n        self, df: pd.DataFrame\n    ) -&gt; Tuple[pd.DataFrame, Optional[List[Tuple[str, int, int]]]]:\n        # Standardization logic\n        if self.cols_to_scale is not None:\n            self.standardize_cols = (\n                self.cols_to_scale\n                if self.cols_to_scale != \"all\"\n                else self.continuous_cols\n            )\n        elif self.scale:\n            if self.already_standard is not None:\n                self.standardize_cols = [\n                    c for c in self.continuous_cols if c not in self.already_standard\n                ]\n            else:\n                self.standardize_cols = self.continuous_cols\n        else:\n            self.standardize_cols = None\n\n        # Quantization logic\n        if self.quantization_setup is not None:\n            # the quantized columns are then treated as categorical\n            quant_cont_embed_input: Optional[List[Tuple[str, int, int]]] = []\n            if isinstance(self.quantization_setup, int):\n                self.cols_and_bins: Optional[Dict[str, Union[int, List[float]]]] = {}\n                for col in self.continuous_cols:\n                    self.cols_and_bins[col] = self.quantization_setup\n                    quant_cont_embed_input.append(\n                        (\n                            col,\n                            self.quantization_setup,\n                            (\n                                embed_sz_rule(\n                                    self.quantization_setup + 1, self.embedding_rule  # type: ignore[arg-type]\n                                )\n                                if self.auto_embed_dim\n                                else self.default_embed_dim\n                            ),\n                        )\n                    )\n            else:\n                for col, val in self.quantization_setup.items():\n                    if isinstance(val, int):\n                        quant_cont_embed_input.append(\n                            (\n                                col,\n                                val,\n                                (\n                                    embed_sz_rule(\n                                        val + 1, self.embedding_rule  # type: ignore[arg-type]\n                                    )\n                                    if self.auto_embed_dim\n                                    else self.default_embed_dim\n                                ),\n                            )\n                        )\n                    else:\n                        quant_cont_embed_input.append(\n                            (\n                                col,\n                                len(val) - 1,\n                                (\n                                    embed_sz_rule(len(val), self.embedding_rule)  # type: ignore[arg-type]\n                                    if self.auto_embed_dim\n                                    else self.default_embed_dim\n                                ),\n                            )\n                        )\n\n                self.cols_and_bins = self.quantization_setup.copy()\n        else:\n            self.cols_and_bins = None\n            quant_cont_embed_input = None\n\n        return df[self.continuous_cols], quant_cont_embed_input\n\n    def _check_inputs(self, cat_embed_cols):  # noqa: C901\n        if self.scale or self.already_standard is not None:\n            warnings.warn(\n                \"'scale' and 'already_standard' will be deprecated in the next release. \"\n                \"Please use 'cols_to_scale' instead\",\n                DeprecationWarning,\n                stacklevel=2,\n            )\n\n        if self.scale:\n            if self.already_standard is not None:\n                standardize_cols = [\n                    c for c in self.continuous_cols if c not in self.already_standard\n                ]\n            else:\n                standardize_cols = self.continuous_cols\n        elif self.cols_to_scale is not None:\n            standardize_cols = self.cols_to_scale\n        else:\n            standardize_cols = None\n\n        if standardize_cols is not None:\n            if isinstance(self.quantization_setup, int):\n                cols_to_quantize_and_standardize = [\n                    c for c in standardize_cols if c in self.continuous_cols\n                ]\n            elif isinstance(self.quantization_setup, dict):\n                cols_to_quantize_and_standardize = [\n                    c for c in standardize_cols if c in self.quantization_setup\n                ]\n            else:\n                cols_to_quantize_and_standardize = None\n            if cols_to_quantize_and_standardize is not None:\n                warnings.warn(\n                    f\"the following columns: {cols_to_quantize_and_standardize} will be first scaled\"\n                    \" using a StandardScaler and then quantized. Make sure this is what you really want\"\n                )\n\n        if self.with_cls_token and not self.with_attention:\n            warnings.warn(\n                \"If 'with_cls_token' is set to 'True', 'with_attention' will be automatically \"\n                \"to 'True' if is 'False'\",\n                UserWarning,\n            )\n            self.with_attention = True\n\n        if (cat_embed_cols is None) and (self.continuous_cols is None):\n            raise ValueError(\n                \"'cat_embed_cols' and 'continuous_cols' are 'None'. Please, define at least one of the two.\"\n            )\n\n        if (\n            cat_embed_cols is not None\n            and self.continuous_cols is not None\n            and len(np.intersect1d(cat_embed_cols, self.continuous_cols)) &gt; 0\n        ):\n            overlapping_cols = list(\n                np.intersect1d(cat_embed_cols, self.continuous_cols)\n            )\n            raise ValueError(\n                \"Currently passing columns as both categorical and continuum is not supported.\"\n                \" Please, choose one or the other for the following columns: {}\".format(\n                    \", \".join(overlapping_cols)\n                )\n            )\n        transformer_error_message = (\n            \"If with_attention is 'True' cat_embed_cols must be a list \"\n            \" of strings with the columns to be encoded as embeddings.\"\n        )\n        if (\n            self.with_attention\n            and cat_embed_cols is not None\n            and isinstance(cat_embed_cols[0], tuple)\n        ):\n            raise ValueError(transformer_error_message)\n\n    def __repr__(self) -&gt; str:  # noqa: C901\n        list_of_params: List[str] = []\n        if self.cat_embed_cols is not None:\n            list_of_params.append(\"cat_embed_cols={cat_embed_cols}\")\n        if self.continuous_cols is not None:\n            list_of_params.append(\"continuous_cols={continuous_cols}\")\n        if self.quantization_setup is not None:\n            list_of_params.append(\"quantization_setup={quantization_setup}\")\n        if self.cols_to_scale is not None:\n            list_of_params.append(\"cols_to_scale={cols_to_scale}\")\n        if not self.auto_embed_dim:\n            list_of_params.append(\"auto_embed_dim={auto_embed_dim}\")\n        if self.embedding_rule != \"fastai_new\":\n            list_of_params.append(\"embedding_rule='{embedding_rule}'\")\n        if self.default_embed_dim != 16:\n            list_of_params.append(\"default_embed_dim={default_embed_dim}\")\n        if self.with_attention:\n            list_of_params.append(\"with_attention={with_attention}\")\n        if self.with_cls_token:\n            list_of_params.append(\"with_cls_token={with_cls_token}\")\n        if self.shared_embed:\n            list_of_params.append(\"shared_embed={shared_embed}\")\n        if self.verbose != 1:\n            list_of_params.append(\"verbose={verbose}\")\n        if self.scale:\n            list_of_params.append(\"scale={scale}\")\n        if self.already_standard is not None:\n            list_of_params.append(\"already_standard={already_standard}\")\n        if len(self.quant_args) &gt; 0:\n            list_of_params.append(\n                \", \".join([f\"{k}\" + \"=\" + f\"{v}\" for k, v in self.quant_args.items()])\n            )\n        if len(self.scale_args) &gt; 0:\n            list_of_params.append(\n                \", \".join([f\"{k}\" + \"=\" + f\"{v}\" for k, v in self.scale_args.items()])\n            )\n        all_params = \", \".join(list_of_params)\n        return f\"TabPreprocessor({all_params.format(**self.__dict__)})\"\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor.fit","title":"fit","text":"<pre><code>fit(df)\n</code></pre> <p>Fits the Preprocessor and creates required attributes</p> <p>Parameters:</p> Name Type Description Default <code>df</code> <code>DataFrame</code> <p>Input pandas dataframe</p> required <p>Returns:</p> Type Description <code>TabPreprocessor</code> <p><code>TabPreprocessor</code> fitted object</p> Source code in <code>pytorch_widedeep/preprocessing/tab_preprocessor.py</code> <pre><code>def fit(self, df: pd.DataFrame) -&gt; BasePreprocessor:  # noqa: C901\n    \"\"\"Fits the Preprocessor and creates required attributes\n\n    Parameters\n    ----------\n    df: pd.DataFrame\n        Input pandas dataframe\n\n    Returns\n    -------\n    TabPreprocessor\n        `TabPreprocessor` fitted object\n    \"\"\"\n\n    df_adj = self._insert_cls_token(df) if self.with_cls_token else df.copy()\n\n    self.column_idx: Dict[str, int] = {}\n\n    # Categorical embeddings logic\n    if self.cat_embed_cols is not None or self.quantization_setup is not None:\n        self.cat_embed_input: List[Union[Tuple[str, int], Tuple[str, int, int]]] = (\n            []\n        )\n\n    if self.cat_embed_cols is not None:\n        df_cat, cat_embed_dim = self._prepare_categorical(df_adj)\n\n        self.label_encoder = LabelEncoder(\n            columns_to_encode=df_cat.columns.tolist(),\n            shared_embed=self.shared_embed,\n            with_attention=self.with_attention,\n        )\n        self.label_encoder.fit(df_cat)\n\n        for k, v in self.label_encoder.encoding_dict.items():\n            if self.with_attention:\n                self.cat_embed_input.append((k, len(v)))\n            else:\n                self.cat_embed_input.append((k, len(v), cat_embed_dim[k]))\n\n        self.column_idx.update({k: v for v, k in enumerate(df_cat.columns)})\n\n    # Continuous columns logic\n    if self.continuous_cols is not None:\n        df_cont, cont_embed_dim = self._prepare_continuous(df_adj)\n\n        # Standardization logic\n        if self.standardize_cols is not None:\n            self.scaler = StandardScaler(**self.scale_args).fit(\n                df_cont[self.standardize_cols].values\n            )\n        elif self.verbose:\n            warnings.warn(\"Continuous columns will not be normalised\")\n\n        # Quantization logic\n        if self.cols_and_bins is not None:\n            # we do not run 'Quantizer.fit' here since in the wild case\n            # someone wants standardization and quantization for the same\n            # columns, the Quantizer will run on the scaled data\n            self.quantizer = Quantizer(self.cols_and_bins, **self.quant_args)\n\n            if self.with_attention:\n                for col, n_cat, _ in cont_embed_dim:\n                    self.cat_embed_input.append((col, n_cat))\n            else:\n                self.cat_embed_input.extend(cont_embed_dim)\n\n        self.column_idx.update(\n            {k: v + len(self.column_idx) for v, k in enumerate(df_cont)}\n        )\n\n    self.is_fitted = True\n\n    return self\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor.transform","title":"transform","text":"<pre><code>transform(df)\n</code></pre> <p>Returns the processed <code>dataframe</code> as a np.ndarray</p> <p>Parameters:</p> Name Type Description Default <code>df</code> <code>DataFrame</code> <p>Input pandas dataframe</p> required <p>Returns:</p> Type Description <code>ndarray</code> <p>transformed input dataframe</p> Source code in <code>pytorch_widedeep/preprocessing/tab_preprocessor.py</code> <pre><code>def transform(self, df: pd.DataFrame) -&gt; np.ndarray:  # noqa: C901\n    \"\"\"Returns the processed `dataframe` as a np.ndarray\n\n    Parameters\n    ----------\n    df: pd.DataFrame\n        Input pandas dataframe\n\n    Returns\n    -------\n    np.ndarray\n        transformed input dataframe\n    \"\"\"\n    check_is_fitted(self, condition=self.is_fitted)\n\n    df_adj = self._insert_cls_token(df) if self.with_cls_token else df.copy()\n\n    if self.cat_embed_cols is not None:\n        df_cat = df_adj[self.cat_cols]\n        df_cat = self.label_encoder.transform(df_cat)\n    if self.continuous_cols is not None:\n        df_cont = df_adj[self.continuous_cols]\n        # Standardization logic\n        if self.standardize_cols:\n            df_cont[self.standardize_cols] = self.scaler.transform(\n                df_cont[self.standardize_cols].values\n            )\n        # Quantization logic\n        if self.cols_and_bins is not None:\n            # Adjustment so I don't have to override the method\n            # in 'ChunkTabPreprocessor'\n            if self.quantizer.is_fitted:\n                df_cont = self.quantizer.transform(df_cont)\n            else:\n                df_cont = self.quantizer.fit_transform(df_cont)\n    try:\n        df_deep = pd.concat([df_cat, df_cont], axis=1)\n    except NameError:\n        try:\n            df_deep = df_cat.copy()\n        except NameError:\n            df_deep = df_cont.copy()\n\n    return df_deep.values\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor.inverse_transform","title":"inverse_transform","text":"<pre><code>inverse_transform(encoded)\n</code></pre> <p>Takes as input the output from the <code>transform</code> method and it will return the original values.</p> <p>Parameters:</p> Name Type Description Default <code>encoded</code> <code>ndarray</code> <p>array with the output of the <code>transform</code> method</p> required <p>Returns:</p> Type Description <code>DataFrame</code> <p>Pandas dataframe with the original values</p> Source code in <code>pytorch_widedeep/preprocessing/tab_preprocessor.py</code> <pre><code>def inverse_transform(self, encoded: np.ndarray) -&gt; pd.DataFrame:  # noqa: C901\n    r\"\"\"Takes as input the output from the `transform` method and it will\n    return the original values.\n\n    Parameters\n    ----------\n    encoded: np.ndarray\n        array with the output of the `transform` method\n\n    Returns\n    -------\n    pd.DataFrame\n        Pandas dataframe with the original values\n    \"\"\"\n    decoded = pd.DataFrame(encoded, columns=list(self.column_idx.keys()))\n    # embeddings back to original category\n    if self.cat_embed_cols is not None:\n        decoded = self.label_encoder.inverse_transform(decoded)\n    if self.continuous_cols is not None:\n        # quantized cols to the mid point\n        if self.cols_and_bins is not None:\n            if self.verbose:\n                print(\n                    \"Note that quantized cols will be turned into the mid point of \"\n                    \"the corresponding bin\"\n                )\n            for k, v in self.quantizer.inversed_bins.items():\n                decoded[k] = decoded[k].map(v)\n        # continuous_cols back to non-standarised\n        try:\n            decoded[self.standardize_cols] = self.scaler.inverse_transform(\n                decoded[self.standardize_cols]\n            )\n        except Exception:  # KeyError:\n            pass\n\n    if \"cls_token\" in decoded.columns:\n        decoded.drop(\"cls_token\", axis=1, inplace=True)\n\n    return decoded\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor.fit_transform","title":"fit_transform","text":"<pre><code>fit_transform(df)\n</code></pre> <p>Combines <code>fit</code> and <code>transform</code></p> <p>Parameters:</p> Name Type Description Default <code>df</code> <code>DataFrame</code> <p>Input pandas dataframe</p> required <p>Returns:</p> Type Description <code>ndarray</code> <p>transformed input dataframe</p> Source code in <code>pytorch_widedeep/preprocessing/tab_preprocessor.py</code> <pre><code>def fit_transform(self, df: pd.DataFrame) -&gt; np.ndarray:\n    \"\"\"Combines `fit` and `transform`\n\n    Parameters\n    ----------\n    df: pd.DataFrame\n        Input pandas dataframe\n\n    Returns\n    -------\n    np.ndarray\n        transformed input dataframe\n    \"\"\"\n    return self.fit(df).transform(df)\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.tab_preprocessor.Quantizer","title":"Quantizer","text":"<p>Helper class to perform the quantization of continuous columns. It is included in this docs for completion, since depending on the value of the parameter <code>'quantization_setup'</code> of the <code>TabPreprocessor</code> class, that class might have an attribute of type <code>Quantizer</code>. However, this class is designed to always run internally within the <code>TabPreprocessor</code> class.</p> <p>Parameters:</p> Name Type Description Default <code>quantization_setup</code> <code>Dict[str, Union[int, List[float]]]</code> <p>Dictionary where the keys are the column names to quantize and the values are the either integers indicating the number of bins or a list of scalars indicating the bin edges.</p> required Source code in <code>pytorch_widedeep/preprocessing/tab_preprocessor.py</code> <pre><code>class Quantizer:\n    \"\"\"Helper class to perform the quantization of continuous columns. It is\n    included in this docs for completion, since depending on the value of the\n    parameter `'quantization_setup'` of the `TabPreprocessor` class, that\n    class might have an attribute of type `Quantizer`. However, this class is\n    designed to always run internally within the `TabPreprocessor` class.\n\n    Parameters\n    ----------\n    quantization_setup: Dict, default = None\n        Dictionary where the keys are the column names to quantize and the\n        values are the either integers indicating the number of bins or a\n        list of scalars indicating the bin edges.\n    \"\"\"\n\n    def __init__(\n        self,\n        quantization_setup: Dict[str, Union[int, List[float]]],\n        **kwargs,\n    ):\n        self.quantization_setup = quantization_setup\n        self.quant_args = kwargs\n\n        self.is_fitted = False\n\n    def fit(self, df: pd.DataFrame) -&gt; \"Quantizer\":\n        self.bins: Dict[str, List[float]] = {}\n        for col, bins in self.quantization_setup.items():\n            _, self.bins[col] = pd.cut(\n                df[col], bins, retbins=True, labels=False, **self.quant_args\n            )\n\n        self.inversed_bins: Dict[str, Dict[int, float]] = {}\n        for col, bins in self.bins.items():\n            self.inversed_bins[col] = {\n                k: v\n                for k, v in list(\n                    zip(\n                        range(1, len(bins) + 1),\n                        [(a + b) / 2.0 for a, b in zip(bins, bins[1:])],\n                    )\n                )\n            }\n            # 0\n            self.inversed_bins[col][0] = np.nan\n\n        self.is_fitted = True\n\n        return self\n\n    def transform(self, df: pd.DataFrame) -&gt; pd.DataFrame:\n        check_is_fitted(self, condition=self.is_fitted)\n\n        dfc = df.copy()\n        for col, bins in self.bins.items():\n            dfc[col] = pd.cut(dfc[col], bins, labels=False, **self.quant_args)\n            # 0 will be left for numbers outside the bins, i.e. smaller than\n            # the smaller boundary or larger than the largest boundary\n            dfc[col] = dfc[col] + 1\n            dfc[col] = dfc[col].fillna(0)\n            dfc[col] = dfc[col].astype(int)\n\n        return dfc\n\n    def fit_transform(self, df: pd.DataFrame) -&gt; pd.DataFrame:\n        return self.fit(df).transform(df)\n\n    def __repr__(self) -&gt; str:\n        return f\"Quantizer(quantization_setup={self.quantization_setup})\"\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor","title":"TextPreprocessor","text":"<p>               Bases: <code>BasePreprocessor</code></p> <p>Preprocessor to prepare the <code>deeptext</code> input dataset</p> <p>Parameters:</p> Name Type Description Default <code>text_col</code> <code>str</code> <p>column in the input dataframe containing the texts</p> required <code>max_vocab</code> <code>int</code> <p>Maximum number of tokens in the vocabulary</p> <code>30000</code> <code>min_freq</code> <code>int</code> <p>Minimum frequency for a token to be part of the vocabulary</p> <code>5</code> <code>maxlen</code> <code>int</code> <p>Maximum length of the tokenized sequences</p> <code>80</code> <code>pad_first</code> <code>bool</code> <p>Indicates whether the padding index will be added at the beginning or the end of the sequences</p> <code>True</code> <code>pad_idx</code> <code>int</code> <p>padding index. Fastai's Tokenizer leaves 0 for the 'unknown' token.</p> <code>1</code> <code>already_processed</code> <code>Optional[bool]</code> <p>Boolean indicating if the sequence of elements is already processed or prepared. If this is the case, this Preprocessor will simply tokenize and pad the sequence. </p> <pre><code>Param aliases: `not_text`. &lt;br/&gt;\n</code></pre> <p>This parameter is thought for those cases where the input sequences are already fully processed or are directly not text (e.g. IDs)</p> <code>False</code> <code>word_vectors_path</code> <code>Optional[str]</code> <p>Path to the pretrained word vectors</p> <code>None</code> <code>n_cpus</code> <code>Optional[int]</code> <p>number of CPUs to used during the tokenization process</p> <code>None</code> <code>verbose</code> <code>int</code> <p>Enable verbose output.</p> <code>1</code> <p>Attributes:</p> Name Type Description <code>vocab</code> <code>Vocab</code> <p>an instance of <code>pytorch_widedeep.utils.fastai_transforms.Vocab</code></p> <code>embedding_matrix</code> <code>ndarray</code> <p>Array with the pretrained embeddings</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import pandas as pd\n&gt;&gt;&gt; from pytorch_widedeep.preprocessing import TextPreprocessor\n&gt;&gt;&gt; df_train = pd.DataFrame({'text_column': [\"life is like a box of chocolates\",\n... \"You never know what you're gonna get\"]})\n&gt;&gt;&gt; text_preprocessor = TextPreprocessor(text_col='text_column', max_vocab=25, min_freq=1, maxlen=10)\n&gt;&gt;&gt; text_preprocessor.fit_transform(df_train)\nThe vocabulary contains 24 tokens\narray([[ 1,  1,  1,  1, 10, 11, 12, 13, 14, 15],\n       [ 5,  9, 16, 17, 18,  9, 19, 20, 21, 22]], dtype=int32)\n&gt;&gt;&gt; df_te = pd.DataFrame({'text_column': ['you never know what is in the box']})\n&gt;&gt;&gt; text_preprocessor.transform(df_te)\narray([[ 1,  1,  9, 16, 17, 18, 11,  0,  0, 13]], dtype=int32)\n</code></pre> Source code in <code>pytorch_widedeep/preprocessing/text_preprocessor.py</code> <pre><code>class TextPreprocessor(BasePreprocessor):\n    r\"\"\"Preprocessor to prepare the ``deeptext`` input dataset\n\n    Parameters\n    ----------\n    text_col: str\n        column in the input dataframe containing the texts\n    max_vocab: int, default=30000\n        Maximum number of tokens in the vocabulary\n    min_freq: int, default=5\n        Minimum frequency for a token to be part of the vocabulary\n    maxlen: int, default=80\n        Maximum length of the tokenized sequences\n    pad_first: bool,  default = True\n        Indicates whether the padding index will be added at the beginning or the\n        end of the sequences\n    pad_idx: int, default = 1\n        padding index. Fastai's Tokenizer leaves 0 for the 'unknown' token.\n    already_processed: bool, Optional, default = False\n        Boolean indicating if the sequence of elements is already processed or\n        prepared. If this is the case, this Preprocessor will simply tokenize\n        and pad the sequence. &lt;br/&gt;\n\n            Param aliases: `not_text`. &lt;br/&gt;\n\n        This parameter is thought for those cases where the input sequences\n        are already fully processed or are directly not text (e.g. IDs)\n    word_vectors_path: str, Optional\n        Path to the pretrained word vectors\n    n_cpus: int, Optional, default = None\n        number of CPUs to used during the tokenization process\n    verbose: int, default 1\n        Enable verbose output.\n\n    Attributes\n    ----------\n    vocab: Vocab\n        an instance of `pytorch_widedeep.utils.fastai_transforms.Vocab`\n    embedding_matrix: np.ndarray\n        Array with the pretrained embeddings\n\n    Examples\n    ---------\n    &gt;&gt;&gt; import pandas as pd\n    &gt;&gt;&gt; from pytorch_widedeep.preprocessing import TextPreprocessor\n    &gt;&gt;&gt; df_train = pd.DataFrame({'text_column': [\"life is like a box of chocolates\",\n    ... \"You never know what you're gonna get\"]})\n    &gt;&gt;&gt; text_preprocessor = TextPreprocessor(text_col='text_column', max_vocab=25, min_freq=1, maxlen=10)\n    &gt;&gt;&gt; text_preprocessor.fit_transform(df_train)\n    The vocabulary contains 24 tokens\n    array([[ 1,  1,  1,  1, 10, 11, 12, 13, 14, 15],\n           [ 5,  9, 16, 17, 18,  9, 19, 20, 21, 22]], dtype=int32)\n    &gt;&gt;&gt; df_te = pd.DataFrame({'text_column': ['you never know what is in the box']})\n    &gt;&gt;&gt; text_preprocessor.transform(df_te)\n    array([[ 1,  1,  9, 16, 17, 18, 11,  0,  0, 13]], dtype=int32)\n    \"\"\"\n\n    @alias(\"already_processed\", [\"not_text\"])\n    def __init__(\n        self,\n        text_col: str,\n        max_vocab: int = 30000,\n        min_freq: int = 5,\n        maxlen: int = 80,\n        pad_first: bool = True,\n        pad_idx: int = 1,\n        already_processed: Optional[bool] = False,\n        word_vectors_path: Optional[str] = None,\n        n_cpus: Optional[int] = None,\n        verbose: int = 1,\n    ):\n        super(TextPreprocessor, self).__init__()\n\n        self.text_col = text_col\n        self.max_vocab = max_vocab\n        self.min_freq = min_freq\n        self.maxlen = maxlen\n        self.pad_first = pad_first\n        self.pad_idx = pad_idx\n        self.already_processed = already_processed\n        self.word_vectors_path = word_vectors_path\n        self.verbose = verbose\n        self.n_cpus = n_cpus if n_cpus is not None else os.cpu_count()\n\n        self.is_fitted = False\n\n    def fit(self, df: pd.DataFrame) -&gt; BasePreprocessor:\n        \"\"\"Builds the vocabulary\n\n        Parameters\n        ----------\n        df: pd.DataFrame\n            Input pandas dataframe\n\n        Returns\n        -------\n        TextPreprocessor\n            `TextPreprocessor` fitted object\n        \"\"\"\n        texts = self._read_texts(df)\n\n        tokens = get_texts(texts, self.already_processed, self.n_cpus)\n\n        self.vocab: TVocab = Vocab(\n            max_vocab=self.max_vocab,\n            min_freq=self.min_freq,\n            pad_idx=self.pad_idx,\n        ).fit(\n            tokens,\n        )\n\n        if self.verbose:\n            print(\"The vocabulary contains {} tokens\".format(len(self.vocab.stoi)))\n        if self.word_vectors_path is not None:\n            self.embedding_matrix = build_embeddings_matrix(\n                self.vocab, self.word_vectors_path, self.min_freq\n            )\n\n        self.is_fitted = True\n\n        return self\n\n    def transform(self, df: pd.DataFrame) -&gt; np.ndarray:\n        \"\"\"Returns the padded, _'numericalised'_ sequences\n\n        Parameters\n        ----------\n        df: pd.DataFrame\n            Input pandas dataframe\n\n        Returns\n        -------\n        np.ndarray\n            Padded, _'numericalised'_ sequences\n        \"\"\"\n        check_is_fitted(self, attributes=[\"vocab\"])\n        texts = self._read_texts(df)\n        tokens = get_texts(texts, self.already_processed, self.n_cpus)\n        return self._pad_sequences(tokens)\n\n    def transform_sample(self, text: str) -&gt; np.ndarray:\n        \"\"\"Returns the padded, _'numericalised'_ sequence\n\n        Parameters\n        ----------\n        text: str\n            text to be tokenized and padded\n\n        Returns\n        -------\n        np.ndarray\n            Padded, _'numericalised'_ sequence\n        \"\"\"\n        check_is_fitted(self, attributes=[\"vocab\"])\n        tokens = get_texts([text], self.already_processed, self.n_cpus)\n        return self._pad_sequences(tokens)[0]\n\n    def fit_transform(self, df: pd.DataFrame) -&gt; np.ndarray:\n        \"\"\"Combines `fit` and `transform`\n\n        Parameters\n        ----------\n        df: pd.DataFrame\n            Input pandas dataframe\n\n        Returns\n        -------\n        np.ndarray\n            Padded, _'numericalised'_ sequences\n        \"\"\"\n        return self.fit(df).transform(df)\n\n    def inverse_transform(self, padded_seq: np.ndarray) -&gt; pd.DataFrame:\n        \"\"\"Returns the original text plus the added 'special' tokens\n\n        Parameters\n        ----------\n        padded_seq: np.ndarray\n            array with the output of the `transform` method\n\n        Returns\n        -------\n        pd.DataFrame\n            Pandas dataframe with the original text plus the added 'special' tokens\n        \"\"\"\n        texts = [self.vocab.inverse_transform(num) for num in padded_seq]\n        return pd.DataFrame({self.text_col: texts})\n\n    def _pad_sequences(self, tokens: List[List[str]]) -&gt; np.ndarray:\n        sequences = [self.vocab.transform(t) for t in tokens]\n        padded_seq = np.array(\n            [\n                pad_sequences(\n                    s,\n                    maxlen=self.maxlen,\n                    pad_first=self.pad_first,\n                    pad_idx=self.pad_idx,\n                )\n                for s in sequences\n            ]\n        )\n        return padded_seq\n\n    def _read_texts(\n        self, df: pd.DataFrame, root_dir: Optional[str] = None\n    ) -&gt; List[str]:\n        if root_dir is not None:\n            if not os.path.exists(root_dir):\n                raise ValueError(\n                    \"root_dir does not exist. Please create it before fitting the preprocessor\"\n                )\n            texts_fnames = df[self.text_col].tolist()\n            texts: List[str] = []\n            for texts_fname in texts_fnames:\n                with open(os.path.join(root_dir, texts_fname), \"r\") as f:\n                    texts.append(f.read().replace(\"\\n\", \"\"))\n        else:\n            texts = df[self.text_col].tolist()\n\n        return texts\n\n    def _load_vocab(self, vocab: TVocab) -&gt; None:\n        self.vocab = vocab\n\n    def __repr__(self) -&gt; str:\n        list_of_params: List[str] = [\"text_col={text_col}\"]\n        list_of_params.append(\"max_vocab={max_vocab}\")\n        list_of_params.append(\"min_freq={min_freq}\")\n        list_of_params.append(\"maxlen={maxlen}\")\n        list_of_params.append(\"pad_first={pad_first}\")\n        list_of_params.append(\"pad_idx={pad_idx}\")\n        list_of_params.append(\"already_processed={already_processed}\")\n        if self.word_vectors_path is not None:\n            list_of_params.append(\"word_vectors_path={word_vectors_path}\")\n        if self.n_cpus is not None:\n            list_of_params.append(\"n_cpus={n_cpus}\")\n        if self.verbose is not None:\n            list_of_params.append(\"verbose={verbose}\")\n        all_params = \", \".join(list_of_params)\n        return f\"TextPreprocessor({all_params.format(**self.__dict__)})\"\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor.fit","title":"fit","text":"<pre><code>fit(df)\n</code></pre> <p>Builds the vocabulary</p> <p>Parameters:</p> Name Type Description Default <code>df</code> <code>DataFrame</code> <p>Input pandas dataframe</p> required <p>Returns:</p> Type Description <code>TextPreprocessor</code> <p><code>TextPreprocessor</code> fitted object</p> Source code in <code>pytorch_widedeep/preprocessing/text_preprocessor.py</code> <pre><code>def fit(self, df: pd.DataFrame) -&gt; BasePreprocessor:\n    \"\"\"Builds the vocabulary\n\n    Parameters\n    ----------\n    df: pd.DataFrame\n        Input pandas dataframe\n\n    Returns\n    -------\n    TextPreprocessor\n        `TextPreprocessor` fitted object\n    \"\"\"\n    texts = self._read_texts(df)\n\n    tokens = get_texts(texts, self.already_processed, self.n_cpus)\n\n    self.vocab: TVocab = Vocab(\n        max_vocab=self.max_vocab,\n        min_freq=self.min_freq,\n        pad_idx=self.pad_idx,\n    ).fit(\n        tokens,\n    )\n\n    if self.verbose:\n        print(\"The vocabulary contains {} tokens\".format(len(self.vocab.stoi)))\n    if self.word_vectors_path is not None:\n        self.embedding_matrix = build_embeddings_matrix(\n            self.vocab, self.word_vectors_path, self.min_freq\n        )\n\n    self.is_fitted = True\n\n    return self\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor.transform","title":"transform","text":"<pre><code>transform(df)\n</code></pre> <p>Returns the padded, 'numericalised' sequences</p> <p>Parameters:</p> Name Type Description Default <code>df</code> <code>DataFrame</code> <p>Input pandas dataframe</p> required <p>Returns:</p> Type Description <code>ndarray</code> <p>Padded, 'numericalised' sequences</p> Source code in <code>pytorch_widedeep/preprocessing/text_preprocessor.py</code> <pre><code>def transform(self, df: pd.DataFrame) -&gt; np.ndarray:\n    \"\"\"Returns the padded, _'numericalised'_ sequences\n\n    Parameters\n    ----------\n    df: pd.DataFrame\n        Input pandas dataframe\n\n    Returns\n    -------\n    np.ndarray\n        Padded, _'numericalised'_ sequences\n    \"\"\"\n    check_is_fitted(self, attributes=[\"vocab\"])\n    texts = self._read_texts(df)\n    tokens = get_texts(texts, self.already_processed, self.n_cpus)\n    return self._pad_sequences(tokens)\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor.transform_sample","title":"transform_sample","text":"<pre><code>transform_sample(text)\n</code></pre> <p>Returns the padded, 'numericalised' sequence</p> <p>Parameters:</p> Name Type Description Default <code>text</code> <code>str</code> <p>text to be tokenized and padded</p> required <p>Returns:</p> Type Description <code>ndarray</code> <p>Padded, 'numericalised' sequence</p> Source code in <code>pytorch_widedeep/preprocessing/text_preprocessor.py</code> <pre><code>def transform_sample(self, text: str) -&gt; np.ndarray:\n    \"\"\"Returns the padded, _'numericalised'_ sequence\n\n    Parameters\n    ----------\n    text: str\n        text to be tokenized and padded\n\n    Returns\n    -------\n    np.ndarray\n        Padded, _'numericalised'_ sequence\n    \"\"\"\n    check_is_fitted(self, attributes=[\"vocab\"])\n    tokens = get_texts([text], self.already_processed, self.n_cpus)\n    return self._pad_sequences(tokens)[0]\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor.fit_transform","title":"fit_transform","text":"<pre><code>fit_transform(df)\n</code></pre> <p>Combines <code>fit</code> and <code>transform</code></p> <p>Parameters:</p> Name Type Description Default <code>df</code> <code>DataFrame</code> <p>Input pandas dataframe</p> required <p>Returns:</p> Type Description <code>ndarray</code> <p>Padded, 'numericalised' sequences</p> Source code in <code>pytorch_widedeep/preprocessing/text_preprocessor.py</code> <pre><code>def fit_transform(self, df: pd.DataFrame) -&gt; np.ndarray:\n    \"\"\"Combines `fit` and `transform`\n\n    Parameters\n    ----------\n    df: pd.DataFrame\n        Input pandas dataframe\n\n    Returns\n    -------\n    np.ndarray\n        Padded, _'numericalised'_ sequences\n    \"\"\"\n    return self.fit(df).transform(df)\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor.inverse_transform","title":"inverse_transform","text":"<pre><code>inverse_transform(padded_seq)\n</code></pre> <p>Returns the original text plus the added 'special' tokens</p> <p>Parameters:</p> Name Type Description Default <code>padded_seq</code> <code>ndarray</code> <p>array with the output of the <code>transform</code> method</p> required <p>Returns:</p> Type Description <code>DataFrame</code> <p>Pandas dataframe with the original text plus the added 'special' tokens</p> Source code in <code>pytorch_widedeep/preprocessing/text_preprocessor.py</code> <pre><code>def inverse_transform(self, padded_seq: np.ndarray) -&gt; pd.DataFrame:\n    \"\"\"Returns the original text plus the added 'special' tokens\n\n    Parameters\n    ----------\n    padded_seq: np.ndarray\n        array with the output of the `transform` method\n\n    Returns\n    -------\n    pd.DataFrame\n        Pandas dataframe with the original text plus the added 'special' tokens\n    \"\"\"\n    texts = [self.vocab.inverse_transform(num) for num in padded_seq]\n    return pd.DataFrame({self.text_col: texts})\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor","title":"HFPreprocessor","text":"<p>               Bases: <code>BasePreprocessor</code></p> <p>Text processor to prepare the <code>deeptext</code> input dataset that is a wrapper around HuggingFace's tokenizers.</p> <p>Following the main phylosophy of the <code>pytorch-widedeep</code> library, this class is designed to be as flexible as possible. Therefore, it is coded so that the user can use it as one would use any HuggingFace tokenizers, or following the API call 'protocol' of the rest of the library.</p> <p>Parameters:</p> Name Type Description Default <code>model_name</code> <code>str</code> <p>The model name from the transformers library e.g. 'bert-base-uncased'. Currently supported models are those from the families: BERT, RoBERTa, DistilBERT, ALBERT and ELECTRA.</p> required <code>use_fast_tokenizer</code> <code>bool</code> <p>Whether to use the fast tokenizer from HuggingFace or not</p> <code>False</code> <code>text_col</code> <code>Optional[str]</code> <p>The column in the input dataframe containing the text data. If this tokenizer is used via the <code>fit</code> and <code>transform</code> methods, this argument is mandatory. If the tokenizer is used via the <code>encode</code> method, this argument is not needed since the input text is passed directly to the <code>encode</code> method.</p> <code>None</code> <code>num_workers</code> <code>Optional[int]</code> <p>Number of workers to use when preprocessing the text data. If not None, and <code>use_fast_tokenizer</code> is False, the text data will be preprocessed in parallel using the number of workers specified. If <code>use_fast_tokenizer</code> is True, this argument is ignored.</p> <code>None</code> <code>preprocessing_rules</code> <code>Optional[List[Callable[[str], str]]]</code> <p>A list of functions to be applied to the text data before encoding. This can be useful to clean the text data before encoding. For example, removing html tags, special characters, etc.</p> <code>None</code> <code>tokenizer_params</code> <code>Optional[Dict[str, Any]]</code> <p>Additional parameters to be passed to the HuggingFace's <code>PreTrainedTokenizer</code>. Parameters to the <code>PreTrainedTokenizer</code> can also be passed via the <code>**kwargs</code> argument</p> <code>None</code> <code>encode_params</code> <code>Optional[Dict[str, Any]]</code> <p>Additional parameters to be passed to the <code>batch_encode_plus</code> method of the HuggingFace's <code>PreTrainedTokenizer</code>. If the <code>fit</code> and <code>transform</code> methods are used, the <code>encode_params</code> dict parameter is mandatory. If the <code>encode</code> method is used, this parameter is not needed since the input text is passed directly to the <code>encode</code> method.</p> <code>None</code> <code>**kwargs</code> <p>Additional kwargs to be passed to the model, in particular to the <code>PreTrainedTokenizer</code> class.</p> <code>{}</code> <p>Attributes:</p> Name Type Description <code>is_fitted</code> <code>bool</code> <p>Boolean indicating if the preprocessor has been fitted. This is a HuggingFacea tokenizer, so it is always considered fitted and this attribute is manually set to True internally. This parameter exists for consistency with the rest of the library and because is needed for some functionality in the library.</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import pandas as pd\n&gt;&gt;&gt; from pytorch_widedeep.preprocessing import HFPreprocessor\n&gt;&gt;&gt; df = pd.DataFrame({\"text\": [\"this is the first text\", \"this is the second text\"]})\n&gt;&gt;&gt; hf_processor_1 = HFPreprocessor(model_name=\"bert-base-uncased\", text_col=\"text\")\n&gt;&gt;&gt; X_text_1 = hf_processor_1.fit_transform(df)\n&gt;&gt;&gt; texts = [\"this is a new text\", \"this is another text\"]\n&gt;&gt;&gt; hf_processor_2 = HFPreprocessor(model_name=\"bert-base-uncased\")\n&gt;&gt;&gt; X_text_2 = hf_processor_2.encode(texts, max_length=10, padding=\"max_length\", truncation=True)\n</code></pre> Source code in <code>pytorch_widedeep/preprocessing/hf_preprocessor.py</code> <pre><code>class HFPreprocessor(BasePreprocessor):\n    \"\"\"Text processor to prepare the ``deeptext`` input dataset that is a\n    wrapper around HuggingFace's tokenizers.\n\n    Following the main phylosophy of the `pytorch-widedeep` library, this\n    class is designed to be as flexible as possible. Therefore, it is coded\n    so that the user can use it as one would use any HuggingFace tokenizers,\n    or following the API call 'protocol' of the rest of the library.\n\n    Parameters\n    ----------\n    model_name: str\n        The model name from the transformers library e.g. _'bert-base-uncased'_.\n        Currently supported models are those from the families: BERT, RoBERTa,\n        DistilBERT, ALBERT and ELECTRA.\n    use_fast_tokenizer: bool, default = False\n        Whether to use the fast tokenizer from HuggingFace or not\n    text_col: Optional[str], default = None\n        The column in the input dataframe containing the text data. If this\n        tokenizer is used via the `fit` and `transform` methods, this\n        argument is mandatory. If the tokenizer is used via the `encode`\n        method, this argument is not needed since the input text is passed\n        directly to the `encode` method.\n    num_workers: Optional[int], default = None\n        Number of workers to use when preprocessing the text data. If not\n        None, and `use_fast_tokenizer` is False, the text data will be\n        preprocessed in parallel using the number of workers specified. If\n        `use_fast_tokenizer` is True, this argument is ignored.\n    preprocessing_rules: Optional[List[Callable[[str], str]]], default = None\n        A list of functions to be applied to the text data before encoding.\n        This can be useful to clean the text data before encoding. For\n        example, removing html tags, special characters, etc.\n    tokenizer_params: Optional[Dict[str, Any]], default = None\n        Additional parameters to be passed to the HuggingFace's\n        `PreTrainedTokenizer`. Parameters to the `PreTrainedTokenizer`\n        can also be passed via the `**kwargs` argument\n    encode_params: Optional[Dict[str, Any]], default = None\n        Additional parameters to be passed to the `batch_encode_plus` method\n        of the HuggingFace's `PreTrainedTokenizer`. If the `fit` and `transform`\n        methods are used, the `encode_params` dict parameter is mandatory. If\n        the `encode` method is used, this parameter is not needed since the\n        input text is passed directly to the `encode` method.\n    **kwargs\n        Additional kwargs to be passed to the model, in particular to the\n        `PreTrainedTokenizer` class.\n\n    Attributes\n    ----------\n    is_fitted: bool\n        Boolean indicating if the preprocessor has been fitted. This is a\n        HuggingFacea tokenizer, so it is always considered fitted and this\n        attribute is manually set to True internally. This parameter exists\n        for consistency with the rest of the library and because is needed\n        for some functionality in the library.\n\n    Examples\n    --------\n    &gt;&gt;&gt; import pandas as pd\n    &gt;&gt;&gt; from pytorch_widedeep.preprocessing import HFPreprocessor\n    &gt;&gt;&gt; df = pd.DataFrame({\"text\": [\"this is the first text\", \"this is the second text\"]})\n    &gt;&gt;&gt; hf_processor_1 = HFPreprocessor(model_name=\"bert-base-uncased\", text_col=\"text\")\n    &gt;&gt;&gt; X_text_1 = hf_processor_1.fit_transform(df)\n    &gt;&gt;&gt; texts = [\"this is a new text\", \"this is another text\"]\n    &gt;&gt;&gt; hf_processor_2 = HFPreprocessor(model_name=\"bert-base-uncased\")\n    &gt;&gt;&gt; X_text_2 = hf_processor_2.encode(texts, max_length=10, padding=\"max_length\", truncation=True)\n    \"\"\"\n\n    def __init__(\n        self,\n        model_name: str,\n        *,\n        use_fast_tokenizer: bool = False,\n        text_col: Optional[str] = None,\n        root_dir: Optional[str] = None,\n        num_workers: Optional[int] = None,\n        preprocessing_rules: Optional[List[Callable[[str], str]]] = None,\n        tokenizer_params: Optional[Dict[str, Any]] = None,\n        encode_params: Optional[Dict[str, Any]] = None,\n        **kwargs,\n    ):\n        self.model_name = model_name\n        self.use_fast_tokenizer = use_fast_tokenizer\n        self.text_col = text_col\n        self.root_dir = root_dir\n        self.num_workers = num_workers\n        self.preprocessing_rules = preprocessing_rules\n        self.tokenizer_params = tokenizer_params if tokenizer_params is not None else {}\n        self.encode_params = encode_params if encode_params is not None else {}\n\n        self._multiprocessing = (\n            num_workers is not None and num_workers &gt; 1 and not use_fast_tokenizer\n        )\n\n        if kwargs:\n            self.tokenizer_params.update(kwargs)\n\n        self.tokenizer = get_tokenizer(\n            model_name=self.model_name,\n            use_fast_tokenizer=self.use_fast_tokenizer,\n            **self.tokenizer_params,\n        )\n\n        # A HuggingFace tokenizer is already trained, since we need this\n        # attribute elsewhere in the library, we simply set it to True\n        self.is_fitted = True\n\n    def encode(self, texts: List[str], **kwargs) -&gt; npt.NDArray[np.int64]:\n        \"\"\"\n        Encodes a list of texts. The method is a wrapper around the\n        `batch_encode_plus` method of the HuggingFace's tokenizer.\n\n        if 'use_fast_tokenizer' is True, the method will use the `batch_encode_plus`\n\n        Parameters\n        ----------\n        texts: List[str]\n            List of texts to be encoded\n        **kwargs\n            Additional parameters to be passed to the `batch_encode_plus` method\n            of the HuggingFace's tokenizer. If the 'encode_params' dict was passed\n            when instantiating the class, that dictionaly will be updated with\n            the kwargs passed here.\n\n        Returns\n        -------\n        np.array\n            The encoded texts\n        \"\"\"\n        if kwargs:\n            self.encode_params.update(kwargs)\n\n        if self.preprocessing_rules:\n            if self._multiprocessing:\n                texts = self._process_text_parallel(texts)\n            else:\n                texts = [self._preprocess_text(text) for text in texts]\n\n        if self._multiprocessing:\n            input_ids = self._encode_paralell(texts, **self.encode_params)\n        else:\n            encoded_texts = self.tokenizer.batch_encode_plus(\n                texts,\n                **self.encode_params,\n            )\n            input_ids = encoded_texts.get(\"input_ids\")\n\n        self.is_fitted = True\n\n        try:\n            output = np.array(input_ids)\n        except ValueError:\n            warnings.warn(\n                \"Padding and Truncating parameters were not passed and all input arrays \"\n                \"do not have the same shape. Padding to the longest sequence. \"\n                \"Padding will be done with the index of the pad token for the model\",\n                UserWarning,\n            )\n            max_len = max([len(ids) for ids in input_ids])\n            output = np.array(\n                [\n                    np.pad(ids, (self.tokenizer.pad_token_id, max_len - len(ids)))\n                    for ids in input_ids\n                ]\n            )\n\n        return output\n\n    def decode(\n        self, input_ids: npt.NDArray[np.int64], skip_special_tokens: bool\n    ) -&gt; List[str]:\n        \"\"\"\n        Decodes a list of input_ids. The method is a wrapper around the\n        `convert_ids_to_tokens` and `convert_tokens_to_string` methods of the\n        HuggingFace's tokenizer.\n\n        Parameters\n        ----------\n        input_ids: npt.NDArray[np.int64]\n            The input_ids to be decoded\n        skip_special_tokens: bool\n            Whether to skip the special tokens or not\n\n        Returns\n        -------\n        List[str]\n            The decoded texts\n        \"\"\"\n        texts = [\n            self.tokenizer.convert_tokens_to_string(\n                self.tokenizer.convert_ids_to_tokens(input_ids[i], skip_special_tokens)\n            )\n            for i in range(input_ids.shape[0])\n        ]\n        return texts\n\n    def fit(self, df: pd.DataFrame) -&gt; \"HFPreprocessor\":\n        \"\"\"\n        This method is included for consistency with the rest of the library\n        in general and with the `BasePreprocessor` in particular. HuggingFace's\n        tokenizers and models are already trained. Therefore, the 'fit' method\n        here does nothing other than checking that the 'text_col' parameter is\n        not `None`.\n\n        Parameters\n        ----------\n        df: pd.DataFrame\n            The dataframe containing the text data in the column specified by\n            the 'text_col' parameter\n        \"\"\"\n        if self.text_col is None:\n            raise ValueError(\n                \"'text_col' is None. Please specify the column name containing the text data\"\n                \" if you want to use the 'fit' method\"\n            )\n        return self\n\n    def transform(self, df: pd.DataFrame) -&gt; npt.NDArray[np.int64]:\n        \"\"\"\n        Encodes the text data in the input dataframe. This method simply\n        calls the `encode` method under the hood. Similar to the `fit` method,\n        this method is included for consistency with the rest of the library\n        in general and with the `BasePreprocessor` in particular.\n\n        Parameters\n        ----------\n        df: pd.DataFrame\n            The dataframe containing the text data in the column specified by\n            the 'text_col' parameter\n\n        Returns\n        -------\n        np.array\n            The encoded texts\n        \"\"\"\n        if self.text_col is None:\n            raise ValueError(\n                \"'text_col' is None. Please specify the column name containing the text data\"\n                \" if you want to use the 'fit' method\"\n            )\n\n        texts = self._read_texts(df, self.root_dir)\n\n        return self.encode(texts)\n\n    def transform_sample(self, text: str) -&gt; npt.NDArray[np.int64]:\n        \"\"\"\n        Encodes a single text sample.\n\n        Parameters\n        ----------\n        text: str\n            The text sample to be encoded\n\n        Returns\n        -------\n        np.array\n            The encoded text\n        \"\"\"\n\n        if not self.is_fitted:\n            raise ValueError(\n                \"The `encode` (or `fit`) method must be called before calling `transform_sample`\"\n            )\n        return self.encode([text])[0]\n\n    def fit_transform(self, df: pd.DataFrame) -&gt; npt.NDArray[np.int64]:\n        \"\"\"\n        Encodes the text data in the input dataframe.\n\n        Parameters\n        ----------\n        df: pd.DataFrame\n            The dataframe containing the text data in the column specified by\n            the 'text_col' parameter\n\n        Returns\n        -------\n        np.array\n            The encoded texts\n        \"\"\"\n        return self.fit(df).transform(df)\n\n    def inverse_transform(\n        self, input_ids: npt.NDArray[np.int64], skip_special_tokens: bool\n    ) -&gt; List[str]:\n        \"\"\"\n        Decodes a list of input_ids. The method simply calls the `decode` method\n        under the hood.\n\n        Parameters\n        ----------\n        input_ids: npt.NDArray[np.int64]\n            The input_ids to be decoded\n        skip_special_tokens: bool\n            Whether to skip the special tokens or not\n\n        Returns\n        -------\n        List[str]\n            The decoded texts\n        \"\"\"\n        return self.decode(input_ids, skip_special_tokens)\n\n    def _process_text_parallel(self, texts: List[str]) -&gt; List[str]:\n        num_processes = (\n            self.num_workers if self.num_workers is not None else os.cpu_count()\n        )\n        with ProcessPoolExecutor(max_workers=num_processes) as executor:\n            processed_texts = list(executor.map(self._preprocess_text, texts))\n        return processed_texts\n\n    def _preprocess_text(self, text: str) -&gt; str:\n        for rule in self.preprocessing_rules:\n            text = rule(text)\n        return text\n\n    def _encode_paralell(self, texts: List[str], **kwargs) -&gt; List[List[int]]:\n        num_processes = (\n            self.num_workers if self.num_workers is not None else os.cpu_count()\n        )\n        ptokenizer = partial(self.tokenizer.encode, **kwargs)\n        with ProcessPoolExecutor(max_workers=num_processes) as executor:\n            encoded_texts = list(executor.map(ptokenizer, texts))\n        return encoded_texts\n\n    def _read_texts(\n        self, df: pd.DataFrame, root_dir: Optional[str] = None\n    ) -&gt; List[str]:\n        if root_dir is not None:\n            if not os.path.exists(root_dir):\n                raise ValueError(\n                    \"root_dir does not exist. Please create it before fitting the preprocessor\"\n                )\n            texts_fnames = df[self.text_col].tolist()\n            texts: List[str] = []\n            for texts_fname in texts_fnames:\n                with open(os.path.join(root_dir, texts_fname), \"r\") as f:\n                    texts.append(f.read().replace(\"\\n\", \"\"))\n        else:\n            texts = df[self.text_col].tolist()\n\n        return texts\n\n    def __repr__(self):\n        return (\n            f\"HFPreprocessor(text_col={self.text_col}, model_name={self.model_name}, \"\n            f\"use_fast_tokenizer={self.use_fast_tokenizer}, num_workers={self.num_workers}, \"\n            f\"preprocessing_rules={self.preprocessing_rules}, tokenizer_params={self.tokenizer_params}, \"\n            f\"encode_params={self.encode_params})\"\n        )\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.encode","title":"encode","text":"<pre><code>encode(texts, **kwargs)\n</code></pre> <p>Encodes a list of texts. The method is a wrapper around the <code>batch_encode_plus</code> method of the HuggingFace's tokenizer.</p> <p>if 'use_fast_tokenizer' is True, the method will use the <code>batch_encode_plus</code></p> <p>Parameters:</p> Name Type Description Default <code>texts</code> <code>List[str]</code> <p>List of texts to be encoded</p> required <code>**kwargs</code> <p>Additional parameters to be passed to the <code>batch_encode_plus</code> method of the HuggingFace's tokenizer. If the 'encode_params' dict was passed when instantiating the class, that dictionaly will be updated with the kwargs passed here.</p> <code>{}</code> <p>Returns:</p> Type Description <code>array</code> <p>The encoded texts</p> Source code in <code>pytorch_widedeep/preprocessing/hf_preprocessor.py</code> <pre><code>def encode(self, texts: List[str], **kwargs) -&gt; npt.NDArray[np.int64]:\n    \"\"\"\n    Encodes a list of texts. The method is a wrapper around the\n    `batch_encode_plus` method of the HuggingFace's tokenizer.\n\n    if 'use_fast_tokenizer' is True, the method will use the `batch_encode_plus`\n\n    Parameters\n    ----------\n    texts: List[str]\n        List of texts to be encoded\n    **kwargs\n        Additional parameters to be passed to the `batch_encode_plus` method\n        of the HuggingFace's tokenizer. If the 'encode_params' dict was passed\n        when instantiating the class, that dictionaly will be updated with\n        the kwargs passed here.\n\n    Returns\n    -------\n    np.array\n        The encoded texts\n    \"\"\"\n    if kwargs:\n        self.encode_params.update(kwargs)\n\n    if self.preprocessing_rules:\n        if self._multiprocessing:\n            texts = self._process_text_parallel(texts)\n        else:\n            texts = [self._preprocess_text(text) for text in texts]\n\n    if self._multiprocessing:\n        input_ids = self._encode_paralell(texts, **self.encode_params)\n    else:\n        encoded_texts = self.tokenizer.batch_encode_plus(\n            texts,\n            **self.encode_params,\n        )\n        input_ids = encoded_texts.get(\"input_ids\")\n\n    self.is_fitted = True\n\n    try:\n        output = np.array(input_ids)\n    except ValueError:\n        warnings.warn(\n            \"Padding and Truncating parameters were not passed and all input arrays \"\n            \"do not have the same shape. Padding to the longest sequence. \"\n            \"Padding will be done with the index of the pad token for the model\",\n            UserWarning,\n        )\n        max_len = max([len(ids) for ids in input_ids])\n        output = np.array(\n            [\n                np.pad(ids, (self.tokenizer.pad_token_id, max_len - len(ids)))\n                for ids in input_ids\n            ]\n        )\n\n    return output\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.decode","title":"decode","text":"<pre><code>decode(input_ids, skip_special_tokens)\n</code></pre> <p>Decodes a list of input_ids. The method is a wrapper around the <code>convert_ids_to_tokens</code> and <code>convert_tokens_to_string</code> methods of the HuggingFace's tokenizer.</p> <p>Parameters:</p> Name Type Description Default <code>input_ids</code> <code>NDArray[int64]</code> <p>The input_ids to be decoded</p> required <code>skip_special_tokens</code> <code>bool</code> <p>Whether to skip the special tokens or not</p> required <p>Returns:</p> Type Description <code>List[str]</code> <p>The decoded texts</p> Source code in <code>pytorch_widedeep/preprocessing/hf_preprocessor.py</code> <pre><code>def decode(\n    self, input_ids: npt.NDArray[np.int64], skip_special_tokens: bool\n) -&gt; List[str]:\n    \"\"\"\n    Decodes a list of input_ids. The method is a wrapper around the\n    `convert_ids_to_tokens` and `convert_tokens_to_string` methods of the\n    HuggingFace's tokenizer.\n\n    Parameters\n    ----------\n    input_ids: npt.NDArray[np.int64]\n        The input_ids to be decoded\n    skip_special_tokens: bool\n        Whether to skip the special tokens or not\n\n    Returns\n    -------\n    List[str]\n        The decoded texts\n    \"\"\"\n    texts = [\n        self.tokenizer.convert_tokens_to_string(\n            self.tokenizer.convert_ids_to_tokens(input_ids[i], skip_special_tokens)\n        )\n        for i in range(input_ids.shape[0])\n    ]\n    return texts\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.fit","title":"fit","text":"<pre><code>fit(df)\n</code></pre> <p>This method is included for consistency with the rest of the library in general and with the <code>BasePreprocessor</code> in particular. HuggingFace's tokenizers and models are already trained. Therefore, the 'fit' method here does nothing other than checking that the 'text_col' parameter is not <code>None</code>.</p> <p>Parameters:</p> Name Type Description Default <code>df</code> <code>DataFrame</code> <p>The dataframe containing the text data in the column specified by the 'text_col' parameter</p> required Source code in <code>pytorch_widedeep/preprocessing/hf_preprocessor.py</code> <pre><code>def fit(self, df: pd.DataFrame) -&gt; \"HFPreprocessor\":\n    \"\"\"\n    This method is included for consistency with the rest of the library\n    in general and with the `BasePreprocessor` in particular. HuggingFace's\n    tokenizers and models are already trained. Therefore, the 'fit' method\n    here does nothing other than checking that the 'text_col' parameter is\n    not `None`.\n\n    Parameters\n    ----------\n    df: pd.DataFrame\n        The dataframe containing the text data in the column specified by\n        the 'text_col' parameter\n    \"\"\"\n    if self.text_col is None:\n        raise ValueError(\n            \"'text_col' is None. Please specify the column name containing the text data\"\n            \" if you want to use the 'fit' method\"\n        )\n    return self\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.transform","title":"transform","text":"<pre><code>transform(df)\n</code></pre> <p>Encodes the text data in the input dataframe. This method simply calls the <code>encode</code> method under the hood. Similar to the <code>fit</code> method, this method is included for consistency with the rest of the library in general and with the <code>BasePreprocessor</code> in particular.</p> <p>Parameters:</p> Name Type Description Default <code>df</code> <code>DataFrame</code> <p>The dataframe containing the text data in the column specified by the 'text_col' parameter</p> required <p>Returns:</p> Type Description <code>array</code> <p>The encoded texts</p> Source code in <code>pytorch_widedeep/preprocessing/hf_preprocessor.py</code> <pre><code>def transform(self, df: pd.DataFrame) -&gt; npt.NDArray[np.int64]:\n    \"\"\"\n    Encodes the text data in the input dataframe. This method simply\n    calls the `encode` method under the hood. Similar to the `fit` method,\n    this method is included for consistency with the rest of the library\n    in general and with the `BasePreprocessor` in particular.\n\n    Parameters\n    ----------\n    df: pd.DataFrame\n        The dataframe containing the text data in the column specified by\n        the 'text_col' parameter\n\n    Returns\n    -------\n    np.array\n        The encoded texts\n    \"\"\"\n    if self.text_col is None:\n        raise ValueError(\n            \"'text_col' is None. Please specify the column name containing the text data\"\n            \" if you want to use the 'fit' method\"\n        )\n\n    texts = self._read_texts(df, self.root_dir)\n\n    return self.encode(texts)\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.transform_sample","title":"transform_sample","text":"<pre><code>transform_sample(text)\n</code></pre> <p>Encodes a single text sample.</p> <p>Parameters:</p> Name Type Description Default <code>text</code> <code>str</code> <p>The text sample to be encoded</p> required <p>Returns:</p> Type Description <code>array</code> <p>The encoded text</p> Source code in <code>pytorch_widedeep/preprocessing/hf_preprocessor.py</code> <pre><code>def transform_sample(self, text: str) -&gt; npt.NDArray[np.int64]:\n    \"\"\"\n    Encodes a single text sample.\n\n    Parameters\n    ----------\n    text: str\n        The text sample to be encoded\n\n    Returns\n    -------\n    np.array\n        The encoded text\n    \"\"\"\n\n    if not self.is_fitted:\n        raise ValueError(\n            \"The `encode` (or `fit`) method must be called before calling `transform_sample`\"\n        )\n    return self.encode([text])[0]\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.fit_transform","title":"fit_transform","text":"<pre><code>fit_transform(df)\n</code></pre> <p>Encodes the text data in the input dataframe.</p> <p>Parameters:</p> Name Type Description Default <code>df</code> <code>DataFrame</code> <p>The dataframe containing the text data in the column specified by the 'text_col' parameter</p> required <p>Returns:</p> Type Description <code>array</code> <p>The encoded texts</p> Source code in <code>pytorch_widedeep/preprocessing/hf_preprocessor.py</code> <pre><code>def fit_transform(self, df: pd.DataFrame) -&gt; npt.NDArray[np.int64]:\n    \"\"\"\n    Encodes the text data in the input dataframe.\n\n    Parameters\n    ----------\n    df: pd.DataFrame\n        The dataframe containing the text data in the column specified by\n        the 'text_col' parameter\n\n    Returns\n    -------\n    np.array\n        The encoded texts\n    \"\"\"\n    return self.fit(df).transform(df)\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.inverse_transform","title":"inverse_transform","text":"<pre><code>inverse_transform(input_ids, skip_special_tokens)\n</code></pre> <p>Decodes a list of input_ids. The method simply calls the <code>decode</code> method under the hood.</p> <p>Parameters:</p> Name Type Description Default <code>input_ids</code> <code>NDArray[int64]</code> <p>The input_ids to be decoded</p> required <code>skip_special_tokens</code> <code>bool</code> <p>Whether to skip the special tokens or not</p> required <p>Returns:</p> Type Description <code>List[str]</code> <p>The decoded texts</p> Source code in <code>pytorch_widedeep/preprocessing/hf_preprocessor.py</code> <pre><code>def inverse_transform(\n    self, input_ids: npt.NDArray[np.int64], skip_special_tokens: bool\n) -&gt; List[str]:\n    \"\"\"\n    Decodes a list of input_ids. The method simply calls the `decode` method\n    under the hood.\n\n    Parameters\n    ----------\n    input_ids: npt.NDArray[np.int64]\n        The input_ids to be decoded\n    skip_special_tokens: bool\n        Whether to skip the special tokens or not\n\n    Returns\n    -------\n    List[str]\n        The decoded texts\n    \"\"\"\n    return self.decode(input_ids, skip_special_tokens)\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.image_preprocessor.ImagePreprocessor","title":"ImagePreprocessor","text":"<p>               Bases: <code>BasePreprocessor</code></p> <p>Preprocessor to prepare the <code>deepimage</code> input dataset.</p> <p>The Preprocessing consists simply on resizing according to their aspect ratio</p> <p>Parameters:</p> Name Type Description Default <code>img_col</code> <code>str</code> <p>name of the column with the images filenames</p> required <code>img_path</code> <code>str</code> <p>path to the dicrectory where the images are stored</p> required <code>width</code> <code>int</code> <p>width of the resulting processed image.</p> <code>224</code> <code>height</code> <code>int</code> <p>width of the resulting processed image.</p> <code>224</code> <code>verbose</code> <code>int</code> <p>Enable verbose output.</p> <code>1</code> <p>Attributes:</p> Name Type Description <code>aap</code> <code>AspectAwarePreprocessor</code> <p>an instance of <code>pytorch_widedeep.utils.image_utils.AspectAwarePreprocessor</code></p> <code>spp</code> <code>SimplePreprocessor</code> <p>an instance of <code>pytorch_widedeep.utils.image_utils.SimplePreprocessor</code></p> <code>normalise_metrics</code> <code>Dict</code> <p>Dict containing the normalisation metrics of the image dataset, i.e. mean and std for the R, G and B channels</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import pandas as pd\n&gt;&gt;&gt;\n&gt;&gt;&gt; from pytorch_widedeep.preprocessing import ImagePreprocessor\n&gt;&gt;&gt;\n&gt;&gt;&gt; path_to_image1 = 'tests/test_data_utils/images/galaxy1.png'\n&gt;&gt;&gt; path_to_image2 = 'tests/test_data_utils/images/galaxy2.png'\n&gt;&gt;&gt;\n&gt;&gt;&gt; df_train = pd.DataFrame({'images_column': [path_to_image1]})\n&gt;&gt;&gt; df_test = pd.DataFrame({'images_column': [path_to_image2]})\n&gt;&gt;&gt; img_preprocessor = ImagePreprocessor(img_col='images_column', img_path='.', verbose=0)\n&gt;&gt;&gt; resized_images = img_preprocessor.fit_transform(df_train)\n&gt;&gt;&gt; new_resized_images = img_preprocessor.transform(df_train)\n</code></pre> <p> NOTE: Normalising metrics will only be computed when the <code>fit_transform</code> method is run. Running <code>transform</code> only will not change the computed metrics and running <code>fit</code> only simply instantiates the resizing functions.</p> Source code in <code>pytorch_widedeep/preprocessing/image_preprocessor.py</code> <pre><code>class ImagePreprocessor(BasePreprocessor):\n    r\"\"\"Preprocessor to prepare the ``deepimage`` input dataset.\n\n    The Preprocessing consists simply on resizing according to their\n    aspect ratio\n\n    Parameters\n    ----------\n    img_col: str\n        name of the column with the images filenames\n    img_path: str\n        path to the dicrectory where the images are stored\n    width: int, default=224\n        width of the resulting processed image.\n    height: int, default=224\n        width of the resulting processed image.\n    verbose: int, default 1\n        Enable verbose output.\n\n    Attributes\n    ----------\n    aap: AspectAwarePreprocessor\n        an instance of `pytorch_widedeep.utils.image_utils.AspectAwarePreprocessor`\n    spp: SimplePreprocessor\n        an instance of `pytorch_widedeep.utils.image_utils.SimplePreprocessor`\n    normalise_metrics: Dict\n        Dict containing the normalisation metrics of the image dataset, i.e.\n        mean and std for the R, G and B channels\n\n    Examples\n    --------\n    &gt;&gt;&gt; import pandas as pd\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; from pytorch_widedeep.preprocessing import ImagePreprocessor\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; path_to_image1 = 'tests/test_data_utils/images/galaxy1.png'\n    &gt;&gt;&gt; path_to_image2 = 'tests/test_data_utils/images/galaxy2.png'\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; df_train = pd.DataFrame({'images_column': [path_to_image1]})\n    &gt;&gt;&gt; df_test = pd.DataFrame({'images_column': [path_to_image2]})\n    &gt;&gt;&gt; img_preprocessor = ImagePreprocessor(img_col='images_column', img_path='.', verbose=0)\n    &gt;&gt;&gt; resized_images = img_preprocessor.fit_transform(df_train)\n    &gt;&gt;&gt; new_resized_images = img_preprocessor.transform(df_train)\n\n    :information_source: **NOTE**:\n    Normalising metrics will only be computed when the ``fit_transform``\n    method is run. Running ``transform`` only will not change the computed\n    metrics and running ``fit`` only simply instantiates the resizing\n    functions.\n    \"\"\"\n\n    def __init__(\n        self,\n        img_col: str,\n        img_path: str,\n        width: int = 224,\n        height: int = 224,\n        verbose: int = 1,\n    ):\n        super(ImagePreprocessor, self).__init__()\n\n        self.img_col = img_col\n        self.img_path = img_path\n        self.width = width\n        self.height = height\n        self.verbose = verbose\n\n        self.aap = AspectAwarePreprocessor(self.width, self.height)\n        self.spp = SimplePreprocessor(self.width, self.height)\n\n        self.compute_normalising_computed = False\n\n    def fit(self, df: pd.DataFrame) -&gt; BasePreprocessor:\n        # simply include this method to comply with the BasePreprocessor and\n        # for consistency with the rest of the preprocessors\n        return self\n\n    def transform_sample(self, img: np.ndarray) -&gt; np.ndarray:\n        aspect_r = img.shape[0] / img.shape[1]\n        if aspect_r != 1.0:\n            resized_img = self.aap.preprocess(img)\n        else:\n            resized_img = self.spp.preprocess(img)\n        return resized_img\n\n    def transform(self, df: pd.DataFrame) -&gt; np.ndarray:\n        \"\"\"Resizes the images to the input height and width.\n\n\n        Parameters\n        ----------\n        df: pd.DataFrame\n            Input pandas dataframe with the `img_col`\n\n        Returns\n        -------\n        np.ndarray\n            Resized images to the input height and width\n        \"\"\"\n        image_list = df[self.img_col].tolist()\n        if self.verbose:\n            print(\"Reading Images from {}\".format(self.img_path))\n        imgs = [cv2.imread(\"/\".join([self.img_path, img])) for img in image_list]\n\n        # finding images with different height and width\n        aspect = [(im.shape[0], im.shape[1]) for im in imgs]\n        aspect_r = [a[0] / a[1] for a in aspect]\n        diff_idx = [i for i, r in enumerate(aspect_r) if r != 1.0]\n\n        if self.verbose:\n            print(\"Resizing\")\n        resized_imgs = []\n        for i, img in tqdm(enumerate(imgs), total=len(imgs), disable=self.verbose != 1):\n            if i in diff_idx:\n                resized_imgs.append(self.aap.preprocess(img))\n            else:\n                # if aspect ratio is 1:1, no need for AspectAwarePreprocessor\n                resized_imgs.append(self.spp.preprocess(img))\n\n        if not self.compute_normalising_computed:\n            if self.verbose:\n                print(\"Computing normalisation metrics\")\n            # mean and std deviation will only be computed when the fit method\n            # is called\n            mean_R, mean_G, mean_B = [], [], []\n            std_R, std_G, std_B = [], [], []\n            for rsz_img in resized_imgs:\n                (mean_b, mean_g, mean_r), (std_b, std_g, std_r) = cv2.meanStdDev(\n                    rsz_img\n                )\n                mean_R.append(mean_r)\n                mean_G.append(mean_g)\n                mean_B.append(mean_b)\n                std_R.append(std_r)\n                std_G.append(std_g)\n                std_B.append(std_b)\n            self.normalise_metrics = dict(\n                mean={\n                    \"R\": np.mean(mean_R) / 255.0,\n                    \"G\": np.mean(mean_G) / 255.0,\n                    \"B\": np.mean(mean_B) / 255.0,\n                },\n                std={\n                    \"R\": np.mean(std_R) / 255.0,\n                    \"G\": np.mean(std_G) / 255.0,\n                    \"B\": np.mean(std_B) / 255.0,\n                },\n            )\n            self.compute_normalising_computed = True\n        return np.asarray(resized_imgs)\n\n    def fit_transform(self, df: pd.DataFrame) -&gt; np.ndarray:\n        \"\"\"Combines `fit` and `transform`\n\n        Parameters\n        ----------\n        df: pd.DataFrame\n            Input pandas dataframe\n\n        Returns\n        -------\n        np.ndarray\n            Resized images to the input height and width\n        \"\"\"\n        return self.fit(df).transform(df)\n\n    def inverse_transform(self, transformed_image):\n        raise NotImplementedError(\n            \"'inverse_transform' method is not implemented for 'ImagePreprocessor'\"\n        )\n\n    def __repr__(self) -&gt; str:\n        list_of_params: List[str] = []\n        list_of_params.append(\"img_col={img_col}\")\n        list_of_params.append(\"img_path={img_path}\")\n        list_of_params.append(\"width={width}\")\n        list_of_params.append(\"height={height}\")\n        list_of_params.append(\"verbose={verbose}\")\n        all_params = \", \".join(list_of_params)\n        return f\"ImagePreprocessor({all_params.format(**self.__dict__)})\"\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.image_preprocessor.ImagePreprocessor.transform","title":"transform","text":"<pre><code>transform(df)\n</code></pre> <p>Resizes the images to the input height and width.</p> <p>Parameters:</p> Name Type Description Default <code>df</code> <code>DataFrame</code> <p>Input pandas dataframe with the <code>img_col</code></p> required <p>Returns:</p> Type Description <code>ndarray</code> <p>Resized images to the input height and width</p> Source code in <code>pytorch_widedeep/preprocessing/image_preprocessor.py</code> <pre><code>def transform(self, df: pd.DataFrame) -&gt; np.ndarray:\n    \"\"\"Resizes the images to the input height and width.\n\n\n    Parameters\n    ----------\n    df: pd.DataFrame\n        Input pandas dataframe with the `img_col`\n\n    Returns\n    -------\n    np.ndarray\n        Resized images to the input height and width\n    \"\"\"\n    image_list = df[self.img_col].tolist()\n    if self.verbose:\n        print(\"Reading Images from {}\".format(self.img_path))\n    imgs = [cv2.imread(\"/\".join([self.img_path, img])) for img in image_list]\n\n    # finding images with different height and width\n    aspect = [(im.shape[0], im.shape[1]) for im in imgs]\n    aspect_r = [a[0] / a[1] for a in aspect]\n    diff_idx = [i for i, r in enumerate(aspect_r) if r != 1.0]\n\n    if self.verbose:\n        print(\"Resizing\")\n    resized_imgs = []\n    for i, img in tqdm(enumerate(imgs), total=len(imgs), disable=self.verbose != 1):\n        if i in diff_idx:\n            resized_imgs.append(self.aap.preprocess(img))\n        else:\n            # if aspect ratio is 1:1, no need for AspectAwarePreprocessor\n            resized_imgs.append(self.spp.preprocess(img))\n\n    if not self.compute_normalising_computed:\n        if self.verbose:\n            print(\"Computing normalisation metrics\")\n        # mean and std deviation will only be computed when the fit method\n        # is called\n        mean_R, mean_G, mean_B = [], [], []\n        std_R, std_G, std_B = [], [], []\n        for rsz_img in resized_imgs:\n            (mean_b, mean_g, mean_r), (std_b, std_g, std_r) = cv2.meanStdDev(\n                rsz_img\n            )\n            mean_R.append(mean_r)\n            mean_G.append(mean_g)\n            mean_B.append(mean_b)\n            std_R.append(std_r)\n            std_G.append(std_g)\n            std_B.append(std_b)\n        self.normalise_metrics = dict(\n            mean={\n                \"R\": np.mean(mean_R) / 255.0,\n                \"G\": np.mean(mean_G) / 255.0,\n                \"B\": np.mean(mean_B) / 255.0,\n            },\n            std={\n                \"R\": np.mean(std_R) / 255.0,\n                \"G\": np.mean(std_G) / 255.0,\n                \"B\": np.mean(std_B) / 255.0,\n            },\n        )\n        self.compute_normalising_computed = True\n    return np.asarray(resized_imgs)\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.image_preprocessor.ImagePreprocessor.fit_transform","title":"fit_transform","text":"<pre><code>fit_transform(df)\n</code></pre> <p>Combines <code>fit</code> and <code>transform</code></p> <p>Parameters:</p> Name Type Description Default <code>df</code> <code>DataFrame</code> <p>Input pandas dataframe</p> required <p>Returns:</p> Type Description <code>ndarray</code> <p>Resized images to the input height and width</p> Source code in <code>pytorch_widedeep/preprocessing/image_preprocessor.py</code> <pre><code>def fit_transform(self, df: pd.DataFrame) -&gt; np.ndarray:\n    \"\"\"Combines `fit` and `transform`\n\n    Parameters\n    ----------\n    df: pd.DataFrame\n        Input pandas dataframe\n\n    Returns\n    -------\n    np.ndarray\n        Resized images to the input height and width\n    \"\"\"\n    return self.fit(df).transform(df)\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.din_preprocessor.DINPreprocessor","title":"DINPreprocessor","text":"<p>               Bases: <code>BasePreprocessor</code></p> <p>Preprocessor for Deep Interest Network (DIN) models.</p> <p>This preprocessor handles the preparation of data for DIN models, including sequence building, label encoding, and handling of various types of input columns (categorical, continuous, and sequential).</p> <p>Parameters:</p> Name Type Description Default <code>user_id_col</code> <code>str</code> <p>Name of the column containing user IDs.</p> required <code>item_embed_col</code> <code>Union[str, Tuple[str, int]]</code> <p>Name of the column containing item IDs to be embedded, or a tuple of (column_name, embedding_dim). If embedding_dim is not provided, it will be automatically calculated using a rule of thumb specified by <code>embedding_rule</code>. Aliased as <code>item_id_col</code>.</p> required <code>target_col</code> <code>str</code> <p>Name of the column containing the target variable.</p> required <code>max_seq_length</code> <code>int</code> <p>Maximum length of sequences to be created.</p> required <code>action_col</code> <code>Optional[str]</code> <p>Name of the column containing user actions (if applicable), for example a rating, or purchased/not-purchased, etc). This 'action_col' can also be the same as the 'target_col'.</p> <code>None</code> <code>other_seq_embed_cols</code> <code>Optional[List[str] | List[Tuple[str, int]]]</code> <p>List of other columns to be treated as sequences. Each element in the list can be a string with the name of the column or a tuple with the name of the column and the embedding dimension.</p> <code>None</code> <code>cat_embed_cols</code> <code>Optional[Union[List[str], List[Tuple[str, int]]]]</code> <p>This and the following parameters are identical to those used in the TabPreprocessor. List containing the name of the categorical columns that will be represented by embeddings (e.g. ['education', 'relationship', ...]) or a Tuple with the name and the embedding dimension (e.g.: [ ('education',32), ('relationship',16), ...])</p> <code>None</code> <code>continuous_cols</code> <code>Optional[List[str]]</code> <p>List with the name of the continuous cols</p> <code>None</code> <code>quantization_setup</code> <code>Optional[Union[int, Dict[str, Union[int, List[float]]]]]</code> <p>Continuous columns can be turned into categorical via <code>pd.cut</code>. If <code>quantization_setup</code> is an <code>int</code>, all continuous columns will be quantized using this value as the number of bins. Alternatively, a dictionary where the keys are the column names to quantize and the values are the either integers indicating the number of bins or a list of scalars indicating the bin edges can also be used.</p> <code>None</code> <code>cols_to_scale</code> <code>Optional[Union[List[str], str]]</code> <p>List with the names of the columns that will be standarised via sklearn's <code>StandardScaler</code>. It can also be the string <code>'all'</code> in which case all the continuous cols will be scaled.</p> <code>None</code> <code>auto_embed_dim</code> <code>bool</code> <p>Boolean indicating whether the embedding dimensions will be automatically defined via rule of thumb. See <code>embedding_rule</code> below.</p> <code>True</code> <code>embedding_rule</code> <code>Literal['google', 'fastai_old', 'fastai_new']</code> <p>If <code>auto_embed_dim=True</code>, this is the choice of embedding rule of thumb. Choices are:</p> <ul> <li> <p>fastai_new: \\(min(600, round(1.6     imes n_{cat}^{0.56}))\\)</p> </li> <li> <p>fastai_old: \\(min(50, (n_{cat}//{2})+1)\\)</p> </li> <li> <p>google: \\(min(600, round(n_{cat}^{0.24}))\\)</p> </li> </ul> <code>'fastai_new'</code> <code>default_embed_dim</code> <code>int</code> <p>Dimension for the embeddings if the embedding dimension is not provided in the <code>cat_embed_cols</code> parameter and <code>auto_embed_dim</code> is set to <code>False</code>.</p> <code>16</code> <code>verbose</code> <code>int</code> <p>Verbosity level.</p> <code>1</code> <code>scale</code> <code>bool</code> <p> note: this arg will be removed in upcoming  releases. Please use <code>cols_to_scale</code> instead.  Bool indicating  whether or not to scale/standarise continuous cols. It is important  to emphasize that all the DL models for tabular data in the library  also include the possibility of normalising the input continuous  features via a <code>BatchNorm</code> or a <code>LayerNorm</code>.  Param alias:  <code>scale_cont_cols</code>.</p> <code>False</code> <code>already_standard</code> <code>Optional[List[str]]</code> <p> note: this arg will be removed in upcoming  releases. Please use <code>cols_to_scale</code> instead.  List with the  name of the continuous cols that do not need to be  scaled/standarised.</p> <code>None</code> <p>Other Parameters:</p> Name Type Description <code>**kwargs</code> <p><code>pd.cut</code> and <code>StandardScaler</code> related args</p> <p>Attributes:</p> Name Type Description <code>is_fitted</code> <code>bool</code> <p>Whether the preprocessor has been fitted.</p> <code>has_standard_tab_data</code> <code>bool</code> <p>Whether the data includes standard tabular data.</p> <code>tab_preprocessor</code> <code>TabPreprocessor</code> <p>Preprocessor for standard tabular data (if applicable).</p> <code>din_columns_idx</code> <code>Dict[str, int]</code> <p>Dictionary mapping column names to their indices in the processed data.</p> <code>item_le</code> <code>LabelEncoder</code> <p>Label encoder for item IDs.</p> <code>n_items</code> <code>int</code> <p>Number of unique items.</p> <code>user_behaviour_config</code> <code>Tuple[List[str], int, int]</code> <p>Configuration for user behavior sequences.</p> <code>action_le</code> <code>LabelEncoder</code> <p>Label encoder for action column (if applicable).</p> <code>n_actions</code> <code>int</code> <p>Number of unique actions (if applicable).</p> <code>action_seq_config</code> <code>Tuple[List[str], int]</code> <p>Configuration for action sequences (if applicable).</p> <code>other_seq_le</code> <code>LabelEncoder</code> <p>Label encoder for other sequence columns (if applicable).</p> <code>n_other_seq_cols</code> <code>Dict[str, int]</code> <p>Number of unique values in each other sequence column.</p> <code>other_seq_config</code> <code>List[Tuple[List[str], int, int]]</code> <p>Configuration for other sequence columns.</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import pandas as pd\n&gt;&gt;&gt; from pytorch_widedeep.preprocessing import DINPreprocessor\n&gt;&gt;&gt; data = {\n...    'user_id': [1, 1, 1, 2, 2, 2, 3, 3, 3],\n...    'item_id': [101, 102, 103, 101, 103, 104, 102, 103, 104],\n...    'timestamp': [1, 2, 3, 1, 2, 3, 1, 2, 3],\n...    'category': ['A', 'B', 'A', 'B', 'A', 'C', 'B', 'A', 'C'],\n...    'price': [10.5, 15.0, 12.0, 10.5, 12.0, 20.0, 15.0, 12.0, 20.0],\n...    'rating': [0, 1, 0, 1, 0, 1, 0, 1, 0]\n... }\n&gt;&gt;&gt; df = pd.DataFrame(data)\n&gt;&gt;&gt; din_preprocessor = DINPreprocessor(\n...     user_id_col='user_id',\n...     item_embed_col='item_id',\n...     target_col='rating',\n...     max_seq_length=2,\n...     action_col='rating',\n...     other_seq_embed_cols=['category'],\n...     cat_embed_cols=['user_id'],\n...     continuous_cols=['price'],\n...     cols_to_scale=['price']\n... )\n&gt;&gt;&gt; X, y = din_preprocessor.fit_transform(df)\n</code></pre> Source code in <code>pytorch_widedeep/preprocessing/din_preprocessor.py</code> <pre><code>class DINPreprocessor(BasePreprocessor):\n    \"\"\"\n    Preprocessor for Deep Interest Network (DIN) models.\n\n    This preprocessor handles the preparation of data for DIN models,\n    including sequence building, label encoding, and handling of various\n    types of input columns (categorical, continuous, and sequential).\n\n    Parameters\n    ----------\n    user_id_col : str\n        Name of the column containing user IDs.\n    item_embed_col : Union[str, Tuple[str, int]]\n        Name of the column containing item IDs to be embedded, or a tuple of\n        (column_name, embedding_dim). If embedding_dim is not provided, it\n        will be automatically calculated using a rule of thumb specified by\n        `embedding_rule`. Aliased as `item_id_col`.\n    target_col : str\n        Name of the column containing the target variable.\n    max_seq_length : int\n        Maximum length of sequences to be created.\n    action_col : Optional[str], default=None\n        Name of the column containing user actions (if applicable), for\n        example a rating, or purchased/not-purchased, etc). This 'action_col'\n        can also be the same as the 'target_col'.\n    other_seq_embed_cols : Optional[List[str] | List[Tuple[str, int]]], default=None\n        List of other columns to be treated as sequences. Each element in the\n        list can be a string with the name of the column or a tuple with the\n        name of the column and the embedding dimension.\n\n    cat_embed_cols : Optional[Union[List[str], List[Tuple[str, int]]]], default=None\n        This and the following parameters are identical to those used in the\n        TabPreprocessor.&lt;br&gt;\n        List containing the name of the categorical columns that will be\n        represented by embeddings (e.g. _['education', 'relationship', ...]_) or\n        a Tuple with the name and the embedding dimension (e.g.: _[\n        ('education',32), ('relationship',16), ...]_)\n    continuous_cols: List, default = None\n        List with the name of the continuous cols\n    quantization_setup: int or Dict, default = None\n        Continuous columns can be turned into categorical via `pd.cut`. If\n        `quantization_setup` is an `int`, all continuous columns will be\n        quantized using this value as the number of bins. Alternatively, a\n        dictionary where the keys are the column names to quantize and the\n        values are the either integers indicating the number of bins or a\n        list of scalars indicating the bin edges can also be used.\n    cols_to_scale: List or str, default = None,\n        List with the names of the columns that will be standarised via\n        sklearn's `StandardScaler`. It can also be the string `'all'` in\n        which case all the continuous cols will be scaled.\n    auto_embed_dim: bool, default = True\n        Boolean indicating whether the embedding dimensions will be\n        automatically defined via rule of thumb. See `embedding_rule`\n        below.\n    embedding_rule: str, default = 'fastai_new'\n        If `auto_embed_dim=True`, this is the choice of embedding rule of\n        thumb. Choices are:\n\n        - _fastai_new_: $min(600, round(1.6 \\times n_{cat}^{0.56}))$\n\n        - _fastai_old_: $min(50, (n_{cat}//{2})+1)$\n\n        - _google_: $min(600, round(n_{cat}^{0.24}))$\n    default_embed_dim: int, default=16\n        Dimension for the embeddings if the embedding dimension is not\n        provided in the `cat_embed_cols` parameter and `auto_embed_dim` is\n        set to `False`.\n    verbose : int, default=1\n        Verbosity level.\n    scale: bool, default = False\n        :information_source: **note**: this arg will be removed in upcoming\n         releases. Please use `cols_to_scale` instead. &lt;br/&gt; Bool indicating\n         whether or not to scale/standarise continuous cols. It is important\n         to emphasize that all the DL models for tabular data in the library\n         also include the possibility of normalising the input continuous\n         features via a `BatchNorm` or a `LayerNorm`. &lt;br/&gt; Param alias:\n         `scale_cont_cols`.\n    already_standard: List, default = None\n        :information_source: **note**: this arg will be removed in upcoming\n         releases. Please use `cols_to_scale` instead. &lt;br/&gt; List with the\n         name of the continuous cols that do not need to be\n         scaled/standarised.\n\n    Other Parameters\n    ----------------\n    **kwargs: dict\n        `pd.cut` and `StandardScaler` related args\n\n    Attributes\n    ----------\n    is_fitted : bool\n        Whether the preprocessor has been fitted.\n    has_standard_tab_data : bool\n        Whether the data includes standard tabular data.\n    tab_preprocessor : TabPreprocessor\n        Preprocessor for standard tabular data (if applicable).\n    din_columns_idx : Dict[str, int]\n        Dictionary mapping column names to their indices in the processed data.\n    item_le : LabelEncoder\n        Label encoder for item IDs.\n    n_items : int\n        Number of unique items.\n    user_behaviour_config : Tuple[List[str], int, int]\n        Configuration for user behavior sequences.\n    action_le : LabelEncoder\n        Label encoder for action column (if applicable).\n    n_actions : int\n        Number of unique actions (if applicable).\n    action_seq_config : Tuple[List[str], int]\n        Configuration for action sequences (if applicable).\n    other_seq_le : LabelEncoder\n        Label encoder for other sequence columns (if applicable).\n    n_other_seq_cols : Dict[str, int]\n        Number of unique values in each other sequence column.\n    other_seq_config : List[Tuple[List[str], int, int]]\n        Configuration for other sequence columns.\n\n    Examples\n    --------\n    &gt;&gt;&gt; import pandas as pd\n    &gt;&gt;&gt; from pytorch_widedeep.preprocessing import DINPreprocessor\n    &gt;&gt;&gt; data = {\n    ...    'user_id': [1, 1, 1, 2, 2, 2, 3, 3, 3],\n    ...    'item_id': [101, 102, 103, 101, 103, 104, 102, 103, 104],\n    ...    'timestamp': [1, 2, 3, 1, 2, 3, 1, 2, 3],\n    ...    'category': ['A', 'B', 'A', 'B', 'A', 'C', 'B', 'A', 'C'],\n    ...    'price': [10.5, 15.0, 12.0, 10.5, 12.0, 20.0, 15.0, 12.0, 20.0],\n    ...    'rating': [0, 1, 0, 1, 0, 1, 0, 1, 0]\n    ... }\n    &gt;&gt;&gt; df = pd.DataFrame(data)\n    &gt;&gt;&gt; din_preprocessor = DINPreprocessor(\n    ...     user_id_col='user_id',\n    ...     item_embed_col='item_id',\n    ...     target_col='rating',\n    ...     max_seq_length=2,\n    ...     action_col='rating',\n    ...     other_seq_embed_cols=['category'],\n    ...     cat_embed_cols=['user_id'],\n    ...     continuous_cols=['price'],\n    ...     cols_to_scale=['price']\n    ... )\n    &gt;&gt;&gt; X, y = din_preprocessor.fit_transform(df)\n    \"\"\"\n\n    @alias(\"item_embed_col\", [\"item_id_col\"])\n    def __init__(\n        self,\n        *,\n        user_id_col: str,\n        item_embed_col: Union[str, Tuple[str, int]],\n        target_col: str,\n        max_seq_length: int,\n        action_col: Optional[str] = None,\n        other_seq_embed_cols: Optional[Union[List[str], List[Tuple[str, int]]]] = None,\n        cat_embed_cols: Optional[Union[List[str], List[Tuple[str, int]]]] = None,\n        continuous_cols: Optional[List[str]] = None,\n        quantization_setup: Optional[\n            Union[int, Dict[str, Union[int, List[float]]]]\n        ] = None,\n        cols_to_scale: Optional[Union[List[str], str]] = None,\n        auto_embed_dim: bool = True,\n        embedding_rule: Literal[\"google\", \"fastai_old\", \"fastai_new\"] = \"fastai_new\",\n        default_embed_dim: int = 16,\n        verbose: int = 1,\n        scale: bool = False,\n        already_standard: Optional[List[str]] = None,\n        **kwargs,\n    ):\n        self.user_id_col = user_id_col\n        self.item_embed_col = item_embed_col\n        self.max_seq_length = max_seq_length\n        self.target_col = target_col if target_col is not None else \"target\"\n        self.action_col = action_col\n        self.other_seq_embed_cols = other_seq_embed_cols\n        self.cat_embed_cols = cat_embed_cols\n        self.continuous_cols = continuous_cols\n        self.quantization_setup = quantization_setup\n        self.cols_to_scale = cols_to_scale\n        self.auto_embed_dim = auto_embed_dim\n        self.embedding_rule = embedding_rule\n        self.default_embed_dim = default_embed_dim\n        self.verbose = verbose\n        self.scale = scale\n        self.already_standard = already_standard\n        self.kwargs = kwargs\n\n        self.has_standard_tab_data = bool(self.cat_embed_cols or self.continuous_cols)\n        if self.has_standard_tab_data:\n            self.tab_preprocessor = TabPreprocessor(\n                cat_embed_cols=self.cat_embed_cols,\n                continuous_cols=self.continuous_cols,\n                quantization_setup=self.quantization_setup,\n                cols_to_scale=self.cols_to_scale,\n                auto_embed_dim=self.auto_embed_dim,\n                embedding_rule=self.embedding_rule,\n                default_embed_dim=self.default_embed_dim,\n                verbose=self.verbose,\n                scale=self.scale,\n                already_standard=self.already_standard,\n                **self.kwargs,\n            )\n\n        self.is_fitted = False\n\n    def fit(self, df: pd.DataFrame) -&gt; \"DINPreprocessor\":\n        if self.has_standard_tab_data:\n            self.tab_preprocessor.fit(df)\n            self.din_columns_idx = {\n                col: i for i, col in enumerate(self.tab_preprocessor.column_idx.keys())\n            }\n        else:\n            self.din_columns_idx = {}\n\n        self._fit_label_encoders(df)\n        self.is_fitted = True\n        return self\n\n    def transform(self, df: pd.DataFrame) -&gt; Tuple[np.ndarray, np.ndarray]:\n        _df = self._pre_transform(df)\n        df_w_sequences = self._build_sequences(_df)\n        X_all = self._concatenate_features(df_w_sequences)\n        return X_all, np.array(df_w_sequences[self.target_col].tolist())\n\n    def fit_transform(self, df: pd.DataFrame) -&gt; Tuple[np.ndarray, np.ndarray]:\n        self.fit(df)\n        return self.transform(df)\n\n    def inverse_transform(self, X: np.ndarray) -&gt; pd.DataFrame:\n        raise NotImplementedError(\n            \"inverse_transform is not implemented for this preprocessor\"\n        )\n\n    def _fit_label_encoders(self, df: pd.DataFrame) -&gt; None:\n        self.item_le = LabelEncoder(columns_to_encode=[self._get_item_col()])\n        self.item_le.fit(df)\n        self.n_items = max(self.item_le.encoding_dict[self._get_item_col()].values())\n        user_behaviour_embed_size = self._get_embed_size(\n            self.item_embed_col, self.n_items\n        )\n        self.user_behaviour_config = (\n            [f\"item_{i+1}\" for i in range(self.max_seq_length)],\n            self.n_items,\n            user_behaviour_embed_size,\n        )\n\n        self._update_din_columns_idx(\"item\", self.max_seq_length)\n        self.din_columns_idx[\"target_item\"] = len(self.din_columns_idx)\n\n        if self.action_col:\n            self._fit_action_label_encoder(df)\n        if self.other_seq_embed_cols:\n            self._fit_other_seq_label_encoders(df)\n\n    def _fit_action_label_encoder(self, df: pd.DataFrame) -&gt; None:\n        self.action_le = LabelEncoder(columns_to_encode=[self.action_col])\n        self.action_le.fit(df)\n        self.n_actions = len(self.action_le.encoding_dict[self.action_col])\n        self.action_seq_config = (\n            [f\"action_{i+1}\" for i in range(self.max_seq_length)],\n            self.n_actions,\n        )\n        self._update_din_columns_idx(\"action\", self.max_seq_length)\n\n    def _other_seq_cols_float_warning(self, df: pd.DataFrame) -&gt; None:\n        other_seq_cols = [\n            col[0] if isinstance(col, tuple) else col\n            for col in self.other_seq_embed_cols\n        ]\n        for col in other_seq_cols:\n            if df[col].dtype == float:\n                warnings.warn(\n                    f\"{col} is a float column. It will be converted to integers. \"\n                    \"If this is not what you want, please convert it beforehand.\",\n                    UserWarning,\n                )\n\n    def _convert_other_seq_cols_to_int(self, df: pd.DataFrame) -&gt; None:\n        other_seq_cols = [\n            col[0] if isinstance(col, tuple) else col\n            for col in self.other_seq_embed_cols\n        ]\n        for col in other_seq_cols:\n            if df[col].dtype == float:\n                df[col] = df[col].astype(int)\n\n    def _fit_other_seq_label_encoders(self, df: pd.DataFrame) -&gt; None:\n        self._other_seq_cols_float_warning(df)\n        self._convert_other_seq_cols_to_int(df)\n        self.other_seq_le = LabelEncoder(\n            columns_to_encode=[\n                col[0] if isinstance(col, tuple) else col\n                for col in self.other_seq_embed_cols\n            ]\n        )\n        self.other_seq_le.fit(df)\n        other_seq_cols = [\n            col[0] if isinstance(col, tuple) else col\n            for col in self.other_seq_embed_cols\n        ]\n        self.n_other_seq_cols = {\n            col: max(self.other_seq_le.encoding_dict[col].values())\n            for col in other_seq_cols\n        }\n        other_seq_embed_sizes = {\n            col: self._get_embed_size(col, self.n_other_seq_cols[col])\n            for col in other_seq_cols\n        }\n        self.other_seq_config = [\n            (\n                self._get_seq_col_names(col),\n                self.n_other_seq_cols[col],\n                other_seq_embed_sizes[col],\n            )\n            for col in other_seq_cols\n        ]\n        self._update_din_columns_idx_for_other_seq(other_seq_cols)\n\n    def _get_item_col(self) -&gt; str:\n        return (\n            self.item_embed_col[0]\n            if isinstance(self.item_embed_col, tuple)\n            else self.item_embed_col\n        )\n\n    def _get_embed_size(self, col: Union[str, Tuple[str, int]], n_unique: int) -&gt; int:\n        return col[1] if isinstance(col, tuple) else embed_sz_rule(n_unique)\n\n    def _get_seq_col_names(self, col: str) -&gt; List[str]:\n        return [f\"{col}_{i+1}\" for i in range(self.max_seq_length)]\n\n    def _update_din_columns_idx(self, prefix: str, length: int) -&gt; None:\n        current_len = len(self.din_columns_idx)\n        self.din_columns_idx.update(\n            {f\"{prefix}_{i+1}\": i + current_len for i in range(length)}\n        )\n\n    def _update_din_columns_idx_for_other_seq(self, other_seq_cols: List[str]) -&gt; None:\n        current_len = len(self.din_columns_idx)\n        for col in other_seq_cols:\n            self.din_columns_idx.update(\n                {f\"{col}_{i+1}\": i + current_len for i in range(self.max_seq_length)}\n            )\n            self.din_columns_idx[f\"target_{col}\"] = len(self.din_columns_idx)\n            current_len = len(self.din_columns_idx)\n\n    def _pre_transform(self, df: pd.DataFrame) -&gt; pd.DataFrame:\n        check_is_fitted(self, attributes=[\"din_columns_idx\"])\n        df = self.item_le.transform(df)\n        if self.action_col:\n            df = self.action_le.transform(df)\n        if self.other_seq_embed_cols:\n            df = self.other_seq_le.transform(df)\n        return df\n\n    def _build_sequences(self, df: pd.DataFrame) -&gt; pd.DataFrame:\n        # df.columns.tolist() -&gt; to avoid annoying pandas warning\n        return (\n            df.groupby(self.user_id_col)[df.columns.tolist()]  # type: ignore[index]\n            .apply(self._group_sequences)\n            .reset_index(drop=True)\n        )\n\n    def _group_sequences(self, group: pd.DataFrame) -&gt; pd.DataFrame:\n        item_col = self._get_item_col()\n        items = group[item_col].tolist()\n        targets = group[self.target_col].tolist()\n        drop_cols = [item_col, self.target_col]\n\n        # sequences cannot be built with user, item pairs with only one\n        # interaction\n        if len(items) &lt;= 1:\n            return pd.DataFrame()\n\n        sequences = [\n            self._create_sequence(group, items, targets, i, drop_cols)\n            for i in range(max(1, len(items) - self.max_seq_length))\n        ]\n\n        seq_df = pd.DataFrame(sequences)\n        non_seq_cols = group.drop_duplicates([self.user_id_col]).drop(drop_cols, axis=1)\n        return pd.merge(seq_df, non_seq_cols, on=self.user_id_col)\n\n    def _create_sequence(\n        self,\n        group: pd.DataFrame,\n        items: List[int],\n        targets: List[int],\n        i: int,\n        drop_cols: List[str],\n    ) -&gt; Dict[str, Union[str, List[int]]]:\n        end_idx = min(i + self.max_seq_length, len(items) - 1)\n        item_sequences = items[i:end_idx]\n        target_item = items[end_idx]\n        target = targets[end_idx]\n\n        sequence = {\n            self.user_id_col: group.name,\n            \"items_sequence\": item_sequences,\n            \"target_item\": target_item,\n        }\n\n        if self.action_col:\n            sequence.update(\n                self._create_action_sequence(group, targets, i, target, drop_cols)\n            )\n        else:\n            sequence[self.target_col] = target\n            drop_cols.append(self.target_col)\n\n        if self.other_seq_embed_cols:\n            sequence.update(self._create_other_seq_sequence(group, i, drop_cols))\n\n        return sequence\n\n    def _create_action_sequence(\n        self,\n        group: pd.DataFrame,\n        targets: List[int],\n        i: int,\n        target: int,\n        drop_cols: List[str],\n    ) -&gt; Dict[str, Union[int, List[int]]]:\n        if self.action_col != self.target_col:\n            actions = group[self.action_col].tolist()\n            action_sequences = (\n                actions[i : i + self.max_seq_length]\n                if i + self.max_seq_length &lt; len(actions)\n                else actions[i:-1]\n            )\n            drop_cols.append(self.action_col)\n        else:\n            target -= 1  # the 'transform' method adds 1 as it saves 0 for padding\n            action_sequences = (\n                targets[i : i + self.max_seq_length]\n                if i + self.max_seq_length &lt; len(targets)\n                else targets[i:-1]\n            )\n\n        return {self.target_col: target, \"actions_sequence\": action_sequences}\n\n    def _create_other_seq_sequence(\n        self, group: pd.DataFrame, i: int, drop_cols: List[str]\n    ) -&gt; Dict[str, Union[int, List[int]]]:\n        other_seq_cols = [\n            col[0] if isinstance(col, tuple) else col\n            for col in self.other_seq_embed_cols\n        ]\n        drop_cols += other_seq_cols\n        other_seqs = {col: group[col].tolist() for col in other_seq_cols}\n\n        other_seqs_sequences: Dict[str, Union[int, List[int]]] = {}\n        for col in other_seq_cols:\n            other_seqs_sequences[col] = (\n                other_seqs[col][i : i + self.max_seq_length]\n                if i + self.max_seq_length &lt; len(other_seqs[col])\n                else other_seqs[col][i:-1]\n            )\n\n        sequence: Dict[str, Union[int, List[int]]] = {}\n        for col in other_seq_cols:\n            sequence[f\"{col}_sequence\"] = other_seqs_sequences[col]\n            sequence[f\"target_{col}\"] = (\n                other_seqs[col][i + self.max_seq_length]\n                if i + self.max_seq_length &lt; len(other_seqs[col])\n                else other_seqs[col][-1]\n            )\n\n        return sequence\n\n    def _concatenate_features(self, df_w_sequences: pd.DataFrame) -&gt; np.ndarray:\n        user_behaviour_seq = df_w_sequences[\"items_sequence\"].tolist()\n        X_user_behaviour = np.vstack(\n            [\n                pad_sequences(seq, self.max_seq_length, pad_idx=0)\n                for seq in user_behaviour_seq\n            ]\n        )\n        X_target_item = np.array(df_w_sequences[\"target_item\"].tolist()).reshape(-1, 1)\n        X_all = np.concatenate([X_user_behaviour, X_target_item], axis=1)\n\n        if self.has_standard_tab_data:\n            X_tab = self.tab_preprocessor.transform(df_w_sequences)\n            X_all = np.concatenate([X_tab, X_all], axis=1)\n\n        if self.action_col:\n            action_seq = df_w_sequences[\"actions_sequence\"].tolist()\n            X_actions = np.vstack(\n                [\n                    pad_sequences(seq, self.max_seq_length, pad_idx=0)\n                    for seq in action_seq\n                ]\n            )\n            X_all = np.concatenate([X_all, X_actions], axis=1)\n\n        if self.other_seq_embed_cols:\n            X_all = self._concatenate_other_seq_features(df_w_sequences, X_all)\n\n        assert len(self.din_columns_idx) == X_all.shape[1], (\n            f\"Something went wrong. The number of columns in the final array \"\n            f\"({X_all.shape[1]}) is different from the number of columns in \"\n            f\"self.din_columns_idx ({len(self.din_columns_idx)})\"\n        )\n\n        return X_all\n\n    def _concatenate_other_seq_features(\n        self, df_w_sequences: pd.DataFrame, X_all: np.ndarray\n    ) -&gt; np.ndarray:\n        other_seq_cols = [\n            col[0] if isinstance(col, tuple) else col\n            for col in self.other_seq_embed_cols\n        ]\n        other_seq = {\n            col: df_w_sequences[f\"{col}_sequence\"].tolist() for col in other_seq_cols\n        }\n        other_seq_target = {\n            col: df_w_sequences[f\"target_{col}\"].tolist() for col in other_seq_cols\n        }\n        X_other_seq_arrays = []\n\n        for col in other_seq_cols:\n            X_other_seq_arrays.append(\n                np.vstack(\n                    [\n                        pad_sequences(s, self.max_seq_length, pad_idx=0)\n                        for s in other_seq[col]\n                    ]\n                )\n            )\n            X_other_seq_arrays.append(np.array(other_seq_target[col]).reshape(-1, 1))\n\n        return np.concatenate([X_all] + X_other_seq_arrays, axis=1)\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#chunked-versions","title":"Chunked versions","text":"<p>Chunked versions of the preprocessors are also available. These are useful when the data is too big to fit in memory. See also the <code>load_from_folder</code> module in the library and the corresponding section here in the documentation.</p> <p>Note that there is not a <code>ChunkImagePreprocessor</code>. This is because the processing of the images will occur inside the <code>ImageFromFolder</code> class in the <code>load_from_folder</code> module.</p>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.wide_preprocessor.ChunkWidePreprocessor","title":"ChunkWidePreprocessor","text":"<p>               Bases: <code>WidePreprocessor</code></p> <p>Preprocessor to prepare the wide input dataset</p> <p>This Preprocessor prepares the data for the wide, linear component. This linear model is implemented via an Embedding layer that is connected to the output neuron. <code>ChunkWidePreprocessor</code> numerically encodes all the unique values of all categorical columns <code>wide_cols + crossed_cols</code>. See the Example below.</p> <p>Parameters:</p> Name Type Description Default <code>wide_cols</code> <code>List[str]</code> <p>List of strings with the name of the columns that will label encoded and passed through the <code>wide</code> component</p> required <code>crossed_cols</code> <code>Optional[List[Tuple[str, str]]]</code> <p>List of Tuples with the name of the columns that will be <code>'crossed'</code> and then label encoded. e.g. [('education', 'occupation'), ...]. For binary features, a cross-product transformation is 1 if and only if the constituent features are all 1, and 0 otherwise.</p> <code>None</code> <p>Attributes:</p> Name Type Description <code>wide_crossed_cols</code> <code>List</code> <p>List with the names of all columns that will be label encoded</p> <code>encoding_dict</code> <code>Dict</code> <p>Dictionary where the keys are the result of pasting <code>colname + '_' + column value</code> and the values are the corresponding mapped integer.</p> <code>inverse_encoding_dict</code> <code>Dict</code> <p>the inverse encoding dictionary</p> <code>wide_dim</code> <code>int</code> <p>Dimension of the wide model (i.e. dim of the linear layer)</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import pandas as pd\n&gt;&gt;&gt; from pytorch_widedeep.preprocessing import ChunkWidePreprocessor\n&gt;&gt;&gt; chunk = pd.DataFrame({'color': ['r', 'b', 'g'], 'size': ['s', 'n', 'l']})\n&gt;&gt;&gt; wide_cols = ['color']\n&gt;&gt;&gt; crossed_cols = [('color', 'size')]\n&gt;&gt;&gt; chunk_wide_preprocessor = ChunkWidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols,\n... n_chunks=1)\n&gt;&gt;&gt; X_wide = chunk_wide_preprocessor.fit_transform(chunk)\n</code></pre> Source code in <code>pytorch_widedeep/preprocessing/wide_preprocessor.py</code> <pre><code>class ChunkWidePreprocessor(WidePreprocessor):\n    r\"\"\"Preprocessor to prepare the wide input dataset\n\n    This Preprocessor prepares the data for the wide, linear component.\n    This linear model is implemented via an Embedding layer that is\n    connected to the output neuron. `ChunkWidePreprocessor` numerically\n    encodes all the unique values of all categorical columns `wide_cols +\n    crossed_cols`. See the Example below.\n\n    Parameters\n    ----------\n    wide_cols: List\n        List of strings with the name of the columns that will label\n        encoded and passed through the `wide` component\n    crossed_cols: List, default = None\n        List of Tuples with the name of the columns that will be `'crossed'`\n        and then label encoded. e.g. _[('education', 'occupation'), ...]_. For\n        binary features, a cross-product transformation is 1 if and only if\n        the constituent features are all 1, and 0 otherwise.\n\n    Attributes\n    ----------\n    wide_crossed_cols: List\n        List with the names of all columns that will be label encoded\n    encoding_dict: Dict\n        Dictionary where the keys are the result of pasting `colname + '_' +\n        column value` and the values are the corresponding mapped integer.\n    inverse_encoding_dict: Dict\n        the inverse encoding dictionary\n    wide_dim: int\n        Dimension of the wide model (i.e. dim of the linear layer)\n\n    Examples\n    --------\n    &gt;&gt;&gt; import pandas as pd\n    &gt;&gt;&gt; from pytorch_widedeep.preprocessing import ChunkWidePreprocessor\n    &gt;&gt;&gt; chunk = pd.DataFrame({'color': ['r', 'b', 'g'], 'size': ['s', 'n', 'l']})\n    &gt;&gt;&gt; wide_cols = ['color']\n    &gt;&gt;&gt; crossed_cols = [('color', 'size')]\n    &gt;&gt;&gt; chunk_wide_preprocessor = ChunkWidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols,\n    ... n_chunks=1)\n    &gt;&gt;&gt; X_wide = chunk_wide_preprocessor.fit_transform(chunk)\n    \"\"\"\n\n    def __init__(\n        self,\n        wide_cols: List[str],\n        n_chunks: int,\n        crossed_cols: Optional[List[Tuple[str, str]]] = None,\n    ):\n        super(ChunkWidePreprocessor, self).__init__(wide_cols, crossed_cols)\n\n        self.n_chunks = n_chunks\n\n        self.chunk_counter = 0\n\n        self.is_fitted = False\n\n    def partial_fit(self, chunk: pd.DataFrame) -&gt; \"ChunkWidePreprocessor\":\n        r\"\"\"Fits the Preprocessor and creates required attributes\n\n        Parameters\n        ----------\n        chunk: pd.DataFrame\n            Input pandas dataframe\n\n        Returns\n        -------\n        ChunkWidePreprocessor\n            `ChunkWidePreprocessor` fitted object\n        \"\"\"\n        df_wide = self._prepare_wide(chunk)\n        self.wide_crossed_cols = df_wide.columns.tolist()\n\n        if self.chunk_counter == 0:\n            self.glob_feature_set = set(\n                self._make_global_feature_list(df_wide[self.wide_crossed_cols])\n            )\n        else:\n            self.glob_feature_set.update(\n                self._make_global_feature_list(df_wide[self.wide_crossed_cols])\n            )\n\n        self.chunk_counter += 1\n\n        if self.chunk_counter == self.n_chunks:\n            self.encoding_dict = {v: i + 1 for i, v in enumerate(self.glob_feature_set)}\n            self.wide_dim = len(self.encoding_dict)\n            self.inverse_encoding_dict = {k: v for v, k in self.encoding_dict.items()}\n            self.inverse_encoding_dict[0] = \"unseen\"\n\n            self.is_fitted = True\n\n        return self\n\n    def fit(self, df: pd.DataFrame) -&gt; \"ChunkWidePreprocessor\":\n        \"\"\"\n        Runs `partial_fit`. This is just to override the fit method in the base\n        class. This class is not designed or thought to run fit\n        \"\"\"\n        return self.partial_fit(df)\n\n    def __repr__(self) -&gt; str:\n        list_of_params: List[str] = [\"wide_cols={wide_cols}\"]\n        list_of_params.append(\"n_chunks={n_chunks}\")\n        if self.crossed_cols is not None:\n            list_of_params.append(\"crossed_cols={crossed_cols}\")\n        all_params = \", \".join(list_of_params)\n        return f\"WidePreprocessor({all_params.format(**self.__dict__)})\"\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.wide_preprocessor.ChunkWidePreprocessor.partial_fit","title":"partial_fit","text":"<pre><code>partial_fit(chunk)\n</code></pre> <p>Fits the Preprocessor and creates required attributes</p> <p>Parameters:</p> Name Type Description Default <code>chunk</code> <code>DataFrame</code> <p>Input pandas dataframe</p> required <p>Returns:</p> Type Description <code>ChunkWidePreprocessor</code> <p><code>ChunkWidePreprocessor</code> fitted object</p> Source code in <code>pytorch_widedeep/preprocessing/wide_preprocessor.py</code> <pre><code>def partial_fit(self, chunk: pd.DataFrame) -&gt; \"ChunkWidePreprocessor\":\n    r\"\"\"Fits the Preprocessor and creates required attributes\n\n    Parameters\n    ----------\n    chunk: pd.DataFrame\n        Input pandas dataframe\n\n    Returns\n    -------\n    ChunkWidePreprocessor\n        `ChunkWidePreprocessor` fitted object\n    \"\"\"\n    df_wide = self._prepare_wide(chunk)\n    self.wide_crossed_cols = df_wide.columns.tolist()\n\n    if self.chunk_counter == 0:\n        self.glob_feature_set = set(\n            self._make_global_feature_list(df_wide[self.wide_crossed_cols])\n        )\n    else:\n        self.glob_feature_set.update(\n            self._make_global_feature_list(df_wide[self.wide_crossed_cols])\n        )\n\n    self.chunk_counter += 1\n\n    if self.chunk_counter == self.n_chunks:\n        self.encoding_dict = {v: i + 1 for i, v in enumerate(self.glob_feature_set)}\n        self.wide_dim = len(self.encoding_dict)\n        self.inverse_encoding_dict = {k: v for v, k in self.encoding_dict.items()}\n        self.inverse_encoding_dict[0] = \"unseen\"\n\n        self.is_fitted = True\n\n    return self\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.wide_preprocessor.ChunkWidePreprocessor.fit","title":"fit","text":"<pre><code>fit(df)\n</code></pre> <p>Runs <code>partial_fit</code>. This is just to override the fit method in the base class. This class is not designed or thought to run fit</p> Source code in <code>pytorch_widedeep/preprocessing/wide_preprocessor.py</code> <pre><code>def fit(self, df: pd.DataFrame) -&gt; \"ChunkWidePreprocessor\":\n    \"\"\"\n    Runs `partial_fit`. This is just to override the fit method in the base\n    class. This class is not designed or thought to run fit\n    \"\"\"\n    return self.partial_fit(df)\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.tab_preprocessor.ChunkTabPreprocessor","title":"ChunkTabPreprocessor","text":"<p>               Bases: <code>TabPreprocessor</code></p> <p>Preprocessor to prepare the <code>deeptabular</code> component input dataset</p> <p>Parameters:</p> Name Type Description Default <code>n_chunks</code> <code>int</code> <p>Number of chunks that the tabular dataset is divided by.</p> required <code>cat_embed_cols</code> <code>Optional[Union[List[str], List[Tuple[str, int]]]]</code> <p>List containing the name of the categorical columns that will be represented by embeddings (e.g. ['education', 'relationship', ...]) or a Tuple with the name and the embedding dimension (e.g.: [ ('education',32), ('relationship',16), ...]).  Param alias: <code>embed_cols</code></p> <code>None</code> <code>continuous_cols</code> <code>Optional[List[str]]</code> <p>List with the name of the continuous cols</p> <code>None</code> <code>cols_and_bins</code> <code>Optional[Dict[str, List[float]]]</code> <p>Continuous columns can be turned into categorical via <code>pd.cut</code>. 'cols_and_bins' is dictionary where the keys are the column names to quantize and the values are a list of scalars indicating the bin edges.  Param alias: <code>quantization_setup</code></p> <code>None</code> <code>cols_to_scale</code> <code>Optional[Union[List[str], str]]</code> <p>List with the names of the columns that will be standarised via sklearn's <code>StandardScaler</code></p> <code>None</code> <code>default_embed_dim</code> <code>int</code> <p>Dimension for the embeddings if the embed_dim is not provided in the <code>cat_embed_cols</code> parameter and <code>auto_embed_dim</code> is set to <code>False</code>.</p> <code>16</code> <code>with_attention</code> <code>bool</code> <p>Boolean indicating whether the preprocessed data will be passed to an attention-based model (more precisely a model where all embeddings must have the same dimensions). If <code>True</code>, the param <code>cat_embed_cols</code> must just be a list containing just the categorical column names: e.g. ['education', 'relationship', ...]. This is because they will all be  encoded using embeddings of the same dim, which will be specified  later when the model is defined.  Param alias:  <code>for_transformer</code></p> <code>False</code> <code>with_cls_token</code> <code>bool</code> <p>Boolean indicating if a <code>'[CLS]'</code> token will be added to the dataset when using attention-based models. The final hidden state corresponding to this token is used as the aggregated representation for classification and regression tasks. If not, the categorical (and continuous embeddings if present) will be concatenated before being passed to the final MLP (if present).</p> <code>False</code> <code>shared_embed</code> <code>bool</code> <p>Boolean indicating if the embeddings will be \"shared\" when using attention-based models. The idea behind <code>shared_embed</code> is described in the Appendix A in the TabTransformer paper: 'The goal of having column embedding is to enable the model to distinguish the classes in one column from those in the other columns'. In other words, the idea is to let the model learn which column is embedded at the time. See: <code>pytorch_widedeep.models.transformers._layers.SharedEmbeddings</code>.</p> <code>False</code> <code>verbose</code> <code>int</code> <code>1</code> <code>scale</code> <code>bool</code> <p> note: this arg will be removed in upcoming  releases. Please use <code>cols_to_scale</code> instead.  Bool indicating  whether or not to scale/standarise continuous cols. It is important  to emphasize that all the DL models for tabular data in the library  also include the possibility of normalising the input continuous  features via a <code>BatchNorm</code> or a <code>LayerNorm</code>.  Param alias:  <code>scale_cont_cols</code>.</p> <code>False</code> <code>already_standard</code> <code>Optional[List[str]]</code> <p> note: this arg will be removed in upcoming  releases. Please use <code>cols_to_scale</code> instead.  List with the  name of the continuous cols that do not need to be  scaled/standarised.</p> <code>None</code> <p>Other Parameters:</p> Name Type Description <code>**kwargs</code> <p><code>pd.cut</code> and <code>StandardScaler</code> related args</p> <p>Attributes:</p> Name Type Description <code>cat_cols</code> <code>List[str]</code> <p>List containing the names of the categorical columns after processing.</p> <code>cols_and_bins</code> <code>Optional[Dict[str, Union[int, List[float]]]]</code> <p>Dictionary containing the quantization setup after processing if quantization is requested. This is derived from the quantization_setup parameter.</p> <code>quant_args</code> <code>Dict</code> <p>Dictionary containing arguments passed to pandas.cut() function for quantization.</p> <code>scale_args</code> <code>Dict</code> <p>Dictionary containing arguments passed to StandardScaler.</p> <code>label_encoder</code> <code>LabelEncoder</code> <p>Instance of LabelEncoder used to encode categorical variables. See <code>pytorch_widedeep.utils.dense_utils.LabelEncoder</code>.</p> <code>cat_embed_input</code> <code>List</code> <p>List of tuples with the column name, number of individual values for that column and, if <code>with_attention</code> is set to <code>False</code>, the corresponding embeddings dim, e.g. [('education', 16, 10), ('relationship', 6, 8), ...].</p> <code>standardize_cols</code> <code>List</code> <p>List of the columns that will be standardized.</p> <code>scaler</code> <code>StandardScaler</code> <p>An instance of <code>sklearn.preprocessing.StandardScaler</code> if standardization is requested.</p> <code>column_idx</code> <code>Dict</code> <p>Dictionary where keys are column names and values are column indexes. This is necessary to slice tensors.</p> <code>quantizer</code> <code>Quantizer</code> <p>An instance of <code>Quantizer</code> if quantization is requested.</p> <code>is_fitted</code> <code>bool</code> <p>Boolean indicating if the preprocessor has been fitted.</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import pandas as pd\n&gt;&gt;&gt; import numpy as np\n&gt;&gt;&gt; from pytorch_widedeep.preprocessing import ChunkTabPreprocessor\n&gt;&gt;&gt; np.random.seed(42)\n&gt;&gt;&gt; chunk_df = pd.DataFrame({'cat_col': np.random.choice(['A', 'B', 'C'], size=8),\n... 'cont_col': np.random.uniform(1, 100, size=8)})\n&gt;&gt;&gt; cat_embed_cols = [('cat_col',4)]\n&gt;&gt;&gt; cont_cols = ['cont_col']\n&gt;&gt;&gt; tab_preprocessor = ChunkTabPreprocessor(\n... n_chunks=1, cat_embed_cols=cat_embed_cols, continuous_cols=cont_cols\n... )\n&gt;&gt;&gt; X_tab = tab_preprocessor.fit_transform(chunk_df)\n&gt;&gt;&gt; tab_preprocessor.cat_embed_cols\n[('cat_col', 4)]\n&gt;&gt;&gt; tab_preprocessor.column_idx\n{'cat_col': 0, 'cont_col': 1}\n</code></pre> Source code in <code>pytorch_widedeep/preprocessing/tab_preprocessor.py</code> <pre><code>class ChunkTabPreprocessor(TabPreprocessor):\n    r\"\"\"Preprocessor to prepare the `deeptabular` component input dataset\n\n    Parameters\n    ----------\n    n_chunks: int\n        Number of chunks that the tabular dataset is divided by.\n    cat_embed_cols: List, default = None\n        List containing the name of the categorical columns that will be\n        represented by embeddings (e.g. _['education', 'relationship', ...]_) or\n        a Tuple with the name and the embedding dimension (e.g.: _[\n        ('education',32), ('relationship',16), ...]_). &lt;br/&gt; Param alias:\n        `embed_cols`\n    continuous_cols: List, default = None\n        List with the name of the continuous cols\n    cols_and_bins: Dict, default = None\n        Continuous columns can be turned into categorical via\n        `pd.cut`. 'cols_and_bins' is dictionary where the keys are the column\n        names to quantize and the values are a list of scalars indicating the\n        bin edges. &lt;br/&gt; Param alias: `quantization_setup`\n    cols_to_scale: List, default = None,\n        List with the names of the columns that will be standarised via\n        sklearn's `StandardScaler`\n    default_embed_dim: int, default=16\n        Dimension for the embeddings if the embed_dim is not provided in the\n        `cat_embed_cols` parameter and `auto_embed_dim` is set to\n        `False`.\n    with_attention: bool, default = False\n        Boolean indicating whether the preprocessed data will be passed to an\n        attention-based model (more precisely a model where all embeddings\n        must have the same dimensions). If `True`, the param `cat_embed_cols`\n        must just be a list containing just the categorical column names:\n        e.g.\n        _['education', 'relationship', ...]_. This is because they will all be\n         encoded using embeddings of the same dim, which will be specified\n         later when the model is defined. &lt;br/&gt; Param alias:\n         `for_transformer`\n    with_cls_token: bool, default = False\n        Boolean indicating if a `'[CLS]'` token will be added to the dataset\n        when using attention-based models. The final hidden state\n        corresponding to this token is used as the aggregated representation\n        for classification and regression tasks. If not, the categorical\n        (and continuous embeddings if present) will be concatenated before\n        being passed to the final MLP (if present).\n    shared_embed: bool, default = False\n        Boolean indicating if the embeddings will be \"shared\" when using\n        attention-based models. The idea behind `shared_embed` is\n        described in the Appendix A in the [TabTransformer paper](https://arxiv.org/abs/2012.06678):\n        _'The goal of having column embedding is to enable the model to\n        distinguish the classes in one column from those in the other\n        columns'_. In other words, the idea is to let the model learn which\n        column is embedded at the time. See: `pytorch_widedeep.models.transformers._layers.SharedEmbeddings`.\n    verbose: int, default = 1\n    scale: bool, default = False\n        :information_source: **note**: this arg will be removed in upcoming\n         releases. Please use `cols_to_scale` instead. &lt;br/&gt; Bool indicating\n         whether or not to scale/standarise continuous cols. It is important\n         to emphasize that all the DL models for tabular data in the library\n         also include the possibility of normalising the input continuous\n         features via a `BatchNorm` or a `LayerNorm`. &lt;br/&gt; Param alias:\n         `scale_cont_cols`.\n    already_standard: List, default = None\n        :information_source: **note**: this arg will be removed in upcoming\n         releases. Please use `cols_to_scale` instead. &lt;br/&gt; List with the\n         name of the continuous cols that do not need to be\n         scaled/standarised.\n\n    Other Parameters\n    ----------------\n    **kwargs: dict\n        `pd.cut` and `StandardScaler` related args\n\n    Attributes\n    ----------\n    cat_cols: List[str]\n        List containing the names of the categorical columns after processing.\n    cols_and_bins: Optional[Dict[str, Union[int, List[float]]]]\n        Dictionary containing the quantization setup after processing if\n        quantization is requested. This is derived from the\n        quantization_setup parameter.\n    quant_args: Dict\n        Dictionary containing arguments passed to pandas.cut() function for quantization.\n    scale_args: Dict\n        Dictionary containing arguments passed to StandardScaler.\n    label_encoder: LabelEncoder\n        Instance of LabelEncoder used to encode categorical variables.\n        See `pytorch_widedeep.utils.dense_utils.LabelEncoder`.\n    cat_embed_input: List\n        List of tuples with the column name, number of individual values for\n        that column and, if `with_attention` is set to `False`, the\n        corresponding embeddings dim, e.g. _[('education', 16, 10),\n        ('relationship', 6, 8), ...]_.\n    standardize_cols: List\n        List of the columns that will be standardized.\n    scaler: StandardScaler\n        An instance of `sklearn.preprocessing.StandardScaler` if standardization\n        is requested.\n    column_idx: Dict\n        Dictionary where keys are column names and values are column indexes.\n        This is necessary to slice tensors.\n    quantizer: Quantizer\n        An instance of `Quantizer` if quantization is requested.\n    is_fitted: bool\n        Boolean indicating if the preprocessor has been fitted.\n\n    Examples\n    --------\n    &gt;&gt;&gt; import pandas as pd\n    &gt;&gt;&gt; import numpy as np\n    &gt;&gt;&gt; from pytorch_widedeep.preprocessing import ChunkTabPreprocessor\n    &gt;&gt;&gt; np.random.seed(42)\n    &gt;&gt;&gt; chunk_df = pd.DataFrame({'cat_col': np.random.choice(['A', 'B', 'C'], size=8),\n    ... 'cont_col': np.random.uniform(1, 100, size=8)})\n    &gt;&gt;&gt; cat_embed_cols = [('cat_col',4)]\n    &gt;&gt;&gt; cont_cols = ['cont_col']\n    &gt;&gt;&gt; tab_preprocessor = ChunkTabPreprocessor(\n    ... n_chunks=1, cat_embed_cols=cat_embed_cols, continuous_cols=cont_cols\n    ... )\n    &gt;&gt;&gt; X_tab = tab_preprocessor.fit_transform(chunk_df)\n    &gt;&gt;&gt; tab_preprocessor.cat_embed_cols\n    [('cat_col', 4)]\n    &gt;&gt;&gt; tab_preprocessor.column_idx\n    {'cat_col': 0, 'cont_col': 1}\n    \"\"\"\n\n    @alias(\"with_attention\", [\"for_transformer\"])\n    @alias(\"cat_embed_cols\", [\"embed_cols\"])\n    @alias(\"scale\", [\"scale_cont_cols\"])\n    @alias(\"cols_and_bins\", [\"quantization_setup\"])\n    def __init__(\n        self,\n        n_chunks: int,\n        cat_embed_cols: Optional[Union[List[str], List[Tuple[str, int]]]] = None,\n        continuous_cols: Optional[List[str]] = None,\n        cols_and_bins: Optional[Dict[str, List[float]]] = None,\n        cols_to_scale: Optional[Union[List[str], str]] = None,\n        default_embed_dim: int = 16,\n        with_attention: bool = False,\n        with_cls_token: bool = False,\n        shared_embed: bool = False,\n        verbose: int = 1,\n        *,\n        scale: bool = False,\n        already_standard: Optional[List[str]] = None,\n        **kwargs,\n    ):\n        super(ChunkTabPreprocessor, self).__init__(\n            cat_embed_cols=cat_embed_cols,\n            continuous_cols=continuous_cols,\n            quantization_setup=None,\n            cols_to_scale=cols_to_scale,\n            auto_embed_dim=False,\n            embedding_rule=\"google\",  # does not matter, irrelevant\n            default_embed_dim=default_embed_dim,\n            with_attention=with_attention,\n            with_cls_token=with_cls_token,\n            shared_embed=shared_embed,\n            verbose=verbose,\n            scale=scale,\n            already_standard=already_standard,\n            **kwargs,\n        )\n\n        self.n_chunks = n_chunks\n        self.chunk_counter = 0\n\n        self.cols_and_bins = cols_and_bins  # type: ignore[assignment]\n        if self.cols_and_bins is not None:\n            self.quantizer = Quantizer(self.cols_and_bins, **self.quant_args)\n\n        self.embed_prepared = False\n        self.continuous_prepared = False\n\n    def partial_fit(self, df: pd.DataFrame) -&gt; \"ChunkTabPreprocessor\":  # noqa: C901\n        # df here, and throughout the class, is a chunk of the original df\n        self.chunk_counter += 1\n\n        df_adj = self._insert_cls_token(df) if self.with_cls_token else df.copy()\n\n        self.column_idx: Dict[str, int] = {}\n\n        # Categorical embeddings logic\n        if self.cat_embed_cols is not None:\n            if not self.embed_prepared:\n                df_cat, self.cat_embed_dim = self._prepare_categorical(df_adj)\n                self.label_encoder = LabelEncoder(\n                    columns_to_encode=df_cat.columns.tolist(),\n                    shared_embed=self.shared_embed,\n                    with_attention=self.with_attention,\n                )\n                self.label_encoder.partial_fit(df_cat)\n            else:\n                df_cat = df_adj[self.cat_cols]\n                self.label_encoder.partial_fit(df_cat)\n\n            self.column_idx.update({k: v for v, k in enumerate(df_cat.columns)})\n\n        # Continuous columns logic\n        if self.continuous_cols is not None:\n            if not self.continuous_prepared:\n                df_cont, self.cont_embed_dim = self._prepare_continuous(df_adj)\n            else:\n                df_cont = df[self.continuous_cols]\n\n            self.column_idx.update(\n                {k: v + len(self.column_idx) for v, k in enumerate(df_cont.columns)}\n            )\n\n            if self.standardize_cols is not None:\n                self.scaler.partial_fit(df_cont[self.standardize_cols].values)\n\n        if self.chunk_counter == self.n_chunks:\n            if self.cat_embed_cols is not None or self.cols_and_bins is not None:\n                self.cat_embed_input: List[\n                    Union[Tuple[str, int], Tuple[str, int, int]]\n                ] = []\n\n            if self.cat_embed_cols is not None:\n                for k, v in self.label_encoder.encoding_dict.items():\n                    if self.with_attention:\n                        self.cat_embed_input.append((k, len(v)))\n                    else:\n                        self.cat_embed_input.append((k, len(v), self.cat_embed_dim[k]))\n\n            if self.cols_and_bins is not None:\n                assert self.cont_embed_dim is not None  # just to make mypy happy\n                if self.with_attention:\n                    for col, n_cat, _ in self.cont_embed_dim:\n                        self.cat_embed_input.append((col, n_cat))\n                else:\n                    self.cat_embed_input.extend(self.cont_embed_dim)\n\n            self.is_fitted = True\n\n        return self\n\n    def fit(self, df: pd.DataFrame) -&gt; \"ChunkTabPreprocessor\":\n        # just to override the fit method in the base class. This class is not\n        # designed or thought to run fit\n        return self.partial_fit(df)\n\n    def _prepare_categorical(\n        self, df: pd.DataFrame\n    ) -&gt; Tuple[pd.DataFrame, Dict[str, int]]:\n        # When dealing with chunks we will NOT support the option of\n        # automatically define embeddings as this implies going through the\n        # entire dataset\n        if isinstance(self.cat_embed_cols[0], tuple):\n            self.cat_cols: List[str] = [emb[0] for emb in self.cat_embed_cols]\n            cat_embed_dim: Dict[str, int] = dict(self.cat_embed_cols)  # type: ignore\n        else:\n            self.cat_cols = self.cat_embed_cols  # type: ignore[assignment]\n            cat_embed_dim = {\n                e: self.default_embed_dim for e in self.cat_embed_cols  # type: ignore[misc]\n            }  # type: ignore\n\n        self.embed_prepared = True\n\n        return df[self.cat_cols], cat_embed_dim\n\n    def _prepare_continuous(\n        self, df: pd.DataFrame\n    ) -&gt; Tuple[pd.DataFrame, Optional[List[Tuple[str, int, int]]]]:\n        # Standardization logic\n        if self.cols_to_scale is not None:\n            self.standardize_cols = (\n                self.cols_to_scale\n                if self.cols_to_scale != \"all\"\n                else self.continuous_cols\n            )\n        elif self.scale:\n            if self.already_standard is not None:\n                self.standardize_cols = [\n                    c\n                    for c in self.continuous_cols  # type: ignore[misc]\n                    if c not in self.already_standard\n                ]\n            else:\n                self.standardize_cols = self.continuous_cols\n        else:\n            self.standardize_cols = None\n\n        if self.standardize_cols is not None:\n            self.scaler = StandardScaler(**self.scale_args)\n        elif self.verbose:\n            warnings.warn(\"Continuous columns will not be normalised\")\n\n        # Quantization logic\n        if self.cols_and_bins is not None:\n            # the quantized columns are then treated as categorical\n            quant_cont_embed_input: Optional[List[Tuple[str, int, int]]] = []\n            for col, val in self.cols_and_bins.items():\n                if isinstance(val, int):\n                    quant_cont_embed_input.append(\n                        (\n                            col,\n                            val,\n                            self.default_embed_dim,\n                        )\n                    )\n                else:\n                    quant_cont_embed_input.append(\n                        (\n                            col,\n                            len(val) - 1,\n                            self.default_embed_dim,\n                        )\n                    )\n        else:\n            quant_cont_embed_input = None\n\n        self.continuous_prepared = True\n\n        return df[self.continuous_cols], quant_cont_embed_input\n\n    def __repr__(self) -&gt; str:  # noqa: C901\n        list_of_params: List[str] = []\n        if self.n_chunks is not None:\n            list_of_params.append(\"n_chunks={n_chunks}\")\n        if self.cat_embed_cols is not None:\n            list_of_params.append(\"cat_embed_cols={cat_embed_cols}\")\n        if self.continuous_cols is not None:\n            list_of_params.append(\"continuous_cols={continuous_cols}\")\n        if self.cols_and_bins is not None:\n            list_of_params.append(\"cols_and_bins={cols_and_bins}\")\n        if self.cols_to_scale is not None:\n            list_of_params.append(\"cols_to_scale={cols_to_scale}\")\n        if self.default_embed_dim != 16:\n            list_of_params.append(\"default_embed_dim={default_embed_dim}\")\n        if self.with_attention:\n            list_of_params.append(\"with_attention={with_attention}\")\n        if self.with_cls_token:\n            list_of_params.append(\"with_cls_token={with_cls_token}\")\n        if self.shared_embed:\n            list_of_params.append(\"shared_embed={shared_embed}\")\n        if self.verbose != 1:\n            list_of_params.append(\"verbose={verbose}\")\n        if self.scale:\n            list_of_params.append(\"scale={scale}\")\n        if self.already_standard is not None:\n            list_of_params.append(\"already_standard={already_standard}\")\n        if len(self.quant_args) &gt; 0:\n            list_of_params.append(\n                \", \".join([f\"{k}\" + \"=\" + f\"{v}\" for k, v in self.quant_args.items()])\n            )\n        if len(self.scale_args) &gt; 0:\n            list_of_params.append(\n                \", \".join([f\"{k}\" + \"=\" + f\"{v}\" for k, v in self.scale_args.items()])\n            )\n        all_params = \", \".join(list_of_params)\n        return f\"ChunkTabPreprocessor({all_params.format(**self.__dict__)})\"\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.text_preprocessor.ChunkTextPreprocessor","title":"ChunkTextPreprocessor","text":"<p>               Bases: <code>TextPreprocessor</code></p> <p>Preprocessor to prepare the <code>deeptext</code> input dataset</p> <p>Parameters:</p> Name Type Description Default <code>text_col</code> <code>str</code> <p>column in the input dataframe containing either the texts or the filenames where the text documents are stored</p> required <code>n_chunks</code> <code>int</code> <p>Number of chunks that the text dataset is divided by.</p> required <code>root_dir</code> <code>Optional[str]</code> <p>If 'text_col' contains the filenames with the text documents, this is the path to the directory where those documents are stored.</p> <code>None</code> <code>max_vocab</code> <code>int</code> <p>Maximum number of tokens in the vocabulary</p> <code>30000</code> <code>min_freq</code> <code>int</code> <p>Minimum frequency for a token to be part of the vocabulary</p> <code>5</code> <code>maxlen</code> <code>int</code> <p>Maximum length of the tokenized sequences</p> <code>80</code> <code>pad_first</code> <code>bool</code> <p>Indicates whether the padding index will be added at the beginning or the end of the sequences</p> <code>True</code> <code>pad_idx</code> <code>int</code> <p>padding index. Fastai's Tokenizer leaves 0 for the 'unknown' token.</p> <code>1</code> <code>word_vectors_path</code> <code>Optional[str]</code> <p>Path to the pretrained word vectors</p> <code>None</code> <code>n_cpus</code> <code>Optional[int]</code> <p>number of CPUs to used during the tokenization process</p> <code>None</code> <code>verbose</code> <code>int</code> <p>Enable verbose output.</p> <code>1</code> <p>Attributes:</p> Name Type Description <code>vocab</code> <code>Vocab</code> <p>an instance of <code>pytorch_widedeep.utils.fastai_transforms.ChunkVocab</code></p> <code>embedding_matrix</code> <code>ndarray</code> <p>Array with the pretrained embeddings if <code>word_vectors_path</code> is not None</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import pandas as pd\n&gt;&gt;&gt; from pytorch_widedeep.preprocessing import ChunkTextPreprocessor\n&gt;&gt;&gt; chunk_df = pd.DataFrame({'text_column': [\"life is like a box of chocolates\",\n... \"You never know what you're gonna get\"]})\n&gt;&gt;&gt; chunk_text_preprocessor = ChunkTextPreprocessor(text_col='text_column', n_chunks=1,\n... max_vocab=25, min_freq=1, maxlen=10, verbose=0, n_cpus=1)\n&gt;&gt;&gt; processed_chunk = chunk_text_preprocessor.fit_transform(chunk_df)\n</code></pre> Source code in <code>pytorch_widedeep/preprocessing/text_preprocessor.py</code> <pre><code>class ChunkTextPreprocessor(TextPreprocessor):\n    r\"\"\"Preprocessor to prepare the ``deeptext`` input dataset\n\n    Parameters\n    ----------\n    text_col: str\n        column in the input dataframe containing either the texts or the\n        filenames where the text documents are stored\n    n_chunks: int\n        Number of chunks that the text dataset is divided by.\n    root_dir: str, Optional, default = None\n        If 'text_col' contains the filenames with the text documents, this is\n        the path to the directory where those documents are stored.\n    max_vocab: int, default=30000\n        Maximum number of tokens in the vocabulary\n    min_freq: int, default=5\n        Minimum frequency for a token to be part of the vocabulary\n    maxlen: int, default=80\n        Maximum length of the tokenized sequences\n    pad_first: bool,  default = True\n        Indicates whether the padding index will be added at the beginning or the\n        end of the sequences\n    pad_idx: int, default = 1\n        padding index. Fastai's Tokenizer leaves 0 for the 'unknown' token.\n    word_vectors_path: str, Optional\n        Path to the pretrained word vectors\n    n_cpus: int, Optional, default = None\n        number of CPUs to used during the tokenization process\n    verbose: int, default 1\n        Enable verbose output.\n\n    Attributes\n    ----------\n    vocab: Vocab\n        an instance of `pytorch_widedeep.utils.fastai_transforms.ChunkVocab`\n    embedding_matrix: np.ndarray\n        Array with the pretrained embeddings if `word_vectors_path` is not None\n\n    Examples\n    ---------\n    &gt;&gt;&gt; import pandas as pd\n    &gt;&gt;&gt; from pytorch_widedeep.preprocessing import ChunkTextPreprocessor\n    &gt;&gt;&gt; chunk_df = pd.DataFrame({'text_column': [\"life is like a box of chocolates\",\n    ... \"You never know what you're gonna get\"]})\n    &gt;&gt;&gt; chunk_text_preprocessor = ChunkTextPreprocessor(text_col='text_column', n_chunks=1,\n    ... max_vocab=25, min_freq=1, maxlen=10, verbose=0, n_cpus=1)\n    &gt;&gt;&gt; processed_chunk = chunk_text_preprocessor.fit_transform(chunk_df)\n    \"\"\"\n\n    def __init__(\n        self,\n        text_col: str,\n        n_chunks: int,\n        root_dir: Optional[str] = None,\n        max_vocab: int = 30000,\n        min_freq: int = 5,\n        maxlen: int = 80,\n        pad_first: bool = True,\n        pad_idx: int = 1,\n        already_processed: Optional[bool] = False,\n        word_vectors_path: Optional[str] = None,\n        n_cpus: Optional[int] = None,\n        verbose: int = 1,\n    ):\n        super(ChunkTextPreprocessor, self).__init__(\n            text_col=text_col,\n            max_vocab=max_vocab,\n            min_freq=min_freq,\n            maxlen=maxlen,\n            pad_first=pad_first,\n            pad_idx=pad_idx,\n            already_processed=already_processed,\n            word_vectors_path=word_vectors_path,\n            n_cpus=n_cpus,\n            verbose=verbose,\n        )\n\n        self.n_chunks = n_chunks\n        self.root_dir = root_dir\n\n        self.chunk_counter = 0\n\n        self.is_fitted = False\n\n    def partial_fit(self, df: pd.DataFrame) -&gt; \"ChunkTextPreprocessor\":\n        # df is a chunk of the original dataframe\n        self.chunk_counter += 1\n\n        texts = self._read_texts(df, self.root_dir)\n\n        tokens = get_texts(texts, self.already_processed, self.n_cpus)\n\n        if not hasattr(self, \"vocab\"):\n            self.vocab = ChunkVocab(\n                max_vocab=self.max_vocab,\n                min_freq=self.min_freq,\n                pad_idx=self.pad_idx,\n                n_chunks=self.n_chunks,\n            )\n\n        self.vocab.fit(tokens)\n\n        if self.chunk_counter == self.n_chunks:\n            if self.verbose:\n                print(\"The vocabulary contains {} tokens\".format(len(self.vocab.stoi)))\n            if self.word_vectors_path is not None:\n                self.embedding_matrix = build_embeddings_matrix(\n                    self.vocab, self.word_vectors_path, self.min_freq\n                )\n\n            self.is_fitted = True\n\n        return self\n\n    def fit(self, df: pd.DataFrame) -&gt; \"ChunkTextPreprocessor\":\n        # df is a chunk of the original dataframe\n        return self.partial_fit(df)\n\n    def __repr__(self) -&gt; str:\n        list_of_params: List[str] = [\"text_col='{text_col}'\"]\n        if self.n_chunks is not None:\n            list_of_params.append(\"n_chunks={n_chunks}\")\n        if self.root_dir is not None:\n            list_of_params.append(\"root_dir={root_dir}\")\n        list_of_params.append(\"max_vocab={max_vocab}\")\n        list_of_params.append(\"min_freq={min_freq}\")\n        list_of_params.append(\"maxlen={maxlen}\")\n        list_of_params.append(\"pad_first={pad_first}\")\n        list_of_params.append(\"pad_idx={pad_idx}\")\n        if self.word_vectors_path is not None:\n            list_of_params.append(\"word_vectors_path={word_vectors_path}\")\n        if self.n_cpus is not None:\n            list_of_params.append(\"n_cpus={n_cpus}\")\n        list_of_params.append(\"verbose={verbose}\")\n        all_params = \", \".join(list_of_params)\n        return f\"ChunkTextPreprocessor({all_params.format(**self.__dict__)})\"\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.hf_preprocessor.ChunkHFPreprocessor","title":"ChunkHFPreprocessor","text":"<p>               Bases: <code>HFPreprocessor</code></p> <p>Text processor to prepare the <code>deeptext</code> input dataset that is a wrapper around HuggingFace's tokenizers.</p> <p>Hugginface Tokenizer's are already 'trained'. Therefore, unlike the <code>ChunkTextPreprocessor</code> this is mostly identical to the <code>HFPreprocessor</code> with the only difference that the class needs a 'text_col' parameter to be passed. Also the parameter <code>encode_params</code> is not really optional when using this class. It must be passed containing at least the 'max_length' encoding parameter. This is because we need to ensure that  all sequences have the same length when encoding in chunks.</p> <p>Parameters:</p> Name Type Description Default <code>model_name</code> <code>str</code> <p>The model name from the transformers library e.g. 'bert-base-uncased'. Currently supported models are those from the families: BERT, RoBERTa, DistilBERT, ALBERT and ELECTRA.</p> required <code>text_col</code> <code>str</code> <p>The column in the input dataframe containing the text data. When using the <code>ChunkHFPreprocessor</code> the <code>text_col</code> parameter is mandatory.</p> required <code>root_dir</code> <code>Optional[str]</code> <p>The root directory where the text files are located. This is only needed if the text data is stored in text files. If the text data is stored in a column in the input dataframe, this parameter is not needed.</p> <code>None</code> <code>use_fast_tokenizer</code> <code>bool</code> <p>Whether to use the fast tokenizer from HuggingFace or not</p> <code>True</code> <code>num_workers</code> <code>Optional[int]</code> <p>Number of workers to use when preprocessing the text data. If not None, and <code>use_fast_tokenizer</code> is False, the text data will be preprocessed in parallel using the number of workers specified. If <code>use_fast_tokenizer</code> is True, this argument is ignored.</p> <code>None</code> <code>preprocessing_rules</code> <code>Optional[List[Callable[[str], str]]]</code> <p>A list of functions to be applied to the text data before encoding. This can be useful to clean the text data before encoding. For example, removing html tags, special characters, etc.</p> <code>None</code> <code>tokenizer_params</code> <code>Optional[Dict[str, Any]]</code> <p>Additional parameters to be passed to the HuggingFace's <code>PreTrainedTokenizer</code>.</p> <code>None</code> <code>encode_params</code> <code>Optional[Dict[str, Any]]</code> <p>Additional parameters to be passed to the <code>batch_encode_plus</code> method of the HuggingFace's <code>PreTrainedTokenizer</code>. In the case of the <code>ChunkHFPreprocessor</code>, this parameter is not really <code>Optional</code>. It must be passed containing at least the 'max_length' encoding parameter</p> <code>None</code> <p>Attributes:</p> Name Type Description <code>is_fitted</code> <code>bool</code> <p>Boolean indicating if the preprocessor has been fitted. This is a HuggingFacea tokenizer, so it is always considered fitted and this attribute is manually set to True internally. This parameter exists for consistency with the rest of the library and because is needed for some functionality in the library.</p> Source code in <code>pytorch_widedeep/preprocessing/hf_preprocessor.py</code> <pre><code>class ChunkHFPreprocessor(HFPreprocessor):\n    \"\"\"Text processor to prepare the ``deeptext`` input dataset that is a\n    wrapper around HuggingFace's tokenizers.\n\n    Hugginface Tokenizer's are already 'trained'. Therefore, unlike the\n    `ChunkTextPreprocessor` this is mostly identical to the `HFPreprocessor`\n    with the only difference that the class needs a 'text_col' parameter to\n    be passed. Also the parameter `encode_params` is not really optional when\n    using this class. It must be passed containing at least the\n    'max_length' encoding parameter. This is because we need to ensure that\n     all sequences have the same length when encoding in chunks.\n\n    Parameters\n    ----------\n    model_name: str\n        The model name from the transformers library e.g. _'bert-base-uncased'_.\n        Currently supported models are those from the families: BERT, RoBERTa,\n        DistilBERT, ALBERT and ELECTRA.\n    text_col: str, default = None\n        The column in the input dataframe containing the text data. When using\n        the `ChunkHFPreprocessor` the `text_col` parameter is mandatory.\n    root_dir: Optional[str], default = None\n        The root directory where the text files are located. This is only\n        needed if the text data is stored in text files. If the text data is\n        stored in a column in the input dataframe, this parameter is not\n        needed.\n    use_fast_tokenizer: bool, default = False\n        Whether to use the fast tokenizer from HuggingFace or not\n    num_workers: Optional[int], default = None\n        Number of workers to use when preprocessing the text data. If not\n        None, and `use_fast_tokenizer` is False, the text data will be\n        preprocessed in parallel using the number of workers specified. If\n        `use_fast_tokenizer` is True, this argument is ignored.\n    preprocessing_rules: Optional[List[Callable[[str], str]]], default = None\n        A list of functions to be applied to the text data before encoding.\n        This can be useful to clean the text data before encoding. For\n        example, removing html tags, special characters, etc.\n    tokenizer_params: Optional[Dict[str, Any]], default = None\n        Additional parameters to be passed to the HuggingFace's\n        `PreTrainedTokenizer`.\n    encode_params: Optional[Dict[str, Any]], default = None\n        Additional parameters to be passed to the `batch_encode_plus` method\n        of the HuggingFace's `PreTrainedTokenizer`. In the case of the\n        `ChunkHFPreprocessor`, this parameter is not really `Optional`. It\n        must be passed containing at least the 'max_length' encoding\n        parameter\n\n    Attributes\n    ----------\n    is_fitted: bool\n        Boolean indicating if the preprocessor has been fitted. This is a\n        HuggingFacea tokenizer, so it is always considered fitted and this\n        attribute is manually set to True internally. This parameter exists\n        for consistency with the rest of the library and because is needed\n        for some functionality in the library.\n\n    \"\"\"\n\n    def __init__(\n        self,\n        model_name: str,\n        *,\n        text_col: str,\n        root_dir: Optional[str] = None,\n        use_fast_tokenizer: bool = True,\n        num_workers: Optional[int] = None,\n        preprocessing_rules: Optional[List[Callable[[str], str]]] = None,\n        tokenizer_params: Optional[Dict[str, Any]] = None,\n        encode_params: Optional[Dict[str, Any]] = None,\n    ):\n        super().__init__(\n            model_name=model_name,\n            use_fast_tokenizer=use_fast_tokenizer,\n            text_col=text_col,\n            num_workers=num_workers,\n            preprocessing_rules=preprocessing_rules,\n            tokenizer_params=tokenizer_params,\n            encode_params=encode_params,\n        )\n\n        self.root_dir = root_dir\n\n        # when using in chunks encode_params is not really optional. I will\n        # review types in due time\n        if self.encode_params is None:\n            raise ValueError(\n                \"The 'encode_params' dict must be passed to the ChunkHFTokenizer \"\n                \"containing at least the 'max_length' encoding parameter\"\n            )\n\n        if \"padding\" not in self.encode_params or not self.encode_params[\"padding\"]:\n            self.encode_params[\"padding\"] = True\n\n        if (\n            \"truncation\" not in self.encode_params\n            or not self.encode_params[\"truncation\"]\n        ):\n            self.encode_params[\"truncation\"] = True\n\n    def __repr__(self):\n        return (\n            f\"ChunkHFPreprocessor(text_col={self.text_col}, model_name={self.model_name}, \"\n            f\"use_fast_tokenizer={self.use_fast_tokenizer}, num_workers={self.num_workers}, \"\n            f\"preprocessing_rules={self.preprocessing_rules}, tokenizer_params={self.tokenizer_params}, \"\n            f\"encode_params={self.encode_params}, root_dir={self.root_dir})\"\n        )\n</code></pre>"},{"location":"pytorch-widedeep/self_supervised_pretraining.html","title":"Self Supervised Pre-training for tabular data","text":"<p>In this library we have implemented two methods or routines that allow the user to use self-suerpvised pre-training for all tabular models in the library with the exception of the <code>TabPerceiver</code> (this is a particular model and self-supervised pre-training requires some adjustments that will be implemented in future versions). Please see the examples folder in the repo or the examples section in the docs for details on how to use self-supervised pre-training with this library.</p> <p>The two routines implemented are illustrated in the figures below. The first is from TabNet: Attentive Interpretable Tabular Learning. It is a 'standard' encoder-decoder architecture and and is designed here for models that do not use transformer-based architectures (or when the embeddings can all have different dimensions). The second is from SAINT: Improved Neural Networks for Tabular Data via Row Attention and Contrastive Pre-Training, it is based on Contrastive and Denoising learning and is designed for models that use transformer-based architectures (or when the embeddings all need to have the same dimension):</p> <p> </p> <p>Figure 1. Figure 2 in their paper. The caption of the original paper is included in case it is useful.</p> <p> </p> <p>Figure 2. Figure 1 in their paper. The caption of the original paper is included in case it is useful.</p> <p>Note that the self-supervised pre-trainers described below focus, of course, on the self-supervised pre-training phase, i.e. the left side in Figure 1 and the upper part in Figure 2. When combined with the <code>Trainer</code> described earlier in the documenation, one can reproduce the full process illustrated in the figures above.</p> <p>Also Note that it is beyond the scope of this docs to explain in detail these routines. In addition, to fully utilise the self-supervised trainers implemented in this library a minimum understanding of the processes as described in the papers is required. Therefore, we strongly encourage the users to have a look to the papers.</p>"},{"location":"pytorch-widedeep/self_supervised_pretraining.html#pytorch_widedeep.self_supervised_training.EncoderDecoderTrainer","title":"EncoderDecoderTrainer","text":"<p>               Bases: <code>BaseEncoderDecoderTrainer</code></p> <p>This class implements an Encoder-Decoder self-supervised 'routine' inspired by TabNet: Attentive Interpretable Tabular Learning. See Figure 1 above.</p> <p>Parameters:</p> Name Type Description Default <code>encoder</code> <code>ModelWithoutAttention</code> <p>An instance of a <code>TabMlp</code>, <code>TabResNet</code> or <code>TabNet</code> model</p> required <code>decoder</code> <code>Optional[DecoderWithoutAttention]</code> <p>An instance of  a <code>TabMlpDecoder</code>, <code>TabResNetDecoder</code> or <code>TabNetDecoder</code> model. if <code>None</code> the decoder will be automatically built as a 'simetric' model to the Encoder</p> <code>None</code> <code>masked_prob</code> <code>float</code> <p>Indicates the fraction of elements in the embedding tensor that will be masked and hence used for reconstruction</p> <code>0.2</code> <code>optimizer</code> <code>Optional[Optimizer]</code> <p>An instance of Pytorch's <code>Optimizer</code> object (e.g. <code>torch.optim.Adam ()</code>). if no optimizer is passed it will default to <code>AdamW</code>.</p> <code>None</code> <code>lr_scheduler</code> <code>Optional[LRScheduler]</code> <p>An instance of Pytorch's <code>LRScheduler</code> object (e.g <code>torch.optim.lr_scheduler.StepLR(opt, step_size=5)</code>).</p> <code>None</code> <code>callbacks</code> <code>Optional[List[Callback]]</code> <p>List with <code>Callback</code> objects. The three callbacks available in <code>pytorch-widedeep</code> are: <code>LRHistory</code>, <code>ModelCheckpoint</code> and <code>EarlyStopping</code>. This can also be a custom callback. See <code>pytorch_widedeep.callbacks.Callback</code> or the Examples folder in the repo.</p> <code>None</code> <code>verbose</code> <code>int</code> <p>Setting it to 0 will print nothing during training.</p> <code>1</code> <code>seed</code> <code>int</code> <p>Random seed to be used internally for train_test_split</p> <code>1</code> <p>Other Parameters:</p> Name Type Description <code>**kwargs</code> <p>Other infrequently used arguments that can also be passed as kwargs are:</p> <ul> <li> <p>device: <code>str</code>     string indicating the device. One of 'cpu' or 'gpu'</p> </li> <li> <p>num_workers: <code>int</code>     number of workers to be used internally by the data loaders</p> </li> <li> <p>reducelronplateau_criterion: <code>str</code>     This sets the criterion that will be used by the lr scheduler to     take a step: One of 'loss' or 'metric'. The ReduceLROnPlateau     learning rate is a bit particular.</p> </li> </ul> Source code in <code>pytorch_widedeep/self_supervised_training/encoder_decoder_trainer.py</code> <pre><code>class EncoderDecoderTrainer(BaseEncoderDecoderTrainer):\n    r\"\"\"This class implements an Encoder-Decoder self-supervised 'routine'\n    inspired by\n    [TabNet: Attentive Interpretable Tabular Learning](https://arxiv.org/abs/1908.07442).\n    See Figure 1 above.\n\n    Parameters\n    ----------\n    encoder: ModelWithoutAttention,\n        An instance of a `TabMlp`, `TabResNet` or `TabNet` model\n    decoder: Optional[DecoderWithoutAttention] = None,\n        An instance of  a `TabMlpDecoder`, `TabResNetDecoder` or\n        `TabNetDecoder` model. if `None` the decoder will be automatically\n        built as a '_simetric_' model to the Encoder\n    masked_prob: float = 0.2,\n        Indicates the fraction of elements in the embedding tensor that will\n        be masked and hence used for reconstruction\n    optimizer: Optional[Optimizer] = None,\n        An instance of Pytorch's `Optimizer` object (e.g. `torch.optim.Adam\n        ()`). if no optimizer is passed it will default to `AdamW`.\n    lr_scheduler: Optional[LRScheduler] = None,\n        An instance of Pytorch's `LRScheduler` object\n        (e.g `torch.optim.lr_scheduler.StepLR(opt, step_size=5)`).\n    callbacks: Optional[List[Callback]] = None,\n        List with `Callback` objects. The three callbacks available in\n        `pytorch-widedeep` are: `LRHistory`, `ModelCheckpoint` and\n        `EarlyStopping`. This can also be a custom callback. See\n        `pytorch_widedeep.callbacks.Callback` or the Examples folder in the\n        repo.\n    verbose: int, default=1\n        Setting it to 0 will print nothing during training.\n    seed: int, default=1\n        Random seed to be used internally for train_test_split\n\n    Other Parameters\n    ----------------\n    **kwargs: dict\n        Other infrequently used arguments that can also be passed as kwargs are:\n\n        - **device**: `str`&lt;br/&gt;\n            string indicating the device. One of _'cpu'_ or _'gpu'_\n\n        - **num_workers**: `int`&lt;br/&gt;\n            number of workers to be used internally by the data loaders\n\n        - **reducelronplateau_criterion**: `str`\n            This sets the criterion that will be used by the lr scheduler to\n            take a step: One of _'loss'_ or _'metric'_. The ReduceLROnPlateau\n            learning rate is a bit particular.\n\n    \"\"\"\n\n    def __init__(\n        self,\n        encoder: ModelWithoutAttention,\n        decoder: Optional[DecoderWithoutAttention] = None,\n        masked_prob: float = 0.2,\n        optimizer: Optional[Optimizer] = None,\n        lr_scheduler: Optional[LRScheduler] = None,\n        callbacks: Optional[List[Callback]] = None,\n        verbose: int = 1,\n        seed: int = 1,\n        **kwargs,\n    ):\n        super().__init__(\n            encoder=encoder,\n            decoder=decoder,\n            masked_prob=masked_prob,\n            optimizer=optimizer,\n            lr_scheduler=lr_scheduler,\n            callbacks=callbacks,\n            verbose=verbose,\n            seed=seed,\n            **kwargs,\n        )\n\n    def pretrain(\n        self,\n        X_tab: np.ndarray,\n        X_tab_val: Optional[np.ndarray] = None,\n        val_split: Optional[float] = None,\n        validation_freq: int = 1,\n        n_epochs: int = 1,\n        batch_size: int = 32,\n    ):\n        r\"\"\"Pretrain method. Can also be called using `.fit(&lt;same_args&gt;)`\n\n        Parameters\n        ----------\n        X_tab: np.ndarray,\n            tabular dataset\n        X_tab_val: np.ndarray, Optional, default = None\n            validation data\n        val_split: float, Optional. default=None\n            An alterative to passing the validation set is to use a train/val\n            split fraction via `val_split`\n        validation_freq: int, default=1\n            epochs validation frequency\n        n_epochs: int, default=1\n            number of epochs\n        batch_size: int, default=32\n            batch size\n        \"\"\"\n\n        self.batch_size = batch_size\n\n        train_set, eval_set = self._train_eval_split(X_tab, X_tab_val, val_split)\n        train_loader = DataLoader(\n            dataset=train_set, batch_size=batch_size, num_workers=self.num_workers\n        )\n        train_steps = len(train_loader)\n        if eval_set is not None:\n            eval_loader = DataLoader(\n                dataset=eval_set,\n                batch_size=batch_size,\n                num_workers=self.num_workers,\n                shuffle=False,\n            )\n            eval_steps = len(eval_loader)\n\n        self.callback_container.on_train_begin(\n            {\n                \"batch_size\": batch_size,\n                \"train_steps\": train_steps,\n                \"n_epochs\": n_epochs,\n            }\n        )\n        for epoch in range(n_epochs):\n            epoch_logs: Dict[str, float] = {}\n            self.callback_container.on_epoch_begin(epoch, logs=epoch_logs)\n\n            self.train_running_loss = 0.0\n            with trange(train_steps, disable=self.verbose != 1) as t:\n                for batch_idx, X in zip(t, train_loader):\n                    t.set_description(\"epoch %i\" % (epoch + 1))\n                    train_loss = self._train_step(X[0], batch_idx)\n                    self.callback_container.on_batch_end(batch=batch_idx)\n                    print_loss_and_metric(t, train_loss)\n\n            epoch_logs = save_epoch_logs(epoch_logs, train_loss, None, \"train\")\n\n            on_epoch_end_metric = None\n            if eval_set is not None and epoch % validation_freq == (\n                validation_freq - 1\n            ):\n                self.callback_container.on_eval_begin()\n                self.valid_running_loss = 0.0\n                with trange(eval_steps, disable=self.verbose != 1) as v:\n                    for batch_idx, X in zip(v, eval_loader):\n                        v.set_description(\"valid\")\n                        val_loss = self._eval_step(X[0], batch_idx)\n                        print_loss_and_metric(v, val_loss)\n                epoch_logs = save_epoch_logs(epoch_logs, val_loss, None, \"val\")\n                on_epoch_end_metric = val_loss\n            else:\n                if self.reducelronplateau:\n                    raise NotImplementedError(\n                        \"ReduceLROnPlateau scheduler can be used only with validation data.\"\n                    )\n\n            self.callback_container.on_epoch_end(epoch, epoch_logs, on_epoch_end_metric)\n\n            if self.early_stop:\n                self.callback_container.on_train_end(epoch_logs)\n                break\n\n        self.callback_container.on_train_end(epoch_logs)\n        self._restore_best_weights()\n        self.ed_model.train()\n\n    def fit(\n        self,\n        X_tab: np.ndarray,\n        X_tab_val: Optional[np.ndarray] = None,\n        val_split: Optional[float] = None,\n        validation_freq: int = 1,\n        n_epochs: int = 1,\n        batch_size: int = 32,\n    ):\n        return self.pretrain(\n            X_tab, X_tab_val, val_split, validation_freq, n_epochs, batch_size\n        )\n\n    def explain(self, X_tab: np.ndarray, save_step_masks: bool = False):\n        raise NotImplementedError(\n            \"The 'explain' is currently not implemented for Self Supervised Pretraining\"\n        )\n\n    def _train_step(self, X_tab: Tensor, batch_idx: int) -&gt; float:\n        X = X_tab.to(self.device)\n\n        self.optimizer.zero_grad()\n        x_embed, x_embed_rec, mask = self.ed_model(X)\n        loss = self.loss_fn(x_embed, x_embed_rec, mask)\n        loss.backward()\n        self.optimizer.step()\n\n        self.train_running_loss += loss.item()\n        avg_loss = self.train_running_loss / (batch_idx + 1)\n\n        return avg_loss\n\n    def _eval_step(self, X_tab: Tensor, batch_idx: int) -&gt; float:\n        self.ed_model.eval()\n\n        with torch.no_grad():\n            X = X_tab.to(self.device)\n\n            x_embed, x_embed_rec, mask = self.ed_model(X)\n            loss = self.loss_fn(x_embed, x_embed_rec, mask)\n\n            self.valid_running_loss += loss.item()\n            avg_loss = self.valid_running_loss / (batch_idx + 1)\n\n        return avg_loss\n\n    def _train_eval_split(\n        self,\n        X: np.ndarray,\n        X_tab_val: Optional[np.ndarray] = None,\n        val_split: Optional[float] = None,\n    ) -&gt; Tuple[TensorDataset, Optional[TensorDataset]]:\n        if X_tab_val is not None:\n            train_set = TensorDataset(torch.from_numpy(X))\n            eval_set = TensorDataset(torch.from_numpy(X_tab_val))\n        elif val_split is not None:\n            X_tr, X_tab_val = train_test_split(  # type: ignore\n                X, test_size=val_split, random_state=self.seed\n            )\n            train_set = TensorDataset(torch.from_numpy(X_tr))\n            eval_set = TensorDataset(torch.from_numpy(X_tab_val))\n        else:\n            train_set = TensorDataset(torch.from_numpy(X))\n            eval_set = None\n\n        return train_set, eval_set\n</code></pre>"},{"location":"pytorch-widedeep/self_supervised_pretraining.html#pytorch_widedeep.self_supervised_training.EncoderDecoderTrainer.pretrain","title":"pretrain","text":"<pre><code>pretrain(\n    X_tab,\n    X_tab_val=None,\n    val_split=None,\n    validation_freq=1,\n    n_epochs=1,\n    batch_size=32,\n)\n</code></pre> <p>Pretrain method. Can also be called using <code>.fit(&lt;same_args&gt;)</code></p> <p>Parameters:</p> Name Type Description Default <code>X_tab</code> <code>ndarray</code> <p>tabular dataset</p> required <code>X_tab_val</code> <code>Optional[ndarray]</code> <p>validation data</p> <code>None</code> <code>val_split</code> <code>Optional[float]</code> <p>An alterative to passing the validation set is to use a train/val split fraction via <code>val_split</code></p> <code>None</code> <code>validation_freq</code> <code>int</code> <p>epochs validation frequency</p> <code>1</code> <code>n_epochs</code> <code>int</code> <p>number of epochs</p> <code>1</code> <code>batch_size</code> <code>int</code> <p>batch size</p> <code>32</code> Source code in <code>pytorch_widedeep/self_supervised_training/encoder_decoder_trainer.py</code> <pre><code>def pretrain(\n    self,\n    X_tab: np.ndarray,\n    X_tab_val: Optional[np.ndarray] = None,\n    val_split: Optional[float] = None,\n    validation_freq: int = 1,\n    n_epochs: int = 1,\n    batch_size: int = 32,\n):\n    r\"\"\"Pretrain method. Can also be called using `.fit(&lt;same_args&gt;)`\n\n    Parameters\n    ----------\n    X_tab: np.ndarray,\n        tabular dataset\n    X_tab_val: np.ndarray, Optional, default = None\n        validation data\n    val_split: float, Optional. default=None\n        An alterative to passing the validation set is to use a train/val\n        split fraction via `val_split`\n    validation_freq: int, default=1\n        epochs validation frequency\n    n_epochs: int, default=1\n        number of epochs\n    batch_size: int, default=32\n        batch size\n    \"\"\"\n\n    self.batch_size = batch_size\n\n    train_set, eval_set = self._train_eval_split(X_tab, X_tab_val, val_split)\n    train_loader = DataLoader(\n        dataset=train_set, batch_size=batch_size, num_workers=self.num_workers\n    )\n    train_steps = len(train_loader)\n    if eval_set is not None:\n        eval_loader = DataLoader(\n            dataset=eval_set,\n            batch_size=batch_size,\n            num_workers=self.num_workers,\n            shuffle=False,\n        )\n        eval_steps = len(eval_loader)\n\n    self.callback_container.on_train_begin(\n        {\n            \"batch_size\": batch_size,\n            \"train_steps\": train_steps,\n            \"n_epochs\": n_epochs,\n        }\n    )\n    for epoch in range(n_epochs):\n        epoch_logs: Dict[str, float] = {}\n        self.callback_container.on_epoch_begin(epoch, logs=epoch_logs)\n\n        self.train_running_loss = 0.0\n        with trange(train_steps, disable=self.verbose != 1) as t:\n            for batch_idx, X in zip(t, train_loader):\n                t.set_description(\"epoch %i\" % (epoch + 1))\n                train_loss = self._train_step(X[0], batch_idx)\n                self.callback_container.on_batch_end(batch=batch_idx)\n                print_loss_and_metric(t, train_loss)\n\n        epoch_logs = save_epoch_logs(epoch_logs, train_loss, None, \"train\")\n\n        on_epoch_end_metric = None\n        if eval_set is not None and epoch % validation_freq == (\n            validation_freq - 1\n        ):\n            self.callback_container.on_eval_begin()\n            self.valid_running_loss = 0.0\n            with trange(eval_steps, disable=self.verbose != 1) as v:\n                for batch_idx, X in zip(v, eval_loader):\n                    v.set_description(\"valid\")\n                    val_loss = self._eval_step(X[0], batch_idx)\n                    print_loss_and_metric(v, val_loss)\n            epoch_logs = save_epoch_logs(epoch_logs, val_loss, None, \"val\")\n            on_epoch_end_metric = val_loss\n        else:\n            if self.reducelronplateau:\n                raise NotImplementedError(\n                    \"ReduceLROnPlateau scheduler can be used only with validation data.\"\n                )\n\n        self.callback_container.on_epoch_end(epoch, epoch_logs, on_epoch_end_metric)\n\n        if self.early_stop:\n            self.callback_container.on_train_end(epoch_logs)\n            break\n\n    self.callback_container.on_train_end(epoch_logs)\n    self._restore_best_weights()\n    self.ed_model.train()\n</code></pre>"},{"location":"pytorch-widedeep/self_supervised_pretraining.html#pytorch_widedeep.self_supervised_training.ContrastiveDenoisingTrainer","title":"ContrastiveDenoisingTrainer","text":"<p>               Bases: <code>BaseContrastiveDenoisingTrainer</code></p> <p>This class trains a Contrastive, Denoising Self Supervised 'routine' that is based on the one described in SAINT: Improved Neural Networks for Tabular Data via Row Attention and Contrastive Pre-Training, their Figure 1.</p> <p>Parameters:</p> Name Type Description Default <code>model</code> <code>ModelWithAttention</code> <p>An instance of a <code>TabTransformer</code>, <code>SAINT</code>, <code>FTTransformer</code>, <code>TabFastFormer</code>, <code>TabPerceiver</code>, <code>ContextAttentionMLP</code> and <code>SelfAttentionMLP</code>.</p> required <code>preprocessor</code> <code>TabPreprocessor</code> <p>A fitted <code>TabPreprocessor</code> object. See <code>pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor</code></p> required <code>optimizer</code> <code>Optional[Optimizer]</code> <p>An instance of Pytorch's <code>Optimizer</code> object (e.g. <code>torch.optim.Adam ()</code>). if no optimizer is passed it will default to <code>AdamW</code>.</p> <code>None</code> <code>lr_scheduler</code> <code>Optional[LRScheduler]</code> <p>An instance of Pytorch's <code>LRScheduler</code> object (e.g <code>torch.optim.lr_scheduler.StepLR(opt, step_size=5)</code>).</p> <code>None</code> <code>callbacks</code> <code>Optional[List[Callback]]</code> <p>List with <code>Callback</code> objects. The three callbacks available in <code>pytorch-widedeep</code> are: <code>LRHistory</code>, <code>ModelCheckpoint</code> and <code>EarlyStopping</code>. This can also be a custom callback. See <code>pytorch_widedeep.callbacks.Callback</code> or the Examples folder in the repo.</p> <code>None</code> <code>loss_type</code> <code>Literal[contrastive, denoising, both]</code> <p>One of 'contrastive', 'denoising' or 'both'. See SAINT: Improved Neural Networks for Tabular Data via Row Attention and Contrastive Pre-Training, their figure (1) and their equation (5).</p> <code>'both'</code> <code>projection_head1_dims</code> <code>Optional[List[int]]</code> <p>The projection heads are simply MLPs. This parameter is a list of integers with the dimensions of the MLP hidden layers. See the paper for details. Note that setting up this parameter requires some knowledge of the architecture one is using. For example, if we are representing the features with embeddings of dim 32 (i.e. the so called dimension of the model is 32), then the first dimension of the projection head must be 32 (e.g. [32, 16])</p> <code>None</code> <code>projection_head2_dims</code> <code>Optional[List[int]]</code> <p>Same as 'projection_head1_dims' for the second head</p> <code>None</code> <code>projection_heads_activation</code> <code>str</code> <p>Activation function for the projection heads</p> <code>'relu'</code> <code>cat_mlp_type</code> <code>Literal[single, multiple]</code> <p>If 'denoising' loss is used, one can choose two types of 'stacked' MLPs to process the output from the transformer-based encoder that receives 'corrupted' (cut-mixed and mixed-up) features. These are 'single' or 'multiple'. The former approach will apply a single MLP to all the categorical features while the latter will use one MLP per categorical feature</p> <code>'multiple'</code> <code>cont_mlp_type</code> <code>Literal[single, multiple]</code> <p>Same as 'cat_mlp_type' but for the continuous features</p> <code>'multiple'</code> <code>denoise_mlps_activation</code> <code>str</code> <p>activation function for the so called 'denoising mlps'.</p> <code>'relu'</code> <code>verbose</code> <code>int</code> <p>Setting it to 0 will print nothing during training.</p> <code>1</code> <code>seed</code> <code>int</code> <p>Random seed to be used internally for train_test_split</p> <code>1</code> <p>Other Parameters:</p> Name Type Description <code>**kwargs</code> <p>Other infrequently used arguments that can also be passed as kwargs are:</p> <ul> <li> <p>device: <code>str</code>     string indicating the device. One of 'cpu' or 'gpu'</p> </li> <li> <p>num_workers: <code>int</code>     number of workers to be used internally by the data loaders</p> </li> <li> <p>reducelronplateau_criterion: <code>str</code>     This sets the criterion that will be used by the lr scheduler to     take a step: One of 'loss' or 'metric'. The ReduceLROnPlateau     learning rate is a bit particular.</p> </li> </ul> Source code in <code>pytorch_widedeep/self_supervised_training/contrastive_denoising_trainer.py</code> <pre><code>class ContrastiveDenoisingTrainer(BaseContrastiveDenoisingTrainer):\n    r\"\"\"This class trains a Contrastive, Denoising Self Supervised 'routine' that\n    is based on the one described in\n    [SAINT: Improved Neural Networks for Tabular Data via Row Attention and\n    Contrastive Pre-Training](https://arxiv.org/abs/2106.01342), their Figure 1.\n\n    Parameters\n    ----------\n    model: ModelWithAttention,\n        An instance of a `TabTransformer`, `SAINT`, `FTTransformer`,\n        `TabFastFormer`, `TabPerceiver`, `ContextAttentionMLP` and\n        `SelfAttentionMLP`.\n    preprocessor: `TabPreprocessor`\n        A fitted `TabPreprocessor` object. See\n        `pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor`\n    optimizer: Optional[Optimizer] = None,\n        An instance of Pytorch's `Optimizer` object (e.g. `torch.optim.Adam\n        ()`). if no optimizer is passed it will default to `AdamW`.\n    lr_scheduler: Optional[LRScheduler] = None,\n        An instance of Pytorch's `LRScheduler` object\n        (e.g `torch.optim.lr_scheduler.StepLR(opt, step_size=5)`).\n    callbacks: Optional[List[Callback]] = None,\n        List with `Callback` objects. The three callbacks available in\n        `pytorch-widedeep` are: `LRHistory`, `ModelCheckpoint` and\n        `EarlyStopping`. This can also be a custom callback. See\n        `pytorch_widedeep.callbacks.Callback` or the Examples folder in the\n        repo.\n    loss_type: str, default = \"both\"\n        One of '_contrastive_', '_denoising_' or '_both_'. See [SAINT: Improved\n        Neural Networks for Tabular Data via Row Attention and Contrastive\n        Pre-Training](https://arxiv.org/abs/2203.05556), their figure (1)\n        and their equation (5).\n    projection_head1_dims: list, Optional, default = None\n        The projection heads are simply MLPs. This parameter is a list\n        of integers with the dimensions of the MLP hidden layers. See the\n        [paper](https://arxiv.org/abs/2203.05556) for details. Note that\n        setting up this parameter requires some knowledge of the architecture\n        one is using. For example, if we are representing the features with\n        embeddings of dim 32 (i.e. the so called dimension of the model is\n        32), then the first dimension of the projection head must be 32 (e.g.\n        [32, 16])\n    projection_head2_dims: list, Optional, default = None\n        Same as '_projection_head1_dims_' for the second head\n    projection_heads_activation: str, default = \"relu\"\n        Activation function for the projection heads\n    cat_mlp_type: str, default = \"multiple\"\n        If '_denoising_' loss is used, one can choose two types of 'stacked'\n        MLPs to process the output from the transformer-based encoder that\n        receives 'corrupted' (cut-mixed and mixed-up) features. These\n        are '_single_' or '_multiple_'. The former approach will apply a single\n        MLP to all the categorical features while the latter will use one MLP\n        per categorical feature\n    cont_mlp_type: str, default = \"multiple\"\n        Same as 'cat_mlp_type' but for the continuous features\n    denoise_mlps_activation: str, default = \"relu\"\n        activation function for the so called 'denoising mlps'.\n    verbose: int, default=1\n        Setting it to 0 will print nothing during training.\n    seed: int, default=1\n        Random seed to be used internally for train_test_split\n\n    Other Parameters\n    ----------------\n    **kwargs: dict\n        Other infrequently used arguments that can also be passed as kwargs are:\n\n        - **device**: `str`&lt;br/&gt;\n            string indicating the device. One of _'cpu'_ or _'gpu'_\n\n        - **num_workers**: `int`&lt;br/&gt;\n            number of workers to be used internally by the data loaders\n\n        - **reducelronplateau_criterion**: `str`\n            This sets the criterion that will be used by the lr scheduler to\n            take a step: One of _'loss'_ or _'metric'_. The ReduceLROnPlateau\n            learning rate is a bit particular.\n\n    \"\"\"\n\n    def __init__(\n        self,\n        model: ModelWithAttention,\n        preprocessor: TabPreprocessor,\n        optimizer: Optional[Optimizer] = None,\n        lr_scheduler: Optional[LRScheduler] = None,\n        callbacks: Optional[List[Callback]] = None,\n        loss_type: Literal[\"contrastive\", \"denoising\", \"both\"] = \"both\",\n        projection_head1_dims: Optional[List[int]] = None,\n        projection_head2_dims: Optional[List[int]] = None,\n        projection_heads_activation: str = \"relu\",\n        cat_mlp_type: Literal[\"single\", \"multiple\"] = \"multiple\",\n        cont_mlp_type: Literal[\"single\", \"multiple\"] = \"multiple\",\n        denoise_mlps_activation: str = \"relu\",\n        verbose: int = 1,\n        seed: int = 1,\n        **kwargs,\n    ):\n        super().__init__(\n            model=model,\n            preprocessor=preprocessor,\n            loss_type=loss_type,\n            optimizer=optimizer,\n            lr_scheduler=lr_scheduler,\n            callbacks=callbacks,\n            projection_head1_dims=projection_head1_dims,\n            projection_head2_dims=projection_head2_dims,\n            projection_heads_activation=projection_heads_activation,\n            cat_mlp_type=cat_mlp_type,\n            cont_mlp_type=cont_mlp_type,\n            denoise_mlps_activation=denoise_mlps_activation,\n            verbose=verbose,\n            seed=seed,\n            **kwargs,\n        )\n\n    def pretrain(\n        self,\n        X_tab: np.ndarray,\n        X_tab_val: Optional[np.ndarray] = None,\n        val_split: Optional[float] = None,\n        validation_freq: int = 1,\n        n_epochs: int = 1,\n        batch_size: int = 32,\n    ):\n        r\"\"\"Pretrain method. Can also be called using `.fit(&lt;same_args&gt;)`\n\n        Parameters\n        ----------\n        X_tab: np.ndarray,\n            tabular dataset\n        X_tab_val: np.ndarray, Optional, default = None\n            validation data. Note that, although it is possible to use\n            contrastive-denoising training with a validation set, such set\n            must include feature values that are _all_ seen in the training\n            set in the case of the categorical columns. This is because the\n            values of the columns themselves will be used as targets when\n            computing the loss. Therefore, if a new category is present in\n            the validation set that was not seen in training this will\n            effectively be like trying to predict a new, never seen category\n            (and Pytorch will throw an error)\n        val_split: float, Optional. default=None\n            An alterative to passing the validation set is to use a train/val\n            split fraction via `val_split`\n        validation_freq: int, default=1\n            epochs validation frequency\n        n_epochs: int, default=1\n            number of epochs\n        batch_size: int, default=32\n            batch size\n        \"\"\"\n\n        self.batch_size = batch_size\n\n        train_set, eval_set = self._train_eval_split(X_tab, X_tab_val, val_split)\n        train_loader = DataLoader(\n            dataset=train_set, batch_size=batch_size, num_workers=self.num_workers\n        )\n        train_steps = len(train_loader)\n        if eval_set is not None:\n            eval_loader = DataLoader(\n                dataset=eval_set,\n                batch_size=batch_size,\n                num_workers=self.num_workers,\n                shuffle=False,\n            )\n            eval_steps = len(eval_loader)\n\n        self.callback_container.on_train_begin(\n            {\n                \"batch_size\": batch_size,\n                \"train_steps\": train_steps,\n                \"n_epochs\": n_epochs,\n            }\n        )\n        for epoch in range(n_epochs):\n            epoch_logs: Dict[str, float] = {}\n            self.callback_container.on_epoch_begin(epoch, logs=epoch_logs)\n\n            self.train_running_loss = 0.0\n            with trange(train_steps, disable=self.verbose != 1) as t:\n                for batch_idx, X in zip(t, train_loader):\n                    t.set_description(\"epoch %i\" % (epoch + 1))\n                    train_loss = self._train_step(X[0], batch_idx)\n                    self.callback_container.on_batch_end(batch=batch_idx)\n                    print_loss_and_metric(t, train_loss)\n\n            epoch_logs = save_epoch_logs(epoch_logs, train_loss, None, \"train\")\n\n            on_epoch_end_metric = None\n            if eval_set is not None and epoch % validation_freq == (\n                validation_freq - 1\n            ):\n                self.callback_container.on_eval_begin()\n                self.valid_running_loss = 0.0\n                with trange(eval_steps, disable=self.verbose != 1) as v:\n                    for batch_idx, X in zip(v, eval_loader):\n                        v.set_description(\"valid\")\n                        val_loss = self._eval_step(X[0], batch_idx)\n                        print_loss_and_metric(v, val_loss)\n                epoch_logs = save_epoch_logs(epoch_logs, val_loss, None, \"val\")\n                on_epoch_end_metric = val_loss\n            else:\n                if self.reducelronplateau:\n                    raise NotImplementedError(\n                        \"ReduceLROnPlateau scheduler can be used only with validation data.\"\n                    )\n\n            self.callback_container.on_epoch_end(epoch, epoch_logs, on_epoch_end_metric)\n\n            if self.early_stop:\n                self.callback_container.on_train_end(epoch_logs)\n                break\n\n        self.callback_container.on_train_end(epoch_logs)\n        self._restore_best_weights()\n        self.cd_model.train()\n\n    def fit(\n        self,\n        X_tab: np.ndarray,\n        X_tab_val: Optional[np.ndarray] = None,\n        val_split: Optional[float] = None,\n        validation_freq: int = 1,\n        n_epochs: int = 1,\n        batch_size: int = 32,\n    ):\n        return self.pretrain(\n            X_tab, X_tab_val, val_split, validation_freq, n_epochs, batch_size\n        )\n\n    def _train_step(self, X_tab: Tensor, batch_idx: int) -&gt; float:\n        X = X_tab.to(self.device)\n\n        self.optimizer.zero_grad()\n        g_projs, cat_x_and_x_, cont_x_and_x_ = self.cd_model(X)\n        loss = self._compute_loss(g_projs, cat_x_and_x_, cont_x_and_x_)\n        loss.backward()\n        self.optimizer.step()\n\n        self.train_running_loss += loss.item()\n        avg_loss = self.train_running_loss / (batch_idx + 1)\n\n        return avg_loss\n\n    def _eval_step(self, X_tab: Tensor, batch_idx: int) -&gt; float:\n        self.cd_model.eval()\n\n        with torch.no_grad():\n            X = X_tab.to(self.device)\n\n            g_projs, cat_x_and_x_, cont_x_and_x_ = self.cd_model(X)\n            loss = self._compute_loss(g_projs, cat_x_and_x_, cont_x_and_x_)\n\n            self.valid_running_loss += loss.item()\n            avg_loss = self.valid_running_loss / (batch_idx + 1)\n\n        return avg_loss\n\n    def _train_eval_split(\n        self,\n        X: np.ndarray,\n        X_tab_val: Optional[np.ndarray] = None,\n        val_split: Optional[float] = None,\n    ) -&gt; Tuple[TensorDataset, Optional[TensorDataset]]:\n        if X_tab_val is not None:\n            train_set = TensorDataset(torch.from_numpy(X))\n            eval_set = TensorDataset(torch.from_numpy(X_tab_val))\n        elif val_split is not None:\n            X_tr, X_tab_val = train_test_split(  # type: ignore\n                X, test_size=val_split, random_state=self.seed\n            )\n            train_set = TensorDataset(torch.from_numpy(X_tr))\n            eval_set = TensorDataset(torch.from_numpy(X_tab_val))\n        else:\n            train_set = TensorDataset(torch.from_numpy(X))\n            eval_set = None\n\n        return train_set, eval_set\n</code></pre>"},{"location":"pytorch-widedeep/self_supervised_pretraining.html#pytorch_widedeep.self_supervised_training.ContrastiveDenoisingTrainer.pretrain","title":"pretrain","text":"<pre><code>pretrain(\n    X_tab,\n    X_tab_val=None,\n    val_split=None,\n    validation_freq=1,\n    n_epochs=1,\n    batch_size=32,\n)\n</code></pre> <p>Pretrain method. Can also be called using <code>.fit(&lt;same_args&gt;)</code></p> <p>Parameters:</p> Name Type Description Default <code>X_tab</code> <code>ndarray</code> <p>tabular dataset</p> required <code>X_tab_val</code> <code>Optional[ndarray]</code> <p>validation data. Note that, although it is possible to use contrastive-denoising training with a validation set, such set must include feature values that are all seen in the training set in the case of the categorical columns. This is because the values of the columns themselves will be used as targets when computing the loss. Therefore, if a new category is present in the validation set that was not seen in training this will effectively be like trying to predict a new, never seen category (and Pytorch will throw an error)</p> <code>None</code> <code>val_split</code> <code>Optional[float]</code> <p>An alterative to passing the validation set is to use a train/val split fraction via <code>val_split</code></p> <code>None</code> <code>validation_freq</code> <code>int</code> <p>epochs validation frequency</p> <code>1</code> <code>n_epochs</code> <code>int</code> <p>number of epochs</p> <code>1</code> <code>batch_size</code> <code>int</code> <p>batch size</p> <code>32</code> Source code in <code>pytorch_widedeep/self_supervised_training/contrastive_denoising_trainer.py</code> <pre><code>def pretrain(\n    self,\n    X_tab: np.ndarray,\n    X_tab_val: Optional[np.ndarray] = None,\n    val_split: Optional[float] = None,\n    validation_freq: int = 1,\n    n_epochs: int = 1,\n    batch_size: int = 32,\n):\n    r\"\"\"Pretrain method. Can also be called using `.fit(&lt;same_args&gt;)`\n\n    Parameters\n    ----------\n    X_tab: np.ndarray,\n        tabular dataset\n    X_tab_val: np.ndarray, Optional, default = None\n        validation data. Note that, although it is possible to use\n        contrastive-denoising training with a validation set, such set\n        must include feature values that are _all_ seen in the training\n        set in the case of the categorical columns. This is because the\n        values of the columns themselves will be used as targets when\n        computing the loss. Therefore, if a new category is present in\n        the validation set that was not seen in training this will\n        effectively be like trying to predict a new, never seen category\n        (and Pytorch will throw an error)\n    val_split: float, Optional. default=None\n        An alterative to passing the validation set is to use a train/val\n        split fraction via `val_split`\n    validation_freq: int, default=1\n        epochs validation frequency\n    n_epochs: int, default=1\n        number of epochs\n    batch_size: int, default=32\n        batch size\n    \"\"\"\n\n    self.batch_size = batch_size\n\n    train_set, eval_set = self._train_eval_split(X_tab, X_tab_val, val_split)\n    train_loader = DataLoader(\n        dataset=train_set, batch_size=batch_size, num_workers=self.num_workers\n    )\n    train_steps = len(train_loader)\n    if eval_set is not None:\n        eval_loader = DataLoader(\n            dataset=eval_set,\n            batch_size=batch_size,\n            num_workers=self.num_workers,\n            shuffle=False,\n        )\n        eval_steps = len(eval_loader)\n\n    self.callback_container.on_train_begin(\n        {\n            \"batch_size\": batch_size,\n            \"train_steps\": train_steps,\n            \"n_epochs\": n_epochs,\n        }\n    )\n    for epoch in range(n_epochs):\n        epoch_logs: Dict[str, float] = {}\n        self.callback_container.on_epoch_begin(epoch, logs=epoch_logs)\n\n        self.train_running_loss = 0.0\n        with trange(train_steps, disable=self.verbose != 1) as t:\n            for batch_idx, X in zip(t, train_loader):\n                t.set_description(\"epoch %i\" % (epoch + 1))\n                train_loss = self._train_step(X[0], batch_idx)\n                self.callback_container.on_batch_end(batch=batch_idx)\n                print_loss_and_metric(t, train_loss)\n\n        epoch_logs = save_epoch_logs(epoch_logs, train_loss, None, \"train\")\n\n        on_epoch_end_metric = None\n        if eval_set is not None and epoch % validation_freq == (\n            validation_freq - 1\n        ):\n            self.callback_container.on_eval_begin()\n            self.valid_running_loss = 0.0\n            with trange(eval_steps, disable=self.verbose != 1) as v:\n                for batch_idx, X in zip(v, eval_loader):\n                    v.set_description(\"valid\")\n                    val_loss = self._eval_step(X[0], batch_idx)\n                    print_loss_and_metric(v, val_loss)\n            epoch_logs = save_epoch_logs(epoch_logs, val_loss, None, \"val\")\n            on_epoch_end_metric = val_loss\n        else:\n            if self.reducelronplateau:\n                raise NotImplementedError(\n                    \"ReduceLROnPlateau scheduler can be used only with validation data.\"\n                )\n\n        self.callback_container.on_epoch_end(epoch, epoch_logs, on_epoch_end_metric)\n\n        if self.early_stop:\n            self.callback_container.on_train_end(epoch_logs)\n            break\n\n    self.callback_container.on_train_end(epoch_logs)\n    self._restore_best_weights()\n    self.cd_model.train()\n</code></pre>"},{"location":"pytorch-widedeep/tab2vec.html","title":"Tab2Vec","text":""},{"location":"pytorch-widedeep/tab2vec.html#pytorch_widedeep.tab2vec.Tab2Vec","title":"Tab2Vec","text":"<p>Class to transform an input dataframe into vectorized form.</p> <p>This class will take an input dataframe in the form of the dataframe used for training, and it will turn it into a vectorised form based on the processing applied by the model to the categorical and continuous columns.</p> <p> NOTE: Currently this class is only implemented  for the deeptabular component. Therefore, if the input dataframe has a  text column or a column with the path to images, these will be ignored.  We will be adding these functionalities in future versions</p> <p>Parameters:</p> Name Type Description Default <code>model</code> <code>Union[WideDeep, BayesianWide, BayesianTabMlp]</code> <p><code>WideDeep</code>, <code>BayesianWide</code> or <code>BayesianTabMlp</code> model. Must be trained.</p> required <code>tab_preprocessor</code> <code>TabPreprocessor</code> <p><code>TabPreprocessor</code> object. Must be fitted.</p> required <code>return_dataframe</code> <code>bool</code> <p>Boolean indicating of the returned object(s) will be array(s) or pandas dataframe(s)</p> <code>False</code> <p>Attributes:</p> Name Type Description <code>vectorizer</code> <code>Module</code> <p>Torch module with the categorical and continuous encoding process</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import string\n&gt;&gt;&gt; from random import choices\n&gt;&gt;&gt; import numpy as np\n&gt;&gt;&gt; import pandas as pd\n&gt;&gt;&gt; from pytorch_widedeep import Tab2Vec\n&gt;&gt;&gt; from pytorch_widedeep.models import TabMlp, WideDeep\n&gt;&gt;&gt; from pytorch_widedeep.preprocessing import TabPreprocessor\n&gt;&gt;&gt;\n&gt;&gt;&gt; colnames = list(string.ascii_lowercase)[:4]\n&gt;&gt;&gt; cat_col1_vals = [\"a\", \"b\", \"c\"]\n&gt;&gt;&gt; cat_col2_vals = [\"d\", \"e\", \"f\"]\n&gt;&gt;&gt;\n&gt;&gt;&gt; # Create the toy input dataframe and a toy dataframe to be vectorised\n&gt;&gt;&gt; cat_inp = [np.array(choices(c, k=5)) for c in [cat_col1_vals, cat_col2_vals]]\n&gt;&gt;&gt; cont_inp = [np.round(np.random.rand(5), 2) for _ in range(2)]\n&gt;&gt;&gt; df_inp = pd.DataFrame(np.vstack(cat_inp + cont_inp).transpose(), columns=colnames)\n&gt;&gt;&gt; cat_t2v = [np.array(choices(c, k=5)) for c in [cat_col1_vals, cat_col2_vals]]\n&gt;&gt;&gt; cont_t2v = [np.round(np.random.rand(5), 2) for _ in range(2)]\n&gt;&gt;&gt; df_t2v = pd.DataFrame(np.vstack(cat_t2v + cont_t2v).transpose(), columns=colnames)\n&gt;&gt;&gt;\n&gt;&gt;&gt; # fit the TabPreprocessor\n&gt;&gt;&gt; embed_cols = [(\"a\", 2), (\"b\", 4)]\n&gt;&gt;&gt; cont_cols = [\"c\", \"d\"]\n&gt;&gt;&gt; tab_preprocessor = TabPreprocessor(cat_embed_cols=embed_cols, continuous_cols=cont_cols)\n&gt;&gt;&gt; X_tab = tab_preprocessor.fit_transform(df_inp)\n&gt;&gt;&gt;\n&gt;&gt;&gt; # define the model (and let's assume we train it)\n&gt;&gt;&gt; tabmlp = TabMlp(\n... column_idx=tab_preprocessor.column_idx,\n... cat_embed_input=tab_preprocessor.cat_embed_input,\n... continuous_cols=tab_preprocessor.continuous_cols,\n... mlp_hidden_dims=[8, 4])\n&gt;&gt;&gt; model = WideDeep(deeptabular=tabmlp)\n&gt;&gt;&gt; # ...train the model...\n&gt;&gt;&gt;\n&gt;&gt;&gt; # vectorise the dataframe\n&gt;&gt;&gt; t2v = Tab2Vec(tab_preprocessor, model, device=\"cpu\")\n&gt;&gt;&gt; X_vec = t2v.transform(df_t2v)\n</code></pre> Source code in <code>pytorch_widedeep/tab2vec.py</code> <pre><code>class Tab2Vec:\n    r\"\"\"Class to transform an input dataframe into vectorized form.\n\n    This class will take an input dataframe in the form of the dataframe used\n    for training, and it will turn it into a vectorised form based on the\n    processing applied by the model to the categorical and continuous\n    columns.\n\n    :information_source: **NOTE**: Currently this class is only implemented\n     for the deeptabular component. Therefore, if the input dataframe has a\n     text column or a column with the path to images, these will be ignored.\n     We will be adding these functionalities in future versions\n\n    Parameters\n    ----------\n    model: `WideDeep`, `BayesianWide` or `BayesianTabMlp`\n        `WideDeep`, `BayesianWide` or `BayesianTabMlp` model. Must be trained.\n    tab_preprocessor: `TabPreprocessor`\n        `TabPreprocessor` object. Must be fitted.\n    return_dataframe: bool\n        Boolean indicating of the returned object(s) will be array(s) or\n        pandas dataframe(s)\n\n    Attributes\n    ----------\n    vectorizer: nn.Module\n        Torch module with the categorical and continuous encoding process\n\n    Examples\n    --------\n    &gt;&gt;&gt; import string\n    &gt;&gt;&gt; from random import choices\n    &gt;&gt;&gt; import numpy as np\n    &gt;&gt;&gt; import pandas as pd\n    &gt;&gt;&gt; from pytorch_widedeep import Tab2Vec\n    &gt;&gt;&gt; from pytorch_widedeep.models import TabMlp, WideDeep\n    &gt;&gt;&gt; from pytorch_widedeep.preprocessing import TabPreprocessor\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; colnames = list(string.ascii_lowercase)[:4]\n    &gt;&gt;&gt; cat_col1_vals = [\"a\", \"b\", \"c\"]\n    &gt;&gt;&gt; cat_col2_vals = [\"d\", \"e\", \"f\"]\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; # Create the toy input dataframe and a toy dataframe to be vectorised\n    &gt;&gt;&gt; cat_inp = [np.array(choices(c, k=5)) for c in [cat_col1_vals, cat_col2_vals]]\n    &gt;&gt;&gt; cont_inp = [np.round(np.random.rand(5), 2) for _ in range(2)]\n    &gt;&gt;&gt; df_inp = pd.DataFrame(np.vstack(cat_inp + cont_inp).transpose(), columns=colnames)\n    &gt;&gt;&gt; cat_t2v = [np.array(choices(c, k=5)) for c in [cat_col1_vals, cat_col2_vals]]\n    &gt;&gt;&gt; cont_t2v = [np.round(np.random.rand(5), 2) for _ in range(2)]\n    &gt;&gt;&gt; df_t2v = pd.DataFrame(np.vstack(cat_t2v + cont_t2v).transpose(), columns=colnames)\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; # fit the TabPreprocessor\n    &gt;&gt;&gt; embed_cols = [(\"a\", 2), (\"b\", 4)]\n    &gt;&gt;&gt; cont_cols = [\"c\", \"d\"]\n    &gt;&gt;&gt; tab_preprocessor = TabPreprocessor(cat_embed_cols=embed_cols, continuous_cols=cont_cols)\n    &gt;&gt;&gt; X_tab = tab_preprocessor.fit_transform(df_inp)\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; # define the model (and let's assume we train it)\n    &gt;&gt;&gt; tabmlp = TabMlp(\n    ... column_idx=tab_preprocessor.column_idx,\n    ... cat_embed_input=tab_preprocessor.cat_embed_input,\n    ... continuous_cols=tab_preprocessor.continuous_cols,\n    ... mlp_hidden_dims=[8, 4])\n    &gt;&gt;&gt; model = WideDeep(deeptabular=tabmlp)\n    &gt;&gt;&gt; # ...train the model...\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; # vectorise the dataframe\n    &gt;&gt;&gt; t2v = Tab2Vec(tab_preprocessor, model, device=\"cpu\")\n    &gt;&gt;&gt; X_vec = t2v.transform(df_t2v)\n    \"\"\"\n\n    def __init__(\n        self,\n        tab_preprocessor: TabPreprocessor,\n        model: Union[WideDeep, BayesianWide, BayesianTabMlp],\n        device: str | torch.device,\n        return_dataframe: bool = False,\n        verbose: bool = False,\n    ):\n        super(Tab2Vec, self).__init__()\n\n        self._check_inputs(tab_preprocessor, model, verbose)\n\n        if isinstance(device, str):\n            self.device = torch.device(device)\n        else:\n            self.device = device\n\n        self.tab_preprocessor = tab_preprocessor\n        self.return_dataframe = return_dataframe\n        self.verbose = verbose\n\n        self.vectorizer = self._set_vectorizer(model)\n\n        self._set_dim_attributes(tab_preprocessor, model)\n\n    def fit(self, df: pd.DataFrame, target_col: Optional[str] = None) -&gt; \"Tab2Vec\":\n        r\"\"\"This is an empty method i.e. Returns the unchanged object itself. Is\n        only included for consistency in case `Tab2Vec` is used as part of a\n        Pipeline\n\n        Parameters\n        ----------\n        df: pd.DataFrame\n            DataFrame to be vectorised, i.e. the categorical and continuous\n            columns will be encoded based on the processing applied within\n            the model\n        target_col: str, Optional\n            Column name of the target_col variable. If `None` only the array of\n            predictors will be returned\n\n        Returns\n        -------\n        Tab2Vec\n        \"\"\"\n\n        return self\n\n    def transform(\n        self,\n        df: pd.DataFrame,\n        target_col: Optional[str] = None,\n    ) -&gt; Union[\n        np.ndarray,\n        Tuple[np.ndarray, np.ndarray],\n        pd.DataFrame,\n        Tuple[pd.DataFrame, pd.Series],\n    ]:\n        r\"\"\"Transforms the input dataframe into vectorized form. If a target\n        column name is passed the target values will be returned separately\n        in their corresponding type (np.ndarray or pd.DataFrame)\n\n        Parameters\n        ----------\n        df: pd.DataFrame\n            DataFrame to be vectorised, i.e. the categorical and continuous\n            columns will be encoded based on the processing applied within\n            the model\n        target_col: str, Optional\n            Column name of the target_col variable. If `None` only the array of\n            predictors will be returned\n\n        Returns\n        -------\n        Union[np.ndarray, Tuple[np.ndarray, np.ndarray], pd.DataFrame, Tuple[pd.DataFrame, pd.Series]\n            Returns eiter a numpy array with the vectorised values, or a Tuple\n            of numpy arrays with the vectorised values and the target. The\n            same applies to dataframes in case we choose to set\n            `return_dataframe = True`\n        \"\"\"\n\n        X_tab = self.tab_preprocessor.transform(df)\n\n        if self.device.type == \"mps\":\n            X = torch.from_numpy(X_tab.astype(\"float32\")).to(self.device)\n        else:\n            X = torch.from_numpy(X_tab.astype(\"float\")).to(self.device)\n\n        with torch.no_grad():\n            if self.is_tab_transformer:\n                x_vec, x_cont_not_embed = self.vectorizer(X)\n            else:\n                x_vec = self.vectorizer(X)\n                x_cont_not_embed = None\n\n        if self.tab_preprocessor.with_cls_token:\n            x_vec = x_vec[:, 1:, :]\n\n        if self.tab_preprocessor.with_attention:\n            x_vec = einops.rearrange(x_vec, \"s c e -&gt; s (c e)\")\n\n        if x_cont_not_embed is not None:\n            x_vec = torch.cat([x_vec, x_cont_not_embed], 1).detach().cpu().numpy()\n        else:\n            x_vec = x_vec.detach().cpu().numpy()\n\n        if self.return_dataframe:\n            new_colnames = self._new_colnames()\n            if target_col:\n                return pd.DataFrame(data=x_vec, columns=new_colnames), df[[target_col]]\n            else:\n                return pd.DataFrame(data=x_vec, columns=new_colnames)\n        else:\n            if target_col:\n                return x_vec, df[target_col].values\n            else:\n                return x_vec\n\n    def fit_transform(\n        self, df: pd.DataFrame, target_col: Optional[str] = None\n    ) -&gt; Union[\n        np.ndarray,\n        Tuple[np.ndarray, np.ndarray],\n        pd.DataFrame,\n        Tuple[pd.DataFrame, pd.Series],\n    ]:\n        r\"\"\"Combines `fit` and `transform`\"\"\"\n        return self.fit(df, target_col).transform(df, target_col)\n\n    def _new_colnames(self) -&gt; List[str]:\n        if self.tab_preprocessor.with_attention:\n            return self._new_colnames_with_attention()\n        else:\n            return self._new_colnames_without_attention()\n\n    def _new_colnames_with_attention(\n        self,\n    ) -&gt; List[str]:\n        cat_cols: List[str] = (\n            [\n                ei[0]\n                for ei in self.tab_preprocessor.cat_embed_input\n                if ei[0] != \"cls_token\"\n            ]\n            if self.tab_preprocessor.cat_embed_input is not None\n            else []\n        )\n\n        cont_cols: List[str] = (\n            self.tab_preprocessor.continuous_cols\n            if self.tab_preprocessor.continuous_cols is not None\n            else []\n        )\n\n        if self.are_cont_embed:\n            all_embed_cols = cat_cols + cont_cols\n        else:\n            all_embed_cols = cat_cols\n\n        new_colnames = []\n        for colname in all_embed_cols:\n            assert self.input_dim is not None\n            new_colnames.extend(\n                [\n                    \"_\".join([colname, \"embed\", str(i + 1)])\n                    for i in range(self.input_dim)\n                ]\n            )\n\n        if not self.are_cont_embed:\n            new_colnames += cont_cols\n\n        return new_colnames\n\n    def _new_colnames_without_attention(\n        self,\n    ) -&gt; List[str]:\n        new_colnames: List[str] = []\n\n        if self.tab_preprocessor.cat_embed_input is not None:\n            for colname, _, embedding_dim in self.tab_preprocessor.cat_embed_input:  # type: ignore[misc]\n                new_colnames.extend(\n                    [\n                        \"_\".join([colname, \"embed\", str(i + 1)])\n                        for i in range(embedding_dim)\n                    ]\n                )\n\n        if self.tab_preprocessor.continuous_cols is not None:\n            if self.are_cont_embed:\n                for colname in self.tab_preprocessor.continuous_cols:\n                    assert self.cont_embed_dim is not None\n                    new_colnames.extend(\n                        [\n                            \"_\".join([colname, \"embed\", str(i + 1)])\n                            for i in range(self.cont_embed_dim)\n                        ]\n                    )\n            else:\n                new_colnames += self.tab_preprocessor.continuous_cols\n\n        return new_colnames\n\n    def _check_inputs(\n        self,\n        tab_preprocessor: TabPreprocessor,\n        model: Union[WideDeep, BayesianWide, BayesianTabMlp],\n        verbose: bool,\n    ) -&gt; None:\n        if not isinstance(model, BaseBayesianModel):\n            if verbose:\n                if model.deepimage is not None or model.deeptext is not None:\n                    warnings.warn(\n                        \"Currently 'Tab2Vec' is only implemented for the 'deeptabular' component.\"\n                    )\n\n            if model.deeptabular is None:\n                raise RuntimeError(\n                    \"Currently 'Tab2Vec' is only implemented for the 'deeptabular' component.\"\n                )\n            if not tab_preprocessor.is_fitted:\n                raise RuntimeError(\n                    \"The 'tab_preprocessor' must be fitted before is passed to 'Tab2Vec'\"\n                )\n\n    def _set_vectorizer(\n        self, model: Union[WideDeep, BayesianWide, BayesianTabMlp]\n    ) -&gt; Callable:\n        if isinstance(model, BaseBayesianModel):\n            vectorizer = model._get_embeddings\n            self.is_tab_transformer = False\n        elif model.deeptabular[0].__class__.__name__.lower() == \"tabtransformer\":  # type: ignore\n            self.is_tab_transformer = True\n            vectorizer = model.deeptabular[0]._get_embeddings_tt  # type: ignore\n        else:\n            self.is_tab_transformer = False\n            vectorizer = model.deeptabular[0]._get_embeddings  # type: ignore\n        return vectorizer\n\n    def _set_dim_attributes(\n        self,\n        tab_preprocessor: TabPreprocessor,\n        model: Union[WideDeep, BayesianWide, BayesianTabMlp],\n    ) -&gt; None:\n        if isinstance(model, BaseBayesianModel):\n            self.are_cont_embed: bool = (\n                model.embed_continuous if model.embed_continuous else False\n            )\n            self.input_dim: Optional[int] = None\n            self.cont_embed_dim: Optional[int] = model.cont_embed_dim\n        else:\n            self.are_cont_embed = (\n                model.deeptabular[0].embed_continuous_method is not None  # type: ignore\n            )\n            if tab_preprocessor.with_attention:\n                self.input_dim = model.deeptabular[0].input_dim  # type: ignore\n                self.cont_embed_dim = None\n            else:\n                self.input_dim = None\n                self.cont_embed_dim = model.deeptabular[0].cont_embed_dim  # type: ignore\n</code></pre>"},{"location":"pytorch-widedeep/tab2vec.html#pytorch_widedeep.tab2vec.Tab2Vec.fit","title":"fit","text":"<pre><code>fit(df, target_col=None)\n</code></pre> <p>This is an empty method i.e. Returns the unchanged object itself. Is only included for consistency in case <code>Tab2Vec</code> is used as part of a Pipeline</p> <p>Parameters:</p> Name Type Description Default <code>df</code> <code>DataFrame</code> <p>DataFrame to be vectorised, i.e. the categorical and continuous columns will be encoded based on the processing applied within the model</p> required <code>target_col</code> <code>Optional[str]</code> <p>Column name of the target_col variable. If <code>None</code> only the array of predictors will be returned</p> <code>None</code> <p>Returns:</p> Type Description <code>Tab2Vec</code> Source code in <code>pytorch_widedeep/tab2vec.py</code> <pre><code>def fit(self, df: pd.DataFrame, target_col: Optional[str] = None) -&gt; \"Tab2Vec\":\n    r\"\"\"This is an empty method i.e. Returns the unchanged object itself. Is\n    only included for consistency in case `Tab2Vec` is used as part of a\n    Pipeline\n\n    Parameters\n    ----------\n    df: pd.DataFrame\n        DataFrame to be vectorised, i.e. the categorical and continuous\n        columns will be encoded based on the processing applied within\n        the model\n    target_col: str, Optional\n        Column name of the target_col variable. If `None` only the array of\n        predictors will be returned\n\n    Returns\n    -------\n    Tab2Vec\n    \"\"\"\n\n    return self\n</code></pre>"},{"location":"pytorch-widedeep/tab2vec.html#pytorch_widedeep.tab2vec.Tab2Vec.transform","title":"transform","text":"<pre><code>transform(df, target_col=None)\n</code></pre> <p>Transforms the input dataframe into vectorized form. If a target column name is passed the target values will be returned separately in their corresponding type (np.ndarray or pd.DataFrame)</p> <p>Parameters:</p> Name Type Description Default <code>df</code> <code>DataFrame</code> <p>DataFrame to be vectorised, i.e. the categorical and continuous columns will be encoded based on the processing applied within the model</p> required <code>target_col</code> <code>Optional[str]</code> <p>Column name of the target_col variable. If <code>None</code> only the array of predictors will be returned</p> <code>None</code> <p>Returns:</p> Type Description <code>Union[np.ndarray, Tuple[np.ndarray, np.ndarray], pd.DataFrame, Tuple[pd.DataFrame, pd.Series]</code> <p>Returns eiter a numpy array with the vectorised values, or a Tuple of numpy arrays with the vectorised values and the target. The same applies to dataframes in case we choose to set <code>return_dataframe = True</code></p> Source code in <code>pytorch_widedeep/tab2vec.py</code> <pre><code>def transform(\n    self,\n    df: pd.DataFrame,\n    target_col: Optional[str] = None,\n) -&gt; Union[\n    np.ndarray,\n    Tuple[np.ndarray, np.ndarray],\n    pd.DataFrame,\n    Tuple[pd.DataFrame, pd.Series],\n]:\n    r\"\"\"Transforms the input dataframe into vectorized form. If a target\n    column name is passed the target values will be returned separately\n    in their corresponding type (np.ndarray or pd.DataFrame)\n\n    Parameters\n    ----------\n    df: pd.DataFrame\n        DataFrame to be vectorised, i.e. the categorical and continuous\n        columns will be encoded based on the processing applied within\n        the model\n    target_col: str, Optional\n        Column name of the target_col variable. If `None` only the array of\n        predictors will be returned\n\n    Returns\n    -------\n    Union[np.ndarray, Tuple[np.ndarray, np.ndarray], pd.DataFrame, Tuple[pd.DataFrame, pd.Series]\n        Returns eiter a numpy array with the vectorised values, or a Tuple\n        of numpy arrays with the vectorised values and the target. The\n        same applies to dataframes in case we choose to set\n        `return_dataframe = True`\n    \"\"\"\n\n    X_tab = self.tab_preprocessor.transform(df)\n\n    if self.device.type == \"mps\":\n        X = torch.from_numpy(X_tab.astype(\"float32\")).to(self.device)\n    else:\n        X = torch.from_numpy(X_tab.astype(\"float\")).to(self.device)\n\n    with torch.no_grad():\n        if self.is_tab_transformer:\n            x_vec, x_cont_not_embed = self.vectorizer(X)\n        else:\n            x_vec = self.vectorizer(X)\n            x_cont_not_embed = None\n\n    if self.tab_preprocessor.with_cls_token:\n        x_vec = x_vec[:, 1:, :]\n\n    if self.tab_preprocessor.with_attention:\n        x_vec = einops.rearrange(x_vec, \"s c e -&gt; s (c e)\")\n\n    if x_cont_not_embed is not None:\n        x_vec = torch.cat([x_vec, x_cont_not_embed], 1).detach().cpu().numpy()\n    else:\n        x_vec = x_vec.detach().cpu().numpy()\n\n    if self.return_dataframe:\n        new_colnames = self._new_colnames()\n        if target_col:\n            return pd.DataFrame(data=x_vec, columns=new_colnames), df[[target_col]]\n        else:\n            return pd.DataFrame(data=x_vec, columns=new_colnames)\n    else:\n        if target_col:\n            return x_vec, df[target_col].values\n        else:\n            return x_vec\n</code></pre>"},{"location":"pytorch-widedeep/tab2vec.html#pytorch_widedeep.tab2vec.Tab2Vec.fit_transform","title":"fit_transform","text":"<pre><code>fit_transform(df, target_col=None)\n</code></pre> <p>Combines <code>fit</code> and <code>transform</code></p> Source code in <code>pytorch_widedeep/tab2vec.py</code> <pre><code>def fit_transform(\n    self, df: pd.DataFrame, target_col: Optional[str] = None\n) -&gt; Union[\n    np.ndarray,\n    Tuple[np.ndarray, np.ndarray],\n    pd.DataFrame,\n    Tuple[pd.DataFrame, pd.Series],\n]:\n    r\"\"\"Combines `fit` and `transform`\"\"\"\n    return self.fit(df, target_col).transform(df, target_col)\n</code></pre>"},{"location":"pytorch-widedeep/the_rec_module.html","title":"The <code>rec</code> module","text":"<p>This module contains models are that specifically designed for recommendation systems. While the rest of the models can be accessed from the <code>pytorch_widedeep.models</code> module, models in this module need to be specifically imported from the <code>rec</code> module, e.g.:</p> <pre><code>from pytorch_widedeep.models.rec import DeepFactorizationMachine\n</code></pre> <p>The list of models here is not meant to be exhaustive, but it includes some common architectures such as factorization machines, field aware factorization machines or extreme factorization machines. More models will be added in the future.</p>"},{"location":"pytorch-widedeep/the_rec_module.html#pytorch_widedeep.models.rec.deepfm.DeepFactorizationMachine","title":"DeepFactorizationMachine","text":"<p>               Bases: <code>BaseTabularModelWithAttention</code></p> <p>Deep Factorization Machine (DeepFM) for recommendation systems, which is an adaptation of 'Factorization Machines' by Steffen Rendle. Presented in 'DeepFM: A Factorization-Machine based Neural Network for CTR Prediction' by Huifeng Guo, Ruiming Tang, Yunming Yey, Zhenguo Li, Xiuqiang He. 2017.</p> <p>The implementation in this library takes advantage of all the functionalities available to encode categorical and continuous features. The model can be used with only the factorization machine</p> <p>Note that this class implements only the 'Deep' component of the model described in the paper. The linear component is not implemented 'internally' and, if one wants to include it, it can be easily added using the 'wide' (aka linear) component available in this library. See the examples in the examples folder.</p> <p>Parameters:</p> Name Type Description Default <code>column_idx</code> <code>Dict[str, int]</code> <p>Dictionary mapping column names to their corresponding index.</p> required <code>num_factors</code> <code>int</code> <p>Number of factors for the factorization machine.</p> required <code>reduce_sum</code> <code>bool</code> <p>Whether to reduce the sum in the factorization machine output.</p> <code>True</code> <code>cat_embed_input</code> <code>Optional[List[Tuple[str, int]]]</code> <p>List of tuples with categorical column names and number of unique values.</p> <code>None</code> <code>cat_embed_dropout</code> <code>Optional[float]</code> <p>Categorical embeddings dropout. If <code>None</code>, it will default to 0.</p> <code>None</code> <code>use_cat_bias</code> <code>Optional[bool]</code> <p>Boolean indicating if bias will be used for the categorical embeddings. If <code>None</code>, it will default to 'False'.</p> <code>None</code> <code>cat_embed_activation</code> <code>Optional[str]</code> <p>Activation function for the categorical embeddings, if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> <code>None</code> <code>continuous_cols</code> <code>Optional[List[str]]</code> <p>List with the name of the numeric (aka continuous) columns</p> <code>None</code> <code>cont_norm_layer</code> <code>Optional[Literal['batchnorm', 'layernorm']]</code> <p>Type of normalization layer applied to the continuous features. Options are: 'layernorm' and 'batchnorm'. if <code>None</code>, no normalization layer will be used.</p> <code>None</code> <code>embed_continuous_method</code> <code>Optional[Literal['piecewise', 'periodic', 'standard']]</code> <p>Method to use to embed the continuous features. Options are: 'standard', 'periodic' or 'piecewise'. The 'standard' embedding method is based on the FT-Transformer implementation presented in the paper: Revisiting Deep Learning Models for Tabular Data. The 'periodic' and_'piecewise'_ methods were presented in the paper: On Embeddings for Numerical Features in Tabular Deep Learning. Please, read the papers for details.</p> <code>'standard'</code> <code>cont_embed_dropout</code> <code>Optional[float]</code> <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p> <code>None</code> <code>cont_embed_activation</code> <code>Optional[str]</code> <p>Activation function for the continuous embeddings if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported. If <code>None</code>, no activation function will be applied.</p> <code>None</code> <code>quantization_setup</code> <code>Optional[Dict[str, List[float]]]</code> <p>This parameter is used when the 'piecewise' method is used to embed the continuous cols. It is a dict where keys are the name of the continuous columns and values are lists with the boundaries for the quantization of the continuous_cols. See the examples for details. If If the 'piecewise' method is used, this parameter is required.</p> <code>None</code> <code>n_frequencies</code> <code>Optional[int]</code> <p>This is the so called 'k' in their paper On Embeddings for Numerical Features in Tabular Deep Learning, and is the number of 'frequencies' that will be used to represent each continuous column. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>sigma</code> <code>Optional[float]</code> <p>This is the sigma parameter in the paper mentioned when describing the previous parameters and it is used to initialise the 'frequency weights'. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>share_last_layer</code> <code>Optional[bool]</code> <p>This parameter is not present in the before mentioned paper but it is implemented in the official repo. If <code>True</code> the linear layer that turns the frequencies into embeddings will be shared across the continuous columns. If <code>False</code> a different linear layer will be used for each continuous column. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>full_embed_dropout</code> <code>Optional[bool]</code> <p>If <code>True</code>, the full embedding corresponding to a column will be masked out/dropout. If <code>None</code>, it will default to <code>False</code>.</p> <code>None</code> <code>mlp_hidden_dims</code> <code>Optional[List[int]]</code> <p>List with the number of neurons per dense layer in the mlp.</p> <code>None</code> <code>mlp_activation</code> <code>Optional[str]</code> <p>Activation function for the dense layers of the MLP. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> <code>None</code> <code>mlp_dropout</code> <code>Optional[float]</code> <p>float or List of floats with the dropout between the dense layers. e.g: [0.5,0.5]</p> <code>None</code> <code>mlp_batchnorm</code> <code>Optional[bool]</code> <p>Boolean indicating whether or not batch normalization will be applied to the dense layers</p> <code>None</code> <code>mlp_batchnorm_last</code> <code>Optional[bool]</code> <p>Boolean indicating whether or not batch normalization will be applied to the last of the dense layers</p> <code>None</code> <code>mlp_linear_first</code> <code>Optional[bool]</code> <p>Boolean indicating the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code></p> <code>None</code> <p>Attributes:</p> Name Type Description <code>mlp</code> <code>MLP</code> <p>MLP component of the model if the mlp_hidden_dims parameter is not None. If None, the model will only return the output of the factorization machine.</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; from typing import Dict, List, Tuple\n&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from torch import Tensor\n&gt;&gt;&gt; from pytorch_widedeep.models.rec import DeepFactorizationMachine\n&gt;&gt;&gt; X = torch.randint(0, 10, (16, 2))\n&gt;&gt;&gt; column_idx: Dict[str, int] = {\"col1\": 0, \"col2\": 1}\n&gt;&gt;&gt; cat_embed_input: List[Tuple[str, int]] = [(\"col1\", 10), (\"col2\", 10)]\n&gt;&gt;&gt; fm = DeepFactorizationMachine(\n...     column_idx=column_idx,\n...     num_factors=8,\n...     cat_embed_input=cat_embed_input,\n...     mlp_hidden_dims=[16, 8]\n... )\n&gt;&gt;&gt; out = fm(X)\n</code></pre> Source code in <code>pytorch_widedeep/models/rec/deepfm.py</code> <pre><code>class DeepFactorizationMachine(BaseTabularModelWithAttention):\n    \"\"\"\n    Deep Factorization Machine (DeepFM) for recommendation systems, which is\n    an adaptation of 'Factorization Machines' by Steffen Rendle. Presented\n    in 'DeepFM: A Factorization-Machine based Neural Network for CTR\n    Prediction' by Huifeng Guo, Ruiming Tang, Yunming Yey, Zhenguo Li,\n    Xiuqiang He. 2017.\n\n    The implementation in this library takes advantage of all the\n    functionalities available to encode categorical and continuous features.\n    The model can be used with only the factorization machine\n\n    Note that this class implements only the 'Deep' component of the model\n    described in the paper. The linear component is not\n    implemented 'internally' and, if one wants to include it, it can be\n    easily added using the 'wide' (aka linear) component available in this\n    library. See the examples in the examples folder.\n\n    Parameters\n    ----------\n    column_idx : Dict[str, int]\n        Dictionary mapping column names to their corresponding index.\n    num_factors : int\n        Number of factors for the factorization machine.\n    reduce_sum : bool, default=True\n        Whether to reduce the sum in the factorization machine output.\n    cat_embed_input : Optional[List[Tuple[str, int]]], default=None\n        List of tuples with categorical column names and number of unique values.\n    cat_embed_dropout : Optional[float], default=None\n        Categorical embeddings dropout. If `None`, it will default\n        to 0.\n    use_cat_bias : Optional[bool], default=None\n        Boolean indicating if bias will be used for the categorical embeddings.\n        If `None`, it will default to 'False'.\n    cat_embed_activation : Optional[str], default=None\n        Activation function for the categorical embeddings, if any. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported\n    continuous_cols : Optional[List[str]], default=None\n        List with the name of the numeric (aka continuous) columns\n    cont_norm_layer : Optional[Literal[\"batchnorm\", \"layernorm\"]], default=None\n        Type of normalization layer applied to the continuous features.\n        Options are: _'layernorm'_ and _'batchnorm'_. if `None`, no\n        normalization layer will be used.\n    embed_continuous_method: Optional[Literal[\"piecewise\", \"periodic\", \"standard\"]], default=\"standard\"\n        Method to use to embed the continuous features. Options are:\n        _'standard'_, _'periodic'_ or _'piecewise'_. The _'standard'_\n        embedding method is based on the FT-Transformer implementation\n        presented in the paper: [Revisiting Deep Learning Models for\n        Tabular Data](https://arxiv.org/abs/2106.11959v5). The _'periodic'_\n        and_'piecewise'_ methods were presented in the paper: [On Embeddings for\n        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556).\n        Please, read the papers for details.\n    cont_embed_dropout : Optional[float], default=None\n        Dropout for the continuous embeddings. If `None`, it will default to 0.0\n    cont_embed_activation : Optional[str], default=None\n        Activation function for the continuous embeddings if any. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported.\n        If `None`, no activation function will be applied.\n    quantization_setup : Optional[Dict[str, List[float]]], default=None\n        This parameter is used when the _'piecewise'_ method is used to embed\n        the continuous cols. It is a dict where keys are the name of the continuous\n        columns and values are lists with the boundaries for the quantization\n        of the continuous_cols. See the examples for details. If\n        If the _'piecewise'_ method is used, this parameter is required.\n    n_frequencies : Optional[int], default=None\n        This is the so called _'k'_ in their paper [On Embeddings for\n        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556),\n        and is the number of 'frequencies' that will be used to represent each\n        continuous column. See their Eq 2 in the paper for details. If\n        the _'periodic'_ method is used, this parameter is required.\n    sigma : Optional[float], default=None\n        This is the sigma parameter in the paper mentioned when describing the\n        previous parameters and it is used to initialise the 'frequency\n        weights'. See their Eq 2 in the paper for details. If\n        the _'periodic'_ method is used, this parameter is required.\n    share_last_layer : Optional[bool], default=None\n        This parameter is not present in the before mentioned paper but it is implemented in\n        the [official repo](https://github.com/yandex-research/rtdl-num-embeddings/tree/main).\n        If `True` the linear layer that turns the frequencies into embeddings\n        will be shared across the continuous columns. If `False` a different\n        linear layer will be used for each continuous column.\n        If the _'periodic'_ method is used, this parameter is required.\n    full_embed_dropout: bool, Optional, default = None,\n        If `True`, the full embedding corresponding to a column will be masked\n        out/dropout. If `None`, it will default to `False`.\n    mlp_hidden_dims: List, default = [200, 100]\n        List with the number of neurons per dense layer in the mlp.\n    mlp_activation: str, default = \"relu\"\n        Activation function for the dense layers of the MLP. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported\n    mlp_dropout: float or List, default = 0.1\n        float or List of floats with the dropout between the dense layers.\n        e.g: _[0.5,0.5]_\n    mlp_batchnorm: bool, default = False\n        Boolean indicating whether or not batch normalization will be applied\n        to the dense layers\n    mlp_batchnorm_last: bool, default = False\n        Boolean indicating whether or not batch normalization will be applied\n        to the last of the dense layers\n    mlp_linear_first: bool, default = False\n        Boolean indicating the order of the operations in the dense\n        layer. If `True: [LIN -&gt; ACT -&gt; BN -&gt; DP]`. If `False: [BN -&gt; DP -&gt;\n        LIN -&gt; ACT]`\n\n    Attributes\n    ----------\n    mlp: MLP\n        MLP component of the model if the mlp_hidden_dims parameter is not\n        None. If None, the model will only return the output of the\n        factorization machine.\n\n    Examples\n    --------\n    &gt;&gt;&gt; from typing import Dict, List, Tuple\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from torch import Tensor\n    &gt;&gt;&gt; from pytorch_widedeep.models.rec import DeepFactorizationMachine\n    &gt;&gt;&gt; X = torch.randint(0, 10, (16, 2))\n    &gt;&gt;&gt; column_idx: Dict[str, int] = {\"col1\": 0, \"col2\": 1}\n    &gt;&gt;&gt; cat_embed_input: List[Tuple[str, int]] = [(\"col1\", 10), (\"col2\", 10)]\n    &gt;&gt;&gt; fm = DeepFactorizationMachine(\n    ...     column_idx=column_idx,\n    ...     num_factors=8,\n    ...     cat_embed_input=cat_embed_input,\n    ...     mlp_hidden_dims=[16, 8]\n    ... )\n    &gt;&gt;&gt; out = fm(X)\n    \"\"\"\n\n    def __init__(\n        self,\n        *,\n        column_idx: Dict[str, int],\n        num_factors: int,\n        reduce_sum: bool = True,\n        cat_embed_input: Optional[List[Tuple[str, int]]] = None,\n        cat_embed_dropout: Optional[float] = None,\n        use_cat_bias: Optional[bool] = None,\n        cat_embed_activation: Optional[str] = None,\n        continuous_cols: Optional[List[str]] = None,\n        cont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\n        embed_continuous_method: Optional[\n            Literal[\"piecewise\", \"periodic\", \"standard\"]\n        ] = \"standard\",\n        cont_embed_dropout: Optional[float] = None,\n        cont_embed_activation: Optional[str] = None,\n        quantization_setup: Optional[Dict[str, List[float]]] = None,\n        n_frequencies: Optional[int] = None,\n        sigma: Optional[float] = None,\n        share_last_layer: Optional[bool] = None,\n        full_embed_dropout: Optional[bool] = None,\n        mlp_hidden_dims: Optional[List[int]] = None,\n        mlp_activation: Optional[str] = None,\n        mlp_dropout: Optional[float] = None,\n        mlp_batchnorm: Optional[bool] = None,\n        mlp_batchnorm_last: Optional[bool] = None,\n        mlp_linear_first: Optional[bool] = None,\n    ):\n        super(DeepFactorizationMachine, self).__init__(\n            column_idx=column_idx,\n            input_dim=num_factors,\n            cat_embed_input=cat_embed_input,\n            cat_embed_dropout=cat_embed_dropout,\n            use_cat_bias=use_cat_bias,\n            cat_embed_activation=cat_embed_activation,\n            shared_embed=False,\n            add_shared_embed=None,\n            frac_shared_embed=None,\n            continuous_cols=continuous_cols,\n            cont_norm_layer=cont_norm_layer,\n            embed_continuous_method=embed_continuous_method,\n            cont_embed_dropout=cont_embed_dropout,\n            cont_embed_activation=cont_embed_activation,\n            quantization_setup=quantization_setup,\n            n_frequencies=n_frequencies,\n            sigma=sigma,\n            share_last_layer=share_last_layer,\n            full_embed_dropout=full_embed_dropout,\n        )\n\n        self.reduce_sum = reduce_sum\n\n        self.mlp_hidden_dims = mlp_hidden_dims\n        self.mlp_activation = mlp_activation\n        self.mlp_dropout = mlp_dropout\n        self.mlp_batchnorm = mlp_batchnorm\n        self.mlp_batchnorm_last = mlp_batchnorm_last\n        self.mlp_linear_first = mlp_linear_first\n\n        if self.mlp_hidden_dims is not None:\n\n            if self.mlp_hidden_dims[-1] != self.input_dim:\n                d_hidden = (\n                    [self.input_dim * len(self.column_idx)]\n                    + self.mlp_hidden_dims\n                    + [self.input_dim]\n                )\n            else:\n                d_hidden = [\n                    self.input_dim * len(self.column_idx)\n                ] + self.mlp_hidden_dims\n\n            self.mlp = MLP(\n                d_hidden=d_hidden,\n                activation=(\n                    \"relu\" if self.mlp_activation is None else self.mlp_activation\n                ),\n                dropout=0.0 if self.mlp_dropout is None else self.mlp_dropout,\n                batchnorm=False if self.mlp_batchnorm is None else self.mlp_batchnorm,\n                batchnorm_last=(\n                    False\n                    if self.mlp_batchnorm_last is None\n                    else self.mlp_batchnorm_last\n                ),\n                linear_first=(\n                    False if self.mlp_linear_first is None else self.mlp_linear_first\n                ),\n            )\n        else:\n            self.mlp = None\n\n    def forward(self, X: Tensor) -&gt; Tensor:\n        embed = self._get_embeddings(X)\n        fm_output = factorization_machine(embed, self.reduce_sum)\n\n        if self.mlp is None:\n            return fm_output\n\n        mlp_input = embed.view(embed.size(0), -1)\n        mlp_output = self.mlp(mlp_input)\n\n        if self.reduce_sum:\n            mlp_output = mlp_output.sum(1, keepdim=True)\n\n        return fm_output + mlp_output\n\n    @property\n    def output_dim(self) -&gt; int:\n        if self.reduce_sum:\n            return 1\n        else:\n            return self.input_dim\n</code></pre>"},{"location":"pytorch-widedeep/the_rec_module.html#pytorch_widedeep.models.rec.deepffm.DeepFieldAwareFactorizationMachine","title":"DeepFieldAwareFactorizationMachine","text":"<p>               Bases: <code>BaseTabularModelWithAttention</code></p> <p>Deep Field Aware Factorization Machine (DeepFFM) for recommendation systems. Adaptation of the paper 'Field-aware Factorization Machines in a Real-world Online Advertising System', Juan et al. 2017.</p> <p>This class implements only the 'Deep' component of the model described in the paper. The linear component is not implemented 'internally' and, if one wants to include it, it can be easily added using the 'wide'/linear component in this library. See the examples in the examples folder.</p> <p>Note that in this case, only categorical features are accepted. This is because the embeddings of each feature will be learned using all other features. Therefore these embeddings have to be all of the same nature. This does not occur if we mix categorical and continuous features.</p> <p>Parameters:</p> Name Type Description Default <code>column_idx</code> <code>Dict[str, int]</code> <p>Dictionary mapping column names to their corresponding index.</p> required <code>num_factors</code> <code>int</code> <p>Number of factors for the factorization machine.</p> required <code>reduce_sum</code> <code>bool</code> <p>Whether to reduce the sum in the factorization machine output.</p> <code>True</code> <code>cat_embed_input</code> <code>Optional[List[Tuple[str, int]]]</code> <p>List of tuples with categorical column names and number of unique values.</p> <code>None</code> <code>cat_embed_dropout</code> <code>Optional[float]</code> <p>Categorical embeddings dropout. If <code>None</code>, it will default to 0.</p> <code>None</code> <code>use_cat_bias</code> <code>Optional[bool]</code> <p>Boolean indicating if bias will be used for the categorical embeddings. If <code>None</code>, it will default to 'False'.</p> <code>None</code> <code>cat_embed_activation</code> <code>Optional[str]</code> <p>Activation function for the categorical embeddings, if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> <code>None</code> <code>mlp_hidden_dims</code> <code>Optional[List[int]]</code> <p>List with the number of neurons per dense layer in the mlp.</p> <code>None</code> <code>mlp_activation</code> <code>Optional[str]</code> <p>Activation function for the dense layers of the MLP. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> <code>None</code> <code>mlp_dropout</code> <code>Optional[float]</code> <p>float or List of floats with the dropout between the dense layers. e.g: [0.5,0.5]</p> <code>None</code> <code>mlp_batchnorm</code> <code>Optional[bool]</code> <p>Boolean indicating whether or not batch normalization will be applied to the dense layers</p> <code>None</code> <code>mlp_batchnorm_last</code> <code>Optional[bool]</code> <p>Boolean indicating whether or not batch normalization will be applied to the last of the dense layers</p> <code>None</code> <code>mlp_linear_first</code> <code>Optional[bool]</code> <p>Boolean indicating the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code></p> <code>None</code> <p>Attributes:</p> Name Type Description <code>n_features</code> <code>int</code> <p>Number of unique features/columns</p> <code>n_tokens</code> <code>int</code> <p>Number of unique values (tokens) in the full dataset (corpus)</p> <code>encoders</code> <code>ModuleList</code> <p>List of <code>BaseTabularModelWithAttention</code> instances. One per categorical column</p> <code>mlp</code> <code>Module</code> <p>Multi-layer perceptron. If <code>None</code> the output will be the output of the factorization machine (i.e. the sum of the interactions)</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from torch import Tensor\n&gt;&gt;&gt; from typing import Dict, List, Tuple\n&gt;&gt;&gt; from pytorch_widedeep.models.rec import DeepFieldAwareFactorizationMachine\n&gt;&gt;&gt; X = torch.randint(0, 10, (16, 2))\n&gt;&gt;&gt; column_idx: Dict[str, int] = {\"col1\": 0, \"col2\": 1}\n&gt;&gt;&gt; cat_embed_input: List[Tuple[str, int]] = [(\"col1\", 10), (\"col2\", 10)]\n&gt;&gt;&gt; ffm = DeepFieldAwareFactorizationMachine(\n...     column_idx=column_idx,\n...     num_factors=4,\n...     cat_embed_input=cat_embed_input,\n...     mlp_hidden_dims=[16, 8]\n... )\n&gt;&gt;&gt; output = ffm(X)\n</code></pre> Source code in <code>pytorch_widedeep/models/rec/deepffm.py</code> <pre><code>class DeepFieldAwareFactorizationMachine(BaseTabularModelWithAttention):\n    \"\"\"\n    Deep Field Aware Factorization Machine (DeepFFM) for recommendation\n    systems. Adaptation of the paper 'Field-aware Factorization Machines in a\n    Real-world Online Advertising System', Juan et al. 2017.\n\n    This class implements only the 'Deep' component of the model described in\n    the paper. The linear component is not implemented 'internally' and, if\n    one wants to include it, it can be easily added using the 'wide'/linear\n    component in this library. See the examples in the examples folder.\n\n    Note that in this case, only categorical features are accepted. This is\n    because the embeddings of each feature will be learned using all other\n    features. Therefore these embeddings have to be all of the same nature.\n    This does not occur if we mix categorical and continuous features.\n\n    Parameters\n    ----------\n    column_idx : Dict[str, int]\n        Dictionary mapping column names to their corresponding index.\n    num_factors : int\n        Number of factors for the factorization machine.\n    reduce_sum : bool, default=True\n        Whether to reduce the sum in the factorization machine output.\n    cat_embed_input : Optional[List[Tuple[str, int]]], default=None\n        List of tuples with categorical column names and number of unique values.\n    cat_embed_dropout : Optional[float], default=None\n        Categorical embeddings dropout. If `None`, it will default\n        to 0.\n    use_cat_bias : Optional[bool], default=None\n        Boolean indicating if bias will be used for the categorical embeddings.\n        If `None`, it will default to 'False'.\n    cat_embed_activation : Optional[str], default=None\n        Activation function for the categorical embeddings, if any. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported\n    mlp_hidden_dims: List, default = [200, 100]\n        List with the number of neurons per dense layer in the mlp.\n    mlp_activation: str, default = \"relu\"\n        Activation function for the dense layers of the MLP. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported\n    mlp_dropout: float or List, default = 0.1\n        float or List of floats with the dropout between the dense layers.\n        e.g: _[0.5,0.5]_\n    mlp_batchnorm: bool, default = False\n        Boolean indicating whether or not batch normalization will be applied\n        to the dense layers\n    mlp_batchnorm_last: bool, default = False\n        Boolean indicating whether or not batch normalization will be applied\n        to the last of the dense layers\n    mlp_linear_first: bool, default = False\n        Boolean indicating the order of the operations in the dense\n        layer. If `True: [LIN -&gt; ACT -&gt; BN -&gt; DP]`. If `False: [BN -&gt; DP -&gt;\n        LIN -&gt; ACT]`\n\n    Attributes\n    ----------\n    n_features: int\n        Number of unique features/columns\n    n_tokens: int\n        Number of unique values (tokens) in the full dataset (corpus)\n    encoders: nn.ModuleList\n        List of `BaseTabularModelWithAttention` instances. One per categorical\n        column\n    mlp: nn.Module\n        Multi-layer perceptron. If `None` the output will be the output of the\n        factorization machine (i.e. the sum of the interactions)\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from torch import Tensor\n    &gt;&gt;&gt; from typing import Dict, List, Tuple\n    &gt;&gt;&gt; from pytorch_widedeep.models.rec import DeepFieldAwareFactorizationMachine\n    &gt;&gt;&gt; X = torch.randint(0, 10, (16, 2))\n    &gt;&gt;&gt; column_idx: Dict[str, int] = {\"col1\": 0, \"col2\": 1}\n    &gt;&gt;&gt; cat_embed_input: List[Tuple[str, int]] = [(\"col1\", 10), (\"col2\", 10)]\n    &gt;&gt;&gt; ffm = DeepFieldAwareFactorizationMachine(\n    ...     column_idx=column_idx,\n    ...     num_factors=4,\n    ...     cat_embed_input=cat_embed_input,\n    ...     mlp_hidden_dims=[16, 8]\n    ... )\n    &gt;&gt;&gt; output = ffm(X)\n    \"\"\"\n\n    def __init__(\n        self,\n        *,\n        column_idx: Dict[str, int],\n        num_factors: int,\n        cat_embed_input: List[Tuple[str, int]],\n        reduce_sum: bool = True,\n        cat_embed_dropout: Optional[float] = None,\n        use_cat_bias: Optional[bool] = None,\n        cat_embed_activation: Optional[str] = None,\n        mlp_hidden_dims: Optional[List[int]] = None,\n        mlp_activation: Optional[str] = None,\n        mlp_dropout: Optional[float] = None,\n        mlp_batchnorm: Optional[bool] = None,\n        mlp_batchnorm_last: Optional[bool] = None,\n        mlp_linear_first: Optional[bool] = None,\n    ):\n        super(DeepFieldAwareFactorizationMachine, self).__init__(\n            column_idx=column_idx,\n            input_dim=num_factors,\n            cat_embed_input=cat_embed_input,\n            cat_embed_dropout=cat_embed_dropout,\n            use_cat_bias=use_cat_bias,\n            cat_embed_activation=cat_embed_activation,\n            shared_embed=False,\n            add_shared_embed=None,\n            frac_shared_embed=None,\n            continuous_cols=None,\n            cont_norm_layer=None,\n            embed_continuous_method=None,\n            cont_embed_dropout=None,\n            cont_embed_activation=None,\n            quantization_setup=None,\n            n_frequencies=None,\n            sigma=None,\n            share_last_layer=None,\n            full_embed_dropout=None,\n        )\n\n        self.reduce_sum = reduce_sum\n\n        self.mlp_hidden_dims = mlp_hidden_dims\n        self.mlp_activation = mlp_activation\n        self.mlp_dropout = mlp_dropout\n        self.mlp_batchnorm = mlp_batchnorm\n        self.mlp_batchnorm_last = mlp_batchnorm_last\n        self.mlp_linear_first = mlp_linear_first\n\n        self.n_features = len(self.column_idx)\n        self.n_tokens = sum([ei[1] for ei in cat_embed_input])\n\n        self.encoders = nn.ModuleList(\n            [\n                BaseTabularModelWithAttention(**config)\n                for config in self._get_encoder_configs()\n            ]\n        )\n\n        if self.mlp_hidden_dims is not None:\n            d_hidden = [\n                self.n_features * (self.n_features - 1) // 2 * num_factors\n            ] + self.mlp_hidden_dims\n            self.mlp = MLP(\n                d_hidden=d_hidden,\n                activation=(\n                    \"relu\" if self.mlp_activation is None else self.mlp_activation\n                ),\n                dropout=0.0 if self.mlp_dropout is None else self.mlp_dropout,\n                batchnorm=False if self.mlp_batchnorm is None else self.mlp_batchnorm,\n                batchnorm_last=(\n                    False\n                    if self.mlp_batchnorm_last is None\n                    else self.mlp_batchnorm_last\n                ),\n                linear_first=(\n                    False if self.mlp_linear_first is None else self.mlp_linear_first\n                ),\n            )\n        else:\n            self.mlp = None\n\n    def forward(self, X: Tensor) -&gt; Tensor:\n\n        interactions_l: List[Tensor] = []\n        for i in range(len(self.column_idx)):\n            for j in range(i + 1, len(self.column_idx)):\n                # the syntax [i] and [j] is to keep the shape of the tensors\n                # as they are sliced within '_get_embeddings'. This will\n                # return a tensor of shape (b, 1, embed_dim). Then it has to\n                # be squeezed to (b, embed_dim)  before multiplied\n                embed_i = self.encoders[i]._get_embeddings(X[:, [i]]).squeeze(1)\n                embed_j = self.encoders[j]._get_embeddings(X[:, [j]]).squeeze(1)\n                interactions_l.append(embed_i * embed_j)\n\n        interactions = torch.cat(interactions_l, dim=1)\n\n        if self.mlp is not None:\n            interactions = interactions.view(X.size(0), -1)\n            deep_out = self.mlp(interactions)\n        else:\n            deep_out = interactions\n\n        if self.reduce_sum:\n            deep_out = deep_out.sum(dim=1, keepdim=True)\n\n        return deep_out\n\n    def _get_encoder_configs(self) -&gt; List[Dict[str, Any]]:\n        config: List[Dict[str, Any]] = []\n        for col, _ in self.column_idx.items():\n            cat_embed_input = [(col, self.n_tokens)]\n            _config = {\n                \"column_idx\": {col: 0},\n                \"input_dim\": self.input_dim,\n                \"cat_embed_input\": cat_embed_input,\n                \"cat_embed_dropout\": self.cat_embed_dropout,\n                \"use_cat_bias\": self.use_cat_bias,\n                \"cat_embed_activation\": self.cat_embed_activation,\n                \"shared_embed\": None,\n                \"add_shared_embed\": None,\n                \"frac_shared_embed\": None,\n                \"continuous_cols\": None,\n                \"cont_norm_layer\": None,\n                \"embed_continuous_method\": None,\n                \"cont_embed_dropout\": None,\n                \"cont_embed_activation\": None,\n                \"quantization_setup\": None,\n                \"n_frequencies\": None,\n                \"sigma\": None,\n                \"share_last_layer\": None,\n                \"full_embed_dropout\": None,\n            }\n\n            config.append(_config)\n\n        return config\n\n    @property\n    def output_dim(self) -&gt; int:\n        if self.reduce_sum:\n            return 1\n        elif self.mlp_hidden_dims is not None:\n            return self.mlp_hidden_dims[-1]\n        else:\n            return self.n_features * (self.n_features - 1) // 2 * self.input_dim\n</code></pre>"},{"location":"pytorch-widedeep/the_rec_module.html#pytorch_widedeep.models.rec.din.DeepInterestNetwork","title":"DeepInterestNetwork","text":"<p>               Bases: <code>BaseWDModelComponent</code></p> <p>Adaptation of the Deep Interest Network (DIN) for recommendation systems as described in the paper: 'Deep Interest Network for Click-Through Rate Prediction' by Guorui Zhou et al. 2018.</p> <p>Note that all the categorical- and continuous-related parameters refer to the categorical and continuous columns that are not part of the sequential columns and will be treated as standard tabular data.</p> <p>This model requires some specific data preparation that allows for quite a lot of flexibility. Therefore, I have included a preprocessor (<code>DINPreprocessor</code>) in the preprocessing module that will take care of the data preparation.</p> <p>Parameters:</p> Name Type Description Default <code>column_idx</code> <code>Dict[str, int]</code> <p>Dictionary mapping column names to their corresponding index.</p> required <code>target_item_col</code> <code>str</code> <p>Name of the target item column. Note that this is not the target column. This algorithm relies on a sequence representation of interactions. The target item would be the next item in the sequence of interactions (e.g. item 6<sup>th</sup> in a sequence of 5 items), and our goal is to predict a given action on it.</p> <code>'target_item'</code> <code>user_behavior_confiq</code> <code>Tuple[List[str], int, int]</code> <p>Configuration for user behavior sequence columns. Tuple containing: - List of column names that correspond to the user behavior sequence - Number of unique feature values (n_tokens) - Embedding dimension Example: <code>([\"item_1\", \"item_2\", \"item_3\"], 5, 8)</code></p> required <code>action_seq_config</code> <code>Optional[Tuple[List[str], int]]</code> <p>Configuration for a so-called action sequence columns (for example a rating, or purchased/not-purchased, etc). Tuple containing: - List of column names - Number of unique feature values (n_tokens) This action will always be learned as a 1d embedding and will be combined with the user behaviour. For example, imagine that the action is purchased/not-purchased. then per item in the user behaviour sequence there will be a binary action to learn 0/1. Such action will be represented by a float number that will multiply the corresponding item embedding in the user behaviour sequence. Example: <code>([\"rating_1\", \"rating_2\", \"rating_3\"], 5)</code> Internally, the embedding dimension will be set to 1</p> <code>None</code> <code>other_seq_cols_confiq</code> <code>Optional[List[Tuple[List[str], int, int]]]</code> <p>Configuration for other sequential columns. List of tuples containing: - List of column names that correspond to the sequential column - Number of unique feature values (n_tokens) - Embedding dimension Example: <code>[([\"seq1_col1\", \"seq1_col2\"], 5, 8), ([\"seq2_col1\", \"seq2_col2\"], 5, 8)]</code></p> <code>None</code> <code>attention_unit_activation</code> <code>Literal['prelu', 'dice']</code> <p>Activation function to use in the attention unit.</p> <code>\"prelu\"</code> <code>cat_embed_input</code> <code>Optional[List[Tuple[str, int, int]]]</code> <p>Configuration for other columns. List of tuples containing: - Column name - Number of unique feature values (n_tokens) - Embedding dimension</p> <p>Note: From here in advance the remaining parameters are related to the categorical and continuous columns that are not part of the sequential columns and will be treated as standard tabular data.</p> <code>None</code> <code>cat_embed_dropout</code> <code>Optional[float]</code> <p>Categorical embeddings dropout. If <code>None</code>, it will default to 0.</p> <code>None</code> <code>use_cat_bias</code> <code>Optional[bool]</code> <p>Boolean indicating if bias will be used for the categorical embeddings. If <code>None</code>, it will default to 'False'.</p> <code>None</code> <code>cat_embed_activation</code> <code>Optional[str]</code> <p>Activation function for the categorical embeddings, if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> <code>None</code> <code>continuous_cols</code> <code>Optional[List[str]]</code> <p>List with the name of the numeric (aka continuous) columns</p> <code>None</code> <code>cont_norm_layer</code> <code>Optional[Literal['batchnorm', 'layernorm']]</code> <p>Type of normalization layer applied to the continuous features. Options are: 'layernorm' and 'batchnorm'. if <code>None</code>, no normalization layer will be used.</p> <code>None</code> <code>embed_continuous</code> <code>Optional[bool]</code> <p>Boolean indicating if the continuous columns will be embedded using one of the available methods: 'standard', 'periodic' or 'piecewise'. If <code>None</code>, it will default to 'False'. NOTE: This parameter is deprecated and it  will be removed in future releases. Please, use the  <code>embed_continuous_method</code> parameter instead.</p> <code>None</code> <code>embed_continuous_method</code> <code>Optional[Literal['piecewise', 'periodic', 'standard']]</code> <p>Method to use to embed the continuous features. Options are: 'standard', 'periodic' or 'piecewise'. The 'standard' embedding method is based on the FT-Transformer implementation presented in the paper: Revisiting Deep Learning Models for Tabular Data. The 'periodic' and_'piecewise'_ methods were presented in the paper: On Embeddings for Numerical Features in Tabular Deep Learning. Please, read the papers for details.</p> <code>None</code> <code>cont_embed_dim</code> <code>Optional[int]</code> <p>Size of the continuous embeddings. If the continuous columns are embedded, <code>cont_embed_dim</code> must be passed.</p> <code>None</code> <code>cont_embed_dropout</code> <code>Optional[float]</code> <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p> <code>None</code> <code>cont_embed_activation</code> <code>Optional[str]</code> <p>Activation function for the continuous embeddings if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported. If <code>None</code>, no activation function will be applied.</p> <code>None</code> <code>quantization_setup</code> <code>Optional[Dict[str, List[float]]]</code> <p>This parameter is used when the 'piecewise' method is used to embed the continuous cols. It is a dict where keys are the name of the continuous columns and values are lists with the boundaries for the quantization of the continuous_cols. See the examples for details. If If the 'piecewise' method is used, this parameter is required.</p> <code>None</code> <code>n_frequencies</code> <code>Optional[int]</code> <p>This is the so called 'k' in their paper On Embeddings for Numerical Features in Tabular Deep Learning, and is the number of 'frequencies' that will be used to represent each continuous column. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>sigma</code> <code>Optional[float]</code> <p>This is the sigma parameter in the paper mentioned when describing the previous parameters and it is used to initialise the 'frequency weights'. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>share_last_layer</code> <code>Optional[bool]</code> <p>This parameter is not present in the before mentioned paper but it is implemented in the official repo. If <code>True</code> the linear layer that turns the frequencies into embeddings will be shared across the continuous columns. If <code>False</code> a different linear layer will be used for each continuous column. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>full_embed_dropout</code> <code>Optional[bool]</code> <p>If <code>True</code>, the full embedding corresponding to a column will be masked out/dropout. If <code>None</code>, it will default to <code>False</code>.</p> <code>None</code> <code>mlp_hidden_dims</code> <code>Optional[List[int]]</code> <p>List with the number of neurons per dense layer in the mlp.</p> <code>None</code> <code>mlp_activation</code> <code>Optional[str]</code> <p>Activation function for the dense layers of the MLP. Currently 'tanh', 'relu', 'leaky_relu', 'gelu' and 'preglu' are   supported</p> <code>None</code> <code>mlp_dropout</code> <code>Optional[float]</code> <p>float or List of floats with the dropout between the dense layers. e.g: [0.5,0.5]</p> <code>None</code> <code>mlp_batchnorm</code> <code>Optional[bool]</code> <p>Boolean indicating whether or not batch normalization will be applied to the dense layers</p> <code>None</code> <code>mlp_batchnorm_last</code> <code>Optional[bool]</code> <p>Boolean indicating whether or not batch normalization will be applied to the last of the dense layers</p> <code>None</code> <code>mlp_linear_first</code> <code>Optional[bool]</code> <p>Boolean indicating the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code></p> <code>None</code> <p>Attributes:</p> Name Type Description <code>user_behavior_indexes</code> <code>List[int]</code> <p>List with the indexes of the user behavior columns</p> <code>user_behavior_embed</code> <code>BaseTabularModelWithAttention</code> <p>Embedding layer for the user</p> <code>action_seq_indexes</code> <code>List[int]</code> <p>List with the indexes of the rating sequence columns if the action_seq_config parameter is not None</p> <code>action_embed</code> <code>BaseTabularModelWithAttention</code> <p>Embedding layer for the rating sequence columns if the action_seq_config parameter is not None</p> <code>other_seq_cols_indexes</code> <code>Dict[str, List[int]]</code> <p>Dictionary with the indexes of the other sequential columns if the other_seq_cols_confiq parameter is not None</p> <code>other_seq_cols_embed</code> <code>ModuleDict</code> <p>Dictionary with the embedding layers for the other sequential columns if the other_seq_cols_confiq parameter is not None</p> <code>other_cols_idx</code> <code>List[int]</code> <p>List with the indexes of the other columns if the other_cols_config parameter is not None</p> <code>other_col_embed</code> <code>BaseTabularModel</code> <p>Embedding layer for the other columns if the other_cols_config parameter is not None</p> <code>mlp</code> <code>Optional[MLP]</code> <p>MLP component of the model. If None, no MLP will be used. This should almost always be not None.</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; import numpy as np\n&gt;&gt;&gt; from torch import Tensor\n&gt;&gt;&gt; from typing import Dict, List, Tuple\n&gt;&gt;&gt; from pytorch_widedeep.models.rec import DeepInterestNetwork\n&gt;&gt;&gt; np_seed = np.random.seed(42)\n&gt;&gt;&gt; torch_seed = torch.manual_seed(42)\n&gt;&gt;&gt; num_users = 10\n&gt;&gt;&gt; num_items = 5\n&gt;&gt;&gt; num_contexts = 3\n&gt;&gt;&gt; seq_length = 3\n&gt;&gt;&gt; num_samples = 10\n&gt;&gt;&gt; user_ids = np.random.randint(0, num_users, num_samples)\n&gt;&gt;&gt; target_item_ids = np.random.randint(0, num_items, num_samples)\n&gt;&gt;&gt; context_ids = np.random.randint(0, num_contexts, num_samples)\n&gt;&gt;&gt; user_behavior = np.array(\n...     [\n...         np.random.choice(num_items, seq_length, replace=False)\n...         for _ in range(num_samples)\n...     ]\n... )\n&gt;&gt;&gt; X_arr = np.column_stack((user_ids, target_item_ids, context_ids, user_behavior))\n&gt;&gt;&gt; X = torch.tensor(X_arr, dtype=torch.long)\n&gt;&gt;&gt; column_idx: Dict[str, int] = {\n...     \"user_id\": 0,\n...     \"target_item\": 1,\n...     \"context\": 2,\n...     \"item_1\": 3,\n...     \"item_2\": 4,\n...     \"item_3\": 5,\n... }\n&gt;&gt;&gt; user_behavior_config: Tuple[List[str], int, int] = (\n...     [\"item_1\", \"item_2\", \"item_3\"],\n...     num_items,\n...     8,\n... )\n&gt;&gt;&gt; cat_embed_input: List[Tuple[str, int, int]] = [\n...     (\"user_id\", num_users, 8),\n...     (\"context\", num_contexts, 4),\n... ]\n&gt;&gt;&gt; model = DeepInterestNetwork(\n...     column_idx=column_idx,\n...     target_item_col=\"target_item\",\n...     user_behavior_confiq=user_behavior_config,\n...     cat_embed_input=cat_embed_input,\n...     mlp_hidden_dims=[16, 8],\n... )\n&gt;&gt;&gt; output = model(X)\n</code></pre> Source code in <code>pytorch_widedeep/models/rec/din.py</code> <pre><code>class DeepInterestNetwork(BaseWDModelComponent):\n    \"\"\"\n    Adaptation of the Deep Interest Network (DIN) for recommendation systems\n    as described in the paper: 'Deep Interest Network for Click-Through Rate\n    Prediction' by Guorui Zhou et al. 2018.\n\n    Note that all the categorical- and continuous-related parameters refer to\n    the categorical and continuous columns that are not part of the\n    sequential columns and will be treated as standard tabular data.\n\n    This model requires some specific data preparation that allows for quite a\n    lot of flexibility. Therefore, I have included a preprocessor\n    (`DINPreprocessor`) in the preprocessing module that will take care of\n    the data preparation.\n\n    Parameters\n    ----------\n    column_idx : Dict[str, int]\n        Dictionary mapping column names to their corresponding index.\n    target_item_col : str\n        Name of the target item column. Note that this is not the target\n        column. This algorithm relies on a sequence representation of\n        interactions. The target item would be the next item in the sequence\n        of interactions (e.g. item 6th in a sequence of 5 items), and our\n        goal is to predict a given action on it.\n    user_behavior_confiq : Tuple[List[str], int, int]\n        Configuration for user behavior sequence columns. Tuple containing:\n        - List of column names that correspond to the user behavior sequence&lt;br/&gt;\n        - Number of unique feature values (n_tokens)&lt;br/&gt;\n        - Embedding dimension&lt;br/&gt;\n        Example: `([\"item_1\", \"item_2\", \"item_3\"], 5, 8)`\n    action_seq_config : Optional[Tuple[List[str], int]], default=None\n        Configuration for a so-called action sequence columns (for example a\n        rating, or purchased/not-purchased, etc). Tuple containing:&lt;br/&gt;\n        - List of column names&lt;br/&gt;\n        - Number of unique feature values (n_tokens)&lt;br/&gt;\n        This action will **always** be learned as a 1d embedding and will be\n        combined with the user behaviour. For example, imagine that the\n        action is purchased/not-purchased. then per item in the user\n        behaviour sequence there will be a binary action to learn 0/1. Such\n        action will be represented by a float number that will multiply the\n        corresponding item embedding in the user behaviour sequence.&lt;br/&gt;\n        Example: `([\"rating_1\", \"rating_2\", \"rating_3\"], 5)`&lt;br/&gt;\n        Internally, the embedding dimension will be set to 1\n    other_seq_cols_confiq : Optional[List[Tuple[List[str], int, int]]], default=None\n        Configuration for other sequential columns. List of tuples containing:&lt;br/&gt;\n        - List of column names that correspond to the sequential column&lt;br/&gt;\n        - Number of unique feature values (n_tokens)&lt;br/&gt;\n        - Embedding dimension&lt;br/&gt;\n        Example: `[([\"seq1_col1\", \"seq1_col2\"], 5, 8), ([\"seq2_col1\", \"seq2_col2\"], 5, 8)]`\n    attention_unit_activation : Literal[\"prelu\", \"dice\"], default=\"prelu\"\n        Activation function to use in the attention unit.\n    cat_embed_input : Optional[List[Tuple[str, int, int]]], default=None\n        Configuration for other columns. List of tuples containing:&lt;br/&gt;\n        - Column name&lt;br/&gt;\n        - Number of unique feature values (n_tokens)&lt;br/&gt;\n        - Embedding dimension&lt;br/&gt;\n\n        **Note**: From here in advance the remaining parameters are related to\n        the categorical and continuous columns that are not part of the\n        sequential columns and will be treated as standard tabular data.\n\n    cat_embed_dropout : Optional[float], default=None\n        Categorical embeddings dropout. If `None`, it will default\n        to 0.\n    use_cat_bias : Optional[bool], default=None\n        Boolean indicating if bias will be used for the categorical embeddings.\n        If `None`, it will default to 'False'.\n    cat_embed_activation : Optional[str], default=None\n        Activation function for the categorical embeddings, if any. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported\n    continuous_cols : Optional[List[str]], default=None\n        List with the name of the numeric (aka continuous) columns\n    cont_norm_layer : Optional[Literal[\"batchnorm\", \"layernorm\"]], default=None\n        Type of normalization layer applied to the continuous features.\n        Options are: _'layernorm'_ and _'batchnorm'_. if `None`, no\n        normalization layer will be used.\n    embed_continuous : Optional[bool], default=None\n        Boolean indicating if the continuous columns will be embedded using\n        one of the available methods: _'standard'_, _'periodic'_\n        or _'piecewise'_. If `None`, it will default to 'False'.&lt;br/&gt;\n        :information_source: **NOTE**: This parameter is deprecated and it\n         will be removed in future releases. Please, use the\n         `embed_continuous_method` parameter instead.\n    embed_continuous_method : Optional[Literal[\"piecewise\", \"periodic\", \"standard\"]], default=None\n        Method to use to embed the continuous features. Options are:\n        _'standard'_, _'periodic'_ or _'piecewise'_. The _'standard'_\n        embedding method is based on the FT-Transformer implementation\n        presented in the paper: [Revisiting Deep Learning Models for\n        Tabular Data](https://arxiv.org/abs/2106.11959v5). The _'periodic'_\n        and_'piecewise'_ methods were presented in the paper: [On Embeddings for\n        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556).\n        Please, read the papers for details.\n    cont_embed_dim : Optional[int], default=None\n        Size of the continuous embeddings. If the continuous columns are\n        embedded, `cont_embed_dim` must be passed.\n    cont_embed_dropout : Optional[float], default=None\n        Dropout for the continuous embeddings. If `None`, it will default to 0.0\n    cont_embed_activation : Optional[str], default=None\n        Activation function for the continuous embeddings if any. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported.\n        If `None`, no activation function will be applied.\n    quantization_setup : Optional[Dict[str, List[float]]], default=None\n        This parameter is used when the _'piecewise'_ method is used to embed\n        the continuous cols. It is a dict where keys are the name of the continuous\n        columns and values are lists with the boundaries for the quantization\n        of the continuous_cols. See the examples for details. If\n        If the _'piecewise'_ method is used, this parameter is required.\n    n_frequencies : Optional[int], default=None\n        This is the so called _'k'_ in their paper [On Embeddings for\n        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556),\n        and is the number of 'frequencies' that will be used to represent each\n        continuous column. See their Eq 2 in the paper for details. If\n        the _'periodic'_ method is used, this parameter is required.\n    sigma : Optional[float], default=None\n        This is the sigma parameter in the paper mentioned when describing the\n        previous parameters and it is used to initialise the 'frequency\n        weights'. See their Eq 2 in the paper for details. If\n        the _'periodic'_ method is used, this parameter is required.\n    share_last_layer : Optional[bool], default=None\n        This parameter is not present in the before mentioned paper but it is implemented in\n        the [official repo](https://github.com/yandex-research/rtdl-num-embeddings/tree/main).\n        If `True` the linear layer that turns the frequencies into embeddings\n        will be shared across the continuous columns. If `False` a different\n        linear layer will be used for each continuous column.\n        If the _'periodic'_ method is used, this parameter is required.\n    full_embed_dropout: bool, Optional, default = None,\n        If `True`, the full embedding corresponding to a column will be masked\n        out/dropout. If `None`, it will default to `False`.\n    mlp_hidden_dims: List, default = [200, 100]\n        List with the number of neurons per dense layer in the mlp.\n    mlp_activation: str, default = \"relu\"\n        Activation function for the dense layers of the MLP. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_, _'gelu'_ and _'preglu'_ are\n          supported\n    mlp_dropout: float or List, default = 0.1\n        float or List of floats with the dropout between the dense layers.\n        e.g: _[0.5,0.5]_\n    mlp_batchnorm: bool, default = False\n        Boolean indicating whether or not batch normalization will be applied\n        to the dense layers\n    mlp_batchnorm_last: bool, default = False\n        Boolean indicating whether or not batch normalization will be applied\n        to the last of the dense layers\n    mlp_linear_first: bool, default = False\n        Boolean indicating the order of the operations in the dense\n        layer. If `True: [LIN -&gt; ACT -&gt; BN -&gt; DP]`. If `False: [BN -&gt; DP -&gt;\n        LIN -&gt; ACT]`\n\n    Attributes\n    ----------\n    user_behavior_indexes: List[int]\n        List with the indexes of the user behavior columns\n    user_behavior_embed: BaseTabularModelWithAttention\n        Embedding layer for the user\n    action_seq_indexes: List[int]\n        List with the indexes of the rating sequence columns if the\n        action_seq_config parameter is not None\n    action_embed: BaseTabularModelWithAttention\n        Embedding layer for the rating sequence columns if the\n        action_seq_config parameter is not None\n    other_seq_cols_indexes: Dict[str, List[int]]\n        Dictionary with the indexes of the other sequential columns if the\n        other_seq_cols_confiq parameter is not None\n    other_seq_cols_embed: nn.ModuleDict\n        Dictionary with the embedding layers for the other sequential columns\n        if the other_seq_cols_confiq parameter is not None\n    other_cols_idx: List[int]\n        List with the indexes of the other columns if the other_cols_config\n        parameter is not None\n    other_col_embed: BaseTabularModel\n        Embedding layer for the other columns if the other_cols_config\n        parameter is not None\n    mlp: Optional[MLP]\n        MLP component of the model. If None, no MLP will be used. This should\n        almost always be not None.\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; import numpy as np\n    &gt;&gt;&gt; from torch import Tensor\n    &gt;&gt;&gt; from typing import Dict, List, Tuple\n    &gt;&gt;&gt; from pytorch_widedeep.models.rec import DeepInterestNetwork\n    &gt;&gt;&gt; np_seed = np.random.seed(42)\n    &gt;&gt;&gt; torch_seed = torch.manual_seed(42)\n    &gt;&gt;&gt; num_users = 10\n    &gt;&gt;&gt; num_items = 5\n    &gt;&gt;&gt; num_contexts = 3\n    &gt;&gt;&gt; seq_length = 3\n    &gt;&gt;&gt; num_samples = 10\n    &gt;&gt;&gt; user_ids = np.random.randint(0, num_users, num_samples)\n    &gt;&gt;&gt; target_item_ids = np.random.randint(0, num_items, num_samples)\n    &gt;&gt;&gt; context_ids = np.random.randint(0, num_contexts, num_samples)\n    &gt;&gt;&gt; user_behavior = np.array(\n    ...     [\n    ...         np.random.choice(num_items, seq_length, replace=False)\n    ...         for _ in range(num_samples)\n    ...     ]\n    ... )\n    &gt;&gt;&gt; X_arr = np.column_stack((user_ids, target_item_ids, context_ids, user_behavior))\n    &gt;&gt;&gt; X = torch.tensor(X_arr, dtype=torch.long)\n    &gt;&gt;&gt; column_idx: Dict[str, int] = {\n    ...     \"user_id\": 0,\n    ...     \"target_item\": 1,\n    ...     \"context\": 2,\n    ...     \"item_1\": 3,\n    ...     \"item_2\": 4,\n    ...     \"item_3\": 5,\n    ... }\n    &gt;&gt;&gt; user_behavior_config: Tuple[List[str], int, int] = (\n    ...     [\"item_1\", \"item_2\", \"item_3\"],\n    ...     num_items,\n    ...     8,\n    ... )\n    &gt;&gt;&gt; cat_embed_input: List[Tuple[str, int, int]] = [\n    ...     (\"user_id\", num_users, 8),\n    ...     (\"context\", num_contexts, 4),\n    ... ]\n    &gt;&gt;&gt; model = DeepInterestNetwork(\n    ...     column_idx=column_idx,\n    ...     target_item_col=\"target_item\",\n    ...     user_behavior_confiq=user_behavior_config,\n    ...     cat_embed_input=cat_embed_input,\n    ...     mlp_hidden_dims=[16, 8],\n    ... )\n    &gt;&gt;&gt; output = model(X)\n    \"\"\"\n\n    def __init__(\n        self,\n        *,\n        column_idx: Dict[str, int],\n        user_behavior_confiq: Tuple[List[str], int, int],\n        target_item_col: str = \"target_item\",\n        action_seq_config: Optional[Tuple[List[str], int]] = None,\n        other_seq_cols_confiq: Optional[List[Tuple[List[str], int, int]]] = None,\n        attention_unit_activation: Literal[\"prelu\", \"dice\"] = \"prelu\",\n        cat_embed_input: Optional[List[Tuple[str, int, int]]] = None,\n        cat_embed_dropout: Optional[float] = None,\n        use_cat_bias: Optional[bool] = None,\n        cat_embed_activation: Optional[str] = None,\n        continuous_cols: Optional[List[str]] = None,\n        cont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\n        embed_continuous: Optional[bool] = None,\n        embed_continuous_method: Optional[\n            Literal[\"standard\", \"piecewise\", \"periodic\"]\n        ] = None,\n        cont_embed_dim: Optional[int] = None,\n        cont_embed_dropout: Optional[float] = None,\n        cont_embed_activation: Optional[str] = None,\n        quantization_setup: Optional[Dict[str, List[float]]] = None,\n        n_frequencies: Optional[int] = None,\n        sigma: Optional[float] = None,\n        share_last_layer: Optional[bool] = None,\n        full_embed_dropout: Optional[bool] = None,\n        mlp_hidden_dims: Optional[List[int]] = None,\n        mlp_activation: Optional[str] = None,\n        mlp_dropout: Optional[float] = None,\n        mlp_batchnorm: Optional[bool] = None,\n        mlp_batchnorm_last: Optional[bool] = None,\n        mlp_linear_first: Optional[bool] = None,\n    ):\n        super(DeepInterestNetwork, self).__init__()\n\n        self.column_idx = {\n            k: v for k, v in sorted(column_idx.items(), key=lambda x: x[1])\n        }\n\n        self.column_idx = column_idx\n        self.target_item_col = target_item_col\n        self.user_behavior_confiq = user_behavior_confiq\n        self.action_seq_config = (\n            (action_seq_config[0], action_seq_config[1], 1)\n            if action_seq_config is not None\n            else None\n        )\n        self.other_seq_cols_confiq = other_seq_cols_confiq\n        self.cat_embed_input = cat_embed_input\n        self.attention_unit_activation = attention_unit_activation\n\n        self.cat_embed_dropout = cat_embed_dropout\n        self.use_cat_bias = use_cat_bias\n        self.cat_embed_activation = cat_embed_activation\n        self.continuous_cols = continuous_cols\n        self.cont_norm_layer = cont_norm_layer\n        self.embed_continuous = embed_continuous\n        self.embed_continuous_method = embed_continuous_method\n        self.cont_embed_dim = cont_embed_dim\n        self.cont_embed_dropout = cont_embed_dropout\n        self.cont_embed_activation = cont_embed_activation\n        self.quantization_setup = quantization_setup\n        self.n_frequencies = n_frequencies\n        self.sigma = sigma\n        self.share_last_layer = share_last_layer\n        self.full_embed_dropout = full_embed_dropout\n\n        self.mlp_hidden_dims = mlp_hidden_dims\n        self.mlp_activation = mlp_activation\n        self.mlp_dropout = mlp_dropout\n        self.mlp_batchnorm = mlp_batchnorm\n        self.mlp_batchnorm_last = mlp_batchnorm_last\n        self.mlp_linear_first = mlp_linear_first\n\n        self.target_item_idx = self.column_idx[target_item_col]\n\n        self.user_behavior_indexes, self.user_behavior_embed = (\n            self.set_user_behavior_indexes_and_embed(user_behavior_confiq)\n        )\n        self.user_behavior_dim = user_behavior_confiq[2]\n\n        if self.action_seq_config is not None:\n            self.action_seq_indexes, self.action_embed = (\n                self._set_rating_indexes_and_embed(self.action_seq_config)\n            )\n        else:\n            self.action_embed = None\n\n        if self.other_seq_cols_confiq is not None:\n            (\n                self.other_seq_cols_indexes,\n                self.other_seq_cols_embed,\n                self.other_seq_dim,\n            ) = self._set_other_seq_cols_indexes_embed_and_dim(\n                self.other_seq_cols_confiq\n            )\n        else:\n            self.other_seq_cols_embed = None\n            self.other_seq_dim = 0\n\n        if self.cat_embed_input is not None or self.continuous_cols is not None:\n            self.other_cols_idx, self.other_col_embed, self.other_cols_dim = (\n                self._set_other_cols_idx_embed_and_dim(\n                    self.cat_embed_input, self.continuous_cols\n                )\n            )\n        else:\n            self.other_col_embed = None\n            self.other_cols_dim = 0\n\n        self.attention = ActivationUnit(\n            user_behavior_confiq[2], attention_unit_activation\n        )\n\n        if self.mlp_hidden_dims is not None:\n            mlp_input_dim = (\n                self.user_behavior_dim * 2 + self.other_seq_dim + self.other_cols_dim\n            )\n            self.mlp = MLP(\n                d_hidden=[mlp_input_dim] + self.mlp_hidden_dims,\n                activation=(\n                    \"relu\" if self.mlp_activation is None else self.mlp_activation\n                ),\n                dropout=0.0 if self.mlp_dropout is None else self.mlp_dropout,\n                batchnorm=False if self.mlp_batchnorm is None else self.mlp_batchnorm,\n                batchnorm_last=(\n                    False\n                    if self.mlp_batchnorm_last is None\n                    else self.mlp_batchnorm_last\n                ),\n                linear_first=(\n                    False if self.mlp_linear_first is None else self.mlp_linear_first\n                ),\n            )\n        else:\n            self.mlp = None\n\n    def forward(self, X: Tensor) -&gt; Tensor:\n\n        X_target_item = X[:, [self.target_item_idx]]\n        item_embed = self.user_behavior_embed.cat_embed.embed(X_target_item.long())\n\n        X_user_behavior = X[:, self.user_behavior_indexes]\n        user_behavior_embed = self.user_behavior_embed._get_embeddings(X_user_behavior)\n        # 0 is the padding index\n        mask = (X_user_behavior != 0).float().to(X.device)\n\n        if self.action_embed is not None:\n            X_rating = X[:, self.action_seq_indexes]\n            action_embed = self.action_embed._get_embeddings(X_rating)\n            user_behavior_embed = user_behavior_embed * action_embed\n\n        attention_scores = self.attention(item_embed, user_behavior_embed)\n        attention_scores = attention_scores * mask\n        user_interest = (attention_scores.unsqueeze(-1) * user_behavior_embed).sum(1)\n\n        deep_out = torch.cat([item_embed.squeeze(1), user_interest], dim=1)\n\n        if self.other_seq_cols_embed is not None:\n            X_other_seq: Dict[str, Tensor] = {\n                col: X[:, idx] for col, idx in self.other_seq_cols_indexes.items()\n            }\n            other_seq_embed = torch.cat(\n                [\n                    self.other_seq_cols_embed[col]._get_embeddings(X_other_seq[col])\n                    for col in self.other_seq_cols_indexes.keys()\n                ],\n                dim=-1,\n            ).sum(1)\n            deep_out = torch.cat([deep_out, other_seq_embed], dim=1)\n\n        if self.other_col_embed is not None:\n            X_other_cols = X[:, self.other_cols_idx]\n            other_cols_embed = self.other_col_embed._get_embeddings(X_other_cols)\n            deep_out = torch.cat([deep_out, other_cols_embed], dim=1)\n\n        if self.mlp is not None:\n            deep_out = self.mlp(deep_out)\n\n        return deep_out\n\n    @staticmethod\n    def _get_seq_cols_embed_confiq(tup: Tuple[List[str], int, int]) -&gt; Dict[str, Any]:\n        # tup[0] is the list of columns\n        # tup[1] is the number of unique feat value or \"n_tokens\"\n        # tup[2] is the embedding dimension\n\n        # Once sliced, the indexes will go from 0 to len(tup[0]) it is assumed\n        # that the columns in tup[0] are ordered of appearance in the input\n        # data\n        column_idx = {col: i for i, col in enumerate(tup[0])}\n\n        # This is a hack so that I can use any BaseTabularModelWithAttention.\n        # For this model to work 'cat_embed_input' is normally a List of\n        # Tuples where the first element is the column name and the second is\n        # the number of unique values for that column. That second elements\n        # is added internally to compute what one could call \"n_tokens\". Here\n        # I'm passing that value as the second element of the first tuple\n        # and then adding 0s for the rest of the columns\n        cat_embed_input = [(tup[0][0], tup[1])] + [(col, 0) for col in tup[0][1:]]\n\n        input_dim = tup[2]\n\n        col_config = {\n            \"column_idx\": column_idx,\n            \"input_dim\": input_dim,\n            \"cat_embed_input\": cat_embed_input,\n            \"cat_embed_dropout\": None,\n            \"use_cat_bias\": None,\n            \"cat_embed_activation\": None,\n            \"shared_embed\": None,\n            \"add_shared_embed\": None,\n            \"frac_shared_embed\": None,\n            \"continuous_cols\": None,\n            \"cont_norm_layer\": None,\n            \"embed_continuous_method\": None,\n            \"cont_embed_dropout\": None,\n            \"cont_embed_activation\": None,\n            \"quantization_setup\": None,\n            \"n_frequencies\": None,\n            \"sigma\": None,\n            \"share_last_layer\": None,\n            \"full_embed_dropout\": None,\n        }\n\n        return col_config\n\n    def _get_other_cols_embed_config(\n        self,\n        cat_embed_input: Optional[List[Tuple[str, int, int]]],\n        continuous_cols: Optional[List[str]],\n        column_idx: Dict[str, int],\n    ) -&gt; Dict[str, Any]:\n\n        cols_config = {\n            \"column_idx\": {col: i for i, col in enumerate(column_idx.keys())},\n            \"cat_embed_input\": cat_embed_input,\n            \"cat_embed_dropout\": self.cat_embed_dropout,\n            \"use_cat_bias\": self.use_cat_bias,\n            \"cat_embed_activation\": self.cat_embed_activation,\n            \"continuous_cols\": continuous_cols,\n            \"cont_norm_layer\": self.cont_norm_layer,\n            \"embed_continuous\": self.embed_continuous,\n            \"embed_continuous_method\": self.embed_continuous_method,\n            \"cont_embed_dim\": self.cont_embed_dim,\n            \"cont_embed_dropout\": self.cont_embed_dropout,\n            \"cont_embed_activation\": self.cont_embed_activation,\n            \"quantization_setup\": self.quantization_setup,\n            \"n_frequencies\": self.n_frequencies,\n            \"sigma\": self.sigma,\n            \"share_last_layer\": self.share_last_layer,\n            \"full_embed_dropout\": self.full_embed_dropout,\n        }\n\n        return cols_config\n\n    def set_user_behavior_indexes_and_embed(\n        self, user_behavior_confiq: Tuple[List[str], int, int]\n    ) -&gt; Tuple[List[int], BaseTabularModelWithAttention]:\n        user_behavior_indexes = [\n            self.column_idx[col] for col in user_behavior_confiq[0]\n        ]\n        user_behavior_embed = BaseTabularModelWithAttention(\n            **self._get_seq_cols_embed_confiq(user_behavior_confiq)\n        )\n        return user_behavior_indexes, user_behavior_embed\n\n    def _set_rating_indexes_and_embed(\n        self, action_seq_config: Tuple[List[str], int, int]\n    ) -&gt; Tuple[List[int], BaseTabularModelWithAttention]:\n        action_seq_indexes = [self.column_idx[col] for col in action_seq_config[0]]\n        action_embed = BaseTabularModelWithAttention(\n            **self._get_seq_cols_embed_confiq(action_seq_config)\n        )\n        return action_seq_indexes, action_embed\n\n    def _set_other_seq_cols_indexes_embed_and_dim(\n        self, other_seq_cols_confiq: List[Tuple[List[str], int, int]]\n    ) -&gt; Tuple[Dict[str, List[int]], nn.ModuleDict, int]:\n        other_seq_cols_indexes: Dict[str, List[int]] = {}\n        for i, el in enumerate(other_seq_cols_confiq):\n            key = f\"seq_{i}\"\n            idxs = [self.column_idx[col] for col in el[0]]\n            other_seq_cols_indexes[key] = idxs\n        other_seq_cols_config = {\n            f\"seq_{i}\": self._get_seq_cols_embed_confiq(el)\n            for i, el in enumerate(other_seq_cols_confiq)\n        }\n        other_seq_cols_embed = nn.ModuleDict(\n            {\n                key: BaseTabularModelWithAttention(**config)\n                for key, config in other_seq_cols_config.items()\n            }\n        )\n        other_seq_dim = sum([el[2] for el in other_seq_cols_confiq])\n\n        return other_seq_cols_indexes, other_seq_cols_embed, other_seq_dim\n\n    def _set_other_cols_idx_embed_and_dim(\n        self,\n        cat_embed_input: Optional[List[Tuple[str, int, int]]],\n        continuous_cols: Optional[List[str]],\n    ) -&gt; Tuple[List[int], BaseTabularModelWithoutAttention, int]:\n        other_cols_idx: Dict[str, int] = {}\n        if cat_embed_input is not None:\n            other_cols_idx = {\n                col: self.column_idx[col] for col in [el[0] for el in cat_embed_input]\n            }\n        if continuous_cols is not None:\n            other_cols_idx.update(\n                {col: self.column_idx[col] for col in continuous_cols}\n            )\n\n        sorted_other_cols_idx = {\n            k: v for k, v in sorted(other_cols_idx.items(), key=lambda x: x[1])\n        }\n\n        other_col_embed = BaseTabularModelWithoutAttention(\n            **self._get_other_cols_embed_config(\n                cat_embed_input, continuous_cols, sorted_other_cols_idx\n            )\n        )\n\n        other_cols_dim = other_col_embed.output_dim\n\n        return list(other_cols_idx.values()), other_col_embed, other_cols_dim\n\n    @property\n    def output_dim(self) -&gt; int:\n        if self.mlp_hidden_dims is not None:\n            return self.mlp_hidden_dims[-1]\n        else:\n            return self.user_behavior_dim * 2 + self.other_seq_dim + self.other_cols_dim\n</code></pre>"},{"location":"pytorch-widedeep/the_rec_module.html#pytorch_widedeep.models.rec.xdeepfm.ExtremeDeepFactorizationMachine","title":"ExtremeDeepFactorizationMachine","text":"<p>               Bases: <code>BaseTabularModelWithAttention</code></p> <p>Adaptation of 'xDeepFM implementation: xDeepFM: Combining Explicit and Implicit Feature Interactions for Recommender Systems' by Jianxun Lian, Xiaohuan Zhou, Fuzheng Zhang, Zhongxia Chen, Xing Xie, Guangzhong Sun and Enhong Chen, 2018</p> <p>The implementation in this library takes advantage of all the functionalities available to encode categorical and continuous features. The model can be used with only the factorization machine</p> <p>Note that this class implements only the 'Deep' component of the model described in the paper. The linear component is not implemented 'internally' and, if one wants to include it, it can be easily added using the 'wide'/linear component in this library. See the examples in the examples folder.</p> <p>Parameters:</p> Name Type Description Default <code>column_idx</code> <code>Dict[str, int]</code> <p>Dictionary mapping column names to their corresponding index.</p> required <code>input_dim</code> <code>int</code> <p>Embedding input dimensions</p> required <code>reduce_sum</code> <code>bool</code> <p>Whether to reduce the sum in the factorization machine output.</p> <code>True</code> <code>cin_layer_dims</code> <code>List[int]</code> <p>List with the number of units per CIN layer. e.g: [128, 64]</p> required <code>cat_embed_input</code> <code>Optional[List[Tuple[str, int]]]</code> <p>List of tuples with categorical column names and number of unique values.</p> <code>None</code> <code>cat_embed_dropout</code> <code>Optional[float]</code> <p>Categorical embeddings dropout. If <code>None</code>, it will default to 0.</p> <code>None</code> <code>use_cat_bias</code> <code>Optional[bool]</code> <p>Boolean indicating if bias will be used for the categorical embeddings. If <code>None</code>, it will default to 'False'.</p> <code>None</code> <code>cat_embed_activation</code> <code>Optional[str]</code> <p>Activation function for the categorical embeddings, if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> <code>None</code> <code>continuous_cols</code> <code>Optional[List[str]]</code> <p>List with the name of the numeric (aka continuous) columns</p> <code>None</code> <code>cont_norm_layer</code> <code>Optional[Literal['batchnorm', 'layernorm']]</code> <p>Type of normalization layer applied to the continuous features. Options are: 'layernorm' and 'batchnorm'. if <code>None</code>, no normalization layer will be used.</p> <code>None</code> <code>embed_continuous_method</code> <code>Optional[Literal['piecewise', 'periodic', 'standard']]</code> <p>Method to use to embed the continuous features. Options are: 'standard', 'periodic' or 'piecewise'. The 'standard' embedding method is based on the FT-Transformer implementation presented in the paper: Revisiting Deep Learning Models for Tabular Data. The 'periodic' and_'piecewise'_ methods were presented in the paper: On Embeddings for Numerical Features in Tabular Deep Learning. Please, read the papers for details.</p> <code>'standard'</code> <code>cont_embed_dropout</code> <code>Optional[float]</code> <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p> <code>None</code> <code>cont_embed_activation</code> <code>Optional[str]</code> <p>Activation function for the continuous embeddings if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported. If <code>None</code>, no activation function will be applied.</p> <code>None</code> <code>quantization_setup</code> <code>Optional[Dict[str, List[float]]]</code> <p>This parameter is used when the 'piecewise' method is used to embed the continuous cols. It is a dict where keys are the name of the continuous columns and values are lists with the boundaries for the quantization of the continuous_cols. See the examples for details. If If the 'piecewise' method is used, this parameter is required.</p> <code>None</code> <code>n_frequencies</code> <code>Optional[int]</code> <p>This is the so called 'k' in their paper On Embeddings for Numerical Features in Tabular Deep Learning, and is the number of 'frequencies' that will be used to represent each continuous column. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>sigma</code> <code>Optional[float]</code> <p>This is the sigma parameter in the paper mentioned when describing the previous parameters and it is used to initialise the 'frequency weights'. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>share_last_layer</code> <code>Optional[bool]</code> <p>This parameter is not present in the before mentioned paper but it is implemented in the official repo. If <code>True</code> the linear layer that turns the frequencies into embeddings will be shared across the continuous columns. If <code>False</code> a different linear layer will be used for each continuous column. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>full_embed_dropout</code> <code>Optional[bool]</code> <p>If <code>True</code>, the full embedding corresponding to a column will be masked out/dropout. If <code>None</code>, it will default to <code>False</code>.</p> <code>None</code> <code>mlp_hidden_dims</code> <code>Optional[List[int]]</code> <p>List with the number of neurons per dense layer in the mlp.</p> <code>None</code> <code>mlp_activation</code> <code>Optional[str]</code> <p>Activation function for the dense layers of the MLP. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> <code>None</code> <code>mlp_dropout</code> <code>Optional[float]</code> <p>float or List of floats with the dropout between the dense layers. e.g: [0.5,0.5]</p> <code>None</code> <code>mlp_batchnorm</code> <code>Optional[bool]</code> <p>Boolean indicating whether or not batch normalization will be applied to the dense layers</p> <code>None</code> <code>mlp_batchnorm_last</code> <code>Optional[bool]</code> <p>Boolean indicating whether or not batch normalization will be applied to the last of the dense layers</p> <code>None</code> <code>mlp_linear_first</code> <code>Optional[bool]</code> <p>Boolean indicating the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code></p> <code>None</code> <p>Attributes:</p> Name Type Description <code>n_features</code> <code>int</code> <p>Number of unique features/columns</p> <code>cin</code> <code>CompressedInteractionNetwork</code> <p>Instance of the <code>CompressedInteractionNetwork</code> class</p> <code>mlp</code> <code>MLP</code> <p>Instance of the <code>MLP</code> class if <code>mlp_hidden_dims</code> is not None. If None, the model will return directly the output of the <code>CIN</code></p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models.rec import ExtremeDeepFactorizationMachine\n&gt;&gt;&gt; X_tab = torch.randint(0, 10, (16, 2))\n&gt;&gt;&gt; column_idx = {\"col1\": 0, \"col2\": 1}\n&gt;&gt;&gt; cat_embed_input = [(\"col1\", 10), (\"col2\", 10)]\n&gt;&gt;&gt; xdeepfm = ExtremeDeepFactorizationMachine(\n...     column_idx=column_idx,\n...     input_dim=4,\n...     cin_layer_dims=[8, 16],\n...     cat_embed_input=cat_embed_input,\n...     mlp_hidden_dims=[16, 8]\n... )\n&gt;&gt;&gt; output = xdeepfm(X_tab)\n</code></pre> Source code in <code>pytorch_widedeep/models/rec/xdeepfm.py</code> <pre><code>class ExtremeDeepFactorizationMachine(BaseTabularModelWithAttention):\n    \"\"\"\n    Adaptation of 'xDeepFM implementation: xDeepFM: Combining Explicit and\n    Implicit Feature Interactions for Recommender Systems' by Jianxun Lian,\n    Xiaohuan Zhou, Fuzheng Zhang, Zhongxia Chen, Xing Xie, Guangzhong Sun and\n    Enhong Chen, 2018\n\n    The implementation in this library takes advantage of all the\n    functionalities available to encode categorical and continuous features.\n    The model can be used with only the factorization machine\n\n    Note that this class implements only the 'Deep' component of the model\n    described in the paper. The linear component is not\n    implemented 'internally' and, if one wants to include it, it can be\n    easily added using the 'wide'/linear component in this library. See the\n    examples in the examples folder.\n\n    Parameters\n    ----------\n    column_idx : Dict[str, int]\n        Dictionary mapping column names to their corresponding index.\n    input_dim : int\n        Embedding input dimensions\n    reduce_sum : bool, default=True\n        Whether to reduce the sum in the factorization machine output.\n    cin_layer_dims : List[int]\n        List with the number of units per CIN layer. e.g: _[128, 64]_\n    cat_embed_input : Optional[List[Tuple[str, int]]], default=None\n        List of tuples with categorical column names and number of unique values.\n    cat_embed_dropout : Optional[float], default=None\n        Categorical embeddings dropout. If `None`, it will default\n        to 0.\n    use_cat_bias : Optional[bool], default=None\n        Boolean indicating if bias will be used for the categorical embeddings.\n        If `None`, it will default to 'False'.\n    cat_embed_activation : Optional[str], default=None\n        Activation function for the categorical embeddings, if any. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported\n    continuous_cols : Optional[List[str]], default=None\n        List with the name of the numeric (aka continuous) columns\n    cont_norm_layer : Optional[Literal[\"batchnorm\", \"layernorm\"]], default=None\n        Type of normalization layer applied to the continuous features.\n        Options are: _'layernorm'_ and _'batchnorm'_. if `None`, no\n        normalization layer will be used.\n    embed_continuous_method: Optional[Literal[\"piecewise\", \"periodic\", \"standard\"]], default=\"standard\"\n        Method to use to embed the continuous features. Options are:\n        _'standard'_, _'periodic'_ or _'piecewise'_. The _'standard'_\n        embedding method is based on the FT-Transformer implementation\n        presented in the paper: [Revisiting Deep Learning Models for\n        Tabular Data](https://arxiv.org/abs/2106.11959v5). The _'periodic'_\n        and_'piecewise'_ methods were presented in the paper: [On Embeddings for\n        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556).\n        Please, read the papers for details.\n    cont_embed_dropout : Optional[float], default=None\n        Dropout for the continuous embeddings. If `None`, it will default to 0.0\n    cont_embed_activation : Optional[str], default=None\n        Activation function for the continuous embeddings if any. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported.\n        If `None`, no activation function will be applied.\n    quantization_setup : Optional[Dict[str, List[float]]], default=None\n        This parameter is used when the _'piecewise'_ method is used to embed\n        the continuous cols. It is a dict where keys are the name of the continuous\n        columns and values are lists with the boundaries for the quantization\n        of the continuous_cols. See the examples for details. If\n        If the _'piecewise'_ method is used, this parameter is required.\n    n_frequencies : Optional[int], default=None\n        This is the so called _'k'_ in their paper [On Embeddings for\n        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556),\n        and is the number of 'frequencies' that will be used to represent each\n        continuous column. See their Eq 2 in the paper for details. If\n        the _'periodic'_ method is used, this parameter is required.\n    sigma : Optional[float], default=None\n        This is the sigma parameter in the paper mentioned when describing the\n        previous parameters and it is used to initialise the 'frequency\n        weights'. See their Eq 2 in the paper for details. If\n        the _'periodic'_ method is used, this parameter is required.\n    share_last_layer : Optional[bool], default=None\n        This parameter is not present in the before mentioned paper but it is implemented in\n        the [official repo](https://github.com/yandex-research/rtdl-num-embeddings/tree/main).\n        If `True` the linear layer that turns the frequencies into embeddings\n        will be shared across the continuous columns. If `False` a different\n        linear layer will be used for each continuous column.\n        If the _'periodic'_ method is used, this parameter is required.\n    full_embed_dropout: bool, Optional, default = None,\n        If `True`, the full embedding corresponding to a column will be masked\n        out/dropout. If `None`, it will default to `False`.\n    mlp_hidden_dims: List, default = [200, 100]\n        List with the number of neurons per dense layer in the mlp.\n    mlp_activation: str, default = \"relu\"\n        Activation function for the dense layers of the MLP. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported\n    mlp_dropout: float or List, default = 0.1\n        float or List of floats with the dropout between the dense layers.\n        e.g: _[0.5,0.5]_\n    mlp_batchnorm: bool, default = False\n        Boolean indicating whether or not batch normalization will be applied\n        to the dense layers\n    mlp_batchnorm_last: bool, default = False\n        Boolean indicating whether or not batch normalization will be applied\n        to the last of the dense layers\n    mlp_linear_first: bool, default = False\n        Boolean indicating the order of the operations in the dense\n        layer. If `True: [LIN -&gt; ACT -&gt; BN -&gt; DP]`. If `False: [BN -&gt; DP -&gt;\n        LIN -&gt; ACT]`\n\n    Attributes\n    ----------\n    n_features: int\n        Number of unique features/columns\n    cin: CompressedInteractionNetwork\n        Instance of the `CompressedInteractionNetwork` class\n    mlp: MLP\n        Instance of the `MLP` class if `mlp_hidden_dims` is not None. If None,\n        the model will return directly the output of the `CIN`\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.models.rec import ExtremeDeepFactorizationMachine\n    &gt;&gt;&gt; X_tab = torch.randint(0, 10, (16, 2))\n    &gt;&gt;&gt; column_idx = {\"col1\": 0, \"col2\": 1}\n    &gt;&gt;&gt; cat_embed_input = [(\"col1\", 10), (\"col2\", 10)]\n    &gt;&gt;&gt; xdeepfm = ExtremeDeepFactorizationMachine(\n    ...     column_idx=column_idx,\n    ...     input_dim=4,\n    ...     cin_layer_dims=[8, 16],\n    ...     cat_embed_input=cat_embed_input,\n    ...     mlp_hidden_dims=[16, 8]\n    ... )\n    &gt;&gt;&gt; output = xdeepfm(X_tab)\n    \"\"\"\n\n    def __init__(\n        self,\n        *,\n        column_idx: Dict[str, int],\n        input_dim: int,\n        reduce_sum: bool = True,\n        cin_layer_dims: List[int],\n        cat_embed_input: List[Tuple[str, int]],\n        cat_embed_dropout: Optional[float] = None,\n        use_cat_bias: Optional[bool] = None,\n        cat_embed_activation: Optional[str] = None,\n        continuous_cols: Optional[List[str]] = None,\n        cont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\n        embed_continuous_method: Optional[\n            Literal[\"piecewise\", \"periodic\", \"standard\"]\n        ] = \"standard\",\n        cont_embed_dropout: Optional[float] = None,\n        cont_embed_activation: Optional[str] = None,\n        quantization_setup: Optional[Dict[str, List[float]]] = None,\n        n_frequencies: Optional[int] = None,\n        sigma: Optional[float] = None,\n        share_last_layer: Optional[bool] = None,\n        full_embed_dropout: Optional[bool] = None,\n        mlp_hidden_dims: Optional[List[int]] = None,\n        mlp_activation: Optional[str] = None,\n        mlp_dropout: Optional[float] = None,\n        mlp_batchnorm: Optional[bool] = None,\n        mlp_batchnorm_last: Optional[bool] = None,\n        mlp_linear_first: Optional[bool] = None,\n    ):\n        super(ExtremeDeepFactorizationMachine, self).__init__(\n            column_idx=column_idx,\n            input_dim=input_dim,\n            cat_embed_input=cat_embed_input,\n            cat_embed_dropout=cat_embed_dropout,\n            use_cat_bias=use_cat_bias,\n            cat_embed_activation=cat_embed_activation,\n            shared_embed=False,\n            add_shared_embed=None,\n            frac_shared_embed=None,\n            continuous_cols=continuous_cols,\n            cont_norm_layer=cont_norm_layer,\n            embed_continuous_method=embed_continuous_method,\n            cont_embed_dropout=cont_embed_dropout,\n            cont_embed_activation=cont_embed_activation,\n            quantization_setup=quantization_setup,\n            n_frequencies=n_frequencies,\n            sigma=sigma,\n            share_last_layer=share_last_layer,\n            full_embed_dropout=full_embed_dropout,\n        )\n\n        self.mlp_hidden_dims = mlp_hidden_dims\n        self.mlp_activation = mlp_activation\n        self.mlp_dropout = mlp_dropout\n        self.mlp_batchnorm = mlp_batchnorm\n        self.mlp_batchnorm_last = mlp_batchnorm_last\n        self.mlp_linear_first = mlp_linear_first\n\n        self.reduce_sum = reduce_sum\n        self.cin_layer_dims = cin_layer_dims\n\n        self.n_features = len(self.column_idx)\n\n        self.cin = CompressedInteractionNetwork(\n            num_cols=self.n_features, cin_layer_dims=self.cin_layer_dims\n        )\n\n        if self.mlp_hidden_dims is not None:\n            if (\n                self.mlp_hidden_dims[-1] != sum(self.cin_layer_dims)\n                and not self.reduce_sum\n            ):\n                d_hidden = (\n                    [sum(self.cin_layer_dims)]\n                    + self.mlp_hidden_dims\n                    + [sum(self.cin_layer_dims)]\n                )\n            else:\n                d_hidden = [sum(self.cin_layer_dims)] + self.mlp_hidden_dims\n\n            self.mlp = MLP(\n                d_hidden=d_hidden,\n                activation=(\n                    \"relu\" if self.mlp_activation is None else self.mlp_activation\n                ),\n                dropout=0.0 if self.mlp_dropout is None else self.mlp_dropout,\n                batchnorm=False if self.mlp_batchnorm is None else self.mlp_batchnorm,\n                batchnorm_last=(\n                    False\n                    if self.mlp_batchnorm_last is None\n                    else self.mlp_batchnorm_last\n                ),\n                linear_first=(\n                    False if self.mlp_linear_first is None else self.mlp_linear_first\n                ),\n            )\n        else:\n            self.mlp = None\n\n    def forward(self, X: Tensor) -&gt; Tensor:\n\n        embeddings = self._get_embeddings(X)\n        cin_out = self.cin(embeddings)\n\n        if self.mlp is None:\n            if self.reduce_sum:\n                return cin_out.sum(dim=1, keepdim=True)\n            return cin_out\n\n        mlp_out = self.mlp(cin_out)\n\n        if self.reduce_sum:\n            cin_out = cin_out.sum(dim=1, keepdim=True)\n            mlp_out = mlp_out.sum(dim=1, keepdim=True)\n\n        return mlp_out + cin_out\n\n    @property\n    def output_dim(self):\n        if self.reduce_sum:\n            return 1\n        else:\n            return sum(self.cin_layer_dims)\n</code></pre>"},{"location":"pytorch-widedeep/the_rec_module.html#pytorch_widedeep.models.rec.autoint.AutoInt","title":"AutoInt","text":"<p>               Bases: <code>BaseTabularModelWithAttention</code></p> <p>Defines an <code>AutoInt</code> model that can be used as the <code>deeptabular</code> component of a Wide &amp; Deep model or independently by itself.</p> <p>This class implements the AutoInt: Automatic Feature Interaction Learning via Self-Attentive Neural Networks architecture, which learns feature interactions through multi-head self-attention networks.</p> <p>Parameters:</p> Name Type Description Default <code>column_idx</code> <code>Dict[str, int]</code> <p>Dict containing the index of the columns that will be passed through the <code>AutoInt</code> model. Required to slice the tensors. e.g. {'education': 0, 'relationship': 1, 'workclass': 2, ...}.</p> required <code>input_dim</code> <code>int</code> <p>Dimension of the input embeddings</p> required <code>num_heads</code> <code>int</code> <p>Number of attention heads</p> <code>4</code> <code>num_layers</code> <code>int</code> <p>Number of interacting layers (attention + residual)</p> <code>2</code> <code>reduction</code> <code>Literal['mean', 'cat']</code> <p>How to reduce the output of the attention layers. Options are: 'mean': mean of attention outputs 'cat': concatenation of attention outputs</p> <code>'mean'</code> <code>cat_embed_input</code> <code>Optional[List[Tuple[str, int]]]</code> <p>List of Tuples with the column name, number of unique values and embedding dimension. e.g. [(education, 11, 32), ...]</p> <code>None</code> <code>cat_embed_dropout</code> <code>Optional[float]</code> <p>Categorical embeddings dropout. If <code>None</code>, it will default to 0.</p> <code>None</code> <code>use_cat_bias</code> <code>Optional[bool]</code> <p>Boolean indicating if bias will be used for the categorical embeddings</p> <code>None</code> <code>cat_embed_activation</code> <code>Optional[str]</code> <p>Activation function for the categorical embeddings</p> <code>None</code> <code>continuous_cols</code> <code>Optional[List[str]]</code> <p>List with the name of the numeric (aka continuous) columns</p> <code>None</code> <code>cont_norm_layer</code> <code>Optional[Literal['batchnorm', 'layernorm']]</code> <p>Type of normalization layer applied to the continuous features. Options are: 'layernorm' and 'batchnorm'. if <code>None</code>, no normalization layer will be used.</p> <code>None</code> <code>embed_continuous_method</code> <code>Optional[Literal['standard', 'piecewise', 'periodic']]</code> <p>Method to use to embed the continuous features. Options are: 'standard', 'periodic' or 'piecewise'. The 'standard' embedding method is based on the FT-Transformer implementation presented in the paper: Revisiting Deep Learning Models for Tabular Data. The 'periodic' and_'piecewise'_ methods were presented in the paper: On Embeddings for Numerical Features in Tabular Deep Learning. Please, read the papers for details.</p> <code>None</code> <code>cont_embed_dropout</code> <code>Optional[float]</code> <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p> <code>None</code> <code>cont_embed_activation</code> <code>Optional[str]</code> <p>Activation function for the continuous embeddings if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported. If <code>None</code>, no activation function will be applied.</p> <code>None</code> <code>quantization_setup</code> <code>Optional[Dict[str, List[float]]]</code> <p>This parameter is used when the 'piecewise' method is used to embed the continuous cols. It is a dict where keys are the name of the continuous columns and values are lists with the boundaries for the quantization of the continuous_cols. See the examples for details. If If the 'piecewise' method is used, this parameter is required.</p> <code>None</code> <code>n_frequencies</code> <code>Optional[int]</code> <p>This is the so called 'k' in their paper On Embeddings for Numerical Features in Tabular Deep Learning, and is the number of 'frequencies' that will be used to represent each continuous column. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>sigma</code> <code>Optional[float]</code> <p>This is the sigma parameter in the paper mentioned when describing the previous parameters and it is used to initialise the 'frequency weights'. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>share_last_layer</code> <code>Optional[bool]</code> <p>This parameter is not present in the before mentioned paper but it is implemented in the official repo. If <code>True</code> the linear layer that turns the frequencies into embeddings will be shared across the continuous columns. If <code>False</code> a different linear layer will be used for each continuous column. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>full_embed_dropout</code> <code>Optional[bool]</code> <p>If <code>True</code>, the full embedding corresponding to a column will be masked out/dropout. If <code>None</code>, it will default to <code>False</code>.</p> <code>None</code> <p>Attributes:</p> Name Type Description <code>attention_layers</code> <code>ModuleList</code> <p>List of multi-head attention layers</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models.rec import AutoInt\n&gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n&gt;&gt;&gt; colnames = [\"a\", \"b\", \"c\", \"d\", \"e\"]\n&gt;&gt;&gt; cat_embed_input = [(u, i, j) for u, i, j in zip(colnames[:4], [4] * 4, [8] * 4)]\n&gt;&gt;&gt; column_idx = {k: v for v, k in enumerate(colnames)}\n&gt;&gt;&gt; model = AutoInt(\n...     column_idx=column_idx,\n...     input_dim=32,\n...     cat_embed_input=cat_embed_input,\n...     continuous_cols=[\"e\"],\n...     embed_continuous_method=\"standard\",\n...     num_heads=4,\n...     num_layers=2\n... )\n&gt;&gt;&gt; out = model(X_tab)\n</code></pre> Source code in <code>pytorch_widedeep/models/rec/autoint.py</code> <pre><code>class AutoInt(BaseTabularModelWithAttention):\n    r\"\"\"Defines an `AutoInt` model that can be used as the `deeptabular` component\n    of a Wide &amp; Deep model or independently by itself.\n\n    This class implements the [AutoInt: Automatic Feature Interaction Learning via Self-Attentive\n    Neural Networks](https://arxiv.org/abs/1810.11921) architecture, which learns feature\n    interactions through multi-head self-attention networks.\n\n    Parameters\n    ----------\n    column_idx: Dict\n        Dict containing the index of the columns that will be passed through\n        the `AutoInt` model. Required to slice the tensors. e.g.\n        _{'education': 0, 'relationship': 1, 'workclass': 2, ...}_.\n    input_dim: int\n        Dimension of the input embeddings\n    num_heads: int, default = 4\n        Number of attention heads\n    num_layers: int, default = 2\n        Number of interacting layers (attention + residual)\n    reduction: str, default = \"mean\"\n        How to reduce the output of the attention layers. Options are:\n        _'mean'_: mean of attention outputs\n        _'cat'_: concatenation of attention outputs\n    cat_embed_input: List, Optional, default = None\n        List of Tuples with the column name, number of unique values and\n        embedding dimension. e.g. _[(education, 11, 32), ...]_\n    cat_embed_dropout: float, Optional, default = None\n        Categorical embeddings dropout. If `None`, it will default to 0.\n    use_cat_bias: bool, Optional, default = None,\n        Boolean indicating if bias will be used for the categorical embeddings\n    cat_embed_activation: Optional, str, default = None,\n        Activation function for the categorical embeddings\n    continuous_cols : Optional[List[str]], default=None\n        List with the name of the numeric (aka continuous) columns\n    cont_norm_layer : Optional[Literal[\"batchnorm\", \"layernorm\"]], default=None\n        Type of normalization layer applied to the continuous features.\n        Options are: _'layernorm'_ and _'batchnorm'_. if `None`, no\n        normalization layer will be used.\n    embed_continuous_method: Optional[Literal[\"piecewise\", \"periodic\", \"standard\"]], default=\"standard\"\n        Method to use to embed the continuous features. Options are:\n        _'standard'_, _'periodic'_ or _'piecewise'_. The _'standard'_\n        embedding method is based on the FT-Transformer implementation\n        presented in the paper: [Revisiting Deep Learning Models for\n        Tabular Data](https://arxiv.org/abs/2106.11959v5). The _'periodic'_\n        and_'piecewise'_ methods were presented in the paper: [On Embeddings for\n        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556).\n        Please, read the papers for details.\n    cont_embed_dropout : Optional[float], default=None\n        Dropout for the continuous embeddings. If `None`, it will default to 0.0\n    cont_embed_activation : Optional[str], default=None\n        Activation function for the continuous embeddings if any. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported.\n        If `None`, no activation function will be applied.\n    quantization_setup : Optional[Dict[str, List[float]]], default=None\n        This parameter is used when the _'piecewise'_ method is used to embed\n        the continuous cols. It is a dict where keys are the name of the continuous\n        columns and values are lists with the boundaries for the quantization\n        of the continuous_cols. See the examples for details. If\n        If the _'piecewise'_ method is used, this parameter is required.\n    n_frequencies : Optional[int], default=None\n        This is the so called _'k'_ in their paper [On Embeddings for\n        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556),\n        and is the number of 'frequencies' that will be used to represent each\n        continuous column. See their Eq 2 in the paper for details. If\n        the _'periodic'_ method is used, this parameter is required.\n    sigma : Optional[float], default=None\n        This is the sigma parameter in the paper mentioned when describing the\n        previous parameters and it is used to initialise the 'frequency\n        weights'. See their Eq 2 in the paper for details. If\n        the _'periodic'_ method is used, this parameter is required.\n    share_last_layer : Optional[bool], default=None\n        This parameter is not present in the before mentioned paper but it is implemented in\n        the [official repo](https://github.com/yandex-research/rtdl-num-embeddings/tree/main).\n        If `True` the linear layer that turns the frequencies into embeddings\n        will be shared across the continuous columns. If `False` a different\n        linear layer will be used for each continuous column.\n        If the _'periodic'_ method is used, this parameter is required.\n    full_embed_dropout: bool, Optional, default = None,\n        If `True`, the full embedding corresponding to a column will be masked\n        out/dropout. If `None`, it will default to `False`.\n\n    Attributes\n    ----------\n    attention_layers: nn.ModuleList\n        List of multi-head attention layers\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.models.rec import AutoInt\n    &gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n    &gt;&gt;&gt; colnames = [\"a\", \"b\", \"c\", \"d\", \"e\"]\n    &gt;&gt;&gt; cat_embed_input = [(u, i, j) for u, i, j in zip(colnames[:4], [4] * 4, [8] * 4)]\n    &gt;&gt;&gt; column_idx = {k: v for v, k in enumerate(colnames)}\n    &gt;&gt;&gt; model = AutoInt(\n    ...     column_idx=column_idx,\n    ...     input_dim=32,\n    ...     cat_embed_input=cat_embed_input,\n    ...     continuous_cols=[\"e\"],\n    ...     embed_continuous_method=\"standard\",\n    ...     num_heads=4,\n    ...     num_layers=2\n    ... )\n    &gt;&gt;&gt; out = model(X_tab)\n    \"\"\"\n\n    def __init__(\n        self,\n        *,\n        column_idx: Dict[str, int],\n        input_dim: int,\n        num_heads: int = 4,\n        num_layers: int = 2,\n        reduction: Literal[\"mean\", \"cat\"] = \"mean\",\n        cat_embed_input: Optional[List[Tuple[str, int]]] = None,\n        cat_embed_dropout: Optional[float] = None,\n        use_cat_bias: Optional[bool] = None,\n        cat_embed_activation: Optional[str] = None,\n        continuous_cols: Optional[List[str]] = None,\n        cont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\n        embed_continuous_method: Optional[\n            Literal[\"standard\", \"piecewise\", \"periodic\"]\n        ] = None,\n        cont_embed_dropout: Optional[float] = None,\n        cont_embed_activation: Optional[str] = None,\n        quantization_setup: Optional[Dict[str, List[float]]] = None,\n        n_frequencies: Optional[int] = None,\n        sigma: Optional[float] = None,\n        share_last_layer: Optional[bool] = None,\n        full_embed_dropout: Optional[bool] = None,\n    ):\n        super(AutoInt, self).__init__(\n            column_idx=column_idx,\n            input_dim=input_dim,\n            cat_embed_input=cat_embed_input,\n            cat_embed_dropout=cat_embed_dropout,\n            use_cat_bias=use_cat_bias,\n            cat_embed_activation=cat_embed_activation,\n            shared_embed=None,\n            add_shared_embed=None,\n            frac_shared_embed=None,\n            continuous_cols=continuous_cols,\n            cont_norm_layer=cont_norm_layer,\n            embed_continuous_method=embed_continuous_method,\n            cont_embed_dropout=cont_embed_dropout,\n            cont_embed_activation=cont_embed_activation,\n            quantization_setup=quantization_setup,\n            n_frequencies=n_frequencies,\n            sigma=sigma,\n            share_last_layer=share_last_layer,\n            full_embed_dropout=full_embed_dropout,\n        )\n\n        self.num_heads = num_heads\n        self.num_layers = num_layers\n        self.reduction = reduction\n\n        self.attention_layers = nn.ModuleList(\n            [\n                nn.MultiheadAttention(\n                    embed_dim=input_dim,\n                    num_heads=num_heads,\n                    batch_first=True,\n                )\n                for _ in range(num_layers)\n            ]\n        )\n\n    def forward(self, X: torch.Tensor) -&gt; torch.Tensor:\n        x = self._get_embeddings(X)\n\n        for layer in self.attention_layers:\n            attn_output, _ = layer(x, x, x)\n            x = attn_output + x\n\n        if self.reduction == \"mean\":\n            out = x.mean(dim=1)\n        else:\n            out = x.reshape(x.size(0), -1)\n\n        return out\n\n    @property\n    def output_dim(self) -&gt; int:\n        if self.reduction == \"mean\":\n            return self.input_dim\n        else:\n            return self.input_dim * len(self.column_idx)\n</code></pre>"},{"location":"pytorch-widedeep/the_rec_module.html#pytorch_widedeep.models.rec.autoint_plus.AutoIntPlus","title":"AutoIntPlus","text":"<p>               Bases: <code>BaseTabularModelWithAttention</code></p> <p>Defines an <code>AutoIntPlus</code> model that can be used as the <code>deeptabular</code> component of a Wide &amp; Deep model or independently by itself.</p> <p>This class implements an enhanced version of the AutoInt architecture, adding a parallel or stacked deep network and an optional gating mechanism to control the contribution of the attention-based and MLP branches.</p> <p>Parameters:</p> Name Type Description Default <code>column_idx</code> <code>Dict[str, int]</code> <p>Dict containing the index of the columns that will be passed through the model. Required to slice the tensors. e.g. {'education': 0, 'relationship': 1, 'workclass': 2, ...}.</p> required <code>input_dim</code> <code>int</code> <p>Dimension of the input embeddings</p> required <code>num_heads</code> <code>int</code> <p>Number of attention heads</p> <code>4</code> <code>num_layers</code> <code>int</code> <p>Number of interacting layers (attention + residual)</p> <code>2</code> <code>reduction</code> <code>Literal['mean', 'cat']</code> <p>How to reduce the output of the attention layers. Options are: 'mean': mean of attention outputs 'cat': concatenation of attention outputs</p> <code>'mean'</code> <code>structure</code> <code>Literal['stacked', 'parallel']</code> <p>Structure of the model. Either 'parallel' or 'stacked'. If 'parallel', the output will be the concatenation of the attention and deep networks. If 'stacked', the attention output will be fed into the deep network.</p> <code>'parallel'</code> <code>gated</code> <code>bool</code> <p>If True and structure is 'parallel', uses a gating mechanism to combine the attention and deep networks. Note: requires reduction='mean'.</p> <code>True</code> <code>cat_embed_input</code> <code>Optional[List[Tuple[str, int]]]</code> <p>List of Tuples with the column name, number of unique values and embedding dimension. e.g. [(education, 11, 32), ...]</p> <code>None</code> <code>cat_embed_dropout</code> <code>Optional[float]</code> <p>Categorical embeddings dropout. If <code>None</code>, it will default to 0.</p> <code>None</code> <code>use_cat_bias</code> <code>Optional[bool]</code> <p>Boolean indicating if bias will be used for the categorical embeddings</p> <code>None</code> <code>cat_embed_activation</code> <code>Optional[str]</code> <p>Activation function for the categorical embeddings</p> <code>None</code> <code>continuous_cols</code> <code>Optional[List[str]]</code> <p>List with the name of the numeric (aka continuous) columns</p> <code>None</code> <code>cont_norm_layer</code> <code>Optional[Literal['batchnorm', 'layernorm']]</code> <p>Type of normalization layer applied to the continuous features. Options are: 'layernorm' and 'batchnorm'. if <code>None</code>, no normalization layer will be used.</p> <code>None</code> <code>embed_continuous_method</code> <code>Optional[Literal['standard', 'piecewise', 'periodic']]</code> <p>Method to use to embed the continuous features. Options are: 'standard', 'periodic' or 'piecewise'. The 'standard' embedding method is based on the FT-Transformer implementation presented in the paper: Revisiting Deep Learning Models for Tabular Data. The 'periodic' and_'piecewise'_ methods were presented in the paper: On Embeddings for Numerical Features in Tabular Deep Learning. Please, read the papers for details.</p> <code>None</code> <code>cont_embed_dropout</code> <code>Optional[float]</code> <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p> <code>None</code> <code>cont_embed_activation</code> <code>Optional[str]</code> <p>Activation function for the continuous embeddings if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported. If <code>None</code>, no activation function will be applied.</p> <code>None</code> <code>quantization_setup</code> <code>Optional[Dict[str, List[float]]]</code> <p>This parameter is used when the 'piecewise' method is used to embed the continuous cols. It is a dict where keys are the name of the continuous columns and values are lists with the boundaries for the quantization of the continuous_cols. See the examples for details. If If the 'piecewise' method is used, this parameter is required.</p> <code>None</code> <code>n_frequencies</code> <code>Optional[int]</code> <p>This is the so called 'k' in their paper On Embeddings for Numerical Features in Tabular Deep Learning, and is the number of 'frequencies' that will be used to represent each continuous column. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>sigma</code> <code>Optional[float]</code> <p>This is the sigma parameter in the paper mentioned when describing the previous parameters and it is used to initialise the 'frequency weights'. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>share_last_layer</code> <code>Optional[bool]</code> <p>This parameter is not present in the before mentioned paper but it is implemented in the official repo. If <code>True</code> the linear layer that turns the frequencies into embeddings will be shared across the continuous columns. If <code>False</code> a different linear layer will be used for each continuous column. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>full_embed_dropout</code> <code>Optional[bool]</code> <p>If <code>True</code>, the full embedding corresponding to a column will be masked out/dropout. If <code>None</code>, it will default to <code>False</code>.</p> <code>None</code> <code>mlp_hidden_dims</code> <code>List[int]</code> <p>List with the number of neurons per dense layer in the mlp.</p> <code>[100, 100]</code> <code>mlp_activation</code> <code>str</code> <p>Activation function for the dense layers of the MLP. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> <code>'relu'</code> <code>mlp_dropout</code> <code>Union[float, List[float]]</code> <p>float or List of floats with the dropout between the dense layers. e.g: [0.5,0.5]</p> <code>0.1</code> <code>mlp_batchnorm</code> <code>bool</code> <p>Boolean indicating whether or not batch normalization will be applied to the dense layers</p> <code>False</code> <code>mlp_batchnorm_last</code> <code>bool</code> <p>Boolean indicating whether or not batch normalization will be applied to the last of the dense layers</p> <code>False</code> <code>mlp_linear_first</code> <code>bool</code> <p>Boolean indicating the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code></p> <code>True</code> <p>Attributes:</p> Name Type Description <code>attention_layers</code> <code>ModuleList</code> <p>List of multi-head attention layers</p> <code>deep_network</code> <code>Module</code> <p>The deep network component (MLP)</p> <code>gate</code> <code>(Module, optional)</code> <p>The gating network (if gated=True)</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models.rec import AutoIntPlus\n&gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n&gt;&gt;&gt; colnames = [\"a\", \"b\", \"c\", \"d\", \"e\"]\n&gt;&gt;&gt; cat_embed_input = [(u, i, j) for u, i, j in zip(colnames[:4], [4] * 4, [8] * 4)]\n&gt;&gt;&gt; column_idx = {k: v for v, k in enumerate(colnames)}\n&gt;&gt;&gt; model = AutoIntPlus(\n...     column_idx=column_idx,\n...     input_dim=32,\n...     cat_embed_input=cat_embed_input,\n...     continuous_cols=[\"e\"],\n...     embed_continuous_method=\"standard\",\n...     num_heads=4,\n...     num_layers=2,\n...     structure=\"parallel\",\n...     gated=True,\n...     mlp_hidden_dims=[64, 32]\n... )\n&gt;&gt;&gt; out = model(X_tab)\n</code></pre> Source code in <code>pytorch_widedeep/models/rec/autoint_plus.py</code> <pre><code>class AutoIntPlus(BaseTabularModelWithAttention):\n    r\"\"\"Defines an `AutoIntPlus` model that can be used as the `deeptabular` component\n    of a Wide &amp; Deep model or independently by itself.\n\n    This class implements an enhanced version of the [AutoInt](https://arxiv.org/abs/1810.11921)\n    architecture, adding a parallel or stacked deep network and an optional gating mechanism\n    to control the contribution of the attention-based and MLP branches.\n\n    Parameters\n    ----------\n    column_idx: Dict\n        Dict containing the index of the columns that will be passed through\n        the model. Required to slice the tensors. e.g.\n        _{'education': 0, 'relationship': 1, 'workclass': 2, ...}_.\n    input_dim: int\n        Dimension of the input embeddings\n    num_heads: int, default = 4\n        Number of attention heads\n    num_layers: int, default = 2\n        Number of interacting layers (attention + residual)\n    reduction: str, default = \"mean\"\n        How to reduce the output of the attention layers. Options are:\n        _'mean'_: mean of attention outputs\n        _'cat'_: concatenation of attention outputs\n    structure: str, default = \"parallel\"\n        Structure of the model. Either _'parallel'_ or _'stacked'_. If _'parallel'_,\n        the output will be the concatenation of the attention and deep networks.\n        If _'stacked'_, the attention output will be fed into the deep network.\n    gated: bool, default = True\n        If True and structure is 'parallel', uses a gating mechanism to combine\n        the attention and deep networks. Note: requires reduction='mean'.\n    cat_embed_input: List, Optional, default = None\n        List of Tuples with the column name, number of unique values and\n        embedding dimension. e.g. _[(education, 11, 32), ...]_\n    cat_embed_dropout: float, Optional, default = None\n        Categorical embeddings dropout. If `None`, it will default to 0.\n    use_cat_bias: bool, Optional, default = None,\n        Boolean indicating if bias will be used for the categorical embeddings\n    cat_embed_activation: Optional, str, default = None,\n        Activation function for the categorical embeddings\n    continuous_cols : Optional[List[str]], default=None\n        List with the name of the numeric (aka continuous) columns\n    cont_norm_layer : Optional[Literal[\"batchnorm\", \"layernorm\"]], default=None\n        Type of normalization layer applied to the continuous features.\n        Options are: _'layernorm'_ and _'batchnorm'_. if `None`, no\n        normalization layer will be used.\n    embed_continuous_method: Optional[Literal[\"piecewise\", \"periodic\", \"standard\"]], default=\"standard\"\n        Method to use to embed the continuous features. Options are:\n        _'standard'_, _'periodic'_ or _'piecewise'_. The _'standard'_\n        embedding method is based on the FT-Transformer implementation\n        presented in the paper: [Revisiting Deep Learning Models for\n        Tabular Data](https://arxiv.org/abs/2106.11959v5). The _'periodic'_\n        and_'piecewise'_ methods were presented in the paper: [On Embeddings for\n        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556).\n        Please, read the papers for details.\n    cont_embed_dropout : Optional[float], default=None\n        Dropout for the continuous embeddings. If `None`, it will default to 0.0\n    cont_embed_activation : Optional[str], default=None\n        Activation function for the continuous embeddings if any. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported.\n        If `None`, no activation function will be applied.\n    quantization_setup : Optional[Dict[str, List[float]]], default=None\n        This parameter is used when the _'piecewise'_ method is used to embed\n        the continuous cols. It is a dict where keys are the name of the continuous\n        columns and values are lists with the boundaries for the quantization\n        of the continuous_cols. See the examples for details. If\n        If the _'piecewise'_ method is used, this parameter is required.\n    n_frequencies : Optional[int], default=None\n        This is the so called _'k'_ in their paper [On Embeddings for\n        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556),\n        and is the number of 'frequencies' that will be used to represent each\n        continuous column. See their Eq 2 in the paper for details. If\n        the _'periodic'_ method is used, this parameter is required.\n    sigma : Optional[float], default=None\n        This is the sigma parameter in the paper mentioned when describing the\n        previous parameters and it is used to initialise the 'frequency\n        weights'. See their Eq 2 in the paper for details. If\n        the _'periodic'_ method is used, this parameter is required.\n    share_last_layer : Optional[bool], default=None\n        This parameter is not present in the before mentioned paper but it is implemented in\n        the [official repo](https://github.com/yandex-research/rtdl-num-embeddings/tree/main).\n        If `True` the linear layer that turns the frequencies into embeddings\n        will be shared across the continuous columns. If `False` a different\n        linear layer will be used for each continuous column.\n        If the _'periodic'_ method is used, this parameter is required.\n    full_embed_dropout: bool, Optional, default = None,\n        If `True`, the full embedding corresponding to a column will be masked\n        out/dropout. If `None`, it will default to `False`.\n    mlp_hidden_dims: List, default = [200, 100]\n        List with the number of neurons per dense layer in the mlp.\n    mlp_activation: str, default = \"relu\"\n        Activation function for the dense layers of the MLP. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported\n    mlp_dropout: float or List, default = 0.1\n        float or List of floats with the dropout between the dense layers.\n        e.g: _[0.5,0.5]_\n    mlp_batchnorm: bool, default = False\n        Boolean indicating whether or not batch normalization will be applied\n        to the dense layers\n    mlp_batchnorm_last: bool, default = False\n        Boolean indicating whether or not batch normalization will be applied\n        to the last of the dense layers\n    mlp_linear_first: bool, default = False\n        Boolean indicating the order of the operations in the dense\n        layer. If `True: [LIN -&gt; ACT -&gt; BN -&gt; DP]`. If `False: [BN -&gt; DP -&gt;\n        LIN -&gt; ACT]`\n\n    Attributes\n    ----------\n    attention_layers: nn.ModuleList\n        List of multi-head attention layers\n    deep_network: nn.Module\n        The deep network component (MLP)\n    gate: nn.Module, optional\n        The gating network (if gated=True)\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.models.rec import AutoIntPlus\n    &gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n    &gt;&gt;&gt; colnames = [\"a\", \"b\", \"c\", \"d\", \"e\"]\n    &gt;&gt;&gt; cat_embed_input = [(u, i, j) for u, i, j in zip(colnames[:4], [4] * 4, [8] * 4)]\n    &gt;&gt;&gt; column_idx = {k: v for v, k in enumerate(colnames)}\n    &gt;&gt;&gt; model = AutoIntPlus(\n    ...     column_idx=column_idx,\n    ...     input_dim=32,\n    ...     cat_embed_input=cat_embed_input,\n    ...     continuous_cols=[\"e\"],\n    ...     embed_continuous_method=\"standard\",\n    ...     num_heads=4,\n    ...     num_layers=2,\n    ...     structure=\"parallel\",\n    ...     gated=True,\n    ...     mlp_hidden_dims=[64, 32]\n    ... )\n    &gt;&gt;&gt; out = model(X_tab)\n    \"\"\"\n\n    def __init__(\n        self,\n        *,\n        column_idx: Dict[str, int],\n        input_dim: int,\n        num_heads: int = 4,\n        num_layers: int = 2,\n        reduction: Literal[\"mean\", \"cat\"] = \"mean\",\n        structure: Literal[\"stacked\", \"parallel\"] = \"parallel\",\n        gated: bool = True,\n        cat_embed_input: Optional[List[Tuple[str, int]]] = None,\n        cat_embed_dropout: Optional[float] = None,\n        use_cat_bias: Optional[bool] = None,\n        cat_embed_activation: Optional[str] = None,\n        continuous_cols: Optional[List[str]] = None,\n        cont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\n        embed_continuous_method: Optional[\n            Literal[\"standard\", \"piecewise\", \"periodic\"]\n        ] = None,\n        cont_embed_dropout: Optional[float] = None,\n        cont_embed_activation: Optional[str] = None,\n        quantization_setup: Optional[Dict[str, List[float]]] = None,\n        n_frequencies: Optional[int] = None,\n        sigma: Optional[float] = None,\n        share_last_layer: Optional[bool] = None,\n        full_embed_dropout: Optional[bool] = None,\n        mlp_hidden_dims: List[int] = [100, 100],\n        mlp_activation: str = \"relu\",\n        mlp_dropout: Union[float, List[float]] = 0.1,\n        mlp_batchnorm: bool = False,\n        mlp_batchnorm_last: bool = False,\n        mlp_linear_first: bool = True,\n    ):\n        super(AutoIntPlus, self).__init__(\n            column_idx=column_idx,\n            input_dim=input_dim,\n            cat_embed_input=cat_embed_input,\n            cat_embed_dropout=cat_embed_dropout,\n            use_cat_bias=use_cat_bias,\n            cat_embed_activation=cat_embed_activation,\n            shared_embed=None,\n            add_shared_embed=None,\n            frac_shared_embed=None,\n            continuous_cols=continuous_cols,\n            cont_norm_layer=cont_norm_layer,\n            embed_continuous_method=embed_continuous_method,\n            cont_embed_dropout=cont_embed_dropout,\n            cont_embed_activation=cont_embed_activation,\n            quantization_setup=quantization_setup,\n            n_frequencies=n_frequencies,\n            sigma=sigma,\n            share_last_layer=share_last_layer,\n            full_embed_dropout=full_embed_dropout,\n        )\n\n        self.input_dim = input_dim\n        self.num_heads = num_heads\n        self.num_layers = num_layers\n        self.reduction = reduction\n        self.structure = structure\n        self.gated = gated\n\n        self.mlp_hidden_dims = mlp_hidden_dims\n        self.mlp_activation = mlp_activation\n        self.mlp_dropout = mlp_dropout\n        self.mlp_batchnorm = mlp_batchnorm\n        self.mlp_batchnorm_last = mlp_batchnorm_last\n        self.mlp_linear_first = mlp_linear_first\n\n        self.attention_layers = nn.ModuleList(\n            [\n                nn.MultiheadAttention(\n                    embed_dim=self.input_dim,\n                    num_heads=self.num_heads,\n                    batch_first=True,\n                )\n                for _ in range(num_layers)\n            ]\n        )\n\n        if self.gated:\n            self.gate = self._build_gate()\n\n        deep_network_inp_dim = self._get_deep_network_input_dim()\n        self.deep_network = MLP(\n            [deep_network_inp_dim] + self.mlp_hidden_dims,\n            self.mlp_activation,\n            self.mlp_dropout,\n            self.mlp_batchnorm,\n            self.mlp_batchnorm_last,\n            self.mlp_linear_first,\n        )\n\n    def forward(self, X: torch.Tensor) -&gt; torch.Tensor:\n        x = self._get_embeddings(X)\n        attn_output = self._apply_attention_layers(x)\n        reduced_attn_output = self._reduce_attention_output(attn_output)\n        if self.structure == \"parallel\":\n            return self._parallel_output(reduced_attn_output, x)\n        else:  # structure == \"stacked\"\n            return self.deep_network(reduced_attn_output)\n\n    def _get_deep_network_input_dim(self) -&gt; int:\n        if self.structure == \"parallel\":\n            return self.input_dim * len(self.column_idx)\n        elif self.reduction == \"mean\":  # structure == \"stacked\"\n            return self.input_dim\n        else:\n            return self.input_dim * len(self.column_idx)\n\n    def _build_gate(self) -&gt; nn.Linear:\n        self._setup_gating()\n        return nn.Linear(self.input_dim * 2, self.input_dim)\n\n    def _setup_gating(self):\n        if self.structure == \"stacked\":\n            raise ValueError(\n                \"Gating is not supported for stacked structure. Set `gated=False`.\"\n            )\n        if self.reduction != \"mean\":\n            raise ValueError(\n                \"When using a gated structure, the reduction must be 'mean'.\"\n            )\n        if self.mlp_hidden_dims[-1] != self.input_dim:\n            self.mlp_hidden_dims = self.mlp_hidden_dims + [self.input_dim]\n            UserWarning(\n                \"When using a gated structure, the last hidden layer of \"\n                \"the MLP must have the same dimension as the input. \"\n                \"The last hidden layer has been set to the input dimension.\"\n            )\n\n    def _apply_attention_layers(self, x: torch.Tensor) -&gt; torch.Tensor:\n        attn_output = x\n        for layer in self.attention_layers:\n            layer_out, _ = layer(attn_output, attn_output, attn_output)\n            attn_output = layer_out + attn_output\n        return attn_output\n\n    def _reduce_attention_output(self, attn_output: torch.Tensor) -&gt; torch.Tensor:\n        return (\n            attn_output.mean(dim=1)\n            if self.reduction == \"mean\"\n            else attn_output.flatten(1)\n        )\n\n    def _parallel_output(\n        self, attn_output: torch.Tensor, x: torch.Tensor\n    ) -&gt; torch.Tensor:\n        deep_output = self.deep_network(x.reshape(x.size(0), -1))\n\n        if not self.gated:\n            return torch.cat([attn_output, deep_output], dim=1)\n\n        combined_features = torch.cat([attn_output, deep_output], dim=-1)\n        gate_weights = torch.sigmoid(self.gate(combined_features))\n        return gate_weights * attn_output + (1 - gate_weights) * deep_output\n\n    @property\n    def output_dim(self) -&gt; int:\n        if self.structure == \"parallel\":\n            if self.gated:\n                return self.input_dim\n            else:\n                if self.reduction == \"mean\":\n                    return self.mlp_hidden_dims[-1] + self.input_dim\n                else:\n                    return self.mlp_hidden_dims[-1] + self.input_dim * len(\n                        self.column_idx\n                    )\n        else:\n            return self.mlp_hidden_dims[-1]\n</code></pre>"},{"location":"pytorch-widedeep/the_rec_module.html#pytorch_widedeep.models.rec.basic_transformer.Transformer","title":"Transformer","text":"<p>               Bases: <code>Module</code></p> <p>Basic Encoder-Only Transformer Model for sequence classification/regression. As all other models in the library this model can be used as the <code>deeptext</code> component of a Wide &amp; Deep model or independently by itself.</p> <p> NOTE: This model is introduced in the context of recommendation systems and thought for sequences of any nature (e.g. items). It can, of course, still be used for text.</p> <p>Parameters:</p> Name Type Description Default <code>vocab_size</code> <code>int</code> <p>Number of words in the vocabulary</p> required <code>input_dim</code> <code>int</code> <p>Dimension of the token embeddings</p> <p>Param aliases: <code>embed_dim</code>, <code>d_model</code>. </p> required <code>seq_length</code> <code>int</code> <p>Input sequence length</p> required <code>n_heads</code> <code>int</code> <p>Number of attention heads per Transformer block</p> required <code>n_blocks</code> <code>int</code> <p>Number of Transformer blocks</p> required <code>attn_dropout</code> <code>float</code> <p>Dropout that will be applied to the Multi-Head Attention layers</p> <code>0.1</code> <code>ff_dropout</code> <code>float</code> <p>Dropout that will be applied to the FeedForward network</p> <code>0.1</code> <code>ff_factor</code> <code>int</code> <p>Multiplicative factor applied to the first layer of the FF network in each Transformer block, This is normally set to 4.</p> <code>4</code> <code>activation</code> <code>str</code> <p>Transformer Encoder activation function. 'tanh', 'relu', 'leaky_relu', 'gelu', 'geglu' and 'reglu' are supported</p> <code>'gelu'</code> <code>padding_idx</code> <code>int</code> <p>index of the padding token in the padded-tokenised sequences.</p> <code>0</code> <code>with_cls_token</code> <code>bool</code> <p>Boolean indicating if a <code>'[CLS]'</code> token is included in the tokenized sequences. If present, the final hidden state corresponding to this token is used as the aggregated representation for classification and regression tasks. NOTE: if included in the tokenized sequences it must be inserted as the first token in the sequences.</p> <code>False</code> <code>with_pos_encoding</code> <code>bool</code> <p>Boolean indicating if positional encoding will be used</p> <code>True</code> <code>pos_encoding_dropout</code> <code>float</code> <p>Positional encoding dropout</p> <code>0.1</code> <code>pos_encoder</code> <code>Optional[Module]</code> <p>This model uses by default a standard positional encoding approach. However, any custom positional encoder can also be used and pass to the Transformer model via the 'pos_encoder' parameter</p> <code>None</code> <p>Attributes:</p> Name Type Description <code>embedding</code> <code>Module</code> <p>Standard token embedding layer</p> <code>pos_encoder</code> <code>Module</code> <p>Positional Encoder</p> <code>encoder</code> <code>Module</code> <p>Sequence of Transformer blocks</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models.rec import Transformer\n&gt;&gt;&gt; X = torch.cat((torch.zeros([5,1]), torch.empty(5, 4).random_(1,4)), axis=1)\n&gt;&gt;&gt; model = Transformer(vocab_size=4, seq_length=5, input_dim=8, n_heads=1, n_blocks=1)\n&gt;&gt;&gt; out = model(X)\n</code></pre> Source code in <code>pytorch_widedeep/models/rec/basic_transformer.py</code> <pre><code>class Transformer(nn.Module):\n    r\"\"\"\n    Basic Encoder-Only Transformer Model for sequence\n    classification/regression. As all other models in the library this model\n    can be used as the `deeptext` component of a Wide &amp; Deep model or\n    independently by itself.\n\n    :information_source: **NOTE**:\n    This model is introduced in the context of recommendation systems and\n    thought for sequences of any nature (e.g. items). It can, of course,\n    still be used for text.\n\n    Parameters\n    ----------\n    vocab_size: int\n        Number of words in the vocabulary\n    input_dim: int\n        Dimension of the token embeddings\n\n        Param aliases: `embed_dim`, `d_model`. &lt;br/&gt;\n    seq_length: int, Optional, default = None\n        Input sequence length\n    n_heads: int, default = 8\n        Number of attention heads per Transformer block\n    n_blocks: int, default = 4\n        Number of Transformer blocks\n    attn_dropout: float, default = 0.2\n        Dropout that will be applied to the Multi-Head Attention layers\n    ff_dropout: float, default = 0.1\n        Dropout that will be applied to the FeedForward network\n    ff_factor: float, default = 4\n        Multiplicative factor applied to the first layer of the FF network in\n        each Transformer block, This is normally set to 4.\n    activation: str, default = \"gelu\"\n        Transformer Encoder activation function. _'tanh'_, _'relu'_,\n        _'leaky_relu'_, _'gelu'_, _'geglu'_ and _'reglu'_ are supported\n    padding_idx: int, default = 0\n        index of the padding token in the padded-tokenised sequences.\n    with_cls_token: bool, default = False\n        Boolean indicating if a `'[CLS]'` token is included in the tokenized\n        sequences. If present, the final hidden state corresponding to this\n        token is used as the aggregated representation for classification and\n        regression tasks. **NOTE**: if included in the tokenized sequences it\n        must be inserted as the first token in the sequences.\n    with_pos_encoding: bool, default = True\n        Boolean indicating if positional encoding will be used\n    pos_encoding_dropout: float, default = 0.1\n        Positional encoding dropout\n    pos_encoder: nn.Module, Optional, default = None\n        This model uses by default a standard positional encoding approach.\n        However, any custom positional encoder can also be used and pass to\n        the Transformer model via the 'pos_encoder' parameter\n\n    Attributes\n    ----------\n    embedding: nn.Module\n        Standard token embedding layer\n    pos_encoder: nn.Module\n        Positional Encoder\n    encoder: nn.Module\n        Sequence of Transformer blocks\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.models.rec import Transformer\n    &gt;&gt;&gt; X = torch.cat((torch.zeros([5,1]), torch.empty(5, 4).random_(1,4)), axis=1)\n    &gt;&gt;&gt; model = Transformer(vocab_size=4, seq_length=5, input_dim=8, n_heads=1, n_blocks=1)\n    &gt;&gt;&gt; out = model(X)\n    \"\"\"\n\n    @alias(\"input_dim\", [\"embed_dim\", \"d_model\"])\n    @alias(\"seq_length\", [\"max_length\", \"maxlen\"])\n    def __init__(\n        self,\n        *,\n        vocab_size: int,\n        seq_length: int,\n        input_dim: int,\n        n_heads: int,\n        n_blocks: int,\n        attn_dropout: float = 0.1,\n        ff_dropout: float = 0.1,\n        ff_factor: int = 4,\n        activation: str = \"gelu\",\n        use_linear_attention: bool = False,\n        use_flash_attention: bool = False,\n        padding_idx: int = 0,\n        with_cls_token: bool = False,\n        with_pos_encoding: bool = True,\n        pos_encoding_dropout: float = 0.1,\n        pos_encoder: Optional[nn.Module] = None,\n    ):\n        super().__init__()\n\n        self.input_dim = input_dim\n        self.seq_length = seq_length\n        self.n_heads = n_heads\n        self.n_blocks = n_blocks\n        self.attn_dropout = attn_dropout\n        self.ff_dropout = ff_dropout\n        self.ff_factor = ff_factor\n        self.activation = activation\n        self.use_linear_attention = use_linear_attention\n        self.use_flash_attention = use_flash_attention\n        self.padding_idx = padding_idx\n        self.with_cls_token = with_cls_token\n        self.with_pos_encoding = with_pos_encoding\n        self.pos_encoding_dropout = pos_encoding_dropout\n\n        self.embedding = nn.Embedding(\n            vocab_size, input_dim, padding_idx=self.padding_idx\n        )\n\n        if with_pos_encoding:\n            if pos_encoder is not None:\n                self.pos_encoder: Union[nn.Module, nn.Identity, PositionalEncoding] = (\n                    pos_encoder\n                )\n            else:\n                self.pos_encoder = PositionalEncoding(\n                    input_dim, pos_encoding_dropout, seq_length\n                )\n        else:\n            self.pos_encoder = nn.Identity()\n\n        self.encoder = nn.Sequential()\n        for i in range(n_blocks):\n            self.encoder.add_module(\n                \"transformer_block\" + str(i),\n                TransformerEncoder(\n                    input_dim,\n                    n_heads,\n                    False,  # use_qkv_bias\n                    attn_dropout,\n                    ff_dropout,\n                    ff_factor,\n                    activation,\n                    use_linear_attention,\n                    use_flash_attention,\n                ),\n            )\n\n    def forward(self, X: Tensor) -&gt; Tensor:\n        x = self.embedding(X.long())\n        x = self.pos_encoder(x)\n        x = self.encoder(x)\n        if self.with_cls_token:\n            x = x[:, 0, :]\n        else:\n            x = x.flatten(1)\n        return x\n\n    @property\n    def output_dim(self) -&gt; int:\n        if self.with_cls_token:\n            output_dim = self.input_dim\n        else:\n            output_dim = self.input_dim * self.seq_length\n        return output_dim\n</code></pre>"},{"location":"pytorch-widedeep/the_rec_module.html#pytorch_widedeep.models.rec.dcn.DeepCrossNetwork","title":"DeepCrossNetwork","text":"<p>               Bases: <code>BaseTabularModelWithoutAttention</code></p> <p>Defines a <code>DeepCrossNetwork</code> model that can be used as the <code>deeptabular</code> component of a Wide &amp; Deep model or independently by itself.</p> <p>This class implements the Deep &amp; Cross Network for Ad Click Predictions architecture, which automatically combines features to generate feature interactions in an explicit fashion and at each layer.</p> <p>The cross layer implements the following equation:</p> \\[x_{l+1} = x_0 \\odot (W_l x_l + b_l) + x_l\\] <p>where:</p> <ul> <li>\\(\\odot\\) represents element-wise multiplication</li> <li>\\(x_l\\), \\(x_{l+1}\\) are the outputs from the \\(l^{th}\\) and \\((l+1)^{th}\\) cross layers</li> <li>\\(W_l\\), \\(b_l\\) are the weight and bias parameters</li> </ul> <p>Parameters:</p> Name Type Description Default <code>column_idx</code> <code>Dict[str, int]</code> <p>Dict containing the index of the columns that will be passed through the <code>DeepCrossNetwork</code> model. Required to slice the tensors. e.g. {'education': 0, 'relationship': 1, 'workclass': 2, ...}.</p> required <code>n_cross_layers</code> <code>int</code> <p>Number of cross layers in the cross network</p> <code>3</code> <code>cat_embed_input</code> <code>Optional[List[Tuple[str, int, int]]]</code> <p>List of Tuples with the column name, number of unique values and embedding dimension. e.g. [(education, 11, 32), ...]</p> <code>None</code> <code>cat_embed_dropout</code> <code>Optional[float]</code> <p>Categorical embeddings dropout. If <code>None</code>, it will default to 0.</p> <code>None</code> <code>use_cat_bias</code> <code>Optional[bool]</code> <p>Boolean indicating if bias will be used for the categorical embeddings</p> <code>None</code> <code>cat_embed_activation</code> <code>Optional[str]</code> <p>Activation function for the categorical embeddings</p> <code>None</code> <code>continuous_cols</code> <code>Optional[List[str]]</code> <p>List with the name of the numeric (aka continuous) columns</p> <code>None</code> <code>cont_norm_layer</code> <code>Optional[Literal['batchnorm', 'layernorm']]</code> <p>Type of normalization layer applied to the continuous features. Options are: 'layernorm' and 'batchnorm'. if <code>None</code>, no normalization layer will be used.</p> <code>None</code> <code>embed_continuous_method</code> <code>Optional[Literal['standard', 'piecewise', 'periodic']]</code> <p>Method to use to embed the continuous features. Options are: 'standard', 'periodic' or 'piecewise'. The 'standard' embedding method is based on the FT-Transformer implementation presented in the paper: Revisiting Deep Learning Models for Tabular Data. The 'periodic' and_'piecewise'_ methods were presented in the paper: On Embeddings for Numerical Features in Tabular Deep Learning. Please, read the papers for details.</p> <code>None</code> <code>cont_embed_dropout</code> <code>Optional[float]</code> <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p> <code>None</code> <code>cont_embed_activation</code> <code>Optional[str]</code> <p>Activation function for the continuous embeddings if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported. If <code>None</code>, no activation function will be applied.</p> <code>None</code> <code>quantization_setup</code> <code>Optional[Dict[str, List[float]]]</code> <p>This parameter is used when the 'piecewise' method is used to embed the continuous cols. It is a dict where keys are the name of the continuous columns and values are lists with the boundaries for the quantization of the continuous_cols. See the examples for details. If If the 'piecewise' method is used, this parameter is required.</p> <code>None</code> <code>n_frequencies</code> <code>Optional[int]</code> <p>This is the so called 'k' in their paper On Embeddings for Numerical Features in Tabular Deep Learning, and is the number of 'frequencies' that will be used to represent each continuous column. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>sigma</code> <code>Optional[float]</code> <p>This is the sigma parameter in the paper mentioned when describing the previous parameters and it is used to initialise the 'frequency weights'. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>share_last_layer</code> <code>Optional[bool]</code> <p>This parameter is not present in the before mentioned paper but it is implemented in the official repo. If <code>True</code> the linear layer that turns the frequencies into embeddings will be shared across the continuous columns. If <code>False</code> a different linear layer will be used for each continuous column. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>full_embed_dropout</code> <code>Optional[bool]</code> <p>If <code>True</code>, the full embedding corresponding to a column will be masked out/dropout. If <code>None</code>, it will default to <code>False</code>.</p> <code>None</code> <code>mlp_hidden_dims</code> <code>List[int]</code> <p>List with the number of neurons per dense layer in the mlp.</p> <code>[200, 100]</code> <code>mlp_activation</code> <code>str</code> <p>Activation function for the dense layers of the MLP. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> <code>'relu'</code> <code>mlp_dropout</code> <code>Union[float, List[float]]</code> <p>float or List of floats with the dropout between the dense layers. e.g: [0.5,0.5]</p> <code>0.1</code> <code>mlp_batchnorm</code> <code>bool</code> <p>Boolean indicating whether or not batch normalization will be applied to the dense layers</p> <code>False</code> <code>mlp_batchnorm_last</code> <code>bool</code> <p>Boolean indicating whether or not batch normalization will be applied to the last of the dense layers</p> <code>False</code> <code>mlp_linear_first</code> <code>bool</code> <p>Boolean indicating the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code></p> <code>True</code> <p>Attributes:</p> Name Type Description <code>cross_network</code> <code>Module</code> <p>The cross network component</p> <code>deep_network</code> <code>Module</code> <p>The deep network component (MLP)</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models.rec import DeepCrossNetwork\n&gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n&gt;&gt;&gt; colnames = [\"a\", \"b\", \"c\", \"d\", \"e\"]\n&gt;&gt;&gt; cat_embed_input = [(u, i, j) for u, i, j in zip(colnames[:4], [4] * 4, [8] * 4)]\n&gt;&gt;&gt; column_idx = {k: v for v, k in enumerate(colnames)}\n&gt;&gt;&gt; model = DeepCrossNetwork(\n...     column_idx=column_idx,\n...     cat_embed_input=cat_embed_input,\n...     continuous_cols=[\"e\"],\n...     n_cross_layers=2,\n...     mlp_hidden_dims=[16, 8]\n... )\n&gt;&gt;&gt; out = model(X_tab)\n</code></pre> Source code in <code>pytorch_widedeep/models/rec/dcn.py</code> <pre><code>class DeepCrossNetwork(BaseTabularModelWithoutAttention):\n    r\"\"\"Defines a `DeepCrossNetwork` model that can be used as the `deeptabular`\n    component of a Wide &amp; Deep model or independently by itself.\n\n    This class implements the [Deep &amp; Cross Network for Ad Click Predictions](https://arxiv.org/abs/1708.05123)\n    architecture, which automatically combines features to generate feature interactions\n    in an explicit fashion and at each layer.\n\n    The cross layer implements the following equation:\n\n    $$x_{l+1} = x_0 \\odot (W_l x_l + b_l) + x_l$$\n\n    where:\n\n    * $\\odot$ represents element-wise multiplication\n    * $x_l$, $x_{l+1}$ are the outputs from the $l^{th}$ and $(l+1)^{th}$ cross layers\n    * $W_l$, $b_l$ are the weight and bias parameters\n\n    Parameters\n    ----------\n    column_idx: Dict\n        Dict containing the index of the columns that will be passed through\n        the `DeepCrossNetwork` model. Required to slice the tensors. e.g.\n        _{'education': 0, 'relationship': 1, 'workclass': 2, ...}_.\n    n_cross_layers: int, default = 3\n        Number of cross layers in the cross network\n    cat_embed_input: List, Optional, default = None\n        List of Tuples with the column name, number of unique values and\n        embedding dimension. e.g. _[(education, 11, 32), ...]_\n    cat_embed_dropout: float, Optional, default = None\n        Categorical embeddings dropout. If `None`, it will default to 0.\n    use_cat_bias: bool, Optional, default = None,\n        Boolean indicating if bias will be used for the categorical embeddings\n    cat_embed_activation: Optional, str, default = None,\n        Activation function for the categorical embeddings\n    continuous_cols : Optional[List[str]], default=None\n        List with the name of the numeric (aka continuous) columns\n    cont_norm_layer : Optional[Literal[\"batchnorm\", \"layernorm\"]], default=None\n        Type of normalization layer applied to the continuous features.\n        Options are: _'layernorm'_ and _'batchnorm'_. if `None`, no\n        normalization layer will be used.\n    embed_continuous_method: Optional[Literal[\"piecewise\", \"periodic\", \"standard\"]], default=\"standard\"\n        Method to use to embed the continuous features. Options are:\n        _'standard'_, _'periodic'_ or _'piecewise'_. The _'standard'_\n        embedding method is based on the FT-Transformer implementation\n        presented in the paper: [Revisiting Deep Learning Models for\n        Tabular Data](https://arxiv.org/abs/2106.11959v5). The _'periodic'_\n        and_'piecewise'_ methods were presented in the paper: [On Embeddings for\n        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556).\n        Please, read the papers for details.\n    cont_embed_dropout : Optional[float], default=None\n        Dropout for the continuous embeddings. If `None`, it will default to 0.0\n    cont_embed_activation : Optional[str], default=None\n        Activation function for the continuous embeddings if any. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported.\n        If `None`, no activation function will be applied.\n    quantization_setup : Optional[Dict[str, List[float]]], default=None\n        This parameter is used when the _'piecewise'_ method is used to embed\n        the continuous cols. It is a dict where keys are the name of the continuous\n        columns and values are lists with the boundaries for the quantization\n        of the continuous_cols. See the examples for details. If\n        If the _'piecewise'_ method is used, this parameter is required.\n    n_frequencies : Optional[int], default=None\n        This is the so called _'k'_ in their paper [On Embeddings for\n        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556),\n        and is the number of 'frequencies' that will be used to represent each\n        continuous column. See their Eq 2 in the paper for details. If\n        the _'periodic'_ method is used, this parameter is required.\n    sigma : Optional[float], default=None\n        This is the sigma parameter in the paper mentioned when describing the\n        previous parameters and it is used to initialise the 'frequency\n        weights'. See their Eq 2 in the paper for details. If\n        the _'periodic'_ method is used, this parameter is required.\n    share_last_layer : Optional[bool], default=None\n        This parameter is not present in the before mentioned paper but it is implemented in\n        the [official repo](https://github.com/yandex-research/rtdl-num-embeddings/tree/main).\n        If `True` the linear layer that turns the frequencies into embeddings\n        will be shared across the continuous columns. If `False` a different\n        linear layer will be used for each continuous column.\n        If the _'periodic'_ method is used, this parameter is required.\n    full_embed_dropout: bool, Optional, default = None,\n        If `True`, the full embedding corresponding to a column will be masked\n        out/dropout. If `None`, it will default to `False`.\n    mlp_hidden_dims: List, default = [200, 100]\n        List with the number of neurons per dense layer in the mlp.\n    mlp_activation: str, default = \"relu\"\n        Activation function for the dense layers of the MLP. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported\n    mlp_dropout: float or List, default = 0.1\n        float or List of floats with the dropout between the dense layers.\n        e.g: _[0.5,0.5]_\n    mlp_batchnorm: bool, default = False\n        Boolean indicating whether or not batch normalization will be applied\n        to the dense layers\n    mlp_batchnorm_last: bool, default = False\n        Boolean indicating whether or not batch normalization will be applied\n        to the last of the dense layers\n    mlp_linear_first: bool, default = False\n        Boolean indicating the order of the operations in the dense\n        layer. If `True: [LIN -&gt; ACT -&gt; BN -&gt; DP]`. If `False: [BN -&gt; DP -&gt;\n        LIN -&gt; ACT]`\n\n    Attributes\n    ----------\n    cross_network: nn.Module\n        The cross network component\n    deep_network: nn.Module\n        The deep network component (MLP)\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.models.rec import DeepCrossNetwork\n    &gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n    &gt;&gt;&gt; colnames = [\"a\", \"b\", \"c\", \"d\", \"e\"]\n    &gt;&gt;&gt; cat_embed_input = [(u, i, j) for u, i, j in zip(colnames[:4], [4] * 4, [8] * 4)]\n    &gt;&gt;&gt; column_idx = {k: v for v, k in enumerate(colnames)}\n    &gt;&gt;&gt; model = DeepCrossNetwork(\n    ...     column_idx=column_idx,\n    ...     cat_embed_input=cat_embed_input,\n    ...     continuous_cols=[\"e\"],\n    ...     n_cross_layers=2,\n    ...     mlp_hidden_dims=[16, 8]\n    ... )\n    &gt;&gt;&gt; out = model(X_tab)\n    \"\"\"\n\n    def __init__(\n        self,\n        *,\n        column_idx: Dict[str, int],\n        n_cross_layers: int = 3,\n        cat_embed_input: Optional[List[Tuple[str, int, int]]] = None,\n        cat_embed_dropout: Optional[float] = None,\n        use_cat_bias: Optional[bool] = None,\n        cat_embed_activation: Optional[str] = None,\n        continuous_cols: Optional[List[str]] = None,\n        cont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\n        embed_continuous: Optional[bool] = None,\n        embed_continuous_method: Optional[\n            Literal[\"standard\", \"piecewise\", \"periodic\"]\n        ] = None,\n        cont_embed_dim: Optional[int] = None,\n        cont_embed_dropout: Optional[float] = None,\n        cont_embed_activation: Optional[str] = None,\n        quantization_setup: Optional[Dict[str, List[float]]] = None,\n        n_frequencies: Optional[int] = None,\n        sigma: Optional[float] = None,\n        share_last_layer: Optional[bool] = None,\n        full_embed_dropout: Optional[bool] = None,\n        mlp_hidden_dims: List[int] = [200, 100],\n        mlp_activation: str = \"relu\",\n        mlp_dropout: Union[float, List[float]] = 0.1,\n        mlp_batchnorm: bool = False,\n        mlp_batchnorm_last: bool = False,\n        mlp_linear_first: bool = True,\n    ):\n        super(DeepCrossNetwork, self).__init__(\n            column_idx=column_idx,\n            cat_embed_input=cat_embed_input,\n            cat_embed_dropout=cat_embed_dropout,\n            use_cat_bias=use_cat_bias,\n            cat_embed_activation=cat_embed_activation,\n            continuous_cols=continuous_cols,\n            cont_norm_layer=cont_norm_layer,\n            embed_continuous=embed_continuous,\n            embed_continuous_method=embed_continuous_method,\n            cont_embed_dim=cont_embed_dim,\n            cont_embed_dropout=cont_embed_dropout,\n            cont_embed_activation=cont_embed_activation,\n            quantization_setup=quantization_setup,\n            n_frequencies=n_frequencies,\n            sigma=sigma,\n            share_last_layer=share_last_layer,\n            full_embed_dropout=full_embed_dropout,\n        )\n\n        self.mlp_hidden_dims = mlp_hidden_dims\n        self.mlp_activation = mlp_activation\n        self.mlp_dropout = mlp_dropout\n        self.mlp_batchnorm = mlp_batchnorm\n        self.mlp_batchnorm_last = mlp_batchnorm_last\n        self.mlp_linear_first = mlp_linear_first\n\n        embeddings_output_dim = self.cat_out_dim + self.cont_out_dim\n        self.deep_network = MLP(\n            [embeddings_output_dim] + self.mlp_hidden_dims,\n            self.mlp_activation,\n            self.mlp_dropout,\n            self.mlp_batchnorm,\n            self.mlp_batchnorm_last,\n            self.mlp_linear_first,\n        )\n        self.cross_network = CrossNetwork(embeddings_output_dim, n_cross_layers)\n\n    def forward(self, X: torch.Tensor) -&gt; torch.Tensor:\n        x = self._get_embeddings(X)\n        cross_output = self.cross_network(x)\n        deep_output = self.deep_network(x)\n        return torch.cat([cross_output, deep_output], dim=1)\n\n    @property\n    def output_dim(self) -&gt; int:\n        return self.mlp_hidden_dims[-1] + self.cat_out_dim + self.cont_out_dim\n</code></pre>"},{"location":"pytorch-widedeep/the_rec_module.html#pytorch_widedeep.models.rec.dcnv2.DeepCrossNetworkV2","title":"DeepCrossNetworkV2","text":"<p>               Bases: <code>BaseTabularModelWithoutAttention</code></p> <p>Defines a <code>DeepCrossNetworkV2</code> model that can be used as the <code>deeptabular</code> component of a Wide &amp; Deep model or independently by itself.</p> <p>This class implements the DCN V2: Improved Deep &amp; Cross Network and Practical Lessons for Web-scale Learning to Rank Systems architecture, which enhances the original DCN by introducing a more expressive cross network that uses multiple experts and matrix decomposition techniques to improve model capacity while maintaining computational efficiency.</p> <p>The cross layer implements the following equation:</p> \\[E_i(x_l) = x_0 \\odot (U_l^i \\cdot g(C_l^i \\cdot g(V_l^iT x_l)) + b_l)\\] <p>where:</p> <ul> <li>\\(\\odot\\) represents element-wise multiplication</li> <li>\\(U_l^i\\), \\(C_l^i\\), \\(V_l^i\\) are the decomposed weight matrices for expert \\(i\\) at layer \\(l\\)</li> <li>\\(g\\) is the activation function (ReLU)</li> <li>\\(b_l\\) is the bias term</li> </ul> <p>Parameters:</p> Name Type Description Default <code>column_idx</code> <code>Dict[str, int]</code> <p>Dict containing the index of the columns that will be passed through the <code>DeepCrossNetworkV2</code> model. Required to slice the tensors. e.g. {'education': 0, 'relationship': 1, 'workclass': 2, ...}.</p> required <code>num_cross_layers</code> <code>int</code> <p>Number of cross layers in the cross network</p> <code>2</code> <code>low_rank</code> <code>Optional[int]</code> <p>Rank of the weight matrix decomposition. If None, full-rank weights are used</p> <code>None</code> <code>num_experts</code> <code>int</code> <p>Number of expert networks in mixture of experts</p> <code>2</code> <code>expert_dropout</code> <code>float</code> <p>Dropout rate for expert outputs</p> <code>0.0</code> <code>structure</code> <code>Literal['stacked', 'parallel']</code> <p>Structure of the model. Either 'parallel' or 'stacked'. If 'parallel', the output will be the concatenation of the cross network and deep network outputs. If 'stacked', the cross network output will be fed into the deep network.</p> <code>'parallel'</code> <code>cat_embed_input</code> <code>Optional[List[Tuple[str, int, int]]]</code> <p>List of Tuples with the column name, number of unique values and embedding dimension. e.g. [(education, 11, 32), ...]</p> <code>None</code> <code>cat_embed_dropout</code> <code>Optional[float]</code> <p>Categorical embeddings dropout. If <code>None</code>, it will default to 0.</p> <code>None</code> <code>use_cat_bias</code> <code>Optional[bool]</code> <p>Boolean indicating if bias will be used for the categorical embeddings</p> <code>None</code> <code>cat_embed_activation</code> <code>Optional[str]</code> <p>Activation function for the categorical embeddings</p> <code>None</code> <code>continuous_cols</code> <code>Optional[List[str]]</code> <p>List with the name of the numeric (aka continuous) columns</p> <code>None</code> <code>cont_norm_layer</code> <code>Optional[Literal['batchnorm', 'layernorm']]</code> <p>Type of normalization layer applied to the continuous features. Options are: 'layernorm' and 'batchnorm'. if <code>None</code>, no normalization layer will be used.</p> <code>None</code> <code>embed_continuous_method</code> <code>Optional[Literal['standard', 'piecewise', 'periodic']]</code> <p>Method to use to embed the continuous features. Options are: 'standard', 'periodic' or 'piecewise'. The 'standard' embedding method is based on the FT-Transformer implementation presented in the paper: Revisiting Deep Learning Models for Tabular Data. The 'periodic' and_'piecewise'_ methods were presented in the paper: On Embeddings for Numerical Features in Tabular Deep Learning. Please, read the papers for details.</p> <code>None</code> <code>cont_embed_dropout</code> <code>Optional[float]</code> <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p> <code>None</code> <code>cont_embed_activation</code> <code>Optional[str]</code> <p>Activation function for the continuous embeddings if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported. If <code>None</code>, no activation function will be applied.</p> <code>None</code> <code>quantization_setup</code> <code>Optional[Dict[str, List[float]]]</code> <p>This parameter is used when the 'piecewise' method is used to embed the continuous cols. It is a dict where keys are the name of the continuous columns and values are lists with the boundaries for the quantization of the continuous_cols. See the examples for details. If If the 'piecewise' method is used, this parameter is required.</p> <code>None</code> <code>n_frequencies</code> <code>Optional[int]</code> <p>This is the so called 'k' in their paper On Embeddings for Numerical Features in Tabular Deep Learning, and is the number of 'frequencies' that will be used to represent each continuous column. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>sigma</code> <code>Optional[float]</code> <p>This is the sigma parameter in the paper mentioned when describing the previous parameters and it is used to initialise the 'frequency weights'. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>share_last_layer</code> <code>Optional[bool]</code> <p>This parameter is not present in the before mentioned paper but it is implemented in the official repo. If <code>True</code> the linear layer that turns the frequencies into embeddings will be shared across the continuous columns. If <code>False</code> a different linear layer will be used for each continuous column. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>full_embed_dropout</code> <code>Optional[bool]</code> <p>If <code>True</code>, the full embedding corresponding to a column will be masked out/dropout. If <code>None</code>, it will default to <code>False</code>.</p> <code>None</code> <code>mlp_hidden_dims</code> <code>List[int]</code> <p>List with the number of neurons per dense layer in the mlp.</p> <code>[200, 100]</code> <code>mlp_activation</code> <code>str</code> <p>Activation function for the dense layers of the MLP. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> <code>'relu'</code> <code>mlp_dropout</code> <code>Union[float, List[float]]</code> <p>float or List of floats with the dropout between the dense layers. e.g: [0.5,0.5]</p> <code>0.1</code> <code>mlp_batchnorm</code> <code>bool</code> <p>Boolean indicating whether or not batch normalization will be applied to the dense layers</p> <code>False</code> <code>mlp_batchnorm_last</code> <code>bool</code> <p>Boolean indicating whether or not batch normalization will be applied to the last of the dense layers</p> <code>False</code> <code>mlp_linear_first</code> <code>bool</code> <p>Boolean indicating the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code></p> <code>True</code> <p>Attributes:</p> Name Type Description <code>cross_network</code> <code>Module</code> <p>The cross network component with mixture of experts</p> <code>deep_network</code> <code>Module</code> <p>The deep network component (MLP)</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models.rec import DeepCrossNetworkV2\n&gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n&gt;&gt;&gt; colnames = [\"a\", \"b\", \"c\", \"d\", \"e\"]\n&gt;&gt;&gt; cat_embed_input = [(u, i, j) for u, i, j in zip(colnames[:4], [4] * 4, [8] * 4)]\n&gt;&gt;&gt; column_idx = {k: v for v, k in enumerate(colnames)}\n&gt;&gt;&gt; model = DeepCrossNetworkV2(\n...     column_idx=column_idx,\n...     cat_embed_input=cat_embed_input,\n...     continuous_cols=[\"e\"],\n...     num_cross_layers=2,\n...     low_rank=32,\n...     num_experts=4,\n...     mlp_hidden_dims=[16, 8]\n... )\n&gt;&gt;&gt; out = model(X_tab)\n</code></pre> Source code in <code>pytorch_widedeep/models/rec/dcnv2.py</code> <pre><code>class DeepCrossNetworkV2(BaseTabularModelWithoutAttention):\n    r\"\"\"Defines a `DeepCrossNetworkV2` model that can be used as the `deeptabular`\n    component of a Wide &amp; Deep model or independently by itself.\n\n    This class implements the\n    [DCN V2: Improved Deep &amp; Cross Network and Practical Lessons for Web-scale Learning to Rank Systems](https://arxiv.org/pdf/2008.13535)\n    architecture, which enhances the original DCN by introducing a more expressive cross\n    network that uses multiple experts and matrix decomposition techniques to improve\n    model capacity while maintaining computational efficiency.\n\n    The cross layer implements the following equation:\n\n    $$E_i(x_l) = x_0 \\odot (U_l^i \\cdot g(C_l^i \\cdot g(V_l^iT x_l)) + b_l)$$\n\n    where:\n\n    * $\\odot$ represents element-wise multiplication\n    * $U_l^i$, $C_l^i$, $V_l^i$ are the decomposed weight matrices for expert $i$ at layer $l$\n    * $g$ is the activation function (ReLU)\n    * $b_l$ is the bias term\n\n    Parameters\n    ----------\n    column_idx: Dict\n        Dict containing the index of the columns that will be passed through\n        the `DeepCrossNetworkV2` model. Required to slice the tensors. e.g.\n        _{'education': 0, 'relationship': 1, 'workclass': 2, ...}_.\n    num_cross_layers: int, default = 2\n        Number of cross layers in the cross network\n    low_rank: int, Optional, default = None\n        Rank of the weight matrix decomposition. If None, full-rank weights are used\n    num_experts: int, default = 2\n        Number of expert networks in mixture of experts\n    expert_dropout: float, default = 0.0\n        Dropout rate for expert outputs\n    structure: str, default = \"parallel\"\n        Structure of the model. Either _'parallel'_ or _'stacked'_. If _'parallel'_,\n        the output will be the concatenation of the cross network and deep network\n        outputs. If _'stacked'_, the cross network output will be fed into the deep\n        network.\n    cat_embed_input: List, Optional, default = None\n        List of Tuples with the column name, number of unique values and\n        embedding dimension. e.g. _[(education, 11, 32), ...]_\n    cat_embed_dropout: float, Optional, default = None\n        Categorical embeddings dropout. If `None`, it will default to 0.\n    use_cat_bias: bool, Optional, default = None,\n        Boolean indicating if bias will be used for the categorical embeddings\n    cat_embed_activation: Optional, str, default = None,\n        Activation function for the categorical embeddings\n    continuous_cols : Optional[List[str]], default=None\n        List with the name of the numeric (aka continuous) columns\n    cont_norm_layer : Optional[Literal[\"batchnorm\", \"layernorm\"]], default=None\n        Type of normalization layer applied to the continuous features.\n        Options are: _'layernorm'_ and _'batchnorm'_. if `None`, no\n        normalization layer will be used.\n    embed_continuous_method: Optional[Literal[\"piecewise\", \"periodic\", \"standard\"]], default=\"standard\"\n        Method to use to embed the continuous features. Options are:\n        _'standard'_, _'periodic'_ or _'piecewise'_. The _'standard'_\n        embedding method is based on the FT-Transformer implementation\n        presented in the paper: [Revisiting Deep Learning Models for\n        Tabular Data](https://arxiv.org/abs/2106.11959v5). The _'periodic'_\n        and_'piecewise'_ methods were presented in the paper: [On Embeddings for\n        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556).\n        Please, read the papers for details.\n    cont_embed_dropout : Optional[float], default=None\n        Dropout for the continuous embeddings. If `None`, it will default to 0.0\n    cont_embed_activation : Optional[str], default=None\n        Activation function for the continuous embeddings if any. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported.\n        If `None`, no activation function will be applied.\n    quantization_setup : Optional[Dict[str, List[float]]], default=None\n        This parameter is used when the _'piecewise'_ method is used to embed\n        the continuous cols. It is a dict where keys are the name of the continuous\n        columns and values are lists with the boundaries for the quantization\n        of the continuous_cols. See the examples for details. If\n        If the _'piecewise'_ method is used, this parameter is required.\n    n_frequencies : Optional[int], default=None\n        This is the so called _'k'_ in their paper [On Embeddings for\n        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556),\n        and is the number of 'frequencies' that will be used to represent each\n        continuous column. See their Eq 2 in the paper for details. If\n        the _'periodic'_ method is used, this parameter is required.\n    sigma : Optional[float], default=None\n        This is the sigma parameter in the paper mentioned when describing the\n        previous parameters and it is used to initialise the 'frequency\n        weights'. See their Eq 2 in the paper for details. If\n        the _'periodic'_ method is used, this parameter is required.\n    share_last_layer : Optional[bool], default=None\n        This parameter is not present in the before mentioned paper but it is implemented in\n        the [official repo](https://github.com/yandex-research/rtdl-num-embeddings/tree/main).\n        If `True` the linear layer that turns the frequencies into embeddings\n        will be shared across the continuous columns. If `False` a different\n        linear layer will be used for each continuous column.\n        If the _'periodic'_ method is used, this parameter is required.\n    full_embed_dropout: bool, Optional, default = None,\n        If `True`, the full embedding corresponding to a column will be masked\n        out/dropout. If `None`, it will default to `False`.\n    mlp_hidden_dims: List, default = [200, 100]\n        List with the number of neurons per dense layer in the mlp.\n    mlp_activation: str, default = \"relu\"\n        Activation function for the dense layers of the MLP. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported\n    mlp_dropout: float or List, default = 0.1\n        float or List of floats with the dropout between the dense layers.\n        e.g: _[0.5,0.5]_\n    mlp_batchnorm: bool, default = False\n        Boolean indicating whether or not batch normalization will be applied\n        to the dense layers\n    mlp_batchnorm_last: bool, default = False\n        Boolean indicating whether or not batch normalization will be applied\n        to the last of the dense layers\n    mlp_linear_first: bool, default = False\n        Boolean indicating the order of the operations in the dense\n        layer. If `True: [LIN -&gt; ACT -&gt; BN -&gt; DP]`. If `False: [BN -&gt; DP -&gt;\n        LIN -&gt; ACT]`\n    Attributes\n    ----------\n    cross_network: nn.Module\n        The cross network component with mixture of experts\n    deep_network: nn.Module\n        The deep network component (MLP)\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.models.rec import DeepCrossNetworkV2\n    &gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n    &gt;&gt;&gt; colnames = [\"a\", \"b\", \"c\", \"d\", \"e\"]\n    &gt;&gt;&gt; cat_embed_input = [(u, i, j) for u, i, j in zip(colnames[:4], [4] * 4, [8] * 4)]\n    &gt;&gt;&gt; column_idx = {k: v for v, k in enumerate(colnames)}\n    &gt;&gt;&gt; model = DeepCrossNetworkV2(\n    ...     column_idx=column_idx,\n    ...     cat_embed_input=cat_embed_input,\n    ...     continuous_cols=[\"e\"],\n    ...     num_cross_layers=2,\n    ...     low_rank=32,\n    ...     num_experts=4,\n    ...     mlp_hidden_dims=[16, 8]\n    ... )\n    &gt;&gt;&gt; out = model(X_tab)\n    \"\"\"\n\n    def __init__(\n        self,\n        *,\n        column_idx: Dict[str, int],\n        num_cross_layers: int = 2,\n        low_rank: Optional[int] = None,\n        num_experts: int = 2,\n        expert_dropout: float = 0.0,\n        structure: Literal[\"stacked\", \"parallel\"] = \"parallel\",\n        cat_embed_input: Optional[List[Tuple[str, int, int]]] = None,\n        cat_embed_dropout: Optional[float] = None,\n        use_cat_bias: Optional[bool] = None,\n        cat_embed_activation: Optional[str] = None,\n        continuous_cols: Optional[List[str]] = None,\n        cont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\n        embed_continuous: Optional[bool] = None,\n        embed_continuous_method: Optional[\n            Literal[\"standard\", \"piecewise\", \"periodic\"]\n        ] = None,\n        cont_embed_dim: Optional[int] = None,\n        cont_embed_dropout: Optional[float] = None,\n        cont_embed_activation: Optional[str] = None,\n        quantization_setup: Optional[Dict[str, List[float]]] = None,\n        n_frequencies: Optional[int] = None,\n        sigma: Optional[float] = None,\n        share_last_layer: Optional[bool] = None,\n        full_embed_dropout: Optional[bool] = None,\n        mlp_hidden_dims: List[int] = [200, 100],\n        mlp_activation: str = \"relu\",\n        mlp_dropout: Union[float, List[float]] = 0.1,\n        mlp_batchnorm: bool = False,\n        mlp_batchnorm_last: bool = False,\n        mlp_linear_first: bool = True,\n    ):\n        super(DeepCrossNetworkV2, self).__init__(\n            column_idx=column_idx,\n            cat_embed_input=cat_embed_input,\n            cat_embed_dropout=cat_embed_dropout,\n            use_cat_bias=use_cat_bias,\n            cat_embed_activation=cat_embed_activation,\n            continuous_cols=continuous_cols,\n            cont_norm_layer=cont_norm_layer,\n            embed_continuous=embed_continuous,\n            embed_continuous_method=embed_continuous_method,\n            cont_embed_dim=cont_embed_dim,\n            cont_embed_dropout=cont_embed_dropout,\n            cont_embed_activation=cont_embed_activation,\n            quantization_setup=quantization_setup,\n            n_frequencies=n_frequencies,\n            sigma=sigma,\n            share_last_layer=share_last_layer,\n            full_embed_dropout=full_embed_dropout,\n        )\n\n        self.num_cross_layers = num_cross_layers\n        self.num_experts = num_experts\n        self.low_rank = low_rank\n        self.expert_dropout = expert_dropout\n        self.structure = structure\n\n        self.mlp_hidden_dims = mlp_hidden_dims\n        self.mlp_activation = mlp_activation\n        self.mlp_dropout = mlp_dropout\n        self.mlp_batchnorm = mlp_batchnorm\n        self.mlp_batchnorm_last = mlp_batchnorm_last\n        self.mlp_linear_first = mlp_linear_first\n\n        embeddings_output_dim = self.cat_out_dim + self.cont_out_dim\n        self.deep_network = MLP(\n            [embeddings_output_dim] + self.mlp_hidden_dims,\n            self.mlp_activation,\n            self.mlp_dropout,\n            self.mlp_batchnorm,\n            self.mlp_batchnorm_last,\n            self.mlp_linear_first,\n        )\n\n        self.cross_network = CrossNetworkV2(\n            input_dim=embeddings_output_dim,\n            num_layers=self.num_cross_layers,\n            low_rank=self.low_rank,\n            num_experts=self.num_experts,\n            expert_dropout=self.expert_dropout,\n        )\n\n    def forward(self, X: torch.Tensor) -&gt; torch.Tensor:\n        x = self._get_embeddings(X)\n        cross_output = self.cross_network(x)\n\n        if self.structure == \"stacked\":\n            deep_output = self.deep_network(cross_output)\n            return deep_output\n        else:  # parallel\n            deep_output = self.deep_network(x)\n            return torch.cat([cross_output, deep_output], dim=1)\n\n    @property\n    def output_dim(self) -&gt; int:\n        if self.structure == \"stacked\":\n            return self.mlp_hidden_dims[-1]\n        else:  # parallel\n            return self.mlp_hidden_dims[-1] + self.cat_out_dim + self.cont_out_dim\n</code></pre>"},{"location":"pytorch-widedeep/the_rec_module.html#pytorch_widedeep.models.rec.gdcn.GatedDeepCrossNetwork","title":"GatedDeepCrossNetwork","text":"<p>               Bases: <code>BaseTabularModelWithoutAttention</code></p> <p>Defines a <code>GatedDeepCrossNetwork</code> model that can be used as the <code>deeptabular</code> component of a Wide &amp; Deep model or independently by itself.</p> <p>This class implements the Gated Deep &amp; Cross Network (GDCN) architecture as described in the paper Towards Deeper, Lighter and Interpretable Cross Network for CTR Prediction. The GDCN enhances the original DCN by introducing a gating mechanism in the cross network. The gating mechanism controls feature interactions by learning which interactions are more important.</p> <p>The cross layer implements the following equation:</p> \\[c_{i+1} = c_0 \\odot (W^c \\times c_i + b) \\odot \\sigma(W^g \\times c_i) + c_i\\] <p>where:</p> <ul> <li>\\(\\odot\\) represents element-wise multiplication</li> <li>\\(W^c\\) and \\(W^g\\) are the cross and gate weight matrices respectively</li> <li>\\(\\sigma\\) is the sigmoid activation function</li> </ul> <p>Parameters:</p> Name Type Description Default <code>column_idx</code> <code>Dict[str, int]</code> <p>Dict containing the index of the columns that will be passed through the <code>GatedDeepCrossNetwork</code> model. Required to slice the tensors. e.g. {'education': 0, 'relationship': 1, 'workclass': 2, ...}.</p> required <code>num_cross_layers</code> <code>int</code> <p>Number of cross layers in the cross network</p> <code>3</code> <code>structure</code> <code>Literal['stacked', 'parallel']</code> <p>Structure of the model. Either 'parallel' or 'stacked'. If 'parallel', the output will be the concatenation of the cross network and deep network outputs. If 'stacked', the cross network output will be fed into the deep network.</p> <code>'parallel'</code> <code>cat_embed_input</code> <code>Optional[List[Tuple[str, int, int]]]</code> <p>List of Tuples with the column name, number of unique values and embedding dimension. e.g. [(education, 11, 32), ...]</p> <code>None</code> <code>cat_embed_dropout</code> <code>Optional[float]</code> <p>Categorical embeddings dropout. If <code>None</code>, it will default to 0.</p> <code>None</code> <code>use_cat_bias</code> <code>Optional[bool]</code> <p>Boolean indicating if bias will be used for the categorical embeddings. If <code>None</code>, it will default to 'False'.</p> <code>None</code> <code>cat_embed_activation</code> <code>Optional[str]</code> <p>Activation function for the categorical embeddings, if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> <code>None</code> <code>continuous_cols</code> <code>Optional[List[str]]</code> <p>List with the name of the numeric (aka continuous) columns</p> <code>None</code> <code>cont_norm_layer</code> <code>Optional[Literal['batchnorm', 'layernorm']]</code> <p>Type of normalization layer applied to the continuous features. Options are: 'layernorm' and 'batchnorm'. if <code>None</code>, no normalization layer will be used.</p> <code>None</code> <code>embed_continuous_method</code> <code>Optional[Literal['standard', 'piecewise', 'periodic']]</code> <p>Method to use to embed the continuous features. Options are: 'standard', 'periodic' or 'piecewise'. The 'standard' embedding method is based on the FT-Transformer implementation presented in the paper: Revisiting Deep Learning Models for Tabular Data. The 'periodic' and_'piecewise'_ methods were presented in the paper: On Embeddings for Numerical Features in Tabular Deep Learning. Please, read the papers for details.</p> <code>None</code> <code>cont_embed_dropout</code> <code>Optional[float]</code> <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p> <code>None</code> <code>cont_embed_activation</code> <code>Optional[str]</code> <p>Activation function for the continuous embeddings if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported. If <code>None</code>, no activation function will be applied.</p> <code>None</code> <code>quantization_setup</code> <code>Optional[Dict[str, List[float]]]</code> <p>This parameter is used when the 'piecewise' method is used to embed the continuous cols. It is a dict where keys are the name of the continuous columns and values are lists with the boundaries for the quantization of the continuous_cols. See the examples for details. If If the 'piecewise' method is used, this parameter is required.</p> <code>None</code> <code>n_frequencies</code> <code>Optional[int]</code> <p>This is the so called 'k' in their paper On Embeddings for Numerical Features in Tabular Deep Learning, and is the number of 'frequencies' that will be used to represent each continuous column. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>sigma</code> <code>Optional[float]</code> <p>This is the sigma parameter in the paper mentioned when describing the previous parameters and it is used to initialise the 'frequency weights'. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>share_last_layer</code> <code>Optional[bool]</code> <p>This parameter is not present in the before mentioned paper but it is implemented in the official repo. If <code>True</code> the linear layer that turns the frequencies into embeddings will be shared across the continuous columns. If <code>False</code> a different linear layer will be used for each continuous column. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>full_embed_dropout</code> <code>Optional[bool]</code> <p>If <code>True</code>, the full embedding corresponding to a column will be masked out/dropout. If <code>None</code>, it will default to <code>False</code>.</p> <code>None</code> <code>mlp_hidden_dims</code> <code>List[int]</code> <p>List with the number of neurons per dense layer in the mlp.</p> <code>[200, 100]</code> <code>mlp_activation</code> <code>str</code> <p>Activation function for the dense layers of the MLP. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> <code>'relu'</code> <code>mlp_dropout</code> <code>Union[float, List[float]]</code> <p>float or List of floats with the dropout between the dense layers. e.g: [0.5,0.5]</p> <code>0.1</code> <code>mlp_batchnorm</code> <code>bool</code> <p>Boolean indicating whether or not batch normalization will be applied to the dense layers</p> <code>False</code> <code>mlp_batchnorm_last</code> <code>bool</code> <p>Boolean indicating whether or not batch normalization will be applied to the last of the dense layers</p> <code>False</code> <code>mlp_linear_first</code> <code>bool</code> <p>Boolean indicating the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code></p> <code>True</code> <p>Attributes:</p> Name Type Description <code>cross_network</code> <code>Module</code> <p>The gated cross network component</p> <code>deep_network</code> <code>Module</code> <p>The deep network component (MLP)</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models.rec import GatedDeepCrossNetwork\n&gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n&gt;&gt;&gt; colnames = [\"a\", \"b\", \"c\", \"d\", \"e\"]\n&gt;&gt;&gt; cat_embed_input = [(u, i, j) for u, i, j in zip(colnames[:4], [4] * 4, [8] * 4)]\n&gt;&gt;&gt; column_idx = {k: v for v, k in enumerate(colnames)}\n&gt;&gt;&gt; model = GatedDeepCrossNetwork(\n...     column_idx=column_idx,\n...     cat_embed_input=cat_embed_input,\n...     continuous_cols=[\"e\"],\n...     num_cross_layers=2,\n...     mlp_hidden_dims=[16, 8]\n... )\n&gt;&gt;&gt; out = model(X_tab)\n</code></pre> Source code in <code>pytorch_widedeep/models/rec/gdcn.py</code> <pre><code>class GatedDeepCrossNetwork(BaseTabularModelWithoutAttention):\n    r\"\"\"Defines a `GatedDeepCrossNetwork` model that can be used as the `deeptabular`\n    component of a Wide &amp; Deep model or independently by itself.\n\n    This class implements the Gated Deep &amp; Cross Network (GDCN) architecture\n    as described in the paper\n    [Towards Deeper, Lighter and Interpretable Cross Network for CTR Prediction](https://arxiv.org/pdf/2311.04635).\n    The GDCN enhances the original DCN by introducing a gating mechanism in\n    the cross network. The gating mechanism controls feature interactions by\n    learning which interactions are more important.\n\n    The cross layer implements the following equation:\n\n    $$c_{i+1} = c_0 \\odot (W^c \\times c_i + b) \\odot \\sigma(W^g \\times c_i) + c_i$$\n\n    where:\n\n    * $\\odot$ represents element-wise multiplication\n    * $W^c$ and $W^g$ are the cross and gate weight matrices respectively\n    * $\\sigma$ is the sigmoid activation function\n\n    Parameters\n    ----------\n    column_idx: Dict\n        Dict containing the index of the columns that will be passed through\n        the `GatedDeepCrossNetwork` model. Required to slice the tensors. e.g.\n        _{'education': 0, 'relationship': 1, 'workclass': 2, ...}_.\n    num_cross_layers: int, default = 3\n        Number of cross layers in the cross network\n    structure: str, default = \"parallel\"\n        Structure of the model. Either _'parallel'_ or _'stacked'_. If _'parallel'_,\n        the output will be the concatenation of the cross network and deep network\n        outputs. If _'stacked'_, the cross network output will be fed into the deep\n        network.\n    cat_embed_input: List, Optional, default = None\n        List of Tuples with the column name, number of unique values and\n        embedding dimension. e.g. _[(education, 11, 32), ...]_\n    cat_embed_dropout: float, Optional, default = None\n        Categorical embeddings dropout. If `None`, it will default to 0.\n    use_cat_bias: bool, Optional, default = None,\n        Boolean indicating if bias will be used for the categorical embeddings.\n        If `None`, it will default to 'False'.\n    cat_embed_activation: Optional, str, default = None,\n        Activation function for the categorical embeddings, if any. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported\n    continuous_cols : Optional[List[str]], default=None\n        List with the name of the numeric (aka continuous) columns\n    cont_norm_layer : Optional[Literal[\"batchnorm\", \"layernorm\"]], default=None\n        Type of normalization layer applied to the continuous features.\n        Options are: _'layernorm'_ and _'batchnorm'_. if `None`, no\n        normalization layer will be used.\n    embed_continuous_method: Optional[Literal[\"piecewise\", \"periodic\", \"standard\"]], default=\"standard\"\n        Method to use to embed the continuous features. Options are:\n        _'standard'_, _'periodic'_ or _'piecewise'_. The _'standard'_\n        embedding method is based on the FT-Transformer implementation\n        presented in the paper: [Revisiting Deep Learning Models for\n        Tabular Data](https://arxiv.org/abs/2106.11959v5). The _'periodic'_\n        and_'piecewise'_ methods were presented in the paper: [On Embeddings for\n        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556).\n        Please, read the papers for details.\n    cont_embed_dropout : Optional[float], default=None\n        Dropout for the continuous embeddings. If `None`, it will default to 0.0\n    cont_embed_activation : Optional[str], default=None\n        Activation function for the continuous embeddings if any. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported.\n        If `None`, no activation function will be applied.\n    quantization_setup : Optional[Dict[str, List[float]]], default=None\n        This parameter is used when the _'piecewise'_ method is used to embed\n        the continuous cols. It is a dict where keys are the name of the continuous\n        columns and values are lists with the boundaries for the quantization\n        of the continuous_cols. See the examples for details. If\n        If the _'piecewise'_ method is used, this parameter is required.\n    n_frequencies : Optional[int], default=None\n        This is the so called _'k'_ in their paper [On Embeddings for\n        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556),\n        and is the number of 'frequencies' that will be used to represent each\n        continuous column. See their Eq 2 in the paper for details. If\n        the _'periodic'_ method is used, this parameter is required.\n    sigma : Optional[float], default=None\n        This is the sigma parameter in the paper mentioned when describing the\n        previous parameters and it is used to initialise the 'frequency\n        weights'. See their Eq 2 in the paper for details. If\n        the _'periodic'_ method is used, this parameter is required.\n    share_last_layer : Optional[bool], default=None\n        This parameter is not present in the before mentioned paper but it is implemented in\n        the [official repo](https://github.com/yandex-research/rtdl-num-embeddings/tree/main).\n        If `True` the linear layer that turns the frequencies into embeddings\n        will be shared across the continuous columns. If `False` a different\n        linear layer will be used for each continuous column.\n        If the _'periodic'_ method is used, this parameter is required.\n    full_embed_dropout: bool, Optional, default = None,\n        If `True`, the full embedding corresponding to a column will be masked\n        out/dropout. If `None`, it will default to `False`.\n    mlp_hidden_dims: List, default = [200, 100]\n        List with the number of neurons per dense layer in the mlp.\n    mlp_activation: str, default = \"relu\"\n        Activation function for the dense layers of the MLP. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported\n    mlp_dropout: float or List, default = 0.1\n        float or List of floats with the dropout between the dense layers.\n        e.g: _[0.5,0.5]_\n    mlp_batchnorm: bool, default = False\n        Boolean indicating whether or not batch normalization will be applied\n        to the dense layers\n    mlp_batchnorm_last: bool, default = False\n        Boolean indicating whether or not batch normalization will be applied\n        to the last of the dense layers\n    mlp_linear_first: bool, default = False\n        Boolean indicating the order of the operations in the dense\n        layer. If `True: [LIN -&gt; ACT -&gt; BN -&gt; DP]`. If `False: [BN -&gt; DP -&gt;\n        LIN -&gt; ACT]`\n\n    Attributes\n    ----------\n    cross_network: nn.Module\n        The gated cross network component\n    deep_network: nn.Module\n        The deep network component (MLP)\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.models.rec import GatedDeepCrossNetwork\n    &gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n    &gt;&gt;&gt; colnames = [\"a\", \"b\", \"c\", \"d\", \"e\"]\n    &gt;&gt;&gt; cat_embed_input = [(u, i, j) for u, i, j in zip(colnames[:4], [4] * 4, [8] * 4)]\n    &gt;&gt;&gt; column_idx = {k: v for v, k in enumerate(colnames)}\n    &gt;&gt;&gt; model = GatedDeepCrossNetwork(\n    ...     column_idx=column_idx,\n    ...     cat_embed_input=cat_embed_input,\n    ...     continuous_cols=[\"e\"],\n    ...     num_cross_layers=2,\n    ...     mlp_hidden_dims=[16, 8]\n    ... )\n    &gt;&gt;&gt; out = model(X_tab)\n    \"\"\"\n\n    def __init__(\n        self,\n        *,\n        column_idx: Dict[str, int],\n        num_cross_layers: int = 3,\n        structure: Literal[\"stacked\", \"parallel\"] = \"parallel\",\n        cat_embed_input: Optional[List[Tuple[str, int, int]]] = None,\n        cat_embed_dropout: Optional[float] = None,\n        use_cat_bias: Optional[bool] = None,\n        cat_embed_activation: Optional[str] = None,\n        continuous_cols: Optional[List[str]] = None,\n        cont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\n        embed_continuous: Optional[bool] = None,\n        embed_continuous_method: Optional[\n            Literal[\"standard\", \"piecewise\", \"periodic\"]\n        ] = None,\n        cont_embed_dim: Optional[int] = None,\n        cont_embed_dropout: Optional[float] = None,\n        cont_embed_activation: Optional[str] = None,\n        quantization_setup: Optional[Dict[str, List[float]]] = None,\n        n_frequencies: Optional[int] = None,\n        sigma: Optional[float] = None,\n        share_last_layer: Optional[bool] = None,\n        full_embed_dropout: Optional[bool] = None,\n        mlp_hidden_dims: List[int] = [200, 100],\n        mlp_activation: str = \"relu\",\n        mlp_dropout: Union[float, List[float]] = 0.1,\n        mlp_batchnorm: bool = False,\n        mlp_batchnorm_last: bool = False,\n        mlp_linear_first: bool = True,\n    ):\n        super(GatedDeepCrossNetwork, self).__init__(\n            column_idx=column_idx,\n            cat_embed_input=cat_embed_input,\n            cat_embed_dropout=cat_embed_dropout,\n            use_cat_bias=use_cat_bias,\n            cat_embed_activation=cat_embed_activation,\n            continuous_cols=continuous_cols,\n            cont_norm_layer=cont_norm_layer,\n            embed_continuous=embed_continuous,\n            embed_continuous_method=embed_continuous_method,\n            cont_embed_dim=cont_embed_dim,\n            cont_embed_dropout=cont_embed_dropout,\n            cont_embed_activation=cont_embed_activation,\n            quantization_setup=quantization_setup,\n            n_frequencies=n_frequencies,\n            sigma=sigma,\n            share_last_layer=share_last_layer,\n            full_embed_dropout=full_embed_dropout,\n        )\n\n        self.num_cross_layers = num_cross_layers\n        self.structure = structure\n\n        self.mlp_hidden_dims = mlp_hidden_dims\n        self.mlp_activation = mlp_activation\n        self.mlp_dropout = mlp_dropout\n        self.mlp_batchnorm = mlp_batchnorm\n        self.mlp_batchnorm_last = mlp_batchnorm_last\n        self.mlp_linear_first = mlp_linear_first\n\n        embeddings_output_dim = self.cat_out_dim + self.cont_out_dim\n        self.deep_network = MLP(\n            [embeddings_output_dim] + self.mlp_hidden_dims,\n            self.mlp_activation,\n            self.mlp_dropout,\n            self.mlp_batchnorm,\n            self.mlp_batchnorm_last,\n            self.mlp_linear_first,\n        )\n\n        self.cross_network = GatedCrossNetwork(embeddings_output_dim, num_cross_layers)\n\n    def forward(self, X: torch.Tensor) -&gt; torch.Tensor:\n        x = self._get_embeddings(X)\n        cross_output = self.cross_network(x)\n\n        if self.structure == \"stacked\":\n            deep_output = self.deep_network(cross_output)\n            return deep_output\n        else:  # parallel\n            deep_output = self.deep_network(x)\n            return torch.cat([cross_output, deep_output], dim=1)\n\n    @property\n    def output_dim(self) -&gt; int:\n        if self.structure == \"stacked\":\n            return self.mlp_hidden_dims[-1]\n        else:  # parallel\n            return self.mlp_hidden_dims[-1] + self.cat_out_dim + self.cont_out_dim\n</code></pre>"},{"location":"pytorch-widedeep/trainer.html","title":"Training multimodal Deep Learning Models","text":"<p>Here is the documentation for the <code>Trainer</code> class, that will do all the heavy lifting.</p> <p>Trainer is also available from <code>pytorch-widedeep</code> directly, for example, one could do:</p> <pre><code>    from pytorch-widedeep.training import Trainer\n</code></pre> <p>or also:</p> <pre><code>    from pytorch-widedeep import Trainer\n</code></pre>"},{"location":"pytorch-widedeep/trainer.html#pytorch_widedeep.training.Trainer","title":"Trainer","text":"<p>               Bases: <code>BaseTrainer</code></p> <p>Class to set the of attributes that will be used during the training process.</p> <p>Parameters:</p> Name Type Description Default <code>model</code> <code>WideDeep</code> <p>An object of class <code>WideDeep</code></p> required <code>objective</code> <code>str</code> <p>Defines the objective, loss or cost function. </p> <p>Param aliases: <code>loss_function</code>, <code>loss_fn</code>, <code>loss</code>, <code>cost_function</code>, <code>cost_fn</code>, <code>cost</code>. </p> <p>Possible values are:</p> <ul> <li> <p><code>binary</code>, aliases: <code>logistic</code>, <code>binary_logloss</code>, <code>binary_cross_entropy</code></p> </li> <li> <p><code>binary_focal_loss</code></p> </li> <li> <p><code>multiclass</code>, aliases: <code>multi_logloss</code>, <code>cross_entropy</code>, <code>categorical_cross_entropy</code>,</p> </li> <li> <p><code>multiclass_focal_loss</code></p> </li> <li> <p><code>regression</code>, aliases: <code>mse</code>, <code>l2</code>, <code>mean_squared_error</code></p> </li> <li> <p><code>mean_absolute_error</code>, aliases: <code>mae</code>, <code>l1</code></p> </li> <li> <p><code>mean_squared_log_error</code>, aliases: <code>msle</code></p> </li> <li> <p><code>root_mean_squared_error</code>, aliases:  <code>rmse</code></p> </li> <li> <p><code>root_mean_squared_log_error</code>, aliases: <code>rmsle</code></p> </li> <li> <p><code>zero_inflated_lognormal</code>, aliases: <code>ziln</code></p> </li> <li> <p><code>quantile</code></p> </li> <li> <p><code>tweedie</code></p> </li> <li> <p><code>multitarget</code>, aliases: <code>multi_target</code></p> </li> </ul> <p>NOTE: For <code>multitarget</code> a custom loss function must be passed</p> required <code>custom_loss_function</code> <code>Optional[Module]</code> <p>It is possible to pass a custom loss function. See for example <code>pytorch_widedeep.losses.FocalLoss</code> for the required structure of the object or the Examples section in this documentation or in the repo. Note that if <code>custom_loss_function</code> is not <code>None</code>, <code>objective</code> must be 'binary', 'multiclass' or 'regression', consistent with the loss function</p> <code>None</code> <code>optimizers</code> <code>Optional[Union[Optimizer, Dict[str, Union[Optimizer, List[Optimizer]]]]]</code> <ul> <li>An instance of Pytorch's <code>Optimizer</code> object   (e.g. <code>torch.optim.Adam()</code>) or</li> <li>a dictionary where there keys are the model components (i.e.   'wide', 'deeptabular', 'deeptext', 'deepimage'   and/or 'deephead')  and the values are the corresponding   optimizers or list of optimizers if multiple models are used for   the given data mode (e.g. two text columns/models for the deeptext   component). If multiple optimizers are used the   dictionary MUST contain an optimizer per model component.</li> </ul> <p>if no optimizers are passed it will default to <code>Adam</code> for all model components</p> <code>None</code> <code>lr_schedulers</code> <code>Optional[Union[LRScheduler, Dict[str, Union[LRScheduler, List[LRScheduler]]]]]</code> <ul> <li>An instance of Pytorch's <code>LRScheduler</code> object (e.g   <code>torch.optim.lr_scheduler.StepLR(opt, step_size=5)</code>) or</li> <li>a dictionary where there keys are the model componenst (i.e. 'wide',   'deeptabular', 'deeptext', 'deepimage' and/or 'deephead') and the   values are the corresponding learning rate schedulers or list of     learning rate schedulers if multiple models are used for the given     data mode (e.g. two text columns/models for the deeptext component).</li> </ul> <code>None</code> <code>initializers</code> <code>Optional[Union[Initializer, Dict[str, Union[Initializer, List[Initializer]]]]]</code> <ul> <li>An instance of an <code>Initializer</code> object see <code>pytorch-widedeep.initializers</code> or</li> <li>a dictionary where there keys are the model components (i.e. 'wide',   'deeptabular', 'deeptext', 'deepimage' and/or 'deephead')   and the values are the corresponding initializers or list of     initializers if multiple models are used for the given data mode (e.g.     two text columns/models for the deeptext component).</li> </ul> <code>None</code> <code>transforms</code> <code>Optional[List[Transforms]]</code> <p>List with <code>torchvision.transforms</code> to be applied to the image component of the model (i.e. <code>deepimage</code>) See torchvision transforms.</p> <code>None</code> <code>callbacks</code> <code>Optional[List[Callback]]</code> <p>List with <code>Callback</code> objects. The three callbacks available in <code>pytorch-widedeep</code> are: <code>LRHistory</code>, <code>ModelCheckpoint</code> and <code>EarlyStopping</code>. The <code>History</code> and the <code>LRShedulerCallback</code> callbacks are used by default. This can also be a custom callback as long as the object of type <code>Callback</code>. See <code>pytorch_widedeep.callbacks.Callback</code> or the examples folder in the repo.</p> <code>None</code> <code>metrics</code> <code>Optional[Union[List[Metric], List[Metric]]]</code> <ul> <li>List of objects of type <code>Metric</code>. Metrics available are:   <code>Accuracy</code>, <code>Precision</code>, <code>Recall</code>, <code>FBetaScore</code>,   <code>F1Score</code> and <code>R2Score</code>. This can also be a custom metric as long   as it is an object of type <code>Metric</code>. See   <code>pytorch_widedeep.metrics.Metric</code> or the examples folder in the   repo</li> <li>List of objects of type <code>torchmetrics.Metric</code>. This can be any   metric from torchmetrics library   Examples.   This can also be a custom metric as long as   it is an object of type <code>Metric</code>. See   the instructions.</li> </ul> <code>None</code> <code>verbose</code> <code>int</code> <p>Verbosity level. If set to 0 nothing will be printed during training</p> <code>1</code> <code>seed</code> <code>int</code> <p>Random seed to be used internally for train/test split</p> <code>1</code> <p>Other Parameters:</p> Name Type Description <code>**kwargs</code> <p>Other infrequently used arguments that can also be passed as kwargs are:</p> <ul> <li> <p>device: <code>str</code>     string indicating the device. One of 'cpu', 'gpu' or 'mps' if     run on a Mac with Apple silicon or AMD GPU(s)</p> </li> <li> <p>num_workers: <code>int</code>     number of workers to be used internally by the data loaders</p> </li> <li> <p>lambda_sparse: <code>float</code>     lambda sparse parameter in case the <code>deeptabular</code> component is <code>TabNet</code></p> </li> <li> <p>class_weight: <code>List[float]</code>     This is the <code>weight</code> or <code>pos_weight</code> parameter in     <code>CrossEntropyLoss</code> and <code>BCEWithLogitsLoss</code>, depending on whether</p> </li> <li>reducelronplateau_criterion: <code>str</code>     This sets the criterion that will be used by the lr scheduler to     take a step: One of 'loss' or 'metric'. The ReduceLROnPlateau     learning rate is a bit particular.</li> </ul> <p>Attributes:</p> Name Type Description <code>cyclic_lr</code> <code>bool</code> <p>Attribute that indicates if any of the lr_schedulers is cyclic_lr (i.e. <code>CyclicLR</code> or <code>OneCycleLR</code>). See Pytorch schedulers.</p> <code>feature_importance</code> <code>dict</code> <p>dict where the keys are the column names and the values are the corresponding feature importances. This attribute will only exist if the <code>deeptabular</code> component is a Tabnet model.</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from torchvision.transforms import ToTensor\n&gt;&gt;&gt;\n&gt;&gt;&gt; # wide deep imports\n&gt;&gt;&gt; from pytorch_widedeep.callbacks import EarlyStopping, LRHistory\n&gt;&gt;&gt; from pytorch_widedeep.initializers import KaimingNormal, KaimingUniform, Normal, Uniform\n&gt;&gt;&gt; from pytorch_widedeep.models import TabResnet, Vision, BasicRNN, Wide, WideDeep\n&gt;&gt;&gt; from pytorch_widedeep import Trainer\n&gt;&gt;&gt;\n&gt;&gt;&gt; embed_input = [(u, i, j) for u, i, j in zip([\"a\", \"b\", \"c\"][:4], [4] * 3, [8] * 3)]\n&gt;&gt;&gt; column_idx = {k: v for v, k in enumerate([\"a\", \"b\", \"c\"])}\n&gt;&gt;&gt; wide = Wide(10, 1)\n&gt;&gt;&gt;\n&gt;&gt;&gt; # build the model\n&gt;&gt;&gt; deeptabular = TabResnet(blocks_dims=[8, 4], column_idx=column_idx, cat_embed_input=embed_input)\n&gt;&gt;&gt; deeptext = BasicRNN(vocab_size=10, embed_dim=4, padding_idx=0)\n&gt;&gt;&gt; deepimage = Vision()\n&gt;&gt;&gt; model = WideDeep(wide=wide, deeptabular=deeptabular, deeptext=deeptext, deepimage=deepimage)\n&gt;&gt;&gt;\n&gt;&gt;&gt; # set optimizers and schedulers\n&gt;&gt;&gt; wide_opt = torch.optim.Adam(model.wide.parameters())\n&gt;&gt;&gt; deep_opt = torch.optim.AdamW(model.deeptabular.parameters())\n&gt;&gt;&gt; text_opt = torch.optim.Adam(model.deeptext.parameters())\n&gt;&gt;&gt; img_opt = torch.optim.AdamW(model.deepimage.parameters())\n&gt;&gt;&gt;\n&gt;&gt;&gt; wide_sch = torch.optim.lr_scheduler.StepLR(wide_opt, step_size=5)\n&gt;&gt;&gt; deep_sch = torch.optim.lr_scheduler.StepLR(deep_opt, step_size=3)\n&gt;&gt;&gt; text_sch = torch.optim.lr_scheduler.StepLR(text_opt, step_size=5)\n&gt;&gt;&gt; img_sch = torch.optim.lr_scheduler.StepLR(img_opt, step_size=3)\n&gt;&gt;&gt;\n&gt;&gt;&gt; optimizers = {\"wide\": wide_opt, \"deeptabular\": deep_opt, \"deeptext\": text_opt, \"deepimage\": img_opt}\n&gt;&gt;&gt; schedulers = {\"wide\": wide_sch, \"deeptabular\": deep_sch, \"deeptext\": text_sch, \"deepimage\": img_sch}\n&gt;&gt;&gt;\n&gt;&gt;&gt; # set initializers and callbacks\n&gt;&gt;&gt; initializers = {\"wide\": Uniform, \"deeptabular\": Normal, \"deeptext\": KaimingNormal, \"deepimage\": KaimingUniform}\n&gt;&gt;&gt; transforms = [ToTensor]\n&gt;&gt;&gt; callbacks = [LRHistory(n_epochs=4), EarlyStopping]\n&gt;&gt;&gt;\n&gt;&gt;&gt; # set the trainer\n&gt;&gt;&gt; trainer = Trainer(model, objective=\"regression\", initializers=initializers, optimizers=optimizers,\n... lr_schedulers=schedulers, callbacks=callbacks, transforms=transforms)\n</code></pre> Source code in <code>pytorch_widedeep/training/trainer.py</code> <pre><code>class Trainer(BaseTrainer):\n    r\"\"\"Class to set the of attributes that will be used during the\n    training process.\n\n    Parameters\n    ----------\n    model: `WideDeep`\n        An object of class `WideDeep`\n    objective: str\n        Defines the objective, loss or cost function. &lt;br/&gt;\n\n        Param aliases: `loss_function`, `loss_fn`, `loss`,\n        `cost_function`, `cost_fn`, `cost`. &lt;br/&gt;\n\n        Possible values are:\n\n        - `binary`, aliases: `logistic`, `binary_logloss`, `binary_cross_entropy`\n\n        - `binary_focal_loss`\n\n        - `multiclass`, aliases: `multi_logloss`, `cross_entropy`, `categorical_cross_entropy`,\n\n        - `multiclass_focal_loss`\n\n        - `regression`, aliases: `mse`, `l2`, `mean_squared_error`\n\n        - `mean_absolute_error`, aliases: `mae`, `l1`\n\n        - `mean_squared_log_error`, aliases: `msle`\n\n        - `root_mean_squared_error`, aliases:  `rmse`\n\n        - `root_mean_squared_log_error`, aliases: `rmsle`\n\n        - `zero_inflated_lognormal`, aliases: `ziln`\n\n        - `quantile`\n\n        - `tweedie`\n\n        - `multitarget`, aliases: `multi_target`\n\n        **NOTE**: For `multitarget` a custom loss function must be passed\n    custom_loss_function: `nn.Module`. Optional, default = None\n        It is possible to pass a custom loss function. See for example\n        `pytorch_widedeep.losses.FocalLoss` for the required structure of the\n        object or the Examples section in this documentation or in the repo.\n        Note that if `custom_loss_function` is not `None`, `objective` must\n        be _'binary'_, _'multiclass'_ or _'regression'_, consistent with the\n        loss function\n    optimizers: `Optimizer` or dict. Optional, default=None\n        - An instance of Pytorch's `Optimizer` object\n          (e.g. `torch.optim.Adam()`) or\n        - a dictionary where there keys are the model components (i.e.\n          _'wide'_, _'deeptabular'_, _'deeptext'_, _'deepimage'_\n          and/or _'deephead'_)  and the values are the corresponding\n          optimizers or list of optimizers if multiple models are used for\n          the given data mode (e.g. two text columns/models for the deeptext\n          component). If multiple optimizers are used the\n          dictionary **MUST** contain an optimizer per model component.\n\n        if no optimizers are passed it will default to `Adam` for all\n        model components\n    lr_schedulers: `LRScheduler` or dict. Optional, default=None\n        - An instance of Pytorch's `LRScheduler` object (e.g\n          `torch.optim.lr_scheduler.StepLR(opt, step_size=5)`) or\n        - a dictionary where there keys are the model componenst (i.e. _'wide'_,\n          _'deeptabular'_, _'deeptext'_, _'deepimage'_ and/or _'deephead'_) and the\n          values are the corresponding learning rate schedulers or list of\n            learning rate schedulers if multiple models are used for the given\n            data mode (e.g. two text columns/models for the deeptext component).\n    initializers: `Initializer` or dict. Optional, default=None\n        - An instance of an `Initializer` object see `pytorch-widedeep.initializers` or\n        - a dictionary where there keys are the model components (i.e. _'wide'_,\n          _'deeptabular'_, _'deeptext'_, _'deepimage'_ and/or _'deephead'_)\n          and the values are the corresponding initializers or list of\n            initializers if multiple models are used for the given data mode (e.g.\n            two text columns/models for the deeptext component).\n    transforms: List. Optional, default=None\n        List with `torchvision.transforms` to be applied to the image\n        component of the model (i.e. `deepimage`) See\n        [torchvision transforms](https://pytorch.org/docs/stable/torchvision/transforms.html).\n    callbacks: List. Optional, default=None\n        List with `Callback` objects. The three callbacks available in\n        `pytorch-widedeep` are: `LRHistory`, `ModelCheckpoint` and\n        `EarlyStopping`. The `History` and the `LRShedulerCallback` callbacks\n        are used by default. This can also be a custom callback as long as\n        the object of type `Callback`. See\n        `pytorch_widedeep.callbacks.Callback` or the examples folder in the\n        repo.\n    metrics: List. Optional, default=None\n        - List of objects of type `Metric`. Metrics available are:\n          `Accuracy`, `Precision`, `Recall`, `FBetaScore`,\n          `F1Score` and `R2Score`. This can also be a custom metric as long\n          as it is an object of type `Metric`. See\n          `pytorch_widedeep.metrics.Metric` or the examples folder in the\n          repo\n        - List of objects of type `torchmetrics.Metric`. This can be any\n          metric from torchmetrics library\n          [Examples](https://torchmetrics.readthedocs.io/en/latest/).\n          This can also be a custom metric as long as\n          it is an object of type `Metric`. See\n          [the instructions](https://torchmetrics.readthedocs.io/en/latest/).\n    verbose: int, default=1\n        Verbosity level. If set to 0 nothing will be printed during training\n    seed: int, default=1\n        Random seed to be used internally for train/test split\n\n    Other Parameters\n    ----------------\n    **kwargs: dict\n        Other infrequently used arguments that can also be passed as kwargs are:\n\n        - **device**: `str`&lt;br/&gt;\n            string indicating the device. One of _'cpu'_, _'gpu'_ or 'mps' if\n            run on a Mac with Apple silicon or AMD GPU(s)\n\n        - **num_workers**: `int`&lt;br/&gt;\n            number of workers to be used internally by the data loaders\n\n        - **lambda_sparse**: `float`&lt;br/&gt;\n            lambda sparse parameter in case the `deeptabular` component is `TabNet`\n\n        - **class_weight**: `List[float]`&lt;br/&gt;\n            This is the `weight` or `pos_weight` parameter in\n            `CrossEntropyLoss` and `BCEWithLogitsLoss`, depending on whether\n        - **reducelronplateau_criterion**: `str`\n            This sets the criterion that will be used by the lr scheduler to\n            take a step: One of _'loss'_ or _'metric'_. The ReduceLROnPlateau\n            learning rate is a bit particular.\n\n    Attributes\n    ----------\n    cyclic_lr: bool\n        Attribute that indicates if any of the lr_schedulers is cyclic_lr\n        (i.e. `CyclicLR` or\n        `OneCycleLR`). See [Pytorch schedulers](https://pytorch.org/docs/stable/optim.html).\n    feature_importance: dict\n        dict where the keys are the column names and the values are the\n        corresponding feature importances. This attribute will only exist\n        if the `deeptabular` component is a Tabnet model.\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from torchvision.transforms import ToTensor\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; # wide deep imports\n    &gt;&gt;&gt; from pytorch_widedeep.callbacks import EarlyStopping, LRHistory\n    &gt;&gt;&gt; from pytorch_widedeep.initializers import KaimingNormal, KaimingUniform, Normal, Uniform\n    &gt;&gt;&gt; from pytorch_widedeep.models import TabResnet, Vision, BasicRNN, Wide, WideDeep\n    &gt;&gt;&gt; from pytorch_widedeep import Trainer\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; embed_input = [(u, i, j) for u, i, j in zip([\"a\", \"b\", \"c\"][:4], [4] * 3, [8] * 3)]\n    &gt;&gt;&gt; column_idx = {k: v for v, k in enumerate([\"a\", \"b\", \"c\"])}\n    &gt;&gt;&gt; wide = Wide(10, 1)\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; # build the model\n    &gt;&gt;&gt; deeptabular = TabResnet(blocks_dims=[8, 4], column_idx=column_idx, cat_embed_input=embed_input)\n    &gt;&gt;&gt; deeptext = BasicRNN(vocab_size=10, embed_dim=4, padding_idx=0)\n    &gt;&gt;&gt; deepimage = Vision()\n    &gt;&gt;&gt; model = WideDeep(wide=wide, deeptabular=deeptabular, deeptext=deeptext, deepimage=deepimage)\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; # set optimizers and schedulers\n    &gt;&gt;&gt; wide_opt = torch.optim.Adam(model.wide.parameters())\n    &gt;&gt;&gt; deep_opt = torch.optim.AdamW(model.deeptabular.parameters())\n    &gt;&gt;&gt; text_opt = torch.optim.Adam(model.deeptext.parameters())\n    &gt;&gt;&gt; img_opt = torch.optim.AdamW(model.deepimage.parameters())\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; wide_sch = torch.optim.lr_scheduler.StepLR(wide_opt, step_size=5)\n    &gt;&gt;&gt; deep_sch = torch.optim.lr_scheduler.StepLR(deep_opt, step_size=3)\n    &gt;&gt;&gt; text_sch = torch.optim.lr_scheduler.StepLR(text_opt, step_size=5)\n    &gt;&gt;&gt; img_sch = torch.optim.lr_scheduler.StepLR(img_opt, step_size=3)\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; optimizers = {\"wide\": wide_opt, \"deeptabular\": deep_opt, \"deeptext\": text_opt, \"deepimage\": img_opt}\n    &gt;&gt;&gt; schedulers = {\"wide\": wide_sch, \"deeptabular\": deep_sch, \"deeptext\": text_sch, \"deepimage\": img_sch}\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; # set initializers and callbacks\n    &gt;&gt;&gt; initializers = {\"wide\": Uniform, \"deeptabular\": Normal, \"deeptext\": KaimingNormal, \"deepimage\": KaimingUniform}\n    &gt;&gt;&gt; transforms = [ToTensor]\n    &gt;&gt;&gt; callbacks = [LRHistory(n_epochs=4), EarlyStopping]\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; # set the trainer\n    &gt;&gt;&gt; trainer = Trainer(model, objective=\"regression\", initializers=initializers, optimizers=optimizers,\n    ... lr_schedulers=schedulers, callbacks=callbacks, transforms=transforms)\n    \"\"\"\n\n    @alias(  # noqa: C901\n        \"objective\",\n        [\"loss_function\", \"loss_fn\", \"loss\", \"cost_function\", \"cost_fn\", \"cost\"],\n    )\n    @alias(\"metrics\", [\"train_metrics\"])\n    def __init__(\n        self,\n        model: WideDeep,\n        objective: str,\n        custom_loss_function: Optional[nn.Module] = None,\n        optimizers: Optional[\n            Union[Optimizer, Dict[str, Union[Optimizer, List[Optimizer]]]]\n        ] = None,\n        lr_schedulers: Optional[\n            Union[LRScheduler, Dict[str, Union[LRScheduler, List[LRScheduler]]]]\n        ] = None,\n        initializers: Optional[\n            Union[Initializer, Dict[str, Union[Initializer, List[Initializer]]]]\n        ] = None,\n        transforms: Optional[List[Transforms]] = None,\n        callbacks: Optional[List[Callback]] = None,\n        metrics: Optional[Union[List[Metric], List[TorchMetric]]] = None,\n        eval_metrics: Optional[Union[List[Metric], List[TorchMetric]]] = None,\n        verbose: int = 1,\n        seed: int = 1,\n        **kwargs,\n    ):\n        super().__init__(\n            model=model,\n            objective=objective,\n            custom_loss_function=custom_loss_function,\n            optimizers=optimizers,\n            lr_schedulers=lr_schedulers,\n            initializers=initializers,\n            transforms=transforms,\n            callbacks=callbacks,\n            metrics=metrics,\n            eval_metrics=eval_metrics,\n            verbose=verbose,\n            seed=seed,\n            **kwargs,\n        )\n\n    @alias(\"finetune\", [\"warmup\"])\n    @alias(\"train_dataloader\", [\"custom_dataloader\"])\n    @alias(\"eval_dataloader\", [\"custom_eval_dataloader\"])\n    def fit(  # noqa: C901\n        self,\n        X_wide: Optional[np.ndarray] = None,\n        X_tab: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n        X_text: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n        X_img: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n        X_train: Optional[Dict[str, Union[np.ndarray, List[np.ndarray]]]] = None,\n        X_val: Optional[Dict[str, Union[np.ndarray, List[np.ndarray]]]] = None,\n        val_split: Optional[float] = None,\n        target: Optional[np.ndarray] = None,\n        n_epochs: int = 1,\n        validation_freq: int = 1,\n        batch_size: int = 32,\n        train_dataloader: Optional[CustomDataLoader] = None,\n        eval_dataloader: Optional[CustomDataLoader] = None,\n        feature_importance_sample_size: Optional[int] = None,\n        finetune: bool = False,\n        **kwargs,\n    ):\n        r\"\"\"Fit method.\n\n        The input datasets can be passed either directly via numpy arrays\n        (`X_wide`, `X_tab`, `X_text` or `X_img`) or alternatively, in\n        dictionaries (`X_train` or `X_val`).\n\n        Parameters\n        ----------\n        X_wide: np.ndarray, Optional. default=None\n            Input for the `wide` model component.\n            See `pytorch_widedeep.preprocessing.WidePreprocessor`\n        X_tab: np.ndarray, Optional. default=None\n            Input for the `deeptabular` model component.\n            See `pytorch_widedeep.preprocessing.TabPreprocessor`. If multiple\n            tabular models are used for different columns, this should be a\n            list of numpy arrays\n        X_text: Union[np.ndarray, List[np.ndarray]], Optional. default=None\n            Input for the `deeptext` model component.\n            See `pytorch_widedeep.preprocessing.TextPreprocessor`.\n            If multiple text columns/models are used, this should be a list of\n            numpy arrays\n        X_img: np.ndarray, Optional. default=None\n            Input for the `deepimage` model component.\n            See `pytorch_widedeep.preprocessing.ImagePreprocessor`.\n            If multiple image columns/models are used, this should be a list of\n            numpy arrays\n        X_train: Dict, Optional. default=None\n            The training dataset can also be passed in a dictionary. Keys are\n            _'X_wide'_, _'X_tab'_, _'X_text'_, _'X_img'_ and _'target'_. Values\n            are the corresponding matrices. Note that of multiple text or image\n            columns/models are used, the corresponding values should be lists\n            of numpy arrays\n        X_val: Dict, Optional. default=None\n            The validation dataset can also be passed in a dictionary. Keys\n            are _'X_wide'_, _'X_tab'_, _'X_text'_, _'X_img'_ and _'target'_.\n            Values are the corresponding matrices. Note that of multiple text\n            or image columns/models are used, the corresponding values should\n            be lists of numpy arrays\n        val_split: float, Optional. default=None\n            train/val split fraction\n        target: np.ndarray, Optional. default=None\n            target values\n        n_epochs: int, default=1\n            number of epochs\n        validation_freq: int, default=1\n            epochs validation frequency\n        batch_size: int, default=32\n            batch size\n        custom_dataloader: `DataLoader`, Optional, default=None\n            object of class `torch.utils.data.DataLoader`. Available\n            predefined dataloaders are in `pytorch-widedeep.dataloaders`.If\n            `None`, a standard torch `DataLoader` is used.\n        finetune: bool, default=False\n            fine-tune individual model components. This functionality can also\n            be used to 'warm-up' (and hence the alias `warmup`) individual\n            components before the joined training starts, and hence its\n            alias. See the Examples folder in the repo for more details\n\n            `pytorch_widedeep` implements 3 fine-tune routines.\n\n            - fine-tune all trainable layers at once. This routine is\n              inspired by the work of Howard &amp; Sebastian Ruder 2018 in their\n              [ULMfit paper](https://arxiv.org/abs/1801.06146). Using a\n              Slanted Triangular learing (see\n              [Leslie N. Smith paper](https://arxiv.org/pdf/1506.01186.pdf) ) ,\n              the process is the following: *i*) the learning rate will\n              gradually increase for 10% of the training steps from max_lr/10\n              to max_lr. *ii*) It will then gradually decrease to max_lr/10\n              for the remaining 90% of the steps. The optimizer used in the\n              process is `Adam`.\n\n            and two gradual fine-tune routines, where only certain layers are\n            trained at a time.\n\n            - The so called `Felbo` gradual fine-tune rourine, based on the the\n              Felbo et al., 2017 [DeepEmoji paper](https://arxiv.org/abs/1708.00524).\n            - The `Howard` routine based on the work of Howard &amp; Sebastian Ruder 2018 in their\n              [ULMfit paper](https://arxiv.org/abs/1801.06146&gt;).\n\n            For details on how these routines work, please see the Examples\n            section in this documentation and the Examples folder in the repo. &lt;br/&gt;\n            Param Alias: `warmup`\n\n        Other Parameters\n        ----------------\n        **kwargs:\n            Other keyword arguments are:\n\n            - **DataLoader related parameters**:&lt;br/&gt;\n                For example,  `sampler`, `batch_sampler`, `collate_fn`, etc.\n                Please, see the pytorch\n                [DataLoader docs](https://pytorch.org/docs/stable/data.html#torch.utils.data.DataLoader)\n                for details.\n\n            - **Finetune related parameters**:&lt;br/&gt;\n                see the source code at `pytorch_widedeep._finetune`. Namely, these are:\n\n                - `finetune_epochs` (`int`):\n                    number of epochs use for fine tuning\n                - `finetune_max_lr` (`float`):\n                   max lr during fine tuning\n                - `routine` (`str`):\n                   one of _'howard'_ or _'felbo'_\n                - `deeptabular_gradual` (`bool`):\n                   boolean indicating if the `deeptabular` component will be fine tuned gradually\n                - `deeptabular_layers` (`Optional[Union[List[nn.Module], List[List[nn.Module]]]]`):\n                   List of pytorch modules indicating the layers of the\n                   `deeptabular` that will be fine tuned\n                - `deeptabular_max_lr` (`Union[float, List[float]]`):\n                   max lr for the `deeptabular` componet during fine tuning\n                - `deeptext_gradual` (`bool`):\n                   same as `deeptabular_gradual` but for the `deeptext` component\n                - `deeptext_layers` (`Optional[Union[List[nn.Module], List[List[nn.Module]]]]`):\n                   same as `deeptabular_gradual` but for the `deeptext` component.\n                   If there are multiple text columns/models, this should be a list of lists\n                - `deeptext_max_lr` (`Union[float, List[float]]`):\n                   same as `deeptabular_gradual` but for the `deeptext` component\n                   If there are multiple text columns/models, this should be a list of floats\n                - `deepimage_gradual` (`bool`):\n                   same as `deeptext_layers` but for the `deepimage` component\n                - `deepimage_layers` (`Optional[Union[List[nn.Module], List[List[nn.Module]]]]`):\n                   same as `deeptext_layers` but for the `deepimage` component\n                - `deepimage_max_lr` (`Union[float, List[float]]`):\n                    same as `deeptext_layers` but for the `deepimage` component\n\n        Examples\n        --------\n\n        For a series of comprehensive examples on how to use the `fit` method, please see the\n        [Examples](https://github.com/jrzaurin/pytorch-widedeep/tree/master/examples)\n        folder in the repo\n        \"\"\"\n        dataloader_args, finetune_args = self._extract_kwargs(kwargs)\n        self.batch_size = batch_size\n        train_set, eval_set = wd_train_val_split(\n            self.seed,\n            self.method,  # type: ignore\n            X_wide,\n            X_tab,\n            X_text,\n            X_img,\n            X_train,\n            X_val,\n            val_split,\n            target,\n            self.transforms,\n        )\n        train_loader = self._set_dataloader(\n            train_dataloader, train_set, batch_size, dataloader_args\n        )\n        eval_loader = (\n            self._set_dataloader(eval_dataloader, eval_set, batch_size, dataloader_args)\n            if eval_set is not None\n            else None\n        )\n\n        if finetune:\n            self.with_finetuning: bool = True\n            self._do_finetune(train_loader, **finetune_args)\n            if self.verbose:\n                print(\n                    \"Fine-tuning (or warmup) of individual components completed. \"\n                    \"Training the whole model for {} epochs\".format(n_epochs)\n                )\n        else:\n            self.with_finetuning = False\n\n        self.callback_container.on_train_begin(\n            {\n                \"batch_size\": batch_size,\n                \"train_steps\": len(train_loader),\n                \"n_epochs\": n_epochs,\n            }\n        )\n        for epoch in range(n_epochs):\n            epoch_logs = self._train_epoch(train_loader, epoch)\n            if eval_loader is not None and epoch % validation_freq == (\n                validation_freq - 1\n            ):\n                epoch_logs, on_epoch_end_metric = self._eval_epoch(\n                    eval_loader, epoch_logs\n                )\n            else:\n                on_epoch_end_metric = None\n                if self.reducelronplateau:\n                    raise NotImplementedError(\n                        \"ReduceLROnPlateau scheduler can be used only with validation data.\"\n                    )\n            self.callback_container.on_epoch_end(epoch, epoch_logs, on_epoch_end_metric)\n            if self.early_stop:\n                # self.callback_container.on_train_end(epoch_logs)\n                break\n\n        self.callback_container.on_train_end(epoch_logs)\n        if feature_importance_sample_size is not None:\n            self.feature_importance = FeatureImportance(\n                self.device, feature_importance_sample_size\n            ).feature_importance(train_loader, self.model)\n        self._restore_best_weights()\n        self.model.train()\n\n    def predict(  # type: ignore[override, return]\n        self,\n        X_wide: Optional[np.ndarray] = None,\n        X_tab: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n        X_text: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n        X_img: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n        X_test: Optional[Dict[str, Union[np.ndarray, List[np.ndarray]]]] = None,\n        batch_size: Optional[int] = None,\n    ) -&gt; np.ndarray:\n        r\"\"\"Returns the predictions\n\n        The input datasets can be passed either directly via numpy arrays\n        (`X_wide`, `X_tab`, `X_text` or `X_img`) or alternatively, in\n        a dictionary (`X_test`)\n\n\n        Parameters\n        ----------\n        X_wide: np.ndarray, Optional. default=None\n            Input for the `wide` model component.\n            See `pytorch_widedeep.preprocessing.WidePreprocessor`\n        X_tab: np.ndarray or List[np.ndarray], Optional. default=None\n            Input for the `deeptabular` model component.\n            See `pytorch_widedeep.preprocessing.TabPreprocessor`\n        X_text: np.ndarray or List[np.ndarray], Optional. default=None\n            Input for the `deeptext` model component.\n            See `pytorch_widedeep.preprocessing.TextPreprocessor`\n        X_img: np.ndarray or List[np.ndarray], Optional. default=None\n            Input for the `deepimage` model component.\n            See `pytorch_widedeep.preprocessing.ImagePreprocessor`\n        X_test: Dict, Optional. default=None\n            The test dataset can also be passed in a dictionary. Keys are\n            `X_wide`, _'X_tab'_, _'X_text'_, _'X_img'_ and _'target'_. Values\n            are the corresponding matrices.\n        batch_size: int, default = 256\n            If a trainer is used to predict after having trained a model, the\n            `batch_size` needs to be defined as it will not be defined as\n            the `Trainer` is instantiated\n\n        Returns\n        -------\n        np.ndarray:\n            array with the predictions\n        \"\"\"\n        preds_l = self._predict(X_wide, X_tab, X_text, X_img, X_test, batch_size)\n        if self.method == \"regression\":\n            return np.vstack(preds_l).squeeze(1)\n        if self.method == \"binary\":\n            preds = np.vstack(preds_l).squeeze(1)\n            return (preds &gt; 0.5).astype(\"int\")\n        if self.method == \"qregression\":\n            return np.vstack(preds_l)\n        if self.method == \"multiclass\":\n            preds = np.vstack(preds_l)\n            return np.argmax(preds, 1)  # type: ignore[return-value]\n        if self.method == \"multitarget\":\n            if self.loss_fn.__class__.__name__ in [\n                \"MultiTargetClassificationLoss\",\n                \"MutilTargetRegressionAndClassificationLoss\",\n            ]:\n                raise ValueError(\n                    \"MultiTargetClassificationLoss and MutilTargetRegressionAndClassificationLoss \"\n                    \"are not supported by predict method. Please use predict_proba method instead.\"\n                )\n            return np.vstack(preds_l)\n\n    def predict_uncertainty(  # type: ignore[return]\n        self,\n        X_wide: Optional[np.ndarray] = None,\n        X_tab: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n        X_text: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n        X_img: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n        X_test: Optional[Dict[str, Union[np.ndarray, List[np.ndarray]]]] = None,\n        batch_size: Optional[int] = None,\n        uncertainty_granularity: int = 1000,\n    ) -&gt; np.ndarray:\n        r\"\"\"Returns the predicted ucnertainty of the model for the test dataset\n        using a Monte Carlo method during which dropout layers are activated\n        in the evaluation/prediction phase and each sample is predicted N\n        times (`uncertainty_granularity` times).\n\n        This is based on\n        [Dropout as a Bayesian Approximation: Representing\n        Model Uncertainty in Deep Learning](https://arxiv.org/abs/1506.02142?context=stat).\n\n        Parameters\n        ----------\n        X_wide: np.ndarray, Optional. default=None\n            Input for the `wide` model component.\n            See `pytorch_widedeep.preprocessing.WidePreprocessor`\n        X_tab: np.ndarray or List[np.ndarray], Optional. default=None\n            Input for the `deeptabular` model component.\n            See `pytorch_widedeep.preprocessing.TabPreprocessor`\n        X_text: np.ndarray or List[np.ndarray], Optional. default=None\n            Input for the `deeptext` model component.\n            See `pytorch_widedeep.preprocessing.TextPreprocessor`\n        X_img: np.ndarray or List[np.ndarray], Optional. default=None\n            Input for the `deepimage` model component.\n            See `pytorch_widedeep.preprocessing.ImagePreprocessor`\n        X_test: Dict, Optional. default=None\n            The test dataset can also be passed in a dictionary. Keys are\n            _'X_wide'_, _'X_tab'_, _'X_text'_, _'X_img'_ and _'target'_. Values\n            are the corresponding matrices.\n        batch_size: int, default = 256\n            If a trainer is used to predict after having trained a model, the\n            `batch_size` needs to be defined as it will not be defined as\n            the `Trainer` is instantiated\n        uncertainty_granularity: int default = 1000\n            number of times the model does prediction for each sample\n\n        Returns\n        -------\n        np.ndarray:\n            - if `method = regression`, it will return an array with `(max, min, mean, stdev)`\n              values for each sample.\n            - if `method = binary` it will return an array with\n              `(mean_cls_0_prob, mean_cls_1_prob, predicted_cls)` for each sample.\n            - if `method = multiclass` it will return an array with\n              `(mean_cls_0_prob, mean_cls_1_prob, mean_cls_2_prob, ... , predicted_cls)`\n              values for each sample.\n\n        \"\"\"\n        preds_l = self._predict(\n            X_wide,\n            X_tab,\n            X_text,\n            X_img,\n            X_test,\n            batch_size,\n            uncertainty_granularity,\n            uncertainty=True,\n        )\n        preds = np.vstack(preds_l)\n        samples_num = int(preds.shape[0] / uncertainty_granularity)\n        if self.method == \"regression\":\n            preds = preds.squeeze(1)\n            preds = preds.reshape((uncertainty_granularity, samples_num))\n            return np.array(\n                (\n                    preds.max(axis=0),\n                    preds.min(axis=0),\n                    preds.mean(axis=0),\n                    preds.std(axis=0),\n                )\n            ).T\n        if self.method == \"qregression\":\n            raise ValueError(\n                \"Currently predict_uncertainty is not supported for qregression method\"\n            )\n        if self.method == \"binary\":\n            preds = preds.squeeze(1)\n            preds = preds.reshape((uncertainty_granularity, samples_num))\n            preds = preds.mean(axis=0)\n            probs = np.zeros([preds.shape[0], 3])\n            probs[:, 0] = 1 - preds\n            probs[:, 1] = preds\n            return probs\n        if self.method == \"multiclass\":\n            preds = preds.reshape(uncertainty_granularity, samples_num, preds.shape[1])\n            preds = preds.mean(axis=0)\n            preds = np.hstack((preds, np.vstack(np.argmax(preds, 1))))\n            return preds\n\n        if self.method == \"multitarget\":\n            raise ValueError(\n                \"Currently predict_uncertainty is not supported for multitarget method\"\n            )\n\n    def predict_proba(  # type: ignore[override, return]  # noqa: C901\n        self,\n        X_wide: Optional[np.ndarray] = None,\n        X_tab: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n        X_text: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n        X_img: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n        X_test: Optional[Dict[str, Union[np.ndarray, List[np.ndarray]]]] = None,\n        batch_size: Optional[int] = None,\n    ) -&gt; np.ndarray:\n        r\"\"\"Returns the predicted probabilities for the test dataset for  binary\n        and multiclass methods\n\n        The input datasets can be passed either directly via numpy arrays\n        (`X_wide`, `X_tab`, `X_text` or `X_img`) or alternatively, in\n        a dictionary (`X_test`)\n\n        Parameters\n        ----------\n        X_wide: np.ndarray, Optional. default=None\n            Input for the `wide` model component.\n            See `pytorch_widedeep.preprocessing.WidePreprocessor`\n        X_tab: np.ndarray or List[np.ndarray], Optional. default=None\n            Input for the `deeptabular` model component.\n            See `pytorch_widedeep.preprocessing.TabPreprocessor`\n        X_text: np.ndarray or List[np.ndarray], Optional. default=None\n            Input for the `deeptext` model component.\n            See `pytorch_widedeep.preprocessing.TextPreprocessor`\n        X_img: np.ndarray or List[np.ndarray], Optional. default=None\n            Input for the `deepimage` model component.\n            See `pytorch_widedeep.preprocessing.ImagePreprocessor`\n        X_test: Dict, Optional. default=None\n            The test dataset can also be passed in a dictionary. Keys are\n            `X_wide`, _'X_tab'_, _'X_text'_, _'X_img'_ and _'target'_. Values\n            are the corresponding matrices.\n        batch_size: int, default = 256\n            If a trainer is used to predict after having trained a model, the\n            `batch_size` needs to be defined as it will not be defined as\n            the `Trainer` is instantiated\n\n        Returns\n        -------\n        np.ndarray\n            array with the probabilities per class\n        \"\"\"\n\n        preds_l = self._predict(X_wide, X_tab, X_text, X_img, X_test, batch_size)\n        if self.method == \"binary\":\n            preds = np.vstack(preds_l).squeeze(1)\n            probs = np.zeros([preds.shape[0], 2])\n            probs[:, 0] = 1 - preds\n            probs[:, 1] = preds\n            return probs\n        if self.method == \"multiclass\":\n            return np.vstack(preds_l)\n        if self.method == \"multitarget\":\n            return np.vstack(preds_l)\n\n    def explain(self, X_tab: np.ndarray, save_step_masks: Optional[bool] = None):\n        # TO DO: Add docs to this, to the feat imp parameter and the all\n        # related classes\n        explainer = Explainer(self.device)\n\n        res = explainer.explain(\n            self.model, X_tab, self.num_workers, self.batch_size, save_step_masks\n        )\n\n        return res\n\n    def save(\n        self,\n        path: str,\n        save_state_dict: bool = False,\n        save_optimizer: bool = False,\n        model_filename: str = \"wd_model.pt\",\n    ):\n        r\"\"\"Saves the model, training and evaluation history, and the\n        `feature_importance` attribute (if the `deeptabular` component is a\n        Tabnet model) to disk\n\n        The `Trainer` class is built so that it 'just' trains a model. With\n        that in mind, all the torch related parameters (such as optimizers,\n        learning rate schedulers, initializers, etc) have to be defined\n        externally and then passed to the `Trainer`. As a result, the\n        `Trainer` does not generate any attribute or additional data\n        products that need to be saved other than the `model` object itself,\n        which can be saved as any other torch model (e.g. `torch.save(model,\n        path)`).\n\n        The exception is Tabnet. If the `deeptabular` component is a Tabnet\n        model, an attribute (a dict) called `feature_importance` will be\n        created at the end of the training process. Therefore, a `save`\n        method was created that will save the feature importance dictionary\n        to a json file and, since we are here, the model weights, training\n        history and learning rate history.\n\n        Parameters\n        ----------\n        path: str\n            path to the directory where the model and the feature importance\n            attribute will be saved.\n        save_state_dict: bool, default = False\n            Boolean indicating whether to save directly the model\n            (and optimizer) or the model's (and optimizer's) state\n            dictionary\n        save_optimizer: bool, default = False\n            Boolean indicating whether to save the optimizer\n        model_filename: str, Optional, default = \"wd_model.pt\"\n            filename where the model weights will be store\n        \"\"\"\n\n        self._save_history(path)\n\n        self._save_model_and_optimizer(\n            path, save_state_dict, save_optimizer, model_filename\n        )\n\n        if self.model.is_tabnet:\n            with open(Path(path) / \"feature_importance.json\", \"w\") as fi:\n                json.dump(self.feature_importance, fi)\n\n    def _set_dataloader(\n        self,\n        dataloader: Optional[CustomDataLoader],\n        dataset: WideDeepDataset,\n        batch_size: int,\n        dataloader_args: Dict[str, Any],\n    ) -&gt; Union[DataLoader, CustomDataLoader]:\n        if dataloader is not None:\n            dataloader.set_dataset(dataset)\n            return dataloader\n        else:\n            # var name 'loader' to avoid reassigment and type errors\n            loader = DataLoader(\n                dataset=dataset,\n                batch_size=batch_size,\n                num_workers=self.num_workers,\n                **dataloader_args,\n            )\n        return loader\n\n    @alias(\"n_epochs\", [\"finetune_epochs\", \"warmup_epochs\"])\n    @alias(\"max_lr\", [\"finetune_max_lr\", \"warmup_max_lr\"])\n    def _do_finetune(\n        self,\n        loader: DataLoader,\n        n_epochs: int = 5,\n        max_lr: float = 0.01,\n        routine: Literal[\"howard\", \"felbo\"] = \"howard\",\n        deeptabular_gradual: bool = False,\n        deeptabular_layers: Optional[\n            Union[List[nn.Module], List[List[nn.Module]]]\n        ] = None,\n        deeptabular_max_lr: Union[float, List[float]] = 0.01,\n        deeptext_gradual: bool = False,\n        deeptext_layers: Optional[Union[List[nn.Module], List[List[nn.Module]]]] = None,\n        deeptext_max_lr: Union[float, List[float]] = 0.01,\n        deepimage_gradual: bool = False,\n        deepimage_layers: Optional[\n            Union[List[nn.Module], List[List[nn.Module]]]\n        ] = None,\n        deepimage_max_lr: Union[float, List[float]] = 0.01,\n    ):\n        r\"\"\"\n        Simple wrap-up to individually fine-tune model components\n        \"\"\"\n        if self.model.deephead is not None:\n            raise ValueError(\n                \"Currently warming up is only supported without a fully connected 'DeepHead'\"\n            )\n\n        finetuner = FineTune(self.loss_fn, self.metric, self.method, self.verbose)  # type: ignore[arg-type]\n        if self.model.wide:\n            finetuner.finetune_all(self.model.wide, \"wide\", loader, n_epochs, max_lr)\n\n        if self.model.deeptabular:\n            if deeptabular_gradual:\n                assert (\n                    deeptabular_layers is not None\n                ), \"deeptabular_layers must be passed if deeptabular_gradual=True\"\n                finetuner.finetune_gradual(\n                    self.model.deeptabular,\n                    \"deeptabular\",\n                    loader,\n                    deeptabular_max_lr,\n                    deeptabular_layers,\n                    routine,\n                )\n            else:\n                finetuner.finetune_all(\n                    self.model.deeptabular, \"deeptabular\", loader, n_epochs, max_lr\n                )\n\n        if self.model.deeptext:\n            if deeptext_gradual:\n                assert (\n                    deeptext_layers is not None\n                ), \"deeptext_layers must be passed if deeptext_gradual=True\"\n                finetuner.finetune_gradual(\n                    self.model.deeptext,\n                    \"deeptext\",\n                    loader,\n                    deeptext_max_lr,\n                    deeptext_layers,\n                    routine,\n                )\n            else:\n                finetuner.finetune_all(\n                    self.model.deeptext, \"deeptext\", loader, n_epochs, max_lr\n                )\n\n        if self.model.deepimage:\n            if deepimage_gradual:\n                assert (\n                    deepimage_layers is not None\n                ), \"deepimage_layers must be passed if deepimage_gradual=True\"\n                finetuner.finetune_gradual(\n                    self.model.deepimage,\n                    \"deepimage\",\n                    loader,\n                    deepimage_max_lr,\n                    deepimage_layers,\n                    routine,\n                )\n            else:\n                finetuner.finetune_all(\n                    self.model.deepimage, \"deepimage\", loader, n_epochs, max_lr\n                )\n\n    def _train_epoch(\n        self,\n        train_loader: DataLoader,\n        epoch: int,\n    ):\n        epoch_logs: Dict[str, float] = {}\n        self.callback_container.on_epoch_begin(epoch, logs=epoch_logs)\n        self.train_running_loss = 0.0\n\n        train_steps = len(train_loader)\n        with trange(train_steps, disable=self.verbose != 1) as t:\n            for batch_idx, (data, targett) in zip(t, train_loader):\n                t.set_description(\"epoch %i\" % (epoch + 1))\n                train_score, train_loss = self._train_step(data, targett, batch_idx)\n                print_loss_and_metric(t, train_loss, train_score)\n                self.callback_container.on_batch_end(batch=batch_idx)\n\n        epoch_logs = save_epoch_logs(epoch_logs, train_loss, train_score, \"train\")\n\n        return epoch_logs\n\n    def _train_step(\n        self,\n        data: Dict[str, Union[Tensor, List[Tensor]]],\n        target: Tensor,\n        batch_idx: int,\n    ):\n\n        self.model.train()\n\n        X: Dict[str, Union[Tensor, List[Tensor]]] = {}\n        for k, v in data.items():\n            if isinstance(v, list):\n                X[k] = [to_device(i, self.device) for i in v]\n            else:\n                X[k] = to_device(v, self.device)\n        y = (\n            target.view(-1, 1).float()\n            if self.method not in [\"multiclass\", \"qregression\", \"multitarget\"]\n            else target\n        )\n        y = to_device(y, self.device)\n\n        self.optimizer.zero_grad()\n\n        y_pred = self.model(X)\n\n        if self.model.is_tabnet:\n            loss = self.loss_fn(y_pred[0], y) - self.lambda_sparse * y_pred[1]\n            score = self._get_score(y_pred[0], y, is_train=True)\n        else:\n            loss = self.loss_fn(y_pred, y)\n            score = self._get_score(y_pred, y, is_train=True)\n\n        loss.backward()\n        self.optimizer.step()\n\n        self.train_running_loss += loss.item()\n        avg_loss = self.train_running_loss / (batch_idx + 1)\n\n        return score, avg_loss\n\n    def _eval_epoch(\n        self,\n        eval_loader: DataLoader,\n        epoch_logs: Dict[str, float],\n    ) -&gt; Tuple[Dict[str, float], Optional[float]]:\n        self.callback_container.on_eval_begin()\n        self.valid_running_loss = 0.0\n\n        eval_steps = len(eval_loader)\n        with trange(eval_steps, disable=self.verbose != 1) as v:\n            for i, (data, targett) in zip(v, eval_loader):\n                v.set_description(\"valid\")\n                val_score, val_loss = self._eval_step(data, targett, i)\n                print_loss_and_metric(v, val_loss, val_score)\n\n        epoch_logs = save_epoch_logs(epoch_logs, val_loss, val_score, \"val\")\n\n        if self.reducelronplateau:\n            if self.reducelronplateau_criterion == \"loss\":\n                on_epoch_end_metric = val_loss\n            else:\n                on_epoch_end_metric = val_score[self.reducelronplateau_criterion]\n        else:\n            on_epoch_end_metric = None\n\n        return epoch_logs, on_epoch_end_metric\n\n    def _eval_step(\n        self,\n        data: Dict[str, Union[Tensor, List[Tensor]]],\n        target: Tensor,\n        batch_idx: int,\n    ):\n        self.model.eval()\n        with torch.no_grad():\n            X: Dict[str, Union[Tensor, List[Tensor]]] = {}\n            for k, v in data.items():\n                if isinstance(v, list):\n                    X[k] = [to_device(i, self.device) for i in v]\n                else:\n                    X[k] = to_device(v, self.device)\n            y = (\n                target.view(-1, 1).float()\n                if self.method not in [\"multiclass\", \"qregression\", \"multitarget\"]\n                else target\n            )\n            y = to_device(y, self.device)\n\n            y_pred = self.model(X)\n            if self.model.is_tabnet:\n                loss = self.loss_fn(y_pred[0], y) - self.lambda_sparse * y_pred[1]\n                score = self._get_score(y_pred[0], y, is_train=False)\n            else:\n                score = self._get_score(y_pred, y, is_train=False)\n                loss = self.loss_fn(y_pred, y)\n\n            self.valid_running_loss += loss.item()\n            avg_loss = self.valid_running_loss / (batch_idx + 1)\n\n        self.model.train()\n        return score, avg_loss\n\n    def _get_score(\n        self, y_pred: Tensor, y: Tensor, is_train: bool\n    ) -&gt; Optional[Dict[str, float]]:\n\n        score = None\n        metric = None\n\n        if self.metric and not self.eval_metric:\n            metric = self.metric\n        elif self.metric and self.eval_metric:\n            metric = self.metric if is_train else self.eval_metric\n        elif not self.metric and self.eval_metric:\n            metric = None if is_train else self.eval_metric\n\n        if metric is not None:\n            if self.method == \"regression\":\n                score = metric(y_pred, y)\n            if self.method == \"binary\":\n                score = metric(torch.sigmoid(y_pred), y)\n            if self.method == \"qregression\":\n                score = metric(y_pred, y)\n            if self.method == \"multiclass\":\n                score = metric(F.softmax(y_pred, dim=1), y)\n        return score\n\n    def _predict(  # type: ignore[override, return]  # noqa: C901\n        self,\n        X_wide: Optional[np.ndarray] = None,\n        X_tab: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n        X_text: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n        X_img: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n        X_test: Optional[Dict[str, Union[np.ndarray, List[np.ndarray]]]] = None,\n        batch_size: Optional[int] = None,\n        uncertainty_granularity: int = 1000,\n        uncertainty: bool = False,\n    ) -&gt; List:\n        r\"\"\"Private method to avoid code repetition in predict and\n        predict_proba. For parameter information, please, see the .predict()\n        method documentation\n        \"\"\"\n        if X_test is not None:\n            test_set = WideDeepDataset(**X_test)  # type: ignore[arg-type]\n        else:\n            load_dict: Dict[str, Union[np.ndarray, List[np.ndarray]]] = {}\n            if X_wide is not None:\n                load_dict = {\"X_wide\": X_wide}\n            if X_tab is not None:\n                load_dict.update({\"X_tab\": X_tab})\n            if X_text is not None:\n                load_dict.update({\"X_text\": X_text})\n            if X_img is not None:\n                load_dict.update({\"X_img\": X_img})\n            test_set = WideDeepDataset(**load_dict)  # type: ignore[arg-type]\n\n        if not hasattr(self, \"batch_size\"):\n            assert batch_size is not None, (\n                \"'batch_size' has not be previosly set in this Trainer and must be\"\n                \" specified via the 'batch_size' argument in this predict call\"\n            )\n            self.batch_size = batch_size\n\n        test_loader = DataLoader(\n            dataset=test_set,\n            batch_size=self.batch_size,\n            num_workers=self.num_workers,\n            shuffle=False,\n        )\n        test_steps = (len(test_loader.dataset) // test_loader.batch_size) + 1  # type: ignore[arg-type]\n\n        self.model.eval()\n        preds_l = []\n\n        if uncertainty:\n            for m in self.model.modules():\n                if m.__class__.__name__.startswith(\"Dropout\"):\n                    m.train()\n            prediction_iters = uncertainty_granularity\n        else:\n            prediction_iters = 1\n\n        with torch.no_grad():\n            with trange(uncertainty_granularity, disable=uncertainty is False) as t:\n                for _, _ in zip(t, range(prediction_iters)):\n                    t.set_description(\"predict_UncertaintyIter\")\n\n                    with trange(\n                        test_steps, disable=self.verbose != 1 or uncertainty is True\n                    ) as tt:\n                        for _, data in zip(tt, test_loader):\n                            tt.set_description(\"predict\")\n                            X: Dict[str, Union[Tensor, List[Tensor]]] = {}\n                            for k, v in data.items():\n                                if isinstance(v, list):\n                                    X[k] = [to_device(i, self.device) for i in v]\n                                else:\n                                    X[k] = to_device(v, self.device)\n                            preds = (\n                                self.model(X)\n                                if not self.model.is_tabnet\n                                else self.model(X)[0]\n                            )\n                            if self.method == \"binary\":\n                                preds = torch.sigmoid(preds)\n                            if self.method == \"multiclass\":\n                                preds = F.softmax(preds, dim=1)\n                            if self.method == \"regression\" and isinstance(\n                                self.loss_fn, ZILNLoss\n                            ):\n                                preds = self._predict_ziln(preds)\n                            preds = preds.cpu().data.numpy()\n                            preds_l.append(preds)\n        self.model.train()\n        return preds_l\n\n    @staticmethod\n    def _predict_ziln(preds: Tensor) -&gt; Tensor:\n        # Legacy implementation. It will be removed in future versions\n        \"\"\"Calculates predicted mean of zero inflated lognormal logits.\n\n        Adjusted implementaion of `code\n        &lt;https://github.com/google/lifetime_value/blob/master/lifetime_value/zero_inflated_lognormal.py&gt;`\n\n        Arguments:\n            preds: [batch_size, 3] tensor of logits.\n        Returns:\n            ziln_preds: [batch_size, 1] tensor of predicted mean.\n        \"\"\"\n        positive_probs = torch.sigmoid(preds[..., :1])\n        loc = preds[..., 1:2]\n        scale = F.softplus(preds[..., 2:])\n        ziln_preds = positive_probs * torch.exp(loc + 0.5 * torch.square(scale))\n        return ziln_preds\n\n    @staticmethod\n    def _extract_kwargs(kwargs):\n        dataloader_params = [\n            \"shuffle\",\n            \"sampler\",\n            \"batch_sampler\",\n            \"num_workers\",\n            \"collate_fn\",\n            \"pin_memory\",\n            \"drop_last\",\n            \"timeout\",\n            \"worker_init_fn\",\n            \"generator\",\n            \"prefetch_factor\",\n            \"persistent_workers\",\n            \"oversample_mul\",\n        ]\n        finetune_params = [\n            \"n_epochs\",\n            \"finetune_epochs\",\n            \"warmup_epochs\",\n            \"max_lr\",\n            \"finetune_max_lr\",\n            \"warmup_max_lr\",\n            \"routine\",\n            \"deeptabular_gradual\",\n            \"deeptabular_layers\",\n            \"deeptabular_max_lr\",\n            \"deeptext_gradual\",\n            \"deeptext_layers\",\n            \"deeptext_max_lr\",\n            \"deepimage_gradual\",\n            \"deepimage_layers\",\n            \"deepimage_max_lr\",\n        ]\n\n        dataloader_args, finetune_args = {}, {}\n        for k, v in kwargs.items():\n            if k in dataloader_params:\n                dataloader_args[k] = v\n            if k in finetune_params:\n                finetune_args[k] = v\n\n        return dataloader_args, finetune_args\n</code></pre>"},{"location":"pytorch-widedeep/trainer.html#pytorch_widedeep.training.Trainer.fit","title":"fit","text":"<pre><code>fit(\n    X_wide=None,\n    X_tab=None,\n    X_text=None,\n    X_img=None,\n    X_train=None,\n    X_val=None,\n    val_split=None,\n    target=None,\n    n_epochs=1,\n    validation_freq=1,\n    batch_size=32,\n    train_dataloader=None,\n    eval_dataloader=None,\n    feature_importance_sample_size=None,\n    finetune=False,\n    **kwargs\n)\n</code></pre> <p>Fit method.</p> <p>The input datasets can be passed either directly via numpy arrays (<code>X_wide</code>, <code>X_tab</code>, <code>X_text</code> or <code>X_img</code>) or alternatively, in dictionaries (<code>X_train</code> or <code>X_val</code>).</p> <p>Parameters:</p> Name Type Description Default <code>X_wide</code> <code>Optional[ndarray]</code> <p>Input for the <code>wide</code> model component. See <code>pytorch_widedeep.preprocessing.WidePreprocessor</code></p> <code>None</code> <code>X_tab</code> <code>Optional[Union[ndarray, List[ndarray]]]</code> <p>Input for the <code>deeptabular</code> model component. See <code>pytorch_widedeep.preprocessing.TabPreprocessor</code>. If multiple tabular models are used for different columns, this should be a list of numpy arrays</p> <code>None</code> <code>X_text</code> <code>Optional[Union[ndarray, List[ndarray]]]</code> <p>Input for the <code>deeptext</code> model component. See <code>pytorch_widedeep.preprocessing.TextPreprocessor</code>. If multiple text columns/models are used, this should be a list of numpy arrays</p> <code>None</code> <code>X_img</code> <code>Optional[Union[ndarray, List[ndarray]]]</code> <p>Input for the <code>deepimage</code> model component. See <code>pytorch_widedeep.preprocessing.ImagePreprocessor</code>. If multiple image columns/models are used, this should be a list of numpy arrays</p> <code>None</code> <code>X_train</code> <code>Optional[Dict[str, Union[ndarray, List[ndarray]]]]</code> <p>The training dataset can also be passed in a dictionary. Keys are 'X_wide', 'X_tab', 'X_text', 'X_img' and 'target'. Values are the corresponding matrices. Note that of multiple text or image columns/models are used, the corresponding values should be lists of numpy arrays</p> <code>None</code> <code>X_val</code> <code>Optional[Dict[str, Union[ndarray, List[ndarray]]]]</code> <p>The validation dataset can also be passed in a dictionary. Keys are 'X_wide', 'X_tab', 'X_text', 'X_img' and 'target'. Values are the corresponding matrices. Note that of multiple text or image columns/models are used, the corresponding values should be lists of numpy arrays</p> <code>None</code> <code>val_split</code> <code>Optional[float]</code> <p>train/val split fraction</p> <code>None</code> <code>target</code> <code>Optional[ndarray]</code> <p>target values</p> <code>None</code> <code>n_epochs</code> <code>int</code> <p>number of epochs</p> <code>1</code> <code>validation_freq</code> <code>int</code> <p>epochs validation frequency</p> <code>1</code> <code>batch_size</code> <code>int</code> <p>batch size</p> <code>32</code> <code>custom_dataloader</code> <p>object of class <code>torch.utils.data.DataLoader</code>. Available predefined dataloaders are in <code>pytorch-widedeep.dataloaders</code>.If <code>None</code>, a standard torch <code>DataLoader</code> is used.</p> required <code>finetune</code> <code>bool</code> <p>fine-tune individual model components. This functionality can also be used to 'warm-up' (and hence the alias <code>warmup</code>) individual components before the joined training starts, and hence its alias. See the Examples folder in the repo for more details</p> <p><code>pytorch_widedeep</code> implements 3 fine-tune routines.</p> <ul> <li>fine-tune all trainable layers at once. This routine is   inspired by the work of Howard &amp; Sebastian Ruder 2018 in their   ULMfit paper. Using a   Slanted Triangular learing (see   Leslie N. Smith paper ) ,   the process is the following: i) the learning rate will   gradually increase for 10% of the training steps from max_lr/10   to max_lr. ii) It will then gradually decrease to max_lr/10   for the remaining 90% of the steps. The optimizer used in the   process is <code>Adam</code>.</li> </ul> <p>and two gradual fine-tune routines, where only certain layers are trained at a time.</p> <ul> <li>The so called <code>Felbo</code> gradual fine-tune rourine, based on the the   Felbo et al., 2017 DeepEmoji paper.</li> <li>The <code>Howard</code> routine based on the work of Howard &amp; Sebastian Ruder 2018 in their   ULMfit paper.</li> </ul> <p>For details on how these routines work, please see the Examples section in this documentation and the Examples folder in the repo.  Param Alias: <code>warmup</code></p> <code>False</code> <p>Other Parameters:</p> Name Type Description <code>**kwargs</code> <p>Other keyword arguments are:</p> <ul> <li> <p>DataLoader related parameters:     For example,  <code>sampler</code>, <code>batch_sampler</code>, <code>collate_fn</code>, etc.     Please, see the pytorch     DataLoader docs     for details.</p> </li> <li> <p>Finetune related parameters:     see the source code at <code>pytorch_widedeep._finetune</code>. Namely, these are:</p> <ul> <li><code>finetune_epochs</code> (<code>int</code>):     number of epochs use for fine tuning</li> <li><code>finetune_max_lr</code> (<code>float</code>):    max lr during fine tuning</li> <li><code>routine</code> (<code>str</code>):    one of 'howard' or 'felbo'</li> <li><code>deeptabular_gradual</code> (<code>bool</code>):    boolean indicating if the <code>deeptabular</code> component will be fine tuned gradually</li> <li><code>deeptabular_layers</code> (<code>Optional[Union[List[nn.Module], List[List[nn.Module]]]]</code>):    List of pytorch modules indicating the layers of the    <code>deeptabular</code> that will be fine tuned</li> <li><code>deeptabular_max_lr</code> (<code>Union[float, List[float]]</code>):    max lr for the <code>deeptabular</code> componet during fine tuning</li> <li><code>deeptext_gradual</code> (<code>bool</code>):    same as <code>deeptabular_gradual</code> but for the <code>deeptext</code> component</li> <li><code>deeptext_layers</code> (<code>Optional[Union[List[nn.Module], List[List[nn.Module]]]]</code>):    same as <code>deeptabular_gradual</code> but for the <code>deeptext</code> component.    If there are multiple text columns/models, this should be a list of lists</li> <li><code>deeptext_max_lr</code> (<code>Union[float, List[float]]</code>):    same as <code>deeptabular_gradual</code> but for the <code>deeptext</code> component    If there are multiple text columns/models, this should be a list of floats</li> <li><code>deepimage_gradual</code> (<code>bool</code>):    same as <code>deeptext_layers</code> but for the <code>deepimage</code> component</li> <li><code>deepimage_layers</code> (<code>Optional[Union[List[nn.Module], List[List[nn.Module]]]]</code>):    same as <code>deeptext_layers</code> but for the <code>deepimage</code> component</li> <li><code>deepimage_max_lr</code> (<code>Union[float, List[float]]</code>):     same as <code>deeptext_layers</code> but for the <code>deepimage</code> component</li> </ul> </li> </ul> <p>Examples:</p> <p>For a series of comprehensive examples on how to use the <code>fit</code> method, please see the Examples folder in the repo</p> Source code in <code>pytorch_widedeep/training/trainer.py</code> <pre><code>@alias(\"finetune\", [\"warmup\"])\n@alias(\"train_dataloader\", [\"custom_dataloader\"])\n@alias(\"eval_dataloader\", [\"custom_eval_dataloader\"])\ndef fit(  # noqa: C901\n    self,\n    X_wide: Optional[np.ndarray] = None,\n    X_tab: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n    X_text: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n    X_img: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n    X_train: Optional[Dict[str, Union[np.ndarray, List[np.ndarray]]]] = None,\n    X_val: Optional[Dict[str, Union[np.ndarray, List[np.ndarray]]]] = None,\n    val_split: Optional[float] = None,\n    target: Optional[np.ndarray] = None,\n    n_epochs: int = 1,\n    validation_freq: int = 1,\n    batch_size: int = 32,\n    train_dataloader: Optional[CustomDataLoader] = None,\n    eval_dataloader: Optional[CustomDataLoader] = None,\n    feature_importance_sample_size: Optional[int] = None,\n    finetune: bool = False,\n    **kwargs,\n):\n    r\"\"\"Fit method.\n\n    The input datasets can be passed either directly via numpy arrays\n    (`X_wide`, `X_tab`, `X_text` or `X_img`) or alternatively, in\n    dictionaries (`X_train` or `X_val`).\n\n    Parameters\n    ----------\n    X_wide: np.ndarray, Optional. default=None\n        Input for the `wide` model component.\n        See `pytorch_widedeep.preprocessing.WidePreprocessor`\n    X_tab: np.ndarray, Optional. default=None\n        Input for the `deeptabular` model component.\n        See `pytorch_widedeep.preprocessing.TabPreprocessor`. If multiple\n        tabular models are used for different columns, this should be a\n        list of numpy arrays\n    X_text: Union[np.ndarray, List[np.ndarray]], Optional. default=None\n        Input for the `deeptext` model component.\n        See `pytorch_widedeep.preprocessing.TextPreprocessor`.\n        If multiple text columns/models are used, this should be a list of\n        numpy arrays\n    X_img: np.ndarray, Optional. default=None\n        Input for the `deepimage` model component.\n        See `pytorch_widedeep.preprocessing.ImagePreprocessor`.\n        If multiple image columns/models are used, this should be a list of\n        numpy arrays\n    X_train: Dict, Optional. default=None\n        The training dataset can also be passed in a dictionary. Keys are\n        _'X_wide'_, _'X_tab'_, _'X_text'_, _'X_img'_ and _'target'_. Values\n        are the corresponding matrices. Note that of multiple text or image\n        columns/models are used, the corresponding values should be lists\n        of numpy arrays\n    X_val: Dict, Optional. default=None\n        The validation dataset can also be passed in a dictionary. Keys\n        are _'X_wide'_, _'X_tab'_, _'X_text'_, _'X_img'_ and _'target'_.\n        Values are the corresponding matrices. Note that of multiple text\n        or image columns/models are used, the corresponding values should\n        be lists of numpy arrays\n    val_split: float, Optional. default=None\n        train/val split fraction\n    target: np.ndarray, Optional. default=None\n        target values\n    n_epochs: int, default=1\n        number of epochs\n    validation_freq: int, default=1\n        epochs validation frequency\n    batch_size: int, default=32\n        batch size\n    custom_dataloader: `DataLoader`, Optional, default=None\n        object of class `torch.utils.data.DataLoader`. Available\n        predefined dataloaders are in `pytorch-widedeep.dataloaders`.If\n        `None`, a standard torch `DataLoader` is used.\n    finetune: bool, default=False\n        fine-tune individual model components. This functionality can also\n        be used to 'warm-up' (and hence the alias `warmup`) individual\n        components before the joined training starts, and hence its\n        alias. See the Examples folder in the repo for more details\n\n        `pytorch_widedeep` implements 3 fine-tune routines.\n\n        - fine-tune all trainable layers at once. This routine is\n          inspired by the work of Howard &amp; Sebastian Ruder 2018 in their\n          [ULMfit paper](https://arxiv.org/abs/1801.06146). Using a\n          Slanted Triangular learing (see\n          [Leslie N. Smith paper](https://arxiv.org/pdf/1506.01186.pdf) ) ,\n          the process is the following: *i*) the learning rate will\n          gradually increase for 10% of the training steps from max_lr/10\n          to max_lr. *ii*) It will then gradually decrease to max_lr/10\n          for the remaining 90% of the steps. The optimizer used in the\n          process is `Adam`.\n\n        and two gradual fine-tune routines, where only certain layers are\n        trained at a time.\n\n        - The so called `Felbo` gradual fine-tune rourine, based on the the\n          Felbo et al., 2017 [DeepEmoji paper](https://arxiv.org/abs/1708.00524).\n        - The `Howard` routine based on the work of Howard &amp; Sebastian Ruder 2018 in their\n          [ULMfit paper](https://arxiv.org/abs/1801.06146&gt;).\n\n        For details on how these routines work, please see the Examples\n        section in this documentation and the Examples folder in the repo. &lt;br/&gt;\n        Param Alias: `warmup`\n\n    Other Parameters\n    ----------------\n    **kwargs:\n        Other keyword arguments are:\n\n        - **DataLoader related parameters**:&lt;br/&gt;\n            For example,  `sampler`, `batch_sampler`, `collate_fn`, etc.\n            Please, see the pytorch\n            [DataLoader docs](https://pytorch.org/docs/stable/data.html#torch.utils.data.DataLoader)\n            for details.\n\n        - **Finetune related parameters**:&lt;br/&gt;\n            see the source code at `pytorch_widedeep._finetune`. Namely, these are:\n\n            - `finetune_epochs` (`int`):\n                number of epochs use for fine tuning\n            - `finetune_max_lr` (`float`):\n               max lr during fine tuning\n            - `routine` (`str`):\n               one of _'howard'_ or _'felbo'_\n            - `deeptabular_gradual` (`bool`):\n               boolean indicating if the `deeptabular` component will be fine tuned gradually\n            - `deeptabular_layers` (`Optional[Union[List[nn.Module], List[List[nn.Module]]]]`):\n               List of pytorch modules indicating the layers of the\n               `deeptabular` that will be fine tuned\n            - `deeptabular_max_lr` (`Union[float, List[float]]`):\n               max lr for the `deeptabular` componet during fine tuning\n            - `deeptext_gradual` (`bool`):\n               same as `deeptabular_gradual` but for the `deeptext` component\n            - `deeptext_layers` (`Optional[Union[List[nn.Module], List[List[nn.Module]]]]`):\n               same as `deeptabular_gradual` but for the `deeptext` component.\n               If there are multiple text columns/models, this should be a list of lists\n            - `deeptext_max_lr` (`Union[float, List[float]]`):\n               same as `deeptabular_gradual` but for the `deeptext` component\n               If there are multiple text columns/models, this should be a list of floats\n            - `deepimage_gradual` (`bool`):\n               same as `deeptext_layers` but for the `deepimage` component\n            - `deepimage_layers` (`Optional[Union[List[nn.Module], List[List[nn.Module]]]]`):\n               same as `deeptext_layers` but for the `deepimage` component\n            - `deepimage_max_lr` (`Union[float, List[float]]`):\n                same as `deeptext_layers` but for the `deepimage` component\n\n    Examples\n    --------\n\n    For a series of comprehensive examples on how to use the `fit` method, please see the\n    [Examples](https://github.com/jrzaurin/pytorch-widedeep/tree/master/examples)\n    folder in the repo\n    \"\"\"\n    dataloader_args, finetune_args = self._extract_kwargs(kwargs)\n    self.batch_size = batch_size\n    train_set, eval_set = wd_train_val_split(\n        self.seed,\n        self.method,  # type: ignore\n        X_wide,\n        X_tab,\n        X_text,\n        X_img,\n        X_train,\n        X_val,\n        val_split,\n        target,\n        self.transforms,\n    )\n    train_loader = self._set_dataloader(\n        train_dataloader, train_set, batch_size, dataloader_args\n    )\n    eval_loader = (\n        self._set_dataloader(eval_dataloader, eval_set, batch_size, dataloader_args)\n        if eval_set is not None\n        else None\n    )\n\n    if finetune:\n        self.with_finetuning: bool = True\n        self._do_finetune(train_loader, **finetune_args)\n        if self.verbose:\n            print(\n                \"Fine-tuning (or warmup) of individual components completed. \"\n                \"Training the whole model for {} epochs\".format(n_epochs)\n            )\n    else:\n        self.with_finetuning = False\n\n    self.callback_container.on_train_begin(\n        {\n            \"batch_size\": batch_size,\n            \"train_steps\": len(train_loader),\n            \"n_epochs\": n_epochs,\n        }\n    )\n    for epoch in range(n_epochs):\n        epoch_logs = self._train_epoch(train_loader, epoch)\n        if eval_loader is not None and epoch % validation_freq == (\n            validation_freq - 1\n        ):\n            epoch_logs, on_epoch_end_metric = self._eval_epoch(\n                eval_loader, epoch_logs\n            )\n        else:\n            on_epoch_end_metric = None\n            if self.reducelronplateau:\n                raise NotImplementedError(\n                    \"ReduceLROnPlateau scheduler can be used only with validation data.\"\n                )\n        self.callback_container.on_epoch_end(epoch, epoch_logs, on_epoch_end_metric)\n        if self.early_stop:\n            # self.callback_container.on_train_end(epoch_logs)\n            break\n\n    self.callback_container.on_train_end(epoch_logs)\n    if feature_importance_sample_size is not None:\n        self.feature_importance = FeatureImportance(\n            self.device, feature_importance_sample_size\n        ).feature_importance(train_loader, self.model)\n    self._restore_best_weights()\n    self.model.train()\n</code></pre>"},{"location":"pytorch-widedeep/trainer.html#pytorch_widedeep.training.Trainer.predict","title":"predict","text":"<pre><code>predict(\n    X_wide=None,\n    X_tab=None,\n    X_text=None,\n    X_img=None,\n    X_test=None,\n    batch_size=None,\n)\n</code></pre> <p>Returns the predictions</p> <p>The input datasets can be passed either directly via numpy arrays (<code>X_wide</code>, <code>X_tab</code>, <code>X_text</code> or <code>X_img</code>) or alternatively, in a dictionary (<code>X_test</code>)</p> <p>Parameters:</p> Name Type Description Default <code>X_wide</code> <code>Optional[ndarray]</code> <p>Input for the <code>wide</code> model component. See <code>pytorch_widedeep.preprocessing.WidePreprocessor</code></p> <code>None</code> <code>X_tab</code> <code>Optional[Union[ndarray, List[ndarray]]]</code> <p>Input for the <code>deeptabular</code> model component. See <code>pytorch_widedeep.preprocessing.TabPreprocessor</code></p> <code>None</code> <code>X_text</code> <code>Optional[Union[ndarray, List[ndarray]]]</code> <p>Input for the <code>deeptext</code> model component. See <code>pytorch_widedeep.preprocessing.TextPreprocessor</code></p> <code>None</code> <code>X_img</code> <code>Optional[Union[ndarray, List[ndarray]]]</code> <p>Input for the <code>deepimage</code> model component. See <code>pytorch_widedeep.preprocessing.ImagePreprocessor</code></p> <code>None</code> <code>X_test</code> <code>Optional[Dict[str, Union[ndarray, List[ndarray]]]]</code> <p>The test dataset can also be passed in a dictionary. Keys are <code>X_wide</code>, 'X_tab', 'X_text', 'X_img' and 'target'. Values are the corresponding matrices.</p> <code>None</code> <code>batch_size</code> <code>Optional[int]</code> <p>If a trainer is used to predict after having trained a model, the <code>batch_size</code> needs to be defined as it will not be defined as the <code>Trainer</code> is instantiated</p> <code>None</code> <p>Returns:</p> Type Description <code>np.ndarray:</code> <p>array with the predictions</p> Source code in <code>pytorch_widedeep/training/trainer.py</code> <pre><code>def predict(  # type: ignore[override, return]\n    self,\n    X_wide: Optional[np.ndarray] = None,\n    X_tab: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n    X_text: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n    X_img: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n    X_test: Optional[Dict[str, Union[np.ndarray, List[np.ndarray]]]] = None,\n    batch_size: Optional[int] = None,\n) -&gt; np.ndarray:\n    r\"\"\"Returns the predictions\n\n    The input datasets can be passed either directly via numpy arrays\n    (`X_wide`, `X_tab`, `X_text` or `X_img`) or alternatively, in\n    a dictionary (`X_test`)\n\n\n    Parameters\n    ----------\n    X_wide: np.ndarray, Optional. default=None\n        Input for the `wide` model component.\n        See `pytorch_widedeep.preprocessing.WidePreprocessor`\n    X_tab: np.ndarray or List[np.ndarray], Optional. default=None\n        Input for the `deeptabular` model component.\n        See `pytorch_widedeep.preprocessing.TabPreprocessor`\n    X_text: np.ndarray or List[np.ndarray], Optional. default=None\n        Input for the `deeptext` model component.\n        See `pytorch_widedeep.preprocessing.TextPreprocessor`\n    X_img: np.ndarray or List[np.ndarray], Optional. default=None\n        Input for the `deepimage` model component.\n        See `pytorch_widedeep.preprocessing.ImagePreprocessor`\n    X_test: Dict, Optional. default=None\n        The test dataset can also be passed in a dictionary. Keys are\n        `X_wide`, _'X_tab'_, _'X_text'_, _'X_img'_ and _'target'_. Values\n        are the corresponding matrices.\n    batch_size: int, default = 256\n        If a trainer is used to predict after having trained a model, the\n        `batch_size` needs to be defined as it will not be defined as\n        the `Trainer` is instantiated\n\n    Returns\n    -------\n    np.ndarray:\n        array with the predictions\n    \"\"\"\n    preds_l = self._predict(X_wide, X_tab, X_text, X_img, X_test, batch_size)\n    if self.method == \"regression\":\n        return np.vstack(preds_l).squeeze(1)\n    if self.method == \"binary\":\n        preds = np.vstack(preds_l).squeeze(1)\n        return (preds &gt; 0.5).astype(\"int\")\n    if self.method == \"qregression\":\n        return np.vstack(preds_l)\n    if self.method == \"multiclass\":\n        preds = np.vstack(preds_l)\n        return np.argmax(preds, 1)  # type: ignore[return-value]\n    if self.method == \"multitarget\":\n        if self.loss_fn.__class__.__name__ in [\n            \"MultiTargetClassificationLoss\",\n            \"MutilTargetRegressionAndClassificationLoss\",\n        ]:\n            raise ValueError(\n                \"MultiTargetClassificationLoss and MutilTargetRegressionAndClassificationLoss \"\n                \"are not supported by predict method. Please use predict_proba method instead.\"\n            )\n        return np.vstack(preds_l)\n</code></pre>"},{"location":"pytorch-widedeep/trainer.html#pytorch_widedeep.training.Trainer.predict_uncertainty","title":"predict_uncertainty","text":"<pre><code>predict_uncertainty(\n    X_wide=None,\n    X_tab=None,\n    X_text=None,\n    X_img=None,\n    X_test=None,\n    batch_size=None,\n    uncertainty_granularity=1000,\n)\n</code></pre> <p>Returns the predicted ucnertainty of the model for the test dataset using a Monte Carlo method during which dropout layers are activated in the evaluation/prediction phase and each sample is predicted N times (<code>uncertainty_granularity</code> times).</p> <p>This is based on Dropout as a Bayesian Approximation: Representing Model Uncertainty in Deep Learning.</p> <p>Parameters:</p> Name Type Description Default <code>X_wide</code> <code>Optional[ndarray]</code> <p>Input for the <code>wide</code> model component. See <code>pytorch_widedeep.preprocessing.WidePreprocessor</code></p> <code>None</code> <code>X_tab</code> <code>Optional[Union[ndarray, List[ndarray]]]</code> <p>Input for the <code>deeptabular</code> model component. See <code>pytorch_widedeep.preprocessing.TabPreprocessor</code></p> <code>None</code> <code>X_text</code> <code>Optional[Union[ndarray, List[ndarray]]]</code> <p>Input for the <code>deeptext</code> model component. See <code>pytorch_widedeep.preprocessing.TextPreprocessor</code></p> <code>None</code> <code>X_img</code> <code>Optional[Union[ndarray, List[ndarray]]]</code> <p>Input for the <code>deepimage</code> model component. See <code>pytorch_widedeep.preprocessing.ImagePreprocessor</code></p> <code>None</code> <code>X_test</code> <code>Optional[Dict[str, Union[ndarray, List[ndarray]]]]</code> <p>The test dataset can also be passed in a dictionary. Keys are 'X_wide', 'X_tab', 'X_text', 'X_img' and 'target'. Values are the corresponding matrices.</p> <code>None</code> <code>batch_size</code> <code>Optional[int]</code> <p>If a trainer is used to predict after having trained a model, the <code>batch_size</code> needs to be defined as it will not be defined as the <code>Trainer</code> is instantiated</p> <code>None</code> <code>uncertainty_granularity</code> <code>int</code> <p>number of times the model does prediction for each sample</p> <code>1000</code> <p>Returns:</p> Type Description <code>np.ndarray:</code> <ul> <li>if <code>method = regression</code>, it will return an array with <code>(max, min, mean, stdev)</code>   values for each sample.</li> <li>if <code>method = binary</code> it will return an array with   <code>(mean_cls_0_prob, mean_cls_1_prob, predicted_cls)</code> for each sample.</li> <li>if <code>method = multiclass</code> it will return an array with   <code>(mean_cls_0_prob, mean_cls_1_prob, mean_cls_2_prob, ... , predicted_cls)</code>   values for each sample.</li> </ul> Source code in <code>pytorch_widedeep/training/trainer.py</code> <pre><code>def predict_uncertainty(  # type: ignore[return]\n    self,\n    X_wide: Optional[np.ndarray] = None,\n    X_tab: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n    X_text: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n    X_img: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n    X_test: Optional[Dict[str, Union[np.ndarray, List[np.ndarray]]]] = None,\n    batch_size: Optional[int] = None,\n    uncertainty_granularity: int = 1000,\n) -&gt; np.ndarray:\n    r\"\"\"Returns the predicted ucnertainty of the model for the test dataset\n    using a Monte Carlo method during which dropout layers are activated\n    in the evaluation/prediction phase and each sample is predicted N\n    times (`uncertainty_granularity` times).\n\n    This is based on\n    [Dropout as a Bayesian Approximation: Representing\n    Model Uncertainty in Deep Learning](https://arxiv.org/abs/1506.02142?context=stat).\n\n    Parameters\n    ----------\n    X_wide: np.ndarray, Optional. default=None\n        Input for the `wide` model component.\n        See `pytorch_widedeep.preprocessing.WidePreprocessor`\n    X_tab: np.ndarray or List[np.ndarray], Optional. default=None\n        Input for the `deeptabular` model component.\n        See `pytorch_widedeep.preprocessing.TabPreprocessor`\n    X_text: np.ndarray or List[np.ndarray], Optional. default=None\n        Input for the `deeptext` model component.\n        See `pytorch_widedeep.preprocessing.TextPreprocessor`\n    X_img: np.ndarray or List[np.ndarray], Optional. default=None\n        Input for the `deepimage` model component.\n        See `pytorch_widedeep.preprocessing.ImagePreprocessor`\n    X_test: Dict, Optional. default=None\n        The test dataset can also be passed in a dictionary. Keys are\n        _'X_wide'_, _'X_tab'_, _'X_text'_, _'X_img'_ and _'target'_. Values\n        are the corresponding matrices.\n    batch_size: int, default = 256\n        If a trainer is used to predict after having trained a model, the\n        `batch_size` needs to be defined as it will not be defined as\n        the `Trainer` is instantiated\n    uncertainty_granularity: int default = 1000\n        number of times the model does prediction for each sample\n\n    Returns\n    -------\n    np.ndarray:\n        - if `method = regression`, it will return an array with `(max, min, mean, stdev)`\n          values for each sample.\n        - if `method = binary` it will return an array with\n          `(mean_cls_0_prob, mean_cls_1_prob, predicted_cls)` for each sample.\n        - if `method = multiclass` it will return an array with\n          `(mean_cls_0_prob, mean_cls_1_prob, mean_cls_2_prob, ... , predicted_cls)`\n          values for each sample.\n\n    \"\"\"\n    preds_l = self._predict(\n        X_wide,\n        X_tab,\n        X_text,\n        X_img,\n        X_test,\n        batch_size,\n        uncertainty_granularity,\n        uncertainty=True,\n    )\n    preds = np.vstack(preds_l)\n    samples_num = int(preds.shape[0] / uncertainty_granularity)\n    if self.method == \"regression\":\n        preds = preds.squeeze(1)\n        preds = preds.reshape((uncertainty_granularity, samples_num))\n        return np.array(\n            (\n                preds.max(axis=0),\n                preds.min(axis=0),\n                preds.mean(axis=0),\n                preds.std(axis=0),\n            )\n        ).T\n    if self.method == \"qregression\":\n        raise ValueError(\n            \"Currently predict_uncertainty is not supported for qregression method\"\n        )\n    if self.method == \"binary\":\n        preds = preds.squeeze(1)\n        preds = preds.reshape((uncertainty_granularity, samples_num))\n        preds = preds.mean(axis=0)\n        probs = np.zeros([preds.shape[0], 3])\n        probs[:, 0] = 1 - preds\n        probs[:, 1] = preds\n        return probs\n    if self.method == \"multiclass\":\n        preds = preds.reshape(uncertainty_granularity, samples_num, preds.shape[1])\n        preds = preds.mean(axis=0)\n        preds = np.hstack((preds, np.vstack(np.argmax(preds, 1))))\n        return preds\n\n    if self.method == \"multitarget\":\n        raise ValueError(\n            \"Currently predict_uncertainty is not supported for multitarget method\"\n        )\n</code></pre>"},{"location":"pytorch-widedeep/trainer.html#pytorch_widedeep.training.Trainer.predict_proba","title":"predict_proba","text":"<pre><code>predict_proba(\n    X_wide=None,\n    X_tab=None,\n    X_text=None,\n    X_img=None,\n    X_test=None,\n    batch_size=None,\n)\n</code></pre> <p>Returns the predicted probabilities for the test dataset for  binary and multiclass methods</p> <p>The input datasets can be passed either directly via numpy arrays (<code>X_wide</code>, <code>X_tab</code>, <code>X_text</code> or <code>X_img</code>) or alternatively, in a dictionary (<code>X_test</code>)</p> <p>Parameters:</p> Name Type Description Default <code>X_wide</code> <code>Optional[ndarray]</code> <p>Input for the <code>wide</code> model component. See <code>pytorch_widedeep.preprocessing.WidePreprocessor</code></p> <code>None</code> <code>X_tab</code> <code>Optional[Union[ndarray, List[ndarray]]]</code> <p>Input for the <code>deeptabular</code> model component. See <code>pytorch_widedeep.preprocessing.TabPreprocessor</code></p> <code>None</code> <code>X_text</code> <code>Optional[Union[ndarray, List[ndarray]]]</code> <p>Input for the <code>deeptext</code> model component. See <code>pytorch_widedeep.preprocessing.TextPreprocessor</code></p> <code>None</code> <code>X_img</code> <code>Optional[Union[ndarray, List[ndarray]]]</code> <p>Input for the <code>deepimage</code> model component. See <code>pytorch_widedeep.preprocessing.ImagePreprocessor</code></p> <code>None</code> <code>X_test</code> <code>Optional[Dict[str, Union[ndarray, List[ndarray]]]]</code> <p>The test dataset can also be passed in a dictionary. Keys are <code>X_wide</code>, 'X_tab', 'X_text', 'X_img' and 'target'. Values are the corresponding matrices.</p> <code>None</code> <code>batch_size</code> <code>Optional[int]</code> <p>If a trainer is used to predict after having trained a model, the <code>batch_size</code> needs to be defined as it will not be defined as the <code>Trainer</code> is instantiated</p> <code>None</code> <p>Returns:</p> Type Description <code>ndarray</code> <p>array with the probabilities per class</p> Source code in <code>pytorch_widedeep/training/trainer.py</code> <pre><code>def predict_proba(  # type: ignore[override, return]  # noqa: C901\n    self,\n    X_wide: Optional[np.ndarray] = None,\n    X_tab: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n    X_text: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n    X_img: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n    X_test: Optional[Dict[str, Union[np.ndarray, List[np.ndarray]]]] = None,\n    batch_size: Optional[int] = None,\n) -&gt; np.ndarray:\n    r\"\"\"Returns the predicted probabilities for the test dataset for  binary\n    and multiclass methods\n\n    The input datasets can be passed either directly via numpy arrays\n    (`X_wide`, `X_tab`, `X_text` or `X_img`) or alternatively, in\n    a dictionary (`X_test`)\n\n    Parameters\n    ----------\n    X_wide: np.ndarray, Optional. default=None\n        Input for the `wide` model component.\n        See `pytorch_widedeep.preprocessing.WidePreprocessor`\n    X_tab: np.ndarray or List[np.ndarray], Optional. default=None\n        Input for the `deeptabular` model component.\n        See `pytorch_widedeep.preprocessing.TabPreprocessor`\n    X_text: np.ndarray or List[np.ndarray], Optional. default=None\n        Input for the `deeptext` model component.\n        See `pytorch_widedeep.preprocessing.TextPreprocessor`\n    X_img: np.ndarray or List[np.ndarray], Optional. default=None\n        Input for the `deepimage` model component.\n        See `pytorch_widedeep.preprocessing.ImagePreprocessor`\n    X_test: Dict, Optional. default=None\n        The test dataset can also be passed in a dictionary. Keys are\n        `X_wide`, _'X_tab'_, _'X_text'_, _'X_img'_ and _'target'_. Values\n        are the corresponding matrices.\n    batch_size: int, default = 256\n        If a trainer is used to predict after having trained a model, the\n        `batch_size` needs to be defined as it will not be defined as\n        the `Trainer` is instantiated\n\n    Returns\n    -------\n    np.ndarray\n        array with the probabilities per class\n    \"\"\"\n\n    preds_l = self._predict(X_wide, X_tab, X_text, X_img, X_test, batch_size)\n    if self.method == \"binary\":\n        preds = np.vstack(preds_l).squeeze(1)\n        probs = np.zeros([preds.shape[0], 2])\n        probs[:, 0] = 1 - preds\n        probs[:, 1] = preds\n        return probs\n    if self.method == \"multiclass\":\n        return np.vstack(preds_l)\n    if self.method == \"multitarget\":\n        return np.vstack(preds_l)\n</code></pre>"},{"location":"pytorch-widedeep/trainer.html#pytorch_widedeep.training.Trainer.save","title":"save","text":"<pre><code>save(\n    path,\n    save_state_dict=False,\n    save_optimizer=False,\n    model_filename=\"wd_model.pt\",\n)\n</code></pre> <p>Saves the model, training and evaluation history, and the <code>feature_importance</code> attribute (if the <code>deeptabular</code> component is a Tabnet model) to disk</p> <p>The <code>Trainer</code> class is built so that it 'just' trains a model. With that in mind, all the torch related parameters (such as optimizers, learning rate schedulers, initializers, etc) have to be defined externally and then passed to the <code>Trainer</code>. As a result, the <code>Trainer</code> does not generate any attribute or additional data products that need to be saved other than the <code>model</code> object itself, which can be saved as any other torch model (e.g. <code>torch.save(model, path)</code>).</p> <p>The exception is Tabnet. If the <code>deeptabular</code> component is a Tabnet model, an attribute (a dict) called <code>feature_importance</code> will be created at the end of the training process. Therefore, a <code>save</code> method was created that will save the feature importance dictionary to a json file and, since we are here, the model weights, training history and learning rate history.</p> <p>Parameters:</p> Name Type Description Default <code>path</code> <code>str</code> <p>path to the directory where the model and the feature importance attribute will be saved.</p> required <code>save_state_dict</code> <code>bool</code> <p>Boolean indicating whether to save directly the model (and optimizer) or the model's (and optimizer's) state dictionary</p> <code>False</code> <code>save_optimizer</code> <code>bool</code> <p>Boolean indicating whether to save the optimizer</p> <code>False</code> <code>model_filename</code> <code>str</code> <p>filename where the model weights will be store</p> <code>'wd_model.pt'</code> Source code in <code>pytorch_widedeep/training/trainer.py</code> <pre><code>def save(\n    self,\n    path: str,\n    save_state_dict: bool = False,\n    save_optimizer: bool = False,\n    model_filename: str = \"wd_model.pt\",\n):\n    r\"\"\"Saves the model, training and evaluation history, and the\n    `feature_importance` attribute (if the `deeptabular` component is a\n    Tabnet model) to disk\n\n    The `Trainer` class is built so that it 'just' trains a model. With\n    that in mind, all the torch related parameters (such as optimizers,\n    learning rate schedulers, initializers, etc) have to be defined\n    externally and then passed to the `Trainer`. As a result, the\n    `Trainer` does not generate any attribute or additional data\n    products that need to be saved other than the `model` object itself,\n    which can be saved as any other torch model (e.g. `torch.save(model,\n    path)`).\n\n    The exception is Tabnet. If the `deeptabular` component is a Tabnet\n    model, an attribute (a dict) called `feature_importance` will be\n    created at the end of the training process. Therefore, a `save`\n    method was created that will save the feature importance dictionary\n    to a json file and, since we are here, the model weights, training\n    history and learning rate history.\n\n    Parameters\n    ----------\n    path: str\n        path to the directory where the model and the feature importance\n        attribute will be saved.\n    save_state_dict: bool, default = False\n        Boolean indicating whether to save directly the model\n        (and optimizer) or the model's (and optimizer's) state\n        dictionary\n    save_optimizer: bool, default = False\n        Boolean indicating whether to save the optimizer\n    model_filename: str, Optional, default = \"wd_model.pt\"\n        filename where the model weights will be store\n    \"\"\"\n\n    self._save_history(path)\n\n    self._save_model_and_optimizer(\n        path, save_state_dict, save_optimizer, model_filename\n    )\n\n    if self.model.is_tabnet:\n        with open(Path(path) / \"feature_importance.json\", \"w\") as fi:\n            json.dump(self.feature_importance, fi)\n</code></pre>"},{"location":"pytorch-widedeep/utils/index.html","title":"The <code>utils</code> module","text":"<p>These are a series of utilities that might be useful for a number of preprocessing tasks, even not directly related to <code>pytorch-widedeep</code>. All the classes and functions discussed here are available directly from the <code>utils</code> module. For example, the <code>LabelEncoder</code> within the <code>deeptabular_utils</code> submodule can be imported as:</p> <pre><code>from pytorch_widedeep.utils import LabelEncoder\n</code></pre> <p>These are classes and functions that are internally used in the library. We include them here in case the user finds them useful for other purposes.</p>"},{"location":"pytorch-widedeep/utils/deeptabular_utils.html","title":"deeptabular utils","text":""},{"location":"pytorch-widedeep/utils/deeptabular_utils.html#pytorch_widedeep.utils.deeptabular_utils.LabelEncoder","title":"LabelEncoder","text":"<p>Label Encode categorical values for multiple columns at once</p> <p> NOTE: LabelEncoder reserves 0 for <code>unseen</code> new categories. This is convenient when defining the embedding layers, since we can just set padding idx to 0.</p> <p>Parameters:</p> Name Type Description Default <code>columns_to_encode</code> <code>Optional[List[str]]</code> <p>List of strings containing the names of the columns to encode. If <code>None</code> all columns of type <code>object</code> in the dataframe will be label encoded.</p> <code>None</code> <code>with_attention</code> <code>bool</code> <p>Boolean indicating whether the preprocessed data will be passed to an attention-based model. Aliased as <code>for_transformer</code>.</p> <code>False</code> <code>shared_embed</code> <code>bool</code> <p>Boolean indicating if the embeddings will be \"shared\" when using attention-based models. The idea behind <code>shared_embed</code> is described in the Appendix A in the TabTransformer paper: 'The goal of having column embedding is to enable the model to distinguish the classes in one column from those in the other columns'. In other words, the idea is to let the model learn which column is embedded at the time. See: <code>pytorch_widedeep.models.transformers._layers.SharedEmbeddings</code>.</p> <code>False</code> <p>Attributes:</p> Name Type Description <code>encoding_dict</code> <code>Dict</code> <p>Dictionary containing the encoding mappings in the format, e.g. :  <code>{'colname1': {'cat1': 1, 'cat2': 2, ...}, 'colname2': {'cat1': 1, 'cat2': 2, ...}, ...}</code></p> <code>inverse_encoding_dict</code> <code>Dict</code> <p>Dictionary containing the inverse encoding mappings in the format, e.g. :  <code>{'colname1': {1: 'cat1', 2: 'cat2', ...}, 'colname2': {1: 'cat1', 2: 'cat2', ...}, ...}</code></p> Source code in <code>pytorch_widedeep/utils/deeptabular_utils.py</code> <pre><code>class LabelEncoder:\n    r\"\"\"Label Encode categorical values for multiple columns at once\n\n    :information_source: **NOTE**:\n    LabelEncoder reserves 0 for `unseen` new categories. This is convenient\n    when defining the embedding layers, since we can just set padding idx to 0.\n\n    Parameters\n    ----------\n    columns_to_encode: list, Optional, default = None\n        List of strings containing the names of the columns to encode. If\n        `None` all columns of type `object` in the dataframe will be label\n        encoded.\n    with_attention: bool, default = False\n        Boolean indicating whether the preprocessed data will be passed to an\n        attention-based model. Aliased as `for_transformer`.\n    shared_embed: bool, default = False\n        Boolean indicating if the embeddings will be \"_shared_\" when using\n        attention-based models. The idea behind `shared_embed` is described\n        in the Appendix A in the [TabTransformer paper](https://arxiv.org/abs/2012.06678):\n        '_The goal of having column embedding is to enable the model to\n        distinguish the classes in one column from those in the\n        other columns_'. In other words, the idea is to let the model learn\n        which column is embedded at the time. See: `pytorch_widedeep.models.transformers._layers.SharedEmbeddings`.\n\n    Attributes\n    ----------\n    encoding_dict : Dict\n        Dictionary containing the encoding mappings in the format, e.g. : &lt;br/&gt;\n        `{'colname1': {'cat1': 1, 'cat2': 2, ...}, 'colname2': {'cat1': 1, 'cat2': 2, ...}, ...}`\n    inverse_encoding_dict : Dict\n        Dictionary containing the inverse encoding mappings in the format, e.g. : &lt;br/&gt;\n        `{'colname1': {1: 'cat1', 2: 'cat2', ...}, 'colname2': {1: 'cat1', 2: 'cat2', ...}, ...}`\n\n    \"\"\"\n\n    @alias(\"with_attention\", [\"for_transformer\"])\n    def __init__(\n        self,\n        columns_to_encode: Optional[List[str]] = None,\n        with_attention: bool = False,\n        shared_embed: bool = False,\n    ):\n        self.columns_to_encode = columns_to_encode\n\n        self.shared_embed = shared_embed\n        self.with_attention = with_attention\n\n        self.reset_embed_idx = not self.with_attention or self.shared_embed\n\n    def partial_fit(self, df: pd.DataFrame) -&gt; \"LabelEncoder\":  # noqa: C901\n        \"\"\"Main method. Creates encoding attributes.\n\n        Returns\n        -------\n        LabelEncoder\n            `LabelEncoder` fitted object\n        \"\"\"\n        # here df is a chunk of the data. this is meant to be run when the\n        # data is large and we pass a chunk at a time. Therefore, we do not\n        # copy the input chunk as mutating a chunk is ok\n        if self.columns_to_encode is None:\n            self.columns_to_encode = list(df.select_dtypes(include=[\"object\"]).columns)\n        else:\n            # sanity check to make sure all categorical columns are in an adequate\n            # format\n            for col in self.columns_to_encode:\n                df[col] = df[col].astype(\"O\")\n\n        unique_column_vals: Dict[str, List[str]] = {}\n        for c in self.columns_to_encode:\n            unique_column_vals[c] = df[c].unique().tolist()\n\n        if not hasattr(self, \"encoding_dict\"):\n            # we run the method 'partial_fit' for the 1st time\n            self.encoding_dict: Dict[str, Dict[str, int]] = {}\n            if \"cls_token\" in unique_column_vals and self.shared_embed:\n                self.encoding_dict[\"cls_token\"] = {\"[CLS]\": 0}\n                del unique_column_vals[\"cls_token\"]\n\n            # leave 0 for padding/\"unseen\" categories. Also we need an\n            # attribute to keep track of the encoding in case we use\n            # attention and we do not re-start the index/counter\n            self.cum_idx: int = 1\n            for k, v in unique_column_vals.items():\n                self.encoding_dict[k] = {o: i + self.cum_idx for i, o in enumerate(v)}\n                self.cum_idx = 1 if self.reset_embed_idx else self.cum_idx + len(v)\n        else:\n            # the 'partial_fit' method has already run.\n            # \"cls_token\" will have been added already\n            if \"cls_token\" in unique_column_vals and self.shared_embed:\n                del unique_column_vals[\"cls_token\"]\n\n            # Classes in the new df/chunk of the dataset that have not been seen\n            # before\n            unseen_classes: Dict[str, List[str]] = {}\n            for c in self.columns_to_encode:\n                unseen_classes[c] = list(\n                    np.setdiff1d(\n                        unique_column_vals[c], list(self.encoding_dict[c].keys())\n                    )\n                )\n\n            # leave 0 for padding/\"unseen\" categories\n            for k, v in unique_column_vals.items():\n                # if we use attention we need to start encoding from the\n                # last 'overall' encoding index. Otherwise, we use the max\n                # encoding index per categorical col\n                _idx = (\n                    max(self.encoding_dict[k].values()) + 1\n                    if self.reset_embed_idx\n                    else self.cum_idx\n                )\n                if len(unseen_classes[k]) != 0:\n                    for i, o in enumerate(unseen_classes[k]):\n                        if o not in self.encoding_dict[k]:\n                            self.encoding_dict[k][o] = i + _idx\n                    # if self.reset_embed_idx is True it will be 1 anyway\n                    self.cum_idx = (\n                        1\n                        if self.reset_embed_idx\n                        else self.cum_idx + len(unseen_classes[k])\n                    )\n\n        return self\n\n    def fit(self, df: pd.DataFrame) -&gt; \"LabelEncoder\":\n        \"\"\"Simply runs the `partial_fit` method when the data fits in memory\n\n        Returns\n        -------\n        LabelEncoder\n            `LabelEncoder` fitted object\n        \"\"\"\n        # this is meant to be run when the data fits in memory and therefore,\n        # we do not want to mutate the original df, so we copy it\n        self.partial_fit(df.copy())\n\n        self.inverse_encoding_dict = self.create_inverse_encoding_dict()\n\n        return self\n\n    def transform(self, df: pd.DataFrame) -&gt; pd.DataFrame:\n        \"\"\"Label Encoded the categories in `columns_to_encode`\n\n        Returns\n        -------\n        pd.DataFrame\n            label-encoded dataframe\n        \"\"\"\n        try:\n            self.encoding_dict\n        except AttributeError:\n            raise NotFittedError(\n                \"This LabelEncoder instance is not fitted yet. \"\n                \"Call 'fit' with appropriate arguments before using this LabelEncoder.\"\n            )\n\n        df_inp = df.copy()\n        # sanity check to make sure all categorical columns are in an adequate\n        # format\n        for col in self.columns_to_encode:  # type: ignore\n            df_inp[col] = df_inp[col].astype(\"O\")\n\n        for k, v in self.encoding_dict.items():\n            df_inp[k] = df_inp[k].apply(lambda x: v[x] if x in v.keys() else 0)\n\n        return df_inp\n\n    def fit_transform(self, df: pd.DataFrame) -&gt; pd.DataFrame:\n        \"\"\"Combines `fit` and `transform`\n\n        Examples\n        --------\n\n        &gt;&gt;&gt; import pandas as pd\n        &gt;&gt;&gt; from pytorch_widedeep.utils import LabelEncoder\n        &gt;&gt;&gt; df = pd.DataFrame({'col1': [1,2,3], 'col2': ['me', 'you', 'him']})\n        &gt;&gt;&gt; columns_to_encode = ['col2']\n        &gt;&gt;&gt; encoder = LabelEncoder(columns_to_encode)\n        &gt;&gt;&gt; encoder.fit_transform(df)\n           col1  col2\n        0     1     1\n        1     2     2\n        2     3     3\n        &gt;&gt;&gt; encoder.encoding_dict\n        {'col2': {'me': 1, 'you': 2, 'him': 3}}\n\n        Returns\n        -------\n        pd.DataFrame\n            label-encoded dataframe\n        \"\"\"\n        return self.fit(df).transform(df)\n\n    def create_inverse_encoding_dict(self) -&gt; Dict[str, Dict[int, str]]:\n        inverse_encoding_dict = dict()\n        for c in self.encoding_dict:\n            inverse_encoding_dict[c] = {v: k for k, v in self.encoding_dict[c].items()}\n            inverse_encoding_dict[c][0] = \"unseen\"\n        return inverse_encoding_dict\n\n    def inverse_transform(self, df: pd.DataFrame) -&gt; pd.DataFrame:\n        \"\"\"Returns the original categories\n\n        Examples\n        --------\n\n        &gt;&gt;&gt; import pandas as pd\n        &gt;&gt;&gt; from pytorch_widedeep.utils import LabelEncoder\n        &gt;&gt;&gt; df = pd.DataFrame({'col1': [1,2,3], 'col2': ['me', 'you', 'him']})\n        &gt;&gt;&gt; columns_to_encode = ['col2']\n        &gt;&gt;&gt; encoder = LabelEncoder(columns_to_encode)\n        &gt;&gt;&gt; df_enc = encoder.fit_transform(df)\n        &gt;&gt;&gt; encoder.inverse_transform(df_enc)\n           col1 col2\n        0     1   me\n        1     2  you\n        2     3  him\n\n        Returns\n        -------\n        pd.DataFrame\n            DataFrame with original categories\n        \"\"\"\n\n        if not hasattr(self, \"inverse_encoding_dict\"):\n            self.inverse_encoding_dict = self.create_inverse_encoding_dict()\n\n        for k, v in self.inverse_encoding_dict.items():\n            df[k] = df[k].apply(lambda x: v[x])\n\n        return df\n\n    def __repr__(self) -&gt; str:\n        list_of_params: List[str] = []\n        if self.columns_to_encode is not None:\n            list_of_params.append(\"columns_to_encode={columns_to_encode}\")\n        if self.with_attention:\n            list_of_params.append(\"with_attention={with_attention}\")\n        if self.shared_embed:\n            list_of_params.append(\"shared_embed={shared_embed}\")\n        all_params = \", \".join(list_of_params)\n        return f\"LabelEncoder({all_params.format(**self.__dict__)})\"\n</code></pre>"},{"location":"pytorch-widedeep/utils/deeptabular_utils.html#pytorch_widedeep.utils.deeptabular_utils.LabelEncoder.partial_fit","title":"partial_fit","text":"<pre><code>partial_fit(df)\n</code></pre> <p>Main method. Creates encoding attributes.</p> <p>Returns:</p> Type Description <code>LabelEncoder</code> <p><code>LabelEncoder</code> fitted object</p> Source code in <code>pytorch_widedeep/utils/deeptabular_utils.py</code> <pre><code>def partial_fit(self, df: pd.DataFrame) -&gt; \"LabelEncoder\":  # noqa: C901\n    \"\"\"Main method. Creates encoding attributes.\n\n    Returns\n    -------\n    LabelEncoder\n        `LabelEncoder` fitted object\n    \"\"\"\n    # here df is a chunk of the data. this is meant to be run when the\n    # data is large and we pass a chunk at a time. Therefore, we do not\n    # copy the input chunk as mutating a chunk is ok\n    if self.columns_to_encode is None:\n        self.columns_to_encode = list(df.select_dtypes(include=[\"object\"]).columns)\n    else:\n        # sanity check to make sure all categorical columns are in an adequate\n        # format\n        for col in self.columns_to_encode:\n            df[col] = df[col].astype(\"O\")\n\n    unique_column_vals: Dict[str, List[str]] = {}\n    for c in self.columns_to_encode:\n        unique_column_vals[c] = df[c].unique().tolist()\n\n    if not hasattr(self, \"encoding_dict\"):\n        # we run the method 'partial_fit' for the 1st time\n        self.encoding_dict: Dict[str, Dict[str, int]] = {}\n        if \"cls_token\" in unique_column_vals and self.shared_embed:\n            self.encoding_dict[\"cls_token\"] = {\"[CLS]\": 0}\n            del unique_column_vals[\"cls_token\"]\n\n        # leave 0 for padding/\"unseen\" categories. Also we need an\n        # attribute to keep track of the encoding in case we use\n        # attention and we do not re-start the index/counter\n        self.cum_idx: int = 1\n        for k, v in unique_column_vals.items():\n            self.encoding_dict[k] = {o: i + self.cum_idx for i, o in enumerate(v)}\n            self.cum_idx = 1 if self.reset_embed_idx else self.cum_idx + len(v)\n    else:\n        # the 'partial_fit' method has already run.\n        # \"cls_token\" will have been added already\n        if \"cls_token\" in unique_column_vals and self.shared_embed:\n            del unique_column_vals[\"cls_token\"]\n\n        # Classes in the new df/chunk of the dataset that have not been seen\n        # before\n        unseen_classes: Dict[str, List[str]] = {}\n        for c in self.columns_to_encode:\n            unseen_classes[c] = list(\n                np.setdiff1d(\n                    unique_column_vals[c], list(self.encoding_dict[c].keys())\n                )\n            )\n\n        # leave 0 for padding/\"unseen\" categories\n        for k, v in unique_column_vals.items():\n            # if we use attention we need to start encoding from the\n            # last 'overall' encoding index. Otherwise, we use the max\n            # encoding index per categorical col\n            _idx = (\n                max(self.encoding_dict[k].values()) + 1\n                if self.reset_embed_idx\n                else self.cum_idx\n            )\n            if len(unseen_classes[k]) != 0:\n                for i, o in enumerate(unseen_classes[k]):\n                    if o not in self.encoding_dict[k]:\n                        self.encoding_dict[k][o] = i + _idx\n                # if self.reset_embed_idx is True it will be 1 anyway\n                self.cum_idx = (\n                    1\n                    if self.reset_embed_idx\n                    else self.cum_idx + len(unseen_classes[k])\n                )\n\n    return self\n</code></pre>"},{"location":"pytorch-widedeep/utils/deeptabular_utils.html#pytorch_widedeep.utils.deeptabular_utils.LabelEncoder.fit","title":"fit","text":"<pre><code>fit(df)\n</code></pre> <p>Simply runs the <code>partial_fit</code> method when the data fits in memory</p> <p>Returns:</p> Type Description <code>LabelEncoder</code> <p><code>LabelEncoder</code> fitted object</p> Source code in <code>pytorch_widedeep/utils/deeptabular_utils.py</code> <pre><code>def fit(self, df: pd.DataFrame) -&gt; \"LabelEncoder\":\n    \"\"\"Simply runs the `partial_fit` method when the data fits in memory\n\n    Returns\n    -------\n    LabelEncoder\n        `LabelEncoder` fitted object\n    \"\"\"\n    # this is meant to be run when the data fits in memory and therefore,\n    # we do not want to mutate the original df, so we copy it\n    self.partial_fit(df.copy())\n\n    self.inverse_encoding_dict = self.create_inverse_encoding_dict()\n\n    return self\n</code></pre>"},{"location":"pytorch-widedeep/utils/deeptabular_utils.html#pytorch_widedeep.utils.deeptabular_utils.LabelEncoder.transform","title":"transform","text":"<pre><code>transform(df)\n</code></pre> <p>Label Encoded the categories in <code>columns_to_encode</code></p> <p>Returns:</p> Type Description <code>DataFrame</code> <p>label-encoded dataframe</p> Source code in <code>pytorch_widedeep/utils/deeptabular_utils.py</code> <pre><code>def transform(self, df: pd.DataFrame) -&gt; pd.DataFrame:\n    \"\"\"Label Encoded the categories in `columns_to_encode`\n\n    Returns\n    -------\n    pd.DataFrame\n        label-encoded dataframe\n    \"\"\"\n    try:\n        self.encoding_dict\n    except AttributeError:\n        raise NotFittedError(\n            \"This LabelEncoder instance is not fitted yet. \"\n            \"Call 'fit' with appropriate arguments before using this LabelEncoder.\"\n        )\n\n    df_inp = df.copy()\n    # sanity check to make sure all categorical columns are in an adequate\n    # format\n    for col in self.columns_to_encode:  # type: ignore\n        df_inp[col] = df_inp[col].astype(\"O\")\n\n    for k, v in self.encoding_dict.items():\n        df_inp[k] = df_inp[k].apply(lambda x: v[x] if x in v.keys() else 0)\n\n    return df_inp\n</code></pre>"},{"location":"pytorch-widedeep/utils/deeptabular_utils.html#pytorch_widedeep.utils.deeptabular_utils.LabelEncoder.fit_transform","title":"fit_transform","text":"<pre><code>fit_transform(df)\n</code></pre> <p>Combines <code>fit</code> and <code>transform</code></p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import pandas as pd\n&gt;&gt;&gt; from pytorch_widedeep.utils import LabelEncoder\n&gt;&gt;&gt; df = pd.DataFrame({'col1': [1,2,3], 'col2': ['me', 'you', 'him']})\n&gt;&gt;&gt; columns_to_encode = ['col2']\n&gt;&gt;&gt; encoder = LabelEncoder(columns_to_encode)\n&gt;&gt;&gt; encoder.fit_transform(df)\n   col1  col2\n0     1     1\n1     2     2\n2     3     3\n&gt;&gt;&gt; encoder.encoding_dict\n{'col2': {'me': 1, 'you': 2, 'him': 3}}\n</code></pre> <p>Returns:</p> Type Description <code>DataFrame</code> <p>label-encoded dataframe</p> Source code in <code>pytorch_widedeep/utils/deeptabular_utils.py</code> <pre><code>def fit_transform(self, df: pd.DataFrame) -&gt; pd.DataFrame:\n    \"\"\"Combines `fit` and `transform`\n\n    Examples\n    --------\n\n    &gt;&gt;&gt; import pandas as pd\n    &gt;&gt;&gt; from pytorch_widedeep.utils import LabelEncoder\n    &gt;&gt;&gt; df = pd.DataFrame({'col1': [1,2,3], 'col2': ['me', 'you', 'him']})\n    &gt;&gt;&gt; columns_to_encode = ['col2']\n    &gt;&gt;&gt; encoder = LabelEncoder(columns_to_encode)\n    &gt;&gt;&gt; encoder.fit_transform(df)\n       col1  col2\n    0     1     1\n    1     2     2\n    2     3     3\n    &gt;&gt;&gt; encoder.encoding_dict\n    {'col2': {'me': 1, 'you': 2, 'him': 3}}\n\n    Returns\n    -------\n    pd.DataFrame\n        label-encoded dataframe\n    \"\"\"\n    return self.fit(df).transform(df)\n</code></pre>"},{"location":"pytorch-widedeep/utils/deeptabular_utils.html#pytorch_widedeep.utils.deeptabular_utils.LabelEncoder.inverse_transform","title":"inverse_transform","text":"<pre><code>inverse_transform(df)\n</code></pre> <p>Returns the original categories</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import pandas as pd\n&gt;&gt;&gt; from pytorch_widedeep.utils import LabelEncoder\n&gt;&gt;&gt; df = pd.DataFrame({'col1': [1,2,3], 'col2': ['me', 'you', 'him']})\n&gt;&gt;&gt; columns_to_encode = ['col2']\n&gt;&gt;&gt; encoder = LabelEncoder(columns_to_encode)\n&gt;&gt;&gt; df_enc = encoder.fit_transform(df)\n&gt;&gt;&gt; encoder.inverse_transform(df_enc)\n   col1 col2\n0     1   me\n1     2  you\n2     3  him\n</code></pre> <p>Returns:</p> Type Description <code>DataFrame</code> <p>DataFrame with original categories</p> Source code in <code>pytorch_widedeep/utils/deeptabular_utils.py</code> <pre><code>def inverse_transform(self, df: pd.DataFrame) -&gt; pd.DataFrame:\n    \"\"\"Returns the original categories\n\n    Examples\n    --------\n\n    &gt;&gt;&gt; import pandas as pd\n    &gt;&gt;&gt; from pytorch_widedeep.utils import LabelEncoder\n    &gt;&gt;&gt; df = pd.DataFrame({'col1': [1,2,3], 'col2': ['me', 'you', 'him']})\n    &gt;&gt;&gt; columns_to_encode = ['col2']\n    &gt;&gt;&gt; encoder = LabelEncoder(columns_to_encode)\n    &gt;&gt;&gt; df_enc = encoder.fit_transform(df)\n    &gt;&gt;&gt; encoder.inverse_transform(df_enc)\n       col1 col2\n    0     1   me\n    1     2  you\n    2     3  him\n\n    Returns\n    -------\n    pd.DataFrame\n        DataFrame with original categories\n    \"\"\"\n\n    if not hasattr(self, \"inverse_encoding_dict\"):\n        self.inverse_encoding_dict = self.create_inverse_encoding_dict()\n\n    for k, v in self.inverse_encoding_dict.items():\n        df[k] = df[k].apply(lambda x: v[x])\n\n    return df\n</code></pre>"},{"location":"pytorch-widedeep/utils/fastai_transforms.html","title":"Fastai transforms","text":"<p>We directly copied and pasted part of the <code>transforms.py</code> module from the <code>fastai</code> library (from an old version). The reason to do such a thing is because <code>pytorch_widedeep</code> only needs the <code>Tokenizer</code> and the <code>Vocab</code> classes there. This way we avoid extra dependencies. Credit for all the code in the <code>fastai_transforms</code> module in this <code>pytorch-widedeep</code> package goes to Jeremy Howard and the <code>fastai</code> team. I only include the documentation here for completion, but I strongly advise the user to read the <code>fastai</code> documentation.</p>"},{"location":"pytorch-widedeep/utils/fastai_transforms.html#pytorch_widedeep.utils.fastai_transforms.Tokenizer","title":"Tokenizer","text":"<p>Class to combine a series of rules and a tokenizer function to tokenize text with multiprocessing.</p> <p>Setting some of the parameters of this class require perhaps some familiarity with the source code.</p> <p>Parameters:</p> Name Type Description Default <code>tok_func</code> <code>Callable</code> <p>Tokenizer Object. See <code>pytorch_widedeep.utils.fastai_transforms.SpacyTokenizer</code></p> <code>SpacyTokenizer</code> <code>lang</code> <code>str</code> <p>Text's Language</p> <code>'en'</code> <code>pre_rules</code> <code>Optional[ListRules]</code> <p>Custom type: <code>Collection[Callable[[str], str]]</code>. These are <code>Callable</code> objects that will be applied to the text (str) directly as <code>rule(tok)</code> before being tokenized.</p> <code>None</code> <code>post_rules</code> <code>Optional[ListRules]</code> <p>Custom type: <code>Collection[Callable[[str], str]]</code>. These are <code>Callable</code> objects that will be applied to the tokens as <code>rule(tokens)</code> after the text has been tokenized.</p> <code>None</code> <code>special_cases</code> <code>Optional[Collection[str]]</code> <p>special cases to be added to the tokenizer via <code>Spacy</code>'s <code>add_special_case</code> method</p> <code>None</code> <code>n_cpus</code> <code>Optional[int]</code> <p>number of CPUs to used during the tokenization process</p> <code>None</code> Source code in <code>pytorch_widedeep/utils/fastai_transforms.py</code> <pre><code>class Tokenizer:\n    r\"\"\"Class to combine a series of rules and a tokenizer function to tokenize\n    text with multiprocessing.\n\n    Setting some of the parameters of this class require perhaps some\n    familiarity with the source code.\n\n    Parameters\n    ----------\n    tok_func: Callable, default = ``SpacyTokenizer``\n        Tokenizer Object. See `pytorch_widedeep.utils.fastai_transforms.SpacyTokenizer`\n    lang: str, default = \"en\"\n        Text's Language\n    pre_rules: ListRules, Optional, default = None\n        Custom type: ``Collection[Callable[[str], str]]``. These are\n        `Callable` objects that will be applied to the text (str) directly as\n        `rule(tok)` before being tokenized.\n    post_rules: ListRules, Optional, default = None\n        Custom type: ``Collection[Callable[[str], str]]``. These are\n        `Callable` objects that will be applied to the tokens as\n        `rule(tokens)` after the text has been tokenized.\n    special_cases: Collection, Optional, default= None\n        special cases to be added to the tokenizer via ``Spacy``'s\n        ``add_special_case`` method\n    n_cpus: int, Optional, default = None\n        number of CPUs to used during the tokenization process\n    \"\"\"\n\n    def __init__(\n        self,\n        tok_func: Callable = SpacyTokenizer,\n        lang: str = \"en\",\n        pre_rules: Optional[ListRules] = None,\n        post_rules: Optional[ListRules] = None,\n        special_cases: Optional[Collection[str]] = None,\n        n_cpus: Optional[int] = None,\n    ):\n        self.tok_func, self.lang, self.special_cases = tok_func, lang, special_cases\n        self.pre_rules = ifnone(pre_rules, defaults.text_pre_rules)\n        self.post_rules = ifnone(post_rules, defaults.text_post_rules)\n        self.special_cases = (\n            special_cases if special_cases is not None else defaults.text_spec_tok\n        )\n        self.n_cpus = ifnone(n_cpus, defaults.cpus)\n\n    def __repr__(self) -&gt; str:\n        res = f\"Tokenizer {self.tok_func.__name__} in {self.lang} with the following rules:\\n\"\n        for rule in self.pre_rules:\n            res += f\" - {rule.__name__}\\n\"\n        for rule in self.post_rules:\n            res += f\" - {rule.__name__}\\n\"\n        return res\n\n    def process_text(self, t: str, tok: BaseTokenizer) -&gt; List[str]:\n        r\"\"\"Process and tokenize one text ``t`` with tokenizer ``tok``.\n\n        Parameters\n        ----------\n        t: str\n            text to be processed and tokenized\n        tok: ``BaseTokenizer``\n            Instance of `BaseTokenizer`. See\n            `pytorch_widedeep.utils.fastai_transforms.BaseTokenizer`\n\n        Returns\n        -------\n        List[str]\n            List of tokens\n        \"\"\"\n        for rule in self.pre_rules:\n            t = rule(t)\n        toks = tok.tokenizer(t)\n        for rule in self.post_rules:\n            toks = rule(toks)\n        return toks\n\n    def _process_all_1(self, texts: Collection[str]) -&gt; List[List[str]]:\n        \"\"\"Process a list of ``texts`` in one process.\"\"\"\n\n        tok = self.tok_func(self.lang)\n        if self.special_cases:\n            tok.add_special_cases(self.special_cases)\n        return [self.process_text(str(t), tok) for t in texts]\n\n    def process_all(self, texts: Collection[str]) -&gt; List[List[str]]:\n        r\"\"\"Process a list of texts. Parallel execution of ``process_text``.\n\n        Examples\n        --------\n        &gt;&gt;&gt; from pytorch_widedeep.utils import Tokenizer\n        &gt;&gt;&gt; texts = ['Machine learning is great', 'but building stuff is even better']\n        &gt;&gt;&gt; tok = Tokenizer()\n        &gt;&gt;&gt; tok.process_all(texts)\n        [['xxmaj', 'machine', 'learning', 'is', 'great'], ['but', 'building', 'stuff', 'is', 'even', 'better']]\n\n        :information_source: **NOTE**:\n        Note the token ``TK_MAJ`` (`xxmaj`), used to indicate the\n        next word begins with a capital in the original text. For more\n        details of special tokens please see the [``fastai`` docs](https://docs.fast.ai/text.core.html#Tokenizing).\n\n        Returns\n        -------\n        List[List[str]]\n            List containing lists of tokens. One list per \"_document_\"\n\n        \"\"\"\n\n        if self.n_cpus &lt;= 1:\n            return self._process_all_1(texts)\n        with ProcessPoolExecutor(self.n_cpus) as e:\n            return sum(\n                e.map(self._process_all_1, partition_by_cores(texts, self.n_cpus)), []\n            )\n</code></pre>"},{"location":"pytorch-widedeep/utils/fastai_transforms.html#pytorch_widedeep.utils.fastai_transforms.Tokenizer.process_text","title":"process_text","text":"<pre><code>process_text(t, tok)\n</code></pre> <p>Process and tokenize one text <code>t</code> with tokenizer <code>tok</code>.</p> <p>Parameters:</p> Name Type Description Default <code>t</code> <code>str</code> <p>text to be processed and tokenized</p> required <code>tok</code> <code>BaseTokenizer</code> <p>Instance of <code>BaseTokenizer</code>. See <code>pytorch_widedeep.utils.fastai_transforms.BaseTokenizer</code></p> required <p>Returns:</p> Type Description <code>List[str]</code> <p>List of tokens</p> Source code in <code>pytorch_widedeep/utils/fastai_transforms.py</code> <pre><code>def process_text(self, t: str, tok: BaseTokenizer) -&gt; List[str]:\n    r\"\"\"Process and tokenize one text ``t`` with tokenizer ``tok``.\n\n    Parameters\n    ----------\n    t: str\n        text to be processed and tokenized\n    tok: ``BaseTokenizer``\n        Instance of `BaseTokenizer`. See\n        `pytorch_widedeep.utils.fastai_transforms.BaseTokenizer`\n\n    Returns\n    -------\n    List[str]\n        List of tokens\n    \"\"\"\n    for rule in self.pre_rules:\n        t = rule(t)\n    toks = tok.tokenizer(t)\n    for rule in self.post_rules:\n        toks = rule(toks)\n    return toks\n</code></pre>"},{"location":"pytorch-widedeep/utils/fastai_transforms.html#pytorch_widedeep.utils.fastai_transforms.Tokenizer.process_all","title":"process_all","text":"<pre><code>process_all(texts)\n</code></pre> <p>Process a list of texts. Parallel execution of <code>process_text</code>.</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; from pytorch_widedeep.utils import Tokenizer\n&gt;&gt;&gt; texts = ['Machine learning is great', 'but building stuff is even better']\n&gt;&gt;&gt; tok = Tokenizer()\n&gt;&gt;&gt; tok.process_all(texts)\n[['xxmaj', 'machine', 'learning', 'is', 'great'], ['but', 'building', 'stuff', 'is', 'even', 'better']]\n</code></pre> <p> NOTE: Note the token <code>TK_MAJ</code> (<code>xxmaj</code>), used to indicate the next word begins with a capital in the original text. For more details of special tokens please see the <code>fastai</code> docs.</p> <p>Returns:</p> Type Description <code>List[List[str]]</code> <p>List containing lists of tokens. One list per \"document\"</p> Source code in <code>pytorch_widedeep/utils/fastai_transforms.py</code> <pre><code>def process_all(self, texts: Collection[str]) -&gt; List[List[str]]:\n    r\"\"\"Process a list of texts. Parallel execution of ``process_text``.\n\n    Examples\n    --------\n    &gt;&gt;&gt; from pytorch_widedeep.utils import Tokenizer\n    &gt;&gt;&gt; texts = ['Machine learning is great', 'but building stuff is even better']\n    &gt;&gt;&gt; tok = Tokenizer()\n    &gt;&gt;&gt; tok.process_all(texts)\n    [['xxmaj', 'machine', 'learning', 'is', 'great'], ['but', 'building', 'stuff', 'is', 'even', 'better']]\n\n    :information_source: **NOTE**:\n    Note the token ``TK_MAJ`` (`xxmaj`), used to indicate the\n    next word begins with a capital in the original text. For more\n    details of special tokens please see the [``fastai`` docs](https://docs.fast.ai/text.core.html#Tokenizing).\n\n    Returns\n    -------\n    List[List[str]]\n        List containing lists of tokens. One list per \"_document_\"\n\n    \"\"\"\n\n    if self.n_cpus &lt;= 1:\n        return self._process_all_1(texts)\n    with ProcessPoolExecutor(self.n_cpus) as e:\n        return sum(\n            e.map(self._process_all_1, partition_by_cores(texts, self.n_cpus)), []\n        )\n</code></pre>"},{"location":"pytorch-widedeep/utils/fastai_transforms.html#pytorch_widedeep.utils.fastai_transforms.Vocab","title":"Vocab","text":"<p>Contains the correspondence between numbers and tokens.</p> <p>Parameters:</p> Name Type Description Default <code>max_vocab</code> <code>int</code> <p>maximum vocabulary size</p> required <code>min_freq</code> <code>int</code> <p>minimum frequency for a token to be considereds</p> required <code>pad_idx</code> <code>Optional[int]</code> <p>padding index. If <code>None</code>, Fastai's Tokenizer leaves the 0 index for the unknown token ('xxunk') and defaults to 1 for the padding token ('xxpad').</p> <code>None</code> <p>Attributes:</p> Name Type Description <code>itos</code> <code>Collection</code> <p><code>index to str</code>. Collection of strings that are the tokens of the vocabulary</p> <code>stoi</code> <code>defaultdict</code> <p><code>str to index</code>. Dictionary containing the tokens of the vocabulary and their corresponding index</p> Source code in <code>pytorch_widedeep/utils/fastai_transforms.py</code> <pre><code>class Vocab:\n    r\"\"\"Contains the correspondence between numbers and tokens.\n\n    Parameters\n    ----------\n    max_vocab: int\n        maximum vocabulary size\n    min_freq: int\n        minimum frequency for a token to be considereds\n    pad_idx: int, Optional, default = None\n        padding index. If `None`, Fastai's Tokenizer leaves the 0 index\n        for the unknown token (_'xxunk'_) and defaults to 1 for the padding\n        token (_'xxpad'_).\n\n    Attributes\n    ----------\n    itos: Collection\n        `index to str`. Collection of strings that are the tokens of the\n        vocabulary\n    stoi: defaultdict\n        `str to index`. Dictionary containing the tokens of the vocabulary and\n        their corresponding index\n    \"\"\"\n\n    def __init__(\n        self,\n        max_vocab: int,\n        min_freq: int,\n        pad_idx: Optional[int] = None,\n        special_cases: Optional[Collection[str]] = None,\n    ):\n        self.max_vocab = max_vocab\n        self.min_freq = min_freq\n        self.pad_idx = pad_idx\n        self.special_cases = (\n            special_cases if special_cases is not None else defaults.text_spec_tok\n        )\n\n    def create(\n        self,\n        tokens: Tokens,\n    ) -&gt; \"Vocab\":\n        r\"\"\"Create a vocabulary object from a set of tokens.\n\n        Parameters\n        ----------\n        tokens: Tokens\n            Custom type: ``Collection[Collection[str]]``  see\n            `pytorch_widedeep.wdtypes`. Collection of collection of\n            strings (e.g. list of tokenized sentences)\n\n        Examples\n        --------\n        &gt;&gt;&gt; from pytorch_widedeep.utils import Tokenizer, Vocab\n        &gt;&gt;&gt; texts = ['Machine learning is great', 'but building stuff is even better']\n        &gt;&gt;&gt; tokens = Tokenizer().process_all(texts)\n        &gt;&gt;&gt; vocab = Vocab(max_vocab=18, min_freq=1).create(tokens)\n        &gt;&gt;&gt; vocab.numericalize(['machine', 'learning', 'is', 'great'])\n        [10, 11, 9, 12]\n        &gt;&gt;&gt; vocab.textify([10, 11, 9, 12])\n        'machine learning is great'\n\n        :information_source: **NOTE**:\n        Note the many special tokens that ``fastai``'s' tokenizer adds. These\n        are particularly useful when building Language models and/or in\n        classification/Regression tasks. Please see the [``fastai`` docs](https://docs.fast.ai/text.core.html#Tokenizing).\n\n        Returns\n        -------\n        Vocab\n            An instance of a `Vocab` object\n        \"\"\"\n\n        freq = Counter(p for o in tokens for p in o)\n        itos = [o for o, c in freq.most_common(self.max_vocab) if c &gt;= self.min_freq]\n        for o in reversed(self.special_cases):\n            if o in itos:\n                itos.remove(o)\n            itos.insert(0, o)\n\n        if self.pad_idx is not None and self.pad_idx != 1:\n            itos.remove(PAD)\n            itos.insert(self.pad_idx, PAD)\n            # get the new 'xxunk' index\n            xxunk_idx = np.where([el == \"xxunk\" for el in itos])[0][0]\n        else:\n            xxunk_idx = 0\n\n        itos = itos[: self.max_vocab]\n        if (\n            len(itos) &lt; self.max_vocab\n        ):  # Make sure vocab size is a multiple of 8 for fast mixed precision training\n            while len(itos) % 8 != 0:\n                itos.append(\"xxfake\")\n\n        self.itos = itos\n        self.stoi = defaultdict(\n            lambda: xxunk_idx, {v: k for k, v in enumerate(self.itos)}\n        )\n\n        return self\n\n    def fit(\n        self,\n        tokens: Tokens,\n    ) -&gt; \"Vocab\":\n        \"\"\"\n        Calls the `create` method. I simply want to honor fast ai naming, but\n        for consistency with the rest of the library I am including a fit method\n        \"\"\"\n        return self.create(tokens)\n\n    def numericalize(self, t: Collection[str]) -&gt; List[int]:\n        \"\"\"Convert a list of tokens ``t`` to their ids.\n\n        Returns\n        -------\n        List[int]\n            List of '_numericalsed_' tokens\n        \"\"\"\n        return [self.stoi[w] for w in t]\n\n    def transform(self, t: Collection[str]) -&gt; List[int]:\n        \"\"\"\n        Calls the `numericalize` method. I simply want to honor fast ai naming,\n        but for consistency with the rest of the library I am including a\n        transform method\n        \"\"\"\n        return self.numericalize(t)\n\n    def textify(self, nums: Collection[int], sep=\" \") -&gt; Union[str, List[str]]:\n        \"\"\"Convert a list of ``nums`` (or indexes) to their tokens.\n\n        Returns\n        -------\n        List[str]\n            List of tokens\n        \"\"\"\n        return (\n            sep.join([self.itos[i] for i in nums])\n            if sep is not None\n            else [self.itos[i] for i in nums]\n        )\n\n    def inverse_transform(\n        self, nums: Collection[int], sep=\" \"\n    ) -&gt; Union[str, List[str]]:\n        \"\"\"\n        Calls the `textify` method. I simply want to honor fast ai naming, but\n        for consistency with the rest of the library I am including an\n        inverse_transform method\n        \"\"\"\n        # I simply want to honor fast ai naming, but for consistency with the\n        # rest of the library I am including an inverse_transform method\n        return self.textify(nums, sep)\n\n    def __getstate__(self):\n        return {\"itos\": self.itos}\n\n    def __setstate__(self, state: dict):\n        self.itos = state[\"itos\"]\n        self.stoi = defaultdict(int, {v: k for k, v in enumerate(self.itos)})\n</code></pre>"},{"location":"pytorch-widedeep/utils/fastai_transforms.html#pytorch_widedeep.utils.fastai_transforms.Vocab.create","title":"create","text":"<pre><code>create(tokens)\n</code></pre> <p>Create a vocabulary object from a set of tokens.</p> <p>Parameters:</p> Name Type Description Default <code>tokens</code> <code>Tokens</code> <p>Custom type: <code>Collection[Collection[str]]</code>  see <code>pytorch_widedeep.wdtypes</code>. Collection of collection of strings (e.g. list of tokenized sentences)</p> required <p>Examples:</p> <pre><code>&gt;&gt;&gt; from pytorch_widedeep.utils import Tokenizer, Vocab\n&gt;&gt;&gt; texts = ['Machine learning is great', 'but building stuff is even better']\n&gt;&gt;&gt; tokens = Tokenizer().process_all(texts)\n&gt;&gt;&gt; vocab = Vocab(max_vocab=18, min_freq=1).create(tokens)\n&gt;&gt;&gt; vocab.numericalize(['machine', 'learning', 'is', 'great'])\n[10, 11, 9, 12]\n&gt;&gt;&gt; vocab.textify([10, 11, 9, 12])\n'machine learning is great'\n</code></pre> <p> NOTE: Note the many special tokens that <code>fastai</code>'s' tokenizer adds. These are particularly useful when building Language models and/or in classification/Regression tasks. Please see the <code>fastai</code> docs.</p> <p>Returns:</p> Type Description <code>Vocab</code> <p>An instance of a <code>Vocab</code> object</p> Source code in <code>pytorch_widedeep/utils/fastai_transforms.py</code> <pre><code>def create(\n    self,\n    tokens: Tokens,\n) -&gt; \"Vocab\":\n    r\"\"\"Create a vocabulary object from a set of tokens.\n\n    Parameters\n    ----------\n    tokens: Tokens\n        Custom type: ``Collection[Collection[str]]``  see\n        `pytorch_widedeep.wdtypes`. Collection of collection of\n        strings (e.g. list of tokenized sentences)\n\n    Examples\n    --------\n    &gt;&gt;&gt; from pytorch_widedeep.utils import Tokenizer, Vocab\n    &gt;&gt;&gt; texts = ['Machine learning is great', 'but building stuff is even better']\n    &gt;&gt;&gt; tokens = Tokenizer().process_all(texts)\n    &gt;&gt;&gt; vocab = Vocab(max_vocab=18, min_freq=1).create(tokens)\n    &gt;&gt;&gt; vocab.numericalize(['machine', 'learning', 'is', 'great'])\n    [10, 11, 9, 12]\n    &gt;&gt;&gt; vocab.textify([10, 11, 9, 12])\n    'machine learning is great'\n\n    :information_source: **NOTE**:\n    Note the many special tokens that ``fastai``'s' tokenizer adds. These\n    are particularly useful when building Language models and/or in\n    classification/Regression tasks. Please see the [``fastai`` docs](https://docs.fast.ai/text.core.html#Tokenizing).\n\n    Returns\n    -------\n    Vocab\n        An instance of a `Vocab` object\n    \"\"\"\n\n    freq = Counter(p for o in tokens for p in o)\n    itos = [o for o, c in freq.most_common(self.max_vocab) if c &gt;= self.min_freq]\n    for o in reversed(self.special_cases):\n        if o in itos:\n            itos.remove(o)\n        itos.insert(0, o)\n\n    if self.pad_idx is not None and self.pad_idx != 1:\n        itos.remove(PAD)\n        itos.insert(self.pad_idx, PAD)\n        # get the new 'xxunk' index\n        xxunk_idx = np.where([el == \"xxunk\" for el in itos])[0][0]\n    else:\n        xxunk_idx = 0\n\n    itos = itos[: self.max_vocab]\n    if (\n        len(itos) &lt; self.max_vocab\n    ):  # Make sure vocab size is a multiple of 8 for fast mixed precision training\n        while len(itos) % 8 != 0:\n            itos.append(\"xxfake\")\n\n    self.itos = itos\n    self.stoi = defaultdict(\n        lambda: xxunk_idx, {v: k for k, v in enumerate(self.itos)}\n    )\n\n    return self\n</code></pre>"},{"location":"pytorch-widedeep/utils/fastai_transforms.html#pytorch_widedeep.utils.fastai_transforms.Vocab.fit","title":"fit","text":"<pre><code>fit(tokens)\n</code></pre> <p>Calls the <code>create</code> method. I simply want to honor fast ai naming, but for consistency with the rest of the library I am including a fit method</p> Source code in <code>pytorch_widedeep/utils/fastai_transforms.py</code> <pre><code>def fit(\n    self,\n    tokens: Tokens,\n) -&gt; \"Vocab\":\n    \"\"\"\n    Calls the `create` method. I simply want to honor fast ai naming, but\n    for consistency with the rest of the library I am including a fit method\n    \"\"\"\n    return self.create(tokens)\n</code></pre>"},{"location":"pytorch-widedeep/utils/fastai_transforms.html#pytorch_widedeep.utils.fastai_transforms.Vocab.numericalize","title":"numericalize","text":"<pre><code>numericalize(t)\n</code></pre> <p>Convert a list of tokens <code>t</code> to their ids.</p> <p>Returns:</p> Type Description <code>List[int]</code> <p>List of 'numericalsed' tokens</p> Source code in <code>pytorch_widedeep/utils/fastai_transforms.py</code> <pre><code>def numericalize(self, t: Collection[str]) -&gt; List[int]:\n    \"\"\"Convert a list of tokens ``t`` to their ids.\n\n    Returns\n    -------\n    List[int]\n        List of '_numericalsed_' tokens\n    \"\"\"\n    return [self.stoi[w] for w in t]\n</code></pre>"},{"location":"pytorch-widedeep/utils/fastai_transforms.html#pytorch_widedeep.utils.fastai_transforms.Vocab.transform","title":"transform","text":"<pre><code>transform(t)\n</code></pre> <p>Calls the <code>numericalize</code> method. I simply want to honor fast ai naming, but for consistency with the rest of the library I am including a transform method</p> Source code in <code>pytorch_widedeep/utils/fastai_transforms.py</code> <pre><code>def transform(self, t: Collection[str]) -&gt; List[int]:\n    \"\"\"\n    Calls the `numericalize` method. I simply want to honor fast ai naming,\n    but for consistency with the rest of the library I am including a\n    transform method\n    \"\"\"\n    return self.numericalize(t)\n</code></pre>"},{"location":"pytorch-widedeep/utils/fastai_transforms.html#pytorch_widedeep.utils.fastai_transforms.Vocab.textify","title":"textify","text":"<pre><code>textify(nums, sep=' ')\n</code></pre> <p>Convert a list of <code>nums</code> (or indexes) to their tokens.</p> <p>Returns:</p> Type Description <code>List[str]</code> <p>List of tokens</p> Source code in <code>pytorch_widedeep/utils/fastai_transforms.py</code> <pre><code>def textify(self, nums: Collection[int], sep=\" \") -&gt; Union[str, List[str]]:\n    \"\"\"Convert a list of ``nums`` (or indexes) to their tokens.\n\n    Returns\n    -------\n    List[str]\n        List of tokens\n    \"\"\"\n    return (\n        sep.join([self.itos[i] for i in nums])\n        if sep is not None\n        else [self.itos[i] for i in nums]\n    )\n</code></pre>"},{"location":"pytorch-widedeep/utils/fastai_transforms.html#pytorch_widedeep.utils.fastai_transforms.Vocab.inverse_transform","title":"inverse_transform","text":"<pre><code>inverse_transform(nums, sep=' ')\n</code></pre> <p>Calls the <code>textify</code> method. I simply want to honor fast ai naming, but for consistency with the rest of the library I am including an inverse_transform method</p> Source code in <code>pytorch_widedeep/utils/fastai_transforms.py</code> <pre><code>def inverse_transform(\n    self, nums: Collection[int], sep=\" \"\n) -&gt; Union[str, List[str]]:\n    \"\"\"\n    Calls the `textify` method. I simply want to honor fast ai naming, but\n    for consistency with the rest of the library I am including an\n    inverse_transform method\n    \"\"\"\n    # I simply want to honor fast ai naming, but for consistency with the\n    # rest of the library I am including an inverse_transform method\n    return self.textify(nums, sep)\n</code></pre>"},{"location":"pytorch-widedeep/utils/image_utils.html","title":"Image utils","text":"<p>SimplePreprocessor and AspectAwarePreprocessor are directly taked from the great series of Books `Deep Learning for Computer Vision by Adrian. Therefore, all credit for the code in the <code>image_utils</code> module goes to Adrian Rosebrock.</p>"},{"location":"pytorch-widedeep/utils/image_utils.html#pytorch_widedeep.utils.image_utils.AspectAwarePreprocessor","title":"AspectAwarePreprocessor","text":"<p>Class to resize an image to a certain width and height taking into account the image aspect ratio</p> <p>Parameters:</p> Name Type Description Default <code>width</code> <code>int</code> <p>output width</p> required <code>height</code> <code>int</code> <p>output height</p> required <code>inter</code> <p><code>opencv</code> interpolation method. See <code>opencv</code> <code>InterpolationFlags</code>.</p> <code>INTER_AREA</code> Source code in <code>pytorch_widedeep/utils/image_utils.py</code> <pre><code>class AspectAwarePreprocessor:\n    r\"\"\"Class to resize an image to a certain width and height taking into account\n    the image aspect ratio\n\n    Parameters\n    ----------\n    width: int\n        output width\n    height: int\n        output height\n    inter: interpolation method,  default = ``cv2.INTER_AREA``\n        ``opencv`` interpolation method. See ``opencv``\n        `InterpolationFlags`.\n    \"\"\"\n\n    def __init__(self, width: int, height: int, inter=cv2.INTER_AREA):\n        self.width = width\n        self.height = height\n        self.inter = inter\n\n    def preprocess(self, image: np.ndarray) -&gt; np.ndarray:\n        r\"\"\"Returns the resized input image taking into account the image aspect ratio\n\n        Parameters\n        ----------\n        image: np.ndarray\n            Input image to be resized\n\n        Examples\n        --------\n        &gt;&gt;&gt; import cv2\n        &gt;&gt;&gt; from pytorch_widedeep.utils import AspectAwarePreprocessor\n        &gt;&gt;&gt; img = cv2.imread(\"tests/test_data_utils/images/galaxy1.png\")\n        &gt;&gt;&gt; img.shape\n        (694, 890, 3)\n        &gt;&gt;&gt; app = AspectAwarePreprocessor(width=224, height=224)\n        &gt;&gt;&gt; resized_img = app.preprocess(img)\n        &gt;&gt;&gt; resized_img.shape\n        (224, 224, 3)\n\n        Returns\n        -------\n        np.ndarray\n            Resized image according to its original image aspect ratio\n        \"\"\"\n        (h, w) = image.shape[:2]\n        dW = 0\n        dH = 0\n\n        if w &lt; h:\n            image = resize(image, width=self.width, inter=self.inter)\n            dH = int((image.shape[0] - self.height) / 2.0)\n        else:\n            image = resize(image, height=self.height, inter=self.inter)\n            dW = int((image.shape[1] - self.width) / 2.0)\n\n        (h, w) = image.shape[:2]\n        image = image[dH : h - dH, dW : w - dW]\n\n        resized_image = cv2.resize(\n            image, (self.width, self.height), interpolation=self.inter\n        )\n\n        return resized_image\n</code></pre>"},{"location":"pytorch-widedeep/utils/image_utils.html#pytorch_widedeep.utils.image_utils.AspectAwarePreprocessor.preprocess","title":"preprocess","text":"<pre><code>preprocess(image)\n</code></pre> <p>Returns the resized input image taking into account the image aspect ratio</p> <p>Parameters:</p> Name Type Description Default <code>image</code> <code>ndarray</code> <p>Input image to be resized</p> required <p>Examples:</p> <pre><code>&gt;&gt;&gt; import cv2\n&gt;&gt;&gt; from pytorch_widedeep.utils import AspectAwarePreprocessor\n&gt;&gt;&gt; img = cv2.imread(\"tests/test_data_utils/images/galaxy1.png\")\n&gt;&gt;&gt; img.shape\n(694, 890, 3)\n&gt;&gt;&gt; app = AspectAwarePreprocessor(width=224, height=224)\n&gt;&gt;&gt; resized_img = app.preprocess(img)\n&gt;&gt;&gt; resized_img.shape\n(224, 224, 3)\n</code></pre> <p>Returns:</p> Type Description <code>ndarray</code> <p>Resized image according to its original image aspect ratio</p> Source code in <code>pytorch_widedeep/utils/image_utils.py</code> <pre><code>def preprocess(self, image: np.ndarray) -&gt; np.ndarray:\n    r\"\"\"Returns the resized input image taking into account the image aspect ratio\n\n    Parameters\n    ----------\n    image: np.ndarray\n        Input image to be resized\n\n    Examples\n    --------\n    &gt;&gt;&gt; import cv2\n    &gt;&gt;&gt; from pytorch_widedeep.utils import AspectAwarePreprocessor\n    &gt;&gt;&gt; img = cv2.imread(\"tests/test_data_utils/images/galaxy1.png\")\n    &gt;&gt;&gt; img.shape\n    (694, 890, 3)\n    &gt;&gt;&gt; app = AspectAwarePreprocessor(width=224, height=224)\n    &gt;&gt;&gt; resized_img = app.preprocess(img)\n    &gt;&gt;&gt; resized_img.shape\n    (224, 224, 3)\n\n    Returns\n    -------\n    np.ndarray\n        Resized image according to its original image aspect ratio\n    \"\"\"\n    (h, w) = image.shape[:2]\n    dW = 0\n    dH = 0\n\n    if w &lt; h:\n        image = resize(image, width=self.width, inter=self.inter)\n        dH = int((image.shape[0] - self.height) / 2.0)\n    else:\n        image = resize(image, height=self.height, inter=self.inter)\n        dW = int((image.shape[1] - self.width) / 2.0)\n\n    (h, w) = image.shape[:2]\n    image = image[dH : h - dH, dW : w - dW]\n\n    resized_image = cv2.resize(\n        image, (self.width, self.height), interpolation=self.inter\n    )\n\n    return resized_image\n</code></pre>"},{"location":"pytorch-widedeep/utils/image_utils.html#pytorch_widedeep.utils.image_utils.SimplePreprocessor","title":"SimplePreprocessor","text":"<p>Class to resize an image to a certain width and height</p> <p>Parameters:</p> Name Type Description Default <code>width</code> <code>int</code> <p>output width</p> required <code>height</code> <code>int</code> <p>output height</p> required <code>inter</code> <p><code>opencv</code> interpolation method. See <code>opencv</code> <code>InterpolationFlags</code>.</p> <code>INTER_AREA</code> Source code in <code>pytorch_widedeep/utils/image_utils.py</code> <pre><code>class SimplePreprocessor:\n    r\"\"\"Class to resize an image to a certain width and height\n\n    Parameters\n    ----------\n    width: int\n        output width\n    height: int\n        output height\n    inter: interpolation method, default = ``cv2.INTER_AREA``\n        ``opencv`` interpolation method. See ``opencv``\n        `InterpolationFlags`.\n    \"\"\"\n\n    def __init__(self, width: int, height: int, inter=cv2.INTER_AREA):\n        self.width = width\n        self.height = height\n        self.inter = inter\n\n    def preprocess(self, image: np.ndarray) -&gt; np.ndarray:\n        r\"\"\"Returns the resized input image\n\n        Parameters\n        ----------\n        image: np.ndarray\n            Input image to be resized\n\n        Returns\n        -------\n        np.ndarray\n            Resized image\n\n        \"\"\"\n        resized_image = cv2.resize(\n            image, (self.width, self.height), interpolation=self.inter\n        )\n\n        return resized_image\n</code></pre>"},{"location":"pytorch-widedeep/utils/image_utils.html#pytorch_widedeep.utils.image_utils.SimplePreprocessor.preprocess","title":"preprocess","text":"<pre><code>preprocess(image)\n</code></pre> <p>Returns the resized input image</p> <p>Parameters:</p> Name Type Description Default <code>image</code> <code>ndarray</code> <p>Input image to be resized</p> required <p>Returns:</p> Type Description <code>ndarray</code> <p>Resized image</p> Source code in <code>pytorch_widedeep/utils/image_utils.py</code> <pre><code>def preprocess(self, image: np.ndarray) -&gt; np.ndarray:\n    r\"\"\"Returns the resized input image\n\n    Parameters\n    ----------\n    image: np.ndarray\n        Input image to be resized\n\n    Returns\n    -------\n    np.ndarray\n        Resized image\n\n    \"\"\"\n    resized_image = cv2.resize(\n        image, (self.width, self.height), interpolation=self.inter\n    )\n\n    return resized_image\n</code></pre>"},{"location":"pytorch-widedeep/utils/text_utils.html","title":"Text utils","text":"<p>Collection of helper function that facilitate processing text.</p>"},{"location":"pytorch-widedeep/utils/text_utils.html#pytorch_widedeep.utils.text_utils.simple_preprocess","title":"simple_preprocess","text":"<pre><code>simple_preprocess(\n    doc, lower=False, deacc=False, min_len=2, max_len=15\n)\n</code></pre> <p>This is <code>Gensim</code>'s <code>simple_preprocess</code> with a <code>lower</code> param to indicate wether or not to lower case all the token in the doc</p> <p>For more information see: <code>Gensim</code> utils module</p> <p>Parameters:</p> Name Type Description Default <code>doc</code> <code>str</code> <p>Input document.</p> required <code>lower</code> <code>bool</code> <p>Lower case tokens in the input doc</p> <code>False</code> <code>deacc</code> <code>bool</code> <p>Remove accent marks from tokens using <code>Gensim</code>'s <code>deaccent</code></p> <code>False</code> <code>min_len</code> <code>int</code> <p>Minimum length of token (inclusive). Shorter tokens are discarded.</p> <code>2</code> <code>max_len</code> <code>int</code> <p>Maximum length of token in result (inclusive). Longer tokens are discarded.</p> <code>15</code> <p>Examples:</p> <pre><code>&gt;&gt;&gt; from pytorch_widedeep.utils import simple_preprocess\n&gt;&gt;&gt; simple_preprocess('Machine learning is great')\n['Machine', 'learning', 'is', 'great']\n</code></pre> <p>Returns:</p> Type Description <code>List[str]</code> <p>List with the processed tokens</p> Source code in <code>pytorch_widedeep/utils/text_utils.py</code> <pre><code>def simple_preprocess(\n    doc: str,\n    lower: bool = False,\n    deacc: bool = False,\n    min_len: int = 2,\n    max_len: int = 15,\n) -&gt; List[str]:\n    r\"\"\"\n    This is `Gensim`'s `simple_preprocess` with a `lower` param to\n    indicate wether or not to lower case all the token in the doc\n\n    For more information see: `Gensim` [utils module](https://radimrehurek.com/gensim/utils.html)\n\n    Parameters\n    ----------\n    doc: str\n        Input document.\n    lower: bool, default = False\n        Lower case tokens in the input doc\n    deacc: bool, default = False\n        Remove accent marks from tokens using `Gensim`'s `deaccent`\n    min_len: int, default = 2\n        Minimum length of token (inclusive). Shorter tokens are discarded.\n    max_len: int, default = 15\n        Maximum length of token in result (inclusive). Longer tokens are discarded.\n\n    Examples\n    --------\n    &gt;&gt;&gt; from pytorch_widedeep.utils import simple_preprocess\n    &gt;&gt;&gt; simple_preprocess('Machine learning is great')\n    ['Machine', 'learning', 'is', 'great']\n\n    Returns\n    -------\n    List[str]\n        List with the processed tokens\n    \"\"\"\n    tokens = [\n        token\n        for token in tokenize(doc, lower=lower, deacc=deacc, errors=\"ignore\")\n        if min_len &lt;= len(token) &lt;= max_len and not token.startswith(\"_\")\n    ]\n    return tokens\n</code></pre>"},{"location":"pytorch-widedeep/utils/text_utils.html#pytorch_widedeep.utils.text_utils.get_texts","title":"get_texts","text":"<pre><code>get_texts(texts, already_processed=False, n_cpus=None)\n</code></pre> <p>Tokenization using <code>Fastai</code>'s <code>Tokenizer</code> because it does a series of very convenients things during the tokenization process</p> <p>See <code>pytorch_widedeep.utils.fastai_utils.Tokenizer</code></p> <p> NOTE: <code>get_texts</code> uses <code>pytorch_widedeep.utils.fastai_transforms.Tokenizer</code>. Such tokenizer uses a series of convenient processing steps, including the  addition of some special tokens, such as <code>TK_MAJ</code> (<code>xxmaj</code>), used to indicate the next word begins with a capital in the original text. For more details of special tokens please see the <code>fastai</code> `docs</p> <p>Parameters:</p> Name Type Description Default <code>texts</code> <code>List[str]</code> <p>List of str with the texts (or documents). One str per document</p> required <code>already_processed</code> <code>Optional[bool]</code> <p>Boolean indicating if the text is already processed and we simply want to tokenize it. This parameter is thought for those cases where the input sequences might not be text (but IDs, or anything else) and we just want to tokenize it</p> <code>False</code> <code>n_cpus</code> <code>Optional[int]</code> <p>number of CPUs to used during the tokenization process</p> <code>None</code> <p>Examples:</p> <pre><code>&gt;&gt;&gt; from pytorch_widedeep.utils import get_texts\n&gt;&gt;&gt; texts = ['Machine learning is great', 'but building stuff is even better']\n&gt;&gt;&gt; get_texts(texts)\n[['xxmaj', 'machine', 'learning', 'is', 'great'], ['but', 'building', 'stuff', 'is', 'even', 'better']]\n</code></pre> <p>Returns:</p> Type Description <code>List[List[str]]</code> <p>List of lists, one list per 'document' containing its corresponding tokens</p> Source code in <code>pytorch_widedeep/utils/text_utils.py</code> <pre><code>def get_texts(\n    texts: List[str],\n    already_processed: Optional[bool] = False,\n    n_cpus: Optional[int] = None,\n) -&gt; List[List[str]]:\n    r\"\"\"Tokenization using `Fastai`'s `Tokenizer` because it does a\n    series of very convenients things during the tokenization process\n\n    See `pytorch_widedeep.utils.fastai_utils.Tokenizer`\n\n    :information_source: **NOTE**:\n    `get_texts` uses `pytorch_widedeep.utils.fastai_transforms.Tokenizer`.\n    Such tokenizer uses a series of convenient processing steps, including\n    the  addition of some special tokens, such as `TK_MAJ` (`xxmaj`), used to\n    indicate the next word begins with a capital in the original text. For more\n    details of special tokens please see the [`fastai` `docs](https://docs.fast.ai/text.core.html#Tokenizing)\n\n    Parameters\n    ----------\n    texts: List\n        List of str with the texts (or documents). One str per document\n    already_processed: bool, Optional, default = False\n        Boolean indicating if the text is already processed and we simply want\n        to tokenize it. This parameter is thought for those cases where the\n        input sequences might not be text (but IDs, or anything else) and we\n        just want to tokenize it\n    n_cpus: int, Optional, default = None\n        number of CPUs to used during the tokenization process\n\n    Examples\n    --------\n    &gt;&gt;&gt; from pytorch_widedeep.utils import get_texts\n    &gt;&gt;&gt; texts = ['Machine learning is great', 'but building stuff is even better']\n    &gt;&gt;&gt; get_texts(texts)\n    [['xxmaj', 'machine', 'learning', 'is', 'great'], ['but', 'building', 'stuff', 'is', 'even', 'better']]\n\n    Returns\n    -------\n    List[List[str]]\n        List of lists, one list per '_document_' containing its corresponding tokens\n    \"\"\"\n\n    num_cpus = n_cpus if n_cpus is not None else os.cpu_count()\n\n    if not already_processed:\n        processed_texts = [\" \".join(simple_preprocess(t)) for t in texts]\n    else:\n        processed_texts = texts\n    tok = Tokenizer(n_cpus=num_cpus).process_all(processed_texts)\n    return tok\n</code></pre>"},{"location":"pytorch-widedeep/utils/text_utils.html#pytorch_widedeep.utils.text_utils.pad_sequences","title":"pad_sequences","text":"<pre><code>pad_sequences(seq, maxlen, pad_first=True, pad_idx=1)\n</code></pre> <p>Given a List of tokenized and <code>numericalised</code> sequences it will return padded sequences according to the input parameters.</p> <p>Parameters:</p> Name Type Description Default <code>seq</code> <code>List[int]</code> <p>List of int with the <code>numericalised</code> tokens</p> required <code>maxlen</code> <code>int</code> <p>Maximum length of the padded sequences</p> required <code>pad_first</code> <code>bool</code> <p>Indicates whether the padding index will be added at the beginning or the end of the sequences</p> <code>True</code> <code>pad_idx</code> <code>int</code> <p>padding index. Fastai's Tokenizer leaves 0 for the 'unknown' token.</p> <code>1</code> <p>Examples:</p> <pre><code>&gt;&gt;&gt; from pytorch_widedeep.utils import pad_sequences\n&gt;&gt;&gt; seq = [1,2,3]\n&gt;&gt;&gt; pad_sequences(seq, maxlen=5, pad_idx=0)\narray([0, 0, 1, 2, 3], dtype=int32)\n</code></pre> <p>Returns:</p> Type Description <code>ndarray</code> <p>numpy array with the padded sequences</p> Source code in <code>pytorch_widedeep/utils/text_utils.py</code> <pre><code>def pad_sequences(\n    seq: List[int], maxlen: int, pad_first: bool = True, pad_idx: int = 1\n) -&gt; np.ndarray:\n    r\"\"\"\n    Given a List of tokenized and `numericalised` sequences it will return\n    padded sequences according to the input parameters.\n\n    Parameters\n    ----------\n    seq: List\n        List of int with the `numericalised` tokens\n    maxlen: int\n        Maximum length of the padded sequences\n    pad_first: bool,  default = True\n        Indicates whether the padding index will be added at the beginning or the\n        end of the sequences\n    pad_idx: int, default = 1\n        padding index. Fastai's Tokenizer leaves 0 for the 'unknown' token.\n\n    Examples\n    --------\n    &gt;&gt;&gt; from pytorch_widedeep.utils import pad_sequences\n    &gt;&gt;&gt; seq = [1,2,3]\n    &gt;&gt;&gt; pad_sequences(seq, maxlen=5, pad_idx=0)\n    array([0, 0, 1, 2, 3], dtype=int32)\n\n    Returns\n    -------\n    np.ndarray\n        numpy array with the padded sequences\n    \"\"\"\n    if len(seq) == 0:\n        return np.zeros(maxlen, dtype=\"int32\") + pad_idx\n    elif len(seq) &gt;= maxlen:\n        res = np.array(seq[-maxlen:]).astype(\"int32\")\n        return res\n    else:\n        res = np.zeros(maxlen, dtype=\"int32\") + pad_idx\n        if pad_first:\n            res[-len(seq) :] = seq\n        else:\n            res[: len(seq) :] = seq\n        return res\n</code></pre>"},{"location":"pytorch-widedeep/utils/text_utils.html#pytorch_widedeep.utils.text_utils.build_embeddings_matrix","title":"build_embeddings_matrix","text":"<pre><code>build_embeddings_matrix(\n    vocab, word_vectors_path, min_freq, verbose=1\n)\n</code></pre> <p>Build the embedding matrix using pretrained word vectors.</p> <p>Returns pretrained word embeddings. If a word in our vocabulary is not among the pretrained embeddings it will be assigned the mean pretrained word-embeddings vector</p> <p>Parameters:</p> Name Type Description Default <code>vocab</code> <code>Union[Vocab, ChunkVocab]</code> <p>see <code>pytorch_widedeep.utils.fastai_utils.Vocab</code></p> required <code>word_vectors_path</code> <code>str</code> <p>path to the pretrained word embeddings</p> required <code>min_freq</code> <code>int</code> <p>minimum frequency required for a word to be in the vocabulary</p> required <code>verbose</code> <code>int</code> <p>level of verbosity. Set to 0 for no verbosity</p> <code>1</code> <p>Returns:</p> Type Description <code>ndarray</code> <p>Pretrained word embeddings</p> Source code in <code>pytorch_widedeep/utils/text_utils.py</code> <pre><code>def build_embeddings_matrix(\n    vocab: Union[Vocab, ChunkVocab],\n    word_vectors_path: str,\n    min_freq: int,\n    verbose: int = 1,\n) -&gt; np.ndarray:  # pragma: no cover\n    r\"\"\"Build the embedding matrix using pretrained word vectors.\n\n    Returns pretrained word embeddings. If a word in our vocabulary is not\n    among the pretrained embeddings it will be assigned the mean pretrained\n    word-embeddings vector\n\n    Parameters\n    ----------\n    vocab: Vocab\n        see `pytorch_widedeep.utils.fastai_utils.Vocab`\n    word_vectors_path: str\n        path to the pretrained word embeddings\n    min_freq: int\n        minimum frequency required for a word to be in the vocabulary\n    verbose: int,  default=1\n        level of verbosity. Set to 0 for no verbosity\n\n    Returns\n    -------\n    np.ndarray\n        Pretrained word embeddings\n    \"\"\"\n    if not os.path.isfile(word_vectors_path):\n        raise FileNotFoundError(\"{} not found\".format(word_vectors_path))\n    if verbose:\n        print(\"Indexing word vectors...\")\n\n    embeddings_index = {}\n    f = open(word_vectors_path)\n    for line in f:\n        values = line.split()\n        word = values[0]\n        coefs = np.asarray(values[1:], dtype=\"float32\")\n        embeddings_index[word] = coefs\n    f.close()\n\n    if verbose:\n        print(\"Loaded {} word vectors\".format(len(embeddings_index)))\n        print(\"Preparing embeddings matrix...\")\n\n    mean_word_vector = np.mean(list(embeddings_index.values()), axis=0)  # type: ignore[arg-type]\n    embedding_dim = len(list(embeddings_index.values())[0])\n    num_words = len(vocab.itos)\n    embedding_matrix = np.zeros((num_words, embedding_dim))\n    found_words = 0\n    for i, word in enumerate(vocab.itos):\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[i] = embedding_vector\n            found_words += 1\n        else:\n            embedding_matrix[i] = mean_word_vector\n\n    if verbose:\n        print(\n            \"{} words in the vocabulary had {} vectors and appear more than {} times\".format(\n                found_words, word_vectors_path, min_freq\n            )\n        )\n\n    return embedding_matrix.astype(\"float32\")\n</code></pre>"}]}
\ No newline at end of file
diff --git a/mkdocs/site/sitemap.xml b/mkdocs/site/sitemap.xml
index 5a8ffe99..51b090e4 100644
--- a/mkdocs/site/sitemap.xml
+++ b/mkdocs/site/sitemap.xml
@@ -2,178 +2,178 @@
 <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/index.html</loc>
-         <lastmod>2024-10-21</lastmod>
+         <lastmod>2024-11-01</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/contributing.html</loc>
-         <lastmod>2024-10-21</lastmod>
+         <lastmod>2024-11-01</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/installation.html</loc>
-         <lastmod>2024-10-21</lastmod>
+         <lastmod>2024-11-01</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/quick_start.html</loc>
-         <lastmod>2024-10-21</lastmod>
+         <lastmod>2024-11-01</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/examples/01_preprocessors_and_utils.html</loc>
-         <lastmod>2024-10-21</lastmod>
+         <lastmod>2024-11-01</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/examples/02_model_components.html</loc>
-         <lastmod>2024-10-21</lastmod>
+         <lastmod>2024-11-01</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/examples/03_binary_classification_with_defaults.html</loc>
-         <lastmod>2024-10-21</lastmod>
+         <lastmod>2024-11-01</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/examples/04_regression_with_images_and_text.html</loc>
-         <lastmod>2024-10-21</lastmod>
+         <lastmod>2024-11-01</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/examples/05_save_and_load_model_and_artifacts.html</loc>
-         <lastmod>2024-10-21</lastmod>
+         <lastmod>2024-11-01</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/examples/06_finetune_and_warmup.html</loc>
-         <lastmod>2024-10-21</lastmod>
+         <lastmod>2024-11-01</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/examples/07_custom_components.html</loc>
-         <lastmod>2024-10-21</lastmod>
+         <lastmod>2024-11-01</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/examples/08_custom_dataLoader_imbalanced_dataset.html</loc>
-         <lastmod>2024-10-21</lastmod>
+         <lastmod>2024-11-01</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/examples/09_extracting_embeddings.html</loc>
-         <lastmod>2024-10-21</lastmod>
+         <lastmod>2024-11-01</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/examples/10_3rd_party_integration-RayTune_WnB.html</loc>
-         <lastmod>2024-10-21</lastmod>
+         <lastmod>2024-11-01</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/examples/11_auc_multiclass.html</loc>
-         <lastmod>2024-10-21</lastmod>
+         <lastmod>2024-11-01</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html</loc>
-         <lastmod>2024-10-21</lastmod>
+         <lastmod>2024-11-01</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/examples/13_model_uncertainty_prediction.html</loc>
-         <lastmod>2024-10-21</lastmod>
+         <lastmod>2024-11-01</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/examples/14_bayesian_models.html</loc>
-         <lastmod>2024-10-21</lastmod>
+         <lastmod>2024-11-01</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/examples/15_Self_Supervised_Pretraning_pt1.html</loc>
-         <lastmod>2024-10-21</lastmod>
+         <lastmod>2024-11-01</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/examples/15_Self_Supervised_Pretraning_pt2.html</loc>
-         <lastmod>2024-10-21</lastmod>
+         <lastmod>2024-11-01</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/examples/16_Usign_a_custom_hugging_face_model.html</loc>
-         <lastmod>2024-10-21</lastmod>
+         <lastmod>2024-11-01</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/examples/17_feature_importance_via_attention_weights.html</loc>
-         <lastmod>2024-10-21</lastmod>
+         <lastmod>2024-11-01</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/examples/18_wide_and_deep_for_recsys_pt1.html</loc>
-         <lastmod>2024-10-21</lastmod>
+         <lastmod>2024-11-01</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/examples/18_wide_and_deep_for_recsys_pt2.html</loc>
-         <lastmod>2024-10-21</lastmod>
+         <lastmod>2024-11-01</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/examples/19_load_from_folder_functionality.html</loc>
-         <lastmod>2024-10-21</lastmod>
+         <lastmod>2024-11-01</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/examples/20_Using_huggingface_within_widedeep.html</loc>
-         <lastmod>2024-10-21</lastmod>
+         <lastmod>2024-11-01</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/pytorch-widedeep/bayesian_models.html</loc>
-         <lastmod>2024-10-21</lastmod>
+         <lastmod>2024-11-01</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/pytorch-widedeep/bayesian_trainer.html</loc>
-         <lastmod>2024-10-21</lastmod>
+         <lastmod>2024-11-01</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/pytorch-widedeep/callbacks.html</loc>
-         <lastmod>2024-10-21</lastmod>
+         <lastmod>2024-11-01</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/pytorch-widedeep/dataloaders.html</loc>
-         <lastmod>2024-10-21</lastmod>
+         <lastmod>2024-11-01</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/pytorch-widedeep/load_from_folder.html</loc>
-         <lastmod>2024-10-21</lastmod>
+         <lastmod>2024-11-01</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/pytorch-widedeep/losses.html</loc>
-         <lastmod>2024-10-21</lastmod>
+         <lastmod>2024-11-01</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/pytorch-widedeep/metrics.html</loc>
-         <lastmod>2024-10-21</lastmod>
+         <lastmod>2024-11-01</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/pytorch-widedeep/model_components.html</loc>
-         <lastmod>2024-10-21</lastmod>
+         <lastmod>2024-11-01</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/pytorch-widedeep/preprocessing.html</loc>
-         <lastmod>2024-10-21</lastmod>
+         <lastmod>2024-11-01</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/pytorch-widedeep/self_supervised_pretraining.html</loc>
-         <lastmod>2024-10-21</lastmod>
+         <lastmod>2024-11-01</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/pytorch-widedeep/tab2vec.html</loc>
-         <lastmod>2024-10-21</lastmod>
+         <lastmod>2024-11-01</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/pytorch-widedeep/the_rec_module.html</loc>
-         <lastmod>2024-10-21</lastmod>
+         <lastmod>2024-11-01</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/pytorch-widedeep/trainer.html</loc>
-         <lastmod>2024-10-21</lastmod>
+         <lastmod>2024-11-01</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/pytorch-widedeep/utils/index.html</loc>
-         <lastmod>2024-10-21</lastmod>
+         <lastmod>2024-11-01</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/pytorch-widedeep/utils/deeptabular_utils.html</loc>
-         <lastmod>2024-10-21</lastmod>
+         <lastmod>2024-11-01</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/pytorch-widedeep/utils/fastai_transforms.html</loc>
-         <lastmod>2024-10-21</lastmod>
+         <lastmod>2024-11-01</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/pytorch-widedeep/utils/image_utils.html</loc>
-         <lastmod>2024-10-21</lastmod>
+         <lastmod>2024-11-01</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/pytorch-widedeep/utils/text_utils.html</loc>
-         <lastmod>2024-10-21</lastmod>
+         <lastmod>2024-11-01</lastmod>
     </url>
 </urlset>
\ No newline at end of file
diff --git a/mkdocs/site/sitemap.xml.gz b/mkdocs/site/sitemap.xml.gz
index fc00092d75d415151a7962dbe6542540569ce4f3..cd6a171e0efd519ae7446666c59c2683e87f0e6a 100644
GIT binary patch
literal 796
zcmV+%1LOQ3iwFn+8YE`||8r?{Wo=<_E_iKh0M(gMYuhjo$KUrUO5PL4X}h+gO<@m%
zK^bG~80_V^$Y)zrmOP!DG~a$FCv8Vd_p(85eu=F((huGJ@BS#4*Y}+}T2o{j{MDqK
z%_m3XO{lDIt|s4ZKTOXj*ULATLv&2(NM81QHEC1o-xbAXvzf6?)M3xF5SxNcD-T@k
z2aEaaVzPX5Bv+nbcQ;;_E|_JWg=Lz4n?f}0bYm;3i27M1tWrx=Fg&xNu)d=ES(`d{
zS>)gSJ`Hl}LbY7X7bnxQoX$&mWk0rCw|HFmZxBrIDcaSLWLpkPVm&8x4wJ;|up~bR
zYwkd3Vmd4c-DB4~;$mJxAE}SQ5OatO=qr$zIo1NW0GXQ{n9%jXlTT{Mj$vgz#w{3G
zhiz@mm}0Q8sfCJa9Go856NuDA3GySM?XaO;z!Tl8Nyt-RT+@i<0#^Iz=SNYEt<mV&
z)3;DtPigQw+zrNV=(WI|ff+c3PN8tlAJ8gHcq2ikDA~9|hu%;%j!Ng$LVE$CjwpF2
zjSQ$;QI$Krmdz~Za2zY>MI2kOK8eW3Dm49q+uNL(-~D?ns3qWFpp&v_$C9aqYXLt#
z-h8?V%n+h&?kFO|nqfZ!JkFqm_jq3v2a!umC{f?$E?!x4c#HEyc>*ik5?k~;Ts7f7
zg|Fml_&W3y*OsN?e2ydv_gUb5D*qGCg%-}Y@QrQb0UyeGJ2XvZlx9u4*lR&OgPJf6
zk)+@BL9`<Zg0)4!l%zwB-iU0|s_B*G`B;5Mf018=Iz(}DhPSLKRsTq-CQ&bTj<SxD
zqjhlEcGOZsbJ)?@v{k~nm}fzWe)c*VWt1(u^-5aN^Y05)mHu@)DV}23qcS{$&aKeg
zY1_8!!`<l4wUIrSu{u0v(za)vSO;s=QK>rHd4czIHg@n+%ol8&%^^RT!+tb}*6Slp
zxLT~qXrpQ=dz9R=21nZ9RCcQIU|7SS3$KvnTS>OTVbpMl{R(9*OQFqX#IqRXPABDf
aJgj&Z^O^O(z#Bi9(()H#HlZBY8vp=oqmCc|

literal 795
zcmV+$1LXW4iwFpSnH6UO|8r?{Wo=<_E_iKh0M(gIZ`&{ohVT6qMed1{^m}MiV21$%
zieXq=4A_NWWZD)gOP(kv&A%Vzr0vk6yKF!>Ut%kc^g!fGzECc&A6s|0qR2M*tI=#S
z9UYQ4p|rlf8r|Q08lR7@7YCPpbWG_`UiN%7YEtSx6vcYIp0G{SV#kvZ>w--q4_xd9
zi|OQIv^Y4FD^IY!8?H+i%p%XiGEKKhA)02qwk4HB-6Rs0si86$p4d=WU((~GNv*pq
z^6!412068%T+FBQ<MC`dp3mf!-Pm^B;(6h}K`_CmXqSDGZP_o0^_<W-OcJmClKkwg
zc>tk_X}=)!h;8SHi|GuyNL>tum_uYhUxLKUu@=BN$lT<>gtiNwd{RSp1WW5NZotSo
zY%6Pq6oa)*4U|-2@ASYPL!>%Nke>-{i#2Tnp6F3cLY@HQiUuqfu-rvIKZ<H>g+|Yw
zo<e0krQUCG*BINr(*kz}rsot|g~B<1LMt)hjRcvZWaAPYdPC(fDxFgc?FEQBqU4=4
zGN5)zW$yG^Hgh(GqgX;G;@E)oNkl$Wq477|+~v&t?muclodNa+S}BWmD4ANg=J4b5
z&6k_N3?bV3fg&=j7<Mzj^9)LOk9I||7rDfQ67@~);-xkFw>U49$FRf=u|>bdRTJ(L
zxFuJ?ZQoH`S(b|PHIgXYXMuNV_MdRhwQ!!oJ=@v?K9%*Ruj|Yx&6>8c*MfQm6=CWl
zNx$iWXh#$TD~o_BNrxP~5!t#?(<^7^L-iT_MSc;g5XH$E-ms=r{UfECM7`KL$|_2Z
zR>5W4QArKWVMk}vMhWM9ngu2L+39GIQMT;XD``cqzb{l(`uFLic#2_<%J2+2w?y-x
zZQHUBcY{0EM)q39>hPFJ+n%*z9jsADrRr?w4c^Py*xpYu->`8uhx}p=yTKe<ua7k0
za=s#?jjEySQF6=b9chD8*{OzuVGVySyhE06CE0q1QNtnjJCv0yg*KZJ&tjBYos`4z
Zu;N|JXV(7$Z}?zJi(iqXm8{qs000Y=kO2Sy

diff --git a/mkdocs/sources/index.md b/mkdocs/sources/index.md
index 77d61195..fb6aebfb 100644
--- a/mkdocs/sources/index.md
+++ b/mkdocs/sources/index.md
@@ -31,7 +31,7 @@ The content of this document is organized as follows:
 
 - [**pytorch-widedeep**](#pytorch-widedeep)
     - [Introduction](#introduction)
-    - [Architectures](#Architectures)
+    - [Architectures](#architectures)
     - [The ``deeptabular`` component](#the-deeptabular-component)
     - [The ``rec`` module](#the-rec-module)
     - [Text and Images](#text-and-images)
diff --git a/mkdocs/sources/pytorch-widedeep/bayesian_models.md b/mkdocs/sources/pytorch-widedeep/bayesian_models.md
index 2b48c267..63ea1837 100644
--- a/mkdocs/sources/pytorch-widedeep/bayesian_models.md
+++ b/mkdocs/sources/pytorch-widedeep/bayesian_models.md
@@ -10,13 +10,5 @@ The models in this module are based on the publication:
 
 
 ::: pytorch_widedeep.bayesian_models.tabular.bayesian_linear.bayesian_wide.BayesianWide
-    selection:
-        filters:
-            - "!^_"  # exclude all members starting with _
-            - "!^forward$"
 
 ::: pytorch_widedeep.bayesian_models.tabular.bayesian_mlp.bayesian_tab_mlp.BayesianTabMlp
-    selection:
-        filters:
-            - "!^_"  # exclude all members starting with _
-            - "!^forward$"
diff --git a/mkdocs/sources/pytorch-widedeep/load_from_folder.md b/mkdocs/sources/pytorch-widedeep/load_from_folder.md
index 4cc7523c..c61bc5f1 100644
--- a/mkdocs/sources/pytorch-widedeep/load_from_folder.md
+++ b/mkdocs/sources/pytorch-widedeep/load_from_folder.md
@@ -19,18 +19,27 @@ disk, including the extension (`.jpg`, `.png`, etc...).
 the texts themselves or the names of the files containing the texts as stored
 in disk.
 
-The tabular data might or might not fit in disk itself. If it does
-not, please see the ``ChunkPreprocessor`` utilities at the[``preprocessing``]
-(preprocessing.md) module and the examples folder in the repo, which
-illustrate such case. Finally note that only `csv` format is currently
-supported in that case(more formats coming soon).
+The tabular data might or might not fit in disk itself. If it does not, please
+see the ``ChunkPreprocessor`` utilities at the ``preprocessing`` module and
+the examples folder in the repo, which illustrate such case. Finally note
+that only `csv` format is currently supported in that case (more formats
+might come soon).
 
 ::: pytorch_widedeep.load_from_folder.tabular.tabular_from_folder.TabFromFolder
+    options:
+        members:
+            - "get_item"
 
 ::: pytorch_widedeep.load_from_folder.tabular.tabular_from_folder.WideFromFolder
 
 ::: pytorch_widedeep.load_from_folder.text.text_from_folder.TextFromFolder
+    options:
+        members:
+            - "get_item"
 
 ::: pytorch_widedeep.load_from_folder.image.image_from_folder.ImageFromFolder
+    options:
+        members:
+            - "get_item"
 
 ::: pytorch_widedeep.load_from_folder.wd_dataset_from_folder.WideDeepDatasetFromFolder
diff --git a/mkdocs/sources/pytorch-widedeep/model_components.md b/mkdocs/sources/pytorch-widedeep/model_components.md
index ab42e7c9..c59332aa 100644
--- a/mkdocs/sources/pytorch-widedeep/model_components.md
+++ b/mkdocs/sources/pytorch-widedeep/model_components.md
@@ -12,7 +12,6 @@ self-supervised pre-training with tabular data.
     selection:
         filters:
             - "!^forward$"
-            - "!^_reset_parameters$"
 
 ::: pytorch_widedeep.models.tabular.mlp.tab_mlp.TabMlp
     selection:
@@ -82,42 +81,35 @@ self-supervised pre-training with tabular data.
 ::: pytorch_widedeep.models.text.rnns.basic_rnn.BasicRNN
     selection:
         filters:
-            - "!^_"  # exclude all members starting with _
             - "!^forward$"
 
 ::: pytorch_widedeep.models.text.rnns.attentive_rnn.AttentiveRNN
     selection:
         filters:
-            - "!^_"  # exclude all members starting with _
             - "!^forward$"
 
 ::: pytorch_widedeep.models.text.rnns.stacked_attentive_rnn.StackedAttentiveRNN
     selection:
         filters:
-            - "!^_"  # exclude all members starting with _
             - "!^forward$"
 
 ::: pytorch_widedeep.models.text.huggingface_transformers.hf_model.HFModel
     selection:
         filters:
-            - "!^_"  # exclude all members starting with _
             - "!^forward$"
 
 
 ::: pytorch_widedeep.models.image.vision.Vision
     selection:
         filters:
-            - "!^_"  # exclude all members starting with _
             - "!^forward$"
 
 ::: pytorch_widedeep.models.model_fusion.ModelFuser
     selection:
         filters:
-            - "!^_"  # exclude all members starting with _
             - "!^forward$"
 
 ::: pytorch_widedeep.models.wide_deep.WideDeep
     selection:
         filters:
-            - "!^_"  # exclude all members starting with _
             - "!^forward$"
diff --git a/mkdocs/sources/pytorch-widedeep/the_rec_module.md b/mkdocs/sources/pytorch-widedeep/the_rec_module.md
index 46ccb90c..cc17c5be 100644
--- a/mkdocs/sources/pytorch-widedeep/the_rec_module.md
+++ b/mkdocs/sources/pytorch-widedeep/the_rec_module.md
@@ -18,25 +18,48 @@ added in the future.
         filters:
             - "!^forward$"
 
-
 ::: pytorch_widedeep.models.rec.deepffm.DeepFieldAwareFactorizationMachine
     selection:
         filters:
             - "!^forward$"
 
+::: pytorch_widedeep.models.rec.din.DeepInterestNetwork
+    selection:
+        filters:
+            - "!^forward$"
 
 ::: pytorch_widedeep.models.rec.xdeepfm.ExtremeDeepFactorizationMachine
     selection:
         filters:
             - "!^forward$"
 
+::: pytorch_widedeep.models.rec.autoint.AutoInt
+    selection:
+        filters:
+            - "!^forward$"
 
-::: pytorch_widedeep.models.rec.din.DeepInterestNetwork
+::: pytorch_widedeep.models.rec.autoint_plus.AutoIntPlus
     selection:
         filters:
             - "!^forward$"
 
+
 ::: pytorch_widedeep.models.rec.basic_transformer.Transformer
     selection:
         filters:
             - "!^forward$"
+
+::: pytorch_widedeep.models.rec.dcn.DeepCrossNetwork
+    selection:
+        filters:
+            - "!^forward$"
+
+::: pytorch_widedeep.models.rec.dcnv2.DeepCrossNetworkV2
+    selection:
+        filters:
+            - "!^forward$"
+
+::: pytorch_widedeep.models.rec.gdcn.GatedDeepCrossNetwork
+    selection:
+        filters:
+            - "!^forward$"
diff --git a/mkdocs/sources/pytorch-widedeep/utils/fastai_transforms.md b/mkdocs/sources/pytorch-widedeep/utils/fastai_transforms.md
index e58f79f9..16a77819 100644
--- a/mkdocs/sources/pytorch-widedeep/utils/fastai_transforms.md
+++ b/mkdocs/sources/pytorch-widedeep/utils/fastai_transforms.md
@@ -1,9 +1,9 @@
 # Fastai transforms
 
-I directly copied and pasted part of the ``transforms.py`` module from
+We directly copied and pasted part of the ``transforms.py`` module from
 the ``fastai`` library (from an old version). The reason to do such a thing is because
 ``pytorch_widedeep`` only needs the ``Tokenizer`` and the ``Vocab`` classes
-there. This way I avoid extra dependencies. Credit for all the code in the
+there. This way we avoid extra dependencies. Credit for all the code in the
 ``fastai_transforms`` module in this ``pytorch-widedeep`` package goes to
 Jeremy Howard and the `fastai` team. I only include the documentation here for
 completion, but I strongly advise the user to read the ``fastai`` [documentation](https://docs.fast.ai/).
diff --git a/pytorch_widedeep/load_from_folder/image/image_from_folder.py b/pytorch_widedeep/load_from_folder/image/image_from_folder.py
index dee911ae..a6f9dbb6 100644
--- a/pytorch_widedeep/load_from_folder/image/image_from_folder.py
+++ b/pytorch_widedeep/load_from_folder/image/image_from_folder.py
@@ -138,6 +138,22 @@ def __init__(
     def get_item(
         self, fname: Union[str, List[str]]
     ) -> Union[np.ndarray, List[np.ndarray]]:
+        """
+        This method is used to load the image dataset(s) from disk.
+
+        Parameters
+        ----------
+        fname: Union[str, List[str]]
+            the name of the image file(s) to load. If a list is provided, the
+            method will return a list of numpy arrays. Each element in the list
+            corresponds to the image in the different image columns.
+
+        Returns
+        -------
+        Union[np.ndarray, List[np.ndarray]]
+            a numpy array or a list of numpy arrays representing the image(s)
+        """
+
         if isinstance(fname, list):
             if not isinstance(self.directory, list):
                 _directory = [self.directory] * len(fname)
diff --git a/pytorch_widedeep/load_from_folder/tabular/tabular_from_folder.py b/pytorch_widedeep/load_from_folder/tabular/tabular_from_folder.py
index 83fb1e60..8b308f8f 100644
--- a/pytorch_widedeep/load_from_folder/tabular/tabular_from_folder.py
+++ b/pytorch_widedeep/load_from_folder/tabular/tabular_from_folder.py
@@ -108,6 +108,21 @@ def get_item(self, idx: int) -> Tuple[  # noqa: C901
         Optional[Union[str, List[str]]],
         Optional[Union[int, float]],
     ]:
+        """
+        This method is used to retrieve a sample from the csv file
+
+        Parameters
+        ----------
+        idx: int
+            the index of the sample to retrieve
+
+        Returns
+        -------
+        Tuple
+            a tuple with the processed tabular data, the text data and/or the
+            image data, and the target variable
+        """
+
         path = os.path.join(self.directory, self.fname)
 
         try:
diff --git a/pytorch_widedeep/load_from_folder/text/text_from_folder.py b/pytorch_widedeep/load_from_folder/text/text_from_folder.py
index 4e00da08..7be40b0c 100644
--- a/pytorch_widedeep/load_from_folder/text/text_from_folder.py
+++ b/pytorch_widedeep/load_from_folder/text/text_from_folder.py
@@ -23,7 +23,7 @@ class TextFromFolder:
 
     Parameters
     ----------
-    preprocessor: Union[TextPreprocessor, ChunkTextPreprocessor, HFPreprocessor, ChunkHFPreprocessor]
+    preprocessor:
         The preprocessor used to process the text. It must be fitted before using
         this class
     """
@@ -56,6 +56,21 @@ def __init__(
     def get_item(
         self, text: Union[str, List[str]]
     ) -> Union[np.ndarray, List[np.ndarray]]:
+        """
+        Given a text or a list of texts corresponding to different text
+        columns, this method will return the processed text or a list of
+        processed texts
+
+        Parameters
+        ----------
+        text: Union[str, List[str]]
+            The text or list of texts to be processed
+
+        Returns
+        -------
+        Union[np.ndarray, List[np.ndarray]]
+            The processed text or a list of processed texts
+        """
         if isinstance(self.preprocessor, list):
             assert isinstance(text, list)
             processed_sample: Union[np.ndarray, List[np.ndarray]] = [
diff --git a/pytorch_widedeep/load_from_folder/wd_dataset_from_folder.py b/pytorch_widedeep/load_from_folder/wd_dataset_from_folder.py
index 1005eae9..31bdd4cb 100644
--- a/pytorch_widedeep/load_from_folder/wd_dataset_from_folder.py
+++ b/pytorch_widedeep/load_from_folder/wd_dataset_from_folder.py
@@ -34,13 +34,13 @@ class WideDeepDatasetFromFolder(Dataset):
     img_from_folder: Optional[ImageFromFolder], default = None
         Instance of the `ImageFromFolder` class
     reference: Type["WideDeepDatasetFromFolder"], default = None
-        If not None, the 'text_from_folder' and 'img_from_folder' objects will
-        be retrieved from the reference class. This is useful when we want to
-        use a `WideDeepDatasetFromFolder` class used for a train dataset as a
-        reference for the validation and test datasets. In this case, the
-        `text_from_folder` and `img_from_folder` objects will be the same for
-        all three datasets, so there is no need to create a new instance for
-        each dataset.
+        If `reference` not None, the `text_from_folder` and `img_from_folder`
+        objects will be retrieved from the reference class. This is useful
+        when we want to use a `WideDeepDatasetFromFolder` class used for a
+        train dataset as a reference for the validation and test datasets. In
+        this case, the `text_from_folder` and `img_from_folder` objects will
+        be the same for all three datasets, so there is no need to create a
+        new instance for each dataset.
     """
 
     def __init__(
diff --git a/pytorch_widedeep/metrics.py b/pytorch_widedeep/metrics.py
index 673cb95a..11de4906 100644
--- a/pytorch_widedeep/metrics.py
+++ b/pytorch_widedeep/metrics.py
@@ -845,3 +845,19 @@ def __call__(self, y_pred: Tensor, y_true: Tensor) -> np.ndarray:
 RankingMetrics = Union[
     BinaryNDCG_at_k, MAP_at_k, HitRatio_at_k, Precision_at_k, Recall_at_k
 ]
+
+
+# tnis is just because we want to support python 3.9. In 3.10 we could define
+# a type hint like Union[BinaryNDCG_at_k, MAP_at_k, HitRatio_at_k,
+# Precision_at_k, Recall_at_k]
+def is_ranking_metric(metric) -> bool:
+    return isinstance(
+        metric,
+        (
+            BinaryNDCG_at_k,
+            MAP_at_k,
+            HitRatio_at_k,
+            Precision_at_k,
+            Recall_at_k,
+        ),
+    )
diff --git a/pytorch_widedeep/models/model_fusion.py b/pytorch_widedeep/models/model_fusion.py
index b303802a..af112a82 100644
--- a/pytorch_widedeep/models/model_fusion.py
+++ b/pytorch_widedeep/models/model_fusion.py
@@ -179,57 +179,55 @@ def __init__(
 
     def forward(self, X: List[Tensor]) -> Tensor:  # noqa: C901
         if self.fusion_method == "head":
-            return self.head(
-                torch.cat([model(x) for model, x in zip(self.models, X)], -1)
-            )
+            return self._head_fusion(X)
         elif self.fusion_method == "dot":
-            assert len(X) == 2, (
-                "When using 'dot' as fusion_method, only two models "
-                " can be fused. Accordingly, only two inputs should be provided"
-            )
-            outputs = [model(x) for model, x in zip(self.models, X)]
-            return torch.bmm(outputs[1].unsqueeze(1), outputs[0].unsqueeze(2)).view(
-                -1, 1
-            )
+            return self._dot_fusion(X)
         else:
-            if isinstance(self.fusion_method, str):
-                fusion_methods = [self.fusion_method]
-            else:
-                fusion_methods = self.fusion_method  # type: ignore
-
-            fused_outputs: List[Tensor] = []
-            for fm in fusion_methods:
-                if fm == "concatenate":
-                    out = torch.cat([model(x) for model, x in zip(self.models, X)], 1)
-                else:
+            return self._other_fusions(X)
+
+    def _head_fusion(self, X: List[Tensor]) -> Tensor:
+        return self.head(torch.cat([model(x) for model, x in zip(self.models, X)], -1))
+
+    def _dot_fusion(self, X: List[Tensor]) -> Tensor:
+        assert (
+            len(X) == 2
+        ), "When using 'dot' as fusion_method, only two models can be fused"
+        outputs = [model(x) for model, x in zip(self.models, X)]
+        return torch.bmm(outputs[1].unsqueeze(1), outputs[0].unsqueeze(2)).view(-1, 1)
+
+    def _other_fusions(self, X: List[Tensor]) -> Tensor:
+        fusion_methods = (
+            [self.fusion_method]
+            if isinstance(self.fusion_method, str)
+            else self.fusion_method
+        )
+        fused_outputs = [self._apply_fusion_method(fm, X) for fm in fusion_methods]  # type: ignore[attr-defined]
+        return (
+            fused_outputs[0] if len(fused_outputs) == 1 else torch.cat(fused_outputs, 1)
+        )
 
-                    model_outputs = [model(x) for model, x in zip(self.models, X)]
-                    projections = self.project(model_outputs)
-
-                    if fm == "mean":
-                        out = torch.mean(torch.stack(projections, -1), -1)
-                    elif fm == "max":
-                        out, _ = torch.max(torch.stack(projections, -1), -1)
-                    elif fm == "min":
-                        out, _ = torch.min(torch.stack(projections, -1), -1)
-                    elif fm == "sum":
-                        out = torch.sum(torch.stack(projections, -1), -1)
-                    elif fm == "mult":
-                        out = torch.prod(torch.stack(projections, -1), -1)
-                    else:
-                        # This should never happen, but avoids type errors
-                        raise ValueError(
-                            "fusion_method must be one of ['concatenate', 'mean', 'max', 'sum', 'mult', 'dot', 'head'] "
-                            "or a list of those"
-                        )
-                fused_outputs.append(out)
-
-            if len(fused_outputs) == 1:
-                return fused_outputs[0]
-            else:
-                return torch.cat(fused_outputs, 1)
+    def _apply_fusion_method(self, fusion_method: str, X: List[Tensor]) -> Tensor:
+        if fusion_method == "concatenate":
+            return torch.cat([model(x) for model, x in zip(self.models, X)], 1)
+
+        model_outputs = [model(x) for model, x in zip(self.models, X)]
+        projections = self._project(model_outputs)
+        stacked = torch.stack(projections, -1)
+
+        if fusion_method == "mean":
+            return torch.mean(stacked, -1)
+        elif fusion_method == "max":
+            return torch.max(stacked, -1)[0]
+        elif fusion_method == "min":
+            return torch.min(stacked, -1)[0]
+        elif fusion_method == "sum":
+            return torch.sum(stacked, -1)
+        elif fusion_method == "mult":
+            return torch.prod(stacked, -1)
+        else:
+            raise ValueError(f"Unsupported fusion method: {fusion_method}")
 
-    def project(self, X: List[Tensor]) -> List[Tensor]:
+    def _project(self, X: List[Tensor]) -> List[Tensor]:
         r"""Projects the output of the models to a common dimension."""
 
         if self.all_output_dim_equal and self.projection_method is None:
@@ -298,94 +296,74 @@ def output_dim(self) -> int:
 
         return output_dim
 
-    def check_input_parameters(self):  # noqa: C901
+    def check_input_parameters(self):
+        self._validate_tabnet()
+        self._validate_fusion_methods()
+        self._validate_projection_requirements()
+        self._validate_head_dot_exclusivity()
 
+    def _validate_tabnet(self):
         if any(isinstance(model, TabNet) for model in self.models):
             raise ValueError(
                 "TabNet is not supported in ModelFuser. "
                 "Please, use another model for tabular data"
             )
 
+    def _validate_fusion_methods(self):
+        valid_methods = [
+            "concatenate",
+            "min",
+            "max",
+            "mean",
+            "sum",
+            "mult",
+            "dot",
+            "head",
+        ]
+
         if isinstance(self.fusion_method, str):
-            if not any(
-                x == self.fusion_method
-                for x in [
-                    "concatenate",
-                    "min",
-                    "max",
-                    "mean",
-                    "sum",
-                    "dot",
-                    "mult",
-                    "head",
-                ]
-            ):
+            if self.fusion_method not in valid_methods:
                 raise ValueError(
                     "fusion_method must be one of ['concatenate', 'mean', 'max', 'sum', 'mult', 'dot', 'head'] "
                     "or a list of any those but 'dot'"
                 )
-
-            if (
-                any(x in self.fusion_method for x in ["min", "max", "mean"])
-                and not self.all_output_dim_equal
-            ):
-                if self.projection_method is None:
-                    raise ValueError(
-                        "If 'fusion_method' is not 'concatenate' or 'head', "
-                        " and the output dimensions of the models are not equal, "
-                        "'projection_method' must be provided"
-                    )
-                elif self.projection_method not in ["min", "max", "mean"]:
-                    raise ValueError(
-                        "projection_method must be one of ['min', 'max', 'mean']"
-                    )
         else:
-            if not all(
-                any(
-                    x == fm
-                    for x in [
-                        "concatenate",
-                        "min",
-                        "max",
-                        "mean",
-                        "sum",
-                        "mult",
-                        "dot",
-                        "head",
-                    ]
-                )
-                for fm in self.fusion_method
-            ):
+            if not all(fm in valid_methods for fm in self.fusion_method):
                 raise ValueError(
                     "fusion_method must be one of ['concatenate', 'mean', 'max', 'sum', 'mult', 'dot', 'head'] "
                     "or a list of those but 'dot'"
                 )
 
-            if (
-                all(
-                    any(x in fm for x in ["min", "max", "mean"])
-                    for fm in self.fusion_method
-                )
-                and not self.all_output_dim_equal
-            ):
-                if self.projection_method is None:
-                    raise ValueError(
-                        "If 'fusion_method' is not 'concatenate' or 'head', "
-                        " and the output dimensions of the models are not equal, "
-                        "'projection_method' must be provided"
-                    )
-                elif self.projection_method not in ["min", "max", "mean"]:
-                    raise ValueError(
-                        "projection_method must be one of ['min', 'max', 'mean']"
-                    )
+    def _validate_projection_requirements(self):
+        needs_projection = not self.all_output_dim_equal
+        has_size_dependent_method = False
 
-        if any(x in self.fusion_method for x in ["head", "dot"]) and isinstance(
-            self.fusion_method, list
-        ):
-            raise ValueError(
-                "When using 'head' or 'dot' as fusion_method, no other method should be provided"
+        if isinstance(self.fusion_method, str):
+            has_size_dependent_method = self.fusion_method in ["min", "max", "mean"]
+        else:
+            has_size_dependent_method = all(
+                fm in ["min", "max", "mean"] for fm in self.fusion_method
             )
 
+        if has_size_dependent_method and needs_projection:
+            if self.projection_method is None:
+                raise ValueError(
+                    "If 'fusion_method' is not 'concatenate' or 'head', "
+                    "and the output dimensions of the models are not equal, "
+                    "'projection_method' must be provided"
+                )
+            elif self.projection_method not in ["min", "max", "mean"]:
+                raise ValueError(
+                    "projection_method must be one of ['min', 'max', 'mean']"
+                )
+
+    def _validate_head_dot_exclusivity(self):
+        if isinstance(self.fusion_method, list):
+            if any(method in ["head", "dot"] for method in self.fusion_method):
+                raise ValueError(
+                    "When using 'head' or 'dot' as fusion_method, no other method should be provided"
+                )
+
     def __repr__(self):
         if self.projection_method is not None:
             proj = f"{self.projection_method}"
diff --git a/pytorch_widedeep/models/rec/autoint.py b/pytorch_widedeep/models/rec/autoint.py
index 95265c69..5ff30428 100644
--- a/pytorch_widedeep/models/rec/autoint.py
+++ b/pytorch_widedeep/models/rec/autoint.py
@@ -41,28 +41,54 @@ class AutoInt(BaseTabularModelWithAttention):
         Boolean indicating if bias will be used for the categorical embeddings
     cat_embed_activation: Optional, str, default = None,
         Activation function for the categorical embeddings
-    continuous_cols: List, Optional, default = None
+    continuous_cols : Optional[List[str]], default=None
         List with the name of the numeric (aka continuous) columns
-    cont_norm_layer: str, Optional, default = None
+    cont_norm_layer : Optional[Literal["batchnorm", "layernorm"]], default=None
         Type of normalization layer applied to the continuous features.
-        Options are: _'layernorm'_ and _'batchnorm'_
-    embed_continuous_method: Optional, str, default = None,
+        Options are: _'layernorm'_ and _'batchnorm'_. if `None`, no
+        normalization layer will be used.
+    embed_continuous_method: Optional[Literal["piecewise", "periodic", "standard"]], default="standard"
         Method to use to embed the continuous features. Options are:
-        _'standard'_, _'periodic'_ or _'piecewise'_
-    cont_embed_dropout: float, Optional, default = None,
-        Dropout for the continuous embeddings
-    cont_embed_activation: Optional, str, default = None,
-        Activation function for the continuous embeddings
-    quantization_setup: Dict[str, List[float]], Optional, default = None,
-        Setup for the piecewise embeddings quantization
-    n_frequencies: int, Optional, default = None,
-        Number of frequencies for periodic embeddings
-    sigma: float, Optional, default = None,
-        Sigma parameter for periodic embeddings
-    share_last_layer: bool, Optional, default = None,
-        Whether to share the last layer in periodic embeddings
+        _'standard'_, _'periodic'_ or _'piecewise'_. The _'standard'_
+        embedding method is based on the FT-Transformer implementation
+        presented in the paper: [Revisiting Deep Learning Models for
+        Tabular Data](https://arxiv.org/abs/2106.11959v5). The _'periodic'_
+        and_'piecewise'_ methods were presented in the paper: [On Embeddings for
+        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556).
+        Please, read the papers for details.
+    cont_embed_dropout : Optional[float], default=None
+        Dropout for the continuous embeddings. If `None`, it will default to 0.0
+    cont_embed_activation : Optional[str], default=None
+        Activation function for the continuous embeddings if any. Currently
+        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported.
+        If `None`, no activation function will be applied.
+    quantization_setup : Optional[Dict[str, List[float]]], default=None
+        This parameter is used when the _'piecewise'_ method is used to embed
+        the continuous cols. It is a dict where keys are the name of the continuous
+        columns and values are lists with the boundaries for the quantization
+        of the continuous_cols. See the examples for details. If
+        If the _'piecewise'_ method is used, this parameter is required.
+    n_frequencies : Optional[int], default=None
+        This is the so called _'k'_ in their paper [On Embeddings for
+        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556),
+        and is the number of 'frequencies' that will be used to represent each
+        continuous column. See their Eq 2 in the paper for details. If
+        the _'periodic'_ method is used, this parameter is required.
+    sigma : Optional[float], default=None
+        This is the sigma parameter in the paper mentioned when describing the
+        previous parameters and it is used to initialise the 'frequency
+        weights'. See their Eq 2 in the paper for details. If
+        the _'periodic'_ method is used, this parameter is required.
+    share_last_layer : Optional[bool], default=None
+        This parameter is not present in the before mentioned paper but it is implemented in
+        the [official repo](https://github.com/yandex-research/rtdl-num-embeddings/tree/main).
+        If `True` the linear layer that turns the frequencies into embeddings
+        will be shared across the continuous columns. If `False` a different
+        linear layer will be used for each continuous column.
+        If the _'periodic'_ method is used, this parameter is required.
     full_embed_dropout: bool, Optional, default = None,
-        If True, drops the entire embedding for a column
+        If `True`, the full embedding corresponding to a column will be masked
+        out/dropout. If `None`, it will default to `False`.
 
     Attributes
     ----------
diff --git a/pytorch_widedeep/models/rec/autoint_plus.py b/pytorch_widedeep/models/rec/autoint_plus.py
index 6e6d9588..75134d23 100644
--- a/pytorch_widedeep/models/rec/autoint_plus.py
+++ b/pytorch_widedeep/models/rec/autoint_plus.py
@@ -49,41 +49,72 @@ class AutoIntPlus(BaseTabularModelWithAttention):
         Boolean indicating if bias will be used for the categorical embeddings
     cat_embed_activation: Optional, str, default = None,
         Activation function for the categorical embeddings
-    continuous_cols: List, Optional, default = None
+    continuous_cols : Optional[List[str]], default=None
         List with the name of the numeric (aka continuous) columns
-    cont_norm_layer: str, Optional, default = None
+    cont_norm_layer : Optional[Literal["batchnorm", "layernorm"]], default=None
         Type of normalization layer applied to the continuous features.
-        Options are: _'layernorm'_ and _'batchnorm'_
-    embed_continuous_method: Optional, str, default = None,
+        Options are: _'layernorm'_ and _'batchnorm'_. if `None`, no
+        normalization layer will be used.
+    embed_continuous_method: Optional[Literal["piecewise", "periodic", "standard"]], default="standard"
         Method to use to embed the continuous features. Options are:
-        _'standard'_, _'periodic'_ or _'piecewise'_
-    cont_embed_dropout: float, Optional, default = None,
-        Dropout for the continuous embeddings
-    cont_embed_activation: Optional, str, default = None,
-        Activation function for the continuous embeddings
-    quantization_setup: Dict[str, List[float]], Optional, default = None,
-        Setup for the piecewise embeddings quantization
-    n_frequencies: int, Optional, default = None,
-        Number of frequencies for periodic embeddings
-    sigma: float, Optional, default = None,
-        Sigma parameter for periodic embeddings
-    share_last_layer: bool, Optional, default = None,
-        Whether to share the last layer in periodic embeddings
+        _'standard'_, _'periodic'_ or _'piecewise'_. The _'standard'_
+        embedding method is based on the FT-Transformer implementation
+        presented in the paper: [Revisiting Deep Learning Models for
+        Tabular Data](https://arxiv.org/abs/2106.11959v5). The _'periodic'_
+        and_'piecewise'_ methods were presented in the paper: [On Embeddings for
+        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556).
+        Please, read the papers for details.
+    cont_embed_dropout : Optional[float], default=None
+        Dropout for the continuous embeddings. If `None`, it will default to 0.0
+    cont_embed_activation : Optional[str], default=None
+        Activation function for the continuous embeddings if any. Currently
+        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported.
+        If `None`, no activation function will be applied.
+    quantization_setup : Optional[Dict[str, List[float]]], default=None
+        This parameter is used when the _'piecewise'_ method is used to embed
+        the continuous cols. It is a dict where keys are the name of the continuous
+        columns and values are lists with the boundaries for the quantization
+        of the continuous_cols. See the examples for details. If
+        If the _'piecewise'_ method is used, this parameter is required.
+    n_frequencies : Optional[int], default=None
+        This is the so called _'k'_ in their paper [On Embeddings for
+        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556),
+        and is the number of 'frequencies' that will be used to represent each
+        continuous column. See their Eq 2 in the paper for details. If
+        the _'periodic'_ method is used, this parameter is required.
+    sigma : Optional[float], default=None
+        This is the sigma parameter in the paper mentioned when describing the
+        previous parameters and it is used to initialise the 'frequency
+        weights'. See their Eq 2 in the paper for details. If
+        the _'periodic'_ method is used, this parameter is required.
+    share_last_layer : Optional[bool], default=None
+        This parameter is not present in the before mentioned paper but it is implemented in
+        the [official repo](https://github.com/yandex-research/rtdl-num-embeddings/tree/main).
+        If `True` the linear layer that turns the frequencies into embeddings
+        will be shared across the continuous columns. If `False` a different
+        linear layer will be used for each continuous column.
+        If the _'periodic'_ method is used, this parameter is required.
     full_embed_dropout: bool, Optional, default = None,
-        If True, drops the entire embedding for a column
-    mlp_hidden_dims: List, default = [100, 100]
-        List with the number of neurons per dense layer in the deep network
+        If `True`, the full embedding corresponding to a column will be masked
+        out/dropout. If `None`, it will default to `False`.
+    mlp_hidden_dims: List, default = [200, 100]
+        List with the number of neurons per dense layer in the mlp.
     mlp_activation: str, default = "relu"
-        Activation function for the dense layers
+        Activation function for the dense layers of the MLP. Currently
+        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported
     mlp_dropout: float or List, default = 0.1
-        Dropout between the dense layers
+        float or List of floats with the dropout between the dense layers.
+        e.g: _[0.5,0.5]_
     mlp_batchnorm: bool, default = False
-        If True, applies batch normalization in the dense layers
+        Boolean indicating whether or not batch normalization will be applied
+        to the dense layers
     mlp_batchnorm_last: bool, default = False
-        If True, applies batch normalization in the last dense layer
-    mlp_linear_first: bool, default = True
-        If True, applies the order: [Linear -> Activation -> BatchNorm -> Dropout]
-        If False: [BatchNorm -> Dropout -> Linear -> Activation]
+        Boolean indicating whether or not batch normalization will be applied
+        to the last of the dense layers
+    mlp_linear_first: bool, default = False
+        Boolean indicating the order of the operations in the dense
+        layer. If `True: [LIN -> ACT -> BN -> DP]`. If `False: [BN -> DP ->
+        LIN -> ACT]`
 
     Attributes
     ----------
diff --git a/pytorch_widedeep/models/rec/dcn.py b/pytorch_widedeep/models/rec/dcn.py
index a985f106..36cc8135 100644
--- a/pytorch_widedeep/models/rec/dcn.py
+++ b/pytorch_widedeep/models/rec/dcn.py
@@ -60,45 +60,72 @@ class DeepCrossNetwork(BaseTabularModelWithoutAttention):
         Boolean indicating if bias will be used for the categorical embeddings
     cat_embed_activation: Optional, str, default = None,
         Activation function for the categorical embeddings
-    continuous_cols: List, Optional, default = None
+    continuous_cols : Optional[List[str]], default=None
         List with the name of the numeric (aka continuous) columns
-    cont_norm_layer: str, Optional, default = None
+    cont_norm_layer : Optional[Literal["batchnorm", "layernorm"]], default=None
         Type of normalization layer applied to the continuous features.
-        Options are: _'layernorm'_ and _'batchnorm'_
-    embed_continuous: bool, Optional, default = None,
-        Boolean indicating if the continuous columns will be embedded
-    embed_continuous_method: Optional, str, default = None,
+        Options are: _'layernorm'_ and _'batchnorm'_. if `None`, no
+        normalization layer will be used.
+    embed_continuous_method: Optional[Literal["piecewise", "periodic", "standard"]], default="standard"
         Method to use to embed the continuous features. Options are:
-        _'standard'_, _'periodic'_ or _'piecewise'_
-    cont_embed_dim: int, Optional, default = None,
-        Size of the continuous embeddings
-    cont_embed_dropout: float, Optional, default = None,
-        Dropout for the continuous embeddings
-    cont_embed_activation: Optional, str, default = None,
-        Activation function for the continuous embeddings
-    quantization_setup: Dict[str, List[float]], Optional, default = None,
-        Setup for the piecewise embeddings quantization
-    n_frequencies: int, Optional, default = None,
-        Number of frequencies for periodic embeddings
-    sigma: float, Optional, default = None,
-        Sigma parameter for periodic embeddings
-    share_last_layer: bool, Optional, default = None,
-        Whether to share the last layer in periodic embeddings
+        _'standard'_, _'periodic'_ or _'piecewise'_. The _'standard'_
+        embedding method is based on the FT-Transformer implementation
+        presented in the paper: [Revisiting Deep Learning Models for
+        Tabular Data](https://arxiv.org/abs/2106.11959v5). The _'periodic'_
+        and_'piecewise'_ methods were presented in the paper: [On Embeddings for
+        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556).
+        Please, read the papers for details.
+    cont_embed_dropout : Optional[float], default=None
+        Dropout for the continuous embeddings. If `None`, it will default to 0.0
+    cont_embed_activation : Optional[str], default=None
+        Activation function for the continuous embeddings if any. Currently
+        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported.
+        If `None`, no activation function will be applied.
+    quantization_setup : Optional[Dict[str, List[float]]], default=None
+        This parameter is used when the _'piecewise'_ method is used to embed
+        the continuous cols. It is a dict where keys are the name of the continuous
+        columns and values are lists with the boundaries for the quantization
+        of the continuous_cols. See the examples for details. If
+        If the _'piecewise'_ method is used, this parameter is required.
+    n_frequencies : Optional[int], default=None
+        This is the so called _'k'_ in their paper [On Embeddings for
+        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556),
+        and is the number of 'frequencies' that will be used to represent each
+        continuous column. See their Eq 2 in the paper for details. If
+        the _'periodic'_ method is used, this parameter is required.
+    sigma : Optional[float], default=None
+        This is the sigma parameter in the paper mentioned when describing the
+        previous parameters and it is used to initialise the 'frequency
+        weights'. See their Eq 2 in the paper for details. If
+        the _'periodic'_ method is used, this parameter is required.
+    share_last_layer : Optional[bool], default=None
+        This parameter is not present in the before mentioned paper but it is implemented in
+        the [official repo](https://github.com/yandex-research/rtdl-num-embeddings/tree/main).
+        If `True` the linear layer that turns the frequencies into embeddings
+        will be shared across the continuous columns. If `False` a different
+        linear layer will be used for each continuous column.
+        If the _'periodic'_ method is used, this parameter is required.
     full_embed_dropout: bool, Optional, default = None,
-        If True, drops the entire embedding for a column
+        If `True`, the full embedding corresponding to a column will be masked
+        out/dropout. If `None`, it will default to `False`.
     mlp_hidden_dims: List, default = [200, 100]
-        List with the number of neurons per dense layer in the deep network
+        List with the number of neurons per dense layer in the mlp.
     mlp_activation: str, default = "relu"
-        Activation function for the dense layers
+        Activation function for the dense layers of the MLP. Currently
+        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported
     mlp_dropout: float or List, default = 0.1
-        Dropout between the dense layers
+        float or List of floats with the dropout between the dense layers.
+        e.g: _[0.5,0.5]_
     mlp_batchnorm: bool, default = False
-        If True, applies batch normalization in the dense layers
+        Boolean indicating whether or not batch normalization will be applied
+        to the dense layers
     mlp_batchnorm_last: bool, default = False
-        If True, applies batch normalization in the last dense layer
-    mlp_linear_first: bool, default = True
-        If True, applies the order: [Linear -> Activation -> BatchNorm -> Dropout]
-        If False: [BatchNorm -> Dropout -> Linear -> Activation]
+        Boolean indicating whether or not batch normalization will be applied
+        to the last of the dense layers
+    mlp_linear_first: bool, default = False
+        Boolean indicating the order of the operations in the dense
+        layer. If `True: [LIN -> ACT -> BN -> DP]`. If `False: [BN -> DP ->
+        LIN -> ACT]`
 
     Attributes
     ----------
diff --git a/pytorch_widedeep/models/rec/dcnv2.py b/pytorch_widedeep/models/rec/dcnv2.py
index cb82823c..3ddbba39 100644
--- a/pytorch_widedeep/models/rec/dcnv2.py
+++ b/pytorch_widedeep/models/rec/dcnv2.py
@@ -168,46 +168,72 @@ class DeepCrossNetworkV2(BaseTabularModelWithoutAttention):
         Boolean indicating if bias will be used for the categorical embeddings
     cat_embed_activation: Optional, str, default = None,
         Activation function for the categorical embeddings
-    continuous_cols: List, Optional, default = None
+    continuous_cols : Optional[List[str]], default=None
         List with the name of the numeric (aka continuous) columns
-    cont_norm_layer: str, Optional, default = None,
+    cont_norm_layer : Optional[Literal["batchnorm", "layernorm"]], default=None
         Type of normalization layer applied to the continuous features.
-        Options are: _'layernorm'_ and _'batchnorm'_
-    embed_continuous: bool, Optional, default = None,
-        Boolean indicating if the continuous columns will be embedded
-    embed_continuous_method: Optional, str, default = None,
+        Options are: _'layernorm'_ and _'batchnorm'_. if `None`, no
+        normalization layer will be used.
+    embed_continuous_method: Optional[Literal["piecewise", "periodic", "standard"]], default="standard"
         Method to use to embed the continuous features. Options are:
-        _'standard'_, _'periodic'_ or _'piecewise'_
-    cont_embed_dim: int, Optional, default = None,
-        Size of the continuous embeddings
-    cont_embed_dropout: float, Optional, default = None,
-        Dropout for the continuous embeddings
-    cont_embed_activation: Optional, str, default = None,
-        Activation function for the continuous embeddings
-    quantization_setup: Dict[str, List[float]], Optional, default = None,
-        Setup for the piecewise embeddings quantization
-    n_frequencies: int, Optional, default = None,
-        Number of frequencies for periodic embeddings
-    sigma: float, Optional, default = None,
-        Sigma parameter for periodic embeddings
-    share_last_layer: bool, Optional, default = None,
-        Whether to share the last layer in periodic embeddings
+        _'standard'_, _'periodic'_ or _'piecewise'_. The _'standard'_
+        embedding method is based on the FT-Transformer implementation
+        presented in the paper: [Revisiting Deep Learning Models for
+        Tabular Data](https://arxiv.org/abs/2106.11959v5). The _'periodic'_
+        and_'piecewise'_ methods were presented in the paper: [On Embeddings for
+        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556).
+        Please, read the papers for details.
+    cont_embed_dropout : Optional[float], default=None
+        Dropout for the continuous embeddings. If `None`, it will default to 0.0
+    cont_embed_activation : Optional[str], default=None
+        Activation function for the continuous embeddings if any. Currently
+        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported.
+        If `None`, no activation function will be applied.
+    quantization_setup : Optional[Dict[str, List[float]]], default=None
+        This parameter is used when the _'piecewise'_ method is used to embed
+        the continuous cols. It is a dict where keys are the name of the continuous
+        columns and values are lists with the boundaries for the quantization
+        of the continuous_cols. See the examples for details. If
+        If the _'piecewise'_ method is used, this parameter is required.
+    n_frequencies : Optional[int], default=None
+        This is the so called _'k'_ in their paper [On Embeddings for
+        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556),
+        and is the number of 'frequencies' that will be used to represent each
+        continuous column. See their Eq 2 in the paper for details. If
+        the _'periodic'_ method is used, this parameter is required.
+    sigma : Optional[float], default=None
+        This is the sigma parameter in the paper mentioned when describing the
+        previous parameters and it is used to initialise the 'frequency
+        weights'. See their Eq 2 in the paper for details. If
+        the _'periodic'_ method is used, this parameter is required.
+    share_last_layer : Optional[bool], default=None
+        This parameter is not present in the before mentioned paper but it is implemented in
+        the [official repo](https://github.com/yandex-research/rtdl-num-embeddings/tree/main).
+        If `True` the linear layer that turns the frequencies into embeddings
+        will be shared across the continuous columns. If `False` a different
+        linear layer will be used for each continuous column.
+        If the _'periodic'_ method is used, this parameter is required.
     full_embed_dropout: bool, Optional, default = None,
-        If True, drops the entire embedding for a column
+        If `True`, the full embedding corresponding to a column will be masked
+        out/dropout. If `None`, it will default to `False`.
     mlp_hidden_dims: List, default = [200, 100]
-        List with the number of neurons per dense layer in the deep network
+        List with the number of neurons per dense layer in the mlp.
     mlp_activation: str, default = "relu"
-        Activation function for the dense layers
+        Activation function for the dense layers of the MLP. Currently
+        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported
     mlp_dropout: float or List, default = 0.1
-        Dropout between the dense layers
+        float or List of floats with the dropout between the dense layers.
+        e.g: _[0.5,0.5]_
     mlp_batchnorm: bool, default = False
-        If True, applies batch normalization in the dense layers
+        Boolean indicating whether or not batch normalization will be applied
+        to the dense layers
     mlp_batchnorm_last: bool, default = False
-        If True, applies batch normalization in the last dense layer
-    mlp_linear_first: bool, default = True
-        If True, applies the order: [Linear -> Activation -> BatchNorm -> Dropout]
-        If False: [BatchNorm -> Dropout -> Linear -> Activation]
-
+        Boolean indicating whether or not batch normalization will be applied
+        to the last of the dense layers
+    mlp_linear_first: bool, default = False
+        Boolean indicating the order of the operations in the dense
+        layer. If `True: [LIN -> ACT -> BN -> DP]`. If `False: [BN -> DP ->
+        LIN -> ACT]`
     Attributes
     ----------
     cross_network: nn.Module
diff --git a/pytorch_widedeep/models/rec/deepfm.py b/pytorch_widedeep/models/rec/deepfm.py
index 95f9896b..f7e02ff5 100644
--- a/pytorch_widedeep/models/rec/deepfm.py
+++ b/pytorch_widedeep/models/rec/deepfm.py
@@ -60,7 +60,7 @@ class DeepFactorizationMachine(BaseTabularModelWithAttention):
         Type of normalization layer applied to the continuous features.
         Options are: _'layernorm'_ and _'batchnorm'_. if `None`, no
         normalization layer will be used.
-    embed_continuous_method : Optional[Literal["piecewise", "periodic"]], default="piecewise"
+    embed_continuous_method: Optional[Literal["piecewise", "periodic", "standard"]], default="standard"
         Method to use to embed the continuous features. Options are:
         _'standard'_, _'periodic'_ or _'piecewise'_. The _'standard'_
         embedding method is based on the FT-Transformer implementation
@@ -69,9 +69,6 @@ class DeepFactorizationMachine(BaseTabularModelWithAttention):
         and_'piecewise'_ methods were presented in the paper: [On Embeddings for
         Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556).
         Please, read the papers for details.
-    cont_embed_dim : Optional[int], default=None
-        Size of the continuous embeddings. If the continuous columns are
-        embedded, `cont_embed_dim` must be passed.
     cont_embed_dropout : Optional[float], default=None
         Dropout for the continuous embeddings. If `None`, it will default to 0.0
     cont_embed_activation : Optional[str], default=None
@@ -162,8 +159,8 @@ def __init__(
         continuous_cols: Optional[List[str]] = None,
         cont_norm_layer: Optional[Literal["batchnorm", "layernorm"]] = None,
         embed_continuous_method: Optional[
-            Literal["piecewise", "periodic"]
-        ] = "piecewise",
+            Literal["piecewise", "periodic", "standard"]
+        ] = "standard",
         cont_embed_dropout: Optional[float] = None,
         cont_embed_activation: Optional[str] = None,
         quantization_setup: Optional[Dict[str, List[float]]] = None,
diff --git a/pytorch_widedeep/models/rec/din.py b/pytorch_widedeep/models/rec/din.py
index 94ed0911..d220657f 100644
--- a/pytorch_widedeep/models/rec/din.py
+++ b/pytorch_widedeep/models/rec/din.py
@@ -1,4 +1,4 @@
-from typing import Any, Dict, List, Tuple, Literal, Optional
+from typing import Any, Dict, List, Tuple, Union, Literal, Optional
 
 import torch
 import torch.nn.functional as F
@@ -36,7 +36,7 @@ def __init__(
         self.proj_dim = proj_dim if proj_dim is not None else embed_dim
         self.linear_in = nn.Linear(embed_dim * 4, self.proj_dim)
         if activation == "prelu":
-            self.activation: nn.PReLU | Dice = nn.PReLU()
+            self.activation: Union[nn.PReLU, Dice] = nn.PReLU()
         elif activation == "dice":
             self.activation = Dice(self.proj_dim)
         self.linear_out = nn.Linear(self.proj_dim, 1)
@@ -80,7 +80,11 @@ class DeepInterestNetwork(BaseWDModelComponent):
     column_idx : Dict[str, int]
         Dictionary mapping column names to their corresponding index.
     target_item_col : str
-        Name of the target item column.
+        Name of the target item column. Note that this is not the target
+        column. This algorithm relies on a sequence representation of
+        interactions. The target item would be the next item in the sequence
+        of interactions (e.g. item 6th in a sequence of 5 items), and our
+        goal is to predict a given action on it.
     user_behavior_confiq : Tuple[List[str], int, int]
         Configuration for user behavior sequence columns. Tuple containing:
         - List of column names that correspond to the user behavior sequence<br/>
@@ -140,7 +144,7 @@ class DeepInterestNetwork(BaseWDModelComponent):
         :information_source: **NOTE**: This parameter is deprecated and it
          will be removed in future releases. Please, use the
          `embed_continuous_method` parameter instead.
-    embed_continuous_method : Optional[Literal["piecewise", "periodic"]], default="piecewise"
+    embed_continuous_method : Optional[Literal["piecewise", "periodic", "standard"]], default=None
         Method to use to embed the continuous features. Options are:
         _'standard'_, _'periodic'_ or _'piecewise'_. The _'standard'_
         embedding method is based on the FT-Transformer implementation
diff --git a/pytorch_widedeep/models/rec/gdcn.py b/pytorch_widedeep/models/rec/gdcn.py
index 10f0a9a7..e9b75b2c 100644
--- a/pytorch_widedeep/models/rec/gdcn.py
+++ b/pytorch_widedeep/models/rec/gdcn.py
@@ -101,46 +101,72 @@ class GatedDeepCrossNetwork(BaseTabularModelWithoutAttention):
     cat_embed_activation: Optional, str, default = None,
         Activation function for the categorical embeddings, if any. Currently
         _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported
-    continuous_cols: List, Optional, default = None
+    continuous_cols : Optional[List[str]], default=None
         List with the name of the numeric (aka continuous) columns
-    cont_norm_layer: str, Optional, default = None
+    cont_norm_layer : Optional[Literal["batchnorm", "layernorm"]], default=None
         Type of normalization layer applied to the continuous features.
         Options are: _'layernorm'_ and _'batchnorm'_. if `None`, no
         normalization layer will be used.
-    embed_continuous: bool, Optional, default = None,
-        Boolean indicating if the continuous columns will be embedded
-    embed_continuous_method: Optional, str, default = None,
+    embed_continuous_method: Optional[Literal["piecewise", "periodic", "standard"]], default="standard"
         Method to use to embed the continuous features. Options are:
-        _'standard'_, _'periodic'_ or _'piecewise'_.
-    cont_embed_dim: int, Optional, default = None,
-        Size of the continuous embeddings
-    cont_embed_dropout: float, Optional, default = None,
-        Dropout for the continuous embeddings
-    cont_embed_activation: Optional, str, default = None,
-        Activation function for the continuous embeddings
-    quantization_setup: Dict[str, List[float]], Optional, default = None,
-        Setup for the piecewise embeddings quantization
-    n_frequencies: int, Optional, default = None,
-        Number of frequencies for periodic embeddings
-    sigma: float, Optional, default = None,
-        Sigma parameter for periodic embeddings
-    share_last_layer: bool, Optional, default = None,
-        Whether to share the last layer in periodic embeddings
+        _'standard'_, _'periodic'_ or _'piecewise'_. The _'standard'_
+        embedding method is based on the FT-Transformer implementation
+        presented in the paper: [Revisiting Deep Learning Models for
+        Tabular Data](https://arxiv.org/abs/2106.11959v5). The _'periodic'_
+        and_'piecewise'_ methods were presented in the paper: [On Embeddings for
+        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556).
+        Please, read the papers for details.
+    cont_embed_dropout : Optional[float], default=None
+        Dropout for the continuous embeddings. If `None`, it will default to 0.0
+    cont_embed_activation : Optional[str], default=None
+        Activation function for the continuous embeddings if any. Currently
+        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported.
+        If `None`, no activation function will be applied.
+    quantization_setup : Optional[Dict[str, List[float]]], default=None
+        This parameter is used when the _'piecewise'_ method is used to embed
+        the continuous cols. It is a dict where keys are the name of the continuous
+        columns and values are lists with the boundaries for the quantization
+        of the continuous_cols. See the examples for details. If
+        If the _'piecewise'_ method is used, this parameter is required.
+    n_frequencies : Optional[int], default=None
+        This is the so called _'k'_ in their paper [On Embeddings for
+        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556),
+        and is the number of 'frequencies' that will be used to represent each
+        continuous column. See their Eq 2 in the paper for details. If
+        the _'periodic'_ method is used, this parameter is required.
+    sigma : Optional[float], default=None
+        This is the sigma parameter in the paper mentioned when describing the
+        previous parameters and it is used to initialise the 'frequency
+        weights'. See their Eq 2 in the paper for details. If
+        the _'periodic'_ method is used, this parameter is required.
+    share_last_layer : Optional[bool], default=None
+        This parameter is not present in the before mentioned paper but it is implemented in
+        the [official repo](https://github.com/yandex-research/rtdl-num-embeddings/tree/main).
+        If `True` the linear layer that turns the frequencies into embeddings
+        will be shared across the continuous columns. If `False` a different
+        linear layer will be used for each continuous column.
+        If the _'periodic'_ method is used, this parameter is required.
     full_embed_dropout: bool, Optional, default = None,
-        If True, drops the entire embedding for a column
+        If `True`, the full embedding corresponding to a column will be masked
+        out/dropout. If `None`, it will default to `False`.
     mlp_hidden_dims: List, default = [200, 100]
-        List with the number of neurons per dense layer in the deep network
+        List with the number of neurons per dense layer in the mlp.
     mlp_activation: str, default = "relu"
-        Activation function for the dense layers
+        Activation function for the dense layers of the MLP. Currently
+        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported
     mlp_dropout: float or List, default = 0.1
-        Dropout between the dense layers
+        float or List of floats with the dropout between the dense layers.
+        e.g: _[0.5,0.5]_
     mlp_batchnorm: bool, default = False
-        If True, applies batch normalization in the dense layers
+        Boolean indicating whether or not batch normalization will be applied
+        to the dense layers
     mlp_batchnorm_last: bool, default = False
-        If True, applies batch normalization in the last dense layer
-    mlp_linear_first: bool, default = True
-        If True, applies the order: [Linear -> Activation -> BatchNorm -> Dropout]
-        If False: [BatchNorm -> Dropout -> Linear -> Activation]
+        Boolean indicating whether or not batch normalization will be applied
+        to the last of the dense layers
+    mlp_linear_first: bool, default = False
+        Boolean indicating the order of the operations in the dense
+        layer. If `True: [LIN -> ACT -> BN -> DP]`. If `False: [BN -> DP ->
+        LIN -> ACT]`
 
     Attributes
     ----------
diff --git a/pytorch_widedeep/models/rec/xdeepfm.py b/pytorch_widedeep/models/rec/xdeepfm.py
index 8834e707..40ed789e 100644
--- a/pytorch_widedeep/models/rec/xdeepfm.py
+++ b/pytorch_widedeep/models/rec/xdeepfm.py
@@ -81,7 +81,7 @@ class ExtremeDeepFactorizationMachine(BaseTabularModelWithAttention):
         Type of normalization layer applied to the continuous features.
         Options are: _'layernorm'_ and _'batchnorm'_. if `None`, no
         normalization layer will be used.
-    embed_continuous_method : Optional[Literal["piecewise", "periodic"]], default="piecewise"
+    embed_continuous_method: Optional[Literal["piecewise", "periodic", "standard"]], default="standard"
         Method to use to embed the continuous features. Options are:
         _'standard'_, _'periodic'_ or _'piecewise'_. The _'standard'_
         embedding method is based on the FT-Transformer implementation
@@ -90,9 +90,6 @@ class ExtremeDeepFactorizationMachine(BaseTabularModelWithAttention):
         and_'piecewise'_ methods were presented in the paper: [On Embeddings for
         Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556).
         Please, read the papers for details.
-    cont_embed_dim : Optional[int], default=None
-        Size of the continuous embeddings. If the continuous columns are
-        embedded, `cont_embed_dim` must be passed.
     cont_embed_dropout : Optional[float], default=None
         Dropout for the continuous embeddings. If `None`, it will default to 0.0
     cont_embed_activation : Optional[str], default=None
@@ -186,8 +183,8 @@ def __init__(
         continuous_cols: Optional[List[str]] = None,
         cont_norm_layer: Optional[Literal["batchnorm", "layernorm"]] = None,
         embed_continuous_method: Optional[
-            Literal["piecewise", "periodic"]
-        ] = "piecewise",
+            Literal["piecewise", "periodic", "standard"]
+        ] = "standard",
         cont_embed_dropout: Optional[float] = None,
         cont_embed_activation: Optional[str] = None,
         quantization_setup: Optional[Dict[str, List[float]]] = None,
diff --git a/pytorch_widedeep/preprocessing/din_preprocessor.py b/pytorch_widedeep/preprocessing/din_preprocessor.py
index f66bbdf7..6a291b1b 100644
--- a/pytorch_widedeep/preprocessing/din_preprocessor.py
+++ b/pytorch_widedeep/preprocessing/din_preprocessor.py
@@ -26,55 +26,94 @@ class DINPreprocessor(BasePreprocessor):
     including sequence building, label encoding, and handling of various
     types of input columns (categorical, continuous, and sequential).
 
-    Parameters:
-    -----------
+    Parameters
+    ----------
     user_id_col : str
         Name of the column containing user IDs.
     item_embed_col : Union[str, Tuple[str, int]]
         Name of the column containing item IDs to be embedded, or a tuple of
-        (column_name, embedding_dim).
+        (column_name, embedding_dim). If embedding_dim is not provided, it
+        will be automatically calculated using a rule of thumb specified by
+        `embedding_rule`. Aliased as `item_id_col`.
     target_col : str
         Name of the column containing the target variable.
     max_seq_length : int
         Maximum length of sequences to be created.
     action_col : Optional[str], default=None
-        Name of the column containing user actions (if applicable).
+        Name of the column containing user actions (if applicable), for
+        example a rating, or purchased/not-purchased, etc). This 'action_col'
+        can also be the same as the 'target_col'.
     other_seq_embed_cols : Optional[List[str] | List[Tuple[str, int]]], default=None
-        List of other columns to be treated as sequences.
+        List of other columns to be treated as sequences. Each element in the
+        list can be a string with the name of the column or a tuple with the
+        name of the column and the embedding dimension.
+
     cat_embed_cols : Optional[Union[List[str], List[Tuple[str, int]]]], default=None
-        List of categorical columns to be represented by embeddings.
+        This and the following parameters are identical to those used in the
+        TabPreprocessor.<br>
+        List containing the name of the categorical columns that will be
+        represented by embeddings (e.g. _['education', 'relationship', ...]_) or
+        a Tuple with the name and the embedding dimension (e.g.: _[
+        ('education',32), ('relationship',16), ...]_)
     continuous_cols: List, default = None
-        List with the name of the continuous cols.
-    quantization_setup: int or Dict[str, Union[int, List[float]]], default=None
-        Continuous columns can be turned into categorical via `pd.cut`.
+        List with the name of the continuous cols
+    quantization_setup: int or Dict, default = None
+        Continuous columns can be turned into categorical via `pd.cut`. If
+        `quantization_setup` is an `int`, all continuous columns will be
+        quantized using this value as the number of bins. Alternatively, a
+        dictionary where the keys are the column names to quantize and the
+        values are the either integers indicating the number of bins or a
+        list of scalars indicating the bin edges can also be used.
     cols_to_scale: List or str, default = None,
-        List with the names of the columns that will be standardized via
-        sklearn's `StandardScaler`.
+        List with the names of the columns that will be standarised via
+        sklearn's `StandardScaler`. It can also be the string `'all'` in
+        which case all the continuous cols will be scaled.
     auto_embed_dim: bool, default = True
         Boolean indicating whether the embedding dimensions will be
-        automatically defined via rule of thumb.
+        automatically defined via rule of thumb. See `embedding_rule`
+        below.
     embedding_rule: str, default = 'fastai_new'
-        Rule of thumb for embedding size.
+        If `auto_embed_dim=True`, this is the choice of embedding rule of
+        thumb. Choices are:
+
+        - _fastai_new_: $min(600, round(1.6 \times n_{cat}^{0.56}))$
+
+        - _fastai_old_: $min(50, (n_{cat}//{2})+1)$
+
+        - _google_: $min(600, round(n_{cat}^{0.24}))$
     default_embed_dim: int, default=16
-        Default dimension for the embeddings.
+        Dimension for the embeddings if the embedding dimension is not
+        provided in the `cat_embed_cols` parameter and `auto_embed_dim` is
+        set to `False`.
     verbose : int, default=1
         Verbosity level.
     scale: bool, default = False
-        Boolean indicating whether or not to scale/standardize continuous cols.
+        :information_source: **note**: this arg will be removed in upcoming
+         releases. Please use `cols_to_scale` instead. <br/> Bool indicating
+         whether or not to scale/standarise continuous cols. It is important
+         to emphasize that all the DL models for tabular data in the library
+         also include the possibility of normalising the input continuous
+         features via a `BatchNorm` or a `LayerNorm`. <br/> Param alias:
+         `scale_cont_cols`.
     already_standard: List, default = None
-        List with the name of the continuous cols that do not need to be
-        scaled/standardized.
-    **kwargs :
-        Additional keyword arguments to be passed to the TabPreprocessor.
-
-    Attributes:
-    -----------
+        :information_source: **note**: this arg will be removed in upcoming
+         releases. Please use `cols_to_scale` instead. <br/> List with the
+         name of the continuous cols that do not need to be
+         scaled/standarised.
+
+    Other Parameters
+    ----------------
+    **kwargs: dict
+        `pd.cut` and `StandardScaler` related args
+
+    Attributes
+    ----------
     is_fitted : bool
         Whether the preprocessor has been fitted.
     has_standard_tab_data : bool
         Whether the data includes standard tabular data.
     tab_preprocessor : TabPreprocessor
-        Preprocessor for standard tabular data.
+        Preprocessor for standard tabular data (if applicable).
     din_columns_idx : Dict[str, int]
         Dictionary mapping column names to their indices in the processed data.
     item_le : LabelEncoder
@@ -96,8 +135,8 @@ class DINPreprocessor(BasePreprocessor):
     other_seq_config : List[Tuple[List[str], int, int]]
         Configuration for other sequence columns.
 
-    Examples:
-    ---------
+    Examples
+    --------
     >>> import pandas as pd
     >>> from pytorch_widedeep.preprocessing import DINPreprocessor
     >>> data = {
diff --git a/pytorch_widedeep/preprocessing/tab_preprocessor.py b/pytorch_widedeep/preprocessing/tab_preprocessor.py
index 4490fe46..18762691 100644
--- a/pytorch_widedeep/preprocessing/tab_preprocessor.py
+++ b/pytorch_widedeep/preprocessing/tab_preprocessor.py
@@ -116,7 +116,7 @@ class TabPreprocessor(BasePreprocessor):
         List containing the name of the categorical columns that will be
         represented by embeddings (e.g. _['education', 'relationship', ...]_) or
         a Tuple with the name and the embedding dimension (e.g.: _[
-        ('education',32), ('relationship',16), ...]_)
+        ('education',32), ('relationship',16), ...]_).<br/> Param alias: `embed_cols`
     continuous_cols: List, default = None
         List with the name of the continuous cols
     quantization_setup: int or Dict, default = None
@@ -125,7 +125,8 @@ class TabPreprocessor(BasePreprocessor):
         quantized using this value as the number of bins. Alternatively, a
         dictionary where the keys are the column names to quantize and the
         values are the either integers indicating the number of bins or a
-        list of scalars indicating the bin edges can also be used.
+        list of scalars indicating the bin edges can also be used.<br/>
+        Param alias: `cols_and_bins`
     cols_to_scale: List or str, default = None,
         List with the names of the columns that will be standarised via
         sklearn's `StandardScaler`. It can also be the string `'all'` in
@@ -156,7 +157,7 @@ class TabPreprocessor(BasePreprocessor):
         _['education', 'relationship', ...]_. This is because they will all be
          encoded using embeddings of the same dim, which will be specified
          later when the model is defined. <br/> Param alias:
-         `for_transformer`
+         `for_transformer`, `for_matrix_factorization`, `for_mf`
     with_cls_token: bool, default = False
         Boolean indicating if a `'[CLS]'` token will be added to the dataset
         when using attention-based models. The final hidden state
@@ -194,26 +195,36 @@ class TabPreprocessor(BasePreprocessor):
 
     Attributes
     ----------
-    embed_dim: Dict
-        Dictionary where keys are the embed cols and values are the embedding
-        dimensions. If `with_attention` is set to `True` this attribute
-        is not generated during the `fit` process
+    cat_cols: List[str]
+        List containing the names of the categorical columns after processing.
+    cols_and_bins: Optional[Dict[str, Union[int, List[float]]]]
+        Dictionary containing the quantization setup after processing if
+        quantization is requested. This is derived from the
+        quantization_setup parameter.
+    quant_args: Dict
+        Dictionary containing arguments passed to pandas.cut() function for quantization.
+    scale_args: Dict
+        Dictionary containing arguments passed to StandardScaler.
     label_encoder: LabelEncoder
-        see `pytorch_widedeep.utils.dense_utils.LabelEncder`
+        Instance of LabelEncoder used to encode categorical variables.
+        See `pytorch_widedeep.utils.dense_utils.LabelEncoder`.
     cat_embed_input: List
-        List of Tuples with the column name, number of individual values for
-        that column and, If `with_attention` is set to `False`, the
+        List of tuples with the column name, number of individual values for
+        that column and, if `with_attention` is set to `False`, the
         corresponding embeddings dim, e.g. _[('education', 16, 10),
         ('relationship', 6, 8), ...]_.
     standardize_cols: List
-        List of the columns that will be standarized
+        List of the columns that will be standardized.
     scaler: StandardScaler
-        an instance of `sklearn.preprocessing.StandardScaler`
+        An instance of `sklearn.preprocessing.StandardScaler` if standardization
+        is requested.
     column_idx: Dict
         Dictionary where keys are column names and values are column indexes.
-        This is neccesary to slice tensors
+        This is necessary to slice tensors.
     quantizer: Quantizer
-        an instance of `Quantizer`
+        An instance of `Quantizer` if quantization is requested.
+    is_fitted: bool
+        Boolean indicating if the preprocessor has been fitted.
 
     Examples
     --------
@@ -711,14 +722,15 @@ class ChunkTabPreprocessor(TabPreprocessor):
         List containing the name of the categorical columns that will be
         represented by embeddings (e.g. _['education', 'relationship', ...]_) or
         a Tuple with the name and the embedding dimension (e.g.: _[
-        ('education',32), ('relationship',16), ...]_)
+        ('education',32), ('relationship',16), ...]_). <br/> Param alias:
+        `embed_cols`
     continuous_cols: List, default = None
         List with the name of the continuous cols
     cols_and_bins: Dict, default = None
         Continuous columns can be turned into categorical via
         `pd.cut`. 'cols_and_bins' is dictionary where the keys are the column
         names to quantize and the values are a list of scalars indicating the
-        bin edges.
+        bin edges. <br/> Param alias: `quantization_setup`
     cols_to_scale: List, default = None,
         List with the names of the columns that will be standarised via
         sklearn's `StandardScaler`
@@ -773,27 +785,36 @@ class ChunkTabPreprocessor(TabPreprocessor):
 
     Attributes
     ----------
-    embed_dim: Dict
-        Dictionary where keys are the embed cols and values are the embedding
-        dimensions. If `with_attention` is set to `True` this attribute
-        is not generated during the `fit` process
+    cat_cols: List[str]
+        List containing the names of the categorical columns after processing.
+    cols_and_bins: Optional[Dict[str, Union[int, List[float]]]]
+        Dictionary containing the quantization setup after processing if
+        quantization is requested. This is derived from the
+        quantization_setup parameter.
+    quant_args: Dict
+        Dictionary containing arguments passed to pandas.cut() function for quantization.
+    scale_args: Dict
+        Dictionary containing arguments passed to StandardScaler.
     label_encoder: LabelEncoder
-        see `pytorch_widedeep.utils.dense_utils.LabelEncder`
+        Instance of LabelEncoder used to encode categorical variables.
+        See `pytorch_widedeep.utils.dense_utils.LabelEncoder`.
     cat_embed_input: List
-        List of Tuples with the column name, number of individual values for
-        that column and, If `with_attention` is set to `False`, the
+        List of tuples with the column name, number of individual values for
+        that column and, if `with_attention` is set to `False`, the
         corresponding embeddings dim, e.g. _[('education', 16, 10),
         ('relationship', 6, 8), ...]_.
     standardize_cols: List
-        List of the columns that will be standarized
+        List of the columns that will be standardized.
     scaler: StandardScaler
-        an instance of `sklearn.preprocessing.StandardScaler`
-        if 'cols_to_scale' is not None or 'scale' is 'True'
+        An instance of `sklearn.preprocessing.StandardScaler` if standardization
+        is requested.
     column_idx: Dict
         Dictionary where keys are column names and values are column indexes.
-        This is neccesary to slice tensors
+        This is necessary to slice tensors.
     quantizer: Quantizer
-        an instance of `Quantizer`
+        An instance of `Quantizer` if quantization is requested.
+    is_fitted: bool
+        Boolean indicating if the preprocessor has been fitted.
 
     Examples
     --------
diff --git a/pytorch_widedeep/self_supervised_training/_base_contrastive_denoising_trainer.py b/pytorch_widedeep/self_supervised_training/_base_contrastive_denoising_trainer.py
index 11c9b835..0756e18e 100644
--- a/pytorch_widedeep/self_supervised_training/_base_contrastive_denoising_trainer.py
+++ b/pytorch_widedeep/self_supervised_training/_base_contrastive_denoising_trainer.py
@@ -105,9 +105,9 @@ def pretrain(
     def save(
         self,
         path: str,
-        save_state_dict: bool,
-        save_optimizer: bool,
-        model_filename: str,
+        save_state_dict: bool = False,
+        save_optimizer: bool = False,
+        model_filename: str = "wd_model.pt",
     ):
         r"""Saves the model, training and evaluation history (if any) to disk
 
diff --git a/pytorch_widedeep/self_supervised_training/_base_encoder_decoder_trainer.py b/pytorch_widedeep/self_supervised_training/_base_encoder_decoder_trainer.py
index 97491b4f..05ad1750 100644
--- a/pytorch_widedeep/self_supervised_training/_base_encoder_decoder_trainer.py
+++ b/pytorch_widedeep/self_supervised_training/_base_encoder_decoder_trainer.py
@@ -80,9 +80,9 @@ def pretrain(
     def save(
         self,
         path: str,
-        save_state_dict: bool,
-        save_optimizer: bool,
-        model_filename: str,
+        save_state_dict: bool = False,
+        save_optimizer: bool = False,
+        model_filename: str = "wd_model.pt",
     ):
         r"""Saves the model, training and evaluation history (if any) to disk
 
diff --git a/pytorch_widedeep/self_supervised_training/contrastive_denoising_trainer.py b/pytorch_widedeep/self_supervised_training/contrastive_denoising_trainer.py
index 15c4d6ad..90f2f90a 100644
--- a/pytorch_widedeep/self_supervised_training/contrastive_denoising_trainer.py
+++ b/pytorch_widedeep/self_supervised_training/contrastive_denoising_trainer.py
@@ -17,6 +17,7 @@
 )
 from pytorch_widedeep.callbacks import Callback
 from pytorch_widedeep.preprocessing import TabPreprocessor
+from pytorch_widedeep.utils.general_utils import to_device
 from pytorch_widedeep.training._trainer_utils import (
     save_epoch_logs,
     print_loss_and_metric,
@@ -257,7 +258,7 @@ def fit(
         )
 
     def _train_step(self, X_tab: Tensor, batch_idx: int) -> float:
-        X = X_tab.to(self.device)
+        X = to_device(X_tab, self.device)
 
         self.optimizer.zero_grad()
         g_projs, cat_x_and_x_, cont_x_and_x_ = self.cd_model(X)
@@ -274,7 +275,7 @@ def _eval_step(self, X_tab: Tensor, batch_idx: int) -> float:
         self.cd_model.eval()
 
         with torch.no_grad():
-            X = X_tab.to(self.device)
+            X = to_device(X_tab, self.device)
 
             g_projs, cat_x_and_x_, cont_x_and_x_ = self.cd_model(X)
             loss = self._compute_loss(g_projs, cat_x_and_x_, cont_x_and_x_)
diff --git a/pytorch_widedeep/self_supervised_training/encoder_decoder_trainer.py b/pytorch_widedeep/self_supervised_training/encoder_decoder_trainer.py
index 6b2ce440..d03185f2 100644
--- a/pytorch_widedeep/self_supervised_training/encoder_decoder_trainer.py
+++ b/pytorch_widedeep/self_supervised_training/encoder_decoder_trainer.py
@@ -16,6 +16,7 @@
     DecoderWithoutAttention,
 )
 from pytorch_widedeep.callbacks import Callback
+from pytorch_widedeep.utils.general_utils import to_device
 from pytorch_widedeep.training._trainer_utils import (
     save_epoch_logs,
     print_loss_and_metric,
@@ -214,7 +215,7 @@ def explain(self, X_tab: np.ndarray, save_step_masks: bool = False):
         )
 
     def _train_step(self, X_tab: Tensor, batch_idx: int) -> float:
-        X = X_tab.to(self.device)
+        X = to_device(X_tab, self.device)
 
         self.optimizer.zero_grad()
         x_embed, x_embed_rec, mask = self.ed_model(X)
@@ -231,7 +232,7 @@ def _eval_step(self, X_tab: Tensor, batch_idx: int) -> float:
         self.ed_model.eval()
 
         with torch.no_grad():
-            X = X_tab.to(self.device)
+            X = to_device(X_tab, self.device)
 
             x_embed, x_embed_rec, mask = self.ed_model(X)
             loss = self.loss_fn(x_embed, x_embed_rec, mask)
diff --git a/pytorch_widedeep/tab2vec.py b/pytorch_widedeep/tab2vec.py
index c10ddb56..c70eecf7 100644
--- a/pytorch_widedeep/tab2vec.py
+++ b/pytorch_widedeep/tab2vec.py
@@ -85,7 +85,7 @@ def __init__(
         self,
         tab_preprocessor: TabPreprocessor,
         model: Union[WideDeep, BayesianWide, BayesianTabMlp],
-        device: str | torch.device,
+        device: Union[str, torch.device],
         return_dataframe: bool = False,
         verbose: bool = False,
     ):
diff --git a/pytorch_widedeep/training/_base_trainer.py b/pytorch_widedeep/training/_base_trainer.py
index 2f15a3bb..6d24f690 100644
--- a/pytorch_widedeep/training/_base_trainer.py
+++ b/pytorch_widedeep/training/_base_trainer.py
@@ -10,7 +10,7 @@
 from torchmetrics import Metric as TorchMetric
 from torch.optim.lr_scheduler import ReduceLROnPlateau
 
-from pytorch_widedeep.metrics import Metric, RankingMetrics, MultipleMetrics
+from pytorch_widedeep.metrics import Metric, MultipleMetrics, is_ranking_metric
 from pytorch_widedeep.wdtypes import (
     Any,
     Dict,
@@ -304,9 +304,7 @@ def _set_callbacks_and_metrics(
         if metrics is not None:
             self.metric = MultipleMetrics(metrics)
             if (
-                any(
-                    [isinstance(m, RankingMetrics) for m in self.metric._metrics]  # type: ignore[arg-type, misc]
-                )
+                any([is_ranking_metric(m) for m in self.metric._metrics])
                 and self.verbose
             ):
                 UserWarning(
@@ -320,9 +318,7 @@ def _set_callbacks_and_metrics(
         if eval_metrics is not None:
             self.eval_metric = MultipleMetrics(eval_metrics)
             if (
-                any(
-                    [isinstance(m, RankingMetrics) for m in self.eval_metric._metrics]  # type: ignore[arg-type, misc]
-                )
+                any([is_ranking_metric(m) for m in self.eval_metric._metrics])
                 and self.verbose
             ):
                 UserWarning(
diff --git a/pytorch_widedeep/training/_wd_dataset.py b/pytorch_widedeep/training/_wd_dataset.py
index 217d9f73..2abb78d4 100644
--- a/pytorch_widedeep/training/_wd_dataset.py
+++ b/pytorch_widedeep/training/_wd_dataset.py
@@ -237,7 +237,7 @@ def __len__(self):
 def group_collate_fn(batch: List[Tuple[Bunch, np.ndarray]]):
 
     all_targets: List[np.ndarray] = []
-    all_features: Dict[str, List[np.ndarray] | List[List[np.ndarray]]] = {
+    all_features: Dict[str, Union[List[np.ndarray], List[List[np.ndarray]]]] = {
         "wide": [],
         "deeptabular": [],
         "deeptext": [],
@@ -250,7 +250,7 @@ def group_collate_fn(batch: List[Tuple[Bunch, np.ndarray]]):
         all_targets.append(targets)
     all_features = {k: v for k, v in all_features.items() if v}
 
-    batch_features: Dict[str, torch.Tensor | List[torch.Tensor]] = {}
+    batch_features: Dict[str, Union[torch.Tensor, List[torch.Tensor]]] = {}
 
     for key, value in all_features.items():
         if isinstance(value[0], list):
diff --git a/pytorch_widedeep/training/trainer.py b/pytorch_widedeep/training/trainer.py
index e259268d..997c5e7b 100644
--- a/pytorch_widedeep/training/trainer.py
+++ b/pytorch_widedeep/training/trainer.py
@@ -787,7 +787,7 @@ def _set_dataloader(
         dataset: WideDeepDataset,
         batch_size: int,
         dataloader_args: Dict[str, Any],
-    ) -> DataLoader | CustomDataLoader:
+    ) -> Union[DataLoader, CustomDataLoader]:
         if dataloader is not None:
             dataloader.set_dataset(dataset)
             return dataloader
diff --git a/pytorch_widedeep/utils/image_utils.py b/pytorch_widedeep/utils/image_utils.py
index 9dfce505..8355be68 100644
--- a/pytorch_widedeep/utils/image_utils.py
+++ b/pytorch_widedeep/utils/image_utils.py
@@ -9,11 +9,43 @@
 
 import cv2
 import numpy as np
-import imutils
 
 __all__ = ["AspectAwarePreprocessor", "SimplePreprocessor"]
 
 
+# credit for the code here to ADRIAN ROSEBROCK
+def resize(image, width=None, height=None, inter=cv2.INTER_AREA):
+    # initialize the dimensions of the image to be resized and
+    # grab the image size
+    dim = None
+    (h, w) = image.shape[:2]
+
+    # if both the width and height are None, then return the
+    # original image
+    if width is None and height is None:
+        return image
+
+    # check to see if the width is None
+    if width is None:
+        # calculate the ratio of the height and construct the
+        # dimensions
+        r = height / float(h)
+        dim = (int(w * r), height)
+
+    # otherwise, the height is None
+    else:
+        # calculate the ratio of the width and construct the
+        # dimensions
+        r = width / float(w)
+        dim = (width, int(h * r))
+
+    # resize the image
+    resized = cv2.resize(image, dim, interpolation=inter)
+
+    # return the resized image
+    return resized
+
+
 class AspectAwarePreprocessor:
     r"""Class to resize an image to a certain width and height taking into account
     the image aspect ratio
@@ -64,10 +96,10 @@ def preprocess(self, image: np.ndarray) -> np.ndarray:
         dH = 0
 
         if w < h:
-            image = imutils.resize(image, width=self.width, inter=self.inter)
+            image = resize(image, width=self.width, inter=self.inter)
             dH = int((image.shape[0] - self.height) / 2.0)
         else:
-            image = imutils.resize(image, height=self.height, inter=self.inter)
+            image = resize(image, height=self.height, inter=self.inter)
             dW = int((image.shape[1] - self.width) / 2.0)
 
         (h, w) = image.shape[:2]
diff --git a/pytorch_widedeep/utils/text_utils.py b/pytorch_widedeep/utils/text_utils.py
index 353b188b..e4bbbaef 100644
--- a/pytorch_widedeep/utils/text_utils.py
+++ b/pytorch_widedeep/utils/text_utils.py
@@ -64,6 +64,13 @@ def get_texts(
 
     See `pytorch_widedeep.utils.fastai_utils.Tokenizer`
 
+    :information_source: **NOTE**:
+    `get_texts` uses `pytorch_widedeep.utils.fastai_transforms.Tokenizer`.
+    Such tokenizer uses a series of convenient processing steps, including
+    the  addition of some special tokens, such as `TK_MAJ` (`xxmaj`), used to
+    indicate the next word begins with a capital in the original text. For more
+    details of special tokens please see the [`fastai` `docs](https://docs.fast.ai/text.core.html#Tokenizing)
+
     Parameters
     ----------
     texts: List
@@ -87,13 +94,6 @@ def get_texts(
     -------
     List[List[str]]
         List of lists, one list per '_document_' containing its corresponding tokens
-
-    :information_source: **NOTE**:
-    `get_texts` uses `pytorch_widedeep.utils.fastai_transforms.Tokenizer`.
-    Such tokenizer uses a series of convenient processing steps, including
-    the  addition of some special tokens, such as `TK_MAJ` (`xxmaj`), used to
-    indicate the next word begins with a capital in the original text. For more
-    details of special tokens please see the [`fastai` `docs](https://docs.fast.ai/text.core.html#Tokenizing)
     """
 
     num_cpus = n_cpus if n_cpus is not None else os.cpu_count()
diff --git a/requirements.txt b/requirements.txt
index 95257a66..848670ce 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -5,7 +5,6 @@ scikit-learn>=1.0.2
 gensim
 spacy
 opencv-contrib-python>=4.9.0.80
-imutils
 tqdm
 torch >= 2.0.0
 torchvision >= 0.15.0
diff --git a/tests/test_data_utils/test_du_image.py b/tests/test_data_utils/test_du_image.py
index ee8ceaf5..470e807a 100644
--- a/tests/test_data_utils/test_du_image.py
+++ b/tests/test_data_utils/test_du_image.py
@@ -7,6 +7,7 @@
 
 from pytorch_widedeep.utils import SimplePreprocessor, AspectAwarePreprocessor
 from pytorch_widedeep.preprocessing import ImagePreprocessor
+from pytorch_widedeep.utils.image_utils import resize
 
 full_path = os.path.realpath(__file__)
 path = os.path.split(full_path)[0]
@@ -53,3 +54,65 @@ def test_sizes():
 def test_notimplementederror():
     with pytest.raises(NotImplementedError):
         org_df = processor.inverse_transform(X_imgs)  # noqa: F841
+
+
+###############################################################################
+# testthe resize function
+###############################################################################
+
+
+@pytest.fixture
+def sample_image():
+    # Create a sample 100x200 RGB image
+    return np.zeros((100, 200, 3), dtype=np.uint8)
+
+
+def test_resize_width_only(sample_image):
+    # Test resizing with only width specified
+    width = 100
+    resized = resize(sample_image, width=width)
+
+    # Check dimensions
+    assert resized.shape[1] == width  # width should match
+    assert resized.shape[2] == 3  # channels should remain unchanged
+    # Height should be proportionally scaled (50 in this case)
+    assert resized.shape[0] == 50
+
+
+def test_resize_height_only(sample_image):
+    # Test resizing with only height specified
+    height = 50
+    resized = resize(sample_image, height=height)
+
+    # Check dimensions
+    assert resized.shape[0] == height  # height should match
+    assert resized.shape[2] == 3  # channels should remain unchanged
+    # Width should be proportionally scaled (100 in this case)
+    assert resized.shape[1] == 100
+
+
+def test_resize_none_dimensions(sample_image):
+    # Test when both width and height are None
+    resized = resize(sample_image, width=None, height=None)
+
+    # Image should remain unchanged
+    assert np.array_equal(resized, sample_image)
+    assert resized.shape == sample_image.shape
+
+
+def test_resize_different_interpolation(sample_image):
+    # Test with different interpolation method
+    width = 100
+    resized = resize(sample_image, width=width, inter=cv2.INTER_LINEAR)
+
+    assert resized.shape[1] == width
+    assert resized.shape[2] == 3
+    assert resized.shape[0] == 50
+
+
+def test_resize_maintains_type(sample_image):
+    # Test that the output maintains the same dtype as input
+    width = 100
+    resized = resize(sample_image, width=width)
+
+    assert resized.dtype == sample_image.dtype
diff --git a/tests/test_rec/test_autoint.py b/tests/test_rec/test_autoint.py
index 0294236b..d0bb3432 100644
--- a/tests/test_rec/test_autoint.py
+++ b/tests/test_rec/test_autoint.py
@@ -103,3 +103,52 @@ def test_autoint_full_process(cat_embed_cols):
         and "train_loss" in trainer.history
         and "val_loss" in trainer.history
     )
+
+
+@pytest.mark.parametrize(
+    "embed_continuous_method", ["periodic", "piecewise", "standard"]
+)
+def test_autoint_cont_embed_methods(
+    embed_continuous_method, cat_embed_cols, continuous_cols
+):
+    tab_preprocessor = TabPreprocessor(
+        cat_embed_cols=cat_embed_cols,
+        continuous_cols=continuous_cols,
+        with_attention=True,
+    )
+
+    X_tab = tab_preprocessor.fit_transform(train)
+    X_tab_tnsr = torch.tensor(X_tab, dtype=torch.float32)
+
+    if embed_continuous_method == "periodic":
+        model = AutoInt(
+            column_idx=tab_preprocessor.column_idx,
+            input_dim=16,
+            cat_embed_input=tab_preprocessor.cat_embed_input,
+            num_heads=2,
+            num_layers=2,
+            continuous_cols=continuous_cols,
+            embed_continuous_method=embed_continuous_method,
+            n_frequencies=4,
+            sigma=0.1,
+            share_last_layer=False,
+        )
+    else:
+        quantization_setup = {
+            "item_price": list(train["item_price"].quantile([0.25, 0.5, 0.75]).values),
+            "user_age": list(train["user_age"].quantile([0.25, 0.5, 0.75]).values),
+        }
+        model = AutoInt(
+            column_idx=tab_preprocessor.column_idx,
+            input_dim=16,
+            cat_embed_input=tab_preprocessor.cat_embed_input,
+            num_heads=2,
+            num_layers=2,
+            continuous_cols=continuous_cols,
+            embed_continuous_method=embed_continuous_method,
+            quantization_setup=quantization_setup,
+        )
+
+    res = model(X_tab_tnsr)
+
+    assert res.shape == (X_tab_tnsr.shape[0], model.output_dim)
diff --git a/tests/test_rec/test_autoint_plus.py b/tests/test_rec/test_autoint_plus.py
index 3819d9d0..9c5b6bb4 100644
--- a/tests/test_rec/test_autoint_plus.py
+++ b/tests/test_rec/test_autoint_plus.py
@@ -44,6 +44,7 @@ def test_autoint_plus_reduction(
         gated=gated,
         continuous_cols=continuous_cols,
         embed_continuous_method="standard",
+        mlp_hidden_dims=[32, 16],
     )
 
     res = model(X_tab_tnsr)
@@ -72,6 +73,7 @@ def test_autoint_plus_gating_validation_errors(cat_embed_cols, continuous_cols):
             gated=True,
             continuous_cols=continuous_cols,
             embed_continuous_method="standard",
+            mlp_hidden_dims=[32, 16],
         )
 
     # Test 2: ValueError when using gated with reduction="cat"
@@ -87,6 +89,7 @@ def test_autoint_plus_gating_validation_errors(cat_embed_cols, continuous_cols):
             gated=True,
             continuous_cols=continuous_cols,
             embed_continuous_method="standard",
+            mlp_hidden_dims=[32, 16],
         )
 
 
@@ -111,6 +114,7 @@ def test_autoint_plus_with_wide_component(cat_embed_cols):
         cat_embed_input=tab_preprocessor.cat_embed_input,
         structure="parallel",
         gated=True,
+        mlp_hidden_dims=[32, 16],
     )
 
     linear = Wide(input_dim=X_wide.max())
@@ -153,6 +157,7 @@ def test_autoint_plus_full_process(structure, gated, reduction, cat_embed_cols):
         structure=structure,
         gated=gated,
         reduction=reduction,
+        mlp_hidden_dims=[32, 16],
     )
 
     model = WideDeep(deeptabular=auto_int_plus)
@@ -171,3 +176,54 @@ def test_autoint_plus_full_process(structure, gated, reduction, cat_embed_cols):
         and "train_loss" in trainer.history
         and "val_loss" in trainer.history
     )
+
+
+@pytest.mark.parametrize(
+    "embed_continuous_method", ["periodic", "piecewise", "standard"]
+)
+def test_autoint_plus_cont_embed_methods(
+    embed_continuous_method, cat_embed_cols, continuous_cols
+):
+    tab_preprocessor = TabPreprocessor(
+        cat_embed_cols=cat_embed_cols,
+        continuous_cols=continuous_cols,
+        with_attention=True,
+    )
+
+    X_tab = tab_preprocessor.fit_transform(train)
+    X_tab_tnsr = torch.tensor(X_tab, dtype=torch.float32)
+
+    if embed_continuous_method == "periodic":
+        model = AutoIntPlus(
+            column_idx=tab_preprocessor.column_idx,
+            input_dim=16,
+            cat_embed_input=tab_preprocessor.cat_embed_input,
+            num_heads=2,
+            num_layers=2,
+            continuous_cols=continuous_cols,
+            embed_continuous_method=embed_continuous_method,
+            n_frequencies=4,
+            sigma=0.1,
+            share_last_layer=False,
+            mlp_hidden_dims=[32, 16],
+        )
+    else:
+        quantization_setup = {
+            "item_price": list(train["item_price"].quantile([0.25, 0.5, 0.75]).values),
+            "user_age": list(train["user_age"].quantile([0.25, 0.5, 0.75]).values),
+        }
+        model = AutoIntPlus(
+            column_idx=tab_preprocessor.column_idx,
+            input_dim=16,
+            cat_embed_input=tab_preprocessor.cat_embed_input,
+            num_heads=2,
+            num_layers=2,
+            continuous_cols=continuous_cols,
+            embed_continuous_method=embed_continuous_method,
+            quantization_setup=quantization_setup,
+            mlp_hidden_dims=[32, 16],
+        )
+
+    res = model(X_tab_tnsr)
+
+    assert res.shape == (X_tab_tnsr.shape[0], model.output_dim)
diff --git a/tests/test_rec/test_deepfm.py b/tests/test_rec/test_deepfm.py
index 55592bec..489e660f 100644
--- a/tests/test_rec/test_deepfm.py
+++ b/tests/test_rec/test_deepfm.py
@@ -54,7 +54,9 @@ def test_deepfm_reduce_sum(
         assert res.shape == (X_tab_tnsr.shape[0], model.output_dim)
 
 
-@pytest.mark.parametrize("embed_continuous_method", ["periodic", "piecewise"])
+@pytest.mark.parametrize(
+    "embed_continuous_method", ["periodic", "piecewise", "standard"]
+)
 @pytest.mark.parametrize("reduce_sum", [True, False])
 def test_deepfm_cont_embed_methods(
     embed_continuous_method, reduce_sum, cat_embed_cols, continuous_cols
diff --git a/tests/test_rec/test_xdeepfm.py b/tests/test_rec/test_xdeepfm.py
index 0b2273c0..58bbba02 100644
--- a/tests/test_rec/test_xdeepfm.py
+++ b/tests/test_rec/test_xdeepfm.py
@@ -55,7 +55,9 @@ def test_xdeepfm_reduce_sum(
         assert res.shape == (X_tab_tnsr.shape[0], xdeepfm.output_dim)
 
 
-@pytest.mark.parametrize("embed_continuous_method", ["periodic", "piecewise"])
+@pytest.mark.parametrize(
+    "embed_continuous_method", ["periodic", "piecewise", "standard"]
+)
 @pytest.mark.parametrize("reduce_sum", [True, False])
 def test_xdeepfm_cont_embed_methods(
     embed_continuous_method, reduce_sum, cat_embed_cols, continuous_cols

From 389d55e133f6566d663453527f48922ba4689a68 Mon Sep 17 00:00:00 2001
From: Javier <jrzaurin@gmail.com>
Date: Sat, 2 Nov 2024 16:00:39 +0000
Subject: [PATCH 22/24] comment out a type error (ill come back). Only thing
 left is to review the examples, both notebooks and scripts and publihs

---
 pytorch_widedeep/utils/fastai_transforms.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pytorch_widedeep/utils/fastai_transforms.py b/pytorch_widedeep/utils/fastai_transforms.py
index 667f5212..a19cef30 100644
--- a/pytorch_widedeep/utils/fastai_transforms.py
+++ b/pytorch_widedeep/utils/fastai_transforms.py
@@ -414,7 +414,7 @@ def create(
 
         freq = Counter(p for o in tokens for p in o)
         itos = [o for o, c in freq.most_common(self.max_vocab) if c >= self.min_freq]
-        for o in reversed(self.special_cases):
+        for o in reversed(self.special_cases):  # type: ignore[arg-type]
             if o in itos:
                 itos.remove(o)
             itos.insert(0, o)

From 8f504b7c33c861f4e379b2bb18aff013a681fb33 Mon Sep 17 00:00:00 2001
From: Javier <jrzaurin@gmail.com>
Date: Wed, 6 Nov 2024 13:55:47 +0000
Subject: [PATCH 23/24] reviewed the examples and changed the multiprocessing
 approach of the FastaiTokenizer

---
 VERSION                                       |    2 +-
 ...custom_dataLoader_imbalanced_dataset.ipynb |  150 +-
 .../notebooks/09_extracting_embeddings.ipynb  |   32 +-
 .../13_model_uncertainty_prediction.ipynb     |  711 +------
 .../15_Self_Supervised_Pretraning_pt2.ipynb   |   60 +-
 examples/scripts/adult_census.py              |    4 +-
 .../scripts/adult_census_attention_mlp.py     |    3 -
 .../scripts/adult_census_bayesian_tabmlp.py   |    2 -
 .../adult_census_cont_den_full_example.py     |    2 -
 .../adult_census_cont_den_run_all_models.py   |   12 +-
 .../adult_census_enc_dec_full_example.py      |    2 -
 .../adult_census_enc_dec_run_all_models.py    |    2 -
 .../adult_census_feature_importance.py        |    2 -
 ...adult_census_linear_and_flash_attention.py |    2 -
 examples/scripts/adult_census_tabnet.py       |    4 +-
 examples/scripts/adult_census_transformers.py |    2 -
 .../scripts/airbnb_all_modes_multiclass.py    |    2 -
 examples/scripts/airbnb_all_modes_regr.py     |    2 -
 .../scripts/airbnb_load_from_folder_regr.py   |    3 -
 examples/scripts/bio_imbalanced_loader.py     |   55 +-
 examples/scripts/ecomerce_reviews_bert.py     |    2 +
 examples/scripts/tmp.py                       |   64 +
 mkdocs/site/404.html                          |    6 +-
 mkdocs/site/assets/_mkdocstrings.css          |   26 +-
 mkdocs/site/contributing.html                 |    6 +-
 mkdocs/site/contributing.md                   |    7 -
 mkdocs/site/index.html                        |    9 +-
 mkdocs/site/index.md                          |  890 --------
 mkdocs/site/installation.html                 |    6 +-
 mkdocs/site/installation.md                   |   48 -
 .../pytorch-widedeep/bayesian_models.html     |  112 +-
 .../site/pytorch-widedeep/bayesian_models.md  |   14 -
 .../pytorch-widedeep/bayesian_trainer.html    |  131 +-
 .../site/pytorch-widedeep/bayesian_trainer.md |   11 -
 mkdocs/site/pytorch-widedeep/callbacks.html   |   85 +-
 mkdocs/site/pytorch-widedeep/callbacks.md     |   14 -
 mkdocs/site/pytorch-widedeep/dataloaders.html |   15 +-
 mkdocs/site/pytorch-widedeep/dataloaders.md   |    9 -
 mkdocs/site/pytorch-widedeep/losses.html      |  370 +++-
 mkdocs/site/pytorch-widedeep/losses.md        |   66 -
 mkdocs/site/pytorch-widedeep/metrics.html     |   60 +-
 mkdocs/site/pytorch-widedeep/metrics.md       |   43 -
 .../pytorch-widedeep/model_components.html    | 1821 +++++++++++++----
 .../site/pytorch-widedeep/model_components.md |  115 --
 .../site/pytorch-widedeep/preprocessing.html  |  517 +++--
 mkdocs/site/pytorch-widedeep/preprocessing.md |   44 -
 .../self_supervised_pretraining.html          |  184 +-
 .../self_supervised_pretraining.md            |   65 -
 mkdocs/site/pytorch-widedeep/tab2vec.html     |   41 +-
 mkdocs/site/pytorch-widedeep/tab2vec.md       |    3 -
 mkdocs/site/pytorch-widedeep/trainer.html     |  201 +-
 mkdocs/site/pytorch-widedeep/trainer.md       |   21 -
 .../utils/deeptabular_utils.html              |   23 +-
 .../utils/deeptabular_utils.md                |    3 -
 .../utils/fastai_transforms.html              |  155 +-
 .../utils/fastai_transforms.md                |   13 -
 .../pytorch-widedeep/utils/image_utils.html   |   48 +-
 .../pytorch-widedeep/utils/image_utils.md     |   11 -
 mkdocs/site/pytorch-widedeep/utils/index.html |    6 +-
 mkdocs/site/pytorch-widedeep/utils/index.md   |   15 -
 .../pytorch-widedeep/utils/text_utils.html    |   70 +-
 .../site/pytorch-widedeep/utils/text_utils.md |   11 -
 mkdocs/site/quick_start.html                  |    6 +-
 mkdocs/site/quick_start.md                    |  105 -
 mkdocs/site/search/search_index.json          |    2 +-
 mkdocs/site/sitemap.xml                       |   88 +-
 mkdocs/site/sitemap.xml.gz                    |  Bin 796 -> 796 bytes
 ...custom_dataLoader_imbalanced_dataset.ipynb |  150 +-
 .../examples/09_extracting_embeddings.ipynb   |   32 +-
 .../13_model_uncertainty_prediction.ipynb     |  711 +------
 .../15_Self_Supervised_Pretraning_pt2.ipynb   |   60 +-
 pytorch_widedeep/utils/fastai_transforms.py   |   12 +-
 pytorch_widedeep/version.py                   |    2 +-
 73 files changed, 3497 insertions(+), 4086 deletions(-)
 create mode 100644 examples/scripts/tmp.py
 delete mode 100644 mkdocs/site/contributing.md
 delete mode 100644 mkdocs/site/index.md
 delete mode 100644 mkdocs/site/installation.md
 delete mode 100644 mkdocs/site/pytorch-widedeep/bayesian_models.md
 delete mode 100644 mkdocs/site/pytorch-widedeep/bayesian_trainer.md
 delete mode 100644 mkdocs/site/pytorch-widedeep/callbacks.md
 delete mode 100644 mkdocs/site/pytorch-widedeep/dataloaders.md
 delete mode 100644 mkdocs/site/pytorch-widedeep/losses.md
 delete mode 100644 mkdocs/site/pytorch-widedeep/metrics.md
 delete mode 100644 mkdocs/site/pytorch-widedeep/model_components.md
 delete mode 100644 mkdocs/site/pytorch-widedeep/preprocessing.md
 delete mode 100644 mkdocs/site/pytorch-widedeep/self_supervised_pretraining.md
 delete mode 100644 mkdocs/site/pytorch-widedeep/tab2vec.md
 delete mode 100644 mkdocs/site/pytorch-widedeep/trainer.md
 delete mode 100644 mkdocs/site/pytorch-widedeep/utils/deeptabular_utils.md
 delete mode 100644 mkdocs/site/pytorch-widedeep/utils/fastai_transforms.md
 delete mode 100644 mkdocs/site/pytorch-widedeep/utils/image_utils.md
 delete mode 100644 mkdocs/site/pytorch-widedeep/utils/index.md
 delete mode 100644 mkdocs/site/pytorch-widedeep/utils/text_utils.md
 delete mode 100644 mkdocs/site/quick_start.md

diff --git a/VERSION b/VERSION
index 9edc58bb..9f05f9f2 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-1.6.4
+1.6.5
diff --git a/examples/notebooks/08_custom_dataLoader_imbalanced_dataset.ipynb b/examples/notebooks/08_custom_dataLoader_imbalanced_dataset.ipynb
index ced37a9d..7d662a39 100644
--- a/examples/notebooks/08_custom_dataLoader_imbalanced_dataset.ipynb
+++ b/examples/notebooks/08_custom_dataLoader_imbalanced_dataset.ipynb
@@ -33,34 +33,16 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": 15,
    "id": "5d189eca",
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "/Users/javierrodriguezzaurin/.pyenv/versions/3.10.13/envs/widedeep310/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
-      "  from .autonotebook import tqdm as notebook_tqdm\n"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
-    "import numpy as np\n",
-    "import pandas as pd\n",
-    "import torch\n",
-    "from torch.optim import SGD, lr_scheduler\n",
-    "\n",
     "from pytorch_widedeep import Trainer\n",
     "from pytorch_widedeep.preprocessing import TabPreprocessor\n",
     "from pytorch_widedeep.models import TabMlp, WideDeep\n",
-    "from pytorch_widedeep.dataloaders import DataLoaderImbalanced, DataLoaderDefault\n",
-    "from torchmetrics import F1Score as F1_torchmetrics\n",
-    "from torchmetrics import Accuracy as Accuracy_torchmetrics\n",
-    "from torchmetrics import Precision as Precision_torchmetrics\n",
-    "from torchmetrics import Recall as Recall_torchmetrics\n",
-    "from pytorch_widedeep.metrics import Accuracy, Recall, Precision, F1Score, R2Score\n",
+    "from pytorch_widedeep.dataloaders import DataLoaderImbalanced\n",
+    "from pytorch_widedeep.metrics import Accuracy, Precision\n",
     "from pytorch_widedeep.initializers import XavierNormal\n",
     "from pytorch_widedeep.datasets import load_bio_kdd04\n",
     "\n",
@@ -72,16 +54,12 @@
     "\n",
     "import warnings\n",
     "\n",
-    "warnings.filterwarnings(\"ignore\", category=DeprecationWarning)\n",
-    "\n",
-    "# increase displayed columns in jupyter notebook\n",
-    "pd.set_option(\"display.max_columns\", 200)\n",
-    "pd.set_option(\"display.max_rows\", 300)"
+    "warnings.filterwarnings(\"ignore\", category=DeprecationWarning)"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": 10,
    "id": "e72ebad3",
    "metadata": {},
    "outputs": [],
@@ -102,7 +80,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": 11,
    "id": "81cf3284",
    "metadata": {},
    "outputs": [],
@@ -121,7 +99,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": 12,
    "id": "d202b17c",
    "metadata": {},
    "outputs": [
@@ -135,55 +113,33 @@
        "      (encoder): MLP(\n",
        "        (mlp): Sequential(\n",
        "          (dense_layer_0): Sequential(\n",
-       "            (0): Linear(in_features=74, out_features=148, bias=True)\n",
+       "            (0): Linear(in_features=74, out_features=64, bias=True)\n",
        "            (1): ReLU(inplace=True)\n",
        "            (2): Dropout(p=0.1, inplace=False)\n",
        "          )\n",
        "          (dense_layer_1): Sequential(\n",
-       "            (0): Linear(in_features=148, out_features=118, bias=True)\n",
-       "            (1): ReLU(inplace=True)\n",
-       "            (2): Dropout(p=0.1, inplace=False)\n",
-       "          )\n",
-       "          (dense_layer_2): Sequential(\n",
-       "            (0): Linear(in_features=118, out_features=89, bias=True)\n",
-       "            (1): ReLU(inplace=True)\n",
-       "            (2): Dropout(p=0.1, inplace=False)\n",
-       "          )\n",
-       "          (dense_layer_3): Sequential(\n",
-       "            (0): Linear(in_features=89, out_features=59, bias=True)\n",
-       "            (1): ReLU(inplace=True)\n",
-       "            (2): Dropout(p=0.1, inplace=False)\n",
-       "          )\n",
-       "          (dense_layer_4): Sequential(\n",
-       "            (0): Linear(in_features=59, out_features=30, bias=True)\n",
+       "            (0): Linear(in_features=64, out_features=32, bias=True)\n",
        "            (1): ReLU(inplace=True)\n",
        "            (2): Dropout(p=0.1, inplace=False)\n",
        "          )\n",
        "        )\n",
        "      )\n",
        "    )\n",
-       "    (1): Linear(in_features=30, out_features=1, bias=True)\n",
+       "    (1): Linear(in_features=32, out_features=1, bias=True)\n",
        "  )\n",
        ")"
       ]
      },
-     "execution_count": 4,
+     "execution_count": 12,
      "metadata": {},
      "output_type": "execute_result"
     }
    ],
    "source": [
-    "# Define the model\n",
-    "input_layer = len(tab_preprocessor.continuous_cols)\n",
-    "output_layer = 1\n",
-    "hidden_layers = np.linspace(\n",
-    "    input_layer * 2, output_layer, 5, endpoint=False, dtype=int\n",
-    ").tolist()\n",
-    "\n",
     "deeptabular = TabMlp(\n",
     "    column_idx=tab_preprocessor.column_idx,\n",
     "    continuous_cols=tab_preprocessor.continuous_cols,\n",
-    "    mlp_hidden_dims=hidden_layers,\n",
+    "    mlp_hidden_dims=[64, 32],\n",
     ")\n",
     "model = WideDeep(deeptabular=deeptabular)\n",
     "model"
@@ -191,39 +147,22 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": 13,
    "id": "c6262897",
    "metadata": {},
    "outputs": [],
    "source": [
-    "# Metrics from pytorch-widedeep\n",
-    "accuracy = Accuracy(top_k=2)\n",
-    "precision = Precision(average=False)\n",
-    "\n",
-    "# # Metrics from torchmetrics\n",
-    "# accuracy = Accuracy_torchmetrics(average=None, num_classes=1)\n",
-    "# precision = Precision_torchmetrics(average=\"micro\", num_classes=1)\n",
-    "\n",
-    "# Optimizers\n",
-    "deep_opt = SGD(model.deeptabular.parameters(), lr=0.1)\n",
-    "\n",
-    "# LR Scheduler\n",
-    "deep_sch = lr_scheduler.StepLR(deep_opt, step_size=3)\n",
-    "\n",
     "trainer = Trainer(\n",
     "    model,\n",
     "    objective=\"binary\",\n",
-    "    lr_schedulers={\"deeptabular\": deep_sch},\n",
-    "    initializers={\"deeptabular\": XavierNormal},\n",
-    "    optimizers={\"deeptabular\": deep_opt},\n",
-    "    metrics=[accuracy, precision],\n",
+    "    metrics=[Accuracy(), Precision()],\n",
     "    verbose=1,\n",
     ")"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 6,
+   "execution_count": 16,
    "id": "3c34b76f",
    "metadata": {},
    "outputs": [
@@ -231,22 +170,9 @@
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "epoch 1: 100%|███████████████████████████████████████| 325/325 [00:02<00:00, 153.99it/s, loss=0.163, metrics={'acc': 0.9363, 'prec': [0.9358]}]\n",
-      "valid: 100%|███████████████████████████████████████████| 456/456 [00:02<00:00, 205.93it/s, loss=0.1, metrics={'acc': 0.9501, 'prec': [0.1447]}]\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Training time[s]: 0:00:04\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "predict: 100%|██████████████████████████████████████████████████████████████████████████████████████████████| 456/456 [00:01<00:00, 368.16it/s]\n"
+      "epoch 1: 100%|██████████| 2074/2074 [00:10<00:00, 194.90it/s, loss=0.115, metrics={'acc': 0.959, 'prec': 0.9637}] \n",
+      "valid: 100%|██████████| 258/258 [00:00<00:00, 330.04it/s, loss=0.189, metrics={'acc': 0.9147, 'prec': 0.9825}]\n",
+      "predict: 100%|██████████| 456/456 [00:00<00:00, 679.61it/s]\n"
      ]
     },
     {
@@ -255,39 +181,33 @@
      "text": [
       "              precision    recall  f1-score   support\n",
       "\n",
-      "           0       1.00      0.95      0.97     14446\n",
-      "           1       0.15      0.95      0.25       130\n",
+      "           0       1.00      0.98      0.99     14446\n",
+      "           1       0.26      0.94      0.41       130\n",
       "\n",
-      "    accuracy                           0.95     14576\n",
-      "   macro avg       0.57      0.95      0.61     14576\n",
-      "weighted avg       0.99      0.95      0.97     14576\n",
-      "\n",
-      "Actual predicted values:\n",
-      "(array([0, 1]), array([13736,   840]))\n"
+      "    accuracy                           0.98     14576\n",
+      "   macro avg       0.63      0.96      0.70     14576\n",
+      "weighted avg       0.99      0.98      0.98     14576\n",
+      "\n"
      ]
     }
    ],
    "source": [
-    "start = time.time()\n",
+    "train_dataloader = DataLoaderImbalanced(kwargs={\"oversample_mul\": 5})\n",
+    "\n",
+    "# in reality one should not really do this, but just to illustrate\n",
+    "eval_dataloader = DataLoaderImbalanced(kwargs={\"oversample_mul\": 5})\n",
+    "\n",
     "trainer.fit(\n",
     "    X_train={\"X_tab\": X_tab_train, \"target\": y_train},\n",
     "    X_val={\"X_tab\": X_tab_valid, \"target\": y_valid},\n",
     "    n_epochs=1,\n",
     "    batch_size=32,\n",
-    "    custom_dataloader=DataLoaderImbalanced,\n",
-    "    oversample_mul=5,\n",
+    "    train_dataloader=train_dataloader,\n",
+    "    eval_dataloader=eval_dataloader,\n",
     ")\n",
-    "print(\n",
-    "    \"Training time[s]: {}\".format(\n",
-    "        datetime.timedelta(seconds=round(time.time() - start))\n",
-    "    )\n",
-    ")\n",
-    "\n",
-    "pd.DataFrame(trainer.history)\n",
     "\n",
-    "df_pred = trainer.predict(X_tab=X_tab_test)\n",
-    "print(classification_report(df_test[\"target\"].to_list(), df_pred))\n",
-    "print(\"Actual predicted values:\\n{}\".format(np.unique(df_pred, return_counts=True)))"
+    "preds = trainer.predict(X_tab=X_tab_test)\n",
+    "print(classification_report(df_test[\"target\"].to_list(), preds))"
    ]
   }
  ],
@@ -307,7 +227,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.10.13"
+   "version": "3.10.15"
   }
  },
  "nbformat": 4,
diff --git a/examples/notebooks/09_extracting_embeddings.ipynb b/examples/notebooks/09_extracting_embeddings.ipynb
index e26646da..73dcd457 100644
--- a/examples/notebooks/09_extracting_embeddings.ipynb
+++ b/examples/notebooks/09_extracting_embeddings.ipynb
@@ -13,7 +13,16 @@
    "cell_type": "code",
    "execution_count": 1,
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/Users/javierrodriguezzaurin/.pyenv/versions/3.10.15/envs/widedeep310/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
+      "  from .autonotebook import tqdm as notebook_tqdm\n"
+     ]
+    }
+   ],
    "source": [
     "import numpy as np\n",
     "import pandas as pd\n",
@@ -393,7 +402,7 @@
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "/Users/javierrodriguezzaurin/Projects/pytorch-widedeep/pytorch_widedeep/preprocessing/tab_preprocessor.py:358: UserWarning: Continuous columns will not be normalised\n",
+      "/Users/javierrodriguezzaurin/Projects/pytorch-widedeep/pytorch_widedeep/preprocessing/tab_preprocessor.py:364: UserWarning: Continuous columns will not be normalised\n",
       "  warnings.warn(\"Continuous columns will not be normalised\")\n"
      ]
     }
@@ -411,16 +420,7 @@
    "cell_type": "code",
    "execution_count": 6,
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "/Users/javierrodriguezzaurin/Projects/pytorch-widedeep/pytorch_widedeep/utils/general_utils.py:12: DeprecationWarning: The 'embed_continuous' parameter is deprecated and will be removed in the next release. Please use 'embed_continuous_method' instead See the documentation for more details.\n",
-      "  return func(*args, **kwargs)\n"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
     "ft_transformer = FTTransformer(\n",
     "    column_idx=tab_preprocessor.column_idx,\n",
@@ -442,8 +442,8 @@
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "epoch 1: 100%|█████████████████████████████████████████████████████████████| 153/153 [00:03<00:00, 41.47it/s, loss=221, metrics={'acc': 0.686}]\n",
-      "valid: 100%|█████████████████████████████████████████████████████████████████| 39/39 [00:00<00:00, 86.24it/s, loss=9.28, metrics={'acc': 0.76}]\n"
+      "epoch 1: 100%|██████████| 153/153 [00:02<00:00, 56.56it/s, loss=48.7, metrics={'acc': 0.7371}]\n",
+      "valid: 100%|██████████| 39/39 [00:00<00:00, 129.23it/s, loss=7.44, metrics={'acc': 0.6928}]\n"
      ]
     }
    ],
@@ -459,7 +459,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "t2v = Tab2Vec(model=model, tab_preprocessor=tab_preprocessor)"
+    "t2v = Tab2Vec(model=model, tab_preprocessor=tab_preprocessor, device=\"mps\")"
    ]
   },
   {
@@ -527,7 +527,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.10.13"
+   "version": "3.10.15"
   },
   "vscode": {
    "interpreter": {
diff --git a/examples/notebooks/13_model_uncertainty_prediction.ipynb b/examples/notebooks/13_model_uncertainty_prediction.ipynb
index 0d450d2d..59ccce20 100644
--- a/examples/notebooks/13_model_uncertainty_prediction.ipynb
+++ b/examples/notebooks/13_model_uncertainty_prediction.ipynb
@@ -40,22 +40,25 @@
    "execution_count": 1,
    "id": "9c4e5042",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/Users/javierrodriguezzaurin/.pyenv/versions/3.10.15/envs/widedeep310/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
+      "  from .autonotebook import tqdm as notebook_tqdm\n"
+     ]
+    }
+   ],
    "source": [
-    "import numpy as np\n",
     "import pandas as pd\n",
-    "import torch\n",
-    "from torch.optim import SGD, lr_scheduler\n",
+    "import numpy as np\n",
     "\n",
     "from pytorch_widedeep import Trainer\n",
     "from pytorch_widedeep.preprocessing import TabPreprocessor\n",
     "from pytorch_widedeep.models import TabMlp, WideDeep\n",
-    "from pytorch_widedeep.dataloaders import DataLoaderImbalanced, DataLoaderDefault\n",
-    "from torchmetrics import F1Score as F1_torchmetrics\n",
-    "from torchmetrics import Accuracy as Accuracy_torchmetrics\n",
-    "from torchmetrics import Precision as Precision_torchmetrics\n",
-    "from torchmetrics import Recall as Recall_torchmetrics\n",
-    "from pytorch_widedeep.metrics import Accuracy, Recall, Precision, F1Score, R2Score\n",
+    "from pytorch_widedeep.dataloaders import DataLoaderImbalanced\n",
+    "from pytorch_widedeep.metrics import Accuracy, Recall, Precision, F1Score\n",
     "from pytorch_widedeep.initializers import XavierNormal\n",
     "from pytorch_widedeep.datasets import load_bio_kdd04\n",
     "\n",
@@ -67,11 +70,7 @@
     "\n",
     "import warnings\n",
     "\n",
-    "warnings.filterwarnings(\"ignore\", category=DeprecationWarning)\n",
-    "\n",
-    "# increase displayed columns in jupyter notebook\n",
-    "pd.set_option(\"display.max_columns\", 200)\n",
-    "pd.set_option(\"display.max_rows\", 300)"
+    "warnings.filterwarnings(\"ignore\", category=DeprecationWarning)"
    ]
   },
   {
@@ -111,63 +110,7 @@
        "      <th>8</th>\n",
        "      <th>9</th>\n",
        "      <th>10</th>\n",
-       "      <th>11</th>\n",
-       "      <th>12</th>\n",
-       "      <th>13</th>\n",
-       "      <th>14</th>\n",
-       "      <th>15</th>\n",
-       "      <th>16</th>\n",
-       "      <th>17</th>\n",
-       "      <th>18</th>\n",
-       "      <th>19</th>\n",
-       "      <th>20</th>\n",
-       "      <th>21</th>\n",
-       "      <th>22</th>\n",
-       "      <th>23</th>\n",
-       "      <th>24</th>\n",
-       "      <th>25</th>\n",
-       "      <th>26</th>\n",
-       "      <th>27</th>\n",
-       "      <th>28</th>\n",
-       "      <th>29</th>\n",
-       "      <th>30</th>\n",
-       "      <th>31</th>\n",
-       "      <th>32</th>\n",
-       "      <th>33</th>\n",
-       "      <th>34</th>\n",
-       "      <th>35</th>\n",
-       "      <th>36</th>\n",
-       "      <th>37</th>\n",
-       "      <th>38</th>\n",
-       "      <th>39</th>\n",
-       "      <th>40</th>\n",
-       "      <th>41</th>\n",
-       "      <th>42</th>\n",
-       "      <th>43</th>\n",
-       "      <th>44</th>\n",
-       "      <th>45</th>\n",
-       "      <th>46</th>\n",
-       "      <th>47</th>\n",
-       "      <th>48</th>\n",
-       "      <th>49</th>\n",
-       "      <th>50</th>\n",
-       "      <th>51</th>\n",
-       "      <th>52</th>\n",
-       "      <th>53</th>\n",
-       "      <th>54</th>\n",
-       "      <th>55</th>\n",
-       "      <th>56</th>\n",
-       "      <th>57</th>\n",
-       "      <th>58</th>\n",
-       "      <th>59</th>\n",
-       "      <th>60</th>\n",
-       "      <th>61</th>\n",
-       "      <th>62</th>\n",
-       "      <th>63</th>\n",
-       "      <th>64</th>\n",
-       "      <th>65</th>\n",
-       "      <th>66</th>\n",
-       "      <th>67</th>\n",
+       "      <th>...</th>\n",
        "      <th>68</th>\n",
        "      <th>69</th>\n",
        "      <th>70</th>\n",
@@ -193,63 +136,7 @@
        "      <td>20.0</td>\n",
        "      <td>1256.8</td>\n",
        "      <td>-0.89</td>\n",
-       "      <td>0.33</td>\n",
-       "      <td>11.0</td>\n",
-       "      <td>-55.0</td>\n",
-       "      <td>267.2</td>\n",
-       "      <td>0.52</td>\n",
-       "      <td>0.05</td>\n",
-       "      <td>-2.36</td>\n",
-       "      <td>49.6</td>\n",
-       "      <td>252.0</td>\n",
-       "      <td>0.43</td>\n",
-       "      <td>1.16</td>\n",
-       "      <td>-2.06</td>\n",
-       "      <td>-33.0</td>\n",
-       "      <td>-123.2</td>\n",
-       "      <td>1.60</td>\n",
-       "      <td>-0.49</td>\n",
-       "      <td>-6.06</td>\n",
-       "      <td>65.0</td>\n",
-       "      <td>296.1</td>\n",
-       "      <td>-0.28</td>\n",
-       "      <td>-0.26</td>\n",
-       "      <td>-3.83</td>\n",
-       "      <td>-22.6</td>\n",
-       "      <td>-170.0</td>\n",
-       "      <td>3.06</td>\n",
-       "      <td>-1.05</td>\n",
-       "      <td>-3.29</td>\n",
-       "      <td>22.9</td>\n",
-       "      <td>286.3</td>\n",
-       "      <td>0.12</td>\n",
-       "      <td>2.58</td>\n",
-       "      <td>4.08</td>\n",
-       "      <td>-33.0</td>\n",
-       "      <td>-178.9</td>\n",
-       "      <td>1.88</td>\n",
-       "      <td>0.53</td>\n",
-       "      <td>-7.0</td>\n",
-       "      <td>-44.0</td>\n",
-       "      <td>1987.0</td>\n",
-       "      <td>-5.41</td>\n",
-       "      <td>0.95</td>\n",
-       "      <td>-4.0</td>\n",
-       "      <td>-57.0</td>\n",
-       "      <td>722.9</td>\n",
-       "      <td>-3.26</td>\n",
-       "      <td>-0.55</td>\n",
-       "      <td>-7.5</td>\n",
-       "      <td>125.5</td>\n",
-       "      <td>1547.2</td>\n",
-       "      <td>-0.36</td>\n",
-       "      <td>1.12</td>\n",
-       "      <td>9.0</td>\n",
-       "      <td>-37.0</td>\n",
-       "      <td>72.5</td>\n",
-       "      <td>0.47</td>\n",
-       "      <td>0.74</td>\n",
-       "      <td>-11.0</td>\n",
+       "      <td>...</td>\n",
        "      <td>-8.0</td>\n",
        "      <td>1595.1</td>\n",
        "      <td>-1.64</td>\n",
@@ -273,63 +160,7 @@
        "      <td>9.5</td>\n",
        "      <td>608.1</td>\n",
        "      <td>0.50</td>\n",
-       "      <td>0.07</td>\n",
-       "      <td>20.5</td>\n",
-       "      <td>-52.5</td>\n",
-       "      <td>521.6</td>\n",
-       "      <td>-1.08</td>\n",
-       "      <td>0.58</td>\n",
-       "      <td>-0.02</td>\n",
-       "      <td>-3.2</td>\n",
-       "      <td>103.6</td>\n",
-       "      <td>-0.95</td>\n",
-       "      <td>0.23</td>\n",
-       "      <td>-2.87</td>\n",
-       "      <td>-25.9</td>\n",
-       "      <td>-52.2</td>\n",
-       "      <td>-0.21</td>\n",
-       "      <td>0.87</td>\n",
-       "      <td>-1.81</td>\n",
-       "      <td>10.4</td>\n",
-       "      <td>62.0</td>\n",
-       "      <td>-0.28</td>\n",
-       "      <td>-0.04</td>\n",
-       "      <td>1.48</td>\n",
-       "      <td>-17.6</td>\n",
-       "      <td>-198.3</td>\n",
-       "      <td>3.43</td>\n",
-       "      <td>2.84</td>\n",
-       "      <td>5.87</td>\n",
-       "      <td>-16.9</td>\n",
-       "      <td>72.6</td>\n",
-       "      <td>-0.31</td>\n",
-       "      <td>2.79</td>\n",
-       "      <td>2.71</td>\n",
-       "      <td>-33.5</td>\n",
-       "      <td>-11.6</td>\n",
-       "      <td>-1.11</td>\n",
-       "      <td>4.01</td>\n",
-       "      <td>5.0</td>\n",
-       "      <td>-57.0</td>\n",
-       "      <td>666.3</td>\n",
-       "      <td>1.13</td>\n",
-       "      <td>4.38</td>\n",
-       "      <td>5.0</td>\n",
-       "      <td>-64.0</td>\n",
-       "      <td>39.3</td>\n",
-       "      <td>1.07</td>\n",
-       "      <td>-0.16</td>\n",
-       "      <td>32.5</td>\n",
-       "      <td>100.0</td>\n",
-       "      <td>1893.7</td>\n",
-       "      <td>-2.80</td>\n",
-       "      <td>-0.22</td>\n",
-       "      <td>2.5</td>\n",
-       "      <td>-28.5</td>\n",
-       "      <td>45.0</td>\n",
-       "      <td>0.58</td>\n",
-       "      <td>0.41</td>\n",
-       "      <td>-19.0</td>\n",
+       "      <td>...</td>\n",
        "      <td>-6.0</td>\n",
        "      <td>762.9</td>\n",
        "      <td>0.29</td>\n",
@@ -353,63 +184,7 @@
        "      <td>58.5</td>\n",
        "      <td>1623.6</td>\n",
        "      <td>-1.40</td>\n",
-       "      <td>0.02</td>\n",
-       "      <td>-6.5</td>\n",
-       "      <td>-48.0</td>\n",
-       "      <td>621.0</td>\n",
-       "      <td>-1.20</td>\n",
-       "      <td>0.14</td>\n",
-       "      <td>-0.20</td>\n",
-       "      <td>73.6</td>\n",
-       "      <td>609.1</td>\n",
-       "      <td>-0.44</td>\n",
-       "      <td>-0.58</td>\n",
-       "      <td>-0.04</td>\n",
-       "      <td>-23.0</td>\n",
-       "      <td>-27.4</td>\n",
-       "      <td>-0.72</td>\n",
-       "      <td>-1.04</td>\n",
-       "      <td>-1.09</td>\n",
-       "      <td>91.1</td>\n",
-       "      <td>635.6</td>\n",
-       "      <td>-0.88</td>\n",
-       "      <td>0.24</td>\n",
-       "      <td>0.59</td>\n",
-       "      <td>-18.7</td>\n",
-       "      <td>-7.2</td>\n",
-       "      <td>-0.60</td>\n",
-       "      <td>-2.82</td>\n",
-       "      <td>-0.71</td>\n",
-       "      <td>52.4</td>\n",
-       "      <td>504.1</td>\n",
-       "      <td>0.89</td>\n",
-       "      <td>-0.67</td>\n",
-       "      <td>-9.30</td>\n",
-       "      <td>-20.8</td>\n",
-       "      <td>-25.7</td>\n",
-       "      <td>-0.77</td>\n",
-       "      <td>-0.85</td>\n",
-       "      <td>0.0</td>\n",
-       "      <td>-20.0</td>\n",
-       "      <td>2259.0</td>\n",
-       "      <td>-0.94</td>\n",
-       "      <td>1.15</td>\n",
-       "      <td>-4.0</td>\n",
-       "      <td>-44.0</td>\n",
-       "      <td>-22.7</td>\n",
-       "      <td>0.94</td>\n",
-       "      <td>-0.98</td>\n",
-       "      <td>-19.0</td>\n",
-       "      <td>105.0</td>\n",
-       "      <td>1267.9</td>\n",
-       "      <td>1.03</td>\n",
-       "      <td>1.27</td>\n",
-       "      <td>11.0</td>\n",
-       "      <td>-39.5</td>\n",
-       "      <td>82.3</td>\n",
-       "      <td>0.47</td>\n",
-       "      <td>-0.19</td>\n",
-       "      <td>-10.0</td>\n",
+       "      <td>...</td>\n",
        "      <td>7.0</td>\n",
        "      <td>1491.8</td>\n",
        "      <td>0.32</td>\n",
@@ -433,63 +208,7 @@
        "      <td>46.0</td>\n",
        "      <td>1921.6</td>\n",
        "      <td>-1.36</td>\n",
-       "      <td>-0.47</td>\n",
-       "      <td>-32.0</td>\n",
-       "      <td>-51.5</td>\n",
-       "      <td>560.9</td>\n",
-       "      <td>-0.29</td>\n",
-       "      <td>-0.10</td>\n",
-       "      <td>-1.11</td>\n",
-       "      <td>124.3</td>\n",
-       "      <td>791.6</td>\n",
-       "      <td>0.00</td>\n",
-       "      <td>0.39</td>\n",
-       "      <td>-1.85</td>\n",
-       "      <td>-21.7</td>\n",
-       "      <td>-44.9</td>\n",
-       "      <td>-0.21</td>\n",
-       "      <td>0.02</td>\n",
-       "      <td>0.89</td>\n",
-       "      <td>133.9</td>\n",
-       "      <td>797.8</td>\n",
-       "      <td>-0.08</td>\n",
-       "      <td>1.06</td>\n",
-       "      <td>-0.26</td>\n",
-       "      <td>-16.4</td>\n",
-       "      <td>-74.1</td>\n",
-       "      <td>0.97</td>\n",
-       "      <td>-0.80</td>\n",
-       "      <td>-0.41</td>\n",
-       "      <td>66.9</td>\n",
-       "      <td>955.3</td>\n",
-       "      <td>-1.90</td>\n",
-       "      <td>1.28</td>\n",
-       "      <td>-6.65</td>\n",
-       "      <td>-28.1</td>\n",
-       "      <td>47.5</td>\n",
-       "      <td>-1.91</td>\n",
-       "      <td>1.42</td>\n",
-       "      <td>1.0</td>\n",
-       "      <td>-30.0</td>\n",
-       "      <td>1846.7</td>\n",
-       "      <td>0.76</td>\n",
-       "      <td>1.10</td>\n",
-       "      <td>-4.0</td>\n",
-       "      <td>-52.0</td>\n",
-       "      <td>-53.9</td>\n",
-       "      <td>1.71</td>\n",
-       "      <td>-0.22</td>\n",
-       "      <td>-12.0</td>\n",
-       "      <td>97.5</td>\n",
-       "      <td>1969.8</td>\n",
-       "      <td>-1.70</td>\n",
-       "      <td>0.16</td>\n",
-       "      <td>-1.0</td>\n",
-       "      <td>-32.5</td>\n",
-       "      <td>255.9</td>\n",
-       "      <td>-0.46</td>\n",
-       "      <td>1.57</td>\n",
-       "      <td>10.0</td>\n",
+       "      <td>...</td>\n",
        "      <td>6.0</td>\n",
        "      <td>2047.7</td>\n",
        "      <td>-0.98</td>\n",
@@ -513,63 +232,7 @@
        "      <td>12.0</td>\n",
        "      <td>464.8</td>\n",
        "      <td>0.88</td>\n",
-       "      <td>0.19</td>\n",
-       "      <td>8.0</td>\n",
-       "      <td>-51.5</td>\n",
-       "      <td>98.1</td>\n",
-       "      <td>1.09</td>\n",
-       "      <td>-0.33</td>\n",
-       "      <td>-2.16</td>\n",
-       "      <td>-3.9</td>\n",
-       "      <td>102.7</td>\n",
-       "      <td>0.39</td>\n",
-       "      <td>-1.22</td>\n",
-       "      <td>-3.39</td>\n",
-       "      <td>-15.2</td>\n",
-       "      <td>-42.2</td>\n",
-       "      <td>-1.18</td>\n",
-       "      <td>-1.11</td>\n",
-       "      <td>-3.55</td>\n",
-       "      <td>8.9</td>\n",
-       "      <td>141.3</td>\n",
-       "      <td>-0.16</td>\n",
-       "      <td>-0.43</td>\n",
-       "      <td>-4.15</td>\n",
-       "      <td>-12.9</td>\n",
-       "      <td>-13.4</td>\n",
-       "      <td>-1.32</td>\n",
-       "      <td>-0.98</td>\n",
-       "      <td>-3.69</td>\n",
-       "      <td>8.8</td>\n",
-       "      <td>136.1</td>\n",
-       "      <td>-0.30</td>\n",
-       "      <td>4.13</td>\n",
-       "      <td>1.89</td>\n",
-       "      <td>-13.0</td>\n",
-       "      <td>-18.7</td>\n",
-       "      <td>-1.37</td>\n",
-       "      <td>-0.93</td>\n",
-       "      <td>0.0</td>\n",
-       "      <td>-1.0</td>\n",
-       "      <td>810.1</td>\n",
-       "      <td>-2.29</td>\n",
-       "      <td>6.72</td>\n",
-       "      <td>1.0</td>\n",
-       "      <td>-23.0</td>\n",
-       "      <td>-29.7</td>\n",
-       "      <td>0.58</td>\n",
-       "      <td>-1.10</td>\n",
-       "      <td>-18.5</td>\n",
-       "      <td>33.5</td>\n",
-       "      <td>206.8</td>\n",
-       "      <td>1.84</td>\n",
-       "      <td>-0.13</td>\n",
-       "      <td>4.0</td>\n",
-       "      <td>-29.0</td>\n",
-       "      <td>30.1</td>\n",
-       "      <td>0.80</td>\n",
-       "      <td>-0.24</td>\n",
-       "      <td>5.0</td>\n",
+       "      <td>...</td>\n",
        "      <td>-14.0</td>\n",
        "      <td>479.5</td>\n",
        "      <td>0.68</td>\n",
@@ -583,6 +246,7 @@
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
+       "<p>5 rows × 77 columns</p>\n",
        "</div>"
       ],
       "text/plain": [
@@ -593,47 +257,14 @@
        "3         279    261535       0  41.0  27.91 -0.35   3.0  46.0  1921.6 -1.36   \n",
        "4         279    261536       0  50.0  28.00 -1.32  -9.0  12.0   464.8  0.88   \n",
        "\n",
-       "     11    12    13     14    15    16    17     18     19    20    21    22  \\\n",
-       "0  0.33  11.0 -55.0  267.2  0.52  0.05 -2.36   49.6  252.0  0.43  1.16 -2.06   \n",
-       "1  0.07  20.5 -52.5  521.6 -1.08  0.58 -0.02   -3.2  103.6 -0.95  0.23 -2.87   \n",
-       "2  0.02  -6.5 -48.0  621.0 -1.20  0.14 -0.20   73.6  609.1 -0.44 -0.58 -0.04   \n",
-       "3 -0.47 -32.0 -51.5  560.9 -0.29 -0.10 -1.11  124.3  791.6  0.00  0.39 -1.85   \n",
-       "4  0.19   8.0 -51.5   98.1  1.09 -0.33 -2.16   -3.9  102.7  0.39 -1.22 -3.39   \n",
-       "\n",
-       "     23     24    25    26    27     28     29    30    31    32    33     34  \\\n",
-       "0 -33.0 -123.2  1.60 -0.49 -6.06   65.0  296.1 -0.28 -0.26 -3.83 -22.6 -170.0   \n",
-       "1 -25.9  -52.2 -0.21  0.87 -1.81   10.4   62.0 -0.28 -0.04  1.48 -17.6 -198.3   \n",
-       "2 -23.0  -27.4 -0.72 -1.04 -1.09   91.1  635.6 -0.88  0.24  0.59 -18.7   -7.2   \n",
-       "3 -21.7  -44.9 -0.21  0.02  0.89  133.9  797.8 -0.08  1.06 -0.26 -16.4  -74.1   \n",
-       "4 -15.2  -42.2 -1.18 -1.11 -3.55    8.9  141.3 -0.16 -0.43 -4.15 -12.9  -13.4   \n",
-       "\n",
-       "     35    36    37    38     39    40    41    42    43     44    45    46  \\\n",
-       "0  3.06 -1.05 -3.29  22.9  286.3  0.12  2.58  4.08 -33.0 -178.9  1.88  0.53   \n",
-       "1  3.43  2.84  5.87 -16.9   72.6 -0.31  2.79  2.71 -33.5  -11.6 -1.11  4.01   \n",
-       "2 -0.60 -2.82 -0.71  52.4  504.1  0.89 -0.67 -9.30 -20.8  -25.7 -0.77 -0.85   \n",
-       "3  0.97 -0.80 -0.41  66.9  955.3 -1.90  1.28 -6.65 -28.1   47.5 -1.91  1.42   \n",
-       "4 -1.32 -0.98 -3.69   8.8  136.1 -0.30  4.13  1.89 -13.0  -18.7 -1.37 -0.93   \n",
+       "   ...    68      69    70    71   72    73     74    75    76    77  \n",
+       "0  ...  -8.0  1595.1 -1.64  2.83 -2.0 -50.0  445.2 -0.35  0.26  0.76  \n",
+       "1  ...  -6.0   762.9  0.29  0.82 -3.0 -35.0  140.3  1.16  0.39  0.73  \n",
+       "2  ...   7.0  1491.8  0.32 -1.29  0.0 -34.0  658.2 -0.76  0.26  0.24  \n",
+       "3  ...   6.0  2047.7 -0.98  1.53  0.0 -49.0  554.2 -0.83  0.39  0.73  \n",
+       "4  ... -14.0   479.5  0.68 -0.59  2.0 -36.0   -6.9  2.02  0.14 -0.23  \n",
        "\n",
-       "    47    48      49    50    51   52    53     54    55    56    57     58  \\\n",
-       "0 -7.0 -44.0  1987.0 -5.41  0.95 -4.0 -57.0  722.9 -3.26 -0.55  -7.5  125.5   \n",
-       "1  5.0 -57.0   666.3  1.13  4.38  5.0 -64.0   39.3  1.07 -0.16  32.5  100.0   \n",
-       "2  0.0 -20.0  2259.0 -0.94  1.15 -4.0 -44.0  -22.7  0.94 -0.98 -19.0  105.0   \n",
-       "3  1.0 -30.0  1846.7  0.76  1.10 -4.0 -52.0  -53.9  1.71 -0.22 -12.0   97.5   \n",
-       "4  0.0  -1.0   810.1 -2.29  6.72  1.0 -23.0  -29.7  0.58 -1.10 -18.5   33.5   \n",
-       "\n",
-       "       59    60    61    62    63     64    65    66    67    68      69  \\\n",
-       "0  1547.2 -0.36  1.12   9.0 -37.0   72.5  0.47  0.74 -11.0  -8.0  1595.1   \n",
-       "1  1893.7 -2.80 -0.22   2.5 -28.5   45.0  0.58  0.41 -19.0  -6.0   762.9   \n",
-       "2  1267.9  1.03  1.27  11.0 -39.5   82.3  0.47 -0.19 -10.0   7.0  1491.8   \n",
-       "3  1969.8 -1.70  0.16  -1.0 -32.5  255.9 -0.46  1.57  10.0   6.0  2047.7   \n",
-       "4   206.8  1.84 -0.13   4.0 -29.0   30.1  0.80 -0.24   5.0 -14.0   479.5   \n",
-       "\n",
-       "     70    71   72    73     74    75    76    77  \n",
-       "0 -1.64  2.83 -2.0 -50.0  445.2 -0.35  0.26  0.76  \n",
-       "1  0.29  0.82 -3.0 -35.0  140.3  1.16  0.39  0.73  \n",
-       "2  0.32 -1.29  0.0 -34.0  658.2 -0.76  0.26  0.24  \n",
-       "3 -0.98  1.53  0.0 -49.0  554.2 -0.83  0.39  0.73  \n",
-       "4  0.68 -0.59  2.0 -36.0   -6.9  2.02  0.14 -0.23  "
+       "[5 rows x 77 columns]"
       ]
      },
      "execution_count": 2,
@@ -649,31 +280,6 @@
   {
    "cell_type": "code",
    "execution_count": 3,
-   "id": "d818e9f2",
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "target\n",
-       "0    144455\n",
-       "1      1296\n",
-       "Name: count, dtype: int64"
-      ]
-     },
-     "execution_count": 3,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "# imbalance of the classes\n",
-    "df[\"target\"].value_counts()"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 4,
    "id": "38cdc663",
    "metadata": {},
    "outputs": [],
@@ -684,7 +290,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": 4,
    "id": "039147fc",
    "metadata": {},
    "outputs": [],
@@ -707,7 +313,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 6,
+   "execution_count": 5,
    "id": "95358c9c",
    "metadata": {},
    "outputs": [],
@@ -717,7 +323,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 7,
+   "execution_count": 6,
    "id": "4bf475a3",
    "metadata": {},
    "outputs": [],
@@ -744,21 +350,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 8,
-   "id": "439c350e",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "input_layer = len(tab_preprocessor.continuous_cols)\n",
-    "output_layer = 1\n",
-    "hidden_layers = np.linspace(\n",
-    "    input_layer * 2, output_layer, 5, endpoint=False, dtype=int\n",
-    ").tolist()"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 9,
+   "execution_count": 7,
    "id": "c8334b9b",
    "metadata": {},
    "outputs": [
@@ -772,48 +364,33 @@
        "      (encoder): MLP(\n",
        "        (mlp): Sequential(\n",
        "          (dense_layer_0): Sequential(\n",
-       "            (0): Linear(in_features=74, out_features=148, bias=True)\n",
+       "            (0): Linear(in_features=74, out_features=64, bias=True)\n",
        "            (1): ReLU(inplace=True)\n",
        "            (2): Dropout(p=0.1, inplace=False)\n",
        "          )\n",
        "          (dense_layer_1): Sequential(\n",
-       "            (0): Linear(in_features=148, out_features=118, bias=True)\n",
-       "            (1): ReLU(inplace=True)\n",
-       "            (2): Dropout(p=0.1, inplace=False)\n",
-       "          )\n",
-       "          (dense_layer_2): Sequential(\n",
-       "            (0): Linear(in_features=118, out_features=89, bias=True)\n",
-       "            (1): ReLU(inplace=True)\n",
-       "            (2): Dropout(p=0.1, inplace=False)\n",
-       "          )\n",
-       "          (dense_layer_3): Sequential(\n",
-       "            (0): Linear(in_features=89, out_features=59, bias=True)\n",
-       "            (1): ReLU(inplace=True)\n",
-       "            (2): Dropout(p=0.1, inplace=False)\n",
-       "          )\n",
-       "          (dense_layer_4): Sequential(\n",
-       "            (0): Linear(in_features=59, out_features=30, bias=True)\n",
+       "            (0): Linear(in_features=64, out_features=32, bias=True)\n",
        "            (1): ReLU(inplace=True)\n",
        "            (2): Dropout(p=0.1, inplace=False)\n",
        "          )\n",
        "        )\n",
        "      )\n",
        "    )\n",
-       "    (1): Linear(in_features=30, out_features=1, bias=True)\n",
+       "    (1): Linear(in_features=32, out_features=1, bias=True)\n",
        "  )\n",
        ")"
       ]
      },
-     "execution_count": 9,
+     "execution_count": 7,
      "metadata": {},
      "output_type": "execute_result"
     }
    ],
    "source": [
     "deeptabular = TabMlp(\n",
-    "    mlp_hidden_dims=hidden_layers,\n",
     "    column_idx=tab_preprocessor.column_idx,\n",
     "    continuous_cols=tab_preprocessor.continuous_cols,\n",
+    "    mlp_hidden_dims=[64, 32],\n",
     ")\n",
     "model = WideDeep(deeptabular=deeptabular, pred_dim=1)\n",
     "model"
@@ -821,58 +398,22 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 10,
-   "id": "eabe6cc2",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# # Metrics from torchmetrics\n",
-    "# accuracy = Accuracy_torchmetrics(average=None, num_classes=1)\n",
-    "# precision = Precision_torchmetrics(average=\"micro\", num_classes=1)\n",
-    "# f1 = F1_torchmetrics(average=None, num_classes=1)\n",
-    "# recall = Recall_torchmetrics(average=None, num_classes=1)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 11,
-   "id": "54b0bd74",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# Metrics from pytorch-widedeep\n",
-    "accuracy = Accuracy(top_k=2)\n",
-    "precision = Precision(average=False)\n",
-    "recall = Recall(average=True)\n",
-    "f1 = F1Score(average=False)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 12,
+   "execution_count": 8,
    "id": "64561e0d",
    "metadata": {},
    "outputs": [],
    "source": [
-    "# Optimizers\n",
-    "deep_opt = SGD(model.deeptabular.parameters(), lr=0.1)\n",
-    "# LR Scheduler\n",
-    "deep_sch = lr_scheduler.StepLR(deep_opt, step_size=3)\n",
-    "\n",
     "trainer = Trainer(\n",
     "    model,\n",
     "    objective=\"binary\",\n",
-    "    lr_schedulers={\"deeptabular\": deep_sch},\n",
-    "    initializers={\"deeptabular\": XavierNormal},\n",
-    "    optimizers={\"deeptabular\": deep_opt},\n",
-    "    metrics=[accuracy, precision, recall, f1],\n",
+    "    metrics=[Accuracy(), Precision(), F1Score(), Recall()],\n",
     "    verbose=1,\n",
     ")"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 13,
+   "execution_count": 9,
    "id": "edf1284f",
    "metadata": {},
    "outputs": [
@@ -880,26 +421,12 @@
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "epoch 1: 100%|████████| 208/208 [00:01<00:00, 130.75it/s, loss=0.187, metrics={'acc': 0.9214, 'prec': [0.9149], 'rec': 0.9318, 'f1': [0.9233]}]\n",
-      "valid: 100%|████████████| 292/292 [00:01<00:00, 173.71it/s, loss=0.106, metrics={'acc': 0.9499, 'prec': [0.1435], 'rec': 0.938, 'f1': [0.249]}]\n",
-      "epoch 2: 100%|████████| 208/208 [00:01<00:00, 139.68it/s, loss=0.109, metrics={'acc': 0.9559, 'prec': [0.9537], 'rec': 0.9572, 'f1': [0.9554]}]\n",
-      "valid: 100%|█████████| 292/292 [00:01<00:00, 177.53it/s, loss=0.0888, metrics={'acc': 0.9602, 'prec': [0.1755], 'rec': 0.9457, 'f1': [0.2961]}]\n",
-      "epoch 3: 100%|█████████| 208/208 [00:01<00:00, 141.63it/s, loss=0.08, metrics={'acc': 0.9706, 'prec': [0.9648], 'rec': 0.9766, 'f1': [0.9707]}]\n",
-      "valid: 100%|█████████| 292/292 [00:01<00:00, 165.36it/s, loss=0.0969, metrics={'acc': 0.9564, 'prec': [0.1636], 'rec': 0.9535, 'f1': [0.2792]}]"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Training time[s]: 0:00:10\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "\n"
+      "epoch 1: 100%|██████████| 3644/3644 [00:20<00:00, 175.39it/s, loss=0.0222, metrics={'acc': 0.9945, 'prec': 0.7565, 'f1': 0.6419, 'rec': 0.5574}]\n",
+      "valid: 100%|██████████| 456/456 [00:01<00:00, 252.36it/s, loss=0.0125, metrics={'acc': 0.9969, 'prec': 0.92, 'f1': 0.8035, 'rec': 0.7132}]  \n",
+      "epoch 2: 100%|██████████| 3644/3644 [00:20<00:00, 177.43it/s, loss=0.0119, metrics={'acc': 0.9968, 'prec': 0.9209, 'f1': 0.793, 'rec': 0.6962}] \n",
+      "valid: 100%|██████████| 456/456 [00:01<00:00, 255.61it/s, loss=0.0121, metrics={'acc': 0.997, 'prec': 0.8972, 'f1': 0.8136, 'rec': 0.7442}] \n",
+      "epoch 3: 100%|██████████| 3644/3644 [00:20<00:00, 176.07it/s, loss=0.0103, metrics={'acc': 0.9973, 'prec': 0.9312, 'f1': 0.8351, 'rec': 0.757}] \n",
+      "valid: 100%|██████████| 456/456 [00:01<00:00, 259.70it/s, loss=0.0119, metrics={'acc': 0.997, 'prec': 0.8909, 'f1': 0.8201, 'rec': 0.7597}] \n"
      ]
     }
    ],
@@ -909,20 +436,13 @@
     "    X_train={\"X_tab\": X_tab_train, \"target\": y_train},\n",
     "    X_val={\"X_tab\": X_tab_valid, \"target\": y_valid},\n",
     "    n_epochs=3,\n",
-    "    batch_size=50,\n",
-    "    custom_dataloader=DataLoaderImbalanced,\n",
-    "    oversample_mul=5,\n",
-    ")\n",
-    "print(\n",
-    "    \"Training time[s]: {}\".format(\n",
-    "        datetime.timedelta(seconds=round(time.time() - start))\n",
-    "    )\n",
+    "    batch_size=32,\n",
     ")"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 14,
+   "execution_count": 10,
    "id": "5bbc47ca",
    "metadata": {},
    "outputs": [
@@ -950,77 +470,72 @@
        "      <th>train_loss</th>\n",
        "      <th>train_acc</th>\n",
        "      <th>train_prec</th>\n",
-       "      <th>train_rec</th>\n",
        "      <th>train_f1</th>\n",
+       "      <th>train_rec</th>\n",
        "      <th>val_loss</th>\n",
        "      <th>val_acc</th>\n",
        "      <th>val_prec</th>\n",
-       "      <th>val_rec</th>\n",
        "      <th>val_f1</th>\n",
+       "      <th>val_rec</th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
        "      <th>0</th>\n",
-       "      <td>0.186707</td>\n",
-       "      <td>0.921408</td>\n",
-       "      <td>[0.9149412512779236]</td>\n",
-       "      <td>0.931801</td>\n",
-       "      <td>[0.9232940673828125]</td>\n",
-       "      <td>0.106023</td>\n",
-       "      <td>0.949914</td>\n",
-       "      <td>[0.14353498816490173]</td>\n",
-       "      <td>0.937984</td>\n",
-       "      <td>[0.24897116422653198]</td>\n",
+       "      <td>0.022229</td>\n",
+       "      <td>0.994468</td>\n",
+       "      <td>0.756545</td>\n",
+       "      <td>0.641866</td>\n",
+       "      <td>0.557377</td>\n",
+       "      <td>0.012473</td>\n",
+       "      <td>0.996913</td>\n",
+       "      <td>0.920000</td>\n",
+       "      <td>0.803493</td>\n",
+       "      <td>0.713178</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>1</th>\n",
-       "      <td>0.109498</td>\n",
-       "      <td>0.955931</td>\n",
-       "      <td>[0.9536514282226562]</td>\n",
-       "      <td>0.957193</td>\n",
-       "      <td>[0.9554190039634705]</td>\n",
-       "      <td>0.088787</td>\n",
-       "      <td>0.960206</td>\n",
-       "      <td>[0.17553956806659698]</td>\n",
-       "      <td>0.945736</td>\n",
-       "      <td>[0.29611650109291077]</td>\n",
+       "      <td>0.011912</td>\n",
+       "      <td>0.996767</td>\n",
+       "      <td>0.920918</td>\n",
+       "      <td>0.792971</td>\n",
+       "      <td>0.696239</td>\n",
+       "      <td>0.012088</td>\n",
+       "      <td>0.996981</td>\n",
+       "      <td>0.897196</td>\n",
+       "      <td>0.813559</td>\n",
+       "      <td>0.744186</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>2</th>\n",
-       "      <td>0.079979</td>\n",
-       "      <td>0.970588</td>\n",
-       "      <td>[0.9648183584213257]</td>\n",
-       "      <td>0.976582</td>\n",
-       "      <td>[0.9706646203994751]</td>\n",
-       "      <td>0.096858</td>\n",
-       "      <td>0.956432</td>\n",
-       "      <td>[0.1635638326406479]</td>\n",
-       "      <td>0.953488</td>\n",
-       "      <td>[0.279228150844574]</td>\n",
+       "      <td>0.010341</td>\n",
+       "      <td>0.997341</td>\n",
+       "      <td>0.931198</td>\n",
+       "      <td>0.835106</td>\n",
+       "      <td>0.756991</td>\n",
+       "      <td>0.011884</td>\n",
+       "      <td>0.997050</td>\n",
+       "      <td>0.890909</td>\n",
+       "      <td>0.820084</td>\n",
+       "      <td>0.759690</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "</div>"
       ],
       "text/plain": [
-       "   train_loss  train_acc            train_prec  train_rec  \\\n",
-       "0    0.186707   0.921408  [0.9149412512779236]   0.931801   \n",
-       "1    0.109498   0.955931  [0.9536514282226562]   0.957193   \n",
-       "2    0.079979   0.970588  [0.9648183584213257]   0.976582   \n",
-       "\n",
-       "               train_f1  val_loss   val_acc               val_prec   val_rec  \\\n",
-       "0  [0.9232940673828125]  0.106023  0.949914  [0.14353498816490173]  0.937984   \n",
-       "1  [0.9554190039634705]  0.088787  0.960206  [0.17553956806659698]  0.945736   \n",
-       "2  [0.9706646203994751]  0.096858  0.956432   [0.1635638326406479]  0.953488   \n",
+       "   train_loss  train_acc  train_prec  train_f1  train_rec  val_loss   val_acc  \\\n",
+       "0    0.022229   0.994468    0.756545  0.641866   0.557377  0.012473  0.996913   \n",
+       "1    0.011912   0.996767    0.920918  0.792971   0.696239  0.012088  0.996981   \n",
+       "2    0.010341   0.997341    0.931198  0.835106   0.756991  0.011884  0.997050   \n",
        "\n",
-       "                  val_f1  \n",
-       "0  [0.24897116422653198]  \n",
-       "1  [0.29611650109291077]  \n",
-       "2    [0.279228150844574]  "
+       "   val_prec    val_f1   val_rec  \n",
+       "0  0.920000  0.803493  0.713178  \n",
+       "1  0.897196  0.813559  0.744186  \n",
+       "2  0.890909  0.820084  0.759690  "
       ]
      },
-     "execution_count": 14,
+     "execution_count": 10,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -1039,7 +554,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 15,
+   "execution_count": 11,
    "id": "2fc67e9b",
    "metadata": {
     "scrolled": true
@@ -1049,7 +564,7 @@
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "predict: 100%|██████████████████████████████████████████████████████████████████████████████████████████████| 292/292 [00:00<00:00, 346.55it/s]\n"
+      "predict: 100%|██████████| 456/456 [00:00<00:00, 689.36it/s]\n"
      ]
     },
     {
@@ -1058,15 +573,15 @@
      "text": [
       "              precision    recall  f1-score   support\n",
       "\n",
-      "           0       1.00      0.96      0.98     14446\n",
-      "           1       0.17      0.95      0.29       130\n",
+      "           0       1.00      1.00      1.00     14446\n",
+      "           1       0.91      0.78      0.84       130\n",
       "\n",
-      "    accuracy                           0.96     14576\n",
-      "   macro avg       0.58      0.95      0.63     14576\n",
-      "weighted avg       0.99      0.96      0.97     14576\n",
+      "    accuracy                           1.00     14576\n",
+      "   macro avg       0.95      0.89      0.92     14576\n",
+      "weighted avg       1.00      1.00      1.00     14576\n",
       "\n",
       "Actual predicted values:\n",
-      "(array([0, 1]), array([13845,   731]))\n"
+      "(array([0, 1]), array([14465,   111]))\n"
      ]
     }
    ],
@@ -1086,7 +601,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 16,
+   "execution_count": 12,
    "id": "c7317302",
    "metadata": {},
    "outputs": [
@@ -1094,7 +609,7 @@
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "predict_UncertaintyIter: 100%|█████████████████████████████████████████████████████████████████████████████████| 10/10 [00:03<00:00,  3.25it/s]"
+      "predict_UncertaintyIter: 100%|██████████| 10/10 [00:05<00:00,  1.86it/s]"
      ]
     },
     {
@@ -1103,12 +618,12 @@
      "text": [
       "              precision    recall  f1-score   support\n",
       "\n",
-      "           0       1.00      0.96      0.98     14446\n",
-      "           1       0.17      0.95      0.29       130\n",
+      "           0       1.00      1.00      1.00     14446\n",
+      "           1       0.91      0.78      0.84       130\n",
       "\n",
-      "    accuracy                           0.96     14576\n",
-      "   macro avg       0.58      0.95      0.63     14576\n",
-      "weighted avg       0.99      0.96      0.97     14576\n",
+      "    accuracy                           1.00     14576\n",
+      "   macro avg       0.95      0.89      0.92     14576\n",
+      "weighted avg       1.00      1.00      1.00     14576\n",
       "\n",
       "Actual predicted values:\n",
       "(array([0.]), array([14576]))\n"
@@ -1134,23 +649,23 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 17,
+   "execution_count": 13,
    "id": "f63ca87a",
    "metadata": {},
    "outputs": [
     {
      "data": {
       "text/plain": [
-       "array([[9.98401165e-01, 1.59881881e-03, 0.00000000e+00],\n",
-       "       [9.99941409e-01, 5.85634953e-05, 0.00000000e+00],\n",
-       "       [9.97351170e-01, 2.64881272e-03, 0.00000000e+00],\n",
+       "array([[9.99999821e-01, 1.77245539e-07, 0.00000000e+00],\n",
+       "       [1.00000000e+00, 8.29310925e-11, 0.00000000e+00],\n",
+       "       [9.99995947e-01, 4.06420531e-06, 0.00000000e+00],\n",
        "       ...,\n",
-       "       [9.99494374e-01, 5.05603210e-04, 0.00000000e+00],\n",
-       "       [9.99981642e-01, 1.83574630e-05, 0.00000000e+00],\n",
-       "       [9.99996483e-01, 3.52600046e-06, 0.00000000e+00]])"
+       "       [9.99999940e-01, 3.85314713e-08, 0.00000000e+00],\n",
+       "       [1.00000000e+00, 2.98146707e-09, 0.00000000e+00],\n",
+       "       [1.00000000e+00, 1.21332046e-12, 0.00000000e+00]])"
       ]
      },
-     "execution_count": 17,
+     "execution_count": 13,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -1179,7 +694,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.10.13"
+   "version": "3.10.15"
   },
   "toc": {
    "base_numbering": 1,
diff --git a/examples/notebooks/15_Self_Supervised_Pretraning_pt2.ipynb b/examples/notebooks/15_Self_Supervised_Pretraning_pt2.ipynb
index 78d8f884..b39f1f78 100644
--- a/examples/notebooks/15_Self_Supervised_Pretraning_pt2.ipynb
+++ b/examples/notebooks/15_Self_Supervised_Pretraning_pt2.ipynb
@@ -58,7 +58,16 @@
    "execution_count": 1,
    "id": "e5b94ab2",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/Users/javierrodriguezzaurin/.pyenv/versions/3.10.15/envs/widedeep310/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
+      "  from .autonotebook import tqdm as notebook_tqdm\n"
+     ]
+    }
+   ],
    "source": [
     "import torch\n",
     "from sklearn.metrics import accuracy_score\n",
@@ -84,7 +93,7 @@
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "/Users/javierrodriguezzaurin/Projects/pytorch-widedeep/pytorch_widedeep/preprocessing/tab_preprocessor.py:358: UserWarning: Continuous columns will not be normalised\n",
+      "/Users/javierrodriguezzaurin/Projects/pytorch-widedeep/pytorch_widedeep/preprocessing/tab_preprocessor.py:364: UserWarning: Continuous columns will not be normalised\n",
       "  warnings.warn(\"Continuous columns will not be normalised\")\n"
      ]
     }
@@ -154,11 +163,11 @@
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "epoch 1: 100%|█████████████████████████████████████████████████████████████████████████████████████| 153/153 [00:13<00:00, 11.73it/s, loss=579]\n",
-      "epoch 2: 100%|█████████████████████████████████████████████████████████████████████████████████████| 153/153 [00:12<00:00, 12.56it/s, loss=143]\n",
-      "epoch 3: 100%|█████████████████████████████████████████████████████████████████████████████████████| 153/153 [00:12<00:00, 12.49it/s, loss=141]\n",
-      "epoch 4: 100%|█████████████████████████████████████████████████████████████████████████████████████| 153/153 [00:11<00:00, 12.77it/s, loss=138]\n",
-      "epoch 5: 100%|█████████████████████████████████████████████████████████████████████████████████████| 153/153 [00:11<00:00, 13.29it/s, loss=137]\n"
+      "epoch 1: 100%|██████████| 153/153 [00:22<00:00,  6.91it/s, loss=656]   \n",
+      "epoch 2: 100%|██████████| 153/153 [00:07<00:00, 20.84it/s, loss=141]\n",
+      "epoch 3: 100%|██████████| 153/153 [00:07<00:00, 20.75it/s, loss=137]\n",
+      "epoch 4: 100%|██████████| 153/153 [00:07<00:00, 21.14it/s, loss=135]\n",
+      "epoch 5: 100%|██████████| 153/153 [00:07<00:00, 21.05it/s, loss=134]\n"
      ]
     }
    ],
@@ -197,7 +206,16 @@
    "execution_count": 6,
    "id": "b60be9fd",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/var/folders/pd/_2wz_qt16yq1fk6jn_xtxqk40000gn/T/ipykernel_71058/884975850.py:2: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.\n",
+      "  contrastive_denoising_model = torch.load(\n"
+     ]
+    }
+   ],
    "source": [
     "# some time has passed, we load the model with torch as usual:\n",
     "contrastive_denoising_model = torch.load(\n",
@@ -224,7 +242,7 @@
       "text/plain": [
        "FTTransformer(\n",
        "  (cat_embed): SameSizeCatEmbeddings(\n",
-       "    (embed): Embedding(323, 32, padding_idx=0)\n",
+       "    (embed): Embedding(322, 32, padding_idx=0)\n",
        "    (dropout): Dropout(p=0.0, inplace=False)\n",
        "  )\n",
        "  (cont_norm): Identity()\n",
@@ -330,19 +348,19 @@
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "epoch 1: 100%|██████████████████████████████████████████████████████████| 153/153 [00:05<00:00, 27.19it/s, loss=0.383, metrics={'acc': 0.8176}]\n",
-      "epoch 2: 100%|██████████████████████████████████████████████████████████| 153/153 [00:05<00:00, 26.08it/s, loss=0.325, metrics={'acc': 0.8502}]\n",
-      "epoch 3: 100%|██████████████████████████████████████████████████████████| 153/153 [00:05<00:00, 26.56it/s, loss=0.306, metrics={'acc': 0.8601}]\n",
-      "epoch 4: 100%|██████████████████████████████████████████████████████████| 153/153 [00:05<00:00, 27.41it/s, loss=0.295, metrics={'acc': 0.8641}]\n",
-      "epoch 5: 100%|██████████████████████████████████████████████████████████| 153/153 [00:06<00:00, 24.70it/s, loss=0.289, metrics={'acc': 0.8656}]\n",
-      "predict: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████| 39/39 [00:00<00:00, 97.26it/s]"
+      "epoch 1: 100%|██████████| 153/153 [00:03<00:00, 50.99it/s, loss=0.383, metrics={'acc': 0.8218}]\n",
+      "epoch 2: 100%|██████████| 153/153 [00:02<00:00, 53.22it/s, loss=0.328, metrics={'acc': 0.8505}]\n",
+      "epoch 3: 100%|██████████| 153/153 [00:02<00:00, 54.50it/s, loss=0.309, metrics={'acc': 0.8588}]\n",
+      "epoch 4: 100%|██████████| 153/153 [00:02<00:00, 53.90it/s, loss=0.299, metrics={'acc': 0.8641}]\n",
+      "epoch 5: 100%|██████████| 153/153 [00:02<00:00, 54.02it/s, loss=0.292, metrics={'acc': 0.8665}]\n",
+      "predict: 100%|██████████| 39/39 [00:00<00:00, 88.79it/s] "
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "0.8695874705701709\n"
+      "0.8679496366055891\n"
      ]
     },
     {
@@ -368,14 +386,6 @@
     "test_acc = accuracy_score(target_te, preds)\n",
     "print(test_acc)"
    ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "2f61e849-6c90-4b51-9e91-67007025222d",
-   "metadata": {},
-   "outputs": [],
-   "source": []
   }
  ],
  "metadata": {
@@ -394,7 +404,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.10.13"
+   "version": "3.10.15"
   }
  },
  "nbformat": 4,
diff --git a/examples/scripts/adult_census.py b/examples/scripts/adult_census.py
index 17173c41..f71bf6ab 100644
--- a/examples/scripts/adult_census.py
+++ b/examples/scripts/adult_census.py
@@ -10,8 +10,6 @@
 from pytorch_widedeep.initializers import XavierNormal, KaimingNormal
 from pytorch_widedeep.preprocessing import TabPreprocessor, WidePreprocessor
 
-use_cuda = torch.cuda.is_available()
-
 if __name__ == "__main__":
     df: pd.DataFrame = load_adult(as_frame=True)
     df.columns = [c.replace("-", "_") for c in df.columns]
@@ -90,6 +88,8 @@
     for tab_model in [tab_mlp, tab_resnet]:
         model = WideDeep(wide=wide, deeptabular=tab_model)  # type: ignore[arg-type]
 
+        # Over engineered example simply to show how to use the Trainer class
+        # and its parameters
         wide_opt = torch.optim.Adam(model.wide.parameters(), lr=0.01)
         deep_opt = torch.optim.AdamW(model.deeptabular.parameters(), lr=0.01)
         wide_sch = torch.optim.lr_scheduler.StepLR(wide_opt, step_size=2)
diff --git a/examples/scripts/adult_census_attention_mlp.py b/examples/scripts/adult_census_attention_mlp.py
index 85562f63..1b3df068 100644
--- a/examples/scripts/adult_census_attention_mlp.py
+++ b/examples/scripts/adult_census_attention_mlp.py
@@ -1,5 +1,4 @@
 import numpy as np
-import torch
 import pandas as pd
 
 from pytorch_widedeep import Trainer
@@ -8,8 +7,6 @@
 from pytorch_widedeep.datasets import load_adult
 from pytorch_widedeep.preprocessing import TabPreprocessor
 
-use_cuda = torch.cuda.is_available()
-
 if __name__ == "__main__":
     df: pd.DataFrame = load_adult(as_frame=True)
     df.columns = [c.replace("-", "_") for c in df.columns]
diff --git a/examples/scripts/adult_census_bayesian_tabmlp.py b/examples/scripts/adult_census_bayesian_tabmlp.py
index 682f2808..d04c0f52 100644
--- a/examples/scripts/adult_census_bayesian_tabmlp.py
+++ b/examples/scripts/adult_census_bayesian_tabmlp.py
@@ -9,8 +9,6 @@
 from pytorch_widedeep.preprocessing import TabPreprocessor, WidePreprocessor
 from pytorch_widedeep.bayesian_models import BayesianWide, BayesianTabMlp
 
-use_cuda = torch.cuda.is_available()
-
 if __name__ == "__main__":
     df: pd.DataFrame = load_adult(as_frame=True)
     df.columns = [c.replace("-", "_") for c in df.columns]
diff --git a/examples/scripts/adult_census_cont_den_full_example.py b/examples/scripts/adult_census_cont_den_full_example.py
index a661d011..a8f9a053 100644
--- a/examples/scripts/adult_census_cont_den_full_example.py
+++ b/examples/scripts/adult_census_cont_den_full_example.py
@@ -10,8 +10,6 @@
 from pytorch_widedeep.preprocessing import TabPreprocessor
 from pytorch_widedeep.self_supervised_training import ContrastiveDenoisingTrainer
 
-use_cuda = torch.cuda.is_available()
-
 if __name__ == "__main__":
     df: pd.DataFrame = load_adult(as_frame=True)
     df.columns = [c.replace("-", "_") for c in df.columns]
diff --git a/examples/scripts/adult_census_cont_den_run_all_models.py b/examples/scripts/adult_census_cont_den_run_all_models.py
index b69174bb..adc21c7d 100644
--- a/examples/scripts/adult_census_cont_den_run_all_models.py
+++ b/examples/scripts/adult_census_cont_den_run_all_models.py
@@ -16,8 +16,6 @@
 from pytorch_widedeep.preprocessing import TabPreprocessor
 from pytorch_widedeep.self_supervised_training import ContrastiveDenoisingTrainer
 
-use_cuda = torch.cuda.is_available()
-
 if __name__ == "__main__":
     df: pd.DataFrame = load_adult(as_frame=True)
     df.columns = [c.replace("-", "_") for c in df.columns]
@@ -71,7 +69,7 @@
                 n_blocks=4,
             )
         if transf_model == "saint":
-            model = SAINT(
+            model = SAINT(  # type: ignore[assignment]
                 column_idx=processor.column_idx,
                 cat_embed_input=processor.cat_embed_input,
                 continuous_cols=continuous_cols,
@@ -79,7 +77,7 @@
                 n_blocks=4,
             )
         if transf_model == "tab_fastformer":
-            model = TabFastFormer(
+            model = TabFastFormer(  # type: ignore[assignment]
                 column_idx=processor.column_idx,
                 cat_embed_input=processor.cat_embed_input,
                 continuous_cols=continuous_cols,
@@ -89,7 +87,7 @@
                 share_weights=False,
             )
         if transf_model == "ft_transformer":
-            model = FTTransformer(
+            model = FTTransformer(  # type: ignore[assignment]
                 column_idx=processor.column_idx,
                 cat_embed_input=processor.cat_embed_input,
                 continuous_cols=continuous_cols,
@@ -117,7 +115,7 @@
         X_tab = processor.fit_transform(df)
 
         if attn_model == "context_attention":
-            model = ContextAttentionMLP(
+            model = ContextAttentionMLP(  # type: ignore[assignment]
                 column_idx=processor.column_idx,
                 cat_embed_input=processor.cat_embed_input,
                 continuous_cols=continuous_cols,
@@ -126,7 +124,7 @@
                 n_blocks=3,
             )
         if attn_model == "self_attention":
-            model = SelfAttentionMLP(
+            model = SelfAttentionMLP(  # type: ignore[assignment]
                 column_idx=processor.column_idx,
                 cat_embed_input=processor.cat_embed_input,
                 continuous_cols=continuous_cols,
diff --git a/examples/scripts/adult_census_enc_dec_full_example.py b/examples/scripts/adult_census_enc_dec_full_example.py
index 972e833d..fc0de9fc 100644
--- a/examples/scripts/adult_census_enc_dec_full_example.py
+++ b/examples/scripts/adult_census_enc_dec_full_example.py
@@ -10,8 +10,6 @@
 from pytorch_widedeep.preprocessing import TabPreprocessor
 from pytorch_widedeep.self_supervised_training import EncoderDecoderTrainer
 
-use_cuda = torch.cuda.is_available()
-
 if __name__ == "__main__":
     df: pd.DataFrame = load_adult(as_frame=True)
     df.columns = [c.replace("-", "_") for c in df.columns]
diff --git a/examples/scripts/adult_census_enc_dec_run_all_models.py b/examples/scripts/adult_census_enc_dec_run_all_models.py
index 2c77cffc..4182d051 100644
--- a/examples/scripts/adult_census_enc_dec_run_all_models.py
+++ b/examples/scripts/adult_census_enc_dec_run_all_models.py
@@ -10,8 +10,6 @@
 from pytorch_widedeep.preprocessing import TabPreprocessor
 from pytorch_widedeep.self_supervised_training import EncoderDecoderTrainer
 
-use_cuda = torch.cuda.is_available()
-
 if __name__ == "__main__":
     df: pd.DataFrame = load_adult(as_frame=True)
     df.columns = [c.replace("-", "_") for c in df.columns]
diff --git a/examples/scripts/adult_census_feature_importance.py b/examples/scripts/adult_census_feature_importance.py
index 2973c6f4..83be1506 100644
--- a/examples/scripts/adult_census_feature_importance.py
+++ b/examples/scripts/adult_census_feature_importance.py
@@ -16,8 +16,6 @@
 from pytorch_widedeep.datasets import load_adult
 from pytorch_widedeep.preprocessing import TabPreprocessor
 
-# use_cuda = torch.cuda.is_available()
-
 df: pd.DataFrame = load_adult(as_frame=True)
 df.columns = [c.replace("-", "_") for c in df.columns]
 df["income_label"] = (df["income"].apply(lambda x: ">50K" in x)).astype(int)
diff --git a/examples/scripts/adult_census_linear_and_flash_attention.py b/examples/scripts/adult_census_linear_and_flash_attention.py
index c46be8e1..2c2e05bc 100644
--- a/examples/scripts/adult_census_linear_and_flash_attention.py
+++ b/examples/scripts/adult_census_linear_and_flash_attention.py
@@ -9,8 +9,6 @@
 from pytorch_widedeep.datasets import load_adult
 from pytorch_widedeep.preprocessing import TabPreprocessor
 
-# use_cuda = torch.cuda.is_available()
-
 df: pd.DataFrame = load_adult(as_frame=True)
 df.columns = [c.replace("-", "_") for c in df.columns]
 df["income_label"] = (df["income"].apply(lambda x: ">50K" in x)).astype(int)
diff --git a/examples/scripts/adult_census_tabnet.py b/examples/scripts/adult_census_tabnet.py
index e3f2cfe7..15162a67 100644
--- a/examples/scripts/adult_census_tabnet.py
+++ b/examples/scripts/adult_census_tabnet.py
@@ -9,8 +9,6 @@
 from pytorch_widedeep.callbacks import LRHistory, EarlyStopping, ModelCheckpoint
 from pytorch_widedeep.preprocessing import TabPreprocessor
 
-use_cuda = torch.cuda.is_available()
-
 if __name__ == "__main__":
     df: pd.DataFrame = load_adult(as_frame=True)
     df.columns = [c.replace("-", "_") for c in df.columns]
@@ -19,7 +17,7 @@
     )
     df["income_label"] = (df["income"].apply(lambda x: ">50K" in x)).astype(int)
     df.drop("income", axis=1, inplace=True)
-    df.head()
+    # df.head()
 
     cat_embed_cols = [
         "age_buckets",
diff --git a/examples/scripts/adult_census_transformers.py b/examples/scripts/adult_census_transformers.py
index 1718f0d0..67a00338 100644
--- a/examples/scripts/adult_census_transformers.py
+++ b/examples/scripts/adult_census_transformers.py
@@ -18,8 +18,6 @@
 from pytorch_widedeep.initializers import XavierNormal, KaimingNormal
 from pytorch_widedeep.preprocessing import TabPreprocessor, WidePreprocessor
 
-use_cuda = torch.cuda.is_available()
-
 if __name__ == "__main__":
     df: pd.DataFrame = load_adult(as_frame=True)
     df.columns = [c.replace("-", "_") for c in df.columns]
diff --git a/examples/scripts/airbnb_all_modes_multiclass.py b/examples/scripts/airbnb_all_modes_multiclass.py
index 54d3eb0f..f2eefddc 100644
--- a/examples/scripts/airbnb_all_modes_multiclass.py
+++ b/examples/scripts/airbnb_all_modes_multiclass.py
@@ -9,8 +9,6 @@
 from pytorch_widedeep.metrics import F1Score, Precision
 from pytorch_widedeep.preprocessing import TabPreprocessor, WidePreprocessor
 
-use_cuda = torch.cuda.is_available()
-
 if __name__ == "__main__":
     # The airbnb dataset, which you could get from here:
     # http://insideairbnb.com/get-the-data.html, is too big to be included in
diff --git a/examples/scripts/airbnb_all_modes_regr.py b/examples/scripts/airbnb_all_modes_regr.py
index 2e23af6c..8d9df1fc 100644
--- a/examples/scripts/airbnb_all_modes_regr.py
+++ b/examples/scripts/airbnb_all_modes_regr.py
@@ -25,8 +25,6 @@
     ImagePreprocessor,
 )
 
-use_cuda = torch.cuda.is_available()
-
 if __name__ == "__main__":
     # The airbnb dataset, which you could get from here:
     # http://insideairbnb.com/get-the-data.html, is too big to be included in
diff --git a/examples/scripts/airbnb_load_from_folder_regr.py b/examples/scripts/airbnb_load_from_folder_regr.py
index 040433fc..df4d4161 100644
--- a/examples/scripts/airbnb_load_from_folder_regr.py
+++ b/examples/scripts/airbnb_load_from_folder_regr.py
@@ -1,5 +1,4 @@
 import numpy as np
-import torch
 import pandas as pd
 from torch.utils.data import DataLoader
 
@@ -19,8 +18,6 @@
     WideDeepDatasetFromFolder,
 )
 
-use_cuda = torch.cuda.is_available()
-
 if __name__ == "__main__":
     # The airbnb dataset, which you could get from here:
     # http://insideairbnb.com/get-the-data.html, is too big to be included in
diff --git a/examples/scripts/bio_imbalanced_loader.py b/examples/scripts/bio_imbalanced_loader.py
index abe834ef..631afb56 100644
--- a/examples/scripts/bio_imbalanced_loader.py
+++ b/examples/scripts/bio_imbalanced_loader.py
@@ -1,19 +1,13 @@
-import time
-import datetime
 import warnings
 
-import numpy as np
 import pandas as pd
-from torch.optim import SGD, lr_scheduler
-from sklearn.metrics import classification_report
 from sklearn.model_selection import train_test_split
 
 from pytorch_widedeep import Trainer
 from pytorch_widedeep.models import TabMlp, WideDeep
-from pytorch_widedeep.metrics import Recall, F1Score, Accuracy, Precision
+from pytorch_widedeep.metrics import Accuracy, Precision
 from pytorch_widedeep.datasets import load_bio_kdd04
 from pytorch_widedeep.dataloaders import DataLoaderImbalanced
-from pytorch_widedeep.initializers import XavierNormal
 from pytorch_widedeep.preprocessing import TabPreprocessor
 
 warnings.filterwarnings("ignore", category=DeprecationWarning)
@@ -43,59 +37,32 @@
     y_valid = df_valid["target"].values
     y_test = df_test["target"].values
 
-    # Define the model
-    input_layer = len(tab_preprocessor.continuous_cols)
-    output_layer = 1
-    hidden_layers = np.linspace(
-        input_layer * 2, output_layer, 5, endpoint=False, dtype=int
-    ).tolist()
-
     deeptabular = TabMlp(
         column_idx=tab_preprocessor.column_idx,
         continuous_cols=tab_preprocessor.continuous_cols,
-        mlp_hidden_dims=hidden_layers,
+        mlp_hidden_dims=[64, 32],
     )
     model = WideDeep(deeptabular=deeptabular)
 
-    # Metrics from pytorch-widedeep
-    accuracy = Accuracy(top_k=2)
-    precision = Precision(average=False)
-    recall = Recall(average=True)
-    f1 = F1Score(average=False)
-
-    # Optimizers
-    deep_opt = SGD(model.deeptabular.parameters(), lr=0.1)
-
-    # LR Scheduler
-    deep_sch = lr_scheduler.StepLR(deep_opt, step_size=3)
-
     trainer = Trainer(
         model,
         objective="binary",
-        lr_schedulers={"deeptabular": deep_sch},
-        initializers={"deeptabular": XavierNormal},
-        optimizers={"deeptabular": deep_opt},
-        metrics=[accuracy, precision],
+        metrics=[Accuracy, Precision],
         verbose=1,
     )
 
-    start = time.time()
+    train_dataloader = DataLoaderImbalanced(kwargs={"oversample_mul": 5})
+
+    # in reality one should not really do this, but just to illustrate
+    eval_dataloader = DataLoaderImbalanced(kwargs={"oversample_mul": 5})
+
     trainer.fit(
         X_train={"X_tab": X_tab_train, "target": y_train},
         X_val={"X_tab": X_tab_valid, "target": y_valid},
         n_epochs=1,
         batch_size=32,
-        custom_dataloader=DataLoaderImbalanced,
-        **{"oversample_mul": 5},
+        train_dataloader=train_dataloader,
+        eval_dataloader=eval_dataloader,
     )
-    print(
-        "Training time[s]: {}".format(
-            datetime.timedelta(seconds=round(time.time() - start))
-        )
-    )
-
-    pd.DataFrame(trainer.history)
 
-    df_pred = trainer.predict(X_tab=X_tab_test)
-    print(classification_report(df_test["target"].to_list(), df_pred))
-    print("Actual predicted values:\n{}".format(np.unique(df_pred, return_counts=True)))
+    preds = trainer.predict(X_tab=X_tab_test)
diff --git a/examples/scripts/ecomerce_reviews_bert.py b/examples/scripts/ecomerce_reviews_bert.py
index e6304515..67cfe3c4 100644
--- a/examples/scripts/ecomerce_reviews_bert.py
+++ b/examples/scripts/ecomerce_reviews_bert.py
@@ -1,3 +1,5 @@
+# HT models are available in the library, but in this script we show how
+# to use a custom model with the WideDeep class.
 import numpy as np
 import torch
 import pandas as pd
diff --git a/examples/scripts/tmp.py b/examples/scripts/tmp.py
new file mode 100644
index 00000000..970b4c38
--- /dev/null
+++ b/examples/scripts/tmp.py
@@ -0,0 +1,64 @@
+import numpy as np
+import pandas as pd
+
+from pytorch_widedeep import Trainer
+from pytorch_widedeep.models import TabMlp, WideDeep
+from pytorch_widedeep.metrics import Accuracy, Precision
+from pytorch_widedeep.datasets import load_adult
+from pytorch_widedeep.preprocessing import TabPreprocessor
+
+if __name__ == "__main__":
+    df: pd.DataFrame = load_adult(as_frame=True)
+    df.columns = [c.replace("-", "_") for c in df.columns]
+    df["age_buckets"] = pd.cut(
+        df.age, bins=[16, 25, 30, 35, 40, 45, 50, 55, 60, 91], labels=np.arange(9)
+    )
+    df["income_label"] = (df["income"].apply(lambda x: ">50K" in x)).astype(int)
+    df.drop("income", axis=1, inplace=True)
+
+    cat_embed_cols = [
+        "workclass",
+        "education",
+        "marital_status",
+        "occupation",
+        "relationship",
+        "race",
+        "gender",
+        "capital_gain",
+        "capital_loss",
+        "native_country",
+    ]
+    continuous_cols = ["age", "hours_per_week"]
+    target = "income_label"
+    target = df[target].values
+
+    tab_preprocessor = TabPreprocessor(
+        cat_embed_cols=cat_embed_cols,
+        continuous_cols=continuous_cols,
+        scale=True,  # type: ignore[arg-type]
+    )
+    X_tab = tab_preprocessor.fit_transform(df)
+
+    tab_mlp = TabMlp(
+        column_idx=tab_preprocessor.column_idx,
+        cat_embed_input=tab_preprocessor.cat_embed_input,
+        continuous_cols=continuous_cols,
+        mlp_hidden_dims=[200, 100],
+        mlp_dropout=[0.2, 0.2],
+    )
+
+    model = WideDeep(deeptabular=tab_mlp)
+
+    trainer = Trainer(
+        model,
+        objective="binary",
+        metrics=[Accuracy, Precision],
+    )
+
+    trainer.fit(
+        X_tab=X_tab,
+        target=target,
+        n_epochs=2,
+        batch_size=256,
+        val_split=0.2,
+    )
diff --git a/mkdocs/site/404.html b/mkdocs/site/404.html
index b12bf418..ba48759e 100644
--- a/mkdocs/site/404.html
+++ b/mkdocs/site/404.html
@@ -12,7 +12,7 @@
       
       
       <link rel="icon" href="/assets/images/favicon.ico">
-      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.35">
+      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.43">
     
     
       
@@ -20,7 +20,7 @@
       
     
     
-      <link rel="stylesheet" href="/assets/stylesheets/main.35f28582.min.css">
+      <link rel="stylesheet" href="/assets/stylesheets/main.0253249f.min.css">
       
         
         <link rel="stylesheet" href="/assets/stylesheets/palette.06af60db.min.css">
@@ -1577,7 +1577,7 @@ <h1>404 - Not found</h1>
     <script id="__config" type="application/json">{"base": "/", "features": ["navigation.tabs", "navigation.tabs.sticky", "navigation.indexes", "navigation.expand", "toc.integrate"], "search": "/assets/javascripts/workers/search.6ce7567c.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version": "Select version"}}</script>
     
     
-      <script src="/assets/javascripts/bundle.56dfad97.min.js"></script>
+      <script src="/assets/javascripts/bundle.83f73b43.min.js"></script>
       
         <script src="/stylesheets/extra.js"></script>
       
diff --git a/mkdocs/site/assets/_mkdocstrings.css b/mkdocs/site/assets/_mkdocstrings.css
index 85449ec7..b500381b 100644
--- a/mkdocs/site/assets/_mkdocstrings.css
+++ b/mkdocs/site/assets/_mkdocstrings.css
@@ -26,20 +26,33 @@
   float: right;
 }
 
+/* Parameter headings must be inline, not blocks. */
+.doc-heading-parameter {
+  display: inline;
+}
+
+/* Prefer space on the right, not the left of parameter permalinks. */
+.doc-heading-parameter .headerlink {
+  margin-left: 0 !important;
+  margin-right: 0.2rem;
+}
+
 /* Backward-compatibility: docstring section titles in bold. */
 .doc-section-title {
   font-weight: bold;
 }
 
 /* Symbols in Navigation and ToC. */
-:root,
+:root, :host,
 [data-md-color-scheme="default"] {
+  --doc-symbol-parameter-fg-color: #df50af;
   --doc-symbol-attribute-fg-color: #953800;
   --doc-symbol-function-fg-color: #8250df;
   --doc-symbol-method-fg-color: #8250df;
   --doc-symbol-class-fg-color: #0550ae;
   --doc-symbol-module-fg-color: #5cad0f;
 
+  --doc-symbol-parameter-bg-color: #df50af1a;
   --doc-symbol-attribute-bg-color: #9538001a;
   --doc-symbol-function-bg-color: #8250df1a;
   --doc-symbol-method-bg-color: #8250df1a;
@@ -48,12 +61,14 @@
 }
 
 [data-md-color-scheme="slate"] {
+  --doc-symbol-parameter-fg-color: #ffa8cc;
   --doc-symbol-attribute-fg-color: #ffa657;
   --doc-symbol-function-fg-color: #d2a8ff;
   --doc-symbol-method-fg-color: #d2a8ff;
   --doc-symbol-class-fg-color: #79c0ff;
   --doc-symbol-module-fg-color: #baff79;
 
+  --doc-symbol-parameter-bg-color: #ffa8cc1a;
   --doc-symbol-attribute-bg-color: #ffa6571a;
   --doc-symbol-function-bg-color: #d2a8ff1a;
   --doc-symbol-method-bg-color: #d2a8ff1a;
@@ -68,6 +83,15 @@ code.doc-symbol {
   font-weight: bold;
 }
 
+code.doc-symbol-parameter {
+  color: var(--doc-symbol-parameter-fg-color);
+  background-color: var(--doc-symbol-parameter-bg-color);
+}
+
+code.doc-symbol-parameter::after {
+  content: "param";
+}
+
 code.doc-symbol-attribute {
   color: var(--doc-symbol-attribute-fg-color);
   background-color: var(--doc-symbol-attribute-bg-color);
diff --git a/mkdocs/site/contributing.html b/mkdocs/site/contributing.html
index 0f4867ab..fd18d032 100644
--- a/mkdocs/site/contributing.html
+++ b/mkdocs/site/contributing.html
@@ -16,7 +16,7 @@
       
       
       <link rel="icon" href="assets/images/favicon.ico">
-      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.35">
+      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.43">
     
     
       
@@ -24,7 +24,7 @@
       
     
     
-      <link rel="stylesheet" href="assets/stylesheets/main.35f28582.min.css">
+      <link rel="stylesheet" href="assets/stylesheets/main.0253249f.min.css">
       
         
         <link rel="stylesheet" href="assets/stylesheets/palette.06af60db.min.css">
@@ -1640,7 +1640,7 @@ <h1>Contributing</h1>
     <script id="__config" type="application/json">{"base": ".", "features": ["navigation.tabs", "navigation.tabs.sticky", "navigation.indexes", "navigation.expand", "toc.integrate"], "search": "assets/javascripts/workers/search.6ce7567c.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version": "Select version"}}</script>
     
     
-      <script src="assets/javascripts/bundle.56dfad97.min.js"></script>
+      <script src="assets/javascripts/bundle.83f73b43.min.js"></script>
       
         <script src="stylesheets/extra.js"></script>
       
diff --git a/mkdocs/site/contributing.md b/mkdocs/site/contributing.md
deleted file mode 100644
index 5aee05c7..00000000
--- a/mkdocs/site/contributing.md
+++ /dev/null
@@ -1,7 +0,0 @@
-Pytorch-widedeep is being developed and used by many active community members. Your help is very valuable to make it better for everyone.
-
-- Check for the [Roadmap](https://github.com/users/jrzaurin/projects/3) or [Open an issue](https://github.com/jrzaurin/pytorch-widedeep/issues) to report problems or recommend new features and submit a draft pull requests, which will be changed to pull request after intial review
-- Contribute to the [tests](https://github.com/jrzaurin/pytorch-widedeep/tree/master/tests) to make it more reliable.
-- Contribute to the [documentation](https://github.com/jrzaurin/pytorch-widedeep/tree/master/docs) to make it clearer for everyone.
-- Contribute to the [examples](https://github.com/jrzaurin/pytorch-widedeep/tree/master/examples) to share your experience with other users.
-- Join the dicussion on [slack](https://join.slack.com/t/pytorch-widedeep/shared_invite/zt-soss7stf-iXpVuLeKZz8lGTnxxtHtTw)
\ No newline at end of file
diff --git a/mkdocs/site/index.html b/mkdocs/site/index.html
index 71873d46..0beb6e5c 100644
--- a/mkdocs/site/index.html
+++ b/mkdocs/site/index.html
@@ -16,7 +16,7 @@
       
       
       <link rel="icon" href="assets/images/favicon.ico">
-      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.35">
+      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.43">
     
     
       
@@ -24,7 +24,7 @@
       
     
     
-      <link rel="stylesheet" href="assets/stylesheets/main.35f28582.min.css">
+      <link rel="stylesheet" href="assets/stylesheets/main.0253249f.min.css">
       
         
         <link rel="stylesheet" href="assets/stylesheets/palette.06af60db.min.css">
@@ -2382,8 +2382,7 @@ <h4 id="apa">APA<a class="headerlink" href="#apa" title="Permanent link">&para;<
       
         <a href="mailto:jrzaurin@gmail.com">Javier</a>, 
         <a href="mailto:mulinka.pavol@gmail.com">Pavol Mulinka</a>, 
-        <a href="mailto:javierrodriguezzaurin@javiers-macbook-pro.local">Javier Rodriguez Zaurin</a>, 
-        <a href="mailto:not.committed.yet">Not Committed Yet</a>
+        <a href="mailto:javierrodriguezzaurin@javiers-macbook-pro.local">Javier Rodriguez Zaurin</a>
     </nav>
   </span>
 
@@ -2451,7 +2450,7 @@ <h4 id="apa">APA<a class="headerlink" href="#apa" title="Permanent link">&para;<
     <script id="__config" type="application/json">{"base": ".", "features": ["navigation.tabs", "navigation.tabs.sticky", "navigation.indexes", "navigation.expand", "toc.integrate"], "search": "assets/javascripts/workers/search.6ce7567c.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version": "Select version"}}</script>
     
     
-      <script src="assets/javascripts/bundle.56dfad97.min.js"></script>
+      <script src="assets/javascripts/bundle.83f73b43.min.js"></script>
       
         <script src="stylesheets/extra.js"></script>
       
diff --git a/mkdocs/site/index.md b/mkdocs/site/index.md
deleted file mode 100644
index fb6aebfb..00000000
--- a/mkdocs/site/index.md
+++ /dev/null
@@ -1,890 +0,0 @@
-
-<p align="center">
-  <img width="300" src="docs/figures/widedeep_logo.png">
-</p>
-
-[![PyPI version](https://badge.fury.io/py/pytorch-widedeep.svg)](https://pypi.org/project/pytorch-widedeep/)
-[![Python 3.7 3.8 3.9 3.10](https://img.shields.io/badge/python-3.7%20%7C%203.8%20%7C%203.9%20%7C%203.10-blue.svg)](https://pypi.org/project/pytorch-widedeep/)
-[![Build Status](https://github.com/jrzaurin/pytorch-widedeep/actions/workflows/build.yml/badge.svg)](https://github.com/jrzaurin/pytorch-widedeep/actions)
-[![Documentation Status](https://readthedocs.org/projects/pytorch-widedeep/badge/?version=latest)](https://pytorch-widedeep.readthedocs.io/en/latest/?badge=latest)
-[![codecov](https://codecov.io/gh/jrzaurin/pytorch-widedeep/branch/master/graph/badge.svg)](https://codecov.io/gh/jrzaurin/pytorch-widedeep)
-[![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)
-[![Maintenance](https://img.shields.io/badge/Maintained%3F-yes-green.svg)](https://github.com/jrzaurin/pytorch-widedeep/graphs/commit-activity)
-[![contributions welcome](https://img.shields.io/badge/contributions-welcome-brightgreen.svg?style=flat)](https://github.com/jrzaurin/pytorch-widedeep/issues)
-[![Slack](https://img.shields.io/badge/slack-chat-green.svg?logo=slack)](https://join.slack.com/t/pytorch-widedeep/shared_invite/zt-soss7stf-iXpVuLeKZz8lGTnxxtHtTw)
-[![DOI](https://joss.theoj.org/papers/10.21105/joss.05027/status.svg)](https://doi.org/10.21105/joss.05027)
-
-# **pytorch-widedeep**
-
-A flexible package for multimodal-deep-learning to combine tabular data with
-text and images using Wide and Deep models in Pytorch
-
-**Documentation:** [https://pytorch-widedeep.readthedocs.io](https://pytorch-widedeep.readthedocs.io/en/latest/index.html)
-
-**Companion posts and tutorials:** [infinitoml](https://jrzaurin.github.io/infinitoml/)
-
-**Experiments and comparison with `LightGBM`**: [TabularDL vs LightGBM](https://github.com/jrzaurin/tabulardl-benchmark)
-
-**Slack**: if you want to contribute or just want to chat with us, join [slack](https://join.slack.com/t/pytorch-widedeep/shared_invite/zt-soss7stf-iXpVuLeKZz8lGTnxxtHtTw)
-
-The content of this document is organized as follows:
-
-- [**pytorch-widedeep**](#pytorch-widedeep)
-    - [Introduction](#introduction)
-    - [Architectures](#architectures)
-    - [The ``deeptabular`` component](#the-deeptabular-component)
-    - [The ``rec`` module](#the-rec-module)
-    - [Text and Images](#text-and-images)
-    - [Acknowledgments](#acknowledgments)
-    - [License](#license)
-    - [Cite](#cite)
-        - [BibTex](#bibtex)
-        - [APA](#apa)
-
-
-### Introduction
-
-``pytorch-widedeep`` is based on Google's [Wide and Deep Algorithm](https://arxiv.org/abs/1606.07792),
-adjusted for multi-modal datasets.
-
-In general terms, `pytorch-widedeep` is a package to use deep learning with
-tabular data. In particular, is intended to facilitate the combination of
-text and images with corresponding tabular data using wide and deep models.
-With that in mind there are a number of architectures that can be implemented
-with the library. The main components of those architectures are shown in the
-Figure below:
-
-
-<p align="center">
-  <img width="750" src="docs/figures/widedeep_arch_new.png">
-</p>
-
-
-In math terms, and following the notation in the
-[paper](https://arxiv.org/abs/1606.07792), the expression for the architecture
-without a ``deephead`` component can be formulated as:
-
-<p align="center">
-  <img width="500" src="docs/figures/architecture_1_math.png">
-</p>
-
-
-Where &sigma; is the sigmoid function, *'W'* are the weight matrices applied to the wide model and to the final
-activations of the deep models, *'a'* are these final activations,
-&phi;(x) are the cross product transformations of the original features *'x'*, and
-, and *'b'* is the bias term.
-In case you are wondering what are *"cross product transformations"*, here is
-a quote taken directly from the paper: *"For binary features, a cross-product
-transformation (e.g., “AND(gender=female, language=en)”) is 1 if and only if
-the constituent features (“gender=female” and “language=en”) are all 1, and 0
-otherwise".*
-
-It is perfectly possible to use custom models (and not necessarily those in
-the library) as long as the the custom models have a property called
-``output_dim`` with the size of the last layer of activations, so that
-``WideDeep`` can be constructed. Examples on how to use custom components can
-be found in the Examples folder and the section below.
-
-### Architectures
-
-The `pytorch-widedeep` library offers a number of different architectures. In
-this section we will show some of them in their simplest form (i.e. with
-default param values in most cases) with their corresponding code snippets.
-Note that **all** the snippets below shoud run locally. For a more detailed
-explanation of the different components and their parameters, please refer to
-the documentation.
-
-For the examples below we will be using a toy dataset generated as follows:
-
-```python
-import os
-import random
-
-import numpy as np
-import pandas as pd
-from PIL import Image
-from faker import Faker
-
-
-def create_and_save_random_image(image_number, size=(32, 32)):
-
-    if not os.path.exists("images"):
-        os.makedirs("images")
-
-    array = np.random.randint(0, 256, (size[0], size[1], 3), dtype=np.uint8)
-
-    image = Image.fromarray(array)
-
-    image_name = f"image_{image_number}.png"
-    image.save(os.path.join("images", image_name))
-
-    return image_name
-
-
-fake = Faker()
-
-cities = ["New York", "Los Angeles", "Chicago", "Houston"]
-names = ["Alice", "Bob", "Charlie", "David", "Eva"]
-
-data = {
-    "city": [random.choice(cities) for _ in range(100)],
-    "name": [random.choice(names) for _ in range(100)],
-    "age": [random.uniform(18, 70) for _ in range(100)],
-    "height": [random.uniform(150, 200) for _ in range(100)],
-    "sentence": [fake.sentence() for _ in range(100)],
-    "other_sentence": [fake.sentence() for _ in range(100)],
-    "image_name": [create_and_save_random_image(i) for i in range(100)],
-    "target": [random.choice([0, 1]) for _ in range(100)],
-}
-
-df = pd.DataFrame(data)
-```
-
-This will create a 100 rows dataframe and a dir in your local folder, called
-`images` with 100 random images (or images with just noise).
-
-Perhaps the simplest architecture would be just one component, `wide`,
-`deeptabular`, `deeptext` or `deepimage` on their own, which is also
-possible, but let's start the examples with a standard Wide and Deep
-architecture. From there, how to build a model comprised only of one
-component will be straightforward.
-
-Note that the examples shown below would be almost identical using any of the
-models available in the library. For example, `TabMlp` can be replaced by
-`TabResnet`, `TabNet`, `TabTransformer`, etc. Similarly, `BasicRNN` can be
-replaced by `AttentiveRNN`, `StackedAttentiveRNN`, or `HFModel` with
-their corresponding parameters and preprocessor in the case of the Hugging
-Face models.
-
-**1. Wide and Tabular component (aka deeptabular)**
-
-
-<p align="center">
-  <img width="400" src="docs/figures/arch_1.png">
-</p>
-
-
-```python
-from pytorch_widedeep.preprocessing import TabPreprocessor, WidePreprocessor
-from pytorch_widedeep.models import Wide, TabMlp, WideDeep
-from pytorch_widedeep.training import Trainer
-
-# Wide
-wide_cols = ["city"]
-crossed_cols = [("city", "name")]
-wide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols)
-X_wide = wide_preprocessor.fit_transform(df)
-wide = Wide(input_dim=np.unique(X_wide).shape[0])
-
-# Tabular
-tab_preprocessor = TabPreprocessor(
-    embed_cols=["city", "name"], continuous_cols=["age", "height"]
-)
-X_tab = tab_preprocessor.fit_transform(df)
-tab_mlp = TabMlp(
-    column_idx=tab_preprocessor.column_idx,
-    cat_embed_input=tab_preprocessor.cat_embed_input,
-    continuous_cols=tab_preprocessor.continuous_cols,
-    mlp_hidden_dims=[64, 32],
-)
-
-# WideDeep
-model = WideDeep(wide=wide, deeptabular=tab_mlp)
-
-# Train
-trainer = Trainer(model, objective="binary")
-
-trainer.fit(
-    X_wide=X_wide,
-    X_tab=X_tab,
-    target=df["target"].values,
-    n_epochs=1,
-    batch_size=32,
-)
-```
-
-**2. Tabular and Text data**
-
-<p align="center">
-  <img width="400" src="docs/figures/arch_2.png">
-</p>
-
-
-```python
-from pytorch_widedeep.preprocessing import TabPreprocessor, TextPreprocessor
-from pytorch_widedeep.models import TabMlp, BasicRNN, WideDeep
-from pytorch_widedeep.training import Trainer
-
-# Tabular
-tab_preprocessor = TabPreprocessor(
-    embed_cols=["city", "name"], continuous_cols=["age", "height"]
-)
-X_tab = tab_preprocessor.fit_transform(df)
-tab_mlp = TabMlp(
-    column_idx=tab_preprocessor.column_idx,
-    cat_embed_input=tab_preprocessor.cat_embed_input,
-    continuous_cols=tab_preprocessor.continuous_cols,
-    mlp_hidden_dims=[64, 32],
-)
-
-# Text
-text_preprocessor = TextPreprocessor(
-    text_col="sentence", maxlen=20, max_vocab=100, n_cpus=1
-)
-X_text = text_preprocessor.fit_transform(df)
-rnn = BasicRNN(
-    vocab_size=len(text_preprocessor.vocab.itos),
-    embed_dim=16,
-    hidden_dim=8,
-    n_layers=1,
-)
-
-# WideDeep
-model = WideDeep(deeptabular=tab_mlp, deeptext=rnn)
-
-# Train
-trainer = Trainer(model, objective="binary")
-
-trainer.fit(
-    X_tab=X_tab,
-    X_text=X_text,
-    target=df["target"].values,
-    n_epochs=1,
-    batch_size=32,
-)
-```
-
-**3. Tabular and text with a FC head on top via the `head_hidden_dims` param
-  in `WideDeep`**
-
-<p align="center">
-  <img width="400" src="docs/figures/arch_3.png">
-</p>
-
-```python
-from pytorch_widedeep.preprocessing import TabPreprocessor, TextPreprocessor
-from pytorch_widedeep.models import TabMlp, BasicRNN, WideDeep
-from pytorch_widedeep.training import Trainer
-
-# Tabular
-tab_preprocessor = TabPreprocessor(
-    embed_cols=["city", "name"], continuous_cols=["age", "height"]
-)
-X_tab = tab_preprocessor.fit_transform(df)
-tab_mlp = TabMlp(
-    column_idx=tab_preprocessor.column_idx,
-    cat_embed_input=tab_preprocessor.cat_embed_input,
-    continuous_cols=tab_preprocessor.continuous_cols,
-    mlp_hidden_dims=[64, 32],
-)
-
-# Text
-text_preprocessor = TextPreprocessor(
-    text_col="sentence", maxlen=20, max_vocab=100, n_cpus=1
-)
-X_text = text_preprocessor.fit_transform(df)
-rnn = BasicRNN(
-    vocab_size=len(text_preprocessor.vocab.itos),
-    embed_dim=16,
-    hidden_dim=8,
-    n_layers=1,
-)
-
-# WideDeep
-model = WideDeep(deeptabular=tab_mlp, deeptext=rnn, head_hidden_dims=[32, 16])
-
-# Train
-trainer = Trainer(model, objective="binary")
-
-trainer.fit(
-    X_tab=X_tab,
-    X_text=X_text,
-    target=df["target"].values,
-    n_epochs=1,
-    batch_size=32,
-)
-```
-
-**4. Tabular and multiple text columns that are passed directly to
-  `WideDeep`**
-
-<p align="center">
-  <img width="500" src="docs/figures/arch_4.png">
-</p>
-
-```python
-from pytorch_widedeep.preprocessing import TabPreprocessor, TextPreprocessor
-from pytorch_widedeep.models import TabMlp, BasicRNN, WideDeep
-from pytorch_widedeep.training import Trainer
-
-
-# Tabular
-tab_preprocessor = TabPreprocessor(
-    embed_cols=["city", "name"], continuous_cols=["age", "height"]
-)
-X_tab = tab_preprocessor.fit_transform(df)
-tab_mlp = TabMlp(
-    column_idx=tab_preprocessor.column_idx,
-    cat_embed_input=tab_preprocessor.cat_embed_input,
-    continuous_cols=tab_preprocessor.continuous_cols,
-    mlp_hidden_dims=[64, 32],
-)
-
-# Text
-text_preprocessor_1 = TextPreprocessor(
-    text_col="sentence", maxlen=20, max_vocab=100, n_cpus=1
-)
-X_text_1 = text_preprocessor_1.fit_transform(df)
-text_preprocessor_2 = TextPreprocessor(
-    text_col="other_sentence", maxlen=20, max_vocab=100, n_cpus=1
-)
-X_text_2 = text_preprocessor_2.fit_transform(df)
-rnn_1 = BasicRNN(
-    vocab_size=len(text_preprocessor_1.vocab.itos),
-    embed_dim=16,
-    hidden_dim=8,
-    n_layers=1,
-)
-rnn_2 = BasicRNN(
-    vocab_size=len(text_preprocessor_2.vocab.itos),
-    embed_dim=16,
-    hidden_dim=8,
-    n_layers=1,
-)
-
-# WideDeep
-model = WideDeep(deeptabular=tab_mlp, deeptext=[rnn_1, rnn_2])
-
-# Train
-trainer = Trainer(model, objective="binary")
-
-trainer.fit(
-    X_tab=X_tab,
-    X_text=[X_text_1, X_text_2],
-    target=df["target"].values,
-    n_epochs=1,
-    batch_size=32,
-)
-```
-
-**5. Tabular data and multiple text columns that are fused via a the library's
-  `ModelFuser` class**
-
-<p align="center">
-    <img width="500" src="docs/figures/arch_5.png">
-</p>
-
-```python
-from pytorch_widedeep.preprocessing import TabPreprocessor, TextPreprocessor
-from pytorch_widedeep.models import TabMlp, BasicRNN, WideDeep, ModelFuser
-from pytorch_widedeep import Trainer
-
-# Tabular
-tab_preprocessor = TabPreprocessor(
-    embed_cols=["city", "name"], continuous_cols=["age", "height"]
-)
-X_tab = tab_preprocessor.fit_transform(df)
-tab_mlp = TabMlp(
-    column_idx=tab_preprocessor.column_idx,
-    cat_embed_input=tab_preprocessor.cat_embed_input,
-    continuous_cols=tab_preprocessor.continuous_cols,
-    mlp_hidden_dims=[64, 32],
-)
-
-# Text
-text_preprocessor_1 = TextPreprocessor(
-    text_col="sentence", maxlen=20, max_vocab=100, n_cpus=1
-)
-X_text_1 = text_preprocessor_1.fit_transform(df)
-text_preprocessor_2 = TextPreprocessor(
-    text_col="other_sentence", maxlen=20, max_vocab=100, n_cpus=1
-)
-X_text_2 = text_preprocessor_2.fit_transform(df)
-
-rnn_1 = BasicRNN(
-    vocab_size=len(text_preprocessor_1.vocab.itos),
-    embed_dim=16,
-    hidden_dim=8,
-    n_layers=1,
-)
-rnn_2 = BasicRNN(
-    vocab_size=len(text_preprocessor_2.vocab.itos),
-    embed_dim=16,
-    hidden_dim=8,
-    n_layers=1,
-)
-
-models_fuser = ModelFuser(models=[rnn_1, rnn_2], fusion_method="mult")
-
-# WideDeep
-model = WideDeep(deeptabular=tab_mlp, deeptext=models_fuser)
-
-# Train
-trainer = Trainer(model, objective="binary")
-
-trainer.fit(
-    X_tab=X_tab,
-    X_text=[X_text_1, X_text_2],
-    target=df["target"].values,
-    n_epochs=1,
-    batch_size=32,
-)
-```
-
-**6. Tabular and multiple text columns, with an image column. The text columns
-  are fused via the library's `ModelFuser` and then all fused via the
-  deephead paramenter in `WideDeep` which is a custom `ModelFuser` coded by
-  the user**
-
-This is perhaps the less elegant solution as it involves a custom component by
-the user and slicing the 'incoming' tensor. In the future, we will include a
-`TextAndImageModelFuser` to make this process more straightforward. Still, is not
-really complicated and it is a good example of how to use custom components in
-`pytorch-widedeep`.
-
-Note that the only requirement for the custom component is that it has a
-property called `output_dim` that returns the size of the last layer of
-activations. In other words, it does not need to inherit from
-`BaseWDModelComponent`. This base class simply checks the existence of such
-property and avoids some typing errors internally.
-
-
-<p align="center">
-    <img width="600" src="docs/figures/arch_6.png">
-</p>
-
-
-```python
-import torch
-
-from pytorch_widedeep.preprocessing import TabPreprocessor, TextPreprocessor, ImagePreprocessor
-from pytorch_widedeep.models import TabMlp, BasicRNN, WideDeep, ModelFuser, Vision
-from pytorch_widedeep.models._base_wd_model_component import BaseWDModelComponent
-from pytorch_widedeep import Trainer
-
-# Tabular
-tab_preprocessor = TabPreprocessor(
-    embed_cols=["city", "name"], continuous_cols=["age", "height"]
-)
-X_tab = tab_preprocessor.fit_transform(df)
-tab_mlp = TabMlp(
-    column_idx=tab_preprocessor.column_idx,
-    cat_embed_input=tab_preprocessor.cat_embed_input,
-    continuous_cols=tab_preprocessor.continuous_cols,
-    mlp_hidden_dims=[16, 8],
-)
-
-# Text
-text_preprocessor_1 = TextPreprocessor(
-    text_col="sentence", maxlen=20, max_vocab=100, n_cpus=1
-)
-X_text_1 = text_preprocessor_1.fit_transform(df)
-text_preprocessor_2 = TextPreprocessor(
-    text_col="other_sentence", maxlen=20, max_vocab=100, n_cpus=1
-)
-X_text_2 = text_preprocessor_2.fit_transform(df)
-rnn_1 = BasicRNN(
-    vocab_size=len(text_preprocessor_1.vocab.itos),
-    embed_dim=16,
-    hidden_dim=8,
-    n_layers=1,
-)
-rnn_2 = BasicRNN(
-    vocab_size=len(text_preprocessor_2.vocab.itos),
-    embed_dim=16,
-    hidden_dim=8,
-    n_layers=1,
-)
-models_fuser = ModelFuser(
-    models=[rnn_1, rnn_2],
-    fusion_method="mult",
-)
-
-# Image
-image_preprocessor = ImagePreprocessor(img_col="image_name", img_path="images")
-X_img = image_preprocessor.fit_transform(df)
-vision = Vision(pretrained_model_setup="resnet18", head_hidden_dims=[16, 8])
-
-# deephead (custom model fuser)
-class MyModelFuser(BaseWDModelComponent):
-    """
-    Simply a Linear + Relu sequence on top of the text + images followed by a
-    Linear -> Relu -> Linear for the concatenation of tabular slice of the
-    tensor and the output of the text and image sequential model
-    """
-    def __init__(
-        self,
-        tab_incoming_dim: int,
-        text_incoming_dim: int,
-        image_incoming_dim: int,
-        output_units: int,
-    ):
-
-        super(MyModelFuser, self).__init__()
-
-        self.tab_incoming_dim = tab_incoming_dim
-        self.text_incoming_dim = text_incoming_dim
-        self.image_incoming_dim = image_incoming_dim
-        self.output_units = output_units
-        self.text_and_image_fuser = torch.nn.Sequential(
-            torch.nn.Linear(text_incoming_dim + image_incoming_dim, output_units),
-            torch.nn.ReLU(),
-        )
-        self.out = torch.nn.Sequential(
-            torch.nn.Linear(output_units + tab_incoming_dim, output_units * 4),
-            torch.nn.ReLU(),
-            torch.nn.Linear(output_units * 4, output_units),
-        )
-
-    def forward(self, X: torch.Tensor) -> torch.Tensor:
-        tab_slice = slice(0, self.tab_incoming_dim)
-        text_slice = slice(
-            self.tab_incoming_dim, self.tab_incoming_dim + self.text_incoming_dim
-        )
-        image_slice = slice(
-            self.tab_incoming_dim + self.text_incoming_dim,
-            self.tab_incoming_dim + self.text_incoming_dim + self.image_incoming_dim,
-        )
-        X_tab = X[:, tab_slice]
-        X_text = X[:, text_slice]
-        X_img = X[:, image_slice]
-        X_text_and_image = self.text_and_image_fuser(torch.cat([X_text, X_img], dim=1))
-        return self.out(torch.cat([X_tab, X_text_and_image], dim=1))
-
-    @property
-    def output_dim(self):
-        return self.output_units
-
-deephead = MyModelFuser(
-    tab_incoming_dim=tab_mlp.output_dim,
-    text_incoming_dim=models_fuser.output_dim,
-    image_incoming_dim=vision.output_dim,
-    output_units=8,
-)
-
-# WideDeep
-model = WideDeep(
-    deeptabular=tab_mlp,
-    deeptext=models_fuser,
-    deepimage=vision,
-    deephead=deephead,
-)
-
-# Train
-trainer = Trainer(model, objective="binary")
-
-trainer.fit(
-    X_tab=X_tab,
-    X_text=[X_text_1, X_text_2],
-    X_img=X_img,
-    target=df["target"].values,
-    n_epochs=1,
-    batch_size=32,
-)
-```
-
-**7. A two-tower model**
-
-This is a popular model in the context of recommendation systems. Let's say we
-have a tabular dataset formed my triples (user features, item features,
-target). We can create a two-tower model where the user and item features are
-passed through two separate models and then "fused" via a dot product.
-
-<p align="center">
-  <img width="350" src="docs/figures/arch_7.png">
-</p>
-
-
-```python
-import numpy as np
-import pandas as pd
-
-from pytorch_widedeep import Trainer
-from pytorch_widedeep.preprocessing import TabPreprocessor
-from pytorch_widedeep.models import TabMlp, WideDeep, ModelFuser
-
-# Let's create the interaction dataset
-# user_features dataframe
-np.random.seed(42)
-user_ids = np.arange(1, 101)
-ages = np.random.randint(18, 60, size=100)
-genders = np.random.choice(["male", "female"], size=100)
-locations = np.random.choice(["city_a", "city_b", "city_c", "city_d"], size=100)
-user_features = pd.DataFrame(
-    {"id": user_ids, "age": ages, "gender": genders, "location": locations}
-)
-
-# item_features dataframe
-item_ids = np.arange(1, 101)
-prices = np.random.uniform(10, 500, size=100).round(2)
-colors = np.random.choice(["red", "blue", "green", "black"], size=100)
-categories = np.random.choice(["electronics", "clothing", "home", "toys"], size=100)
-
-item_features = pd.DataFrame(
-    {"id": item_ids, "price": prices, "color": colors, "category": categories}
-)
-
-# Interactions dataframe
-interaction_user_ids = np.random.choice(user_ids, size=1000)
-interaction_item_ids = np.random.choice(item_ids, size=1000)
-purchased = np.random.choice([0, 1], size=1000, p=[0.7, 0.3])
-interactions = pd.DataFrame(
-    {
-        "user_id": interaction_user_ids,
-        "item_id": interaction_item_ids,
-        "purchased": purchased,
-    }
-)
-user_item_purchased = interactions.merge(
-    user_features, left_on="user_id", right_on="id"
-).merge(item_features, left_on="item_id", right_on="id")
-
-# Users
-tab_preprocessor_user = TabPreprocessor(
-    cat_embed_cols=["gender", "location"],
-    continuous_cols=["age"],
-)
-X_user = tab_preprocessor_user.fit_transform(user_item_purchased)
-tab_mlp_user = TabMlp(
-    column_idx=tab_preprocessor_user.column_idx,
-    cat_embed_input=tab_preprocessor_user.cat_embed_input,
-    continuous_cols=["age"],
-    mlp_hidden_dims=[16, 8],
-    mlp_dropout=[0.2, 0.2],
-)
-
-# Items
-tab_preprocessor_item = TabPreprocessor(
-    cat_embed_cols=["color", "category"],
-    continuous_cols=["price"],
-)
-X_item = tab_preprocessor_item.fit_transform(user_item_purchased)
-tab_mlp_item = TabMlp(
-    column_idx=tab_preprocessor_item.column_idx,
-    cat_embed_input=tab_preprocessor_item.cat_embed_input,
-    continuous_cols=["price"],
-    mlp_hidden_dims=[16, 8],
-    mlp_dropout=[0.2, 0.2],
-)
-
-two_tower_model = ModelFuser([tab_mlp_user, tab_mlp_item], fusion_method="dot")
-
-model = WideDeep(deeptabular=two_tower_model)
-
-trainer = Trainer(model, objective="binary")
-
-trainer.fit(
-    X_tab=[X_user, X_item],
-    target=interactions.purchased.values,
-    n_epochs=1,
-    batch_size=32,
-)
-```
-
-**8. Tabular with a multi-target loss**
-
-This one is "a bonus" to illustrate the use of multi-target losses, more than
-actually a different architecture.
-
-<p align="center">
-  <img width="200" src="docs/figures/arch_8.png">
-</p>
-
-
-```python
-from pytorch_widedeep.preprocessing import TabPreprocessor, TextPreprocessor, ImagePreprocessor
-from pytorch_widedeep.models import TabMlp, BasicRNN, WideDeep, ModelFuser, Vision
-from pytorch_widedeep.losses_multitarget import MultiTargetClassificationLoss
-from pytorch_widedeep.models._base_wd_model_component import BaseWDModelComponent
-from pytorch_widedeep import Trainer
-
-# let's add a second target to the dataframe
-df["target2"] = [random.choice([0, 1]) for _ in range(100)]
-
-# Tabular
-tab_preprocessor = TabPreprocessor(
-    embed_cols=["city", "name"], continuous_cols=["age", "height"]
-)
-X_tab = tab_preprocessor.fit_transform(df)
-tab_mlp = TabMlp(
-    column_idx=tab_preprocessor.column_idx,
-    cat_embed_input=tab_preprocessor.cat_embed_input,
-    continuous_cols=tab_preprocessor.continuous_cols,
-    mlp_hidden_dims=[64, 32],
-)
-
-# 'pred_dim=2' because we have two binary targets. For other types of targets,
-#  please, see the documentation
-model = WideDeep(deeptabular=tab_mlp, pred_dim=2).
-
-loss = MultiTargetClassificationLoss(binary_config=[0, 1], reduction="mean")
-
-# When a multi-target loss is used, 'custom_loss_function' must not be None.
-# See the docs
-trainer = Trainer(model, objective="multitarget", custom_loss_function=loss)
-
-trainer.fit(
-    X_tab=X_tab,
-    target=df[["target", "target2"]].values,
-    n_epochs=1,
-    batch_size=32,
-)
-```
-
-### The ``deeptabular`` component
-
-It is important to emphasize again that **each individual component, `wide`,
-`deeptabular`, `deeptext` and `deepimage`, can be used independently** and in
-isolation. For example, one could use only `wide`, which is in simply a
-linear model. In fact, one of the most interesting functionalities
-in``pytorch-widedeep`` would be the use of the ``deeptabular`` component on
-its own, i.e. what one might normally refer as Deep Learning for Tabular
-Data. Currently, ``pytorch-widedeep`` offers the following different models
-for that component:
-
-0. **Wide**: a simple linear model where the nonlinearities are captured via
-cross-product transformations, as explained before.
-1. **TabMlp**: a simple MLP that receives embeddings representing the
-categorical features, concatenated with the continuous features, which can
-also be embedded.
-2. **TabResnet**: similar to the previous model but the embeddings are
-passed through a series of ResNet blocks built with dense layers.
-3. **TabNet**: details on TabNet can be found in
-[TabNet: Attentive Interpretable Tabular Learning](https://arxiv.org/abs/1908.07442)
-
-Two simpler attention based models that we call:
-
-4. **ContextAttentionMLP**: MLP with at attention mechanism "on top" that is based on
-    [Hierarchical Attention Networks for Document Classification](https://www.cs.cmu.edu/~./hovy/papers/16HLT-hierarchical-attention-networks.pd)
-5. **SelfAttentionMLP**: MLP with an attention mechanism that is a simplified
-    version of a transformer block that we refer as "query-key self-attention".
-
-The ``Tabformer`` family, i.e. Transformers for Tabular data:
-
-6. **TabTransformer**: details on the TabTransformer can be found in
-[TabTransformer: Tabular Data Modeling Using Contextual Embeddings](https://arxiv.org/pdf/2012.06678.pdf).
-7. **SAINT**: Details on SAINT can be found in
-[SAINT: Improved Neural Networks for Tabular Data via Row Attention and Contrastive Pre-Training](https://arxiv.org/abs/2106.01342).
-8. **FT-Transformer**: details on the FT-Transformer can be found in
-[Revisiting Deep Learning Models for Tabular Data](https://arxiv.org/abs/2106.11959).
-9. **TabFastFormer**: adaptation of the FastFormer for tabular data. Details
-on the Fasformer can be found in
-[FastFormers: Highly Efficient Transformer Models for Natural Language Understanding](https://arxiv.org/abs/2010.13382)
-10. **TabPerceiver**: adaptation of the Perceiver for tabular data. Details on
-the Perceiver can be found in
-[Perceiver: General Perception with Iterative Attention](https://arxiv.org/abs/2103.03206)
-
-And probabilistic DL models for tabular data based on
-[Weight Uncertainty in Neural Networks](https://arxiv.org/abs/1505.05424):
-
-11. **BayesianWide**: Probabilistic adaptation of the `Wide` model.
-12. **BayesianTabMlp**: Probabilistic adaptation of the `TabMlp` model
-
-Note that while there are scientific publications for the TabTransformer,
-SAINT and FT-Transformer, the TabFasfFormer and TabPerceiver are our own
-adaptation of those algorithms for tabular data.
-
-In addition, Self-Supervised pre-training can be used for all `deeptabular`
-models, with the exception of the `TabPerceiver`. Self-Supervised
-pre-training can be used via two methods or routines which we refer as:
-encoder-decoder method and constrastive-denoising method. Please, see the
-documentation and the examples for details on this functionality, and all
-other options in the library.
-
-### The ``rec`` module
-
-This module was introduced as an extension to the existing components in the
-library, addressing questions and issues related to recommendation systems.
-While still under active development, it currently includes a select number
-of powerful recommendation models.
-
-It's worth noting that this library already supported the implementation of
-various recommendation algorithms using existing components. For example,
-models like Wide and Deep, Two-Tower, or Neural Collaborative Filtering could
-be constructed using the library's core functionalities.
-
-The recommendation algorithms in the `rec` module are:
-
-1. [DeepFM: A Factorization-Machine based Neural Network for CTR Prediction](https://arxiv.org/abs/1703.04247)
-2. (Deep) Field Aware Factorization Machine (FFM): a Deep Learning version of the algorithm presented in [Field-aware Factorization Machines in a Real-world Online Advertising System](https://arxiv.org/abs/1701.04099)
-3. [xDeepFM: Combining Explicit and Implicit Feature Interactions for Recommender Systems](https://arxiv.org/pdf/1803.05170)
-4. [Deep Interest Network for Click-Through Rate Prediction](https://arxiv.org/abs/1706.06978)
-
-These can all be used as the `deeptabular` component in the `WideDeep` model.
-See the examples for more details.
-
-### Text and Images
-For the text component, `deeptext`, the library offers the following models:
-
-1. **BasicRNN**: a simple RNN 2. **AttentiveRNN**: a RNN with an attention
-mechanism based on the
-[Hierarchical Attention Networks for DocumentClassification](https://www.cs.cmu.edu/~./hovy/papers/16HLT-hierarchical-attention-networks.pd)
-3. **StackedAttentiveRNN**: a stack of AttentiveRNNs
-4. **HFModel**: a wrapper around Hugging Face Transfomer-based models. At the moment
-only models from the families BERT, RoBERTa, DistilBERT, ALBERT and ELECTRA
-are supported. This is because this library is designed to address
-classification and regression tasks and these are the most 'popular'
-encoder-only models, which have proved to be those that work best for these
-tasks. If there is demand for other models, they will be included in the
-future.
-
-For the image component, `deepimage`, the library supports models from the
-following families:
-'resnet', 'shufflenet', 'resnext', 'wide_resnet', 'regnet', 'densenet', 'mobilenetv3',
- 'mobilenetv2', 'mnasnet', 'efficientnet' and 'squeezenet'.  These are
- offered via `torchvision` and wrapped up in the `Vision` class.
-
-### Acknowledgments
-
-This library takes from a series of other libraries, so I think it is just
-fair to mention them here in the README (specific mentions are also included
-in the code).
-
-The `Callbacks` and `Initializers` structure and code is inspired by the
-[`torchsample`](https://github.com/ncullen93/torchsample) library, which in
-itself partially inspired by [`Keras`](https://keras.io/).
-
-The `TextProcessor` class in this library uses the
-[`fastai`](https://docs.fast.ai/text.transform.html#BaseTokenizer.tokenizer)'s
-`Tokenizer` and `Vocab`. The code at `utils.fastai_transforms` is a minor
-adaptation of their code so it functions within this library. To my experience
-their `Tokenizer` is the best in class.
-
-The `ImageProcessor` class in this library uses code from the fantastic [Deep
-Learning for Computer
-Vision](https://www.pyimagesearch.com/deep-learning-computer-vision-python-book/)
-(DL4CV) book by Adrian Rosebrock.
-
-### License
-
-This work is dual-licensed under Apache 2.0 and MIT (or any later version).
-You can choose between one of them if you use this work.
-
-`SPDX-License-Identifier: Apache-2.0 AND MIT`
-
-### Cite
-
-#### BibTex
-
-```
-@article{Zaurin_pytorch-widedeep_A_flexible_2023,
-author = {Zaurin, Javier Rodriguez and Mulinka, Pavol},
-doi = {10.21105/joss.05027},
-journal = {Journal of Open Source Software},
-month = jun,
-number = {86},
-pages = {5027},
-title = {{pytorch-widedeep: A flexible package for multimodal deep learning}},
-url = {https://joss.theoj.org/papers/10.21105/joss.05027},
-volume = {8},
-year = {2023}
-}
-```
-
-#### APA
-
-```
-Zaurin, J. R., & Mulinka, P. (2023). pytorch-widedeep: A flexible package for
-multimodal deep learning. Journal of Open Source Software, 8(86), 5027.
-https://doi.org/10.21105/joss.05027
-```
diff --git a/mkdocs/site/installation.html b/mkdocs/site/installation.html
index 9394411d..f01e814c 100644
--- a/mkdocs/site/installation.html
+++ b/mkdocs/site/installation.html
@@ -18,7 +18,7 @@
       
       
       <link rel="icon" href="assets/images/favicon.ico">
-      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.35">
+      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.43">
     
     
       
@@ -26,7 +26,7 @@
       
     
     
-      <link rel="stylesheet" href="assets/stylesheets/main.35f28582.min.css">
+      <link rel="stylesheet" href="assets/stylesheets/main.0253249f.min.css">
       
         
         <link rel="stylesheet" href="assets/stylesheets/palette.06af60db.min.css">
@@ -1717,7 +1717,7 @@ <h2 id="dependencies">Dependencies<a class="headerlink" href="#dependencies" tit
     <script id="__config" type="application/json">{"base": ".", "features": ["navigation.tabs", "navigation.tabs.sticky", "navigation.indexes", "navigation.expand", "toc.integrate"], "search": "assets/javascripts/workers/search.6ce7567c.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version": "Select version"}}</script>
     
     
-      <script src="assets/javascripts/bundle.56dfad97.min.js"></script>
+      <script src="assets/javascripts/bundle.83f73b43.min.js"></script>
       
         <script src="stylesheets/extra.js"></script>
       
diff --git a/mkdocs/site/installation.md b/mkdocs/site/installation.md
deleted file mode 100644
index 2de21b28..00000000
--- a/mkdocs/site/installation.md
+++ /dev/null
@@ -1,48 +0,0 @@
-# Installation
-
-This section explains how to install ``pytorch-widedeep``.
-
-For the latest stable release, execute:
-
-```bash
-pip install pytorch-widedeep
-```
-
-For the bleeding-edge version, execute:
-
-```bash
-pip install git+https://github.com/jrzaurin/pytorch-widedeep.git
-```
-
-For developer install
-
-```bash
-# Clone the repository
-git clone https://github.com/jrzaurin/pytorch-widedeep
-cd pytorch-widedeep
-
-# Install in dev mode
-pip install -e .
-```
-
-## Dependencies
-
-* pandas>=1.3.5
-* numpy>=1.21.6
-* scipy>=1.7.3,<=1.12.0
-* scikit-learn>=1.0.2
-* gensim
-* spacy
-* opencv-contrib-python
-* imutils
-* tqdm
-* torch >= 2.0.0
-* torchvision >= 0.15.0
-* einops
-* wrapt
-* torchmetrics
-* pyarrow
-* fastparquet>=0.8.1
-* transformers
-* sentence-transformers
-* sentencepiece
\ No newline at end of file
diff --git a/mkdocs/site/pytorch-widedeep/bayesian_models.html b/mkdocs/site/pytorch-widedeep/bayesian_models.html
index 9b055c9c..3029b146 100644
--- a/mkdocs/site/pytorch-widedeep/bayesian_models.html
+++ b/mkdocs/site/pytorch-widedeep/bayesian_models.html
@@ -18,7 +18,7 @@
       
       
       <link rel="icon" href="../assets/images/favicon.ico">
-      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.35">
+      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.43">
     
     
       
@@ -26,7 +26,7 @@
       
     
     
-      <link rel="stylesheet" href="../assets/stylesheets/main.35f28582.min.css">
+      <link rel="stylesheet" href="../assets/stylesheets/main.0253249f.min.css">
       
         
         <link rel="stylesheet" href="../assets/stylesheets/palette.06af60db.min.css">
@@ -1643,7 +1643,9 @@ <h2 id="pytorch_widedeep.bayesian_models.tabular.bayesian_linear.bayesian_wide.B
       </thead>
       <tbody>
           <tr class="doc-section-item">
-            <td><code>input_dim</code></td>
+            <td>
+                <code>input_dim</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -1660,7 +1662,9 @@ <h2 id="pytorch_widedeep.bayesian_models.tabular.bayesian_linear.bayesian_wide.B
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>pred_dim</code></td>
+            <td>
+                <code>pred_dim</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -1674,7 +1678,9 @@ <h2 id="pytorch_widedeep.bayesian_models.tabular.bayesian_linear.bayesian_wide.B
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>prior_sigma_1</code></td>
+            <td>
+                <code>prior_sigma_1</code>
+            </td>
             <td>
                   <code>float</code>
             </td>
@@ -1697,7 +1703,9 @@ <h2 id="pytorch_widedeep.bayesian_models.tabular.bayesian_linear.bayesian_wide.B
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>prior_sigma_2</code></td>
+            <td>
+                <code>prior_sigma_2</code>
+            </td>
             <td>
                   <code>float</code>
             </td>
@@ -1713,7 +1721,9 @@ <h2 id="pytorch_widedeep.bayesian_models.tabular.bayesian_linear.bayesian_wide.B
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>prior_pi</code></td>
+            <td>
+                <code>prior_pi</code>
+            </td>
             <td>
                   <code>float</code>
             </td>
@@ -1728,7 +1738,9 @@ <h2 id="pytorch_widedeep.bayesian_models.tabular.bayesian_linear.bayesian_wide.B
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>posterior_mu_init</code></td>
+            <td>
+                <code>posterior_mu_init</code>
+            </td>
             <td>
                   <code>float</code>
             </td>
@@ -1756,7 +1768,9 @@ <h2 id="pytorch_widedeep.bayesian_models.tabular.bayesian_linear.bayesian_wide.B
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>posterior_rho_init</code></td>
+            <td>
+                <code>posterior_rho_init</code>
+            </td>
             <td>
                   <code>float</code>
             </td>
@@ -1808,6 +1822,11 @@ <h2 id="pytorch_widedeep.bayesian_models.tabular.bayesian_linear.bayesian_wide.B
 <span class="gp">&gt;&gt;&gt; </span><span class="n">out</span> <span class="o">=</span> <span class="n">wide</span><span class="p">(</span><span class="n">X</span><span class="p">)</span>
 </code></pre></div>
 
+
+
+
+
+
               <details class="quote">
                 <summary>Source code in <code>pytorch_widedeep/bayesian_models/tabular/bayesian_linear/bayesian_wide.py</code></summary>
                 <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">  9</span>
@@ -2064,7 +2083,9 @@ <h2 id="pytorch_widedeep.bayesian_models.tabular.bayesian_mlp.bayesian_tab_mlp.B
       </thead>
       <tbody>
           <tr class="doc-section-item">
-            <td><code>column_idx</code></td>
+            <td>
+                <code>column_idx</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, int]</code>
             </td>
@@ -2080,7 +2101,9 @@ <h2 id="pytorch_widedeep.bayesian_models.tabular.bayesian_mlp.bayesian_tab_mlp.B
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>cat_embed_input</code></td>
+            <td>
+                <code>cat_embed_input</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.wdtypes.Tuple">Tuple</span>[str, int, int]]]</code>
             </td>
@@ -2095,7 +2118,9 @@ <h2 id="pytorch_widedeep.bayesian_models.tabular.bayesian_mlp.bayesian_tab_mlp.B
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>cat_embed_activation</code></td>
+            <td>
+                <code>cat_embed_activation</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>
             </td>
@@ -2110,7 +2135,9 @@ <h2 id="pytorch_widedeep.bayesian_models.tabular.bayesian_mlp.bayesian_tab_mlp.B
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>continuous_cols</code></td>
+            <td>
+                <code>continuous_cols</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[str]]</code>
             </td>
@@ -2124,7 +2151,9 @@ <h2 id="pytorch_widedeep.bayesian_models.tabular.bayesian_mlp.bayesian_tab_mlp.B
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>cont_norm_layer</code></td>
+            <td>
+                <code>cont_norm_layer</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[batchnorm, layernorm]]</code>
             </td>
@@ -2139,7 +2168,9 @@ <h2 id="pytorch_widedeep.bayesian_models.tabular.bayesian_mlp.bayesian_tab_mlp.B
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>embed_continuous</code></td>
+            <td>
+                <code>embed_continuous</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
             </td>
@@ -2154,7 +2185,9 @@ <h2 id="pytorch_widedeep.bayesian_models.tabular.bayesian_mlp.bayesian_tab_mlp.B
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>cont_embed_dim</code></td>
+            <td>
+                <code>cont_embed_dim</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[int]</code>
             </td>
@@ -2168,7 +2201,9 @@ <h2 id="pytorch_widedeep.bayesian_models.tabular.bayesian_mlp.bayesian_tab_mlp.B
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>cont_embed_dropout</code></td>
+            <td>
+                <code>cont_embed_dropout</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>
             </td>
@@ -2182,7 +2217,9 @@ <h2 id="pytorch_widedeep.bayesian_models.tabular.bayesian_mlp.bayesian_tab_mlp.B
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>use_cont_bias</code></td>
+            <td>
+                <code>use_cont_bias</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
             </td>
@@ -2196,7 +2233,9 @@ <h2 id="pytorch_widedeep.bayesian_models.tabular.bayesian_mlp.bayesian_tab_mlp.B
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>cont_embed_activation</code></td>
+            <td>
+                <code>cont_embed_activation</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>
             </td>
@@ -2211,7 +2250,9 @@ <h2 id="pytorch_widedeep.bayesian_models.tabular.bayesian_mlp.bayesian_tab_mlp.B
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>mlp_hidden_dims</code></td>
+            <td>
+                <code>mlp_hidden_dims</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.List">List</span>[int]</code>
             </td>
@@ -2225,7 +2266,9 @@ <h2 id="pytorch_widedeep.bayesian_models.tabular.bayesian_mlp.bayesian_tab_mlp.B
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>mlp_activation</code></td>
+            <td>
+                <code>mlp_activation</code>
+            </td>
             <td>
                   <code>str</code>
             </td>
@@ -2240,7 +2283,9 @@ <h2 id="pytorch_widedeep.bayesian_models.tabular.bayesian_mlp.bayesian_tab_mlp.B
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>prior_sigma_1</code></td>
+            <td>
+                <code>prior_sigma_1</code>
+            </td>
             <td>
                   <code>float</code>
             </td>
@@ -2263,7 +2308,9 @@ <h2 id="pytorch_widedeep.bayesian_models.tabular.bayesian_mlp.bayesian_tab_mlp.B
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>prior_sigma_2</code></td>
+            <td>
+                <code>prior_sigma_2</code>
+            </td>
             <td>
                   <code>float</code>
             </td>
@@ -2279,7 +2326,9 @@ <h2 id="pytorch_widedeep.bayesian_models.tabular.bayesian_mlp.bayesian_tab_mlp.B
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>prior_pi</code></td>
+            <td>
+                <code>prior_pi</code>
+            </td>
             <td>
                   <code>float</code>
             </td>
@@ -2295,7 +2344,9 @@ <h2 id="pytorch_widedeep.bayesian_models.tabular.bayesian_mlp.bayesian_tab_mlp.B
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>posterior_mu_init</code></td>
+            <td>
+                <code>posterior_mu_init</code>
+            </td>
             <td>
                   <code>float</code>
             </td>
@@ -2323,7 +2374,9 @@ <h2 id="pytorch_widedeep.bayesian_models.tabular.bayesian_mlp.bayesian_tab_mlp.B
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>posterior_rho_init</code></td>
+            <td>
+                <code>posterior_rho_init</code>
+            </td>
             <td>
                   <code>float</code>
             </td>
@@ -2391,6 +2444,11 @@ <h2 id="pytorch_widedeep.bayesian_models.tabular.bayesian_mlp.bayesian_tab_mlp.B
 <span class="gp">&gt;&gt;&gt; </span><span class="n">out</span> <span class="o">=</span> <span class="n">model</span><span class="p">(</span><span class="n">X_tab</span><span class="p">)</span>
 </code></pre></div>
 
+
+
+
+
+
               <details class="quote">
                 <summary>Source code in <code>pytorch_widedeep/bayesian_models/tabular/bayesian_mlp/bayesian_tab_mlp.py</code></summary>
                 <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 16</span>
@@ -3001,7 +3059,7 @@ <h2 id="pytorch_widedeep.bayesian_models.tabular.bayesian_mlp.bayesian_tab_mlp.B
     <script id="__config" type="application/json">{"base": "..", "features": ["navigation.tabs", "navigation.tabs.sticky", "navigation.indexes", "navigation.expand", "toc.integrate"], "search": "../assets/javascripts/workers/search.6ce7567c.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version": "Select version"}}</script>
     
     
-      <script src="../assets/javascripts/bundle.56dfad97.min.js"></script>
+      <script src="../assets/javascripts/bundle.83f73b43.min.js"></script>
       
         <script src="../stylesheets/extra.js"></script>
       
diff --git a/mkdocs/site/pytorch-widedeep/bayesian_models.md b/mkdocs/site/pytorch-widedeep/bayesian_models.md
deleted file mode 100644
index 63ea1837..00000000
--- a/mkdocs/site/pytorch-widedeep/bayesian_models.md
+++ /dev/null
@@ -1,14 +0,0 @@
-# The ``bayesian models`` module
-
-This module contains the two Bayesian Models available in this library, namely
-the bayesian version of the ``Wide`` and ``TabMlp`` models, referred as
-``BayesianWide`` and ``BayesianTabMlp``. These models are very useful in
-scenarios where getting a measure of uncertainty is important.
-
-The models in this module are based on the publication:
-[Weight Uncertainty in Neural Networks](https://arxiv.org/abs/1505.05424?context=cs).
-
-
-::: pytorch_widedeep.bayesian_models.tabular.bayesian_linear.bayesian_wide.BayesianWide
-
-::: pytorch_widedeep.bayesian_models.tabular.bayesian_mlp.bayesian_tab_mlp.BayesianTabMlp
diff --git a/mkdocs/site/pytorch-widedeep/bayesian_trainer.html b/mkdocs/site/pytorch-widedeep/bayesian_trainer.html
index aa29a595..9103419f 100644
--- a/mkdocs/site/pytorch-widedeep/bayesian_trainer.html
+++ b/mkdocs/site/pytorch-widedeep/bayesian_trainer.html
@@ -18,7 +18,7 @@
       
       
       <link rel="icon" href="../assets/images/favicon.ico">
-      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.35">
+      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.43">
     
     
       
@@ -26,7 +26,7 @@
       
     
     
-      <link rel="stylesheet" href="../assets/stylesheets/main.35f28582.min.css">
+      <link rel="stylesheet" href="../assets/stylesheets/main.0253249f.min.css">
       
         
         <link rel="stylesheet" href="../assets/stylesheets/palette.06af60db.min.css">
@@ -1630,7 +1630,9 @@ <h2 id="pytorch_widedeep.training.BayesianTrainer" class="doc doc-heading">
       </thead>
       <tbody>
           <tr class="doc-section-item">
-            <td><code>model</code></td>
+            <td>
+                <code>model</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.bayesian_models._base_bayesian_model.BaseBayesianModel">BaseBayesianModel</span></code>
             </td>
@@ -1645,7 +1647,9 @@ <h2 id="pytorch_widedeep.training.BayesianTrainer" class="doc doc-heading">
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>objective</code></td>
+            <td>
+                <code>objective</code>
+            </td>
             <td>
                   <code>str</code>
             </td>
@@ -1662,7 +1666,9 @@ <h2 id="pytorch_widedeep.training.BayesianTrainer" class="doc doc-heading">
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>custom_loss_function</code></td>
+            <td>
+                <code>custom_loss_function</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Module">Module</span>]</code>
             </td>
@@ -1679,7 +1685,9 @@ <h2 id="pytorch_widedeep.training.BayesianTrainer" class="doc doc-heading">
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>optimizer</code></td>
+            <td>
+                <code>optimizer</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Optimizer">Optimizer</span>]</code>
             </td>
@@ -1694,7 +1702,9 @@ <h2 id="pytorch_widedeep.training.BayesianTrainer" class="doc doc-heading">
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>lr_scheduler</code></td>
+            <td>
+                <code>lr_scheduler</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.LRScheduler">LRScheduler</span>]</code>
             </td>
@@ -1709,7 +1719,9 @@ <h2 id="pytorch_widedeep.training.BayesianTrainer" class="doc doc-heading">
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>callbacks</code></td>
+            <td>
+                <code>callbacks</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.callbacks.Callback">Callback</span>]]</code>
             </td>
@@ -1727,7 +1739,9 @@ <h2 id="pytorch_widedeep.training.BayesianTrainer" class="doc doc-heading">
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>metrics</code></td>
+            <td>
+                <code>metrics</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.metrics.Metric">Metric</span>], <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="torchmetrics.Metric">Metric</span>]]]</code>
             </td>
@@ -1752,7 +1766,9 @@ <h2 id="pytorch_widedeep.training.BayesianTrainer" class="doc doc-heading">
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>verbose</code></td>
+            <td>
+                <code>verbose</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -1766,7 +1782,9 @@ <h2 id="pytorch_widedeep.training.BayesianTrainer" class="doc doc-heading">
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>seed</code></td>
+            <td>
+                <code>seed</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -1855,6 +1873,11 @@ <h2 id="pytorch_widedeep.training.BayesianTrainer" class="doc doc-heading">
       </tbody>
     </table>
 
+
+
+
+
+
               <details class="quote">
                 <summary>Source code in <code>pytorch_widedeep/training/bayesian_trainer.py</code></summary>
                 <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 33</span>
@@ -2862,7 +2885,9 @@ <h3 id="pytorch_widedeep.training.BayesianTrainer.fit" class="doc doc-heading">
       </thead>
       <tbody>
           <tr class="doc-section-item">
-            <td><code>X_tab</code></td>
+            <td>
+                <code>X_tab</code>
+            </td>
             <td>
                   <code><span title="numpy.ndarray">ndarray</span></code>
             </td>
@@ -2876,7 +2901,9 @@ <h3 id="pytorch_widedeep.training.BayesianTrainer.fit" class="doc doc-heading">
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>target</code></td>
+            <td>
+                <code>target</code>
+            </td>
             <td>
                   <code><span title="numpy.ndarray">ndarray</span></code>
             </td>
@@ -2890,7 +2917,9 @@ <h3 id="pytorch_widedeep.training.BayesianTrainer.fit" class="doc doc-heading">
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>X_tab_val</code></td>
+            <td>
+                <code>X_tab_val</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="numpy.ndarray">ndarray</span>]</code>
             </td>
@@ -2904,7 +2933,9 @@ <h3 id="pytorch_widedeep.training.BayesianTrainer.fit" class="doc doc-heading">
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>target_val</code></td>
+            <td>
+                <code>target_val</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="numpy.ndarray">ndarray</span>]</code>
             </td>
@@ -2918,7 +2949,9 @@ <h3 id="pytorch_widedeep.training.BayesianTrainer.fit" class="doc doc-heading">
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>val_split</code></td>
+            <td>
+                <code>val_split</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>
             </td>
@@ -2933,7 +2966,9 @@ <h3 id="pytorch_widedeep.training.BayesianTrainer.fit" class="doc doc-heading">
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>n_epochs</code></td>
+            <td>
+                <code>n_epochs</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -2947,7 +2982,9 @@ <h3 id="pytorch_widedeep.training.BayesianTrainer.fit" class="doc doc-heading">
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>validation_freq</code></td>
+            <td>
+                <code>validation_freq</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -2961,7 +2998,9 @@ <h3 id="pytorch_widedeep.training.BayesianTrainer.fit" class="doc doc-heading">
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>batch_size</code></td>
+            <td>
+                <code>batch_size</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -2975,7 +3014,9 @@ <h3 id="pytorch_widedeep.training.BayesianTrainer.fit" class="doc doc-heading">
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>n_train_samples</code></td>
+            <td>
+                <code>n_train_samples</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -2990,7 +3031,9 @@ <h3 id="pytorch_widedeep.training.BayesianTrainer.fit" class="doc doc-heading">
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>n_val_samples</code></td>
+            <td>
+                <code>n_val_samples</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -3274,7 +3317,9 @@ <h3 id="pytorch_widedeep.training.BayesianTrainer.predict" class="doc doc-headin
       </thead>
       <tbody>
           <tr class="doc-section-item">
-            <td><code>X_tab</code></td>
+            <td>
+                <code>X_tab</code>
+            </td>
             <td>
                   <code><span title="numpy.ndarray">ndarray</span></code>
             </td>
@@ -3288,7 +3333,9 @@ <h3 id="pytorch_widedeep.training.BayesianTrainer.predict" class="doc doc-headin
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>n_samples</code></td>
+            <td>
+                <code>n_samples</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -3303,7 +3350,9 @@ <h3 id="pytorch_widedeep.training.BayesianTrainer.predict" class="doc doc-headin
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>return_samples</code></td>
+            <td>
+                <code>return_samples</code>
+            </td>
             <td>
                   <code>bool</code>
             </td>
@@ -3317,7 +3366,9 @@ <h3 id="pytorch_widedeep.training.BayesianTrainer.predict" class="doc doc-headin
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>batch_size</code></td>
+            <td>
+                <code>batch_size</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -3467,7 +3518,9 @@ <h3 id="pytorch_widedeep.training.BayesianTrainer.predict_proba" class="doc doc-
       </thead>
       <tbody>
           <tr class="doc-section-item">
-            <td><code>X_tab</code></td>
+            <td>
+                <code>X_tab</code>
+            </td>
             <td>
                   <code><span title="numpy.ndarray">ndarray</span></code>
             </td>
@@ -3481,7 +3534,9 @@ <h3 id="pytorch_widedeep.training.BayesianTrainer.predict_proba" class="doc doc-
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>n_samples</code></td>
+            <td>
+                <code>n_samples</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -3496,7 +3551,9 @@ <h3 id="pytorch_widedeep.training.BayesianTrainer.predict_proba" class="doc doc-
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>return_samples</code></td>
+            <td>
+                <code>return_samples</code>
+            </td>
             <td>
                   <code>bool</code>
             </td>
@@ -3510,7 +3567,9 @@ <h3 id="pytorch_widedeep.training.BayesianTrainer.predict_proba" class="doc doc-
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>batch_size</code></td>
+            <td>
+                <code>batch_size</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -3683,7 +3742,9 @@ <h3 id="pytorch_widedeep.training.BayesianTrainer.save" class="doc doc-heading">
       </thead>
       <tbody>
           <tr class="doc-section-item">
-            <td><code>path</code></td>
+            <td>
+                <code>path</code>
+            </td>
             <td>
                   <code>str</code>
             </td>
@@ -3698,7 +3759,9 @@ <h3 id="pytorch_widedeep.training.BayesianTrainer.save" class="doc doc-heading">
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>save_state_dict</code></td>
+            <td>
+                <code>save_state_dict</code>
+            </td>
             <td>
                   <code>bool</code>
             </td>
@@ -3713,7 +3776,9 @@ <h3 id="pytorch_widedeep.training.BayesianTrainer.save" class="doc doc-heading">
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>model_filename</code></td>
+            <td>
+                <code>model_filename</code>
+            </td>
             <td>
                   <code>str</code>
             </td>
@@ -3929,7 +3994,7 @@ <h3 id="pytorch_widedeep.training.BayesianTrainer.save" class="doc doc-heading">
     <script id="__config" type="application/json">{"base": "..", "features": ["navigation.tabs", "navigation.tabs.sticky", "navigation.indexes", "navigation.expand", "toc.integrate"], "search": "../assets/javascripts/workers/search.6ce7567c.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version": "Select version"}}</script>
     
     
-      <script src="../assets/javascripts/bundle.56dfad97.min.js"></script>
+      <script src="../assets/javascripts/bundle.83f73b43.min.js"></script>
       
         <script src="../stylesheets/extra.js"></script>
       
diff --git a/mkdocs/site/pytorch-widedeep/bayesian_trainer.md b/mkdocs/site/pytorch-widedeep/bayesian_trainer.md
deleted file mode 100644
index 6e74934d..00000000
--- a/mkdocs/site/pytorch-widedeep/bayesian_trainer.md
+++ /dev/null
@@ -1,11 +0,0 @@
-# Training Deep Learning Probabilistic Models
-
-
-::: pytorch_widedeep.training.BayesianTrainer
-	<!-- selection:
-		members:
-			- fit
-			- predict
-			- predict_uncertainty
-			- predict_proba
-			- get_embeddings -->
diff --git a/mkdocs/site/pytorch-widedeep/callbacks.html b/mkdocs/site/pytorch-widedeep/callbacks.html
index ed2cfa86..53a1b388 100644
--- a/mkdocs/site/pytorch-widedeep/callbacks.html
+++ b/mkdocs/site/pytorch-widedeep/callbacks.html
@@ -18,7 +18,7 @@
       
       
       <link rel="icon" href="../assets/images/favicon.ico">
-      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.35">
+      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.43">
     
     
       
@@ -26,7 +26,7 @@
       
     
     
-      <link rel="stylesheet" href="../assets/stylesheets/main.35f28582.min.css">
+      <link rel="stylesheet" href="../assets/stylesheets/main.0253249f.min.css">
       
         
         <link rel="stylesheet" href="../assets/stylesheets/palette.06af60db.min.css">
@@ -1653,7 +1653,9 @@ <h2 id="pytorch_widedeep.callbacks.LRHistory" class="doc doc-heading">
       </thead>
       <tbody>
           <tr class="doc-section-item">
-            <td><code>n_epochs</code></td>
+            <td>
+                <code>n_epochs</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -1683,6 +1685,11 @@ <h2 id="pytorch_widedeep.callbacks.LRHistory" class="doc doc-heading">
 <span class="gp">&gt;&gt;&gt; </span><span class="n">trainer</span> <span class="o">=</span> <span class="n">Trainer</span><span class="p">(</span><span class="n">model</span><span class="p">,</span> <span class="n">objective</span><span class="o">=</span><span class="s2">&quot;regression&quot;</span><span class="p">,</span> <span class="n">callbacks</span><span class="o">=</span><span class="p">[</span><span class="n">LRHistory</span><span class="p">(</span><span class="n">n_epochs</span><span class="o">=</span><span class="mi">10</span><span class="p">)])</span>
 </code></pre></div>
 
+
+
+
+
+
               <details class="quote">
                 <summary>Source code in <code>pytorch_widedeep/callbacks.py</code></summary>
                 <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">251</span>
@@ -1990,7 +1997,9 @@ <h2 id="pytorch_widedeep.callbacks.ModelCheckpoint" class="doc doc-heading">
       </thead>
       <tbody>
           <tr class="doc-section-item">
-            <td><code>filepath</code></td>
+            <td>
+                <code>filepath</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>
             </td>
@@ -2009,7 +2018,9 @@ <h2 id="pytorch_widedeep.callbacks.ModelCheckpoint" class="doc doc-heading">
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>monitor</code></td>
+            <td>
+                <code>monitor</code>
+            </td>
             <td>
                   <code>str</code>
             </td>
@@ -2024,7 +2035,9 @@ <h2 id="pytorch_widedeep.callbacks.ModelCheckpoint" class="doc doc-heading">
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>min_delta</code></td>
+            <td>
+                <code>min_delta</code>
+            </td>
             <td>
                   <code>float</code>
             </td>
@@ -2040,7 +2053,9 @@ <h2 id="pytorch_widedeep.callbacks.ModelCheckpoint" class="doc doc-heading">
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>verbose</code></td>
+            <td>
+                <code>verbose</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -2054,7 +2069,9 @@ <h2 id="pytorch_widedeep.callbacks.ModelCheckpoint" class="doc doc-heading">
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>save_best_only</code></td>
+            <td>
+                <code>save_best_only</code>
+            </td>
             <td>
                   <code>bool</code>
             </td>
@@ -2069,7 +2086,9 @@ <h2 id="pytorch_widedeep.callbacks.ModelCheckpoint" class="doc doc-heading">
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>mode</code></td>
+            <td>
+                <code>mode</code>
+            </td>
             <td>
                   <code>str</code>
             </td>
@@ -2088,7 +2107,9 @@ <h2 id="pytorch_widedeep.callbacks.ModelCheckpoint" class="doc doc-heading">
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>period</code></td>
+            <td>
+                <code>period</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -2102,7 +2123,9 @@ <h2 id="pytorch_widedeep.callbacks.ModelCheckpoint" class="doc doc-heading">
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>max_save</code></td>
+            <td>
+                <code>max_save</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -2183,6 +2206,11 @@ <h2 id="pytorch_widedeep.callbacks.ModelCheckpoint" class="doc doc-heading">
 <span class="gp">&gt;&gt;&gt; </span><span class="n">trainer</span> <span class="o">=</span> <span class="n">Trainer</span><span class="p">(</span><span class="n">model</span><span class="p">,</span> <span class="n">objective</span><span class="o">=</span><span class="s2">&quot;regression&quot;</span><span class="p">,</span> <span class="n">callbacks</span><span class="o">=</span><span class="p">[</span><span class="n">ModelCheckpoint</span><span class="p">(</span><span class="n">filepath</span><span class="o">=</span><span class="s1">&#39;checkpoints/weights_out&#39;</span><span class="p">)])</span>
 </code></pre></div>
 
+
+
+
+
+
               <details class="quote">
                 <summary>Source code in <code>pytorch_widedeep/callbacks.py</code></summary>
                 <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">377</span>
@@ -2644,7 +2672,9 @@ <h2 id="pytorch_widedeep.callbacks.EarlyStopping" class="doc doc-heading">
       </thead>
       <tbody>
           <tr class="doc-section-item">
-            <td><code>monitor</code></td>
+            <td>
+                <code>monitor</code>
+            </td>
             <td>
                   <code>str</code>
             </td>
@@ -2659,7 +2689,9 @@ <h2 id="pytorch_widedeep.callbacks.EarlyStopping" class="doc doc-heading">
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>min_delta</code></td>
+            <td>
+                <code>min_delta</code>
+            </td>
             <td>
                   <code>float</code>
             </td>
@@ -2675,7 +2707,9 @@ <h2 id="pytorch_widedeep.callbacks.EarlyStopping" class="doc doc-heading">
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>patience</code></td>
+            <td>
+                <code>patience</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -2690,7 +2724,9 @@ <h2 id="pytorch_widedeep.callbacks.EarlyStopping" class="doc doc-heading">
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>verbose</code></td>
+            <td>
+                <code>verbose</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -2704,7 +2740,9 @@ <h2 id="pytorch_widedeep.callbacks.EarlyStopping" class="doc doc-heading">
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>mode</code></td>
+            <td>
+                <code>mode</code>
+            </td>
             <td>
                   <code>str</code>
             </td>
@@ -2722,7 +2760,9 @@ <h2 id="pytorch_widedeep.callbacks.EarlyStopping" class="doc doc-heading">
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>baseline</code></td>
+            <td>
+                <code>baseline</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>
             </td>
@@ -2737,7 +2777,9 @@ <h2 id="pytorch_widedeep.callbacks.EarlyStopping" class="doc doc-heading">
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>restore_best_weights</code></td>
+            <td>
+                <code>restore_best_weights</code>
+            </td>
             <td>
                   <code>bool</code>
             </td>
@@ -2805,6 +2847,11 @@ <h2 id="pytorch_widedeep.callbacks.EarlyStopping" class="doc doc-heading">
 <span class="gp">&gt;&gt;&gt; </span><span class="n">trainer</span> <span class="o">=</span> <span class="n">Trainer</span><span class="p">(</span><span class="n">model</span><span class="p">,</span> <span class="n">objective</span><span class="o">=</span><span class="s2">&quot;regression&quot;</span><span class="p">,</span> <span class="n">callbacks</span><span class="o">=</span><span class="p">[</span><span class="n">EarlyStopping</span><span class="p">(</span><span class="n">patience</span><span class="o">=</span><span class="mi">10</span><span class="p">)])</span>
 </code></pre></div>
 
+
+
+
+
+
               <details class="quote">
                 <summary>Source code in <code>pytorch_widedeep/callbacks.py</code></summary>
                 <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">580</span>
@@ -3241,7 +3288,7 @@ <h2 id="pytorch_widedeep.callbacks.EarlyStopping" class="doc doc-heading">
     <script id="__config" type="application/json">{"base": "..", "features": ["navigation.tabs", "navigation.tabs.sticky", "navigation.indexes", "navigation.expand", "toc.integrate"], "search": "../assets/javascripts/workers/search.6ce7567c.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version": "Select version"}}</script>
     
     
-      <script src="../assets/javascripts/bundle.56dfad97.min.js"></script>
+      <script src="../assets/javascripts/bundle.83f73b43.min.js"></script>
       
         <script src="../stylesheets/extra.js"></script>
       
diff --git a/mkdocs/site/pytorch-widedeep/callbacks.md b/mkdocs/site/pytorch-widedeep/callbacks.md
deleted file mode 100644
index 950423d6..00000000
--- a/mkdocs/site/pytorch-widedeep/callbacks.md
+++ /dev/null
@@ -1,14 +0,0 @@
-# Callbacks
-
-Here are the 4 callbacks available to the user in `pytorch-widedepp`:
-`LRHistory`, `ModelCheckpoint`, `EarlyStopping` and `RayTuneReporter`.
-
-:information_source:  **NOTE**: other callbacks , like `History`, run always
- by default. In particular, the `History` callback saves the metrics in the
- `history` attribute of the `Trainer`.
-
-::: pytorch_widedeep.callbacks.LRHistory
-
-::: pytorch_widedeep.callbacks.ModelCheckpoint
-
-::: pytorch_widedeep.callbacks.EarlyStopping
diff --git a/mkdocs/site/pytorch-widedeep/dataloaders.html b/mkdocs/site/pytorch-widedeep/dataloaders.html
index 357c490d..b8df0e5a 100644
--- a/mkdocs/site/pytorch-widedeep/dataloaders.html
+++ b/mkdocs/site/pytorch-widedeep/dataloaders.html
@@ -18,7 +18,7 @@
       
       
       <link rel="icon" href="../assets/images/favicon.ico">
-      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.35">
+      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.43">
     
     
       
@@ -26,7 +26,7 @@
       
     
     
-      <link rel="stylesheet" href="../assets/stylesheets/main.35f28582.min.css">
+      <link rel="stylesheet" href="../assets/stylesheets/main.0253249f.min.css">
       
         
         <link rel="stylesheet" href="../assets/stylesheets/palette.06af60db.min.css">
@@ -1634,7 +1634,9 @@ <h2 id="pytorch_widedeep.dataloaders.DataLoaderImbalanced" class="doc doc-headin
       </thead>
       <tbody>
           <tr class="doc-section-item">
-            <td><code>dataset</code></td>
+            <td>
+                <code>dataset</code>
+            </td>
             <td>
                   <code><span title="typing.Optional">Optional</span>[<span title="pytorch_widedeep.training._wd_dataset.WideDeepDataset">WideDeepDataset</span>]</code>
             </td>
@@ -1694,6 +1696,11 @@ <h2 id="pytorch_widedeep.dataloaders.DataLoaderImbalanced" class="doc doc-headin
       </tbody>
     </table>
 
+
+
+
+
+
               <details class="quote">
                 <summary>Source code in <code>pytorch_widedeep/dataloaders.py</code></summary>
                 <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 79</span>
@@ -1924,7 +1931,7 @@ <h2 id="pytorch_widedeep.dataloaders.DataLoaderImbalanced" class="doc doc-headin
     <script id="__config" type="application/json">{"base": "..", "features": ["navigation.tabs", "navigation.tabs.sticky", "navigation.indexes", "navigation.expand", "toc.integrate"], "search": "../assets/javascripts/workers/search.6ce7567c.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version": "Select version"}}</script>
     
     
-      <script src="../assets/javascripts/bundle.56dfad97.min.js"></script>
+      <script src="../assets/javascripts/bundle.83f73b43.min.js"></script>
       
         <script src="../stylesheets/extra.js"></script>
       
diff --git a/mkdocs/site/pytorch-widedeep/dataloaders.md b/mkdocs/site/pytorch-widedeep/dataloaders.md
deleted file mode 100644
index 43f82a9c..00000000
--- a/mkdocs/site/pytorch-widedeep/dataloaders.md
+++ /dev/null
@@ -1,9 +0,0 @@
-# Dataloaders
-
----
-:information_source: **NOTE**: This module should contain custom dataloaders
- that the user might want to implement. At the moment `pytorch-widedeep`
- offers one custom dataloader, `DataLoaderImbalanced`.
----
-
-::: pytorch_widedeep.dataloaders.DataLoaderImbalanced
diff --git a/mkdocs/site/pytorch-widedeep/losses.html b/mkdocs/site/pytorch-widedeep/losses.html
index 6147a17f..d6788a35 100644
--- a/mkdocs/site/pytorch-widedeep/losses.html
+++ b/mkdocs/site/pytorch-widedeep/losses.html
@@ -18,7 +18,7 @@
       
       
       <link rel="icon" href="../assets/images/favicon.ico">
-      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.35">
+      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.43">
     
     
       
@@ -26,7 +26,7 @@
       
     
     
-      <link rel="stylesheet" href="../assets/stylesheets/main.35f28582.min.css">
+      <link rel="stylesheet" href="../assets/stylesheets/main.0253249f.min.css">
       
         
         <link rel="stylesheet" href="../assets/stylesheets/palette.06af60db.min.css">
@@ -1802,6 +1802,11 @@ <h2 id="pytorch_widedeep.losses.MSELoss" class="doc doc-heading">
 
         <p>Mean square error loss</p>
 
+
+
+
+
+
               <details class="quote">
                 <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
                 <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">10</span>
@@ -1911,7 +1916,9 @@ <h3 id="pytorch_widedeep.losses.MSELoss.forward" class="doc doc-heading">
       </thead>
       <tbody>
           <tr class="doc-section-item">
-            <td><code>input</code></td>
+            <td>
+                <code>input</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>
             </td>
@@ -1925,7 +1932,9 @@ <h3 id="pytorch_widedeep.losses.MSELoss.forward" class="doc doc-heading">
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>target</code></td>
+            <td>
+                <code>target</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>
             </td>
@@ -2032,6 +2041,11 @@ <h2 id="pytorch_widedeep.losses.MSLELoss" class="doc doc-heading">
 
         <p>Mean square log error loss</p>
 
+
+
+
+
+
               <details class="quote">
                 <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
                 <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">45</span>
@@ -2149,7 +2163,9 @@ <h3 id="pytorch_widedeep.losses.MSLELoss.forward" class="doc doc-heading">
       </thead>
       <tbody>
           <tr class="doc-section-item">
-            <td><code>input</code></td>
+            <td>
+                <code>input</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>
             </td>
@@ -2163,7 +2179,9 @@ <h3 id="pytorch_widedeep.losses.MSLELoss.forward" class="doc doc-heading">
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>target</code></td>
+            <td>
+                <code>target</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>
             </td>
@@ -2284,6 +2302,11 @@ <h2 id="pytorch_widedeep.losses.RMSELoss" class="doc doc-heading">
 
         <p>Root mean square error loss</p>
 
+
+
+
+
+
               <details class="quote">
                 <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
                 <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 84</span>
@@ -2385,7 +2408,9 @@ <h3 id="pytorch_widedeep.losses.RMSELoss.forward" class="doc doc-heading">
       </thead>
       <tbody>
           <tr class="doc-section-item">
-            <td><code>input</code></td>
+            <td>
+                <code>input</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>
             </td>
@@ -2399,7 +2424,9 @@ <h3 id="pytorch_widedeep.losses.RMSELoss.forward" class="doc doc-heading">
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>target</code></td>
+            <td>
+                <code>target</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>
             </td>
@@ -2498,6 +2525,11 @@ <h2 id="pytorch_widedeep.losses.RMSLELoss" class="doc doc-heading">
 
         <p>Root mean square log error loss</p>
 
+
+
+
+
+
               <details class="quote">
                 <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
                 <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">115</span>
@@ -2607,7 +2639,9 @@ <h3 id="pytorch_widedeep.losses.RMSLELoss.forward" class="doc doc-heading">
       </thead>
       <tbody>
           <tr class="doc-section-item">
-            <td><code>input</code></td>
+            <td>
+                <code>input</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>
             </td>
@@ -2621,7 +2655,9 @@ <h3 id="pytorch_widedeep.losses.RMSLELoss.forward" class="doc doc-heading">
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>target</code></td>
+            <td>
+                <code>target</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>
             </td>
@@ -2752,7 +2788,9 @@ <h2 id="pytorch_widedeep.losses.QuantileLoss" class="doc doc-heading">
       </thead>
       <tbody>
           <tr class="doc-section-item">
-            <td><code>quantiles</code></td>
+            <td>
+                <code>quantiles</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.List">List</span>[float]</code>
             </td>
@@ -2768,6 +2806,11 @@ <h2 id="pytorch_widedeep.losses.QuantileLoss" class="doc doc-heading">
       </tbody>
     </table>
 
+
+
+
+
+
               <details class="quote">
                 <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
                 <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">150</span>
@@ -2929,7 +2972,9 @@ <h3 id="pytorch_widedeep.losses.QuantileLoss.forward" class="doc doc-heading">
       </thead>
       <tbody>
           <tr class="doc-section-item">
-            <td><code>input</code></td>
+            <td>
+                <code>input</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>
             </td>
@@ -2943,7 +2988,9 @@ <h3 id="pytorch_widedeep.losses.QuantileLoss.forward" class="doc doc-heading">
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>target</code></td>
+            <td>
+                <code>target</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>
             </td>
@@ -3098,7 +3145,9 @@ <h2 id="pytorch_widedeep.losses.FocalLoss" class="doc doc-heading">
       </thead>
       <tbody>
           <tr class="doc-section-item">
-            <td><code>alpha</code></td>
+            <td>
+                <code>alpha</code>
+            </td>
             <td>
                   <code>float</code>
             </td>
@@ -3112,7 +3161,9 @@ <h2 id="pytorch_widedeep.losses.FocalLoss" class="doc doc-heading">
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>gamma</code></td>
+            <td>
+                <code>gamma</code>
+            </td>
             <td>
                   <code>float</code>
             </td>
@@ -3128,6 +3179,11 @@ <h2 id="pytorch_widedeep.losses.FocalLoss" class="doc doc-heading">
       </tbody>
     </table>
 
+
+
+
+
+
               <details class="quote">
                 <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
                 <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">211</span>
@@ -3313,7 +3369,9 @@ <h3 id="pytorch_widedeep.losses.FocalLoss.forward" class="doc doc-heading">
       </thead>
       <tbody>
           <tr class="doc-section-item">
-            <td><code>input</code></td>
+            <td>
+                <code>input</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>
             </td>
@@ -3327,7 +3385,9 @@ <h3 id="pytorch_widedeep.losses.FocalLoss.forward" class="doc doc-heading">
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>target</code></td>
+            <td>
+                <code>target</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>
             </td>
@@ -3473,6 +3533,11 @@ <h2 id="pytorch_widedeep.losses.BayesianSELoss" class="doc doc-heading">
 the original publication
 <a href="https://arxiv.org/abs/1505.05424">Weight Uncertainty in Neural Networks</a>.</p>
 
+
+
+
+
+
               <details class="quote">
                 <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
                 <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">304</span>
@@ -3570,7 +3635,9 @@ <h3 id="pytorch_widedeep.losses.BayesianSELoss.forward" class="doc doc-heading">
       </thead>
       <tbody>
           <tr class="doc-section-item">
-            <td><code>input</code></td>
+            <td>
+                <code>input</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>
             </td>
@@ -3584,7 +3651,9 @@ <h3 id="pytorch_widedeep.losses.BayesianSELoss.forward" class="doc doc-heading">
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>target</code></td>
+            <td>
+                <code>target</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>
             </td>
@@ -3681,6 +3750,11 @@ <h2 id="pytorch_widedeep.losses.TweedieLoss" class="doc doc-heading">
 <a href="https://towardsdatascience.com/tweedie-loss-function-for-right-skewed-data-2c5ca470678f">this post</a>
 and the <a href="https://arxiv.org/abs/1811.10192">original publication</a> for details.</p>
 
+
+
+
+
+
               <details class="quote">
                 <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
                 <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">333</span>
@@ -3822,7 +3896,9 @@ <h3 id="pytorch_widedeep.losses.TweedieLoss.forward" class="doc doc-heading">
       </thead>
       <tbody>
           <tr class="doc-section-item">
-            <td><code>input</code></td>
+            <td>
+                <code>input</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>
             </td>
@@ -3836,7 +3912,9 @@ <h3 id="pytorch_widedeep.losses.TweedieLoss.forward" class="doc doc-heading">
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>target</code></td>
+            <td>
+                <code>target</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>
             </td>
@@ -3850,7 +3928,9 @@ <h3 id="pytorch_widedeep.losses.TweedieLoss.forward" class="doc doc-heading">
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>p</code></td>
+            <td>
+                <code>p</code>
+            </td>
             <td>
                   <code>float</code>
             </td>
@@ -3987,6 +4067,11 @@ <h2 id="pytorch_widedeep.losses.ZILNLoss" class="doc doc-heading">
 and the corresponding
 <a href="https://github.com/google/lifetime_value/blob/master/lifetime_value/zero_inflated_lognormal.py">code</a>.</p>
 
+
+
+
+
+
               <details class="quote">
                 <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
                 <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">384</span>
@@ -4190,7 +4275,9 @@ <h3 id="pytorch_widedeep.losses.ZILNLoss.forward" class="doc doc-heading">
       </thead>
       <tbody>
           <tr class="doc-section-item">
-            <td><code>input</code></td>
+            <td>
+                <code>input</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>
             </td>
@@ -4204,7 +4291,9 @@ <h3 id="pytorch_widedeep.losses.ZILNLoss.forward" class="doc doc-heading">
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>target</code></td>
+            <td>
+                <code>target</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>
             </td>
@@ -4363,6 +4452,11 @@ <h2 id="pytorch_widedeep.losses.L1Loss" class="doc doc-heading">
 
         <p>L1 loss</p>
 
+
+
+
+
+
               <details class="quote">
                 <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
                 <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">466</span>
@@ -4466,7 +4560,9 @@ <h3 id="pytorch_widedeep.losses.L1Loss.forward" class="doc doc-heading">
       </thead>
       <tbody>
           <tr class="doc-section-item">
-            <td><code>input</code></td>
+            <td>
+                <code>input</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>
             </td>
@@ -4480,7 +4576,9 @@ <h3 id="pytorch_widedeep.losses.L1Loss.forward" class="doc doc-heading">
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>target</code></td>
+            <td>
+                <code>target</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>
             </td>
@@ -4596,7 +4694,9 @@ <h2 id="pytorch_widedeep.losses.FocalR_L1Loss" class="doc doc-heading">
       </thead>
       <tbody>
           <tr class="doc-section-item">
-            <td><code>beta</code></td>
+            <td>
+                <code>beta</code>
+            </td>
             <td>
                   <code>float</code>
             </td>
@@ -4610,7 +4710,9 @@ <h2 id="pytorch_widedeep.losses.FocalR_L1Loss" class="doc doc-heading">
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>gamma</code></td>
+            <td>
+                <code>gamma</code>
+            </td>
             <td>
                   <code>float</code>
             </td>
@@ -4624,7 +4726,9 @@ <h2 id="pytorch_widedeep.losses.FocalR_L1Loss" class="doc doc-heading">
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>activation_fn</code></td>
+            <td>
+                <code>activation_fn</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[sigmoid, tanh]</code>
             </td>
@@ -4642,6 +4746,11 @@ <h2 id="pytorch_widedeep.losses.FocalR_L1Loss" class="doc doc-heading">
       </tbody>
     </table>
 
+
+
+
+
+
               <details class="quote">
                 <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
                 <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">498</span>
@@ -4811,7 +4920,9 @@ <h3 id="pytorch_widedeep.losses.FocalR_L1Loss.forward" class="doc doc-heading">
       </thead>
       <tbody>
           <tr class="doc-section-item">
-            <td><code>input</code></td>
+            <td>
+                <code>input</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>
             </td>
@@ -4825,7 +4936,9 @@ <h3 id="pytorch_widedeep.losses.FocalR_L1Loss.forward" class="doc doc-heading">
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>target</code></td>
+            <td>
+                <code>target</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>
             </td>
@@ -4969,7 +5082,9 @@ <h2 id="pytorch_widedeep.losses.FocalR_MSELoss" class="doc doc-heading">
       </thead>
       <tbody>
           <tr class="doc-section-item">
-            <td><code>beta</code></td>
+            <td>
+                <code>beta</code>
+            </td>
             <td>
                   <code>float</code>
             </td>
@@ -4983,7 +5098,9 @@ <h2 id="pytorch_widedeep.losses.FocalR_MSELoss" class="doc doc-heading">
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>gamma</code></td>
+            <td>
+                <code>gamma</code>
+            </td>
             <td>
                   <code>float</code>
             </td>
@@ -4997,7 +5114,9 @@ <h2 id="pytorch_widedeep.losses.FocalR_MSELoss" class="doc doc-heading">
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>activation_fn</code></td>
+            <td>
+                <code>activation_fn</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[sigmoid, tanh]</code>
             </td>
@@ -5015,6 +5134,11 @@ <h2 id="pytorch_widedeep.losses.FocalR_MSELoss" class="doc doc-heading">
       </tbody>
     </table>
 
+
+
+
+
+
               <details class="quote">
                 <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
                 <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">563</span>
@@ -5184,7 +5308,9 @@ <h3 id="pytorch_widedeep.losses.FocalR_MSELoss.forward" class="doc doc-heading">
       </thead>
       <tbody>
           <tr class="doc-section-item">
-            <td><code>input</code></td>
+            <td>
+                <code>input</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>
             </td>
@@ -5198,7 +5324,9 @@ <h3 id="pytorch_widedeep.losses.FocalR_MSELoss.forward" class="doc doc-heading">
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>target</code></td>
+            <td>
+                <code>target</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>
             </td>
@@ -5342,7 +5470,9 @@ <h2 id="pytorch_widedeep.losses.FocalR_RMSELoss" class="doc doc-heading">
       </thead>
       <tbody>
           <tr class="doc-section-item">
-            <td><code>beta</code></td>
+            <td>
+                <code>beta</code>
+            </td>
             <td>
                   <code>float</code>
             </td>
@@ -5356,7 +5486,9 @@ <h2 id="pytorch_widedeep.losses.FocalR_RMSELoss" class="doc doc-heading">
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>gamma</code></td>
+            <td>
+                <code>gamma</code>
+            </td>
             <td>
                   <code>float</code>
             </td>
@@ -5370,7 +5502,9 @@ <h2 id="pytorch_widedeep.losses.FocalR_RMSELoss" class="doc doc-heading">
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>activation_fn</code></td>
+            <td>
+                <code>activation_fn</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[sigmoid, tanh]</code>
             </td>
@@ -5388,6 +5522,11 @@ <h2 id="pytorch_widedeep.losses.FocalR_RMSELoss" class="doc doc-heading">
       </tbody>
     </table>
 
+
+
+
+
+
               <details class="quote">
                 <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
                 <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">628</span>
@@ -5557,7 +5696,9 @@ <h3 id="pytorch_widedeep.losses.FocalR_RMSELoss.forward" class="doc doc-heading"
       </thead>
       <tbody>
           <tr class="doc-section-item">
-            <td><code>input</code></td>
+            <td>
+                <code>input</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>
             </td>
@@ -5571,7 +5712,9 @@ <h3 id="pytorch_widedeep.losses.FocalR_RMSELoss.forward" class="doc doc-heading"
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>target</code></td>
+            <td>
+                <code>target</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>
             </td>
@@ -5702,6 +5845,11 @@ <h2 id="pytorch_widedeep.losses.HuberLoss" class="doc doc-heading">
         <p>Hubbler Loss</p>
 <p>Based on <a href="https://arxiv.org/abs/2102.09554">Delving into Deep Imbalanced Regression</a>.</p>
 
+
+
+
+
+
               <details class="quote">
                 <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
                 <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">693</span>
@@ -5823,7 +5971,9 @@ <h3 id="pytorch_widedeep.losses.HuberLoss.forward" class="doc doc-heading">
       </thead>
       <tbody>
           <tr class="doc-section-item">
-            <td><code>input</code></td>
+            <td>
+                <code>input</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>
             </td>
@@ -5837,7 +5987,9 @@ <h3 id="pytorch_widedeep.losses.HuberLoss.forward" class="doc doc-heading">
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>target</code></td>
+            <td>
+                <code>target</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>
             </td>
@@ -5976,7 +6128,9 @@ <h2 id="pytorch_widedeep.losses.InfoNCELoss" class="doc doc-heading">
       </thead>
       <tbody>
           <tr class="doc-section-item">
-            <td><code>temperature</code></td>
+            <td>
+                <code>temperature</code>
+            </td>
             <td>
                   <code>float</code>
             </td>
@@ -5990,7 +6144,9 @@ <h2 id="pytorch_widedeep.losses.InfoNCELoss" class="doc doc-heading">
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>reduction</code></td>
+            <td>
+                <code>reduction</code>
+            </td>
             <td>
                   <code>str</code>
             </td>
@@ -6006,6 +6162,11 @@ <h2 id="pytorch_widedeep.losses.InfoNCELoss" class="doc doc-heading">
       </tbody>
     </table>
 
+
+
+
+
+
               <details class="quote">
                 <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
                 <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">734</span>
@@ -6169,7 +6330,9 @@ <h3 id="pytorch_widedeep.losses.InfoNCELoss.forward" class="doc doc-heading">
       </thead>
       <tbody>
           <tr class="doc-section-item">
-            <td><code>g_projs</code></td>
+            <td>
+                <code>g_projs</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Tuple">Tuple</span>[<span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span>, <span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span>]</code>
             </td>
@@ -6313,7 +6476,9 @@ <h2 id="pytorch_widedeep.losses.DenoisingLoss" class="doc doc-heading">
       </thead>
       <tbody>
           <tr class="doc-section-item">
-            <td><code>lambda_cat</code></td>
+            <td>
+                <code>lambda_cat</code>
+            </td>
             <td>
                   <code>float</code>
             </td>
@@ -6328,7 +6493,9 @@ <h2 id="pytorch_widedeep.losses.DenoisingLoss" class="doc doc-heading">
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>lambda_cont</code></td>
+            <td>
+                <code>lambda_cont</code>
+            </td>
             <td>
                   <code>float</code>
             </td>
@@ -6343,7 +6510,9 @@ <h2 id="pytorch_widedeep.losses.DenoisingLoss" class="doc doc-heading">
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>reduction</code></td>
+            <td>
+                <code>reduction</code>
+            </td>
             <td>
                   <code>str</code>
             </td>
@@ -6359,6 +6528,11 @@ <h2 id="pytorch_widedeep.losses.DenoisingLoss" class="doc doc-heading">
       </tbody>
     </table>
 
+
+
+
+
+
               <details class="quote">
                 <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
                 <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">796</span>
@@ -6622,7 +6796,9 @@ <h3 id="pytorch_widedeep.losses.DenoisingLoss.forward" class="doc doc-heading">
       </thead>
       <tbody>
           <tr class="doc-section-item">
-            <td><code>x_cat_and_cat_</code></td>
+            <td>
+                <code>x_cat_and_cat_</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.wdtypes.Tuple">Tuple</span>[<span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span>, <span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span>]], <span title="pytorch_widedeep.wdtypes.Tuple">Tuple</span>[<span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span>, <span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span>]]]</code>
             </td>
@@ -6640,7 +6816,9 @@ <h3 id="pytorch_widedeep.losses.DenoisingLoss.forward" class="doc doc-heading">
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>x_cont_and_cont_</code></td>
+            <td>
+                <code>x_cont_and_cont_</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.wdtypes.Tuple">Tuple</span>[<span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span>, <span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span>]], <span title="pytorch_widedeep.wdtypes.Tuple">Tuple</span>[<span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span>, <span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span>]]]</code>
             </td>
@@ -6806,7 +6984,9 @@ <h2 id="pytorch_widedeep.losses.EncoderDecoderLoss" class="doc doc-heading">
       </thead>
       <tbody>
           <tr class="doc-section-item">
-            <td><code>eps</code></td>
+            <td>
+                <code>eps</code>
+            </td>
             <td>
                   <code>float</code>
             </td>
@@ -6822,6 +7002,11 @@ <h2 id="pytorch_widedeep.losses.EncoderDecoderLoss" class="doc doc-heading">
       </tbody>
     </table>
 
+
+
+
+
+
               <details class="quote">
                 <summary>Source code in <code>pytorch_widedeep/losses.py</code></summary>
                 <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">908</span>
@@ -6991,7 +7176,9 @@ <h3 id="pytorch_widedeep.losses.EncoderDecoderLoss.forward" class="doc doc-headi
       </thead>
       <tbody>
           <tr class="doc-section-item">
-            <td><code>x_true</code></td>
+            <td>
+                <code>x_true</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>
             </td>
@@ -7005,7 +7192,9 @@ <h3 id="pytorch_widedeep.losses.EncoderDecoderLoss.forward" class="doc doc-headi
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>x_pred</code></td>
+            <td>
+                <code>x_pred</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>
             </td>
@@ -7019,7 +7208,9 @@ <h3 id="pytorch_widedeep.losses.EncoderDecoderLoss.forward" class="doc doc-headi
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>mask</code></td>
+            <td>
+                <code>mask</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Tensor">Tensor</span></code>
             </td>
@@ -7175,7 +7366,9 @@ <h2 id="pytorch_widedeep.losses_multitarget.MultiTargetRegressionLoss" class="do
       </thead>
       <tbody>
           <tr class="doc-section-item">
-            <td><code>weights</code></td>
+            <td>
+                <code>weights</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[float]]</code>
             </td>
@@ -7191,7 +7384,9 @@ <h2 id="pytorch_widedeep.losses_multitarget.MultiTargetRegressionLoss" class="do
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>reduction</code></td>
+            <td>
+                <code>reduction</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[mean, sum]</code>
             </td>
@@ -7220,6 +7415,11 @@ <h2 id="pytorch_widedeep.losses_multitarget.MultiTargetRegressionLoss" class="do
 <span class="gp">&gt;&gt;&gt; </span><span class="n">output</span> <span class="o">=</span> <span class="n">loss</span><span class="p">(</span><span class="nb">input</span><span class="p">,</span> <span class="n">target</span><span class="p">)</span>
 </code></pre></div>
 
+
+
+
+
+
               <details class="quote">
                 <summary>Source code in <code>pytorch_widedeep/losses_multitarget.py</code></summary>
                 <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">11</span>
@@ -7400,7 +7600,9 @@ <h2 id="pytorch_widedeep.losses_multitarget.MultiTargetClassificationLoss" class
       </thead>
       <tbody>
           <tr class="doc-section-item">
-            <td><code>binary_config</code></td>
+            <td>
+                <code>binary_config</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[int, <span title="pytorch_widedeep.wdtypes.Tuple">Tuple</span>[int, float]]]]</code>
             </td>
@@ -7417,7 +7619,9 @@ <h2 id="pytorch_widedeep.losses_multitarget.MultiTargetClassificationLoss" class
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>multiclass_config</code></td>
+            <td>
+                <code>multiclass_config</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="pytorch_widedeep.wdtypes.Tuple">Tuple</span>[int, int], <span title="pytorch_widedeep.wdtypes.Tuple">Tuple</span>[int, int, <span title="pytorch_widedeep.wdtypes.List">List</span>[float]]]]]</code>
             </td>
@@ -7434,7 +7638,9 @@ <h2 id="pytorch_widedeep.losses_multitarget.MultiTargetClassificationLoss" class
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>weights</code></td>
+            <td>
+                <code>weights</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[float]]</code>
             </td>
@@ -7450,7 +7656,9 @@ <h2 id="pytorch_widedeep.losses_multitarget.MultiTargetClassificationLoss" class
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>reduction</code></td>
+            <td>
+                <code>reduction</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[mean, sum]</code>
             </td>
@@ -7468,7 +7676,9 @@ <h2 id="pytorch_widedeep.losses_multitarget.MultiTargetClassificationLoss" class
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>binary_trick</code></td>
+            <td>
+                <code>binary_trick</code>
+            </td>
             <td>
                   <code>bool</code>
             </td>
@@ -7513,6 +7723,11 @@ <h2 id="pytorch_widedeep.losses_multitarget.MultiTargetClassificationLoss" class
 <span class="gp">&gt;&gt;&gt; </span><span class="n">output_3</span> <span class="o">=</span> <span class="n">loss_3</span><span class="p">(</span><span class="n">input_binary_trick</span><span class="p">,</span> <span class="n">target</span><span class="p">)</span>
 </code></pre></div>
 
+
+
+
+
+
               <details class="quote">
                 <summary>Source code in <code>pytorch_widedeep/losses_multitarget.py</code></summary>
                 <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 74</span>
@@ -8007,7 +8222,9 @@ <h2 id="pytorch_widedeep.losses_multitarget.MutilTargetRegressionAndClassificati
       </thead>
       <tbody>
           <tr class="doc-section-item">
-            <td><code>regression_config</code></td>
+            <td>
+                <code>regression_config</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.List">List</span>[int]</code>
             </td>
@@ -8021,7 +8238,9 @@ <h2 id="pytorch_widedeep.losses_multitarget.MutilTargetRegressionAndClassificati
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>binary_config</code></td>
+            <td>
+                <code>binary_config</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[int, <span title="pytorch_widedeep.wdtypes.Tuple">Tuple</span>[int, float]]]]</code>
             </td>
@@ -8038,7 +8257,9 @@ <h2 id="pytorch_widedeep.losses_multitarget.MutilTargetRegressionAndClassificati
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>multiclass_config</code></td>
+            <td>
+                <code>multiclass_config</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="pytorch_widedeep.wdtypes.Tuple">Tuple</span>[int, int], <span title="pytorch_widedeep.wdtypes.Tuple">Tuple</span>[int, int, <span title="pytorch_widedeep.wdtypes.List">List</span>[float]]]]]</code>
             </td>
@@ -8055,7 +8276,9 @@ <h2 id="pytorch_widedeep.losses_multitarget.MutilTargetRegressionAndClassificati
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>weights</code></td>
+            <td>
+                <code>weights</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[float]]</code>
             </td>
@@ -8071,7 +8294,9 @@ <h2 id="pytorch_widedeep.losses_multitarget.MutilTargetRegressionAndClassificati
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>reduction</code></td>
+            <td>
+                <code>reduction</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[mean, sum]</code>
             </td>
@@ -8088,7 +8313,9 @@ <h2 id="pytorch_widedeep.losses_multitarget.MutilTargetRegressionAndClassificati
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>binary_trick</code></td>
+            <td>
+                <code>binary_trick</code>
+            </td>
             <td>
                   <code>bool</code>
             </td>
@@ -8128,6 +8355,11 @@ <h2 id="pytorch_widedeep.losses_multitarget.MutilTargetRegressionAndClassificati
 <span class="gp">&gt;&gt;&gt; </span><span class="n">output</span> <span class="o">=</span> <span class="n">loss</span><span class="p">(</span><span class="nb">input</span><span class="p">,</span> <span class="n">target</span><span class="p">)</span>
 </code></pre></div>
 
+
+
+
+
+
               <details class="quote">
                 <summary>Source code in <code>pytorch_widedeep/losses_multitarget.py</code></summary>
                 <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">294</span>
@@ -8736,7 +8968,7 @@ <h2 id="pytorch_widedeep.losses_multitarget.MutilTargetRegressionAndClassificati
     <script id="__config" type="application/json">{"base": "..", "features": ["navigation.tabs", "navigation.tabs.sticky", "navigation.indexes", "navigation.expand", "toc.integrate"], "search": "../assets/javascripts/workers/search.6ce7567c.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version": "Select version"}}</script>
     
     
-      <script src="../assets/javascripts/bundle.56dfad97.min.js"></script>
+      <script src="../assets/javascripts/bundle.83f73b43.min.js"></script>
       
         <script src="../stylesheets/extra.js"></script>
       
diff --git a/mkdocs/site/pytorch-widedeep/losses.md b/mkdocs/site/pytorch-widedeep/losses.md
deleted file mode 100644
index bd467e2f..00000000
--- a/mkdocs/site/pytorch-widedeep/losses.md
+++ /dev/null
@@ -1,66 +0,0 @@
-# Losses
-
-`pytorch-widedeep` accepts a number of losses and objectives that can be
-passed to the `Trainer` class via the parameter `objective`
-(see `pytorch-widedeep.training.Trainer`). For most cases the loss function
-that `pytorch-widedeep` will use internally is already implemented in
-Pytorch.
-
-In addition, `pytorch-widedeep` implements a series of  "custom" loss
-functions. These are described below for completion since, as mentioned
-before, they are used internally by the `Trainer`. Of course, onen could
-always use them on their own and can be imported as:
-
-``
-from pytorch_widedeep.losses import FocalLoss
-``
-
----
-:information_source: **NOTE**:  Losses in this module expect the predictions
- and ground truth to have the same dimensions for regression and binary
- classification problems $(N_{samples}, 1)$. In the case of multiclass
- classification problems the ground truth is expected to be a 1D tensor with
- the corresponding classes. See Examples below
----
-
-::: pytorch_widedeep.losses.MSELoss
-
-::: pytorch_widedeep.losses.MSLELoss
-
-::: pytorch_widedeep.losses.RMSELoss
-
-::: pytorch_widedeep.losses.RMSLELoss
-
-::: pytorch_widedeep.losses.QuantileLoss
-
-::: pytorch_widedeep.losses.FocalLoss
-
-::: pytorch_widedeep.losses.BayesianSELoss
-
-::: pytorch_widedeep.losses.TweedieLoss
-
-::: pytorch_widedeep.losses.ZILNLoss
-
-::: pytorch_widedeep.losses.L1Loss
-
-::: pytorch_widedeep.losses.FocalR_L1Loss
-
-::: pytorch_widedeep.losses.FocalR_MSELoss
-
-::: pytorch_widedeep.losses.FocalR_RMSELoss
-
-::: pytorch_widedeep.losses.HuberLoss
-
-::: pytorch_widedeep.losses.InfoNCELoss
-
-::: pytorch_widedeep.losses.DenoisingLoss
-
-::: pytorch_widedeep.losses.EncoderDecoderLoss
-
-::: pytorch_widedeep.losses_multitarget.MultiTargetRegressionLoss
-
-::: pytorch_widedeep.losses_multitarget.MultiTargetClassificationLoss
-
-::: pytorch_widedeep.losses_multitarget.MutilTargetRegressionAndClassificationLoss
-
-
diff --git a/mkdocs/site/pytorch-widedeep/metrics.html b/mkdocs/site/pytorch-widedeep/metrics.html
index ca85170a..3e1c9993 100644
--- a/mkdocs/site/pytorch-widedeep/metrics.html
+++ b/mkdocs/site/pytorch-widedeep/metrics.html
@@ -18,7 +18,7 @@
       
       
       <link rel="icon" href="../assets/images/favicon.ico">
-      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.35">
+      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.43">
     
     
       
@@ -26,7 +26,7 @@
       
     
     
-      <link rel="stylesheet" href="../assets/stylesheets/main.35f28582.min.css">
+      <link rel="stylesheet" href="../assets/stylesheets/main.0253249f.min.css">
       
         
         <link rel="stylesheet" href="../assets/stylesheets/palette.06af60db.min.css">
@@ -1696,7 +1696,9 @@ <h2 id="pytorch_widedeep.metrics.Accuracy" class="doc doc-heading">
       </thead>
       <tbody>
           <tr class="doc-section-item">
-            <td><code>top_k</code></td>
+            <td>
+                <code>top_k</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -1732,6 +1734,11 @@ <h2 id="pytorch_widedeep.metrics.Accuracy" class="doc doc-heading">
 <span class="go">array(0.66666667)</span>
 </code></pre></div>
 
+
+
+
+
+
               <details class="quote">
                 <summary>Source code in <code>pytorch_widedeep/metrics.py</code></summary>
                 <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 48</span>
@@ -1935,7 +1942,9 @@ <h2 id="pytorch_widedeep.metrics.Precision" class="doc doc-heading">
       </thead>
       <tbody>
           <tr class="doc-section-item">
-            <td><code>average</code></td>
+            <td>
+                <code>average</code>
+            </td>
             <td>
                   <code>bool</code>
             </td>
@@ -1971,6 +1980,11 @@ <h2 id="pytorch_widedeep.metrics.Precision" class="doc doc-heading">
 <span class="go">array(0.33333334)</span>
 </code></pre></div>
 
+
+
+
+
+
               <details class="quote">
                 <summary>Source code in <code>pytorch_widedeep/metrics.py</code></summary>
                 <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">107</span>
@@ -2188,7 +2202,9 @@ <h2 id="pytorch_widedeep.metrics.Recall" class="doc doc-heading">
       </thead>
       <tbody>
           <tr class="doc-section-item">
-            <td><code>average</code></td>
+            <td>
+                <code>average</code>
+            </td>
             <td>
                   <code>bool</code>
             </td>
@@ -2224,6 +2240,11 @@ <h2 id="pytorch_widedeep.metrics.Recall" class="doc doc-heading">
 <span class="go">array(0.33333334)</span>
 </code></pre></div>
 
+
+
+
+
+
               <details class="quote">
                 <summary>Source code in <code>pytorch_widedeep/metrics.py</code></summary>
                 <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">173</span>
@@ -2444,7 +2465,9 @@ <h2 id="pytorch_widedeep.metrics.FBetaScore" class="doc doc-heading">
       </thead>
       <tbody>
           <tr class="doc-section-item">
-            <td><code>beta</code></td>
+            <td>
+                <code>beta</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -2458,7 +2481,9 @@ <h2 id="pytorch_widedeep.metrics.FBetaScore" class="doc doc-heading">
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>average</code></td>
+            <td>
+                <code>average</code>
+            </td>
             <td>
                   <code>bool</code>
             </td>
@@ -2494,6 +2519,11 @@ <h2 id="pytorch_widedeep.metrics.FBetaScore" class="doc doc-heading">
 <span class="go">array(0.33333334)</span>
 </code></pre></div>
 
+
+
+
+
+
               <details class="quote">
                 <summary>Source code in <code>pytorch_widedeep/metrics.py</code></summary>
                 <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">239</span>
@@ -2707,7 +2737,9 @@ <h2 id="pytorch_widedeep.metrics.F1Score" class="doc doc-heading">
       </thead>
       <tbody>
           <tr class="doc-section-item">
-            <td><code>average</code></td>
+            <td>
+                <code>average</code>
+            </td>
             <td>
                   <code>bool</code>
             </td>
@@ -2743,6 +2775,11 @@ <h2 id="pytorch_widedeep.metrics.F1Score" class="doc doc-heading">
 <span class="go">array(0.33333334)</span>
 </code></pre></div>
 
+
+
+
+
+
               <details class="quote">
                 <summary>Source code in <code>pytorch_widedeep/metrics.py</code></summary>
                 <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">303</span>
@@ -2922,6 +2959,11 @@ <h2 id="pytorch_widedeep.metrics.R2Score" class="doc doc-heading">
 <span class="go">array(0.94860814)</span>
 </code></pre></div>
 
+
+
+
+
+
               <details class="quote">
                 <summary>Source code in <code>pytorch_widedeep/metrics.py</code></summary>
                 <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">348</span>
@@ -3174,7 +3216,7 @@ <h3 id="pytorch_widedeep.metrics.R2Score.reset" class="doc doc-heading">
     <script id="__config" type="application/json">{"base": "..", "features": ["navigation.tabs", "navigation.tabs.sticky", "navigation.indexes", "navigation.expand", "toc.integrate"], "search": "../assets/javascripts/workers/search.6ce7567c.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version": "Select version"}}</script>
     
     
-      <script src="../assets/javascripts/bundle.56dfad97.min.js"></script>
+      <script src="../assets/javascripts/bundle.83f73b43.min.js"></script>
       
         <script src="../stylesheets/extra.js"></script>
       
diff --git a/mkdocs/site/pytorch-widedeep/metrics.md b/mkdocs/site/pytorch-widedeep/metrics.md
deleted file mode 100644
index 02866314..00000000
--- a/mkdocs/site/pytorch-widedeep/metrics.md
+++ /dev/null
@@ -1,43 +0,0 @@
-# Metrics
-
----
-:information_source: **NOTE**: metrics in this module expect the predictions
- and ground truth to have the same dimensions for regression and binary
- classification problems: $(N_{samples}, 1)$. In the case of multiclass
- classification problems the ground truth is expected to be a 1D tensor with
- the corresponding classes. See Examples below
----
-
-We have added the possibility of using the metrics available at the
-[torchmetrics](https://torchmetrics.readthedocs.io/en/latest/) library. Note
-that this library is still in its early versions and therefore this option
-should be used with caution. To use ``torchmetrics`` simply import them and
-use them as any of the ``pytorch-widedeep`` metrics described below.
-
-```python
-from torchmetrics import Accuracy, Precision
-
-accuracy = Accuracy(average=None, num_classes=2)
-precision = Precision(average='micro', num_classes=2)
-
-trainer = Trainer(model, objective="binary", metrics=[accuracy, precision])
-```
-
-A functioning example for ``pytorch-widedeep`` using ``torchmetrics`` can be
-found in the [Examples folder](https://github.com/jrzaurin/pytorch-widedeep/blob/master/examples)
-
-:information_source: **NOTE**: the forward method for all metrics in this
- module takes two tensors, `y_pred` and `y_true` (in that order). Therefore,
- we do not include the method in the documentation.
-
-::: pytorch_widedeep.metrics.Accuracy
-
-::: pytorch_widedeep.metrics.Precision
-
-::: pytorch_widedeep.metrics.Recall
-
-::: pytorch_widedeep.metrics.FBetaScore
-
-::: pytorch_widedeep.metrics.F1Score
-
-::: pytorch_widedeep.metrics.R2Score
diff --git a/mkdocs/site/pytorch-widedeep/model_components.html b/mkdocs/site/pytorch-widedeep/model_components.html
index 4138c3dc..bec355bd 100644
--- a/mkdocs/site/pytorch-widedeep/model_components.html
+++ b/mkdocs/site/pytorch-widedeep/model_components.html
@@ -18,7 +18,7 @@
       
       
       <link rel="icon" href="../assets/images/favicon.ico">
-      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.35">
+      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.43">
     
     
       
@@ -26,7 +26,7 @@
       
     
     
-      <link rel="stylesheet" href="../assets/stylesheets/main.35f28582.min.css">
+      <link rel="stylesheet" href="../assets/stylesheets/main.0253249f.min.css">
       
         
         <link rel="stylesheet" href="../assets/stylesheets/palette.06af60db.min.css">
@@ -1815,7 +1815,9 @@ <h2 id="pytorch_widedeep.models.tabular.linear.wide.Wide" class="doc doc-heading
       </thead>
       <tbody>
           <tr class="doc-section-item">
-            <td><code>input_dim</code></td>
+            <td>
+                <code>input_dim</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -1833,7 +1835,9 @@ <h2 id="pytorch_widedeep.models.tabular.linear.wide.Wide" class="doc doc-heading
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>pred_dim</code></td>
+            <td>
+                <code>pred_dim</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -1886,6 +1890,11 @@ <h2 id="pytorch_widedeep.models.tabular.linear.wide.Wide" class="doc doc-heading
 <span class="gp">&gt;&gt;&gt; </span><span class="n">out</span> <span class="o">=</span> <span class="n">wide</span><span class="p">(</span><span class="n">X</span><span class="p">)</span>
 </code></pre></div>
 
+
+
+
+
+
               <details class="quote">
                 <summary>Source code in <code>pytorch_widedeep/models/tabular/linear/wide.py</code></summary>
                 <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">10</span>
@@ -2101,7 +2110,9 @@ <h2 id="pytorch_widedeep.models.tabular.mlp.tab_mlp.TabMlp" class="doc doc-headi
       </thead>
       <tbody>
           <tr class="doc-section-item">
-            <td><code>column_idx</code></td>
+            <td>
+                <code>column_idx</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, int]</code>
             </td>
@@ -2117,7 +2128,9 @@ <h2 id="pytorch_widedeep.models.tabular.mlp.tab_mlp.TabMlp" class="doc doc-headi
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>cat_embed_input</code></td>
+            <td>
+                <code>cat_embed_input</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.wdtypes.Tuple">Tuple</span>[str, int, int]]]</code>
             </td>
@@ -2132,7 +2145,9 @@ <h2 id="pytorch_widedeep.models.tabular.mlp.tab_mlp.TabMlp" class="doc doc-headi
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>cat_embed_dropout</code></td>
+            <td>
+                <code>cat_embed_dropout</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>
             </td>
@@ -2147,7 +2162,9 @@ <h2 id="pytorch_widedeep.models.tabular.mlp.tab_mlp.TabMlp" class="doc doc-headi
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>use_cat_bias</code></td>
+            <td>
+                <code>use_cat_bias</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
             </td>
@@ -2162,7 +2179,9 @@ <h2 id="pytorch_widedeep.models.tabular.mlp.tab_mlp.TabMlp" class="doc doc-headi
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>cat_embed_activation</code></td>
+            <td>
+                <code>cat_embed_activation</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>
             </td>
@@ -2177,7 +2196,9 @@ <h2 id="pytorch_widedeep.models.tabular.mlp.tab_mlp.TabMlp" class="doc doc-headi
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>continuous_cols</code></td>
+            <td>
+                <code>continuous_cols</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[str]]</code>
             </td>
@@ -2191,7 +2212,9 @@ <h2 id="pytorch_widedeep.models.tabular.mlp.tab_mlp.TabMlp" class="doc doc-headi
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>cont_norm_layer</code></td>
+            <td>
+                <code>cont_norm_layer</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[batchnorm, layernorm]]</code>
             </td>
@@ -2207,7 +2230,9 @@ <h2 id="pytorch_widedeep.models.tabular.mlp.tab_mlp.TabMlp" class="doc doc-headi
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>embed_continuous</code></td>
+            <td>
+                <code>embed_continuous</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
             </td>
@@ -2226,7 +2251,9 @@ <h2 id="pytorch_widedeep.models.tabular.mlp.tab_mlp.TabMlp" class="doc doc-headi
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>embed_continuous_method</code></td>
+            <td>
+                <code>embed_continuous_method</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[standard, piecewise, periodic]]</code>
             </td>
@@ -2247,7 +2274,9 @@ <h2 id="pytorch_widedeep.models.tabular.mlp.tab_mlp.TabMlp" class="doc doc-headi
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>cont_embed_dim</code></td>
+            <td>
+                <code>cont_embed_dim</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[int]</code>
             </td>
@@ -2262,7 +2291,9 @@ <h2 id="pytorch_widedeep.models.tabular.mlp.tab_mlp.TabMlp" class="doc doc-headi
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>cont_embed_dropout</code></td>
+            <td>
+                <code>cont_embed_dropout</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>
             </td>
@@ -2276,7 +2307,9 @@ <h2 id="pytorch_widedeep.models.tabular.mlp.tab_mlp.TabMlp" class="doc doc-headi
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>cont_embed_activation</code></td>
+            <td>
+                <code>cont_embed_activation</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>
             </td>
@@ -2292,7 +2325,9 @@ <h2 id="pytorch_widedeep.models.tabular.mlp.tab_mlp.TabMlp" class="doc doc-headi
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>quantization_setup</code></td>
+            <td>
+                <code>quantization_setup</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, <span title="pytorch_widedeep.wdtypes.List">List</span>[float]]]</code>
             </td>
@@ -2310,7 +2345,9 @@ <h2 id="pytorch_widedeep.models.tabular.mlp.tab_mlp.TabMlp" class="doc doc-headi
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>n_frequencies</code></td>
+            <td>
+                <code>n_frequencies</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[int]</code>
             </td>
@@ -2328,7 +2365,9 @@ <h2 id="pytorch_widedeep.models.tabular.mlp.tab_mlp.TabMlp" class="doc doc-headi
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>sigma</code></td>
+            <td>
+                <code>sigma</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>
             </td>
@@ -2345,7 +2384,9 @@ <h2 id="pytorch_widedeep.models.tabular.mlp.tab_mlp.TabMlp" class="doc doc-headi
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>share_last_layer</code></td>
+            <td>
+                <code>share_last_layer</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
             </td>
@@ -2364,7 +2405,9 @@ <h2 id="pytorch_widedeep.models.tabular.mlp.tab_mlp.TabMlp" class="doc doc-headi
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>full_embed_dropout</code></td>
+            <td>
+                <code>full_embed_dropout</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
             </td>
@@ -2379,7 +2422,9 @@ <h2 id="pytorch_widedeep.models.tabular.mlp.tab_mlp.TabMlp" class="doc doc-headi
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>mlp_hidden_dims</code></td>
+            <td>
+                <code>mlp_hidden_dims</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.List">List</span>[int]</code>
             </td>
@@ -2393,7 +2438,9 @@ <h2 id="pytorch_widedeep.models.tabular.mlp.tab_mlp.TabMlp" class="doc doc-headi
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>mlp_activation</code></td>
+            <td>
+                <code>mlp_activation</code>
+            </td>
             <td>
                   <code>str</code>
             </td>
@@ -2408,7 +2455,9 @@ <h2 id="pytorch_widedeep.models.tabular.mlp.tab_mlp.TabMlp" class="doc doc-headi
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>mlp_dropout</code></td>
+            <td>
+                <code>mlp_dropout</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Union">Union</span>[float, <span title="pytorch_widedeep.wdtypes.List">List</span>[float]]</code>
             </td>
@@ -2423,7 +2472,9 @@ <h2 id="pytorch_widedeep.models.tabular.mlp.tab_mlp.TabMlp" class="doc doc-headi
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>mlp_batchnorm</code></td>
+            <td>
+                <code>mlp_batchnorm</code>
+            </td>
             <td>
                   <code>bool</code>
             </td>
@@ -2438,7 +2489,9 @@ <h2 id="pytorch_widedeep.models.tabular.mlp.tab_mlp.TabMlp" class="doc doc-headi
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>mlp_batchnorm_last</code></td>
+            <td>
+                <code>mlp_batchnorm_last</code>
+            </td>
             <td>
                   <code>bool</code>
             </td>
@@ -2453,7 +2506,9 @@ <h2 id="pytorch_widedeep.models.tabular.mlp.tab_mlp.TabMlp" class="doc doc-headi
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>mlp_linear_first</code></td>
+            <td>
+                <code>mlp_linear_first</code>
+            </td>
             <td>
                   <code>bool</code>
             </td>
@@ -2510,6 +2565,11 @@ <h2 id="pytorch_widedeep.models.tabular.mlp.tab_mlp.TabMlp" class="doc doc-headi
 <span class="gp">&gt;&gt;&gt; </span><span class="n">out</span> <span class="o">=</span> <span class="n">model</span><span class="p">(</span><span class="n">X_tab</span><span class="p">)</span>
 </code></pre></div>
 
+
+
+
+
+
               <details class="quote">
                 <summary>Source code in <code>pytorch_widedeep/models/tabular/mlp/tab_mlp.py</code></summary>
                 <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 10</span>
@@ -3002,7 +3062,9 @@ <h2 id="pytorch_widedeep.models.tabular.mlp.tab_mlp.TabMlpDecoder" class="doc do
       </thead>
       <tbody>
           <tr class="doc-section-item">
-            <td><code>embed_dim</code></td>
+            <td>
+                <code>embed_dim</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -3016,7 +3078,9 @@ <h2 id="pytorch_widedeep.models.tabular.mlp.tab_mlp.TabMlpDecoder" class="doc do
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>mlp_hidden_dims</code></td>
+            <td>
+                <code>mlp_hidden_dims</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.List">List</span>[int]</code>
             </td>
@@ -3030,7 +3094,9 @@ <h2 id="pytorch_widedeep.models.tabular.mlp.tab_mlp.TabMlpDecoder" class="doc do
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>mlp_activation</code></td>
+            <td>
+                <code>mlp_activation</code>
+            </td>
             <td>
                   <code>str</code>
             </td>
@@ -3045,7 +3111,9 @@ <h2 id="pytorch_widedeep.models.tabular.mlp.tab_mlp.TabMlpDecoder" class="doc do
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>mlp_dropout</code></td>
+            <td>
+                <code>mlp_dropout</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Union">Union</span>[float, <span title="pytorch_widedeep.wdtypes.List">List</span>[float]]</code>
             </td>
@@ -3060,7 +3128,9 @@ <h2 id="pytorch_widedeep.models.tabular.mlp.tab_mlp.TabMlpDecoder" class="doc do
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>mlp_batchnorm</code></td>
+            <td>
+                <code>mlp_batchnorm</code>
+            </td>
             <td>
                   <code>bool</code>
             </td>
@@ -3075,7 +3145,9 @@ <h2 id="pytorch_widedeep.models.tabular.mlp.tab_mlp.TabMlpDecoder" class="doc do
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>mlp_batchnorm_last</code></td>
+            <td>
+                <code>mlp_batchnorm_last</code>
+            </td>
             <td>
                   <code>bool</code>
             </td>
@@ -3090,7 +3162,9 @@ <h2 id="pytorch_widedeep.models.tabular.mlp.tab_mlp.TabMlpDecoder" class="doc do
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>mlp_linear_first</code></td>
+            <td>
+                <code>mlp_linear_first</code>
+            </td>
             <td>
                   <code>bool</code>
             </td>
@@ -3144,6 +3218,11 @@ <h2 id="pytorch_widedeep.models.tabular.mlp.tab_mlp.TabMlpDecoder" class="doc do
 <span class="go">torch.Size([3, 32])</span>
 </code></pre></div>
 
+
+
+
+
+
               <details class="quote">
                 <summary>Source code in <code>pytorch_widedeep/models/tabular/mlp/tab_mlp.py</code></summary>
                 <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">223</span>
@@ -3368,7 +3447,9 @@ <h2 id="pytorch_widedeep.models.tabular.resnet.tab_resnet.TabResnet" class="doc
       </thead>
       <tbody>
           <tr class="doc-section-item">
-            <td><code>column_idx</code></td>
+            <td>
+                <code>column_idx</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, int]</code>
             </td>
@@ -3384,7 +3465,9 @@ <h2 id="pytorch_widedeep.models.tabular.resnet.tab_resnet.TabResnet" class="doc
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>cat_embed_input</code></td>
+            <td>
+                <code>cat_embed_input</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.wdtypes.Tuple">Tuple</span>[str, int, int]]]</code>
             </td>
@@ -3399,7 +3482,9 @@ <h2 id="pytorch_widedeep.models.tabular.resnet.tab_resnet.TabResnet" class="doc
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>cat_embed_dropout</code></td>
+            <td>
+                <code>cat_embed_dropout</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>
             </td>
@@ -3414,7 +3499,9 @@ <h2 id="pytorch_widedeep.models.tabular.resnet.tab_resnet.TabResnet" class="doc
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>use_cat_bias</code></td>
+            <td>
+                <code>use_cat_bias</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
             </td>
@@ -3429,7 +3516,9 @@ <h2 id="pytorch_widedeep.models.tabular.resnet.tab_resnet.TabResnet" class="doc
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>cat_embed_activation</code></td>
+            <td>
+                <code>cat_embed_activation</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>
             </td>
@@ -3444,7 +3533,9 @@ <h2 id="pytorch_widedeep.models.tabular.resnet.tab_resnet.TabResnet" class="doc
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>continuous_cols</code></td>
+            <td>
+                <code>continuous_cols</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[str]]</code>
             </td>
@@ -3458,7 +3549,9 @@ <h2 id="pytorch_widedeep.models.tabular.resnet.tab_resnet.TabResnet" class="doc
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>cont_norm_layer</code></td>
+            <td>
+                <code>cont_norm_layer</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[batchnorm, layernorm]]</code>
             </td>
@@ -3474,7 +3567,9 @@ <h2 id="pytorch_widedeep.models.tabular.resnet.tab_resnet.TabResnet" class="doc
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>embed_continuous</code></td>
+            <td>
+                <code>embed_continuous</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
             </td>
@@ -3493,7 +3588,9 @@ <h2 id="pytorch_widedeep.models.tabular.resnet.tab_resnet.TabResnet" class="doc
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>embed_continuous_method</code></td>
+            <td>
+                <code>embed_continuous_method</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[standard, piecewise, periodic]]</code>
             </td>
@@ -3514,7 +3611,9 @@ <h2 id="pytorch_widedeep.models.tabular.resnet.tab_resnet.TabResnet" class="doc
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>cont_embed_dim</code></td>
+            <td>
+                <code>cont_embed_dim</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[int]</code>
             </td>
@@ -3529,7 +3628,9 @@ <h2 id="pytorch_widedeep.models.tabular.resnet.tab_resnet.TabResnet" class="doc
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>cont_embed_dropout</code></td>
+            <td>
+                <code>cont_embed_dropout</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>
             </td>
@@ -3543,7 +3644,9 @@ <h2 id="pytorch_widedeep.models.tabular.resnet.tab_resnet.TabResnet" class="doc
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>cont_embed_activation</code></td>
+            <td>
+                <code>cont_embed_activation</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>
             </td>
@@ -3559,7 +3662,9 @@ <h2 id="pytorch_widedeep.models.tabular.resnet.tab_resnet.TabResnet" class="doc
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>quantization_setup</code></td>
+            <td>
+                <code>quantization_setup</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, <span title="pytorch_widedeep.wdtypes.List">List</span>[float]]]</code>
             </td>
@@ -3577,7 +3682,9 @@ <h2 id="pytorch_widedeep.models.tabular.resnet.tab_resnet.TabResnet" class="doc
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>n_frequencies</code></td>
+            <td>
+                <code>n_frequencies</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[int]</code>
             </td>
@@ -3595,7 +3702,9 @@ <h2 id="pytorch_widedeep.models.tabular.resnet.tab_resnet.TabResnet" class="doc
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>sigma</code></td>
+            <td>
+                <code>sigma</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>
             </td>
@@ -3612,7 +3721,9 @@ <h2 id="pytorch_widedeep.models.tabular.resnet.tab_resnet.TabResnet" class="doc
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>share_last_layer</code></td>
+            <td>
+                <code>share_last_layer</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
             </td>
@@ -3631,7 +3742,9 @@ <h2 id="pytorch_widedeep.models.tabular.resnet.tab_resnet.TabResnet" class="doc
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>full_embed_dropout</code></td>
+            <td>
+                <code>full_embed_dropout</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
             </td>
@@ -3646,7 +3759,9 @@ <h2 id="pytorch_widedeep.models.tabular.resnet.tab_resnet.TabResnet" class="doc
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>blocks_dims</code></td>
+            <td>
+                <code>blocks_dims</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.List">List</span>[int]</code>
             </td>
@@ -3665,7 +3780,9 @@ <h2 id="pytorch_widedeep.models.tabular.resnet.tab_resnet.TabResnet" class="doc
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>blocks_dropout</code></td>
+            <td>
+                <code>blocks_dropout</code>
+            </td>
             <td>
                   <code>float</code>
             </td>
@@ -3679,7 +3796,9 @@ <h2 id="pytorch_widedeep.models.tabular.resnet.tab_resnet.TabResnet" class="doc
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>simplify_blocks</code></td>
+            <td>
+                <code>simplify_blocks</code>
+            </td>
             <td>
                   <code>bool</code>
             </td>
@@ -3695,7 +3814,9 @@ <h2 id="pytorch_widedeep.models.tabular.resnet.tab_resnet.TabResnet" class="doc
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>mlp_hidden_dims</code></td>
+            <td>
+                <code>mlp_hidden_dims</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[int]]</code>
             </td>
@@ -3711,7 +3832,9 @@ <h2 id="pytorch_widedeep.models.tabular.resnet.tab_resnet.TabResnet" class="doc
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>mlp_activation</code></td>
+            <td>
+                <code>mlp_activation</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>
             </td>
@@ -3728,7 +3851,9 @@ <h2 id="pytorch_widedeep.models.tabular.resnet.tab_resnet.TabResnet" class="doc
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>mlp_dropout</code></td>
+            <td>
+                <code>mlp_dropout</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>
             </td>
@@ -3744,7 +3869,9 @@ <h2 id="pytorch_widedeep.models.tabular.resnet.tab_resnet.TabResnet" class="doc
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>mlp_batchnorm</code></td>
+            <td>
+                <code>mlp_batchnorm</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
             </td>
@@ -3761,7 +3888,9 @@ <h2 id="pytorch_widedeep.models.tabular.resnet.tab_resnet.TabResnet" class="doc
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>mlp_batchnorm_last</code></td>
+            <td>
+                <code>mlp_batchnorm_last</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
             </td>
@@ -3778,7 +3907,9 @@ <h2 id="pytorch_widedeep.models.tabular.resnet.tab_resnet.TabResnet" class="doc
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>mlp_linear_first</code></td>
+            <td>
+                <code>mlp_linear_first</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
             </td>
@@ -3850,6 +3981,11 @@ <h2 id="pytorch_widedeep.models.tabular.resnet.tab_resnet.TabResnet" class="doc
 <span class="gp">&gt;&gt;&gt; </span><span class="n">out</span> <span class="o">=</span> <span class="n">model</span><span class="p">(</span><span class="n">X_deep</span><span class="p">)</span>
 </code></pre></div>
 
+
+
+
+
+
               <details class="quote">
                 <summary>Source code in <code>pytorch_widedeep/models/tabular/resnet/tab_resnet.py</code></summary>
                 <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 11</span>
@@ -4476,7 +4612,9 @@ <h2 id="pytorch_widedeep.models.tabular.resnet.tab_resnet.TabResnetDecoder" clas
       </thead>
       <tbody>
           <tr class="doc-section-item">
-            <td><code>embed_dim</code></td>
+            <td>
+                <code>embed_dim</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -4490,7 +4628,9 @@ <h2 id="pytorch_widedeep.models.tabular.resnet.tab_resnet.TabResnetDecoder" clas
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>blocks_dims</code></td>
+            <td>
+                <code>blocks_dims</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.List">List</span>[int]</code>
             </td>
@@ -4509,7 +4649,9 @@ <h2 id="pytorch_widedeep.models.tabular.resnet.tab_resnet.TabResnetDecoder" clas
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>blocks_dropout</code></td>
+            <td>
+                <code>blocks_dropout</code>
+            </td>
             <td>
                   <code>float</code>
             </td>
@@ -4523,7 +4665,9 @@ <h2 id="pytorch_widedeep.models.tabular.resnet.tab_resnet.TabResnetDecoder" clas
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>simplify_blocks</code></td>
+            <td>
+                <code>simplify_blocks</code>
+            </td>
             <td>
                   <code>bool</code>
             </td>
@@ -4539,7 +4683,9 @@ <h2 id="pytorch_widedeep.models.tabular.resnet.tab_resnet.TabResnetDecoder" clas
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>mlp_hidden_dims</code></td>
+            <td>
+                <code>mlp_hidden_dims</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[int]]</code>
             </td>
@@ -4555,7 +4701,9 @@ <h2 id="pytorch_widedeep.models.tabular.resnet.tab_resnet.TabResnetDecoder" clas
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>mlp_activation</code></td>
+            <td>
+                <code>mlp_activation</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>
             </td>
@@ -4572,7 +4720,9 @@ <h2 id="pytorch_widedeep.models.tabular.resnet.tab_resnet.TabResnetDecoder" clas
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>mlp_dropout</code></td>
+            <td>
+                <code>mlp_dropout</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>
             </td>
@@ -4588,7 +4738,9 @@ <h2 id="pytorch_widedeep.models.tabular.resnet.tab_resnet.TabResnetDecoder" clas
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>mlp_batchnorm</code></td>
+            <td>
+                <code>mlp_batchnorm</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
             </td>
@@ -4605,7 +4757,9 @@ <h2 id="pytorch_widedeep.models.tabular.resnet.tab_resnet.TabResnetDecoder" clas
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>mlp_batchnorm_last</code></td>
+            <td>
+                <code>mlp_batchnorm_last</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
             </td>
@@ -4622,7 +4776,9 @@ <h2 id="pytorch_widedeep.models.tabular.resnet.tab_resnet.TabResnetDecoder" clas
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>mlp_linear_first</code></td>
+            <td>
+                <code>mlp_linear_first</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
             </td>
@@ -4692,6 +4848,11 @@ <h2 id="pytorch_widedeep.models.tabular.resnet.tab_resnet.TabResnetDecoder" clas
 <span class="go">torch.Size([3, 32])</span>
 </code></pre></div>
 
+
+
+
+
+
               <details class="quote">
                 <summary>Source code in <code>pytorch_widedeep/models/tabular/resnet/tab_resnet.py</code></summary>
                 <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">284</span>
@@ -5041,7 +5202,9 @@ <h2 id="pytorch_widedeep.models.tabular.tabnet.tab_net.TabNet" class="doc doc-he
       </thead>
       <tbody>
           <tr class="doc-section-item">
-            <td><code>column_idx</code></td>
+            <td>
+                <code>column_idx</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, int]</code>
             </td>
@@ -5057,7 +5220,9 @@ <h2 id="pytorch_widedeep.models.tabular.tabnet.tab_net.TabNet" class="doc doc-he
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>cat_embed_input</code></td>
+            <td>
+                <code>cat_embed_input</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.wdtypes.Tuple">Tuple</span>[str, int, int]]]</code>
             </td>
@@ -5072,7 +5237,9 @@ <h2 id="pytorch_widedeep.models.tabular.tabnet.tab_net.TabNet" class="doc doc-he
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>cat_embed_dropout</code></td>
+            <td>
+                <code>cat_embed_dropout</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>
             </td>
@@ -5087,7 +5254,9 @@ <h2 id="pytorch_widedeep.models.tabular.tabnet.tab_net.TabNet" class="doc doc-he
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>use_cat_bias</code></td>
+            <td>
+                <code>use_cat_bias</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
             </td>
@@ -5102,7 +5271,9 @@ <h2 id="pytorch_widedeep.models.tabular.tabnet.tab_net.TabNet" class="doc doc-he
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>cat_embed_activation</code></td>
+            <td>
+                <code>cat_embed_activation</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>
             </td>
@@ -5117,7 +5288,9 @@ <h2 id="pytorch_widedeep.models.tabular.tabnet.tab_net.TabNet" class="doc doc-he
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>continuous_cols</code></td>
+            <td>
+                <code>continuous_cols</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[str]]</code>
             </td>
@@ -5131,7 +5304,9 @@ <h2 id="pytorch_widedeep.models.tabular.tabnet.tab_net.TabNet" class="doc doc-he
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>cont_norm_layer</code></td>
+            <td>
+                <code>cont_norm_layer</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[batchnorm, layernorm]]</code>
             </td>
@@ -5147,7 +5322,9 @@ <h2 id="pytorch_widedeep.models.tabular.tabnet.tab_net.TabNet" class="doc doc-he
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>embed_continuous</code></td>
+            <td>
+                <code>embed_continuous</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
             </td>
@@ -5166,7 +5343,9 @@ <h2 id="pytorch_widedeep.models.tabular.tabnet.tab_net.TabNet" class="doc doc-he
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>embed_continuous_method</code></td>
+            <td>
+                <code>embed_continuous_method</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[standard, piecewise, periodic]]</code>
             </td>
@@ -5187,7 +5366,9 @@ <h2 id="pytorch_widedeep.models.tabular.tabnet.tab_net.TabNet" class="doc doc-he
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>cont_embed_dim</code></td>
+            <td>
+                <code>cont_embed_dim</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[int]</code>
             </td>
@@ -5202,7 +5383,9 @@ <h2 id="pytorch_widedeep.models.tabular.tabnet.tab_net.TabNet" class="doc doc-he
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>cont_embed_dropout</code></td>
+            <td>
+                <code>cont_embed_dropout</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>
             </td>
@@ -5216,7 +5399,9 @@ <h2 id="pytorch_widedeep.models.tabular.tabnet.tab_net.TabNet" class="doc doc-he
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>cont_embed_activation</code></td>
+            <td>
+                <code>cont_embed_activation</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>
             </td>
@@ -5232,7 +5417,9 @@ <h2 id="pytorch_widedeep.models.tabular.tabnet.tab_net.TabNet" class="doc doc-he
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>quantization_setup</code></td>
+            <td>
+                <code>quantization_setup</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, <span title="pytorch_widedeep.wdtypes.List">List</span>[float]]]</code>
             </td>
@@ -5250,7 +5437,9 @@ <h2 id="pytorch_widedeep.models.tabular.tabnet.tab_net.TabNet" class="doc doc-he
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>n_frequencies</code></td>
+            <td>
+                <code>n_frequencies</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[int]</code>
             </td>
@@ -5268,7 +5457,9 @@ <h2 id="pytorch_widedeep.models.tabular.tabnet.tab_net.TabNet" class="doc doc-he
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>sigma</code></td>
+            <td>
+                <code>sigma</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>
             </td>
@@ -5285,7 +5476,9 @@ <h2 id="pytorch_widedeep.models.tabular.tabnet.tab_net.TabNet" class="doc doc-he
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>share_last_layer</code></td>
+            <td>
+                <code>share_last_layer</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
             </td>
@@ -5304,7 +5497,9 @@ <h2 id="pytorch_widedeep.models.tabular.tabnet.tab_net.TabNet" class="doc doc-he
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>full_embed_dropout</code></td>
+            <td>
+                <code>full_embed_dropout</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
             </td>
@@ -5319,7 +5514,9 @@ <h2 id="pytorch_widedeep.models.tabular.tabnet.tab_net.TabNet" class="doc doc-he
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>n_steps</code></td>
+            <td>
+                <code>n_steps</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -5335,7 +5532,9 @@ <h2 id="pytorch_widedeep.models.tabular.tabnet.tab_net.TabNet" class="doc doc-he
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>step_dim</code></td>
+            <td>
+                <code>step_dim</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -5350,7 +5549,9 @@ <h2 id="pytorch_widedeep.models.tabular.tabnet.tab_net.TabNet" class="doc doc-he
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>attn_dim</code></td>
+            <td>
+                <code>attn_dim</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -5364,7 +5565,9 @@ <h2 id="pytorch_widedeep.models.tabular.tabnet.tab_net.TabNet" class="doc doc-he
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>dropout</code></td>
+            <td>
+                <code>dropout</code>
+            </td>
             <td>
                   <code>float</code>
             </td>
@@ -5378,7 +5581,9 @@ <h2 id="pytorch_widedeep.models.tabular.tabnet.tab_net.TabNet" class="doc doc-he
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>n_glu_step_dependent</code></td>
+            <td>
+                <code>n_glu_step_dependent</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -5392,7 +5597,9 @@ <h2 id="pytorch_widedeep.models.tabular.tabnet.tab_net.TabNet" class="doc doc-he
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>n_glu_shared</code></td>
+            <td>
+                <code>n_glu_shared</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -5407,7 +5614,9 @@ <h2 id="pytorch_widedeep.models.tabular.tabnet.tab_net.TabNet" class="doc doc-he
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>ghost_bn</code></td>
+            <td>
+                <code>ghost_bn</code>
+            </td>
             <td>
                   <code>bool</code>
             </td>
@@ -5422,7 +5631,9 @@ <h2 id="pytorch_widedeep.models.tabular.tabnet.tab_net.TabNet" class="doc doc-he
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>virtual_batch_size</code></td>
+            <td>
+                <code>virtual_batch_size</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -5436,7 +5647,9 @@ <h2 id="pytorch_widedeep.models.tabular.tabnet.tab_net.TabNet" class="doc doc-he
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>momentum</code></td>
+            <td>
+                <code>momentum</code>
+            </td>
             <td>
                   <code>float</code>
             </td>
@@ -5452,7 +5665,9 @@ <h2 id="pytorch_widedeep.models.tabular.tabnet.tab_net.TabNet" class="doc doc-he
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>gamma</code></td>
+            <td>
+                <code>gamma</code>
+            </td>
             <td>
                   <code>float</code>
             </td>
@@ -5469,7 +5684,9 @@ <h2 id="pytorch_widedeep.models.tabular.tabnet.tab_net.TabNet" class="doc doc-he
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>epsilon</code></td>
+            <td>
+                <code>epsilon</code>
+            </td>
             <td>
                   <code>float</code>
             </td>
@@ -5483,7 +5700,9 @@ <h2 id="pytorch_widedeep.models.tabular.tabnet.tab_net.TabNet" class="doc doc-he
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>mask_type</code></td>
+            <td>
+                <code>mask_type</code>
+            </td>
             <td>
                   <code>str</code>
             </td>
@@ -5534,6 +5753,11 @@ <h2 id="pytorch_widedeep.models.tabular.tabnet.tab_net.TabNet" class="doc doc-he
 <span class="gp">&gt;&gt;&gt; </span><span class="n">out</span> <span class="o">=</span> <span class="n">model</span><span class="p">(</span><span class="n">X_tab</span><span class="p">)</span>
 </code></pre></div>
 
+
+
+
+
+
               <details class="quote">
                 <summary>Source code in <code>pytorch_widedeep/models/tabular/tabnet/tab_net.py</code></summary>
                 <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 15</span>
@@ -6103,7 +6327,9 @@ <h2 id="pytorch_widedeep.models.tabular.tabnet.tab_net.TabNetDecoder" class="doc
       </thead>
       <tbody>
           <tr class="doc-section-item">
-            <td><code>embed_dim</code></td>
+            <td>
+                <code>embed_dim</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -6117,7 +6343,9 @@ <h2 id="pytorch_widedeep.models.tabular.tabnet.tab_net.TabNetDecoder" class="doc
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>n_steps</code></td>
+            <td>
+                <code>n_steps</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -6133,7 +6361,9 @@ <h2 id="pytorch_widedeep.models.tabular.tabnet.tab_net.TabNetDecoder" class="doc
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>step_dim</code></td>
+            <td>
+                <code>step_dim</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -6148,7 +6378,9 @@ <h2 id="pytorch_widedeep.models.tabular.tabnet.tab_net.TabNetDecoder" class="doc
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>dropout</code></td>
+            <td>
+                <code>dropout</code>
+            </td>
             <td>
                   <code>float</code>
             </td>
@@ -6162,7 +6394,9 @@ <h2 id="pytorch_widedeep.models.tabular.tabnet.tab_net.TabNetDecoder" class="doc
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>n_glu_step_dependent</code></td>
+            <td>
+                <code>n_glu_step_dependent</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -6176,7 +6410,9 @@ <h2 id="pytorch_widedeep.models.tabular.tabnet.tab_net.TabNetDecoder" class="doc
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>n_glu_shared</code></td>
+            <td>
+                <code>n_glu_shared</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -6191,7 +6427,9 @@ <h2 id="pytorch_widedeep.models.tabular.tabnet.tab_net.TabNetDecoder" class="doc
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>ghost_bn</code></td>
+            <td>
+                <code>ghost_bn</code>
+            </td>
             <td>
                   <code>bool</code>
             </td>
@@ -6206,7 +6444,9 @@ <h2 id="pytorch_widedeep.models.tabular.tabnet.tab_net.TabNetDecoder" class="doc
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>virtual_batch_size</code></td>
+            <td>
+                <code>virtual_batch_size</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -6220,7 +6460,9 @@ <h2 id="pytorch_widedeep.models.tabular.tabnet.tab_net.TabNetDecoder" class="doc
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>momentum</code></td>
+            <td>
+                <code>momentum</code>
+            </td>
             <td>
                   <code>float</code>
             </td>
@@ -6275,6 +6517,11 @@ <h2 id="pytorch_widedeep.models.tabular.tabnet.tab_net.TabNetDecoder" class="doc
 <span class="go">torch.Size([3, 32])</span>
 </code></pre></div>
 
+
+
+
+
+
               <details class="quote">
                 <summary>Source code in <code>pytorch_widedeep/models/tabular/tabnet/tab_net.py</code></summary>
                 <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">280</span>
@@ -6561,7 +6808,9 @@ <h2 id="pytorch_widedeep.models.tabular.mlp.context_attention_mlp.ContextAttenti
       </thead>
       <tbody>
           <tr class="doc-section-item">
-            <td><code>column_idx</code></td>
+            <td>
+                <code>column_idx</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, int]</code>
             </td>
@@ -6577,7 +6826,9 @@ <h2 id="pytorch_widedeep.models.tabular.mlp.context_attention_mlp.ContextAttenti
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>cat_embed_input</code></td>
+            <td>
+                <code>cat_embed_input</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.wdtypes.Tuple">Tuple</span>[str, int]]]</code>
             </td>
@@ -6592,7 +6843,9 @@ <h2 id="pytorch_widedeep.models.tabular.mlp.context_attention_mlp.ContextAttenti
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>cat_embed_dropout</code></td>
+            <td>
+                <code>cat_embed_dropout</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>
             </td>
@@ -6607,7 +6860,9 @@ <h2 id="pytorch_widedeep.models.tabular.mlp.context_attention_mlp.ContextAttenti
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>use_cat_bias</code></td>
+            <td>
+                <code>use_cat_bias</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
             </td>
@@ -6622,9 +6877,11 @@ <h2 id="pytorch_widedeep.models.tabular.mlp.context_attention_mlp.ContextAttenti
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>cat_embed_activation</code></td>
             <td>
-                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>
+                <code>cat_embed_activation</code>
+            </td>
+            <td>
+                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>
             </td>
             <td>
               <div class="doc-md-description">
@@ -6637,7 +6894,9 @@ <h2 id="pytorch_widedeep.models.tabular.mlp.context_attention_mlp.ContextAttenti
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>shared_embed</code></td>
+            <td>
+                <code>shared_embed</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
             </td>
@@ -6656,7 +6915,9 @@ <h2 id="pytorch_widedeep.models.tabular.mlp.context_attention_mlp.ContextAttenti
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>add_shared_embed</code></td>
+            <td>
+                <code>add_shared_embed</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
             </td>
@@ -6674,7 +6935,9 @@ <h2 id="pytorch_widedeep.models.tabular.mlp.context_attention_mlp.ContextAttenti
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>frac_shared_embed</code></td>
+            <td>
+                <code>frac_shared_embed</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>
             </td>
@@ -6690,7 +6953,9 @@ <h2 id="pytorch_widedeep.models.tabular.mlp.context_attention_mlp.ContextAttenti
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>continuous_cols</code></td>
+            <td>
+                <code>continuous_cols</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[str]]</code>
             </td>
@@ -6704,7 +6969,9 @@ <h2 id="pytorch_widedeep.models.tabular.mlp.context_attention_mlp.ContextAttenti
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>cont_norm_layer</code></td>
+            <td>
+                <code>cont_norm_layer</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[batchnorm, layernorm]]</code>
             </td>
@@ -6720,7 +6987,9 @@ <h2 id="pytorch_widedeep.models.tabular.mlp.context_attention_mlp.ContextAttenti
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>embed_continuous_method</code></td>
+            <td>
+                <code>embed_continuous_method</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[standard, piecewise, periodic]]</code>
             </td>
@@ -6741,7 +7010,9 @@ <h2 id="pytorch_widedeep.models.tabular.mlp.context_attention_mlp.ContextAttenti
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>cont_embed_dropout</code></td>
+            <td>
+                <code>cont_embed_dropout</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>
             </td>
@@ -6755,7 +7026,9 @@ <h2 id="pytorch_widedeep.models.tabular.mlp.context_attention_mlp.ContextAttenti
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>cont_embed_activation</code></td>
+            <td>
+                <code>cont_embed_activation</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>
             </td>
@@ -6771,7 +7044,9 @@ <h2 id="pytorch_widedeep.models.tabular.mlp.context_attention_mlp.ContextAttenti
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>quantization_setup</code></td>
+            <td>
+                <code>quantization_setup</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, <span title="pytorch_widedeep.wdtypes.List">List</span>[float]]]</code>
             </td>
@@ -6789,7 +7064,9 @@ <h2 id="pytorch_widedeep.models.tabular.mlp.context_attention_mlp.ContextAttenti
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>n_frequencies</code></td>
+            <td>
+                <code>n_frequencies</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[int]</code>
             </td>
@@ -6807,7 +7084,9 @@ <h2 id="pytorch_widedeep.models.tabular.mlp.context_attention_mlp.ContextAttenti
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>sigma</code></td>
+            <td>
+                <code>sigma</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>
             </td>
@@ -6824,7 +7103,9 @@ <h2 id="pytorch_widedeep.models.tabular.mlp.context_attention_mlp.ContextAttenti
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>share_last_layer</code></td>
+            <td>
+                <code>share_last_layer</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
             </td>
@@ -6843,7 +7124,9 @@ <h2 id="pytorch_widedeep.models.tabular.mlp.context_attention_mlp.ContextAttenti
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>full_embed_dropout</code></td>
+            <td>
+                <code>full_embed_dropout</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
             </td>
@@ -6858,7 +7141,9 @@ <h2 id="pytorch_widedeep.models.tabular.mlp.context_attention_mlp.ContextAttenti
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>input_dim</code></td>
+            <td>
+                <code>input_dim</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -6873,7 +7158,9 @@ <h2 id="pytorch_widedeep.models.tabular.mlp.context_attention_mlp.ContextAttenti
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>attn_dropout</code></td>
+            <td>
+                <code>attn_dropout</code>
+            </td>
             <td>
                   <code>float</code>
             </td>
@@ -6887,7 +7174,9 @@ <h2 id="pytorch_widedeep.models.tabular.mlp.context_attention_mlp.ContextAttenti
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>with_addnorm</code></td>
+            <td>
+                <code>with_addnorm</code>
+            </td>
             <td>
                   <code>bool</code>
             </td>
@@ -6902,7 +7191,9 @@ <h2 id="pytorch_widedeep.models.tabular.mlp.context_attention_mlp.ContextAttenti
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>attn_activation</code></td>
+            <td>
+                <code>attn_activation</code>
+            </td>
             <td>
                   <code>str</code>
             </td>
@@ -6918,7 +7209,9 @@ <h2 id="pytorch_widedeep.models.tabular.mlp.context_attention_mlp.ContextAttenti
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>n_blocks</code></td>
+            <td>
+                <code>n_blocks</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -6971,6 +7264,11 @@ <h2 id="pytorch_widedeep.models.tabular.mlp.context_attention_mlp.ContextAttenti
 <span class="gp">&gt;&gt;&gt; </span><span class="n">out</span> <span class="o">=</span> <span class="n">model</span><span class="p">(</span><span class="n">X_tab</span><span class="p">)</span>
 </code></pre></div>
 
+
+
+
+
+
               <details class="quote">
                 <summary>Source code in <code>pytorch_widedeep/models/tabular/mlp/context_attention_mlp.py</code></summary>
                 <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 10</span>
@@ -7570,7 +7868,9 @@ <h2 id="pytorch_widedeep.models.tabular.mlp.self_attention_mlp.SelfAttentionMLP"
       </thead>
       <tbody>
           <tr class="doc-section-item">
-            <td><code>column_idx</code></td>
+            <td>
+                <code>column_idx</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, int]</code>
             </td>
@@ -7586,7 +7886,9 @@ <h2 id="pytorch_widedeep.models.tabular.mlp.self_attention_mlp.SelfAttentionMLP"
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>cat_embed_input</code></td>
+            <td>
+                <code>cat_embed_input</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.wdtypes.Tuple">Tuple</span>[str, int]]]</code>
             </td>
@@ -7601,7 +7903,9 @@ <h2 id="pytorch_widedeep.models.tabular.mlp.self_attention_mlp.SelfAttentionMLP"
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>cat_embed_dropout</code></td>
+            <td>
+                <code>cat_embed_dropout</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>
             </td>
@@ -7616,7 +7920,9 @@ <h2 id="pytorch_widedeep.models.tabular.mlp.self_attention_mlp.SelfAttentionMLP"
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>use_cat_bias</code></td>
+            <td>
+                <code>use_cat_bias</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
             </td>
@@ -7631,7 +7937,9 @@ <h2 id="pytorch_widedeep.models.tabular.mlp.self_attention_mlp.SelfAttentionMLP"
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>cat_embed_activation</code></td>
+            <td>
+                <code>cat_embed_activation</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>
             </td>
@@ -7646,7 +7954,9 @@ <h2 id="pytorch_widedeep.models.tabular.mlp.self_attention_mlp.SelfAttentionMLP"
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>shared_embed</code></td>
+            <td>
+                <code>shared_embed</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
             </td>
@@ -7665,7 +7975,9 @@ <h2 id="pytorch_widedeep.models.tabular.mlp.self_attention_mlp.SelfAttentionMLP"
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>add_shared_embed</code></td>
+            <td>
+                <code>add_shared_embed</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
             </td>
@@ -7683,7 +7995,9 @@ <h2 id="pytorch_widedeep.models.tabular.mlp.self_attention_mlp.SelfAttentionMLP"
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>frac_shared_embed</code></td>
+            <td>
+                <code>frac_shared_embed</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>
             </td>
@@ -7699,7 +8013,9 @@ <h2 id="pytorch_widedeep.models.tabular.mlp.self_attention_mlp.SelfAttentionMLP"
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>continuous_cols</code></td>
+            <td>
+                <code>continuous_cols</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[str]]</code>
             </td>
@@ -7713,7 +8029,9 @@ <h2 id="pytorch_widedeep.models.tabular.mlp.self_attention_mlp.SelfAttentionMLP"
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>cont_norm_layer</code></td>
+            <td>
+                <code>cont_norm_layer</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[batchnorm, layernorm]]</code>
             </td>
@@ -7729,7 +8047,9 @@ <h2 id="pytorch_widedeep.models.tabular.mlp.self_attention_mlp.SelfAttentionMLP"
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>embed_continuous_method</code></td>
+            <td>
+                <code>embed_continuous_method</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[standard, piecewise, periodic]]</code>
             </td>
@@ -7750,7 +8070,9 @@ <h2 id="pytorch_widedeep.models.tabular.mlp.self_attention_mlp.SelfAttentionMLP"
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>cont_embed_dropout</code></td>
+            <td>
+                <code>cont_embed_dropout</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>
             </td>
@@ -7764,7 +8086,9 @@ <h2 id="pytorch_widedeep.models.tabular.mlp.self_attention_mlp.SelfAttentionMLP"
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>cont_embed_activation</code></td>
+            <td>
+                <code>cont_embed_activation</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>
             </td>
@@ -7780,7 +8104,9 @@ <h2 id="pytorch_widedeep.models.tabular.mlp.self_attention_mlp.SelfAttentionMLP"
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>quantization_setup</code></td>
+            <td>
+                <code>quantization_setup</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, <span title="pytorch_widedeep.wdtypes.List">List</span>[float]]]</code>
             </td>
@@ -7798,7 +8124,9 @@ <h2 id="pytorch_widedeep.models.tabular.mlp.self_attention_mlp.SelfAttentionMLP"
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>n_frequencies</code></td>
+            <td>
+                <code>n_frequencies</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[int]</code>
             </td>
@@ -7816,7 +8144,9 @@ <h2 id="pytorch_widedeep.models.tabular.mlp.self_attention_mlp.SelfAttentionMLP"
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>sigma</code></td>
+            <td>
+                <code>sigma</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>
             </td>
@@ -7833,7 +8163,9 @@ <h2 id="pytorch_widedeep.models.tabular.mlp.self_attention_mlp.SelfAttentionMLP"
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>share_last_layer</code></td>
+            <td>
+                <code>share_last_layer</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
             </td>
@@ -7852,7 +8184,9 @@ <h2 id="pytorch_widedeep.models.tabular.mlp.self_attention_mlp.SelfAttentionMLP"
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>full_embed_dropout</code></td>
+            <td>
+                <code>full_embed_dropout</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
             </td>
@@ -7867,7 +8201,9 @@ <h2 id="pytorch_widedeep.models.tabular.mlp.self_attention_mlp.SelfAttentionMLP"
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>input_dim</code></td>
+            <td>
+                <code>input_dim</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -7882,7 +8218,9 @@ <h2 id="pytorch_widedeep.models.tabular.mlp.self_attention_mlp.SelfAttentionMLP"
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>attn_dropout</code></td>
+            <td>
+                <code>attn_dropout</code>
+            </td>
             <td>
                   <code>float</code>
             </td>
@@ -7896,7 +8234,9 @@ <h2 id="pytorch_widedeep.models.tabular.mlp.self_attention_mlp.SelfAttentionMLP"
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>n_heads</code></td>
+            <td>
+                <code>n_heads</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -7910,7 +8250,9 @@ <h2 id="pytorch_widedeep.models.tabular.mlp.self_attention_mlp.SelfAttentionMLP"
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>use_bias</code></td>
+            <td>
+                <code>use_bias</code>
+            </td>
             <td>
                   <code>bool</code>
             </td>
@@ -7925,7 +8267,9 @@ <h2 id="pytorch_widedeep.models.tabular.mlp.self_attention_mlp.SelfAttentionMLP"
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>with_addnorm</code></td>
+            <td>
+                <code>with_addnorm</code>
+            </td>
             <td>
                   <code>bool</code>
             </td>
@@ -7939,7 +8283,9 @@ <h2 id="pytorch_widedeep.models.tabular.mlp.self_attention_mlp.SelfAttentionMLP"
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>attn_activation</code></td>
+            <td>
+                <code>attn_activation</code>
+            </td>
             <td>
                   <code>str</code>
             </td>
@@ -7955,7 +8301,9 @@ <h2 id="pytorch_widedeep.models.tabular.mlp.self_attention_mlp.SelfAttentionMLP"
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>n_blocks</code></td>
+            <td>
+                <code>n_blocks</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -8019,6 +8367,11 @@ <h2 id="pytorch_widedeep.models.tabular.mlp.self_attention_mlp.SelfAttentionMLP"
 <span class="gp">&gt;&gt;&gt; </span><span class="n">out</span> <span class="o">=</span> <span class="n">model</span><span class="p">(</span><span class="n">X_tab</span><span class="p">)</span>
 </code></pre></div>
 
+
+
+
+
+
               <details class="quote">
                 <summary>Source code in <code>pytorch_widedeep/models/tabular/mlp/self_attention_mlp.py</code></summary>
                 <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 10</span>
@@ -8659,7 +9012,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_transformer.TabTransfor
       </thead>
       <tbody>
           <tr class="doc-section-item">
-            <td><code>column_idx</code></td>
+            <td>
+                <code>column_idx</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, int]</code>
             </td>
@@ -8675,7 +9030,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_transformer.TabTransfor
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>cat_embed_input</code></td>
+            <td>
+                <code>cat_embed_input</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.wdtypes.Tuple">Tuple</span>[str, int]]]</code>
             </td>
@@ -8690,7 +9047,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_transformer.TabTransfor
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>cat_embed_dropout</code></td>
+            <td>
+                <code>cat_embed_dropout</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>
             </td>
@@ -8705,7 +9064,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_transformer.TabTransfor
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>use_cat_bias</code></td>
+            <td>
+                <code>use_cat_bias</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
             </td>
@@ -8720,7 +9081,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_transformer.TabTransfor
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>cat_embed_activation</code></td>
+            <td>
+                <code>cat_embed_activation</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>
             </td>
@@ -8735,7 +9098,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_transformer.TabTransfor
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>shared_embed</code></td>
+            <td>
+                <code>shared_embed</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
             </td>
@@ -8754,7 +9119,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_transformer.TabTransfor
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>add_shared_embed</code></td>
+            <td>
+                <code>add_shared_embed</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
             </td>
@@ -8772,7 +9139,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_transformer.TabTransfor
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>frac_shared_embed</code></td>
+            <td>
+                <code>frac_shared_embed</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>
             </td>
@@ -8788,7 +9157,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_transformer.TabTransfor
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>continuous_cols</code></td>
+            <td>
+                <code>continuous_cols</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[str]]</code>
             </td>
@@ -8802,7 +9173,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_transformer.TabTransfor
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>cont_norm_layer</code></td>
+            <td>
+                <code>cont_norm_layer</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[batchnorm, layernorm]]</code>
             </td>
@@ -8818,7 +9191,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_transformer.TabTransfor
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>embed_continuous_method</code></td>
+            <td>
+                <code>embed_continuous_method</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[standard, piecewise, periodic]]</code>
             </td>
@@ -8839,7 +9214,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_transformer.TabTransfor
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>cont_embed_dropout</code></td>
+            <td>
+                <code>cont_embed_dropout</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>
             </td>
@@ -8853,7 +9230,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_transformer.TabTransfor
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>cont_embed_activation</code></td>
+            <td>
+                <code>cont_embed_activation</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>
             </td>
@@ -8869,7 +9248,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_transformer.TabTransfor
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>quantization_setup</code></td>
+            <td>
+                <code>quantization_setup</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, <span title="pytorch_widedeep.wdtypes.List">List</span>[float]]]</code>
             </td>
@@ -8887,7 +9268,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_transformer.TabTransfor
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>n_frequencies</code></td>
+            <td>
+                <code>n_frequencies</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[int]</code>
             </td>
@@ -8905,7 +9288,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_transformer.TabTransfor
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>sigma</code></td>
+            <td>
+                <code>sigma</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>
             </td>
@@ -8922,7 +9307,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_transformer.TabTransfor
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>share_last_layer</code></td>
+            <td>
+                <code>share_last_layer</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
             </td>
@@ -8941,7 +9328,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_transformer.TabTransfor
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>full_embed_dropout</code></td>
+            <td>
+                <code>full_embed_dropout</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
             </td>
@@ -8956,7 +9345,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_transformer.TabTransfor
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>input_dim</code></td>
+            <td>
+                <code>input_dim</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -8971,7 +9362,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_transformer.TabTransfor
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>n_heads</code></td>
+            <td>
+                <code>n_heads</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -8985,7 +9378,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_transformer.TabTransfor
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>use_qkv_bias</code></td>
+            <td>
+                <code>use_qkv_bias</code>
+            </td>
             <td>
                   <code>bool</code>
             </td>
@@ -9000,7 +9395,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_transformer.TabTransfor
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>n_blocks</code></td>
+            <td>
+                <code>n_blocks</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -9014,7 +9411,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_transformer.TabTransfor
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>attn_dropout</code></td>
+            <td>
+                <code>attn_dropout</code>
+            </td>
             <td>
                   <code>float</code>
             </td>
@@ -9028,7 +9427,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_transformer.TabTransfor
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>ff_dropout</code></td>
+            <td>
+                <code>ff_dropout</code>
+            </td>
             <td>
                   <code>float</code>
             </td>
@@ -9042,7 +9443,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_transformer.TabTransfor
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>ff_factor</code></td>
+            <td>
+                <code>ff_factor</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -9057,7 +9460,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_transformer.TabTransfor
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>transformer_activation</code></td>
+            <td>
+                <code>transformer_activation</code>
+            </td>
             <td>
                   <code>str</code>
             </td>
@@ -9072,7 +9477,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_transformer.TabTransfor
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>use_linear_attention</code></td>
+            <td>
+                <code>use_linear_attention</code>
+            </td>
             <td>
                   <code>bool</code>
             </td>
@@ -9091,7 +9498,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_transformer.TabTransfor
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>use_flash_attention</code></td>
+            <td>
+                <code>use_flash_attention</code>
+            </td>
             <td>
                   <code>bool</code>
             </td>
@@ -9107,7 +9516,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_transformer.TabTransfor
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>mlp_hidden_dims</code></td>
+            <td>
+                <code>mlp_hidden_dims</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[int]]</code>
             </td>
@@ -9123,7 +9534,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_transformer.TabTransfor
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>mlp_activation</code></td>
+            <td>
+                <code>mlp_activation</code>
+            </td>
             <td>
                   <code>str</code>
             </td>
@@ -9140,7 +9553,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_transformer.TabTransfor
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>mlp_dropout</code></td>
+            <td>
+                <code>mlp_dropout</code>
+            </td>
             <td>
                   <code>float</code>
             </td>
@@ -9156,7 +9571,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_transformer.TabTransfor
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>mlp_batchnorm</code></td>
+            <td>
+                <code>mlp_batchnorm</code>
+            </td>
             <td>
                   <code>bool</code>
             </td>
@@ -9173,7 +9590,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_transformer.TabTransfor
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>mlp_batchnorm_last</code></td>
+            <td>
+                <code>mlp_batchnorm_last</code>
+            </td>
             <td>
                   <code>bool</code>
             </td>
@@ -9190,7 +9609,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_transformer.TabTransfor
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>mlp_linear_first</code></td>
+            <td>
+                <code>mlp_linear_first</code>
+            </td>
             <td>
                   <code>bool</code>
             </td>
@@ -9259,6 +9680,11 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_transformer.TabTransfor
 <span class="gp">&gt;&gt;&gt; </span><span class="n">out</span> <span class="o">=</span> <span class="n">model</span><span class="p">(</span><span class="n">X_tab</span><span class="p">)</span>
 </code></pre></div>
 
+
+
+
+
+
               <details class="quote">
                 <summary>Source code in <code>pytorch_widedeep/models/tabular/transformers/tab_transformer.py</code></summary>
                 <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 12</span>
@@ -10219,7 +10645,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.saint.SAINT" class="doc doc
       </thead>
       <tbody>
           <tr class="doc-section-item">
-            <td><code>column_idx</code></td>
+            <td>
+                <code>column_idx</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, int]</code>
             </td>
@@ -10235,7 +10663,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.saint.SAINT" class="doc doc
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>cat_embed_input</code></td>
+            <td>
+                <code>cat_embed_input</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.wdtypes.Tuple">Tuple</span>[str, int]]]</code>
             </td>
@@ -10250,7 +10680,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.saint.SAINT" class="doc doc
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>cat_embed_dropout</code></td>
+            <td>
+                <code>cat_embed_dropout</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>
             </td>
@@ -10265,7 +10697,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.saint.SAINT" class="doc doc
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>use_cat_bias</code></td>
+            <td>
+                <code>use_cat_bias</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
             </td>
@@ -10280,7 +10714,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.saint.SAINT" class="doc doc
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>cat_embed_activation</code></td>
+            <td>
+                <code>cat_embed_activation</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>
             </td>
@@ -10295,7 +10731,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.saint.SAINT" class="doc doc
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>shared_embed</code></td>
+            <td>
+                <code>shared_embed</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
             </td>
@@ -10314,7 +10752,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.saint.SAINT" class="doc doc
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>add_shared_embed</code></td>
+            <td>
+                <code>add_shared_embed</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
             </td>
@@ -10332,7 +10772,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.saint.SAINT" class="doc doc
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>frac_shared_embed</code></td>
+            <td>
+                <code>frac_shared_embed</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>
             </td>
@@ -10348,7 +10790,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.saint.SAINT" class="doc doc
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>continuous_cols</code></td>
+            <td>
+                <code>continuous_cols</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[str]]</code>
             </td>
@@ -10362,7 +10806,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.saint.SAINT" class="doc doc
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>cont_norm_layer</code></td>
+            <td>
+                <code>cont_norm_layer</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[batchnorm, layernorm]]</code>
             </td>
@@ -10378,7 +10824,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.saint.SAINT" class="doc doc
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>embed_continuous_method</code></td>
+            <td>
+                <code>embed_continuous_method</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[standard, piecewise, periodic]]</code>
             </td>
@@ -10399,7 +10847,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.saint.SAINT" class="doc doc
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>cont_embed_dropout</code></td>
+            <td>
+                <code>cont_embed_dropout</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>
             </td>
@@ -10413,7 +10863,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.saint.SAINT" class="doc doc
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>cont_embed_activation</code></td>
+            <td>
+                <code>cont_embed_activation</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>
             </td>
@@ -10429,7 +10881,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.saint.SAINT" class="doc doc
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>quantization_setup</code></td>
+            <td>
+                <code>quantization_setup</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, <span title="pytorch_widedeep.wdtypes.List">List</span>[float]]]</code>
             </td>
@@ -10447,7 +10901,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.saint.SAINT" class="doc doc
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>n_frequencies</code></td>
+            <td>
+                <code>n_frequencies</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[int]</code>
             </td>
@@ -10465,7 +10921,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.saint.SAINT" class="doc doc
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>sigma</code></td>
+            <td>
+                <code>sigma</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>
             </td>
@@ -10482,7 +10940,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.saint.SAINT" class="doc doc
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>share_last_layer</code></td>
+            <td>
+                <code>share_last_layer</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
             </td>
@@ -10501,7 +10961,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.saint.SAINT" class="doc doc
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>full_embed_dropout</code></td>
+            <td>
+                <code>full_embed_dropout</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
             </td>
@@ -10516,7 +10978,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.saint.SAINT" class="doc doc
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>input_dim</code></td>
+            <td>
+                <code>input_dim</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -10531,7 +10995,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.saint.SAINT" class="doc doc
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>n_heads</code></td>
+            <td>
+                <code>n_heads</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -10545,7 +11011,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.saint.SAINT" class="doc doc
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>use_qkv_bias</code></td>
+            <td>
+                <code>use_qkv_bias</code>
+            </td>
             <td>
                   <code>bool</code>
             </td>
@@ -10560,7 +11028,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.saint.SAINT" class="doc doc
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>n_blocks</code></td>
+            <td>
+                <code>n_blocks</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -10574,7 +11044,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.saint.SAINT" class="doc doc
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>attn_dropout</code></td>
+            <td>
+                <code>attn_dropout</code>
+            </td>
             <td>
                   <code>float</code>
             </td>
@@ -10589,7 +11061,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.saint.SAINT" class="doc doc
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>ff_dropout</code></td>
+            <td>
+                <code>ff_dropout</code>
+            </td>
             <td>
                   <code>float</code>
             </td>
@@ -10603,7 +11077,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.saint.SAINT" class="doc doc
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>ff_factor</code></td>
+            <td>
+                <code>ff_factor</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -10618,7 +11094,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.saint.SAINT" class="doc doc
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>transformer_activation</code></td>
+            <td>
+                <code>transformer_activation</code>
+            </td>
             <td>
                   <code>str</code>
             </td>
@@ -10633,7 +11111,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.saint.SAINT" class="doc doc
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>mlp_hidden_dims</code></td>
+            <td>
+                <code>mlp_hidden_dims</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[int]]</code>
             </td>
@@ -10649,7 +11129,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.saint.SAINT" class="doc doc
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>mlp_activation</code></td>
+            <td>
+                <code>mlp_activation</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>
             </td>
@@ -10666,7 +11148,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.saint.SAINT" class="doc doc
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>mlp_dropout</code></td>
+            <td>
+                <code>mlp_dropout</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>
             </td>
@@ -10682,7 +11166,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.saint.SAINT" class="doc doc
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>mlp_batchnorm</code></td>
+            <td>
+                <code>mlp_batchnorm</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
             </td>
@@ -10699,7 +11185,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.saint.SAINT" class="doc doc
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>mlp_batchnorm_last</code></td>
+            <td>
+                <code>mlp_batchnorm_last</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
             </td>
@@ -10716,7 +11204,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.saint.SAINT" class="doc doc
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>mlp_linear_first</code></td>
+            <td>
+                <code>mlp_linear_first</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
             </td>
@@ -10785,6 +11275,11 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.saint.SAINT" class="doc doc
 <span class="gp">&gt;&gt;&gt; </span><span class="n">out</span> <span class="o">=</span> <span class="n">model</span><span class="p">(</span><span class="n">X_tab</span><span class="p">)</span>
 </code></pre></div>
 
+
+
+
+
+
               <details class="quote">
                 <summary>Source code in <code>pytorch_widedeep/models/tabular/transformers/saint.py</code></summary>
                 <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 11</span>
@@ -11571,7 +12066,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.ft_transformer.FTTransforme
       </thead>
       <tbody>
           <tr class="doc-section-item">
-            <td><code>column_idx</code></td>
+            <td>
+                <code>column_idx</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, int]</code>
             </td>
@@ -11587,7 +12084,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.ft_transformer.FTTransforme
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>cat_embed_input</code></td>
+            <td>
+                <code>cat_embed_input</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.wdtypes.Tuple">Tuple</span>[str, int]]]</code>
             </td>
@@ -11602,7 +12101,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.ft_transformer.FTTransforme
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>cat_embed_dropout</code></td>
+            <td>
+                <code>cat_embed_dropout</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>
             </td>
@@ -11617,7 +12118,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.ft_transformer.FTTransforme
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>use_cat_bias</code></td>
+            <td>
+                <code>use_cat_bias</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
             </td>
@@ -11632,7 +12135,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.ft_transformer.FTTransforme
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>cat_embed_activation</code></td>
+            <td>
+                <code>cat_embed_activation</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>
             </td>
@@ -11647,7 +12152,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.ft_transformer.FTTransforme
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>shared_embed</code></td>
+            <td>
+                <code>shared_embed</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
             </td>
@@ -11666,7 +12173,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.ft_transformer.FTTransforme
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>add_shared_embed</code></td>
+            <td>
+                <code>add_shared_embed</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
             </td>
@@ -11684,7 +12193,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.ft_transformer.FTTransforme
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>frac_shared_embed</code></td>
+            <td>
+                <code>frac_shared_embed</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>
             </td>
@@ -11700,7 +12211,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.ft_transformer.FTTransforme
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>continuous_cols</code></td>
+            <td>
+                <code>continuous_cols</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[str]]</code>
             </td>
@@ -11714,7 +12227,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.ft_transformer.FTTransforme
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>cont_norm_layer</code></td>
+            <td>
+                <code>cont_norm_layer</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[batchnorm, layernorm]]</code>
             </td>
@@ -11730,7 +12245,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.ft_transformer.FTTransforme
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>embed_continuous_method</code></td>
+            <td>
+                <code>embed_continuous_method</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[standard, piecewise, periodic]]</code>
             </td>
@@ -11751,7 +12268,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.ft_transformer.FTTransforme
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>cont_embed_dropout</code></td>
+            <td>
+                <code>cont_embed_dropout</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>
             </td>
@@ -11765,7 +12284,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.ft_transformer.FTTransforme
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>cont_embed_activation</code></td>
+            <td>
+                <code>cont_embed_activation</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>
             </td>
@@ -11781,7 +12302,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.ft_transformer.FTTransforme
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>quantization_setup</code></td>
+            <td>
+                <code>quantization_setup</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, <span title="pytorch_widedeep.wdtypes.List">List</span>[float]]]</code>
             </td>
@@ -11799,7 +12322,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.ft_transformer.FTTransforme
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>n_frequencies</code></td>
+            <td>
+                <code>n_frequencies</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[int]</code>
             </td>
@@ -11817,7 +12342,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.ft_transformer.FTTransforme
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>sigma</code></td>
+            <td>
+                <code>sigma</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>
             </td>
@@ -11834,7 +12361,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.ft_transformer.FTTransforme
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>share_last_layer</code></td>
+            <td>
+                <code>share_last_layer</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
             </td>
@@ -11853,7 +12382,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.ft_transformer.FTTransforme
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>full_embed_dropout</code></td>
+            <td>
+                <code>full_embed_dropout</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
             </td>
@@ -11868,7 +12399,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.ft_transformer.FTTransforme
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>input_dim</code></td>
+            <td>
+                <code>input_dim</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -11883,7 +12416,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.ft_transformer.FTTransforme
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>kv_compression_factor</code></td>
+            <td>
+                <code>kv_compression_factor</code>
+            </td>
             <td>
                   <code>float</code>
             </td>
@@ -11902,7 +12437,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.ft_transformer.FTTransforme
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>kv_sharing</code></td>
+            <td>
+                <code>kv_sharing</code>
+            </td>
             <td>
                   <code>bool</code>
             </td>
@@ -11918,7 +12455,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.ft_transformer.FTTransforme
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>n_heads</code></td>
+            <td>
+                <code>n_heads</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -11932,7 +12471,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.ft_transformer.FTTransforme
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>use_qkv_bias</code></td>
+            <td>
+                <code>use_qkv_bias</code>
+            </td>
             <td>
                   <code>bool</code>
             </td>
@@ -11947,7 +12488,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.ft_transformer.FTTransforme
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>n_blocks</code></td>
+            <td>
+                <code>n_blocks</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -11961,7 +12504,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.ft_transformer.FTTransforme
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>attn_dropout</code></td>
+            <td>
+                <code>attn_dropout</code>
+            </td>
             <td>
                   <code>float</code>
             </td>
@@ -11975,7 +12520,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.ft_transformer.FTTransforme
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>ff_dropout</code></td>
+            <td>
+                <code>ff_dropout</code>
+            </td>
             <td>
                   <code>float</code>
             </td>
@@ -11989,7 +12536,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.ft_transformer.FTTransforme
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>ff_factor</code></td>
+            <td>
+                <code>ff_factor</code>
+            </td>
             <td>
                   <code>float</code>
             </td>
@@ -12005,7 +12554,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.ft_transformer.FTTransforme
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>transformer_activation</code></td>
+            <td>
+                <code>transformer_activation</code>
+            </td>
             <td>
                   <code>str</code>
             </td>
@@ -12020,7 +12571,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.ft_transformer.FTTransforme
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>mlp_hidden_dims</code></td>
+            <td>
+                <code>mlp_hidden_dims</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[int]]</code>
             </td>
@@ -12036,7 +12589,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.ft_transformer.FTTransforme
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>mlp_activation</code></td>
+            <td>
+                <code>mlp_activation</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>
             </td>
@@ -12053,7 +12608,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.ft_transformer.FTTransforme
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>mlp_dropout</code></td>
+            <td>
+                <code>mlp_dropout</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>
             </td>
@@ -12069,7 +12626,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.ft_transformer.FTTransforme
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>mlp_batchnorm</code></td>
+            <td>
+                <code>mlp_batchnorm</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
             </td>
@@ -12086,7 +12645,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.ft_transformer.FTTransforme
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>mlp_batchnorm_last</code></td>
+            <td>
+                <code>mlp_batchnorm_last</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
             </td>
@@ -12103,7 +12664,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.ft_transformer.FTTransforme
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>mlp_linear_first</code></td>
+            <td>
+                <code>mlp_linear_first</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
             </td>
@@ -12172,6 +12735,11 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.ft_transformer.FTTransforme
 <span class="gp">&gt;&gt;&gt; </span><span class="n">out</span> <span class="o">=</span> <span class="n">model</span><span class="p">(</span><span class="n">X_tab</span><span class="p">)</span>
 </code></pre></div>
 
+
+
+
+
+
               <details class="quote">
                 <summary>Source code in <code>pytorch_widedeep/models/tabular/transformers/ft_transformer.py</code></summary>
                 <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 11</span>
@@ -12966,7 +13534,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_perceiver.TabPerceiver"
       </thead>
       <tbody>
           <tr class="doc-section-item">
-            <td><code>column_idx</code></td>
+            <td>
+                <code>column_idx</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, int]</code>
             </td>
@@ -12982,7 +13552,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_perceiver.TabPerceiver"
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>cat_embed_input</code></td>
+            <td>
+                <code>cat_embed_input</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.wdtypes.Tuple">Tuple</span>[str, int]]]</code>
             </td>
@@ -12997,7 +13569,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_perceiver.TabPerceiver"
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>cat_embed_dropout</code></td>
+            <td>
+                <code>cat_embed_dropout</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>
             </td>
@@ -13012,7 +13586,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_perceiver.TabPerceiver"
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>use_cat_bias</code></td>
+            <td>
+                <code>use_cat_bias</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
             </td>
@@ -13027,7 +13603,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_perceiver.TabPerceiver"
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>cat_embed_activation</code></td>
+            <td>
+                <code>cat_embed_activation</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>
             </td>
@@ -13042,7 +13620,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_perceiver.TabPerceiver"
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>shared_embed</code></td>
+            <td>
+                <code>shared_embed</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
             </td>
@@ -13061,7 +13641,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_perceiver.TabPerceiver"
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>add_shared_embed</code></td>
+            <td>
+                <code>add_shared_embed</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
             </td>
@@ -13079,7 +13661,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_perceiver.TabPerceiver"
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>frac_shared_embed</code></td>
+            <td>
+                <code>frac_shared_embed</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>
             </td>
@@ -13095,7 +13679,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_perceiver.TabPerceiver"
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>continuous_cols</code></td>
+            <td>
+                <code>continuous_cols</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[str]]</code>
             </td>
@@ -13109,7 +13695,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_perceiver.TabPerceiver"
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>cont_norm_layer</code></td>
+            <td>
+                <code>cont_norm_layer</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[batchnorm, layernorm]]</code>
             </td>
@@ -13125,7 +13713,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_perceiver.TabPerceiver"
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>embed_continuous_method</code></td>
+            <td>
+                <code>embed_continuous_method</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[standard, piecewise, periodic]]</code>
             </td>
@@ -13146,7 +13736,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_perceiver.TabPerceiver"
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>cont_embed_dropout</code></td>
+            <td>
+                <code>cont_embed_dropout</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>
             </td>
@@ -13160,7 +13752,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_perceiver.TabPerceiver"
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>cont_embed_activation</code></td>
+            <td>
+                <code>cont_embed_activation</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>
             </td>
@@ -13176,7 +13770,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_perceiver.TabPerceiver"
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>quantization_setup</code></td>
+            <td>
+                <code>quantization_setup</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, <span title="pytorch_widedeep.wdtypes.List">List</span>[float]]]</code>
             </td>
@@ -13194,7 +13790,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_perceiver.TabPerceiver"
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>n_frequencies</code></td>
+            <td>
+                <code>n_frequencies</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[int]</code>
             </td>
@@ -13212,7 +13810,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_perceiver.TabPerceiver"
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>sigma</code></td>
+            <td>
+                <code>sigma</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>
             </td>
@@ -13229,7 +13829,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_perceiver.TabPerceiver"
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>share_last_layer</code></td>
+            <td>
+                <code>share_last_layer</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
             </td>
@@ -13248,7 +13850,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_perceiver.TabPerceiver"
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>full_embed_dropout</code></td>
+            <td>
+                <code>full_embed_dropout</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
             </td>
@@ -13263,7 +13867,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_perceiver.TabPerceiver"
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>input_dim</code></td>
+            <td>
+                <code>input_dim</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -13278,7 +13884,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_perceiver.TabPerceiver"
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>n_cross_attns</code></td>
+            <td>
+                <code>n_cross_attns</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -13298,7 +13906,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_perceiver.TabPerceiver"
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>n_cross_attn_heads</code></td>
+            <td>
+                <code>n_cross_attn_heads</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -13312,7 +13922,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_perceiver.TabPerceiver"
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>n_latents</code></td>
+            <td>
+                <code>n_latents</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -13330,7 +13942,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_perceiver.TabPerceiver"
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>latent_dim</code></td>
+            <td>
+                <code>latent_dim</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -13344,7 +13958,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_perceiver.TabPerceiver"
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>n_latent_heads</code></td>
+            <td>
+                <code>n_latent_heads</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -13358,7 +13974,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_perceiver.TabPerceiver"
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>n_latent_blocks</code></td>
+            <td>
+                <code>n_latent_blocks</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -13373,7 +13991,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_perceiver.TabPerceiver"
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>n_perceiver_blocks</code></td>
+            <td>
+                <code>n_perceiver_blocks</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -13388,7 +14008,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_perceiver.TabPerceiver"
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>share_weights</code></td>
+            <td>
+                <code>share_weights</code>
+            </td>
             <td>
                   <code>bool</code>
             </td>
@@ -13403,7 +14025,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_perceiver.TabPerceiver"
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>attn_dropout</code></td>
+            <td>
+                <code>attn_dropout</code>
+            </td>
             <td>
                   <code>float</code>
             </td>
@@ -13417,7 +14041,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_perceiver.TabPerceiver"
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>ff_dropout</code></td>
+            <td>
+                <code>ff_dropout</code>
+            </td>
             <td>
                   <code>float</code>
             </td>
@@ -13431,7 +14057,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_perceiver.TabPerceiver"
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>ff_factor</code></td>
+            <td>
+                <code>ff_factor</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -13446,7 +14074,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_perceiver.TabPerceiver"
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>transformer_activation</code></td>
+            <td>
+                <code>transformer_activation</code>
+            </td>
             <td>
                   <code>str</code>
             </td>
@@ -13461,7 +14091,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_perceiver.TabPerceiver"
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>mlp_hidden_dims</code></td>
+            <td>
+                <code>mlp_hidden_dims</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[int]]</code>
             </td>
@@ -13477,7 +14109,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_perceiver.TabPerceiver"
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>mlp_activation</code></td>
+            <td>
+                <code>mlp_activation</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>
             </td>
@@ -13494,7 +14128,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_perceiver.TabPerceiver"
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>mlp_dropout</code></td>
+            <td>
+                <code>mlp_dropout</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>
             </td>
@@ -13510,7 +14146,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_perceiver.TabPerceiver"
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>mlp_batchnorm</code></td>
+            <td>
+                <code>mlp_batchnorm</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
             </td>
@@ -13527,7 +14165,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_perceiver.TabPerceiver"
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>mlp_batchnorm_last</code></td>
+            <td>
+                <code>mlp_batchnorm_last</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
             </td>
@@ -13544,7 +14184,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_perceiver.TabPerceiver"
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>mlp_linear_first</code></td>
+            <td>
+                <code>mlp_linear_first</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
             </td>
@@ -13626,6 +14268,11 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_perceiver.TabPerceiver"
 <span class="gp">&gt;&gt;&gt; </span><span class="n">out</span> <span class="o">=</span> <span class="n">model</span><span class="p">(</span><span class="n">X_tab</span><span class="p">)</span>
 </code></pre></div>
 
+
+
+
+
+
               <details class="quote">
                 <summary>Source code in <code>pytorch_widedeep/models/tabular/transformers/tab_perceiver.py</code></summary>
                 <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 13</span>
@@ -14624,7 +15271,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_fastformer.TabFastForme
       </thead>
       <tbody>
           <tr class="doc-section-item">
-            <td><code>column_idx</code></td>
+            <td>
+                <code>column_idx</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, int]</code>
             </td>
@@ -14640,7 +15289,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_fastformer.TabFastForme
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>cat_embed_input</code></td>
+            <td>
+                <code>cat_embed_input</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.wdtypes.Tuple">Tuple</span>[str, int]]]</code>
             </td>
@@ -14655,7 +15306,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_fastformer.TabFastForme
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>cat_embed_dropout</code></td>
+            <td>
+                <code>cat_embed_dropout</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>
             </td>
@@ -14670,7 +15323,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_fastformer.TabFastForme
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>use_cat_bias</code></td>
+            <td>
+                <code>use_cat_bias</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
             </td>
@@ -14685,7 +15340,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_fastformer.TabFastForme
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>cat_embed_activation</code></td>
+            <td>
+                <code>cat_embed_activation</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>
             </td>
@@ -14700,7 +15357,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_fastformer.TabFastForme
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>shared_embed</code></td>
+            <td>
+                <code>shared_embed</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
             </td>
@@ -14719,7 +15378,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_fastformer.TabFastForme
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>add_shared_embed</code></td>
+            <td>
+                <code>add_shared_embed</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
             </td>
@@ -14737,7 +15398,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_fastformer.TabFastForme
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>frac_shared_embed</code></td>
+            <td>
+                <code>frac_shared_embed</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>
             </td>
@@ -14753,7 +15416,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_fastformer.TabFastForme
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>continuous_cols</code></td>
+            <td>
+                <code>continuous_cols</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[str]]</code>
             </td>
@@ -14767,7 +15432,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_fastformer.TabFastForme
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>cont_norm_layer</code></td>
+            <td>
+                <code>cont_norm_layer</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[batchnorm, layernorm]]</code>
             </td>
@@ -14783,7 +15450,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_fastformer.TabFastForme
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>embed_continuous_method</code></td>
+            <td>
+                <code>embed_continuous_method</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[standard, piecewise, periodic]]</code>
             </td>
@@ -14804,7 +15473,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_fastformer.TabFastForme
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>cont_embed_dropout</code></td>
+            <td>
+                <code>cont_embed_dropout</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>
             </td>
@@ -14818,7 +15489,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_fastformer.TabFastForme
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>cont_embed_activation</code></td>
+            <td>
+                <code>cont_embed_activation</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>
             </td>
@@ -14834,7 +15507,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_fastformer.TabFastForme
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>quantization_setup</code></td>
+            <td>
+                <code>quantization_setup</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, <span title="pytorch_widedeep.wdtypes.List">List</span>[float]]]</code>
             </td>
@@ -14852,7 +15527,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_fastformer.TabFastForme
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>n_frequencies</code></td>
+            <td>
+                <code>n_frequencies</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[int]</code>
             </td>
@@ -14870,7 +15547,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_fastformer.TabFastForme
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>sigma</code></td>
+            <td>
+                <code>sigma</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>
             </td>
@@ -14887,7 +15566,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_fastformer.TabFastForme
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>share_last_layer</code></td>
+            <td>
+                <code>share_last_layer</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
             </td>
@@ -14906,7 +15587,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_fastformer.TabFastForme
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>full_embed_dropout</code></td>
+            <td>
+                <code>full_embed_dropout</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
             </td>
@@ -14921,7 +15604,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_fastformer.TabFastForme
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>input_dim</code></td>
+            <td>
+                <code>input_dim</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -14936,7 +15621,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_fastformer.TabFastForme
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>n_heads</code></td>
+            <td>
+                <code>n_heads</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -14950,7 +15637,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_fastformer.TabFastForme
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>use_bias</code></td>
+            <td>
+                <code>use_bias</code>
+            </td>
             <td>
                   <code>bool</code>
             </td>
@@ -14965,7 +15654,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_fastformer.TabFastForme
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>n_blocks</code></td>
+            <td>
+                <code>n_blocks</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -14979,7 +15670,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_fastformer.TabFastForme
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>attn_dropout</code></td>
+            <td>
+                <code>attn_dropout</code>
+            </td>
             <td>
                   <code>float</code>
             </td>
@@ -14993,7 +15686,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_fastformer.TabFastForme
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>ff_dropout</code></td>
+            <td>
+                <code>ff_dropout</code>
+            </td>
             <td>
                   <code>float</code>
             </td>
@@ -15007,7 +15702,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_fastformer.TabFastForme
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>ff_factor</code></td>
+            <td>
+                <code>ff_factor</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -15022,7 +15719,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_fastformer.TabFastForme
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>share_qv_weights</code></td>
+            <td>
+                <code>share_qv_weights</code>
+            </td>
             <td>
                   <code>bool</code>
             </td>
@@ -15037,7 +15736,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_fastformer.TabFastForme
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>share_weights</code></td>
+            <td>
+                <code>share_weights</code>
+            </td>
             <td>
                   <code>bool</code>
             </td>
@@ -15055,7 +15756,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_fastformer.TabFastForme
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>transformer_activation</code></td>
+            <td>
+                <code>transformer_activation</code>
+            </td>
             <td>
                   <code>str</code>
             </td>
@@ -15070,7 +15773,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_fastformer.TabFastForme
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>mlp_hidden_dims</code></td>
+            <td>
+                <code>mlp_hidden_dims</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[int]]</code>
             </td>
@@ -15085,7 +15790,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_fastformer.TabFastForme
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>mlp_hidden_dims</code></td>
+            <td>
+                <code>mlp_hidden_dims</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[int]]</code>
             </td>
@@ -15101,7 +15808,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_fastformer.TabFastForme
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>mlp_activation</code></td>
+            <td>
+                <code>mlp_activation</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>
             </td>
@@ -15118,7 +15827,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_fastformer.TabFastForme
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>mlp_dropout</code></td>
+            <td>
+                <code>mlp_dropout</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>
             </td>
@@ -15134,7 +15845,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_fastformer.TabFastForme
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>mlp_batchnorm</code></td>
+            <td>
+                <code>mlp_batchnorm</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
             </td>
@@ -15151,7 +15864,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_fastformer.TabFastForme
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>mlp_batchnorm_last</code></td>
+            <td>
+                <code>mlp_batchnorm_last</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
             </td>
@@ -15168,7 +15883,9 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_fastformer.TabFastForme
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>mlp_linear_first</code></td>
+            <td>
+                <code>mlp_linear_first</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
             </td>
@@ -15237,6 +15954,11 @@ <h2 id="pytorch_widedeep.models.tabular.transformers.tab_fastformer.TabFastForme
 <span class="gp">&gt;&gt;&gt; </span><span class="n">out</span> <span class="o">=</span> <span class="n">model</span><span class="p">(</span><span class="n">X_tab</span><span class="p">)</span>
 </code></pre></div>
 
+
+
+
+
+
               <details class="quote">
                 <summary>Source code in <code>pytorch_widedeep/models/tabular/transformers/tab_fastformer.py</code></summary>
                 <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 11</span>
@@ -16070,7 +16792,9 @@ <h2 id="pytorch_widedeep.models.text.rnns.basic_rnn.BasicRNN" class="doc doc-hea
       </thead>
       <tbody>
           <tr class="doc-section-item">
-            <td><code>vocab_size</code></td>
+            <td>
+                <code>vocab_size</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -16084,7 +16808,9 @@ <h2 id="pytorch_widedeep.models.text.rnns.basic_rnn.BasicRNN" class="doc doc-hea
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>embed_dim</code></td>
+            <td>
+                <code>embed_dim</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[int]</code>
             </td>
@@ -16099,7 +16825,9 @@ <h2 id="pytorch_widedeep.models.text.rnns.basic_rnn.BasicRNN" class="doc doc-hea
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>embed_matrix</code></td>
+            <td>
+                <code>embed_matrix</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="numpy.ndarray">ndarray</span>]</code>
             </td>
@@ -16113,7 +16841,9 @@ <h2 id="pytorch_widedeep.models.text.rnns.basic_rnn.BasicRNN" class="doc doc-hea
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>embed_trainable</code></td>
+            <td>
+                <code>embed_trainable</code>
+            </td>
             <td>
                   <code>bool</code>
             </td>
@@ -16127,7 +16857,9 @@ <h2 id="pytorch_widedeep.models.text.rnns.basic_rnn.BasicRNN" class="doc doc-hea
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>rnn_type</code></td>
+            <td>
+                <code>rnn_type</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[lstm, gru]</code>
             </td>
@@ -16141,7 +16873,9 @@ <h2 id="pytorch_widedeep.models.text.rnns.basic_rnn.BasicRNN" class="doc doc-hea
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>hidden_dim</code></td>
+            <td>
+                <code>hidden_dim</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -16155,7 +16889,9 @@ <h2 id="pytorch_widedeep.models.text.rnns.basic_rnn.BasicRNN" class="doc doc-hea
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>n_layers</code></td>
+            <td>
+                <code>n_layers</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -16169,7 +16905,9 @@ <h2 id="pytorch_widedeep.models.text.rnns.basic_rnn.BasicRNN" class="doc doc-hea
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>rnn_dropout</code></td>
+            <td>
+                <code>rnn_dropout</code>
+            </td>
             <td>
                   <code>float</code>
             </td>
@@ -16183,7 +16921,9 @@ <h2 id="pytorch_widedeep.models.text.rnns.basic_rnn.BasicRNN" class="doc doc-hea
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>bidirectional</code></td>
+            <td>
+                <code>bidirectional</code>
+            </td>
             <td>
                   <code>bool</code>
             </td>
@@ -16197,7 +16937,9 @@ <h2 id="pytorch_widedeep.models.text.rnns.basic_rnn.BasicRNN" class="doc doc-hea
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>use_hidden_state</code></td>
+            <td>
+                <code>use_hidden_state</code>
+            </td>
             <td>
                   <code>bool</code>
             </td>
@@ -16212,7 +16954,9 @@ <h2 id="pytorch_widedeep.models.text.rnns.basic_rnn.BasicRNN" class="doc doc-hea
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>padding_idx</code></td>
+            <td>
+                <code>padding_idx</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -16229,7 +16973,9 @@ <h2 id="pytorch_widedeep.models.text.rnns.basic_rnn.BasicRNN" class="doc doc-hea
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>head_hidden_dims</code></td>
+            <td>
+                <code>head_hidden_dims</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[int]]</code>
             </td>
@@ -16243,7 +16989,9 @@ <h2 id="pytorch_widedeep.models.text.rnns.basic_rnn.BasicRNN" class="doc doc-hea
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>head_activation</code></td>
+            <td>
+                <code>head_activation</code>
+            </td>
             <td>
                   <code>str</code>
             </td>
@@ -16258,7 +17006,9 @@ <h2 id="pytorch_widedeep.models.text.rnns.basic_rnn.BasicRNN" class="doc doc-hea
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>head_dropout</code></td>
+            <td>
+                <code>head_dropout</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>
             </td>
@@ -16272,7 +17022,9 @@ <h2 id="pytorch_widedeep.models.text.rnns.basic_rnn.BasicRNN" class="doc doc-hea
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>head_batchnorm</code></td>
+            <td>
+                <code>head_batchnorm</code>
+            </td>
             <td>
                   <code>bool</code>
             </td>
@@ -16287,7 +17039,9 @@ <h2 id="pytorch_widedeep.models.text.rnns.basic_rnn.BasicRNN" class="doc doc-hea
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>head_batchnorm_last</code></td>
+            <td>
+                <code>head_batchnorm_last</code>
+            </td>
             <td>
                   <code>bool</code>
             </td>
@@ -16302,7 +17056,9 @@ <h2 id="pytorch_widedeep.models.text.rnns.basic_rnn.BasicRNN" class="doc doc-hea
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>head_linear_first</code></td>
+            <td>
+                <code>head_linear_first</code>
+            </td>
             <td>
                   <code>bool</code>
             </td>
@@ -16377,6 +17133,11 @@ <h2 id="pytorch_widedeep.models.text.rnns.basic_rnn.BasicRNN" class="doc doc-hea
 <span class="gp">&gt;&gt;&gt; </span><span class="n">out</span> <span class="o">=</span> <span class="n">model</span><span class="p">(</span><span class="n">X_text</span><span class="p">)</span>
 </code></pre></div>
 
+
+
+
+
+
               <details class="quote">
                 <summary>Source code in <code>pytorch_widedeep/models/text/rnns/basic_rnn.py</code></summary>
                 <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 12</span>
@@ -16967,7 +17728,9 @@ <h2 id="pytorch_widedeep.models.text.rnns.attentive_rnn.AttentiveRNN" class="doc
       </thead>
       <tbody>
           <tr class="doc-section-item">
-            <td><code>vocab_size</code></td>
+            <td>
+                <code>vocab_size</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -16981,7 +17744,9 @@ <h2 id="pytorch_widedeep.models.text.rnns.attentive_rnn.AttentiveRNN" class="doc
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>embed_dim</code></td>
+            <td>
+                <code>embed_dim</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[int]</code>
             </td>
@@ -16996,7 +17761,9 @@ <h2 id="pytorch_widedeep.models.text.rnns.attentive_rnn.AttentiveRNN" class="doc
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>embed_matrix</code></td>
+            <td>
+                <code>embed_matrix</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="numpy.ndarray">ndarray</span>]</code>
             </td>
@@ -17010,7 +17777,9 @@ <h2 id="pytorch_widedeep.models.text.rnns.attentive_rnn.AttentiveRNN" class="doc
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>embed_trainable</code></td>
+            <td>
+                <code>embed_trainable</code>
+            </td>
             <td>
                   <code>bool</code>
             </td>
@@ -17024,7 +17793,9 @@ <h2 id="pytorch_widedeep.models.text.rnns.attentive_rnn.AttentiveRNN" class="doc
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>rnn_type</code></td>
+            <td>
+                <code>rnn_type</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[lstm, gru]</code>
             </td>
@@ -17038,7 +17809,9 @@ <h2 id="pytorch_widedeep.models.text.rnns.attentive_rnn.AttentiveRNN" class="doc
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>hidden_dim</code></td>
+            <td>
+                <code>hidden_dim</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -17052,7 +17825,9 @@ <h2 id="pytorch_widedeep.models.text.rnns.attentive_rnn.AttentiveRNN" class="doc
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>n_layers</code></td>
+            <td>
+                <code>n_layers</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -17066,7 +17841,9 @@ <h2 id="pytorch_widedeep.models.text.rnns.attentive_rnn.AttentiveRNN" class="doc
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>rnn_dropout</code></td>
+            <td>
+                <code>rnn_dropout</code>
+            </td>
             <td>
                   <code>float</code>
             </td>
@@ -17080,7 +17857,9 @@ <h2 id="pytorch_widedeep.models.text.rnns.attentive_rnn.AttentiveRNN" class="doc
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>bidirectional</code></td>
+            <td>
+                <code>bidirectional</code>
+            </td>
             <td>
                   <code>bool</code>
             </td>
@@ -17094,7 +17873,9 @@ <h2 id="pytorch_widedeep.models.text.rnns.attentive_rnn.AttentiveRNN" class="doc
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>use_hidden_state</code></td>
+            <td>
+                <code>use_hidden_state</code>
+            </td>
             <td>
                   <code>bool</code>
             </td>
@@ -17109,7 +17890,9 @@ <h2 id="pytorch_widedeep.models.text.rnns.attentive_rnn.AttentiveRNN" class="doc
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>padding_idx</code></td>
+            <td>
+                <code>padding_idx</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -17126,7 +17909,9 @@ <h2 id="pytorch_widedeep.models.text.rnns.attentive_rnn.AttentiveRNN" class="doc
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>attn_concatenate</code></td>
+            <td>
+                <code>attn_concatenate</code>
+            </td>
             <td>
                   <code>bool</code>
             </td>
@@ -17142,7 +17927,9 @@ <h2 id="pytorch_widedeep.models.text.rnns.attentive_rnn.AttentiveRNN" class="doc
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>attn_dropout</code></td>
+            <td>
+                <code>attn_dropout</code>
+            </td>
             <td>
                   <code>float</code>
             </td>
@@ -17156,7 +17943,9 @@ <h2 id="pytorch_widedeep.models.text.rnns.attentive_rnn.AttentiveRNN" class="doc
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>head_hidden_dims</code></td>
+            <td>
+                <code>head_hidden_dims</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[int]]</code>
             </td>
@@ -17170,7 +17959,9 @@ <h2 id="pytorch_widedeep.models.text.rnns.attentive_rnn.AttentiveRNN" class="doc
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>head_activation</code></td>
+            <td>
+                <code>head_activation</code>
+            </td>
             <td>
                   <code>str</code>
             </td>
@@ -17185,7 +17976,9 @@ <h2 id="pytorch_widedeep.models.text.rnns.attentive_rnn.AttentiveRNN" class="doc
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>head_dropout</code></td>
+            <td>
+                <code>head_dropout</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>
             </td>
@@ -17199,7 +17992,9 @@ <h2 id="pytorch_widedeep.models.text.rnns.attentive_rnn.AttentiveRNN" class="doc
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>head_batchnorm</code></td>
+            <td>
+                <code>head_batchnorm</code>
+            </td>
             <td>
                   <code>bool</code>
             </td>
@@ -17214,7 +18009,9 @@ <h2 id="pytorch_widedeep.models.text.rnns.attentive_rnn.AttentiveRNN" class="doc
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>head_batchnorm_last</code></td>
+            <td>
+                <code>head_batchnorm_last</code>
+            </td>
             <td>
                   <code>bool</code>
             </td>
@@ -17229,7 +18026,9 @@ <h2 id="pytorch_widedeep.models.text.rnns.attentive_rnn.AttentiveRNN" class="doc
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>head_linear_first</code></td>
+            <td>
+                <code>head_linear_first</code>
+            </td>
             <td>
                   <code>bool</code>
             </td>
@@ -17304,6 +18103,11 @@ <h2 id="pytorch_widedeep.models.text.rnns.attentive_rnn.AttentiveRNN" class="doc
 <span class="gp">&gt;&gt;&gt; </span><span class="n">out</span> <span class="o">=</span> <span class="n">model</span><span class="p">(</span><span class="n">X_text</span><span class="p">)</span>
 </code></pre></div>
 
+
+
+
+
+
               <details class="quote">
                 <summary>Source code in <code>pytorch_widedeep/models/text/rnns/attentive_rnn.py</code></summary>
                 <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 10</span>
@@ -17738,7 +18542,9 @@ <h2 id="pytorch_widedeep.models.text.rnns.stacked_attentive_rnn.StackedAttentive
       </thead>
       <tbody>
           <tr class="doc-section-item">
-            <td><code>vocab_size</code></td>
+            <td>
+                <code>vocab_size</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -17752,7 +18558,9 @@ <h2 id="pytorch_widedeep.models.text.rnns.stacked_attentive_rnn.StackedAttentive
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>embed_dim</code></td>
+            <td>
+                <code>embed_dim</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[int]</code>
             </td>
@@ -17767,7 +18575,9 @@ <h2 id="pytorch_widedeep.models.text.rnns.stacked_attentive_rnn.StackedAttentive
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>embed_matrix</code></td>
+            <td>
+                <code>embed_matrix</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="numpy.ndarray">ndarray</span>]</code>
             </td>
@@ -17781,7 +18591,9 @@ <h2 id="pytorch_widedeep.models.text.rnns.stacked_attentive_rnn.StackedAttentive
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>embed_trainable</code></td>
+            <td>
+                <code>embed_trainable</code>
+            </td>
             <td>
                   <code>bool</code>
             </td>
@@ -17795,7 +18607,9 @@ <h2 id="pytorch_widedeep.models.text.rnns.stacked_attentive_rnn.StackedAttentive
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>rnn_type</code></td>
+            <td>
+                <code>rnn_type</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[lstm, gru]</code>
             </td>
@@ -17809,7 +18623,9 @@ <h2 id="pytorch_widedeep.models.text.rnns.stacked_attentive_rnn.StackedAttentive
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>hidden_dim</code></td>
+            <td>
+                <code>hidden_dim</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -17823,7 +18639,9 @@ <h2 id="pytorch_widedeep.models.text.rnns.stacked_attentive_rnn.StackedAttentive
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>bidirectional</code></td>
+            <td>
+                <code>bidirectional</code>
+            </td>
             <td>
                   <code>bool</code>
             </td>
@@ -17837,7 +18655,9 @@ <h2 id="pytorch_widedeep.models.text.rnns.stacked_attentive_rnn.StackedAttentive
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>padding_idx</code></td>
+            <td>
+                <code>padding_idx</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -17854,7 +18674,9 @@ <h2 id="pytorch_widedeep.models.text.rnns.stacked_attentive_rnn.StackedAttentive
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>n_blocks</code></td>
+            <td>
+                <code>n_blocks</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -17869,7 +18691,9 @@ <h2 id="pytorch_widedeep.models.text.rnns.stacked_attentive_rnn.StackedAttentive
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>attn_concatenate</code></td>
+            <td>
+                <code>attn_concatenate</code>
+            </td>
             <td>
                   <code>bool</code>
             </td>
@@ -17885,7 +18709,9 @@ <h2 id="pytorch_widedeep.models.text.rnns.stacked_attentive_rnn.StackedAttentive
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>attn_dropout</code></td>
+            <td>
+                <code>attn_dropout</code>
+            </td>
             <td>
                   <code>float</code>
             </td>
@@ -17899,7 +18725,9 @@ <h2 id="pytorch_widedeep.models.text.rnns.stacked_attentive_rnn.StackedAttentive
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>with_addnorm</code></td>
+            <td>
+                <code>with_addnorm</code>
+            </td>
             <td>
                   <code>bool</code>
             </td>
@@ -17914,7 +18742,9 @@ <h2 id="pytorch_widedeep.models.text.rnns.stacked_attentive_rnn.StackedAttentive
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>head_hidden_dims</code></td>
+            <td>
+                <code>head_hidden_dims</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[int]]</code>
             </td>
@@ -17928,7 +18758,9 @@ <h2 id="pytorch_widedeep.models.text.rnns.stacked_attentive_rnn.StackedAttentive
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>head_activation</code></td>
+            <td>
+                <code>head_activation</code>
+            </td>
             <td>
                   <code>str</code>
             </td>
@@ -17943,7 +18775,9 @@ <h2 id="pytorch_widedeep.models.text.rnns.stacked_attentive_rnn.StackedAttentive
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>head_dropout</code></td>
+            <td>
+                <code>head_dropout</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>
             </td>
@@ -17957,7 +18791,9 @@ <h2 id="pytorch_widedeep.models.text.rnns.stacked_attentive_rnn.StackedAttentive
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>head_batchnorm</code></td>
+            <td>
+                <code>head_batchnorm</code>
+            </td>
             <td>
                   <code>bool</code>
             </td>
@@ -17972,7 +18808,9 @@ <h2 id="pytorch_widedeep.models.text.rnns.stacked_attentive_rnn.StackedAttentive
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>head_batchnorm_last</code></td>
+            <td>
+                <code>head_batchnorm_last</code>
+            </td>
             <td>
                   <code>bool</code>
             </td>
@@ -17987,7 +18825,9 @@ <h2 id="pytorch_widedeep.models.text.rnns.stacked_attentive_rnn.StackedAttentive
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>head_linear_first</code></td>
+            <td>
+                <code>head_linear_first</code>
+            </td>
             <td>
                   <code>bool</code>
             </td>
@@ -18062,6 +18902,11 @@ <h2 id="pytorch_widedeep.models.text.rnns.stacked_attentive_rnn.StackedAttentive
 <span class="gp">&gt;&gt;&gt; </span><span class="n">out</span> <span class="o">=</span> <span class="n">model</span><span class="p">(</span><span class="n">X_text</span><span class="p">)</span>
 </code></pre></div>
 
+
+
+
+
+
               <details class="quote">
                 <summary>Source code in <code>pytorch_widedeep/models/text/rnns/stacked_attentive_rnn.py</code></summary>
                 <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 13</span>
@@ -18715,7 +19560,9 @@ <h2 id="pytorch_widedeep.models.text.huggingface_transformers.hf_model.HFModel"
       </thead>
       <tbody>
           <tr class="doc-section-item">
-            <td><code>model_name</code></td>
+            <td>
+                <code>model_name</code>
+            </td>
             <td>
                   <code>str</code>
             </td>
@@ -18731,7 +19578,9 @@ <h2 id="pytorch_widedeep.models.text.huggingface_transformers.hf_model.HFModel"
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>use_cls_token</code></td>
+            <td>
+                <code>use_cls_token</code>
+            </td>
             <td>
                   <code>bool</code>
             </td>
@@ -18746,7 +19595,9 @@ <h2 id="pytorch_widedeep.models.text.huggingface_transformers.hf_model.HFModel"
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>trainable_parameters</code></td>
+            <td>
+                <code>trainable_parameters</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[str]]</code>
             </td>
@@ -18761,7 +19612,9 @@ <h2 id="pytorch_widedeep.models.text.huggingface_transformers.hf_model.HFModel"
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>head_hidden_dims</code></td>
+            <td>
+                <code>head_hidden_dims</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[int]]</code>
             </td>
@@ -18775,7 +19628,9 @@ <h2 id="pytorch_widedeep.models.text.huggingface_transformers.hf_model.HFModel"
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>head_activation</code></td>
+            <td>
+                <code>head_activation</code>
+            </td>
             <td>
                   <code>str</code>
             </td>
@@ -18790,7 +19645,9 @@ <h2 id="pytorch_widedeep.models.text.huggingface_transformers.hf_model.HFModel"
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>head_dropout</code></td>
+            <td>
+                <code>head_dropout</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>
             </td>
@@ -18804,7 +19661,9 @@ <h2 id="pytorch_widedeep.models.text.huggingface_transformers.hf_model.HFModel"
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>head_batchnorm</code></td>
+            <td>
+                <code>head_batchnorm</code>
+            </td>
             <td>
                   <code>bool</code>
             </td>
@@ -18819,7 +19678,9 @@ <h2 id="pytorch_widedeep.models.text.huggingface_transformers.hf_model.HFModel"
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>head_batchnorm_last</code></td>
+            <td>
+                <code>head_batchnorm_last</code>
+            </td>
             <td>
                   <code>bool</code>
             </td>
@@ -18834,7 +19695,9 @@ <h2 id="pytorch_widedeep.models.text.huggingface_transformers.hf_model.HFModel"
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>head_linear_first</code></td>
+            <td>
+                <code>head_linear_first</code>
+            </td>
             <td>
                   <code>bool</code>
             </td>
@@ -18850,7 +19713,9 @@ <h2 id="pytorch_widedeep.models.text.huggingface_transformers.hf_model.HFModel"
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>verbose</code></td>
+            <td>
+                <code>verbose</code>
+            </td>
             <td>
                   <code>bool</code>
             </td>
@@ -18864,7 +19729,9 @@ <h2 id="pytorch_widedeep.models.text.huggingface_transformers.hf_model.HFModel"
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>**kwargs</code></td>
+            <td>
+                <code>**kwargs</code>
+            </td>
             <td>
             </td>
             <td>
@@ -18914,6 +19781,11 @@ <h2 id="pytorch_widedeep.models.text.huggingface_transformers.hf_model.HFModel"
 <span class="gp">&gt;&gt;&gt; </span><span class="n">out</span> <span class="o">=</span> <span class="n">model</span><span class="p">(</span><span class="n">X_text</span><span class="p">)</span>
 </code></pre></div>
 
+
+
+
+
+
               <details class="quote">
                 <summary>Source code in <code>pytorch_widedeep/models/text/huggingface_transformers/hf_model.py</code></summary>
                 <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 12</span>
@@ -19349,7 +20221,9 @@ <h2 id="pytorch_widedeep.models.image.vision.Vision" class="doc doc-heading">
       </thead>
       <tbody>
           <tr class="doc-section-item">
-            <td><code>pretrained_model_setup</code></td>
+            <td>
+                <code>pretrained_model_setup</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Union">Union</span>[str, <span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, <span title="pytorch_widedeep.wdtypes.Union">Union</span>[str, <span title="pytorch_widedeep.wdtypes.WeightsEnum">WeightsEnum</span>]]]</code>
             </td>
@@ -19372,7 +20246,9 @@ <h2 id="pytorch_widedeep.models.image.vision.Vision" class="doc doc-heading">
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>n_trainable</code></td>
+            <td>
+                <code>n_trainable</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[int]</code>
             </td>
@@ -19389,7 +20265,9 @@ <h2 id="pytorch_widedeep.models.image.vision.Vision" class="doc doc-heading">
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>trainable_params</code></td>
+            <td>
+                <code>trainable_params</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[str]]</code>
             </td>
@@ -19409,7 +20287,9 @@ <h2 id="pytorch_widedeep.models.image.vision.Vision" class="doc doc-heading">
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>channel_sizes</code></td>
+            <td>
+                <code>channel_sizes</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.List">List</span>[int]</code>
             </td>
@@ -19424,7 +20304,9 @@ <h2 id="pytorch_widedeep.models.image.vision.Vision" class="doc doc-heading">
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>kernel_sizes</code></td>
+            <td>
+                <code>kernel_sizes</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Union">Union</span>[int, <span title="pytorch_widedeep.wdtypes.List">List</span>[int]]</code>
             </td>
@@ -19439,7 +20321,9 @@ <h2 id="pytorch_widedeep.models.image.vision.Vision" class="doc doc-heading">
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>strides</code></td>
+            <td>
+                <code>strides</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Union">Union</span>[int, <span title="pytorch_widedeep.wdtypes.List">List</span>[int]]</code>
             </td>
@@ -19454,7 +20338,9 @@ <h2 id="pytorch_widedeep.models.image.vision.Vision" class="doc doc-heading">
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>head_hidden_dims</code></td>
+            <td>
+                <code>head_hidden_dims</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[int]]</code>
             </td>
@@ -19468,7 +20354,9 @@ <h2 id="pytorch_widedeep.models.image.vision.Vision" class="doc doc-heading">
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>head_activation</code></td>
+            <td>
+                <code>head_activation</code>
+            </td>
             <td>
                   <code>str</code>
             </td>
@@ -19483,7 +20371,9 @@ <h2 id="pytorch_widedeep.models.image.vision.Vision" class="doc doc-heading">
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>head_dropout</code></td>
+            <td>
+                <code>head_dropout</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Union">Union</span>[float, <span title="pytorch_widedeep.wdtypes.List">List</span>[float]]</code>
             </td>
@@ -19497,7 +20387,9 @@ <h2 id="pytorch_widedeep.models.image.vision.Vision" class="doc doc-heading">
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>head_batchnorm</code></td>
+            <td>
+                <code>head_batchnorm</code>
+            </td>
             <td>
                   <code>bool</code>
             </td>
@@ -19512,7 +20404,9 @@ <h2 id="pytorch_widedeep.models.image.vision.Vision" class="doc doc-heading">
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>head_batchnorm_last</code></td>
+            <td>
+                <code>head_batchnorm_last</code>
+            </td>
             <td>
                   <code>bool</code>
             </td>
@@ -19527,7 +20421,9 @@ <h2 id="pytorch_widedeep.models.image.vision.Vision" class="doc doc-heading">
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>head_linear_first</code></td>
+            <td>
+                <code>head_linear_first</code>
+            </td>
             <td>
                   <code>bool</code>
             </td>
@@ -19579,6 +20475,11 @@ <h2 id="pytorch_widedeep.models.image.vision.Vision" class="doc doc-heading">
 <span class="gp">&gt;&gt;&gt; </span><span class="n">out</span> <span class="o">=</span> <span class="n">model</span><span class="p">(</span><span class="n">X_img</span><span class="p">)</span>
 </code></pre></div>
 
+
+
+
+
+
               <details class="quote">
                 <summary>Source code in <code>pytorch_widedeep/models/image/vision.py</code></summary>
                 <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 42</span>
@@ -20208,7 +21109,9 @@ <h2 id="pytorch_widedeep.models.model_fusion.ModelFuser" class="doc doc-heading"
       </thead>
       <tbody>
           <tr class="doc-section-item">
-            <td><code>models</code></td>
+            <td>
+                <code>models</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.models._base_wd_model_component.BaseWDModelComponent">BaseWDModelComponent</span>]</code>
             </td>
@@ -20222,7 +21125,9 @@ <h2 id="pytorch_widedeep.models.model_fusion.ModelFuser" class="doc doc-heading"
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>fusion_method</code></td>
+            <td>
+                <code>fusion_method</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[concatenate, mean, max, sum, mult, dot, <span title="pytorch_widedeep.models.model_fusion.ModelFuser.head">head</span>], <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[concatenate, mean, max, sum, mult, <span title="pytorch_widedeep.models.model_fusion.ModelFuser.head">head</span>]]]</code>
             </td>
@@ -20240,7 +21145,9 @@ <h2 id="pytorch_widedeep.models.model_fusion.ModelFuser" class="doc doc-heading"
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>projection_method</code></td>
+            <td>
+                <code>projection_method</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[min, max, mean]]</code>
             </td>
@@ -20256,7 +21163,9 @@ <h2 id="pytorch_widedeep.models.model_fusion.ModelFuser" class="doc doc-heading"
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>custom_head</code></td>
+            <td>
+                <code>custom_head</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="pytorch_widedeep.models._base_wd_model_component.BaseWDModelComponent">BaseWDModelComponent</span>, <span title="torch.nn.Module">Module</span>]]</code>
             </td>
@@ -20272,7 +21181,9 @@ <h2 id="pytorch_widedeep.models.model_fusion.ModelFuser" class="doc doc-heading"
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>head_hidden_dims</code></td>
+            <td>
+                <code>head_hidden_dims</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[int]]</code>
             </td>
@@ -20287,7 +21198,9 @@ <h2 id="pytorch_widedeep.models.model_fusion.ModelFuser" class="doc doc-heading"
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>head_activation</code></td>
+            <td>
+                <code>head_activation</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>
             </td>
@@ -20301,7 +21214,9 @@ <h2 id="pytorch_widedeep.models.model_fusion.ModelFuser" class="doc doc-heading"
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>head_dropout</code></td>
+            <td>
+                <code>head_dropout</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>
             </td>
@@ -20315,7 +21230,9 @@ <h2 id="pytorch_widedeep.models.model_fusion.ModelFuser" class="doc doc-heading"
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>head_batchnorm</code></td>
+            <td>
+                <code>head_batchnorm</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
             </td>
@@ -20329,7 +21246,9 @@ <h2 id="pytorch_widedeep.models.model_fusion.ModelFuser" class="doc doc-heading"
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>head_batchnorm_last</code></td>
+            <td>
+                <code>head_batchnorm_last</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
             </td>
@@ -20344,7 +21263,9 @@ <h2 id="pytorch_widedeep.models.model_fusion.ModelFuser" class="doc doc-heading"
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>head_linear_first</code></td>
+            <td>
+                <code>head_linear_first</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[bool]</code>
             </td>
@@ -20432,6 +21353,11 @@ <h2 id="pytorch_widedeep.models.model_fusion.ModelFuser" class="doc doc-heading"
 <span class="gp">&gt;&gt;&gt; </span><span class="n">out</span> <span class="o">=</span> <span class="n">fused_model</span><span class="p">([</span><span class="n">X_text1_tnsr</span><span class="p">,</span> <span class="n">X_text2_tnsr</span><span class="p">])</span>
 </code></pre></div>
 
+
+
+
+
+
               <details class="quote">
                 <summary>Source code in <code>pytorch_widedeep/models/model_fusion.py</code></summary>
                 <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 10</span>
@@ -21256,7 +22182,9 @@ <h2 id="pytorch_widedeep.models.wide_deep.WideDeep" class="doc doc-heading">
       </thead>
       <tbody>
           <tr class="doc-section-item">
-            <td><code>wide</code></td>
+            <td>
+                <code>wide</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="torch.nn.Module">Module</span>]</code>
             </td>
@@ -21271,7 +22199,9 @@ <h2 id="pytorch_widedeep.models.wide_deep.WideDeep" class="doc doc-heading">
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>deeptabular</code></td>
+            <td>
+                <code>deeptabular</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="pytorch_widedeep.models._base_wd_model_component.BaseWDModelComponent">BaseWDModelComponent</span>, <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.models._base_wd_model_component.BaseWDModelComponent">BaseWDModelComponent</span>]]]</code>
             </td>
@@ -21289,7 +22219,9 @@ <h2 id="pytorch_widedeep.models.wide_deep.WideDeep" class="doc doc-heading">
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>deeptext</code></td>
+            <td>
+                <code>deeptext</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="pytorch_widedeep.models._base_wd_model_component.BaseWDModelComponent">BaseWDModelComponent</span>, <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.models._base_wd_model_component.BaseWDModelComponent">BaseWDModelComponent</span>]]]</code>
             </td>
@@ -21306,7 +22238,9 @@ <h2 id="pytorch_widedeep.models.wide_deep.WideDeep" class="doc doc-heading">
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>deepimage</code></td>
+            <td>
+                <code>deepimage</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="pytorch_widedeep.models._base_wd_model_component.BaseWDModelComponent">BaseWDModelComponent</span>, <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.models._base_wd_model_component.BaseWDModelComponent">BaseWDModelComponent</span>]]]</code>
             </td>
@@ -21323,7 +22257,9 @@ <h2 id="pytorch_widedeep.models.wide_deep.WideDeep" class="doc doc-heading">
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>deephead</code></td>
+            <td>
+                <code>deephead</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.models._base_wd_model_component.BaseWDModelComponent">BaseWDModelComponent</span>]</code>
             </td>
@@ -21339,7 +22275,9 @@ <h2 id="pytorch_widedeep.models.wide_deep.WideDeep" class="doc doc-heading">
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>head_hidden_dims</code></td>
+            <td>
+                <code>head_hidden_dims</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[int]]</code>
             </td>
@@ -21353,7 +22291,9 @@ <h2 id="pytorch_widedeep.models.wide_deep.WideDeep" class="doc doc-heading">
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>head_activation</code></td>
+            <td>
+                <code>head_activation</code>
+            </td>
             <td>
                   <code>str</code>
             </td>
@@ -21368,7 +22308,9 @@ <h2 id="pytorch_widedeep.models.wide_deep.WideDeep" class="doc doc-heading">
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>head_dropout</code></td>
+            <td>
+                <code>head_dropout</code>
+            </td>
             <td>
                   <code>float</code>
             </td>
@@ -21382,7 +22324,9 @@ <h2 id="pytorch_widedeep.models.wide_deep.WideDeep" class="doc doc-heading">
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>head_batchnorm</code></td>
+            <td>
+                <code>head_batchnorm</code>
+            </td>
             <td>
                   <code>bool</code>
             </td>
@@ -21397,7 +22341,9 @@ <h2 id="pytorch_widedeep.models.wide_deep.WideDeep" class="doc doc-heading">
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>head_batchnorm_last</code></td>
+            <td>
+                <code>head_batchnorm_last</code>
+            </td>
             <td>
                   <code>bool</code>
             </td>
@@ -21412,7 +22358,9 @@ <h2 id="pytorch_widedeep.models.wide_deep.WideDeep" class="doc doc-heading">
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>head_linear_first</code></td>
+            <td>
+                <code>head_linear_first</code>
+            </td>
             <td>
                   <code>bool</code>
             </td>
@@ -21428,7 +22376,9 @@ <h2 id="pytorch_widedeep.models.wide_deep.WideDeep" class="doc doc-heading">
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>enforce_positive</code></td>
+            <td>
+                <code>enforce_positive</code>
+            </td>
             <td>
                   <code>bool</code>
             </td>
@@ -21445,7 +22395,9 @@ <h2 id="pytorch_widedeep.models.wide_deep.WideDeep" class="doc doc-heading">
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>enforce_positive_activation</code></td>
+            <td>
+                <code>enforce_positive_activation</code>
+            </td>
             <td>
                   <code>str</code>
             </td>
@@ -21460,7 +22412,9 @@ <h2 id="pytorch_widedeep.models.wide_deep.WideDeep" class="doc doc-heading">
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>pred_dim</code></td>
+            <td>
+                <code>pred_dim</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -21498,6 +22452,11 @@ <h2 id="pytorch_widedeep.models.wide_deep.WideDeep" class="doc doc-heading">
  the size of these last layers of activations. See for example
  <code>pytorch_widedeep.models.tab_mlp.TabMlp</code></p>
 
+
+
+
+
+
               <details class="quote">
                 <summary>Source code in <code>pytorch_widedeep/models/wide_deep.py</code></summary>
                 <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 19</span>
@@ -22700,7 +23659,7 @@ <h2 id="pytorch_widedeep.models.wide_deep.WideDeep" class="doc doc-heading">
     <script id="__config" type="application/json">{"base": "..", "features": ["navigation.tabs", "navigation.tabs.sticky", "navigation.indexes", "navigation.expand", "toc.integrate"], "search": "../assets/javascripts/workers/search.6ce7567c.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version": "Select version"}}</script>
     
     
-      <script src="../assets/javascripts/bundle.56dfad97.min.js"></script>
+      <script src="../assets/javascripts/bundle.83f73b43.min.js"></script>
       
         <script src="../stylesheets/extra.js"></script>
       
diff --git a/mkdocs/site/pytorch-widedeep/model_components.md b/mkdocs/site/pytorch-widedeep/model_components.md
deleted file mode 100644
index c59332aa..00000000
--- a/mkdocs/site/pytorch-widedeep/model_components.md
+++ /dev/null
@@ -1,115 +0,0 @@
-# The ``models`` module
-
-This module contains the models that can be used as the four main components
-that will comprise a Wide and Deep model (``wide``, ``deeptabular``,
-``deeptext``, ``deepimage``), as well as the ``WideDeep`` "constructor"
-class. Note that each of the four components can be used independently. It
-also contains all the documentation for the models that can be used for
-self-supervised pre-training with tabular data.
-
-
-::: pytorch_widedeep.models.tabular.linear.wide.Wide
-    selection:
-        filters:
-            - "!^forward$"
-
-::: pytorch_widedeep.models.tabular.mlp.tab_mlp.TabMlp
-    selection:
-        filters:
-            - "!^forward$"
-
-::: pytorch_widedeep.models.tabular.mlp.tab_mlp.TabMlpDecoder
-    selection:
-        filters:
-            - "!^forward$"
-
-::: pytorch_widedeep.models.tabular.resnet.tab_resnet.TabResnet
-    selection:
-        filters:
-            - "!^forward$"
-
-::: pytorch_widedeep.models.tabular.resnet.tab_resnet.TabResnetDecoder
-    selection:
-        filters:
-            - "!^forward$"
-
-::: pytorch_widedeep.models.tabular.tabnet.tab_net.TabNet
-    selection:
-        filters:
-            - "!^forward$"
-
-::: pytorch_widedeep.models.tabular.tabnet.tab_net.TabNetDecoder
-    selection:
-        filters:
-            - "!^forward$"
-
-::: pytorch_widedeep.models.tabular.mlp.context_attention_mlp.ContextAttentionMLP
-    selection:
-        filters:
-            - "!^forward$"
-
-::: pytorch_widedeep.models.tabular.mlp.self_attention_mlp.SelfAttentionMLP
-    selection:
-        filters:
-            - "!^forward$"
-
-::: pytorch_widedeep.models.tabular.transformers.tab_transformer.TabTransformer
-    selection:
-        filters:
-            - "!^forward$"
-
-::: pytorch_widedeep.models.tabular.transformers.saint.SAINT
-    selection:
-        filters:
-            - "!^forward$"
-
-::: pytorch_widedeep.models.tabular.transformers.ft_transformer.FTTransformer
-    selection:
-        filters:
-            - "!^forward$"
-
-::: pytorch_widedeep.models.tabular.transformers.tab_perceiver.TabPerceiver
-    selection:
-        filters:
-            - "!^forward$"
-
-::: pytorch_widedeep.models.tabular.transformers.tab_fastformer.TabFastFormer
-    selection:
-        filters:
-            - "!^forward$"
-
-::: pytorch_widedeep.models.text.rnns.basic_rnn.BasicRNN
-    selection:
-        filters:
-            - "!^forward$"
-
-::: pytorch_widedeep.models.text.rnns.attentive_rnn.AttentiveRNN
-    selection:
-        filters:
-            - "!^forward$"
-
-::: pytorch_widedeep.models.text.rnns.stacked_attentive_rnn.StackedAttentiveRNN
-    selection:
-        filters:
-            - "!^forward$"
-
-::: pytorch_widedeep.models.text.huggingface_transformers.hf_model.HFModel
-    selection:
-        filters:
-            - "!^forward$"
-
-
-::: pytorch_widedeep.models.image.vision.Vision
-    selection:
-        filters:
-            - "!^forward$"
-
-::: pytorch_widedeep.models.model_fusion.ModelFuser
-    selection:
-        filters:
-            - "!^forward$"
-
-::: pytorch_widedeep.models.wide_deep.WideDeep
-    selection:
-        filters:
-            - "!^forward$"
diff --git a/mkdocs/site/pytorch-widedeep/preprocessing.html b/mkdocs/site/pytorch-widedeep/preprocessing.html
index 217aa63f..5e26cbeb 100644
--- a/mkdocs/site/pytorch-widedeep/preprocessing.html
+++ b/mkdocs/site/pytorch-widedeep/preprocessing.html
@@ -18,7 +18,7 @@
       
       
       <link rel="icon" href="../assets/images/favicon.ico">
-      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.35">
+      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.43">
     
     
       
@@ -26,7 +26,7 @@
       
     
     
-      <link rel="stylesheet" href="../assets/stylesheets/main.35f28582.min.css">
+      <link rel="stylesheet" href="../assets/stylesheets/main.0253249f.min.css">
       
         
         <link rel="stylesheet" href="../assets/stylesheets/palette.06af60db.min.css">
@@ -1738,7 +1738,9 @@ <h2 id="pytorch_widedeep.preprocessing.wide_preprocessor.WidePreprocessor" class
       </thead>
       <tbody>
           <tr class="doc-section-item">
-            <td><code>wide_cols</code></td>
+            <td>
+                <code>wide_cols</code>
+            </td>
             <td>
                   <code><span title="typing.List">List</span>[str]</code>
             </td>
@@ -1753,7 +1755,9 @@ <h2 id="pytorch_widedeep.preprocessing.wide_preprocessor.WidePreprocessor" class
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>crossed_cols</code></td>
+            <td>
+                <code>crossed_cols</code>
+            </td>
             <td>
                   <code><span title="typing.Optional">Optional</span>[<span title="typing.List">List</span>[<span title="typing.Tuple">Tuple</span>[str, str]]]</code>
             </td>
@@ -1853,6 +1857,11 @@ <h2 id="pytorch_widedeep.preprocessing.wide_preprocessor.WidePreprocessor" class
 <span class="go">2     g        g-l</span>
 </code></pre></div>
 
+
+
+
+
+
               <details class="quote">
                 <summary>Source code in <code>pytorch_widedeep/preprocessing/wide_preprocessor.py</code></summary>
                 <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 12</span>
@@ -2287,7 +2296,9 @@ <h3 id="pytorch_widedeep.preprocessing.wide_preprocessor.WidePreprocessor.fit" c
       </thead>
       <tbody>
           <tr class="doc-section-item">
-            <td><code>df</code></td>
+            <td>
+                <code>df</code>
+            </td>
             <td>
                   <code><span title="pandas.DataFrame">DataFrame</span></code>
             </td>
@@ -2414,7 +2425,9 @@ <h3 id="pytorch_widedeep.preprocessing.wide_preprocessor.WidePreprocessor.transf
       </thead>
       <tbody>
           <tr class="doc-section-item">
-            <td><code>df</code></td>
+            <td>
+                <code>df</code>
+            </td>
             <td>
                   <code><span title="pandas.DataFrame">DataFrame</span></code>
             </td>
@@ -2537,7 +2550,9 @@ <h3 id="pytorch_widedeep.preprocessing.wide_preprocessor.WidePreprocessor.invers
       </thead>
       <tbody>
           <tr class="doc-section-item">
-            <td><code>encoded</code></td>
+            <td>
+                <code>encoded</code>
+            </td>
             <td>
                   <code><span title="numpy.ndarray">ndarray</span></code>
             </td>
@@ -2664,7 +2679,9 @@ <h3 id="pytorch_widedeep.preprocessing.wide_preprocessor.WidePreprocessor.fit_tr
       </thead>
       <tbody>
           <tr class="doc-section-item">
-            <td><code>df</code></td>
+            <td>
+                <code>df</code>
+            </td>
             <td>
                   <code><span title="pandas.DataFrame">DataFrame</span></code>
             </td>
@@ -2777,7 +2794,9 @@ <h2 id="pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor" class="
       </thead>
       <tbody>
           <tr class="doc-section-item">
-            <td><code>cat_embed_cols</code></td>
+            <td>
+                <code>cat_embed_cols</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[str], <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.wdtypes.Tuple">Tuple</span>[str, int]]]]</code>
             </td>
@@ -2794,7 +2813,9 @@ <h2 id="pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor" class="
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>continuous_cols</code></td>
+            <td>
+                <code>continuous_cols</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[str]]</code>
             </td>
@@ -2808,7 +2829,9 @@ <h2 id="pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor" class="
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>quantization_setup</code></td>
+            <td>
+                <code>quantization_setup</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[int, <span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, <span title="pytorch_widedeep.wdtypes.Union">Union</span>[int, <span title="pytorch_widedeep.wdtypes.List">List</span>[float]]]]]</code>
             </td>
@@ -2828,7 +2851,9 @@ <h2 id="pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor" class="
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>cols_to_scale</code></td>
+            <td>
+                <code>cols_to_scale</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[str], str]]</code>
             </td>
@@ -2844,7 +2869,9 @@ <h2 id="pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor" class="
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>auto_embed_dim</code></td>
+            <td>
+                <code>auto_embed_dim</code>
+            </td>
             <td>
                   <code>bool</code>
             </td>
@@ -2860,7 +2887,9 @@ <h2 id="pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor" class="
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>embedding_rule</code></td>
+            <td>
+                <code>embedding_rule</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[google, fastai_old, fastai_new]</code>
             </td>
@@ -2886,7 +2915,9 @@ <h2 id="pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor" class="
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>default_embed_dim</code></td>
+            <td>
+                <code>default_embed_dim</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -2902,7 +2933,9 @@ <h2 id="pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor" class="
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>with_attention</code></td>
+            <td>
+                <code>with_attention</code>
+            </td>
             <td>
                   <code>bool</code>
             </td>
@@ -2924,7 +2957,9 @@ <h2 id="pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor" class="
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>with_cls_token</code></td>
+            <td>
+                <code>with_cls_token</code>
+            </td>
             <td>
                   <code>bool</code>
             </td>
@@ -2943,7 +2978,9 @@ <h2 id="pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor" class="
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>shared_embed</code></td>
+            <td>
+                <code>shared_embed</code>
+            </td>
             <td>
                   <code>bool</code>
             </td>
@@ -2963,7 +3000,9 @@ <h2 id="pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor" class="
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>verbose</code></td>
+            <td>
+                <code>verbose</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -2977,7 +3016,9 @@ <h2 id="pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor" class="
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>scale</code></td>
+            <td>
+                <code>scale</code>
+            </td>
             <td>
                   <code>bool</code>
             </td>
@@ -2997,7 +3038,9 @@ <h2 id="pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor" class="
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>already_standard</code></td>
+            <td>
+                <code>already_standard</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[str]]</code>
             </td>
@@ -3207,6 +3250,11 @@ <h2 id="pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor" class="
 <span class="gp">&gt;&gt;&gt; </span><span class="n">ft_cont_df2</span> <span class="o">=</span> <span class="n">tab_preprocessor2</span><span class="o">.</span><span class="n">fit_transform</span><span class="p">(</span><span class="n">cont_df</span><span class="p">)</span>
 </code></pre></div>
 
+
+
+
+
+
               <details class="quote">
                 <summary>Source code in <code>pytorch_widedeep/preprocessing/tab_preprocessor.py</code></summary>
                 <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">110</span>
@@ -4455,7 +4503,9 @@ <h3 id="pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor.fit" cla
       </thead>
       <tbody>
           <tr class="doc-section-item">
-            <td><code>df</code></td>
+            <td>
+                <code>df</code>
+            </td>
             <td>
                   <code><span title="pandas.DataFrame">DataFrame</span></code>
             </td>
@@ -4677,7 +4727,9 @@ <h3 id="pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor.transfor
       </thead>
       <tbody>
           <tr class="doc-section-item">
-            <td><code>df</code></td>
+            <td>
+                <code>df</code>
+            </td>
             <td>
                   <code><span title="pandas.DataFrame">DataFrame</span></code>
             </td>
@@ -4840,7 +4892,9 @@ <h3 id="pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor.inverse_
       </thead>
       <tbody>
           <tr class="doc-section-item">
-            <td><code>encoded</code></td>
+            <td>
+                <code>encoded</code>
+            </td>
             <td>
                   <code><span title="numpy.ndarray">ndarray</span></code>
             </td>
@@ -4994,7 +5048,9 @@ <h3 id="pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor.fit_tran
       </thead>
       <tbody>
           <tr class="doc-section-item">
-            <td><code>df</code></td>
+            <td>
+                <code>df</code>
+            </td>
             <td>
                   <code><span title="pandas.DataFrame">DataFrame</span></code>
             </td>
@@ -5109,7 +5165,9 @@ <h2 id="pytorch_widedeep.preprocessing.tab_preprocessor.Quantizer" class="doc do
       </thead>
       <tbody>
           <tr class="doc-section-item">
-            <td><code>quantization_setup</code></td>
+            <td>
+                <code>quantization_setup</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, <span title="pytorch_widedeep.wdtypes.Union">Union</span>[int, <span title="pytorch_widedeep.wdtypes.List">List</span>[float]]]</code>
             </td>
@@ -5127,6 +5185,11 @@ <h2 id="pytorch_widedeep.preprocessing.tab_preprocessor.Quantizer" class="doc do
       </tbody>
     </table>
 
+
+
+
+
+
               <details class="quote">
                 <summary>Source code in <code>pytorch_widedeep/preprocessing/tab_preprocessor.py</code></summary>
                 <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 39</span>
@@ -5320,7 +5383,9 @@ <h2 id="pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor" class
       </thead>
       <tbody>
           <tr class="doc-section-item">
-            <td><code>text_col</code></td>
+            <td>
+                <code>text_col</code>
+            </td>
             <td>
                   <code>str</code>
             </td>
@@ -5334,7 +5399,9 @@ <h2 id="pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor" class
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>max_vocab</code></td>
+            <td>
+                <code>max_vocab</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -5348,7 +5415,9 @@ <h2 id="pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor" class
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>min_freq</code></td>
+            <td>
+                <code>min_freq</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -5362,7 +5431,9 @@ <h2 id="pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor" class
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>maxlen</code></td>
+            <td>
+                <code>maxlen</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -5376,7 +5447,9 @@ <h2 id="pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor" class
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>pad_first</code></td>
+            <td>
+                <code>pad_first</code>
+            </td>
             <td>
                   <code>bool</code>
             </td>
@@ -5391,7 +5464,9 @@ <h2 id="pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor" class
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>pad_idx</code></td>
+            <td>
+                <code>pad_idx</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -5405,7 +5480,9 @@ <h2 id="pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor" class
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>already_processed</code></td>
+            <td>
+                <code>already_processed</code>
+            </td>
             <td>
                   <code><span title="typing.Optional">Optional</span>[bool]</code>
             </td>
@@ -5425,7 +5502,9 @@ <h2 id="pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor" class
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>word_vectors_path</code></td>
+            <td>
+                <code>word_vectors_path</code>
+            </td>
             <td>
                   <code><span title="typing.Optional">Optional</span>[str]</code>
             </td>
@@ -5439,7 +5518,9 @@ <h2 id="pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor" class
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>n_cpus</code></td>
+            <td>
+                <code>n_cpus</code>
+            </td>
             <td>
                   <code><span title="typing.Optional">Optional</span>[int]</code>
             </td>
@@ -5453,7 +5534,9 @@ <h2 id="pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor" class
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>verbose</code></td>
+            <td>
+                <code>verbose</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -5521,6 +5604,11 @@ <h2 id="pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor" class
 <span class="go">array([[ 1,  1,  9, 16, 17, 18, 11,  0,  0, 13]], dtype=int32)</span>
 </code></pre></div>
 
+
+
+
+
+
               <details class="quote">
                 <summary>Source code in <code>pytorch_widedeep/preprocessing/text_preprocessor.py</code></summary>
                 <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 22</span>
@@ -6045,7 +6133,9 @@ <h3 id="pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor.fit" c
       </thead>
       <tbody>
           <tr class="doc-section-item">
-            <td><code>df</code></td>
+            <td>
+                <code>df</code>
+            </td>
             <td>
                   <code><span title="pandas.DataFrame">DataFrame</span></code>
             </td>
@@ -6189,7 +6279,9 @@ <h3 id="pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor.transf
       </thead>
       <tbody>
           <tr class="doc-section-item">
-            <td><code>df</code></td>
+            <td>
+                <code>df</code>
+            </td>
             <td>
                   <code><span title="pandas.DataFrame">DataFrame</span></code>
             </td>
@@ -6297,7 +6389,9 @@ <h3 id="pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor.transf
       </thead>
       <tbody>
           <tr class="doc-section-item">
-            <td><code>text</code></td>
+            <td>
+                <code>text</code>
+            </td>
             <td>
                   <code>str</code>
             </td>
@@ -6403,7 +6497,9 @@ <h3 id="pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor.fit_tr
       </thead>
       <tbody>
           <tr class="doc-section-item">
-            <td><code>df</code></td>
+            <td>
+                <code>df</code>
+            </td>
             <td>
                   <code><span title="pandas.DataFrame">DataFrame</span></code>
             </td>
@@ -6505,7 +6601,9 @@ <h3 id="pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor.invers
       </thead>
       <tbody>
           <tr class="doc-section-item">
-            <td><code>padded_seq</code></td>
+            <td>
+                <code>padded_seq</code>
+            </td>
             <td>
                   <code><span title="numpy.ndarray">ndarray</span></code>
             </td>
@@ -6625,7 +6723,9 @@ <h2 id="pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor" class="do
       </thead>
       <tbody>
           <tr class="doc-section-item">
-            <td><code>model_name</code></td>
+            <td>
+                <code>model_name</code>
+            </td>
             <td>
                   <code>str</code>
             </td>
@@ -6641,7 +6741,9 @@ <h2 id="pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor" class="do
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>use_fast_tokenizer</code></td>
+            <td>
+                <code>use_fast_tokenizer</code>
+            </td>
             <td>
                   <code>bool</code>
             </td>
@@ -6655,7 +6757,9 @@ <h2 id="pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor" class="do
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>text_col</code></td>
+            <td>
+                <code>text_col</code>
+            </td>
             <td>
                   <code><span title="typing.Optional">Optional</span>[str]</code>
             </td>
@@ -6673,7 +6777,9 @@ <h2 id="pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor" class="do
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>num_workers</code></td>
+            <td>
+                <code>num_workers</code>
+            </td>
             <td>
                   <code><span title="typing.Optional">Optional</span>[int]</code>
             </td>
@@ -6690,7 +6796,9 @@ <h2 id="pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor" class="do
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>preprocessing_rules</code></td>
+            <td>
+                <code>preprocessing_rules</code>
+            </td>
             <td>
                   <code><span title="typing.Optional">Optional</span>[<span title="typing.List">List</span>[<span title="typing.Callable">Callable</span>[[str], str]]]</code>
             </td>
@@ -6706,7 +6814,9 @@ <h2 id="pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor" class="do
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>tokenizer_params</code></td>
+            <td>
+                <code>tokenizer_params</code>
+            </td>
             <td>
                   <code><span title="typing.Optional">Optional</span>[<span title="typing.Dict">Dict</span>[str, <span title="typing.Any">Any</span>]]</code>
             </td>
@@ -6722,7 +6832,9 @@ <h2 id="pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor" class="do
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>encode_params</code></td>
+            <td>
+                <code>encode_params</code>
+            </td>
             <td>
                   <code><span title="typing.Optional">Optional</span>[<span title="typing.Dict">Dict</span>[str, <span title="typing.Any">Any</span>]]</code>
             </td>
@@ -6740,7 +6852,9 @@ <h2 id="pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor" class="do
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>**kwargs</code></td>
+            <td>
+                <code>**kwargs</code>
+            </td>
             <td>
             </td>
             <td>
@@ -6797,6 +6911,11 @@ <h2 id="pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor" class="do
 <span class="gp">&gt;&gt;&gt; </span><span class="n">X_text_2</span> <span class="o">=</span> <span class="n">hf_processor_2</span><span class="o">.</span><span class="n">encode</span><span class="p">(</span><span class="n">texts</span><span class="p">,</span> <span class="n">max_length</span><span class="o">=</span><span class="mi">10</span><span class="p">,</span> <span class="n">padding</span><span class="o">=</span><span class="s2">&quot;max_length&quot;</span><span class="p">,</span> <span class="n">truncation</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
 </code></pre></div>
 
+
+
+
+
+
               <details class="quote">
                 <summary>Source code in <code>pytorch_widedeep/preprocessing/hf_preprocessor.py</code></summary>
                 <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 17</span>
@@ -7543,7 +7662,9 @@ <h3 id="pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.encode" cl
       </thead>
       <tbody>
           <tr class="doc-section-item">
-            <td><code>texts</code></td>
+            <td>
+                <code>texts</code>
+            </td>
             <td>
                   <code><span title="typing.List">List</span>[str]</code>
             </td>
@@ -7557,7 +7678,9 @@ <h3 id="pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.encode" cl
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>**kwargs</code></td>
+            <td>
+                <code>**kwargs</code>
+            </td>
             <td>
             </td>
             <td>
@@ -7755,7 +7878,9 @@ <h3 id="pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.decode" cl
       </thead>
       <tbody>
           <tr class="doc-section-item">
-            <td><code>input_ids</code></td>
+            <td>
+                <code>input_ids</code>
+            </td>
             <td>
                   <code><span title="numpy.typing.NDArray">NDArray</span>[<span title="numpy.int64">int64</span>]</code>
             </td>
@@ -7769,7 +7894,9 @@ <h3 id="pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.decode" cl
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>skip_special_tokens</code></td>
+            <td>
+                <code>skip_special_tokens</code>
+            </td>
             <td>
                   <code>bool</code>
             </td>
@@ -7901,7 +8028,9 @@ <h3 id="pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.fit" class
       </thead>
       <tbody>
           <tr class="doc-section-item">
-            <td><code>df</code></td>
+            <td>
+                <code>df</code>
+            </td>
             <td>
                   <code><span title="pandas.DataFrame">DataFrame</span></code>
             </td>
@@ -7996,7 +8125,9 @@ <h3 id="pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.transform"
       </thead>
       <tbody>
           <tr class="doc-section-item">
-            <td><code>df</code></td>
+            <td>
+                <code>df</code>
+            </td>
             <td>
                   <code><span title="pandas.DataFrame">DataFrame</span></code>
             </td>
@@ -8125,7 +8256,9 @@ <h3 id="pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.transform_
       </thead>
       <tbody>
           <tr class="doc-section-item">
-            <td><code>text</code></td>
+            <td>
+                <code>text</code>
+            </td>
             <td>
                   <code>str</code>
             </td>
@@ -8239,7 +8372,9 @@ <h3 id="pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.fit_transf
       </thead>
       <tbody>
           <tr class="doc-section-item">
-            <td><code>df</code></td>
+            <td>
+                <code>df</code>
+            </td>
             <td>
                   <code><span title="pandas.DataFrame">DataFrame</span></code>
             </td>
@@ -8347,7 +8482,9 @@ <h3 id="pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.inverse_tr
       </thead>
       <tbody>
           <tr class="doc-section-item">
-            <td><code>input_ids</code></td>
+            <td>
+                <code>input_ids</code>
+            </td>
             <td>
                   <code><span title="numpy.typing.NDArray">NDArray</span>[<span title="numpy.int64">int64</span>]</code>
             </td>
@@ -8361,7 +8498,9 @@ <h3 id="pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.inverse_tr
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>skip_special_tokens</code></td>
+            <td>
+                <code>skip_special_tokens</code>
+            </td>
             <td>
                   <code>bool</code>
             </td>
@@ -8488,7 +8627,9 @@ <h2 id="pytorch_widedeep.preprocessing.image_preprocessor.ImagePreprocessor" cla
       </thead>
       <tbody>
           <tr class="doc-section-item">
-            <td><code>img_col</code></td>
+            <td>
+                <code>img_col</code>
+            </td>
             <td>
                   <code>str</code>
             </td>
@@ -8502,7 +8643,9 @@ <h2 id="pytorch_widedeep.preprocessing.image_preprocessor.ImagePreprocessor" cla
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>img_path</code></td>
+            <td>
+                <code>img_path</code>
+            </td>
             <td>
                   <code>str</code>
             </td>
@@ -8516,7 +8659,9 @@ <h2 id="pytorch_widedeep.preprocessing.image_preprocessor.ImagePreprocessor" cla
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>width</code></td>
+            <td>
+                <code>width</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -8530,7 +8675,9 @@ <h2 id="pytorch_widedeep.preprocessing.image_preprocessor.ImagePreprocessor" cla
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>height</code></td>
+            <td>
+                <code>height</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -8544,7 +8691,9 @@ <h2 id="pytorch_widedeep.preprocessing.image_preprocessor.ImagePreprocessor" cla
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>verbose</code></td>
+            <td>
+                <code>verbose</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -8629,6 +8778,11 @@ <h2 id="pytorch_widedeep.preprocessing.image_preprocessor.ImagePreprocessor" cla
 metrics and running <code>fit</code> only simply instantiates the resizing
 functions.</p>
 
+
+
+
+
+
               <details class="quote">
                 <summary>Source code in <code>pytorch_widedeep/preprocessing/image_preprocessor.py</code></summary>
                 <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 15</span>
@@ -9033,7 +9187,9 @@ <h3 id="pytorch_widedeep.preprocessing.image_preprocessor.ImagePreprocessor.tran
       </thead>
       <tbody>
           <tr class="doc-section-item">
-            <td><code>df</code></td>
+            <td>
+                <code>df</code>
+            </td>
             <td>
                   <code><span title="pandas.DataFrame">DataFrame</span></code>
             </td>
@@ -9237,7 +9393,9 @@ <h3 id="pytorch_widedeep.preprocessing.image_preprocessor.ImagePreprocessor.fit_
       </thead>
       <tbody>
           <tr class="doc-section-item">
-            <td><code>df</code></td>
+            <td>
+                <code>df</code>
+            </td>
             <td>
                   <code><span title="pandas.DataFrame">DataFrame</span></code>
             </td>
@@ -9353,7 +9511,9 @@ <h2 id="pytorch_widedeep.preprocessing.din_preprocessor.DINPreprocessor" class="
       </thead>
       <tbody>
           <tr class="doc-section-item">
-            <td><code>user_id_col</code></td>
+            <td>
+                <code>user_id_col</code>
+            </td>
             <td>
                   <code>str</code>
             </td>
@@ -9367,7 +9527,9 @@ <h2 id="pytorch_widedeep.preprocessing.din_preprocessor.DINPreprocessor" class="
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>item_embed_col</code></td>
+            <td>
+                <code>item_embed_col</code>
+            </td>
             <td>
                   <code><span title="typing.Union">Union</span>[str, <span title="typing.Tuple">Tuple</span>[str, int]]</code>
             </td>
@@ -9384,7 +9546,9 @@ <h2 id="pytorch_widedeep.preprocessing.din_preprocessor.DINPreprocessor" class="
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>target_col</code></td>
+            <td>
+                <code>target_col</code>
+            </td>
             <td>
                   <code>str</code>
             </td>
@@ -9398,7 +9562,9 @@ <h2 id="pytorch_widedeep.preprocessing.din_preprocessor.DINPreprocessor" class="
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>max_seq_length</code></td>
+            <td>
+                <code>max_seq_length</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -9412,7 +9578,9 @@ <h2 id="pytorch_widedeep.preprocessing.din_preprocessor.DINPreprocessor" class="
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>action_col</code></td>
+            <td>
+                <code>action_col</code>
+            </td>
             <td>
                   <code><span title="typing.Optional">Optional</span>[str]</code>
             </td>
@@ -9428,7 +9596,9 @@ <h2 id="pytorch_widedeep.preprocessing.din_preprocessor.DINPreprocessor" class="
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>other_seq_embed_cols</code></td>
+            <td>
+                <code>other_seq_embed_cols</code>
+            </td>
             <td>
                   <code><span title="typing.Optional">Optional</span>[<span title="typing.List">List</span>[str] | <span title="typing.List">List</span>[<span title="typing.Tuple">Tuple</span>[str, int]]]</code>
             </td>
@@ -9444,7 +9614,9 @@ <h2 id="pytorch_widedeep.preprocessing.din_preprocessor.DINPreprocessor" class="
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>cat_embed_cols</code></td>
+            <td>
+                <code>cat_embed_cols</code>
+            </td>
             <td>
                   <code><span title="typing.Optional">Optional</span>[<span title="typing.Union">Union</span>[<span title="typing.List">List</span>[str], <span title="typing.List">List</span>[<span title="typing.Tuple">Tuple</span>[str, int]]]]</code>
             </td>
@@ -9463,7 +9635,9 @@ <h2 id="pytorch_widedeep.preprocessing.din_preprocessor.DINPreprocessor" class="
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>continuous_cols</code></td>
+            <td>
+                <code>continuous_cols</code>
+            </td>
             <td>
                   <code><span title="typing.Optional">Optional</span>[<span title="typing.List">List</span>[str]]</code>
             </td>
@@ -9477,7 +9651,9 @@ <h2 id="pytorch_widedeep.preprocessing.din_preprocessor.DINPreprocessor" class="
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>quantization_setup</code></td>
+            <td>
+                <code>quantization_setup</code>
+            </td>
             <td>
                   <code><span title="typing.Optional">Optional</span>[<span title="typing.Union">Union</span>[int, <span title="typing.Dict">Dict</span>[str, <span title="typing.Union">Union</span>[int, <span title="typing.List">List</span>[float]]]]]</code>
             </td>
@@ -9496,7 +9672,9 @@ <h2 id="pytorch_widedeep.preprocessing.din_preprocessor.DINPreprocessor" class="
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>cols_to_scale</code></td>
+            <td>
+                <code>cols_to_scale</code>
+            </td>
             <td>
                   <code><span title="typing.Optional">Optional</span>[<span title="typing.Union">Union</span>[<span title="typing.List">List</span>[str], str]]</code>
             </td>
@@ -9512,7 +9690,9 @@ <h2 id="pytorch_widedeep.preprocessing.din_preprocessor.DINPreprocessor" class="
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>auto_embed_dim</code></td>
+            <td>
+                <code>auto_embed_dim</code>
+            </td>
             <td>
                   <code>bool</code>
             </td>
@@ -9528,7 +9708,9 @@ <h2 id="pytorch_widedeep.preprocessing.din_preprocessor.DINPreprocessor" class="
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>embedding_rule</code></td>
+            <td>
+                <code>embedding_rule</code>
+            </td>
             <td>
                   <code><span title="typing.Literal">Literal</span>[&#39;google&#39;, &#39;fastai_old&#39;, &#39;fastai_new&#39;]</code>
             </td>
@@ -9554,7 +9736,9 @@ <h2 id="pytorch_widedeep.preprocessing.din_preprocessor.DINPreprocessor" class="
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>default_embed_dim</code></td>
+            <td>
+                <code>default_embed_dim</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -9570,7 +9754,9 @@ <h2 id="pytorch_widedeep.preprocessing.din_preprocessor.DINPreprocessor" class="
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>verbose</code></td>
+            <td>
+                <code>verbose</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -9584,7 +9770,9 @@ <h2 id="pytorch_widedeep.preprocessing.din_preprocessor.DINPreprocessor" class="
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>scale</code></td>
+            <td>
+                <code>scale</code>
+            </td>
             <td>
                   <code>bool</code>
             </td>
@@ -9604,7 +9792,9 @@ <h2 id="pytorch_widedeep.preprocessing.din_preprocessor.DINPreprocessor" class="
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>already_standard</code></td>
+            <td>
+                <code>already_standard</code>
+            </td>
             <td>
                   <code><span title="typing.Optional">Optional</span>[<span title="typing.List">List</span>[str]]</code>
             </td>
@@ -9831,6 +10021,11 @@ <h2 id="pytorch_widedeep.preprocessing.din_preprocessor.DINPreprocessor" class="
 <span class="gp">&gt;&gt;&gt; </span><span class="n">X</span><span class="p">,</span> <span class="n">y</span> <span class="o">=</span> <span class="n">din_preprocessor</span><span class="o">.</span><span class="n">fit_transform</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
 </code></pre></div>
 
+
+
+
+
+
               <details class="quote">
                 <summary>Source code in <code>pytorch_widedeep/preprocessing/din_preprocessor.py</code></summary>
                 <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 21</span>
@@ -10962,7 +11157,9 @@ <h2 id="pytorch_widedeep.preprocessing.wide_preprocessor.ChunkWidePreprocessor"
       </thead>
       <tbody>
           <tr class="doc-section-item">
-            <td><code>wide_cols</code></td>
+            <td>
+                <code>wide_cols</code>
+            </td>
             <td>
                   <code><span title="typing.List">List</span>[str]</code>
             </td>
@@ -10977,7 +11174,9 @@ <h2 id="pytorch_widedeep.preprocessing.wide_preprocessor.ChunkWidePreprocessor"
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>crossed_cols</code></td>
+            <td>
+                <code>crossed_cols</code>
+            </td>
             <td>
                   <code><span title="typing.Optional">Optional</span>[<span title="typing.List">List</span>[<span title="typing.Tuple">Tuple</span>[str, str]]]</code>
             </td>
@@ -11067,6 +11266,11 @@ <h2 id="pytorch_widedeep.preprocessing.wide_preprocessor.ChunkWidePreprocessor"
 <span class="gp">&gt;&gt;&gt; </span><span class="n">X_wide</span> <span class="o">=</span> <span class="n">chunk_wide_preprocessor</span><span class="o">.</span><span class="n">fit_transform</span><span class="p">(</span><span class="n">chunk</span><span class="p">)</span>
 </code></pre></div>
 
+
+
+
+
+
               <details class="quote">
                 <summary>Source code in <code>pytorch_widedeep/preprocessing/wide_preprocessor.py</code></summary>
                 <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">209</span>
@@ -11329,7 +11533,9 @@ <h3 id="pytorch_widedeep.preprocessing.wide_preprocessor.ChunkWidePreprocessor.p
       </thead>
       <tbody>
           <tr class="doc-section-item">
-            <td><code>chunk</code></td>
+            <td>
+                <code>chunk</code>
+            </td>
             <td>
                   <code><span title="pandas.DataFrame">DataFrame</span></code>
             </td>
@@ -11521,7 +11727,9 @@ <h2 id="pytorch_widedeep.preprocessing.tab_preprocessor.ChunkTabPreprocessor" cl
       </thead>
       <tbody>
           <tr class="doc-section-item">
-            <td><code>n_chunks</code></td>
+            <td>
+                <code>n_chunks</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -11535,7 +11743,9 @@ <h2 id="pytorch_widedeep.preprocessing.tab_preprocessor.ChunkTabPreprocessor" cl
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>cat_embed_cols</code></td>
+            <td>
+                <code>cat_embed_cols</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[str], <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.wdtypes.Tuple">Tuple</span>[str, int]]]]</code>
             </td>
@@ -11553,7 +11763,9 @@ <h2 id="pytorch_widedeep.preprocessing.tab_preprocessor.ChunkTabPreprocessor" cl
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>continuous_cols</code></td>
+            <td>
+                <code>continuous_cols</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[str]]</code>
             </td>
@@ -11567,7 +11779,9 @@ <h2 id="pytorch_widedeep.preprocessing.tab_preprocessor.ChunkTabPreprocessor" cl
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>cols_and_bins</code></td>
+            <td>
+                <code>cols_and_bins</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, <span title="pytorch_widedeep.wdtypes.List">List</span>[float]]]</code>
             </td>
@@ -11584,7 +11798,9 @@ <h2 id="pytorch_widedeep.preprocessing.tab_preprocessor.ChunkTabPreprocessor" cl
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>cols_to_scale</code></td>
+            <td>
+                <code>cols_to_scale</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[str], str]]</code>
             </td>
@@ -11599,7 +11815,9 @@ <h2 id="pytorch_widedeep.preprocessing.tab_preprocessor.ChunkTabPreprocessor" cl
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>default_embed_dim</code></td>
+            <td>
+                <code>default_embed_dim</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -11615,7 +11833,9 @@ <h2 id="pytorch_widedeep.preprocessing.tab_preprocessor.ChunkTabPreprocessor" cl
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>with_attention</code></td>
+            <td>
+                <code>with_attention</code>
+            </td>
             <td>
                   <code>bool</code>
             </td>
@@ -11637,7 +11857,9 @@ <h2 id="pytorch_widedeep.preprocessing.tab_preprocessor.ChunkTabPreprocessor" cl
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>with_cls_token</code></td>
+            <td>
+                <code>with_cls_token</code>
+            </td>
             <td>
                   <code>bool</code>
             </td>
@@ -11656,7 +11878,9 @@ <h2 id="pytorch_widedeep.preprocessing.tab_preprocessor.ChunkTabPreprocessor" cl
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>shared_embed</code></td>
+            <td>
+                <code>shared_embed</code>
+            </td>
             <td>
                   <code>bool</code>
             </td>
@@ -11676,7 +11900,9 @@ <h2 id="pytorch_widedeep.preprocessing.tab_preprocessor.ChunkTabPreprocessor" cl
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>verbose</code></td>
+            <td>
+                <code>verbose</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -11690,7 +11916,9 @@ <h2 id="pytorch_widedeep.preprocessing.tab_preprocessor.ChunkTabPreprocessor" cl
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>scale</code></td>
+            <td>
+                <code>scale</code>
+            </td>
             <td>
                   <code>bool</code>
             </td>
@@ -11710,7 +11938,9 @@ <h2 id="pytorch_widedeep.preprocessing.tab_preprocessor.ChunkTabPreprocessor" cl
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>already_standard</code></td>
+            <td>
+                <code>already_standard</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[str]]</code>
             </td>
@@ -11916,6 +12146,11 @@ <h2 id="pytorch_widedeep.preprocessing.tab_preprocessor.ChunkTabPreprocessor" cl
 <span class="go">{&#39;cat_col&#39;: 0, &#39;cont_col&#39;: 1}</span>
 </code></pre></div>
 
+
+
+
+
+
               <details class="quote">
                 <summary>Source code in <code>pytorch_widedeep/preprocessing/tab_preprocessor.py</code></summary>
                 <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 714</span>
@@ -12673,7 +12908,9 @@ <h2 id="pytorch_widedeep.preprocessing.text_preprocessor.ChunkTextPreprocessor"
       </thead>
       <tbody>
           <tr class="doc-section-item">
-            <td><code>text_col</code></td>
+            <td>
+                <code>text_col</code>
+            </td>
             <td>
                   <code>str</code>
             </td>
@@ -12688,7 +12925,9 @@ <h2 id="pytorch_widedeep.preprocessing.text_preprocessor.ChunkTextPreprocessor"
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>n_chunks</code></td>
+            <td>
+                <code>n_chunks</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -12702,7 +12941,9 @@ <h2 id="pytorch_widedeep.preprocessing.text_preprocessor.ChunkTextPreprocessor"
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>root_dir</code></td>
+            <td>
+                <code>root_dir</code>
+            </td>
             <td>
                   <code><span title="typing.Optional">Optional</span>[str]</code>
             </td>
@@ -12717,7 +12958,9 @@ <h2 id="pytorch_widedeep.preprocessing.text_preprocessor.ChunkTextPreprocessor"
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>max_vocab</code></td>
+            <td>
+                <code>max_vocab</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -12731,7 +12974,9 @@ <h2 id="pytorch_widedeep.preprocessing.text_preprocessor.ChunkTextPreprocessor"
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>min_freq</code></td>
+            <td>
+                <code>min_freq</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -12745,7 +12990,9 @@ <h2 id="pytorch_widedeep.preprocessing.text_preprocessor.ChunkTextPreprocessor"
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>maxlen</code></td>
+            <td>
+                <code>maxlen</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -12759,7 +13006,9 @@ <h2 id="pytorch_widedeep.preprocessing.text_preprocessor.ChunkTextPreprocessor"
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>pad_first</code></td>
+            <td>
+                <code>pad_first</code>
+            </td>
             <td>
                   <code>bool</code>
             </td>
@@ -12774,7 +13023,9 @@ <h2 id="pytorch_widedeep.preprocessing.text_preprocessor.ChunkTextPreprocessor"
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>pad_idx</code></td>
+            <td>
+                <code>pad_idx</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -12788,7 +13039,9 @@ <h2 id="pytorch_widedeep.preprocessing.text_preprocessor.ChunkTextPreprocessor"
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>word_vectors_path</code></td>
+            <td>
+                <code>word_vectors_path</code>
+            </td>
             <td>
                   <code><span title="typing.Optional">Optional</span>[str]</code>
             </td>
@@ -12802,7 +13055,9 @@ <h2 id="pytorch_widedeep.preprocessing.text_preprocessor.ChunkTextPreprocessor"
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>n_cpus</code></td>
+            <td>
+                <code>n_cpus</code>
+            </td>
             <td>
                   <code><span title="typing.Optional">Optional</span>[int]</code>
             </td>
@@ -12816,7 +13071,9 @@ <h2 id="pytorch_widedeep.preprocessing.text_preprocessor.ChunkTextPreprocessor"
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>verbose</code></td>
+            <td>
+                <code>verbose</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -12879,6 +13136,11 @@ <h2 id="pytorch_widedeep.preprocessing.text_preprocessor.ChunkTextPreprocessor"
 <span class="gp">&gt;&gt;&gt; </span><span class="n">processed_chunk</span> <span class="o">=</span> <span class="n">chunk_text_preprocessor</span><span class="o">.</span><span class="n">fit_transform</span><span class="p">(</span><span class="n">chunk_df</span><span class="p">)</span>
 </code></pre></div>
 
+
+
+
+
+
               <details class="quote">
                 <summary>Source code in <code>pytorch_widedeep/preprocessing/text_preprocessor.py</code></summary>
                 <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">264</span>
@@ -13214,7 +13476,9 @@ <h2 id="pytorch_widedeep.preprocessing.hf_preprocessor.ChunkHFPreprocessor" clas
       </thead>
       <tbody>
           <tr class="doc-section-item">
-            <td><code>model_name</code></td>
+            <td>
+                <code>model_name</code>
+            </td>
             <td>
                   <code>str</code>
             </td>
@@ -13230,7 +13494,9 @@ <h2 id="pytorch_widedeep.preprocessing.hf_preprocessor.ChunkHFPreprocessor" clas
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>text_col</code></td>
+            <td>
+                <code>text_col</code>
+            </td>
             <td>
                   <code>str</code>
             </td>
@@ -13245,7 +13511,9 @@ <h2 id="pytorch_widedeep.preprocessing.hf_preprocessor.ChunkHFPreprocessor" clas
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>root_dir</code></td>
+            <td>
+                <code>root_dir</code>
+            </td>
             <td>
                   <code><span title="typing.Optional">Optional</span>[str]</code>
             </td>
@@ -13262,7 +13530,9 @@ <h2 id="pytorch_widedeep.preprocessing.hf_preprocessor.ChunkHFPreprocessor" clas
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>use_fast_tokenizer</code></td>
+            <td>
+                <code>use_fast_tokenizer</code>
+            </td>
             <td>
                   <code>bool</code>
             </td>
@@ -13276,7 +13546,9 @@ <h2 id="pytorch_widedeep.preprocessing.hf_preprocessor.ChunkHFPreprocessor" clas
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>num_workers</code></td>
+            <td>
+                <code>num_workers</code>
+            </td>
             <td>
                   <code><span title="typing.Optional">Optional</span>[int]</code>
             </td>
@@ -13293,7 +13565,9 @@ <h2 id="pytorch_widedeep.preprocessing.hf_preprocessor.ChunkHFPreprocessor" clas
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>preprocessing_rules</code></td>
+            <td>
+                <code>preprocessing_rules</code>
+            </td>
             <td>
                   <code><span title="typing.Optional">Optional</span>[<span title="typing.List">List</span>[<span title="typing.Callable">Callable</span>[[str], str]]]</code>
             </td>
@@ -13309,7 +13583,9 @@ <h2 id="pytorch_widedeep.preprocessing.hf_preprocessor.ChunkHFPreprocessor" clas
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>tokenizer_params</code></td>
+            <td>
+                <code>tokenizer_params</code>
+            </td>
             <td>
                   <code><span title="typing.Optional">Optional</span>[<span title="typing.Dict">Dict</span>[str, <span title="typing.Any">Any</span>]]</code>
             </td>
@@ -13324,7 +13600,9 @@ <h2 id="pytorch_widedeep.preprocessing.hf_preprocessor.ChunkHFPreprocessor" clas
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>encode_params</code></td>
+            <td>
+                <code>encode_params</code>
+            </td>
             <td>
                   <code><span title="typing.Optional">Optional</span>[<span title="typing.Dict">Dict</span>[str, <span title="typing.Any">Any</span>]]</code>
             </td>
@@ -13373,6 +13651,11 @@ <h2 id="pytorch_widedeep.preprocessing.hf_preprocessor.ChunkHFPreprocessor" clas
       </tbody>
     </table>
 
+
+
+
+
+
               <details class="quote">
                 <summary>Source code in <code>pytorch_widedeep/preprocessing/hf_preprocessor.py</code></summary>
                 <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">369</span>
@@ -13699,7 +13982,7 @@ <h2 id="pytorch_widedeep.preprocessing.hf_preprocessor.ChunkHFPreprocessor" clas
     <script id="__config" type="application/json">{"base": "..", "features": ["navigation.tabs", "navigation.tabs.sticky", "navigation.indexes", "navigation.expand", "toc.integrate"], "search": "../assets/javascripts/workers/search.6ce7567c.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version": "Select version"}}</script>
     
     
-      <script src="../assets/javascripts/bundle.56dfad97.min.js"></script>
+      <script src="../assets/javascripts/bundle.83f73b43.min.js"></script>
       
         <script src="../stylesheets/extra.js"></script>
       
diff --git a/mkdocs/site/pytorch-widedeep/preprocessing.md b/mkdocs/site/pytorch-widedeep/preprocessing.md
deleted file mode 100644
index 520dd914..00000000
--- a/mkdocs/site/pytorch-widedeep/preprocessing.md
+++ /dev/null
@@ -1,44 +0,0 @@
-# The ``preprocessing`` module
-
-This module contains the classes that are used to prepare the data before
-being passed to the models. There is one Preprocessor per data mode or model
-component (``wide``, ``deeptabular``, ``deepimage`` and ``deeptext``) with
-the exception of the ``deeptext`` component. In this case, two processors are
-available: one for the case when no Hugging Face model is used
-(`TextPreprocessor`) and another one when a Hugging Face model is used
-(`HFPreprocessor`).
-
-
-::: pytorch_widedeep.preprocessing.wide_preprocessor.WidePreprocessor
-
-::: pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor
-
-::: pytorch_widedeep.preprocessing.tab_preprocessor.Quantizer
-
-::: pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor
-
-::: pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor
-
-::: pytorch_widedeep.preprocessing.image_preprocessor.ImagePreprocessor
-
-::: pytorch_widedeep.preprocessing.din_preprocessor.DINPreprocessor
-
-
-## Chunked versions
-
-Chunked versions of the preprocessors are also available. These are useful
-when the data is too big to fit in memory. See also the [``load_from_folder``](load_from_folder.md)
-module in the library and the corresponding section here in the documentation.
-
-Note that there is not a ``ChunkImagePreprocessor``. This is because the
-processing of the images will occur inside the `ImageFromFolder` class in
-the [``load_from_folder``](load_from_folder.md) module.
-
-
-::: pytorch_widedeep.preprocessing.wide_preprocessor.ChunkWidePreprocessor
-
-::: pytorch_widedeep.preprocessing.tab_preprocessor.ChunkTabPreprocessor
-
-::: pytorch_widedeep.preprocessing.text_preprocessor.ChunkTextPreprocessor
-
-::: pytorch_widedeep.preprocessing.hf_preprocessor.ChunkHFPreprocessor
diff --git a/mkdocs/site/pytorch-widedeep/self_supervised_pretraining.html b/mkdocs/site/pytorch-widedeep/self_supervised_pretraining.html
index 5c447b2f..8877984c 100644
--- a/mkdocs/site/pytorch-widedeep/self_supervised_pretraining.html
+++ b/mkdocs/site/pytorch-widedeep/self_supervised_pretraining.html
@@ -18,7 +18,7 @@
       
       
       <link rel="icon" href="../assets/images/favicon.ico">
-      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.35">
+      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.43">
     
     
       
@@ -26,7 +26,7 @@
       
     
     
-      <link rel="stylesheet" href="../assets/stylesheets/main.35f28582.min.css">
+      <link rel="stylesheet" href="../assets/stylesheets/main.0253249f.min.css">
       
         
         <link rel="stylesheet" href="../assets/stylesheets/palette.06af60db.min.css">
@@ -1678,7 +1678,9 @@ <h2 id="pytorch_widedeep.self_supervised_training.EncoderDecoderTrainer" class="
       </thead>
       <tbody>
           <tr class="doc-section-item">
-            <td><code>encoder</code></td>
+            <td>
+                <code>encoder</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.ModelWithoutAttention">ModelWithoutAttention</span></code>
             </td>
@@ -1692,7 +1694,9 @@ <h2 id="pytorch_widedeep.self_supervised_training.EncoderDecoderTrainer" class="
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>decoder</code></td>
+            <td>
+                <code>decoder</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.DecoderWithoutAttention">DecoderWithoutAttention</span>]</code>
             </td>
@@ -1708,7 +1712,9 @@ <h2 id="pytorch_widedeep.self_supervised_training.EncoderDecoderTrainer" class="
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>masked_prob</code></td>
+            <td>
+                <code>masked_prob</code>
+            </td>
             <td>
                   <code>float</code>
             </td>
@@ -1723,7 +1729,9 @@ <h2 id="pytorch_widedeep.self_supervised_training.EncoderDecoderTrainer" class="
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>optimizer</code></td>
+            <td>
+                <code>optimizer</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Optimizer">Optimizer</span>]</code>
             </td>
@@ -1738,7 +1746,9 @@ <h2 id="pytorch_widedeep.self_supervised_training.EncoderDecoderTrainer" class="
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>lr_scheduler</code></td>
+            <td>
+                <code>lr_scheduler</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.LRScheduler">LRScheduler</span>]</code>
             </td>
@@ -1753,7 +1763,9 @@ <h2 id="pytorch_widedeep.self_supervised_training.EncoderDecoderTrainer" class="
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>callbacks</code></td>
+            <td>
+                <code>callbacks</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.callbacks.Callback">Callback</span>]]</code>
             </td>
@@ -1771,7 +1783,9 @@ <h2 id="pytorch_widedeep.self_supervised_training.EncoderDecoderTrainer" class="
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>verbose</code></td>
+            <td>
+                <code>verbose</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -1785,7 +1799,9 @@ <h2 id="pytorch_widedeep.self_supervised_training.EncoderDecoderTrainer" class="
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>seed</code></td>
+            <td>
+                <code>seed</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -1841,10 +1857,14 @@ <h2 id="pytorch_widedeep.self_supervised_training.EncoderDecoderTrainer" class="
       </tbody>
     </table>
 
+
+
+
+
+
               <details class="quote">
                 <summary>Source code in <code>pytorch_widedeep/self_supervised_training/encoder_decoder_trainer.py</code></summary>
-                <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 28</span>
-<span class="normal"> 29</span>
+                <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 29</span>
 <span class="normal"> 30</span>
 <span class="normal"> 31</span>
 <span class="normal"> 32</span>
@@ -2078,7 +2098,8 @@ <h2 id="pytorch_widedeep.self_supervised_training.EncoderDecoderTrainer" class="
 <span class="normal">260</span>
 <span class="normal">261</span>
 <span class="normal">262</span>
-<span class="normal">263</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">class</span> <span class="nc">EncoderDecoderTrainer</span><span class="p">(</span><span class="n">BaseEncoderDecoderTrainer</span><span class="p">):</span>
+<span class="normal">263</span>
+<span class="normal">264</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">class</span> <span class="nc">EncoderDecoderTrainer</span><span class="p">(</span><span class="n">BaseEncoderDecoderTrainer</span><span class="p">):</span>
 <span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;This class implements an Encoder-Decoder self-supervised &#39;routine&#39;</span>
 <span class="sd">    inspired by</span>
 <span class="sd">    [TabNet: Attentive Interpretable Tabular Learning](https://arxiv.org/abs/1908.07442).</span>
@@ -2267,7 +2288,7 @@ <h2 id="pytorch_widedeep.self_supervised_training.EncoderDecoderTrainer" class="
         <span class="p">)</span>
 
     <span class="k">def</span> <span class="nf">_train_step</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">X_tab</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">,</span> <span class="n">batch_idx</span><span class="p">:</span> <span class="nb">int</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">float</span><span class="p">:</span>
-        <span class="n">X</span> <span class="o">=</span> <span class="n">X_tab</span><span class="o">.</span><span class="n">to</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">device</span><span class="p">)</span>
+        <span class="n">X</span> <span class="o">=</span> <span class="n">to_device</span><span class="p">(</span><span class="n">X_tab</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">device</span><span class="p">)</span>
 
         <span class="bp">self</span><span class="o">.</span><span class="n">optimizer</span><span class="o">.</span><span class="n">zero_grad</span><span class="p">()</span>
         <span class="n">x_embed</span><span class="p">,</span> <span class="n">x_embed_rec</span><span class="p">,</span> <span class="n">mask</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">ed_model</span><span class="p">(</span><span class="n">X</span><span class="p">)</span>
@@ -2284,7 +2305,7 @@ <h2 id="pytorch_widedeep.self_supervised_training.EncoderDecoderTrainer" class="
         <span class="bp">self</span><span class="o">.</span><span class="n">ed_model</span><span class="o">.</span><span class="n">eval</span><span class="p">()</span>
 
         <span class="k">with</span> <span class="n">torch</span><span class="o">.</span><span class="n">no_grad</span><span class="p">():</span>
-            <span class="n">X</span> <span class="o">=</span> <span class="n">X_tab</span><span class="o">.</span><span class="n">to</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">device</span><span class="p">)</span>
+            <span class="n">X</span> <span class="o">=</span> <span class="n">to_device</span><span class="p">(</span><span class="n">X_tab</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">device</span><span class="p">)</span>
 
             <span class="n">x_embed</span><span class="p">,</span> <span class="n">x_embed_rec</span><span class="p">,</span> <span class="n">mask</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">ed_model</span><span class="p">(</span><span class="n">X</span><span class="p">)</span>
             <span class="n">loss</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">loss_fn</span><span class="p">(</span><span class="n">x_embed</span><span class="p">,</span> <span class="n">x_embed_rec</span><span class="p">,</span> <span class="n">mask</span><span class="p">)</span>
@@ -2364,7 +2385,9 @@ <h3 id="pytorch_widedeep.self_supervised_training.EncoderDecoderTrainer.pretrain
       </thead>
       <tbody>
           <tr class="doc-section-item">
-            <td><code>X_tab</code></td>
+            <td>
+                <code>X_tab</code>
+            </td>
             <td>
                   <code><span title="numpy.ndarray">ndarray</span></code>
             </td>
@@ -2378,7 +2401,9 @@ <h3 id="pytorch_widedeep.self_supervised_training.EncoderDecoderTrainer.pretrain
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>X_tab_val</code></td>
+            <td>
+                <code>X_tab_val</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="numpy.ndarray">ndarray</span>]</code>
             </td>
@@ -2392,7 +2417,9 @@ <h3 id="pytorch_widedeep.self_supervised_training.EncoderDecoderTrainer.pretrain
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>val_split</code></td>
+            <td>
+                <code>val_split</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>
             </td>
@@ -2407,7 +2434,9 @@ <h3 id="pytorch_widedeep.self_supervised_training.EncoderDecoderTrainer.pretrain
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>validation_freq</code></td>
+            <td>
+                <code>validation_freq</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -2421,7 +2450,9 @@ <h3 id="pytorch_widedeep.self_supervised_training.EncoderDecoderTrainer.pretrain
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>n_epochs</code></td>
+            <td>
+                <code>n_epochs</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -2435,7 +2466,9 @@ <h3 id="pytorch_widedeep.self_supervised_training.EncoderDecoderTrainer.pretrain
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>batch_size</code></td>
+            <td>
+                <code>batch_size</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -2453,8 +2486,7 @@ <h3 id="pytorch_widedeep.self_supervised_training.EncoderDecoderTrainer.pretrain
 
             <details class="quote">
               <summary>Source code in <code>pytorch_widedeep/self_supervised_training/encoder_decoder_trainer.py</code></summary>
-              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">104</span>
-<span class="normal">105</span>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">105</span>
 <span class="normal">106</span>
 <span class="normal">107</span>
 <span class="normal">108</span>
@@ -2545,7 +2577,8 @@ <h3 id="pytorch_widedeep.self_supervised_training.EncoderDecoderTrainer.pretrain
 <span class="normal">193</span>
 <span class="normal">194</span>
 <span class="normal">195</span>
-<span class="normal">196</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">pretrain</span><span class="p">(</span>
+<span class="normal">196</span>
+<span class="normal">197</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">pretrain</span><span class="p">(</span>
     <span class="bp">self</span><span class="p">,</span>
     <span class="n">X_tab</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">,</span>
     <span class="n">X_tab_val</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
@@ -2686,7 +2719,9 @@ <h2 id="pytorch_widedeep.self_supervised_training.ContrastiveDenoisingTrainer" c
       </thead>
       <tbody>
           <tr class="doc-section-item">
-            <td><code>model</code></td>
+            <td>
+                <code>model</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.ModelWithAttention">ModelWithAttention</span></code>
             </td>
@@ -2702,7 +2737,9 @@ <h2 id="pytorch_widedeep.self_supervised_training.ContrastiveDenoisingTrainer" c
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>preprocessor</code></td>
+            <td>
+                <code>preprocessor</code>
+            </td>
             <td>
                   <code><a class="autorefs autorefs-internal" title="pytorch_widedeep.preprocessing.TabPreprocessor" href="preprocessing.html#pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor">TabPreprocessor</a></code>
             </td>
@@ -2717,7 +2754,9 @@ <h2 id="pytorch_widedeep.self_supervised_training.ContrastiveDenoisingTrainer" c
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>optimizer</code></td>
+            <td>
+                <code>optimizer</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Optimizer">Optimizer</span>]</code>
             </td>
@@ -2732,7 +2771,9 @@ <h2 id="pytorch_widedeep.self_supervised_training.ContrastiveDenoisingTrainer" c
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>lr_scheduler</code></td>
+            <td>
+                <code>lr_scheduler</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.LRScheduler">LRScheduler</span>]</code>
             </td>
@@ -2747,7 +2788,9 @@ <h2 id="pytorch_widedeep.self_supervised_training.ContrastiveDenoisingTrainer" c
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>callbacks</code></td>
+            <td>
+                <code>callbacks</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.callbacks.Callback">Callback</span>]]</code>
             </td>
@@ -2765,7 +2808,9 @@ <h2 id="pytorch_widedeep.self_supervised_training.ContrastiveDenoisingTrainer" c
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>loss_type</code></td>
+            <td>
+                <code>loss_type</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[contrastive, denoising, both]</code>
             </td>
@@ -2782,7 +2827,9 @@ <h2 id="pytorch_widedeep.self_supervised_training.ContrastiveDenoisingTrainer" c
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>projection_head1_dims</code></td>
+            <td>
+                <code>projection_head1_dims</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[int]]</code>
             </td>
@@ -2803,7 +2850,9 @@ <h2 id="pytorch_widedeep.self_supervised_training.ContrastiveDenoisingTrainer" c
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>projection_head2_dims</code></td>
+            <td>
+                <code>projection_head2_dims</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[int]]</code>
             </td>
@@ -2817,7 +2866,9 @@ <h2 id="pytorch_widedeep.self_supervised_training.ContrastiveDenoisingTrainer" c
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>projection_heads_activation</code></td>
+            <td>
+                <code>projection_heads_activation</code>
+            </td>
             <td>
                   <code>str</code>
             </td>
@@ -2831,7 +2882,9 @@ <h2 id="pytorch_widedeep.self_supervised_training.ContrastiveDenoisingTrainer" c
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>cat_mlp_type</code></td>
+            <td>
+                <code>cat_mlp_type</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[single, multiple]</code>
             </td>
@@ -2850,7 +2903,9 @@ <h2 id="pytorch_widedeep.self_supervised_training.ContrastiveDenoisingTrainer" c
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>cont_mlp_type</code></td>
+            <td>
+                <code>cont_mlp_type</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[single, multiple]</code>
             </td>
@@ -2864,7 +2919,9 @@ <h2 id="pytorch_widedeep.self_supervised_training.ContrastiveDenoisingTrainer" c
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>denoise_mlps_activation</code></td>
+            <td>
+                <code>denoise_mlps_activation</code>
+            </td>
             <td>
                   <code>str</code>
             </td>
@@ -2878,7 +2935,9 @@ <h2 id="pytorch_widedeep.self_supervised_training.ContrastiveDenoisingTrainer" c
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>verbose</code></td>
+            <td>
+                <code>verbose</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -2892,7 +2951,9 @@ <h2 id="pytorch_widedeep.self_supervised_training.ContrastiveDenoisingTrainer" c
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>seed</code></td>
+            <td>
+                <code>seed</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -2948,10 +3009,14 @@ <h2 id="pytorch_widedeep.self_supervised_training.ContrastiveDenoisingTrainer" c
       </tbody>
     </table>
 
+
+
+
+
+
               <details class="quote">
                 <summary>Source code in <code>pytorch_widedeep/self_supervised_training/contrastive_denoising_trainer.py</code></summary>
-                <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 29</span>
-<span class="normal"> 30</span>
+                <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 30</span>
 <span class="normal"> 31</span>
 <span class="normal"> 32</span>
 <span class="normal"> 33</span>
@@ -3227,7 +3292,8 @@ <h2 id="pytorch_widedeep.self_supervised_training.ContrastiveDenoisingTrainer" c
 <span class="normal">303</span>
 <span class="normal">304</span>
 <span class="normal">305</span>
-<span class="normal">306</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">class</span> <span class="nc">ContrastiveDenoisingTrainer</span><span class="p">(</span><span class="n">BaseContrastiveDenoisingTrainer</span><span class="p">):</span>
+<span class="normal">306</span>
+<span class="normal">307</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">class</span> <span class="nc">ContrastiveDenoisingTrainer</span><span class="p">(</span><span class="n">BaseContrastiveDenoisingTrainer</span><span class="p">):</span>
 <span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;This class trains a Contrastive, Denoising Self Supervised &#39;routine&#39; that</span>
 <span class="sd">    is based on the one described in</span>
 <span class="sd">    [SAINT: Improved Neural Networks for Tabular Data via Row Attention and</span>
@@ -3458,7 +3524,7 @@ <h2 id="pytorch_widedeep.self_supervised_training.ContrastiveDenoisingTrainer" c
         <span class="p">)</span>
 
     <span class="k">def</span> <span class="nf">_train_step</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">X_tab</span><span class="p">:</span> <span class="n">Tensor</span><span class="p">,</span> <span class="n">batch_idx</span><span class="p">:</span> <span class="nb">int</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">float</span><span class="p">:</span>
-        <span class="n">X</span> <span class="o">=</span> <span class="n">X_tab</span><span class="o">.</span><span class="n">to</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">device</span><span class="p">)</span>
+        <span class="n">X</span> <span class="o">=</span> <span class="n">to_device</span><span class="p">(</span><span class="n">X_tab</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">device</span><span class="p">)</span>
 
         <span class="bp">self</span><span class="o">.</span><span class="n">optimizer</span><span class="o">.</span><span class="n">zero_grad</span><span class="p">()</span>
         <span class="n">g_projs</span><span class="p">,</span> <span class="n">cat_x_and_x_</span><span class="p">,</span> <span class="n">cont_x_and_x_</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">cd_model</span><span class="p">(</span><span class="n">X</span><span class="p">)</span>
@@ -3475,7 +3541,7 @@ <h2 id="pytorch_widedeep.self_supervised_training.ContrastiveDenoisingTrainer" c
         <span class="bp">self</span><span class="o">.</span><span class="n">cd_model</span><span class="o">.</span><span class="n">eval</span><span class="p">()</span>
 
         <span class="k">with</span> <span class="n">torch</span><span class="o">.</span><span class="n">no_grad</span><span class="p">():</span>
-            <span class="n">X</span> <span class="o">=</span> <span class="n">X_tab</span><span class="o">.</span><span class="n">to</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">device</span><span class="p">)</span>
+            <span class="n">X</span> <span class="o">=</span> <span class="n">to_device</span><span class="p">(</span><span class="n">X_tab</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">device</span><span class="p">)</span>
 
             <span class="n">g_projs</span><span class="p">,</span> <span class="n">cat_x_and_x_</span><span class="p">,</span> <span class="n">cont_x_and_x_</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">cd_model</span><span class="p">(</span><span class="n">X</span><span class="p">)</span>
             <span class="n">loss</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_compute_loss</span><span class="p">(</span><span class="n">g_projs</span><span class="p">,</span> <span class="n">cat_x_and_x_</span><span class="p">,</span> <span class="n">cont_x_and_x_</span><span class="p">)</span>
@@ -3555,7 +3621,9 @@ <h3 id="pytorch_widedeep.self_supervised_training.ContrastiveDenoisingTrainer.pr
       </thead>
       <tbody>
           <tr class="doc-section-item">
-            <td><code>X_tab</code></td>
+            <td>
+                <code>X_tab</code>
+            </td>
             <td>
                   <code><span title="numpy.ndarray">ndarray</span></code>
             </td>
@@ -3569,7 +3637,9 @@ <h3 id="pytorch_widedeep.self_supervised_training.ContrastiveDenoisingTrainer.pr
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>X_tab_val</code></td>
+            <td>
+                <code>X_tab_val</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="numpy.ndarray">ndarray</span>]</code>
             </td>
@@ -3591,7 +3661,9 @@ <h3 id="pytorch_widedeep.self_supervised_training.ContrastiveDenoisingTrainer.pr
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>val_split</code></td>
+            <td>
+                <code>val_split</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>
             </td>
@@ -3606,7 +3678,9 @@ <h3 id="pytorch_widedeep.self_supervised_training.ContrastiveDenoisingTrainer.pr
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>validation_freq</code></td>
+            <td>
+                <code>validation_freq</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -3620,7 +3694,9 @@ <h3 id="pytorch_widedeep.self_supervised_training.ContrastiveDenoisingTrainer.pr
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>n_epochs</code></td>
+            <td>
+                <code>n_epochs</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -3634,7 +3710,9 @@ <h3 id="pytorch_widedeep.self_supervised_training.ContrastiveDenoisingTrainer.pr
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>batch_size</code></td>
+            <td>
+                <code>batch_size</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -3652,8 +3730,7 @@ <h3 id="pytorch_widedeep.self_supervised_training.ContrastiveDenoisingTrainer.pr
 
             <details class="quote">
               <summary>Source code in <code>pytorch_widedeep/self_supervised_training/contrastive_denoising_trainer.py</code></summary>
-              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">144</span>
-<span class="normal">145</span>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">145</span>
 <span class="normal">146</span>
 <span class="normal">147</span>
 <span class="normal">148</span>
@@ -3752,7 +3829,8 @@ <h3 id="pytorch_widedeep.self_supervised_training.ContrastiveDenoisingTrainer.pr
 <span class="normal">241</span>
 <span class="normal">242</span>
 <span class="normal">243</span>
-<span class="normal">244</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">pretrain</span><span class="p">(</span>
+<span class="normal">244</span>
+<span class="normal">245</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">pretrain</span><span class="p">(</span>
     <span class="bp">self</span><span class="p">,</span>
     <span class="n">X_tab</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">,</span>
     <span class="n">X_tab_val</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
@@ -3957,7 +4035,7 @@ <h3 id="pytorch_widedeep.self_supervised_training.ContrastiveDenoisingTrainer.pr
     <script id="__config" type="application/json">{"base": "..", "features": ["navigation.tabs", "navigation.tabs.sticky", "navigation.indexes", "navigation.expand", "toc.integrate"], "search": "../assets/javascripts/workers/search.6ce7567c.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version": "Select version"}}</script>
     
     
-      <script src="../assets/javascripts/bundle.56dfad97.min.js"></script>
+      <script src="../assets/javascripts/bundle.83f73b43.min.js"></script>
       
         <script src="../stylesheets/extra.js"></script>
       
diff --git a/mkdocs/site/pytorch-widedeep/self_supervised_pretraining.md b/mkdocs/site/pytorch-widedeep/self_supervised_pretraining.md
deleted file mode 100644
index 05e39060..00000000
--- a/mkdocs/site/pytorch-widedeep/self_supervised_pretraining.md
+++ /dev/null
@@ -1,65 +0,0 @@
-# Self Supervised Pre-training for tabular data
-
-In this library we have implemented two methods or routines that allow the
-user to use self-suerpvised pre-training for all tabular models in the library
-with the exception of the `TabPerceiver` (this is a particular model and
-self-supervised pre-training requires some adjustments that will be
-implemented in future versions). Please see the examples folder in the repo
-or the examples section in the docs for details on how to use self-supervised
-pre-training with this library.
-
-
-The two routines implemented are illustrated in the figures below. The first
-is from [TabNet: Attentive Interpretable Tabular Learning](https://arxiv.org/abs/1908.07442).
-It is a *'standard'* encoder-decoder architecture and and is designed here for
-models that do not use transformer-based architectures (or when the
-embeddings can all have different dimensions). The second is from
-[SAINT: Improved Neural Networks for Tabular Data via Row Attention and
-Contrastive Pre-Training](https://arxiv.org/abs/2203.05556), it is based on
-Contrastive and Denoising learning and is designed for models that use
-transformer-based architectures (or when the embeddings all need to have the
-same dimension):
-
-<p align="center">
-  <img width="750" src="../docs/figures/self_supervised_tabnet.png">
-</p>
-
-Figure 1. Figure 2 in their [paper](https://arxiv.org/abs/1908.07442). The
-caption of the original paper is included in case it is useful.
-
-<p align="center">
-  <img width="700" src="../docs/figures/self_supervised_saint.png">
-</p>
-
-Figure 2. Figure 1 in their [paper](https://arxiv.org/abs/2203.05556). The
-caption of the original paper is included in case it is useful.
-
-Note that the self-supervised pre-trainers described below focus, of course,
-on the self-supervised pre-training phase, i.e. the left side in Figure 1 and
-the upper part in Figure 2. When combined with the `Trainer` described
-earlier in the documenation, one can reproduce the full process illustrated
-in the figures above.
-
-Also Note that it is beyond the scope of this docs to explain in detail these
-routines. In addition, to fully utilise the self-supervised trainers
-implemented in this library a minimum understanding of the processes as
-described in the papers is required. Therefore, we strongly encourage the
-users to have a look to the papers.
-
-
-::: pytorch_widedeep.self_supervised_training.EncoderDecoderTrainer
-    selection:
-        filters:
-            - "!^_"  # exclude all members starting with _
-            - "!^explain"
-            - "!^fit"
-
-
-::: pytorch_widedeep.self_supervised_training.ContrastiveDenoisingTrainer
-    selection:
-        filters:
-            - "!^_"  # exclude all members starting with _
-            - "!^explain"
-            - "!^fit"
-
-
diff --git a/mkdocs/site/pytorch-widedeep/tab2vec.html b/mkdocs/site/pytorch-widedeep/tab2vec.html
index 37e611a0..3bf92916 100644
--- a/mkdocs/site/pytorch-widedeep/tab2vec.html
+++ b/mkdocs/site/pytorch-widedeep/tab2vec.html
@@ -18,7 +18,7 @@
       
       
       <link rel="icon" href="../assets/images/favicon.ico">
-      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.35">
+      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.43">
     
     
       
@@ -26,7 +26,7 @@
       
     
     
-      <link rel="stylesheet" href="../assets/stylesheets/main.35f28582.min.css">
+      <link rel="stylesheet" href="../assets/stylesheets/main.0253249f.min.css">
       
         
         <link rel="stylesheet" href="../assets/stylesheets/palette.06af60db.min.css">
@@ -1633,7 +1633,9 @@ <h2 id="pytorch_widedeep.tab2vec.Tab2Vec" class="doc doc-heading">
       </thead>
       <tbody>
           <tr class="doc-section-item">
-            <td><code>model</code></td>
+            <td>
+                <code>model</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Union">Union</span>[<a class="autorefs autorefs-internal" title="pytorch_widedeep.wdtypes.WideDeep" href="model_components.html#pytorch_widedeep.models.wide_deep.WideDeep">WideDeep</a>, <a class="autorefs autorefs-internal" title="pytorch_widedeep.bayesian_models.BayesianWide" href="bayesian_models.html#pytorch_widedeep.bayesian_models.tabular.bayesian_linear.bayesian_wide.BayesianWide">BayesianWide</a>, <a class="autorefs autorefs-internal" title="pytorch_widedeep.bayesian_models.BayesianTabMlp" href="bayesian_models.html#pytorch_widedeep.bayesian_models.tabular.bayesian_mlp.bayesian_tab_mlp.BayesianTabMlp">BayesianTabMlp</a>]</code>
             </td>
@@ -1647,7 +1649,9 @@ <h2 id="pytorch_widedeep.tab2vec.Tab2Vec" class="doc doc-heading">
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>tab_preprocessor</code></td>
+            <td>
+                <code>tab_preprocessor</code>
+            </td>
             <td>
                   <code><a class="autorefs autorefs-internal" title="pytorch_widedeep.preprocessing.TabPreprocessor" href="preprocessing.html#pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor">TabPreprocessor</a></code>
             </td>
@@ -1661,7 +1665,9 @@ <h2 id="pytorch_widedeep.tab2vec.Tab2Vec" class="doc doc-heading">
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>return_dataframe</code></td>
+            <td>
+                <code>return_dataframe</code>
+            </td>
             <td>
                   <code>bool</code>
             </td>
@@ -1745,6 +1751,11 @@ <h2 id="pytorch_widedeep.tab2vec.Tab2Vec" class="doc doc-heading">
 <span class="gp">&gt;&gt;&gt; </span><span class="n">X_vec</span> <span class="o">=</span> <span class="n">t2v</span><span class="o">.</span><span class="n">transform</span><span class="p">(</span><span class="n">df_t2v</span><span class="p">)</span>
 </code></pre></div>
 
+
+
+
+
+
               <details class="quote">
                 <summary>Source code in <code>pytorch_widedeep/tab2vec.py</code></summary>
                 <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 14</span>
@@ -2149,7 +2160,7 @@ <h2 id="pytorch_widedeep.tab2vec.Tab2Vec" class="doc doc-heading">
         <span class="bp">self</span><span class="p">,</span>
         <span class="n">tab_preprocessor</span><span class="p">:</span> <span class="n">TabPreprocessor</span><span class="p">,</span>
         <span class="n">model</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="n">WideDeep</span><span class="p">,</span> <span class="n">BayesianWide</span><span class="p">,</span> <span class="n">BayesianTabMlp</span><span class="p">],</span>
-        <span class="n">device</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="n">torch</span><span class="o">.</span><span class="n">device</span><span class="p">,</span>
+        <span class="n">device</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">torch</span><span class="o">.</span><span class="n">device</span><span class="p">],</span>
         <span class="n">return_dataframe</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
         <span class="n">verbose</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
     <span class="p">):</span>
@@ -2449,7 +2460,9 @@ <h3 id="pytorch_widedeep.tab2vec.Tab2Vec.fit" class="doc doc-heading">
       </thead>
       <tbody>
           <tr class="doc-section-item">
-            <td><code>df</code></td>
+            <td>
+                <code>df</code>
+            </td>
             <td>
                   <code><span title="pandas.DataFrame">DataFrame</span></code>
             </td>
@@ -2465,7 +2478,9 @@ <h3 id="pytorch_widedeep.tab2vec.Tab2Vec.fit" class="doc doc-heading">
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>target_col</code></td>
+            <td>
+                <code>target_col</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>
             </td>
@@ -2584,7 +2599,9 @@ <h3 id="pytorch_widedeep.tab2vec.Tab2Vec.transform" class="doc doc-heading">
       </thead>
       <tbody>
           <tr class="doc-section-item">
-            <td><code>df</code></td>
+            <td>
+                <code>df</code>
+            </td>
             <td>
                   <code><span title="pandas.DataFrame">DataFrame</span></code>
             </td>
@@ -2600,7 +2617,9 @@ <h3 id="pytorch_widedeep.tab2vec.Tab2Vec.transform" class="doc doc-heading">
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>target_col</code></td>
+            <td>
+                <code>target_col</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[str]</code>
             </td>
@@ -2927,7 +2946,7 @@ <h3 id="pytorch_widedeep.tab2vec.Tab2Vec.fit_transform" class="doc doc-heading">
     <script id="__config" type="application/json">{"base": "..", "features": ["navigation.tabs", "navigation.tabs.sticky", "navigation.indexes", "navigation.expand", "toc.integrate"], "search": "../assets/javascripts/workers/search.6ce7567c.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version": "Select version"}}</script>
     
     
-      <script src="../assets/javascripts/bundle.56dfad97.min.js"></script>
+      <script src="../assets/javascripts/bundle.83f73b43.min.js"></script>
       
         <script src="../stylesheets/extra.js"></script>
       
diff --git a/mkdocs/site/pytorch-widedeep/tab2vec.md b/mkdocs/site/pytorch-widedeep/tab2vec.md
deleted file mode 100644
index 78271b63..00000000
--- a/mkdocs/site/pytorch-widedeep/tab2vec.md
+++ /dev/null
@@ -1,3 +0,0 @@
-# Tab2Vec
-
-::: pytorch_widedeep.tab2vec.Tab2Vec
diff --git a/mkdocs/site/pytorch-widedeep/trainer.html b/mkdocs/site/pytorch-widedeep/trainer.html
index c9ac062d..39f867d1 100644
--- a/mkdocs/site/pytorch-widedeep/trainer.html
+++ b/mkdocs/site/pytorch-widedeep/trainer.html
@@ -18,7 +18,7 @@
       
       
       <link rel="icon" href="../assets/images/favicon.ico">
-      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.35">
+      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.43">
     
     
       
@@ -26,7 +26,7 @@
       
     
     
-      <link rel="stylesheet" href="../assets/stylesheets/main.35f28582.min.css">
+      <link rel="stylesheet" href="../assets/stylesheets/main.0253249f.min.css">
       
         
         <link rel="stylesheet" href="../assets/stylesheets/palette.06af60db.min.css">
@@ -1635,7 +1635,9 @@ <h2 id="pytorch_widedeep.training.Trainer" class="doc doc-heading">
       </thead>
       <tbody>
           <tr class="doc-section-item">
-            <td><code>model</code></td>
+            <td>
+                <code>model</code>
+            </td>
             <td>
                   <code><a class="autorefs autorefs-internal" title="pytorch_widedeep.wdtypes.WideDeep" href="model_components.html#pytorch_widedeep.models.wide_deep.WideDeep">WideDeep</a></code>
             </td>
@@ -1649,7 +1651,9 @@ <h2 id="pytorch_widedeep.training.Trainer" class="doc doc-heading">
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>objective</code></td>
+            <td>
+                <code>objective</code>
+            </td>
             <td>
                   <code>str</code>
             </td>
@@ -1708,7 +1712,9 @@ <h2 id="pytorch_widedeep.training.Trainer" class="doc doc-heading">
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>custom_loss_function</code></td>
+            <td>
+                <code>custom_loss_function</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="torch.nn.Module">Module</span>]</code>
             </td>
@@ -1727,7 +1733,9 @@ <h2 id="pytorch_widedeep.training.Trainer" class="doc doc-heading">
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>optimizers</code></td>
+            <td>
+                <code>optimizers</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="pytorch_widedeep.wdtypes.Optimizer">Optimizer</span>, <span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, <span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="pytorch_widedeep.wdtypes.Optimizer">Optimizer</span>, <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.wdtypes.Optimizer">Optimizer</span>]]]]]</code>
             </td>
@@ -1753,7 +1761,9 @@ <h2 id="pytorch_widedeep.training.Trainer" class="doc doc-heading">
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>lr_schedulers</code></td>
+            <td>
+                <code>lr_schedulers</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="pytorch_widedeep.wdtypes.LRScheduler">LRScheduler</span>, <span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, <span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="pytorch_widedeep.wdtypes.LRScheduler">LRScheduler</span>, <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.wdtypes.LRScheduler">LRScheduler</span>]]]]]</code>
             </td>
@@ -1775,7 +1785,9 @@ <h2 id="pytorch_widedeep.training.Trainer" class="doc doc-heading">
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>initializers</code></td>
+            <td>
+                <code>initializers</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="pytorch_widedeep.initializers.Initializer">Initializer</span>, <span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, <span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="pytorch_widedeep.initializers.Initializer">Initializer</span>, <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.initializers.Initializer">Initializer</span>]]]]]</code>
             </td>
@@ -1796,7 +1808,9 @@ <h2 id="pytorch_widedeep.training.Trainer" class="doc doc-heading">
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>transforms</code></td>
+            <td>
+                <code>transforms</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.wdtypes.Transforms">Transforms</span>]]</code>
             </td>
@@ -1812,7 +1826,9 @@ <h2 id="pytorch_widedeep.training.Trainer" class="doc doc-heading">
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>callbacks</code></td>
+            <td>
+                <code>callbacks</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.callbacks.Callback">Callback</span>]]</code>
             </td>
@@ -1832,7 +1848,9 @@ <h2 id="pytorch_widedeep.training.Trainer" class="doc doc-heading">
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>metrics</code></td>
+            <td>
+                <code>metrics</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.metrics.Metric">Metric</span>], <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="torchmetrics.Metric">Metric</span>]]]</code>
             </td>
@@ -1859,7 +1877,9 @@ <h2 id="pytorch_widedeep.training.Trainer" class="doc doc-heading">
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>verbose</code></td>
+            <td>
+                <code>verbose</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -1873,7 +1893,9 @@ <h2 id="pytorch_widedeep.training.Trainer" class="doc doc-heading">
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>seed</code></td>
+            <td>
+                <code>seed</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -2022,6 +2044,11 @@ <h2 id="pytorch_widedeep.training.Trainer" class="doc doc-heading">
 <span class="gp">... </span><span class="n">lr_schedulers</span><span class="o">=</span><span class="n">schedulers</span><span class="p">,</span> <span class="n">callbacks</span><span class="o">=</span><span class="n">callbacks</span><span class="p">,</span> <span class="n">transforms</span><span class="o">=</span><span class="n">transforms</span><span class="p">)</span>
 </code></pre></div>
 
+
+
+
+
+
               <details class="quote">
                 <summary>Source code in <code>pytorch_widedeep/training/trainer.py</code></summary>
                 <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">  43</span>
@@ -3920,7 +3947,7 @@ <h2 id="pytorch_widedeep.training.Trainer" class="doc doc-heading">
         <span class="n">dataset</span><span class="p">:</span> <span class="n">WideDeepDataset</span><span class="p">,</span>
         <span class="n">batch_size</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span>
         <span class="n">dataloader_args</span><span class="p">:</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">],</span>
-    <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">DataLoader</span> <span class="o">|</span> <span class="n">CustomDataLoader</span><span class="p">:</span>
+    <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Union</span><span class="p">[</span><span class="n">DataLoader</span><span class="p">,</span> <span class="n">CustomDataLoader</span><span class="p">]:</span>
         <span class="k">if</span> <span class="n">dataloader</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
             <span class="n">dataloader</span><span class="o">.</span><span class="n">set_dataset</span><span class="p">(</span><span class="n">dataset</span><span class="p">)</span>
             <span class="k">return</span> <span class="n">dataloader</span>
@@ -4386,7 +4413,9 @@ <h3 id="pytorch_widedeep.training.Trainer.fit" class="doc doc-heading">
       </thead>
       <tbody>
           <tr class="doc-section-item">
-            <td><code>X_wide</code></td>
+            <td>
+                <code>X_wide</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="numpy.ndarray">ndarray</span>]</code>
             </td>
@@ -4401,7 +4430,9 @@ <h3 id="pytorch_widedeep.training.Trainer.fit" class="doc doc-heading">
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>X_tab</code></td>
+            <td>
+                <code>X_tab</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="numpy.ndarray">ndarray</span>, <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="numpy.ndarray">ndarray</span>]]]</code>
             </td>
@@ -4418,7 +4449,9 @@ <h3 id="pytorch_widedeep.training.Trainer.fit" class="doc doc-heading">
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>X_text</code></td>
+            <td>
+                <code>X_text</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="numpy.ndarray">ndarray</span>, <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="numpy.ndarray">ndarray</span>]]]</code>
             </td>
@@ -4435,7 +4468,9 @@ <h3 id="pytorch_widedeep.training.Trainer.fit" class="doc doc-heading">
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>X_img</code></td>
+            <td>
+                <code>X_img</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="numpy.ndarray">ndarray</span>, <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="numpy.ndarray">ndarray</span>]]]</code>
             </td>
@@ -4452,7 +4487,9 @@ <h3 id="pytorch_widedeep.training.Trainer.fit" class="doc doc-heading">
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>X_train</code></td>
+            <td>
+                <code>X_train</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, <span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="numpy.ndarray">ndarray</span>, <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="numpy.ndarray">ndarray</span>]]]]</code>
             </td>
@@ -4470,7 +4507,9 @@ <h3 id="pytorch_widedeep.training.Trainer.fit" class="doc doc-heading">
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>X_val</code></td>
+            <td>
+                <code>X_val</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, <span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="numpy.ndarray">ndarray</span>, <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="numpy.ndarray">ndarray</span>]]]]</code>
             </td>
@@ -4488,7 +4527,9 @@ <h3 id="pytorch_widedeep.training.Trainer.fit" class="doc doc-heading">
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>val_split</code></td>
+            <td>
+                <code>val_split</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[float]</code>
             </td>
@@ -4502,7 +4543,9 @@ <h3 id="pytorch_widedeep.training.Trainer.fit" class="doc doc-heading">
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>target</code></td>
+            <td>
+                <code>target</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="numpy.ndarray">ndarray</span>]</code>
             </td>
@@ -4516,7 +4559,9 @@ <h3 id="pytorch_widedeep.training.Trainer.fit" class="doc doc-heading">
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>n_epochs</code></td>
+            <td>
+                <code>n_epochs</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -4530,7 +4575,9 @@ <h3 id="pytorch_widedeep.training.Trainer.fit" class="doc doc-heading">
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>validation_freq</code></td>
+            <td>
+                <code>validation_freq</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -4544,7 +4591,9 @@ <h3 id="pytorch_widedeep.training.Trainer.fit" class="doc doc-heading">
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>batch_size</code></td>
+            <td>
+                <code>batch_size</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -4558,7 +4607,9 @@ <h3 id="pytorch_widedeep.training.Trainer.fit" class="doc doc-heading">
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>custom_dataloader</code></td>
+            <td>
+                <code>custom_dataloader</code>
+            </td>
             <td>
             </td>
             <td>
@@ -4573,7 +4624,9 @@ <h3 id="pytorch_widedeep.training.Trainer.fit" class="doc doc-heading">
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>finetune</code></td>
+            <td>
+                <code>finetune</code>
+            </td>
             <td>
                   <code>bool</code>
             </td>
@@ -5177,7 +5230,9 @@ <h3 id="pytorch_widedeep.training.Trainer.predict" class="doc doc-heading">
       </thead>
       <tbody>
           <tr class="doc-section-item">
-            <td><code>X_wide</code></td>
+            <td>
+                <code>X_wide</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="numpy.ndarray">ndarray</span>]</code>
             </td>
@@ -5192,7 +5247,9 @@ <h3 id="pytorch_widedeep.training.Trainer.predict" class="doc doc-heading">
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>X_tab</code></td>
+            <td>
+                <code>X_tab</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="numpy.ndarray">ndarray</span>, <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="numpy.ndarray">ndarray</span>]]]</code>
             </td>
@@ -5207,7 +5264,9 @@ <h3 id="pytorch_widedeep.training.Trainer.predict" class="doc doc-heading">
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>X_text</code></td>
+            <td>
+                <code>X_text</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="numpy.ndarray">ndarray</span>, <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="numpy.ndarray">ndarray</span>]]]</code>
             </td>
@@ -5222,7 +5281,9 @@ <h3 id="pytorch_widedeep.training.Trainer.predict" class="doc doc-heading">
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>X_img</code></td>
+            <td>
+                <code>X_img</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="numpy.ndarray">ndarray</span>, <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="numpy.ndarray">ndarray</span>]]]</code>
             </td>
@@ -5237,7 +5298,9 @@ <h3 id="pytorch_widedeep.training.Trainer.predict" class="doc doc-heading">
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>X_test</code></td>
+            <td>
+                <code>X_test</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, <span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="numpy.ndarray">ndarray</span>, <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="numpy.ndarray">ndarray</span>]]]]</code>
             </td>
@@ -5253,7 +5316,9 @@ <h3 id="pytorch_widedeep.training.Trainer.predict" class="doc doc-heading">
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>batch_size</code></td>
+            <td>
+                <code>batch_size</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[int]</code>
             </td>
@@ -5473,7 +5538,9 @@ <h3 id="pytorch_widedeep.training.Trainer.predict_uncertainty" class="doc doc-he
       </thead>
       <tbody>
           <tr class="doc-section-item">
-            <td><code>X_wide</code></td>
+            <td>
+                <code>X_wide</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="numpy.ndarray">ndarray</span>]</code>
             </td>
@@ -5488,7 +5555,9 @@ <h3 id="pytorch_widedeep.training.Trainer.predict_uncertainty" class="doc doc-he
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>X_tab</code></td>
+            <td>
+                <code>X_tab</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="numpy.ndarray">ndarray</span>, <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="numpy.ndarray">ndarray</span>]]]</code>
             </td>
@@ -5503,7 +5572,9 @@ <h3 id="pytorch_widedeep.training.Trainer.predict_uncertainty" class="doc doc-he
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>X_text</code></td>
+            <td>
+                <code>X_text</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="numpy.ndarray">ndarray</span>, <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="numpy.ndarray">ndarray</span>]]]</code>
             </td>
@@ -5518,7 +5589,9 @@ <h3 id="pytorch_widedeep.training.Trainer.predict_uncertainty" class="doc doc-he
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>X_img</code></td>
+            <td>
+                <code>X_img</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="numpy.ndarray">ndarray</span>, <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="numpy.ndarray">ndarray</span>]]]</code>
             </td>
@@ -5533,7 +5606,9 @@ <h3 id="pytorch_widedeep.training.Trainer.predict_uncertainty" class="doc doc-he
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>X_test</code></td>
+            <td>
+                <code>X_test</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, <span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="numpy.ndarray">ndarray</span>, <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="numpy.ndarray">ndarray</span>]]]]</code>
             </td>
@@ -5549,7 +5624,9 @@ <h3 id="pytorch_widedeep.training.Trainer.predict_uncertainty" class="doc doc-he
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>batch_size</code></td>
+            <td>
+                <code>batch_size</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[int]</code>
             </td>
@@ -5565,7 +5642,9 @@ <h3 id="pytorch_widedeep.training.Trainer.predict_uncertainty" class="doc doc-he
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>uncertainty_granularity</code></td>
+            <td>
+                <code>uncertainty_granularity</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -5860,7 +5939,9 @@ <h3 id="pytorch_widedeep.training.Trainer.predict_proba" class="doc doc-heading"
       </thead>
       <tbody>
           <tr class="doc-section-item">
-            <td><code>X_wide</code></td>
+            <td>
+                <code>X_wide</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="numpy.ndarray">ndarray</span>]</code>
             </td>
@@ -5875,7 +5956,9 @@ <h3 id="pytorch_widedeep.training.Trainer.predict_proba" class="doc doc-heading"
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>X_tab</code></td>
+            <td>
+                <code>X_tab</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="numpy.ndarray">ndarray</span>, <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="numpy.ndarray">ndarray</span>]]]</code>
             </td>
@@ -5890,7 +5973,9 @@ <h3 id="pytorch_widedeep.training.Trainer.predict_proba" class="doc doc-heading"
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>X_text</code></td>
+            <td>
+                <code>X_text</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="numpy.ndarray">ndarray</span>, <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="numpy.ndarray">ndarray</span>]]]</code>
             </td>
@@ -5905,7 +5990,9 @@ <h3 id="pytorch_widedeep.training.Trainer.predict_proba" class="doc doc-heading"
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>X_img</code></td>
+            <td>
+                <code>X_img</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="numpy.ndarray">ndarray</span>, <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="numpy.ndarray">ndarray</span>]]]</code>
             </td>
@@ -5920,7 +6007,9 @@ <h3 id="pytorch_widedeep.training.Trainer.predict_proba" class="doc doc-heading"
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>X_test</code></td>
+            <td>
+                <code>X_test</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, <span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="numpy.ndarray">ndarray</span>, <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="numpy.ndarray">ndarray</span>]]]]</code>
             </td>
@@ -5936,7 +6025,9 @@ <h3 id="pytorch_widedeep.training.Trainer.predict_proba" class="doc doc-heading"
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>batch_size</code></td>
+            <td>
+                <code>batch_size</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[int]</code>
             </td>
@@ -6145,7 +6236,9 @@ <h3 id="pytorch_widedeep.training.Trainer.save" class="doc doc-heading">
       </thead>
       <tbody>
           <tr class="doc-section-item">
-            <td><code>path</code></td>
+            <td>
+                <code>path</code>
+            </td>
             <td>
                   <code>str</code>
             </td>
@@ -6160,7 +6253,9 @@ <h3 id="pytorch_widedeep.training.Trainer.save" class="doc doc-heading">
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>save_state_dict</code></td>
+            <td>
+                <code>save_state_dict</code>
+            </td>
             <td>
                   <code>bool</code>
             </td>
@@ -6176,7 +6271,9 @@ <h3 id="pytorch_widedeep.training.Trainer.save" class="doc doc-heading">
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>save_optimizer</code></td>
+            <td>
+                <code>save_optimizer</code>
+            </td>
             <td>
                   <code>bool</code>
             </td>
@@ -6190,7 +6287,9 @@ <h3 id="pytorch_widedeep.training.Trainer.save" class="doc doc-heading">
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>model_filename</code></td>
+            <td>
+                <code>model_filename</code>
+            </td>
             <td>
                   <code>str</code>
             </td>
@@ -6413,7 +6512,7 @@ <h3 id="pytorch_widedeep.training.Trainer.save" class="doc doc-heading">
     <script id="__config" type="application/json">{"base": "..", "features": ["navigation.tabs", "navigation.tabs.sticky", "navigation.indexes", "navigation.expand", "toc.integrate"], "search": "../assets/javascripts/workers/search.6ce7567c.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version": "Select version"}}</script>
     
     
-      <script src="../assets/javascripts/bundle.56dfad97.min.js"></script>
+      <script src="../assets/javascripts/bundle.83f73b43.min.js"></script>
       
         <script src="../stylesheets/extra.js"></script>
       
diff --git a/mkdocs/site/pytorch-widedeep/trainer.md b/mkdocs/site/pytorch-widedeep/trainer.md
deleted file mode 100644
index 656e63aa..00000000
--- a/mkdocs/site/pytorch-widedeep/trainer.md
+++ /dev/null
@@ -1,21 +0,0 @@
-# Training multimodal Deep Learning Models
-
-Here is the documentation for the ``Trainer`` class, that will do all the heavy lifting.
-
-Trainer is also available from ``pytorch-widedeep`` directly, for example, one could do:
-
-```
-    from pytorch-widedeep.training import Trainer
-```
-
-or also:
-
-```
-    from pytorch-widedeep import Trainer
-```
-
-::: pytorch_widedeep.training.Trainer
-    selection:
-        filters:
-            - "!^_"  # exclude all members starting with _
-            - "!^get_embeddings$"
diff --git a/mkdocs/site/pytorch-widedeep/utils/deeptabular_utils.html b/mkdocs/site/pytorch-widedeep/utils/deeptabular_utils.html
index 64eb03e9..fe01be47 100644
--- a/mkdocs/site/pytorch-widedeep/utils/deeptabular_utils.html
+++ b/mkdocs/site/pytorch-widedeep/utils/deeptabular_utils.html
@@ -18,7 +18,7 @@
       
       
       <link rel="icon" href="../../assets/images/favicon.ico">
-      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.35">
+      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.43">
     
     
       
@@ -26,7 +26,7 @@
       
     
     
-      <link rel="stylesheet" href="../../assets/stylesheets/main.35f28582.min.css">
+      <link rel="stylesheet" href="../../assets/stylesheets/main.0253249f.min.css">
       
         
         <link rel="stylesheet" href="../../assets/stylesheets/palette.06af60db.min.css">
@@ -1628,7 +1628,9 @@ <h2 id="pytorch_widedeep.utils.deeptabular_utils.LabelEncoder" class="doc doc-he
       </thead>
       <tbody>
           <tr class="doc-section-item">
-            <td><code>columns_to_encode</code></td>
+            <td>
+                <code>columns_to_encode</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[str]]</code>
             </td>
@@ -1644,7 +1646,9 @@ <h2 id="pytorch_widedeep.utils.deeptabular_utils.LabelEncoder" class="doc doc-he
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>with_attention</code></td>
+            <td>
+                <code>with_attention</code>
+            </td>
             <td>
                   <code>bool</code>
             </td>
@@ -1659,7 +1663,9 @@ <h2 id="pytorch_widedeep.utils.deeptabular_utils.LabelEncoder" class="doc doc-he
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>shared_embed</code></td>
+            <td>
+                <code>shared_embed</code>
+            </td>
             <td>
                   <code>bool</code>
             </td>
@@ -1719,6 +1725,11 @@ <h2 id="pytorch_widedeep.utils.deeptabular_utils.LabelEncoder" class="doc doc-he
       </tbody>
     </table>
 
+
+
+
+
+
               <details class="quote">
                 <summary>Source code in <code>pytorch_widedeep/utils/deeptabular_utils.py</code></summary>
                 <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 14</span>
@@ -2920,7 +2931,7 @@ <h3 id="pytorch_widedeep.utils.deeptabular_utils.LabelEncoder.inverse_transform"
     <script id="__config" type="application/json">{"base": "../..", "features": ["navigation.tabs", "navigation.tabs.sticky", "navigation.indexes", "navigation.expand", "toc.integrate"], "search": "../../assets/javascripts/workers/search.6ce7567c.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version": "Select version"}}</script>
     
     
-      <script src="../../assets/javascripts/bundle.56dfad97.min.js"></script>
+      <script src="../../assets/javascripts/bundle.83f73b43.min.js"></script>
       
         <script src="../../stylesheets/extra.js"></script>
       
diff --git a/mkdocs/site/pytorch-widedeep/utils/deeptabular_utils.md b/mkdocs/site/pytorch-widedeep/utils/deeptabular_utils.md
deleted file mode 100644
index edc0ede1..00000000
--- a/mkdocs/site/pytorch-widedeep/utils/deeptabular_utils.md
+++ /dev/null
@@ -1,3 +0,0 @@
-# deeptabular utils
-
-::: pytorch_widedeep.utils.deeptabular_utils.LabelEncoder
diff --git a/mkdocs/site/pytorch-widedeep/utils/fastai_transforms.html b/mkdocs/site/pytorch-widedeep/utils/fastai_transforms.html
index 900ecef5..1e8bf7dd 100644
--- a/mkdocs/site/pytorch-widedeep/utils/fastai_transforms.html
+++ b/mkdocs/site/pytorch-widedeep/utils/fastai_transforms.html
@@ -18,7 +18,7 @@
       
       
       <link rel="icon" href="../../assets/images/favicon.ico">
-      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.35">
+      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.43">
     
     
       
@@ -26,7 +26,7 @@
       
     
     
-      <link rel="stylesheet" href="../../assets/stylesheets/main.35f28582.min.css">
+      <link rel="stylesheet" href="../../assets/stylesheets/main.0253249f.min.css">
       
         
         <link rel="stylesheet" href="../../assets/stylesheets/palette.06af60db.min.css">
@@ -1644,7 +1644,9 @@ <h2 id="pytorch_widedeep.utils.fastai_transforms.Tokenizer" class="doc doc-headi
       </thead>
       <tbody>
           <tr class="doc-section-item">
-            <td><code>tok_func</code></td>
+            <td>
+                <code>tok_func</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Callable">Callable</span></code>
             </td>
@@ -1658,7 +1660,9 @@ <h2 id="pytorch_widedeep.utils.fastai_transforms.Tokenizer" class="doc doc-headi
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>lang</code></td>
+            <td>
+                <code>lang</code>
+            </td>
             <td>
                   <code>str</code>
             </td>
@@ -1672,7 +1676,9 @@ <h2 id="pytorch_widedeep.utils.fastai_transforms.Tokenizer" class="doc doc-headi
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>pre_rules</code></td>
+            <td>
+                <code>pre_rules</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.ListRules">ListRules</span>]</code>
             </td>
@@ -1688,7 +1694,9 @@ <h2 id="pytorch_widedeep.utils.fastai_transforms.Tokenizer" class="doc doc-headi
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>post_rules</code></td>
+            <td>
+                <code>post_rules</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.ListRules">ListRules</span>]</code>
             </td>
@@ -1704,7 +1712,9 @@ <h2 id="pytorch_widedeep.utils.fastai_transforms.Tokenizer" class="doc doc-headi
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>special_cases</code></td>
+            <td>
+                <code>special_cases</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Collection">Collection</span>[str]]</code>
             </td>
@@ -1719,7 +1729,9 @@ <h2 id="pytorch_widedeep.utils.fastai_transforms.Tokenizer" class="doc doc-headi
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>n_cpus</code></td>
+            <td>
+                <code>n_cpus</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[int]</code>
             </td>
@@ -1735,6 +1747,11 @@ <h2 id="pytorch_widedeep.utils.fastai_transforms.Tokenizer" class="doc doc-headi
       </tbody>
     </table>
 
+
+
+
+
+
               <details class="quote">
                 <summary>Source code in <code>pytorch_widedeep/utils/fastai_transforms.py</code></summary>
                 <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">227</span>
@@ -1849,7 +1866,9 @@ <h2 id="pytorch_widedeep.utils.fastai_transforms.Tokenizer" class="doc doc-headi
 <span class="normal">336</span>
 <span class="normal">337</span>
 <span class="normal">338</span>
-<span class="normal">339</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">class</span> <span class="nc">Tokenizer</span><span class="p">:</span>
+<span class="normal">339</span>
+<span class="normal">340</span>
+<span class="normal">341</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">class</span> <span class="nc">Tokenizer</span><span class="p">:</span>
 <span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Class to combine a series of rules and a tokenizer function to tokenize</span>
 <span class="sd">    text with multiprocessing.</span>
 
@@ -1958,10 +1977,12 @@ <h2 id="pytorch_widedeep.utils.fastai_transforms.Tokenizer" class="doc doc-headi
 
         <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">n_cpus</span> <span class="o">&lt;=</span> <span class="mi">1</span><span class="p">:</span>
             <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_process_all_1</span><span class="p">(</span><span class="n">texts</span><span class="p">)</span>
-        <span class="k">with</span> <span class="n">ProcessPoolExecutor</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">n_cpus</span><span class="p">)</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
-            <span class="k">return</span> <span class="nb">sum</span><span class="p">(</span>
-                <span class="n">e</span><span class="o">.</span><span class="n">map</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_process_all_1</span><span class="p">,</span> <span class="n">partition_by_cores</span><span class="p">(</span><span class="n">texts</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">n_cpus</span><span class="p">)),</span> <span class="p">[]</span>
-            <span class="p">)</span>
+
+        <span class="k">with</span> <span class="n">Pool</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">n_cpus</span><span class="p">)</span> <span class="k">as</span> <span class="n">p</span><span class="p">:</span>
+            <span class="n">partitioned_texts</span> <span class="o">=</span> <span class="n">partition_by_cores</span><span class="p">(</span><span class="n">texts</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">n_cpus</span><span class="p">)</span>
+            <span class="n">results</span> <span class="o">=</span> <span class="n">p</span><span class="o">.</span><span class="n">map</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_process_all_1</span><span class="p">,</span> <span class="n">partitioned_texts</span><span class="p">)</span>
+            <span class="n">res</span> <span class="o">=</span> <span class="nb">sum</span><span class="p">(</span><span class="n">results</span><span class="p">,</span> <span class="p">[])</span>
+        <span class="k">return</span> <span class="n">res</span>
 </code></pre></div></td></tr></table></div>
               </details>
 
@@ -2005,7 +2026,9 @@ <h3 id="pytorch_widedeep.utils.fastai_transforms.Tokenizer.process_text" class="
       </thead>
       <tbody>
           <tr class="doc-section-item">
-            <td><code>t</code></td>
+            <td>
+                <code>t</code>
+            </td>
             <td>
                   <code>str</code>
             </td>
@@ -2019,7 +2042,9 @@ <h3 id="pytorch_widedeep.utils.fastai_transforms.Tokenizer.process_text" class="
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>tok</code></td>
+            <td>
+                <code>tok</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.utils.fastai_transforms.BaseTokenizer">BaseTokenizer</span></code>
             </td>
@@ -2191,7 +2216,9 @@ <h3 id="pytorch_widedeep.utils.fastai_transforms.Tokenizer.process_all" class="d
 <span class="normal">336</span>
 <span class="normal">337</span>
 <span class="normal">338</span>
-<span class="normal">339</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">process_all</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">texts</span><span class="p">:</span> <span class="n">Collection</span><span class="p">[</span><span class="nb">str</span><span class="p">])</span> <span class="o">-&gt;</span> <span class="n">List</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]]:</span>
+<span class="normal">339</span>
+<span class="normal">340</span>
+<span class="normal">341</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">process_all</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">texts</span><span class="p">:</span> <span class="n">Collection</span><span class="p">[</span><span class="nb">str</span><span class="p">])</span> <span class="o">-&gt;</span> <span class="n">List</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]]:</span>
 <span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Process a list of texts. Parallel execution of ``process_text``.</span>
 
 <span class="sd">    Examples</span>
@@ -2216,10 +2243,12 @@ <h3 id="pytorch_widedeep.utils.fastai_transforms.Tokenizer.process_all" class="d
 
     <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">n_cpus</span> <span class="o">&lt;=</span> <span class="mi">1</span><span class="p">:</span>
         <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_process_all_1</span><span class="p">(</span><span class="n">texts</span><span class="p">)</span>
-    <span class="k">with</span> <span class="n">ProcessPoolExecutor</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">n_cpus</span><span class="p">)</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
-        <span class="k">return</span> <span class="nb">sum</span><span class="p">(</span>
-            <span class="n">e</span><span class="o">.</span><span class="n">map</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_process_all_1</span><span class="p">,</span> <span class="n">partition_by_cores</span><span class="p">(</span><span class="n">texts</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">n_cpus</span><span class="p">)),</span> <span class="p">[]</span>
-        <span class="p">)</span>
+
+    <span class="k">with</span> <span class="n">Pool</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">n_cpus</span><span class="p">)</span> <span class="k">as</span> <span class="n">p</span><span class="p">:</span>
+        <span class="n">partitioned_texts</span> <span class="o">=</span> <span class="n">partition_by_cores</span><span class="p">(</span><span class="n">texts</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">n_cpus</span><span class="p">)</span>
+        <span class="n">results</span> <span class="o">=</span> <span class="n">p</span><span class="o">.</span><span class="n">map</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_process_all_1</span><span class="p">,</span> <span class="n">partitioned_texts</span><span class="p">)</span>
+        <span class="n">res</span> <span class="o">=</span> <span class="nb">sum</span><span class="p">(</span><span class="n">results</span><span class="p">,</span> <span class="p">[])</span>
+    <span class="k">return</span> <span class="n">res</span>
 </code></pre></div></td></tr></table></div>
             </details>
     </div>
@@ -2263,7 +2292,9 @@ <h2 id="pytorch_widedeep.utils.fastai_transforms.Vocab" class="doc doc-heading">
       </thead>
       <tbody>
           <tr class="doc-section-item">
-            <td><code>max_vocab</code></td>
+            <td>
+                <code>max_vocab</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -2277,7 +2308,9 @@ <h2 id="pytorch_widedeep.utils.fastai_transforms.Vocab" class="doc doc-heading">
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>min_freq</code></td>
+            <td>
+                <code>min_freq</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -2291,7 +2324,9 @@ <h2 id="pytorch_widedeep.utils.fastai_transforms.Vocab" class="doc doc-heading">
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>pad_idx</code></td>
+            <td>
+                <code>pad_idx</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[int]</code>
             </td>
@@ -2347,11 +2382,14 @@ <h2 id="pytorch_widedeep.utils.fastai_transforms.Vocab" class="doc doc-heading">
       </tbody>
     </table>
 
+
+
+
+
+
               <details class="quote">
                 <summary>Source code in <code>pytorch_widedeep/utils/fastai_transforms.py</code></summary>
-                <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">342</span>
-<span class="normal">343</span>
-<span class="normal">344</span>
+                <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">344</span>
 <span class="normal">345</span>
 <span class="normal">346</span>
 <span class="normal">347</span>
@@ -2510,7 +2548,9 @@ <h2 id="pytorch_widedeep.utils.fastai_transforms.Vocab" class="doc doc-heading">
 <span class="normal">500</span>
 <span class="normal">501</span>
 <span class="normal">502</span>
-<span class="normal">503</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">class</span> <span class="nc">Vocab</span><span class="p">:</span>
+<span class="normal">503</span>
+<span class="normal">504</span>
+<span class="normal">505</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">class</span> <span class="nc">Vocab</span><span class="p">:</span>
 <span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Contains the correspondence between numbers and tokens.</span>
 
 <span class="sd">    Parameters</span>
@@ -2585,7 +2625,7 @@ <h2 id="pytorch_widedeep.utils.fastai_transforms.Vocab" class="doc doc-heading">
 
         <span class="n">freq</span> <span class="o">=</span> <span class="n">Counter</span><span class="p">(</span><span class="n">p</span> <span class="k">for</span> <span class="n">o</span> <span class="ow">in</span> <span class="n">tokens</span> <span class="k">for</span> <span class="n">p</span> <span class="ow">in</span> <span class="n">o</span><span class="p">)</span>
         <span class="n">itos</span> <span class="o">=</span> <span class="p">[</span><span class="n">o</span> <span class="k">for</span> <span class="n">o</span><span class="p">,</span> <span class="n">c</span> <span class="ow">in</span> <span class="n">freq</span><span class="o">.</span><span class="n">most_common</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">max_vocab</span><span class="p">)</span> <span class="k">if</span> <span class="n">c</span> <span class="o">&gt;=</span> <span class="bp">self</span><span class="o">.</span><span class="n">min_freq</span><span class="p">]</span>
-        <span class="k">for</span> <span class="n">o</span> <span class="ow">in</span> <span class="nb">reversed</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">special_cases</span><span class="p">):</span>
+        <span class="k">for</span> <span class="n">o</span> <span class="ow">in</span> <span class="nb">reversed</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">special_cases</span><span class="p">):</span>  <span class="c1"># type: ignore[arg-type]</span>
             <span class="k">if</span> <span class="n">o</span> <span class="ow">in</span> <span class="n">itos</span><span class="p">:</span>
                 <span class="n">itos</span><span class="o">.</span><span class="n">remove</span><span class="p">(</span><span class="n">o</span><span class="p">)</span>
             <span class="n">itos</span><span class="o">.</span><span class="n">insert</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="n">o</span><span class="p">)</span>
@@ -2715,7 +2755,9 @@ <h3 id="pytorch_widedeep.utils.fastai_transforms.Vocab.create" class="doc doc-he
       </thead>
       <tbody>
           <tr class="doc-section-item">
-            <td><code>tokens</code></td>
+            <td>
+                <code>tokens</code>
+            </td>
             <td>
                   <code><span title="pytorch_widedeep.wdtypes.Tokens">Tokens</span></code>
             </td>
@@ -2774,9 +2816,7 @@ <h3 id="pytorch_widedeep.utils.fastai_transforms.Vocab.create" class="doc doc-he
 
             <details class="quote">
               <summary>Source code in <code>pytorch_widedeep/utils/fastai_transforms.py</code></summary>
-              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">380</span>
-<span class="normal">381</span>
-<span class="normal">382</span>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">382</span>
 <span class="normal">383</span>
 <span class="normal">384</span>
 <span class="normal">385</span>
@@ -2836,7 +2876,9 @@ <h3 id="pytorch_widedeep.utils.fastai_transforms.Vocab.create" class="doc doc-he
 <span class="normal">439</span>
 <span class="normal">440</span>
 <span class="normal">441</span>
-<span class="normal">442</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">create</span><span class="p">(</span>
+<span class="normal">442</span>
+<span class="normal">443</span>
+<span class="normal">444</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">create</span><span class="p">(</span>
     <span class="bp">self</span><span class="p">,</span>
     <span class="n">tokens</span><span class="p">:</span> <span class="n">Tokens</span><span class="p">,</span>
 <span class="p">)</span> <span class="o">-&gt;</span> <span class="s2">&quot;Vocab&quot;</span><span class="p">:</span>
@@ -2873,7 +2915,7 @@ <h3 id="pytorch_widedeep.utils.fastai_transforms.Vocab.create" class="doc doc-he
 
     <span class="n">freq</span> <span class="o">=</span> <span class="n">Counter</span><span class="p">(</span><span class="n">p</span> <span class="k">for</span> <span class="n">o</span> <span class="ow">in</span> <span class="n">tokens</span> <span class="k">for</span> <span class="n">p</span> <span class="ow">in</span> <span class="n">o</span><span class="p">)</span>
     <span class="n">itos</span> <span class="o">=</span> <span class="p">[</span><span class="n">o</span> <span class="k">for</span> <span class="n">o</span><span class="p">,</span> <span class="n">c</span> <span class="ow">in</span> <span class="n">freq</span><span class="o">.</span><span class="n">most_common</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">max_vocab</span><span class="p">)</span> <span class="k">if</span> <span class="n">c</span> <span class="o">&gt;=</span> <span class="bp">self</span><span class="o">.</span><span class="n">min_freq</span><span class="p">]</span>
-    <span class="k">for</span> <span class="n">o</span> <span class="ow">in</span> <span class="nb">reversed</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">special_cases</span><span class="p">):</span>
+    <span class="k">for</span> <span class="n">o</span> <span class="ow">in</span> <span class="nb">reversed</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">special_cases</span><span class="p">):</span>  <span class="c1"># type: ignore[arg-type]</span>
         <span class="k">if</span> <span class="n">o</span> <span class="ow">in</span> <span class="n">itos</span><span class="p">:</span>
             <span class="n">itos</span><span class="o">.</span><span class="n">remove</span><span class="p">(</span><span class="n">o</span><span class="p">)</span>
         <span class="n">itos</span><span class="o">.</span><span class="n">insert</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="n">o</span><span class="p">)</span>
@@ -2923,15 +2965,15 @@ <h3 id="pytorch_widedeep.utils.fastai_transforms.Vocab.fit" class="doc doc-headi
 
             <details class="quote">
               <summary>Source code in <code>pytorch_widedeep/utils/fastai_transforms.py</code></summary>
-              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">444</span>
-<span class="normal">445</span>
-<span class="normal">446</span>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">446</span>
 <span class="normal">447</span>
 <span class="normal">448</span>
 <span class="normal">449</span>
 <span class="normal">450</span>
 <span class="normal">451</span>
-<span class="normal">452</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">fit</span><span class="p">(</span>
+<span class="normal">452</span>
+<span class="normal">453</span>
+<span class="normal">454</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">fit</span><span class="p">(</span>
     <span class="bp">self</span><span class="p">,</span>
     <span class="n">tokens</span><span class="p">:</span> <span class="n">Tokens</span><span class="p">,</span>
 <span class="p">)</span> <span class="o">-&gt;</span> <span class="s2">&quot;Vocab&quot;</span><span class="p">:</span>
@@ -2986,15 +3028,15 @@ <h3 id="pytorch_widedeep.utils.fastai_transforms.Vocab.numericalize" class="doc
 
             <details class="quote">
               <summary>Source code in <code>pytorch_widedeep/utils/fastai_transforms.py</code></summary>
-              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">454</span>
-<span class="normal">455</span>
-<span class="normal">456</span>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">456</span>
 <span class="normal">457</span>
 <span class="normal">458</span>
 <span class="normal">459</span>
 <span class="normal">460</span>
 <span class="normal">461</span>
-<span class="normal">462</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">numericalize</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">t</span><span class="p">:</span> <span class="n">Collection</span><span class="p">[</span><span class="nb">str</span><span class="p">])</span> <span class="o">-&gt;</span> <span class="n">List</span><span class="p">[</span><span class="nb">int</span><span class="p">]:</span>
+<span class="normal">462</span>
+<span class="normal">463</span>
+<span class="normal">464</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">numericalize</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">t</span><span class="p">:</span> <span class="n">Collection</span><span class="p">[</span><span class="nb">str</span><span class="p">])</span> <span class="o">-&gt;</span> <span class="n">List</span><span class="p">[</span><span class="nb">int</span><span class="p">]:</span>
 <span class="w">    </span><span class="sd">&quot;&quot;&quot;Convert a list of tokens ``t`` to their ids.</span>
 
 <span class="sd">    Returns</span>
@@ -3028,13 +3070,13 @@ <h3 id="pytorch_widedeep.utils.fastai_transforms.Vocab.transform" class="doc doc
 
             <details class="quote">
               <summary>Source code in <code>pytorch_widedeep/utils/fastai_transforms.py</code></summary>
-              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">464</span>
-<span class="normal">465</span>
-<span class="normal">466</span>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">466</span>
 <span class="normal">467</span>
 <span class="normal">468</span>
 <span class="normal">469</span>
-<span class="normal">470</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">transform</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">t</span><span class="p">:</span> <span class="n">Collection</span><span class="p">[</span><span class="nb">str</span><span class="p">])</span> <span class="o">-&gt;</span> <span class="n">List</span><span class="p">[</span><span class="nb">int</span><span class="p">]:</span>
+<span class="normal">470</span>
+<span class="normal">471</span>
+<span class="normal">472</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">transform</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">t</span><span class="p">:</span> <span class="n">Collection</span><span class="p">[</span><span class="nb">str</span><span class="p">])</span> <span class="o">-&gt;</span> <span class="n">List</span><span class="p">[</span><span class="nb">int</span><span class="p">]:</span>
 <span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
 <span class="sd">    Calls the `numericalize` method. I simply want to honor fast ai naming,</span>
 <span class="sd">    but for consistency with the rest of the library I am including a</span>
@@ -3087,9 +3129,7 @@ <h3 id="pytorch_widedeep.utils.fastai_transforms.Vocab.textify" class="doc doc-h
 
             <details class="quote">
               <summary>Source code in <code>pytorch_widedeep/utils/fastai_transforms.py</code></summary>
-              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">472</span>
-<span class="normal">473</span>
-<span class="normal">474</span>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">474</span>
 <span class="normal">475</span>
 <span class="normal">476</span>
 <span class="normal">477</span>
@@ -3099,7 +3139,9 @@ <h3 id="pytorch_widedeep.utils.fastai_transforms.Vocab.textify" class="doc doc-h
 <span class="normal">481</span>
 <span class="normal">482</span>
 <span class="normal">483</span>
-<span class="normal">484</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">textify</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">nums</span><span class="p">:</span> <span class="n">Collection</span><span class="p">[</span><span class="nb">int</span><span class="p">],</span> <span class="n">sep</span><span class="o">=</span><span class="s2">&quot; &quot;</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Union</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]]:</span>
+<span class="normal">484</span>
+<span class="normal">485</span>
+<span class="normal">486</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">textify</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">nums</span><span class="p">:</span> <span class="n">Collection</span><span class="p">[</span><span class="nb">int</span><span class="p">],</span> <span class="n">sep</span><span class="o">=</span><span class="s2">&quot; &quot;</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Union</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]]:</span>
 <span class="w">    </span><span class="sd">&quot;&quot;&quot;Convert a list of ``nums`` (or indexes) to their tokens.</span>
 
 <span class="sd">    Returns</span>
@@ -3137,9 +3179,7 @@ <h3 id="pytorch_widedeep.utils.fastai_transforms.Vocab.inverse_transform" class=
 
             <details class="quote">
               <summary>Source code in <code>pytorch_widedeep/utils/fastai_transforms.py</code></summary>
-              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">486</span>
-<span class="normal">487</span>
-<span class="normal">488</span>
+              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">488</span>
 <span class="normal">489</span>
 <span class="normal">490</span>
 <span class="normal">491</span>
@@ -3147,7 +3187,9 @@ <h3 id="pytorch_widedeep.utils.fastai_transforms.Vocab.inverse_transform" class=
 <span class="normal">493</span>
 <span class="normal">494</span>
 <span class="normal">495</span>
-<span class="normal">496</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">inverse_transform</span><span class="p">(</span>
+<span class="normal">496</span>
+<span class="normal">497</span>
+<span class="normal">498</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">inverse_transform</span><span class="p">(</span>
     <span class="bp">self</span><span class="p">,</span> <span class="n">nums</span><span class="p">:</span> <span class="n">Collection</span><span class="p">[</span><span class="nb">int</span><span class="p">],</span> <span class="n">sep</span><span class="o">=</span><span class="s2">&quot; &quot;</span>
 <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Union</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]]:</span>
 <span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
@@ -3194,8 +3236,7 @@ <h3 id="pytorch_widedeep.utils.fastai_transforms.Vocab.inverse_transform" class=
     <nav>
       
         <a href="mailto:jrzaurin@gmail.com">Javier</a>, 
-        <a href="mailto:mulinka.pavol@gmail.com">Pavol Mulinka</a>, 
-        <a href="mailto:not.committed.yet">Not Committed Yet</a>
+        <a href="mailto:mulinka.pavol@gmail.com">Pavol Mulinka</a>
     </nav>
   </span>
 
@@ -3263,7 +3304,7 @@ <h3 id="pytorch_widedeep.utils.fastai_transforms.Vocab.inverse_transform" class=
     <script id="__config" type="application/json">{"base": "../..", "features": ["navigation.tabs", "navigation.tabs.sticky", "navigation.indexes", "navigation.expand", "toc.integrate"], "search": "../../assets/javascripts/workers/search.6ce7567c.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version": "Select version"}}</script>
     
     
-      <script src="../../assets/javascripts/bundle.56dfad97.min.js"></script>
+      <script src="../../assets/javascripts/bundle.83f73b43.min.js"></script>
       
         <script src="../../stylesheets/extra.js"></script>
       
diff --git a/mkdocs/site/pytorch-widedeep/utils/fastai_transforms.md b/mkdocs/site/pytorch-widedeep/utils/fastai_transforms.md
deleted file mode 100644
index 16a77819..00000000
--- a/mkdocs/site/pytorch-widedeep/utils/fastai_transforms.md
+++ /dev/null
@@ -1,13 +0,0 @@
-# Fastai transforms
-
-We directly copied and pasted part of the ``transforms.py`` module from
-the ``fastai`` library (from an old version). The reason to do such a thing is because
-``pytorch_widedeep`` only needs the ``Tokenizer`` and the ``Vocab`` classes
-there. This way we avoid extra dependencies. Credit for all the code in the
-``fastai_transforms`` module in this ``pytorch-widedeep`` package goes to
-Jeremy Howard and the `fastai` team. I only include the documentation here for
-completion, but I strongly advise the user to read the ``fastai`` [documentation](https://docs.fast.ai/).
-
-::: pytorch_widedeep.utils.fastai_transforms.Tokenizer
-
-::: pytorch_widedeep.utils.fastai_transforms.Vocab
diff --git a/mkdocs/site/pytorch-widedeep/utils/image_utils.html b/mkdocs/site/pytorch-widedeep/utils/image_utils.html
index d4f337c5..ad290e00 100644
--- a/mkdocs/site/pytorch-widedeep/utils/image_utils.html
+++ b/mkdocs/site/pytorch-widedeep/utils/image_utils.html
@@ -18,7 +18,7 @@
       
       
       <link rel="icon" href="../../assets/images/favicon.ico">
-      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.35">
+      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.43">
     
     
       
@@ -26,7 +26,7 @@
       
     
     
-      <link rel="stylesheet" href="../../assets/stylesheets/main.35f28582.min.css">
+      <link rel="stylesheet" href="../../assets/stylesheets/main.0253249f.min.css">
       
         
         <link rel="stylesheet" href="../../assets/stylesheets/palette.06af60db.min.css">
@@ -1640,7 +1640,9 @@ <h2 id="pytorch_widedeep.utils.image_utils.AspectAwarePreprocessor" class="doc d
       </thead>
       <tbody>
           <tr class="doc-section-item">
-            <td><code>width</code></td>
+            <td>
+                <code>width</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -1654,7 +1656,9 @@ <h2 id="pytorch_widedeep.utils.image_utils.AspectAwarePreprocessor" class="doc d
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>height</code></td>
+            <td>
+                <code>height</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -1668,7 +1672,9 @@ <h2 id="pytorch_widedeep.utils.image_utils.AspectAwarePreprocessor" class="doc d
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>inter</code></td>
+            <td>
+                <code>inter</code>
+            </td>
             <td>
             </td>
             <td>
@@ -1684,6 +1690,11 @@ <h2 id="pytorch_widedeep.utils.image_utils.AspectAwarePreprocessor" class="doc d
       </tbody>
     </table>
 
+
+
+
+
+
               <details class="quote">
                 <summary>Source code in <code>pytorch_widedeep/utils/image_utils.py</code></summary>
                 <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 49</span>
@@ -1856,7 +1867,9 @@ <h3 id="pytorch_widedeep.utils.image_utils.AspectAwarePreprocessor.preprocess" c
       </thead>
       <tbody>
           <tr class="doc-section-item">
-            <td><code>image</code></td>
+            <td>
+                <code>image</code>
+            </td>
             <td>
                   <code><span title="numpy.ndarray">ndarray</span></code>
             </td>
@@ -2040,7 +2053,9 @@ <h2 id="pytorch_widedeep.utils.image_utils.SimplePreprocessor" class="doc doc-he
       </thead>
       <tbody>
           <tr class="doc-section-item">
-            <td><code>width</code></td>
+            <td>
+                <code>width</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -2054,7 +2069,9 @@ <h2 id="pytorch_widedeep.utils.image_utils.SimplePreprocessor" class="doc doc-he
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>height</code></td>
+            <td>
+                <code>height</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -2068,7 +2085,9 @@ <h2 id="pytorch_widedeep.utils.image_utils.SimplePreprocessor" class="doc doc-he
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>inter</code></td>
+            <td>
+                <code>inter</code>
+            </td>
             <td>
             </td>
             <td>
@@ -2084,6 +2103,11 @@ <h2 id="pytorch_widedeep.utils.image_utils.SimplePreprocessor" class="doc doc-he
       </tbody>
     </table>
 
+
+
+
+
+
               <details class="quote">
                 <summary>Source code in <code>pytorch_widedeep/utils/image_utils.py</code></summary>
                 <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">115</span>
@@ -2204,7 +2228,9 @@ <h3 id="pytorch_widedeep.utils.image_utils.SimplePreprocessor.preprocess" class=
       </thead>
       <tbody>
           <tr class="doc-section-item">
-            <td><code>image</code></td>
+            <td>
+                <code>image</code>
+            </td>
             <td>
                   <code><span title="numpy.ndarray">ndarray</span></code>
             </td>
@@ -2385,7 +2411,7 @@ <h3 id="pytorch_widedeep.utils.image_utils.SimplePreprocessor.preprocess" class=
     <script id="__config" type="application/json">{"base": "../..", "features": ["navigation.tabs", "navigation.tabs.sticky", "navigation.indexes", "navigation.expand", "toc.integrate"], "search": "../../assets/javascripts/workers/search.6ce7567c.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version": "Select version"}}</script>
     
     
-      <script src="../../assets/javascripts/bundle.56dfad97.min.js"></script>
+      <script src="../../assets/javascripts/bundle.83f73b43.min.js"></script>
       
         <script src="../../stylesheets/extra.js"></script>
       
diff --git a/mkdocs/site/pytorch-widedeep/utils/image_utils.md b/mkdocs/site/pytorch-widedeep/utils/image_utils.md
deleted file mode 100644
index 206eb4b3..00000000
--- a/mkdocs/site/pytorch-widedeep/utils/image_utils.md
+++ /dev/null
@@ -1,11 +0,0 @@
-# Image utils
-
-[SimplePreprocessor][pytorch_widedeep.utils.image_utils.SimplePreprocessor] and
-[AspectAwarePreprocessor][pytorch_widedeep.utils.image_utils.AspectAwarePreprocessor] are directly
-taked from the great series of Books `Deep Learning for [Computer Vision](https://www.pyimagesearch.com/deep-learning-computer-vision-python-book/)
-by [Adrian](https://www.pyimagesearch.com/). Therefore, all credit for the
-code in the ``image_utils`` module goes to [Adrian Rosebrock](https://www.pyimagesearch.com/).
-
-::: pytorch_widedeep.utils.image_utils.AspectAwarePreprocessor
-
-::: pytorch_widedeep.utils.image_utils.SimplePreprocessor
diff --git a/mkdocs/site/pytorch-widedeep/utils/index.html b/mkdocs/site/pytorch-widedeep/utils/index.html
index 72689bfa..c116b4da 100644
--- a/mkdocs/site/pytorch-widedeep/utils/index.html
+++ b/mkdocs/site/pytorch-widedeep/utils/index.html
@@ -18,7 +18,7 @@
       
       
       <link rel="icon" href="../../assets/images/favicon.ico">
-      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.35">
+      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.43">
     
     
       
@@ -26,7 +26,7 @@
       
     
     
-      <link rel="stylesheet" href="../../assets/stylesheets/main.35f28582.min.css">
+      <link rel="stylesheet" href="../../assets/stylesheets/main.0253249f.min.css">
       
         
         <link rel="stylesheet" href="../../assets/stylesheets/palette.06af60db.min.css">
@@ -1644,7 +1644,7 @@ <h1 id="the-utils-module">The <code>utils</code> module<a class="headerlink" hre
     <script id="__config" type="application/json">{"base": "../..", "features": ["navigation.tabs", "navigation.tabs.sticky", "navigation.indexes", "navigation.expand", "toc.integrate"], "search": "../../assets/javascripts/workers/search.6ce7567c.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version": "Select version"}}</script>
     
     
-      <script src="../../assets/javascripts/bundle.56dfad97.min.js"></script>
+      <script src="../../assets/javascripts/bundle.83f73b43.min.js"></script>
       
         <script src="../../stylesheets/extra.js"></script>
       
diff --git a/mkdocs/site/pytorch-widedeep/utils/index.md b/mkdocs/site/pytorch-widedeep/utils/index.md
deleted file mode 100644
index c4b7aa55..00000000
--- a/mkdocs/site/pytorch-widedeep/utils/index.md
+++ /dev/null
@@ -1,15 +0,0 @@
-# The ``utils`` module
-
-
-These are a series of utilities that might be useful for a number of
-preprocessing tasks, even not directly related to ``pytorch-widedeep``. All
-the classes and functions discussed here are available directly from the
-``utils`` module. For example, the ``LabelEncoder`` within the
-``deeptabular_utils`` submodule can be imported as:
-
-```
-from pytorch_widedeep.utils import LabelEncoder
-```
-
-These are classes and functions that are internally used in the library. We
-include them here in case the user finds them useful for other purposes.
diff --git a/mkdocs/site/pytorch-widedeep/utils/text_utils.html b/mkdocs/site/pytorch-widedeep/utils/text_utils.html
index e54ba5a9..b0279a2b 100644
--- a/mkdocs/site/pytorch-widedeep/utils/text_utils.html
+++ b/mkdocs/site/pytorch-widedeep/utils/text_utils.html
@@ -18,7 +18,7 @@
       
       
       <link rel="icon" href="../../assets/images/favicon.ico">
-      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.35">
+      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.43">
     
     
       
@@ -26,7 +26,7 @@
       
     
     
-      <link rel="stylesheet" href="../../assets/stylesheets/main.35f28582.min.css">
+      <link rel="stylesheet" href="../../assets/stylesheets/main.0253249f.min.css">
       
         
         <link rel="stylesheet" href="../../assets/stylesheets/palette.06af60db.min.css">
@@ -1656,7 +1656,9 @@ <h2 id="pytorch_widedeep.utils.text_utils.simple_preprocess" class="doc doc-head
       </thead>
       <tbody>
           <tr class="doc-section-item">
-            <td><code>doc</code></td>
+            <td>
+                <code>doc</code>
+            </td>
             <td>
                   <code>str</code>
             </td>
@@ -1670,7 +1672,9 @@ <h2 id="pytorch_widedeep.utils.text_utils.simple_preprocess" class="doc doc-head
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>lower</code></td>
+            <td>
+                <code>lower</code>
+            </td>
             <td>
                   <code>bool</code>
             </td>
@@ -1684,7 +1688,9 @@ <h2 id="pytorch_widedeep.utils.text_utils.simple_preprocess" class="doc doc-head
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>deacc</code></td>
+            <td>
+                <code>deacc</code>
+            </td>
             <td>
                   <code>bool</code>
             </td>
@@ -1698,7 +1704,9 @@ <h2 id="pytorch_widedeep.utils.text_utils.simple_preprocess" class="doc doc-head
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>min_len</code></td>
+            <td>
+                <code>min_len</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -1712,7 +1720,9 @@ <h2 id="pytorch_widedeep.utils.text_utils.simple_preprocess" class="doc doc-head
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>max_len</code></td>
+            <td>
+                <code>max_len</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -1887,7 +1897,9 @@ <h2 id="pytorch_widedeep.utils.text_utils.get_texts" class="doc doc-heading">
       </thead>
       <tbody>
           <tr class="doc-section-item">
-            <td><code>texts</code></td>
+            <td>
+                <code>texts</code>
+            </td>
             <td>
                   <code><span title="typing.List">List</span>[str]</code>
             </td>
@@ -1901,7 +1913,9 @@ <h2 id="pytorch_widedeep.utils.text_utils.get_texts" class="doc doc-heading">
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>already_processed</code></td>
+            <td>
+                <code>already_processed</code>
+            </td>
             <td>
                   <code><span title="typing.Optional">Optional</span>[bool]</code>
             </td>
@@ -1918,7 +1932,9 @@ <h2 id="pytorch_widedeep.utils.text_utils.get_texts" class="doc doc-heading">
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>n_cpus</code></td>
+            <td>
+                <code>n_cpus</code>
+            </td>
             <td>
                   <code><span title="typing.Optional">Optional</span>[int]</code>
             </td>
@@ -2101,7 +2117,9 @@ <h2 id="pytorch_widedeep.utils.text_utils.pad_sequences" class="doc doc-heading"
       </thead>
       <tbody>
           <tr class="doc-section-item">
-            <td><code>seq</code></td>
+            <td>
+                <code>seq</code>
+            </td>
             <td>
                   <code><span title="typing.List">List</span>[int]</code>
             </td>
@@ -2115,7 +2133,9 @@ <h2 id="pytorch_widedeep.utils.text_utils.pad_sequences" class="doc doc-heading"
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>maxlen</code></td>
+            <td>
+                <code>maxlen</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -2129,7 +2149,9 @@ <h2 id="pytorch_widedeep.utils.text_utils.pad_sequences" class="doc doc-heading"
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>pad_first</code></td>
+            <td>
+                <code>pad_first</code>
+            </td>
             <td>
                   <code>bool</code>
             </td>
@@ -2144,7 +2166,9 @@ <h2 id="pytorch_widedeep.utils.text_utils.pad_sequences" class="doc doc-heading"
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>pad_idx</code></td>
+            <td>
+                <code>pad_idx</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -2317,7 +2341,9 @@ <h2 id="pytorch_widedeep.utils.text_utils.build_embeddings_matrix" class="doc do
       </thead>
       <tbody>
           <tr class="doc-section-item">
-            <td><code>vocab</code></td>
+            <td>
+                <code>vocab</code>
+            </td>
             <td>
                   <code><span title="typing.Union">Union</span>[<a class="autorefs autorefs-internal" title="pytorch_widedeep.utils.fastai_transforms.Vocab" href="fastai_transforms.html#pytorch_widedeep.utils.fastai_transforms.Vocab">Vocab</a>, <span title="pytorch_widedeep.utils.fastai_transforms.ChunkVocab">ChunkVocab</span>]</code>
             </td>
@@ -2331,7 +2357,9 @@ <h2 id="pytorch_widedeep.utils.text_utils.build_embeddings_matrix" class="doc do
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>word_vectors_path</code></td>
+            <td>
+                <code>word_vectors_path</code>
+            </td>
             <td>
                   <code>str</code>
             </td>
@@ -2345,7 +2373,9 @@ <h2 id="pytorch_widedeep.utils.text_utils.build_embeddings_matrix" class="doc do
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>min_freq</code></td>
+            <td>
+                <code>min_freq</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -2359,7 +2389,9 @@ <h2 id="pytorch_widedeep.utils.text_utils.build_embeddings_matrix" class="doc do
             </td>
           </tr>
           <tr class="doc-section-item">
-            <td><code>verbose</code></td>
+            <td>
+                <code>verbose</code>
+            </td>
             <td>
                   <code>int</code>
             </td>
@@ -2628,7 +2660,7 @@ <h2 id="pytorch_widedeep.utils.text_utils.build_embeddings_matrix" class="doc do
     <script id="__config" type="application/json">{"base": "../..", "features": ["navigation.tabs", "navigation.tabs.sticky", "navigation.indexes", "navigation.expand", "toc.integrate"], "search": "../../assets/javascripts/workers/search.6ce7567c.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version": "Select version"}}</script>
     
     
-      <script src="../../assets/javascripts/bundle.56dfad97.min.js"></script>
+      <script src="../../assets/javascripts/bundle.83f73b43.min.js"></script>
       
         <script src="../../stylesheets/extra.js"></script>
       
diff --git a/mkdocs/site/pytorch-widedeep/utils/text_utils.md b/mkdocs/site/pytorch-widedeep/utils/text_utils.md
deleted file mode 100644
index 44897720..00000000
--- a/mkdocs/site/pytorch-widedeep/utils/text_utils.md
+++ /dev/null
@@ -1,11 +0,0 @@
-# Text utils
-
-Collection of helper function that facilitate processing text.
-
-::: pytorch_widedeep.utils.text_utils.simple_preprocess
-
-::: pytorch_widedeep.utils.text_utils.get_texts
-
-::: pytorch_widedeep.utils.text_utils.pad_sequences
-
-::: pytorch_widedeep.utils.text_utils.build_embeddings_matrix
diff --git a/mkdocs/site/quick_start.html b/mkdocs/site/quick_start.html
index 385f038d..63a5b58f 100644
--- a/mkdocs/site/quick_start.html
+++ b/mkdocs/site/quick_start.html
@@ -18,7 +18,7 @@
       
       
       <link rel="icon" href="assets/images/favicon.ico">
-      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.35">
+      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.43">
     
     
       
@@ -26,7 +26,7 @@
       
     
     
-      <link rel="stylesheet" href="assets/stylesheets/main.35f28582.min.css">
+      <link rel="stylesheet" href="assets/stylesheets/main.0253249f.min.css">
       
         
         <link rel="stylesheet" href="assets/stylesheets/palette.06af60db.min.css">
@@ -1742,7 +1742,7 @@ <h1 id="quick-start">Quick Start<a class="headerlink" href="#quick-start" title=
     <script id="__config" type="application/json">{"base": ".", "features": ["navigation.tabs", "navigation.tabs.sticky", "navigation.indexes", "navigation.expand", "toc.integrate"], "search": "assets/javascripts/workers/search.6ce7567c.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version": "Select version"}}</script>
     
     
-      <script src="assets/javascripts/bundle.56dfad97.min.js"></script>
+      <script src="assets/javascripts/bundle.83f73b43.min.js"></script>
       
         <script src="stylesheets/extra.js"></script>
       
diff --git a/mkdocs/site/quick_start.md b/mkdocs/site/quick_start.md
deleted file mode 100644
index dec31788..00000000
--- a/mkdocs/site/quick_start.md
+++ /dev/null
@@ -1,105 +0,0 @@
-# Quick Start
-
-This is an example of a binary classification with the
-[adult census](https://www.kaggle.com/wenruliu/adult-income-dataset?select=adult.csv)
-dataset using a combination of a wide and deep model (in this case a so called
-``deeptabular`` model) with defaults settings.
-
-```python
-import numpy as np
-import torch
-from sklearn.model_selection import train_test_split
-
-from pytorch_widedeep import Trainer
-from pytorch_widedeep.preprocessing import WidePreprocessor, TabPreprocessor
-from pytorch_widedeep.models import Wide, TabMlp, WideDeep
-from pytorch_widedeep.metrics import Accuracy
-from pytorch_widedeep.datasets import load_adult
-
-
-df = load_adult(as_frame=True)
-df["income_label"] = (df["income"].apply(lambda x: ">50K" in x)).astype(int)
-df.drop("income", axis=1, inplace=True)
-df_train, df_test = train_test_split(df, test_size=0.2, stratify=df.income_label)
-
-# Define the 'column set up'
-wide_cols = [
-    "education",
-    "relationship",
-    "workclass",
-    "occupation",
-    "native-country",
-    "gender",
-]
-crossed_cols = [("education", "occupation"), ("native-country", "occupation")]
-
-cat_embed_cols = [
-    "workclass",
-    "education",
-    "marital-status",
-    "occupation",
-    "relationship",
-    "race",
-    "gender",
-    "capital-gain",
-    "capital-loss",
-    "native-country",
-]
-continuous_cols = ["age", "hours-per-week"]
-target = "income_label"
-target = df_train[target].values
-
-# prepare the data
-wide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols)
-X_wide = wide_preprocessor.fit_transform(df_train)
-
-tab_preprocessor = TabPreprocessor(
-    cat_embed_cols=cat_embed_cols, continuous_cols=continuous_cols  # type: ignore[arg-type]
-)
-X_tab = tab_preprocessor.fit_transform(df_train)
-
-# build the model
-wide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1)
-tab_mlp = TabMlp(
-    column_idx=tab_preprocessor.column_idx,
-    cat_embed_input=tab_preprocessor.cat_embed_input,
-    continuous_cols=continuous_cols,
-)
-model = WideDeep(wide=wide, deeptabular=tab_mlp)
-
-# train and validate
-trainer = Trainer(model, objective="binary", metrics=[Accuracy])
-trainer.fit(
-    X_wide=X_wide,
-    X_tab=X_tab,
-    target=target,
-    n_epochs=5,
-    batch_size=256,
-)
-
-# predict on test
-X_wide_te = wide_preprocessor.transform(df_test)
-X_tab_te = tab_preprocessor.transform(df_test)
-preds = trainer.predict(X_wide=X_wide_te, X_tab=X_tab_te)
-
-# Save and load
-
-# Option 1: this will also save training history and lr history if the
-# LRHistory callback is used
-trainer.save(path="model_weights", save_state_dict=True)
-
-# Option 2: save as any other torch model
-torch.save(model.state_dict(), "model_weights/wd_model.pt")
-
-# From here in advance, Option 1 or 2 are the same. I assume the user has
-# prepared the data and defined the new model components:
-# 1. Build the model
-model_new = WideDeep(wide=wide, deeptabular=tab_mlp)
-model_new.load_state_dict(torch.load("model_weights/wd_model.pt"))
-
-# 2. Instantiate the trainer
-trainer_new = Trainer(model_new, objective="binary")
-
-# 3. Either start the fit or directly predict
-preds = trainer_new.predict(X_wide=X_wide, X_tab=X_tab, batch_size=32)
-```
\ No newline at end of file
diff --git a/mkdocs/site/search/search_index.json b/mkdocs/site/search/search_index.json
index 73418793..510beedf 100644
--- a/mkdocs/site/search/search_index.json
+++ b/mkdocs/site/search/search_index.json
@@ -1 +1 @@
-{"config":{"lang":["en"],"separator":"[\\s\\-]+","pipeline":["stopWordFilter"]},"docs":[{"location":"index.html","title":"Home","text":""},{"location":"index.html#pytorch-widedeep","title":"pytorch-widedeep","text":"<p>A flexible package for multimodal-deep-learning to combine tabular data with text and images using Wide and Deep models in Pytorch</p> <p>Documentation: https://pytorch-widedeep.readthedocs.io</p> <p>Companion posts and tutorials: infinitoml</p> <p>Experiments and comparison with <code>LightGBM</code>: TabularDL vs LightGBM</p> <p>Slack: if you want to contribute or just want to chat with us, join slack</p> <p>The content of this document is organized as follows:</p> <ul> <li>pytorch-widedeep<ul> <li>Introduction</li> <li>Architectures</li> <li>The <code>deeptabular</code> component</li> <li>The <code>rec</code> module</li> <li>Text and Images</li> <li>Acknowledgments</li> <li>License</li> <li>Cite<ul> <li>BibTex</li> <li>APA</li> </ul> </li> </ul> </li> </ul>"},{"location":"index.html#introduction","title":"Introduction","text":"<p><code>pytorch-widedeep</code> is based on Google's Wide and Deep Algorithm, adjusted for multi-modal datasets.</p> <p>In general terms, <code>pytorch-widedeep</code> is a package to use deep learning with tabular data. In particular, is intended to facilitate the combination of text and images with corresponding tabular data using wide and deep models. With that in mind there are a number of architectures that can be implemented with the library. The main components of those architectures are shown in the Figure below:</p> <p> </p> <p>In math terms, and following the notation in the paper, the expression for the architecture without a <code>deephead</code> component can be formulated as:</p> <p> </p> <p>Where \u03c3 is the sigmoid function, 'W' are the weight matrices applied to the wide model and to the final activations of the deep models, 'a' are these final activations, \u03c6(x) are the cross product transformations of the original features 'x', and , and 'b' is the bias term. In case you are wondering what are \"cross product transformations\", here is a quote taken directly from the paper: \"For binary features, a cross-product transformation (e.g., \u201cAND(gender=female, language=en)\u201d) is 1 if and only if the constituent features (\u201cgender=female\u201d and \u201clanguage=en\u201d) are all 1, and 0 otherwise\".</p> <p>It is perfectly possible to use custom models (and not necessarily those in the library) as long as the the custom models have a property called <code>output_dim</code> with the size of the last layer of activations, so that <code>WideDeep</code> can be constructed. Examples on how to use custom components can be found in the Examples folder and the section below.</p>"},{"location":"index.html#architectures","title":"Architectures","text":"<p>The <code>pytorch-widedeep</code> library offers a number of different architectures. In this section we will show some of them in their simplest form (i.e. with default param values in most cases) with their corresponding code snippets. Note that all the snippets below shoud run locally. For a more detailed explanation of the different components and their parameters, please refer to the documentation.</p> <p>For the examples below we will be using a toy dataset generated as follows:</p> <pre><code>import os\nimport random\n\nimport numpy as np\nimport pandas as pd\nfrom PIL import Image\nfrom faker import Faker\n\n\ndef create_and_save_random_image(image_number, size=(32, 32)):\n\n    if not os.path.exists(\"images\"):\n        os.makedirs(\"images\")\n\n    array = np.random.randint(0, 256, (size[0], size[1], 3), dtype=np.uint8)\n\n    image = Image.fromarray(array)\n\n    image_name = f\"image_{image_number}.png\"\n    image.save(os.path.join(\"images\", image_name))\n\n    return image_name\n\n\nfake = Faker()\n\ncities = [\"New York\", \"Los Angeles\", \"Chicago\", \"Houston\"]\nnames = [\"Alice\", \"Bob\", \"Charlie\", \"David\", \"Eva\"]\n\ndata = {\n    \"city\": [random.choice(cities) for _ in range(100)],\n    \"name\": [random.choice(names) for _ in range(100)],\n    \"age\": [random.uniform(18, 70) for _ in range(100)],\n    \"height\": [random.uniform(150, 200) for _ in range(100)],\n    \"sentence\": [fake.sentence() for _ in range(100)],\n    \"other_sentence\": [fake.sentence() for _ in range(100)],\n    \"image_name\": [create_and_save_random_image(i) for i in range(100)],\n    \"target\": [random.choice([0, 1]) for _ in range(100)],\n}\n\ndf = pd.DataFrame(data)\n</code></pre> <p>This will create a 100 rows dataframe and a dir in your local folder, called <code>images</code> with 100 random images (or images with just noise).</p> <p>Perhaps the simplest architecture would be just one component, <code>wide</code>, <code>deeptabular</code>, <code>deeptext</code> or <code>deepimage</code> on their own, which is also possible, but let's start the examples with a standard Wide and Deep architecture. From there, how to build a model comprised only of one component will be straightforward.</p> <p>Note that the examples shown below would be almost identical using any of the models available in the library. For example, <code>TabMlp</code> can be replaced by <code>TabResnet</code>, <code>TabNet</code>, <code>TabTransformer</code>, etc. Similarly, <code>BasicRNN</code> can be replaced by <code>AttentiveRNN</code>, <code>StackedAttentiveRNN</code>, or <code>HFModel</code> with their corresponding parameters and preprocessor in the case of the Hugging Face models.</p> <p>1. Wide and Tabular component (aka deeptabular)</p> <p> </p> <pre><code>from pytorch_widedeep.preprocessing import TabPreprocessor, WidePreprocessor\nfrom pytorch_widedeep.models import Wide, TabMlp, WideDeep\nfrom pytorch_widedeep.training import Trainer\n\n# Wide\nwide_cols = [\"city\"]\ncrossed_cols = [(\"city\", \"name\")]\nwide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols)\nX_wide = wide_preprocessor.fit_transform(df)\nwide = Wide(input_dim=np.unique(X_wide).shape[0])\n\n# Tabular\ntab_preprocessor = TabPreprocessor(\n    embed_cols=[\"city\", \"name\"], continuous_cols=[\"age\", \"height\"]\n)\nX_tab = tab_preprocessor.fit_transform(df)\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=tab_preprocessor.continuous_cols,\n    mlp_hidden_dims=[64, 32],\n)\n\n# WideDeep\nmodel = WideDeep(wide=wide, deeptabular=tab_mlp)\n\n# Train\ntrainer = Trainer(model, objective=\"binary\")\n\ntrainer.fit(\n    X_wide=X_wide,\n    X_tab=X_tab,\n    target=df[\"target\"].values,\n    n_epochs=1,\n    batch_size=32,\n)\n</code></pre> <p>2. Tabular and Text data</p> <p> </p> <pre><code>from pytorch_widedeep.preprocessing import TabPreprocessor, TextPreprocessor\nfrom pytorch_widedeep.models import TabMlp, BasicRNN, WideDeep\nfrom pytorch_widedeep.training import Trainer\n\n# Tabular\ntab_preprocessor = TabPreprocessor(\n    embed_cols=[\"city\", \"name\"], continuous_cols=[\"age\", \"height\"]\n)\nX_tab = tab_preprocessor.fit_transform(df)\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=tab_preprocessor.continuous_cols,\n    mlp_hidden_dims=[64, 32],\n)\n\n# Text\ntext_preprocessor = TextPreprocessor(\n    text_col=\"sentence\", maxlen=20, max_vocab=100, n_cpus=1\n)\nX_text = text_preprocessor.fit_transform(df)\nrnn = BasicRNN(\n    vocab_size=len(text_preprocessor.vocab.itos),\n    embed_dim=16,\n    hidden_dim=8,\n    n_layers=1,\n)\n\n# WideDeep\nmodel = WideDeep(deeptabular=tab_mlp, deeptext=rnn)\n\n# Train\ntrainer = Trainer(model, objective=\"binary\")\n\ntrainer.fit(\n    X_tab=X_tab,\n    X_text=X_text,\n    target=df[\"target\"].values,\n    n_epochs=1,\n    batch_size=32,\n)\n</code></pre> <p>3. Tabular and text with a FC head on top via the <code>head_hidden_dims</code> param   in <code>WideDeep</code></p> <p> </p> <pre><code>from pytorch_widedeep.preprocessing import TabPreprocessor, TextPreprocessor\nfrom pytorch_widedeep.models import TabMlp, BasicRNN, WideDeep\nfrom pytorch_widedeep.training import Trainer\n\n# Tabular\ntab_preprocessor = TabPreprocessor(\n    embed_cols=[\"city\", \"name\"], continuous_cols=[\"age\", \"height\"]\n)\nX_tab = tab_preprocessor.fit_transform(df)\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=tab_preprocessor.continuous_cols,\n    mlp_hidden_dims=[64, 32],\n)\n\n# Text\ntext_preprocessor = TextPreprocessor(\n    text_col=\"sentence\", maxlen=20, max_vocab=100, n_cpus=1\n)\nX_text = text_preprocessor.fit_transform(df)\nrnn = BasicRNN(\n    vocab_size=len(text_preprocessor.vocab.itos),\n    embed_dim=16,\n    hidden_dim=8,\n    n_layers=1,\n)\n\n# WideDeep\nmodel = WideDeep(deeptabular=tab_mlp, deeptext=rnn, head_hidden_dims=[32, 16])\n\n# Train\ntrainer = Trainer(model, objective=\"binary\")\n\ntrainer.fit(\n    X_tab=X_tab,\n    X_text=X_text,\n    target=df[\"target\"].values,\n    n_epochs=1,\n    batch_size=32,\n)\n</code></pre> <p>4. Tabular and multiple text columns that are passed directly to   <code>WideDeep</code></p> <p> </p> <pre><code>from pytorch_widedeep.preprocessing import TabPreprocessor, TextPreprocessor\nfrom pytorch_widedeep.models import TabMlp, BasicRNN, WideDeep\nfrom pytorch_widedeep.training import Trainer\n\n\n# Tabular\ntab_preprocessor = TabPreprocessor(\n    embed_cols=[\"city\", \"name\"], continuous_cols=[\"age\", \"height\"]\n)\nX_tab = tab_preprocessor.fit_transform(df)\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=tab_preprocessor.continuous_cols,\n    mlp_hidden_dims=[64, 32],\n)\n\n# Text\ntext_preprocessor_1 = TextPreprocessor(\n    text_col=\"sentence\", maxlen=20, max_vocab=100, n_cpus=1\n)\nX_text_1 = text_preprocessor_1.fit_transform(df)\ntext_preprocessor_2 = TextPreprocessor(\n    text_col=\"other_sentence\", maxlen=20, max_vocab=100, n_cpus=1\n)\nX_text_2 = text_preprocessor_2.fit_transform(df)\nrnn_1 = BasicRNN(\n    vocab_size=len(text_preprocessor_1.vocab.itos),\n    embed_dim=16,\n    hidden_dim=8,\n    n_layers=1,\n)\nrnn_2 = BasicRNN(\n    vocab_size=len(text_preprocessor_2.vocab.itos),\n    embed_dim=16,\n    hidden_dim=8,\n    n_layers=1,\n)\n\n# WideDeep\nmodel = WideDeep(deeptabular=tab_mlp, deeptext=[rnn_1, rnn_2])\n\n# Train\ntrainer = Trainer(model, objective=\"binary\")\n\ntrainer.fit(\n    X_tab=X_tab,\n    X_text=[X_text_1, X_text_2],\n    target=df[\"target\"].values,\n    n_epochs=1,\n    batch_size=32,\n)\n</code></pre> <p>5. Tabular data and multiple text columns that are fused via a the library's   <code>ModelFuser</code> class</p> <p> </p> <pre><code>from pytorch_widedeep.preprocessing import TabPreprocessor, TextPreprocessor\nfrom pytorch_widedeep.models import TabMlp, BasicRNN, WideDeep, ModelFuser\nfrom pytorch_widedeep import Trainer\n\n# Tabular\ntab_preprocessor = TabPreprocessor(\n    embed_cols=[\"city\", \"name\"], continuous_cols=[\"age\", \"height\"]\n)\nX_tab = tab_preprocessor.fit_transform(df)\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=tab_preprocessor.continuous_cols,\n    mlp_hidden_dims=[64, 32],\n)\n\n# Text\ntext_preprocessor_1 = TextPreprocessor(\n    text_col=\"sentence\", maxlen=20, max_vocab=100, n_cpus=1\n)\nX_text_1 = text_preprocessor_1.fit_transform(df)\ntext_preprocessor_2 = TextPreprocessor(\n    text_col=\"other_sentence\", maxlen=20, max_vocab=100, n_cpus=1\n)\nX_text_2 = text_preprocessor_2.fit_transform(df)\n\nrnn_1 = BasicRNN(\n    vocab_size=len(text_preprocessor_1.vocab.itos),\n    embed_dim=16,\n    hidden_dim=8,\n    n_layers=1,\n)\nrnn_2 = BasicRNN(\n    vocab_size=len(text_preprocessor_2.vocab.itos),\n    embed_dim=16,\n    hidden_dim=8,\n    n_layers=1,\n)\n\nmodels_fuser = ModelFuser(models=[rnn_1, rnn_2], fusion_method=\"mult\")\n\n# WideDeep\nmodel = WideDeep(deeptabular=tab_mlp, deeptext=models_fuser)\n\n# Train\ntrainer = Trainer(model, objective=\"binary\")\n\ntrainer.fit(\n    X_tab=X_tab,\n    X_text=[X_text_1, X_text_2],\n    target=df[\"target\"].values,\n    n_epochs=1,\n    batch_size=32,\n)\n</code></pre> <p>6. Tabular and multiple text columns, with an image column. The text columns   are fused via the library's <code>ModelFuser</code> and then all fused via the   deephead paramenter in <code>WideDeep</code> which is a custom <code>ModelFuser</code> coded by   the user</p> <p>This is perhaps the less elegant solution as it involves a custom component by the user and slicing the 'incoming' tensor. In the future, we will include a <code>TextAndImageModelFuser</code> to make this process more straightforward. Still, is not really complicated and it is a good example of how to use custom components in <code>pytorch-widedeep</code>.</p> <p>Note that the only requirement for the custom component is that it has a property called <code>output_dim</code> that returns the size of the last layer of activations. In other words, it does not need to inherit from <code>BaseWDModelComponent</code>. This base class simply checks the existence of such property and avoids some typing errors internally.</p> <p> </p> <pre><code>import torch\n\nfrom pytorch_widedeep.preprocessing import TabPreprocessor, TextPreprocessor, ImagePreprocessor\nfrom pytorch_widedeep.models import TabMlp, BasicRNN, WideDeep, ModelFuser, Vision\nfrom pytorch_widedeep.models._base_wd_model_component import BaseWDModelComponent\nfrom pytorch_widedeep import Trainer\n\n# Tabular\ntab_preprocessor = TabPreprocessor(\n    embed_cols=[\"city\", \"name\"], continuous_cols=[\"age\", \"height\"]\n)\nX_tab = tab_preprocessor.fit_transform(df)\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=tab_preprocessor.continuous_cols,\n    mlp_hidden_dims=[16, 8],\n)\n\n# Text\ntext_preprocessor_1 = TextPreprocessor(\n    text_col=\"sentence\", maxlen=20, max_vocab=100, n_cpus=1\n)\nX_text_1 = text_preprocessor_1.fit_transform(df)\ntext_preprocessor_2 = TextPreprocessor(\n    text_col=\"other_sentence\", maxlen=20, max_vocab=100, n_cpus=1\n)\nX_text_2 = text_preprocessor_2.fit_transform(df)\nrnn_1 = BasicRNN(\n    vocab_size=len(text_preprocessor_1.vocab.itos),\n    embed_dim=16,\n    hidden_dim=8,\n    n_layers=1,\n)\nrnn_2 = BasicRNN(\n    vocab_size=len(text_preprocessor_2.vocab.itos),\n    embed_dim=16,\n    hidden_dim=8,\n    n_layers=1,\n)\nmodels_fuser = ModelFuser(\n    models=[rnn_1, rnn_2],\n    fusion_method=\"mult\",\n)\n\n# Image\nimage_preprocessor = ImagePreprocessor(img_col=\"image_name\", img_path=\"images\")\nX_img = image_preprocessor.fit_transform(df)\nvision = Vision(pretrained_model_setup=\"resnet18\", head_hidden_dims=[16, 8])\n\n# deephead (custom model fuser)\nclass MyModelFuser(BaseWDModelComponent):\n    \"\"\"\n    Simply a Linear + Relu sequence on top of the text + images followed by a\n    Linear -&gt; Relu -&gt; Linear for the concatenation of tabular slice of the\n    tensor and the output of the text and image sequential model\n    \"\"\"\n    def __init__(\n        self,\n        tab_incoming_dim: int,\n        text_incoming_dim: int,\n        image_incoming_dim: int,\n        output_units: int,\n    ):\n\n        super(MyModelFuser, self).__init__()\n\n        self.tab_incoming_dim = tab_incoming_dim\n        self.text_incoming_dim = text_incoming_dim\n        self.image_incoming_dim = image_incoming_dim\n        self.output_units = output_units\n        self.text_and_image_fuser = torch.nn.Sequential(\n            torch.nn.Linear(text_incoming_dim + image_incoming_dim, output_units),\n            torch.nn.ReLU(),\n        )\n        self.out = torch.nn.Sequential(\n            torch.nn.Linear(output_units + tab_incoming_dim, output_units * 4),\n            torch.nn.ReLU(),\n            torch.nn.Linear(output_units * 4, output_units),\n        )\n\n    def forward(self, X: torch.Tensor) -&gt; torch.Tensor:\n        tab_slice = slice(0, self.tab_incoming_dim)\n        text_slice = slice(\n            self.tab_incoming_dim, self.tab_incoming_dim + self.text_incoming_dim\n        )\n        image_slice = slice(\n            self.tab_incoming_dim + self.text_incoming_dim,\n            self.tab_incoming_dim + self.text_incoming_dim + self.image_incoming_dim,\n        )\n        X_tab = X[:, tab_slice]\n        X_text = X[:, text_slice]\n        X_img = X[:, image_slice]\n        X_text_and_image = self.text_and_image_fuser(torch.cat([X_text, X_img], dim=1))\n        return self.out(torch.cat([X_tab, X_text_and_image], dim=1))\n\n    @property\n    def output_dim(self):\n        return self.output_units\n\ndeephead = MyModelFuser(\n    tab_incoming_dim=tab_mlp.output_dim,\n    text_incoming_dim=models_fuser.output_dim,\n    image_incoming_dim=vision.output_dim,\n    output_units=8,\n)\n\n# WideDeep\nmodel = WideDeep(\n    deeptabular=tab_mlp,\n    deeptext=models_fuser,\n    deepimage=vision,\n    deephead=deephead,\n)\n\n# Train\ntrainer = Trainer(model, objective=\"binary\")\n\ntrainer.fit(\n    X_tab=X_tab,\n    X_text=[X_text_1, X_text_2],\n    X_img=X_img,\n    target=df[\"target\"].values,\n    n_epochs=1,\n    batch_size=32,\n)\n</code></pre> <p>7. A two-tower model</p> <p>This is a popular model in the context of recommendation systems. Let's say we have a tabular dataset formed my triples (user features, item features, target). We can create a two-tower model where the user and item features are passed through two separate models and then \"fused\" via a dot product.</p> <p> </p> <pre><code>import numpy as np\nimport pandas as pd\n\nfrom pytorch_widedeep import Trainer\nfrom pytorch_widedeep.preprocessing import TabPreprocessor\nfrom pytorch_widedeep.models import TabMlp, WideDeep, ModelFuser\n\n# Let's create the interaction dataset\n# user_features dataframe\nnp.random.seed(42)\nuser_ids = np.arange(1, 101)\nages = np.random.randint(18, 60, size=100)\ngenders = np.random.choice([\"male\", \"female\"], size=100)\nlocations = np.random.choice([\"city_a\", \"city_b\", \"city_c\", \"city_d\"], size=100)\nuser_features = pd.DataFrame(\n    {\"id\": user_ids, \"age\": ages, \"gender\": genders, \"location\": locations}\n)\n\n# item_features dataframe\nitem_ids = np.arange(1, 101)\nprices = np.random.uniform(10, 500, size=100).round(2)\ncolors = np.random.choice([\"red\", \"blue\", \"green\", \"black\"], size=100)\ncategories = np.random.choice([\"electronics\", \"clothing\", \"home\", \"toys\"], size=100)\n\nitem_features = pd.DataFrame(\n    {\"id\": item_ids, \"price\": prices, \"color\": colors, \"category\": categories}\n)\n\n# Interactions dataframe\ninteraction_user_ids = np.random.choice(user_ids, size=1000)\ninteraction_item_ids = np.random.choice(item_ids, size=1000)\npurchased = np.random.choice([0, 1], size=1000, p=[0.7, 0.3])\ninteractions = pd.DataFrame(\n    {\n        \"user_id\": interaction_user_ids,\n        \"item_id\": interaction_item_ids,\n        \"purchased\": purchased,\n    }\n)\nuser_item_purchased = interactions.merge(\n    user_features, left_on=\"user_id\", right_on=\"id\"\n).merge(item_features, left_on=\"item_id\", right_on=\"id\")\n\n# Users\ntab_preprocessor_user = TabPreprocessor(\n    cat_embed_cols=[\"gender\", \"location\"],\n    continuous_cols=[\"age\"],\n)\nX_user = tab_preprocessor_user.fit_transform(user_item_purchased)\ntab_mlp_user = TabMlp(\n    column_idx=tab_preprocessor_user.column_idx,\n    cat_embed_input=tab_preprocessor_user.cat_embed_input,\n    continuous_cols=[\"age\"],\n    mlp_hidden_dims=[16, 8],\n    mlp_dropout=[0.2, 0.2],\n)\n\n# Items\ntab_preprocessor_item = TabPreprocessor(\n    cat_embed_cols=[\"color\", \"category\"],\n    continuous_cols=[\"price\"],\n)\nX_item = tab_preprocessor_item.fit_transform(user_item_purchased)\ntab_mlp_item = TabMlp(\n    column_idx=tab_preprocessor_item.column_idx,\n    cat_embed_input=tab_preprocessor_item.cat_embed_input,\n    continuous_cols=[\"price\"],\n    mlp_hidden_dims=[16, 8],\n    mlp_dropout=[0.2, 0.2],\n)\n\ntwo_tower_model = ModelFuser([tab_mlp_user, tab_mlp_item], fusion_method=\"dot\")\n\nmodel = WideDeep(deeptabular=two_tower_model)\n\ntrainer = Trainer(model, objective=\"binary\")\n\ntrainer.fit(\n    X_tab=[X_user, X_item],\n    target=interactions.purchased.values,\n    n_epochs=1,\n    batch_size=32,\n)\n</code></pre> <p>8. Tabular with a multi-target loss</p> <p>This one is \"a bonus\" to illustrate the use of multi-target losses, more than actually a different architecture.</p> <p> </p> <pre><code>from pytorch_widedeep.preprocessing import TabPreprocessor, TextPreprocessor, ImagePreprocessor\nfrom pytorch_widedeep.models import TabMlp, BasicRNN, WideDeep, ModelFuser, Vision\nfrom pytorch_widedeep.losses_multitarget import MultiTargetClassificationLoss\nfrom pytorch_widedeep.models._base_wd_model_component import BaseWDModelComponent\nfrom pytorch_widedeep import Trainer\n\n# let's add a second target to the dataframe\ndf[\"target2\"] = [random.choice([0, 1]) for _ in range(100)]\n\n# Tabular\ntab_preprocessor = TabPreprocessor(\n    embed_cols=[\"city\", \"name\"], continuous_cols=[\"age\", \"height\"]\n)\nX_tab = tab_preprocessor.fit_transform(df)\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=tab_preprocessor.continuous_cols,\n    mlp_hidden_dims=[64, 32],\n)\n\n# 'pred_dim=2' because we have two binary targets. For other types of targets,\n#  please, see the documentation\nmodel = WideDeep(deeptabular=tab_mlp, pred_dim=2).\n\nloss = MultiTargetClassificationLoss(binary_config=[0, 1], reduction=\"mean\")\n\n# When a multi-target loss is used, 'custom_loss_function' must not be None.\n# See the docs\ntrainer = Trainer(model, objective=\"multitarget\", custom_loss_function=loss)\n\ntrainer.fit(\n    X_tab=X_tab,\n    target=df[[\"target\", \"target2\"]].values,\n    n_epochs=1,\n    batch_size=32,\n)\n</code></pre>"},{"location":"index.html#the-deeptabular-component","title":"The <code>deeptabular</code> component","text":"<p>It is important to emphasize again that each individual component, <code>wide</code>, <code>deeptabular</code>, <code>deeptext</code> and <code>deepimage</code>, can be used independently and in isolation. For example, one could use only <code>wide</code>, which is in simply a linear model. In fact, one of the most interesting functionalities in<code>pytorch-widedeep</code> would be the use of the <code>deeptabular</code> component on its own, i.e. what one might normally refer as Deep Learning for Tabular Data. Currently, <code>pytorch-widedeep</code> offers the following different models for that component:</p> <ol> <li>Wide: a simple linear model where the nonlinearities are captured via cross-product transformations, as explained before.</li> <li>TabMlp: a simple MLP that receives embeddings representing the categorical features, concatenated with the continuous features, which can also be embedded.</li> <li>TabResnet: similar to the previous model but the embeddings are passed through a series of ResNet blocks built with dense layers.</li> <li>TabNet: details on TabNet can be found in TabNet: Attentive Interpretable Tabular Learning</li> </ol> <p>Two simpler attention based models that we call:</p> <ol> <li>ContextAttentionMLP: MLP with at attention mechanism \"on top\" that is based on     Hierarchical Attention Networks for Document Classification</li> <li>SelfAttentionMLP: MLP with an attention mechanism that is a simplified     version of a transformer block that we refer as \"query-key self-attention\".</li> </ol> <p>The <code>Tabformer</code> family, i.e. Transformers for Tabular data:</p> <ol> <li>TabTransformer: details on the TabTransformer can be found in TabTransformer: Tabular Data Modeling Using Contextual Embeddings.</li> <li>SAINT: Details on SAINT can be found in SAINT: Improved Neural Networks for Tabular Data via Row Attention and Contrastive Pre-Training.</li> <li>FT-Transformer: details on the FT-Transformer can be found in Revisiting Deep Learning Models for Tabular Data.</li> <li>TabFastFormer: adaptation of the FastFormer for tabular data. Details on the Fasformer can be found in FastFormers: Highly Efficient Transformer Models for Natural Language Understanding</li> <li>TabPerceiver: adaptation of the Perceiver for tabular data. Details on the Perceiver can be found in Perceiver: General Perception with Iterative Attention</li> </ol> <p>And probabilistic DL models for tabular data based on Weight Uncertainty in Neural Networks:</p> <ol> <li>BayesianWide: Probabilistic adaptation of the <code>Wide</code> model.</li> <li>BayesianTabMlp: Probabilistic adaptation of the <code>TabMlp</code> model</li> </ol> <p>Note that while there are scientific publications for the TabTransformer, SAINT and FT-Transformer, the TabFasfFormer and TabPerceiver are our own adaptation of those algorithms for tabular data.</p> <p>In addition, Self-Supervised pre-training can be used for all <code>deeptabular</code> models, with the exception of the <code>TabPerceiver</code>. Self-Supervised pre-training can be used via two methods or routines which we refer as: encoder-decoder method and constrastive-denoising method. Please, see the documentation and the examples for details on this functionality, and all other options in the library.</p>"},{"location":"index.html#the-rec-module","title":"The <code>rec</code> module","text":"<p>This module was introduced as an extension to the existing components in the library, addressing questions and issues related to recommendation systems. While still under active development, it currently includes a select number of powerful recommendation models.</p> <p>It's worth noting that this library already supported the implementation of various recommendation algorithms using existing components. For example, models like Wide and Deep, Two-Tower, or Neural Collaborative Filtering could be constructed using the library's core functionalities.</p> <p>The recommendation algorithms in the <code>rec</code> module are:</p> <ol> <li>DeepFM: A Factorization-Machine based Neural Network for CTR Prediction</li> <li>(Deep) Field Aware Factorization Machine (FFM): a Deep Learning version of the algorithm presented in Field-aware Factorization Machines in a Real-world Online Advertising System</li> <li>xDeepFM: Combining Explicit and Implicit Feature Interactions for Recommender Systems</li> <li>Deep Interest Network for Click-Through Rate Prediction</li> </ol> <p>These can all be used as the <code>deeptabular</code> component in the <code>WideDeep</code> model. See the examples for more details.</p>"},{"location":"index.html#text-and-images","title":"Text and Images","text":"<p>For the text component, <code>deeptext</code>, the library offers the following models:</p> <ol> <li>BasicRNN: a simple RNN 2. AttentiveRNN: a RNN with an attention mechanism based on the Hierarchical Attention Networks for DocumentClassification</li> <li>StackedAttentiveRNN: a stack of AttentiveRNNs</li> <li>HFModel: a wrapper around Hugging Face Transfomer-based models. At the moment only models from the families BERT, RoBERTa, DistilBERT, ALBERT and ELECTRA are supported. This is because this library is designed to address classification and regression tasks and these are the most 'popular' encoder-only models, which have proved to be those that work best for these tasks. If there is demand for other models, they will be included in the future.</li> </ol> <p>For the image component, <code>deepimage</code>, the library supports models from the following families: 'resnet', 'shufflenet', 'resnext', 'wide_resnet', 'regnet', 'densenet', 'mobilenetv3',  'mobilenetv2', 'mnasnet', 'efficientnet' and 'squeezenet'.  These are  offered via <code>torchvision</code> and wrapped up in the <code>Vision</code> class.</p>"},{"location":"index.html#acknowledgments","title":"Acknowledgments","text":"<p>This library takes from a series of other libraries, so I think it is just fair to mention them here in the README (specific mentions are also included in the code).</p> <p>The <code>Callbacks</code> and <code>Initializers</code> structure and code is inspired by the <code>torchsample</code> library, which in itself partially inspired by <code>Keras</code>.</p> <p>The <code>TextProcessor</code> class in this library uses the <code>fastai</code>'s <code>Tokenizer</code> and <code>Vocab</code>. The code at <code>utils.fastai_transforms</code> is a minor adaptation of their code so it functions within this library. To my experience their <code>Tokenizer</code> is the best in class.</p> <p>The <code>ImageProcessor</code> class in this library uses code from the fantastic Deep Learning for Computer Vision (DL4CV) book by Adrian Rosebrock.</p>"},{"location":"index.html#license","title":"License","text":"<p>This work is dual-licensed under Apache 2.0 and MIT (or any later version). You can choose between one of them if you use this work.</p> <p><code>SPDX-License-Identifier: Apache-2.0 AND MIT</code></p>"},{"location":"index.html#cite","title":"Cite","text":""},{"location":"index.html#bibtex","title":"BibTex","text":"<pre><code>@article{Zaurin_pytorch-widedeep_A_flexible_2023,\nauthor = {Zaurin, Javier Rodriguez and Mulinka, Pavol},\ndoi = {10.21105/joss.05027},\njournal = {Journal of Open Source Software},\nmonth = jun,\nnumber = {86},\npages = {5027},\ntitle = {{pytorch-widedeep: A flexible package for multimodal deep learning}},\nurl = {https://joss.theoj.org/papers/10.21105/joss.05027},\nvolume = {8},\nyear = {2023}\n}\n</code></pre>"},{"location":"index.html#apa","title":"APA","text":"<pre><code>Zaurin, J. R., &amp; Mulinka, P. (2023). pytorch-widedeep: A flexible package for\nmultimodal deep learning. Journal of Open Source Software, 8(86), 5027.\nhttps://doi.org/10.21105/joss.05027\n</code></pre>"},{"location":"contributing.html","title":"Contributing","text":"<p>Pytorch-widedeep is being developed and used by many active community members. Your help is very valuable to make it better for everyone.</p> <ul> <li>Check for the Roadmap or Open an issue to report problems or recommend new features and submit a draft pull requests, which will be changed to pull request after intial review</li> <li>Contribute to the tests to make it more reliable.</li> <li>Contribute to the documentation to make it clearer for everyone.</li> <li>Contribute to the examples to share your experience with other users.</li> <li>Join the dicussion on slack</li> </ul>"},{"location":"installation.html","title":"Installation","text":"<p>This section explains how to install <code>pytorch-widedeep</code>.</p> <p>For the latest stable release, execute:</p> <pre><code>pip install pytorch-widedeep\n</code></pre> <p>For the bleeding-edge version, execute:</p> <pre><code>pip install git+https://github.com/jrzaurin/pytorch-widedeep.git\n</code></pre> <p>For developer install</p> <pre><code># Clone the repository\ngit clone https://github.com/jrzaurin/pytorch-widedeep\ncd pytorch-widedeep\n\n# Install in dev mode\npip install -e .\n</code></pre>"},{"location":"installation.html#dependencies","title":"Dependencies","text":"<ul> <li>pandas&gt;=1.3.5</li> <li>numpy&gt;=1.21.6</li> <li>scipy&gt;=1.7.3,&lt;=1.12.0</li> <li>scikit-learn&gt;=1.0.2</li> <li>gensim</li> <li>spacy</li> <li>opencv-contrib-python</li> <li>imutils</li> <li>tqdm</li> <li>torch &gt;= 2.0.0</li> <li>torchvision &gt;= 0.15.0</li> <li>einops</li> <li>wrapt</li> <li>torchmetrics</li> <li>pyarrow</li> <li>fastparquet&gt;=0.8.1</li> <li>transformers</li> <li>sentence-transformers</li> <li>sentencepiece</li> </ul>"},{"location":"quick_start.html","title":"Quick Start","text":"<p>This is an example of a binary classification with the adult census dataset using a combination of a wide and deep model (in this case a so called <code>deeptabular</code> model) with defaults settings.</p> <pre><code>import numpy as np\nimport torch\nfrom sklearn.model_selection import train_test_split\n\nfrom pytorch_widedeep import Trainer\nfrom pytorch_widedeep.preprocessing import WidePreprocessor, TabPreprocessor\nfrom pytorch_widedeep.models import Wide, TabMlp, WideDeep\nfrom pytorch_widedeep.metrics import Accuracy\nfrom pytorch_widedeep.datasets import load_adult\n\n\ndf = load_adult(as_frame=True)\ndf[\"income_label\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int)\ndf.drop(\"income\", axis=1, inplace=True)\ndf_train, df_test = train_test_split(df, test_size=0.2, stratify=df.income_label)\n\n# Define the 'column set up'\nwide_cols = [\n    \"education\",\n    \"relationship\",\n    \"workclass\",\n    \"occupation\",\n    \"native-country\",\n    \"gender\",\n]\ncrossed_cols = [(\"education\", \"occupation\"), (\"native-country\", \"occupation\")]\n\ncat_embed_cols = [\n    \"workclass\",\n    \"education\",\n    \"marital-status\",\n    \"occupation\",\n    \"relationship\",\n    \"race\",\n    \"gender\",\n    \"capital-gain\",\n    \"capital-loss\",\n    \"native-country\",\n]\ncontinuous_cols = [\"age\", \"hours-per-week\"]\ntarget = \"income_label\"\ntarget = df_train[target].values\n\n# prepare the data\nwide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols)\nX_wide = wide_preprocessor.fit_transform(df_train)\n\ntab_preprocessor = TabPreprocessor(\n    cat_embed_cols=cat_embed_cols, continuous_cols=continuous_cols  # type: ignore[arg-type]\n)\nX_tab = tab_preprocessor.fit_transform(df_train)\n\n# build the model\nwide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1)\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=continuous_cols,\n)\nmodel = WideDeep(wide=wide, deeptabular=tab_mlp)\n\n# train and validate\ntrainer = Trainer(model, objective=\"binary\", metrics=[Accuracy])\ntrainer.fit(\n    X_wide=X_wide,\n    X_tab=X_tab,\n    target=target,\n    n_epochs=5,\n    batch_size=256,\n)\n\n# predict on test\nX_wide_te = wide_preprocessor.transform(df_test)\nX_tab_te = tab_preprocessor.transform(df_test)\npreds = trainer.predict(X_wide=X_wide_te, X_tab=X_tab_te)\n\n# Save and load\n\n# Option 1: this will also save training history and lr history if the\n# LRHistory callback is used\ntrainer.save(path=\"model_weights\", save_state_dict=True)\n\n# Option 2: save as any other torch model\ntorch.save(model.state_dict(), \"model_weights/wd_model.pt\")\n\n# From here in advance, Option 1 or 2 are the same. I assume the user has\n# prepared the data and defined the new model components:\n# 1. Build the model\nmodel_new = WideDeep(wide=wide, deeptabular=tab_mlp)\nmodel_new.load_state_dict(torch.load(\"model_weights/wd_model.pt\"))\n\n# 2. Instantiate the trainer\ntrainer_new = Trainer(model_new, objective=\"binary\")\n\n# 3. Either start the fit or directly predict\npreds = trainer_new.predict(X_wide=X_wide, X_tab=X_tab, batch_size=32)\n</code></pre>"},{"location":"examples/01_preprocessors_and_utils.html","title":"01_preprocessors_and_utils","text":"<p>For example</p> In\u00a0[1]: Copied! <pre>import numpy as np\nimport pandas as pd\nimport pytorch_widedeep as wd\n\nfrom pytorch_widedeep.datasets import load_adult\nfrom pytorch_widedeep.preprocessing import WidePreprocessor\n</pre> import numpy as np import pandas as pd import pytorch_widedeep as wd  from pytorch_widedeep.datasets import load_adult from pytorch_widedeep.preprocessing import WidePreprocessor <pre>/Users/javierrodriguezzaurin/.pyenv/versions/3.10.13/envs/widedeep310/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n  from .autonotebook import tqdm as notebook_tqdm\n</pre> In\u00a0[2]: Copied! <pre>df = load_adult(as_frame=True)\ndf.head()\n</pre> df = load_adult(as_frame=True) df.head() Out[2]: age workclass fnlwgt education educational-num marital-status occupation relationship race gender capital-gain capital-loss hours-per-week native-country income 0 25 Private 226802 11th 7 Never-married Machine-op-inspct Own-child Black Male 0 0 40 United-States &lt;=50K 1 38 Private 89814 HS-grad 9 Married-civ-spouse Farming-fishing Husband White Male 0 0 50 United-States &lt;=50K 2 28 Local-gov 336951 Assoc-acdm 12 Married-civ-spouse Protective-serv Husband White Male 0 0 40 United-States &gt;50K 3 44 Private 160323 Some-college 10 Married-civ-spouse Machine-op-inspct Husband Black Male 7688 0 40 United-States &gt;50K 4 18 ? 103497 Some-college 10 Never-married ? Own-child White Female 0 0 30 United-States &lt;=50K In\u00a0[3]: Copied! <pre>wide_cols = [\n    \"education\",\n    \"relationship\",\n    \"workclass\",\n    \"occupation\",\n    \"native-country\",\n    \"gender\",\n]\ncrossed_cols = [(\"education\", \"occupation\"), (\"native-country\", \"occupation\")]\n</pre> wide_cols = [     \"education\",     \"relationship\",     \"workclass\",     \"occupation\",     \"native-country\",     \"gender\", ] crossed_cols = [(\"education\", \"occupation\"), (\"native-country\", \"occupation\")] In\u00a0[4]: Copied! <pre>wide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols)\nX_wide = wide_preprocessor.fit_transform(df)\n# From here on, any new observation can be prepared by simply running `.transform`\n# new_X_wide = wide_preprocessor.transform(new_df)\n</pre> wide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols) X_wide = wide_preprocessor.fit_transform(df) # From here on, any new observation can be prepared by simply running `.transform` # new_X_wide = wide_preprocessor.transform(new_df) In\u00a0[5]: Copied! <pre>X_wide\n</pre> X_wide Out[5]: <pre>array([[  1,  17,  23, ...,  89,  91, 316],\n       [  2,  18,  23, ...,  89,  92, 317],\n       [  3,  18,  24, ...,  89,  93, 318],\n       ...,\n       [  2,  20,  23, ...,  90, 103, 323],\n       [  2,  17,  23, ...,  89, 103, 323],\n       [  2,  21,  29, ...,  90, 115, 324]])</pre> <p>Note that the label encoding starts from <code>1</code>. This is because it is convenient to leave <code>0</code> for padding, i.e. unknown categories. Let's take from example the first entry</p> In\u00a0[6]: Copied! <pre>X_wide[0]\n</pre> X_wide[0] Out[6]: <pre>array([  1,  17,  23,  32,  47,  89,  91, 316])</pre> In\u00a0[7]: Copied! <pre>wide_preprocessor.inverse_transform(X_wide[:1])\n</pre> wide_preprocessor.inverse_transform(X_wide[:1]) Out[7]: education relationship workclass occupation native-country gender education_occupation native-country_occupation 0 11th Own-child Private Machine-op-inspct United-States Male 11th-Machine-op-inspct United-States-Machine-op-inspct <p>As we can see, <code>wide_preprocessor</code> numerically encodes the <code>wide_cols</code> and the <code>crossed_cols</code>, which can be recovered using the method <code>inverse_transform</code>.</p> In\u00a0[8]: Copied! <pre>from pytorch_widedeep.preprocessing import TabPreprocessor\n</pre> from pytorch_widedeep.preprocessing import TabPreprocessor In\u00a0[9]: Copied! <pre># cat_embed_cols = [(column_name, embed_dim), ...]\ncat_embed_cols = [\n    (\"education\", 10),\n    (\"relationship\", 8),\n    (\"workclass\", 10),\n    (\"occupation\", 10),\n    (\"native-country\", 10),\n]\ncontinuous_cols = [\"age\", \"hours-per-week\"]\n</pre> # cat_embed_cols = [(column_name, embed_dim), ...] cat_embed_cols = [     (\"education\", 10),     (\"relationship\", 8),     (\"workclass\", 10),     (\"occupation\", 10),     (\"native-country\", 10), ] continuous_cols = [\"age\", \"hours-per-week\"] In\u00a0[10]: Copied! <pre>tab_preprocessor = TabPreprocessor(\n    cat_embed_cols=cat_embed_cols,\n    continuous_cols=continuous_cols,\n    cols_to_scale=[\"age\"],  # or scale=True or cols_to_scale=continuous_cols\n)\nX_tab = tab_preprocessor.fit_transform(df)\n# From here on, any new observation can be prepared by simply running `.transform`\n# new_X_deep = deep_preprocessor.transform(new_df)\n</pre> tab_preprocessor = TabPreprocessor(     cat_embed_cols=cat_embed_cols,     continuous_cols=continuous_cols,     cols_to_scale=[\"age\"],  # or scale=True or cols_to_scale=continuous_cols ) X_tab = tab_preprocessor.fit_transform(df) # From here on, any new observation can be prepared by simply running `.transform` # new_X_deep = deep_preprocessor.transform(new_df) In\u00a0[11]: Copied! <pre>X_tab\n</pre> X_tab Out[11]: <pre>array([[ 1.00000000e+00,  1.00000000e+00,  1.00000000e+00, ...,\n         1.00000000e+00, -9.95128932e-01,  4.00000000e+01],\n       [ 2.00000000e+00,  2.00000000e+00,  1.00000000e+00, ...,\n         1.00000000e+00, -4.69415091e-02,  5.00000000e+01],\n       [ 3.00000000e+00,  2.00000000e+00,  2.00000000e+00, ...,\n         1.00000000e+00, -7.76316450e-01,  4.00000000e+01],\n       ...,\n       [ 2.00000000e+00,  4.00000000e+00,  1.00000000e+00, ...,\n         1.00000000e+00,  1.41180837e+00,  4.00000000e+01],\n       [ 2.00000000e+00,  1.00000000e+00,  1.00000000e+00, ...,\n         1.00000000e+00, -1.21394141e+00,  2.00000000e+01],\n       [ 2.00000000e+00,  5.00000000e+00,  7.00000000e+00, ...,\n         1.00000000e+00,  9.74183408e-01,  4.00000000e+01]])</pre> <p>Note that the label encoding starts from <code>1</code>. This is because it is convenient to leave <code>0</code> for padding, i.e. unknown categories. Let's take from example the first entry</p> In\u00a0[12]: Copied! <pre>X_tab[0]\n</pre> X_tab[0] Out[12]: <pre>array([ 1.        ,  1.        ,  1.        ,  1.        ,  1.        ,\n       -0.99512893, 40.        ])</pre> In\u00a0[13]: Copied! <pre>tab_preprocessor.inverse_transform(X_tab[:1])\n</pre> tab_preprocessor.inverse_transform(X_tab[:1]) Out[13]: education relationship workclass occupation native-country age hours-per-week 0 11th Own-child Private Machine-op-inspct United-States 25.0 40.0 <p>The <code>TabPreprocessor</code> will have a series of useful attributes that can later be used when instantiating the different Tabular Models, such us for example, the column indexes (used to slice the tensors, internally in the models) or the categorical embeddings set up</p> In\u00a0[14]: Copied! <pre>tab_preprocessor.column_idx\n</pre> tab_preprocessor.column_idx Out[14]: <pre>{'education': 0,\n 'relationship': 1,\n 'workclass': 2,\n 'occupation': 3,\n 'native-country': 4,\n 'age': 5,\n 'hours-per-week': 6}</pre> In\u00a0[15]: Copied! <pre># column name, num unique, embedding dim\ntab_preprocessor.cat_embed_input\n</pre> # column name, num unique, embedding dim tab_preprocessor.cat_embed_input Out[15]: <pre>[('education', 16, 10),\n ('relationship', 6, 8),\n ('workclass', 9, 10),\n ('occupation', 15, 10),\n ('native-country', 42, 10)]</pre> <p>As I mentioned, there is more one can do, such as for example, quantize (or bucketize) the continuous cols. For this we could use the <code>quantization_setup</code> param. This parameter accepts a number of different inputs and uses <code>pd.cut</code> under the hood to quantize the continuous cols. For more info, please, read the docs. Let's use it here to quantize \"age\" and \"hours-per-week\" in 4 and 5 \"buckets\" respectively</p> In\u00a0[16]: Copied! <pre>quantization_setup = {\n    \"age\": 4,\n    \"hours-per-week\": 5,\n}  # you can also pass a list of floats with the boundaries if you wanted\nquant_tab_preprocessor = TabPreprocessor(\n    cat_embed_cols=cat_embed_cols,\n    continuous_cols=continuous_cols,\n    quantization_setup=quantization_setup,\n)\nqX_tab = quant_tab_preprocessor.fit_transform(df)\n</pre> quantization_setup = {     \"age\": 4,     \"hours-per-week\": 5, }  # you can also pass a list of floats with the boundaries if you wanted quant_tab_preprocessor = TabPreprocessor(     cat_embed_cols=cat_embed_cols,     continuous_cols=continuous_cols,     quantization_setup=quantization_setup, ) qX_tab = quant_tab_preprocessor.fit_transform(df) <pre>/Users/javierrodriguezzaurin/Projects/pytorch-widedeep/pytorch_widedeep/preprocessing/tab_preprocessor.py:358: UserWarning: Continuous columns will not be normalised\n  warnings.warn(\"Continuous columns will not be normalised\")\n</pre> In\u00a0[17]: Copied! <pre>qX_tab\n</pre> qX_tab Out[17]: <pre>array([[1, 1, 1, ..., 1, 1, 2],\n       [2, 2, 1, ..., 1, 2, 3],\n       [3, 2, 2, ..., 1, 1, 2],\n       ...,\n       [2, 4, 1, ..., 1, 3, 2],\n       [2, 1, 1, ..., 1, 1, 1],\n       [2, 5, 7, ..., 1, 2, 2]])</pre> <p>Note that the continuous columns that have been bucketised into quantiles are treated as any other categorical column</p> In\u00a0[18]: Copied! <pre>quant_tab_preprocessor.cat_embed_input\n</pre> quant_tab_preprocessor.cat_embed_input Out[18]: <pre>[('education', 16, 10),\n ('relationship', 6, 8),\n ('workclass', 9, 10),\n ('occupation', 15, 10),\n ('native-country', 42, 10),\n ('age', 4, 4),\n ('hours-per-week', 5, 4)]</pre> <p>Where the column 'age' has now 4 categories, which will be encoded using embeddings of 4 dims. Note that, as any other categorical columns, the categorical \"counter\" starts with 1. This is because all incoming values that are lower/higher than the existing lowest/highest value in the train (or already seen) dataset, will be encoded as 0.</p> In\u00a0[19]: Copied! <pre>np.unique(qX_tab[:, quant_tab_preprocessor.column_idx[\"age\"]])\n</pre> np.unique(qX_tab[:, quant_tab_preprocessor.column_idx[\"age\"]]) Out[19]: <pre>array([1, 2, 3, 4])</pre> <p>Finally, if we now wanted to <code>inverse_transform</code> the transformed array into the original dataframe, we could still do it, but the continuous, bucketized columns will be transformed back to the middle of their quantile/bucket range</p> In\u00a0[20]: Copied! <pre>df_decoded = quant_tab_preprocessor.inverse_transform(qX_tab)\n</pre> df_decoded = quant_tab_preprocessor.inverse_transform(qX_tab) <pre>Note that quantized cols will be turned into the mid point of the corresponding bin\n</pre> In\u00a0[21]: Copied! <pre>df.head(2)\n</pre> df.head(2) Out[21]: age workclass fnlwgt education educational-num marital-status occupation relationship race gender capital-gain capital-loss hours-per-week native-country income 0 25 Private 226802 11th 7 Never-married Machine-op-inspct Own-child Black Male 0 0 40 United-States &lt;=50K 1 38 Private 89814 HS-grad 9 Married-civ-spouse Farming-fishing Husband White Male 0 0 50 United-States &lt;=50K In\u00a0[22]: Copied! <pre>df_decoded.head(2)\n</pre> df_decoded.head(2) Out[22]: education relationship workclass occupation native-country age hours-per-week 0 11th Own-child Private Machine-op-inspct United-States 26.0885 30.4 1 HS-grad Husband Private Farming-fishing United-States 44.3750 50.0 <p>there is one final comment to make regarding to the <code>inverse_transform</code> functionality. As we mentioned before, the encoding <code>0</code> is reserved for values that fall outside the range covered by the data we used to run the <code>fit</code> method. For example</p> In\u00a0[23]: Copied! <pre>df.age.min(), df.age.max()\n</pre> df.age.min(), df.age.max() Out[23]: <pre>(17, 90)</pre> <p>All future age values outside that range will be encoded as 0 and decoded as <code>NaN</code></p> In\u00a0[24]: Copied! <pre>tmp_df = df.head(1).copy()\ntmp_df.loc[:, \"age\"] = 5\n</pre> tmp_df = df.head(1).copy() tmp_df.loc[:, \"age\"] = 5 In\u00a0[25]: Copied! <pre>tmp_df\n</pre> tmp_df Out[25]: age workclass fnlwgt education educational-num marital-status occupation relationship race gender capital-gain capital-loss hours-per-week native-country income 0 5 Private 226802 11th 7 Never-married Machine-op-inspct Own-child Black Male 0 0 40 United-States &lt;=50K In\u00a0[26]: Copied! <pre># quant_tab_preprocessor has already been fitted with a data that has an age range between 17 and 90\ntmp_qX_tab = quant_tab_preprocessor.transform(tmp_df)\n</pre> # quant_tab_preprocessor has already been fitted with a data that has an age range between 17 and 90 tmp_qX_tab = quant_tab_preprocessor.transform(tmp_df) In\u00a0[27]: Copied! <pre>tmp_qX_tab\n</pre> tmp_qX_tab Out[27]: <pre>array([[1, 1, 1, 1, 1, 0, 2]])</pre> In\u00a0[28]: Copied! <pre>quant_tab_preprocessor.inverse_transform(tmp_qX_tab)\n</pre> quant_tab_preprocessor.inverse_transform(tmp_qX_tab) <pre>Note that quantized cols will be turned into the mid point of the corresponding bin\n</pre> Out[28]: education relationship workclass occupation native-country age hours-per-week 0 11th Own-child Private Machine-op-inspct United-States NaN 30.4 In\u00a0[29]: Copied! <pre>from pytorch_widedeep.preprocessing import TextPreprocessor\n</pre> from pytorch_widedeep.preprocessing import TextPreprocessor In\u00a0[30]: Copied! <pre># The airbnb dataset, which you could get from here:\n# http://insideairbnb.com/get-the-data.html, is too big to be included in\n# our datasets module (when including images). Therefore, go there,\n# download it, and use the download_images.py script to get the images\n# and the airbnb_data_processing.py to process the data. We'll find\n# better datasets in the future ;). Note that here we are only using a\n# small sample to illustrate the use, so PLEASE ignore the results, just\n# focus on usage\ndf = pd.read_csv(\"../tmp_data/airbnb/airbnb_sample.csv\")\n</pre> # The airbnb dataset, which you could get from here: # http://insideairbnb.com/get-the-data.html, is too big to be included in # our datasets module (when including images). Therefore, go there, # download it, and use the download_images.py script to get the images # and the airbnb_data_processing.py to process the data. We'll find # better datasets in the future ;). Note that here we are only using a # small sample to illustrate the use, so PLEASE ignore the results, just # focus on usage df = pd.read_csv(\"../tmp_data/airbnb/airbnb_sample.csv\") In\u00a0[31]: Copied! <pre>texts = df.description.tolist()\ntexts[:2]\n</pre> texts = df.description.tolist() texts[:2] Out[31]: <pre>[\"My bright double bedroom with a large window has a relaxed feeling! It comfortably fits one or two and is centrally located just two blocks from Finsbury Park. Enjoy great restaurants in the area and easy access to easy transport tubes, trains and buses. Babies and children of all ages are welcome. Hello Everyone, I'm offering my lovely double bedroom in Finsbury Park area (zone 2) for let in a shared apartment.  You will share the apartment with me and it is fully furnished with a self catering kitchen. Two people can easily sleep well as the room has a queen size bed. I also have a travel cot for a baby for guest with small children.  I will require a deposit up front as a security gesture on both our parts and will be given back to you when you return the keys.  I trust anyone who will be responding to this add would treat my home with care and respect .  Best Wishes  Alina Guest will have access to the self catering kitchen and bathroom. There is the flat is equipped wifi internet,\",\n \"Lots of windows and light.  St Luke's Gardens are at the end of the block, and the river not too far the other way. Ten minutes walk if you go slowly. Buses to everywhere round the corner and shops, restaurants, pubs, the cinema and Waitrose . Bright Chelsea Apartment  This is a bright one bedroom ground floor apartment in an interesting listed building. There is one double bedroom and a living room/kitchen The apartment has a full  bathroom and the kitchen is fully equipped. Two wardrobes are available exclusively for guests and bedside tables and two long drawers. This sunny convenient compact flat is just around the corner from the Waitrose supermarket and all sorts of shops, cinemas, restaurants and pubs.  This is a lovely part of London. There is a fun farmers market in the King's Road at the weekend.  Buses to everywhere are just round the corner, and two underground stations are within ten minutes walk. There is a very nice pub round by St. Luke's gardens, 4 mins slow walk, the \"]</pre> In\u00a0[32]: Copied! <pre>text_preprocessor = TextPreprocessor(text_col=\"description\")\nX_text = text_preprocessor.fit_transform(df)\n# From here on, any new observation can be prepared by simply running `.transform`\n# new_X_text = text_preprocessor.transform(new_df)\n</pre> text_preprocessor = TextPreprocessor(text_col=\"description\") X_text = text_preprocessor.fit_transform(df) # From here on, any new observation can be prepared by simply running `.transform` # new_X_text = text_preprocessor.transform(new_df) <pre>The vocabulary contains 2192 tokens\n</pre> In\u00a0[33]: Copied! <pre>print(X_text[0])\n</pre> print(X_text[0]) <pre>[  29   48   37  367  818   17  910   17  177   15  122  349   53  879\n 1174  126  393   40  911    0   23  228   71  819    9   53   55 1380\n  225   11   18  308   18 1564   10  755    0  942  239   53   55    0\n   11   36 1013  277 1974   70   62   15 1475    9  943    5  251    5\n    0    5    0    5  177   53   37   75   11   10  294  726   32    9\n   42    5   25   12   10   22   12  136  100  145]\n</pre> In\u00a0[34]: Copied! <pre>from pytorch_widedeep.preprocessing import ImagePreprocessor\n</pre> from pytorch_widedeep.preprocessing import ImagePreprocessor In\u00a0[35]: Copied! <pre>image_preprocessor = wd.preprocessing.ImagePreprocessor(\n    img_col=\"id\", img_path=\"../tmp_data/airbnb/property_picture/\"\n)\nX_images = image_preprocessor.fit_transform(df)\n# From here on, any new observation can be prepared by simply running `.transform`\n# new_X_images = image_preprocessor.transform(new_df)\n</pre> image_preprocessor = wd.preprocessing.ImagePreprocessor(     img_col=\"id\", img_path=\"../tmp_data/airbnb/property_picture/\" ) X_images = image_preprocessor.fit_transform(df) # From here on, any new observation can be prepared by simply running `.transform` # new_X_images = image_preprocessor.transform(new_df) <pre>Reading Images from ../tmp_data/airbnb/property_picture/\nResizing\n</pre> <pre>100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 1001/1001 [00:01&lt;00:00, 667.89it/s]\n</pre> <pre>Computing normalisation metrics\n</pre> In\u00a0[36]: Copied! <pre>X_images[0].shape\n</pre> X_images[0].shape Out[36]: <pre>(224, 224, 3)</pre>"},{"location":"examples/01_preprocessors_and_utils.html#processors-and-utils","title":"Processors and Utils\u00b6","text":"<p>Description of the main tools and utilities that one needs to prepare the data for a <code>WideDeep</code> model constructor.</p>"},{"location":"examples/01_preprocessors_and_utils.html#the-preprocessing-module","title":"The <code>preprocessing</code>  module\u00b6","text":"<p>There are 4 preprocessors, corresponding to 4 main components of the <code>WideDeep</code> model. These are</p> <ul> <li><code>WidePreprocessor</code></li> <li><code>TabPreprocessor</code></li> <li><code>TextPreprocessor</code></li> <li><code>ImagePreprocessor</code></li> </ul> <p>Behind the scenes, these preprocessors use a series of helper funcions and classes that are in the <code>utils</code> module. If you were interested please go and have a look to the documentation</p>"},{"location":"examples/01_preprocessors_and_utils.html#1-widepreprocessor","title":"1. WidePreprocessor\u00b6","text":"<p>The <code>wide</code> component of the model is a linear model that in principle, could be implemented as a linear layer receiving the result of on one-hot encoding categorical columns. However, this is not memory efficient. Therefore, we implement a liner layer as an Embedding layer plus a bias. I will explain in a bit more detail later.</p> <p>With that in mind, <code>WidePreprocessor</code> simply encodes the categories numerically so that they are the indexes of the lookup table that is an Embedding layer.</p>"},{"location":"examples/01_preprocessors_and_utils.html#2-tabpreprocessor","title":"2. TabPreprocessor\u00b6","text":"<p>The <code>TabPreprocessor</code> has a lot of different functionalities. Let's explore some of them in detail. In its basic use, the <code>TabPreprocessor</code> simply label encodes the categorical columns and normalises the numerical ones (unless otherwised specified).</p>"},{"location":"examples/01_preprocessors_and_utils.html#3-textpreprocessor","title":"3. TextPreprocessor\u00b6","text":"<p>This preprocessor returns the tokenised, padded sequences that will be directly fed to the stack of LSTMs.</p>"},{"location":"examples/01_preprocessors_and_utils.html#4-imagepreprocessor","title":"4. ImagePreprocessor\u00b6","text":"<p><code>ImagePreprocessor</code> simply resizes the images, being aware of the aspect ratio.</p>"},{"location":"examples/02_model_components.html","title":"02_model_components","text":"In\u00a0[1]: Copied! <pre>import torch\nimport pandas as pd\nimport numpy as np\n\nfrom torch import nn\n</pre> import torch import pandas as pd import numpy as np  from torch import nn In\u00a0[2]: Copied! <pre>df = pd.DataFrame({\"color\": [\"r\", \"b\", \"g\"], \"size\": [\"s\", \"n\", \"l\"]})\ndf.head()\n</pre> df = pd.DataFrame({\"color\": [\"r\", \"b\", \"g\"], \"size\": [\"s\", \"n\", \"l\"]}) df.head() Out[2]: color size 0 r s 1 b n 2 g l <p>one hot encoded, the first observation would be</p> In\u00a0[3]: Copied! <pre>obs_0_oh = (np.array([1.0, 0.0, 0.0, 1.0, 0.0, 0.0])).astype(\"float32\")\n</pre> obs_0_oh = (np.array([1.0, 0.0, 0.0, 1.0, 0.0, 0.0])).astype(\"float32\") <p>if we simply numerically encode (label encode or <code>le</code>) the values:</p> In\u00a0[4]: Copied! <pre>obs_0_le = (np.array([0, 3])).astype(\"int64\")\n</pre> obs_0_le = (np.array([0, 3])).astype(\"int64\") <p>Note that in the functioning implementation of the package we start from 1, saving 0 for padding, i.e. unseen values.</p> <p>Now, let's see if the two implementations are equivalent</p> In\u00a0[5]: Copied! <pre># we have 6 different values. Let's assume we are performing a regression, so pred_dim = 1\nlin = nn.Linear(6, 1)\n</pre> # we have 6 different values. Let's assume we are performing a regression, so pred_dim = 1 lin = nn.Linear(6, 1) In\u00a0[6]: Copied! <pre>emb = nn.Embedding(6, 1)\nemb.weight = nn.Parameter(lin.weight.reshape_as(emb.weight))\n</pre> emb = nn.Embedding(6, 1) emb.weight = nn.Parameter(lin.weight.reshape_as(emb.weight)) In\u00a0[7]: Copied! <pre>lin(torch.tensor(obs_0_oh))\n</pre> lin(torch.tensor(obs_0_oh)) Out[7]: <pre>tensor([-0.5181], grad_fn=&lt;ViewBackward0&gt;)</pre> In\u00a0[8]: Copied! <pre>emb(torch.tensor(obs_0_le)).sum() + lin.bias\n</pre> emb(torch.tensor(obs_0_le)).sum() + lin.bias Out[8]: <pre>tensor([-0.5181], grad_fn=&lt;AddBackward0&gt;)</pre> <p>And this is precisely how the linear model <code>Wide</code> is implemented</p> In\u00a0[9]: Copied! <pre>from pytorch_widedeep.models import Wide\n</pre> from pytorch_widedeep.models import Wide <pre>/Users/javierrodriguezzaurin/.pyenv/versions/3.10.13/envs/widedeep310/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n  from .autonotebook import tqdm as notebook_tqdm\n</pre> In\u00a0[10]: Copied! <pre># ?Wide\n</pre> # ?Wide In\u00a0[11]: Copied! <pre>wide = Wide(input_dim=10, pred_dim=1)\nwide\n</pre> wide = Wide(input_dim=10, pred_dim=1) wide Out[11]: <pre>Wide(\n  (wide_linear): Embedding(11, 1, padding_idx=0)\n)</pre> <p>Note that even though the input dim is 10, the Embedding layer has 11 weights. Again, this is because we save <code>0</code> for padding, which is used for unseen values during the encoding process.</p> <p>As I mentioned, <code>deeptabular</code> has enough complexity on its own and it will be described in a separated notebook. Let's then jump to <code>deeptext</code>.</p> In\u00a0[12]: Copied! <pre>from pytorch_widedeep.preprocessing import TabPreprocessor\nfrom pytorch_widedeep.models import TabMlp\n</pre> from pytorch_widedeep.preprocessing import TabPreprocessor from pytorch_widedeep.models import TabMlp In\u00a0[13]: Copied! <pre>data = {\n    \"cat1\": np.random.choice([\"A\", \"B\", \"C\"], size=20),\n    \"cat2\": np.random.choice([\"X\", \"Y\"], size=20),\n    \"cont1\": np.random.rand(20),\n    \"cont2\": np.random.rand(20),\n}\n\ndf = pd.DataFrame(data)\n</pre> data = {     \"cat1\": np.random.choice([\"A\", \"B\", \"C\"], size=20),     \"cat2\": np.random.choice([\"X\", \"Y\"], size=20),     \"cont1\": np.random.rand(20),     \"cont2\": np.random.rand(20), }  df = pd.DataFrame(data) In\u00a0[14]: Copied! <pre>df.head()\n</pre> df.head() Out[14]: cat1 cat2 cont1 cont2 0 A Y 0.789347 0.561789 1 C X 0.050822 0.061538 2 A Y 0.863784 0.241967 3 C X 0.917848 0.644658 4 C Y 0.042328 0.417303 In\u00a0[15]: Copied! <pre># see the docs for details on all params/options\ntab_preprocessor = TabPreprocessor(\n    cat_embed_cols=[\"cat1\", \"cat2\"],\n    continuous_cols=[\"cont1\", \"cont2\"],\n    embedding_rule=\"fastai\",\n)\n</pre> # see the docs for details on all params/options tab_preprocessor = TabPreprocessor(     cat_embed_cols=[\"cat1\", \"cat2\"],     continuous_cols=[\"cont1\", \"cont2\"],     embedding_rule=\"fastai\", ) In\u00a0[16]: Copied! <pre>X_tab = tab_preprocessor.fit_transform(df)\n</pre> X_tab = tab_preprocessor.fit_transform(df) <pre>/Users/javierrodriguezzaurin/Projects/pytorch-widedeep/pytorch_widedeep/preprocessing/tab_preprocessor.py:358: UserWarning: Continuous columns will not be normalised\n  warnings.warn(\"Continuous columns will not be normalised\")\n</pre> In\u00a0[17]: Copied! <pre># toy example just to build a model.\ntabmlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=tab_preprocessor.continuous_cols,\n    embed_continuous_method=\"standard\",\n    cont_embed_dim=4,\n    mlp_hidden_dims=[8, 4],\n    mlp_linear_first=True,\n)\ntabmlp\n</pre> # toy example just to build a model. tabmlp = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     continuous_cols=tab_preprocessor.continuous_cols,     embed_continuous_method=\"standard\",     cont_embed_dim=4,     mlp_hidden_dims=[8, 4],     mlp_linear_first=True, ) tabmlp Out[17]: <pre>TabMlp(\n  (cat_embed): DiffSizeCatEmbeddings(\n    (embed_layers): ModuleDict(\n      (emb_layer_cat1): Embedding(4, 3, padding_idx=0)\n      (emb_layer_cat2): Embedding(3, 2, padding_idx=0)\n    )\n    (embedding_dropout): Dropout(p=0.0, inplace=False)\n  )\n  (cont_norm): Identity()\n  (cont_embed): ContEmbeddings(\n    INFO: [ContLinear = weight(n_cont_cols, embed_dim) + bias(n_cont_cols, embed_dim)]\n    (linear): ContLinear(n_cont_cols=2, embed_dim=4, embed_dropout=0.0)\n    (dropout): Dropout(p=0.0, inplace=False)\n  )\n  (encoder): MLP(\n    (mlp): Sequential(\n      (dense_layer_0): Sequential(\n        (0): Linear(in_features=13, out_features=8, bias=True)\n        (1): ReLU(inplace=True)\n        (2): Dropout(p=0.1, inplace=False)\n      )\n      (dense_layer_1): Sequential(\n        (0): Linear(in_features=8, out_features=4, bias=True)\n        (1): ReLU(inplace=True)\n        (2): Dropout(p=0.1, inplace=False)\n      )\n    )\n  )\n)</pre> <p>Lets describe a bit the model: first we have what we call a <code>DiffSizeCatEmbeddings</code>, where categorical columns with different number of unique categories will be encoded with embeddings of different dimensions. Then the continuous columns will not be normalised (the normalised layer is just the identity) and they will be embedded via a \"standard\" method, using a so-called <code>ContLinear</code> layer. This layer displays some <code>INFO</code> that tells us what it is (<code>ContLinear = weight(n_cont_cols, embed_dim) + bias(n_cont_cols, embed_dim)]</code>). There are two other options available to embed the continuous cols based on the paper On Embeddings for Numerical Features in Tabular Deep Learning. These are <code>PieceWise</code> and <code>Periodic</code> and all available via the <code>embed_continuous_method</code> param, which can adopt values <code>\"standard\", \"piecewise\"</code> and <code>\"periodic\"</code>. The embedded categorical and continuous columns will be then concatenated ($3 + 2 + (4 * 2) = 13$ input dims) and passed to an MLP.</p> In\u00a0[18]: Copied! <pre>from pytorch_widedeep.models import BasicRNN\n</pre> from pytorch_widedeep.models import BasicRNN In\u00a0[19]: Copied! <pre>basic_rnn = BasicRNN(vocab_size=4, hidden_dim=4, n_layers=1, padding_idx=0, embed_dim=4)\n</pre> basic_rnn = BasicRNN(vocab_size=4, hidden_dim=4, n_layers=1, padding_idx=0, embed_dim=4) <pre>/Users/javierrodriguezzaurin/.pyenv/versions/3.10.13/envs/widedeep310/lib/python3.10/site-packages/torch/nn/modules/rnn.py:82: UserWarning: dropout option adds dropout after all but last recurrent layer, so non-zero dropout expects num_layers greater than 1, but got dropout=0.1 and num_layers=1\n  warnings.warn(\"dropout option adds dropout after all but last \"\n</pre> In\u00a0[20]: Copied! <pre>basic_rnn\n</pre> basic_rnn Out[20]: <pre>BasicRNN(\n  (word_embed): Embedding(4, 4, padding_idx=0)\n  (rnn): LSTM(4, 4, batch_first=True, dropout=0.1)\n  (rnn_mlp): Identity()\n)</pre> <p>You could, if you wanted, add a Fully Connected Head (FC-Head) on top of it</p> In\u00a0[21]: Copied! <pre>from pytorch_widedeep.models import Vision\n</pre> from pytorch_widedeep.models import Vision In\u00a0[22]: Copied! <pre>resnet = Vision(pretrained_model_setup=\"resnet18\", n_trainable=0)\n</pre> resnet = Vision(pretrained_model_setup=\"resnet18\", n_trainable=0) In\u00a0[23]: Copied! <pre>resnet\n</pre> resnet Out[23]: <pre>Vision(\n  (features): Sequential(\n    (0): Conv2d(3, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False)\n    (1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n    (2): ReLU(inplace=True)\n    (3): MaxPool2d(kernel_size=3, stride=2, padding=1, dilation=1, ceil_mode=False)\n    (4): Sequential(\n      (0): BasicBlock(\n        (conv1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n        (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (relu): ReLU(inplace=True)\n        (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n        (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n      )\n      (1): BasicBlock(\n        (conv1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n        (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (relu): ReLU(inplace=True)\n        (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n        (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n      )\n    )\n    (5): Sequential(\n      (0): BasicBlock(\n        (conv1): Conv2d(64, 128, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n        (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (relu): ReLU(inplace=True)\n        (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n        (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (downsample): Sequential(\n          (0): Conv2d(64, 128, kernel_size=(1, 1), stride=(2, 2), bias=False)\n          (1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        )\n      )\n      (1): BasicBlock(\n        (conv1): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n        (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (relu): ReLU(inplace=True)\n        (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n        (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n      )\n    )\n    (6): Sequential(\n      (0): BasicBlock(\n        (conv1): Conv2d(128, 256, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n        (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (relu): ReLU(inplace=True)\n        (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n        (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (downsample): Sequential(\n          (0): Conv2d(128, 256, kernel_size=(1, 1), stride=(2, 2), bias=False)\n          (1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        )\n      )\n      (1): BasicBlock(\n        (conv1): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n        (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (relu): ReLU(inplace=True)\n        (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n        (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n      )\n    )\n    (7): Sequential(\n      (0): BasicBlock(\n        (conv1): Conv2d(256, 512, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n        (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (relu): ReLU(inplace=True)\n        (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n        (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (downsample): Sequential(\n          (0): Conv2d(256, 512, kernel_size=(1, 1), stride=(2, 2), bias=False)\n          (1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        )\n      )\n      (1): BasicBlock(\n        (conv1): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n        (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (relu): ReLU(inplace=True)\n        (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n        (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n      )\n    )\n    (8): AdaptiveAvgPool2d(output_size=(1, 1))\n  )\n)</pre>"},{"location":"examples/02_model_components.html#model-components","title":"Model Components\u00b6","text":"<p>The main components of a <code>WideDeep</code> (i.e. Multimodal) model are tabular data, text and images, which are feed into the model via so called <code>wide</code>, <code>deeptabular</code>, <code>deeptext</code> and <code>deepimage</code> model components</p>"},{"location":"examples/02_model_components.html#1-wide","title":"1. <code>wide</code>\u00b6","text":"<p>The <code>wide</code> component is a Linear layer \"plugged\" into the output neuron(s). Here, the non-linearities are captured via crossed columns. Crossed columns are, quoting directly the paper: \"For binary features, a cross-product transformation (e.g., \u201cAND(gender=female, language=en)\u201d) is 1 if and only if the constituent features (\u201cgender=female\u201d and \u201clanguage=en\u201d) are all 1, and 0 otherwise\".</p> <p>The only particularity of our implementation is that we have implemented the linear layer via an Embedding layer plus a bias. While the implementations are equivalent, the latter is faster and far more memory efficient, since we do not need to one hot encode the categorical features.</p> <p>Let's assume we the following dataset:</p>"},{"location":"examples/02_model_components.html#2-deeptabular","title":"2. <code>deeptabular</code>\u00b6","text":"<p>The <code>deeptabular</code> model alone is what normally would be referred as Deep Learning for tabular data. As mentioned a number of times throughout the library, each component can be used independently. Therefore, if you wanted to use any of the models below alone, it is perfectly possible. There are just a couple of simple requirement that will be covered in a later notebook.</p> <p>By the time of writing, there are a number of models available in <code>pytorch-widedeep</code> to do DL for tabular data. These are:</p> <ol> <li><code>TabMlp</code></li> <li><code>ContextAttentionMLP</code></li> <li><code>SelfAttentionMLP</code></li> <li><code>TabResnet</code></li> <li><code>Tabnet</code></li> <li><code>TabTransformer</code></li> <li><code>FT-Tabransformer</code></li> <li><code>SAINT</code></li> <li><code>TabFastFormer</code></li> <li><code>TabPerceiver</code></li> </ol> <p>Let's have a look to one of them. For more information on each of these models, please, have a look to the documentation</p>"},{"location":"examples/02_model_components.html#3-deeptext","title":"3. <code>deeptext</code>\u00b6","text":"<p>At the time of writing, <code>pytorch-widedeep</code> offers three models that can be passed to <code>WideDeep</code> as the <code>deeptext</code> component. These are:</p> <ol> <li>BasicRNN</li> <li>AttentiveRNN</li> <li>StackedAttentiveRNN</li> </ol> <p>For details on each of these models, please, have a look to the documentation of the package.</p> <p>We will soon integrate with Hugginface, but let me insist. It is perfectly possible to use custom models for each component, please, have a look to the corresponding notebook. In general, simply, build them and pass them as the corresponding parameters. Note that the custom models MUST return a last layer of activations (i.e. not the final prediction) so that  these activations are collected by <code>WideDeep</code> and combined accordingly. In  addition, the models MUST also contain an attribute <code>output_dim</code> with the size of these last layers of activations.</p> <p>Let's have a look to the <code>BasicRNN</code> model</p>"},{"location":"examples/02_model_components.html#4-deepimage","title":"4. <code>deepimage</code>\u00b6","text":"<p>At the time of writing <code>pytorch-widedeep</code> is integrated with torchvision via the <code>Vision</code> class. This means that the it is possible to use a variant of the following architectures:</p> <ol> <li>resnet</li> <li>shufflenet</li> <li>resnext</li> <li>wide_resnet</li> <li>regnet</li> <li>densenet</li> <li>mobilenet</li> <li>mnasnet</li> <li>efficientnet</li> <li>squeezenet</li> </ol> <p>The user can choose which layers will be trainable. Alternatively, in none of these architectures is useful, one could use a simple, fully trained CNN (please see the package documentation) or pass a custom model.</p> <p>let's have a look</p>"},{"location":"examples/03_binary_classification_with_defaults.html","title":"03_binary_classification_with_defaults","text":"In\u00a0[1]: Copied! <pre>import numpy as np\nimport pandas as pd\nimport torch\n\nfrom pytorch_widedeep.preprocessing import WidePreprocessor, TabPreprocessor\nfrom pytorch_widedeep.training import Trainer\nfrom pytorch_widedeep.models import Wide, TabMlp, WideDeep\nfrom pytorch_widedeep.metrics import Accuracy, Precision\nfrom pytorch_widedeep.datasets import load_adult\n</pre> import numpy as np import pandas as pd import torch  from pytorch_widedeep.preprocessing import WidePreprocessor, TabPreprocessor from pytorch_widedeep.training import Trainer from pytorch_widedeep.models import Wide, TabMlp, WideDeep from pytorch_widedeep.metrics import Accuracy, Precision from pytorch_widedeep.datasets import load_adult <pre>/Users/javierrodriguezzaurin/.pyenv/versions/3.10.13/envs/widedeep310/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n  from .autonotebook import tqdm as notebook_tqdm\n</pre> In\u00a0[2]: Copied! <pre>df = load_adult(as_frame=True)\ndf.head()\n</pre> df = load_adult(as_frame=True) df.head() Out[2]: age workclass fnlwgt education educational-num marital-status occupation relationship race gender capital-gain capital-loss hours-per-week native-country income 0 25 Private 226802 11th 7 Never-married Machine-op-inspct Own-child Black Male 0 0 40 United-States &lt;=50K 1 38 Private 89814 HS-grad 9 Married-civ-spouse Farming-fishing Husband White Male 0 0 50 United-States &lt;=50K 2 28 Local-gov 336951 Assoc-acdm 12 Married-civ-spouse Protective-serv Husband White Male 0 0 40 United-States &gt;50K 3 44 Private 160323 Some-college 10 Married-civ-spouse Machine-op-inspct Husband Black Male 7688 0 40 United-States &gt;50K 4 18 ? 103497 Some-college 10 Never-married ? Own-child White Female 0 0 30 United-States &lt;=50K In\u00a0[3]: Copied! <pre># For convenience, we'll replace '-' with '_'\ndf.columns = [c.replace(\"-\", \"_\") for c in df.columns]\n# binary target\ndf[\"income_label\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int)\ndf.drop(\"income\", axis=1, inplace=True)\ndf.head()\n</pre> # For convenience, we'll replace '-' with '_' df.columns = [c.replace(\"-\", \"_\") for c in df.columns] # binary target df[\"income_label\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int) df.drop(\"income\", axis=1, inplace=True) df.head() Out[3]: age workclass fnlwgt education educational_num marital_status occupation relationship race gender capital_gain capital_loss hours_per_week native_country income_label 0 25 Private 226802 11th 7 Never-married Machine-op-inspct Own-child Black Male 0 0 40 United-States 0 1 38 Private 89814 HS-grad 9 Married-civ-spouse Farming-fishing Husband White Male 0 0 50 United-States 0 2 28 Local-gov 336951 Assoc-acdm 12 Married-civ-spouse Protective-serv Husband White Male 0 0 40 United-States 1 3 44 Private 160323 Some-college 10 Married-civ-spouse Machine-op-inspct Husband Black Male 7688 0 40 United-States 1 4 18 ? 103497 Some-college 10 Never-married ? Own-child White Female 0 0 30 United-States 0 In\u00a0[4]: Copied! <pre>df.drop([\"fnlwgt\", \"educational_num\"], axis=1, inplace=True)\n</pre> df.drop([\"fnlwgt\", \"educational_num\"], axis=1, inplace=True) In\u00a0[5]: Copied! <pre># Define wide, crossed and deep tabular columns\nwide_cols = [\n    \"workclass\",\n    \"education\",\n    \"marital_status\",\n    \"occupation\",\n    \"relationship\",\n    \"race\",\n    \"gender\",\n    \"native_country\",\n]\ncrossed_cols = [(\"education\", \"occupation\"), (\"native_country\", \"occupation\")]\n</pre> # Define wide, crossed and deep tabular columns wide_cols = [     \"workclass\",     \"education\",     \"marital_status\",     \"occupation\",     \"relationship\",     \"race\",     \"gender\",     \"native_country\", ] crossed_cols = [(\"education\", \"occupation\"), (\"native_country\", \"occupation\")] In\u00a0[6]: Copied! <pre>cat_embed_cols = [\n    \"workclass\",\n    \"education\",\n    \"marital_status\",\n    \"occupation\",\n    \"relationship\",\n    \"race\",\n    \"gender\",\n    \"capital_gain\",\n    \"capital_loss\",\n    \"native_country\",\n]\ncontinuous_cols = [\"age\", \"hours_per_week\"]\n</pre> cat_embed_cols = [     \"workclass\",     \"education\",     \"marital_status\",     \"occupation\",     \"relationship\",     \"race\",     \"gender\",     \"capital_gain\",     \"capital_loss\",     \"native_country\", ] continuous_cols = [\"age\", \"hours_per_week\"] In\u00a0[7]: Copied! <pre># TARGET\ntarget_col = \"income_label\"\ntarget = df[target_col].values\n</pre> # TARGET target_col = \"income_label\" target = df[target_col].values <p>let's see what the preprocessors do</p> In\u00a0[8]: Copied! <pre># wide\nwide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols)\nX_wide = wide_preprocessor.fit_transform(df)\n</pre> # wide wide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols) X_wide = wide_preprocessor.fit_transform(df) In\u00a0[9]: Copied! <pre># # wide_preprocessor has an attribute called encoding_dict with the encoding dictionary\n# wide_preprocessor.encoding_dict\n</pre> # # wide_preprocessor has an attribute called encoding_dict with the encoding dictionary # wide_preprocessor.encoding_dict In\u00a0[10]: Copied! <pre># deeptabular\ntab_preprocessor = TabPreprocessor(\n    embed_cols=cat_embed_cols,\n    continuous_cols=continuous_cols,\n    cols_to_scale=continuous_cols,\n)\nX_tab = tab_preprocessor.fit_transform(df)\n</pre> # deeptabular tab_preprocessor = TabPreprocessor(     embed_cols=cat_embed_cols,     continuous_cols=continuous_cols,     cols_to_scale=continuous_cols, ) X_tab = tab_preprocessor.fit_transform(df) In\u00a0[11]: Copied! <pre># check the docs to understand the useful attributes that the tab_preprocessor has. For example,\n# as well as an encoding dictionary, tab_preprocessor has an attribute called cat_embed_input\n# that specifies the categortical columns that will be represented as embeddings, the number\n# of different categories per feature, and the dimension of the embeddings as defined by some\n# of the internal rules of thumb that the preprocessor has (have a look to the docs)\ntab_preprocessor.cat_embed_input\n</pre> # check the docs to understand the useful attributes that the tab_preprocessor has. For example, # as well as an encoding dictionary, tab_preprocessor has an attribute called cat_embed_input # that specifies the categortical columns that will be represented as embeddings, the number # of different categories per feature, and the dimension of the embeddings as defined by some # of the internal rules of thumb that the preprocessor has (have a look to the docs) tab_preprocessor.cat_embed_input Out[11]: <pre>[('workclass', 9, 5),\n ('education', 16, 8),\n ('marital_status', 7, 5),\n ('occupation', 15, 7),\n ('relationship', 6, 4),\n ('race', 5, 4),\n ('gender', 2, 2),\n ('capital_gain', 123, 24),\n ('capital_loss', 99, 21),\n ('native_country', 42, 13)]</pre> In\u00a0[12]: Copied! <pre>print(X_wide)\nprint(X_wide.shape)\n</pre> print(X_wide) print(X_wide.shape) <pre>[[  1  10  26 ...  61 103 328]\n [  1  11  27 ...  61 104 329]\n [  2  12  27 ...  61 105 330]\n ...\n [  1  11  28 ...  61 115 335]\n [  1  11  26 ...  61 115 335]\n [  7  11  27 ...  61 127 336]]\n(48842, 10)\n</pre> In\u00a0[13]: Copied! <pre>print(X_tab)\nprint(X_tab.shape)\n</pre> print(X_tab) print(X_tab.shape) <pre>[[ 1.          1.          1.         ...  1.         -0.99512893\n  -0.03408696]\n [ 1.          2.          2.         ...  1.         -0.04694151\n   0.77292975]\n [ 2.          3.          2.         ...  1.         -0.77631645\n  -0.03408696]\n ...\n [ 1.          2.          3.         ...  1.          1.41180837\n  -0.03408696]\n [ 1.          2.          1.         ...  1.         -1.21394141\n  -1.64812038]\n [ 7.          2.          2.         ...  1.          0.97418341\n  -0.03408696]]\n(48842, 12)\n</pre> In\u00a0[14]: Copied! <pre>wide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1)\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    cat_embed_dropout=0.1,\n    continuous_cols=continuous_cols,\n    mlp_hidden_dims=[400, 200],\n    mlp_dropout=0.5,\n    mlp_activation=\"leaky_relu\",\n)\n</pre> wide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1) tab_mlp = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     cat_embed_dropout=0.1,     continuous_cols=continuous_cols,     mlp_hidden_dims=[400, 200],     mlp_dropout=0.5,     mlp_activation=\"leaky_relu\", ) <p>Let's first find out how a linear model performs</p> In\u00a0[15]: Copied! <pre>wide\n</pre> wide Out[15]: <pre>Wide(\n  (wide_linear): Embedding(809, 1, padding_idx=0)\n)</pre> <p>Before being passed to the Trainer, the models need to be \"constructed\" with the <code>WideDeep</code> constructor class. For the particular case of the wide/linear model, not much really happens</p> In\u00a0[16]: Copied! <pre>lin_model = WideDeep(wide=wide)\n</pre> lin_model = WideDeep(wide=wide) In\u00a0[17]: Copied! <pre>lin_model\n</pre> lin_model Out[17]: <pre>WideDeep(\n  (wide): Wide(\n    (wide_linear): Embedding(809, 1, padding_idx=0)\n  )\n)</pre> In\u00a0[18]: Copied! <pre>lin_trainer = Trainer(\n    model=lin_model,\n    objective=\"binary\",\n    optimizers=torch.optim.AdamW(lin_model.parameters(), lr=0.01),\n    metrics=[Accuracy, Precision],\n)\n</pre> lin_trainer = Trainer(     model=lin_model,     objective=\"binary\",     optimizers=torch.optim.AdamW(lin_model.parameters(), lr=0.01),     metrics=[Accuracy, Precision], ) In\u00a0[19]: Copied! <pre>lin_trainer.fit(X_wide=X_wide, target=target, n_epochs=4, batch_size=128, val_split=0.2)\n</pre> lin_trainer.fit(X_wide=X_wide, target=target, n_epochs=4, batch_size=128, val_split=0.2) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 306/306 [00:02&lt;00:00, 109.04it/s, loss=0.426, metrics={'acc': 0.7983, 'prec': 0.6152}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 77/77 [00:00&lt;00:00, 102.46it/s, loss=0.366, metrics={'acc': 0.832, 'prec': 0.6916}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 306/306 [00:02&lt;00:00, 130.27it/s, loss=0.364, metrics={'acc': 0.8305, 'prec': 0.6933}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 77/77 [00:00&lt;00:00, 150.46it/s, loss=0.361, metrics={'acc': 0.8357, 'prec': 0.6982}]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 306/306 [00:02&lt;00:00, 133.19it/s, loss=0.359, metrics={'acc': 0.8329, 'prec': 0.6994}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 77/77 [00:00&lt;00:00, 145.75it/s, loss=0.361, metrics={'acc': 0.836, 'prec': 0.7009}]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 306/306 [00:02&lt;00:00, 130.91it/s, loss=0.358, metrics={'acc': 0.8333, 'prec': 0.7005}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 77/77 [00:00&lt;00:00, 155.08it/s, loss=0.361, metrics={'acc': 0.8364, 'prec': 0.702}]\n</pre> <p>Bear in mind that <code>wide</code> is a linear model where the non-linearities are captured via the crossed columns. For the crossed-columns to be effective one needs proper business knowledge. There is no magic formula to produce them</p> <p>Let's have a look to the tabular model by itself</p> In\u00a0[20]: Copied! <pre>tab_model = WideDeep(deeptabular=tab_mlp)\n</pre> tab_model = WideDeep(deeptabular=tab_mlp) In\u00a0[21]: Copied! <pre>tab_model\n</pre> tab_model Out[21]: <pre>WideDeep(\n  (deeptabular): Sequential(\n    (0): TabMlp(\n      (cat_embed): DiffSizeCatEmbeddings(\n        (embed_layers): ModuleDict(\n          (emb_layer_workclass): Embedding(10, 5, padding_idx=0)\n          (emb_layer_education): Embedding(17, 8, padding_idx=0)\n          (emb_layer_marital_status): Embedding(8, 5, padding_idx=0)\n          (emb_layer_occupation): Embedding(16, 7, padding_idx=0)\n          (emb_layer_relationship): Embedding(7, 4, padding_idx=0)\n          (emb_layer_race): Embedding(6, 4, padding_idx=0)\n          (emb_layer_gender): Embedding(3, 2, padding_idx=0)\n          (emb_layer_capital_gain): Embedding(124, 24, padding_idx=0)\n          (emb_layer_capital_loss): Embedding(100, 21, padding_idx=0)\n          (emb_layer_native_country): Embedding(43, 13, padding_idx=0)\n        )\n        (embedding_dropout): Dropout(p=0.1, inplace=False)\n      )\n      (cont_norm): Identity()\n      (encoder): MLP(\n        (mlp): Sequential(\n          (dense_layer_0): Sequential(\n            (0): Linear(in_features=95, out_features=400, bias=True)\n            (1): LeakyReLU(negative_slope=0.01, inplace=True)\n            (2): Dropout(p=0.5, inplace=False)\n          )\n          (dense_layer_1): Sequential(\n            (0): Linear(in_features=400, out_features=200, bias=True)\n            (1): LeakyReLU(negative_slope=0.01, inplace=True)\n            (2): Dropout(p=0.5, inplace=False)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=200, out_features=1, bias=True)\n  )\n)</pre> <p>You can see how the <code>WideDeep</code> class has added a final prediction layer that collects the activations from the last layer of the model and plugs them into the output neuron. If this was a multiclass classification problem, the prediction dimension (i.e. the size of that final layer) needs to be specified via the <code>pred_dim</code> when instantiating the <code>WideDeep</code> class, as we will see later</p> In\u00a0[22]: Copied! <pre>tab_trainer = Trainer(\n    model=tab_model,\n    objective=\"binary\",\n    optimizers=torch.optim.AdamW(tab_model.parameters(), lr=0.001),\n    metrics=[Accuracy, Precision],\n)\n</pre> tab_trainer = Trainer(     model=tab_model,     objective=\"binary\",     optimizers=torch.optim.AdamW(tab_model.parameters(), lr=0.001),     metrics=[Accuracy, Precision], ) In\u00a0[23]: Copied! <pre>tab_trainer.fit(X_tab=X_tab, target=target, n_epochs=4, batch_size=128, val_split=0.2)\n</pre> tab_trainer.fit(X_tab=X_tab, target=target, n_epochs=4, batch_size=128, val_split=0.2) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 306/306 [00:03&lt;00:00, 97.00it/s, loss=0.37, metrics={'acc': 0.8267, 'prec': 0.7037}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 77/77 [00:00&lt;00:00, 134.91it/s, loss=0.313, metrics={'acc': 0.8588, 'prec': 0.7577}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 306/306 [00:03&lt;00:00, 86.86it/s, loss=0.319, metrics={'acc': 0.8514, 'prec': 0.761}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 77/77 [00:01&lt;00:00, 73.13it/s, loss=0.296, metrics={'acc': 0.8675, 'prec': 0.7685}]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 306/306 [00:03&lt;00:00, 79.07it/s, loss=0.305, metrics={'acc': 0.8574, 'prec': 0.7646}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 77/77 [00:00&lt;00:00, 130.11it/s, loss=0.289, metrics={'acc': 0.8696, 'prec': 0.7765}]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 306/306 [00:03&lt;00:00, 87.39it/s, loss=0.296, metrics={'acc': 0.8622, 'prec': 0.7769}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 77/77 [00:00&lt;00:00, 90.63it/s, loss=0.285, metrics={'acc': 0.8697, 'prec': 0.7741}]\n</pre> <p>The best result I ever obtained with <code>LightGBM</code> on this dataset is 0.8782...so we are pretty close.</p> <p>Let's combine the <code>wide</code> and <code>tab_mlp</code> components see if it helps</p> In\u00a0[24]: Copied! <pre>wide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1)\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    cat_embed_dropout=0.1,\n    continuous_cols=continuous_cols,\n    mlp_hidden_dims=[400, 200],\n    mlp_dropout=0.5,\n    mlp_activation=\"leaky_relu\",\n)\nwd_model = WideDeep(wide=wide, deeptabular=tab_mlp)\n</pre> wide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1) tab_mlp = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     cat_embed_dropout=0.1,     continuous_cols=continuous_cols,     mlp_hidden_dims=[400, 200],     mlp_dropout=0.5,     mlp_activation=\"leaky_relu\", ) wd_model = WideDeep(wide=wide, deeptabular=tab_mlp) In\u00a0[25]: Copied! <pre>wd_trainer = Trainer(\n    model=wd_model,\n    objective=\"binary\",\n    optimizers=torch.optim.AdamW(wd_model.parameters(), lr=0.001),\n    metrics=[Accuracy, Precision],\n)\n</pre> wd_trainer = Trainer(     model=wd_model,     objective=\"binary\",     optimizers=torch.optim.AdamW(wd_model.parameters(), lr=0.001),     metrics=[Accuracy, Precision], ) In\u00a0[26]: Copied! <pre>wd_trainer.fit(\n    X_wide=X_wide, X_tab=X_tab, target=target, n_epochs=4, batch_size=128, val_split=0.2\n)\n</pre> wd_trainer.fit(     X_wide=X_wide, X_tab=X_tab, target=target, n_epochs=4, batch_size=128, val_split=0.2 ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 306/306 [00:03&lt;00:00, 77.48it/s, loss=0.418, metrics={'acc': 0.8047, 'prec': 0.6154}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 77/77 [00:00&lt;00:00, 110.51it/s, loss=0.321, metrics={'acc': 0.8521, 'prec': 0.7059}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 306/306 [00:03&lt;00:00, 82.70it/s, loss=0.333, metrics={'acc': 0.8428, 'prec': 0.7141}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 77/77 [00:00&lt;00:00, 112.52it/s, loss=0.299, metrics={'acc': 0.866, 'prec': 0.7447}]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 306/306 [00:04&lt;00:00, 74.34it/s, loss=0.312, metrics={'acc': 0.8533, 'prec': 0.7404}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 77/77 [00:00&lt;00:00, 89.86it/s, loss=0.29, metrics={'acc': 0.8683, 'prec': 0.7496}]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 306/306 [00:04&lt;00:00, 65.32it/s, loss=0.301, metrics={'acc': 0.8591, 'prec': 0.7542}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 77/77 [00:00&lt;00:00, 86.81it/s, loss=0.286, metrics={'acc': 0.8712, 'prec': 0.7552}]\n</pre> <p>For this particular case, the combination of both did not lead to better results that using just the tab_mlp model, when using only 4 epochs.</p> <p>Note that we have use a <code>TabMlp</code> model, but we could use any other model in the library using the same syntax</p> In\u00a0[27]: Copied! <pre>from pytorch_widedeep.models import TabTransformer\n</pre> from pytorch_widedeep.models import TabTransformer <p>The parameters for the <code>TabTransformer</code> are this</p> <pre><code>column_idx: Dict[str, int],\ncat_embed_input: Optional[List[Tuple[str, int]]] = None,\ncat_embed_dropout: Optional[float] = None,\nuse_cat_bias: Optional[bool] = None,\ncat_embed_activation: Optional[str] = None,\nshared_embed: Optional[bool] = None,\nadd_shared_embed: Optional[bool] = None,\nfrac_shared_embed: Optional[float] = None,\ncontinuous_cols: Optional[List[str]] = None,\ncont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\nembed_continuous: Optional[bool] = None,\nembed_continuous_method: Optional[Literal[\"standard\", \"piecewise\", \"periodic\"]] = None,\ncont_embed_dropout: Optional[float] = None,\ncont_embed_activation: Optional[str] = None,\nquantization_setup: Optional[Dict[str, List[float]]] = None,\nn_frequencies: Optional[int] = None,\nsigma: Optional[float] = None,\nshare_last_layer: Optional[bool] = None,\nfull_embed_dropout: Optional[bool] = None,\ninput_dim: int = 32,\nn_heads: int = 8,\nuse_qkv_bias: bool = False,\nn_blocks: int = 4,\nattn_dropout: float = 0.2,\nff_dropout: float = 0.1,\nff_factor: int = 4,\ntransformer_activation: str = \"gelu\",\nuse_linear_attention: bool = False,\nuse_flash_attention: bool = False,\nmlp_hidden_dims: Optional[List[int]] = None,\nmlp_activation: str = \"relu\",\nmlp_dropout: float = 0.1,\nmlp_batchnorm: bool = False,\nmlp_batchnorm_last: bool = False,\nmlp_linear_first: bool = True,\n</code></pre> <p>Please, see the documentation for details on each one of them, for now let's see how one could use a <code>TabTransformer</code> model in a few lines of code</p> In\u00a0[28]: Copied! <pre>tab_transformer = TabTransformer(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    cat_embed_dropout=0.1,\n    continuous_cols=continuous_cols,\n    embed_continuous_method=\"standard\",\n    cont_norm_layer=\"layernorm\",\n    cont_embed_dropout=0.2,\n    cont_embed_activation=\"leaky_relu\",\n    n_heads=4,\n    ff_dropout=0.2,\n    mlp_dropout=0.5,\n    mlp_activation=\"leaky_relu\",\n    mlp_linear_first=\"True\",\n)\n</pre> tab_transformer = TabTransformer(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     cat_embed_dropout=0.1,     continuous_cols=continuous_cols,     embed_continuous_method=\"standard\",     cont_norm_layer=\"layernorm\",     cont_embed_dropout=0.2,     cont_embed_activation=\"leaky_relu\",     n_heads=4,     ff_dropout=0.2,     mlp_dropout=0.5,     mlp_activation=\"leaky_relu\",     mlp_linear_first=\"True\", ) In\u00a0[29]: Copied! <pre>tab_model = WideDeep(deeptabular=tab_transformer)\n</pre> tab_model = WideDeep(deeptabular=tab_transformer) In\u00a0[30]: Copied! <pre>tab_model\n</pre> tab_model Out[30]: <pre>WideDeep(\n  (deeptabular): Sequential(\n    (0): TabTransformer(\n      (cat_embed): SameSizeCatEmbeddings(\n        (embed): Embedding(325, 32, padding_idx=0)\n        (dropout): Dropout(p=0.1, inplace=False)\n      )\n      (cont_norm): LayerNorm((2,), eps=1e-05, elementwise_affine=True)\n      (cont_embed): ContEmbeddings(\n        INFO: [ContLinear = weight(n_cont_cols, embed_dim) + bias(n_cont_cols, embed_dim)]\n        (linear): ContLinear(n_cont_cols=2, embed_dim=32, embed_dropout=0.2)\n        (activation_fn): LeakyReLU(negative_slope=0.01, inplace=True)\n        (dropout): Dropout(p=0.2, inplace=False)\n      )\n      (encoder): Sequential(\n        (transformer_block0): TransformerEncoder(\n          (attn): MultiHeadedAttention(\n            (dropout): Dropout(p=0.2, inplace=False)\n            (q_proj): Linear(in_features=32, out_features=32, bias=False)\n            (kv_proj): Linear(in_features=32, out_features=64, bias=False)\n            (out_proj): Linear(in_features=32, out_features=32, bias=False)\n          )\n          (ff): FeedForward(\n            (w_1): Linear(in_features=32, out_features=128, bias=True)\n            (w_2): Linear(in_features=128, out_features=32, bias=True)\n            (dropout): Dropout(p=0.2, inplace=False)\n            (activation): GELU(approximate='none')\n          )\n          (attn_addnorm): AddNorm(\n            (dropout): Dropout(p=0.2, inplace=False)\n            (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n          )\n          (ff_addnorm): AddNorm(\n            (dropout): Dropout(p=0.2, inplace=False)\n            (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n          )\n        )\n        (transformer_block1): TransformerEncoder(\n          (attn): MultiHeadedAttention(\n            (dropout): Dropout(p=0.2, inplace=False)\n            (q_proj): Linear(in_features=32, out_features=32, bias=False)\n            (kv_proj): Linear(in_features=32, out_features=64, bias=False)\n            (out_proj): Linear(in_features=32, out_features=32, bias=False)\n          )\n          (ff): FeedForward(\n            (w_1): Linear(in_features=32, out_features=128, bias=True)\n            (w_2): Linear(in_features=128, out_features=32, bias=True)\n            (dropout): Dropout(p=0.2, inplace=False)\n            (activation): GELU(approximate='none')\n          )\n          (attn_addnorm): AddNorm(\n            (dropout): Dropout(p=0.2, inplace=False)\n            (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n          )\n          (ff_addnorm): AddNorm(\n            (dropout): Dropout(p=0.2, inplace=False)\n            (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n          )\n        )\n        (transformer_block2): TransformerEncoder(\n          (attn): MultiHeadedAttention(\n            (dropout): Dropout(p=0.2, inplace=False)\n            (q_proj): Linear(in_features=32, out_features=32, bias=False)\n            (kv_proj): Linear(in_features=32, out_features=64, bias=False)\n            (out_proj): Linear(in_features=32, out_features=32, bias=False)\n          )\n          (ff): FeedForward(\n            (w_1): Linear(in_features=32, out_features=128, bias=True)\n            (w_2): Linear(in_features=128, out_features=32, bias=True)\n            (dropout): Dropout(p=0.2, inplace=False)\n            (activation): GELU(approximate='none')\n          )\n          (attn_addnorm): AddNorm(\n            (dropout): Dropout(p=0.2, inplace=False)\n            (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n          )\n          (ff_addnorm): AddNorm(\n            (dropout): Dropout(p=0.2, inplace=False)\n            (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n          )\n        )\n        (transformer_block3): TransformerEncoder(\n          (attn): MultiHeadedAttention(\n            (dropout): Dropout(p=0.2, inplace=False)\n            (q_proj): Linear(in_features=32, out_features=32, bias=False)\n            (kv_proj): Linear(in_features=32, out_features=64, bias=False)\n            (out_proj): Linear(in_features=32, out_features=32, bias=False)\n          )\n          (ff): FeedForward(\n            (w_1): Linear(in_features=32, out_features=128, bias=True)\n            (w_2): Linear(in_features=128, out_features=32, bias=True)\n            (dropout): Dropout(p=0.2, inplace=False)\n            (activation): GELU(approximate='none')\n          )\n          (attn_addnorm): AddNorm(\n            (dropout): Dropout(p=0.2, inplace=False)\n            (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n          )\n          (ff_addnorm): AddNorm(\n            (dropout): Dropout(p=0.2, inplace=False)\n            (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=384, out_features=1, bias=True)\n  )\n)</pre> In\u00a0[31]: Copied! <pre>tab_trainer = Trainer(\n    model=tab_model,\n    objective=\"binary\",\n    optimizers=torch.optim.AdamW(tab_model.parameters(), lr=0.001),\n    metrics=[Accuracy, Precision],\n)\n</pre> tab_trainer = Trainer(     model=tab_model,     objective=\"binary\",     optimizers=torch.optim.AdamW(tab_model.parameters(), lr=0.001),     metrics=[Accuracy, Precision], ) In\u00a0[32]: Copied! <pre>tab_trainer.fit(X_tab=X_tab, target=target, n_epochs=1, batch_size=128, val_split=0.2)\n</pre> tab_trainer.fit(X_tab=X_tab, target=target, n_epochs=1, batch_size=128, val_split=0.2) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 306/306 [00:11&lt;00:00, 27.57it/s, loss=0.359, metrics={'acc': 0.8334, 'prec': 0.7082}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 77/77 [00:01&lt;00:00, 57.89it/s, loss=0.33, metrics={'acc': 0.8536, 'prec': 0.7152}]\n</pre>"},{"location":"examples/03_binary_classification_with_defaults.html#simple-binary-classification-with-defaults","title":"Simple Binary Classification with defaults\u00b6","text":"<p>In this notebook we will train a Wide and Deep model and simply a \"Deep\" model using the well known adult dataset</p>"},{"location":"examples/03_binary_classification_with_defaults.html#preparing-the-data","title":"Preparing the data\u00b6","text":""},{"location":"examples/03_binary_classification_with_defaults.html#defining-the-model","title":"Defining the model\u00b6","text":""},{"location":"examples/04_regression_with_images_and_text.html","title":"04_regression_with_images_and_text","text":"In\u00a0[1]: Copied! <pre>import numpy as np\nimport pandas as pd\nimport os\nimport torch\nfrom torchvision.transforms import ToTensor, Normalize\n\nfrom pytorch_widedeep import Trainer\nfrom pytorch_widedeep.preprocessing import (\n    WidePreprocessor,\n    TabPreprocessor,\n    TextPreprocessor,\n    ImagePreprocessor,\n)\nfrom pytorch_widedeep.models import (\n    Wide,\n    TabMlp,\n    Vision,\n    BasicRNN,\n    WideDeep,\n)\nfrom pytorch_widedeep.losses import RMSELoss\nfrom pytorch_widedeep.initializers import *\nfrom pytorch_widedeep.callbacks import *\n</pre> import numpy as np import pandas as pd import os import torch from torchvision.transforms import ToTensor, Normalize  from pytorch_widedeep import Trainer from pytorch_widedeep.preprocessing import (     WidePreprocessor,     TabPreprocessor,     TextPreprocessor,     ImagePreprocessor, ) from pytorch_widedeep.models import (     Wide,     TabMlp,     Vision,     BasicRNN,     WideDeep, ) from pytorch_widedeep.losses import RMSELoss from pytorch_widedeep.initializers import * from pytorch_widedeep.callbacks import * <pre>/Users/javierrodriguezzaurin/.pyenv/versions/3.10.13/envs/widedeep310/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n  from .autonotebook import tqdm as notebook_tqdm\n</pre> In\u00a0[2]: Copied! <pre>df = pd.read_csv(\"../tmp_data/airbnb/airbnb_sample.csv\")\ndf.head()\n</pre> df = pd.read_csv(\"../tmp_data/airbnb/airbnb_sample.csv\") df.head() Out[2]: id host_id description host_listings_count host_identity_verified neighbourhood_cleansed latitude longitude is_location_exact property_type ... amenity_wide_entrance amenity_wide_entrance_for_guests amenity_wide_entryway amenity_wide_hallways amenity_wifi amenity_window_guards amenity_wine_cooler security_deposit extra_people yield 0 13913.jpg 54730 My bright double bedroom with a large window h... 4.0 f Islington 51.56802 -0.11121 t apartment ... 1 0 0 0 1 0 0 100.0 15.0 12.00 1 15400.jpg 60302 Lots of windows and light.  St Luke's Gardens ... 1.0 t Kensington and Chelsea 51.48796 -0.16898 t apartment ... 0 0 0 0 1 0 0 150.0 0.0 109.50 2 17402.jpg 67564 Open from June 2018 after a 3-year break, we a... 19.0 t Westminster 51.52098 -0.14002 t apartment ... 0 0 0 0 1 0 0 350.0 10.0 149.65 3 24328.jpg 41759 Artist house, bright high ceiling rooms, priva... 2.0 t Wandsworth 51.47298 -0.16376 t other ... 0 0 0 0 1 0 0 250.0 0.0 215.60 4 25023.jpg 102813 Large, all comforts, 2-bed flat; first floor; ... 1.0 f Wandsworth 51.44687 -0.21874 t apartment ... 0 0 0 0 1 0 0 250.0 11.0 79.35 <p>5 rows \u00d7 223 columns</p> In\u00a0[3]: Copied! <pre># There are a number of columns that are already binary. Therefore, no need to one hot encode them\ncrossed_cols = [(\"property_type\", \"room_type\")]\nalready_dummies = [c for c in df.columns if \"amenity\" in c] + [\"has_house_rules\"]\nwide_cols = [\n    \"is_location_exact\",\n    \"property_type\",\n    \"room_type\",\n    \"host_gender\",\n    \"instant_bookable\",\n] + already_dummies\n\ncat_embed_cols = [(c, 16) for c in df.columns if \"catg\" in c] + [\n    (\"neighbourhood_cleansed\", 64),\n    (\"cancellation_policy\", 16),\n]\ncontinuous_cols = [\"latitude\", \"longitude\", \"security_deposit\", \"extra_people\"]\n\n# text and image colnames\ntext_col = \"description\"\nimg_col = \"id\"\n\n# path to pretrained word embeddings and the images\nword_vectors_path = \"../tmp_data/glove.6B/glove.6B.100d.txt\"\nimg_path = \"../tmp_data/airbnb/property_picture\"\n\n# target\ntarget_col = \"yield\"\n</pre> # There are a number of columns that are already binary. Therefore, no need to one hot encode them crossed_cols = [(\"property_type\", \"room_type\")] already_dummies = [c for c in df.columns if \"amenity\" in c] + [\"has_house_rules\"] wide_cols = [     \"is_location_exact\",     \"property_type\",     \"room_type\",     \"host_gender\",     \"instant_bookable\", ] + already_dummies  cat_embed_cols = [(c, 16) for c in df.columns if \"catg\" in c] + [     (\"neighbourhood_cleansed\", 64),     (\"cancellation_policy\", 16), ] continuous_cols = [\"latitude\", \"longitude\", \"security_deposit\", \"extra_people\"]  # text and image colnames text_col = \"description\" img_col = \"id\"  # path to pretrained word embeddings and the images word_vectors_path = \"../tmp_data/glove.6B/glove.6B.100d.txt\" img_path = \"../tmp_data/airbnb/property_picture\"  # target target_col = \"yield\" In\u00a0[4]: Copied! <pre>target = df[target_col].values\n</pre> target = df[target_col].values In\u00a0[5]: Copied! <pre>wide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols)\nX_wide = wide_preprocessor.fit_transform(df)\n</pre> wide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols) X_wide = wide_preprocessor.fit_transform(df) In\u00a0[6]: Copied! <pre>tab_preprocessor = TabPreprocessor(\n    cat_embed_cols=cat_embed_cols,\n    continuous_cols=continuous_cols,\n)\nX_tab = tab_preprocessor.fit_transform(df)\n</pre> tab_preprocessor = TabPreprocessor(     cat_embed_cols=cat_embed_cols,     continuous_cols=continuous_cols, ) X_tab = tab_preprocessor.fit_transform(df) <pre>/Users/javierrodriguezzaurin/Projects/pytorch-widedeep/pytorch_widedeep/preprocessing/tab_preprocessor.py:358: UserWarning: Continuous columns will not be normalised\n  warnings.warn(\"Continuous columns will not be normalised\")\n</pre> In\u00a0[7]: Copied! <pre>text_preprocessor = TextPreprocessor(\n    word_vectors_path=word_vectors_path, text_col=text_col\n)\nX_text = text_preprocessor.fit_transform(df)\n</pre> text_preprocessor = TextPreprocessor(     word_vectors_path=word_vectors_path, text_col=text_col ) X_text = text_preprocessor.fit_transform(df) <pre>The vocabulary contains 2192 tokens\nIndexing word vectors...\nLoaded 400000 word vectors\nPreparing embeddings matrix...\n2175 words in the vocabulary had ../tmp_data/glove.6B/glove.6B.100d.txt vectors and appear more than 5 times\n</pre> In\u00a0[8]: Copied! <pre>image_processor = ImagePreprocessor(img_col=img_col, img_path=img_path)\nX_images = image_processor.fit_transform(df)\n</pre> image_processor = ImagePreprocessor(img_col=img_col, img_path=img_path) X_images = image_processor.fit_transform(df) <pre>Reading Images from ../tmp_data/airbnb/property_picture\nResizing\n</pre> <pre>100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 1001/1001 [00:01&lt;00:00, 638.00it/s]\n</pre> <pre>Computing normalisation metrics\n</pre> In\u00a0[9]: Copied! <pre># Linear model\nwide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1)\n\n# DeepDense: 2 Dense layers\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=continuous_cols,\n    mlp_hidden_dims=[128, 64],\n    mlp_dropout=0.1,\n)\n\n# DeepText: a stack of 2 LSTMs\nbasic_rnn = BasicRNN(\n    vocab_size=len(text_preprocessor.vocab.itos),\n    embed_matrix=text_preprocessor.embedding_matrix,\n    n_layers=2,\n    hidden_dim=64,\n    rnn_dropout=0.5,\n)\n\n# Pretrained Resnet 18\nresnet = Vision(pretrained_model_setup=\"resnet18\", n_trainable=4)\n</pre> # Linear model wide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1)  # DeepDense: 2 Dense layers tab_mlp = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     continuous_cols=continuous_cols,     mlp_hidden_dims=[128, 64],     mlp_dropout=0.1, )  # DeepText: a stack of 2 LSTMs basic_rnn = BasicRNN(     vocab_size=len(text_preprocessor.vocab.itos),     embed_matrix=text_preprocessor.embedding_matrix,     n_layers=2,     hidden_dim=64,     rnn_dropout=0.5, )  # Pretrained Resnet 18 resnet = Vision(pretrained_model_setup=\"resnet18\", n_trainable=4) <p>Combine them all with the \"collector\" class <code>WideDeep</code></p> In\u00a0[10]: Copied! <pre>model = WideDeep(\n    wide=wide,\n    deeptabular=tab_mlp,\n    deeptext=basic_rnn,\n    deepimage=resnet,\n    head_hidden_dims=[256, 128],\n)\n</pre> model = WideDeep(     wide=wide,     deeptabular=tab_mlp,     deeptext=basic_rnn,     deepimage=resnet,     head_hidden_dims=[256, 128], ) In\u00a0[11]: Copied! <pre>trainer = Trainer(model, objective=\"rmse\")\n</pre> trainer = Trainer(model, objective=\"rmse\") In\u00a0[12]: Copied! <pre>trainer.fit(\n    X_wide=X_wide,\n    X_tab=X_tab,\n    X_text=X_text,\n    X_img=X_images,\n    target=target,\n    n_epochs=1,\n    batch_size=32,\n    val_split=0.2,\n)\n</pre> trainer.fit(     X_wide=X_wide,     X_tab=X_tab,     X_text=X_text,     X_img=X_images,     target=target,     n_epochs=1,     batch_size=32,     val_split=0.2, ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 25/25 [00:19&lt;00:00,  1.28it/s, loss=115]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 7/7 [00:04&lt;00:00,  1.62it/s, loss=94.1]\n</pre> <p>Both, the Text and Image components allow FC-heads on their own (have a look to the documentation).</p> <p>Now let's go \"kaggle crazy\". Let's use different optimizers, initializers and schedulers for different components. Moreover, let's use a different learning rate for different parameter groups, for the <code>deeptabular</code> component</p> In\u00a0[13]: Copied! <pre>deep_params = []\nfor childname, child in model.named_children():\n    if childname == \"deeptabular\":\n        for n, p in child.named_parameters():\n            if \"embed_layer\" in n:\n                deep_params.append({\"params\": p, \"lr\": 1e-4})\n            else:\n                deep_params.append({\"params\": p, \"lr\": 1e-3})\n</pre> deep_params = [] for childname, child in model.named_children():     if childname == \"deeptabular\":         for n, p in child.named_parameters():             if \"embed_layer\" in n:                 deep_params.append({\"params\": p, \"lr\": 1e-4})             else:                 deep_params.append({\"params\": p, \"lr\": 1e-3}) In\u00a0[14]: Copied! <pre>wide_opt = torch.optim.Adam(model.wide.parameters(), lr=0.03)\ndeep_opt = torch.optim.Adam(deep_params)\ntext_opt = torch.optim.AdamW(model.deeptext.parameters())\nimg_opt = torch.optim.AdamW(model.deepimage.parameters())\nhead_opt = torch.optim.Adam(model.deephead.parameters())\n</pre> wide_opt = torch.optim.Adam(model.wide.parameters(), lr=0.03) deep_opt = torch.optim.Adam(deep_params) text_opt = torch.optim.AdamW(model.deeptext.parameters()) img_opt = torch.optim.AdamW(model.deepimage.parameters()) head_opt = torch.optim.Adam(model.deephead.parameters()) In\u00a0[15]: Copied! <pre>wide_sch = torch.optim.lr_scheduler.StepLR(wide_opt, step_size=5)\ndeep_sch = torch.optim.lr_scheduler.MultiStepLR(deep_opt, milestones=[3, 8])\ntext_sch = torch.optim.lr_scheduler.StepLR(text_opt, step_size=5)\nimg_sch = torch.optim.lr_scheduler.MultiStepLR(deep_opt, milestones=[3, 8])\nhead_sch = torch.optim.lr_scheduler.StepLR(head_opt, step_size=5)\n</pre> wide_sch = torch.optim.lr_scheduler.StepLR(wide_opt, step_size=5) deep_sch = torch.optim.lr_scheduler.MultiStepLR(deep_opt, milestones=[3, 8]) text_sch = torch.optim.lr_scheduler.StepLR(text_opt, step_size=5) img_sch = torch.optim.lr_scheduler.MultiStepLR(deep_opt, milestones=[3, 8]) head_sch = torch.optim.lr_scheduler.StepLR(head_opt, step_size=5) In\u00a0[16]: Copied! <pre># remember, one optimizer per model components, for lr_schedures and initializers is not neccesary\noptimizers = {\n    \"wide\": wide_opt,\n    \"deeptabular\": deep_opt,\n    \"deeptext\": text_opt,\n    \"deepimage\": img_opt,\n    \"deephead\": head_opt,\n}\nschedulers = {\n    \"wide\": wide_sch,\n    \"deeptabular\": deep_sch,\n    \"deeptext\": text_sch,\n    \"deepimage\": img_sch,\n    \"deephead\": head_sch,\n}\n\n# Now...we have used pretrained word embeddings, so you do not want to\n# initialise these  embeddings. However you might still want to initialise the\n# other layers in the DeepText component. No probs, you can do that with the\n# parameter pattern and your knowledge on regular  expressions. Here we are\n# telling to the KaimingNormal initializer to NOT touch the  parameters whose\n# name contains the string word_embed.\ninitializers = {\n    \"wide\": KaimingNormal,\n    \"deeptabular\": KaimingNormal,\n    \"deeptext\": KaimingNormal(pattern=r\"^(?!.*word_embed).*$\"),\n    \"deepimage\": KaimingNormal,\n}\n\nmean = [0.406, 0.456, 0.485]  # BGR\nstd = [0.225, 0.224, 0.229]  # BGR\ntransforms = [ToTensor, Normalize(mean=mean, std=std)]\ncallbacks = [\n    LRHistory(n_epochs=10),\n    EarlyStopping,\n    ModelCheckpoint(filepath=\"model_weights/wd_out\"),\n]\n</pre> # remember, one optimizer per model components, for lr_schedures and initializers is not neccesary optimizers = {     \"wide\": wide_opt,     \"deeptabular\": deep_opt,     \"deeptext\": text_opt,     \"deepimage\": img_opt,     \"deephead\": head_opt, } schedulers = {     \"wide\": wide_sch,     \"deeptabular\": deep_sch,     \"deeptext\": text_sch,     \"deepimage\": img_sch,     \"deephead\": head_sch, }  # Now...we have used pretrained word embeddings, so you do not want to # initialise these  embeddings. However you might still want to initialise the # other layers in the DeepText component. No probs, you can do that with the # parameter pattern and your knowledge on regular  expressions. Here we are # telling to the KaimingNormal initializer to NOT touch the  parameters whose # name contains the string word_embed. initializers = {     \"wide\": KaimingNormal,     \"deeptabular\": KaimingNormal,     \"deeptext\": KaimingNormal(pattern=r\"^(?!.*word_embed).*$\"),     \"deepimage\": KaimingNormal, }  mean = [0.406, 0.456, 0.485]  # BGR std = [0.225, 0.224, 0.229]  # BGR transforms = [ToTensor, Normalize(mean=mean, std=std)] callbacks = [     LRHistory(n_epochs=10),     EarlyStopping,     ModelCheckpoint(filepath=\"model_weights/wd_out\"), ] In\u00a0[17]: Copied! <pre>trainer = Trainer(\n    model,\n    objective=\"rmse\",\n    initializers=initializers,\n    optimizers=optimizers,\n    lr_schedulers=schedulers,\n    callbacks=callbacks,\n    transforms=transforms,\n)\n</pre> trainer = Trainer(     model,     objective=\"rmse\",     initializers=initializers,     optimizers=optimizers,     lr_schedulers=schedulers,     callbacks=callbacks,     transforms=transforms, ) <pre>/Users/javierrodriguezzaurin/Projects/pytorch-widedeep/pytorch_widedeep/initializers.py:34: UserWarning: No initializer found for deephead\n  warnings.warn(\n</pre> In\u00a0[18]: Copied! <pre>trainer.fit(\n    X_wide=X_wide,\n    X_tab=X_tab,\n    X_text=X_text,\n    X_img=X_images,\n    target=target,\n    n_epochs=1,\n    batch_size=32,\n    val_split=0.2,\n)\n</pre> trainer.fit(     X_wide=X_wide,     X_tab=X_tab,     X_text=X_text,     X_img=X_images,     target=target,     n_epochs=1,     batch_size=32,     val_split=0.2, ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 25/25 [00:19&lt;00:00,  1.25it/s, loss=101]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 7/7 [00:04&lt;00:00,  1.62it/s, loss=90.6]\n</pre> <pre>Model weights after training corresponds to the those of the final epoch which might not be the best performing weights. Use the 'ModelCheckpoint' Callback to restore the best epoch weights.\n</pre> <p>we have only run one epoch, but let's check that the LRHistory callback records the lr values for each group</p> In\u00a0[19]: Copied! <pre>trainer.lr_history\n</pre> trainer.lr_history Out[19]: <pre>{'lr_wide_0': [0.03, 0.03],\n 'lr_deeptabular_0': [0.0001, 0.0001],\n 'lr_deeptabular_1': [0.0001, 0.0001],\n 'lr_deeptabular_2': [0.0001, 0.0001],\n 'lr_deeptabular_3': [0.0001, 0.0001],\n 'lr_deeptabular_4': [0.0001, 0.0001],\n 'lr_deeptabular_5': [0.0001, 0.0001],\n 'lr_deeptabular_6': [0.0001, 0.0001],\n 'lr_deeptabular_7': [0.0001, 0.0001],\n 'lr_deeptabular_8': [0.0001, 0.0001],\n 'lr_deeptabular_9': [0.001, 0.001],\n 'lr_deeptabular_10': [0.001, 0.001],\n 'lr_deeptabular_11': [0.001, 0.001],\n 'lr_deeptabular_12': [0.001, 0.001],\n 'lr_deeptext_0': [0.001, 0.001],\n 'lr_deepimage_0': [0.001, 0.001],\n 'lr_deephead_0': [0.001, 0.001]}</pre>"},{"location":"examples/04_regression_with_images_and_text.html#regression-with-images-and-text","title":"Regression with Images and Text\u00b6","text":"<p>In this notebook we will go through a series of examples on how to combine all Wide &amp; Deep components.</p> <p>To that aim I will use the Airbnb listings dataset for London, which you can download from here. I use this dataset simply because it contains tabular data, images and text.</p> <p>I have taken a sample of 1000 listings to keep the data tractable in this notebook. Also, I have preprocessed the data and prepared it for this exercise. All preprocessing steps can be found in the notebook <code>airbnb_data_preprocessing.ipynb</code> in this <code>examples</code> folder.</p>"},{"location":"examples/04_regression_with_images_and_text.html#regression-with-the-defaults","title":"Regression with the defaults\u00b6","text":"<p>The set up</p>"},{"location":"examples/04_regression_with_images_and_text.html#prepare-the-data","title":"Prepare the data\u00b6","text":"<p>I will focus here on how to prepare the data and run the model. Check notebooks 1 and 2 to see what's going on behind the scences</p> <p>Preparing the data is rather simple</p>"},{"location":"examples/04_regression_with_images_and_text.html#build-the-model-components","title":"Build the model components\u00b6","text":""},{"location":"examples/04_regression_with_images_and_text.html#build-the-trainer-and-fit","title":"Build the trainer and fit\u00b6","text":""},{"location":"examples/05_save_and_load_model_and_artifacts.html","title":"05_save_and_load_model_and_artifacts","text":"In\u00a0[1]: Copied! <pre>import pickle\nimport numpy as np\nimport pandas as pd\nimport torch\nimport shutil\n\nfrom pytorch_widedeep.preprocessing import TabPreprocessor\nfrom pytorch_widedeep.training import Trainer\nfrom pytorch_widedeep.callbacks import EarlyStopping, ModelCheckpoint, LRHistory\nfrom pytorch_widedeep.models import TabMlp, WideDeep\nfrom pytorch_widedeep.metrics import Accuracy\nfrom pytorch_widedeep.datasets import load_adult\nfrom sklearn.model_selection import train_test_split\n</pre> import pickle import numpy as np import pandas as pd import torch import shutil  from pytorch_widedeep.preprocessing import TabPreprocessor from pytorch_widedeep.training import Trainer from pytorch_widedeep.callbacks import EarlyStopping, ModelCheckpoint, LRHistory from pytorch_widedeep.models import TabMlp, WideDeep from pytorch_widedeep.metrics import Accuracy from pytorch_widedeep.datasets import load_adult from sklearn.model_selection import train_test_split <pre>/Users/javierrodriguezzaurin/.pyenv/versions/3.10.13/envs/widedeep310/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n  from .autonotebook import tqdm as notebook_tqdm\n</pre> In\u00a0[2]: Copied! <pre>df = load_adult(as_frame=True)\ndf.head()\n</pre> df = load_adult(as_frame=True) df.head() Out[2]: age workclass fnlwgt education educational-num marital-status occupation relationship race gender capital-gain capital-loss hours-per-week native-country income 0 25 Private 226802 11th 7 Never-married Machine-op-inspct Own-child Black Male 0 0 40 United-States &lt;=50K 1 38 Private 89814 HS-grad 9 Married-civ-spouse Farming-fishing Husband White Male 0 0 50 United-States &lt;=50K 2 28 Local-gov 336951 Assoc-acdm 12 Married-civ-spouse Protective-serv Husband White Male 0 0 40 United-States &gt;50K 3 44 Private 160323 Some-college 10 Married-civ-spouse Machine-op-inspct Husband Black Male 7688 0 40 United-States &gt;50K 4 18 ? 103497 Some-college 10 Never-married ? Own-child White Female 0 0 30 United-States &lt;=50K In\u00a0[3]: Copied! <pre># For convenience, we'll replace '-' with '_'\ndf.columns = [c.replace(\"-\", \"_\") for c in df.columns]\n# binary target\ndf[\"target\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int)\ndf.drop(\"income\", axis=1, inplace=True)\ndf.head()\n</pre> # For convenience, we'll replace '-' with '_' df.columns = [c.replace(\"-\", \"_\") for c in df.columns] # binary target df[\"target\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int) df.drop(\"income\", axis=1, inplace=True) df.head() Out[3]: age workclass fnlwgt education educational_num marital_status occupation relationship race gender capital_gain capital_loss hours_per_week native_country target 0 25 Private 226802 11th 7 Never-married Machine-op-inspct Own-child Black Male 0 0 40 United-States 0 1 38 Private 89814 HS-grad 9 Married-civ-spouse Farming-fishing Husband White Male 0 0 50 United-States 0 2 28 Local-gov 336951 Assoc-acdm 12 Married-civ-spouse Protective-serv Husband White Male 0 0 40 United-States 1 3 44 Private 160323 Some-college 10 Married-civ-spouse Machine-op-inspct Husband Black Male 7688 0 40 United-States 1 4 18 ? 103497 Some-college 10 Never-married ? Own-child White Female 0 0 30 United-States 0 In\u00a0[4]: Copied! <pre>train, valid = train_test_split(df, test_size=0.2, stratify=df.target)\n# the test data will be used lately as if it was \"fresh\", new data coming after some time...\nvalid, test = train_test_split(valid, test_size=0.5, stratify=valid.target)\n</pre> train, valid = train_test_split(df, test_size=0.2, stratify=df.target) # the test data will be used lately as if it was \"fresh\", new data coming after some time... valid, test = train_test_split(valid, test_size=0.5, stratify=valid.target) In\u00a0[5]: Copied! <pre>print(f\"train shape: {train.shape}\")\nprint(f\"valid shape: {valid.shape}\")\nprint(f\"test shape: {test.shape}\")\n</pre> print(f\"train shape: {train.shape}\") print(f\"valid shape: {valid.shape}\") print(f\"test shape: {test.shape}\") <pre>train shape: (39073, 15)\nvalid shape: (4884, 15)\ntest shape: (4885, 15)\n</pre> In\u00a0[6]: Copied! <pre>cat_embed_cols = [\n    \"workclass\",\n    \"education\",\n    \"marital_status\",\n    \"occupation\",\n    \"relationship\",\n    \"race\",\n    \"gender\",\n    \"capital_gain\",\n    \"capital_loss\",\n    \"native_country\",\n]\ncontinuous_cols = [\"age\", \"hours_per_week\"]\n</pre> cat_embed_cols = [     \"workclass\",     \"education\",     \"marital_status\",     \"occupation\",     \"relationship\",     \"race\",     \"gender\",     \"capital_gain\",     \"capital_loss\",     \"native_country\", ] continuous_cols = [\"age\", \"hours_per_week\"] In\u00a0[7]: Copied! <pre>tab_preprocessor = TabPreprocessor(\n    embed_cols=cat_embed_cols,\n    continuous_cols=continuous_cols,\n)\nX_tab_train = tab_preprocessor.fit_transform(train)\ny_train = train.target.values\nX_tab_valid = tab_preprocessor.transform(valid)\ny_valid = valid.target.values\n</pre> tab_preprocessor = TabPreprocessor(     embed_cols=cat_embed_cols,     continuous_cols=continuous_cols, ) X_tab_train = tab_preprocessor.fit_transform(train) y_train = train.target.values X_tab_valid = tab_preprocessor.transform(valid) y_valid = valid.target.values <pre>/Users/javierrodriguezzaurin/Projects/pytorch-widedeep/pytorch_widedeep/preprocessing/tab_preprocessor.py:358: UserWarning: Continuous columns will not be normalised\n  warnings.warn(\"Continuous columns will not be normalised\")\n</pre> In\u00a0[8]: Copied! <pre>tab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    cat_embed_dropout=0.1,\n    continuous_cols=continuous_cols,\n    cont_norm_layer=\"layernorm\",\n    embed_continuous_method=\"standard\",\n    cont_embed_dim=8,\n    mlp_hidden_dims=[64, 32],\n    mlp_dropout=0.2,\n    mlp_activation=\"leaky_relu\",\n)\nmodel = WideDeep(deeptabular=tab_mlp)\n</pre> tab_mlp = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     cat_embed_dropout=0.1,     continuous_cols=continuous_cols,     cont_norm_layer=\"layernorm\",     embed_continuous_method=\"standard\",     cont_embed_dim=8,     mlp_hidden_dims=[64, 32],     mlp_dropout=0.2,     mlp_activation=\"leaky_relu\", ) model = WideDeep(deeptabular=tab_mlp) In\u00a0[9]: Copied! <pre>model\n</pre> model Out[9]: <pre>WideDeep(\n  (deeptabular): Sequential(\n    (0): TabMlp(\n      (cat_embed): DiffSizeCatEmbeddings(\n        (embed_layers): ModuleDict(\n          (emb_layer_workclass): Embedding(10, 5, padding_idx=0)\n          (emb_layer_education): Embedding(17, 8, padding_idx=0)\n          (emb_layer_marital_status): Embedding(8, 5, padding_idx=0)\n          (emb_layer_occupation): Embedding(16, 7, padding_idx=0)\n          (emb_layer_relationship): Embedding(7, 4, padding_idx=0)\n          (emb_layer_race): Embedding(6, 4, padding_idx=0)\n          (emb_layer_gender): Embedding(3, 2, padding_idx=0)\n          (emb_layer_capital_gain): Embedding(122, 23, padding_idx=0)\n          (emb_layer_capital_loss): Embedding(97, 21, padding_idx=0)\n          (emb_layer_native_country): Embedding(43, 13, padding_idx=0)\n        )\n        (embedding_dropout): Dropout(p=0.1, inplace=False)\n      )\n      (cont_norm): LayerNorm((2,), eps=1e-05, elementwise_affine=True)\n      (cont_embed): ContEmbeddings(\n        INFO: [ContLinear = weight(n_cont_cols, embed_dim) + bias(n_cont_cols, embed_dim)]\n        (linear): ContLinear(n_cont_cols=2, embed_dim=8, embed_dropout=0.0)\n        (dropout): Dropout(p=0.0, inplace=False)\n      )\n      (encoder): MLP(\n        (mlp): Sequential(\n          (dense_layer_0): Sequential(\n            (0): Linear(in_features=108, out_features=64, bias=True)\n            (1): LeakyReLU(negative_slope=0.01, inplace=True)\n            (2): Dropout(p=0.2, inplace=False)\n          )\n          (dense_layer_1): Sequential(\n            (0): Linear(in_features=64, out_features=32, bias=True)\n            (1): LeakyReLU(negative_slope=0.01, inplace=True)\n            (2): Dropout(p=0.2, inplace=False)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=32, out_features=1, bias=True)\n  )\n)</pre> In\u00a0[10]: Copied! <pre>early_stopping = EarlyStopping()\nmodel_checkpoint = ModelCheckpoint(\n    filepath=\"tmp_dir/adult_tabmlp_model\",\n    save_best_only=True,\n    verbose=1,\n    max_save=1,\n)\n\ntrainer = Trainer(\n    model,\n    objective=\"binary\",\n    callbacks=[early_stopping, model_checkpoint],\n    metrics=[Accuracy],\n)\n\ntrainer.fit(\n    X_train={\"X_tab\": X_tab_train, \"target\": y_train},\n    X_val={\"X_tab\": X_tab_valid, \"target\": y_valid},\n    n_epochs=4,\n    batch_size=256,\n)\n</pre> early_stopping = EarlyStopping() model_checkpoint = ModelCheckpoint(     filepath=\"tmp_dir/adult_tabmlp_model\",     save_best_only=True,     verbose=1,     max_save=1, )  trainer = Trainer(     model,     objective=\"binary\",     callbacks=[early_stopping, model_checkpoint],     metrics=[Accuracy], )  trainer.fit(     X_train={\"X_tab\": X_tab_train, \"target\": y_train},     X_val={\"X_tab\": X_tab_valid, \"target\": y_valid},     n_epochs=4,     batch_size=256, ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:02&lt;00:00, 76.25it/s, loss=0.452, metrics={'acc': 0.7867}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 20/20 [00:00&lt;00:00, 125.36it/s, loss=0.335, metrics={'acc': 0.8532}]\n</pre> <pre>\nEpoch 1: val_loss improved from inf to 0.33532 Saving model to tmp_dir/adult_tabmlp_model_1.p\n</pre> <pre>epoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 76.98it/s, loss=0.355, metrics={'acc': 0.8401}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 20/20 [00:00&lt;00:00, 106.51it/s, loss=0.303, metrics={'acc': 0.8665}]\n</pre> <pre>\nEpoch 2: val_loss improved from 0.33532 to 0.30273 Saving model to tmp_dir/adult_tabmlp_model_2.p\n</pre> <pre>epoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 82.71it/s, loss=0.332, metrics={'acc': 0.849}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 20/20 [00:00&lt;00:00, 107.80it/s, loss=0.288, metrics={'acc': 0.8757}]\n</pre> <pre>\nEpoch 3: val_loss improved from 0.30273 to 0.28791 Saving model to tmp_dir/adult_tabmlp_model_3.p\n</pre> <pre>epoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 79.02it/s, loss=0.32, metrics={'acc': 0.8541}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 20/20 [00:00&lt;00:00, 127.07it/s, loss=0.282, metrics={'acc': 0.8763}]</pre> <pre>\nEpoch 4: val_loss improved from 0.28791 to 0.28238 Saving model to tmp_dir/adult_tabmlp_model_4.p\nModel weights restored to best epoch: 4\n</pre> <pre>\n</pre> In\u00a0[11]: Copied! <pre>torch.save(model, \"tmp_dir/model_saved_option_1.pt\")\n</pre> torch.save(model, \"tmp_dir/model_saved_option_1.pt\") In\u00a0[12]: Copied! <pre>torch.save(model.state_dict(), \"tmp_dir/model_state_dict_saved_option_1.pt\")\n</pre> torch.save(model.state_dict(), \"tmp_dir/model_state_dict_saved_option_1.pt\") In\u00a0[13]: Copied! <pre>trainer.save(path=\"tmp_dir/\", model_filename=\"model_saved_option_2.pt\")\n</pre> trainer.save(path=\"tmp_dir/\", model_filename=\"model_saved_option_2.pt\") <p>or the state dict</p> In\u00a0[14]: Copied! <pre>trainer.save(\n    path=\"tmp_dir/\",\n    model_filename=\"model_state_dict_saved_option_2.pt\",\n    save_state_dict=True,\n)\n</pre> trainer.save(     path=\"tmp_dir/\",     model_filename=\"model_state_dict_saved_option_2.pt\",     save_state_dict=True, ) In\u00a0[15]: Copied! <pre>%%bash\n\nls tmp_dir/\n</pre> %%bash  ls tmp_dir/ <pre>adult_tabmlp_model_4.p\nhistory\nmodel_saved_option_1.pt\nmodel_saved_option_2.pt\nmodel_state_dict_saved_option_1.pt\nmodel_state_dict_saved_option_2.pt\n</pre> In\u00a0[16]: Copied! <pre>%%bash\n\nls tmp_dir/history/\n</pre> %%bash  ls tmp_dir/history/ <pre>train_eval_history.json\n</pre> <p>Note that since we have used the <code>ModelCheckpoint</code> Callback, <code>adult_tabmlp_model_2.p</code> is the model state dict of the model at epoch 2, i.e. same as <code>model_state_dict_saved_option_1.p</code> or <code>model_state_dict_saved_option_2.p</code>.</p> In\u00a0[17]: Copied! <pre>with open(\"tmp_dir/tab_preproc.pkl\", \"wb\") as dp:\n    pickle.dump(tab_preprocessor, dp)\n</pre> with open(\"tmp_dir/tab_preproc.pkl\", \"wb\") as dp:     pickle.dump(tab_preprocessor, dp) In\u00a0[18]: Copied! <pre>with open(\"tmp_dir/eary_stop.pkl\", \"wb\") as es:\n    pickle.dump(early_stopping, es)\n</pre> with open(\"tmp_dir/eary_stop.pkl\", \"wb\") as es:     pickle.dump(early_stopping, es) In\u00a0[19]: Copied! <pre>%%bash\n\nls tmp_dir/\n</pre> %%bash  ls tmp_dir/ <pre>adult_tabmlp_model_4.p\neary_stop.pkl\nhistory\nmodel_saved_option_1.pt\nmodel_saved_option_2.pt\nmodel_state_dict_saved_option_1.pt\nmodel_state_dict_saved_option_2.pt\ntab_preproc.pkl\n</pre> <p>And that is pretty much all you need to resume training or directly predict, let's see</p> In\u00a0[20]: Copied! <pre>test.head()\n</pre> test.head() Out[20]: age workclass fnlwgt education educational_num marital_status occupation relationship race gender capital_gain capital_loss hours_per_week native_country target 10103 43 Private 198282 HS-grad 9 Married-civ-spouse Craft-repair Husband White Male 0 0 40 United-States 1 31799 20 Private 228686 11th 7 Married-civ-spouse Other-service Husband White Male 0 0 40 United-States 0 19971 26 Private 291968 HS-grad 9 Married-civ-spouse Transport-moving Husband White Male 0 0 44 United-States 0 3039 48 Private 175958 Bachelors 13 Divorced Prof-specialty Not-in-family White Male 0 0 30 United-States 0 20725 18 Private 232024 11th 7 Never-married Machine-op-inspct Own-child White Male 0 0 55 United-States 0 In\u00a0[21]: Copied! <pre>with open(\"tmp_dir/tab_preproc.pkl\", \"rb\") as tp:\n    tab_preprocessor_new = pickle.load(tp)\n</pre> with open(\"tmp_dir/tab_preproc.pkl\", \"rb\") as tp:     tab_preprocessor_new = pickle.load(tp) In\u00a0[22]: Copied! <pre>X_test_tab = tab_preprocessor_new.transform(test)\ny_test = test.target\n</pre> X_test_tab = tab_preprocessor_new.transform(test) y_test = test.target In\u00a0[23]: Copied! <pre>tab_mlp_new = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    cat_embed_dropout=0.1,\n    continuous_cols=continuous_cols,\n    cont_norm_layer=\"layernorm\",\n    embed_continuous_method=\"standard\",\n    cont_embed_dim=8,\n    mlp_hidden_dims=[64, 32],\n    mlp_dropout=0.2,\n    mlp_activation=\"leaky_relu\",\n)\nnew_model = WideDeep(deeptabular=tab_mlp)\n</pre> tab_mlp_new = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     cat_embed_dropout=0.1,     continuous_cols=continuous_cols,     cont_norm_layer=\"layernorm\",     embed_continuous_method=\"standard\",     cont_embed_dim=8,     mlp_hidden_dims=[64, 32],     mlp_dropout=0.2,     mlp_activation=\"leaky_relu\", ) new_model = WideDeep(deeptabular=tab_mlp) In\u00a0[24]: Copied! <pre>new_model.load_state_dict(torch.load(\"tmp_dir/model_state_dict_saved_option_2.pt\"))\n</pre> new_model.load_state_dict(torch.load(\"tmp_dir/model_state_dict_saved_option_2.pt\")) Out[24]: <pre>&lt;All keys matched successfully&gt;</pre> In\u00a0[25]: Copied! <pre>trainer = Trainer(\n    model,\n    objective=\"binary\",\n)\n</pre> trainer = Trainer(     model,     objective=\"binary\", ) In\u00a0[26]: Copied! <pre>preds = trainer.predict(X_tab=X_test_tab, batch_size=32)\n</pre> preds = trainer.predict(X_tab=X_test_tab, batch_size=32) <pre>predict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:00&lt;00:00, 309.83it/s]\n</pre> In\u00a0[27]: Copied! <pre>from sklearn.metrics import accuracy_score\n</pre> from sklearn.metrics import accuracy_score In\u00a0[28]: Copied! <pre>accuracy_score(y_test, preds)\n</pre> accuracy_score(y_test, preds) Out[28]: <pre>0.8595701125895598</pre> In\u00a0[29]: Copied! <pre>shutil.rmtree(\"tmp_dir/\")\n</pre> shutil.rmtree(\"tmp_dir/\")"},{"location":"examples/05_save_and_load_model_and_artifacts.html#save-and-load-model-and-artifacts","title":"Save and load model and artifacts\u00b6","text":"<p>In this notebook I will show the different options to save and load a model, as well as some additional objects produced during training.</p> <p>On a given day, you train a model...</p>"},{"location":"examples/05_save_and_load_model_and_artifacts.html#save-model-option-1","title":"Save model: option 1\u00b6","text":"<p>save (and load) a model as you woud do with any other torch model</p>"},{"location":"examples/05_save_and_load_model_and_artifacts.html#save-model-option-2","title":"Save model: option 2\u00b6","text":"<p>use the <code>trainer</code>. The <code>trainer</code> will also save the training history and the learning rate history (if learning rate schedulers are used)</p>"},{"location":"examples/05_save_and_load_model_and_artifacts.html#save-preprocessors-and-callbacks","title":"Save preprocessors and callbacks\u00b6","text":"<p>...just pickle them</p>"},{"location":"examples/05_save_and_load_model_and_artifacts.html#run-new-experiment-prepare-new-dataset-load-model-and-predict","title":"Run New experiment: prepare new dataset, load model, and predict\u00b6","text":""},{"location":"examples/06_finetune_and_warmup.html","title":"06_finetune_and_warmup","text":"In\u00a0[1]: Copied! <pre>import numpy as np\nimport pandas as pd\nimport torch\n\nfrom pytorch_widedeep import Trainer\nfrom pytorch_widedeep.preprocessing import WidePreprocessor, TabPreprocessor\nfrom pytorch_widedeep.models import Wide, TabMlp, TabResnet, WideDeep\nfrom pytorch_widedeep.metrics import Accuracy\nfrom pytorch_widedeep.datasets import load_adult\n</pre> import numpy as np import pandas as pd import torch  from pytorch_widedeep import Trainer from pytorch_widedeep.preprocessing import WidePreprocessor, TabPreprocessor from pytorch_widedeep.models import Wide, TabMlp, TabResnet, WideDeep from pytorch_widedeep.metrics import Accuracy from pytorch_widedeep.datasets import load_adult <pre>/Users/javierrodriguezzaurin/.pyenv/versions/3.10.13/envs/widedeep310/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n  from .autonotebook import tqdm as notebook_tqdm\n</pre> In\u00a0[2]: Copied! <pre>df = load_adult(as_frame=True)\n# For convenience, we'll replace '-' with '_'\ndf.columns = [c.replace(\"-\", \"_\") for c in df.columns]\n# binary target\ndf[\"income_label\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int)\ndf.drop(\"income\", axis=1, inplace=True)\ndf.head()\n</pre> df = load_adult(as_frame=True) # For convenience, we'll replace '-' with '_' df.columns = [c.replace(\"-\", \"_\") for c in df.columns] # binary target df[\"income_label\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int) df.drop(\"income\", axis=1, inplace=True) df.head() Out[2]: age workclass fnlwgt education educational_num marital_status occupation relationship race gender capital_gain capital_loss hours_per_week native_country income_label 0 25 Private 226802 11th 7 Never-married Machine-op-inspct Own-child Black Male 0 0 40 United-States 0 1 38 Private 89814 HS-grad 9 Married-civ-spouse Farming-fishing Husband White Male 0 0 50 United-States 0 2 28 Local-gov 336951 Assoc-acdm 12 Married-civ-spouse Protective-serv Husband White Male 0 0 40 United-States 1 3 44 Private 160323 Some-college 10 Married-civ-spouse Machine-op-inspct Husband Black Male 7688 0 40 United-States 1 4 18 ? 103497 Some-college 10 Never-married ? Own-child White Female 0 0 30 United-States 0 In\u00a0[3]: Copied! <pre># Define wide, crossed and deep tabular columns\nwide_cols = [\n    \"workclass\",\n    \"education\",\n    \"marital_status\",\n    \"occupation\",\n    \"relationship\",\n    \"race\",\n    \"gender\",\n    \"native_country\",\n]\ncrossed_cols = [(\"education\", \"occupation\"), (\"native_country\", \"occupation\")]\ncat_embed_cols = [\n    \"workclass\",\n    \"education\",\n    \"marital_status\",\n    \"occupation\",\n    \"relationship\",\n    \"race\",\n    \"gender\",\n    \"capital_gain\",\n    \"capital_loss\",\n    \"native_country\",\n]\ncontinuous_cols = [\"age\", \"hours_per_week\"]\ntarget_col = \"income_label\"\ntarget = df[target_col].values\n</pre> # Define wide, crossed and deep tabular columns wide_cols = [     \"workclass\",     \"education\",     \"marital_status\",     \"occupation\",     \"relationship\",     \"race\",     \"gender\",     \"native_country\", ] crossed_cols = [(\"education\", \"occupation\"), (\"native_country\", \"occupation\")] cat_embed_cols = [     \"workclass\",     \"education\",     \"marital_status\",     \"occupation\",     \"relationship\",     \"race\",     \"gender\",     \"capital_gain\",     \"capital_loss\",     \"native_country\", ] continuous_cols = [\"age\", \"hours_per_week\"] target_col = \"income_label\" target = df[target_col].values In\u00a0[4]: Copied! <pre># TARGET\ntarget = df[target_col].values\n\n# WIDE\nwide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols)\nX_wide = wide_preprocessor.fit_transform(df)\n\n# DEEP\ntab_preprocessor = TabPreprocessor(\n    cat_embed_cols=cat_embed_cols, continuous_cols=continuous_cols\n)\nX_tab = tab_preprocessor.fit_transform(df)\n</pre> # TARGET target = df[target_col].values  # WIDE wide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols) X_wide = wide_preprocessor.fit_transform(df)  # DEEP tab_preprocessor = TabPreprocessor(     cat_embed_cols=cat_embed_cols, continuous_cols=continuous_cols ) X_tab = tab_preprocessor.fit_transform(df) <pre>/Users/javierrodriguezzaurin/Projects/pytorch-widedeep/pytorch_widedeep/preprocessing/tab_preprocessor.py:358: UserWarning: Continuous columns will not be normalised\n  warnings.warn(\"Continuous columns will not be normalised\")\n</pre> In\u00a0[5]: Copied! <pre>wide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1)\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    cat_embed_dropout=0.1,\n    continuous_cols=continuous_cols,\n    embed_continuous_method=\"standard\",\n    cont_embed_dim=8,\n    mlp_hidden_dims=[64, 32],\n    mlp_dropout=0.2,\n    mlp_activation=\"leaky_relu\",\n)\nmodel = WideDeep(wide=wide, deeptabular=tab_mlp)\n</pre> wide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1) tab_mlp = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     cat_embed_dropout=0.1,     continuous_cols=continuous_cols,     embed_continuous_method=\"standard\",     cont_embed_dim=8,     mlp_hidden_dims=[64, 32],     mlp_dropout=0.2,     mlp_activation=\"leaky_relu\", ) model = WideDeep(wide=wide, deeptabular=tab_mlp) In\u00a0[6]: Copied! <pre>trainer = Trainer(\n    model,\n    objective=\"binary\",\n    optimizers=torch.optim.Adam(model.parameters(), lr=0.01),\n    metrics=[Accuracy],\n)\n</pre> trainer = Trainer(     model,     objective=\"binary\",     optimizers=torch.optim.Adam(model.parameters(), lr=0.01),     metrics=[Accuracy], ) In\u00a0[7]: Copied! <pre>trainer.fit(\n    X_wide=X_wide, X_tab=X_tab, target=target, n_epochs=2, val_split=0.2, batch_size=256\n)\n</pre> trainer.fit(     X_wide=X_wide, X_tab=X_tab, target=target, n_epochs=2, val_split=0.2, batch_size=256 ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:02&lt;00:00, 74.26it/s, loss=0.399, metrics={'acc': 0.8163}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 39/39 [00:00&lt;00:00, 91.03it/s, loss=0.296, metrics={'acc': 0.8677}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 81.31it/s, loss=0.3, metrics={'acc': 0.8614}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 39/39 [00:00&lt;00:00, 106.45it/s, loss=0.285, metrics={'acc': 0.8721}]\n</pre> In\u00a0[8]: Copied! <pre>trainer.save(path=\"models_dir/\", save_state_dict=True, model_filename=\"model_1.pt\")\n</pre> trainer.save(path=\"models_dir/\", save_state_dict=True, model_filename=\"model_1.pt\") <p>Now time goes by...and we want to fine-tune the model to another, new dataset (for example, a dataset that is identical to the one you used to train the previous model but for another country).</p> <p>Here I will use the same dataset just for illustration purposes, but the flow would be identical to that new dataset</p> In\u00a0[9]: Copied! <pre>wide_1 = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1)\ntab_mlp_1 = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    cat_embed_dropout=0.1,\n    continuous_cols=continuous_cols,\n    embed_continuous_method=\"standard\",\n    cont_embed_dim=8,\n    mlp_hidden_dims=[64, 32],\n    mlp_dropout=0.2,\n    mlp_activation=\"leaky_relu\",\n)\nmodel_1 = WideDeep(wide=wide, deeptabular=tab_mlp)\n</pre> wide_1 = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1) tab_mlp_1 = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     cat_embed_dropout=0.1,     continuous_cols=continuous_cols,     embed_continuous_method=\"standard\",     cont_embed_dim=8,     mlp_hidden_dims=[64, 32],     mlp_dropout=0.2,     mlp_activation=\"leaky_relu\", ) model_1 = WideDeep(wide=wide, deeptabular=tab_mlp) In\u00a0[10]: Copied! <pre>model_1.load_state_dict(torch.load(\"models_dir/model_1.pt\"))\n</pre> model_1.load_state_dict(torch.load(\"models_dir/model_1.pt\")) Out[10]: <pre>&lt;All keys matched successfully&gt;</pre> In\u00a0[11]: Copied! <pre>trainer_1 = Trainer(model_1, objective=\"binary\", metrics=[Accuracy])\n</pre> trainer_1 = Trainer(model_1, objective=\"binary\", metrics=[Accuracy]) In\u00a0[12]: Copied! <pre>trainer_1.fit(\n    X_wide=X_wide,\n    X_tab=X_tab,\n    target=target,\n    n_epochs=2,\n    batch_size=256,\n    finetune=True,\n    finetune_epochs=2,\n)\n</pre> trainer_1.fit(     X_wide=X_wide,     X_tab=X_tab,     target=target,     n_epochs=2,     batch_size=256,     finetune=True,     finetune_epochs=2, ) <pre>Training wide for 2 epochs\n</pre> <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 191/191 [00:01&lt;00:00, 97.37it/s, loss=0.39, metrics={'acc': 0.8152}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 191/191 [00:01&lt;00:00, 104.04it/s, loss=0.359, metrics={'acc': 0.824}]\n</pre> <pre>Training deeptabular for 2 epochs\n</pre> <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 191/191 [00:02&lt;00:00, 83.83it/s, loss=0.297, metrics={'acc': 0.8365}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 191/191 [00:02&lt;00:00, 82.78it/s, loss=0.283, metrics={'acc': 0.8445}]\n</pre> <pre>Fine-tuning (or warmup) of individual components completed. Training the whole model for 2 epochs\n</pre> <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 191/191 [00:02&lt;00:00, 72.84it/s, loss=0.281, metrics={'acc': 0.8716}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 191/191 [00:02&lt;00:00, 77.46it/s, loss=0.273, metrics={'acc': 0.8744}]\n</pre> <p>Note that, as I describe above, in scenario 2, we can just use this to warm up models before they joined training begins:</p> In\u00a0[13]: Copied! <pre>wide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1)\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    cat_embed_dropout=0.1,\n    continuous_cols=continuous_cols,\n    embed_continuous_method=\"standard\",\n    cont_embed_dim=8,\n    mlp_hidden_dims=[64, 32],\n    mlp_dropout=0.2,\n    mlp_activation=\"leaky_relu\",\n)\nmodel = WideDeep(wide=wide, deeptabular=tab_mlp)\n</pre> wide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1) tab_mlp = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     cat_embed_dropout=0.1,     continuous_cols=continuous_cols,     embed_continuous_method=\"standard\",     cont_embed_dim=8,     mlp_hidden_dims=[64, 32],     mlp_dropout=0.2,     mlp_activation=\"leaky_relu\", ) model = WideDeep(wide=wide, deeptabular=tab_mlp) In\u00a0[14]: Copied! <pre>trainer_2 = Trainer(model, objective=\"binary\", metrics=[Accuracy])\n</pre> trainer_2 = Trainer(model, objective=\"binary\", metrics=[Accuracy]) In\u00a0[15]: Copied! <pre>trainer_2.fit(\n    X_wide=X_wide,\n    X_tab=X_tab,\n    target=target,\n    val_split=0.1,\n    warmup=True,\n    warmup_epochs=2,\n    n_epochs=2,\n    batch_size=256,\n)\n</pre> trainer_2.fit(     X_wide=X_wide,     X_tab=X_tab,     target=target,     val_split=0.1,     warmup=True,     warmup_epochs=2,     n_epochs=2,     batch_size=256, ) <pre>Training wide for 2 epochs\n</pre> <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:01&lt;00:00, 102.49it/s, loss=0.52, metrics={'acc': 0.7519}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:01&lt;00:00, 98.15it/s, loss=0.381, metrics={'acc': 0.7891}]\n</pre> <pre>Training deeptabular for 2 epochs\n</pre> <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:02&lt;00:00, 82.97it/s, loss=0.356, metrics={'acc': 0.8043}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:02&lt;00:00, 80.27it/s, loss=0.295, metrics={'acc': 0.8195}]\n</pre> <pre>Fine-tuning (or warmup) of individual components completed. Training the whole model for 2 epochs\n</pre> <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:02&lt;00:00, 77.27it/s, loss=0.291, metrics={'acc': 0.8667}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 20/20 [00:00&lt;00:00, 89.57it/s, loss=0.289, metrics={'acc': 0.8665}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:02&lt;00:00, 72.69it/s, loss=0.283, metrics={'acc': 0.8693}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 20/20 [00:00&lt;00:00, 91.81it/s, loss=0.284, metrics={'acc': 0.869}]\n</pre> <p>We need to explicitly indicate</p> <ol> <li><p>That we want fine-tune</p> </li> <li><p>The components that we want to individually fine-tune</p> </li> <li><p>In case of gradual fine-tuning, the routine (\"felbo\" or \"howard\")</p> </li> <li><p>The layers we want to fine-tune.</p> </li> </ol> <p>For example</p> In\u00a0[16]: Copied! <pre>wide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1)\ntab_resnet = TabResnet(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    cat_embed_dropout=0.1,\n    continuous_cols=continuous_cols,\n    blocks_dims=[200, 200, 200],\n)\nmodel = WideDeep(wide=wide, deeptabular=tab_resnet)\n</pre> wide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1) tab_resnet = TabResnet(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     cat_embed_dropout=0.1,     continuous_cols=continuous_cols,     blocks_dims=[200, 200, 200], ) model = WideDeep(wide=wide, deeptabular=tab_resnet) In\u00a0[17]: Copied! <pre>model\n</pre> model Out[17]: <pre>WideDeep(\n  (wide): Wide(\n    (wide_linear): Embedding(809, 1, padding_idx=0)\n  )\n  (deeptabular): Sequential(\n    (0): TabResnet(\n      (cat_embed): DiffSizeCatEmbeddings(\n        (embed_layers): ModuleDict(\n          (emb_layer_workclass): Embedding(10, 5, padding_idx=0)\n          (emb_layer_education): Embedding(17, 8, padding_idx=0)\n          (emb_layer_marital_status): Embedding(8, 5, padding_idx=0)\n          (emb_layer_occupation): Embedding(16, 7, padding_idx=0)\n          (emb_layer_relationship): Embedding(7, 4, padding_idx=0)\n          (emb_layer_race): Embedding(6, 4, padding_idx=0)\n          (emb_layer_gender): Embedding(3, 2, padding_idx=0)\n          (emb_layer_capital_gain): Embedding(124, 24, padding_idx=0)\n          (emb_layer_capital_loss): Embedding(100, 21, padding_idx=0)\n          (emb_layer_native_country): Embedding(43, 13, padding_idx=0)\n        )\n        (embedding_dropout): Dropout(p=0.1, inplace=False)\n      )\n      (cont_norm): Identity()\n      (encoder): DenseResnet(\n        (dense_resnet): Sequential(\n          (lin_inp): Linear(in_features=95, out_features=200, bias=False)\n          (bn_inp): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n          (block_0): BasicBlock(\n            (lin1): Linear(in_features=200, out_features=200, bias=False)\n            (bn1): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n            (leaky_relu): LeakyReLU(negative_slope=0.01, inplace=True)\n            (dp): Dropout(p=0.1, inplace=False)\n            (lin2): Linear(in_features=200, out_features=200, bias=False)\n            (bn2): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n          )\n          (block_1): BasicBlock(\n            (lin1): Linear(in_features=200, out_features=200, bias=False)\n            (bn1): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n            (leaky_relu): LeakyReLU(negative_slope=0.01, inplace=True)\n            (dp): Dropout(p=0.1, inplace=False)\n            (lin2): Linear(in_features=200, out_features=200, bias=False)\n            (bn2): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=200, out_features=1, bias=True)\n  )\n)</pre> <p>let's first train as usual</p> In\u00a0[18]: Copied! <pre>trainer_3 = Trainer(model, objective=\"binary\", metrics=[Accuracy])\n</pre> trainer_3 = Trainer(model, objective=\"binary\", metrics=[Accuracy]) In\u00a0[19]: Copied! <pre>trainer_3.fit(\n    X_wide=X_wide, X_tab=X_tab, target=target, val_split=0.1, n_epochs=2, batch_size=256\n)\n</pre> trainer_3.fit(     X_wide=X_wide, X_tab=X_tab, target=target, val_split=0.1, n_epochs=2, batch_size=256 ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:03&lt;00:00, 54.23it/s, loss=0.382, metrics={'acc': 0.8239}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 20/20 [00:00&lt;00:00, 84.72it/s, loss=0.331, metrics={'acc': 0.8526}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:03&lt;00:00, 54.35it/s, loss=0.33, metrics={'acc': 0.8465}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 20/20 [00:00&lt;00:00, 68.15it/s, loss=0.312, metrics={'acc': 0.8604}]\n</pre> In\u00a0[20]: Copied! <pre>trainer_3.save(path=\"models_dir\", save_state_dict=True, model_filename=\"model_3.pt\")\n</pre> trainer_3.save(path=\"models_dir\", save_state_dict=True, model_filename=\"model_3.pt\") <p>Now we are going to fine-tune the model components, and in the case of the <code>deeptabular</code> component, we will fine-tune the resnet-blocks and the linear layer but NOT the embeddings.</p> <p>For this, we need to access the model component's children: <code>deeptabular</code> $\\rightarrow$ <code>tab_resnet</code> $\\rightarrow$ <code>dense_resnet</code> $\\rightarrow$ <code>blocks</code></p> In\u00a0[21]: Copied! <pre>wide_3 = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1)\ntab_resnet_3 = TabResnet(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    cat_embed_dropout=0.1,\n    continuous_cols=continuous_cols,\n    blocks_dims=[200, 200, 200],\n)\nmodel_3 = WideDeep(wide=wide, deeptabular=tab_resnet)\n</pre> wide_3 = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1) tab_resnet_3 = TabResnet(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     cat_embed_dropout=0.1,     continuous_cols=continuous_cols,     blocks_dims=[200, 200, 200], ) model_3 = WideDeep(wide=wide, deeptabular=tab_resnet) In\u00a0[22]: Copied! <pre>model_3.load_state_dict(torch.load(\"models_dir/model_3.pt\"))\n</pre> model_3.load_state_dict(torch.load(\"models_dir/model_3.pt\")) Out[22]: <pre>&lt;All keys matched successfully&gt;</pre> In\u00a0[23]: Copied! <pre>model_3\n</pre> model_3 Out[23]: <pre>WideDeep(\n  (wide): Wide(\n    (wide_linear): Embedding(809, 1, padding_idx=0)\n  )\n  (deeptabular): Sequential(\n    (0): TabResnet(\n      (cat_embed): DiffSizeCatEmbeddings(\n        (embed_layers): ModuleDict(\n          (emb_layer_workclass): Embedding(10, 5, padding_idx=0)\n          (emb_layer_education): Embedding(17, 8, padding_idx=0)\n          (emb_layer_marital_status): Embedding(8, 5, padding_idx=0)\n          (emb_layer_occupation): Embedding(16, 7, padding_idx=0)\n          (emb_layer_relationship): Embedding(7, 4, padding_idx=0)\n          (emb_layer_race): Embedding(6, 4, padding_idx=0)\n          (emb_layer_gender): Embedding(3, 2, padding_idx=0)\n          (emb_layer_capital_gain): Embedding(124, 24, padding_idx=0)\n          (emb_layer_capital_loss): Embedding(100, 21, padding_idx=0)\n          (emb_layer_native_country): Embedding(43, 13, padding_idx=0)\n        )\n        (embedding_dropout): Dropout(p=0.1, inplace=False)\n      )\n      (cont_norm): Identity()\n      (encoder): DenseResnet(\n        (dense_resnet): Sequential(\n          (lin_inp): Linear(in_features=95, out_features=200, bias=False)\n          (bn_inp): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n          (block_0): BasicBlock(\n            (lin1): Linear(in_features=200, out_features=200, bias=False)\n            (bn1): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n            (leaky_relu): LeakyReLU(negative_slope=0.01, inplace=True)\n            (dp): Dropout(p=0.1, inplace=False)\n            (lin2): Linear(in_features=200, out_features=200, bias=False)\n            (bn2): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n          )\n          (block_1): BasicBlock(\n            (lin1): Linear(in_features=200, out_features=200, bias=False)\n            (bn1): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n            (leaky_relu): LeakyReLU(negative_slope=0.01, inplace=True)\n            (dp): Dropout(p=0.1, inplace=False)\n            (lin2): Linear(in_features=200, out_features=200, bias=False)\n            (bn2): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=200, out_features=1, bias=True)\n  )\n)</pre> In\u00a0[24]: Copied! <pre>tab_lin_layer = list(model_3.deeptabular.children())[1]\n</pre> tab_lin_layer = list(model_3.deeptabular.children())[1] In\u00a0[25]: Copied! <pre>tab_lin_layer\n</pre> tab_lin_layer Out[25]: <pre>Linear(in_features=200, out_features=1, bias=True)</pre> In\u00a0[26]: Copied! <pre>tab_deep_layers = []\nfor n1, c1 in model_3.deeptabular.named_children():\n    if (\n        n1 == \"0\"\n    ):  # 0 is the model component and 1 is always the prediction layer added by the `WideDeep` class\n        for n2, c2 in c1.named_children():\n            if n2 == \"encoder\":  # TabResnet\n                for _, c3 in c2.named_children():\n                    for n4, c4 in c3.named_children():  # dense_resnet\n                        if \"block\" in n4:\n                            tab_deep_layers.append((n4, c4))\n</pre> tab_deep_layers = [] for n1, c1 in model_3.deeptabular.named_children():     if (         n1 == \"0\"     ):  # 0 is the model component and 1 is always the prediction layer added by the `WideDeep` class         for n2, c2 in c1.named_children():             if n2 == \"encoder\":  # TabResnet                 for _, c3 in c2.named_children():                     for n4, c4 in c3.named_children():  # dense_resnet                         if \"block\" in n4:                             tab_deep_layers.append((n4, c4)) In\u00a0[27]: Copied! <pre>tab_deep_layers\n</pre> tab_deep_layers Out[27]: <pre>[('block_0',\n  BasicBlock(\n    (lin1): Linear(in_features=200, out_features=200, bias=False)\n    (bn1): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n    (leaky_relu): LeakyReLU(negative_slope=0.01, inplace=True)\n    (dp): Dropout(p=0.1, inplace=False)\n    (lin2): Linear(in_features=200, out_features=200, bias=False)\n    (bn2): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n  )),\n ('block_1',\n  BasicBlock(\n    (lin1): Linear(in_features=200, out_features=200, bias=False)\n    (bn1): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n    (leaky_relu): LeakyReLU(negative_slope=0.01, inplace=True)\n    (dp): Dropout(p=0.1, inplace=False)\n    (lin2): Linear(in_features=200, out_features=200, bias=False)\n    (bn2): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n  ))]</pre> <p>Now remember, we need to pass ONLY LAYERS (before I included the name for clarity) the layers in WARM UP ORDER, therefore:</p> In\u00a0[28]: Copied! <pre>tab_deep_layers = [el[1] for el in tab_deep_layers][::-1]\n</pre> tab_deep_layers = [el[1] for el in tab_deep_layers][::-1] In\u00a0[29]: Copied! <pre>tab_layers = [tab_lin_layer] + tab_deep_layers[::-1]\n</pre> tab_layers = [tab_lin_layer] + tab_deep_layers[::-1] In\u00a0[30]: Copied! <pre>tab_layers\n</pre> tab_layers Out[30]: <pre>[Linear(in_features=200, out_features=1, bias=True),\n BasicBlock(\n   (lin1): Linear(in_features=200, out_features=200, bias=False)\n   (bn1): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n   (leaky_relu): LeakyReLU(negative_slope=0.01, inplace=True)\n   (dp): Dropout(p=0.1, inplace=False)\n   (lin2): Linear(in_features=200, out_features=200, bias=False)\n   (bn2): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n ),\n BasicBlock(\n   (lin1): Linear(in_features=200, out_features=200, bias=False)\n   (bn1): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n   (leaky_relu): LeakyReLU(negative_slope=0.01, inplace=True)\n   (dp): Dropout(p=0.1, inplace=False)\n   (lin2): Linear(in_features=200, out_features=200, bias=False)\n   (bn2): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n )]</pre> <p>And now simply</p> In\u00a0[31]: Copied! <pre>trainer_4 = Trainer(model_3, objective=\"binary\", metrics=[Accuracy])\n</pre> trainer_4 = Trainer(model_3, objective=\"binary\", metrics=[Accuracy]) In\u00a0[32]: Copied! <pre>trainer_4.fit(\n    X_wide=X_wide,\n    X_tab=X_tab,\n    target=target,\n    val_split=0.1,\n    finetune=True,\n    finetune_epochs=2,\n    deeptabular_gradual=True,\n    deeptabular_layers=tab_layers,\n    deeptabular_max_lr=0.01,\n    n_epochs=2,\n    batch_size=256,\n)\n</pre> trainer_4.fit(     X_wide=X_wide,     X_tab=X_tab,     target=target,     val_split=0.1,     finetune=True,     finetune_epochs=2,     deeptabular_gradual=True,     deeptabular_layers=tab_layers,     deeptabular_max_lr=0.01,     n_epochs=2,     batch_size=256, ) <pre>Training wide for 2 epochs\n</pre> <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:01&lt;00:00, 95.17it/s, loss=0.504, metrics={'acc': 0.7523}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:01&lt;00:00, 99.83it/s, loss=0.384, metrics={'acc': 0.789}]\n</pre> <pre>Training deeptabular, layer 1 of 3\n</pre> <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:02&lt;00:00, 72.31it/s, loss=0.317, metrics={'acc': 0.8098}]\n</pre> <pre>Training deeptabular, layer 2 of 3\n</pre> <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:02&lt;00:00, 65.97it/s, loss=0.312, metrics={'acc': 0.8214}]\n</pre> <pre>Training deeptabular, layer 3 of 3\n</pre> <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:02&lt;00:00, 63.92it/s, loss=0.306, metrics={'acc': 0.8284}]\n</pre> <pre>Fine-tuning (or warmup) of individual components completed. Training the whole model for 2 epochs\n</pre> <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:03&lt;00:00, 57.26it/s, loss=0.292, metrics={'acc': 0.8664}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 20/20 [00:00&lt;00:00, 84.56it/s, loss=0.292, metrics={'acc': 0.8696}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:03&lt;00:00, 53.61it/s, loss=0.282, metrics={'acc': 0.8693}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 20/20 [00:00&lt;00:00, 80.59it/s, loss=0.289, metrics={'acc': 0.8719}]\n</pre> <p>Finally, there is one more use case I would like to consider. The case where we train only one component and we just want to fine-tune and stop the training afterwards, since there is no joined training. This is a simple as</p> In\u00a0[33]: Copied! <pre>tab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    cat_embed_dropout=0.1,\n    continuous_cols=continuous_cols,\n    mlp_hidden_dims=[64, 32],\n    mlp_dropout=0.2,\n    mlp_activation=\"leaky_relu\",\n)\nmodel = WideDeep(deeptabular=tab_mlp)\n</pre> tab_mlp = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     cat_embed_dropout=0.1,     continuous_cols=continuous_cols,     mlp_hidden_dims=[64, 32],     mlp_dropout=0.2,     mlp_activation=\"leaky_relu\", ) model = WideDeep(deeptabular=tab_mlp) In\u00a0[34]: Copied! <pre>trainer_5 = Trainer(\n    model,\n    objective=\"binary\",\n    optimizers=torch.optim.Adam(model.parameters(), lr=0.01),\n    metrics=[Accuracy],\n)\n</pre> trainer_5 = Trainer(     model,     objective=\"binary\",     optimizers=torch.optim.Adam(model.parameters(), lr=0.01),     metrics=[Accuracy], ) In\u00a0[35]: Copied! <pre>trainer_5.fit(\n    X_wide=X_wide, X_tab=X_tab, target=target, val_split=0.1, n_epochs=1, batch_size=256\n)\n</pre> trainer_5.fit(     X_wide=X_wide, X_tab=X_tab, target=target, val_split=0.1, n_epochs=1, batch_size=256 ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:02&lt;00:00, 73.69it/s, loss=0.365, metrics={'acc': 0.8331}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 20/20 [00:00&lt;00:00, 92.56it/s, loss=0.299, metrics={'acc': 0.8673}]\n</pre> In\u00a0[36]: Copied! <pre>trainer_5.save(path=\"models_dir\", save_state_dict=True, model_filename=\"model_5.pt\")\n</pre> trainer_5.save(path=\"models_dir\", save_state_dict=True, model_filename=\"model_5.pt\") In\u00a0[37]: Copied! <pre>tab_mlp_5 = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    cat_embed_dropout=0.1,\n    continuous_cols=continuous_cols,\n    mlp_hidden_dims=[64, 32],\n    mlp_dropout=0.2,\n    mlp_activation=\"leaky_relu\",\n)\nmodel_5 = WideDeep(deeptabular=tab_mlp_5)\n</pre> tab_mlp_5 = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     cat_embed_dropout=0.1,     continuous_cols=continuous_cols,     mlp_hidden_dims=[64, 32],     mlp_dropout=0.2,     mlp_activation=\"leaky_relu\", ) model_5 = WideDeep(deeptabular=tab_mlp_5) In\u00a0[38]: Copied! <pre>model_5.load_state_dict(torch.load(\"models_dir/model_5.pt\"))\n</pre> model_5.load_state_dict(torch.load(\"models_dir/model_5.pt\")) Out[38]: <pre>&lt;All keys matched successfully&gt;</pre> <p>...times go by...</p> In\u00a0[39]: Copied! <pre>trainer_6 = Trainer(\n    model_5,\n    objective=\"binary\",\n    optimizers=torch.optim.Adam(model.parameters(), lr=0.01),\n    metrics=[Accuracy],\n)\n</pre> trainer_6 = Trainer(     model_5,     objective=\"binary\",     optimizers=torch.optim.Adam(model.parameters(), lr=0.01),     metrics=[Accuracy], ) In\u00a0[40]: Copied! <pre>trainer_6.fit(\n    X_wide=X_wide,\n    X_tab=X_tab,\n    target=target,\n    val_split=0.1,\n    finetune=True,\n    finetune_epochs=2,\n    finetune_max_lr=0.01,\n    stop_after_finetuning=True,\n    batch_size=256,\n)\n</pre> trainer_6.fit(     X_wide=X_wide,     X_tab=X_tab,     target=target,     val_split=0.1,     finetune=True,     finetune_epochs=2,     finetune_max_lr=0.01,     stop_after_finetuning=True,     batch_size=256, ) <pre>Training deeptabular for 2 epochs\n</pre> <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:02&lt;00:00, 73.86it/s, loss=0.298, metrics={'acc': 0.8652}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:02&lt;00:00, 75.45it/s, loss=0.286, metrics={'acc': 0.8669}]\n</pre> <pre>Fine-tuning (or warmup) of individual components completed. Training the whole model for 1 epochs\n</pre> <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:02&lt;00:00, 76.29it/s, loss=0.282, metrics={'acc': 0.8698}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 20/20 [00:00&lt;00:00, 84.93it/s, loss=0.281, metrics={'acc': 0.8749}]\n</pre> In\u00a0[42]: Copied! <pre>import shutil\n\nshutil.rmtree(\"models_dir/\")\nshutil.rmtree(\"model_weights/\")\n</pre> import shutil  shutil.rmtree(\"models_dir/\") shutil.rmtree(\"model_weights/\") In\u00a0[\u00a0]: Copied! <pre>\n</pre>"},{"location":"examples/06_finetune_and_warmup.html#the-finetunewarm-up-option","title":"The FineTune/Warm Up option\u00b6","text":"<p>Let's place ourselves in two possible scenarios.</p> <ol> <li><p>Let's assume we have run a model and we want to just transfer the learnings (you know...transfer-learning) to another dataset, or simply we have received new data and we do not want to start the training of each component from scratch. Simply, we want to load the pre-trained weights and fine-tune.</p> </li> <li><p>We just want to \"warm up\" individual model components individually before the joined training begins.</p> </li> </ol> <p>This can be done with the <code>finetune</code> set of parameters. There are 3 fine-tuning routines:</p> <ol> <li>Fine-tune all trainable layers at once with a triangular one-cycle learning rate (referred as slanted triangular learning rates in Howard &amp; Ruder 2018)</li> <li>Gradual fine-tuning inspired by the work of Felbo et al., 2017</li> <li>Gradual fine-tuning based on the work of Howard &amp; Ruder 2018</li> </ol> <p>Currently fine-tunning is only supported without a fully connected head, i.e. if <code>deephead=None</code>. In addition, <code>Felbo</code> and <code>Howard</code> routines only applied, of course, to the <code>deeptabular</code>, <code>deeptext</code> and <code>deepimage</code> models. The <code>wide</code> component can also be fine-tuned, but only in an \"all at once\" mode.</p>"},{"location":"examples/06_finetune_and_warmup.html#fine-tune-or-warm-up-all-at-once","title":"Fine-tune or warm-up all at once\u00b6","text":"<p>Here, the model components will be trained for <code>finetune_epochs</code> using a triangular one-cycle learning rate (slanted triangular learning rate) ranging from <code>finetune_max_lr/10</code> to <code>finetune_max_lr</code> (default is 0.01). 10% of the training steps are used to increase the learning rate which then decreases for the remaining 90%.</p> <p>Here all trainable layers are fine-tuned.</p> <p>Let's have a look to one example.</p>"},{"location":"examples/06_finetune_and_warmup.html#fine-tune-gradually-the-felbo-and-the-howard-routines","title":"Fine-tune Gradually: The \"felbo\"  and the \"howard\" routines\u00b6","text":"<p>The Felbo routine can be illustrated as follows:</p> <p> </p> <p>Figure 1. The figure can be described as follows: fine-tune (or train) the last layer for one epoch using a one cycle triangular learning rate. Then fine-tune the next deeper layer for one epoch, with a learning rate that is a factor of 2.5 lower than the previous learning rate (the 2.5 factor is fixed) while freezing the already warmed up layer(s). Repeat untill all individual layers are warmed. Then warm one last epoch with all warmed layers trainable. The vanishing color gradient in the figure attempts to illustrate the decreasing learning rate.</p> <p>Note that this is not identical to the Fine-Tunning routine described in Felbo et al, 2017, this is why I used the word 'inspired'.</p> <p>The Howard routine can be illustrated as follows:</p> <p> </p> <p>Figure 2. The figure can be described as follows: fine-tune (or train) the last layer for one epoch using a one cycle triangular learning rate. Then fine-tune the next deeper layer for one epoch, with a learning rate that is a factor of 2.5 lower than the previous learning rate (the 2.5 factor is fixed) while keeping the already warmed up layer(s) trainable. Repeat. The vanishing color gradient in the figure attempts to illustrate the decreasing learning rate.</p> <p>Note that I write \"fine-tune (or train) the last layer for one epoch [...]\". However, in practice the user will have to specify the order of the layers to be fine-tuned. This is another reason why I wrote that the fine-tune routines I have implemented are inspired by the work of Felbo and Howard and not identical to their implemenations.</p> <p>The <code>felbo</code> and <code>howard</code> routines can be accessed with via the <code>fine-tune</code> parameters.</p>"},{"location":"examples/07_custom_components.html","title":"07_custom_components","text":"In\u00a0[1]: Copied! <pre>import numpy as np\nimport pandas as pd\nimport os\nimport torch\n\nfrom torch import Tensor\nfrom pytorch_widedeep import Trainer\nfrom pytorch_widedeep.preprocessing import (\n    WidePreprocessor,\n    TabPreprocessor,\n    TextPreprocessor,\n    ImagePreprocessor,\n)\nfrom pytorch_widedeep.models import (\n    Wide,\n    TabMlp,\n    Vision,\n    BasicRNN,\n    WideDeep,\n)\nfrom pytorch_widedeep.losses import RMSELoss\nfrom pytorch_widedeep.initializers import *\nfrom pytorch_widedeep.callbacks import *\nfrom pytorch_widedeep.datasets import load_adult\n</pre> import numpy as np import pandas as pd import os import torch  from torch import Tensor from pytorch_widedeep import Trainer from pytorch_widedeep.preprocessing import (     WidePreprocessor,     TabPreprocessor,     TextPreprocessor,     ImagePreprocessor, ) from pytorch_widedeep.models import (     Wide,     TabMlp,     Vision,     BasicRNN,     WideDeep, ) from pytorch_widedeep.losses import RMSELoss from pytorch_widedeep.initializers import * from pytorch_widedeep.callbacks import * from pytorch_widedeep.datasets import load_adult <pre>/Users/javierrodriguezzaurin/.pyenv/versions/3.10.13/envs/widedeep310/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n  from .autonotebook import tqdm as notebook_tqdm\n</pre> In\u00a0[2]: Copied! <pre>df = pd.read_csv(\"../tmp_data/airbnb/airbnb_sample.csv\")\ndf.head()\n</pre> df = pd.read_csv(\"../tmp_data/airbnb/airbnb_sample.csv\") df.head() Out[2]: id host_id description host_listings_count host_identity_verified neighbourhood_cleansed latitude longitude is_location_exact property_type ... amenity_wide_entrance amenity_wide_entrance_for_guests amenity_wide_entryway amenity_wide_hallways amenity_wifi amenity_window_guards amenity_wine_cooler security_deposit extra_people yield 0 13913.jpg 54730 My bright double bedroom with a large window h... 4.0 f Islington 51.56802 -0.11121 t apartment ... 1 0 0 0 1 0 0 100.0 15.0 12.00 1 15400.jpg 60302 Lots of windows and light.  St Luke's Gardens ... 1.0 t Kensington and Chelsea 51.48796 -0.16898 t apartment ... 0 0 0 0 1 0 0 150.0 0.0 109.50 2 17402.jpg 67564 Open from June 2018 after a 3-year break, we a... 19.0 t Westminster 51.52098 -0.14002 t apartment ... 0 0 0 0 1 0 0 350.0 10.0 149.65 3 24328.jpg 41759 Artist house, bright high ceiling rooms, priva... 2.0 t Wandsworth 51.47298 -0.16376 t other ... 0 0 0 0 1 0 0 250.0 0.0 215.60 4 25023.jpg 102813 Large, all comforts, 2-bed flat; first floor; ... 1.0 f Wandsworth 51.44687 -0.21874 t apartment ... 0 0 0 0 1 0 0 250.0 11.0 79.35 <p>5 rows \u00d7 223 columns</p> In\u00a0[3]: Copied! <pre># There are a number of columns that are already binary. Therefore, no need to one hot encode them\ncrossed_cols = [(\"property_type\", \"room_type\")]\nalready_dummies = [c for c in df.columns if \"amenity\" in c] + [\"has_house_rules\"]\nwide_cols = [\n    \"is_location_exact\",\n    \"property_type\",\n    \"room_type\",\n    \"host_gender\",\n    \"instant_bookable\",\n] + already_dummies\n\ncat_embed_cols = [(c, 16) for c in df.columns if \"catg\" in c] + [\n    (\"neighbourhood_cleansed\", 64),\n    (\"cancellation_policy\", 16),\n]\ncontinuous_cols = [\"latitude\", \"longitude\", \"security_deposit\", \"extra_people\"]\n# it does not make sense to standarised Latitude and Longitude\nalready_standard = [\"latitude\", \"longitude\"]\n\n# text and image colnames\ntext_col = \"description\"\nimg_col = \"id\"\n\n# path to pretrained word embeddings and the images\nword_vectors_path = \"../tmp_data/glove.6B/glove.6B.100d.txt\"\nimg_path = \"../tmp_data/airbnb/property_picture\"\n\n# target\ntarget_col = \"yield\"\n</pre> # There are a number of columns that are already binary. Therefore, no need to one hot encode them crossed_cols = [(\"property_type\", \"room_type\")] already_dummies = [c for c in df.columns if \"amenity\" in c] + [\"has_house_rules\"] wide_cols = [     \"is_location_exact\",     \"property_type\",     \"room_type\",     \"host_gender\",     \"instant_bookable\", ] + already_dummies  cat_embed_cols = [(c, 16) for c in df.columns if \"catg\" in c] + [     (\"neighbourhood_cleansed\", 64),     (\"cancellation_policy\", 16), ] continuous_cols = [\"latitude\", \"longitude\", \"security_deposit\", \"extra_people\"] # it does not make sense to standarised Latitude and Longitude already_standard = [\"latitude\", \"longitude\"]  # text and image colnames text_col = \"description\" img_col = \"id\"  # path to pretrained word embeddings and the images word_vectors_path = \"../tmp_data/glove.6B/glove.6B.100d.txt\" img_path = \"../tmp_data/airbnb/property_picture\"  # target target_col = \"yield\" In\u00a0[4]: Copied! <pre>target = df[target_col].values\n</pre> target = df[target_col].values In\u00a0[5]: Copied! <pre>wide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols)\nX_wide = wide_preprocessor.fit_transform(df)\n\ntab_preprocessor = TabPreprocessor(\n    embed_cols=cat_embed_cols, continuous_cols=continuous_cols\n)\nX_tab = tab_preprocessor.fit_transform(df)\n\ntext_preprocessor = TextPreprocessor(\n    word_vectors_path=word_vectors_path, text_col=text_col\n)\nX_text = text_preprocessor.fit_transform(df)\n\nimage_processor = ImagePreprocessor(img_col=img_col, img_path=img_path)\nX_images = image_processor.fit_transform(df)\n</pre> wide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols) X_wide = wide_preprocessor.fit_transform(df)  tab_preprocessor = TabPreprocessor(     embed_cols=cat_embed_cols, continuous_cols=continuous_cols ) X_tab = tab_preprocessor.fit_transform(df)  text_preprocessor = TextPreprocessor(     word_vectors_path=word_vectors_path, text_col=text_col ) X_text = text_preprocessor.fit_transform(df)  image_processor = ImagePreprocessor(img_col=img_col, img_path=img_path) X_images = image_processor.fit_transform(df) <pre>/Users/javierrodriguezzaurin/Projects/pytorch-widedeep/pytorch_widedeep/preprocessing/tab_preprocessor.py:358: UserWarning: Continuous columns will not be normalised\n  warnings.warn(\"Continuous columns will not be normalised\")\n</pre> <pre>The vocabulary contains 2192 tokens\nIndexing word vectors...\nLoaded 400000 word vectors\nPreparing embeddings matrix...\n2175 words in the vocabulary had ../tmp_data/glove.6B/glove.6B.100d.txt vectors and appear more than 5 times\nReading Images from ../tmp_data/airbnb/property_picture\nResizing\n</pre> <pre>100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 1001/1001 [00:02&lt;00:00, 497.80it/s]\n</pre> <pre>Computing normalisation metrics\n</pre> <p>Now we are ready to build a wide and deep model. Three of the four components we will use are included in this package, and they will be combined with a custom <code>deeptext</code> component. Then the fit process will run with a custom loss function.</p> <p>Let's have a look</p> In\u00a0[6]: Copied! <pre># Linear model\nwide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1)\n\n# DeepDense: 2 Dense layers\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    cat_embed_dropout=0.1,\n    continuous_cols=continuous_cols,\n    mlp_hidden_dims=[128, 64],\n    mlp_dropout=0.1,\n)\n\n# Pretrained Resnet 18\nresnet = Vision(pretrained_model_name=\"resnet18\", n_trainable=0)\n</pre> # Linear model wide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1)  # DeepDense: 2 Dense layers tab_mlp = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     cat_embed_dropout=0.1,     continuous_cols=continuous_cols,     mlp_hidden_dims=[128, 64],     mlp_dropout=0.1, )  # Pretrained Resnet 18 resnet = Vision(pretrained_model_name=\"resnet18\", n_trainable=0) In\u00a0[7]: Copied! <pre>class MyDeepText(nn.Module):\n    def __init__(self, vocab_size, padding_idx=1, embed_dim=100, hidden_dim=64):\n        super(MyDeepText, self).__init__()\n\n        # word/token embeddings\n        self.word_embed = nn.Embedding(vocab_size, embed_dim, padding_idx=padding_idx)\n\n        # stack of RNNs\n        self.rnn = nn.GRU(\n            embed_dim,\n            hidden_dim,\n            num_layers=2,\n            bidirectional=True,\n            batch_first=True,\n        )\n\n        # Remember, this MUST be defined. If not WideDeep will through an error\n        self.output_dim = hidden_dim * 2\n\n    def forward(self, X):\n        embed = self.word_embed(X.long())\n        o, h = self.rnn(embed)\n        return torch.cat((h[-2], h[-1]), dim=1)\n</pre> class MyDeepText(nn.Module):     def __init__(self, vocab_size, padding_idx=1, embed_dim=100, hidden_dim=64):         super(MyDeepText, self).__init__()          # word/token embeddings         self.word_embed = nn.Embedding(vocab_size, embed_dim, padding_idx=padding_idx)          # stack of RNNs         self.rnn = nn.GRU(             embed_dim,             hidden_dim,             num_layers=2,             bidirectional=True,             batch_first=True,         )          # Remember, this MUST be defined. If not WideDeep will through an error         self.output_dim = hidden_dim * 2      def forward(self, X):         embed = self.word_embed(X.long())         o, h = self.rnn(embed)         return torch.cat((h[-2], h[-1]), dim=1) In\u00a0[8]: Copied! <pre>mydeeptext = MyDeepText(vocab_size=len(text_preprocessor.vocab.itos))\n</pre> mydeeptext = MyDeepText(vocab_size=len(text_preprocessor.vocab.itos)) In\u00a0[9]: Copied! <pre>model = WideDeep(wide=wide, deeptabular=tab_mlp, deeptext=mydeeptext, deepimage=resnet)\n</pre> model = WideDeep(wide=wide, deeptabular=tab_mlp, deeptext=mydeeptext, deepimage=resnet) In\u00a0[10]: Copied! <pre>class RMSELoss(nn.Module):\n    def __init__(self):\n        \"\"\"root mean squared error\"\"\"\n        super().__init__()\n        self.mse = nn.MSELoss()\n\n    def forward(self, input: Tensor, target: Tensor) -&gt; Tensor:\n        return torch.sqrt(self.mse(input, target))\n</pre> class RMSELoss(nn.Module):     def __init__(self):         \"\"\"root mean squared error\"\"\"         super().__init__()         self.mse = nn.MSELoss()      def forward(self, input: Tensor, target: Tensor) -&gt; Tensor:         return torch.sqrt(self.mse(input, target)) <p>and now we just instantiate the <code>Trainer</code> as usual. Needless to say, but this runs with 1000 random observations, so loss and metric values are meaningless. This is just an example</p> In\u00a0[11]: Copied! <pre>trainer = Trainer(model, objective=\"regression\", custom_loss_function=RMSELoss())\n</pre> trainer = Trainer(model, objective=\"regression\", custom_loss_function=RMSELoss()) In\u00a0[12]: Copied! <pre>trainer.fit(\n    X_wide=X_wide,\n    X_tab=X_tab,\n    X_text=X_text,\n    X_img=X_images,\n    target=target,\n    n_epochs=1,\n    batch_size=32,\n    val_split=0.2,\n)\n</pre> trainer.fit(     X_wide=X_wide,     X_tab=X_tab,     X_text=X_text,     X_img=X_images,     target=target,     n_epochs=1,     batch_size=32,     val_split=0.2, ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 25/25 [00:23&lt;00:00,  1.07it/s, loss=126]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 7/7 [00:05&lt;00:00,  1.24it/s, loss=97.4]\n</pre> <p>In addition to model components and loss functions, we can also use custom callbacks or custom metrics. The former need to be of type <code>Callback</code> and the latter need to be of type <code>Metric</code>. See:</p> <pre>pytorch-widedeep.callbacks\n</pre> <p>and</p> <pre>pytorch-widedeep.metrics\n</pre> <p>For this example let me use the adult dataset. Again, we first prepare the data as usual</p> In\u00a0[13]: Copied! <pre>df = load_adult(as_frame=True)\ndf.head()\n</pre> df = load_adult(as_frame=True) df.head() Out[13]: age workclass fnlwgt education educational-num marital-status occupation relationship race gender capital-gain capital-loss hours-per-week native-country income 0 25 Private 226802 11th 7 Never-married Machine-op-inspct Own-child Black Male 0 0 40 United-States &lt;=50K 1 38 Private 89814 HS-grad 9 Married-civ-spouse Farming-fishing Husband White Male 0 0 50 United-States &lt;=50K 2 28 Local-gov 336951 Assoc-acdm 12 Married-civ-spouse Protective-serv Husband White Male 0 0 40 United-States &gt;50K 3 44 Private 160323 Some-college 10 Married-civ-spouse Machine-op-inspct Husband Black Male 7688 0 40 United-States &gt;50K 4 18 ? 103497 Some-college 10 Never-married ? Own-child White Female 0 0 30 United-States &lt;=50K In\u00a0[14]: Copied! <pre># For convenience, we'll replace '-' with '_'\ndf.columns = [c.replace(\"-\", \"_\") for c in df.columns]\n# binary target\ndf[\"income_label\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int)\ndf.drop(\"income\", axis=1, inplace=True)\ndf.head()\n</pre> # For convenience, we'll replace '-' with '_' df.columns = [c.replace(\"-\", \"_\") for c in df.columns] # binary target df[\"income_label\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int) df.drop(\"income\", axis=1, inplace=True) df.head() Out[14]: age workclass fnlwgt education educational_num marital_status occupation relationship race gender capital_gain capital_loss hours_per_week native_country income_label 0 25 Private 226802 11th 7 Never-married Machine-op-inspct Own-child Black Male 0 0 40 United-States 0 1 38 Private 89814 HS-grad 9 Married-civ-spouse Farming-fishing Husband White Male 0 0 50 United-States 0 2 28 Local-gov 336951 Assoc-acdm 12 Married-civ-spouse Protective-serv Husband White Male 0 0 40 United-States 1 3 44 Private 160323 Some-college 10 Married-civ-spouse Machine-op-inspct Husband Black Male 7688 0 40 United-States 1 4 18 ? 103497 Some-college 10 Never-married ? Own-child White Female 0 0 30 United-States 0 In\u00a0[15]: Copied! <pre># Define wide, crossed and deep tabular columns\nwide_cols = [\n    \"workclass\",\n    \"education\",\n    \"marital_status\",\n    \"occupation\",\n    \"relationship\",\n    \"race\",\n    \"gender\",\n    \"native_country\",\n]\ncrossed_cols = [(\"education\", \"occupation\"), (\"native_country\", \"occupation\")]\ncat_embed_cols = [\n    \"workclass\",\n    \"education\",\n    \"marital_status\",\n    \"occupation\",\n    \"relationship\",\n    \"race\",\n    \"gender\",\n    \"capital_gain\",\n    \"capital_loss\",\n    \"native_country\",\n]\ncontinuous_cols = [\"age\", \"hours_per_week\"]\ntarget_col = \"income_label\"\ntarget = df[target_col].values\n</pre> # Define wide, crossed and deep tabular columns wide_cols = [     \"workclass\",     \"education\",     \"marital_status\",     \"occupation\",     \"relationship\",     \"race\",     \"gender\",     \"native_country\", ] crossed_cols = [(\"education\", \"occupation\"), (\"native_country\", \"occupation\")] cat_embed_cols = [     \"workclass\",     \"education\",     \"marital_status\",     \"occupation\",     \"relationship\",     \"race\",     \"gender\",     \"capital_gain\",     \"capital_loss\",     \"native_country\", ] continuous_cols = [\"age\", \"hours_per_week\"] target_col = \"income_label\" target = df[target_col].values In\u00a0[16]: Copied! <pre># wide\nwide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols)\nX_wide = wide_preprocessor.fit_transform(df)\n\n# deeptabular\ntab_preprocessor = TabPreprocessor(\n    embed_cols=cat_embed_cols, continuous_cols=continuous_cols\n)\nX_tab = tab_preprocessor.fit_transform(df)\n</pre> # wide wide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols) X_wide = wide_preprocessor.fit_transform(df)  # deeptabular tab_preprocessor = TabPreprocessor(     embed_cols=cat_embed_cols, continuous_cols=continuous_cols ) X_tab = tab_preprocessor.fit_transform(df) <pre>/Users/javierrodriguezzaurin/Projects/pytorch-widedeep/pytorch_widedeep/preprocessing/tab_preprocessor.py:358: UserWarning: Continuous columns will not be normalised\n  warnings.warn(\"Continuous columns will not be normalised\")\n</pre> In\u00a0[17]: Copied! <pre>wide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1)\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=continuous_cols,\n    mlp_hidden_dims=[128, 64],\n    mlp_dropout=0.2,\n    mlp_activation=\"leaky_relu\",\n)\nmodel = WideDeep(wide=wide, deeptabular=tab_mlp)\n</pre> wide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1) tab_mlp = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     continuous_cols=continuous_cols,     mlp_hidden_dims=[128, 64],     mlp_dropout=0.2,     mlp_activation=\"leaky_relu\", ) model = WideDeep(wide=wide, deeptabular=tab_mlp) In\u00a0[18]: Copied! <pre>from pytorch_widedeep.metrics import Metric\n</pre> from pytorch_widedeep.metrics import Metric In\u00a0[19]: Copied! <pre>class Accuracy(Metric):\n    def __init__(self, top_k: int = 1):\n        super(Accuracy, self).__init__()\n\n        self.top_k = top_k\n        self.correct_count = 0\n        self.total_count = 0\n\n        # \u00a0metric name needs to be defined\n        self._name = \"acc\"\n\n    def reset(self):\n        self.correct_count = 0\n        self.total_count = 0\n\n    def __call__(self, y_pred: Tensor, y_true: Tensor) -&gt; np.ndarray:\n        num_classes = y_pred.size(1)\n\n        if num_classes == 1:\n            y_pred = y_pred.round()\n            y_true = y_true\n        elif num_classes &gt; 1:\n            y_pred = y_pred.topk(self.top_k, 1)[1]\n            y_true = y_true.view(-1, 1).expand_as(y_pred)\n\n        self.correct_count += y_pred.eq(y_true).sum().item()\n        self.total_count += len(y_pred)\n        accuracy = float(self.correct_count) / float(self.total_count)\n        return np.array(accuracy)\n</pre> class Accuracy(Metric):     def __init__(self, top_k: int = 1):         super(Accuracy, self).__init__()          self.top_k = top_k         self.correct_count = 0         self.total_count = 0          # \u00a0metric name needs to be defined         self._name = \"acc\"      def reset(self):         self.correct_count = 0         self.total_count = 0      def __call__(self, y_pred: Tensor, y_true: Tensor) -&gt; np.ndarray:         num_classes = y_pred.size(1)          if num_classes == 1:             y_pred = y_pred.round()             y_true = y_true         elif num_classes &gt; 1:             y_pred = y_pred.topk(self.top_k, 1)[1]             y_true = y_true.view(-1, 1).expand_as(y_pred)          self.correct_count += y_pred.eq(y_true).sum().item()         self.total_count += len(y_pred)         accuracy = float(self.correct_count) / float(self.total_count)         return np.array(accuracy) In\u00a0[20]: Copied! <pre># have a look to the class\nfrom pytorch_widedeep.callbacks import Callback\n</pre> # have a look to the class from pytorch_widedeep.callbacks import Callback In\u00a0[21]: Copied! <pre>class SillyCallback(Callback):\n    def on_train_begin(self, logs=None):\n        # recordings will be the trainer object attributes\n        self.trainer.silly_callback = {}\n\n        self.trainer.silly_callback[\"beginning\"] = []\n        self.trainer.silly_callback[\"end\"] = []\n\n    def on_epoch_begin(self, epoch, logs=None):\n        self.trainer.silly_callback[\"beginning\"].append(epoch + 1)\n\n    def on_epoch_end(self, epoch, logs=None, metric=None):\n        self.trainer.silly_callback[\"end\"].append(epoch + 1)\n</pre> class SillyCallback(Callback):     def on_train_begin(self, logs=None):         # recordings will be the trainer object attributes         self.trainer.silly_callback = {}          self.trainer.silly_callback[\"beginning\"] = []         self.trainer.silly_callback[\"end\"] = []      def on_epoch_begin(self, epoch, logs=None):         self.trainer.silly_callback[\"beginning\"].append(epoch + 1)      def on_epoch_end(self, epoch, logs=None, metric=None):         self.trainer.silly_callback[\"end\"].append(epoch + 1) <p>and now, as usual:</p> In\u00a0[22]: Copied! <pre>trainer = Trainer(\n    model, objective=\"binary\", metrics=[Accuracy], callbacks=[SillyCallback]\n)\n</pre> trainer = Trainer(     model, objective=\"binary\", metrics=[Accuracy], callbacks=[SillyCallback] ) In\u00a0[23]: Copied! <pre>trainer.fit(\n    X_wide=X_wide, X_tab=X_tab, target=target, n_epochs=5, batch_size=64, val_split=0.2\n)\n</pre> trainer.fit(     X_wide=X_wide, X_tab=X_tab, target=target, n_epochs=5, batch_size=64, val_split=0.2 ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 611/611 [00:06&lt;00:00, 94.39it/s, loss=0.411, metrics={'acc': 0.814}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 121.91it/s, loss=0.327, metrics={'acc': 0.8449}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 611/611 [00:07&lt;00:00, 85.39it/s, loss=0.324, metrics={'acc': 0.8495}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 88.68it/s, loss=0.298, metrics={'acc': 0.8612}]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 611/611 [00:08&lt;00:00, 74.35it/s, loss=0.302, metrics={'acc': 0.8593}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 100.51it/s, loss=0.29, metrics={'acc': 0.8665}]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 611/611 [00:08&lt;00:00, 73.83it/s, loss=0.292, metrics={'acc': 0.8637}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 105.98it/s, loss=0.286, metrics={'acc': 0.8695}]\nepoch 5: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 611/611 [00:08&lt;00:00, 72.15it/s, loss=0.286, metrics={'acc': 0.866}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 92.27it/s, loss=0.284, metrics={'acc': 0.8698}]\n</pre> In\u00a0[24]: Copied! <pre>trainer.silly_callback\n</pre> trainer.silly_callback Out[24]: <pre>{'beginning': [1, 2, 3, 4, 5], 'end': [1, 2, 3, 4, 5]}</pre>"},{"location":"examples/07_custom_components.html#custom-components","title":"Custom components\u00b6","text":"<p>As I mentioned earlier in the example notebooks, and also in the <code>README</code>, it is possible to customise almost every component in <code>pytorch-widedeep</code>.</p> <p>Let's now go through a couple of simple examples to illustrate how that could be done.</p> <p>First let's load and process the data \"as usual\", let's start with a regression and the airbnb dataset.</p>"},{"location":"examples/07_custom_components.html#custom-deeptext","title":"Custom <code>deeptext</code>\u00b6","text":"<p>Standard Pytorch model</p>"},{"location":"examples/07_custom_components.html#custom-loss-function","title":"Custom loss function\u00b6","text":"<p>Loss functions must simply inherit pytorch's <code>nn.Module</code>. For example, let's say we want to use <code>RMSE</code> (note that this is already available in the package, but I will pass it here as a custom loss for illustration purposes)</p>"},{"location":"examples/07_custom_components.html#custom-metric","title":"Custom metric\u00b6","text":"<p>Let's say we want to use our own accuracy metric (again, this is already available in the package, but I will pass it here as a custom loss for illustration purposes).</p> <p>This could be done as:</p>"},{"location":"examples/07_custom_components.html#custom-callback","title":"Custom Callback\u00b6","text":"<p>Let's code a callback that records the current epoch at the beginning and the end of each epoch (silly, but you know, this is just an example)</p>"},{"location":"examples/08_custom_dataLoader_imbalanced_dataset.html","title":"08_custom_dataLoader_imbalanced_dataset","text":"<ul> <li>In this notebook we will use the higly imbalanced Protein Homology Dataset from KDD cup 2004</li> </ul> <pre><code>* The first element of each line is a BLOCK ID that denotes to which native sequence this example belongs. There is a unique BLOCK ID for each native sequence. BLOCK IDs are integers running from 1 to 303 (one for each native sequence, i.e. for each query). BLOCK IDs were assigned before the blocks were split into the train and test sets, so they do not run consecutively in either file.\n* The second element of each line is an EXAMPLE ID that uniquely describes the example. You will need this EXAMPLE ID and the BLOCK ID when you submit results.\n* The third element is the class of the example. Proteins that are homologous to the native sequence are denoted by 1, non-homologous proteins (i.e. decoys) by 0. Test examples have a \"?\" in this position.\n* All following elements are feature values. There are 74 feature values in each line. The features describe the match (e.g. the score of a sequence alignment) between the native protein sequence and the sequence that is tested for homology.\n</code></pre> In\u00a0[1]: Copied! <pre>import numpy as np\nimport pandas as pd\nimport torch\nfrom torch.optim import SGD, lr_scheduler\n\nfrom pytorch_widedeep import Trainer\nfrom pytorch_widedeep.preprocessing import TabPreprocessor\nfrom pytorch_widedeep.models import TabMlp, WideDeep\nfrom pytorch_widedeep.dataloaders import DataLoaderImbalanced, DataLoaderDefault\nfrom torchmetrics import F1Score as F1_torchmetrics\nfrom torchmetrics import Accuracy as Accuracy_torchmetrics\nfrom torchmetrics import Precision as Precision_torchmetrics\nfrom torchmetrics import Recall as Recall_torchmetrics\nfrom pytorch_widedeep.metrics import Accuracy, Recall, Precision, F1Score, R2Score\nfrom pytorch_widedeep.initializers import XavierNormal\nfrom pytorch_widedeep.datasets import load_bio_kdd04\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import classification_report\n\nimport time\nimport datetime\n\nimport warnings\n\nwarnings.filterwarnings(\"ignore\", category=DeprecationWarning)\n\n# increase displayed columns in jupyter notebook\npd.set_option(\"display.max_columns\", 200)\npd.set_option(\"display.max_rows\", 300)\n</pre> import numpy as np import pandas as pd import torch from torch.optim import SGD, lr_scheduler  from pytorch_widedeep import Trainer from pytorch_widedeep.preprocessing import TabPreprocessor from pytorch_widedeep.models import TabMlp, WideDeep from pytorch_widedeep.dataloaders import DataLoaderImbalanced, DataLoaderDefault from torchmetrics import F1Score as F1_torchmetrics from torchmetrics import Accuracy as Accuracy_torchmetrics from torchmetrics import Precision as Precision_torchmetrics from torchmetrics import Recall as Recall_torchmetrics from pytorch_widedeep.metrics import Accuracy, Recall, Precision, F1Score, R2Score from pytorch_widedeep.initializers import XavierNormal from pytorch_widedeep.datasets import load_bio_kdd04  from sklearn.model_selection import train_test_split from sklearn.metrics import classification_report  import time import datetime  import warnings  warnings.filterwarnings(\"ignore\", category=DeprecationWarning)  # increase displayed columns in jupyter notebook pd.set_option(\"display.max_columns\", 200) pd.set_option(\"display.max_rows\", 300) <pre>/Users/javierrodriguezzaurin/.pyenv/versions/3.10.13/envs/widedeep310/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n  from .autonotebook import tqdm as notebook_tqdm\n</pre> In\u00a0[2]: Copied! <pre>df = load_bio_kdd04(as_frame=True)\n# drop columns we won't need in this example\ndf.drop(columns=[\"EXAMPLE_ID\", \"BLOCK_ID\"], inplace=True)\n\ndf_train, df_valid = train_test_split(\n    df, test_size=0.2, stratify=df[\"target\"], random_state=1\n)\ndf_valid, df_test = train_test_split(\n    df_valid, test_size=0.5, stratify=df_valid[\"target\"], random_state=1\n)\n\ncontinuous_cols = df.drop(columns=[\"target\"]).columns.values.tolist()\n</pre> df = load_bio_kdd04(as_frame=True) # drop columns we won't need in this example df.drop(columns=[\"EXAMPLE_ID\", \"BLOCK_ID\"], inplace=True)  df_train, df_valid = train_test_split(     df, test_size=0.2, stratify=df[\"target\"], random_state=1 ) df_valid, df_test = train_test_split(     df_valid, test_size=0.5, stratify=df_valid[\"target\"], random_state=1 )  continuous_cols = df.drop(columns=[\"target\"]).columns.values.tolist() In\u00a0[3]: Copied! <pre># deeptabular\ntab_preprocessor = TabPreprocessor(continuous_cols=continuous_cols, scale=True)\nX_tab_train = tab_preprocessor.fit_transform(df_train)\nX_tab_valid = tab_preprocessor.transform(df_valid)\nX_tab_test = tab_preprocessor.transform(df_test)\n\n# target\ny_train = df_train[\"target\"].values\ny_valid = df_valid[\"target\"].values\ny_test = df_test[\"target\"].values\n</pre> # deeptabular tab_preprocessor = TabPreprocessor(continuous_cols=continuous_cols, scale=True) X_tab_train = tab_preprocessor.fit_transform(df_train) X_tab_valid = tab_preprocessor.transform(df_valid) X_tab_test = tab_preprocessor.transform(df_test)  # target y_train = df_train[\"target\"].values y_valid = df_valid[\"target\"].values y_test = df_test[\"target\"].values In\u00a0[4]: Copied! <pre># Define the model\ninput_layer = len(tab_preprocessor.continuous_cols)\noutput_layer = 1\nhidden_layers = np.linspace(\n    input_layer * 2, output_layer, 5, endpoint=False, dtype=int\n).tolist()\n\ndeeptabular = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    continuous_cols=tab_preprocessor.continuous_cols,\n    mlp_hidden_dims=hidden_layers,\n)\nmodel = WideDeep(deeptabular=deeptabular)\nmodel\n</pre> # Define the model input_layer = len(tab_preprocessor.continuous_cols) output_layer = 1 hidden_layers = np.linspace(     input_layer * 2, output_layer, 5, endpoint=False, dtype=int ).tolist()  deeptabular = TabMlp(     column_idx=tab_preprocessor.column_idx,     continuous_cols=tab_preprocessor.continuous_cols,     mlp_hidden_dims=hidden_layers, ) model = WideDeep(deeptabular=deeptabular) model Out[4]: <pre>WideDeep(\n  (deeptabular): Sequential(\n    (0): TabMlp(\n      (cont_norm): Identity()\n      (encoder): MLP(\n        (mlp): Sequential(\n          (dense_layer_0): Sequential(\n            (0): Linear(in_features=74, out_features=148, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_1): Sequential(\n            (0): Linear(in_features=148, out_features=118, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_2): Sequential(\n            (0): Linear(in_features=118, out_features=89, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_3): Sequential(\n            (0): Linear(in_features=89, out_features=59, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_4): Sequential(\n            (0): Linear(in_features=59, out_features=30, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=30, out_features=1, bias=True)\n  )\n)</pre> In\u00a0[5]: Copied! <pre># Metrics from pytorch-widedeep\naccuracy = Accuracy(top_k=2)\nprecision = Precision(average=False)\n\n# # Metrics from torchmetrics\n# accuracy = Accuracy_torchmetrics(average=None, num_classes=1)\n# precision = Precision_torchmetrics(average=\"micro\", num_classes=1)\n\n# Optimizers\ndeep_opt = SGD(model.deeptabular.parameters(), lr=0.1)\n\n# LR Scheduler\ndeep_sch = lr_scheduler.StepLR(deep_opt, step_size=3)\n\ntrainer = Trainer(\n    model,\n    objective=\"binary\",\n    lr_schedulers={\"deeptabular\": deep_sch},\n    initializers={\"deeptabular\": XavierNormal},\n    optimizers={\"deeptabular\": deep_opt},\n    metrics=[accuracy, precision],\n    verbose=1,\n)\n</pre> # Metrics from pytorch-widedeep accuracy = Accuracy(top_k=2) precision = Precision(average=False)  # # Metrics from torchmetrics # accuracy = Accuracy_torchmetrics(average=None, num_classes=1) # precision = Precision_torchmetrics(average=\"micro\", num_classes=1)  # Optimizers deep_opt = SGD(model.deeptabular.parameters(), lr=0.1)  # LR Scheduler deep_sch = lr_scheduler.StepLR(deep_opt, step_size=3)  trainer = Trainer(     model,     objective=\"binary\",     lr_schedulers={\"deeptabular\": deep_sch},     initializers={\"deeptabular\": XavierNormal},     optimizers={\"deeptabular\": deep_opt},     metrics=[accuracy, precision],     verbose=1, ) In\u00a0[6]: Copied! <pre>start = time.time()\ntrainer.fit(\n    X_train={\"X_tab\": X_tab_train, \"target\": y_train},\n    X_val={\"X_tab\": X_tab_valid, \"target\": y_valid},\n    n_epochs=1,\n    batch_size=32,\n    custom_dataloader=DataLoaderImbalanced,\n    oversample_mul=5,\n)\nprint(\n    \"Training time[s]: {}\".format(\n        datetime.timedelta(seconds=round(time.time() - start))\n    )\n)\n\npd.DataFrame(trainer.history)\n\ndf_pred = trainer.predict(X_tab=X_tab_test)\nprint(classification_report(df_test[\"target\"].to_list(), df_pred))\nprint(\"Actual predicted values:\\n{}\".format(np.unique(df_pred, return_counts=True)))\n</pre> start = time.time() trainer.fit(     X_train={\"X_tab\": X_tab_train, \"target\": y_train},     X_val={\"X_tab\": X_tab_valid, \"target\": y_valid},     n_epochs=1,     batch_size=32,     custom_dataloader=DataLoaderImbalanced,     oversample_mul=5, ) print(     \"Training time[s]: {}\".format(         datetime.timedelta(seconds=round(time.time() - start))     ) )  pd.DataFrame(trainer.history)  df_pred = trainer.predict(X_tab=X_tab_test) print(classification_report(df_test[\"target\"].to_list(), df_pred)) print(\"Actual predicted values:\\n{}\".format(np.unique(df_pred, return_counts=True))) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 325/325 [00:02&lt;00:00, 153.99it/s, loss=0.163, metrics={'acc': 0.9363, 'prec': [0.9358]}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 456/456 [00:02&lt;00:00, 205.93it/s, loss=0.1, metrics={'acc': 0.9501, 'prec': [0.1447]}]\n</pre> <pre>Training time[s]: 0:00:04\n</pre> <pre>predict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 456/456 [00:01&lt;00:00, 368.16it/s]\n</pre> <pre>              precision    recall  f1-score   support\n\n           0       1.00      0.95      0.97     14446\n           1       0.15      0.95      0.25       130\n\n    accuracy                           0.95     14576\n   macro avg       0.57      0.95      0.61     14576\nweighted avg       0.99      0.95      0.97     14576\n\nActual predicted values:\n(array([0, 1]), array([13736,   840]))\n</pre>"},{"location":"examples/08_custom_dataLoader_imbalanced_dataset.html#custom-dataloader-for-imbalanced-dataset","title":"Custom DataLoader for Imbalanced dataset\u00b6","text":""},{"location":"examples/08_custom_dataLoader_imbalanced_dataset.html#initial-imports","title":"Initial imports\u00b6","text":""},{"location":"examples/09_extracting_embeddings.html","title":"09_extracting_embeddings","text":"In\u00a0[1]: Copied! <pre>import numpy as np\nimport pandas as pd\nimport torch\n\nfrom pytorch_widedeep.preprocessing import TabPreprocessor\nfrom pytorch_widedeep.training import Trainer\nfrom pytorch_widedeep.models import FTTransformer, WideDeep\nfrom pytorch_widedeep.metrics import Accuracy\nfrom pytorch_widedeep import Tab2Vec\nfrom pytorch_widedeep.datasets import load_adult\n</pre> import numpy as np import pandas as pd import torch  from pytorch_widedeep.preprocessing import TabPreprocessor from pytorch_widedeep.training import Trainer from pytorch_widedeep.models import FTTransformer, WideDeep from pytorch_widedeep.metrics import Accuracy from pytorch_widedeep import Tab2Vec from pytorch_widedeep.datasets import load_adult In\u00a0[2]: Copied! <pre>df = load_adult(as_frame=True)\ndf.head()\n</pre> df = load_adult(as_frame=True) df.head() Out[2]: age workclass fnlwgt education educational-num marital-status occupation relationship race gender capital-gain capital-loss hours-per-week native-country income 0 25 Private 226802 11th 7 Never-married Machine-op-inspct Own-child Black Male 0 0 40 United-States &lt;=50K 1 38 Private 89814 HS-grad 9 Married-civ-spouse Farming-fishing Husband White Male 0 0 50 United-States &lt;=50K 2 28 Local-gov 336951 Assoc-acdm 12 Married-civ-spouse Protective-serv Husband White Male 0 0 40 United-States &gt;50K 3 44 Private 160323 Some-college 10 Married-civ-spouse Machine-op-inspct Husband Black Male 7688 0 40 United-States &gt;50K 4 18 ? 103497 Some-college 10 Never-married ? Own-child White Female 0 0 30 United-States &lt;=50K In\u00a0[3]: Copied! <pre># For convenience, we'll replace '-' with '_'\ndf.columns = [c.replace(\"-\", \"_\") for c in df.columns]\n# binary target\ndf[\"target\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int)\ndf.drop([\"income\", \"educational_num\"], axis=1, inplace=True)\n\ndf.head()\n</pre> # For convenience, we'll replace '-' with '_' df.columns = [c.replace(\"-\", \"_\") for c in df.columns] # binary target df[\"target\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int) df.drop([\"income\", \"educational_num\"], axis=1, inplace=True)  df.head() Out[3]: age workclass fnlwgt education marital_status occupation relationship race gender capital_gain capital_loss hours_per_week native_country target 0 25 Private 226802 11th Never-married Machine-op-inspct Own-child Black Male 0 0 40 United-States 0 1 38 Private 89814 HS-grad Married-civ-spouse Farming-fishing Husband White Male 0 0 50 United-States 0 2 28 Local-gov 336951 Assoc-acdm Married-civ-spouse Protective-serv Husband White Male 0 0 40 United-States 1 3 44 Private 160323 Some-college Married-civ-spouse Machine-op-inspct Husband Black Male 7688 0 40 United-States 1 4 18 ? 103497 Some-college Never-married ? Own-child White Female 0 0 30 United-States 0 In\u00a0[4]: Copied! <pre>cat_cols, cont_cols = [], []\nfor col in df.columns:\n    # 50 is just a random number I choose here for this example\n    if df[col].dtype == \"O\" or df[col].nunique() &lt; 50 and col != \"target\":\n        cat_cols.append(col)\n    elif col != \"target\":\n        cont_cols.append(col)\ntarget_col = \"target\"\n</pre> cat_cols, cont_cols = [], [] for col in df.columns:     # 50 is just a random number I choose here for this example     if df[col].dtype == \"O\" or df[col].nunique() &lt; 50 and col != \"target\":         cat_cols.append(col)     elif col != \"target\":         cont_cols.append(col) target_col = \"target\" In\u00a0[5]: Copied! <pre>target = df[target_col].values\n\ntab_preprocessor = TabPreprocessor(\n    embed_cols=cat_cols, continuous_cols=cont_cols, for_transformer=True\n)\nX_tab = tab_preprocessor.fit_transform(df)\n</pre> target = df[target_col].values  tab_preprocessor = TabPreprocessor(     embed_cols=cat_cols, continuous_cols=cont_cols, for_transformer=True ) X_tab = tab_preprocessor.fit_transform(df) <pre>/Users/javierrodriguezzaurin/Projects/pytorch-widedeep/pytorch_widedeep/preprocessing/tab_preprocessor.py:358: UserWarning: Continuous columns will not be normalised\n  warnings.warn(\"Continuous columns will not be normalised\")\n</pre> In\u00a0[6]: Copied! <pre>ft_transformer = FTTransformer(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=tab_preprocessor.continuous_cols,\n    embed_continuous_method=\"standard\",\n    n_blocks=2,\n    n_heads=4,\n    input_dim=16,\n)\n</pre> ft_transformer = FTTransformer(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     continuous_cols=tab_preprocessor.continuous_cols,     embed_continuous_method=\"standard\",     n_blocks=2,     n_heads=4,     input_dim=16, ) <pre>/Users/javierrodriguezzaurin/Projects/pytorch-widedeep/pytorch_widedeep/utils/general_utils.py:12: DeprecationWarning: The 'embed_continuous' parameter is deprecated and will be removed in the next release. Please use 'embed_continuous_method' instead See the documentation for more details.\n  return func(*args, **kwargs)\n</pre> In\u00a0[7]: Copied! <pre>model = WideDeep(deeptabular=ft_transformer)\ntrainer = Trainer(model, objective=\"binary\", metrics=[Accuracy])\ntrainer.fit(X_tab=X_tab, target=target, n_epochs=1, batch_size=256, val_split=0.2)\n</pre> model = WideDeep(deeptabular=ft_transformer) trainer = Trainer(model, objective=\"binary\", metrics=[Accuracy]) trainer.fit(X_tab=X_tab, target=target, n_epochs=1, batch_size=256, val_split=0.2) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:03&lt;00:00, 41.47it/s, loss=221, metrics={'acc': 0.686}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 39/39 [00:00&lt;00:00, 86.24it/s, loss=9.28, metrics={'acc': 0.76}]\n</pre> In\u00a0[8]: Copied! <pre>t2v = Tab2Vec(model=model, tab_preprocessor=tab_preprocessor)\n</pre> t2v = Tab2Vec(model=model, tab_preprocessor=tab_preprocessor) In\u00a0[9]: Copied! <pre># assuming is a test set with target col\nX_vec, y = t2v.transform(df.sample(100), target_col=\"target\")\n</pre> # assuming is a test set with target col X_vec, y = t2v.transform(df.sample(100), target_col=\"target\") In\u00a0[10]: Copied! <pre># X vec is the dataframe turned into the embeddings\nX_vec.shape\n</pre> # X vec is the dataframe turned into the embeddings X_vec.shape Out[10]: <pre>(100, 208)</pre> <p><code>208 = input_dim (16) * n_cols (13)</code></p> In\u00a0[11]: Copied! <pre># ...or if we don't have target col\nX_vec = t2v.transform(df.sample(100))\n</pre> # ...or if we don't have target col X_vec = t2v.transform(df.sample(100))"},{"location":"examples/09_extracting_embeddings.html#extracting-embeddings","title":"Extracting embeddings\u00b6","text":"<p>This notebook is a simple guide to extracting learned feature embeddings using Tab2Vec</p>"},{"location":"examples/10_3rd_party_integration-RayTune_WnB.html","title":"10_3rd_party_integration-RayTune_WnB","text":"In\u00a0[11]: Copied! <pre>from typing import Optional, Dict\nimport os\n\nimport numpy as np\nimport pandas as pd\nimport torch\nimport wandb\nfrom torch.optim import SGD, lr_scheduler\n\nfrom pytorch_widedeep import Trainer\nfrom pytorch_widedeep.preprocessing import TabPreprocessor\nfrom pytorch_widedeep.models import TabMlp, WideDeep\nfrom torchmetrics import F1Score as F1_torchmetrics\nfrom torchmetrics import Accuracy as Accuracy_torchmetrics\nfrom torchmetrics import Precision as Precision_torchmetrics\nfrom torchmetrics import Recall as Recall_torchmetrics\nfrom pytorch_widedeep.metrics import Accuracy, Recall, Precision, F1Score, R2Score\nfrom pytorch_widedeep.initializers import XavierNormal\nfrom pytorch_widedeep.callbacks import (\n    EarlyStopping,\n    ModelCheckpoint,\n    Callback,\n)\nfrom pytorch_widedeep.datasets import load_bio_kdd04\n\nfrom sklearn.model_selection import train_test_split\nimport warnings\n\nwarnings.filterwarnings(\"ignore\", category=DeprecationWarning)\n\nfrom ray import tune\nfrom ray.tune.schedulers import AsyncHyperBandScheduler\nfrom ray.tune import JupyterNotebookReporter\nfrom ray.air.integrations.wandb import WandbLoggerCallback\n\n# from ray.tune.integration.wandb import wandb_mixin\n\nimport tracemalloc\n\ntracemalloc.start()\n\n# increase displayed columns in jupyter notebook\npd.set_option(\"display.max_columns\", 200)\npd.set_option(\"display.max_rows\", 300)\n</pre> from typing import Optional, Dict import os  import numpy as np import pandas as pd import torch import wandb from torch.optim import SGD, lr_scheduler  from pytorch_widedeep import Trainer from pytorch_widedeep.preprocessing import TabPreprocessor from pytorch_widedeep.models import TabMlp, WideDeep from torchmetrics import F1Score as F1_torchmetrics from torchmetrics import Accuracy as Accuracy_torchmetrics from torchmetrics import Precision as Precision_torchmetrics from torchmetrics import Recall as Recall_torchmetrics from pytorch_widedeep.metrics import Accuracy, Recall, Precision, F1Score, R2Score from pytorch_widedeep.initializers import XavierNormal from pytorch_widedeep.callbacks import (     EarlyStopping,     ModelCheckpoint,     Callback, ) from pytorch_widedeep.datasets import load_bio_kdd04  from sklearn.model_selection import train_test_split import warnings  warnings.filterwarnings(\"ignore\", category=DeprecationWarning)  from ray import tune from ray.tune.schedulers import AsyncHyperBandScheduler from ray.tune import JupyterNotebookReporter from ray.air.integrations.wandb import WandbLoggerCallback  # from ray.tune.integration.wandb import wandb_mixin  import tracemalloc  tracemalloc.start()  # increase displayed columns in jupyter notebook pd.set_option(\"display.max_columns\", 200) pd.set_option(\"display.max_rows\", 300) In\u00a0[12]: Copied! <pre>class RayTuneReporter(Callback):\n    r\"\"\"Callback that allows reporting history and lr_history values to RayTune\n    during Hyperparameter tuning\n\n    Callbacks are passed as input parameters to the ``Trainer`` class. See\n    :class:`pytorch_widedeep.trainer.Trainer`\n\n    For examples see the examples folder at:\n\n        .. code-block:: bash\n\n            /examples/12_HyperParameter_tuning_w_RayTune.ipynb\n    \"\"\"\n\n    def on_epoch_end(\n        self, epoch: int, logs: Optional[Dict] = None, metric: Optional[float] = None\n    ):\n        report_dict = {}\n        for k, v in self.trainer.history.items():\n            report_dict.update({k: v[-1]})\n        if hasattr(self.trainer, \"lr_history\"):\n            for k, v in self.trainer.lr_history.items():\n                report_dict.update({k: v[-1]})\n        tune.report(report_dict)\n\n\nclass WnBReportBest(Callback):\n    r\"\"\"Callback that allows reporting best performance of a run to WnB\n    during Hyperparameter tuning. It is an adjusted pytorch_widedeep.callbacks.ModelCheckpoint\n    with added WnB and removed checkpoint saving.\n\n    Callbacks are passed as input parameters to the ``Trainer`` class.\n\n    Parameters\n    ----------\n    wb: obj\n        Weights&amp;Biases API interface to report single best result usable for\n        comparisson of multiple paramater combinations by, for example,\n        `parallel coordinates\n        &lt;https://docs.wandb.ai/ref/app/features/panels/parallel-coordinates&gt;`_.\n        E.g W&amp;B summary report `wandb.run.summary[\"best\"]`.\n    monitor: str, default=\"loss\"\n        quantity to monitor. Typically `'val_loss'` or metric name\n        (e.g. `'val_acc'`)\n    mode: str, default=\"auto\"\n        If ``save_best_only=True``, the decision to overwrite the current save\n        file is made based on either the maximization or the minimization of\n        the monitored quantity. For `'acc'`, this should be `'max'`, for\n        `'loss'` this should be `'min'`, etc. In `'auto'` mode, the\n        direction is automatically inferred from the name of the monitored\n        quantity.\n\n    \"\"\"\n\n    def __init__(\n        self,\n        wb: object,\n        monitor: str = \"val_loss\",\n        mode: str = \"auto\",\n    ):\n        super(WnBReportBest, self).__init__()\n\n        self.monitor = monitor\n        self.mode = mode\n        self.wb = wb\n\n        if self.mode not in [\"auto\", \"min\", \"max\"]:\n            warnings.warn(\n                \"WnBReportBest mode %s is unknown, \"\n                \"fallback to auto mode.\" % (self.mode),\n                RuntimeWarning,\n            )\n            self.mode = \"auto\"\n        if self.mode == \"min\":\n            self.monitor_op = np.less\n            self.best = np.Inf\n        elif self.mode == \"max\":\n            self.monitor_op = np.greater  # type: ignore[assignment]\n            self.best = -np.Inf\n        else:\n            if self._is_metric(self.monitor):\n                self.monitor_op = np.greater  # type: ignore[assignment]\n                self.best = -np.Inf\n            else:\n                self.monitor_op = np.less\n                self.best = np.Inf\n\n    def on_epoch_end(  # noqa: C901\n        self, epoch: int, logs: Optional[Dict] = None, metric: Optional[float] = None\n    ):\n        logs = logs or {}\n        current = logs.get(self.monitor)\n        if current is not None:\n            if self.monitor_op(current, self.best):\n                self.wb.run.summary[\"best\"] = current  # type: ignore[attr-defined]\n                self.best = current\n                self.best_epoch = epoch\n\n    @staticmethod\n    def _is_metric(monitor: str):\n        \"copied from pytorch_widedeep.callbacks\"\n        if any([s in monitor for s in [\"acc\", \"prec\", \"rec\", \"fscore\", \"f1\", \"f2\"]]):\n            return True\n        else:\n            return False\n</pre> class RayTuneReporter(Callback):     r\"\"\"Callback that allows reporting history and lr_history values to RayTune     during Hyperparameter tuning      Callbacks are passed as input parameters to the ``Trainer`` class. See     :class:`pytorch_widedeep.trainer.Trainer`      For examples see the examples folder at:          .. code-block:: bash              /examples/12_HyperParameter_tuning_w_RayTune.ipynb     \"\"\"      def on_epoch_end(         self, epoch: int, logs: Optional[Dict] = None, metric: Optional[float] = None     ):         report_dict = {}         for k, v in self.trainer.history.items():             report_dict.update({k: v[-1]})         if hasattr(self.trainer, \"lr_history\"):             for k, v in self.trainer.lr_history.items():                 report_dict.update({k: v[-1]})         tune.report(report_dict)   class WnBReportBest(Callback):     r\"\"\"Callback that allows reporting best performance of a run to WnB     during Hyperparameter tuning. It is an adjusted pytorch_widedeep.callbacks.ModelCheckpoint     with added WnB and removed checkpoint saving.      Callbacks are passed as input parameters to the ``Trainer`` class.      Parameters     ----------     wb: obj         Weights&amp;Biases API interface to report single best result usable for         comparisson of multiple paramater combinations by, for example,         `parallel coordinates         `_.         E.g W&amp;B summary report `wandb.run.summary[\"best\"]`.     monitor: str, default=\"loss\"         quantity to monitor. Typically `'val_loss'` or metric name         (e.g. `'val_acc'`)     mode: str, default=\"auto\"         If ``save_best_only=True``, the decision to overwrite the current save         file is made based on either the maximization or the minimization of         the monitored quantity. For `'acc'`, this should be `'max'`, for         `'loss'` this should be `'min'`, etc. In `'auto'` mode, the         direction is automatically inferred from the name of the monitored         quantity.      \"\"\"      def __init__(         self,         wb: object,         monitor: str = \"val_loss\",         mode: str = \"auto\",     ):         super(WnBReportBest, self).__init__()          self.monitor = monitor         self.mode = mode         self.wb = wb          if self.mode not in [\"auto\", \"min\", \"max\"]:             warnings.warn(                 \"WnBReportBest mode %s is unknown, \"                 \"fallback to auto mode.\" % (self.mode),                 RuntimeWarning,             )             self.mode = \"auto\"         if self.mode == \"min\":             self.monitor_op = np.less             self.best = np.Inf         elif self.mode == \"max\":             self.monitor_op = np.greater  # type: ignore[assignment]             self.best = -np.Inf         else:             if self._is_metric(self.monitor):                 self.monitor_op = np.greater  # type: ignore[assignment]                 self.best = -np.Inf             else:                 self.monitor_op = np.less                 self.best = np.Inf      def on_epoch_end(  # noqa: C901         self, epoch: int, logs: Optional[Dict] = None, metric: Optional[float] = None     ):         logs = logs or {}         current = logs.get(self.monitor)         if current is not None:             if self.monitor_op(current, self.best):                 self.wb.run.summary[\"best\"] = current  # type: ignore[attr-defined]                 self.best = current                 self.best_epoch = epoch      @staticmethod     def _is_metric(monitor: str):         \"copied from pytorch_widedeep.callbacks\"         if any([s in monitor for s in [\"acc\", \"prec\", \"rec\", \"fscore\", \"f1\", \"f2\"]]):             return True         else:             return False In\u00a0[13]: Copied! <pre>df = load_bio_kdd04(as_frame=True)\ndf.head()\n</pre> df = load_bio_kdd04(as_frame=True) df.head() Out[13]: EXAMPLE_ID BLOCK_ID target 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 0 279 261532 0 52.0 32.69 0.30 2.5 20.0 1256.8 -0.89 0.33 11.0 -55.0 267.2 0.52 0.05 -2.36 49.6 252.0 0.43 1.16 -2.06 -33.0 -123.2 1.60 -0.49 -6.06 65.0 296.1 -0.28 -0.26 -3.83 -22.6 -170.0 3.06 -1.05 -3.29 22.9 286.3 0.12 2.58 4.08 -33.0 -178.9 1.88 0.53 -7.0 -44.0 1987.0 -5.41 0.95 -4.0 -57.0 722.9 -3.26 -0.55 -7.5 125.5 1547.2 -0.36 1.12 9.0 -37.0 72.5 0.47 0.74 -11.0 -8.0 1595.1 -1.64 2.83 -2.0 -50.0 445.2 -0.35 0.26 0.76 1 279 261533 0 58.0 33.33 0.00 16.5 9.5 608.1 0.50 0.07 20.5 -52.5 521.6 -1.08 0.58 -0.02 -3.2 103.6 -0.95 0.23 -2.87 -25.9 -52.2 -0.21 0.87 -1.81 10.4 62.0 -0.28 -0.04 1.48 -17.6 -198.3 3.43 2.84 5.87 -16.9 72.6 -0.31 2.79 2.71 -33.5 -11.6 -1.11 4.01 5.0 -57.0 666.3 1.13 4.38 5.0 -64.0 39.3 1.07 -0.16 32.5 100.0 1893.7 -2.80 -0.22 2.5 -28.5 45.0 0.58 0.41 -19.0 -6.0 762.9 0.29 0.82 -3.0 -35.0 140.3 1.16 0.39 0.73 2 279 261534 0 77.0 27.27 -0.91 6.0 58.5 1623.6 -1.40 0.02 -6.5 -48.0 621.0 -1.20 0.14 -0.20 73.6 609.1 -0.44 -0.58 -0.04 -23.0 -27.4 -0.72 -1.04 -1.09 91.1 635.6 -0.88 0.24 0.59 -18.7 -7.2 -0.60 -2.82 -0.71 52.4 504.1 0.89 -0.67 -9.30 -20.8 -25.7 -0.77 -0.85 0.0 -20.0 2259.0 -0.94 1.15 -4.0 -44.0 -22.7 0.94 -0.98 -19.0 105.0 1267.9 1.03 1.27 11.0 -39.5 82.3 0.47 -0.19 -10.0 7.0 1491.8 0.32 -1.29 0.0 -34.0 658.2 -0.76 0.26 0.24 3 279 261535 0 41.0 27.91 -0.35 3.0 46.0 1921.6 -1.36 -0.47 -32.0 -51.5 560.9 -0.29 -0.10 -1.11 124.3 791.6 0.00 0.39 -1.85 -21.7 -44.9 -0.21 0.02 0.89 133.9 797.8 -0.08 1.06 -0.26 -16.4 -74.1 0.97 -0.80 -0.41 66.9 955.3 -1.90 1.28 -6.65 -28.1 47.5 -1.91 1.42 1.0 -30.0 1846.7 0.76 1.10 -4.0 -52.0 -53.9 1.71 -0.22 -12.0 97.5 1969.8 -1.70 0.16 -1.0 -32.5 255.9 -0.46 1.57 10.0 6.0 2047.7 -0.98 1.53 0.0 -49.0 554.2 -0.83 0.39 0.73 4 279 261536 0 50.0 28.00 -1.32 -9.0 12.0 464.8 0.88 0.19 8.0 -51.5 98.1 1.09 -0.33 -2.16 -3.9 102.7 0.39 -1.22 -3.39 -15.2 -42.2 -1.18 -1.11 -3.55 8.9 141.3 -0.16 -0.43 -4.15 -12.9 -13.4 -1.32 -0.98 -3.69 8.8 136.1 -0.30 4.13 1.89 -13.0 -18.7 -1.37 -0.93 0.0 -1.0 810.1 -2.29 6.72 1.0 -23.0 -29.7 0.58 -1.10 -18.5 33.5 206.8 1.84 -0.13 4.0 -29.0 30.1 0.80 -0.24 5.0 -14.0 479.5 0.68 -0.59 2.0 -36.0 -6.9 2.02 0.14 -0.23 In\u00a0[14]: Copied! <pre># imbalance of the classes\ndf[\"target\"].value_counts()\n</pre> # imbalance of the classes df[\"target\"].value_counts() Out[14]: <pre>target\n0    144455\n1      1296\nName: count, dtype: int64</pre> In\u00a0[15]: Copied! <pre># drop columns we won't need in this example\ndf.drop(columns=[\"EXAMPLE_ID\", \"BLOCK_ID\"], inplace=True)\n</pre> # drop columns we won't need in this example df.drop(columns=[\"EXAMPLE_ID\", \"BLOCK_ID\"], inplace=True) In\u00a0[16]: Copied! <pre>df_train, df_valid = train_test_split(\n    df, test_size=0.2, stratify=df[\"target\"], random_state=1\n)\ndf_valid, df_test = train_test_split(\n    df_valid, test_size=0.5, stratify=df_valid[\"target\"], random_state=1\n)\n</pre> df_train, df_valid = train_test_split(     df, test_size=0.2, stratify=df[\"target\"], random_state=1 ) df_valid, df_test = train_test_split(     df_valid, test_size=0.5, stratify=df_valid[\"target\"], random_state=1 ) In\u00a0[17]: Copied! <pre>continuous_cols = df.drop(columns=[\"target\"]).columns.values.tolist()\n</pre> continuous_cols = df.drop(columns=[\"target\"]).columns.values.tolist() In\u00a0[18]: Copied! <pre># deeptabular\ntab_preprocessor = TabPreprocessor(continuous_cols=continuous_cols, scale=True)\nX_tab_train = tab_preprocessor.fit_transform(df_train)\nX_tab_valid = tab_preprocessor.transform(df_valid)\nX_tab_test = tab_preprocessor.transform(df_test)\n\n# target\ny_train = df_train[\"target\"].values\ny_valid = df_valid[\"target\"].values\ny_test = df_test[\"target\"].values\n</pre> # deeptabular tab_preprocessor = TabPreprocessor(continuous_cols=continuous_cols, scale=True) X_tab_train = tab_preprocessor.fit_transform(df_train) X_tab_valid = tab_preprocessor.transform(df_valid) X_tab_test = tab_preprocessor.transform(df_test)  # target y_train = df_train[\"target\"].values y_valid = df_valid[\"target\"].values y_test = df_test[\"target\"].values In\u00a0[19]: Copied! <pre>input_layer = len(tab_preprocessor.continuous_cols)\noutput_layer = 1\nhidden_layers = np.linspace(\n    input_layer * 2, output_layer, 5, endpoint=False, dtype=int\n).tolist()\n</pre> input_layer = len(tab_preprocessor.continuous_cols) output_layer = 1 hidden_layers = np.linspace(     input_layer * 2, output_layer, 5, endpoint=False, dtype=int ).tolist() In\u00a0[20]: Copied! <pre>deeptabular = TabMlp(\n    mlp_hidden_dims=hidden_layers,\n    column_idx=tab_preprocessor.column_idx,\n    continuous_cols=tab_preprocessor.continuous_cols,\n)\nmodel = WideDeep(deeptabular=deeptabular)\nmodel\n</pre> deeptabular = TabMlp(     mlp_hidden_dims=hidden_layers,     column_idx=tab_preprocessor.column_idx,     continuous_cols=tab_preprocessor.continuous_cols, ) model = WideDeep(deeptabular=deeptabular) model Out[20]: <pre>WideDeep(\n  (deeptabular): Sequential(\n    (0): TabMlp(\n      (cont_norm): Identity()\n      (encoder): MLP(\n        (mlp): Sequential(\n          (dense_layer_0): Sequential(\n            (0): Linear(in_features=74, out_features=148, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_1): Sequential(\n            (0): Linear(in_features=148, out_features=118, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_2): Sequential(\n            (0): Linear(in_features=118, out_features=89, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_3): Sequential(\n            (0): Linear(in_features=89, out_features=59, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_4): Sequential(\n            (0): Linear(in_features=59, out_features=30, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=30, out_features=1, bias=True)\n  )\n)</pre> In\u00a0[24]: Copied! <pre># Metrics from torchmetrics\naccuracy = Accuracy_torchmetrics(average=None, num_classes=1, task=\"binary\")\nprecision = Precision_torchmetrics(average=\"micro\", num_classes=1, task=\"binary\")\nf1 = F1_torchmetrics(average=None, num_classes=1, task=\"binary\")\nrecall = Recall_torchmetrics(average=None, num_classes=1, task=\"binary\")\n</pre> # Metrics from torchmetrics accuracy = Accuracy_torchmetrics(average=None, num_classes=1, task=\"binary\") precision = Precision_torchmetrics(average=\"micro\", num_classes=1, task=\"binary\") f1 = F1_torchmetrics(average=None, num_classes=1, task=\"binary\") recall = Recall_torchmetrics(average=None, num_classes=1, task=\"binary\") <p>Note:</p> <p>Following cells includes usage of both <code>RayTuneReporter</code> and <code>WnBReportBest</code> callbacks. In case you want to use just <code>RayTuneReporter</code>, remove following:</p> <ul> <li>wandb from config</li> <li><code>WandbLoggerCallback</code></li> <li><code>WnBReportBest</code></li> <li><code>@wandb_mixin</code> decorator</li> </ul> <p>We do not see strong reason to use WnB without RayTune for a single paramater combination run, but it is possible:</p> <ul> <li>option01: define paramaters in config only for a single value <code>tune.grid_search([1000])</code> (single value RayTune run)</li> <li>option02: define WnB callback that reports currnet validation/training loss, metrics, etc. at the end of batch, ie. do not report to WnB at <code>epoch_end</code> as in <code>WnBReportBest</code> but at the <code>on_batch_end</code>, see <code>pytorch_widedeep.callbacks.Callback</code></li> </ul> In\u00a0[26]: Copied! <pre>config = {\n    \"batch_size\": tune.grid_search([1000, 5000]),\n    \"wandb\": {\n        \"project\": \"test\",\n        # \"api_key_file\": os.getcwd() + \"/wandb_api.key\",\n        \"api_key\": \"WNB_API_KEY\",\n    },\n}\n\n# Optimizers\ndeep_opt = SGD(model.deeptabular.parameters(), lr=0.1)\n# LR Scheduler\ndeep_sch = lr_scheduler.StepLR(deep_opt, step_size=3)\n\n\n@wandb_mixin\ndef training_function(config, X_train, X_val):\n    early_stopping = EarlyStopping()\n    model_checkpoint = ModelCheckpoint(save_best_only=True)\n    # Hyperparameters\n    batch_size = config[\"batch_size\"]\n    trainer = Trainer(\n        model,\n        objective=\"binary_focal_loss\",\n        callbacks=[\n            RayTuneReporter,\n            WnBReportBest(wb=wandb),\n            early_stopping,\n            model_checkpoint,\n        ],\n        lr_schedulers={\"deeptabular\": deep_sch},\n        initializers={\"deeptabular\": XavierNormal},\n        optimizers={\"deeptabular\": deep_opt},\n        metrics=[accuracy, precision, recall, f1],\n        verbose=0,\n    )\n\n    trainer.fit(X_train=X_train, X_val=X_val, n_epochs=5, batch_size=batch_size)\n\n\nX_train = {\"X_tab\": X_tab_train, \"target\": y_train}\nX_val = {\"X_tab\": X_tab_valid, \"target\": y_valid}\n\nasha_scheduler = AsyncHyperBandScheduler(\n    time_attr=\"training_iteration\",\n    metric=\"_metric/val_loss\",\n    mode=\"min\",\n    max_t=100,\n    grace_period=10,\n    reduction_factor=3,\n    brackets=1,\n)\n\nanalysis = tune.run(\n    tune.with_parameters(training_function, X_train=X_train, X_val=X_val),\n    resources_per_trial={\"cpu\": 1, \"gpu\": 0},\n    progress_reporter=JupyterNotebookReporter(overwrite=True),\n    scheduler=asha_scheduler,\n    config=config,\n    callbacks=[\n        WandbLoggerCallback(\n            project=config[\"wandb\"][\"project\"],\n            # api_key_file=config[\"wandb\"][\"api_key_file\"],\n            api_key=config[\"wandb\"][\"api_key\"],\n            log_config=True,\n        )\n    ],\n)\n</pre> config = {     \"batch_size\": tune.grid_search([1000, 5000]),     \"wandb\": {         \"project\": \"test\",         # \"api_key_file\": os.getcwd() + \"/wandb_api.key\",         \"api_key\": \"WNB_API_KEY\",     }, }  # Optimizers deep_opt = SGD(model.deeptabular.parameters(), lr=0.1) # LR Scheduler deep_sch = lr_scheduler.StepLR(deep_opt, step_size=3)   @wandb_mixin def training_function(config, X_train, X_val):     early_stopping = EarlyStopping()     model_checkpoint = ModelCheckpoint(save_best_only=True)     # Hyperparameters     batch_size = config[\"batch_size\"]     trainer = Trainer(         model,         objective=\"binary_focal_loss\",         callbacks=[             RayTuneReporter,             WnBReportBest(wb=wandb),             early_stopping,             model_checkpoint,         ],         lr_schedulers={\"deeptabular\": deep_sch},         initializers={\"deeptabular\": XavierNormal},         optimizers={\"deeptabular\": deep_opt},         metrics=[accuracy, precision, recall, f1],         verbose=0,     )      trainer.fit(X_train=X_train, X_val=X_val, n_epochs=5, batch_size=batch_size)   X_train = {\"X_tab\": X_tab_train, \"target\": y_train} X_val = {\"X_tab\": X_tab_valid, \"target\": y_valid}  asha_scheduler = AsyncHyperBandScheduler(     time_attr=\"training_iteration\",     metric=\"_metric/val_loss\",     mode=\"min\",     max_t=100,     grace_period=10,     reduction_factor=3,     brackets=1, )  analysis = tune.run(     tune.with_parameters(training_function, X_train=X_train, X_val=X_val),     resources_per_trial={\"cpu\": 1, \"gpu\": 0},     progress_reporter=JupyterNotebookReporter(overwrite=True),     scheduler=asha_scheduler,     config=config,     callbacks=[         WandbLoggerCallback(             project=config[\"wandb\"][\"project\"],             # api_key_file=config[\"wandb\"][\"api_key_file\"],             api_key=config[\"wandb\"][\"api_key\"],             log_config=True,         )     ], ) <pre>/Users/javierrodriguezzaurin/.pyenv/versions/3.10.13/lib/python3.10/tempfile.py:860: ResourceWarning: Implicitly cleaning up &lt;TemporaryDirectory '/var/folders/_2/lrjn1qn54c758tdtktr1bvkc0000gn/T/tmp60pfyl1kwandb'&gt;\n  _warnings.warn(warn_message, ResourceWarning)\n/Users/javierrodriguezzaurin/.pyenv/versions/3.10.13/lib/python3.10/tempfile.py:860: ResourceWarning: Implicitly cleaning up &lt;TemporaryDirectory '/var/folders/_2/lrjn1qn54c758tdtktr1bvkc0000gn/T/tmpnjv2rg1wwandb-artifacts'&gt;\n  _warnings.warn(warn_message, ResourceWarning)\n/Users/javierrodriguezzaurin/.pyenv/versions/3.10.13/lib/python3.10/tempfile.py:860: ResourceWarning: Implicitly cleaning up &lt;TemporaryDirectory '/var/folders/_2/lrjn1qn54c758tdtktr1bvkc0000gn/T/tmpgebu5k1kwandb-media'&gt;\n  _warnings.warn(warn_message, ResourceWarning)\n/Users/javierrodriguezzaurin/.pyenv/versions/3.10.13/lib/python3.10/tempfile.py:860: ResourceWarning: Implicitly cleaning up &lt;TemporaryDirectory '/var/folders/_2/lrjn1qn54c758tdtktr1bvkc0000gn/T/tmpxy9y2yriwandb-media'&gt;\n  _warnings.warn(warn_message, ResourceWarning)\n</pre> In\u00a0[14]: Copied! <pre>analysis.results\n</pre> analysis.results Out[14]: <pre>{'fc9a8_00000': {'_metric': {'train_loss': 0.006297602537127896,\n   'train_Accuracy': 0.9925042986869812,\n   'train_Precision': 0.9939393997192383,\n   'train_Recall': 0.15814851224422455,\n   'train_F1Score': 0.2728785574436188,\n   'val_loss': 0.005045663565397263,\n   'val_Accuracy': 0.9946483969688416,\n   'val_Precision': 1.0,\n   'val_Recall': 0.39534884691238403,\n   'val_F1Score': 0.5666667222976685},\n  'time_this_iter_s': 2.388202428817749,\n  'done': True,\n  'timesteps_total': None,\n  'episodes_total': None,\n  'training_iteration': 5,\n  'trial_id': 'fc9a8_00000',\n  'experiment_id': 'baad1d4f3d924b48b9ece1b9f26c80cc',\n  'date': '2022-07-31_14-06-51',\n  'timestamp': 1659276411,\n  'time_total_s': 12.656474113464355,\n  'pid': 1813,\n  'hostname': 'jupyter-5uperpalo',\n  'node_ip': '10.32.44.172',\n  'config': {'batch_size': 1000},\n  'time_since_restore': 12.656474113464355,\n  'timesteps_since_restore': 0,\n  'iterations_since_restore': 5,\n  'warmup_time': 0.8006253242492676,\n  'experiment_tag': '0_batch_size=1000'},\n 'fc9a8_00001': {'_metric': {'train_loss': 0.02519632239515583,\n   'train_Accuracy': 0.9910891652107239,\n   'train_Precision': 0.25,\n   'train_Recall': 0.0009643201483413577,\n   'train_F1Score': 0.0019212296465411782,\n   'val_loss': 0.02578434906899929,\n   'val_Accuracy': 0.9911492466926575,\n   'val_Precision': 0.0,\n   'val_Recall': 0.0,\n   'val_F1Score': 0.0},\n  'time_this_iter_s': 4.113586902618408,\n  'done': True,\n  'timesteps_total': None,\n  'episodes_total': None,\n  'training_iteration': 5,\n  'trial_id': 'fc9a8_00001',\n  'experiment_id': 'f2e54a6a5780429fbf0db0746853347e',\n  'date': '2022-07-31_14-06-56',\n  'timestamp': 1659276416,\n  'time_total_s': 12.926990509033203,\n  'pid': 1962,\n  'hostname': 'jupyter-5uperpalo',\n  'node_ip': '10.32.44.172',\n  'config': {'batch_size': 5000},\n  'time_since_restore': 12.926990509033203,\n  'timesteps_since_restore': 0,\n  'iterations_since_restore': 5,\n  'warmup_time': 0.9253025054931641,\n  'experiment_tag': '1_batch_size=5000'}}</pre> <p>Using Weights and Biases logging you can create parallel coordinates graphs that map parametr combinations to the best(lowest) loss achieved during the training of the networks</p> <p></p> <p>local visualization of raytune reults using tensorboard</p> In\u00a0[23]: Copied! <pre>%load_ext tensorboard\n%tensorboard --logdir ~/ray_results\n</pre> %load_ext tensorboard %tensorboard --logdir ~/ray_results"},{"location":"examples/10_3rd_party_integration-RayTune_WnB.html#3rd-party-integration-raytune-weights-biases","title":"3rd party integration - RayTune, Weights &amp; Biases\u00b6","text":"<p>This notebook provides guideline for integration of external library functions in the model training process through <code>Callback</code> objects, a popular concept of using objects as arguments for other objects.</p> <p>[DISCLAIMER]</p> <p>We show integration of RayTune (a hyperparameter tuning framework) and Weights &amp; Biases (ML projects experiment tracking and versioning solution) in the <code>pytorch_widedeep</code> model training process. We did not include <code>RayTuneReporter</code> and <code>WnBReportBest</code> in the library code to minimize the dependencies on other libraries that are not directly included in the model design and training.</p>"},{"location":"examples/10_3rd_party_integration-RayTune_WnB.html#initial-imports","title":"Initial imports\u00b6","text":""},{"location":"examples/10_3rd_party_integration-RayTune_WnB.html#preparing-the-data","title":"Preparing the data\u00b6","text":""},{"location":"examples/10_3rd_party_integration-RayTune_WnB.html#define-the-model","title":"Define the model\u00b6","text":""},{"location":"examples/11_auc_multiclass.html","title":"11_auc_multiclass","text":"In\u00a0[1]: Copied! <pre>import numpy as np\nimport pandas as pd\nfrom torch.optim import SGD, lr_scheduler\n\nfrom pytorch_widedeep import Trainer\nfrom pytorch_widedeep.preprocessing import TabPreprocessor\nfrom pytorch_widedeep.models import TabMlp, WideDeep\nfrom torchmetrics import AUROC\nfrom pytorch_widedeep.initializers import XavierNormal\nfrom pytorch_widedeep.datasets import load_ecoli\nfrom pytorch_widedeep.utils import LabelEncoder\n\nfrom sklearn.model_selection import train_test_split\n\n# increase displayed columns in jupyter notebook\npd.set_option(\"display.max_columns\", 200)\npd.set_option(\"display.max_rows\", 300)\n</pre> import numpy as np import pandas as pd from torch.optim import SGD, lr_scheduler  from pytorch_widedeep import Trainer from pytorch_widedeep.preprocessing import TabPreprocessor from pytorch_widedeep.models import TabMlp, WideDeep from torchmetrics import AUROC from pytorch_widedeep.initializers import XavierNormal from pytorch_widedeep.datasets import load_ecoli from pytorch_widedeep.utils import LabelEncoder  from sklearn.model_selection import train_test_split  # increase displayed columns in jupyter notebook pd.set_option(\"display.max_columns\", 200) pd.set_option(\"display.max_rows\", 300) In\u00a0[2]: Copied! <pre>df = load_ecoli(as_frame=True)\ndf.head()\n</pre> df = load_ecoli(as_frame=True) df.head() Out[2]: SequenceName mcg gvh lip chg aac alm1 alm2 class 0 AAT_ECOLI 0.49 0.29 0.48 0.5 0.56 0.24 0.35 cp 1 ACEA_ECOLI 0.07 0.40 0.48 0.5 0.54 0.35 0.44 cp 2 ACEK_ECOLI 0.56 0.40 0.48 0.5 0.49 0.37 0.46 cp 3 ACKA_ECOLI 0.59 0.49 0.48 0.5 0.52 0.45 0.36 cp 4 ADI_ECOLI 0.23 0.32 0.48 0.5 0.55 0.25 0.35 cp In\u00a0[3]: Copied! <pre># imbalance of the classes\ndf[\"class\"].value_counts()\n</pre> # imbalance of the classes df[\"class\"].value_counts() Out[3]: <pre>class\ncp     143\nim      77\npp      52\nimU     35\nom      20\nomL      5\nimS      2\nimL      2\nName: count, dtype: int64</pre> In\u00a0[4]: Copied! <pre>df = df.loc[~df[\"class\"].isin([\"omL\", \"imS\", \"imL\"])]\ndf.reset_index(inplace=True, drop=True)\n</pre> df = df.loc[~df[\"class\"].isin([\"omL\", \"imS\", \"imL\"])] df.reset_index(inplace=True, drop=True) In\u00a0[5]: Copied! <pre>encoder = LabelEncoder([\"class\"])\ndf_enc = encoder.fit_transform(df)\ndf_enc[\"class\"] = df_enc[\"class\"] - 1\n</pre> encoder = LabelEncoder([\"class\"]) df_enc = encoder.fit_transform(df) df_enc[\"class\"] = df_enc[\"class\"] - 1 In\u00a0[6]: Copied! <pre># drop columns we won't need in this example\ndf_enc = df_enc.drop(columns=[\"SequenceName\"])\n</pre> # drop columns we won't need in this example df_enc = df_enc.drop(columns=[\"SequenceName\"]) In\u00a0[7]: Copied! <pre>df_train, df_valid = train_test_split(\n    df_enc, test_size=0.2, stratify=df_enc[\"class\"], random_state=1\n)\ndf_valid, df_test = train_test_split(\n    df_valid, test_size=0.5, stratify=df_valid[\"class\"], random_state=1\n)\n</pre> df_train, df_valid = train_test_split(     df_enc, test_size=0.2, stratify=df_enc[\"class\"], random_state=1 ) df_valid, df_test = train_test_split(     df_valid, test_size=0.5, stratify=df_valid[\"class\"], random_state=1 ) In\u00a0[8]: Copied! <pre>continuous_cols = df_enc.drop(columns=[\"class\"]).columns.values.tolist()\n</pre> continuous_cols = df_enc.drop(columns=[\"class\"]).columns.values.tolist() In\u00a0[9]: Copied! <pre># deeptabular\ntab_preprocessor = TabPreprocessor(continuous_cols=continuous_cols, scale=True)\nX_tab_train = tab_preprocessor.fit_transform(df_train)\nX_tab_valid = tab_preprocessor.transform(df_valid)\nX_tab_test = tab_preprocessor.transform(df_test)\n\n# target\ny_train = df_train[\"class\"].values\ny_valid = df_valid[\"class\"].values\ny_test = df_test[\"class\"].values\n\nX_train = {\"X_tab\": X_tab_train, \"target\": y_train}\nX_val = {\"X_tab\": X_tab_valid, \"target\": y_valid}\n</pre> # deeptabular tab_preprocessor = TabPreprocessor(continuous_cols=continuous_cols, scale=True) X_tab_train = tab_preprocessor.fit_transform(df_train) X_tab_valid = tab_preprocessor.transform(df_valid) X_tab_test = tab_preprocessor.transform(df_test)  # target y_train = df_train[\"class\"].values y_valid = df_valid[\"class\"].values y_test = df_test[\"class\"].values  X_train = {\"X_tab\": X_tab_train, \"target\": y_train} X_val = {\"X_tab\": X_tab_valid, \"target\": y_valid} <pre>/Users/javierrodriguezzaurin/Projects/pytorch-widedeep/pytorch_widedeep/preprocessing/tab_preprocessor.py:295: DeprecationWarning: 'scale' and 'already_standard' will be deprecated in the next release. Please use 'cols_to_scale' instead\n  self._check_inputs(cat_embed_cols)\n</pre> In\u00a0[10]: Copied! <pre>deeptabular = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    continuous_cols=tab_preprocessor.continuous_cols,\n)\nmodel = WideDeep(deeptabular=deeptabular, pred_dim=df_enc[\"class\"].nunique())\nmodel\n</pre> deeptabular = TabMlp(     column_idx=tab_preprocessor.column_idx,     continuous_cols=tab_preprocessor.continuous_cols, ) model = WideDeep(deeptabular=deeptabular, pred_dim=df_enc[\"class\"].nunique()) model Out[10]: <pre>WideDeep(\n  (deeptabular): Sequential(\n    (0): TabMlp(\n      (cont_norm): Identity()\n      (encoder): MLP(\n        (mlp): Sequential(\n          (dense_layer_0): Sequential(\n            (0): Linear(in_features=7, out_features=200, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_1): Sequential(\n            (0): Linear(in_features=200, out_features=100, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=100, out_features=5, bias=True)\n  )\n)</pre> In\u00a0[11]: Copied! <pre>auroc = AUROC(num_classes=df_enc[\"class\"].nunique(), task=\"multiclass\")\n</pre> auroc = AUROC(num_classes=df_enc[\"class\"].nunique(), task=\"multiclass\") In\u00a0[12]: Copied! <pre># Optimizers\ndeep_opt = SGD(model.deeptabular.parameters(), lr=0.1)\n# LR Scheduler\ndeep_sch = lr_scheduler.StepLR(deep_opt, step_size=3)\n# Hyperparameters\ntrainer = Trainer(\n    model,\n    objective=\"multiclass_focal_loss\",\n    lr_schedulers={\"deeptabular\": deep_sch},\n    initializers={\"deeptabular\": XavierNormal},\n    optimizers={\"deeptabular\": deep_opt},\n    metrics=[auroc],\n)\n\ntrainer.fit(X_train=X_train, X_val=X_val, n_epochs=5, batch_size=50)\n</pre> # Optimizers deep_opt = SGD(model.deeptabular.parameters(), lr=0.1) # LR Scheduler deep_sch = lr_scheduler.StepLR(deep_opt, step_size=3) # Hyperparameters trainer = Trainer(     model,     objective=\"multiclass_focal_loss\",     lr_schedulers={\"deeptabular\": deep_sch},     initializers={\"deeptabular\": XavierNormal},     optimizers={\"deeptabular\": deep_opt},     metrics=[auroc], )  trainer.fit(X_train=X_train, X_val=X_val, n_epochs=5, batch_size=50) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 6/6 [00:00&lt;00:00, 54.59it/s, loss=0.109, metrics={'MulticlassAUROC': 0.314}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 1/1 [00:00&lt;00:00, 98.35it/s, loss=0.105, metrics={'MulticlassAUROC': 0.2558}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 6/6 [00:00&lt;00:00, 91.55it/s, loss=0.105, metrics={'MulticlassAUROC': 0.3546}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 1/1 [00:00&lt;00:00, 111.68it/s, loss=0.101, metrics={'MulticlassAUROC': 0.2737}]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 6/6 [00:00&lt;00:00, 62.55it/s, loss=0.1, metrics={'MulticlassAUROC': 0.3795}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 1/1 [00:00&lt;00:00, 108.51it/s, loss=0.0966, metrics={'MulticlassAUROC': 0.3053}]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 6/6 [00:00&lt;00:00, 99.35it/s, loss=0.0965, metrics={'MulticlassAUROC': 0.3809}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 1/1 [00:00&lt;00:00, 117.73it/s, loss=0.0962, metrics={'MulticlassAUROC': 0.3089}]\nepoch 5: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 6/6 [00:00&lt;00:00, 110.56it/s, loss=0.0967, metrics={'MulticlassAUROC': 0.3509}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 1/1 [00:00&lt;00:00, 127.35it/s, loss=0.0958, metrics={'MulticlassAUROC': 0.3089}]\n</pre> In\u00a0[\u00a0]: Copied! <pre>\n</pre>"},{"location":"examples/11_auc_multiclass.html#auc-multiclass-computation","title":"AUC multiclass computation\u00b6","text":""},{"location":"examples/11_auc_multiclass.html#initial-imports","title":"Initial imports\u00b6","text":""},{"location":"examples/11_auc_multiclass.html#preparing-the-data","title":"Preparing the data\u00b6","text":""},{"location":"examples/11_auc_multiclass.html#define-the-model","title":"Define the model\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html","title":"12_ZILNLoss_origkeras_vs_pytorch_widedeep","text":"In\u00a0[1]: Copied! <pre># @title Copyright 2019 The Lifetime Value Authors.\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     https://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n# ============================================================================\n</pre> # @title Copyright 2019 The Lifetime Value Authors. # Licensed under the Apache License, Version 2.0 (the \"License\"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # #     https://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an \"AS IS\" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ Run in Google Colab View source on GitHub In\u00a0[3]: Copied! <pre>import os\n\nimport numpy as np\nimport pandas as pd\nfrom scipy import stats\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport tensorflow as tf\nimport tensorflow_probability as tfp\nfrom typing import Sequence\n\n# install and import ltv\n!pip install -q git+https://github.com/google/lifetime_value\nimport lifetime_value as ltv\n</pre> import os  import numpy as np import pandas as pd from scipy import stats import matplotlib.pyplot as plt import seaborn as sns import tensorflow as tf import tensorflow_probability as tfp from typing import Sequence  # install and import ltv !pip install -q git+https://github.com/google/lifetime_value import lifetime_value as ltv In\u00a0[\u00a0]: Copied! <pre>tfd = tfp.distributions\n%config InlineBackend.figure_format='retina'\nsns.set_style(\"whitegrid\")\n</pre> tfd = tfp.distributions %config InlineBackend.figure_format='retina' sns.set_style(\"whitegrid\") In\u00a0[\u00a0]: Copied! <pre>MODEL = \"dnn\"\nLOSS = \"ziln\"  # @param { isTemplate: true, type: 'string'} ['mse', 'ziln']\nLEARNING_RATE = 0.001  # @param { isTemplate: true}\nVERSION = 0  # @param { isTemplate: true, type: 'integer'}\nOUTPUT_CSV_FOLDER = \"/tmp/lifetime-value/kdd_cup_98/result\"  # @param { isTemplate: true, type: 'string'}\n</pre> MODEL = \"dnn\" LOSS = \"ziln\"  # @param { isTemplate: true, type: 'string'} ['mse', 'ziln'] LEARNING_RATE = 0.001  # @param { isTemplate: true} VERSION = 0  # @param { isTemplate: true, type: 'integer'} OUTPUT_CSV_FOLDER = \"/tmp/lifetime-value/kdd_cup_98/result\"  # @param { isTemplate: true, type: 'string'} <p>Download kdd_cup_98 data to /tmp/lifetime-value/kdd_cup_98</p> In\u00a0[\u00a0]: Copied! <pre>%%bash\nmkdir -p /tmp/lifetime-value/kdd_cup_98\nwget https://kdd.ics.uci.edu/databases/kddcup98/epsilon_mirror/cup98lrn.zip -P /tmp/lifetime-value/kdd_cup_98/\nwget https://kdd.ics.uci.edu/databases/kddcup98/epsilon_mirror/cup98val.zip -P /tmp/lifetime-value/kdd_cup_98/\nwget https://kdd.ics.uci.edu/databases/kddcup98/epsilon_mirror/valtargt.txt -P /tmp/lifetime-value/kdd_cup_98/\ncd /tmp/lifetime-value/kdd_cup_98/\nunzip cup98lrn.zip\nunzip cup98val.zip\n</pre> %%bash mkdir -p /tmp/lifetime-value/kdd_cup_98 wget https://kdd.ics.uci.edu/databases/kddcup98/epsilon_mirror/cup98lrn.zip -P /tmp/lifetime-value/kdd_cup_98/ wget https://kdd.ics.uci.edu/databases/kddcup98/epsilon_mirror/cup98val.zip -P /tmp/lifetime-value/kdd_cup_98/ wget https://kdd.ics.uci.edu/databases/kddcup98/epsilon_mirror/valtargt.txt -P /tmp/lifetime-value/kdd_cup_98/ cd /tmp/lifetime-value/kdd_cup_98/ unzip cup98lrn.zip unzip cup98val.zip In\u00a0[\u00a0]: Copied! <pre>df_train = pd.read_csv(\"/tmp/lifetime-value/kdd_cup_98/cup98LRN.txt\")\nnum_train = df_train.shape[0]\ndf_eval = pd.read_csv(\"/tmp/lifetime-value/kdd_cup_98/cup98VAL.txt\")\ndf_eval_target = pd.read_csv(\"/tmp/lifetime-value/kdd_cup_98/valtargt.txt\")\ndf_eval = df_eval.merge(df_eval_target, on=\"CONTROLN\")\n</pre> df_train = pd.read_csv(\"/tmp/lifetime-value/kdd_cup_98/cup98LRN.txt\") num_train = df_train.shape[0] df_eval = pd.read_csv(\"/tmp/lifetime-value/kdd_cup_98/cup98VAL.txt\") df_eval_target = pd.read_csv(\"/tmp/lifetime-value/kdd_cup_98/valtargt.txt\") df_eval = df_eval.merge(df_eval_target, on=\"CONTROLN\") In\u00a0[\u00a0]: Copied! <pre>df = pd.concat([df_train, df_eval], axis=0, sort=True)\n</pre> df = pd.concat([df_train, df_eval], axis=0, sort=True) In\u00a0[\u00a0]: Copied! <pre>y = df[\"TARGET_D\"][:num_train]\n</pre> y = df[\"TARGET_D\"][:num_train] In\u00a0[\u00a0]: Copied! <pre>def plot_hist_log_scale(y):\n    max_val = y.max() + 1.0\n    ax = pd.Series(y).hist(\n        figsize=(8, 5), bins=10 ** np.linspace(0.0, np.log10(max_val), 20)\n    )\n\n    plt.xlabel(\"Donation ($)\")\n    plt.ylabel(\"Count\")\n    # plt.title('Histogram of LTV')\n    plt.xticks(rotation=\"horizontal\")\n    plt.legend(loc=\"upper left\")\n    ax.set_xscale(\"log\")\n    ax.grid(False)\n    # Hide the right and top spines\n    ax.spines[\"right\"].set_visible(False)\n    ax.spines[\"top\"].set_visible(False)\n    # Only show ticks on the left and bottom spines\n    ax.yaxis.set_ticks_position(\"left\")\n    ax.xaxis.set_ticks_position(\"bottom\")\n    plt.show()\n\n    fig = ax.get_figure()\n    output_file = tf.io.gfile.GFile(\n        \"/tmp/lifetime-value/kdd_cup_98/histogram_kdd98_log_scale.pdf\", \"wb\"\n    )\n    fig.savefig(output_file, bbox_inches=\"tight\", format=\"pdf\")\n</pre> def plot_hist_log_scale(y):     max_val = y.max() + 1.0     ax = pd.Series(y).hist(         figsize=(8, 5), bins=10 ** np.linspace(0.0, np.log10(max_val), 20)     )      plt.xlabel(\"Donation ($)\")     plt.ylabel(\"Count\")     # plt.title('Histogram of LTV')     plt.xticks(rotation=\"horizontal\")     plt.legend(loc=\"upper left\")     ax.set_xscale(\"log\")     ax.grid(False)     # Hide the right and top spines     ax.spines[\"right\"].set_visible(False)     ax.spines[\"top\"].set_visible(False)     # Only show ticks on the left and bottom spines     ax.yaxis.set_ticks_position(\"left\")     ax.xaxis.set_ticks_position(\"bottom\")     plt.show()      fig = ax.get_figure()     output_file = tf.io.gfile.GFile(         \"/tmp/lifetime-value/kdd_cup_98/histogram_kdd98_log_scale.pdf\", \"wb\"     )     fig.savefig(output_file, bbox_inches=\"tight\", format=\"pdf\") In\u00a0[\u00a0]: Copied! <pre>plot_hist_log_scale(y[y &gt; 0])\n</pre> plot_hist_log_scale(y[y &gt; 0]) In\u00a0[\u00a0]: Copied! <pre>VOCAB_FEATURES = [\n    \"ODATEDW\",  # date of donor's first gift (YYMM)\n    \"OSOURCE\",  # donor acquisition mailing list\n    \"TCODE\",  # donor title code\n    \"STATE\",\n    \"ZIP\",\n    \"DOMAIN\",  # urbanicity level and socio-economic status of the neighborhood\n    \"CLUSTER\",  # socio-economic status\n    \"GENDER\",\n    \"MAXADATE\",  # date of the most recent promotion received\n    \"MINRDATE\",\n    \"LASTDATE\",\n    \"FISTDATE\",\n    \"RFA_2A\",\n]\n</pre> VOCAB_FEATURES = [     \"ODATEDW\",  # date of donor's first gift (YYMM)     \"OSOURCE\",  # donor acquisition mailing list     \"TCODE\",  # donor title code     \"STATE\",     \"ZIP\",     \"DOMAIN\",  # urbanicity level and socio-economic status of the neighborhood     \"CLUSTER\",  # socio-economic status     \"GENDER\",     \"MAXADATE\",  # date of the most recent promotion received     \"MINRDATE\",     \"LASTDATE\",     \"FISTDATE\",     \"RFA_2A\", ] In\u00a0[\u00a0]: Copied! <pre>df[\"ODATEDW\"] = df[\"ODATEDW\"].astype(\"str\")\ndf[\"TCODE\"] = df[\"TCODE\"].apply(lambda x: \"{:03d}\".format(x // 1000 if x &gt; 1000 else x))\ndf[\"ZIP\"] = df[\"ZIP\"].str.slice(0, 5)\ndf[\"MAXADATE\"] = df[\"MAXADATE\"].astype(\"str\")\ndf[\"MINRDATE\"] = df[\"MINRDATE\"].astype(\"str\")\ndf[\"LASTDATE\"] = df[\"LASTDATE\"].astype(\"str\")\ndf[\"FISTDATE\"] = df[\"FISTDATE\"].astype(\"str\")\n</pre> df[\"ODATEDW\"] = df[\"ODATEDW\"].astype(\"str\") df[\"TCODE\"] = df[\"TCODE\"].apply(lambda x: \"{:03d}\".format(x // 1000 if x &gt; 1000 else x)) df[\"ZIP\"] = df[\"ZIP\"].str.slice(0, 5) df[\"MAXADATE\"] = df[\"MAXADATE\"].astype(\"str\") df[\"MINRDATE\"] = df[\"MINRDATE\"].astype(\"str\") df[\"LASTDATE\"] = df[\"LASTDATE\"].astype(\"str\") df[\"FISTDATE\"] = df[\"FISTDATE\"].astype(\"str\") In\u00a0[\u00a0]: Copied! <pre>def label_encoding(y, frequency_threshold=100):\n    value_counts = pd.value_counts(y)\n    categories = value_counts[value_counts &gt;= frequency_threshold].index.to_numpy()\n    # 0 indicates the unknown category.\n    return pd.Categorical(y, categories=categories).codes + 1\n</pre> def label_encoding(y, frequency_threshold=100):     value_counts = pd.value_counts(y)     categories = value_counts[value_counts &gt;= frequency_threshold].index.to_numpy()     # 0 indicates the unknown category.     return pd.Categorical(y, categories=categories).codes + 1 In\u00a0[\u00a0]: Copied! <pre>for key in VOCAB_FEATURES:\n    df[key] = label_encoding(df[key])\n</pre> for key in VOCAB_FEATURES:     df[key] = label_encoding(df[key]) In\u00a0[\u00a0]: Copied! <pre>MAIL_ORDER_RESPONSES = [\n    \"MBCRAFT\",\n    \"MBGARDEN\",\n    \"MBBOOKS\",\n    \"MBCOLECT\",\n    \"MAGFAML\",\n    \"MAGFEM\",\n    \"MAGMALE\",\n    \"PUBGARDN\",\n    \"PUBCULIN\",\n    \"PUBHLTH\",\n    \"PUBDOITY\",\n    \"PUBNEWFN\",\n    \"PUBPHOTO\",\n    \"PUBOPP\",\n    \"RFA_2F\",\n]\n</pre> MAIL_ORDER_RESPONSES = [     \"MBCRAFT\",     \"MBGARDEN\",     \"MBBOOKS\",     \"MBCOLECT\",     \"MAGFAML\",     \"MAGFEM\",     \"MAGMALE\",     \"PUBGARDN\",     \"PUBCULIN\",     \"PUBHLTH\",     \"PUBDOITY\",     \"PUBNEWFN\",     \"PUBPHOTO\",     \"PUBOPP\",     \"RFA_2F\", ] In\u00a0[\u00a0]: Copied! <pre>INDICATOR_FEATURES = [\n    \"AGE\",  # age decile, 0 indicates unknown\n    \"NUMCHLD\",\n    \"INCOME\",\n    \"WEALTH1\",\n    \"HIT\",\n] + MAIL_ORDER_RESPONSES\n</pre> INDICATOR_FEATURES = [     \"AGE\",  # age decile, 0 indicates unknown     \"NUMCHLD\",     \"INCOME\",     \"WEALTH1\",     \"HIT\", ] + MAIL_ORDER_RESPONSES In\u00a0[\u00a0]: Copied! <pre>df[\"AGE\"] = pd.qcut(df[\"AGE\"].values, 10).codes + 1\ndf[\"NUMCHLD\"] = df[\"NUMCHLD\"].apply(lambda x: 0 if np.isnan(x) else int(x))\ndf[\"INCOME\"] = df[\"INCOME\"].apply(lambda x: 0 if np.isnan(x) else int(x))\ndf[\"WEALTH1\"] = df[\"WEALTH1\"].apply(lambda x: 0 if np.isnan(x) else int(x) + 1)\ndf[\"HIT\"] = pd.qcut(df[\"HIT\"].values, q=50, duplicates=\"drop\").codes\n\nfor col in MAIL_ORDER_RESPONSES:\n    df[col] = pd.qcut(df[col].values, q=20, duplicates=\"drop\").codes + 1\n</pre> df[\"AGE\"] = pd.qcut(df[\"AGE\"].values, 10).codes + 1 df[\"NUMCHLD\"] = df[\"NUMCHLD\"].apply(lambda x: 0 if np.isnan(x) else int(x)) df[\"INCOME\"] = df[\"INCOME\"].apply(lambda x: 0 if np.isnan(x) else int(x)) df[\"WEALTH1\"] = df[\"WEALTH1\"].apply(lambda x: 0 if np.isnan(x) else int(x) + 1) df[\"HIT\"] = pd.qcut(df[\"HIT\"].values, q=50, duplicates=\"drop\").codes  for col in MAIL_ORDER_RESPONSES:     df[col] = pd.qcut(df[col].values, q=20, duplicates=\"drop\").codes + 1 In\u00a0[\u00a0]: Copied! <pre>NUMERIC_FEATURES = [\n    # binary\n    \"MAILCODE\",  # bad address\n    \"NOEXCH\",  # do not exchange\n    \"RECINHSE\",  # donor has given to PVA's in house program\n    \"RECP3\",  # donor has given to PVA's P3 program\n    \"RECPGVG\",  # planned giving record\n    \"RECSWEEP\",  # sweepstakes record\n    \"HOMEOWNR\",  # home owner\n    \"CHILD03\",\n    \"CHILD07\",\n    \"CHILD12\",\n    \"CHILD18\",\n    # continuous\n    \"CARDPROM\",\n    \"NUMPROM\",\n    \"CARDPM12\",\n    \"NUMPRM12\",\n    \"RAMNTALL\",\n    \"NGIFTALL\",\n    \"MINRAMNT\",\n    \"MAXRAMNT\",\n    \"LASTGIFT\",\n    \"AVGGIFT\",\n]\n</pre> NUMERIC_FEATURES = [     # binary     \"MAILCODE\",  # bad address     \"NOEXCH\",  # do not exchange     \"RECINHSE\",  # donor has given to PVA's in house program     \"RECP3\",  # donor has given to PVA's P3 program     \"RECPGVG\",  # planned giving record     \"RECSWEEP\",  # sweepstakes record     \"HOMEOWNR\",  # home owner     \"CHILD03\",     \"CHILD07\",     \"CHILD12\",     \"CHILD18\",     # continuous     \"CARDPROM\",     \"NUMPROM\",     \"CARDPM12\",     \"NUMPRM12\",     \"RAMNTALL\",     \"NGIFTALL\",     \"MINRAMNT\",     \"MAXRAMNT\",     \"LASTGIFT\",     \"AVGGIFT\", ] In\u00a0[\u00a0]: Copied! <pre>df[\"MAILCODE\"] = (df[\"MAILCODE\"] == \"B\").astype(\"float32\")\ndf[\"PVASTATE\"] = df[\"PVASTATE\"].isin([\"P\", \"E\"]).astype(\"float32\")\ndf[\"NOEXCH\"] = df[\"NOEXCH\"].isin([\"X\", \"1\"]).astype(\"float32\")\ndf[\"RECINHSE\"] = (df[\"RECINHSE\"] == \"X\").astype(\"float32\")\ndf[\"RECP3\"] = (df[\"RECP3\"] == \"X\").astype(\"float32\")\ndf[\"RECPGVG\"] = (df[\"RECPGVG\"] == \"X\").astype(\"float32\")\ndf[\"RECSWEEP\"] = (df[\"RECSWEEP\"] == \"X\").astype(\"float32\")\ndf[\"HOMEOWNR\"] = (df[\"HOMEOWNR\"] == \"H\").astype(\"float32\")\ndf[\"CHILD03\"] = df[\"CHILD03\"].isin([\"M\", \"F\", \"B\"]).astype(\"float32\")\ndf[\"CHILD07\"] = df[\"CHILD07\"].isin([\"M\", \"F\", \"B\"]).astype(\"float32\")\ndf[\"CHILD12\"] = df[\"CHILD12\"].isin([\"M\", \"F\", \"B\"]).astype(\"float32\")\ndf[\"CHILD18\"] = df[\"CHILD18\"].isin([\"M\", \"F\", \"B\"]).astype(\"float32\")\n\ndf[\"CARDPROM\"] = df[\"CARDPROM\"] / 100\ndf[\"NUMPROM\"] = df[\"NUMPROM\"] / 100\ndf[\"CARDPM12\"] = df[\"CARDPM12\"] / 100\ndf[\"NUMPRM12\"] = df[\"NUMPRM12\"] / 100\ndf[\"RAMNTALL\"] = np.log1p(df[\"RAMNTALL\"])\ndf[\"NGIFTALL\"] = np.log1p(df[\"NGIFTALL\"])\ndf[\"MINRAMNT\"] = np.log1p(df[\"MINRAMNT\"])\ndf[\"MAXRAMNT\"] = np.log1p(df[\"MAXRAMNT\"])\ndf[\"LASTGIFT\"] = np.log1p(df[\"LASTGIFT\"])\ndf[\"AVGGIFT\"] = np.log1p(df[\"AVGGIFT\"])\n</pre> df[\"MAILCODE\"] = (df[\"MAILCODE\"] == \"B\").astype(\"float32\") df[\"PVASTATE\"] = df[\"PVASTATE\"].isin([\"P\", \"E\"]).astype(\"float32\") df[\"NOEXCH\"] = df[\"NOEXCH\"].isin([\"X\", \"1\"]).astype(\"float32\") df[\"RECINHSE\"] = (df[\"RECINHSE\"] == \"X\").astype(\"float32\") df[\"RECP3\"] = (df[\"RECP3\"] == \"X\").astype(\"float32\") df[\"RECPGVG\"] = (df[\"RECPGVG\"] == \"X\").astype(\"float32\") df[\"RECSWEEP\"] = (df[\"RECSWEEP\"] == \"X\").astype(\"float32\") df[\"HOMEOWNR\"] = (df[\"HOMEOWNR\"] == \"H\").astype(\"float32\") df[\"CHILD03\"] = df[\"CHILD03\"].isin([\"M\", \"F\", \"B\"]).astype(\"float32\") df[\"CHILD07\"] = df[\"CHILD07\"].isin([\"M\", \"F\", \"B\"]).astype(\"float32\") df[\"CHILD12\"] = df[\"CHILD12\"].isin([\"M\", \"F\", \"B\"]).astype(\"float32\") df[\"CHILD18\"] = df[\"CHILD18\"].isin([\"M\", \"F\", \"B\"]).astype(\"float32\")  df[\"CARDPROM\"] = df[\"CARDPROM\"] / 100 df[\"NUMPROM\"] = df[\"NUMPROM\"] / 100 df[\"CARDPM12\"] = df[\"CARDPM12\"] / 100 df[\"NUMPRM12\"] = df[\"NUMPRM12\"] / 100 df[\"RAMNTALL\"] = np.log1p(df[\"RAMNTALL\"]) df[\"NGIFTALL\"] = np.log1p(df[\"NGIFTALL\"]) df[\"MINRAMNT\"] = np.log1p(df[\"MINRAMNT\"]) df[\"MAXRAMNT\"] = np.log1p(df[\"MAXRAMNT\"]) df[\"LASTGIFT\"] = np.log1p(df[\"LASTGIFT\"]) df[\"AVGGIFT\"] = np.log1p(df[\"AVGGIFT\"]) In\u00a0[\u00a0]: Copied! <pre>CATEGORICAL_FEATURES = VOCAB_FEATURES + INDICATOR_FEATURES\nALL_FEATURES = CATEGORICAL_FEATURES + NUMERIC_FEATURES\n</pre> CATEGORICAL_FEATURES = VOCAB_FEATURES + INDICATOR_FEATURES ALL_FEATURES = CATEGORICAL_FEATURES + NUMERIC_FEATURES In\u00a0[\u00a0]: Copied! <pre>def dnn_split(df):\n    df_train = df.iloc[:num_train]\n    df_eval = df.iloc[num_train:]\n\n    def feature_dict(df):\n        features = {k: v.values for k, v in dict(df[CATEGORICAL_FEATURES]).items()}\n        features[\"numeric\"] = df[NUMERIC_FEATURES].astype(\"float32\").values\n        return features\n\n    x_train, y_train = (\n        feature_dict(df_train),\n        df_train[\"TARGET_D\"].astype(\"float32\").values,\n    )\n    x_eval, y_eval = feature_dict(df_eval), df_eval[\"TARGET_D\"].astype(\"float32\").values\n\n    return x_train, x_eval, y_train, y_eval\n</pre> def dnn_split(df):     df_train = df.iloc[:num_train]     df_eval = df.iloc[num_train:]      def feature_dict(df):         features = {k: v.values for k, v in dict(df[CATEGORICAL_FEATURES]).items()}         features[\"numeric\"] = df[NUMERIC_FEATURES].astype(\"float32\").values         return features      x_train, y_train = (         feature_dict(df_train),         df_train[\"TARGET_D\"].astype(\"float32\").values,     )     x_eval, y_eval = feature_dict(df_eval), df_eval[\"TARGET_D\"].astype(\"float32\").values      return x_train, x_eval, y_train, y_eval In\u00a0[\u00a0]: Copied! <pre>def embedding_dim(x):\n    return int(x**0.25) + 1\n\n\ndef embedding_layer(vocab_size):\n    return tf.keras.Sequential(\n        [\n            tf.keras.layers.Embedding(\n                input_dim=vocab_size,\n                output_dim=embedding_dim(vocab_size),\n                input_length=1,\n            ),\n            tf.keras.layers.Flatten(),\n        ]\n    )\n\n\ndef dnn_model(output_units):\n    numeric_input = tf.keras.layers.Input(\n        shape=(len(NUMERIC_FEATURES),), name=\"numeric\"\n    )\n\n    embedding_inputs = [\n        tf.keras.layers.Input(shape=(1,), name=key, dtype=np.int64)\n        for key in CATEGORICAL_FEATURES\n    ]\n\n    embedding_outputs = [\n        embedding_layer(vocab_size=df[key].max() + 1)(input)\n        for key, input in zip(CATEGORICAL_FEATURES, embedding_inputs)\n    ]\n\n    deep_input = tf.keras.layers.concatenate([numeric_input] + embedding_outputs)\n    deep_model = tf.keras.Sequential(\n        [\n            tf.keras.layers.Dense(128, activation=\"relu\"),\n            tf.keras.layers.Dense(128, activation=\"relu\"),\n            tf.keras.layers.Dense(64, activation=\"relu\"),\n            tf.keras.layers.Dense(64, activation=\"relu\"),\n            tf.keras.layers.Dense(units=output_units),\n        ]\n    )\n    return tf.keras.Model(\n        inputs=[numeric_input] + embedding_inputs, outputs=deep_model(deep_input)\n    )\n</pre> def embedding_dim(x):     return int(x**0.25) + 1   def embedding_layer(vocab_size):     return tf.keras.Sequential(         [             tf.keras.layers.Embedding(                 input_dim=vocab_size,                 output_dim=embedding_dim(vocab_size),                 input_length=1,             ),             tf.keras.layers.Flatten(),         ]     )   def dnn_model(output_units):     numeric_input = tf.keras.layers.Input(         shape=(len(NUMERIC_FEATURES),), name=\"numeric\"     )      embedding_inputs = [         tf.keras.layers.Input(shape=(1,), name=key, dtype=np.int64)         for key in CATEGORICAL_FEATURES     ]      embedding_outputs = [         embedding_layer(vocab_size=df[key].max() + 1)(input)         for key, input in zip(CATEGORICAL_FEATURES, embedding_inputs)     ]      deep_input = tf.keras.layers.concatenate([numeric_input] + embedding_outputs)     deep_model = tf.keras.Sequential(         [             tf.keras.layers.Dense(128, activation=\"relu\"),             tf.keras.layers.Dense(128, activation=\"relu\"),             tf.keras.layers.Dense(64, activation=\"relu\"),             tf.keras.layers.Dense(64, activation=\"relu\"),             tf.keras.layers.Dense(units=output_units),         ]     )     return tf.keras.Model(         inputs=[numeric_input] + embedding_inputs, outputs=deep_model(deep_input)     ) In\u00a0[\u00a0]: Copied! <pre>if LOSS == \"mse\":\n    loss = tf.keras.losses.MeanSquaredError()\n    output_units = 1\n\nif LOSS == \"ziln\":\n    loss = ltv.zero_inflated_lognormal_loss\n    output_units = 3\n</pre> if LOSS == \"mse\":     loss = tf.keras.losses.MeanSquaredError()     output_units = 1  if LOSS == \"ziln\":     loss = ltv.zero_inflated_lognormal_loss     output_units = 3 In\u00a0[\u00a0]: Copied! <pre>x_train, x_eval, y_train, y_eval = dnn_split(df)\nmodel = dnn_model(output_units)\n</pre> x_train, x_eval, y_train, y_eval = dnn_split(df) model = dnn_model(output_units) In\u00a0[\u00a0]: Copied! <pre>model.compile(optimizer=tf.keras.optimizers.Nadam(lr=LEARNING_RATE), loss=loss)\n</pre> model.compile(optimizer=tf.keras.optimizers.Nadam(lr=LEARNING_RATE), loss=loss) In\u00a0[\u00a0]: Copied! <pre>callbacks = [\n    tf.keras.callbacks.ReduceLROnPlateau(monitor=\"val_loss\", min_lr=1e-6),\n    tf.keras.callbacks.EarlyStopping(monitor=\"val_loss\", patience=10),\n]\n</pre> callbacks = [     tf.keras.callbacks.ReduceLROnPlateau(monitor=\"val_loss\", min_lr=1e-6),     tf.keras.callbacks.EarlyStopping(monitor=\"val_loss\", patience=10), ] In\u00a0[\u00a0]: Copied! <pre>history = model.fit(\n    x=x_train,\n    y=y_train,\n    batch_size=2048,\n    epochs=200,\n    verbose=2,\n    callbacks=callbacks,\n    validation_data=(x_eval, y_eval),\n).history\n</pre> history = model.fit(     x=x_train,     y=y_train,     batch_size=2048,     epochs=200,     verbose=2,     callbacks=callbacks,     validation_data=(x_eval, y_eval), ).history In\u00a0[\u00a0]: Copied! <pre>pd.DataFrame(history)[[\"loss\", \"val_loss\"]].plot();\n</pre> pd.DataFrame(history)[[\"loss\", \"val_loss\"]].plot(); In\u00a0[\u00a0]: Copied! <pre>if LOSS == \"mse\":\n    y_pred = model.predict(x=x_eval, batch_size=1024).flatten()\n\nif LOSS == \"ziln\":\n    logits = model.predict(x=x_eval, batch_size=1024)\n    y_pred = ltv.zero_inflated_lognormal_pred(logits).numpy().flatten()\n</pre> if LOSS == \"mse\":     y_pred = model.predict(x=x_eval, batch_size=1024).flatten()  if LOSS == \"ziln\":     logits = model.predict(x=x_eval, batch_size=1024)     y_pred = ltv.zero_inflated_lognormal_pred(logits).numpy().flatten() In\u00a0[\u00a0]: Copied! <pre>from pytorch_widedeep.preprocessing import TabPreprocessor\nfrom pytorch_widedeep.training import Trainer\nfrom pytorch_widedeep.models import TabMlp, WideDeep\nfrom torch.optim.lr_scheduler import ReduceLROnPlateau\nfrom pytorch_widedeep.callbacks import EarlyStopping\nfrom torch.optim import NAdam\n</pre> from pytorch_widedeep.preprocessing import TabPreprocessor from pytorch_widedeep.training import Trainer from pytorch_widedeep.models import TabMlp, WideDeep from torch.optim.lr_scheduler import ReduceLROnPlateau from pytorch_widedeep.callbacks import EarlyStopping from torch.optim import NAdam In\u00a0[\u00a0]: Copied! <pre># CATEGORICAL_FEATURES\nNUMERICAL_FEATURES = [\"num\" + str(i) for i in range(21)]\nx_train_pyt_num = pd.DataFrame(columns=NUMERICAL_FEATURES, data=x_train[\"numeric\"])\nx_train_pyt_cat = pd.DataFrame(\n    {key: value for key, value in x_train.items() if key not in [\"numeric\"]}\n)\n\nx_eval_pyt_num = pd.DataFrame(columns=NUMERICAL_FEATURES, data=x_eval[\"numeric\"])\nx_eval_pyt_cat = pd.DataFrame(\n    {key: value for key, value in x_eval.items() if key not in [\"numeric\"]}\n)\n</pre> # CATEGORICAL_FEATURES NUMERICAL_FEATURES = [\"num\" + str(i) for i in range(21)] x_train_pyt_num = pd.DataFrame(columns=NUMERICAL_FEATURES, data=x_train[\"numeric\"]) x_train_pyt_cat = pd.DataFrame(     {key: value for key, value in x_train.items() if key not in [\"numeric\"]} )  x_eval_pyt_num = pd.DataFrame(columns=NUMERICAL_FEATURES, data=x_eval[\"numeric\"]) x_eval_pyt_cat = pd.DataFrame(     {key: value for key, value in x_eval.items() if key not in [\"numeric\"]} ) In\u00a0[\u00a0]: Copied! <pre>x_train_pyt = pd.concat([x_train_pyt_num, x_train_pyt_cat], axis=1)\nx_eval_pyt = pd.concat([x_eval_pyt_num, x_eval_pyt_cat], axis=1)\n</pre> x_train_pyt = pd.concat([x_train_pyt_num, x_train_pyt_cat], axis=1) x_eval_pyt = pd.concat([x_eval_pyt_num, x_eval_pyt_cat], axis=1) In\u00a0[\u00a0]: Copied! <pre>embed_input = [\n    (u, int(x_train_pyt[u].nunique() ** 0.25) + 1) for u in CATEGORICAL_FEATURES\n]\n</pre> embed_input = [     (u, int(x_train_pyt[u].nunique() ** 0.25) + 1) for u in CATEGORICAL_FEATURES ] In\u00a0[\u00a0]: Copied! <pre># deeptabular\ntab_preprocessor = TabPreprocessor(\n    embed_cols=embed_input,\n    continuous_cols=NUMERICAL_FEATURES,\n    shared_embed=False,\n    scale=False,\n)\nX_tab_train = tab_preprocessor.fit_transform(x_train_pyt)\nX_tab_valid = tab_preprocessor.transform(x_eval_pyt)\nX_tab_test = tab_preprocessor.transform(x_eval_pyt)\n\n# target\ny_train = y_train\ny_valid = y_eval\ny_test = y_train\n\nX_train = {\"X_tab\": X_tab_train, \"target\": y_train}\nX_val = {\"X_tab\": X_tab_valid, \"target\": y_valid}\nX_test = {\"X_tab\": X_tab_test}\n\ndeeptabular = TabMlp(\n    mlp_hidden_dims=[128, 128, 64, 64],\n    column_idx=tab_preprocessor.column_idx,\n    embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=tab_preprocessor.continuous_cols,\n)\n\nmodel = WideDeep(deeptabular=deeptabular, pred_dim=3)\n\ndeep_opt = NAdam(model.deeptabular.parameters(), lr=LEARNING_RATE)\ncallbacks = [EarlyStopping()]\ndeep_sch = ReduceLROnPlateau(deep_opt, min_lr=1e-6)\n\nobjective = \"ziln\"\n\ntrainer = Trainer(\n    model,\n    callbacks=callbacks,\n    lr_schedulers={\"deeptabular\": deep_sch},\n    objective=objective,\n    optimizers={\"deeptabular\": deep_opt},\n)\n\ntrainer.fit(\n    X_train=X_train,\n    X_val=X_val,\n    n_epochs=200,\n    batch_size=2048,\n)\n\ny_pred_pytorch = trainer.predict(X_test=X_test)\n</pre> # deeptabular tab_preprocessor = TabPreprocessor(     embed_cols=embed_input,     continuous_cols=NUMERICAL_FEATURES,     shared_embed=False,     scale=False, ) X_tab_train = tab_preprocessor.fit_transform(x_train_pyt) X_tab_valid = tab_preprocessor.transform(x_eval_pyt) X_tab_test = tab_preprocessor.transform(x_eval_pyt)  # target y_train = y_train y_valid = y_eval y_test = y_train  X_train = {\"X_tab\": X_tab_train, \"target\": y_train} X_val = {\"X_tab\": X_tab_valid, \"target\": y_valid} X_test = {\"X_tab\": X_tab_test}  deeptabular = TabMlp(     mlp_hidden_dims=[128, 128, 64, 64],     column_idx=tab_preprocessor.column_idx,     embed_input=tab_preprocessor.cat_embed_input,     continuous_cols=tab_preprocessor.continuous_cols, )  model = WideDeep(deeptabular=deeptabular, pred_dim=3)  deep_opt = NAdam(model.deeptabular.parameters(), lr=LEARNING_RATE) callbacks = [EarlyStopping()] deep_sch = ReduceLROnPlateau(deep_opt, min_lr=1e-6)  objective = \"ziln\"  trainer = Trainer(     model,     callbacks=callbacks,     lr_schedulers={\"deeptabular\": deep_sch},     objective=objective,     optimizers={\"deeptabular\": deep_opt}, )  trainer.fit(     X_train=X_train,     X_val=X_val,     n_epochs=200,     batch_size=2048, )  y_pred_pytorch = trainer.predict(X_test=X_test) In\u00a0[\u00a0]: Copied! <pre>pd.DataFrame(trainer.history)[[\"train_loss\", \"val_loss\"]].plot();\n</pre> pd.DataFrame(trainer.history)[[\"train_loss\", \"val_loss\"]].plot(); In\u00a0[\u00a0]: Copied! <pre>from sklearn.metrics import mean_squared_error\n\nmean_squared_error(y_pred, y_pred_pytorch)\n</pre> from sklearn.metrics import mean_squared_error  mean_squared_error(y_pred, y_pred_pytorch) In\u00a0[\u00a0]: Copied! <pre>unit_costs = [0.4, 0.5, 0.6, 0.68, 0.7, 0.8, 0.9, 1.0]\n</pre> unit_costs = [0.4, 0.5, 0.6, 0.68, 0.7, 0.8, 0.9, 1.0] In\u00a0[\u00a0]: Copied! <pre>num_mailed = [np.sum(y_pred &gt; v) for v in unit_costs]\nnum_mailed\n</pre> num_mailed = [np.sum(y_pred &gt; v) for v in unit_costs] num_mailed In\u00a0[\u00a0]: Copied! <pre>baseline_total_profit = np.sum(y_eval - 0.68)\nbaseline_total_profit\n</pre> baseline_total_profit = np.sum(y_eval - 0.68) baseline_total_profit In\u00a0[\u00a0]: Copied! <pre>total_profits = [np.sum(y_eval[y_pred &gt; v] - v) for v in unit_costs]\ntotal_profits\n</pre> total_profits = [np.sum(y_eval[y_pred &gt; v] - v) for v in unit_costs] total_profits In\u00a0[\u00a0]: Copied! <pre>gain = pd.DataFrame(\n    {\n        \"lorenz\": ltv.cumulative_true(y_eval, y_eval),\n        \"baseline\": ltv.cumulative_true(y_eval, x_eval[\"numeric\"][:, 19]),\n        \"model\": ltv.cumulative_true(y_eval, y_pred),\n    }\n)\n</pre> gain = pd.DataFrame(     {         \"lorenz\": ltv.cumulative_true(y_eval, y_eval),         \"baseline\": ltv.cumulative_true(y_eval, x_eval[\"numeric\"][:, 19]),         \"model\": ltv.cumulative_true(y_eval, y_pred),     } ) In\u00a0[\u00a0]: Copied! <pre>num_customers = np.float32(gain.shape[0])\ngain[\"cumulative_customer\"] = (np.arange(num_customers) + 1.0) / num_customers\n</pre> num_customers = np.float32(gain.shape[0]) gain[\"cumulative_customer\"] = (np.arange(num_customers) + 1.0) / num_customers In\u00a0[\u00a0]: Copied! <pre>ax = gain[\n    [\n        \"cumulative_customer\",\n        \"lorenz\",\n        \"baseline\",\n        \"model\",\n    ]\n].plot(x=\"cumulative_customer\", figsize=(8, 5), legend=True)\n\nax.legend([\"Groundtruth\", \"Baseline\", \"Model\"], loc=\"lower right\")\n\nax.set_xlabel(\"Cumulative Fraction of Customers\")\nax.set_xticks(np.arange(0, 1.1, 0.1))\nax.set_xlim((0, 1.0))\n\nax.set_ylabel(\"Cumulative Fraction of Total Lifetime Value\")\nax.set_yticks(np.arange(0, 1.1, 0.1))\nax.set_ylim((0, 1.05))\nax.set_title(\"Gain Chart\");\n</pre> ax = gain[     [         \"cumulative_customer\",         \"lorenz\",         \"baseline\",         \"model\",     ] ].plot(x=\"cumulative_customer\", figsize=(8, 5), legend=True)  ax.legend([\"Groundtruth\", \"Baseline\", \"Model\"], loc=\"lower right\")  ax.set_xlabel(\"Cumulative Fraction of Customers\") ax.set_xticks(np.arange(0, 1.1, 0.1)) ax.set_xlim((0, 1.0))  ax.set_ylabel(\"Cumulative Fraction of Total Lifetime Value\") ax.set_yticks(np.arange(0, 1.1, 0.1)) ax.set_ylim((0, 1.05)) ax.set_title(\"Gain Chart\"); In\u00a0[\u00a0]: Copied! <pre>gini = ltv.gini_from_gain(gain[[\"lorenz\", \"baseline\", \"model\"]])\ngini\n</pre> gini = ltv.gini_from_gain(gain[[\"lorenz\", \"baseline\", \"model\"]]) gini In\u00a0[\u00a0]: Copied! <pre>df_decile = ltv.decile_stats(y_eval, y_pred)\ndf_decile\n</pre> df_decile = ltv.decile_stats(y_eval, y_pred) df_decile In\u00a0[\u00a0]: Copied! <pre>ax = df_decile[[\"label_mean\", \"pred_mean\"]].plot.bar(rot=0)\n\nax.set_title(\"Decile Chart\")\nax.set_xlabel(\"Prediction bucket\")\nax.set_ylabel(\"Average bucket value\")\nax.legend([\"Label\", \"Prediction\"], loc=\"upper left\");\n</pre> ax = df_decile[[\"label_mean\", \"pred_mean\"]].plot.bar(rot=0)  ax.set_title(\"Decile Chart\") ax.set_xlabel(\"Prediction bucket\") ax.set_ylabel(\"Average bucket value\") ax.legend([\"Label\", \"Prediction\"], loc=\"upper left\"); In\u00a0[\u00a0]: Copied! <pre>def spearmanr(x1: Sequence[float], x2: Sequence[float]) -&gt; float:\n    \"\"\"Calculates spearmanr rank correlation coefficient.\n\n    See https://docs.scipy.org/doc/scipy/reference/stats.html.\n\n    Args:\n      x1: 1D array_like.\n      x2: 1D array_like.\n\n    Returns:\n      correlation: float.\n    \"\"\"\n    return stats.spearmanr(x1, x2, nan_policy=\"raise\")[0]\n\n\nspearman_corr = spearmanr(y_eval, y_pred)\nspearman_corr\n</pre> def spearmanr(x1: Sequence[float], x2: Sequence[float]) -&gt; float:     \"\"\"Calculates spearmanr rank correlation coefficient.      See https://docs.scipy.org/doc/scipy/reference/stats.html.      Args:       x1: 1D array_like.       x2: 1D array_like.      Returns:       correlation: float.     \"\"\"     return stats.spearmanr(x1, x2, nan_policy=\"raise\")[0]   spearman_corr = spearmanr(y_eval, y_pred) spearman_corr In\u00a0[\u00a0]: Copied! <pre>df_metrics = pd.DataFrame(\n    {\n        \"model\": MODEL,\n        \"loss_function\": LOSS,\n        \"train_loss\": history[\"loss\"][-1],\n        \"eval_loss\": history[\"val_loss\"][-1],\n        \"label_positive\": np.mean(y_eval &gt; 0),\n        \"label_mean\": y_eval.mean(),\n        \"pred_mean\": y_pred.mean(),\n        \"decile_mape\": df_decile[\"decile_mape\"].mean(),\n        \"baseline_gini\": gini[\"normalized\"][1],\n        \"gini\": gini[\"normalized\"][2],\n        \"spearman_corr\": spearman_corr,\n    },\n    index=[VERSION],\n)\n</pre> df_metrics = pd.DataFrame(     {         \"model\": MODEL,         \"loss_function\": LOSS,         \"train_loss\": history[\"loss\"][-1],         \"eval_loss\": history[\"val_loss\"][-1],         \"label_positive\": np.mean(y_eval &gt; 0),         \"label_mean\": y_eval.mean(),         \"pred_mean\": y_pred.mean(),         \"decile_mape\": df_decile[\"decile_mape\"].mean(),         \"baseline_gini\": gini[\"normalized\"][1],         \"gini\": gini[\"normalized\"][2],         \"spearman_corr\": spearman_corr,     },     index=[VERSION], ) In\u00a0[\u00a0]: Copied! <pre>for unit_cost, total_profit in zip(unit_costs, total_profits):\n    df_metrics[\"total_profit_{:02d}\".format(int(unit_cost * 100))] = total_profit\n</pre> for unit_cost, total_profit in zip(unit_costs, total_profits):     df_metrics[\"total_profit_{:02d}\".format(int(unit_cost * 100))] = total_profit In\u00a0[\u00a0]: Copied! <pre>df_metrics.T\n</pre> df_metrics.T In\u00a0[\u00a0]: Copied! <pre>output_path = OUTPUT_CSV_FOLDER\n</pre> output_path = OUTPUT_CSV_FOLDER In\u00a0[\u00a0]: Copied! <pre>if not os.path.isdir(output_path):\n    os.makedirs(output_path)\n</pre> if not os.path.isdir(output_path):     os.makedirs(output_path) In\u00a0[\u00a0]: Copied! <pre>output_file = os.path.join(\n    output_path, \"{}_regression_{}_{}.csv\".format(MODEL, LOSS, VERSION)\n)\n</pre> output_file = os.path.join(     output_path, \"{}_regression_{}_{}.csv\".format(MODEL, LOSS, VERSION) ) In\u00a0[\u00a0]: Copied! <pre>df_metrics.to_csv(output_file, index=False)\n</pre> df_metrics.to_csv(output_file, index=False)"},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#zilnloss","title":"ZILNLoss\u00b6","text":"<p>[DISCLAIMER]</p> <p>Purpose of this notebook is to check if ZILNloss implemented originaly Keras give same results in pytorch-widedeep implemenatation</p>"},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#kdd-cup-98-ltv-prediction","title":"KDD Cup 98 LTV Prediction\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#configs","title":"Configs\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#load-data","title":"Load data\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#label-distribution","title":"Label distribution\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#preprocess-features","title":"Preprocess features\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#vocab","title":"Vocab\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#indicator","title":"Indicator\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#numeric","title":"Numeric\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#all","title":"All\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#traineval-split","title":"Train/eval split\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#model","title":"Model\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#loss","title":"Loss\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#train","title":"Train\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#eval","title":"Eval\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#pytorch-widedeep-approach","title":"Pytorch-widedeep approach\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#appendix","title":"Appendix\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#total-profit","title":"Total Profit\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#gini-coefficient","title":"Gini Coefficient\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#calibration","title":"Calibration\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#rank-correlation","title":"Rank Correlation\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#all-metrics-together","title":"All metrics together\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#save","title":"Save\u00b6","text":""},{"location":"examples/13_model_uncertainty_prediction.html","title":"13_model_uncertainty_prediction","text":"<ul> <li>In this notebook we will use the higly imbalanced Protein Homology Dataset from KDD cup 2004</li> </ul> <pre><code>* The first element of each line is a BLOCK ID that denotes to which native sequence this example belongs. There is a unique BLOCK ID for each native sequence. BLOCK IDs are integers running from 1 to 303 (one for each native sequence, i.e. for each query). BLOCK IDs were assigned before the blocks were split into the train and test sets, so they do not run consecutively in either file.\n* The second element of each line is an EXAMPLE ID that uniquely describes the example. You will need this EXAMPLE ID and the BLOCK ID when you submit results.\n* The third element is the class of the example. Proteins that are homologous to the native sequence are denoted by 1, non-homologous proteins (i.e. decoys) by 0. Test examples have a \"?\" in this position.\n* All following elements are feature values. There are 74 feature values in each line. The features describe the match (e.g. the score of a sequence alignment) between the native protein sequence and the sequence that is tested for homology.\n</code></pre> In\u00a0[1]: Copied! <pre>import numpy as np\nimport pandas as pd\nimport torch\nfrom torch.optim import SGD, lr_scheduler\n\nfrom pytorch_widedeep import Trainer\nfrom pytorch_widedeep.preprocessing import TabPreprocessor\nfrom pytorch_widedeep.models import TabMlp, WideDeep\nfrom pytorch_widedeep.dataloaders import DataLoaderImbalanced, DataLoaderDefault\nfrom torchmetrics import F1Score as F1_torchmetrics\nfrom torchmetrics import Accuracy as Accuracy_torchmetrics\nfrom torchmetrics import Precision as Precision_torchmetrics\nfrom torchmetrics import Recall as Recall_torchmetrics\nfrom pytorch_widedeep.metrics import Accuracy, Recall, Precision, F1Score, R2Score\nfrom pytorch_widedeep.initializers import XavierNormal\nfrom pytorch_widedeep.datasets import load_bio_kdd04\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import classification_report\n\nimport time\nimport datetime\n\nimport warnings\n\nwarnings.filterwarnings(\"ignore\", category=DeprecationWarning)\n\n# increase displayed columns in jupyter notebook\npd.set_option(\"display.max_columns\", 200)\npd.set_option(\"display.max_rows\", 300)\n</pre> import numpy as np import pandas as pd import torch from torch.optim import SGD, lr_scheduler  from pytorch_widedeep import Trainer from pytorch_widedeep.preprocessing import TabPreprocessor from pytorch_widedeep.models import TabMlp, WideDeep from pytorch_widedeep.dataloaders import DataLoaderImbalanced, DataLoaderDefault from torchmetrics import F1Score as F1_torchmetrics from torchmetrics import Accuracy as Accuracy_torchmetrics from torchmetrics import Precision as Precision_torchmetrics from torchmetrics import Recall as Recall_torchmetrics from pytorch_widedeep.metrics import Accuracy, Recall, Precision, F1Score, R2Score from pytorch_widedeep.initializers import XavierNormal from pytorch_widedeep.datasets import load_bio_kdd04  from sklearn.model_selection import train_test_split from sklearn.metrics import classification_report  import time import datetime  import warnings  warnings.filterwarnings(\"ignore\", category=DeprecationWarning)  # increase displayed columns in jupyter notebook pd.set_option(\"display.max_columns\", 200) pd.set_option(\"display.max_rows\", 300) In\u00a0[2]: Copied! <pre>df = load_bio_kdd04(as_frame=True)\ndf.head()\n</pre> df = load_bio_kdd04(as_frame=True) df.head() Out[2]: EXAMPLE_ID BLOCK_ID target 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 0 279 261532 0 52.0 32.69 0.30 2.5 20.0 1256.8 -0.89 0.33 11.0 -55.0 267.2 0.52 0.05 -2.36 49.6 252.0 0.43 1.16 -2.06 -33.0 -123.2 1.60 -0.49 -6.06 65.0 296.1 -0.28 -0.26 -3.83 -22.6 -170.0 3.06 -1.05 -3.29 22.9 286.3 0.12 2.58 4.08 -33.0 -178.9 1.88 0.53 -7.0 -44.0 1987.0 -5.41 0.95 -4.0 -57.0 722.9 -3.26 -0.55 -7.5 125.5 1547.2 -0.36 1.12 9.0 -37.0 72.5 0.47 0.74 -11.0 -8.0 1595.1 -1.64 2.83 -2.0 -50.0 445.2 -0.35 0.26 0.76 1 279 261533 0 58.0 33.33 0.00 16.5 9.5 608.1 0.50 0.07 20.5 -52.5 521.6 -1.08 0.58 -0.02 -3.2 103.6 -0.95 0.23 -2.87 -25.9 -52.2 -0.21 0.87 -1.81 10.4 62.0 -0.28 -0.04 1.48 -17.6 -198.3 3.43 2.84 5.87 -16.9 72.6 -0.31 2.79 2.71 -33.5 -11.6 -1.11 4.01 5.0 -57.0 666.3 1.13 4.38 5.0 -64.0 39.3 1.07 -0.16 32.5 100.0 1893.7 -2.80 -0.22 2.5 -28.5 45.0 0.58 0.41 -19.0 -6.0 762.9 0.29 0.82 -3.0 -35.0 140.3 1.16 0.39 0.73 2 279 261534 0 77.0 27.27 -0.91 6.0 58.5 1623.6 -1.40 0.02 -6.5 -48.0 621.0 -1.20 0.14 -0.20 73.6 609.1 -0.44 -0.58 -0.04 -23.0 -27.4 -0.72 -1.04 -1.09 91.1 635.6 -0.88 0.24 0.59 -18.7 -7.2 -0.60 -2.82 -0.71 52.4 504.1 0.89 -0.67 -9.30 -20.8 -25.7 -0.77 -0.85 0.0 -20.0 2259.0 -0.94 1.15 -4.0 -44.0 -22.7 0.94 -0.98 -19.0 105.0 1267.9 1.03 1.27 11.0 -39.5 82.3 0.47 -0.19 -10.0 7.0 1491.8 0.32 -1.29 0.0 -34.0 658.2 -0.76 0.26 0.24 3 279 261535 0 41.0 27.91 -0.35 3.0 46.0 1921.6 -1.36 -0.47 -32.0 -51.5 560.9 -0.29 -0.10 -1.11 124.3 791.6 0.00 0.39 -1.85 -21.7 -44.9 -0.21 0.02 0.89 133.9 797.8 -0.08 1.06 -0.26 -16.4 -74.1 0.97 -0.80 -0.41 66.9 955.3 -1.90 1.28 -6.65 -28.1 47.5 -1.91 1.42 1.0 -30.0 1846.7 0.76 1.10 -4.0 -52.0 -53.9 1.71 -0.22 -12.0 97.5 1969.8 -1.70 0.16 -1.0 -32.5 255.9 -0.46 1.57 10.0 6.0 2047.7 -0.98 1.53 0.0 -49.0 554.2 -0.83 0.39 0.73 4 279 261536 0 50.0 28.00 -1.32 -9.0 12.0 464.8 0.88 0.19 8.0 -51.5 98.1 1.09 -0.33 -2.16 -3.9 102.7 0.39 -1.22 -3.39 -15.2 -42.2 -1.18 -1.11 -3.55 8.9 141.3 -0.16 -0.43 -4.15 -12.9 -13.4 -1.32 -0.98 -3.69 8.8 136.1 -0.30 4.13 1.89 -13.0 -18.7 -1.37 -0.93 0.0 -1.0 810.1 -2.29 6.72 1.0 -23.0 -29.7 0.58 -1.10 -18.5 33.5 206.8 1.84 -0.13 4.0 -29.0 30.1 0.80 -0.24 5.0 -14.0 479.5 0.68 -0.59 2.0 -36.0 -6.9 2.02 0.14 -0.23 In\u00a0[3]: Copied! <pre># imbalance of the classes\ndf[\"target\"].value_counts()\n</pre> # imbalance of the classes df[\"target\"].value_counts() Out[3]: <pre>target\n0    144455\n1      1296\nName: count, dtype: int64</pre> In\u00a0[4]: Copied! <pre># drop columns we won't need in this example\ndf.drop(columns=[\"EXAMPLE_ID\", \"BLOCK_ID\"], inplace=True)\n</pre> # drop columns we won't need in this example df.drop(columns=[\"EXAMPLE_ID\", \"BLOCK_ID\"], inplace=True) In\u00a0[5]: Copied! <pre>df_train, df_valid = train_test_split(\n    df, test_size=0.2, stratify=df[\"target\"], random_state=1\n)\ndf_valid, df_test = train_test_split(\n    df_valid, test_size=0.5, stratify=df_valid[\"target\"], random_state=1\n)\n</pre> df_train, df_valid = train_test_split(     df, test_size=0.2, stratify=df[\"target\"], random_state=1 ) df_valid, df_test = train_test_split(     df_valid, test_size=0.5, stratify=df_valid[\"target\"], random_state=1 ) In\u00a0[6]: Copied! <pre>continuous_cols = df.drop(columns=[\"target\"]).columns.values.tolist()\n</pre> continuous_cols = df.drop(columns=[\"target\"]).columns.values.tolist() In\u00a0[7]: Copied! <pre># deeptabular\ntab_preprocessor = TabPreprocessor(continuous_cols=continuous_cols, scale=True)\nX_tab_train = tab_preprocessor.fit_transform(df_train)\nX_tab_valid = tab_preprocessor.transform(df_valid)\nX_tab_test = tab_preprocessor.transform(df_test)\n\n# target\ny_train = df_train[\"target\"].values\ny_valid = df_valid[\"target\"].values\ny_test = df_test[\"target\"].values\n</pre> # deeptabular tab_preprocessor = TabPreprocessor(continuous_cols=continuous_cols, scale=True) X_tab_train = tab_preprocessor.fit_transform(df_train) X_tab_valid = tab_preprocessor.transform(df_valid) X_tab_test = tab_preprocessor.transform(df_test)  # target y_train = df_train[\"target\"].values y_valid = df_valid[\"target\"].values y_test = df_test[\"target\"].values In\u00a0[8]: Copied! <pre>input_layer = len(tab_preprocessor.continuous_cols)\noutput_layer = 1\nhidden_layers = np.linspace(\n    input_layer * 2, output_layer, 5, endpoint=False, dtype=int\n).tolist()\n</pre> input_layer = len(tab_preprocessor.continuous_cols) output_layer = 1 hidden_layers = np.linspace(     input_layer * 2, output_layer, 5, endpoint=False, dtype=int ).tolist() In\u00a0[9]: Copied! <pre>deeptabular = TabMlp(\n    mlp_hidden_dims=hidden_layers,\n    column_idx=tab_preprocessor.column_idx,\n    continuous_cols=tab_preprocessor.continuous_cols,\n)\nmodel = WideDeep(deeptabular=deeptabular, pred_dim=1)\nmodel\n</pre> deeptabular = TabMlp(     mlp_hidden_dims=hidden_layers,     column_idx=tab_preprocessor.column_idx,     continuous_cols=tab_preprocessor.continuous_cols, ) model = WideDeep(deeptabular=deeptabular, pred_dim=1) model Out[9]: <pre>WideDeep(\n  (deeptabular): Sequential(\n    (0): TabMlp(\n      (cont_norm): Identity()\n      (encoder): MLP(\n        (mlp): Sequential(\n          (dense_layer_0): Sequential(\n            (0): Linear(in_features=74, out_features=148, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_1): Sequential(\n            (0): Linear(in_features=148, out_features=118, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_2): Sequential(\n            (0): Linear(in_features=118, out_features=89, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_3): Sequential(\n            (0): Linear(in_features=89, out_features=59, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_4): Sequential(\n            (0): Linear(in_features=59, out_features=30, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=30, out_features=1, bias=True)\n  )\n)</pre> In\u00a0[10]: Copied! <pre># # Metrics from torchmetrics\n# accuracy = Accuracy_torchmetrics(average=None, num_classes=1)\n# precision = Precision_torchmetrics(average=\"micro\", num_classes=1)\n# f1 = F1_torchmetrics(average=None, num_classes=1)\n# recall = Recall_torchmetrics(average=None, num_classes=1)\n</pre> # # Metrics from torchmetrics # accuracy = Accuracy_torchmetrics(average=None, num_classes=1) # precision = Precision_torchmetrics(average=\"micro\", num_classes=1) # f1 = F1_torchmetrics(average=None, num_classes=1) # recall = Recall_torchmetrics(average=None, num_classes=1) In\u00a0[11]: Copied! <pre># Metrics from pytorch-widedeep\naccuracy = Accuracy(top_k=2)\nprecision = Precision(average=False)\nrecall = Recall(average=True)\nf1 = F1Score(average=False)\n</pre> # Metrics from pytorch-widedeep accuracy = Accuracy(top_k=2) precision = Precision(average=False) recall = Recall(average=True) f1 = F1Score(average=False) In\u00a0[12]: Copied! <pre># Optimizers\ndeep_opt = SGD(model.deeptabular.parameters(), lr=0.1)\n# LR Scheduler\ndeep_sch = lr_scheduler.StepLR(deep_opt, step_size=3)\n\ntrainer = Trainer(\n    model,\n    objective=\"binary\",\n    lr_schedulers={\"deeptabular\": deep_sch},\n    initializers={\"deeptabular\": XavierNormal},\n    optimizers={\"deeptabular\": deep_opt},\n    metrics=[accuracy, precision, recall, f1],\n    verbose=1,\n)\n</pre> # Optimizers deep_opt = SGD(model.deeptabular.parameters(), lr=0.1) # LR Scheduler deep_sch = lr_scheduler.StepLR(deep_opt, step_size=3)  trainer = Trainer(     model,     objective=\"binary\",     lr_schedulers={\"deeptabular\": deep_sch},     initializers={\"deeptabular\": XavierNormal},     optimizers={\"deeptabular\": deep_opt},     metrics=[accuracy, precision, recall, f1],     verbose=1, ) In\u00a0[13]: Copied! <pre>start = time.time()\ntrainer.fit(\n    X_train={\"X_tab\": X_tab_train, \"target\": y_train},\n    X_val={\"X_tab\": X_tab_valid, \"target\": y_valid},\n    n_epochs=3,\n    batch_size=50,\n    custom_dataloader=DataLoaderImbalanced,\n    oversample_mul=5,\n)\nprint(\n    \"Training time[s]: {}\".format(\n        datetime.timedelta(seconds=round(time.time() - start))\n    )\n)\n</pre> start = time.time() trainer.fit(     X_train={\"X_tab\": X_tab_train, \"target\": y_train},     X_val={\"X_tab\": X_tab_valid, \"target\": y_valid},     n_epochs=3,     batch_size=50,     custom_dataloader=DataLoaderImbalanced,     oversample_mul=5, ) print(     \"Training time[s]: {}\".format(         datetime.timedelta(seconds=round(time.time() - start))     ) ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 208/208 [00:01&lt;00:00, 130.75it/s, loss=0.187, metrics={'acc': 0.9214, 'prec': [0.9149], 'rec': 0.9318, 'f1': [0.9233]}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 292/292 [00:01&lt;00:00, 173.71it/s, loss=0.106, metrics={'acc': 0.9499, 'prec': [0.1435], 'rec': 0.938, 'f1': [0.249]}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 208/208 [00:01&lt;00:00, 139.68it/s, loss=0.109, metrics={'acc': 0.9559, 'prec': [0.9537], 'rec': 0.9572, 'f1': [0.9554]}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 292/292 [00:01&lt;00:00, 177.53it/s, loss=0.0888, metrics={'acc': 0.9602, 'prec': [0.1755], 'rec': 0.9457, 'f1': [0.2961]}]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 208/208 [00:01&lt;00:00, 141.63it/s, loss=0.08, metrics={'acc': 0.9706, 'prec': [0.9648], 'rec': 0.9766, 'f1': [0.9707]}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 292/292 [00:01&lt;00:00, 165.36it/s, loss=0.0969, metrics={'acc': 0.9564, 'prec': [0.1636], 'rec': 0.9535, 'f1': [0.2792]}]</pre> <pre>Training time[s]: 0:00:10\n</pre> <pre>\n</pre> In\u00a0[14]: Copied! <pre>pd.DataFrame(trainer.history)\n</pre> pd.DataFrame(trainer.history) Out[14]: train_loss train_acc train_prec train_rec train_f1 val_loss val_acc val_prec val_rec val_f1 0 0.186707 0.921408 [0.9149412512779236] 0.931801 [0.9232940673828125] 0.106023 0.949914 [0.14353498816490173] 0.937984 [0.24897116422653198] 1 0.109498 0.955931 [0.9536514282226562] 0.957193 [0.9554190039634705] 0.088787 0.960206 [0.17553956806659698] 0.945736 [0.29611650109291077] 2 0.079979 0.970588 [0.9648183584213257] 0.976582 [0.9706646203994751] 0.096858 0.956432 [0.1635638326406479] 0.953488 [0.279228150844574] In\u00a0[15]: Copied! <pre>df_pred = trainer.predict(X_tab=X_tab_test)\nprint(classification_report(df_test[\"target\"].to_list(), df_pred))\nprint(\"Actual predicted values:\\n{}\".format(np.unique(df_pred, return_counts=True)))\n</pre> df_pred = trainer.predict(X_tab=X_tab_test) print(classification_report(df_test[\"target\"].to_list(), df_pred)) print(\"Actual predicted values:\\n{}\".format(np.unique(df_pred, return_counts=True))) <pre>predict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 292/292 [00:00&lt;00:00, 346.55it/s]\n</pre> <pre>              precision    recall  f1-score   support\n\n           0       1.00      0.96      0.98     14446\n           1       0.17      0.95      0.29       130\n\n    accuracy                           0.96     14576\n   macro avg       0.58      0.95      0.63     14576\nweighted avg       0.99      0.96      0.97     14576\n\nActual predicted values:\n(array([0, 1]), array([13845,   731]))\n</pre> In\u00a0[16]: Copied! <pre>df_pred_unc = trainer.predict_uncertainty(X_tab=X_tab_test, uncertainty_granularity=10)\nprint(classification_report(df_test[\"target\"].to_list(), df_pred))\nprint(\n    \"Actual predicted values:\\n{}\".format(\n        np.unique(df_pred_unc[:, -1], return_counts=True)\n    )\n)\n</pre> df_pred_unc = trainer.predict_uncertainty(X_tab=X_tab_test, uncertainty_granularity=10) print(classification_report(df_test[\"target\"].to_list(), df_pred)) print(     \"Actual predicted values:\\n{}\".format(         np.unique(df_pred_unc[:, -1], return_counts=True)     ) ) <pre>predict_UncertaintyIter: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 10/10 [00:03&lt;00:00,  3.25it/s]</pre> <pre>              precision    recall  f1-score   support\n\n           0       1.00      0.96      0.98     14446\n           1       0.17      0.95      0.29       130\n\n    accuracy                           0.96     14576\n   macro avg       0.58      0.95      0.63     14576\nweighted avg       0.99      0.96      0.97     14576\n\nActual predicted values:\n(array([0.]), array([14576]))\n</pre> <pre>\n</pre> In\u00a0[17]: Copied! <pre>df_pred_unc\n</pre> df_pred_unc Out[17]: <pre>array([[9.98401165e-01, 1.59881881e-03, 0.00000000e+00],\n       [9.99941409e-01, 5.85634953e-05, 0.00000000e+00],\n       [9.97351170e-01, 2.64881272e-03, 0.00000000e+00],\n       ...,\n       [9.99494374e-01, 5.05603210e-04, 0.00000000e+00],\n       [9.99981642e-01, 1.83574630e-05, 0.00000000e+00],\n       [9.99996483e-01, 3.52600046e-06, 0.00000000e+00]])</pre>"},{"location":"examples/13_model_uncertainty_prediction.html#model-uncertainty-prediction","title":"Model Uncertainty prediction\u00b6","text":"<p>Note:</p> <p>This notebook extends the \"Custom DataLoader for Imbalanced dataset\" notebook</p>"},{"location":"examples/13_model_uncertainty_prediction.html#initial-imports","title":"Initial imports\u00b6","text":""},{"location":"examples/13_model_uncertainty_prediction.html#preparing-the-data","title":"Preparing the data\u00b6","text":""},{"location":"examples/13_model_uncertainty_prediction.html#define-the-model","title":"Define the model\u00b6","text":""},{"location":"examples/13_model_uncertainty_prediction.html#normal-prediction","title":"\"Normal\" prediction\u00b6","text":""},{"location":"examples/13_model_uncertainty_prediction.html#prediction-using-uncertainty","title":"Prediction using uncertainty\u00b6","text":""},{"location":"examples/14_bayesian_models.html","title":"14_bayesian_models","text":"In\u00a0[1]: Copied! <pre>import numpy as np\nimport torch\nimport pandas as pd\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import accuracy_score\n\nfrom pytorch_widedeep.metrics import Accuracy\nfrom pytorch_widedeep.datasets import load_adult\nfrom pytorch_widedeep.callbacks import EarlyStopping, ModelCheckpoint\nfrom pytorch_widedeep.preprocessing import TabPreprocessor, WidePreprocessor\nfrom pytorch_widedeep.bayesian_models import BayesianWide, BayesianTabMlp\nfrom pytorch_widedeep.training.bayesian_trainer import BayesianTrainer\n</pre> import numpy as np import torch import pandas as pd  from sklearn.model_selection import train_test_split from sklearn.metrics import accuracy_score  from pytorch_widedeep.metrics import Accuracy from pytorch_widedeep.datasets import load_adult from pytorch_widedeep.callbacks import EarlyStopping, ModelCheckpoint from pytorch_widedeep.preprocessing import TabPreprocessor, WidePreprocessor from pytorch_widedeep.bayesian_models import BayesianWide, BayesianTabMlp from pytorch_widedeep.training.bayesian_trainer import BayesianTrainer <p>The first few things to do we know them very well, like with any other model described in any of the other notebooks</p> In\u00a0[2]: Copied! <pre>df = load_adult(as_frame=True)\ndf.columns = [c.replace(\"-\", \"_\") for c in df.columns]\ndf[\"age_buckets\"] = pd.cut(\n    df.age, bins=[16, 25, 30, 35, 40, 45, 50, 55, 60, 91], labels=np.arange(9)\n)\ndf[\"income_label\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int)\ndf.drop(\"income\", axis=1, inplace=True)\ndf.head()\n</pre> df = load_adult(as_frame=True) df.columns = [c.replace(\"-\", \"_\") for c in df.columns] df[\"age_buckets\"] = pd.cut(     df.age, bins=[16, 25, 30, 35, 40, 45, 50, 55, 60, 91], labels=np.arange(9) ) df[\"income_label\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int) df.drop(\"income\", axis=1, inplace=True) df.head() Out[2]: age workclass fnlwgt education educational_num marital_status occupation relationship race gender capital_gain capital_loss hours_per_week native_country age_buckets income_label 0 25 Private 226802 11th 7 Never-married Machine-op-inspct Own-child Black Male 0 0 40 United-States 0 0 1 38 Private 89814 HS-grad 9 Married-civ-spouse Farming-fishing Husband White Male 0 0 50 United-States 3 0 2 28 Local-gov 336951 Assoc-acdm 12 Married-civ-spouse Protective-serv Husband White Male 0 0 40 United-States 1 1 3 44 Private 160323 Some-college 10 Married-civ-spouse Machine-op-inspct Husband Black Male 7688 0 40 United-States 4 1 4 18 ? 103497 Some-college 10 Never-married ? Own-child White Female 0 0 30 United-States 0 0 In\u00a0[3]: Copied! <pre>train, test = train_test_split(df, test_size=0.2, stratify=df.income_label)\n</pre> train, test = train_test_split(df, test_size=0.2, stratify=df.income_label) In\u00a0[4]: Copied! <pre>wide_cols = [\n    \"age_buckets\",\n    \"education\",\n    \"relationship\",\n    \"workclass\",\n    \"occupation\",\n    \"native_country\",\n    \"gender\",\n]\ncrossed_cols = [(\"education\", \"occupation\"), (\"native_country\", \"occupation\")]\n\ncat_embed_cols = [\n    \"workclass\",\n    \"education\",\n    \"marital_status\",\n    \"occupation\",\n    \"relationship\",\n    \"race\",\n    \"gender\",\n    \"capital_gain\",\n    \"capital_loss\",\n    \"native_country\",\n]\ncontinuous_cols = [\"age\", \"hours_per_week\"]\n\ntarget = train[\"income_label\"].values\n</pre> wide_cols = [     \"age_buckets\",     \"education\",     \"relationship\",     \"workclass\",     \"occupation\",     \"native_country\",     \"gender\", ] crossed_cols = [(\"education\", \"occupation\"), (\"native_country\", \"occupation\")]  cat_embed_cols = [     \"workclass\",     \"education\",     \"marital_status\",     \"occupation\",     \"relationship\",     \"race\",     \"gender\",     \"capital_gain\",     \"capital_loss\",     \"native_country\", ] continuous_cols = [\"age\", \"hours_per_week\"]  target = train[\"income_label\"].values In\u00a0[5]: Copied! <pre>wide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols)\nX_tab = wide_preprocessor.fit_transform(train)\n</pre> wide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols) X_tab = wide_preprocessor.fit_transform(train) In\u00a0[6]: Copied! <pre>model = BayesianWide(\n    input_dim=np.unique(X_tab).shape[0],\n    prior_sigma_1=1.0,\n    prior_sigma_2=0.002,\n    prior_pi=0.8,\n    posterior_mu_init=0,\n    posterior_rho_init=-7.0,\n    pred_dim=1,  # here the models are NOT passed to a WideDeep constructor class so the output dim MUST be specified\n)\n</pre> model = BayesianWide(     input_dim=np.unique(X_tab).shape[0],     prior_sigma_1=1.0,     prior_sigma_2=0.002,     prior_pi=0.8,     posterior_mu_init=0,     posterior_rho_init=-7.0,     pred_dim=1,  # here the models are NOT passed to a WideDeep constructor class so the output dim MUST be specified ) In\u00a0[7]: Copied! <pre>trainer = BayesianTrainer(\n    model,\n    objective=\"binary\",\n    optimizer=torch.optim.Adam(model.parameters(), lr=0.01),\n    metrics=[Accuracy],\n)\n</pre> trainer = BayesianTrainer(     model,     objective=\"binary\",     optimizer=torch.optim.Adam(model.parameters(), lr=0.01),     metrics=[Accuracy], ) In\u00a0[8]: Copied! <pre>trainer.fit(\n    X_tab=X_tab,\n    target=target,\n    val_split=0.2,\n    n_epochs=2,\n    batch_size=256,\n)\n</pre> trainer.fit(     X_tab=X_tab,     target=target,     val_split=0.2,     n_epochs=2,     batch_size=256, ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 123/123 [00:00&lt;00:00, 124.32it/s, loss=163, metrics={'acc': 0.7813}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 31/31 [00:00&lt;00:00, 238.67it/s, loss=141, metrics={'acc': 0.8219}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 123/123 [00:00&lt;00:00, 132.81it/s, loss=140, metrics={'acc': 0.8285}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 31/31 [00:00&lt;00:00, 190.16it/s, loss=140, metrics={'acc': 0.8298}]\n</pre> In\u00a0[9]: Copied! <pre>tab_preprocessor = TabPreprocessor(\n    cat_embed_cols=cat_embed_cols, continuous_cols=continuous_cols\n)\nX_tab = tab_preprocessor.fit_transform(train)\n</pre> tab_preprocessor = TabPreprocessor(     cat_embed_cols=cat_embed_cols, continuous_cols=continuous_cols ) X_tab = tab_preprocessor.fit_transform(train) <pre>/Users/javierrodriguezzaurin/Projects/pytorch-widedeep/pytorch_widedeep/preprocessing/tab_preprocessor.py:358: UserWarning: Continuous columns will not be normalised\n  warnings.warn(\"Continuous columns will not be normalised\")\n</pre> In\u00a0[10]: Copied! <pre>model = BayesianTabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=continuous_cols,\n    #     embed_continuous_method = \"standard\",\n    #     cont_embed_activation=\"leaky_relu\",\n    #     cont_embed_dim = 8,\n    mlp_hidden_dims=[128, 64],\n    prior_sigma_1=1.0,\n    prior_sigma_2=0.002,\n    prior_pi=0.8,\n    posterior_mu_init=0,\n    posterior_rho_init=-7.0,\n    pred_dim=1,\n)\n</pre> model = BayesianTabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     continuous_cols=continuous_cols,     #     embed_continuous_method = \"standard\",     #     cont_embed_activation=\"leaky_relu\",     #     cont_embed_dim = 8,     mlp_hidden_dims=[128, 64],     prior_sigma_1=1.0,     prior_sigma_2=0.002,     prior_pi=0.8,     posterior_mu_init=0,     posterior_rho_init=-7.0,     pred_dim=1, ) In\u00a0[11]: Copied! <pre>trainer = BayesianTrainer(\n    model,\n    objective=\"binary\",\n    optimizer=torch.optim.Adam(model.parameters(), lr=0.01),\n    metrics=[Accuracy],\n)\n</pre> trainer = BayesianTrainer(     model,     objective=\"binary\",     optimizer=torch.optim.Adam(model.parameters(), lr=0.01),     metrics=[Accuracy], ) In\u00a0[12]: Copied! <pre>trainer.fit(\n    X_tab=X_tab,\n    target=target,\n    val_split=0.2,\n    n_epochs=2,\n    batch_size=256,\n)\n</pre> trainer.fit(     X_tab=X_tab,     target=target,     val_split=0.2,     n_epochs=2,     batch_size=256, ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 123/123 [00:04&lt;00:00, 28.74it/s, loss=2e+3, metrics={'acc': 0.8007}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 31/31 [00:00&lt;00:00, 136.89it/s, loss=1.75e+3, metrics={'acc': 0.8418}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 123/123 [00:04&lt;00:00, 29.41it/s, loss=1.73e+3, metrics={'acc': 0.8596}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 31/31 [00:00&lt;00:00, 143.87it/s, loss=1.71e+3, metrics={'acc': 0.8569}]\n</pre> <p>These models are powerful beyond the success metrics because they give us a sense of uncertainty as we predict. Let's have a look</p> In\u00a0[13]: Copied! <pre>X_tab_test = tab_preprocessor.transform(test)\n</pre> X_tab_test = tab_preprocessor.transform(test) In\u00a0[14]: Copied! <pre>preds = trainer.predict(X_tab_test, return_samples=True, n_samples=5)\n</pre> preds = trainer.predict(X_tab_test, return_samples=True, n_samples=5) <pre>predict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 39/39 [00:01&lt;00:00, 33.92it/s]\n</pre> In\u00a0[15]: Copied! <pre>preds.shape\n</pre> preds.shape Out[15]: <pre>(5, 9769)</pre> <p>as we can see the prediction have shape <code>(5, 9769)</code>, one set of predictions each time we have internally run predict (i.e. sample the network and predict, defined by the parameter <code>n_samples</code>). This gives us an idea of how certain the model is about a certain prediction.</p> <p>Similarly, we could obtain the probabilities</p> In\u00a0[16]: Copied! <pre>probs = trainer.predict_proba(X_tab_test, return_samples=True, n_samples=5)\n</pre> probs = trainer.predict_proba(X_tab_test, return_samples=True, n_samples=5) <pre>predict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 39/39 [00:01&lt;00:00, 32.79it/s]\n</pre> In\u00a0[17]: Copied! <pre>probs.shape\n</pre> probs.shape Out[17]: <pre>(5, 9769, 2)</pre> <p>And we could see how the model performs each time we sampled the network</p> In\u00a0[18]: Copied! <pre>for p in preds:\n    print(accuracy_score(p, test[\"income_label\"].values))\n</pre> for p in preds:     print(accuracy_score(p, test[\"income_label\"].values)) <pre>0.8559729757395844\n0.8564847988535162\n0.8567918927218753\n0.8562800696079435\n0.8558706111167981\n</pre> In\u00a0[\u00a0]: Copied! <pre>\n</pre>"},{"location":"examples/14_bayesian_models.html#the-bayesian-models","title":"The Bayesian Models\u00b6","text":"<p>Perhaps one of the most interesting functionality in the library is the access to full Bayesian models in almost exactly the same way one would use any of the other models in the library.</p> <p>Note however that the Bayesian models are ONLY available for tabular data and, at the moment, we do not support combining them to form a Wide and Deep model.</p> <p>The implementation in this library is based on the publication: Weight Uncertainty in Neural Networks, by Blundell et al., 2015. Code-wise, our implementation is inspired by a number of source:</p> <ol> <li>https://joshfeldman.net/WeightUncertainty/</li> <li>https://www.nitarshan.com/bayes-by-backprop/</li> <li>https://github.com/piEsposito/blitz-bayesian-deep-learning</li> <li>https://github.com/zackchase/mxnet-the-straight-dope/tree/master/chapter18_variational-methods-and-uncertainty</li> </ol> <p>The two Bayesian models available in the library are:</p> <ol> <li>BayesianWide: this is a linear model where the non-linearities are captured via crossed-columns</li> <li>BayesianMLP: this is a standard MLP that receives categorical embeddings and continuous cols (embedded or not) which are the passed through a series of dense layers. All parameters in the model are probabilistic.</li> </ol>"},{"location":"examples/14_bayesian_models.html#1-bayesianwide","title":"1. <code>BayesianWide</code>\u00b6","text":""},{"location":"examples/14_bayesian_models.html#2-bayesiantabmlp","title":"2. <code>BayesianTabMlp</code>\u00b6","text":""},{"location":"examples/15_Self_Supervised_Pretraning_pt1.html","title":"15_Self-Supervised Pre-Training pt 1","text":"In\u00a0[1]: Copied! <pre>import torch\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.model_selection import train_test_split\n\nfrom pytorch_widedeep import Trainer\nfrom pytorch_widedeep.models import TabMlp, WideDeep\nfrom pytorch_widedeep.metrics import Accuracy\nfrom pytorch_widedeep.datasets import load_adult\nfrom pytorch_widedeep.preprocessing import TabPreprocessor\nfrom pytorch_widedeep.self_supervised_training import EncoderDecoderTrainer\n</pre> import torch from sklearn.metrics import accuracy_score from sklearn.model_selection import train_test_split  from pytorch_widedeep import Trainer from pytorch_widedeep.models import TabMlp, WideDeep from pytorch_widedeep.metrics import Accuracy from pytorch_widedeep.datasets import load_adult from pytorch_widedeep.preprocessing import TabPreprocessor from pytorch_widedeep.self_supervised_training import EncoderDecoderTrainer In\u00a0[2]: Copied! <pre>df = load_adult(as_frame=True)\ndf.columns = [c.replace(\"-\", \"_\") for c in df.columns]\ndf[\"income_label\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int)\ndf.drop(\"income\", axis=1, inplace=True)\n</pre> df = load_adult(as_frame=True) df.columns = [c.replace(\"-\", \"_\") for c in df.columns] df[\"income_label\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int) df.drop(\"income\", axis=1, inplace=True) In\u00a0[3]: Copied! <pre># one could chose to use a validation set for early stopping, hyperparam\n# optimization, etc. This is just an example, so we simply use train/test\n# split\ndf_tr, df_te = train_test_split(df, test_size=0.2, stratify=df.income_label)\n</pre> # one could chose to use a validation set for early stopping, hyperparam # optimization, etc. This is just an example, so we simply use train/test # split df_tr, df_te = train_test_split(df, test_size=0.2, stratify=df.income_label) In\u00a0[4]: Copied! <pre>df_tr.head(2)\n</pre> df_tr.head(2) Out[4]: age workclass fnlwgt education educational_num marital_status occupation relationship race gender capital_gain capital_loss hours_per_week native_country income_label 9042 26 Local-gov 250551 HS-grad 9 Married-civ-spouse Craft-repair Own-child Black Male 0 0 40 United-States 0 25322 50 Private 34832 Bachelors 13 Married-civ-spouse Tech-support Husband White Male 15024 0 40 United-States 1 In\u00a0[5]: Copied! <pre># As always, we need to define which cols will be represented as embeddings\n# and which one will be continuous features\ncat_embed_cols = [\n    \"workclass\",\n    \"education\",\n    \"marital_status\",\n    \"occupation\",\n    \"relationship\",\n    \"race\",\n    \"gender\",\n    \"capital_gain\",\n    \"capital_loss\",\n    \"native_country\",\n]\ncontinuous_cols = [\"age\", \"hours_per_week\"]\ntarget_col = \"income_label\"\n</pre> # As always, we need to define which cols will be represented as embeddings # and which one will be continuous features cat_embed_cols = [     \"workclass\",     \"education\",     \"marital_status\",     \"occupation\",     \"relationship\",     \"race\",     \"gender\",     \"capital_gain\",     \"capital_loss\",     \"native_country\", ] continuous_cols = [\"age\", \"hours_per_week\"] target_col = \"income_label\" In\u00a0[6]: Copied! <pre># We prepare the data to be passed to the model\ntab_preprocessor = TabPreprocessor(\n    cat_embed_cols=cat_embed_cols, continuous_cols=continuous_cols\n)\nX_tab = tab_preprocessor.fit_transform(df_tr)\ntarget = df_tr[target_col].values\n</pre> # We prepare the data to be passed to the model tab_preprocessor = TabPreprocessor(     cat_embed_cols=cat_embed_cols, continuous_cols=continuous_cols ) X_tab = tab_preprocessor.fit_transform(df_tr) target = df_tr[target_col].values <pre>/Users/javierrodriguezzaurin/Projects/pytorch-widedeep/pytorch_widedeep/preprocessing/tab_preprocessor.py:358: UserWarning: Continuous columns will not be normalised\n  warnings.warn(\"Continuous columns will not be normalised\")\n</pre> In\u00a0[7]: Copied! <pre>X_tab[:5]\n</pre> X_tab[:5] Out[7]: <pre>array([[ 1,  1,  1,  1,  1,  1,  1,  1,  1,  1, 26, 40],\n       [ 2,  2,  1,  2,  2,  2,  1,  2,  1,  1, 50, 40],\n       [ 2,  1,  1,  3,  2,  2,  1,  1,  2,  1, 39, 46],\n       [ 2,  3,  2,  4,  1,  2,  2,  1,  1,  1, 17, 10],\n       [ 3,  4,  2,  1,  1,  2,  1,  1,  1,  1, 32, 20]])</pre> In\u00a0[8]: Copied! <pre># We define a model that will act as the encoder in the encoder/decoder\n# architecture. This could be any of: TabMlp, TabResnet or TabNet\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=tab_preprocessor.continuous_cols,\n)\n</pre> # We define a model that will act as the encoder in the encoder/decoder # architecture. This could be any of: TabMlp, TabResnet or TabNet tab_mlp = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     continuous_cols=tab_preprocessor.continuous_cols, ) In\u00a0[9]: Copied! <pre>tab_mlp\n</pre> tab_mlp Out[9]: <pre>TabMlp(\n  (cat_embed): DiffSizeCatEmbeddings(\n    (embed_layers): ModuleDict(\n      (emb_layer_workclass): Embedding(10, 5, padding_idx=0)\n      (emb_layer_education): Embedding(17, 8, padding_idx=0)\n      (emb_layer_marital_status): Embedding(8, 5, padding_idx=0)\n      (emb_layer_occupation): Embedding(16, 7, padding_idx=0)\n      (emb_layer_relationship): Embedding(7, 4, padding_idx=0)\n      (emb_layer_race): Embedding(6, 4, padding_idx=0)\n      (emb_layer_gender): Embedding(3, 2, padding_idx=0)\n      (emb_layer_capital_gain): Embedding(124, 24, padding_idx=0)\n      (emb_layer_capital_loss): Embedding(98, 21, padding_idx=0)\n      (emb_layer_native_country): Embedding(42, 13, padding_idx=0)\n    )\n    (embedding_dropout): Dropout(p=0.0, inplace=False)\n  )\n  (cont_norm): Identity()\n  (encoder): MLP(\n    (mlp): Sequential(\n      (dense_layer_0): Sequential(\n        (0): Linear(in_features=95, out_features=200, bias=True)\n        (1): ReLU(inplace=True)\n        (2): Dropout(p=0.1, inplace=False)\n      )\n      (dense_layer_1): Sequential(\n        (0): Linear(in_features=200, out_features=100, bias=True)\n        (1): ReLU(inplace=True)\n        (2): Dropout(p=0.1, inplace=False)\n      )\n    )\n  )\n)</pre> In\u00a0[10]: Copied! <pre># If we do not pass a custom decoder, which is perfectly possible via the\n# decoder param,  the EncoderDecoderTrainer will automatically build a\n# decoder which will be the 'mirror' image of the encoder\nencoder_decoder_trainer = EncoderDecoderTrainer(encoder=tab_mlp)\n</pre> # If we do not pass a custom decoder, which is perfectly possible via the # decoder param,  the EncoderDecoderTrainer will automatically build a # decoder which will be the 'mirror' image of the encoder encoder_decoder_trainer = EncoderDecoderTrainer(encoder=tab_mlp) In\u00a0[11]: Copied! <pre># let's have a look to the encoder_decoder_model (aka ed_model)\nencoder_decoder_trainer.ed_model\n</pre> # let's have a look to the encoder_decoder_model (aka ed_model) encoder_decoder_trainer.ed_model Out[11]: <pre>EncoderDecoderModel(\n  (encoder): TabMlp(\n    (cat_embed): DiffSizeCatEmbeddings(\n      (embed_layers): ModuleDict(\n        (emb_layer_workclass): Embedding(10, 5, padding_idx=0)\n        (emb_layer_education): Embedding(17, 8, padding_idx=0)\n        (emb_layer_marital_status): Embedding(8, 5, padding_idx=0)\n        (emb_layer_occupation): Embedding(16, 7, padding_idx=0)\n        (emb_layer_relationship): Embedding(7, 4, padding_idx=0)\n        (emb_layer_race): Embedding(6, 4, padding_idx=0)\n        (emb_layer_gender): Embedding(3, 2, padding_idx=0)\n        (emb_layer_capital_gain): Embedding(124, 24, padding_idx=0)\n        (emb_layer_capital_loss): Embedding(98, 21, padding_idx=0)\n        (emb_layer_native_country): Embedding(42, 13, padding_idx=0)\n      )\n      (embedding_dropout): Dropout(p=0.0, inplace=False)\n    )\n    (cont_norm): Identity()\n    (encoder): MLP(\n      (mlp): Sequential(\n        (dense_layer_0): Sequential(\n          (0): Linear(in_features=95, out_features=200, bias=True)\n          (1): ReLU(inplace=True)\n          (2): Dropout(p=0.1, inplace=False)\n        )\n        (dense_layer_1): Sequential(\n          (0): Linear(in_features=200, out_features=100, bias=True)\n          (1): ReLU(inplace=True)\n          (2): Dropout(p=0.1, inplace=False)\n        )\n      )\n    )\n  )\n  (decoder): TabMlpDecoder(\n    (decoder): MLP(\n      (mlp): Sequential(\n        (dense_layer_0): Sequential(\n          (0): Linear(in_features=100, out_features=200, bias=True)\n          (1): ReLU(inplace=True)\n          (2): Dropout(p=0.1, inplace=False)\n        )\n        (dense_layer_1): Sequential(\n          (0): Linear(in_features=200, out_features=95, bias=True)\n          (1): ReLU(inplace=True)\n          (2): Dropout(p=0.1, inplace=False)\n        )\n      )\n    )\n  )\n  (masker): RandomObfuscator()\n)</pre> <p>Ignoring the <code>masker</code>, which just...well...masks, the <code>ed_model</code> consists of:</p> <ol> <li>An encoder model that is a <code>TabMlp</code> model that is in itself comprised by an Embedding layer (or rather a collection of them, referred as <code>cat_and_cont_embed</code>) and an encoder (a simple MLP, referred as <code>encoder</code>)</li> <li>A decoder which is just an \"inverted\" MLP (referred as <code>decoder</code>)</li> </ol> In\u00a0[12]: Copied! <pre># And we just...pretrain\nencoder_decoder_trainer.pretrain(X_tab, n_epochs=5, batch_size=256)\n</pre> # And we just...pretrain encoder_decoder_trainer.pretrain(X_tab, n_epochs=5, batch_size=256) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 82.90it/s, loss=4.07]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 89.87it/s, loss=3.09]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 92.86it/s, loss=2.53]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 91.24it/s, loss=2.09]\nepoch 5: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 91.38it/s, loss=1.78]\n</pre> <p>At this point we have two options, we could either save the model for later use or we could continue to supervised training. The latter is rather simple, after running:</p> <pre>encoder_decoder_trainer.pretrain(X_tab, n_epochs=5, batch_size=256)\n</pre> <p>you just have to</p> <pre>model = WideDeep(deeptabular=tab_mlp)\ntrainer = Trainer(model=model, objective=\"binary\", metrics=[Accuracy])\n\ntrainer.fit(X_tab=X_tab, target=target, n_epochs=5, batch_size=256)\n\n# And, you know...we get a test metric\nX_tab_te = tab_preprocessor.transform(df_te)\ntarget_te = df_te[target_col].values\n\npreds = trainer.predict(X_tab=X_tab_te)\ntest_acc = accuracy_score(target_te, preds)\n</pre> <p>Let's say that in any case, we are 'decent' scientists/people and we want to save the model:</p> In\u00a0[13]: Copied! <pre>encoder_decoder_trainer.save(\n    path=\"pretrained_weights\", model_filename=\"encoder_decoder_model.pt\"\n)\n</pre> encoder_decoder_trainer.save(     path=\"pretrained_weights\", model_filename=\"encoder_decoder_model.pt\" ) <p>some time has passed...</p> In\u00a0[14]: Copied! <pre>encoder_decoder_model = torch.load(\"pretrained_weights/encoder_decoder_model.pt\")\n</pre> encoder_decoder_model = torch.load(\"pretrained_weights/encoder_decoder_model.pt\") <p>Now, AND THIS IS IMPORTANT We have loaded the encoder AND the decoder. To proceed to the supervised training we ONLY need the encoder</p> In\u00a0[15]: Copied! <pre>pretrained_encoder = encoder_decoder_model.encoder\n</pre> pretrained_encoder = encoder_decoder_model.encoder In\u00a0[16]: Copied! <pre>pretrained_encoder\n</pre> pretrained_encoder Out[16]: <pre>TabMlp(\n  (cat_embed): DiffSizeCatEmbeddings(\n    (embed_layers): ModuleDict(\n      (emb_layer_workclass): Embedding(10, 5, padding_idx=0)\n      (emb_layer_education): Embedding(17, 8, padding_idx=0)\n      (emb_layer_marital_status): Embedding(8, 5, padding_idx=0)\n      (emb_layer_occupation): Embedding(16, 7, padding_idx=0)\n      (emb_layer_relationship): Embedding(7, 4, padding_idx=0)\n      (emb_layer_race): Embedding(6, 4, padding_idx=0)\n      (emb_layer_gender): Embedding(3, 2, padding_idx=0)\n      (emb_layer_capital_gain): Embedding(124, 24, padding_idx=0)\n      (emb_layer_capital_loss): Embedding(98, 21, padding_idx=0)\n      (emb_layer_native_country): Embedding(42, 13, padding_idx=0)\n    )\n    (embedding_dropout): Dropout(p=0.0, inplace=False)\n  )\n  (cont_norm): Identity()\n  (encoder): MLP(\n    (mlp): Sequential(\n      (dense_layer_0): Sequential(\n        (0): Linear(in_features=95, out_features=200, bias=True)\n        (1): ReLU(inplace=True)\n        (2): Dropout(p=0.1, inplace=False)\n      )\n      (dense_layer_1): Sequential(\n        (0): Linear(in_features=200, out_features=100, bias=True)\n        (1): ReLU(inplace=True)\n        (2): Dropout(p=0.1, inplace=False)\n      )\n    )\n  )\n)</pre> In\u00a0[17]: Copied! <pre># and as always, ANY supervised model in this library has to go throuth the WideDeep class:\nmodel = WideDeep(deeptabular=pretrained_encoder)\ntrainer = Trainer(model=model, objective=\"binary\", metrics=[Accuracy])\n\ntrainer.fit(X_tab=X_tab, target=target, n_epochs=5, batch_size=256)\n\nX_tab_te = tab_preprocessor.transform(df_te)\ntarget_te = df_te[target_col].values\n\npreds = trainer.predict(X_tab=X_tab_te)\ntest_acc = accuracy_score(target_te, preds)\nprint(test_acc)\n</pre> # and as always, ANY supervised model in this library has to go throuth the WideDeep class: model = WideDeep(deeptabular=pretrained_encoder) trainer = Trainer(model=model, objective=\"binary\", metrics=[Accuracy])  trainer.fit(X_tab=X_tab, target=target, n_epochs=5, batch_size=256)  X_tab_te = tab_preprocessor.transform(df_te) target_te = df_te[target_col].values  preds = trainer.predict(X_tab=X_tab_te) test_acc = accuracy_score(target_te, preds) print(test_acc) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 88.04it/s, loss=0.374, metrics={'acc': 0.8253}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 85.63it/s, loss=0.324, metrics={'acc': 0.8491}]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 87.56it/s, loss=0.301, metrics={'acc': 0.8608}]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:02&lt;00:00, 73.38it/s, loss=0.29, metrics={'acc': 0.8655}]\nepoch 5: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 78.68it/s, loss=0.284, metrics={'acc': 0.8686}]\npredict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 39/39 [00:00&lt;00:00, 173.02it/s]\n</pre> <pre>0.8730678677449074\n</pre> <p>As we mentioned before, we can also use a <code>TabResNet</code> or <code>TabNet</code> model and a custom decoder. Let's have a look:</p> In\u00a0[18]: Copied! <pre>from pytorch_widedeep.models import TabResnet as TabResnetEncoder, TabResnetDecoder\n</pre> from pytorch_widedeep.models import TabResnet as TabResnetEncoder, TabResnetDecoder In\u00a0[19]: Copied! <pre>resnet_encoder = TabResnetEncoder(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=continuous_cols,\n    blocks_dims=[200, 100, 100],\n)\n</pre> resnet_encoder = TabResnetEncoder(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     continuous_cols=continuous_cols,     blocks_dims=[200, 100, 100], ) <p>let's have a look to the model</p> In\u00a0[20]: Copied! <pre>resnet_encoder\n</pre> resnet_encoder Out[20]: <pre>TabResnet(\n  (cat_embed): DiffSizeCatEmbeddings(\n    (embed_layers): ModuleDict(\n      (emb_layer_workclass): Embedding(10, 5, padding_idx=0)\n      (emb_layer_education): Embedding(17, 8, padding_idx=0)\n      (emb_layer_marital_status): Embedding(8, 5, padding_idx=0)\n      (emb_layer_occupation): Embedding(16, 7, padding_idx=0)\n      (emb_layer_relationship): Embedding(7, 4, padding_idx=0)\n      (emb_layer_race): Embedding(6, 4, padding_idx=0)\n      (emb_layer_gender): Embedding(3, 2, padding_idx=0)\n      (emb_layer_capital_gain): Embedding(124, 24, padding_idx=0)\n      (emb_layer_capital_loss): Embedding(98, 21, padding_idx=0)\n      (emb_layer_native_country): Embedding(42, 13, padding_idx=0)\n    )\n    (embedding_dropout): Dropout(p=0.0, inplace=False)\n  )\n  (cont_norm): Identity()\n  (encoder): DenseResnet(\n    (dense_resnet): Sequential(\n      (lin_inp): Linear(in_features=95, out_features=200, bias=False)\n      (bn_inp): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n      (block_0): BasicBlock(\n        (resize): Sequential(\n          (0): Linear(in_features=200, out_features=100, bias=False)\n          (1): BatchNorm1d(100, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        )\n        (lin1): Linear(in_features=200, out_features=100, bias=False)\n        (bn1): BatchNorm1d(100, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (leaky_relu): LeakyReLU(negative_slope=0.01, inplace=True)\n        (dp): Dropout(p=0.1, inplace=False)\n        (lin2): Linear(in_features=100, out_features=100, bias=False)\n        (bn2): BatchNorm1d(100, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n      )\n      (block_1): BasicBlock(\n        (lin1): Linear(in_features=100, out_features=100, bias=False)\n        (bn1): BatchNorm1d(100, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (leaky_relu): LeakyReLU(negative_slope=0.01, inplace=True)\n        (dp): Dropout(p=0.1, inplace=False)\n        (lin2): Linear(in_features=100, out_features=100, bias=False)\n        (bn2): BatchNorm1d(100, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n      )\n    )\n  )\n)</pre> <p>As we can see, the tensor we are trying to reconstruct, the embeddings, is of size <code>94</code> (this number is stored in the attribute: <code>esnet_encoder.cat_and_cont_embed.output_dim</code>), with that information we could build or own decoder as:</p> In\u00a0[21]: Copied! <pre># for all possible params see the docs\nresnet_decoder = TabResnetDecoder(\n    embed_dim=resnet_encoder.cat_out_dim + resnet_encoder.cont_out_dim,\n    blocks_dims=[100, 100, 200],\n)\n</pre> # for all possible params see the docs resnet_decoder = TabResnetDecoder(     embed_dim=resnet_encoder.cat_out_dim + resnet_encoder.cont_out_dim,     blocks_dims=[100, 100, 200], ) In\u00a0[22]: Copied! <pre>resnet_decoder\n</pre> resnet_decoder Out[22]: <pre>TabResnetDecoder(\n  (decoder): DenseResnet(\n    (dense_resnet): Sequential(\n      (block_0): BasicBlock(\n        (lin1): Linear(in_features=100, out_features=100, bias=False)\n        (bn1): BatchNorm1d(100, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (leaky_relu): LeakyReLU(negative_slope=0.01, inplace=True)\n        (dp): Dropout(p=0.1, inplace=False)\n        (lin2): Linear(in_features=100, out_features=100, bias=False)\n        (bn2): BatchNorm1d(100, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n      )\n      (block_1): BasicBlock(\n        (resize): Sequential(\n          (0): Linear(in_features=100, out_features=200, bias=False)\n          (1): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        )\n        (lin1): Linear(in_features=100, out_features=200, bias=False)\n        (bn1): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (leaky_relu): LeakyReLU(negative_slope=0.01, inplace=True)\n        (dp): Dropout(p=0.1, inplace=False)\n        (lin2): Linear(in_features=200, out_features=200, bias=False)\n        (bn2): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n      )\n    )\n  )\n  (reconstruction_layer): Linear(in_features=200, out_features=95, bias=False)\n)</pre> <p>and now:</p> In\u00a0[23]: Copied! <pre>ec_trainer = EncoderDecoderTrainer(\n    encoder=resnet_encoder,\n    decoder=resnet_decoder,\n    masked_prob=0.2,\n)\nec_trainer.pretrain(X_tab, n_epochs=5, batch_size=256)\n</pre> ec_trainer = EncoderDecoderTrainer(     encoder=resnet_encoder,     decoder=resnet_decoder,     masked_prob=0.2, ) ec_trainer.pretrain(X_tab, n_epochs=5, batch_size=256) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:03&lt;00:00, 46.89it/s, loss=1.52]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:03&lt;00:00, 46.78it/s, loss=0.81]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:03&lt;00:00, 39.82it/s, loss=0.56]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:03&lt;00:00, 46.73it/s, loss=0.417]\nepoch 5: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:03&lt;00:00, 46.24it/s, loss=0.329]\n</pre> In\u00a0[24]: Copied! <pre># and as always, ANY supervised model in this library has to go throuth the WideDeep class:\nmodel = WideDeep(deeptabular=resnet_encoder)\ntrainer = Trainer(model=model, objective=\"binary\", metrics=[Accuracy])\n\ntrainer.fit(X_tab=X_tab, target=target, n_epochs=5, batch_size=256)\n\nX_tab_te = tab_preprocessor.transform(df_te)\ntarget_te = df_te[target_col].values\n\npreds = trainer.predict(X_tab=X_tab_te)\ntest_acc = accuracy_score(target_te, preds)\nprint(test_acc)\n</pre> # and as always, ANY supervised model in this library has to go throuth the WideDeep class: model = WideDeep(deeptabular=resnet_encoder) trainer = Trainer(model=model, objective=\"binary\", metrics=[Accuracy])  trainer.fit(X_tab=X_tab, target=target, n_epochs=5, batch_size=256)  X_tab_te = tab_preprocessor.transform(df_te) target_te = df_te[target_col].values  preds = trainer.predict(X_tab=X_tab_te) test_acc = accuracy_score(target_te, preds) print(test_acc) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:02&lt;00:00, 58.63it/s, loss=0.335, metrics={'acc': 0.8442}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:02&lt;00:00, 58.02it/s, loss=0.296, metrics={'acc': 0.864}]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:02&lt;00:00, 55.91it/s, loss=0.283, metrics={'acc': 0.8687}]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:02&lt;00:00, 55.00it/s, loss=0.276, metrics={'acc': 0.871}]\nepoch 5: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:02&lt;00:00, 51.95it/s, loss=0.272, metrics={'acc': 0.8732}]\npredict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 39/39 [00:00&lt;00:00, 120.15it/s]\n</pre> <pre>0.8725560446309756\n</pre> In\u00a0[\u00a0]: Copied! <pre>\n</pre>"},{"location":"examples/15_Self_Supervised_Pretraning_pt1.html#self-supervised-pretraining-for-tabular-data","title":"Self Supervised Pretraining for Tabular Data\u00b6","text":"<p>We have implemented two Self Supervised Pre-training routines that allow the user to pre-train all tabular models in the library with the exception of the TabPerceiver (which is a special monster).</p> <p>The two routines implemented are illustrated in the figures below. The 1st is from TabNet: Attentive Interpretable Tabular Learning and is designed for models that do not use transformer-based architectures, while the second is from SAINT: Improved Neural Networks for Tabular Data via Row Attention and Contrastive Pre-Training, and is designed for models that use transformer-based architectures.</p> <p> </p> <p>Fig 1. Figure 2 in their paper. I have included de original caption in case is useful, althought the Figure itself is pretty self explanatory</p> <p> </p> <p>Fig 2. Figure 1 in their paper. Here the caption is necessary \ud83d\ude0f</p> <p>It is beyond the scope of this notebook to explain in detail those implementations. Therefore, we strongly recommend the user to go and read the papers if this functionality is of interest to her/him.</p> <p>One thing is worth noticing however. As seen in Fig 1(the TabNet paper's Fig 2) the masking of the input features happens in the feature space. However, the implementation in this library is inspired by that at the dreamquark-ai repo, which is in itself inspired by the original implementation (by the way, at this point I will write it once again. All TabNet related things in this library are inspired when not directly based in the code in that repo, therefore, ALL CREDIT TO THE GUYS AT dreamquark-ai).</p> <p>In that implementation the masking happens in the embedding space, and currently does not mask the entire embedding (i.e. categorical feature). We decided to release as it is in this version and we will implement the exact same process described in the paper in future releases.</p> <p>Having said all of the above let's see how to use self supervision for tabular data with <code>pytorch-widedeep</code>. We will concentrate in this notebook on the 1st of the two approaches (the 'TabNet approach'). For details on the second approach please see <code>16_Self_Supervised_Pretraning_pt2</code>.</p>"},{"location":"examples/15_Self_Supervised_Pretraning_pt1.html#self-supervision-for-non-transformer-based-models","title":"Self Supervision for non-transformer-based models..\u00b6","text":"<p>...or in general, for models where the embeddigns can have all different dimensions. In this library, these are: <code>TabMlp</code>, <code>TabResNet</code> and <code>TabNet</code></p> <p>As shown in Figure, this is an encoder-encoder approach where we learn to predict values in the incoming data that have been masked. However, as I mentioned before, our implementation is a bit different, and the masking occurs in th embedding space.</p> <p>Nonetheless, the code below illustrates how to use this encoder-decoder approach with <code>pytorch-widedeep</code></p>"},{"location":"examples/15_Self_Supervised_Pretraning_pt2.html","title":"15_Self-Supervised Pre-Training pt 2","text":"In\u00a0[1]: Copied! <pre>import torch\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.model_selection import train_test_split\n\nfrom pytorch_widedeep import Trainer\nfrom pytorch_widedeep.models import WideDeep, FTTransformer\nfrom pytorch_widedeep.metrics import Accuracy\nfrom pytorch_widedeep.datasets import load_adult\nfrom pytorch_widedeep.preprocessing import TabPreprocessor\nfrom pytorch_widedeep.self_supervised_training import (\n    ContrastiveDenoisingTrainer,\n)\n</pre> import torch from sklearn.metrics import accuracy_score from sklearn.model_selection import train_test_split  from pytorch_widedeep import Trainer from pytorch_widedeep.models import WideDeep, FTTransformer from pytorch_widedeep.metrics import Accuracy from pytorch_widedeep.datasets import load_adult from pytorch_widedeep.preprocessing import TabPreprocessor from pytorch_widedeep.self_supervised_training import (     ContrastiveDenoisingTrainer, ) In\u00a0[2]: Copied! <pre>df = load_adult(as_frame=True)\ndf.columns = [c.replace(\"-\", \"_\") for c in df.columns]\ndf[\"income_label\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int)\ndf.drop(\"income\", axis=1, inplace=True)\n\n# one could chose to use a validation set for early stopping, hyperparam\n# optimization, etc. This is just an example, so we simply use train/test\n# split\ndf_tr, df_te = train_test_split(df, test_size=0.2, stratify=df.income_label)\n\ncat_embed_cols = [\n    \"workclass\",\n    \"education\",\n    \"marital_status\",\n    \"occupation\",\n    \"relationship\",\n    \"race\",\n    \"gender\",\n    \"capital_gain\",\n    \"capital_loss\",\n    \"native_country\",\n]\ncontinuous_cols = [\"age\", \"hours_per_week\"]\ntarget_col = \"income_label\"\n\ntab_preprocessor = TabPreprocessor(\n    cat_embed_cols=cat_embed_cols,\n    continuous_cols=continuous_cols,\n    with_attention=True,\n    with_cls_token=True,  # this is optional\n)\nX_tab = tab_preprocessor.fit_transform(df_tr)\ntarget = df_tr[target_col].values\n</pre> df = load_adult(as_frame=True) df.columns = [c.replace(\"-\", \"_\") for c in df.columns] df[\"income_label\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int) df.drop(\"income\", axis=1, inplace=True)  # one could chose to use a validation set for early stopping, hyperparam # optimization, etc. This is just an example, so we simply use train/test # split df_tr, df_te = train_test_split(df, test_size=0.2, stratify=df.income_label)  cat_embed_cols = [     \"workclass\",     \"education\",     \"marital_status\",     \"occupation\",     \"relationship\",     \"race\",     \"gender\",     \"capital_gain\",     \"capital_loss\",     \"native_country\", ] continuous_cols = [\"age\", \"hours_per_week\"] target_col = \"income_label\"  tab_preprocessor = TabPreprocessor(     cat_embed_cols=cat_embed_cols,     continuous_cols=continuous_cols,     with_attention=True,     with_cls_token=True,  # this is optional ) X_tab = tab_preprocessor.fit_transform(df_tr) target = df_tr[target_col].values <pre>/Users/javierrodriguezzaurin/Projects/pytorch-widedeep/pytorch_widedeep/preprocessing/tab_preprocessor.py:358: UserWarning: Continuous columns will not be normalised\n  warnings.warn(\"Continuous columns will not be normalised\")\n</pre> In\u00a0[3]: Copied! <pre>ft_transformer = FTTransformer(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=tab_preprocessor.continuous_cols,\n    embed_continuous_method=\"standard\",\n    input_dim=32,\n    kv_compression_factor=0.5,\n    n_blocks=3,\n    n_heads=4,\n)\n</pre> ft_transformer = FTTransformer(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     continuous_cols=tab_preprocessor.continuous_cols,     embed_continuous_method=\"standard\",     input_dim=32,     kv_compression_factor=0.5,     n_blocks=3,     n_heads=4, ) In\u00a0[4]: Copied! <pre># for a full list of the params for the the ContrastiveDenoisingTrainer (which are many) please see the docs.\n# Note that using these params involves some knowledge of the routine and the architecture of the model used\ncontrastive_denoising_trainer = ContrastiveDenoisingTrainer(\n    model=ft_transformer,\n    preprocessor=tab_preprocessor,\n)\ncontrastive_denoising_trainer.pretrain(X_tab, n_epochs=5, batch_size=256)\n</pre> # for a full list of the params for the the ContrastiveDenoisingTrainer (which are many) please see the docs. # Note that using these params involves some knowledge of the routine and the architecture of the model used contrastive_denoising_trainer = ContrastiveDenoisingTrainer(     model=ft_transformer,     preprocessor=tab_preprocessor, ) contrastive_denoising_trainer.pretrain(X_tab, n_epochs=5, batch_size=256) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:13&lt;00:00, 11.73it/s, loss=579]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:12&lt;00:00, 12.56it/s, loss=143]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:12&lt;00:00, 12.49it/s, loss=141]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:11&lt;00:00, 12.77it/s, loss=138]\nepoch 5: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:11&lt;00:00, 13.29it/s, loss=137]\n</pre> In\u00a0[5]: Copied! <pre>contrastive_denoising_trainer.save(\n    path=\"pretrained_weights\", model_filename=\"contrastive_denoising_model.pt\"\n)\n</pre> contrastive_denoising_trainer.save(     path=\"pretrained_weights\", model_filename=\"contrastive_denoising_model.pt\" ) <p>some time has passed</p> In\u00a0[6]: Copied! <pre># some time has passed, we load the model with torch as usual:\ncontrastive_denoising_model = torch.load(\n    \"pretrained_weights/contrastive_denoising_model.pt\"\n)\n</pre> # some time has passed, we load the model with torch as usual: contrastive_denoising_model = torch.load(     \"pretrained_weights/contrastive_denoising_model.pt\" ) <p>NOW, AND THIS IS IMPORTANT! We have loaded the entire contrastive, denoising model. To proceed to the supervised training we ONLY need the attention-based model, which is the 'model' attribute of the trainer, let's have a look</p> In\u00a0[7]: Copied! <pre>contrastive_denoising_model.model\n</pre> contrastive_denoising_model.model Out[7]: <pre>FTTransformer(\n  (cat_embed): SameSizeCatEmbeddings(\n    (embed): Embedding(323, 32, padding_idx=0)\n    (dropout): Dropout(p=0.0, inplace=False)\n  )\n  (cont_norm): Identity()\n  (cont_embed): ContEmbeddings(\n    INFO: [ContLinear = weight(n_cont_cols, embed_dim) + bias(n_cont_cols, embed_dim)]\n    (linear): ContLinear(n_cont_cols=2, embed_dim=32, embed_dropout=0.0)\n    (dropout): Dropout(p=0.0, inplace=False)\n  )\n  (encoder): Sequential(\n    (fttransformer_block0): FTTransformerEncoder(\n      (attn): LinearAttentionLinformer(\n        (dropout): Dropout(p=0.2, inplace=False)\n        (qkv_proj): Linear(in_features=32, out_features=96, bias=False)\n        (out_proj): Linear(in_features=32, out_features=32, bias=False)\n      )\n      (ff): FeedForward(\n        (w_1): Linear(in_features=32, out_features=84, bias=True)\n        (w_2): Linear(in_features=42, out_features=32, bias=True)\n        (dropout): Dropout(p=0.1, inplace=False)\n        (activation): REGLU()\n      )\n      (attn_normadd): NormAdd(\n        (dropout): Dropout(p=0.2, inplace=False)\n        (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n      )\n      (ff_normadd): NormAdd(\n        (dropout): Dropout(p=0.1, inplace=False)\n        (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n      )\n    )\n    (fttransformer_block1): FTTransformerEncoder(\n      (attn): LinearAttentionLinformer(\n        (dropout): Dropout(p=0.2, inplace=False)\n        (qkv_proj): Linear(in_features=32, out_features=96, bias=False)\n        (out_proj): Linear(in_features=32, out_features=32, bias=False)\n      )\n      (ff): FeedForward(\n        (w_1): Linear(in_features=32, out_features=84, bias=True)\n        (w_2): Linear(in_features=42, out_features=32, bias=True)\n        (dropout): Dropout(p=0.1, inplace=False)\n        (activation): REGLU()\n      )\n      (attn_normadd): NormAdd(\n        (dropout): Dropout(p=0.2, inplace=False)\n        (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n      )\n      (ff_normadd): NormAdd(\n        (dropout): Dropout(p=0.1, inplace=False)\n        (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n      )\n    )\n    (fttransformer_block2): FTTransformerEncoder(\n      (attn): LinearAttentionLinformer(\n        (dropout): Dropout(p=0.2, inplace=False)\n        (qkv_proj): Linear(in_features=32, out_features=96, bias=False)\n        (out_proj): Linear(in_features=32, out_features=32, bias=False)\n      )\n      (ff): FeedForward(\n        (w_1): Linear(in_features=32, out_features=84, bias=True)\n        (w_2): Linear(in_features=42, out_features=32, bias=True)\n        (dropout): Dropout(p=0.1, inplace=False)\n        (activation): REGLU()\n      )\n      (attn_normadd): NormAdd(\n        (dropout): Dropout(p=0.2, inplace=False)\n        (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n      )\n      (ff_normadd): NormAdd(\n        (dropout): Dropout(p=0.1, inplace=False)\n        (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n      )\n    )\n  )\n)</pre> In\u00a0[8]: Copied! <pre>pretrained_model = contrastive_denoising_model.model\n</pre> pretrained_model = contrastive_denoising_model.model In\u00a0[9]: Copied! <pre># and as always, ANY supervised model in this library has to go throuth the WideDeep class:\nmodel = WideDeep(deeptabular=pretrained_model)\ntrainer = Trainer(model=model, objective=\"binary\", metrics=[Accuracy])\n\ntrainer.fit(X_tab=X_tab, target=target, n_epochs=5, batch_size=256)\n\n# And, you know...we get a test metric\nX_tab_te = tab_preprocessor.transform(df_te)\ntarget_te = df_te[target_col].values\n\npreds = trainer.predict(X_tab=X_tab_te)\ntest_acc = accuracy_score(target_te, preds)\nprint(test_acc)\n</pre> # and as always, ANY supervised model in this library has to go throuth the WideDeep class: model = WideDeep(deeptabular=pretrained_model) trainer = Trainer(model=model, objective=\"binary\", metrics=[Accuracy])  trainer.fit(X_tab=X_tab, target=target, n_epochs=5, batch_size=256)  # And, you know...we get a test metric X_tab_te = tab_preprocessor.transform(df_te) target_te = df_te[target_col].values  preds = trainer.predict(X_tab=X_tab_te) test_acc = accuracy_score(target_te, preds) print(test_acc) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:05&lt;00:00, 27.19it/s, loss=0.383, metrics={'acc': 0.8176}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:05&lt;00:00, 26.08it/s, loss=0.325, metrics={'acc': 0.8502}]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:05&lt;00:00, 26.56it/s, loss=0.306, metrics={'acc': 0.8601}]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:05&lt;00:00, 27.41it/s, loss=0.295, metrics={'acc': 0.8641}]\nepoch 5: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:06&lt;00:00, 24.70it/s, loss=0.289, metrics={'acc': 0.8656}]\npredict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 39/39 [00:00&lt;00:00, 97.26it/s]</pre> <pre>0.8695874705701709\n</pre> <pre>\n</pre> In\u00a0[\u00a0]: Copied! <pre>\n</pre>"},{"location":"examples/15_Self_Supervised_Pretraning_pt2.html#self-supervised-pretraining-for-tabular-data","title":"Self Supervised Pretraining for Tabular Data\u00b6","text":"<p>We have implemented two Self Supervised Pre-training routines that allow the user to pre-train all tabular models in the library with the exception of the TabPerceiver (which is a special monster).</p> <p>The two routines implemented are illustrated in the figures below. The 1st is from TabNet: Attentive Interpretable Tabular Learning and is designed for models that do not use transformer-based architectures, while the second is from SAINT: Improved Neural Networks for Tabular Data via Row Attention and Contrastive Pre-Training, and is designed for models that use transformer-based architectures.</p> <p> </p> <p>Fig 1. Figure 2 in their paper. I have included de original caption in case is useful, althought the Figure itself is pretty self explanatory</p> <p> </p> <p>Fig 2. Figure 1 in their paper. Here the caption is necessary \ud83d\ude0f</p> <p>It is beyond the scope of this notebook to explain in detail those implementations. Therefore, we strongly recommend the user to go and read the papers if this functionality is of interest to her/him.</p> <p>One thing is worth noticing however. As seen in Fig 1(the TabNet paper's Fig 2) the masking of the input features happens in the feature space. However, the implementation in this library is inspired by that at the dreamquark-ai repo, which is in itself inspired by the original implementation (by the way, at this point I will write it once again. All TabNet related things in this library are inspired when not directly based in the code in that repo, therefore, ALL CREDIT TO THE GUYS AT dreamquark-ai).</p> <p>In that implementation the masking happens in the embedding space, and currently does not mask the entire embedding (i.e. categorical feature). We decided to release as it is in this version and we will implement the exact same process described in the paper in future releases.</p> <p>Having said all of the above let's see how to use self supervision for tabular data with <code>pytorch-widedeep</code>. We will concentrate in this notebook on the 2nd of the two approaches (the 'SAINT approach'). For details on the 1st approach (the 'TabNet' approach) please see <code>16_Self_Supervised_Pretraning_pt1</code>.</p>"},{"location":"examples/15_Self_Supervised_Pretraning_pt2.html#self-supervision-transformer-based-models","title":"Self Supervision transformer-based models..\u00b6","text":"<p>...or in general, for models where the embeddigns have all the same dimensions. In this library, these are:</p> <ul> <li>TabTransformer</li> <li>FTTransformer</li> <li>SAINT</li> <li>TabFastFormer</li> </ul> <p>Note that there is one additional Transformer-based model, the <code>TabPerceiver</code>, however this is a \"particular\" model and at the moment we do not support self supervision for it, but it will come.</p> <p>Let see at one example using the <code>FTTransformer</code>.</p>"},{"location":"examples/16_Usign_a_custom_hugging_face_model.html","title":"16_Usign-a-custom-hugging-face-model","text":"In\u00a0[1]: Copied! <pre>import numpy as np\nimport torch\nimport lightgbm as lgb\nfrom lightgbm import Dataset as lgbDataset\nfrom scipy.sparse import hstack, csr_matrix\nfrom sklearn.metrics import (\n    f1_score,\n    recall_score,\n    accuracy_score,\n    precision_score,\n    confusion_matrix,\n)\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.feature_extraction.text import TfidfVectorizer\n\nfrom torch import Tensor, nn\nfrom transformers import DistilBertModel, DistilBertTokenizer\nfrom pytorch_widedeep import Trainer\nfrom pytorch_widedeep.models import TabMlp, BasicRNN, WideDeep\nfrom pytorch_widedeep.metrics import F1Score, Accuracy\nfrom pytorch_widedeep.utils import Tokenizer, LabelEncoder\nfrom pytorch_widedeep.preprocessing import TextPreprocessor, TabPreprocessor\nfrom pytorch_widedeep.datasets import load_womens_ecommerce\nfrom pytorch_widedeep.utils.fastai_transforms import (\n    fix_html,\n    spec_add_spaces,\n    rm_useless_spaces,\n)\n</pre> import numpy as np import torch import lightgbm as lgb from lightgbm import Dataset as lgbDataset from scipy.sparse import hstack, csr_matrix from sklearn.metrics import (     f1_score,     recall_score,     accuracy_score,     precision_score,     confusion_matrix, ) from sklearn.model_selection import train_test_split from sklearn.feature_extraction.text import TfidfVectorizer  from torch import Tensor, nn from transformers import DistilBertModel, DistilBertTokenizer from pytorch_widedeep import Trainer from pytorch_widedeep.models import TabMlp, BasicRNN, WideDeep from pytorch_widedeep.metrics import F1Score, Accuracy from pytorch_widedeep.utils import Tokenizer, LabelEncoder from pytorch_widedeep.preprocessing import TextPreprocessor, TabPreprocessor from pytorch_widedeep.datasets import load_womens_ecommerce from pytorch_widedeep.utils.fastai_transforms import (     fix_html,     spec_add_spaces,     rm_useless_spaces, ) <p>Let's load the data and have a look:</p> In\u00a0[2]: Copied! <pre>df = load_womens_ecommerce(as_frame=True)\n\ndf.columns = [c.replace(\" \", \"_\").lower() for c in df.columns]\n\n# classes from [0,num_class)\ndf[\"rating\"] = (df[\"rating\"] - 1).astype(\"int64\")\n\n# group reviews with 1 and 2 scores into one class\ndf.loc[df.rating == 0, \"rating\"] = 1\n\n# and back again to [0,num_class)\ndf[\"rating\"] = (df[\"rating\"] - 1).astype(\"int64\")\n\n# drop short reviews\ndf = df[~df.review_text.isna()]\ndf[\"review_length\"] = df.review_text.apply(lambda x: len(x.split(\" \")))\ndf = df[df.review_length &gt;= 5]\ndf = df.drop(\"review_length\", axis=1).reset_index(drop=True)\n</pre> df = load_womens_ecommerce(as_frame=True)  df.columns = [c.replace(\" \", \"_\").lower() for c in df.columns]  # classes from [0,num_class) df[\"rating\"] = (df[\"rating\"] - 1).astype(\"int64\")  # group reviews with 1 and 2 scores into one class df.loc[df.rating == 0, \"rating\"] = 1  # and back again to [0,num_class) df[\"rating\"] = (df[\"rating\"] - 1).astype(\"int64\")  # drop short reviews df = df[~df.review_text.isna()] df[\"review_length\"] = df.review_text.apply(lambda x: len(x.split(\" \"))) df = df[df.review_length &gt;= 5] df = df.drop(\"review_length\", axis=1).reset_index(drop=True) In\u00a0[3]: Copied! <pre>df.head()\n</pre> df.head() Out[3]: clothing_id age title review_text rating recommended_ind positive_feedback_count division_name department_name class_name 0 767 33 None Absolutely wonderful - silky and sexy and comf... 2 1 0 Initmates Intimate Intimates 1 1080 34 None Love this dress!  it's sooo pretty.  i happene... 3 1 4 General Dresses Dresses 2 1077 60 Some major design flaws I had such high hopes for this dress and reall... 1 0 0 General Dresses Dresses 3 1049 50 My favorite buy! I love, love, love this jumpsuit. it's fun, fl... 3 1 0 General Petite Bottoms Pants 4 847 47 Flattering shirt This shirt is very flattering to all due to th... 3 1 6 General Tops Blouses <p>So, we will use the <code>review_text</code> column to predict the <code>rating</code>. Later on, we will try to combine it with some other columns (like <code>division_name</code> and <code>age</code>) see if these help.</p> <p>Let's first have a look to the distribution of ratings</p> In\u00a0[4]: Copied! <pre>df.rating.value_counts()\n</pre> df.rating.value_counts() Out[4]: <pre>rating\n3    12515\n2     4904\n1     2820\n0     2369\nName: count, dtype: int64</pre> <p>This shows that we could have perhaps grouped rating scores of 1, 2 and 3 into 1...but anyway, let's just move on with those 4 classes.</p> <p>We are not going to carry any hyperparameter optimization here, so, we will only need a train and a test set (i.e.  no need of a validation set for the example in this notebook)</p> In\u00a0[5]: Copied! <pre>train, test = train_test_split(df, train_size=0.8, random_state=1, stratify=df.rating)\n</pre> train, test = train_test_split(df, train_size=0.8, random_state=1, stratify=df.rating) <p>Let's see what we have to beat. What metrics would we obtain if we always predict the most common rating (3)?</p> In\u00a0[6]: Copied! <pre>most_common_pred = [train.rating.value_counts().index[0]] * len(test)\n\nmost_common_acc = accuracy_score(test.rating, most_common_pred)\nmost_common_f1 = f1_score(test.rating, most_common_pred, average=\"weighted\")\n</pre> most_common_pred = [train.rating.value_counts().index[0]] * len(test)  most_common_acc = accuracy_score(test.rating, most_common_pred) most_common_f1 = f1_score(test.rating, most_common_pred, average=\"weighted\") In\u00a0[7]: Copied! <pre>print(f\"Accuracy: {most_common_acc}. F1 Score: {most_common_f1}\")\n</pre> print(f\"Accuracy: {most_common_acc}. F1 Score: {most_common_f1}\") <pre>Accuracy: 0.553516143299425. F1 Score: 0.3944344218301668\n</pre> <p>ok, these are our \"baseline\" metrics.</p> <p>Let's start by using simply tf-idf + lightGBM</p> In\u00a0[8]: Copied! <pre># ?Tokenizer\n</pre> # ?Tokenizer In\u00a0[9]: Copied! <pre># this Tokenizer is part of our utils module but of course, any valid tokenizer can be used here.\n\n# When using notebooks there seems to be an issue related with multiprocessing (and sometimes tqdm)\n# that can only be solved by using only one CPU\ntok = Tokenizer(n_cpus=1)\ntok_reviews_tr = tok.process_all(train.review_text.tolist())\ntok_reviews_te = tok.process_all(test.review_text.tolist())\n</pre> # this Tokenizer is part of our utils module but of course, any valid tokenizer can be used here.  # When using notebooks there seems to be an issue related with multiprocessing (and sometimes tqdm) # that can only be solved by using only one CPU tok = Tokenizer(n_cpus=1) tok_reviews_tr = tok.process_all(train.review_text.tolist()) tok_reviews_te = tok.process_all(test.review_text.tolist()) In\u00a0[10]: Copied! <pre>vectorizer = TfidfVectorizer(\n    max_features=5000, preprocessor=lambda x: x, tokenizer=lambda x: x, min_df=5\n)\n\nX_text_tr = vectorizer.fit_transform(tok_reviews_tr)\nX_text_te = vectorizer.transform(tok_reviews_te)\n</pre> vectorizer = TfidfVectorizer(     max_features=5000, preprocessor=lambda x: x, tokenizer=lambda x: x, min_df=5 )  X_text_tr = vectorizer.fit_transform(tok_reviews_tr) X_text_te = vectorizer.transform(tok_reviews_te) <pre>/Users/javierrodriguezzaurin/.pyenv/versions/3.10.13/envs/widedeep310/lib/python3.10/site-packages/sklearn/feature_extraction/text.py:525: UserWarning: The parameter 'token_pattern' will not be used since 'tokenizer' is not None'\n  warnings.warn(\n</pre> In\u00a0[11]: Copied! <pre>X_text_tr\n</pre> X_text_tr Out[11]: <pre>&lt;18086x4566 sparse matrix of type '&lt;class 'numpy.float64'&gt;'\n\twith 884074 stored elements in Compressed Sparse Row format&gt;</pre> <p>We now move our matrices to lightGBM <code>Dataset</code> format</p> In\u00a0[12]: Copied! <pre>lgbtrain_text = lgbDataset(\n    X_text_tr,\n    train.rating.values,\n    free_raw_data=False,\n)\n\nlgbtest_text = lgbDataset(\n    X_text_te,\n    test.rating.values,\n    reference=lgbtrain_text,\n    free_raw_data=False,\n)\n</pre> lgbtrain_text = lgbDataset(     X_text_tr,     train.rating.values,     free_raw_data=False, )  lgbtest_text = lgbDataset(     X_text_te,     test.rating.values,     reference=lgbtrain_text,     free_raw_data=False, ) <p>and off we go. By the way, I think as we run the next cell, we should appreciate how fast lightGBM runs. Yes, the input is a sparse matrix, but still, trains on 18086x4566 in a matter of secs</p> In\u00a0[\u00a0]: Copied! <pre>lgb_text_model = lgb.train(\n    {\"objective\": \"multiclass\", \"num_classes\": 4},\n    lgbtrain_text,\n    valid_sets=[lgbtest_text, lgbtrain_text],\n    valid_names=[\"test\", \"train\"],\n)\n</pre> lgb_text_model = lgb.train(     {\"objective\": \"multiclass\", \"num_classes\": 4},     lgbtrain_text,     valid_sets=[lgbtest_text, lgbtrain_text],     valid_names=[\"test\", \"train\"], ) In\u00a0[14]: Copied! <pre>preds_text = lgb_text_model.predict(X_text_te)\npred_text_class = np.argmax(preds_text, 1)\n</pre> preds_text = lgb_text_model.predict(X_text_te) pred_text_class = np.argmax(preds_text, 1) In\u00a0[15]: Copied! <pre>acc_text = accuracy_score(lgbtest_text.label, pred_text_class)\nf1_text = f1_score(lgbtest_text.label, pred_text_class, average=\"weighted\")\ncm_text = confusion_matrix(lgbtest_text.label, pred_text_class)\n</pre> acc_text = accuracy_score(lgbtest_text.label, pred_text_class) f1_text = f1_score(lgbtest_text.label, pred_text_class, average=\"weighted\") cm_text = confusion_matrix(lgbtest_text.label, pred_text_class) In\u00a0[16]: Copied! <pre>print(f\"LightGBM Accuracy: {acc_text}. LightGBM F1 Score: {f1_text}\")\n</pre> print(f\"LightGBM Accuracy: {acc_text}. LightGBM F1 Score: {f1_text}\") <pre>LightGBM Accuracy: 0.6444051304732419. LightGBM F1 Score: 0.617154488246181\n</pre> In\u00a0[17]: Copied! <pre>print(f\"LightGBM Confusion Matrix: \\n {cm_text}\")\n</pre> print(f\"LightGBM Confusion Matrix: \\n {cm_text}\") <pre>LightGBM Confusion Matrix: \n [[ 199  135   61   79]\n [ 123  169  149  123]\n [  30   94  279  578]\n [  16   30  190 2267]]\n</pre> <p>Ok, so, with no hyperparameter optimization lightGBM gets an accuracy of 0.64 and a F1 score of 0.62. This is significantly better than predicting always the most popular.</p> <p>Let's see if in this implementation, some additional features, like <code>age</code> or <code>class_name</code> are of any help</p> In\u00a0[18]: Copied! <pre>tab_cols = [\n    \"age\",\n    \"division_name\",\n    \"department_name\",\n    \"class_name\",\n]\n\nfor tab_df in [train, test]:\n    for c in [\"division_name\", \"department_name\", \"class_name\"]:\n        tab_df[c] = tab_df[c].str.lower()\n        tab_df[c].fillna(\"missing\", inplace=True)\n</pre> tab_cols = [     \"age\",     \"division_name\",     \"department_name\",     \"class_name\", ]  for tab_df in [train, test]:     for c in [\"division_name\", \"department_name\", \"class_name\"]:         tab_df[c] = tab_df[c].str.lower()         tab_df[c].fillna(\"missing\", inplace=True) In\u00a0[19]: Copied! <pre># This is our LabelEncoder. A class that is designed to work with the models in this library but\n# can be used for general purposes\nle = LabelEncoder(columns_to_encode=[\"division_name\", \"department_name\", \"class_name\"])\ntrain_tab_le = le.fit_transform(train)\ntest_tab_le = le.transform(test)\n</pre> # This is our LabelEncoder. A class that is designed to work with the models in this library but # can be used for general purposes le = LabelEncoder(columns_to_encode=[\"division_name\", \"department_name\", \"class_name\"]) train_tab_le = le.fit_transform(train) test_tab_le = le.transform(test) In\u00a0[20]: Copied! <pre>train_tab_le.head()\n</pre> train_tab_le.head() Out[20]: clothing_id age title review_text rating recommended_ind positive_feedback_count division_name department_name class_name 4541 836 35 None Bought this on sale in my reg size- 10. im 5'9... 2 1 2 1 1 1 18573 1022 25 Look like \"mom jeans\" Maybe i just have the wrong body type for thes... 1 0 0 2 2 2 1058 815 39 Ig brought me here Love the way this top layers under my jackets ... 2 1 0 1 1 1 12132 984 47 Runs small especially the arms I love this jacket. it's the prettiest and mos... 3 1 0 1 3 3 20756 1051 42 True red, true beauty. These pants are gorgeous--the fabric has a sat... 3 1 0 2 2 4 <p>let's for example have a look to the encodings for the categorical feature <code>class_name</code></p> In\u00a0[21]: Copied! <pre>le.encoding_dict[\"class_name\"]\n</pre> le.encoding_dict[\"class_name\"] Out[21]: <pre>{'blouses': 1,\n 'jeans': 2,\n 'jackets': 3,\n 'pants': 4,\n 'knits': 5,\n 'dresses': 6,\n 'skirts': 7,\n 'sweaters': 8,\n 'fine gauge': 9,\n 'legwear': 10,\n 'lounge': 11,\n 'shorts': 12,\n 'outerwear': 13,\n 'intimates': 14,\n 'swim': 15,\n 'trend': 16,\n 'sleep': 17,\n 'layering': 18,\n 'missing': 19,\n 'casual bottoms': 20,\n 'chemises': 21}</pre> In\u00a0[22]: Copied! <pre># tabular training and test sets\nX_tab_tr = csr_matrix(train_tab_le[tab_cols].values)\nX_tab_te = csr_matrix(test_tab_le[tab_cols].values)\n\n# text + tabular training and test sets\nX_tab_text_tr = hstack((X_tab_tr, X_text_tr))\nX_tab_text_te = hstack((X_tab_te, X_text_te))\n</pre> # tabular training and test sets X_tab_tr = csr_matrix(train_tab_le[tab_cols].values) X_tab_te = csr_matrix(test_tab_le[tab_cols].values)  # text + tabular training and test sets X_tab_text_tr = hstack((X_tab_tr, X_text_tr)) X_tab_text_te = hstack((X_tab_te, X_text_te)) In\u00a0[23]: Copied! <pre>X_tab_tr\n</pre> X_tab_tr Out[23]: <pre>&lt;18086x4 sparse matrix of type '&lt;class 'numpy.int64'&gt;'\n\twith 72344 stored elements in Compressed Sparse Row format&gt;</pre> In\u00a0[24]: Copied! <pre>X_tab_text_tr\n</pre> X_tab_text_tr Out[24]: <pre>&lt;18086x4570 sparse matrix of type '&lt;class 'numpy.float64'&gt;'\n\twith 956418 stored elements in Compressed Sparse Row format&gt;</pre> In\u00a0[25]: Copied! <pre>lgbtrain_tab_text = lgbDataset(\n    X_tab_text_tr,\n    train.rating.values,\n    categorical_feature=[0, 1, 2, 3],\n    free_raw_data=False,\n)\n\nlgbtest_tab_text = lgbDataset(\n    X_tab_text_te,\n    test.rating.values,\n    reference=lgbtrain_tab_text,\n    free_raw_data=False,\n)\n</pre> lgbtrain_tab_text = lgbDataset(     X_tab_text_tr,     train.rating.values,     categorical_feature=[0, 1, 2, 3],     free_raw_data=False, )  lgbtest_tab_text = lgbDataset(     X_tab_text_te,     test.rating.values,     reference=lgbtrain_tab_text,     free_raw_data=False, ) In\u00a0[26]: Copied! <pre>lgb_tab_text_model = lgb.train(\n    {\"objective\": \"multiclass\", \"num_classes\": 4},\n    lgbtrain_tab_text,\n    valid_sets=[lgbtrain_tab_text, lgbtest_tab_text],\n    valid_names=[\"test\", \"train\"],\n    verbose_eval=False,\n)\n</pre> lgb_tab_text_model = lgb.train(     {\"objective\": \"multiclass\", \"num_classes\": 4},     lgbtrain_tab_text,     valid_sets=[lgbtrain_tab_text, lgbtest_tab_text],     valid_names=[\"test\", \"train\"],     verbose_eval=False, ) <pre>/opt/conda/envs/wd38/lib/python3.8/site-packages/lightgbm/basic.py:2065: UserWarning: Using categorical_feature in Dataset.\n  _log_warning('Using categorical_feature in Dataset.')\n/opt/conda/envs/wd38/lib/python3.8/site-packages/lightgbm/basic.py:2068: UserWarning: categorical_feature in Dataset is overridden.\nNew categorical_feature is [0, 1, 2, 3]\n  _log_warning('categorical_feature in Dataset is overridden.\\n'\n/opt/conda/envs/wd38/lib/python3.8/site-packages/lightgbm/engine.py:239: UserWarning: 'verbose_eval' argument is deprecated and will be removed in a future release of LightGBM. Pass 'log_evaluation()' callback via 'callbacks' argument instead.\n  _log_warning(\"'verbose_eval' argument is deprecated and will be removed in a future release of LightGBM. \"\n</pre> <pre>[LightGBM] [Warning] Auto-choosing col-wise multi-threading, the overhead of testing was 0.138280 seconds.\nYou can set `force_col_wise=true` to remove the overhead.\n[LightGBM] [Info] Total Bins 143432\n[LightGBM] [Info] Number of data points in the train set: 18086, number of used features: 2289\n[LightGBM] [Info] Start training from score -2.255919\n[LightGBM] [Info] Start training from score -2.081545\n[LightGBM] [Info] Start training from score -1.528281\n[LightGBM] [Info] Start training from score -0.591354\n</pre> <pre>/opt/conda/envs/wd38/lib/python3.8/site-packages/lightgbm/basic.py:1780: UserWarning: Overriding the parameters from Reference Dataset.\n  _log_warning('Overriding the parameters from Reference Dataset.')\n/opt/conda/envs/wd38/lib/python3.8/site-packages/lightgbm/basic.py:1513: UserWarning: categorical_column in param dict is overridden.\n  _log_warning(f'{cat_alias} in param dict is overridden.')\n</pre> In\u00a0[27]: Copied! <pre>preds_tab_text = lgb_tab_text_model.predict(X_tab_text_te)\npreds_tab_text_class = np.argmax(preds_tab_text, 1)\n\nacc_tab_text = accuracy_score(lgbtest_tab_text.label, preds_tab_text_class)\nf1_tab_text = f1_score(lgbtest_tab_text.label, preds_tab_text_class, average=\"weighted\")\ncm_tab_text = confusion_matrix(lgbtest_tab_text.label, preds_tab_text_class)\n</pre> preds_tab_text = lgb_tab_text_model.predict(X_tab_text_te) preds_tab_text_class = np.argmax(preds_tab_text, 1)  acc_tab_text = accuracy_score(lgbtest_tab_text.label, preds_tab_text_class) f1_tab_text = f1_score(lgbtest_tab_text.label, preds_tab_text_class, average=\"weighted\") cm_tab_text = confusion_matrix(lgbtest_tab_text.label, preds_tab_text_class) In\u00a0[28]: Copied! <pre>print(\n    f\"LightGBM text + tabular Accuracy: {acc_tab_text}. LightGBM text + tabular F1 Score: {f1_tab_text}\"\n)\n</pre> print(     f\"LightGBM text + tabular Accuracy: {acc_tab_text}. LightGBM text + tabular F1 Score: {f1_tab_text}\" ) <pre>LightGBM text + tabular Accuracy: 0.6382131800088456. LightGBM text + tabular F1 Score: 0.6080251307242649\n</pre> In\u00a0[29]: Copied! <pre>print(f\"LightGBM text + tabular Confusion Matrix:\\n {cm_tab_text}\")\n</pre> print(f\"LightGBM text + tabular Confusion Matrix:\\n {cm_tab_text}\") <pre>LightGBM text + tabular Confusion Matrix:\n [[ 193  123   68   90]\n [ 123  146  157  138]\n [  37   90  272  582]\n [  16   37  175 2275]]\n</pre> <p>So, in this set up, the addition tabular columns do not help performance.</p> In\u00a0[30]: Copied! <pre>text_preprocessor = TextPreprocessor(\n    text_col=\"review_text\", max_vocab=5000, min_freq=5, maxlen=90, n_cpus=1\n)\n\nwd_X_text_tr = text_preprocessor.fit_transform(train)\nwd_X_text_te = text_preprocessor.transform(test)\n</pre> text_preprocessor = TextPreprocessor(     text_col=\"review_text\", max_vocab=5000, min_freq=5, maxlen=90, n_cpus=1 )  wd_X_text_tr = text_preprocessor.fit_transform(train) wd_X_text_te = text_preprocessor.transform(test) <pre>The vocabulary contains 4328 tokens\n</pre> In\u00a0[31]: Copied! <pre>basic_rnn = BasicRNN(\n    vocab_size=len(text_preprocessor.vocab.itos),\n    embed_dim=300,\n    hidden_dim=64,\n    n_layers=3,\n    rnn_dropout=0.2,\n    head_hidden_dims=[32],\n)\n\n\nwd_text_model = WideDeep(deeptext=basic_rnn, pred_dim=4)\n</pre> basic_rnn = BasicRNN(     vocab_size=len(text_preprocessor.vocab.itos),     embed_dim=300,     hidden_dim=64,     n_layers=3,     rnn_dropout=0.2,     head_hidden_dims=[32], )   wd_text_model = WideDeep(deeptext=basic_rnn, pred_dim=4) In\u00a0[32]: Copied! <pre>wd_text_model\n</pre> wd_text_model Out[32]: <pre>WideDeep(\n  (deeptext): Sequential(\n    (0): BasicRNN(\n      (word_embed): Embedding(4328, 300, padding_idx=1)\n      (rnn): LSTM(300, 64, num_layers=3, batch_first=True, dropout=0.2)\n      (rnn_mlp): MLP(\n        (mlp): Sequential(\n          (dense_layer_0): Sequential(\n            (0): Linear(in_features=64, out_features=32, bias=True)\n            (1): ReLU(inplace=True)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=32, out_features=4, bias=True)\n  )\n)</pre> In\u00a0[33]: Copied! <pre>text_trainer = Trainer(\n    wd_text_model,\n    objective=\"multiclass\",\n    metrics=[Accuracy, F1Score(average=True)],\n    num_workers=0,  # As in the case of the tokenizer, in notebook I need to set this to 0 for the Trainer to work\n)\n</pre> text_trainer = Trainer(     wd_text_model,     objective=\"multiclass\",     metrics=[Accuracy, F1Score(average=True)],     num_workers=0,  # As in the case of the tokenizer, in notebook I need to set this to 0 for the Trainer to work ) In\u00a0[34]: Copied! <pre>text_trainer.fit(\n    X_text=wd_X_text_tr,\n    target=train.rating.values,\n    n_epochs=5,\n    batch_size=256,\n)\n</pre> text_trainer.fit(     X_text=wd_X_text_tr,     target=train.rating.values,     n_epochs=5,     batch_size=256, ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 71/71 [00:01&lt;00:00, 52.39it/s, loss=1.16, metrics={'acc': 0.5349, 'f1': 0.2011}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 71/71 [00:01&lt;00:00, 70.35it/s, loss=0.964, metrics={'acc': 0.5827, 'f1': 0.3005}]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 71/71 [00:01&lt;00:00, 70.33it/s, loss=0.845, metrics={'acc': 0.6252, 'f1': 0.4133}]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 71/71 [00:01&lt;00:00, 69.99it/s, loss=0.765, metrics={'acc': 0.6575, 'f1': 0.4875}]\nepoch 5: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 71/71 [00:01&lt;00:00, 69.55it/s, loss=0.709, metrics={'acc': 0.6879, 'f1': 0.5423}]\n</pre> In\u00a0[35]: Copied! <pre>wd_pred_text = text_trainer.predict_proba(X_text=wd_X_text_te)\nwd_pred_text_class = np.argmax(wd_pred_text, 1)\n</pre> wd_pred_text = text_trainer.predict_proba(X_text=wd_X_text_te) wd_pred_text_class = np.argmax(wd_pred_text, 1) <pre>predict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 18/18 [00:00&lt;00:00, 211.51it/s]\n</pre> In\u00a0[36]: Copied! <pre>wd_acc_text = accuracy_score(test.rating, wd_pred_text_class)\nwd_f1_text = f1_score(test.rating, wd_pred_text_class, average=\"weighted\")\nwd_cm_text = confusion_matrix(test.rating, wd_pred_text_class)\n</pre> wd_acc_text = accuracy_score(test.rating, wd_pred_text_class) wd_f1_text = f1_score(test.rating, wd_pred_text_class, average=\"weighted\") wd_cm_text = confusion_matrix(test.rating, wd_pred_text_class) In\u00a0[37]: Copied! <pre>print(f\"Basic RNN Accuracy: {wd_acc_text}. Basic RNN F1 Score: {wd_f1_text}\")\n</pre> print(f\"Basic RNN Accuracy: {wd_acc_text}. Basic RNN F1 Score: {wd_f1_text}\") <pre>Basic RNN Accuracy: 0.6076957098628926. Basic RNN F1 Score: 0.6017335854471788\n</pre> In\u00a0[38]: Copied! <pre>print(f\"Basic RNN Confusion Matrix:\\n {wd_cm_text}\")\n</pre> print(f\"Basic RNN Confusion Matrix:\\n {wd_cm_text}\") <pre>Basic RNN Confusion Matrix:\n [[ 327   76   62    9]\n [ 285  115  117   47]\n [ 131  122  315  413]\n [  42   69  401 1991]]\n</pre> <p>The performance is very similar to that of using simply tf-idf and lightgbm. Let see if adding tabular features helps when using <code>pytorch-widedeep</code></p> In\u00a0[39]: Copied! <pre># ?TabPreprocessor\n</pre> # ?TabPreprocessor In\u00a0[40]: Copied! <pre>tab_preprocessor = TabPreprocessor(cat_embed_cols=tab_cols)\n\nwd_X_tab_tr = tab_preprocessor.fit_transform(train)\nwd_X_tab_te = tab_preprocessor.transform(test)\n</pre> tab_preprocessor = TabPreprocessor(cat_embed_cols=tab_cols)  wd_X_tab_tr = tab_preprocessor.fit_transform(train) wd_X_tab_te = tab_preprocessor.transform(test) In\u00a0[41]: Copied! <pre># ?TabMlp\n</pre> # ?TabMlp In\u00a0[42]: Copied! <pre>tab_model = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    mlp_hidden_dims=[100, 50],\n)\n</pre> tab_model = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     mlp_hidden_dims=[100, 50], ) In\u00a0[43]: Copied! <pre>tab_model\n</pre> tab_model Out[43]: <pre>TabMlp(\n  (cat_and_cont_embed): DiffSizeCatAndContEmbeddings(\n    (cat_embed): DiffSizeCatEmbeddings(\n      (embed_layers): ModuleDict(\n        (emb_layer_age): Embedding(78, 18, padding_idx=0)\n        (emb_layer_division_name): Embedding(5, 3, padding_idx=0)\n        (emb_layer_department_name): Embedding(8, 5, padding_idx=0)\n        (emb_layer_class_name): Embedding(22, 9, padding_idx=0)\n      )\n      (embedding_dropout): Dropout(p=0.1, inplace=False)\n    )\n  )\n  (encoder): MLP(\n    (mlp): Sequential(\n      (dense_layer_0): Sequential(\n        (0): Dropout(p=0.1, inplace=False)\n        (1): Linear(in_features=35, out_features=100, bias=True)\n        (2): ReLU(inplace=True)\n      )\n      (dense_layer_1): Sequential(\n        (0): Dropout(p=0.1, inplace=False)\n        (1): Linear(in_features=100, out_features=50, bias=True)\n        (2): ReLU(inplace=True)\n      )\n    )\n  )\n)</pre> In\u00a0[44]: Copied! <pre>text_model = BasicRNN(\n    vocab_size=len(text_preprocessor.vocab.itos),\n    embed_dim=300,\n    hidden_dim=64,\n    n_layers=3,\n    rnn_dropout=0.2,\n    head_hidden_dims=[32],\n)\n</pre> text_model = BasicRNN(     vocab_size=len(text_preprocessor.vocab.itos),     embed_dim=300,     hidden_dim=64,     n_layers=3,     rnn_dropout=0.2,     head_hidden_dims=[32], ) In\u00a0[45]: Copied! <pre>wd_tab_and_text_model = WideDeep(deeptabular=tab_model, deeptext=text_model, pred_dim=4)\n</pre> wd_tab_and_text_model = WideDeep(deeptabular=tab_model, deeptext=text_model, pred_dim=4) In\u00a0[46]: Copied! <pre>wd_tab_and_text_model\n</pre> wd_tab_and_text_model Out[46]: <pre>WideDeep(\n  (deeptabular): Sequential(\n    (0): TabMlp(\n      (cat_and_cont_embed): DiffSizeCatAndContEmbeddings(\n        (cat_embed): DiffSizeCatEmbeddings(\n          (embed_layers): ModuleDict(\n            (emb_layer_age): Embedding(78, 18, padding_idx=0)\n            (emb_layer_division_name): Embedding(5, 3, padding_idx=0)\n            (emb_layer_department_name): Embedding(8, 5, padding_idx=0)\n            (emb_layer_class_name): Embedding(22, 9, padding_idx=0)\n          )\n          (embedding_dropout): Dropout(p=0.1, inplace=False)\n        )\n      )\n      (encoder): MLP(\n        (mlp): Sequential(\n          (dense_layer_0): Sequential(\n            (0): Dropout(p=0.1, inplace=False)\n            (1): Linear(in_features=35, out_features=100, bias=True)\n            (2): ReLU(inplace=True)\n          )\n          (dense_layer_1): Sequential(\n            (0): Dropout(p=0.1, inplace=False)\n            (1): Linear(in_features=100, out_features=50, bias=True)\n            (2): ReLU(inplace=True)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=50, out_features=4, bias=True)\n  )\n  (deeptext): Sequential(\n    (0): BasicRNN(\n      (word_embed): Embedding(4328, 300, padding_idx=1)\n      (rnn): LSTM(300, 64, num_layers=3, batch_first=True, dropout=0.2)\n      (rnn_mlp): MLP(\n        (mlp): Sequential(\n          (dense_layer_0): Sequential(\n            (0): Linear(in_features=64, out_features=32, bias=True)\n            (1): ReLU(inplace=True)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=32, out_features=4, bias=True)\n  )\n)</pre> In\u00a0[47]: Copied! <pre>tab_and_text_trainer = Trainer(\n    wd_tab_and_text_model,\n    objective=\"multiclass\",\n    metrics=[Accuracy, F1Score(average=True)],\n    num_workers=0,  # As in the case of the tokenizer, in notebook I need to set this to 0 for the Trainer to work\n)\n</pre> tab_and_text_trainer = Trainer(     wd_tab_and_text_model,     objective=\"multiclass\",     metrics=[Accuracy, F1Score(average=True)],     num_workers=0,  # As in the case of the tokenizer, in notebook I need to set this to 0 for the Trainer to work ) In\u00a0[48]: Copied! <pre>tab_and_text_trainer.fit(\n    X_tab=wd_X_tab_tr,\n    X_text=wd_X_text_tr,\n    target=train.rating.values,\n    n_epochs=5,\n    batch_size=256,\n)\n</pre> tab_and_text_trainer.fit(     X_tab=wd_X_tab_tr,     X_text=wd_X_text_tr,     target=train.rating.values,     n_epochs=5,     batch_size=256, ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 71/71 [00:01&lt;00:00, 52.04it/s, loss=1.13, metrics={'acc': 0.538, 'f1': 0.1911}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 71/71 [00:01&lt;00:00, 52.28it/s, loss=0.936, metrics={'acc': 0.5887, 'f1': 0.3507}]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 71/71 [00:01&lt;00:00, 52.26it/s, loss=0.825, metrics={'acc': 0.6394, 'f1': 0.4545}]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 71/71 [00:01&lt;00:00, 51.33it/s, loss=0.757, metrics={'acc': 0.6696, 'f1': 0.5214}]\nepoch 5: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 71/71 [00:01&lt;00:00, 50.39it/s, loss=0.702, metrics={'acc': 0.6963, 'f1': 0.5654}]\n</pre> In\u00a0[49]: Copied! <pre>wd_pred_tab_and_text = tab_and_text_trainer.predict_proba(\n    X_tab=wd_X_tab_te, X_text=wd_X_text_te\n)\nwd_pred_tab_and_text_class = np.argmax(wd_pred_tab_and_text, 1)\n</pre> wd_pred_tab_and_text = tab_and_text_trainer.predict_proba(     X_tab=wd_X_tab_te, X_text=wd_X_text_te ) wd_pred_tab_and_text_class = np.argmax(wd_pred_tab_and_text, 1) <pre>predict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 18/18 [00:00&lt;00:00, 136.94it/s]\n</pre> In\u00a0[50]: Copied! <pre>wd_acc_tab_and_text = accuracy_score(test.rating, wd_pred_tab_and_text_class)\nwd_f1_tab_and_text = f1_score(\n    test.rating, wd_pred_tab_and_text_class, average=\"weighted\"\n)\nwd_cm_tab_and_text = confusion_matrix(test.rating, wd_pred_tab_and_text_class)\n</pre> wd_acc_tab_and_text = accuracy_score(test.rating, wd_pred_tab_and_text_class) wd_f1_tab_and_text = f1_score(     test.rating, wd_pred_tab_and_text_class, average=\"weighted\" ) wd_cm_tab_and_text = confusion_matrix(test.rating, wd_pred_tab_and_text_class) In\u00a0[51]: Copied! <pre>print(\n    f\"Basic RNN + Tabular  Accuracy: {wd_acc_tab_and_text}. Basic RNN + TabularF1 Score: {wd_f1_tab_and_text}\"\n)\nprint(f\"Basic RNN + Tabular  Confusion Matrix:\\n {wd_cm_tab_and_text}\")\n</pre> print(     f\"Basic RNN + Tabular  Accuracy: {wd_acc_tab_and_text}. Basic RNN + TabularF1 Score: {wd_f1_tab_and_text}\" ) print(f\"Basic RNN + Tabular  Confusion Matrix:\\n {wd_cm_tab_and_text}\") <pre>Basic RNN + Tabular  Accuracy: 0.6333480760725343. Basic RNN + TabularF1 Score: 0.6332310089593208\nBasic RNN + Tabular  Confusion Matrix:\n [[ 267  132   65   10]\n [ 198  168  159   39]\n [  57  113  410  401]\n [  12   58  414 2019]]\n</pre> <p>We are going to \"manually\" code the Tokenizer and the model and see how they can be used as part of the process along with the <code>pytorch-widedeep</code> library.</p> <p>Tokenizer:</p> In\u00a0[52]: Copied! <pre>class BertTokenizer(object):\n    def __init__(\n        self,\n        pretrained_tokenizer=\"distilbert-base-uncased\",\n        do_lower_case=True,\n        max_length=90,\n    ):\n        super(BertTokenizer, self).__init__()\n        self.pretrained_tokenizer = pretrained_tokenizer\n        self.do_lower_case = do_lower_case\n        self.max_length = max_length\n\n    def fit(self, texts):\n        self.tokenizer = DistilBertTokenizer.from_pretrained(\n            self.pretrained_tokenizer, do_lower_case=self.do_lower_case\n        )\n\n        return self\n\n    def transform(self, texts):\n        input_ids = []\n        for text in texts:\n            encoded_sent = self.tokenizer.encode_plus(\n                text=self._pre_rules(text),\n                add_special_tokens=True,\n                max_length=self.max_length,\n                padding=\"max_length\",\n                truncation=True,\n            )\n\n            input_ids.append(encoded_sent.get(\"input_ids\"))\n        return np.stack(input_ids)\n\n    def fit_transform(self, texts):\n        return self.fit(texts).transform(texts)\n\n    @staticmethod\n    def _pre_rules(text):\n        return fix_html(rm_useless_spaces(spec_add_spaces(text)))\n</pre> class BertTokenizer(object):     def __init__(         self,         pretrained_tokenizer=\"distilbert-base-uncased\",         do_lower_case=True,         max_length=90,     ):         super(BertTokenizer, self).__init__()         self.pretrained_tokenizer = pretrained_tokenizer         self.do_lower_case = do_lower_case         self.max_length = max_length      def fit(self, texts):         self.tokenizer = DistilBertTokenizer.from_pretrained(             self.pretrained_tokenizer, do_lower_case=self.do_lower_case         )          return self      def transform(self, texts):         input_ids = []         for text in texts:             encoded_sent = self.tokenizer.encode_plus(                 text=self._pre_rules(text),                 add_special_tokens=True,                 max_length=self.max_length,                 padding=\"max_length\",                 truncation=True,             )              input_ids.append(encoded_sent.get(\"input_ids\"))         return np.stack(input_ids)      def fit_transform(self, texts):         return self.fit(texts).transform(texts)      @staticmethod     def _pre_rules(text):         return fix_html(rm_useless_spaces(spec_add_spaces(text))) <p>Model:</p> In\u00a0[53]: Copied! <pre>class BertModel(nn.Module):\n    def __init__(\n        self,\n        model_name: str = \"distilbert-base-uncased\",\n        freeze_bert: bool = False,\n    ):\n        super(BertModel, self).__init__()\n\n        self.bert = DistilBertModel.from_pretrained(\n            model_name,\n        )\n\n        if freeze_bert:\n            for param in self.bert.parameters():\n                param.requires_grad = False\n\n    def forward(self, X_inp: Tensor) -&gt; Tensor:\n        attn_mask = (X_inp != 0).type(torch.int8)\n        outputs = self.bert(input_ids=X_inp, attention_mask=attn_mask)\n        return outputs[0][:, 0, :]\n\n    @property\n    def output_dim(self) -&gt; int:\n        # This is THE ONLY requirement for any model to work with pytorch-widedeep. Must\n        # have a 'output_dim' property so the WideDeep class knows the incoming dims\n        # from the custom model. in this case, I hardcoded it\n        return 768\n</pre> class BertModel(nn.Module):     def __init__(         self,         model_name: str = \"distilbert-base-uncased\",         freeze_bert: bool = False,     ):         super(BertModel, self).__init__()          self.bert = DistilBertModel.from_pretrained(             model_name,         )          if freeze_bert:             for param in self.bert.parameters():                 param.requires_grad = False      def forward(self, X_inp: Tensor) -&gt; Tensor:         attn_mask = (X_inp != 0).type(torch.int8)         outputs = self.bert(input_ids=X_inp, attention_mask=attn_mask)         return outputs[0][:, 0, :]      @property     def output_dim(self) -&gt; int:         # This is THE ONLY requirement for any model to work with pytorch-widedeep. Must         # have a 'output_dim' property so the WideDeep class knows the incoming dims         # from the custom model. in this case, I hardcoded it         return 768 In\u00a0[54]: Copied! <pre>bert_tokenizer = BertTokenizer()\nX_bert_tr = bert_tokenizer.fit_transform(train[\"review_text\"].tolist())\nX_bert_te = bert_tokenizer.transform(test[\"review_text\"].tolist())\n</pre> bert_tokenizer = BertTokenizer() X_bert_tr = bert_tokenizer.fit_transform(train[\"review_text\"].tolist()) X_bert_te = bert_tokenizer.transform(test[\"review_text\"].tolist()) <p>As I mentioned a number of times in the documentation and examples, <code>pytorch-widedeep</code> is designed for flexibility. For any of the data modes (tabular, text and images) there are available components/models in the library. However, the user can choose to use any model they want with the only requirement that such model must have a <code>output_dim</code> property.</p> <p>With that in mind, the <code>BertModel</code> class defined above can be used by <code>pytorch-widedeep</code> as any other of the internal components. In other words, simply...pass it to the <code>WideDeep</code> class. In this case we are going to add a FC-head as part of the classifier.</p> In\u00a0[55]: Copied! <pre>bert_model = BertModel(freeze_bert=True)\nwd_bert_model = WideDeep(\n    deeptext=bert_model,\n    head_hidden_dims=[256, 128, 64],\n    pred_dim=4,\n)\n</pre> bert_model = BertModel(freeze_bert=True) wd_bert_model = WideDeep(     deeptext=bert_model,     head_hidden_dims=[256, 128, 64],     pred_dim=4, ) <pre>Some weights of the model checkpoint at distilbert-base-uncased were not used when initializing DistilBertModel: ['vocab_projector.bias', 'vocab_layer_norm.bias', 'vocab_transform.weight', 'vocab_transform.bias', 'vocab_projector.weight', 'vocab_layer_norm.weight']\n- This IS expected if you are initializing DistilBertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n- This IS NOT expected if you are initializing DistilBertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n</pre> In\u00a0[56]: Copied! <pre>wd_bert_model\n</pre> wd_bert_model Out[56]: <pre>WideDeep(\n  (deeptext): BertModel(\n    (bert): DistilBertModel(\n      (embeddings): Embeddings(\n        (word_embeddings): Embedding(30522, 768, padding_idx=0)\n        (position_embeddings): Embedding(512, 768)\n        (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n        (dropout): Dropout(p=0.1, inplace=False)\n      )\n      (transformer): Transformer(\n        (layer): ModuleList(\n          (0): TransformerBlock(\n            (attention): MultiHeadSelfAttention(\n              (dropout): Dropout(p=0.1, inplace=False)\n              (q_lin): Linear(in_features=768, out_features=768, bias=True)\n              (k_lin): Linear(in_features=768, out_features=768, bias=True)\n              (v_lin): Linear(in_features=768, out_features=768, bias=True)\n              (out_lin): Linear(in_features=768, out_features=768, bias=True)\n            )\n            (sa_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n            (ffn): FFN(\n              (dropout): Dropout(p=0.1, inplace=False)\n              (lin1): Linear(in_features=768, out_features=3072, bias=True)\n              (lin2): Linear(in_features=3072, out_features=768, bias=True)\n              (activation): GELUActivation()\n            )\n            (output_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n          )\n          (1): TransformerBlock(\n            (attention): MultiHeadSelfAttention(\n              (dropout): Dropout(p=0.1, inplace=False)\n              (q_lin): Linear(in_features=768, out_features=768, bias=True)\n              (k_lin): Linear(in_features=768, out_features=768, bias=True)\n              (v_lin): Linear(in_features=768, out_features=768, bias=True)\n              (out_lin): Linear(in_features=768, out_features=768, bias=True)\n            )\n            (sa_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n            (ffn): FFN(\n              (dropout): Dropout(p=0.1, inplace=False)\n              (lin1): Linear(in_features=768, out_features=3072, bias=True)\n              (lin2): Linear(in_features=3072, out_features=768, bias=True)\n              (activation): GELUActivation()\n            )\n            (output_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n          )\n          (2): TransformerBlock(\n            (attention): MultiHeadSelfAttention(\n              (dropout): Dropout(p=0.1, inplace=False)\n              (q_lin): Linear(in_features=768, out_features=768, bias=True)\n              (k_lin): Linear(in_features=768, out_features=768, bias=True)\n              (v_lin): Linear(in_features=768, out_features=768, bias=True)\n              (out_lin): Linear(in_features=768, out_features=768, bias=True)\n            )\n            (sa_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n            (ffn): FFN(\n              (dropout): Dropout(p=0.1, inplace=False)\n              (lin1): Linear(in_features=768, out_features=3072, bias=True)\n              (lin2): Linear(in_features=3072, out_features=768, bias=True)\n              (activation): GELUActivation()\n            )\n            (output_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n          )\n          (3): TransformerBlock(\n            (attention): MultiHeadSelfAttention(\n              (dropout): Dropout(p=0.1, inplace=False)\n              (q_lin): Linear(in_features=768, out_features=768, bias=True)\n              (k_lin): Linear(in_features=768, out_features=768, bias=True)\n              (v_lin): Linear(in_features=768, out_features=768, bias=True)\n              (out_lin): Linear(in_features=768, out_features=768, bias=True)\n            )\n            (sa_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n            (ffn): FFN(\n              (dropout): Dropout(p=0.1, inplace=False)\n              (lin1): Linear(in_features=768, out_features=3072, bias=True)\n              (lin2): Linear(in_features=3072, out_features=768, bias=True)\n              (activation): GELUActivation()\n            )\n            (output_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n          )\n          (4): TransformerBlock(\n            (attention): MultiHeadSelfAttention(\n              (dropout): Dropout(p=0.1, inplace=False)\n              (q_lin): Linear(in_features=768, out_features=768, bias=True)\n              (k_lin): Linear(in_features=768, out_features=768, bias=True)\n              (v_lin): Linear(in_features=768, out_features=768, bias=True)\n              (out_lin): Linear(in_features=768, out_features=768, bias=True)\n            )\n            (sa_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n            (ffn): FFN(\n              (dropout): Dropout(p=0.1, inplace=False)\n              (lin1): Linear(in_features=768, out_features=3072, bias=True)\n              (lin2): Linear(in_features=3072, out_features=768, bias=True)\n              (activation): GELUActivation()\n            )\n            (output_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n          )\n          (5): TransformerBlock(\n            (attention): MultiHeadSelfAttention(\n              (dropout): Dropout(p=0.1, inplace=False)\n              (q_lin): Linear(in_features=768, out_features=768, bias=True)\n              (k_lin): Linear(in_features=768, out_features=768, bias=True)\n              (v_lin): Linear(in_features=768, out_features=768, bias=True)\n              (out_lin): Linear(in_features=768, out_features=768, bias=True)\n            )\n            (sa_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n            (ffn): FFN(\n              (dropout): Dropout(p=0.1, inplace=False)\n              (lin1): Linear(in_features=768, out_features=3072, bias=True)\n              (lin2): Linear(in_features=3072, out_features=768, bias=True)\n              (activation): GELUActivation()\n            )\n            (output_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n          )\n        )\n      )\n    )\n  )\n  (deephead): Sequential(\n    (0): MLP(\n      (mlp): Sequential(\n        (dense_layer_0): Sequential(\n          (0): Linear(in_features=768, out_features=256, bias=True)\n          (1): ReLU(inplace=True)\n          (2): Dropout(p=0.1, inplace=False)\n        )\n        (dense_layer_1): Sequential(\n          (0): Linear(in_features=256, out_features=128, bias=True)\n          (1): ReLU(inplace=True)\n          (2): Dropout(p=0.1, inplace=False)\n        )\n        (dense_layer_2): Sequential(\n          (0): Linear(in_features=128, out_features=64, bias=True)\n          (1): ReLU(inplace=True)\n          (2): Dropout(p=0.1, inplace=False)\n        )\n      )\n    )\n    (1): Linear(in_features=64, out_features=4, bias=True)\n  )\n)</pre> In\u00a0[57]: Copied! <pre>wd_bert_trainer = Trainer(\n    wd_bert_model,\n    objective=\"multiclass\",\n    metrics=[Accuracy, F1Score(average=True)],\n    num_workers=0,  # As in the case of the tokenizer, in notebook I need to set this to 0 for the Trainer to work\n)\n\nwd_bert_trainer.fit(\n    X_text=X_bert_tr,\n    target=train.rating.values,\n    n_epochs=3,\n    batch_size=64,\n)\n</pre> wd_bert_trainer = Trainer(     wd_bert_model,     objective=\"multiclass\",     metrics=[Accuracy, F1Score(average=True)],     num_workers=0,  # As in the case of the tokenizer, in notebook I need to set this to 0 for the Trainer to work )  wd_bert_trainer.fit(     X_text=X_bert_tr,     target=train.rating.values,     n_epochs=3,     batch_size=64, ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 283/283 [00:14&lt;00:00, 19.68it/s, loss=0.968, metrics={'acc': 0.5879, 'f1': 0.3591}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 283/283 [00:14&lt;00:00, 19.63it/s, loss=0.884, metrics={'acc': 0.6178, 'f1': 0.4399}]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 283/283 [00:14&lt;00:00, 19.55it/s, loss=0.87, metrics={'acc': 0.6234, 'f1': 0.4527}]\n</pre> In\u00a0[58]: Copied! <pre>wd_bert_pred_text = wd_bert_trainer.predict_proba(X_text=X_bert_te)\nwd_bert_pred_text_class = np.argmax(wd_bert_pred_text, 1)\n</pre> wd_bert_pred_text = wd_bert_trainer.predict_proba(X_text=X_bert_te) wd_bert_pred_text_class = np.argmax(wd_bert_pred_text, 1) <pre>predict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 71/71 [00:03&lt;00:00, 21.97it/s]\n</pre> In\u00a0[59]: Copied! <pre>wd_bert_acc = accuracy_score(test.rating, wd_bert_pred_text_class)\nwd_bert_f1 = f1_score(test.rating, wd_bert_pred_text_class, average=\"weighted\")\nwd_bert_cm = confusion_matrix(test.rating, wd_bert_pred_text_class)\n</pre> wd_bert_acc = accuracy_score(test.rating, wd_bert_pred_text_class) wd_bert_f1 = f1_score(test.rating, wd_bert_pred_text_class, average=\"weighted\") wd_bert_cm = confusion_matrix(test.rating, wd_bert_pred_text_class) In\u00a0[60]: Copied! <pre>print(f\"Distilbert Accuracy: {wd_bert_acc}. Distilbert F1 Score: {wd_bert_f1}\")\nprint(f\"Distilbert Confusion Matrix:\\n {wd_bert_cm}\")\n</pre> print(f\"Distilbert Accuracy: {wd_bert_acc}. Distilbert F1 Score: {wd_bert_f1}\") print(f\"Distilbert Confusion Matrix:\\n {wd_bert_cm}\") <pre>Distilbert Accuracy: 0.6326846528084918. Distilbert F1 Score: 0.5796652991272998\nDistilbert Confusion Matrix:\n [[ 287   75   22   90]\n [ 197  136   62  169]\n [  68  119  123  671]\n [  40   64   84 2315]]\n</pre> <p>Now, adding a tabular model follows the exact same process as the one described in section 2.</p> In\u00a0[61]: Copied! <pre>tab_model = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    mlp_hidden_dims=[100, 50],\n)\n</pre> tab_model = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     mlp_hidden_dims=[100, 50], ) In\u00a0[62]: Copied! <pre>wd_tab_bert_model = WideDeep(\n    deeptabular=tab_model,\n    deeptext=bert_model,\n    head_hidden_dims=[256, 128, 64],\n    pred_dim=4,\n)\n</pre> wd_tab_bert_model = WideDeep(     deeptabular=tab_model,     deeptext=bert_model,     head_hidden_dims=[256, 128, 64],     pred_dim=4, ) In\u00a0[63]: Copied! <pre>wd_tab_bert_trainer = Trainer(\n    wd_tab_bert_model,\n    objective=\"multiclass\",\n    metrics=[Accuracy, F1Score(average=True)],\n    num_workers=0,  # As in the case of the tokenizer, in notebook I need to set this to 0 for the Trainer to work\n)\n</pre> wd_tab_bert_trainer = Trainer(     wd_tab_bert_model,     objective=\"multiclass\",     metrics=[Accuracy, F1Score(average=True)],     num_workers=0,  # As in the case of the tokenizer, in notebook I need to set this to 0 for the Trainer to work ) In\u00a0[64]: Copied! <pre>wd_tab_bert_trainer.fit(\n    X_tab=wd_X_tab_tr,\n    X_text=X_bert_tr,\n    target=train.rating.values,\n    n_epochs=3,\n    batch_size=64,\n)\n</pre> wd_tab_bert_trainer.fit(     X_tab=wd_X_tab_tr,     X_text=X_bert_tr,     target=train.rating.values,     n_epochs=3,     batch_size=64, ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 283/283 [00:15&lt;00:00, 18.15it/s, loss=0.974, metrics={'acc': 0.5838, 'f1': 0.3404}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 283/283 [00:15&lt;00:00, 18.38it/s, loss=0.885, metrics={'acc': 0.618, 'f1': 0.4378}]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 283/283 [00:15&lt;00:00, 18.40it/s, loss=0.868, metrics={'acc': 0.6252, 'f1': 0.4575}]\n</pre> In\u00a0[65]: Copied! <pre>wd_tab_bert_pred_text = wd_tab_bert_trainer.predict_proba(\n    X_tab=wd_X_tab_te, X_text=X_bert_te\n)\nwd_tab_bert_pred_text_class = np.argmax(wd_tab_bert_pred_text, 1)\n</pre> wd_tab_bert_pred_text = wd_tab_bert_trainer.predict_proba(     X_tab=wd_X_tab_te, X_text=X_bert_te ) wd_tab_bert_pred_text_class = np.argmax(wd_tab_bert_pred_text, 1) <pre>predict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 71/71 [00:03&lt;00:00, 21.32it/s]\n</pre> In\u00a0[66]: Copied! <pre>wd_tab_bert_acc = accuracy_score(test.rating, wd_tab_bert_pred_text_class)\nwd_tab_bert_f1 = f1_score(test.rating, wd_tab_bert_pred_text_class, average=\"weighted\")\nwd_tab_bert_cm = confusion_matrix(test.rating, wd_tab_bert_pred_text_class)\n</pre> wd_tab_bert_acc = accuracy_score(test.rating, wd_tab_bert_pred_text_class) wd_tab_bert_f1 = f1_score(test.rating, wd_tab_bert_pred_text_class, average=\"weighted\") wd_tab_bert_cm = confusion_matrix(test.rating, wd_tab_bert_pred_text_class) In\u00a0[67]: Copied! <pre>print(\n    f\"Distilbert + Tabular Accuracy: {wd_tab_bert_acc}. Distilbert+ Tabular F1 Score: {wd_tab_bert_f1}\"\n)\nprint(f\"Distilbert + Tabular Confusion Matrix:\\n {wd_tab_bert_cm}\")\n</pre> print(     f\"Distilbert + Tabular Accuracy: {wd_tab_bert_acc}. Distilbert+ Tabular F1 Score: {wd_tab_bert_f1}\" ) print(f\"Distilbert + Tabular Confusion Matrix:\\n {wd_tab_bert_cm}\") <pre>Distilbert + Tabular Accuracy: 0.6242812914639541. Distilbert+ Tabular F1 Score: 0.5508351761564895\nDistilbert + Tabular Confusion Matrix:\n [[ 297   56   11  110]\n [ 229   91   38  206]\n [  86   90   71  734]\n [  49   48   42 2364]]\n</pre>"},{"location":"examples/16_Usign_a_custom_hugging_face_model.html#using-a-hugginface-model","title":"Using a Hugginface model\u00b6","text":"<p>In this notebook we will show how to use an \"external\" Hugginface model along with any other model in the libray. In particular we will show how to combine it with a tabular DL model.</p> <p>Since we are here, we will also compare the performance of a few models on a text classification problem.</p> <p>The notebook will go as follows:</p> <ol> <li>Text classification using tf-idf + LightGBM</li> <li>Text classification using a basic RNN</li> <li>Text classification using Distilbert</li> </ol> <p>In all 3 cases we will add some tabular features to see if these help.</p> <p>In general, I would not pay much attention to the results since I have placed no effort in getting the best possible results (i.e. no hyperparameter optimization or trying different architectures, for example).</p> <p>Let's go</p>"},{"location":"examples/16_Usign_a_custom_hugging_face_model.html#1-text-classification-using-tf-idf-lightgbm","title":"1. Text classification using tf-idf + LightGBM\u00b6","text":""},{"location":"examples/16_Usign_a_custom_hugging_face_model.html#2-text-classification-using-pytorch-widedeeps-built-in-models-a-basic-rnn","title":"2. Text classification using pytorch-widedeep's built-in models (a basic RNN)\u00b6","text":"<p>Moving on now to fully using <code>pytorch-widedeep</code> in this dataset, let's have a look on how one could use a simple RNN to predict the ratings with the library.</p>"},{"location":"examples/16_Usign_a_custom_hugging_face_model.html#3-text-classification-using-a-hugginface-model-as-a-custom-model-in-pytorch-widedeeps","title":"3. Text classification using a Hugginface model as a custom model in pytorch-widedeep's\u00b6","text":""},{"location":"examples/17_feature_importance_via_attention_weights.html","title":"17_feature_importance_via_attention_weights","text":"In\u00a0[1]: Copied! <pre>import torch\n\nimport numpy as np\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import accuracy_score\n\n\nfrom pytorch_widedeep import Trainer\nfrom pytorch_widedeep.models import TabTransformer, ContextAttentionMLP, WideDeep\nfrom pytorch_widedeep.callbacks import EarlyStopping\nfrom pytorch_widedeep.metrics import Accuracy\nfrom pytorch_widedeep.datasets import load_adult\nfrom pytorch_widedeep.preprocessing import TabPreprocessor\n</pre> import torch  import numpy as np from sklearn.model_selection import train_test_split from sklearn.metrics import accuracy_score   from pytorch_widedeep import Trainer from pytorch_widedeep.models import TabTransformer, ContextAttentionMLP, WideDeep from pytorch_widedeep.callbacks import EarlyStopping from pytorch_widedeep.metrics import Accuracy from pytorch_widedeep.datasets import load_adult from pytorch_widedeep.preprocessing import TabPreprocessor In\u00a0[2]: Copied! <pre># use_cuda = torch.cuda.is_available()\ndf = load_adult(as_frame=True)\ndf.columns = [c.replace(\"-\", \"_\") for c in df.columns]\ndf[\"income_label\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int)\ndf.drop([\"income\", \"fnlwgt\", \"educational_num\"], axis=1, inplace=True)\ntarget_colname = \"income_label\"\n</pre> # use_cuda = torch.cuda.is_available() df = load_adult(as_frame=True) df.columns = [c.replace(\"-\", \"_\") for c in df.columns] df[\"income_label\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int) df.drop([\"income\", \"fnlwgt\", \"educational_num\"], axis=1, inplace=True) target_colname = \"income_label\" In\u00a0[3]: Copied! <pre>df.head()\n</pre> df.head() Out[3]: age workclass education marital_status occupation relationship race gender capital_gain capital_loss hours_per_week native_country income_label 0 25 Private 11th Never-married Machine-op-inspct Own-child Black Male 0 0 40 United-States 0 1 38 Private HS-grad Married-civ-spouse Farming-fishing Husband White Male 0 0 50 United-States 0 2 28 Local-gov Assoc-acdm Married-civ-spouse Protective-serv Husband White Male 0 0 40 United-States 1 3 44 Private Some-college Married-civ-spouse Machine-op-inspct Husband Black Male 7688 0 40 United-States 1 4 18 ? Some-college Never-married ? Own-child White Female 0 0 30 United-States 0 In\u00a0[4]: Copied! <pre>cat_embed_cols = []\nfor col in df.columns:\n    if df[col].dtype == \"O\" or df[col].nunique() &lt; 200 and col != target_colname:\n        cat_embed_cols.append(col)\n</pre> cat_embed_cols = [] for col in df.columns:     if df[col].dtype == \"O\" or df[col].nunique() &lt; 200 and col != target_colname:         cat_embed_cols.append(col) In\u00a0[5]: Copied! <pre># all cols will be categorical\nassert len(cat_embed_cols) == df.shape[1] - 1\n</pre> # all cols will be categorical assert len(cat_embed_cols) == df.shape[1] - 1 In\u00a0[6]: Copied! <pre>train, test = train_test_split(\n    df, test_size=0.1, random_state=1, stratify=df[[target_colname]]\n)\n</pre> train, test = train_test_split(     df, test_size=0.1, random_state=1, stratify=df[[target_colname]] ) In\u00a0[7]: Copied! <pre>tab_preprocessor = TabPreprocessor(cat_embed_cols=cat_embed_cols, with_attention=True)\n</pre> tab_preprocessor = TabPreprocessor(cat_embed_cols=cat_embed_cols, with_attention=True) In\u00a0[8]: Copied! <pre>X_tab_train = tab_preprocessor.fit_transform(train)\nX_tab_test = tab_preprocessor.transform(test)\ntarget = train[target_colname].values\n</pre> X_tab_train = tab_preprocessor.fit_transform(train) X_tab_test = tab_preprocessor.transform(test) target = train[target_colname].values In\u00a0[9]: Copied! <pre>tab_transformer = TabTransformer(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    embed_continuous_method=\"standard\",\n    input_dim=8,\n    n_heads=2,\n    n_blocks=1,\n    attn_dropout=0.1,\n    transformer_activation=\"relu\",\n)\n</pre> tab_transformer = TabTransformer(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     embed_continuous_method=\"standard\",     input_dim=8,     n_heads=2,     n_blocks=1,     attn_dropout=0.1,     transformer_activation=\"relu\", ) In\u00a0[10]: Copied! <pre>model = WideDeep(deeptabular=tab_transformer)\n</pre> model = WideDeep(deeptabular=tab_transformer) In\u00a0[11]: Copied! <pre>optimizer = torch.optim.Adam(model.parameters(), lr=0.01, weight_decay=0.0)\n</pre> optimizer = torch.optim.Adam(model.parameters(), lr=0.01, weight_decay=0.0) In\u00a0[12]: Copied! <pre>lr_scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(\n    optimizer,\n    threshold=0.001,\n    threshold_mode=\"abs\",\n    patience=10,\n)\n</pre> lr_scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(     optimizer,     threshold=0.001,     threshold_mode=\"abs\",     patience=10, ) In\u00a0[13]: Copied! <pre>early_stopping = EarlyStopping(\n    min_delta=0.001, patience=30, restore_best_weights=True, verbose=True\n)\n</pre> early_stopping = EarlyStopping(     min_delta=0.001, patience=30, restore_best_weights=True, verbose=True ) In\u00a0[14]: Copied! <pre>trainer = Trainer(\n    model,\n    objective=\"binary\",\n    optimizers=optimizer,\n    lr_schedulers=lr_scheduler,\n    reducelronplateau_criterion=\"loss\",\n    callbacks=[early_stopping],\n    metrics=[Accuracy],\n)\n</pre> trainer = Trainer(     model,     objective=\"binary\",     optimizers=optimizer,     lr_schedulers=lr_scheduler,     reducelronplateau_criterion=\"loss\",     callbacks=[early_stopping],     metrics=[Accuracy], ) <p>The feature importances will be computed after training, using a sample of the training dataset of size <code>feature_importance_sample_size</code></p> In\u00a0[15]: Copied! <pre>trainer.fit(\n    X_tab=X_tab_train,\n    target=target,\n    val_split=0.2,\n    n_epochs=100,\n    batch_size=128,\n    validation_freq=1,\n    feature_importance_sample_size=1000,\n)\n</pre> trainer.fit(     X_tab=X_tab_train,     target=target,     val_split=0.2,     n_epochs=100,     batch_size=128,     validation_freq=1,     feature_importance_sample_size=1000, ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 81.80it/s, loss=0.334, metrics={'acc': 0.847}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 111.34it/s, loss=0.294, metrics={'acc': 0.8669}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 83.02it/s, loss=0.293, metrics={'acc': 0.8656}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 124.03it/s, loss=0.283, metrics={'acc': 0.8678}]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 87.69it/s, loss=0.282, metrics={'acc': 0.8703}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 118.22it/s, loss=0.279, metrics={'acc': 0.8717}]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 79.24it/s, loss=0.277, metrics={'acc': 0.8718}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 115.29it/s, loss=0.277, metrics={'acc': 0.8731}]\nepoch 5: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 83.76it/s, loss=0.275, metrics={'acc': 0.8727}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 120.80it/s, loss=0.276, metrics={'acc': 0.8727}]\nepoch 6: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 82.78it/s, loss=0.273, metrics={'acc': 0.873}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 100.43it/s, loss=0.276, metrics={'acc': 0.871}]\nepoch 7: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 76.35it/s, loss=0.271, metrics={'acc': 0.8742}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 116.14it/s, loss=0.275, metrics={'acc': 0.8726}]\nepoch 8: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 74.29it/s, loss=0.271, metrics={'acc': 0.875}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 117.46it/s, loss=0.276, metrics={'acc': 0.8718}]\nepoch 9: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 77.10it/s, loss=0.27, metrics={'acc': 0.8761}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 105.49it/s, loss=0.275, metrics={'acc': 0.8728}]\nepoch 10: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 70.40it/s, loss=0.269, metrics={'acc': 0.8747}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 105.47it/s, loss=0.275, metrics={'acc': 0.8726}]\nepoch 11: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 72.83it/s, loss=0.268, metrics={'acc': 0.8742}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 105.03it/s, loss=0.274, metrics={'acc': 0.873}]\nepoch 12: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 71.86it/s, loss=0.267, metrics={'acc': 0.8743}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 106.61it/s, loss=0.274, metrics={'acc': 0.8734}]\nepoch 13: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 72.39it/s, loss=0.267, metrics={'acc': 0.876}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 108.05it/s, loss=0.275, metrics={'acc': 0.8717}]\nepoch 14: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 73.36it/s, loss=0.265, metrics={'acc': 0.8767}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 109.60it/s, loss=0.276, metrics={'acc': 0.8747}]\nepoch 15: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 73.34it/s, loss=0.264, metrics={'acc': 0.876}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 117.55it/s, loss=0.276, metrics={'acc': 0.8706}]\nepoch 16: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 75.35it/s, loss=0.264, metrics={'acc': 0.8777}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 122.08it/s, loss=0.275, metrics={'acc': 0.8753}]\nepoch 17: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 75.96it/s, loss=0.263, metrics={'acc': 0.877}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 120.83it/s, loss=0.277, metrics={'acc': 0.8739}]\nepoch 18: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 75.38it/s, loss=0.263, metrics={'acc': 0.8779}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 118.18it/s, loss=0.278, metrics={'acc': 0.8714}]\nepoch 19: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 75.61it/s, loss=0.261, metrics={'acc': 0.8784}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 119.19it/s, loss=0.278, metrics={'acc': 0.8712}]\nepoch 20: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 72.43it/s, loss=0.261, metrics={'acc': 0.8791}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 118.87it/s, loss=0.28, metrics={'acc': 0.873}]\nepoch 21: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 73.97it/s, loss=0.26, metrics={'acc': 0.8787}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 107.50it/s, loss=0.279, metrics={'acc': 0.8732}]\nepoch 22: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 71.76it/s, loss=0.253, metrics={'acc': 0.8816}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 108.11it/s, loss=0.279, metrics={'acc': 0.8707}]\nepoch 23: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 71.92it/s, loss=0.252, metrics={'acc': 0.8828}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 100.14it/s, loss=0.28, metrics={'acc': 0.8711}]\nepoch 24: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 75.04it/s, loss=0.252, metrics={'acc': 0.8829}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 98.36it/s, loss=0.28, metrics={'acc': 0.8708}]\nepoch 25: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 76.31it/s, loss=0.251, metrics={'acc': 0.883}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 122.97it/s, loss=0.281, metrics={'acc': 0.8709}]\nepoch 26: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 76.63it/s, loss=0.25, metrics={'acc': 0.8834}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 123.07it/s, loss=0.281, metrics={'acc': 0.8698}]\nepoch 27: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 78.37it/s, loss=0.251, metrics={'acc': 0.884}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 114.75it/s, loss=0.281, metrics={'acc': 0.87}]\nepoch 28: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 77.19it/s, loss=0.25, metrics={'acc': 0.883}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 117.99it/s, loss=0.282, metrics={'acc': 0.8699}]\nepoch 29: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 76.19it/s, loss=0.25, metrics={'acc': 0.8829}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 111.11it/s, loss=0.282, metrics={'acc': 0.8695}]\nepoch 30: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 77.57it/s, loss=0.249, metrics={'acc': 0.8839}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 115.64it/s, loss=0.283, metrics={'acc': 0.8689}]\nepoch 31: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 79.55it/s, loss=0.249, metrics={'acc': 0.8846}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 125.10it/s, loss=0.283, metrics={'acc': 0.869}]\nepoch 32: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 82.56it/s, loss=0.248, metrics={'acc': 0.8841}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 118.45it/s, loss=0.284, metrics={'acc': 0.8687}]\nepoch 33: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 81.06it/s, loss=0.248, metrics={'acc': 0.8848}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 129.26it/s, loss=0.284, metrics={'acc': 0.8689}]\nepoch 34: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 80.53it/s, loss=0.248, metrics={'acc': 0.8854}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 120.61it/s, loss=0.283, metrics={'acc': 0.869}]\nepoch 35: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 80.78it/s, loss=0.248, metrics={'acc': 0.8853}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 127.31it/s, loss=0.283, metrics={'acc': 0.8694}]\nepoch 36: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 82.51it/s, loss=0.248, metrics={'acc': 0.8863}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 125.94it/s, loss=0.283, metrics={'acc': 0.8693}]\nepoch 37: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 81.35it/s, loss=0.247, metrics={'acc': 0.8844}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 126.77it/s, loss=0.283, metrics={'acc': 0.8692}]\nepoch 38: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 80.62it/s, loss=0.248, metrics={'acc': 0.8837}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 116.62it/s, loss=0.283, metrics={'acc': 0.8692}]\nepoch 39: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 79.82it/s, loss=0.248, metrics={'acc': 0.8842}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 123.64it/s, loss=0.283, metrics={'acc': 0.8695}]\nepoch 40: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 76.86it/s, loss=0.247, metrics={'acc': 0.8855}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 120.34it/s, loss=0.283, metrics={'acc': 0.8692}]\n</pre> <pre>Best Epoch: 10. Best val_loss: 0.27451\nRestoring model weights from the end of the best epoch\n</pre> In\u00a0[16]: Copied! <pre>trainer.feature_importance\n</pre> trainer.feature_importance Out[16]: <pre>{'age': 0.09718182,\n 'workclass': 0.090637445,\n 'education': 0.08910798,\n 'marital_status': 0.08971319,\n 'occupation': 0.12546304,\n 'relationship': 0.086381145,\n 'race': 0.050686445,\n 'gender': 0.05116429,\n 'capital_gain': 0.08165918,\n 'capital_loss': 0.07702667,\n 'hours_per_week': 0.08205996,\n 'native_country': 0.07891885}</pre> In\u00a0[17]: Copied! <pre>preds = trainer.predict(X_tab=X_tab_test)\n</pre> preds = trainer.predict(X_tab=X_tab_test) <pre>predict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 39/39 [00:00&lt;00:00, 199.63it/s]\n</pre> In\u00a0[18]: Copied! <pre>accuracy_score(preds, test.income_label)\n</pre> accuracy_score(preds, test.income_label) Out[18]: <pre>0.8685772773797339</pre> In\u00a0[19]: Copied! <pre>test.reset_index(drop=True, inplace=True)\n</pre> test.reset_index(drop=True, inplace=True) In\u00a0[20]: Copied! <pre>test[test.income_label == 0].head(1)\n</pre> test[test.income_label == 0].head(1) Out[20]: age workclass education marital_status occupation relationship race gender capital_gain capital_loss hours_per_week native_country income_label 0 26 Private Some-college Never-married Exec-managerial Not-in-family White Male 0 0 60 United-States 0 In\u00a0[21]: Copied! <pre>test[test.income_label == 1].head(1)\n</pre> test[test.income_label == 1].head(1) Out[21]: age workclass education marital_status occupation relationship race gender capital_gain capital_loss hours_per_week native_country income_label 3 36 Local-gov Doctorate Married-civ-spouse Prof-specialty Husband White Male 0 1887 50 United-States 1 <p>To get the feature importance of a test dataset, simply use the <code>explain</code> method</p> In\u00a0[22]: Copied! <pre>feat_imp_per_sample = trainer.explain(X_tab_test, save_step_masks=False)\n</pre> feat_imp_per_sample = trainer.explain(X_tab_test, save_step_masks=False) In\u00a0[23]: Copied! <pre>list(test.iloc[0].index[np.argsort(-feat_imp_per_sample[0])])\n</pre> list(test.iloc[0].index[np.argsort(-feat_imp_per_sample[0])]) Out[23]: <pre>['hours_per_week',\n 'education',\n 'relationship',\n 'occupation',\n 'workclass',\n 'capital_gain',\n 'native_country',\n 'marital_status',\n 'capital_loss',\n 'age',\n 'race',\n 'gender']</pre> In\u00a0[24]: Copied! <pre>list(test.iloc[3].index[np.argsort(-feat_imp_per_sample[3])])\n</pre> list(test.iloc[3].index[np.argsort(-feat_imp_per_sample[3])]) Out[24]: <pre>['age',\n 'capital_loss',\n 'hours_per_week',\n 'marital_status',\n 'native_country',\n 'relationship',\n 'race',\n 'education',\n 'occupation',\n 'capital_gain',\n 'gender',\n 'workclass']</pre> <p>We could do the same with the <code>ContextAttentionMLP</code></p> In\u00a0[25]: Copied! <pre>context_attn_mlp = ContextAttentionMLP(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    cat_embed_dropout=0.0,\n    input_dim=16,\n    attn_dropout=0.1,\n    attn_activation=\"relu\",\n)\n</pre> context_attn_mlp = ContextAttentionMLP(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     cat_embed_dropout=0.0,     input_dim=16,     attn_dropout=0.1,     attn_activation=\"relu\", ) In\u00a0[26]: Copied! <pre>mlp_model = WideDeep(deeptabular=context_attn_mlp)\n</pre> mlp_model = WideDeep(deeptabular=context_attn_mlp) In\u00a0[27]: Copied! <pre>mlp_optimizer = torch.optim.Adam(mlp_model.parameters(), lr=0.01, weight_decay=0.0)\n</pre> mlp_optimizer = torch.optim.Adam(mlp_model.parameters(), lr=0.01, weight_decay=0.0) In\u00a0[28]: Copied! <pre>mlp_lr_scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(\n    mlp_optimizer,\n    threshold=0.001,\n    threshold_mode=\"abs\",\n    patience=10,\n)\n</pre> mlp_lr_scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(     mlp_optimizer,     threshold=0.001,     threshold_mode=\"abs\",     patience=10, ) In\u00a0[29]: Copied! <pre>mlp_early_stopping = EarlyStopping(\n    min_delta=0.001, patience=30, restore_best_weights=True, verbose=True\n)\n</pre> mlp_early_stopping = EarlyStopping(     min_delta=0.001, patience=30, restore_best_weights=True, verbose=True ) In\u00a0[30]: Copied! <pre>mlp_trainer = Trainer(\n    mlp_model,\n    objective=\"binary\",\n    optimizers=mlp_optimizer,\n    lr_schedulers=mlp_lr_scheduler,\n    reducelronplateau_criterion=\"loss\",\n    callbacks=[mlp_early_stopping],\n    metrics=[Accuracy],\n)\n</pre> mlp_trainer = Trainer(     mlp_model,     objective=\"binary\",     optimizers=mlp_optimizer,     lr_schedulers=mlp_lr_scheduler,     reducelronplateau_criterion=\"loss\",     callbacks=[mlp_early_stopping],     metrics=[Accuracy], ) In\u00a0[31]: Copied! <pre>mlp_trainer.fit(\n    X_tab=X_tab_train,\n    target=target,\n    val_split=0.2,\n    n_epochs=100,\n    batch_size=128,\n    validation_freq=1,\n    feature_importance_sample_size=1000,\n)\n</pre> mlp_trainer.fit(     X_tab=X_tab_train,     target=target,     val_split=0.2,     n_epochs=100,     batch_size=128,     validation_freq=1,     feature_importance_sample_size=1000, ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 73.11it/s, loss=0.405, metrics={'acc': 0.8094}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 119.26it/s, loss=0.309, metrics={'acc': 0.8583}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 71.70it/s, loss=0.332, metrics={'acc': 0.8447}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 116.36it/s, loss=0.293, metrics={'acc': 0.8646}]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 78.42it/s, loss=0.319, metrics={'acc': 0.8505}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 118.05it/s, loss=0.293, metrics={'acc': 0.8654}]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 79.00it/s, loss=0.312, metrics={'acc': 0.8554}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 116.49it/s, loss=0.291, metrics={'acc': 0.8661}]\nepoch 5: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 79.50it/s, loss=0.308, metrics={'acc': 0.8583}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 120.48it/s, loss=0.287, metrics={'acc': 0.8669}]\nepoch 6: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 80.84it/s, loss=0.303, metrics={'acc': 0.8605}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 128.70it/s, loss=0.288, metrics={'acc': 0.8673}]\nepoch 7: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 79.93it/s, loss=0.301, metrics={'acc': 0.8597}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 121.47it/s, loss=0.298, metrics={'acc': 0.8628}]\nepoch 8: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 80.56it/s, loss=0.3, metrics={'acc': 0.8592}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 119.84it/s, loss=0.281, metrics={'acc': 0.8718}]\nepoch 9: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 79.17it/s, loss=0.298, metrics={'acc': 0.8619}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 126.32it/s, loss=0.28, metrics={'acc': 0.8716}]\nepoch 10: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 82.13it/s, loss=0.297, metrics={'acc': 0.8615}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 120.50it/s, loss=0.281, metrics={'acc': 0.8718}]\nepoch 11: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 82.54it/s, loss=0.293, metrics={'acc': 0.8641}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 122.57it/s, loss=0.284, metrics={'acc': 0.867}]\nepoch 12: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 80.92it/s, loss=0.293, metrics={'acc': 0.863}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 126.42it/s, loss=0.282, metrics={'acc': 0.8701}]\nepoch 13: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 79.61it/s, loss=0.293, metrics={'acc': 0.8635}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 117.56it/s, loss=0.276, metrics={'acc': 0.8719}]\nepoch 14: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 74.92it/s, loss=0.29, metrics={'acc': 0.8633}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 112.06it/s, loss=0.286, metrics={'acc': 0.8669}]\nepoch 15: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 76.83it/s, loss=0.291, metrics={'acc': 0.865}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 112.88it/s, loss=0.282, metrics={'acc': 0.8677}]\nepoch 16: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 76.74it/s, loss=0.29, metrics={'acc': 0.8653}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 120.85it/s, loss=0.285, metrics={'acc': 0.8672}]\nepoch 17: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 75.99it/s, loss=0.29, metrics={'acc': 0.865}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 113.53it/s, loss=0.282, metrics={'acc': 0.8681}]\nepoch 18: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 71.22it/s, loss=0.288, metrics={'acc': 0.8651}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 112.89it/s, loss=0.288, metrics={'acc': 0.8676}]\nepoch 19: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 75.85it/s, loss=0.29, metrics={'acc': 0.8661}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 114.26it/s, loss=0.284, metrics={'acc': 0.8662}]\nepoch 20: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 75.80it/s, loss=0.289, metrics={'acc': 0.8661}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 119.44it/s, loss=0.281, metrics={'acc': 0.8703}]\nepoch 21: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 75.72it/s, loss=0.29, metrics={'acc': 0.8661}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 113.04it/s, loss=0.285, metrics={'acc': 0.8648}]\nepoch 22: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 74.86it/s, loss=0.289, metrics={'acc': 0.8656}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 111.75it/s, loss=0.282, metrics={'acc': 0.8666}]\nepoch 23: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 76.68it/s, loss=0.289, metrics={'acc': 0.8668}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 115.86it/s, loss=0.282, metrics={'acc': 0.8724}]\nepoch 24: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 75.93it/s, loss=0.288, metrics={'acc': 0.8653}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 114.69it/s, loss=0.285, metrics={'acc': 0.8656}]\nepoch 25: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 79.99it/s, loss=0.284, metrics={'acc': 0.8671}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 118.27it/s, loss=0.277, metrics={'acc': 0.8707}]\nepoch 26: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 79.86it/s, loss=0.282, metrics={'acc': 0.8686}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 123.94it/s, loss=0.276, metrics={'acc': 0.8712}]\nepoch 27: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 77.86it/s, loss=0.283, metrics={'acc': 0.8691}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 114.11it/s, loss=0.277, metrics={'acc': 0.8716}]\nepoch 28: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 77.43it/s, loss=0.281, metrics={'acc': 0.8696}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 114.64it/s, loss=0.277, metrics={'acc': 0.8712}]\nepoch 29: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 77.39it/s, loss=0.281, metrics={'acc': 0.8696}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 115.83it/s, loss=0.277, metrics={'acc': 0.872}]\nepoch 30: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 74.93it/s, loss=0.28, metrics={'acc': 0.8706}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 113.07it/s, loss=0.275, metrics={'acc': 0.8714}]\nepoch 31: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 77.35it/s, loss=0.281, metrics={'acc': 0.8697}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 112.68it/s, loss=0.276, metrics={'acc': 0.872}]\nepoch 32: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 76.72it/s, loss=0.28, metrics={'acc': 0.8693}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 122.50it/s, loss=0.276, metrics={'acc': 0.8709}]\nepoch 33: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 76.68it/s, loss=0.28, metrics={'acc': 0.8716}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 110.07it/s, loss=0.277, metrics={'acc': 0.8709}]\nepoch 34: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 73.58it/s, loss=0.279, metrics={'acc': 0.8704}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 111.07it/s, loss=0.274, metrics={'acc': 0.8719}]\nepoch 35: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 74.03it/s, loss=0.28, metrics={'acc': 0.8687}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 110.15it/s, loss=0.276, metrics={'acc': 0.871}]\nepoch 36: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 73.11it/s, loss=0.279, metrics={'acc': 0.8706}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 111.10it/s, loss=0.278, metrics={'acc': 0.8705}]\nepoch 37: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 74.30it/s, loss=0.279, metrics={'acc': 0.869}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 110.75it/s, loss=0.279, metrics={'acc': 0.8702}]\nepoch 38: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 72.34it/s, loss=0.28, metrics={'acc': 0.8691}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 117.79it/s, loss=0.277, metrics={'acc': 0.8698}]\nepoch 39: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 75.49it/s, loss=0.279, metrics={'acc': 0.8694}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 115.25it/s, loss=0.279, metrics={'acc': 0.87}]\nepoch 40: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 75.51it/s, loss=0.28, metrics={'acc': 0.8694}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 110.90it/s, loss=0.277, metrics={'acc': 0.8694}]\nepoch 41: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 75.35it/s, loss=0.278, metrics={'acc': 0.8716}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 115.74it/s, loss=0.28, metrics={'acc': 0.8675}]\nepoch 42: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 76.35it/s, loss=0.279, metrics={'acc': 0.8695}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 122.76it/s, loss=0.277, metrics={'acc': 0.8699}]\nepoch 43: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:04&lt;00:00, 66.14it/s, loss=0.279, metrics={'acc': 0.8681}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 106.20it/s, loss=0.277, metrics={'acc': 0.8714}]\nepoch 44: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 73.37it/s, loss=0.279, metrics={'acc': 0.8704}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 115.18it/s, loss=0.277, metrics={'acc': 0.8716}]\nepoch 45: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 76.23it/s, loss=0.278, metrics={'acc': 0.8702}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 124.83it/s, loss=0.278, metrics={'acc': 0.8707}]\nepoch 46: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 77.12it/s, loss=0.278, metrics={'acc': 0.8704}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 126.62it/s, loss=0.279, metrics={'acc': 0.8693}]\nepoch 47: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 78.55it/s, loss=0.276, metrics={'acc': 0.8713}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 117.99it/s, loss=0.279, metrics={'acc': 0.8691}]\nepoch 48: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 77.25it/s, loss=0.278, metrics={'acc': 0.8719}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 124.52it/s, loss=0.278, metrics={'acc': 0.8695}]\nepoch 49: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 78.35it/s, loss=0.277, metrics={'acc': 0.8721}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 119.82it/s, loss=0.279, metrics={'acc': 0.8691}]\nepoch 50: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 77.15it/s, loss=0.277, metrics={'acc': 0.8717}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 122.62it/s, loss=0.278, metrics={'acc': 0.8699}]\nepoch 51: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 78.55it/s, loss=0.277, metrics={'acc': 0.8713}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 117.63it/s, loss=0.278, metrics={'acc': 0.87}]\nepoch 52: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 77.04it/s, loss=0.276, metrics={'acc': 0.8721}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 126.39it/s, loss=0.278, metrics={'acc': 0.8697}]\nepoch 53: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 78.15it/s, loss=0.277, metrics={'acc': 0.8721}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 127.56it/s, loss=0.278, metrics={'acc': 0.8699}]\nepoch 54: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 77.41it/s, loss=0.277, metrics={'acc': 0.8711}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 118.95it/s, loss=0.278, metrics={'acc': 0.8698}]\nepoch 55: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 76.35it/s, loss=0.277, metrics={'acc': 0.8718}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 126.90it/s, loss=0.278, metrics={'acc': 0.8699}]\nepoch 56: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 77.83it/s, loss=0.277, metrics={'acc': 0.8707}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 115.13it/s, loss=0.279, metrics={'acc': 0.8691}]\nepoch 57: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 75.20it/s, loss=0.277, metrics={'acc': 0.8722}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 123.16it/s, loss=0.279, metrics={'acc': 0.8691}]\nepoch 58: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 73.33it/s, loss=0.276, metrics={'acc': 0.871}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 123.37it/s, loss=0.278, metrics={'acc': 0.8691}]\nepoch 59: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 78.41it/s, loss=0.277, metrics={'acc': 0.8714}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 125.17it/s, loss=0.278, metrics={'acc': 0.8695}]\nepoch 60: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 79.26it/s, loss=0.276, metrics={'acc': 0.8721}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 120.60it/s, loss=0.278, metrics={'acc': 0.869}]\nepoch 61: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 75.88it/s, loss=0.278, metrics={'acc': 0.8703}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 124.47it/s, loss=0.278, metrics={'acc': 0.8692}]\nepoch 62: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 77.13it/s, loss=0.276, metrics={'acc': 0.8711}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 125.80it/s, loss=0.278, metrics={'acc': 0.8691}]\nepoch 63: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 77.20it/s, loss=0.277, metrics={'acc': 0.8715}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 118.50it/s, loss=0.278, metrics={'acc': 0.8695}]\nepoch 64: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 78.11it/s, loss=0.276, metrics={'acc': 0.8719}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 114.52it/s, loss=0.278, metrics={'acc': 0.869}]\n</pre> <pre>Best Epoch: 34. Best val_loss: 0.27449\nRestoring model weights from the end of the best epoch\n</pre> In\u00a0[32]: Copied! <pre>mlp_trainer.feature_importance\n</pre> mlp_trainer.feature_importance Out[32]: <pre>{'age': 0.116632804,\n 'workclass': 0.050255153,\n 'education': 0.094621316,\n 'marital_status': 0.12328919,\n 'occupation': 0.107893184,\n 'relationship': 0.11747801,\n 'race': 0.054717205,\n 'gender': 0.07514235,\n 'capital_gain': 0.059732802,\n 'capital_loss': 0.06738944,\n 'hours_per_week': 0.0610674,\n 'native_country': 0.07178114}</pre> In\u00a0[33]: Copied! <pre>mlp_preds = mlp_trainer.predict(X_tab=X_tab_test)\n</pre> mlp_preds = mlp_trainer.predict(X_tab=X_tab_test) <pre>predict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 39/39 [00:00&lt;00:00, 212.38it/s]\n</pre> In\u00a0[34]: Copied! <pre>accuracy_score(mlp_preds, test.income_label)\n</pre> accuracy_score(mlp_preds, test.income_label) Out[34]: <pre>0.8726714431934494</pre>"},{"location":"examples/17_feature_importance_via_attention_weights.html#feature-importance-via-the-attention-weights","title":"Feature Importance via the attention weights\u00b6","text":"<p>I will start by saying that I consider this feature of the library purely experimental. First of all I think there are multiple ways one could address finding the features importances for these models. However, and more importantly, one has to bear in mind that even tree-based algorithms on the same dataset produce different feature importances. This is more \"dramatic\" if one uses different techniques, such as shap or feature permutation (see for example this and references therein). All this to say that, sometimes, feature importance is just a measure contained within the experiment run, and for the model used.</p> <p>With that in mind, each instantiation of a deep tabular model, that has millions of trainable parameters, will potentially produce a different set of feature importances, even if the model has the same architecture. Moreover, this effect will become more apparent if the dataset is relatively easy and there are dependent/related columns so that one could get to the same success metric with different parameters.</p> <p>In summary, feature importances are implemented in this librray for all attention-based models for tabular data, with the exception of the <code>TabPerceiver</code>. However this functionality has to be used and interpreted with care and consider of value within the 'universe' (or context) of the model with which these features were produced.</p> <p>Nonetheless, let's have a look to how one would access to the feature importances when using this library.</p>"},{"location":"examples/18_wide_and_deep_for_recsys_pt1.html","title":"18_wide_and_deep_for_recsys_pt1","text":"<p>The goal of this, and the companion (part 2) notebooks is to illustrate how one could use this library in the context of recommendation systems. In particular, this notebook and the scripts at the <code>wide_deep_for_recsys</code> dir are a response to this issue. Therefore, we will use the Kaggle notebook referred in that issue here.</p> <p>In order to keep the length of the notebook tractable, we will split this exercise in 2. In this first notebook we will prepare the data in almost the exact same way as it is done in the Kaggle notebook and also show how one could use <code>pytorch-widedeep</code> to build a model almost identical to the one in that notebook.</p> <p>In a second notebook, we will show how one could use this library to implement other models, still following the same problem formulation.</p> In\u00a0[1]: Copied! <pre>from pathlib import Path\nimport warnings\n\nimport pandas as pd\nfrom sklearn.model_selection import train_test_split\n\nfrom pytorch_widedeep.datasets import load_movielens100k\n</pre> from pathlib import Path import warnings  import pandas as pd from sklearn.model_selection import train_test_split  from pytorch_widedeep.datasets import load_movielens100k In\u00a0[2]: Copied! <pre>warnings.filterwarnings(\"ignore\")\n</pre> warnings.filterwarnings(\"ignore\") In\u00a0[3]: Copied! <pre>save_path = Path(\"prepared_data\")\nif not save_path.exists():\n    save_path.mkdir(parents=True, exist_ok=True)\n</pre> save_path = Path(\"prepared_data\") if not save_path.exists():     save_path.mkdir(parents=True, exist_ok=True) In\u00a0[4]: Copied! <pre>data, users, items = load_movielens100k(as_frame=True)\n</pre> data, users, items = load_movielens100k(as_frame=True) In\u00a0[5]: Copied! <pre># Alternatively, as specified in the docs: 'The last 19 fields are the genres' so:\n# list_of_genres = items.columns.tolist()[-19:]\nlist_of_genres = [\n    \"unknown\",\n    \"Action\",\n    \"Adventure\",\n    \"Animation\",\n    \"Children's\",\n    \"Comedy\",\n    \"Crime\",\n    \"Documentary\",\n    \"Drama\",\n    \"Fantasy\",\n    \"Film-Noir\",\n    \"Horror\",\n    \"Musical\",\n    \"Mystery\",\n    \"Romance\",\n    \"Sci-Fi\",\n    \"Thriller\",\n    \"War\",\n    \"Western\",\n]\n</pre> # Alternatively, as specified in the docs: 'The last 19 fields are the genres' so: # list_of_genres = items.columns.tolist()[-19:] list_of_genres = [     \"unknown\",     \"Action\",     \"Adventure\",     \"Animation\",     \"Children's\",     \"Comedy\",     \"Crime\",     \"Documentary\",     \"Drama\",     \"Fantasy\",     \"Film-Noir\",     \"Horror\",     \"Musical\",     \"Mystery\",     \"Romance\",     \"Sci-Fi\",     \"Thriller\",     \"War\",     \"Western\", ] <p>Let's first start by loading the interactions, user and item data</p> In\u00a0[6]: Copied! <pre>data.head()\n</pre> data.head() Out[6]: user_id movie_id rating timestamp 0 196 242 3 881250949 1 186 302 3 891717742 2 22 377 1 878887116 3 244 51 2 880606923 4 166 346 1 886397596 In\u00a0[7]: Copied! <pre>users.head()\n</pre> users.head() Out[7]: user_id age gender occupation zip_code 0 1 24 M technician 85711 1 2 53 F other 94043 2 3 23 M writer 32067 3 4 24 M technician 43537 4 5 33 F other 15213 In\u00a0[8]: Copied! <pre>items.head()\n</pre> items.head() Out[8]: movie_id movie_title release_date video_release_date IMDb_URL unknown Action Adventure Animation Children's ... Fantasy Film-Noir Horror Musical Mystery Romance Sci-Fi Thriller War Western 0 1 Toy Story (1995) 01-Jan-1995 NaN http://us.imdb.com/M/title-exact?Toy%20Story%2... 0 0 0 1 1 ... 0 0 0 0 0 0 0 0 0 0 1 2 GoldenEye (1995) 01-Jan-1995 NaN http://us.imdb.com/M/title-exact?GoldenEye%20(... 0 1 1 0 0 ... 0 0 0 0 0 0 0 1 0 0 2 3 Four Rooms (1995) 01-Jan-1995 NaN http://us.imdb.com/M/title-exact?Four%20Rooms%... 0 0 0 0 0 ... 0 0 0 0 0 0 0 1 0 0 3 4 Get Shorty (1995) 01-Jan-1995 NaN http://us.imdb.com/M/title-exact?Get%20Shorty%... 0 1 0 0 0 ... 0 0 0 0 0 0 0 0 0 0 4 5 Copycat (1995) 01-Jan-1995 NaN http://us.imdb.com/M/title-exact?Copycat%20(1995) 0 0 0 0 0 ... 0 0 0 0 0 0 0 1 0 0 <p>5 rows \u00d7 24 columns</p> In\u00a0[9]: Copied! <pre># adding a column with the number of movies watched per user\ndataset = data.sort_values([\"user_id\", \"timestamp\"]).reset_index(drop=True)\ndataset[\"one\"] = 1\ndataset[\"num_watched\"] = dataset.groupby(\"user_id\")[\"one\"].cumsum()\ndataset.drop(\"one\", axis=1, inplace=True)\ndataset.head()\n</pre> # adding a column with the number of movies watched per user dataset = data.sort_values([\"user_id\", \"timestamp\"]).reset_index(drop=True) dataset[\"one\"] = 1 dataset[\"num_watched\"] = dataset.groupby(\"user_id\")[\"one\"].cumsum() dataset.drop(\"one\", axis=1, inplace=True) dataset.head() Out[9]: user_id movie_id rating timestamp num_watched 0 1 168 5 874965478 1 1 1 172 5 874965478 2 2 1 165 5 874965518 3 3 1 156 4 874965556 4 4 1 196 5 874965677 5 In\u00a0[10]: Copied! <pre># adding a column with the mean rating at a point in time per user\ndataset[\"mean_rate\"] = (\n    dataset.groupby(\"user_id\")[\"rating\"].cumsum() / dataset[\"num_watched\"]\n)\ndataset.head()\n</pre> # adding a column with the mean rating at a point in time per user dataset[\"mean_rate\"] = (     dataset.groupby(\"user_id\")[\"rating\"].cumsum() / dataset[\"num_watched\"] ) dataset.head() Out[10]: user_id movie_id rating timestamp num_watched mean_rate 0 1 168 5 874965478 1 5.00 1 1 172 5 874965478 2 5.00 2 1 165 5 874965518 3 5.00 3 1 156 4 874965556 4 4.75 4 1 196 5 874965677 5 4.80 In\u00a0[11]: Copied! <pre>dataset[\"target\"] = dataset.groupby(\"user_id\")[\"movie_id\"].shift(-1)\n</pre> dataset[\"target\"] = dataset.groupby(\"user_id\")[\"movie_id\"].shift(-1) <p>Following the same processing used by the author in the before-mentioned Kaggle notebook, we build sequences of previous movies watched</p> In\u00a0[12]: Copied! <pre># Here the author builds the sequences\ndataset[\"prev_movies\"] = dataset[\"movie_id\"].apply(lambda x: str(x))\ndataset[\"prev_movies\"] = (\n    dataset.groupby(\"user_id\")[\"prev_movies\"]\n    .apply(lambda x: (x + \" \").cumsum().str.strip())\n    .reset_index(drop=True)\n)\ndataset[\"prev_movies\"] = dataset[\"prev_movies\"].apply(lambda x: x.split())\ndataset.head()\n</pre> # Here the author builds the sequences dataset[\"prev_movies\"] = dataset[\"movie_id\"].apply(lambda x: str(x)) dataset[\"prev_movies\"] = (     dataset.groupby(\"user_id\")[\"prev_movies\"]     .apply(lambda x: (x + \" \").cumsum().str.strip())     .reset_index(drop=True) ) dataset[\"prev_movies\"] = dataset[\"prev_movies\"].apply(lambda x: x.split()) dataset.head() Out[12]: user_id movie_id rating timestamp num_watched mean_rate target prev_movies 0 1 168 5 874965478 1 5.00 172.0 [168] 1 1 172 5 874965478 2 5.00 165.0 [168, 172] 2 1 165 5 874965518 3 5.00 156.0 [168, 172, 165] 3 1 156 4 874965556 4 4.75 196.0 [168, 172, 165, 156] 4 1 196 5 874965677 5 4.80 166.0 [168, 172, 165, 156, 196] <p>And now we add a <code>genre_rate</code> as the mean of all movies rated for a given genre per user</p> In\u00a0[13]: Copied! <pre>dataset = dataset.merge(items[[\"movie_id\"] + list_of_genres], on=\"movie_id\", how=\"left\")\nfor genre in list_of_genres:\n    dataset[f\"{genre}_rate\"] = dataset[genre] * dataset[\"rating\"]\n    dataset[genre] = dataset.groupby(\"user_id\")[genre].cumsum()\n    dataset[f\"{genre}_rate\"] = (\n        dataset.groupby(\"user_id\")[f\"{genre}_rate\"].cumsum() / dataset[genre]\n    )\ndataset[list_of_genres] = dataset[list_of_genres].apply(\n    lambda x: x / dataset[\"num_watched\"]\n)\ndataset.head()\n</pre> dataset = dataset.merge(items[[\"movie_id\"] + list_of_genres], on=\"movie_id\", how=\"left\") for genre in list_of_genres:     dataset[f\"{genre}_rate\"] = dataset[genre] * dataset[\"rating\"]     dataset[genre] = dataset.groupby(\"user_id\")[genre].cumsum()     dataset[f\"{genre}_rate\"] = (         dataset.groupby(\"user_id\")[f\"{genre}_rate\"].cumsum() / dataset[genre]     ) dataset[list_of_genres] = dataset[list_of_genres].apply(     lambda x: x / dataset[\"num_watched\"] ) dataset.head() Out[13]: user_id movie_id rating timestamp num_watched mean_rate target prev_movies unknown Action ... Fantasy_rate Film-Noir_rate Horror_rate Musical_rate Mystery_rate Romance_rate Sci-Fi_rate Thriller_rate War_rate Western_rate 0 1 168 5 874965478 1 5.00 172.0 [168] 0.0 0.000000 ... NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN 1 1 172 5 874965478 2 5.00 165.0 [168, 172] 0.0 0.500000 ... NaN NaN NaN NaN NaN 5.0 5.0 NaN 5.0 NaN 2 1 165 5 874965518 3 5.00 156.0 [168, 172, 165] 0.0 0.333333 ... NaN NaN NaN NaN NaN 5.0 5.0 NaN 5.0 NaN 3 1 156 4 874965556 4 4.75 196.0 [168, 172, 165, 156] 0.0 0.250000 ... NaN NaN NaN NaN NaN 5.0 5.0 4.0 5.0 NaN 4 1 196 5 874965677 5 4.80 166.0 [168, 172, 165, 156, 196] 0.0 0.200000 ... NaN NaN NaN NaN NaN 5.0 5.0 4.0 5.0 NaN <p>5 rows \u00d7 46 columns</p> <p>Adding user features</p> In\u00a0[14]: Copied! <pre>dataset = dataset.merge(users, on=\"user_id\", how=\"left\")\ndataset.head()\n</pre> dataset = dataset.merge(users, on=\"user_id\", how=\"left\") dataset.head() Out[14]: user_id movie_id rating timestamp num_watched mean_rate target prev_movies unknown Action ... Mystery_rate Romance_rate Sci-Fi_rate Thriller_rate War_rate Western_rate age gender occupation zip_code 0 1 168 5 874965478 1 5.00 172.0 [168] 0.0 0.000000 ... NaN NaN NaN NaN NaN NaN 24 M technician 85711 1 1 172 5 874965478 2 5.00 165.0 [168, 172] 0.0 0.500000 ... NaN 5.0 5.0 NaN 5.0 NaN 24 M technician 85711 2 1 165 5 874965518 3 5.00 156.0 [168, 172, 165] 0.0 0.333333 ... NaN 5.0 5.0 NaN 5.0 NaN 24 M technician 85711 3 1 156 4 874965556 4 4.75 196.0 [168, 172, 165, 156] 0.0 0.250000 ... NaN 5.0 5.0 4.0 5.0 NaN 24 M technician 85711 4 1 196 5 874965677 5 4.80 166.0 [168, 172, 165, 156, 196] 0.0 0.200000 ... NaN 5.0 5.0 4.0 5.0 NaN 24 M technician 85711 <p>5 rows \u00d7 50 columns</p> <p>Again, we use the same settings as those in the Kaggle notebook, but <code>COLD_START_TRESH</code> is pretty aggressive</p> In\u00a0[15]: Copied! <pre>COLD_START_TRESH = 5\n\nfiltred_data = dataset[\n    (dataset[\"num_watched\"] &gt;= COLD_START_TRESH) &amp; ~(dataset[\"target\"].isna())\n].sort_values(\"timestamp\")\ntrain_data, _test_data = train_test_split(filtred_data, test_size=0.2, shuffle=False)\nvalid_data, test_data = train_test_split(_test_data, test_size=0.5, shuffle=False)\n</pre> COLD_START_TRESH = 5  filtred_data = dataset[     (dataset[\"num_watched\"] &gt;= COLD_START_TRESH) &amp; ~(dataset[\"target\"].isna()) ].sort_values(\"timestamp\") train_data, _test_data = train_test_split(filtred_data, test_size=0.2, shuffle=False) valid_data, test_data = train_test_split(_test_data, test_size=0.5, shuffle=False) In\u00a0[16]: Copied! <pre>cols_to_drop = [\n    # \"rating\",\n    \"timestamp\",\n    \"num_watched\",\n]\n\ndf_train = train_data.drop(cols_to_drop, axis=1)\ndf_valid = valid_data.drop(cols_to_drop, axis=1)\ndf_test = test_data.drop(cols_to_drop, axis=1)\n\ndf_train.to_pickle(save_path / \"df_train.pkl\")\ndf_valid.to_pickle(save_path / \"df_valid.pkl\")\ndf_test.to_pickle(save_path / \"df_test.pkl\")\n</pre> cols_to_drop = [     # \"rating\",     \"timestamp\",     \"num_watched\", ]  df_train = train_data.drop(cols_to_drop, axis=1) df_valid = valid_data.drop(cols_to_drop, axis=1) df_test = test_data.drop(cols_to_drop, axis=1)  df_train.to_pickle(save_path / \"df_train.pkl\") df_valid.to_pickle(save_path / \"df_valid.pkl\") df_test.to_pickle(save_path / \"df_test.pkl\") <p>Let's now build a model that is nearly identical to the one use in the Kaggle notebook</p> In\u00a0[17]: Copied! <pre>import numpy as np\nimport torch\nfrom torch import nn\nfrom scipy.sparse import coo_matrix\n\nfrom pytorch_widedeep import Trainer\nfrom pytorch_widedeep.models import TabMlp, BasicRNN, WideDeep\nfrom pytorch_widedeep.preprocessing import TabPreprocessor\n</pre> import numpy as np import torch from torch import nn from scipy.sparse import coo_matrix  from pytorch_widedeep import Trainer from pytorch_widedeep.models import TabMlp, BasicRNN, WideDeep from pytorch_widedeep.preprocessing import TabPreprocessor In\u00a0[18]: Copied! <pre>device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n\nsave_path = Path(\"prepared_data\")\n\nPAD_IDX = 0\n</pre> device = \"cuda\" if torch.cuda.is_available() else \"cpu\"  save_path = Path(\"prepared_data\")  PAD_IDX = 0 <p>Let's use some of the functions the author of the kaggle's notebook uses to prepare the data</p> In\u00a0[19]: Copied! <pre>def get_coo_indexes(lil):\n    rows = []\n    cols = []\n    for i, el in enumerate(lil):\n        if type(el) != list:\n            el = [el]\n        for j in el:\n            rows.append(i)\n            cols.append(j)\n    return rows, cols\n\n\ndef get_sparse_features(series, shape):\n    coo_indexes = get_coo_indexes(series.tolist())\n    sparse_df = coo_matrix(\n        (np.ones(len(coo_indexes[0])), (coo_indexes[0], coo_indexes[1])), shape=shape\n    )\n    return sparse_df\n\n\ndef sparse_to_idx(data, pad_idx=-1):\n    indexes = data.nonzero()\n    indexes_df = pd.DataFrame()\n    indexes_df[\"rows\"] = indexes[0]\n    indexes_df[\"cols\"] = indexes[1]\n    mdf = indexes_df.groupby(\"rows\").apply(lambda x: x[\"cols\"].tolist())\n    max_len = mdf.apply(lambda x: len(x)).max()\n    return mdf.apply(lambda x: pd.Series(x + [pad_idx] * (max_len - len(x)))).values\n</pre> def get_coo_indexes(lil):     rows = []     cols = []     for i, el in enumerate(lil):         if type(el) != list:             el = [el]         for j in el:             rows.append(i)             cols.append(j)     return rows, cols   def get_sparse_features(series, shape):     coo_indexes = get_coo_indexes(series.tolist())     sparse_df = coo_matrix(         (np.ones(len(coo_indexes[0])), (coo_indexes[0], coo_indexes[1])), shape=shape     )     return sparse_df   def sparse_to_idx(data, pad_idx=-1):     indexes = data.nonzero()     indexes_df = pd.DataFrame()     indexes_df[\"rows\"] = indexes[0]     indexes_df[\"cols\"] = indexes[1]     mdf = indexes_df.groupby(\"rows\").apply(lambda x: x[\"cols\"].tolist())     max_len = mdf.apply(lambda x: len(x)).max()     return mdf.apply(lambda x: pd.Series(x + [pad_idx] * (max_len - len(x)))).values <p>For the time being, we will not use a validation set for hyperparameter optimization, and we will simply concatenate the validation and the test set in one test set. I simply splitted the data into train/valid/test in case the reader wants to actually do hyperparameter optimization (and because I know in the future I will).</p> <p>There is also another caveat worth mentioning, related to the indexing of the movies. To build the matrices of movies watched, we use the entire dataset. A more realistic (and correct) approach would be to use ONLY the movies that appear in the training set and consider <code>unknown</code> or <code>unseen</code> those in the testing set that have not been seen during training. Nonetheless, this will not affect the purposes of this notebook, which is to illustrate how one could use <code>pytorch-widedeep</code> to build a recommendation algorithm. However, if one wanted to explore the performance of different algorithms in a \"proper\" way, these \"details\" need to be accounted for.</p> In\u00a0[20]: Copied! <pre>df_test = pd.concat([df_valid, df_test], ignore_index=True)\n</pre> df_test = pd.concat([df_valid, df_test], ignore_index=True) In\u00a0[21]: Copied! <pre>id_cols = [\"user_id\", \"movie_id\"]\nmax_movie_index = max(df_train.movie_id.max(), df_test.movie_id.max())\n</pre> id_cols = [\"user_id\", \"movie_id\"] max_movie_index = max(df_train.movie_id.max(), df_test.movie_id.max()) In\u00a0[22]: Copied! <pre>X_train = df_train.drop(id_cols + [\"rating\", \"prev_movies\", \"target\"], axis=1)\ny_train = np.array(df_train.target.values, dtype=\"int64\")\ntrain_movies_watched = get_sparse_features(\n    df_train[\"prev_movies\"], (len(df_train), max_movie_index + 1)\n)\n\nX_test = df_test.drop(id_cols + [\"rating\", \"prev_movies\", \"target\"], axis=1)\ny_test = np.array(df_test.target.values, dtype=\"int64\")\ntest_movies_watched = get_sparse_features(\n    df_test[\"prev_movies\"], (len(df_test), max_movie_index + 1)\n)\n</pre> X_train = df_train.drop(id_cols + [\"rating\", \"prev_movies\", \"target\"], axis=1) y_train = np.array(df_train.target.values, dtype=\"int64\") train_movies_watched = get_sparse_features(     df_train[\"prev_movies\"], (len(df_train), max_movie_index + 1) )  X_test = df_test.drop(id_cols + [\"rating\", \"prev_movies\", \"target\"], axis=1) y_test = np.array(df_test.target.values, dtype=\"int64\") test_movies_watched = get_sparse_features(     df_test[\"prev_movies\"], (len(df_test), max_movie_index + 1) ) <p>let's have a look to the information in each dataset</p> In\u00a0[23]: Copied! <pre>X_train.head()\n</pre> X_train.head() Out[23]: mean_rate unknown Action Adventure Animation Children's Comedy Crime Documentary Drama ... Mystery_rate Romance_rate Sci-Fi_rate Thriller_rate War_rate Western_rate age gender occupation zip_code 25423 4.000000 0.0 0.400000 0.200000 0.0 0.0 0.400000 0.0 0.0 0.200000 ... NaN 4.0 4.0 4.000000 4.0 NaN 21 M student 48823 25425 4.000000 0.0 0.285714 0.142857 0.0 0.0 0.428571 0.0 0.0 0.285714 ... NaN 4.0 4.0 4.000000 4.0 NaN 21 M student 48823 25424 4.000000 0.0 0.333333 0.166667 0.0 0.0 0.333333 0.0 0.0 0.333333 ... NaN 4.0 4.0 4.000000 4.0 NaN 21 M student 48823 25426 3.875000 0.0 0.250000 0.125000 0.0 0.0 0.375000 0.0 0.0 0.250000 ... NaN 4.0 4.0 3.666667 4.0 NaN 21 M student 48823 25427 3.888889 0.0 0.222222 0.111111 0.0 0.0 0.333333 0.0 0.0 0.333333 ... NaN 4.0 4.0 3.666667 4.0 NaN 21 M student 48823 <p>5 rows \u00d7 43 columns</p> In\u00a0[24]: Copied! <pre>y_train\n</pre> y_train Out[24]: <pre>array([772, 288, 108, ..., 183, 432, 509])</pre> In\u00a0[25]: Copied! <pre>train_movies_watched\n</pre> train_movies_watched Out[25]: <pre>&lt;76228x1683 sparse matrix of type '&lt;class 'numpy.float64'&gt;'\n\twith 7957390 stored elements in COOrdinate format&gt;</pre> In\u00a0[26]: Copied! <pre>sorted(df_train.prev_movies.tolist()[0])\n</pre> sorted(df_train.prev_movies.tolist()[0]) Out[26]: <pre>['173', '185', '255', '286', '298']</pre> In\u00a0[27]: Copied! <pre>np.where(train_movies_watched.todense()[0])\n</pre> np.where(train_movies_watched.todense()[0]) Out[27]: <pre>(array([0, 0, 0, 0, 0]), array([173, 185, 255, 286, 298]))</pre> <p>And from now on is when the specifics related to this library start to appear. The only component that is going to be a bit different is the so-called tabular component, referred as <code>continuous</code> in the notebook.</p> <p>In the case of <code>pytorch-widedeep</code> we have the <code>TabPreprocessor</code> that allows for a lot of flexibility as to how we would like to process the tabular component of this Wide and Deep model. In other words, here our tabular component is a bit more elaborated than that in the notebook, just a bit...</p> In\u00a0[28]: Copied! <pre>cat_cols = [\"gender\", \"occupation\", \"zip_code\"]\ncont_cols = [c for c in X_train if c not in cat_cols]\ntab_preprocessor = TabPreprocessor(\n    cat_embed_cols=cat_cols,\n    continuous_cols=cont_cols,\n)\n</pre> cat_cols = [\"gender\", \"occupation\", \"zip_code\"] cont_cols = [c for c in X_train if c not in cat_cols] tab_preprocessor = TabPreprocessor(     cat_embed_cols=cat_cols,     continuous_cols=cont_cols, ) In\u00a0[29]: Copied! <pre>X_train_tab = tab_preprocessor.fit_transform(X_train.fillna(0))\nX_test_tab = tab_preprocessor.transform(X_test.fillna(0))\n</pre> X_train_tab = tab_preprocessor.fit_transform(X_train.fillna(0)) X_test_tab = tab_preprocessor.transform(X_test.fillna(0)) <p>Now, in the notebook, the author moves the sparse matrices to sparse tensors and then turns them into dense tensors. In reality, this is not neccessary, one could feed sparse tensors to <code>nn.Linear</code> layers in pytorch. Nonetheless, this is not the most efficient implementation and is the reason why in our library the wide, linear component is implemented as an embedding layer.</p> <p>Nonetheless, to reproduce the notebook the best we can and because currently the <code>Wide</code> model in <code>pytorch-widedeep</code> is not designed to receive sparse tensors (we might consider implementing this functionality), we will turn the sparse COO matrices into dense arrays. We will then code a fairly simple, custom <code>Wide</code> component.</p> In\u00a0[30]: Copied! <pre>X_train_wide = np.array(train_movies_watched.todense())\nX_test_wide = np.array(test_movies_watched.todense())\n</pre> X_train_wide = np.array(train_movies_watched.todense()) X_test_wide = np.array(test_movies_watched.todense()) <p>Finally, the author of the notebook uses a simple <code>Embedding</code> layer to encode the sequences of movies watched, the <code>prev_movies</code> columns. In my opinion, there is an element of information redundancy here. This is because the wide and text components have implicitely the same information, but in different form. Moreover, both of the models used for these two components ignore the sequential element in the data. Nonetheless, we want to reproduce the Kaggle notebook as close as possible, AND as one can explore later (by simply performing simple ablation studies), the wide component seems to carry most of the predictive power.</p> In\u00a0[31]: Copied! <pre>X_train_text = sparse_to_idx(train_movies_watched, pad_idx=PAD_IDX)\nX_test_text = sparse_to_idx(test_movies_watched, pad_idx=PAD_IDX)\n</pre> X_train_text = sparse_to_idx(train_movies_watched, pad_idx=PAD_IDX) X_test_text = sparse_to_idx(test_movies_watched, pad_idx=PAD_IDX) <p>Let's now build the models</p> In\u00a0[32]: Copied! <pre>class Wide(nn.Module):\n    def __init__(self, input_dim: int, pred_dim: int):\n        super().__init__()\n\n        self.input_dim = input_dim\n        self.pred_dim = pred_dim\n\n        # When I coded the library I never though that someone would want to code\n        # their own wide component. However, if you do, the wide component must have\n        # a 'wide_linear' attribute. In other words, the linear layer must be\n        # called 'wide_linear'\n        self.wide_linear = nn.Linear(input_dim, pred_dim)\n\n    def forward(self, X):\n        out = self.wide_linear(X.type(torch.float32))\n        return out\n\n\nwide = Wide(X_train_wide.shape[1], max_movie_index + 1)\n</pre> class Wide(nn.Module):     def __init__(self, input_dim: int, pred_dim: int):         super().__init__()          self.input_dim = input_dim         self.pred_dim = pred_dim          # When I coded the library I never though that someone would want to code         # their own wide component. However, if you do, the wide component must have         # a 'wide_linear' attribute. In other words, the linear layer must be         # called 'wide_linear'         self.wide_linear = nn.Linear(input_dim, pred_dim)      def forward(self, X):         out = self.wide_linear(X.type(torch.float32))         return out   wide = Wide(X_train_wide.shape[1], max_movie_index + 1) In\u00a0[33]: Copied! <pre>wide\n</pre> wide Out[33]: <pre>Wide(\n  (wide_linear): Linear(in_features=1683, out_features=1683, bias=True)\n)</pre> In\u00a0[34]: Copied! <pre>class SimpleEmbed(nn.Module):\n    def __init__(self, vocab_size: int, embed_dim: int, pad_idx: int):\n        super().__init__()\n\n        self.vocab_size = vocab_size\n        self.embed_dim = embed_dim\n        self.pad_idx = pad_idx\n\n        # The sequences of movies watched are simply embedded in the Kaggle\n        # notebook. No RNN, Transformer or any model is used\n        self.embed = nn.Embedding(vocab_size, embed_dim, padding_idx=pad_idx)\n\n    def forward(self, X):\n        embed = self.embed(X)\n        embed_mean = torch.mean(embed, dim=1)\n        return embed_mean\n\n    @property\n    def output_dim(self) -&gt; int:\n        # All deep components in a custom 'pytorch-widedeep' model must have\n        # an output_dim property\n        return self.embed_dim\n\n\n#  In the notebook the author uses simply embeddings\nsimple_embed = SimpleEmbed(max_movie_index + 1, 16, 0)\n</pre> class SimpleEmbed(nn.Module):     def __init__(self, vocab_size: int, embed_dim: int, pad_idx: int):         super().__init__()          self.vocab_size = vocab_size         self.embed_dim = embed_dim         self.pad_idx = pad_idx          # The sequences of movies watched are simply embedded in the Kaggle         # notebook. No RNN, Transformer or any model is used         self.embed = nn.Embedding(vocab_size, embed_dim, padding_idx=pad_idx)      def forward(self, X):         embed = self.embed(X)         embed_mean = torch.mean(embed, dim=1)         return embed_mean      @property     def output_dim(self) -&gt; int:         # All deep components in a custom 'pytorch-widedeep' model must have         # an output_dim property         return self.embed_dim   #  In the notebook the author uses simply embeddings simple_embed = SimpleEmbed(max_movie_index + 1, 16, 0) In\u00a0[35]: Copied! <pre>simple_embed\n</pre> simple_embed Out[35]: <pre>SimpleEmbed(\n  (embed): Embedding(1683, 16, padding_idx=0)\n)</pre> <p>Maybe one would like to use an RNN to account for the sequence nature of the problem. If that was the case it would be as easy as:</p> In\u00a0[36]: Copied! <pre>basic_rnn = BasicRNN(\n    vocab_size=max_movie_index + 1,\n    embed_dim=16,\n    hidden_dim=32,\n    n_layers=2,\n    rnn_type=\"gru\",\n)\n</pre> basic_rnn = BasicRNN(     vocab_size=max_movie_index + 1,     embed_dim=16,     hidden_dim=32,     n_layers=2,     rnn_type=\"gru\", ) <p>And finally, the tabular component, which is the notebook is simply a stak of linear + Rely layers. In our case we have an embedding layer before the linear layers to encode categorial and numerical cols</p> In\u00a0[37]: Copied! <pre>tab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=tab_preprocessor.continuous_cols,\n    cont_norm_layer=None,\n    mlp_hidden_dims=[1024, 512, 256],\n    mlp_activation=\"relu\",\n)\n</pre> tab_mlp = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     continuous_cols=tab_preprocessor.continuous_cols,     cont_norm_layer=None,     mlp_hidden_dims=[1024, 512, 256],     mlp_activation=\"relu\", ) In\u00a0[38]: Copied! <pre>tab_mlp\n</pre> tab_mlp Out[38]: <pre>TabMlp(\n  (cat_embed): DiffSizeCatEmbeddings(\n    (embed_layers): ModuleDict(\n      (emb_layer_gender): Embedding(3, 2, padding_idx=0)\n      (emb_layer_occupation): Embedding(22, 9, padding_idx=0)\n      (emb_layer_zip_code): Embedding(648, 60, padding_idx=0)\n    )\n    (embedding_dropout): Dropout(p=0.0, inplace=False)\n  )\n  (cont_norm): Identity()\n  (encoder): MLP(\n    (mlp): Sequential(\n      (dense_layer_0): Sequential(\n        (0): Linear(in_features=111, out_features=1024, bias=True)\n        (1): ReLU(inplace=True)\n        (2): Dropout(p=0.1, inplace=False)\n      )\n      (dense_layer_1): Sequential(\n        (0): Linear(in_features=1024, out_features=512, bias=True)\n        (1): ReLU(inplace=True)\n        (2): Dropout(p=0.1, inplace=False)\n      )\n      (dense_layer_2): Sequential(\n        (0): Linear(in_features=512, out_features=256, bias=True)\n        (1): ReLU(inplace=True)\n        (2): Dropout(p=0.1, inplace=False)\n      )\n    )\n  )\n)</pre> <p>Finally, we simply wrap up all models with the <code>WideDeep</code> 'collector' class and we are ready to train.</p> In\u00a0[39]: Copied! <pre>wide_deep_model = WideDeep(\n    wide=wide, deeptabular=tab_mlp, deeptext=simple_embed, pred_dim=max_movie_index + 1\n)\n</pre> wide_deep_model = WideDeep(     wide=wide, deeptabular=tab_mlp, deeptext=simple_embed, pred_dim=max_movie_index + 1 ) In\u00a0[40]: Copied! <pre>wide_deep_model\n</pre> wide_deep_model Out[40]: <pre>WideDeep(\n  (wide): Wide(\n    (wide_linear): Linear(in_features=1683, out_features=1683, bias=True)\n  )\n  (deeptabular): Sequential(\n    (0): TabMlp(\n      (cat_embed): DiffSizeCatEmbeddings(\n        (embed_layers): ModuleDict(\n          (emb_layer_gender): Embedding(3, 2, padding_idx=0)\n          (emb_layer_occupation): Embedding(22, 9, padding_idx=0)\n          (emb_layer_zip_code): Embedding(648, 60, padding_idx=0)\n        )\n        (embedding_dropout): Dropout(p=0.0, inplace=False)\n      )\n      (cont_norm): Identity()\n      (encoder): MLP(\n        (mlp): Sequential(\n          (dense_layer_0): Sequential(\n            (0): Linear(in_features=111, out_features=1024, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_1): Sequential(\n            (0): Linear(in_features=1024, out_features=512, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_2): Sequential(\n            (0): Linear(in_features=512, out_features=256, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=256, out_features=1683, bias=True)\n  )\n  (deeptext): Sequential(\n    (0): SimpleEmbed(\n      (embed): Embedding(1683, 16, padding_idx=0)\n    )\n    (1): Linear(in_features=16, out_features=1683, bias=True)\n  )\n)</pre> <p>Note that the main difference between this wide and deep model and the Wide and Deep model in the Kaggle notebook is that in that notebook, the author concatenates the embedings and the tabular features, then passes this concatenation through a stack of linear + Relu layers with a final output dim of 256. Then concatenates this output with the binary features and connects this concatenation with the final linear layer (so the final weights are of dim (batch_size, 256 + 1683)). Our implementation follows the notation of the original paper and instead of concatenating the tabular, text and wide components and then connect them to the output neurons, we first compute their output, and then add it (see here: https://arxiv.org/pdf/1606.07792.pdf, their Eq 3). Note that this is effectively the same, with the caveat that while in one case one initialises a big weight matrix \"at once\", in our implementation we initialise different matrices for different components. Anyway, let's give it a go.</p> In\u00a0[41]: Copied! <pre>trainer = Trainer(\n    model=wide_deep_model,\n    objective=\"multiclass\",\n    custom_loss_function=nn.CrossEntropyLoss(ignore_index=PAD_IDX),\n    optimizers=torch.optim.Adam(wide_deep_model.parameters(), lr=1e-3),\n)\n</pre> trainer = Trainer(     model=wide_deep_model,     objective=\"multiclass\",     custom_loss_function=nn.CrossEntropyLoss(ignore_index=PAD_IDX),     optimizers=torch.optim.Adam(wide_deep_model.parameters(), lr=1e-3), ) In\u00a0[42]: Copied! <pre>trainer.fit(\n    X_train={\n        \"X_wide\": X_train_wide,\n        \"X_tab\": X_train_tab,\n        \"X_text\": X_train_text,\n        \"target\": y_train,\n    },\n    X_val={\n        \"X_wide\": X_test_wide,\n        \"X_tab\": X_test_tab,\n        \"X_text\": X_test_text,\n        \"target\": y_test,\n    },\n    n_epochs=5,\n    batch_size=512,\n    shuffle=False,\n)\n</pre> trainer.fit(     X_train={         \"X_wide\": X_train_wide,         \"X_tab\": X_train_tab,         \"X_text\": X_train_text,         \"target\": y_train,     },     X_val={         \"X_wide\": X_test_wide,         \"X_tab\": X_test_tab,         \"X_text\": X_test_text,         \"target\": y_test,     },     n_epochs=5,     batch_size=512,     shuffle=False, ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 149/149 [00:19&lt;00:00,  7.66it/s, loss=6.66]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 38/38 [00:02&lt;00:00, 18.75it/s, loss=6.6]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 149/149 [00:21&lt;00:00,  6.95it/s, loss=5.97]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 38/38 [00:01&lt;00:00, 21.03it/s, loss=6.52]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 149/149 [00:19&lt;00:00,  7.51it/s, loss=5.65]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 38/38 [00:01&lt;00:00, 20.16it/s, loss=6.53]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 149/149 [00:23&lt;00:00,  6.29it/s, loss=5.41]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 38/38 [00:02&lt;00:00, 13.97it/s, loss=6.57]\nepoch 5: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 149/149 [00:19&lt;00:00,  7.58it/s, loss=5.2]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 38/38 [00:02&lt;00:00, 18.82it/s, loss=6.63]\n</pre> <p>Now one could continue to the 'compare' metrics section of the Kaggle notebook. However, for the purposes of illustrating how one could use <code>pytorch-widedeep</code> to build recommendation algorithms we consider this notebook completed and move onto part 2</p>"},{"location":"examples/18_wide_and_deep_for_recsys_pt1.html#problem-formulation","title":"Problem formulation\u00b6","text":"<p>In this particular exercise the problem is formulated as predicting the next movie that will be watched (in consequence the last interactions will be discarded)</p>"},{"location":"examples/18_wide_and_deep_for_recsys_pt2.html","title":"18_wide_and_deep_for_recsys_pt2","text":"<p>This is the second of the two notebooks where we aim to illustrate how one could use this library to build recommendation algorithms using the example in this Kaggle notebook as guidance. In the previous notebook we used <code>pytorch-widedeep</code> to build a model that replicated almost exactly that in the notebook. In this, shorter notebook we will show how one could use the library to explore other models, following the same problem formulation, this is: given a state of a user at a certain point in time having watched a series of movies, our goal is to predict which movie the user will watch next.</p> <p>Assuming that one has read (and run) the previous notebook, the required data will be stored in a local dir called <code>prepared_data</code>, so let's read it:</p> In\u00a0[1]: Copied! <pre>from pathlib import Path\n\nimport numpy as np\nimport torch\nimport pandas as pd\nfrom torch import nn\n\nfrom pytorch_widedeep import Trainer\nfrom pytorch_widedeep.utils import pad_sequences\nfrom pytorch_widedeep.models import TabMlp, WideDeep, Transformer\nfrom pytorch_widedeep.preprocessing import TabPreprocessor\n</pre> from pathlib import Path  import numpy as np import torch import pandas as pd from torch import nn  from pytorch_widedeep import Trainer from pytorch_widedeep.utils import pad_sequences from pytorch_widedeep.models import TabMlp, WideDeep, Transformer from pytorch_widedeep.preprocessing import TabPreprocessor In\u00a0[2]: Copied! <pre>save_path = Path(\"prepared_data\")\n\nPAD_IDX = 0\n\nid_cols = [\"user_id\", \"movie_id\"]\n\ndf_train = pd.read_pickle(save_path / \"df_train.pkl\")\ndf_valid = pd.read_pickle(save_path / \"df_valid.pkl\")\ndf_test = pd.read_pickle(save_path / \"df_test.pkl\")\n</pre> save_path = Path(\"prepared_data\")  PAD_IDX = 0  id_cols = [\"user_id\", \"movie_id\"]  df_train = pd.read_pickle(save_path / \"df_train.pkl\") df_valid = pd.read_pickle(save_path / \"df_valid.pkl\") df_test = pd.read_pickle(save_path / \"df_test.pkl\") <p>...remember that in the previous notebook we explained that we are not  going to use a validation set here (in a real-world example, or simply a more realistic example, one should always use it).</p> In\u00a0[3]: Copied! <pre>df_test = pd.concat([df_valid, df_test], ignore_index=True)\n</pre> df_test = pd.concat([df_valid, df_test], ignore_index=True) <p>Also remember that, in the previous notebook we discussed that the <code>'maxlen'</code> and <code>'max_movie_index'</code> parameters should be computed using only the train set. In particular, to properly do the tokenization, one would have to use ONLY train tokens and add a token for new 'unknown'/'unseen' movies in the test set. This can also be done with this library or manually, so I will leave it to the reader to implement that tokenzation appraoch.</p> In\u00a0[4]: Copied! <pre>maxlen = max(\n    df_train.prev_movies.apply(lambda x: len(x)).max(),\n    df_test.prev_movies.apply(lambda x: len(x)).max(),\n)\n\nmax_movie_index = max(df_train.movie_id.max(), df_test.movie_id.max())\n</pre> maxlen = max(     df_train.prev_movies.apply(lambda x: len(x)).max(),     df_test.prev_movies.apply(lambda x: len(x)).max(), )  max_movie_index = max(df_train.movie_id.max(), df_test.movie_id.max()) <p>From now one things are pretty simple, moreover bearing in mind that in this example we are not going to use a wide component since, in pple, one would believe that the information in that component is also 'carried' by the movie sequences (However in the previous notebook, if one performs ablation studies, these suggest that most of the prediction power comes from the linear, wide model).</p> <p>In the example here we are going to explore one (of many) possibilities. We are simply going to encode the triplet <code>(user, item, rating)</code> and use it as a <code>deeptabular</code> component and the sequences of previously watched movies as the <code>deeptext</code> component. For the <code>deeptext</code> component we are going to use a basic encoder-only transformer model.</p> <p>Let's start with the tabular data preparation</p> In\u00a0[5]: Copied! <pre>df_train_user_item = df_train[[\"user_id\", \"movie_id\", \"rating\"]]\ntrain_movies_sequences = df_train.prev_movies.apply(\n    lambda x: [int(el) for el in x]\n).to_list()\ny_train = df_train.target.values.astype(int)\n\ndf_test_user_item = df_train[[\"user_id\", \"movie_id\", \"rating\"]]\ntest_movies_sequences = df_test.prev_movies.apply(\n    lambda x: [int(el) for el in x]\n).to_list()\ny_test = df_test.target.values.astype(int)\n\ntab_preprocessor = tab_preprocessor = TabPreprocessor(\n    cat_embed_cols=[\"user_id\", \"movie_id\", \"rating\"],\n)\nX_train_tab = tab_preprocessor.fit_transform(df_train_user_item)\nX_test_tab = tab_preprocessor.transform(df_test_user_item)\n</pre> df_train_user_item = df_train[[\"user_id\", \"movie_id\", \"rating\"]] train_movies_sequences = df_train.prev_movies.apply(     lambda x: [int(el) for el in x] ).to_list() y_train = df_train.target.values.astype(int)  df_test_user_item = df_train[[\"user_id\", \"movie_id\", \"rating\"]] test_movies_sequences = df_test.prev_movies.apply(     lambda x: [int(el) for el in x] ).to_list() y_test = df_test.target.values.astype(int)  tab_preprocessor = tab_preprocessor = TabPreprocessor(     cat_embed_cols=[\"user_id\", \"movie_id\", \"rating\"], ) X_train_tab = tab_preprocessor.fit_transform(df_train_user_item) X_test_tab = tab_preprocessor.transform(df_test_user_item) <p>And not the text component, simply padding the sequences:</p> In\u00a0[6]: Copied! <pre>X_train_text = np.array(\n    [\n        pad_sequences(\n            s,\n            maxlen=maxlen,\n            pad_first=False,\n            pad_idx=PAD_IDX,\n        )\n        for s in train_movies_sequences\n    ]\n)\nX_test_text = np.array(\n    [\n        pad_sequences(\n            s,\n            maxlen=maxlen,\n            pad_first=False,\n            pad_idx=0,\n        )\n        for s in test_movies_sequences\n    ]\n)\n</pre> X_train_text = np.array(     [         pad_sequences(             s,             maxlen=maxlen,             pad_first=False,             pad_idx=PAD_IDX,         )         for s in train_movies_sequences     ] ) X_test_text = np.array(     [         pad_sequences(             s,             maxlen=maxlen,             pad_first=False,             pad_idx=0,         )         for s in test_movies_sequences     ] ) <p>We now define the model components and the wide and deep model.</p> In\u00a0[7]: Copied! <pre>tab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    mlp_hidden_dims=[1024, 512, 256],\n    mlp_activation=\"relu\",\n)\n\n# plenty of options here, see the docs\ntransformer = Transformer(\n    vocab_size=max_movie_index + 1,\n    embed_dim=32,\n    n_heads=2,\n    n_blocks=2,\n    seq_length=maxlen,\n)\n\nwide_deep_model = WideDeep(\n    deeptabular=tab_mlp, deeptext=transformer, pred_dim=max_movie_index + 1\n)\n</pre> tab_mlp = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     mlp_hidden_dims=[1024, 512, 256],     mlp_activation=\"relu\", )  # plenty of options here, see the docs transformer = Transformer(     vocab_size=max_movie_index + 1,     embed_dim=32,     n_heads=2,     n_blocks=2,     seq_length=maxlen, )  wide_deep_model = WideDeep(     deeptabular=tab_mlp, deeptext=transformer, pred_dim=max_movie_index + 1 ) In\u00a0[8]: Copied! <pre>wide_deep_model\n</pre> wide_deep_model Out[8]: <pre>WideDeep(\n  (deeptabular): Sequential(\n    (0): TabMlp(\n      (cat_embed): DiffSizeCatEmbeddings(\n        (embed_layers): ModuleDict(\n          (emb_layer_user_id): Embedding(749, 65, padding_idx=0)\n          (emb_layer_movie_id): Embedding(1612, 100, padding_idx=0)\n          (emb_layer_rating): Embedding(6, 4, padding_idx=0)\n        )\n        (embedding_dropout): Dropout(p=0.0, inplace=False)\n      )\n      (encoder): MLP(\n        (mlp): Sequential(\n          (dense_layer_0): Sequential(\n            (0): Linear(in_features=169, out_features=1024, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_1): Sequential(\n            (0): Linear(in_features=1024, out_features=512, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_2): Sequential(\n            (0): Linear(in_features=512, out_features=256, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=256, out_features=1683, bias=True)\n  )\n  (deeptext): Sequential(\n    (0): Transformer(\n      (embedding): Embedding(1683, 32, padding_idx=0)\n      (pos_encoder): PositionalEncoding(\n        (dropout): Dropout(p=0.1, inplace=False)\n      )\n      (encoder): Sequential(\n        (transformer_block0): TransformerEncoder(\n          (attn): MultiHeadedAttention(\n            (dropout): Dropout(p=0.1, inplace=False)\n            (q_proj): Linear(in_features=32, out_features=32, bias=False)\n            (kv_proj): Linear(in_features=32, out_features=64, bias=False)\n            (out_proj): Linear(in_features=32, out_features=32, bias=False)\n          )\n          (ff): FeedForward(\n            (w_1): Linear(in_features=32, out_features=128, bias=True)\n            (w_2): Linear(in_features=128, out_features=32, bias=True)\n            (dropout): Dropout(p=0.1, inplace=False)\n            (activation): GELU(approximate='none')\n          )\n          (attn_addnorm): AddNorm(\n            (dropout): Dropout(p=0.1, inplace=False)\n            (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n          )\n          (ff_addnorm): AddNorm(\n            (dropout): Dropout(p=0.1, inplace=False)\n            (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n          )\n        )\n        (transformer_block1): TransformerEncoder(\n          (attn): MultiHeadedAttention(\n            (dropout): Dropout(p=0.1, inplace=False)\n            (q_proj): Linear(in_features=32, out_features=32, bias=False)\n            (kv_proj): Linear(in_features=32, out_features=64, bias=False)\n            (out_proj): Linear(in_features=32, out_features=32, bias=False)\n          )\n          (ff): FeedForward(\n            (w_1): Linear(in_features=32, out_features=128, bias=True)\n            (w_2): Linear(in_features=128, out_features=32, bias=True)\n            (dropout): Dropout(p=0.1, inplace=False)\n            (activation): GELU(approximate='none')\n          )\n          (attn_addnorm): AddNorm(\n            (dropout): Dropout(p=0.1, inplace=False)\n            (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n          )\n          (ff_addnorm): AddNorm(\n            (dropout): Dropout(p=0.1, inplace=False)\n            (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=23552, out_features=1683, bias=True)\n  )\n)</pre> <p>And as in the previous notebook, let's train (you will need a GPU for this)</p> In\u00a0[\u00a0]: Copied! <pre>trainer = Trainer(\n    model=wide_deep_model,\n    objective=\"multiclass\",\n    custom_loss_function=nn.CrossEntropyLoss(ignore_index=PAD_IDX),\n    optimizers=torch.optim.Adam(wide_deep_model.parameters(), lr=1e-3),\n)\n\ntrainer.fit(\n    X_train={\n        \"X_tab\": X_train_tab,\n        \"X_text\": X_train_text,\n        \"target\": y_train,\n    },\n    X_val={\n        \"X_tab\": X_test_tab,\n        \"X_text\": X_test_text,\n        \"target\": y_test,\n    },\n    n_epochs=10,\n    batch_size=521,\n    shuffle=False,\n)\n</pre> trainer = Trainer(     model=wide_deep_model,     objective=\"multiclass\",     custom_loss_function=nn.CrossEntropyLoss(ignore_index=PAD_IDX),     optimizers=torch.optim.Adam(wide_deep_model.parameters(), lr=1e-3), )  trainer.fit(     X_train={         \"X_tab\": X_train_tab,         \"X_text\": X_train_text,         \"target\": y_train,     },     X_val={         \"X_tab\": X_test_tab,         \"X_text\": X_test_text,         \"target\": y_test,     },     n_epochs=10,     batch_size=521,     shuffle=False, ) <pre>epoch 1:   0%|                                                                                                         | 0/147 [00:34&lt;?, ?it/s]\n</pre> In\u00a0[\u00a0]: Copied! <pre>\n</pre>"},{"location":"examples/19_load_from_folder_functionality.html","title":"19_load_from_folder_functionality","text":"<p>In this notebook I want to illustrate how one can use our <code>[...]FromFolder</code> functionalities along with the <code>[...]ChunkPreProcessors</code> in those cases where the dataset is too bit to fit in memory.</p> <p>These functionalities in the library have been designed for the following scenarop</p> In\u00a0[1]: Copied! <pre>import numpy as np\nimport torch\nimport pandas as pd\nfrom torch.utils.data import DataLoader\n\nfrom pytorch_widedeep.models import TabMlp, Vision, BasicRNN, WideDeep\nfrom pytorch_widedeep.training import TrainerFromFolder\nfrom pytorch_widedeep.callbacks import EarlyStopping, ModelCheckpoint\nfrom pytorch_widedeep.preprocessing import (\n    TabPreprocessor,\n    TextPreprocessor,\n    ImagePreprocessor,\n    ChunkTabPreprocessor,\n    ChunkTextPreprocessor,\n)\nfrom pytorch_widedeep.load_from_folder import (\n    TabFromFolder,\n    TextFromFolder,\n    ImageFromFolder,\n    WideDeepDatasetFromFolder,\n)\n</pre> import numpy as np import torch import pandas as pd from torch.utils.data import DataLoader  from pytorch_widedeep.models import TabMlp, Vision, BasicRNN, WideDeep from pytorch_widedeep.training import TrainerFromFolder from pytorch_widedeep.callbacks import EarlyStopping, ModelCheckpoint from pytorch_widedeep.preprocessing import (     TabPreprocessor,     TextPreprocessor,     ImagePreprocessor,     ChunkTabPreprocessor,     ChunkTextPreprocessor, ) from pytorch_widedeep.load_from_folder import (     TabFromFolder,     TextFromFolder,     ImageFromFolder,     WideDeepDatasetFromFolder, ) In\u00a0[2]: Copied! <pre># in my case, I place the data in a folder I call tmp_data, let's see how it looks\nairbnb_data = pd.read_csv(\"../tmp_data/airbnb/airbnb_sample.csv\")\nairbnb_data.sample(5)\n</pre> # in my case, I place the data in a folder I call tmp_data, let's see how it looks airbnb_data = pd.read_csv(\"../tmp_data/airbnb/airbnb_sample.csv\") airbnb_data.sample(5) Out[2]: id host_id description host_listings_count host_identity_verified neighbourhood_cleansed latitude longitude is_location_exact property_type ... amenity_wide_entrance amenity_wide_entrance_for_guests amenity_wide_entryway amenity_wide_hallways amenity_wifi amenity_window_guards amenity_wine_cooler security_deposit extra_people yield 39 53242.jpg 247650 A lovely big bright bedroom in a 2 bedroom fla... 2.0 t Lambeth 51.47075 -0.12913 t apartment ... 0 0 0 0 1 0 0 250.0 5.0 9.75 214 236716.jpg 1241070 We offer a warm welcome in our quiet double ro... 1.0 t Hackney 51.56593 -0.07482 t other ... 0 0 0 0 1 0 0 200.0 10.0 76.50 400 346523.jpg 1756532 Available for you to rent is a cozy studio in ... 2.0 t Kensington and Chelsea 51.48311 -0.18428 t other ... 0 0 0 0 1 0 0 0.0 50.0 180.90 512 389627.jpg 1949299 This gorgeous studio flat is situated in the v... 1.0 t Westminster 51.51838 -0.14238 f apartment ... 0 0 0 0 1 0 0 250.0 25.0 276.90 504 388767.jpg 1945165 If you want to experience London at it's best ... 2.0 f Camden 51.54293 -0.14073 t apartment ... 0 0 0 0 1 0 0 150.0 10.0 591.10 <p>5 rows \u00d7 223 columns</p> In\u00a0[3]: Copied! <pre># for example\nfrom IPython.display import Image\n\npil_img = Image(filename=\"../tmp_data/airbnb/property_picture/272908.jpg\")\ndisplay(pil_img)\n</pre> # for example from IPython.display import Image  pil_img = Image(filename=\"../tmp_data/airbnb/property_picture/272908.jpg\") display(pil_img) In\u00a0[4]: Copied! <pre># And the description for the property that that picture belongs to is:\nairbnb_data[airbnb_data.id == \"272908.jpg\"].description.tolist()\n</pre> # And the description for the property that that picture belongs to is: airbnb_data[airbnb_data.id == \"272908.jpg\"].description.tolist() Out[4]: <pre>[\"Bright, sunny beautiful room that will give you the perfect base to explore all of London. Come and explore one of London's best neighbourhoods - Herne Hill! As mentioned in (Website hidden by Airbnb)   (Website hidden by Airbnb)  WiFi availability with a fully stocked and clean uplifting home. Lovely sunny, airy and big double bedroom on a leafy south-London street.    Note: This room comes with a reserved Off-Street parking spot! The room is on the first floor and boasts an enormous Super King bed, gorgeous wooden floors, tall ceilings and large windows which let in the sunshine almost all day. (Yoga May or meditation cushion available on request) The flat is bright and airy and big! So lots of space for all.  Location wise you are only 10 minutes walk to either Herne Hill or West Dulwich stations, both of which will take you to Victoria and the city within minutes. You can also hop on a bus right outside the flat that will take you to Brixton tube station within 8 minutes where you \"]</pre> <p>Ok, so we have tabular data where one column is <code>description</code> and another <code>id</code>, points towards the images stored in disk. Now, remember the following, because this will appear a few times in the notebook: our \"reference dataset\" is the tabular data.</p> <p>Therefore, since I want to illustrate a \"semi-realistic\" case, if we need to split the data into training, validation and test datasets, these datasets needs to be separetely stored in disk. In my case I have done this and in the <code>tmp_data/airbnb</code> dir I have the following:</p> <pre>../tmp_data/airbnb\n\u251c\u2500\u2500 airbnb_sample.csv\n\u251c\u2500\u2500 airbnb_sample_eval.csv\n\u251c\u2500\u2500 airbnb_sample_test.csv\n\u251c\u2500\u2500 airbnb_sample_train.csv\n\u2514\u2500\u2500 property_picture\n</pre> <p>Where <code>airbnb_sample.csv</code> is the full sample (1001 rows) and the <code>train</code>, <code>eval</code> and <code>test</code> set is the corresponding splits. In a realistic example, the full sample would be the 'gigantic' dataset and the rest the corresponding splits. One has to do this 'offline', prior to start the coding.</p> <p>Also, one thing that one needs to know is the number of total observations/rows, as well as the splits. In our case the train size is 800, and the eval and test sizes are 100 and 101 respectively.</p> <p>With all that info, let's star</p> In\u00a0[5]: Copied! <pre># path to the tabular data and the splits\ndata_path = \"../tmp_data/airbnb/\"\ntrain_fname = \"airbnb_sample_train.csv\"\neval_fname = \"airbnb_sample_eval.csv\"\ntest_fname = \"airbnb_sample_test.csv\"\n\n# split sizes\ntrain_size = 800\neval_size = 100\ntest_size = 101\n\n# number of chunks for the Chunk Preprocessors\nchunksize = 100\nn_chunks = int(np.ceil(train_size / chunksize))\n\n# path to the image dataset and name of the image col\nimg_path = \"../tmp_data/airbnb/property_picture/\"\nimg_col = \"id\"\n\n# name of the text col\ntext_col = \"description\"\n\n# mane of the target\ntarget_col = \"yield\"\n\n# definition of the categorical and continuous cols for the TabPreprocessor\ncat_embed_cols = [\n    \"host_listings_count\",\n    \"neighbourhood_cleansed\",\n    \"is_location_exact\",\n    \"property_type\",\n    \"room_type\",\n    \"accommodates\",\n    \"bathrooms\",\n    \"bedrooms\",\n    \"beds\",\n    \"guests_included\",\n    \"minimum_nights\",\n    \"instant_bookable\",\n    \"cancellation_policy\",\n    \"has_house_rules\",\n    \"host_gender\",\n    \"accommodates_catg\",\n    \"guests_included_catg\",\n    \"minimum_nights_catg\",\n    \"host_listings_count_catg\",\n    \"bathrooms_catg\",\n    \"bedrooms_catg\",\n    \"beds_catg\",\n    \"security_deposit\",\n    \"extra_people\",\n]\ncont_cols = [\"latitude\", \"longitude\"]\n</pre> # path to the tabular data and the splits data_path = \"../tmp_data/airbnb/\" train_fname = \"airbnb_sample_train.csv\" eval_fname = \"airbnb_sample_eval.csv\" test_fname = \"airbnb_sample_test.csv\"  # split sizes train_size = 800 eval_size = 100 test_size = 101  # number of chunks for the Chunk Preprocessors chunksize = 100 n_chunks = int(np.ceil(train_size / chunksize))  # path to the image dataset and name of the image col img_path = \"../tmp_data/airbnb/property_picture/\" img_col = \"id\"  # name of the text col text_col = \"description\"  # mane of the target target_col = \"yield\"  # definition of the categorical and continuous cols for the TabPreprocessor cat_embed_cols = [     \"host_listings_count\",     \"neighbourhood_cleansed\",     \"is_location_exact\",     \"property_type\",     \"room_type\",     \"accommodates\",     \"bathrooms\",     \"bedrooms\",     \"beds\",     \"guests_included\",     \"minimum_nights\",     \"instant_bookable\",     \"cancellation_policy\",     \"has_house_rules\",     \"host_gender\",     \"accommodates_catg\",     \"guests_included_catg\",     \"minimum_nights_catg\",     \"host_listings_count_catg\",     \"bathrooms_catg\",     \"bedrooms_catg\",     \"beds_catg\",     \"security_deposit\",     \"extra_people\", ] cont_cols = [\"latitude\", \"longitude\"] In\u00a0[6]: Copied! <pre>tab_preprocessor = TabPreprocessor(\n    embed_cols=cat_embed_cols,\n    continuous_cols=cont_cols,\n    default_embed_dim=8,\n    verbose=0,\n)\n\ntext_preprocessor = TextPreprocessor(\n    text_col=text_col,\n    n_cpus=1,\n)\n\nimg_preprocessor = ImagePreprocessor(\n    img_col=img_col,\n    img_path=img_path,\n)\n</pre> tab_preprocessor = TabPreprocessor(     embed_cols=cat_embed_cols,     continuous_cols=cont_cols,     default_embed_dim=8,     verbose=0, )  text_preprocessor = TextPreprocessor(     text_col=text_col,     n_cpus=1, )  img_preprocessor = ImagePreprocessor(     img_col=img_col,     img_path=img_path, ) In\u00a0[7]: Copied! <pre>tab_preprocessor.fit(airbnb_data)\ntext_preprocessor.fit(airbnb_data)\nimg_preprocessor.fit(airbnb_data)\n</pre> tab_preprocessor.fit(airbnb_data) text_preprocessor.fit(airbnb_data) img_preprocessor.fit(airbnb_data) <pre>The vocabulary contains 2192 tokens\n</pre> Out[7]: <pre>ImagePreprocessor(img_col=id, img_path=../tmp_data/airbnb/property_picture/, width=224, height=224, verbose=1)</pre> In\u00a0[8]: Copied! <pre>chunk_tab_preprocessor = ChunkTabPreprocessor(\n    embed_cols=cat_embed_cols,\n    continuous_cols=cont_cols,\n    n_chunks=n_chunks,\n    default_embed_dim=8,\n    verbose=0,\n)\n\nchunk_text_preprocessor = ChunkTextPreprocessor(\n    n_chunks=n_chunks,\n    text_col=text_col,\n    n_cpus=1,\n    verbose=0,\n)\n\nfor i, chunk in enumerate(\n    pd.read_csv(\"/\".join([data_path, train_fname]), chunksize=chunksize)\n):\n    print(f\"chunk in loop: {i + 1}\")\n    chunk_tab_preprocessor.fit(chunk)\n    chunk_text_preprocessor.fit(chunk)\n</pre> chunk_tab_preprocessor = ChunkTabPreprocessor(     embed_cols=cat_embed_cols,     continuous_cols=cont_cols,     n_chunks=n_chunks,     default_embed_dim=8,     verbose=0, )  chunk_text_preprocessor = ChunkTextPreprocessor(     n_chunks=n_chunks,     text_col=text_col,     n_cpus=1,     verbose=0, )  for i, chunk in enumerate(     pd.read_csv(\"/\".join([data_path, train_fname]), chunksize=chunksize) ):     print(f\"chunk in loop: {i + 1}\")     chunk_tab_preprocessor.fit(chunk)     chunk_text_preprocessor.fit(chunk) <pre>chunk in loop: 1\nchunk in loop: 2\nchunk in loop: 3\nchunk in loop: 4\nchunk in loop: 5\nchunk in loop: 6\nchunk in loop: 7\nchunk in loop: 8\n</pre> In\u00a0[9]: Copied! <pre>train_tab_folder = TabFromFolder(\n    fname=train_fname,\n    directory=data_path,\n    target_col=target_col,\n    preprocessor=tab_preprocessor,\n    text_col=text_col,\n    img_col=img_col,\n)\n\n# Note how we can use the `train_tab_folder` as reference so we don't have to\n# define all parameters again\neval_tab_folder = TabFromFolder(fname=eval_fname, reference=train_tab_folder)\n\n# Note that for the test set we can ignore the target as no metric will be\n# computed by the `predict` method\ntest_tab_folder = TabFromFolder(\n    fname=test_fname, reference=train_tab_folder, ignore_target=True\n)\n</pre> train_tab_folder = TabFromFolder(     fname=train_fname,     directory=data_path,     target_col=target_col,     preprocessor=tab_preprocessor,     text_col=text_col,     img_col=img_col, )  # Note how we can use the `train_tab_folder` as reference so we don't have to # define all parameters again eval_tab_folder = TabFromFolder(fname=eval_fname, reference=train_tab_folder)  # Note that for the test set we can ignore the target as no metric will be # computed by the `predict` method test_tab_folder = TabFromFolder(     fname=test_fname, reference=train_tab_folder, ignore_target=True ) In\u00a0[10]: Copied! <pre># for the text and image datasets we do not need to specify eval or test loaders\ntext_folder = TextFromFolder(preprocessor=text_preprocessor)\nimg_folder = ImageFromFolder(preprocessor=img_preprocessor)\n</pre> # for the text and image datasets we do not need to specify eval or test loaders text_folder = TextFromFolder(preprocessor=text_preprocessor) img_folder = ImageFromFolder(preprocessor=img_preprocessor) In\u00a0[11]: Copied! <pre>train_dataset_folder = WideDeepDatasetFromFolder(\n    n_samples=train_size,\n    tab_from_folder=train_tab_folder,\n    text_from_folder=text_folder,\n    img_from_folder=img_folder,\n)\n\n# Note that the eval and test loaders only need their corresponding\n# `TabFromFolder` classes. The rest of the parameters can be defined\n# via a `reference` `TabFromFolder` class\neval_dataset_folder = WideDeepDatasetFromFolder(\n    n_samples=eval_size,\n    tab_from_folder=eval_tab_folder,\n    reference=train_dataset_folder,\n)\n\ntest_dataset_folder = WideDeepDatasetFromFolder(\n    n_samples=test_size,\n    tab_from_folder=test_tab_folder,\n    reference=train_dataset_folder,\n)\n</pre> train_dataset_folder = WideDeepDatasetFromFolder(     n_samples=train_size,     tab_from_folder=train_tab_folder,     text_from_folder=text_folder,     img_from_folder=img_folder, )  # Note that the eval and test loaders only need their corresponding # `TabFromFolder` classes. The rest of the parameters can be defined # via a `reference` `TabFromFolder` class eval_dataset_folder = WideDeepDatasetFromFolder(     n_samples=eval_size,     tab_from_folder=eval_tab_folder,     reference=train_dataset_folder, )  test_dataset_folder = WideDeepDatasetFromFolder(     n_samples=test_size,     tab_from_folder=test_tab_folder,     reference=train_dataset_folder, ) In\u00a0[12]: Copied! <pre>train_loader = DataLoader(train_dataset_folder, batch_size=16, num_workers=1)\neval_loader = DataLoader(eval_dataset_folder, batch_size=16, num_workers=1)\ntest_loader = DataLoader(test_dataset_folder, batch_size=16, num_workers=1)\n</pre> train_loader = DataLoader(train_dataset_folder, batch_size=16, num_workers=1) eval_loader = DataLoader(eval_dataset_folder, batch_size=16, num_workers=1) test_loader = DataLoader(test_dataset_folder, batch_size=16, num_workers=1) <p>And from here on is business as usual:</p> In\u00a0[13]: Copied! <pre># for example\nbasic_rnn = BasicRNN(\n    vocab_size=len(text_preprocessor.vocab.itos),\n    embed_dim=32,\n    hidden_dim=64,\n    n_layers=2,\n)\n\ndeepimage = Vision()\n\ndeepdense = TabMlp(\n    mlp_hidden_dims=[32, 16],\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=cont_cols,\n)\n\nmodel = WideDeep(\n    deeptabular=deepdense,\n    deeptext=basic_rnn,\n    deepimage=deepimage,\n)\n\nmodel\n</pre> # for example basic_rnn = BasicRNN(     vocab_size=len(text_preprocessor.vocab.itos),     embed_dim=32,     hidden_dim=64,     n_layers=2, )  deepimage = Vision()  deepdense = TabMlp(     mlp_hidden_dims=[32, 16],     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     continuous_cols=cont_cols, )  model = WideDeep(     deeptabular=deepdense,     deeptext=basic_rnn,     deepimage=deepimage, )  model Out[13]: <pre>WideDeep(\n  (deeptabular): Sequential(\n    (0): TabMlp(\n      (cat_embed): DiffSizeCatEmbeddings(\n        (embed_layers): ModuleDict(\n          (emb_layer_host_listings_count): Embedding(28, 10, padding_idx=0)\n          (emb_layer_neighbourhood_cleansed): Embedding(33, 11, padding_idx=0)\n          (emb_layer_is_location_exact): Embedding(3, 2, padding_idx=0)\n          (emb_layer_property_type): Embedding(4, 3, padding_idx=0)\n          (emb_layer_room_type): Embedding(4, 3, padding_idx=0)\n          (emb_layer_accommodates): Embedding(14, 7, padding_idx=0)\n          (emb_layer_bathrooms): Embedding(11, 6, padding_idx=0)\n          (emb_layer_bedrooms): Embedding(7, 4, padding_idx=0)\n          (emb_layer_beds): Embedding(11, 6, padding_idx=0)\n          (emb_layer_guests_included): Embedding(11, 6, padding_idx=0)\n          (emb_layer_minimum_nights): Embedding(25, 9, padding_idx=0)\n          (emb_layer_instant_bookable): Embedding(3, 2, padding_idx=0)\n          (emb_layer_cancellation_policy): Embedding(6, 4, padding_idx=0)\n          (emb_layer_has_house_rules): Embedding(3, 2, padding_idx=0)\n          (emb_layer_host_gender): Embedding(4, 3, padding_idx=0)\n          (emb_layer_accommodates_catg): Embedding(4, 3, padding_idx=0)\n          (emb_layer_guests_included_catg): Embedding(4, 3, padding_idx=0)\n          (emb_layer_minimum_nights_catg): Embedding(4, 3, padding_idx=0)\n          (emb_layer_host_listings_count_catg): Embedding(5, 3, padding_idx=0)\n          (emb_layer_bathrooms_catg): Embedding(4, 3, padding_idx=0)\n          (emb_layer_bedrooms_catg): Embedding(5, 3, padding_idx=0)\n          (emb_layer_beds_catg): Embedding(5, 3, padding_idx=0)\n          (emb_layer_security_deposit): Embedding(53, 15, padding_idx=0)\n          (emb_layer_extra_people): Embedding(39, 12, padding_idx=0)\n        )\n        (embedding_dropout): Dropout(p=0.0, inplace=False)\n      )\n      (cont_norm): Identity()\n      (encoder): MLP(\n        (mlp): Sequential(\n          (dense_layer_0): Sequential(\n            (0): Linear(in_features=128, out_features=32, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_1): Sequential(\n            (0): Linear(in_features=32, out_features=16, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=16, out_features=1, bias=True)\n  )\n  (deeptext): Sequential(\n    (0): BasicRNN(\n      (word_embed): Embedding(2192, 32, padding_idx=1)\n      (rnn): LSTM(32, 64, num_layers=2, batch_first=True, dropout=0.1)\n      (rnn_mlp): Identity()\n    )\n    (1): Linear(in_features=64, out_features=1, bias=True)\n  )\n  (deepimage): Sequential(\n    (0): Vision(\n      (features): Sequential(\n        (conv_layer_0): Sequential(\n          (0): Conv2d(3, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False)\n          (1): BatchNorm2d(64, eps=1e-05, momentum=0.01, affine=True, track_running_stats=True)\n          (2): LeakyReLU(negative_slope=0.1, inplace=True)\n          (maxpool): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n        )\n        (conv_layer_1): Sequential(\n          (0): Conv2d(64, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n          (1): BatchNorm2d(128, eps=1e-05, momentum=0.01, affine=True, track_running_stats=True)\n          (2): LeakyReLU(negative_slope=0.1, inplace=True)\n        )\n        (conv_layer_2): Sequential(\n          (0): Conv2d(128, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n          (1): BatchNorm2d(256, eps=1e-05, momentum=0.01, affine=True, track_running_stats=True)\n          (2): LeakyReLU(negative_slope=0.1, inplace=True)\n        )\n        (conv_layer_3): Sequential(\n          (0): Conv2d(256, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n          (1): BatchNorm2d(512, eps=1e-05, momentum=0.01, affine=True, track_running_stats=True)\n          (2): LeakyReLU(negative_slope=0.1, inplace=True)\n          (adaptiveavgpool): AdaptiveAvgPool2d(output_size=(1, 1))\n        )\n      )\n    )\n    (1): Linear(in_features=512, out_features=1, bias=True)\n  )\n)</pre> In\u00a0[14]: Copied! <pre>trainer = TrainerFromFolder(\n    model,\n    objective=\"regression\",\n)\n\ntrainer.fit(\n    train_loader=train_loader,\n    eval_loader=eval_loader,\n)\n</pre> trainer = TrainerFromFolder(     model,     objective=\"regression\", )  trainer.fit(     train_loader=train_loader,     eval_loader=eval_loader, ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 50/50 [03:41&lt;00:00,  4.42s/it, loss=1.64e+4]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 7/7 [00:23&lt;00:00,  3.30s/it, loss=6.27e+3]\n</pre> In\u00a0[15]: Copied! <pre>preds = trainer.predict(test_loader=test_loader)\n</pre> preds = trainer.predict(test_loader=test_loader) <pre>predict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 7/7 [00:22&lt;00:00,  3.26s/it]\n</pre> <p>Note that in the case of predict you could also choose to do this</p> In\u00a0[16]: Copied! <pre>df_test = pd.read_csv(\"/\".join([data_path, test_fname]))\n</pre> df_test = pd.read_csv(\"/\".join([data_path, test_fname])) In\u00a0[17]: Copied! <pre># if the images for the test set fit in memory\nX_tab_test = chunk_tab_preprocessor.transform(df_test)\nX_text_test = chunk_text_preprocessor.transform(df_test)\nX_img_test = img_preprocessor.transform(df_test)\n</pre> # if the images for the test set fit in memory X_tab_test = chunk_tab_preprocessor.transform(df_test) X_text_test = chunk_text_preprocessor.transform(df_test) X_img_test = img_preprocessor.transform(df_test) <pre>Reading Images from ../tmp_data/airbnb/property_picture/\nResizing\n</pre> <pre>100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 101/101 [00:00&lt;00:00, 708.23it/s]</pre> <pre>Computing normalisation metrics\n</pre> <pre>\n</pre> In\u00a0[18]: Copied! <pre>preds = trainer.predict(\n    X_tab=X_tab_test, X_text=X_text_test, X_img=X_img_test, batch_size=32\n)\n</pre> preds = trainer.predict(     X_tab=X_tab_test, X_text=X_text_test, X_img=X_img_test, batch_size=32 ) <pre>predict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 4/4 [00:03&lt;00:00,  1.14it/s]\n</pre>"},{"location":"examples/19_load_from_folder_functionality.html#scenario","title":"Scenario\u00b6","text":"<p>We have a tabular dataset combined with images and text and either some, or all these datasets do not fit in memory. Note that the tabular dataset MUST ALWAYS be present as it is considered the rerefence. This is, if we have an image dataset, the tabular dataset must contain a column that points to the image file names as stored in disk. Similarly, if we have a text dataset, then the tabular dataset must contain a column with the texts themselves or a the file names of the text files as stored in disk.</p> <p>If you only have text and/or images and not a tabular component, I would suggest using other libraries (such as hugginface probably).</p> <p>Within this scenario, they are two possible scenarios that we will cover here:</p> <ol> <li><p>The tabular data itsel fits in memory and is only the images that do not: in this case you could use the 'standard' <code>Preprocessors</code> in the library and off you go, move directly to the <code>[...]FromFolder</code> functionalities</p> </li> <li><p>The tabular data is also very large and does not fit in memory, so we have to process it in chuncks. For this second case I have created the so called <code>Chunk[...]Preprocessor</code> (Wide, Tab and Text).</p> </li> </ol> <p>Note that at the moment ONLY csv format is allowed for the tabular file. More formats will be supported in the future.</p> <p>Let's see a complete example to illustrate how each of these cases would be addressed with the new functionalities in the library. For this example we will use a sample of the airbnb dataset</p> <p>The airbnb dataset, which you could get from here, is too big to be included in our datasets module (when including images). Therefore, what I did was, go there, download it, and use the download_images.py script to get the images and the airbnb_data_processing.py to process the data. I did this ages ago and I believe the format of the dataset might be different now. Nonetheless, I will show samples of the dataset as we go through so you can extrapolate the content of this notebook to your particular problem.</p> <p>In the future we will find better datasets\ud83d\ude42. Finally, note that here we are only using a small sample to illustrate the use, so PLEASE ignore the results, just focus on usage.</p>"},{"location":"examples/19_load_from_folder_functionality.html#setting-variables-and-constants","title":"Setting variables and constants\u00b6","text":""},{"location":"examples/19_load_from_folder_functionality.html#step-1-the-preprocessors","title":"Step 1: the preprocessors\u00b6","text":""},{"location":"examples/19_load_from_folder_functionality.html#scenario-1-only-the-images-do-not-fit-in-disk","title":"Scenario 1: only the images do not fit in disk\u00b6","text":"<p>In this case we can prepare the data in the 'standard' way</p>"},{"location":"examples/19_load_from_folder_functionality.html#scenario-2-the-tabular-data-is-also-huge","title":"Scenario 2: the tabular data is also huge\u00b6","text":"<p>Then we need to prepare it in chunks. Note that, unfortunately, the tabular and text preprocessors need to see the whole dataset once. This is because to process tabular or text data we need to encode values. For those encodings to be consistent they need to have seen the whole dataset. Alternatively, one could code a solution with some streaming encoder for both datasets. However, such implementation is not trivial for this library (and in general). I also don't think that having to see the whole data once is such a big limitation. Let's see how is done.</p> <p>Note that I have not mentioned the image dataset. This is because the processing of the image dataset does not require any form of encoding and in consequence can be done 'on the fly'. Therefore, no <code>ChunkImgPreprocessor</code> processor is needed.</p>"},{"location":"examples/19_load_from_folder_functionality.html#step-2-the-fromfolder-classes","title":"Step 2: the <code>[...]FromFolder</code> classes\u00b6","text":"<p>Once we have the preprocessors, we need to instantiate the classes that will enable us to load the data from their respective folders. From now on I am going to proceed with the <code>chunk_tab_preprocessor</code>, <code>chunk_text_preprocessor</code> and <code>img_preprocessor</code>, but the code would be identical if instead of the first two preprocessors we decided to use the <code>tab_preprocessor</code> and <code>text_preprocessor</code>.</p> <p>Once more, our reference datasets are the tabular datasets, which we have splitted in train, eval and test prior to start the coding. Therefore, we will eventually need a loader for each split</p>"},{"location":"examples/19_load_from_folder_functionality.html#step-3-pytorch-datasets-and-dataloaders","title":"Step 3: pytorch datasets and dataloaders\u00b6","text":"<p>From here in advance, is all very 'standard' if you are familiar with pytorch. One needs to define a class that inherits from the <code>Dataset</code> class in pytorch. Then this will be passed to a <code>DataLoader</code> class and we are ready to train. Our <code>Dataset</code> child class is <code>WideDeepDatasetFromFolder</code>. This class will use the tabular dataset and the corresponding text and image columns to load the adequate data in the batches</p> <p>Let's do it</p>"},{"location":"examples/19_load_from_folder_functionality.html#step-4-define-the-model","title":"Step 4: define the model\u00b6","text":""},{"location":"examples/19_load_from_folder_functionality.html#step-5-fit-and-predict","title":"Step 5: fit and predict\u00b6","text":""},{"location":"examples/20_Using_huggingface_within_widedeep.html","title":"20-Using-huggingface-within-widedeep","text":"<p>In this notebook we will show how to use Hugginface's tokenizers and models as they are integrated within the library. In notebook number 17 you can find examples on how to code your own, custom, Hugginface (hereafter HF) model and use it in combination of any other model in the library</p> In\u00a0[1]: Copied! <pre>import numpy as np\nimport pandas as pd\nfrom sklearn.metrics import f1_score, accuracy_score\nfrom sklearn.model_selection import train_test_split\n\nfrom pytorch_widedeep import Trainer\nfrom pytorch_widedeep.models import HFModel, WideDeep\nfrom pytorch_widedeep.metrics import F1Score, Accuracy\nfrom pytorch_widedeep.datasets import load_womens_ecommerce\nfrom pytorch_widedeep.preprocessing import HFPreprocessor\n</pre> import numpy as np import pandas as pd from sklearn.metrics import f1_score, accuracy_score from sklearn.model_selection import train_test_split  from pytorch_widedeep import Trainer from pytorch_widedeep.models import HFModel, WideDeep from pytorch_widedeep.metrics import F1Score, Accuracy from pytorch_widedeep.datasets import load_womens_ecommerce from pytorch_widedeep.preprocessing import HFPreprocessor <pre>/Users/javierrodriguezzaurin/.pyenv/versions/3.10.13/envs/widedeep310/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n  from .autonotebook import tqdm as notebook_tqdm\n</pre> In\u00a0[2]: Copied! <pre>df: pd.DataFrame = load_womens_ecommerce(as_frame=True)  # type: ignore\n</pre> df: pd.DataFrame = load_womens_ecommerce(as_frame=True)  # type: ignore In\u00a0[3]: Copied! <pre>df.shape\n</pre> df.shape Out[3]: <pre>(23486, 10)</pre> In\u00a0[4]: Copied! <pre>df.sample(3)\n</pre> df.sample(3) Out[4]: Clothing ID Age Title Review Text Rating Recommended IND Positive Feedback Count Division Name Department Name Class Name 7004 862 43 Cute and feminine Loved this sweater wrap and bought it in both ... 5 1 2 General Tops Knits 12508 975 66 Love it The linen fabric is elegantly thin feels and l... 5 1 3 General Jackets Jackets 10288 950 41 Perfect for fall This sweater is just as pictured. the fit is t... 5 1 0 General Tops Sweaters In\u00a0[5]: Copied! <pre># Let's do some mild preprocessing\ndf.columns = [c.replace(\" \", \"_\").lower() for c in df.columns]\n\n# classes from [0,num_class)\ndf[\"rating\"] = (df[\"rating\"] - 1).astype(\"int64\")\n\n# group reviews with 1 and 2 scores into one class\ndf.loc[df.rating == 0, \"rating\"] = 1\n\n# and back again to [0,num_class)\ndf[\"rating\"] = (df[\"rating\"] - 1).astype(\"int64\")\n</pre> # Let's do some mild preprocessing df.columns = [c.replace(\" \", \"_\").lower() for c in df.columns]  # classes from [0,num_class) df[\"rating\"] = (df[\"rating\"] - 1).astype(\"int64\")  # group reviews with 1 and 2 scores into one class df.loc[df.rating == 0, \"rating\"] = 1  # and back again to [0,num_class) df[\"rating\"] = (df[\"rating\"] - 1).astype(\"int64\") In\u00a0[6]: Copied! <pre># drop short reviews\ndf = df[~df.review_text.isna()]\ndf[\"review_length\"] = df.review_text.apply(lambda x: len(x.split(\" \")))\ndf = df[df.review_length &gt;= 5]\ndf = df.drop(\"review_length\", axis=1).reset_index(drop=True)\n</pre> # drop short reviews df = df[~df.review_text.isna()] df[\"review_length\"] = df.review_text.apply(lambda x: len(x.split(\" \"))) df = df[df.review_length &gt;= 5] df = df.drop(\"review_length\", axis=1).reset_index(drop=True) In\u00a0[7]: Copied! <pre>df.shape\n</pre> df.shape Out[7]: <pre>(22608, 10)</pre> In\u00a0[8]: Copied! <pre># if you run this on a CPU, you might want to subsample the dataset. With that in mind I am simply going to stratify-sample to the minimum category occurrence and then sample at random\n# If you run this on a GPU you can comment out the following two cells\ndf.rating.value_counts()\n</pre> # if you run this on a CPU, you might want to subsample the dataset. With that in mind I am simply going to stratify-sample to the minimum category occurrence and then sample at random # If you run this on a GPU you can comment out the following two cells df.rating.value_counts() Out[8]: <pre>rating\n3    12515\n2     4904\n1     2820\n0     2369\nName: count, dtype: int64</pre> In\u00a0[9]: Copied! <pre>df = (\n    df.groupby(\"rating\", group_keys=False)\n    .apply(lambda x: x.sample(min(len(x), 2369)))\n    .sample(1000)\n)\n</pre> df = (     df.groupby(\"rating\", group_keys=False)     .apply(lambda x: x.sample(min(len(x), 2369)))     .sample(1000) ) <pre>/var/folders/_2/lrjn1qn54c758tdtktr1bvkc0000gn/T/ipykernel_5886/895673206.py:3: DeprecationWarning: DataFrameGroupBy.apply operated on the grouping columns. This behavior is deprecated, and in a future version of pandas the grouping columns will be excluded from the operation. Either pass `include_groups=False` to exclude the groupings or explicitly select the grouping columns after groupby to silence this warning.\n  .apply(lambda x: x.sample(min(len(x), 2369)))\n</pre> In\u00a0[10]: Copied! <pre>train, test = train_test_split(df, train_size=0.8, random_state=1, stratify=df.rating)\n\n# possible model names currently supported in the library\nmodel_names = [\n    \"distilbert-base-uncased\",\n    \"bert-base-uncased\",\n    \"FacebookAI/roberta-base\",\n    \"albert-base-v2\",\n    \"google/electra-base-discriminator\",\n]\n\n# Let's choose one. The syntax is the same for all the models\nmodel_name = \"distilbert-base-uncased\"\n</pre> train, test = train_test_split(df, train_size=0.8, random_state=1, stratify=df.rating)  # possible model names currently supported in the library model_names = [     \"distilbert-base-uncased\",     \"bert-base-uncased\",     \"FacebookAI/roberta-base\",     \"albert-base-v2\",     \"google/electra-base-discriminator\", ]  # Let's choose one. The syntax is the same for all the models model_name = \"distilbert-base-uncased\" <p>Now we can use the <code>HFPreprocessor</code> class. As most things in this library, the integration with HF has been coded aiming for a flexible use. With this in mind, there are two ways one can use a <code>HFPreprocessor</code> class.</p> <ol> <li>Passing a <code>text_col</code> and <code>encode_params</code> as the class is instantiated and then using the <code>fit</code> and <code>transform</code> as with any other preprocessor in the library</li> <li>Without passing  <code>text_col</code> and <code>encode_params</code> as the class is instantiated and using the <code>encode</code> method of the <code>HFPreprocessor</code> which is simply a wrapper around the encode method of HF's tokenizers</li> </ol> <p>Let's have a look</p> In\u00a0[11]: Copied! <pre>tokenizer1 = HFPreprocessor(\n    model_name=model_name,\n    text_col=\"review_text\",\n    num_workers=1,\n    encode_params={\n        \"max_length\": 90,\n        \"padding\": \"max_length\",\n        \"truncation\": True,\n        \"add_special_tokens\": True,\n    },\n)\n\nX_text_tr1 = tokenizer1.fit_transform(train)\nX_text_te1 = tokenizer1.transform(test)\n</pre> tokenizer1 = HFPreprocessor(     model_name=model_name,     text_col=\"review_text\",     num_workers=1,     encode_params={         \"max_length\": 90,         \"padding\": \"max_length\",         \"truncation\": True,         \"add_special_tokens\": True,     }, )  X_text_tr1 = tokenizer1.fit_transform(train) X_text_te1 = tokenizer1.transform(test) In\u00a0[12]: Copied! <pre>tokenizer2 = HFPreprocessor(\n    model_name=model_name,\n    num_workers=1,\n)\n\nX_text_tr2 = tokenizer2.encode(\n    train.review_text.tolist(),\n    max_length=90,\n    padding=\"max_length\",\n    truncation=True,\n    add_special_tokens=True,\n)\nX_text_te2 = tokenizer2.encode(\n    test.review_text.tolist(),\n    max_length=90,\n    padding=\"max_length\",\n    truncation=True,\n    add_special_tokens=True,\n)\n</pre> tokenizer2 = HFPreprocessor(     model_name=model_name,     num_workers=1, )  X_text_tr2 = tokenizer2.encode(     train.review_text.tolist(),     max_length=90,     padding=\"max_length\",     truncation=True,     add_special_tokens=True, ) X_text_te2 = tokenizer2.encode(     test.review_text.tolist(),     max_length=90,     padding=\"max_length\",     truncation=True,     add_special_tokens=True, ) In\u00a0[13]: Copied! <pre>all(X_text_tr1[0] == X_text_tr2[0])\n</pre> all(X_text_tr1[0] == X_text_tr2[0]) Out[13]: <pre>True</pre> In\u00a0[14]: Copied! <pre># Now we define a model which is as easy as:\n# Note that this will instantiation will lead to NO parameter trainable in the HF model.\n# If you want to fine-tune the HF model, you can set the trainable parameters via the 'trainable_parameters' argument.\n# Alternatively, you can use a head (MLP) via the 'head'-related arguments (see the docs for more details)\nhf_model = HFModel(model_name=model_name)\n</pre> # Now we define a model which is as easy as: # Note that this will instantiation will lead to NO parameter trainable in the HF model. # If you want to fine-tune the HF model, you can set the trainable parameters via the 'trainable_parameters' argument. # Alternatively, you can use a head (MLP) via the 'head'-related arguments (see the docs for more details) hf_model = HFModel(model_name=model_name) In\u00a0[15]: Copied! <pre># And from here on is the same as any other WideDeep model\nmodel = WideDeep(\n    deeptext=hf_model,\n    pred_dim=4,\n)\n\ntrainer = Trainer(\n    model,\n    objective=\"multiclass\",\n    metrics=[Accuracy(), F1Score(average=True)],\n)\n\ntrainer.fit(\n    X_text=X_text_tr2,\n    target=train.rating.values,\n    n_epochs=1,\n    batch_size=64,\n)\n# If you run this on a CPU and you sampled the data, the metrics will not be better than a random guess. Remember, this is just a demo\n</pre> # And from here on is the same as any other WideDeep model model = WideDeep(     deeptext=hf_model,     pred_dim=4, )  trainer = Trainer(     model,     objective=\"multiclass\",     metrics=[Accuracy(), F1Score(average=True)], )  trainer.fit(     X_text=X_text_tr2,     target=train.rating.values,     n_epochs=1,     batch_size=64, ) # If you run this on a CPU and you sampled the data, the metrics will not be better than a random guess. Remember, this is just a demo <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 13/13 [02:06&lt;00:00,  9.75s/it, loss=3.2, metrics={'acc': 0.235, 'f1': 0.2336}]  \n</pre> In\u00a0[17]: Copied! <pre>preds_text = trainer.predict_proba(X_text=X_text_te2)\npred_text_class = np.argmax(preds_text, 1)\n\nacc_text = accuracy_score(test.rating, pred_text_class)\nf1_text = f1_score(test.rating, pred_text_class, average=\"weighted\")\nprint(f\"Accuracy: {acc_text:.4f}\")\nprint(f\"F1: {f1_text:.4f}\")\n</pre> preds_text = trainer.predict_proba(X_text=X_text_te2) pred_text_class = np.argmax(preds_text, 1)  acc_text = accuracy_score(test.rating, pred_text_class) f1_text = f1_score(test.rating, pred_text_class, average=\"weighted\") print(f\"Accuracy: {acc_text:.4f}\") print(f\"F1: {f1_text:.4f}\") <pre>predict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 4/4 [00:05&lt;00:00,  1.43s/it]</pre> <pre>Accuracy: 0.2500\nF1: 0.1000\n</pre> <pre>\n</pre>"},{"location":"pytorch-widedeep/bayesian_models.html","title":"The <code>bayesian models</code> module","text":"<p>This module contains the two Bayesian Models available in this library, namely the bayesian version of the <code>Wide</code> and <code>TabMlp</code> models, referred as <code>BayesianWide</code> and <code>BayesianTabMlp</code>. These models are very useful in scenarios where getting a measure of uncertainty is important.</p> <p>The models in this module are based on the publication: Weight Uncertainty in Neural Networks.</p>"},{"location":"pytorch-widedeep/bayesian_models.html#pytorch_widedeep.bayesian_models.tabular.bayesian_linear.bayesian_wide.BayesianWide","title":"BayesianWide","text":"<p>               Bases: <code>BaseBayesianModel</code></p> <p>Defines a <code>Wide</code> model. This is a linear model where the non-linearlities are captured via crossed-columns</p> <p>Parameters:</p> Name Type Description Default <code>input_dim</code> <code>int</code> <p>size of the Embedding layer. <code>input_dim</code> is the summation of all the individual values for all the features that go through the wide component. For example, if the wide component receives 2 features with 5 individual values each, <code>input_dim = 10</code></p> required <code>pred_dim</code> <code>int</code> <p>size of the ouput tensor containing the predictions</p> <code>1</code> <code>prior_sigma_1</code> <code>float</code> <p>The prior weight distribution is a scaled mixture of two Gaussian densities:</p> \\[    \\begin{aligned}    P(\\mathbf{w}) = \\prod_{i=j} \\pi N (\\mathbf{w}_j | 0, \\sigma_{1}^{2}) + (1 - \\pi) N (\\mathbf{w}_j | 0, \\sigma_{2}^{2})    \\end{aligned} \\] <p><code>prior_sigma_1</code> is the prior of the sigma parameter for the first of the two Gaussians that will be mixed to produce the prior weight distribution.</p> <code>1.0</code> <code>prior_sigma_2</code> <code>float</code> <p>Prior of the sigma parameter for the second of the two Gaussian distributions that will be mixed to produce the prior weight distribution</p> <code>0.002</code> <code>prior_pi</code> <code>float</code> <p>Scaling factor that will be used to mix the Gaussians to produce the prior weight distribution</p> <code>0.8</code> <code>posterior_mu_init</code> <code>float</code> <p>The posterior sample of the weights is defined as:</p> \\[    \\begin{aligned}    \\mathbf{w} &amp;= \\mu + log(1 + exp(\\rho))    \\end{aligned} \\] <p>where:</p> \\[    \\begin{aligned}    \\mathcal{N}(x\\vert \\mu, \\sigma) &amp;= \\frac{1}{\\sqrt{2\\pi}\\sigma}e^{-\\frac{(x-\\mu)^2}{2\\sigma^2}}\\\\    \\log{\\mathcal{N}(x\\vert \\mu, \\sigma)} &amp;= -\\log{\\sqrt{2\\pi}} -\\log{\\sigma} -\\frac{(x-\\mu)^2}{2\\sigma^2}\\\\    \\end{aligned} \\] <p>\\(\\mu\\) is initialised using a normal distributtion with mean <code>posterior_mu_init</code> and std equal to 0.1.</p> <code>0.0</code> <code>posterior_rho_init</code> <code>float</code> <p>As in the case of \\(\\mu\\), \\(\\rho\\) is initialised using a normal distributtion with mean <code>posterior_rho_init</code> and std equal to 0.1.</p> <code>-7.0</code> <p>Attributes:</p> Name Type Description <code>bayesian_wide_linear</code> <code>Module</code> <p>the linear layer that comprises the wide branch of the model</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.bayesian_models import BayesianWide\n&gt;&gt;&gt; X = torch.empty(4, 4).random_(6)\n&gt;&gt;&gt; wide = BayesianWide(input_dim=int(X.max().item()), pred_dim=1)\n&gt;&gt;&gt; out = wide(X)\n</code></pre> Source code in <code>pytorch_widedeep/bayesian_models/tabular/bayesian_linear/bayesian_wide.py</code> <pre><code>class BayesianWide(BaseBayesianModel):\n    r\"\"\"Defines a `Wide` model. This is a linear model where the\n    non-linearlities are captured via crossed-columns\n\n    Parameters\n    ----------\n    input_dim: int\n        size of the Embedding layer. `input_dim` is the summation of all the\n        individual values for all the features that go through the wide\n        component. For example, if the wide component receives 2 features with\n        5 individual values each, `input_dim = 10`\n    pred_dim: int\n        size of the ouput tensor containing the predictions\n    prior_sigma_1: float, default = 1.0\n        The prior weight distribution is a scaled mixture of two Gaussian\n        densities:\n\n        $$\n           \\begin{aligned}\n           P(\\mathbf{w}) = \\prod_{i=j} \\pi N (\\mathbf{w}_j | 0, \\sigma_{1}^{2}) + (1 - \\pi) N (\\mathbf{w}_j | 0, \\sigma_{2}^{2})\n           \\end{aligned}\n        $$\n\n        `prior_sigma_1` is the prior of the sigma parameter for the first of the two\n        Gaussians that will be mixed to produce the prior weight\n        distribution.\n    prior_sigma_2: float, default = 0.002\n        Prior of the sigma parameter for the second of the two Gaussian\n        distributions that will be mixed to produce the prior weight\n        distribution\n    prior_pi: float, default = 0.8\n        Scaling factor that will be used to mix the Gaussians to produce the\n        prior weight distribution\n    posterior_mu_init: float = 0.0\n        The posterior sample of the weights is defined as:\n\n        $$\n           \\begin{aligned}\n           \\mathbf{w} &amp;= \\mu + log(1 + exp(\\rho))\n           \\end{aligned}\n        $$\n\n        where:\n\n        $$\n           \\begin{aligned}\n           \\mathcal{N}(x\\vert \\mu, \\sigma) &amp;= \\frac{1}{\\sqrt{2\\pi}\\sigma}e^{-\\frac{(x-\\mu)^2}{2\\sigma^2}}\\\\\n           \\log{\\mathcal{N}(x\\vert \\mu, \\sigma)} &amp;= -\\log{\\sqrt{2\\pi}} -\\log{\\sigma} -\\frac{(x-\\mu)^2}{2\\sigma^2}\\\\\n           \\end{aligned}\n        $$\n\n        $\\mu$ is initialised using a normal distributtion with mean\n        `posterior_mu_init` and std equal to 0.1.\n    posterior_rho_init: float = -7.0\n        As in the case of $\\mu$, $\\rho$ is initialised using a\n        normal distributtion with mean `posterior_rho_init` and std equal to\n        0.1.\n\n    Attributes\n    -----------\n    bayesian_wide_linear: nn.Module\n        the linear layer that comprises the wide branch of the model\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.bayesian_models import BayesianWide\n    &gt;&gt;&gt; X = torch.empty(4, 4).random_(6)\n    &gt;&gt;&gt; wide = BayesianWide(input_dim=int(X.max().item()), pred_dim=1)\n    &gt;&gt;&gt; out = wide(X)\n    \"\"\"\n\n    def __init__(\n        self,\n        input_dim: int,\n        pred_dim: int = 1,\n        prior_sigma_1: float = 1.0,\n        prior_sigma_2: float = 0.002,\n        prior_pi: float = 0.8,\n        posterior_mu_init: float = 0.0,\n        posterior_rho_init: float = -7.0,\n    ):\n        super(BayesianWide, self).__init__()\n        #  Embeddings: val + 1 because 0 is reserved for padding/unseen cateogories.\n        self.bayesian_wide_linear = bnn.BayesianEmbedding(\n            n_embed=input_dim + 1,\n            embed_dim=pred_dim,\n            padding_idx=0,\n            prior_sigma_1=prior_sigma_1,\n            prior_sigma_2=prior_sigma_2,\n            prior_pi=prior_pi,\n            posterior_mu_init=posterior_mu_init,\n            posterior_rho_init=posterior_rho_init,\n        )\n        self.bias = nn.Parameter(torch.zeros(pred_dim))\n\n    def forward(self, X: Tensor) -&gt; Tensor:\n        out = self.bayesian_wide_linear(X.long()).sum(dim=1) + self.bias\n        return out\n</code></pre>"},{"location":"pytorch-widedeep/bayesian_models.html#pytorch_widedeep.bayesian_models.tabular.bayesian_mlp.bayesian_tab_mlp.BayesianTabMlp","title":"BayesianTabMlp","text":"<p>               Bases: <code>BaseBayesianModel</code></p> <p>Defines a <code>BayesianTabMlp</code> model.</p> <p>This class combines embedding representations of the categorical features with numerical (aka continuous) features, embedded or not. These are then passed through a series of probabilistic dense layers (i.e. a MLP).</p> <p>Parameters:</p> Name Type Description Default <code>column_idx</code> <code>Dict[str, int]</code> <p>Dict containing the index of the columns that will be passed through the <code>TabMlp</code> model. Required to slice the tensors. e.g. {'education': 0, 'relationship': 1, 'workclass': 2, ...}</p> required <code>cat_embed_input</code> <code>Optional[List[Tuple[str, int, int]]]</code> <p>List of Tuples with the column name, number of unique values and embedding dimension. e.g. [(education, 11, 32), ...]</p> <code>None</code> <code>cat_embed_activation</code> <code>Optional[str]</code> <p>Activation function for the categorical embeddings, if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> <code>None</code> <code>continuous_cols</code> <code>Optional[List[str]]</code> <p>List with the name of the numeric (aka continuous) columns</p> <code>None</code> <code>cont_norm_layer</code> <code>Optional[Literal[batchnorm, layernorm]]</code> <p>Type of normalization layer applied to the continuous features. Options are: 'layernorm', 'batchnorm' or None.</p> <code>None</code> <code>embed_continuous</code> <code>Optional[bool]</code> <p>Boolean indicating if the continuous columns will be embedded (i.e. passed each through a linear layer with or without activation)</p> <code>None</code> <code>cont_embed_dim</code> <code>Optional[int]</code> <p>Size of the continuous embeddings</p> <code>None</code> <code>cont_embed_dropout</code> <code>Optional[float]</code> <p>Dropout for the continuous embeddings</p> <code>None</code> <code>use_cont_bias</code> <code>Optional[bool]</code> <p>Boolean indicating if bias will be used for the continuous embeddings</p> <code>None</code> <code>cont_embed_activation</code> <code>Optional[str]</code> <p>Activation function for the continuous embeddings if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> <code>None</code> <code>mlp_hidden_dims</code> <code>List[int]</code> <p>List with the number of neurons per dense layer in the mlp.</p> <code>[200, 100]</code> <code>mlp_activation</code> <code>str</code> <p>Activation function for the dense layers of the MLP. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> <code>'leaky_relu'</code> <code>prior_sigma_1</code> <code>float</code> <p>The prior weight distribution is a scaled mixture of two Gaussian densities:</p> \\[    \\begin{aligned}    P(\\mathbf{w}) = \\prod_{i=j} \\pi N (\\mathbf{w}_j | 0, \\sigma_{1}^{2}) + (1 - \\pi) N (\\mathbf{w}_j | 0, \\sigma_{2}^{2})    \\end{aligned} \\] <p><code>prior_sigma_1</code> is the prior of the sigma parameter for the first of the two Gaussians that will be mixed to produce the prior weight distribution.</p> <code>1</code> <code>prior_sigma_2</code> <code>float</code> <p>Prior of the sigma parameter for the second of the two Gaussian distributions that will be mixed to produce the prior weight distribution for each Bayesian linear and embedding layer</p> <code>0.002</code> <code>prior_pi</code> <code>float</code> <p>Scaling factor that will be used to mix the Gaussians to produce the prior weight distribution ffor each Bayesian linear and embedding layer</p> <code>0.8</code> <code>posterior_mu_init</code> <code>float</code> <p>The posterior sample of the weights is defined as:</p> <p>$$    \\begin{aligned}    \\mathbf{w} &amp;= \\mu + log(1 + exp(\\rho))    \\end{aligned} $$ where:</p> \\[    \\begin{aligned}    \\mathcal{N}(x\\vert \\mu, \\sigma) &amp;= \\frac{1}{\\sqrt{2\\pi}\\sigma}e^{-\\frac{(x-\\mu)^2}{2\\sigma^2}}\\\\    \\log{\\mathcal{N}(x\\vert \\mu, \\sigma)} &amp;= -\\log{\\sqrt{2\\pi}} -\\log{\\sigma} -\\frac{(x-\\mu)^2}{2\\sigma^2}\\\\    \\end{aligned} \\] <p>\\(\\mu\\) is initialised using a normal distributtion with mean <code>posterior_mu_init</code> and std equal to 0.1.</p> <code>0.0</code> <code>posterior_rho_init</code> <code>float</code> <p>As in the case of \\(\\mu\\), \\(\\rho\\) is initialised using a normal distributtion with mean <code>posterior_rho_init</code> and std equal to 0.1.</p> <code>-7.0</code> <p>Attributes:</p> Name Type Description <code>bayesian_cat_and_cont_embed</code> <code>Module</code> <p>This is the module that processes the categorical and continuous columns</p> <code>bayesian_tab_mlp</code> <code>Sequential</code> <p>mlp model that will receive the concatenation of the embeddings and the continuous columns</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.bayesian_models import BayesianTabMlp\n&gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n&gt;&gt;&gt; colnames = ['a', 'b', 'c', 'd', 'e']\n&gt;&gt;&gt; cat_embed_input = [(u,i,j) for u,i,j in zip(colnames[:4], [4]*4, [8]*4)]\n&gt;&gt;&gt; column_idx = {k:v for v,k in enumerate(colnames)}\n&gt;&gt;&gt; model = BayesianTabMlp(mlp_hidden_dims=[8,4], column_idx=column_idx, cat_embed_input=cat_embed_input,\n... continuous_cols = ['e'])\n&gt;&gt;&gt; out = model(X_tab)\n</code></pre> Source code in <code>pytorch_widedeep/bayesian_models/tabular/bayesian_mlp/bayesian_tab_mlp.py</code> <pre><code>class BayesianTabMlp(BaseBayesianModel):\n    r\"\"\"Defines a `BayesianTabMlp` model.\n\n    This class combines embedding representations of the categorical features\n    with numerical (aka continuous) features, embedded or not. These are then\n    passed through a series of probabilistic dense layers (i.e. a MLP).\n\n    Parameters\n    ----------\n    column_idx: Dict\n        Dict containing the index of the columns that will be passed through\n        the `TabMlp` model. Required to slice the tensors. e.g. _{'education':\n        0, 'relationship': 1, 'workclass': 2, ...}_\n    cat_embed_input: List, Optional, default = None\n        List of Tuples with the column name, number of unique values and\n        embedding dimension. e.g. _[(education, 11, 32), ...]_\n    cat_embed_activation: Optional, str, default = None,\n        Activation function for the categorical embeddings, if any. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported\n    continuous_cols: List, Optional, default = None\n        List with the name of the numeric (aka continuous) columns\n    cont_norm_layer: str, default =  \"batchnorm\"\n        Type of normalization layer applied to the continuous features. Options\n        are: 'layernorm', 'batchnorm' or None.\n    embed_continuous: bool, default = False,\n        Boolean indicating if the continuous columns will be embedded\n        (i.e. passed each through a linear layer with or without activation)\n    cont_embed_dim: int, default = 32,\n        Size of the continuous embeddings\n    cont_embed_dropout: float, default = 0.1,\n        Dropout for the continuous embeddings\n    use_cont_bias: bool, default = True,\n        Boolean indicating if bias will be used for the continuous embeddings\n    cont_embed_activation: Optional, str, default = None,\n        Activation function for the continuous embeddings if any. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported\n    mlp_hidden_dims: List, default = [200, 100]\n        List with the number of neurons per dense layer in the mlp.\n    mlp_activation: str, default = \"relu\"\n        Activation function for the dense layers of the MLP. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported\n    prior_sigma_1: float, default = 1.0\n        The prior weight distribution is a scaled mixture of two Gaussian\n        densities:\n\n        $$\n           \\begin{aligned}\n           P(\\mathbf{w}) = \\prod_{i=j} \\pi N (\\mathbf{w}_j | 0, \\sigma_{1}^{2}) + (1 - \\pi) N (\\mathbf{w}_j | 0, \\sigma_{2}^{2})\n           \\end{aligned}\n        $$\n\n        `prior_sigma_1` is the prior of the sigma parameter for the first of the two\n        Gaussians that will be mixed to produce the prior weight\n        distribution.\n    prior_sigma_2: float, default = 0.002\n        Prior of the sigma parameter for the second of the two Gaussian\n        distributions that will be mixed to produce the prior weight\n        distribution for each Bayesian linear and embedding layer\n    prior_pi: float, default = 0.8\n        Scaling factor that will be used to mix the Gaussians to produce the\n        prior weight distribution ffor each Bayesian linear and embedding\n        layer\n    posterior_mu_init: float = 0.0\n        The posterior sample of the weights is defined as:\n\n        $$\n           \\begin{aligned}\n           \\mathbf{w} &amp;= \\mu + log(1 + exp(\\rho))\n           \\end{aligned}\n        $$\n        where:\n\n        $$\n           \\begin{aligned}\n           \\mathcal{N}(x\\vert \\mu, \\sigma) &amp;= \\frac{1}{\\sqrt{2\\pi}\\sigma}e^{-\\frac{(x-\\mu)^2}{2\\sigma^2}}\\\\\n           \\log{\\mathcal{N}(x\\vert \\mu, \\sigma)} &amp;= -\\log{\\sqrt{2\\pi}} -\\log{\\sigma} -\\frac{(x-\\mu)^2}{2\\sigma^2}\\\\\n           \\end{aligned}\n        $$\n\n        $\\mu$ is initialised using a normal distributtion with mean\n        `posterior_mu_init` and std equal to 0.1.\n    posterior_rho_init: float = -7.0\n        As in the case of $\\mu$, $\\rho$ is initialised using a\n        normal distributtion with mean `posterior_rho_init` and std equal to\n        0.1.\n\n    Attributes\n    ----------\n    bayesian_cat_and_cont_embed: nn.Module\n        This is the module that processes the categorical and continuous columns\n    bayesian_tab_mlp: nn.Sequential\n        mlp model that will receive the concatenation of the embeddings and\n        the continuous columns\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.bayesian_models import BayesianTabMlp\n    &gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n    &gt;&gt;&gt; colnames = ['a', 'b', 'c', 'd', 'e']\n    &gt;&gt;&gt; cat_embed_input = [(u,i,j) for u,i,j in zip(colnames[:4], [4]*4, [8]*4)]\n    &gt;&gt;&gt; column_idx = {k:v for v,k in enumerate(colnames)}\n    &gt;&gt;&gt; model = BayesianTabMlp(mlp_hidden_dims=[8,4], column_idx=column_idx, cat_embed_input=cat_embed_input,\n    ... continuous_cols = ['e'])\n    &gt;&gt;&gt; out = model(X_tab)\n    \"\"\"\n\n    def __init__(\n        self,\n        column_idx: Dict[str, int],\n        *,\n        cat_embed_input: Optional[List[Tuple[str, int, int]]] = None,\n        cat_embed_activation: Optional[str] = None,\n        continuous_cols: Optional[List[str]] = None,\n        embed_continuous: Optional[bool] = None,\n        cont_embed_dim: Optional[int] = None,\n        cont_embed_dropout: Optional[float] = None,\n        cont_embed_activation: Optional[str] = None,\n        use_cont_bias: Optional[bool] = None,\n        cont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\n        mlp_hidden_dims: List[int] = [200, 100],\n        mlp_activation: str = \"leaky_relu\",\n        prior_sigma_1: float = 1,\n        prior_sigma_2: float = 0.002,\n        prior_pi: float = 0.8,\n        posterior_mu_init: float = 0.0,\n        posterior_rho_init: float = -7.0,\n        pred_dim=1,  # Bayesian models will require their own trainer and need the output layer\n    ):\n        super(BayesianTabMlp, self).__init__()\n\n        self.column_idx = column_idx\n        self.cat_embed_input = cat_embed_input\n        self.cat_embed_activation = cat_embed_activation\n\n        self.continuous_cols = continuous_cols\n        self.cont_norm_layer = cont_norm_layer\n        self.embed_continuous = embed_continuous\n        self.cont_embed_dim = cont_embed_dim\n        self.cont_embed_dropout = cont_embed_dropout\n        self.use_cont_bias = use_cont_bias\n        self.cont_embed_activation = cont_embed_activation\n\n        self.mlp_hidden_dims = mlp_hidden_dims\n        self.mlp_activation = mlp_activation\n\n        self.prior_sigma_1 = prior_sigma_1\n        self.prior_sigma_2 = prior_sigma_2\n        self.prior_pi = prior_pi\n        self.posterior_mu_init = posterior_mu_init\n        self.posterior_rho_init = posterior_rho_init\n\n        self.pred_dim = pred_dim\n\n        allowed_activations = [\"relu\", \"leaky_relu\", \"tanh\", \"gelu\"]\n        if self.mlp_activation not in allowed_activations:\n            raise ValueError(\n                \"Currently, only the following activation functions are supported \"\n                \"for the Bayesian MLP's dense layers: {}. Got '{}' instead\".format(\n                    \", \".join(allowed_activations),\n                    self.mlp_activation,\n                )\n            )\n\n        # Categorical\n        if self.cat_embed_input is not None:\n            self.cat_embed = BayesianDiffSizeCatEmbeddings(\n                column_idx=self.column_idx,\n                embed_input=self.cat_embed_input,\n                prior_sigma_1=self.prior_sigma_1,\n                prior_sigma_2=self.prior_sigma_2,\n                prior_pi=self.prior_pi,\n                posterior_mu_init=self.posterior_mu_init,\n                posterior_rho_init=self.posterior_rho_init,\n                activation_fn=self.cat_embed_activation,\n            )\n            self.cat_out_dim = int(np.sum([embed[2] for embed in self.cat_embed_input]))\n        else:\n            self.cat_out_dim = 0\n\n        # Continuous\n        if self.continuous_cols is not None:\n            self.cont_idx = [self.column_idx[col] for col in self.continuous_cols]\n            if cont_norm_layer == \"layernorm\":\n                self.cont_norm: NormLayers = nn.LayerNorm(len(self.continuous_cols))\n            elif cont_norm_layer == \"batchnorm\":\n                self.cont_norm = nn.BatchNorm1d(len(self.continuous_cols))\n            else:\n                self.cont_norm = nn.Identity()\n            if self.embed_continuous:\n                assert self.cont_embed_dim is not None, (\n                    \"If 'embed_continuous' is True, 'cont_embed_dim' must be \"\n                    \"provided\"\n                )\n                self.cont_embed = BayesianContEmbeddings(\n                    n_cont_cols=len(self.continuous_cols),\n                    embed_dim=self.cont_embed_dim,\n                    prior_sigma_1=self.prior_sigma_1,\n                    prior_sigma_2=self.prior_sigma_2,\n                    prior_pi=self.prior_pi,\n                    posterior_mu_init=self.posterior_mu_init,\n                    posterior_rho_init=self.posterior_rho_init,\n                    use_bias=(\n                        False if self.use_cont_bias is None else self.use_cont_bias\n                    ),\n                    activation_fn=self.cont_embed_activation,\n                )\n                self.cont_out_dim = len(self.continuous_cols) * self.cont_embed_dim\n            else:\n                self.cont_out_dim = len(self.continuous_cols)\n        else:\n            self.cont_out_dim = 0\n\n        self.output_dim = self.cat_out_dim + self.cont_out_dim\n\n        mlp_hidden_dims = [self.output_dim] + mlp_hidden_dims + [pred_dim]\n        self.bayesian_tab_mlp = BayesianMLP(\n            mlp_hidden_dims,\n            mlp_activation,\n            True,  # use_bias\n            prior_sigma_1,\n            prior_sigma_2,\n            prior_pi,\n            posterior_mu_init,\n            posterior_rho_init,\n        )\n\n    def forward(self, X: Tensor) -&gt; Tensor:\n        x = self._get_embeddings(X)\n        x = self.bayesian_tab_mlp(x)\n        return x\n\n    def _get_embeddings(self, X: Tensor) -&gt; Tensor:\n        tensors_to_concat: List[Tensor] = []\n        if self.cat_embed_input is not None:\n            x_cat = self.cat_embed(X)\n            tensors_to_concat.append(x_cat)\n\n        if self.continuous_cols is not None:\n            x_cont = self.cont_norm((X[:, self.cont_idx].float()))\n            if self.embed_continuous:\n                x_cont = self.cont_embed(x_cont)\n                x_cont = einops.rearrange(x_cont, \"b s d -&gt; b (s d)\")\n            tensors_to_concat.append(x_cont)\n\n        x = torch.cat(tensors_to_concat, 1)\n\n        return x\n</code></pre>"},{"location":"pytorch-widedeep/bayesian_trainer.html","title":"Training Deep Learning Probabilistic Models","text":""},{"location":"pytorch-widedeep/bayesian_trainer.html#pytorch_widedeep.training.BayesianTrainer","title":"BayesianTrainer","text":"<p>               Bases: <code>BaseBayesianTrainer</code></p> <p>Class to set the of attributes that will be used during the training process.</p> <p>Both the Bayesian models and the Trainer in this repo are based on the paper: Weight Uncertainty in Neural Networks.</p> <p>Parameters:</p> Name Type Description Default <code>model</code> <code>BaseBayesianModel</code> <p>An object of class <code>BaseBayesianModel</code>. See the <code>Model Components</code> section here in the docs.</p> required <code>objective</code> <code>str</code> <p>Defines the objective, loss or cost function. Param aliases: <code>loss_function</code>, <code>loss_fn</code>, <code>loss</code>, <code>cost_function</code>, <code>cost_fn</code>, <code>cost</code> Possible values are: 'binary', 'multiclass', 'regression'</p> required <code>custom_loss_function</code> <code>Optional[Module]</code> <p>If none of the loss functions available suits the user, it is possible to pass a custom loss function. See for example <code>pytorch_widedeep.losses.FocalLoss</code> for the required structure of the object or the Examples folder in the repo.</p> <code>None</code> <code>optimizer</code> <code>Optional[Optimizer]</code> <p>An instance of Pytorch's <code>Optimizer</code> object(e.g. <code>torch.optim.Adam ()</code>). if no optimizer is passed it will default to <code>AdamW</code>.</p> <code>None</code> <code>lr_scheduler</code> <code>Optional[LRScheduler]</code> <p>An instance of Pytorch's <code>LRScheduler</code> object (e.g <code>torch.optim.lr_scheduler.StepLR(opt, step_size=5)</code>).</p> <code>None</code> <code>callbacks</code> <code>Optional[List[Callback]]</code> <p>List with <code>Callback</code> objects. The three callbacks available in <code>pytorch-widedeep</code> are: <code>LRHistory</code>, <code>ModelCheckpoint</code> and <code>EarlyStopping</code>. This can also be a custom callback. See <code>pytorch_widedeep.callbacks.Callback</code> or the Examples folder in the repo.</p> <code>None</code> <code>metrics</code> <code>Optional[Union[List[Metric], List[Metric]]]</code> <ul> <li>List of objects of type <code>Metric</code>. Metrics available are:   <code>Accuracy</code>, <code>Precision</code>, <code>Recall</code>, <code>FBetaScore</code>,   <code>F1Score</code> and <code>R2Score</code>. This can also be a custom metric as   long as it is an object of type <code>Metric</code>. See   <code>pytorch_widedeep.metrics.Metric</code> or the Examples folder in the repo</li> <li>List of objects of type <code>torchmetrics.Metric</code>. This can be any   metric from torchmetrics library Examples   classification-metrics&gt;<code>_. It can also be a torchmetric custom metric as   long as it is an object of type</code>Metric<code>.   See</code>the instructions</li> </ul> <code>None</code> <code>verbose</code> <code>int</code> <p>Setting it to 0 will print nothing during training.</p> <code>1</code> <code>seed</code> <code>int</code> <p>Random seed to be used internally for train_test_split</p> <code>1</code> <p>Other Parameters:</p> Name Type Description <code>**kwargs</code> <p>Other infrequently used arguments that can also be passed as kwargs are:</p> <ul> <li> <p>device: <code>str</code>     string indicating the device. One of 'cpu', 'gpu' or 'mps' if     run on a Mac with Apple silicon or AMD GPU(s)</p> </li> <li> <p>num_workers: <code>int</code>     number of workers to be used internally by the data loaders</p> </li> <li> <p>class_weight: <code>List[float]</code>     This is the <code>weight</code> or <code>pos_weight</code> parameter in     <code>CrossEntropyLoss</code> and <code>BCEWithLogitsLoss</code>, depending on whether</p> </li> <li> <p>reducelronplateau_criterion: <code>str</code>     This sets the criterion that will be used by the lr scheduler to     take a step: One of 'loss' or 'metric'. The ReduceLROnPlateau     learning rate is a bit particular.</p> </li> </ul> <p>Attributes:</p> Name Type Description <code>cyclic_lr</code> <code>bool</code> <p>Attribute that indicates if  the lr_scheduler is cyclic_lr (i.e. <code>CyclicLR</code> or <code>OneCycleLR</code>). See <code>Pytorch schedulers &lt;https://pytorch.org/docs/stable/optim.html&gt;</code>_.</p> Source code in <code>pytorch_widedeep/training/bayesian_trainer.py</code> <pre><code>class BayesianTrainer(BaseBayesianTrainer):\n    r\"\"\"Class to set the of attributes that will be used during the\n    training process.\n\n    Both the Bayesian models and the Trainer in this repo are based on the paper:\n    [Weight Uncertainty in Neural Networks](https://arxiv.org/pdf/1505.05424.pdf).\n\n    Parameters\n    ----------\n    model: `BaseBayesianModel`\n        An object of class `BaseBayesianModel`. See the `Model Components`\n        section here in the docs.\n    objective: str\n        Defines the objective, loss or cost function.&lt;br/&gt;\n        Param aliases: `loss_function`, `loss_fn`, `loss`,\n        `cost_function`, `cost_fn`, `cost`&lt;br/&gt;\n        Possible values are: _'binary'_, _'multiclass'_, _'regression'_\n    custom_loss_function: `nn.Module`, optional, default = None\n        If none of the loss functions available suits the user, it is possible\n        to pass a custom loss function. See for example\n        `pytorch_widedeep.losses.FocalLoss` for the required structure of the\n        object or the Examples folder in the repo.\n    optimizer: `Optimzer`, optional, default= None\n        An instance of Pytorch's `Optimizer` object(e.g. `torch.optim.Adam\n        ()`). if no optimizer is passed it will default to `AdamW`.\n    lr_scheduler: `LRScheduler`, optional, default=None\n        An instance of Pytorch's `LRScheduler` object\n        (e.g `torch.optim.lr_scheduler.StepLR(opt, step_size=5)`).\n    callbacks: List, optional, default=None\n        List with `Callback` objects. The three callbacks available in\n        `pytorch-widedeep` are: `LRHistory`, `ModelCheckpoint` and\n        `EarlyStopping`. This can also be a custom callback. See\n        `pytorch_widedeep.callbacks.Callback` or the Examples folder in the\n        repo.\n    metrics: List, optional, default=None\n        - List of objects of type `Metric`. Metrics available are:\n          `Accuracy`, `Precision`, `Recall`, `FBetaScore`,\n          `F1Score` and `R2Score`. This can also be a custom metric as\n          long as it is an object of type `Metric`. See\n          `pytorch_widedeep.metrics.Metric` or the Examples folder in the repo\n        - List of objects of type `torchmetrics.Metric`. This can be any\n          metric from torchmetrics library [Examples](https://lightning.ai/docs/torchmetrics)\n          classification-metrics&gt;`_. It can also be a torchmetric custom metric as\n          long as it is an object of type `Metric`.\n          See `the [instructions]((https://lightning.ai/docs/torchmetrics))\n    verbose: int, default=1\n        Setting it to 0 will print nothing during training.\n    seed: int, default=1\n        Random seed to be used internally for train_test_split\n\n    Other Parameters\n    ----------------\n    **kwargs: dict\n        Other infrequently used arguments that can also be passed as kwargs are:\n\n        - **device**: `str`&lt;br/&gt;\n            string indicating the device. One of _'cpu'_, _'gpu'_ or 'mps' if\n            run on a Mac with Apple silicon or AMD GPU(s)\n\n        - **num_workers**: `int`&lt;br/&gt;\n            number of workers to be used internally by the data loaders\n\n        - **class_weight**: `List[float]`&lt;br/&gt;\n            This is the `weight` or `pos_weight` parameter in\n            `CrossEntropyLoss` and `BCEWithLogitsLoss`, depending on whether\n\n        - **reducelronplateau_criterion**: `str`\n            This sets the criterion that will be used by the lr scheduler to\n            take a step: One of _'loss'_ or _'metric'_. The ReduceLROnPlateau\n            learning rate is a bit particular.\n\n    Attributes\n    ----------\n    cyclic_lr: bool\n        Attribute that indicates if  the lr_scheduler is cyclic_lr\n        (i.e. `CyclicLR` or `OneCycleLR`). See `Pytorch schedulers\n        &lt;https://pytorch.org/docs/stable/optim.html&gt;`_.\n    \"\"\"\n\n    @alias(  # noqa: C901\n        \"objective\",\n        [\"loss_function\", \"loss_fn\", \"loss\", \"cost_function\", \"cost_fn\", \"cost\"],\n    )\n    def __init__(\n        self,\n        model: BaseBayesianModel,\n        objective: str,\n        custom_loss_function: Optional[Module] = None,\n        optimizer: Optional[Optimizer] = None,\n        lr_scheduler: Optional[LRScheduler] = None,\n        callbacks: Optional[List[Callback]] = None,\n        metrics: Optional[Union[List[Metric], List[TorchMetric]]] = None,\n        verbose: int = 1,\n        seed: int = 1,\n        **kwargs,\n    ):\n        super().__init__(\n            model=model,\n            objective=objective,\n            custom_loss_function=custom_loss_function,\n            optimizer=optimizer,\n            lr_scheduler=lr_scheduler,\n            callbacks=callbacks,\n            metrics=metrics,\n            verbose=verbose,\n            seed=seed,\n            **kwargs,\n        )\n\n    def fit(  # noqa: C901\n        self,\n        X_tab: np.ndarray,\n        target: np.ndarray,\n        X_tab_val: Optional[np.ndarray] = None,\n        target_val: Optional[np.ndarray] = None,\n        val_split: Optional[float] = None,\n        n_epochs: int = 1,\n        validation_freq: int = 1,\n        batch_size: int = 32,\n        n_train_samples: int = 2,\n        n_val_samples: int = 2,\n    ):\n        r\"\"\"Fit method.\n\n        Parameters\n        ----------\n        X_tab: np.ndarray,\n            tabular dataset\n        target: np.ndarray\n            target values\n        X_tab_val: np.ndarray, Optional, default = None\n            validation data\n        target_val: np.ndarray, Optional, default = None\n            validation target values\n        val_split: float, Optional. default=None\n            An alterative to passing the validation set is to use a train/val\n            split fraction via `val_split`\n        n_epochs: int, default=1\n            number of epochs\n        validation_freq: int, default=1\n            epochs validation frequency\n        batch_size: int, default=32\n            batch size\n        n_train_samples: int, default=2\n            number of samples to average over during the training process.\n            See [Weight Uncertainty in Neural Networks](https://arxiv.org/pdf/1505.05424.pdf) for details.\n        n_val_samples: int, default=2\n            number of samples to average over during the validation process.\n            See [Weight Uncertainty in Neural Networks](https://arxiv.org/pdf/1505.05424.pdf) for details.\n        \"\"\"\n\n        self.batch_size = batch_size\n\n        train_set, eval_set = tabular_train_val_split(\n            self.seed, self.objective, X_tab, target, X_tab_val, target_val, val_split\n        )\n        train_loader = DataLoader(\n            dataset=train_set, batch_size=batch_size, num_workers=self.num_workers\n        )\n        train_steps = len(train_loader)\n\n        if eval_set is not None:\n            eval_loader = DataLoader(\n                dataset=eval_set,\n                batch_size=batch_size,\n                num_workers=self.num_workers,\n                shuffle=False,\n            )\n            eval_steps = len(eval_loader)\n\n        self.callback_container.on_train_begin(\n            {\n                \"batch_size\": batch_size,\n                \"train_steps\": train_steps,\n                \"n_epochs\": n_epochs,\n            }\n        )\n        for epoch in range(n_epochs):\n            epoch_logs: Dict[str, float] = {}\n            self.callback_container.on_epoch_begin(epoch, logs=epoch_logs)\n\n            self.train_running_loss = 0.0\n            with trange(train_steps, disable=self.verbose != 1) as t:\n                for batch_idx, (X, y) in zip(t, train_loader):\n                    t.set_description(\"epoch %i\" % (epoch + 1))\n                    train_score, train_loss = self._train_step(\n                        X, y, n_train_samples, train_steps, batch_idx\n                    )\n                    print_loss_and_metric(t, train_loss, train_score)\n                    self.callback_container.on_batch_end(batch=batch_idx)\n            epoch_logs = save_epoch_logs(epoch_logs, train_loss, train_score, \"train\")\n\n            on_epoch_end_metric = None\n            if eval_set is not None and epoch % validation_freq == (\n                validation_freq - 1\n            ):\n                self.callback_container.on_eval_begin()\n                self.valid_running_loss = 0.0\n                with trange(eval_steps, disable=self.verbose != 1) as v:\n                    for i, (X, y) in zip(v, eval_loader):\n                        v.set_description(\"valid\")\n                        val_score, val_loss = self._eval_step(\n                            X, y, n_val_samples, train_steps, i\n                        )\n                        print_loss_and_metric(v, val_loss, val_score)\n                epoch_logs = save_epoch_logs(epoch_logs, val_loss, val_score, \"val\")\n\n                if self.reducelronplateau:\n                    if self.reducelronplateau_criterion == \"loss\":\n                        on_epoch_end_metric = val_loss\n                    else:\n                        on_epoch_end_metric = val_score[\n                            self.reducelronplateau_criterion\n                        ]\n\n            self.callback_container.on_epoch_end(epoch, epoch_logs, on_epoch_end_metric)\n\n            if self.early_stop:\n                self.callback_container.on_train_end(epoch_logs)\n                break\n\n        self.callback_container.on_train_end(epoch_logs)\n        self._restore_best_weights()\n        self.model.train()\n\n    def predict(  # type: ignore[return]\n        self,\n        X_tab: np.ndarray,\n        n_samples: int = 5,\n        return_samples: bool = False,\n        batch_size: int = 256,\n    ) -&gt; np.ndarray:\n        r\"\"\"Returns the predictions\n\n        Parameters\n        ----------\n        X_tab: np.ndarray,\n            tabular dataset\n        n_samples: int, default=5\n            number of samples that will be either returned or averaged to\n            produce an overal prediction\n        return_samples: bool, default = False\n            Boolean indicating whether the n samples will be averaged or directly returned\n        batch_size: int, default = 256\n            batch size\n\n        Returns\n        -------\n        np.ndarray:\n            array with the predictions\n        \"\"\"\n\n        preds_l = self._predict(X_tab, n_samples, return_samples, batch_size)\n        preds = np.hstack(preds_l) if return_samples else np.vstack(preds_l)\n        axis = 2 if return_samples else 1\n\n        if self.objective == \"regression\":\n            return preds.squeeze(axis)\n        if self.objective == \"binary\":\n            return (preds.squeeze(axis) &gt; 0.5).astype(\"int\")\n        if self.objective == \"multiclass\":\n            return np.argmax(preds, axis)\n\n    def predict_proba(  # type: ignore[return]\n        self,\n        X_tab: np.ndarray,\n        n_samples: int = 5,\n        return_samples: bool = False,\n        batch_size: int = 256,\n    ) -&gt; np.ndarray:\n        r\"\"\"Returns the predicted probabilities\n\n        Parameters\n        ----------\n        X_tab: np.ndarray,\n            tabular dataset\n        n_samples: int, default=5\n            number of samples that will be either returned or averaged to\n            produce an overal prediction\n        return_samples: bool, default = False\n            Boolean indicating whether the n samples will be averaged or directly returned\n        batch_size: int, default = 256\n            batch size\n\n        Returns\n        -------\n        np.ndarray\n            array with the probabilities per class\n        \"\"\"\n        preds_l = self._predict(X_tab, n_samples, return_samples, batch_size)\n        preds = np.hstack(preds_l) if return_samples else np.vstack(preds_l)\n\n        if self.objective == \"binary\":\n            if return_samples:\n                preds = preds.squeeze(2)\n                probs = np.zeros([n_samples, preds.shape[1], 2])\n                for i in range(n_samples):\n                    probs[i, :, 0] = 1 - preds[i]\n                    probs[i, :, 1] = preds[i]\n            else:\n                preds = preds.squeeze(1)\n                probs = np.zeros([preds.shape[0], 2])\n                probs[:, 0] = 1 - preds\n                probs[:, 1] = preds\n            return probs\n        if self.objective == \"multiclass\":\n            return preds\n\n    def save(\n        self,\n        path: str,\n        save_state_dict: bool = False,\n        model_filename: str = \"bayesian_model.pt\",\n    ):\n        r\"\"\"Saves the model, training and evaluation history to disk\n\n        The `Trainer` class is built so that it 'just' trains a model. With\n        that in mind, all the torch related parameters (such as optimizers or\n        learning rate schedulers) have to be defined externally and then\n        passed to the `Trainer`. As a result, the `Trainer` does not\n        generate any attribute or additional data products that need to be\n        saved other than the `model` object itself, which can be saved as\n        any other torch model (e.g. `torch.save(model, path)`).\n\n        Parameters\n        ----------\n        path: str\n            path to the directory where the model and the feature importance\n            attribute will be saved.\n        save_state_dict: bool, default = False\n            Boolean indicating whether to save directly the model or the\n            model's state dictionary\n        model_filename: str, Optional, default = \"wd_model.pt\"\n            filename where the model weights will be store\n        \"\"\"\n\n        save_dir = Path(path)\n        history_dir = save_dir / \"history\"\n        history_dir.mkdir(exist_ok=True, parents=True)\n\n        # the trainer is run with the History Callback by default\n        with open(history_dir / \"train_eval_history.json\", \"w\") as teh:\n            json.dump(self.history, teh)  # type: ignore[attr-defined]\n\n        has_lr_history = any(\n            [clbk.__class__.__name__ == \"LRHistory\" for clbk in self.callbacks]\n        )\n        if self.lr_scheduler is not None and has_lr_history:\n            with open(history_dir / \"lr_history.json\", \"w\") as lrh:\n                json.dump(self.lr_history, lrh)  # type: ignore[attr-defined]\n\n        model_path = save_dir / model_filename\n        if save_state_dict:\n            torch.save(self.model.state_dict(), model_path)\n        else:\n            torch.save(self.model, model_path)\n\n    def _train_step(\n        self,\n        X_tab: Tensor,\n        target: Tensor,\n        n_samples: int,\n        n_batches: int,\n        batch_idx: int,\n    ):\n        self.model.train()\n\n        X = to_device(X_tab, self.device)\n        y = target.view(-1, 1).float() if self.objective != \"multiclass\" else target\n        y = to_device(y, self.device)\n\n        self.optimizer.zero_grad()\n        y_pred, loss = self.model.sample_elbo(X, y, self.loss_fn, n_samples, n_batches)  # type: ignore[arg-type]\n\n        y_pred = y_pred.mean(dim=0)\n        score = self._get_score(y_pred, y)\n\n        loss.backward()\n        self.optimizer.step()\n\n        self.train_running_loss += loss.item()\n        avg_loss = self.train_running_loss / (batch_idx + 1)\n\n        return score, avg_loss\n\n    def _eval_step(\n        self,\n        X_tab: Tensor,\n        target: Tensor,\n        n_samples: int,\n        n_batches: int,\n        batch_idx: int,\n    ):\n        self.model.eval()\n        with torch.no_grad():\n            X = to_device(X_tab, self.device)\n            y = target.view(-1, 1).float() if self.objective != \"multiclass\" else target\n            y = to_device(y, self.device)\n\n            y_pred, loss = self.model.sample_elbo(\n                X,  # type: ignore[arg-type]\n                y,\n                self.loss_fn,\n                n_samples,\n                n_batches,\n            )\n            y_pred = y_pred.mean(dim=0)\n            score = self._get_score(y_pred, y)\n\n            self.valid_running_loss += loss.item()\n            avg_loss = self.valid_running_loss / (batch_idx + 1)\n\n        return score, avg_loss\n\n    def _get_score(self, y_pred, y):\n        if self.metric is not None:\n            if self.objective == \"regression\":\n                score = self.metric(y_pred, y)\n            if self.objective == \"binary\":\n                score = self.metric(torch.sigmoid(y_pred), y)\n            if self.objective == \"multiclass\":\n                score = self.metric(F.softmax(y_pred, dim=1), y)\n            return score\n        else:\n            return None\n\n    def _predict(  # noqa: C901\n        self,\n        X_tab: np.ndarray,\n        n_samples: int = 5,\n        return_samples: bool = False,\n        batch_size: int = 256,\n    ) -&gt; List:\n        self.batch_size = batch_size\n\n        test_set = TensorDataset(torch.from_numpy(X_tab))\n        test_loader = DataLoader(\n            dataset=test_set,\n            batch_size=self.batch_size,\n            num_workers=self.num_workers,\n            shuffle=False,\n        )\n        test_steps = (len(test_loader.dataset) // test_loader.batch_size) + 1  # type: ignore[arg-type]\n\n        preds_l = []\n        with torch.no_grad():\n            with trange(test_steps, disable=self.verbose != 1) as tt:\n                for _, Xl in zip(tt, test_loader):\n                    tt.set_description(\"predict\")\n\n                    try:\n                        X = Xl[0].to(self.device)\n                    except TypeError:\n                        X = Xl[0].to(self.device, dtype=torch.float32)\n\n                    if return_samples:\n                        preds = torch.stack([self.model(X) for _ in range(n_samples)])\n                    else:\n                        self.model.eval()\n                        preds = self.model(X)\n\n                    if self.objective == \"binary\":\n                        preds = torch.sigmoid(preds)\n                    if self.objective == \"multiclass\":\n                        preds = (\n                            F.softmax(preds, dim=2)\n                            if return_samples\n                            else F.softmax(preds, dim=1)\n                        )\n\n                    preds_cpu = preds.cpu().data.numpy()\n                    preds_l.append(preds_cpu)\n\n        self.model.train()\n\n        return preds_l\n</code></pre>"},{"location":"pytorch-widedeep/bayesian_trainer.html#pytorch_widedeep.training.BayesianTrainer.fit","title":"fit","text":"<pre><code>fit(\n    X_tab,\n    target,\n    X_tab_val=None,\n    target_val=None,\n    val_split=None,\n    n_epochs=1,\n    validation_freq=1,\n    batch_size=32,\n    n_train_samples=2,\n    n_val_samples=2,\n)\n</code></pre> <p>Fit method.</p> <p>Parameters:</p> Name Type Description Default <code>X_tab</code> <code>ndarray</code> <p>tabular dataset</p> required <code>target</code> <code>ndarray</code> <p>target values</p> required <code>X_tab_val</code> <code>Optional[ndarray]</code> <p>validation data</p> <code>None</code> <code>target_val</code> <code>Optional[ndarray]</code> <p>validation target values</p> <code>None</code> <code>val_split</code> <code>Optional[float]</code> <p>An alterative to passing the validation set is to use a train/val split fraction via <code>val_split</code></p> <code>None</code> <code>n_epochs</code> <code>int</code> <p>number of epochs</p> <code>1</code> <code>validation_freq</code> <code>int</code> <p>epochs validation frequency</p> <code>1</code> <code>batch_size</code> <code>int</code> <p>batch size</p> <code>32</code> <code>n_train_samples</code> <code>int</code> <p>number of samples to average over during the training process. See Weight Uncertainty in Neural Networks for details.</p> <code>2</code> <code>n_val_samples</code> <code>int</code> <p>number of samples to average over during the validation process. See Weight Uncertainty in Neural Networks for details.</p> <code>2</code> Source code in <code>pytorch_widedeep/training/bayesian_trainer.py</code> <pre><code>def fit(  # noqa: C901\n    self,\n    X_tab: np.ndarray,\n    target: np.ndarray,\n    X_tab_val: Optional[np.ndarray] = None,\n    target_val: Optional[np.ndarray] = None,\n    val_split: Optional[float] = None,\n    n_epochs: int = 1,\n    validation_freq: int = 1,\n    batch_size: int = 32,\n    n_train_samples: int = 2,\n    n_val_samples: int = 2,\n):\n    r\"\"\"Fit method.\n\n    Parameters\n    ----------\n    X_tab: np.ndarray,\n        tabular dataset\n    target: np.ndarray\n        target values\n    X_tab_val: np.ndarray, Optional, default = None\n        validation data\n    target_val: np.ndarray, Optional, default = None\n        validation target values\n    val_split: float, Optional. default=None\n        An alterative to passing the validation set is to use a train/val\n        split fraction via `val_split`\n    n_epochs: int, default=1\n        number of epochs\n    validation_freq: int, default=1\n        epochs validation frequency\n    batch_size: int, default=32\n        batch size\n    n_train_samples: int, default=2\n        number of samples to average over during the training process.\n        See [Weight Uncertainty in Neural Networks](https://arxiv.org/pdf/1505.05424.pdf) for details.\n    n_val_samples: int, default=2\n        number of samples to average over during the validation process.\n        See [Weight Uncertainty in Neural Networks](https://arxiv.org/pdf/1505.05424.pdf) for details.\n    \"\"\"\n\n    self.batch_size = batch_size\n\n    train_set, eval_set = tabular_train_val_split(\n        self.seed, self.objective, X_tab, target, X_tab_val, target_val, val_split\n    )\n    train_loader = DataLoader(\n        dataset=train_set, batch_size=batch_size, num_workers=self.num_workers\n    )\n    train_steps = len(train_loader)\n\n    if eval_set is not None:\n        eval_loader = DataLoader(\n            dataset=eval_set,\n            batch_size=batch_size,\n            num_workers=self.num_workers,\n            shuffle=False,\n        )\n        eval_steps = len(eval_loader)\n\n    self.callback_container.on_train_begin(\n        {\n            \"batch_size\": batch_size,\n            \"train_steps\": train_steps,\n            \"n_epochs\": n_epochs,\n        }\n    )\n    for epoch in range(n_epochs):\n        epoch_logs: Dict[str, float] = {}\n        self.callback_container.on_epoch_begin(epoch, logs=epoch_logs)\n\n        self.train_running_loss = 0.0\n        with trange(train_steps, disable=self.verbose != 1) as t:\n            for batch_idx, (X, y) in zip(t, train_loader):\n                t.set_description(\"epoch %i\" % (epoch + 1))\n                train_score, train_loss = self._train_step(\n                    X, y, n_train_samples, train_steps, batch_idx\n                )\n                print_loss_and_metric(t, train_loss, train_score)\n                self.callback_container.on_batch_end(batch=batch_idx)\n        epoch_logs = save_epoch_logs(epoch_logs, train_loss, train_score, \"train\")\n\n        on_epoch_end_metric = None\n        if eval_set is not None and epoch % validation_freq == (\n            validation_freq - 1\n        ):\n            self.callback_container.on_eval_begin()\n            self.valid_running_loss = 0.0\n            with trange(eval_steps, disable=self.verbose != 1) as v:\n                for i, (X, y) in zip(v, eval_loader):\n                    v.set_description(\"valid\")\n                    val_score, val_loss = self._eval_step(\n                        X, y, n_val_samples, train_steps, i\n                    )\n                    print_loss_and_metric(v, val_loss, val_score)\n            epoch_logs = save_epoch_logs(epoch_logs, val_loss, val_score, \"val\")\n\n            if self.reducelronplateau:\n                if self.reducelronplateau_criterion == \"loss\":\n                    on_epoch_end_metric = val_loss\n                else:\n                    on_epoch_end_metric = val_score[\n                        self.reducelronplateau_criterion\n                    ]\n\n        self.callback_container.on_epoch_end(epoch, epoch_logs, on_epoch_end_metric)\n\n        if self.early_stop:\n            self.callback_container.on_train_end(epoch_logs)\n            break\n\n    self.callback_container.on_train_end(epoch_logs)\n    self._restore_best_weights()\n    self.model.train()\n</code></pre>"},{"location":"pytorch-widedeep/bayesian_trainer.html#pytorch_widedeep.training.BayesianTrainer.predict","title":"predict","text":"<pre><code>predict(\n    X_tab, n_samples=5, return_samples=False, batch_size=256\n)\n</code></pre> <p>Returns the predictions</p> <p>Parameters:</p> Name Type Description Default <code>X_tab</code> <code>ndarray</code> <p>tabular dataset</p> required <code>n_samples</code> <code>int</code> <p>number of samples that will be either returned or averaged to produce an overal prediction</p> <code>5</code> <code>return_samples</code> <code>bool</code> <p>Boolean indicating whether the n samples will be averaged or directly returned</p> <code>False</code> <code>batch_size</code> <code>int</code> <p>batch size</p> <code>256</code> <p>Returns:</p> Type Description <code>np.ndarray:</code> <p>array with the predictions</p> Source code in <code>pytorch_widedeep/training/bayesian_trainer.py</code> <pre><code>def predict(  # type: ignore[return]\n    self,\n    X_tab: np.ndarray,\n    n_samples: int = 5,\n    return_samples: bool = False,\n    batch_size: int = 256,\n) -&gt; np.ndarray:\n    r\"\"\"Returns the predictions\n\n    Parameters\n    ----------\n    X_tab: np.ndarray,\n        tabular dataset\n    n_samples: int, default=5\n        number of samples that will be either returned or averaged to\n        produce an overal prediction\n    return_samples: bool, default = False\n        Boolean indicating whether the n samples will be averaged or directly returned\n    batch_size: int, default = 256\n        batch size\n\n    Returns\n    -------\n    np.ndarray:\n        array with the predictions\n    \"\"\"\n\n    preds_l = self._predict(X_tab, n_samples, return_samples, batch_size)\n    preds = np.hstack(preds_l) if return_samples else np.vstack(preds_l)\n    axis = 2 if return_samples else 1\n\n    if self.objective == \"regression\":\n        return preds.squeeze(axis)\n    if self.objective == \"binary\":\n        return (preds.squeeze(axis) &gt; 0.5).astype(\"int\")\n    if self.objective == \"multiclass\":\n        return np.argmax(preds, axis)\n</code></pre>"},{"location":"pytorch-widedeep/bayesian_trainer.html#pytorch_widedeep.training.BayesianTrainer.predict_proba","title":"predict_proba","text":"<pre><code>predict_proba(\n    X_tab, n_samples=5, return_samples=False, batch_size=256\n)\n</code></pre> <p>Returns the predicted probabilities</p> <p>Parameters:</p> Name Type Description Default <code>X_tab</code> <code>ndarray</code> <p>tabular dataset</p> required <code>n_samples</code> <code>int</code> <p>number of samples that will be either returned or averaged to produce an overal prediction</p> <code>5</code> <code>return_samples</code> <code>bool</code> <p>Boolean indicating whether the n samples will be averaged or directly returned</p> <code>False</code> <code>batch_size</code> <code>int</code> <p>batch size</p> <code>256</code> <p>Returns:</p> Type Description <code>ndarray</code> <p>array with the probabilities per class</p> Source code in <code>pytorch_widedeep/training/bayesian_trainer.py</code> <pre><code>def predict_proba(  # type: ignore[return]\n    self,\n    X_tab: np.ndarray,\n    n_samples: int = 5,\n    return_samples: bool = False,\n    batch_size: int = 256,\n) -&gt; np.ndarray:\n    r\"\"\"Returns the predicted probabilities\n\n    Parameters\n    ----------\n    X_tab: np.ndarray,\n        tabular dataset\n    n_samples: int, default=5\n        number of samples that will be either returned or averaged to\n        produce an overal prediction\n    return_samples: bool, default = False\n        Boolean indicating whether the n samples will be averaged or directly returned\n    batch_size: int, default = 256\n        batch size\n\n    Returns\n    -------\n    np.ndarray\n        array with the probabilities per class\n    \"\"\"\n    preds_l = self._predict(X_tab, n_samples, return_samples, batch_size)\n    preds = np.hstack(preds_l) if return_samples else np.vstack(preds_l)\n\n    if self.objective == \"binary\":\n        if return_samples:\n            preds = preds.squeeze(2)\n            probs = np.zeros([n_samples, preds.shape[1], 2])\n            for i in range(n_samples):\n                probs[i, :, 0] = 1 - preds[i]\n                probs[i, :, 1] = preds[i]\n        else:\n            preds = preds.squeeze(1)\n            probs = np.zeros([preds.shape[0], 2])\n            probs[:, 0] = 1 - preds\n            probs[:, 1] = preds\n        return probs\n    if self.objective == \"multiclass\":\n        return preds\n</code></pre>"},{"location":"pytorch-widedeep/bayesian_trainer.html#pytorch_widedeep.training.BayesianTrainer.save","title":"save","text":"<pre><code>save(\n    path,\n    save_state_dict=False,\n    model_filename=\"bayesian_model.pt\",\n)\n</code></pre> <p>Saves the model, training and evaluation history to disk</p> <p>The <code>Trainer</code> class is built so that it 'just' trains a model. With that in mind, all the torch related parameters (such as optimizers or learning rate schedulers) have to be defined externally and then passed to the <code>Trainer</code>. As a result, the <code>Trainer</code> does not generate any attribute or additional data products that need to be saved other than the <code>model</code> object itself, which can be saved as any other torch model (e.g. <code>torch.save(model, path)</code>).</p> <p>Parameters:</p> Name Type Description Default <code>path</code> <code>str</code> <p>path to the directory where the model and the feature importance attribute will be saved.</p> required <code>save_state_dict</code> <code>bool</code> <p>Boolean indicating whether to save directly the model or the model's state dictionary</p> <code>False</code> <code>model_filename</code> <code>str</code> <p>filename where the model weights will be store</p> <code>'bayesian_model.pt'</code> Source code in <code>pytorch_widedeep/training/bayesian_trainer.py</code> <pre><code>def save(\n    self,\n    path: str,\n    save_state_dict: bool = False,\n    model_filename: str = \"bayesian_model.pt\",\n):\n    r\"\"\"Saves the model, training and evaluation history to disk\n\n    The `Trainer` class is built so that it 'just' trains a model. With\n    that in mind, all the torch related parameters (such as optimizers or\n    learning rate schedulers) have to be defined externally and then\n    passed to the `Trainer`. As a result, the `Trainer` does not\n    generate any attribute or additional data products that need to be\n    saved other than the `model` object itself, which can be saved as\n    any other torch model (e.g. `torch.save(model, path)`).\n\n    Parameters\n    ----------\n    path: str\n        path to the directory where the model and the feature importance\n        attribute will be saved.\n    save_state_dict: bool, default = False\n        Boolean indicating whether to save directly the model or the\n        model's state dictionary\n    model_filename: str, Optional, default = \"wd_model.pt\"\n        filename where the model weights will be store\n    \"\"\"\n\n    save_dir = Path(path)\n    history_dir = save_dir / \"history\"\n    history_dir.mkdir(exist_ok=True, parents=True)\n\n    # the trainer is run with the History Callback by default\n    with open(history_dir / \"train_eval_history.json\", \"w\") as teh:\n        json.dump(self.history, teh)  # type: ignore[attr-defined]\n\n    has_lr_history = any(\n        [clbk.__class__.__name__ == \"LRHistory\" for clbk in self.callbacks]\n    )\n    if self.lr_scheduler is not None and has_lr_history:\n        with open(history_dir / \"lr_history.json\", \"w\") as lrh:\n            json.dump(self.lr_history, lrh)  # type: ignore[attr-defined]\n\n    model_path = save_dir / model_filename\n    if save_state_dict:\n        torch.save(self.model.state_dict(), model_path)\n    else:\n        torch.save(self.model, model_path)\n</code></pre>"},{"location":"pytorch-widedeep/callbacks.html","title":"Callbacks","text":"<p>Here are the 4 callbacks available to the user in <code>pytorch-widedepp</code>: <code>LRHistory</code>, <code>ModelCheckpoint</code>, <code>EarlyStopping</code> and <code>RayTuneReporter</code>.</p> <p> NOTE: other callbacks , like <code>History</code>, run always  by default. In particular, the <code>History</code> callback saves the metrics in the  <code>history</code> attribute of the <code>Trainer</code>.</p>"},{"location":"pytorch-widedeep/callbacks.html#pytorch_widedeep.callbacks.LRHistory","title":"LRHistory","text":"<p>               Bases: <code>Callback</code></p> <p>Saves the learning rates during training in the <code>lr_history</code> attribute of the <code>Trainer</code>.</p> <p>Callbacks are passed as input parameters to the <code>Trainer</code> class. See <code>pytorch_widedeep.trainer.Trainer</code></p> <p>Parameters:</p> Name Type Description Default <code>n_epochs</code> <code>int</code> <p>number of training epochs</p> required <p>Examples:</p> <pre><code>&gt;&gt;&gt; from pytorch_widedeep.callbacks import LRHistory\n&gt;&gt;&gt; from pytorch_widedeep.models import TabMlp, Wide, WideDeep\n&gt;&gt;&gt; from pytorch_widedeep.training import Trainer\n&gt;&gt;&gt;\n&gt;&gt;&gt; embed_input = [(u, i, j) for u, i, j in zip([\"a\", \"b\", \"c\"][:4], [4] * 3, [8] * 3)]\n&gt;&gt;&gt; column_idx = {k: v for v, k in enumerate([\"a\", \"b\", \"c\"])}\n&gt;&gt;&gt; wide = Wide(10, 1)\n&gt;&gt;&gt; deep = TabMlp(mlp_hidden_dims=[8, 4], column_idx=column_idx, cat_embed_input=embed_input)\n&gt;&gt;&gt; model = WideDeep(wide, deep)\n&gt;&gt;&gt; trainer = Trainer(model, objective=\"regression\", callbacks=[LRHistory(n_epochs=10)])\n</code></pre> Source code in <code>pytorch_widedeep/callbacks.py</code> <pre><code>class LRHistory(Callback):\n    r\"\"\"Saves the learning rates during training in the `lr_history` attribute\n    of the `Trainer`.\n\n    Callbacks are passed as input parameters to the `Trainer` class. See\n    `pytorch_widedeep.trainer.Trainer`\n\n    Parameters\n    ----------\n    n_epochs: int\n        number of training epochs\n\n    Examples\n    --------\n    &gt;&gt;&gt; from pytorch_widedeep.callbacks import LRHistory\n    &gt;&gt;&gt; from pytorch_widedeep.models import TabMlp, Wide, WideDeep\n    &gt;&gt;&gt; from pytorch_widedeep.training import Trainer\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; embed_input = [(u, i, j) for u, i, j in zip([\"a\", \"b\", \"c\"][:4], [4] * 3, [8] * 3)]\n    &gt;&gt;&gt; column_idx = {k: v for v, k in enumerate([\"a\", \"b\", \"c\"])}\n    &gt;&gt;&gt; wide = Wide(10, 1)\n    &gt;&gt;&gt; deep = TabMlp(mlp_hidden_dims=[8, 4], column_idx=column_idx, cat_embed_input=embed_input)\n    &gt;&gt;&gt; model = WideDeep(wide, deep)\n    &gt;&gt;&gt; trainer = Trainer(model, objective=\"regression\", callbacks=[LRHistory(n_epochs=10)])\n    \"\"\"\n\n    def __init__(self, n_epochs: int):\n        super(LRHistory, self).__init__()\n        self.n_epochs = n_epochs\n\n    def on_epoch_begin(self, epoch: int, logs: Optional[Dict] = None):\n        if epoch == 0 and self.trainer.lr_scheduler is not None:\n            self.trainer.lr_history = {}\n            if self._multiple_scheduler(self.trainer.lr_scheduler):\n                self._save_group_lr_mulitple_scheduler(step_location=\"on_epoch_begin\")\n            else:\n                self._save_group_lr(self.trainer.optimizer)\n\n    def on_batch_end(self, batch: int, logs: Optional[Dict] = None):\n        if self.trainer.lr_scheduler is not None:\n            if self._multiple_scheduler(self.trainer.lr_scheduler):\n                self._save_group_lr_mulitple_scheduler(step_location=\"on_batch_end\")\n            elif self.trainer.cyclic_lr:\n                self._save_group_lr(self.trainer.optimizer)\n\n    def on_epoch_end(\n        self, epoch: int, logs: Optional[Dict] = None, metric: Optional[float] = None\n    ):\n        if epoch != (self.n_epochs - 1) and self.trainer.lr_scheduler is not None:\n            if self._multiple_scheduler(self.trainer.lr_scheduler):\n                self._save_group_lr_mulitple_scheduler(step_location=\"on_epoch_end\")\n            elif not self.trainer.cyclic_lr:\n                self._save_group_lr(self.trainer.optimizer)\n\n    def _save_group_lr_mulitple_scheduler(self, step_location: str):\n        for model_name, optimizer in self.trainer.optimizer._optimizers.items():\n            if isinstance(optimizer, list):\n                # then, if it has schedulers, we assume it has to have the\n                # same number of schedulers as optimizers\n                for i, opt in enumerate(optimizer):\n                    if (\n                        step_location == \"on_epoch_begin\"\n                        or (\n                            step_location == \"on_batch_end\"\n                            and self._has_cyclic_scheduler(model_name)\n                        )\n                        or (\n                            step_location == \"on_epoch_end\"\n                            and not self._has_cyclic_scheduler(model_name)\n                        )\n                    ):\n                        self._save_group_lr(opt, model_name, \"_\".join([\"opt\", str(i)]))\n                else:\n                    # do nothing\n                    pass\n            else:\n                if (\n                    step_location == \"on_epoch_begin\"\n                    or (\n                        step_location == \"on_batch_end\"\n                        and self._has_cyclic_scheduler(model_name)\n                    )\n                    or (\n                        step_location == \"on_epoch_end\"\n                        and not self._has_cyclic_scheduler(model_name)\n                    )\n                ):\n                    self._save_group_lr(optimizer, model_name)\n\n    def _save_group_lr(\n        self,\n        opt: Optimizer,\n        suffix: Optional[str] = None,\n        model_name: Optional[str] = None,\n    ):\n        suffix = suffix or \"\"\n        model_name = model_name or \"\"\n        for group_idx, group in enumerate(opt.param_groups):\n            group_name = (\"_\").join(\n                [x for x in [\"lr\", model_name, suffix, str(group_idx)] if x]\n            )\n            self.trainer.lr_history.setdefault(group_name, []).append(group[\"lr\"])\n\n    @staticmethod\n    def _multiple_scheduler(scheduler: LRScheduler) -&gt; bool:\n        return scheduler.__class__.__name__ == \"MultipleLRScheduler\"\n\n    def _has_cyclic_scheduler(self, model_name: str):\n        if model_name in self.trainer.lr_scheduler._schedulers:\n            if isinstance(self.trainer.lr_scheduler._schedulers[model_name], list):\n                return any(\n                    [\n                        self._is_cyclic(s)\n                        for s in self.trainer.lr_scheduler._schedulers[model_name]\n                    ]\n                )\n            else:\n                return self._is_cyclic(\n                    self.trainer.lr_scheduler._schedulers[model_name]\n                )\n\n    @staticmethod\n    def _is_cyclic(scheduler: LRScheduler) -&gt; bool:\n        return \"cycl\" in scheduler.__class__.__name__.lower()\n</code></pre>"},{"location":"pytorch-widedeep/callbacks.html#pytorch_widedeep.callbacks.ModelCheckpoint","title":"ModelCheckpoint","text":"<p>               Bases: <code>Callback</code></p> <p>Saves the model after every epoch.</p> <p>This class is almost identical to the corresponding keras class. Therefore, credit to the Keras Team.</p> <p>Callbacks are passed as input parameters to the <code>Trainer</code> class. See <code>pytorch_widedeep.trainer.Trainer</code></p> <p>Parameters:</p> Name Type Description Default <code>filepath</code> <code>Optional[str]</code> <p>Full path to save the output weights. It must contain only the root of the filenames. Epoch number and <code>.pt</code> extension (for pytorch) will be added. e.g. <code>filepath=\"path/to/output_weights/weights_out\"</code> And the saved files in that directory will be named: 'weights_out_1.pt', 'weights_out_2.pt', .... If set to <code>None</code> the class just report best metric and best_epoch.</p> <code>None</code> <code>monitor</code> <code>str</code> <p>quantity to monitor. Typically 'val_loss' or metric name (e.g. 'val_acc')</p> <code>'val_loss'</code> <code>min_delta</code> <code>float</code> <p>minimum change in the monitored quantity to qualify as an improvement, i.e. an absolute change of less than min_delta, will count as no improvement.</p> <code>0.0</code> <code>verbose</code> <code>int</code> <p>verbosity mode</p> <code>0</code> <code>save_best_only</code> <code>bool</code> <p>the latest best model according to the quantity monitored will not be overwritten.</p> <code>False</code> <code>mode</code> <code>str</code> <p>If <code>save_best_only=True</code>, the decision to overwrite the current save file is made based on either the maximization or the minimization of the monitored quantity. For 'acc', this should be 'max', for 'loss' this should be 'min', etc. In 'auto' mode, the direction is automatically inferred from the name of the monitored quantity.</p> <code>'auto'</code> <code>period</code> <code>int</code> <p>Interval (number of epochs) between checkpoints.</p> <code>1</code> <code>max_save</code> <code>int</code> <p>Maximum number of outputs to save. If -1 will save all outputs</p> <code>-1</code> <p>Attributes:</p> Name Type Description <code>best</code> <code>float</code> <p>best metric</p> <code>best_epoch</code> <code>int</code> <p>best epoch</p> <code>best_state_dict</code> <code>dict</code> <p>best model state dictionary. To restore model to its best state use <code>Trainer.model.load_state_dict (model_checkpoint.best_state_dict)</code> where <code>model_checkpoint</code> is an instance of the class <code>ModelCheckpoint</code>. See the Examples folder in the repo or the Examples section in this documentation for details</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; from pytorch_widedeep.callbacks import ModelCheckpoint\n&gt;&gt;&gt; from pytorch_widedeep.models import TabMlp, Wide, WideDeep\n&gt;&gt;&gt; from pytorch_widedeep.training import Trainer\n&gt;&gt;&gt;\n&gt;&gt;&gt; embed_input = [(u, i, j) for u, i, j in zip([\"a\", \"b\", \"c\"][:4], [4] * 3, [8] * 3)]\n&gt;&gt;&gt; column_idx = {k: v for v, k in enumerate([\"a\", \"b\", \"c\"])}\n&gt;&gt;&gt; wide = Wide(10, 1)\n&gt;&gt;&gt; deep = TabMlp(mlp_hidden_dims=[8, 4], column_idx=column_idx, cat_embed_input=embed_input)\n&gt;&gt;&gt; model = WideDeep(wide, deep)\n&gt;&gt;&gt; trainer = Trainer(model, objective=\"regression\", callbacks=[ModelCheckpoint(filepath='checkpoints/weights_out')])\n</code></pre> Source code in <code>pytorch_widedeep/callbacks.py</code> <pre><code>class ModelCheckpoint(Callback):\n    r\"\"\"Saves the model after every epoch.\n\n    This class is almost identical to the corresponding keras class.\n    Therefore, **credit** to the Keras Team.\n\n    Callbacks are passed as input parameters to the `Trainer` class. See\n    `pytorch_widedeep.trainer.Trainer`\n\n    Parameters\n    ----------\n    filepath: str, default=None\n        Full path to save the output weights. It must contain only the root of\n        the filenames. Epoch number and `.pt` extension (for pytorch) will be\n        added. e.g. `filepath=\"path/to/output_weights/weights_out\"` And the\n        saved files in that directory will be named:\n        _'weights_out_1.pt', 'weights_out_2.pt', ..._. If set to `None` the\n        class just report best metric and best_epoch.\n    monitor: str, default=\"loss\"\n        quantity to monitor. Typically _'val_loss'_ or metric name\n        (e.g. _'val_acc'_)\n    min_delta: float, default=0.\n        minimum change in the monitored quantity to qualify as an\n        improvement, i.e. an absolute change of less than min_delta, will\n        count as no improvement.\n    verbose:int, default=0\n        verbosity mode\n    save_best_only: bool, default=False,\n        the latest best model according to the quantity monitored will not be\n        overwritten.\n    mode: str, default=\"auto\"\n        If `save_best_only=True`, the decision to overwrite the current save\n        file is made based on either the maximization or the minimization of\n        the monitored quantity. For _'acc'_, this should be _'max'_, for\n        _'loss'_ this should be _'min'_, etc. In '_auto'_ mode, the\n        direction is automatically inferred from the name of the monitored\n        quantity.\n    period: int, default=1\n        Interval (number of epochs) between checkpoints.\n    max_save: int, default=-1\n        Maximum number of outputs to save. If -1 will save all outputs\n\n    Attributes\n    ----------\n    best: float\n        best metric\n    best_epoch: int\n        best epoch\n    best_state_dict: dict\n        best model state dictionary.&lt;br/&gt;\n        To restore model to its best state use `Trainer.model.load_state_dict\n        (model_checkpoint.best_state_dict)` where `model_checkpoint` is an\n        instance of the class `ModelCheckpoint`. See the Examples folder in\n        the repo or the Examples section in this documentation for details\n\n    Examples\n    --------\n    &gt;&gt;&gt; from pytorch_widedeep.callbacks import ModelCheckpoint\n    &gt;&gt;&gt; from pytorch_widedeep.models import TabMlp, Wide, WideDeep\n    &gt;&gt;&gt; from pytorch_widedeep.training import Trainer\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; embed_input = [(u, i, j) for u, i, j in zip([\"a\", \"b\", \"c\"][:4], [4] * 3, [8] * 3)]\n    &gt;&gt;&gt; column_idx = {k: v for v, k in enumerate([\"a\", \"b\", \"c\"])}\n    &gt;&gt;&gt; wide = Wide(10, 1)\n    &gt;&gt;&gt; deep = TabMlp(mlp_hidden_dims=[8, 4], column_idx=column_idx, cat_embed_input=embed_input)\n    &gt;&gt;&gt; model = WideDeep(wide, deep)\n    &gt;&gt;&gt; trainer = Trainer(model, objective=\"regression\", callbacks=[ModelCheckpoint(filepath='checkpoints/weights_out')])\n    \"\"\"\n\n    def __init__(\n        self,\n        filepath: Optional[str] = None,\n        monitor: str = \"val_loss\",\n        min_delta: float = 0.0,\n        verbose: int = 0,\n        save_best_only: bool = False,\n        mode: str = \"auto\",\n        period: int = 1,\n        max_save: int = -1,\n    ):\n        super(ModelCheckpoint, self).__init__()\n\n        self.filepath = filepath\n        self.monitor = monitor\n        self.min_delta = min_delta\n        self.verbose = verbose\n        self.save_best_only = save_best_only\n        self.mode = mode\n        self.period = period\n        self.max_save = max_save\n\n        self.epochs_since_last_save = 0\n\n        if self.filepath:\n            if len(self.filepath.split(\"/\")[:-1]) == 0:\n                raise ValueError(\n                    \"'filepath' must be the full path to save the output weights,\"\n                    \" including the root of the filenames. e.g. 'checkpoints/weights_out'\"\n                )\n\n            root_dir = (\"/\").join(self.filepath.split(\"/\")[:-1])\n            if not os.path.exists(root_dir):\n                os.makedirs(root_dir)\n\n        if self.max_save &gt; 0:\n            self.old_files: List[str] = []\n\n        if self.mode not in [\"auto\", \"min\", \"max\"]:\n            warnings.warn(\n                \"ModelCheckpoint mode %s is unknown, \"\n                \"fallback to auto mode.\" % (self.mode),\n                RuntimeWarning,\n            )\n            self.mode = \"auto\"\n        if self.mode == \"min\":\n            self.monitor_op = np.less\n            self.best = np.Inf\n        elif self.mode == \"max\":\n            self.monitor_op = np.greater  # type: ignore[assignment]\n            self.best = -np.Inf\n        else:\n            if _is_metric(self.monitor):\n                self.monitor_op = np.greater  # type: ignore[assignment]\n                self.best = -np.Inf\n            else:\n                self.monitor_op = np.less\n                self.best = np.Inf\n\n        if self.monitor_op == np.greater:\n            self.min_delta *= 1\n        else:\n            self.min_delta *= -1\n\n    def on_epoch_end(  # noqa: C901\n        self, epoch: int, logs: Optional[Dict] = None, metric: Optional[float] = None\n    ):\n        logs = logs or {}\n        self.epochs_since_last_save += 1\n        if self.epochs_since_last_save &gt;= self.period:\n            self.epochs_since_last_save = 0\n            if self.filepath:\n                filepath = \"{}_{}.p\".format(self.filepath, epoch + 1)\n            if self.save_best_only:\n                current = logs.get(self.monitor)\n                if current is None:\n                    warnings.warn(\n                        \"Can save best model only with %s available, \"\n                        \"skipping.\" % (self.monitor),\n                        RuntimeWarning,\n                    )\n                else:\n                    if self.monitor_op(current - self.min_delta, self.best):\n                        if self.verbose &gt; 0:\n                            if self.filepath:\n                                print(\n                                    f\"\\nEpoch {epoch + 1}: {self.monitor} improved from {self.best:.5f} to {current:.5f} \"\n                                    f\"Saving model to {filepath}\"\n                                )\n                            else:\n                                print(\n                                    f\"\\nEpoch {epoch + 1}: {self.monitor} improved from {self.best:.5f} to {current:.5f} \"\n                                )\n                        self.best = current\n                        self.best_epoch = epoch\n                        self.best_state_dict = copy.deepcopy(self.model.state_dict())\n                        if self.filepath:\n                            torch.save(self.best_state_dict, filepath)\n                            if self.max_save &gt; 0:\n                                if len(self.old_files) == self.max_save:\n                                    try:\n                                        os.remove(self.old_files[0])\n                                    except FileNotFoundError:\n                                        pass\n                                    self.old_files = self.old_files[1:]\n                                self.old_files.append(filepath)\n                    else:\n                        if self.verbose &gt; 0:\n                            print(\n                                f\"\\nEpoch {epoch + 1}: {self.monitor} did not improve from {self.best:.5f} \"\n                                f\" considering a 'min_delta' improvement of {self.min_delta:.5f}\"\n                            )\n            if not self.save_best_only and self.filepath:\n                if self.verbose &gt; 0:\n                    print(\"\\nEpoch %05d: saving model to %s\" % (epoch + 1, filepath))\n                torch.save(self.model.state_dict(), filepath)\n                if self.max_save &gt; 0:\n                    if len(self.old_files) == self.max_save:\n                        try:\n                            os.remove(self.old_files[0])\n                        except FileNotFoundError:\n                            pass\n                        self.old_files = self.old_files[1:]\n                    self.old_files.append(filepath)\n\n    def __getstate__(self):\n        d = self.__dict__\n        self_dict = {k: d[k] for k in d if k not in [\"trainer\", \"model\"]}\n        return self_dict\n\n    def __setstate__(self, state):\n        self.__dict__ = state\n</code></pre>"},{"location":"pytorch-widedeep/callbacks.html#pytorch_widedeep.callbacks.EarlyStopping","title":"EarlyStopping","text":"<p>               Bases: <code>Callback</code></p> <p>Stop training when a monitored quantity has stopped improving.</p> <p>This class is almost identical to the corresponding keras class. Therefore, credit to the Keras Team.</p> <p>Callbacks are passed as input parameters to the <code>Trainer</code> class. See <code>pytorch_widedeep.trainer.Trainer</code></p> <p>Parameters:</p> Name Type Description Default <code>monitor</code> <code>str</code> <p>Quantity to monitor. Typically 'val_loss' or metric name (e.g. 'val_acc')</p> <code>'val_loss'</code> <code>min_delta</code> <code>float</code> <p>minimum change in the monitored quantity to qualify as an improvement, i.e. an absolute change of less than min_delta, will count as no improvement.</p> <code>0.0</code> <code>patience</code> <code>int</code> <p>Number of epochs that produced the monitored quantity with no improvement after which training will be stopped.</p> <code>10</code> <code>verbose</code> <code>int</code> <p>verbosity mode.</p> <code>0</code> <code>mode</code> <code>str</code> <p>one of {'auto', 'min', 'max'}. In 'min' mode, training will stop when the quantity monitored has stopped decreasing; in 'max' mode it will stop when the quantity monitored has stopped increasing; in 'auto' mode, the direction is automatically inferred from the name of the monitored quantity.</p> <code>'auto'</code> <code>baseline</code> <code>Optional[float]</code> <p>Baseline value for the monitored quantity to reach. Training will stop if the model does not show improvement over the baseline.</p> <code>None</code> <code>restore_best_weights</code> <code>bool</code> <p>Whether to restore model weights from the epoch with the best value of the monitored quantity. If <code>False</code>, the model weights obtained at the last step of training are used.</p> <code>False</code> <p>Attributes:</p> Name Type Description <code>best</code> <code>float</code> <p>best metric</p> <code>stopped_epoch</code> <code>int</code> <p>epoch when the training stopped</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; from pytorch_widedeep.callbacks import EarlyStopping\n&gt;&gt;&gt; from pytorch_widedeep.models import TabMlp, Wide, WideDeep\n&gt;&gt;&gt; from pytorch_widedeep.training import Trainer\n&gt;&gt;&gt;\n&gt;&gt;&gt; embed_input = [(u, i, j) for u, i, j in zip([\"a\", \"b\", \"c\"][:4], [4] * 3, [8] * 3)]\n&gt;&gt;&gt; column_idx = {k: v for v, k in enumerate([\"a\", \"b\", \"c\"])}\n&gt;&gt;&gt; wide = Wide(10, 1)\n&gt;&gt;&gt; deep = TabMlp(mlp_hidden_dims=[8, 4], column_idx=column_idx, cat_embed_input=embed_input)\n&gt;&gt;&gt; model = WideDeep(wide, deep)\n&gt;&gt;&gt; trainer = Trainer(model, objective=\"regression\", callbacks=[EarlyStopping(patience=10)])\n</code></pre> Source code in <code>pytorch_widedeep/callbacks.py</code> <pre><code>class EarlyStopping(Callback):\n    r\"\"\"Stop training when a monitored quantity has stopped improving.\n\n    This class is almost identical to the corresponding keras class.\n    Therefore, **credit** to the Keras Team.\n\n    Callbacks are passed as input parameters to the `Trainer` class. See\n    `pytorch_widedeep.trainer.Trainer`\n\n    Parameters\n    -----------\n    monitor: str, default='val_loss'.\n        Quantity to monitor. Typically _'val_loss'_ or metric name\n        (e.g. _'val_acc'_)\n    min_delta: float, default=0.\n        minimum change in the monitored quantity to qualify as an\n        improvement, i.e. an absolute change of less than min_delta, will\n        count as no improvement.\n    patience: int, default=10.\n        Number of epochs that produced the monitored quantity with no\n        improvement after which training will be stopped.\n    verbose: int.\n        verbosity mode.\n    mode: str, default='auto'\n        one of _{'auto', 'min', 'max'}_. In _'min'_ mode, training will\n        stop when the quantity monitored has stopped decreasing; in _'max'_\n        mode it will stop when the quantity monitored has stopped increasing;\n        in _'auto'_ mode, the direction is automatically inferred from the\n        name of the monitored quantity.\n    baseline: float, Optional. default=None.\n        Baseline value for the monitored quantity to reach. Training will\n        stop if the model does not show improvement over the baseline.\n    restore_best_weights: bool, default=None\n        Whether to restore model weights from the epoch with the best\n        value of the monitored quantity. If `False`, the model weights\n        obtained at the last step of training are used.\n\n    Attributes\n    ----------\n    best: float\n        best metric\n    stopped_epoch: int\n        epoch when the training stopped\n\n    Examples\n    --------\n    &gt;&gt;&gt; from pytorch_widedeep.callbacks import EarlyStopping\n    &gt;&gt;&gt; from pytorch_widedeep.models import TabMlp, Wide, WideDeep\n    &gt;&gt;&gt; from pytorch_widedeep.training import Trainer\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; embed_input = [(u, i, j) for u, i, j in zip([\"a\", \"b\", \"c\"][:4], [4] * 3, [8] * 3)]\n    &gt;&gt;&gt; column_idx = {k: v for v, k in enumerate([\"a\", \"b\", \"c\"])}\n    &gt;&gt;&gt; wide = Wide(10, 1)\n    &gt;&gt;&gt; deep = TabMlp(mlp_hidden_dims=[8, 4], column_idx=column_idx, cat_embed_input=embed_input)\n    &gt;&gt;&gt; model = WideDeep(wide, deep)\n    &gt;&gt;&gt; trainer = Trainer(model, objective=\"regression\", callbacks=[EarlyStopping(patience=10)])\n    \"\"\"\n\n    def __init__(\n        self,\n        monitor: str = \"val_loss\",\n        min_delta: float = 0.0,\n        patience: int = 10,\n        verbose: int = 0,\n        mode: str = \"auto\",\n        baseline: Optional[float] = None,\n        restore_best_weights: bool = False,\n    ):\n        super(EarlyStopping, self).__init__()\n\n        self.monitor = monitor\n        self.min_delta = min_delta\n        self.patience = patience\n        self.verbose = verbose\n        self.mode = mode\n        self.baseline = baseline\n        self.restore_best_weights = restore_best_weights\n\n        self.wait = 0\n        self.stopped_epoch = 0\n        self.state_dict = None\n\n        if self.mode not in [\"auto\", \"min\", \"max\"]:\n            warnings.warn(\n                \"EarlyStopping mode %s is unknown, \"\n                \"fallback to auto mode.\" % self.mode,\n                RuntimeWarning,\n            )\n            self.mode = \"auto\"\n\n        if self.mode == \"min\":\n            self.monitor_op = np.less\n        elif self.mode == \"max\":\n            self.monitor_op = np.greater  # type: ignore[assignment]\n        else:\n            if _is_metric(self.monitor):\n                self.monitor_op = np.greater  # type: ignore[assignment]\n            else:\n                self.monitor_op = np.less\n\n        if self.monitor_op == np.greater:\n            self.min_delta *= 1\n        else:\n            self.min_delta *= -1\n\n    def on_train_begin(self, logs: Optional[Dict] = None):\n        # Allow instances to be re-used\n        self.wait = 0\n        self.stopped_epoch = 0\n        if self.baseline is not None:\n            self.best = self.baseline\n        else:\n            self.best = np.Inf if self.monitor_op == np.less else -np.Inf\n\n    def on_epoch_end(\n        self, epoch: int, logs: Optional[Dict] = None, metric: Optional[float] = None\n    ):\n        current = self.get_monitor_value(logs)\n        if current is None:\n            return\n\n        if self.monitor_op(current - self.min_delta, self.best):\n            self.best = current\n            self.wait = 0\n            self.best_epoch = epoch\n            if self.restore_best_weights:\n                self.state_dict = copy.deepcopy(self.model.state_dict())\n        else:\n            self.wait += 1\n            if self.wait &gt;= self.patience:\n                self.stopped_epoch = epoch\n                self.trainer.early_stop = True\n\n    def on_train_end(self, logs: Optional[Dict] = None):\n        if self.stopped_epoch &gt; 0 and self.verbose &gt; 0:\n            print(\n                f\"Best Epoch: {self.best_epoch + 1}. Best {self.monitor}: {self.best:.5f}\"\n            )\n        if self.restore_best_weights and self.state_dict is not None:\n            if self.verbose &gt; 0:\n                print(\"Restoring model weights from the end of the best epoch\")\n            self.model.load_state_dict(self.state_dict)\n\n    def get_monitor_value(self, logs):\n        monitor_value = logs.get(self.monitor)\n        if monitor_value is None:\n            warnings.warn(\n                \"Early stopping conditioned on metric `%s` \"\n                \"which is not available. Available metrics are: %s\"\n                % (self.monitor, \",\".join(list(logs.keys()))),\n                RuntimeWarning,\n            )\n        return monitor_value\n\n    def __getstate__(self):\n        d = self.__dict__\n        self_dict = {k: d[k] for k in d if k not in [\"trainer\", \"model\"]}\n        return self_dict\n\n    def __setstate__(self, state):\n        self.__dict__ = state\n</code></pre>"},{"location":"pytorch-widedeep/dataloaders.html","title":"Dataloaders","text":"<p> NOTE: This module should contain custom dataloaders  that the user might want to implement. At the moment <code>pytorch-widedeep</code>  offers one custom dataloader, <code>DataLoaderImbalanced</code>.</p>"},{"location":"pytorch-widedeep/dataloaders.html#pytorch_widedeep.dataloaders.DataLoaderImbalanced","title":"DataLoaderImbalanced","text":"<p>               Bases: <code>CustomDataLoader</code></p> <p>Class to load and shuffle batches with adjusted weights for imbalanced datasets. If the classes do not begin from 0 remapping is necessary. See here.</p> <p>Parameters:</p> Name Type Description Default <code>dataset</code> <code>Optional[WideDeepDataset]</code> <p>see <code>pytorch_widedeep.training._wd_dataset</code></p> <code>None</code> <p>Other Parameters:</p> Name Type Description <code>*args</code> <p>Positional arguments to be passed to the parent CustomDataLoader.</p> <code>**kwargs</code> <p>This can include any parameter that can be passed to the 'standard' pytorch DataLoader and that is not already explicitely passed to the class. In addition, the dictionary can also include the extra parameter <code>oversample_mul</code> which will multiply the number of samples of the minority class to be sampled by the <code>WeightedRandomSampler</code>.</p> <p>In other words, the <code>num_samples</code> param in <code>WeightedRandomSampler</code> will be defined as:</p> \\[ minority \\space class \\space count \\times number \\space of \\space classes \\times oversample\\_mul \\] Source code in <code>pytorch_widedeep/dataloaders.py</code> <pre><code>class DataLoaderImbalanced(CustomDataLoader):\n    r\"\"\"Class to load and shuffle batches with adjusted weights for imbalanced\n    datasets. If the classes do not begin from 0 remapping is necessary. See\n    [here](https://towardsdatascience.com/pytorch-tabular-multiclass-classification-9f8211a123ab).\n\n    Parameters\n    ----------\n    dataset: `WideDeepDataset`\n        see `pytorch_widedeep.training._wd_dataset`\n\n    Other Parameters\n    ----------------\n    *args: Any\n        Positional arguments to be passed to the parent CustomDataLoader.\n    **kwargs: Dict\n        This can include any parameter that can be passed to the _'standard'_\n        pytorch\n        [DataLoader](https://pytorch.org/docs/stable/data.html#torch.utils.data.DataLoader)\n        and that is not already explicitely passed to the class. In addition,\n        the dictionary can also include the extra parameter `oversample_mul` which\n        will multiply the number of samples of the minority class to be sampled by\n        the [`WeightedRandomSampler`](https://pytorch.org/docs/stable/data.html#torch.utils.data.WeightedRandomSampler).\n\n        In other words, the `num_samples` param in `WeightedRandomSampler` will be defined as:\n\n        $$\n        minority \\space class \\space count \\times number \\space of \\space classes \\times oversample\\_mul\n        $$\n    \"\"\"\n\n    def __init__(self, dataset: Optional[WideDeepDataset] = None, *args, **kwargs):\n        self.args = args\n        self.kwargs = kwargs\n\n        if dataset is not None:\n            self._setup_sampler(dataset)\n            super().__init__(dataset, *args, sampler=self.sampler, **kwargs)\n        else:\n            super().__init__()\n\n    def set_dataset(self, dataset: WideDeepDataset):\n        sampler = self._setup_sampler(dataset)\n        # update the kwargs with the new sampler\n        self.kwargs[\"sampler\"] = sampler\n        super().set_dataset(dataset)\n\n    def _setup_sampler(self, dataset: WideDeepDataset) -&gt; WeightedRandomSampler:\n        assert dataset.Y is not None, (\n            \"The 'dataset' instance of WideDeepDataset must contain a \"\n            \"target array 'Y'\"\n        )\n\n        oversample_mul = self.kwargs.pop(\"oversample_mul\", 1)\n        weights, minor_cls_cnt, num_clss = get_class_weights(dataset)\n        num_samples = int(minor_cls_cnt * num_clss * oversample_mul)\n        samples_weight = list(np.array([weights[i] for i in dataset.Y]))\n        sampler = WeightedRandomSampler(samples_weight, num_samples, replacement=True)\n        return sampler\n</code></pre>"},{"location":"pytorch-widedeep/load_from_folder.html","title":"The <code>load_from_folder</code> module","text":"<p>The <code>load_from_folder</code> module contains the classes that are necessary to load data from disk and these are inspired by the <code>ImageFolder</code> class in the <code>torchvision</code> library. This module is designed with one specific case in mind. Such case is the following: given a multi-modal dataset with tabular data, images and text, the images do not fit in memory, and therefore, they have to be loaded from disk. However, as any other functionality in this library, there is some flexibility and some additional cases can also be addressed using this module.</p> <p>For this module to be used, the datasets must be prepared in a certain way:</p> <ol> <li> <p>the tabular data must contain a column with the images names as stored in disk, including the extension (<code>.jpg</code>, <code>.png</code>, etc...).</p> </li> <li> <p>Regarding to the text dataset, the tabular data can contain a column with the texts themselves or the names of the files containing the texts as stored in disk.</p> </li> </ol> <p>The tabular data might or might not fit in disk itself. If it does not, please see the <code>ChunkPreprocessor</code> utilities at the <code>preprocessing</code> module and the examples folder in the repo, which illustrate such case. Finally note that only <code>csv</code> format is currently supported in that case (more formats might come soon).</p>"},{"location":"pytorch-widedeep/load_from_folder.html#pytorch_widedeep.load_from_folder.tabular.tabular_from_folder.TabFromFolder","title":"TabFromFolder","text":"<p>This class is used to load tabular data from disk. The current constrains are:</p> <ol> <li>The only file format supported right now is csv</li> <li>The csv file must contain headers</li> </ol> <p>For examples, please, see the examples folder in the repo.</p> <p>Parameters:</p> Name Type Description Default <code>fname</code> <code>str</code> <p>the name of the csv file</p> required <code>directory</code> <code>Optional[str]</code> <p>the path to the directory where the csv file is located. If None, a <code>TabFromFolder</code> reference object must be provided</p> <code>None</code> <code>target_col</code> <code>Optional[str]</code> <p>the name of the target column. If None, a <code>TabFromFolder</code> reference object must be provided</p> <code>None</code> <code>preprocessor</code> <code>Optional[TabularPreprocessor]</code> <p>a fitted <code>TabularPreprocessor</code> object. If None, a <code>TabFromFolder</code> reference object must be provided</p> <code>None</code> <code>text_col</code> <code>Optional[Union[str, List[str]]]</code> <p>the name of the column with the texts themselves or the names of the files that contain the text dataset. If None, either there is no text column or a <code>TabFromFolder</code> reference object must be provided</p> <code>None</code> <code>img_col</code> <code>Optional[Union[str, List[str]]]</code> <p>the name of the column with the the names of the images. If None, either there is no image column or a <code>TabFromFolder</code> reference object must be provided</p> <code>None</code> <code>ignore_target</code> <code>bool</code> <p>whether to ignore the target column. This is normally set to True when this class is used for a test dataset.</p> <code>False</code> <code>reference</code> <code>Optional[Any]</code> <p>a reference <code>TabFromFolder</code> object. If provided, the <code>TabFromFolder</code> object will be created using the attributes of the reference object. This is useful to instantiate a <code>TabFromFolder</code> object for evaluation or test purposes</p> <code>None</code> <code>verbose</code> <code>Optional[int]</code> <p>verbosity. If 0, no output will be printed during the process.</p> <code>1</code> Source code in <code>pytorch_widedeep/load_from_folder/tabular/tabular_from_folder.py</code> <pre><code>class TabFromFolder:\n    \"\"\"\n    This class is used to load tabular data from disk. The current constrains are:\n\n    1. The only file format supported right now is csv\n    2. The csv file must contain headers\n\n    For examples, please, see the examples folder in the repo.\n\n    Parameters\n    ----------\n    fname: str\n        the name of the csv file\n    directory: str, Optional, default = None\n        the path to the directory where the csv file is located. If None,\n        a `TabFromFolder` reference object must be provided\n    target_col: str, Optional, default = None\n        the name of the target column. If None, a `TabFromFolder` reference\n        object must be provided\n    preprocessor: `TabularPreprocessor`, Optional, default = None\n        a fitted `TabularPreprocessor` object. If None, a `TabFromFolder`\n        reference object must be provided\n    text_col: str, Optional, default = None\n        the name of the column with the texts themselves or the names of the\n        files that contain the text dataset. If None, either there is no text\n        column or a `TabFromFolder` reference object must be provided\n    img_col: str, Optional, default = None\n        the name of the column with the the names of the images. If None,\n        either there is no image column or a `TabFromFolder` reference object\n        must be provided\n    ignore_target: bool, default = False\n        whether to ignore the target column. This is normally set to True when\n        this class is used for a test dataset.\n    reference: `TabFromFolder`, Optional, default = None\n        a reference `TabFromFolder` object. If provided, the `TabFromFolder`\n        object will be created using the attributes of the reference object.\n        This is useful to instantiate a `TabFromFolder` object for evaluation\n        or test purposes\n    verbose: int, default = 1\n        verbosity. If 0, no output will be printed during the process.\n    \"\"\"\n\n    def __init__(\n        self,\n        fname: str,\n        directory: Optional[str] = None,\n        target_col: Optional[str] = None,\n        preprocessor: Optional[TabularPreprocessor] = None,\n        text_col: Optional[Union[str, List[str]]] = None,\n        img_col: Optional[Union[str, List[str]]] = None,\n        ignore_target: bool = False,\n        reference: Optional[Any] = None,  # is Type[\"TabFromFolder\"],\n        verbose: Optional[int] = 1,\n    ):\n        self.fname = fname\n        self.ignore_target = ignore_target\n        self.verbose = verbose\n\n        if reference is not None:\n            (\n                self.directory,\n                self.target_col,\n                self.preprocessor,\n                self.text_col,\n                self.img_col,\n            ) = self._set_from_reference(reference, preprocessor)\n        else:\n            assert (\n                directory is not None\n                and (target_col is not None and not ignore_target)\n                and preprocessor is not None\n            ), (\n                \"if no reference is provided, 'directory', 'target_col' and 'preprocessor' \"\n                \"must be provided\"\n            )\n\n            self.directory = directory\n            self.target_col = target_col\n            self.preprocessor = preprocessor\n            self.text_col = text_col\n            self.img_col = img_col\n\n        assert (\n            self.preprocessor.is_fitted\n        ), \"The preprocessor must be fitted before passing it to this class\"\n\n    def get_item(self, idx: int) -&gt; Tuple[  # noqa: C901\n        np.ndarray,\n        Optional[Union[str, List[str]]],\n        Optional[Union[str, List[str]]],\n        Optional[Union[int, float]],\n    ]:\n        \"\"\"\n        This method is used to retrieve a sample from the csv file\n\n        Parameters\n        ----------\n        idx: int\n            the index of the sample to retrieve\n\n        Returns\n        -------\n        Tuple\n            a tuple with the processed tabular data, the text data and/or the\n            image data, and the target variable\n        \"\"\"\n\n        path = os.path.join(self.directory, self.fname)\n\n        try:\n            if not hasattr(self, \"colnames\"):\n                self.colnames = pd.read_csv(path, nrows=0).columns.tolist()\n\n            # TO DO: we need to look into this as the treatment is different\n            # whether the csv contains headers or not. For the time being we\n            # will require that the csv file has headers\n\n            _sample = pd.read_csv(\n                path, skiprows=lambda x: x != idx + 1, header=None\n            ).values\n            sample = pd.DataFrame(_sample, columns=self.colnames)\n        except Exception:\n            raise ValueError(\"Currently only csv format is supported.\")\n\n        text_fnames_or_text: Optional[Union[str, List[str]]] = None\n        if self.text_col is not None:\n            if isinstance(self.text_col, list):\n                text_fnames_or_text = [\n                    sample[col].to_list()[0] for col in self.text_col\n                ]\n            else:\n                text_fnames_or_text = sample[self.text_col].to_list()[0]\n\n        img_fname: Optional[Union[str, List[str]]] = None\n        if self.img_col is not None:\n            if isinstance(self.img_col, list):\n                img_fname = [sample[col].to_list()[0] for col in self.img_col]\n            else:\n                img_fname = sample[self.img_col].to_list()[0]\n\n        processed_sample = self.preprocessor.transform_sample(sample)\n\n        if not self.ignore_target:\n            target = sample[self.target_col].to_list()[0]\n        else:\n            target = None\n\n        return processed_sample, text_fnames_or_text, img_fname, target\n\n    def _set_from_reference(\n        self,\n        reference: Any,  # is Type[\"TabFromFolder\"],\n        preprocessor: Optional[TabularPreprocessor],\n    ) -&gt; Tuple[\n        str,\n        str,\n        TabularPreprocessor,\n        Optional[Union[str, List[str]]],\n        Optional[Union[str, List[str]]],\n    ]:\n        (\n            directory,\n            target_col,\n            _preprocessor,\n            text_col,\n            img_col,\n        ) = self._get_from_reference(reference)\n\n        if preprocessor is not None:\n            preprocessor = preprocessor\n            if self.verbose:\n                UserWarning(\n                    \"The preprocessor from the reference object is overwritten \"\n                    \"by the provided preprocessor\"\n                )\n        else:\n            preprocessor = _preprocessor\n\n        return directory, target_col, preprocessor, text_col, img_col\n\n    @staticmethod\n    def _get_from_reference(\n        reference: Any,  # is Type[\"TabFromFolder\"],\n    ) -&gt; Tuple[str, str, TabularPreprocessor, Optional[str], Optional[str]]:\n        return (\n            reference.directory,\n            reference.target_col,\n            reference.preprocessor,\n            reference.text_col,\n            reference.img_col,\n        )\n\n    def __repr__(self) -&gt; str:  # noqa: C901\n        list_of_params: List[str] = []\n        if self.fname is not None:\n            list_of_params.append(\"fname={fname}\")\n        if self.directory is not None:\n            list_of_params.append(\"directory={directory}\")\n        if self.target_col is not None:\n            list_of_params.append(\"target_col={target_col}\")\n        if self.preprocessor is not None:\n            list_of_params.append(\n                f\"preprocessor={self.preprocessor.__class__.__name__}\"\n            )\n        if self.text_col is not None:\n            if isinstance(self.text_col, list):\n                list_of_params.append(\n                    f\"text_col={[text_col for text_col in self.text_col]}\"\n                )\n            else:\n                list_of_params.append(\"text_col={text_col}\")\n        if self.img_col is not None:\n            if isinstance(self.img_col, list):\n                list_of_params.append(\n                    f\"img_col={[img_col for img_col in self.img_col]}\"\n                )\n            else:\n                list_of_params.append(\"img_col={img_col}\")\n        if self.ignore_target is not None:\n            list_of_params.append(\"ignore_target={ignore_target}\")\n        if self.verbose is not None:\n            list_of_params.append(\"verbose={verbose}\")\n        all_params = \", \".join(list_of_params)\n        return f\"{self.__class__.__name__}({all_params.format(**self.__dict__)})\"\n</code></pre>"},{"location":"pytorch-widedeep/load_from_folder.html#pytorch_widedeep.load_from_folder.tabular.tabular_from_folder.TabFromFolder.get_item","title":"get_item","text":"<pre><code>get_item(idx)\n</code></pre> <p>This method is used to retrieve a sample from the csv file</p> <p>Parameters:</p> Name Type Description Default <code>idx</code> <code>int</code> <p>the index of the sample to retrieve</p> required <p>Returns:</p> Type Description <code>Tuple</code> <p>a tuple with the processed tabular data, the text data and/or the image data, and the target variable</p> Source code in <code>pytorch_widedeep/load_from_folder/tabular/tabular_from_folder.py</code> <pre><code>def get_item(self, idx: int) -&gt; Tuple[  # noqa: C901\n    np.ndarray,\n    Optional[Union[str, List[str]]],\n    Optional[Union[str, List[str]]],\n    Optional[Union[int, float]],\n]:\n    \"\"\"\n    This method is used to retrieve a sample from the csv file\n\n    Parameters\n    ----------\n    idx: int\n        the index of the sample to retrieve\n\n    Returns\n    -------\n    Tuple\n        a tuple with the processed tabular data, the text data and/or the\n        image data, and the target variable\n    \"\"\"\n\n    path = os.path.join(self.directory, self.fname)\n\n    try:\n        if not hasattr(self, \"colnames\"):\n            self.colnames = pd.read_csv(path, nrows=0).columns.tolist()\n\n        # TO DO: we need to look into this as the treatment is different\n        # whether the csv contains headers or not. For the time being we\n        # will require that the csv file has headers\n\n        _sample = pd.read_csv(\n            path, skiprows=lambda x: x != idx + 1, header=None\n        ).values\n        sample = pd.DataFrame(_sample, columns=self.colnames)\n    except Exception:\n        raise ValueError(\"Currently only csv format is supported.\")\n\n    text_fnames_or_text: Optional[Union[str, List[str]]] = None\n    if self.text_col is not None:\n        if isinstance(self.text_col, list):\n            text_fnames_or_text = [\n                sample[col].to_list()[0] for col in self.text_col\n            ]\n        else:\n            text_fnames_or_text = sample[self.text_col].to_list()[0]\n\n    img_fname: Optional[Union[str, List[str]]] = None\n    if self.img_col is not None:\n        if isinstance(self.img_col, list):\n            img_fname = [sample[col].to_list()[0] for col in self.img_col]\n        else:\n            img_fname = sample[self.img_col].to_list()[0]\n\n    processed_sample = self.preprocessor.transform_sample(sample)\n\n    if not self.ignore_target:\n        target = sample[self.target_col].to_list()[0]\n    else:\n        target = None\n\n    return processed_sample, text_fnames_or_text, img_fname, target\n</code></pre>"},{"location":"pytorch-widedeep/load_from_folder.html#pytorch_widedeep.load_from_folder.tabular.tabular_from_folder.WideFromFolder","title":"WideFromFolder","text":"<p>               Bases: <code>TabFromFolder</code></p> <p>This class is mostly identical to <code>TabFromFolder</code> but exists because we want to separate the treatment of the wide and the deep tabular components</p> <p>Parameters:</p> Name Type Description Default <code>fname</code> <code>str</code> <p>the name of the csv file</p> required <code>directory</code> <code>Optional[str]</code> <p>the path to the directory where the csv file is located. If None, a <code>WideFromFolder</code> reference object must be provided</p> <code>None</code> <code>target_col</code> <code>Optional[str]</code> <p>the name of the target column. If None, a <code>WideFromFolder</code> reference object must be provided</p> <code>None</code> <code>preprocessor</code> <code>Optional[TabularPreprocessor]</code> <p>a fitted <code>TabularPreprocessor</code> object. If None, a <code>WideFromFolder</code> reference object must be provided</p> <code>None</code> <code>text_col</code> <code>Optional[str]</code> <p>the name of the column with the texts themselves or the names of the files that contain the text dataset. If None, either there is no text column or a <code>WideFromFolder</code> reference object must be provided=</p> <code>None</code> <code>img_col</code> <code>Optional[str]</code> <p>the name of the column with the the names of the images. If None, either there is no image column or a <code>WideFromFolder</code> reference object must be provided</p> <code>None</code> <code>ignore_target</code> <code>bool</code> <p>whether to ignore the target column. This is normally used when this class is used for a test dataset.</p> <code>False</code> <code>reference</code> <code>Optional[Any]</code> <p>a reference <code>WideFromFolder</code> object. If provided, the <code>WideFromFolder</code> object will be created using the attributes of the reference object. This is useful to instantiate a <code>WideFromFolder</code> object for evaluation or test purposes</p> <code>None</code> <code>verbose</code> <code>int</code> <p>verbosity. If 0, no output will be printed during the process.</p> <code>1</code> Source code in <code>pytorch_widedeep/load_from_folder/tabular/tabular_from_folder.py</code> <pre><code>class WideFromFolder(TabFromFolder):\n    \"\"\"\n    This class is mostly identical to `TabFromFolder` but exists because we\n    want to separate the treatment of the wide and the deep tabular\n    components\n\n    Parameters\n    ----------\n    fname: str\n        the name of the csv file\n    directory: str, Optional, default = None\n        the path to the directory where the csv file is located. If None,\n        a `WideFromFolder` reference object must be provided\n    target_col: str, Optional, default = None\n        the name of the target column. If None, a `WideFromFolder` reference\n        object must be provided\n    preprocessor: `TabularPreprocessor`, Optional, default = None\n        a fitted `TabularPreprocessor` object. If None, a `WideFromFolder`\n        reference object must be provided\n    text_col: str, Optional, default = None\n        the name of the column with the texts themselves or the names of the\n        files that contain the text dataset. If None, either there is no text\n        column or a `WideFromFolder` reference object must be provided=\n    img_col: str, Optional, default = None\n        the name of the column with the the names of the images. If None,\n        either there is no image column or a `WideFromFolder` reference object\n        must be provided\n    ignore_target: bool, default = False\n        whether to ignore the target column. This is normally used when this\n        class is used for a test dataset.\n    reference: `WideFromFolder`, Optional, default = None\n        a reference `WideFromFolder` object. If provided, the `WideFromFolder`\n        object will be created using the attributes of the reference object.\n        This is useful to instantiate a `WideFromFolder` object for evaluation\n        or test purposes\n    verbose: int, default = 1\n        verbosity. If 0, no output will be printed during the process.\n    \"\"\"\n\n    def __init__(\n        self,\n        fname: str,\n        directory: Optional[str] = None,\n        target_col: Optional[str] = None,\n        preprocessor: Optional[TabularPreprocessor] = None,\n        text_col: Optional[str] = None,\n        img_col: Optional[str] = None,\n        ignore_target: bool = False,\n        reference: Optional[Any] = None,  # is Type[\"WideFromFolder\"],\n        verbose: int = 1,\n    ):\n        super(WideFromFolder, self).__init__(\n            fname=fname,\n            directory=directory,\n            target_col=target_col,\n            preprocessor=preprocessor,\n            text_col=text_col,\n            img_col=img_col,\n            reference=reference,\n            ignore_target=ignore_target,\n            verbose=verbose,\n        )\n</code></pre>"},{"location":"pytorch-widedeep/load_from_folder.html#pytorch_widedeep.load_from_folder.text.text_from_folder.TextFromFolder","title":"TextFromFolder","text":"<p>This class is used to load the text dataset (i.e. the text files) from a folder, or to retrieve the text given a texts column specified within the preprocessor object.</p> <p>For examples, please, see the examples folder in the repo.</p> <p>Parameters:</p> Name Type Description Default <code>preprocessor</code> <code>Union[TextPreprocessor, ChunkTextPreprocessor, HFPreprocessor, ChunkHFPreprocessor, List[TextPreprocessor], List[ChunkTextPreprocessor], List[HFPreprocessor], List[ChunkHFPreprocessor]]</code> <p>The preprocessor used to process the text. It must be fitted before using this class</p> required Source code in <code>pytorch_widedeep/load_from_folder/text/text_from_folder.py</code> <pre><code>class TextFromFolder:\n    \"\"\"\n    This class is used to load the text dataset (i.e. the text files) from a\n    folder, or to retrieve the text given a texts column specified within the\n    preprocessor object.\n\n    For examples, please, see the examples folder in the repo.\n\n    Parameters\n    ----------\n    preprocessor:\n        The preprocessor used to process the text. It must be fitted before using\n        this class\n    \"\"\"\n\n    def __init__(\n        self,\n        preprocessor: Union[\n            TextPreprocessor,\n            ChunkTextPreprocessor,\n            HFPreprocessor,\n            ChunkHFPreprocessor,\n            List[TextPreprocessor],\n            List[ChunkTextPreprocessor],\n            List[HFPreprocessor],\n            List[ChunkHFPreprocessor],\n        ],\n    ):\n        if isinstance(preprocessor, list):\n            for p in preprocessor:\n                assert (\n                    p.is_fitted\n                ), \"All preprocessors must be fitted before using this class\"\n        else:\n            assert (\n                preprocessor.is_fitted\n            ), \"The preprocessor must be fitted before using this class\"\n\n        self.preprocessor = preprocessor\n\n    def get_item(\n        self, text: Union[str, List[str]]\n    ) -&gt; Union[np.ndarray, List[np.ndarray]]:\n        \"\"\"\n        Given a text or a list of texts corresponding to different text\n        columns, this method will return the processed text or a list of\n        processed texts\n\n        Parameters\n        ----------\n        text: Union[str, List[str]]\n            The text or list of texts to be processed\n\n        Returns\n        -------\n        Union[np.ndarray, List[np.ndarray]]\n            The processed text or a list of processed texts\n        \"\"\"\n        if isinstance(self.preprocessor, list):\n            assert isinstance(text, list)\n            processed_sample: Union[np.ndarray, List[np.ndarray]] = [\n                self._preprocess_one_sample(t, self.preprocessor[i])\n                for i, t in enumerate(text)\n            ]\n        else:\n            assert isinstance(text, str)\n            processed_sample = self._preprocess_one_sample(text, self.preprocessor)\n\n        return processed_sample\n\n    def _preprocess_one_sample(\n        self,\n        text: str,\n        preprocessor: Union[\n            TextPreprocessor,\n            ChunkTextPreprocessor,\n            HFPreprocessor,\n            ChunkHFPreprocessor,\n        ],\n    ) -&gt; np.ndarray:\n        if (\n            isinstance(preprocessor, ChunkTextPreprocessor)\n            and preprocessor.root_dir is not None\n        ):\n            path = os.path.join(preprocessor.root_dir, text)\n\n            with open(path, \"r\") as f:\n                sample = f.read().replace(\"\\n\", \"\")\n        else:\n            sample = text\n\n        processed_sample = preprocessor.transform_sample(sample)\n\n        return processed_sample\n\n    def __repr__(self):\n        if isinstance(self.preprocessor, list):\n            return f\"{self.__class__.__name__}({[p.__class__.__name__ for p in self.preprocessor]})\"\n        else:\n            return f\"{self.__class__.__name__}({self.preprocessor.__class__.__name__})\"\n</code></pre>"},{"location":"pytorch-widedeep/load_from_folder.html#pytorch_widedeep.load_from_folder.text.text_from_folder.TextFromFolder.get_item","title":"get_item","text":"<pre><code>get_item(text)\n</code></pre> <p>Given a text or a list of texts corresponding to different text columns, this method will return the processed text or a list of processed texts</p> <p>Parameters:</p> Name Type Description Default <code>text</code> <code>Union[str, List[str]]</code> <p>The text or list of texts to be processed</p> required <p>Returns:</p> Type Description <code>Union[ndarray, List[ndarray]]</code> <p>The processed text or a list of processed texts</p> Source code in <code>pytorch_widedeep/load_from_folder/text/text_from_folder.py</code> <pre><code>def get_item(\n    self, text: Union[str, List[str]]\n) -&gt; Union[np.ndarray, List[np.ndarray]]:\n    \"\"\"\n    Given a text or a list of texts corresponding to different text\n    columns, this method will return the processed text or a list of\n    processed texts\n\n    Parameters\n    ----------\n    text: Union[str, List[str]]\n        The text or list of texts to be processed\n\n    Returns\n    -------\n    Union[np.ndarray, List[np.ndarray]]\n        The processed text or a list of processed texts\n    \"\"\"\n    if isinstance(self.preprocessor, list):\n        assert isinstance(text, list)\n        processed_sample: Union[np.ndarray, List[np.ndarray]] = [\n            self._preprocess_one_sample(t, self.preprocessor[i])\n            for i, t in enumerate(text)\n        ]\n    else:\n        assert isinstance(text, str)\n        processed_sample = self._preprocess_one_sample(text, self.preprocessor)\n\n    return processed_sample\n</code></pre>"},{"location":"pytorch-widedeep/load_from_folder.html#pytorch_widedeep.load_from_folder.image.image_from_folder.ImageFromFolder","title":"ImageFromFolder","text":"<p>This class is used to load the image dataset from disk. It is inspired by the <code>ImageFolder</code> class at the <code>torchvision</code> library. Here, we have simply adapted to work within the context of a Wide and Deep multi-modal model.</p> <p>For examples, please, see the examples folder in the repo.</p> <p>Parameters:</p> Name Type Description Default <code>directory</code> <code>Optional[Union[str, List[str]]]</code> <p>the path to the directory where the images are located. If None, a preprocessor must be provided.</p> <code>None</code> <code>preprocessor</code> <code>Optional[Union[ImagePreprocessor, List[ImagePreprocessor]]]</code> <p>a fitted <code>ImagePreprocessor</code> object.</p> <code>None</code> <code>loader</code> <code>Callable[[str], Any]</code> <p>a function to load a sample given its path.</p> <code>default_loader</code> <code>extensions</code> <code>Optional[Tuple[str, ...]]</code> <p>a tuple with the allowed extensions. If None, IMG_EXTENSIONS will be used where IMG_EXTENSIONS =\".jpg\", \".jpeg\", \".png\", \".ppm\", \".bmp\", \".pgm\", \".tif\", \".tiff\", \".webp\"</p> <code>None</code> <code>transforms</code> <code>Optional[Any]</code> <p>a <code>torchvision.transforms</code> object. If None, this class will simply return an array representation of the PIL Image</p> <code>None</code> Source code in <code>pytorch_widedeep/load_from_folder/image/image_from_folder.py</code> <pre><code>class ImageFromFolder:\n    \"\"\"\n    This class is used to load the image dataset from disk. It is inspired by\n    the `ImageFolder` class at the `torchvision` library. Here, we have\n    simply adapted to work within the context of a Wide and Deep multi-modal\n    model.\n\n    For examples, please, see the examples folder in the repo.\n\n    Parameters\n    ----------\n    directory: str, Optional, default = None\n        the path to the directory where the images are located. If None, a\n        preprocessor must be provided.\n    preprocessor: `ImagePreprocessor`, Optional, default = None\n        a fitted `ImagePreprocessor` object.\n    loader: Callable[[str], Any], Optional, default = default_loader\n        a function to load a sample given its path.\n    extensions: Tuple[str, ...], Optional, default = IMG_EXTENSIONS\n        a tuple with the allowed extensions. If None, IMG_EXTENSIONS will be\n        used where IMG_EXTENSIONS\n        =\".jpg\", \".jpeg\", \".png\", \".ppm\", \".bmp\", \".pgm\", \".tif\", \".tiff\", \".webp\"\n    transforms: Optional[Any], default = None\n        a `torchvision.transforms` object. If None, this class will simply\n        return an array representation of the PIL Image\n    \"\"\"\n\n    def __init__(\n        self,\n        directory: Optional[Union[str, List[str]]] = None,\n        preprocessor: Optional[\n            Union[ImagePreprocessor, List[ImagePreprocessor]]\n        ] = None,\n        loader: Callable[[str], Any] = default_loader,\n        extensions: Optional[Tuple[str, ...]] = None,\n        transforms: Optional[Any] = None,\n    ) -&gt; None:\n        assert (\n            directory is not None or preprocessor is not None\n        ), \"Either a directory or an instance of ImagePreprocessor(s) must be provided\"\n\n        if directory is not None and preprocessor is not None:  # pragma: no cover\n            error_msg = (\n                \"If both 'directory' and 'preprocessor' are provided, the 'img_path' \"\n                \"attribute of the 'preprocessor' must be the same as the 'directory'\"\n            )\n            if isinstance(directory, list):\n                assert isinstance(preprocessor, list)\n                assert len(directory) == len(preprocessor)\n                for d, p in zip(directory, preprocessor):\n                    assert d == p.img_path, error_msg\n            else:\n                assert isinstance(preprocessor, ImagePreprocessor)\n                assert directory == preprocessor.img_path, error_msg\n\n        if directory is not None:\n            self.directory = directory\n        else:\n            assert (\n                preprocessor is not None\n            ), \"Either a directory or an instance of ImagePreprocessor must be provided\"\n            if isinstance(preprocessor, list):\n                self.directory = [p.img_path for p in preprocessor]\n            else:\n                self.directory = preprocessor.img_path\n\n        self.preprocessor = preprocessor\n        self.loader = loader\n        self.extensions = extensions if extensions is not None else IMG_EXTENSIONS\n        self.transforms = transforms\n        if self.transforms:\n            self.transforms_names = [\n                tr.__class__.__name__ for tr in self.transforms.transforms\n            ]\n        else:\n            self.transforms_names = []\n\n            self.transpose = True\n\n    def get_item(\n        self, fname: Union[str, List[str]]\n    ) -&gt; Union[np.ndarray, List[np.ndarray]]:\n        \"\"\"\n        This method is used to load the image dataset(s) from disk.\n\n        Parameters\n        ----------\n        fname: Union[str, List[str]]\n            the name of the image file(s) to load. If a list is provided, the\n            method will return a list of numpy arrays. Each element in the list\n            corresponds to the image in the different image columns.\n\n        Returns\n        -------\n        Union[np.ndarray, List[np.ndarray]]\n            a numpy array or a list of numpy arrays representing the image(s)\n        \"\"\"\n\n        if isinstance(fname, list):\n            if not isinstance(self.directory, list):\n                _directory = [self.directory] * len(fname)\n            else:\n                _directory = self.directory\n            if self.preprocessor is not None:\n                assert isinstance(self.preprocessor, list)\n                processed_sample: Union[np.ndarray, List[np.ndarray]] = [\n                    self._preprocess_one_sample(f, d, p)\n                    for f, d, p in zip(fname, _directory, self.preprocessor)\n                ]\n            else:\n                processed_sample = [\n                    self._preprocess_one_sample(f, d) for f, d in zip(fname, _directory)\n                ]\n        else:\n            assert isinstance(self.directory, str)\n            if self.preprocessor is not None:\n                assert isinstance(self.preprocessor, ImagePreprocessor)\n                processed_sample = self._preprocess_one_sample(\n                    fname, self.directory, self.preprocessor\n                )\n            else:\n                processed_sample = self._preprocess_one_sample(fname, self.directory)\n\n        return processed_sample\n\n    def _preprocess_one_sample(\n        self,\n        fname: str,\n        directory: str,\n        preprocessor: Optional[ImagePreprocessor] = None,\n    ) -&gt; np.ndarray:\n        assert has_file_allowed_extension(fname, self.extensions)\n\n        path = os.path.join(directory, fname)\n        sample = self.loader(path)\n\n        assert isinstance(sample, (Image.Image, np.ndarray)), (  # pragma: no cover\n            \"The loader must return an instance of PIL.Image or np.ndarray, \"\n            f\"got {type(sample)} instead\"\n        )\n\n        if preprocessor is not None:\n            if not isinstance(sample, np.ndarray):\n                processed_sample = preprocessor.transform_sample(np.asarray(sample))\n            else:\n                processed_sample = preprocessor.transform_sample(sample)\n        else:\n            processed_sample = sample  # type: ignore\n\n        prepared_sample = self._prepare_sample(processed_sample)\n\n        return prepared_sample\n\n    def _prepare_sample(  # noqa: C901\n        self, processed_sample: Union[np.ndarray, Image.Image]\n    ) -&gt; np.ndarray:\n        # if an image dataset is used, make sure is in the right format to\n        # be ingested by the conv layers\n\n        if isinstance(processed_sample, Image.Image):\n            processed_sample = np.asarray(processed_sample)\n\n        # if int must be uint8\n        if \"int\" in str(processed_sample.dtype) and \"uint8\" != str(\n            processed_sample.dtype\n        ):\n            processed_sample = processed_sample.astype(\"uint8\")\n\n        # if float must be float32\n        if \"float\" in str(processed_sample.dtype) and \"float32\" != str(\n            processed_sample.dtype\n        ):\n            processed_sample = processed_sample.astype(\"float32\")\n\n        # if there are no transforms, or these do not include ToTensor()\n        # (weird or unexpected case, not sure is even possible) then we need\n        # to  replicate what ToTensor() does -&gt; transpose axis and normalize if\n        # necessary\n        if not self.transforms or \"ToTensor\" not in self.transforms_names:\n            if processed_sample.ndim == 2:\n                processed_sample = processed_sample[:, :, None]\n\n            processed_sample = processed_sample.transpose(2, 0, 1)\n\n            if \"int\" in str(processed_sample.dtype):\n                processed_sample = (processed_sample / processed_sample.max()).astype(\n                    \"float32\"\n                )\n        elif \"ToTensor\" in self.transforms_names:\n            # if ToTensor() is included, simply apply transforms\n            assert self.transforms_names[0] == \"ToTensor\", (\n                \"If ToTensor() is included in the transforms, it must be the \"\n                \"first transform in the list\"\n            )\n            processed_sample = self.transforms(processed_sample)\n        else:\n            # else apply transforms on the result of calling torch.tensor on\n            # processed_sample after all the previous manipulation\n            processed_sample = self.transforms(torch.tensor(processed_sample))\n\n        return processed_sample  # type: ignore\n\n    def __repr__(self) -&gt; str:\n        list_of_params: List[str] = []\n        if self.directory is not None:\n            list_of_params.append(\"directory={directory}\")\n        if self.preprocessor is not None:\n            list_of_params.append(\n                f\"preprocessor={self.preprocessor.__class__.__name__}\"\n            )\n        if self.loader is not None:\n            list_of_params.append(f\"loader={self.loader.__name__}\")\n        if self.extensions is not None:\n            list_of_params.append(\"extensions={extensions}\")\n        if self.transforms is not None:\n            list_of_params.append(f\"transforms={self.transforms_names}\")\n        all_params = \", \".join(list_of_params)\n        return f\"ImageFromFolder({all_params.format(**self.__dict__)})\"\n</code></pre>"},{"location":"pytorch-widedeep/load_from_folder.html#pytorch_widedeep.load_from_folder.image.image_from_folder.ImageFromFolder.get_item","title":"get_item","text":"<pre><code>get_item(fname)\n</code></pre> <p>This method is used to load the image dataset(s) from disk.</p> <p>Parameters:</p> Name Type Description Default <code>fname</code> <code>Union[str, List[str]]</code> <p>the name of the image file(s) to load. If a list is provided, the method will return a list of numpy arrays. Each element in the list corresponds to the image in the different image columns.</p> required <p>Returns:</p> Type Description <code>Union[ndarray, List[ndarray]]</code> <p>a numpy array or a list of numpy arrays representing the image(s)</p> Source code in <code>pytorch_widedeep/load_from_folder/image/image_from_folder.py</code> <pre><code>def get_item(\n    self, fname: Union[str, List[str]]\n) -&gt; Union[np.ndarray, List[np.ndarray]]:\n    \"\"\"\n    This method is used to load the image dataset(s) from disk.\n\n    Parameters\n    ----------\n    fname: Union[str, List[str]]\n        the name of the image file(s) to load. If a list is provided, the\n        method will return a list of numpy arrays. Each element in the list\n        corresponds to the image in the different image columns.\n\n    Returns\n    -------\n    Union[np.ndarray, List[np.ndarray]]\n        a numpy array or a list of numpy arrays representing the image(s)\n    \"\"\"\n\n    if isinstance(fname, list):\n        if not isinstance(self.directory, list):\n            _directory = [self.directory] * len(fname)\n        else:\n            _directory = self.directory\n        if self.preprocessor is not None:\n            assert isinstance(self.preprocessor, list)\n            processed_sample: Union[np.ndarray, List[np.ndarray]] = [\n                self._preprocess_one_sample(f, d, p)\n                for f, d, p in zip(fname, _directory, self.preprocessor)\n            ]\n        else:\n            processed_sample = [\n                self._preprocess_one_sample(f, d) for f, d in zip(fname, _directory)\n            ]\n    else:\n        assert isinstance(self.directory, str)\n        if self.preprocessor is not None:\n            assert isinstance(self.preprocessor, ImagePreprocessor)\n            processed_sample = self._preprocess_one_sample(\n                fname, self.directory, self.preprocessor\n            )\n        else:\n            processed_sample = self._preprocess_one_sample(fname, self.directory)\n\n    return processed_sample\n</code></pre>"},{"location":"pytorch-widedeep/load_from_folder.html#pytorch_widedeep.load_from_folder.wd_dataset_from_folder.WideDeepDatasetFromFolder","title":"WideDeepDatasetFromFolder","text":"<p>               Bases: <code>Dataset</code></p> <p>This class is the Dataset counterpart of the <code>WideDeepDataset</code> class.</p> <p>Given a reference tabular dataset, with columns that indicate the path to the images and to the text files or the texts themselves, it will use the <code>[...]FromFolder</code> classes to load the data consistently from disk per batch.</p> <p>For examples, please, see the examples folder in the repo.</p> <p>Parameters:</p> Name Type Description Default <code>n_samples</code> <code>int</code> <p>Number of samples in the dataset</p> required <code>tab_from_folder</code> <code>Optional[TabFromFolder]</code> <p>Instance of the <code>TabFromFolder</code> class</p> <code>None</code> <code>wide_from_folder</code> <code>Optional[WideFromFolder]</code> <p>Instance of the <code>WideFromFolder</code> class</p> <code>None</code> <code>text_from_folder</code> <code>Optional[TextFromFolder]</code> <p>Instance of the <code>TextFromFolder</code> class</p> <code>None</code> <code>img_from_folder</code> <code>Optional[ImageFromFolder]</code> <p>Instance of the <code>ImageFromFolder</code> class</p> <code>None</code> <code>reference</code> <code>Optional[Any]</code> <p>If <code>reference</code> not None, the <code>text_from_folder</code> and <code>img_from_folder</code> objects will be retrieved from the reference class. This is useful when we want to use a <code>WideDeepDatasetFromFolder</code> class used for a train dataset as a reference for the validation and test datasets. In this case, the <code>text_from_folder</code> and <code>img_from_folder</code> objects will be the same for all three datasets, so there is no need to create a new instance for each dataset.</p> <code>None</code> Source code in <code>pytorch_widedeep/load_from_folder/wd_dataset_from_folder.py</code> <pre><code>class WideDeepDatasetFromFolder(Dataset):\n    \"\"\"\n    This class is the Dataset counterpart of the `WideDeepDataset` class.\n\n    Given a reference tabular dataset, with columns that indicate the path to\n    the images and to the text files or the texts themselves, it will use the\n    `[...]FromFolder` classes to load the data consistently from disk per batch.\n\n    For examples, please, see the examples folder in the repo.\n\n    Parameters\n    ----------\n    n_samples: int\n        Number of samples in the dataset\n    tab_from_folder: TabFromFolder\n        Instance of the `TabFromFolder` class\n    wide_from_folder: Optional[WideFromFolder], default = None\n        Instance of the `WideFromFolder` class\n    text_from_folder: Optional[TextFromFolder], default = None\n        Instance of the `TextFromFolder` class\n    img_from_folder: Optional[ImageFromFolder], default = None\n        Instance of the `ImageFromFolder` class\n    reference: Type[\"WideDeepDatasetFromFolder\"], default = None\n        If `reference` not None, the `text_from_folder` and `img_from_folder`\n        objects will be retrieved from the reference class. This is useful\n        when we want to use a `WideDeepDatasetFromFolder` class used for a\n        train dataset as a reference for the validation and test datasets. In\n        this case, the `text_from_folder` and `img_from_folder` objects will\n        be the same for all three datasets, so there is no need to create a\n        new instance for each dataset.\n    \"\"\"\n\n    def __init__(\n        self,\n        n_samples: int,\n        tab_from_folder: Optional[TabFromFolder] = None,\n        wide_from_folder: Optional[WideFromFolder] = None,\n        text_from_folder: Optional[TextFromFolder] = None,\n        img_from_folder: Optional[ImageFromFolder] = None,\n        reference: Optional[Any] = None,  # is Type[\"WideDeepDatasetFromFolder\"],\n    ):\n        super(WideDeepDatasetFromFolder, self).__init__()\n\n        if tab_from_folder is None and wide_from_folder is None:\n            raise ValueError(\n                \"Either 'tab_from_folder' or 'wide_from_folder' must be not None\"\n            )\n\n        if reference is not None:\n            assert (\n                img_from_folder is None and text_from_folder is None\n            ), \"If reference is not None, 'img_from_folder' and 'text_from_folder' left as None\"\n            self.text_from_folder, self.img_from_folder = self._get_from_reference(\n                reference\n            )\n        else:\n            assert (\n                text_from_folder is not None and img_from_folder is not None\n            ), \"If reference is None, 'img_from_folder' and 'text_from_folder' must be not None\"\n            self.text_from_folder = text_from_folder\n            self.img_from_folder = img_from_folder\n\n        self.n_samples = n_samples\n        self.tab_from_folder = tab_from_folder\n        self.wide_from_folder = wide_from_folder\n\n    def __getitem__(self, idx: int):  # noqa: C901\n        x = (\n            Bunch()\n        )  # for consistency with WideDeepDataset, but this is just a Dict[str, Any]\n\n        if self.tab_from_folder is not None:\n            X_tab, text_fname_or_text, img_fname, y = self.tab_from_folder.get_item(\n                idx=idx\n            )\n            x.deeptabular = X_tab\n\n        if self.wide_from_folder is not None:\n            if self.tab_from_folder is None:\n                (\n                    X_wide,\n                    text_fname_or_text,\n                    img_fname,\n                    y,\n                ) = self.wide_from_folder.get_item(idx=idx)\n            else:\n                X_wide, _, _, _ = self.wide_from_folder.get_item(idx=idx)\n            x.wide = X_wide\n\n        if text_fname_or_text is not None:\n            # These assertions should never be raised, but just in case...\n            assert (\n                self.text_from_folder is not None\n            ), \"text_fname_or_text is not None but self.text_from_folder is None\"\n            X_text = self.text_from_folder.get_item(text_fname_or_text)\n            x.deeptext = X_text\n\n        if img_fname is not None:\n            assert (\n                self.img_from_folder is not None\n            ), \"img_fname is not None but self.img_from_folder is None\"\n            X_img = self.img_from_folder.get_item(img_fname)\n            x.deepimage = X_img\n\n        # We are aware that returning sometimes X and sometimes X, y is not\n        # the best practice, but is the easiest way at this stage\n        if y is not None:\n            return x, y\n        else:\n            return x\n\n    def __len__(self):\n        return self.n_samples\n\n    @staticmethod\n    def _get_from_reference(\n        reference: Type[\"WideDeepDatasetFromFolder\"],\n    ) -&gt; Tuple[Optional[TextFromFolder], Optional[ImageFromFolder]]:\n        return reference.text_from_folder, reference.img_from_folder\n\n    def __repr__(self) -&gt; str:\n        list_of_params: List[str] = []\n        list_of_params.append(\"n_samples={n_samples}\")\n        if self.tab_from_folder is not None:\n            list_of_params.append(\n                f\"tab_from_folder={self.tab_from_folder.__class__.__name__}\"\n            )\n        if self.wide_from_folder is not None:\n            list_of_params.append(\n                f\"wide_from_folder={self.wide_from_folder.__class__.__name__}\"\n            )\n        if self.text_from_folder is not None:\n            list_of_params.append(\n                f\"text_from_folder={self.text_from_folder.__class__.__name__}\"\n            )\n        if self.img_from_folder is not None:\n            list_of_params.append(\n                f\"img_from_folder={self.img_from_folder.__class__.__name__}\"\n            )\n        all_params = \", \".join(list_of_params)\n        return f\"WideDeepDatasetFromFolder({all_params.format(**self.__dict__)})\"\n</code></pre>"},{"location":"pytorch-widedeep/losses.html","title":"Losses","text":"<p><code>pytorch-widedeep</code> accepts a number of losses and objectives that can be passed to the <code>Trainer</code> class via the parameter <code>objective</code> (see <code>pytorch-widedeep.training.Trainer</code>). For most cases the loss function that <code>pytorch-widedeep</code> will use internally is already implemented in Pytorch.</p> <p>In addition, <code>pytorch-widedeep</code> implements a series of  \"custom\" loss functions. These are described below for completion since, as mentioned before, they are used internally by the <code>Trainer</code>. Of course, onen could always use them on their own and can be imported as:</p> <p><code>from pytorch_widedeep.losses import FocalLoss</code></p> <p> NOTE:  Losses in this module expect the predictions  and ground truth to have the same dimensions for regression and binary  classification problems \\((N_{samples}, 1)\\). In the case of multiclass  classification problems the ground truth is expected to be a 1D tensor with  the corresponding classes. See Examples below</p>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.MSELoss","title":"MSELoss","text":"<p>               Bases: <code>Module</code></p> <p>Mean square error loss</p> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>class MSELoss(nn.Module):\n    r\"\"\"Mean square error loss\"\"\"\n\n    # legacy code from when we used to support FDS-LDS and this class could\n    # taked the corresponding params. At this stage probably you want to use\n    # torch.nn.MSELoss\n    def __init__(self):\n        super().__init__()\n\n    def forward(\n        self,\n        input: Tensor,\n        target: Tensor,\n    ) -&gt; Tensor:\n        r\"\"\"\n        Parameters\n        ----------\n        input: Tensor\n            Input tensor with predictions\n        target: Tensor\n            Target tensor with the actual values\n\n        Examples\n        --------\n        &gt;&gt;&gt; import torch\n        &gt;&gt;&gt; from pytorch_widedeep.losses import MSELoss\n        &gt;&gt;&gt;\n        &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n        &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n        &gt;&gt;&gt; loss = MSELoss()(input, target)\n        \"\"\"\n        loss = (input - target) ** 2\n        return torch.mean(loss)\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.MSELoss.forward","title":"forward","text":"<pre><code>forward(input, target)\n</code></pre> <p>Parameters:</p> Name Type Description Default <code>input</code> <code>Tensor</code> <p>Input tensor with predictions</p> required <code>target</code> <code>Tensor</code> <p>Target tensor with the actual values</p> required <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.losses import MSELoss\n&gt;&gt;&gt;\n&gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n&gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n&gt;&gt;&gt; loss = MSELoss()(input, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(\n    self,\n    input: Tensor,\n    target: Tensor,\n) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    input: Tensor\n        Input tensor with predictions\n    target: Tensor\n        Target tensor with the actual values\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.losses import MSELoss\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n    &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n    &gt;&gt;&gt; loss = MSELoss()(input, target)\n    \"\"\"\n    loss = (input - target) ** 2\n    return torch.mean(loss)\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.MSLELoss","title":"MSLELoss","text":"<p>               Bases: <code>Module</code></p> <p>Mean square log error loss</p> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>class MSLELoss(nn.Module):\n    r\"\"\"Mean square log error loss\"\"\"\n\n    def __init__(self):\n        super().__init__()\n\n    def forward(\n        self,\n        input: Tensor,\n        target: Tensor,\n    ) -&gt; Tensor:\n        r\"\"\"\n        Parameters\n        ----------\n        input: Tensor\n            Input tensor with predictions (not probabilities)\n        target: Tensor\n            Target tensor with the actual classes\n\n        Examples\n        --------\n        &gt;&gt;&gt; import torch\n        &gt;&gt;&gt; from pytorch_widedeep.losses import MSLELoss\n        &gt;&gt;&gt;\n        &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n        &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n        &gt;&gt;&gt; loss = MSLELoss()(input, target)\n        \"\"\"\n        assert (\n            input.min() &gt;= 0\n        ), \"\"\"All input values must be &gt;=0, if your model is predicting\n            values &lt;0 try to enforce positive values by activation function\n            on last layer with `trainer.enforce_positive_output=True`\"\"\"\n        assert target.min() &gt;= 0, \"All target values must be &gt;=0\"\n\n        loss = (torch.log(input + 1) - torch.log(target + 1)) ** 2\n        return torch.mean(loss)\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.MSLELoss.forward","title":"forward","text":"<pre><code>forward(input, target)\n</code></pre> <p>Parameters:</p> Name Type Description Default <code>input</code> <code>Tensor</code> <p>Input tensor with predictions (not probabilities)</p> required <code>target</code> <code>Tensor</code> <p>Target tensor with the actual classes</p> required <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.losses import MSLELoss\n&gt;&gt;&gt;\n&gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n&gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n&gt;&gt;&gt; loss = MSLELoss()(input, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(\n    self,\n    input: Tensor,\n    target: Tensor,\n) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    input: Tensor\n        Input tensor with predictions (not probabilities)\n    target: Tensor\n        Target tensor with the actual classes\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.losses import MSLELoss\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n    &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n    &gt;&gt;&gt; loss = MSLELoss()(input, target)\n    \"\"\"\n    assert (\n        input.min() &gt;= 0\n    ), \"\"\"All input values must be &gt;=0, if your model is predicting\n        values &lt;0 try to enforce positive values by activation function\n        on last layer with `trainer.enforce_positive_output=True`\"\"\"\n    assert target.min() &gt;= 0, \"All target values must be &gt;=0\"\n\n    loss = (torch.log(input + 1) - torch.log(target + 1)) ** 2\n    return torch.mean(loss)\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.RMSELoss","title":"RMSELoss","text":"<p>               Bases: <code>Module</code></p> <p>Root mean square error loss</p> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>class RMSELoss(nn.Module):\n    r\"\"\"Root mean square error loss\"\"\"\n\n    # legacy code from when we used to support FDS-LDS and this class could\n    # taked the corresponding params. At this stage probably you want to use\n    # torch.sqrt(nn.MSELoss)\n    def __init__(self):\n        super().__init__()\n\n    def forward(self, input: Tensor, target: Tensor) -&gt; Tensor:\n        r\"\"\"\n        Parameters\n        ----------\n        input: Tensor\n            Input tensor with predictions (not probabilities)\n        target: Tensor\n            Target tensor with the actual classes\n\n        Examples\n        --------\n        &gt;&gt;&gt; import torch\n        &gt;&gt;&gt; from pytorch_widedeep.losses import RMSELoss\n        &gt;&gt;&gt;\n        &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n        &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n        &gt;&gt;&gt; loss = RMSELoss()(input, target)\n        \"\"\"\n        loss = (input - target) ** 2\n        return torch.sqrt(torch.mean(loss))\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.RMSELoss.forward","title":"forward","text":"<pre><code>forward(input, target)\n</code></pre> <p>Parameters:</p> Name Type Description Default <code>input</code> <code>Tensor</code> <p>Input tensor with predictions (not probabilities)</p> required <code>target</code> <code>Tensor</code> <p>Target tensor with the actual classes</p> required <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.losses import RMSELoss\n&gt;&gt;&gt;\n&gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n&gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n&gt;&gt;&gt; loss = RMSELoss()(input, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(self, input: Tensor, target: Tensor) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    input: Tensor\n        Input tensor with predictions (not probabilities)\n    target: Tensor\n        Target tensor with the actual classes\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.losses import RMSELoss\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n    &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n    &gt;&gt;&gt; loss = RMSELoss()(input, target)\n    \"\"\"\n    loss = (input - target) ** 2\n    return torch.sqrt(torch.mean(loss))\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.RMSLELoss","title":"RMSLELoss","text":"<p>               Bases: <code>Module</code></p> <p>Root mean square log error loss</p> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>class RMSLELoss(nn.Module):\n    r\"\"\"Root mean square log error loss\"\"\"\n\n    def __init__(self):\n        super().__init__()\n\n    def forward(self, input: Tensor, target: Tensor) -&gt; Tensor:\n        r\"\"\"\n        Parameters\n        ----------\n        input: Tensor\n            Input tensor with predictions (not probabilities)\n        target: Tensor\n            Target tensor with the actual classes\n\n        Examples\n        --------\n        &gt;&gt;&gt; import torch\n        &gt;&gt;&gt; from pytorch_widedeep.losses import RMSLELoss\n        &gt;&gt;&gt;\n        &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n        &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n        &gt;&gt;&gt; loss = RMSLELoss()(input, target)\n        \"\"\"\n        assert (\n            input.min() &gt;= 0\n        ), \"\"\"All input values must be &gt;=0, if your model is predicting\n            values &lt;0 try to enforce positive values by activation function\n            on last layer with `trainer.enforce_positive_output=True`\"\"\"\n        assert target.min() &gt;= 0, \"All target values must be &gt;=0\"\n\n        loss = (torch.log(input + 1) - torch.log(target + 1)) ** 2\n        return torch.sqrt(torch.mean(loss))\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.RMSLELoss.forward","title":"forward","text":"<pre><code>forward(input, target)\n</code></pre> <p>Parameters:</p> Name Type Description Default <code>input</code> <code>Tensor</code> <p>Input tensor with predictions (not probabilities)</p> required <code>target</code> <code>Tensor</code> <p>Target tensor with the actual classes</p> required <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.losses import RMSLELoss\n&gt;&gt;&gt;\n&gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n&gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n&gt;&gt;&gt; loss = RMSLELoss()(input, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(self, input: Tensor, target: Tensor) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    input: Tensor\n        Input tensor with predictions (not probabilities)\n    target: Tensor\n        Target tensor with the actual classes\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.losses import RMSLELoss\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n    &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n    &gt;&gt;&gt; loss = RMSLELoss()(input, target)\n    \"\"\"\n    assert (\n        input.min() &gt;= 0\n    ), \"\"\"All input values must be &gt;=0, if your model is predicting\n        values &lt;0 try to enforce positive values by activation function\n        on last layer with `trainer.enforce_positive_output=True`\"\"\"\n    assert target.min() &gt;= 0, \"All target values must be &gt;=0\"\n\n    loss = (torch.log(input + 1) - torch.log(target + 1)) ** 2\n    return torch.sqrt(torch.mean(loss))\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.QuantileLoss","title":"QuantileLoss","text":"<p>               Bases: <code>Module</code></p> <p>Quantile loss defined as:</p> \\[ Loss = max(q \\times (y-y_{pred}), (1-q) \\times (y_{pred}-y)) \\] <p>All credits go to the implementation at pytorch-forecasting.</p> <p>Parameters:</p> Name Type Description Default <code>quantiles</code> <code>List[float]</code> <p>List of quantiles</p> <code>[0.02, 0.1, 0.25, 0.5, 0.75, 0.9, 0.98]</code> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>class QuantileLoss(nn.Module):\n    r\"\"\"Quantile loss defined as:\n\n    $$\n    Loss = max(q \\times (y-y_{pred}), (1-q) \\times (y_{pred}-y))\n    $$\n\n    All credits go to the implementation at\n    [pytorch-forecasting](https://pytorch-forecasting.readthedocs.io/en/latest/_modules/pytorch_forecasting/metrics.html#QuantileLoss).\n\n    Parameters\n    ----------\n    quantiles: List, default = [0.02, 0.1, 0.25, 0.5, 0.75, 0.9, 0.98]\n        List of quantiles\n    \"\"\"\n\n    def __init__(\n        self,\n        quantiles: List[float] = [0.02, 0.1, 0.25, 0.5, 0.75, 0.9, 0.98],\n    ):\n        super().__init__()\n        self.quantiles = quantiles\n\n    def forward(self, input: Tensor, target: Tensor) -&gt; Tensor:\n        r\"\"\"\n        Parameters\n        ----------\n        input: Tensor\n            Input tensor with predictions\n        target: Tensor\n            Target tensor with the actual values\n\n        Examples\n        --------\n        &gt;&gt;&gt; import torch\n        &gt;&gt;&gt;\n        &gt;&gt;&gt; from pytorch_widedeep.losses import QuantileLoss\n        &gt;&gt;&gt;\n        &gt;&gt;&gt; # REGRESSION\n        &gt;&gt;&gt; target = torch.tensor([[0.6, 1.5]]).view(-1, 1)\n        &gt;&gt;&gt; input = torch.tensor([[.1, .2,], [.4, .5]])\n        &gt;&gt;&gt; qloss = QuantileLoss([0.25, 0.75])\n        &gt;&gt;&gt; loss = qloss(input, target)\n        \"\"\"\n\n        assert input.shape == torch.Size([target.shape[0], len(self.quantiles)]), (\n            \"The input and target have inconsistent shape. The dimension of the prediction \"\n            \"of the model that is using QuantileLoss must be equal to number of quantiles, \"\n            f\"i.e. {len(self.quantiles)}.\"\n        )\n        target = target.view(-1, 1).float()\n        losses = []\n        for i, q in enumerate(self.quantiles):\n            errors = target - input[..., i]\n            losses.append(torch.max((q - 1) * errors, q * errors).unsqueeze(-1))\n\n        loss = torch.cat(losses, dim=2)\n\n        return torch.mean(loss)\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.QuantileLoss.forward","title":"forward","text":"<pre><code>forward(input, target)\n</code></pre> <p>Parameters:</p> Name Type Description Default <code>input</code> <code>Tensor</code> <p>Input tensor with predictions</p> required <code>target</code> <code>Tensor</code> <p>Target tensor with the actual values</p> required <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt;\n&gt;&gt;&gt; from pytorch_widedeep.losses import QuantileLoss\n&gt;&gt;&gt;\n&gt;&gt;&gt; # REGRESSION\n&gt;&gt;&gt; target = torch.tensor([[0.6, 1.5]]).view(-1, 1)\n&gt;&gt;&gt; input = torch.tensor([[.1, .2,], [.4, .5]])\n&gt;&gt;&gt; qloss = QuantileLoss([0.25, 0.75])\n&gt;&gt;&gt; loss = qloss(input, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(self, input: Tensor, target: Tensor) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    input: Tensor\n        Input tensor with predictions\n    target: Tensor\n        Target tensor with the actual values\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; from pytorch_widedeep.losses import QuantileLoss\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; # REGRESSION\n    &gt;&gt;&gt; target = torch.tensor([[0.6, 1.5]]).view(-1, 1)\n    &gt;&gt;&gt; input = torch.tensor([[.1, .2,], [.4, .5]])\n    &gt;&gt;&gt; qloss = QuantileLoss([0.25, 0.75])\n    &gt;&gt;&gt; loss = qloss(input, target)\n    \"\"\"\n\n    assert input.shape == torch.Size([target.shape[0], len(self.quantiles)]), (\n        \"The input and target have inconsistent shape. The dimension of the prediction \"\n        \"of the model that is using QuantileLoss must be equal to number of quantiles, \"\n        f\"i.e. {len(self.quantiles)}.\"\n    )\n    target = target.view(-1, 1).float()\n    losses = []\n    for i, q in enumerate(self.quantiles):\n        errors = target - input[..., i]\n        losses.append(torch.max((q - 1) * errors, q * errors).unsqueeze(-1))\n\n    loss = torch.cat(losses, dim=2)\n\n    return torch.mean(loss)\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.FocalLoss","title":"FocalLoss","text":"<p>               Bases: <code>Module</code></p> <p>Implementation of the Focal loss for both binary and multiclass classification:</p> \\[ FL(p_t) = \\alpha (1 - p_t)^{\\gamma} log(p_t) \\] <p>where, for a case of a binary classification problem</p> \\[ \\begin{equation} p_t= \\begin{cases}p, &amp; \\text{if $y=1$}.\\\\1-p, &amp; \\text{otherwise}. \\end{cases} \\end{equation} \\] <p>Parameters:</p> Name Type Description Default <code>alpha</code> <code>float</code> <p>Focal Loss <code>alpha</code> parameter</p> <code>0.25</code> <code>gamma</code> <code>float</code> <p>Focal Loss <code>gamma</code> parameter</p> <code>1.0</code> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>class FocalLoss(nn.Module):\n    r\"\"\"Implementation of the [Focal loss](https://arxiv.org/pdf/1708.02002.pdf)\n    for both binary and multiclass classification:\n\n    $$\n    FL(p_t) = \\alpha (1 - p_t)^{\\gamma} log(p_t)\n    $$\n\n    where, for a case of a binary classification problem\n\n    $$\n    \\begin{equation} p_t= \\begin{cases}p, &amp; \\text{if $y=1$}.\\\\1-p, &amp; \\text{otherwise}. \\end{cases} \\end{equation}\n    $$\n\n    Parameters\n    ----------\n    alpha: float\n        Focal Loss `alpha` parameter\n    gamma: float\n        Focal Loss `gamma` parameter\n    \"\"\"\n\n    def __init__(self, alpha: float = 0.25, gamma: float = 1.0):\n        super().__init__()\n        self.alpha = alpha\n        self.gamma = gamma\n\n    def _get_weight(self, p: Tensor, t: Tensor) -&gt; Tensor:\n        pt = p * t + (1 - p) * (1 - t)  # type: ignore\n        w = self.alpha * t + (1 - self.alpha) * (1 - t)  # type: ignore\n        return (w * (1 - pt).pow(self.gamma)).detach()  # type: ignore\n\n    def forward(self, input: Tensor, target: Tensor) -&gt; Tensor:\n        r\"\"\"\n        Parameters\n        ----------\n        input: Tensor\n            Input tensor with predictions (not probabilities)\n        target: Tensor\n            Target tensor with the actual classes\n\n        Examples\n        --------\n        &gt;&gt;&gt; import torch\n        &gt;&gt;&gt;\n        &gt;&gt;&gt; from pytorch_widedeep.losses import FocalLoss\n        &gt;&gt;&gt;\n        &gt;&gt;&gt; # BINARY\n        &gt;&gt;&gt; target = torch.tensor([0, 1, 0, 1]).view(-1, 1)\n        &gt;&gt;&gt; input = torch.tensor([[0.6, 0.7, 0.3, 0.8]]).t()\n        &gt;&gt;&gt; loss = FocalLoss()(input, target)\n        &gt;&gt;&gt;\n        &gt;&gt;&gt; # MULTICLASS\n        &gt;&gt;&gt; target = torch.tensor([1, 0, 2]).view(-1, 1)\n        &gt;&gt;&gt; input = torch.tensor([[0.2, 0.5, 0.3], [0.8, 0.1, 0.1], [0.7, 0.2, 0.1]])\n        &gt;&gt;&gt; loss = FocalLoss()(input, target)\n        \"\"\"\n        input_prob = torch.sigmoid(input)\n        if input.size(1) == 1:\n            input_prob = torch.cat([1 - input_prob, input_prob], axis=1)  # type: ignore\n            num_class = 2\n        else:\n            num_class = input_prob.size(1)\n        binary_target = torch.eye(num_class)[target.squeeze().cpu().long()]\n        binary_target = binary_target.to(input.device)\n        binary_target = binary_target.contiguous()\n        weight = self._get_weight(input_prob, binary_target)\n\n        return F.binary_cross_entropy(\n            input_prob, binary_target, weight, reduction=\"mean\"\n        )\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.FocalLoss.forward","title":"forward","text":"<pre><code>forward(input, target)\n</code></pre> <p>Parameters:</p> Name Type Description Default <code>input</code> <code>Tensor</code> <p>Input tensor with predictions (not probabilities)</p> required <code>target</code> <code>Tensor</code> <p>Target tensor with the actual classes</p> required <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt;\n&gt;&gt;&gt; from pytorch_widedeep.losses import FocalLoss\n&gt;&gt;&gt;\n&gt;&gt;&gt; # BINARY\n&gt;&gt;&gt; target = torch.tensor([0, 1, 0, 1]).view(-1, 1)\n&gt;&gt;&gt; input = torch.tensor([[0.6, 0.7, 0.3, 0.8]]).t()\n&gt;&gt;&gt; loss = FocalLoss()(input, target)\n&gt;&gt;&gt;\n&gt;&gt;&gt; # MULTICLASS\n&gt;&gt;&gt; target = torch.tensor([1, 0, 2]).view(-1, 1)\n&gt;&gt;&gt; input = torch.tensor([[0.2, 0.5, 0.3], [0.8, 0.1, 0.1], [0.7, 0.2, 0.1]])\n&gt;&gt;&gt; loss = FocalLoss()(input, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(self, input: Tensor, target: Tensor) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    input: Tensor\n        Input tensor with predictions (not probabilities)\n    target: Tensor\n        Target tensor with the actual classes\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; from pytorch_widedeep.losses import FocalLoss\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; # BINARY\n    &gt;&gt;&gt; target = torch.tensor([0, 1, 0, 1]).view(-1, 1)\n    &gt;&gt;&gt; input = torch.tensor([[0.6, 0.7, 0.3, 0.8]]).t()\n    &gt;&gt;&gt; loss = FocalLoss()(input, target)\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; # MULTICLASS\n    &gt;&gt;&gt; target = torch.tensor([1, 0, 2]).view(-1, 1)\n    &gt;&gt;&gt; input = torch.tensor([[0.2, 0.5, 0.3], [0.8, 0.1, 0.1], [0.7, 0.2, 0.1]])\n    &gt;&gt;&gt; loss = FocalLoss()(input, target)\n    \"\"\"\n    input_prob = torch.sigmoid(input)\n    if input.size(1) == 1:\n        input_prob = torch.cat([1 - input_prob, input_prob], axis=1)  # type: ignore\n        num_class = 2\n    else:\n        num_class = input_prob.size(1)\n    binary_target = torch.eye(num_class)[target.squeeze().cpu().long()]\n    binary_target = binary_target.to(input.device)\n    binary_target = binary_target.contiguous()\n    weight = self._get_weight(input_prob, binary_target)\n\n    return F.binary_cross_entropy(\n        input_prob, binary_target, weight, reduction=\"mean\"\n    )\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.BayesianSELoss","title":"BayesianSELoss","text":"<p>               Bases: <code>Module</code></p> <p>Squared Loss (log Gaussian) for the case of a regression as specified in the original publication Weight Uncertainty in Neural Networks.</p> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>class BayesianSELoss(nn.Module):\n    r\"\"\"Squared Loss (log Gaussian) for the case of a regression as specified in\n    the original publication\n    [Weight Uncertainty in Neural Networks](https://arxiv.org/abs/1505.05424).\n    \"\"\"\n\n    def __init__(self):\n        super().__init__()\n\n    def forward(self, input: Tensor, target: Tensor) -&gt; Tensor:\n        r\"\"\"\n        Parameters\n        ----------\n        input: Tensor\n            Input tensor with predictions (not probabilities)\n        target: Tensor\n            Target tensor with the actual classes\n\n        Examples\n        --------\n        &gt;&gt;&gt; import torch\n        &gt;&gt;&gt; from pytorch_widedeep.losses import BayesianSELoss\n        &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n        &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n        &gt;&gt;&gt; loss = BayesianSELoss()(input, target)\n        \"\"\"\n        return (0.5 * (input - target) ** 2).sum()\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.BayesianSELoss.forward","title":"forward","text":"<pre><code>forward(input, target)\n</code></pre> <p>Parameters:</p> Name Type Description Default <code>input</code> <code>Tensor</code> <p>Input tensor with predictions (not probabilities)</p> required <code>target</code> <code>Tensor</code> <p>Target tensor with the actual classes</p> required <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.losses import BayesianSELoss\n&gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n&gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n&gt;&gt;&gt; loss = BayesianSELoss()(input, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(self, input: Tensor, target: Tensor) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    input: Tensor\n        Input tensor with predictions (not probabilities)\n    target: Tensor\n        Target tensor with the actual classes\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.losses import BayesianSELoss\n    &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n    &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n    &gt;&gt;&gt; loss = BayesianSELoss()(input, target)\n    \"\"\"\n    return (0.5 * (input - target) ** 2).sum()\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.TweedieLoss","title":"TweedieLoss","text":"<p>               Bases: <code>Module</code></p> <p>Tweedie loss for extremely unbalanced zero-inflated data</p> <p>All credits go to Wenbo Shi. See this post and the original publication for details.</p> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>class TweedieLoss(nn.Module):\n    \"\"\"\n    Tweedie loss for extremely unbalanced zero-inflated data\n\n    All credits go to Wenbo Shi. See\n    [this post](https://towardsdatascience.com/tweedie-loss-function-for-right-skewed-data-2c5ca470678f)\n    and the [original publication](https://arxiv.org/abs/1811.10192) for details.\n    \"\"\"\n\n    def __init__(self):\n        super().__init__()\n\n    def forward(\n        self,\n        input: Tensor,\n        target: Tensor,\n        p: float = 1.5,\n    ) -&gt; Tensor:\n        r\"\"\"\n        Parameters\n        ----------\n        input: Tensor\n            Input tensor with predictions\n        target: Tensor\n            Target tensor with the actual values\n        p: float, default = 1.5\n            the power to be used to compute the loss. See the original\n            publication for details\n\n        Examples\n        --------\n        &gt;&gt;&gt; import torch\n        &gt;&gt;&gt; from pytorch_widedeep.losses import TweedieLoss\n        &gt;&gt;&gt;\n        &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n        &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n        &gt;&gt;&gt; loss = TweedieLoss()(input, target)\n        \"\"\"\n\n        assert (\n            input.min() &gt; 0\n        ), \"\"\"All input values must be &gt;=0, if your model is predicting\n            values &lt;0 try to enforce positive values by activation function\n            on last layer with `trainer.enforce_positive_output=True`\"\"\"\n        assert target.min() &gt;= 0, \"All target values must be &gt;=0\"\n        loss = -target * torch.pow(input, 1 - p) / (1 - p) + torch.pow(input, 2 - p) / (\n            2 - p\n        )\n        return torch.mean(loss)\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.TweedieLoss.forward","title":"forward","text":"<pre><code>forward(input, target, p=1.5)\n</code></pre> <p>Parameters:</p> Name Type Description Default <code>input</code> <code>Tensor</code> <p>Input tensor with predictions</p> required <code>target</code> <code>Tensor</code> <p>Target tensor with the actual values</p> required <code>p</code> <code>float</code> <p>the power to be used to compute the loss. See the original publication for details</p> <code>1.5</code> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.losses import TweedieLoss\n&gt;&gt;&gt;\n&gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n&gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n&gt;&gt;&gt; loss = TweedieLoss()(input, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(\n    self,\n    input: Tensor,\n    target: Tensor,\n    p: float = 1.5,\n) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    input: Tensor\n        Input tensor with predictions\n    target: Tensor\n        Target tensor with the actual values\n    p: float, default = 1.5\n        the power to be used to compute the loss. See the original\n        publication for details\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.losses import TweedieLoss\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n    &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n    &gt;&gt;&gt; loss = TweedieLoss()(input, target)\n    \"\"\"\n\n    assert (\n        input.min() &gt; 0\n    ), \"\"\"All input values must be &gt;=0, if your model is predicting\n        values &lt;0 try to enforce positive values by activation function\n        on last layer with `trainer.enforce_positive_output=True`\"\"\"\n    assert target.min() &gt;= 0, \"All target values must be &gt;=0\"\n    loss = -target * torch.pow(input, 1 - p) / (1 - p) + torch.pow(input, 2 - p) / (\n        2 - p\n    )\n    return torch.mean(loss)\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.ZILNLoss","title":"ZILNLoss","text":"<p>               Bases: <code>Module</code></p> <p>Adjusted implementation of the Zero Inflated LogNormal Loss</p> <p>See A Deep Probabilistic Model for Customer Lifetime Value Prediction and the corresponding code.</p> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>class ZILNLoss(nn.Module):\n    r\"\"\"Adjusted implementation of the Zero Inflated LogNormal Loss\n\n    See [A Deep Probabilistic Model for Customer Lifetime Value Prediction](https://arxiv.org/pdf/1912.07753.pdf)\n    and the corresponding\n    [code](https://github.com/google/lifetime_value/blob/master/lifetime_value/zero_inflated_lognormal.py).\n    \"\"\"\n\n    def __init__(self):\n        super().__init__()\n\n    def forward(self, input: Tensor, target: Tensor) -&gt; Tensor:\n        r\"\"\"\n        Parameters\n        ----------\n        input: Tensor\n            Input tensor with predictions with spape (N,3), where N is the batch size\n        target: Tensor\n            Target tensor with the actual target values\n\n        Examples\n        --------\n        &gt;&gt;&gt; import torch\n        &gt;&gt;&gt; from pytorch_widedeep.losses import ZILNLoss\n        &gt;&gt;&gt;\n        &gt;&gt;&gt; target = torch.tensor([[0., 1.5]]).view(-1, 1)\n        &gt;&gt;&gt; input = torch.tensor([[.1, .2, .3], [.4, .5, .6]])\n        &gt;&gt;&gt; loss = ZILNLoss()(input, target)\n        \"\"\"\n        positive = target &gt; 0\n        positive = positive.float()\n\n        assert input.shape == torch.Size([target.shape[0], 3]), (\n            \"Wrong shape of the 'input' tensor. The pred_dim of the \"\n            \"model that is using ZILNLoss must be equal to 3.\"\n        )\n\n        positive_input = input[..., :1]\n\n        classification_loss = F.binary_cross_entropy_with_logits(\n            positive_input, positive, reduction=\"none\"\n        ).flatten()\n\n        loc = input[..., 1:2]\n\n        # when using max the two input tensors (input and other) have to be of\n        # the same type\n        max_input = F.softplus(input[..., 2:])\n        max_other = self.get_eps(max_input)\n        scale = torch.max(max_input, max_other)\n        safe_labels = positive * target + (1 - positive) * torch.ones_like(target)\n\n        regression_loss = -torch.mean(\n            positive\n            * torch.distributions.log_normal.LogNormal(loc=loc, scale=scale).log_prob(\n                safe_labels\n            ),\n            dim=-1,\n        )\n\n        return torch.mean(classification_loss + regression_loss)\n\n    @staticmethod\n    def get_eps(max_input: Tensor) -&gt; Tensor:\n        if max_input.device.type == \"mps\":\n            # For MPS, use float32 and then convert to the input type\n            eps = torch.finfo(torch.float32).eps\n            max_other = (\n                torch.sqrt(torch.tensor([eps], device=\"cpu\"))\n                .to(max_input.device)\n                .to(max_input.dtype)\n            )\n        else:\n            # For other devices, use the original approach\n            eps = torch.finfo(torch.double).eps\n            max_other = (\n                torch.sqrt(torch.tensor([eps])).to(max_input.device).to(max_input.dtype)\n            )\n\n        return max_other\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.ZILNLoss.forward","title":"forward","text":"<pre><code>forward(input, target)\n</code></pre> <p>Parameters:</p> Name Type Description Default <code>input</code> <code>Tensor</code> <p>Input tensor with predictions with spape (N,3), where N is the batch size</p> required <code>target</code> <code>Tensor</code> <p>Target tensor with the actual target values</p> required <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.losses import ZILNLoss\n&gt;&gt;&gt;\n&gt;&gt;&gt; target = torch.tensor([[0., 1.5]]).view(-1, 1)\n&gt;&gt;&gt; input = torch.tensor([[.1, .2, .3], [.4, .5, .6]])\n&gt;&gt;&gt; loss = ZILNLoss()(input, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(self, input: Tensor, target: Tensor) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    input: Tensor\n        Input tensor with predictions with spape (N,3), where N is the batch size\n    target: Tensor\n        Target tensor with the actual target values\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.losses import ZILNLoss\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; target = torch.tensor([[0., 1.5]]).view(-1, 1)\n    &gt;&gt;&gt; input = torch.tensor([[.1, .2, .3], [.4, .5, .6]])\n    &gt;&gt;&gt; loss = ZILNLoss()(input, target)\n    \"\"\"\n    positive = target &gt; 0\n    positive = positive.float()\n\n    assert input.shape == torch.Size([target.shape[0], 3]), (\n        \"Wrong shape of the 'input' tensor. The pred_dim of the \"\n        \"model that is using ZILNLoss must be equal to 3.\"\n    )\n\n    positive_input = input[..., :1]\n\n    classification_loss = F.binary_cross_entropy_with_logits(\n        positive_input, positive, reduction=\"none\"\n    ).flatten()\n\n    loc = input[..., 1:2]\n\n    # when using max the two input tensors (input and other) have to be of\n    # the same type\n    max_input = F.softplus(input[..., 2:])\n    max_other = self.get_eps(max_input)\n    scale = torch.max(max_input, max_other)\n    safe_labels = positive * target + (1 - positive) * torch.ones_like(target)\n\n    regression_loss = -torch.mean(\n        positive\n        * torch.distributions.log_normal.LogNormal(loc=loc, scale=scale).log_prob(\n            safe_labels\n        ),\n        dim=-1,\n    )\n\n    return torch.mean(classification_loss + regression_loss)\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.L1Loss","title":"L1Loss","text":"<p>               Bases: <code>Module</code></p> <p>L1 loss</p> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>class L1Loss(nn.Module):\n    r\"\"\"L1 loss\"\"\"\n\n    # legacy code from when we used to support FDS-LDS and this class could\n    # taked the corresponding params. At this stage probably you want to use\n    # torch.nn.L1Loss\n    def __init__(self):\n        super().__init__()\n\n    def forward(self, input: Tensor, target: Tensor) -&gt; Tensor:\n        r\"\"\"\n        Parameters\n        ----------\n        input: Tensor\n            Input tensor with predictions\n        target: Tensor\n            Target tensor with the actual values\n\n        Examples\n        --------\n        &gt;&gt;&gt; import torch\n        &gt;&gt;&gt;\n        &gt;&gt;&gt; from pytorch_widedeep.losses import L1Loss\n        &gt;&gt;&gt;\n        &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n        &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n        &gt;&gt;&gt; loss = L1Loss()(input, target)\n        \"\"\"\n        loss = F.l1_loss(input, target, reduction=\"none\")\n        return torch.mean(loss)\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.L1Loss.forward","title":"forward","text":"<pre><code>forward(input, target)\n</code></pre> <p>Parameters:</p> Name Type Description Default <code>input</code> <code>Tensor</code> <p>Input tensor with predictions</p> required <code>target</code> <code>Tensor</code> <p>Target tensor with the actual values</p> required <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt;\n&gt;&gt;&gt; from pytorch_widedeep.losses import L1Loss\n&gt;&gt;&gt;\n&gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n&gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n&gt;&gt;&gt; loss = L1Loss()(input, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(self, input: Tensor, target: Tensor) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    input: Tensor\n        Input tensor with predictions\n    target: Tensor\n        Target tensor with the actual values\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; from pytorch_widedeep.losses import L1Loss\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n    &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n    &gt;&gt;&gt; loss = L1Loss()(input, target)\n    \"\"\"\n    loss = F.l1_loss(input, target, reduction=\"none\")\n    return torch.mean(loss)\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.FocalR_L1Loss","title":"FocalR_L1Loss","text":"<p>               Bases: <code>Module</code></p> <p>Focal-R L1 loss</p> <p>Based on Delving into Deep Imbalanced Regression.</p> <p>Parameters:</p> Name Type Description Default <code>beta</code> <code>float</code> <p>Focal Loss <code>beta</code> parameter in their implementation</p> <code>0.2</code> <code>gamma</code> <code>float</code> <p>Focal Loss <code>gamma</code> parameter</p> <code>1.0</code> <code>activation_fn</code> <code>Literal[sigmoid, tanh]</code> <p>Activation function to be used during the computation of the loss. Possible values are 'sigmoid' and 'tanh'. See the original publication for details.</p> <code>'sigmoid'</code> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>class FocalR_L1Loss(nn.Module):\n    r\"\"\"Focal-R L1 loss\n\n    Based on [Delving into Deep Imbalanced Regression](https://arxiv.org/abs/2102.09554).\n\n    Parameters\n    ----------\n    beta: float\n        Focal Loss `beta` parameter in their implementation\n    gamma: float\n        Focal Loss `gamma` parameter\n    activation_fn: str, default = \"sigmoid\"\n        Activation function to be used during the computation of the loss.\n        Possible values are _'sigmoid'_ and _'tanh'_. See the original\n        publication for details.\n    \"\"\"\n\n    def __init__(\n        self,\n        beta: float = 0.2,\n        gamma: float = 1.0,\n        activation_fn: Literal[\"sigmoid\", \"tanh\"] = \"sigmoid\",\n    ):\n        super().__init__()\n        self.beta = beta\n        self.gamma = gamma\n        self.activation_fn = activation_fn\n\n    def forward(\n        self,\n        input: Tensor,\n        target: Tensor,\n    ) -&gt; Tensor:\n        r\"\"\"\n        Parameters\n        ----------\n        input: Tensor\n            Input tensor with predictions (not probabilities)\n        target: Tensor\n            Target tensor with the actual classes\n\n        Examples\n        --------\n        &gt;&gt;&gt; import torch\n        &gt;&gt;&gt;\n        &gt;&gt;&gt; from pytorch_widedeep.losses import FocalR_L1Loss\n        &gt;&gt;&gt;\n        &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n        &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n        &gt;&gt;&gt; loss = FocalR_L1Loss()(input, target)\n        \"\"\"\n        loss = F.l1_loss(input, target, reduction=\"none\")\n        if self.activation_fn == \"tanh\":\n            loss *= (torch.tanh(self.beta * torch.abs(input - target))) ** self.gamma\n        elif self.activation_fn == \"sigmoid\":\n            loss *= (\n                2 * torch.sigmoid(self.beta * torch.abs(input - target)) - 1\n            ) ** self.gamma\n        else:\n            ValueError(\n                \"Incorrect activation function value - must be in ['sigmoid', 'tanh']\"\n            )\n        return torch.mean(loss)\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.FocalR_L1Loss.forward","title":"forward","text":"<pre><code>forward(input, target)\n</code></pre> <p>Parameters:</p> Name Type Description Default <code>input</code> <code>Tensor</code> <p>Input tensor with predictions (not probabilities)</p> required <code>target</code> <code>Tensor</code> <p>Target tensor with the actual classes</p> required <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt;\n&gt;&gt;&gt; from pytorch_widedeep.losses import FocalR_L1Loss\n&gt;&gt;&gt;\n&gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n&gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n&gt;&gt;&gt; loss = FocalR_L1Loss()(input, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(\n    self,\n    input: Tensor,\n    target: Tensor,\n) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    input: Tensor\n        Input tensor with predictions (not probabilities)\n    target: Tensor\n        Target tensor with the actual classes\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; from pytorch_widedeep.losses import FocalR_L1Loss\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n    &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n    &gt;&gt;&gt; loss = FocalR_L1Loss()(input, target)\n    \"\"\"\n    loss = F.l1_loss(input, target, reduction=\"none\")\n    if self.activation_fn == \"tanh\":\n        loss *= (torch.tanh(self.beta * torch.abs(input - target))) ** self.gamma\n    elif self.activation_fn == \"sigmoid\":\n        loss *= (\n            2 * torch.sigmoid(self.beta * torch.abs(input - target)) - 1\n        ) ** self.gamma\n    else:\n        ValueError(\n            \"Incorrect activation function value - must be in ['sigmoid', 'tanh']\"\n        )\n    return torch.mean(loss)\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.FocalR_MSELoss","title":"FocalR_MSELoss","text":"<p>               Bases: <code>Module</code></p> <p>Focal-R MSE loss</p> <p>Based on Delving into Deep Imbalanced Regression.</p> <p>Parameters:</p> Name Type Description Default <code>beta</code> <code>float</code> <p>Focal Loss <code>beta</code> parameter in their implementation</p> <code>0.2</code> <code>gamma</code> <code>float</code> <p>Focal Loss <code>gamma</code> parameter</p> <code>1.0</code> <code>activation_fn</code> <code>Literal[sigmoid, tanh]</code> <p>Activation function to be used during the computation of the loss. Possible values are 'sigmoid' and 'tanh'. See the original publication for details.</p> <code>'sigmoid'</code> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>class FocalR_MSELoss(nn.Module):\n    r\"\"\"Focal-R MSE loss\n\n    Based on [Delving into Deep Imbalanced Regression](https://arxiv.org/abs/2102.09554).\n\n    Parameters\n    ----------\n    beta: float\n        Focal Loss `beta` parameter in their implementation\n    gamma: float\n        Focal Loss `gamma` parameter\n    activation_fn: str, default = \"sigmoid\"\n        Activation function to be used during the computation of the loss.\n        Possible values are _'sigmoid'_ and _'tanh'_. See the original\n        publication for details.\n    \"\"\"\n\n    def __init__(\n        self,\n        beta: float = 0.2,\n        gamma: float = 1.0,\n        activation_fn: Literal[\"sigmoid\", \"tanh\"] = \"sigmoid\",\n    ):\n        super().__init__()\n        self.beta = beta\n        self.gamma = gamma\n        self.activation_fn = activation_fn\n\n    def forward(\n        self,\n        input: Tensor,\n        target: Tensor,\n    ) -&gt; Tensor:\n        r\"\"\"\n        Parameters\n        ----------\n        input: Tensor\n            Input tensor with predictions (not probabilities)\n        target: Tensor\n            Target tensor with the actual classes\n\n        Examples\n        --------\n        &gt;&gt;&gt; import torch\n        &gt;&gt;&gt;\n        &gt;&gt;&gt; from pytorch_widedeep.losses import FocalR_MSELoss\n        &gt;&gt;&gt;\n        &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n        &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n        &gt;&gt;&gt; loss = FocalR_MSELoss()(input, target)\n        \"\"\"\n        loss = (input - target) ** 2\n        if self.activation_fn == \"tanh\":\n            loss *= (torch.tanh(self.beta * torch.abs(input - target))) ** self.gamma\n        elif self.activation_fn == \"sigmoid\":\n            loss *= (\n                2 * torch.sigmoid(self.beta * torch.abs((input - target) ** 2)) - 1\n            ) ** self.gamma\n        else:\n            ValueError(\n                \"Incorrect activation function value - must be in ['sigmoid', 'tanh']\"\n            )\n        return torch.mean(loss)\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.FocalR_MSELoss.forward","title":"forward","text":"<pre><code>forward(input, target)\n</code></pre> <p>Parameters:</p> Name Type Description Default <code>input</code> <code>Tensor</code> <p>Input tensor with predictions (not probabilities)</p> required <code>target</code> <code>Tensor</code> <p>Target tensor with the actual classes</p> required <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt;\n&gt;&gt;&gt; from pytorch_widedeep.losses import FocalR_MSELoss\n&gt;&gt;&gt;\n&gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n&gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n&gt;&gt;&gt; loss = FocalR_MSELoss()(input, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(\n    self,\n    input: Tensor,\n    target: Tensor,\n) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    input: Tensor\n        Input tensor with predictions (not probabilities)\n    target: Tensor\n        Target tensor with the actual classes\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; from pytorch_widedeep.losses import FocalR_MSELoss\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n    &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n    &gt;&gt;&gt; loss = FocalR_MSELoss()(input, target)\n    \"\"\"\n    loss = (input - target) ** 2\n    if self.activation_fn == \"tanh\":\n        loss *= (torch.tanh(self.beta * torch.abs(input - target))) ** self.gamma\n    elif self.activation_fn == \"sigmoid\":\n        loss *= (\n            2 * torch.sigmoid(self.beta * torch.abs((input - target) ** 2)) - 1\n        ) ** self.gamma\n    else:\n        ValueError(\n            \"Incorrect activation function value - must be in ['sigmoid', 'tanh']\"\n        )\n    return torch.mean(loss)\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.FocalR_RMSELoss","title":"FocalR_RMSELoss","text":"<p>               Bases: <code>Module</code></p> <p>Focal-R RMSE loss</p> <p>Based on Delving into Deep Imbalanced Regression.</p> <p>Parameters:</p> Name Type Description Default <code>beta</code> <code>float</code> <p>Focal Loss <code>beta</code> parameter in their implementation</p> <code>0.2</code> <code>gamma</code> <code>float</code> <p>Focal Loss <code>gamma</code> parameter</p> <code>1.0</code> <code>activation_fn</code> <code>Literal[sigmoid, tanh]</code> <p>Activation function to be used during the computation of the loss. Possible values are 'sigmoid' and 'tanh'. See the original publication for details.</p> <code>'sigmoid'</code> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>class FocalR_RMSELoss(nn.Module):\n    r\"\"\"Focal-R RMSE loss\n\n    Based on [Delving into Deep Imbalanced Regression](https://arxiv.org/abs/2102.09554).\n\n    Parameters\n    ----------\n    beta: float\n        Focal Loss `beta` parameter in their implementation\n    gamma: float\n        Focal Loss `gamma` parameter\n    activation_fn: str, default = \"sigmoid\"\n        Activation function to be used during the computation of the loss.\n        Possible values are _'sigmoid'_ and _'tanh'_. See the original\n        publication for details.\n    \"\"\"\n\n    def __init__(\n        self,\n        beta: float = 0.2,\n        gamma: float = 1.0,\n        activation_fn: Literal[\"sigmoid\", \"tanh\"] = \"sigmoid\",\n    ):\n        super().__init__()\n        self.beta = beta\n        self.gamma = gamma\n        self.activation_fn = activation_fn\n\n    def forward(\n        self,\n        input: Tensor,\n        target: Tensor,\n    ) -&gt; Tensor:\n        r\"\"\"\n        Parameters\n        ----------\n        input: Tensor\n            Input tensor with predictions (not probabilities)\n        target: Tensor\n            Target tensor with the actual classes\n\n        Examples\n        --------\n        &gt;&gt;&gt; import torch\n        &gt;&gt;&gt;\n        &gt;&gt;&gt; from pytorch_widedeep.losses import FocalR_RMSELoss\n        &gt;&gt;&gt;\n        &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n        &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n        &gt;&gt;&gt; loss = FocalR_RMSELoss()(input, target)\n        \"\"\"\n        loss = (input - target) ** 2\n        if self.activation_fn == \"tanh\":\n            loss *= (torch.tanh(self.beta * torch.abs(input - target))) ** self.gamma\n        elif self.activation_fn == \"sigmoid\":\n            loss *= (\n                2 * torch.sigmoid(self.beta * torch.abs((input - target) ** 2)) - 1\n            ) ** self.gamma\n        else:\n            ValueError(\n                \"Incorrect activation function value - must be in ['sigmoid', 'tanh']\"\n            )\n        return torch.sqrt(torch.mean(loss))\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.FocalR_RMSELoss.forward","title":"forward","text":"<pre><code>forward(input, target)\n</code></pre> <p>Parameters:</p> Name Type Description Default <code>input</code> <code>Tensor</code> <p>Input tensor with predictions (not probabilities)</p> required <code>target</code> <code>Tensor</code> <p>Target tensor with the actual classes</p> required <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt;\n&gt;&gt;&gt; from pytorch_widedeep.losses import FocalR_RMSELoss\n&gt;&gt;&gt;\n&gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n&gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n&gt;&gt;&gt; loss = FocalR_RMSELoss()(input, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(\n    self,\n    input: Tensor,\n    target: Tensor,\n) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    input: Tensor\n        Input tensor with predictions (not probabilities)\n    target: Tensor\n        Target tensor with the actual classes\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; from pytorch_widedeep.losses import FocalR_RMSELoss\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n    &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n    &gt;&gt;&gt; loss = FocalR_RMSELoss()(input, target)\n    \"\"\"\n    loss = (input - target) ** 2\n    if self.activation_fn == \"tanh\":\n        loss *= (torch.tanh(self.beta * torch.abs(input - target))) ** self.gamma\n    elif self.activation_fn == \"sigmoid\":\n        loss *= (\n            2 * torch.sigmoid(self.beta * torch.abs((input - target) ** 2)) - 1\n        ) ** self.gamma\n    else:\n        ValueError(\n            \"Incorrect activation function value - must be in ['sigmoid', 'tanh']\"\n        )\n    return torch.sqrt(torch.mean(loss))\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.HuberLoss","title":"HuberLoss","text":"<p>               Bases: <code>Module</code></p> <p>Hubbler Loss</p> <p>Based on Delving into Deep Imbalanced Regression.</p> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>class HuberLoss(nn.Module):\n    r\"\"\"Hubbler Loss\n\n    Based on [Delving into Deep Imbalanced Regression](https://arxiv.org/abs/2102.09554).\n    \"\"\"\n\n    def __init__(self, beta: float = 0.2):\n        super().__init__()\n        self.beta = beta\n\n    def forward(\n        self,\n        input: Tensor,\n        target: Tensor,\n    ) -&gt; Tensor:\n        r\"\"\"\n        Parameters\n        ----------\n        input: Tensor\n            Input tensor with predictions (not probabilities)\n        target: Tensor\n            Target tensor with the actual classes\n\n        Examples\n        --------\n        &gt;&gt;&gt; import torch\n        &gt;&gt;&gt;\n        &gt;&gt;&gt; from pytorch_widedeep.losses import HuberLoss\n        &gt;&gt;&gt;\n        &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n        &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n        &gt;&gt;&gt; loss = HuberLoss()(input, target)\n        \"\"\"\n        l1_loss = torch.abs(input - target)\n        cond = l1_loss &lt; self.beta\n        loss = torch.where(\n            cond, 0.5 * l1_loss**2 / self.beta, l1_loss - 0.5 * self.beta\n        )\n        return torch.mean(loss)\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.HuberLoss.forward","title":"forward","text":"<pre><code>forward(input, target)\n</code></pre> <p>Parameters:</p> Name Type Description Default <code>input</code> <code>Tensor</code> <p>Input tensor with predictions (not probabilities)</p> required <code>target</code> <code>Tensor</code> <p>Target tensor with the actual classes</p> required <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt;\n&gt;&gt;&gt; from pytorch_widedeep.losses import HuberLoss\n&gt;&gt;&gt;\n&gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n&gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n&gt;&gt;&gt; loss = HuberLoss()(input, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(\n    self,\n    input: Tensor,\n    target: Tensor,\n) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    input: Tensor\n        Input tensor with predictions (not probabilities)\n    target: Tensor\n        Target tensor with the actual classes\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; from pytorch_widedeep.losses import HuberLoss\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n    &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n    &gt;&gt;&gt; loss = HuberLoss()(input, target)\n    \"\"\"\n    l1_loss = torch.abs(input - target)\n    cond = l1_loss &lt; self.beta\n    loss = torch.where(\n        cond, 0.5 * l1_loss**2 / self.beta, l1_loss - 0.5 * self.beta\n    )\n    return torch.mean(loss)\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.InfoNCELoss","title":"InfoNCELoss","text":"<p>               Bases: <code>Module</code></p> <p>InfoNCE Loss. Loss applied during the Contrastive Denoising Self Supervised Pre-training routine available in this library</p> <p> NOTE: This loss is in principle not exposed to  the user, as it is used internally in the library, but it is included  here for completion.</p> <p>See SAINT: Improved Neural Networks for Tabular Data via Row Attention and Contrastive Pre-Training and references therein</p> <p>Partially inspired by the code in this repo</p> <p>Parameters:</p> Name Type Description Default <code>temperature</code> <code>float</code> <p>The logits are divided by the temperature before computing the loss value</p> <code>0.1</code> <code>reduction</code> <code>str</code> <p>Loss reduction method</p> <code>'mean'</code> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>class InfoNCELoss(nn.Module):\n    r\"\"\"InfoNCE Loss. Loss applied during the Contrastive Denoising Self\n    Supervised Pre-training routine available in this library\n\n    :information_source: **NOTE**: This loss is in principle not exposed to\n     the user, as it is used internally in the library, but it is included\n     here for completion.\n\n    See [SAINT: Improved Neural Networks for Tabular Data via Row Attention\n    and Contrastive Pre-Training](https://arxiv.org/abs/2106.01342) and\n    references therein\n\n    Partially inspired by the code in this [repo](https://github.com/RElbers/info-nce-pytorch)\n\n    Parameters\n    ----------\n    temperature: float, default = 0.1\n        The logits are divided by the temperature before computing the loss value\n    reduction: str, default = \"mean\"\n        Loss reduction method\n    \"\"\"\n\n    def __init__(self, temperature: float = 0.1, reduction: str = \"mean\"):\n        super(InfoNCELoss, self).__init__()\n\n        self.temperature = temperature\n        self.reduction = reduction\n\n    def forward(self, g_projs: Tuple[Tensor, Tensor]) -&gt; Tensor:\n        r\"\"\"\n        Parameters\n        ----------\n        g_projs: Tuple\n            Tuple with the two tensors corresponding to the output of the two\n            projection heads, as described 'SAINT: Improved Neural Networks\n            for Tabular Data via Row Attention and Contrastive Pre-Training'.\n\n        Examples\n        --------\n        &gt;&gt;&gt; import torch\n        &gt;&gt;&gt; from pytorch_widedeep.losses import InfoNCELoss\n        &gt;&gt;&gt; g_projs = (torch.rand(3, 5, 16), torch.rand(3, 5, 16))\n        &gt;&gt;&gt; loss = InfoNCELoss()\n        &gt;&gt;&gt; res = loss(g_projs)\n        \"\"\"\n        z, z_ = g_projs[0], g_projs[1]\n\n        norm_z = F.normalize(z, dim=-1).flatten(1)\n        norm_z_ = F.normalize(z_, dim=-1).flatten(1)\n\n        logits = (norm_z @ norm_z_.t()) / self.temperature\n        logits_ = (norm_z_ @ norm_z.t()) / self.temperature\n\n        # the target/labels are the entries on the diagonal\n        target = torch.arange(len(norm_z), device=norm_z.device)\n\n        loss = F.cross_entropy(logits, target, reduction=self.reduction)\n        loss_ = F.cross_entropy(logits_, target, reduction=self.reduction)\n\n        return (loss + loss_) / 2.0\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.InfoNCELoss.forward","title":"forward","text":"<pre><code>forward(g_projs)\n</code></pre> <p>Parameters:</p> Name Type Description Default <code>g_projs</code> <code>Tuple[Tensor, Tensor]</code> <p>Tuple with the two tensors corresponding to the output of the two projection heads, as described 'SAINT: Improved Neural Networks for Tabular Data via Row Attention and Contrastive Pre-Training'.</p> required <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.losses import InfoNCELoss\n&gt;&gt;&gt; g_projs = (torch.rand(3, 5, 16), torch.rand(3, 5, 16))\n&gt;&gt;&gt; loss = InfoNCELoss()\n&gt;&gt;&gt; res = loss(g_projs)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(self, g_projs: Tuple[Tensor, Tensor]) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    g_projs: Tuple\n        Tuple with the two tensors corresponding to the output of the two\n        projection heads, as described 'SAINT: Improved Neural Networks\n        for Tabular Data via Row Attention and Contrastive Pre-Training'.\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.losses import InfoNCELoss\n    &gt;&gt;&gt; g_projs = (torch.rand(3, 5, 16), torch.rand(3, 5, 16))\n    &gt;&gt;&gt; loss = InfoNCELoss()\n    &gt;&gt;&gt; res = loss(g_projs)\n    \"\"\"\n    z, z_ = g_projs[0], g_projs[1]\n\n    norm_z = F.normalize(z, dim=-1).flatten(1)\n    norm_z_ = F.normalize(z_, dim=-1).flatten(1)\n\n    logits = (norm_z @ norm_z_.t()) / self.temperature\n    logits_ = (norm_z_ @ norm_z.t()) / self.temperature\n\n    # the target/labels are the entries on the diagonal\n    target = torch.arange(len(norm_z), device=norm_z.device)\n\n    loss = F.cross_entropy(logits, target, reduction=self.reduction)\n    loss_ = F.cross_entropy(logits_, target, reduction=self.reduction)\n\n    return (loss + loss_) / 2.0\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.DenoisingLoss","title":"DenoisingLoss","text":"<p>               Bases: <code>Module</code></p> <p>Denoising Loss. Loss applied during the Contrastive Denoising Self Supervised Pre-training routine available in this library</p> <p> NOTE: This loss is in principle not exposed to  the user, as it is used internally in the library, but it is included  here for completion.</p> <p>See SAINT: Improved Neural Networks for Tabular Data via Row Attention and Contrastive Pre-Training and references therein</p> <p>Parameters:</p> Name Type Description Default <code>lambda_cat</code> <code>float</code> <p>Multiplicative factor that will be applied to loss associated to the categorical features</p> <code>1.0</code> <code>lambda_cont</code> <code>float</code> <p>Multiplicative factor that will be applied to loss associated to the continuous features</p> <code>1.0</code> <code>reduction</code> <code>str</code> <p>Loss reduction method</p> <code>'mean'</code> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>class DenoisingLoss(nn.Module):\n    r\"\"\"Denoising Loss. Loss applied during the Contrastive Denoising Self\n    Supervised Pre-training routine available in this library\n\n    :information_source: **NOTE**: This loss is in principle not exposed to\n     the user, as it is used internally in the library, but it is included\n     here for completion.\n\n    See [SAINT: Improved Neural Networks for Tabular Data via Row Attention\n    and Contrastive Pre-Training](https://arxiv.org/abs/2106.01342) and\n    references therein\n\n    Parameters\n    ----------\n    lambda_cat: float, default = 1.\n        Multiplicative factor that will be applied to loss associated to the\n        categorical features\n    lambda_cont: float, default = 1.\n        Multiplicative factor that will be applied to loss associated to the\n        continuous features\n    reduction: str, default = \"mean\"\n        Loss reduction method\n    \"\"\"\n\n    def __init__(\n        self, lambda_cat: float = 1.0, lambda_cont: float = 1.0, reduction: str = \"mean\"\n    ):\n        super(DenoisingLoss, self).__init__()\n\n        self.lambda_cat = lambda_cat\n        self.lambda_cont = lambda_cont\n        self.reduction = reduction\n\n    def forward(\n        self,\n        x_cat_and_cat_: Optional[\n            Union[List[Tuple[Tensor, Tensor]], Tuple[Tensor, Tensor]]\n        ],\n        x_cont_and_cont_: Optional[\n            Union[List[Tuple[Tensor, Tensor]], Tuple[Tensor, Tensor]]\n        ],\n    ) -&gt; Tensor:\n        r\"\"\"\n        Parameters\n        ----------\n        x_cat_and_cat_: tuple of Tensors or lists of tuples\n            Tuple of tensors containing the raw input features and their\n            encodings, referred in the SAINT paper as $x$ and $x''$\n            respectively. If one denoising MLP is used per categorical\n            feature `x_cat_and_cat_` will be a list of tuples, one per\n            categorical feature\n        x_cont_and_cont_: tuple of Tensors or lists of tuples\n            same as `x_cat_and_cat_` but for continuous columns\n\n        Examples\n        --------\n        &gt;&gt;&gt; import torch\n        &gt;&gt;&gt; from pytorch_widedeep.losses import DenoisingLoss\n        &gt;&gt;&gt; x_cat_and_cat_ = (torch.empty(3).random_(3).long(), torch.randn(3, 3))\n        &gt;&gt;&gt; x_cont_and_cont_ = (torch.randn(3, 1), torch.randn(3, 1))\n        &gt;&gt;&gt; loss = DenoisingLoss()\n        &gt;&gt;&gt; res = loss(x_cat_and_cat_, x_cont_and_cont_)\n        \"\"\"\n\n        loss_cat = (\n            self._compute_cat_loss(x_cat_and_cat_)\n            if x_cat_and_cat_ is not None\n            else torch.tensor(0.0)\n        )\n        loss_cont = (\n            self._compute_cont_loss(x_cont_and_cont_)\n            if x_cont_and_cont_ is not None\n            else torch.tensor(0.0)\n        )\n\n        return self.lambda_cat * loss_cat + self.lambda_cont * loss_cont\n\n    def _compute_cat_loss(\n        self, x_cat_and_cat_: Union[List[Tuple[Tensor, Tensor]], Tuple[Tensor, Tensor]]\n    ) -&gt; Tensor:\n        loss_cat = torch.tensor(0.0, device=self._get_device(x_cat_and_cat_))\n        if isinstance(x_cat_and_cat_, list):\n            for x, x_ in x_cat_and_cat_:\n                loss_cat += F.cross_entropy(x_, x, reduction=self.reduction)\n        elif isinstance(x_cat_and_cat_, tuple):\n            x, x_ = x_cat_and_cat_\n            loss_cat += F.cross_entropy(x_, x, reduction=self.reduction)\n\n        return loss_cat\n\n    def _compute_cont_loss(self, x_cont_and_cont_) -&gt; Tensor:\n        loss_cont = torch.tensor(0.0, device=self._get_device(x_cont_and_cont_))\n        if isinstance(x_cont_and_cont_, list):\n            for x, x_ in x_cont_and_cont_:\n                loss_cont += F.mse_loss(x_, x, reduction=self.reduction)\n        elif isinstance(x_cont_and_cont_, tuple):\n            x, x_ = x_cont_and_cont_\n            loss_cont += F.mse_loss(x_, x, reduction=self.reduction)\n\n        return loss_cont\n\n    @staticmethod\n    def _get_device(\n        x_and_x_: Union[List[Tuple[Tensor, Tensor]], Tuple[Tensor, Tensor]]\n    ):\n        if isinstance(x_and_x_, tuple):\n            device = x_and_x_[0].device\n        elif isinstance(x_and_x_, list):\n            device = x_and_x_[0][0].device\n        return device\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.DenoisingLoss.forward","title":"forward","text":"<pre><code>forward(x_cat_and_cat_, x_cont_and_cont_)\n</code></pre> <p>Parameters:</p> Name Type Description Default <code>x_cat_and_cat_</code> <code>Optional[Union[List[Tuple[Tensor, Tensor]], Tuple[Tensor, Tensor]]]</code> <p>Tuple of tensors containing the raw input features and their encodings, referred in the SAINT paper as \\(x\\) and \\(x''\\) respectively. If one denoising MLP is used per categorical feature <code>x_cat_and_cat_</code> will be a list of tuples, one per categorical feature</p> required <code>x_cont_and_cont_</code> <code>Optional[Union[List[Tuple[Tensor, Tensor]], Tuple[Tensor, Tensor]]]</code> <p>same as <code>x_cat_and_cat_</code> but for continuous columns</p> required <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.losses import DenoisingLoss\n&gt;&gt;&gt; x_cat_and_cat_ = (torch.empty(3).random_(3).long(), torch.randn(3, 3))\n&gt;&gt;&gt; x_cont_and_cont_ = (torch.randn(3, 1), torch.randn(3, 1))\n&gt;&gt;&gt; loss = DenoisingLoss()\n&gt;&gt;&gt; res = loss(x_cat_and_cat_, x_cont_and_cont_)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(\n    self,\n    x_cat_and_cat_: Optional[\n        Union[List[Tuple[Tensor, Tensor]], Tuple[Tensor, Tensor]]\n    ],\n    x_cont_and_cont_: Optional[\n        Union[List[Tuple[Tensor, Tensor]], Tuple[Tensor, Tensor]]\n    ],\n) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    x_cat_and_cat_: tuple of Tensors or lists of tuples\n        Tuple of tensors containing the raw input features and their\n        encodings, referred in the SAINT paper as $x$ and $x''$\n        respectively. If one denoising MLP is used per categorical\n        feature `x_cat_and_cat_` will be a list of tuples, one per\n        categorical feature\n    x_cont_and_cont_: tuple of Tensors or lists of tuples\n        same as `x_cat_and_cat_` but for continuous columns\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.losses import DenoisingLoss\n    &gt;&gt;&gt; x_cat_and_cat_ = (torch.empty(3).random_(3).long(), torch.randn(3, 3))\n    &gt;&gt;&gt; x_cont_and_cont_ = (torch.randn(3, 1), torch.randn(3, 1))\n    &gt;&gt;&gt; loss = DenoisingLoss()\n    &gt;&gt;&gt; res = loss(x_cat_and_cat_, x_cont_and_cont_)\n    \"\"\"\n\n    loss_cat = (\n        self._compute_cat_loss(x_cat_and_cat_)\n        if x_cat_and_cat_ is not None\n        else torch.tensor(0.0)\n    )\n    loss_cont = (\n        self._compute_cont_loss(x_cont_and_cont_)\n        if x_cont_and_cont_ is not None\n        else torch.tensor(0.0)\n    )\n\n    return self.lambda_cat * loss_cat + self.lambda_cont * loss_cont\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.EncoderDecoderLoss","title":"EncoderDecoderLoss","text":"<p>               Bases: <code>Module</code></p> <p>'Standard' Encoder Decoder Loss. Loss applied during the Endoder-Decoder  Self-Supervised Pre-Training routine available in this library</p> <p> NOTE: This loss is in principle not exposed to  the user, as it is used internally in the library, but it is included  here for completion.</p> <p>The implementation of this lost is based on that at the tabnet repo, which is in itself an adaptation of that in the original paper TabNet: Attentive Interpretable Tabular Learning.</p> <p>Parameters:</p> Name Type Description Default <code>eps</code> <code>float</code> <p>Simply a small number to avoid dividing by zero</p> <code>1e-09</code> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>class EncoderDecoderLoss(nn.Module):\n    r\"\"\"'_Standard_' Encoder Decoder Loss. Loss applied during the Endoder-Decoder\n     Self-Supervised Pre-Training routine available in this library\n\n    :information_source: **NOTE**: This loss is in principle not exposed to\n     the user, as it is used internally in the library, but it is included\n     here for completion.\n\n    The implementation of this lost is based on that at the\n    [tabnet repo](https://github.com/dreamquark-ai/tabnet), which is in itself an\n    adaptation of that in the original paper [TabNet: Attentive\n    Interpretable Tabular Learning](https://arxiv.org/abs/1908.07442).\n\n    Parameters\n    ----------\n    eps: float\n        Simply a small number to avoid dividing by zero\n    \"\"\"\n\n    def __init__(self, eps: float = 1e-9):\n        super(EncoderDecoderLoss, self).__init__()\n        self.eps = eps\n\n    def forward(self, x_true: Tensor, x_pred: Tensor, mask: Tensor) -&gt; Tensor:\n        r\"\"\"\n        Parameters\n        ----------\n        x_true: Tensor\n            Embeddings of the input data\n        x_pred: Tensor\n            Reconstructed embeddings\n        mask: Tensor\n            Mask with 1s indicated that the reconstruction, and therefore the\n            loss, is based on those features.\n\n        Examples\n        --------\n        &gt;&gt;&gt; import torch\n        &gt;&gt;&gt; from pytorch_widedeep.losses import EncoderDecoderLoss\n        &gt;&gt;&gt; x_true = torch.rand(3, 3)\n        &gt;&gt;&gt; x_pred = torch.rand(3, 3)\n        &gt;&gt;&gt; mask = torch.empty(3, 3).random_(2)\n        &gt;&gt;&gt; loss = EncoderDecoderLoss()\n        &gt;&gt;&gt; res = loss(x_true, x_pred, mask)\n        \"\"\"\n\n        errors = x_pred - x_true\n\n        reconstruction_errors = torch.mul(errors, mask) ** 2\n\n        x_true_means = torch.mean(x_true, dim=0)\n        x_true_means[x_true_means == 0] = 1\n\n        x_true_stds = torch.std(x_true, dim=0) ** 2\n        x_true_stds[x_true_stds == 0] = x_true_means[x_true_stds == 0]\n\n        features_loss = torch.matmul(reconstruction_errors, 1 / x_true_stds)\n        nb_reconstructed_variables = torch.sum(mask, dim=1)\n        features_loss_norm = features_loss / (nb_reconstructed_variables + self.eps)\n\n        loss = torch.mean(features_loss_norm)\n\n        return loss\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.EncoderDecoderLoss.forward","title":"forward","text":"<pre><code>forward(x_true, x_pred, mask)\n</code></pre> <p>Parameters:</p> Name Type Description Default <code>x_true</code> <code>Tensor</code> <p>Embeddings of the input data</p> required <code>x_pred</code> <code>Tensor</code> <p>Reconstructed embeddings</p> required <code>mask</code> <code>Tensor</code> <p>Mask with 1s indicated that the reconstruction, and therefore the loss, is based on those features.</p> required <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.losses import EncoderDecoderLoss\n&gt;&gt;&gt; x_true = torch.rand(3, 3)\n&gt;&gt;&gt; x_pred = torch.rand(3, 3)\n&gt;&gt;&gt; mask = torch.empty(3, 3).random_(2)\n&gt;&gt;&gt; loss = EncoderDecoderLoss()\n&gt;&gt;&gt; res = loss(x_true, x_pred, mask)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(self, x_true: Tensor, x_pred: Tensor, mask: Tensor) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    x_true: Tensor\n        Embeddings of the input data\n    x_pred: Tensor\n        Reconstructed embeddings\n    mask: Tensor\n        Mask with 1s indicated that the reconstruction, and therefore the\n        loss, is based on those features.\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.losses import EncoderDecoderLoss\n    &gt;&gt;&gt; x_true = torch.rand(3, 3)\n    &gt;&gt;&gt; x_pred = torch.rand(3, 3)\n    &gt;&gt;&gt; mask = torch.empty(3, 3).random_(2)\n    &gt;&gt;&gt; loss = EncoderDecoderLoss()\n    &gt;&gt;&gt; res = loss(x_true, x_pred, mask)\n    \"\"\"\n\n    errors = x_pred - x_true\n\n    reconstruction_errors = torch.mul(errors, mask) ** 2\n\n    x_true_means = torch.mean(x_true, dim=0)\n    x_true_means[x_true_means == 0] = 1\n\n    x_true_stds = torch.std(x_true, dim=0) ** 2\n    x_true_stds[x_true_stds == 0] = x_true_means[x_true_stds == 0]\n\n    features_loss = torch.matmul(reconstruction_errors, 1 / x_true_stds)\n    nb_reconstructed_variables = torch.sum(mask, dim=1)\n    features_loss_norm = features_loss / (nb_reconstructed_variables + self.eps)\n\n    loss = torch.mean(features_loss_norm)\n\n    return loss\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses_multitarget.MultiTargetRegressionLoss","title":"MultiTargetRegressionLoss","text":"<p>               Bases: <code>Module</code></p> <p>This class is a wrapper around the Pytorch MSELoss. It allows for multi-target regression problems. The user can provide a list of weights to apply to each target. The loss can be either the sum or the mean of the individual losses</p> <p>Parameters:</p> Name Type Description Default <code>weights</code> <code>Optional[List[float]]</code> <p>List of weights to apply to the loss associated to each target. The length of the list must match the number of targets. Alias: 'target_weights'</p> <code>None</code> <code>reduction</code> <code>Literal[mean, sum]</code> <p>Specifies the reduction to apply to the loss associated to each target: 'mean' | 'sum'. Note that this is NOT the same as the reduction in the MSELoss. This reduction is applied after the loss for each target has been computed. Alias: 'target_reduction'</p> <code>'mean'</code> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.losses_multitarget import MultiTargetRegressionLoss\n&gt;&gt;&gt; input = torch.randn(3, 2)\n&gt;&gt;&gt; target = torch.randn(3, 2)\n&gt;&gt;&gt; loss = MultiTargetRegressionLoss(weights=[0.5, 0.5], reduction=\"mean\")\n&gt;&gt;&gt; output = loss(input, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses_multitarget.py</code> <pre><code>class MultiTargetRegressionLoss(nn.Module):\n    \"\"\"\n    This class is a wrapper around the Pytorch MSELoss. It allows for multi-target\n    regression problems. The user can provide a list of weights to apply to each\n    target. The loss can be either the sum or the mean of the individual losses\n\n    Parameters\n    ----------\n    weights: Optional[List[float], default = None]\n        List of weights to apply to the loss associated to each target. The\n        length of the list must match the number of targets.\n        Alias: 'target_weights'\n    reduction: Literal[\"mean\", \"sum\"], default = \"mean\n        Specifies the reduction to apply to the loss associated to each\n        target: 'mean' | 'sum'. Note that this is NOT the same as the\n        reduction in the MSELoss. This reduction is applied after the loss\n        for each target has been computed. Alias: 'target_reduction'\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.losses_multitarget import MultiTargetRegressionLoss\n    &gt;&gt;&gt; input = torch.randn(3, 2)\n    &gt;&gt;&gt; target = torch.randn(3, 2)\n    &gt;&gt;&gt; loss = MultiTargetRegressionLoss(weights=[0.5, 0.5], reduction=\"mean\")\n    &gt;&gt;&gt; output = loss(input, target)\n    \"\"\"\n\n    @alias(\"reduction\", [\"target_reduction\"])\n    @alias(\"weights\", [\"target_weights\"])\n    def __init__(\n        self,\n        weights: Optional[List[float]] = None,\n        reduction: Literal[\"mean\", \"sum\"] = \"mean\",\n    ):\n        super(MultiTargetRegressionLoss, self).__init__()\n\n        self.weights = weights\n        self.reduction = reduction\n\n        if self.reduction not in [\"mean\", \"sum\"]:\n            raise ValueError(\"reduction must be either 'mean' or 'sum'\")\n\n    def forward(self, input: Tensor, target: Tensor) -&gt; Tensor:\n\n        assert input.size() == target.size()\n\n        if self.weights is not None:\n\n            assert len(self.weights) == input.size(1), (\n                \"The number of weights must match the number of targets. \"\n                f\"Got {len(self.weights)} weights and {input.size(1)} targets\"\n            )\n\n            loss = F.mse_loss(input, target, reduction=\"none\") * torch.tensor(\n                self.weights\n            ).to(input.device)\n        else:\n            loss = F.mse_loss(input, target, reduction=\"none\")\n\n        return loss.mean() if self.reduction == \"mean\" else loss.sum()\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses_multitarget.MultiTargetClassificationLoss","title":"MultiTargetClassificationLoss","text":"<p>               Bases: <code>Module</code></p> <p>This class is a wrapper around the Pytorch binary_cross_entropy_with_logits and cross_entropy losses. It allows for multi-target classification problems. The user can provide a list of weights to apply to each target. The loss can be either the sum or the mean of the individual losses</p> <p>Parameters:</p> Name Type Description Default <code>binary_config</code> <code>Optional[List[Union[int, Tuple[int, float]]]]</code> <p>List of integers with the index of the target for binary classification or tuples with two elements: the index of the targets or binary classification and the positive weight for binary classification</p> <code>None</code> <code>multiclass_config</code> <code>Optional[List[Union[Tuple[int, int], Tuple[int, int, List[float]]]]]</code> <p>List of tuples with two or three elements: the index of the target and the number of classes for multiclass classification, or a tuple with the index of the target, the number of classes and a list of weights to apply to each class (i.e. the 'weight' parameter in the cross_entropy loss)</p> <code>None</code> <code>weights</code> <code>Optional[List[float]]</code> <p>List of weights to apply to the loss associated to each target. The length of the list must match the number of targets. Alias: 'target_weights'</p> <code>None</code> <code>reduction</code> <code>Literal[mean, sum]</code> <p>Specifies the reduction to apply to the loss associated to each target: 'mean' | 'sum'. Note that this is NOT the same as the reduction in the cross_entropy loss or the binary_cross_entropy_with_logits. This reduction is applied after the loss for each target has been computed. Alias: 'target_reduction'</p> <code>'mean'</code> <code>binary_trick</code> <code>bool</code> <p>If True, each target will be considered independently and the loss will be computed as binary_cross_entropy_with_logits. This is a faster implementation. Note that the 'weights' parameter is not compatible with binary_trick=True. Also note that if binary_trick=True, the 'binary_config' must be a list of integers and the 'multiclass_config' must be a list of tuples with two integers: the index of the target and the number of classes. Finally, if binary_trick=True, the binary targets must be the first targets in the target tensor.</p> <p> NOTE: When using the binary_trick, the binary targets are   considered as 2 classes. Therefore, the pred_dim parametere of the   WideDeep class should be adjusted accordingly (adding 2 to per   binary target). For example, in a problem with a binary target and   a 4 class multiclassification target, the pred_dim should be 6.</p> <code>False</code> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.losses_multitarget import MultiTargetClassificationLoss\n&gt;&gt;&gt; input = torch.randn(5, 4)\n&gt;&gt;&gt; input_binary_trick = torch.randn(5, 5)\n&gt;&gt;&gt; target = torch.stack([torch.tensor([0, 1, 0, 1, 1]), torch.tensor([0, 1, 2, 0, 2])], 1)\n&gt;&gt;&gt; loss_1 = MultiTargetClassificationLoss(binary_config=[0], multiclass_config=[(1, 3)], reduction=\"mean\")\n&gt;&gt;&gt; output_1 = loss_1(input, target)\n&gt;&gt;&gt; loss_2 = MultiTargetClassificationLoss(binary_config=[(0, 0.5)], multiclass_config=[(1, 3, [1., 2., 3.])],\n... reduction=\"sum\", weights=[0.5, 0.5])\n&gt;&gt;&gt; output_2 = loss_2(input, target)\n&gt;&gt;&gt; loss_3 = MultiTargetClassificationLoss(binary_config=[0], multiclass_config=[(1, 3)], binary_trick=True)\n&gt;&gt;&gt; output_3 = loss_3(input_binary_trick, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses_multitarget.py</code> <pre><code>class MultiTargetClassificationLoss(nn.Module):\n    \"\"\"\n    This class is a wrapper around the Pytorch binary_cross_entropy_with_logits and\n    cross_entropy losses. It allows for multi-target classification problems. The\n    user can provide a list of weights to apply to each target. The loss can be\n    either the sum or the mean of the individual losses\n\n    Parameters\n    ----------\n    binary_config: Optional[List[int | Tuple[int, float]]], default = None\n        List of integers with the index of the target for binary\n        classification or tuples with two elements: the index of the targets\n        or binary classification and the positive weight for binary\n        classification\n    multiclass_config: Optional[Tuple[int, int] | Tuple[int, int, List[float]]], default = None\n        List of tuples with two or three elements: the index of the target and the\n        number of classes for multiclass classification, or a tuple with the index of\n        the target, the number of classes and a list of weights to apply to each class\n        (i.e. the 'weight' parameter in the cross_entropy loss)\n    weights: Optional[List[float], default = None]\n        List of weights to apply to the loss associated to each target. The\n        length of the list must match the number of targets.\n        Alias: 'target_weights'\n    reduction: Literal[\"mean\", \"sum\"], default = \"sum\n        Specifies the reduction to apply to the loss associated to each\n        target: 'mean' | 'sum'. Note that this is NOT the same as the\n        reduction in the cross_entropy loss or the\n        binary_cross_entropy_with_logits. This reduction is applied after the\n        loss for each target has been computed. Alias: 'target_reduction'\n    binary_trick: bool, default = False\n        If True, each target will be considered independently and the loss\n        will be computed as binary_cross_entropy_with_logits. This is a\n        faster implementation. Note that the 'weights' parameter is not\n        compatible with binary_trick=True. Also note that if\n        binary_trick=True, the 'binary_config' must be a list of integers and\n        the 'multiclass_config' must be a list of tuples with two integers:\n        the index of the target and the number of classes. Finally, if\n        binary_trick=True, the binary targets must be the first targets in\n        the target tensor.\n\n        :information_source: **NOTE**: When using the binary_trick, the binary targets are\n          considered as 2 classes. Therefore, the pred_dim parametere of the\n          WideDeep class should be adjusted accordingly (adding 2 to per\n          binary target). For example, in a problem with a binary target and\n          a 4 class multiclassification target, the pred_dim should be 6.\n\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.losses_multitarget import MultiTargetClassificationLoss\n    &gt;&gt;&gt; input = torch.randn(5, 4)\n    &gt;&gt;&gt; input_binary_trick = torch.randn(5, 5)\n    &gt;&gt;&gt; target = torch.stack([torch.tensor([0, 1, 0, 1, 1]), torch.tensor([0, 1, 2, 0, 2])], 1)\n    &gt;&gt;&gt; loss_1 = MultiTargetClassificationLoss(binary_config=[0], multiclass_config=[(1, 3)], reduction=\"mean\")\n    &gt;&gt;&gt; output_1 = loss_1(input, target)\n    &gt;&gt;&gt; loss_2 = MultiTargetClassificationLoss(binary_config=[(0, 0.5)], multiclass_config=[(1, 3, [1., 2., 3.])],\n    ... reduction=\"sum\", weights=[0.5, 0.5])\n    &gt;&gt;&gt; output_2 = loss_2(input, target)\n    &gt;&gt;&gt; loss_3 = MultiTargetClassificationLoss(binary_config=[0], multiclass_config=[(1, 3)], binary_trick=True)\n    &gt;&gt;&gt; output_3 = loss_3(input_binary_trick, target)\n    \"\"\"\n\n    @alias(\"reduction\", [\"target_reduction\"])\n    @alias(\"weights\", [\"target_weights\"])\n    def __init__(  # noqa: C901\n        self,\n        binary_config: Optional[List[Union[int, Tuple[int, float]]]] = None,\n        multiclass_config: Optional[\n            List[Union[Tuple[int, int], Tuple[int, int, List[float]]]]\n        ] = None,\n        weights: Optional[List[float]] = None,\n        reduction: Literal[\"mean\", \"sum\"] = \"mean\",\n        binary_trick: bool = False,\n    ):\n        super(MultiTargetClassificationLoss, self).__init__()\n\n        if reduction not in [\"mean\", \"sum\"]:\n            raise ValueError(\"reduction must be either 'mean' or 'sum'\")\n\n        self.binary_config = binary_config\n        self.multiclass_config = multiclass_config\n        self.weights = weights\n        self.reduction = reduction\n        self.binary_trick = binary_trick\n\n        if self.weights is not None:\n            if len(self.weights) != (\n                len(self.binary_config) if self.binary_config is not None else 0\n            ) + (\n                len(self.multiclass_config) if self.multiclass_config is not None else 0\n            ):\n                raise ValueError(\n                    \"The number of weights must match the number of binary and multiclass targets\"\n                )\n\n        if self.binary_trick:\n            self._check_inputs_with_binary_trick()\n            self._binary_config: List[int] = binary_config  # type: ignore[assignment]\n            self._multiclass_config: List[Tuple[int, int]] = self.multiclass_config  # type: ignore[assignment]\n        else:\n            self.binary_config_with_pos_weights = (\n                (self._set_binary_config_without_binary_trick())\n                if self.binary_config is not None\n                else None\n            )\n            self.multiclass_config_with_weights = (\n                (self._set_multiclass_config_without_binary_trick())\n                if self.multiclass_config is not None\n                else None\n            )\n\n    def forward(self, input: Tensor, target: Tensor) -&gt; Tensor:\n        if self.binary_trick:\n            return self._forward_binary_trick(input, target)\n        else:\n            return self._forward_without_binary_trick(input, target)\n\n    def _forward_binary_trick(self, input: Tensor, target: Tensor) -&gt; Tensor:\n        binary_target_tensors: List[Tensor] = []\n        if self._binary_config:\n            for idx in self._binary_config:\n                binary_target_tensors.append(\n                    torch.eye(2)[target[:, idx].long()].to(input.device)\n                )\n        if self._multiclass_config:\n            for idx, n_classes in self._multiclass_config:\n                binary_target_tensors.append(\n                    torch.eye(n_classes)[target[:, idx].long()].to(input.device)\n                )\n        binary_target = torch.cat(binary_target_tensors, 1)\n        return F.binary_cross_entropy_with_logits(input, binary_target)\n\n    def _forward_without_binary_trick(self, input: Tensor, target: Tensor) -&gt; Tensor:\n        losses: List[Tensor] = []\n        if self.binary_config_with_pos_weights:\n            for idx, bpos_weight in self.binary_config_with_pos_weights:\n                _loss = F.binary_cross_entropy_with_logits(\n                    input[:, idx],\n                    target[:, idx].float(),\n                    pos_weight=(\n                        torch.tensor(bpos_weight).to(input.device)\n                        if bpos_weight is not None\n                        else None\n                    ),\n                )\n                losses.append(_loss)\n        if self.multiclass_config_with_weights:\n            for idx, n_classes, mpos_weight in self.multiclass_config_with_weights:\n                _loss = F.cross_entropy(\n                    input[:, idx : idx + n_classes],\n                    target[:, idx].long(),\n                    weight=(\n                        torch.tensor(mpos_weight).to(input.device)\n                        if mpos_weight is not None\n                        else None\n                    ),\n                )\n                losses.append(_loss)\n\n            if self.weights is not None:\n                losses = [l * w for l, w in zip(losses, self.weights)]  # noqa: E741\n\n        return (\n            torch.stack(losses).sum()\n            if self.reduction == \"sum\"\n            else torch.stack(losses).mean()\n        )\n\n    def _check_inputs_with_binary_trick(self):\n        if self.binary_config is not None:\n            if any(isinstance(bc, tuple) for bc in self.binary_config):\n                raise ValueError(\n                    \"binary_trick=True is only compatible with binary_config as a list of integers\"\n                )\n\n        if self.multiclass_config is not None:\n            if not all(len(mc) == 2 for mc in self.multiclass_config):\n                raise ValueError(\n                    \"binary_trick=True is only compatible with multiclass_config as a list of \"\n                    \"tuples with two integers: the index of the target and the number of classes\"\n                )\n\n        if self.binary_config is not None and self.multiclass_config is not None:\n            last_binary_idx = (\n                self.binary_config[-1][0]\n                if isinstance(self.binary_config[-1], tuple)\n                else self.binary_config[-1]\n            )\n            if last_binary_idx &gt;= self.multiclass_config[0][0]:\n                raise ValueError(\n                    \"When using binary_trick=True, the binary targets must be the first targets\"\n                    \" in the target tensor\"\n                )\n\n    def _set_binary_config_without_binary_trick(\n        self,\n    ) -&gt; List[Tuple[int, Optional[float]]]:\n        binary_config_with_pos_weights: List[Tuple[int, Optional[float]]] = []\n        for bc in self.binary_config:\n            if isinstance(bc, tuple):\n                binary_config_with_pos_weights.append(bc)\n            else:\n                binary_config_with_pos_weights.append((bc, None))\n        return binary_config_with_pos_weights\n\n    def _set_multiclass_config_without_binary_trick(\n        self,\n    ) -&gt; List[Tuple[int, int, Optional[List[float]]]]:\n        multiclass_config_with_weights: List[Tuple[int, int, Optional[List[float]]]] = (\n            []\n        )\n        for mc in self.multiclass_config:\n            if len(mc) == 3:\n                multiclass_config_with_weights.append(mc)  # type: ignore[arg-type]\n            else:\n                multiclass_config_with_weights.append((mc[0], mc[1], None))\n        return multiclass_config_with_weights\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses_multitarget.MutilTargetRegressionAndClassificationLoss","title":"MutilTargetRegressionAndClassificationLoss","text":"<p>               Bases: <code>Module</code></p> <p>This class is a wrapper around the MultiTargetRegressionLoss and the MultiTargetClassificationLoss. It allows for multi-target regression and classification problems. The user can provide a list of weights to apply to each target. The loss can be either the sum or the mean of the individual losses</p> <p>Parameters:</p> Name Type Description Default <code>regression_config</code> <code>List[int]</code> <p>List of integers with the indices of the regression targets</p> <code>[]</code> <code>binary_config</code> <code>Optional[List[Union[int, Tuple[int, float]]]]</code> <p>List of integers with the index of the target for binary classification or tuples with two elements: the index of the targets or binary classification and the positive weight for binary classification</p> <code>None</code> <code>multiclass_config</code> <code>Optional[List[Union[Tuple[int, int], Tuple[int, int, List[float]]]]]</code> <p>List of tuples with two or three elements: the index of the target and the number of classes for multiclass classification, or a tuple with the index of the target, the number of classes and a list of weights to apply to each class (i.e. the 'weight' parameter in the cross_entropy loss)</p> <code>None</code> <code>weights</code> <code>Optional[List[float]]</code> <p>List of weights to apply to the loss associated to each target. The length of the list must match the number of targets. Alias: 'target_weights'</p> <code>None</code> <code>reduction</code> <code>Literal[mean, sum]</code> <p>Specifies the reduction to apply to the output: 'mean' | 'sum'. Note that this is NOT the same as the reduction in the cross_entropy loss, the binary_cross_entropy_with_logits or the MSELoss. This reduction is applied after each target has been computed. Alias: 'target_reduction'</p> <code>'mean'</code> <code>binary_trick</code> <code>bool</code> <p>If True, each target will be considered independently and the loss will be computed as binary_cross_entropy_with_logits. This is a faster implementation. Note that the 'weights' parameter is not compatible with binary_trick=True. Also note that if binary_trick=True, the 'binary_config' must be a list of integers and the 'multiclass_config' must be a list of tuples with two integers: the index of the target and the number of classes. Finally, if binary_trick=True, the binary targets must be the first targets in the target tensor.</p> <p> NOTE: When using the binary_trick, the binary targets are   considered as 2 classes. Therefore, the pred_dim parametere of the   WideDeep class should be adjusted accordingly (adding 2 to per   binary target). For example, in a problem with a binary target and   a 4 class multiclassification target, the pred_dim should be 6.</p> <code>False</code> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.losses_multitarget import MutilTargetRegressionAndClassificationLoss\n&gt;&gt;&gt; input = torch.randn(5, 5)\n&gt;&gt;&gt; target = torch.stack([torch.randn(5), torch.tensor([0, 1, 0, 1, 1]), torch.tensor([0, 1, 2, 0, 2])], 1)\n&gt;&gt;&gt; loss = MutilTargetRegressionAndClassificationLoss(regression_config=[0], binary_config=[2],\n... multiclass_config=[(2, 3)], reduction=\"mean\")\n&gt;&gt;&gt; output = loss(input, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses_multitarget.py</code> <pre><code>class MutilTargetRegressionAndClassificationLoss(nn.Module):\n    \"\"\"\n    This class is a wrapper around the MultiTargetRegressionLoss and the\n    MultiTargetClassificationLoss. It allows for multi-target regression and\n    classification problems. The user can provide a list of weights to apply to\n    each target. The loss can be either the sum or the mean of the individual losses\n\n    Parameters\n    ----------\n    regression_config: List[int], default = []\n        List of integers with the indices of the regression targets\n    binary_config: Optional[List[int | Tuple[int, float]]], default = None\n        List of integers with the index of the target for binary\n        classification or tuples with two elements: the index of the targets\n        or binary classification and the positive weight for binary\n        classification\n    multiclass_config: Optional[Tuple[int, int] | Tuple[int, int, List[float]]], default = None\n        List of tuples with two or three elements: the index of the target and the\n        number of classes for multiclass classification, or a tuple with the index of\n        the target, the number of classes and a list of weights to apply to each class\n        (i.e. the 'weight' parameter in the cross_entropy loss)\n    weights: Optional[List[float], default = None]\n        List of weights to apply to the loss associated to each target. The\n        length of the list must match the number of targets.\n        Alias: 'target_weights'\n    reduction: Literal[\"mean\", \"sum\"], default = \"sum\n        Specifies the reduction to apply to the output: 'mean' | 'sum'. Note\n        that this is NOT the same as the reduction in the cross_entropy loss,\n        the binary_cross_entropy_with_logits or the MSELoss. This reduction\n        is applied after each target has been computed. Alias: 'target_reduction'\n    binary_trick: bool, default = False\n        If True, each target will be considered independently and the loss\n        will be computed as binary_cross_entropy_with_logits. This is a\n        faster implementation. Note that the 'weights' parameter is not\n        compatible with binary_trick=True. Also note that if\n        binary_trick=True, the 'binary_config' must be a list of integers and\n        the 'multiclass_config' must be a list of tuples with two integers:\n        the index of the target and the number of classes. Finally, if\n        binary_trick=True, the binary targets must be the first targets in\n        the target tensor.\n\n        :information_source: **NOTE**: When using the binary_trick, the binary targets are\n          considered as 2 classes. Therefore, the pred_dim parametere of the\n          WideDeep class should be adjusted accordingly (adding 2 to per\n          binary target). For example, in a problem with a binary target and\n          a 4 class multiclassification target, the pred_dim should be 6.\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.losses_multitarget import MutilTargetRegressionAndClassificationLoss\n    &gt;&gt;&gt; input = torch.randn(5, 5)\n    &gt;&gt;&gt; target = torch.stack([torch.randn(5), torch.tensor([0, 1, 0, 1, 1]), torch.tensor([0, 1, 2, 0, 2])], 1)\n    &gt;&gt;&gt; loss = MutilTargetRegressionAndClassificationLoss(regression_config=[0], binary_config=[2],\n    ... multiclass_config=[(2, 3)], reduction=\"mean\")\n    &gt;&gt;&gt; output = loss(input, target)\n    \"\"\"\n\n    @alias(\"reduction\", [\"target_reduction\"])\n    @alias(\"weights\", [\"target_weights\"])\n    def __init__(  # noqa: C901\n        self,\n        regression_config: List[int] = [],\n        binary_config: Optional[List[Union[int, Tuple[int, float]]]] = None,\n        multiclass_config: Optional[\n            List[Union[Tuple[int, int], Tuple[int, int, List[float]]]]\n        ] = None,\n        weights: Optional[List[float]] = None,\n        reduction: Literal[\"mean\", \"sum\"] = \"mean\",\n        binary_trick: bool = False,\n    ):\n\n        super(MutilTargetRegressionAndClassificationLoss, self).__init__()\n\n        self.regression_config = regression_config\n\n        assert binary_config is not None or multiclass_config is not None, (\n            \"Either binary_config or multiclass_config must be provided. \"\n            \"Otherwise, use the MultiTargetRegressionLoss\"\n        )\n\n        if binary_trick:\n            self._check_inputs_with_binary_trick(\n                regression_config, binary_config, multiclass_config\n            )\n\n        if weights is not None:\n            if len(weights) != (\n                len(regression_config)\n                + (len(binary_config) if binary_config is not None else 0)\n                + (len(multiclass_config) if multiclass_config is not None else 0)\n            ):\n                raise ValueError(\n                    \"The number of weights must match the number of regression, binary and multiclass targets\"\n                )\n\n            self.weights_regression = self._prepare_weights_for_regression_targets(\n                weights, regression_config\n            )\n            self.weights_binary = self._prepare_weights_per_binary_targets(\n                weights, binary_config\n            )\n            self.weights_multiclass = self._prepare_weights_per_multiclass_targets(\n                weights, multiclass_config\n            )\n            self.weights = weights\n        else:\n            self.weights_regression = None\n            self.weights_binary = None\n            self.weights_multiclass = None\n\n        self.multi_target_regression_loss = MultiTargetRegressionLoss(\n            weights=self.weights_regression, reduction=reduction\n        )\n\n        self.multi_target_classification_loss = MultiTargetClassificationLoss(\n            binary_config=binary_config,\n            multiclass_config=multiclass_config,\n            weights=(\n                self.weights_binary + self.weights_multiclass\n                if self.weights_binary is not None\n                and self.weights_multiclass is not None\n                else (\n                    self.weights_binary\n                    if self.weights_binary is not None\n                    else self.weights_multiclass\n                )\n            ),\n            reduction=reduction,\n            binary_trick=binary_trick,\n        )\n\n    def forward(self, input: Tensor, target: Tensor) -&gt; Tensor:\n\n        regression_loss = self.multi_target_regression_loss(\n            input[:, self.regression_config],\n            target[:, self.regression_config],\n        )\n\n        if self.multi_target_classification_loss.binary_trick:\n            classification_loss = self.multi_target_classification_loss(\n                input[:, len(self.regression_config) :], target\n            )\n        else:\n            classification_loss = self.multi_target_classification_loss(input, target)\n\n        return regression_loss + classification_loss\n\n    def _check_inputs_with_binary_trick(\n        self,\n        regression_config: List[int],\n        binary_config: Optional[List[Union[int, Tuple[int, float]]]],\n        multiclass_config: Optional[\n            List[Union[Tuple[int, int], Tuple[int, int, List[float]]]]\n        ],\n    ) -&gt; None:\n\n        error_msg = \"When using binary_trick=True, the targets order must be: regression, binary and multiclass\"\n\n        first_regression_idx = regression_config[0]\n        last_regression_idx = regression_config[-1]\n        if first_regression_idx != 0:\n            raise ValueError(error_msg)\n\n        if binary_config is not None and multiclass_config is not None:\n            first_binary_idx = (\n                binary_config[0][0]\n                if isinstance(binary_config[0], tuple)\n                else binary_config[0]\n            )\n            last_binary_idx = (\n                binary_config[-1][0]\n                if isinstance(binary_config[-1], tuple)\n                else binary_config[-1]\n            )\n            first_multiclass_idx = multiclass_config[0][0]\n\n            if (first_binary_idx != last_regression_idx + 1) or (\n                last_binary_idx &gt;= first_multiclass_idx\n            ):\n                raise ValueError(error_msg)\n        elif binary_config is not None:\n            first_binary_idx = (\n                binary_config[0][0]\n                if isinstance(binary_config[0], tuple)\n                else binary_config[0]\n            )\n            if first_binary_idx != last_regression_idx + 1:\n                raise ValueError(error_msg)\n        elif multiclass_config is not None:\n            first_multiclass_idx = multiclass_config[0][0]\n            if first_multiclass_idx != last_regression_idx + 1:\n                raise ValueError(error_msg)\n        else:\n            raise ValueError(\n                \"Either binary_config or multiclass_config must be provided. \"\n                \"Otherwise, use the MultiTargetRegressionLoss\"\n            )\n\n    def _prepare_weights_for_regression_targets(\n        self,\n        weights: List[float],\n        regression_config: List[int],\n    ) -&gt; List[float]:\n\n        weights_regression = [\n            w for idx, w in enumerate(weights) if idx in regression_config\n        ]\n\n        return weights_regression\n\n    def _prepare_weights_per_binary_targets(\n        self,\n        weights: List[float],\n        binary_config: Optional[List[Union[int, Tuple[int, float]]]],\n    ) -&gt; Optional[List[float]]:\n\n        if binary_config is not None:\n            binary_idx: List[int] = []\n            for bc in binary_config:\n                if isinstance(bc, tuple):\n                    binary_idx.append(bc[0])\n                else:\n                    binary_idx.append(bc)\n            weights_binary = [w for idx, w in enumerate(weights) if idx in binary_idx]\n        else:\n            weights_binary = None\n\n        return weights_binary\n\n    def _prepare_weights_per_multiclass_targets(\n        self,\n        weights: List[float],\n        multiclass_config: Optional[\n            List[Union[Tuple[int, int], Tuple[int, int, List[float]]]]\n        ],\n    ) -&gt; Optional[List[float]]:\n\n        if multiclass_config is not None:\n            multiclass_idx: List[int] = [mc[0] for mc in multiclass_config]\n            weights_multiclass = [\n                w for idx, w in enumerate(weights) if idx in multiclass_idx\n            ]\n        else:\n            weights_multiclass = None\n\n        return weights_multiclass\n</code></pre>"},{"location":"pytorch-widedeep/metrics.html","title":"Metrics","text":"<p> NOTE: metrics in this module expect the predictions  and ground truth to have the same dimensions for regression and binary  classification problems: \\((N_{samples}, 1)\\). In the case of multiclass  classification problems the ground truth is expected to be a 1D tensor with  the corresponding classes. See Examples below</p> <p>We have added the possibility of using the metrics available at the torchmetrics library. Note that this library is still in its early versions and therefore this option should be used with caution. To use <code>torchmetrics</code> simply import them and use them as any of the <code>pytorch-widedeep</code> metrics described below.</p> <pre><code>from torchmetrics import Accuracy, Precision\n\naccuracy = Accuracy(average=None, num_classes=2)\nprecision = Precision(average='micro', num_classes=2)\n\ntrainer = Trainer(model, objective=\"binary\", metrics=[accuracy, precision])\n</code></pre> <p>A functioning example for <code>pytorch-widedeep</code> using <code>torchmetrics</code> can be found in the Examples folder</p> <p> NOTE: the forward method for all metrics in this  module takes two tensors, <code>y_pred</code> and <code>y_true</code> (in that order). Therefore,  we do not include the method in the documentation.</p>"},{"location":"pytorch-widedeep/metrics.html#pytorch_widedeep.metrics.Accuracy","title":"Accuracy","text":"<p>               Bases: <code>Metric</code></p> <p>Class to calculate the accuracy for both binary and categorical problems</p> <p>Parameters:</p> Name Type Description Default <code>top_k</code> <code>int</code> <p>Accuracy will be computed using the top k most likely classes in multiclass problems</p> <code>1</code> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt;\n&gt;&gt;&gt; from pytorch_widedeep.metrics import Accuracy\n&gt;&gt;&gt;\n&gt;&gt;&gt; acc = Accuracy()\n&gt;&gt;&gt; y_true = torch.tensor([0, 1, 0, 1]).view(-1, 1)\n&gt;&gt;&gt; y_pred = torch.tensor([[0.3, 0.2, 0.6, 0.7]]).view(-1, 1)\n&gt;&gt;&gt; acc(y_pred, y_true)\narray(0.5)\n&gt;&gt;&gt;\n&gt;&gt;&gt; acc = Accuracy(top_k=2)\n&gt;&gt;&gt; y_true = torch.tensor([0, 1, 2])\n&gt;&gt;&gt; y_pred = torch.tensor([[0.3, 0.5, 0.2], [0.1, 0.1, 0.8], [0.1, 0.5, 0.4]])\n&gt;&gt;&gt; acc(y_pred, y_true)\narray(0.66666667)\n</code></pre> Source code in <code>pytorch_widedeep/metrics.py</code> <pre><code>class Accuracy(Metric):\n    r\"\"\"Class to calculate the accuracy for both binary and categorical problems\n\n    Parameters\n    ----------\n    top_k: int, default = 1\n        Accuracy will be computed using the top k most likely classes in\n        multiclass problems\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; from pytorch_widedeep.metrics import Accuracy\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; acc = Accuracy()\n    &gt;&gt;&gt; y_true = torch.tensor([0, 1, 0, 1]).view(-1, 1)\n    &gt;&gt;&gt; y_pred = torch.tensor([[0.3, 0.2, 0.6, 0.7]]).view(-1, 1)\n    &gt;&gt;&gt; acc(y_pred, y_true)\n    array(0.5)\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; acc = Accuracy(top_k=2)\n    &gt;&gt;&gt; y_true = torch.tensor([0, 1, 2])\n    &gt;&gt;&gt; y_pred = torch.tensor([[0.3, 0.5, 0.2], [0.1, 0.1, 0.8], [0.1, 0.5, 0.4]])\n    &gt;&gt;&gt; acc(y_pred, y_true)\n    array(0.66666667)\n    \"\"\"\n\n    def __init__(self, top_k: int = 1):\n        super(Accuracy, self).__init__()\n\n        self.top_k = top_k\n        self.correct_count = 0\n        self.total_count = 0\n        self._name = \"acc\"\n\n    def reset(self):\n        \"\"\"\n        resets counters to 0\n        \"\"\"\n        self.correct_count = 0\n        self.total_count = 0\n\n    def __call__(self, y_pred: Tensor, y_true: Tensor) -&gt; np.ndarray:\n        num_classes = y_pred.size(1)\n\n        if num_classes == 1:\n            y_pred = y_pred.round()\n            y_true = y_true\n        elif num_classes &gt; 1:\n            y_pred = y_pred.topk(self.top_k, 1)[1]\n            y_true = y_true.view(-1, 1).expand_as(y_pred)\n\n        self.correct_count += y_pred.eq(y_true).sum().item()  # type: ignore[assignment]\n        self.total_count += len(y_pred)\n        accuracy = float(self.correct_count) / float(self.total_count)\n        return np.array(accuracy)\n</code></pre>"},{"location":"pytorch-widedeep/metrics.html#pytorch_widedeep.metrics.Accuracy.reset","title":"reset","text":"<pre><code>reset()\n</code></pre> <p>resets counters to 0</p> Source code in <code>pytorch_widedeep/metrics.py</code> <pre><code>def reset(self):\n    \"\"\"\n    resets counters to 0\n    \"\"\"\n    self.correct_count = 0\n    self.total_count = 0\n</code></pre>"},{"location":"pytorch-widedeep/metrics.html#pytorch_widedeep.metrics.Precision","title":"Precision","text":"<p>               Bases: <code>Metric</code></p> <p>Class to calculate the precision for both binary and categorical problems</p> <p>Parameters:</p> Name Type Description Default <code>average</code> <code>bool</code> <p>This applies only to multiclass problems. if <code>True</code> calculate precision for each label, and finds their unweighted mean.</p> <code>True</code> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt;\n&gt;&gt;&gt; from pytorch_widedeep.metrics import Precision\n&gt;&gt;&gt;\n&gt;&gt;&gt; prec = Precision()\n&gt;&gt;&gt; y_true = torch.tensor([0, 1, 0, 1]).view(-1, 1)\n&gt;&gt;&gt; y_pred = torch.tensor([[0.3, 0.2, 0.6, 0.7]]).view(-1, 1)\n&gt;&gt;&gt; prec(y_pred, y_true)\narray(0.5)\n&gt;&gt;&gt;\n&gt;&gt;&gt; prec = Precision(average=True)\n&gt;&gt;&gt; y_true = torch.tensor([0, 1, 2])\n&gt;&gt;&gt; y_pred = torch.tensor([[0.7, 0.1, 0.2], [0.1, 0.1, 0.8], [0.1, 0.5, 0.4]])\n&gt;&gt;&gt; prec(y_pred, y_true)\narray(0.33333334)\n</code></pre> Source code in <code>pytorch_widedeep/metrics.py</code> <pre><code>class Precision(Metric):\n    r\"\"\"Class to calculate the precision for both binary and categorical problems\n\n    Parameters\n    ----------\n    average: bool, default = True\n        This applies only to multiclass problems. if ``True`` calculate\n        precision for each label, and finds their unweighted mean.\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; from pytorch_widedeep.metrics import Precision\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; prec = Precision()\n    &gt;&gt;&gt; y_true = torch.tensor([0, 1, 0, 1]).view(-1, 1)\n    &gt;&gt;&gt; y_pred = torch.tensor([[0.3, 0.2, 0.6, 0.7]]).view(-1, 1)\n    &gt;&gt;&gt; prec(y_pred, y_true)\n    array(0.5)\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; prec = Precision(average=True)\n    &gt;&gt;&gt; y_true = torch.tensor([0, 1, 2])\n    &gt;&gt;&gt; y_pred = torch.tensor([[0.7, 0.1, 0.2], [0.1, 0.1, 0.8], [0.1, 0.5, 0.4]])\n    &gt;&gt;&gt; prec(y_pred, y_true)\n    array(0.33333334)\n    \"\"\"\n\n    def __init__(self, average: bool = True):\n        super(Precision, self).__init__()\n\n        self.average = average\n        self.true_positives = 0\n        self.all_positives = 0\n        self.eps = 1e-20\n        self._name = \"prec\"\n\n    def reset(self):\n        \"\"\"\n        resets counters to 0\n        \"\"\"\n        self.true_positives = 0\n        self.all_positives = 0\n\n    def __call__(self, y_pred: Tensor, y_true: Tensor) -&gt; np.ndarray:\n        num_class = y_pred.size(1)\n\n        if num_class == 1:\n            y_pred = y_pred.round()\n            y_true = y_true\n        elif num_class &gt; 1:\n            y_true = torch.eye(num_class)[y_true.squeeze().cpu().long()]\n            y_pred = y_pred.topk(1, 1)[1].view(-1)\n            y_pred = torch.eye(num_class)[y_pred.cpu().long()]\n\n        self.true_positives += (y_true * y_pred).sum(dim=0)  # type:ignore\n        self.all_positives += y_pred.sum(dim=0)  # type:ignore\n\n        precision = self.true_positives / (self.all_positives + self.eps)\n\n        if self.average:\n            return np.array(precision.mean().item())  # type:ignore\n        else:\n            return precision.detach().cpu().numpy()  # type: ignore[attr-defined]\n</code></pre>"},{"location":"pytorch-widedeep/metrics.html#pytorch_widedeep.metrics.Precision.reset","title":"reset","text":"<pre><code>reset()\n</code></pre> <p>resets counters to 0</p> Source code in <code>pytorch_widedeep/metrics.py</code> <pre><code>def reset(self):\n    \"\"\"\n    resets counters to 0\n    \"\"\"\n    self.true_positives = 0\n    self.all_positives = 0\n</code></pre>"},{"location":"pytorch-widedeep/metrics.html#pytorch_widedeep.metrics.Recall","title":"Recall","text":"<p>               Bases: <code>Metric</code></p> <p>Class to calculate the recall for both binary and categorical problems</p> <p>Parameters:</p> Name Type Description Default <code>average</code> <code>bool</code> <p>This applies only to multiclass problems. if <code>True</code> calculate recall for each label, and finds their unweighted mean.</p> <code>True</code> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt;\n&gt;&gt;&gt; from pytorch_widedeep.metrics import Recall\n&gt;&gt;&gt;\n&gt;&gt;&gt; rec = Recall()\n&gt;&gt;&gt; y_true = torch.tensor([0, 1, 0, 1]).view(-1, 1)\n&gt;&gt;&gt; y_pred = torch.tensor([[0.3, 0.2, 0.6, 0.7]]).view(-1, 1)\n&gt;&gt;&gt; rec(y_pred, y_true)\narray(0.5)\n&gt;&gt;&gt;\n&gt;&gt;&gt; rec = Recall(average=True)\n&gt;&gt;&gt; y_true = torch.tensor([0, 1, 2])\n&gt;&gt;&gt; y_pred = torch.tensor([[0.7, 0.1, 0.2], [0.1, 0.1, 0.8], [0.1, 0.5, 0.4]])\n&gt;&gt;&gt; rec(y_pred, y_true)\narray(0.33333334)\n</code></pre> Source code in <code>pytorch_widedeep/metrics.py</code> <pre><code>class Recall(Metric):\n    r\"\"\"Class to calculate the recall for both binary and categorical problems\n\n    Parameters\n    ----------\n    average: bool, default = True\n        This applies only to multiclass problems. if ``True`` calculate recall\n        for each label, and finds their unweighted mean.\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; from pytorch_widedeep.metrics import Recall\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; rec = Recall()\n    &gt;&gt;&gt; y_true = torch.tensor([0, 1, 0, 1]).view(-1, 1)\n    &gt;&gt;&gt; y_pred = torch.tensor([[0.3, 0.2, 0.6, 0.7]]).view(-1, 1)\n    &gt;&gt;&gt; rec(y_pred, y_true)\n    array(0.5)\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; rec = Recall(average=True)\n    &gt;&gt;&gt; y_true = torch.tensor([0, 1, 2])\n    &gt;&gt;&gt; y_pred = torch.tensor([[0.7, 0.1, 0.2], [0.1, 0.1, 0.8], [0.1, 0.5, 0.4]])\n    &gt;&gt;&gt; rec(y_pred, y_true)\n    array(0.33333334)\n    \"\"\"\n\n    def __init__(self, average: bool = True):\n        super(Recall, self).__init__()\n\n        self.average = average\n        self.true_positives = 0\n        self.actual_positives = 0\n        self.eps = 1e-20\n        self._name = \"rec\"\n\n    def reset(self):\n        \"\"\"\n        resets counters to 0\n        \"\"\"\n        self.true_positives = 0\n        self.actual_positives = 0\n\n    def __call__(self, y_pred: Tensor, y_true: Tensor) -&gt; np.ndarray:\n        num_class = y_pred.size(1)\n\n        if num_class == 1:\n            y_pred = y_pred.round()\n            y_true = y_true\n        elif num_class &gt; 1:\n            y_true = torch.eye(num_class)[y_true.squeeze().cpu().long()]\n            y_pred = y_pred.topk(1, 1)[1].view(-1)\n            y_pred = torch.eye(num_class)[y_pred.cpu().long()]\n\n        self.true_positives += (y_true * y_pred).sum(dim=0)  # type: ignore\n        self.actual_positives += y_true.sum(dim=0)  # type: ignore\n\n        recall = self.true_positives / (self.actual_positives + self.eps)\n\n        if self.average:\n            return np.array(recall.mean().item())  # type:ignore\n        else:\n            return recall.detach().cpu().numpy()  # type: ignore[attr-defined]\n</code></pre>"},{"location":"pytorch-widedeep/metrics.html#pytorch_widedeep.metrics.Recall.reset","title":"reset","text":"<pre><code>reset()\n</code></pre> <p>resets counters to 0</p> Source code in <code>pytorch_widedeep/metrics.py</code> <pre><code>def reset(self):\n    \"\"\"\n    resets counters to 0\n    \"\"\"\n    self.true_positives = 0\n    self.actual_positives = 0\n</code></pre>"},{"location":"pytorch-widedeep/metrics.html#pytorch_widedeep.metrics.FBetaScore","title":"FBetaScore","text":"<p>               Bases: <code>Metric</code></p> <p>Class to calculate the fbeta score for both binary and categorical problems</p> \\[ F_{\\beta} = ((1 + {\\beta}^2) * \\frac{(precision * recall)}{({\\beta}^2 * precision + recall)} \\] <p>Parameters:</p> Name Type Description Default <code>beta</code> <code>int</code> <p>Coefficient to control the balance between precision and recall</p> required <code>average</code> <code>bool</code> <p>This applies only to multiclass problems. if <code>True</code> calculate fbeta for each label, and find their unweighted mean.</p> <code>True</code> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt;\n&gt;&gt;&gt; from pytorch_widedeep.metrics import FBetaScore\n&gt;&gt;&gt;\n&gt;&gt;&gt; fbeta = FBetaScore(beta=2)\n&gt;&gt;&gt; y_true = torch.tensor([0, 1, 0, 1]).view(-1, 1)\n&gt;&gt;&gt; y_pred = torch.tensor([[0.3, 0.2, 0.6, 0.7]]).view(-1, 1)\n&gt;&gt;&gt; fbeta(y_pred, y_true)\narray(0.5)\n&gt;&gt;&gt;\n&gt;&gt;&gt; fbeta = FBetaScore(beta=2)\n&gt;&gt;&gt; y_true = torch.tensor([0, 1, 2])\n&gt;&gt;&gt; y_pred = torch.tensor([[0.7, 0.1, 0.2], [0.1, 0.1, 0.8], [0.1, 0.5, 0.4]])\n&gt;&gt;&gt; fbeta(y_pred, y_true)\narray(0.33333334)\n</code></pre> Source code in <code>pytorch_widedeep/metrics.py</code> <pre><code>class FBetaScore(Metric):\n    r\"\"\"Class to calculate the fbeta score for both binary and categorical problems\n\n    $$\n    F_{\\beta} = ((1 + {\\beta}^2) * \\frac{(precision * recall)}{({\\beta}^2 * precision + recall)}\n    $$\n\n    Parameters\n    ----------\n    beta: int\n        Coefficient to control the balance between precision and recall\n    average: bool, default = True\n        This applies only to multiclass problems. if ``True`` calculate fbeta\n        for each label, and find their unweighted mean.\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; from pytorch_widedeep.metrics import FBetaScore\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; fbeta = FBetaScore(beta=2)\n    &gt;&gt;&gt; y_true = torch.tensor([0, 1, 0, 1]).view(-1, 1)\n    &gt;&gt;&gt; y_pred = torch.tensor([[0.3, 0.2, 0.6, 0.7]]).view(-1, 1)\n    &gt;&gt;&gt; fbeta(y_pred, y_true)\n    array(0.5)\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; fbeta = FBetaScore(beta=2)\n    &gt;&gt;&gt; y_true = torch.tensor([0, 1, 2])\n    &gt;&gt;&gt; y_pred = torch.tensor([[0.7, 0.1, 0.2], [0.1, 0.1, 0.8], [0.1, 0.5, 0.4]])\n    &gt;&gt;&gt; fbeta(y_pred, y_true)\n    array(0.33333334)\n    \"\"\"\n\n    def __init__(self, beta: int, average: bool = True):\n        super(FBetaScore, self).__init__()\n\n        self.beta = beta\n        self.average = average\n        self.precision = Precision(average=False)\n        self.recall = Recall(average=False)\n        self.eps = 1e-20\n        self._name = \"\".join([\"f\", str(self.beta)])\n\n    def reset(self):\n        \"\"\"\n        resets precision and recall\n        \"\"\"\n        self.precision.reset()\n        self.recall.reset()\n\n    def __call__(self, y_pred: Tensor, y_true: Tensor) -&gt; np.ndarray:\n        prec = self.precision(y_pred, y_true)\n        rec = self.recall(y_pred, y_true)\n        beta2 = self.beta**2\n\n        fbeta = ((1 + beta2) * prec * rec) / (beta2 * prec + rec + self.eps)\n\n        if self.average:\n            return np.array(fbeta.mean().item())  # type: ignore[attr-defined]\n        else:\n            return fbeta\n</code></pre>"},{"location":"pytorch-widedeep/metrics.html#pytorch_widedeep.metrics.FBetaScore.reset","title":"reset","text":"<pre><code>reset()\n</code></pre> <p>resets precision and recall</p> Source code in <code>pytorch_widedeep/metrics.py</code> <pre><code>def reset(self):\n    \"\"\"\n    resets precision and recall\n    \"\"\"\n    self.precision.reset()\n    self.recall.reset()\n</code></pre>"},{"location":"pytorch-widedeep/metrics.html#pytorch_widedeep.metrics.F1Score","title":"F1Score","text":"<p>               Bases: <code>Metric</code></p> <p>Class to calculate the f1 score for both binary and categorical problems</p> <p>Parameters:</p> Name Type Description Default <code>average</code> <code>bool</code> <p>This applies only to multiclass problems. if <code>True</code> calculate f1 for each label, and find their unweighted mean.</p> <code>True</code> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt;\n&gt;&gt;&gt; from pytorch_widedeep.metrics import F1Score\n&gt;&gt;&gt;\n&gt;&gt;&gt; f1 = F1Score()\n&gt;&gt;&gt; y_true = torch.tensor([0, 1, 0, 1]).view(-1, 1)\n&gt;&gt;&gt; y_pred = torch.tensor([[0.3, 0.2, 0.6, 0.7]]).view(-1, 1)\n&gt;&gt;&gt; f1(y_pred, y_true)\narray(0.5)\n&gt;&gt;&gt;\n&gt;&gt;&gt; f1 = F1Score()\n&gt;&gt;&gt; y_true = torch.tensor([0, 1, 2])\n&gt;&gt;&gt; y_pred = torch.tensor([[0.7, 0.1, 0.2], [0.1, 0.1, 0.8], [0.1, 0.5, 0.4]])\n&gt;&gt;&gt; f1(y_pred, y_true)\narray(0.33333334)\n</code></pre> Source code in <code>pytorch_widedeep/metrics.py</code> <pre><code>class F1Score(Metric):\n    r\"\"\"Class to calculate the f1 score for both binary and categorical problems\n\n    Parameters\n    ----------\n    average: bool, default = True\n        This applies only to multiclass problems. if ``True`` calculate f1 for\n        each label, and find their unweighted mean.\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; from pytorch_widedeep.metrics import F1Score\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; f1 = F1Score()\n    &gt;&gt;&gt; y_true = torch.tensor([0, 1, 0, 1]).view(-1, 1)\n    &gt;&gt;&gt; y_pred = torch.tensor([[0.3, 0.2, 0.6, 0.7]]).view(-1, 1)\n    &gt;&gt;&gt; f1(y_pred, y_true)\n    array(0.5)\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; f1 = F1Score()\n    &gt;&gt;&gt; y_true = torch.tensor([0, 1, 2])\n    &gt;&gt;&gt; y_pred = torch.tensor([[0.7, 0.1, 0.2], [0.1, 0.1, 0.8], [0.1, 0.5, 0.4]])\n    &gt;&gt;&gt; f1(y_pred, y_true)\n    array(0.33333334)\n    \"\"\"\n\n    def __init__(self, average: bool = True):\n        super(F1Score, self).__init__()\n\n        self.average = average\n        self.f1 = FBetaScore(beta=1, average=self.average)\n        self._name = self.f1._name\n\n    def reset(self):\n        \"\"\"\n        resets counters to 0\n        \"\"\"\n        self.f1.reset()\n\n    def __call__(self, y_pred: Tensor, y_true: Tensor) -&gt; np.ndarray:\n        return self.f1(y_pred, y_true)\n</code></pre>"},{"location":"pytorch-widedeep/metrics.html#pytorch_widedeep.metrics.F1Score.reset","title":"reset","text":"<pre><code>reset()\n</code></pre> <p>resets counters to 0</p> Source code in <code>pytorch_widedeep/metrics.py</code> <pre><code>def reset(self):\n    \"\"\"\n    resets counters to 0\n    \"\"\"\n    self.f1.reset()\n</code></pre>"},{"location":"pytorch-widedeep/metrics.html#pytorch_widedeep.metrics.R2Score","title":"R2Score","text":"<p>               Bases: <code>Metric</code></p> <p>Calculates R-Squared, the coefficient of determination:</p> \\[ R^2 = 1 - \\frac{\\sum_{j=1}^n(y_j - \\hat{y_j})^2}{\\sum_{j=1}^n(y_j - \\bar{y})^2} \\] <p>where \\(\\hat{y_j}\\) is the ground truth, \\(y_j\\) is the predicted value and \\(\\bar{y}\\) is the mean of the ground truth.</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt;\n&gt;&gt;&gt; from pytorch_widedeep.metrics import R2Score\n&gt;&gt;&gt;\n&gt;&gt;&gt; r2 = R2Score()\n&gt;&gt;&gt; y_true = torch.tensor([3, -0.5, 2, 7]).view(-1, 1)\n&gt;&gt;&gt; y_pred = torch.tensor([2.5, 0.0, 2, 8]).view(-1, 1)\n&gt;&gt;&gt; r2(y_pred, y_true)\narray(0.94860814)\n</code></pre> Source code in <code>pytorch_widedeep/metrics.py</code> <pre><code>class R2Score(Metric):\n    r\"\"\"\n    Calculates R-Squared, the\n    [coefficient of determination](https://en.wikipedia.org/wiki/Coefficient_of_determination&gt;):\n\n    $$\n    R^2 = 1 - \\frac{\\sum_{j=1}^n(y_j - \\hat{y_j})^2}{\\sum_{j=1}^n(y_j - \\bar{y})^2}\n    $$\n\n    where $\\hat{y_j}$ is the ground truth, $y_j$ is the predicted value and\n    $\\bar{y}$ is the mean of the ground truth.\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; from pytorch_widedeep.metrics import R2Score\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; r2 = R2Score()\n    &gt;&gt;&gt; y_true = torch.tensor([3, -0.5, 2, 7]).view(-1, 1)\n    &gt;&gt;&gt; y_pred = torch.tensor([2.5, 0.0, 2, 8]).view(-1, 1)\n    &gt;&gt;&gt; r2(y_pred, y_true)\n    array(0.94860814)\n    \"\"\"\n\n    def __init__(self):\n        self.numerator = 0\n        self.denominator = 0\n        self.num_examples = 0\n        self.y_true_sum = 0\n\n        self._name = \"r2\"\n\n    def reset(self):\n        \"\"\"\n        resets counters to 0\n        \"\"\"\n        self.numerator = 0\n        self.denominator = 0\n        self.num_examples = 0\n        self.y_true_sum = 0\n\n    def __call__(self, y_pred: Tensor, y_true: Tensor) -&gt; np.ndarray:\n        self.numerator += ((y_pred - y_true) ** 2).sum().item()\n\n        self.num_examples += y_true.shape[0]\n        self.y_true_sum += y_true.sum().item()\n        y_true_avg = self.y_true_sum / self.num_examples\n        self.denominator += ((y_true - y_true_avg) ** 2).sum().item()\n        return np.array((1 - (self.numerator / self.denominator)))\n</code></pre>"},{"location":"pytorch-widedeep/metrics.html#pytorch_widedeep.metrics.R2Score.reset","title":"reset","text":"<pre><code>reset()\n</code></pre> <p>resets counters to 0</p> Source code in <code>pytorch_widedeep/metrics.py</code> <pre><code>def reset(self):\n    \"\"\"\n    resets counters to 0\n    \"\"\"\n    self.numerator = 0\n    self.denominator = 0\n    self.num_examples = 0\n    self.y_true_sum = 0\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html","title":"The <code>models</code> module","text":"<p>This module contains the models that can be used as the four main components that will comprise a Wide and Deep model (<code>wide</code>, <code>deeptabular</code>, <code>deeptext</code>, <code>deepimage</code>), as well as the <code>WideDeep</code> \"constructor\" class. Note that each of the four components can be used independently. It also contains all the documentation for the models that can be used for self-supervised pre-training with tabular data.</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.linear.wide.Wide","title":"Wide","text":"<p>               Bases: <code>Module</code></p> <p>Defines a <code>Wide</code> (linear) model where the non-linearities are captured via the so-called crossed-columns. This can be used as the <code>wide</code> component of a Wide &amp; Deep model.</p> <p>Parameters:</p> Name Type Description Default <code>input_dim</code> <code>int</code> <p>size of the Linear layer (implemented via an Embedding layer). <code>input_dim</code> is the summation of all the individual values for all the features that go through the wide model. For example, if the wide model receives 2 features with 5 individual values each, <code>input_dim = 10</code></p> required <code>pred_dim</code> <code>int</code> <p>size of the ouput tensor containing the predictions. Note that unlike all the other models, the wide model is connected directly to the output neuron(s) when used to build a Wide and Deep model. Therefore, it requires the <code>pred_dim</code> parameter.</p> <code>1</code> <p>Attributes:</p> Name Type Description <code>wide_linear</code> <code>Module</code> <p>the linear layer that comprises the wide branch of the model</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import Wide\n&gt;&gt;&gt; X = torch.empty(4, 4).random_(20)\n&gt;&gt;&gt; wide = Wide(input_dim=int(X.max().item()), pred_dim=1)\n&gt;&gt;&gt; out = wide(X)\n</code></pre> Source code in <code>pytorch_widedeep/models/tabular/linear/wide.py</code> <pre><code>class Wide(nn.Module):\n    r\"\"\"Defines a `Wide` (linear) model where the non-linearities are\n    captured via the so-called crossed-columns. This can be used as the\n    `wide` component of a Wide &amp; Deep model.\n\n    Parameters\n    -----------\n    input_dim: int\n        size of the Linear layer (implemented via an Embedding layer).\n        `input_dim` is the summation of all the individual values for all the\n        features that go through the wide model. For example, if the wide\n        model receives 2 features with 5 individual values each, `input_dim =\n        10`\n    pred_dim: int, default = 1\n        size of the ouput tensor containing the predictions. Note that unlike\n        all the other models, the wide model is connected directly to the\n        output neuron(s) when used to build a Wide and Deep model. Therefore,\n        it requires the `pred_dim` parameter.\n\n    Attributes\n    -----------\n    wide_linear: nn.Module\n        the linear layer that comprises the wide branch of the model\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.models import Wide\n    &gt;&gt;&gt; X = torch.empty(4, 4).random_(20)\n    &gt;&gt;&gt; wide = Wide(input_dim=int(X.max().item()), pred_dim=1)\n    &gt;&gt;&gt; out = wide(X)\n    \"\"\"\n\n    @alias(\"pred_dim\", [\"pred_size\", \"num_class\"])\n    def __init__(self, input_dim: int, pred_dim: int = 1):\n        super(Wide, self).__init__()\n\n        self.input_dim = input_dim\n        self.pred_dim = pred_dim\n\n        # Embeddings: val + 1 because 0 is reserved for padding/unseen cateogories.\n        self.wide_linear = nn.Embedding(input_dim + 1, pred_dim, padding_idx=0)\n        # (Sum(Embedding) + bias) is equivalent to (OneHotVector + Linear)\n        self.bias = nn.Parameter(torch.zeros(pred_dim))\n        self._reset_parameters()\n\n    def _reset_parameters(self) -&gt; None:\n        r\"\"\"initialize Embedding and bias like nn.Linear. See [original\n        implementation](https://pytorch.org/docs/stable/_modules/torch/nn/modules/linear.html#Linear).\n        \"\"\"\n        nn.init.kaiming_uniform_(self.wide_linear.weight, a=math.sqrt(5))\n        fan_in, _ = nn.init._calculate_fan_in_and_fan_out(self.wide_linear.weight)\n        bound = 1 / math.sqrt(fan_in)\n        nn.init.uniform_(self.bias, -bound, bound)\n\n    def forward(self, X: Tensor) -&gt; Tensor:\n        r\"\"\"Forward pass. Simply connecting the Embedding layer with the ouput\n        neuron(s)\"\"\"\n        out = self.wide_linear(X.long()).sum(dim=1) + self.bias\n        return out\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.linear.wide.Wide.forward","title":"forward","text":"<pre><code>forward(X)\n</code></pre> <p>Forward pass. Simply connecting the Embedding layer with the ouput neuron(s)</p> Source code in <code>pytorch_widedeep/models/tabular/linear/wide.py</code> <pre><code>def forward(self, X: Tensor) -&gt; Tensor:\n    r\"\"\"Forward pass. Simply connecting the Embedding layer with the ouput\n    neuron(s)\"\"\"\n    out = self.wide_linear(X.long()).sum(dim=1) + self.bias\n    return out\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.mlp.tab_mlp.TabMlp","title":"TabMlp","text":"<p>               Bases: <code>BaseTabularModelWithoutAttention</code></p> <p>Defines a <code>TabMlp</code> model that can be used as the <code>deeptabular</code> component of a Wide &amp; Deep model or independently by itself.</p> <p>This class combines embedding representations of the categorical features with numerical (aka continuous) features, embedded or not. These are then passed through a series of dense layers (i.e. a MLP).</p> <p>Most of the parameters for this class are <code>Optional</code> since the use of categorical or continuous is in fact optional (i.e. one can use categorical features only, continuous features only or both).</p> <p>Parameters:</p> Name Type Description Default <code>column_idx</code> <code>Dict[str, int]</code> <p>Dict containing the index of the columns that will be passed through the <code>TabMlp</code> model. Required to slice the tensors. e.g. {'education': 0, 'relationship': 1, 'workclass': 2, ...}.</p> required <code>cat_embed_input</code> <code>Optional[List[Tuple[str, int, int]]]</code> <p>List of Tuples with the column name, number of unique values and embedding dimension. e.g. [(education, 11, 32), ...]</p> <code>None</code> <code>cat_embed_dropout</code> <code>Optional[float]</code> <p>Categorical embeddings dropout. If <code>None</code>, it will default to 0.</p> <code>None</code> <code>use_cat_bias</code> <code>Optional[bool]</code> <p>Boolean indicating if bias will be used for the categorical embeddings. If <code>None</code>, it will default to 'False'.</p> <code>None</code> <code>cat_embed_activation</code> <code>Optional[str]</code> <p>Activation function for the categorical embeddings, if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> <code>None</code> <code>continuous_cols</code> <code>Optional[List[str]]</code> <p>List with the name of the numeric (aka continuous) columns</p> <code>None</code> <code>cont_norm_layer</code> <code>Optional[Literal[batchnorm, layernorm]]</code> <p>Type of normalization layer applied to the continuous features. Options are: 'layernorm' and 'batchnorm'. if <code>None</code>, no normalization layer will be used.</p> <code>None</code> <code>embed_continuous</code> <code>Optional[bool]</code> <p>Boolean indicating if the continuous columns will be embedded using one of the available methods: 'standard', 'periodic' or 'piecewise'. If <code>None</code>, it will default to 'False'. NOTE: This parameter is deprecated and it  will be removed in future releases. Please, use the  <code>embed_continuous_method</code> parameter instead.</p> <code>None</code> <code>embed_continuous_method</code> <code>Optional[Literal[standard, piecewise, periodic]]</code> <p>Method to use to embed the continuous features. Options are: 'standard', 'periodic' or 'piecewise'. The 'standard' embedding method is based on the FT-Transformer implementation presented in the paper: Revisiting Deep Learning Models for Tabular Data. The 'periodic' and_'piecewise'_ methods were presented in the paper: On Embeddings for Numerical Features in Tabular Deep Learning. Please, read the papers for details.</p> <code>None</code> <code>cont_embed_dim</code> <code>Optional[int]</code> <p>Size of the continuous embeddings. If the continuous columns are embedded, <code>cont_embed_dim</code> must be passed.</p> <code>None</code> <code>cont_embed_dropout</code> <code>Optional[float]</code> <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p> <code>None</code> <code>cont_embed_activation</code> <code>Optional[str]</code> <p>Activation function for the continuous embeddings if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported. If <code>None</code>, no activation function will be applied.</p> <code>None</code> <code>quantization_setup</code> <code>Optional[Dict[str, List[float]]]</code> <p>This parameter is used when the 'piecewise' method is used to embed the continuous cols. It is a dict where keys are the name of the continuous columns and values are lists with the boundaries for the quantization of the continuous_cols. See the examples for details. If If the 'piecewise' method is used, this parameter is required.</p> <code>None</code> <code>n_frequencies</code> <code>Optional[int]</code> <p>This is the so called 'k' in their paper On Embeddings for Numerical Features in Tabular Deep Learning, and is the number of 'frequencies' that will be used to represent each continuous column. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>sigma</code> <code>Optional[float]</code> <p>This is the sigma parameter in the paper mentioned when describing the previous parameters and it is used to initialise the 'frequency weights'. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>share_last_layer</code> <code>Optional[bool]</code> <p>This parameter is not present in the before mentioned paper but it is implemented in the official repo. If <code>True</code> the linear layer that turns the frequencies into embeddings will be shared across the continuous columns. If <code>False</code> a different linear layer will be used for each continuous column. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>full_embed_dropout</code> <code>Optional[bool]</code> <p>If <code>True</code>, the full embedding corresponding to a column will be masked out/dropout. If <code>None</code>, it will default to <code>False</code>.</p> <code>None</code> <code>mlp_hidden_dims</code> <code>List[int]</code> <p>List with the number of neurons per dense layer in the mlp.</p> <code>[200, 100]</code> <code>mlp_activation</code> <code>str</code> <p>Activation function for the dense layers of the MLP. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> <code>'relu'</code> <code>mlp_dropout</code> <code>Union[float, List[float]]</code> <p>float or List of floats with the dropout between the dense layers. e.g: [0.5,0.5]</p> <code>0.1</code> <code>mlp_batchnorm</code> <code>bool</code> <p>Boolean indicating whether or not batch normalization will be applied to the dense layers</p> <code>False</code> <code>mlp_batchnorm_last</code> <code>bool</code> <p>Boolean indicating whether or not batch normalization will be applied to the last of the dense layers</p> <code>False</code> <code>mlp_linear_first</code> <code>bool</code> <p>Boolean indicating the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code></p> <code>True</code> <p>Attributes:</p> Name Type Description <code>encoder</code> <code>Module</code> <p>mlp model that will receive the concatenation of the embeddings and the continuous columns</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import TabMlp\n&gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n&gt;&gt;&gt; colnames = [\"a\", \"b\", \"c\", \"d\", \"e\"]\n&gt;&gt;&gt; cat_embed_input = [(u, i, j) for u, i, j in zip(colnames[:4], [4] * 4, [8] * 4)]\n&gt;&gt;&gt; column_idx = {k: v for v, k in enumerate(colnames)}\n&gt;&gt;&gt; model = TabMlp(mlp_hidden_dims=[8, 4], column_idx=column_idx, cat_embed_input=cat_embed_input,\n... continuous_cols=[\"e\"])\n&gt;&gt;&gt; out = model(X_tab)\n</code></pre> Source code in <code>pytorch_widedeep/models/tabular/mlp/tab_mlp.py</code> <pre><code>class TabMlp(BaseTabularModelWithoutAttention):\n    r\"\"\"Defines a `TabMlp` model that can be used as the `deeptabular`\n    component of a Wide &amp; Deep model or independently by itself.\n\n    This class combines embedding representations of the categorical features\n    with numerical (aka continuous) features, embedded or not. These are then\n    passed through a series of dense layers (i.e. a MLP).\n\n    Most of the parameters for this class are `Optional` since the use of\n    categorical or continuous is in fact optional (i.e. one can use\n    categorical features only, continuous features only or both).\n\n    Parameters\n    ----------\n    column_idx: Dict\n        Dict containing the index of the columns that will be passed through\n        the `TabMlp` model. Required to slice the tensors. e.g. _{'education':\n        0, 'relationship': 1, 'workclass': 2, ...}_.\n    cat_embed_input: List, Optional, default = None\n        List of Tuples with the column name, number of unique values and\n        embedding dimension. e.g. _[(education, 11, 32), ...]_\n    cat_embed_dropout: float, Optional, default = None\n        Categorical embeddings dropout. If `None`, it will default\n        to 0.\n    use_cat_bias: bool, Optional, default = None,\n        Boolean indicating if bias will be used for the categorical embeddings.\n        If `None`, it will default to 'False'.\n    cat_embed_activation: Optional, str, default = None,\n        Activation function for the categorical embeddings, if any. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported\n    continuous_cols: List, Optional, default = None\n        List with the name of the numeric (aka continuous) columns\n    cont_norm_layer: str, Optional, default =  None\n        Type of normalization layer applied to the continuous features.\n        Options are: _'layernorm'_ and _'batchnorm'_. if `None`, no\n        normalization layer will be used.\n    embed_continuous: bool, Optional, default = None,\n        Boolean indicating if the continuous columns will be embedded using\n        one of the available methods: _'standard'_, _'periodic'_\n        or _'piecewise'_. If `None`, it will default to 'False'.&lt;br/&gt;\n        :information_source: **NOTE**: This parameter is deprecated and it\n         will be removed in future releases. Please, use the\n         `embed_continuous_method` parameter instead.\n    embed_continuous_method: Optional, str, default = None,\n        Method to use to embed the continuous features. Options are:\n        _'standard'_, _'periodic'_ or _'piecewise'_. The _'standard'_\n        embedding method is based on the FT-Transformer implementation\n        presented in the paper: [Revisiting Deep Learning Models for\n        Tabular Data](https://arxiv.org/abs/2106.11959v5). The _'periodic'_\n        and_'piecewise'_ methods were presented in the paper: [On Embeddings for\n        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556).\n        Please, read the papers for details.\n    cont_embed_dim: int, Optional, default = None,\n        Size of the continuous embeddings. If the continuous columns are\n        embedded, `cont_embed_dim` must be passed.\n    cont_embed_dropout: float, Optional, default = None,\n        Dropout for the continuous embeddings. If `None`, it will default to 0.0\n    cont_embed_activation: Optional, str, default = None,\n        Activation function for the continuous embeddings if any. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported.\n        If `None`, no activation function will be applied.\n    quantization_setup: Dict[str, List[float]], Optional, default = None,\n        This parameter is used when the _'piecewise'_ method is used to embed\n        the continuous cols. It is a dict where keys are the name of the continuous\n        columns and values are lists with the boundaries for the quantization\n        of the continuous_cols. See the examples for details. If\n        If the _'piecewise'_ method is used, this parameter is required.\n    n_frequencies: int, Optional, default = None,\n        This is the so called _'k'_ in their paper [On Embeddings for\n        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556),\n        and is the number of 'frequencies' that will be used to represent each\n        continuous column. See their Eq 2 in the paper for details. If\n        the _'periodic'_ method is used, this parameter is required.\n    sigma: float, Optional, default = None,\n        This is the sigma parameter in the paper mentioned when describing the\n        previous parameters and it is used to initialise the 'frequency\n        weights'. See their Eq 2 in the paper for details. If\n        the _'periodic'_ method is used, this parameter is required.\n    share_last_layer: bool, Optional, default = None,\n        This parameter is not present in the before mentioned paper but it is implemented in\n        the [official repo](https://github.com/yandex-research/rtdl-num-embeddings/tree/main).\n        If `True` the linear layer that turns the frequencies into embeddings\n        will be shared across the continuous columns. If `False` a different\n        linear layer will be used for each continuous column.\n        If the _'periodic'_ method is used, this parameter is required.\n    full_embed_dropout: bool, Optional, default = None,\n        If `True`, the full embedding corresponding to a column will be masked\n        out/dropout. If `None`, it will default to `False`.\n    mlp_hidden_dims: List, default = [200, 100]\n        List with the number of neurons per dense layer in the mlp.\n    mlp_activation: str, default = \"relu\"\n        Activation function for the dense layers of the MLP. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported\n    mlp_dropout: float or List, default = 0.1\n        float or List of floats with the dropout between the dense layers.\n        e.g: _[0.5,0.5]_\n    mlp_batchnorm: bool, default = False\n        Boolean indicating whether or not batch normalization will be applied\n        to the dense layers\n    mlp_batchnorm_last: bool, default = False\n        Boolean indicating whether or not batch normalization will be applied\n        to the last of the dense layers\n    mlp_linear_first: bool, default = False\n        Boolean indicating the order of the operations in the dense\n        layer. If `True: [LIN -&gt; ACT -&gt; BN -&gt; DP]`. If `False: [BN -&gt; DP -&gt;\n        LIN -&gt; ACT]`\n\n    Attributes\n    ----------\n    encoder: nn.Module\n        mlp model that will receive the concatenation of the embeddings and\n        the continuous columns\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.models import TabMlp\n    &gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n    &gt;&gt;&gt; colnames = [\"a\", \"b\", \"c\", \"d\", \"e\"]\n    &gt;&gt;&gt; cat_embed_input = [(u, i, j) for u, i, j in zip(colnames[:4], [4] * 4, [8] * 4)]\n    &gt;&gt;&gt; column_idx = {k: v for v, k in enumerate(colnames)}\n    &gt;&gt;&gt; model = TabMlp(mlp_hidden_dims=[8, 4], column_idx=column_idx, cat_embed_input=cat_embed_input,\n    ... continuous_cols=[\"e\"])\n    &gt;&gt;&gt; out = model(X_tab)\n    \"\"\"\n\n    def __init__(\n        self,\n        column_idx: Dict[str, int],\n        *,\n        cat_embed_input: Optional[List[Tuple[str, int, int]]] = None,\n        cat_embed_dropout: Optional[float] = None,\n        use_cat_bias: Optional[bool] = None,\n        cat_embed_activation: Optional[str] = None,\n        continuous_cols: Optional[List[str]] = None,\n        cont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\n        embed_continuous: Optional[bool] = None,\n        embed_continuous_method: Optional[\n            Literal[\"standard\", \"piecewise\", \"periodic\"]\n        ] = None,\n        cont_embed_dim: Optional[int] = None,\n        cont_embed_dropout: Optional[float] = None,\n        cont_embed_activation: Optional[str] = None,\n        quantization_setup: Optional[Dict[str, List[float]]] = None,\n        n_frequencies: Optional[int] = None,\n        sigma: Optional[float] = None,\n        share_last_layer: Optional[bool] = None,\n        full_embed_dropout: Optional[bool] = None,\n        mlp_hidden_dims: List[int] = [200, 100],\n        mlp_activation: str = \"relu\",\n        mlp_dropout: Union[float, List[float]] = 0.1,\n        mlp_batchnorm: bool = False,\n        mlp_batchnorm_last: bool = False,\n        mlp_linear_first: bool = True,\n    ):\n        super(TabMlp, self).__init__(\n            column_idx=column_idx,\n            cat_embed_input=cat_embed_input,\n            cat_embed_dropout=cat_embed_dropout,\n            use_cat_bias=use_cat_bias,\n            cat_embed_activation=cat_embed_activation,\n            continuous_cols=continuous_cols,\n            cont_norm_layer=cont_norm_layer,\n            embed_continuous=embed_continuous,\n            embed_continuous_method=embed_continuous_method,\n            cont_embed_dim=cont_embed_dim,\n            cont_embed_dropout=cont_embed_dropout,\n            cont_embed_activation=cont_embed_activation,\n            quantization_setup=quantization_setup,\n            n_frequencies=n_frequencies,\n            sigma=sigma,\n            share_last_layer=share_last_layer,\n            full_embed_dropout=full_embed_dropout,\n        )\n\n        self.mlp_hidden_dims = mlp_hidden_dims\n        self.mlp_activation = mlp_activation\n        self.mlp_dropout = mlp_dropout\n        self.mlp_batchnorm = mlp_batchnorm\n        self.mlp_batchnorm_last = mlp_batchnorm_last\n        self.mlp_linear_first = mlp_linear_first\n\n        # Embeddings are instantiated at the base model\n        # Mlp\n        mlp_input_dim = self.cat_out_dim + self.cont_out_dim\n        mlp_hidden_dims = [mlp_input_dim] + mlp_hidden_dims\n        self.encoder = MLP(\n            mlp_hidden_dims,\n            mlp_activation,\n            mlp_dropout,\n            mlp_batchnorm,\n            mlp_batchnorm_last,\n            mlp_linear_first,\n        )\n\n    def forward(self, X: Tensor) -&gt; Tensor:\n        x = self._get_embeddings(X)\n        return self.encoder(x)\n\n    @property\n    def output_dim(self) -&gt; int:\n        r\"\"\"The output dimension of the model. This is a required property\n        neccesary to build the `WideDeep` class\"\"\"\n        return self.mlp_hidden_dims[-1]\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.mlp.tab_mlp.TabMlp.output_dim","title":"output_dim  <code>property</code>","text":"<pre><code>output_dim\n</code></pre> <p>The output dimension of the model. This is a required property neccesary to build the <code>WideDeep</code> class</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.mlp.tab_mlp.TabMlpDecoder","title":"TabMlpDecoder","text":"<p>               Bases: <code>Module</code></p> <p>Companion decoder model for the <code>TabMlp</code> model (which can be considered an encoder itself).</p> <p>This class is designed to be used with the <code>EncoderDecoderTrainer</code> when using self-supervised pre-training (see the corresponding section in the docs). The <code>TabMlpDecoder</code> will receive the output from the MLP and 'reconstruct' the embeddings.</p> <p>Parameters:</p> Name Type Description Default <code>embed_dim</code> <code>int</code> <p>Size of the embeddings tensor that needs to be reconstructed.</p> required <code>mlp_hidden_dims</code> <code>List[int]</code> <p>List with the number of neurons per dense layer in the mlp.</p> <code>[100, 200]</code> <code>mlp_activation</code> <code>str</code> <p>Activation function for the dense layers of the MLP. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> <code>'relu'</code> <code>mlp_dropout</code> <code>Union[float, List[float]]</code> <p>float or List of floats with the dropout between the dense layers. e.g: [0.5,0.5]</p> <code>0.1</code> <code>mlp_batchnorm</code> <code>bool</code> <p>Boolean indicating whether or not batch normalization will be applied to the dense layers</p> <code>False</code> <code>mlp_batchnorm_last</code> <code>bool</code> <p>Boolean indicating whether or not batch normalization will be applied to the last of the dense layers</p> <code>False</code> <code>mlp_linear_first</code> <code>bool</code> <p>Boolean indicating the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code></p> <code>True</code> <p>Attributes:</p> Name Type Description <code>decoder</code> <code>Module</code> <p>mlp model that will receive the output of the encoder</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import TabMlpDecoder\n&gt;&gt;&gt; x_inp = torch.rand(3, 8)\n&gt;&gt;&gt; decoder = TabMlpDecoder(embed_dim=32, mlp_hidden_dims=[8,16])\n&gt;&gt;&gt; res = decoder(x_inp)\n&gt;&gt;&gt; res.shape\ntorch.Size([3, 32])\n</code></pre> Source code in <code>pytorch_widedeep/models/tabular/mlp/tab_mlp.py</code> <pre><code>class TabMlpDecoder(nn.Module):\n    r\"\"\"Companion decoder model for the `TabMlp` model (which can be considered\n    an encoder itself).\n\n    This class is designed to be used with the `EncoderDecoderTrainer` when\n    using self-supervised pre-training (see the corresponding section in the\n    docs). The `TabMlpDecoder` will receive the output from the MLP\n    and '_reconstruct_' the embeddings.\n\n    Parameters\n    ----------\n    embed_dim: int\n        Size of the embeddings tensor that needs to be reconstructed.\n    mlp_hidden_dims: List, default = [200, 100]\n        List with the number of neurons per dense layer in the mlp.\n    mlp_activation: str, default = \"relu\"\n        Activation function for the dense layers of the MLP. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported\n    mlp_dropout: float or List, default = 0.1\n        float or List of floats with the dropout between the dense layers.\n        e.g: _[0.5,0.5]_\n    mlp_batchnorm: bool, default = False\n        Boolean indicating whether or not batch normalization will be applied\n        to the dense layers\n    mlp_batchnorm_last: bool, default = False\n        Boolean indicating whether or not batch normalization will be applied\n        to the last of the dense layers\n    mlp_linear_first: bool, default = False\n        Boolean indicating the order of the operations in the dense\n        layer. If `True: [LIN -&gt; ACT -&gt; BN -&gt; DP]`. If `False: [BN -&gt; DP -&gt;\n        LIN -&gt; ACT]`\n\n    Attributes\n    ----------\n    decoder: nn.Module\n        mlp model that will receive the output of the encoder\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.models import TabMlpDecoder\n    &gt;&gt;&gt; x_inp = torch.rand(3, 8)\n    &gt;&gt;&gt; decoder = TabMlpDecoder(embed_dim=32, mlp_hidden_dims=[8,16])\n    &gt;&gt;&gt; res = decoder(x_inp)\n    &gt;&gt;&gt; res.shape\n    torch.Size([3, 32])\n    \"\"\"\n\n    def __init__(\n        self,\n        embed_dim: int,\n        mlp_hidden_dims: List[int] = [100, 200],\n        mlp_activation: str = \"relu\",\n        mlp_dropout: Union[float, List[float]] = 0.1,\n        mlp_batchnorm: bool = False,\n        mlp_batchnorm_last: bool = False,\n        mlp_linear_first: bool = True,\n    ):\n        super(TabMlpDecoder, self).__init__()\n\n        self.embed_dim = embed_dim\n\n        self.mlp_hidden_dims = mlp_hidden_dims\n        self.mlp_activation = mlp_activation\n        self.mlp_dropout = mlp_dropout\n        self.mlp_batchnorm = mlp_batchnorm\n        self.mlp_batchnorm_last = mlp_batchnorm_last\n        self.mlp_linear_first = mlp_linear_first\n\n        self.decoder = MLP(\n            mlp_hidden_dims + [self.embed_dim],\n            mlp_activation,\n            mlp_dropout,\n            mlp_batchnorm,\n            mlp_batchnorm_last,\n            mlp_linear_first,\n        )\n\n    def forward(self, X: Tensor) -&gt; Tensor:\n        return self.decoder(X)\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.resnet.tab_resnet.TabResnet","title":"TabResnet","text":"<p>               Bases: <code>BaseTabularModelWithoutAttention</code></p> <p>Defines a <code>TabResnet</code> model that can be used as the <code>deeptabular</code> component of a Wide &amp; Deep model or independently by itself.</p> <p>This class combines embedding representations of the categorical features with numerical (aka continuous) features, embedded or not. These are then passed through a series of Resnet blocks. See <code>pytorch_widedeep.models.tab_resnet._layers</code> for details on the structure of each block.</p> <p>Most of the parameters for this class are <code>Optional</code> since the use of categorical or continuous is in fact optional (i.e. one can use categorical features only, continuous features only or both).</p> <p>Parameters:</p> Name Type Description Default <code>column_idx</code> <code>Dict[str, int]</code> <p>Dict containing the index of the columns that will be passed through the <code>TabMlp</code> model. Required to slice the tensors. e.g. {'education': 0, 'relationship': 1, 'workclass': 2, ...}.</p> required <code>cat_embed_input</code> <code>Optional[List[Tuple[str, int, int]]]</code> <p>List of Tuples with the column name, number of unique values and embedding dimension. e.g. [(education, 11, 32), ...]</p> <code>None</code> <code>cat_embed_dropout</code> <code>Optional[float]</code> <p>Categorical embeddings dropout. If <code>None</code>, it will default to 0.</p> <code>None</code> <code>use_cat_bias</code> <code>Optional[bool]</code> <p>Boolean indicating if bias will be used for the categorical embeddings. If <code>None</code>, it will default to 'False'.</p> <code>None</code> <code>cat_embed_activation</code> <code>Optional[str]</code> <p>Activation function for the categorical embeddings, if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> <code>None</code> <code>continuous_cols</code> <code>Optional[List[str]]</code> <p>List with the name of the numeric (aka continuous) columns</p> <code>None</code> <code>cont_norm_layer</code> <code>Optional[Literal[batchnorm, layernorm]]</code> <p>Type of normalization layer applied to the continuous features. Options are: 'layernorm' and 'batchnorm'. if <code>None</code>, no normalization layer will be used.</p> <code>None</code> <code>embed_continuous</code> <code>Optional[bool]</code> <p>Boolean indicating if the continuous columns will be embedded using one of the available methods: 'standard', 'periodic' or 'piecewise'. If <code>None</code>, it will default to 'False'. NOTE: This parameter is deprecated and it  will be removed in future releases. Please, use the  <code>embed_continuous_method</code> parameter instead.</p> <code>None</code> <code>embed_continuous_method</code> <code>Optional[Literal[standard, piecewise, periodic]]</code> <p>Method to use to embed the continuous features. Options are: 'standard', 'periodic' or 'piecewise'. The 'standard' embedding method is based on the FT-Transformer implementation presented in the paper: Revisiting Deep Learning Models for Tabular Data. The 'periodic' and_'piecewise'_ methods were presented in the paper: On Embeddings for Numerical Features in Tabular Deep Learning. Please, read the papers for details.</p> <code>None</code> <code>cont_embed_dim</code> <code>Optional[int]</code> <p>Size of the continuous embeddings. If the continuous columns are embedded, <code>cont_embed_dim</code> must be passed.</p> <code>None</code> <code>cont_embed_dropout</code> <code>Optional[float]</code> <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p> <code>None</code> <code>cont_embed_activation</code> <code>Optional[str]</code> <p>Activation function for the continuous embeddings if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported. If <code>None</code>, no activation function will be applied.</p> <code>None</code> <code>quantization_setup</code> <code>Optional[Dict[str, List[float]]]</code> <p>This parameter is used when the 'piecewise' method is used to embed the continuous cols. It is a dict where keys are the name of the continuous columns and values are lists with the boundaries for the quantization of the continuous_cols. See the examples for details. If If the 'piecewise' method is used, this parameter is required.</p> <code>None</code> <code>n_frequencies</code> <code>Optional[int]</code> <p>This is the so called 'k' in their paper On Embeddings for Numerical Features in Tabular Deep Learning, and is the number of 'frequencies' that will be used to represent each continuous column. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>sigma</code> <code>Optional[float]</code> <p>This is the sigma parameter in the paper mentioned when describing the previous parameters and it is used to initialise the 'frequency weights'. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>share_last_layer</code> <code>Optional[bool]</code> <p>This parameter is not present in the before mentioned paper but it is implemented in the official repo. If <code>True</code> the linear layer that turns the frequencies into embeddings will be shared across the continuous columns. If <code>False</code> a different linear layer will be used for each continuous column. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>full_embed_dropout</code> <code>Optional[bool]</code> <p>If <code>True</code>, the full embedding corresponding to a column will be masked out/dropout. If <code>None</code>, it will default to <code>False</code>.</p> <code>None</code> <code>blocks_dims</code> <code>List[int]</code> <p>List of integers that define the input and output units of each block. For example: [200, 100, 100] will generate 2 blocks. The first will receive a tensor of size 200 and output a tensor of size 100, and the second will receive a tensor of size 100 and output a tensor of size 100. See <code>pytorch_widedeep.models.tab_resnet._layers</code> for details on the structure of each block.</p> <code>[200, 100, 100]</code> <code>blocks_dropout</code> <code>float</code> <p>Block's internal dropout.</p> <code>0.1</code> <code>simplify_blocks</code> <code>bool</code> <p>Boolean indicating if the simplest possible residual blocks (<code>X -&gt; [ [LIN, BN, ACT]  + X ]</code>) will be used instead of a standard one (<code>X -&gt; [ [LIN1, BN1, ACT1] -&gt; [LIN2, BN2]  + X ]</code>).</p> <code>False</code> <code>mlp_hidden_dims</code> <code>Optional[List[int]]</code> <p>List with the number of neurons per dense layer in the MLP. e.g: [64, 32]. If <code>None</code> the  output of the Resnet Blocks will be connected directly to the output neuron(s).</p> <code>None</code> <code>mlp_activation</code> <code>Optional[str]</code> <p>Activation function for the dense layers of the MLP. Currently 'tanh', 'relu', 'leaky'_relu' and _'gelu' are supported. If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to 'relu'.</p> <code>None</code> <code>mlp_dropout</code> <code>Optional[float]</code> <p>float with the dropout between the dense layers of the MLP. If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to 0.0.</p> <code>None</code> <code>mlp_batchnorm</code> <code>Optional[bool]</code> <p>Boolean indicating whether or not batch normalization will be applied to the dense layers If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to False.</p> <code>None</code> <code>mlp_batchnorm_last</code> <code>Optional[bool]</code> <p>Boolean indicating whether or not batch normalization will be applied to the last of the dense layers If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to False.</p> <code>None</code> <code>mlp_linear_first</code> <code>Optional[bool]</code> <p>Boolean indicating the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code> If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to <code>True</code>.</p> <code>None</code> <p>Attributes:</p> Name Type Description <code>encoder</code> <code>Module</code> <p>deep dense Resnet model that will receive the concatenation of the embeddings and the continuous columns</p> <code>mlp</code> <code>Module</code> <p>if <code>mlp_hidden_dims</code> is <code>True</code>, this attribute will be an mlp model that will receive the results of the concatenation of the embeddings and the continuous columns -- if present --.</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import TabResnet\n&gt;&gt;&gt; X_deep = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n&gt;&gt;&gt; colnames = ['a', 'b', 'c', 'd', 'e']\n&gt;&gt;&gt; cat_embed_input = [(u,i,j) for u,i,j in zip(colnames[:4], [4]*4, [8]*4)]\n&gt;&gt;&gt; column_idx = {k:v for v,k in enumerate(colnames)}\n&gt;&gt;&gt; model = TabResnet(blocks_dims=[16,4], column_idx=column_idx, cat_embed_input=cat_embed_input,\n... continuous_cols = ['e'])\n&gt;&gt;&gt; out = model(X_deep)\n</code></pre> Source code in <code>pytorch_widedeep/models/tabular/resnet/tab_resnet.py</code> <pre><code>class TabResnet(BaseTabularModelWithoutAttention):\n    r\"\"\"Defines a `TabResnet` model that can be used as the `deeptabular`\n    component of a Wide &amp; Deep model or independently by itself.\n\n    This class combines embedding representations of the categorical features\n    with numerical (aka continuous) features, embedded or not. These are then\n    passed through a series of Resnet blocks. See\n    `pytorch_widedeep.models.tab_resnet._layers` for details on the\n    structure of each block.\n\n    Most of the parameters for this class are `Optional` since the use of\n    categorical or continuous is in fact optional (i.e. one can use\n    categorical features only, continuous features only or both).\n\n    Parameters\n    ----------\n    column_idx: Dict\n        Dict containing the index of the columns that will be passed through\n        the `TabMlp` model. Required to slice the tensors. e.g. _{'education':\n        0, 'relationship': 1, 'workclass': 2, ...}_.\n    cat_embed_input: List, Optional, default = None\n        List of Tuples with the column name, number of unique values and\n        embedding dimension. e.g. _[(education, 11, 32), ...]_\n    cat_embed_dropout: float, Optional, default = None\n        Categorical embeddings dropout. If `None`, it will default\n        to 0.\n    use_cat_bias: bool, Optional, default = None,\n        Boolean indicating if bias will be used for the categorical embeddings.\n        If `None`, it will default to 'False'.\n    cat_embed_activation: Optional, str, default = None,\n        Activation function for the categorical embeddings, if any. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported\n    continuous_cols: List, Optional, default = None\n        List with the name of the numeric (aka continuous) columns\n    cont_norm_layer: str, Optional, default =  None\n        Type of normalization layer applied to the continuous features.\n        Options are: _'layernorm'_ and _'batchnorm'_. if `None`, no\n        normalization layer will be used.\n    embed_continuous: bool, Optional, default = None,\n        Boolean indicating if the continuous columns will be embedded using\n        one of the available methods: _'standard'_, _'periodic'_\n        or _'piecewise'_. If `None`, it will default to 'False'.&lt;br/&gt;\n        :information_source: **NOTE**: This parameter is deprecated and it\n         will be removed in future releases. Please, use the\n         `embed_continuous_method` parameter instead.\n    embed_continuous_method: Optional, str, default = None,\n        Method to use to embed the continuous features. Options are:\n        _'standard'_, _'periodic'_ or _'piecewise'_. The _'standard'_\n        embedding method is based on the FT-Transformer implementation\n        presented in the paper: [Revisiting Deep Learning Models for\n        Tabular Data](https://arxiv.org/abs/2106.11959v5). The _'periodic'_\n        and_'piecewise'_ methods were presented in the paper: [On Embeddings for\n        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556).\n        Please, read the papers for details.\n    cont_embed_dim: int, Optional, default = None,\n        Size of the continuous embeddings. If the continuous columns are\n        embedded, `cont_embed_dim` must be passed.\n    cont_embed_dropout: float, Optional, default = None,\n        Dropout for the continuous embeddings. If `None`, it will default to 0.0\n    cont_embed_activation: Optional, str, default = None,\n        Activation function for the continuous embeddings if any. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported.\n        If `None`, no activation function will be applied.\n    quantization_setup: Dict[str, List[float]], Optional, default = None,\n        This parameter is used when the _'piecewise'_ method is used to embed\n        the continuous cols. It is a dict where keys are the name of the continuous\n        columns and values are lists with the boundaries for the quantization\n        of the continuous_cols. See the examples for details. If\n        If the _'piecewise'_ method is used, this parameter is required.\n    n_frequencies: int, Optional, default = None,\n        This is the so called _'k'_ in their paper [On Embeddings for\n        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556),\n        and is the number of 'frequencies' that will be used to represent each\n        continuous column. See their Eq 2 in the paper for details. If\n        the _'periodic'_ method is used, this parameter is required.\n    sigma: float, Optional, default = None,\n        This is the sigma parameter in the paper mentioned when describing the\n        previous parameters and it is used to initialise the 'frequency\n        weights'. See their Eq 2 in the paper for details. If\n        the _'periodic'_ method is used, this parameter is required.\n    share_last_layer: bool, Optional, default = None,\n        This parameter is not present in the before mentioned paper but it is implemented in\n        the [official repo](https://github.com/yandex-research/rtdl-num-embeddings/tree/main).\n        If `True` the linear layer that turns the frequencies into embeddings\n        will be shared across the continuous columns. If `False` a different\n        linear layer will be used for each continuous column.\n        If the _'periodic'_ method is used, this parameter is required.\n    full_embed_dropout: bool, Optional, default = None,\n        If `True`, the full embedding corresponding to a column will be masked\n        out/dropout. If `None`, it will default to `False`.\n    blocks_dims: List, default = [200, 100, 100]\n        List of integers that define the input and output units of each block.\n        For example: _[200, 100, 100]_ will generate 2 blocks. The first will\n        receive a tensor of size 200 and output a tensor of size 100, and the\n        second will receive a tensor of size 100 and output a tensor of size\n        100. See `pytorch_widedeep.models.tab_resnet._layers` for\n        details on the structure of each block.\n    blocks_dropout: float, default =  0.1\n        Block's internal dropout.\n    simplify_blocks: bool, default = False,\n        Boolean indicating if the simplest possible residual blocks (`X -&gt; [\n        [LIN, BN, ACT]  + X ]`) will be used instead of a standard one\n        (`X -&gt; [ [LIN1, BN1, ACT1] -&gt; [LIN2, BN2]  + X ]`).\n    mlp_hidden_dims: List, Optional, default = None\n        List with the number of neurons per dense layer in the MLP. e.g:\n        _[64, 32]_. If `None` the  output of the Resnet Blocks will be\n        connected directly to the output neuron(s).\n    mlp_activation: str, Optional, default = None\n        Activation function for the dense layers of the MLP. Currently\n        _'tanh'_, _'relu'_, _'leaky'_relu' and _'gelu'_ are supported.\n        If 'mlp_hidden_dims' is not `None` and this parameter is `None`, it\n        will default to _'relu'_.\n    mlp_dropout: float, Optional, default = None\n        float with the dropout between the dense layers of the MLP.\n        If 'mlp_hidden_dims' is not `None` and this parameter is `None`, it\n        will default to 0.0.\n    mlp_batchnorm: bool, Optional, default = None\n        Boolean indicating whether or not batch normalization will be applied\n        to the dense layers\n        If 'mlp_hidden_dims' is not `None` and this parameter is `None`, it\n        will default to False.\n    mlp_batchnorm_last: bool, Optional, default = None\n        Boolean indicating whether or not batch normalization will be applied\n        to the last of the dense layers\n        If 'mlp_hidden_dims' is not `None` and this parameter is `None`, it\n        will default to False.\n    mlp_linear_first: bool, Optional, default = None\n        Boolean indicating the order of the operations in the dense\n        layer. If `True: [LIN -&gt; ACT -&gt; BN -&gt; DP]`. If `False: [BN -&gt; DP -&gt;\n        LIN -&gt; ACT]`\n        If 'mlp_hidden_dims' is not `None` and this parameter is `None`, it\n        will default to `True`.\n\n    Attributes\n    ----------\n    encoder: nn.Module\n        deep dense Resnet model that will receive the concatenation of the\n        embeddings and the continuous columns\n    mlp: nn.Module\n        if `mlp_hidden_dims` is `True`, this attribute will be an mlp\n        model that will receive the results of the concatenation of the\n        embeddings and the continuous columns -- if present --.\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.models import TabResnet\n    &gt;&gt;&gt; X_deep = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n    &gt;&gt;&gt; colnames = ['a', 'b', 'c', 'd', 'e']\n    &gt;&gt;&gt; cat_embed_input = [(u,i,j) for u,i,j in zip(colnames[:4], [4]*4, [8]*4)]\n    &gt;&gt;&gt; column_idx = {k:v for v,k in enumerate(colnames)}\n    &gt;&gt;&gt; model = TabResnet(blocks_dims=[16,4], column_idx=column_idx, cat_embed_input=cat_embed_input,\n    ... continuous_cols = ['e'])\n    &gt;&gt;&gt; out = model(X_deep)\n    \"\"\"\n\n    def __init__(\n        self,\n        column_idx: Dict[str, int],\n        *,\n        cat_embed_input: Optional[List[Tuple[str, int, int]]] = None,\n        cat_embed_dropout: Optional[float] = None,\n        use_cat_bias: Optional[bool] = None,\n        cat_embed_activation: Optional[str] = None,\n        continuous_cols: Optional[List[str]] = None,\n        cont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\n        embed_continuous: Optional[bool] = None,\n        embed_continuous_method: Optional[\n            Literal[\"standard\", \"piecewise\", \"periodic\"]\n        ] = None,\n        cont_embed_dim: Optional[int] = None,\n        cont_embed_dropout: Optional[float] = None,\n        cont_embed_activation: Optional[str] = None,\n        quantization_setup: Optional[Dict[str, List[float]]] = None,\n        n_frequencies: Optional[int] = None,\n        sigma: Optional[float] = None,\n        share_last_layer: Optional[bool] = None,\n        full_embed_dropout: Optional[bool] = None,\n        blocks_dims: List[int] = [200, 100, 100],\n        blocks_dropout: float = 0.1,\n        simplify_blocks: bool = False,\n        mlp_hidden_dims: Optional[List[int]] = None,\n        mlp_activation: Optional[str] = None,\n        mlp_dropout: Optional[float] = None,\n        mlp_batchnorm: Optional[bool] = None,\n        mlp_batchnorm_last: Optional[bool] = None,\n        mlp_linear_first: Optional[bool] = None,\n    ):\n        super(TabResnet, self).__init__(\n            column_idx=column_idx,\n            cat_embed_input=cat_embed_input,\n            cat_embed_dropout=cat_embed_dropout,\n            use_cat_bias=use_cat_bias,\n            cat_embed_activation=cat_embed_activation,\n            continuous_cols=continuous_cols,\n            cont_norm_layer=cont_norm_layer,\n            embed_continuous=embed_continuous,\n            embed_continuous_method=embed_continuous_method,\n            cont_embed_dim=cont_embed_dim,\n            cont_embed_dropout=cont_embed_dropout,\n            cont_embed_activation=cont_embed_activation,\n            quantization_setup=quantization_setup,\n            n_frequencies=n_frequencies,\n            sigma=sigma,\n            share_last_layer=share_last_layer,\n            full_embed_dropout=full_embed_dropout,\n        )\n\n        if len(blocks_dims) &lt; 2:\n            raise ValueError(\n                \"'blocks' must contain at least two elements, e.g. [256, 128]\"\n            )\n\n        self.blocks_dims = blocks_dims\n        self.blocks_dropout = blocks_dropout\n        self.simplify_blocks = simplify_blocks\n\n        self.mlp_hidden_dims = mlp_hidden_dims\n        self.mlp_activation = mlp_activation\n        self.mlp_dropout = mlp_dropout\n        self.mlp_batchnorm = mlp_batchnorm\n        self.mlp_batchnorm_last = mlp_batchnorm_last\n        self.mlp_linear_first = mlp_linear_first\n\n        # Embeddings are instantiated at the base model\n\n        # Resnet\n        dense_resnet_input_dim = self.cat_out_dim + self.cont_out_dim\n        self.encoder = DenseResnet(\n            dense_resnet_input_dim, blocks_dims, blocks_dropout, self.simplify_blocks\n        )\n\n        # Mlp: adding an MLP on top of the Resnet blocks is optional and\n        # therefore all related params are optional\n        if self.mlp_hidden_dims is not None:\n            self.mlp = MLP(\n                d_hidden=[self.blocks_dims[-1]] + self.mlp_hidden_dims,\n                activation=(\n                    \"relu\" if self.mlp_activation is None else self.mlp_activation\n                ),\n                dropout=0.0 if self.mlp_dropout is None else self.mlp_dropout,\n                batchnorm=False if self.mlp_batchnorm is None else self.mlp_batchnorm,\n                batchnorm_last=(\n                    False\n                    if self.mlp_batchnorm_last is None\n                    else self.mlp_batchnorm_last\n                ),\n                linear_first=(\n                    True if self.mlp_linear_first is None else self.mlp_linear_first\n                ),\n            )\n        else:\n            self.mlp = None\n\n    def forward(self, X: Tensor) -&gt; Tensor:\n        x = self._get_embeddings(X)\n        x = self.encoder(x)\n        if self.mlp is not None:\n            x = self.mlp(x)\n        return x\n\n    @property\n    def output_dim(self) -&gt; int:\n        r\"\"\"The output dimension of the model. This is a required property\n        neccesary to build the `WideDeep` class\n        \"\"\"\n        return (\n            self.mlp_hidden_dims[-1]\n            if self.mlp_hidden_dims is not None\n            else self.blocks_dims[-1]\n        )\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.resnet.tab_resnet.TabResnet.output_dim","title":"output_dim  <code>property</code>","text":"<pre><code>output_dim\n</code></pre> <p>The output dimension of the model. This is a required property neccesary to build the <code>WideDeep</code> class</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.resnet.tab_resnet.TabResnetDecoder","title":"TabResnetDecoder","text":"<p>               Bases: <code>Module</code></p> <p>Companion decoder model for the <code>TabResnet</code> model (which can be considered an encoder itself)</p> <p>This class is designed to be used with the <code>EncoderDecoderTrainer</code> when using self-supervised pre-training (see the corresponding section in the docs). This class will receive the output from the ResNet blocks or the MLP(if present) and 'reconstruct' the embeddings.</p> <p>Parameters:</p> Name Type Description Default <code>embed_dim</code> <code>int</code> <p>Size of the embeddings tensor to be reconstructed.</p> required <code>blocks_dims</code> <code>List[int]</code> <p>List of integers that define the input and output units of each block. For example: [200, 100, 100] will generate 2 blocks. The first will receive a tensor of size 200 and output a tensor of size 100, and the second will receive a tensor of size 100 and output a tensor of size 100. See <code>pytorch_widedeep.models.tab_resnet._layers</code> for details on the structure of each block.</p> <code>[100, 100, 200]</code> <code>blocks_dropout</code> <code>float</code> <p>Block's internal dropout.</p> <code>0.1</code> <code>simplify_blocks</code> <code>bool</code> <p>Boolean indicating if the simplest possible residual blocks (<code>X -&gt; [ [LIN, BN, ACT]  + X ]</code>) will be used instead of a standard one (<code>X -&gt; [ [LIN1, BN1, ACT1] -&gt; [LIN2, BN2]  + X ]</code>).</p> <code>False</code> <code>mlp_hidden_dims</code> <code>Optional[List[int]]</code> <p>List with the number of neurons per dense layer in the MLP. e.g: [64, 32]. If <code>None</code> the  output of the Resnet Blocks will be connected directly to the output neuron(s).</p> <code>None</code> <code>mlp_activation</code> <code>Optional[str]</code> <p>Activation function for the dense layers of the MLP. Currently 'tanh', 'relu', 'leaky'_relu' and _'gelu' are supported. If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to 'relu'.</p> <code>None</code> <code>mlp_dropout</code> <code>Optional[float]</code> <p>float with the dropout between the dense layers of the MLP. If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to 0.0.</p> <code>None</code> <code>mlp_batchnorm</code> <code>Optional[bool]</code> <p>Boolean indicating whether or not batch normalization will be applied to the dense layers If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to False.</p> <code>None</code> <code>mlp_batchnorm_last</code> <code>Optional[bool]</code> <p>Boolean indicating whether or not batch normalization will be applied to the last of the dense layers If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to False.</p> <code>None</code> <code>mlp_linear_first</code> <code>Optional[bool]</code> <p>Boolean indicating the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code> If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to <code>True</code>.</p> <code>None</code> <p>Attributes:</p> Name Type Description <code>decoder</code> <code>Module</code> <p>deep dense Resnet model that will receive the output of the encoder IF <code>mlp_hidden_dims</code> is None</p> <code>mlp</code> <code>Module</code> <p>if <code>mlp_hidden_dims</code> is not None, the overall decoder will consist in an MLP that will receive the output of the encoder followed by the deep dense Resnet.</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import TabResnetDecoder\n&gt;&gt;&gt; x_inp = torch.rand(3, 8)\n&gt;&gt;&gt; decoder = TabResnetDecoder(embed_dim=32, blocks_dims=[8, 16, 16])\n&gt;&gt;&gt; res = decoder(x_inp)\n&gt;&gt;&gt; res.shape\ntorch.Size([3, 32])\n</code></pre> Source code in <code>pytorch_widedeep/models/tabular/resnet/tab_resnet.py</code> <pre><code>class TabResnetDecoder(nn.Module):\n    r\"\"\"Companion decoder model for the `TabResnet` model (which can be\n    considered an encoder itself)\n\n    This class is designed to be used with the `EncoderDecoderTrainer` when\n    using self-supervised pre-training (see the corresponding section in the\n    docs). This class will receive the output from the ResNet blocks or the\n    MLP(if present) and '_reconstruct_' the embeddings.\n\n    Parameters\n    ----------\n    embed_dim: int\n        Size of the embeddings tensor to be reconstructed.\n    blocks_dims: List, default = [200, 100, 100]\n        List of integers that define the input and output units of each block.\n        For example: _[200, 100, 100]_ will generate 2 blocks. The first will\n        receive a tensor of size 200 and output a tensor of size 100, and the\n        second will receive a tensor of size 100 and output a tensor of size\n        100. See `pytorch_widedeep.models.tab_resnet._layers` for\n        details on the structure of each block.\n    blocks_dropout: float, default =  0.1\n        Block's internal dropout.\n    simplify_blocks: bool, default = False,\n        Boolean indicating if the simplest possible residual blocks (`X -&gt; [\n        [LIN, BN, ACT]  + X ]`) will be used instead of a standard one\n        (`X -&gt; [ [LIN1, BN1, ACT1] -&gt; [LIN2, BN2]  + X ]`).\n    mlp_hidden_dims: List, Optional, default = None\n        List with the number of neurons per dense layer in the MLP. e.g:\n        _[64, 32]_. If `None` the  output of the Resnet Blocks will be\n        connected directly to the output neuron(s).\n    mlp_activation: str, Optional, default = None\n        Activation function for the dense layers of the MLP. Currently\n        _'tanh'_, _'relu'_, _'leaky'_relu' and _'gelu'_ are supported.\n        If 'mlp_hidden_dims' is not `None` and this parameter is `None`, it\n        will default to _'relu'_.\n    mlp_dropout: float, Optional, default = None\n        float with the dropout between the dense layers of the MLP.\n        If 'mlp_hidden_dims' is not `None` and this parameter is `None`, it\n        will default to 0.0.\n    mlp_batchnorm: bool, Optional, default = None\n        Boolean indicating whether or not batch normalization will be applied\n        to the dense layers\n        If 'mlp_hidden_dims' is not `None` and this parameter is `None`, it\n        will default to False.\n    mlp_batchnorm_last: bool, Optional, default = None\n        Boolean indicating whether or not batch normalization will be applied\n        to the last of the dense layers\n        If 'mlp_hidden_dims' is not `None` and this parameter is `None`, it\n        will default to False.\n    mlp_linear_first: bool, Optional, default = None\n        Boolean indicating the order of the operations in the dense\n        layer. If `True: [LIN -&gt; ACT -&gt; BN -&gt; DP]`. If `False: [BN -&gt; DP -&gt;\n        LIN -&gt; ACT]`\n        If 'mlp_hidden_dims' is not `None` and this parameter is `None`, it\n        will default to `True`.\n\n    Attributes\n    ----------\n    decoder: nn.Module\n        deep dense Resnet model that will receive the output of the encoder IF\n        `mlp_hidden_dims` is None\n    mlp: nn.Module\n        if `mlp_hidden_dims` is not None, the overall decoder will consist\n        in an MLP that will receive the output of the encoder followed by the\n        deep dense Resnet.\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.models import TabResnetDecoder\n    &gt;&gt;&gt; x_inp = torch.rand(3, 8)\n    &gt;&gt;&gt; decoder = TabResnetDecoder(embed_dim=32, blocks_dims=[8, 16, 16])\n    &gt;&gt;&gt; res = decoder(x_inp)\n    &gt;&gt;&gt; res.shape\n    torch.Size([3, 32])\n    \"\"\"\n\n    def __init__(\n        self,\n        embed_dim: int,\n        blocks_dims: List[int] = [100, 100, 200],\n        blocks_dropout: float = 0.1,\n        simplify_blocks: bool = False,\n        mlp_hidden_dims: Optional[List[int]] = None,\n        mlp_activation: Optional[str] = None,\n        mlp_dropout: Optional[float] = None,\n        mlp_batchnorm: Optional[bool] = None,\n        mlp_batchnorm_last: Optional[bool] = None,\n        mlp_linear_first: Optional[bool] = None,\n    ):\n        super(TabResnetDecoder, self).__init__()\n\n        if len(blocks_dims) &lt; 2:\n            raise ValueError(\n                \"'blocks' must contain at least two elements, e.g. [256, 128]\"\n            )\n\n        self.embed_dim = embed_dim\n\n        self.blocks_dims = blocks_dims\n        self.blocks_dropout = blocks_dropout\n        self.simplify_blocks = simplify_blocks\n\n        self.mlp_hidden_dims = mlp_hidden_dims\n        self.mlp_activation = mlp_activation\n        self.mlp_dropout = mlp_dropout\n        self.mlp_batchnorm = mlp_batchnorm\n        self.mlp_batchnorm_last = mlp_batchnorm_last\n        self.mlp_linear_first = mlp_linear_first\n\n        if self.mlp_hidden_dims is not None:\n            self.mlp = MLP(\n                d_hidden=[self.mlp_first_hidden_dim] + self.mlp_hidden_dims,\n                activation=(\n                    \"relu\" if self.mlp_activation is None else self.mlp_activation\n                ),\n                dropout=0.0 if self.mlp_dropout is None else self.mlp_dropout,\n                batchnorm=False if self.mlp_batchnorm is None else self.mlp_batchnorm,\n                batchnorm_last=(\n                    False\n                    if self.mlp_batchnorm_last is None\n                    else self.mlp_batchnorm_last\n                ),\n                linear_first=(\n                    True if self.mlp_linear_first is None else self.mlp_linear_first\n                ),\n            )\n            self.decoder = DenseResnet(\n                self.mlp_hidden_dims[-1],\n                blocks_dims,\n                blocks_dropout,\n                self.simplify_blocks,\n            )\n        else:\n            self.mlp = None\n            self.decoder = DenseResnet(\n                blocks_dims[0], blocks_dims, blocks_dropout, self.simplify_blocks\n            )\n\n        self.reconstruction_layer = nn.Linear(blocks_dims[-1], embed_dim, bias=False)\n\n    def forward(self, X: Tensor) -&gt; Tensor:\n        x = self.mlp(X) if self.mlp is not None else X\n        return self.reconstruction_layer(self.decoder(x))\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.tabnet.tab_net.TabNet","title":"TabNet","text":"<p>               Bases: <code>BaseTabularModelWithoutAttention</code></p> <p>Defines a TabNet model that can be used as the <code>deeptabular</code> component of a Wide &amp; Deep model or independently by itself.</p> <p>The implementation in this library is fully based on that here by the dreamquark-ai team, simply adapted so that it can work within the <code>WideDeep</code> frame. Therefore, ALL CREDIT TO THE DREAMQUARK-AI TEAM.</p> <p>Parameters:</p> Name Type Description Default <code>column_idx</code> <code>Dict[str, int]</code> <p>Dict containing the index of the columns that will be passed through the <code>TabMlp</code> model. Required to slice the tensors. e.g. {'education': 0, 'relationship': 1, 'workclass': 2, ...}.</p> required <code>cat_embed_input</code> <code>Optional[List[Tuple[str, int, int]]]</code> <p>List of Tuples with the column name, number of unique values and embedding dimension. e.g. [(education, 11, 32), ...]</p> <code>None</code> <code>cat_embed_dropout</code> <code>Optional[float]</code> <p>Categorical embeddings dropout. If <code>None</code>, it will default to 0.</p> <code>None</code> <code>use_cat_bias</code> <code>Optional[bool]</code> <p>Boolean indicating if bias will be used for the categorical embeddings. If <code>None</code>, it will default to 'False'.</p> <code>None</code> <code>cat_embed_activation</code> <code>Optional[str]</code> <p>Activation function for the categorical embeddings, if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> <code>None</code> <code>continuous_cols</code> <code>Optional[List[str]]</code> <p>List with the name of the numeric (aka continuous) columns</p> <code>None</code> <code>cont_norm_layer</code> <code>Optional[Literal[batchnorm, layernorm]]</code> <p>Type of normalization layer applied to the continuous features. Options are: 'layernorm' and 'batchnorm'. if <code>None</code>, no normalization layer will be used.</p> <code>None</code> <code>embed_continuous</code> <code>Optional[bool]</code> <p>Boolean indicating if the continuous columns will be embedded using one of the available methods: 'standard', 'periodic' or 'piecewise'. If <code>None</code>, it will default to 'False'. NOTE: This parameter is deprecated and it  will be removed in future releases. Please, use the  <code>embed_continuous_method</code> parameter instead.</p> <code>None</code> <code>embed_continuous_method</code> <code>Optional[Literal[standard, piecewise, periodic]]</code> <p>Method to use to embed the continuous features. Options are: 'standard', 'periodic' or 'piecewise'. The 'standard' embedding method is based on the FT-Transformer implementation presented in the paper: Revisiting Deep Learning Models for Tabular Data. The 'periodic' and_'piecewise'_ methods were presented in the paper: On Embeddings for Numerical Features in Tabular Deep Learning. Please, read the papers for details.</p> <code>None</code> <code>cont_embed_dim</code> <code>Optional[int]</code> <p>Size of the continuous embeddings. If the continuous columns are embedded, <code>cont_embed_dim</code> must be passed.</p> <code>None</code> <code>cont_embed_dropout</code> <code>Optional[float]</code> <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p> <code>None</code> <code>cont_embed_activation</code> <code>Optional[str]</code> <p>Activation function for the continuous embeddings if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported. If <code>None</code>, no activation function will be applied.</p> <code>None</code> <code>quantization_setup</code> <code>Optional[Dict[str, List[float]]]</code> <p>This parameter is used when the 'piecewise' method is used to embed the continuous cols. It is a dict where keys are the name of the continuous columns and values are lists with the boundaries for the quantization of the continuous_cols. See the examples for details. If If the 'piecewise' method is used, this parameter is required.</p> <code>None</code> <code>n_frequencies</code> <code>Optional[int]</code> <p>This is the so called 'k' in their paper On Embeddings for Numerical Features in Tabular Deep Learning, and is the number of 'frequencies' that will be used to represent each continuous column. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>sigma</code> <code>Optional[float]</code> <p>This is the sigma parameter in the paper mentioned when describing the previous parameters and it is used to initialise the 'frequency weights'. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>share_last_layer</code> <code>Optional[bool]</code> <p>This parameter is not present in the before mentioned paper but it is implemented in the official repo. If <code>True</code> the linear layer that turns the frequencies into embeddings will be shared across the continuous columns. If <code>False</code> a different linear layer will be used for each continuous column. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>full_embed_dropout</code> <code>Optional[bool]</code> <p>If <code>True</code>, the full embedding corresponding to a column will be masked out/dropout. If <code>None</code>, it will default to <code>False</code>.</p> <code>None</code> <code>n_steps</code> <code>int</code> <p>number of decision steps. For a better understanding of the function of <code>n_steps</code> and the upcoming parameters, please see the paper.</p> <code>3</code> <code>step_dim</code> <code>int</code> <p>Step's output dimension. This is the output dimension that <code>WideDeep</code> will collect and connect to the output neuron(s).</p> <code>8</code> <code>attn_dim</code> <code>int</code> <p>Attention dimension</p> <code>8</code> <code>dropout</code> <code>float</code> <p>GLU block's internal dropout</p> <code>0.0</code> <code>n_glu_step_dependent</code> <code>int</code> <p>number of GLU Blocks (<code>[FC -&gt; BN -&gt; GLU]</code>) that are step dependent</p> <code>2</code> <code>n_glu_shared</code> <code>int</code> <p>number of GLU Blocks (<code>[FC -&gt; BN -&gt; GLU]</code>) that will be shared across decision steps</p> <code>2</code> <code>ghost_bn</code> <code>bool</code> <p>Boolean indicating if Ghost Batch Normalization will be used.</p> <code>True</code> <code>virtual_batch_size</code> <code>int</code> <p>Batch size when using Ghost Batch Normalization</p> <code>128</code> <code>momentum</code> <code>float</code> <p>Ghost Batch Normalization's momentum. The dreamquark-ai advises for very low values. However high values are used in the original publication. During our tests higher values lead to better results</p> <code>0.02</code> <code>gamma</code> <code>float</code> <p>Relaxation parameter in the paper. When gamma = 1, a feature is enforced to be used only at one decision step. As gamma increases, more flexibility is provided to use a feature at multiple decision steps</p> <code>1.3</code> <code>epsilon</code> <code>float</code> <p>Float to avoid log(0). Always keep low</p> <code>1e-15</code> <code>mask_type</code> <code>str</code> <p>Mask function to use. Either 'sparsemax' or 'entmax'</p> <code>'sparsemax'</code> <p>Attributes:</p> Name Type Description <code>encoder</code> <code>Module</code> <p>the TabNet encoder. For details see the original publication.</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n&gt;&gt;&gt; colnames = [\"a\", \"b\", \"c\", \"d\", \"e\"]\n&gt;&gt;&gt; cat_embed_input = [(u, i, j) for u, i, j in zip(colnames[:4], [4] * 4, [8] * 4)]\n&gt;&gt;&gt; column_idx = {k: v for v, k in enumerate(colnames)}\n&gt;&gt;&gt; model = TabNet(column_idx=column_idx, cat_embed_input=cat_embed_input, continuous_cols=[\"e\"])\n&gt;&gt;&gt; out = model(X_tab)\n</code></pre> Source code in <code>pytorch_widedeep/models/tabular/tabnet/tab_net.py</code> <pre><code>class TabNet(BaseTabularModelWithoutAttention):\n    r\"\"\"Defines a [TabNet model](https://arxiv.org/abs/1908.07442) that\n    can be used as the `deeptabular` component of a Wide &amp; Deep model or\n    independently by itself.\n\n    The implementation in this library is fully based on that\n    [here](https://github.com/dreamquark-ai/tabnet) by the dreamquark-ai team,\n    simply adapted so that it can work within the `WideDeep` frame.\n    Therefore, **ALL CREDIT TO THE DREAMQUARK-AI TEAM**.\n\n    Parameters\n    ----------\n    column_idx: Dict\n        Dict containing the index of the columns that will be passed through\n        the `TabMlp` model. Required to slice the tensors. e.g. _{'education':\n        0, 'relationship': 1, 'workclass': 2, ...}_.\n    cat_embed_input: List, Optional, default = None\n        List of Tuples with the column name, number of unique values and\n        embedding dimension. e.g. _[(education, 11, 32), ...]_\n    cat_embed_dropout: float, Optional, default = None\n        Categorical embeddings dropout. If `None`, it will default\n        to 0.\n    use_cat_bias: bool, Optional, default = None,\n        Boolean indicating if bias will be used for the categorical embeddings.\n        If `None`, it will default to 'False'.\n    cat_embed_activation: Optional, str, default = None,\n        Activation function for the categorical embeddings, if any. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported\n    continuous_cols: List, Optional, default = None\n        List with the name of the numeric (aka continuous) columns\n    cont_norm_layer: str, Optional, default =  None\n        Type of normalization layer applied to the continuous features.\n        Options are: _'layernorm'_ and _'batchnorm'_. if `None`, no\n        normalization layer will be used.\n    embed_continuous: bool, Optional, default = None,\n        Boolean indicating if the continuous columns will be embedded using\n        one of the available methods: _'standard'_, _'periodic'_\n        or _'piecewise'_. If `None`, it will default to 'False'.&lt;br/&gt;\n        :information_source: **NOTE**: This parameter is deprecated and it\n         will be removed in future releases. Please, use the\n         `embed_continuous_method` parameter instead.\n    embed_continuous_method: Optional, str, default = None,\n        Method to use to embed the continuous features. Options are:\n        _'standard'_, _'periodic'_ or _'piecewise'_. The _'standard'_\n        embedding method is based on the FT-Transformer implementation\n        presented in the paper: [Revisiting Deep Learning Models for\n        Tabular Data](https://arxiv.org/abs/2106.11959v5). The _'periodic'_\n        and_'piecewise'_ methods were presented in the paper: [On Embeddings for\n        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556).\n        Please, read the papers for details.\n    cont_embed_dim: int, Optional, default = None,\n        Size of the continuous embeddings. If the continuous columns are\n        embedded, `cont_embed_dim` must be passed.\n    cont_embed_dropout: float, Optional, default = None,\n        Dropout for the continuous embeddings. If `None`, it will default to 0.0\n    cont_embed_activation: Optional, str, default = None,\n        Activation function for the continuous embeddings if any. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported.\n        If `None`, no activation function will be applied.\n    quantization_setup: Dict[str, List[float]], Optional, default = None,\n        This parameter is used when the _'piecewise'_ method is used to embed\n        the continuous cols. It is a dict where keys are the name of the continuous\n        columns and values are lists with the boundaries for the quantization\n        of the continuous_cols. See the examples for details. If\n        If the _'piecewise'_ method is used, this parameter is required.\n    n_frequencies: int, Optional, default = None,\n        This is the so called _'k'_ in their paper [On Embeddings for\n        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556),\n        and is the number of 'frequencies' that will be used to represent each\n        continuous column. See their Eq 2 in the paper for details. If\n        the _'periodic'_ method is used, this parameter is required.\n    sigma: float, Optional, default = None,\n        This is the sigma parameter in the paper mentioned when describing the\n        previous parameters and it is used to initialise the 'frequency\n        weights'. See their Eq 2 in the paper for details. If\n        the _'periodic'_ method is used, this parameter is required.\n    share_last_layer: bool, Optional, default = None,\n        This parameter is not present in the before mentioned paper but it is implemented in\n        the [official repo](https://github.com/yandex-research/rtdl-num-embeddings/tree/main).\n        If `True` the linear layer that turns the frequencies into embeddings\n        will be shared across the continuous columns. If `False` a different\n        linear layer will be used for each continuous column.\n        If the _'periodic'_ method is used, this parameter is required.\n    full_embed_dropout: bool, Optional, default = None,\n        If `True`, the full embedding corresponding to a column will be masked\n        out/dropout. If `None`, it will default to `False`.\n    n_steps: int, default = 3\n        number of decision steps. For a better understanding of the function\n        of `n_steps` and the upcoming parameters, please see the\n        [paper](https://arxiv.org/abs/1908.07442).\n    step_dim: int, default = 8\n        Step's output dimension. This is the output dimension that\n        `WideDeep` will collect and connect to the output neuron(s).\n    attn_dim: int, default = 8\n        Attention dimension\n    dropout: float, default = 0.0\n        GLU block's internal dropout\n    n_glu_step_dependent: int, default = 2\n        number of GLU Blocks (`[FC -&gt; BN -&gt; GLU]`) that are step dependent\n    n_glu_shared: int, default = 2\n        number of GLU Blocks (`[FC -&gt; BN -&gt; GLU]`) that will be shared\n        across decision steps\n    ghost_bn: bool, default=True\n        Boolean indicating if [Ghost Batch Normalization](https://arxiv.org/abs/1705.08741)\n        will be used.\n    virtual_batch_size: int, default = 128\n        Batch size when using Ghost Batch Normalization\n    momentum: float, default = 0.02\n        Ghost Batch Normalization's momentum. The dreamquark-ai advises for\n        very low values. However high values are used in the original\n        publication. During our tests higher values lead to better results\n    gamma: float, default = 1.3\n        Relaxation parameter in the paper. When gamma = 1, a feature is\n        enforced to be used only at one decision step. As gamma increases,\n        more flexibility is provided to use a feature at multiple decision\n        steps\n    epsilon: float, default = 1e-15\n        Float to avoid log(0). Always keep low\n    mask_type: str, default = \"sparsemax\"\n        Mask function to use. Either _'sparsemax'_ or _'entmax'_\n\n    Attributes\n    ----------\n    encoder: nn.Module\n        the TabNet encoder. For details see the [original publication](https://arxiv.org/abs/1908.07442).\n\n    Examples\n    --------\n    &gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n    &gt;&gt;&gt; colnames = [\"a\", \"b\", \"c\", \"d\", \"e\"]\n    &gt;&gt;&gt; cat_embed_input = [(u, i, j) for u, i, j in zip(colnames[:4], [4] * 4, [8] * 4)]\n    &gt;&gt;&gt; column_idx = {k: v for v, k in enumerate(colnames)}\n    &gt;&gt;&gt; model = TabNet(column_idx=column_idx, cat_embed_input=cat_embed_input, continuous_cols=[\"e\"])\n    &gt;&gt;&gt; out = model(X_tab)\n    \"\"\"\n\n    def __init__(\n        self,\n        column_idx: Dict[str, int],\n        *,\n        cat_embed_input: Optional[List[Tuple[str, int, int]]] = None,\n        cat_embed_dropout: Optional[float] = None,\n        use_cat_bias: Optional[bool] = None,\n        cat_embed_activation: Optional[str] = None,\n        continuous_cols: Optional[List[str]] = None,\n        cont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\n        embed_continuous: Optional[bool] = None,\n        embed_continuous_method: Optional[\n            Literal[\"standard\", \"piecewise\", \"periodic\"]\n        ] = None,\n        cont_embed_dim: Optional[int] = None,\n        cont_embed_dropout: Optional[float] = None,\n        cont_embed_activation: Optional[str] = None,\n        quantization_setup: Optional[Dict[str, List[float]]] = None,\n        n_frequencies: Optional[int] = None,\n        sigma: Optional[float] = None,\n        share_last_layer: Optional[bool] = None,\n        full_embed_dropout: Optional[bool] = None,\n        n_steps: int = 3,\n        step_dim: int = 8,\n        attn_dim: int = 8,\n        dropout: float = 0.0,\n        n_glu_step_dependent: int = 2,\n        n_glu_shared: int = 2,\n        ghost_bn: bool = True,\n        virtual_batch_size: int = 128,\n        momentum: float = 0.02,\n        gamma: float = 1.3,\n        epsilon: float = 1e-15,\n        mask_type: str = \"sparsemax\",\n    ):\n        super(TabNet, self).__init__(\n            column_idx=column_idx,\n            cat_embed_input=cat_embed_input,\n            cat_embed_dropout=cat_embed_dropout,\n            use_cat_bias=use_cat_bias,\n            cat_embed_activation=cat_embed_activation,\n            continuous_cols=continuous_cols,\n            cont_norm_layer=cont_norm_layer,\n            embed_continuous=embed_continuous,\n            embed_continuous_method=embed_continuous_method,\n            cont_embed_dim=cont_embed_dim,\n            cont_embed_dropout=cont_embed_dropout,\n            cont_embed_activation=cont_embed_activation,\n            quantization_setup=quantization_setup,\n            n_frequencies=n_frequencies,\n            sigma=sigma,\n            share_last_layer=share_last_layer,\n            full_embed_dropout=full_embed_dropout,\n        )\n\n        self.n_steps = n_steps\n        self.step_dim = step_dim\n        self.attn_dim = attn_dim\n        self.dropout = dropout\n        self.n_glu_step_dependent = n_glu_step_dependent\n        self.n_glu_shared = n_glu_shared\n        self.ghost_bn = ghost_bn\n        self.virtual_batch_size = virtual_batch_size\n        self.momentum = momentum\n        self.gamma = gamma\n        self.epsilon = epsilon\n        self.mask_type = mask_type\n\n        # Embeddings are instantiated at the base model\n        self.embed_out_dim = self.cat_out_dim + self.cont_out_dim\n\n        # TabNet\n        self.encoder = TabNetEncoder(\n            self.embed_out_dim,\n            n_steps,\n            step_dim,\n            attn_dim,\n            dropout,\n            n_glu_step_dependent,\n            n_glu_shared,\n            ghost_bn,\n            virtual_batch_size,\n            momentum,\n            gamma,\n            epsilon,\n            mask_type,\n        )\n\n    def forward(\n        self, X: Tensor, prior: Optional[Tensor] = None\n    ) -&gt; Tuple[Tensor, Tensor]:\n        x = self._get_embeddings(X)\n        steps_output, M_loss = self.encoder(x, prior)\n        res = torch.sum(torch.stack(steps_output, dim=0), dim=0)\n        return (res, M_loss)\n\n    def forward_masks(self, X: Tensor) -&gt; Tuple[Tensor, Dict[int, Tensor]]:\n        x = self._get_embeddings(X)\n        return self.encoder.forward_masks(x)\n\n    @property\n    def output_dim(self) -&gt; int:\n        r\"\"\"The output dimension of the model. This is a required property\n        neccesary to build the `WideDeep` class\n        \"\"\"\n        return self.step_dim\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.tabnet.tab_net.TabNet.output_dim","title":"output_dim  <code>property</code>","text":"<pre><code>output_dim\n</code></pre> <p>The output dimension of the model. This is a required property neccesary to build the <code>WideDeep</code> class</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.tabnet.tab_net.TabNetDecoder","title":"TabNetDecoder","text":"<p>               Bases: <code>Module</code></p> <p>Companion decoder model for the <code>TabNet</code> model (which can be considered an encoder itself)</p> <p>This class is designed to be used with the <code>EncoderDecoderTrainer</code> when using self-supervised pre-training (see the corresponding section in the docs). This class will receive the output from the <code>TabNet</code> encoder (i.e. the output from the so called 'steps') and 'reconstruct' the embeddings.</p> <p>Parameters:</p> Name Type Description Default <code>embed_dim</code> <code>int</code> <p>Size of the embeddings tensor to be reconstructed.</p> required <code>n_steps</code> <code>int</code> <p>number of decision steps. For a better understanding of the function of <code>n_steps</code> and the upcoming parameters, please see the paper.</p> <code>3</code> <code>step_dim</code> <code>int</code> <p>Step's output dimension. This is the output dimension that <code>WideDeep</code> will collect and connect to the output neuron(s).</p> <code>8</code> <code>dropout</code> <code>float</code> <p>GLU block's internal dropout</p> <code>0.0</code> <code>n_glu_step_dependent</code> <code>int</code> <p>number of GLU Blocks (<code>[FC -&gt; BN -&gt; GLU]</code>) that are step dependent</p> <code>2</code> <code>n_glu_shared</code> <code>int</code> <p>number of GLU Blocks (<code>[FC -&gt; BN -&gt; GLU]</code>) that will be shared across decision steps</p> <code>2</code> <code>ghost_bn</code> <code>bool</code> <p>Boolean indicating if Ghost Batch Normalization will be used.</p> <code>True</code> <code>virtual_batch_size</code> <code>int</code> <p>Batch size when using Ghost Batch Normalization</p> <code>128</code> <code>momentum</code> <code>float</code> <p>Ghost Batch Normalization's momentum. The dreamquark-ai advises for very low values. However high values are used in the original publication. During our tests higher values lead to better results</p> <code>0.02</code> <p>Attributes:</p> Name Type Description <code>decoder</code> <code>Module</code> <p>decoder that will receive the output from the encoder's steps and will reconstruct the embeddings</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import TabNetDecoder\n&gt;&gt;&gt; x_inp = [torch.rand(3, 8), torch.rand(3, 8), torch.rand(3, 8)]\n&gt;&gt;&gt; decoder = TabNetDecoder(embed_dim=32, ghost_bn=False)\n&gt;&gt;&gt; res = decoder(x_inp)\n&gt;&gt;&gt; res.shape\ntorch.Size([3, 32])\n</code></pre> Source code in <code>pytorch_widedeep/models/tabular/tabnet/tab_net.py</code> <pre><code>class TabNetDecoder(nn.Module):\n    r\"\"\"Companion decoder model for the `TabNet` model (which can be\n    considered an encoder itself)\n\n    This class is designed to be used with the `EncoderDecoderTrainer` when\n    using self-supervised pre-training (see the corresponding section in the\n    docs). This class will receive the output from the `TabNet` encoder\n    (i.e. the output from the so called 'steps') and '_reconstruct_' the\n    embeddings.\n\n    Parameters\n    ----------\n    embed_dim: int\n        Size of the embeddings tensor to be reconstructed.\n    n_steps: int, default = 3\n        number of decision steps. For a better understanding of the function\n        of `n_steps` and the upcoming parameters, please see the\n        [paper](https://arxiv.org/abs/1908.07442).\n    step_dim: int, default = 8\n        Step's output dimension. This is the output dimension that\n        `WideDeep` will collect and connect to the output neuron(s).\n    dropout: float, default = 0.0\n        GLU block's internal dropout\n    n_glu_step_dependent: int, default = 2\n        number of GLU Blocks (`[FC -&gt; BN -&gt; GLU]`) that are step dependent\n    n_glu_shared: int, default = 2\n        number of GLU Blocks (`[FC -&gt; BN -&gt; GLU]`) that will be shared\n        across decision steps\n    ghost_bn: bool, default=True\n        Boolean indicating if [Ghost Batch Normalization](https://arxiv.org/abs/1705.08741)\n        will be used.\n    virtual_batch_size: int, default = 128\n        Batch size when using Ghost Batch Normalization\n    momentum: float, default = 0.02\n        Ghost Batch Normalization's momentum. The dreamquark-ai advises for\n        very low values. However high values are used in the original\n        publication. During our tests higher values lead to better results\n\n    Attributes\n    ----------\n    decoder: nn.Module\n        decoder that will receive the output from the encoder's steps and will\n        reconstruct the embeddings\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.models import TabNetDecoder\n    &gt;&gt;&gt; x_inp = [torch.rand(3, 8), torch.rand(3, 8), torch.rand(3, 8)]\n    &gt;&gt;&gt; decoder = TabNetDecoder(embed_dim=32, ghost_bn=False)\n    &gt;&gt;&gt; res = decoder(x_inp)\n    &gt;&gt;&gt; res.shape\n    torch.Size([3, 32])\n    \"\"\"\n\n    def __init__(\n        self,\n        embed_dim: int,\n        n_steps: int = 3,\n        step_dim: int = 8,\n        dropout: float = 0.0,\n        n_glu_step_dependent: int = 2,\n        n_glu_shared: int = 2,\n        ghost_bn: bool = True,\n        virtual_batch_size: int = 128,\n        momentum: float = 0.02,\n    ):\n        super(TabNetDecoder, self).__init__()\n\n        self.n_steps = n_steps\n        self.step_dim = step_dim\n        self.dropout = dropout\n        self.n_glu_step_dependent = n_glu_step_dependent\n        self.n_glu_shared = n_glu_shared\n        self.ghost_bn = ghost_bn\n        self.virtual_batch_size = virtual_batch_size\n        self.momentum = momentum\n\n        shared_layers = nn.ModuleList()\n        for i in range(n_glu_shared):\n            if i == 0:\n                shared_layers.append(nn.Linear(step_dim, 2 * step_dim, bias=False))\n            else:\n                shared_layers.append(nn.Linear(step_dim, 2 * step_dim, bias=False))\n\n        self.decoder = nn.ModuleList()\n        for step in range(n_steps):\n            transformer = FeatTransformer(\n                step_dim,\n                step_dim,\n                dropout,\n                shared_layers,\n                n_glu_step_dependent,\n                ghost_bn,\n                virtual_batch_size,\n                momentum=momentum,\n            )\n            self.decoder.append(transformer)\n\n        self.reconstruction_layer = nn.Linear(step_dim, embed_dim, bias=False)\n        initialize_non_glu(self.reconstruction_layer, step_dim, embed_dim)\n\n    def forward(self, X: List[Tensor]) -&gt; Tensor:\n        out = torch.tensor(0.0)\n        for i, x in enumerate(X):\n            x = self.decoder[i](x)\n            out = torch.add(out, x)\n        out = self.reconstruction_layer(out)\n        return out\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.mlp.context_attention_mlp.ContextAttentionMLP","title":"ContextAttentionMLP","text":"<p>               Bases: <code>BaseTabularModelWithAttention</code></p> <p>Defines a <code>ContextAttentionMLP</code> model that can be used as the <code>deeptabular</code> component of a Wide &amp; Deep model or independently by itself.</p> <p>This class combines embedding representations of the categorical features with numerical (aka continuous) features that are also embedded. These are then passed through a series of attention blocks. Each attention block is comprised by a <code>ContextAttentionEncoder</code>. Such encoder is in part inspired by the attention mechanism described in Hierarchical Attention Networks for Document Classification. See <code>pytorch_widedeep.models.tabular.mlp._attention_layers</code> for details.</p> <p>Most of the parameters for this class are <code>Optional</code> since the use of categorical or continuous is in fact optional (i.e. one can use categorical features only, continuous features only or both).</p> <p>Parameters:</p> Name Type Description Default <code>column_idx</code> <code>Dict[str, int]</code> <p>Dict containing the index of the columns that will be passed through the <code>TabMlp</code> model. Required to slice the tensors. e.g. {'education': 0, 'relationship': 1, 'workclass': 2, ...}.</p> required <code>cat_embed_input</code> <code>Optional[List[Tuple[str, int]]]</code> <p>List of Tuples with the column name and number of unique values and embedding dimension. e.g. [(education, 11), ...]</p> <code>None</code> <code>cat_embed_dropout</code> <code>Optional[float]</code> <p>Categorical embeddings dropout. If <code>None</code>, it will default to 0.</p> <code>None</code> <code>use_cat_bias</code> <code>Optional[bool]</code> <p>Boolean indicating if bias will be used for the categorical embeddings. If <code>None</code>, it will default to 'False'.</p> <code>None</code> <code>cat_embed_activation</code> <code>Optional[str]</code> <p>Activation function for the categorical embeddings, if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> <code>None</code> <code>shared_embed</code> <code>Optional[bool]</code> <p>Boolean indicating if the embeddings will be \"shared\". The idea behind <code>shared_embed</code> is described in the Appendix A in the TabTransformer paper: 'The goal of having column embedding is to enable the model to distinguish the classes in one column from those in the other columns'. In other words, the idea is to let the model learn which column is embedded at the time. See: <code>pytorch_widedeep.models.transformers._layers.SharedEmbeddings</code>.</p> <code>None</code> <code>add_shared_embed</code> <code>Optional[bool]</code> <p>The two embedding sharing strategies are: 1) add the shared embeddings to the column embeddings or 2) to replace the first <code>frac_shared_embed</code> with the shared embeddings. See <code>pytorch_widedeep.models.embeddings_layers.SharedEmbeddings</code> If 'None' is passed, it will default to 'False'.</p> <code>None</code> <code>frac_shared_embed</code> <code>Optional[float]</code> <p>The fraction of embeddings that will be shared (if <code>add_shared_embed = False</code>) by all the different categories for one particular column. If 'None' is passed, it will default to 0.0.</p> <code>None</code> <code>continuous_cols</code> <code>Optional[List[str]]</code> <p>List with the name of the numeric (aka continuous) columns</p> <code>None</code> <code>cont_norm_layer</code> <code>Optional[Literal[batchnorm, layernorm]]</code> <p>Type of normalization layer applied to the continuous features. Options are: 'layernorm' and 'batchnorm'. if <code>None</code>, no normalization layer will be used.</p> <code>None</code> <code>embed_continuous_method</code> <code>Optional[Literal[standard, piecewise, periodic]]</code> <p>Method to use to embed the continuous features. Options are: 'standard', 'periodic' or 'piecewise'. The 'standard' embedding method is based on the FT-Transformer implementation presented in the paper: Revisiting Deep Learning Models for Tabular Data. The 'periodic' and_'piecewise'_ methods were presented in the paper: On Embeddings for Numerical Features in Tabular Deep Learning. Please, read the papers for details.</p> <code>'standard'</code> <code>cont_embed_dropout</code> <code>Optional[float]</code> <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p> <code>None</code> <code>cont_embed_activation</code> <code>Optional[str]</code> <p>Activation function for the continuous embeddings if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported. If <code>None</code>, no activation function will be applied.</p> <code>None</code> <code>quantization_setup</code> <code>Optional[Dict[str, List[float]]]</code> <p>This parameter is used when the 'piecewise' method is used to embed the continuous cols. It is a dict where keys are the name of the continuous columns and values are lists with the boundaries for the quantization of the continuous_cols. See the examples for details. If If the 'piecewise' method is used, this parameter is required.</p> <code>None</code> <code>n_frequencies</code> <code>Optional[int]</code> <p>This is the so called 'k' in their paper On Embeddings for Numerical Features in Tabular Deep Learning, and is the number of 'frequencies' that will be used to represent each continuous column. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>sigma</code> <code>Optional[float]</code> <p>This is the sigma parameter in the paper mentioned when describing the previous parameters and it is used to initialise the 'frequency weights'. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>share_last_layer</code> <code>Optional[bool]</code> <p>This parameter is not present in the before mentioned paper but it is implemented in the official repo. If <code>True</code> the linear layer that turns the frequencies into embeddings will be shared across the continuous columns. If <code>False</code> a different linear layer will be used for each continuous column. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>full_embed_dropout</code> <code>Optional[bool]</code> <p>If <code>True</code>, the full embedding corresponding to a column will be masked out/dropout. If <code>None</code>, it will default to <code>False</code>.</p> <code>None</code> <code>input_dim</code> <code>int</code> <p>The so-called dimension of the model. Is the number of embeddings used to encode the categorical and/or continuous columns</p> <code>32</code> <code>attn_dropout</code> <code>float</code> <p>Dropout for each attention block</p> <code>0.2</code> <code>with_addnorm</code> <code>bool</code> <p>Boolean indicating if residual connections will be used in the attention blocks</p> <code>False</code> <code>attn_activation</code> <code>str</code> <p>String indicating the activation function to be applied to the dense layer in each attention encoder. 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported.</p> <code>'leaky_relu'</code> <code>n_blocks</code> <code>int</code> <p>Number of attention blocks</p> <code>3</code> <p>Attributes:</p> Name Type Description <code>encoder</code> <code>Module</code> <p>Sequence of attention encoders.</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import ContextAttentionMLP\n&gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n&gt;&gt;&gt; colnames = ['a', 'b', 'c', 'd', 'e']\n&gt;&gt;&gt; cat_embed_input = [(u,i,j) for u,i,j in zip(colnames[:4], [4]*4, [8]*4)]\n&gt;&gt;&gt; column_idx = {k:v for v,k in enumerate(colnames)}\n&gt;&gt;&gt; model = ContextAttentionMLP(column_idx=column_idx, cat_embed_input=cat_embed_input, continuous_cols = ['e'])\n&gt;&gt;&gt; out = model(X_tab)\n</code></pre> Source code in <code>pytorch_widedeep/models/tabular/mlp/context_attention_mlp.py</code> <pre><code>class ContextAttentionMLP(BaseTabularModelWithAttention):\n    r\"\"\"Defines a `ContextAttentionMLP` model that can be used as the\n    `deeptabular` component of a Wide &amp; Deep model or independently by\n    itself.\n\n    This class combines embedding representations of the categorical features\n    with numerical (aka continuous) features that are also embedded. These\n    are then passed through a series of attention blocks. Each attention\n    block is comprised by a `ContextAttentionEncoder`. Such encoder is in\n    part inspired by the attention mechanism described in\n    [Hierarchical Attention Networks for Document\n    Classification](https://www.cs.cmu.edu/~./hovy/papers/16HLT-hierarchical-attention-networks.pdf).\n    See `pytorch_widedeep.models.tabular.mlp._attention_layers` for details.\n\n    Most of the parameters for this class are `Optional` since the use of\n    categorical or continuous is in fact optional (i.e. one can use\n    categorical features only, continuous features only or both).\n\n    Parameters\n    ----------\n    column_idx: Dict\n        Dict containing the index of the columns that will be passed through\n        the `TabMlp` model. Required to slice the tensors. e.g. _{'education':\n        0, 'relationship': 1, 'workclass': 2, ...}_.\n    cat_embed_input: List, Optional, default = None\n        List of Tuples with the column name and number of unique values and\n        embedding dimension. e.g. _[(education, 11), ...]_\n    cat_embed_dropout: float, Optional, default = None\n        Categorical embeddings dropout. If `None`, it will default\n        to 0.\n    use_cat_bias: bool, Optional, default = None,\n        Boolean indicating if bias will be used for the categorical embeddings.\n        If `None`, it will default to 'False'.\n    cat_embed_activation: Optional, str, default = None,\n        Activation function for the categorical embeddings, if any. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported\n    shared_embed: bool, Optional, default = None\n        Boolean indicating if the embeddings will be \"shared\". The idea behind `shared_embed` is\n        described in the Appendix A in the [TabTransformer paper](https://arxiv.org/abs/2012.06678):\n        _'The goal of having column embedding is to enable the model to\n        distinguish the classes in one column from those in the other\n        columns'_. In other words, the idea is to let the model learn which\n        column is embedded at the time. See: `pytorch_widedeep.models.transformers._layers.SharedEmbeddings`.\n    add_shared_embed: bool, Optional, default = None\n        The two embedding sharing strategies are: 1) add the shared embeddings\n        to the column embeddings or 2) to replace the first\n        `frac_shared_embed` with the shared embeddings.\n        See `pytorch_widedeep.models.embeddings_layers.SharedEmbeddings`\n        If 'None' is passed, it will default to 'False'.\n    frac_shared_embed: float, Optional, default = None\n        The fraction of embeddings that will be shared (if `add_shared_embed\n        = False`) by all the different categories for one particular\n        column. If 'None' is passed, it will default to 0.0.\n    continuous_cols: List, Optional, default = None\n        List with the name of the numeric (aka continuous) columns\n    cont_norm_layer: str, Optional, default =  None\n        Type of normalization layer applied to the continuous features.\n        Options are: _'layernorm'_ and _'batchnorm'_. if `None`, no\n        normalization layer will be used.\n    embed_continuous_method: Optional, str, default = None,\n        Method to use to embed the continuous features. Options are:\n        _'standard'_, _'periodic'_ or _'piecewise'_. The _'standard'_\n        embedding method is based on the FT-Transformer implementation\n        presented in the paper: [Revisiting Deep Learning Models for\n        Tabular Data](https://arxiv.org/abs/2106.11959v5). The _'periodic'_\n        and_'piecewise'_ methods were presented in the paper: [On Embeddings for\n        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556).\n        Please, read the papers for details.\n    cont_embed_dropout: float, Optional, default = None,\n        Dropout for the continuous embeddings. If `None`, it will default to 0.0\n    cont_embed_activation: Optional, str, default = None,\n        Activation function for the continuous embeddings if any. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported.\n        If `None`, no activation function will be applied.\n    quantization_setup: Dict[str, List[float]], Optional, default = None,\n        This parameter is used when the _'piecewise'_ method is used to embed\n        the continuous cols. It is a dict where keys are the name of the continuous\n        columns and values are lists with the boundaries for the quantization\n        of the continuous_cols. See the examples for details. If\n        If the _'piecewise'_ method is used, this parameter is required.\n    n_frequencies: int, Optional, default = None,\n        This is the so called _'k'_ in their paper [On Embeddings for\n        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556),\n        and is the number of 'frequencies' that will be used to represent each\n        continuous column. See their Eq 2 in the paper for details. If\n        the _'periodic'_ method is used, this parameter is required.\n    sigma: float, Optional, default = None,\n        This is the sigma parameter in the paper mentioned when describing the\n        previous parameters and it is used to initialise the 'frequency\n        weights'. See their Eq 2 in the paper for details. If\n        the _'periodic'_ method is used, this parameter is required.\n    share_last_layer: bool, Optional, default = None,\n        This parameter is not present in the before mentioned paper but it is implemented in\n        the [official repo](https://github.com/yandex-research/rtdl-num-embeddings/tree/main).\n        If `True` the linear layer that turns the frequencies into embeddings\n        will be shared across the continuous columns. If `False` a different\n        linear layer will be used for each continuous column.\n        If the _'periodic'_ method is used, this parameter is required.\n    full_embed_dropout: bool, Optional, default = None,\n        If `True`, the full embedding corresponding to a column will be masked\n        out/dropout. If `None`, it will default to `False`.\n    input_dim: int, default = 32\n        The so-called *dimension of the model*. Is the number of embeddings\n        used to encode the categorical and/or continuous columns\n    attn_dropout: float, default = 0.2\n        Dropout for each attention block\n    with_addnorm: bool = False,\n        Boolean indicating if residual connections will be used in the\n        attention blocks\n    attn_activation: str, default = \"leaky_relu\"\n        String indicating the activation function to be applied to the dense\n        layer in each attention encoder. _'tanh'_, _'relu'_, _'leaky_relu'_\n        and _'gelu'_ are supported.\n    n_blocks: int, default = 3\n        Number of attention blocks\n\n    Attributes\n    ----------\n    encoder: nn.Module\n        Sequence of attention encoders.\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.models import ContextAttentionMLP\n    &gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n    &gt;&gt;&gt; colnames = ['a', 'b', 'c', 'd', 'e']\n    &gt;&gt;&gt; cat_embed_input = [(u,i,j) for u,i,j in zip(colnames[:4], [4]*4, [8]*4)]\n    &gt;&gt;&gt; column_idx = {k:v for v,k in enumerate(colnames)}\n    &gt;&gt;&gt; model = ContextAttentionMLP(column_idx=column_idx, cat_embed_input=cat_embed_input, continuous_cols = ['e'])\n    &gt;&gt;&gt; out = model(X_tab)\n    \"\"\"\n\n    def __init__(\n        self,\n        column_idx: Dict[str, int],\n        *,\n        cat_embed_input: Optional[List[Tuple[str, int]]] = None,\n        cat_embed_dropout: Optional[float] = None,\n        use_cat_bias: Optional[bool] = None,\n        cat_embed_activation: Optional[str] = None,\n        shared_embed: Optional[bool] = None,\n        add_shared_embed: Optional[bool] = None,\n        frac_shared_embed: Optional[float] = None,\n        continuous_cols: Optional[List[str]] = None,\n        cont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\n        embed_continuous_method: Optional[\n            Literal[\"standard\", \"piecewise\", \"periodic\"]\n        ] = \"standard\",\n        cont_embed_dropout: Optional[float] = None,\n        cont_embed_activation: Optional[str] = None,\n        quantization_setup: Optional[Dict[str, List[float]]] = None,\n        n_frequencies: Optional[int] = None,\n        sigma: Optional[float] = None,\n        share_last_layer: Optional[bool] = None,\n        full_embed_dropout: Optional[bool] = None,\n        input_dim: int = 32,\n        attn_dropout: float = 0.2,\n        with_addnorm: bool = False,\n        attn_activation: str = \"leaky_relu\",\n        n_blocks: int = 3,\n    ):\n        super(ContextAttentionMLP, self).__init__(\n            column_idx=column_idx,\n            cat_embed_input=cat_embed_input,\n            cat_embed_dropout=cat_embed_dropout,\n            use_cat_bias=use_cat_bias,\n            cat_embed_activation=cat_embed_activation,\n            shared_embed=shared_embed,\n            add_shared_embed=add_shared_embed,\n            frac_shared_embed=frac_shared_embed,\n            continuous_cols=continuous_cols,\n            cont_norm_layer=cont_norm_layer,\n            embed_continuous_method=embed_continuous_method,\n            cont_embed_dropout=cont_embed_dropout,\n            cont_embed_activation=cont_embed_activation,\n            input_dim=input_dim,\n            quantization_setup=quantization_setup,\n            n_frequencies=n_frequencies,\n            sigma=sigma,\n            share_last_layer=share_last_layer,\n            full_embed_dropout=full_embed_dropout,\n        )\n\n        self.attn_dropout = attn_dropout\n        self.with_addnorm = with_addnorm\n        self.attn_activation = attn_activation\n        self.n_blocks = n_blocks\n\n        self.with_cls_token = \"cls_token\" in column_idx\n        self.n_cat = len(cat_embed_input) if cat_embed_input is not None else 0\n        self.n_cont = len(continuous_cols) if continuous_cols is not None else 0\n\n        # Embeddings are instantiated at the base model\n        # Attention Blocks\n        self.encoder = nn.Sequential()\n        for i in range(n_blocks):\n            self.encoder.add_module(\n                \"attention_block\" + str(i),\n                ContextAttentionEncoder(\n                    input_dim,\n                    attn_dropout,\n                    with_addnorm,\n                    attn_activation,\n                ),\n            )\n\n    def forward(self, X: Tensor) -&gt; Tensor:\n        x = self._get_embeddings(X)\n        x = self.encoder(x)\n        if self.with_cls_token:\n            out = x[:, 0, :]\n        else:\n            out = x.flatten(1)\n        return out\n\n    @property\n    def output_dim(self) -&gt; int:\n        r\"\"\"The output dimension of the model. This is a required property\n        neccesary to build the `WideDeep` class\n        \"\"\"\n        return (\n            self.input_dim\n            if self.with_cls_token\n            else ((self.n_cat + self.n_cont) * self.input_dim)\n        )\n\n    @property\n    def attention_weights(self) -&gt; List[Tensor]:\n        r\"\"\"List with the attention weights per block\n\n        The shape of the attention weights is $(N, F)$, where $N$ is the batch\n        size and $F$ is the number of features/columns in the dataset\n        \"\"\"\n        return [blk.attn.attn_weights for blk in self.encoder]\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.mlp.context_attention_mlp.ContextAttentionMLP.output_dim","title":"output_dim  <code>property</code>","text":"<pre><code>output_dim\n</code></pre> <p>The output dimension of the model. This is a required property neccesary to build the <code>WideDeep</code> class</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.mlp.context_attention_mlp.ContextAttentionMLP.attention_weights","title":"attention_weights  <code>property</code>","text":"<pre><code>attention_weights\n</code></pre> <p>List with the attention weights per block</p> <p>The shape of the attention weights is \\((N, F)\\), where \\(N\\) is the batch size and \\(F\\) is the number of features/columns in the dataset</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.mlp.self_attention_mlp.SelfAttentionMLP","title":"SelfAttentionMLP","text":"<p>               Bases: <code>BaseTabularModelWithAttention</code></p> <p>Defines a <code>SelfAttentionMLP</code> model that can be used as the deeptabular component of a Wide &amp; Deep model or independently by itself.</p> <p>This class combines embedding representations of the categorical features with numerical (aka continuous) features that are also embedded. These are then passed through a series of attention blocks. Each attention block is comprised by what we would refer as a simplified <code>SelfAttentionEncoder</code>. See <code>pytorch_widedeep.models.tabular.mlp._attention_layers</code> for details. The reason to use a simplified version of self attention is because we observed that the 'standard' attention mechanism used in the TabTransformer has a notable tendency to overfit.</p> <p>In more detail, this model only uses Q and K (and not V). If we think about it as in terms of text (and intuitively), the Softmax(QK^T) is the attention mechanism that tells us how much, at each position in the input sentence, each word is represented or 'expressed'. We refer to that as 'attention weights'. These attention weighst are normally multiplied by a Value matrix to further strength the focus on the words that each word should be attending to (again, intuitively).</p> <p>In this implementation we skip this last multiplication and instead we multiply the attention weights directly by the input tensor. This is a simplification that we expect is beneficial in terms of avoiding overfitting for tabular data.</p> <p>Most of the parameters for this class are <code>Optional</code> since the use of categorical or continuous is in fact optional (i.e. one can use categorical features only, continuous features only or both).</p> <p>Parameters:</p> Name Type Description Default <code>column_idx</code> <code>Dict[str, int]</code> <p>Dict containing the index of the columns that will be passed through the <code>TabMlp</code> model. Required to slice the tensors. e.g. {'education': 0, 'relationship': 1, 'workclass': 2, ...}.</p> required <code>cat_embed_input</code> <code>Optional[List[Tuple[str, int]]]</code> <p>List of Tuples with the column name and number of unique values and embedding dimension. e.g. [(education, 11), ...]</p> <code>None</code> <code>cat_embed_dropout</code> <code>Optional[float]</code> <p>Categorical embeddings dropout. If <code>None</code>, it will default to 0.</p> <code>None</code> <code>use_cat_bias</code> <code>Optional[bool]</code> <p>Boolean indicating if bias will be used for the categorical embeddings. If <code>None</code>, it will default to 'False'.</p> <code>None</code> <code>cat_embed_activation</code> <code>Optional[str]</code> <p>Activation function for the categorical embeddings, if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> <code>None</code> <code>shared_embed</code> <code>Optional[bool]</code> <p>Boolean indicating if the embeddings will be \"shared\". The idea behind <code>shared_embed</code> is described in the Appendix A in the TabTransformer paper: 'The goal of having column embedding is to enable the model to distinguish the classes in one column from those in the other columns'. In other words, the idea is to let the model learn which column is embedded at the time. See: <code>pytorch_widedeep.models.transformers._layers.SharedEmbeddings</code>.</p> <code>None</code> <code>add_shared_embed</code> <code>Optional[bool]</code> <p>The two embedding sharing strategies are: 1) add the shared embeddings to the column embeddings or 2) to replace the first <code>frac_shared_embed</code> with the shared embeddings. See <code>pytorch_widedeep.models.embeddings_layers.SharedEmbeddings</code> If 'None' is passed, it will default to 'False'.</p> <code>None</code> <code>frac_shared_embed</code> <code>Optional[float]</code> <p>The fraction of embeddings that will be shared (if <code>add_shared_embed = False</code>) by all the different categories for one particular column. If 'None' is passed, it will default to 0.0.</p> <code>None</code> <code>continuous_cols</code> <code>Optional[List[str]]</code> <p>List with the name of the numeric (aka continuous) columns</p> <code>None</code> <code>cont_norm_layer</code> <code>Optional[Literal[batchnorm, layernorm]]</code> <p>Type of normalization layer applied to the continuous features. Options are: 'layernorm' and 'batchnorm'. if <code>None</code>, no normalization layer will be used.</p> <code>None</code> <code>embed_continuous_method</code> <code>Optional[Literal[standard, piecewise, periodic]]</code> <p>Method to use to embed the continuous features. Options are: 'standard', 'periodic' or 'piecewise'. The 'standard' embedding method is based on the FT-Transformer implementation presented in the paper: Revisiting Deep Learning Models for Tabular Data. The 'periodic' and_'piecewise'_ methods were presented in the paper: On Embeddings for Numerical Features in Tabular Deep Learning. Please, read the papers for details.</p> <code>'standard'</code> <code>cont_embed_dropout</code> <code>Optional[float]</code> <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p> <code>None</code> <code>cont_embed_activation</code> <code>Optional[str]</code> <p>Activation function for the continuous embeddings if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported. If <code>None</code>, no activation function will be applied.</p> <code>None</code> <code>quantization_setup</code> <code>Optional[Dict[str, List[float]]]</code> <p>This parameter is used when the 'piecewise' method is used to embed the continuous cols. It is a dict where keys are the name of the continuous columns and values are lists with the boundaries for the quantization of the continuous_cols. See the examples for details. If If the 'piecewise' method is used, this parameter is required.</p> <code>None</code> <code>n_frequencies</code> <code>Optional[int]</code> <p>This is the so called 'k' in their paper On Embeddings for Numerical Features in Tabular Deep Learning, and is the number of 'frequencies' that will be used to represent each continuous column. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>sigma</code> <code>Optional[float]</code> <p>This is the sigma parameter in the paper mentioned when describing the previous parameters and it is used to initialise the 'frequency weights'. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>share_last_layer</code> <code>Optional[bool]</code> <p>This parameter is not present in the before mentioned paper but it is implemented in the official repo. If <code>True</code> the linear layer that turns the frequencies into embeddings will be shared across the continuous columns. If <code>False</code> a different linear layer will be used for each continuous column. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>full_embed_dropout</code> <code>Optional[bool]</code> <p>If <code>True</code>, the full embedding corresponding to a column will be masked out/dropout. If <code>None</code>, it will default to <code>False</code>.</p> <code>None</code> <code>input_dim</code> <code>int</code> <p>The so-called dimension of the model. Is the number of embeddings used to encode the categorical and/or continuous columns</p> <code>32</code> <code>attn_dropout</code> <code>float</code> <p>Dropout for each attention block</p> <code>0.2</code> <code>n_heads</code> <code>int</code> <p>Number of attention heads per attention block.</p> <code>8</code> <code>use_bias</code> <code>bool</code> <p>Boolean indicating whether or not to use bias in the Q, K projection layers.</p> <code>False</code> <code>with_addnorm</code> <code>bool</code> <p>Boolean indicating if residual connections will be used in the attention blocks</p> <code>False</code> <code>attn_activation</code> <code>str</code> <p>String indicating the activation function to be applied to the dense layer in each attention encoder. 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported.</p> <code>'leaky_relu'</code> <code>n_blocks</code> <code>int</code> <p>Number of attention blocks</p> <code>3</code> <p>Attributes:</p> Name Type Description <code>cat_and_cont_embed</code> <code>Module</code> <p>This is the module that processes the categorical and continuous columns</p> <code>encoder</code> <code>Module</code> <p>Sequence of attention encoders.</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import SelfAttentionMLP\n&gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n&gt;&gt;&gt; colnames = ['a', 'b', 'c', 'd', 'e']\n&gt;&gt;&gt; cat_embed_input = [(u,i,j) for u,i,j in zip(colnames[:4], [4]*4, [8]*4)]\n&gt;&gt;&gt; column_idx = {k:v for v,k in enumerate(colnames)}\n&gt;&gt;&gt; model = SelfAttentionMLP(column_idx=column_idx, cat_embed_input=cat_embed_input, continuous_cols = ['e'])\n&gt;&gt;&gt; out = model(X_tab)\n</code></pre> Source code in <code>pytorch_widedeep/models/tabular/mlp/self_attention_mlp.py</code> <pre><code>class SelfAttentionMLP(BaseTabularModelWithAttention):\n    r\"\"\"Defines a `SelfAttentionMLP` model that can be used as the\n    deeptabular component of a Wide &amp; Deep model or independently by\n    itself.\n\n    This class combines embedding representations of the categorical features\n    with numerical (aka continuous) features that are also embedded. These\n    are then passed through a series of attention blocks. Each attention\n    block is comprised by what we would refer as a simplified\n    `SelfAttentionEncoder`. See\n    `pytorch_widedeep.models.tabular.mlp._attention_layers` for details. The\n    reason to use a simplified version of self attention is because we\n    observed that the '_standard_' attention mechanism used in the\n    TabTransformer has a notable tendency to overfit.\n\n    In more detail, this model only uses Q and K (and not V). If we think\n    about it as in terms of text (and intuitively), the Softmax(QK^T) is the\n    attention mechanism that tells us how much, at each position in the input\n    sentence, each word is represented or 'expressed'. We refer to that\n    as 'attention weights'. These attention weighst are normally multiplied\n    by a Value matrix to further strength the focus on the words that each\n    word should be attending to (again, intuitively).\n\n    In this implementation we skip this last multiplication and instead we\n    multiply the attention weights directly by the input tensor. This is a\n    simplification that we expect is beneficial in terms of avoiding\n    overfitting for tabular data.\n\n    Most of the parameters for this class are `Optional` since the use of\n    categorical or continuous is in fact optional (i.e. one can use\n    categorical features only, continuous features only or both).\n\n    Parameters\n    ----------\n    column_idx: Dict\n        Dict containing the index of the columns that will be passed through\n        the `TabMlp` model. Required to slice the tensors. e.g. _{'education':\n        0, 'relationship': 1, 'workclass': 2, ...}_.\n    cat_embed_input: List, Optional, default = None\n        List of Tuples with the column name and number of unique values and\n        embedding dimension. e.g. _[(education, 11), ...]_\n    cat_embed_dropout: float, Optional, default = None\n        Categorical embeddings dropout. If `None`, it will default\n        to 0.\n    use_cat_bias: bool, Optional, default = None,\n        Boolean indicating if bias will be used for the categorical embeddings.\n        If `None`, it will default to 'False'.\n    cat_embed_activation: Optional, str, default = None,\n        Activation function for the categorical embeddings, if any. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported\n    shared_embed: bool, Optional, default = None\n        Boolean indicating if the embeddings will be \"shared\". The idea behind `shared_embed` is\n        described in the Appendix A in the [TabTransformer paper](https://arxiv.org/abs/2012.06678):\n        _'The goal of having column embedding is to enable the model to\n        distinguish the classes in one column from those in the other\n        columns'_. In other words, the idea is to let the model learn which\n        column is embedded at the time. See: `pytorch_widedeep.models.transformers._layers.SharedEmbeddings`.\n    add_shared_embed: bool, Optional, default = None\n        The two embedding sharing strategies are: 1) add the shared embeddings\n        to the column embeddings or 2) to replace the first\n        `frac_shared_embed` with the shared embeddings.\n        See `pytorch_widedeep.models.embeddings_layers.SharedEmbeddings`\n        If 'None' is passed, it will default to 'False'.\n    frac_shared_embed: float, Optional, default = None\n        The fraction of embeddings that will be shared (if `add_shared_embed\n        = False`) by all the different categories for one particular\n        column. If 'None' is passed, it will default to 0.0.\n    continuous_cols: List, Optional, default = None\n        List with the name of the numeric (aka continuous) columns\n    cont_norm_layer: str, Optional, default =  None\n        Type of normalization layer applied to the continuous features.\n        Options are: _'layernorm'_ and _'batchnorm'_. if `None`, no\n        normalization layer will be used.\n    embed_continuous_method: Optional, str, default = None,\n        Method to use to embed the continuous features. Options are:\n        _'standard'_, _'periodic'_ or _'piecewise'_. The _'standard'_\n        embedding method is based on the FT-Transformer implementation\n        presented in the paper: [Revisiting Deep Learning Models for\n        Tabular Data](https://arxiv.org/abs/2106.11959v5). The _'periodic'_\n        and_'piecewise'_ methods were presented in the paper: [On Embeddings for\n        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556).\n        Please, read the papers for details.\n    cont_embed_dropout: float, Optional, default = None,\n        Dropout for the continuous embeddings. If `None`, it will default to 0.0\n    cont_embed_activation: Optional, str, default = None,\n        Activation function for the continuous embeddings if any. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported.\n        If `None`, no activation function will be applied.\n    quantization_setup: Dict[str, List[float]], Optional, default = None,\n        This parameter is used when the _'piecewise'_ method is used to embed\n        the continuous cols. It is a dict where keys are the name of the continuous\n        columns and values are lists with the boundaries for the quantization\n        of the continuous_cols. See the examples for details. If\n        If the _'piecewise'_ method is used, this parameter is required.\n    n_frequencies: int, Optional, default = None,\n        This is the so called _'k'_ in their paper [On Embeddings for\n        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556),\n        and is the number of 'frequencies' that will be used to represent each\n        continuous column. See their Eq 2 in the paper for details. If\n        the _'periodic'_ method is used, this parameter is required.\n    sigma: float, Optional, default = None,\n        This is the sigma parameter in the paper mentioned when describing the\n        previous parameters and it is used to initialise the 'frequency\n        weights'. See their Eq 2 in the paper for details. If\n        the _'periodic'_ method is used, this parameter is required.\n    share_last_layer: bool, Optional, default = None,\n        This parameter is not present in the before mentioned paper but it is implemented in\n        the [official repo](https://github.com/yandex-research/rtdl-num-embeddings/tree/main).\n        If `True` the linear layer that turns the frequencies into embeddings\n        will be shared across the continuous columns. If `False` a different\n        linear layer will be used for each continuous column.\n        If the _'periodic'_ method is used, this parameter is required.\n    full_embed_dropout: bool, Optional, default = None,\n        If `True`, the full embedding corresponding to a column will be masked\n        out/dropout. If `None`, it will default to `False`.\n    input_dim: int, default = 32\n        The so-called *dimension of the model*. Is the number of\n        embeddings used to encode the categorical and/or continuous columns\n    attn_dropout: float, default = 0.2\n        Dropout for each attention block\n    n_heads: int, default = 8\n        Number of attention heads per attention block.\n    use_bias: bool, default = False\n        Boolean indicating whether or not to use bias in the Q, K projection\n        layers.\n    with_addnorm: bool = False,\n        Boolean indicating if residual connections will be used in the attention blocks\n    attn_activation: str, default = \"leaky_relu\"\n        String indicating the activation function to be applied to the dense\n        layer in each attention encoder. _'tanh'_, _'relu'_, _'leaky_relu'_\n        and _'gelu'_ are supported.\n    n_blocks: int, default = 3\n        Number of attention blocks\n\n    Attributes\n    ----------\n    cat_and_cont_embed: nn.Module\n        This is the module that processes the categorical and continuous columns\n    encoder: nn.Module\n        Sequence of attention encoders.\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.models import SelfAttentionMLP\n    &gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n    &gt;&gt;&gt; colnames = ['a', 'b', 'c', 'd', 'e']\n    &gt;&gt;&gt; cat_embed_input = [(u,i,j) for u,i,j in zip(colnames[:4], [4]*4, [8]*4)]\n    &gt;&gt;&gt; column_idx = {k:v for v,k in enumerate(colnames)}\n    &gt;&gt;&gt; model = SelfAttentionMLP(column_idx=column_idx, cat_embed_input=cat_embed_input, continuous_cols = ['e'])\n    &gt;&gt;&gt; out = model(X_tab)\n    \"\"\"\n\n    def __init__(\n        self,\n        column_idx: Dict[str, int],\n        *,\n        cat_embed_input: Optional[List[Tuple[str, int]]] = None,\n        cat_embed_dropout: Optional[float] = None,\n        use_cat_bias: Optional[bool] = None,\n        cat_embed_activation: Optional[str] = None,\n        shared_embed: Optional[bool] = None,\n        add_shared_embed: Optional[bool] = None,\n        frac_shared_embed: Optional[float] = None,\n        continuous_cols: Optional[List[str]] = None,\n        cont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\n        embed_continuous_method: Optional[\n            Literal[\"standard\", \"piecewise\", \"periodic\"]\n        ] = \"standard\",\n        cont_embed_dropout: Optional[float] = None,\n        cont_embed_activation: Optional[str] = None,\n        quantization_setup: Optional[Dict[str, List[float]]] = None,\n        n_frequencies: Optional[int] = None,\n        sigma: Optional[float] = None,\n        share_last_layer: Optional[bool] = None,\n        full_embed_dropout: Optional[bool] = None,\n        input_dim: int = 32,\n        attn_dropout: float = 0.2,\n        n_heads: int = 8,\n        use_bias: bool = False,\n        with_addnorm: bool = False,\n        attn_activation: str = \"leaky_relu\",\n        n_blocks: int = 3,\n    ):\n        super(SelfAttentionMLP, self).__init__(\n            column_idx=column_idx,\n            input_dim=input_dim,\n            cat_embed_input=cat_embed_input,\n            cat_embed_dropout=cat_embed_dropout,\n            use_cat_bias=use_cat_bias,\n            cat_embed_activation=cat_embed_activation,\n            shared_embed=shared_embed,\n            add_shared_embed=add_shared_embed,\n            frac_shared_embed=frac_shared_embed,\n            continuous_cols=continuous_cols,\n            cont_norm_layer=cont_norm_layer,\n            embed_continuous_method=embed_continuous_method,\n            cont_embed_dropout=cont_embed_dropout,\n            cont_embed_activation=cont_embed_activation,\n            quantization_setup=quantization_setup,\n            n_frequencies=n_frequencies,\n            sigma=sigma,\n            share_last_layer=share_last_layer,\n            full_embed_dropout=full_embed_dropout,\n        )\n\n        self.attn_dropout = attn_dropout\n        self.n_heads = n_heads\n        self.use_bias = use_bias\n        self.with_addnorm = with_addnorm\n        self.attn_activation = attn_activation\n        self.n_blocks = n_blocks\n\n        self.with_cls_token = \"cls_token\" in column_idx\n        self.n_cat = len(cat_embed_input) if cat_embed_input is not None else 0\n        self.n_cont = len(continuous_cols) if continuous_cols is not None else 0\n\n        # Embeddings are instantiated at the base model\n        # Attention Blocks\n        self.encoder = nn.Sequential()\n        for i in range(n_blocks):\n            self.encoder.add_module(\n                \"attention_block\" + str(i),\n                SelfAttentionEncoder(\n                    input_dim,\n                    attn_dropout,\n                    use_bias,\n                    n_heads,\n                    with_addnorm,\n                    attn_activation,\n                ),\n            )\n\n    def forward(self, X: Tensor) -&gt; Tensor:\n        x = self._get_embeddings(X)\n        x = self.encoder(x)\n        if self.with_cls_token:\n            out = x[:, 0, :]\n        else:\n            out = x.flatten(1)\n        return out\n\n    @property\n    def output_dim(self) -&gt; int:\n        r\"\"\"The output dimension of the model. This is a required property\n        neccesary to build the WideDeep class\n        \"\"\"\n        return (\n            self.input_dim\n            if self.with_cls_token\n            else ((self.n_cat + self.n_cont) * self.input_dim)\n        )\n\n    @property\n    def attention_weights(self) -&gt; List[Tensor]:\n        r\"\"\"List with the attention weights per block\n\n        The shape of the attention weights is $(N, H, F, F)$, where $N$ is the\n        batch size, $H$ is the number of attention heads and $F$ is the\n        number of features/columns in the dataset\n        \"\"\"\n        return [blk.attn.attn_weights for blk in self.encoder]\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.mlp.self_attention_mlp.SelfAttentionMLP.output_dim","title":"output_dim  <code>property</code>","text":"<pre><code>output_dim\n</code></pre> <p>The output dimension of the model. This is a required property neccesary to build the WideDeep class</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.mlp.self_attention_mlp.SelfAttentionMLP.attention_weights","title":"attention_weights  <code>property</code>","text":"<pre><code>attention_weights\n</code></pre> <p>List with the attention weights per block</p> <p>The shape of the attention weights is \\((N, H, F, F)\\), where \\(N\\) is the batch size, \\(H\\) is the number of attention heads and \\(F\\) is the number of features/columns in the dataset</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.transformers.tab_transformer.TabTransformer","title":"TabTransformer","text":"<p>               Bases: <code>BaseTabularModelWithAttention</code></p> <p>Defines our adptation of the TabTransformer model that can be used as the <code>deeptabular</code> component of a Wide &amp; Deep model or independently by itself.</p> <p>Most of the parameters for this class are <code>Optional</code> since the use of categorical or continuous is in fact optional (i.e. one can use categorical features only, continuous features only or both).</p> <p> NOTE: This is an enhanced adaptation of the model described in the paper. It can be considered as the flagship of our transformer family of models for tabular data and offers mutiple, additional features relative to the original publication(and some other models in the library)</p> <p>Parameters:</p> Name Type Description Default <code>column_idx</code> <code>Dict[str, int]</code> <p>Dict containing the index of the columns that will be passed through the <code>TabMlp</code> model. Required to slice the tensors. e.g. {'education': 0, 'relationship': 1, 'workclass': 2, ...}.</p> required <code>cat_embed_input</code> <code>Optional[List[Tuple[str, int]]]</code> <p>List of Tuples with the column name and number of unique values and embedding dimension. e.g. [(education, 11), ...]</p> <code>None</code> <code>cat_embed_dropout</code> <code>Optional[float]</code> <p>Categorical embeddings dropout. If <code>None</code>, it will default to 0.</p> <code>None</code> <code>use_cat_bias</code> <code>Optional[bool]</code> <p>Boolean indicating if bias will be used for the categorical embeddings. If <code>None</code>, it will default to 'False'.</p> <code>None</code> <code>cat_embed_activation</code> <code>Optional[str]</code> <p>Activation function for the categorical embeddings, if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> <code>None</code> <code>shared_embed</code> <code>Optional[bool]</code> <p>Boolean indicating if the embeddings will be \"shared\". The idea behind <code>shared_embed</code> is described in the Appendix A in the TabTransformer paper: 'The goal of having column embedding is to enable the model to distinguish the classes in one column from those in the other columns'. In other words, the idea is to let the model learn which column is embedded at the time. See: <code>pytorch_widedeep.models.transformers._layers.SharedEmbeddings</code>.</p> <code>None</code> <code>add_shared_embed</code> <code>Optional[bool]</code> <p>The two embedding sharing strategies are: 1) add the shared embeddings to the column embeddings or 2) to replace the first <code>frac_shared_embed</code> with the shared embeddings. See <code>pytorch_widedeep.models.embeddings_layers.SharedEmbeddings</code> If 'None' is passed, it will default to 'False'.</p> <code>None</code> <code>frac_shared_embed</code> <code>Optional[float]</code> <p>The fraction of embeddings that will be shared (if <code>add_shared_embed = False</code>) by all the different categories for one particular column. If 'None' is passed, it will default to 0.0.</p> <code>None</code> <code>continuous_cols</code> <code>Optional[List[str]]</code> <p>List with the name of the numeric (aka continuous) columns</p> <code>None</code> <code>cont_norm_layer</code> <code>Optional[Literal[batchnorm, layernorm]]</code> <p>Type of normalization layer applied to the continuous features. Options are: 'layernorm' and 'batchnorm'. if <code>None</code>, no normalization layer will be used.</p> <code>None</code> <code>embed_continuous_method</code> <code>Optional[Literal[standard, piecewise, periodic]]</code> <p>Method to use to embed the continuous features. Options are: 'standard', 'periodic' or 'piecewise'. The 'standard' embedding method is based on the FT-Transformer implementation presented in the paper: Revisiting Deep Learning Models for Tabular Data. The 'periodic' and_'piecewise'_ methods were presented in the paper: On Embeddings for Numerical Features in Tabular Deep Learning. Please, read the papers for details.</p> <code>None</code> <code>cont_embed_dropout</code> <code>Optional[float]</code> <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p> <code>None</code> <code>cont_embed_activation</code> <code>Optional[str]</code> <p>Activation function for the continuous embeddings if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported. If <code>None</code>, no activation function will be applied.</p> <code>None</code> <code>quantization_setup</code> <code>Optional[Dict[str, List[float]]]</code> <p>This parameter is used when the 'piecewise' method is used to embed the continuous cols. It is a dict where keys are the name of the continuous columns and values are lists with the boundaries for the quantization of the continuous_cols. See the examples for details. If If the 'piecewise' method is used, this parameter is required.</p> <code>None</code> <code>n_frequencies</code> <code>Optional[int]</code> <p>This is the so called 'k' in their paper On Embeddings for Numerical Features in Tabular Deep Learning, and is the number of 'frequencies' that will be used to represent each continuous column. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>sigma</code> <code>Optional[float]</code> <p>This is the sigma parameter in the paper mentioned when describing the previous parameters and it is used to initialise the 'frequency weights'. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>share_last_layer</code> <code>Optional[bool]</code> <p>This parameter is not present in the before mentioned paper but it is implemented in the official repo. If <code>True</code> the linear layer that turns the frequencies into embeddings will be shared across the continuous columns. If <code>False</code> a different linear layer will be used for each continuous column. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>full_embed_dropout</code> <code>Optional[bool]</code> <p>If <code>True</code>, the full embedding corresponding to a column will be masked out/dropout. If <code>None</code>, it will default to <code>False</code>.</p> <code>None</code> <code>input_dim</code> <code>int</code> <p>The so-called dimension of the model. Is the number of embeddings used to encode the categorical and/or continuous columns</p> <code>32</code> <code>n_heads</code> <code>int</code> <p>Number of attention heads per Transformer block</p> <code>8</code> <code>use_qkv_bias</code> <code>bool</code> <p>Boolean indicating whether or not to use bias in the Q, K, and V projection layers.</p> <code>False</code> <code>n_blocks</code> <code>int</code> <p>Number of Transformer blocks</p> <code>4</code> <code>attn_dropout</code> <code>float</code> <p>Dropout that will be applied to the Multi-Head Attention layers</p> <code>0.2</code> <code>ff_dropout</code> <code>float</code> <p>Dropout that will be applied to the FeedForward network</p> <code>0.1</code> <code>ff_factor</code> <code>int</code> <p>Multiplicative factor applied to the first layer of the FF network in each Transformer block, This is normally set to 4.</p> <code>4</code> <code>transformer_activation</code> <code>str</code> <p>Transformer Encoder activation function. 'tanh', 'relu', 'leaky_relu', 'gelu', 'geglu' and 'reglu' are supported</p> <code>'gelu'</code> <code>use_linear_attention</code> <code>bool</code> <p>Boolean indicating if Linear Attention (from Transformers are RNNs: Fast Autoregressive Transformers with Linear Attention) will be used. The inclusing of this mode of attention is inspired by this post, where the Uber team finds that this attention mechanism leads to the best results for their tabular data.</p> <code>False</code> <code>use_flash_attention</code> <code>bool</code> <p>Boolean indicating if Flash Attention will be used. </p> <code>False</code> <code>mlp_hidden_dims</code> <code>Optional[List[int]]</code> <p>List with the number of neurons per dense layer in the MLP. e.g: [64, 32]. If not provided no MLP on top of the final Transformer block will be used.</p> <code>None</code> <code>mlp_activation</code> <code>str</code> <p>Activation function for the dense layers of the MLP. Currently 'tanh', 'relu', 'leaky'_relu' and _'gelu' are supported. If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to 'relu'.</p> <code>'relu'</code> <code>mlp_dropout</code> <code>float</code> <p>float with the dropout between the dense layers of the MLP. If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to 0.0.</p> <code>0.1</code> <code>mlp_batchnorm</code> <code>bool</code> <p>Boolean indicating whether or not batch normalization will be applied to the dense layers If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to False.</p> <code>False</code> <code>mlp_batchnorm_last</code> <code>bool</code> <p>Boolean indicating whether or not batch normalization will be applied to the last of the dense layers If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to False.</p> <code>False</code> <code>mlp_linear_first</code> <code>bool</code> <p>Boolean indicating the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code> If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to <code>True</code>.</p> <code>True</code> <p>Attributes:</p> Name Type Description <code>encoder</code> <code>Module</code> <p>Sequence of Transformer blocks</p> <code>mlp</code> <code>Module</code> <p>MLP component in the model</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import TabTransformer\n&gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n&gt;&gt;&gt; colnames = ['a', 'b', 'c', 'd', 'e']\n&gt;&gt;&gt; cat_embed_input = [(u,i) for u,i in zip(colnames[:4], [4]*4)]\n&gt;&gt;&gt; continuous_cols = ['e']\n&gt;&gt;&gt; column_idx = {k:v for v,k in enumerate(colnames)}\n&gt;&gt;&gt; model = TabTransformer(column_idx=column_idx, cat_embed_input=cat_embed_input, continuous_cols=continuous_cols)\n&gt;&gt;&gt; out = model(X_tab)\n</code></pre> Source code in <code>pytorch_widedeep/models/tabular/transformers/tab_transformer.py</code> <pre><code>class TabTransformer(BaseTabularModelWithAttention):\n    r\"\"\"Defines our adptation of the\n    [TabTransformer model](https://arxiv.org/abs/2012.06678)\n    that can be used as the `deeptabular` component of a\n    Wide &amp; Deep model or independently by itself.\n\n    Most of the parameters for this class are `Optional` since the use of\n    categorical or continuous is in fact optional (i.e. one can use\n    categorical features only, continuous features only or both).\n\n    :information_source: **NOTE**:\n    This is an enhanced adaptation of the model described in the paper. It can\n    be considered as the flagship of our transformer family of models for\n    tabular data and offers mutiple, additional features relative to the\n    original publication(and some other models in the library)\n\n    Parameters\n    ----------\n    column_idx: Dict\n        Dict containing the index of the columns that will be passed through\n        the `TabMlp` model. Required to slice the tensors. e.g. _{'education':\n        0, 'relationship': 1, 'workclass': 2, ...}_.\n    cat_embed_input: List, Optional, default = None\n        List of Tuples with the column name and number of unique values and\n        embedding dimension. e.g. _[(education, 11), ...]_\n    cat_embed_dropout: float, Optional, default = None\n        Categorical embeddings dropout. If `None`, it will default\n        to 0.\n    use_cat_bias: bool, Optional, default = None,\n        Boolean indicating if bias will be used for the categorical embeddings.\n        If `None`, it will default to 'False'.\n    cat_embed_activation: Optional, str, default = None,\n        Activation function for the categorical embeddings, if any. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported\n    shared_embed: bool, Optional, default = None\n        Boolean indicating if the embeddings will be \"shared\". The idea behind `shared_embed` is\n        described in the Appendix A in the [TabTransformer paper](https://arxiv.org/abs/2012.06678):\n        _'The goal of having column embedding is to enable the model to\n        distinguish the classes in one column from those in the other\n        columns'_. In other words, the idea is to let the model learn which\n        column is embedded at the time. See: `pytorch_widedeep.models.transformers._layers.SharedEmbeddings`.\n    add_shared_embed: bool, Optional, default = None\n        The two embedding sharing strategies are: 1) add the shared embeddings\n        to the column embeddings or 2) to replace the first\n        `frac_shared_embed` with the shared embeddings.\n        See `pytorch_widedeep.models.embeddings_layers.SharedEmbeddings`\n        If 'None' is passed, it will default to 'False'.\n    frac_shared_embed: float, Optional, default = None\n        The fraction of embeddings that will be shared (if `add_shared_embed\n        = False`) by all the different categories for one particular\n        column. If 'None' is passed, it will default to 0.0.\n    continuous_cols: List, Optional, default = None\n        List with the name of the numeric (aka continuous) columns\n    cont_norm_layer: str, Optional, default =  None\n        Type of normalization layer applied to the continuous features.\n        Options are: _'layernorm'_ and _'batchnorm'_. if `None`, no\n        normalization layer will be used.\n    embed_continuous_method: Optional, str, default = None,\n        Method to use to embed the continuous features. Options are:\n        _'standard'_, _'periodic'_ or _'piecewise'_. The _'standard'_\n        embedding method is based on the FT-Transformer implementation\n        presented in the paper: [Revisiting Deep Learning Models for\n        Tabular Data](https://arxiv.org/abs/2106.11959v5). The _'periodic'_\n        and_'piecewise'_ methods were presented in the paper: [On Embeddings for\n        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556).\n        Please, read the papers for details.\n    cont_embed_dropout: float, Optional, default = None,\n        Dropout for the continuous embeddings. If `None`, it will default to 0.0\n    cont_embed_activation: Optional, str, default = None,\n        Activation function for the continuous embeddings if any. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported.\n        If `None`, no activation function will be applied.\n    quantization_setup: Dict[str, List[float]], Optional, default = None,\n        This parameter is used when the _'piecewise'_ method is used to embed\n        the continuous cols. It is a dict where keys are the name of the continuous\n        columns and values are lists with the boundaries for the quantization\n        of the continuous_cols. See the examples for details. If\n        If the _'piecewise'_ method is used, this parameter is required.\n    n_frequencies: int, Optional, default = None,\n        This is the so called _'k'_ in their paper [On Embeddings for\n        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556),\n        and is the number of 'frequencies' that will be used to represent each\n        continuous column. See their Eq 2 in the paper for details. If\n        the _'periodic'_ method is used, this parameter is required.\n    sigma: float, Optional, default = None,\n        This is the sigma parameter in the paper mentioned when describing the\n        previous parameters and it is used to initialise the 'frequency\n        weights'. See their Eq 2 in the paper for details. If\n        the _'periodic'_ method is used, this parameter is required.\n    share_last_layer: bool, Optional, default = None,\n        This parameter is not present in the before mentioned paper but it is implemented in\n        the [official repo](https://github.com/yandex-research/rtdl-num-embeddings/tree/main).\n        If `True` the linear layer that turns the frequencies into embeddings\n        will be shared across the continuous columns. If `False` a different\n        linear layer will be used for each continuous column.\n        If the _'periodic'_ method is used, this parameter is required.\n    full_embed_dropout: bool, Optional, default = None,\n        If `True`, the full embedding corresponding to a column will be masked\n        out/dropout. If `None`, it will default to `False`.\n    input_dim: int, default = 32\n        The so-called *dimension of the model*. Is the number of\n        embeddings used to encode the categorical and/or continuous columns\n    n_heads: int, default = 8\n        Number of attention heads per Transformer block\n    use_qkv_bias: bool, default = False\n        Boolean indicating whether or not to use bias in the Q, K, and V\n        projection layers.\n    n_blocks: int, default = 4\n        Number of Transformer blocks\n    attn_dropout: float, default = 0.2\n        Dropout that will be applied to the Multi-Head Attention layers\n    ff_dropout: float, default = 0.1\n        Dropout that will be applied to the FeedForward network\n    ff_factor: float, default = 4\n        Multiplicative factor applied to the first layer of the FF network in\n        each Transformer block, This is normally set to 4.\n    transformer_activation: str, default = \"gelu\"\n        Transformer Encoder activation function. _'tanh'_, _'relu'_,\n        _'leaky_relu'_, _'gelu'_, _'geglu'_ and _'reglu'_ are supported\n    use_linear_attention: Boolean, default = False,\n        Boolean indicating if Linear Attention (from [Transformers are RNNs:\n        Fast Autoregressive Transformers with Linear Attention](https://arxiv.org/abs/2006.16236))\n        will be used. The inclusing of this mode of attention is inspired by\n        [this post](https://www.uber.com/en-GB/blog/deepeta-how-uber-predicts-arrival-times/),\n        where the Uber team finds that this attention mechanism leads to the\n        best results for their tabular data.\n    use_flash_attention: Boolean, default = False,\n        Boolean indicating if\n        [Flash Attention](https://pytorch.org/docs/stable/generated/torch.nn.functional.scaled_dot_product_attention.html)\n        will be used. &lt;br/&gt;\n    mlp_hidden_dims: List, Optional, default = None\n        List with the number of neurons per dense layer in the MLP. e.g:\n        _[64, 32]_. If not provided no MLP on top of the final\n        Transformer block will be used.\n    mlp_activation: str, Optional, default = None\n        Activation function for the dense layers of the MLP. Currently\n        _'tanh'_, _'relu'_, _'leaky'_relu' and _'gelu'_ are supported.\n        If 'mlp_hidden_dims' is not `None` and this parameter is `None`, it\n        will default to _'relu'_.\n    mlp_dropout: float, Optional, default = None\n        float with the dropout between the dense layers of the MLP.\n        If 'mlp_hidden_dims' is not `None` and this parameter is `None`, it\n        will default to 0.0.\n    mlp_batchnorm: bool, Optional, default = None\n        Boolean indicating whether or not batch normalization will be applied\n        to the dense layers\n        If 'mlp_hidden_dims' is not `None` and this parameter is `None`, it\n        will default to False.\n    mlp_batchnorm_last: bool, Optional, default = None\n        Boolean indicating whether or not batch normalization will be applied\n        to the last of the dense layers\n        If 'mlp_hidden_dims' is not `None` and this parameter is `None`, it\n        will default to False.\n    mlp_linear_first: bool, Optional, default = None\n        Boolean indicating the order of the operations in the dense\n        layer. If `True: [LIN -&gt; ACT -&gt; BN -&gt; DP]`. If `False: [BN -&gt; DP -&gt;\n        LIN -&gt; ACT]`\n        If 'mlp_hidden_dims' is not `None` and this parameter is `None`, it\n        will default to `True`.\n\n    Attributes\n    ----------\n    encoder: nn.Module\n        Sequence of Transformer blocks\n    mlp: nn.Module\n        MLP component in the model\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.models import TabTransformer\n    &gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n    &gt;&gt;&gt; colnames = ['a', 'b', 'c', 'd', 'e']\n    &gt;&gt;&gt; cat_embed_input = [(u,i) for u,i in zip(colnames[:4], [4]*4)]\n    &gt;&gt;&gt; continuous_cols = ['e']\n    &gt;&gt;&gt; column_idx = {k:v for v,k in enumerate(colnames)}\n    &gt;&gt;&gt; model = TabTransformer(column_idx=column_idx, cat_embed_input=cat_embed_input, continuous_cols=continuous_cols)\n    &gt;&gt;&gt; out = model(X_tab)\n    \"\"\"\n\n    def __init__(\n        self,\n        column_idx: Dict[str, int],\n        *,\n        cat_embed_input: Optional[List[Tuple[str, int]]] = None,\n        cat_embed_dropout: Optional[float] = None,\n        use_cat_bias: Optional[bool] = None,\n        cat_embed_activation: Optional[str] = None,\n        shared_embed: Optional[bool] = None,\n        add_shared_embed: Optional[bool] = None,\n        frac_shared_embed: Optional[float] = None,\n        continuous_cols: Optional[List[str]] = None,\n        cont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\n        embed_continuous: Optional[bool] = None,\n        embed_continuous_method: Optional[\n            Literal[\"standard\", \"piecewise\", \"periodic\"]\n        ] = None,\n        cont_embed_dropout: Optional[float] = None,\n        cont_embed_activation: Optional[str] = None,\n        quantization_setup: Optional[Dict[str, List[float]]] = None,\n        n_frequencies: Optional[int] = None,\n        sigma: Optional[float] = None,\n        share_last_layer: Optional[bool] = None,\n        full_embed_dropout: Optional[bool] = None,\n        input_dim: int = 32,\n        n_heads: int = 8,\n        use_qkv_bias: bool = False,\n        n_blocks: int = 4,\n        attn_dropout: float = 0.2,\n        ff_dropout: float = 0.1,\n        ff_factor: int = 4,\n        transformer_activation: str = \"gelu\",\n        use_linear_attention: bool = False,\n        use_flash_attention: bool = False,\n        mlp_hidden_dims: Optional[List[int]] = None,\n        mlp_activation: str = \"relu\",\n        mlp_dropout: float = 0.1,\n        mlp_batchnorm: bool = False,\n        mlp_batchnorm_last: bool = False,\n        mlp_linear_first: bool = True,\n    ):\n        super(TabTransformer, self).__init__(\n            column_idx=column_idx,\n            cat_embed_input=cat_embed_input,\n            cat_embed_dropout=cat_embed_dropout,\n            use_cat_bias=use_cat_bias,\n            cat_embed_activation=cat_embed_activation,\n            shared_embed=shared_embed,\n            add_shared_embed=add_shared_embed,\n            frac_shared_embed=frac_shared_embed,\n            continuous_cols=continuous_cols,\n            cont_norm_layer=cont_norm_layer,\n            # UGLY hack to be able to use the base class __init__ method\n            embed_continuous_method=(\n                \"standard\"\n                if embed_continuous_method is None\n                else embed_continuous_method\n            ),\n            cont_embed_dropout=cont_embed_dropout,\n            cont_embed_activation=cont_embed_activation,\n            quantization_setup=quantization_setup,\n            n_frequencies=n_frequencies,\n            sigma=sigma,\n            share_last_layer=share_last_layer,\n            input_dim=input_dim,\n            full_embed_dropout=full_embed_dropout,\n        )\n\n        # we overwrite the embed_continuous_method parameter that was set\n        # to 'standard' if embed_continuous_method is None. In other words,\n        # this will already have the right value if the user provided a value\n        # for embed_continuous_method, otherwise will be \"standard\" when it\n        # should be None\n        self.embed_continuous_method = embed_continuous_method\n\n        if embed_continuous is not None:\n            self.embed_continuous = embed_continuous\n            if embed_continuous and embed_continuous_method is None:\n                raise ValueError(\n                    \"If 'embed_continuous' is True, 'embed_continuous_method' must be \"\n                    \"one of 'standard', 'piecewise' or 'periodic'.\"\n                )\n        elif self.embed_continuous_method is not None:\n            self.embed_continuous = True\n        else:\n            self.embed_continuous = False\n\n        self.n_heads = n_heads\n        self.use_qkv_bias = use_qkv_bias\n        self.n_blocks = n_blocks\n        self.attn_dropout = attn_dropout\n        self.ff_dropout = ff_dropout\n        self.transformer_activation = transformer_activation\n        self.use_linear_attention = use_linear_attention\n        self.use_flash_attention = use_flash_attention\n        self.ff_factor = ff_factor\n\n        self.mlp_hidden_dims = mlp_hidden_dims\n        self.mlp_activation = mlp_activation\n        self.mlp_dropout = mlp_dropout\n        self.mlp_batchnorm = mlp_batchnorm\n        self.mlp_batchnorm_last = mlp_batchnorm_last\n        self.mlp_linear_first = mlp_linear_first\n\n        self.with_cls_token = \"cls_token\" in column_idx\n        self.n_cat = len(cat_embed_input) if cat_embed_input is not None else 0\n        self.n_cont = len(continuous_cols) if continuous_cols is not None else 0\n\n        if self.n_cont and not self.n_cat and not self.embed_continuous:\n            raise ValueError(\n                \"If only continuous features are used 'embed_continuous' must be set to 'True'\"\n            )\n\n        # Embeddings are instantiated at the base model\n        # Transformer blocks\n        self.encoder = nn.Sequential()\n        for i in range(n_blocks):\n            self.encoder.add_module(\n                \"transformer_block\" + str(i),\n                TransformerEncoder(\n                    input_dim,\n                    n_heads,\n                    use_qkv_bias,\n                    attn_dropout,\n                    ff_dropout,\n                    ff_factor,\n                    transformer_activation,\n                    use_linear_attention,\n                    use_flash_attention,\n                ),\n            )\n\n        self.mlp_first_hidden_dim = self._mlp_first_hidden_dim()\n\n        if self.mlp_hidden_dims is not None:\n            self.mlp = MLP(\n                d_hidden=[self.mlp_first_hidden_dim] + self.mlp_hidden_dims,\n                activation=(\n                    \"relu\" if self.mlp_activation is None else self.mlp_activation\n                ),\n                dropout=0.0 if self.mlp_dropout is None else self.mlp_dropout,\n                batchnorm=False if self.mlp_batchnorm is None else self.mlp_batchnorm,\n                batchnorm_last=(\n                    False\n                    if self.mlp_batchnorm_last is None\n                    else self.mlp_batchnorm_last\n                ),\n                linear_first=(\n                    False if self.mlp_linear_first is None else self.mlp_linear_first\n                ),\n            )\n        else:\n            self.mlp = None\n\n    def forward(self, X: Tensor) -&gt; Tensor:\n        x_embed, x_cont_not_embed = self._get_embeddings_tt(X)\n        x = self.encoder(x_embed)\n\n        if self.with_cls_token:\n            x = x[:, 0, :]\n        else:\n            x = x.flatten(1)\n\n        if x_cont_not_embed is not None:\n            x = torch.cat([x, x_cont_not_embed], 1)\n\n        if self.mlp is not None:\n            x = self.mlp(x)\n        return x\n\n    def _mlp_first_hidden_dim(self) -&gt; int:\n        if self.with_cls_token:\n            if self.embed_continuous:\n                attn_output_dim = self.input_dim\n            else:\n                attn_output_dim = self.input_dim + self.n_cont\n        elif self.embed_continuous:\n            attn_output_dim = (self.n_cat + self.n_cont) * self.input_dim\n        else:\n            attn_output_dim = self.n_cat * self.input_dim + self.n_cont\n\n        return attn_output_dim\n\n    @property\n    def output_dim(self) -&gt; int:\n        r\"\"\"The output dimension of the model. This is a required property\n        neccesary to build the `WideDeep` class\n        \"\"\"\n        return (\n            self.mlp_hidden_dims[-1]\n            if self.mlp_hidden_dims is not None\n            else self.mlp_first_hidden_dim\n        )\n\n    @property\n    def attention_weights(self) -&gt; List[Tensor]:\n        r\"\"\"List with the attention weights per block\n\n        The shape of the attention weights is $(N, H, F, F)$, where $N$ is the\n        batch size, $H$ is the number of attention heads and $F$ is the\n        number of features/columns in the dataset\n\n        :information_source: **NOTE**:\n        if flash attention or linear attention\n        are used, no attention weights are saved during the training process\n        and calling this property will throw a ValueError\n        \"\"\"\n        if self.use_flash_attention or self.use_linear_attention:\n            raise ValueError(\n                \"Extraction of the attention weights is not supported for \"\n                \"linear or flash attention\"\n            )\n\n        return [blk.attn.attn_weights for blk in self.encoder]\n\n    def _get_embeddings_tt(self, X: Tensor) -&gt; Tuple[Tensor, Optional[Tensor]]:  # type: ignore[override]\n        if self.n_cont and self.embed_continuous and self.n_cat:\n            # there are continuous and categorical features and the continuous\n            # features are embedded\n            x_embed = self._get_embeddings(X)\n            x_cont_not_embed = None\n        elif self.n_cont and self.embed_continuous:\n            # there are only continuous features and they are embedded\n            x_embed = self.cont_norm((X[:, self.cont_idx].float()))\n            x_embed = self.cont_embed(x_embed)\n            x_cont_not_embed = None\n        elif self.n_cont and self.n_cat:\n            # there are continuous and categorical features but the continuous\n            # features are not embedded\n            x_embed = self.cat_embed(X)\n            x_cont_not_embed = self.cont_norm((X[:, self.cont_idx].float()))\n        elif self.n_cat:\n            # there are only categorical features\n            x_embed = self.cat_embed(X)\n            x_cont_not_embed = None\n        else:\n            raise ValueError(\n                \"Using only continuous features that are not embedded is \"\n                \"is not supported in this implementation. Please set 'embed_continuous_method' \"\n                \"to one of ['standard', 'piecewise', 'periodic']\"\n            )\n        return x_embed, x_cont_not_embed\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.transformers.tab_transformer.TabTransformer.output_dim","title":"output_dim  <code>property</code>","text":"<pre><code>output_dim\n</code></pre> <p>The output dimension of the model. This is a required property neccesary to build the <code>WideDeep</code> class</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.transformers.tab_transformer.TabTransformer.attention_weights","title":"attention_weights  <code>property</code>","text":"<pre><code>attention_weights\n</code></pre> <p>List with the attention weights per block</p> <p>The shape of the attention weights is \\((N, H, F, F)\\), where \\(N\\) is the batch size, \\(H\\) is the number of attention heads and \\(F\\) is the number of features/columns in the dataset</p> <p> NOTE: if flash attention or linear attention are used, no attention weights are saved during the training process and calling this property will throw a ValueError</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.transformers.saint.SAINT","title":"SAINT","text":"<p>               Bases: <code>BaseTabularModelWithAttention</code></p> <p>Defines a SAINT model that can be used as the <code>deeptabular</code> component of a Wide &amp; Deep model or independently by itself.</p> <p>Most of the parameters for this class are <code>Optional</code> since the use of categorical or continuous is in fact optional (i.e. one can use categorical features only, continuous features only or both).</p> <p> NOTE: This is an slightly modified and enhanced  version of the model described in the paper,</p> <p>Parameters:</p> Name Type Description Default <code>column_idx</code> <code>Dict[str, int]</code> <p>Dict containing the index of the columns that will be passed through the <code>TabMlp</code> model. Required to slice the tensors. e.g. {'education': 0, 'relationship': 1, 'workclass': 2, ...}.</p> required <code>cat_embed_input</code> <code>Optional[List[Tuple[str, int]]]</code> <p>List of Tuples with the column name and number of unique values and embedding dimension. e.g. [(education, 11), ...]</p> <code>None</code> <code>cat_embed_dropout</code> <code>Optional[float]</code> <p>Categorical embeddings dropout. If <code>None</code>, it will default to 0.</p> <code>None</code> <code>use_cat_bias</code> <code>Optional[bool]</code> <p>Boolean indicating if bias will be used for the categorical embeddings. If <code>None</code>, it will default to 'False'.</p> <code>None</code> <code>cat_embed_activation</code> <code>Optional[str]</code> <p>Activation function for the categorical embeddings, if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> <code>None</code> <code>shared_embed</code> <code>Optional[bool]</code> <p>Boolean indicating if the embeddings will be \"shared\". The idea behind <code>shared_embed</code> is described in the Appendix A in the TabTransformer paper: 'The goal of having column embedding is to enable the model to distinguish the classes in one column from those in the other columns'. In other words, the idea is to let the model learn which column is embedded at the time. See: <code>pytorch_widedeep.models.transformers._layers.SharedEmbeddings</code>.</p> <code>None</code> <code>add_shared_embed</code> <code>Optional[bool]</code> <p>The two embedding sharing strategies are: 1) add the shared embeddings to the column embeddings or 2) to replace the first <code>frac_shared_embed</code> with the shared embeddings. See <code>pytorch_widedeep.models.embeddings_layers.SharedEmbeddings</code> If 'None' is passed, it will default to 'False'.</p> <code>None</code> <code>frac_shared_embed</code> <code>Optional[float]</code> <p>The fraction of embeddings that will be shared (if <code>add_shared_embed = False</code>) by all the different categories for one particular column. If 'None' is passed, it will default to 0.0.</p> <code>None</code> <code>continuous_cols</code> <code>Optional[List[str]]</code> <p>List with the name of the numeric (aka continuous) columns</p> <code>None</code> <code>cont_norm_layer</code> <code>Optional[Literal[batchnorm, layernorm]]</code> <p>Type of normalization layer applied to the continuous features. Options are: 'layernorm' and 'batchnorm'. if <code>None</code>, no normalization layer will be used.</p> <code>None</code> <code>embed_continuous_method</code> <code>Optional[Literal[standard, piecewise, periodic]]</code> <p>Method to use to embed the continuous features. Options are: 'standard', 'periodic' or 'piecewise'. The 'standard' embedding method is based on the FT-Transformer implementation presented in the paper: Revisiting Deep Learning Models for Tabular Data. The 'periodic' and_'piecewise'_ methods were presented in the paper: On Embeddings for Numerical Features in Tabular Deep Learning. Please, read the papers for details.</p> <code>'standard'</code> <code>cont_embed_dropout</code> <code>Optional[float]</code> <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p> <code>None</code> <code>cont_embed_activation</code> <code>Optional[str]</code> <p>Activation function for the continuous embeddings if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported. If <code>None</code>, no activation function will be applied.</p> <code>None</code> <code>quantization_setup</code> <code>Optional[Dict[str, List[float]]]</code> <p>This parameter is used when the 'piecewise' method is used to embed the continuous cols. It is a dict where keys are the name of the continuous columns and values are lists with the boundaries for the quantization of the continuous_cols. See the examples for details. If If the 'piecewise' method is used, this parameter is required.</p> <code>None</code> <code>n_frequencies</code> <code>Optional[int]</code> <p>This is the so called 'k' in their paper On Embeddings for Numerical Features in Tabular Deep Learning, and is the number of 'frequencies' that will be used to represent each continuous column. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>sigma</code> <code>Optional[float]</code> <p>This is the sigma parameter in the paper mentioned when describing the previous parameters and it is used to initialise the 'frequency weights'. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>share_last_layer</code> <code>Optional[bool]</code> <p>This parameter is not present in the before mentioned paper but it is implemented in the official repo. If <code>True</code> the linear layer that turns the frequencies into embeddings will be shared across the continuous columns. If <code>False</code> a different linear layer will be used for each continuous column. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>full_embed_dropout</code> <code>Optional[bool]</code> <p>If <code>True</code>, the full embedding corresponding to a column will be masked out/dropout. If <code>None</code>, it will default to <code>False</code>.</p> <code>None</code> <code>input_dim</code> <code>int</code> <p>The so-called dimension of the model. Is the number of embeddings used to encode the categorical and/or continuous columns</p> <code>32</code> <code>n_heads</code> <code>int</code> <p>Number of attention heads per Transformer block</p> <code>8</code> <code>use_qkv_bias</code> <code>bool</code> <p>Boolean indicating whether or not to use bias in the Q, K, and V projection layers</p> <code>False</code> <code>n_blocks</code> <code>int</code> <p>Number of SAINT-Transformer blocks.</p> <code>2</code> <code>attn_dropout</code> <code>float</code> <p>Dropout that will be applied to the Multi-Head Attention column and row layers</p> <code>0.1</code> <code>ff_dropout</code> <code>float</code> <p>Dropout that will be applied to the FeedForward network</p> <code>0.2</code> <code>ff_factor</code> <code>int</code> <p>Multiplicative factor applied to the first layer of the FF network in each Transformer block, This is normally set to 4.</p> <code>4</code> <code>transformer_activation</code> <code>str</code> <p>Transformer Encoder activation function. 'tanh', 'relu', 'leaky_relu', 'gelu', 'geglu' and 'reglu' are supported</p> <code>'gelu'</code> <code>mlp_hidden_dims</code> <code>Optional[List[int]]</code> <p>List with the number of neurons per dense layer in the MLP. e.g: [64, 32]. If not provided no MLP on top of the final Transformer block will be used.</p> <code>None</code> <code>mlp_activation</code> <code>Optional[str]</code> <p>Activation function for the dense layers of the MLP. Currently 'tanh', 'relu', 'leaky'_relu' and _'gelu' are supported. If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to 'relu'.</p> <code>None</code> <code>mlp_dropout</code> <code>Optional[float]</code> <p>float with the dropout between the dense layers of the MLP. If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to 0.0.</p> <code>None</code> <code>mlp_batchnorm</code> <code>Optional[bool]</code> <p>Boolean indicating whether or not batch normalization will be applied to the dense layers If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to False.</p> <code>None</code> <code>mlp_batchnorm_last</code> <code>Optional[bool]</code> <p>Boolean indicating whether or not batch normalization will be applied to the last of the dense layers If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to False.</p> <code>None</code> <code>mlp_linear_first</code> <code>Optional[bool]</code> <p>Boolean indicating the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code> If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to <code>True</code>.</p> <code>None</code> <p>Attributes:</p> Name Type Description <code>encoder</code> <code>Module</code> <p>Sequence of SAINT-Transformer blocks</p> <code>mlp</code> <code>Module</code> <p>MLP component in the model</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import SAINT\n&gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n&gt;&gt;&gt; colnames = ['a', 'b', 'c', 'd', 'e']\n&gt;&gt;&gt; cat_embed_input = [(u,i) for u,i in zip(colnames[:4], [4]*4)]\n&gt;&gt;&gt; continuous_cols = ['e']\n&gt;&gt;&gt; column_idx = {k:v for v,k in enumerate(colnames)}\n&gt;&gt;&gt; model = SAINT(column_idx=column_idx, cat_embed_input=cat_embed_input, continuous_cols=continuous_cols)\n&gt;&gt;&gt; out = model(X_tab)\n</code></pre> Source code in <code>pytorch_widedeep/models/tabular/transformers/saint.py</code> <pre><code>class SAINT(BaseTabularModelWithAttention):\n    r\"\"\"Defines a [SAINT model](https://arxiv.org/abs/2106.01342) that\n    can be used as the `deeptabular` component of a Wide &amp; Deep model or\n    independently by itself.\n\n    Most of the parameters for this class are `Optional` since the use of\n    categorical or continuous is in fact optional (i.e. one can use\n    categorical features only, continuous features only or both).\n\n    :information_source: **NOTE**: This is an slightly modified and enhanced\n     version of the model described in the paper,\n\n    Parameters\n    ----------\n    column_idx: Dict\n        Dict containing the index of the columns that will be passed through\n        the `TabMlp` model. Required to slice the tensors. e.g. _{'education':\n        0, 'relationship': 1, 'workclass': 2, ...}_.\n    cat_embed_input: List, Optional, default = None\n        List of Tuples with the column name and number of unique values and\n        embedding dimension. e.g. _[(education, 11), ...]_\n    cat_embed_dropout: float, Optional, default = None\n        Categorical embeddings dropout. If `None`, it will default\n        to 0.\n    use_cat_bias: bool, Optional, default = None,\n        Boolean indicating if bias will be used for the categorical embeddings.\n        If `None`, it will default to 'False'.\n    cat_embed_activation: Optional, str, default = None,\n        Activation function for the categorical embeddings, if any. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported\n    shared_embed: bool, Optional, default = None\n        Boolean indicating if the embeddings will be \"shared\". The idea behind `shared_embed` is\n        described in the Appendix A in the [TabTransformer paper](https://arxiv.org/abs/2012.06678):\n        _'The goal of having column embedding is to enable the model to\n        distinguish the classes in one column from those in the other\n        columns'_. In other words, the idea is to let the model learn which\n        column is embedded at the time. See: `pytorch_widedeep.models.transformers._layers.SharedEmbeddings`.\n    add_shared_embed: bool, Optional, default = None\n        The two embedding sharing strategies are: 1) add the shared embeddings\n        to the column embeddings or 2) to replace the first\n        `frac_shared_embed` with the shared embeddings.\n        See `pytorch_widedeep.models.embeddings_layers.SharedEmbeddings`\n        If 'None' is passed, it will default to 'False'.\n    frac_shared_embed: float, Optional, default = None\n        The fraction of embeddings that will be shared (if `add_shared_embed\n        = False`) by all the different categories for one particular\n        column. If 'None' is passed, it will default to 0.0.\n    continuous_cols: List, Optional, default = None\n        List with the name of the numeric (aka continuous) columns\n    cont_norm_layer: str, Optional, default =  None\n        Type of normalization layer applied to the continuous features.\n        Options are: _'layernorm'_ and _'batchnorm'_. if `None`, no\n        normalization layer will be used.\n    embed_continuous_method: Optional, str, default = None,\n        Method to use to embed the continuous features. Options are:\n        _'standard'_, _'periodic'_ or _'piecewise'_. The _'standard'_\n        embedding method is based on the FT-Transformer implementation\n        presented in the paper: [Revisiting Deep Learning Models for\n        Tabular Data](https://arxiv.org/abs/2106.11959v5). The _'periodic'_\n        and_'piecewise'_ methods were presented in the paper: [On Embeddings for\n        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556).\n        Please, read the papers for details.\n    cont_embed_dropout: float, Optional, default = None,\n        Dropout for the continuous embeddings. If `None`, it will default to 0.0\n    cont_embed_activation: Optional, str, default = None,\n        Activation function for the continuous embeddings if any. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported.\n        If `None`, no activation function will be applied.\n    quantization_setup: Dict[str, List[float]], Optional, default = None,\n        This parameter is used when the _'piecewise'_ method is used to embed\n        the continuous cols. It is a dict where keys are the name of the continuous\n        columns and values are lists with the boundaries for the quantization\n        of the continuous_cols. See the examples for details. If\n        If the _'piecewise'_ method is used, this parameter is required.\n    n_frequencies: int, Optional, default = None,\n        This is the so called _'k'_ in their paper [On Embeddings for\n        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556),\n        and is the number of 'frequencies' that will be used to represent each\n        continuous column. See their Eq 2 in the paper for details. If\n        the _'periodic'_ method is used, this parameter is required.\n    sigma: float, Optional, default = None,\n        This is the sigma parameter in the paper mentioned when describing the\n        previous parameters and it is used to initialise the 'frequency\n        weights'. See their Eq 2 in the paper for details. If\n        the _'periodic'_ method is used, this parameter is required.\n    share_last_layer: bool, Optional, default = None,\n        This parameter is not present in the before mentioned paper but it is implemented in\n        the [official repo](https://github.com/yandex-research/rtdl-num-embeddings/tree/main).\n        If `True` the linear layer that turns the frequencies into embeddings\n        will be shared across the continuous columns. If `False` a different\n        linear layer will be used for each continuous column.\n        If the _'periodic'_ method is used, this parameter is required.\n    full_embed_dropout: bool, Optional, default = None,\n        If `True`, the full embedding corresponding to a column will be masked\n        out/dropout. If `None`, it will default to `False`.\n    input_dim: int, default = 32\n        The so-called *dimension of the model*. Is the number of\n        embeddings used to encode the categorical and/or continuous columns\n    n_heads: int, default = 8\n        Number of attention heads per Transformer block\n    use_qkv_bias: bool, default = False\n        Boolean indicating whether or not to use bias in the Q, K, and V\n        projection layers\n    n_blocks: int, default = 2\n        Number of SAINT-Transformer blocks.\n    attn_dropout: float, default = 0.2\n        Dropout that will be applied to the Multi-Head Attention column and\n        row layers\n    ff_dropout: float, default = 0.1\n        Dropout that will be applied to the FeedForward network\n    ff_factor: float, default = 4\n        Multiplicative factor applied to the first layer of the FF network in\n        each Transformer block, This is normally set to 4.\n    transformer_activation: str, default = \"gelu\"\n        Transformer Encoder activation function. _'tanh'_, _'relu'_,\n        _'leaky_relu'_, _'gelu'_, _'geglu'_ and _'reglu'_ are supported\n    mlp_hidden_dims: List, Optional, default = None\n        List with the number of neurons per dense layer in the MLP. e.g:\n        _[64, 32]_. If not provided no MLP on top of the final\n        Transformer block will be used.\n    mlp_activation: str, Optional, default = None\n        Activation function for the dense layers of the MLP. Currently\n        _'tanh'_, _'relu'_, _'leaky'_relu' and _'gelu'_ are supported.\n        If 'mlp_hidden_dims' is not `None` and this parameter is `None`, it\n        will default to _'relu'_.\n    mlp_dropout: float, Optional, default = None\n        float with the dropout between the dense layers of the MLP.\n        If 'mlp_hidden_dims' is not `None` and this parameter is `None`, it\n        will default to 0.0.\n    mlp_batchnorm: bool, Optional, default = None\n        Boolean indicating whether or not batch normalization will be applied\n        to the dense layers\n        If 'mlp_hidden_dims' is not `None` and this parameter is `None`, it\n        will default to False.\n    mlp_batchnorm_last: bool, Optional, default = None\n        Boolean indicating whether or not batch normalization will be applied\n        to the last of the dense layers\n        If 'mlp_hidden_dims' is not `None` and this parameter is `None`, it\n        will default to False.\n    mlp_linear_first: bool, Optional, default = None\n        Boolean indicating the order of the operations in the dense\n        layer. If `True: [LIN -&gt; ACT -&gt; BN -&gt; DP]`. If `False: [BN -&gt; DP -&gt;\n        LIN -&gt; ACT]`\n        If 'mlp_hidden_dims' is not `None` and this parameter is `None`, it\n        will default to `True`.\n\n    Attributes\n    ----------\n    encoder: nn.Module\n        Sequence of SAINT-Transformer blocks\n    mlp: nn.Module\n        MLP component in the model\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.models import SAINT\n    &gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n    &gt;&gt;&gt; colnames = ['a', 'b', 'c', 'd', 'e']\n    &gt;&gt;&gt; cat_embed_input = [(u,i) for u,i in zip(colnames[:4], [4]*4)]\n    &gt;&gt;&gt; continuous_cols = ['e']\n    &gt;&gt;&gt; column_idx = {k:v for v,k in enumerate(colnames)}\n    &gt;&gt;&gt; model = SAINT(column_idx=column_idx, cat_embed_input=cat_embed_input, continuous_cols=continuous_cols)\n    &gt;&gt;&gt; out = model(X_tab)\n    \"\"\"\n\n    def __init__(\n        self,\n        column_idx: Dict[str, int],\n        *,\n        cat_embed_input: Optional[List[Tuple[str, int]]] = None,\n        cat_embed_dropout: Optional[float] = None,\n        use_cat_bias: Optional[bool] = None,\n        cat_embed_activation: Optional[str] = None,\n        shared_embed: Optional[bool] = None,\n        add_shared_embed: Optional[bool] = None,\n        frac_shared_embed: Optional[float] = None,\n        continuous_cols: Optional[List[str]] = None,\n        cont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\n        embed_continuous_method: Optional[\n            Literal[\"standard\", \"piecewise\", \"periodic\"]\n        ] = \"standard\",\n        cont_embed_dropout: Optional[float] = None,\n        cont_embed_activation: Optional[str] = None,\n        quantization_setup: Optional[Dict[str, List[float]]] = None,\n        n_frequencies: Optional[int] = None,\n        sigma: Optional[float] = None,\n        share_last_layer: Optional[bool] = None,\n        full_embed_dropout: Optional[bool] = None,\n        input_dim: int = 32,\n        use_qkv_bias: bool = False,\n        n_heads: int = 8,\n        n_blocks: int = 2,\n        attn_dropout: float = 0.1,\n        ff_dropout: float = 0.2,\n        ff_factor: int = 4,\n        transformer_activation: str = \"gelu\",\n        mlp_hidden_dims: Optional[List[int]] = None,\n        mlp_activation: Optional[str] = None,\n        mlp_dropout: Optional[float] = None,\n        mlp_batchnorm: Optional[bool] = None,\n        mlp_batchnorm_last: Optional[bool] = None,\n        mlp_linear_first: Optional[bool] = None,\n    ):\n        super(SAINT, self).__init__(\n            column_idx=column_idx,\n            cat_embed_input=cat_embed_input,\n            cat_embed_dropout=cat_embed_dropout,\n            use_cat_bias=use_cat_bias,\n            cat_embed_activation=cat_embed_activation,\n            shared_embed=shared_embed,\n            add_shared_embed=add_shared_embed,\n            frac_shared_embed=frac_shared_embed,\n            continuous_cols=continuous_cols,\n            cont_norm_layer=cont_norm_layer,\n            embed_continuous_method=embed_continuous_method,\n            cont_embed_dropout=cont_embed_dropout,\n            cont_embed_activation=cont_embed_activation,\n            input_dim=input_dim,\n            quantization_setup=quantization_setup,\n            n_frequencies=n_frequencies,\n            sigma=sigma,\n            share_last_layer=share_last_layer,\n            full_embed_dropout=full_embed_dropout,\n        )\n\n        self.use_qkv_bias = use_qkv_bias\n        self.n_heads = n_heads\n        self.n_blocks = n_blocks\n        self.attn_dropout = attn_dropout\n        self.ff_dropout = ff_dropout\n        self.ff_factor = ff_factor\n        self.transformer_activation = transformer_activation\n\n        self.mlp_hidden_dims = mlp_hidden_dims\n        self.mlp_activation = mlp_activation\n        self.mlp_dropout = mlp_dropout\n        self.mlp_batchnorm = mlp_batchnorm\n        self.mlp_batchnorm_last = mlp_batchnorm_last\n        self.mlp_linear_first = mlp_linear_first\n\n        self.with_cls_token = \"cls_token\" in column_idx\n        self.n_cat = len(cat_embed_input) if cat_embed_input is not None else 0\n        self.n_cont = len(continuous_cols) if continuous_cols is not None else 0\n        self.n_feats = self.n_cat + self.n_cont\n\n        # Embeddings are instantiated at the base model\n        # Transformer blocks\n        self.encoder = nn.Sequential()\n        for i in range(n_blocks):\n            self.encoder.add_module(\n                \"saint_block\" + str(i),\n                SaintEncoder(\n                    input_dim,\n                    n_heads,\n                    use_qkv_bias,\n                    attn_dropout,\n                    ff_dropout,\n                    ff_factor,\n                    transformer_activation,\n                    self.n_feats,\n                ),\n            )\n\n        self.mlp_first_hidden_dim = (\n            self.input_dim if self.with_cls_token else (self.n_feats * self.input_dim)\n        )\n\n        # Mlp: adding an MLP on top of the Resnet blocks is optional and\n        # therefore all related params are optional\n        if self.mlp_hidden_dims is not None:\n            self.mlp = MLP(\n                d_hidden=[self.mlp_first_hidden_dim] + self.mlp_hidden_dims,\n                activation=(\n                    \"relu\" if self.mlp_activation is None else self.mlp_activation\n                ),\n                dropout=0.0 if self.mlp_dropout is None else self.mlp_dropout,\n                batchnorm=False if self.mlp_batchnorm is None else self.mlp_batchnorm,\n                batchnorm_last=(\n                    False\n                    if self.mlp_batchnorm_last is None\n                    else self.mlp_batchnorm_last\n                ),\n                linear_first=(\n                    False if self.mlp_linear_first is None else self.mlp_linear_first\n                ),\n            )\n        else:\n            self.mlp = None\n\n    def forward(self, X: Tensor) -&gt; Tensor:\n        x = self._get_embeddings(X)\n        x = self.encoder(x)\n        if self.with_cls_token:\n            x = x[:, 0, :]\n        else:\n            x = x.flatten(1)\n        if self.mlp is not None:\n            x = self.mlp(x)\n        return x\n\n    @property\n    def output_dim(self) -&gt; int:\n        r\"\"\"The output dimension of the model. This is a required property\n        neccesary to build the `WideDeep` class\n        \"\"\"\n        return (\n            self.mlp_hidden_dims[-1]\n            if self.mlp_hidden_dims is not None\n            else self.mlp_first_hidden_dim\n        )\n\n    @property\n    def attention_weights(self) -&gt; List[Tuple[Tensor, Tensor]]:\n        r\"\"\"List with the attention weights. Each element of the list is a tuple\n        where the first and the second elements are the column and row\n        attention weights respectively\n\n        The shape of the attention weights is:\n\n        - column attention: $(N, H, F, F)$\n\n        - row attention: $(1, H, N, N)$\n\n        where $N$ is the batch size, $H$ is the number of heads and $F$ is the\n        number of features/columns in the dataset\n        \"\"\"\n        attention_weights = []\n        for blk in self.encoder:\n            attention_weights.append(\n                (blk.col_attn.attn_weights, blk.row_attn.attn_weights)\n            )\n        return attention_weights\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.transformers.saint.SAINT.output_dim","title":"output_dim  <code>property</code>","text":"<pre><code>output_dim\n</code></pre> <p>The output dimension of the model. This is a required property neccesary to build the <code>WideDeep</code> class</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.transformers.saint.SAINT.attention_weights","title":"attention_weights  <code>property</code>","text":"<pre><code>attention_weights\n</code></pre> <p>List with the attention weights. Each element of the list is a tuple where the first and the second elements are the column and row attention weights respectively</p> <p>The shape of the attention weights is:</p> <ul> <li> <p>column attention: \\((N, H, F, F)\\)</p> </li> <li> <p>row attention: \\((1, H, N, N)\\)</p> </li> </ul> <p>where \\(N\\) is the batch size, \\(H\\) is the number of heads and \\(F\\) is the number of features/columns in the dataset</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.transformers.ft_transformer.FTTransformer","title":"FTTransformer","text":"<p>               Bases: <code>BaseTabularModelWithAttention</code></p> <p>Defines a FTTransformer model that can be used as the <code>deeptabular</code> component of a Wide &amp; Deep model or independently by itself.</p> <p>Most of the parameters for this class are <code>Optional</code> since the use of categorical or continuous is in fact optional (i.e. one can use categorical features only, continuous features only or both).</p> <p>Parameters:</p> Name Type Description Default <code>column_idx</code> <code>Dict[str, int]</code> <p>Dict containing the index of the columns that will be passed through the <code>TabMlp</code> model. Required to slice the tensors. e.g. {'education': 0, 'relationship': 1, 'workclass': 2, ...}.</p> required <code>cat_embed_input</code> <code>Optional[List[Tuple[str, int]]]</code> <p>List of Tuples with the column name and number of unique values and embedding dimension. e.g. [(education, 11), ...]</p> <code>None</code> <code>cat_embed_dropout</code> <code>Optional[float]</code> <p>Categorical embeddings dropout. If <code>None</code>, it will default to 0.</p> <code>None</code> <code>use_cat_bias</code> <code>Optional[bool]</code> <p>Boolean indicating if bias will be used for the categorical embeddings. If <code>None</code>, it will default to 'False'.</p> <code>None</code> <code>cat_embed_activation</code> <code>Optional[str]</code> <p>Activation function for the categorical embeddings, if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> <code>None</code> <code>shared_embed</code> <code>Optional[bool]</code> <p>Boolean indicating if the embeddings will be \"shared\". The idea behind <code>shared_embed</code> is described in the Appendix A in the TabTransformer paper: 'The goal of having column embedding is to enable the model to distinguish the classes in one column from those in the other columns'. In other words, the idea is to let the model learn which column is embedded at the time. See: <code>pytorch_widedeep.models.transformers._layers.SharedEmbeddings</code>.</p> <code>None</code> <code>add_shared_embed</code> <code>Optional[bool]</code> <p>The two embedding sharing strategies are: 1) add the shared embeddings to the column embeddings or 2) to replace the first <code>frac_shared_embed</code> with the shared embeddings. See <code>pytorch_widedeep.models.embeddings_layers.SharedEmbeddings</code> If 'None' is passed, it will default to 'False'.</p> <code>None</code> <code>frac_shared_embed</code> <code>Optional[float]</code> <p>The fraction of embeddings that will be shared (if <code>add_shared_embed = False</code>) by all the different categories for one particular column. If 'None' is passed, it will default to 0.0.</p> <code>None</code> <code>continuous_cols</code> <code>Optional[List[str]]</code> <p>List with the name of the numeric (aka continuous) columns</p> <code>None</code> <code>cont_norm_layer</code> <code>Optional[Literal[batchnorm, layernorm]]</code> <p>Type of normalization layer applied to the continuous features. Options are: 'layernorm' and 'batchnorm'. if <code>None</code>, no normalization layer will be used.</p> <code>None</code> <code>embed_continuous_method</code> <code>Optional[Literal[standard, piecewise, periodic]]</code> <p>Method to use to embed the continuous features. Options are: 'standard', 'periodic' or 'piecewise'. The 'standard' embedding method is based on the FT-Transformer implementation presented in the paper: Revisiting Deep Learning Models for Tabular Data. The 'periodic' and_'piecewise'_ methods were presented in the paper: On Embeddings for Numerical Features in Tabular Deep Learning. Please, read the papers for details.</p> <code>'standard'</code> <code>cont_embed_dropout</code> <code>Optional[float]</code> <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p> <code>None</code> <code>cont_embed_activation</code> <code>Optional[str]</code> <p>Activation function for the continuous embeddings if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported. If <code>None</code>, no activation function will be applied.</p> <code>None</code> <code>quantization_setup</code> <code>Optional[Dict[str, List[float]]]</code> <p>This parameter is used when the 'piecewise' method is used to embed the continuous cols. It is a dict where keys are the name of the continuous columns and values are lists with the boundaries for the quantization of the continuous_cols. See the examples for details. If If the 'piecewise' method is used, this parameter is required.</p> <code>None</code> <code>n_frequencies</code> <code>Optional[int]</code> <p>This is the so called 'k' in their paper On Embeddings for Numerical Features in Tabular Deep Learning, and is the number of 'frequencies' that will be used to represent each continuous column. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>sigma</code> <code>Optional[float]</code> <p>This is the sigma parameter in the paper mentioned when describing the previous parameters and it is used to initialise the 'frequency weights'. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>share_last_layer</code> <code>Optional[bool]</code> <p>This parameter is not present in the before mentioned paper but it is implemented in the official repo. If <code>True</code> the linear layer that turns the frequencies into embeddings will be shared across the continuous columns. If <code>False</code> a different linear layer will be used for each continuous column. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>full_embed_dropout</code> <code>Optional[bool]</code> <p>If <code>True</code>, the full embedding corresponding to a column will be masked out/dropout. If <code>None</code>, it will default to <code>False</code>.</p> <code>None</code> <code>input_dim</code> <code>int</code> <p>The so-called dimension of the model. Is the number of embeddings used to encode the categorical and/or continuous columns.</p> <code>64</code> <code>kv_compression_factor</code> <code>float</code> <p>By default, the FTTransformer uses Linear Attention (See Linformer: Self-Attention with Linear Complexity ). The compression factor that will be used to reduce the input sequence length. If we denote the resulting sequence length as \\(k = int(kv_{compression \\space factor} \\times s)\\) where \\(s\\) is the input sequence length.</p> <code>0.5</code> <code>kv_sharing</code> <code>bool</code> <p>Boolean indicating if the \\(E\\) and \\(F\\) projection matrices will share weights.  See Linformer: Self-Attention with Linear Complexity for details</p> <code>False</code> <code>n_heads</code> <code>int</code> <p>Number of attention heads per FTTransformer block</p> <code>8</code> <code>use_qkv_bias</code> <code>bool</code> <p>Boolean indicating whether or not to use bias in the Q, K, and V projection layers</p> <code>False</code> <code>n_blocks</code> <code>int</code> <p>Number of FTTransformer blocks</p> <code>4</code> <code>attn_dropout</code> <code>float</code> <p>Dropout that will be applied to the Linear-Attention layers</p> <code>0.2</code> <code>ff_dropout</code> <code>float</code> <p>Dropout that will be applied to the FeedForward network</p> <code>0.1</code> <code>ff_factor</code> <code>float</code> <p>Multiplicative factor applied to the first layer of the FF network in each Transformer block, This is normally set to 4, but they use 4/3 in the paper.</p> <code>1.33</code> <code>transformer_activation</code> <code>str</code> <p>Transformer Encoder activation function. 'tanh', 'relu', 'leaky_relu', 'gelu', 'geglu' and 'reglu' are supported</p> <code>'reglu'</code> <code>mlp_hidden_dims</code> <code>Optional[List[int]]</code> <p>List with the number of neurons per dense layer in the MLP. e.g: [64, 32]. If not provided no MLP on top of the final FTTransformer block will be used.</p> <code>None</code> <code>mlp_activation</code> <code>Optional[str]</code> <p>Activation function for the dense layers of the MLP. Currently 'tanh', 'relu', 'leaky'_relu' and _'gelu' are supported. If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to 'relu'.</p> <code>None</code> <code>mlp_dropout</code> <code>Optional[float]</code> <p>float with the dropout between the dense layers of the MLP. If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to 0.0.</p> <code>None</code> <code>mlp_batchnorm</code> <code>Optional[bool]</code> <p>Boolean indicating whether or not batch normalization will be applied to the dense layers If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to False.</p> <code>None</code> <code>mlp_batchnorm_last</code> <code>Optional[bool]</code> <p>Boolean indicating whether or not batch normalization will be applied to the last of the dense layers If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to False.</p> <code>None</code> <code>mlp_linear_first</code> <code>Optional[bool]</code> <p>Boolean indicating the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code> If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to <code>True</code>.</p> <code>None</code> <p>Attributes:</p> Name Type Description <code>encoder</code> <code>Module</code> <p>Sequence of FTTransformer blocks</p> <code>mlp</code> <code>Module</code> <p>MLP component in the model</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import FTTransformer\n&gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n&gt;&gt;&gt; colnames = ['a', 'b', 'c', 'd', 'e']\n&gt;&gt;&gt; cat_embed_input = [(u,i) for u,i in zip(colnames[:4], [4]*4)]\n&gt;&gt;&gt; continuous_cols = ['e']\n&gt;&gt;&gt; column_idx = {k:v for v,k in enumerate(colnames)}\n&gt;&gt;&gt; model = FTTransformer(column_idx=column_idx, cat_embed_input=cat_embed_input, continuous_cols=continuous_cols)\n&gt;&gt;&gt; out = model(X_tab)\n</code></pre> Source code in <code>pytorch_widedeep/models/tabular/transformers/ft_transformer.py</code> <pre><code>class FTTransformer(BaseTabularModelWithAttention):\n    r\"\"\"Defines a [FTTransformer model](https://arxiv.org/abs/2106.11959) that\n    can be used as the `deeptabular` component of a Wide &amp; Deep model or\n    independently by itself.\n\n    Most of the parameters for this class are `Optional` since the use of\n    categorical or continuous is in fact optional (i.e. one can use\n    categorical features only, continuous features only or both).\n\n    Parameters\n    ----------\n    column_idx: Dict\n        Dict containing the index of the columns that will be passed through\n        the `TabMlp` model. Required to slice the tensors. e.g. _{'education':\n        0, 'relationship': 1, 'workclass': 2, ...}_.\n    cat_embed_input: List, Optional, default = None\n        List of Tuples with the column name and number of unique values and\n        embedding dimension. e.g. _[(education, 11), ...]_\n    cat_embed_dropout: float, Optional, default = None\n        Categorical embeddings dropout. If `None`, it will default\n        to 0.\n    use_cat_bias: bool, Optional, default = None,\n        Boolean indicating if bias will be used for the categorical embeddings.\n        If `None`, it will default to 'False'.\n    cat_embed_activation: Optional, str, default = None,\n        Activation function for the categorical embeddings, if any. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported\n    shared_embed: bool, Optional, default = None\n        Boolean indicating if the embeddings will be \"shared\". The idea behind `shared_embed` is\n        described in the Appendix A in the [TabTransformer paper](https://arxiv.org/abs/2012.06678):\n        _'The goal of having column embedding is to enable the model to\n        distinguish the classes in one column from those in the other\n        columns'_. In other words, the idea is to let the model learn which\n        column is embedded at the time. See: `pytorch_widedeep.models.transformers._layers.SharedEmbeddings`.\n    add_shared_embed: bool, Optional, default = None\n        The two embedding sharing strategies are: 1) add the shared embeddings\n        to the column embeddings or 2) to replace the first\n        `frac_shared_embed` with the shared embeddings.\n        See `pytorch_widedeep.models.embeddings_layers.SharedEmbeddings`\n        If 'None' is passed, it will default to 'False'.\n    frac_shared_embed: float, Optional, default = None\n        The fraction of embeddings that will be shared (if `add_shared_embed\n        = False`) by all the different categories for one particular\n        column. If 'None' is passed, it will default to 0.0.\n    continuous_cols: List, Optional, default = None\n        List with the name of the numeric (aka continuous) columns\n    cont_norm_layer: str, Optional, default =  None\n        Type of normalization layer applied to the continuous features.\n        Options are: _'layernorm'_ and _'batchnorm'_. if `None`, no\n        normalization layer will be used.\n    embed_continuous_method: Optional, str, default = None,\n        Method to use to embed the continuous features. Options are:\n        _'standard'_, _'periodic'_ or _'piecewise'_. The _'standard'_\n        embedding method is based on the FT-Transformer implementation\n        presented in the paper: [Revisiting Deep Learning Models for\n        Tabular Data](https://arxiv.org/abs/2106.11959v5). The _'periodic'_\n        and_'piecewise'_ methods were presented in the paper: [On Embeddings for\n        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556).\n        Please, read the papers for details.\n    cont_embed_dropout: float, Optional, default = None,\n        Dropout for the continuous embeddings. If `None`, it will default to 0.0\n    cont_embed_activation: Optional, str, default = None,\n        Activation function for the continuous embeddings if any. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported.\n        If `None`, no activation function will be applied.\n    quantization_setup: Dict[str, List[float]], Optional, default = None,\n        This parameter is used when the _'piecewise'_ method is used to embed\n        the continuous cols. It is a dict where keys are the name of the continuous\n        columns and values are lists with the boundaries for the quantization\n        of the continuous_cols. See the examples for details. If\n        If the _'piecewise'_ method is used, this parameter is required.\n    n_frequencies: int, Optional, default = None,\n        This is the so called _'k'_ in their paper [On Embeddings for\n        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556),\n        and is the number of 'frequencies' that will be used to represent each\n        continuous column. See their Eq 2 in the paper for details. If\n        the _'periodic'_ method is used, this parameter is required.\n    sigma: float, Optional, default = None,\n        This is the sigma parameter in the paper mentioned when describing the\n        previous parameters and it is used to initialise the 'frequency\n        weights'. See their Eq 2 in the paper for details. If\n        the _'periodic'_ method is used, this parameter is required.\n    share_last_layer: bool, Optional, default = None,\n        This parameter is not present in the before mentioned paper but it is implemented in\n        the [official repo](https://github.com/yandex-research/rtdl-num-embeddings/tree/main).\n        If `True` the linear layer that turns the frequencies into embeddings\n        will be shared across the continuous columns. If `False` a different\n        linear layer will be used for each continuous column.\n        If the _'periodic'_ method is used, this parameter is required.\n    full_embed_dropout: bool, Optional, default = None,\n        If `True`, the full embedding corresponding to a column will be masked\n        out/dropout. If `None`, it will default to `False`.\n    input_dim: int, default = 64\n        The so-called *dimension of the model*. Is the number of embeddings used to encode\n        the categorical and/or continuous columns.\n    kv_compression_factor: int, default = 0.5\n        By default, the FTTransformer uses Linear Attention\n        (See [Linformer: Self-Attention with Linear Complexity](https://arxiv.org/abs/2006.04768&gt;) ).\n        The compression factor that will be used to reduce the input sequence\n        length. If we denote the resulting sequence length as\n        $k = int(kv_{compression \\space factor} \\times s)$\n        where $s$ is the input sequence length.\n    kv_sharing: bool, default = False\n        Boolean indicating if the $E$ and $F$ projection matrices\n        will share weights.  See [Linformer: Self-Attention with Linear\n        Complexity](https://arxiv.org/abs/2006.04768) for details\n    n_heads: int, default = 8\n        Number of attention heads per FTTransformer block\n    use_qkv_bias: bool, default = False\n        Boolean indicating whether or not to use bias in the Q, K, and V\n        projection layers\n    n_blocks: int, default = 4\n        Number of FTTransformer blocks\n    attn_dropout: float, default = 0.2\n        Dropout that will be applied to the Linear-Attention layers\n    ff_dropout: float, default = 0.1\n        Dropout that will be applied to the FeedForward network\n    ff_factor: float, default = 4 / 3\n        Multiplicative factor applied to the first layer of the FF network in\n        each Transformer block, This is normally set to 4, but they use 4/3\n        in the paper.\n    transformer_activation: str, default = \"gelu\"\n        Transformer Encoder activation function. _'tanh'_, _'relu'_,\n        _'leaky_relu'_, _'gelu'_, _'geglu'_ and _'reglu'_ are supported\n    mlp_hidden_dims: List, Optional, default = None\n        List with the number of neurons per dense layer in the MLP. e.g:\n        _[64, 32]_. If not provided no MLP on top of the final\n        FTTransformer block will be used.\n    mlp_activation: str, Optional, default = None\n        Activation function for the dense layers of the MLP. Currently\n        _'tanh'_, _'relu'_, _'leaky'_relu' and _'gelu'_ are supported.\n        If 'mlp_hidden_dims' is not `None` and this parameter is `None`, it\n        will default to _'relu'_.\n    mlp_dropout: float, Optional, default = None\n        float with the dropout between the dense layers of the MLP.\n        If 'mlp_hidden_dims' is not `None` and this parameter is `None`, it\n        will default to 0.0.\n    mlp_batchnorm: bool, Optional, default = None\n        Boolean indicating whether or not batch normalization will be applied\n        to the dense layers\n        If 'mlp_hidden_dims' is not `None` and this parameter is `None`, it\n        will default to False.\n    mlp_batchnorm_last: bool, Optional, default = None\n        Boolean indicating whether or not batch normalization will be applied\n        to the last of the dense layers\n        If 'mlp_hidden_dims' is not `None` and this parameter is `None`, it\n        will default to False.\n    mlp_linear_first: bool, Optional, default = None\n        Boolean indicating the order of the operations in the dense\n        layer. If `True: [LIN -&gt; ACT -&gt; BN -&gt; DP]`. If `False: [BN -&gt; DP -&gt;\n        LIN -&gt; ACT]`\n        If 'mlp_hidden_dims' is not `None` and this parameter is `None`, it\n        will default to `True`.\n\n    Attributes\n    ----------\n    encoder: nn.Module\n        Sequence of FTTransformer blocks\n    mlp: nn.Module\n        MLP component in the model\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.models import FTTransformer\n    &gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n    &gt;&gt;&gt; colnames = ['a', 'b', 'c', 'd', 'e']\n    &gt;&gt;&gt; cat_embed_input = [(u,i) for u,i in zip(colnames[:4], [4]*4)]\n    &gt;&gt;&gt; continuous_cols = ['e']\n    &gt;&gt;&gt; column_idx = {k:v for v,k in enumerate(colnames)}\n    &gt;&gt;&gt; model = FTTransformer(column_idx=column_idx, cat_embed_input=cat_embed_input, continuous_cols=continuous_cols)\n    &gt;&gt;&gt; out = model(X_tab)\n    \"\"\"\n\n    def __init__(\n        self,\n        column_idx: Dict[str, int],\n        *,\n        cat_embed_input: Optional[List[Tuple[str, int]]] = None,\n        cat_embed_dropout: Optional[float] = None,\n        use_cat_bias: Optional[bool] = None,\n        cat_embed_activation: Optional[str] = None,\n        shared_embed: Optional[bool] = None,\n        add_shared_embed: Optional[bool] = None,\n        frac_shared_embed: Optional[float] = None,\n        continuous_cols: Optional[List[str]] = None,\n        cont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\n        embed_continuous_method: Optional[\n            Literal[\"standard\", \"piecewise\", \"periodic\"]\n        ] = \"standard\",\n        cont_embed_dropout: Optional[float] = None,\n        cont_embed_activation: Optional[str] = None,\n        quantization_setup: Optional[Dict[str, List[float]]] = None,\n        n_frequencies: Optional[int] = None,\n        sigma: Optional[float] = None,\n        share_last_layer: Optional[bool] = None,\n        full_embed_dropout: Optional[bool] = None,\n        input_dim: int = 64,\n        kv_compression_factor: float = 0.5,\n        kv_sharing: bool = False,\n        use_qkv_bias: bool = False,\n        n_heads: int = 8,\n        n_blocks: int = 4,\n        attn_dropout: float = 0.2,\n        ff_dropout: float = 0.1,\n        ff_factor: float = 1.33,\n        transformer_activation: str = \"reglu\",\n        mlp_hidden_dims: Optional[List[int]] = None,\n        mlp_activation: Optional[str] = None,\n        mlp_dropout: Optional[float] = None,\n        mlp_batchnorm: Optional[bool] = None,\n        mlp_batchnorm_last: Optional[bool] = None,\n        mlp_linear_first: Optional[bool] = None,\n    ):\n        super(FTTransformer, self).__init__(\n            column_idx=column_idx,\n            cat_embed_input=cat_embed_input,\n            cat_embed_dropout=cat_embed_dropout,\n            use_cat_bias=use_cat_bias,\n            cat_embed_activation=cat_embed_activation,\n            shared_embed=shared_embed,\n            add_shared_embed=add_shared_embed,\n            frac_shared_embed=frac_shared_embed,\n            continuous_cols=continuous_cols,\n            cont_norm_layer=cont_norm_layer,\n            embed_continuous_method=embed_continuous_method,\n            cont_embed_dropout=cont_embed_dropout,\n            cont_embed_activation=cont_embed_activation,\n            input_dim=input_dim,\n            quantization_setup=quantization_setup,\n            n_frequencies=n_frequencies,\n            sigma=sigma,\n            share_last_layer=share_last_layer,\n            full_embed_dropout=full_embed_dropout,\n        )\n\n        self.kv_compression_factor = kv_compression_factor\n        self.kv_sharing = kv_sharing\n        self.use_qkv_bias = use_qkv_bias\n        self.n_heads = n_heads\n        self.n_blocks = n_blocks\n        self.attn_dropout = attn_dropout\n        self.ff_dropout = ff_dropout\n        self.ff_factor = ff_factor\n        self.transformer_activation = transformer_activation\n\n        self.mlp_hidden_dims = mlp_hidden_dims\n        self.mlp_activation = mlp_activation\n        self.mlp_dropout = mlp_dropout\n        self.mlp_batchnorm = mlp_batchnorm\n        self.mlp_batchnorm_last = mlp_batchnorm_last\n        self.mlp_linear_first = mlp_linear_first\n\n        self.with_cls_token = \"cls_token\" in column_idx\n        self.n_cat = len(cat_embed_input) if cat_embed_input is not None else 0\n        self.n_cont = len(continuous_cols) if continuous_cols is not None else 0\n        self.n_feats = self.n_cat + self.n_cont\n\n        # Embeddings are instantiated at the base model\n        # Transformer blocks\n        is_first = True\n        self.encoder = nn.Sequential()\n        for i in range(n_blocks):\n            self.encoder.add_module(\n                \"fttransformer_block\" + str(i),\n                FTTransformerEncoder(\n                    input_dim,\n                    self.n_feats,\n                    n_heads,\n                    use_qkv_bias,\n                    attn_dropout,\n                    ff_dropout,\n                    ff_factor,\n                    kv_compression_factor,\n                    kv_sharing,\n                    transformer_activation,\n                    is_first,\n                ),\n            )\n            is_first = False\n\n        self.mlp_first_hidden_dim = (\n            self.input_dim if self.with_cls_token else (self.n_feats * self.input_dim)\n        )\n\n        # Mlp: adding an MLP on top of the Resnet blocks is optional and\n        # therefore all related params are optional\n        if self.mlp_hidden_dims is not None:\n            self.mlp = MLP(\n                d_hidden=[self.mlp_first_hidden_dim] + self.mlp_hidden_dims,\n                activation=(\n                    \"relu\" if self.mlp_activation is None else self.mlp_activation\n                ),\n                dropout=0.0 if self.mlp_dropout is None else self.mlp_dropout,\n                batchnorm=False if self.mlp_batchnorm is None else self.mlp_batchnorm,\n                batchnorm_last=(\n                    False\n                    if self.mlp_batchnorm_last is None\n                    else self.mlp_batchnorm_last\n                ),\n                linear_first=(\n                    False if self.mlp_linear_first is None else self.mlp_linear_first\n                ),\n            )\n        else:\n            self.mlp = None\n\n    def forward(self, X: Tensor) -&gt; Tensor:\n        x = self._get_embeddings(X)\n        x = self.encoder(x)\n        if self.with_cls_token:\n            x = x[:, 0, :]\n        else:\n            x = x.flatten(1)\n        if self.mlp is not None:\n            x = self.mlp(x)\n        return x\n\n    @property\n    def output_dim(self) -&gt; int:\n        r\"\"\"The output dimension of the model. This is a required property\n        neccesary to build the `WideDeep` class\n        \"\"\"\n        return (\n            self.mlp_hidden_dims[-1]\n            if self.mlp_hidden_dims is not None\n            else self.mlp_first_hidden_dim\n        )\n\n    @property\n    def attention_weights(self) -&gt; List[Tensor]:\n        r\"\"\"List with the attention weights per block\n\n        The shape of the attention weights is: $(N, H, F, k)$, where $N$ is\n        the batch size, $H$ is the number of attention heads, $F$ is the\n        number of features/columns and $k$ is the reduced sequence length or\n        dimension, i.e. $k = int(kv_{compression \\space factor} \\times s)$\n        \"\"\"\n        return [blk.attn.attn_weights for blk in self.encoder]\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.transformers.ft_transformer.FTTransformer.output_dim","title":"output_dim  <code>property</code>","text":"<pre><code>output_dim\n</code></pre> <p>The output dimension of the model. This is a required property neccesary to build the <code>WideDeep</code> class</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.transformers.ft_transformer.FTTransformer.attention_weights","title":"attention_weights  <code>property</code>","text":"<pre><code>attention_weights\n</code></pre> <p>List with the attention weights per block</p> <p>The shape of the attention weights is: \\((N, H, F, k)\\), where \\(N\\) is the batch size, \\(H\\) is the number of attention heads, \\(F\\) is the number of features/columns and \\(k\\) is the reduced sequence length or dimension, i.e. \\(k = int(kv_{compression \\space factor} \\times s)\\)</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.transformers.tab_perceiver.TabPerceiver","title":"TabPerceiver","text":"<p>               Bases: <code>BaseTabularModelWithAttention</code></p> <p>Defines an adaptation of a Perceiver  that can be used as the <code>deeptabular</code> component of a Wide &amp; Deep model  or independently by itself.</p> <p>Most of the parameters for this class are <code>Optional</code> since the use of categorical or continuous is in fact optional (i.e. one can use categorical features only, continuous features only or both).</p> <p> NOTE: while there are scientific publications for  the <code>TabTransformer</code>, <code>SAINT</code> and <code>FTTransformer</code>, the <code>TabPerceiver</code>  and the <code>TabFastFormer</code> are our own adaptations of the  Perceiver and the  FastFormer for tabular data.</p> <p>Parameters:</p> Name Type Description Default <code>column_idx</code> <code>Dict[str, int]</code> <p>Dict containing the index of the columns that will be passed through the <code>TabMlp</code> model. Required to slice the tensors. e.g. {'education': 0, 'relationship': 1, 'workclass': 2, ...}.</p> required <code>cat_embed_input</code> <code>Optional[List[Tuple[str, int]]]</code> <p>List of Tuples with the column name and number of unique values and embedding dimension. e.g. [(education, 11), ...]</p> <code>None</code> <code>cat_embed_dropout</code> <code>Optional[float]</code> <p>Categorical embeddings dropout. If <code>None</code>, it will default to 0.</p> <code>None</code> <code>use_cat_bias</code> <code>Optional[bool]</code> <p>Boolean indicating if bias will be used for the categorical embeddings. If <code>None</code>, it will default to 'False'.</p> <code>None</code> <code>cat_embed_activation</code> <code>Optional[str]</code> <p>Activation function for the categorical embeddings, if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> <code>None</code> <code>shared_embed</code> <code>Optional[bool]</code> <p>Boolean indicating if the embeddings will be \"shared\". The idea behind <code>shared_embed</code> is described in the Appendix A in the TabTransformer paper: 'The goal of having column embedding is to enable the model to distinguish the classes in one column from those in the other columns'. In other words, the idea is to let the model learn which column is embedded at the time. See: <code>pytorch_widedeep.models.transformers._layers.SharedEmbeddings</code>.</p> <code>None</code> <code>add_shared_embed</code> <code>Optional[bool]</code> <p>The two embedding sharing strategies are: 1) add the shared embeddings to the column embeddings or 2) to replace the first <code>frac_shared_embed</code> with the shared embeddings. See <code>pytorch_widedeep.models.embeddings_layers.SharedEmbeddings</code> If 'None' is passed, it will default to 'False'.</p> <code>None</code> <code>frac_shared_embed</code> <code>Optional[float]</code> <p>The fraction of embeddings that will be shared (if <code>add_shared_embed = False</code>) by all the different categories for one particular column. If 'None' is passed, it will default to 0.0.</p> <code>None</code> <code>continuous_cols</code> <code>Optional[List[str]]</code> <p>List with the name of the numeric (aka continuous) columns</p> <code>None</code> <code>cont_norm_layer</code> <code>Optional[Literal[batchnorm, layernorm]]</code> <p>Type of normalization layer applied to the continuous features. Options are: 'layernorm' and 'batchnorm'. if <code>None</code>, no normalization layer will be used.</p> <code>None</code> <code>embed_continuous_method</code> <code>Optional[Literal[standard, piecewise, periodic]]</code> <p>Method to use to embed the continuous features. Options are: 'standard', 'periodic' or 'piecewise'. The 'standard' embedding method is based on the FT-Transformer implementation presented in the paper: Revisiting Deep Learning Models for Tabular Data. The 'periodic' and_'piecewise'_ methods were presented in the paper: On Embeddings for Numerical Features in Tabular Deep Learning. Please, read the papers for details.</p> <code>'standard'</code> <code>cont_embed_dropout</code> <code>Optional[float]</code> <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p> <code>None</code> <code>cont_embed_activation</code> <code>Optional[str]</code> <p>Activation function for the continuous embeddings if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported. If <code>None</code>, no activation function will be applied.</p> <code>None</code> <code>quantization_setup</code> <code>Optional[Dict[str, List[float]]]</code> <p>This parameter is used when the 'piecewise' method is used to embed the continuous cols. It is a dict where keys are the name of the continuous columns and values are lists with the boundaries for the quantization of the continuous_cols. See the examples for details. If If the 'piecewise' method is used, this parameter is required.</p> <code>None</code> <code>n_frequencies</code> <code>Optional[int]</code> <p>This is the so called 'k' in their paper On Embeddings for Numerical Features in Tabular Deep Learning, and is the number of 'frequencies' that will be used to represent each continuous column. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>sigma</code> <code>Optional[float]</code> <p>This is the sigma parameter in the paper mentioned when describing the previous parameters and it is used to initialise the 'frequency weights'. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>share_last_layer</code> <code>Optional[bool]</code> <p>This parameter is not present in the before mentioned paper but it is implemented in the official repo. If <code>True</code> the linear layer that turns the frequencies into embeddings will be shared across the continuous columns. If <code>False</code> a different linear layer will be used for each continuous column. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>full_embed_dropout</code> <code>Optional[bool]</code> <p>If <code>True</code>, the full embedding corresponding to a column will be masked out/dropout. If <code>None</code>, it will default to <code>False</code>.</p> <code>None</code> <code>input_dim</code> <code>int</code> <p>The so-called dimension of the model. Is the number of embeddings used to encode the categorical and/or continuous columns.</p> <code>32</code> <code>n_cross_attns</code> <code>int</code> <p>Number of times each perceiver block will cross attend to the input data (i.e. number of cross attention components per perceiver block). This should normally be 1. However, in the paper they describe some architectures (normally computer vision-related problems) where the Perceiver attends multiple times to the input array. Therefore, maybe multiple cross attention to the input array is also useful in some cases for tabular data  .</p> <code>1</code> <code>n_cross_attn_heads</code> <code>int</code> <p>Number of attention heads for the cross attention component</p> <code>4</code> <code>n_latents</code> <code>int</code> <p>Number of latents. This is the \\(N\\) parameter in the paper. As indicated in the paper, this number should be significantly lower than \\(M\\) (the number of columns in the dataset). Setting \\(N\\) closer to \\(M\\) defies the main purpose of the Perceiver, which is to overcome the transformer quadratic bottleneck</p> <code>16</code> <code>latent_dim</code> <code>int</code> <p>Latent dimension.</p> <code>128</code> <code>n_latent_heads</code> <code>int</code> <p>Number of attention heads per Latent Transformer</p> <code>4</code> <code>n_latent_blocks</code> <code>int</code> <p>Number of transformer encoder blocks (normalised MHA + normalised FF) per Latent Transformer</p> <code>4</code> <code>n_perceiver_blocks</code> <code>int</code> <p>Number of Perceiver blocks defined as [Cross Attention + Latent Transformer]</p> <code>4</code> <code>share_weights</code> <code>bool</code> <p>Boolean indicating if the weights will be shared between Perceiver blocks</p> <code>False</code> <code>attn_dropout</code> <code>float</code> <p>Dropout that will be applied to the Multi-Head Attention layers</p> <code>0.1</code> <code>ff_dropout</code> <code>float</code> <p>Dropout that will be applied to the FeedForward network</p> <code>0.1</code> <code>ff_factor</code> <code>int</code> <p>Multiplicative factor applied to the first layer of the FF network in each Transformer block, This is normally set to 4.</p> <code>4</code> <code>transformer_activation</code> <code>str</code> <p>Transformer Encoder activation function. 'tanh', 'relu', 'leaky_relu', 'gelu', 'geglu' and 'reglu' are supported</p> <code>'geglu'</code> <code>mlp_hidden_dims</code> <code>Optional[List[int]]</code> <p>List with the number of neurons per dense layer in the MLP. e.g: [64, 32]. If not provided no MLP on top of the final Transformer block will be used.</p> <code>None</code> <code>mlp_activation</code> <code>Optional[str]</code> <p>Activation function for the dense layers of the MLP. Currently 'tanh', 'relu', 'leaky'_relu' and _'gelu' are supported. If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to 'relu'.</p> <code>None</code> <code>mlp_dropout</code> <code>Optional[float]</code> <p>float with the dropout between the dense layers of the MLP. If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to 0.0.</p> <code>None</code> <code>mlp_batchnorm</code> <code>Optional[bool]</code> <p>Boolean indicating whether or not batch normalization will be applied to the dense layers If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to False.</p> <code>None</code> <code>mlp_batchnorm_last</code> <code>Optional[bool]</code> <p>Boolean indicating whether or not batch normalization will be applied to the last of the dense layers If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to False.</p> <code>None</code> <code>mlp_linear_first</code> <code>Optional[bool]</code> <p>Boolean indicating the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code> If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to <code>True</code>.</p> <code>None</code> <p>Attributes:</p> Name Type Description <code>encoder</code> <code>ModuleDict</code> <p>ModuleDict with the Perceiver blocks</p> <code>latents</code> <code>Parameter</code> <p>Latents that will be used for prediction</p> <code>mlp</code> <code>Module</code> <p>MLP component in the model</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import TabPerceiver\n&gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n&gt;&gt;&gt; colnames = ['a', 'b', 'c', 'd', 'e']\n&gt;&gt;&gt; cat_embed_input = [(u,i) for u,i in zip(colnames[:4], [4]*4)]\n&gt;&gt;&gt; continuous_cols = ['e']\n&gt;&gt;&gt; column_idx = {k:v for v,k in enumerate(colnames)}\n&gt;&gt;&gt; model = TabPerceiver(column_idx=column_idx, cat_embed_input=cat_embed_input,\n... continuous_cols=continuous_cols, n_latents=2, latent_dim=16,\n... n_perceiver_blocks=2)\n&gt;&gt;&gt; out = model(X_tab)\n</code></pre> Source code in <code>pytorch_widedeep/models/tabular/transformers/tab_perceiver.py</code> <pre><code>class TabPerceiver(BaseTabularModelWithAttention):\n    r\"\"\"Defines an adaptation of a [Perceiver](https://arxiv.org/abs/2103.03206)\n     that can be used as the `deeptabular` component of a Wide &amp; Deep model\n     or independently by itself.\n\n    Most of the parameters for this class are `Optional` since the use of\n    categorical or continuous is in fact optional (i.e. one can use\n    categorical features only, continuous features only or both).\n\n    :information_source: **NOTE**: while there are scientific publications for\n     the `TabTransformer`, `SAINT` and `FTTransformer`, the `TabPerceiver`\n     and the `TabFastFormer` are our own adaptations of the\n     [Perceiver](https://arxiv.org/abs/2103.03206) and the\n     [FastFormer](https://arxiv.org/abs/2108.09084) for tabular data.\n\n    Parameters\n    ----------\n    column_idx: Dict\n        Dict containing the index of the columns that will be passed through\n        the `TabMlp` model. Required to slice the tensors. e.g. _{'education':\n        0, 'relationship': 1, 'workclass': 2, ...}_.\n    cat_embed_input: List, Optional, default = None\n        List of Tuples with the column name and number of unique values and\n        embedding dimension. e.g. _[(education, 11), ...]_\n    cat_embed_dropout: float, Optional, default = None\n        Categorical embeddings dropout. If `None`, it will default\n        to 0.\n    use_cat_bias: bool, Optional, default = None,\n        Boolean indicating if bias will be used for the categorical embeddings.\n        If `None`, it will default to 'False'.\n    cat_embed_activation: Optional, str, default = None,\n        Activation function for the categorical embeddings, if any. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported\n    shared_embed: bool, Optional, default = None\n        Boolean indicating if the embeddings will be \"shared\". The idea behind `shared_embed` is\n        described in the Appendix A in the [TabTransformer paper](https://arxiv.org/abs/2012.06678):\n        _'The goal of having column embedding is to enable the model to\n        distinguish the classes in one column from those in the other\n        columns'_. In other words, the idea is to let the model learn which\n        column is embedded at the time. See: `pytorch_widedeep.models.transformers._layers.SharedEmbeddings`.\n    add_shared_embed: bool, Optional, default = None\n        The two embedding sharing strategies are: 1) add the shared embeddings\n        to the column embeddings or 2) to replace the first\n        `frac_shared_embed` with the shared embeddings.\n        See `pytorch_widedeep.models.embeddings_layers.SharedEmbeddings`\n        If 'None' is passed, it will default to 'False'.\n    frac_shared_embed: float, Optional, default = None\n        The fraction of embeddings that will be shared (if `add_shared_embed\n        = False`) by all the different categories for one particular\n        column. If 'None' is passed, it will default to 0.0.\n    continuous_cols: List, Optional, default = None\n        List with the name of the numeric (aka continuous) columns\n    cont_norm_layer: str, Optional, default =  None\n        Type of normalization layer applied to the continuous features.\n        Options are: _'layernorm'_ and _'batchnorm'_. if `None`, no\n        normalization layer will be used.\n    embed_continuous_method: Optional, str, default = None,\n        Method to use to embed the continuous features. Options are:\n        _'standard'_, _'periodic'_ or _'piecewise'_. The _'standard'_\n        embedding method is based on the FT-Transformer implementation\n        presented in the paper: [Revisiting Deep Learning Models for\n        Tabular Data](https://arxiv.org/abs/2106.11959v5). The _'periodic'_\n        and_'piecewise'_ methods were presented in the paper: [On Embeddings for\n        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556).\n        Please, read the papers for details.\n    cont_embed_dropout: float, Optional, default = None,\n        Dropout for the continuous embeddings. If `None`, it will default to 0.0\n    cont_embed_activation: Optional, str, default = None,\n        Activation function for the continuous embeddings if any. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported.\n        If `None`, no activation function will be applied.\n    quantization_setup: Dict[str, List[float]], Optional, default = None,\n        This parameter is used when the _'piecewise'_ method is used to embed\n        the continuous cols. It is a dict where keys are the name of the continuous\n        columns and values are lists with the boundaries for the quantization\n        of the continuous_cols. See the examples for details. If\n        If the _'piecewise'_ method is used, this parameter is required.\n    n_frequencies: int, Optional, default = None,\n        This is the so called _'k'_ in their paper [On Embeddings for\n        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556),\n        and is the number of 'frequencies' that will be used to represent each\n        continuous column. See their Eq 2 in the paper for details. If\n        the _'periodic'_ method is used, this parameter is required.\n    sigma: float, Optional, default = None,\n        This is the sigma parameter in the paper mentioned when describing the\n        previous parameters and it is used to initialise the 'frequency\n        weights'. See their Eq 2 in the paper for details. If\n        the _'periodic'_ method is used, this parameter is required.\n    share_last_layer: bool, Optional, default = None,\n        This parameter is not present in the before mentioned paper but it is implemented in\n        the [official repo](https://github.com/yandex-research/rtdl-num-embeddings/tree/main).\n        If `True` the linear layer that turns the frequencies into embeddings\n        will be shared across the continuous columns. If `False` a different\n        linear layer will be used for each continuous column.\n        If the _'periodic'_ method is used, this parameter is required.\n    full_embed_dropout: bool, Optional, default = None,\n        If `True`, the full embedding corresponding to a column will be masked\n        out/dropout. If `None`, it will default to `False`.\n    input_dim: int, default = 32\n        The so-called *dimension of the model*. Is the number of embeddings\n        used to encode the categorical and/or continuous columns.\n    n_cross_attns: int, default = 1\n        Number of times each perceiver block will cross attend to the input\n        data (i.e. number of cross attention components per perceiver block).\n        This should normally be 1. However, in the paper they describe some\n        architectures (normally computer vision-related problems) where the\n        Perceiver attends multiple times to the input array. Therefore, maybe\n        multiple cross attention to the input array is also useful in some\n        cases for tabular data :shrug: .\n    n_cross_attn_heads: int, default = 4\n        Number of attention heads for the cross attention component\n    n_latents: int, default = 16\n        Number of latents. This is the $N$ parameter in the paper. As\n        indicated in the paper, this number should be significantly lower\n        than $M$ (the number of columns in the dataset). Setting $N$ closer\n        to $M$ defies the main purpose of the Perceiver, which is to overcome\n        the transformer quadratic bottleneck\n    latent_dim: int, default = 128\n        Latent dimension.\n    n_latent_heads: int, default = 4\n        Number of attention heads per Latent Transformer\n    n_latent_blocks: int, default = 4\n        Number of transformer encoder blocks (normalised MHA + normalised FF)\n        per Latent Transformer\n    n_perceiver_blocks: int, default = 4\n        Number of Perceiver blocks defined as [Cross Attention + Latent\n        Transformer]\n    share_weights: Boolean, default = False\n        Boolean indicating if the weights will be shared between Perceiver\n        blocks\n    attn_dropout: float, default = 0.2\n        Dropout that will be applied to the Multi-Head Attention layers\n    ff_dropout: float, default = 0.1\n        Dropout that will be applied to the FeedForward network\n    ff_factor: float, default = 4\n        Multiplicative factor applied to the first layer of the FF network in\n        each Transformer block, This is normally set to 4.\n    transformer_activation: str, default = \"gelu\"\n        Transformer Encoder activation function. _'tanh'_, _'relu'_,\n        _'leaky_relu'_, _'gelu'_, _'geglu'_ and _'reglu'_ are supported\n    mlp_hidden_dims: List, Optional, default = None\n        List with the number of neurons per dense layer in the MLP. e.g:\n        _[64, 32]_. If not provided no MLP on top of the final\n        Transformer block will be used.\n    mlp_activation: str, Optional, default = None\n        Activation function for the dense layers of the MLP. Currently\n        _'tanh'_, _'relu'_, _'leaky'_relu' and _'gelu'_ are supported.\n        If 'mlp_hidden_dims' is not `None` and this parameter is `None`, it\n        will default to _'relu'_.\n    mlp_dropout: float, Optional, default = None\n        float with the dropout between the dense layers of the MLP.\n        If 'mlp_hidden_dims' is not `None` and this parameter is `None`, it\n        will default to 0.0.\n    mlp_batchnorm: bool, Optional, default = None\n        Boolean indicating whether or not batch normalization will be applied\n        to the dense layers\n        If 'mlp_hidden_dims' is not `None` and this parameter is `None`, it\n        will default to False.\n    mlp_batchnorm_last: bool, Optional, default = None\n        Boolean indicating whether or not batch normalization will be applied\n        to the last of the dense layers\n        If 'mlp_hidden_dims' is not `None` and this parameter is `None`, it\n        will default to False.\n    mlp_linear_first: bool, Optional, default = None\n        Boolean indicating the order of the operations in the dense\n        layer. If `True: [LIN -&gt; ACT -&gt; BN -&gt; DP]`. If `False: [BN -&gt; DP -&gt;\n        LIN -&gt; ACT]`\n        If 'mlp_hidden_dims' is not `None` and this parameter is `None`, it\n        will default to `True`.\n\n    Attributes\n    ----------\n    encoder: nn.ModuleDict\n        ModuleDict with the Perceiver blocks\n    latents: nn.Parameter\n        Latents that will be used for prediction\n    mlp: nn.Module\n        MLP component in the model\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.models import TabPerceiver\n    &gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n    &gt;&gt;&gt; colnames = ['a', 'b', 'c', 'd', 'e']\n    &gt;&gt;&gt; cat_embed_input = [(u,i) for u,i in zip(colnames[:4], [4]*4)]\n    &gt;&gt;&gt; continuous_cols = ['e']\n    &gt;&gt;&gt; column_idx = {k:v for v,k in enumerate(colnames)}\n    &gt;&gt;&gt; model = TabPerceiver(column_idx=column_idx, cat_embed_input=cat_embed_input,\n    ... continuous_cols=continuous_cols, n_latents=2, latent_dim=16,\n    ... n_perceiver_blocks=2)\n    &gt;&gt;&gt; out = model(X_tab)\n    \"\"\"\n\n    def __init__(\n        self,\n        column_idx: Dict[str, int],\n        *,\n        cat_embed_input: Optional[List[Tuple[str, int]]] = None,\n        cat_embed_dropout: Optional[float] = None,\n        use_cat_bias: Optional[bool] = None,\n        cat_embed_activation: Optional[str] = None,\n        shared_embed: Optional[bool] = None,\n        add_shared_embed: Optional[bool] = None,\n        frac_shared_embed: Optional[float] = None,\n        continuous_cols: Optional[List[str]] = None,\n        cont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\n        embed_continuous_method: Optional[\n            Literal[\"standard\", \"piecewise\", \"periodic\"]\n        ] = \"standard\",\n        cont_embed_dropout: Optional[float] = None,\n        cont_embed_activation: Optional[str] = None,\n        quantization_setup: Optional[Dict[str, List[float]]] = None,\n        n_frequencies: Optional[int] = None,\n        sigma: Optional[float] = None,\n        share_last_layer: Optional[bool] = None,\n        full_embed_dropout: Optional[bool] = None,\n        input_dim: int = 32,\n        n_cross_attns: int = 1,\n        n_cross_attn_heads: int = 4,\n        n_latents: int = 16,\n        latent_dim: int = 128,\n        n_latent_heads: int = 4,\n        n_latent_blocks: int = 4,\n        n_perceiver_blocks: int = 4,\n        share_weights: bool = False,\n        attn_dropout: float = 0.1,\n        ff_dropout: float = 0.1,\n        ff_factor: int = 4,\n        transformer_activation: str = \"geglu\",\n        mlp_hidden_dims: Optional[List[int]] = None,\n        mlp_activation: Optional[str] = None,\n        mlp_dropout: Optional[float] = None,\n        mlp_batchnorm: Optional[bool] = None,\n        mlp_batchnorm_last: Optional[bool] = None,\n        mlp_linear_first: Optional[bool] = None,\n    ):\n        super(TabPerceiver, self).__init__(\n            column_idx=column_idx,\n            cat_embed_input=cat_embed_input,\n            cat_embed_dropout=cat_embed_dropout,\n            use_cat_bias=use_cat_bias,\n            cat_embed_activation=cat_embed_activation,\n            shared_embed=shared_embed,\n            add_shared_embed=add_shared_embed,\n            frac_shared_embed=frac_shared_embed,\n            continuous_cols=continuous_cols,\n            cont_norm_layer=cont_norm_layer,\n            embed_continuous_method=embed_continuous_method,\n            cont_embed_dropout=cont_embed_dropout,\n            cont_embed_activation=cont_embed_activation,\n            input_dim=input_dim,\n            quantization_setup=quantization_setup,\n            n_frequencies=n_frequencies,\n            sigma=sigma,\n            share_last_layer=share_last_layer,\n            full_embed_dropout=full_embed_dropout,\n        )\n\n        self.n_cross_attns = n_cross_attns\n        self.n_cross_attn_heads = n_cross_attn_heads\n        self.n_latents = n_latents\n        self.latent_dim = latent_dim\n        self.n_latent_heads = n_latent_heads\n        self.n_latent_blocks = n_latent_blocks\n        self.n_perceiver_blocks = n_perceiver_blocks\n        self.share_weights = share_weights\n        self.attn_dropout = attn_dropout\n        self.ff_dropout = ff_dropout\n        self.ff_factor = ff_factor\n        self.transformer_activation = transformer_activation\n\n        self.mlp_hidden_dims = mlp_hidden_dims\n        self.mlp_activation = mlp_activation\n        self.mlp_dropout = mlp_dropout\n        self.mlp_batchnorm = mlp_batchnorm\n        self.mlp_batchnorm_last = mlp_batchnorm_last\n        self.mlp_linear_first = mlp_linear_first\n\n        # Embeddings are instantiated at the base model\n        # Transformer blocks\n        self.latents = nn.init.trunc_normal_(\n            nn.Parameter(torch.empty(n_latents, latent_dim))\n        )\n\n        self.encoder = nn.ModuleDict()\n        first_perceiver_block = self._build_perceiver_block()\n        self.encoder[\"perceiver_block0\"] = first_perceiver_block\n\n        if share_weights:\n            for n in range(1, n_perceiver_blocks):\n                self.encoder[\"perceiver_block\" + str(n)] = first_perceiver_block\n        else:\n            for n in range(1, n_perceiver_blocks):\n                self.encoder[\"perceiver_block\" + str(n)] = self._build_perceiver_block()\n\n        self.mlp_first_hidden_dim = self.latent_dim\n\n        # Mlp\n        if self.mlp_hidden_dims is not None:\n            self.mlp = MLP(\n                d_hidden=[self.mlp_first_hidden_dim] + self.mlp_hidden_dims,\n                activation=(\n                    \"relu\" if self.mlp_activation is None else self.mlp_activation\n                ),\n                dropout=0.0 if self.mlp_dropout is None else self.mlp_dropout,\n                batchnorm=False if self.mlp_batchnorm is None else self.mlp_batchnorm,\n                batchnorm_last=(\n                    False\n                    if self.mlp_batchnorm_last is None\n                    else self.mlp_batchnorm_last\n                ),\n                linear_first=(\n                    False if self.mlp_linear_first is None else self.mlp_linear_first\n                ),\n            )\n        else:\n            self.mlp = None\n\n    def forward(self, X: Tensor) -&gt; Tensor:\n        x_emb = self._get_embeddings(X)\n\n        x = einops.repeat(self.latents, \"n d -&gt; b n d\", b=X.shape[0])\n\n        for n in range(self.n_perceiver_blocks):\n            cross_attns = self.encoder[\"perceiver_block\" + str(n)][\"cross_attns\"]\n            latent_transformer = self.encoder[\"perceiver_block\" + str(n)][\n                \"latent_transformer\"\n            ]\n            for cross_attn in cross_attns:\n                x = cross_attn(x, x_emb)\n            x = latent_transformer(x)\n\n        # average along the latent index axis\n        x = x.mean(dim=1)\n\n        if self.mlp is not None:\n            x = self.mlp(x)\n\n        return x\n\n    @property\n    def output_dim(self) -&gt; int:\n        r\"\"\"The output dimension of the model. This is a required property\n        neccesary to build the `WideDeep` class\n        \"\"\"\n        return (\n            self.mlp_hidden_dims[-1]\n            if self.mlp_hidden_dims is not None\n            else self.mlp_first_hidden_dim\n        )\n\n    @property\n    def attention_weights(self) -&gt; List:\n        r\"\"\"List with the attention weights. If the weights are not shared\n        between perceiver blocks each element of the list will be a list\n        itself containing the Cross Attention and Latent Transformer\n        attention weights respectively\n\n        The shape of the attention weights is:\n\n        - Cross Attention: $(N, C, L, F)$\n\n        - Latent Attention: $(N, T, L, L)$\n\n        WHere $N$ is the batch size, $C$ is the number of Cross Attention\n        heads, $L$ is the number of Latents, $F$ is the number of\n        features/columns in the dataset and $T$ is the number of Latent\n        Attention heads\n        \"\"\"\n        if self.share_weights:\n            cross_attns = self.encoder[\"perceiver_block0\"][\"cross_attns\"]\n            latent_transformer = self.encoder[\"perceiver_block0\"][\"latent_transformer\"]\n            attention_weights = self._extract_attn_weights(\n                cross_attns, latent_transformer\n            )\n        else:\n            attention_weights = []\n            for n in range(self.n_perceiver_blocks):\n                cross_attns = self.encoder[\"perceiver_block\" + str(n)][\"cross_attns\"]\n                latent_transformer = self.encoder[\"perceiver_block\" + str(n)][\n                    \"latent_transformer\"\n                ]\n                attention_weights.append(\n                    self._extract_attn_weights(cross_attns, latent_transformer)\n                )\n        return attention_weights\n\n    def _build_perceiver_block(self) -&gt; nn.ModuleDict:\n        perceiver_block = nn.ModuleDict()\n\n        # Cross Attention\n        cross_attns = nn.ModuleList()\n        for _ in range(self.n_cross_attns):\n            cross_attns.append(\n                PerceiverEncoder(\n                    self.input_dim,\n                    self.n_cross_attn_heads,\n                    False,  # use_bias\n                    self.attn_dropout,\n                    self.ff_dropout,\n                    self.ff_factor,\n                    self.transformer_activation,\n                    self.latent_dim,  # q_dim,\n                ),\n            )\n        perceiver_block[\"cross_attns\"] = cross_attns\n\n        # Latent Transformer\n        latent_transformer = nn.Sequential()\n        for i in range(self.n_latent_blocks):\n            latent_transformer.add_module(\n                \"latent_block\" + str(i),\n                PerceiverEncoder(\n                    self.latent_dim,  # input_dim\n                    self.n_latent_heads,\n                    False,  # use_bias\n                    self.attn_dropout,\n                    self.ff_dropout,\n                    self.ff_factor,\n                    self.transformer_activation,\n                ),\n            )\n        perceiver_block[\"latent_transformer\"] = latent_transformer\n\n        return perceiver_block\n\n    @staticmethod\n    def _extract_attn_weights(cross_attns, latent_transformer) -&gt; List:\n        attention_weights = []\n        for cross_attn in cross_attns:\n            attention_weights.append(cross_attn.attn.attn_weights)\n        for latent_block in latent_transformer:\n            attention_weights.append(latent_block.attn.attn_weights)\n        return attention_weights\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.transformers.tab_perceiver.TabPerceiver.output_dim","title":"output_dim  <code>property</code>","text":"<pre><code>output_dim\n</code></pre> <p>The output dimension of the model. This is a required property neccesary to build the <code>WideDeep</code> class</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.transformers.tab_perceiver.TabPerceiver.attention_weights","title":"attention_weights  <code>property</code>","text":"<pre><code>attention_weights\n</code></pre> <p>List with the attention weights. If the weights are not shared between perceiver blocks each element of the list will be a list itself containing the Cross Attention and Latent Transformer attention weights respectively</p> <p>The shape of the attention weights is:</p> <ul> <li> <p>Cross Attention: \\((N, C, L, F)\\)</p> </li> <li> <p>Latent Attention: \\((N, T, L, L)\\)</p> </li> </ul> <p>WHere \\(N\\) is the batch size, \\(C\\) is the number of Cross Attention heads, \\(L\\) is the number of Latents, \\(F\\) is the number of features/columns in the dataset and \\(T\\) is the number of Latent Attention heads</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.transformers.tab_fastformer.TabFastFormer","title":"TabFastFormer","text":"<p>               Bases: <code>BaseTabularModelWithAttention</code></p> <p>Defines an adaptation of a FastFormer that can be used as the <code>deeptabular</code> component of a Wide &amp; Deep model or independently by itself.</p> <p>Most of the parameters for this class are <code>Optional</code> since the use of categorical or continuous is in fact optional (i.e. one can use categorical features only, continuous features only or both).</p> <p> NOTE: while there are scientific publications for  the <code>TabTransformer</code>, <code>SAINT</code> and <code>FTTransformer</code>, the <code>TabPerceiver</code>  and the <code>TabFastFormer</code> are our own adaptations of the  Perceiver and the  FastFormer for tabular data.</p> <p>Parameters:</p> Name Type Description Default <code>column_idx</code> <code>Dict[str, int]</code> <p>Dict containing the index of the columns that will be passed through the <code>TabMlp</code> model. Required to slice the tensors. e.g. {'education': 0, 'relationship': 1, 'workclass': 2, ...}.</p> required <code>cat_embed_input</code> <code>Optional[List[Tuple[str, int]]]</code> <p>List of Tuples with the column name and number of unique values and embedding dimension. e.g. [(education, 11), ...]</p> <code>None</code> <code>cat_embed_dropout</code> <code>Optional[float]</code> <p>Categorical embeddings dropout. If <code>None</code>, it will default to 0.</p> <code>None</code> <code>use_cat_bias</code> <code>Optional[bool]</code> <p>Boolean indicating if bias will be used for the categorical embeddings. If <code>None</code>, it will default to 'False'.</p> <code>None</code> <code>cat_embed_activation</code> <code>Optional[str]</code> <p>Activation function for the categorical embeddings, if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> <code>None</code> <code>shared_embed</code> <code>Optional[bool]</code> <p>Boolean indicating if the embeddings will be \"shared\". The idea behind <code>shared_embed</code> is described in the Appendix A in the TabTransformer paper: 'The goal of having column embedding is to enable the model to distinguish the classes in one column from those in the other columns'. In other words, the idea is to let the model learn which column is embedded at the time. See: <code>pytorch_widedeep.models.transformers._layers.SharedEmbeddings</code>.</p> <code>None</code> <code>add_shared_embed</code> <code>Optional[bool]</code> <p>The two embedding sharing strategies are: 1) add the shared embeddings to the column embeddings or 2) to replace the first <code>frac_shared_embed</code> with the shared embeddings. See <code>pytorch_widedeep.models.embeddings_layers.SharedEmbeddings</code> If 'None' is passed, it will default to 'False'.</p> <code>None</code> <code>frac_shared_embed</code> <code>Optional[float]</code> <p>The fraction of embeddings that will be shared (if <code>add_shared_embed = False</code>) by all the different categories for one particular column. If 'None' is passed, it will default to 0.0.</p> <code>None</code> <code>continuous_cols</code> <code>Optional[List[str]]</code> <p>List with the name of the numeric (aka continuous) columns</p> <code>None</code> <code>cont_norm_layer</code> <code>Optional[Literal[batchnorm, layernorm]]</code> <p>Type of normalization layer applied to the continuous features. Options are: 'layernorm' and 'batchnorm'. if <code>None</code>, no normalization layer will be used.</p> <code>None</code> <code>embed_continuous_method</code> <code>Optional[Literal[standard, piecewise, periodic]]</code> <p>Method to use to embed the continuous features. Options are: 'standard', 'periodic' or 'piecewise'. The 'standard' embedding method is based on the FT-Transformer implementation presented in the paper: Revisiting Deep Learning Models for Tabular Data. The 'periodic' and_'piecewise'_ methods were presented in the paper: On Embeddings for Numerical Features in Tabular Deep Learning. Please, read the papers for details.</p> <code>'standard'</code> <code>cont_embed_dropout</code> <code>Optional[float]</code> <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p> <code>None</code> <code>cont_embed_activation</code> <code>Optional[str]</code> <p>Activation function for the continuous embeddings if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported. If <code>None</code>, no activation function will be applied.</p> <code>None</code> <code>quantization_setup</code> <code>Optional[Dict[str, List[float]]]</code> <p>This parameter is used when the 'piecewise' method is used to embed the continuous cols. It is a dict where keys are the name of the continuous columns and values are lists with the boundaries for the quantization of the continuous_cols. See the examples for details. If If the 'piecewise' method is used, this parameter is required.</p> <code>None</code> <code>n_frequencies</code> <code>Optional[int]</code> <p>This is the so called 'k' in their paper On Embeddings for Numerical Features in Tabular Deep Learning, and is the number of 'frequencies' that will be used to represent each continuous column. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>sigma</code> <code>Optional[float]</code> <p>This is the sigma parameter in the paper mentioned when describing the previous parameters and it is used to initialise the 'frequency weights'. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>share_last_layer</code> <code>Optional[bool]</code> <p>This parameter is not present in the before mentioned paper but it is implemented in the official repo. If <code>True</code> the linear layer that turns the frequencies into embeddings will be shared across the continuous columns. If <code>False</code> a different linear layer will be used for each continuous column. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>full_embed_dropout</code> <code>Optional[bool]</code> <p>If <code>True</code>, the full embedding corresponding to a column will be masked out/dropout. If <code>None</code>, it will default to <code>False</code>.</p> <code>None</code> <code>input_dim</code> <code>int</code> <p>The so-called dimension of the model. Is the number of embeddings used to encode the categorical and/or continuous columns</p> <code>32</code> <code>n_heads</code> <code>int</code> <p>Number of attention heads per FastFormer block</p> <code>8</code> <code>use_bias</code> <code>bool</code> <p>Boolean indicating whether or not to use bias in the Q, K, and V projection layers</p> <code>False</code> <code>n_blocks</code> <code>int</code> <p>Number of FastFormer blocks</p> <code>4</code> <code>attn_dropout</code> <code>float</code> <p>Dropout that will be applied to the Additive Attention layers</p> <code>0.1</code> <code>ff_dropout</code> <code>float</code> <p>Dropout that will be applied to the FeedForward network</p> <code>0.2</code> <code>ff_factor</code> <code>int</code> <p>Multiplicative factor applied to the first layer of the FF network in each Transformer block, This is normally set to 4.</p> <code>4</code> <code>share_qv_weights</code> <code>bool</code> <p>Following the paper, this is a boolean indicating if the Value (\\(V\\)) and the Query (\\(Q\\)) transformation parameters will be shared.</p> <code>False</code> <code>share_weights</code> <code>bool</code> <p>In addition to sharing the \\(V\\) and \\(Q\\) transformation parameters, the parameters across different Fastformer layers can also be shared. Please, see <code>pytorch_widedeep/models/tabular/transformers/tab_fastformer.py</code> for details</p> <code>False</code> <code>transformer_activation</code> <code>str</code> <p>Transformer Encoder activation function. 'tanh', 'relu', 'leaky_relu', 'gelu', 'geglu' and 'reglu' are supported</p> <code>'relu'</code> <code>mlp_hidden_dims</code> <code>Optional[List[int]]</code> <p>MLP hidden dimensions. If not provided no MLP on top of the final FTTransformer block will be used</p> <code>None</code> <code>mlp_hidden_dims</code> <code>Optional[List[int]]</code> <p>List with the number of neurons per dense layer in the MLP. e.g: [64, 32]. If not provided no MLP on top of the final Transformer block will be used.</p> <code>None</code> <code>mlp_activation</code> <code>Optional[str]</code> <p>Activation function for the dense layers of the MLP. Currently 'tanh', 'relu', 'leaky'_relu' and _'gelu' are supported. If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to 'relu'.</p> <code>None</code> <code>mlp_dropout</code> <code>Optional[float]</code> <p>float with the dropout between the dense layers of the MLP. If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to 0.0.</p> <code>None</code> <code>mlp_batchnorm</code> <code>Optional[bool]</code> <p>Boolean indicating whether or not batch normalization will be applied to the dense layers If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to False.</p> <code>None</code> <code>mlp_batchnorm_last</code> <code>Optional[bool]</code> <p>Boolean indicating whether or not batch normalization will be applied to the last of the dense layers If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to False.</p> <code>None</code> <code>mlp_linear_first</code> <code>Optional[bool]</code> <p>Boolean indicating the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code> If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to <code>True</code>.</p> <code>None</code> <p>Attributes:</p> Name Type Description <code>encoder</code> <code>Module</code> <p>Sequence of FasFormer blocks.</p> <code>mlp</code> <code>Module</code> <p>MLP component in the model</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import TabFastFormer\n&gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n&gt;&gt;&gt; colnames = ['a', 'b', 'c', 'd', 'e']\n&gt;&gt;&gt; cat_embed_input = [(u,i) for u,i in zip(colnames[:4], [4]*4)]\n&gt;&gt;&gt; continuous_cols = ['e']\n&gt;&gt;&gt; column_idx = {k:v for v,k in enumerate(colnames)}\n&gt;&gt;&gt; model = TabFastFormer(column_idx=column_idx, cat_embed_input=cat_embed_input, continuous_cols=continuous_cols)\n&gt;&gt;&gt; out = model(X_tab)\n</code></pre> Source code in <code>pytorch_widedeep/models/tabular/transformers/tab_fastformer.py</code> <pre><code>class TabFastFormer(BaseTabularModelWithAttention):\n    r\"\"\"Defines an adaptation of a [FastFormer](https://arxiv.org/abs/2108.09084)\n    that can be used as the `deeptabular` component of a Wide &amp; Deep model\n    or independently by itself.\n\n    Most of the parameters for this class are `Optional` since the use of\n    categorical or continuous is in fact optional (i.e. one can use\n    categorical features only, continuous features only or both).\n\n    :information_source: **NOTE**: while there are scientific publications for\n     the `TabTransformer`, `SAINT` and `FTTransformer`, the `TabPerceiver`\n     and the `TabFastFormer` are our own adaptations of the\n     [Perceiver](https://arxiv.org/abs/2103.03206) and the\n     [FastFormer](https://arxiv.org/abs/2108.09084) for tabular data.\n\n    Parameters\n    ----------\n    column_idx: Dict\n        Dict containing the index of the columns that will be passed through\n        the `TabMlp` model. Required to slice the tensors. e.g. _{'education':\n        0, 'relationship': 1, 'workclass': 2, ...}_.\n    cat_embed_input: List, Optional, default = None\n        List of Tuples with the column name and number of unique values and\n        embedding dimension. e.g. _[(education, 11), ...]_\n    cat_embed_dropout: float, Optional, default = None\n        Categorical embeddings dropout. If `None`, it will default\n        to 0.\n    use_cat_bias: bool, Optional, default = None,\n        Boolean indicating if bias will be used for the categorical embeddings.\n        If `None`, it will default to 'False'.\n    cat_embed_activation: Optional, str, default = None,\n        Activation function for the categorical embeddings, if any. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported\n    shared_embed: bool, Optional, default = None\n        Boolean indicating if the embeddings will be \"shared\". The idea behind `shared_embed` is\n        described in the Appendix A in the [TabTransformer paper](https://arxiv.org/abs/2012.06678):\n        _'The goal of having column embedding is to enable the model to\n        distinguish the classes in one column from those in the other\n        columns'_. In other words, the idea is to let the model learn which\n        column is embedded at the time. See: `pytorch_widedeep.models.transformers._layers.SharedEmbeddings`.\n    add_shared_embed: bool, Optional, default = None\n        The two embedding sharing strategies are: 1) add the shared embeddings\n        to the column embeddings or 2) to replace the first\n        `frac_shared_embed` with the shared embeddings.\n        See `pytorch_widedeep.models.embeddings_layers.SharedEmbeddings`\n        If 'None' is passed, it will default to 'False'.\n    frac_shared_embed: float, Optional, default = None\n        The fraction of embeddings that will be shared (if `add_shared_embed\n        = False`) by all the different categories for one particular\n        column. If 'None' is passed, it will default to 0.0.\n    continuous_cols: List, Optional, default = None\n        List with the name of the numeric (aka continuous) columns\n    cont_norm_layer: str, Optional, default =  None\n        Type of normalization layer applied to the continuous features.\n        Options are: _'layernorm'_ and _'batchnorm'_. if `None`, no\n        normalization layer will be used.\n    embed_continuous_method: Optional, str, default = None,\n        Method to use to embed the continuous features. Options are:\n        _'standard'_, _'periodic'_ or _'piecewise'_. The _'standard'_\n        embedding method is based on the FT-Transformer implementation\n        presented in the paper: [Revisiting Deep Learning Models for\n        Tabular Data](https://arxiv.org/abs/2106.11959v5). The _'periodic'_\n        and_'piecewise'_ methods were presented in the paper: [On Embeddings for\n        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556).\n        Please, read the papers for details.\n    cont_embed_dropout: float, Optional, default = None,\n        Dropout for the continuous embeddings. If `None`, it will default to 0.0\n    cont_embed_activation: Optional, str, default = None,\n        Activation function for the continuous embeddings if any. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported.\n        If `None`, no activation function will be applied.\n    quantization_setup: Dict[str, List[float]], Optional, default = None,\n        This parameter is used when the _'piecewise'_ method is used to embed\n        the continuous cols. It is a dict where keys are the name of the continuous\n        columns and values are lists with the boundaries for the quantization\n        of the continuous_cols. See the examples for details. If\n        If the _'piecewise'_ method is used, this parameter is required.\n    n_frequencies: int, Optional, default = None,\n        This is the so called _'k'_ in their paper [On Embeddings for\n        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556),\n        and is the number of 'frequencies' that will be used to represent each\n        continuous column. See their Eq 2 in the paper for details. If\n        the _'periodic'_ method is used, this parameter is required.\n    sigma: float, Optional, default = None,\n        This is the sigma parameter in the paper mentioned when describing the\n        previous parameters and it is used to initialise the 'frequency\n        weights'. See their Eq 2 in the paper for details. If\n        the _'periodic'_ method is used, this parameter is required.\n    share_last_layer: bool, Optional, default = None,\n        This parameter is not present in the before mentioned paper but it is implemented in\n        the [official repo](https://github.com/yandex-research/rtdl-num-embeddings/tree/main).\n        If `True` the linear layer that turns the frequencies into embeddings\n        will be shared across the continuous columns. If `False` a different\n        linear layer will be used for each continuous column.\n        If the _'periodic'_ method is used, this parameter is required.\n    full_embed_dropout: bool, Optional, default = None,\n        If `True`, the full embedding corresponding to a column will be masked\n        out/dropout. If `None`, it will default to `False`.\n    input_dim: int, default = 32\n        The so-called *dimension of the model*. Is the number of\n        embeddings used to encode the categorical and/or continuous columns\n    n_heads: int, default = 8\n        Number of attention heads per FastFormer block\n    use_bias: bool, default = False\n        Boolean indicating whether or not to use bias in the Q, K, and V\n        projection layers\n    n_blocks: int, default = 4\n        Number of FastFormer blocks\n    attn_dropout: float, default = 0.2\n        Dropout that will be applied to the Additive Attention layers\n    ff_dropout: float, default = 0.1\n        Dropout that will be applied to the FeedForward network\n    ff_factor: float, default = 4\n        Multiplicative factor applied to the first layer of the FF network in\n        each Transformer block, This is normally set to 4.\n    share_qv_weights: bool, default = False\n        Following the paper, this is a boolean indicating if the Value ($V$) and\n        the Query ($Q$) transformation parameters will be shared.\n    share_weights: bool, default = False\n        In addition to sharing the $V$ and $Q$ transformation parameters, the\n        parameters across different Fastformer layers can also be shared.\n        Please, see\n        `pytorch_widedeep/models/tabular/transformers/tab_fastformer.py` for\n        details\n    transformer_activation: str, default = \"gelu\"\n        Transformer Encoder activation function. _'tanh'_, _'relu'_,\n        _'leaky_relu'_, _'gelu'_, _'geglu'_ and _'reglu'_ are supported\n    mlp_hidden_dims: List, Optional, default = None\n        MLP hidden dimensions. If not provided no MLP on top of the final\n        FTTransformer block will be used\n    mlp_hidden_dims: List, Optional, default = None\n        List with the number of neurons per dense layer in the MLP. e.g:\n        _[64, 32]_. If not provided no MLP on top of the final\n        Transformer block will be used.\n    mlp_activation: str, Optional, default = None\n        Activation function for the dense layers of the MLP. Currently\n        _'tanh'_, _'relu'_, _'leaky'_relu' and _'gelu'_ are supported.\n        If 'mlp_hidden_dims' is not `None` and this parameter is `None`, it\n        will default to _'relu'_.\n    mlp_dropout: float, Optional, default = None\n        float with the dropout between the dense layers of the MLP.\n        If 'mlp_hidden_dims' is not `None` and this parameter is `None`, it\n        will default to 0.0.\n    mlp_batchnorm: bool, Optional, default = None\n        Boolean indicating whether or not batch normalization will be applied\n        to the dense layers\n        If 'mlp_hidden_dims' is not `None` and this parameter is `None`, it\n        will default to False.\n    mlp_batchnorm_last: bool, Optional, default = None\n        Boolean indicating whether or not batch normalization will be applied\n        to the last of the dense layers\n        If 'mlp_hidden_dims' is not `None` and this parameter is `None`, it\n        will default to False.\n    mlp_linear_first: bool, Optional, default = None\n        Boolean indicating the order of the operations in the dense\n        layer. If `True: [LIN -&gt; ACT -&gt; BN -&gt; DP]`. If `False: [BN -&gt; DP -&gt;\n        LIN -&gt; ACT]`\n        If 'mlp_hidden_dims' is not `None` and this parameter is `None`, it\n        will default to `True`.\n\n    Attributes\n    ----------\n    encoder: nn.Module\n        Sequence of FasFormer blocks.\n    mlp: nn.Module\n        MLP component in the model\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.models import TabFastFormer\n    &gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n    &gt;&gt;&gt; colnames = ['a', 'b', 'c', 'd', 'e']\n    &gt;&gt;&gt; cat_embed_input = [(u,i) for u,i in zip(colnames[:4], [4]*4)]\n    &gt;&gt;&gt; continuous_cols = ['e']\n    &gt;&gt;&gt; column_idx = {k:v for v,k in enumerate(colnames)}\n    &gt;&gt;&gt; model = TabFastFormer(column_idx=column_idx, cat_embed_input=cat_embed_input, continuous_cols=continuous_cols)\n    &gt;&gt;&gt; out = model(X_tab)\n    \"\"\"\n\n    def __init__(\n        self,\n        column_idx: Dict[str, int],\n        *,\n        cat_embed_input: Optional[List[Tuple[str, int]]] = None,\n        cat_embed_dropout: Optional[float] = None,\n        use_cat_bias: Optional[bool] = None,\n        cat_embed_activation: Optional[str] = None,\n        shared_embed: Optional[bool] = None,\n        add_shared_embed: Optional[bool] = None,\n        frac_shared_embed: Optional[float] = None,\n        continuous_cols: Optional[List[str]] = None,\n        cont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\n        embed_continuous_method: Optional[\n            Literal[\"standard\", \"piecewise\", \"periodic\"]\n        ] = \"standard\",\n        cont_embed_dropout: Optional[float] = None,\n        cont_embed_activation: Optional[str] = None,\n        quantization_setup: Optional[Dict[str, List[float]]] = None,\n        n_frequencies: Optional[int] = None,\n        sigma: Optional[float] = None,\n        share_last_layer: Optional[bool] = None,\n        full_embed_dropout: Optional[bool] = None,\n        input_dim: int = 32,\n        n_heads: int = 8,\n        use_bias: bool = False,\n        n_blocks: int = 4,\n        attn_dropout: float = 0.1,\n        ff_dropout: float = 0.2,\n        ff_factor: int = 4,\n        share_qv_weights: bool = False,\n        share_weights: bool = False,\n        transformer_activation: str = \"relu\",\n        mlp_hidden_dims: Optional[List[int]] = None,\n        mlp_activation: Optional[str] = None,\n        mlp_dropout: Optional[float] = None,\n        mlp_batchnorm: Optional[bool] = None,\n        mlp_batchnorm_last: Optional[bool] = None,\n        mlp_linear_first: Optional[bool] = None,\n    ):\n        super(TabFastFormer, self).__init__(\n            column_idx=column_idx,\n            cat_embed_input=cat_embed_input,\n            cat_embed_dropout=cat_embed_dropout,\n            use_cat_bias=use_cat_bias,\n            cat_embed_activation=cat_embed_activation,\n            shared_embed=shared_embed,\n            add_shared_embed=add_shared_embed,\n            frac_shared_embed=frac_shared_embed,\n            continuous_cols=continuous_cols,\n            cont_norm_layer=cont_norm_layer,\n            embed_continuous_method=embed_continuous_method,\n            cont_embed_dropout=cont_embed_dropout,\n            cont_embed_activation=cont_embed_activation,\n            input_dim=input_dim,\n            quantization_setup=quantization_setup,\n            n_frequencies=n_frequencies,\n            sigma=sigma,\n            share_last_layer=share_last_layer,\n            full_embed_dropout=full_embed_dropout,\n        )\n\n        self.n_heads = n_heads\n        self.use_bias = use_bias\n        self.n_blocks = n_blocks\n        self.attn_dropout = attn_dropout\n        self.ff_dropout = ff_dropout\n        self.ff_factor = ff_factor\n        self.share_qv_weights = share_qv_weights\n        self.share_weights = share_weights\n        self.transformer_activation = transformer_activation\n\n        self.mlp_hidden_dims = mlp_hidden_dims\n        self.mlp_activation = mlp_activation\n        self.mlp_dropout = mlp_dropout\n        self.mlp_batchnorm = mlp_batchnorm\n        self.mlp_batchnorm_last = mlp_batchnorm_last\n        self.mlp_linear_first = mlp_linear_first\n\n        self.with_cls_token = \"cls_token\" in column_idx\n        self.n_cat = len(cat_embed_input) if cat_embed_input is not None else 0\n        self.n_cont = len(continuous_cols) if continuous_cols is not None else 0\n        self.n_feats = self.n_cat + self.n_cont\n\n        # Embeddings are instantiated at the base model\n        # Transformer blocks\n        self.encoder = nn.Sequential()\n        first_fastformer_block = FastFormerEncoder(\n            input_dim,\n            n_heads,\n            use_bias,\n            attn_dropout,\n            ff_dropout,\n            ff_factor,\n            share_qv_weights,\n            transformer_activation,\n        )\n        self.encoder.add_module(\"fastformer_block0\", first_fastformer_block)\n        for i in range(1, n_blocks):\n            if share_weights:\n                self.encoder.add_module(\n                    \"fastformer_block\" + str(i), first_fastformer_block\n                )\n            else:\n                self.encoder.add_module(\n                    \"fastformer_block\" + str(i),\n                    FastFormerEncoder(\n                        input_dim,\n                        n_heads,\n                        use_bias,\n                        attn_dropout,\n                        ff_dropout,\n                        ff_factor,\n                        share_qv_weights,\n                        transformer_activation,\n                    ),\n                )\n\n        self.mlp_first_hidden_dim = (\n            self.input_dim if self.with_cls_token else (self.n_feats * self.input_dim)\n        )\n\n        # Mlp: adding an MLP on top of the Resnet blocks is optional and\n        # therefore all related params are optional\n        if self.mlp_hidden_dims is not None:\n            self.mlp = MLP(\n                d_hidden=[self.mlp_first_hidden_dim] + self.mlp_hidden_dims,\n                activation=(\n                    \"relu\" if self.mlp_activation is None else self.mlp_activation\n                ),\n                dropout=0.0 if self.mlp_dropout is None else self.mlp_dropout,\n                batchnorm=False if self.mlp_batchnorm is None else self.mlp_batchnorm,\n                batchnorm_last=(\n                    False\n                    if self.mlp_batchnorm_last is None\n                    else self.mlp_batchnorm_last\n                ),\n                linear_first=(\n                    False if self.mlp_linear_first is None else self.mlp_linear_first\n                ),\n            )\n        else:\n            self.mlp = None\n\n    def forward(self, X: Tensor) -&gt; Tensor:\n        x = self._get_embeddings(X)\n        x = self.encoder(x)\n        if self.with_cls_token:\n            x = x[:, 0, :]\n        else:\n            x = x.flatten(1)\n        if self.mlp is not None:\n            x = self.mlp(x)\n        return x\n\n    @property\n    def output_dim(self) -&gt; int:\n        r\"\"\"The output dimension of the model. This is a required property\n        neccesary to build the `WideDeep` class\n        \"\"\"\n        return (\n            self.mlp_hidden_dims[-1]\n            if self.mlp_hidden_dims is not None\n            else self.mlp_first_hidden_dim\n        )\n\n    @property\n    def attention_weights(self) -&gt; List[Tuple[Tensor, Tensor]]:\n        r\"\"\"List with the attention weights. Each element of the list is a\n        tuple where the first and second elements are the $\\alpha$\n        and $\\beta$ attention weights in the paper.\n\n        The shape of the attention weights is $(N, H, F)$ where $N$ is the\n        batch size, $H$ is the number of attention heads and $F$ is the\n        number of features/columns in the dataset\n        \"\"\"\n        if self.share_weights:\n            attention_weights = [self.encoder[0].attn.attn_weight]\n        else:\n            attention_weights = [blk.attn.attn_weights for blk in self.encoder]\n        return attention_weights\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.transformers.tab_fastformer.TabFastFormer.output_dim","title":"output_dim  <code>property</code>","text":"<pre><code>output_dim\n</code></pre> <p>The output dimension of the model. This is a required property neccesary to build the <code>WideDeep</code> class</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.transformers.tab_fastformer.TabFastFormer.attention_weights","title":"attention_weights  <code>property</code>","text":"<pre><code>attention_weights\n</code></pre> <p>List with the attention weights. Each element of the list is a tuple where the first and second elements are the \\(\\alpha\\) and \\(\\beta\\) attention weights in the paper.</p> <p>The shape of the attention weights is \\((N, H, F)\\) where \\(N\\) is the batch size, \\(H\\) is the number of attention heads and \\(F\\) is the number of features/columns in the dataset</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.text.rnns.basic_rnn.BasicRNN","title":"BasicRNN","text":"<p>               Bases: <code>BaseWDModelComponent</code></p> <p>Standard text classifier/regressor comprised by a stack of RNNs (LSTMs or GRUs) that can be used as the <code>deeptext</code> component of a Wide &amp; Deep model or independently by itself.</p> <p>In addition, there is the option to add a Fully Connected (FC) set of dense layers on top of the stack of RNNs</p> <p>Parameters:</p> Name Type Description Default <code>vocab_size</code> <code>int</code> <p>Number of words in the vocabulary</p> required <code>embed_dim</code> <code>Optional[int]</code> <p>Dimension of the word embeddings if non-pretained word vectors are used</p> <code>None</code> <code>embed_matrix</code> <code>Optional[ndarray]</code> <p>Pretrained word embeddings</p> <code>None</code> <code>embed_trainable</code> <code>bool</code> <p>Boolean indicating if the pretrained embeddings are trainable</p> <code>True</code> <code>rnn_type</code> <code>Literal[lstm, gru]</code> <p>String indicating the type of RNN to use. One of 'lstm' or 'gru'</p> <code>'lstm'</code> <code>hidden_dim</code> <code>int</code> <p>Hidden dim of the RNN</p> <code>64</code> <code>n_layers</code> <code>int</code> <p>Number of recurrent layers</p> <code>3</code> <code>rnn_dropout</code> <code>float</code> <p>Dropout for each RNN layer except the last layer</p> <code>0.0</code> <code>bidirectional</code> <code>bool</code> <p>Boolean indicating whether the staked RNNs are bidirectional</p> <code>False</code> <code>use_hidden_state</code> <code>bool</code> <p>Boolean indicating whether to use the final hidden state or the RNN's output as predicting features. Typically the former is used.</p> <code>True</code> <code>padding_idx</code> <code>int</code> <p>index of the padding token in the padded-tokenised sequences. The <code>TextPreprocessor</code> class within this library uses fastai's tokenizer where the token index 0 is reserved for the 'unknown' word token. Therefore, the default value is set to 1.</p> <code>1</code> <code>head_hidden_dims</code> <code>Optional[List[int]]</code> <p>List with the sizes of the dense layers in the head e.g: [128, 64]</p> <code>None</code> <code>head_activation</code> <code>str</code> <p>Activation function for the dense layers in the head. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> <code>'relu'</code> <code>head_dropout</code> <code>Optional[float]</code> <p>Dropout of the dense layers in the head</p> <code>None</code> <code>head_batchnorm</code> <code>bool</code> <p>Boolean indicating whether or not to include batch normalization in the dense layers that form the 'rnn_mlp'</p> <code>False</code> <code>head_batchnorm_last</code> <code>bool</code> <p>Boolean indicating whether or not to apply batch normalization to the last of the dense layers in the head</p> <code>False</code> <code>head_linear_first</code> <code>bool</code> <p>Boolean indicating whether the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code></p> <code>False</code> <p>Attributes:</p> Name Type Description <code>word_embed</code> <code>Module</code> <p>word embedding matrix</p> <code>rnn</code> <code>Module</code> <p>Stack of RNNs</p> <code>rnn_mlp</code> <code>Module</code> <p>Stack of dense layers on top of the RNN. This will only exists if <code>head_layers_dim</code> is not None</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import BasicRNN\n&gt;&gt;&gt; X_text = torch.cat((torch.zeros([5,1]), torch.empty(5, 4).random_(1,4)), axis=1)\n&gt;&gt;&gt; model = BasicRNN(vocab_size=4, hidden_dim=4, n_layers=2, padding_idx=0, embed_dim=4)\n&gt;&gt;&gt; out = model(X_text)\n</code></pre> Source code in <code>pytorch_widedeep/models/text/rnns/basic_rnn.py</code> <pre><code>class BasicRNN(BaseWDModelComponent):\n    r\"\"\"Standard text classifier/regressor comprised by a stack of RNNs\n    (LSTMs or GRUs) that can be used as the `deeptext` component of a Wide &amp;\n    Deep model or independently by itself.\n\n    In addition, there is the option to add a Fully Connected (FC) set of\n    dense layers on top of the stack of RNNs\n\n    Parameters\n    ----------\n    vocab_size: int\n        Number of words in the vocabulary\n    embed_dim: int, Optional, default = None\n        Dimension of the word embeddings if non-pretained word vectors are\n        used\n    embed_matrix: np.ndarray, Optional, default = None\n        Pretrained word embeddings\n    embed_trainable: bool, default = True\n        Boolean indicating if the pretrained embeddings are trainable\n    rnn_type: str, default = 'lstm'\n        String indicating the type of RNN to use. One of _'lstm'_ or _'gru'_\n    hidden_dim: int, default = 64\n        Hidden dim of the RNN\n    n_layers: int, default = 3\n        Number of recurrent layers\n    rnn_dropout: float, default = 0.1\n        Dropout for each RNN layer except the last layer\n    bidirectional: bool, default = True\n        Boolean indicating whether the staked RNNs are bidirectional\n    use_hidden_state: str, default = True\n        Boolean indicating whether to use the final hidden state or the RNN's\n        output as predicting features. Typically the former is used.\n    padding_idx: int, default = 1\n        index of the padding token in the padded-tokenised sequences. The\n        `TextPreprocessor` class within this library uses fastai's tokenizer\n        where the token index 0 is reserved for the _'unknown'_ word token.\n        Therefore, the default value is set to 1.\n    head_hidden_dims: List, Optional, default = None\n        List with the sizes of the dense layers in the head e.g: _[128, 64]_\n    head_activation: str, default = \"relu\"\n        Activation function for the dense layers in the head. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported\n    head_dropout: float, Optional, default = None\n        Dropout of the dense layers in the head\n    head_batchnorm: bool, default = False\n        Boolean indicating whether or not to include batch normalization in\n        the dense layers that form the _'rnn_mlp'_\n    head_batchnorm_last: bool, default = False\n        Boolean indicating whether or not to apply batch normalization to the\n        last of the dense layers in the head\n    head_linear_first: bool, default = False\n        Boolean indicating whether the order of the operations in the dense\n        layer. If `True: [LIN -&gt; ACT -&gt; BN -&gt; DP]`. If `False: [BN -&gt; DP -&gt;\n        LIN -&gt; ACT]`\n\n    Attributes\n    ----------\n    word_embed: nn.Module\n        word embedding matrix\n    rnn: nn.Module\n        Stack of RNNs\n    rnn_mlp: nn.Module\n        Stack of dense layers on top of the RNN. This will only exists if\n        `head_layers_dim` is not None\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.models import BasicRNN\n    &gt;&gt;&gt; X_text = torch.cat((torch.zeros([5,1]), torch.empty(5, 4).random_(1,4)), axis=1)\n    &gt;&gt;&gt; model = BasicRNN(vocab_size=4, hidden_dim=4, n_layers=2, padding_idx=0, embed_dim=4)\n    &gt;&gt;&gt; out = model(X_text)\n    \"\"\"\n\n    def __init__(\n        self,\n        vocab_size: int,\n        embed_dim: Optional[int] = None,\n        embed_matrix: Optional[np.ndarray] = None,\n        embed_trainable: bool = True,\n        rnn_type: Literal[\"lstm\", \"gru\"] = \"lstm\",\n        hidden_dim: int = 64,\n        n_layers: int = 3,\n        rnn_dropout: float = 0.0,\n        bidirectional: bool = False,\n        use_hidden_state: bool = True,\n        padding_idx: int = 1,\n        head_hidden_dims: Optional[List[int]] = None,\n        head_activation: str = \"relu\",\n        head_dropout: Optional[float] = None,\n        head_batchnorm: bool = False,\n        head_batchnorm_last: bool = False,\n        head_linear_first: bool = False,\n    ):\n        super(BasicRNN, self).__init__()\n\n        if embed_dim is None and embed_matrix is None:\n            raise ValueError(\n                \"If no 'embed_matrix' is passed, the embedding dimension must\"\n                \"be specified with 'embed_dim'\"\n            )\n\n        if rnn_type.lower() not in [\"lstm\", \"gru\"]:\n            raise ValueError(\n                f\"'rnn_type' must be 'lstm' or 'gru', got {rnn_type} instead\"\n            )\n\n        if (\n            embed_dim is not None\n            and embed_matrix is not None\n            and not embed_dim == embed_matrix.shape[1]\n        ):\n            warnings.warn(\n                \"the input embedding dimension {} and the dimension of the \"\n                \"pretrained embeddings {} do not match. The pretrained embeddings \"\n                \"dimension ({}) will be used\".format(\n                    embed_dim, embed_matrix.shape[1], embed_matrix.shape[1]\n                ),\n                UserWarning,\n            )\n\n        self.vocab_size = vocab_size\n        self.embed_trainable = embed_trainable\n        self.embed_dim = embed_dim\n\n        self.rnn_type = rnn_type\n        self.hidden_dim = hidden_dim\n        self.n_layers = n_layers\n        self.rnn_dropout = rnn_dropout\n        self.bidirectional = bidirectional\n        self.use_hidden_state = use_hidden_state\n        self.padding_idx = padding_idx\n\n        self.head_hidden_dims = head_hidden_dims\n        self.head_activation = head_activation\n        self.head_dropout = head_dropout\n        self.head_batchnorm = head_batchnorm\n        self.head_batchnorm_last = head_batchnorm_last\n        self.head_linear_first = head_linear_first\n\n        # Embeddings\n        if embed_matrix is not None:\n            self.word_embed, self.embed_dim = self._set_embeddings(embed_matrix)\n        else:\n            assert self.embed_dim is not None\n            self.word_embed = nn.Embedding(\n                self.vocab_size, self.embed_dim, padding_idx=self.padding_idx\n            )\n\n        # RNN\n        rnn_params = {\n            \"input_size\": self.embed_dim,\n            \"hidden_size\": hidden_dim,\n            \"num_layers\": n_layers,\n            \"bidirectional\": bidirectional,\n            \"dropout\": rnn_dropout,\n            \"batch_first\": True,\n        }\n        if self.rnn_type.lower() == \"lstm\":\n            self.rnn: Union[nn.LSTM, nn.GRU] = nn.LSTM(**rnn_params)\n        elif self.rnn_type.lower() == \"gru\":\n            self.rnn = nn.GRU(**rnn_params)\n        else:\n            raise ValueError(\n                f\"'rnn_type' must be 'lstm' or 'gru', got {self.rnn_type} instead\"\n            )\n\n        self.rnn_output_dim = hidden_dim * 2 if bidirectional else hidden_dim\n\n        # FC-Head (Mlp)\n        if self.head_hidden_dims is not None:\n            head_hidden_dims = [self.rnn_output_dim] + self.head_hidden_dims\n            self.rnn_mlp: Union[MLP, nn.Identity] = MLP(\n                head_hidden_dims,\n                head_activation,\n                head_dropout,\n                head_batchnorm,\n                head_batchnorm_last,\n                head_linear_first,\n            )\n        else:\n            # simple hack to add readability in the forward pass\n            self.rnn_mlp = nn.Identity()\n\n    def forward(self, X: Tensor) -&gt; Tensor:\n        embed = self.word_embed(X.long())\n\n        if self.rnn_type.lower() == \"lstm\":\n            o, (h, c) = self.rnn(embed)\n        elif self.rnn_type.lower() == \"gru\":\n            o, h = self.rnn(embed)\n        else:\n            raise ValueError(\n                f\"'rnn_type' must be 'lstm' or 'gru', got {self.rnn_type} instead\"\n            )\n\n        processed_outputs = self._process_rnn_outputs(o, h)\n\n        return self.rnn_mlp(processed_outputs)\n\n    @property\n    def output_dim(self) -&gt; int:\n        r\"\"\"The output dimension of the model. This is a required property\n        neccesary to build the `WideDeep` class\n        \"\"\"\n        return (\n            self.head_hidden_dims[-1]\n            if self.head_hidden_dims is not None\n            else self.rnn_output_dim\n        )\n\n    def _set_embeddings(\n        self, embed_matrix: Optional[np.ndarray] = None\n    ) -&gt; Tuple[nn.Module, int]:\n        if embed_matrix is not None:\n            assert (\n                embed_matrix.dtype == \"float32\"\n            ), \"'embed_matrix' must be of dtype 'float32', got dtype '{}'\".format(\n                str(embed_matrix.dtype)\n            )\n            word_embed = nn.Embedding(\n                self.vocab_size, embed_matrix.shape[1], padding_idx=self.padding_idx\n            )\n            if self.embed_trainable:\n                word_embed.weight = nn.Parameter(\n                    torch.tensor(embed_matrix), requires_grad=True\n                )\n            else:\n                word_embed.weight = nn.Parameter(\n                    torch.tensor(embed_matrix), requires_grad=False\n                )\n            embed_dim = embed_matrix.shape[1]\n        else:\n            assert self.embed_dim is not None\n            word_embed = nn.Embedding(\n                self.vocab_size, self.embed_dim, padding_idx=self.padding_idx\n            )\n            embed_dim = self.embed_dim\n\n        return word_embed, embed_dim\n\n    def _process_rnn_outputs(self, output: Tensor, hidden: Tensor) -&gt; Tensor:\n        output = output.permute(1, 0, 2)\n        if self.bidirectional:\n            processed_outputs = (\n                torch.cat((hidden[-2], hidden[-1]), dim=1)\n                if self.use_hidden_state\n                else output[-1]\n            )\n        else:\n            processed_outputs = hidden[-1] if self.use_hidden_state else output[-1]\n\n        return processed_outputs\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.text.rnns.basic_rnn.BasicRNN.output_dim","title":"output_dim  <code>property</code>","text":"<pre><code>output_dim\n</code></pre> <p>The output dimension of the model. This is a required property neccesary to build the <code>WideDeep</code> class</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.text.rnns.attentive_rnn.AttentiveRNN","title":"AttentiveRNN","text":"<p>               Bases: <code>BasicRNN</code></p> <p>Text classifier/regressor comprised by a stack of RNNs (LSTMs or GRUs) plus an attention layer. This model can be used as the <code>deeptext</code> component of a Wide &amp; Deep model or independently by itself.</p> <p>In addition, there is the option to add a Fully Connected (FC) set of dense layers on top of attention layer</p> <p>Parameters:</p> Name Type Description Default <code>vocab_size</code> <code>int</code> <p>Number of words in the vocabulary</p> required <code>embed_dim</code> <code>Optional[int]</code> <p>Dimension of the word embeddings if non-pretained word vectors are used</p> <code>None</code> <code>embed_matrix</code> <code>Optional[ndarray]</code> <p>Pretrained word embeddings</p> <code>None</code> <code>embed_trainable</code> <code>bool</code> <p>Boolean indicating if the pretrained embeddings are trainable</p> <code>True</code> <code>rnn_type</code> <code>Literal[lstm, gru]</code> <p>String indicating the type of RNN to use. One of 'lstm' or 'gru'</p> <code>'lstm'</code> <code>hidden_dim</code> <code>int</code> <p>Hidden dim of the RNN</p> <code>64</code> <code>n_layers</code> <code>int</code> <p>Number of recurrent layers</p> <code>3</code> <code>rnn_dropout</code> <code>float</code> <p>Dropout for each RNN layer except the last layer</p> <code>0.1</code> <code>bidirectional</code> <code>bool</code> <p>Boolean indicating whether the staked RNNs are bidirectional</p> <code>False</code> <code>use_hidden_state</code> <code>bool</code> <p>Boolean indicating whether to use the final hidden state or the RNN's output as predicting features. Typically the former is used.</p> <code>True</code> <code>padding_idx</code> <code>int</code> <p>index of the padding token in the padded-tokenised sequences. The <code>TextPreprocessor</code> class within this library uses fastai's tokenizer where the token index 0 is reserved for the 'unknown' word token. Therefore, the default value is set to 1.</p> <code>1</code> <code>attn_concatenate</code> <code>bool</code> <p>Boolean indicating if the input to the attention mechanism will be the output of the RNN or the output of the RNN concatenated with the last hidden state.</p> <code>True</code> <code>attn_dropout</code> <code>float</code> <p>Internal dropout for the attention mechanism</p> <code>0.1</code> <code>head_hidden_dims</code> <code>Optional[List[int]]</code> <p>List with the sizes of the dense layers in the head e.g: [128, 64]</p> <code>None</code> <code>head_activation</code> <code>str</code> <p>Activation function for the dense layers in the head. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> <code>'relu'</code> <code>head_dropout</code> <code>Optional[float]</code> <p>Dropout of the dense layers in the head</p> <code>None</code> <code>head_batchnorm</code> <code>bool</code> <p>Boolean indicating whether or not to include batch normalization in the dense layers that form the 'rnn_mlp'</p> <code>False</code> <code>head_batchnorm_last</code> <code>bool</code> <p>Boolean indicating whether or not to apply batch normalization to the last of the dense layers in the head</p> <code>False</code> <code>head_linear_first</code> <code>bool</code> <p>Boolean indicating whether the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code></p> <code>False</code> <p>Attributes:</p> Name Type Description <code>word_embed</code> <code>Module</code> <p>word embedding matrix</p> <code>rnn</code> <code>Module</code> <p>Stack of RNNs</p> <code>rnn_mlp</code> <code>Module</code> <p>Stack of dense layers on top of the RNN. This will only exists if <code>head_layers_dim</code> is not <code>None</code></p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import AttentiveRNN\n&gt;&gt;&gt; X_text = torch.cat((torch.zeros([5,1]), torch.empty(5, 4).random_(1,4)), axis=1)\n&gt;&gt;&gt; model = AttentiveRNN(vocab_size=4, hidden_dim=4, n_layers=2, padding_idx=0, embed_dim=4)\n&gt;&gt;&gt; out = model(X_text)\n</code></pre> Source code in <code>pytorch_widedeep/models/text/rnns/attentive_rnn.py</code> <pre><code>class AttentiveRNN(BasicRNN):\n    r\"\"\"Text classifier/regressor comprised by a stack of RNNs\n    (LSTMs or GRUs) plus an attention layer. This model can be used as the\n    `deeptext` component of a Wide &amp; Deep model or independently by\n    itself.\n\n    In addition, there is the option to add a Fully Connected (FC) set of dense\n    layers on top of attention layer\n\n    Parameters\n    ----------\n    vocab_size: int\n        Number of words in the vocabulary\n    embed_dim: int, Optional, default = None\n        Dimension of the word embeddings if non-pretained word vectors are\n        used\n    embed_matrix: np.ndarray, Optional, default = None\n        Pretrained word embeddings\n    embed_trainable: bool, default = True\n        Boolean indicating if the pretrained embeddings are trainable\n    rnn_type: str, default = 'lstm'\n        String indicating the type of RNN to use. One of _'lstm'_ or _'gru'_\n    hidden_dim: int, default = 64\n        Hidden dim of the RNN\n    n_layers: int, default = 3\n        Number of recurrent layers\n    rnn_dropout: float, default = 0.1\n        Dropout for each RNN layer except the last layer\n    bidirectional: bool, default = True\n        Boolean indicating whether the staked RNNs are bidirectional\n    use_hidden_state: str, default = True\n        Boolean indicating whether to use the final hidden state or the RNN's\n        output as predicting features. Typically the former is used.\n    padding_idx: int, default = 1\n        index of the padding token in the padded-tokenised sequences. The\n        `TextPreprocessor` class within this library uses fastai's\n        tokenizer where the token index 0 is reserved for the _'unknown'_\n        word token. Therefore, the default value is set to 1.\n    attn_concatenate: bool, default = True\n        Boolean indicating if the input to the attention mechanism will be the\n        output of the RNN or the output of the RNN concatenated with the last\n        hidden state.\n    attn_dropout: float, default = 0.1\n        Internal dropout for the attention mechanism\n    head_hidden_dims: List, Optional, default = None\n        List with the sizes of the dense layers in the head e.g: _[128, 64]_\n    head_activation: str, default = \"relu\"\n        Activation function for the dense layers in the head. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported\n    head_dropout: float, Optional, default = None\n        Dropout of the dense layers in the head\n    head_batchnorm: bool, default = False\n        Boolean indicating whether or not to include batch normalization in\n        the dense layers that form the _'rnn_mlp'_\n    head_batchnorm_last: bool, default = False\n        Boolean indicating whether or not to apply batch normalization to the\n        last of the dense layers in the head\n    head_linear_first: bool, default = False\n        Boolean indicating whether the order of the operations in the dense\n        layer. If `True: [LIN -&gt; ACT -&gt; BN -&gt; DP]`. If `False: [BN -&gt; DP -&gt;\n        LIN -&gt; ACT]`\n\n    Attributes\n    ----------\n    word_embed: nn.Module\n        word embedding matrix\n    rnn: nn.Module\n        Stack of RNNs\n    rnn_mlp: nn.Module\n        Stack of dense layers on top of the RNN. This will only exists if\n        `head_layers_dim` is not `None`\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.models import AttentiveRNN\n    &gt;&gt;&gt; X_text = torch.cat((torch.zeros([5,1]), torch.empty(5, 4).random_(1,4)), axis=1)\n    &gt;&gt;&gt; model = AttentiveRNN(vocab_size=4, hidden_dim=4, n_layers=2, padding_idx=0, embed_dim=4)\n    &gt;&gt;&gt; out = model(X_text)\n    \"\"\"\n\n    def __init__(\n        self,\n        vocab_size: int,\n        embed_dim: Optional[int] = None,\n        embed_matrix: Optional[np.ndarray] = None,\n        embed_trainable: bool = True,\n        rnn_type: Literal[\"lstm\", \"gru\"] = \"lstm\",\n        hidden_dim: int = 64,\n        n_layers: int = 3,\n        rnn_dropout: float = 0.1,\n        bidirectional: bool = False,\n        use_hidden_state: bool = True,\n        padding_idx: int = 1,\n        attn_concatenate: bool = True,\n        attn_dropout: float = 0.1,\n        head_hidden_dims: Optional[List[int]] = None,\n        head_activation: str = \"relu\",\n        head_dropout: Optional[float] = None,\n        head_batchnorm: bool = False,\n        head_batchnorm_last: bool = False,\n        head_linear_first: bool = False,\n    ):\n        super(AttentiveRNN, self).__init__(\n            vocab_size=vocab_size,\n            embed_dim=embed_dim,\n            embed_matrix=embed_matrix,\n            embed_trainable=embed_trainable,\n            rnn_type=rnn_type,\n            hidden_dim=hidden_dim,\n            n_layers=n_layers,\n            rnn_dropout=rnn_dropout,\n            bidirectional=bidirectional,\n            use_hidden_state=use_hidden_state,\n            padding_idx=padding_idx,\n            head_hidden_dims=head_hidden_dims,\n            head_activation=head_activation,\n            head_dropout=head_dropout,\n            head_batchnorm=head_batchnorm,\n            head_batchnorm_last=head_batchnorm_last,\n            head_linear_first=head_linear_first,\n        )\n\n        # Embeddings and RNN defined in the BasicRNN inherited class\n\n        # Attention\n        self.attn_concatenate = attn_concatenate\n        self.attn_dropout = attn_dropout\n\n        if bidirectional and attn_concatenate:\n            self.rnn_output_dim = hidden_dim * 4\n        elif bidirectional or attn_concatenate:\n            self.rnn_output_dim = hidden_dim * 2\n        else:\n            self.rnn_output_dim = hidden_dim\n        self.attn = ContextAttention(\n            self.rnn_output_dim, attn_dropout, sum_along_seq=True\n        )\n\n        # FC-Head (Mlp)\n        if self.head_hidden_dims is not None:\n            head_hidden_dims = [self.rnn_output_dim] + self.head_hidden_dims\n            self.rnn_mlp = MLP(\n                head_hidden_dims,\n                head_activation,\n                head_dropout,\n                head_batchnorm,\n                head_batchnorm_last,\n                head_linear_first,\n            )\n\n    def _process_rnn_outputs(self, output: Tensor, hidden: Tensor) -&gt; Tensor:\n        if self.attn_concatenate:\n            if self.bidirectional:\n                bi_hidden = torch.cat((hidden[-2], hidden[-1]), dim=1)\n                attn_inp = torch.cat(\n                    [output, bi_hidden.unsqueeze(1).expand_as(output)], dim=2\n                )\n            else:\n                attn_inp = torch.cat(\n                    [output, hidden[-1].unsqueeze(1).expand_as(output)], dim=2\n                )\n        else:\n            attn_inp = output\n\n        return self.attn(attn_inp)\n\n    @property\n    def attention_weights(self) -&gt; List:\n        r\"\"\"List with the attention weights\n\n        The shape of the attention weights is $(N, S)$, where $N$ is the batch\n        size and $S$ is the length of the sequence\n        \"\"\"\n        return self.attn.attn_weights\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.text.rnns.attentive_rnn.AttentiveRNN.attention_weights","title":"attention_weights  <code>property</code>","text":"<pre><code>attention_weights\n</code></pre> <p>List with the attention weights</p> <p>The shape of the attention weights is \\((N, S)\\), where \\(N\\) is the batch size and \\(S\\) is the length of the sequence</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.text.rnns.stacked_attentive_rnn.StackedAttentiveRNN","title":"StackedAttentiveRNN","text":"<p>               Bases: <code>BaseWDModelComponent</code></p> <p>Text classifier/regressor comprised by a stack of blocks: <code>[RNN + Attention]</code>. This can be used as the <code>deeptext</code> component of a Wide &amp; Deep model or independently by itself.</p> <p>In addition, there is the option to add a Fully Connected (FC) set of dense layers on top of the attentiob blocks</p> <p>Parameters:</p> Name Type Description Default <code>vocab_size</code> <code>int</code> <p>Number of words in the vocabulary</p> required <code>embed_dim</code> <code>Optional[int]</code> <p>Dimension of the word embeddings if non-pretained word vectors are used</p> <code>None</code> <code>embed_matrix</code> <code>Optional[ndarray]</code> <p>Pretrained word embeddings</p> <code>None</code> <code>embed_trainable</code> <code>bool</code> <p>Boolean indicating if the pretrained embeddings are trainable</p> <code>True</code> <code>rnn_type</code> <code>Literal[lstm, gru]</code> <p>String indicating the type of RNN to use. One of 'lstm' or 'gru'</p> <code>'lstm'</code> <code>hidden_dim</code> <code>int</code> <p>Hidden dim of the RNN</p> <code>64</code> <code>bidirectional</code> <code>bool</code> <p>Boolean indicating whether the staked RNNs are bidirectional</p> <code>False</code> <code>padding_idx</code> <code>int</code> <p>index of the padding token in the padded-tokenised sequences. The <code>TextPreprocessor</code> class within this library uses fastai's tokenizer where the token index 0 is reserved for the 'unknown' word token. Therefore, the default value is set to 1.</p> <code>1</code> <code>n_blocks</code> <code>int</code> <p>Number of attention blocks. Each block is comprised by an RNN and a Context Attention Encoder</p> <code>3</code> <code>attn_concatenate</code> <code>bool</code> <p>Boolean indicating if the input to the attention mechanism will be the output of the RNN or the output of the RNN concatenated with the last hidden state or simply</p> <code>False</code> <code>attn_dropout</code> <code>float</code> <p>Internal dropout for the attention mechanism</p> <code>0.1</code> <code>with_addnorm</code> <code>bool</code> <p>Boolean indicating if the output of each block will be added to the input and normalised</p> <code>False</code> <code>head_hidden_dims</code> <code>Optional[List[int]]</code> <p>List with the sizes of the dense layers in the head e.g: [128, 64]</p> <code>None</code> <code>head_activation</code> <code>str</code> <p>Activation function for the dense layers in the head. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> <code>'relu'</code> <code>head_dropout</code> <code>Optional[float]</code> <p>Dropout of the dense layers in the head</p> <code>None</code> <code>head_batchnorm</code> <code>bool</code> <p>Boolean indicating whether or not to include batch normalization in the dense layers that form the 'rnn_mlp'</p> <code>False</code> <code>head_batchnorm_last</code> <code>bool</code> <p>Boolean indicating whether or not to apply batch normalization to the last of the dense layers in the head</p> <code>False</code> <code>head_linear_first</code> <code>bool</code> <p>Boolean indicating whether the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code></p> <code>False</code> <p>Attributes:</p> Name Type Description <code>word_embed</code> <code>Module</code> <p>word embedding matrix</p> <code>rnn</code> <code>Module</code> <p>Stack of RNNs</p> <code>rnn_mlp</code> <code>Module</code> <p>Stack of dense layers on top of the RNN. This will only exists if <code>head_layers_dim</code> is not <code>None</code></p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import StackedAttentiveRNN\n&gt;&gt;&gt; X_text = torch.cat((torch.zeros([5,1]), torch.empty(5, 4).random_(1,4)), axis=1)\n&gt;&gt;&gt; model = StackedAttentiveRNN(vocab_size=4, hidden_dim=4, padding_idx=0, embed_dim=4)\n&gt;&gt;&gt; out = model(X_text)\n</code></pre> Source code in <code>pytorch_widedeep/models/text/rnns/stacked_attentive_rnn.py</code> <pre><code>class StackedAttentiveRNN(BaseWDModelComponent):\n    r\"\"\"Text classifier/regressor comprised by a stack of blocks:\n    `[RNN + Attention]`. This can be used as the `deeptext` component of a\n    Wide &amp; Deep model or independently by itself.\n\n    In addition, there is the option to add a Fully Connected (FC) set of\n    dense layers on top of the attentiob blocks\n\n    Parameters\n    ----------\n    vocab_size: int\n        Number of words in the vocabulary\n    embed_dim: int, Optional, default = None\n        Dimension of the word embeddings if non-pretained word vectors are\n        used\n    embed_matrix: np.ndarray, Optional, default = None\n        Pretrained word embeddings\n    embed_trainable: bool, default = True\n        Boolean indicating if the pretrained embeddings are trainable\n    rnn_type: str, default = 'lstm'\n        String indicating the type of RNN to use. One of 'lstm' or 'gru'\n    hidden_dim: int, default = 64\n        Hidden dim of the RNN\n    bidirectional: bool, default = True\n        Boolean indicating whether the staked RNNs are bidirectional\n    padding_idx: int, default = 1\n        index of the padding token in the padded-tokenised sequences. The\n        `TextPreprocessor` class within this library uses fastai's\n        tokenizer where the token index 0 is reserved for the _'unknown'_\n        word token. Therefore, the default value is set to 1.\n    n_blocks: int, default = 3\n        Number of attention blocks. Each block is comprised by an RNN and a\n        Context Attention Encoder\n    attn_concatenate: bool, default = True\n        Boolean indicating if the input to the attention mechanism will be the\n        output of the RNN or the output of the RNN concatenated with the last\n        hidden state or simply\n    attn_dropout: float, default = 0.1\n        Internal dropout for the attention mechanism\n    with_addnorm: bool, default = False\n        Boolean indicating if the output of each block will be added to the\n        input and normalised\n    head_hidden_dims: List, Optional, default = None\n        List with the sizes of the dense layers in the head e.g: [128, 64]\n    head_activation: str, default = \"relu\"\n        Activation function for the dense layers in the head. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported\n    head_dropout: float, Optional, default = None\n        Dropout of the dense layers in the head\n    head_batchnorm: bool, default = False\n        Boolean indicating whether or not to include batch normalization in\n        the dense layers that form the _'rnn_mlp'_\n    head_batchnorm_last: bool, default = False\n        Boolean indicating whether or not to apply batch normalization to the\n        last of the dense layers in the head\n    head_linear_first: bool, default = False\n        Boolean indicating whether the order of the operations in the dense\n        layer. If `True: [LIN -&gt; ACT -&gt; BN -&gt; DP]`. If `False: [BN -&gt; DP -&gt;\n        LIN -&gt; ACT]`\n\n    Attributes\n    ----------\n    word_embed: nn.Module\n        word embedding matrix\n    rnn: nn.Module\n        Stack of RNNs\n    rnn_mlp: nn.Module\n        Stack of dense layers on top of the RNN. This will only exists if\n        `head_layers_dim` is not `None`\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.models import StackedAttentiveRNN\n    &gt;&gt;&gt; X_text = torch.cat((torch.zeros([5,1]), torch.empty(5, 4).random_(1,4)), axis=1)\n    &gt;&gt;&gt; model = StackedAttentiveRNN(vocab_size=4, hidden_dim=4, padding_idx=0, embed_dim=4)\n    &gt;&gt;&gt; out = model(X_text)\n    \"\"\"\n\n    def __init__(\n        self,\n        vocab_size: int,\n        embed_dim: Optional[int] = None,\n        embed_matrix: Optional[np.ndarray] = None,\n        embed_trainable: bool = True,\n        rnn_type: Literal[\"lstm\", \"gru\"] = \"lstm\",\n        hidden_dim: int = 64,\n        bidirectional: bool = False,\n        padding_idx: int = 1,\n        n_blocks: int = 3,\n        attn_concatenate: bool = False,\n        attn_dropout: float = 0.1,\n        with_addnorm: bool = False,\n        head_hidden_dims: Optional[List[int]] = None,\n        head_activation: str = \"relu\",\n        head_dropout: Optional[float] = None,\n        head_batchnorm: bool = False,\n        head_batchnorm_last: bool = False,\n        head_linear_first: bool = False,\n    ):\n        super(StackedAttentiveRNN, self).__init__()\n\n        if (\n            embed_dim is not None\n            and embed_matrix is not None\n            and not embed_dim == embed_matrix.shape[1]\n        ):\n            warnings.warn(\n                \"the input embedding dimension {} and the dimension of the \"\n                \"pretrained embeddings {} do not match. The pretrained embeddings \"\n                \"dimension ({}) will be used\".format(\n                    embed_dim, embed_matrix.shape[1], embed_matrix.shape[1]\n                ),\n                UserWarning,\n            )\n\n        if rnn_type.lower() not in [\"lstm\", \"gru\"]:\n            raise ValueError(\n                f\"'rnn_type' must be 'lstm' or 'gru', got {rnn_type} instead\"\n            )\n\n        self.vocab_size = vocab_size\n        self.embed_trainable = embed_trainable\n        self.embed_dim = embed_dim\n\n        self.rnn_type = rnn_type\n        self.hidden_dim = hidden_dim\n        self.bidirectional = bidirectional\n        self.padding_idx = padding_idx\n\n        self.n_blocks = n_blocks\n        self.attn_concatenate = attn_concatenate\n        self.attn_dropout = attn_dropout\n        self.with_addnorm = with_addnorm\n\n        self.head_hidden_dims = head_hidden_dims\n        self.head_activation = head_activation\n        self.head_dropout = head_dropout\n        self.head_batchnorm = head_batchnorm\n        self.head_batchnorm_last = head_batchnorm_last\n        self.head_linear_first = head_linear_first\n\n        # Embeddings\n        self.word_embed, self.embed_dim = self._set_embeddings(embed_matrix)\n\n        # Linear Projection: if embed_dim is different that the input of the\n        # attention blocks we add a linear projection\n        if bidirectional and attn_concatenate:\n            self.rnn_output_dim = hidden_dim * 4\n        elif bidirectional or attn_concatenate:\n            self.rnn_output_dim = hidden_dim * 2\n        else:\n            self.rnn_output_dim = hidden_dim\n\n        if self.rnn_output_dim != self.embed_dim:\n            self.embed_proj: Union[nn.Linear, nn.Identity] = nn.Linear(\n                self.embed_dim, self.rnn_output_dim\n            )\n        else:\n            self.embed_proj = nn.Identity()\n\n        # RNN\n        rnn_params = {\n            \"input_size\": self.rnn_output_dim,\n            \"hidden_size\": hidden_dim,\n            \"bidirectional\": bidirectional,\n            \"batch_first\": True,\n        }\n        if self.rnn_type.lower() == \"lstm\":\n            self.rnn: Union[nn.LSTM, nn.GRU] = nn.LSTM(**rnn_params)\n        elif self.rnn_type.lower() == \"gru\":\n            self.rnn = nn.GRU(**rnn_params)\n\n        # FC-Head (Mlp)\n        self.attention_blks = nn.ModuleList()\n        for i in range(n_blocks):\n            self.attention_blks.append(\n                ContextAttentionEncoder(\n                    self.rnn,\n                    self.rnn_output_dim,\n                    attn_dropout,\n                    attn_concatenate,\n                    with_addnorm=with_addnorm if i != n_blocks - 1 else False,\n                    sum_along_seq=i == n_blocks - 1,\n                )\n            )\n\n        # Mlp\n        if self.head_hidden_dims is not None:\n            head_hidden_dims = [self.rnn_output_dim] + self.head_hidden_dims\n            self.rnn_mlp: Union[MLP, nn.Identity] = MLP(\n                head_hidden_dims,\n                head_activation,\n                head_dropout,\n                head_batchnorm,\n                head_batchnorm_last,\n                head_linear_first,\n            )\n        else:\n            # simple hack to add readability in the forward pass\n            self.rnn_mlp = nn.Identity()\n\n    def forward(self, X: Tensor) -&gt; Tensor:  # type: ignore\n        x = self.embed_proj(self.word_embed(X.long()))\n\n        h = nn.init.zeros_(\n            torch.Tensor(2 if self.bidirectional else 1, X.shape[0], self.hidden_dim)\n        ).to(x.device)\n        if self.rnn_type == \"lstm\":\n            c = nn.init.zeros_(\n                torch.Tensor(\n                    2 if self.bidirectional else 1, X.shape[0], self.hidden_dim\n                )\n            ).to(x.device)\n        else:\n            c = None\n\n        for blk in self.attention_blks:\n            x, h, c = blk(x, h, c)\n\n        return self.rnn_mlp(x)\n\n    @property\n    def output_dim(self) -&gt; int:\n        r\"\"\"The output dimension of the model. This is a required property\n        neccesary to build the `WideDeep` class\n        \"\"\"\n        return (\n            self.head_hidden_dims[-1]\n            if self.head_hidden_dims is not None\n            else self.rnn_output_dim\n        )\n\n    @property\n    def attention_weights(self) -&gt; List:\n        r\"\"\"List with the attention weights per block\n\n        The shape of the attention weights is $(N, S)$ Where $N$ is the batch\n        size and $S$ is the length of the sequence\n        \"\"\"\n        return [blk.attn.attn_weights for blk in self.attention_blks]\n\n    def _set_embeddings(\n        self, embed_matrix: Union[Any, np.ndarray]\n    ) -&gt; Tuple[nn.Module, int]:\n        if isinstance(embed_matrix, np.ndarray):\n            assert (\n                embed_matrix.dtype == \"float32\"\n            ), \"'embed_matrix' must be of dtype 'float32', got dtype '{}'\".format(\n                str(embed_matrix.dtype)\n            )\n            word_embed = nn.Embedding(\n                self.vocab_size, embed_matrix.shape[1], padding_idx=self.padding_idx\n            )\n            if self.embed_trainable:\n                word_embed.weight = nn.Parameter(\n                    torch.tensor(embed_matrix), requires_grad=True\n                )\n            else:\n                word_embed.weight = nn.Parameter(\n                    torch.tensor(embed_matrix), requires_grad=False\n                )\n            embed_dim = embed_matrix.shape[1]\n        else:\n            assert self.embed_dim is not None\n            word_embed = nn.Embedding(\n                self.vocab_size, self.embed_dim, padding_idx=self.padding_idx\n            )\n            embed_dim = self.embed_dim\n\n        return word_embed, embed_dim\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.text.rnns.stacked_attentive_rnn.StackedAttentiveRNN.output_dim","title":"output_dim  <code>property</code>","text":"<pre><code>output_dim\n</code></pre> <p>The output dimension of the model. This is a required property neccesary to build the <code>WideDeep</code> class</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.text.rnns.stacked_attentive_rnn.StackedAttentiveRNN.attention_weights","title":"attention_weights  <code>property</code>","text":"<pre><code>attention_weights\n</code></pre> <p>List with the attention weights per block</p> <p>The shape of the attention weights is \\((N, S)\\) Where \\(N\\) is the batch size and \\(S\\) is the length of the sequence</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.text.huggingface_transformers.hf_model.HFModel","title":"HFModel","text":"<p>               Bases: <code>BaseWDModelComponent</code></p> <p>This class is a wrapper around the Hugging Face transformers library. It can be used as the text component of a Wide &amp; Deep model or independently by itself.</p> <p>At the moment only models from the families BERT, RoBERTa, DistilBERT, ALBERT and ELECTRA are supported. This is because this library is designed to address classification and regression tasks and these are the most 'popular' encoder-only models, which have proved to be those that work best for these tasks.</p> <p>Parameters:</p> Name Type Description Default <code>model_name</code> <code>str</code> <p>The model name from the transformers library e.g. 'bert-base-uncased'. Currently supported models are those from the families: BERT, RoBERTa, DistilBERT, ALBERT and ELECTRA.</p> required <code>use_cls_token</code> <code>bool</code> <p>Boolean indicating whether to use the [CLS] token or the mean of the sequence of hidden states as the sentence embedding</p> <code>True</code> <code>trainable_parameters</code> <code>Optional[List[str]]</code> <p>List with the names of the model parameters that will be trained. If None, none of the parameters will be trainable</p> <code>None</code> <code>head_hidden_dims</code> <code>Optional[List[int]]</code> <p>List with the sizes of the dense layers in the head e.g: [128, 64]</p> <code>None</code> <code>head_activation</code> <code>str</code> <p>Activation function for the dense layers in the head. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> <code>'relu'</code> <code>head_dropout</code> <code>Optional[float]</code> <p>Dropout of the dense layers in the head</p> <code>None</code> <code>head_batchnorm</code> <code>bool</code> <p>Boolean indicating whether or not to include batch normalization in the dense layers that form the head</p> <code>False</code> <code>head_batchnorm_last</code> <code>bool</code> <p>Boolean indicating whether or not to apply batch normalization to the last of the dense layers in the head</p> <code>False</code> <code>head_linear_first</code> <code>bool</code> <p>Boolean indicating whether the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code></p> <code>False</code> <code>verbose</code> <code>bool</code> <p>If True, it will print information about the model</p> <code>False</code> <code>**kwargs</code> <p>Additional kwargs to be passed to the model</p> <code>{}</code> <p>Attributes:</p> Name Type Description <code>head</code> <code>Module</code> <p>Stack of dense layers on top of the transformer. This will only exists if <code>head_layers_dim</code> is not None</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import HFModel\n&gt;&gt;&gt; X_text = torch.cat((torch.zeros([5,1]), torch.empty(5, 4).random_(1,4)), axis=1).long()\n&gt;&gt;&gt; model = HFModel(model_name='bert-base-uncased')\n&gt;&gt;&gt; out = model(X_text)\n</code></pre> Source code in <code>pytorch_widedeep/models/text/huggingface_transformers/hf_model.py</code> <pre><code>class HFModel(BaseWDModelComponent):\n    \"\"\"This class is a wrapper around the Hugging Face transformers library. It\n    can be used as the text component of a Wide &amp; Deep model or independently\n    by itself.\n\n    At the moment only models from the families BERT, RoBERTa, DistilBERT,\n    ALBERT and ELECTRA are supported. This is because this library is\n    designed to address classification and regression tasks and these are the\n    most 'popular' encoder-only models, which have proved to be those that\n    work best for these tasks.\n\n    Parameters\n    ----------\n    model_name: str\n        The model name from the transformers library e.g. 'bert-base-uncased'.\n        Currently supported models are those from the families: BERT, RoBERTa,\n        DistilBERT, ALBERT and ELECTRA.\n    use_cls_token: bool, default = True\n        Boolean indicating whether to use the [CLS] token or the mean of the\n        sequence of hidden states as the sentence embedding\n    trainable_parameters: List, Optional, default = None\n        List with the names of the model parameters that will be trained. If\n        None, none of the parameters will be trainable\n    head_hidden_dims: List, Optional, default = None\n        List with the sizes of the dense layers in the head e.g: _[128, 64]_\n    head_activation: str, default = \"relu\"\n        Activation function for the dense layers in the head. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported\n    head_dropout: float, Optional, default = None\n        Dropout of the dense layers in the head\n    head_batchnorm: bool, default = False\n        Boolean indicating whether or not to include batch normalization in the\n        dense layers that form the head\n    head_batchnorm_last: bool, default = False\n        Boolean indicating whether or not to apply batch normalization to the\n        last of the dense layers in the head\n    head_linear_first: bool, default = False\n        Boolean indicating whether the order of the operations in the dense\n        layer. If `True: [LIN -&gt; ACT -&gt; BN -&gt; DP]`. If `False: [BN -&gt; DP -&gt;\n        LIN -&gt; ACT]`\n    verbose: bool, default = False\n        If True, it will print information about the model\n    **kwargs\n        Additional kwargs to be passed to the model\n\n    Attributes\n    ----------\n    head: nn.Module\n        Stack of dense layers on top of the transformer. This will only exists\n        if `head_layers_dim` is not None\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.models import HFModel\n    &gt;&gt;&gt; X_text = torch.cat((torch.zeros([5,1]), torch.empty(5, 4).random_(1,4)), axis=1).long()\n    &gt;&gt;&gt; model = HFModel(model_name='bert-base-uncased')\n    &gt;&gt;&gt; out = model(X_text)\n    \"\"\"\n\n    @alias(\"use_cls_token\", [\"use_special_token\"])\n    def __init__(\n        self,\n        model_name: str,\n        use_cls_token: bool = True,\n        trainable_parameters: Optional[List[str]] = None,\n        head_hidden_dims: Optional[List[int]] = None,\n        head_activation: str = \"relu\",\n        head_dropout: Optional[float] = None,\n        head_batchnorm: bool = False,\n        head_batchnorm_last: bool = False,\n        head_linear_first: bool = False,\n        verbose: bool = False,\n        **kwargs,\n    ):\n        super().__init__()\n\n        # TO DO: add warning regarging ELECTRA as ELECTRA does not have a cls\n        # token.  Research what happens with ELECTRA\n        self.model_name = model_name\n        self.use_cls_token = use_cls_token\n        self.trainable_parameters = trainable_parameters\n        self.head_hidden_dims = head_hidden_dims\n        self.head_activation = head_activation\n        self.head_dropout = head_dropout\n        self.head_batchnorm = head_batchnorm\n        self.head_batchnorm_last = head_batchnorm_last\n        self.head_linear_first = head_linear_first\n        self.verbose = verbose\n        self.kwargs = kwargs\n\n        if self.verbose and self.use_cls_token:\n            warnings.warn(\n                \"The model will use the [CLS] token. Make sure the tokenizer \"\n                \"was run with add_special_tokens=True\",\n                UserWarning,\n            )\n\n        self.model_class = get_model_class(model_name)\n\n        self.config, self.model = get_config_and_model(self.model_name)\n\n        self.output_attention_weights = kwargs.get(\"output_attentions\", False)\n\n        if self.trainable_parameters is not None:\n            for n, p in self.model.named_parameters():\n                p.requires_grad = any([tl in n for tl in self.trainable_parameters])\n\n        # FC-Head (Mlp). Note that the FC head will always be trainable\n        if self.head_hidden_dims is not None:\n            head_hidden_dims = [self.config.hidden_size] + self.head_hidden_dims\n            self.head = MLP(\n                head_hidden_dims,\n                head_activation,\n                head_dropout,\n                head_batchnorm,\n                head_batchnorm_last,\n                head_linear_first,\n            )\n\n    def forward(self, X: Tensor) -&gt; Tensor:\n\n        # this is inefficient since the attention mask is returned by the\n        # tokenizer, but all models in this library use a forward pass that\n        # takes ONLY an input tensor. A fix will be addressed in a future\n        attn_mask = (X != 0).type(torch.int8)\n\n        output = self.model(input_ids=X, attention_mask=attn_mask, **self.kwargs)\n\n        if self.output_attention_weights:\n            # TO CONSIDER: attention weights as a returned object and not an\n            # attribute\n            self.attn_weights = output[\"attentions\"]\n\n        if self.use_cls_token:\n            output = output[0][:, 0, :]\n        else:\n            # Here one can choose to flatten, but unless the sequence length\n            # is very small, flatten will result in a very large output\n            # tensor.\n            output = output[0].mean(dim=1)\n\n        if self.head_hidden_dims is not None:\n            output = self.head(output)\n\n        return output\n\n    @property\n    def output_dim(self) -&gt; int:\n        return (\n            self.head_hidden_dims[-1]\n            if self.head_hidden_dims is not None\n            else self.config.hidden_size\n        )\n\n    @property\n    def attention_weight(self) -&gt; Tensor:\n        r\"\"\"Returns the attention weights if the model was created with the\n        output_attention_weights=True argument. If not, it will raise an\n        AttributeError.\n\n        The shape of the attention weights is $(N, H, F, F)$, where $N$ is the\n        batch size, $H$ is the number of attention heads and $F$ is the\n        sequence length.\n        \"\"\"\n        if not self.output_attention_weights:\n            raise AttributeError(\n                \"The output_attention_weights attribute was not set to True when creating the model object \"\n                \"Please pass an output_attention_weights=True argument when creating the HFModel object\"\n            )\n        return self.attn_weights\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.text.huggingface_transformers.hf_model.HFModel.attention_weight","title":"attention_weight  <code>property</code>","text":"<pre><code>attention_weight\n</code></pre> <p>Returns the attention weights if the model was created with the output_attention_weights=True argument. If not, it will raise an AttributeError.</p> <p>The shape of the attention weights is \\((N, H, F, F)\\), where \\(N\\) is the batch size, \\(H\\) is the number of attention heads and \\(F\\) is the sequence length.</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.image.vision.Vision","title":"Vision","text":"<p>               Bases: <code>BaseWDModelComponent</code></p> <p>Defines a standard image classifier/regressor using a pretrained network or a sequence of convolution layers that can be used as the <code>deepimage</code> component of a Wide &amp; Deep model or independently by itself.</p> <p> NOTE: this class represents the integration  between <code>pytorch-widedeep</code> and <code>torchvision</code>. New architectures will be  available as they are added to <code>torchvision</code>. In a distant future we aim  to bring transformer-based architectures as well. However, simple  CNN-based architectures (and even MLP-based) seem to produce SoTA  results. For the time being, we describe below the options available  through this class</p> <p>Parameters:</p> Name Type Description Default <code>pretrained_model_setup</code> <code>Union[str, Dict[str, Union[str, WeightsEnum]]]</code> <p>Name of the pretrained model. Should be a variant of the following architectures: 'resnet', 'shufflenet', 'resnext', 'wide_resnet', 'regnet', 'densenet', 'mobilenetv3', 'mobilenetv2', 'mnasnet', 'efficientnet' and 'squeezenet'. if <code>pretrained_model_setup = None</code> a basic, fully trainable CNN will be used. Alternatively, since Torchvision 0.13 one can use pretrained models with different weigths. Therefore, <code>pretrained_model_setup</code> can also be dictionary with the name of the model and the weights (e.g. <code>{'resnet50': ResNet50_Weights.DEFAULT}</code> or <code>{'resnet50': \"IMAGENET1K_V2\"}</code>).  Aliased as <code>pretrained_model_name</code>.</p> <code>None</code> <code>n_trainable</code> <code>Optional[int]</code> <p>Number of trainable layers starting from the layer closer to the output neuron(s). Note that this number DOES NOT take into account the so-called 'head' which is ALWAYS trainable. If <code>trainable_params</code> is not None this parameter will be ignored</p> <code>None</code> <code>trainable_params</code> <code>Optional[List[str]]</code> <p>List of strings containing the names (or substring within the name) of the parameters that will be trained. For example, if we use a 'resnet18' pretrained model and we set <code>trainable_params = ['layer4']</code> only the parameters of 'layer4' of the network (and the head, as mentioned before) will be trained. Note that setting this or the previous parameter involves some knowledge of the architecture used.</p> <code>None</code> <code>channel_sizes</code> <code>List[int]</code> <p>List of integers with the channel sizes of a CNN in case we choose not to use a pretrained model</p> <code>[64, 128, 256, 512]</code> <code>kernel_sizes</code> <code>Union[int, List[int]]</code> <p>List of integers with the kernel sizes of a CNN in case we choose not to use a pretrained model. Must be of length equal to <code>len(channel_sizes) - 1</code>.</p> <code>[7, 3, 3, 3]</code> <code>strides</code> <code>Union[int, List[int]]</code> <p>List of integers with the stride sizes of a CNN in case we choose not to use a pretrained model. Must be of length equal to <code>len(channel_sizes) - 1</code>.</p> <code>[2, 1, 1, 1]</code> <code>head_hidden_dims</code> <code>Optional[List[int]]</code> <p>List with the number of neurons per dense layer in the head. e.g: [64,32]</p> <code>None</code> <code>head_activation</code> <code>str</code> <p>Activation function for the dense layers in the head. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> <code>'relu'</code> <code>head_dropout</code> <code>Union[float, List[float]]</code> <p>float indicating the dropout between the dense layers.</p> <code>0.1</code> <code>head_batchnorm</code> <code>bool</code> <p>Boolean indicating whether or not batch normalization will be applied to the dense layers</p> <code>False</code> <code>head_batchnorm_last</code> <code>bool</code> <p>Boolean indicating whether or not batch normalization will be applied to the last of the dense layers</p> <code>False</code> <code>head_linear_first</code> <code>bool</code> <p>Boolean indicating the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code></p> <code>False</code> <p>Attributes:</p> Name Type Description <code>features</code> <code>Module</code> <p>The pretrained model or Standard CNN plus the optional head</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import Vision\n&gt;&gt;&gt; X_img = torch.rand((2,3,224,224))\n&gt;&gt;&gt; model = Vision(channel_sizes=[64, 128], kernel_sizes = [3, 3], strides=[1, 1], head_hidden_dims=[32, 8])\n&gt;&gt;&gt; out = model(X_img)\n</code></pre> Source code in <code>pytorch_widedeep/models/image/vision.py</code> <pre><code>class Vision(BaseWDModelComponent):\n    r\"\"\"Defines a standard image classifier/regressor using a pretrained\n    network or a sequence of convolution layers that can be used as the\n    `deepimage` component of a Wide &amp; Deep model or independently by\n    itself.\n\n    :information_source: **NOTE**: this class represents the integration\n     between `pytorch-widedeep` and `torchvision`. New architectures will be\n     available as they are added to `torchvision`. In a distant future we aim\n     to bring transformer-based architectures as well. However, simple\n     CNN-based architectures (and even MLP-based) seem to produce SoTA\n     results. For the time being, we describe below the options available\n     through this class\n\n    Parameters\n    ----------\n    pretrained_model_setup: Optional, str or dict, default = None\n        Name of the pretrained model. Should be a variant of the following\n        architectures: _'resnet'_, _'shufflenet'_, _'resnext'_,\n        _'wide_resnet'_, _'regnet'_, _'densenet'_, _'mobilenetv3'_,\n        _'mobilenetv2'_, _'mnasnet'_, _'efficientnet'_ and _'squeezenet'_. if\n        `pretrained_model_setup = None` a basic, fully trainable CNN will be\n        used. Alternatively, since Torchvision 0.13 one can use pretrained\n        models with different weigths. Therefore, `pretrained_model_setup` can\n        also be dictionary with the name of the model and the weights (e.g.\n        `{'resnet50': ResNet50_Weights.DEFAULT}` or\n        `{'resnet50': \"IMAGENET1K_V2\"}`). &lt;br/&gt; Aliased as `pretrained_model_name`.\n    n_trainable: Optional, int, default = None\n        Number of trainable layers starting from the layer closer to the\n        output neuron(s). Note that this number DOES NOT take into account\n        the so-called _'head'_ which is ALWAYS trainable. If\n        `trainable_params` is not None this parameter will be ignored\n    trainable_params: Optional, list, default = None\n        List of strings containing the names (or substring within the name) of\n        the parameters that will be trained. For example, if we use a\n        _'resnet18'_ pretrained model and we set `trainable_params =\n        ['layer4']` only the parameters of _'layer4'_ of the network\n        (and the head, as mentioned before) will be trained. Note that\n        setting this or the previous parameter involves some knowledge of\n        the architecture used.\n    channel_sizes: list, default = [64, 128, 256, 512]\n        List of integers with the channel sizes of a CNN in case we choose not\n        to use a pretrained model\n    kernel_sizes: list or int, default = 3\n        List of integers with the kernel sizes of a CNN in case we choose not\n        to use a pretrained model. Must be of length equal to `len(channel_sizes) - 1`.\n    strides: list or int, default = 1\n        List of integers with the stride sizes of a CNN in case we choose not\n        to use a pretrained model. Must be of length equal to `len(channel_sizes) - 1`.\n    head_hidden_dims: Optional, list, default = None\n        List with the number of neurons per dense layer in the head. e.g: _[64,32]_\n    head_activation: str, default = \"relu\"\n        Activation function for the dense layers in the head. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported\n    head_dropout: float, default = 0.1\n        float indicating the dropout between the dense layers.\n    head_batchnorm: bool, default = False\n        Boolean indicating whether or not batch normalization will be applied\n        to the dense layers\n    head_batchnorm_last: bool, default = False\n        Boolean indicating whether or not batch normalization will be applied\n        to the last of the dense layers\n    head_linear_first: bool, default = False\n        Boolean indicating the order of the operations in the dense\n        layer. If `True: [LIN -&gt; ACT -&gt; BN -&gt; DP]`. If `False: [BN -&gt; DP -&gt;\n        LIN -&gt; ACT]`\n\n    Attributes\n    ----------\n    features: nn.Module\n        The pretrained model or Standard CNN plus the optional head\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.models import Vision\n    &gt;&gt;&gt; X_img = torch.rand((2,3,224,224))\n    &gt;&gt;&gt; model = Vision(channel_sizes=[64, 128], kernel_sizes = [3, 3], strides=[1, 1], head_hidden_dims=[32, 8])\n    &gt;&gt;&gt; out = model(X_img)\n    \"\"\"\n\n    @alias(\"pretrained_model_setup\", [\"pretrained_model_name\"])\n    def __init__(\n        self,\n        pretrained_model_setup: Union[str, Dict[str, Union[str, WeightsEnum]]] = None,\n        n_trainable: Optional[int] = None,\n        trainable_params: Optional[List[str]] = None,\n        channel_sizes: List[int] = [64, 128, 256, 512],\n        kernel_sizes: Union[int, List[int]] = [7, 3, 3, 3],\n        strides: Union[int, List[int]] = [2, 1, 1, 1],\n        head_hidden_dims: Optional[List[int]] = None,\n        head_activation: str = \"relu\",\n        head_dropout: Union[float, List[float]] = 0.1,\n        head_batchnorm: bool = False,\n        head_batchnorm_last: bool = False,\n        head_linear_first: bool = False,\n    ):\n        super(Vision, self).__init__()\n\n        self._check_pretrained_model_setup(\n            pretrained_model_setup, n_trainable, trainable_params\n        )\n\n        self.pretrained_model_setup = pretrained_model_setup\n        self.n_trainable = n_trainable\n        self.trainable_params = trainable_params\n        self.channel_sizes = channel_sizes\n        self.kernel_sizes = kernel_sizes\n        self.strides = strides\n        self.head_hidden_dims = head_hidden_dims\n        self.head_activation = head_activation\n        self.head_dropout = head_dropout\n        self.head_batchnorm = head_batchnorm\n        self.head_batchnorm_last = head_batchnorm_last\n        self.head_linear_first = head_linear_first\n\n        self.features, self.backbone_output_dim = self._get_features()\n\n        if pretrained_model_setup is not None:\n            self._freeze(self.features)\n\n        if self.head_hidden_dims is not None:\n            head_hidden_dims = [self.backbone_output_dim] + self.head_hidden_dims\n            self.vision_mlp = MLP(\n                head_hidden_dims,\n                self.head_activation,\n                self.head_dropout,\n                self.head_batchnorm,\n                self.head_batchnorm_last,\n                self.head_linear_first,\n            )\n\n    def forward(self, X: Tensor) -&gt; Tensor:\n        x = self.features(X)\n\n        if len(x.shape) &gt; 2:\n            if x.shape[2] &gt; 1:\n                x = nn.functional.adaptive_avg_pool2d(x, (1, 1))\n            x = torch.flatten(x, 1)\n\n        if self.head_hidden_dims is not None:\n            x = self.vision_mlp(x)\n\n        return x\n\n    @property\n    def output_dim(self) -&gt; int:\n        r\"\"\"The output dimension of the model. This is a required property\n        neccesary to build the `WideDeep` class\n        \"\"\"\n        return (\n            self.head_hidden_dims[-1]\n            if self.head_hidden_dims is not None\n            else self.backbone_output_dim\n        )\n\n    def _get_features(self) -&gt; Tuple[nn.Module, int]:\n        if self.pretrained_model_setup is not None:\n            if isinstance(self.pretrained_model_setup, str):\n                if self.pretrained_model_setup in allowed_pretrained_models.keys():\n                    model = allowed_pretrained_models[self.pretrained_model_setup]\n                    pretrained_model = torchvision.models.__dict__[model](\n                        weights=torchvision.models.get_model_weights(model).DEFAULT\n                    )\n                    warnings.warn(\n                        f\"{self.pretrained_model_setup} defaulting to {model}\",\n                        UserWarning,\n                    )\n                else:\n                    pretrained_model = torchvision.models.__dict__[\n                        self.pretrained_model_setup\n                    ](weights=\"IMAGENET1K_V1\")\n\n            elif isinstance(self.pretrained_model_setup, Dict):\n                model_name = next(iter(self.pretrained_model_setup))\n                model_weights = self.pretrained_model_setup[model_name]\n\n                if model_name in allowed_pretrained_models.keys():\n                    model_name = allowed_pretrained_models[model_name]\n\n                pretrained_model = torchvision.models.__dict__[model_name](\n                    weights=model_weights\n                )\n            output_dim: int = self.get_backbone_output_dim(pretrained_model)\n            features = nn.Sequential(*(list(pretrained_model.children())[:-1]))\n\n        else:\n            features = self._basic_cnn()\n            output_dim = self.channel_sizes[-1]\n\n        return features, output_dim\n\n    def _basic_cnn(self):\n        channel_sizes = [3] + self.channel_sizes\n        kernel_sizes = (\n            [self.kernel_sizes] * len(self.channel_sizes)\n            if isinstance(self.kernel_sizes, int)\n            else self.kernel_sizes\n        )\n        strides = (\n            [self.strides] * len(self.channel_sizes)\n            if isinstance(self.strides, int)\n            else self.strides\n        )\n\n        BasicCNN = nn.Sequential()\n        for i in range(1, len(channel_sizes)):\n            BasicCNN.add_module(\n                \"conv_layer_{}\".format(i - 1),\n                conv_layer(\n                    channel_sizes[i - 1],\n                    channel_sizes[i],\n                    kernel_sizes[i - 1],\n                    strides[i - 1],\n                    maxpool=i == 1,\n                    adaptiveavgpool=i == len(channel_sizes) - 1,\n                ),\n            )\n        return BasicCNN\n\n    def _freeze(self, features):\n        if self.trainable_params is not None:\n            for name, param in features.named_parameters():\n                for tl in self.trainable_params:\n                    param.requires_grad = tl in name\n        elif self.n_trainable is not None:\n            for i, (name, param) in enumerate(\n                reversed(list(features.named_parameters()))\n            ):\n                param.requires_grad = i &lt; self.n_trainable\n        else:\n            warnings.warn(\n                \"Both 'trainable_params' and 'n_trainable' are 'None' and the entire network will be trained\",\n                UserWarning,\n            )\n\n    @staticmethod\n    def get_backbone_output_dim(features):\n        try:\n            return features.fc.in_features\n        except AttributeError:\n            try:\n                features.classifier.__dict__[\"_modules\"][\"0\"].in_features\n            except AttributeError:\n                try:\n                    return features.classifier.__dict__[\"_modules\"][\"1\"].in_features\n                except AttributeError:\n                    return features.classifier.__dict__[\"_modules\"][\"1\"].in_channels\n\n    @staticmethod\n    def _check_pretrained_model_setup(\n        pretrained_model_setup, n_trainable, trainable_params\n    ):\n        if pretrained_model_setup is not None:\n            if isinstance(pretrained_model_setup, str):\n                pretrained_model_name = pretrained_model_setup\n            elif isinstance(pretrained_model_setup, Dict):\n                pretrained_model_name = list(pretrained_model_setup.keys())[0]\n        else:\n            pretrained_model_name = None\n\n        if pretrained_model_name is not None:\n            valid_pretrained_model_name = any(\n                [name in pretrained_model_name for name in allowed_pretrained_models]\n            )\n            if not valid_pretrained_model_name:\n                raise ValueError(\n                    f\"{pretrained_model_setup} is not among the allowed pretrained models.\"\n                    f\" These are {allowed_pretrained_models.keys()}. Please choose a variant of these architectures\"\n                )\n            if n_trainable is not None and trainable_params is not None:\n                raise UserWarning(\n                    \"Both 'n_trainable' and 'trainable_params' are not None. 'trainable_params' will be used\"\n                )\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.image.vision.Vision.output_dim","title":"output_dim  <code>property</code>","text":"<pre><code>output_dim\n</code></pre> <p>The output dimension of the model. This is a required property neccesary to build the <code>WideDeep</code> class</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.model_fusion.ModelFuser","title":"ModelFuser","text":"<p>               Bases: <code>BaseWDModelComponent</code></p> <p>This class is a wrapper around a list of models that are associated to the different text and/or image columns (and datasets) The class is designed to 'fuse' the models using a variety of methods.</p> <p>Parameters:</p> Name Type Description Default <code>models</code> <code>List[BaseWDModelComponent]</code> <p>List of models whose outputs will be fused</p> required <code>fusion_method</code> <code>Union[Literal[concatenate, mean, max, sum, mult, dot, head], List[Literal[concatenate, mean, max, sum, mult, head]]]</code> <p>Method to fuse the output of the models. It can be one of ['concatenate', 'mean', 'max', 'sum', 'mult', 'dot', 'head'] or a list of those, but 'dot'. If a list is provided the output of the models will be fused using all the methods in the list and the final output will be the concatenation of the outputs of each method</p> required <code>projection_method</code> <code>Optional[Literal[min, max, mean]]</code> <p>If the fusion_method is not 'concatenate', this parameter will determine how to project the output of the models to a common dimension. It can be one of ['min', 'max', 'mean']. Default is None</p> <code>None</code> <code>custom_head</code> <code>Optional[Union[BaseWDModelComponent, Module]]</code> <p>Custom head to be used to fuse the output of the models. If provided, this will take precedence over head_hidden_dims. Also, if provided, 'projection_method' will be ignored.</p> <code>None</code> <code>head_hidden_dims</code> <code>Optional[List[int]]</code> <p>List with the number of neurons per layer in the custom head. If custom_head is provided, this parameter will be ignored</p> <code>None</code> <code>head_activation</code> <code>Optional[str]</code> <p>Activation function to be used in the custom head. Default is None</p> <code>None</code> <code>head_dropout</code> <code>Optional[float]</code> <p>Dropout to be used in the custom head. Default is None</p> <code>None</code> <code>head_batchnorm</code> <code>Optional[bool]</code> <p>Whether to use batchnorm in the custom head. Default is None</p> <code>None</code> <code>head_batchnorm_last</code> <code>Optional[bool]</code> <p>Whether or not batch normalization will be applied to the last of the dense layers</p> <code>None</code> <code>head_linear_first</code> <code>Optional[bool]</code> <p>Boolean indicating the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code></p> <code>None</code> <p>Attributes:</p> Name Type Description <code>head</code> <code>Module or BaseWDModelComponent</code> <p>Custom head to be used to fuse the output of the models. If custom_head is provided, this will take precedence over head_hidden_dims</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; from pytorch_widedeep.preprocessing import TextPreprocessor\n&gt;&gt;&gt; from pytorch_widedeep.models import BasicRNN, ModelFuser\n&gt;&gt;&gt; import torch\n&gt;&gt;&gt; import pandas as pd\n&gt;&gt;&gt;\n&gt;&gt;&gt; df = pd.DataFrame({'text_col1': ['hello world', 'this is a test'],\n... 'text_col2': ['goodbye world', 'this is another test']})\n&gt;&gt;&gt; text_preprocessor_1 = TextPreprocessor(\n...     text_col=\"text_col1\",\n...     max_vocab=10,\n...     min_freq=1,\n...     maxlen=5,\n...     n_cpus=1,\n...     verbose=0)\n&gt;&gt;&gt; text_preprocessor_2 = TextPreprocessor(\n...     text_col=\"text_col2\",\n...     max_vocab=10,\n...     min_freq=1,\n...     maxlen=5,\n...     n_cpus=1,\n...     verbose=0)\n&gt;&gt;&gt; X_text1 = text_preprocessor_1.fit_transform(df)\n&gt;&gt;&gt; X_text2 = text_preprocessor_2.fit_transform(df)\n&gt;&gt;&gt; X_text1_tnsr = torch.from_numpy(X_text1)\n&gt;&gt;&gt; X_text2_tnsr = torch.from_numpy(X_text2)\n&gt;&gt;&gt; rnn1 = BasicRNN(\n...     vocab_size=len(text_preprocessor_1.vocab.itos),\n...     embed_dim=4,\n...     hidden_dim=4,\n...     n_layers=1,\n...     bidirectional=False)\n&gt;&gt;&gt; rnn2 = BasicRNN(\n...     vocab_size=len(text_preprocessor_2.vocab.itos),\n...     embed_dim=4,\n...     hidden_dim=4,\n...     n_layers=1,\n...     bidirectional=False)\n&gt;&gt;&gt; fused_model = ModelFuser(models=[rnn1, rnn2], fusion_method='concatenate')\n&gt;&gt;&gt; out = fused_model([X_text1_tnsr, X_text2_tnsr])\n</code></pre> Source code in <code>pytorch_widedeep/models/model_fusion.py</code> <pre><code>class ModelFuser(BaseWDModelComponent):\n    \"\"\"\n    This class is a wrapper around a list of models that are associated to the\n    different text and/or image columns (and datasets) The class is designed\n    to 'fuse' the models using a variety of methods.\n\n    Parameters\n    ----------\n    models: List[BaseWDModelComponent]\n        List of models whose outputs will be fused\n    fusion_method: Union[str, List[str]]\n        Method to fuse the output of the models. It can be one of\n        ['concatenate', 'mean', 'max', 'sum', 'mult', 'dot', 'head'] or a\n        list of those, but 'dot'. If a list is provided the output of the\n        models will be fused using all the methods in the list and the final\n        output will be the concatenation of the outputs of each method\n    projection_method: Optional[str]\n        If the fusion_method is not 'concatenate', this parameter will\n        determine how to project the output of the models to a common\n        dimension. It can be one of ['min', 'max', 'mean']. Default is None\n    custom_head: Optional[BaseWDModelComponent | nn.Module]\n        Custom head to be used to fuse the output of the models. If provided,\n        this will take precedence over head_hidden_dims. Also, if\n        provided, 'projection_method' will be ignored.\n    head_hidden_dims: Optional[List[int]]\n        List with the number of neurons per layer in the custom head. If\n        custom_head is provided, this parameter will be ignored\n    head_activation: Optional[str]\n        Activation function to be used in the custom head. Default is None\n    head_dropout: Optional[float]\n        Dropout to be used in the custom head. Default is None\n    head_batchnorm: Optional[bool]\n        Whether to use batchnorm in the custom head. Default is None\n    head_batchnorm_last: Optional[bool]\n        Whether or not batch normalization will be applied to the last of the\n        dense layers\n    head_linear_first: Optional[bool]\n        Boolean indicating the order of the operations in the dense\n        layer. If `True: [LIN -&gt; ACT -&gt; BN -&gt; DP]`. If `False: [BN -&gt; DP -&gt;\n        LIN -&gt; ACT]`\n\n    Attributes\n    ----------\n    head: nn.Module or BaseWDModelComponent\n        Custom head to be used to fuse the output of the models. If\n        custom_head is provided, this will take precedence over\n        head_hidden_dims\n\n    Examples\n    --------\n    &gt;&gt;&gt; from pytorch_widedeep.preprocessing import TextPreprocessor\n    &gt;&gt;&gt; from pytorch_widedeep.models import BasicRNN, ModelFuser\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; import pandas as pd\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; df = pd.DataFrame({'text_col1': ['hello world', 'this is a test'],\n    ... 'text_col2': ['goodbye world', 'this is another test']})\n    &gt;&gt;&gt; text_preprocessor_1 = TextPreprocessor(\n    ...     text_col=\"text_col1\",\n    ...     max_vocab=10,\n    ...     min_freq=1,\n    ...     maxlen=5,\n    ...     n_cpus=1,\n    ...     verbose=0)\n    &gt;&gt;&gt; text_preprocessor_2 = TextPreprocessor(\n    ...     text_col=\"text_col2\",\n    ...     max_vocab=10,\n    ...     min_freq=1,\n    ...     maxlen=5,\n    ...     n_cpus=1,\n    ...     verbose=0)\n    &gt;&gt;&gt; X_text1 = text_preprocessor_1.fit_transform(df)\n    &gt;&gt;&gt; X_text2 = text_preprocessor_2.fit_transform(df)\n    &gt;&gt;&gt; X_text1_tnsr = torch.from_numpy(X_text1)\n    &gt;&gt;&gt; X_text2_tnsr = torch.from_numpy(X_text2)\n    &gt;&gt;&gt; rnn1 = BasicRNN(\n    ...     vocab_size=len(text_preprocessor_1.vocab.itos),\n    ...     embed_dim=4,\n    ...     hidden_dim=4,\n    ...     n_layers=1,\n    ...     bidirectional=False)\n    &gt;&gt;&gt; rnn2 = BasicRNN(\n    ...     vocab_size=len(text_preprocessor_2.vocab.itos),\n    ...     embed_dim=4,\n    ...     hidden_dim=4,\n    ...     n_layers=1,\n    ...     bidirectional=False)\n    &gt;&gt;&gt; fused_model = ModelFuser(models=[rnn1, rnn2], fusion_method='concatenate')\n    &gt;&gt;&gt; out = fused_model([X_text1_tnsr, X_text2_tnsr])\n    \"\"\"\n\n    def __init__(\n        self,\n        models: List[BaseWDModelComponent],\n        *,\n        fusion_method: Union[\n            Literal[\n                \"concatenate\",\n                \"mean\",\n                \"max\",\n                \"sum\",\n                \"mult\",\n                \"dot\",\n                \"head\",\n            ],\n            List[Literal[\"concatenate\", \"mean\", \"max\", \"sum\", \"mult\", \"head\"]],\n        ],\n        projection_method: Optional[Literal[\"min\", \"max\", \"mean\"]] = None,\n        custom_head: Optional[Union[BaseWDModelComponent, nn.Module]] = None,\n        head_hidden_dims: Optional[List[int]] = None,\n        head_activation: Optional[str] = None,\n        head_dropout: Optional[float] = None,\n        head_batchnorm: Optional[bool] = None,\n        head_batchnorm_last: Optional[bool] = None,\n        head_linear_first: Optional[bool] = None,\n    ) -&gt; None:\n        super(ModelFuser, self).__init__()\n\n        self.models = nn.ModuleList(models)\n        self.fusion_method = fusion_method\n        self.projection_method = projection_method\n\n        self.all_output_dim_equal = all(\n            model.output_dim == self.models[0].output_dim for model in self.models\n        )\n\n        self.check_input_parameters()\n\n        if self.fusion_method == \"head\":\n            assert (\n                head_hidden_dims is not None or custom_head is not None\n            ), \"When using 'head' as fusion_method, either head_hidden_dims or custom_head must be provided\"\n            if custom_head is not None:\n                # custom_head takes precedence over head_hidden_dims (in case\n                # both are provided)\n                assert hasattr(\n                    custom_head, \"output_dim\"\n                ), \"custom_head must have an 'output_dim' property\"\n                self.head: Union[BaseWDModelComponent, nn.Module] = custom_head\n            else:\n                assert head_hidden_dims is not None\n                self.head_hidden_dims = head_hidden_dims\n                self.head_activation = head_activation\n                self.head_dropout = head_dropout\n                self.head_batchnorm = head_batchnorm\n                self.head_batchnorm_last = head_batchnorm_last\n                self.head_linear_first = head_linear_first\n\n                self.head = MLP(\n                    d_hidden=[sum([model.output_dim for model in self.models])]\n                    + self.head_hidden_dims,\n                    activation=(\n                        \"relu\" if self.head_activation is None else self.head_activation\n                    ),\n                    dropout=0.0 if self.head_dropout is None else self.head_dropout,\n                    batchnorm=(\n                        False if self.head_batchnorm is None else self.head_batchnorm\n                    ),\n                    batchnorm_last=(\n                        False\n                        if self.head_batchnorm_last is None\n                        else self.head_batchnorm_last\n                    ),\n                    linear_first=(\n                        True\n                        if self.head_linear_first is None\n                        else self.head_linear_first\n                    ),\n                )\n\n    def forward(self, X: List[Tensor]) -&gt; Tensor:  # noqa: C901\n        if self.fusion_method == \"head\":\n            return self._head_fusion(X)\n        elif self.fusion_method == \"dot\":\n            return self._dot_fusion(X)\n        else:\n            return self._other_fusions(X)\n\n    def _head_fusion(self, X: List[Tensor]) -&gt; Tensor:\n        return self.head(torch.cat([model(x) for model, x in zip(self.models, X)], -1))\n\n    def _dot_fusion(self, X: List[Tensor]) -&gt; Tensor:\n        assert (\n            len(X) == 2\n        ), \"When using 'dot' as fusion_method, only two models can be fused\"\n        outputs = [model(x) for model, x in zip(self.models, X)]\n        return torch.bmm(outputs[1].unsqueeze(1), outputs[0].unsqueeze(2)).view(-1, 1)\n\n    def _other_fusions(self, X: List[Tensor]) -&gt; Tensor:\n        fusion_methods = (\n            [self.fusion_method]\n            if isinstance(self.fusion_method, str)\n            else self.fusion_method\n        )\n        fused_outputs = [self._apply_fusion_method(fm, X) for fm in fusion_methods]  # type: ignore[attr-defined]\n        return (\n            fused_outputs[0] if len(fused_outputs) == 1 else torch.cat(fused_outputs, 1)\n        )\n\n    def _apply_fusion_method(self, fusion_method: str, X: List[Tensor]) -&gt; Tensor:\n        if fusion_method == \"concatenate\":\n            return torch.cat([model(x) for model, x in zip(self.models, X)], 1)\n\n        model_outputs = [model(x) for model, x in zip(self.models, X)]\n        projections = self._project(model_outputs)\n        stacked = torch.stack(projections, -1)\n\n        if fusion_method == \"mean\":\n            return torch.mean(stacked, -1)\n        elif fusion_method == \"max\":\n            return torch.max(stacked, -1)[0]\n        elif fusion_method == \"min\":\n            return torch.min(stacked, -1)[0]\n        elif fusion_method == \"sum\":\n            return torch.sum(stacked, -1)\n        elif fusion_method == \"mult\":\n            return torch.prod(stacked, -1)\n        else:\n            raise ValueError(f\"Unsupported fusion method: {fusion_method}\")\n\n    def _project(self, X: List[Tensor]) -&gt; List[Tensor]:\n        r\"\"\"Projects the output of the models to a common dimension.\"\"\"\n\n        if self.all_output_dim_equal and self.projection_method is None:\n            return X\n\n        output_dims = [model.output_dim for model in self.models]\n\n        if self.projection_method == \"min\":\n            proj_dim = min(output_dims)\n            idx = output_dims.index(proj_dim)\n        elif self.projection_method == \"max\":\n            proj_dim = max(output_dims)\n            idx = output_dims.index(proj_dim)\n        elif self.projection_method == \"mean\":\n            proj_dim = int(sum(output_dims) / len(output_dims))\n            idx = None\n        else:\n            raise ValueError(\"projection_method must be one of ['min', 'max', 'mean']\")\n\n        x_proj: List[Tensor] = []\n        for i, x in enumerate(X):\n            if i == idx:\n                x_proj.append(x)\n            else:\n                x_proj.append(\n                    nn.Linear(output_dims[i], proj_dim, bias=False, device=x.device)(x)\n                )\n\n        return x_proj\n\n    @property\n    def output_dim(self) -&gt; int:\n        r\"\"\"Returns the output dimension of the model.\"\"\"\n        if self.fusion_method == \"head\":\n            output_dim = (\n                self.head_hidden_dims[-1]\n                if hasattr(self, \"head_hidden_dims\")\n                else self.head.output_dim\n            )\n        elif self.fusion_method == \"dot\":\n            output_dim = 1\n        else:\n            output_dim = 0\n            if isinstance(self.fusion_method, str):\n                fusion_methods = [self.fusion_method]\n            else:\n                fusion_methods = self.fusion_method  # type: ignore\n            for fm in fusion_methods:\n                if fm == \"concatenate\":\n                    output_dim += sum([model.output_dim for model in self.models])\n                elif self.projection_method == \"mean\":\n                    output_dim += int(\n                        sum([model.output_dim for model in self.models])\n                        / len(self.models)\n                    )\n                elif self.projection_method == \"min\":\n                    output_dim += min([model.output_dim for model in self.models])\n                elif self.projection_method == \"max\":\n                    output_dim += max([model.output_dim for model in self.models])\n                elif self.all_output_dim_equal:\n                    output_dim += self.models[0].output_dim\n                else:\n                    raise ValueError(\n                        \"projection_method must be one of ['min', 'max', 'mean']\"\n                    )\n\n        return output_dim\n\n    def check_input_parameters(self):\n        self._validate_tabnet()\n        self._validate_fusion_methods()\n        self._validate_projection_requirements()\n        self._validate_head_dot_exclusivity()\n\n    def _validate_tabnet(self):\n        if any(isinstance(model, TabNet) for model in self.models):\n            raise ValueError(\n                \"TabNet is not supported in ModelFuser. \"\n                \"Please, use another model for tabular data\"\n            )\n\n    def _validate_fusion_methods(self):\n        valid_methods = [\n            \"concatenate\",\n            \"min\",\n            \"max\",\n            \"mean\",\n            \"sum\",\n            \"mult\",\n            \"dot\",\n            \"head\",\n        ]\n\n        if isinstance(self.fusion_method, str):\n            if self.fusion_method not in valid_methods:\n                raise ValueError(\n                    \"fusion_method must be one of ['concatenate', 'mean', 'max', 'sum', 'mult', 'dot', 'head'] \"\n                    \"or a list of any those but 'dot'\"\n                )\n        else:\n            if not all(fm in valid_methods for fm in self.fusion_method):\n                raise ValueError(\n                    \"fusion_method must be one of ['concatenate', 'mean', 'max', 'sum', 'mult', 'dot', 'head'] \"\n                    \"or a list of those but 'dot'\"\n                )\n\n    def _validate_projection_requirements(self):\n        needs_projection = not self.all_output_dim_equal\n        has_size_dependent_method = False\n\n        if isinstance(self.fusion_method, str):\n            has_size_dependent_method = self.fusion_method in [\"min\", \"max\", \"mean\"]\n        else:\n            has_size_dependent_method = all(\n                fm in [\"min\", \"max\", \"mean\"] for fm in self.fusion_method\n            )\n\n        if has_size_dependent_method and needs_projection:\n            if self.projection_method is None:\n                raise ValueError(\n                    \"If 'fusion_method' is not 'concatenate' or 'head', \"\n                    \"and the output dimensions of the models are not equal, \"\n                    \"'projection_method' must be provided\"\n                )\n            elif self.projection_method not in [\"min\", \"max\", \"mean\"]:\n                raise ValueError(\n                    \"projection_method must be one of ['min', 'max', 'mean']\"\n                )\n\n    def _validate_head_dot_exclusivity(self):\n        if isinstance(self.fusion_method, list):\n            if any(method in [\"head\", \"dot\"] for method in self.fusion_method):\n                raise ValueError(\n                    \"When using 'head' or 'dot' as fusion_method, no other method should be provided\"\n                )\n\n    def __repr__(self):\n        if self.projection_method is not None:\n            proj = f\"{self.projection_method}\"\n        else:\n            proj = \"\"\n\n        return f\"Fusion method: {self.fusion_method}. Projection method: {proj}\\nFused Models:\\n{self.models}\"\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.model_fusion.ModelFuser.output_dim","title":"output_dim  <code>property</code>","text":"<pre><code>output_dim\n</code></pre> <p>Returns the output dimension of the model.</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.wide_deep.WideDeep","title":"WideDeep","text":"<p>               Bases: <code>Module</code></p> <p>Main collector class that combines all <code>wide</code>, <code>deeptabular</code> <code>deeptext</code> and <code>deepimage</code> models.</p> <p>Note that all models described so far in this library must be passed to the <code>WideDeep</code> class once constructed. This is because the models output the last layer before the prediction layer. Such prediction layer is added by the <code>WideDeep</code> class as it collects the components for every data mode.</p> <p>There are two options to combine these models that correspond to the two main architectures that <code>pytorch-widedeep</code> can build.</p> <ul> <li> <p>Directly connecting the output of the model components to an ouput neuron(s).</p> </li> <li> <p>Adding a <code>Fully-Connected Head</code> (FC-Head) on top of the deep models.   This FC-Head will combine the output form the <code>deeptabular</code>, <code>deeptext</code> and   <code>deepimage</code> and will be then connected to the output neuron(s).</p> </li> </ul> <p>Parameters:</p> Name Type Description Default <code>wide</code> <code>Optional[Module]</code> <p><code>Wide</code> model. This is a linear model where the non-linearities are captured via crossed-columns.</p> <code>None</code> <code>deeptabular</code> <code>Optional[Union[BaseWDModelComponent, List[BaseWDModelComponent]]]</code> <p>Currently this library implements a number of possible architectures for the <code>deeptabular</code> component. See the documenation of the package. Note that <code>deeptabular</code> can be a list of models. This is useful when using multiple tabular inputs (e.g. for example in the context of a two-tower model for recommendation systems)</p> <code>None</code> <code>deeptext</code> <code>Optional[Union[BaseWDModelComponent, List[BaseWDModelComponent]]]</code> <p>Currently this library implements a number of possible architectures for the <code>deeptext</code> component. See the documenation of the package. Note that <code>deeptext</code> can be a list of models. This is useful when using multiple text inputs.</p> <code>None</code> <code>deepimage</code> <code>Optional[Union[BaseWDModelComponent, List[BaseWDModelComponent]]]</code> <p>Currently this library uses <code>torchvision</code> and implements a number of possible architectures for the <code>deepimage</code> component. See the documenation of the package. Note that <code>deepimage</code> can be a list of models. This is useful when using multiple image inputs.</p> <code>None</code> <code>deephead</code> <code>Optional[BaseWDModelComponent]</code> <p>Alternatively, the user can pass a custom model that will receive the output of the deep component. If <code>deephead</code> is not None all the previous fc-head parameters will be ignored</p> <code>None</code> <code>head_hidden_dims</code> <code>Optional[List[int]]</code> <p>List with the sizes of the dense layers in the head e.g: [128, 64]</p> <code>None</code> <code>head_activation</code> <code>str</code> <p>Activation function for the dense layers in the head. Currently <code>'tanh'</code>, <code>'relu'</code>, <code>'leaky_relu'</code> and <code>'gelu'</code> are supported</p> <code>'relu'</code> <code>head_dropout</code> <code>float</code> <p>Dropout of the dense layers in the head</p> <code>0.1</code> <code>head_batchnorm</code> <code>bool</code> <p>Boolean indicating whether or not to include batch normalization in the dense layers that form the <code>'rnn_mlp'</code></p> <code>False</code> <code>head_batchnorm_last</code> <code>bool</code> <p>Boolean indicating whether or not to apply batch normalization to the last of the dense layers in the head</p> <code>False</code> <code>head_linear_first</code> <code>bool</code> <p>Boolean indicating whether the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code></p> <code>True</code> <code>enforce_positive</code> <code>bool</code> <p>Boolean indicating if the output from the final layer must be positive. This is important if you are using loss functions with non-negative input restrictions, e.g. RMSLE, or if you know your predictions are bounded in between 0 and inf</p> <code>False</code> <code>enforce_positive_activation</code> <code>str</code> <p>Activation function to enforce that the final layer has a positive output. <code>'softplus'</code> or <code>'relu'</code> are supported.</p> <code>'softplus'</code> <code>pred_dim</code> <code>int</code> <p>Size of the final wide and deep output layer containing the predictions. <code>1</code> for regression and binary classification or number of classes for multiclass classification.</p> <code>1</code> <p>Examples:</p> <pre><code>&gt;&gt;&gt; from pytorch_widedeep.models import TabResnet, Vision, BasicRNN, Wide, WideDeep\n&gt;&gt;&gt; embed_input = [(u, i, j) for u, i, j in zip([\"a\", \"b\", \"c\"][:4], [4] * 3, [8] * 3)]\n&gt;&gt;&gt; column_idx = {k: v for v, k in enumerate([\"a\", \"b\", \"c\"])}\n&gt;&gt;&gt; wide = Wide(10, 1)\n&gt;&gt;&gt; deeptabular = TabResnet(blocks_dims=[8, 4], column_idx=column_idx, cat_embed_input=embed_input)\n&gt;&gt;&gt; deeptext = BasicRNN(vocab_size=10, embed_dim=4, padding_idx=0)\n&gt;&gt;&gt; deepimage = Vision()\n&gt;&gt;&gt; model = WideDeep(wide=wide, deeptabular=deeptabular, deeptext=deeptext, deepimage=deepimage)\n</code></pre> <p> NOTE: It is possible to use custom components to  build Wide &amp; Deep models. Simply, build them and pass them as the  corresponding parameters. Note that the custom models MUST return a last  layer of activations(i.e. not the final prediction) so that  these  activations are collected by <code>WideDeep</code> and combined accordingly. In  addition, the models MUST also contain an attribute <code>output_dim</code> with  the size of these last layers of activations. See for example  <code>pytorch_widedeep.models.tab_mlp.TabMlp</code></p> Source code in <code>pytorch_widedeep/models/wide_deep.py</code> <pre><code>class WideDeep(nn.Module):\n    r\"\"\"Main collector class that combines all `wide`, `deeptabular`\n    `deeptext` and `deepimage` models.\n\n    Note that all models described so far in this library must be passed to\n    the `WideDeep` class once constructed. This is because the models output\n    the last layer before the prediction layer. Such prediction layer is\n    added by the `WideDeep` class as it collects the components for every\n    data mode.\n\n    There are two options to combine these models that correspond to the\n    two main architectures that `pytorch-widedeep` can build.\n\n    - Directly connecting the output of the model components to an ouput neuron(s).\n\n    - Adding a `Fully-Connected Head` (FC-Head) on top of the deep models.\n      This FC-Head will combine the output form the `deeptabular`, `deeptext` and\n      `deepimage` and will be then connected to the output neuron(s).\n\n    Parameters\n    ----------\n    wide: nn.Module, Optional, default = None\n        `Wide` model. This is a linear model where the non-linearities are\n        captured via crossed-columns.\n    deeptabular: BaseWDModelComponent, Optional, default = None\n        Currently this library implements a number of possible architectures\n        for the `deeptabular` component. See the documenation of the\n        package. Note that `deeptabular` can be a list of models. This is\n        useful when using multiple tabular inputs (e.g. for example in the\n        context of a two-tower model for recommendation systems)\n    deeptext: BaseWDModelComponent | List[BaseWDModelComponent], Optional, default = None\n        Currently this library implements a number of possible architectures\n        for the `deeptext` component. See the documenation of the\n        package. Note that `deeptext` can be a list of models. This is useful\n        when using multiple text inputs.\n    deepimage: BaseWDModelComponent | List[BaseWDModelComponent], Optional, default = None\n        Currently this library uses `torchvision` and implements a number of\n        possible architectures for the `deepimage` component. See the\n        documenation of the package. Note that `deepimage` can be a list of\n        models. This is useful when using multiple image inputs.\n    deephead: BaseWDModelComponent, Optional, default = None\n        Alternatively, the user can pass a custom model that will receive the\n        output of the deep component. If `deephead` is not None all the\n        previous fc-head parameters will be ignored\n    head_hidden_dims: List, Optional, default = None\n        List with the sizes of the dense layers in the head e.g: [128, 64]\n    head_activation: str, default = \"relu\"\n        Activation function for the dense layers in the head. Currently\n        `'tanh'`, `'relu'`, `'leaky_relu'` and `'gelu'` are supported\n    head_dropout: float, Optional, default = None\n        Dropout of the dense layers in the head\n    head_batchnorm: bool, default = False\n        Boolean indicating whether or not to include batch normalization in\n        the dense layers that form the `'rnn_mlp'`\n    head_batchnorm_last: bool, default = False\n        Boolean indicating whether or not to apply batch normalization to the\n        last of the dense layers in the head\n    head_linear_first: bool, default = False\n        Boolean indicating whether the order of the operations in the dense\n        layer. If `True: [LIN -&gt; ACT -&gt; BN -&gt; DP]`. If `False: [BN -&gt; DP -&gt;\n        LIN -&gt; ACT]`\n    enforce_positive: bool, default = False\n        Boolean indicating if the output from the final layer must be\n        positive. This is important if you are using loss functions with\n        non-negative input restrictions, e.g. RMSLE, or if you know your\n        predictions are bounded in between 0 and inf\n    enforce_positive_activation: str, default = \"softplus\"\n        Activation function to enforce that the final layer has a positive\n        output. `'softplus'` or `'relu'` are supported.\n    pred_dim: int, default = 1\n        Size of the final wide and deep output layer containing the\n        predictions. `1` for regression and binary classification or number\n        of classes for multiclass classification.\n\n\n    Examples\n    --------\n\n    &gt;&gt;&gt; from pytorch_widedeep.models import TabResnet, Vision, BasicRNN, Wide, WideDeep\n    &gt;&gt;&gt; embed_input = [(u, i, j) for u, i, j in zip([\"a\", \"b\", \"c\"][:4], [4] * 3, [8] * 3)]\n    &gt;&gt;&gt; column_idx = {k: v for v, k in enumerate([\"a\", \"b\", \"c\"])}\n    &gt;&gt;&gt; wide = Wide(10, 1)\n    &gt;&gt;&gt; deeptabular = TabResnet(blocks_dims=[8, 4], column_idx=column_idx, cat_embed_input=embed_input)\n    &gt;&gt;&gt; deeptext = BasicRNN(vocab_size=10, embed_dim=4, padding_idx=0)\n    &gt;&gt;&gt; deepimage = Vision()\n    &gt;&gt;&gt; model = WideDeep(wide=wide, deeptabular=deeptabular, deeptext=deeptext, deepimage=deepimage)\n\n\n    :information_source: **NOTE**: It is possible to use custom components to\n     build Wide &amp; Deep models. Simply, build them and pass them as the\n     corresponding parameters. Note that the custom models MUST return a last\n     layer of activations(i.e. not the final prediction) so that  these\n     activations are collected by `WideDeep` and combined accordingly. In\n     addition, the models MUST also contain an attribute `output_dim` with\n     the size of these last layers of activations. See for example\n     `pytorch_widedeep.models.tab_mlp.TabMlp`\n    \"\"\"\n\n    @alias(  # noqa: C901\n        \"pred_dim\",\n        [\"num_class\", \"pred_size\"],\n    )\n    def __init__(\n        self,\n        wide: Optional[nn.Module] = None,\n        deeptabular: Optional[\n            Union[BaseWDModelComponent, List[BaseWDModelComponent]]\n        ] = None,\n        deeptext: Optional[\n            Union[BaseWDModelComponent, List[BaseWDModelComponent]]\n        ] = None,\n        deepimage: Optional[\n            Union[BaseWDModelComponent, List[BaseWDModelComponent]]\n        ] = None,\n        deephead: Optional[BaseWDModelComponent] = None,\n        head_hidden_dims: Optional[List[int]] = None,\n        head_activation: str = \"relu\",\n        head_dropout: float = 0.1,\n        head_batchnorm: bool = False,\n        head_batchnorm_last: bool = False,\n        head_linear_first: bool = True,\n        enforce_positive: bool = False,\n        enforce_positive_activation: str = \"softplus\",\n        pred_dim: int = 1,\n    ):\n        super(WideDeep, self).__init__()\n\n        self._check_inputs(\n            wide,\n            deeptabular,\n            deeptext,\n            deepimage,\n            deephead,\n            head_hidden_dims,\n            pred_dim,\n        )\n\n        # this attribute will be eventually over-written by the Trainer's\n        # device. Acts here as a 'placeholder'.\n        self.wd_device: Optional[str] = None\n\n        # required as attribute just in case we pass a deephead\n        self.pred_dim = pred_dim\n\n        self.enforce_positive = enforce_positive\n\n        # better to set this attribute already here\n        if isinstance(deeptabular, list):\n            self.is_tabnet = False\n        else:\n            self.is_tabnet = deeptabular.__class__.__name__ == \"TabNet\"\n\n        # The main 5 components of the wide and deep assemble: wide,\n        # deeptabular, deeptext, deepimage and deephead\n        self.with_deephead = deephead is not None or head_hidden_dims is not None\n        if deephead is None and head_hidden_dims is not None:\n            self.deephead = self._build_deephead(\n                deeptabular,\n                deeptext,\n                deepimage,\n                head_hidden_dims,\n                head_activation,\n                head_dropout,\n                head_batchnorm,\n                head_batchnorm_last,\n                head_linear_first,\n            )\n        elif deephead is not None:\n            self.deephead = nn.Sequential(\n                deephead, nn.Linear(deephead.output_dim, self.pred_dim)\n            )\n        else:\n            # for consistency with other components we default to None\n            self.deephead = None\n\n        self.wide = wide\n        self.deeptabular, self.deeptext, self.deepimage = self._set_model_components(\n            deeptabular, deeptext, deepimage\n        )\n\n        if self.enforce_positive:\n            self.enf_pos = get_activation_fn(enforce_positive_activation)\n\n    def forward(\n        self,\n        X: Dict[str, Union[Tensor, List[Tensor]]],\n        y: Optional[Tensor] = None,\n    ) -&gt; Union[Tensor, Tuple[Tensor, Tensor]]:\n\n        wide_out = self._forward_wide(X)\n        if self.with_deephead:\n            deep = self._forward_deephead(X, wide_out)\n        else:\n            deep = self._forward_deep(X, wide_out)\n\n        if self.enforce_positive:\n            return self.enf_pos(deep)\n        else:\n            return deep\n\n    def _build_deephead(\n        self,\n        deeptabular: Optional[Union[BaseWDModelComponent, List[BaseWDModelComponent]]],\n        deeptext: Optional[Union[BaseWDModelComponent, List[BaseWDModelComponent]]],\n        deepimage: Optional[Union[BaseWDModelComponent, List[BaseWDModelComponent]]],\n        head_hidden_dims: List[int],\n        head_activation: str,\n        head_dropout: float,\n        head_batchnorm: bool,\n        head_batchnorm_last: bool,\n        head_linear_first: bool,\n    ) -&gt; nn.Sequential:\n        deep_dim = 0\n        if deeptabular is not None:\n            if isinstance(deeptabular, list):\n                for dt in deeptabular:\n                    deep_dim += dt.output_dim\n            else:\n                deep_dim += deeptabular.output_dim\n        if deeptext is not None:\n            if isinstance(deeptext, list):\n                for dt in deeptext:\n                    deep_dim += dt.output_dim\n            else:\n                deep_dim += deeptext.output_dim\n        if deepimage is not None:\n            if isinstance(deepimage, list):\n                for di in deepimage:\n                    deep_dim += di.output_dim\n            else:\n                deep_dim += deepimage.output_dim\n\n        head_hidden_dims = [deep_dim] + head_hidden_dims\n        deephead = nn.Sequential(\n            MLP(\n                head_hidden_dims,\n                head_activation,\n                head_dropout,\n                head_batchnorm,\n                head_batchnorm_last,\n                head_linear_first,\n            ),\n            nn.Linear(head_hidden_dims[-1], self.pred_dim),\n        )\n\n        return deephead\n\n    def _set_model_components(  # noqa: C901\n        self,\n        deeptabular: Optional[Union[BaseWDModelComponent, List[BaseWDModelComponent]]],\n        deeptext: Optional[Union[BaseWDModelComponent, List[BaseWDModelComponent]]],\n        deepimage: Optional[Union[BaseWDModelComponent, List[BaseWDModelComponent]]],\n    ) -&gt; Tuple[\n        Optional[Union[nn.ModuleList, WDModel]],\n        Optional[Union[nn.ModuleList, WDModel]],\n        Optional[Union[nn.ModuleList, WDModel]],\n    ]:\n        if deeptabular is not None:\n            deeptabular_ = self._set_model_component(deeptabular, is_deeptabular=True)\n        else:\n            deeptabular_ = None\n\n        if deeptext is not None:\n            deeptext_ = self._set_model_component(deeptext)\n        else:\n            deeptext_ = None\n\n        if deepimage is not None:\n            deepimage_ = self._set_model_component(deepimage)\n        else:\n            deepimage_ = None\n\n        return deeptabular_, deeptext_, deepimage_\n\n    def _forward_wide(self, X: Dict[str, Union[Tensor, List[Tensor]]]) -&gt; Tensor:\n        if self.wide is not None:\n            out = self.wide(X[\"wide\"])\n        else:\n            first_model_mode = list(X.keys())[0]\n            if isinstance(X[first_model_mode], list):\n                batch_size = X[first_model_mode][0].size(0)\n            else:\n                batch_size = X[first_model_mode].size(0)  # type: ignore[union-attr]\n            out = torch.zeros(batch_size, self.pred_dim).to(self.wd_device)\n\n        return out\n\n    def _forward_deep(\n        self, X: Dict[str, Union[Tensor, List[Tensor]]], wide_out: Tensor\n    ) -&gt; Union[Tensor, Tuple[Tensor, Tensor]]:\n        if self.deeptabular is not None:\n            if self.is_tabnet:\n                tab_out, M_loss = self.deeptabular(X[\"deeptabular\"])\n                wide_out.add_(tab_out)\n            else:\n                wide_out = self._forward_component(\n                    X, self.deeptabular, \"deeptabular\", wide_out\n                )\n\n        if self.deeptext is not None:\n            wide_out = self._forward_component(X, self.deeptext, \"deeptext\", wide_out)\n\n        if self.deepimage is not None:\n            wide_out = self._forward_component(X, self.deepimage, \"deepimage\", wide_out)\n\n        if self.is_tabnet:\n            res: Union[Tensor, Tuple[Tensor, Tensor]] = (wide_out, M_loss)\n        else:\n            res = wide_out\n\n        return res\n\n    def _forward_deephead(\n        self, X: Dict[str, Union[Tensor, List[Tensor]]], wide_out: Tensor\n    ) -&gt; Union[Tensor, Tuple[Tensor, Tensor]]:\n        deepside = torch.FloatTensor().to(self.wd_device)\n\n        if self.deeptabular is not None:\n            if self.is_tabnet:\n                deepside, M_loss = self.deeptabular(X[\"deeptabular\"])\n            else:\n                deepside = self._forward_component_with_head(\n                    X, self.deeptabular, \"deeptabular\", deepside\n                )\n\n        if self.deeptext is not None:\n            deepside = self._forward_component_with_head(\n                X, self.deeptext, \"deeptext\", deepside\n            )\n\n        if self.deepimage is not None:\n            deepside = self._forward_component_with_head(\n                X, self.deepimage, \"deepimage\", deepside\n            )\n\n        # assertion to avoid type issues\n        assert self.deephead is not None\n        deepside_out = self.deephead(deepside)\n\n        if self.is_tabnet:\n            res: Union[Tensor, Tuple[Tensor, Tensor]] = (\n                wide_out.add_(deepside_out),\n                M_loss,\n            )\n        else:\n            res = wide_out.add_(deepside_out)\n\n        return res\n\n    def _forward_component(\n        self,\n        X: Dict[str, Union[Tensor, List[Tensor]]],\n        component: Union[nn.ModuleList, WDModel],\n        component_type: Literal[\"deeptabular\", \"deeptext\", \"deepimage\"],\n        wide_out: Tensor,\n    ) -&gt; Tensor:\n        if isinstance(component, nn.ModuleList):\n            component_out = torch.add(  # type: ignore[call-overload]\n                *[cp(X[component_type][i]) for i, cp in enumerate(component)]\n            )\n        else:\n            component_out = component(X[component_type])\n\n        return wide_out.add_(component_out)\n\n    def _forward_component_with_head(\n        self,\n        X: Dict[str, Union[Tensor, List[Tensor]]],\n        component: Union[nn.ModuleList, WDModel],\n        component_type: Literal[\"deeptabular\", \"deeptext\", \"deepimage\"],\n        deepside: Tensor,\n    ) -&gt; Tensor:\n        if isinstance(component, nn.ModuleList):\n            component_out = torch.cat(  # type: ignore[call-overload]\n                [cp(X[component_type][i]) for i, cp in enumerate(component)], axis=1\n            )\n        else:\n            component_out = component(X[component_type])\n\n        return torch.cat([deepside, component_out], axis=1)  # type: ignore[call-overload]\n\n    def _set_model_component(\n        self,\n        component: Union[BaseWDModelComponent, List[BaseWDModelComponent]],\n        is_deeptabular: bool = False,\n    ) -&gt; Union[nn.ModuleList, WDModel]:\n        if isinstance(component, list):\n            component_: Optional[Union[nn.ModuleList, WDModel]] = nn.ModuleList()\n            for cp in component:\n                if self.with_deephead or cp.output_dim == 1:\n                    component_.append(cp)\n                else:\n                    component_.append(\n                        nn.Sequential(cp, nn.Linear(cp.output_dim, self.pred_dim))\n                    )\n        elif self.with_deephead or component.output_dim == 1:\n            component_ = component\n        elif is_deeptabular and self.is_tabnet:\n            component_ = nn.Sequential(\n                component, TabNetPredLayer(component.output_dim, self.pred_dim)\n            )\n        else:\n            component_ = nn.Sequential(\n                component, nn.Linear(component.output_dim, self.pred_dim)\n            )\n\n        return component_\n\n    @staticmethod  # noqa: C901\n    def _check_inputs(  # noqa: C901\n        wide,\n        deeptabular,\n        deeptext,\n        deepimage,\n        deephead,\n        head_hidden_dims,\n        pred_dim,\n    ):\n        if wide is not None:\n            assert wide.wide_linear.weight.size(1) == pred_dim, (\n                \"the 'pred_dim' of the wide component ({}) must be equal to the 'pred_dim' \"\n                \"of the deep component and the overall model itself ({})\".format(\n                    wide.wide_linear.weight.size(1), pred_dim\n                )\n            )\n\n        if deeptabular is not None:\n            err_msg = (\n                \"deeptabular model must have an 'output_dim' attribute or property.\"\n            )\n            if isinstance(deeptabular, list):\n                all_have_output_dim = all(\n                    hasattr(dt, \"output_dim\") for dt in deeptabular\n                )\n                if not all_have_output_dim:\n                    raise AttributeError(err_msg)\n            else:\n                if not hasattr(deeptabular, \"output_dim\"):\n                    raise AttributeError(err_msg)\n                # the following assertion is thought for those cases where we\n                # use fusion with 'dot product' so that the output_dim will\n                # be 1 and the pred_dim is not 1\n                if deeptabular.output_dim == 1:\n                    assert pred_dim == 1, \"If 'output_dim' is 1, 'pred_dim' must be 1\"\n\n        if deeptabular is not None:\n            is_tabnet = False\n            if isinstance(deeptabular, list):\n                is_any_tabnet = any(\n                    dt.__class__.__name__ == \"TabNet\" for dt in deeptabular\n                )\n                if is_any_tabnet:\n                    raise ValueError(\n                        \"Currently TabNet is not supported as a component of a multiple \"\n                        \"tabular component model.\"\n                    )\n            else:\n                is_tabnet = deeptabular.__class__.__name__ == \"TabNet\"\n            has_wide_text_or_image = (\n                wide is not None or deeptext is not None or deepimage is not None\n            )\n            if is_tabnet and has_wide_text_or_image:\n                warnings.warn(\n                    \"'WideDeep' is a model comprised by multiple components and the 'deeptabular'\"\n                    \" component is 'TabNet'. We recommend using 'TabNet' in isolation.\"\n                    \" The reasons are: i)'TabNet' uses sparse regularization which partially losses\"\n                    \" its purpose when used in combination with other components.\"\n                    \" If you still want to use a multiple component model with 'TabNet',\"\n                    \" consider setting 'lambda_sparse' to 0 during training. ii) The feature\"\n                    \" importances will be computed only for TabNet but the model will comprise multiple\"\n                    \" components. Therefore, such importances will partially lose their 'meaning'.\",\n                    UserWarning,\n                )\n\n        if deeptext is not None:\n            err_msg = \"deeptext model must have an 'output_dim' attribute or property.\"\n            if isinstance(deeptext, list):\n                all_have_output_dim = all(hasattr(dt, \"output_dim\") for dt in deeptext)\n                if not all_have_output_dim:\n                    raise AttributeError(err_msg)\n            else:\n                if not hasattr(deeptext, \"output_dim\"):\n                    raise AttributeError(err_msg)\n                if deeptext.output_dim == 1:\n                    assert pred_dim == 1, \"If 'output_dim' is 1, 'pred_dim' must be 1\"\n\n        if deepimage is not None:\n            err_msg = \"deepimage model must have an 'output_dim' attribute or property.\"\n            if isinstance(deepimage, list):\n                all_have_output_dim = all(hasattr(di, \"output_dim\") for di in deepimage)\n                if not all_have_output_dim:\n                    raise AttributeError(err_msg)\n            else:\n                if not hasattr(deepimage, \"output_dim\"):\n                    raise AttributeError(err_msg)\n                if deepimage.output_dim == 1:\n                    assert pred_dim == 1, \"If 'output_dim' is 1, 'pred_dim' must be 1\"\n\n        if deephead is not None and head_hidden_dims is not None:\n            raise ValueError(\n                \"both 'deephead' and 'head_hidden_dims' are not None. Use one of the other, but not both\"\n            )\n        if (\n            head_hidden_dims is not None\n            and not deeptabular\n            and not deeptext\n            and not deepimage\n        ):\n            raise ValueError(\n                \"if 'head_hidden_dims' is not None, at least one deep component must be used\"\n            )\n\n        if deephead is not None:\n            if not hasattr(deephead, \"output_dim\"):\n                raise AttributeError(\n                    \"As any other custom model passed to 'WideDeep', 'deephead' must have an \"\n                    \"'output_dim' attribute or property. \"\n                )\n            deephead_inp_feat = next(deephead.parameters()).size(1)\n            output_dim = 0\n            if deeptabular is not None:\n                if isinstance(deeptabular, list):\n                    for dt in deeptabular:\n                        output_dim += dt.output_dim\n                else:\n                    output_dim += deeptabular.output_dim\n            if deeptext is not None:\n                if isinstance(deeptext, list):\n                    for dt in deeptext:\n                        output_dim += dt.output_dim\n                else:\n                    output_dim += deeptext.output_dim\n            if deepimage is not None:\n                if isinstance(deepimage, list):\n                    for di in deepimage:\n                        output_dim += di.output_dim\n                else:\n                    output_dim += deepimage.output_dim\n            if deephead_inp_feat != output_dim:\n                warnings.warn(\n                    \"A custom 'deephead' is used and it seems that the input features \"\n                    \"do not match the output of the deep components\",\n                    UserWarning,\n                )\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html","title":"The <code>preprocessing</code> module","text":"<p>This module contains the classes that are used to prepare the data before being passed to the models. There is one Preprocessor per data mode or model component (<code>wide</code>, <code>deeptabular</code>, <code>deepimage</code> and <code>deeptext</code>) with the exception of the <code>deeptext</code> component. In this case, two processors are available: one for the case when no Hugging Face model is used (<code>TextPreprocessor</code>) and another one when a Hugging Face model is used (<code>HFPreprocessor</code>).</p>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.wide_preprocessor.WidePreprocessor","title":"WidePreprocessor","text":"<p>               Bases: <code>BasePreprocessor</code></p> <p>Preprocessor to prepare the wide input dataset</p> <p>This Preprocessor prepares the data for the wide, linear component. This linear model is implemented via an Embedding layer that is connected to the output neuron. <code>WidePreprocessor</code> numerically encodes all the unique values of all categorical columns <code>wide_cols + crossed_cols</code>. See the Example below.</p> <p>Parameters:</p> Name Type Description Default <code>wide_cols</code> <code>List[str]</code> <p>List of strings with the name of the columns that will label encoded and passed through the <code>wide</code> component</p> required <code>crossed_cols</code> <code>Optional[List[Tuple[str, str]]]</code> <p>List of Tuples with the name of the columns that will be <code>'crossed'</code> and then label encoded. e.g. [('education', 'occupation'), ...]. For binary features, a cross-product transformation is 1 if and only if the constituent features are all 1, and 0 otherwise.</p> <code>None</code> <p>Attributes:</p> Name Type Description <code>wide_crossed_cols</code> <code>List</code> <p>List with the names of all columns that will be label encoded</p> <code>encoding_dict</code> <code>Dict</code> <p>Dictionary where the keys are the result of pasting <code>colname + '_' + column value</code> and the values are the corresponding mapped integer.</p> <code>inverse_encoding_dict</code> <code>Dict</code> <p>the inverse encoding dictionary</p> <code>wide_dim</code> <code>int</code> <p>Dimension of the wide model (i.e. dim of the linear layer)</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import pandas as pd\n&gt;&gt;&gt; from pytorch_widedeep.preprocessing import WidePreprocessor\n&gt;&gt;&gt; df = pd.DataFrame({'color': ['r', 'b', 'g'], 'size': ['s', 'n', 'l']})\n&gt;&gt;&gt; wide_cols = ['color']\n&gt;&gt;&gt; crossed_cols = [('color', 'size')]\n&gt;&gt;&gt; wide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols)\n&gt;&gt;&gt; X_wide = wide_preprocessor.fit_transform(df)\n&gt;&gt;&gt; X_wide\narray([[1, 4],\n       [2, 5],\n       [3, 6]])\n&gt;&gt;&gt; wide_preprocessor.encoding_dict\n{'color_r': 1, 'color_b': 2, 'color_g': 3, 'color_size_r-s': 4, 'color_size_b-n': 5, 'color_size_g-l': 6}\n&gt;&gt;&gt; wide_preprocessor.inverse_transform(X_wide)\n  color color_size\n0     r        r-s\n1     b        b-n\n2     g        g-l\n</code></pre> Source code in <code>pytorch_widedeep/preprocessing/wide_preprocessor.py</code> <pre><code>class WidePreprocessor(BasePreprocessor):\n    r\"\"\"Preprocessor to prepare the wide input dataset\n\n    This Preprocessor prepares the data for the wide, linear component.\n    This linear model is implemented via an Embedding layer that is\n    connected to the output neuron. `WidePreprocessor` numerically\n    encodes all the unique values of all categorical columns `wide_cols +\n    crossed_cols`. See the Example below.\n\n    Parameters\n    ----------\n    wide_cols: List\n        List of strings with the name of the columns that will label\n        encoded and passed through the `wide` component\n    crossed_cols: List, default = None\n        List of Tuples with the name of the columns that will be `'crossed'`\n        and then label encoded. e.g. _[('education', 'occupation'), ...]_. For\n        binary features, a cross-product transformation is 1 if and only if\n        the constituent features are all 1, and 0 otherwise.\n\n    Attributes\n    ----------\n    wide_crossed_cols: List\n        List with the names of all columns that will be label encoded\n    encoding_dict: Dict\n        Dictionary where the keys are the result of pasting `colname + '_' +\n        column value` and the values are the corresponding mapped integer.\n    inverse_encoding_dict: Dict\n        the inverse encoding dictionary\n    wide_dim: int\n        Dimension of the wide model (i.e. dim of the linear layer)\n\n    Examples\n    --------\n    &gt;&gt;&gt; import pandas as pd\n    &gt;&gt;&gt; from pytorch_widedeep.preprocessing import WidePreprocessor\n    &gt;&gt;&gt; df = pd.DataFrame({'color': ['r', 'b', 'g'], 'size': ['s', 'n', 'l']})\n    &gt;&gt;&gt; wide_cols = ['color']\n    &gt;&gt;&gt; crossed_cols = [('color', 'size')]\n    &gt;&gt;&gt; wide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols)\n    &gt;&gt;&gt; X_wide = wide_preprocessor.fit_transform(df)\n    &gt;&gt;&gt; X_wide\n    array([[1, 4],\n           [2, 5],\n           [3, 6]])\n    &gt;&gt;&gt; wide_preprocessor.encoding_dict\n    {'color_r': 1, 'color_b': 2, 'color_g': 3, 'color_size_r-s': 4, 'color_size_b-n': 5, 'color_size_g-l': 6}\n    &gt;&gt;&gt; wide_preprocessor.inverse_transform(X_wide)\n      color color_size\n    0     r        r-s\n    1     b        b-n\n    2     g        g-l\n    \"\"\"\n\n    def __init__(\n        self, wide_cols: List[str], crossed_cols: Optional[List[Tuple[str, str]]] = None\n    ):\n        super(WidePreprocessor, self).__init__()\n\n        self.wide_cols = wide_cols\n        self.crossed_cols = crossed_cols\n\n        self.is_fitted = False\n\n    def fit(self, df: pd.DataFrame) -&gt; \"WidePreprocessor\":\n        r\"\"\"Fits the Preprocessor and creates required attributes\n\n        Parameters\n        ----------\n        df: pd.DataFrame\n            Input pandas dataframe\n\n        Returns\n        -------\n        WidePreprocessor\n            `WidePreprocessor` fitted object\n        \"\"\"\n        df_wide = self._prepare_wide(df)\n        self.wide_crossed_cols = df_wide.columns.tolist()\n        glob_feature_list = self._make_global_feature_list(\n            df_wide[self.wide_crossed_cols]\n        )\n        # leave 0 for padding/\"unseen\" categories\n        self.encoding_dict = {v: i + 1 for i, v in enumerate(glob_feature_list)}\n        self.wide_dim = len(self.encoding_dict)\n        self.inverse_encoding_dict = {k: v for v, k in self.encoding_dict.items()}\n        self.inverse_encoding_dict[0] = \"unseen\"\n\n        self.is_fitted = True\n\n        return self\n\n    def transform(self, df: pd.DataFrame) -&gt; np.ndarray:\n        r\"\"\"\n        Parameters\n        ----------\n        df: pd.DataFrame\n            Input pandas dataframe\n\n        Returns\n        -------\n        np.ndarray\n            transformed input dataframe\n        \"\"\"\n        check_is_fitted(self, attributes=[\"encoding_dict\"])\n        df_wide = self._prepare_wide(df)\n        encoded = np.zeros([len(df_wide), len(self.wide_crossed_cols)])\n        for col_i, col in enumerate(self.wide_crossed_cols):\n            encoded[:, col_i] = df_wide[col].apply(\n                lambda x: (\n                    self.encoding_dict[col + \"_\" + str(x)]\n                    if col + \"_\" + str(x) in self.encoding_dict\n                    else 0\n                )\n            )\n        return encoded.astype(\"int64\")\n\n    def transform_sample(self, df: pd.DataFrame) -&gt; np.ndarray:\n        return self.transform(df)[0]\n\n    def inverse_transform(self, encoded: np.ndarray) -&gt; pd.DataFrame:\n        r\"\"\"Takes as input the output from the `transform` method and it will\n        return the original values.\n\n        Parameters\n        ----------\n        encoded: np.ndarray\n            numpy array with the encoded values that are the output from the\n            `transform` method\n\n        Returns\n        -------\n        pd.DataFrame\n            Pandas dataframe with the original values\n        \"\"\"\n        decoded = pd.DataFrame(encoded, columns=self.wide_crossed_cols)\n\n        if pd.__version__ &gt;= \"2.1.0\":\n            decoded = decoded.map(lambda x: self.inverse_encoding_dict[x])\n        else:\n            decoded = decoded.applymap(lambda x: self.inverse_encoding_dict[x])\n\n        for col in decoded.columns:\n            rm_str = \"\".join([col, \"_\"])\n            decoded[col] = decoded[col].apply(lambda x: x.replace(rm_str, \"\"))\n        return decoded\n\n    def fit_transform(self, df: pd.DataFrame) -&gt; np.ndarray:\n        \"\"\"Combines `fit` and `transform`\n\n        Parameters\n        ----------\n        df: pd.DataFrame\n            Input pandas dataframe\n\n        Returns\n        -------\n        np.ndarray\n            transformed input dataframe\n        \"\"\"\n        return self.fit(df).transform(df)\n\n    def _make_global_feature_list(self, df: pd.DataFrame) -&gt; List:\n        glob_feature_list = []\n        for column in df.columns:\n            glob_feature_list += self._make_column_feature_list(df[column])\n        return glob_feature_list\n\n    def _make_column_feature_list(self, s: pd.Series) -&gt; List:\n        return [s.name + \"_\" + str(x) for x in s.unique()]\n\n    def _cross_cols(self, df: pd.DataFrame):\n        df_cc = df.copy()\n        crossed_colnames = []\n        for cols in self.crossed_cols:\n            for c in cols:\n                df_cc[c] = df_cc[c].astype(\"str\")\n            colname = \"_\".join(cols)\n            df_cc[colname] = df_cc[list(cols)].apply(lambda x: \"-\".join(x), axis=1)\n            crossed_colnames.append(colname)\n        return df_cc[crossed_colnames]\n\n    def _prepare_wide(self, df: pd.DataFrame):\n        if self.crossed_cols is not None:\n            df_cc = self._cross_cols(df)\n            return pd.concat([df[self.wide_cols], df_cc], axis=1)\n        else:\n            return df.copy()[self.wide_cols]\n\n    def __repr__(self) -&gt; str:\n        list_of_params: List[str] = [\"wide_cols={wide_cols}\"]\n        if self.crossed_cols is not None:\n            list_of_params.append(\"crossed_cols={crossed_cols}\")\n        all_params = \", \".join(list_of_params)\n        return f\"WidePreprocessor({all_params.format(**self.__dict__)})\"\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.wide_preprocessor.WidePreprocessor.fit","title":"fit","text":"<pre><code>fit(df)\n</code></pre> <p>Fits the Preprocessor and creates required attributes</p> <p>Parameters:</p> Name Type Description Default <code>df</code> <code>DataFrame</code> <p>Input pandas dataframe</p> required <p>Returns:</p> Type Description <code>WidePreprocessor</code> <p><code>WidePreprocessor</code> fitted object</p> Source code in <code>pytorch_widedeep/preprocessing/wide_preprocessor.py</code> <pre><code>def fit(self, df: pd.DataFrame) -&gt; \"WidePreprocessor\":\n    r\"\"\"Fits the Preprocessor and creates required attributes\n\n    Parameters\n    ----------\n    df: pd.DataFrame\n        Input pandas dataframe\n\n    Returns\n    -------\n    WidePreprocessor\n        `WidePreprocessor` fitted object\n    \"\"\"\n    df_wide = self._prepare_wide(df)\n    self.wide_crossed_cols = df_wide.columns.tolist()\n    glob_feature_list = self._make_global_feature_list(\n        df_wide[self.wide_crossed_cols]\n    )\n    # leave 0 for padding/\"unseen\" categories\n    self.encoding_dict = {v: i + 1 for i, v in enumerate(glob_feature_list)}\n    self.wide_dim = len(self.encoding_dict)\n    self.inverse_encoding_dict = {k: v for v, k in self.encoding_dict.items()}\n    self.inverse_encoding_dict[0] = \"unseen\"\n\n    self.is_fitted = True\n\n    return self\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.wide_preprocessor.WidePreprocessor.transform","title":"transform","text":"<pre><code>transform(df)\n</code></pre> <p>Parameters:</p> Name Type Description Default <code>df</code> <code>DataFrame</code> <p>Input pandas dataframe</p> required <p>Returns:</p> Type Description <code>ndarray</code> <p>transformed input dataframe</p> Source code in <code>pytorch_widedeep/preprocessing/wide_preprocessor.py</code> <pre><code>def transform(self, df: pd.DataFrame) -&gt; np.ndarray:\n    r\"\"\"\n    Parameters\n    ----------\n    df: pd.DataFrame\n        Input pandas dataframe\n\n    Returns\n    -------\n    np.ndarray\n        transformed input dataframe\n    \"\"\"\n    check_is_fitted(self, attributes=[\"encoding_dict\"])\n    df_wide = self._prepare_wide(df)\n    encoded = np.zeros([len(df_wide), len(self.wide_crossed_cols)])\n    for col_i, col in enumerate(self.wide_crossed_cols):\n        encoded[:, col_i] = df_wide[col].apply(\n            lambda x: (\n                self.encoding_dict[col + \"_\" + str(x)]\n                if col + \"_\" + str(x) in self.encoding_dict\n                else 0\n            )\n        )\n    return encoded.astype(\"int64\")\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.wide_preprocessor.WidePreprocessor.inverse_transform","title":"inverse_transform","text":"<pre><code>inverse_transform(encoded)\n</code></pre> <p>Takes as input the output from the <code>transform</code> method and it will return the original values.</p> <p>Parameters:</p> Name Type Description Default <code>encoded</code> <code>ndarray</code> <p>numpy array with the encoded values that are the output from the <code>transform</code> method</p> required <p>Returns:</p> Type Description <code>DataFrame</code> <p>Pandas dataframe with the original values</p> Source code in <code>pytorch_widedeep/preprocessing/wide_preprocessor.py</code> <pre><code>def inverse_transform(self, encoded: np.ndarray) -&gt; pd.DataFrame:\n    r\"\"\"Takes as input the output from the `transform` method and it will\n    return the original values.\n\n    Parameters\n    ----------\n    encoded: np.ndarray\n        numpy array with the encoded values that are the output from the\n        `transform` method\n\n    Returns\n    -------\n    pd.DataFrame\n        Pandas dataframe with the original values\n    \"\"\"\n    decoded = pd.DataFrame(encoded, columns=self.wide_crossed_cols)\n\n    if pd.__version__ &gt;= \"2.1.0\":\n        decoded = decoded.map(lambda x: self.inverse_encoding_dict[x])\n    else:\n        decoded = decoded.applymap(lambda x: self.inverse_encoding_dict[x])\n\n    for col in decoded.columns:\n        rm_str = \"\".join([col, \"_\"])\n        decoded[col] = decoded[col].apply(lambda x: x.replace(rm_str, \"\"))\n    return decoded\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.wide_preprocessor.WidePreprocessor.fit_transform","title":"fit_transform","text":"<pre><code>fit_transform(df)\n</code></pre> <p>Combines <code>fit</code> and <code>transform</code></p> <p>Parameters:</p> Name Type Description Default <code>df</code> <code>DataFrame</code> <p>Input pandas dataframe</p> required <p>Returns:</p> Type Description <code>ndarray</code> <p>transformed input dataframe</p> Source code in <code>pytorch_widedeep/preprocessing/wide_preprocessor.py</code> <pre><code>def fit_transform(self, df: pd.DataFrame) -&gt; np.ndarray:\n    \"\"\"Combines `fit` and `transform`\n\n    Parameters\n    ----------\n    df: pd.DataFrame\n        Input pandas dataframe\n\n    Returns\n    -------\n    np.ndarray\n        transformed input dataframe\n    \"\"\"\n    return self.fit(df).transform(df)\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor","title":"TabPreprocessor","text":"<p>               Bases: <code>BasePreprocessor</code></p> <p>Preprocessor to prepare the <code>deeptabular</code> component input dataset</p> <p>Parameters:</p> Name Type Description Default <code>cat_embed_cols</code> <code>Optional[Union[List[str], List[Tuple[str, int]]]]</code> <p>List containing the name of the categorical columns that will be represented by embeddings (e.g. ['education', 'relationship', ...]) or a Tuple with the name and the embedding dimension (e.g.: [ ('education',32), ('relationship',16), ...]). Param alias: <code>embed_cols</code></p> <code>None</code> <code>continuous_cols</code> <code>Optional[List[str]]</code> <p>List with the name of the continuous cols</p> <code>None</code> <code>quantization_setup</code> <code>Optional[Union[int, Dict[str, Union[int, List[float]]]]]</code> <p>Continuous columns can be turned into categorical via <code>pd.cut</code>. If <code>quantization_setup</code> is an <code>int</code>, all continuous columns will be quantized using this value as the number of bins. Alternatively, a dictionary where the keys are the column names to quantize and the values are the either integers indicating the number of bins or a list of scalars indicating the bin edges can also be used. Param alias: <code>cols_and_bins</code></p> <code>None</code> <code>cols_to_scale</code> <code>Optional[Union[List[str], str]]</code> <p>List with the names of the columns that will be standarised via sklearn's <code>StandardScaler</code>. It can also be the string <code>'all'</code> in which case all the continuous cols will be scaled.</p> <code>None</code> <code>auto_embed_dim</code> <code>bool</code> <p>Boolean indicating whether the embedding dimensions will be automatically defined via rule of thumb. See <code>embedding_rule</code> below.</p> <code>True</code> <code>embedding_rule</code> <code>Literal[google, fastai_old, fastai_new]</code> <p>If <code>auto_embed_dim=True</code>, this is the choice of embedding rule of thumb. Choices are:</p> <ul> <li> <p>fastai_new: \\(min(600, round(1.6 \\times n_{cat}^{0.56}))\\)</p> </li> <li> <p>fastai_old: \\(min(50, (n_{cat}//{2})+1)\\)</p> </li> <li> <p>google: \\(min(600, round(n_{cat}^{0.24}))\\)</p> </li> </ul> <code>'fastai_new'</code> <code>default_embed_dim</code> <code>int</code> <p>Dimension for the embeddings if the embedding dimension is not provided in the <code>cat_embed_cols</code> parameter and <code>auto_embed_dim</code> is set to <code>False</code>.</p> <code>16</code> <code>with_attention</code> <code>bool</code> <p>Boolean indicating whether the preprocessed data will be passed to an attention-based model (more precisely a model where all embeddings must have the same dimensions). If <code>True</code>, the param <code>cat_embed_cols</code> must just be a list containing just the categorical column names: e.g. ['education', 'relationship', ...]. This is because they will all be  encoded using embeddings of the same dim, which will be specified  later when the model is defined.  Param alias:  <code>for_transformer</code>, <code>for_matrix_factorization</code>, <code>for_mf</code></p> <code>False</code> <code>with_cls_token</code> <code>bool</code> <p>Boolean indicating if a <code>'[CLS]'</code> token will be added to the dataset when using attention-based models. The final hidden state corresponding to this token is used as the aggregated representation for classification and regression tasks. If not, the categorical and/or continuous embeddings will be concatenated before being passed to the final MLP (if present).</p> <code>False</code> <code>shared_embed</code> <code>bool</code> <p>Boolean indicating if the embeddings will be \"shared\" when using attention-based models. The idea behind <code>shared_embed</code> is described in the Appendix A in the TabTransformer paper: 'The goal of having column embedding is to enable the model to distinguish the classes in one column from those in the other columns'. In other words, the idea is to let the model learn which column is embedded at the time. See: <code>pytorch_widedeep.models.transformers._layers.SharedEmbeddings</code>.</p> <code>False</code> <code>verbose</code> <code>int</code> <code>1</code> <code>scale</code> <code>bool</code> <p> note: this arg will be removed in upcoming  releases. Please use <code>cols_to_scale</code> instead.  Bool indicating  whether or not to scale/standarise continuous cols. It is important  to emphasize that all the DL models for tabular data in the library  also include the possibility of normalising the input continuous  features via a <code>BatchNorm</code> or a <code>LayerNorm</code>.  Param alias:  <code>scale_cont_cols</code>.</p> <code>False</code> <code>already_standard</code> <code>Optional[List[str]]</code> <p> note: this arg will be removed in upcoming  releases. Please use <code>cols_to_scale</code> instead.  List with the  name of the continuous cols that do not need to be  scaled/standarised.</p> <code>None</code> <p>Other Parameters:</p> Name Type Description <code>**kwargs</code> <p><code>pd.cut</code> and <code>StandardScaler</code> related args</p> <p>Attributes:</p> Name Type Description <code>cat_cols</code> <code>List[str]</code> <p>List containing the names of the categorical columns after processing.</p> <code>cols_and_bins</code> <code>Optional[Dict[str, Union[int, List[float]]]]</code> <p>Dictionary containing the quantization setup after processing if quantization is requested. This is derived from the quantization_setup parameter.</p> <code>quant_args</code> <code>Dict</code> <p>Dictionary containing arguments passed to pandas.cut() function for quantization.</p> <code>scale_args</code> <code>Dict</code> <p>Dictionary containing arguments passed to StandardScaler.</p> <code>label_encoder</code> <code>LabelEncoder</code> <p>Instance of LabelEncoder used to encode categorical variables. See <code>pytorch_widedeep.utils.dense_utils.LabelEncoder</code>.</p> <code>cat_embed_input</code> <code>List</code> <p>List of tuples with the column name, number of individual values for that column and, if <code>with_attention</code> is set to <code>False</code>, the corresponding embeddings dim, e.g. [('education', 16, 10), ('relationship', 6, 8), ...].</p> <code>standardize_cols</code> <code>List</code> <p>List of the columns that will be standardized.</p> <code>scaler</code> <code>StandardScaler</code> <p>An instance of <code>sklearn.preprocessing.StandardScaler</code> if standardization is requested.</p> <code>column_idx</code> <code>Dict</code> <p>Dictionary where keys are column names and values are column indexes. This is necessary to slice tensors.</p> <code>quantizer</code> <code>Quantizer</code> <p>An instance of <code>Quantizer</code> if quantization is requested.</p> <code>is_fitted</code> <code>bool</code> <p>Boolean indicating if the preprocessor has been fitted.</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import pandas as pd\n&gt;&gt;&gt; import numpy as np\n&gt;&gt;&gt; from pytorch_widedeep.preprocessing import TabPreprocessor\n&gt;&gt;&gt; df = pd.DataFrame({'color': ['r', 'b', 'g'], 'size': ['s', 'n', 'l'], 'age': [25, 40, 55]})\n&gt;&gt;&gt; cat_embed_cols = [('color',5), ('size',5)]\n&gt;&gt;&gt; cont_cols = ['age']\n&gt;&gt;&gt; deep_preprocessor = TabPreprocessor(cat_embed_cols=cat_embed_cols, continuous_cols=cont_cols)\n&gt;&gt;&gt; X_tab = deep_preprocessor.fit_transform(df)\n&gt;&gt;&gt; deep_preprocessor.cat_embed_cols\n[('color', 5), ('size', 5)]\n&gt;&gt;&gt; deep_preprocessor.column_idx\n{'color': 0, 'size': 1, 'age': 2}\n&gt;&gt;&gt; cont_df = pd.DataFrame({\"col1\": np.random.rand(10), \"col2\": np.random.rand(10) + 1})\n&gt;&gt;&gt; cont_cols = [\"col1\", \"col2\"]\n&gt;&gt;&gt; tab_preprocessor = TabPreprocessor(continuous_cols=cont_cols, quantization_setup=3)\n&gt;&gt;&gt; ft_cont_df = tab_preprocessor.fit_transform(cont_df)\n&gt;&gt;&gt; # or...\n&gt;&gt;&gt; quantization_setup = {'col1': [0., 0.4, 1.], 'col2': [1., 1.4, 2.]}\n&gt;&gt;&gt; tab_preprocessor2 = TabPreprocessor(continuous_cols=cont_cols, quantization_setup=quantization_setup)\n&gt;&gt;&gt; ft_cont_df2 = tab_preprocessor2.fit_transform(cont_df)\n</code></pre> Source code in <code>pytorch_widedeep/preprocessing/tab_preprocessor.py</code> <pre><code>class TabPreprocessor(BasePreprocessor):\n    r\"\"\"Preprocessor to prepare the `deeptabular` component input dataset\n\n    Parameters\n    ----------\n    cat_embed_cols: List, default = None\n        List containing the name of the categorical columns that will be\n        represented by embeddings (e.g. _['education', 'relationship', ...]_) or\n        a Tuple with the name and the embedding dimension (e.g.: _[\n        ('education',32), ('relationship',16), ...]_).&lt;br/&gt; Param alias: `embed_cols`\n    continuous_cols: List, default = None\n        List with the name of the continuous cols\n    quantization_setup: int or Dict, default = None\n        Continuous columns can be turned into categorical via `pd.cut`. If\n        `quantization_setup` is an `int`, all continuous columns will be\n        quantized using this value as the number of bins. Alternatively, a\n        dictionary where the keys are the column names to quantize and the\n        values are the either integers indicating the number of bins or a\n        list of scalars indicating the bin edges can also be used.&lt;br/&gt;\n        Param alias: `cols_and_bins`\n    cols_to_scale: List or str, default = None,\n        List with the names of the columns that will be standarised via\n        sklearn's `StandardScaler`. It can also be the string `'all'` in\n        which case all the continuous cols will be scaled.\n    auto_embed_dim: bool, default = True\n        Boolean indicating whether the embedding dimensions will be\n        automatically defined via rule of thumb. See `embedding_rule`\n        below.\n    embedding_rule: str, default = 'fastai_new'\n        If `auto_embed_dim=True`, this is the choice of embedding rule of\n        thumb. Choices are:\n\n        - _fastai_new_: $min(600, round(1.6 \\times n_{cat}^{0.56}))$\n\n        - _fastai_old_: $min(50, (n_{cat}//{2})+1)$\n\n        - _google_: $min(600, round(n_{cat}^{0.24}))$\n    default_embed_dim: int, default=16\n        Dimension for the embeddings if the embedding dimension is not\n        provided in the `cat_embed_cols` parameter and `auto_embed_dim` is\n        set to `False`.\n    with_attention: bool, default = False\n        Boolean indicating whether the preprocessed data will be passed to an\n        attention-based model (more precisely a model where all embeddings\n        must have the same dimensions). If `True`, the param `cat_embed_cols`\n        must just be a list containing just the categorical column names:\n        e.g.\n        _['education', 'relationship', ...]_. This is because they will all be\n         encoded using embeddings of the same dim, which will be specified\n         later when the model is defined. &lt;br/&gt; Param alias:\n         `for_transformer`, `for_matrix_factorization`, `for_mf`\n    with_cls_token: bool, default = False\n        Boolean indicating if a `'[CLS]'` token will be added to the dataset\n        when using attention-based models. The final hidden state\n        corresponding to this token is used as the aggregated representation\n        for classification and regression tasks. If not, the categorical\n        and/or continuous embeddings will be concatenated before being passed\n        to the final MLP (if present).\n    shared_embed: bool, default = False\n        Boolean indicating if the embeddings will be \"shared\" when using\n        attention-based models. The idea behind `shared_embed` is\n        described in the Appendix A in the [TabTransformer paper](https://arxiv.org/abs/2012.06678):\n        _'The goal of having column embedding is to enable the model to\n        distinguish the classes in one column from those in the other\n        columns'_. In other words, the idea is to let the model learn which\n        column is embedded at the time. See: `pytorch_widedeep.models.transformers._layers.SharedEmbeddings`.\n    verbose: int, default = 1\n    scale: bool, default = False\n        :information_source: **note**: this arg will be removed in upcoming\n         releases. Please use `cols_to_scale` instead. &lt;br/&gt; Bool indicating\n         whether or not to scale/standarise continuous cols. It is important\n         to emphasize that all the DL models for tabular data in the library\n         also include the possibility of normalising the input continuous\n         features via a `BatchNorm` or a `LayerNorm`. &lt;br/&gt; Param alias:\n         `scale_cont_cols`.\n    already_standard: List, default = None\n        :information_source: **note**: this arg will be removed in upcoming\n         releases. Please use `cols_to_scale` instead. &lt;br/&gt; List with the\n         name of the continuous cols that do not need to be\n         scaled/standarised.\n\n    Other Parameters\n    ----------------\n    **kwargs: dict\n        `pd.cut` and `StandardScaler` related args\n\n    Attributes\n    ----------\n    cat_cols: List[str]\n        List containing the names of the categorical columns after processing.\n    cols_and_bins: Optional[Dict[str, Union[int, List[float]]]]\n        Dictionary containing the quantization setup after processing if\n        quantization is requested. This is derived from the\n        quantization_setup parameter.\n    quant_args: Dict\n        Dictionary containing arguments passed to pandas.cut() function for quantization.\n    scale_args: Dict\n        Dictionary containing arguments passed to StandardScaler.\n    label_encoder: LabelEncoder\n        Instance of LabelEncoder used to encode categorical variables.\n        See `pytorch_widedeep.utils.dense_utils.LabelEncoder`.\n    cat_embed_input: List\n        List of tuples with the column name, number of individual values for\n        that column and, if `with_attention` is set to `False`, the\n        corresponding embeddings dim, e.g. _[('education', 16, 10),\n        ('relationship', 6, 8), ...]_.\n    standardize_cols: List\n        List of the columns that will be standardized.\n    scaler: StandardScaler\n        An instance of `sklearn.preprocessing.StandardScaler` if standardization\n        is requested.\n    column_idx: Dict\n        Dictionary where keys are column names and values are column indexes.\n        This is necessary to slice tensors.\n    quantizer: Quantizer\n        An instance of `Quantizer` if quantization is requested.\n    is_fitted: bool\n        Boolean indicating if the preprocessor has been fitted.\n\n    Examples\n    --------\n    &gt;&gt;&gt; import pandas as pd\n    &gt;&gt;&gt; import numpy as np\n    &gt;&gt;&gt; from pytorch_widedeep.preprocessing import TabPreprocessor\n    &gt;&gt;&gt; df = pd.DataFrame({'color': ['r', 'b', 'g'], 'size': ['s', 'n', 'l'], 'age': [25, 40, 55]})\n    &gt;&gt;&gt; cat_embed_cols = [('color',5), ('size',5)]\n    &gt;&gt;&gt; cont_cols = ['age']\n    &gt;&gt;&gt; deep_preprocessor = TabPreprocessor(cat_embed_cols=cat_embed_cols, continuous_cols=cont_cols)\n    &gt;&gt;&gt; X_tab = deep_preprocessor.fit_transform(df)\n    &gt;&gt;&gt; deep_preprocessor.cat_embed_cols\n    [('color', 5), ('size', 5)]\n    &gt;&gt;&gt; deep_preprocessor.column_idx\n    {'color': 0, 'size': 1, 'age': 2}\n    &gt;&gt;&gt; cont_df = pd.DataFrame({\"col1\": np.random.rand(10), \"col2\": np.random.rand(10) + 1})\n    &gt;&gt;&gt; cont_cols = [\"col1\", \"col2\"]\n    &gt;&gt;&gt; tab_preprocessor = TabPreprocessor(continuous_cols=cont_cols, quantization_setup=3)\n    &gt;&gt;&gt; ft_cont_df = tab_preprocessor.fit_transform(cont_df)\n    &gt;&gt;&gt; # or...\n    &gt;&gt;&gt; quantization_setup = {'col1': [0., 0.4, 1.], 'col2': [1., 1.4, 2.]}\n    &gt;&gt;&gt; tab_preprocessor2 = TabPreprocessor(continuous_cols=cont_cols, quantization_setup=quantization_setup)\n    &gt;&gt;&gt; ft_cont_df2 = tab_preprocessor2.fit_transform(cont_df)\n    \"\"\"\n\n    @alias(\"with_attention\", [\"for_transformer\", \"for_matrix_factorization\", \"for_mf\"])\n    @alias(\"cat_embed_cols\", [\"embed_cols\"])\n    @alias(\"scale\", [\"scale_cont_cols\"])\n    @alias(\"quantization_setup\", [\"cols_and_bins\"])\n    def __init__(\n        self,\n        cat_embed_cols: Optional[Union[List[str], List[Tuple[str, int]]]] = None,\n        continuous_cols: Optional[List[str]] = None,\n        quantization_setup: Optional[\n            Union[int, Dict[str, Union[int, List[float]]]]\n        ] = None,\n        cols_to_scale: Optional[Union[List[str], str]] = None,\n        auto_embed_dim: bool = True,\n        embedding_rule: Literal[\"google\", \"fastai_old\", \"fastai_new\"] = \"fastai_new\",\n        default_embed_dim: int = 16,\n        with_attention: bool = False,\n        with_cls_token: bool = False,\n        shared_embed: bool = False,\n        verbose: int = 1,\n        *,\n        scale: bool = False,\n        already_standard: Optional[List[str]] = None,\n        **kwargs,\n    ):\n        super(TabPreprocessor, self).__init__()\n\n        self.continuous_cols = continuous_cols\n        self.quantization_setup = quantization_setup\n        self.cols_to_scale = cols_to_scale\n        self.scale = scale\n        self.already_standard = already_standard\n        self.auto_embed_dim = auto_embed_dim\n        self.embedding_rule = embedding_rule\n        self.default_embed_dim = default_embed_dim\n        self.with_attention = with_attention\n        self.with_cls_token = with_cls_token\n        self.shared_embed = shared_embed\n        self.verbose = verbose\n\n        self.quant_args = {\n            k: v for k, v in kwargs.items() if k in pd.cut.__code__.co_varnames\n        }\n        self.scale_args = {\n            k: v for k, v in kwargs.items() if k in StandardScaler().get_params()\n        }\n\n        self._check_inputs(cat_embed_cols)\n\n        if with_cls_token:\n            self.cat_embed_cols = (\n                [\"cls_token\"] + cat_embed_cols  # type: ignore[operator]\n                if cat_embed_cols is not None\n                else [\"cls_token\"]\n            )\n        else:\n            self.cat_embed_cols = cat_embed_cols  # type: ignore[assignment]\n\n        self.is_fitted = False\n\n    def fit(self, df: pd.DataFrame) -&gt; BasePreprocessor:  # noqa: C901\n        \"\"\"Fits the Preprocessor and creates required attributes\n\n        Parameters\n        ----------\n        df: pd.DataFrame\n            Input pandas dataframe\n\n        Returns\n        -------\n        TabPreprocessor\n            `TabPreprocessor` fitted object\n        \"\"\"\n\n        df_adj = self._insert_cls_token(df) if self.with_cls_token else df.copy()\n\n        self.column_idx: Dict[str, int] = {}\n\n        # Categorical embeddings logic\n        if self.cat_embed_cols is not None or self.quantization_setup is not None:\n            self.cat_embed_input: List[Union[Tuple[str, int], Tuple[str, int, int]]] = (\n                []\n            )\n\n        if self.cat_embed_cols is not None:\n            df_cat, cat_embed_dim = self._prepare_categorical(df_adj)\n\n            self.label_encoder = LabelEncoder(\n                columns_to_encode=df_cat.columns.tolist(),\n                shared_embed=self.shared_embed,\n                with_attention=self.with_attention,\n            )\n            self.label_encoder.fit(df_cat)\n\n            for k, v in self.label_encoder.encoding_dict.items():\n                if self.with_attention:\n                    self.cat_embed_input.append((k, len(v)))\n                else:\n                    self.cat_embed_input.append((k, len(v), cat_embed_dim[k]))\n\n            self.column_idx.update({k: v for v, k in enumerate(df_cat.columns)})\n\n        # Continuous columns logic\n        if self.continuous_cols is not None:\n            df_cont, cont_embed_dim = self._prepare_continuous(df_adj)\n\n            # Standardization logic\n            if self.standardize_cols is not None:\n                self.scaler = StandardScaler(**self.scale_args).fit(\n                    df_cont[self.standardize_cols].values\n                )\n            elif self.verbose:\n                warnings.warn(\"Continuous columns will not be normalised\")\n\n            # Quantization logic\n            if self.cols_and_bins is not None:\n                # we do not run 'Quantizer.fit' here since in the wild case\n                # someone wants standardization and quantization for the same\n                # columns, the Quantizer will run on the scaled data\n                self.quantizer = Quantizer(self.cols_and_bins, **self.quant_args)\n\n                if self.with_attention:\n                    for col, n_cat, _ in cont_embed_dim:\n                        self.cat_embed_input.append((col, n_cat))\n                else:\n                    self.cat_embed_input.extend(cont_embed_dim)\n\n            self.column_idx.update(\n                {k: v + len(self.column_idx) for v, k in enumerate(df_cont)}\n            )\n\n        self.is_fitted = True\n\n        return self\n\n    def transform(self, df: pd.DataFrame) -&gt; np.ndarray:  # noqa: C901\n        \"\"\"Returns the processed `dataframe` as a np.ndarray\n\n        Parameters\n        ----------\n        df: pd.DataFrame\n            Input pandas dataframe\n\n        Returns\n        -------\n        np.ndarray\n            transformed input dataframe\n        \"\"\"\n        check_is_fitted(self, condition=self.is_fitted)\n\n        df_adj = self._insert_cls_token(df) if self.with_cls_token else df.copy()\n\n        if self.cat_embed_cols is not None:\n            df_cat = df_adj[self.cat_cols]\n            df_cat = self.label_encoder.transform(df_cat)\n        if self.continuous_cols is not None:\n            df_cont = df_adj[self.continuous_cols]\n            # Standardization logic\n            if self.standardize_cols:\n                df_cont[self.standardize_cols] = self.scaler.transform(\n                    df_cont[self.standardize_cols].values\n                )\n            # Quantization logic\n            if self.cols_and_bins is not None:\n                # Adjustment so I don't have to override the method\n                # in 'ChunkTabPreprocessor'\n                if self.quantizer.is_fitted:\n                    df_cont = self.quantizer.transform(df_cont)\n                else:\n                    df_cont = self.quantizer.fit_transform(df_cont)\n        try:\n            df_deep = pd.concat([df_cat, df_cont], axis=1)\n        except NameError:\n            try:\n                df_deep = df_cat.copy()\n            except NameError:\n                df_deep = df_cont.copy()\n\n        return df_deep.values\n\n    def transform_sample(self, df: pd.DataFrame) -&gt; np.ndarray:\n        return self.transform(df).astype(\"float\")[0]\n\n    def inverse_transform(self, encoded: np.ndarray) -&gt; pd.DataFrame:  # noqa: C901\n        r\"\"\"Takes as input the output from the `transform` method and it will\n        return the original values.\n\n        Parameters\n        ----------\n        encoded: np.ndarray\n            array with the output of the `transform` method\n\n        Returns\n        -------\n        pd.DataFrame\n            Pandas dataframe with the original values\n        \"\"\"\n        decoded = pd.DataFrame(encoded, columns=list(self.column_idx.keys()))\n        # embeddings back to original category\n        if self.cat_embed_cols is not None:\n            decoded = self.label_encoder.inverse_transform(decoded)\n        if self.continuous_cols is not None:\n            # quantized cols to the mid point\n            if self.cols_and_bins is not None:\n                if self.verbose:\n                    print(\n                        \"Note that quantized cols will be turned into the mid point of \"\n                        \"the corresponding bin\"\n                    )\n                for k, v in self.quantizer.inversed_bins.items():\n                    decoded[k] = decoded[k].map(v)\n            # continuous_cols back to non-standarised\n            try:\n                decoded[self.standardize_cols] = self.scaler.inverse_transform(\n                    decoded[self.standardize_cols]\n                )\n            except Exception:  # KeyError:\n                pass\n\n        if \"cls_token\" in decoded.columns:\n            decoded.drop(\"cls_token\", axis=1, inplace=True)\n\n        return decoded\n\n    def fit_transform(self, df: pd.DataFrame) -&gt; np.ndarray:\n        \"\"\"Combines `fit` and `transform`\n\n        Parameters\n        ----------\n        df: pd.DataFrame\n            Input pandas dataframe\n\n        Returns\n        -------\n        np.ndarray\n            transformed input dataframe\n        \"\"\"\n        return self.fit(df).transform(df)\n\n    def _insert_cls_token(self, df: pd.DataFrame) -&gt; pd.DataFrame:\n        df_cls = df.copy()\n        df_cls.insert(loc=0, column=\"cls_token\", value=\"[CLS]\")\n        return df_cls\n\n    def _prepare_categorical(\n        self, df: pd.DataFrame\n    ) -&gt; Tuple[pd.DataFrame, Dict[str, int]]:\n        if isinstance(self.cat_embed_cols[0], tuple):\n            self.cat_cols: List[str] = [emb[0] for emb in self.cat_embed_cols]\n            cat_embed_dim: Dict[str, int] = dict(self.cat_embed_cols)  # type: ignore\n        else:\n            self.cat_cols = self.cat_embed_cols  # type: ignore[assignment]\n            if self.auto_embed_dim:\n                assert isinstance(self.cat_embed_cols[0], str), (\n                    \"If 'auto_embed_dim' is 'True' and 'with_attention' is 'False', \"\n                    \"'cat_embed_cols' must be a list of strings with the columns to \"\n                    \"be encoded as embeddings.\"\n                )\n                n_cats: Dict[str, int] = {col: df[col].nunique() for col in self.cat_embed_cols}  # type: ignore[misc]\n                cat_embed_dim = {\n                    col: embed_sz_rule(n_cat, self.embedding_rule)  # type: ignore[misc]\n                    for col, n_cat in n_cats.items()\n                }\n            else:\n                cat_embed_dim = {\n                    e: self.default_embed_dim for e in self.cat_embed_cols  # type: ignore[misc]\n                }  # type: ignore\n        return df[self.cat_cols], cat_embed_dim\n\n    def _prepare_continuous(\n        self, df: pd.DataFrame\n    ) -&gt; Tuple[pd.DataFrame, Optional[List[Tuple[str, int, int]]]]:\n        # Standardization logic\n        if self.cols_to_scale is not None:\n            self.standardize_cols = (\n                self.cols_to_scale\n                if self.cols_to_scale != \"all\"\n                else self.continuous_cols\n            )\n        elif self.scale:\n            if self.already_standard is not None:\n                self.standardize_cols = [\n                    c for c in self.continuous_cols if c not in self.already_standard\n                ]\n            else:\n                self.standardize_cols = self.continuous_cols\n        else:\n            self.standardize_cols = None\n\n        # Quantization logic\n        if self.quantization_setup is not None:\n            # the quantized columns are then treated as categorical\n            quant_cont_embed_input: Optional[List[Tuple[str, int, int]]] = []\n            if isinstance(self.quantization_setup, int):\n                self.cols_and_bins: Optional[Dict[str, Union[int, List[float]]]] = {}\n                for col in self.continuous_cols:\n                    self.cols_and_bins[col] = self.quantization_setup\n                    quant_cont_embed_input.append(\n                        (\n                            col,\n                            self.quantization_setup,\n                            (\n                                embed_sz_rule(\n                                    self.quantization_setup + 1, self.embedding_rule  # type: ignore[arg-type]\n                                )\n                                if self.auto_embed_dim\n                                else self.default_embed_dim\n                            ),\n                        )\n                    )\n            else:\n                for col, val in self.quantization_setup.items():\n                    if isinstance(val, int):\n                        quant_cont_embed_input.append(\n                            (\n                                col,\n                                val,\n                                (\n                                    embed_sz_rule(\n                                        val + 1, self.embedding_rule  # type: ignore[arg-type]\n                                    )\n                                    if self.auto_embed_dim\n                                    else self.default_embed_dim\n                                ),\n                            )\n                        )\n                    else:\n                        quant_cont_embed_input.append(\n                            (\n                                col,\n                                len(val) - 1,\n                                (\n                                    embed_sz_rule(len(val), self.embedding_rule)  # type: ignore[arg-type]\n                                    if self.auto_embed_dim\n                                    else self.default_embed_dim\n                                ),\n                            )\n                        )\n\n                self.cols_and_bins = self.quantization_setup.copy()\n        else:\n            self.cols_and_bins = None\n            quant_cont_embed_input = None\n\n        return df[self.continuous_cols], quant_cont_embed_input\n\n    def _check_inputs(self, cat_embed_cols):  # noqa: C901\n        if self.scale or self.already_standard is not None:\n            warnings.warn(\n                \"'scale' and 'already_standard' will be deprecated in the next release. \"\n                \"Please use 'cols_to_scale' instead\",\n                DeprecationWarning,\n                stacklevel=2,\n            )\n\n        if self.scale:\n            if self.already_standard is not None:\n                standardize_cols = [\n                    c for c in self.continuous_cols if c not in self.already_standard\n                ]\n            else:\n                standardize_cols = self.continuous_cols\n        elif self.cols_to_scale is not None:\n            standardize_cols = self.cols_to_scale\n        else:\n            standardize_cols = None\n\n        if standardize_cols is not None:\n            if isinstance(self.quantization_setup, int):\n                cols_to_quantize_and_standardize = [\n                    c for c in standardize_cols if c in self.continuous_cols\n                ]\n            elif isinstance(self.quantization_setup, dict):\n                cols_to_quantize_and_standardize = [\n                    c for c in standardize_cols if c in self.quantization_setup\n                ]\n            else:\n                cols_to_quantize_and_standardize = None\n            if cols_to_quantize_and_standardize is not None:\n                warnings.warn(\n                    f\"the following columns: {cols_to_quantize_and_standardize} will be first scaled\"\n                    \" using a StandardScaler and then quantized. Make sure this is what you really want\"\n                )\n\n        if self.with_cls_token and not self.with_attention:\n            warnings.warn(\n                \"If 'with_cls_token' is set to 'True', 'with_attention' will be automatically \"\n                \"to 'True' if is 'False'\",\n                UserWarning,\n            )\n            self.with_attention = True\n\n        if (cat_embed_cols is None) and (self.continuous_cols is None):\n            raise ValueError(\n                \"'cat_embed_cols' and 'continuous_cols' are 'None'. Please, define at least one of the two.\"\n            )\n\n        if (\n            cat_embed_cols is not None\n            and self.continuous_cols is not None\n            and len(np.intersect1d(cat_embed_cols, self.continuous_cols)) &gt; 0\n        ):\n            overlapping_cols = list(\n                np.intersect1d(cat_embed_cols, self.continuous_cols)\n            )\n            raise ValueError(\n                \"Currently passing columns as both categorical and continuum is not supported.\"\n                \" Please, choose one or the other for the following columns: {}\".format(\n                    \", \".join(overlapping_cols)\n                )\n            )\n        transformer_error_message = (\n            \"If with_attention is 'True' cat_embed_cols must be a list \"\n            \" of strings with the columns to be encoded as embeddings.\"\n        )\n        if (\n            self.with_attention\n            and cat_embed_cols is not None\n            and isinstance(cat_embed_cols[0], tuple)\n        ):\n            raise ValueError(transformer_error_message)\n\n    def __repr__(self) -&gt; str:  # noqa: C901\n        list_of_params: List[str] = []\n        if self.cat_embed_cols is not None:\n            list_of_params.append(\"cat_embed_cols={cat_embed_cols}\")\n        if self.continuous_cols is not None:\n            list_of_params.append(\"continuous_cols={continuous_cols}\")\n        if self.quantization_setup is not None:\n            list_of_params.append(\"quantization_setup={quantization_setup}\")\n        if self.cols_to_scale is not None:\n            list_of_params.append(\"cols_to_scale={cols_to_scale}\")\n        if not self.auto_embed_dim:\n            list_of_params.append(\"auto_embed_dim={auto_embed_dim}\")\n        if self.embedding_rule != \"fastai_new\":\n            list_of_params.append(\"embedding_rule='{embedding_rule}'\")\n        if self.default_embed_dim != 16:\n            list_of_params.append(\"default_embed_dim={default_embed_dim}\")\n        if self.with_attention:\n            list_of_params.append(\"with_attention={with_attention}\")\n        if self.with_cls_token:\n            list_of_params.append(\"with_cls_token={with_cls_token}\")\n        if self.shared_embed:\n            list_of_params.append(\"shared_embed={shared_embed}\")\n        if self.verbose != 1:\n            list_of_params.append(\"verbose={verbose}\")\n        if self.scale:\n            list_of_params.append(\"scale={scale}\")\n        if self.already_standard is not None:\n            list_of_params.append(\"already_standard={already_standard}\")\n        if len(self.quant_args) &gt; 0:\n            list_of_params.append(\n                \", \".join([f\"{k}\" + \"=\" + f\"{v}\" for k, v in self.quant_args.items()])\n            )\n        if len(self.scale_args) &gt; 0:\n            list_of_params.append(\n                \", \".join([f\"{k}\" + \"=\" + f\"{v}\" for k, v in self.scale_args.items()])\n            )\n        all_params = \", \".join(list_of_params)\n        return f\"TabPreprocessor({all_params.format(**self.__dict__)})\"\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor.fit","title":"fit","text":"<pre><code>fit(df)\n</code></pre> <p>Fits the Preprocessor and creates required attributes</p> <p>Parameters:</p> Name Type Description Default <code>df</code> <code>DataFrame</code> <p>Input pandas dataframe</p> required <p>Returns:</p> Type Description <code>TabPreprocessor</code> <p><code>TabPreprocessor</code> fitted object</p> Source code in <code>pytorch_widedeep/preprocessing/tab_preprocessor.py</code> <pre><code>def fit(self, df: pd.DataFrame) -&gt; BasePreprocessor:  # noqa: C901\n    \"\"\"Fits the Preprocessor and creates required attributes\n\n    Parameters\n    ----------\n    df: pd.DataFrame\n        Input pandas dataframe\n\n    Returns\n    -------\n    TabPreprocessor\n        `TabPreprocessor` fitted object\n    \"\"\"\n\n    df_adj = self._insert_cls_token(df) if self.with_cls_token else df.copy()\n\n    self.column_idx: Dict[str, int] = {}\n\n    # Categorical embeddings logic\n    if self.cat_embed_cols is not None or self.quantization_setup is not None:\n        self.cat_embed_input: List[Union[Tuple[str, int], Tuple[str, int, int]]] = (\n            []\n        )\n\n    if self.cat_embed_cols is not None:\n        df_cat, cat_embed_dim = self._prepare_categorical(df_adj)\n\n        self.label_encoder = LabelEncoder(\n            columns_to_encode=df_cat.columns.tolist(),\n            shared_embed=self.shared_embed,\n            with_attention=self.with_attention,\n        )\n        self.label_encoder.fit(df_cat)\n\n        for k, v in self.label_encoder.encoding_dict.items():\n            if self.with_attention:\n                self.cat_embed_input.append((k, len(v)))\n            else:\n                self.cat_embed_input.append((k, len(v), cat_embed_dim[k]))\n\n        self.column_idx.update({k: v for v, k in enumerate(df_cat.columns)})\n\n    # Continuous columns logic\n    if self.continuous_cols is not None:\n        df_cont, cont_embed_dim = self._prepare_continuous(df_adj)\n\n        # Standardization logic\n        if self.standardize_cols is not None:\n            self.scaler = StandardScaler(**self.scale_args).fit(\n                df_cont[self.standardize_cols].values\n            )\n        elif self.verbose:\n            warnings.warn(\"Continuous columns will not be normalised\")\n\n        # Quantization logic\n        if self.cols_and_bins is not None:\n            # we do not run 'Quantizer.fit' here since in the wild case\n            # someone wants standardization and quantization for the same\n            # columns, the Quantizer will run on the scaled data\n            self.quantizer = Quantizer(self.cols_and_bins, **self.quant_args)\n\n            if self.with_attention:\n                for col, n_cat, _ in cont_embed_dim:\n                    self.cat_embed_input.append((col, n_cat))\n            else:\n                self.cat_embed_input.extend(cont_embed_dim)\n\n        self.column_idx.update(\n            {k: v + len(self.column_idx) for v, k in enumerate(df_cont)}\n        )\n\n    self.is_fitted = True\n\n    return self\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor.transform","title":"transform","text":"<pre><code>transform(df)\n</code></pre> <p>Returns the processed <code>dataframe</code> as a np.ndarray</p> <p>Parameters:</p> Name Type Description Default <code>df</code> <code>DataFrame</code> <p>Input pandas dataframe</p> required <p>Returns:</p> Type Description <code>ndarray</code> <p>transformed input dataframe</p> Source code in <code>pytorch_widedeep/preprocessing/tab_preprocessor.py</code> <pre><code>def transform(self, df: pd.DataFrame) -&gt; np.ndarray:  # noqa: C901\n    \"\"\"Returns the processed `dataframe` as a np.ndarray\n\n    Parameters\n    ----------\n    df: pd.DataFrame\n        Input pandas dataframe\n\n    Returns\n    -------\n    np.ndarray\n        transformed input dataframe\n    \"\"\"\n    check_is_fitted(self, condition=self.is_fitted)\n\n    df_adj = self._insert_cls_token(df) if self.with_cls_token else df.copy()\n\n    if self.cat_embed_cols is not None:\n        df_cat = df_adj[self.cat_cols]\n        df_cat = self.label_encoder.transform(df_cat)\n    if self.continuous_cols is not None:\n        df_cont = df_adj[self.continuous_cols]\n        # Standardization logic\n        if self.standardize_cols:\n            df_cont[self.standardize_cols] = self.scaler.transform(\n                df_cont[self.standardize_cols].values\n            )\n        # Quantization logic\n        if self.cols_and_bins is not None:\n            # Adjustment so I don't have to override the method\n            # in 'ChunkTabPreprocessor'\n            if self.quantizer.is_fitted:\n                df_cont = self.quantizer.transform(df_cont)\n            else:\n                df_cont = self.quantizer.fit_transform(df_cont)\n    try:\n        df_deep = pd.concat([df_cat, df_cont], axis=1)\n    except NameError:\n        try:\n            df_deep = df_cat.copy()\n        except NameError:\n            df_deep = df_cont.copy()\n\n    return df_deep.values\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor.inverse_transform","title":"inverse_transform","text":"<pre><code>inverse_transform(encoded)\n</code></pre> <p>Takes as input the output from the <code>transform</code> method and it will return the original values.</p> <p>Parameters:</p> Name Type Description Default <code>encoded</code> <code>ndarray</code> <p>array with the output of the <code>transform</code> method</p> required <p>Returns:</p> Type Description <code>DataFrame</code> <p>Pandas dataframe with the original values</p> Source code in <code>pytorch_widedeep/preprocessing/tab_preprocessor.py</code> <pre><code>def inverse_transform(self, encoded: np.ndarray) -&gt; pd.DataFrame:  # noqa: C901\n    r\"\"\"Takes as input the output from the `transform` method and it will\n    return the original values.\n\n    Parameters\n    ----------\n    encoded: np.ndarray\n        array with the output of the `transform` method\n\n    Returns\n    -------\n    pd.DataFrame\n        Pandas dataframe with the original values\n    \"\"\"\n    decoded = pd.DataFrame(encoded, columns=list(self.column_idx.keys()))\n    # embeddings back to original category\n    if self.cat_embed_cols is not None:\n        decoded = self.label_encoder.inverse_transform(decoded)\n    if self.continuous_cols is not None:\n        # quantized cols to the mid point\n        if self.cols_and_bins is not None:\n            if self.verbose:\n                print(\n                    \"Note that quantized cols will be turned into the mid point of \"\n                    \"the corresponding bin\"\n                )\n            for k, v in self.quantizer.inversed_bins.items():\n                decoded[k] = decoded[k].map(v)\n        # continuous_cols back to non-standarised\n        try:\n            decoded[self.standardize_cols] = self.scaler.inverse_transform(\n                decoded[self.standardize_cols]\n            )\n        except Exception:  # KeyError:\n            pass\n\n    if \"cls_token\" in decoded.columns:\n        decoded.drop(\"cls_token\", axis=1, inplace=True)\n\n    return decoded\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor.fit_transform","title":"fit_transform","text":"<pre><code>fit_transform(df)\n</code></pre> <p>Combines <code>fit</code> and <code>transform</code></p> <p>Parameters:</p> Name Type Description Default <code>df</code> <code>DataFrame</code> <p>Input pandas dataframe</p> required <p>Returns:</p> Type Description <code>ndarray</code> <p>transformed input dataframe</p> Source code in <code>pytorch_widedeep/preprocessing/tab_preprocessor.py</code> <pre><code>def fit_transform(self, df: pd.DataFrame) -&gt; np.ndarray:\n    \"\"\"Combines `fit` and `transform`\n\n    Parameters\n    ----------\n    df: pd.DataFrame\n        Input pandas dataframe\n\n    Returns\n    -------\n    np.ndarray\n        transformed input dataframe\n    \"\"\"\n    return self.fit(df).transform(df)\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.tab_preprocessor.Quantizer","title":"Quantizer","text":"<p>Helper class to perform the quantization of continuous columns. It is included in this docs for completion, since depending on the value of the parameter <code>'quantization_setup'</code> of the <code>TabPreprocessor</code> class, that class might have an attribute of type <code>Quantizer</code>. However, this class is designed to always run internally within the <code>TabPreprocessor</code> class.</p> <p>Parameters:</p> Name Type Description Default <code>quantization_setup</code> <code>Dict[str, Union[int, List[float]]]</code> <p>Dictionary where the keys are the column names to quantize and the values are the either integers indicating the number of bins or a list of scalars indicating the bin edges.</p> required Source code in <code>pytorch_widedeep/preprocessing/tab_preprocessor.py</code> <pre><code>class Quantizer:\n    \"\"\"Helper class to perform the quantization of continuous columns. It is\n    included in this docs for completion, since depending on the value of the\n    parameter `'quantization_setup'` of the `TabPreprocessor` class, that\n    class might have an attribute of type `Quantizer`. However, this class is\n    designed to always run internally within the `TabPreprocessor` class.\n\n    Parameters\n    ----------\n    quantization_setup: Dict, default = None\n        Dictionary where the keys are the column names to quantize and the\n        values are the either integers indicating the number of bins or a\n        list of scalars indicating the bin edges.\n    \"\"\"\n\n    def __init__(\n        self,\n        quantization_setup: Dict[str, Union[int, List[float]]],\n        **kwargs,\n    ):\n        self.quantization_setup = quantization_setup\n        self.quant_args = kwargs\n\n        self.is_fitted = False\n\n    def fit(self, df: pd.DataFrame) -&gt; \"Quantizer\":\n        self.bins: Dict[str, List[float]] = {}\n        for col, bins in self.quantization_setup.items():\n            _, self.bins[col] = pd.cut(\n                df[col], bins, retbins=True, labels=False, **self.quant_args\n            )\n\n        self.inversed_bins: Dict[str, Dict[int, float]] = {}\n        for col, bins in self.bins.items():\n            self.inversed_bins[col] = {\n                k: v\n                for k, v in list(\n                    zip(\n                        range(1, len(bins) + 1),\n                        [(a + b) / 2.0 for a, b in zip(bins, bins[1:])],\n                    )\n                )\n            }\n            # 0\n            self.inversed_bins[col][0] = np.nan\n\n        self.is_fitted = True\n\n        return self\n\n    def transform(self, df: pd.DataFrame) -&gt; pd.DataFrame:\n        check_is_fitted(self, condition=self.is_fitted)\n\n        dfc = df.copy()\n        for col, bins in self.bins.items():\n            dfc[col] = pd.cut(dfc[col], bins, labels=False, **self.quant_args)\n            # 0 will be left for numbers outside the bins, i.e. smaller than\n            # the smaller boundary or larger than the largest boundary\n            dfc[col] = dfc[col] + 1\n            dfc[col] = dfc[col].fillna(0)\n            dfc[col] = dfc[col].astype(int)\n\n        return dfc\n\n    def fit_transform(self, df: pd.DataFrame) -&gt; pd.DataFrame:\n        return self.fit(df).transform(df)\n\n    def __repr__(self) -&gt; str:\n        return f\"Quantizer(quantization_setup={self.quantization_setup})\"\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor","title":"TextPreprocessor","text":"<p>               Bases: <code>BasePreprocessor</code></p> <p>Preprocessor to prepare the <code>deeptext</code> input dataset</p> <p>Parameters:</p> Name Type Description Default <code>text_col</code> <code>str</code> <p>column in the input dataframe containing the texts</p> required <code>max_vocab</code> <code>int</code> <p>Maximum number of tokens in the vocabulary</p> <code>30000</code> <code>min_freq</code> <code>int</code> <p>Minimum frequency for a token to be part of the vocabulary</p> <code>5</code> <code>maxlen</code> <code>int</code> <p>Maximum length of the tokenized sequences</p> <code>80</code> <code>pad_first</code> <code>bool</code> <p>Indicates whether the padding index will be added at the beginning or the end of the sequences</p> <code>True</code> <code>pad_idx</code> <code>int</code> <p>padding index. Fastai's Tokenizer leaves 0 for the 'unknown' token.</p> <code>1</code> <code>already_processed</code> <code>Optional[bool]</code> <p>Boolean indicating if the sequence of elements is already processed or prepared. If this is the case, this Preprocessor will simply tokenize and pad the sequence. </p> <pre><code>Param aliases: `not_text`. &lt;br/&gt;\n</code></pre> <p>This parameter is thought for those cases where the input sequences are already fully processed or are directly not text (e.g. IDs)</p> <code>False</code> <code>word_vectors_path</code> <code>Optional[str]</code> <p>Path to the pretrained word vectors</p> <code>None</code> <code>n_cpus</code> <code>Optional[int]</code> <p>number of CPUs to used during the tokenization process</p> <code>None</code> <code>verbose</code> <code>int</code> <p>Enable verbose output.</p> <code>1</code> <p>Attributes:</p> Name Type Description <code>vocab</code> <code>Vocab</code> <p>an instance of <code>pytorch_widedeep.utils.fastai_transforms.Vocab</code></p> <code>embedding_matrix</code> <code>ndarray</code> <p>Array with the pretrained embeddings</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import pandas as pd\n&gt;&gt;&gt; from pytorch_widedeep.preprocessing import TextPreprocessor\n&gt;&gt;&gt; df_train = pd.DataFrame({'text_column': [\"life is like a box of chocolates\",\n... \"You never know what you're gonna get\"]})\n&gt;&gt;&gt; text_preprocessor = TextPreprocessor(text_col='text_column', max_vocab=25, min_freq=1, maxlen=10)\n&gt;&gt;&gt; text_preprocessor.fit_transform(df_train)\nThe vocabulary contains 24 tokens\narray([[ 1,  1,  1,  1, 10, 11, 12, 13, 14, 15],\n       [ 5,  9, 16, 17, 18,  9, 19, 20, 21, 22]], dtype=int32)\n&gt;&gt;&gt; df_te = pd.DataFrame({'text_column': ['you never know what is in the box']})\n&gt;&gt;&gt; text_preprocessor.transform(df_te)\narray([[ 1,  1,  9, 16, 17, 18, 11,  0,  0, 13]], dtype=int32)\n</code></pre> Source code in <code>pytorch_widedeep/preprocessing/text_preprocessor.py</code> <pre><code>class TextPreprocessor(BasePreprocessor):\n    r\"\"\"Preprocessor to prepare the ``deeptext`` input dataset\n\n    Parameters\n    ----------\n    text_col: str\n        column in the input dataframe containing the texts\n    max_vocab: int, default=30000\n        Maximum number of tokens in the vocabulary\n    min_freq: int, default=5\n        Minimum frequency for a token to be part of the vocabulary\n    maxlen: int, default=80\n        Maximum length of the tokenized sequences\n    pad_first: bool,  default = True\n        Indicates whether the padding index will be added at the beginning or the\n        end of the sequences\n    pad_idx: int, default = 1\n        padding index. Fastai's Tokenizer leaves 0 for the 'unknown' token.\n    already_processed: bool, Optional, default = False\n        Boolean indicating if the sequence of elements is already processed or\n        prepared. If this is the case, this Preprocessor will simply tokenize\n        and pad the sequence. &lt;br/&gt;\n\n            Param aliases: `not_text`. &lt;br/&gt;\n\n        This parameter is thought for those cases where the input sequences\n        are already fully processed or are directly not text (e.g. IDs)\n    word_vectors_path: str, Optional\n        Path to the pretrained word vectors\n    n_cpus: int, Optional, default = None\n        number of CPUs to used during the tokenization process\n    verbose: int, default 1\n        Enable verbose output.\n\n    Attributes\n    ----------\n    vocab: Vocab\n        an instance of `pytorch_widedeep.utils.fastai_transforms.Vocab`\n    embedding_matrix: np.ndarray\n        Array with the pretrained embeddings\n\n    Examples\n    ---------\n    &gt;&gt;&gt; import pandas as pd\n    &gt;&gt;&gt; from pytorch_widedeep.preprocessing import TextPreprocessor\n    &gt;&gt;&gt; df_train = pd.DataFrame({'text_column': [\"life is like a box of chocolates\",\n    ... \"You never know what you're gonna get\"]})\n    &gt;&gt;&gt; text_preprocessor = TextPreprocessor(text_col='text_column', max_vocab=25, min_freq=1, maxlen=10)\n    &gt;&gt;&gt; text_preprocessor.fit_transform(df_train)\n    The vocabulary contains 24 tokens\n    array([[ 1,  1,  1,  1, 10, 11, 12, 13, 14, 15],\n           [ 5,  9, 16, 17, 18,  9, 19, 20, 21, 22]], dtype=int32)\n    &gt;&gt;&gt; df_te = pd.DataFrame({'text_column': ['you never know what is in the box']})\n    &gt;&gt;&gt; text_preprocessor.transform(df_te)\n    array([[ 1,  1,  9, 16, 17, 18, 11,  0,  0, 13]], dtype=int32)\n    \"\"\"\n\n    @alias(\"already_processed\", [\"not_text\"])\n    def __init__(\n        self,\n        text_col: str,\n        max_vocab: int = 30000,\n        min_freq: int = 5,\n        maxlen: int = 80,\n        pad_first: bool = True,\n        pad_idx: int = 1,\n        already_processed: Optional[bool] = False,\n        word_vectors_path: Optional[str] = None,\n        n_cpus: Optional[int] = None,\n        verbose: int = 1,\n    ):\n        super(TextPreprocessor, self).__init__()\n\n        self.text_col = text_col\n        self.max_vocab = max_vocab\n        self.min_freq = min_freq\n        self.maxlen = maxlen\n        self.pad_first = pad_first\n        self.pad_idx = pad_idx\n        self.already_processed = already_processed\n        self.word_vectors_path = word_vectors_path\n        self.verbose = verbose\n        self.n_cpus = n_cpus if n_cpus is not None else os.cpu_count()\n\n        self.is_fitted = False\n\n    def fit(self, df: pd.DataFrame) -&gt; BasePreprocessor:\n        \"\"\"Builds the vocabulary\n\n        Parameters\n        ----------\n        df: pd.DataFrame\n            Input pandas dataframe\n\n        Returns\n        -------\n        TextPreprocessor\n            `TextPreprocessor` fitted object\n        \"\"\"\n        texts = self._read_texts(df)\n\n        tokens = get_texts(texts, self.already_processed, self.n_cpus)\n\n        self.vocab: TVocab = Vocab(\n            max_vocab=self.max_vocab,\n            min_freq=self.min_freq,\n            pad_idx=self.pad_idx,\n        ).fit(\n            tokens,\n        )\n\n        if self.verbose:\n            print(\"The vocabulary contains {} tokens\".format(len(self.vocab.stoi)))\n        if self.word_vectors_path is not None:\n            self.embedding_matrix = build_embeddings_matrix(\n                self.vocab, self.word_vectors_path, self.min_freq\n            )\n\n        self.is_fitted = True\n\n        return self\n\n    def transform(self, df: pd.DataFrame) -&gt; np.ndarray:\n        \"\"\"Returns the padded, _'numericalised'_ sequences\n\n        Parameters\n        ----------\n        df: pd.DataFrame\n            Input pandas dataframe\n\n        Returns\n        -------\n        np.ndarray\n            Padded, _'numericalised'_ sequences\n        \"\"\"\n        check_is_fitted(self, attributes=[\"vocab\"])\n        texts = self._read_texts(df)\n        tokens = get_texts(texts, self.already_processed, self.n_cpus)\n        return self._pad_sequences(tokens)\n\n    def transform_sample(self, text: str) -&gt; np.ndarray:\n        \"\"\"Returns the padded, _'numericalised'_ sequence\n\n        Parameters\n        ----------\n        text: str\n            text to be tokenized and padded\n\n        Returns\n        -------\n        np.ndarray\n            Padded, _'numericalised'_ sequence\n        \"\"\"\n        check_is_fitted(self, attributes=[\"vocab\"])\n        tokens = get_texts([text], self.already_processed, self.n_cpus)\n        return self._pad_sequences(tokens)[0]\n\n    def fit_transform(self, df: pd.DataFrame) -&gt; np.ndarray:\n        \"\"\"Combines `fit` and `transform`\n\n        Parameters\n        ----------\n        df: pd.DataFrame\n            Input pandas dataframe\n\n        Returns\n        -------\n        np.ndarray\n            Padded, _'numericalised'_ sequences\n        \"\"\"\n        return self.fit(df).transform(df)\n\n    def inverse_transform(self, padded_seq: np.ndarray) -&gt; pd.DataFrame:\n        \"\"\"Returns the original text plus the added 'special' tokens\n\n        Parameters\n        ----------\n        padded_seq: np.ndarray\n            array with the output of the `transform` method\n\n        Returns\n        -------\n        pd.DataFrame\n            Pandas dataframe with the original text plus the added 'special' tokens\n        \"\"\"\n        texts = [self.vocab.inverse_transform(num) for num in padded_seq]\n        return pd.DataFrame({self.text_col: texts})\n\n    def _pad_sequences(self, tokens: List[List[str]]) -&gt; np.ndarray:\n        sequences = [self.vocab.transform(t) for t in tokens]\n        padded_seq = np.array(\n            [\n                pad_sequences(\n                    s,\n                    maxlen=self.maxlen,\n                    pad_first=self.pad_first,\n                    pad_idx=self.pad_idx,\n                )\n                for s in sequences\n            ]\n        )\n        return padded_seq\n\n    def _read_texts(\n        self, df: pd.DataFrame, root_dir: Optional[str] = None\n    ) -&gt; List[str]:\n        if root_dir is not None:\n            if not os.path.exists(root_dir):\n                raise ValueError(\n                    \"root_dir does not exist. Please create it before fitting the preprocessor\"\n                )\n            texts_fnames = df[self.text_col].tolist()\n            texts: List[str] = []\n            for texts_fname in texts_fnames:\n                with open(os.path.join(root_dir, texts_fname), \"r\") as f:\n                    texts.append(f.read().replace(\"\\n\", \"\"))\n        else:\n            texts = df[self.text_col].tolist()\n\n        return texts\n\n    def _load_vocab(self, vocab: TVocab) -&gt; None:\n        self.vocab = vocab\n\n    def __repr__(self) -&gt; str:\n        list_of_params: List[str] = [\"text_col={text_col}\"]\n        list_of_params.append(\"max_vocab={max_vocab}\")\n        list_of_params.append(\"min_freq={min_freq}\")\n        list_of_params.append(\"maxlen={maxlen}\")\n        list_of_params.append(\"pad_first={pad_first}\")\n        list_of_params.append(\"pad_idx={pad_idx}\")\n        list_of_params.append(\"already_processed={already_processed}\")\n        if self.word_vectors_path is not None:\n            list_of_params.append(\"word_vectors_path={word_vectors_path}\")\n        if self.n_cpus is not None:\n            list_of_params.append(\"n_cpus={n_cpus}\")\n        if self.verbose is not None:\n            list_of_params.append(\"verbose={verbose}\")\n        all_params = \", \".join(list_of_params)\n        return f\"TextPreprocessor({all_params.format(**self.__dict__)})\"\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor.fit","title":"fit","text":"<pre><code>fit(df)\n</code></pre> <p>Builds the vocabulary</p> <p>Parameters:</p> Name Type Description Default <code>df</code> <code>DataFrame</code> <p>Input pandas dataframe</p> required <p>Returns:</p> Type Description <code>TextPreprocessor</code> <p><code>TextPreprocessor</code> fitted object</p> Source code in <code>pytorch_widedeep/preprocessing/text_preprocessor.py</code> <pre><code>def fit(self, df: pd.DataFrame) -&gt; BasePreprocessor:\n    \"\"\"Builds the vocabulary\n\n    Parameters\n    ----------\n    df: pd.DataFrame\n        Input pandas dataframe\n\n    Returns\n    -------\n    TextPreprocessor\n        `TextPreprocessor` fitted object\n    \"\"\"\n    texts = self._read_texts(df)\n\n    tokens = get_texts(texts, self.already_processed, self.n_cpus)\n\n    self.vocab: TVocab = Vocab(\n        max_vocab=self.max_vocab,\n        min_freq=self.min_freq,\n        pad_idx=self.pad_idx,\n    ).fit(\n        tokens,\n    )\n\n    if self.verbose:\n        print(\"The vocabulary contains {} tokens\".format(len(self.vocab.stoi)))\n    if self.word_vectors_path is not None:\n        self.embedding_matrix = build_embeddings_matrix(\n            self.vocab, self.word_vectors_path, self.min_freq\n        )\n\n    self.is_fitted = True\n\n    return self\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor.transform","title":"transform","text":"<pre><code>transform(df)\n</code></pre> <p>Returns the padded, 'numericalised' sequences</p> <p>Parameters:</p> Name Type Description Default <code>df</code> <code>DataFrame</code> <p>Input pandas dataframe</p> required <p>Returns:</p> Type Description <code>ndarray</code> <p>Padded, 'numericalised' sequences</p> Source code in <code>pytorch_widedeep/preprocessing/text_preprocessor.py</code> <pre><code>def transform(self, df: pd.DataFrame) -&gt; np.ndarray:\n    \"\"\"Returns the padded, _'numericalised'_ sequences\n\n    Parameters\n    ----------\n    df: pd.DataFrame\n        Input pandas dataframe\n\n    Returns\n    -------\n    np.ndarray\n        Padded, _'numericalised'_ sequences\n    \"\"\"\n    check_is_fitted(self, attributes=[\"vocab\"])\n    texts = self._read_texts(df)\n    tokens = get_texts(texts, self.already_processed, self.n_cpus)\n    return self._pad_sequences(tokens)\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor.transform_sample","title":"transform_sample","text":"<pre><code>transform_sample(text)\n</code></pre> <p>Returns the padded, 'numericalised' sequence</p> <p>Parameters:</p> Name Type Description Default <code>text</code> <code>str</code> <p>text to be tokenized and padded</p> required <p>Returns:</p> Type Description <code>ndarray</code> <p>Padded, 'numericalised' sequence</p> Source code in <code>pytorch_widedeep/preprocessing/text_preprocessor.py</code> <pre><code>def transform_sample(self, text: str) -&gt; np.ndarray:\n    \"\"\"Returns the padded, _'numericalised'_ sequence\n\n    Parameters\n    ----------\n    text: str\n        text to be tokenized and padded\n\n    Returns\n    -------\n    np.ndarray\n        Padded, _'numericalised'_ sequence\n    \"\"\"\n    check_is_fitted(self, attributes=[\"vocab\"])\n    tokens = get_texts([text], self.already_processed, self.n_cpus)\n    return self._pad_sequences(tokens)[0]\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor.fit_transform","title":"fit_transform","text":"<pre><code>fit_transform(df)\n</code></pre> <p>Combines <code>fit</code> and <code>transform</code></p> <p>Parameters:</p> Name Type Description Default <code>df</code> <code>DataFrame</code> <p>Input pandas dataframe</p> required <p>Returns:</p> Type Description <code>ndarray</code> <p>Padded, 'numericalised' sequences</p> Source code in <code>pytorch_widedeep/preprocessing/text_preprocessor.py</code> <pre><code>def fit_transform(self, df: pd.DataFrame) -&gt; np.ndarray:\n    \"\"\"Combines `fit` and `transform`\n\n    Parameters\n    ----------\n    df: pd.DataFrame\n        Input pandas dataframe\n\n    Returns\n    -------\n    np.ndarray\n        Padded, _'numericalised'_ sequences\n    \"\"\"\n    return self.fit(df).transform(df)\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor.inverse_transform","title":"inverse_transform","text":"<pre><code>inverse_transform(padded_seq)\n</code></pre> <p>Returns the original text plus the added 'special' tokens</p> <p>Parameters:</p> Name Type Description Default <code>padded_seq</code> <code>ndarray</code> <p>array with the output of the <code>transform</code> method</p> required <p>Returns:</p> Type Description <code>DataFrame</code> <p>Pandas dataframe with the original text plus the added 'special' tokens</p> Source code in <code>pytorch_widedeep/preprocessing/text_preprocessor.py</code> <pre><code>def inverse_transform(self, padded_seq: np.ndarray) -&gt; pd.DataFrame:\n    \"\"\"Returns the original text plus the added 'special' tokens\n\n    Parameters\n    ----------\n    padded_seq: np.ndarray\n        array with the output of the `transform` method\n\n    Returns\n    -------\n    pd.DataFrame\n        Pandas dataframe with the original text plus the added 'special' tokens\n    \"\"\"\n    texts = [self.vocab.inverse_transform(num) for num in padded_seq]\n    return pd.DataFrame({self.text_col: texts})\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor","title":"HFPreprocessor","text":"<p>               Bases: <code>BasePreprocessor</code></p> <p>Text processor to prepare the <code>deeptext</code> input dataset that is a wrapper around HuggingFace's tokenizers.</p> <p>Following the main phylosophy of the <code>pytorch-widedeep</code> library, this class is designed to be as flexible as possible. Therefore, it is coded so that the user can use it as one would use any HuggingFace tokenizers, or following the API call 'protocol' of the rest of the library.</p> <p>Parameters:</p> Name Type Description Default <code>model_name</code> <code>str</code> <p>The model name from the transformers library e.g. 'bert-base-uncased'. Currently supported models are those from the families: BERT, RoBERTa, DistilBERT, ALBERT and ELECTRA.</p> required <code>use_fast_tokenizer</code> <code>bool</code> <p>Whether to use the fast tokenizer from HuggingFace or not</p> <code>False</code> <code>text_col</code> <code>Optional[str]</code> <p>The column in the input dataframe containing the text data. If this tokenizer is used via the <code>fit</code> and <code>transform</code> methods, this argument is mandatory. If the tokenizer is used via the <code>encode</code> method, this argument is not needed since the input text is passed directly to the <code>encode</code> method.</p> <code>None</code> <code>num_workers</code> <code>Optional[int]</code> <p>Number of workers to use when preprocessing the text data. If not None, and <code>use_fast_tokenizer</code> is False, the text data will be preprocessed in parallel using the number of workers specified. If <code>use_fast_tokenizer</code> is True, this argument is ignored.</p> <code>None</code> <code>preprocessing_rules</code> <code>Optional[List[Callable[[str], str]]]</code> <p>A list of functions to be applied to the text data before encoding. This can be useful to clean the text data before encoding. For example, removing html tags, special characters, etc.</p> <code>None</code> <code>tokenizer_params</code> <code>Optional[Dict[str, Any]]</code> <p>Additional parameters to be passed to the HuggingFace's <code>PreTrainedTokenizer</code>. Parameters to the <code>PreTrainedTokenizer</code> can also be passed via the <code>**kwargs</code> argument</p> <code>None</code> <code>encode_params</code> <code>Optional[Dict[str, Any]]</code> <p>Additional parameters to be passed to the <code>batch_encode_plus</code> method of the HuggingFace's <code>PreTrainedTokenizer</code>. If the <code>fit</code> and <code>transform</code> methods are used, the <code>encode_params</code> dict parameter is mandatory. If the <code>encode</code> method is used, this parameter is not needed since the input text is passed directly to the <code>encode</code> method.</p> <code>None</code> <code>**kwargs</code> <p>Additional kwargs to be passed to the model, in particular to the <code>PreTrainedTokenizer</code> class.</p> <code>{}</code> <p>Attributes:</p> Name Type Description <code>is_fitted</code> <code>bool</code> <p>Boolean indicating if the preprocessor has been fitted. This is a HuggingFacea tokenizer, so it is always considered fitted and this attribute is manually set to True internally. This parameter exists for consistency with the rest of the library and because is needed for some functionality in the library.</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import pandas as pd\n&gt;&gt;&gt; from pytorch_widedeep.preprocessing import HFPreprocessor\n&gt;&gt;&gt; df = pd.DataFrame({\"text\": [\"this is the first text\", \"this is the second text\"]})\n&gt;&gt;&gt; hf_processor_1 = HFPreprocessor(model_name=\"bert-base-uncased\", text_col=\"text\")\n&gt;&gt;&gt; X_text_1 = hf_processor_1.fit_transform(df)\n&gt;&gt;&gt; texts = [\"this is a new text\", \"this is another text\"]\n&gt;&gt;&gt; hf_processor_2 = HFPreprocessor(model_name=\"bert-base-uncased\")\n&gt;&gt;&gt; X_text_2 = hf_processor_2.encode(texts, max_length=10, padding=\"max_length\", truncation=True)\n</code></pre> Source code in <code>pytorch_widedeep/preprocessing/hf_preprocessor.py</code> <pre><code>class HFPreprocessor(BasePreprocessor):\n    \"\"\"Text processor to prepare the ``deeptext`` input dataset that is a\n    wrapper around HuggingFace's tokenizers.\n\n    Following the main phylosophy of the `pytorch-widedeep` library, this\n    class is designed to be as flexible as possible. Therefore, it is coded\n    so that the user can use it as one would use any HuggingFace tokenizers,\n    or following the API call 'protocol' of the rest of the library.\n\n    Parameters\n    ----------\n    model_name: str\n        The model name from the transformers library e.g. _'bert-base-uncased'_.\n        Currently supported models are those from the families: BERT, RoBERTa,\n        DistilBERT, ALBERT and ELECTRA.\n    use_fast_tokenizer: bool, default = False\n        Whether to use the fast tokenizer from HuggingFace or not\n    text_col: Optional[str], default = None\n        The column in the input dataframe containing the text data. If this\n        tokenizer is used via the `fit` and `transform` methods, this\n        argument is mandatory. If the tokenizer is used via the `encode`\n        method, this argument is not needed since the input text is passed\n        directly to the `encode` method.\n    num_workers: Optional[int], default = None\n        Number of workers to use when preprocessing the text data. If not\n        None, and `use_fast_tokenizer` is False, the text data will be\n        preprocessed in parallel using the number of workers specified. If\n        `use_fast_tokenizer` is True, this argument is ignored.\n    preprocessing_rules: Optional[List[Callable[[str], str]]], default = None\n        A list of functions to be applied to the text data before encoding.\n        This can be useful to clean the text data before encoding. For\n        example, removing html tags, special characters, etc.\n    tokenizer_params: Optional[Dict[str, Any]], default = None\n        Additional parameters to be passed to the HuggingFace's\n        `PreTrainedTokenizer`. Parameters to the `PreTrainedTokenizer`\n        can also be passed via the `**kwargs` argument\n    encode_params: Optional[Dict[str, Any]], default = None\n        Additional parameters to be passed to the `batch_encode_plus` method\n        of the HuggingFace's `PreTrainedTokenizer`. If the `fit` and `transform`\n        methods are used, the `encode_params` dict parameter is mandatory. If\n        the `encode` method is used, this parameter is not needed since the\n        input text is passed directly to the `encode` method.\n    **kwargs\n        Additional kwargs to be passed to the model, in particular to the\n        `PreTrainedTokenizer` class.\n\n    Attributes\n    ----------\n    is_fitted: bool\n        Boolean indicating if the preprocessor has been fitted. This is a\n        HuggingFacea tokenizer, so it is always considered fitted and this\n        attribute is manually set to True internally. This parameter exists\n        for consistency with the rest of the library and because is needed\n        for some functionality in the library.\n\n    Examples\n    --------\n    &gt;&gt;&gt; import pandas as pd\n    &gt;&gt;&gt; from pytorch_widedeep.preprocessing import HFPreprocessor\n    &gt;&gt;&gt; df = pd.DataFrame({\"text\": [\"this is the first text\", \"this is the second text\"]})\n    &gt;&gt;&gt; hf_processor_1 = HFPreprocessor(model_name=\"bert-base-uncased\", text_col=\"text\")\n    &gt;&gt;&gt; X_text_1 = hf_processor_1.fit_transform(df)\n    &gt;&gt;&gt; texts = [\"this is a new text\", \"this is another text\"]\n    &gt;&gt;&gt; hf_processor_2 = HFPreprocessor(model_name=\"bert-base-uncased\")\n    &gt;&gt;&gt; X_text_2 = hf_processor_2.encode(texts, max_length=10, padding=\"max_length\", truncation=True)\n    \"\"\"\n\n    def __init__(\n        self,\n        model_name: str,\n        *,\n        use_fast_tokenizer: bool = False,\n        text_col: Optional[str] = None,\n        root_dir: Optional[str] = None,\n        num_workers: Optional[int] = None,\n        preprocessing_rules: Optional[List[Callable[[str], str]]] = None,\n        tokenizer_params: Optional[Dict[str, Any]] = None,\n        encode_params: Optional[Dict[str, Any]] = None,\n        **kwargs,\n    ):\n        self.model_name = model_name\n        self.use_fast_tokenizer = use_fast_tokenizer\n        self.text_col = text_col\n        self.root_dir = root_dir\n        self.num_workers = num_workers\n        self.preprocessing_rules = preprocessing_rules\n        self.tokenizer_params = tokenizer_params if tokenizer_params is not None else {}\n        self.encode_params = encode_params if encode_params is not None else {}\n\n        self._multiprocessing = (\n            num_workers is not None and num_workers &gt; 1 and not use_fast_tokenizer\n        )\n\n        if kwargs:\n            self.tokenizer_params.update(kwargs)\n\n        self.tokenizer = get_tokenizer(\n            model_name=self.model_name,\n            use_fast_tokenizer=self.use_fast_tokenizer,\n            **self.tokenizer_params,\n        )\n\n        # A HuggingFace tokenizer is already trained, since we need this\n        # attribute elsewhere in the library, we simply set it to True\n        self.is_fitted = True\n\n    def encode(self, texts: List[str], **kwargs) -&gt; npt.NDArray[np.int64]:\n        \"\"\"\n        Encodes a list of texts. The method is a wrapper around the\n        `batch_encode_plus` method of the HuggingFace's tokenizer.\n\n        if 'use_fast_tokenizer' is True, the method will use the `batch_encode_plus`\n\n        Parameters\n        ----------\n        texts: List[str]\n            List of texts to be encoded\n        **kwargs\n            Additional parameters to be passed to the `batch_encode_plus` method\n            of the HuggingFace's tokenizer. If the 'encode_params' dict was passed\n            when instantiating the class, that dictionaly will be updated with\n            the kwargs passed here.\n\n        Returns\n        -------\n        np.array\n            The encoded texts\n        \"\"\"\n        if kwargs:\n            self.encode_params.update(kwargs)\n\n        if self.preprocessing_rules:\n            if self._multiprocessing:\n                texts = self._process_text_parallel(texts)\n            else:\n                texts = [self._preprocess_text(text) for text in texts]\n\n        if self._multiprocessing:\n            input_ids = self._encode_paralell(texts, **self.encode_params)\n        else:\n            encoded_texts = self.tokenizer.batch_encode_plus(\n                texts,\n                **self.encode_params,\n            )\n            input_ids = encoded_texts.get(\"input_ids\")\n\n        self.is_fitted = True\n\n        try:\n            output = np.array(input_ids)\n        except ValueError:\n            warnings.warn(\n                \"Padding and Truncating parameters were not passed and all input arrays \"\n                \"do not have the same shape. Padding to the longest sequence. \"\n                \"Padding will be done with the index of the pad token for the model\",\n                UserWarning,\n            )\n            max_len = max([len(ids) for ids in input_ids])\n            output = np.array(\n                [\n                    np.pad(ids, (self.tokenizer.pad_token_id, max_len - len(ids)))\n                    for ids in input_ids\n                ]\n            )\n\n        return output\n\n    def decode(\n        self, input_ids: npt.NDArray[np.int64], skip_special_tokens: bool\n    ) -&gt; List[str]:\n        \"\"\"\n        Decodes a list of input_ids. The method is a wrapper around the\n        `convert_ids_to_tokens` and `convert_tokens_to_string` methods of the\n        HuggingFace's tokenizer.\n\n        Parameters\n        ----------\n        input_ids: npt.NDArray[np.int64]\n            The input_ids to be decoded\n        skip_special_tokens: bool\n            Whether to skip the special tokens or not\n\n        Returns\n        -------\n        List[str]\n            The decoded texts\n        \"\"\"\n        texts = [\n            self.tokenizer.convert_tokens_to_string(\n                self.tokenizer.convert_ids_to_tokens(input_ids[i], skip_special_tokens)\n            )\n            for i in range(input_ids.shape[0])\n        ]\n        return texts\n\n    def fit(self, df: pd.DataFrame) -&gt; \"HFPreprocessor\":\n        \"\"\"\n        This method is included for consistency with the rest of the library\n        in general and with the `BasePreprocessor` in particular. HuggingFace's\n        tokenizers and models are already trained. Therefore, the 'fit' method\n        here does nothing other than checking that the 'text_col' parameter is\n        not `None`.\n\n        Parameters\n        ----------\n        df: pd.DataFrame\n            The dataframe containing the text data in the column specified by\n            the 'text_col' parameter\n        \"\"\"\n        if self.text_col is None:\n            raise ValueError(\n                \"'text_col' is None. Please specify the column name containing the text data\"\n                \" if you want to use the 'fit' method\"\n            )\n        return self\n\n    def transform(self, df: pd.DataFrame) -&gt; npt.NDArray[np.int64]:\n        \"\"\"\n        Encodes the text data in the input dataframe. This method simply\n        calls the `encode` method under the hood. Similar to the `fit` method,\n        this method is included for consistency with the rest of the library\n        in general and with the `BasePreprocessor` in particular.\n\n        Parameters\n        ----------\n        df: pd.DataFrame\n            The dataframe containing the text data in the column specified by\n            the 'text_col' parameter\n\n        Returns\n        -------\n        np.array\n            The encoded texts\n        \"\"\"\n        if self.text_col is None:\n            raise ValueError(\n                \"'text_col' is None. Please specify the column name containing the text data\"\n                \" if you want to use the 'fit' method\"\n            )\n\n        texts = self._read_texts(df, self.root_dir)\n\n        return self.encode(texts)\n\n    def transform_sample(self, text: str) -&gt; npt.NDArray[np.int64]:\n        \"\"\"\n        Encodes a single text sample.\n\n        Parameters\n        ----------\n        text: str\n            The text sample to be encoded\n\n        Returns\n        -------\n        np.array\n            The encoded text\n        \"\"\"\n\n        if not self.is_fitted:\n            raise ValueError(\n                \"The `encode` (or `fit`) method must be called before calling `transform_sample`\"\n            )\n        return self.encode([text])[0]\n\n    def fit_transform(self, df: pd.DataFrame) -&gt; npt.NDArray[np.int64]:\n        \"\"\"\n        Encodes the text data in the input dataframe.\n\n        Parameters\n        ----------\n        df: pd.DataFrame\n            The dataframe containing the text data in the column specified by\n            the 'text_col' parameter\n\n        Returns\n        -------\n        np.array\n            The encoded texts\n        \"\"\"\n        return self.fit(df).transform(df)\n\n    def inverse_transform(\n        self, input_ids: npt.NDArray[np.int64], skip_special_tokens: bool\n    ) -&gt; List[str]:\n        \"\"\"\n        Decodes a list of input_ids. The method simply calls the `decode` method\n        under the hood.\n\n        Parameters\n        ----------\n        input_ids: npt.NDArray[np.int64]\n            The input_ids to be decoded\n        skip_special_tokens: bool\n            Whether to skip the special tokens or not\n\n        Returns\n        -------\n        List[str]\n            The decoded texts\n        \"\"\"\n        return self.decode(input_ids, skip_special_tokens)\n\n    def _process_text_parallel(self, texts: List[str]) -&gt; List[str]:\n        num_processes = (\n            self.num_workers if self.num_workers is not None else os.cpu_count()\n        )\n        with ProcessPoolExecutor(max_workers=num_processes) as executor:\n            processed_texts = list(executor.map(self._preprocess_text, texts))\n        return processed_texts\n\n    def _preprocess_text(self, text: str) -&gt; str:\n        for rule in self.preprocessing_rules:\n            text = rule(text)\n        return text\n\n    def _encode_paralell(self, texts: List[str], **kwargs) -&gt; List[List[int]]:\n        num_processes = (\n            self.num_workers if self.num_workers is not None else os.cpu_count()\n        )\n        ptokenizer = partial(self.tokenizer.encode, **kwargs)\n        with ProcessPoolExecutor(max_workers=num_processes) as executor:\n            encoded_texts = list(executor.map(ptokenizer, texts))\n        return encoded_texts\n\n    def _read_texts(\n        self, df: pd.DataFrame, root_dir: Optional[str] = None\n    ) -&gt; List[str]:\n        if root_dir is not None:\n            if not os.path.exists(root_dir):\n                raise ValueError(\n                    \"root_dir does not exist. Please create it before fitting the preprocessor\"\n                )\n            texts_fnames = df[self.text_col].tolist()\n            texts: List[str] = []\n            for texts_fname in texts_fnames:\n                with open(os.path.join(root_dir, texts_fname), \"r\") as f:\n                    texts.append(f.read().replace(\"\\n\", \"\"))\n        else:\n            texts = df[self.text_col].tolist()\n\n        return texts\n\n    def __repr__(self):\n        return (\n            f\"HFPreprocessor(text_col={self.text_col}, model_name={self.model_name}, \"\n            f\"use_fast_tokenizer={self.use_fast_tokenizer}, num_workers={self.num_workers}, \"\n            f\"preprocessing_rules={self.preprocessing_rules}, tokenizer_params={self.tokenizer_params}, \"\n            f\"encode_params={self.encode_params})\"\n        )\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.encode","title":"encode","text":"<pre><code>encode(texts, **kwargs)\n</code></pre> <p>Encodes a list of texts. The method is a wrapper around the <code>batch_encode_plus</code> method of the HuggingFace's tokenizer.</p> <p>if 'use_fast_tokenizer' is True, the method will use the <code>batch_encode_plus</code></p> <p>Parameters:</p> Name Type Description Default <code>texts</code> <code>List[str]</code> <p>List of texts to be encoded</p> required <code>**kwargs</code> <p>Additional parameters to be passed to the <code>batch_encode_plus</code> method of the HuggingFace's tokenizer. If the 'encode_params' dict was passed when instantiating the class, that dictionaly will be updated with the kwargs passed here.</p> <code>{}</code> <p>Returns:</p> Type Description <code>array</code> <p>The encoded texts</p> Source code in <code>pytorch_widedeep/preprocessing/hf_preprocessor.py</code> <pre><code>def encode(self, texts: List[str], **kwargs) -&gt; npt.NDArray[np.int64]:\n    \"\"\"\n    Encodes a list of texts. The method is a wrapper around the\n    `batch_encode_plus` method of the HuggingFace's tokenizer.\n\n    if 'use_fast_tokenizer' is True, the method will use the `batch_encode_plus`\n\n    Parameters\n    ----------\n    texts: List[str]\n        List of texts to be encoded\n    **kwargs\n        Additional parameters to be passed to the `batch_encode_plus` method\n        of the HuggingFace's tokenizer. If the 'encode_params' dict was passed\n        when instantiating the class, that dictionaly will be updated with\n        the kwargs passed here.\n\n    Returns\n    -------\n    np.array\n        The encoded texts\n    \"\"\"\n    if kwargs:\n        self.encode_params.update(kwargs)\n\n    if self.preprocessing_rules:\n        if self._multiprocessing:\n            texts = self._process_text_parallel(texts)\n        else:\n            texts = [self._preprocess_text(text) for text in texts]\n\n    if self._multiprocessing:\n        input_ids = self._encode_paralell(texts, **self.encode_params)\n    else:\n        encoded_texts = self.tokenizer.batch_encode_plus(\n            texts,\n            **self.encode_params,\n        )\n        input_ids = encoded_texts.get(\"input_ids\")\n\n    self.is_fitted = True\n\n    try:\n        output = np.array(input_ids)\n    except ValueError:\n        warnings.warn(\n            \"Padding and Truncating parameters were not passed and all input arrays \"\n            \"do not have the same shape. Padding to the longest sequence. \"\n            \"Padding will be done with the index of the pad token for the model\",\n            UserWarning,\n        )\n        max_len = max([len(ids) for ids in input_ids])\n        output = np.array(\n            [\n                np.pad(ids, (self.tokenizer.pad_token_id, max_len - len(ids)))\n                for ids in input_ids\n            ]\n        )\n\n    return output\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.decode","title":"decode","text":"<pre><code>decode(input_ids, skip_special_tokens)\n</code></pre> <p>Decodes a list of input_ids. The method is a wrapper around the <code>convert_ids_to_tokens</code> and <code>convert_tokens_to_string</code> methods of the HuggingFace's tokenizer.</p> <p>Parameters:</p> Name Type Description Default <code>input_ids</code> <code>NDArray[int64]</code> <p>The input_ids to be decoded</p> required <code>skip_special_tokens</code> <code>bool</code> <p>Whether to skip the special tokens or not</p> required <p>Returns:</p> Type Description <code>List[str]</code> <p>The decoded texts</p> Source code in <code>pytorch_widedeep/preprocessing/hf_preprocessor.py</code> <pre><code>def decode(\n    self, input_ids: npt.NDArray[np.int64], skip_special_tokens: bool\n) -&gt; List[str]:\n    \"\"\"\n    Decodes a list of input_ids. The method is a wrapper around the\n    `convert_ids_to_tokens` and `convert_tokens_to_string` methods of the\n    HuggingFace's tokenizer.\n\n    Parameters\n    ----------\n    input_ids: npt.NDArray[np.int64]\n        The input_ids to be decoded\n    skip_special_tokens: bool\n        Whether to skip the special tokens or not\n\n    Returns\n    -------\n    List[str]\n        The decoded texts\n    \"\"\"\n    texts = [\n        self.tokenizer.convert_tokens_to_string(\n            self.tokenizer.convert_ids_to_tokens(input_ids[i], skip_special_tokens)\n        )\n        for i in range(input_ids.shape[0])\n    ]\n    return texts\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.fit","title":"fit","text":"<pre><code>fit(df)\n</code></pre> <p>This method is included for consistency with the rest of the library in general and with the <code>BasePreprocessor</code> in particular. HuggingFace's tokenizers and models are already trained. Therefore, the 'fit' method here does nothing other than checking that the 'text_col' parameter is not <code>None</code>.</p> <p>Parameters:</p> Name Type Description Default <code>df</code> <code>DataFrame</code> <p>The dataframe containing the text data in the column specified by the 'text_col' parameter</p> required Source code in <code>pytorch_widedeep/preprocessing/hf_preprocessor.py</code> <pre><code>def fit(self, df: pd.DataFrame) -&gt; \"HFPreprocessor\":\n    \"\"\"\n    This method is included for consistency with the rest of the library\n    in general and with the `BasePreprocessor` in particular. HuggingFace's\n    tokenizers and models are already trained. Therefore, the 'fit' method\n    here does nothing other than checking that the 'text_col' parameter is\n    not `None`.\n\n    Parameters\n    ----------\n    df: pd.DataFrame\n        The dataframe containing the text data in the column specified by\n        the 'text_col' parameter\n    \"\"\"\n    if self.text_col is None:\n        raise ValueError(\n            \"'text_col' is None. Please specify the column name containing the text data\"\n            \" if you want to use the 'fit' method\"\n        )\n    return self\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.transform","title":"transform","text":"<pre><code>transform(df)\n</code></pre> <p>Encodes the text data in the input dataframe. This method simply calls the <code>encode</code> method under the hood. Similar to the <code>fit</code> method, this method is included for consistency with the rest of the library in general and with the <code>BasePreprocessor</code> in particular.</p> <p>Parameters:</p> Name Type Description Default <code>df</code> <code>DataFrame</code> <p>The dataframe containing the text data in the column specified by the 'text_col' parameter</p> required <p>Returns:</p> Type Description <code>array</code> <p>The encoded texts</p> Source code in <code>pytorch_widedeep/preprocessing/hf_preprocessor.py</code> <pre><code>def transform(self, df: pd.DataFrame) -&gt; npt.NDArray[np.int64]:\n    \"\"\"\n    Encodes the text data in the input dataframe. This method simply\n    calls the `encode` method under the hood. Similar to the `fit` method,\n    this method is included for consistency with the rest of the library\n    in general and with the `BasePreprocessor` in particular.\n\n    Parameters\n    ----------\n    df: pd.DataFrame\n        The dataframe containing the text data in the column specified by\n        the 'text_col' parameter\n\n    Returns\n    -------\n    np.array\n        The encoded texts\n    \"\"\"\n    if self.text_col is None:\n        raise ValueError(\n            \"'text_col' is None. Please specify the column name containing the text data\"\n            \" if you want to use the 'fit' method\"\n        )\n\n    texts = self._read_texts(df, self.root_dir)\n\n    return self.encode(texts)\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.transform_sample","title":"transform_sample","text":"<pre><code>transform_sample(text)\n</code></pre> <p>Encodes a single text sample.</p> <p>Parameters:</p> Name Type Description Default <code>text</code> <code>str</code> <p>The text sample to be encoded</p> required <p>Returns:</p> Type Description <code>array</code> <p>The encoded text</p> Source code in <code>pytorch_widedeep/preprocessing/hf_preprocessor.py</code> <pre><code>def transform_sample(self, text: str) -&gt; npt.NDArray[np.int64]:\n    \"\"\"\n    Encodes a single text sample.\n\n    Parameters\n    ----------\n    text: str\n        The text sample to be encoded\n\n    Returns\n    -------\n    np.array\n        The encoded text\n    \"\"\"\n\n    if not self.is_fitted:\n        raise ValueError(\n            \"The `encode` (or `fit`) method must be called before calling `transform_sample`\"\n        )\n    return self.encode([text])[0]\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.fit_transform","title":"fit_transform","text":"<pre><code>fit_transform(df)\n</code></pre> <p>Encodes the text data in the input dataframe.</p> <p>Parameters:</p> Name Type Description Default <code>df</code> <code>DataFrame</code> <p>The dataframe containing the text data in the column specified by the 'text_col' parameter</p> required <p>Returns:</p> Type Description <code>array</code> <p>The encoded texts</p> Source code in <code>pytorch_widedeep/preprocessing/hf_preprocessor.py</code> <pre><code>def fit_transform(self, df: pd.DataFrame) -&gt; npt.NDArray[np.int64]:\n    \"\"\"\n    Encodes the text data in the input dataframe.\n\n    Parameters\n    ----------\n    df: pd.DataFrame\n        The dataframe containing the text data in the column specified by\n        the 'text_col' parameter\n\n    Returns\n    -------\n    np.array\n        The encoded texts\n    \"\"\"\n    return self.fit(df).transform(df)\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.inverse_transform","title":"inverse_transform","text":"<pre><code>inverse_transform(input_ids, skip_special_tokens)\n</code></pre> <p>Decodes a list of input_ids. The method simply calls the <code>decode</code> method under the hood.</p> <p>Parameters:</p> Name Type Description Default <code>input_ids</code> <code>NDArray[int64]</code> <p>The input_ids to be decoded</p> required <code>skip_special_tokens</code> <code>bool</code> <p>Whether to skip the special tokens or not</p> required <p>Returns:</p> Type Description <code>List[str]</code> <p>The decoded texts</p> Source code in <code>pytorch_widedeep/preprocessing/hf_preprocessor.py</code> <pre><code>def inverse_transform(\n    self, input_ids: npt.NDArray[np.int64], skip_special_tokens: bool\n) -&gt; List[str]:\n    \"\"\"\n    Decodes a list of input_ids. The method simply calls the `decode` method\n    under the hood.\n\n    Parameters\n    ----------\n    input_ids: npt.NDArray[np.int64]\n        The input_ids to be decoded\n    skip_special_tokens: bool\n        Whether to skip the special tokens or not\n\n    Returns\n    -------\n    List[str]\n        The decoded texts\n    \"\"\"\n    return self.decode(input_ids, skip_special_tokens)\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.image_preprocessor.ImagePreprocessor","title":"ImagePreprocessor","text":"<p>               Bases: <code>BasePreprocessor</code></p> <p>Preprocessor to prepare the <code>deepimage</code> input dataset.</p> <p>The Preprocessing consists simply on resizing according to their aspect ratio</p> <p>Parameters:</p> Name Type Description Default <code>img_col</code> <code>str</code> <p>name of the column with the images filenames</p> required <code>img_path</code> <code>str</code> <p>path to the dicrectory where the images are stored</p> required <code>width</code> <code>int</code> <p>width of the resulting processed image.</p> <code>224</code> <code>height</code> <code>int</code> <p>width of the resulting processed image.</p> <code>224</code> <code>verbose</code> <code>int</code> <p>Enable verbose output.</p> <code>1</code> <p>Attributes:</p> Name Type Description <code>aap</code> <code>AspectAwarePreprocessor</code> <p>an instance of <code>pytorch_widedeep.utils.image_utils.AspectAwarePreprocessor</code></p> <code>spp</code> <code>SimplePreprocessor</code> <p>an instance of <code>pytorch_widedeep.utils.image_utils.SimplePreprocessor</code></p> <code>normalise_metrics</code> <code>Dict</code> <p>Dict containing the normalisation metrics of the image dataset, i.e. mean and std for the R, G and B channels</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import pandas as pd\n&gt;&gt;&gt;\n&gt;&gt;&gt; from pytorch_widedeep.preprocessing import ImagePreprocessor\n&gt;&gt;&gt;\n&gt;&gt;&gt; path_to_image1 = 'tests/test_data_utils/images/galaxy1.png'\n&gt;&gt;&gt; path_to_image2 = 'tests/test_data_utils/images/galaxy2.png'\n&gt;&gt;&gt;\n&gt;&gt;&gt; df_train = pd.DataFrame({'images_column': [path_to_image1]})\n&gt;&gt;&gt; df_test = pd.DataFrame({'images_column': [path_to_image2]})\n&gt;&gt;&gt; img_preprocessor = ImagePreprocessor(img_col='images_column', img_path='.', verbose=0)\n&gt;&gt;&gt; resized_images = img_preprocessor.fit_transform(df_train)\n&gt;&gt;&gt; new_resized_images = img_preprocessor.transform(df_train)\n</code></pre> <p> NOTE: Normalising metrics will only be computed when the <code>fit_transform</code> method is run. Running <code>transform</code> only will not change the computed metrics and running <code>fit</code> only simply instantiates the resizing functions.</p> Source code in <code>pytorch_widedeep/preprocessing/image_preprocessor.py</code> <pre><code>class ImagePreprocessor(BasePreprocessor):\n    r\"\"\"Preprocessor to prepare the ``deepimage`` input dataset.\n\n    The Preprocessing consists simply on resizing according to their\n    aspect ratio\n\n    Parameters\n    ----------\n    img_col: str\n        name of the column with the images filenames\n    img_path: str\n        path to the dicrectory where the images are stored\n    width: int, default=224\n        width of the resulting processed image.\n    height: int, default=224\n        width of the resulting processed image.\n    verbose: int, default 1\n        Enable verbose output.\n\n    Attributes\n    ----------\n    aap: AspectAwarePreprocessor\n        an instance of `pytorch_widedeep.utils.image_utils.AspectAwarePreprocessor`\n    spp: SimplePreprocessor\n        an instance of `pytorch_widedeep.utils.image_utils.SimplePreprocessor`\n    normalise_metrics: Dict\n        Dict containing the normalisation metrics of the image dataset, i.e.\n        mean and std for the R, G and B channels\n\n    Examples\n    --------\n    &gt;&gt;&gt; import pandas as pd\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; from pytorch_widedeep.preprocessing import ImagePreprocessor\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; path_to_image1 = 'tests/test_data_utils/images/galaxy1.png'\n    &gt;&gt;&gt; path_to_image2 = 'tests/test_data_utils/images/galaxy2.png'\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; df_train = pd.DataFrame({'images_column': [path_to_image1]})\n    &gt;&gt;&gt; df_test = pd.DataFrame({'images_column': [path_to_image2]})\n    &gt;&gt;&gt; img_preprocessor = ImagePreprocessor(img_col='images_column', img_path='.', verbose=0)\n    &gt;&gt;&gt; resized_images = img_preprocessor.fit_transform(df_train)\n    &gt;&gt;&gt; new_resized_images = img_preprocessor.transform(df_train)\n\n    :information_source: **NOTE**:\n    Normalising metrics will only be computed when the ``fit_transform``\n    method is run. Running ``transform`` only will not change the computed\n    metrics and running ``fit`` only simply instantiates the resizing\n    functions.\n    \"\"\"\n\n    def __init__(\n        self,\n        img_col: str,\n        img_path: str,\n        width: int = 224,\n        height: int = 224,\n        verbose: int = 1,\n    ):\n        super(ImagePreprocessor, self).__init__()\n\n        self.img_col = img_col\n        self.img_path = img_path\n        self.width = width\n        self.height = height\n        self.verbose = verbose\n\n        self.aap = AspectAwarePreprocessor(self.width, self.height)\n        self.spp = SimplePreprocessor(self.width, self.height)\n\n        self.compute_normalising_computed = False\n\n    def fit(self, df: pd.DataFrame) -&gt; BasePreprocessor:\n        # simply include this method to comply with the BasePreprocessor and\n        # for consistency with the rest of the preprocessors\n        return self\n\n    def transform_sample(self, img: np.ndarray) -&gt; np.ndarray:\n        aspect_r = img.shape[0] / img.shape[1]\n        if aspect_r != 1.0:\n            resized_img = self.aap.preprocess(img)\n        else:\n            resized_img = self.spp.preprocess(img)\n        return resized_img\n\n    def transform(self, df: pd.DataFrame) -&gt; np.ndarray:\n        \"\"\"Resizes the images to the input height and width.\n\n\n        Parameters\n        ----------\n        df: pd.DataFrame\n            Input pandas dataframe with the `img_col`\n\n        Returns\n        -------\n        np.ndarray\n            Resized images to the input height and width\n        \"\"\"\n        image_list = df[self.img_col].tolist()\n        if self.verbose:\n            print(\"Reading Images from {}\".format(self.img_path))\n        imgs = [cv2.imread(\"/\".join([self.img_path, img])) for img in image_list]\n\n        # finding images with different height and width\n        aspect = [(im.shape[0], im.shape[1]) for im in imgs]\n        aspect_r = [a[0] / a[1] for a in aspect]\n        diff_idx = [i for i, r in enumerate(aspect_r) if r != 1.0]\n\n        if self.verbose:\n            print(\"Resizing\")\n        resized_imgs = []\n        for i, img in tqdm(enumerate(imgs), total=len(imgs), disable=self.verbose != 1):\n            if i in diff_idx:\n                resized_imgs.append(self.aap.preprocess(img))\n            else:\n                # if aspect ratio is 1:1, no need for AspectAwarePreprocessor\n                resized_imgs.append(self.spp.preprocess(img))\n\n        if not self.compute_normalising_computed:\n            if self.verbose:\n                print(\"Computing normalisation metrics\")\n            # mean and std deviation will only be computed when the fit method\n            # is called\n            mean_R, mean_G, mean_B = [], [], []\n            std_R, std_G, std_B = [], [], []\n            for rsz_img in resized_imgs:\n                (mean_b, mean_g, mean_r), (std_b, std_g, std_r) = cv2.meanStdDev(\n                    rsz_img\n                )\n                mean_R.append(mean_r)\n                mean_G.append(mean_g)\n                mean_B.append(mean_b)\n                std_R.append(std_r)\n                std_G.append(std_g)\n                std_B.append(std_b)\n            self.normalise_metrics = dict(\n                mean={\n                    \"R\": np.mean(mean_R) / 255.0,\n                    \"G\": np.mean(mean_G) / 255.0,\n                    \"B\": np.mean(mean_B) / 255.0,\n                },\n                std={\n                    \"R\": np.mean(std_R) / 255.0,\n                    \"G\": np.mean(std_G) / 255.0,\n                    \"B\": np.mean(std_B) / 255.0,\n                },\n            )\n            self.compute_normalising_computed = True\n        return np.asarray(resized_imgs)\n\n    def fit_transform(self, df: pd.DataFrame) -&gt; np.ndarray:\n        \"\"\"Combines `fit` and `transform`\n\n        Parameters\n        ----------\n        df: pd.DataFrame\n            Input pandas dataframe\n\n        Returns\n        -------\n        np.ndarray\n            Resized images to the input height and width\n        \"\"\"\n        return self.fit(df).transform(df)\n\n    def inverse_transform(self, transformed_image):\n        raise NotImplementedError(\n            \"'inverse_transform' method is not implemented for 'ImagePreprocessor'\"\n        )\n\n    def __repr__(self) -&gt; str:\n        list_of_params: List[str] = []\n        list_of_params.append(\"img_col={img_col}\")\n        list_of_params.append(\"img_path={img_path}\")\n        list_of_params.append(\"width={width}\")\n        list_of_params.append(\"height={height}\")\n        list_of_params.append(\"verbose={verbose}\")\n        all_params = \", \".join(list_of_params)\n        return f\"ImagePreprocessor({all_params.format(**self.__dict__)})\"\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.image_preprocessor.ImagePreprocessor.transform","title":"transform","text":"<pre><code>transform(df)\n</code></pre> <p>Resizes the images to the input height and width.</p> <p>Parameters:</p> Name Type Description Default <code>df</code> <code>DataFrame</code> <p>Input pandas dataframe with the <code>img_col</code></p> required <p>Returns:</p> Type Description <code>ndarray</code> <p>Resized images to the input height and width</p> Source code in <code>pytorch_widedeep/preprocessing/image_preprocessor.py</code> <pre><code>def transform(self, df: pd.DataFrame) -&gt; np.ndarray:\n    \"\"\"Resizes the images to the input height and width.\n\n\n    Parameters\n    ----------\n    df: pd.DataFrame\n        Input pandas dataframe with the `img_col`\n\n    Returns\n    -------\n    np.ndarray\n        Resized images to the input height and width\n    \"\"\"\n    image_list = df[self.img_col].tolist()\n    if self.verbose:\n        print(\"Reading Images from {}\".format(self.img_path))\n    imgs = [cv2.imread(\"/\".join([self.img_path, img])) for img in image_list]\n\n    # finding images with different height and width\n    aspect = [(im.shape[0], im.shape[1]) for im in imgs]\n    aspect_r = [a[0] / a[1] for a in aspect]\n    diff_idx = [i for i, r in enumerate(aspect_r) if r != 1.0]\n\n    if self.verbose:\n        print(\"Resizing\")\n    resized_imgs = []\n    for i, img in tqdm(enumerate(imgs), total=len(imgs), disable=self.verbose != 1):\n        if i in diff_idx:\n            resized_imgs.append(self.aap.preprocess(img))\n        else:\n            # if aspect ratio is 1:1, no need for AspectAwarePreprocessor\n            resized_imgs.append(self.spp.preprocess(img))\n\n    if not self.compute_normalising_computed:\n        if self.verbose:\n            print(\"Computing normalisation metrics\")\n        # mean and std deviation will only be computed when the fit method\n        # is called\n        mean_R, mean_G, mean_B = [], [], []\n        std_R, std_G, std_B = [], [], []\n        for rsz_img in resized_imgs:\n            (mean_b, mean_g, mean_r), (std_b, std_g, std_r) = cv2.meanStdDev(\n                rsz_img\n            )\n            mean_R.append(mean_r)\n            mean_G.append(mean_g)\n            mean_B.append(mean_b)\n            std_R.append(std_r)\n            std_G.append(std_g)\n            std_B.append(std_b)\n        self.normalise_metrics = dict(\n            mean={\n                \"R\": np.mean(mean_R) / 255.0,\n                \"G\": np.mean(mean_G) / 255.0,\n                \"B\": np.mean(mean_B) / 255.0,\n            },\n            std={\n                \"R\": np.mean(std_R) / 255.0,\n                \"G\": np.mean(std_G) / 255.0,\n                \"B\": np.mean(std_B) / 255.0,\n            },\n        )\n        self.compute_normalising_computed = True\n    return np.asarray(resized_imgs)\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.image_preprocessor.ImagePreprocessor.fit_transform","title":"fit_transform","text":"<pre><code>fit_transform(df)\n</code></pre> <p>Combines <code>fit</code> and <code>transform</code></p> <p>Parameters:</p> Name Type Description Default <code>df</code> <code>DataFrame</code> <p>Input pandas dataframe</p> required <p>Returns:</p> Type Description <code>ndarray</code> <p>Resized images to the input height and width</p> Source code in <code>pytorch_widedeep/preprocessing/image_preprocessor.py</code> <pre><code>def fit_transform(self, df: pd.DataFrame) -&gt; np.ndarray:\n    \"\"\"Combines `fit` and `transform`\n\n    Parameters\n    ----------\n    df: pd.DataFrame\n        Input pandas dataframe\n\n    Returns\n    -------\n    np.ndarray\n        Resized images to the input height and width\n    \"\"\"\n    return self.fit(df).transform(df)\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.din_preprocessor.DINPreprocessor","title":"DINPreprocessor","text":"<p>               Bases: <code>BasePreprocessor</code></p> <p>Preprocessor for Deep Interest Network (DIN) models.</p> <p>This preprocessor handles the preparation of data for DIN models, including sequence building, label encoding, and handling of various types of input columns (categorical, continuous, and sequential).</p> <p>Parameters:</p> Name Type Description Default <code>user_id_col</code> <code>str</code> <p>Name of the column containing user IDs.</p> required <code>item_embed_col</code> <code>Union[str, Tuple[str, int]]</code> <p>Name of the column containing item IDs to be embedded, or a tuple of (column_name, embedding_dim). If embedding_dim is not provided, it will be automatically calculated using a rule of thumb specified by <code>embedding_rule</code>. Aliased as <code>item_id_col</code>.</p> required <code>target_col</code> <code>str</code> <p>Name of the column containing the target variable.</p> required <code>max_seq_length</code> <code>int</code> <p>Maximum length of sequences to be created.</p> required <code>action_col</code> <code>Optional[str]</code> <p>Name of the column containing user actions (if applicable), for example a rating, or purchased/not-purchased, etc). This 'action_col' can also be the same as the 'target_col'.</p> <code>None</code> <code>other_seq_embed_cols</code> <code>Optional[List[str] | List[Tuple[str, int]]]</code> <p>List of other columns to be treated as sequences. Each element in the list can be a string with the name of the column or a tuple with the name of the column and the embedding dimension.</p> <code>None</code> <code>cat_embed_cols</code> <code>Optional[Union[List[str], List[Tuple[str, int]]]]</code> <p>This and the following parameters are identical to those used in the TabPreprocessor. List containing the name of the categorical columns that will be represented by embeddings (e.g. ['education', 'relationship', ...]) or a Tuple with the name and the embedding dimension (e.g.: [ ('education',32), ('relationship',16), ...])</p> <code>None</code> <code>continuous_cols</code> <code>Optional[List[str]]</code> <p>List with the name of the continuous cols</p> <code>None</code> <code>quantization_setup</code> <code>Optional[Union[int, Dict[str, Union[int, List[float]]]]]</code> <p>Continuous columns can be turned into categorical via <code>pd.cut</code>. If <code>quantization_setup</code> is an <code>int</code>, all continuous columns will be quantized using this value as the number of bins. Alternatively, a dictionary where the keys are the column names to quantize and the values are the either integers indicating the number of bins or a list of scalars indicating the bin edges can also be used.</p> <code>None</code> <code>cols_to_scale</code> <code>Optional[Union[List[str], str]]</code> <p>List with the names of the columns that will be standarised via sklearn's <code>StandardScaler</code>. It can also be the string <code>'all'</code> in which case all the continuous cols will be scaled.</p> <code>None</code> <code>auto_embed_dim</code> <code>bool</code> <p>Boolean indicating whether the embedding dimensions will be automatically defined via rule of thumb. See <code>embedding_rule</code> below.</p> <code>True</code> <code>embedding_rule</code> <code>Literal['google', 'fastai_old', 'fastai_new']</code> <p>If <code>auto_embed_dim=True</code>, this is the choice of embedding rule of thumb. Choices are:</p> <ul> <li> <p>fastai_new: \\(min(600, round(1.6     imes n_{cat}^{0.56}))\\)</p> </li> <li> <p>fastai_old: \\(min(50, (n_{cat}//{2})+1)\\)</p> </li> <li> <p>google: \\(min(600, round(n_{cat}^{0.24}))\\)</p> </li> </ul> <code>'fastai_new'</code> <code>default_embed_dim</code> <code>int</code> <p>Dimension for the embeddings if the embedding dimension is not provided in the <code>cat_embed_cols</code> parameter and <code>auto_embed_dim</code> is set to <code>False</code>.</p> <code>16</code> <code>verbose</code> <code>int</code> <p>Verbosity level.</p> <code>1</code> <code>scale</code> <code>bool</code> <p> note: this arg will be removed in upcoming  releases. Please use <code>cols_to_scale</code> instead.  Bool indicating  whether or not to scale/standarise continuous cols. It is important  to emphasize that all the DL models for tabular data in the library  also include the possibility of normalising the input continuous  features via a <code>BatchNorm</code> or a <code>LayerNorm</code>.  Param alias:  <code>scale_cont_cols</code>.</p> <code>False</code> <code>already_standard</code> <code>Optional[List[str]]</code> <p> note: this arg will be removed in upcoming  releases. Please use <code>cols_to_scale</code> instead.  List with the  name of the continuous cols that do not need to be  scaled/standarised.</p> <code>None</code> <p>Other Parameters:</p> Name Type Description <code>**kwargs</code> <p><code>pd.cut</code> and <code>StandardScaler</code> related args</p> <p>Attributes:</p> Name Type Description <code>is_fitted</code> <code>bool</code> <p>Whether the preprocessor has been fitted.</p> <code>has_standard_tab_data</code> <code>bool</code> <p>Whether the data includes standard tabular data.</p> <code>tab_preprocessor</code> <code>TabPreprocessor</code> <p>Preprocessor for standard tabular data (if applicable).</p> <code>din_columns_idx</code> <code>Dict[str, int]</code> <p>Dictionary mapping column names to their indices in the processed data.</p> <code>item_le</code> <code>LabelEncoder</code> <p>Label encoder for item IDs.</p> <code>n_items</code> <code>int</code> <p>Number of unique items.</p> <code>user_behaviour_config</code> <code>Tuple[List[str], int, int]</code> <p>Configuration for user behavior sequences.</p> <code>action_le</code> <code>LabelEncoder</code> <p>Label encoder for action column (if applicable).</p> <code>n_actions</code> <code>int</code> <p>Number of unique actions (if applicable).</p> <code>action_seq_config</code> <code>Tuple[List[str], int]</code> <p>Configuration for action sequences (if applicable).</p> <code>other_seq_le</code> <code>LabelEncoder</code> <p>Label encoder for other sequence columns (if applicable).</p> <code>n_other_seq_cols</code> <code>Dict[str, int]</code> <p>Number of unique values in each other sequence column.</p> <code>other_seq_config</code> <code>List[Tuple[List[str], int, int]]</code> <p>Configuration for other sequence columns.</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import pandas as pd\n&gt;&gt;&gt; from pytorch_widedeep.preprocessing import DINPreprocessor\n&gt;&gt;&gt; data = {\n...    'user_id': [1, 1, 1, 2, 2, 2, 3, 3, 3],\n...    'item_id': [101, 102, 103, 101, 103, 104, 102, 103, 104],\n...    'timestamp': [1, 2, 3, 1, 2, 3, 1, 2, 3],\n...    'category': ['A', 'B', 'A', 'B', 'A', 'C', 'B', 'A', 'C'],\n...    'price': [10.5, 15.0, 12.0, 10.5, 12.0, 20.0, 15.0, 12.0, 20.0],\n...    'rating': [0, 1, 0, 1, 0, 1, 0, 1, 0]\n... }\n&gt;&gt;&gt; df = pd.DataFrame(data)\n&gt;&gt;&gt; din_preprocessor = DINPreprocessor(\n...     user_id_col='user_id',\n...     item_embed_col='item_id',\n...     target_col='rating',\n...     max_seq_length=2,\n...     action_col='rating',\n...     other_seq_embed_cols=['category'],\n...     cat_embed_cols=['user_id'],\n...     continuous_cols=['price'],\n...     cols_to_scale=['price']\n... )\n&gt;&gt;&gt; X, y = din_preprocessor.fit_transform(df)\n</code></pre> Source code in <code>pytorch_widedeep/preprocessing/din_preprocessor.py</code> <pre><code>class DINPreprocessor(BasePreprocessor):\n    \"\"\"\n    Preprocessor for Deep Interest Network (DIN) models.\n\n    This preprocessor handles the preparation of data for DIN models,\n    including sequence building, label encoding, and handling of various\n    types of input columns (categorical, continuous, and sequential).\n\n    Parameters\n    ----------\n    user_id_col : str\n        Name of the column containing user IDs.\n    item_embed_col : Union[str, Tuple[str, int]]\n        Name of the column containing item IDs to be embedded, or a tuple of\n        (column_name, embedding_dim). If embedding_dim is not provided, it\n        will be automatically calculated using a rule of thumb specified by\n        `embedding_rule`. Aliased as `item_id_col`.\n    target_col : str\n        Name of the column containing the target variable.\n    max_seq_length : int\n        Maximum length of sequences to be created.\n    action_col : Optional[str], default=None\n        Name of the column containing user actions (if applicable), for\n        example a rating, or purchased/not-purchased, etc). This 'action_col'\n        can also be the same as the 'target_col'.\n    other_seq_embed_cols : Optional[List[str] | List[Tuple[str, int]]], default=None\n        List of other columns to be treated as sequences. Each element in the\n        list can be a string with the name of the column or a tuple with the\n        name of the column and the embedding dimension.\n\n    cat_embed_cols : Optional[Union[List[str], List[Tuple[str, int]]]], default=None\n        This and the following parameters are identical to those used in the\n        TabPreprocessor.&lt;br&gt;\n        List containing the name of the categorical columns that will be\n        represented by embeddings (e.g. _['education', 'relationship', ...]_) or\n        a Tuple with the name and the embedding dimension (e.g.: _[\n        ('education',32), ('relationship',16), ...]_)\n    continuous_cols: List, default = None\n        List with the name of the continuous cols\n    quantization_setup: int or Dict, default = None\n        Continuous columns can be turned into categorical via `pd.cut`. If\n        `quantization_setup` is an `int`, all continuous columns will be\n        quantized using this value as the number of bins. Alternatively, a\n        dictionary where the keys are the column names to quantize and the\n        values are the either integers indicating the number of bins or a\n        list of scalars indicating the bin edges can also be used.\n    cols_to_scale: List or str, default = None,\n        List with the names of the columns that will be standarised via\n        sklearn's `StandardScaler`. It can also be the string `'all'` in\n        which case all the continuous cols will be scaled.\n    auto_embed_dim: bool, default = True\n        Boolean indicating whether the embedding dimensions will be\n        automatically defined via rule of thumb. See `embedding_rule`\n        below.\n    embedding_rule: str, default = 'fastai_new'\n        If `auto_embed_dim=True`, this is the choice of embedding rule of\n        thumb. Choices are:\n\n        - _fastai_new_: $min(600, round(1.6 \\times n_{cat}^{0.56}))$\n\n        - _fastai_old_: $min(50, (n_{cat}//{2})+1)$\n\n        - _google_: $min(600, round(n_{cat}^{0.24}))$\n    default_embed_dim: int, default=16\n        Dimension for the embeddings if the embedding dimension is not\n        provided in the `cat_embed_cols` parameter and `auto_embed_dim` is\n        set to `False`.\n    verbose : int, default=1\n        Verbosity level.\n    scale: bool, default = False\n        :information_source: **note**: this arg will be removed in upcoming\n         releases. Please use `cols_to_scale` instead. &lt;br/&gt; Bool indicating\n         whether or not to scale/standarise continuous cols. It is important\n         to emphasize that all the DL models for tabular data in the library\n         also include the possibility of normalising the input continuous\n         features via a `BatchNorm` or a `LayerNorm`. &lt;br/&gt; Param alias:\n         `scale_cont_cols`.\n    already_standard: List, default = None\n        :information_source: **note**: this arg will be removed in upcoming\n         releases. Please use `cols_to_scale` instead. &lt;br/&gt; List with the\n         name of the continuous cols that do not need to be\n         scaled/standarised.\n\n    Other Parameters\n    ----------------\n    **kwargs: dict\n        `pd.cut` and `StandardScaler` related args\n\n    Attributes\n    ----------\n    is_fitted : bool\n        Whether the preprocessor has been fitted.\n    has_standard_tab_data : bool\n        Whether the data includes standard tabular data.\n    tab_preprocessor : TabPreprocessor\n        Preprocessor for standard tabular data (if applicable).\n    din_columns_idx : Dict[str, int]\n        Dictionary mapping column names to their indices in the processed data.\n    item_le : LabelEncoder\n        Label encoder for item IDs.\n    n_items : int\n        Number of unique items.\n    user_behaviour_config : Tuple[List[str], int, int]\n        Configuration for user behavior sequences.\n    action_le : LabelEncoder\n        Label encoder for action column (if applicable).\n    n_actions : int\n        Number of unique actions (if applicable).\n    action_seq_config : Tuple[List[str], int]\n        Configuration for action sequences (if applicable).\n    other_seq_le : LabelEncoder\n        Label encoder for other sequence columns (if applicable).\n    n_other_seq_cols : Dict[str, int]\n        Number of unique values in each other sequence column.\n    other_seq_config : List[Tuple[List[str], int, int]]\n        Configuration for other sequence columns.\n\n    Examples\n    --------\n    &gt;&gt;&gt; import pandas as pd\n    &gt;&gt;&gt; from pytorch_widedeep.preprocessing import DINPreprocessor\n    &gt;&gt;&gt; data = {\n    ...    'user_id': [1, 1, 1, 2, 2, 2, 3, 3, 3],\n    ...    'item_id': [101, 102, 103, 101, 103, 104, 102, 103, 104],\n    ...    'timestamp': [1, 2, 3, 1, 2, 3, 1, 2, 3],\n    ...    'category': ['A', 'B', 'A', 'B', 'A', 'C', 'B', 'A', 'C'],\n    ...    'price': [10.5, 15.0, 12.0, 10.5, 12.0, 20.0, 15.0, 12.0, 20.0],\n    ...    'rating': [0, 1, 0, 1, 0, 1, 0, 1, 0]\n    ... }\n    &gt;&gt;&gt; df = pd.DataFrame(data)\n    &gt;&gt;&gt; din_preprocessor = DINPreprocessor(\n    ...     user_id_col='user_id',\n    ...     item_embed_col='item_id',\n    ...     target_col='rating',\n    ...     max_seq_length=2,\n    ...     action_col='rating',\n    ...     other_seq_embed_cols=['category'],\n    ...     cat_embed_cols=['user_id'],\n    ...     continuous_cols=['price'],\n    ...     cols_to_scale=['price']\n    ... )\n    &gt;&gt;&gt; X, y = din_preprocessor.fit_transform(df)\n    \"\"\"\n\n    @alias(\"item_embed_col\", [\"item_id_col\"])\n    def __init__(\n        self,\n        *,\n        user_id_col: str,\n        item_embed_col: Union[str, Tuple[str, int]],\n        target_col: str,\n        max_seq_length: int,\n        action_col: Optional[str] = None,\n        other_seq_embed_cols: Optional[Union[List[str], List[Tuple[str, int]]]] = None,\n        cat_embed_cols: Optional[Union[List[str], List[Tuple[str, int]]]] = None,\n        continuous_cols: Optional[List[str]] = None,\n        quantization_setup: Optional[\n            Union[int, Dict[str, Union[int, List[float]]]]\n        ] = None,\n        cols_to_scale: Optional[Union[List[str], str]] = None,\n        auto_embed_dim: bool = True,\n        embedding_rule: Literal[\"google\", \"fastai_old\", \"fastai_new\"] = \"fastai_new\",\n        default_embed_dim: int = 16,\n        verbose: int = 1,\n        scale: bool = False,\n        already_standard: Optional[List[str]] = None,\n        **kwargs,\n    ):\n        self.user_id_col = user_id_col\n        self.item_embed_col = item_embed_col\n        self.max_seq_length = max_seq_length\n        self.target_col = target_col if target_col is not None else \"target\"\n        self.action_col = action_col\n        self.other_seq_embed_cols = other_seq_embed_cols\n        self.cat_embed_cols = cat_embed_cols\n        self.continuous_cols = continuous_cols\n        self.quantization_setup = quantization_setup\n        self.cols_to_scale = cols_to_scale\n        self.auto_embed_dim = auto_embed_dim\n        self.embedding_rule = embedding_rule\n        self.default_embed_dim = default_embed_dim\n        self.verbose = verbose\n        self.scale = scale\n        self.already_standard = already_standard\n        self.kwargs = kwargs\n\n        self.has_standard_tab_data = bool(self.cat_embed_cols or self.continuous_cols)\n        if self.has_standard_tab_data:\n            self.tab_preprocessor = TabPreprocessor(\n                cat_embed_cols=self.cat_embed_cols,\n                continuous_cols=self.continuous_cols,\n                quantization_setup=self.quantization_setup,\n                cols_to_scale=self.cols_to_scale,\n                auto_embed_dim=self.auto_embed_dim,\n                embedding_rule=self.embedding_rule,\n                default_embed_dim=self.default_embed_dim,\n                verbose=self.verbose,\n                scale=self.scale,\n                already_standard=self.already_standard,\n                **self.kwargs,\n            )\n\n        self.is_fitted = False\n\n    def fit(self, df: pd.DataFrame) -&gt; \"DINPreprocessor\":\n        if self.has_standard_tab_data:\n            self.tab_preprocessor.fit(df)\n            self.din_columns_idx = {\n                col: i for i, col in enumerate(self.tab_preprocessor.column_idx.keys())\n            }\n        else:\n            self.din_columns_idx = {}\n\n        self._fit_label_encoders(df)\n        self.is_fitted = True\n        return self\n\n    def transform(self, df: pd.DataFrame) -&gt; Tuple[np.ndarray, np.ndarray]:\n        _df = self._pre_transform(df)\n        df_w_sequences = self._build_sequences(_df)\n        X_all = self._concatenate_features(df_w_sequences)\n        return X_all, np.array(df_w_sequences[self.target_col].tolist())\n\n    def fit_transform(self, df: pd.DataFrame) -&gt; Tuple[np.ndarray, np.ndarray]:\n        self.fit(df)\n        return self.transform(df)\n\n    def inverse_transform(self, X: np.ndarray) -&gt; pd.DataFrame:\n        raise NotImplementedError(\n            \"inverse_transform is not implemented for this preprocessor\"\n        )\n\n    def _fit_label_encoders(self, df: pd.DataFrame) -&gt; None:\n        self.item_le = LabelEncoder(columns_to_encode=[self._get_item_col()])\n        self.item_le.fit(df)\n        self.n_items = max(self.item_le.encoding_dict[self._get_item_col()].values())\n        user_behaviour_embed_size = self._get_embed_size(\n            self.item_embed_col, self.n_items\n        )\n        self.user_behaviour_config = (\n            [f\"item_{i+1}\" for i in range(self.max_seq_length)],\n            self.n_items,\n            user_behaviour_embed_size,\n        )\n\n        self._update_din_columns_idx(\"item\", self.max_seq_length)\n        self.din_columns_idx[\"target_item\"] = len(self.din_columns_idx)\n\n        if self.action_col:\n            self._fit_action_label_encoder(df)\n        if self.other_seq_embed_cols:\n            self._fit_other_seq_label_encoders(df)\n\n    def _fit_action_label_encoder(self, df: pd.DataFrame) -&gt; None:\n        self.action_le = LabelEncoder(columns_to_encode=[self.action_col])\n        self.action_le.fit(df)\n        self.n_actions = len(self.action_le.encoding_dict[self.action_col])\n        self.action_seq_config = (\n            [f\"action_{i+1}\" for i in range(self.max_seq_length)],\n            self.n_actions,\n        )\n        self._update_din_columns_idx(\"action\", self.max_seq_length)\n\n    def _other_seq_cols_float_warning(self, df: pd.DataFrame) -&gt; None:\n        other_seq_cols = [\n            col[0] if isinstance(col, tuple) else col\n            for col in self.other_seq_embed_cols\n        ]\n        for col in other_seq_cols:\n            if df[col].dtype == float:\n                warnings.warn(\n                    f\"{col} is a float column. It will be converted to integers. \"\n                    \"If this is not what you want, please convert it beforehand.\",\n                    UserWarning,\n                )\n\n    def _convert_other_seq_cols_to_int(self, df: pd.DataFrame) -&gt; None:\n        other_seq_cols = [\n            col[0] if isinstance(col, tuple) else col\n            for col in self.other_seq_embed_cols\n        ]\n        for col in other_seq_cols:\n            if df[col].dtype == float:\n                df[col] = df[col].astype(int)\n\n    def _fit_other_seq_label_encoders(self, df: pd.DataFrame) -&gt; None:\n        self._other_seq_cols_float_warning(df)\n        self._convert_other_seq_cols_to_int(df)\n        self.other_seq_le = LabelEncoder(\n            columns_to_encode=[\n                col[0] if isinstance(col, tuple) else col\n                for col in self.other_seq_embed_cols\n            ]\n        )\n        self.other_seq_le.fit(df)\n        other_seq_cols = [\n            col[0] if isinstance(col, tuple) else col\n            for col in self.other_seq_embed_cols\n        ]\n        self.n_other_seq_cols = {\n            col: max(self.other_seq_le.encoding_dict[col].values())\n            for col in other_seq_cols\n        }\n        other_seq_embed_sizes = {\n            col: self._get_embed_size(col, self.n_other_seq_cols[col])\n            for col in other_seq_cols\n        }\n        self.other_seq_config = [\n            (\n                self._get_seq_col_names(col),\n                self.n_other_seq_cols[col],\n                other_seq_embed_sizes[col],\n            )\n            for col in other_seq_cols\n        ]\n        self._update_din_columns_idx_for_other_seq(other_seq_cols)\n\n    def _get_item_col(self) -&gt; str:\n        return (\n            self.item_embed_col[0]\n            if isinstance(self.item_embed_col, tuple)\n            else self.item_embed_col\n        )\n\n    def _get_embed_size(self, col: Union[str, Tuple[str, int]], n_unique: int) -&gt; int:\n        return col[1] if isinstance(col, tuple) else embed_sz_rule(n_unique)\n\n    def _get_seq_col_names(self, col: str) -&gt; List[str]:\n        return [f\"{col}_{i+1}\" for i in range(self.max_seq_length)]\n\n    def _update_din_columns_idx(self, prefix: str, length: int) -&gt; None:\n        current_len = len(self.din_columns_idx)\n        self.din_columns_idx.update(\n            {f\"{prefix}_{i+1}\": i + current_len for i in range(length)}\n        )\n\n    def _update_din_columns_idx_for_other_seq(self, other_seq_cols: List[str]) -&gt; None:\n        current_len = len(self.din_columns_idx)\n        for col in other_seq_cols:\n            self.din_columns_idx.update(\n                {f\"{col}_{i+1}\": i + current_len for i in range(self.max_seq_length)}\n            )\n            self.din_columns_idx[f\"target_{col}\"] = len(self.din_columns_idx)\n            current_len = len(self.din_columns_idx)\n\n    def _pre_transform(self, df: pd.DataFrame) -&gt; pd.DataFrame:\n        check_is_fitted(self, attributes=[\"din_columns_idx\"])\n        df = self.item_le.transform(df)\n        if self.action_col:\n            df = self.action_le.transform(df)\n        if self.other_seq_embed_cols:\n            df = self.other_seq_le.transform(df)\n        return df\n\n    def _build_sequences(self, df: pd.DataFrame) -&gt; pd.DataFrame:\n        # df.columns.tolist() -&gt; to avoid annoying pandas warning\n        return (\n            df.groupby(self.user_id_col)[df.columns.tolist()]  # type: ignore[index]\n            .apply(self._group_sequences)\n            .reset_index(drop=True)\n        )\n\n    def _group_sequences(self, group: pd.DataFrame) -&gt; pd.DataFrame:\n        item_col = self._get_item_col()\n        items = group[item_col].tolist()\n        targets = group[self.target_col].tolist()\n        drop_cols = [item_col, self.target_col]\n\n        # sequences cannot be built with user, item pairs with only one\n        # interaction\n        if len(items) &lt;= 1:\n            return pd.DataFrame()\n\n        sequences = [\n            self._create_sequence(group, items, targets, i, drop_cols)\n            for i in range(max(1, len(items) - self.max_seq_length))\n        ]\n\n        seq_df = pd.DataFrame(sequences)\n        non_seq_cols = group.drop_duplicates([self.user_id_col]).drop(drop_cols, axis=1)\n        return pd.merge(seq_df, non_seq_cols, on=self.user_id_col)\n\n    def _create_sequence(\n        self,\n        group: pd.DataFrame,\n        items: List[int],\n        targets: List[int],\n        i: int,\n        drop_cols: List[str],\n    ) -&gt; Dict[str, Union[str, List[int]]]:\n        end_idx = min(i + self.max_seq_length, len(items) - 1)\n        item_sequences = items[i:end_idx]\n        target_item = items[end_idx]\n        target = targets[end_idx]\n\n        sequence = {\n            self.user_id_col: group.name,\n            \"items_sequence\": item_sequences,\n            \"target_item\": target_item,\n        }\n\n        if self.action_col:\n            sequence.update(\n                self._create_action_sequence(group, targets, i, target, drop_cols)\n            )\n        else:\n            sequence[self.target_col] = target\n            drop_cols.append(self.target_col)\n\n        if self.other_seq_embed_cols:\n            sequence.update(self._create_other_seq_sequence(group, i, drop_cols))\n\n        return sequence\n\n    def _create_action_sequence(\n        self,\n        group: pd.DataFrame,\n        targets: List[int],\n        i: int,\n        target: int,\n        drop_cols: List[str],\n    ) -&gt; Dict[str, Union[int, List[int]]]:\n        if self.action_col != self.target_col:\n            actions = group[self.action_col].tolist()\n            action_sequences = (\n                actions[i : i + self.max_seq_length]\n                if i + self.max_seq_length &lt; len(actions)\n                else actions[i:-1]\n            )\n            drop_cols.append(self.action_col)\n        else:\n            target -= 1  # the 'transform' method adds 1 as it saves 0 for padding\n            action_sequences = (\n                targets[i : i + self.max_seq_length]\n                if i + self.max_seq_length &lt; len(targets)\n                else targets[i:-1]\n            )\n\n        return {self.target_col: target, \"actions_sequence\": action_sequences}\n\n    def _create_other_seq_sequence(\n        self, group: pd.DataFrame, i: int, drop_cols: List[str]\n    ) -&gt; Dict[str, Union[int, List[int]]]:\n        other_seq_cols = [\n            col[0] if isinstance(col, tuple) else col\n            for col in self.other_seq_embed_cols\n        ]\n        drop_cols += other_seq_cols\n        other_seqs = {col: group[col].tolist() for col in other_seq_cols}\n\n        other_seqs_sequences: Dict[str, Union[int, List[int]]] = {}\n        for col in other_seq_cols:\n            other_seqs_sequences[col] = (\n                other_seqs[col][i : i + self.max_seq_length]\n                if i + self.max_seq_length &lt; len(other_seqs[col])\n                else other_seqs[col][i:-1]\n            )\n\n        sequence: Dict[str, Union[int, List[int]]] = {}\n        for col in other_seq_cols:\n            sequence[f\"{col}_sequence\"] = other_seqs_sequences[col]\n            sequence[f\"target_{col}\"] = (\n                other_seqs[col][i + self.max_seq_length]\n                if i + self.max_seq_length &lt; len(other_seqs[col])\n                else other_seqs[col][-1]\n            )\n\n        return sequence\n\n    def _concatenate_features(self, df_w_sequences: pd.DataFrame) -&gt; np.ndarray:\n        user_behaviour_seq = df_w_sequences[\"items_sequence\"].tolist()\n        X_user_behaviour = np.vstack(\n            [\n                pad_sequences(seq, self.max_seq_length, pad_idx=0)\n                for seq in user_behaviour_seq\n            ]\n        )\n        X_target_item = np.array(df_w_sequences[\"target_item\"].tolist()).reshape(-1, 1)\n        X_all = np.concatenate([X_user_behaviour, X_target_item], axis=1)\n\n        if self.has_standard_tab_data:\n            X_tab = self.tab_preprocessor.transform(df_w_sequences)\n            X_all = np.concatenate([X_tab, X_all], axis=1)\n\n        if self.action_col:\n            action_seq = df_w_sequences[\"actions_sequence\"].tolist()\n            X_actions = np.vstack(\n                [\n                    pad_sequences(seq, self.max_seq_length, pad_idx=0)\n                    for seq in action_seq\n                ]\n            )\n            X_all = np.concatenate([X_all, X_actions], axis=1)\n\n        if self.other_seq_embed_cols:\n            X_all = self._concatenate_other_seq_features(df_w_sequences, X_all)\n\n        assert len(self.din_columns_idx) == X_all.shape[1], (\n            f\"Something went wrong. The number of columns in the final array \"\n            f\"({X_all.shape[1]}) is different from the number of columns in \"\n            f\"self.din_columns_idx ({len(self.din_columns_idx)})\"\n        )\n\n        return X_all\n\n    def _concatenate_other_seq_features(\n        self, df_w_sequences: pd.DataFrame, X_all: np.ndarray\n    ) -&gt; np.ndarray:\n        other_seq_cols = [\n            col[0] if isinstance(col, tuple) else col\n            for col in self.other_seq_embed_cols\n        ]\n        other_seq = {\n            col: df_w_sequences[f\"{col}_sequence\"].tolist() for col in other_seq_cols\n        }\n        other_seq_target = {\n            col: df_w_sequences[f\"target_{col}\"].tolist() for col in other_seq_cols\n        }\n        X_other_seq_arrays = []\n\n        for col in other_seq_cols:\n            X_other_seq_arrays.append(\n                np.vstack(\n                    [\n                        pad_sequences(s, self.max_seq_length, pad_idx=0)\n                        for s in other_seq[col]\n                    ]\n                )\n            )\n            X_other_seq_arrays.append(np.array(other_seq_target[col]).reshape(-1, 1))\n\n        return np.concatenate([X_all] + X_other_seq_arrays, axis=1)\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#chunked-versions","title":"Chunked versions","text":"<p>Chunked versions of the preprocessors are also available. These are useful when the data is too big to fit in memory. See also the <code>load_from_folder</code> module in the library and the corresponding section here in the documentation.</p> <p>Note that there is not a <code>ChunkImagePreprocessor</code>. This is because the processing of the images will occur inside the <code>ImageFromFolder</code> class in the <code>load_from_folder</code> module.</p>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.wide_preprocessor.ChunkWidePreprocessor","title":"ChunkWidePreprocessor","text":"<p>               Bases: <code>WidePreprocessor</code></p> <p>Preprocessor to prepare the wide input dataset</p> <p>This Preprocessor prepares the data for the wide, linear component. This linear model is implemented via an Embedding layer that is connected to the output neuron. <code>ChunkWidePreprocessor</code> numerically encodes all the unique values of all categorical columns <code>wide_cols + crossed_cols</code>. See the Example below.</p> <p>Parameters:</p> Name Type Description Default <code>wide_cols</code> <code>List[str]</code> <p>List of strings with the name of the columns that will label encoded and passed through the <code>wide</code> component</p> required <code>crossed_cols</code> <code>Optional[List[Tuple[str, str]]]</code> <p>List of Tuples with the name of the columns that will be <code>'crossed'</code> and then label encoded. e.g. [('education', 'occupation'), ...]. For binary features, a cross-product transformation is 1 if and only if the constituent features are all 1, and 0 otherwise.</p> <code>None</code> <p>Attributes:</p> Name Type Description <code>wide_crossed_cols</code> <code>List</code> <p>List with the names of all columns that will be label encoded</p> <code>encoding_dict</code> <code>Dict</code> <p>Dictionary where the keys are the result of pasting <code>colname + '_' + column value</code> and the values are the corresponding mapped integer.</p> <code>inverse_encoding_dict</code> <code>Dict</code> <p>the inverse encoding dictionary</p> <code>wide_dim</code> <code>int</code> <p>Dimension of the wide model (i.e. dim of the linear layer)</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import pandas as pd\n&gt;&gt;&gt; from pytorch_widedeep.preprocessing import ChunkWidePreprocessor\n&gt;&gt;&gt; chunk = pd.DataFrame({'color': ['r', 'b', 'g'], 'size': ['s', 'n', 'l']})\n&gt;&gt;&gt; wide_cols = ['color']\n&gt;&gt;&gt; crossed_cols = [('color', 'size')]\n&gt;&gt;&gt; chunk_wide_preprocessor = ChunkWidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols,\n... n_chunks=1)\n&gt;&gt;&gt; X_wide = chunk_wide_preprocessor.fit_transform(chunk)\n</code></pre> Source code in <code>pytorch_widedeep/preprocessing/wide_preprocessor.py</code> <pre><code>class ChunkWidePreprocessor(WidePreprocessor):\n    r\"\"\"Preprocessor to prepare the wide input dataset\n\n    This Preprocessor prepares the data for the wide, linear component.\n    This linear model is implemented via an Embedding layer that is\n    connected to the output neuron. `ChunkWidePreprocessor` numerically\n    encodes all the unique values of all categorical columns `wide_cols +\n    crossed_cols`. See the Example below.\n\n    Parameters\n    ----------\n    wide_cols: List\n        List of strings with the name of the columns that will label\n        encoded and passed through the `wide` component\n    crossed_cols: List, default = None\n        List of Tuples with the name of the columns that will be `'crossed'`\n        and then label encoded. e.g. _[('education', 'occupation'), ...]_. For\n        binary features, a cross-product transformation is 1 if and only if\n        the constituent features are all 1, and 0 otherwise.\n\n    Attributes\n    ----------\n    wide_crossed_cols: List\n        List with the names of all columns that will be label encoded\n    encoding_dict: Dict\n        Dictionary where the keys are the result of pasting `colname + '_' +\n        column value` and the values are the corresponding mapped integer.\n    inverse_encoding_dict: Dict\n        the inverse encoding dictionary\n    wide_dim: int\n        Dimension of the wide model (i.e. dim of the linear layer)\n\n    Examples\n    --------\n    &gt;&gt;&gt; import pandas as pd\n    &gt;&gt;&gt; from pytorch_widedeep.preprocessing import ChunkWidePreprocessor\n    &gt;&gt;&gt; chunk = pd.DataFrame({'color': ['r', 'b', 'g'], 'size': ['s', 'n', 'l']})\n    &gt;&gt;&gt; wide_cols = ['color']\n    &gt;&gt;&gt; crossed_cols = [('color', 'size')]\n    &gt;&gt;&gt; chunk_wide_preprocessor = ChunkWidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols,\n    ... n_chunks=1)\n    &gt;&gt;&gt; X_wide = chunk_wide_preprocessor.fit_transform(chunk)\n    \"\"\"\n\n    def __init__(\n        self,\n        wide_cols: List[str],\n        n_chunks: int,\n        crossed_cols: Optional[List[Tuple[str, str]]] = None,\n    ):\n        super(ChunkWidePreprocessor, self).__init__(wide_cols, crossed_cols)\n\n        self.n_chunks = n_chunks\n\n        self.chunk_counter = 0\n\n        self.is_fitted = False\n\n    def partial_fit(self, chunk: pd.DataFrame) -&gt; \"ChunkWidePreprocessor\":\n        r\"\"\"Fits the Preprocessor and creates required attributes\n\n        Parameters\n        ----------\n        chunk: pd.DataFrame\n            Input pandas dataframe\n\n        Returns\n        -------\n        ChunkWidePreprocessor\n            `ChunkWidePreprocessor` fitted object\n        \"\"\"\n        df_wide = self._prepare_wide(chunk)\n        self.wide_crossed_cols = df_wide.columns.tolist()\n\n        if self.chunk_counter == 0:\n            self.glob_feature_set = set(\n                self._make_global_feature_list(df_wide[self.wide_crossed_cols])\n            )\n        else:\n            self.glob_feature_set.update(\n                self._make_global_feature_list(df_wide[self.wide_crossed_cols])\n            )\n\n        self.chunk_counter += 1\n\n        if self.chunk_counter == self.n_chunks:\n            self.encoding_dict = {v: i + 1 for i, v in enumerate(self.glob_feature_set)}\n            self.wide_dim = len(self.encoding_dict)\n            self.inverse_encoding_dict = {k: v for v, k in self.encoding_dict.items()}\n            self.inverse_encoding_dict[0] = \"unseen\"\n\n            self.is_fitted = True\n\n        return self\n\n    def fit(self, df: pd.DataFrame) -&gt; \"ChunkWidePreprocessor\":\n        \"\"\"\n        Runs `partial_fit`. This is just to override the fit method in the base\n        class. This class is not designed or thought to run fit\n        \"\"\"\n        return self.partial_fit(df)\n\n    def __repr__(self) -&gt; str:\n        list_of_params: List[str] = [\"wide_cols={wide_cols}\"]\n        list_of_params.append(\"n_chunks={n_chunks}\")\n        if self.crossed_cols is not None:\n            list_of_params.append(\"crossed_cols={crossed_cols}\")\n        all_params = \", \".join(list_of_params)\n        return f\"WidePreprocessor({all_params.format(**self.__dict__)})\"\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.wide_preprocessor.ChunkWidePreprocessor.partial_fit","title":"partial_fit","text":"<pre><code>partial_fit(chunk)\n</code></pre> <p>Fits the Preprocessor and creates required attributes</p> <p>Parameters:</p> Name Type Description Default <code>chunk</code> <code>DataFrame</code> <p>Input pandas dataframe</p> required <p>Returns:</p> Type Description <code>ChunkWidePreprocessor</code> <p><code>ChunkWidePreprocessor</code> fitted object</p> Source code in <code>pytorch_widedeep/preprocessing/wide_preprocessor.py</code> <pre><code>def partial_fit(self, chunk: pd.DataFrame) -&gt; \"ChunkWidePreprocessor\":\n    r\"\"\"Fits the Preprocessor and creates required attributes\n\n    Parameters\n    ----------\n    chunk: pd.DataFrame\n        Input pandas dataframe\n\n    Returns\n    -------\n    ChunkWidePreprocessor\n        `ChunkWidePreprocessor` fitted object\n    \"\"\"\n    df_wide = self._prepare_wide(chunk)\n    self.wide_crossed_cols = df_wide.columns.tolist()\n\n    if self.chunk_counter == 0:\n        self.glob_feature_set = set(\n            self._make_global_feature_list(df_wide[self.wide_crossed_cols])\n        )\n    else:\n        self.glob_feature_set.update(\n            self._make_global_feature_list(df_wide[self.wide_crossed_cols])\n        )\n\n    self.chunk_counter += 1\n\n    if self.chunk_counter == self.n_chunks:\n        self.encoding_dict = {v: i + 1 for i, v in enumerate(self.glob_feature_set)}\n        self.wide_dim = len(self.encoding_dict)\n        self.inverse_encoding_dict = {k: v for v, k in self.encoding_dict.items()}\n        self.inverse_encoding_dict[0] = \"unseen\"\n\n        self.is_fitted = True\n\n    return self\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.wide_preprocessor.ChunkWidePreprocessor.fit","title":"fit","text":"<pre><code>fit(df)\n</code></pre> <p>Runs <code>partial_fit</code>. This is just to override the fit method in the base class. This class is not designed or thought to run fit</p> Source code in <code>pytorch_widedeep/preprocessing/wide_preprocessor.py</code> <pre><code>def fit(self, df: pd.DataFrame) -&gt; \"ChunkWidePreprocessor\":\n    \"\"\"\n    Runs `partial_fit`. This is just to override the fit method in the base\n    class. This class is not designed or thought to run fit\n    \"\"\"\n    return self.partial_fit(df)\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.tab_preprocessor.ChunkTabPreprocessor","title":"ChunkTabPreprocessor","text":"<p>               Bases: <code>TabPreprocessor</code></p> <p>Preprocessor to prepare the <code>deeptabular</code> component input dataset</p> <p>Parameters:</p> Name Type Description Default <code>n_chunks</code> <code>int</code> <p>Number of chunks that the tabular dataset is divided by.</p> required <code>cat_embed_cols</code> <code>Optional[Union[List[str], List[Tuple[str, int]]]]</code> <p>List containing the name of the categorical columns that will be represented by embeddings (e.g. ['education', 'relationship', ...]) or a Tuple with the name and the embedding dimension (e.g.: [ ('education',32), ('relationship',16), ...]).  Param alias: <code>embed_cols</code></p> <code>None</code> <code>continuous_cols</code> <code>Optional[List[str]]</code> <p>List with the name of the continuous cols</p> <code>None</code> <code>cols_and_bins</code> <code>Optional[Dict[str, List[float]]]</code> <p>Continuous columns can be turned into categorical via <code>pd.cut</code>. 'cols_and_bins' is dictionary where the keys are the column names to quantize and the values are a list of scalars indicating the bin edges.  Param alias: <code>quantization_setup</code></p> <code>None</code> <code>cols_to_scale</code> <code>Optional[Union[List[str], str]]</code> <p>List with the names of the columns that will be standarised via sklearn's <code>StandardScaler</code></p> <code>None</code> <code>default_embed_dim</code> <code>int</code> <p>Dimension for the embeddings if the embed_dim is not provided in the <code>cat_embed_cols</code> parameter and <code>auto_embed_dim</code> is set to <code>False</code>.</p> <code>16</code> <code>with_attention</code> <code>bool</code> <p>Boolean indicating whether the preprocessed data will be passed to an attention-based model (more precisely a model where all embeddings must have the same dimensions). If <code>True</code>, the param <code>cat_embed_cols</code> must just be a list containing just the categorical column names: e.g. ['education', 'relationship', ...]. This is because they will all be  encoded using embeddings of the same dim, which will be specified  later when the model is defined.  Param alias:  <code>for_transformer</code></p> <code>False</code> <code>with_cls_token</code> <code>bool</code> <p>Boolean indicating if a <code>'[CLS]'</code> token will be added to the dataset when using attention-based models. The final hidden state corresponding to this token is used as the aggregated representation for classification and regression tasks. If not, the categorical (and continuous embeddings if present) will be concatenated before being passed to the final MLP (if present).</p> <code>False</code> <code>shared_embed</code> <code>bool</code> <p>Boolean indicating if the embeddings will be \"shared\" when using attention-based models. The idea behind <code>shared_embed</code> is described in the Appendix A in the TabTransformer paper: 'The goal of having column embedding is to enable the model to distinguish the classes in one column from those in the other columns'. In other words, the idea is to let the model learn which column is embedded at the time. See: <code>pytorch_widedeep.models.transformers._layers.SharedEmbeddings</code>.</p> <code>False</code> <code>verbose</code> <code>int</code> <code>1</code> <code>scale</code> <code>bool</code> <p> note: this arg will be removed in upcoming  releases. Please use <code>cols_to_scale</code> instead.  Bool indicating  whether or not to scale/standarise continuous cols. It is important  to emphasize that all the DL models for tabular data in the library  also include the possibility of normalising the input continuous  features via a <code>BatchNorm</code> or a <code>LayerNorm</code>.  Param alias:  <code>scale_cont_cols</code>.</p> <code>False</code> <code>already_standard</code> <code>Optional[List[str]]</code> <p> note: this arg will be removed in upcoming  releases. Please use <code>cols_to_scale</code> instead.  List with the  name of the continuous cols that do not need to be  scaled/standarised.</p> <code>None</code> <p>Other Parameters:</p> Name Type Description <code>**kwargs</code> <p><code>pd.cut</code> and <code>StandardScaler</code> related args</p> <p>Attributes:</p> Name Type Description <code>cat_cols</code> <code>List[str]</code> <p>List containing the names of the categorical columns after processing.</p> <code>cols_and_bins</code> <code>Optional[Dict[str, Union[int, List[float]]]]</code> <p>Dictionary containing the quantization setup after processing if quantization is requested. This is derived from the quantization_setup parameter.</p> <code>quant_args</code> <code>Dict</code> <p>Dictionary containing arguments passed to pandas.cut() function for quantization.</p> <code>scale_args</code> <code>Dict</code> <p>Dictionary containing arguments passed to StandardScaler.</p> <code>label_encoder</code> <code>LabelEncoder</code> <p>Instance of LabelEncoder used to encode categorical variables. See <code>pytorch_widedeep.utils.dense_utils.LabelEncoder</code>.</p> <code>cat_embed_input</code> <code>List</code> <p>List of tuples with the column name, number of individual values for that column and, if <code>with_attention</code> is set to <code>False</code>, the corresponding embeddings dim, e.g. [('education', 16, 10), ('relationship', 6, 8), ...].</p> <code>standardize_cols</code> <code>List</code> <p>List of the columns that will be standardized.</p> <code>scaler</code> <code>StandardScaler</code> <p>An instance of <code>sklearn.preprocessing.StandardScaler</code> if standardization is requested.</p> <code>column_idx</code> <code>Dict</code> <p>Dictionary where keys are column names and values are column indexes. This is necessary to slice tensors.</p> <code>quantizer</code> <code>Quantizer</code> <p>An instance of <code>Quantizer</code> if quantization is requested.</p> <code>is_fitted</code> <code>bool</code> <p>Boolean indicating if the preprocessor has been fitted.</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import pandas as pd\n&gt;&gt;&gt; import numpy as np\n&gt;&gt;&gt; from pytorch_widedeep.preprocessing import ChunkTabPreprocessor\n&gt;&gt;&gt; np.random.seed(42)\n&gt;&gt;&gt; chunk_df = pd.DataFrame({'cat_col': np.random.choice(['A', 'B', 'C'], size=8),\n... 'cont_col': np.random.uniform(1, 100, size=8)})\n&gt;&gt;&gt; cat_embed_cols = [('cat_col',4)]\n&gt;&gt;&gt; cont_cols = ['cont_col']\n&gt;&gt;&gt; tab_preprocessor = ChunkTabPreprocessor(\n... n_chunks=1, cat_embed_cols=cat_embed_cols, continuous_cols=cont_cols\n... )\n&gt;&gt;&gt; X_tab = tab_preprocessor.fit_transform(chunk_df)\n&gt;&gt;&gt; tab_preprocessor.cat_embed_cols\n[('cat_col', 4)]\n&gt;&gt;&gt; tab_preprocessor.column_idx\n{'cat_col': 0, 'cont_col': 1}\n</code></pre> Source code in <code>pytorch_widedeep/preprocessing/tab_preprocessor.py</code> <pre><code>class ChunkTabPreprocessor(TabPreprocessor):\n    r\"\"\"Preprocessor to prepare the `deeptabular` component input dataset\n\n    Parameters\n    ----------\n    n_chunks: int\n        Number of chunks that the tabular dataset is divided by.\n    cat_embed_cols: List, default = None\n        List containing the name of the categorical columns that will be\n        represented by embeddings (e.g. _['education', 'relationship', ...]_) or\n        a Tuple with the name and the embedding dimension (e.g.: _[\n        ('education',32), ('relationship',16), ...]_). &lt;br/&gt; Param alias:\n        `embed_cols`\n    continuous_cols: List, default = None\n        List with the name of the continuous cols\n    cols_and_bins: Dict, default = None\n        Continuous columns can be turned into categorical via\n        `pd.cut`. 'cols_and_bins' is dictionary where the keys are the column\n        names to quantize and the values are a list of scalars indicating the\n        bin edges. &lt;br/&gt; Param alias: `quantization_setup`\n    cols_to_scale: List, default = None,\n        List with the names of the columns that will be standarised via\n        sklearn's `StandardScaler`\n    default_embed_dim: int, default=16\n        Dimension for the embeddings if the embed_dim is not provided in the\n        `cat_embed_cols` parameter and `auto_embed_dim` is set to\n        `False`.\n    with_attention: bool, default = False\n        Boolean indicating whether the preprocessed data will be passed to an\n        attention-based model (more precisely a model where all embeddings\n        must have the same dimensions). If `True`, the param `cat_embed_cols`\n        must just be a list containing just the categorical column names:\n        e.g.\n        _['education', 'relationship', ...]_. This is because they will all be\n         encoded using embeddings of the same dim, which will be specified\n         later when the model is defined. &lt;br/&gt; Param alias:\n         `for_transformer`\n    with_cls_token: bool, default = False\n        Boolean indicating if a `'[CLS]'` token will be added to the dataset\n        when using attention-based models. The final hidden state\n        corresponding to this token is used as the aggregated representation\n        for classification and regression tasks. If not, the categorical\n        (and continuous embeddings if present) will be concatenated before\n        being passed to the final MLP (if present).\n    shared_embed: bool, default = False\n        Boolean indicating if the embeddings will be \"shared\" when using\n        attention-based models. The idea behind `shared_embed` is\n        described in the Appendix A in the [TabTransformer paper](https://arxiv.org/abs/2012.06678):\n        _'The goal of having column embedding is to enable the model to\n        distinguish the classes in one column from those in the other\n        columns'_. In other words, the idea is to let the model learn which\n        column is embedded at the time. See: `pytorch_widedeep.models.transformers._layers.SharedEmbeddings`.\n    verbose: int, default = 1\n    scale: bool, default = False\n        :information_source: **note**: this arg will be removed in upcoming\n         releases. Please use `cols_to_scale` instead. &lt;br/&gt; Bool indicating\n         whether or not to scale/standarise continuous cols. It is important\n         to emphasize that all the DL models for tabular data in the library\n         also include the possibility of normalising the input continuous\n         features via a `BatchNorm` or a `LayerNorm`. &lt;br/&gt; Param alias:\n         `scale_cont_cols`.\n    already_standard: List, default = None\n        :information_source: **note**: this arg will be removed in upcoming\n         releases. Please use `cols_to_scale` instead. &lt;br/&gt; List with the\n         name of the continuous cols that do not need to be\n         scaled/standarised.\n\n    Other Parameters\n    ----------------\n    **kwargs: dict\n        `pd.cut` and `StandardScaler` related args\n\n    Attributes\n    ----------\n    cat_cols: List[str]\n        List containing the names of the categorical columns after processing.\n    cols_and_bins: Optional[Dict[str, Union[int, List[float]]]]\n        Dictionary containing the quantization setup after processing if\n        quantization is requested. This is derived from the\n        quantization_setup parameter.\n    quant_args: Dict\n        Dictionary containing arguments passed to pandas.cut() function for quantization.\n    scale_args: Dict\n        Dictionary containing arguments passed to StandardScaler.\n    label_encoder: LabelEncoder\n        Instance of LabelEncoder used to encode categorical variables.\n        See `pytorch_widedeep.utils.dense_utils.LabelEncoder`.\n    cat_embed_input: List\n        List of tuples with the column name, number of individual values for\n        that column and, if `with_attention` is set to `False`, the\n        corresponding embeddings dim, e.g. _[('education', 16, 10),\n        ('relationship', 6, 8), ...]_.\n    standardize_cols: List\n        List of the columns that will be standardized.\n    scaler: StandardScaler\n        An instance of `sklearn.preprocessing.StandardScaler` if standardization\n        is requested.\n    column_idx: Dict\n        Dictionary where keys are column names and values are column indexes.\n        This is necessary to slice tensors.\n    quantizer: Quantizer\n        An instance of `Quantizer` if quantization is requested.\n    is_fitted: bool\n        Boolean indicating if the preprocessor has been fitted.\n\n    Examples\n    --------\n    &gt;&gt;&gt; import pandas as pd\n    &gt;&gt;&gt; import numpy as np\n    &gt;&gt;&gt; from pytorch_widedeep.preprocessing import ChunkTabPreprocessor\n    &gt;&gt;&gt; np.random.seed(42)\n    &gt;&gt;&gt; chunk_df = pd.DataFrame({'cat_col': np.random.choice(['A', 'B', 'C'], size=8),\n    ... 'cont_col': np.random.uniform(1, 100, size=8)})\n    &gt;&gt;&gt; cat_embed_cols = [('cat_col',4)]\n    &gt;&gt;&gt; cont_cols = ['cont_col']\n    &gt;&gt;&gt; tab_preprocessor = ChunkTabPreprocessor(\n    ... n_chunks=1, cat_embed_cols=cat_embed_cols, continuous_cols=cont_cols\n    ... )\n    &gt;&gt;&gt; X_tab = tab_preprocessor.fit_transform(chunk_df)\n    &gt;&gt;&gt; tab_preprocessor.cat_embed_cols\n    [('cat_col', 4)]\n    &gt;&gt;&gt; tab_preprocessor.column_idx\n    {'cat_col': 0, 'cont_col': 1}\n    \"\"\"\n\n    @alias(\"with_attention\", [\"for_transformer\"])\n    @alias(\"cat_embed_cols\", [\"embed_cols\"])\n    @alias(\"scale\", [\"scale_cont_cols\"])\n    @alias(\"cols_and_bins\", [\"quantization_setup\"])\n    def __init__(\n        self,\n        n_chunks: int,\n        cat_embed_cols: Optional[Union[List[str], List[Tuple[str, int]]]] = None,\n        continuous_cols: Optional[List[str]] = None,\n        cols_and_bins: Optional[Dict[str, List[float]]] = None,\n        cols_to_scale: Optional[Union[List[str], str]] = None,\n        default_embed_dim: int = 16,\n        with_attention: bool = False,\n        with_cls_token: bool = False,\n        shared_embed: bool = False,\n        verbose: int = 1,\n        *,\n        scale: bool = False,\n        already_standard: Optional[List[str]] = None,\n        **kwargs,\n    ):\n        super(ChunkTabPreprocessor, self).__init__(\n            cat_embed_cols=cat_embed_cols,\n            continuous_cols=continuous_cols,\n            quantization_setup=None,\n            cols_to_scale=cols_to_scale,\n            auto_embed_dim=False,\n            embedding_rule=\"google\",  # does not matter, irrelevant\n            default_embed_dim=default_embed_dim,\n            with_attention=with_attention,\n            with_cls_token=with_cls_token,\n            shared_embed=shared_embed,\n            verbose=verbose,\n            scale=scale,\n            already_standard=already_standard,\n            **kwargs,\n        )\n\n        self.n_chunks = n_chunks\n        self.chunk_counter = 0\n\n        self.cols_and_bins = cols_and_bins  # type: ignore[assignment]\n        if self.cols_and_bins is not None:\n            self.quantizer = Quantizer(self.cols_and_bins, **self.quant_args)\n\n        self.embed_prepared = False\n        self.continuous_prepared = False\n\n    def partial_fit(self, df: pd.DataFrame) -&gt; \"ChunkTabPreprocessor\":  # noqa: C901\n        # df here, and throughout the class, is a chunk of the original df\n        self.chunk_counter += 1\n\n        df_adj = self._insert_cls_token(df) if self.with_cls_token else df.copy()\n\n        self.column_idx: Dict[str, int] = {}\n\n        # Categorical embeddings logic\n        if self.cat_embed_cols is not None:\n            if not self.embed_prepared:\n                df_cat, self.cat_embed_dim = self._prepare_categorical(df_adj)\n                self.label_encoder = LabelEncoder(\n                    columns_to_encode=df_cat.columns.tolist(),\n                    shared_embed=self.shared_embed,\n                    with_attention=self.with_attention,\n                )\n                self.label_encoder.partial_fit(df_cat)\n            else:\n                df_cat = df_adj[self.cat_cols]\n                self.label_encoder.partial_fit(df_cat)\n\n            self.column_idx.update({k: v for v, k in enumerate(df_cat.columns)})\n\n        # Continuous columns logic\n        if self.continuous_cols is not None:\n            if not self.continuous_prepared:\n                df_cont, self.cont_embed_dim = self._prepare_continuous(df_adj)\n            else:\n                df_cont = df[self.continuous_cols]\n\n            self.column_idx.update(\n                {k: v + len(self.column_idx) for v, k in enumerate(df_cont.columns)}\n            )\n\n            if self.standardize_cols is not None:\n                self.scaler.partial_fit(df_cont[self.standardize_cols].values)\n\n        if self.chunk_counter == self.n_chunks:\n            if self.cat_embed_cols is not None or self.cols_and_bins is not None:\n                self.cat_embed_input: List[\n                    Union[Tuple[str, int], Tuple[str, int, int]]\n                ] = []\n\n            if self.cat_embed_cols is not None:\n                for k, v in self.label_encoder.encoding_dict.items():\n                    if self.with_attention:\n                        self.cat_embed_input.append((k, len(v)))\n                    else:\n                        self.cat_embed_input.append((k, len(v), self.cat_embed_dim[k]))\n\n            if self.cols_and_bins is not None:\n                assert self.cont_embed_dim is not None  # just to make mypy happy\n                if self.with_attention:\n                    for col, n_cat, _ in self.cont_embed_dim:\n                        self.cat_embed_input.append((col, n_cat))\n                else:\n                    self.cat_embed_input.extend(self.cont_embed_dim)\n\n            self.is_fitted = True\n\n        return self\n\n    def fit(self, df: pd.DataFrame) -&gt; \"ChunkTabPreprocessor\":\n        # just to override the fit method in the base class. This class is not\n        # designed or thought to run fit\n        return self.partial_fit(df)\n\n    def _prepare_categorical(\n        self, df: pd.DataFrame\n    ) -&gt; Tuple[pd.DataFrame, Dict[str, int]]:\n        # When dealing with chunks we will NOT support the option of\n        # automatically define embeddings as this implies going through the\n        # entire dataset\n        if isinstance(self.cat_embed_cols[0], tuple):\n            self.cat_cols: List[str] = [emb[0] for emb in self.cat_embed_cols]\n            cat_embed_dim: Dict[str, int] = dict(self.cat_embed_cols)  # type: ignore\n        else:\n            self.cat_cols = self.cat_embed_cols  # type: ignore[assignment]\n            cat_embed_dim = {\n                e: self.default_embed_dim for e in self.cat_embed_cols  # type: ignore[misc]\n            }  # type: ignore\n\n        self.embed_prepared = True\n\n        return df[self.cat_cols], cat_embed_dim\n\n    def _prepare_continuous(\n        self, df: pd.DataFrame\n    ) -&gt; Tuple[pd.DataFrame, Optional[List[Tuple[str, int, int]]]]:\n        # Standardization logic\n        if self.cols_to_scale is not None:\n            self.standardize_cols = (\n                self.cols_to_scale\n                if self.cols_to_scale != \"all\"\n                else self.continuous_cols\n            )\n        elif self.scale:\n            if self.already_standard is not None:\n                self.standardize_cols = [\n                    c\n                    for c in self.continuous_cols  # type: ignore[misc]\n                    if c not in self.already_standard\n                ]\n            else:\n                self.standardize_cols = self.continuous_cols\n        else:\n            self.standardize_cols = None\n\n        if self.standardize_cols is not None:\n            self.scaler = StandardScaler(**self.scale_args)\n        elif self.verbose:\n            warnings.warn(\"Continuous columns will not be normalised\")\n\n        # Quantization logic\n        if self.cols_and_bins is not None:\n            # the quantized columns are then treated as categorical\n            quant_cont_embed_input: Optional[List[Tuple[str, int, int]]] = []\n            for col, val in self.cols_and_bins.items():\n                if isinstance(val, int):\n                    quant_cont_embed_input.append(\n                        (\n                            col,\n                            val,\n                            self.default_embed_dim,\n                        )\n                    )\n                else:\n                    quant_cont_embed_input.append(\n                        (\n                            col,\n                            len(val) - 1,\n                            self.default_embed_dim,\n                        )\n                    )\n        else:\n            quant_cont_embed_input = None\n\n        self.continuous_prepared = True\n\n        return df[self.continuous_cols], quant_cont_embed_input\n\n    def __repr__(self) -&gt; str:  # noqa: C901\n        list_of_params: List[str] = []\n        if self.n_chunks is not None:\n            list_of_params.append(\"n_chunks={n_chunks}\")\n        if self.cat_embed_cols is not None:\n            list_of_params.append(\"cat_embed_cols={cat_embed_cols}\")\n        if self.continuous_cols is not None:\n            list_of_params.append(\"continuous_cols={continuous_cols}\")\n        if self.cols_and_bins is not None:\n            list_of_params.append(\"cols_and_bins={cols_and_bins}\")\n        if self.cols_to_scale is not None:\n            list_of_params.append(\"cols_to_scale={cols_to_scale}\")\n        if self.default_embed_dim != 16:\n            list_of_params.append(\"default_embed_dim={default_embed_dim}\")\n        if self.with_attention:\n            list_of_params.append(\"with_attention={with_attention}\")\n        if self.with_cls_token:\n            list_of_params.append(\"with_cls_token={with_cls_token}\")\n        if self.shared_embed:\n            list_of_params.append(\"shared_embed={shared_embed}\")\n        if self.verbose != 1:\n            list_of_params.append(\"verbose={verbose}\")\n        if self.scale:\n            list_of_params.append(\"scale={scale}\")\n        if self.already_standard is not None:\n            list_of_params.append(\"already_standard={already_standard}\")\n        if len(self.quant_args) &gt; 0:\n            list_of_params.append(\n                \", \".join([f\"{k}\" + \"=\" + f\"{v}\" for k, v in self.quant_args.items()])\n            )\n        if len(self.scale_args) &gt; 0:\n            list_of_params.append(\n                \", \".join([f\"{k}\" + \"=\" + f\"{v}\" for k, v in self.scale_args.items()])\n            )\n        all_params = \", \".join(list_of_params)\n        return f\"ChunkTabPreprocessor({all_params.format(**self.__dict__)})\"\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.text_preprocessor.ChunkTextPreprocessor","title":"ChunkTextPreprocessor","text":"<p>               Bases: <code>TextPreprocessor</code></p> <p>Preprocessor to prepare the <code>deeptext</code> input dataset</p> <p>Parameters:</p> Name Type Description Default <code>text_col</code> <code>str</code> <p>column in the input dataframe containing either the texts or the filenames where the text documents are stored</p> required <code>n_chunks</code> <code>int</code> <p>Number of chunks that the text dataset is divided by.</p> required <code>root_dir</code> <code>Optional[str]</code> <p>If 'text_col' contains the filenames with the text documents, this is the path to the directory where those documents are stored.</p> <code>None</code> <code>max_vocab</code> <code>int</code> <p>Maximum number of tokens in the vocabulary</p> <code>30000</code> <code>min_freq</code> <code>int</code> <p>Minimum frequency for a token to be part of the vocabulary</p> <code>5</code> <code>maxlen</code> <code>int</code> <p>Maximum length of the tokenized sequences</p> <code>80</code> <code>pad_first</code> <code>bool</code> <p>Indicates whether the padding index will be added at the beginning or the end of the sequences</p> <code>True</code> <code>pad_idx</code> <code>int</code> <p>padding index. Fastai's Tokenizer leaves 0 for the 'unknown' token.</p> <code>1</code> <code>word_vectors_path</code> <code>Optional[str]</code> <p>Path to the pretrained word vectors</p> <code>None</code> <code>n_cpus</code> <code>Optional[int]</code> <p>number of CPUs to used during the tokenization process</p> <code>None</code> <code>verbose</code> <code>int</code> <p>Enable verbose output.</p> <code>1</code> <p>Attributes:</p> Name Type Description <code>vocab</code> <code>Vocab</code> <p>an instance of <code>pytorch_widedeep.utils.fastai_transforms.ChunkVocab</code></p> <code>embedding_matrix</code> <code>ndarray</code> <p>Array with the pretrained embeddings if <code>word_vectors_path</code> is not None</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import pandas as pd\n&gt;&gt;&gt; from pytorch_widedeep.preprocessing import ChunkTextPreprocessor\n&gt;&gt;&gt; chunk_df = pd.DataFrame({'text_column': [\"life is like a box of chocolates\",\n... \"You never know what you're gonna get\"]})\n&gt;&gt;&gt; chunk_text_preprocessor = ChunkTextPreprocessor(text_col='text_column', n_chunks=1,\n... max_vocab=25, min_freq=1, maxlen=10, verbose=0, n_cpus=1)\n&gt;&gt;&gt; processed_chunk = chunk_text_preprocessor.fit_transform(chunk_df)\n</code></pre> Source code in <code>pytorch_widedeep/preprocessing/text_preprocessor.py</code> <pre><code>class ChunkTextPreprocessor(TextPreprocessor):\n    r\"\"\"Preprocessor to prepare the ``deeptext`` input dataset\n\n    Parameters\n    ----------\n    text_col: str\n        column in the input dataframe containing either the texts or the\n        filenames where the text documents are stored\n    n_chunks: int\n        Number of chunks that the text dataset is divided by.\n    root_dir: str, Optional, default = None\n        If 'text_col' contains the filenames with the text documents, this is\n        the path to the directory where those documents are stored.\n    max_vocab: int, default=30000\n        Maximum number of tokens in the vocabulary\n    min_freq: int, default=5\n        Minimum frequency for a token to be part of the vocabulary\n    maxlen: int, default=80\n        Maximum length of the tokenized sequences\n    pad_first: bool,  default = True\n        Indicates whether the padding index will be added at the beginning or the\n        end of the sequences\n    pad_idx: int, default = 1\n        padding index. Fastai's Tokenizer leaves 0 for the 'unknown' token.\n    word_vectors_path: str, Optional\n        Path to the pretrained word vectors\n    n_cpus: int, Optional, default = None\n        number of CPUs to used during the tokenization process\n    verbose: int, default 1\n        Enable verbose output.\n\n    Attributes\n    ----------\n    vocab: Vocab\n        an instance of `pytorch_widedeep.utils.fastai_transforms.ChunkVocab`\n    embedding_matrix: np.ndarray\n        Array with the pretrained embeddings if `word_vectors_path` is not None\n\n    Examples\n    ---------\n    &gt;&gt;&gt; import pandas as pd\n    &gt;&gt;&gt; from pytorch_widedeep.preprocessing import ChunkTextPreprocessor\n    &gt;&gt;&gt; chunk_df = pd.DataFrame({'text_column': [\"life is like a box of chocolates\",\n    ... \"You never know what you're gonna get\"]})\n    &gt;&gt;&gt; chunk_text_preprocessor = ChunkTextPreprocessor(text_col='text_column', n_chunks=1,\n    ... max_vocab=25, min_freq=1, maxlen=10, verbose=0, n_cpus=1)\n    &gt;&gt;&gt; processed_chunk = chunk_text_preprocessor.fit_transform(chunk_df)\n    \"\"\"\n\n    def __init__(\n        self,\n        text_col: str,\n        n_chunks: int,\n        root_dir: Optional[str] = None,\n        max_vocab: int = 30000,\n        min_freq: int = 5,\n        maxlen: int = 80,\n        pad_first: bool = True,\n        pad_idx: int = 1,\n        already_processed: Optional[bool] = False,\n        word_vectors_path: Optional[str] = None,\n        n_cpus: Optional[int] = None,\n        verbose: int = 1,\n    ):\n        super(ChunkTextPreprocessor, self).__init__(\n            text_col=text_col,\n            max_vocab=max_vocab,\n            min_freq=min_freq,\n            maxlen=maxlen,\n            pad_first=pad_first,\n            pad_idx=pad_idx,\n            already_processed=already_processed,\n            word_vectors_path=word_vectors_path,\n            n_cpus=n_cpus,\n            verbose=verbose,\n        )\n\n        self.n_chunks = n_chunks\n        self.root_dir = root_dir\n\n        self.chunk_counter = 0\n\n        self.is_fitted = False\n\n    def partial_fit(self, df: pd.DataFrame) -&gt; \"ChunkTextPreprocessor\":\n        # df is a chunk of the original dataframe\n        self.chunk_counter += 1\n\n        texts = self._read_texts(df, self.root_dir)\n\n        tokens = get_texts(texts, self.already_processed, self.n_cpus)\n\n        if not hasattr(self, \"vocab\"):\n            self.vocab = ChunkVocab(\n                max_vocab=self.max_vocab,\n                min_freq=self.min_freq,\n                pad_idx=self.pad_idx,\n                n_chunks=self.n_chunks,\n            )\n\n        self.vocab.fit(tokens)\n\n        if self.chunk_counter == self.n_chunks:\n            if self.verbose:\n                print(\"The vocabulary contains {} tokens\".format(len(self.vocab.stoi)))\n            if self.word_vectors_path is not None:\n                self.embedding_matrix = build_embeddings_matrix(\n                    self.vocab, self.word_vectors_path, self.min_freq\n                )\n\n            self.is_fitted = True\n\n        return self\n\n    def fit(self, df: pd.DataFrame) -&gt; \"ChunkTextPreprocessor\":\n        # df is a chunk of the original dataframe\n        return self.partial_fit(df)\n\n    def __repr__(self) -&gt; str:\n        list_of_params: List[str] = [\"text_col='{text_col}'\"]\n        if self.n_chunks is not None:\n            list_of_params.append(\"n_chunks={n_chunks}\")\n        if self.root_dir is not None:\n            list_of_params.append(\"root_dir={root_dir}\")\n        list_of_params.append(\"max_vocab={max_vocab}\")\n        list_of_params.append(\"min_freq={min_freq}\")\n        list_of_params.append(\"maxlen={maxlen}\")\n        list_of_params.append(\"pad_first={pad_first}\")\n        list_of_params.append(\"pad_idx={pad_idx}\")\n        if self.word_vectors_path is not None:\n            list_of_params.append(\"word_vectors_path={word_vectors_path}\")\n        if self.n_cpus is not None:\n            list_of_params.append(\"n_cpus={n_cpus}\")\n        list_of_params.append(\"verbose={verbose}\")\n        all_params = \", \".join(list_of_params)\n        return f\"ChunkTextPreprocessor({all_params.format(**self.__dict__)})\"\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.hf_preprocessor.ChunkHFPreprocessor","title":"ChunkHFPreprocessor","text":"<p>               Bases: <code>HFPreprocessor</code></p> <p>Text processor to prepare the <code>deeptext</code> input dataset that is a wrapper around HuggingFace's tokenizers.</p> <p>Hugginface Tokenizer's are already 'trained'. Therefore, unlike the <code>ChunkTextPreprocessor</code> this is mostly identical to the <code>HFPreprocessor</code> with the only difference that the class needs a 'text_col' parameter to be passed. Also the parameter <code>encode_params</code> is not really optional when using this class. It must be passed containing at least the 'max_length' encoding parameter. This is because we need to ensure that  all sequences have the same length when encoding in chunks.</p> <p>Parameters:</p> Name Type Description Default <code>model_name</code> <code>str</code> <p>The model name from the transformers library e.g. 'bert-base-uncased'. Currently supported models are those from the families: BERT, RoBERTa, DistilBERT, ALBERT and ELECTRA.</p> required <code>text_col</code> <code>str</code> <p>The column in the input dataframe containing the text data. When using the <code>ChunkHFPreprocessor</code> the <code>text_col</code> parameter is mandatory.</p> required <code>root_dir</code> <code>Optional[str]</code> <p>The root directory where the text files are located. This is only needed if the text data is stored in text files. If the text data is stored in a column in the input dataframe, this parameter is not needed.</p> <code>None</code> <code>use_fast_tokenizer</code> <code>bool</code> <p>Whether to use the fast tokenizer from HuggingFace or not</p> <code>True</code> <code>num_workers</code> <code>Optional[int]</code> <p>Number of workers to use when preprocessing the text data. If not None, and <code>use_fast_tokenizer</code> is False, the text data will be preprocessed in parallel using the number of workers specified. If <code>use_fast_tokenizer</code> is True, this argument is ignored.</p> <code>None</code> <code>preprocessing_rules</code> <code>Optional[List[Callable[[str], str]]]</code> <p>A list of functions to be applied to the text data before encoding. This can be useful to clean the text data before encoding. For example, removing html tags, special characters, etc.</p> <code>None</code> <code>tokenizer_params</code> <code>Optional[Dict[str, Any]]</code> <p>Additional parameters to be passed to the HuggingFace's <code>PreTrainedTokenizer</code>.</p> <code>None</code> <code>encode_params</code> <code>Optional[Dict[str, Any]]</code> <p>Additional parameters to be passed to the <code>batch_encode_plus</code> method of the HuggingFace's <code>PreTrainedTokenizer</code>. In the case of the <code>ChunkHFPreprocessor</code>, this parameter is not really <code>Optional</code>. It must be passed containing at least the 'max_length' encoding parameter</p> <code>None</code> <p>Attributes:</p> Name Type Description <code>is_fitted</code> <code>bool</code> <p>Boolean indicating if the preprocessor has been fitted. This is a HuggingFacea tokenizer, so it is always considered fitted and this attribute is manually set to True internally. This parameter exists for consistency with the rest of the library and because is needed for some functionality in the library.</p> Source code in <code>pytorch_widedeep/preprocessing/hf_preprocessor.py</code> <pre><code>class ChunkHFPreprocessor(HFPreprocessor):\n    \"\"\"Text processor to prepare the ``deeptext`` input dataset that is a\n    wrapper around HuggingFace's tokenizers.\n\n    Hugginface Tokenizer's are already 'trained'. Therefore, unlike the\n    `ChunkTextPreprocessor` this is mostly identical to the `HFPreprocessor`\n    with the only difference that the class needs a 'text_col' parameter to\n    be passed. Also the parameter `encode_params` is not really optional when\n    using this class. It must be passed containing at least the\n    'max_length' encoding parameter. This is because we need to ensure that\n     all sequences have the same length when encoding in chunks.\n\n    Parameters\n    ----------\n    model_name: str\n        The model name from the transformers library e.g. _'bert-base-uncased'_.\n        Currently supported models are those from the families: BERT, RoBERTa,\n        DistilBERT, ALBERT and ELECTRA.\n    text_col: str, default = None\n        The column in the input dataframe containing the text data. When using\n        the `ChunkHFPreprocessor` the `text_col` parameter is mandatory.\n    root_dir: Optional[str], default = None\n        The root directory where the text files are located. This is only\n        needed if the text data is stored in text files. If the text data is\n        stored in a column in the input dataframe, this parameter is not\n        needed.\n    use_fast_tokenizer: bool, default = False\n        Whether to use the fast tokenizer from HuggingFace or not\n    num_workers: Optional[int], default = None\n        Number of workers to use when preprocessing the text data. If not\n        None, and `use_fast_tokenizer` is False, the text data will be\n        preprocessed in parallel using the number of workers specified. If\n        `use_fast_tokenizer` is True, this argument is ignored.\n    preprocessing_rules: Optional[List[Callable[[str], str]]], default = None\n        A list of functions to be applied to the text data before encoding.\n        This can be useful to clean the text data before encoding. For\n        example, removing html tags, special characters, etc.\n    tokenizer_params: Optional[Dict[str, Any]], default = None\n        Additional parameters to be passed to the HuggingFace's\n        `PreTrainedTokenizer`.\n    encode_params: Optional[Dict[str, Any]], default = None\n        Additional parameters to be passed to the `batch_encode_plus` method\n        of the HuggingFace's `PreTrainedTokenizer`. In the case of the\n        `ChunkHFPreprocessor`, this parameter is not really `Optional`. It\n        must be passed containing at least the 'max_length' encoding\n        parameter\n\n    Attributes\n    ----------\n    is_fitted: bool\n        Boolean indicating if the preprocessor has been fitted. This is a\n        HuggingFacea tokenizer, so it is always considered fitted and this\n        attribute is manually set to True internally. This parameter exists\n        for consistency with the rest of the library and because is needed\n        for some functionality in the library.\n\n    \"\"\"\n\n    def __init__(\n        self,\n        model_name: str,\n        *,\n        text_col: str,\n        root_dir: Optional[str] = None,\n        use_fast_tokenizer: bool = True,\n        num_workers: Optional[int] = None,\n        preprocessing_rules: Optional[List[Callable[[str], str]]] = None,\n        tokenizer_params: Optional[Dict[str, Any]] = None,\n        encode_params: Optional[Dict[str, Any]] = None,\n    ):\n        super().__init__(\n            model_name=model_name,\n            use_fast_tokenizer=use_fast_tokenizer,\n            text_col=text_col,\n            num_workers=num_workers,\n            preprocessing_rules=preprocessing_rules,\n            tokenizer_params=tokenizer_params,\n            encode_params=encode_params,\n        )\n\n        self.root_dir = root_dir\n\n        # when using in chunks encode_params is not really optional. I will\n        # review types in due time\n        if self.encode_params is None:\n            raise ValueError(\n                \"The 'encode_params' dict must be passed to the ChunkHFTokenizer \"\n                \"containing at least the 'max_length' encoding parameter\"\n            )\n\n        if \"padding\" not in self.encode_params or not self.encode_params[\"padding\"]:\n            self.encode_params[\"padding\"] = True\n\n        if (\n            \"truncation\" not in self.encode_params\n            or not self.encode_params[\"truncation\"]\n        ):\n            self.encode_params[\"truncation\"] = True\n\n    def __repr__(self):\n        return (\n            f\"ChunkHFPreprocessor(text_col={self.text_col}, model_name={self.model_name}, \"\n            f\"use_fast_tokenizer={self.use_fast_tokenizer}, num_workers={self.num_workers}, \"\n            f\"preprocessing_rules={self.preprocessing_rules}, tokenizer_params={self.tokenizer_params}, \"\n            f\"encode_params={self.encode_params}, root_dir={self.root_dir})\"\n        )\n</code></pre>"},{"location":"pytorch-widedeep/self_supervised_pretraining.html","title":"Self Supervised Pre-training for tabular data","text":"<p>In this library we have implemented two methods or routines that allow the user to use self-suerpvised pre-training for all tabular models in the library with the exception of the <code>TabPerceiver</code> (this is a particular model and self-supervised pre-training requires some adjustments that will be implemented in future versions). Please see the examples folder in the repo or the examples section in the docs for details on how to use self-supervised pre-training with this library.</p> <p>The two routines implemented are illustrated in the figures below. The first is from TabNet: Attentive Interpretable Tabular Learning. It is a 'standard' encoder-decoder architecture and and is designed here for models that do not use transformer-based architectures (or when the embeddings can all have different dimensions). The second is from SAINT: Improved Neural Networks for Tabular Data via Row Attention and Contrastive Pre-Training, it is based on Contrastive and Denoising learning and is designed for models that use transformer-based architectures (or when the embeddings all need to have the same dimension):</p> <p> </p> <p>Figure 1. Figure 2 in their paper. The caption of the original paper is included in case it is useful.</p> <p> </p> <p>Figure 2. Figure 1 in their paper. The caption of the original paper is included in case it is useful.</p> <p>Note that the self-supervised pre-trainers described below focus, of course, on the self-supervised pre-training phase, i.e. the left side in Figure 1 and the upper part in Figure 2. When combined with the <code>Trainer</code> described earlier in the documenation, one can reproduce the full process illustrated in the figures above.</p> <p>Also Note that it is beyond the scope of this docs to explain in detail these routines. In addition, to fully utilise the self-supervised trainers implemented in this library a minimum understanding of the processes as described in the papers is required. Therefore, we strongly encourage the users to have a look to the papers.</p>"},{"location":"pytorch-widedeep/self_supervised_pretraining.html#pytorch_widedeep.self_supervised_training.EncoderDecoderTrainer","title":"EncoderDecoderTrainer","text":"<p>               Bases: <code>BaseEncoderDecoderTrainer</code></p> <p>This class implements an Encoder-Decoder self-supervised 'routine' inspired by TabNet: Attentive Interpretable Tabular Learning. See Figure 1 above.</p> <p>Parameters:</p> Name Type Description Default <code>encoder</code> <code>ModelWithoutAttention</code> <p>An instance of a <code>TabMlp</code>, <code>TabResNet</code> or <code>TabNet</code> model</p> required <code>decoder</code> <code>Optional[DecoderWithoutAttention]</code> <p>An instance of  a <code>TabMlpDecoder</code>, <code>TabResNetDecoder</code> or <code>TabNetDecoder</code> model. if <code>None</code> the decoder will be automatically built as a 'simetric' model to the Encoder</p> <code>None</code> <code>masked_prob</code> <code>float</code> <p>Indicates the fraction of elements in the embedding tensor that will be masked and hence used for reconstruction</p> <code>0.2</code> <code>optimizer</code> <code>Optional[Optimizer]</code> <p>An instance of Pytorch's <code>Optimizer</code> object (e.g. <code>torch.optim.Adam ()</code>). if no optimizer is passed it will default to <code>AdamW</code>.</p> <code>None</code> <code>lr_scheduler</code> <code>Optional[LRScheduler]</code> <p>An instance of Pytorch's <code>LRScheduler</code> object (e.g <code>torch.optim.lr_scheduler.StepLR(opt, step_size=5)</code>).</p> <code>None</code> <code>callbacks</code> <code>Optional[List[Callback]]</code> <p>List with <code>Callback</code> objects. The three callbacks available in <code>pytorch-widedeep</code> are: <code>LRHistory</code>, <code>ModelCheckpoint</code> and <code>EarlyStopping</code>. This can also be a custom callback. See <code>pytorch_widedeep.callbacks.Callback</code> or the Examples folder in the repo.</p> <code>None</code> <code>verbose</code> <code>int</code> <p>Setting it to 0 will print nothing during training.</p> <code>1</code> <code>seed</code> <code>int</code> <p>Random seed to be used internally for train_test_split</p> <code>1</code> <p>Other Parameters:</p> Name Type Description <code>**kwargs</code> <p>Other infrequently used arguments that can also be passed as kwargs are:</p> <ul> <li> <p>device: <code>str</code>     string indicating the device. One of 'cpu' or 'gpu'</p> </li> <li> <p>num_workers: <code>int</code>     number of workers to be used internally by the data loaders</p> </li> <li> <p>reducelronplateau_criterion: <code>str</code>     This sets the criterion that will be used by the lr scheduler to     take a step: One of 'loss' or 'metric'. The ReduceLROnPlateau     learning rate is a bit particular.</p> </li> </ul> Source code in <code>pytorch_widedeep/self_supervised_training/encoder_decoder_trainer.py</code> <pre><code>class EncoderDecoderTrainer(BaseEncoderDecoderTrainer):\n    r\"\"\"This class implements an Encoder-Decoder self-supervised 'routine'\n    inspired by\n    [TabNet: Attentive Interpretable Tabular Learning](https://arxiv.org/abs/1908.07442).\n    See Figure 1 above.\n\n    Parameters\n    ----------\n    encoder: ModelWithoutAttention,\n        An instance of a `TabMlp`, `TabResNet` or `TabNet` model\n    decoder: Optional[DecoderWithoutAttention] = None,\n        An instance of  a `TabMlpDecoder`, `TabResNetDecoder` or\n        `TabNetDecoder` model. if `None` the decoder will be automatically\n        built as a '_simetric_' model to the Encoder\n    masked_prob: float = 0.2,\n        Indicates the fraction of elements in the embedding tensor that will\n        be masked and hence used for reconstruction\n    optimizer: Optional[Optimizer] = None,\n        An instance of Pytorch's `Optimizer` object (e.g. `torch.optim.Adam\n        ()`). if no optimizer is passed it will default to `AdamW`.\n    lr_scheduler: Optional[LRScheduler] = None,\n        An instance of Pytorch's `LRScheduler` object\n        (e.g `torch.optim.lr_scheduler.StepLR(opt, step_size=5)`).\n    callbacks: Optional[List[Callback]] = None,\n        List with `Callback` objects. The three callbacks available in\n        `pytorch-widedeep` are: `LRHistory`, `ModelCheckpoint` and\n        `EarlyStopping`. This can also be a custom callback. See\n        `pytorch_widedeep.callbacks.Callback` or the Examples folder in the\n        repo.\n    verbose: int, default=1\n        Setting it to 0 will print nothing during training.\n    seed: int, default=1\n        Random seed to be used internally for train_test_split\n\n    Other Parameters\n    ----------------\n    **kwargs: dict\n        Other infrequently used arguments that can also be passed as kwargs are:\n\n        - **device**: `str`&lt;br/&gt;\n            string indicating the device. One of _'cpu'_ or _'gpu'_\n\n        - **num_workers**: `int`&lt;br/&gt;\n            number of workers to be used internally by the data loaders\n\n        - **reducelronplateau_criterion**: `str`\n            This sets the criterion that will be used by the lr scheduler to\n            take a step: One of _'loss'_ or _'metric'_. The ReduceLROnPlateau\n            learning rate is a bit particular.\n\n    \"\"\"\n\n    def __init__(\n        self,\n        encoder: ModelWithoutAttention,\n        decoder: Optional[DecoderWithoutAttention] = None,\n        masked_prob: float = 0.2,\n        optimizer: Optional[Optimizer] = None,\n        lr_scheduler: Optional[LRScheduler] = None,\n        callbacks: Optional[List[Callback]] = None,\n        verbose: int = 1,\n        seed: int = 1,\n        **kwargs,\n    ):\n        super().__init__(\n            encoder=encoder,\n            decoder=decoder,\n            masked_prob=masked_prob,\n            optimizer=optimizer,\n            lr_scheduler=lr_scheduler,\n            callbacks=callbacks,\n            verbose=verbose,\n            seed=seed,\n            **kwargs,\n        )\n\n    def pretrain(\n        self,\n        X_tab: np.ndarray,\n        X_tab_val: Optional[np.ndarray] = None,\n        val_split: Optional[float] = None,\n        validation_freq: int = 1,\n        n_epochs: int = 1,\n        batch_size: int = 32,\n    ):\n        r\"\"\"Pretrain method. Can also be called using `.fit(&lt;same_args&gt;)`\n\n        Parameters\n        ----------\n        X_tab: np.ndarray,\n            tabular dataset\n        X_tab_val: np.ndarray, Optional, default = None\n            validation data\n        val_split: float, Optional. default=None\n            An alterative to passing the validation set is to use a train/val\n            split fraction via `val_split`\n        validation_freq: int, default=1\n            epochs validation frequency\n        n_epochs: int, default=1\n            number of epochs\n        batch_size: int, default=32\n            batch size\n        \"\"\"\n\n        self.batch_size = batch_size\n\n        train_set, eval_set = self._train_eval_split(X_tab, X_tab_val, val_split)\n        train_loader = DataLoader(\n            dataset=train_set, batch_size=batch_size, num_workers=self.num_workers\n        )\n        train_steps = len(train_loader)\n        if eval_set is not None:\n            eval_loader = DataLoader(\n                dataset=eval_set,\n                batch_size=batch_size,\n                num_workers=self.num_workers,\n                shuffle=False,\n            )\n            eval_steps = len(eval_loader)\n\n        self.callback_container.on_train_begin(\n            {\n                \"batch_size\": batch_size,\n                \"train_steps\": train_steps,\n                \"n_epochs\": n_epochs,\n            }\n        )\n        for epoch in range(n_epochs):\n            epoch_logs: Dict[str, float] = {}\n            self.callback_container.on_epoch_begin(epoch, logs=epoch_logs)\n\n            self.train_running_loss = 0.0\n            with trange(train_steps, disable=self.verbose != 1) as t:\n                for batch_idx, X in zip(t, train_loader):\n                    t.set_description(\"epoch %i\" % (epoch + 1))\n                    train_loss = self._train_step(X[0], batch_idx)\n                    self.callback_container.on_batch_end(batch=batch_idx)\n                    print_loss_and_metric(t, train_loss)\n\n            epoch_logs = save_epoch_logs(epoch_logs, train_loss, None, \"train\")\n\n            on_epoch_end_metric = None\n            if eval_set is not None and epoch % validation_freq == (\n                validation_freq - 1\n            ):\n                self.callback_container.on_eval_begin()\n                self.valid_running_loss = 0.0\n                with trange(eval_steps, disable=self.verbose != 1) as v:\n                    for batch_idx, X in zip(v, eval_loader):\n                        v.set_description(\"valid\")\n                        val_loss = self._eval_step(X[0], batch_idx)\n                        print_loss_and_metric(v, val_loss)\n                epoch_logs = save_epoch_logs(epoch_logs, val_loss, None, \"val\")\n                on_epoch_end_metric = val_loss\n            else:\n                if self.reducelronplateau:\n                    raise NotImplementedError(\n                        \"ReduceLROnPlateau scheduler can be used only with validation data.\"\n                    )\n\n            self.callback_container.on_epoch_end(epoch, epoch_logs, on_epoch_end_metric)\n\n            if self.early_stop:\n                self.callback_container.on_train_end(epoch_logs)\n                break\n\n        self.callback_container.on_train_end(epoch_logs)\n        self._restore_best_weights()\n        self.ed_model.train()\n\n    def fit(\n        self,\n        X_tab: np.ndarray,\n        X_tab_val: Optional[np.ndarray] = None,\n        val_split: Optional[float] = None,\n        validation_freq: int = 1,\n        n_epochs: int = 1,\n        batch_size: int = 32,\n    ):\n        return self.pretrain(\n            X_tab, X_tab_val, val_split, validation_freq, n_epochs, batch_size\n        )\n\n    def explain(self, X_tab: np.ndarray, save_step_masks: bool = False):\n        raise NotImplementedError(\n            \"The 'explain' is currently not implemented for Self Supervised Pretraining\"\n        )\n\n    def _train_step(self, X_tab: Tensor, batch_idx: int) -&gt; float:\n        X = X_tab.to(self.device)\n\n        self.optimizer.zero_grad()\n        x_embed, x_embed_rec, mask = self.ed_model(X)\n        loss = self.loss_fn(x_embed, x_embed_rec, mask)\n        loss.backward()\n        self.optimizer.step()\n\n        self.train_running_loss += loss.item()\n        avg_loss = self.train_running_loss / (batch_idx + 1)\n\n        return avg_loss\n\n    def _eval_step(self, X_tab: Tensor, batch_idx: int) -&gt; float:\n        self.ed_model.eval()\n\n        with torch.no_grad():\n            X = X_tab.to(self.device)\n\n            x_embed, x_embed_rec, mask = self.ed_model(X)\n            loss = self.loss_fn(x_embed, x_embed_rec, mask)\n\n            self.valid_running_loss += loss.item()\n            avg_loss = self.valid_running_loss / (batch_idx + 1)\n\n        return avg_loss\n\n    def _train_eval_split(\n        self,\n        X: np.ndarray,\n        X_tab_val: Optional[np.ndarray] = None,\n        val_split: Optional[float] = None,\n    ) -&gt; Tuple[TensorDataset, Optional[TensorDataset]]:\n        if X_tab_val is not None:\n            train_set = TensorDataset(torch.from_numpy(X))\n            eval_set = TensorDataset(torch.from_numpy(X_tab_val))\n        elif val_split is not None:\n            X_tr, X_tab_val = train_test_split(  # type: ignore\n                X, test_size=val_split, random_state=self.seed\n            )\n            train_set = TensorDataset(torch.from_numpy(X_tr))\n            eval_set = TensorDataset(torch.from_numpy(X_tab_val))\n        else:\n            train_set = TensorDataset(torch.from_numpy(X))\n            eval_set = None\n\n        return train_set, eval_set\n</code></pre>"},{"location":"pytorch-widedeep/self_supervised_pretraining.html#pytorch_widedeep.self_supervised_training.EncoderDecoderTrainer.pretrain","title":"pretrain","text":"<pre><code>pretrain(\n    X_tab,\n    X_tab_val=None,\n    val_split=None,\n    validation_freq=1,\n    n_epochs=1,\n    batch_size=32,\n)\n</code></pre> <p>Pretrain method. Can also be called using <code>.fit(&lt;same_args&gt;)</code></p> <p>Parameters:</p> Name Type Description Default <code>X_tab</code> <code>ndarray</code> <p>tabular dataset</p> required <code>X_tab_val</code> <code>Optional[ndarray]</code> <p>validation data</p> <code>None</code> <code>val_split</code> <code>Optional[float]</code> <p>An alterative to passing the validation set is to use a train/val split fraction via <code>val_split</code></p> <code>None</code> <code>validation_freq</code> <code>int</code> <p>epochs validation frequency</p> <code>1</code> <code>n_epochs</code> <code>int</code> <p>number of epochs</p> <code>1</code> <code>batch_size</code> <code>int</code> <p>batch size</p> <code>32</code> Source code in <code>pytorch_widedeep/self_supervised_training/encoder_decoder_trainer.py</code> <pre><code>def pretrain(\n    self,\n    X_tab: np.ndarray,\n    X_tab_val: Optional[np.ndarray] = None,\n    val_split: Optional[float] = None,\n    validation_freq: int = 1,\n    n_epochs: int = 1,\n    batch_size: int = 32,\n):\n    r\"\"\"Pretrain method. Can also be called using `.fit(&lt;same_args&gt;)`\n\n    Parameters\n    ----------\n    X_tab: np.ndarray,\n        tabular dataset\n    X_tab_val: np.ndarray, Optional, default = None\n        validation data\n    val_split: float, Optional. default=None\n        An alterative to passing the validation set is to use a train/val\n        split fraction via `val_split`\n    validation_freq: int, default=1\n        epochs validation frequency\n    n_epochs: int, default=1\n        number of epochs\n    batch_size: int, default=32\n        batch size\n    \"\"\"\n\n    self.batch_size = batch_size\n\n    train_set, eval_set = self._train_eval_split(X_tab, X_tab_val, val_split)\n    train_loader = DataLoader(\n        dataset=train_set, batch_size=batch_size, num_workers=self.num_workers\n    )\n    train_steps = len(train_loader)\n    if eval_set is not None:\n        eval_loader = DataLoader(\n            dataset=eval_set,\n            batch_size=batch_size,\n            num_workers=self.num_workers,\n            shuffle=False,\n        )\n        eval_steps = len(eval_loader)\n\n    self.callback_container.on_train_begin(\n        {\n            \"batch_size\": batch_size,\n            \"train_steps\": train_steps,\n            \"n_epochs\": n_epochs,\n        }\n    )\n    for epoch in range(n_epochs):\n        epoch_logs: Dict[str, float] = {}\n        self.callback_container.on_epoch_begin(epoch, logs=epoch_logs)\n\n        self.train_running_loss = 0.0\n        with trange(train_steps, disable=self.verbose != 1) as t:\n            for batch_idx, X in zip(t, train_loader):\n                t.set_description(\"epoch %i\" % (epoch + 1))\n                train_loss = self._train_step(X[0], batch_idx)\n                self.callback_container.on_batch_end(batch=batch_idx)\n                print_loss_and_metric(t, train_loss)\n\n        epoch_logs = save_epoch_logs(epoch_logs, train_loss, None, \"train\")\n\n        on_epoch_end_metric = None\n        if eval_set is not None and epoch % validation_freq == (\n            validation_freq - 1\n        ):\n            self.callback_container.on_eval_begin()\n            self.valid_running_loss = 0.0\n            with trange(eval_steps, disable=self.verbose != 1) as v:\n                for batch_idx, X in zip(v, eval_loader):\n                    v.set_description(\"valid\")\n                    val_loss = self._eval_step(X[0], batch_idx)\n                    print_loss_and_metric(v, val_loss)\n            epoch_logs = save_epoch_logs(epoch_logs, val_loss, None, \"val\")\n            on_epoch_end_metric = val_loss\n        else:\n            if self.reducelronplateau:\n                raise NotImplementedError(\n                    \"ReduceLROnPlateau scheduler can be used only with validation data.\"\n                )\n\n        self.callback_container.on_epoch_end(epoch, epoch_logs, on_epoch_end_metric)\n\n        if self.early_stop:\n            self.callback_container.on_train_end(epoch_logs)\n            break\n\n    self.callback_container.on_train_end(epoch_logs)\n    self._restore_best_weights()\n    self.ed_model.train()\n</code></pre>"},{"location":"pytorch-widedeep/self_supervised_pretraining.html#pytorch_widedeep.self_supervised_training.ContrastiveDenoisingTrainer","title":"ContrastiveDenoisingTrainer","text":"<p>               Bases: <code>BaseContrastiveDenoisingTrainer</code></p> <p>This class trains a Contrastive, Denoising Self Supervised 'routine' that is based on the one described in SAINT: Improved Neural Networks for Tabular Data via Row Attention and Contrastive Pre-Training, their Figure 1.</p> <p>Parameters:</p> Name Type Description Default <code>model</code> <code>ModelWithAttention</code> <p>An instance of a <code>TabTransformer</code>, <code>SAINT</code>, <code>FTTransformer</code>, <code>TabFastFormer</code>, <code>TabPerceiver</code>, <code>ContextAttentionMLP</code> and <code>SelfAttentionMLP</code>.</p> required <code>preprocessor</code> <code>TabPreprocessor</code> <p>A fitted <code>TabPreprocessor</code> object. See <code>pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor</code></p> required <code>optimizer</code> <code>Optional[Optimizer]</code> <p>An instance of Pytorch's <code>Optimizer</code> object (e.g. <code>torch.optim.Adam ()</code>). if no optimizer is passed it will default to <code>AdamW</code>.</p> <code>None</code> <code>lr_scheduler</code> <code>Optional[LRScheduler]</code> <p>An instance of Pytorch's <code>LRScheduler</code> object (e.g <code>torch.optim.lr_scheduler.StepLR(opt, step_size=5)</code>).</p> <code>None</code> <code>callbacks</code> <code>Optional[List[Callback]]</code> <p>List with <code>Callback</code> objects. The three callbacks available in <code>pytorch-widedeep</code> are: <code>LRHistory</code>, <code>ModelCheckpoint</code> and <code>EarlyStopping</code>. This can also be a custom callback. See <code>pytorch_widedeep.callbacks.Callback</code> or the Examples folder in the repo.</p> <code>None</code> <code>loss_type</code> <code>Literal[contrastive, denoising, both]</code> <p>One of 'contrastive', 'denoising' or 'both'. See SAINT: Improved Neural Networks for Tabular Data via Row Attention and Contrastive Pre-Training, their figure (1) and their equation (5).</p> <code>'both'</code> <code>projection_head1_dims</code> <code>Optional[List[int]]</code> <p>The projection heads are simply MLPs. This parameter is a list of integers with the dimensions of the MLP hidden layers. See the paper for details. Note that setting up this parameter requires some knowledge of the architecture one is using. For example, if we are representing the features with embeddings of dim 32 (i.e. the so called dimension of the model is 32), then the first dimension of the projection head must be 32 (e.g. [32, 16])</p> <code>None</code> <code>projection_head2_dims</code> <code>Optional[List[int]]</code> <p>Same as 'projection_head1_dims' for the second head</p> <code>None</code> <code>projection_heads_activation</code> <code>str</code> <p>Activation function for the projection heads</p> <code>'relu'</code> <code>cat_mlp_type</code> <code>Literal[single, multiple]</code> <p>If 'denoising' loss is used, one can choose two types of 'stacked' MLPs to process the output from the transformer-based encoder that receives 'corrupted' (cut-mixed and mixed-up) features. These are 'single' or 'multiple'. The former approach will apply a single MLP to all the categorical features while the latter will use one MLP per categorical feature</p> <code>'multiple'</code> <code>cont_mlp_type</code> <code>Literal[single, multiple]</code> <p>Same as 'cat_mlp_type' but for the continuous features</p> <code>'multiple'</code> <code>denoise_mlps_activation</code> <code>str</code> <p>activation function for the so called 'denoising mlps'.</p> <code>'relu'</code> <code>verbose</code> <code>int</code> <p>Setting it to 0 will print nothing during training.</p> <code>1</code> <code>seed</code> <code>int</code> <p>Random seed to be used internally for train_test_split</p> <code>1</code> <p>Other Parameters:</p> Name Type Description <code>**kwargs</code> <p>Other infrequently used arguments that can also be passed as kwargs are:</p> <ul> <li> <p>device: <code>str</code>     string indicating the device. One of 'cpu' or 'gpu'</p> </li> <li> <p>num_workers: <code>int</code>     number of workers to be used internally by the data loaders</p> </li> <li> <p>reducelronplateau_criterion: <code>str</code>     This sets the criterion that will be used by the lr scheduler to     take a step: One of 'loss' or 'metric'. The ReduceLROnPlateau     learning rate is a bit particular.</p> </li> </ul> Source code in <code>pytorch_widedeep/self_supervised_training/contrastive_denoising_trainer.py</code> <pre><code>class ContrastiveDenoisingTrainer(BaseContrastiveDenoisingTrainer):\n    r\"\"\"This class trains a Contrastive, Denoising Self Supervised 'routine' that\n    is based on the one described in\n    [SAINT: Improved Neural Networks for Tabular Data via Row Attention and\n    Contrastive Pre-Training](https://arxiv.org/abs/2106.01342), their Figure 1.\n\n    Parameters\n    ----------\n    model: ModelWithAttention,\n        An instance of a `TabTransformer`, `SAINT`, `FTTransformer`,\n        `TabFastFormer`, `TabPerceiver`, `ContextAttentionMLP` and\n        `SelfAttentionMLP`.\n    preprocessor: `TabPreprocessor`\n        A fitted `TabPreprocessor` object. See\n        `pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor`\n    optimizer: Optional[Optimizer] = None,\n        An instance of Pytorch's `Optimizer` object (e.g. `torch.optim.Adam\n        ()`). if no optimizer is passed it will default to `AdamW`.\n    lr_scheduler: Optional[LRScheduler] = None,\n        An instance of Pytorch's `LRScheduler` object\n        (e.g `torch.optim.lr_scheduler.StepLR(opt, step_size=5)`).\n    callbacks: Optional[List[Callback]] = None,\n        List with `Callback` objects. The three callbacks available in\n        `pytorch-widedeep` are: `LRHistory`, `ModelCheckpoint` and\n        `EarlyStopping`. This can also be a custom callback. See\n        `pytorch_widedeep.callbacks.Callback` or the Examples folder in the\n        repo.\n    loss_type: str, default = \"both\"\n        One of '_contrastive_', '_denoising_' or '_both_'. See [SAINT: Improved\n        Neural Networks for Tabular Data via Row Attention and Contrastive\n        Pre-Training](https://arxiv.org/abs/2203.05556), their figure (1)\n        and their equation (5).\n    projection_head1_dims: list, Optional, default = None\n        The projection heads are simply MLPs. This parameter is a list\n        of integers with the dimensions of the MLP hidden layers. See the\n        [paper](https://arxiv.org/abs/2203.05556) for details. Note that\n        setting up this parameter requires some knowledge of the architecture\n        one is using. For example, if we are representing the features with\n        embeddings of dim 32 (i.e. the so called dimension of the model is\n        32), then the first dimension of the projection head must be 32 (e.g.\n        [32, 16])\n    projection_head2_dims: list, Optional, default = None\n        Same as '_projection_head1_dims_' for the second head\n    projection_heads_activation: str, default = \"relu\"\n        Activation function for the projection heads\n    cat_mlp_type: str, default = \"multiple\"\n        If '_denoising_' loss is used, one can choose two types of 'stacked'\n        MLPs to process the output from the transformer-based encoder that\n        receives 'corrupted' (cut-mixed and mixed-up) features. These\n        are '_single_' or '_multiple_'. The former approach will apply a single\n        MLP to all the categorical features while the latter will use one MLP\n        per categorical feature\n    cont_mlp_type: str, default = \"multiple\"\n        Same as 'cat_mlp_type' but for the continuous features\n    denoise_mlps_activation: str, default = \"relu\"\n        activation function for the so called 'denoising mlps'.\n    verbose: int, default=1\n        Setting it to 0 will print nothing during training.\n    seed: int, default=1\n        Random seed to be used internally for train_test_split\n\n    Other Parameters\n    ----------------\n    **kwargs: dict\n        Other infrequently used arguments that can also be passed as kwargs are:\n\n        - **device**: `str`&lt;br/&gt;\n            string indicating the device. One of _'cpu'_ or _'gpu'_\n\n        - **num_workers**: `int`&lt;br/&gt;\n            number of workers to be used internally by the data loaders\n\n        - **reducelronplateau_criterion**: `str`\n            This sets the criterion that will be used by the lr scheduler to\n            take a step: One of _'loss'_ or _'metric'_. The ReduceLROnPlateau\n            learning rate is a bit particular.\n\n    \"\"\"\n\n    def __init__(\n        self,\n        model: ModelWithAttention,\n        preprocessor: TabPreprocessor,\n        optimizer: Optional[Optimizer] = None,\n        lr_scheduler: Optional[LRScheduler] = None,\n        callbacks: Optional[List[Callback]] = None,\n        loss_type: Literal[\"contrastive\", \"denoising\", \"both\"] = \"both\",\n        projection_head1_dims: Optional[List[int]] = None,\n        projection_head2_dims: Optional[List[int]] = None,\n        projection_heads_activation: str = \"relu\",\n        cat_mlp_type: Literal[\"single\", \"multiple\"] = \"multiple\",\n        cont_mlp_type: Literal[\"single\", \"multiple\"] = \"multiple\",\n        denoise_mlps_activation: str = \"relu\",\n        verbose: int = 1,\n        seed: int = 1,\n        **kwargs,\n    ):\n        super().__init__(\n            model=model,\n            preprocessor=preprocessor,\n            loss_type=loss_type,\n            optimizer=optimizer,\n            lr_scheduler=lr_scheduler,\n            callbacks=callbacks,\n            projection_head1_dims=projection_head1_dims,\n            projection_head2_dims=projection_head2_dims,\n            projection_heads_activation=projection_heads_activation,\n            cat_mlp_type=cat_mlp_type,\n            cont_mlp_type=cont_mlp_type,\n            denoise_mlps_activation=denoise_mlps_activation,\n            verbose=verbose,\n            seed=seed,\n            **kwargs,\n        )\n\n    def pretrain(\n        self,\n        X_tab: np.ndarray,\n        X_tab_val: Optional[np.ndarray] = None,\n        val_split: Optional[float] = None,\n        validation_freq: int = 1,\n        n_epochs: int = 1,\n        batch_size: int = 32,\n    ):\n        r\"\"\"Pretrain method. Can also be called using `.fit(&lt;same_args&gt;)`\n\n        Parameters\n        ----------\n        X_tab: np.ndarray,\n            tabular dataset\n        X_tab_val: np.ndarray, Optional, default = None\n            validation data. Note that, although it is possible to use\n            contrastive-denoising training with a validation set, such set\n            must include feature values that are _all_ seen in the training\n            set in the case of the categorical columns. This is because the\n            values of the columns themselves will be used as targets when\n            computing the loss. Therefore, if a new category is present in\n            the validation set that was not seen in training this will\n            effectively be like trying to predict a new, never seen category\n            (and Pytorch will throw an error)\n        val_split: float, Optional. default=None\n            An alterative to passing the validation set is to use a train/val\n            split fraction via `val_split`\n        validation_freq: int, default=1\n            epochs validation frequency\n        n_epochs: int, default=1\n            number of epochs\n        batch_size: int, default=32\n            batch size\n        \"\"\"\n\n        self.batch_size = batch_size\n\n        train_set, eval_set = self._train_eval_split(X_tab, X_tab_val, val_split)\n        train_loader = DataLoader(\n            dataset=train_set, batch_size=batch_size, num_workers=self.num_workers\n        )\n        train_steps = len(train_loader)\n        if eval_set is not None:\n            eval_loader = DataLoader(\n                dataset=eval_set,\n                batch_size=batch_size,\n                num_workers=self.num_workers,\n                shuffle=False,\n            )\n            eval_steps = len(eval_loader)\n\n        self.callback_container.on_train_begin(\n            {\n                \"batch_size\": batch_size,\n                \"train_steps\": train_steps,\n                \"n_epochs\": n_epochs,\n            }\n        )\n        for epoch in range(n_epochs):\n            epoch_logs: Dict[str, float] = {}\n            self.callback_container.on_epoch_begin(epoch, logs=epoch_logs)\n\n            self.train_running_loss = 0.0\n            with trange(train_steps, disable=self.verbose != 1) as t:\n                for batch_idx, X in zip(t, train_loader):\n                    t.set_description(\"epoch %i\" % (epoch + 1))\n                    train_loss = self._train_step(X[0], batch_idx)\n                    self.callback_container.on_batch_end(batch=batch_idx)\n                    print_loss_and_metric(t, train_loss)\n\n            epoch_logs = save_epoch_logs(epoch_logs, train_loss, None, \"train\")\n\n            on_epoch_end_metric = None\n            if eval_set is not None and epoch % validation_freq == (\n                validation_freq - 1\n            ):\n                self.callback_container.on_eval_begin()\n                self.valid_running_loss = 0.0\n                with trange(eval_steps, disable=self.verbose != 1) as v:\n                    for batch_idx, X in zip(v, eval_loader):\n                        v.set_description(\"valid\")\n                        val_loss = self._eval_step(X[0], batch_idx)\n                        print_loss_and_metric(v, val_loss)\n                epoch_logs = save_epoch_logs(epoch_logs, val_loss, None, \"val\")\n                on_epoch_end_metric = val_loss\n            else:\n                if self.reducelronplateau:\n                    raise NotImplementedError(\n                        \"ReduceLROnPlateau scheduler can be used only with validation data.\"\n                    )\n\n            self.callback_container.on_epoch_end(epoch, epoch_logs, on_epoch_end_metric)\n\n            if self.early_stop:\n                self.callback_container.on_train_end(epoch_logs)\n                break\n\n        self.callback_container.on_train_end(epoch_logs)\n        self._restore_best_weights()\n        self.cd_model.train()\n\n    def fit(\n        self,\n        X_tab: np.ndarray,\n        X_tab_val: Optional[np.ndarray] = None,\n        val_split: Optional[float] = None,\n        validation_freq: int = 1,\n        n_epochs: int = 1,\n        batch_size: int = 32,\n    ):\n        return self.pretrain(\n            X_tab, X_tab_val, val_split, validation_freq, n_epochs, batch_size\n        )\n\n    def _train_step(self, X_tab: Tensor, batch_idx: int) -&gt; float:\n        X = X_tab.to(self.device)\n\n        self.optimizer.zero_grad()\n        g_projs, cat_x_and_x_, cont_x_and_x_ = self.cd_model(X)\n        loss = self._compute_loss(g_projs, cat_x_and_x_, cont_x_and_x_)\n        loss.backward()\n        self.optimizer.step()\n\n        self.train_running_loss += loss.item()\n        avg_loss = self.train_running_loss / (batch_idx + 1)\n\n        return avg_loss\n\n    def _eval_step(self, X_tab: Tensor, batch_idx: int) -&gt; float:\n        self.cd_model.eval()\n\n        with torch.no_grad():\n            X = X_tab.to(self.device)\n\n            g_projs, cat_x_and_x_, cont_x_and_x_ = self.cd_model(X)\n            loss = self._compute_loss(g_projs, cat_x_and_x_, cont_x_and_x_)\n\n            self.valid_running_loss += loss.item()\n            avg_loss = self.valid_running_loss / (batch_idx + 1)\n\n        return avg_loss\n\n    def _train_eval_split(\n        self,\n        X: np.ndarray,\n        X_tab_val: Optional[np.ndarray] = None,\n        val_split: Optional[float] = None,\n    ) -&gt; Tuple[TensorDataset, Optional[TensorDataset]]:\n        if X_tab_val is not None:\n            train_set = TensorDataset(torch.from_numpy(X))\n            eval_set = TensorDataset(torch.from_numpy(X_tab_val))\n        elif val_split is not None:\n            X_tr, X_tab_val = train_test_split(  # type: ignore\n                X, test_size=val_split, random_state=self.seed\n            )\n            train_set = TensorDataset(torch.from_numpy(X_tr))\n            eval_set = TensorDataset(torch.from_numpy(X_tab_val))\n        else:\n            train_set = TensorDataset(torch.from_numpy(X))\n            eval_set = None\n\n        return train_set, eval_set\n</code></pre>"},{"location":"pytorch-widedeep/self_supervised_pretraining.html#pytorch_widedeep.self_supervised_training.ContrastiveDenoisingTrainer.pretrain","title":"pretrain","text":"<pre><code>pretrain(\n    X_tab,\n    X_tab_val=None,\n    val_split=None,\n    validation_freq=1,\n    n_epochs=1,\n    batch_size=32,\n)\n</code></pre> <p>Pretrain method. Can also be called using <code>.fit(&lt;same_args&gt;)</code></p> <p>Parameters:</p> Name Type Description Default <code>X_tab</code> <code>ndarray</code> <p>tabular dataset</p> required <code>X_tab_val</code> <code>Optional[ndarray]</code> <p>validation data. Note that, although it is possible to use contrastive-denoising training with a validation set, such set must include feature values that are all seen in the training set in the case of the categorical columns. This is because the values of the columns themselves will be used as targets when computing the loss. Therefore, if a new category is present in the validation set that was not seen in training this will effectively be like trying to predict a new, never seen category (and Pytorch will throw an error)</p> <code>None</code> <code>val_split</code> <code>Optional[float]</code> <p>An alterative to passing the validation set is to use a train/val split fraction via <code>val_split</code></p> <code>None</code> <code>validation_freq</code> <code>int</code> <p>epochs validation frequency</p> <code>1</code> <code>n_epochs</code> <code>int</code> <p>number of epochs</p> <code>1</code> <code>batch_size</code> <code>int</code> <p>batch size</p> <code>32</code> Source code in <code>pytorch_widedeep/self_supervised_training/contrastive_denoising_trainer.py</code> <pre><code>def pretrain(\n    self,\n    X_tab: np.ndarray,\n    X_tab_val: Optional[np.ndarray] = None,\n    val_split: Optional[float] = None,\n    validation_freq: int = 1,\n    n_epochs: int = 1,\n    batch_size: int = 32,\n):\n    r\"\"\"Pretrain method. Can also be called using `.fit(&lt;same_args&gt;)`\n\n    Parameters\n    ----------\n    X_tab: np.ndarray,\n        tabular dataset\n    X_tab_val: np.ndarray, Optional, default = None\n        validation data. Note that, although it is possible to use\n        contrastive-denoising training with a validation set, such set\n        must include feature values that are _all_ seen in the training\n        set in the case of the categorical columns. This is because the\n        values of the columns themselves will be used as targets when\n        computing the loss. Therefore, if a new category is present in\n        the validation set that was not seen in training this will\n        effectively be like trying to predict a new, never seen category\n        (and Pytorch will throw an error)\n    val_split: float, Optional. default=None\n        An alterative to passing the validation set is to use a train/val\n        split fraction via `val_split`\n    validation_freq: int, default=1\n        epochs validation frequency\n    n_epochs: int, default=1\n        number of epochs\n    batch_size: int, default=32\n        batch size\n    \"\"\"\n\n    self.batch_size = batch_size\n\n    train_set, eval_set = self._train_eval_split(X_tab, X_tab_val, val_split)\n    train_loader = DataLoader(\n        dataset=train_set, batch_size=batch_size, num_workers=self.num_workers\n    )\n    train_steps = len(train_loader)\n    if eval_set is not None:\n        eval_loader = DataLoader(\n            dataset=eval_set,\n            batch_size=batch_size,\n            num_workers=self.num_workers,\n            shuffle=False,\n        )\n        eval_steps = len(eval_loader)\n\n    self.callback_container.on_train_begin(\n        {\n            \"batch_size\": batch_size,\n            \"train_steps\": train_steps,\n            \"n_epochs\": n_epochs,\n        }\n    )\n    for epoch in range(n_epochs):\n        epoch_logs: Dict[str, float] = {}\n        self.callback_container.on_epoch_begin(epoch, logs=epoch_logs)\n\n        self.train_running_loss = 0.0\n        with trange(train_steps, disable=self.verbose != 1) as t:\n            for batch_idx, X in zip(t, train_loader):\n                t.set_description(\"epoch %i\" % (epoch + 1))\n                train_loss = self._train_step(X[0], batch_idx)\n                self.callback_container.on_batch_end(batch=batch_idx)\n                print_loss_and_metric(t, train_loss)\n\n        epoch_logs = save_epoch_logs(epoch_logs, train_loss, None, \"train\")\n\n        on_epoch_end_metric = None\n        if eval_set is not None and epoch % validation_freq == (\n            validation_freq - 1\n        ):\n            self.callback_container.on_eval_begin()\n            self.valid_running_loss = 0.0\n            with trange(eval_steps, disable=self.verbose != 1) as v:\n                for batch_idx, X in zip(v, eval_loader):\n                    v.set_description(\"valid\")\n                    val_loss = self._eval_step(X[0], batch_idx)\n                    print_loss_and_metric(v, val_loss)\n            epoch_logs = save_epoch_logs(epoch_logs, val_loss, None, \"val\")\n            on_epoch_end_metric = val_loss\n        else:\n            if self.reducelronplateau:\n                raise NotImplementedError(\n                    \"ReduceLROnPlateau scheduler can be used only with validation data.\"\n                )\n\n        self.callback_container.on_epoch_end(epoch, epoch_logs, on_epoch_end_metric)\n\n        if self.early_stop:\n            self.callback_container.on_train_end(epoch_logs)\n            break\n\n    self.callback_container.on_train_end(epoch_logs)\n    self._restore_best_weights()\n    self.cd_model.train()\n</code></pre>"},{"location":"pytorch-widedeep/tab2vec.html","title":"Tab2Vec","text":""},{"location":"pytorch-widedeep/tab2vec.html#pytorch_widedeep.tab2vec.Tab2Vec","title":"Tab2Vec","text":"<p>Class to transform an input dataframe into vectorized form.</p> <p>This class will take an input dataframe in the form of the dataframe used for training, and it will turn it into a vectorised form based on the processing applied by the model to the categorical and continuous columns.</p> <p> NOTE: Currently this class is only implemented  for the deeptabular component. Therefore, if the input dataframe has a  text column or a column with the path to images, these will be ignored.  We will be adding these functionalities in future versions</p> <p>Parameters:</p> Name Type Description Default <code>model</code> <code>Union[WideDeep, BayesianWide, BayesianTabMlp]</code> <p><code>WideDeep</code>, <code>BayesianWide</code> or <code>BayesianTabMlp</code> model. Must be trained.</p> required <code>tab_preprocessor</code> <code>TabPreprocessor</code> <p><code>TabPreprocessor</code> object. Must be fitted.</p> required <code>return_dataframe</code> <code>bool</code> <p>Boolean indicating of the returned object(s) will be array(s) or pandas dataframe(s)</p> <code>False</code> <p>Attributes:</p> Name Type Description <code>vectorizer</code> <code>Module</code> <p>Torch module with the categorical and continuous encoding process</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import string\n&gt;&gt;&gt; from random import choices\n&gt;&gt;&gt; import numpy as np\n&gt;&gt;&gt; import pandas as pd\n&gt;&gt;&gt; from pytorch_widedeep import Tab2Vec\n&gt;&gt;&gt; from pytorch_widedeep.models import TabMlp, WideDeep\n&gt;&gt;&gt; from pytorch_widedeep.preprocessing import TabPreprocessor\n&gt;&gt;&gt;\n&gt;&gt;&gt; colnames = list(string.ascii_lowercase)[:4]\n&gt;&gt;&gt; cat_col1_vals = [\"a\", \"b\", \"c\"]\n&gt;&gt;&gt; cat_col2_vals = [\"d\", \"e\", \"f\"]\n&gt;&gt;&gt;\n&gt;&gt;&gt; # Create the toy input dataframe and a toy dataframe to be vectorised\n&gt;&gt;&gt; cat_inp = [np.array(choices(c, k=5)) for c in [cat_col1_vals, cat_col2_vals]]\n&gt;&gt;&gt; cont_inp = [np.round(np.random.rand(5), 2) for _ in range(2)]\n&gt;&gt;&gt; df_inp = pd.DataFrame(np.vstack(cat_inp + cont_inp).transpose(), columns=colnames)\n&gt;&gt;&gt; cat_t2v = [np.array(choices(c, k=5)) for c in [cat_col1_vals, cat_col2_vals]]\n&gt;&gt;&gt; cont_t2v = [np.round(np.random.rand(5), 2) for _ in range(2)]\n&gt;&gt;&gt; df_t2v = pd.DataFrame(np.vstack(cat_t2v + cont_t2v).transpose(), columns=colnames)\n&gt;&gt;&gt;\n&gt;&gt;&gt; # fit the TabPreprocessor\n&gt;&gt;&gt; embed_cols = [(\"a\", 2), (\"b\", 4)]\n&gt;&gt;&gt; cont_cols = [\"c\", \"d\"]\n&gt;&gt;&gt; tab_preprocessor = TabPreprocessor(cat_embed_cols=embed_cols, continuous_cols=cont_cols)\n&gt;&gt;&gt; X_tab = tab_preprocessor.fit_transform(df_inp)\n&gt;&gt;&gt;\n&gt;&gt;&gt; # define the model (and let's assume we train it)\n&gt;&gt;&gt; tabmlp = TabMlp(\n... column_idx=tab_preprocessor.column_idx,\n... cat_embed_input=tab_preprocessor.cat_embed_input,\n... continuous_cols=tab_preprocessor.continuous_cols,\n... mlp_hidden_dims=[8, 4])\n&gt;&gt;&gt; model = WideDeep(deeptabular=tabmlp)\n&gt;&gt;&gt; # ...train the model...\n&gt;&gt;&gt;\n&gt;&gt;&gt; # vectorise the dataframe\n&gt;&gt;&gt; t2v = Tab2Vec(tab_preprocessor, model, device=\"cpu\")\n&gt;&gt;&gt; X_vec = t2v.transform(df_t2v)\n</code></pre> Source code in <code>pytorch_widedeep/tab2vec.py</code> <pre><code>class Tab2Vec:\n    r\"\"\"Class to transform an input dataframe into vectorized form.\n\n    This class will take an input dataframe in the form of the dataframe used\n    for training, and it will turn it into a vectorised form based on the\n    processing applied by the model to the categorical and continuous\n    columns.\n\n    :information_source: **NOTE**: Currently this class is only implemented\n     for the deeptabular component. Therefore, if the input dataframe has a\n     text column or a column with the path to images, these will be ignored.\n     We will be adding these functionalities in future versions\n\n    Parameters\n    ----------\n    model: `WideDeep`, `BayesianWide` or `BayesianTabMlp`\n        `WideDeep`, `BayesianWide` or `BayesianTabMlp` model. Must be trained.\n    tab_preprocessor: `TabPreprocessor`\n        `TabPreprocessor` object. Must be fitted.\n    return_dataframe: bool\n        Boolean indicating of the returned object(s) will be array(s) or\n        pandas dataframe(s)\n\n    Attributes\n    ----------\n    vectorizer: nn.Module\n        Torch module with the categorical and continuous encoding process\n\n    Examples\n    --------\n    &gt;&gt;&gt; import string\n    &gt;&gt;&gt; from random import choices\n    &gt;&gt;&gt; import numpy as np\n    &gt;&gt;&gt; import pandas as pd\n    &gt;&gt;&gt; from pytorch_widedeep import Tab2Vec\n    &gt;&gt;&gt; from pytorch_widedeep.models import TabMlp, WideDeep\n    &gt;&gt;&gt; from pytorch_widedeep.preprocessing import TabPreprocessor\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; colnames = list(string.ascii_lowercase)[:4]\n    &gt;&gt;&gt; cat_col1_vals = [\"a\", \"b\", \"c\"]\n    &gt;&gt;&gt; cat_col2_vals = [\"d\", \"e\", \"f\"]\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; # Create the toy input dataframe and a toy dataframe to be vectorised\n    &gt;&gt;&gt; cat_inp = [np.array(choices(c, k=5)) for c in [cat_col1_vals, cat_col2_vals]]\n    &gt;&gt;&gt; cont_inp = [np.round(np.random.rand(5), 2) for _ in range(2)]\n    &gt;&gt;&gt; df_inp = pd.DataFrame(np.vstack(cat_inp + cont_inp).transpose(), columns=colnames)\n    &gt;&gt;&gt; cat_t2v = [np.array(choices(c, k=5)) for c in [cat_col1_vals, cat_col2_vals]]\n    &gt;&gt;&gt; cont_t2v = [np.round(np.random.rand(5), 2) for _ in range(2)]\n    &gt;&gt;&gt; df_t2v = pd.DataFrame(np.vstack(cat_t2v + cont_t2v).transpose(), columns=colnames)\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; # fit the TabPreprocessor\n    &gt;&gt;&gt; embed_cols = [(\"a\", 2), (\"b\", 4)]\n    &gt;&gt;&gt; cont_cols = [\"c\", \"d\"]\n    &gt;&gt;&gt; tab_preprocessor = TabPreprocessor(cat_embed_cols=embed_cols, continuous_cols=cont_cols)\n    &gt;&gt;&gt; X_tab = tab_preprocessor.fit_transform(df_inp)\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; # define the model (and let's assume we train it)\n    &gt;&gt;&gt; tabmlp = TabMlp(\n    ... column_idx=tab_preprocessor.column_idx,\n    ... cat_embed_input=tab_preprocessor.cat_embed_input,\n    ... continuous_cols=tab_preprocessor.continuous_cols,\n    ... mlp_hidden_dims=[8, 4])\n    &gt;&gt;&gt; model = WideDeep(deeptabular=tabmlp)\n    &gt;&gt;&gt; # ...train the model...\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; # vectorise the dataframe\n    &gt;&gt;&gt; t2v = Tab2Vec(tab_preprocessor, model, device=\"cpu\")\n    &gt;&gt;&gt; X_vec = t2v.transform(df_t2v)\n    \"\"\"\n\n    def __init__(\n        self,\n        tab_preprocessor: TabPreprocessor,\n        model: Union[WideDeep, BayesianWide, BayesianTabMlp],\n        device: str | torch.device,\n        return_dataframe: bool = False,\n        verbose: bool = False,\n    ):\n        super(Tab2Vec, self).__init__()\n\n        self._check_inputs(tab_preprocessor, model, verbose)\n\n        if isinstance(device, str):\n            self.device = torch.device(device)\n        else:\n            self.device = device\n\n        self.tab_preprocessor = tab_preprocessor\n        self.return_dataframe = return_dataframe\n        self.verbose = verbose\n\n        self.vectorizer = self._set_vectorizer(model)\n\n        self._set_dim_attributes(tab_preprocessor, model)\n\n    def fit(self, df: pd.DataFrame, target_col: Optional[str] = None) -&gt; \"Tab2Vec\":\n        r\"\"\"This is an empty method i.e. Returns the unchanged object itself. Is\n        only included for consistency in case `Tab2Vec` is used as part of a\n        Pipeline\n\n        Parameters\n        ----------\n        df: pd.DataFrame\n            DataFrame to be vectorised, i.e. the categorical and continuous\n            columns will be encoded based on the processing applied within\n            the model\n        target_col: str, Optional\n            Column name of the target_col variable. If `None` only the array of\n            predictors will be returned\n\n        Returns\n        -------\n        Tab2Vec\n        \"\"\"\n\n        return self\n\n    def transform(\n        self,\n        df: pd.DataFrame,\n        target_col: Optional[str] = None,\n    ) -&gt; Union[\n        np.ndarray,\n        Tuple[np.ndarray, np.ndarray],\n        pd.DataFrame,\n        Tuple[pd.DataFrame, pd.Series],\n    ]:\n        r\"\"\"Transforms the input dataframe into vectorized form. If a target\n        column name is passed the target values will be returned separately\n        in their corresponding type (np.ndarray or pd.DataFrame)\n\n        Parameters\n        ----------\n        df: pd.DataFrame\n            DataFrame to be vectorised, i.e. the categorical and continuous\n            columns will be encoded based on the processing applied within\n            the model\n        target_col: str, Optional\n            Column name of the target_col variable. If `None` only the array of\n            predictors will be returned\n\n        Returns\n        -------\n        Union[np.ndarray, Tuple[np.ndarray, np.ndarray], pd.DataFrame, Tuple[pd.DataFrame, pd.Series]\n            Returns eiter a numpy array with the vectorised values, or a Tuple\n            of numpy arrays with the vectorised values and the target. The\n            same applies to dataframes in case we choose to set\n            `return_dataframe = True`\n        \"\"\"\n\n        X_tab = self.tab_preprocessor.transform(df)\n\n        if self.device.type == \"mps\":\n            X = torch.from_numpy(X_tab.astype(\"float32\")).to(self.device)\n        else:\n            X = torch.from_numpy(X_tab.astype(\"float\")).to(self.device)\n\n        with torch.no_grad():\n            if self.is_tab_transformer:\n                x_vec, x_cont_not_embed = self.vectorizer(X)\n            else:\n                x_vec = self.vectorizer(X)\n                x_cont_not_embed = None\n\n        if self.tab_preprocessor.with_cls_token:\n            x_vec = x_vec[:, 1:, :]\n\n        if self.tab_preprocessor.with_attention:\n            x_vec = einops.rearrange(x_vec, \"s c e -&gt; s (c e)\")\n\n        if x_cont_not_embed is not None:\n            x_vec = torch.cat([x_vec, x_cont_not_embed], 1).detach().cpu().numpy()\n        else:\n            x_vec = x_vec.detach().cpu().numpy()\n\n        if self.return_dataframe:\n            new_colnames = self._new_colnames()\n            if target_col:\n                return pd.DataFrame(data=x_vec, columns=new_colnames), df[[target_col]]\n            else:\n                return pd.DataFrame(data=x_vec, columns=new_colnames)\n        else:\n            if target_col:\n                return x_vec, df[target_col].values\n            else:\n                return x_vec\n\n    def fit_transform(\n        self, df: pd.DataFrame, target_col: Optional[str] = None\n    ) -&gt; Union[\n        np.ndarray,\n        Tuple[np.ndarray, np.ndarray],\n        pd.DataFrame,\n        Tuple[pd.DataFrame, pd.Series],\n    ]:\n        r\"\"\"Combines `fit` and `transform`\"\"\"\n        return self.fit(df, target_col).transform(df, target_col)\n\n    def _new_colnames(self) -&gt; List[str]:\n        if self.tab_preprocessor.with_attention:\n            return self._new_colnames_with_attention()\n        else:\n            return self._new_colnames_without_attention()\n\n    def _new_colnames_with_attention(\n        self,\n    ) -&gt; List[str]:\n        cat_cols: List[str] = (\n            [\n                ei[0]\n                for ei in self.tab_preprocessor.cat_embed_input\n                if ei[0] != \"cls_token\"\n            ]\n            if self.tab_preprocessor.cat_embed_input is not None\n            else []\n        )\n\n        cont_cols: List[str] = (\n            self.tab_preprocessor.continuous_cols\n            if self.tab_preprocessor.continuous_cols is not None\n            else []\n        )\n\n        if self.are_cont_embed:\n            all_embed_cols = cat_cols + cont_cols\n        else:\n            all_embed_cols = cat_cols\n\n        new_colnames = []\n        for colname in all_embed_cols:\n            assert self.input_dim is not None\n            new_colnames.extend(\n                [\n                    \"_\".join([colname, \"embed\", str(i + 1)])\n                    for i in range(self.input_dim)\n                ]\n            )\n\n        if not self.are_cont_embed:\n            new_colnames += cont_cols\n\n        return new_colnames\n\n    def _new_colnames_without_attention(\n        self,\n    ) -&gt; List[str]:\n        new_colnames: List[str] = []\n\n        if self.tab_preprocessor.cat_embed_input is not None:\n            for colname, _, embedding_dim in self.tab_preprocessor.cat_embed_input:  # type: ignore[misc]\n                new_colnames.extend(\n                    [\n                        \"_\".join([colname, \"embed\", str(i + 1)])\n                        for i in range(embedding_dim)\n                    ]\n                )\n\n        if self.tab_preprocessor.continuous_cols is not None:\n            if self.are_cont_embed:\n                for colname in self.tab_preprocessor.continuous_cols:\n                    assert self.cont_embed_dim is not None\n                    new_colnames.extend(\n                        [\n                            \"_\".join([colname, \"embed\", str(i + 1)])\n                            for i in range(self.cont_embed_dim)\n                        ]\n                    )\n            else:\n                new_colnames += self.tab_preprocessor.continuous_cols\n\n        return new_colnames\n\n    def _check_inputs(\n        self,\n        tab_preprocessor: TabPreprocessor,\n        model: Union[WideDeep, BayesianWide, BayesianTabMlp],\n        verbose: bool,\n    ) -&gt; None:\n        if not isinstance(model, BaseBayesianModel):\n            if verbose:\n                if model.deepimage is not None or model.deeptext is not None:\n                    warnings.warn(\n                        \"Currently 'Tab2Vec' is only implemented for the 'deeptabular' component.\"\n                    )\n\n            if model.deeptabular is None:\n                raise RuntimeError(\n                    \"Currently 'Tab2Vec' is only implemented for the 'deeptabular' component.\"\n                )\n            if not tab_preprocessor.is_fitted:\n                raise RuntimeError(\n                    \"The 'tab_preprocessor' must be fitted before is passed to 'Tab2Vec'\"\n                )\n\n    def _set_vectorizer(\n        self, model: Union[WideDeep, BayesianWide, BayesianTabMlp]\n    ) -&gt; Callable:\n        if isinstance(model, BaseBayesianModel):\n            vectorizer = model._get_embeddings\n            self.is_tab_transformer = False\n        elif model.deeptabular[0].__class__.__name__.lower() == \"tabtransformer\":  # type: ignore\n            self.is_tab_transformer = True\n            vectorizer = model.deeptabular[0]._get_embeddings_tt  # type: ignore\n        else:\n            self.is_tab_transformer = False\n            vectorizer = model.deeptabular[0]._get_embeddings  # type: ignore\n        return vectorizer\n\n    def _set_dim_attributes(\n        self,\n        tab_preprocessor: TabPreprocessor,\n        model: Union[WideDeep, BayesianWide, BayesianTabMlp],\n    ) -&gt; None:\n        if isinstance(model, BaseBayesianModel):\n            self.are_cont_embed: bool = (\n                model.embed_continuous if model.embed_continuous else False\n            )\n            self.input_dim: Optional[int] = None\n            self.cont_embed_dim: Optional[int] = model.cont_embed_dim\n        else:\n            self.are_cont_embed = (\n                model.deeptabular[0].embed_continuous_method is not None  # type: ignore\n            )\n            if tab_preprocessor.with_attention:\n                self.input_dim = model.deeptabular[0].input_dim  # type: ignore\n                self.cont_embed_dim = None\n            else:\n                self.input_dim = None\n                self.cont_embed_dim = model.deeptabular[0].cont_embed_dim  # type: ignore\n</code></pre>"},{"location":"pytorch-widedeep/tab2vec.html#pytorch_widedeep.tab2vec.Tab2Vec.fit","title":"fit","text":"<pre><code>fit(df, target_col=None)\n</code></pre> <p>This is an empty method i.e. Returns the unchanged object itself. Is only included for consistency in case <code>Tab2Vec</code> is used as part of a Pipeline</p> <p>Parameters:</p> Name Type Description Default <code>df</code> <code>DataFrame</code> <p>DataFrame to be vectorised, i.e. the categorical and continuous columns will be encoded based on the processing applied within the model</p> required <code>target_col</code> <code>Optional[str]</code> <p>Column name of the target_col variable. If <code>None</code> only the array of predictors will be returned</p> <code>None</code> <p>Returns:</p> Type Description <code>Tab2Vec</code> Source code in <code>pytorch_widedeep/tab2vec.py</code> <pre><code>def fit(self, df: pd.DataFrame, target_col: Optional[str] = None) -&gt; \"Tab2Vec\":\n    r\"\"\"This is an empty method i.e. Returns the unchanged object itself. Is\n    only included for consistency in case `Tab2Vec` is used as part of a\n    Pipeline\n\n    Parameters\n    ----------\n    df: pd.DataFrame\n        DataFrame to be vectorised, i.e. the categorical and continuous\n        columns will be encoded based on the processing applied within\n        the model\n    target_col: str, Optional\n        Column name of the target_col variable. If `None` only the array of\n        predictors will be returned\n\n    Returns\n    -------\n    Tab2Vec\n    \"\"\"\n\n    return self\n</code></pre>"},{"location":"pytorch-widedeep/tab2vec.html#pytorch_widedeep.tab2vec.Tab2Vec.transform","title":"transform","text":"<pre><code>transform(df, target_col=None)\n</code></pre> <p>Transforms the input dataframe into vectorized form. If a target column name is passed the target values will be returned separately in their corresponding type (np.ndarray or pd.DataFrame)</p> <p>Parameters:</p> Name Type Description Default <code>df</code> <code>DataFrame</code> <p>DataFrame to be vectorised, i.e. the categorical and continuous columns will be encoded based on the processing applied within the model</p> required <code>target_col</code> <code>Optional[str]</code> <p>Column name of the target_col variable. If <code>None</code> only the array of predictors will be returned</p> <code>None</code> <p>Returns:</p> Type Description <code>Union[np.ndarray, Tuple[np.ndarray, np.ndarray], pd.DataFrame, Tuple[pd.DataFrame, pd.Series]</code> <p>Returns eiter a numpy array with the vectorised values, or a Tuple of numpy arrays with the vectorised values and the target. The same applies to dataframes in case we choose to set <code>return_dataframe = True</code></p> Source code in <code>pytorch_widedeep/tab2vec.py</code> <pre><code>def transform(\n    self,\n    df: pd.DataFrame,\n    target_col: Optional[str] = None,\n) -&gt; Union[\n    np.ndarray,\n    Tuple[np.ndarray, np.ndarray],\n    pd.DataFrame,\n    Tuple[pd.DataFrame, pd.Series],\n]:\n    r\"\"\"Transforms the input dataframe into vectorized form. If a target\n    column name is passed the target values will be returned separately\n    in their corresponding type (np.ndarray or pd.DataFrame)\n\n    Parameters\n    ----------\n    df: pd.DataFrame\n        DataFrame to be vectorised, i.e. the categorical and continuous\n        columns will be encoded based on the processing applied within\n        the model\n    target_col: str, Optional\n        Column name of the target_col variable. If `None` only the array of\n        predictors will be returned\n\n    Returns\n    -------\n    Union[np.ndarray, Tuple[np.ndarray, np.ndarray], pd.DataFrame, Tuple[pd.DataFrame, pd.Series]\n        Returns eiter a numpy array with the vectorised values, or a Tuple\n        of numpy arrays with the vectorised values and the target. The\n        same applies to dataframes in case we choose to set\n        `return_dataframe = True`\n    \"\"\"\n\n    X_tab = self.tab_preprocessor.transform(df)\n\n    if self.device.type == \"mps\":\n        X = torch.from_numpy(X_tab.astype(\"float32\")).to(self.device)\n    else:\n        X = torch.from_numpy(X_tab.astype(\"float\")).to(self.device)\n\n    with torch.no_grad():\n        if self.is_tab_transformer:\n            x_vec, x_cont_not_embed = self.vectorizer(X)\n        else:\n            x_vec = self.vectorizer(X)\n            x_cont_not_embed = None\n\n    if self.tab_preprocessor.with_cls_token:\n        x_vec = x_vec[:, 1:, :]\n\n    if self.tab_preprocessor.with_attention:\n        x_vec = einops.rearrange(x_vec, \"s c e -&gt; s (c e)\")\n\n    if x_cont_not_embed is not None:\n        x_vec = torch.cat([x_vec, x_cont_not_embed], 1).detach().cpu().numpy()\n    else:\n        x_vec = x_vec.detach().cpu().numpy()\n\n    if self.return_dataframe:\n        new_colnames = self._new_colnames()\n        if target_col:\n            return pd.DataFrame(data=x_vec, columns=new_colnames), df[[target_col]]\n        else:\n            return pd.DataFrame(data=x_vec, columns=new_colnames)\n    else:\n        if target_col:\n            return x_vec, df[target_col].values\n        else:\n            return x_vec\n</code></pre>"},{"location":"pytorch-widedeep/tab2vec.html#pytorch_widedeep.tab2vec.Tab2Vec.fit_transform","title":"fit_transform","text":"<pre><code>fit_transform(df, target_col=None)\n</code></pre> <p>Combines <code>fit</code> and <code>transform</code></p> Source code in <code>pytorch_widedeep/tab2vec.py</code> <pre><code>def fit_transform(\n    self, df: pd.DataFrame, target_col: Optional[str] = None\n) -&gt; Union[\n    np.ndarray,\n    Tuple[np.ndarray, np.ndarray],\n    pd.DataFrame,\n    Tuple[pd.DataFrame, pd.Series],\n]:\n    r\"\"\"Combines `fit` and `transform`\"\"\"\n    return self.fit(df, target_col).transform(df, target_col)\n</code></pre>"},{"location":"pytorch-widedeep/the_rec_module.html","title":"The <code>rec</code> module","text":"<p>This module contains models are that specifically designed for recommendation systems. While the rest of the models can be accessed from the <code>pytorch_widedeep.models</code> module, models in this module need to be specifically imported from the <code>rec</code> module, e.g.:</p> <pre><code>from pytorch_widedeep.models.rec import DeepFactorizationMachine\n</code></pre> <p>The list of models here is not meant to be exhaustive, but it includes some common architectures such as factorization machines, field aware factorization machines or extreme factorization machines. More models will be added in the future.</p>"},{"location":"pytorch-widedeep/the_rec_module.html#pytorch_widedeep.models.rec.deepfm.DeepFactorizationMachine","title":"DeepFactorizationMachine","text":"<p>               Bases: <code>BaseTabularModelWithAttention</code></p> <p>Deep Factorization Machine (DeepFM) for recommendation systems, which is an adaptation of 'Factorization Machines' by Steffen Rendle. Presented in 'DeepFM: A Factorization-Machine based Neural Network for CTR Prediction' by Huifeng Guo, Ruiming Tang, Yunming Yey, Zhenguo Li, Xiuqiang He. 2017.</p> <p>The implementation in this library takes advantage of all the functionalities available to encode categorical and continuous features. The model can be used with only the factorization machine</p> <p>Note that this class implements only the 'Deep' component of the model described in the paper. The linear component is not implemented 'internally' and, if one wants to include it, it can be easily added using the 'wide' (aka linear) component available in this library. See the examples in the examples folder.</p> <p>Parameters:</p> Name Type Description Default <code>column_idx</code> <code>Dict[str, int]</code> <p>Dictionary mapping column names to their corresponding index.</p> required <code>num_factors</code> <code>int</code> <p>Number of factors for the factorization machine.</p> required <code>reduce_sum</code> <code>bool</code> <p>Whether to reduce the sum in the factorization machine output.</p> <code>True</code> <code>cat_embed_input</code> <code>Optional[List[Tuple[str, int]]]</code> <p>List of tuples with categorical column names and number of unique values.</p> <code>None</code> <code>cat_embed_dropout</code> <code>Optional[float]</code> <p>Categorical embeddings dropout. If <code>None</code>, it will default to 0.</p> <code>None</code> <code>use_cat_bias</code> <code>Optional[bool]</code> <p>Boolean indicating if bias will be used for the categorical embeddings. If <code>None</code>, it will default to 'False'.</p> <code>None</code> <code>cat_embed_activation</code> <code>Optional[str]</code> <p>Activation function for the categorical embeddings, if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> <code>None</code> <code>continuous_cols</code> <code>Optional[List[str]]</code> <p>List with the name of the numeric (aka continuous) columns</p> <code>None</code> <code>cont_norm_layer</code> <code>Optional[Literal['batchnorm', 'layernorm']]</code> <p>Type of normalization layer applied to the continuous features. Options are: 'layernorm' and 'batchnorm'. if <code>None</code>, no normalization layer will be used.</p> <code>None</code> <code>embed_continuous_method</code> <code>Optional[Literal['piecewise', 'periodic', 'standard']]</code> <p>Method to use to embed the continuous features. Options are: 'standard', 'periodic' or 'piecewise'. The 'standard' embedding method is based on the FT-Transformer implementation presented in the paper: Revisiting Deep Learning Models for Tabular Data. The 'periodic' and_'piecewise'_ methods were presented in the paper: On Embeddings for Numerical Features in Tabular Deep Learning. Please, read the papers for details.</p> <code>'standard'</code> <code>cont_embed_dropout</code> <code>Optional[float]</code> <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p> <code>None</code> <code>cont_embed_activation</code> <code>Optional[str]</code> <p>Activation function for the continuous embeddings if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported. If <code>None</code>, no activation function will be applied.</p> <code>None</code> <code>quantization_setup</code> <code>Optional[Dict[str, List[float]]]</code> <p>This parameter is used when the 'piecewise' method is used to embed the continuous cols. It is a dict where keys are the name of the continuous columns and values are lists with the boundaries for the quantization of the continuous_cols. See the examples for details. If If the 'piecewise' method is used, this parameter is required.</p> <code>None</code> <code>n_frequencies</code> <code>Optional[int]</code> <p>This is the so called 'k' in their paper On Embeddings for Numerical Features in Tabular Deep Learning, and is the number of 'frequencies' that will be used to represent each continuous column. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>sigma</code> <code>Optional[float]</code> <p>This is the sigma parameter in the paper mentioned when describing the previous parameters and it is used to initialise the 'frequency weights'. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>share_last_layer</code> <code>Optional[bool]</code> <p>This parameter is not present in the before mentioned paper but it is implemented in the official repo. If <code>True</code> the linear layer that turns the frequencies into embeddings will be shared across the continuous columns. If <code>False</code> a different linear layer will be used for each continuous column. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>full_embed_dropout</code> <code>Optional[bool]</code> <p>If <code>True</code>, the full embedding corresponding to a column will be masked out/dropout. If <code>None</code>, it will default to <code>False</code>.</p> <code>None</code> <code>mlp_hidden_dims</code> <code>Optional[List[int]]</code> <p>List with the number of neurons per dense layer in the mlp.</p> <code>None</code> <code>mlp_activation</code> <code>Optional[str]</code> <p>Activation function for the dense layers of the MLP. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> <code>None</code> <code>mlp_dropout</code> <code>Optional[float]</code> <p>float or List of floats with the dropout between the dense layers. e.g: [0.5,0.5]</p> <code>None</code> <code>mlp_batchnorm</code> <code>Optional[bool]</code> <p>Boolean indicating whether or not batch normalization will be applied to the dense layers</p> <code>None</code> <code>mlp_batchnorm_last</code> <code>Optional[bool]</code> <p>Boolean indicating whether or not batch normalization will be applied to the last of the dense layers</p> <code>None</code> <code>mlp_linear_first</code> <code>Optional[bool]</code> <p>Boolean indicating the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code></p> <code>None</code> <p>Attributes:</p> Name Type Description <code>mlp</code> <code>MLP</code> <p>MLP component of the model if the mlp_hidden_dims parameter is not None. If None, the model will only return the output of the factorization machine.</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; from typing import Dict, List, Tuple\n&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from torch import Tensor\n&gt;&gt;&gt; from pytorch_widedeep.models.rec import DeepFactorizationMachine\n&gt;&gt;&gt; X = torch.randint(0, 10, (16, 2))\n&gt;&gt;&gt; column_idx: Dict[str, int] = {\"col1\": 0, \"col2\": 1}\n&gt;&gt;&gt; cat_embed_input: List[Tuple[str, int]] = [(\"col1\", 10), (\"col2\", 10)]\n&gt;&gt;&gt; fm = DeepFactorizationMachine(\n...     column_idx=column_idx,\n...     num_factors=8,\n...     cat_embed_input=cat_embed_input,\n...     mlp_hidden_dims=[16, 8]\n... )\n&gt;&gt;&gt; out = fm(X)\n</code></pre> Source code in <code>pytorch_widedeep/models/rec/deepfm.py</code> <pre><code>class DeepFactorizationMachine(BaseTabularModelWithAttention):\n    \"\"\"\n    Deep Factorization Machine (DeepFM) for recommendation systems, which is\n    an adaptation of 'Factorization Machines' by Steffen Rendle. Presented\n    in 'DeepFM: A Factorization-Machine based Neural Network for CTR\n    Prediction' by Huifeng Guo, Ruiming Tang, Yunming Yey, Zhenguo Li,\n    Xiuqiang He. 2017.\n\n    The implementation in this library takes advantage of all the\n    functionalities available to encode categorical and continuous features.\n    The model can be used with only the factorization machine\n\n    Note that this class implements only the 'Deep' component of the model\n    described in the paper. The linear component is not\n    implemented 'internally' and, if one wants to include it, it can be\n    easily added using the 'wide' (aka linear) component available in this\n    library. See the examples in the examples folder.\n\n    Parameters\n    ----------\n    column_idx : Dict[str, int]\n        Dictionary mapping column names to their corresponding index.\n    num_factors : int\n        Number of factors for the factorization machine.\n    reduce_sum : bool, default=True\n        Whether to reduce the sum in the factorization machine output.\n    cat_embed_input : Optional[List[Tuple[str, int]]], default=None\n        List of tuples with categorical column names and number of unique values.\n    cat_embed_dropout : Optional[float], default=None\n        Categorical embeddings dropout. If `None`, it will default\n        to 0.\n    use_cat_bias : Optional[bool], default=None\n        Boolean indicating if bias will be used for the categorical embeddings.\n        If `None`, it will default to 'False'.\n    cat_embed_activation : Optional[str], default=None\n        Activation function for the categorical embeddings, if any. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported\n    continuous_cols : Optional[List[str]], default=None\n        List with the name of the numeric (aka continuous) columns\n    cont_norm_layer : Optional[Literal[\"batchnorm\", \"layernorm\"]], default=None\n        Type of normalization layer applied to the continuous features.\n        Options are: _'layernorm'_ and _'batchnorm'_. if `None`, no\n        normalization layer will be used.\n    embed_continuous_method: Optional[Literal[\"piecewise\", \"periodic\", \"standard\"]], default=\"standard\"\n        Method to use to embed the continuous features. Options are:\n        _'standard'_, _'periodic'_ or _'piecewise'_. The _'standard'_\n        embedding method is based on the FT-Transformer implementation\n        presented in the paper: [Revisiting Deep Learning Models for\n        Tabular Data](https://arxiv.org/abs/2106.11959v5). The _'periodic'_\n        and_'piecewise'_ methods were presented in the paper: [On Embeddings for\n        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556).\n        Please, read the papers for details.\n    cont_embed_dropout : Optional[float], default=None\n        Dropout for the continuous embeddings. If `None`, it will default to 0.0\n    cont_embed_activation : Optional[str], default=None\n        Activation function for the continuous embeddings if any. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported.\n        If `None`, no activation function will be applied.\n    quantization_setup : Optional[Dict[str, List[float]]], default=None\n        This parameter is used when the _'piecewise'_ method is used to embed\n        the continuous cols. It is a dict where keys are the name of the continuous\n        columns and values are lists with the boundaries for the quantization\n        of the continuous_cols. See the examples for details. If\n        If the _'piecewise'_ method is used, this parameter is required.\n    n_frequencies : Optional[int], default=None\n        This is the so called _'k'_ in their paper [On Embeddings for\n        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556),\n        and is the number of 'frequencies' that will be used to represent each\n        continuous column. See their Eq 2 in the paper for details. If\n        the _'periodic'_ method is used, this parameter is required.\n    sigma : Optional[float], default=None\n        This is the sigma parameter in the paper mentioned when describing the\n        previous parameters and it is used to initialise the 'frequency\n        weights'. See their Eq 2 in the paper for details. If\n        the _'periodic'_ method is used, this parameter is required.\n    share_last_layer : Optional[bool], default=None\n        This parameter is not present in the before mentioned paper but it is implemented in\n        the [official repo](https://github.com/yandex-research/rtdl-num-embeddings/tree/main).\n        If `True` the linear layer that turns the frequencies into embeddings\n        will be shared across the continuous columns. If `False` a different\n        linear layer will be used for each continuous column.\n        If the _'periodic'_ method is used, this parameter is required.\n    full_embed_dropout: bool, Optional, default = None,\n        If `True`, the full embedding corresponding to a column will be masked\n        out/dropout. If `None`, it will default to `False`.\n    mlp_hidden_dims: List, default = [200, 100]\n        List with the number of neurons per dense layer in the mlp.\n    mlp_activation: str, default = \"relu\"\n        Activation function for the dense layers of the MLP. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported\n    mlp_dropout: float or List, default = 0.1\n        float or List of floats with the dropout between the dense layers.\n        e.g: _[0.5,0.5]_\n    mlp_batchnorm: bool, default = False\n        Boolean indicating whether or not batch normalization will be applied\n        to the dense layers\n    mlp_batchnorm_last: bool, default = False\n        Boolean indicating whether or not batch normalization will be applied\n        to the last of the dense layers\n    mlp_linear_first: bool, default = False\n        Boolean indicating the order of the operations in the dense\n        layer. If `True: [LIN -&gt; ACT -&gt; BN -&gt; DP]`. If `False: [BN -&gt; DP -&gt;\n        LIN -&gt; ACT]`\n\n    Attributes\n    ----------\n    mlp: MLP\n        MLP component of the model if the mlp_hidden_dims parameter is not\n        None. If None, the model will only return the output of the\n        factorization machine.\n\n    Examples\n    --------\n    &gt;&gt;&gt; from typing import Dict, List, Tuple\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from torch import Tensor\n    &gt;&gt;&gt; from pytorch_widedeep.models.rec import DeepFactorizationMachine\n    &gt;&gt;&gt; X = torch.randint(0, 10, (16, 2))\n    &gt;&gt;&gt; column_idx: Dict[str, int] = {\"col1\": 0, \"col2\": 1}\n    &gt;&gt;&gt; cat_embed_input: List[Tuple[str, int]] = [(\"col1\", 10), (\"col2\", 10)]\n    &gt;&gt;&gt; fm = DeepFactorizationMachine(\n    ...     column_idx=column_idx,\n    ...     num_factors=8,\n    ...     cat_embed_input=cat_embed_input,\n    ...     mlp_hidden_dims=[16, 8]\n    ... )\n    &gt;&gt;&gt; out = fm(X)\n    \"\"\"\n\n    def __init__(\n        self,\n        *,\n        column_idx: Dict[str, int],\n        num_factors: int,\n        reduce_sum: bool = True,\n        cat_embed_input: Optional[List[Tuple[str, int]]] = None,\n        cat_embed_dropout: Optional[float] = None,\n        use_cat_bias: Optional[bool] = None,\n        cat_embed_activation: Optional[str] = None,\n        continuous_cols: Optional[List[str]] = None,\n        cont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\n        embed_continuous_method: Optional[\n            Literal[\"piecewise\", \"periodic\", \"standard\"]\n        ] = \"standard\",\n        cont_embed_dropout: Optional[float] = None,\n        cont_embed_activation: Optional[str] = None,\n        quantization_setup: Optional[Dict[str, List[float]]] = None,\n        n_frequencies: Optional[int] = None,\n        sigma: Optional[float] = None,\n        share_last_layer: Optional[bool] = None,\n        full_embed_dropout: Optional[bool] = None,\n        mlp_hidden_dims: Optional[List[int]] = None,\n        mlp_activation: Optional[str] = None,\n        mlp_dropout: Optional[float] = None,\n        mlp_batchnorm: Optional[bool] = None,\n        mlp_batchnorm_last: Optional[bool] = None,\n        mlp_linear_first: Optional[bool] = None,\n    ):\n        super(DeepFactorizationMachine, self).__init__(\n            column_idx=column_idx,\n            input_dim=num_factors,\n            cat_embed_input=cat_embed_input,\n            cat_embed_dropout=cat_embed_dropout,\n            use_cat_bias=use_cat_bias,\n            cat_embed_activation=cat_embed_activation,\n            shared_embed=False,\n            add_shared_embed=None,\n            frac_shared_embed=None,\n            continuous_cols=continuous_cols,\n            cont_norm_layer=cont_norm_layer,\n            embed_continuous_method=embed_continuous_method,\n            cont_embed_dropout=cont_embed_dropout,\n            cont_embed_activation=cont_embed_activation,\n            quantization_setup=quantization_setup,\n            n_frequencies=n_frequencies,\n            sigma=sigma,\n            share_last_layer=share_last_layer,\n            full_embed_dropout=full_embed_dropout,\n        )\n\n        self.reduce_sum = reduce_sum\n\n        self.mlp_hidden_dims = mlp_hidden_dims\n        self.mlp_activation = mlp_activation\n        self.mlp_dropout = mlp_dropout\n        self.mlp_batchnorm = mlp_batchnorm\n        self.mlp_batchnorm_last = mlp_batchnorm_last\n        self.mlp_linear_first = mlp_linear_first\n\n        if self.mlp_hidden_dims is not None:\n\n            if self.mlp_hidden_dims[-1] != self.input_dim:\n                d_hidden = (\n                    [self.input_dim * len(self.column_idx)]\n                    + self.mlp_hidden_dims\n                    + [self.input_dim]\n                )\n            else:\n                d_hidden = [\n                    self.input_dim * len(self.column_idx)\n                ] + self.mlp_hidden_dims\n\n            self.mlp = MLP(\n                d_hidden=d_hidden,\n                activation=(\n                    \"relu\" if self.mlp_activation is None else self.mlp_activation\n                ),\n                dropout=0.0 if self.mlp_dropout is None else self.mlp_dropout,\n                batchnorm=False if self.mlp_batchnorm is None else self.mlp_batchnorm,\n                batchnorm_last=(\n                    False\n                    if self.mlp_batchnorm_last is None\n                    else self.mlp_batchnorm_last\n                ),\n                linear_first=(\n                    False if self.mlp_linear_first is None else self.mlp_linear_first\n                ),\n            )\n        else:\n            self.mlp = None\n\n    def forward(self, X: Tensor) -&gt; Tensor:\n        embed = self._get_embeddings(X)\n        fm_output = factorization_machine(embed, self.reduce_sum)\n\n        if self.mlp is None:\n            return fm_output\n\n        mlp_input = embed.view(embed.size(0), -1)\n        mlp_output = self.mlp(mlp_input)\n\n        if self.reduce_sum:\n            mlp_output = mlp_output.sum(1, keepdim=True)\n\n        return fm_output + mlp_output\n\n    @property\n    def output_dim(self) -&gt; int:\n        if self.reduce_sum:\n            return 1\n        else:\n            return self.input_dim\n</code></pre>"},{"location":"pytorch-widedeep/the_rec_module.html#pytorch_widedeep.models.rec.deepffm.DeepFieldAwareFactorizationMachine","title":"DeepFieldAwareFactorizationMachine","text":"<p>               Bases: <code>BaseTabularModelWithAttention</code></p> <p>Deep Field Aware Factorization Machine (DeepFFM) for recommendation systems. Adaptation of the paper 'Field-aware Factorization Machines in a Real-world Online Advertising System', Juan et al. 2017.</p> <p>This class implements only the 'Deep' component of the model described in the paper. The linear component is not implemented 'internally' and, if one wants to include it, it can be easily added using the 'wide'/linear component in this library. See the examples in the examples folder.</p> <p>Note that in this case, only categorical features are accepted. This is because the embeddings of each feature will be learned using all other features. Therefore these embeddings have to be all of the same nature. This does not occur if we mix categorical and continuous features.</p> <p>Parameters:</p> Name Type Description Default <code>column_idx</code> <code>Dict[str, int]</code> <p>Dictionary mapping column names to their corresponding index.</p> required <code>num_factors</code> <code>int</code> <p>Number of factors for the factorization machine.</p> required <code>reduce_sum</code> <code>bool</code> <p>Whether to reduce the sum in the factorization machine output.</p> <code>True</code> <code>cat_embed_input</code> <code>Optional[List[Tuple[str, int]]]</code> <p>List of tuples with categorical column names and number of unique values.</p> <code>None</code> <code>cat_embed_dropout</code> <code>Optional[float]</code> <p>Categorical embeddings dropout. If <code>None</code>, it will default to 0.</p> <code>None</code> <code>use_cat_bias</code> <code>Optional[bool]</code> <p>Boolean indicating if bias will be used for the categorical embeddings. If <code>None</code>, it will default to 'False'.</p> <code>None</code> <code>cat_embed_activation</code> <code>Optional[str]</code> <p>Activation function for the categorical embeddings, if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> <code>None</code> <code>mlp_hidden_dims</code> <code>Optional[List[int]]</code> <p>List with the number of neurons per dense layer in the mlp.</p> <code>None</code> <code>mlp_activation</code> <code>Optional[str]</code> <p>Activation function for the dense layers of the MLP. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> <code>None</code> <code>mlp_dropout</code> <code>Optional[float]</code> <p>float or List of floats with the dropout between the dense layers. e.g: [0.5,0.5]</p> <code>None</code> <code>mlp_batchnorm</code> <code>Optional[bool]</code> <p>Boolean indicating whether or not batch normalization will be applied to the dense layers</p> <code>None</code> <code>mlp_batchnorm_last</code> <code>Optional[bool]</code> <p>Boolean indicating whether or not batch normalization will be applied to the last of the dense layers</p> <code>None</code> <code>mlp_linear_first</code> <code>Optional[bool]</code> <p>Boolean indicating the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code></p> <code>None</code> <p>Attributes:</p> Name Type Description <code>n_features</code> <code>int</code> <p>Number of unique features/columns</p> <code>n_tokens</code> <code>int</code> <p>Number of unique values (tokens) in the full dataset (corpus)</p> <code>encoders</code> <code>ModuleList</code> <p>List of <code>BaseTabularModelWithAttention</code> instances. One per categorical column</p> <code>mlp</code> <code>Module</code> <p>Multi-layer perceptron. If <code>None</code> the output will be the output of the factorization machine (i.e. the sum of the interactions)</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from torch import Tensor\n&gt;&gt;&gt; from typing import Dict, List, Tuple\n&gt;&gt;&gt; from pytorch_widedeep.models.rec import DeepFieldAwareFactorizationMachine\n&gt;&gt;&gt; X = torch.randint(0, 10, (16, 2))\n&gt;&gt;&gt; column_idx: Dict[str, int] = {\"col1\": 0, \"col2\": 1}\n&gt;&gt;&gt; cat_embed_input: List[Tuple[str, int]] = [(\"col1\", 10), (\"col2\", 10)]\n&gt;&gt;&gt; ffm = DeepFieldAwareFactorizationMachine(\n...     column_idx=column_idx,\n...     num_factors=4,\n...     cat_embed_input=cat_embed_input,\n...     mlp_hidden_dims=[16, 8]\n... )\n&gt;&gt;&gt; output = ffm(X)\n</code></pre> Source code in <code>pytorch_widedeep/models/rec/deepffm.py</code> <pre><code>class DeepFieldAwareFactorizationMachine(BaseTabularModelWithAttention):\n    \"\"\"\n    Deep Field Aware Factorization Machine (DeepFFM) for recommendation\n    systems. Adaptation of the paper 'Field-aware Factorization Machines in a\n    Real-world Online Advertising System', Juan et al. 2017.\n\n    This class implements only the 'Deep' component of the model described in\n    the paper. The linear component is not implemented 'internally' and, if\n    one wants to include it, it can be easily added using the 'wide'/linear\n    component in this library. See the examples in the examples folder.\n\n    Note that in this case, only categorical features are accepted. This is\n    because the embeddings of each feature will be learned using all other\n    features. Therefore these embeddings have to be all of the same nature.\n    This does not occur if we mix categorical and continuous features.\n\n    Parameters\n    ----------\n    column_idx : Dict[str, int]\n        Dictionary mapping column names to their corresponding index.\n    num_factors : int\n        Number of factors for the factorization machine.\n    reduce_sum : bool, default=True\n        Whether to reduce the sum in the factorization machine output.\n    cat_embed_input : Optional[List[Tuple[str, int]]], default=None\n        List of tuples with categorical column names and number of unique values.\n    cat_embed_dropout : Optional[float], default=None\n        Categorical embeddings dropout. If `None`, it will default\n        to 0.\n    use_cat_bias : Optional[bool], default=None\n        Boolean indicating if bias will be used for the categorical embeddings.\n        If `None`, it will default to 'False'.\n    cat_embed_activation : Optional[str], default=None\n        Activation function for the categorical embeddings, if any. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported\n    mlp_hidden_dims: List, default = [200, 100]\n        List with the number of neurons per dense layer in the mlp.\n    mlp_activation: str, default = \"relu\"\n        Activation function for the dense layers of the MLP. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported\n    mlp_dropout: float or List, default = 0.1\n        float or List of floats with the dropout between the dense layers.\n        e.g: _[0.5,0.5]_\n    mlp_batchnorm: bool, default = False\n        Boolean indicating whether or not batch normalization will be applied\n        to the dense layers\n    mlp_batchnorm_last: bool, default = False\n        Boolean indicating whether or not batch normalization will be applied\n        to the last of the dense layers\n    mlp_linear_first: bool, default = False\n        Boolean indicating the order of the operations in the dense\n        layer. If `True: [LIN -&gt; ACT -&gt; BN -&gt; DP]`. If `False: [BN -&gt; DP -&gt;\n        LIN -&gt; ACT]`\n\n    Attributes\n    ----------\n    n_features: int\n        Number of unique features/columns\n    n_tokens: int\n        Number of unique values (tokens) in the full dataset (corpus)\n    encoders: nn.ModuleList\n        List of `BaseTabularModelWithAttention` instances. One per categorical\n        column\n    mlp: nn.Module\n        Multi-layer perceptron. If `None` the output will be the output of the\n        factorization machine (i.e. the sum of the interactions)\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from torch import Tensor\n    &gt;&gt;&gt; from typing import Dict, List, Tuple\n    &gt;&gt;&gt; from pytorch_widedeep.models.rec import DeepFieldAwareFactorizationMachine\n    &gt;&gt;&gt; X = torch.randint(0, 10, (16, 2))\n    &gt;&gt;&gt; column_idx: Dict[str, int] = {\"col1\": 0, \"col2\": 1}\n    &gt;&gt;&gt; cat_embed_input: List[Tuple[str, int]] = [(\"col1\", 10), (\"col2\", 10)]\n    &gt;&gt;&gt; ffm = DeepFieldAwareFactorizationMachine(\n    ...     column_idx=column_idx,\n    ...     num_factors=4,\n    ...     cat_embed_input=cat_embed_input,\n    ...     mlp_hidden_dims=[16, 8]\n    ... )\n    &gt;&gt;&gt; output = ffm(X)\n    \"\"\"\n\n    def __init__(\n        self,\n        *,\n        column_idx: Dict[str, int],\n        num_factors: int,\n        cat_embed_input: List[Tuple[str, int]],\n        reduce_sum: bool = True,\n        cat_embed_dropout: Optional[float] = None,\n        use_cat_bias: Optional[bool] = None,\n        cat_embed_activation: Optional[str] = None,\n        mlp_hidden_dims: Optional[List[int]] = None,\n        mlp_activation: Optional[str] = None,\n        mlp_dropout: Optional[float] = None,\n        mlp_batchnorm: Optional[bool] = None,\n        mlp_batchnorm_last: Optional[bool] = None,\n        mlp_linear_first: Optional[bool] = None,\n    ):\n        super(DeepFieldAwareFactorizationMachine, self).__init__(\n            column_idx=column_idx,\n            input_dim=num_factors,\n            cat_embed_input=cat_embed_input,\n            cat_embed_dropout=cat_embed_dropout,\n            use_cat_bias=use_cat_bias,\n            cat_embed_activation=cat_embed_activation,\n            shared_embed=False,\n            add_shared_embed=None,\n            frac_shared_embed=None,\n            continuous_cols=None,\n            cont_norm_layer=None,\n            embed_continuous_method=None,\n            cont_embed_dropout=None,\n            cont_embed_activation=None,\n            quantization_setup=None,\n            n_frequencies=None,\n            sigma=None,\n            share_last_layer=None,\n            full_embed_dropout=None,\n        )\n\n        self.reduce_sum = reduce_sum\n\n        self.mlp_hidden_dims = mlp_hidden_dims\n        self.mlp_activation = mlp_activation\n        self.mlp_dropout = mlp_dropout\n        self.mlp_batchnorm = mlp_batchnorm\n        self.mlp_batchnorm_last = mlp_batchnorm_last\n        self.mlp_linear_first = mlp_linear_first\n\n        self.n_features = len(self.column_idx)\n        self.n_tokens = sum([ei[1] for ei in cat_embed_input])\n\n        self.encoders = nn.ModuleList(\n            [\n                BaseTabularModelWithAttention(**config)\n                for config in self._get_encoder_configs()\n            ]\n        )\n\n        if self.mlp_hidden_dims is not None:\n            d_hidden = [\n                self.n_features * (self.n_features - 1) // 2 * num_factors\n            ] + self.mlp_hidden_dims\n            self.mlp = MLP(\n                d_hidden=d_hidden,\n                activation=(\n                    \"relu\" if self.mlp_activation is None else self.mlp_activation\n                ),\n                dropout=0.0 if self.mlp_dropout is None else self.mlp_dropout,\n                batchnorm=False if self.mlp_batchnorm is None else self.mlp_batchnorm,\n                batchnorm_last=(\n                    False\n                    if self.mlp_batchnorm_last is None\n                    else self.mlp_batchnorm_last\n                ),\n                linear_first=(\n                    False if self.mlp_linear_first is None else self.mlp_linear_first\n                ),\n            )\n        else:\n            self.mlp = None\n\n    def forward(self, X: Tensor) -&gt; Tensor:\n\n        interactions_l: List[Tensor] = []\n        for i in range(len(self.column_idx)):\n            for j in range(i + 1, len(self.column_idx)):\n                # the syntax [i] and [j] is to keep the shape of the tensors\n                # as they are sliced within '_get_embeddings'. This will\n                # return a tensor of shape (b, 1, embed_dim). Then it has to\n                # be squeezed to (b, embed_dim)  before multiplied\n                embed_i = self.encoders[i]._get_embeddings(X[:, [i]]).squeeze(1)\n                embed_j = self.encoders[j]._get_embeddings(X[:, [j]]).squeeze(1)\n                interactions_l.append(embed_i * embed_j)\n\n        interactions = torch.cat(interactions_l, dim=1)\n\n        if self.mlp is not None:\n            interactions = interactions.view(X.size(0), -1)\n            deep_out = self.mlp(interactions)\n        else:\n            deep_out = interactions\n\n        if self.reduce_sum:\n            deep_out = deep_out.sum(dim=1, keepdim=True)\n\n        return deep_out\n\n    def _get_encoder_configs(self) -&gt; List[Dict[str, Any]]:\n        config: List[Dict[str, Any]] = []\n        for col, _ in self.column_idx.items():\n            cat_embed_input = [(col, self.n_tokens)]\n            _config = {\n                \"column_idx\": {col: 0},\n                \"input_dim\": self.input_dim,\n                \"cat_embed_input\": cat_embed_input,\n                \"cat_embed_dropout\": self.cat_embed_dropout,\n                \"use_cat_bias\": self.use_cat_bias,\n                \"cat_embed_activation\": self.cat_embed_activation,\n                \"shared_embed\": None,\n                \"add_shared_embed\": None,\n                \"frac_shared_embed\": None,\n                \"continuous_cols\": None,\n                \"cont_norm_layer\": None,\n                \"embed_continuous_method\": None,\n                \"cont_embed_dropout\": None,\n                \"cont_embed_activation\": None,\n                \"quantization_setup\": None,\n                \"n_frequencies\": None,\n                \"sigma\": None,\n                \"share_last_layer\": None,\n                \"full_embed_dropout\": None,\n            }\n\n            config.append(_config)\n\n        return config\n\n    @property\n    def output_dim(self) -&gt; int:\n        if self.reduce_sum:\n            return 1\n        elif self.mlp_hidden_dims is not None:\n            return self.mlp_hidden_dims[-1]\n        else:\n            return self.n_features * (self.n_features - 1) // 2 * self.input_dim\n</code></pre>"},{"location":"pytorch-widedeep/the_rec_module.html#pytorch_widedeep.models.rec.din.DeepInterestNetwork","title":"DeepInterestNetwork","text":"<p>               Bases: <code>BaseWDModelComponent</code></p> <p>Adaptation of the Deep Interest Network (DIN) for recommendation systems as described in the paper: 'Deep Interest Network for Click-Through Rate Prediction' by Guorui Zhou et al. 2018.</p> <p>Note that all the categorical- and continuous-related parameters refer to the categorical and continuous columns that are not part of the sequential columns and will be treated as standard tabular data.</p> <p>This model requires some specific data preparation that allows for quite a lot of flexibility. Therefore, I have included a preprocessor (<code>DINPreprocessor</code>) in the preprocessing module that will take care of the data preparation.</p> <p>Parameters:</p> Name Type Description Default <code>column_idx</code> <code>Dict[str, int]</code> <p>Dictionary mapping column names to their corresponding index.</p> required <code>target_item_col</code> <code>str</code> <p>Name of the target item column. Note that this is not the target column. This algorithm relies on a sequence representation of interactions. The target item would be the next item in the sequence of interactions (e.g. item 6<sup>th</sup> in a sequence of 5 items), and our goal is to predict a given action on it.</p> <code>'target_item'</code> <code>user_behavior_confiq</code> <code>Tuple[List[str], int, int]</code> <p>Configuration for user behavior sequence columns. Tuple containing: - List of column names that correspond to the user behavior sequence - Number of unique feature values (n_tokens) - Embedding dimension Example: <code>([\"item_1\", \"item_2\", \"item_3\"], 5, 8)</code></p> required <code>action_seq_config</code> <code>Optional[Tuple[List[str], int]]</code> <p>Configuration for a so-called action sequence columns (for example a rating, or purchased/not-purchased, etc). Tuple containing: - List of column names - Number of unique feature values (n_tokens) This action will always be learned as a 1d embedding and will be combined with the user behaviour. For example, imagine that the action is purchased/not-purchased. then per item in the user behaviour sequence there will be a binary action to learn 0/1. Such action will be represented by a float number that will multiply the corresponding item embedding in the user behaviour sequence. Example: <code>([\"rating_1\", \"rating_2\", \"rating_3\"], 5)</code> Internally, the embedding dimension will be set to 1</p> <code>None</code> <code>other_seq_cols_confiq</code> <code>Optional[List[Tuple[List[str], int, int]]]</code> <p>Configuration for other sequential columns. List of tuples containing: - List of column names that correspond to the sequential column - Number of unique feature values (n_tokens) - Embedding dimension Example: <code>[([\"seq1_col1\", \"seq1_col2\"], 5, 8), ([\"seq2_col1\", \"seq2_col2\"], 5, 8)]</code></p> <code>None</code> <code>attention_unit_activation</code> <code>Literal['prelu', 'dice']</code> <p>Activation function to use in the attention unit.</p> <code>\"prelu\"</code> <code>cat_embed_input</code> <code>Optional[List[Tuple[str, int, int]]]</code> <p>Configuration for other columns. List of tuples containing: - Column name - Number of unique feature values (n_tokens) - Embedding dimension</p> <p>Note: From here in advance the remaining parameters are related to the categorical and continuous columns that are not part of the sequential columns and will be treated as standard tabular data.</p> <code>None</code> <code>cat_embed_dropout</code> <code>Optional[float]</code> <p>Categorical embeddings dropout. If <code>None</code>, it will default to 0.</p> <code>None</code> <code>use_cat_bias</code> <code>Optional[bool]</code> <p>Boolean indicating if bias will be used for the categorical embeddings. If <code>None</code>, it will default to 'False'.</p> <code>None</code> <code>cat_embed_activation</code> <code>Optional[str]</code> <p>Activation function for the categorical embeddings, if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> <code>None</code> <code>continuous_cols</code> <code>Optional[List[str]]</code> <p>List with the name of the numeric (aka continuous) columns</p> <code>None</code> <code>cont_norm_layer</code> <code>Optional[Literal['batchnorm', 'layernorm']]</code> <p>Type of normalization layer applied to the continuous features. Options are: 'layernorm' and 'batchnorm'. if <code>None</code>, no normalization layer will be used.</p> <code>None</code> <code>embed_continuous</code> <code>Optional[bool]</code> <p>Boolean indicating if the continuous columns will be embedded using one of the available methods: 'standard', 'periodic' or 'piecewise'. If <code>None</code>, it will default to 'False'. NOTE: This parameter is deprecated and it  will be removed in future releases. Please, use the  <code>embed_continuous_method</code> parameter instead.</p> <code>None</code> <code>embed_continuous_method</code> <code>Optional[Literal['piecewise', 'periodic', 'standard']]</code> <p>Method to use to embed the continuous features. Options are: 'standard', 'periodic' or 'piecewise'. The 'standard' embedding method is based on the FT-Transformer implementation presented in the paper: Revisiting Deep Learning Models for Tabular Data. The 'periodic' and_'piecewise'_ methods were presented in the paper: On Embeddings for Numerical Features in Tabular Deep Learning. Please, read the papers for details.</p> <code>None</code> <code>cont_embed_dim</code> <code>Optional[int]</code> <p>Size of the continuous embeddings. If the continuous columns are embedded, <code>cont_embed_dim</code> must be passed.</p> <code>None</code> <code>cont_embed_dropout</code> <code>Optional[float]</code> <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p> <code>None</code> <code>cont_embed_activation</code> <code>Optional[str]</code> <p>Activation function for the continuous embeddings if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported. If <code>None</code>, no activation function will be applied.</p> <code>None</code> <code>quantization_setup</code> <code>Optional[Dict[str, List[float]]]</code> <p>This parameter is used when the 'piecewise' method is used to embed the continuous cols. It is a dict where keys are the name of the continuous columns and values are lists with the boundaries for the quantization of the continuous_cols. See the examples for details. If If the 'piecewise' method is used, this parameter is required.</p> <code>None</code> <code>n_frequencies</code> <code>Optional[int]</code> <p>This is the so called 'k' in their paper On Embeddings for Numerical Features in Tabular Deep Learning, and is the number of 'frequencies' that will be used to represent each continuous column. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>sigma</code> <code>Optional[float]</code> <p>This is the sigma parameter in the paper mentioned when describing the previous parameters and it is used to initialise the 'frequency weights'. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>share_last_layer</code> <code>Optional[bool]</code> <p>This parameter is not present in the before mentioned paper but it is implemented in the official repo. If <code>True</code> the linear layer that turns the frequencies into embeddings will be shared across the continuous columns. If <code>False</code> a different linear layer will be used for each continuous column. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>full_embed_dropout</code> <code>Optional[bool]</code> <p>If <code>True</code>, the full embedding corresponding to a column will be masked out/dropout. If <code>None</code>, it will default to <code>False</code>.</p> <code>None</code> <code>mlp_hidden_dims</code> <code>Optional[List[int]]</code> <p>List with the number of neurons per dense layer in the mlp.</p> <code>None</code> <code>mlp_activation</code> <code>Optional[str]</code> <p>Activation function for the dense layers of the MLP. Currently 'tanh', 'relu', 'leaky_relu', 'gelu' and 'preglu' are   supported</p> <code>None</code> <code>mlp_dropout</code> <code>Optional[float]</code> <p>float or List of floats with the dropout between the dense layers. e.g: [0.5,0.5]</p> <code>None</code> <code>mlp_batchnorm</code> <code>Optional[bool]</code> <p>Boolean indicating whether or not batch normalization will be applied to the dense layers</p> <code>None</code> <code>mlp_batchnorm_last</code> <code>Optional[bool]</code> <p>Boolean indicating whether or not batch normalization will be applied to the last of the dense layers</p> <code>None</code> <code>mlp_linear_first</code> <code>Optional[bool]</code> <p>Boolean indicating the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code></p> <code>None</code> <p>Attributes:</p> Name Type Description <code>user_behavior_indexes</code> <code>List[int]</code> <p>List with the indexes of the user behavior columns</p> <code>user_behavior_embed</code> <code>BaseTabularModelWithAttention</code> <p>Embedding layer for the user</p> <code>action_seq_indexes</code> <code>List[int]</code> <p>List with the indexes of the rating sequence columns if the action_seq_config parameter is not None</p> <code>action_embed</code> <code>BaseTabularModelWithAttention</code> <p>Embedding layer for the rating sequence columns if the action_seq_config parameter is not None</p> <code>other_seq_cols_indexes</code> <code>Dict[str, List[int]]</code> <p>Dictionary with the indexes of the other sequential columns if the other_seq_cols_confiq parameter is not None</p> <code>other_seq_cols_embed</code> <code>ModuleDict</code> <p>Dictionary with the embedding layers for the other sequential columns if the other_seq_cols_confiq parameter is not None</p> <code>other_cols_idx</code> <code>List[int]</code> <p>List with the indexes of the other columns if the other_cols_config parameter is not None</p> <code>other_col_embed</code> <code>BaseTabularModel</code> <p>Embedding layer for the other columns if the other_cols_config parameter is not None</p> <code>mlp</code> <code>Optional[MLP]</code> <p>MLP component of the model. If None, no MLP will be used. This should almost always be not None.</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; import numpy as np\n&gt;&gt;&gt; from torch import Tensor\n&gt;&gt;&gt; from typing import Dict, List, Tuple\n&gt;&gt;&gt; from pytorch_widedeep.models.rec import DeepInterestNetwork\n&gt;&gt;&gt; np_seed = np.random.seed(42)\n&gt;&gt;&gt; torch_seed = torch.manual_seed(42)\n&gt;&gt;&gt; num_users = 10\n&gt;&gt;&gt; num_items = 5\n&gt;&gt;&gt; num_contexts = 3\n&gt;&gt;&gt; seq_length = 3\n&gt;&gt;&gt; num_samples = 10\n&gt;&gt;&gt; user_ids = np.random.randint(0, num_users, num_samples)\n&gt;&gt;&gt; target_item_ids = np.random.randint(0, num_items, num_samples)\n&gt;&gt;&gt; context_ids = np.random.randint(0, num_contexts, num_samples)\n&gt;&gt;&gt; user_behavior = np.array(\n...     [\n...         np.random.choice(num_items, seq_length, replace=False)\n...         for _ in range(num_samples)\n...     ]\n... )\n&gt;&gt;&gt; X_arr = np.column_stack((user_ids, target_item_ids, context_ids, user_behavior))\n&gt;&gt;&gt; X = torch.tensor(X_arr, dtype=torch.long)\n&gt;&gt;&gt; column_idx: Dict[str, int] = {\n...     \"user_id\": 0,\n...     \"target_item\": 1,\n...     \"context\": 2,\n...     \"item_1\": 3,\n...     \"item_2\": 4,\n...     \"item_3\": 5,\n... }\n&gt;&gt;&gt; user_behavior_config: Tuple[List[str], int, int] = (\n...     [\"item_1\", \"item_2\", \"item_3\"],\n...     num_items,\n...     8,\n... )\n&gt;&gt;&gt; cat_embed_input: List[Tuple[str, int, int]] = [\n...     (\"user_id\", num_users, 8),\n...     (\"context\", num_contexts, 4),\n... ]\n&gt;&gt;&gt; model = DeepInterestNetwork(\n...     column_idx=column_idx,\n...     target_item_col=\"target_item\",\n...     user_behavior_confiq=user_behavior_config,\n...     cat_embed_input=cat_embed_input,\n...     mlp_hidden_dims=[16, 8],\n... )\n&gt;&gt;&gt; output = model(X)\n</code></pre> Source code in <code>pytorch_widedeep/models/rec/din.py</code> <pre><code>class DeepInterestNetwork(BaseWDModelComponent):\n    \"\"\"\n    Adaptation of the Deep Interest Network (DIN) for recommendation systems\n    as described in the paper: 'Deep Interest Network for Click-Through Rate\n    Prediction' by Guorui Zhou et al. 2018.\n\n    Note that all the categorical- and continuous-related parameters refer to\n    the categorical and continuous columns that are not part of the\n    sequential columns and will be treated as standard tabular data.\n\n    This model requires some specific data preparation that allows for quite a\n    lot of flexibility. Therefore, I have included a preprocessor\n    (`DINPreprocessor`) in the preprocessing module that will take care of\n    the data preparation.\n\n    Parameters\n    ----------\n    column_idx : Dict[str, int]\n        Dictionary mapping column names to their corresponding index.\n    target_item_col : str\n        Name of the target item column. Note that this is not the target\n        column. This algorithm relies on a sequence representation of\n        interactions. The target item would be the next item in the sequence\n        of interactions (e.g. item 6th in a sequence of 5 items), and our\n        goal is to predict a given action on it.\n    user_behavior_confiq : Tuple[List[str], int, int]\n        Configuration for user behavior sequence columns. Tuple containing:\n        - List of column names that correspond to the user behavior sequence&lt;br/&gt;\n        - Number of unique feature values (n_tokens)&lt;br/&gt;\n        - Embedding dimension&lt;br/&gt;\n        Example: `([\"item_1\", \"item_2\", \"item_3\"], 5, 8)`\n    action_seq_config : Optional[Tuple[List[str], int]], default=None\n        Configuration for a so-called action sequence columns (for example a\n        rating, or purchased/not-purchased, etc). Tuple containing:&lt;br/&gt;\n        - List of column names&lt;br/&gt;\n        - Number of unique feature values (n_tokens)&lt;br/&gt;\n        This action will **always** be learned as a 1d embedding and will be\n        combined with the user behaviour. For example, imagine that the\n        action is purchased/not-purchased. then per item in the user\n        behaviour sequence there will be a binary action to learn 0/1. Such\n        action will be represented by a float number that will multiply the\n        corresponding item embedding in the user behaviour sequence.&lt;br/&gt;\n        Example: `([\"rating_1\", \"rating_2\", \"rating_3\"], 5)`&lt;br/&gt;\n        Internally, the embedding dimension will be set to 1\n    other_seq_cols_confiq : Optional[List[Tuple[List[str], int, int]]], default=None\n        Configuration for other sequential columns. List of tuples containing:&lt;br/&gt;\n        - List of column names that correspond to the sequential column&lt;br/&gt;\n        - Number of unique feature values (n_tokens)&lt;br/&gt;\n        - Embedding dimension&lt;br/&gt;\n        Example: `[([\"seq1_col1\", \"seq1_col2\"], 5, 8), ([\"seq2_col1\", \"seq2_col2\"], 5, 8)]`\n    attention_unit_activation : Literal[\"prelu\", \"dice\"], default=\"prelu\"\n        Activation function to use in the attention unit.\n    cat_embed_input : Optional[List[Tuple[str, int, int]]], default=None\n        Configuration for other columns. List of tuples containing:&lt;br/&gt;\n        - Column name&lt;br/&gt;\n        - Number of unique feature values (n_tokens)&lt;br/&gt;\n        - Embedding dimension&lt;br/&gt;\n\n        **Note**: From here in advance the remaining parameters are related to\n        the categorical and continuous columns that are not part of the\n        sequential columns and will be treated as standard tabular data.\n\n    cat_embed_dropout : Optional[float], default=None\n        Categorical embeddings dropout. If `None`, it will default\n        to 0.\n    use_cat_bias : Optional[bool], default=None\n        Boolean indicating if bias will be used for the categorical embeddings.\n        If `None`, it will default to 'False'.\n    cat_embed_activation : Optional[str], default=None\n        Activation function for the categorical embeddings, if any. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported\n    continuous_cols : Optional[List[str]], default=None\n        List with the name of the numeric (aka continuous) columns\n    cont_norm_layer : Optional[Literal[\"batchnorm\", \"layernorm\"]], default=None\n        Type of normalization layer applied to the continuous features.\n        Options are: _'layernorm'_ and _'batchnorm'_. if `None`, no\n        normalization layer will be used.\n    embed_continuous : Optional[bool], default=None\n        Boolean indicating if the continuous columns will be embedded using\n        one of the available methods: _'standard'_, _'periodic'_\n        or _'piecewise'_. If `None`, it will default to 'False'.&lt;br/&gt;\n        :information_source: **NOTE**: This parameter is deprecated and it\n         will be removed in future releases. Please, use the\n         `embed_continuous_method` parameter instead.\n    embed_continuous_method : Optional[Literal[\"piecewise\", \"periodic\", \"standard\"]], default=None\n        Method to use to embed the continuous features. Options are:\n        _'standard'_, _'periodic'_ or _'piecewise'_. The _'standard'_\n        embedding method is based on the FT-Transformer implementation\n        presented in the paper: [Revisiting Deep Learning Models for\n        Tabular Data](https://arxiv.org/abs/2106.11959v5). The _'periodic'_\n        and_'piecewise'_ methods were presented in the paper: [On Embeddings for\n        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556).\n        Please, read the papers for details.\n    cont_embed_dim : Optional[int], default=None\n        Size of the continuous embeddings. If the continuous columns are\n        embedded, `cont_embed_dim` must be passed.\n    cont_embed_dropout : Optional[float], default=None\n        Dropout for the continuous embeddings. If `None`, it will default to 0.0\n    cont_embed_activation : Optional[str], default=None\n        Activation function for the continuous embeddings if any. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported.\n        If `None`, no activation function will be applied.\n    quantization_setup : Optional[Dict[str, List[float]]], default=None\n        This parameter is used when the _'piecewise'_ method is used to embed\n        the continuous cols. It is a dict where keys are the name of the continuous\n        columns and values are lists with the boundaries for the quantization\n        of the continuous_cols. See the examples for details. If\n        If the _'piecewise'_ method is used, this parameter is required.\n    n_frequencies : Optional[int], default=None\n        This is the so called _'k'_ in their paper [On Embeddings for\n        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556),\n        and is the number of 'frequencies' that will be used to represent each\n        continuous column. See their Eq 2 in the paper for details. If\n        the _'periodic'_ method is used, this parameter is required.\n    sigma : Optional[float], default=None\n        This is the sigma parameter in the paper mentioned when describing the\n        previous parameters and it is used to initialise the 'frequency\n        weights'. See their Eq 2 in the paper for details. If\n        the _'periodic'_ method is used, this parameter is required.\n    share_last_layer : Optional[bool], default=None\n        This parameter is not present in the before mentioned paper but it is implemented in\n        the [official repo](https://github.com/yandex-research/rtdl-num-embeddings/tree/main).\n        If `True` the linear layer that turns the frequencies into embeddings\n        will be shared across the continuous columns. If `False` a different\n        linear layer will be used for each continuous column.\n        If the _'periodic'_ method is used, this parameter is required.\n    full_embed_dropout: bool, Optional, default = None,\n        If `True`, the full embedding corresponding to a column will be masked\n        out/dropout. If `None`, it will default to `False`.\n    mlp_hidden_dims: List, default = [200, 100]\n        List with the number of neurons per dense layer in the mlp.\n    mlp_activation: str, default = \"relu\"\n        Activation function for the dense layers of the MLP. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_, _'gelu'_ and _'preglu'_ are\n          supported\n    mlp_dropout: float or List, default = 0.1\n        float or List of floats with the dropout between the dense layers.\n        e.g: _[0.5,0.5]_\n    mlp_batchnorm: bool, default = False\n        Boolean indicating whether or not batch normalization will be applied\n        to the dense layers\n    mlp_batchnorm_last: bool, default = False\n        Boolean indicating whether or not batch normalization will be applied\n        to the last of the dense layers\n    mlp_linear_first: bool, default = False\n        Boolean indicating the order of the operations in the dense\n        layer. If `True: [LIN -&gt; ACT -&gt; BN -&gt; DP]`. If `False: [BN -&gt; DP -&gt;\n        LIN -&gt; ACT]`\n\n    Attributes\n    ----------\n    user_behavior_indexes: List[int]\n        List with the indexes of the user behavior columns\n    user_behavior_embed: BaseTabularModelWithAttention\n        Embedding layer for the user\n    action_seq_indexes: List[int]\n        List with the indexes of the rating sequence columns if the\n        action_seq_config parameter is not None\n    action_embed: BaseTabularModelWithAttention\n        Embedding layer for the rating sequence columns if the\n        action_seq_config parameter is not None\n    other_seq_cols_indexes: Dict[str, List[int]]\n        Dictionary with the indexes of the other sequential columns if the\n        other_seq_cols_confiq parameter is not None\n    other_seq_cols_embed: nn.ModuleDict\n        Dictionary with the embedding layers for the other sequential columns\n        if the other_seq_cols_confiq parameter is not None\n    other_cols_idx: List[int]\n        List with the indexes of the other columns if the other_cols_config\n        parameter is not None\n    other_col_embed: BaseTabularModel\n        Embedding layer for the other columns if the other_cols_config\n        parameter is not None\n    mlp: Optional[MLP]\n        MLP component of the model. If None, no MLP will be used. This should\n        almost always be not None.\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; import numpy as np\n    &gt;&gt;&gt; from torch import Tensor\n    &gt;&gt;&gt; from typing import Dict, List, Tuple\n    &gt;&gt;&gt; from pytorch_widedeep.models.rec import DeepInterestNetwork\n    &gt;&gt;&gt; np_seed = np.random.seed(42)\n    &gt;&gt;&gt; torch_seed = torch.manual_seed(42)\n    &gt;&gt;&gt; num_users = 10\n    &gt;&gt;&gt; num_items = 5\n    &gt;&gt;&gt; num_contexts = 3\n    &gt;&gt;&gt; seq_length = 3\n    &gt;&gt;&gt; num_samples = 10\n    &gt;&gt;&gt; user_ids = np.random.randint(0, num_users, num_samples)\n    &gt;&gt;&gt; target_item_ids = np.random.randint(0, num_items, num_samples)\n    &gt;&gt;&gt; context_ids = np.random.randint(0, num_contexts, num_samples)\n    &gt;&gt;&gt; user_behavior = np.array(\n    ...     [\n    ...         np.random.choice(num_items, seq_length, replace=False)\n    ...         for _ in range(num_samples)\n    ...     ]\n    ... )\n    &gt;&gt;&gt; X_arr = np.column_stack((user_ids, target_item_ids, context_ids, user_behavior))\n    &gt;&gt;&gt; X = torch.tensor(X_arr, dtype=torch.long)\n    &gt;&gt;&gt; column_idx: Dict[str, int] = {\n    ...     \"user_id\": 0,\n    ...     \"target_item\": 1,\n    ...     \"context\": 2,\n    ...     \"item_1\": 3,\n    ...     \"item_2\": 4,\n    ...     \"item_3\": 5,\n    ... }\n    &gt;&gt;&gt; user_behavior_config: Tuple[List[str], int, int] = (\n    ...     [\"item_1\", \"item_2\", \"item_3\"],\n    ...     num_items,\n    ...     8,\n    ... )\n    &gt;&gt;&gt; cat_embed_input: List[Tuple[str, int, int]] = [\n    ...     (\"user_id\", num_users, 8),\n    ...     (\"context\", num_contexts, 4),\n    ... ]\n    &gt;&gt;&gt; model = DeepInterestNetwork(\n    ...     column_idx=column_idx,\n    ...     target_item_col=\"target_item\",\n    ...     user_behavior_confiq=user_behavior_config,\n    ...     cat_embed_input=cat_embed_input,\n    ...     mlp_hidden_dims=[16, 8],\n    ... )\n    &gt;&gt;&gt; output = model(X)\n    \"\"\"\n\n    def __init__(\n        self,\n        *,\n        column_idx: Dict[str, int],\n        user_behavior_confiq: Tuple[List[str], int, int],\n        target_item_col: str = \"target_item\",\n        action_seq_config: Optional[Tuple[List[str], int]] = None,\n        other_seq_cols_confiq: Optional[List[Tuple[List[str], int, int]]] = None,\n        attention_unit_activation: Literal[\"prelu\", \"dice\"] = \"prelu\",\n        cat_embed_input: Optional[List[Tuple[str, int, int]]] = None,\n        cat_embed_dropout: Optional[float] = None,\n        use_cat_bias: Optional[bool] = None,\n        cat_embed_activation: Optional[str] = None,\n        continuous_cols: Optional[List[str]] = None,\n        cont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\n        embed_continuous: Optional[bool] = None,\n        embed_continuous_method: Optional[\n            Literal[\"standard\", \"piecewise\", \"periodic\"]\n        ] = None,\n        cont_embed_dim: Optional[int] = None,\n        cont_embed_dropout: Optional[float] = None,\n        cont_embed_activation: Optional[str] = None,\n        quantization_setup: Optional[Dict[str, List[float]]] = None,\n        n_frequencies: Optional[int] = None,\n        sigma: Optional[float] = None,\n        share_last_layer: Optional[bool] = None,\n        full_embed_dropout: Optional[bool] = None,\n        mlp_hidden_dims: Optional[List[int]] = None,\n        mlp_activation: Optional[str] = None,\n        mlp_dropout: Optional[float] = None,\n        mlp_batchnorm: Optional[bool] = None,\n        mlp_batchnorm_last: Optional[bool] = None,\n        mlp_linear_first: Optional[bool] = None,\n    ):\n        super(DeepInterestNetwork, self).__init__()\n\n        self.column_idx = {\n            k: v for k, v in sorted(column_idx.items(), key=lambda x: x[1])\n        }\n\n        self.column_idx = column_idx\n        self.target_item_col = target_item_col\n        self.user_behavior_confiq = user_behavior_confiq\n        self.action_seq_config = (\n            (action_seq_config[0], action_seq_config[1], 1)\n            if action_seq_config is not None\n            else None\n        )\n        self.other_seq_cols_confiq = other_seq_cols_confiq\n        self.cat_embed_input = cat_embed_input\n        self.attention_unit_activation = attention_unit_activation\n\n        self.cat_embed_dropout = cat_embed_dropout\n        self.use_cat_bias = use_cat_bias\n        self.cat_embed_activation = cat_embed_activation\n        self.continuous_cols = continuous_cols\n        self.cont_norm_layer = cont_norm_layer\n        self.embed_continuous = embed_continuous\n        self.embed_continuous_method = embed_continuous_method\n        self.cont_embed_dim = cont_embed_dim\n        self.cont_embed_dropout = cont_embed_dropout\n        self.cont_embed_activation = cont_embed_activation\n        self.quantization_setup = quantization_setup\n        self.n_frequencies = n_frequencies\n        self.sigma = sigma\n        self.share_last_layer = share_last_layer\n        self.full_embed_dropout = full_embed_dropout\n\n        self.mlp_hidden_dims = mlp_hidden_dims\n        self.mlp_activation = mlp_activation\n        self.mlp_dropout = mlp_dropout\n        self.mlp_batchnorm = mlp_batchnorm\n        self.mlp_batchnorm_last = mlp_batchnorm_last\n        self.mlp_linear_first = mlp_linear_first\n\n        self.target_item_idx = self.column_idx[target_item_col]\n\n        self.user_behavior_indexes, self.user_behavior_embed = (\n            self.set_user_behavior_indexes_and_embed(user_behavior_confiq)\n        )\n        self.user_behavior_dim = user_behavior_confiq[2]\n\n        if self.action_seq_config is not None:\n            self.action_seq_indexes, self.action_embed = (\n                self._set_rating_indexes_and_embed(self.action_seq_config)\n            )\n        else:\n            self.action_embed = None\n\n        if self.other_seq_cols_confiq is not None:\n            (\n                self.other_seq_cols_indexes,\n                self.other_seq_cols_embed,\n                self.other_seq_dim,\n            ) = self._set_other_seq_cols_indexes_embed_and_dim(\n                self.other_seq_cols_confiq\n            )\n        else:\n            self.other_seq_cols_embed = None\n            self.other_seq_dim = 0\n\n        if self.cat_embed_input is not None or self.continuous_cols is not None:\n            self.other_cols_idx, self.other_col_embed, self.other_cols_dim = (\n                self._set_other_cols_idx_embed_and_dim(\n                    self.cat_embed_input, self.continuous_cols\n                )\n            )\n        else:\n            self.other_col_embed = None\n            self.other_cols_dim = 0\n\n        self.attention = ActivationUnit(\n            user_behavior_confiq[2], attention_unit_activation\n        )\n\n        if self.mlp_hidden_dims is not None:\n            mlp_input_dim = (\n                self.user_behavior_dim * 2 + self.other_seq_dim + self.other_cols_dim\n            )\n            self.mlp = MLP(\n                d_hidden=[mlp_input_dim] + self.mlp_hidden_dims,\n                activation=(\n                    \"relu\" if self.mlp_activation is None else self.mlp_activation\n                ),\n                dropout=0.0 if self.mlp_dropout is None else self.mlp_dropout,\n                batchnorm=False if self.mlp_batchnorm is None else self.mlp_batchnorm,\n                batchnorm_last=(\n                    False\n                    if self.mlp_batchnorm_last is None\n                    else self.mlp_batchnorm_last\n                ),\n                linear_first=(\n                    False if self.mlp_linear_first is None else self.mlp_linear_first\n                ),\n            )\n        else:\n            self.mlp = None\n\n    def forward(self, X: Tensor) -&gt; Tensor:\n\n        X_target_item = X[:, [self.target_item_idx]]\n        item_embed = self.user_behavior_embed.cat_embed.embed(X_target_item.long())\n\n        X_user_behavior = X[:, self.user_behavior_indexes]\n        user_behavior_embed = self.user_behavior_embed._get_embeddings(X_user_behavior)\n        # 0 is the padding index\n        mask = (X_user_behavior != 0).float().to(X.device)\n\n        if self.action_embed is not None:\n            X_rating = X[:, self.action_seq_indexes]\n            action_embed = self.action_embed._get_embeddings(X_rating)\n            user_behavior_embed = user_behavior_embed * action_embed\n\n        attention_scores = self.attention(item_embed, user_behavior_embed)\n        attention_scores = attention_scores * mask\n        user_interest = (attention_scores.unsqueeze(-1) * user_behavior_embed).sum(1)\n\n        deep_out = torch.cat([item_embed.squeeze(1), user_interest], dim=1)\n\n        if self.other_seq_cols_embed is not None:\n            X_other_seq: Dict[str, Tensor] = {\n                col: X[:, idx] for col, idx in self.other_seq_cols_indexes.items()\n            }\n            other_seq_embed = torch.cat(\n                [\n                    self.other_seq_cols_embed[col]._get_embeddings(X_other_seq[col])\n                    for col in self.other_seq_cols_indexes.keys()\n                ],\n                dim=-1,\n            ).sum(1)\n            deep_out = torch.cat([deep_out, other_seq_embed], dim=1)\n\n        if self.other_col_embed is not None:\n            X_other_cols = X[:, self.other_cols_idx]\n            other_cols_embed = self.other_col_embed._get_embeddings(X_other_cols)\n            deep_out = torch.cat([deep_out, other_cols_embed], dim=1)\n\n        if self.mlp is not None:\n            deep_out = self.mlp(deep_out)\n\n        return deep_out\n\n    @staticmethod\n    def _get_seq_cols_embed_confiq(tup: Tuple[List[str], int, int]) -&gt; Dict[str, Any]:\n        # tup[0] is the list of columns\n        # tup[1] is the number of unique feat value or \"n_tokens\"\n        # tup[2] is the embedding dimension\n\n        # Once sliced, the indexes will go from 0 to len(tup[0]) it is assumed\n        # that the columns in tup[0] are ordered of appearance in the input\n        # data\n        column_idx = {col: i for i, col in enumerate(tup[0])}\n\n        # This is a hack so that I can use any BaseTabularModelWithAttention.\n        # For this model to work 'cat_embed_input' is normally a List of\n        # Tuples where the first element is the column name and the second is\n        # the number of unique values for that column. That second elements\n        # is added internally to compute what one could call \"n_tokens\". Here\n        # I'm passing that value as the second element of the first tuple\n        # and then adding 0s for the rest of the columns\n        cat_embed_input = [(tup[0][0], tup[1])] + [(col, 0) for col in tup[0][1:]]\n\n        input_dim = tup[2]\n\n        col_config = {\n            \"column_idx\": column_idx,\n            \"input_dim\": input_dim,\n            \"cat_embed_input\": cat_embed_input,\n            \"cat_embed_dropout\": None,\n            \"use_cat_bias\": None,\n            \"cat_embed_activation\": None,\n            \"shared_embed\": None,\n            \"add_shared_embed\": None,\n            \"frac_shared_embed\": None,\n            \"continuous_cols\": None,\n            \"cont_norm_layer\": None,\n            \"embed_continuous_method\": None,\n            \"cont_embed_dropout\": None,\n            \"cont_embed_activation\": None,\n            \"quantization_setup\": None,\n            \"n_frequencies\": None,\n            \"sigma\": None,\n            \"share_last_layer\": None,\n            \"full_embed_dropout\": None,\n        }\n\n        return col_config\n\n    def _get_other_cols_embed_config(\n        self,\n        cat_embed_input: Optional[List[Tuple[str, int, int]]],\n        continuous_cols: Optional[List[str]],\n        column_idx: Dict[str, int],\n    ) -&gt; Dict[str, Any]:\n\n        cols_config = {\n            \"column_idx\": {col: i for i, col in enumerate(column_idx.keys())},\n            \"cat_embed_input\": cat_embed_input,\n            \"cat_embed_dropout\": self.cat_embed_dropout,\n            \"use_cat_bias\": self.use_cat_bias,\n            \"cat_embed_activation\": self.cat_embed_activation,\n            \"continuous_cols\": continuous_cols,\n            \"cont_norm_layer\": self.cont_norm_layer,\n            \"embed_continuous\": self.embed_continuous,\n            \"embed_continuous_method\": self.embed_continuous_method,\n            \"cont_embed_dim\": self.cont_embed_dim,\n            \"cont_embed_dropout\": self.cont_embed_dropout,\n            \"cont_embed_activation\": self.cont_embed_activation,\n            \"quantization_setup\": self.quantization_setup,\n            \"n_frequencies\": self.n_frequencies,\n            \"sigma\": self.sigma,\n            \"share_last_layer\": self.share_last_layer,\n            \"full_embed_dropout\": self.full_embed_dropout,\n        }\n\n        return cols_config\n\n    def set_user_behavior_indexes_and_embed(\n        self, user_behavior_confiq: Tuple[List[str], int, int]\n    ) -&gt; Tuple[List[int], BaseTabularModelWithAttention]:\n        user_behavior_indexes = [\n            self.column_idx[col] for col in user_behavior_confiq[0]\n        ]\n        user_behavior_embed = BaseTabularModelWithAttention(\n            **self._get_seq_cols_embed_confiq(user_behavior_confiq)\n        )\n        return user_behavior_indexes, user_behavior_embed\n\n    def _set_rating_indexes_and_embed(\n        self, action_seq_config: Tuple[List[str], int, int]\n    ) -&gt; Tuple[List[int], BaseTabularModelWithAttention]:\n        action_seq_indexes = [self.column_idx[col] for col in action_seq_config[0]]\n        action_embed = BaseTabularModelWithAttention(\n            **self._get_seq_cols_embed_confiq(action_seq_config)\n        )\n        return action_seq_indexes, action_embed\n\n    def _set_other_seq_cols_indexes_embed_and_dim(\n        self, other_seq_cols_confiq: List[Tuple[List[str], int, int]]\n    ) -&gt; Tuple[Dict[str, List[int]], nn.ModuleDict, int]:\n        other_seq_cols_indexes: Dict[str, List[int]] = {}\n        for i, el in enumerate(other_seq_cols_confiq):\n            key = f\"seq_{i}\"\n            idxs = [self.column_idx[col] for col in el[0]]\n            other_seq_cols_indexes[key] = idxs\n        other_seq_cols_config = {\n            f\"seq_{i}\": self._get_seq_cols_embed_confiq(el)\n            for i, el in enumerate(other_seq_cols_confiq)\n        }\n        other_seq_cols_embed = nn.ModuleDict(\n            {\n                key: BaseTabularModelWithAttention(**config)\n                for key, config in other_seq_cols_config.items()\n            }\n        )\n        other_seq_dim = sum([el[2] for el in other_seq_cols_confiq])\n\n        return other_seq_cols_indexes, other_seq_cols_embed, other_seq_dim\n\n    def _set_other_cols_idx_embed_and_dim(\n        self,\n        cat_embed_input: Optional[List[Tuple[str, int, int]]],\n        continuous_cols: Optional[List[str]],\n    ) -&gt; Tuple[List[int], BaseTabularModelWithoutAttention, int]:\n        other_cols_idx: Dict[str, int] = {}\n        if cat_embed_input is not None:\n            other_cols_idx = {\n                col: self.column_idx[col] for col in [el[0] for el in cat_embed_input]\n            }\n        if continuous_cols is not None:\n            other_cols_idx.update(\n                {col: self.column_idx[col] for col in continuous_cols}\n            )\n\n        sorted_other_cols_idx = {\n            k: v for k, v in sorted(other_cols_idx.items(), key=lambda x: x[1])\n        }\n\n        other_col_embed = BaseTabularModelWithoutAttention(\n            **self._get_other_cols_embed_config(\n                cat_embed_input, continuous_cols, sorted_other_cols_idx\n            )\n        )\n\n        other_cols_dim = other_col_embed.output_dim\n\n        return list(other_cols_idx.values()), other_col_embed, other_cols_dim\n\n    @property\n    def output_dim(self) -&gt; int:\n        if self.mlp_hidden_dims is not None:\n            return self.mlp_hidden_dims[-1]\n        else:\n            return self.user_behavior_dim * 2 + self.other_seq_dim + self.other_cols_dim\n</code></pre>"},{"location":"pytorch-widedeep/the_rec_module.html#pytorch_widedeep.models.rec.xdeepfm.ExtremeDeepFactorizationMachine","title":"ExtremeDeepFactorizationMachine","text":"<p>               Bases: <code>BaseTabularModelWithAttention</code></p> <p>Adaptation of 'xDeepFM implementation: xDeepFM: Combining Explicit and Implicit Feature Interactions for Recommender Systems' by Jianxun Lian, Xiaohuan Zhou, Fuzheng Zhang, Zhongxia Chen, Xing Xie, Guangzhong Sun and Enhong Chen, 2018</p> <p>The implementation in this library takes advantage of all the functionalities available to encode categorical and continuous features. The model can be used with only the factorization machine</p> <p>Note that this class implements only the 'Deep' component of the model described in the paper. The linear component is not implemented 'internally' and, if one wants to include it, it can be easily added using the 'wide'/linear component in this library. See the examples in the examples folder.</p> <p>Parameters:</p> Name Type Description Default <code>column_idx</code> <code>Dict[str, int]</code> <p>Dictionary mapping column names to their corresponding index.</p> required <code>input_dim</code> <code>int</code> <p>Embedding input dimensions</p> required <code>reduce_sum</code> <code>bool</code> <p>Whether to reduce the sum in the factorization machine output.</p> <code>True</code> <code>cin_layer_dims</code> <code>List[int]</code> <p>List with the number of units per CIN layer. e.g: [128, 64]</p> required <code>cat_embed_input</code> <code>Optional[List[Tuple[str, int]]]</code> <p>List of tuples with categorical column names and number of unique values.</p> <code>None</code> <code>cat_embed_dropout</code> <code>Optional[float]</code> <p>Categorical embeddings dropout. If <code>None</code>, it will default to 0.</p> <code>None</code> <code>use_cat_bias</code> <code>Optional[bool]</code> <p>Boolean indicating if bias will be used for the categorical embeddings. If <code>None</code>, it will default to 'False'.</p> <code>None</code> <code>cat_embed_activation</code> <code>Optional[str]</code> <p>Activation function for the categorical embeddings, if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> <code>None</code> <code>continuous_cols</code> <code>Optional[List[str]]</code> <p>List with the name of the numeric (aka continuous) columns</p> <code>None</code> <code>cont_norm_layer</code> <code>Optional[Literal['batchnorm', 'layernorm']]</code> <p>Type of normalization layer applied to the continuous features. Options are: 'layernorm' and 'batchnorm'. if <code>None</code>, no normalization layer will be used.</p> <code>None</code> <code>embed_continuous_method</code> <code>Optional[Literal['piecewise', 'periodic', 'standard']]</code> <p>Method to use to embed the continuous features. Options are: 'standard', 'periodic' or 'piecewise'. The 'standard' embedding method is based on the FT-Transformer implementation presented in the paper: Revisiting Deep Learning Models for Tabular Data. The 'periodic' and_'piecewise'_ methods were presented in the paper: On Embeddings for Numerical Features in Tabular Deep Learning. Please, read the papers for details.</p> <code>'standard'</code> <code>cont_embed_dropout</code> <code>Optional[float]</code> <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p> <code>None</code> <code>cont_embed_activation</code> <code>Optional[str]</code> <p>Activation function for the continuous embeddings if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported. If <code>None</code>, no activation function will be applied.</p> <code>None</code> <code>quantization_setup</code> <code>Optional[Dict[str, List[float]]]</code> <p>This parameter is used when the 'piecewise' method is used to embed the continuous cols. It is a dict where keys are the name of the continuous columns and values are lists with the boundaries for the quantization of the continuous_cols. See the examples for details. If If the 'piecewise' method is used, this parameter is required.</p> <code>None</code> <code>n_frequencies</code> <code>Optional[int]</code> <p>This is the so called 'k' in their paper On Embeddings for Numerical Features in Tabular Deep Learning, and is the number of 'frequencies' that will be used to represent each continuous column. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>sigma</code> <code>Optional[float]</code> <p>This is the sigma parameter in the paper mentioned when describing the previous parameters and it is used to initialise the 'frequency weights'. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>share_last_layer</code> <code>Optional[bool]</code> <p>This parameter is not present in the before mentioned paper but it is implemented in the official repo. If <code>True</code> the linear layer that turns the frequencies into embeddings will be shared across the continuous columns. If <code>False</code> a different linear layer will be used for each continuous column. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>full_embed_dropout</code> <code>Optional[bool]</code> <p>If <code>True</code>, the full embedding corresponding to a column will be masked out/dropout. If <code>None</code>, it will default to <code>False</code>.</p> <code>None</code> <code>mlp_hidden_dims</code> <code>Optional[List[int]]</code> <p>List with the number of neurons per dense layer in the mlp.</p> <code>None</code> <code>mlp_activation</code> <code>Optional[str]</code> <p>Activation function for the dense layers of the MLP. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> <code>None</code> <code>mlp_dropout</code> <code>Optional[float]</code> <p>float or List of floats with the dropout between the dense layers. e.g: [0.5,0.5]</p> <code>None</code> <code>mlp_batchnorm</code> <code>Optional[bool]</code> <p>Boolean indicating whether or not batch normalization will be applied to the dense layers</p> <code>None</code> <code>mlp_batchnorm_last</code> <code>Optional[bool]</code> <p>Boolean indicating whether or not batch normalization will be applied to the last of the dense layers</p> <code>None</code> <code>mlp_linear_first</code> <code>Optional[bool]</code> <p>Boolean indicating the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code></p> <code>None</code> <p>Attributes:</p> Name Type Description <code>n_features</code> <code>int</code> <p>Number of unique features/columns</p> <code>cin</code> <code>CompressedInteractionNetwork</code> <p>Instance of the <code>CompressedInteractionNetwork</code> class</p> <code>mlp</code> <code>MLP</code> <p>Instance of the <code>MLP</code> class if <code>mlp_hidden_dims</code> is not None. If None, the model will return directly the output of the <code>CIN</code></p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models.rec import ExtremeDeepFactorizationMachine\n&gt;&gt;&gt; X_tab = torch.randint(0, 10, (16, 2))\n&gt;&gt;&gt; column_idx = {\"col1\": 0, \"col2\": 1}\n&gt;&gt;&gt; cat_embed_input = [(\"col1\", 10), (\"col2\", 10)]\n&gt;&gt;&gt; xdeepfm = ExtremeDeepFactorizationMachine(\n...     column_idx=column_idx,\n...     input_dim=4,\n...     cin_layer_dims=[8, 16],\n...     cat_embed_input=cat_embed_input,\n...     mlp_hidden_dims=[16, 8]\n... )\n&gt;&gt;&gt; output = xdeepfm(X_tab)\n</code></pre> Source code in <code>pytorch_widedeep/models/rec/xdeepfm.py</code> <pre><code>class ExtremeDeepFactorizationMachine(BaseTabularModelWithAttention):\n    \"\"\"\n    Adaptation of 'xDeepFM implementation: xDeepFM: Combining Explicit and\n    Implicit Feature Interactions for Recommender Systems' by Jianxun Lian,\n    Xiaohuan Zhou, Fuzheng Zhang, Zhongxia Chen, Xing Xie, Guangzhong Sun and\n    Enhong Chen, 2018\n\n    The implementation in this library takes advantage of all the\n    functionalities available to encode categorical and continuous features.\n    The model can be used with only the factorization machine\n\n    Note that this class implements only the 'Deep' component of the model\n    described in the paper. The linear component is not\n    implemented 'internally' and, if one wants to include it, it can be\n    easily added using the 'wide'/linear component in this library. See the\n    examples in the examples folder.\n\n    Parameters\n    ----------\n    column_idx : Dict[str, int]\n        Dictionary mapping column names to their corresponding index.\n    input_dim : int\n        Embedding input dimensions\n    reduce_sum : bool, default=True\n        Whether to reduce the sum in the factorization machine output.\n    cin_layer_dims : List[int]\n        List with the number of units per CIN layer. e.g: _[128, 64]_\n    cat_embed_input : Optional[List[Tuple[str, int]]], default=None\n        List of tuples with categorical column names and number of unique values.\n    cat_embed_dropout : Optional[float], default=None\n        Categorical embeddings dropout. If `None`, it will default\n        to 0.\n    use_cat_bias : Optional[bool], default=None\n        Boolean indicating if bias will be used for the categorical embeddings.\n        If `None`, it will default to 'False'.\n    cat_embed_activation : Optional[str], default=None\n        Activation function for the categorical embeddings, if any. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported\n    continuous_cols : Optional[List[str]], default=None\n        List with the name of the numeric (aka continuous) columns\n    cont_norm_layer : Optional[Literal[\"batchnorm\", \"layernorm\"]], default=None\n        Type of normalization layer applied to the continuous features.\n        Options are: _'layernorm'_ and _'batchnorm'_. if `None`, no\n        normalization layer will be used.\n    embed_continuous_method: Optional[Literal[\"piecewise\", \"periodic\", \"standard\"]], default=\"standard\"\n        Method to use to embed the continuous features. Options are:\n        _'standard'_, _'periodic'_ or _'piecewise'_. The _'standard'_\n        embedding method is based on the FT-Transformer implementation\n        presented in the paper: [Revisiting Deep Learning Models for\n        Tabular Data](https://arxiv.org/abs/2106.11959v5). The _'periodic'_\n        and_'piecewise'_ methods were presented in the paper: [On Embeddings for\n        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556).\n        Please, read the papers for details.\n    cont_embed_dropout : Optional[float], default=None\n        Dropout for the continuous embeddings. If `None`, it will default to 0.0\n    cont_embed_activation : Optional[str], default=None\n        Activation function for the continuous embeddings if any. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported.\n        If `None`, no activation function will be applied.\n    quantization_setup : Optional[Dict[str, List[float]]], default=None\n        This parameter is used when the _'piecewise'_ method is used to embed\n        the continuous cols. It is a dict where keys are the name of the continuous\n        columns and values are lists with the boundaries for the quantization\n        of the continuous_cols. See the examples for details. If\n        If the _'piecewise'_ method is used, this parameter is required.\n    n_frequencies : Optional[int], default=None\n        This is the so called _'k'_ in their paper [On Embeddings for\n        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556),\n        and is the number of 'frequencies' that will be used to represent each\n        continuous column. See their Eq 2 in the paper for details. If\n        the _'periodic'_ method is used, this parameter is required.\n    sigma : Optional[float], default=None\n        This is the sigma parameter in the paper mentioned when describing the\n        previous parameters and it is used to initialise the 'frequency\n        weights'. See their Eq 2 in the paper for details. If\n        the _'periodic'_ method is used, this parameter is required.\n    share_last_layer : Optional[bool], default=None\n        This parameter is not present in the before mentioned paper but it is implemented in\n        the [official repo](https://github.com/yandex-research/rtdl-num-embeddings/tree/main).\n        If `True` the linear layer that turns the frequencies into embeddings\n        will be shared across the continuous columns. If `False` a different\n        linear layer will be used for each continuous column.\n        If the _'periodic'_ method is used, this parameter is required.\n    full_embed_dropout: bool, Optional, default = None,\n        If `True`, the full embedding corresponding to a column will be masked\n        out/dropout. If `None`, it will default to `False`.\n    mlp_hidden_dims: List, default = [200, 100]\n        List with the number of neurons per dense layer in the mlp.\n    mlp_activation: str, default = \"relu\"\n        Activation function for the dense layers of the MLP. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported\n    mlp_dropout: float or List, default = 0.1\n        float or List of floats with the dropout between the dense layers.\n        e.g: _[0.5,0.5]_\n    mlp_batchnorm: bool, default = False\n        Boolean indicating whether or not batch normalization will be applied\n        to the dense layers\n    mlp_batchnorm_last: bool, default = False\n        Boolean indicating whether or not batch normalization will be applied\n        to the last of the dense layers\n    mlp_linear_first: bool, default = False\n        Boolean indicating the order of the operations in the dense\n        layer. If `True: [LIN -&gt; ACT -&gt; BN -&gt; DP]`. If `False: [BN -&gt; DP -&gt;\n        LIN -&gt; ACT]`\n\n    Attributes\n    ----------\n    n_features: int\n        Number of unique features/columns\n    cin: CompressedInteractionNetwork\n        Instance of the `CompressedInteractionNetwork` class\n    mlp: MLP\n        Instance of the `MLP` class if `mlp_hidden_dims` is not None. If None,\n        the model will return directly the output of the `CIN`\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.models.rec import ExtremeDeepFactorizationMachine\n    &gt;&gt;&gt; X_tab = torch.randint(0, 10, (16, 2))\n    &gt;&gt;&gt; column_idx = {\"col1\": 0, \"col2\": 1}\n    &gt;&gt;&gt; cat_embed_input = [(\"col1\", 10), (\"col2\", 10)]\n    &gt;&gt;&gt; xdeepfm = ExtremeDeepFactorizationMachine(\n    ...     column_idx=column_idx,\n    ...     input_dim=4,\n    ...     cin_layer_dims=[8, 16],\n    ...     cat_embed_input=cat_embed_input,\n    ...     mlp_hidden_dims=[16, 8]\n    ... )\n    &gt;&gt;&gt; output = xdeepfm(X_tab)\n    \"\"\"\n\n    def __init__(\n        self,\n        *,\n        column_idx: Dict[str, int],\n        input_dim: int,\n        reduce_sum: bool = True,\n        cin_layer_dims: List[int],\n        cat_embed_input: List[Tuple[str, int]],\n        cat_embed_dropout: Optional[float] = None,\n        use_cat_bias: Optional[bool] = None,\n        cat_embed_activation: Optional[str] = None,\n        continuous_cols: Optional[List[str]] = None,\n        cont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\n        embed_continuous_method: Optional[\n            Literal[\"piecewise\", \"periodic\", \"standard\"]\n        ] = \"standard\",\n        cont_embed_dropout: Optional[float] = None,\n        cont_embed_activation: Optional[str] = None,\n        quantization_setup: Optional[Dict[str, List[float]]] = None,\n        n_frequencies: Optional[int] = None,\n        sigma: Optional[float] = None,\n        share_last_layer: Optional[bool] = None,\n        full_embed_dropout: Optional[bool] = None,\n        mlp_hidden_dims: Optional[List[int]] = None,\n        mlp_activation: Optional[str] = None,\n        mlp_dropout: Optional[float] = None,\n        mlp_batchnorm: Optional[bool] = None,\n        mlp_batchnorm_last: Optional[bool] = None,\n        mlp_linear_first: Optional[bool] = None,\n    ):\n        super(ExtremeDeepFactorizationMachine, self).__init__(\n            column_idx=column_idx,\n            input_dim=input_dim,\n            cat_embed_input=cat_embed_input,\n            cat_embed_dropout=cat_embed_dropout,\n            use_cat_bias=use_cat_bias,\n            cat_embed_activation=cat_embed_activation,\n            shared_embed=False,\n            add_shared_embed=None,\n            frac_shared_embed=None,\n            continuous_cols=continuous_cols,\n            cont_norm_layer=cont_norm_layer,\n            embed_continuous_method=embed_continuous_method,\n            cont_embed_dropout=cont_embed_dropout,\n            cont_embed_activation=cont_embed_activation,\n            quantization_setup=quantization_setup,\n            n_frequencies=n_frequencies,\n            sigma=sigma,\n            share_last_layer=share_last_layer,\n            full_embed_dropout=full_embed_dropout,\n        )\n\n        self.mlp_hidden_dims = mlp_hidden_dims\n        self.mlp_activation = mlp_activation\n        self.mlp_dropout = mlp_dropout\n        self.mlp_batchnorm = mlp_batchnorm\n        self.mlp_batchnorm_last = mlp_batchnorm_last\n        self.mlp_linear_first = mlp_linear_first\n\n        self.reduce_sum = reduce_sum\n        self.cin_layer_dims = cin_layer_dims\n\n        self.n_features = len(self.column_idx)\n\n        self.cin = CompressedInteractionNetwork(\n            num_cols=self.n_features, cin_layer_dims=self.cin_layer_dims\n        )\n\n        if self.mlp_hidden_dims is not None:\n            if (\n                self.mlp_hidden_dims[-1] != sum(self.cin_layer_dims)\n                and not self.reduce_sum\n            ):\n                d_hidden = (\n                    [sum(self.cin_layer_dims)]\n                    + self.mlp_hidden_dims\n                    + [sum(self.cin_layer_dims)]\n                )\n            else:\n                d_hidden = [sum(self.cin_layer_dims)] + self.mlp_hidden_dims\n\n            self.mlp = MLP(\n                d_hidden=d_hidden,\n                activation=(\n                    \"relu\" if self.mlp_activation is None else self.mlp_activation\n                ),\n                dropout=0.0 if self.mlp_dropout is None else self.mlp_dropout,\n                batchnorm=False if self.mlp_batchnorm is None else self.mlp_batchnorm,\n                batchnorm_last=(\n                    False\n                    if self.mlp_batchnorm_last is None\n                    else self.mlp_batchnorm_last\n                ),\n                linear_first=(\n                    False if self.mlp_linear_first is None else self.mlp_linear_first\n                ),\n            )\n        else:\n            self.mlp = None\n\n    def forward(self, X: Tensor) -&gt; Tensor:\n\n        embeddings = self._get_embeddings(X)\n        cin_out = self.cin(embeddings)\n\n        if self.mlp is None:\n            if self.reduce_sum:\n                return cin_out.sum(dim=1, keepdim=True)\n            return cin_out\n\n        mlp_out = self.mlp(cin_out)\n\n        if self.reduce_sum:\n            cin_out = cin_out.sum(dim=1, keepdim=True)\n            mlp_out = mlp_out.sum(dim=1, keepdim=True)\n\n        return mlp_out + cin_out\n\n    @property\n    def output_dim(self):\n        if self.reduce_sum:\n            return 1\n        else:\n            return sum(self.cin_layer_dims)\n</code></pre>"},{"location":"pytorch-widedeep/the_rec_module.html#pytorch_widedeep.models.rec.autoint.AutoInt","title":"AutoInt","text":"<p>               Bases: <code>BaseTabularModelWithAttention</code></p> <p>Defines an <code>AutoInt</code> model that can be used as the <code>deeptabular</code> component of a Wide &amp; Deep model or independently by itself.</p> <p>This class implements the AutoInt: Automatic Feature Interaction Learning via Self-Attentive Neural Networks architecture, which learns feature interactions through multi-head self-attention networks.</p> <p>Parameters:</p> Name Type Description Default <code>column_idx</code> <code>Dict[str, int]</code> <p>Dict containing the index of the columns that will be passed through the <code>AutoInt</code> model. Required to slice the tensors. e.g. {'education': 0, 'relationship': 1, 'workclass': 2, ...}.</p> required <code>input_dim</code> <code>int</code> <p>Dimension of the input embeddings</p> required <code>num_heads</code> <code>int</code> <p>Number of attention heads</p> <code>4</code> <code>num_layers</code> <code>int</code> <p>Number of interacting layers (attention + residual)</p> <code>2</code> <code>reduction</code> <code>Literal['mean', 'cat']</code> <p>How to reduce the output of the attention layers. Options are: 'mean': mean of attention outputs 'cat': concatenation of attention outputs</p> <code>'mean'</code> <code>cat_embed_input</code> <code>Optional[List[Tuple[str, int]]]</code> <p>List of Tuples with the column name, number of unique values and embedding dimension. e.g. [(education, 11, 32), ...]</p> <code>None</code> <code>cat_embed_dropout</code> <code>Optional[float]</code> <p>Categorical embeddings dropout. If <code>None</code>, it will default to 0.</p> <code>None</code> <code>use_cat_bias</code> <code>Optional[bool]</code> <p>Boolean indicating if bias will be used for the categorical embeddings</p> <code>None</code> <code>cat_embed_activation</code> <code>Optional[str]</code> <p>Activation function for the categorical embeddings</p> <code>None</code> <code>continuous_cols</code> <code>Optional[List[str]]</code> <p>List with the name of the numeric (aka continuous) columns</p> <code>None</code> <code>cont_norm_layer</code> <code>Optional[Literal['batchnorm', 'layernorm']]</code> <p>Type of normalization layer applied to the continuous features. Options are: 'layernorm' and 'batchnorm'. if <code>None</code>, no normalization layer will be used.</p> <code>None</code> <code>embed_continuous_method</code> <code>Optional[Literal['standard', 'piecewise', 'periodic']]</code> <p>Method to use to embed the continuous features. Options are: 'standard', 'periodic' or 'piecewise'. The 'standard' embedding method is based on the FT-Transformer implementation presented in the paper: Revisiting Deep Learning Models for Tabular Data. The 'periodic' and_'piecewise'_ methods were presented in the paper: On Embeddings for Numerical Features in Tabular Deep Learning. Please, read the papers for details.</p> <code>None</code> <code>cont_embed_dropout</code> <code>Optional[float]</code> <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p> <code>None</code> <code>cont_embed_activation</code> <code>Optional[str]</code> <p>Activation function for the continuous embeddings if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported. If <code>None</code>, no activation function will be applied.</p> <code>None</code> <code>quantization_setup</code> <code>Optional[Dict[str, List[float]]]</code> <p>This parameter is used when the 'piecewise' method is used to embed the continuous cols. It is a dict where keys are the name of the continuous columns and values are lists with the boundaries for the quantization of the continuous_cols. See the examples for details. If If the 'piecewise' method is used, this parameter is required.</p> <code>None</code> <code>n_frequencies</code> <code>Optional[int]</code> <p>This is the so called 'k' in their paper On Embeddings for Numerical Features in Tabular Deep Learning, and is the number of 'frequencies' that will be used to represent each continuous column. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>sigma</code> <code>Optional[float]</code> <p>This is the sigma parameter in the paper mentioned when describing the previous parameters and it is used to initialise the 'frequency weights'. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>share_last_layer</code> <code>Optional[bool]</code> <p>This parameter is not present in the before mentioned paper but it is implemented in the official repo. If <code>True</code> the linear layer that turns the frequencies into embeddings will be shared across the continuous columns. If <code>False</code> a different linear layer will be used for each continuous column. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>full_embed_dropout</code> <code>Optional[bool]</code> <p>If <code>True</code>, the full embedding corresponding to a column will be masked out/dropout. If <code>None</code>, it will default to <code>False</code>.</p> <code>None</code> <p>Attributes:</p> Name Type Description <code>attention_layers</code> <code>ModuleList</code> <p>List of multi-head attention layers</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models.rec import AutoInt\n&gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n&gt;&gt;&gt; colnames = [\"a\", \"b\", \"c\", \"d\", \"e\"]\n&gt;&gt;&gt; cat_embed_input = [(u, i, j) for u, i, j in zip(colnames[:4], [4] * 4, [8] * 4)]\n&gt;&gt;&gt; column_idx = {k: v for v, k in enumerate(colnames)}\n&gt;&gt;&gt; model = AutoInt(\n...     column_idx=column_idx,\n...     input_dim=32,\n...     cat_embed_input=cat_embed_input,\n...     continuous_cols=[\"e\"],\n...     embed_continuous_method=\"standard\",\n...     num_heads=4,\n...     num_layers=2\n... )\n&gt;&gt;&gt; out = model(X_tab)\n</code></pre> Source code in <code>pytorch_widedeep/models/rec/autoint.py</code> <pre><code>class AutoInt(BaseTabularModelWithAttention):\n    r\"\"\"Defines an `AutoInt` model that can be used as the `deeptabular` component\n    of a Wide &amp; Deep model or independently by itself.\n\n    This class implements the [AutoInt: Automatic Feature Interaction Learning via Self-Attentive\n    Neural Networks](https://arxiv.org/abs/1810.11921) architecture, which learns feature\n    interactions through multi-head self-attention networks.\n\n    Parameters\n    ----------\n    column_idx: Dict\n        Dict containing the index of the columns that will be passed through\n        the `AutoInt` model. Required to slice the tensors. e.g.\n        _{'education': 0, 'relationship': 1, 'workclass': 2, ...}_.\n    input_dim: int\n        Dimension of the input embeddings\n    num_heads: int, default = 4\n        Number of attention heads\n    num_layers: int, default = 2\n        Number of interacting layers (attention + residual)\n    reduction: str, default = \"mean\"\n        How to reduce the output of the attention layers. Options are:\n        _'mean'_: mean of attention outputs\n        _'cat'_: concatenation of attention outputs\n    cat_embed_input: List, Optional, default = None\n        List of Tuples with the column name, number of unique values and\n        embedding dimension. e.g. _[(education, 11, 32), ...]_\n    cat_embed_dropout: float, Optional, default = None\n        Categorical embeddings dropout. If `None`, it will default to 0.\n    use_cat_bias: bool, Optional, default = None,\n        Boolean indicating if bias will be used for the categorical embeddings\n    cat_embed_activation: Optional, str, default = None,\n        Activation function for the categorical embeddings\n    continuous_cols : Optional[List[str]], default=None\n        List with the name of the numeric (aka continuous) columns\n    cont_norm_layer : Optional[Literal[\"batchnorm\", \"layernorm\"]], default=None\n        Type of normalization layer applied to the continuous features.\n        Options are: _'layernorm'_ and _'batchnorm'_. if `None`, no\n        normalization layer will be used.\n    embed_continuous_method: Optional[Literal[\"piecewise\", \"periodic\", \"standard\"]], default=\"standard\"\n        Method to use to embed the continuous features. Options are:\n        _'standard'_, _'periodic'_ or _'piecewise'_. The _'standard'_\n        embedding method is based on the FT-Transformer implementation\n        presented in the paper: [Revisiting Deep Learning Models for\n        Tabular Data](https://arxiv.org/abs/2106.11959v5). The _'periodic'_\n        and_'piecewise'_ methods were presented in the paper: [On Embeddings for\n        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556).\n        Please, read the papers for details.\n    cont_embed_dropout : Optional[float], default=None\n        Dropout for the continuous embeddings. If `None`, it will default to 0.0\n    cont_embed_activation : Optional[str], default=None\n        Activation function for the continuous embeddings if any. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported.\n        If `None`, no activation function will be applied.\n    quantization_setup : Optional[Dict[str, List[float]]], default=None\n        This parameter is used when the _'piecewise'_ method is used to embed\n        the continuous cols. It is a dict where keys are the name of the continuous\n        columns and values are lists with the boundaries for the quantization\n        of the continuous_cols. See the examples for details. If\n        If the _'piecewise'_ method is used, this parameter is required.\n    n_frequencies : Optional[int], default=None\n        This is the so called _'k'_ in their paper [On Embeddings for\n        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556),\n        and is the number of 'frequencies' that will be used to represent each\n        continuous column. See their Eq 2 in the paper for details. If\n        the _'periodic'_ method is used, this parameter is required.\n    sigma : Optional[float], default=None\n        This is the sigma parameter in the paper mentioned when describing the\n        previous parameters and it is used to initialise the 'frequency\n        weights'. See their Eq 2 in the paper for details. If\n        the _'periodic'_ method is used, this parameter is required.\n    share_last_layer : Optional[bool], default=None\n        This parameter is not present in the before mentioned paper but it is implemented in\n        the [official repo](https://github.com/yandex-research/rtdl-num-embeddings/tree/main).\n        If `True` the linear layer that turns the frequencies into embeddings\n        will be shared across the continuous columns. If `False` a different\n        linear layer will be used for each continuous column.\n        If the _'periodic'_ method is used, this parameter is required.\n    full_embed_dropout: bool, Optional, default = None,\n        If `True`, the full embedding corresponding to a column will be masked\n        out/dropout. If `None`, it will default to `False`.\n\n    Attributes\n    ----------\n    attention_layers: nn.ModuleList\n        List of multi-head attention layers\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.models.rec import AutoInt\n    &gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n    &gt;&gt;&gt; colnames = [\"a\", \"b\", \"c\", \"d\", \"e\"]\n    &gt;&gt;&gt; cat_embed_input = [(u, i, j) for u, i, j in zip(colnames[:4], [4] * 4, [8] * 4)]\n    &gt;&gt;&gt; column_idx = {k: v for v, k in enumerate(colnames)}\n    &gt;&gt;&gt; model = AutoInt(\n    ...     column_idx=column_idx,\n    ...     input_dim=32,\n    ...     cat_embed_input=cat_embed_input,\n    ...     continuous_cols=[\"e\"],\n    ...     embed_continuous_method=\"standard\",\n    ...     num_heads=4,\n    ...     num_layers=2\n    ... )\n    &gt;&gt;&gt; out = model(X_tab)\n    \"\"\"\n\n    def __init__(\n        self,\n        *,\n        column_idx: Dict[str, int],\n        input_dim: int,\n        num_heads: int = 4,\n        num_layers: int = 2,\n        reduction: Literal[\"mean\", \"cat\"] = \"mean\",\n        cat_embed_input: Optional[List[Tuple[str, int]]] = None,\n        cat_embed_dropout: Optional[float] = None,\n        use_cat_bias: Optional[bool] = None,\n        cat_embed_activation: Optional[str] = None,\n        continuous_cols: Optional[List[str]] = None,\n        cont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\n        embed_continuous_method: Optional[\n            Literal[\"standard\", \"piecewise\", \"periodic\"]\n        ] = None,\n        cont_embed_dropout: Optional[float] = None,\n        cont_embed_activation: Optional[str] = None,\n        quantization_setup: Optional[Dict[str, List[float]]] = None,\n        n_frequencies: Optional[int] = None,\n        sigma: Optional[float] = None,\n        share_last_layer: Optional[bool] = None,\n        full_embed_dropout: Optional[bool] = None,\n    ):\n        super(AutoInt, self).__init__(\n            column_idx=column_idx,\n            input_dim=input_dim,\n            cat_embed_input=cat_embed_input,\n            cat_embed_dropout=cat_embed_dropout,\n            use_cat_bias=use_cat_bias,\n            cat_embed_activation=cat_embed_activation,\n            shared_embed=None,\n            add_shared_embed=None,\n            frac_shared_embed=None,\n            continuous_cols=continuous_cols,\n            cont_norm_layer=cont_norm_layer,\n            embed_continuous_method=embed_continuous_method,\n            cont_embed_dropout=cont_embed_dropout,\n            cont_embed_activation=cont_embed_activation,\n            quantization_setup=quantization_setup,\n            n_frequencies=n_frequencies,\n            sigma=sigma,\n            share_last_layer=share_last_layer,\n            full_embed_dropout=full_embed_dropout,\n        )\n\n        self.num_heads = num_heads\n        self.num_layers = num_layers\n        self.reduction = reduction\n\n        self.attention_layers = nn.ModuleList(\n            [\n                nn.MultiheadAttention(\n                    embed_dim=input_dim,\n                    num_heads=num_heads,\n                    batch_first=True,\n                )\n                for _ in range(num_layers)\n            ]\n        )\n\n    def forward(self, X: torch.Tensor) -&gt; torch.Tensor:\n        x = self._get_embeddings(X)\n\n        for layer in self.attention_layers:\n            attn_output, _ = layer(x, x, x)\n            x = attn_output + x\n\n        if self.reduction == \"mean\":\n            out = x.mean(dim=1)\n        else:\n            out = x.reshape(x.size(0), -1)\n\n        return out\n\n    @property\n    def output_dim(self) -&gt; int:\n        if self.reduction == \"mean\":\n            return self.input_dim\n        else:\n            return self.input_dim * len(self.column_idx)\n</code></pre>"},{"location":"pytorch-widedeep/the_rec_module.html#pytorch_widedeep.models.rec.autoint_plus.AutoIntPlus","title":"AutoIntPlus","text":"<p>               Bases: <code>BaseTabularModelWithAttention</code></p> <p>Defines an <code>AutoIntPlus</code> model that can be used as the <code>deeptabular</code> component of a Wide &amp; Deep model or independently by itself.</p> <p>This class implements an enhanced version of the AutoInt architecture, adding a parallel or stacked deep network and an optional gating mechanism to control the contribution of the attention-based and MLP branches.</p> <p>Parameters:</p> Name Type Description Default <code>column_idx</code> <code>Dict[str, int]</code> <p>Dict containing the index of the columns that will be passed through the model. Required to slice the tensors. e.g. {'education': 0, 'relationship': 1, 'workclass': 2, ...}.</p> required <code>input_dim</code> <code>int</code> <p>Dimension of the input embeddings</p> required <code>num_heads</code> <code>int</code> <p>Number of attention heads</p> <code>4</code> <code>num_layers</code> <code>int</code> <p>Number of interacting layers (attention + residual)</p> <code>2</code> <code>reduction</code> <code>Literal['mean', 'cat']</code> <p>How to reduce the output of the attention layers. Options are: 'mean': mean of attention outputs 'cat': concatenation of attention outputs</p> <code>'mean'</code> <code>structure</code> <code>Literal['stacked', 'parallel']</code> <p>Structure of the model. Either 'parallel' or 'stacked'. If 'parallel', the output will be the concatenation of the attention and deep networks. If 'stacked', the attention output will be fed into the deep network.</p> <code>'parallel'</code> <code>gated</code> <code>bool</code> <p>If True and structure is 'parallel', uses a gating mechanism to combine the attention and deep networks. Note: requires reduction='mean'.</p> <code>True</code> <code>cat_embed_input</code> <code>Optional[List[Tuple[str, int]]]</code> <p>List of Tuples with the column name, number of unique values and embedding dimension. e.g. [(education, 11, 32), ...]</p> <code>None</code> <code>cat_embed_dropout</code> <code>Optional[float]</code> <p>Categorical embeddings dropout. If <code>None</code>, it will default to 0.</p> <code>None</code> <code>use_cat_bias</code> <code>Optional[bool]</code> <p>Boolean indicating if bias will be used for the categorical embeddings</p> <code>None</code> <code>cat_embed_activation</code> <code>Optional[str]</code> <p>Activation function for the categorical embeddings</p> <code>None</code> <code>continuous_cols</code> <code>Optional[List[str]]</code> <p>List with the name of the numeric (aka continuous) columns</p> <code>None</code> <code>cont_norm_layer</code> <code>Optional[Literal['batchnorm', 'layernorm']]</code> <p>Type of normalization layer applied to the continuous features. Options are: 'layernorm' and 'batchnorm'. if <code>None</code>, no normalization layer will be used.</p> <code>None</code> <code>embed_continuous_method</code> <code>Optional[Literal['standard', 'piecewise', 'periodic']]</code> <p>Method to use to embed the continuous features. Options are: 'standard', 'periodic' or 'piecewise'. The 'standard' embedding method is based on the FT-Transformer implementation presented in the paper: Revisiting Deep Learning Models for Tabular Data. The 'periodic' and_'piecewise'_ methods were presented in the paper: On Embeddings for Numerical Features in Tabular Deep Learning. Please, read the papers for details.</p> <code>None</code> <code>cont_embed_dropout</code> <code>Optional[float]</code> <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p> <code>None</code> <code>cont_embed_activation</code> <code>Optional[str]</code> <p>Activation function for the continuous embeddings if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported. If <code>None</code>, no activation function will be applied.</p> <code>None</code> <code>quantization_setup</code> <code>Optional[Dict[str, List[float]]]</code> <p>This parameter is used when the 'piecewise' method is used to embed the continuous cols. It is a dict where keys are the name of the continuous columns and values are lists with the boundaries for the quantization of the continuous_cols. See the examples for details. If If the 'piecewise' method is used, this parameter is required.</p> <code>None</code> <code>n_frequencies</code> <code>Optional[int]</code> <p>This is the so called 'k' in their paper On Embeddings for Numerical Features in Tabular Deep Learning, and is the number of 'frequencies' that will be used to represent each continuous column. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>sigma</code> <code>Optional[float]</code> <p>This is the sigma parameter in the paper mentioned when describing the previous parameters and it is used to initialise the 'frequency weights'. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>share_last_layer</code> <code>Optional[bool]</code> <p>This parameter is not present in the before mentioned paper but it is implemented in the official repo. If <code>True</code> the linear layer that turns the frequencies into embeddings will be shared across the continuous columns. If <code>False</code> a different linear layer will be used for each continuous column. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>full_embed_dropout</code> <code>Optional[bool]</code> <p>If <code>True</code>, the full embedding corresponding to a column will be masked out/dropout. If <code>None</code>, it will default to <code>False</code>.</p> <code>None</code> <code>mlp_hidden_dims</code> <code>List[int]</code> <p>List with the number of neurons per dense layer in the mlp.</p> <code>[100, 100]</code> <code>mlp_activation</code> <code>str</code> <p>Activation function for the dense layers of the MLP. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> <code>'relu'</code> <code>mlp_dropout</code> <code>Union[float, List[float]]</code> <p>float or List of floats with the dropout between the dense layers. e.g: [0.5,0.5]</p> <code>0.1</code> <code>mlp_batchnorm</code> <code>bool</code> <p>Boolean indicating whether or not batch normalization will be applied to the dense layers</p> <code>False</code> <code>mlp_batchnorm_last</code> <code>bool</code> <p>Boolean indicating whether or not batch normalization will be applied to the last of the dense layers</p> <code>False</code> <code>mlp_linear_first</code> <code>bool</code> <p>Boolean indicating the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code></p> <code>True</code> <p>Attributes:</p> Name Type Description <code>attention_layers</code> <code>ModuleList</code> <p>List of multi-head attention layers</p> <code>deep_network</code> <code>Module</code> <p>The deep network component (MLP)</p> <code>gate</code> <code>(Module, optional)</code> <p>The gating network (if gated=True)</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models.rec import AutoIntPlus\n&gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n&gt;&gt;&gt; colnames = [\"a\", \"b\", \"c\", \"d\", \"e\"]\n&gt;&gt;&gt; cat_embed_input = [(u, i, j) for u, i, j in zip(colnames[:4], [4] * 4, [8] * 4)]\n&gt;&gt;&gt; column_idx = {k: v for v, k in enumerate(colnames)}\n&gt;&gt;&gt; model = AutoIntPlus(\n...     column_idx=column_idx,\n...     input_dim=32,\n...     cat_embed_input=cat_embed_input,\n...     continuous_cols=[\"e\"],\n...     embed_continuous_method=\"standard\",\n...     num_heads=4,\n...     num_layers=2,\n...     structure=\"parallel\",\n...     gated=True,\n...     mlp_hidden_dims=[64, 32]\n... )\n&gt;&gt;&gt; out = model(X_tab)\n</code></pre> Source code in <code>pytorch_widedeep/models/rec/autoint_plus.py</code> <pre><code>class AutoIntPlus(BaseTabularModelWithAttention):\n    r\"\"\"Defines an `AutoIntPlus` model that can be used as the `deeptabular` component\n    of a Wide &amp; Deep model or independently by itself.\n\n    This class implements an enhanced version of the [AutoInt](https://arxiv.org/abs/1810.11921)\n    architecture, adding a parallel or stacked deep network and an optional gating mechanism\n    to control the contribution of the attention-based and MLP branches.\n\n    Parameters\n    ----------\n    column_idx: Dict\n        Dict containing the index of the columns that will be passed through\n        the model. Required to slice the tensors. e.g.\n        _{'education': 0, 'relationship': 1, 'workclass': 2, ...}_.\n    input_dim: int\n        Dimension of the input embeddings\n    num_heads: int, default = 4\n        Number of attention heads\n    num_layers: int, default = 2\n        Number of interacting layers (attention + residual)\n    reduction: str, default = \"mean\"\n        How to reduce the output of the attention layers. Options are:\n        _'mean'_: mean of attention outputs\n        _'cat'_: concatenation of attention outputs\n    structure: str, default = \"parallel\"\n        Structure of the model. Either _'parallel'_ or _'stacked'_. If _'parallel'_,\n        the output will be the concatenation of the attention and deep networks.\n        If _'stacked'_, the attention output will be fed into the deep network.\n    gated: bool, default = True\n        If True and structure is 'parallel', uses a gating mechanism to combine\n        the attention and deep networks. Note: requires reduction='mean'.\n    cat_embed_input: List, Optional, default = None\n        List of Tuples with the column name, number of unique values and\n        embedding dimension. e.g. _[(education, 11, 32), ...]_\n    cat_embed_dropout: float, Optional, default = None\n        Categorical embeddings dropout. If `None`, it will default to 0.\n    use_cat_bias: bool, Optional, default = None,\n        Boolean indicating if bias will be used for the categorical embeddings\n    cat_embed_activation: Optional, str, default = None,\n        Activation function for the categorical embeddings\n    continuous_cols : Optional[List[str]], default=None\n        List with the name of the numeric (aka continuous) columns\n    cont_norm_layer : Optional[Literal[\"batchnorm\", \"layernorm\"]], default=None\n        Type of normalization layer applied to the continuous features.\n        Options are: _'layernorm'_ and _'batchnorm'_. if `None`, no\n        normalization layer will be used.\n    embed_continuous_method: Optional[Literal[\"piecewise\", \"periodic\", \"standard\"]], default=\"standard\"\n        Method to use to embed the continuous features. Options are:\n        _'standard'_, _'periodic'_ or _'piecewise'_. The _'standard'_\n        embedding method is based on the FT-Transformer implementation\n        presented in the paper: [Revisiting Deep Learning Models for\n        Tabular Data](https://arxiv.org/abs/2106.11959v5). The _'periodic'_\n        and_'piecewise'_ methods were presented in the paper: [On Embeddings for\n        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556).\n        Please, read the papers for details.\n    cont_embed_dropout : Optional[float], default=None\n        Dropout for the continuous embeddings. If `None`, it will default to 0.0\n    cont_embed_activation : Optional[str], default=None\n        Activation function for the continuous embeddings if any. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported.\n        If `None`, no activation function will be applied.\n    quantization_setup : Optional[Dict[str, List[float]]], default=None\n        This parameter is used when the _'piecewise'_ method is used to embed\n        the continuous cols. It is a dict where keys are the name of the continuous\n        columns and values are lists with the boundaries for the quantization\n        of the continuous_cols. See the examples for details. If\n        If the _'piecewise'_ method is used, this parameter is required.\n    n_frequencies : Optional[int], default=None\n        This is the so called _'k'_ in their paper [On Embeddings for\n        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556),\n        and is the number of 'frequencies' that will be used to represent each\n        continuous column. See their Eq 2 in the paper for details. If\n        the _'periodic'_ method is used, this parameter is required.\n    sigma : Optional[float], default=None\n        This is the sigma parameter in the paper mentioned when describing the\n        previous parameters and it is used to initialise the 'frequency\n        weights'. See their Eq 2 in the paper for details. If\n        the _'periodic'_ method is used, this parameter is required.\n    share_last_layer : Optional[bool], default=None\n        This parameter is not present in the before mentioned paper but it is implemented in\n        the [official repo](https://github.com/yandex-research/rtdl-num-embeddings/tree/main).\n        If `True` the linear layer that turns the frequencies into embeddings\n        will be shared across the continuous columns. If `False` a different\n        linear layer will be used for each continuous column.\n        If the _'periodic'_ method is used, this parameter is required.\n    full_embed_dropout: bool, Optional, default = None,\n        If `True`, the full embedding corresponding to a column will be masked\n        out/dropout. If `None`, it will default to `False`.\n    mlp_hidden_dims: List, default = [200, 100]\n        List with the number of neurons per dense layer in the mlp.\n    mlp_activation: str, default = \"relu\"\n        Activation function for the dense layers of the MLP. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported\n    mlp_dropout: float or List, default = 0.1\n        float or List of floats with the dropout between the dense layers.\n        e.g: _[0.5,0.5]_\n    mlp_batchnorm: bool, default = False\n        Boolean indicating whether or not batch normalization will be applied\n        to the dense layers\n    mlp_batchnorm_last: bool, default = False\n        Boolean indicating whether or not batch normalization will be applied\n        to the last of the dense layers\n    mlp_linear_first: bool, default = False\n        Boolean indicating the order of the operations in the dense\n        layer. If `True: [LIN -&gt; ACT -&gt; BN -&gt; DP]`. If `False: [BN -&gt; DP -&gt;\n        LIN -&gt; ACT]`\n\n    Attributes\n    ----------\n    attention_layers: nn.ModuleList\n        List of multi-head attention layers\n    deep_network: nn.Module\n        The deep network component (MLP)\n    gate: nn.Module, optional\n        The gating network (if gated=True)\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.models.rec import AutoIntPlus\n    &gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n    &gt;&gt;&gt; colnames = [\"a\", \"b\", \"c\", \"d\", \"e\"]\n    &gt;&gt;&gt; cat_embed_input = [(u, i, j) for u, i, j in zip(colnames[:4], [4] * 4, [8] * 4)]\n    &gt;&gt;&gt; column_idx = {k: v for v, k in enumerate(colnames)}\n    &gt;&gt;&gt; model = AutoIntPlus(\n    ...     column_idx=column_idx,\n    ...     input_dim=32,\n    ...     cat_embed_input=cat_embed_input,\n    ...     continuous_cols=[\"e\"],\n    ...     embed_continuous_method=\"standard\",\n    ...     num_heads=4,\n    ...     num_layers=2,\n    ...     structure=\"parallel\",\n    ...     gated=True,\n    ...     mlp_hidden_dims=[64, 32]\n    ... )\n    &gt;&gt;&gt; out = model(X_tab)\n    \"\"\"\n\n    def __init__(\n        self,\n        *,\n        column_idx: Dict[str, int],\n        input_dim: int,\n        num_heads: int = 4,\n        num_layers: int = 2,\n        reduction: Literal[\"mean\", \"cat\"] = \"mean\",\n        structure: Literal[\"stacked\", \"parallel\"] = \"parallel\",\n        gated: bool = True,\n        cat_embed_input: Optional[List[Tuple[str, int]]] = None,\n        cat_embed_dropout: Optional[float] = None,\n        use_cat_bias: Optional[bool] = None,\n        cat_embed_activation: Optional[str] = None,\n        continuous_cols: Optional[List[str]] = None,\n        cont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\n        embed_continuous_method: Optional[\n            Literal[\"standard\", \"piecewise\", \"periodic\"]\n        ] = None,\n        cont_embed_dropout: Optional[float] = None,\n        cont_embed_activation: Optional[str] = None,\n        quantization_setup: Optional[Dict[str, List[float]]] = None,\n        n_frequencies: Optional[int] = None,\n        sigma: Optional[float] = None,\n        share_last_layer: Optional[bool] = None,\n        full_embed_dropout: Optional[bool] = None,\n        mlp_hidden_dims: List[int] = [100, 100],\n        mlp_activation: str = \"relu\",\n        mlp_dropout: Union[float, List[float]] = 0.1,\n        mlp_batchnorm: bool = False,\n        mlp_batchnorm_last: bool = False,\n        mlp_linear_first: bool = True,\n    ):\n        super(AutoIntPlus, self).__init__(\n            column_idx=column_idx,\n            input_dim=input_dim,\n            cat_embed_input=cat_embed_input,\n            cat_embed_dropout=cat_embed_dropout,\n            use_cat_bias=use_cat_bias,\n            cat_embed_activation=cat_embed_activation,\n            shared_embed=None,\n            add_shared_embed=None,\n            frac_shared_embed=None,\n            continuous_cols=continuous_cols,\n            cont_norm_layer=cont_norm_layer,\n            embed_continuous_method=embed_continuous_method,\n            cont_embed_dropout=cont_embed_dropout,\n            cont_embed_activation=cont_embed_activation,\n            quantization_setup=quantization_setup,\n            n_frequencies=n_frequencies,\n            sigma=sigma,\n            share_last_layer=share_last_layer,\n            full_embed_dropout=full_embed_dropout,\n        )\n\n        self.input_dim = input_dim\n        self.num_heads = num_heads\n        self.num_layers = num_layers\n        self.reduction = reduction\n        self.structure = structure\n        self.gated = gated\n\n        self.mlp_hidden_dims = mlp_hidden_dims\n        self.mlp_activation = mlp_activation\n        self.mlp_dropout = mlp_dropout\n        self.mlp_batchnorm = mlp_batchnorm\n        self.mlp_batchnorm_last = mlp_batchnorm_last\n        self.mlp_linear_first = mlp_linear_first\n\n        self.attention_layers = nn.ModuleList(\n            [\n                nn.MultiheadAttention(\n                    embed_dim=self.input_dim,\n                    num_heads=self.num_heads,\n                    batch_first=True,\n                )\n                for _ in range(num_layers)\n            ]\n        )\n\n        if self.gated:\n            self.gate = self._build_gate()\n\n        deep_network_inp_dim = self._get_deep_network_input_dim()\n        self.deep_network = MLP(\n            [deep_network_inp_dim] + self.mlp_hidden_dims,\n            self.mlp_activation,\n            self.mlp_dropout,\n            self.mlp_batchnorm,\n            self.mlp_batchnorm_last,\n            self.mlp_linear_first,\n        )\n\n    def forward(self, X: torch.Tensor) -&gt; torch.Tensor:\n        x = self._get_embeddings(X)\n        attn_output = self._apply_attention_layers(x)\n        reduced_attn_output = self._reduce_attention_output(attn_output)\n        if self.structure == \"parallel\":\n            return self._parallel_output(reduced_attn_output, x)\n        else:  # structure == \"stacked\"\n            return self.deep_network(reduced_attn_output)\n\n    def _get_deep_network_input_dim(self) -&gt; int:\n        if self.structure == \"parallel\":\n            return self.input_dim * len(self.column_idx)\n        elif self.reduction == \"mean\":  # structure == \"stacked\"\n            return self.input_dim\n        else:\n            return self.input_dim * len(self.column_idx)\n\n    def _build_gate(self) -&gt; nn.Linear:\n        self._setup_gating()\n        return nn.Linear(self.input_dim * 2, self.input_dim)\n\n    def _setup_gating(self):\n        if self.structure == \"stacked\":\n            raise ValueError(\n                \"Gating is not supported for stacked structure. Set `gated=False`.\"\n            )\n        if self.reduction != \"mean\":\n            raise ValueError(\n                \"When using a gated structure, the reduction must be 'mean'.\"\n            )\n        if self.mlp_hidden_dims[-1] != self.input_dim:\n            self.mlp_hidden_dims = self.mlp_hidden_dims + [self.input_dim]\n            UserWarning(\n                \"When using a gated structure, the last hidden layer of \"\n                \"the MLP must have the same dimension as the input. \"\n                \"The last hidden layer has been set to the input dimension.\"\n            )\n\n    def _apply_attention_layers(self, x: torch.Tensor) -&gt; torch.Tensor:\n        attn_output = x\n        for layer in self.attention_layers:\n            layer_out, _ = layer(attn_output, attn_output, attn_output)\n            attn_output = layer_out + attn_output\n        return attn_output\n\n    def _reduce_attention_output(self, attn_output: torch.Tensor) -&gt; torch.Tensor:\n        return (\n            attn_output.mean(dim=1)\n            if self.reduction == \"mean\"\n            else attn_output.flatten(1)\n        )\n\n    def _parallel_output(\n        self, attn_output: torch.Tensor, x: torch.Tensor\n    ) -&gt; torch.Tensor:\n        deep_output = self.deep_network(x.reshape(x.size(0), -1))\n\n        if not self.gated:\n            return torch.cat([attn_output, deep_output], dim=1)\n\n        combined_features = torch.cat([attn_output, deep_output], dim=-1)\n        gate_weights = torch.sigmoid(self.gate(combined_features))\n        return gate_weights * attn_output + (1 - gate_weights) * deep_output\n\n    @property\n    def output_dim(self) -&gt; int:\n        if self.structure == \"parallel\":\n            if self.gated:\n                return self.input_dim\n            else:\n                if self.reduction == \"mean\":\n                    return self.mlp_hidden_dims[-1] + self.input_dim\n                else:\n                    return self.mlp_hidden_dims[-1] + self.input_dim * len(\n                        self.column_idx\n                    )\n        else:\n            return self.mlp_hidden_dims[-1]\n</code></pre>"},{"location":"pytorch-widedeep/the_rec_module.html#pytorch_widedeep.models.rec.basic_transformer.Transformer","title":"Transformer","text":"<p>               Bases: <code>Module</code></p> <p>Basic Encoder-Only Transformer Model for sequence classification/regression. As all other models in the library this model can be used as the <code>deeptext</code> component of a Wide &amp; Deep model or independently by itself.</p> <p> NOTE: This model is introduced in the context of recommendation systems and thought for sequences of any nature (e.g. items). It can, of course, still be used for text.</p> <p>Parameters:</p> Name Type Description Default <code>vocab_size</code> <code>int</code> <p>Number of words in the vocabulary</p> required <code>input_dim</code> <code>int</code> <p>Dimension of the token embeddings</p> <p>Param aliases: <code>embed_dim</code>, <code>d_model</code>. </p> required <code>seq_length</code> <code>int</code> <p>Input sequence length</p> required <code>n_heads</code> <code>int</code> <p>Number of attention heads per Transformer block</p> required <code>n_blocks</code> <code>int</code> <p>Number of Transformer blocks</p> required <code>attn_dropout</code> <code>float</code> <p>Dropout that will be applied to the Multi-Head Attention layers</p> <code>0.1</code> <code>ff_dropout</code> <code>float</code> <p>Dropout that will be applied to the FeedForward network</p> <code>0.1</code> <code>ff_factor</code> <code>int</code> <p>Multiplicative factor applied to the first layer of the FF network in each Transformer block, This is normally set to 4.</p> <code>4</code> <code>activation</code> <code>str</code> <p>Transformer Encoder activation function. 'tanh', 'relu', 'leaky_relu', 'gelu', 'geglu' and 'reglu' are supported</p> <code>'gelu'</code> <code>padding_idx</code> <code>int</code> <p>index of the padding token in the padded-tokenised sequences.</p> <code>0</code> <code>with_cls_token</code> <code>bool</code> <p>Boolean indicating if a <code>'[CLS]'</code> token is included in the tokenized sequences. If present, the final hidden state corresponding to this token is used as the aggregated representation for classification and regression tasks. NOTE: if included in the tokenized sequences it must be inserted as the first token in the sequences.</p> <code>False</code> <code>with_pos_encoding</code> <code>bool</code> <p>Boolean indicating if positional encoding will be used</p> <code>True</code> <code>pos_encoding_dropout</code> <code>float</code> <p>Positional encoding dropout</p> <code>0.1</code> <code>pos_encoder</code> <code>Optional[Module]</code> <p>This model uses by default a standard positional encoding approach. However, any custom positional encoder can also be used and pass to the Transformer model via the 'pos_encoder' parameter</p> <code>None</code> <p>Attributes:</p> Name Type Description <code>embedding</code> <code>Module</code> <p>Standard token embedding layer</p> <code>pos_encoder</code> <code>Module</code> <p>Positional Encoder</p> <code>encoder</code> <code>Module</code> <p>Sequence of Transformer blocks</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models.rec import Transformer\n&gt;&gt;&gt; X = torch.cat((torch.zeros([5,1]), torch.empty(5, 4).random_(1,4)), axis=1)\n&gt;&gt;&gt; model = Transformer(vocab_size=4, seq_length=5, input_dim=8, n_heads=1, n_blocks=1)\n&gt;&gt;&gt; out = model(X)\n</code></pre> Source code in <code>pytorch_widedeep/models/rec/basic_transformer.py</code> <pre><code>class Transformer(nn.Module):\n    r\"\"\"\n    Basic Encoder-Only Transformer Model for sequence\n    classification/regression. As all other models in the library this model\n    can be used as the `deeptext` component of a Wide &amp; Deep model or\n    independently by itself.\n\n    :information_source: **NOTE**:\n    This model is introduced in the context of recommendation systems and\n    thought for sequences of any nature (e.g. items). It can, of course,\n    still be used for text.\n\n    Parameters\n    ----------\n    vocab_size: int\n        Number of words in the vocabulary\n    input_dim: int\n        Dimension of the token embeddings\n\n        Param aliases: `embed_dim`, `d_model`. &lt;br/&gt;\n    seq_length: int, Optional, default = None\n        Input sequence length\n    n_heads: int, default = 8\n        Number of attention heads per Transformer block\n    n_blocks: int, default = 4\n        Number of Transformer blocks\n    attn_dropout: float, default = 0.2\n        Dropout that will be applied to the Multi-Head Attention layers\n    ff_dropout: float, default = 0.1\n        Dropout that will be applied to the FeedForward network\n    ff_factor: float, default = 4\n        Multiplicative factor applied to the first layer of the FF network in\n        each Transformer block, This is normally set to 4.\n    activation: str, default = \"gelu\"\n        Transformer Encoder activation function. _'tanh'_, _'relu'_,\n        _'leaky_relu'_, _'gelu'_, _'geglu'_ and _'reglu'_ are supported\n    padding_idx: int, default = 0\n        index of the padding token in the padded-tokenised sequences.\n    with_cls_token: bool, default = False\n        Boolean indicating if a `'[CLS]'` token is included in the tokenized\n        sequences. If present, the final hidden state corresponding to this\n        token is used as the aggregated representation for classification and\n        regression tasks. **NOTE**: if included in the tokenized sequences it\n        must be inserted as the first token in the sequences.\n    with_pos_encoding: bool, default = True\n        Boolean indicating if positional encoding will be used\n    pos_encoding_dropout: float, default = 0.1\n        Positional encoding dropout\n    pos_encoder: nn.Module, Optional, default = None\n        This model uses by default a standard positional encoding approach.\n        However, any custom positional encoder can also be used and pass to\n        the Transformer model via the 'pos_encoder' parameter\n\n    Attributes\n    ----------\n    embedding: nn.Module\n        Standard token embedding layer\n    pos_encoder: nn.Module\n        Positional Encoder\n    encoder: nn.Module\n        Sequence of Transformer blocks\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.models.rec import Transformer\n    &gt;&gt;&gt; X = torch.cat((torch.zeros([5,1]), torch.empty(5, 4).random_(1,4)), axis=1)\n    &gt;&gt;&gt; model = Transformer(vocab_size=4, seq_length=5, input_dim=8, n_heads=1, n_blocks=1)\n    &gt;&gt;&gt; out = model(X)\n    \"\"\"\n\n    @alias(\"input_dim\", [\"embed_dim\", \"d_model\"])\n    @alias(\"seq_length\", [\"max_length\", \"maxlen\"])\n    def __init__(\n        self,\n        *,\n        vocab_size: int,\n        seq_length: int,\n        input_dim: int,\n        n_heads: int,\n        n_blocks: int,\n        attn_dropout: float = 0.1,\n        ff_dropout: float = 0.1,\n        ff_factor: int = 4,\n        activation: str = \"gelu\",\n        use_linear_attention: bool = False,\n        use_flash_attention: bool = False,\n        padding_idx: int = 0,\n        with_cls_token: bool = False,\n        with_pos_encoding: bool = True,\n        pos_encoding_dropout: float = 0.1,\n        pos_encoder: Optional[nn.Module] = None,\n    ):\n        super().__init__()\n\n        self.input_dim = input_dim\n        self.seq_length = seq_length\n        self.n_heads = n_heads\n        self.n_blocks = n_blocks\n        self.attn_dropout = attn_dropout\n        self.ff_dropout = ff_dropout\n        self.ff_factor = ff_factor\n        self.activation = activation\n        self.use_linear_attention = use_linear_attention\n        self.use_flash_attention = use_flash_attention\n        self.padding_idx = padding_idx\n        self.with_cls_token = with_cls_token\n        self.with_pos_encoding = with_pos_encoding\n        self.pos_encoding_dropout = pos_encoding_dropout\n\n        self.embedding = nn.Embedding(\n            vocab_size, input_dim, padding_idx=self.padding_idx\n        )\n\n        if with_pos_encoding:\n            if pos_encoder is not None:\n                self.pos_encoder: Union[nn.Module, nn.Identity, PositionalEncoding] = (\n                    pos_encoder\n                )\n            else:\n                self.pos_encoder = PositionalEncoding(\n                    input_dim, pos_encoding_dropout, seq_length\n                )\n        else:\n            self.pos_encoder = nn.Identity()\n\n        self.encoder = nn.Sequential()\n        for i in range(n_blocks):\n            self.encoder.add_module(\n                \"transformer_block\" + str(i),\n                TransformerEncoder(\n                    input_dim,\n                    n_heads,\n                    False,  # use_qkv_bias\n                    attn_dropout,\n                    ff_dropout,\n                    ff_factor,\n                    activation,\n                    use_linear_attention,\n                    use_flash_attention,\n                ),\n            )\n\n    def forward(self, X: Tensor) -&gt; Tensor:\n        x = self.embedding(X.long())\n        x = self.pos_encoder(x)\n        x = self.encoder(x)\n        if self.with_cls_token:\n            x = x[:, 0, :]\n        else:\n            x = x.flatten(1)\n        return x\n\n    @property\n    def output_dim(self) -&gt; int:\n        if self.with_cls_token:\n            output_dim = self.input_dim\n        else:\n            output_dim = self.input_dim * self.seq_length\n        return output_dim\n</code></pre>"},{"location":"pytorch-widedeep/the_rec_module.html#pytorch_widedeep.models.rec.dcn.DeepCrossNetwork","title":"DeepCrossNetwork","text":"<p>               Bases: <code>BaseTabularModelWithoutAttention</code></p> <p>Defines a <code>DeepCrossNetwork</code> model that can be used as the <code>deeptabular</code> component of a Wide &amp; Deep model or independently by itself.</p> <p>This class implements the Deep &amp; Cross Network for Ad Click Predictions architecture, which automatically combines features to generate feature interactions in an explicit fashion and at each layer.</p> <p>The cross layer implements the following equation:</p> \\[x_{l+1} = x_0 \\odot (W_l x_l + b_l) + x_l\\] <p>where:</p> <ul> <li>\\(\\odot\\) represents element-wise multiplication</li> <li>\\(x_l\\), \\(x_{l+1}\\) are the outputs from the \\(l^{th}\\) and \\((l+1)^{th}\\) cross layers</li> <li>\\(W_l\\), \\(b_l\\) are the weight and bias parameters</li> </ul> <p>Parameters:</p> Name Type Description Default <code>column_idx</code> <code>Dict[str, int]</code> <p>Dict containing the index of the columns that will be passed through the <code>DeepCrossNetwork</code> model. Required to slice the tensors. e.g. {'education': 0, 'relationship': 1, 'workclass': 2, ...}.</p> required <code>n_cross_layers</code> <code>int</code> <p>Number of cross layers in the cross network</p> <code>3</code> <code>cat_embed_input</code> <code>Optional[List[Tuple[str, int, int]]]</code> <p>List of Tuples with the column name, number of unique values and embedding dimension. e.g. [(education, 11, 32), ...]</p> <code>None</code> <code>cat_embed_dropout</code> <code>Optional[float]</code> <p>Categorical embeddings dropout. If <code>None</code>, it will default to 0.</p> <code>None</code> <code>use_cat_bias</code> <code>Optional[bool]</code> <p>Boolean indicating if bias will be used for the categorical embeddings</p> <code>None</code> <code>cat_embed_activation</code> <code>Optional[str]</code> <p>Activation function for the categorical embeddings</p> <code>None</code> <code>continuous_cols</code> <code>Optional[List[str]]</code> <p>List with the name of the numeric (aka continuous) columns</p> <code>None</code> <code>cont_norm_layer</code> <code>Optional[Literal['batchnorm', 'layernorm']]</code> <p>Type of normalization layer applied to the continuous features. Options are: 'layernorm' and 'batchnorm'. if <code>None</code>, no normalization layer will be used.</p> <code>None</code> <code>embed_continuous_method</code> <code>Optional[Literal['standard', 'piecewise', 'periodic']]</code> <p>Method to use to embed the continuous features. Options are: 'standard', 'periodic' or 'piecewise'. The 'standard' embedding method is based on the FT-Transformer implementation presented in the paper: Revisiting Deep Learning Models for Tabular Data. The 'periodic' and_'piecewise'_ methods were presented in the paper: On Embeddings for Numerical Features in Tabular Deep Learning. Please, read the papers for details.</p> <code>None</code> <code>cont_embed_dropout</code> <code>Optional[float]</code> <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p> <code>None</code> <code>cont_embed_activation</code> <code>Optional[str]</code> <p>Activation function for the continuous embeddings if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported. If <code>None</code>, no activation function will be applied.</p> <code>None</code> <code>quantization_setup</code> <code>Optional[Dict[str, List[float]]]</code> <p>This parameter is used when the 'piecewise' method is used to embed the continuous cols. It is a dict where keys are the name of the continuous columns and values are lists with the boundaries for the quantization of the continuous_cols. See the examples for details. If If the 'piecewise' method is used, this parameter is required.</p> <code>None</code> <code>n_frequencies</code> <code>Optional[int]</code> <p>This is the so called 'k' in their paper On Embeddings for Numerical Features in Tabular Deep Learning, and is the number of 'frequencies' that will be used to represent each continuous column. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>sigma</code> <code>Optional[float]</code> <p>This is the sigma parameter in the paper mentioned when describing the previous parameters and it is used to initialise the 'frequency weights'. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>share_last_layer</code> <code>Optional[bool]</code> <p>This parameter is not present in the before mentioned paper but it is implemented in the official repo. If <code>True</code> the linear layer that turns the frequencies into embeddings will be shared across the continuous columns. If <code>False</code> a different linear layer will be used for each continuous column. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>full_embed_dropout</code> <code>Optional[bool]</code> <p>If <code>True</code>, the full embedding corresponding to a column will be masked out/dropout. If <code>None</code>, it will default to <code>False</code>.</p> <code>None</code> <code>mlp_hidden_dims</code> <code>List[int]</code> <p>List with the number of neurons per dense layer in the mlp.</p> <code>[200, 100]</code> <code>mlp_activation</code> <code>str</code> <p>Activation function for the dense layers of the MLP. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> <code>'relu'</code> <code>mlp_dropout</code> <code>Union[float, List[float]]</code> <p>float or List of floats with the dropout between the dense layers. e.g: [0.5,0.5]</p> <code>0.1</code> <code>mlp_batchnorm</code> <code>bool</code> <p>Boolean indicating whether or not batch normalization will be applied to the dense layers</p> <code>False</code> <code>mlp_batchnorm_last</code> <code>bool</code> <p>Boolean indicating whether or not batch normalization will be applied to the last of the dense layers</p> <code>False</code> <code>mlp_linear_first</code> <code>bool</code> <p>Boolean indicating the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code></p> <code>True</code> <p>Attributes:</p> Name Type Description <code>cross_network</code> <code>Module</code> <p>The cross network component</p> <code>deep_network</code> <code>Module</code> <p>The deep network component (MLP)</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models.rec import DeepCrossNetwork\n&gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n&gt;&gt;&gt; colnames = [\"a\", \"b\", \"c\", \"d\", \"e\"]\n&gt;&gt;&gt; cat_embed_input = [(u, i, j) for u, i, j in zip(colnames[:4], [4] * 4, [8] * 4)]\n&gt;&gt;&gt; column_idx = {k: v for v, k in enumerate(colnames)}\n&gt;&gt;&gt; model = DeepCrossNetwork(\n...     column_idx=column_idx,\n...     cat_embed_input=cat_embed_input,\n...     continuous_cols=[\"e\"],\n...     n_cross_layers=2,\n...     mlp_hidden_dims=[16, 8]\n... )\n&gt;&gt;&gt; out = model(X_tab)\n</code></pre> Source code in <code>pytorch_widedeep/models/rec/dcn.py</code> <pre><code>class DeepCrossNetwork(BaseTabularModelWithoutAttention):\n    r\"\"\"Defines a `DeepCrossNetwork` model that can be used as the `deeptabular`\n    component of a Wide &amp; Deep model or independently by itself.\n\n    This class implements the [Deep &amp; Cross Network for Ad Click Predictions](https://arxiv.org/abs/1708.05123)\n    architecture, which automatically combines features to generate feature interactions\n    in an explicit fashion and at each layer.\n\n    The cross layer implements the following equation:\n\n    $$x_{l+1} = x_0 \\odot (W_l x_l + b_l) + x_l$$\n\n    where:\n\n    * $\\odot$ represents element-wise multiplication\n    * $x_l$, $x_{l+1}$ are the outputs from the $l^{th}$ and $(l+1)^{th}$ cross layers\n    * $W_l$, $b_l$ are the weight and bias parameters\n\n    Parameters\n    ----------\n    column_idx: Dict\n        Dict containing the index of the columns that will be passed through\n        the `DeepCrossNetwork` model. Required to slice the tensors. e.g.\n        _{'education': 0, 'relationship': 1, 'workclass': 2, ...}_.\n    n_cross_layers: int, default = 3\n        Number of cross layers in the cross network\n    cat_embed_input: List, Optional, default = None\n        List of Tuples with the column name, number of unique values and\n        embedding dimension. e.g. _[(education, 11, 32), ...]_\n    cat_embed_dropout: float, Optional, default = None\n        Categorical embeddings dropout. If `None`, it will default to 0.\n    use_cat_bias: bool, Optional, default = None,\n        Boolean indicating if bias will be used for the categorical embeddings\n    cat_embed_activation: Optional, str, default = None,\n        Activation function for the categorical embeddings\n    continuous_cols : Optional[List[str]], default=None\n        List with the name of the numeric (aka continuous) columns\n    cont_norm_layer : Optional[Literal[\"batchnorm\", \"layernorm\"]], default=None\n        Type of normalization layer applied to the continuous features.\n        Options are: _'layernorm'_ and _'batchnorm'_. if `None`, no\n        normalization layer will be used.\n    embed_continuous_method: Optional[Literal[\"piecewise\", \"periodic\", \"standard\"]], default=\"standard\"\n        Method to use to embed the continuous features. Options are:\n        _'standard'_, _'periodic'_ or _'piecewise'_. The _'standard'_\n        embedding method is based on the FT-Transformer implementation\n        presented in the paper: [Revisiting Deep Learning Models for\n        Tabular Data](https://arxiv.org/abs/2106.11959v5). The _'periodic'_\n        and_'piecewise'_ methods were presented in the paper: [On Embeddings for\n        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556).\n        Please, read the papers for details.\n    cont_embed_dropout : Optional[float], default=None\n        Dropout for the continuous embeddings. If `None`, it will default to 0.0\n    cont_embed_activation : Optional[str], default=None\n        Activation function for the continuous embeddings if any. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported.\n        If `None`, no activation function will be applied.\n    quantization_setup : Optional[Dict[str, List[float]]], default=None\n        This parameter is used when the _'piecewise'_ method is used to embed\n        the continuous cols. It is a dict where keys are the name of the continuous\n        columns and values are lists with the boundaries for the quantization\n        of the continuous_cols. See the examples for details. If\n        If the _'piecewise'_ method is used, this parameter is required.\n    n_frequencies : Optional[int], default=None\n        This is the so called _'k'_ in their paper [On Embeddings for\n        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556),\n        and is the number of 'frequencies' that will be used to represent each\n        continuous column. See their Eq 2 in the paper for details. If\n        the _'periodic'_ method is used, this parameter is required.\n    sigma : Optional[float], default=None\n        This is the sigma parameter in the paper mentioned when describing the\n        previous parameters and it is used to initialise the 'frequency\n        weights'. See their Eq 2 in the paper for details. If\n        the _'periodic'_ method is used, this parameter is required.\n    share_last_layer : Optional[bool], default=None\n        This parameter is not present in the before mentioned paper but it is implemented in\n        the [official repo](https://github.com/yandex-research/rtdl-num-embeddings/tree/main).\n        If `True` the linear layer that turns the frequencies into embeddings\n        will be shared across the continuous columns. If `False` a different\n        linear layer will be used for each continuous column.\n        If the _'periodic'_ method is used, this parameter is required.\n    full_embed_dropout: bool, Optional, default = None,\n        If `True`, the full embedding corresponding to a column will be masked\n        out/dropout. If `None`, it will default to `False`.\n    mlp_hidden_dims: List, default = [200, 100]\n        List with the number of neurons per dense layer in the mlp.\n    mlp_activation: str, default = \"relu\"\n        Activation function for the dense layers of the MLP. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported\n    mlp_dropout: float or List, default = 0.1\n        float or List of floats with the dropout between the dense layers.\n        e.g: _[0.5,0.5]_\n    mlp_batchnorm: bool, default = False\n        Boolean indicating whether or not batch normalization will be applied\n        to the dense layers\n    mlp_batchnorm_last: bool, default = False\n        Boolean indicating whether or not batch normalization will be applied\n        to the last of the dense layers\n    mlp_linear_first: bool, default = False\n        Boolean indicating the order of the operations in the dense\n        layer. If `True: [LIN -&gt; ACT -&gt; BN -&gt; DP]`. If `False: [BN -&gt; DP -&gt;\n        LIN -&gt; ACT]`\n\n    Attributes\n    ----------\n    cross_network: nn.Module\n        The cross network component\n    deep_network: nn.Module\n        The deep network component (MLP)\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.models.rec import DeepCrossNetwork\n    &gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n    &gt;&gt;&gt; colnames = [\"a\", \"b\", \"c\", \"d\", \"e\"]\n    &gt;&gt;&gt; cat_embed_input = [(u, i, j) for u, i, j in zip(colnames[:4], [4] * 4, [8] * 4)]\n    &gt;&gt;&gt; column_idx = {k: v for v, k in enumerate(colnames)}\n    &gt;&gt;&gt; model = DeepCrossNetwork(\n    ...     column_idx=column_idx,\n    ...     cat_embed_input=cat_embed_input,\n    ...     continuous_cols=[\"e\"],\n    ...     n_cross_layers=2,\n    ...     mlp_hidden_dims=[16, 8]\n    ... )\n    &gt;&gt;&gt; out = model(X_tab)\n    \"\"\"\n\n    def __init__(\n        self,\n        *,\n        column_idx: Dict[str, int],\n        n_cross_layers: int = 3,\n        cat_embed_input: Optional[List[Tuple[str, int, int]]] = None,\n        cat_embed_dropout: Optional[float] = None,\n        use_cat_bias: Optional[bool] = None,\n        cat_embed_activation: Optional[str] = None,\n        continuous_cols: Optional[List[str]] = None,\n        cont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\n        embed_continuous: Optional[bool] = None,\n        embed_continuous_method: Optional[\n            Literal[\"standard\", \"piecewise\", \"periodic\"]\n        ] = None,\n        cont_embed_dim: Optional[int] = None,\n        cont_embed_dropout: Optional[float] = None,\n        cont_embed_activation: Optional[str] = None,\n        quantization_setup: Optional[Dict[str, List[float]]] = None,\n        n_frequencies: Optional[int] = None,\n        sigma: Optional[float] = None,\n        share_last_layer: Optional[bool] = None,\n        full_embed_dropout: Optional[bool] = None,\n        mlp_hidden_dims: List[int] = [200, 100],\n        mlp_activation: str = \"relu\",\n        mlp_dropout: Union[float, List[float]] = 0.1,\n        mlp_batchnorm: bool = False,\n        mlp_batchnorm_last: bool = False,\n        mlp_linear_first: bool = True,\n    ):\n        super(DeepCrossNetwork, self).__init__(\n            column_idx=column_idx,\n            cat_embed_input=cat_embed_input,\n            cat_embed_dropout=cat_embed_dropout,\n            use_cat_bias=use_cat_bias,\n            cat_embed_activation=cat_embed_activation,\n            continuous_cols=continuous_cols,\n            cont_norm_layer=cont_norm_layer,\n            embed_continuous=embed_continuous,\n            embed_continuous_method=embed_continuous_method,\n            cont_embed_dim=cont_embed_dim,\n            cont_embed_dropout=cont_embed_dropout,\n            cont_embed_activation=cont_embed_activation,\n            quantization_setup=quantization_setup,\n            n_frequencies=n_frequencies,\n            sigma=sigma,\n            share_last_layer=share_last_layer,\n            full_embed_dropout=full_embed_dropout,\n        )\n\n        self.mlp_hidden_dims = mlp_hidden_dims\n        self.mlp_activation = mlp_activation\n        self.mlp_dropout = mlp_dropout\n        self.mlp_batchnorm = mlp_batchnorm\n        self.mlp_batchnorm_last = mlp_batchnorm_last\n        self.mlp_linear_first = mlp_linear_first\n\n        embeddings_output_dim = self.cat_out_dim + self.cont_out_dim\n        self.deep_network = MLP(\n            [embeddings_output_dim] + self.mlp_hidden_dims,\n            self.mlp_activation,\n            self.mlp_dropout,\n            self.mlp_batchnorm,\n            self.mlp_batchnorm_last,\n            self.mlp_linear_first,\n        )\n        self.cross_network = CrossNetwork(embeddings_output_dim, n_cross_layers)\n\n    def forward(self, X: torch.Tensor) -&gt; torch.Tensor:\n        x = self._get_embeddings(X)\n        cross_output = self.cross_network(x)\n        deep_output = self.deep_network(x)\n        return torch.cat([cross_output, deep_output], dim=1)\n\n    @property\n    def output_dim(self) -&gt; int:\n        return self.mlp_hidden_dims[-1] + self.cat_out_dim + self.cont_out_dim\n</code></pre>"},{"location":"pytorch-widedeep/the_rec_module.html#pytorch_widedeep.models.rec.dcnv2.DeepCrossNetworkV2","title":"DeepCrossNetworkV2","text":"<p>               Bases: <code>BaseTabularModelWithoutAttention</code></p> <p>Defines a <code>DeepCrossNetworkV2</code> model that can be used as the <code>deeptabular</code> component of a Wide &amp; Deep model or independently by itself.</p> <p>This class implements the DCN V2: Improved Deep &amp; Cross Network and Practical Lessons for Web-scale Learning to Rank Systems architecture, which enhances the original DCN by introducing a more expressive cross network that uses multiple experts and matrix decomposition techniques to improve model capacity while maintaining computational efficiency.</p> <p>The cross layer implements the following equation:</p> \\[E_i(x_l) = x_0 \\odot (U_l^i \\cdot g(C_l^i \\cdot g(V_l^iT x_l)) + b_l)\\] <p>where:</p> <ul> <li>\\(\\odot\\) represents element-wise multiplication</li> <li>\\(U_l^i\\), \\(C_l^i\\), \\(V_l^i\\) are the decomposed weight matrices for expert \\(i\\) at layer \\(l\\)</li> <li>\\(g\\) is the activation function (ReLU)</li> <li>\\(b_l\\) is the bias term</li> </ul> <p>Parameters:</p> Name Type Description Default <code>column_idx</code> <code>Dict[str, int]</code> <p>Dict containing the index of the columns that will be passed through the <code>DeepCrossNetworkV2</code> model. Required to slice the tensors. e.g. {'education': 0, 'relationship': 1, 'workclass': 2, ...}.</p> required <code>num_cross_layers</code> <code>int</code> <p>Number of cross layers in the cross network</p> <code>2</code> <code>low_rank</code> <code>Optional[int]</code> <p>Rank of the weight matrix decomposition. If None, full-rank weights are used</p> <code>None</code> <code>num_experts</code> <code>int</code> <p>Number of expert networks in mixture of experts</p> <code>2</code> <code>expert_dropout</code> <code>float</code> <p>Dropout rate for expert outputs</p> <code>0.0</code> <code>structure</code> <code>Literal['stacked', 'parallel']</code> <p>Structure of the model. Either 'parallel' or 'stacked'. If 'parallel', the output will be the concatenation of the cross network and deep network outputs. If 'stacked', the cross network output will be fed into the deep network.</p> <code>'parallel'</code> <code>cat_embed_input</code> <code>Optional[List[Tuple[str, int, int]]]</code> <p>List of Tuples with the column name, number of unique values and embedding dimension. e.g. [(education, 11, 32), ...]</p> <code>None</code> <code>cat_embed_dropout</code> <code>Optional[float]</code> <p>Categorical embeddings dropout. If <code>None</code>, it will default to 0.</p> <code>None</code> <code>use_cat_bias</code> <code>Optional[bool]</code> <p>Boolean indicating if bias will be used for the categorical embeddings</p> <code>None</code> <code>cat_embed_activation</code> <code>Optional[str]</code> <p>Activation function for the categorical embeddings</p> <code>None</code> <code>continuous_cols</code> <code>Optional[List[str]]</code> <p>List with the name of the numeric (aka continuous) columns</p> <code>None</code> <code>cont_norm_layer</code> <code>Optional[Literal['batchnorm', 'layernorm']]</code> <p>Type of normalization layer applied to the continuous features. Options are: 'layernorm' and 'batchnorm'. if <code>None</code>, no normalization layer will be used.</p> <code>None</code> <code>embed_continuous_method</code> <code>Optional[Literal['standard', 'piecewise', 'periodic']]</code> <p>Method to use to embed the continuous features. Options are: 'standard', 'periodic' or 'piecewise'. The 'standard' embedding method is based on the FT-Transformer implementation presented in the paper: Revisiting Deep Learning Models for Tabular Data. The 'periodic' and_'piecewise'_ methods were presented in the paper: On Embeddings for Numerical Features in Tabular Deep Learning. Please, read the papers for details.</p> <code>None</code> <code>cont_embed_dropout</code> <code>Optional[float]</code> <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p> <code>None</code> <code>cont_embed_activation</code> <code>Optional[str]</code> <p>Activation function for the continuous embeddings if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported. If <code>None</code>, no activation function will be applied.</p> <code>None</code> <code>quantization_setup</code> <code>Optional[Dict[str, List[float]]]</code> <p>This parameter is used when the 'piecewise' method is used to embed the continuous cols. It is a dict where keys are the name of the continuous columns and values are lists with the boundaries for the quantization of the continuous_cols. See the examples for details. If If the 'piecewise' method is used, this parameter is required.</p> <code>None</code> <code>n_frequencies</code> <code>Optional[int]</code> <p>This is the so called 'k' in their paper On Embeddings for Numerical Features in Tabular Deep Learning, and is the number of 'frequencies' that will be used to represent each continuous column. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>sigma</code> <code>Optional[float]</code> <p>This is the sigma parameter in the paper mentioned when describing the previous parameters and it is used to initialise the 'frequency weights'. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>share_last_layer</code> <code>Optional[bool]</code> <p>This parameter is not present in the before mentioned paper but it is implemented in the official repo. If <code>True</code> the linear layer that turns the frequencies into embeddings will be shared across the continuous columns. If <code>False</code> a different linear layer will be used for each continuous column. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>full_embed_dropout</code> <code>Optional[bool]</code> <p>If <code>True</code>, the full embedding corresponding to a column will be masked out/dropout. If <code>None</code>, it will default to <code>False</code>.</p> <code>None</code> <code>mlp_hidden_dims</code> <code>List[int]</code> <p>List with the number of neurons per dense layer in the mlp.</p> <code>[200, 100]</code> <code>mlp_activation</code> <code>str</code> <p>Activation function for the dense layers of the MLP. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> <code>'relu'</code> <code>mlp_dropout</code> <code>Union[float, List[float]]</code> <p>float or List of floats with the dropout between the dense layers. e.g: [0.5,0.5]</p> <code>0.1</code> <code>mlp_batchnorm</code> <code>bool</code> <p>Boolean indicating whether or not batch normalization will be applied to the dense layers</p> <code>False</code> <code>mlp_batchnorm_last</code> <code>bool</code> <p>Boolean indicating whether or not batch normalization will be applied to the last of the dense layers</p> <code>False</code> <code>mlp_linear_first</code> <code>bool</code> <p>Boolean indicating the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code></p> <code>True</code> <p>Attributes:</p> Name Type Description <code>cross_network</code> <code>Module</code> <p>The cross network component with mixture of experts</p> <code>deep_network</code> <code>Module</code> <p>The deep network component (MLP)</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models.rec import DeepCrossNetworkV2\n&gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n&gt;&gt;&gt; colnames = [\"a\", \"b\", \"c\", \"d\", \"e\"]\n&gt;&gt;&gt; cat_embed_input = [(u, i, j) for u, i, j in zip(colnames[:4], [4] * 4, [8] * 4)]\n&gt;&gt;&gt; column_idx = {k: v for v, k in enumerate(colnames)}\n&gt;&gt;&gt; model = DeepCrossNetworkV2(\n...     column_idx=column_idx,\n...     cat_embed_input=cat_embed_input,\n...     continuous_cols=[\"e\"],\n...     num_cross_layers=2,\n...     low_rank=32,\n...     num_experts=4,\n...     mlp_hidden_dims=[16, 8]\n... )\n&gt;&gt;&gt; out = model(X_tab)\n</code></pre> Source code in <code>pytorch_widedeep/models/rec/dcnv2.py</code> <pre><code>class DeepCrossNetworkV2(BaseTabularModelWithoutAttention):\n    r\"\"\"Defines a `DeepCrossNetworkV2` model that can be used as the `deeptabular`\n    component of a Wide &amp; Deep model or independently by itself.\n\n    This class implements the\n    [DCN V2: Improved Deep &amp; Cross Network and Practical Lessons for Web-scale Learning to Rank Systems](https://arxiv.org/pdf/2008.13535)\n    architecture, which enhances the original DCN by introducing a more expressive cross\n    network that uses multiple experts and matrix decomposition techniques to improve\n    model capacity while maintaining computational efficiency.\n\n    The cross layer implements the following equation:\n\n    $$E_i(x_l) = x_0 \\odot (U_l^i \\cdot g(C_l^i \\cdot g(V_l^iT x_l)) + b_l)$$\n\n    where:\n\n    * $\\odot$ represents element-wise multiplication\n    * $U_l^i$, $C_l^i$, $V_l^i$ are the decomposed weight matrices for expert $i$ at layer $l$\n    * $g$ is the activation function (ReLU)\n    * $b_l$ is the bias term\n\n    Parameters\n    ----------\n    column_idx: Dict\n        Dict containing the index of the columns that will be passed through\n        the `DeepCrossNetworkV2` model. Required to slice the tensors. e.g.\n        _{'education': 0, 'relationship': 1, 'workclass': 2, ...}_.\n    num_cross_layers: int, default = 2\n        Number of cross layers in the cross network\n    low_rank: int, Optional, default = None\n        Rank of the weight matrix decomposition. If None, full-rank weights are used\n    num_experts: int, default = 2\n        Number of expert networks in mixture of experts\n    expert_dropout: float, default = 0.0\n        Dropout rate for expert outputs\n    structure: str, default = \"parallel\"\n        Structure of the model. Either _'parallel'_ or _'stacked'_. If _'parallel'_,\n        the output will be the concatenation of the cross network and deep network\n        outputs. If _'stacked'_, the cross network output will be fed into the deep\n        network.\n    cat_embed_input: List, Optional, default = None\n        List of Tuples with the column name, number of unique values and\n        embedding dimension. e.g. _[(education, 11, 32), ...]_\n    cat_embed_dropout: float, Optional, default = None\n        Categorical embeddings dropout. If `None`, it will default to 0.\n    use_cat_bias: bool, Optional, default = None,\n        Boolean indicating if bias will be used for the categorical embeddings\n    cat_embed_activation: Optional, str, default = None,\n        Activation function for the categorical embeddings\n    continuous_cols : Optional[List[str]], default=None\n        List with the name of the numeric (aka continuous) columns\n    cont_norm_layer : Optional[Literal[\"batchnorm\", \"layernorm\"]], default=None\n        Type of normalization layer applied to the continuous features.\n        Options are: _'layernorm'_ and _'batchnorm'_. if `None`, no\n        normalization layer will be used.\n    embed_continuous_method: Optional[Literal[\"piecewise\", \"periodic\", \"standard\"]], default=\"standard\"\n        Method to use to embed the continuous features. Options are:\n        _'standard'_, _'periodic'_ or _'piecewise'_. The _'standard'_\n        embedding method is based on the FT-Transformer implementation\n        presented in the paper: [Revisiting Deep Learning Models for\n        Tabular Data](https://arxiv.org/abs/2106.11959v5). The _'periodic'_\n        and_'piecewise'_ methods were presented in the paper: [On Embeddings for\n        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556).\n        Please, read the papers for details.\n    cont_embed_dropout : Optional[float], default=None\n        Dropout for the continuous embeddings. If `None`, it will default to 0.0\n    cont_embed_activation : Optional[str], default=None\n        Activation function for the continuous embeddings if any. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported.\n        If `None`, no activation function will be applied.\n    quantization_setup : Optional[Dict[str, List[float]]], default=None\n        This parameter is used when the _'piecewise'_ method is used to embed\n        the continuous cols. It is a dict where keys are the name of the continuous\n        columns and values are lists with the boundaries for the quantization\n        of the continuous_cols. See the examples for details. If\n        If the _'piecewise'_ method is used, this parameter is required.\n    n_frequencies : Optional[int], default=None\n        This is the so called _'k'_ in their paper [On Embeddings for\n        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556),\n        and is the number of 'frequencies' that will be used to represent each\n        continuous column. See their Eq 2 in the paper for details. If\n        the _'periodic'_ method is used, this parameter is required.\n    sigma : Optional[float], default=None\n        This is the sigma parameter in the paper mentioned when describing the\n        previous parameters and it is used to initialise the 'frequency\n        weights'. See their Eq 2 in the paper for details. If\n        the _'periodic'_ method is used, this parameter is required.\n    share_last_layer : Optional[bool], default=None\n        This parameter is not present in the before mentioned paper but it is implemented in\n        the [official repo](https://github.com/yandex-research/rtdl-num-embeddings/tree/main).\n        If `True` the linear layer that turns the frequencies into embeddings\n        will be shared across the continuous columns. If `False` a different\n        linear layer will be used for each continuous column.\n        If the _'periodic'_ method is used, this parameter is required.\n    full_embed_dropout: bool, Optional, default = None,\n        If `True`, the full embedding corresponding to a column will be masked\n        out/dropout. If `None`, it will default to `False`.\n    mlp_hidden_dims: List, default = [200, 100]\n        List with the number of neurons per dense layer in the mlp.\n    mlp_activation: str, default = \"relu\"\n        Activation function for the dense layers of the MLP. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported\n    mlp_dropout: float or List, default = 0.1\n        float or List of floats with the dropout between the dense layers.\n        e.g: _[0.5,0.5]_\n    mlp_batchnorm: bool, default = False\n        Boolean indicating whether or not batch normalization will be applied\n        to the dense layers\n    mlp_batchnorm_last: bool, default = False\n        Boolean indicating whether or not batch normalization will be applied\n        to the last of the dense layers\n    mlp_linear_first: bool, default = False\n        Boolean indicating the order of the operations in the dense\n        layer. If `True: [LIN -&gt; ACT -&gt; BN -&gt; DP]`. If `False: [BN -&gt; DP -&gt;\n        LIN -&gt; ACT]`\n    Attributes\n    ----------\n    cross_network: nn.Module\n        The cross network component with mixture of experts\n    deep_network: nn.Module\n        The deep network component (MLP)\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.models.rec import DeepCrossNetworkV2\n    &gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n    &gt;&gt;&gt; colnames = [\"a\", \"b\", \"c\", \"d\", \"e\"]\n    &gt;&gt;&gt; cat_embed_input = [(u, i, j) for u, i, j in zip(colnames[:4], [4] * 4, [8] * 4)]\n    &gt;&gt;&gt; column_idx = {k: v for v, k in enumerate(colnames)}\n    &gt;&gt;&gt; model = DeepCrossNetworkV2(\n    ...     column_idx=column_idx,\n    ...     cat_embed_input=cat_embed_input,\n    ...     continuous_cols=[\"e\"],\n    ...     num_cross_layers=2,\n    ...     low_rank=32,\n    ...     num_experts=4,\n    ...     mlp_hidden_dims=[16, 8]\n    ... )\n    &gt;&gt;&gt; out = model(X_tab)\n    \"\"\"\n\n    def __init__(\n        self,\n        *,\n        column_idx: Dict[str, int],\n        num_cross_layers: int = 2,\n        low_rank: Optional[int] = None,\n        num_experts: int = 2,\n        expert_dropout: float = 0.0,\n        structure: Literal[\"stacked\", \"parallel\"] = \"parallel\",\n        cat_embed_input: Optional[List[Tuple[str, int, int]]] = None,\n        cat_embed_dropout: Optional[float] = None,\n        use_cat_bias: Optional[bool] = None,\n        cat_embed_activation: Optional[str] = None,\n        continuous_cols: Optional[List[str]] = None,\n        cont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\n        embed_continuous: Optional[bool] = None,\n        embed_continuous_method: Optional[\n            Literal[\"standard\", \"piecewise\", \"periodic\"]\n        ] = None,\n        cont_embed_dim: Optional[int] = None,\n        cont_embed_dropout: Optional[float] = None,\n        cont_embed_activation: Optional[str] = None,\n        quantization_setup: Optional[Dict[str, List[float]]] = None,\n        n_frequencies: Optional[int] = None,\n        sigma: Optional[float] = None,\n        share_last_layer: Optional[bool] = None,\n        full_embed_dropout: Optional[bool] = None,\n        mlp_hidden_dims: List[int] = [200, 100],\n        mlp_activation: str = \"relu\",\n        mlp_dropout: Union[float, List[float]] = 0.1,\n        mlp_batchnorm: bool = False,\n        mlp_batchnorm_last: bool = False,\n        mlp_linear_first: bool = True,\n    ):\n        super(DeepCrossNetworkV2, self).__init__(\n            column_idx=column_idx,\n            cat_embed_input=cat_embed_input,\n            cat_embed_dropout=cat_embed_dropout,\n            use_cat_bias=use_cat_bias,\n            cat_embed_activation=cat_embed_activation,\n            continuous_cols=continuous_cols,\n            cont_norm_layer=cont_norm_layer,\n            embed_continuous=embed_continuous,\n            embed_continuous_method=embed_continuous_method,\n            cont_embed_dim=cont_embed_dim,\n            cont_embed_dropout=cont_embed_dropout,\n            cont_embed_activation=cont_embed_activation,\n            quantization_setup=quantization_setup,\n            n_frequencies=n_frequencies,\n            sigma=sigma,\n            share_last_layer=share_last_layer,\n            full_embed_dropout=full_embed_dropout,\n        )\n\n        self.num_cross_layers = num_cross_layers\n        self.num_experts = num_experts\n        self.low_rank = low_rank\n        self.expert_dropout = expert_dropout\n        self.structure = structure\n\n        self.mlp_hidden_dims = mlp_hidden_dims\n        self.mlp_activation = mlp_activation\n        self.mlp_dropout = mlp_dropout\n        self.mlp_batchnorm = mlp_batchnorm\n        self.mlp_batchnorm_last = mlp_batchnorm_last\n        self.mlp_linear_first = mlp_linear_first\n\n        embeddings_output_dim = self.cat_out_dim + self.cont_out_dim\n        self.deep_network = MLP(\n            [embeddings_output_dim] + self.mlp_hidden_dims,\n            self.mlp_activation,\n            self.mlp_dropout,\n            self.mlp_batchnorm,\n            self.mlp_batchnorm_last,\n            self.mlp_linear_first,\n        )\n\n        self.cross_network = CrossNetworkV2(\n            input_dim=embeddings_output_dim,\n            num_layers=self.num_cross_layers,\n            low_rank=self.low_rank,\n            num_experts=self.num_experts,\n            expert_dropout=self.expert_dropout,\n        )\n\n    def forward(self, X: torch.Tensor) -&gt; torch.Tensor:\n        x = self._get_embeddings(X)\n        cross_output = self.cross_network(x)\n\n        if self.structure == \"stacked\":\n            deep_output = self.deep_network(cross_output)\n            return deep_output\n        else:  # parallel\n            deep_output = self.deep_network(x)\n            return torch.cat([cross_output, deep_output], dim=1)\n\n    @property\n    def output_dim(self) -&gt; int:\n        if self.structure == \"stacked\":\n            return self.mlp_hidden_dims[-1]\n        else:  # parallel\n            return self.mlp_hidden_dims[-1] + self.cat_out_dim + self.cont_out_dim\n</code></pre>"},{"location":"pytorch-widedeep/the_rec_module.html#pytorch_widedeep.models.rec.gdcn.GatedDeepCrossNetwork","title":"GatedDeepCrossNetwork","text":"<p>               Bases: <code>BaseTabularModelWithoutAttention</code></p> <p>Defines a <code>GatedDeepCrossNetwork</code> model that can be used as the <code>deeptabular</code> component of a Wide &amp; Deep model or independently by itself.</p> <p>This class implements the Gated Deep &amp; Cross Network (GDCN) architecture as described in the paper Towards Deeper, Lighter and Interpretable Cross Network for CTR Prediction. The GDCN enhances the original DCN by introducing a gating mechanism in the cross network. The gating mechanism controls feature interactions by learning which interactions are more important.</p> <p>The cross layer implements the following equation:</p> \\[c_{i+1} = c_0 \\odot (W^c \\times c_i + b) \\odot \\sigma(W^g \\times c_i) + c_i\\] <p>where:</p> <ul> <li>\\(\\odot\\) represents element-wise multiplication</li> <li>\\(W^c\\) and \\(W^g\\) are the cross and gate weight matrices respectively</li> <li>\\(\\sigma\\) is the sigmoid activation function</li> </ul> <p>Parameters:</p> Name Type Description Default <code>column_idx</code> <code>Dict[str, int]</code> <p>Dict containing the index of the columns that will be passed through the <code>GatedDeepCrossNetwork</code> model. Required to slice the tensors. e.g. {'education': 0, 'relationship': 1, 'workclass': 2, ...}.</p> required <code>num_cross_layers</code> <code>int</code> <p>Number of cross layers in the cross network</p> <code>3</code> <code>structure</code> <code>Literal['stacked', 'parallel']</code> <p>Structure of the model. Either 'parallel' or 'stacked'. If 'parallel', the output will be the concatenation of the cross network and deep network outputs. If 'stacked', the cross network output will be fed into the deep network.</p> <code>'parallel'</code> <code>cat_embed_input</code> <code>Optional[List[Tuple[str, int, int]]]</code> <p>List of Tuples with the column name, number of unique values and embedding dimension. e.g. [(education, 11, 32), ...]</p> <code>None</code> <code>cat_embed_dropout</code> <code>Optional[float]</code> <p>Categorical embeddings dropout. If <code>None</code>, it will default to 0.</p> <code>None</code> <code>use_cat_bias</code> <code>Optional[bool]</code> <p>Boolean indicating if bias will be used for the categorical embeddings. If <code>None</code>, it will default to 'False'.</p> <code>None</code> <code>cat_embed_activation</code> <code>Optional[str]</code> <p>Activation function for the categorical embeddings, if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> <code>None</code> <code>continuous_cols</code> <code>Optional[List[str]]</code> <p>List with the name of the numeric (aka continuous) columns</p> <code>None</code> <code>cont_norm_layer</code> <code>Optional[Literal['batchnorm', 'layernorm']]</code> <p>Type of normalization layer applied to the continuous features. Options are: 'layernorm' and 'batchnorm'. if <code>None</code>, no normalization layer will be used.</p> <code>None</code> <code>embed_continuous_method</code> <code>Optional[Literal['standard', 'piecewise', 'periodic']]</code> <p>Method to use to embed the continuous features. Options are: 'standard', 'periodic' or 'piecewise'. The 'standard' embedding method is based on the FT-Transformer implementation presented in the paper: Revisiting Deep Learning Models for Tabular Data. The 'periodic' and_'piecewise'_ methods were presented in the paper: On Embeddings for Numerical Features in Tabular Deep Learning. Please, read the papers for details.</p> <code>None</code> <code>cont_embed_dropout</code> <code>Optional[float]</code> <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p> <code>None</code> <code>cont_embed_activation</code> <code>Optional[str]</code> <p>Activation function for the continuous embeddings if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported. If <code>None</code>, no activation function will be applied.</p> <code>None</code> <code>quantization_setup</code> <code>Optional[Dict[str, List[float]]]</code> <p>This parameter is used when the 'piecewise' method is used to embed the continuous cols. It is a dict where keys are the name of the continuous columns and values are lists with the boundaries for the quantization of the continuous_cols. See the examples for details. If If the 'piecewise' method is used, this parameter is required.</p> <code>None</code> <code>n_frequencies</code> <code>Optional[int]</code> <p>This is the so called 'k' in their paper On Embeddings for Numerical Features in Tabular Deep Learning, and is the number of 'frequencies' that will be used to represent each continuous column. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>sigma</code> <code>Optional[float]</code> <p>This is the sigma parameter in the paper mentioned when describing the previous parameters and it is used to initialise the 'frequency weights'. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>share_last_layer</code> <code>Optional[bool]</code> <p>This parameter is not present in the before mentioned paper but it is implemented in the official repo. If <code>True</code> the linear layer that turns the frequencies into embeddings will be shared across the continuous columns. If <code>False</code> a different linear layer will be used for each continuous column. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>full_embed_dropout</code> <code>Optional[bool]</code> <p>If <code>True</code>, the full embedding corresponding to a column will be masked out/dropout. If <code>None</code>, it will default to <code>False</code>.</p> <code>None</code> <code>mlp_hidden_dims</code> <code>List[int]</code> <p>List with the number of neurons per dense layer in the mlp.</p> <code>[200, 100]</code> <code>mlp_activation</code> <code>str</code> <p>Activation function for the dense layers of the MLP. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> <code>'relu'</code> <code>mlp_dropout</code> <code>Union[float, List[float]]</code> <p>float or List of floats with the dropout between the dense layers. e.g: [0.5,0.5]</p> <code>0.1</code> <code>mlp_batchnorm</code> <code>bool</code> <p>Boolean indicating whether or not batch normalization will be applied to the dense layers</p> <code>False</code> <code>mlp_batchnorm_last</code> <code>bool</code> <p>Boolean indicating whether or not batch normalization will be applied to the last of the dense layers</p> <code>False</code> <code>mlp_linear_first</code> <code>bool</code> <p>Boolean indicating the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code></p> <code>True</code> <p>Attributes:</p> Name Type Description <code>cross_network</code> <code>Module</code> <p>The gated cross network component</p> <code>deep_network</code> <code>Module</code> <p>The deep network component (MLP)</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models.rec import GatedDeepCrossNetwork\n&gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n&gt;&gt;&gt; colnames = [\"a\", \"b\", \"c\", \"d\", \"e\"]\n&gt;&gt;&gt; cat_embed_input = [(u, i, j) for u, i, j in zip(colnames[:4], [4] * 4, [8] * 4)]\n&gt;&gt;&gt; column_idx = {k: v for v, k in enumerate(colnames)}\n&gt;&gt;&gt; model = GatedDeepCrossNetwork(\n...     column_idx=column_idx,\n...     cat_embed_input=cat_embed_input,\n...     continuous_cols=[\"e\"],\n...     num_cross_layers=2,\n...     mlp_hidden_dims=[16, 8]\n... )\n&gt;&gt;&gt; out = model(X_tab)\n</code></pre> Source code in <code>pytorch_widedeep/models/rec/gdcn.py</code> <pre><code>class GatedDeepCrossNetwork(BaseTabularModelWithoutAttention):\n    r\"\"\"Defines a `GatedDeepCrossNetwork` model that can be used as the `deeptabular`\n    component of a Wide &amp; Deep model or independently by itself.\n\n    This class implements the Gated Deep &amp; Cross Network (GDCN) architecture\n    as described in the paper\n    [Towards Deeper, Lighter and Interpretable Cross Network for CTR Prediction](https://arxiv.org/pdf/2311.04635).\n    The GDCN enhances the original DCN by introducing a gating mechanism in\n    the cross network. The gating mechanism controls feature interactions by\n    learning which interactions are more important.\n\n    The cross layer implements the following equation:\n\n    $$c_{i+1} = c_0 \\odot (W^c \\times c_i + b) \\odot \\sigma(W^g \\times c_i) + c_i$$\n\n    where:\n\n    * $\\odot$ represents element-wise multiplication\n    * $W^c$ and $W^g$ are the cross and gate weight matrices respectively\n    * $\\sigma$ is the sigmoid activation function\n\n    Parameters\n    ----------\n    column_idx: Dict\n        Dict containing the index of the columns that will be passed through\n        the `GatedDeepCrossNetwork` model. Required to slice the tensors. e.g.\n        _{'education': 0, 'relationship': 1, 'workclass': 2, ...}_.\n    num_cross_layers: int, default = 3\n        Number of cross layers in the cross network\n    structure: str, default = \"parallel\"\n        Structure of the model. Either _'parallel'_ or _'stacked'_. If _'parallel'_,\n        the output will be the concatenation of the cross network and deep network\n        outputs. If _'stacked'_, the cross network output will be fed into the deep\n        network.\n    cat_embed_input: List, Optional, default = None\n        List of Tuples with the column name, number of unique values and\n        embedding dimension. e.g. _[(education, 11, 32), ...]_\n    cat_embed_dropout: float, Optional, default = None\n        Categorical embeddings dropout. If `None`, it will default to 0.\n    use_cat_bias: bool, Optional, default = None,\n        Boolean indicating if bias will be used for the categorical embeddings.\n        If `None`, it will default to 'False'.\n    cat_embed_activation: Optional, str, default = None,\n        Activation function for the categorical embeddings, if any. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported\n    continuous_cols : Optional[List[str]], default=None\n        List with the name of the numeric (aka continuous) columns\n    cont_norm_layer : Optional[Literal[\"batchnorm\", \"layernorm\"]], default=None\n        Type of normalization layer applied to the continuous features.\n        Options are: _'layernorm'_ and _'batchnorm'_. if `None`, no\n        normalization layer will be used.\n    embed_continuous_method: Optional[Literal[\"piecewise\", \"periodic\", \"standard\"]], default=\"standard\"\n        Method to use to embed the continuous features. Options are:\n        _'standard'_, _'periodic'_ or _'piecewise'_. The _'standard'_\n        embedding method is based on the FT-Transformer implementation\n        presented in the paper: [Revisiting Deep Learning Models for\n        Tabular Data](https://arxiv.org/abs/2106.11959v5). The _'periodic'_\n        and_'piecewise'_ methods were presented in the paper: [On Embeddings for\n        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556).\n        Please, read the papers for details.\n    cont_embed_dropout : Optional[float], default=None\n        Dropout for the continuous embeddings. If `None`, it will default to 0.0\n    cont_embed_activation : Optional[str], default=None\n        Activation function for the continuous embeddings if any. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported.\n        If `None`, no activation function will be applied.\n    quantization_setup : Optional[Dict[str, List[float]]], default=None\n        This parameter is used when the _'piecewise'_ method is used to embed\n        the continuous cols. It is a dict where keys are the name of the continuous\n        columns and values are lists with the boundaries for the quantization\n        of the continuous_cols. See the examples for details. If\n        If the _'piecewise'_ method is used, this parameter is required.\n    n_frequencies : Optional[int], default=None\n        This is the so called _'k'_ in their paper [On Embeddings for\n        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556),\n        and is the number of 'frequencies' that will be used to represent each\n        continuous column. See their Eq 2 in the paper for details. If\n        the _'periodic'_ method is used, this parameter is required.\n    sigma : Optional[float], default=None\n        This is the sigma parameter in the paper mentioned when describing the\n        previous parameters and it is used to initialise the 'frequency\n        weights'. See their Eq 2 in the paper for details. If\n        the _'periodic'_ method is used, this parameter is required.\n    share_last_layer : Optional[bool], default=None\n        This parameter is not present in the before mentioned paper but it is implemented in\n        the [official repo](https://github.com/yandex-research/rtdl-num-embeddings/tree/main).\n        If `True` the linear layer that turns the frequencies into embeddings\n        will be shared across the continuous columns. If `False` a different\n        linear layer will be used for each continuous column.\n        If the _'periodic'_ method is used, this parameter is required.\n    full_embed_dropout: bool, Optional, default = None,\n        If `True`, the full embedding corresponding to a column will be masked\n        out/dropout. If `None`, it will default to `False`.\n    mlp_hidden_dims: List, default = [200, 100]\n        List with the number of neurons per dense layer in the mlp.\n    mlp_activation: str, default = \"relu\"\n        Activation function for the dense layers of the MLP. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported\n    mlp_dropout: float or List, default = 0.1\n        float or List of floats with the dropout between the dense layers.\n        e.g: _[0.5,0.5]_\n    mlp_batchnorm: bool, default = False\n        Boolean indicating whether or not batch normalization will be applied\n        to the dense layers\n    mlp_batchnorm_last: bool, default = False\n        Boolean indicating whether or not batch normalization will be applied\n        to the last of the dense layers\n    mlp_linear_first: bool, default = False\n        Boolean indicating the order of the operations in the dense\n        layer. If `True: [LIN -&gt; ACT -&gt; BN -&gt; DP]`. If `False: [BN -&gt; DP -&gt;\n        LIN -&gt; ACT]`\n\n    Attributes\n    ----------\n    cross_network: nn.Module\n        The gated cross network component\n    deep_network: nn.Module\n        The deep network component (MLP)\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.models.rec import GatedDeepCrossNetwork\n    &gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n    &gt;&gt;&gt; colnames = [\"a\", \"b\", \"c\", \"d\", \"e\"]\n    &gt;&gt;&gt; cat_embed_input = [(u, i, j) for u, i, j in zip(colnames[:4], [4] * 4, [8] * 4)]\n    &gt;&gt;&gt; column_idx = {k: v for v, k in enumerate(colnames)}\n    &gt;&gt;&gt; model = GatedDeepCrossNetwork(\n    ...     column_idx=column_idx,\n    ...     cat_embed_input=cat_embed_input,\n    ...     continuous_cols=[\"e\"],\n    ...     num_cross_layers=2,\n    ...     mlp_hidden_dims=[16, 8]\n    ... )\n    &gt;&gt;&gt; out = model(X_tab)\n    \"\"\"\n\n    def __init__(\n        self,\n        *,\n        column_idx: Dict[str, int],\n        num_cross_layers: int = 3,\n        structure: Literal[\"stacked\", \"parallel\"] = \"parallel\",\n        cat_embed_input: Optional[List[Tuple[str, int, int]]] = None,\n        cat_embed_dropout: Optional[float] = None,\n        use_cat_bias: Optional[bool] = None,\n        cat_embed_activation: Optional[str] = None,\n        continuous_cols: Optional[List[str]] = None,\n        cont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\n        embed_continuous: Optional[bool] = None,\n        embed_continuous_method: Optional[\n            Literal[\"standard\", \"piecewise\", \"periodic\"]\n        ] = None,\n        cont_embed_dim: Optional[int] = None,\n        cont_embed_dropout: Optional[float] = None,\n        cont_embed_activation: Optional[str] = None,\n        quantization_setup: Optional[Dict[str, List[float]]] = None,\n        n_frequencies: Optional[int] = None,\n        sigma: Optional[float] = None,\n        share_last_layer: Optional[bool] = None,\n        full_embed_dropout: Optional[bool] = None,\n        mlp_hidden_dims: List[int] = [200, 100],\n        mlp_activation: str = \"relu\",\n        mlp_dropout: Union[float, List[float]] = 0.1,\n        mlp_batchnorm: bool = False,\n        mlp_batchnorm_last: bool = False,\n        mlp_linear_first: bool = True,\n    ):\n        super(GatedDeepCrossNetwork, self).__init__(\n            column_idx=column_idx,\n            cat_embed_input=cat_embed_input,\n            cat_embed_dropout=cat_embed_dropout,\n            use_cat_bias=use_cat_bias,\n            cat_embed_activation=cat_embed_activation,\n            continuous_cols=continuous_cols,\n            cont_norm_layer=cont_norm_layer,\n            embed_continuous=embed_continuous,\n            embed_continuous_method=embed_continuous_method,\n            cont_embed_dim=cont_embed_dim,\n            cont_embed_dropout=cont_embed_dropout,\n            cont_embed_activation=cont_embed_activation,\n            quantization_setup=quantization_setup,\n            n_frequencies=n_frequencies,\n            sigma=sigma,\n            share_last_layer=share_last_layer,\n            full_embed_dropout=full_embed_dropout,\n        )\n\n        self.num_cross_layers = num_cross_layers\n        self.structure = structure\n\n        self.mlp_hidden_dims = mlp_hidden_dims\n        self.mlp_activation = mlp_activation\n        self.mlp_dropout = mlp_dropout\n        self.mlp_batchnorm = mlp_batchnorm\n        self.mlp_batchnorm_last = mlp_batchnorm_last\n        self.mlp_linear_first = mlp_linear_first\n\n        embeddings_output_dim = self.cat_out_dim + self.cont_out_dim\n        self.deep_network = MLP(\n            [embeddings_output_dim] + self.mlp_hidden_dims,\n            self.mlp_activation,\n            self.mlp_dropout,\n            self.mlp_batchnorm,\n            self.mlp_batchnorm_last,\n            self.mlp_linear_first,\n        )\n\n        self.cross_network = GatedCrossNetwork(embeddings_output_dim, num_cross_layers)\n\n    def forward(self, X: torch.Tensor) -&gt; torch.Tensor:\n        x = self._get_embeddings(X)\n        cross_output = self.cross_network(x)\n\n        if self.structure == \"stacked\":\n            deep_output = self.deep_network(cross_output)\n            return deep_output\n        else:  # parallel\n            deep_output = self.deep_network(x)\n            return torch.cat([cross_output, deep_output], dim=1)\n\n    @property\n    def output_dim(self) -&gt; int:\n        if self.structure == \"stacked\":\n            return self.mlp_hidden_dims[-1]\n        else:  # parallel\n            return self.mlp_hidden_dims[-1] + self.cat_out_dim + self.cont_out_dim\n</code></pre>"},{"location":"pytorch-widedeep/trainer.html","title":"Training multimodal Deep Learning Models","text":"<p>Here is the documentation for the <code>Trainer</code> class, that will do all the heavy lifting.</p> <p>Trainer is also available from <code>pytorch-widedeep</code> directly, for example, one could do:</p> <pre><code>    from pytorch-widedeep.training import Trainer\n</code></pre> <p>or also:</p> <pre><code>    from pytorch-widedeep import Trainer\n</code></pre>"},{"location":"pytorch-widedeep/trainer.html#pytorch_widedeep.training.Trainer","title":"Trainer","text":"<p>               Bases: <code>BaseTrainer</code></p> <p>Class to set the of attributes that will be used during the training process.</p> <p>Parameters:</p> Name Type Description Default <code>model</code> <code>WideDeep</code> <p>An object of class <code>WideDeep</code></p> required <code>objective</code> <code>str</code> <p>Defines the objective, loss or cost function. </p> <p>Param aliases: <code>loss_function</code>, <code>loss_fn</code>, <code>loss</code>, <code>cost_function</code>, <code>cost_fn</code>, <code>cost</code>. </p> <p>Possible values are:</p> <ul> <li> <p><code>binary</code>, aliases: <code>logistic</code>, <code>binary_logloss</code>, <code>binary_cross_entropy</code></p> </li> <li> <p><code>binary_focal_loss</code></p> </li> <li> <p><code>multiclass</code>, aliases: <code>multi_logloss</code>, <code>cross_entropy</code>, <code>categorical_cross_entropy</code>,</p> </li> <li> <p><code>multiclass_focal_loss</code></p> </li> <li> <p><code>regression</code>, aliases: <code>mse</code>, <code>l2</code>, <code>mean_squared_error</code></p> </li> <li> <p><code>mean_absolute_error</code>, aliases: <code>mae</code>, <code>l1</code></p> </li> <li> <p><code>mean_squared_log_error</code>, aliases: <code>msle</code></p> </li> <li> <p><code>root_mean_squared_error</code>, aliases:  <code>rmse</code></p> </li> <li> <p><code>root_mean_squared_log_error</code>, aliases: <code>rmsle</code></p> </li> <li> <p><code>zero_inflated_lognormal</code>, aliases: <code>ziln</code></p> </li> <li> <p><code>quantile</code></p> </li> <li> <p><code>tweedie</code></p> </li> <li> <p><code>multitarget</code>, aliases: <code>multi_target</code></p> </li> </ul> <p>NOTE: For <code>multitarget</code> a custom loss function must be passed</p> required <code>custom_loss_function</code> <code>Optional[Module]</code> <p>It is possible to pass a custom loss function. See for example <code>pytorch_widedeep.losses.FocalLoss</code> for the required structure of the object or the Examples section in this documentation or in the repo. Note that if <code>custom_loss_function</code> is not <code>None</code>, <code>objective</code> must be 'binary', 'multiclass' or 'regression', consistent with the loss function</p> <code>None</code> <code>optimizers</code> <code>Optional[Union[Optimizer, Dict[str, Union[Optimizer, List[Optimizer]]]]]</code> <ul> <li>An instance of Pytorch's <code>Optimizer</code> object   (e.g. <code>torch.optim.Adam()</code>) or</li> <li>a dictionary where there keys are the model components (i.e.   'wide', 'deeptabular', 'deeptext', 'deepimage'   and/or 'deephead')  and the values are the corresponding   optimizers or list of optimizers if multiple models are used for   the given data mode (e.g. two text columns/models for the deeptext   component). If multiple optimizers are used the   dictionary MUST contain an optimizer per model component.</li> </ul> <p>if no optimizers are passed it will default to <code>Adam</code> for all model components</p> <code>None</code> <code>lr_schedulers</code> <code>Optional[Union[LRScheduler, Dict[str, Union[LRScheduler, List[LRScheduler]]]]]</code> <ul> <li>An instance of Pytorch's <code>LRScheduler</code> object (e.g   <code>torch.optim.lr_scheduler.StepLR(opt, step_size=5)</code>) or</li> <li>a dictionary where there keys are the model componenst (i.e. 'wide',   'deeptabular', 'deeptext', 'deepimage' and/or 'deephead') and the   values are the corresponding learning rate schedulers or list of     learning rate schedulers if multiple models are used for the given     data mode (e.g. two text columns/models for the deeptext component).</li> </ul> <code>None</code> <code>initializers</code> <code>Optional[Union[Initializer, Dict[str, Union[Initializer, List[Initializer]]]]]</code> <ul> <li>An instance of an <code>Initializer</code> object see <code>pytorch-widedeep.initializers</code> or</li> <li>a dictionary where there keys are the model components (i.e. 'wide',   'deeptabular', 'deeptext', 'deepimage' and/or 'deephead')   and the values are the corresponding initializers or list of     initializers if multiple models are used for the given data mode (e.g.     two text columns/models for the deeptext component).</li> </ul> <code>None</code> <code>transforms</code> <code>Optional[List[Transforms]]</code> <p>List with <code>torchvision.transforms</code> to be applied to the image component of the model (i.e. <code>deepimage</code>) See torchvision transforms.</p> <code>None</code> <code>callbacks</code> <code>Optional[List[Callback]]</code> <p>List with <code>Callback</code> objects. The three callbacks available in <code>pytorch-widedeep</code> are: <code>LRHistory</code>, <code>ModelCheckpoint</code> and <code>EarlyStopping</code>. The <code>History</code> and the <code>LRShedulerCallback</code> callbacks are used by default. This can also be a custom callback as long as the object of type <code>Callback</code>. See <code>pytorch_widedeep.callbacks.Callback</code> or the examples folder in the repo.</p> <code>None</code> <code>metrics</code> <code>Optional[Union[List[Metric], List[Metric]]]</code> <ul> <li>List of objects of type <code>Metric</code>. Metrics available are:   <code>Accuracy</code>, <code>Precision</code>, <code>Recall</code>, <code>FBetaScore</code>,   <code>F1Score</code> and <code>R2Score</code>. This can also be a custom metric as long   as it is an object of type <code>Metric</code>. See   <code>pytorch_widedeep.metrics.Metric</code> or the examples folder in the   repo</li> <li>List of objects of type <code>torchmetrics.Metric</code>. This can be any   metric from torchmetrics library   Examples.   This can also be a custom metric as long as   it is an object of type <code>Metric</code>. See   the instructions.</li> </ul> <code>None</code> <code>verbose</code> <code>int</code> <p>Verbosity level. If set to 0 nothing will be printed during training</p> <code>1</code> <code>seed</code> <code>int</code> <p>Random seed to be used internally for train/test split</p> <code>1</code> <p>Other Parameters:</p> Name Type Description <code>**kwargs</code> <p>Other infrequently used arguments that can also be passed as kwargs are:</p> <ul> <li> <p>device: <code>str</code>     string indicating the device. One of 'cpu', 'gpu' or 'mps' if     run on a Mac with Apple silicon or AMD GPU(s)</p> </li> <li> <p>num_workers: <code>int</code>     number of workers to be used internally by the data loaders</p> </li> <li> <p>lambda_sparse: <code>float</code>     lambda sparse parameter in case the <code>deeptabular</code> component is <code>TabNet</code></p> </li> <li> <p>class_weight: <code>List[float]</code>     This is the <code>weight</code> or <code>pos_weight</code> parameter in     <code>CrossEntropyLoss</code> and <code>BCEWithLogitsLoss</code>, depending on whether</p> </li> <li>reducelronplateau_criterion: <code>str</code>     This sets the criterion that will be used by the lr scheduler to     take a step: One of 'loss' or 'metric'. The ReduceLROnPlateau     learning rate is a bit particular.</li> </ul> <p>Attributes:</p> Name Type Description <code>cyclic_lr</code> <code>bool</code> <p>Attribute that indicates if any of the lr_schedulers is cyclic_lr (i.e. <code>CyclicLR</code> or <code>OneCycleLR</code>). See Pytorch schedulers.</p> <code>feature_importance</code> <code>dict</code> <p>dict where the keys are the column names and the values are the corresponding feature importances. This attribute will only exist if the <code>deeptabular</code> component is a Tabnet model.</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from torchvision.transforms import ToTensor\n&gt;&gt;&gt;\n&gt;&gt;&gt; # wide deep imports\n&gt;&gt;&gt; from pytorch_widedeep.callbacks import EarlyStopping, LRHistory\n&gt;&gt;&gt; from pytorch_widedeep.initializers import KaimingNormal, KaimingUniform, Normal, Uniform\n&gt;&gt;&gt; from pytorch_widedeep.models import TabResnet, Vision, BasicRNN, Wide, WideDeep\n&gt;&gt;&gt; from pytorch_widedeep import Trainer\n&gt;&gt;&gt;\n&gt;&gt;&gt; embed_input = [(u, i, j) for u, i, j in zip([\"a\", \"b\", \"c\"][:4], [4] * 3, [8] * 3)]\n&gt;&gt;&gt; column_idx = {k: v for v, k in enumerate([\"a\", \"b\", \"c\"])}\n&gt;&gt;&gt; wide = Wide(10, 1)\n&gt;&gt;&gt;\n&gt;&gt;&gt; # build the model\n&gt;&gt;&gt; deeptabular = TabResnet(blocks_dims=[8, 4], column_idx=column_idx, cat_embed_input=embed_input)\n&gt;&gt;&gt; deeptext = BasicRNN(vocab_size=10, embed_dim=4, padding_idx=0)\n&gt;&gt;&gt; deepimage = Vision()\n&gt;&gt;&gt; model = WideDeep(wide=wide, deeptabular=deeptabular, deeptext=deeptext, deepimage=deepimage)\n&gt;&gt;&gt;\n&gt;&gt;&gt; # set optimizers and schedulers\n&gt;&gt;&gt; wide_opt = torch.optim.Adam(model.wide.parameters())\n&gt;&gt;&gt; deep_opt = torch.optim.AdamW(model.deeptabular.parameters())\n&gt;&gt;&gt; text_opt = torch.optim.Adam(model.deeptext.parameters())\n&gt;&gt;&gt; img_opt = torch.optim.AdamW(model.deepimage.parameters())\n&gt;&gt;&gt;\n&gt;&gt;&gt; wide_sch = torch.optim.lr_scheduler.StepLR(wide_opt, step_size=5)\n&gt;&gt;&gt; deep_sch = torch.optim.lr_scheduler.StepLR(deep_opt, step_size=3)\n&gt;&gt;&gt; text_sch = torch.optim.lr_scheduler.StepLR(text_opt, step_size=5)\n&gt;&gt;&gt; img_sch = torch.optim.lr_scheduler.StepLR(img_opt, step_size=3)\n&gt;&gt;&gt;\n&gt;&gt;&gt; optimizers = {\"wide\": wide_opt, \"deeptabular\": deep_opt, \"deeptext\": text_opt, \"deepimage\": img_opt}\n&gt;&gt;&gt; schedulers = {\"wide\": wide_sch, \"deeptabular\": deep_sch, \"deeptext\": text_sch, \"deepimage\": img_sch}\n&gt;&gt;&gt;\n&gt;&gt;&gt; # set initializers and callbacks\n&gt;&gt;&gt; initializers = {\"wide\": Uniform, \"deeptabular\": Normal, \"deeptext\": KaimingNormal, \"deepimage\": KaimingUniform}\n&gt;&gt;&gt; transforms = [ToTensor]\n&gt;&gt;&gt; callbacks = [LRHistory(n_epochs=4), EarlyStopping]\n&gt;&gt;&gt;\n&gt;&gt;&gt; # set the trainer\n&gt;&gt;&gt; trainer = Trainer(model, objective=\"regression\", initializers=initializers, optimizers=optimizers,\n... lr_schedulers=schedulers, callbacks=callbacks, transforms=transforms)\n</code></pre> Source code in <code>pytorch_widedeep/training/trainer.py</code> <pre><code>class Trainer(BaseTrainer):\n    r\"\"\"Class to set the of attributes that will be used during the\n    training process.\n\n    Parameters\n    ----------\n    model: `WideDeep`\n        An object of class `WideDeep`\n    objective: str\n        Defines the objective, loss or cost function. &lt;br/&gt;\n\n        Param aliases: `loss_function`, `loss_fn`, `loss`,\n        `cost_function`, `cost_fn`, `cost`. &lt;br/&gt;\n\n        Possible values are:\n\n        - `binary`, aliases: `logistic`, `binary_logloss`, `binary_cross_entropy`\n\n        - `binary_focal_loss`\n\n        - `multiclass`, aliases: `multi_logloss`, `cross_entropy`, `categorical_cross_entropy`,\n\n        - `multiclass_focal_loss`\n\n        - `regression`, aliases: `mse`, `l2`, `mean_squared_error`\n\n        - `mean_absolute_error`, aliases: `mae`, `l1`\n\n        - `mean_squared_log_error`, aliases: `msle`\n\n        - `root_mean_squared_error`, aliases:  `rmse`\n\n        - `root_mean_squared_log_error`, aliases: `rmsle`\n\n        - `zero_inflated_lognormal`, aliases: `ziln`\n\n        - `quantile`\n\n        - `tweedie`\n\n        - `multitarget`, aliases: `multi_target`\n\n        **NOTE**: For `multitarget` a custom loss function must be passed\n    custom_loss_function: `nn.Module`. Optional, default = None\n        It is possible to pass a custom loss function. See for example\n        `pytorch_widedeep.losses.FocalLoss` for the required structure of the\n        object or the Examples section in this documentation or in the repo.\n        Note that if `custom_loss_function` is not `None`, `objective` must\n        be _'binary'_, _'multiclass'_ or _'regression'_, consistent with the\n        loss function\n    optimizers: `Optimizer` or dict. Optional, default=None\n        - An instance of Pytorch's `Optimizer` object\n          (e.g. `torch.optim.Adam()`) or\n        - a dictionary where there keys are the model components (i.e.\n          _'wide'_, _'deeptabular'_, _'deeptext'_, _'deepimage'_\n          and/or _'deephead'_)  and the values are the corresponding\n          optimizers or list of optimizers if multiple models are used for\n          the given data mode (e.g. two text columns/models for the deeptext\n          component). If multiple optimizers are used the\n          dictionary **MUST** contain an optimizer per model component.\n\n        if no optimizers are passed it will default to `Adam` for all\n        model components\n    lr_schedulers: `LRScheduler` or dict. Optional, default=None\n        - An instance of Pytorch's `LRScheduler` object (e.g\n          `torch.optim.lr_scheduler.StepLR(opt, step_size=5)`) or\n        - a dictionary where there keys are the model componenst (i.e. _'wide'_,\n          _'deeptabular'_, _'deeptext'_, _'deepimage'_ and/or _'deephead'_) and the\n          values are the corresponding learning rate schedulers or list of\n            learning rate schedulers if multiple models are used for the given\n            data mode (e.g. two text columns/models for the deeptext component).\n    initializers: `Initializer` or dict. Optional, default=None\n        - An instance of an `Initializer` object see `pytorch-widedeep.initializers` or\n        - a dictionary where there keys are the model components (i.e. _'wide'_,\n          _'deeptabular'_, _'deeptext'_, _'deepimage'_ and/or _'deephead'_)\n          and the values are the corresponding initializers or list of\n            initializers if multiple models are used for the given data mode (e.g.\n            two text columns/models for the deeptext component).\n    transforms: List. Optional, default=None\n        List with `torchvision.transforms` to be applied to the image\n        component of the model (i.e. `deepimage`) See\n        [torchvision transforms](https://pytorch.org/docs/stable/torchvision/transforms.html).\n    callbacks: List. Optional, default=None\n        List with `Callback` objects. The three callbacks available in\n        `pytorch-widedeep` are: `LRHistory`, `ModelCheckpoint` and\n        `EarlyStopping`. The `History` and the `LRShedulerCallback` callbacks\n        are used by default. This can also be a custom callback as long as\n        the object of type `Callback`. See\n        `pytorch_widedeep.callbacks.Callback` or the examples folder in the\n        repo.\n    metrics: List. Optional, default=None\n        - List of objects of type `Metric`. Metrics available are:\n          `Accuracy`, `Precision`, `Recall`, `FBetaScore`,\n          `F1Score` and `R2Score`. This can also be a custom metric as long\n          as it is an object of type `Metric`. See\n          `pytorch_widedeep.metrics.Metric` or the examples folder in the\n          repo\n        - List of objects of type `torchmetrics.Metric`. This can be any\n          metric from torchmetrics library\n          [Examples](https://torchmetrics.readthedocs.io/en/latest/).\n          This can also be a custom metric as long as\n          it is an object of type `Metric`. See\n          [the instructions](https://torchmetrics.readthedocs.io/en/latest/).\n    verbose: int, default=1\n        Verbosity level. If set to 0 nothing will be printed during training\n    seed: int, default=1\n        Random seed to be used internally for train/test split\n\n    Other Parameters\n    ----------------\n    **kwargs: dict\n        Other infrequently used arguments that can also be passed as kwargs are:\n\n        - **device**: `str`&lt;br/&gt;\n            string indicating the device. One of _'cpu'_, _'gpu'_ or 'mps' if\n            run on a Mac with Apple silicon or AMD GPU(s)\n\n        - **num_workers**: `int`&lt;br/&gt;\n            number of workers to be used internally by the data loaders\n\n        - **lambda_sparse**: `float`&lt;br/&gt;\n            lambda sparse parameter in case the `deeptabular` component is `TabNet`\n\n        - **class_weight**: `List[float]`&lt;br/&gt;\n            This is the `weight` or `pos_weight` parameter in\n            `CrossEntropyLoss` and `BCEWithLogitsLoss`, depending on whether\n        - **reducelronplateau_criterion**: `str`\n            This sets the criterion that will be used by the lr scheduler to\n            take a step: One of _'loss'_ or _'metric'_. The ReduceLROnPlateau\n            learning rate is a bit particular.\n\n    Attributes\n    ----------\n    cyclic_lr: bool\n        Attribute that indicates if any of the lr_schedulers is cyclic_lr\n        (i.e. `CyclicLR` or\n        `OneCycleLR`). See [Pytorch schedulers](https://pytorch.org/docs/stable/optim.html).\n    feature_importance: dict\n        dict where the keys are the column names and the values are the\n        corresponding feature importances. This attribute will only exist\n        if the `deeptabular` component is a Tabnet model.\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from torchvision.transforms import ToTensor\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; # wide deep imports\n    &gt;&gt;&gt; from pytorch_widedeep.callbacks import EarlyStopping, LRHistory\n    &gt;&gt;&gt; from pytorch_widedeep.initializers import KaimingNormal, KaimingUniform, Normal, Uniform\n    &gt;&gt;&gt; from pytorch_widedeep.models import TabResnet, Vision, BasicRNN, Wide, WideDeep\n    &gt;&gt;&gt; from pytorch_widedeep import Trainer\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; embed_input = [(u, i, j) for u, i, j in zip([\"a\", \"b\", \"c\"][:4], [4] * 3, [8] * 3)]\n    &gt;&gt;&gt; column_idx = {k: v for v, k in enumerate([\"a\", \"b\", \"c\"])}\n    &gt;&gt;&gt; wide = Wide(10, 1)\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; # build the model\n    &gt;&gt;&gt; deeptabular = TabResnet(blocks_dims=[8, 4], column_idx=column_idx, cat_embed_input=embed_input)\n    &gt;&gt;&gt; deeptext = BasicRNN(vocab_size=10, embed_dim=4, padding_idx=0)\n    &gt;&gt;&gt; deepimage = Vision()\n    &gt;&gt;&gt; model = WideDeep(wide=wide, deeptabular=deeptabular, deeptext=deeptext, deepimage=deepimage)\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; # set optimizers and schedulers\n    &gt;&gt;&gt; wide_opt = torch.optim.Adam(model.wide.parameters())\n    &gt;&gt;&gt; deep_opt = torch.optim.AdamW(model.deeptabular.parameters())\n    &gt;&gt;&gt; text_opt = torch.optim.Adam(model.deeptext.parameters())\n    &gt;&gt;&gt; img_opt = torch.optim.AdamW(model.deepimage.parameters())\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; wide_sch = torch.optim.lr_scheduler.StepLR(wide_opt, step_size=5)\n    &gt;&gt;&gt; deep_sch = torch.optim.lr_scheduler.StepLR(deep_opt, step_size=3)\n    &gt;&gt;&gt; text_sch = torch.optim.lr_scheduler.StepLR(text_opt, step_size=5)\n    &gt;&gt;&gt; img_sch = torch.optim.lr_scheduler.StepLR(img_opt, step_size=3)\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; optimizers = {\"wide\": wide_opt, \"deeptabular\": deep_opt, \"deeptext\": text_opt, \"deepimage\": img_opt}\n    &gt;&gt;&gt; schedulers = {\"wide\": wide_sch, \"deeptabular\": deep_sch, \"deeptext\": text_sch, \"deepimage\": img_sch}\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; # set initializers and callbacks\n    &gt;&gt;&gt; initializers = {\"wide\": Uniform, \"deeptabular\": Normal, \"deeptext\": KaimingNormal, \"deepimage\": KaimingUniform}\n    &gt;&gt;&gt; transforms = [ToTensor]\n    &gt;&gt;&gt; callbacks = [LRHistory(n_epochs=4), EarlyStopping]\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; # set the trainer\n    &gt;&gt;&gt; trainer = Trainer(model, objective=\"regression\", initializers=initializers, optimizers=optimizers,\n    ... lr_schedulers=schedulers, callbacks=callbacks, transforms=transforms)\n    \"\"\"\n\n    @alias(  # noqa: C901\n        \"objective\",\n        [\"loss_function\", \"loss_fn\", \"loss\", \"cost_function\", \"cost_fn\", \"cost\"],\n    )\n    @alias(\"metrics\", [\"train_metrics\"])\n    def __init__(\n        self,\n        model: WideDeep,\n        objective: str,\n        custom_loss_function: Optional[nn.Module] = None,\n        optimizers: Optional[\n            Union[Optimizer, Dict[str, Union[Optimizer, List[Optimizer]]]]\n        ] = None,\n        lr_schedulers: Optional[\n            Union[LRScheduler, Dict[str, Union[LRScheduler, List[LRScheduler]]]]\n        ] = None,\n        initializers: Optional[\n            Union[Initializer, Dict[str, Union[Initializer, List[Initializer]]]]\n        ] = None,\n        transforms: Optional[List[Transforms]] = None,\n        callbacks: Optional[List[Callback]] = None,\n        metrics: Optional[Union[List[Metric], List[TorchMetric]]] = None,\n        eval_metrics: Optional[Union[List[Metric], List[TorchMetric]]] = None,\n        verbose: int = 1,\n        seed: int = 1,\n        **kwargs,\n    ):\n        super().__init__(\n            model=model,\n            objective=objective,\n            custom_loss_function=custom_loss_function,\n            optimizers=optimizers,\n            lr_schedulers=lr_schedulers,\n            initializers=initializers,\n            transforms=transforms,\n            callbacks=callbacks,\n            metrics=metrics,\n            eval_metrics=eval_metrics,\n            verbose=verbose,\n            seed=seed,\n            **kwargs,\n        )\n\n    @alias(\"finetune\", [\"warmup\"])\n    @alias(\"train_dataloader\", [\"custom_dataloader\"])\n    @alias(\"eval_dataloader\", [\"custom_eval_dataloader\"])\n    def fit(  # noqa: C901\n        self,\n        X_wide: Optional[np.ndarray] = None,\n        X_tab: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n        X_text: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n        X_img: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n        X_train: Optional[Dict[str, Union[np.ndarray, List[np.ndarray]]]] = None,\n        X_val: Optional[Dict[str, Union[np.ndarray, List[np.ndarray]]]] = None,\n        val_split: Optional[float] = None,\n        target: Optional[np.ndarray] = None,\n        n_epochs: int = 1,\n        validation_freq: int = 1,\n        batch_size: int = 32,\n        train_dataloader: Optional[CustomDataLoader] = None,\n        eval_dataloader: Optional[CustomDataLoader] = None,\n        feature_importance_sample_size: Optional[int] = None,\n        finetune: bool = False,\n        **kwargs,\n    ):\n        r\"\"\"Fit method.\n\n        The input datasets can be passed either directly via numpy arrays\n        (`X_wide`, `X_tab`, `X_text` or `X_img`) or alternatively, in\n        dictionaries (`X_train` or `X_val`).\n\n        Parameters\n        ----------\n        X_wide: np.ndarray, Optional. default=None\n            Input for the `wide` model component.\n            See `pytorch_widedeep.preprocessing.WidePreprocessor`\n        X_tab: np.ndarray, Optional. default=None\n            Input for the `deeptabular` model component.\n            See `pytorch_widedeep.preprocessing.TabPreprocessor`. If multiple\n            tabular models are used for different columns, this should be a\n            list of numpy arrays\n        X_text: Union[np.ndarray, List[np.ndarray]], Optional. default=None\n            Input for the `deeptext` model component.\n            See `pytorch_widedeep.preprocessing.TextPreprocessor`.\n            If multiple text columns/models are used, this should be a list of\n            numpy arrays\n        X_img: np.ndarray, Optional. default=None\n            Input for the `deepimage` model component.\n            See `pytorch_widedeep.preprocessing.ImagePreprocessor`.\n            If multiple image columns/models are used, this should be a list of\n            numpy arrays\n        X_train: Dict, Optional. default=None\n            The training dataset can also be passed in a dictionary. Keys are\n            _'X_wide'_, _'X_tab'_, _'X_text'_, _'X_img'_ and _'target'_. Values\n            are the corresponding matrices. Note that of multiple text or image\n            columns/models are used, the corresponding values should be lists\n            of numpy arrays\n        X_val: Dict, Optional. default=None\n            The validation dataset can also be passed in a dictionary. Keys\n            are _'X_wide'_, _'X_tab'_, _'X_text'_, _'X_img'_ and _'target'_.\n            Values are the corresponding matrices. Note that of multiple text\n            or image columns/models are used, the corresponding values should\n            be lists of numpy arrays\n        val_split: float, Optional. default=None\n            train/val split fraction\n        target: np.ndarray, Optional. default=None\n            target values\n        n_epochs: int, default=1\n            number of epochs\n        validation_freq: int, default=1\n            epochs validation frequency\n        batch_size: int, default=32\n            batch size\n        custom_dataloader: `DataLoader`, Optional, default=None\n            object of class `torch.utils.data.DataLoader`. Available\n            predefined dataloaders are in `pytorch-widedeep.dataloaders`.If\n            `None`, a standard torch `DataLoader` is used.\n        finetune: bool, default=False\n            fine-tune individual model components. This functionality can also\n            be used to 'warm-up' (and hence the alias `warmup`) individual\n            components before the joined training starts, and hence its\n            alias. See the Examples folder in the repo for more details\n\n            `pytorch_widedeep` implements 3 fine-tune routines.\n\n            - fine-tune all trainable layers at once. This routine is\n              inspired by the work of Howard &amp; Sebastian Ruder 2018 in their\n              [ULMfit paper](https://arxiv.org/abs/1801.06146). Using a\n              Slanted Triangular learing (see\n              [Leslie N. Smith paper](https://arxiv.org/pdf/1506.01186.pdf) ) ,\n              the process is the following: *i*) the learning rate will\n              gradually increase for 10% of the training steps from max_lr/10\n              to max_lr. *ii*) It will then gradually decrease to max_lr/10\n              for the remaining 90% of the steps. The optimizer used in the\n              process is `Adam`.\n\n            and two gradual fine-tune routines, where only certain layers are\n            trained at a time.\n\n            - The so called `Felbo` gradual fine-tune rourine, based on the the\n              Felbo et al., 2017 [DeepEmoji paper](https://arxiv.org/abs/1708.00524).\n            - The `Howard` routine based on the work of Howard &amp; Sebastian Ruder 2018 in their\n              [ULMfit paper](https://arxiv.org/abs/1801.06146&gt;).\n\n            For details on how these routines work, please see the Examples\n            section in this documentation and the Examples folder in the repo. &lt;br/&gt;\n            Param Alias: `warmup`\n\n        Other Parameters\n        ----------------\n        **kwargs:\n            Other keyword arguments are:\n\n            - **DataLoader related parameters**:&lt;br/&gt;\n                For example,  `sampler`, `batch_sampler`, `collate_fn`, etc.\n                Please, see the pytorch\n                [DataLoader docs](https://pytorch.org/docs/stable/data.html#torch.utils.data.DataLoader)\n                for details.\n\n            - **Finetune related parameters**:&lt;br/&gt;\n                see the source code at `pytorch_widedeep._finetune`. Namely, these are:\n\n                - `finetune_epochs` (`int`):\n                    number of epochs use for fine tuning\n                - `finetune_max_lr` (`float`):\n                   max lr during fine tuning\n                - `routine` (`str`):\n                   one of _'howard'_ or _'felbo'_\n                - `deeptabular_gradual` (`bool`):\n                   boolean indicating if the `deeptabular` component will be fine tuned gradually\n                - `deeptabular_layers` (`Optional[Union[List[nn.Module], List[List[nn.Module]]]]`):\n                   List of pytorch modules indicating the layers of the\n                   `deeptabular` that will be fine tuned\n                - `deeptabular_max_lr` (`Union[float, List[float]]`):\n                   max lr for the `deeptabular` componet during fine tuning\n                - `deeptext_gradual` (`bool`):\n                   same as `deeptabular_gradual` but for the `deeptext` component\n                - `deeptext_layers` (`Optional[Union[List[nn.Module], List[List[nn.Module]]]]`):\n                   same as `deeptabular_gradual` but for the `deeptext` component.\n                   If there are multiple text columns/models, this should be a list of lists\n                - `deeptext_max_lr` (`Union[float, List[float]]`):\n                   same as `deeptabular_gradual` but for the `deeptext` component\n                   If there are multiple text columns/models, this should be a list of floats\n                - `deepimage_gradual` (`bool`):\n                   same as `deeptext_layers` but for the `deepimage` component\n                - `deepimage_layers` (`Optional[Union[List[nn.Module], List[List[nn.Module]]]]`):\n                   same as `deeptext_layers` but for the `deepimage` component\n                - `deepimage_max_lr` (`Union[float, List[float]]`):\n                    same as `deeptext_layers` but for the `deepimage` component\n\n        Examples\n        --------\n\n        For a series of comprehensive examples on how to use the `fit` method, please see the\n        [Examples](https://github.com/jrzaurin/pytorch-widedeep/tree/master/examples)\n        folder in the repo\n        \"\"\"\n        dataloader_args, finetune_args = self._extract_kwargs(kwargs)\n        self.batch_size = batch_size\n        train_set, eval_set = wd_train_val_split(\n            self.seed,\n            self.method,  # type: ignore\n            X_wide,\n            X_tab,\n            X_text,\n            X_img,\n            X_train,\n            X_val,\n            val_split,\n            target,\n            self.transforms,\n        )\n        train_loader = self._set_dataloader(\n            train_dataloader, train_set, batch_size, dataloader_args\n        )\n        eval_loader = (\n            self._set_dataloader(eval_dataloader, eval_set, batch_size, dataloader_args)\n            if eval_set is not None\n            else None\n        )\n\n        if finetune:\n            self.with_finetuning: bool = True\n            self._do_finetune(train_loader, **finetune_args)\n            if self.verbose:\n                print(\n                    \"Fine-tuning (or warmup) of individual components completed. \"\n                    \"Training the whole model for {} epochs\".format(n_epochs)\n                )\n        else:\n            self.with_finetuning = False\n\n        self.callback_container.on_train_begin(\n            {\n                \"batch_size\": batch_size,\n                \"train_steps\": len(train_loader),\n                \"n_epochs\": n_epochs,\n            }\n        )\n        for epoch in range(n_epochs):\n            epoch_logs = self._train_epoch(train_loader, epoch)\n            if eval_loader is not None and epoch % validation_freq == (\n                validation_freq - 1\n            ):\n                epoch_logs, on_epoch_end_metric = self._eval_epoch(\n                    eval_loader, epoch_logs\n                )\n            else:\n                on_epoch_end_metric = None\n                if self.reducelronplateau:\n                    raise NotImplementedError(\n                        \"ReduceLROnPlateau scheduler can be used only with validation data.\"\n                    )\n            self.callback_container.on_epoch_end(epoch, epoch_logs, on_epoch_end_metric)\n            if self.early_stop:\n                # self.callback_container.on_train_end(epoch_logs)\n                break\n\n        self.callback_container.on_train_end(epoch_logs)\n        if feature_importance_sample_size is not None:\n            self.feature_importance = FeatureImportance(\n                self.device, feature_importance_sample_size\n            ).feature_importance(train_loader, self.model)\n        self._restore_best_weights()\n        self.model.train()\n\n    def predict(  # type: ignore[override, return]\n        self,\n        X_wide: Optional[np.ndarray] = None,\n        X_tab: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n        X_text: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n        X_img: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n        X_test: Optional[Dict[str, Union[np.ndarray, List[np.ndarray]]]] = None,\n        batch_size: Optional[int] = None,\n    ) -&gt; np.ndarray:\n        r\"\"\"Returns the predictions\n\n        The input datasets can be passed either directly via numpy arrays\n        (`X_wide`, `X_tab`, `X_text` or `X_img`) or alternatively, in\n        a dictionary (`X_test`)\n\n\n        Parameters\n        ----------\n        X_wide: np.ndarray, Optional. default=None\n            Input for the `wide` model component.\n            See `pytorch_widedeep.preprocessing.WidePreprocessor`\n        X_tab: np.ndarray or List[np.ndarray], Optional. default=None\n            Input for the `deeptabular` model component.\n            See `pytorch_widedeep.preprocessing.TabPreprocessor`\n        X_text: np.ndarray or List[np.ndarray], Optional. default=None\n            Input for the `deeptext` model component.\n            See `pytorch_widedeep.preprocessing.TextPreprocessor`\n        X_img: np.ndarray or List[np.ndarray], Optional. default=None\n            Input for the `deepimage` model component.\n            See `pytorch_widedeep.preprocessing.ImagePreprocessor`\n        X_test: Dict, Optional. default=None\n            The test dataset can also be passed in a dictionary. Keys are\n            `X_wide`, _'X_tab'_, _'X_text'_, _'X_img'_ and _'target'_. Values\n            are the corresponding matrices.\n        batch_size: int, default = 256\n            If a trainer is used to predict after having trained a model, the\n            `batch_size` needs to be defined as it will not be defined as\n            the `Trainer` is instantiated\n\n        Returns\n        -------\n        np.ndarray:\n            array with the predictions\n        \"\"\"\n        preds_l = self._predict(X_wide, X_tab, X_text, X_img, X_test, batch_size)\n        if self.method == \"regression\":\n            return np.vstack(preds_l).squeeze(1)\n        if self.method == \"binary\":\n            preds = np.vstack(preds_l).squeeze(1)\n            return (preds &gt; 0.5).astype(\"int\")\n        if self.method == \"qregression\":\n            return np.vstack(preds_l)\n        if self.method == \"multiclass\":\n            preds = np.vstack(preds_l)\n            return np.argmax(preds, 1)  # type: ignore[return-value]\n        if self.method == \"multitarget\":\n            if self.loss_fn.__class__.__name__ in [\n                \"MultiTargetClassificationLoss\",\n                \"MutilTargetRegressionAndClassificationLoss\",\n            ]:\n                raise ValueError(\n                    \"MultiTargetClassificationLoss and MutilTargetRegressionAndClassificationLoss \"\n                    \"are not supported by predict method. Please use predict_proba method instead.\"\n                )\n            return np.vstack(preds_l)\n\n    def predict_uncertainty(  # type: ignore[return]\n        self,\n        X_wide: Optional[np.ndarray] = None,\n        X_tab: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n        X_text: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n        X_img: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n        X_test: Optional[Dict[str, Union[np.ndarray, List[np.ndarray]]]] = None,\n        batch_size: Optional[int] = None,\n        uncertainty_granularity: int = 1000,\n    ) -&gt; np.ndarray:\n        r\"\"\"Returns the predicted ucnertainty of the model for the test dataset\n        using a Monte Carlo method during which dropout layers are activated\n        in the evaluation/prediction phase and each sample is predicted N\n        times (`uncertainty_granularity` times).\n\n        This is based on\n        [Dropout as a Bayesian Approximation: Representing\n        Model Uncertainty in Deep Learning](https://arxiv.org/abs/1506.02142?context=stat).\n\n        Parameters\n        ----------\n        X_wide: np.ndarray, Optional. default=None\n            Input for the `wide` model component.\n            See `pytorch_widedeep.preprocessing.WidePreprocessor`\n        X_tab: np.ndarray or List[np.ndarray], Optional. default=None\n            Input for the `deeptabular` model component.\n            See `pytorch_widedeep.preprocessing.TabPreprocessor`\n        X_text: np.ndarray or List[np.ndarray], Optional. default=None\n            Input for the `deeptext` model component.\n            See `pytorch_widedeep.preprocessing.TextPreprocessor`\n        X_img: np.ndarray or List[np.ndarray], Optional. default=None\n            Input for the `deepimage` model component.\n            See `pytorch_widedeep.preprocessing.ImagePreprocessor`\n        X_test: Dict, Optional. default=None\n            The test dataset can also be passed in a dictionary. Keys are\n            _'X_wide'_, _'X_tab'_, _'X_text'_, _'X_img'_ and _'target'_. Values\n            are the corresponding matrices.\n        batch_size: int, default = 256\n            If a trainer is used to predict after having trained a model, the\n            `batch_size` needs to be defined as it will not be defined as\n            the `Trainer` is instantiated\n        uncertainty_granularity: int default = 1000\n            number of times the model does prediction for each sample\n\n        Returns\n        -------\n        np.ndarray:\n            - if `method = regression`, it will return an array with `(max, min, mean, stdev)`\n              values for each sample.\n            - if `method = binary` it will return an array with\n              `(mean_cls_0_prob, mean_cls_1_prob, predicted_cls)` for each sample.\n            - if `method = multiclass` it will return an array with\n              `(mean_cls_0_prob, mean_cls_1_prob, mean_cls_2_prob, ... , predicted_cls)`\n              values for each sample.\n\n        \"\"\"\n        preds_l = self._predict(\n            X_wide,\n            X_tab,\n            X_text,\n            X_img,\n            X_test,\n            batch_size,\n            uncertainty_granularity,\n            uncertainty=True,\n        )\n        preds = np.vstack(preds_l)\n        samples_num = int(preds.shape[0] / uncertainty_granularity)\n        if self.method == \"regression\":\n            preds = preds.squeeze(1)\n            preds = preds.reshape((uncertainty_granularity, samples_num))\n            return np.array(\n                (\n                    preds.max(axis=0),\n                    preds.min(axis=0),\n                    preds.mean(axis=0),\n                    preds.std(axis=0),\n                )\n            ).T\n        if self.method == \"qregression\":\n            raise ValueError(\n                \"Currently predict_uncertainty is not supported for qregression method\"\n            )\n        if self.method == \"binary\":\n            preds = preds.squeeze(1)\n            preds = preds.reshape((uncertainty_granularity, samples_num))\n            preds = preds.mean(axis=0)\n            probs = np.zeros([preds.shape[0], 3])\n            probs[:, 0] = 1 - preds\n            probs[:, 1] = preds\n            return probs\n        if self.method == \"multiclass\":\n            preds = preds.reshape(uncertainty_granularity, samples_num, preds.shape[1])\n            preds = preds.mean(axis=0)\n            preds = np.hstack((preds, np.vstack(np.argmax(preds, 1))))\n            return preds\n\n        if self.method == \"multitarget\":\n            raise ValueError(\n                \"Currently predict_uncertainty is not supported for multitarget method\"\n            )\n\n    def predict_proba(  # type: ignore[override, return]  # noqa: C901\n        self,\n        X_wide: Optional[np.ndarray] = None,\n        X_tab: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n        X_text: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n        X_img: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n        X_test: Optional[Dict[str, Union[np.ndarray, List[np.ndarray]]]] = None,\n        batch_size: Optional[int] = None,\n    ) -&gt; np.ndarray:\n        r\"\"\"Returns the predicted probabilities for the test dataset for  binary\n        and multiclass methods\n\n        The input datasets can be passed either directly via numpy arrays\n        (`X_wide`, `X_tab`, `X_text` or `X_img`) or alternatively, in\n        a dictionary (`X_test`)\n\n        Parameters\n        ----------\n        X_wide: np.ndarray, Optional. default=None\n            Input for the `wide` model component.\n            See `pytorch_widedeep.preprocessing.WidePreprocessor`\n        X_tab: np.ndarray or List[np.ndarray], Optional. default=None\n            Input for the `deeptabular` model component.\n            See `pytorch_widedeep.preprocessing.TabPreprocessor`\n        X_text: np.ndarray or List[np.ndarray], Optional. default=None\n            Input for the `deeptext` model component.\n            See `pytorch_widedeep.preprocessing.TextPreprocessor`\n        X_img: np.ndarray or List[np.ndarray], Optional. default=None\n            Input for the `deepimage` model component.\n            See `pytorch_widedeep.preprocessing.ImagePreprocessor`\n        X_test: Dict, Optional. default=None\n            The test dataset can also be passed in a dictionary. Keys are\n            `X_wide`, _'X_tab'_, _'X_text'_, _'X_img'_ and _'target'_. Values\n            are the corresponding matrices.\n        batch_size: int, default = 256\n            If a trainer is used to predict after having trained a model, the\n            `batch_size` needs to be defined as it will not be defined as\n            the `Trainer` is instantiated\n\n        Returns\n        -------\n        np.ndarray\n            array with the probabilities per class\n        \"\"\"\n\n        preds_l = self._predict(X_wide, X_tab, X_text, X_img, X_test, batch_size)\n        if self.method == \"binary\":\n            preds = np.vstack(preds_l).squeeze(1)\n            probs = np.zeros([preds.shape[0], 2])\n            probs[:, 0] = 1 - preds\n            probs[:, 1] = preds\n            return probs\n        if self.method == \"multiclass\":\n            return np.vstack(preds_l)\n        if self.method == \"multitarget\":\n            return np.vstack(preds_l)\n\n    def explain(self, X_tab: np.ndarray, save_step_masks: Optional[bool] = None):\n        # TO DO: Add docs to this, to the feat imp parameter and the all\n        # related classes\n        explainer = Explainer(self.device)\n\n        res = explainer.explain(\n            self.model, X_tab, self.num_workers, self.batch_size, save_step_masks\n        )\n\n        return res\n\n    def save(\n        self,\n        path: str,\n        save_state_dict: bool = False,\n        save_optimizer: bool = False,\n        model_filename: str = \"wd_model.pt\",\n    ):\n        r\"\"\"Saves the model, training and evaluation history, and the\n        `feature_importance` attribute (if the `deeptabular` component is a\n        Tabnet model) to disk\n\n        The `Trainer` class is built so that it 'just' trains a model. With\n        that in mind, all the torch related parameters (such as optimizers,\n        learning rate schedulers, initializers, etc) have to be defined\n        externally and then passed to the `Trainer`. As a result, the\n        `Trainer` does not generate any attribute or additional data\n        products that need to be saved other than the `model` object itself,\n        which can be saved as any other torch model (e.g. `torch.save(model,\n        path)`).\n\n        The exception is Tabnet. If the `deeptabular` component is a Tabnet\n        model, an attribute (a dict) called `feature_importance` will be\n        created at the end of the training process. Therefore, a `save`\n        method was created that will save the feature importance dictionary\n        to a json file and, since we are here, the model weights, training\n        history and learning rate history.\n\n        Parameters\n        ----------\n        path: str\n            path to the directory where the model and the feature importance\n            attribute will be saved.\n        save_state_dict: bool, default = False\n            Boolean indicating whether to save directly the model\n            (and optimizer) or the model's (and optimizer's) state\n            dictionary\n        save_optimizer: bool, default = False\n            Boolean indicating whether to save the optimizer\n        model_filename: str, Optional, default = \"wd_model.pt\"\n            filename where the model weights will be store\n        \"\"\"\n\n        self._save_history(path)\n\n        self._save_model_and_optimizer(\n            path, save_state_dict, save_optimizer, model_filename\n        )\n\n        if self.model.is_tabnet:\n            with open(Path(path) / \"feature_importance.json\", \"w\") as fi:\n                json.dump(self.feature_importance, fi)\n\n    def _set_dataloader(\n        self,\n        dataloader: Optional[CustomDataLoader],\n        dataset: WideDeepDataset,\n        batch_size: int,\n        dataloader_args: Dict[str, Any],\n    ) -&gt; Union[DataLoader, CustomDataLoader]:\n        if dataloader is not None:\n            dataloader.set_dataset(dataset)\n            return dataloader\n        else:\n            # var name 'loader' to avoid reassigment and type errors\n            loader = DataLoader(\n                dataset=dataset,\n                batch_size=batch_size,\n                num_workers=self.num_workers,\n                **dataloader_args,\n            )\n        return loader\n\n    @alias(\"n_epochs\", [\"finetune_epochs\", \"warmup_epochs\"])\n    @alias(\"max_lr\", [\"finetune_max_lr\", \"warmup_max_lr\"])\n    def _do_finetune(\n        self,\n        loader: DataLoader,\n        n_epochs: int = 5,\n        max_lr: float = 0.01,\n        routine: Literal[\"howard\", \"felbo\"] = \"howard\",\n        deeptabular_gradual: bool = False,\n        deeptabular_layers: Optional[\n            Union[List[nn.Module], List[List[nn.Module]]]\n        ] = None,\n        deeptabular_max_lr: Union[float, List[float]] = 0.01,\n        deeptext_gradual: bool = False,\n        deeptext_layers: Optional[Union[List[nn.Module], List[List[nn.Module]]]] = None,\n        deeptext_max_lr: Union[float, List[float]] = 0.01,\n        deepimage_gradual: bool = False,\n        deepimage_layers: Optional[\n            Union[List[nn.Module], List[List[nn.Module]]]\n        ] = None,\n        deepimage_max_lr: Union[float, List[float]] = 0.01,\n    ):\n        r\"\"\"\n        Simple wrap-up to individually fine-tune model components\n        \"\"\"\n        if self.model.deephead is not None:\n            raise ValueError(\n                \"Currently warming up is only supported without a fully connected 'DeepHead'\"\n            )\n\n        finetuner = FineTune(self.loss_fn, self.metric, self.method, self.verbose)  # type: ignore[arg-type]\n        if self.model.wide:\n            finetuner.finetune_all(self.model.wide, \"wide\", loader, n_epochs, max_lr)\n\n        if self.model.deeptabular:\n            if deeptabular_gradual:\n                assert (\n                    deeptabular_layers is not None\n                ), \"deeptabular_layers must be passed if deeptabular_gradual=True\"\n                finetuner.finetune_gradual(\n                    self.model.deeptabular,\n                    \"deeptabular\",\n                    loader,\n                    deeptabular_max_lr,\n                    deeptabular_layers,\n                    routine,\n                )\n            else:\n                finetuner.finetune_all(\n                    self.model.deeptabular, \"deeptabular\", loader, n_epochs, max_lr\n                )\n\n        if self.model.deeptext:\n            if deeptext_gradual:\n                assert (\n                    deeptext_layers is not None\n                ), \"deeptext_layers must be passed if deeptext_gradual=True\"\n                finetuner.finetune_gradual(\n                    self.model.deeptext,\n                    \"deeptext\",\n                    loader,\n                    deeptext_max_lr,\n                    deeptext_layers,\n                    routine,\n                )\n            else:\n                finetuner.finetune_all(\n                    self.model.deeptext, \"deeptext\", loader, n_epochs, max_lr\n                )\n\n        if self.model.deepimage:\n            if deepimage_gradual:\n                assert (\n                    deepimage_layers is not None\n                ), \"deepimage_layers must be passed if deepimage_gradual=True\"\n                finetuner.finetune_gradual(\n                    self.model.deepimage,\n                    \"deepimage\",\n                    loader,\n                    deepimage_max_lr,\n                    deepimage_layers,\n                    routine,\n                )\n            else:\n                finetuner.finetune_all(\n                    self.model.deepimage, \"deepimage\", loader, n_epochs, max_lr\n                )\n\n    def _train_epoch(\n        self,\n        train_loader: DataLoader,\n        epoch: int,\n    ):\n        epoch_logs: Dict[str, float] = {}\n        self.callback_container.on_epoch_begin(epoch, logs=epoch_logs)\n        self.train_running_loss = 0.0\n\n        train_steps = len(train_loader)\n        with trange(train_steps, disable=self.verbose != 1) as t:\n            for batch_idx, (data, targett) in zip(t, train_loader):\n                t.set_description(\"epoch %i\" % (epoch + 1))\n                train_score, train_loss = self._train_step(data, targett, batch_idx)\n                print_loss_and_metric(t, train_loss, train_score)\n                self.callback_container.on_batch_end(batch=batch_idx)\n\n        epoch_logs = save_epoch_logs(epoch_logs, train_loss, train_score, \"train\")\n\n        return epoch_logs\n\n    def _train_step(\n        self,\n        data: Dict[str, Union[Tensor, List[Tensor]]],\n        target: Tensor,\n        batch_idx: int,\n    ):\n\n        self.model.train()\n\n        X: Dict[str, Union[Tensor, List[Tensor]]] = {}\n        for k, v in data.items():\n            if isinstance(v, list):\n                X[k] = [to_device(i, self.device) for i in v]\n            else:\n                X[k] = to_device(v, self.device)\n        y = (\n            target.view(-1, 1).float()\n            if self.method not in [\"multiclass\", \"qregression\", \"multitarget\"]\n            else target\n        )\n        y = to_device(y, self.device)\n\n        self.optimizer.zero_grad()\n\n        y_pred = self.model(X)\n\n        if self.model.is_tabnet:\n            loss = self.loss_fn(y_pred[0], y) - self.lambda_sparse * y_pred[1]\n            score = self._get_score(y_pred[0], y, is_train=True)\n        else:\n            loss = self.loss_fn(y_pred, y)\n            score = self._get_score(y_pred, y, is_train=True)\n\n        loss.backward()\n        self.optimizer.step()\n\n        self.train_running_loss += loss.item()\n        avg_loss = self.train_running_loss / (batch_idx + 1)\n\n        return score, avg_loss\n\n    def _eval_epoch(\n        self,\n        eval_loader: DataLoader,\n        epoch_logs: Dict[str, float],\n    ) -&gt; Tuple[Dict[str, float], Optional[float]]:\n        self.callback_container.on_eval_begin()\n        self.valid_running_loss = 0.0\n\n        eval_steps = len(eval_loader)\n        with trange(eval_steps, disable=self.verbose != 1) as v:\n            for i, (data, targett) in zip(v, eval_loader):\n                v.set_description(\"valid\")\n                val_score, val_loss = self._eval_step(data, targett, i)\n                print_loss_and_metric(v, val_loss, val_score)\n\n        epoch_logs = save_epoch_logs(epoch_logs, val_loss, val_score, \"val\")\n\n        if self.reducelronplateau:\n            if self.reducelronplateau_criterion == \"loss\":\n                on_epoch_end_metric = val_loss\n            else:\n                on_epoch_end_metric = val_score[self.reducelronplateau_criterion]\n        else:\n            on_epoch_end_metric = None\n\n        return epoch_logs, on_epoch_end_metric\n\n    def _eval_step(\n        self,\n        data: Dict[str, Union[Tensor, List[Tensor]]],\n        target: Tensor,\n        batch_idx: int,\n    ):\n        self.model.eval()\n        with torch.no_grad():\n            X: Dict[str, Union[Tensor, List[Tensor]]] = {}\n            for k, v in data.items():\n                if isinstance(v, list):\n                    X[k] = [to_device(i, self.device) for i in v]\n                else:\n                    X[k] = to_device(v, self.device)\n            y = (\n                target.view(-1, 1).float()\n                if self.method not in [\"multiclass\", \"qregression\", \"multitarget\"]\n                else target\n            )\n            y = to_device(y, self.device)\n\n            y_pred = self.model(X)\n            if self.model.is_tabnet:\n                loss = self.loss_fn(y_pred[0], y) - self.lambda_sparse * y_pred[1]\n                score = self._get_score(y_pred[0], y, is_train=False)\n            else:\n                score = self._get_score(y_pred, y, is_train=False)\n                loss = self.loss_fn(y_pred, y)\n\n            self.valid_running_loss += loss.item()\n            avg_loss = self.valid_running_loss / (batch_idx + 1)\n\n        self.model.train()\n        return score, avg_loss\n\n    def _get_score(\n        self, y_pred: Tensor, y: Tensor, is_train: bool\n    ) -&gt; Optional[Dict[str, float]]:\n\n        score = None\n        metric = None\n\n        if self.metric and not self.eval_metric:\n            metric = self.metric\n        elif self.metric and self.eval_metric:\n            metric = self.metric if is_train else self.eval_metric\n        elif not self.metric and self.eval_metric:\n            metric = None if is_train else self.eval_metric\n\n        if metric is not None:\n            if self.method == \"regression\":\n                score = metric(y_pred, y)\n            if self.method == \"binary\":\n                score = metric(torch.sigmoid(y_pred), y)\n            if self.method == \"qregression\":\n                score = metric(y_pred, y)\n            if self.method == \"multiclass\":\n                score = metric(F.softmax(y_pred, dim=1), y)\n        return score\n\n    def _predict(  # type: ignore[override, return]  # noqa: C901\n        self,\n        X_wide: Optional[np.ndarray] = None,\n        X_tab: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n        X_text: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n        X_img: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n        X_test: Optional[Dict[str, Union[np.ndarray, List[np.ndarray]]]] = None,\n        batch_size: Optional[int] = None,\n        uncertainty_granularity: int = 1000,\n        uncertainty: bool = False,\n    ) -&gt; List:\n        r\"\"\"Private method to avoid code repetition in predict and\n        predict_proba. For parameter information, please, see the .predict()\n        method documentation\n        \"\"\"\n        if X_test is not None:\n            test_set = WideDeepDataset(**X_test)  # type: ignore[arg-type]\n        else:\n            load_dict: Dict[str, Union[np.ndarray, List[np.ndarray]]] = {}\n            if X_wide is not None:\n                load_dict = {\"X_wide\": X_wide}\n            if X_tab is not None:\n                load_dict.update({\"X_tab\": X_tab})\n            if X_text is not None:\n                load_dict.update({\"X_text\": X_text})\n            if X_img is not None:\n                load_dict.update({\"X_img\": X_img})\n            test_set = WideDeepDataset(**load_dict)  # type: ignore[arg-type]\n\n        if not hasattr(self, \"batch_size\"):\n            assert batch_size is not None, (\n                \"'batch_size' has not be previosly set in this Trainer and must be\"\n                \" specified via the 'batch_size' argument in this predict call\"\n            )\n            self.batch_size = batch_size\n\n        test_loader = DataLoader(\n            dataset=test_set,\n            batch_size=self.batch_size,\n            num_workers=self.num_workers,\n            shuffle=False,\n        )\n        test_steps = (len(test_loader.dataset) // test_loader.batch_size) + 1  # type: ignore[arg-type]\n\n        self.model.eval()\n        preds_l = []\n\n        if uncertainty:\n            for m in self.model.modules():\n                if m.__class__.__name__.startswith(\"Dropout\"):\n                    m.train()\n            prediction_iters = uncertainty_granularity\n        else:\n            prediction_iters = 1\n\n        with torch.no_grad():\n            with trange(uncertainty_granularity, disable=uncertainty is False) as t:\n                for _, _ in zip(t, range(prediction_iters)):\n                    t.set_description(\"predict_UncertaintyIter\")\n\n                    with trange(\n                        test_steps, disable=self.verbose != 1 or uncertainty is True\n                    ) as tt:\n                        for _, data in zip(tt, test_loader):\n                            tt.set_description(\"predict\")\n                            X: Dict[str, Union[Tensor, List[Tensor]]] = {}\n                            for k, v in data.items():\n                                if isinstance(v, list):\n                                    X[k] = [to_device(i, self.device) for i in v]\n                                else:\n                                    X[k] = to_device(v, self.device)\n                            preds = (\n                                self.model(X)\n                                if not self.model.is_tabnet\n                                else self.model(X)[0]\n                            )\n                            if self.method == \"binary\":\n                                preds = torch.sigmoid(preds)\n                            if self.method == \"multiclass\":\n                                preds = F.softmax(preds, dim=1)\n                            if self.method == \"regression\" and isinstance(\n                                self.loss_fn, ZILNLoss\n                            ):\n                                preds = self._predict_ziln(preds)\n                            preds = preds.cpu().data.numpy()\n                            preds_l.append(preds)\n        self.model.train()\n        return preds_l\n\n    @staticmethod\n    def _predict_ziln(preds: Tensor) -&gt; Tensor:\n        # Legacy implementation. It will be removed in future versions\n        \"\"\"Calculates predicted mean of zero inflated lognormal logits.\n\n        Adjusted implementaion of `code\n        &lt;https://github.com/google/lifetime_value/blob/master/lifetime_value/zero_inflated_lognormal.py&gt;`\n\n        Arguments:\n            preds: [batch_size, 3] tensor of logits.\n        Returns:\n            ziln_preds: [batch_size, 1] tensor of predicted mean.\n        \"\"\"\n        positive_probs = torch.sigmoid(preds[..., :1])\n        loc = preds[..., 1:2]\n        scale = F.softplus(preds[..., 2:])\n        ziln_preds = positive_probs * torch.exp(loc + 0.5 * torch.square(scale))\n        return ziln_preds\n\n    @staticmethod\n    def _extract_kwargs(kwargs):\n        dataloader_params = [\n            \"shuffle\",\n            \"sampler\",\n            \"batch_sampler\",\n            \"num_workers\",\n            \"collate_fn\",\n            \"pin_memory\",\n            \"drop_last\",\n            \"timeout\",\n            \"worker_init_fn\",\n            \"generator\",\n            \"prefetch_factor\",\n            \"persistent_workers\",\n            \"oversample_mul\",\n        ]\n        finetune_params = [\n            \"n_epochs\",\n            \"finetune_epochs\",\n            \"warmup_epochs\",\n            \"max_lr\",\n            \"finetune_max_lr\",\n            \"warmup_max_lr\",\n            \"routine\",\n            \"deeptabular_gradual\",\n            \"deeptabular_layers\",\n            \"deeptabular_max_lr\",\n            \"deeptext_gradual\",\n            \"deeptext_layers\",\n            \"deeptext_max_lr\",\n            \"deepimage_gradual\",\n            \"deepimage_layers\",\n            \"deepimage_max_lr\",\n        ]\n\n        dataloader_args, finetune_args = {}, {}\n        for k, v in kwargs.items():\n            if k in dataloader_params:\n                dataloader_args[k] = v\n            if k in finetune_params:\n                finetune_args[k] = v\n\n        return dataloader_args, finetune_args\n</code></pre>"},{"location":"pytorch-widedeep/trainer.html#pytorch_widedeep.training.Trainer.fit","title":"fit","text":"<pre><code>fit(\n    X_wide=None,\n    X_tab=None,\n    X_text=None,\n    X_img=None,\n    X_train=None,\n    X_val=None,\n    val_split=None,\n    target=None,\n    n_epochs=1,\n    validation_freq=1,\n    batch_size=32,\n    train_dataloader=None,\n    eval_dataloader=None,\n    feature_importance_sample_size=None,\n    finetune=False,\n    **kwargs\n)\n</code></pre> <p>Fit method.</p> <p>The input datasets can be passed either directly via numpy arrays (<code>X_wide</code>, <code>X_tab</code>, <code>X_text</code> or <code>X_img</code>) or alternatively, in dictionaries (<code>X_train</code> or <code>X_val</code>).</p> <p>Parameters:</p> Name Type Description Default <code>X_wide</code> <code>Optional[ndarray]</code> <p>Input for the <code>wide</code> model component. See <code>pytorch_widedeep.preprocessing.WidePreprocessor</code></p> <code>None</code> <code>X_tab</code> <code>Optional[Union[ndarray, List[ndarray]]]</code> <p>Input for the <code>deeptabular</code> model component. See <code>pytorch_widedeep.preprocessing.TabPreprocessor</code>. If multiple tabular models are used for different columns, this should be a list of numpy arrays</p> <code>None</code> <code>X_text</code> <code>Optional[Union[ndarray, List[ndarray]]]</code> <p>Input for the <code>deeptext</code> model component. See <code>pytorch_widedeep.preprocessing.TextPreprocessor</code>. If multiple text columns/models are used, this should be a list of numpy arrays</p> <code>None</code> <code>X_img</code> <code>Optional[Union[ndarray, List[ndarray]]]</code> <p>Input for the <code>deepimage</code> model component. See <code>pytorch_widedeep.preprocessing.ImagePreprocessor</code>. If multiple image columns/models are used, this should be a list of numpy arrays</p> <code>None</code> <code>X_train</code> <code>Optional[Dict[str, Union[ndarray, List[ndarray]]]]</code> <p>The training dataset can also be passed in a dictionary. Keys are 'X_wide', 'X_tab', 'X_text', 'X_img' and 'target'. Values are the corresponding matrices. Note that of multiple text or image columns/models are used, the corresponding values should be lists of numpy arrays</p> <code>None</code> <code>X_val</code> <code>Optional[Dict[str, Union[ndarray, List[ndarray]]]]</code> <p>The validation dataset can also be passed in a dictionary. Keys are 'X_wide', 'X_tab', 'X_text', 'X_img' and 'target'. Values are the corresponding matrices. Note that of multiple text or image columns/models are used, the corresponding values should be lists of numpy arrays</p> <code>None</code> <code>val_split</code> <code>Optional[float]</code> <p>train/val split fraction</p> <code>None</code> <code>target</code> <code>Optional[ndarray]</code> <p>target values</p> <code>None</code> <code>n_epochs</code> <code>int</code> <p>number of epochs</p> <code>1</code> <code>validation_freq</code> <code>int</code> <p>epochs validation frequency</p> <code>1</code> <code>batch_size</code> <code>int</code> <p>batch size</p> <code>32</code> <code>custom_dataloader</code> <p>object of class <code>torch.utils.data.DataLoader</code>. Available predefined dataloaders are in <code>pytorch-widedeep.dataloaders</code>.If <code>None</code>, a standard torch <code>DataLoader</code> is used.</p> required <code>finetune</code> <code>bool</code> <p>fine-tune individual model components. This functionality can also be used to 'warm-up' (and hence the alias <code>warmup</code>) individual components before the joined training starts, and hence its alias. See the Examples folder in the repo for more details</p> <p><code>pytorch_widedeep</code> implements 3 fine-tune routines.</p> <ul> <li>fine-tune all trainable layers at once. This routine is   inspired by the work of Howard &amp; Sebastian Ruder 2018 in their   ULMfit paper. Using a   Slanted Triangular learing (see   Leslie N. Smith paper ) ,   the process is the following: i) the learning rate will   gradually increase for 10% of the training steps from max_lr/10   to max_lr. ii) It will then gradually decrease to max_lr/10   for the remaining 90% of the steps. The optimizer used in the   process is <code>Adam</code>.</li> </ul> <p>and two gradual fine-tune routines, where only certain layers are trained at a time.</p> <ul> <li>The so called <code>Felbo</code> gradual fine-tune rourine, based on the the   Felbo et al., 2017 DeepEmoji paper.</li> <li>The <code>Howard</code> routine based on the work of Howard &amp; Sebastian Ruder 2018 in their   ULMfit paper.</li> </ul> <p>For details on how these routines work, please see the Examples section in this documentation and the Examples folder in the repo.  Param Alias: <code>warmup</code></p> <code>False</code> <p>Other Parameters:</p> Name Type Description <code>**kwargs</code> <p>Other keyword arguments are:</p> <ul> <li> <p>DataLoader related parameters:     For example,  <code>sampler</code>, <code>batch_sampler</code>, <code>collate_fn</code>, etc.     Please, see the pytorch     DataLoader docs     for details.</p> </li> <li> <p>Finetune related parameters:     see the source code at <code>pytorch_widedeep._finetune</code>. Namely, these are:</p> <ul> <li><code>finetune_epochs</code> (<code>int</code>):     number of epochs use for fine tuning</li> <li><code>finetune_max_lr</code> (<code>float</code>):    max lr during fine tuning</li> <li><code>routine</code> (<code>str</code>):    one of 'howard' or 'felbo'</li> <li><code>deeptabular_gradual</code> (<code>bool</code>):    boolean indicating if the <code>deeptabular</code> component will be fine tuned gradually</li> <li><code>deeptabular_layers</code> (<code>Optional[Union[List[nn.Module], List[List[nn.Module]]]]</code>):    List of pytorch modules indicating the layers of the    <code>deeptabular</code> that will be fine tuned</li> <li><code>deeptabular_max_lr</code> (<code>Union[float, List[float]]</code>):    max lr for the <code>deeptabular</code> componet during fine tuning</li> <li><code>deeptext_gradual</code> (<code>bool</code>):    same as <code>deeptabular_gradual</code> but for the <code>deeptext</code> component</li> <li><code>deeptext_layers</code> (<code>Optional[Union[List[nn.Module], List[List[nn.Module]]]]</code>):    same as <code>deeptabular_gradual</code> but for the <code>deeptext</code> component.    If there are multiple text columns/models, this should be a list of lists</li> <li><code>deeptext_max_lr</code> (<code>Union[float, List[float]]</code>):    same as <code>deeptabular_gradual</code> but for the <code>deeptext</code> component    If there are multiple text columns/models, this should be a list of floats</li> <li><code>deepimage_gradual</code> (<code>bool</code>):    same as <code>deeptext_layers</code> but for the <code>deepimage</code> component</li> <li><code>deepimage_layers</code> (<code>Optional[Union[List[nn.Module], List[List[nn.Module]]]]</code>):    same as <code>deeptext_layers</code> but for the <code>deepimage</code> component</li> <li><code>deepimage_max_lr</code> (<code>Union[float, List[float]]</code>):     same as <code>deeptext_layers</code> but for the <code>deepimage</code> component</li> </ul> </li> </ul> <p>Examples:</p> <p>For a series of comprehensive examples on how to use the <code>fit</code> method, please see the Examples folder in the repo</p> Source code in <code>pytorch_widedeep/training/trainer.py</code> <pre><code>@alias(\"finetune\", [\"warmup\"])\n@alias(\"train_dataloader\", [\"custom_dataloader\"])\n@alias(\"eval_dataloader\", [\"custom_eval_dataloader\"])\ndef fit(  # noqa: C901\n    self,\n    X_wide: Optional[np.ndarray] = None,\n    X_tab: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n    X_text: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n    X_img: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n    X_train: Optional[Dict[str, Union[np.ndarray, List[np.ndarray]]]] = None,\n    X_val: Optional[Dict[str, Union[np.ndarray, List[np.ndarray]]]] = None,\n    val_split: Optional[float] = None,\n    target: Optional[np.ndarray] = None,\n    n_epochs: int = 1,\n    validation_freq: int = 1,\n    batch_size: int = 32,\n    train_dataloader: Optional[CustomDataLoader] = None,\n    eval_dataloader: Optional[CustomDataLoader] = None,\n    feature_importance_sample_size: Optional[int] = None,\n    finetune: bool = False,\n    **kwargs,\n):\n    r\"\"\"Fit method.\n\n    The input datasets can be passed either directly via numpy arrays\n    (`X_wide`, `X_tab`, `X_text` or `X_img`) or alternatively, in\n    dictionaries (`X_train` or `X_val`).\n\n    Parameters\n    ----------\n    X_wide: np.ndarray, Optional. default=None\n        Input for the `wide` model component.\n        See `pytorch_widedeep.preprocessing.WidePreprocessor`\n    X_tab: np.ndarray, Optional. default=None\n        Input for the `deeptabular` model component.\n        See `pytorch_widedeep.preprocessing.TabPreprocessor`. If multiple\n        tabular models are used for different columns, this should be a\n        list of numpy arrays\n    X_text: Union[np.ndarray, List[np.ndarray]], Optional. default=None\n        Input for the `deeptext` model component.\n        See `pytorch_widedeep.preprocessing.TextPreprocessor`.\n        If multiple text columns/models are used, this should be a list of\n        numpy arrays\n    X_img: np.ndarray, Optional. default=None\n        Input for the `deepimage` model component.\n        See `pytorch_widedeep.preprocessing.ImagePreprocessor`.\n        If multiple image columns/models are used, this should be a list of\n        numpy arrays\n    X_train: Dict, Optional. default=None\n        The training dataset can also be passed in a dictionary. Keys are\n        _'X_wide'_, _'X_tab'_, _'X_text'_, _'X_img'_ and _'target'_. Values\n        are the corresponding matrices. Note that of multiple text or image\n        columns/models are used, the corresponding values should be lists\n        of numpy arrays\n    X_val: Dict, Optional. default=None\n        The validation dataset can also be passed in a dictionary. Keys\n        are _'X_wide'_, _'X_tab'_, _'X_text'_, _'X_img'_ and _'target'_.\n        Values are the corresponding matrices. Note that of multiple text\n        or image columns/models are used, the corresponding values should\n        be lists of numpy arrays\n    val_split: float, Optional. default=None\n        train/val split fraction\n    target: np.ndarray, Optional. default=None\n        target values\n    n_epochs: int, default=1\n        number of epochs\n    validation_freq: int, default=1\n        epochs validation frequency\n    batch_size: int, default=32\n        batch size\n    custom_dataloader: `DataLoader`, Optional, default=None\n        object of class `torch.utils.data.DataLoader`. Available\n        predefined dataloaders are in `pytorch-widedeep.dataloaders`.If\n        `None`, a standard torch `DataLoader` is used.\n    finetune: bool, default=False\n        fine-tune individual model components. This functionality can also\n        be used to 'warm-up' (and hence the alias `warmup`) individual\n        components before the joined training starts, and hence its\n        alias. See the Examples folder in the repo for more details\n\n        `pytorch_widedeep` implements 3 fine-tune routines.\n\n        - fine-tune all trainable layers at once. This routine is\n          inspired by the work of Howard &amp; Sebastian Ruder 2018 in their\n          [ULMfit paper](https://arxiv.org/abs/1801.06146). Using a\n          Slanted Triangular learing (see\n          [Leslie N. Smith paper](https://arxiv.org/pdf/1506.01186.pdf) ) ,\n          the process is the following: *i*) the learning rate will\n          gradually increase for 10% of the training steps from max_lr/10\n          to max_lr. *ii*) It will then gradually decrease to max_lr/10\n          for the remaining 90% of the steps. The optimizer used in the\n          process is `Adam`.\n\n        and two gradual fine-tune routines, where only certain layers are\n        trained at a time.\n\n        - The so called `Felbo` gradual fine-tune rourine, based on the the\n          Felbo et al., 2017 [DeepEmoji paper](https://arxiv.org/abs/1708.00524).\n        - The `Howard` routine based on the work of Howard &amp; Sebastian Ruder 2018 in their\n          [ULMfit paper](https://arxiv.org/abs/1801.06146&gt;).\n\n        For details on how these routines work, please see the Examples\n        section in this documentation and the Examples folder in the repo. &lt;br/&gt;\n        Param Alias: `warmup`\n\n    Other Parameters\n    ----------------\n    **kwargs:\n        Other keyword arguments are:\n\n        - **DataLoader related parameters**:&lt;br/&gt;\n            For example,  `sampler`, `batch_sampler`, `collate_fn`, etc.\n            Please, see the pytorch\n            [DataLoader docs](https://pytorch.org/docs/stable/data.html#torch.utils.data.DataLoader)\n            for details.\n\n        - **Finetune related parameters**:&lt;br/&gt;\n            see the source code at `pytorch_widedeep._finetune`. Namely, these are:\n\n            - `finetune_epochs` (`int`):\n                number of epochs use for fine tuning\n            - `finetune_max_lr` (`float`):\n               max lr during fine tuning\n            - `routine` (`str`):\n               one of _'howard'_ or _'felbo'_\n            - `deeptabular_gradual` (`bool`):\n               boolean indicating if the `deeptabular` component will be fine tuned gradually\n            - `deeptabular_layers` (`Optional[Union[List[nn.Module], List[List[nn.Module]]]]`):\n               List of pytorch modules indicating the layers of the\n               `deeptabular` that will be fine tuned\n            - `deeptabular_max_lr` (`Union[float, List[float]]`):\n               max lr for the `deeptabular` componet during fine tuning\n            - `deeptext_gradual` (`bool`):\n               same as `deeptabular_gradual` but for the `deeptext` component\n            - `deeptext_layers` (`Optional[Union[List[nn.Module], List[List[nn.Module]]]]`):\n               same as `deeptabular_gradual` but for the `deeptext` component.\n               If there are multiple text columns/models, this should be a list of lists\n            - `deeptext_max_lr` (`Union[float, List[float]]`):\n               same as `deeptabular_gradual` but for the `deeptext` component\n               If there are multiple text columns/models, this should be a list of floats\n            - `deepimage_gradual` (`bool`):\n               same as `deeptext_layers` but for the `deepimage` component\n            - `deepimage_layers` (`Optional[Union[List[nn.Module], List[List[nn.Module]]]]`):\n               same as `deeptext_layers` but for the `deepimage` component\n            - `deepimage_max_lr` (`Union[float, List[float]]`):\n                same as `deeptext_layers` but for the `deepimage` component\n\n    Examples\n    --------\n\n    For a series of comprehensive examples on how to use the `fit` method, please see the\n    [Examples](https://github.com/jrzaurin/pytorch-widedeep/tree/master/examples)\n    folder in the repo\n    \"\"\"\n    dataloader_args, finetune_args = self._extract_kwargs(kwargs)\n    self.batch_size = batch_size\n    train_set, eval_set = wd_train_val_split(\n        self.seed,\n        self.method,  # type: ignore\n        X_wide,\n        X_tab,\n        X_text,\n        X_img,\n        X_train,\n        X_val,\n        val_split,\n        target,\n        self.transforms,\n    )\n    train_loader = self._set_dataloader(\n        train_dataloader, train_set, batch_size, dataloader_args\n    )\n    eval_loader = (\n        self._set_dataloader(eval_dataloader, eval_set, batch_size, dataloader_args)\n        if eval_set is not None\n        else None\n    )\n\n    if finetune:\n        self.with_finetuning: bool = True\n        self._do_finetune(train_loader, **finetune_args)\n        if self.verbose:\n            print(\n                \"Fine-tuning (or warmup) of individual components completed. \"\n                \"Training the whole model for {} epochs\".format(n_epochs)\n            )\n    else:\n        self.with_finetuning = False\n\n    self.callback_container.on_train_begin(\n        {\n            \"batch_size\": batch_size,\n            \"train_steps\": len(train_loader),\n            \"n_epochs\": n_epochs,\n        }\n    )\n    for epoch in range(n_epochs):\n        epoch_logs = self._train_epoch(train_loader, epoch)\n        if eval_loader is not None and epoch % validation_freq == (\n            validation_freq - 1\n        ):\n            epoch_logs, on_epoch_end_metric = self._eval_epoch(\n                eval_loader, epoch_logs\n            )\n        else:\n            on_epoch_end_metric = None\n            if self.reducelronplateau:\n                raise NotImplementedError(\n                    \"ReduceLROnPlateau scheduler can be used only with validation data.\"\n                )\n        self.callback_container.on_epoch_end(epoch, epoch_logs, on_epoch_end_metric)\n        if self.early_stop:\n            # self.callback_container.on_train_end(epoch_logs)\n            break\n\n    self.callback_container.on_train_end(epoch_logs)\n    if feature_importance_sample_size is not None:\n        self.feature_importance = FeatureImportance(\n            self.device, feature_importance_sample_size\n        ).feature_importance(train_loader, self.model)\n    self._restore_best_weights()\n    self.model.train()\n</code></pre>"},{"location":"pytorch-widedeep/trainer.html#pytorch_widedeep.training.Trainer.predict","title":"predict","text":"<pre><code>predict(\n    X_wide=None,\n    X_tab=None,\n    X_text=None,\n    X_img=None,\n    X_test=None,\n    batch_size=None,\n)\n</code></pre> <p>Returns the predictions</p> <p>The input datasets can be passed either directly via numpy arrays (<code>X_wide</code>, <code>X_tab</code>, <code>X_text</code> or <code>X_img</code>) or alternatively, in a dictionary (<code>X_test</code>)</p> <p>Parameters:</p> Name Type Description Default <code>X_wide</code> <code>Optional[ndarray]</code> <p>Input for the <code>wide</code> model component. See <code>pytorch_widedeep.preprocessing.WidePreprocessor</code></p> <code>None</code> <code>X_tab</code> <code>Optional[Union[ndarray, List[ndarray]]]</code> <p>Input for the <code>deeptabular</code> model component. See <code>pytorch_widedeep.preprocessing.TabPreprocessor</code></p> <code>None</code> <code>X_text</code> <code>Optional[Union[ndarray, List[ndarray]]]</code> <p>Input for the <code>deeptext</code> model component. See <code>pytorch_widedeep.preprocessing.TextPreprocessor</code></p> <code>None</code> <code>X_img</code> <code>Optional[Union[ndarray, List[ndarray]]]</code> <p>Input for the <code>deepimage</code> model component. See <code>pytorch_widedeep.preprocessing.ImagePreprocessor</code></p> <code>None</code> <code>X_test</code> <code>Optional[Dict[str, Union[ndarray, List[ndarray]]]]</code> <p>The test dataset can also be passed in a dictionary. Keys are <code>X_wide</code>, 'X_tab', 'X_text', 'X_img' and 'target'. Values are the corresponding matrices.</p> <code>None</code> <code>batch_size</code> <code>Optional[int]</code> <p>If a trainer is used to predict after having trained a model, the <code>batch_size</code> needs to be defined as it will not be defined as the <code>Trainer</code> is instantiated</p> <code>None</code> <p>Returns:</p> Type Description <code>np.ndarray:</code> <p>array with the predictions</p> Source code in <code>pytorch_widedeep/training/trainer.py</code> <pre><code>def predict(  # type: ignore[override, return]\n    self,\n    X_wide: Optional[np.ndarray] = None,\n    X_tab: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n    X_text: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n    X_img: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n    X_test: Optional[Dict[str, Union[np.ndarray, List[np.ndarray]]]] = None,\n    batch_size: Optional[int] = None,\n) -&gt; np.ndarray:\n    r\"\"\"Returns the predictions\n\n    The input datasets can be passed either directly via numpy arrays\n    (`X_wide`, `X_tab`, `X_text` or `X_img`) or alternatively, in\n    a dictionary (`X_test`)\n\n\n    Parameters\n    ----------\n    X_wide: np.ndarray, Optional. default=None\n        Input for the `wide` model component.\n        See `pytorch_widedeep.preprocessing.WidePreprocessor`\n    X_tab: np.ndarray or List[np.ndarray], Optional. default=None\n        Input for the `deeptabular` model component.\n        See `pytorch_widedeep.preprocessing.TabPreprocessor`\n    X_text: np.ndarray or List[np.ndarray], Optional. default=None\n        Input for the `deeptext` model component.\n        See `pytorch_widedeep.preprocessing.TextPreprocessor`\n    X_img: np.ndarray or List[np.ndarray], Optional. default=None\n        Input for the `deepimage` model component.\n        See `pytorch_widedeep.preprocessing.ImagePreprocessor`\n    X_test: Dict, Optional. default=None\n        The test dataset can also be passed in a dictionary. Keys are\n        `X_wide`, _'X_tab'_, _'X_text'_, _'X_img'_ and _'target'_. Values\n        are the corresponding matrices.\n    batch_size: int, default = 256\n        If a trainer is used to predict after having trained a model, the\n        `batch_size` needs to be defined as it will not be defined as\n        the `Trainer` is instantiated\n\n    Returns\n    -------\n    np.ndarray:\n        array with the predictions\n    \"\"\"\n    preds_l = self._predict(X_wide, X_tab, X_text, X_img, X_test, batch_size)\n    if self.method == \"regression\":\n        return np.vstack(preds_l).squeeze(1)\n    if self.method == \"binary\":\n        preds = np.vstack(preds_l).squeeze(1)\n        return (preds &gt; 0.5).astype(\"int\")\n    if self.method == \"qregression\":\n        return np.vstack(preds_l)\n    if self.method == \"multiclass\":\n        preds = np.vstack(preds_l)\n        return np.argmax(preds, 1)  # type: ignore[return-value]\n    if self.method == \"multitarget\":\n        if self.loss_fn.__class__.__name__ in [\n            \"MultiTargetClassificationLoss\",\n            \"MutilTargetRegressionAndClassificationLoss\",\n        ]:\n            raise ValueError(\n                \"MultiTargetClassificationLoss and MutilTargetRegressionAndClassificationLoss \"\n                \"are not supported by predict method. Please use predict_proba method instead.\"\n            )\n        return np.vstack(preds_l)\n</code></pre>"},{"location":"pytorch-widedeep/trainer.html#pytorch_widedeep.training.Trainer.predict_uncertainty","title":"predict_uncertainty","text":"<pre><code>predict_uncertainty(\n    X_wide=None,\n    X_tab=None,\n    X_text=None,\n    X_img=None,\n    X_test=None,\n    batch_size=None,\n    uncertainty_granularity=1000,\n)\n</code></pre> <p>Returns the predicted ucnertainty of the model for the test dataset using a Monte Carlo method during which dropout layers are activated in the evaluation/prediction phase and each sample is predicted N times (<code>uncertainty_granularity</code> times).</p> <p>This is based on Dropout as a Bayesian Approximation: Representing Model Uncertainty in Deep Learning.</p> <p>Parameters:</p> Name Type Description Default <code>X_wide</code> <code>Optional[ndarray]</code> <p>Input for the <code>wide</code> model component. See <code>pytorch_widedeep.preprocessing.WidePreprocessor</code></p> <code>None</code> <code>X_tab</code> <code>Optional[Union[ndarray, List[ndarray]]]</code> <p>Input for the <code>deeptabular</code> model component. See <code>pytorch_widedeep.preprocessing.TabPreprocessor</code></p> <code>None</code> <code>X_text</code> <code>Optional[Union[ndarray, List[ndarray]]]</code> <p>Input for the <code>deeptext</code> model component. See <code>pytorch_widedeep.preprocessing.TextPreprocessor</code></p> <code>None</code> <code>X_img</code> <code>Optional[Union[ndarray, List[ndarray]]]</code> <p>Input for the <code>deepimage</code> model component. See <code>pytorch_widedeep.preprocessing.ImagePreprocessor</code></p> <code>None</code> <code>X_test</code> <code>Optional[Dict[str, Union[ndarray, List[ndarray]]]]</code> <p>The test dataset can also be passed in a dictionary. Keys are 'X_wide', 'X_tab', 'X_text', 'X_img' and 'target'. Values are the corresponding matrices.</p> <code>None</code> <code>batch_size</code> <code>Optional[int]</code> <p>If a trainer is used to predict after having trained a model, the <code>batch_size</code> needs to be defined as it will not be defined as the <code>Trainer</code> is instantiated</p> <code>None</code> <code>uncertainty_granularity</code> <code>int</code> <p>number of times the model does prediction for each sample</p> <code>1000</code> <p>Returns:</p> Type Description <code>np.ndarray:</code> <ul> <li>if <code>method = regression</code>, it will return an array with <code>(max, min, mean, stdev)</code>   values for each sample.</li> <li>if <code>method = binary</code> it will return an array with   <code>(mean_cls_0_prob, mean_cls_1_prob, predicted_cls)</code> for each sample.</li> <li>if <code>method = multiclass</code> it will return an array with   <code>(mean_cls_0_prob, mean_cls_1_prob, mean_cls_2_prob, ... , predicted_cls)</code>   values for each sample.</li> </ul> Source code in <code>pytorch_widedeep/training/trainer.py</code> <pre><code>def predict_uncertainty(  # type: ignore[return]\n    self,\n    X_wide: Optional[np.ndarray] = None,\n    X_tab: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n    X_text: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n    X_img: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n    X_test: Optional[Dict[str, Union[np.ndarray, List[np.ndarray]]]] = None,\n    batch_size: Optional[int] = None,\n    uncertainty_granularity: int = 1000,\n) -&gt; np.ndarray:\n    r\"\"\"Returns the predicted ucnertainty of the model for the test dataset\n    using a Monte Carlo method during which dropout layers are activated\n    in the evaluation/prediction phase and each sample is predicted N\n    times (`uncertainty_granularity` times).\n\n    This is based on\n    [Dropout as a Bayesian Approximation: Representing\n    Model Uncertainty in Deep Learning](https://arxiv.org/abs/1506.02142?context=stat).\n\n    Parameters\n    ----------\n    X_wide: np.ndarray, Optional. default=None\n        Input for the `wide` model component.\n        See `pytorch_widedeep.preprocessing.WidePreprocessor`\n    X_tab: np.ndarray or List[np.ndarray], Optional. default=None\n        Input for the `deeptabular` model component.\n        See `pytorch_widedeep.preprocessing.TabPreprocessor`\n    X_text: np.ndarray or List[np.ndarray], Optional. default=None\n        Input for the `deeptext` model component.\n        See `pytorch_widedeep.preprocessing.TextPreprocessor`\n    X_img: np.ndarray or List[np.ndarray], Optional. default=None\n        Input for the `deepimage` model component.\n        See `pytorch_widedeep.preprocessing.ImagePreprocessor`\n    X_test: Dict, Optional. default=None\n        The test dataset can also be passed in a dictionary. Keys are\n        _'X_wide'_, _'X_tab'_, _'X_text'_, _'X_img'_ and _'target'_. Values\n        are the corresponding matrices.\n    batch_size: int, default = 256\n        If a trainer is used to predict after having trained a model, the\n        `batch_size` needs to be defined as it will not be defined as\n        the `Trainer` is instantiated\n    uncertainty_granularity: int default = 1000\n        number of times the model does prediction for each sample\n\n    Returns\n    -------\n    np.ndarray:\n        - if `method = regression`, it will return an array with `(max, min, mean, stdev)`\n          values for each sample.\n        - if `method = binary` it will return an array with\n          `(mean_cls_0_prob, mean_cls_1_prob, predicted_cls)` for each sample.\n        - if `method = multiclass` it will return an array with\n          `(mean_cls_0_prob, mean_cls_1_prob, mean_cls_2_prob, ... , predicted_cls)`\n          values for each sample.\n\n    \"\"\"\n    preds_l = self._predict(\n        X_wide,\n        X_tab,\n        X_text,\n        X_img,\n        X_test,\n        batch_size,\n        uncertainty_granularity,\n        uncertainty=True,\n    )\n    preds = np.vstack(preds_l)\n    samples_num = int(preds.shape[0] / uncertainty_granularity)\n    if self.method == \"regression\":\n        preds = preds.squeeze(1)\n        preds = preds.reshape((uncertainty_granularity, samples_num))\n        return np.array(\n            (\n                preds.max(axis=0),\n                preds.min(axis=0),\n                preds.mean(axis=0),\n                preds.std(axis=0),\n            )\n        ).T\n    if self.method == \"qregression\":\n        raise ValueError(\n            \"Currently predict_uncertainty is not supported for qregression method\"\n        )\n    if self.method == \"binary\":\n        preds = preds.squeeze(1)\n        preds = preds.reshape((uncertainty_granularity, samples_num))\n        preds = preds.mean(axis=0)\n        probs = np.zeros([preds.shape[0], 3])\n        probs[:, 0] = 1 - preds\n        probs[:, 1] = preds\n        return probs\n    if self.method == \"multiclass\":\n        preds = preds.reshape(uncertainty_granularity, samples_num, preds.shape[1])\n        preds = preds.mean(axis=0)\n        preds = np.hstack((preds, np.vstack(np.argmax(preds, 1))))\n        return preds\n\n    if self.method == \"multitarget\":\n        raise ValueError(\n            \"Currently predict_uncertainty is not supported for multitarget method\"\n        )\n</code></pre>"},{"location":"pytorch-widedeep/trainer.html#pytorch_widedeep.training.Trainer.predict_proba","title":"predict_proba","text":"<pre><code>predict_proba(\n    X_wide=None,\n    X_tab=None,\n    X_text=None,\n    X_img=None,\n    X_test=None,\n    batch_size=None,\n)\n</code></pre> <p>Returns the predicted probabilities for the test dataset for  binary and multiclass methods</p> <p>The input datasets can be passed either directly via numpy arrays (<code>X_wide</code>, <code>X_tab</code>, <code>X_text</code> or <code>X_img</code>) or alternatively, in a dictionary (<code>X_test</code>)</p> <p>Parameters:</p> Name Type Description Default <code>X_wide</code> <code>Optional[ndarray]</code> <p>Input for the <code>wide</code> model component. See <code>pytorch_widedeep.preprocessing.WidePreprocessor</code></p> <code>None</code> <code>X_tab</code> <code>Optional[Union[ndarray, List[ndarray]]]</code> <p>Input for the <code>deeptabular</code> model component. See <code>pytorch_widedeep.preprocessing.TabPreprocessor</code></p> <code>None</code> <code>X_text</code> <code>Optional[Union[ndarray, List[ndarray]]]</code> <p>Input for the <code>deeptext</code> model component. See <code>pytorch_widedeep.preprocessing.TextPreprocessor</code></p> <code>None</code> <code>X_img</code> <code>Optional[Union[ndarray, List[ndarray]]]</code> <p>Input for the <code>deepimage</code> model component. See <code>pytorch_widedeep.preprocessing.ImagePreprocessor</code></p> <code>None</code> <code>X_test</code> <code>Optional[Dict[str, Union[ndarray, List[ndarray]]]]</code> <p>The test dataset can also be passed in a dictionary. Keys are <code>X_wide</code>, 'X_tab', 'X_text', 'X_img' and 'target'. Values are the corresponding matrices.</p> <code>None</code> <code>batch_size</code> <code>Optional[int]</code> <p>If a trainer is used to predict after having trained a model, the <code>batch_size</code> needs to be defined as it will not be defined as the <code>Trainer</code> is instantiated</p> <code>None</code> <p>Returns:</p> Type Description <code>ndarray</code> <p>array with the probabilities per class</p> Source code in <code>pytorch_widedeep/training/trainer.py</code> <pre><code>def predict_proba(  # type: ignore[override, return]  # noqa: C901\n    self,\n    X_wide: Optional[np.ndarray] = None,\n    X_tab: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n    X_text: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n    X_img: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n    X_test: Optional[Dict[str, Union[np.ndarray, List[np.ndarray]]]] = None,\n    batch_size: Optional[int] = None,\n) -&gt; np.ndarray:\n    r\"\"\"Returns the predicted probabilities for the test dataset for  binary\n    and multiclass methods\n\n    The input datasets can be passed either directly via numpy arrays\n    (`X_wide`, `X_tab`, `X_text` or `X_img`) or alternatively, in\n    a dictionary (`X_test`)\n\n    Parameters\n    ----------\n    X_wide: np.ndarray, Optional. default=None\n        Input for the `wide` model component.\n        See `pytorch_widedeep.preprocessing.WidePreprocessor`\n    X_tab: np.ndarray or List[np.ndarray], Optional. default=None\n        Input for the `deeptabular` model component.\n        See `pytorch_widedeep.preprocessing.TabPreprocessor`\n    X_text: np.ndarray or List[np.ndarray], Optional. default=None\n        Input for the `deeptext` model component.\n        See `pytorch_widedeep.preprocessing.TextPreprocessor`\n    X_img: np.ndarray or List[np.ndarray], Optional. default=None\n        Input for the `deepimage` model component.\n        See `pytorch_widedeep.preprocessing.ImagePreprocessor`\n    X_test: Dict, Optional. default=None\n        The test dataset can also be passed in a dictionary. Keys are\n        `X_wide`, _'X_tab'_, _'X_text'_, _'X_img'_ and _'target'_. Values\n        are the corresponding matrices.\n    batch_size: int, default = 256\n        If a trainer is used to predict after having trained a model, the\n        `batch_size` needs to be defined as it will not be defined as\n        the `Trainer` is instantiated\n\n    Returns\n    -------\n    np.ndarray\n        array with the probabilities per class\n    \"\"\"\n\n    preds_l = self._predict(X_wide, X_tab, X_text, X_img, X_test, batch_size)\n    if self.method == \"binary\":\n        preds = np.vstack(preds_l).squeeze(1)\n        probs = np.zeros([preds.shape[0], 2])\n        probs[:, 0] = 1 - preds\n        probs[:, 1] = preds\n        return probs\n    if self.method == \"multiclass\":\n        return np.vstack(preds_l)\n    if self.method == \"multitarget\":\n        return np.vstack(preds_l)\n</code></pre>"},{"location":"pytorch-widedeep/trainer.html#pytorch_widedeep.training.Trainer.save","title":"save","text":"<pre><code>save(\n    path,\n    save_state_dict=False,\n    save_optimizer=False,\n    model_filename=\"wd_model.pt\",\n)\n</code></pre> <p>Saves the model, training and evaluation history, and the <code>feature_importance</code> attribute (if the <code>deeptabular</code> component is a Tabnet model) to disk</p> <p>The <code>Trainer</code> class is built so that it 'just' trains a model. With that in mind, all the torch related parameters (such as optimizers, learning rate schedulers, initializers, etc) have to be defined externally and then passed to the <code>Trainer</code>. As a result, the <code>Trainer</code> does not generate any attribute or additional data products that need to be saved other than the <code>model</code> object itself, which can be saved as any other torch model (e.g. <code>torch.save(model, path)</code>).</p> <p>The exception is Tabnet. If the <code>deeptabular</code> component is a Tabnet model, an attribute (a dict) called <code>feature_importance</code> will be created at the end of the training process. Therefore, a <code>save</code> method was created that will save the feature importance dictionary to a json file and, since we are here, the model weights, training history and learning rate history.</p> <p>Parameters:</p> Name Type Description Default <code>path</code> <code>str</code> <p>path to the directory where the model and the feature importance attribute will be saved.</p> required <code>save_state_dict</code> <code>bool</code> <p>Boolean indicating whether to save directly the model (and optimizer) or the model's (and optimizer's) state dictionary</p> <code>False</code> <code>save_optimizer</code> <code>bool</code> <p>Boolean indicating whether to save the optimizer</p> <code>False</code> <code>model_filename</code> <code>str</code> <p>filename where the model weights will be store</p> <code>'wd_model.pt'</code> Source code in <code>pytorch_widedeep/training/trainer.py</code> <pre><code>def save(\n    self,\n    path: str,\n    save_state_dict: bool = False,\n    save_optimizer: bool = False,\n    model_filename: str = \"wd_model.pt\",\n):\n    r\"\"\"Saves the model, training and evaluation history, and the\n    `feature_importance` attribute (if the `deeptabular` component is a\n    Tabnet model) to disk\n\n    The `Trainer` class is built so that it 'just' trains a model. With\n    that in mind, all the torch related parameters (such as optimizers,\n    learning rate schedulers, initializers, etc) have to be defined\n    externally and then passed to the `Trainer`. As a result, the\n    `Trainer` does not generate any attribute or additional data\n    products that need to be saved other than the `model` object itself,\n    which can be saved as any other torch model (e.g. `torch.save(model,\n    path)`).\n\n    The exception is Tabnet. If the `deeptabular` component is a Tabnet\n    model, an attribute (a dict) called `feature_importance` will be\n    created at the end of the training process. Therefore, a `save`\n    method was created that will save the feature importance dictionary\n    to a json file and, since we are here, the model weights, training\n    history and learning rate history.\n\n    Parameters\n    ----------\n    path: str\n        path to the directory where the model and the feature importance\n        attribute will be saved.\n    save_state_dict: bool, default = False\n        Boolean indicating whether to save directly the model\n        (and optimizer) or the model's (and optimizer's) state\n        dictionary\n    save_optimizer: bool, default = False\n        Boolean indicating whether to save the optimizer\n    model_filename: str, Optional, default = \"wd_model.pt\"\n        filename where the model weights will be store\n    \"\"\"\n\n    self._save_history(path)\n\n    self._save_model_and_optimizer(\n        path, save_state_dict, save_optimizer, model_filename\n    )\n\n    if self.model.is_tabnet:\n        with open(Path(path) / \"feature_importance.json\", \"w\") as fi:\n            json.dump(self.feature_importance, fi)\n</code></pre>"},{"location":"pytorch-widedeep/utils/index.html","title":"The <code>utils</code> module","text":"<p>These are a series of utilities that might be useful for a number of preprocessing tasks, even not directly related to <code>pytorch-widedeep</code>. All the classes and functions discussed here are available directly from the <code>utils</code> module. For example, the <code>LabelEncoder</code> within the <code>deeptabular_utils</code> submodule can be imported as:</p> <pre><code>from pytorch_widedeep.utils import LabelEncoder\n</code></pre> <p>These are classes and functions that are internally used in the library. We include them here in case the user finds them useful for other purposes.</p>"},{"location":"pytorch-widedeep/utils/deeptabular_utils.html","title":"deeptabular utils","text":""},{"location":"pytorch-widedeep/utils/deeptabular_utils.html#pytorch_widedeep.utils.deeptabular_utils.LabelEncoder","title":"LabelEncoder","text":"<p>Label Encode categorical values for multiple columns at once</p> <p> NOTE: LabelEncoder reserves 0 for <code>unseen</code> new categories. This is convenient when defining the embedding layers, since we can just set padding idx to 0.</p> <p>Parameters:</p> Name Type Description Default <code>columns_to_encode</code> <code>Optional[List[str]]</code> <p>List of strings containing the names of the columns to encode. If <code>None</code> all columns of type <code>object</code> in the dataframe will be label encoded.</p> <code>None</code> <code>with_attention</code> <code>bool</code> <p>Boolean indicating whether the preprocessed data will be passed to an attention-based model. Aliased as <code>for_transformer</code>.</p> <code>False</code> <code>shared_embed</code> <code>bool</code> <p>Boolean indicating if the embeddings will be \"shared\" when using attention-based models. The idea behind <code>shared_embed</code> is described in the Appendix A in the TabTransformer paper: 'The goal of having column embedding is to enable the model to distinguish the classes in one column from those in the other columns'. In other words, the idea is to let the model learn which column is embedded at the time. See: <code>pytorch_widedeep.models.transformers._layers.SharedEmbeddings</code>.</p> <code>False</code> <p>Attributes:</p> Name Type Description <code>encoding_dict</code> <code>Dict</code> <p>Dictionary containing the encoding mappings in the format, e.g. :  <code>{'colname1': {'cat1': 1, 'cat2': 2, ...}, 'colname2': {'cat1': 1, 'cat2': 2, ...}, ...}</code></p> <code>inverse_encoding_dict</code> <code>Dict</code> <p>Dictionary containing the inverse encoding mappings in the format, e.g. :  <code>{'colname1': {1: 'cat1', 2: 'cat2', ...}, 'colname2': {1: 'cat1', 2: 'cat2', ...}, ...}</code></p> Source code in <code>pytorch_widedeep/utils/deeptabular_utils.py</code> <pre><code>class LabelEncoder:\n    r\"\"\"Label Encode categorical values for multiple columns at once\n\n    :information_source: **NOTE**:\n    LabelEncoder reserves 0 for `unseen` new categories. This is convenient\n    when defining the embedding layers, since we can just set padding idx to 0.\n\n    Parameters\n    ----------\n    columns_to_encode: list, Optional, default = None\n        List of strings containing the names of the columns to encode. If\n        `None` all columns of type `object` in the dataframe will be label\n        encoded.\n    with_attention: bool, default = False\n        Boolean indicating whether the preprocessed data will be passed to an\n        attention-based model. Aliased as `for_transformer`.\n    shared_embed: bool, default = False\n        Boolean indicating if the embeddings will be \"_shared_\" when using\n        attention-based models. The idea behind `shared_embed` is described\n        in the Appendix A in the [TabTransformer paper](https://arxiv.org/abs/2012.06678):\n        '_The goal of having column embedding is to enable the model to\n        distinguish the classes in one column from those in the\n        other columns_'. In other words, the idea is to let the model learn\n        which column is embedded at the time. See: `pytorch_widedeep.models.transformers._layers.SharedEmbeddings`.\n\n    Attributes\n    ----------\n    encoding_dict : Dict\n        Dictionary containing the encoding mappings in the format, e.g. : &lt;br/&gt;\n        `{'colname1': {'cat1': 1, 'cat2': 2, ...}, 'colname2': {'cat1': 1, 'cat2': 2, ...}, ...}`\n    inverse_encoding_dict : Dict\n        Dictionary containing the inverse encoding mappings in the format, e.g. : &lt;br/&gt;\n        `{'colname1': {1: 'cat1', 2: 'cat2', ...}, 'colname2': {1: 'cat1', 2: 'cat2', ...}, ...}`\n\n    \"\"\"\n\n    @alias(\"with_attention\", [\"for_transformer\"])\n    def __init__(\n        self,\n        columns_to_encode: Optional[List[str]] = None,\n        with_attention: bool = False,\n        shared_embed: bool = False,\n    ):\n        self.columns_to_encode = columns_to_encode\n\n        self.shared_embed = shared_embed\n        self.with_attention = with_attention\n\n        self.reset_embed_idx = not self.with_attention or self.shared_embed\n\n    def partial_fit(self, df: pd.DataFrame) -&gt; \"LabelEncoder\":  # noqa: C901\n        \"\"\"Main method. Creates encoding attributes.\n\n        Returns\n        -------\n        LabelEncoder\n            `LabelEncoder` fitted object\n        \"\"\"\n        # here df is a chunk of the data. this is meant to be run when the\n        # data is large and we pass a chunk at a time. Therefore, we do not\n        # copy the input chunk as mutating a chunk is ok\n        if self.columns_to_encode is None:\n            self.columns_to_encode = list(df.select_dtypes(include=[\"object\"]).columns)\n        else:\n            # sanity check to make sure all categorical columns are in an adequate\n            # format\n            for col in self.columns_to_encode:\n                df[col] = df[col].astype(\"O\")\n\n        unique_column_vals: Dict[str, List[str]] = {}\n        for c in self.columns_to_encode:\n            unique_column_vals[c] = df[c].unique().tolist()\n\n        if not hasattr(self, \"encoding_dict\"):\n            # we run the method 'partial_fit' for the 1st time\n            self.encoding_dict: Dict[str, Dict[str, int]] = {}\n            if \"cls_token\" in unique_column_vals and self.shared_embed:\n                self.encoding_dict[\"cls_token\"] = {\"[CLS]\": 0}\n                del unique_column_vals[\"cls_token\"]\n\n            # leave 0 for padding/\"unseen\" categories. Also we need an\n            # attribute to keep track of the encoding in case we use\n            # attention and we do not re-start the index/counter\n            self.cum_idx: int = 1\n            for k, v in unique_column_vals.items():\n                self.encoding_dict[k] = {o: i + self.cum_idx for i, o in enumerate(v)}\n                self.cum_idx = 1 if self.reset_embed_idx else self.cum_idx + len(v)\n        else:\n            # the 'partial_fit' method has already run.\n            # \"cls_token\" will have been added already\n            if \"cls_token\" in unique_column_vals and self.shared_embed:\n                del unique_column_vals[\"cls_token\"]\n\n            # Classes in the new df/chunk of the dataset that have not been seen\n            # before\n            unseen_classes: Dict[str, List[str]] = {}\n            for c in self.columns_to_encode:\n                unseen_classes[c] = list(\n                    np.setdiff1d(\n                        unique_column_vals[c], list(self.encoding_dict[c].keys())\n                    )\n                )\n\n            # leave 0 for padding/\"unseen\" categories\n            for k, v in unique_column_vals.items():\n                # if we use attention we need to start encoding from the\n                # last 'overall' encoding index. Otherwise, we use the max\n                # encoding index per categorical col\n                _idx = (\n                    max(self.encoding_dict[k].values()) + 1\n                    if self.reset_embed_idx\n                    else self.cum_idx\n                )\n                if len(unseen_classes[k]) != 0:\n                    for i, o in enumerate(unseen_classes[k]):\n                        if o not in self.encoding_dict[k]:\n                            self.encoding_dict[k][o] = i + _idx\n                    # if self.reset_embed_idx is True it will be 1 anyway\n                    self.cum_idx = (\n                        1\n                        if self.reset_embed_idx\n                        else self.cum_idx + len(unseen_classes[k])\n                    )\n\n        return self\n\n    def fit(self, df: pd.DataFrame) -&gt; \"LabelEncoder\":\n        \"\"\"Simply runs the `partial_fit` method when the data fits in memory\n\n        Returns\n        -------\n        LabelEncoder\n            `LabelEncoder` fitted object\n        \"\"\"\n        # this is meant to be run when the data fits in memory and therefore,\n        # we do not want to mutate the original df, so we copy it\n        self.partial_fit(df.copy())\n\n        self.inverse_encoding_dict = self.create_inverse_encoding_dict()\n\n        return self\n\n    def transform(self, df: pd.DataFrame) -&gt; pd.DataFrame:\n        \"\"\"Label Encoded the categories in `columns_to_encode`\n\n        Returns\n        -------\n        pd.DataFrame\n            label-encoded dataframe\n        \"\"\"\n        try:\n            self.encoding_dict\n        except AttributeError:\n            raise NotFittedError(\n                \"This LabelEncoder instance is not fitted yet. \"\n                \"Call 'fit' with appropriate arguments before using this LabelEncoder.\"\n            )\n\n        df_inp = df.copy()\n        # sanity check to make sure all categorical columns are in an adequate\n        # format\n        for col in self.columns_to_encode:  # type: ignore\n            df_inp[col] = df_inp[col].astype(\"O\")\n\n        for k, v in self.encoding_dict.items():\n            df_inp[k] = df_inp[k].apply(lambda x: v[x] if x in v.keys() else 0)\n\n        return df_inp\n\n    def fit_transform(self, df: pd.DataFrame) -&gt; pd.DataFrame:\n        \"\"\"Combines `fit` and `transform`\n\n        Examples\n        --------\n\n        &gt;&gt;&gt; import pandas as pd\n        &gt;&gt;&gt; from pytorch_widedeep.utils import LabelEncoder\n        &gt;&gt;&gt; df = pd.DataFrame({'col1': [1,2,3], 'col2': ['me', 'you', 'him']})\n        &gt;&gt;&gt; columns_to_encode = ['col2']\n        &gt;&gt;&gt; encoder = LabelEncoder(columns_to_encode)\n        &gt;&gt;&gt; encoder.fit_transform(df)\n           col1  col2\n        0     1     1\n        1     2     2\n        2     3     3\n        &gt;&gt;&gt; encoder.encoding_dict\n        {'col2': {'me': 1, 'you': 2, 'him': 3}}\n\n        Returns\n        -------\n        pd.DataFrame\n            label-encoded dataframe\n        \"\"\"\n        return self.fit(df).transform(df)\n\n    def create_inverse_encoding_dict(self) -&gt; Dict[str, Dict[int, str]]:\n        inverse_encoding_dict = dict()\n        for c in self.encoding_dict:\n            inverse_encoding_dict[c] = {v: k for k, v in self.encoding_dict[c].items()}\n            inverse_encoding_dict[c][0] = \"unseen\"\n        return inverse_encoding_dict\n\n    def inverse_transform(self, df: pd.DataFrame) -&gt; pd.DataFrame:\n        \"\"\"Returns the original categories\n\n        Examples\n        --------\n\n        &gt;&gt;&gt; import pandas as pd\n        &gt;&gt;&gt; from pytorch_widedeep.utils import LabelEncoder\n        &gt;&gt;&gt; df = pd.DataFrame({'col1': [1,2,3], 'col2': ['me', 'you', 'him']})\n        &gt;&gt;&gt; columns_to_encode = ['col2']\n        &gt;&gt;&gt; encoder = LabelEncoder(columns_to_encode)\n        &gt;&gt;&gt; df_enc = encoder.fit_transform(df)\n        &gt;&gt;&gt; encoder.inverse_transform(df_enc)\n           col1 col2\n        0     1   me\n        1     2  you\n        2     3  him\n\n        Returns\n        -------\n        pd.DataFrame\n            DataFrame with original categories\n        \"\"\"\n\n        if not hasattr(self, \"inverse_encoding_dict\"):\n            self.inverse_encoding_dict = self.create_inverse_encoding_dict()\n\n        for k, v in self.inverse_encoding_dict.items():\n            df[k] = df[k].apply(lambda x: v[x])\n\n        return df\n\n    def __repr__(self) -&gt; str:\n        list_of_params: List[str] = []\n        if self.columns_to_encode is not None:\n            list_of_params.append(\"columns_to_encode={columns_to_encode}\")\n        if self.with_attention:\n            list_of_params.append(\"with_attention={with_attention}\")\n        if self.shared_embed:\n            list_of_params.append(\"shared_embed={shared_embed}\")\n        all_params = \", \".join(list_of_params)\n        return f\"LabelEncoder({all_params.format(**self.__dict__)})\"\n</code></pre>"},{"location":"pytorch-widedeep/utils/deeptabular_utils.html#pytorch_widedeep.utils.deeptabular_utils.LabelEncoder.partial_fit","title":"partial_fit","text":"<pre><code>partial_fit(df)\n</code></pre> <p>Main method. Creates encoding attributes.</p> <p>Returns:</p> Type Description <code>LabelEncoder</code> <p><code>LabelEncoder</code> fitted object</p> Source code in <code>pytorch_widedeep/utils/deeptabular_utils.py</code> <pre><code>def partial_fit(self, df: pd.DataFrame) -&gt; \"LabelEncoder\":  # noqa: C901\n    \"\"\"Main method. Creates encoding attributes.\n\n    Returns\n    -------\n    LabelEncoder\n        `LabelEncoder` fitted object\n    \"\"\"\n    # here df is a chunk of the data. this is meant to be run when the\n    # data is large and we pass a chunk at a time. Therefore, we do not\n    # copy the input chunk as mutating a chunk is ok\n    if self.columns_to_encode is None:\n        self.columns_to_encode = list(df.select_dtypes(include=[\"object\"]).columns)\n    else:\n        # sanity check to make sure all categorical columns are in an adequate\n        # format\n        for col in self.columns_to_encode:\n            df[col] = df[col].astype(\"O\")\n\n    unique_column_vals: Dict[str, List[str]] = {}\n    for c in self.columns_to_encode:\n        unique_column_vals[c] = df[c].unique().tolist()\n\n    if not hasattr(self, \"encoding_dict\"):\n        # we run the method 'partial_fit' for the 1st time\n        self.encoding_dict: Dict[str, Dict[str, int]] = {}\n        if \"cls_token\" in unique_column_vals and self.shared_embed:\n            self.encoding_dict[\"cls_token\"] = {\"[CLS]\": 0}\n            del unique_column_vals[\"cls_token\"]\n\n        # leave 0 for padding/\"unseen\" categories. Also we need an\n        # attribute to keep track of the encoding in case we use\n        # attention and we do not re-start the index/counter\n        self.cum_idx: int = 1\n        for k, v in unique_column_vals.items():\n            self.encoding_dict[k] = {o: i + self.cum_idx for i, o in enumerate(v)}\n            self.cum_idx = 1 if self.reset_embed_idx else self.cum_idx + len(v)\n    else:\n        # the 'partial_fit' method has already run.\n        # \"cls_token\" will have been added already\n        if \"cls_token\" in unique_column_vals and self.shared_embed:\n            del unique_column_vals[\"cls_token\"]\n\n        # Classes in the new df/chunk of the dataset that have not been seen\n        # before\n        unseen_classes: Dict[str, List[str]] = {}\n        for c in self.columns_to_encode:\n            unseen_classes[c] = list(\n                np.setdiff1d(\n                    unique_column_vals[c], list(self.encoding_dict[c].keys())\n                )\n            )\n\n        # leave 0 for padding/\"unseen\" categories\n        for k, v in unique_column_vals.items():\n            # if we use attention we need to start encoding from the\n            # last 'overall' encoding index. Otherwise, we use the max\n            # encoding index per categorical col\n            _idx = (\n                max(self.encoding_dict[k].values()) + 1\n                if self.reset_embed_idx\n                else self.cum_idx\n            )\n            if len(unseen_classes[k]) != 0:\n                for i, o in enumerate(unseen_classes[k]):\n                    if o not in self.encoding_dict[k]:\n                        self.encoding_dict[k][o] = i + _idx\n                # if self.reset_embed_idx is True it will be 1 anyway\n                self.cum_idx = (\n                    1\n                    if self.reset_embed_idx\n                    else self.cum_idx + len(unseen_classes[k])\n                )\n\n    return self\n</code></pre>"},{"location":"pytorch-widedeep/utils/deeptabular_utils.html#pytorch_widedeep.utils.deeptabular_utils.LabelEncoder.fit","title":"fit","text":"<pre><code>fit(df)\n</code></pre> <p>Simply runs the <code>partial_fit</code> method when the data fits in memory</p> <p>Returns:</p> Type Description <code>LabelEncoder</code> <p><code>LabelEncoder</code> fitted object</p> Source code in <code>pytorch_widedeep/utils/deeptabular_utils.py</code> <pre><code>def fit(self, df: pd.DataFrame) -&gt; \"LabelEncoder\":\n    \"\"\"Simply runs the `partial_fit` method when the data fits in memory\n\n    Returns\n    -------\n    LabelEncoder\n        `LabelEncoder` fitted object\n    \"\"\"\n    # this is meant to be run when the data fits in memory and therefore,\n    # we do not want to mutate the original df, so we copy it\n    self.partial_fit(df.copy())\n\n    self.inverse_encoding_dict = self.create_inverse_encoding_dict()\n\n    return self\n</code></pre>"},{"location":"pytorch-widedeep/utils/deeptabular_utils.html#pytorch_widedeep.utils.deeptabular_utils.LabelEncoder.transform","title":"transform","text":"<pre><code>transform(df)\n</code></pre> <p>Label Encoded the categories in <code>columns_to_encode</code></p> <p>Returns:</p> Type Description <code>DataFrame</code> <p>label-encoded dataframe</p> Source code in <code>pytorch_widedeep/utils/deeptabular_utils.py</code> <pre><code>def transform(self, df: pd.DataFrame) -&gt; pd.DataFrame:\n    \"\"\"Label Encoded the categories in `columns_to_encode`\n\n    Returns\n    -------\n    pd.DataFrame\n        label-encoded dataframe\n    \"\"\"\n    try:\n        self.encoding_dict\n    except AttributeError:\n        raise NotFittedError(\n            \"This LabelEncoder instance is not fitted yet. \"\n            \"Call 'fit' with appropriate arguments before using this LabelEncoder.\"\n        )\n\n    df_inp = df.copy()\n    # sanity check to make sure all categorical columns are in an adequate\n    # format\n    for col in self.columns_to_encode:  # type: ignore\n        df_inp[col] = df_inp[col].astype(\"O\")\n\n    for k, v in self.encoding_dict.items():\n        df_inp[k] = df_inp[k].apply(lambda x: v[x] if x in v.keys() else 0)\n\n    return df_inp\n</code></pre>"},{"location":"pytorch-widedeep/utils/deeptabular_utils.html#pytorch_widedeep.utils.deeptabular_utils.LabelEncoder.fit_transform","title":"fit_transform","text":"<pre><code>fit_transform(df)\n</code></pre> <p>Combines <code>fit</code> and <code>transform</code></p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import pandas as pd\n&gt;&gt;&gt; from pytorch_widedeep.utils import LabelEncoder\n&gt;&gt;&gt; df = pd.DataFrame({'col1': [1,2,3], 'col2': ['me', 'you', 'him']})\n&gt;&gt;&gt; columns_to_encode = ['col2']\n&gt;&gt;&gt; encoder = LabelEncoder(columns_to_encode)\n&gt;&gt;&gt; encoder.fit_transform(df)\n   col1  col2\n0     1     1\n1     2     2\n2     3     3\n&gt;&gt;&gt; encoder.encoding_dict\n{'col2': {'me': 1, 'you': 2, 'him': 3}}\n</code></pre> <p>Returns:</p> Type Description <code>DataFrame</code> <p>label-encoded dataframe</p> Source code in <code>pytorch_widedeep/utils/deeptabular_utils.py</code> <pre><code>def fit_transform(self, df: pd.DataFrame) -&gt; pd.DataFrame:\n    \"\"\"Combines `fit` and `transform`\n\n    Examples\n    --------\n\n    &gt;&gt;&gt; import pandas as pd\n    &gt;&gt;&gt; from pytorch_widedeep.utils import LabelEncoder\n    &gt;&gt;&gt; df = pd.DataFrame({'col1': [1,2,3], 'col2': ['me', 'you', 'him']})\n    &gt;&gt;&gt; columns_to_encode = ['col2']\n    &gt;&gt;&gt; encoder = LabelEncoder(columns_to_encode)\n    &gt;&gt;&gt; encoder.fit_transform(df)\n       col1  col2\n    0     1     1\n    1     2     2\n    2     3     3\n    &gt;&gt;&gt; encoder.encoding_dict\n    {'col2': {'me': 1, 'you': 2, 'him': 3}}\n\n    Returns\n    -------\n    pd.DataFrame\n        label-encoded dataframe\n    \"\"\"\n    return self.fit(df).transform(df)\n</code></pre>"},{"location":"pytorch-widedeep/utils/deeptabular_utils.html#pytorch_widedeep.utils.deeptabular_utils.LabelEncoder.inverse_transform","title":"inverse_transform","text":"<pre><code>inverse_transform(df)\n</code></pre> <p>Returns the original categories</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import pandas as pd\n&gt;&gt;&gt; from pytorch_widedeep.utils import LabelEncoder\n&gt;&gt;&gt; df = pd.DataFrame({'col1': [1,2,3], 'col2': ['me', 'you', 'him']})\n&gt;&gt;&gt; columns_to_encode = ['col2']\n&gt;&gt;&gt; encoder = LabelEncoder(columns_to_encode)\n&gt;&gt;&gt; df_enc = encoder.fit_transform(df)\n&gt;&gt;&gt; encoder.inverse_transform(df_enc)\n   col1 col2\n0     1   me\n1     2  you\n2     3  him\n</code></pre> <p>Returns:</p> Type Description <code>DataFrame</code> <p>DataFrame with original categories</p> Source code in <code>pytorch_widedeep/utils/deeptabular_utils.py</code> <pre><code>def inverse_transform(self, df: pd.DataFrame) -&gt; pd.DataFrame:\n    \"\"\"Returns the original categories\n\n    Examples\n    --------\n\n    &gt;&gt;&gt; import pandas as pd\n    &gt;&gt;&gt; from pytorch_widedeep.utils import LabelEncoder\n    &gt;&gt;&gt; df = pd.DataFrame({'col1': [1,2,3], 'col2': ['me', 'you', 'him']})\n    &gt;&gt;&gt; columns_to_encode = ['col2']\n    &gt;&gt;&gt; encoder = LabelEncoder(columns_to_encode)\n    &gt;&gt;&gt; df_enc = encoder.fit_transform(df)\n    &gt;&gt;&gt; encoder.inverse_transform(df_enc)\n       col1 col2\n    0     1   me\n    1     2  you\n    2     3  him\n\n    Returns\n    -------\n    pd.DataFrame\n        DataFrame with original categories\n    \"\"\"\n\n    if not hasattr(self, \"inverse_encoding_dict\"):\n        self.inverse_encoding_dict = self.create_inverse_encoding_dict()\n\n    for k, v in self.inverse_encoding_dict.items():\n        df[k] = df[k].apply(lambda x: v[x])\n\n    return df\n</code></pre>"},{"location":"pytorch-widedeep/utils/fastai_transforms.html","title":"Fastai transforms","text":"<p>We directly copied and pasted part of the <code>transforms.py</code> module from the <code>fastai</code> library (from an old version). The reason to do such a thing is because <code>pytorch_widedeep</code> only needs the <code>Tokenizer</code> and the <code>Vocab</code> classes there. This way we avoid extra dependencies. Credit for all the code in the <code>fastai_transforms</code> module in this <code>pytorch-widedeep</code> package goes to Jeremy Howard and the <code>fastai</code> team. I only include the documentation here for completion, but I strongly advise the user to read the <code>fastai</code> documentation.</p>"},{"location":"pytorch-widedeep/utils/fastai_transforms.html#pytorch_widedeep.utils.fastai_transforms.Tokenizer","title":"Tokenizer","text":"<p>Class to combine a series of rules and a tokenizer function to tokenize text with multiprocessing.</p> <p>Setting some of the parameters of this class require perhaps some familiarity with the source code.</p> <p>Parameters:</p> Name Type Description Default <code>tok_func</code> <code>Callable</code> <p>Tokenizer Object. See <code>pytorch_widedeep.utils.fastai_transforms.SpacyTokenizer</code></p> <code>SpacyTokenizer</code> <code>lang</code> <code>str</code> <p>Text's Language</p> <code>'en'</code> <code>pre_rules</code> <code>Optional[ListRules]</code> <p>Custom type: <code>Collection[Callable[[str], str]]</code>. These are <code>Callable</code> objects that will be applied to the text (str) directly as <code>rule(tok)</code> before being tokenized.</p> <code>None</code> <code>post_rules</code> <code>Optional[ListRules]</code> <p>Custom type: <code>Collection[Callable[[str], str]]</code>. These are <code>Callable</code> objects that will be applied to the tokens as <code>rule(tokens)</code> after the text has been tokenized.</p> <code>None</code> <code>special_cases</code> <code>Optional[Collection[str]]</code> <p>special cases to be added to the tokenizer via <code>Spacy</code>'s <code>add_special_case</code> method</p> <code>None</code> <code>n_cpus</code> <code>Optional[int]</code> <p>number of CPUs to used during the tokenization process</p> <code>None</code> Source code in <code>pytorch_widedeep/utils/fastai_transforms.py</code> <pre><code>class Tokenizer:\n    r\"\"\"Class to combine a series of rules and a tokenizer function to tokenize\n    text with multiprocessing.\n\n    Setting some of the parameters of this class require perhaps some\n    familiarity with the source code.\n\n    Parameters\n    ----------\n    tok_func: Callable, default = ``SpacyTokenizer``\n        Tokenizer Object. See `pytorch_widedeep.utils.fastai_transforms.SpacyTokenizer`\n    lang: str, default = \"en\"\n        Text's Language\n    pre_rules: ListRules, Optional, default = None\n        Custom type: ``Collection[Callable[[str], str]]``. These are\n        `Callable` objects that will be applied to the text (str) directly as\n        `rule(tok)` before being tokenized.\n    post_rules: ListRules, Optional, default = None\n        Custom type: ``Collection[Callable[[str], str]]``. These are\n        `Callable` objects that will be applied to the tokens as\n        `rule(tokens)` after the text has been tokenized.\n    special_cases: Collection, Optional, default= None\n        special cases to be added to the tokenizer via ``Spacy``'s\n        ``add_special_case`` method\n    n_cpus: int, Optional, default = None\n        number of CPUs to used during the tokenization process\n    \"\"\"\n\n    def __init__(\n        self,\n        tok_func: Callable = SpacyTokenizer,\n        lang: str = \"en\",\n        pre_rules: Optional[ListRules] = None,\n        post_rules: Optional[ListRules] = None,\n        special_cases: Optional[Collection[str]] = None,\n        n_cpus: Optional[int] = None,\n    ):\n        self.tok_func, self.lang, self.special_cases = tok_func, lang, special_cases\n        self.pre_rules = ifnone(pre_rules, defaults.text_pre_rules)\n        self.post_rules = ifnone(post_rules, defaults.text_post_rules)\n        self.special_cases = (\n            special_cases if special_cases is not None else defaults.text_spec_tok\n        )\n        self.n_cpus = ifnone(n_cpus, defaults.cpus)\n\n    def __repr__(self) -&gt; str:\n        res = f\"Tokenizer {self.tok_func.__name__} in {self.lang} with the following rules:\\n\"\n        for rule in self.pre_rules:\n            res += f\" - {rule.__name__}\\n\"\n        for rule in self.post_rules:\n            res += f\" - {rule.__name__}\\n\"\n        return res\n\n    def process_text(self, t: str, tok: BaseTokenizer) -&gt; List[str]:\n        r\"\"\"Process and tokenize one text ``t`` with tokenizer ``tok``.\n\n        Parameters\n        ----------\n        t: str\n            text to be processed and tokenized\n        tok: ``BaseTokenizer``\n            Instance of `BaseTokenizer`. See\n            `pytorch_widedeep.utils.fastai_transforms.BaseTokenizer`\n\n        Returns\n        -------\n        List[str]\n            List of tokens\n        \"\"\"\n        for rule in self.pre_rules:\n            t = rule(t)\n        toks = tok.tokenizer(t)\n        for rule in self.post_rules:\n            toks = rule(toks)\n        return toks\n\n    def _process_all_1(self, texts: Collection[str]) -&gt; List[List[str]]:\n        \"\"\"Process a list of ``texts`` in one process.\"\"\"\n\n        tok = self.tok_func(self.lang)\n        if self.special_cases:\n            tok.add_special_cases(self.special_cases)\n        return [self.process_text(str(t), tok) for t in texts]\n\n    def process_all(self, texts: Collection[str]) -&gt; List[List[str]]:\n        r\"\"\"Process a list of texts. Parallel execution of ``process_text``.\n\n        Examples\n        --------\n        &gt;&gt;&gt; from pytorch_widedeep.utils import Tokenizer\n        &gt;&gt;&gt; texts = ['Machine learning is great', 'but building stuff is even better']\n        &gt;&gt;&gt; tok = Tokenizer()\n        &gt;&gt;&gt; tok.process_all(texts)\n        [['xxmaj', 'machine', 'learning', 'is', 'great'], ['but', 'building', 'stuff', 'is', 'even', 'better']]\n\n        :information_source: **NOTE**:\n        Note the token ``TK_MAJ`` (`xxmaj`), used to indicate the\n        next word begins with a capital in the original text. For more\n        details of special tokens please see the [``fastai`` docs](https://docs.fast.ai/text.core.html#Tokenizing).\n\n        Returns\n        -------\n        List[List[str]]\n            List containing lists of tokens. One list per \"_document_\"\n\n        \"\"\"\n\n        if self.n_cpus &lt;= 1:\n            return self._process_all_1(texts)\n        with ProcessPoolExecutor(self.n_cpus) as e:\n            return sum(\n                e.map(self._process_all_1, partition_by_cores(texts, self.n_cpus)), []\n            )\n</code></pre>"},{"location":"pytorch-widedeep/utils/fastai_transforms.html#pytorch_widedeep.utils.fastai_transforms.Tokenizer.process_text","title":"process_text","text":"<pre><code>process_text(t, tok)\n</code></pre> <p>Process and tokenize one text <code>t</code> with tokenizer <code>tok</code>.</p> <p>Parameters:</p> Name Type Description Default <code>t</code> <code>str</code> <p>text to be processed and tokenized</p> required <code>tok</code> <code>BaseTokenizer</code> <p>Instance of <code>BaseTokenizer</code>. See <code>pytorch_widedeep.utils.fastai_transforms.BaseTokenizer</code></p> required <p>Returns:</p> Type Description <code>List[str]</code> <p>List of tokens</p> Source code in <code>pytorch_widedeep/utils/fastai_transforms.py</code> <pre><code>def process_text(self, t: str, tok: BaseTokenizer) -&gt; List[str]:\n    r\"\"\"Process and tokenize one text ``t`` with tokenizer ``tok``.\n\n    Parameters\n    ----------\n    t: str\n        text to be processed and tokenized\n    tok: ``BaseTokenizer``\n        Instance of `BaseTokenizer`. See\n        `pytorch_widedeep.utils.fastai_transforms.BaseTokenizer`\n\n    Returns\n    -------\n    List[str]\n        List of tokens\n    \"\"\"\n    for rule in self.pre_rules:\n        t = rule(t)\n    toks = tok.tokenizer(t)\n    for rule in self.post_rules:\n        toks = rule(toks)\n    return toks\n</code></pre>"},{"location":"pytorch-widedeep/utils/fastai_transforms.html#pytorch_widedeep.utils.fastai_transforms.Tokenizer.process_all","title":"process_all","text":"<pre><code>process_all(texts)\n</code></pre> <p>Process a list of texts. Parallel execution of <code>process_text</code>.</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; from pytorch_widedeep.utils import Tokenizer\n&gt;&gt;&gt; texts = ['Machine learning is great', 'but building stuff is even better']\n&gt;&gt;&gt; tok = Tokenizer()\n&gt;&gt;&gt; tok.process_all(texts)\n[['xxmaj', 'machine', 'learning', 'is', 'great'], ['but', 'building', 'stuff', 'is', 'even', 'better']]\n</code></pre> <p> NOTE: Note the token <code>TK_MAJ</code> (<code>xxmaj</code>), used to indicate the next word begins with a capital in the original text. For more details of special tokens please see the <code>fastai</code> docs.</p> <p>Returns:</p> Type Description <code>List[List[str]]</code> <p>List containing lists of tokens. One list per \"document\"</p> Source code in <code>pytorch_widedeep/utils/fastai_transforms.py</code> <pre><code>def process_all(self, texts: Collection[str]) -&gt; List[List[str]]:\n    r\"\"\"Process a list of texts. Parallel execution of ``process_text``.\n\n    Examples\n    --------\n    &gt;&gt;&gt; from pytorch_widedeep.utils import Tokenizer\n    &gt;&gt;&gt; texts = ['Machine learning is great', 'but building stuff is even better']\n    &gt;&gt;&gt; tok = Tokenizer()\n    &gt;&gt;&gt; tok.process_all(texts)\n    [['xxmaj', 'machine', 'learning', 'is', 'great'], ['but', 'building', 'stuff', 'is', 'even', 'better']]\n\n    :information_source: **NOTE**:\n    Note the token ``TK_MAJ`` (`xxmaj`), used to indicate the\n    next word begins with a capital in the original text. For more\n    details of special tokens please see the [``fastai`` docs](https://docs.fast.ai/text.core.html#Tokenizing).\n\n    Returns\n    -------\n    List[List[str]]\n        List containing lists of tokens. One list per \"_document_\"\n\n    \"\"\"\n\n    if self.n_cpus &lt;= 1:\n        return self._process_all_1(texts)\n    with ProcessPoolExecutor(self.n_cpus) as e:\n        return sum(\n            e.map(self._process_all_1, partition_by_cores(texts, self.n_cpus)), []\n        )\n</code></pre>"},{"location":"pytorch-widedeep/utils/fastai_transforms.html#pytorch_widedeep.utils.fastai_transforms.Vocab","title":"Vocab","text":"<p>Contains the correspondence between numbers and tokens.</p> <p>Parameters:</p> Name Type Description Default <code>max_vocab</code> <code>int</code> <p>maximum vocabulary size</p> required <code>min_freq</code> <code>int</code> <p>minimum frequency for a token to be considereds</p> required <code>pad_idx</code> <code>Optional[int]</code> <p>padding index. If <code>None</code>, Fastai's Tokenizer leaves the 0 index for the unknown token ('xxunk') and defaults to 1 for the padding token ('xxpad').</p> <code>None</code> <p>Attributes:</p> Name Type Description <code>itos</code> <code>Collection</code> <p><code>index to str</code>. Collection of strings that are the tokens of the vocabulary</p> <code>stoi</code> <code>defaultdict</code> <p><code>str to index</code>. Dictionary containing the tokens of the vocabulary and their corresponding index</p> Source code in <code>pytorch_widedeep/utils/fastai_transforms.py</code> <pre><code>class Vocab:\n    r\"\"\"Contains the correspondence between numbers and tokens.\n\n    Parameters\n    ----------\n    max_vocab: int\n        maximum vocabulary size\n    min_freq: int\n        minimum frequency for a token to be considereds\n    pad_idx: int, Optional, default = None\n        padding index. If `None`, Fastai's Tokenizer leaves the 0 index\n        for the unknown token (_'xxunk'_) and defaults to 1 for the padding\n        token (_'xxpad'_).\n\n    Attributes\n    ----------\n    itos: Collection\n        `index to str`. Collection of strings that are the tokens of the\n        vocabulary\n    stoi: defaultdict\n        `str to index`. Dictionary containing the tokens of the vocabulary and\n        their corresponding index\n    \"\"\"\n\n    def __init__(\n        self,\n        max_vocab: int,\n        min_freq: int,\n        pad_idx: Optional[int] = None,\n        special_cases: Optional[Collection[str]] = None,\n    ):\n        self.max_vocab = max_vocab\n        self.min_freq = min_freq\n        self.pad_idx = pad_idx\n        self.special_cases = (\n            special_cases if special_cases is not None else defaults.text_spec_tok\n        )\n\n    def create(\n        self,\n        tokens: Tokens,\n    ) -&gt; \"Vocab\":\n        r\"\"\"Create a vocabulary object from a set of tokens.\n\n        Parameters\n        ----------\n        tokens: Tokens\n            Custom type: ``Collection[Collection[str]]``  see\n            `pytorch_widedeep.wdtypes`. Collection of collection of\n            strings (e.g. list of tokenized sentences)\n\n        Examples\n        --------\n        &gt;&gt;&gt; from pytorch_widedeep.utils import Tokenizer, Vocab\n        &gt;&gt;&gt; texts = ['Machine learning is great', 'but building stuff is even better']\n        &gt;&gt;&gt; tokens = Tokenizer().process_all(texts)\n        &gt;&gt;&gt; vocab = Vocab(max_vocab=18, min_freq=1).create(tokens)\n        &gt;&gt;&gt; vocab.numericalize(['machine', 'learning', 'is', 'great'])\n        [10, 11, 9, 12]\n        &gt;&gt;&gt; vocab.textify([10, 11, 9, 12])\n        'machine learning is great'\n\n        :information_source: **NOTE**:\n        Note the many special tokens that ``fastai``'s' tokenizer adds. These\n        are particularly useful when building Language models and/or in\n        classification/Regression tasks. Please see the [``fastai`` docs](https://docs.fast.ai/text.core.html#Tokenizing).\n\n        Returns\n        -------\n        Vocab\n            An instance of a `Vocab` object\n        \"\"\"\n\n        freq = Counter(p for o in tokens for p in o)\n        itos = [o for o, c in freq.most_common(self.max_vocab) if c &gt;= self.min_freq]\n        for o in reversed(self.special_cases):\n            if o in itos:\n                itos.remove(o)\n            itos.insert(0, o)\n\n        if self.pad_idx is not None and self.pad_idx != 1:\n            itos.remove(PAD)\n            itos.insert(self.pad_idx, PAD)\n            # get the new 'xxunk' index\n            xxunk_idx = np.where([el == \"xxunk\" for el in itos])[0][0]\n        else:\n            xxunk_idx = 0\n\n        itos = itos[: self.max_vocab]\n        if (\n            len(itos) &lt; self.max_vocab\n        ):  # Make sure vocab size is a multiple of 8 for fast mixed precision training\n            while len(itos) % 8 != 0:\n                itos.append(\"xxfake\")\n\n        self.itos = itos\n        self.stoi = defaultdict(\n            lambda: xxunk_idx, {v: k for k, v in enumerate(self.itos)}\n        )\n\n        return self\n\n    def fit(\n        self,\n        tokens: Tokens,\n    ) -&gt; \"Vocab\":\n        \"\"\"\n        Calls the `create` method. I simply want to honor fast ai naming, but\n        for consistency with the rest of the library I am including a fit method\n        \"\"\"\n        return self.create(tokens)\n\n    def numericalize(self, t: Collection[str]) -&gt; List[int]:\n        \"\"\"Convert a list of tokens ``t`` to their ids.\n\n        Returns\n        -------\n        List[int]\n            List of '_numericalsed_' tokens\n        \"\"\"\n        return [self.stoi[w] for w in t]\n\n    def transform(self, t: Collection[str]) -&gt; List[int]:\n        \"\"\"\n        Calls the `numericalize` method. I simply want to honor fast ai naming,\n        but for consistency with the rest of the library I am including a\n        transform method\n        \"\"\"\n        return self.numericalize(t)\n\n    def textify(self, nums: Collection[int], sep=\" \") -&gt; Union[str, List[str]]:\n        \"\"\"Convert a list of ``nums`` (or indexes) to their tokens.\n\n        Returns\n        -------\n        List[str]\n            List of tokens\n        \"\"\"\n        return (\n            sep.join([self.itos[i] for i in nums])\n            if sep is not None\n            else [self.itos[i] for i in nums]\n        )\n\n    def inverse_transform(\n        self, nums: Collection[int], sep=\" \"\n    ) -&gt; Union[str, List[str]]:\n        \"\"\"\n        Calls the `textify` method. I simply want to honor fast ai naming, but\n        for consistency with the rest of the library I am including an\n        inverse_transform method\n        \"\"\"\n        # I simply want to honor fast ai naming, but for consistency with the\n        # rest of the library I am including an inverse_transform method\n        return self.textify(nums, sep)\n\n    def __getstate__(self):\n        return {\"itos\": self.itos}\n\n    def __setstate__(self, state: dict):\n        self.itos = state[\"itos\"]\n        self.stoi = defaultdict(int, {v: k for k, v in enumerate(self.itos)})\n</code></pre>"},{"location":"pytorch-widedeep/utils/fastai_transforms.html#pytorch_widedeep.utils.fastai_transforms.Vocab.create","title":"create","text":"<pre><code>create(tokens)\n</code></pre> <p>Create a vocabulary object from a set of tokens.</p> <p>Parameters:</p> Name Type Description Default <code>tokens</code> <code>Tokens</code> <p>Custom type: <code>Collection[Collection[str]]</code>  see <code>pytorch_widedeep.wdtypes</code>. Collection of collection of strings (e.g. list of tokenized sentences)</p> required <p>Examples:</p> <pre><code>&gt;&gt;&gt; from pytorch_widedeep.utils import Tokenizer, Vocab\n&gt;&gt;&gt; texts = ['Machine learning is great', 'but building stuff is even better']\n&gt;&gt;&gt; tokens = Tokenizer().process_all(texts)\n&gt;&gt;&gt; vocab = Vocab(max_vocab=18, min_freq=1).create(tokens)\n&gt;&gt;&gt; vocab.numericalize(['machine', 'learning', 'is', 'great'])\n[10, 11, 9, 12]\n&gt;&gt;&gt; vocab.textify([10, 11, 9, 12])\n'machine learning is great'\n</code></pre> <p> NOTE: Note the many special tokens that <code>fastai</code>'s' tokenizer adds. These are particularly useful when building Language models and/or in classification/Regression tasks. Please see the <code>fastai</code> docs.</p> <p>Returns:</p> Type Description <code>Vocab</code> <p>An instance of a <code>Vocab</code> object</p> Source code in <code>pytorch_widedeep/utils/fastai_transforms.py</code> <pre><code>def create(\n    self,\n    tokens: Tokens,\n) -&gt; \"Vocab\":\n    r\"\"\"Create a vocabulary object from a set of tokens.\n\n    Parameters\n    ----------\n    tokens: Tokens\n        Custom type: ``Collection[Collection[str]]``  see\n        `pytorch_widedeep.wdtypes`. Collection of collection of\n        strings (e.g. list of tokenized sentences)\n\n    Examples\n    --------\n    &gt;&gt;&gt; from pytorch_widedeep.utils import Tokenizer, Vocab\n    &gt;&gt;&gt; texts = ['Machine learning is great', 'but building stuff is even better']\n    &gt;&gt;&gt; tokens = Tokenizer().process_all(texts)\n    &gt;&gt;&gt; vocab = Vocab(max_vocab=18, min_freq=1).create(tokens)\n    &gt;&gt;&gt; vocab.numericalize(['machine', 'learning', 'is', 'great'])\n    [10, 11, 9, 12]\n    &gt;&gt;&gt; vocab.textify([10, 11, 9, 12])\n    'machine learning is great'\n\n    :information_source: **NOTE**:\n    Note the many special tokens that ``fastai``'s' tokenizer adds. These\n    are particularly useful when building Language models and/or in\n    classification/Regression tasks. Please see the [``fastai`` docs](https://docs.fast.ai/text.core.html#Tokenizing).\n\n    Returns\n    -------\n    Vocab\n        An instance of a `Vocab` object\n    \"\"\"\n\n    freq = Counter(p for o in tokens for p in o)\n    itos = [o for o, c in freq.most_common(self.max_vocab) if c &gt;= self.min_freq]\n    for o in reversed(self.special_cases):\n        if o in itos:\n            itos.remove(o)\n        itos.insert(0, o)\n\n    if self.pad_idx is not None and self.pad_idx != 1:\n        itos.remove(PAD)\n        itos.insert(self.pad_idx, PAD)\n        # get the new 'xxunk' index\n        xxunk_idx = np.where([el == \"xxunk\" for el in itos])[0][0]\n    else:\n        xxunk_idx = 0\n\n    itos = itos[: self.max_vocab]\n    if (\n        len(itos) &lt; self.max_vocab\n    ):  # Make sure vocab size is a multiple of 8 for fast mixed precision training\n        while len(itos) % 8 != 0:\n            itos.append(\"xxfake\")\n\n    self.itos = itos\n    self.stoi = defaultdict(\n        lambda: xxunk_idx, {v: k for k, v in enumerate(self.itos)}\n    )\n\n    return self\n</code></pre>"},{"location":"pytorch-widedeep/utils/fastai_transforms.html#pytorch_widedeep.utils.fastai_transforms.Vocab.fit","title":"fit","text":"<pre><code>fit(tokens)\n</code></pre> <p>Calls the <code>create</code> method. I simply want to honor fast ai naming, but for consistency with the rest of the library I am including a fit method</p> Source code in <code>pytorch_widedeep/utils/fastai_transforms.py</code> <pre><code>def fit(\n    self,\n    tokens: Tokens,\n) -&gt; \"Vocab\":\n    \"\"\"\n    Calls the `create` method. I simply want to honor fast ai naming, but\n    for consistency with the rest of the library I am including a fit method\n    \"\"\"\n    return self.create(tokens)\n</code></pre>"},{"location":"pytorch-widedeep/utils/fastai_transforms.html#pytorch_widedeep.utils.fastai_transforms.Vocab.numericalize","title":"numericalize","text":"<pre><code>numericalize(t)\n</code></pre> <p>Convert a list of tokens <code>t</code> to their ids.</p> <p>Returns:</p> Type Description <code>List[int]</code> <p>List of 'numericalsed' tokens</p> Source code in <code>pytorch_widedeep/utils/fastai_transforms.py</code> <pre><code>def numericalize(self, t: Collection[str]) -&gt; List[int]:\n    \"\"\"Convert a list of tokens ``t`` to their ids.\n\n    Returns\n    -------\n    List[int]\n        List of '_numericalsed_' tokens\n    \"\"\"\n    return [self.stoi[w] for w in t]\n</code></pre>"},{"location":"pytorch-widedeep/utils/fastai_transforms.html#pytorch_widedeep.utils.fastai_transforms.Vocab.transform","title":"transform","text":"<pre><code>transform(t)\n</code></pre> <p>Calls the <code>numericalize</code> method. I simply want to honor fast ai naming, but for consistency with the rest of the library I am including a transform method</p> Source code in <code>pytorch_widedeep/utils/fastai_transforms.py</code> <pre><code>def transform(self, t: Collection[str]) -&gt; List[int]:\n    \"\"\"\n    Calls the `numericalize` method. I simply want to honor fast ai naming,\n    but for consistency with the rest of the library I am including a\n    transform method\n    \"\"\"\n    return self.numericalize(t)\n</code></pre>"},{"location":"pytorch-widedeep/utils/fastai_transforms.html#pytorch_widedeep.utils.fastai_transforms.Vocab.textify","title":"textify","text":"<pre><code>textify(nums, sep=' ')\n</code></pre> <p>Convert a list of <code>nums</code> (or indexes) to their tokens.</p> <p>Returns:</p> Type Description <code>List[str]</code> <p>List of tokens</p> Source code in <code>pytorch_widedeep/utils/fastai_transforms.py</code> <pre><code>def textify(self, nums: Collection[int], sep=\" \") -&gt; Union[str, List[str]]:\n    \"\"\"Convert a list of ``nums`` (or indexes) to their tokens.\n\n    Returns\n    -------\n    List[str]\n        List of tokens\n    \"\"\"\n    return (\n        sep.join([self.itos[i] for i in nums])\n        if sep is not None\n        else [self.itos[i] for i in nums]\n    )\n</code></pre>"},{"location":"pytorch-widedeep/utils/fastai_transforms.html#pytorch_widedeep.utils.fastai_transforms.Vocab.inverse_transform","title":"inverse_transform","text":"<pre><code>inverse_transform(nums, sep=' ')\n</code></pre> <p>Calls the <code>textify</code> method. I simply want to honor fast ai naming, but for consistency with the rest of the library I am including an inverse_transform method</p> Source code in <code>pytorch_widedeep/utils/fastai_transforms.py</code> <pre><code>def inverse_transform(\n    self, nums: Collection[int], sep=\" \"\n) -&gt; Union[str, List[str]]:\n    \"\"\"\n    Calls the `textify` method. I simply want to honor fast ai naming, but\n    for consistency with the rest of the library I am including an\n    inverse_transform method\n    \"\"\"\n    # I simply want to honor fast ai naming, but for consistency with the\n    # rest of the library I am including an inverse_transform method\n    return self.textify(nums, sep)\n</code></pre>"},{"location":"pytorch-widedeep/utils/image_utils.html","title":"Image utils","text":"<p>SimplePreprocessor and AspectAwarePreprocessor are directly taked from the great series of Books `Deep Learning for Computer Vision by Adrian. Therefore, all credit for the code in the <code>image_utils</code> module goes to Adrian Rosebrock.</p>"},{"location":"pytorch-widedeep/utils/image_utils.html#pytorch_widedeep.utils.image_utils.AspectAwarePreprocessor","title":"AspectAwarePreprocessor","text":"<p>Class to resize an image to a certain width and height taking into account the image aspect ratio</p> <p>Parameters:</p> Name Type Description Default <code>width</code> <code>int</code> <p>output width</p> required <code>height</code> <code>int</code> <p>output height</p> required <code>inter</code> <p><code>opencv</code> interpolation method. See <code>opencv</code> <code>InterpolationFlags</code>.</p> <code>INTER_AREA</code> Source code in <code>pytorch_widedeep/utils/image_utils.py</code> <pre><code>class AspectAwarePreprocessor:\n    r\"\"\"Class to resize an image to a certain width and height taking into account\n    the image aspect ratio\n\n    Parameters\n    ----------\n    width: int\n        output width\n    height: int\n        output height\n    inter: interpolation method,  default = ``cv2.INTER_AREA``\n        ``opencv`` interpolation method. See ``opencv``\n        `InterpolationFlags`.\n    \"\"\"\n\n    def __init__(self, width: int, height: int, inter=cv2.INTER_AREA):\n        self.width = width\n        self.height = height\n        self.inter = inter\n\n    def preprocess(self, image: np.ndarray) -&gt; np.ndarray:\n        r\"\"\"Returns the resized input image taking into account the image aspect ratio\n\n        Parameters\n        ----------\n        image: np.ndarray\n            Input image to be resized\n\n        Examples\n        --------\n        &gt;&gt;&gt; import cv2\n        &gt;&gt;&gt; from pytorch_widedeep.utils import AspectAwarePreprocessor\n        &gt;&gt;&gt; img = cv2.imread(\"tests/test_data_utils/images/galaxy1.png\")\n        &gt;&gt;&gt; img.shape\n        (694, 890, 3)\n        &gt;&gt;&gt; app = AspectAwarePreprocessor(width=224, height=224)\n        &gt;&gt;&gt; resized_img = app.preprocess(img)\n        &gt;&gt;&gt; resized_img.shape\n        (224, 224, 3)\n\n        Returns\n        -------\n        np.ndarray\n            Resized image according to its original image aspect ratio\n        \"\"\"\n        (h, w) = image.shape[:2]\n        dW = 0\n        dH = 0\n\n        if w &lt; h:\n            image = resize(image, width=self.width, inter=self.inter)\n            dH = int((image.shape[0] - self.height) / 2.0)\n        else:\n            image = resize(image, height=self.height, inter=self.inter)\n            dW = int((image.shape[1] - self.width) / 2.0)\n\n        (h, w) = image.shape[:2]\n        image = image[dH : h - dH, dW : w - dW]\n\n        resized_image = cv2.resize(\n            image, (self.width, self.height), interpolation=self.inter\n        )\n\n        return resized_image\n</code></pre>"},{"location":"pytorch-widedeep/utils/image_utils.html#pytorch_widedeep.utils.image_utils.AspectAwarePreprocessor.preprocess","title":"preprocess","text":"<pre><code>preprocess(image)\n</code></pre> <p>Returns the resized input image taking into account the image aspect ratio</p> <p>Parameters:</p> Name Type Description Default <code>image</code> <code>ndarray</code> <p>Input image to be resized</p> required <p>Examples:</p> <pre><code>&gt;&gt;&gt; import cv2\n&gt;&gt;&gt; from pytorch_widedeep.utils import AspectAwarePreprocessor\n&gt;&gt;&gt; img = cv2.imread(\"tests/test_data_utils/images/galaxy1.png\")\n&gt;&gt;&gt; img.shape\n(694, 890, 3)\n&gt;&gt;&gt; app = AspectAwarePreprocessor(width=224, height=224)\n&gt;&gt;&gt; resized_img = app.preprocess(img)\n&gt;&gt;&gt; resized_img.shape\n(224, 224, 3)\n</code></pre> <p>Returns:</p> Type Description <code>ndarray</code> <p>Resized image according to its original image aspect ratio</p> Source code in <code>pytorch_widedeep/utils/image_utils.py</code> <pre><code>def preprocess(self, image: np.ndarray) -&gt; np.ndarray:\n    r\"\"\"Returns the resized input image taking into account the image aspect ratio\n\n    Parameters\n    ----------\n    image: np.ndarray\n        Input image to be resized\n\n    Examples\n    --------\n    &gt;&gt;&gt; import cv2\n    &gt;&gt;&gt; from pytorch_widedeep.utils import AspectAwarePreprocessor\n    &gt;&gt;&gt; img = cv2.imread(\"tests/test_data_utils/images/galaxy1.png\")\n    &gt;&gt;&gt; img.shape\n    (694, 890, 3)\n    &gt;&gt;&gt; app = AspectAwarePreprocessor(width=224, height=224)\n    &gt;&gt;&gt; resized_img = app.preprocess(img)\n    &gt;&gt;&gt; resized_img.shape\n    (224, 224, 3)\n\n    Returns\n    -------\n    np.ndarray\n        Resized image according to its original image aspect ratio\n    \"\"\"\n    (h, w) = image.shape[:2]\n    dW = 0\n    dH = 0\n\n    if w &lt; h:\n        image = resize(image, width=self.width, inter=self.inter)\n        dH = int((image.shape[0] - self.height) / 2.0)\n    else:\n        image = resize(image, height=self.height, inter=self.inter)\n        dW = int((image.shape[1] - self.width) / 2.0)\n\n    (h, w) = image.shape[:2]\n    image = image[dH : h - dH, dW : w - dW]\n\n    resized_image = cv2.resize(\n        image, (self.width, self.height), interpolation=self.inter\n    )\n\n    return resized_image\n</code></pre>"},{"location":"pytorch-widedeep/utils/image_utils.html#pytorch_widedeep.utils.image_utils.SimplePreprocessor","title":"SimplePreprocessor","text":"<p>Class to resize an image to a certain width and height</p> <p>Parameters:</p> Name Type Description Default <code>width</code> <code>int</code> <p>output width</p> required <code>height</code> <code>int</code> <p>output height</p> required <code>inter</code> <p><code>opencv</code> interpolation method. See <code>opencv</code> <code>InterpolationFlags</code>.</p> <code>INTER_AREA</code> Source code in <code>pytorch_widedeep/utils/image_utils.py</code> <pre><code>class SimplePreprocessor:\n    r\"\"\"Class to resize an image to a certain width and height\n\n    Parameters\n    ----------\n    width: int\n        output width\n    height: int\n        output height\n    inter: interpolation method, default = ``cv2.INTER_AREA``\n        ``opencv`` interpolation method. See ``opencv``\n        `InterpolationFlags`.\n    \"\"\"\n\n    def __init__(self, width: int, height: int, inter=cv2.INTER_AREA):\n        self.width = width\n        self.height = height\n        self.inter = inter\n\n    def preprocess(self, image: np.ndarray) -&gt; np.ndarray:\n        r\"\"\"Returns the resized input image\n\n        Parameters\n        ----------\n        image: np.ndarray\n            Input image to be resized\n\n        Returns\n        -------\n        np.ndarray\n            Resized image\n\n        \"\"\"\n        resized_image = cv2.resize(\n            image, (self.width, self.height), interpolation=self.inter\n        )\n\n        return resized_image\n</code></pre>"},{"location":"pytorch-widedeep/utils/image_utils.html#pytorch_widedeep.utils.image_utils.SimplePreprocessor.preprocess","title":"preprocess","text":"<pre><code>preprocess(image)\n</code></pre> <p>Returns the resized input image</p> <p>Parameters:</p> Name Type Description Default <code>image</code> <code>ndarray</code> <p>Input image to be resized</p> required <p>Returns:</p> Type Description <code>ndarray</code> <p>Resized image</p> Source code in <code>pytorch_widedeep/utils/image_utils.py</code> <pre><code>def preprocess(self, image: np.ndarray) -&gt; np.ndarray:\n    r\"\"\"Returns the resized input image\n\n    Parameters\n    ----------\n    image: np.ndarray\n        Input image to be resized\n\n    Returns\n    -------\n    np.ndarray\n        Resized image\n\n    \"\"\"\n    resized_image = cv2.resize(\n        image, (self.width, self.height), interpolation=self.inter\n    )\n\n    return resized_image\n</code></pre>"},{"location":"pytorch-widedeep/utils/text_utils.html","title":"Text utils","text":"<p>Collection of helper function that facilitate processing text.</p>"},{"location":"pytorch-widedeep/utils/text_utils.html#pytorch_widedeep.utils.text_utils.simple_preprocess","title":"simple_preprocess","text":"<pre><code>simple_preprocess(\n    doc, lower=False, deacc=False, min_len=2, max_len=15\n)\n</code></pre> <p>This is <code>Gensim</code>'s <code>simple_preprocess</code> with a <code>lower</code> param to indicate wether or not to lower case all the token in the doc</p> <p>For more information see: <code>Gensim</code> utils module</p> <p>Parameters:</p> Name Type Description Default <code>doc</code> <code>str</code> <p>Input document.</p> required <code>lower</code> <code>bool</code> <p>Lower case tokens in the input doc</p> <code>False</code> <code>deacc</code> <code>bool</code> <p>Remove accent marks from tokens using <code>Gensim</code>'s <code>deaccent</code></p> <code>False</code> <code>min_len</code> <code>int</code> <p>Minimum length of token (inclusive). Shorter tokens are discarded.</p> <code>2</code> <code>max_len</code> <code>int</code> <p>Maximum length of token in result (inclusive). Longer tokens are discarded.</p> <code>15</code> <p>Examples:</p> <pre><code>&gt;&gt;&gt; from pytorch_widedeep.utils import simple_preprocess\n&gt;&gt;&gt; simple_preprocess('Machine learning is great')\n['Machine', 'learning', 'is', 'great']\n</code></pre> <p>Returns:</p> Type Description <code>List[str]</code> <p>List with the processed tokens</p> Source code in <code>pytorch_widedeep/utils/text_utils.py</code> <pre><code>def simple_preprocess(\n    doc: str,\n    lower: bool = False,\n    deacc: bool = False,\n    min_len: int = 2,\n    max_len: int = 15,\n) -&gt; List[str]:\n    r\"\"\"\n    This is `Gensim`'s `simple_preprocess` with a `lower` param to\n    indicate wether or not to lower case all the token in the doc\n\n    For more information see: `Gensim` [utils module](https://radimrehurek.com/gensim/utils.html)\n\n    Parameters\n    ----------\n    doc: str\n        Input document.\n    lower: bool, default = False\n        Lower case tokens in the input doc\n    deacc: bool, default = False\n        Remove accent marks from tokens using `Gensim`'s `deaccent`\n    min_len: int, default = 2\n        Minimum length of token (inclusive). Shorter tokens are discarded.\n    max_len: int, default = 15\n        Maximum length of token in result (inclusive). Longer tokens are discarded.\n\n    Examples\n    --------\n    &gt;&gt;&gt; from pytorch_widedeep.utils import simple_preprocess\n    &gt;&gt;&gt; simple_preprocess('Machine learning is great')\n    ['Machine', 'learning', 'is', 'great']\n\n    Returns\n    -------\n    List[str]\n        List with the processed tokens\n    \"\"\"\n    tokens = [\n        token\n        for token in tokenize(doc, lower=lower, deacc=deacc, errors=\"ignore\")\n        if min_len &lt;= len(token) &lt;= max_len and not token.startswith(\"_\")\n    ]\n    return tokens\n</code></pre>"},{"location":"pytorch-widedeep/utils/text_utils.html#pytorch_widedeep.utils.text_utils.get_texts","title":"get_texts","text":"<pre><code>get_texts(texts, already_processed=False, n_cpus=None)\n</code></pre> <p>Tokenization using <code>Fastai</code>'s <code>Tokenizer</code> because it does a series of very convenients things during the tokenization process</p> <p>See <code>pytorch_widedeep.utils.fastai_utils.Tokenizer</code></p> <p> NOTE: <code>get_texts</code> uses <code>pytorch_widedeep.utils.fastai_transforms.Tokenizer</code>. Such tokenizer uses a series of convenient processing steps, including the  addition of some special tokens, such as <code>TK_MAJ</code> (<code>xxmaj</code>), used to indicate the next word begins with a capital in the original text. For more details of special tokens please see the <code>fastai</code> `docs</p> <p>Parameters:</p> Name Type Description Default <code>texts</code> <code>List[str]</code> <p>List of str with the texts (or documents). One str per document</p> required <code>already_processed</code> <code>Optional[bool]</code> <p>Boolean indicating if the text is already processed and we simply want to tokenize it. This parameter is thought for those cases where the input sequences might not be text (but IDs, or anything else) and we just want to tokenize it</p> <code>False</code> <code>n_cpus</code> <code>Optional[int]</code> <p>number of CPUs to used during the tokenization process</p> <code>None</code> <p>Examples:</p> <pre><code>&gt;&gt;&gt; from pytorch_widedeep.utils import get_texts\n&gt;&gt;&gt; texts = ['Machine learning is great', 'but building stuff is even better']\n&gt;&gt;&gt; get_texts(texts)\n[['xxmaj', 'machine', 'learning', 'is', 'great'], ['but', 'building', 'stuff', 'is', 'even', 'better']]\n</code></pre> <p>Returns:</p> Type Description <code>List[List[str]]</code> <p>List of lists, one list per 'document' containing its corresponding tokens</p> Source code in <code>pytorch_widedeep/utils/text_utils.py</code> <pre><code>def get_texts(\n    texts: List[str],\n    already_processed: Optional[bool] = False,\n    n_cpus: Optional[int] = None,\n) -&gt; List[List[str]]:\n    r\"\"\"Tokenization using `Fastai`'s `Tokenizer` because it does a\n    series of very convenients things during the tokenization process\n\n    See `pytorch_widedeep.utils.fastai_utils.Tokenizer`\n\n    :information_source: **NOTE**:\n    `get_texts` uses `pytorch_widedeep.utils.fastai_transforms.Tokenizer`.\n    Such tokenizer uses a series of convenient processing steps, including\n    the  addition of some special tokens, such as `TK_MAJ` (`xxmaj`), used to\n    indicate the next word begins with a capital in the original text. For more\n    details of special tokens please see the [`fastai` `docs](https://docs.fast.ai/text.core.html#Tokenizing)\n\n    Parameters\n    ----------\n    texts: List\n        List of str with the texts (or documents). One str per document\n    already_processed: bool, Optional, default = False\n        Boolean indicating if the text is already processed and we simply want\n        to tokenize it. This parameter is thought for those cases where the\n        input sequences might not be text (but IDs, or anything else) and we\n        just want to tokenize it\n    n_cpus: int, Optional, default = None\n        number of CPUs to used during the tokenization process\n\n    Examples\n    --------\n    &gt;&gt;&gt; from pytorch_widedeep.utils import get_texts\n    &gt;&gt;&gt; texts = ['Machine learning is great', 'but building stuff is even better']\n    &gt;&gt;&gt; get_texts(texts)\n    [['xxmaj', 'machine', 'learning', 'is', 'great'], ['but', 'building', 'stuff', 'is', 'even', 'better']]\n\n    Returns\n    -------\n    List[List[str]]\n        List of lists, one list per '_document_' containing its corresponding tokens\n    \"\"\"\n\n    num_cpus = n_cpus if n_cpus is not None else os.cpu_count()\n\n    if not already_processed:\n        processed_texts = [\" \".join(simple_preprocess(t)) for t in texts]\n    else:\n        processed_texts = texts\n    tok = Tokenizer(n_cpus=num_cpus).process_all(processed_texts)\n    return tok\n</code></pre>"},{"location":"pytorch-widedeep/utils/text_utils.html#pytorch_widedeep.utils.text_utils.pad_sequences","title":"pad_sequences","text":"<pre><code>pad_sequences(seq, maxlen, pad_first=True, pad_idx=1)\n</code></pre> <p>Given a List of tokenized and <code>numericalised</code> sequences it will return padded sequences according to the input parameters.</p> <p>Parameters:</p> Name Type Description Default <code>seq</code> <code>List[int]</code> <p>List of int with the <code>numericalised</code> tokens</p> required <code>maxlen</code> <code>int</code> <p>Maximum length of the padded sequences</p> required <code>pad_first</code> <code>bool</code> <p>Indicates whether the padding index will be added at the beginning or the end of the sequences</p> <code>True</code> <code>pad_idx</code> <code>int</code> <p>padding index. Fastai's Tokenizer leaves 0 for the 'unknown' token.</p> <code>1</code> <p>Examples:</p> <pre><code>&gt;&gt;&gt; from pytorch_widedeep.utils import pad_sequences\n&gt;&gt;&gt; seq = [1,2,3]\n&gt;&gt;&gt; pad_sequences(seq, maxlen=5, pad_idx=0)\narray([0, 0, 1, 2, 3], dtype=int32)\n</code></pre> <p>Returns:</p> Type Description <code>ndarray</code> <p>numpy array with the padded sequences</p> Source code in <code>pytorch_widedeep/utils/text_utils.py</code> <pre><code>def pad_sequences(\n    seq: List[int], maxlen: int, pad_first: bool = True, pad_idx: int = 1\n) -&gt; np.ndarray:\n    r\"\"\"\n    Given a List of tokenized and `numericalised` sequences it will return\n    padded sequences according to the input parameters.\n\n    Parameters\n    ----------\n    seq: List\n        List of int with the `numericalised` tokens\n    maxlen: int\n        Maximum length of the padded sequences\n    pad_first: bool,  default = True\n        Indicates whether the padding index will be added at the beginning or the\n        end of the sequences\n    pad_idx: int, default = 1\n        padding index. Fastai's Tokenizer leaves 0 for the 'unknown' token.\n\n    Examples\n    --------\n    &gt;&gt;&gt; from pytorch_widedeep.utils import pad_sequences\n    &gt;&gt;&gt; seq = [1,2,3]\n    &gt;&gt;&gt; pad_sequences(seq, maxlen=5, pad_idx=0)\n    array([0, 0, 1, 2, 3], dtype=int32)\n\n    Returns\n    -------\n    np.ndarray\n        numpy array with the padded sequences\n    \"\"\"\n    if len(seq) == 0:\n        return np.zeros(maxlen, dtype=\"int32\") + pad_idx\n    elif len(seq) &gt;= maxlen:\n        res = np.array(seq[-maxlen:]).astype(\"int32\")\n        return res\n    else:\n        res = np.zeros(maxlen, dtype=\"int32\") + pad_idx\n        if pad_first:\n            res[-len(seq) :] = seq\n        else:\n            res[: len(seq) :] = seq\n        return res\n</code></pre>"},{"location":"pytorch-widedeep/utils/text_utils.html#pytorch_widedeep.utils.text_utils.build_embeddings_matrix","title":"build_embeddings_matrix","text":"<pre><code>build_embeddings_matrix(\n    vocab, word_vectors_path, min_freq, verbose=1\n)\n</code></pre> <p>Build the embedding matrix using pretrained word vectors.</p> <p>Returns pretrained word embeddings. If a word in our vocabulary is not among the pretrained embeddings it will be assigned the mean pretrained word-embeddings vector</p> <p>Parameters:</p> Name Type Description Default <code>vocab</code> <code>Union[Vocab, ChunkVocab]</code> <p>see <code>pytorch_widedeep.utils.fastai_utils.Vocab</code></p> required <code>word_vectors_path</code> <code>str</code> <p>path to the pretrained word embeddings</p> required <code>min_freq</code> <code>int</code> <p>minimum frequency required for a word to be in the vocabulary</p> required <code>verbose</code> <code>int</code> <p>level of verbosity. Set to 0 for no verbosity</p> <code>1</code> <p>Returns:</p> Type Description <code>ndarray</code> <p>Pretrained word embeddings</p> Source code in <code>pytorch_widedeep/utils/text_utils.py</code> <pre><code>def build_embeddings_matrix(\n    vocab: Union[Vocab, ChunkVocab],\n    word_vectors_path: str,\n    min_freq: int,\n    verbose: int = 1,\n) -&gt; np.ndarray:  # pragma: no cover\n    r\"\"\"Build the embedding matrix using pretrained word vectors.\n\n    Returns pretrained word embeddings. If a word in our vocabulary is not\n    among the pretrained embeddings it will be assigned the mean pretrained\n    word-embeddings vector\n\n    Parameters\n    ----------\n    vocab: Vocab\n        see `pytorch_widedeep.utils.fastai_utils.Vocab`\n    word_vectors_path: str\n        path to the pretrained word embeddings\n    min_freq: int\n        minimum frequency required for a word to be in the vocabulary\n    verbose: int,  default=1\n        level of verbosity. Set to 0 for no verbosity\n\n    Returns\n    -------\n    np.ndarray\n        Pretrained word embeddings\n    \"\"\"\n    if not os.path.isfile(word_vectors_path):\n        raise FileNotFoundError(\"{} not found\".format(word_vectors_path))\n    if verbose:\n        print(\"Indexing word vectors...\")\n\n    embeddings_index = {}\n    f = open(word_vectors_path)\n    for line in f:\n        values = line.split()\n        word = values[0]\n        coefs = np.asarray(values[1:], dtype=\"float32\")\n        embeddings_index[word] = coefs\n    f.close()\n\n    if verbose:\n        print(\"Loaded {} word vectors\".format(len(embeddings_index)))\n        print(\"Preparing embeddings matrix...\")\n\n    mean_word_vector = np.mean(list(embeddings_index.values()), axis=0)  # type: ignore[arg-type]\n    embedding_dim = len(list(embeddings_index.values())[0])\n    num_words = len(vocab.itos)\n    embedding_matrix = np.zeros((num_words, embedding_dim))\n    found_words = 0\n    for i, word in enumerate(vocab.itos):\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[i] = embedding_vector\n            found_words += 1\n        else:\n            embedding_matrix[i] = mean_word_vector\n\n    if verbose:\n        print(\n            \"{} words in the vocabulary had {} vectors and appear more than {} times\".format(\n                found_words, word_vectors_path, min_freq\n            )\n        )\n\n    return embedding_matrix.astype(\"float32\")\n</code></pre>"}]}
\ No newline at end of file
+{"config":{"lang":["en"],"separator":"[\\s\\-]+","pipeline":["stopWordFilter"]},"docs":[{"location":"index.html","title":"Home","text":""},{"location":"index.html#pytorch-widedeep","title":"pytorch-widedeep","text":"<p>A flexible package for multimodal-deep-learning to combine tabular data with text and images using Wide and Deep models in Pytorch</p> <p>Documentation: https://pytorch-widedeep.readthedocs.io</p> <p>Companion posts and tutorials: infinitoml</p> <p>Experiments and comparison with <code>LightGBM</code>: TabularDL vs LightGBM</p> <p>Slack: if you want to contribute or just want to chat with us, join slack</p> <p>The content of this document is organized as follows:</p> <ul> <li>pytorch-widedeep<ul> <li>Introduction</li> <li>Architectures</li> <li>The <code>deeptabular</code> component</li> <li>The <code>rec</code> module</li> <li>Text and Images</li> <li>Acknowledgments</li> <li>License</li> <li>Cite<ul> <li>BibTex</li> <li>APA</li> </ul> </li> </ul> </li> </ul>"},{"location":"index.html#introduction","title":"Introduction","text":"<p><code>pytorch-widedeep</code> is based on Google's Wide and Deep Algorithm, adjusted for multi-modal datasets.</p> <p>In general terms, <code>pytorch-widedeep</code> is a package to use deep learning with tabular data. In particular, is intended to facilitate the combination of text and images with corresponding tabular data using wide and deep models. With that in mind there are a number of architectures that can be implemented with the library. The main components of those architectures are shown in the Figure below:</p> <p> </p> <p>In math terms, and following the notation in the paper, the expression for the architecture without a <code>deephead</code> component can be formulated as:</p> <p> </p> <p>Where \u03c3 is the sigmoid function, 'W' are the weight matrices applied to the wide model and to the final activations of the deep models, 'a' are these final activations, \u03c6(x) are the cross product transformations of the original features 'x', and , and 'b' is the bias term. In case you are wondering what are \"cross product transformations\", here is a quote taken directly from the paper: \"For binary features, a cross-product transformation (e.g., \u201cAND(gender=female, language=en)\u201d) is 1 if and only if the constituent features (\u201cgender=female\u201d and \u201clanguage=en\u201d) are all 1, and 0 otherwise\".</p> <p>It is perfectly possible to use custom models (and not necessarily those in the library) as long as the the custom models have a property called <code>output_dim</code> with the size of the last layer of activations, so that <code>WideDeep</code> can be constructed. Examples on how to use custom components can be found in the Examples folder and the section below.</p>"},{"location":"index.html#architectures","title":"Architectures","text":"<p>The <code>pytorch-widedeep</code> library offers a number of different architectures. In this section we will show some of them in their simplest form (i.e. with default param values in most cases) with their corresponding code snippets. Note that all the snippets below shoud run locally. For a more detailed explanation of the different components and their parameters, please refer to the documentation.</p> <p>For the examples below we will be using a toy dataset generated as follows:</p> <pre><code>import os\nimport random\n\nimport numpy as np\nimport pandas as pd\nfrom PIL import Image\nfrom faker import Faker\n\n\ndef create_and_save_random_image(image_number, size=(32, 32)):\n\n    if not os.path.exists(\"images\"):\n        os.makedirs(\"images\")\n\n    array = np.random.randint(0, 256, (size[0], size[1], 3), dtype=np.uint8)\n\n    image = Image.fromarray(array)\n\n    image_name = f\"image_{image_number}.png\"\n    image.save(os.path.join(\"images\", image_name))\n\n    return image_name\n\n\nfake = Faker()\n\ncities = [\"New York\", \"Los Angeles\", \"Chicago\", \"Houston\"]\nnames = [\"Alice\", \"Bob\", \"Charlie\", \"David\", \"Eva\"]\n\ndata = {\n    \"city\": [random.choice(cities) for _ in range(100)],\n    \"name\": [random.choice(names) for _ in range(100)],\n    \"age\": [random.uniform(18, 70) for _ in range(100)],\n    \"height\": [random.uniform(150, 200) for _ in range(100)],\n    \"sentence\": [fake.sentence() for _ in range(100)],\n    \"other_sentence\": [fake.sentence() for _ in range(100)],\n    \"image_name\": [create_and_save_random_image(i) for i in range(100)],\n    \"target\": [random.choice([0, 1]) for _ in range(100)],\n}\n\ndf = pd.DataFrame(data)\n</code></pre> <p>This will create a 100 rows dataframe and a dir in your local folder, called <code>images</code> with 100 random images (or images with just noise).</p> <p>Perhaps the simplest architecture would be just one component, <code>wide</code>, <code>deeptabular</code>, <code>deeptext</code> or <code>deepimage</code> on their own, which is also possible, but let's start the examples with a standard Wide and Deep architecture. From there, how to build a model comprised only of one component will be straightforward.</p> <p>Note that the examples shown below would be almost identical using any of the models available in the library. For example, <code>TabMlp</code> can be replaced by <code>TabResnet</code>, <code>TabNet</code>, <code>TabTransformer</code>, etc. Similarly, <code>BasicRNN</code> can be replaced by <code>AttentiveRNN</code>, <code>StackedAttentiveRNN</code>, or <code>HFModel</code> with their corresponding parameters and preprocessor in the case of the Hugging Face models.</p> <p>1. Wide and Tabular component (aka deeptabular)</p> <p> </p> <pre><code>from pytorch_widedeep.preprocessing import TabPreprocessor, WidePreprocessor\nfrom pytorch_widedeep.models import Wide, TabMlp, WideDeep\nfrom pytorch_widedeep.training import Trainer\n\n# Wide\nwide_cols = [\"city\"]\ncrossed_cols = [(\"city\", \"name\")]\nwide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols)\nX_wide = wide_preprocessor.fit_transform(df)\nwide = Wide(input_dim=np.unique(X_wide).shape[0])\n\n# Tabular\ntab_preprocessor = TabPreprocessor(\n    embed_cols=[\"city\", \"name\"], continuous_cols=[\"age\", \"height\"]\n)\nX_tab = tab_preprocessor.fit_transform(df)\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=tab_preprocessor.continuous_cols,\n    mlp_hidden_dims=[64, 32],\n)\n\n# WideDeep\nmodel = WideDeep(wide=wide, deeptabular=tab_mlp)\n\n# Train\ntrainer = Trainer(model, objective=\"binary\")\n\ntrainer.fit(\n    X_wide=X_wide,\n    X_tab=X_tab,\n    target=df[\"target\"].values,\n    n_epochs=1,\n    batch_size=32,\n)\n</code></pre> <p>2. Tabular and Text data</p> <p> </p> <pre><code>from pytorch_widedeep.preprocessing import TabPreprocessor, TextPreprocessor\nfrom pytorch_widedeep.models import TabMlp, BasicRNN, WideDeep\nfrom pytorch_widedeep.training import Trainer\n\n# Tabular\ntab_preprocessor = TabPreprocessor(\n    embed_cols=[\"city\", \"name\"], continuous_cols=[\"age\", \"height\"]\n)\nX_tab = tab_preprocessor.fit_transform(df)\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=tab_preprocessor.continuous_cols,\n    mlp_hidden_dims=[64, 32],\n)\n\n# Text\ntext_preprocessor = TextPreprocessor(\n    text_col=\"sentence\", maxlen=20, max_vocab=100, n_cpus=1\n)\nX_text = text_preprocessor.fit_transform(df)\nrnn = BasicRNN(\n    vocab_size=len(text_preprocessor.vocab.itos),\n    embed_dim=16,\n    hidden_dim=8,\n    n_layers=1,\n)\n\n# WideDeep\nmodel = WideDeep(deeptabular=tab_mlp, deeptext=rnn)\n\n# Train\ntrainer = Trainer(model, objective=\"binary\")\n\ntrainer.fit(\n    X_tab=X_tab,\n    X_text=X_text,\n    target=df[\"target\"].values,\n    n_epochs=1,\n    batch_size=32,\n)\n</code></pre> <p>3. Tabular and text with a FC head on top via the <code>head_hidden_dims</code> param   in <code>WideDeep</code></p> <p> </p> <pre><code>from pytorch_widedeep.preprocessing import TabPreprocessor, TextPreprocessor\nfrom pytorch_widedeep.models import TabMlp, BasicRNN, WideDeep\nfrom pytorch_widedeep.training import Trainer\n\n# Tabular\ntab_preprocessor = TabPreprocessor(\n    embed_cols=[\"city\", \"name\"], continuous_cols=[\"age\", \"height\"]\n)\nX_tab = tab_preprocessor.fit_transform(df)\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=tab_preprocessor.continuous_cols,\n    mlp_hidden_dims=[64, 32],\n)\n\n# Text\ntext_preprocessor = TextPreprocessor(\n    text_col=\"sentence\", maxlen=20, max_vocab=100, n_cpus=1\n)\nX_text = text_preprocessor.fit_transform(df)\nrnn = BasicRNN(\n    vocab_size=len(text_preprocessor.vocab.itos),\n    embed_dim=16,\n    hidden_dim=8,\n    n_layers=1,\n)\n\n# WideDeep\nmodel = WideDeep(deeptabular=tab_mlp, deeptext=rnn, head_hidden_dims=[32, 16])\n\n# Train\ntrainer = Trainer(model, objective=\"binary\")\n\ntrainer.fit(\n    X_tab=X_tab,\n    X_text=X_text,\n    target=df[\"target\"].values,\n    n_epochs=1,\n    batch_size=32,\n)\n</code></pre> <p>4. Tabular and multiple text columns that are passed directly to   <code>WideDeep</code></p> <p> </p> <pre><code>from pytorch_widedeep.preprocessing import TabPreprocessor, TextPreprocessor\nfrom pytorch_widedeep.models import TabMlp, BasicRNN, WideDeep\nfrom pytorch_widedeep.training import Trainer\n\n\n# Tabular\ntab_preprocessor = TabPreprocessor(\n    embed_cols=[\"city\", \"name\"], continuous_cols=[\"age\", \"height\"]\n)\nX_tab = tab_preprocessor.fit_transform(df)\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=tab_preprocessor.continuous_cols,\n    mlp_hidden_dims=[64, 32],\n)\n\n# Text\ntext_preprocessor_1 = TextPreprocessor(\n    text_col=\"sentence\", maxlen=20, max_vocab=100, n_cpus=1\n)\nX_text_1 = text_preprocessor_1.fit_transform(df)\ntext_preprocessor_2 = TextPreprocessor(\n    text_col=\"other_sentence\", maxlen=20, max_vocab=100, n_cpus=1\n)\nX_text_2 = text_preprocessor_2.fit_transform(df)\nrnn_1 = BasicRNN(\n    vocab_size=len(text_preprocessor_1.vocab.itos),\n    embed_dim=16,\n    hidden_dim=8,\n    n_layers=1,\n)\nrnn_2 = BasicRNN(\n    vocab_size=len(text_preprocessor_2.vocab.itos),\n    embed_dim=16,\n    hidden_dim=8,\n    n_layers=1,\n)\n\n# WideDeep\nmodel = WideDeep(deeptabular=tab_mlp, deeptext=[rnn_1, rnn_2])\n\n# Train\ntrainer = Trainer(model, objective=\"binary\")\n\ntrainer.fit(\n    X_tab=X_tab,\n    X_text=[X_text_1, X_text_2],\n    target=df[\"target\"].values,\n    n_epochs=1,\n    batch_size=32,\n)\n</code></pre> <p>5. Tabular data and multiple text columns that are fused via a the library's   <code>ModelFuser</code> class</p> <p> </p> <pre><code>from pytorch_widedeep.preprocessing import TabPreprocessor, TextPreprocessor\nfrom pytorch_widedeep.models import TabMlp, BasicRNN, WideDeep, ModelFuser\nfrom pytorch_widedeep import Trainer\n\n# Tabular\ntab_preprocessor = TabPreprocessor(\n    embed_cols=[\"city\", \"name\"], continuous_cols=[\"age\", \"height\"]\n)\nX_tab = tab_preprocessor.fit_transform(df)\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=tab_preprocessor.continuous_cols,\n    mlp_hidden_dims=[64, 32],\n)\n\n# Text\ntext_preprocessor_1 = TextPreprocessor(\n    text_col=\"sentence\", maxlen=20, max_vocab=100, n_cpus=1\n)\nX_text_1 = text_preprocessor_1.fit_transform(df)\ntext_preprocessor_2 = TextPreprocessor(\n    text_col=\"other_sentence\", maxlen=20, max_vocab=100, n_cpus=1\n)\nX_text_2 = text_preprocessor_2.fit_transform(df)\n\nrnn_1 = BasicRNN(\n    vocab_size=len(text_preprocessor_1.vocab.itos),\n    embed_dim=16,\n    hidden_dim=8,\n    n_layers=1,\n)\nrnn_2 = BasicRNN(\n    vocab_size=len(text_preprocessor_2.vocab.itos),\n    embed_dim=16,\n    hidden_dim=8,\n    n_layers=1,\n)\n\nmodels_fuser = ModelFuser(models=[rnn_1, rnn_2], fusion_method=\"mult\")\n\n# WideDeep\nmodel = WideDeep(deeptabular=tab_mlp, deeptext=models_fuser)\n\n# Train\ntrainer = Trainer(model, objective=\"binary\")\n\ntrainer.fit(\n    X_tab=X_tab,\n    X_text=[X_text_1, X_text_2],\n    target=df[\"target\"].values,\n    n_epochs=1,\n    batch_size=32,\n)\n</code></pre> <p>6. Tabular and multiple text columns, with an image column. The text columns   are fused via the library's <code>ModelFuser</code> and then all fused via the   deephead paramenter in <code>WideDeep</code> which is a custom <code>ModelFuser</code> coded by   the user</p> <p>This is perhaps the less elegant solution as it involves a custom component by the user and slicing the 'incoming' tensor. In the future, we will include a <code>TextAndImageModelFuser</code> to make this process more straightforward. Still, is not really complicated and it is a good example of how to use custom components in <code>pytorch-widedeep</code>.</p> <p>Note that the only requirement for the custom component is that it has a property called <code>output_dim</code> that returns the size of the last layer of activations. In other words, it does not need to inherit from <code>BaseWDModelComponent</code>. This base class simply checks the existence of such property and avoids some typing errors internally.</p> <p> </p> <pre><code>import torch\n\nfrom pytorch_widedeep.preprocessing import TabPreprocessor, TextPreprocessor, ImagePreprocessor\nfrom pytorch_widedeep.models import TabMlp, BasicRNN, WideDeep, ModelFuser, Vision\nfrom pytorch_widedeep.models._base_wd_model_component import BaseWDModelComponent\nfrom pytorch_widedeep import Trainer\n\n# Tabular\ntab_preprocessor = TabPreprocessor(\n    embed_cols=[\"city\", \"name\"], continuous_cols=[\"age\", \"height\"]\n)\nX_tab = tab_preprocessor.fit_transform(df)\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=tab_preprocessor.continuous_cols,\n    mlp_hidden_dims=[16, 8],\n)\n\n# Text\ntext_preprocessor_1 = TextPreprocessor(\n    text_col=\"sentence\", maxlen=20, max_vocab=100, n_cpus=1\n)\nX_text_1 = text_preprocessor_1.fit_transform(df)\ntext_preprocessor_2 = TextPreprocessor(\n    text_col=\"other_sentence\", maxlen=20, max_vocab=100, n_cpus=1\n)\nX_text_2 = text_preprocessor_2.fit_transform(df)\nrnn_1 = BasicRNN(\n    vocab_size=len(text_preprocessor_1.vocab.itos),\n    embed_dim=16,\n    hidden_dim=8,\n    n_layers=1,\n)\nrnn_2 = BasicRNN(\n    vocab_size=len(text_preprocessor_2.vocab.itos),\n    embed_dim=16,\n    hidden_dim=8,\n    n_layers=1,\n)\nmodels_fuser = ModelFuser(\n    models=[rnn_1, rnn_2],\n    fusion_method=\"mult\",\n)\n\n# Image\nimage_preprocessor = ImagePreprocessor(img_col=\"image_name\", img_path=\"images\")\nX_img = image_preprocessor.fit_transform(df)\nvision = Vision(pretrained_model_setup=\"resnet18\", head_hidden_dims=[16, 8])\n\n# deephead (custom model fuser)\nclass MyModelFuser(BaseWDModelComponent):\n    \"\"\"\n    Simply a Linear + Relu sequence on top of the text + images followed by a\n    Linear -&gt; Relu -&gt; Linear for the concatenation of tabular slice of the\n    tensor and the output of the text and image sequential model\n    \"\"\"\n    def __init__(\n        self,\n        tab_incoming_dim: int,\n        text_incoming_dim: int,\n        image_incoming_dim: int,\n        output_units: int,\n    ):\n\n        super(MyModelFuser, self).__init__()\n\n        self.tab_incoming_dim = tab_incoming_dim\n        self.text_incoming_dim = text_incoming_dim\n        self.image_incoming_dim = image_incoming_dim\n        self.output_units = output_units\n        self.text_and_image_fuser = torch.nn.Sequential(\n            torch.nn.Linear(text_incoming_dim + image_incoming_dim, output_units),\n            torch.nn.ReLU(),\n        )\n        self.out = torch.nn.Sequential(\n            torch.nn.Linear(output_units + tab_incoming_dim, output_units * 4),\n            torch.nn.ReLU(),\n            torch.nn.Linear(output_units * 4, output_units),\n        )\n\n    def forward(self, X: torch.Tensor) -&gt; torch.Tensor:\n        tab_slice = slice(0, self.tab_incoming_dim)\n        text_slice = slice(\n            self.tab_incoming_dim, self.tab_incoming_dim + self.text_incoming_dim\n        )\n        image_slice = slice(\n            self.tab_incoming_dim + self.text_incoming_dim,\n            self.tab_incoming_dim + self.text_incoming_dim + self.image_incoming_dim,\n        )\n        X_tab = X[:, tab_slice]\n        X_text = X[:, text_slice]\n        X_img = X[:, image_slice]\n        X_text_and_image = self.text_and_image_fuser(torch.cat([X_text, X_img], dim=1))\n        return self.out(torch.cat([X_tab, X_text_and_image], dim=1))\n\n    @property\n    def output_dim(self):\n        return self.output_units\n\ndeephead = MyModelFuser(\n    tab_incoming_dim=tab_mlp.output_dim,\n    text_incoming_dim=models_fuser.output_dim,\n    image_incoming_dim=vision.output_dim,\n    output_units=8,\n)\n\n# WideDeep\nmodel = WideDeep(\n    deeptabular=tab_mlp,\n    deeptext=models_fuser,\n    deepimage=vision,\n    deephead=deephead,\n)\n\n# Train\ntrainer = Trainer(model, objective=\"binary\")\n\ntrainer.fit(\n    X_tab=X_tab,\n    X_text=[X_text_1, X_text_2],\n    X_img=X_img,\n    target=df[\"target\"].values,\n    n_epochs=1,\n    batch_size=32,\n)\n</code></pre> <p>7. A two-tower model</p> <p>This is a popular model in the context of recommendation systems. Let's say we have a tabular dataset formed my triples (user features, item features, target). We can create a two-tower model where the user and item features are passed through two separate models and then \"fused\" via a dot product.</p> <p> </p> <pre><code>import numpy as np\nimport pandas as pd\n\nfrom pytorch_widedeep import Trainer\nfrom pytorch_widedeep.preprocessing import TabPreprocessor\nfrom pytorch_widedeep.models import TabMlp, WideDeep, ModelFuser\n\n# Let's create the interaction dataset\n# user_features dataframe\nnp.random.seed(42)\nuser_ids = np.arange(1, 101)\nages = np.random.randint(18, 60, size=100)\ngenders = np.random.choice([\"male\", \"female\"], size=100)\nlocations = np.random.choice([\"city_a\", \"city_b\", \"city_c\", \"city_d\"], size=100)\nuser_features = pd.DataFrame(\n    {\"id\": user_ids, \"age\": ages, \"gender\": genders, \"location\": locations}\n)\n\n# item_features dataframe\nitem_ids = np.arange(1, 101)\nprices = np.random.uniform(10, 500, size=100).round(2)\ncolors = np.random.choice([\"red\", \"blue\", \"green\", \"black\"], size=100)\ncategories = np.random.choice([\"electronics\", \"clothing\", \"home\", \"toys\"], size=100)\n\nitem_features = pd.DataFrame(\n    {\"id\": item_ids, \"price\": prices, \"color\": colors, \"category\": categories}\n)\n\n# Interactions dataframe\ninteraction_user_ids = np.random.choice(user_ids, size=1000)\ninteraction_item_ids = np.random.choice(item_ids, size=1000)\npurchased = np.random.choice([0, 1], size=1000, p=[0.7, 0.3])\ninteractions = pd.DataFrame(\n    {\n        \"user_id\": interaction_user_ids,\n        \"item_id\": interaction_item_ids,\n        \"purchased\": purchased,\n    }\n)\nuser_item_purchased = interactions.merge(\n    user_features, left_on=\"user_id\", right_on=\"id\"\n).merge(item_features, left_on=\"item_id\", right_on=\"id\")\n\n# Users\ntab_preprocessor_user = TabPreprocessor(\n    cat_embed_cols=[\"gender\", \"location\"],\n    continuous_cols=[\"age\"],\n)\nX_user = tab_preprocessor_user.fit_transform(user_item_purchased)\ntab_mlp_user = TabMlp(\n    column_idx=tab_preprocessor_user.column_idx,\n    cat_embed_input=tab_preprocessor_user.cat_embed_input,\n    continuous_cols=[\"age\"],\n    mlp_hidden_dims=[16, 8],\n    mlp_dropout=[0.2, 0.2],\n)\n\n# Items\ntab_preprocessor_item = TabPreprocessor(\n    cat_embed_cols=[\"color\", \"category\"],\n    continuous_cols=[\"price\"],\n)\nX_item = tab_preprocessor_item.fit_transform(user_item_purchased)\ntab_mlp_item = TabMlp(\n    column_idx=tab_preprocessor_item.column_idx,\n    cat_embed_input=tab_preprocessor_item.cat_embed_input,\n    continuous_cols=[\"price\"],\n    mlp_hidden_dims=[16, 8],\n    mlp_dropout=[0.2, 0.2],\n)\n\ntwo_tower_model = ModelFuser([tab_mlp_user, tab_mlp_item], fusion_method=\"dot\")\n\nmodel = WideDeep(deeptabular=two_tower_model)\n\ntrainer = Trainer(model, objective=\"binary\")\n\ntrainer.fit(\n    X_tab=[X_user, X_item],\n    target=interactions.purchased.values,\n    n_epochs=1,\n    batch_size=32,\n)\n</code></pre> <p>8. Tabular with a multi-target loss</p> <p>This one is \"a bonus\" to illustrate the use of multi-target losses, more than actually a different architecture.</p> <p> </p> <pre><code>from pytorch_widedeep.preprocessing import TabPreprocessor, TextPreprocessor, ImagePreprocessor\nfrom pytorch_widedeep.models import TabMlp, BasicRNN, WideDeep, ModelFuser, Vision\nfrom pytorch_widedeep.losses_multitarget import MultiTargetClassificationLoss\nfrom pytorch_widedeep.models._base_wd_model_component import BaseWDModelComponent\nfrom pytorch_widedeep import Trainer\n\n# let's add a second target to the dataframe\ndf[\"target2\"] = [random.choice([0, 1]) for _ in range(100)]\n\n# Tabular\ntab_preprocessor = TabPreprocessor(\n    embed_cols=[\"city\", \"name\"], continuous_cols=[\"age\", \"height\"]\n)\nX_tab = tab_preprocessor.fit_transform(df)\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=tab_preprocessor.continuous_cols,\n    mlp_hidden_dims=[64, 32],\n)\n\n# 'pred_dim=2' because we have two binary targets. For other types of targets,\n#  please, see the documentation\nmodel = WideDeep(deeptabular=tab_mlp, pred_dim=2).\n\nloss = MultiTargetClassificationLoss(binary_config=[0, 1], reduction=\"mean\")\n\n# When a multi-target loss is used, 'custom_loss_function' must not be None.\n# See the docs\ntrainer = Trainer(model, objective=\"multitarget\", custom_loss_function=loss)\n\ntrainer.fit(\n    X_tab=X_tab,\n    target=df[[\"target\", \"target2\"]].values,\n    n_epochs=1,\n    batch_size=32,\n)\n</code></pre>"},{"location":"index.html#the-deeptabular-component","title":"The <code>deeptabular</code> component","text":"<p>It is important to emphasize again that each individual component, <code>wide</code>, <code>deeptabular</code>, <code>deeptext</code> and <code>deepimage</code>, can be used independently and in isolation. For example, one could use only <code>wide</code>, which is in simply a linear model. In fact, one of the most interesting functionalities in<code>pytorch-widedeep</code> would be the use of the <code>deeptabular</code> component on its own, i.e. what one might normally refer as Deep Learning for Tabular Data. Currently, <code>pytorch-widedeep</code> offers the following different models for that component:</p> <ol> <li>Wide: a simple linear model where the nonlinearities are captured via cross-product transformations, as explained before.</li> <li>TabMlp: a simple MLP that receives embeddings representing the categorical features, concatenated with the continuous features, which can also be embedded.</li> <li>TabResnet: similar to the previous model but the embeddings are passed through a series of ResNet blocks built with dense layers.</li> <li>TabNet: details on TabNet can be found in TabNet: Attentive Interpretable Tabular Learning</li> </ol> <p>Two simpler attention based models that we call:</p> <ol> <li>ContextAttentionMLP: MLP with at attention mechanism \"on top\" that is based on     Hierarchical Attention Networks for Document Classification</li> <li>SelfAttentionMLP: MLP with an attention mechanism that is a simplified     version of a transformer block that we refer as \"query-key self-attention\".</li> </ol> <p>The <code>Tabformer</code> family, i.e. Transformers for Tabular data:</p> <ol> <li>TabTransformer: details on the TabTransformer can be found in TabTransformer: Tabular Data Modeling Using Contextual Embeddings.</li> <li>SAINT: Details on SAINT can be found in SAINT: Improved Neural Networks for Tabular Data via Row Attention and Contrastive Pre-Training.</li> <li>FT-Transformer: details on the FT-Transformer can be found in Revisiting Deep Learning Models for Tabular Data.</li> <li>TabFastFormer: adaptation of the FastFormer for tabular data. Details on the Fasformer can be found in FastFormers: Highly Efficient Transformer Models for Natural Language Understanding</li> <li>TabPerceiver: adaptation of the Perceiver for tabular data. Details on the Perceiver can be found in Perceiver: General Perception with Iterative Attention</li> </ol> <p>And probabilistic DL models for tabular data based on Weight Uncertainty in Neural Networks:</p> <ol> <li>BayesianWide: Probabilistic adaptation of the <code>Wide</code> model.</li> <li>BayesianTabMlp: Probabilistic adaptation of the <code>TabMlp</code> model</li> </ol> <p>Note that while there are scientific publications for the TabTransformer, SAINT and FT-Transformer, the TabFasfFormer and TabPerceiver are our own adaptation of those algorithms for tabular data.</p> <p>In addition, Self-Supervised pre-training can be used for all <code>deeptabular</code> models, with the exception of the <code>TabPerceiver</code>. Self-Supervised pre-training can be used via two methods or routines which we refer as: encoder-decoder method and constrastive-denoising method. Please, see the documentation and the examples for details on this functionality, and all other options in the library.</p>"},{"location":"index.html#the-rec-module","title":"The <code>rec</code> module","text":"<p>This module was introduced as an extension to the existing components in the library, addressing questions and issues related to recommendation systems. While still under active development, it currently includes a select number of powerful recommendation models.</p> <p>It's worth noting that this library already supported the implementation of various recommendation algorithms using existing components. For example, models like Wide and Deep, Two-Tower, or Neural Collaborative Filtering could be constructed using the library's core functionalities.</p> <p>The recommendation algorithms in the <code>rec</code> module are:</p> <ol> <li>DeepFM: A Factorization-Machine based Neural Network for CTR Prediction</li> <li>(Deep) Field Aware Factorization Machine (FFM): a Deep Learning version of the algorithm presented in Field-aware Factorization Machines in a Real-world Online Advertising System</li> <li>xDeepFM: Combining Explicit and Implicit Feature Interactions for Recommender Systems</li> <li>Deep Interest Network for Click-Through Rate Prediction</li> </ol> <p>These can all be used as the <code>deeptabular</code> component in the <code>WideDeep</code> model. See the examples for more details.</p>"},{"location":"index.html#text-and-images","title":"Text and Images","text":"<p>For the text component, <code>deeptext</code>, the library offers the following models:</p> <ol> <li>BasicRNN: a simple RNN 2. AttentiveRNN: a RNN with an attention mechanism based on the Hierarchical Attention Networks for DocumentClassification</li> <li>StackedAttentiveRNN: a stack of AttentiveRNNs</li> <li>HFModel: a wrapper around Hugging Face Transfomer-based models. At the moment only models from the families BERT, RoBERTa, DistilBERT, ALBERT and ELECTRA are supported. This is because this library is designed to address classification and regression tasks and these are the most 'popular' encoder-only models, which have proved to be those that work best for these tasks. If there is demand for other models, they will be included in the future.</li> </ol> <p>For the image component, <code>deepimage</code>, the library supports models from the following families: 'resnet', 'shufflenet', 'resnext', 'wide_resnet', 'regnet', 'densenet', 'mobilenetv3',  'mobilenetv2', 'mnasnet', 'efficientnet' and 'squeezenet'.  These are  offered via <code>torchvision</code> and wrapped up in the <code>Vision</code> class.</p>"},{"location":"index.html#acknowledgments","title":"Acknowledgments","text":"<p>This library takes from a series of other libraries, so I think it is just fair to mention them here in the README (specific mentions are also included in the code).</p> <p>The <code>Callbacks</code> and <code>Initializers</code> structure and code is inspired by the <code>torchsample</code> library, which in itself partially inspired by <code>Keras</code>.</p> <p>The <code>TextProcessor</code> class in this library uses the <code>fastai</code>'s <code>Tokenizer</code> and <code>Vocab</code>. The code at <code>utils.fastai_transforms</code> is a minor adaptation of their code so it functions within this library. To my experience their <code>Tokenizer</code> is the best in class.</p> <p>The <code>ImageProcessor</code> class in this library uses code from the fantastic Deep Learning for Computer Vision (DL4CV) book by Adrian Rosebrock.</p>"},{"location":"index.html#license","title":"License","text":"<p>This work is dual-licensed under Apache 2.0 and MIT (or any later version). You can choose between one of them if you use this work.</p> <p><code>SPDX-License-Identifier: Apache-2.0 AND MIT</code></p>"},{"location":"index.html#cite","title":"Cite","text":""},{"location":"index.html#bibtex","title":"BibTex","text":"<pre><code>@article{Zaurin_pytorch-widedeep_A_flexible_2023,\nauthor = {Zaurin, Javier Rodriguez and Mulinka, Pavol},\ndoi = {10.21105/joss.05027},\njournal = {Journal of Open Source Software},\nmonth = jun,\nnumber = {86},\npages = {5027},\ntitle = {{pytorch-widedeep: A flexible package for multimodal deep learning}},\nurl = {https://joss.theoj.org/papers/10.21105/joss.05027},\nvolume = {8},\nyear = {2023}\n}\n</code></pre>"},{"location":"index.html#apa","title":"APA","text":"<pre><code>Zaurin, J. R., &amp; Mulinka, P. (2023). pytorch-widedeep: A flexible package for\nmultimodal deep learning. Journal of Open Source Software, 8(86), 5027.\nhttps://doi.org/10.21105/joss.05027\n</code></pre>"},{"location":"contributing.html","title":"Contributing","text":"<p>Pytorch-widedeep is being developed and used by many active community members. Your help is very valuable to make it better for everyone.</p> <ul> <li>Check for the Roadmap or Open an issue to report problems or recommend new features and submit a draft pull requests, which will be changed to pull request after intial review</li> <li>Contribute to the tests to make it more reliable.</li> <li>Contribute to the documentation to make it clearer for everyone.</li> <li>Contribute to the examples to share your experience with other users.</li> <li>Join the dicussion on slack</li> </ul>"},{"location":"installation.html","title":"Installation","text":"<p>This section explains how to install <code>pytorch-widedeep</code>.</p> <p>For the latest stable release, execute:</p> <pre><code>pip install pytorch-widedeep\n</code></pre> <p>For the bleeding-edge version, execute:</p> <pre><code>pip install git+https://github.com/jrzaurin/pytorch-widedeep.git\n</code></pre> <p>For developer install</p> <pre><code># Clone the repository\ngit clone https://github.com/jrzaurin/pytorch-widedeep\ncd pytorch-widedeep\n\n# Install in dev mode\npip install -e .\n</code></pre>"},{"location":"installation.html#dependencies","title":"Dependencies","text":"<ul> <li>pandas&gt;=1.3.5</li> <li>numpy&gt;=1.21.6</li> <li>scipy&gt;=1.7.3,&lt;=1.12.0</li> <li>scikit-learn&gt;=1.0.2</li> <li>gensim</li> <li>spacy</li> <li>opencv-contrib-python</li> <li>imutils</li> <li>tqdm</li> <li>torch &gt;= 2.0.0</li> <li>torchvision &gt;= 0.15.0</li> <li>einops</li> <li>wrapt</li> <li>torchmetrics</li> <li>pyarrow</li> <li>fastparquet&gt;=0.8.1</li> <li>transformers</li> <li>sentence-transformers</li> <li>sentencepiece</li> </ul>"},{"location":"quick_start.html","title":"Quick Start","text":"<p>This is an example of a binary classification with the adult census dataset using a combination of a wide and deep model (in this case a so called <code>deeptabular</code> model) with defaults settings.</p> <pre><code>import numpy as np\nimport torch\nfrom sklearn.model_selection import train_test_split\n\nfrom pytorch_widedeep import Trainer\nfrom pytorch_widedeep.preprocessing import WidePreprocessor, TabPreprocessor\nfrom pytorch_widedeep.models import Wide, TabMlp, WideDeep\nfrom pytorch_widedeep.metrics import Accuracy\nfrom pytorch_widedeep.datasets import load_adult\n\n\ndf = load_adult(as_frame=True)\ndf[\"income_label\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int)\ndf.drop(\"income\", axis=1, inplace=True)\ndf_train, df_test = train_test_split(df, test_size=0.2, stratify=df.income_label)\n\n# Define the 'column set up'\nwide_cols = [\n    \"education\",\n    \"relationship\",\n    \"workclass\",\n    \"occupation\",\n    \"native-country\",\n    \"gender\",\n]\ncrossed_cols = [(\"education\", \"occupation\"), (\"native-country\", \"occupation\")]\n\ncat_embed_cols = [\n    \"workclass\",\n    \"education\",\n    \"marital-status\",\n    \"occupation\",\n    \"relationship\",\n    \"race\",\n    \"gender\",\n    \"capital-gain\",\n    \"capital-loss\",\n    \"native-country\",\n]\ncontinuous_cols = [\"age\", \"hours-per-week\"]\ntarget = \"income_label\"\ntarget = df_train[target].values\n\n# prepare the data\nwide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols)\nX_wide = wide_preprocessor.fit_transform(df_train)\n\ntab_preprocessor = TabPreprocessor(\n    cat_embed_cols=cat_embed_cols, continuous_cols=continuous_cols  # type: ignore[arg-type]\n)\nX_tab = tab_preprocessor.fit_transform(df_train)\n\n# build the model\nwide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1)\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=continuous_cols,\n)\nmodel = WideDeep(wide=wide, deeptabular=tab_mlp)\n\n# train and validate\ntrainer = Trainer(model, objective=\"binary\", metrics=[Accuracy])\ntrainer.fit(\n    X_wide=X_wide,\n    X_tab=X_tab,\n    target=target,\n    n_epochs=5,\n    batch_size=256,\n)\n\n# predict on test\nX_wide_te = wide_preprocessor.transform(df_test)\nX_tab_te = tab_preprocessor.transform(df_test)\npreds = trainer.predict(X_wide=X_wide_te, X_tab=X_tab_te)\n\n# Save and load\n\n# Option 1: this will also save training history and lr history if the\n# LRHistory callback is used\ntrainer.save(path=\"model_weights\", save_state_dict=True)\n\n# Option 2: save as any other torch model\ntorch.save(model.state_dict(), \"model_weights/wd_model.pt\")\n\n# From here in advance, Option 1 or 2 are the same. I assume the user has\n# prepared the data and defined the new model components:\n# 1. Build the model\nmodel_new = WideDeep(wide=wide, deeptabular=tab_mlp)\nmodel_new.load_state_dict(torch.load(\"model_weights/wd_model.pt\"))\n\n# 2. Instantiate the trainer\ntrainer_new = Trainer(model_new, objective=\"binary\")\n\n# 3. Either start the fit or directly predict\npreds = trainer_new.predict(X_wide=X_wide, X_tab=X_tab, batch_size=32)\n</code></pre>"},{"location":"examples/01_preprocessors_and_utils.html","title":"01_preprocessors_and_utils","text":"<p>For example</p> In\u00a0[1]: Copied! <pre>import numpy as np\nimport pandas as pd\nimport pytorch_widedeep as wd\n\nfrom pytorch_widedeep.datasets import load_adult\nfrom pytorch_widedeep.preprocessing import WidePreprocessor\n</pre> import numpy as np import pandas as pd import pytorch_widedeep as wd  from pytorch_widedeep.datasets import load_adult from pytorch_widedeep.preprocessing import WidePreprocessor <pre>/Users/javierrodriguezzaurin/.pyenv/versions/3.10.13/envs/widedeep310/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n  from .autonotebook import tqdm as notebook_tqdm\n</pre> In\u00a0[2]: Copied! <pre>df = load_adult(as_frame=True)\ndf.head()\n</pre> df = load_adult(as_frame=True) df.head() Out[2]: age workclass fnlwgt education educational-num marital-status occupation relationship race gender capital-gain capital-loss hours-per-week native-country income 0 25 Private 226802 11th 7 Never-married Machine-op-inspct Own-child Black Male 0 0 40 United-States &lt;=50K 1 38 Private 89814 HS-grad 9 Married-civ-spouse Farming-fishing Husband White Male 0 0 50 United-States &lt;=50K 2 28 Local-gov 336951 Assoc-acdm 12 Married-civ-spouse Protective-serv Husband White Male 0 0 40 United-States &gt;50K 3 44 Private 160323 Some-college 10 Married-civ-spouse Machine-op-inspct Husband Black Male 7688 0 40 United-States &gt;50K 4 18 ? 103497 Some-college 10 Never-married ? Own-child White Female 0 0 30 United-States &lt;=50K In\u00a0[3]: Copied! <pre>wide_cols = [\n    \"education\",\n    \"relationship\",\n    \"workclass\",\n    \"occupation\",\n    \"native-country\",\n    \"gender\",\n]\ncrossed_cols = [(\"education\", \"occupation\"), (\"native-country\", \"occupation\")]\n</pre> wide_cols = [     \"education\",     \"relationship\",     \"workclass\",     \"occupation\",     \"native-country\",     \"gender\", ] crossed_cols = [(\"education\", \"occupation\"), (\"native-country\", \"occupation\")] In\u00a0[4]: Copied! <pre>wide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols)\nX_wide = wide_preprocessor.fit_transform(df)\n# From here on, any new observation can be prepared by simply running `.transform`\n# new_X_wide = wide_preprocessor.transform(new_df)\n</pre> wide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols) X_wide = wide_preprocessor.fit_transform(df) # From here on, any new observation can be prepared by simply running `.transform` # new_X_wide = wide_preprocessor.transform(new_df) In\u00a0[5]: Copied! <pre>X_wide\n</pre> X_wide Out[5]: <pre>array([[  1,  17,  23, ...,  89,  91, 316],\n       [  2,  18,  23, ...,  89,  92, 317],\n       [  3,  18,  24, ...,  89,  93, 318],\n       ...,\n       [  2,  20,  23, ...,  90, 103, 323],\n       [  2,  17,  23, ...,  89, 103, 323],\n       [  2,  21,  29, ...,  90, 115, 324]])</pre> <p>Note that the label encoding starts from <code>1</code>. This is because it is convenient to leave <code>0</code> for padding, i.e. unknown categories. Let's take from example the first entry</p> In\u00a0[6]: Copied! <pre>X_wide[0]\n</pre> X_wide[0] Out[6]: <pre>array([  1,  17,  23,  32,  47,  89,  91, 316])</pre> In\u00a0[7]: Copied! <pre>wide_preprocessor.inverse_transform(X_wide[:1])\n</pre> wide_preprocessor.inverse_transform(X_wide[:1]) Out[7]: education relationship workclass occupation native-country gender education_occupation native-country_occupation 0 11th Own-child Private Machine-op-inspct United-States Male 11th-Machine-op-inspct United-States-Machine-op-inspct <p>As we can see, <code>wide_preprocessor</code> numerically encodes the <code>wide_cols</code> and the <code>crossed_cols</code>, which can be recovered using the method <code>inverse_transform</code>.</p> In\u00a0[8]: Copied! <pre>from pytorch_widedeep.preprocessing import TabPreprocessor\n</pre> from pytorch_widedeep.preprocessing import TabPreprocessor In\u00a0[9]: Copied! <pre># cat_embed_cols = [(column_name, embed_dim), ...]\ncat_embed_cols = [\n    (\"education\", 10),\n    (\"relationship\", 8),\n    (\"workclass\", 10),\n    (\"occupation\", 10),\n    (\"native-country\", 10),\n]\ncontinuous_cols = [\"age\", \"hours-per-week\"]\n</pre> # cat_embed_cols = [(column_name, embed_dim), ...] cat_embed_cols = [     (\"education\", 10),     (\"relationship\", 8),     (\"workclass\", 10),     (\"occupation\", 10),     (\"native-country\", 10), ] continuous_cols = [\"age\", \"hours-per-week\"] In\u00a0[10]: Copied! <pre>tab_preprocessor = TabPreprocessor(\n    cat_embed_cols=cat_embed_cols,\n    continuous_cols=continuous_cols,\n    cols_to_scale=[\"age\"],  # or scale=True or cols_to_scale=continuous_cols\n)\nX_tab = tab_preprocessor.fit_transform(df)\n# From here on, any new observation can be prepared by simply running `.transform`\n# new_X_deep = deep_preprocessor.transform(new_df)\n</pre> tab_preprocessor = TabPreprocessor(     cat_embed_cols=cat_embed_cols,     continuous_cols=continuous_cols,     cols_to_scale=[\"age\"],  # or scale=True or cols_to_scale=continuous_cols ) X_tab = tab_preprocessor.fit_transform(df) # From here on, any new observation can be prepared by simply running `.transform` # new_X_deep = deep_preprocessor.transform(new_df) In\u00a0[11]: Copied! <pre>X_tab\n</pre> X_tab Out[11]: <pre>array([[ 1.00000000e+00,  1.00000000e+00,  1.00000000e+00, ...,\n         1.00000000e+00, -9.95128932e-01,  4.00000000e+01],\n       [ 2.00000000e+00,  2.00000000e+00,  1.00000000e+00, ...,\n         1.00000000e+00, -4.69415091e-02,  5.00000000e+01],\n       [ 3.00000000e+00,  2.00000000e+00,  2.00000000e+00, ...,\n         1.00000000e+00, -7.76316450e-01,  4.00000000e+01],\n       ...,\n       [ 2.00000000e+00,  4.00000000e+00,  1.00000000e+00, ...,\n         1.00000000e+00,  1.41180837e+00,  4.00000000e+01],\n       [ 2.00000000e+00,  1.00000000e+00,  1.00000000e+00, ...,\n         1.00000000e+00, -1.21394141e+00,  2.00000000e+01],\n       [ 2.00000000e+00,  5.00000000e+00,  7.00000000e+00, ...,\n         1.00000000e+00,  9.74183408e-01,  4.00000000e+01]])</pre> <p>Note that the label encoding starts from <code>1</code>. This is because it is convenient to leave <code>0</code> for padding, i.e. unknown categories. Let's take from example the first entry</p> In\u00a0[12]: Copied! <pre>X_tab[0]\n</pre> X_tab[0] Out[12]: <pre>array([ 1.        ,  1.        ,  1.        ,  1.        ,  1.        ,\n       -0.99512893, 40.        ])</pre> In\u00a0[13]: Copied! <pre>tab_preprocessor.inverse_transform(X_tab[:1])\n</pre> tab_preprocessor.inverse_transform(X_tab[:1]) Out[13]: education relationship workclass occupation native-country age hours-per-week 0 11th Own-child Private Machine-op-inspct United-States 25.0 40.0 <p>The <code>TabPreprocessor</code> will have a series of useful attributes that can later be used when instantiating the different Tabular Models, such us for example, the column indexes (used to slice the tensors, internally in the models) or the categorical embeddings set up</p> In\u00a0[14]: Copied! <pre>tab_preprocessor.column_idx\n</pre> tab_preprocessor.column_idx Out[14]: <pre>{'education': 0,\n 'relationship': 1,\n 'workclass': 2,\n 'occupation': 3,\n 'native-country': 4,\n 'age': 5,\n 'hours-per-week': 6}</pre> In\u00a0[15]: Copied! <pre># column name, num unique, embedding dim\ntab_preprocessor.cat_embed_input\n</pre> # column name, num unique, embedding dim tab_preprocessor.cat_embed_input Out[15]: <pre>[('education', 16, 10),\n ('relationship', 6, 8),\n ('workclass', 9, 10),\n ('occupation', 15, 10),\n ('native-country', 42, 10)]</pre> <p>As I mentioned, there is more one can do, such as for example, quantize (or bucketize) the continuous cols. For this we could use the <code>quantization_setup</code> param. This parameter accepts a number of different inputs and uses <code>pd.cut</code> under the hood to quantize the continuous cols. For more info, please, read the docs. Let's use it here to quantize \"age\" and \"hours-per-week\" in 4 and 5 \"buckets\" respectively</p> In\u00a0[16]: Copied! <pre>quantization_setup = {\n    \"age\": 4,\n    \"hours-per-week\": 5,\n}  # you can also pass a list of floats with the boundaries if you wanted\nquant_tab_preprocessor = TabPreprocessor(\n    cat_embed_cols=cat_embed_cols,\n    continuous_cols=continuous_cols,\n    quantization_setup=quantization_setup,\n)\nqX_tab = quant_tab_preprocessor.fit_transform(df)\n</pre> quantization_setup = {     \"age\": 4,     \"hours-per-week\": 5, }  # you can also pass a list of floats with the boundaries if you wanted quant_tab_preprocessor = TabPreprocessor(     cat_embed_cols=cat_embed_cols,     continuous_cols=continuous_cols,     quantization_setup=quantization_setup, ) qX_tab = quant_tab_preprocessor.fit_transform(df) <pre>/Users/javierrodriguezzaurin/Projects/pytorch-widedeep/pytorch_widedeep/preprocessing/tab_preprocessor.py:358: UserWarning: Continuous columns will not be normalised\n  warnings.warn(\"Continuous columns will not be normalised\")\n</pre> In\u00a0[17]: Copied! <pre>qX_tab\n</pre> qX_tab Out[17]: <pre>array([[1, 1, 1, ..., 1, 1, 2],\n       [2, 2, 1, ..., 1, 2, 3],\n       [3, 2, 2, ..., 1, 1, 2],\n       ...,\n       [2, 4, 1, ..., 1, 3, 2],\n       [2, 1, 1, ..., 1, 1, 1],\n       [2, 5, 7, ..., 1, 2, 2]])</pre> <p>Note that the continuous columns that have been bucketised into quantiles are treated as any other categorical column</p> In\u00a0[18]: Copied! <pre>quant_tab_preprocessor.cat_embed_input\n</pre> quant_tab_preprocessor.cat_embed_input Out[18]: <pre>[('education', 16, 10),\n ('relationship', 6, 8),\n ('workclass', 9, 10),\n ('occupation', 15, 10),\n ('native-country', 42, 10),\n ('age', 4, 4),\n ('hours-per-week', 5, 4)]</pre> <p>Where the column 'age' has now 4 categories, which will be encoded using embeddings of 4 dims. Note that, as any other categorical columns, the categorical \"counter\" starts with 1. This is because all incoming values that are lower/higher than the existing lowest/highest value in the train (or already seen) dataset, will be encoded as 0.</p> In\u00a0[19]: Copied! <pre>np.unique(qX_tab[:, quant_tab_preprocessor.column_idx[\"age\"]])\n</pre> np.unique(qX_tab[:, quant_tab_preprocessor.column_idx[\"age\"]]) Out[19]: <pre>array([1, 2, 3, 4])</pre> <p>Finally, if we now wanted to <code>inverse_transform</code> the transformed array into the original dataframe, we could still do it, but the continuous, bucketized columns will be transformed back to the middle of their quantile/bucket range</p> In\u00a0[20]: Copied! <pre>df_decoded = quant_tab_preprocessor.inverse_transform(qX_tab)\n</pre> df_decoded = quant_tab_preprocessor.inverse_transform(qX_tab) <pre>Note that quantized cols will be turned into the mid point of the corresponding bin\n</pre> In\u00a0[21]: Copied! <pre>df.head(2)\n</pre> df.head(2) Out[21]: age workclass fnlwgt education educational-num marital-status occupation relationship race gender capital-gain capital-loss hours-per-week native-country income 0 25 Private 226802 11th 7 Never-married Machine-op-inspct Own-child Black Male 0 0 40 United-States &lt;=50K 1 38 Private 89814 HS-grad 9 Married-civ-spouse Farming-fishing Husband White Male 0 0 50 United-States &lt;=50K In\u00a0[22]: Copied! <pre>df_decoded.head(2)\n</pre> df_decoded.head(2) Out[22]: education relationship workclass occupation native-country age hours-per-week 0 11th Own-child Private Machine-op-inspct United-States 26.0885 30.4 1 HS-grad Husband Private Farming-fishing United-States 44.3750 50.0 <p>there is one final comment to make regarding to the <code>inverse_transform</code> functionality. As we mentioned before, the encoding <code>0</code> is reserved for values that fall outside the range covered by the data we used to run the <code>fit</code> method. For example</p> In\u00a0[23]: Copied! <pre>df.age.min(), df.age.max()\n</pre> df.age.min(), df.age.max() Out[23]: <pre>(17, 90)</pre> <p>All future age values outside that range will be encoded as 0 and decoded as <code>NaN</code></p> In\u00a0[24]: Copied! <pre>tmp_df = df.head(1).copy()\ntmp_df.loc[:, \"age\"] = 5\n</pre> tmp_df = df.head(1).copy() tmp_df.loc[:, \"age\"] = 5 In\u00a0[25]: Copied! <pre>tmp_df\n</pre> tmp_df Out[25]: age workclass fnlwgt education educational-num marital-status occupation relationship race gender capital-gain capital-loss hours-per-week native-country income 0 5 Private 226802 11th 7 Never-married Machine-op-inspct Own-child Black Male 0 0 40 United-States &lt;=50K In\u00a0[26]: Copied! <pre># quant_tab_preprocessor has already been fitted with a data that has an age range between 17 and 90\ntmp_qX_tab = quant_tab_preprocessor.transform(tmp_df)\n</pre> # quant_tab_preprocessor has already been fitted with a data that has an age range between 17 and 90 tmp_qX_tab = quant_tab_preprocessor.transform(tmp_df) In\u00a0[27]: Copied! <pre>tmp_qX_tab\n</pre> tmp_qX_tab Out[27]: <pre>array([[1, 1, 1, 1, 1, 0, 2]])</pre> In\u00a0[28]: Copied! <pre>quant_tab_preprocessor.inverse_transform(tmp_qX_tab)\n</pre> quant_tab_preprocessor.inverse_transform(tmp_qX_tab) <pre>Note that quantized cols will be turned into the mid point of the corresponding bin\n</pre> Out[28]: education relationship workclass occupation native-country age hours-per-week 0 11th Own-child Private Machine-op-inspct United-States NaN 30.4 In\u00a0[29]: Copied! <pre>from pytorch_widedeep.preprocessing import TextPreprocessor\n</pre> from pytorch_widedeep.preprocessing import TextPreprocessor In\u00a0[30]: Copied! <pre># The airbnb dataset, which you could get from here:\n# http://insideairbnb.com/get-the-data.html, is too big to be included in\n# our datasets module (when including images). Therefore, go there,\n# download it, and use the download_images.py script to get the images\n# and the airbnb_data_processing.py to process the data. We'll find\n# better datasets in the future ;). Note that here we are only using a\n# small sample to illustrate the use, so PLEASE ignore the results, just\n# focus on usage\ndf = pd.read_csv(\"../tmp_data/airbnb/airbnb_sample.csv\")\n</pre> # The airbnb dataset, which you could get from here: # http://insideairbnb.com/get-the-data.html, is too big to be included in # our datasets module (when including images). Therefore, go there, # download it, and use the download_images.py script to get the images # and the airbnb_data_processing.py to process the data. We'll find # better datasets in the future ;). Note that here we are only using a # small sample to illustrate the use, so PLEASE ignore the results, just # focus on usage df = pd.read_csv(\"../tmp_data/airbnb/airbnb_sample.csv\") In\u00a0[31]: Copied! <pre>texts = df.description.tolist()\ntexts[:2]\n</pre> texts = df.description.tolist() texts[:2] Out[31]: <pre>[\"My bright double bedroom with a large window has a relaxed feeling! It comfortably fits one or two and is centrally located just two blocks from Finsbury Park. Enjoy great restaurants in the area and easy access to easy transport tubes, trains and buses. Babies and children of all ages are welcome. Hello Everyone, I'm offering my lovely double bedroom in Finsbury Park area (zone 2) for let in a shared apartment.  You will share the apartment with me and it is fully furnished with a self catering kitchen. Two people can easily sleep well as the room has a queen size bed. I also have a travel cot for a baby for guest with small children.  I will require a deposit up front as a security gesture on both our parts and will be given back to you when you return the keys.  I trust anyone who will be responding to this add would treat my home with care and respect .  Best Wishes  Alina Guest will have access to the self catering kitchen and bathroom. There is the flat is equipped wifi internet,\",\n \"Lots of windows and light.  St Luke's Gardens are at the end of the block, and the river not too far the other way. Ten minutes walk if you go slowly. Buses to everywhere round the corner and shops, restaurants, pubs, the cinema and Waitrose . Bright Chelsea Apartment  This is a bright one bedroom ground floor apartment in an interesting listed building. There is one double bedroom and a living room/kitchen The apartment has a full  bathroom and the kitchen is fully equipped. Two wardrobes are available exclusively for guests and bedside tables and two long drawers. This sunny convenient compact flat is just around the corner from the Waitrose supermarket and all sorts of shops, cinemas, restaurants and pubs.  This is a lovely part of London. There is a fun farmers market in the King's Road at the weekend.  Buses to everywhere are just round the corner, and two underground stations are within ten minutes walk. There is a very nice pub round by St. Luke's gardens, 4 mins slow walk, the \"]</pre> In\u00a0[32]: Copied! <pre>text_preprocessor = TextPreprocessor(text_col=\"description\")\nX_text = text_preprocessor.fit_transform(df)\n# From here on, any new observation can be prepared by simply running `.transform`\n# new_X_text = text_preprocessor.transform(new_df)\n</pre> text_preprocessor = TextPreprocessor(text_col=\"description\") X_text = text_preprocessor.fit_transform(df) # From here on, any new observation can be prepared by simply running `.transform` # new_X_text = text_preprocessor.transform(new_df) <pre>The vocabulary contains 2192 tokens\n</pre> In\u00a0[33]: Copied! <pre>print(X_text[0])\n</pre> print(X_text[0]) <pre>[  29   48   37  367  818   17  910   17  177   15  122  349   53  879\n 1174  126  393   40  911    0   23  228   71  819    9   53   55 1380\n  225   11   18  308   18 1564   10  755    0  942  239   53   55    0\n   11   36 1013  277 1974   70   62   15 1475    9  943    5  251    5\n    0    5    0    5  177   53   37   75   11   10  294  726   32    9\n   42    5   25   12   10   22   12  136  100  145]\n</pre> In\u00a0[34]: Copied! <pre>from pytorch_widedeep.preprocessing import ImagePreprocessor\n</pre> from pytorch_widedeep.preprocessing import ImagePreprocessor In\u00a0[35]: Copied! <pre>image_preprocessor = wd.preprocessing.ImagePreprocessor(\n    img_col=\"id\", img_path=\"../tmp_data/airbnb/property_picture/\"\n)\nX_images = image_preprocessor.fit_transform(df)\n# From here on, any new observation can be prepared by simply running `.transform`\n# new_X_images = image_preprocessor.transform(new_df)\n</pre> image_preprocessor = wd.preprocessing.ImagePreprocessor(     img_col=\"id\", img_path=\"../tmp_data/airbnb/property_picture/\" ) X_images = image_preprocessor.fit_transform(df) # From here on, any new observation can be prepared by simply running `.transform` # new_X_images = image_preprocessor.transform(new_df) <pre>Reading Images from ../tmp_data/airbnb/property_picture/\nResizing\n</pre> <pre>100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 1001/1001 [00:01&lt;00:00, 667.89it/s]\n</pre> <pre>Computing normalisation metrics\n</pre> In\u00a0[36]: Copied! <pre>X_images[0].shape\n</pre> X_images[0].shape Out[36]: <pre>(224, 224, 3)</pre>"},{"location":"examples/01_preprocessors_and_utils.html#processors-and-utils","title":"Processors and Utils\u00b6","text":"<p>Description of the main tools and utilities that one needs to prepare the data for a <code>WideDeep</code> model constructor.</p>"},{"location":"examples/01_preprocessors_and_utils.html#the-preprocessing-module","title":"The <code>preprocessing</code>  module\u00b6","text":"<p>There are 4 preprocessors, corresponding to 4 main components of the <code>WideDeep</code> model. These are</p> <ul> <li><code>WidePreprocessor</code></li> <li><code>TabPreprocessor</code></li> <li><code>TextPreprocessor</code></li> <li><code>ImagePreprocessor</code></li> </ul> <p>Behind the scenes, these preprocessors use a series of helper funcions and classes that are in the <code>utils</code> module. If you were interested please go and have a look to the documentation</p>"},{"location":"examples/01_preprocessors_and_utils.html#1-widepreprocessor","title":"1. WidePreprocessor\u00b6","text":"<p>The <code>wide</code> component of the model is a linear model that in principle, could be implemented as a linear layer receiving the result of on one-hot encoding categorical columns. However, this is not memory efficient. Therefore, we implement a liner layer as an Embedding layer plus a bias. I will explain in a bit more detail later.</p> <p>With that in mind, <code>WidePreprocessor</code> simply encodes the categories numerically so that they are the indexes of the lookup table that is an Embedding layer.</p>"},{"location":"examples/01_preprocessors_and_utils.html#2-tabpreprocessor","title":"2. TabPreprocessor\u00b6","text":"<p>The <code>TabPreprocessor</code> has a lot of different functionalities. Let's explore some of them in detail. In its basic use, the <code>TabPreprocessor</code> simply label encodes the categorical columns and normalises the numerical ones (unless otherwised specified).</p>"},{"location":"examples/01_preprocessors_and_utils.html#3-textpreprocessor","title":"3. TextPreprocessor\u00b6","text":"<p>This preprocessor returns the tokenised, padded sequences that will be directly fed to the stack of LSTMs.</p>"},{"location":"examples/01_preprocessors_and_utils.html#4-imagepreprocessor","title":"4. ImagePreprocessor\u00b6","text":"<p><code>ImagePreprocessor</code> simply resizes the images, being aware of the aspect ratio.</p>"},{"location":"examples/02_model_components.html","title":"02_model_components","text":"In\u00a0[1]: Copied! <pre>import torch\nimport pandas as pd\nimport numpy as np\n\nfrom torch import nn\n</pre> import torch import pandas as pd import numpy as np  from torch import nn In\u00a0[2]: Copied! <pre>df = pd.DataFrame({\"color\": [\"r\", \"b\", \"g\"], \"size\": [\"s\", \"n\", \"l\"]})\ndf.head()\n</pre> df = pd.DataFrame({\"color\": [\"r\", \"b\", \"g\"], \"size\": [\"s\", \"n\", \"l\"]}) df.head() Out[2]: color size 0 r s 1 b n 2 g l <p>one hot encoded, the first observation would be</p> In\u00a0[3]: Copied! <pre>obs_0_oh = (np.array([1.0, 0.0, 0.0, 1.0, 0.0, 0.0])).astype(\"float32\")\n</pre> obs_0_oh = (np.array([1.0, 0.0, 0.0, 1.0, 0.0, 0.0])).astype(\"float32\") <p>if we simply numerically encode (label encode or <code>le</code>) the values:</p> In\u00a0[4]: Copied! <pre>obs_0_le = (np.array([0, 3])).astype(\"int64\")\n</pre> obs_0_le = (np.array([0, 3])).astype(\"int64\") <p>Note that in the functioning implementation of the package we start from 1, saving 0 for padding, i.e. unseen values.</p> <p>Now, let's see if the two implementations are equivalent</p> In\u00a0[5]: Copied! <pre># we have 6 different values. Let's assume we are performing a regression, so pred_dim = 1\nlin = nn.Linear(6, 1)\n</pre> # we have 6 different values. Let's assume we are performing a regression, so pred_dim = 1 lin = nn.Linear(6, 1) In\u00a0[6]: Copied! <pre>emb = nn.Embedding(6, 1)\nemb.weight = nn.Parameter(lin.weight.reshape_as(emb.weight))\n</pre> emb = nn.Embedding(6, 1) emb.weight = nn.Parameter(lin.weight.reshape_as(emb.weight)) In\u00a0[7]: Copied! <pre>lin(torch.tensor(obs_0_oh))\n</pre> lin(torch.tensor(obs_0_oh)) Out[7]: <pre>tensor([-0.5181], grad_fn=&lt;ViewBackward0&gt;)</pre> In\u00a0[8]: Copied! <pre>emb(torch.tensor(obs_0_le)).sum() + lin.bias\n</pre> emb(torch.tensor(obs_0_le)).sum() + lin.bias Out[8]: <pre>tensor([-0.5181], grad_fn=&lt;AddBackward0&gt;)</pre> <p>And this is precisely how the linear model <code>Wide</code> is implemented</p> In\u00a0[9]: Copied! <pre>from pytorch_widedeep.models import Wide\n</pre> from pytorch_widedeep.models import Wide <pre>/Users/javierrodriguezzaurin/.pyenv/versions/3.10.13/envs/widedeep310/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n  from .autonotebook import tqdm as notebook_tqdm\n</pre> In\u00a0[10]: Copied! <pre># ?Wide\n</pre> # ?Wide In\u00a0[11]: Copied! <pre>wide = Wide(input_dim=10, pred_dim=1)\nwide\n</pre> wide = Wide(input_dim=10, pred_dim=1) wide Out[11]: <pre>Wide(\n  (wide_linear): Embedding(11, 1, padding_idx=0)\n)</pre> <p>Note that even though the input dim is 10, the Embedding layer has 11 weights. Again, this is because we save <code>0</code> for padding, which is used for unseen values during the encoding process.</p> <p>As I mentioned, <code>deeptabular</code> has enough complexity on its own and it will be described in a separated notebook. Let's then jump to <code>deeptext</code>.</p> In\u00a0[12]: Copied! <pre>from pytorch_widedeep.preprocessing import TabPreprocessor\nfrom pytorch_widedeep.models import TabMlp\n</pre> from pytorch_widedeep.preprocessing import TabPreprocessor from pytorch_widedeep.models import TabMlp In\u00a0[13]: Copied! <pre>data = {\n    \"cat1\": np.random.choice([\"A\", \"B\", \"C\"], size=20),\n    \"cat2\": np.random.choice([\"X\", \"Y\"], size=20),\n    \"cont1\": np.random.rand(20),\n    \"cont2\": np.random.rand(20),\n}\n\ndf = pd.DataFrame(data)\n</pre> data = {     \"cat1\": np.random.choice([\"A\", \"B\", \"C\"], size=20),     \"cat2\": np.random.choice([\"X\", \"Y\"], size=20),     \"cont1\": np.random.rand(20),     \"cont2\": np.random.rand(20), }  df = pd.DataFrame(data) In\u00a0[14]: Copied! <pre>df.head()\n</pre> df.head() Out[14]: cat1 cat2 cont1 cont2 0 A Y 0.789347 0.561789 1 C X 0.050822 0.061538 2 A Y 0.863784 0.241967 3 C X 0.917848 0.644658 4 C Y 0.042328 0.417303 In\u00a0[15]: Copied! <pre># see the docs for details on all params/options\ntab_preprocessor = TabPreprocessor(\n    cat_embed_cols=[\"cat1\", \"cat2\"],\n    continuous_cols=[\"cont1\", \"cont2\"],\n    embedding_rule=\"fastai\",\n)\n</pre> # see the docs for details on all params/options tab_preprocessor = TabPreprocessor(     cat_embed_cols=[\"cat1\", \"cat2\"],     continuous_cols=[\"cont1\", \"cont2\"],     embedding_rule=\"fastai\", ) In\u00a0[16]: Copied! <pre>X_tab = tab_preprocessor.fit_transform(df)\n</pre> X_tab = tab_preprocessor.fit_transform(df) <pre>/Users/javierrodriguezzaurin/Projects/pytorch-widedeep/pytorch_widedeep/preprocessing/tab_preprocessor.py:358: UserWarning: Continuous columns will not be normalised\n  warnings.warn(\"Continuous columns will not be normalised\")\n</pre> In\u00a0[17]: Copied! <pre># toy example just to build a model.\ntabmlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=tab_preprocessor.continuous_cols,\n    embed_continuous_method=\"standard\",\n    cont_embed_dim=4,\n    mlp_hidden_dims=[8, 4],\n    mlp_linear_first=True,\n)\ntabmlp\n</pre> # toy example just to build a model. tabmlp = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     continuous_cols=tab_preprocessor.continuous_cols,     embed_continuous_method=\"standard\",     cont_embed_dim=4,     mlp_hidden_dims=[8, 4],     mlp_linear_first=True, ) tabmlp Out[17]: <pre>TabMlp(\n  (cat_embed): DiffSizeCatEmbeddings(\n    (embed_layers): ModuleDict(\n      (emb_layer_cat1): Embedding(4, 3, padding_idx=0)\n      (emb_layer_cat2): Embedding(3, 2, padding_idx=0)\n    )\n    (embedding_dropout): Dropout(p=0.0, inplace=False)\n  )\n  (cont_norm): Identity()\n  (cont_embed): ContEmbeddings(\n    INFO: [ContLinear = weight(n_cont_cols, embed_dim) + bias(n_cont_cols, embed_dim)]\n    (linear): ContLinear(n_cont_cols=2, embed_dim=4, embed_dropout=0.0)\n    (dropout): Dropout(p=0.0, inplace=False)\n  )\n  (encoder): MLP(\n    (mlp): Sequential(\n      (dense_layer_0): Sequential(\n        (0): Linear(in_features=13, out_features=8, bias=True)\n        (1): ReLU(inplace=True)\n        (2): Dropout(p=0.1, inplace=False)\n      )\n      (dense_layer_1): Sequential(\n        (0): Linear(in_features=8, out_features=4, bias=True)\n        (1): ReLU(inplace=True)\n        (2): Dropout(p=0.1, inplace=False)\n      )\n    )\n  )\n)</pre> <p>Lets describe a bit the model: first we have what we call a <code>DiffSizeCatEmbeddings</code>, where categorical columns with different number of unique categories will be encoded with embeddings of different dimensions. Then the continuous columns will not be normalised (the normalised layer is just the identity) and they will be embedded via a \"standard\" method, using a so-called <code>ContLinear</code> layer. This layer displays some <code>INFO</code> that tells us what it is (<code>ContLinear = weight(n_cont_cols, embed_dim) + bias(n_cont_cols, embed_dim)]</code>). There are two other options available to embed the continuous cols based on the paper On Embeddings for Numerical Features in Tabular Deep Learning. These are <code>PieceWise</code> and <code>Periodic</code> and all available via the <code>embed_continuous_method</code> param, which can adopt values <code>\"standard\", \"piecewise\"</code> and <code>\"periodic\"</code>. The embedded categorical and continuous columns will be then concatenated ($3 + 2 + (4 * 2) = 13$ input dims) and passed to an MLP.</p> In\u00a0[18]: Copied! <pre>from pytorch_widedeep.models import BasicRNN\n</pre> from pytorch_widedeep.models import BasicRNN In\u00a0[19]: Copied! <pre>basic_rnn = BasicRNN(vocab_size=4, hidden_dim=4, n_layers=1, padding_idx=0, embed_dim=4)\n</pre> basic_rnn = BasicRNN(vocab_size=4, hidden_dim=4, n_layers=1, padding_idx=0, embed_dim=4) <pre>/Users/javierrodriguezzaurin/.pyenv/versions/3.10.13/envs/widedeep310/lib/python3.10/site-packages/torch/nn/modules/rnn.py:82: UserWarning: dropout option adds dropout after all but last recurrent layer, so non-zero dropout expects num_layers greater than 1, but got dropout=0.1 and num_layers=1\n  warnings.warn(\"dropout option adds dropout after all but last \"\n</pre> In\u00a0[20]: Copied! <pre>basic_rnn\n</pre> basic_rnn Out[20]: <pre>BasicRNN(\n  (word_embed): Embedding(4, 4, padding_idx=0)\n  (rnn): LSTM(4, 4, batch_first=True, dropout=0.1)\n  (rnn_mlp): Identity()\n)</pre> <p>You could, if you wanted, add a Fully Connected Head (FC-Head) on top of it</p> In\u00a0[21]: Copied! <pre>from pytorch_widedeep.models import Vision\n</pre> from pytorch_widedeep.models import Vision In\u00a0[22]: Copied! <pre>resnet = Vision(pretrained_model_setup=\"resnet18\", n_trainable=0)\n</pre> resnet = Vision(pretrained_model_setup=\"resnet18\", n_trainable=0) In\u00a0[23]: Copied! <pre>resnet\n</pre> resnet Out[23]: <pre>Vision(\n  (features): Sequential(\n    (0): Conv2d(3, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False)\n    (1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n    (2): ReLU(inplace=True)\n    (3): MaxPool2d(kernel_size=3, stride=2, padding=1, dilation=1, ceil_mode=False)\n    (4): Sequential(\n      (0): BasicBlock(\n        (conv1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n        (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (relu): ReLU(inplace=True)\n        (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n        (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n      )\n      (1): BasicBlock(\n        (conv1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n        (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (relu): ReLU(inplace=True)\n        (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n        (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n      )\n    )\n    (5): Sequential(\n      (0): BasicBlock(\n        (conv1): Conv2d(64, 128, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n        (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (relu): ReLU(inplace=True)\n        (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n        (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (downsample): Sequential(\n          (0): Conv2d(64, 128, kernel_size=(1, 1), stride=(2, 2), bias=False)\n          (1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        )\n      )\n      (1): BasicBlock(\n        (conv1): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n        (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (relu): ReLU(inplace=True)\n        (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n        (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n      )\n    )\n    (6): Sequential(\n      (0): BasicBlock(\n        (conv1): Conv2d(128, 256, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n        (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (relu): ReLU(inplace=True)\n        (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n        (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (downsample): Sequential(\n          (0): Conv2d(128, 256, kernel_size=(1, 1), stride=(2, 2), bias=False)\n          (1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        )\n      )\n      (1): BasicBlock(\n        (conv1): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n        (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (relu): ReLU(inplace=True)\n        (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n        (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n      )\n    )\n    (7): Sequential(\n      (0): BasicBlock(\n        (conv1): Conv2d(256, 512, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n        (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (relu): ReLU(inplace=True)\n        (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n        (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (downsample): Sequential(\n          (0): Conv2d(256, 512, kernel_size=(1, 1), stride=(2, 2), bias=False)\n          (1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        )\n      )\n      (1): BasicBlock(\n        (conv1): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n        (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (relu): ReLU(inplace=True)\n        (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n        (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n      )\n    )\n    (8): AdaptiveAvgPool2d(output_size=(1, 1))\n  )\n)</pre>"},{"location":"examples/02_model_components.html#model-components","title":"Model Components\u00b6","text":"<p>The main components of a <code>WideDeep</code> (i.e. Multimodal) model are tabular data, text and images, which are feed into the model via so called <code>wide</code>, <code>deeptabular</code>, <code>deeptext</code> and <code>deepimage</code> model components</p>"},{"location":"examples/02_model_components.html#1-wide","title":"1. <code>wide</code>\u00b6","text":"<p>The <code>wide</code> component is a Linear layer \"plugged\" into the output neuron(s). Here, the non-linearities are captured via crossed columns. Crossed columns are, quoting directly the paper: \"For binary features, a cross-product transformation (e.g., \u201cAND(gender=female, language=en)\u201d) is 1 if and only if the constituent features (\u201cgender=female\u201d and \u201clanguage=en\u201d) are all 1, and 0 otherwise\".</p> <p>The only particularity of our implementation is that we have implemented the linear layer via an Embedding layer plus a bias. While the implementations are equivalent, the latter is faster and far more memory efficient, since we do not need to one hot encode the categorical features.</p> <p>Let's assume we the following dataset:</p>"},{"location":"examples/02_model_components.html#2-deeptabular","title":"2. <code>deeptabular</code>\u00b6","text":"<p>The <code>deeptabular</code> model alone is what normally would be referred as Deep Learning for tabular data. As mentioned a number of times throughout the library, each component can be used independently. Therefore, if you wanted to use any of the models below alone, it is perfectly possible. There are just a couple of simple requirement that will be covered in a later notebook.</p> <p>By the time of writing, there are a number of models available in <code>pytorch-widedeep</code> to do DL for tabular data. These are:</p> <ol> <li><code>TabMlp</code></li> <li><code>ContextAttentionMLP</code></li> <li><code>SelfAttentionMLP</code></li> <li><code>TabResnet</code></li> <li><code>Tabnet</code></li> <li><code>TabTransformer</code></li> <li><code>FT-Tabransformer</code></li> <li><code>SAINT</code></li> <li><code>TabFastFormer</code></li> <li><code>TabPerceiver</code></li> </ol> <p>Let's have a look to one of them. For more information on each of these models, please, have a look to the documentation</p>"},{"location":"examples/02_model_components.html#3-deeptext","title":"3. <code>deeptext</code>\u00b6","text":"<p>At the time of writing, <code>pytorch-widedeep</code> offers three models that can be passed to <code>WideDeep</code> as the <code>deeptext</code> component. These are:</p> <ol> <li>BasicRNN</li> <li>AttentiveRNN</li> <li>StackedAttentiveRNN</li> </ol> <p>For details on each of these models, please, have a look to the documentation of the package.</p> <p>We will soon integrate with Hugginface, but let me insist. It is perfectly possible to use custom models for each component, please, have a look to the corresponding notebook. In general, simply, build them and pass them as the corresponding parameters. Note that the custom models MUST return a last layer of activations (i.e. not the final prediction) so that  these activations are collected by <code>WideDeep</code> and combined accordingly. In  addition, the models MUST also contain an attribute <code>output_dim</code> with the size of these last layers of activations.</p> <p>Let's have a look to the <code>BasicRNN</code> model</p>"},{"location":"examples/02_model_components.html#4-deepimage","title":"4. <code>deepimage</code>\u00b6","text":"<p>At the time of writing <code>pytorch-widedeep</code> is integrated with torchvision via the <code>Vision</code> class. This means that the it is possible to use a variant of the following architectures:</p> <ol> <li>resnet</li> <li>shufflenet</li> <li>resnext</li> <li>wide_resnet</li> <li>regnet</li> <li>densenet</li> <li>mobilenet</li> <li>mnasnet</li> <li>efficientnet</li> <li>squeezenet</li> </ol> <p>The user can choose which layers will be trainable. Alternatively, in none of these architectures is useful, one could use a simple, fully trained CNN (please see the package documentation) or pass a custom model.</p> <p>let's have a look</p>"},{"location":"examples/03_binary_classification_with_defaults.html","title":"03_binary_classification_with_defaults","text":"In\u00a0[1]: Copied! <pre>import numpy as np\nimport pandas as pd\nimport torch\n\nfrom pytorch_widedeep.preprocessing import WidePreprocessor, TabPreprocessor\nfrom pytorch_widedeep.training import Trainer\nfrom pytorch_widedeep.models import Wide, TabMlp, WideDeep\nfrom pytorch_widedeep.metrics import Accuracy, Precision\nfrom pytorch_widedeep.datasets import load_adult\n</pre> import numpy as np import pandas as pd import torch  from pytorch_widedeep.preprocessing import WidePreprocessor, TabPreprocessor from pytorch_widedeep.training import Trainer from pytorch_widedeep.models import Wide, TabMlp, WideDeep from pytorch_widedeep.metrics import Accuracy, Precision from pytorch_widedeep.datasets import load_adult <pre>/Users/javierrodriguezzaurin/.pyenv/versions/3.10.13/envs/widedeep310/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n  from .autonotebook import tqdm as notebook_tqdm\n</pre> In\u00a0[2]: Copied! <pre>df = load_adult(as_frame=True)\ndf.head()\n</pre> df = load_adult(as_frame=True) df.head() Out[2]: age workclass fnlwgt education educational-num marital-status occupation relationship race gender capital-gain capital-loss hours-per-week native-country income 0 25 Private 226802 11th 7 Never-married Machine-op-inspct Own-child Black Male 0 0 40 United-States &lt;=50K 1 38 Private 89814 HS-grad 9 Married-civ-spouse Farming-fishing Husband White Male 0 0 50 United-States &lt;=50K 2 28 Local-gov 336951 Assoc-acdm 12 Married-civ-spouse Protective-serv Husband White Male 0 0 40 United-States &gt;50K 3 44 Private 160323 Some-college 10 Married-civ-spouse Machine-op-inspct Husband Black Male 7688 0 40 United-States &gt;50K 4 18 ? 103497 Some-college 10 Never-married ? Own-child White Female 0 0 30 United-States &lt;=50K In\u00a0[3]: Copied! <pre># For convenience, we'll replace '-' with '_'\ndf.columns = [c.replace(\"-\", \"_\") for c in df.columns]\n# binary target\ndf[\"income_label\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int)\ndf.drop(\"income\", axis=1, inplace=True)\ndf.head()\n</pre> # For convenience, we'll replace '-' with '_' df.columns = [c.replace(\"-\", \"_\") for c in df.columns] # binary target df[\"income_label\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int) df.drop(\"income\", axis=1, inplace=True) df.head() Out[3]: age workclass fnlwgt education educational_num marital_status occupation relationship race gender capital_gain capital_loss hours_per_week native_country income_label 0 25 Private 226802 11th 7 Never-married Machine-op-inspct Own-child Black Male 0 0 40 United-States 0 1 38 Private 89814 HS-grad 9 Married-civ-spouse Farming-fishing Husband White Male 0 0 50 United-States 0 2 28 Local-gov 336951 Assoc-acdm 12 Married-civ-spouse Protective-serv Husband White Male 0 0 40 United-States 1 3 44 Private 160323 Some-college 10 Married-civ-spouse Machine-op-inspct Husband Black Male 7688 0 40 United-States 1 4 18 ? 103497 Some-college 10 Never-married ? Own-child White Female 0 0 30 United-States 0 In\u00a0[4]: Copied! <pre>df.drop([\"fnlwgt\", \"educational_num\"], axis=1, inplace=True)\n</pre> df.drop([\"fnlwgt\", \"educational_num\"], axis=1, inplace=True) In\u00a0[5]: Copied! <pre># Define wide, crossed and deep tabular columns\nwide_cols = [\n    \"workclass\",\n    \"education\",\n    \"marital_status\",\n    \"occupation\",\n    \"relationship\",\n    \"race\",\n    \"gender\",\n    \"native_country\",\n]\ncrossed_cols = [(\"education\", \"occupation\"), (\"native_country\", \"occupation\")]\n</pre> # Define wide, crossed and deep tabular columns wide_cols = [     \"workclass\",     \"education\",     \"marital_status\",     \"occupation\",     \"relationship\",     \"race\",     \"gender\",     \"native_country\", ] crossed_cols = [(\"education\", \"occupation\"), (\"native_country\", \"occupation\")] In\u00a0[6]: Copied! <pre>cat_embed_cols = [\n    \"workclass\",\n    \"education\",\n    \"marital_status\",\n    \"occupation\",\n    \"relationship\",\n    \"race\",\n    \"gender\",\n    \"capital_gain\",\n    \"capital_loss\",\n    \"native_country\",\n]\ncontinuous_cols = [\"age\", \"hours_per_week\"]\n</pre> cat_embed_cols = [     \"workclass\",     \"education\",     \"marital_status\",     \"occupation\",     \"relationship\",     \"race\",     \"gender\",     \"capital_gain\",     \"capital_loss\",     \"native_country\", ] continuous_cols = [\"age\", \"hours_per_week\"] In\u00a0[7]: Copied! <pre># TARGET\ntarget_col = \"income_label\"\ntarget = df[target_col].values\n</pre> # TARGET target_col = \"income_label\" target = df[target_col].values <p>let's see what the preprocessors do</p> In\u00a0[8]: Copied! <pre># wide\nwide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols)\nX_wide = wide_preprocessor.fit_transform(df)\n</pre> # wide wide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols) X_wide = wide_preprocessor.fit_transform(df) In\u00a0[9]: Copied! <pre># # wide_preprocessor has an attribute called encoding_dict with the encoding dictionary\n# wide_preprocessor.encoding_dict\n</pre> # # wide_preprocessor has an attribute called encoding_dict with the encoding dictionary # wide_preprocessor.encoding_dict In\u00a0[10]: Copied! <pre># deeptabular\ntab_preprocessor = TabPreprocessor(\n    embed_cols=cat_embed_cols,\n    continuous_cols=continuous_cols,\n    cols_to_scale=continuous_cols,\n)\nX_tab = tab_preprocessor.fit_transform(df)\n</pre> # deeptabular tab_preprocessor = TabPreprocessor(     embed_cols=cat_embed_cols,     continuous_cols=continuous_cols,     cols_to_scale=continuous_cols, ) X_tab = tab_preprocessor.fit_transform(df) In\u00a0[11]: Copied! <pre># check the docs to understand the useful attributes that the tab_preprocessor has. For example,\n# as well as an encoding dictionary, tab_preprocessor has an attribute called cat_embed_input\n# that specifies the categortical columns that will be represented as embeddings, the number\n# of different categories per feature, and the dimension of the embeddings as defined by some\n# of the internal rules of thumb that the preprocessor has (have a look to the docs)\ntab_preprocessor.cat_embed_input\n</pre> # check the docs to understand the useful attributes that the tab_preprocessor has. For example, # as well as an encoding dictionary, tab_preprocessor has an attribute called cat_embed_input # that specifies the categortical columns that will be represented as embeddings, the number # of different categories per feature, and the dimension of the embeddings as defined by some # of the internal rules of thumb that the preprocessor has (have a look to the docs) tab_preprocessor.cat_embed_input Out[11]: <pre>[('workclass', 9, 5),\n ('education', 16, 8),\n ('marital_status', 7, 5),\n ('occupation', 15, 7),\n ('relationship', 6, 4),\n ('race', 5, 4),\n ('gender', 2, 2),\n ('capital_gain', 123, 24),\n ('capital_loss', 99, 21),\n ('native_country', 42, 13)]</pre> In\u00a0[12]: Copied! <pre>print(X_wide)\nprint(X_wide.shape)\n</pre> print(X_wide) print(X_wide.shape) <pre>[[  1  10  26 ...  61 103 328]\n [  1  11  27 ...  61 104 329]\n [  2  12  27 ...  61 105 330]\n ...\n [  1  11  28 ...  61 115 335]\n [  1  11  26 ...  61 115 335]\n [  7  11  27 ...  61 127 336]]\n(48842, 10)\n</pre> In\u00a0[13]: Copied! <pre>print(X_tab)\nprint(X_tab.shape)\n</pre> print(X_tab) print(X_tab.shape) <pre>[[ 1.          1.          1.         ...  1.         -0.99512893\n  -0.03408696]\n [ 1.          2.          2.         ...  1.         -0.04694151\n   0.77292975]\n [ 2.          3.          2.         ...  1.         -0.77631645\n  -0.03408696]\n ...\n [ 1.          2.          3.         ...  1.          1.41180837\n  -0.03408696]\n [ 1.          2.          1.         ...  1.         -1.21394141\n  -1.64812038]\n [ 7.          2.          2.         ...  1.          0.97418341\n  -0.03408696]]\n(48842, 12)\n</pre> In\u00a0[14]: Copied! <pre>wide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1)\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    cat_embed_dropout=0.1,\n    continuous_cols=continuous_cols,\n    mlp_hidden_dims=[400, 200],\n    mlp_dropout=0.5,\n    mlp_activation=\"leaky_relu\",\n)\n</pre> wide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1) tab_mlp = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     cat_embed_dropout=0.1,     continuous_cols=continuous_cols,     mlp_hidden_dims=[400, 200],     mlp_dropout=0.5,     mlp_activation=\"leaky_relu\", ) <p>Let's first find out how a linear model performs</p> In\u00a0[15]: Copied! <pre>wide\n</pre> wide Out[15]: <pre>Wide(\n  (wide_linear): Embedding(809, 1, padding_idx=0)\n)</pre> <p>Before being passed to the Trainer, the models need to be \"constructed\" with the <code>WideDeep</code> constructor class. For the particular case of the wide/linear model, not much really happens</p> In\u00a0[16]: Copied! <pre>lin_model = WideDeep(wide=wide)\n</pre> lin_model = WideDeep(wide=wide) In\u00a0[17]: Copied! <pre>lin_model\n</pre> lin_model Out[17]: <pre>WideDeep(\n  (wide): Wide(\n    (wide_linear): Embedding(809, 1, padding_idx=0)\n  )\n)</pre> In\u00a0[18]: Copied! <pre>lin_trainer = Trainer(\n    model=lin_model,\n    objective=\"binary\",\n    optimizers=torch.optim.AdamW(lin_model.parameters(), lr=0.01),\n    metrics=[Accuracy, Precision],\n)\n</pre> lin_trainer = Trainer(     model=lin_model,     objective=\"binary\",     optimizers=torch.optim.AdamW(lin_model.parameters(), lr=0.01),     metrics=[Accuracy, Precision], ) In\u00a0[19]: Copied! <pre>lin_trainer.fit(X_wide=X_wide, target=target, n_epochs=4, batch_size=128, val_split=0.2)\n</pre> lin_trainer.fit(X_wide=X_wide, target=target, n_epochs=4, batch_size=128, val_split=0.2) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 306/306 [00:02&lt;00:00, 109.04it/s, loss=0.426, metrics={'acc': 0.7983, 'prec': 0.6152}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 77/77 [00:00&lt;00:00, 102.46it/s, loss=0.366, metrics={'acc': 0.832, 'prec': 0.6916}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 306/306 [00:02&lt;00:00, 130.27it/s, loss=0.364, metrics={'acc': 0.8305, 'prec': 0.6933}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 77/77 [00:00&lt;00:00, 150.46it/s, loss=0.361, metrics={'acc': 0.8357, 'prec': 0.6982}]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 306/306 [00:02&lt;00:00, 133.19it/s, loss=0.359, metrics={'acc': 0.8329, 'prec': 0.6994}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 77/77 [00:00&lt;00:00, 145.75it/s, loss=0.361, metrics={'acc': 0.836, 'prec': 0.7009}]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 306/306 [00:02&lt;00:00, 130.91it/s, loss=0.358, metrics={'acc': 0.8333, 'prec': 0.7005}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 77/77 [00:00&lt;00:00, 155.08it/s, loss=0.361, metrics={'acc': 0.8364, 'prec': 0.702}]\n</pre> <p>Bear in mind that <code>wide</code> is a linear model where the non-linearities are captured via the crossed columns. For the crossed-columns to be effective one needs proper business knowledge. There is no magic formula to produce them</p> <p>Let's have a look to the tabular model by itself</p> In\u00a0[20]: Copied! <pre>tab_model = WideDeep(deeptabular=tab_mlp)\n</pre> tab_model = WideDeep(deeptabular=tab_mlp) In\u00a0[21]: Copied! <pre>tab_model\n</pre> tab_model Out[21]: <pre>WideDeep(\n  (deeptabular): Sequential(\n    (0): TabMlp(\n      (cat_embed): DiffSizeCatEmbeddings(\n        (embed_layers): ModuleDict(\n          (emb_layer_workclass): Embedding(10, 5, padding_idx=0)\n          (emb_layer_education): Embedding(17, 8, padding_idx=0)\n          (emb_layer_marital_status): Embedding(8, 5, padding_idx=0)\n          (emb_layer_occupation): Embedding(16, 7, padding_idx=0)\n          (emb_layer_relationship): Embedding(7, 4, padding_idx=0)\n          (emb_layer_race): Embedding(6, 4, padding_idx=0)\n          (emb_layer_gender): Embedding(3, 2, padding_idx=0)\n          (emb_layer_capital_gain): Embedding(124, 24, padding_idx=0)\n          (emb_layer_capital_loss): Embedding(100, 21, padding_idx=0)\n          (emb_layer_native_country): Embedding(43, 13, padding_idx=0)\n        )\n        (embedding_dropout): Dropout(p=0.1, inplace=False)\n      )\n      (cont_norm): Identity()\n      (encoder): MLP(\n        (mlp): Sequential(\n          (dense_layer_0): Sequential(\n            (0): Linear(in_features=95, out_features=400, bias=True)\n            (1): LeakyReLU(negative_slope=0.01, inplace=True)\n            (2): Dropout(p=0.5, inplace=False)\n          )\n          (dense_layer_1): Sequential(\n            (0): Linear(in_features=400, out_features=200, bias=True)\n            (1): LeakyReLU(negative_slope=0.01, inplace=True)\n            (2): Dropout(p=0.5, inplace=False)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=200, out_features=1, bias=True)\n  )\n)</pre> <p>You can see how the <code>WideDeep</code> class has added a final prediction layer that collects the activations from the last layer of the model and plugs them into the output neuron. If this was a multiclass classification problem, the prediction dimension (i.e. the size of that final layer) needs to be specified via the <code>pred_dim</code> when instantiating the <code>WideDeep</code> class, as we will see later</p> In\u00a0[22]: Copied! <pre>tab_trainer = Trainer(\n    model=tab_model,\n    objective=\"binary\",\n    optimizers=torch.optim.AdamW(tab_model.parameters(), lr=0.001),\n    metrics=[Accuracy, Precision],\n)\n</pre> tab_trainer = Trainer(     model=tab_model,     objective=\"binary\",     optimizers=torch.optim.AdamW(tab_model.parameters(), lr=0.001),     metrics=[Accuracy, Precision], ) In\u00a0[23]: Copied! <pre>tab_trainer.fit(X_tab=X_tab, target=target, n_epochs=4, batch_size=128, val_split=0.2)\n</pre> tab_trainer.fit(X_tab=X_tab, target=target, n_epochs=4, batch_size=128, val_split=0.2) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 306/306 [00:03&lt;00:00, 97.00it/s, loss=0.37, metrics={'acc': 0.8267, 'prec': 0.7037}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 77/77 [00:00&lt;00:00, 134.91it/s, loss=0.313, metrics={'acc': 0.8588, 'prec': 0.7577}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 306/306 [00:03&lt;00:00, 86.86it/s, loss=0.319, metrics={'acc': 0.8514, 'prec': 0.761}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 77/77 [00:01&lt;00:00, 73.13it/s, loss=0.296, metrics={'acc': 0.8675, 'prec': 0.7685}]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 306/306 [00:03&lt;00:00, 79.07it/s, loss=0.305, metrics={'acc': 0.8574, 'prec': 0.7646}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 77/77 [00:00&lt;00:00, 130.11it/s, loss=0.289, metrics={'acc': 0.8696, 'prec': 0.7765}]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 306/306 [00:03&lt;00:00, 87.39it/s, loss=0.296, metrics={'acc': 0.8622, 'prec': 0.7769}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 77/77 [00:00&lt;00:00, 90.63it/s, loss=0.285, metrics={'acc': 0.8697, 'prec': 0.7741}]\n</pre> <p>The best result I ever obtained with <code>LightGBM</code> on this dataset is 0.8782...so we are pretty close.</p> <p>Let's combine the <code>wide</code> and <code>tab_mlp</code> components see if it helps</p> In\u00a0[24]: Copied! <pre>wide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1)\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    cat_embed_dropout=0.1,\n    continuous_cols=continuous_cols,\n    mlp_hidden_dims=[400, 200],\n    mlp_dropout=0.5,\n    mlp_activation=\"leaky_relu\",\n)\nwd_model = WideDeep(wide=wide, deeptabular=tab_mlp)\n</pre> wide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1) tab_mlp = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     cat_embed_dropout=0.1,     continuous_cols=continuous_cols,     mlp_hidden_dims=[400, 200],     mlp_dropout=0.5,     mlp_activation=\"leaky_relu\", ) wd_model = WideDeep(wide=wide, deeptabular=tab_mlp) In\u00a0[25]: Copied! <pre>wd_trainer = Trainer(\n    model=wd_model,\n    objective=\"binary\",\n    optimizers=torch.optim.AdamW(wd_model.parameters(), lr=0.001),\n    metrics=[Accuracy, Precision],\n)\n</pre> wd_trainer = Trainer(     model=wd_model,     objective=\"binary\",     optimizers=torch.optim.AdamW(wd_model.parameters(), lr=0.001),     metrics=[Accuracy, Precision], ) In\u00a0[26]: Copied! <pre>wd_trainer.fit(\n    X_wide=X_wide, X_tab=X_tab, target=target, n_epochs=4, batch_size=128, val_split=0.2\n)\n</pre> wd_trainer.fit(     X_wide=X_wide, X_tab=X_tab, target=target, n_epochs=4, batch_size=128, val_split=0.2 ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 306/306 [00:03&lt;00:00, 77.48it/s, loss=0.418, metrics={'acc': 0.8047, 'prec': 0.6154}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 77/77 [00:00&lt;00:00, 110.51it/s, loss=0.321, metrics={'acc': 0.8521, 'prec': 0.7059}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 306/306 [00:03&lt;00:00, 82.70it/s, loss=0.333, metrics={'acc': 0.8428, 'prec': 0.7141}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 77/77 [00:00&lt;00:00, 112.52it/s, loss=0.299, metrics={'acc': 0.866, 'prec': 0.7447}]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 306/306 [00:04&lt;00:00, 74.34it/s, loss=0.312, metrics={'acc': 0.8533, 'prec': 0.7404}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 77/77 [00:00&lt;00:00, 89.86it/s, loss=0.29, metrics={'acc': 0.8683, 'prec': 0.7496}]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 306/306 [00:04&lt;00:00, 65.32it/s, loss=0.301, metrics={'acc': 0.8591, 'prec': 0.7542}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 77/77 [00:00&lt;00:00, 86.81it/s, loss=0.286, metrics={'acc': 0.8712, 'prec': 0.7552}]\n</pre> <p>For this particular case, the combination of both did not lead to better results that using just the tab_mlp model, when using only 4 epochs.</p> <p>Note that we have use a <code>TabMlp</code> model, but we could use any other model in the library using the same syntax</p> In\u00a0[27]: Copied! <pre>from pytorch_widedeep.models import TabTransformer\n</pre> from pytorch_widedeep.models import TabTransformer <p>The parameters for the <code>TabTransformer</code> are this</p> <pre><code>column_idx: Dict[str, int],\ncat_embed_input: Optional[List[Tuple[str, int]]] = None,\ncat_embed_dropout: Optional[float] = None,\nuse_cat_bias: Optional[bool] = None,\ncat_embed_activation: Optional[str] = None,\nshared_embed: Optional[bool] = None,\nadd_shared_embed: Optional[bool] = None,\nfrac_shared_embed: Optional[float] = None,\ncontinuous_cols: Optional[List[str]] = None,\ncont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\nembed_continuous: Optional[bool] = None,\nembed_continuous_method: Optional[Literal[\"standard\", \"piecewise\", \"periodic\"]] = None,\ncont_embed_dropout: Optional[float] = None,\ncont_embed_activation: Optional[str] = None,\nquantization_setup: Optional[Dict[str, List[float]]] = None,\nn_frequencies: Optional[int] = None,\nsigma: Optional[float] = None,\nshare_last_layer: Optional[bool] = None,\nfull_embed_dropout: Optional[bool] = None,\ninput_dim: int = 32,\nn_heads: int = 8,\nuse_qkv_bias: bool = False,\nn_blocks: int = 4,\nattn_dropout: float = 0.2,\nff_dropout: float = 0.1,\nff_factor: int = 4,\ntransformer_activation: str = \"gelu\",\nuse_linear_attention: bool = False,\nuse_flash_attention: bool = False,\nmlp_hidden_dims: Optional[List[int]] = None,\nmlp_activation: str = \"relu\",\nmlp_dropout: float = 0.1,\nmlp_batchnorm: bool = False,\nmlp_batchnorm_last: bool = False,\nmlp_linear_first: bool = True,\n</code></pre> <p>Please, see the documentation for details on each one of them, for now let's see how one could use a <code>TabTransformer</code> model in a few lines of code</p> In\u00a0[28]: Copied! <pre>tab_transformer = TabTransformer(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    cat_embed_dropout=0.1,\n    continuous_cols=continuous_cols,\n    embed_continuous_method=\"standard\",\n    cont_norm_layer=\"layernorm\",\n    cont_embed_dropout=0.2,\n    cont_embed_activation=\"leaky_relu\",\n    n_heads=4,\n    ff_dropout=0.2,\n    mlp_dropout=0.5,\n    mlp_activation=\"leaky_relu\",\n    mlp_linear_first=\"True\",\n)\n</pre> tab_transformer = TabTransformer(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     cat_embed_dropout=0.1,     continuous_cols=continuous_cols,     embed_continuous_method=\"standard\",     cont_norm_layer=\"layernorm\",     cont_embed_dropout=0.2,     cont_embed_activation=\"leaky_relu\",     n_heads=4,     ff_dropout=0.2,     mlp_dropout=0.5,     mlp_activation=\"leaky_relu\",     mlp_linear_first=\"True\", ) In\u00a0[29]: Copied! <pre>tab_model = WideDeep(deeptabular=tab_transformer)\n</pre> tab_model = WideDeep(deeptabular=tab_transformer) In\u00a0[30]: Copied! <pre>tab_model\n</pre> tab_model Out[30]: <pre>WideDeep(\n  (deeptabular): Sequential(\n    (0): TabTransformer(\n      (cat_embed): SameSizeCatEmbeddings(\n        (embed): Embedding(325, 32, padding_idx=0)\n        (dropout): Dropout(p=0.1, inplace=False)\n      )\n      (cont_norm): LayerNorm((2,), eps=1e-05, elementwise_affine=True)\n      (cont_embed): ContEmbeddings(\n        INFO: [ContLinear = weight(n_cont_cols, embed_dim) + bias(n_cont_cols, embed_dim)]\n        (linear): ContLinear(n_cont_cols=2, embed_dim=32, embed_dropout=0.2)\n        (activation_fn): LeakyReLU(negative_slope=0.01, inplace=True)\n        (dropout): Dropout(p=0.2, inplace=False)\n      )\n      (encoder): Sequential(\n        (transformer_block0): TransformerEncoder(\n          (attn): MultiHeadedAttention(\n            (dropout): Dropout(p=0.2, inplace=False)\n            (q_proj): Linear(in_features=32, out_features=32, bias=False)\n            (kv_proj): Linear(in_features=32, out_features=64, bias=False)\n            (out_proj): Linear(in_features=32, out_features=32, bias=False)\n          )\n          (ff): FeedForward(\n            (w_1): Linear(in_features=32, out_features=128, bias=True)\n            (w_2): Linear(in_features=128, out_features=32, bias=True)\n            (dropout): Dropout(p=0.2, inplace=False)\n            (activation): GELU(approximate='none')\n          )\n          (attn_addnorm): AddNorm(\n            (dropout): Dropout(p=0.2, inplace=False)\n            (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n          )\n          (ff_addnorm): AddNorm(\n            (dropout): Dropout(p=0.2, inplace=False)\n            (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n          )\n        )\n        (transformer_block1): TransformerEncoder(\n          (attn): MultiHeadedAttention(\n            (dropout): Dropout(p=0.2, inplace=False)\n            (q_proj): Linear(in_features=32, out_features=32, bias=False)\n            (kv_proj): Linear(in_features=32, out_features=64, bias=False)\n            (out_proj): Linear(in_features=32, out_features=32, bias=False)\n          )\n          (ff): FeedForward(\n            (w_1): Linear(in_features=32, out_features=128, bias=True)\n            (w_2): Linear(in_features=128, out_features=32, bias=True)\n            (dropout): Dropout(p=0.2, inplace=False)\n            (activation): GELU(approximate='none')\n          )\n          (attn_addnorm): AddNorm(\n            (dropout): Dropout(p=0.2, inplace=False)\n            (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n          )\n          (ff_addnorm): AddNorm(\n            (dropout): Dropout(p=0.2, inplace=False)\n            (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n          )\n        )\n        (transformer_block2): TransformerEncoder(\n          (attn): MultiHeadedAttention(\n            (dropout): Dropout(p=0.2, inplace=False)\n            (q_proj): Linear(in_features=32, out_features=32, bias=False)\n            (kv_proj): Linear(in_features=32, out_features=64, bias=False)\n            (out_proj): Linear(in_features=32, out_features=32, bias=False)\n          )\n          (ff): FeedForward(\n            (w_1): Linear(in_features=32, out_features=128, bias=True)\n            (w_2): Linear(in_features=128, out_features=32, bias=True)\n            (dropout): Dropout(p=0.2, inplace=False)\n            (activation): GELU(approximate='none')\n          )\n          (attn_addnorm): AddNorm(\n            (dropout): Dropout(p=0.2, inplace=False)\n            (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n          )\n          (ff_addnorm): AddNorm(\n            (dropout): Dropout(p=0.2, inplace=False)\n            (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n          )\n        )\n        (transformer_block3): TransformerEncoder(\n          (attn): MultiHeadedAttention(\n            (dropout): Dropout(p=0.2, inplace=False)\n            (q_proj): Linear(in_features=32, out_features=32, bias=False)\n            (kv_proj): Linear(in_features=32, out_features=64, bias=False)\n            (out_proj): Linear(in_features=32, out_features=32, bias=False)\n          )\n          (ff): FeedForward(\n            (w_1): Linear(in_features=32, out_features=128, bias=True)\n            (w_2): Linear(in_features=128, out_features=32, bias=True)\n            (dropout): Dropout(p=0.2, inplace=False)\n            (activation): GELU(approximate='none')\n          )\n          (attn_addnorm): AddNorm(\n            (dropout): Dropout(p=0.2, inplace=False)\n            (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n          )\n          (ff_addnorm): AddNorm(\n            (dropout): Dropout(p=0.2, inplace=False)\n            (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=384, out_features=1, bias=True)\n  )\n)</pre> In\u00a0[31]: Copied! <pre>tab_trainer = Trainer(\n    model=tab_model,\n    objective=\"binary\",\n    optimizers=torch.optim.AdamW(tab_model.parameters(), lr=0.001),\n    metrics=[Accuracy, Precision],\n)\n</pre> tab_trainer = Trainer(     model=tab_model,     objective=\"binary\",     optimizers=torch.optim.AdamW(tab_model.parameters(), lr=0.001),     metrics=[Accuracy, Precision], ) In\u00a0[32]: Copied! <pre>tab_trainer.fit(X_tab=X_tab, target=target, n_epochs=1, batch_size=128, val_split=0.2)\n</pre> tab_trainer.fit(X_tab=X_tab, target=target, n_epochs=1, batch_size=128, val_split=0.2) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 306/306 [00:11&lt;00:00, 27.57it/s, loss=0.359, metrics={'acc': 0.8334, 'prec': 0.7082}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 77/77 [00:01&lt;00:00, 57.89it/s, loss=0.33, metrics={'acc': 0.8536, 'prec': 0.7152}]\n</pre>"},{"location":"examples/03_binary_classification_with_defaults.html#simple-binary-classification-with-defaults","title":"Simple Binary Classification with defaults\u00b6","text":"<p>In this notebook we will train a Wide and Deep model and simply a \"Deep\" model using the well known adult dataset</p>"},{"location":"examples/03_binary_classification_with_defaults.html#preparing-the-data","title":"Preparing the data\u00b6","text":""},{"location":"examples/03_binary_classification_with_defaults.html#defining-the-model","title":"Defining the model\u00b6","text":""},{"location":"examples/04_regression_with_images_and_text.html","title":"04_regression_with_images_and_text","text":"In\u00a0[1]: Copied! <pre>import numpy as np\nimport pandas as pd\nimport os\nimport torch\nfrom torchvision.transforms import ToTensor, Normalize\n\nfrom pytorch_widedeep import Trainer\nfrom pytorch_widedeep.preprocessing import (\n    WidePreprocessor,\n    TabPreprocessor,\n    TextPreprocessor,\n    ImagePreprocessor,\n)\nfrom pytorch_widedeep.models import (\n    Wide,\n    TabMlp,\n    Vision,\n    BasicRNN,\n    WideDeep,\n)\nfrom pytorch_widedeep.losses import RMSELoss\nfrom pytorch_widedeep.initializers import *\nfrom pytorch_widedeep.callbacks import *\n</pre> import numpy as np import pandas as pd import os import torch from torchvision.transforms import ToTensor, Normalize  from pytorch_widedeep import Trainer from pytorch_widedeep.preprocessing import (     WidePreprocessor,     TabPreprocessor,     TextPreprocessor,     ImagePreprocessor, ) from pytorch_widedeep.models import (     Wide,     TabMlp,     Vision,     BasicRNN,     WideDeep, ) from pytorch_widedeep.losses import RMSELoss from pytorch_widedeep.initializers import * from pytorch_widedeep.callbacks import * <pre>/Users/javierrodriguezzaurin/.pyenv/versions/3.10.13/envs/widedeep310/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n  from .autonotebook import tqdm as notebook_tqdm\n</pre> In\u00a0[2]: Copied! <pre>df = pd.read_csv(\"../tmp_data/airbnb/airbnb_sample.csv\")\ndf.head()\n</pre> df = pd.read_csv(\"../tmp_data/airbnb/airbnb_sample.csv\") df.head() Out[2]: id host_id description host_listings_count host_identity_verified neighbourhood_cleansed latitude longitude is_location_exact property_type ... amenity_wide_entrance amenity_wide_entrance_for_guests amenity_wide_entryway amenity_wide_hallways amenity_wifi amenity_window_guards amenity_wine_cooler security_deposit extra_people yield 0 13913.jpg 54730 My bright double bedroom with a large window h... 4.0 f Islington 51.56802 -0.11121 t apartment ... 1 0 0 0 1 0 0 100.0 15.0 12.00 1 15400.jpg 60302 Lots of windows and light.  St Luke's Gardens ... 1.0 t Kensington and Chelsea 51.48796 -0.16898 t apartment ... 0 0 0 0 1 0 0 150.0 0.0 109.50 2 17402.jpg 67564 Open from June 2018 after a 3-year break, we a... 19.0 t Westminster 51.52098 -0.14002 t apartment ... 0 0 0 0 1 0 0 350.0 10.0 149.65 3 24328.jpg 41759 Artist house, bright high ceiling rooms, priva... 2.0 t Wandsworth 51.47298 -0.16376 t other ... 0 0 0 0 1 0 0 250.0 0.0 215.60 4 25023.jpg 102813 Large, all comforts, 2-bed flat; first floor; ... 1.0 f Wandsworth 51.44687 -0.21874 t apartment ... 0 0 0 0 1 0 0 250.0 11.0 79.35 <p>5 rows \u00d7 223 columns</p> In\u00a0[3]: Copied! <pre># There are a number of columns that are already binary. Therefore, no need to one hot encode them\ncrossed_cols = [(\"property_type\", \"room_type\")]\nalready_dummies = [c for c in df.columns if \"amenity\" in c] + [\"has_house_rules\"]\nwide_cols = [\n    \"is_location_exact\",\n    \"property_type\",\n    \"room_type\",\n    \"host_gender\",\n    \"instant_bookable\",\n] + already_dummies\n\ncat_embed_cols = [(c, 16) for c in df.columns if \"catg\" in c] + [\n    (\"neighbourhood_cleansed\", 64),\n    (\"cancellation_policy\", 16),\n]\ncontinuous_cols = [\"latitude\", \"longitude\", \"security_deposit\", \"extra_people\"]\n\n# text and image colnames\ntext_col = \"description\"\nimg_col = \"id\"\n\n# path to pretrained word embeddings and the images\nword_vectors_path = \"../tmp_data/glove.6B/glove.6B.100d.txt\"\nimg_path = \"../tmp_data/airbnb/property_picture\"\n\n# target\ntarget_col = \"yield\"\n</pre> # There are a number of columns that are already binary. Therefore, no need to one hot encode them crossed_cols = [(\"property_type\", \"room_type\")] already_dummies = [c for c in df.columns if \"amenity\" in c] + [\"has_house_rules\"] wide_cols = [     \"is_location_exact\",     \"property_type\",     \"room_type\",     \"host_gender\",     \"instant_bookable\", ] + already_dummies  cat_embed_cols = [(c, 16) for c in df.columns if \"catg\" in c] + [     (\"neighbourhood_cleansed\", 64),     (\"cancellation_policy\", 16), ] continuous_cols = [\"latitude\", \"longitude\", \"security_deposit\", \"extra_people\"]  # text and image colnames text_col = \"description\" img_col = \"id\"  # path to pretrained word embeddings and the images word_vectors_path = \"../tmp_data/glove.6B/glove.6B.100d.txt\" img_path = \"../tmp_data/airbnb/property_picture\"  # target target_col = \"yield\" In\u00a0[4]: Copied! <pre>target = df[target_col].values\n</pre> target = df[target_col].values In\u00a0[5]: Copied! <pre>wide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols)\nX_wide = wide_preprocessor.fit_transform(df)\n</pre> wide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols) X_wide = wide_preprocessor.fit_transform(df) In\u00a0[6]: Copied! <pre>tab_preprocessor = TabPreprocessor(\n    cat_embed_cols=cat_embed_cols,\n    continuous_cols=continuous_cols,\n)\nX_tab = tab_preprocessor.fit_transform(df)\n</pre> tab_preprocessor = TabPreprocessor(     cat_embed_cols=cat_embed_cols,     continuous_cols=continuous_cols, ) X_tab = tab_preprocessor.fit_transform(df) <pre>/Users/javierrodriguezzaurin/Projects/pytorch-widedeep/pytorch_widedeep/preprocessing/tab_preprocessor.py:358: UserWarning: Continuous columns will not be normalised\n  warnings.warn(\"Continuous columns will not be normalised\")\n</pre> In\u00a0[7]: Copied! <pre>text_preprocessor = TextPreprocessor(\n    word_vectors_path=word_vectors_path, text_col=text_col\n)\nX_text = text_preprocessor.fit_transform(df)\n</pre> text_preprocessor = TextPreprocessor(     word_vectors_path=word_vectors_path, text_col=text_col ) X_text = text_preprocessor.fit_transform(df) <pre>The vocabulary contains 2192 tokens\nIndexing word vectors...\nLoaded 400000 word vectors\nPreparing embeddings matrix...\n2175 words in the vocabulary had ../tmp_data/glove.6B/glove.6B.100d.txt vectors and appear more than 5 times\n</pre> In\u00a0[8]: Copied! <pre>image_processor = ImagePreprocessor(img_col=img_col, img_path=img_path)\nX_images = image_processor.fit_transform(df)\n</pre> image_processor = ImagePreprocessor(img_col=img_col, img_path=img_path) X_images = image_processor.fit_transform(df) <pre>Reading Images from ../tmp_data/airbnb/property_picture\nResizing\n</pre> <pre>100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 1001/1001 [00:01&lt;00:00, 638.00it/s]\n</pre> <pre>Computing normalisation metrics\n</pre> In\u00a0[9]: Copied! <pre># Linear model\nwide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1)\n\n# DeepDense: 2 Dense layers\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=continuous_cols,\n    mlp_hidden_dims=[128, 64],\n    mlp_dropout=0.1,\n)\n\n# DeepText: a stack of 2 LSTMs\nbasic_rnn = BasicRNN(\n    vocab_size=len(text_preprocessor.vocab.itos),\n    embed_matrix=text_preprocessor.embedding_matrix,\n    n_layers=2,\n    hidden_dim=64,\n    rnn_dropout=0.5,\n)\n\n# Pretrained Resnet 18\nresnet = Vision(pretrained_model_setup=\"resnet18\", n_trainable=4)\n</pre> # Linear model wide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1)  # DeepDense: 2 Dense layers tab_mlp = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     continuous_cols=continuous_cols,     mlp_hidden_dims=[128, 64],     mlp_dropout=0.1, )  # DeepText: a stack of 2 LSTMs basic_rnn = BasicRNN(     vocab_size=len(text_preprocessor.vocab.itos),     embed_matrix=text_preprocessor.embedding_matrix,     n_layers=2,     hidden_dim=64,     rnn_dropout=0.5, )  # Pretrained Resnet 18 resnet = Vision(pretrained_model_setup=\"resnet18\", n_trainable=4) <p>Combine them all with the \"collector\" class <code>WideDeep</code></p> In\u00a0[10]: Copied! <pre>model = WideDeep(\n    wide=wide,\n    deeptabular=tab_mlp,\n    deeptext=basic_rnn,\n    deepimage=resnet,\n    head_hidden_dims=[256, 128],\n)\n</pre> model = WideDeep(     wide=wide,     deeptabular=tab_mlp,     deeptext=basic_rnn,     deepimage=resnet,     head_hidden_dims=[256, 128], ) In\u00a0[11]: Copied! <pre>trainer = Trainer(model, objective=\"rmse\")\n</pre> trainer = Trainer(model, objective=\"rmse\") In\u00a0[12]: Copied! <pre>trainer.fit(\n    X_wide=X_wide,\n    X_tab=X_tab,\n    X_text=X_text,\n    X_img=X_images,\n    target=target,\n    n_epochs=1,\n    batch_size=32,\n    val_split=0.2,\n)\n</pre> trainer.fit(     X_wide=X_wide,     X_tab=X_tab,     X_text=X_text,     X_img=X_images,     target=target,     n_epochs=1,     batch_size=32,     val_split=0.2, ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 25/25 [00:19&lt;00:00,  1.28it/s, loss=115]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 7/7 [00:04&lt;00:00,  1.62it/s, loss=94.1]\n</pre> <p>Both, the Text and Image components allow FC-heads on their own (have a look to the documentation).</p> <p>Now let's go \"kaggle crazy\". Let's use different optimizers, initializers and schedulers for different components. Moreover, let's use a different learning rate for different parameter groups, for the <code>deeptabular</code> component</p> In\u00a0[13]: Copied! <pre>deep_params = []\nfor childname, child in model.named_children():\n    if childname == \"deeptabular\":\n        for n, p in child.named_parameters():\n            if \"embed_layer\" in n:\n                deep_params.append({\"params\": p, \"lr\": 1e-4})\n            else:\n                deep_params.append({\"params\": p, \"lr\": 1e-3})\n</pre> deep_params = [] for childname, child in model.named_children():     if childname == \"deeptabular\":         for n, p in child.named_parameters():             if \"embed_layer\" in n:                 deep_params.append({\"params\": p, \"lr\": 1e-4})             else:                 deep_params.append({\"params\": p, \"lr\": 1e-3}) In\u00a0[14]: Copied! <pre>wide_opt = torch.optim.Adam(model.wide.parameters(), lr=0.03)\ndeep_opt = torch.optim.Adam(deep_params)\ntext_opt = torch.optim.AdamW(model.deeptext.parameters())\nimg_opt = torch.optim.AdamW(model.deepimage.parameters())\nhead_opt = torch.optim.Adam(model.deephead.parameters())\n</pre> wide_opt = torch.optim.Adam(model.wide.parameters(), lr=0.03) deep_opt = torch.optim.Adam(deep_params) text_opt = torch.optim.AdamW(model.deeptext.parameters()) img_opt = torch.optim.AdamW(model.deepimage.parameters()) head_opt = torch.optim.Adam(model.deephead.parameters()) In\u00a0[15]: Copied! <pre>wide_sch = torch.optim.lr_scheduler.StepLR(wide_opt, step_size=5)\ndeep_sch = torch.optim.lr_scheduler.MultiStepLR(deep_opt, milestones=[3, 8])\ntext_sch = torch.optim.lr_scheduler.StepLR(text_opt, step_size=5)\nimg_sch = torch.optim.lr_scheduler.MultiStepLR(deep_opt, milestones=[3, 8])\nhead_sch = torch.optim.lr_scheduler.StepLR(head_opt, step_size=5)\n</pre> wide_sch = torch.optim.lr_scheduler.StepLR(wide_opt, step_size=5) deep_sch = torch.optim.lr_scheduler.MultiStepLR(deep_opt, milestones=[3, 8]) text_sch = torch.optim.lr_scheduler.StepLR(text_opt, step_size=5) img_sch = torch.optim.lr_scheduler.MultiStepLR(deep_opt, milestones=[3, 8]) head_sch = torch.optim.lr_scheduler.StepLR(head_opt, step_size=5) In\u00a0[16]: Copied! <pre># remember, one optimizer per model components, for lr_schedures and initializers is not neccesary\noptimizers = {\n    \"wide\": wide_opt,\n    \"deeptabular\": deep_opt,\n    \"deeptext\": text_opt,\n    \"deepimage\": img_opt,\n    \"deephead\": head_opt,\n}\nschedulers = {\n    \"wide\": wide_sch,\n    \"deeptabular\": deep_sch,\n    \"deeptext\": text_sch,\n    \"deepimage\": img_sch,\n    \"deephead\": head_sch,\n}\n\n# Now...we have used pretrained word embeddings, so you do not want to\n# initialise these  embeddings. However you might still want to initialise the\n# other layers in the DeepText component. No probs, you can do that with the\n# parameter pattern and your knowledge on regular  expressions. Here we are\n# telling to the KaimingNormal initializer to NOT touch the  parameters whose\n# name contains the string word_embed.\ninitializers = {\n    \"wide\": KaimingNormal,\n    \"deeptabular\": KaimingNormal,\n    \"deeptext\": KaimingNormal(pattern=r\"^(?!.*word_embed).*$\"),\n    \"deepimage\": KaimingNormal,\n}\n\nmean = [0.406, 0.456, 0.485]  # BGR\nstd = [0.225, 0.224, 0.229]  # BGR\ntransforms = [ToTensor, Normalize(mean=mean, std=std)]\ncallbacks = [\n    LRHistory(n_epochs=10),\n    EarlyStopping,\n    ModelCheckpoint(filepath=\"model_weights/wd_out\"),\n]\n</pre> # remember, one optimizer per model components, for lr_schedures and initializers is not neccesary optimizers = {     \"wide\": wide_opt,     \"deeptabular\": deep_opt,     \"deeptext\": text_opt,     \"deepimage\": img_opt,     \"deephead\": head_opt, } schedulers = {     \"wide\": wide_sch,     \"deeptabular\": deep_sch,     \"deeptext\": text_sch,     \"deepimage\": img_sch,     \"deephead\": head_sch, }  # Now...we have used pretrained word embeddings, so you do not want to # initialise these  embeddings. However you might still want to initialise the # other layers in the DeepText component. No probs, you can do that with the # parameter pattern and your knowledge on regular  expressions. Here we are # telling to the KaimingNormal initializer to NOT touch the  parameters whose # name contains the string word_embed. initializers = {     \"wide\": KaimingNormal,     \"deeptabular\": KaimingNormal,     \"deeptext\": KaimingNormal(pattern=r\"^(?!.*word_embed).*$\"),     \"deepimage\": KaimingNormal, }  mean = [0.406, 0.456, 0.485]  # BGR std = [0.225, 0.224, 0.229]  # BGR transforms = [ToTensor, Normalize(mean=mean, std=std)] callbacks = [     LRHistory(n_epochs=10),     EarlyStopping,     ModelCheckpoint(filepath=\"model_weights/wd_out\"), ] In\u00a0[17]: Copied! <pre>trainer = Trainer(\n    model,\n    objective=\"rmse\",\n    initializers=initializers,\n    optimizers=optimizers,\n    lr_schedulers=schedulers,\n    callbacks=callbacks,\n    transforms=transforms,\n)\n</pre> trainer = Trainer(     model,     objective=\"rmse\",     initializers=initializers,     optimizers=optimizers,     lr_schedulers=schedulers,     callbacks=callbacks,     transforms=transforms, ) <pre>/Users/javierrodriguezzaurin/Projects/pytorch-widedeep/pytorch_widedeep/initializers.py:34: UserWarning: No initializer found for deephead\n  warnings.warn(\n</pre> In\u00a0[18]: Copied! <pre>trainer.fit(\n    X_wide=X_wide,\n    X_tab=X_tab,\n    X_text=X_text,\n    X_img=X_images,\n    target=target,\n    n_epochs=1,\n    batch_size=32,\n    val_split=0.2,\n)\n</pre> trainer.fit(     X_wide=X_wide,     X_tab=X_tab,     X_text=X_text,     X_img=X_images,     target=target,     n_epochs=1,     batch_size=32,     val_split=0.2, ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 25/25 [00:19&lt;00:00,  1.25it/s, loss=101]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 7/7 [00:04&lt;00:00,  1.62it/s, loss=90.6]\n</pre> <pre>Model weights after training corresponds to the those of the final epoch which might not be the best performing weights. Use the 'ModelCheckpoint' Callback to restore the best epoch weights.\n</pre> <p>we have only run one epoch, but let's check that the LRHistory callback records the lr values for each group</p> In\u00a0[19]: Copied! <pre>trainer.lr_history\n</pre> trainer.lr_history Out[19]: <pre>{'lr_wide_0': [0.03, 0.03],\n 'lr_deeptabular_0': [0.0001, 0.0001],\n 'lr_deeptabular_1': [0.0001, 0.0001],\n 'lr_deeptabular_2': [0.0001, 0.0001],\n 'lr_deeptabular_3': [0.0001, 0.0001],\n 'lr_deeptabular_4': [0.0001, 0.0001],\n 'lr_deeptabular_5': [0.0001, 0.0001],\n 'lr_deeptabular_6': [0.0001, 0.0001],\n 'lr_deeptabular_7': [0.0001, 0.0001],\n 'lr_deeptabular_8': [0.0001, 0.0001],\n 'lr_deeptabular_9': [0.001, 0.001],\n 'lr_deeptabular_10': [0.001, 0.001],\n 'lr_deeptabular_11': [0.001, 0.001],\n 'lr_deeptabular_12': [0.001, 0.001],\n 'lr_deeptext_0': [0.001, 0.001],\n 'lr_deepimage_0': [0.001, 0.001],\n 'lr_deephead_0': [0.001, 0.001]}</pre>"},{"location":"examples/04_regression_with_images_and_text.html#regression-with-images-and-text","title":"Regression with Images and Text\u00b6","text":"<p>In this notebook we will go through a series of examples on how to combine all Wide &amp; Deep components.</p> <p>To that aim I will use the Airbnb listings dataset for London, which you can download from here. I use this dataset simply because it contains tabular data, images and text.</p> <p>I have taken a sample of 1000 listings to keep the data tractable in this notebook. Also, I have preprocessed the data and prepared it for this exercise. All preprocessing steps can be found in the notebook <code>airbnb_data_preprocessing.ipynb</code> in this <code>examples</code> folder.</p>"},{"location":"examples/04_regression_with_images_and_text.html#regression-with-the-defaults","title":"Regression with the defaults\u00b6","text":"<p>The set up</p>"},{"location":"examples/04_regression_with_images_and_text.html#prepare-the-data","title":"Prepare the data\u00b6","text":"<p>I will focus here on how to prepare the data and run the model. Check notebooks 1 and 2 to see what's going on behind the scences</p> <p>Preparing the data is rather simple</p>"},{"location":"examples/04_regression_with_images_and_text.html#build-the-model-components","title":"Build the model components\u00b6","text":""},{"location":"examples/04_regression_with_images_and_text.html#build-the-trainer-and-fit","title":"Build the trainer and fit\u00b6","text":""},{"location":"examples/05_save_and_load_model_and_artifacts.html","title":"05_save_and_load_model_and_artifacts","text":"In\u00a0[1]: Copied! <pre>import pickle\nimport numpy as np\nimport pandas as pd\nimport torch\nimport shutil\n\nfrom pytorch_widedeep.preprocessing import TabPreprocessor\nfrom pytorch_widedeep.training import Trainer\nfrom pytorch_widedeep.callbacks import EarlyStopping, ModelCheckpoint, LRHistory\nfrom pytorch_widedeep.models import TabMlp, WideDeep\nfrom pytorch_widedeep.metrics import Accuracy\nfrom pytorch_widedeep.datasets import load_adult\nfrom sklearn.model_selection import train_test_split\n</pre> import pickle import numpy as np import pandas as pd import torch import shutil  from pytorch_widedeep.preprocessing import TabPreprocessor from pytorch_widedeep.training import Trainer from pytorch_widedeep.callbacks import EarlyStopping, ModelCheckpoint, LRHistory from pytorch_widedeep.models import TabMlp, WideDeep from pytorch_widedeep.metrics import Accuracy from pytorch_widedeep.datasets import load_adult from sklearn.model_selection import train_test_split <pre>/Users/javierrodriguezzaurin/.pyenv/versions/3.10.13/envs/widedeep310/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n  from .autonotebook import tqdm as notebook_tqdm\n</pre> In\u00a0[2]: Copied! <pre>df = load_adult(as_frame=True)\ndf.head()\n</pre> df = load_adult(as_frame=True) df.head() Out[2]: age workclass fnlwgt education educational-num marital-status occupation relationship race gender capital-gain capital-loss hours-per-week native-country income 0 25 Private 226802 11th 7 Never-married Machine-op-inspct Own-child Black Male 0 0 40 United-States &lt;=50K 1 38 Private 89814 HS-grad 9 Married-civ-spouse Farming-fishing Husband White Male 0 0 50 United-States &lt;=50K 2 28 Local-gov 336951 Assoc-acdm 12 Married-civ-spouse Protective-serv Husband White Male 0 0 40 United-States &gt;50K 3 44 Private 160323 Some-college 10 Married-civ-spouse Machine-op-inspct Husband Black Male 7688 0 40 United-States &gt;50K 4 18 ? 103497 Some-college 10 Never-married ? Own-child White Female 0 0 30 United-States &lt;=50K In\u00a0[3]: Copied! <pre># For convenience, we'll replace '-' with '_'\ndf.columns = [c.replace(\"-\", \"_\") for c in df.columns]\n# binary target\ndf[\"target\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int)\ndf.drop(\"income\", axis=1, inplace=True)\ndf.head()\n</pre> # For convenience, we'll replace '-' with '_' df.columns = [c.replace(\"-\", \"_\") for c in df.columns] # binary target df[\"target\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int) df.drop(\"income\", axis=1, inplace=True) df.head() Out[3]: age workclass fnlwgt education educational_num marital_status occupation relationship race gender capital_gain capital_loss hours_per_week native_country target 0 25 Private 226802 11th 7 Never-married Machine-op-inspct Own-child Black Male 0 0 40 United-States 0 1 38 Private 89814 HS-grad 9 Married-civ-spouse Farming-fishing Husband White Male 0 0 50 United-States 0 2 28 Local-gov 336951 Assoc-acdm 12 Married-civ-spouse Protective-serv Husband White Male 0 0 40 United-States 1 3 44 Private 160323 Some-college 10 Married-civ-spouse Machine-op-inspct Husband Black Male 7688 0 40 United-States 1 4 18 ? 103497 Some-college 10 Never-married ? Own-child White Female 0 0 30 United-States 0 In\u00a0[4]: Copied! <pre>train, valid = train_test_split(df, test_size=0.2, stratify=df.target)\n# the test data will be used lately as if it was \"fresh\", new data coming after some time...\nvalid, test = train_test_split(valid, test_size=0.5, stratify=valid.target)\n</pre> train, valid = train_test_split(df, test_size=0.2, stratify=df.target) # the test data will be used lately as if it was \"fresh\", new data coming after some time... valid, test = train_test_split(valid, test_size=0.5, stratify=valid.target) In\u00a0[5]: Copied! <pre>print(f\"train shape: {train.shape}\")\nprint(f\"valid shape: {valid.shape}\")\nprint(f\"test shape: {test.shape}\")\n</pre> print(f\"train shape: {train.shape}\") print(f\"valid shape: {valid.shape}\") print(f\"test shape: {test.shape}\") <pre>train shape: (39073, 15)\nvalid shape: (4884, 15)\ntest shape: (4885, 15)\n</pre> In\u00a0[6]: Copied! <pre>cat_embed_cols = [\n    \"workclass\",\n    \"education\",\n    \"marital_status\",\n    \"occupation\",\n    \"relationship\",\n    \"race\",\n    \"gender\",\n    \"capital_gain\",\n    \"capital_loss\",\n    \"native_country\",\n]\ncontinuous_cols = [\"age\", \"hours_per_week\"]\n</pre> cat_embed_cols = [     \"workclass\",     \"education\",     \"marital_status\",     \"occupation\",     \"relationship\",     \"race\",     \"gender\",     \"capital_gain\",     \"capital_loss\",     \"native_country\", ] continuous_cols = [\"age\", \"hours_per_week\"] In\u00a0[7]: Copied! <pre>tab_preprocessor = TabPreprocessor(\n    embed_cols=cat_embed_cols,\n    continuous_cols=continuous_cols,\n)\nX_tab_train = tab_preprocessor.fit_transform(train)\ny_train = train.target.values\nX_tab_valid = tab_preprocessor.transform(valid)\ny_valid = valid.target.values\n</pre> tab_preprocessor = TabPreprocessor(     embed_cols=cat_embed_cols,     continuous_cols=continuous_cols, ) X_tab_train = tab_preprocessor.fit_transform(train) y_train = train.target.values X_tab_valid = tab_preprocessor.transform(valid) y_valid = valid.target.values <pre>/Users/javierrodriguezzaurin/Projects/pytorch-widedeep/pytorch_widedeep/preprocessing/tab_preprocessor.py:358: UserWarning: Continuous columns will not be normalised\n  warnings.warn(\"Continuous columns will not be normalised\")\n</pre> In\u00a0[8]: Copied! <pre>tab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    cat_embed_dropout=0.1,\n    continuous_cols=continuous_cols,\n    cont_norm_layer=\"layernorm\",\n    embed_continuous_method=\"standard\",\n    cont_embed_dim=8,\n    mlp_hidden_dims=[64, 32],\n    mlp_dropout=0.2,\n    mlp_activation=\"leaky_relu\",\n)\nmodel = WideDeep(deeptabular=tab_mlp)\n</pre> tab_mlp = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     cat_embed_dropout=0.1,     continuous_cols=continuous_cols,     cont_norm_layer=\"layernorm\",     embed_continuous_method=\"standard\",     cont_embed_dim=8,     mlp_hidden_dims=[64, 32],     mlp_dropout=0.2,     mlp_activation=\"leaky_relu\", ) model = WideDeep(deeptabular=tab_mlp) In\u00a0[9]: Copied! <pre>model\n</pre> model Out[9]: <pre>WideDeep(\n  (deeptabular): Sequential(\n    (0): TabMlp(\n      (cat_embed): DiffSizeCatEmbeddings(\n        (embed_layers): ModuleDict(\n          (emb_layer_workclass): Embedding(10, 5, padding_idx=0)\n          (emb_layer_education): Embedding(17, 8, padding_idx=0)\n          (emb_layer_marital_status): Embedding(8, 5, padding_idx=0)\n          (emb_layer_occupation): Embedding(16, 7, padding_idx=0)\n          (emb_layer_relationship): Embedding(7, 4, padding_idx=0)\n          (emb_layer_race): Embedding(6, 4, padding_idx=0)\n          (emb_layer_gender): Embedding(3, 2, padding_idx=0)\n          (emb_layer_capital_gain): Embedding(122, 23, padding_idx=0)\n          (emb_layer_capital_loss): Embedding(97, 21, padding_idx=0)\n          (emb_layer_native_country): Embedding(43, 13, padding_idx=0)\n        )\n        (embedding_dropout): Dropout(p=0.1, inplace=False)\n      )\n      (cont_norm): LayerNorm((2,), eps=1e-05, elementwise_affine=True)\n      (cont_embed): ContEmbeddings(\n        INFO: [ContLinear = weight(n_cont_cols, embed_dim) + bias(n_cont_cols, embed_dim)]\n        (linear): ContLinear(n_cont_cols=2, embed_dim=8, embed_dropout=0.0)\n        (dropout): Dropout(p=0.0, inplace=False)\n      )\n      (encoder): MLP(\n        (mlp): Sequential(\n          (dense_layer_0): Sequential(\n            (0): Linear(in_features=108, out_features=64, bias=True)\n            (1): LeakyReLU(negative_slope=0.01, inplace=True)\n            (2): Dropout(p=0.2, inplace=False)\n          )\n          (dense_layer_1): Sequential(\n            (0): Linear(in_features=64, out_features=32, bias=True)\n            (1): LeakyReLU(negative_slope=0.01, inplace=True)\n            (2): Dropout(p=0.2, inplace=False)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=32, out_features=1, bias=True)\n  )\n)</pre> In\u00a0[10]: Copied! <pre>early_stopping = EarlyStopping()\nmodel_checkpoint = ModelCheckpoint(\n    filepath=\"tmp_dir/adult_tabmlp_model\",\n    save_best_only=True,\n    verbose=1,\n    max_save=1,\n)\n\ntrainer = Trainer(\n    model,\n    objective=\"binary\",\n    callbacks=[early_stopping, model_checkpoint],\n    metrics=[Accuracy],\n)\n\ntrainer.fit(\n    X_train={\"X_tab\": X_tab_train, \"target\": y_train},\n    X_val={\"X_tab\": X_tab_valid, \"target\": y_valid},\n    n_epochs=4,\n    batch_size=256,\n)\n</pre> early_stopping = EarlyStopping() model_checkpoint = ModelCheckpoint(     filepath=\"tmp_dir/adult_tabmlp_model\",     save_best_only=True,     verbose=1,     max_save=1, )  trainer = Trainer(     model,     objective=\"binary\",     callbacks=[early_stopping, model_checkpoint],     metrics=[Accuracy], )  trainer.fit(     X_train={\"X_tab\": X_tab_train, \"target\": y_train},     X_val={\"X_tab\": X_tab_valid, \"target\": y_valid},     n_epochs=4,     batch_size=256, ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:02&lt;00:00, 76.25it/s, loss=0.452, metrics={'acc': 0.7867}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 20/20 [00:00&lt;00:00, 125.36it/s, loss=0.335, metrics={'acc': 0.8532}]\n</pre> <pre>\nEpoch 1: val_loss improved from inf to 0.33532 Saving model to tmp_dir/adult_tabmlp_model_1.p\n</pre> <pre>epoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 76.98it/s, loss=0.355, metrics={'acc': 0.8401}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 20/20 [00:00&lt;00:00, 106.51it/s, loss=0.303, metrics={'acc': 0.8665}]\n</pre> <pre>\nEpoch 2: val_loss improved from 0.33532 to 0.30273 Saving model to tmp_dir/adult_tabmlp_model_2.p\n</pre> <pre>epoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 82.71it/s, loss=0.332, metrics={'acc': 0.849}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 20/20 [00:00&lt;00:00, 107.80it/s, loss=0.288, metrics={'acc': 0.8757}]\n</pre> <pre>\nEpoch 3: val_loss improved from 0.30273 to 0.28791 Saving model to tmp_dir/adult_tabmlp_model_3.p\n</pre> <pre>epoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 79.02it/s, loss=0.32, metrics={'acc': 0.8541}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 20/20 [00:00&lt;00:00, 127.07it/s, loss=0.282, metrics={'acc': 0.8763}]</pre> <pre>\nEpoch 4: val_loss improved from 0.28791 to 0.28238 Saving model to tmp_dir/adult_tabmlp_model_4.p\nModel weights restored to best epoch: 4\n</pre> <pre>\n</pre> In\u00a0[11]: Copied! <pre>torch.save(model, \"tmp_dir/model_saved_option_1.pt\")\n</pre> torch.save(model, \"tmp_dir/model_saved_option_1.pt\") In\u00a0[12]: Copied! <pre>torch.save(model.state_dict(), \"tmp_dir/model_state_dict_saved_option_1.pt\")\n</pre> torch.save(model.state_dict(), \"tmp_dir/model_state_dict_saved_option_1.pt\") In\u00a0[13]: Copied! <pre>trainer.save(path=\"tmp_dir/\", model_filename=\"model_saved_option_2.pt\")\n</pre> trainer.save(path=\"tmp_dir/\", model_filename=\"model_saved_option_2.pt\") <p>or the state dict</p> In\u00a0[14]: Copied! <pre>trainer.save(\n    path=\"tmp_dir/\",\n    model_filename=\"model_state_dict_saved_option_2.pt\",\n    save_state_dict=True,\n)\n</pre> trainer.save(     path=\"tmp_dir/\",     model_filename=\"model_state_dict_saved_option_2.pt\",     save_state_dict=True, ) In\u00a0[15]: Copied! <pre>%%bash\n\nls tmp_dir/\n</pre> %%bash  ls tmp_dir/ <pre>adult_tabmlp_model_4.p\nhistory\nmodel_saved_option_1.pt\nmodel_saved_option_2.pt\nmodel_state_dict_saved_option_1.pt\nmodel_state_dict_saved_option_2.pt\n</pre> In\u00a0[16]: Copied! <pre>%%bash\n\nls tmp_dir/history/\n</pre> %%bash  ls tmp_dir/history/ <pre>train_eval_history.json\n</pre> <p>Note that since we have used the <code>ModelCheckpoint</code> Callback, <code>adult_tabmlp_model_2.p</code> is the model state dict of the model at epoch 2, i.e. same as <code>model_state_dict_saved_option_1.p</code> or <code>model_state_dict_saved_option_2.p</code>.</p> In\u00a0[17]: Copied! <pre>with open(\"tmp_dir/tab_preproc.pkl\", \"wb\") as dp:\n    pickle.dump(tab_preprocessor, dp)\n</pre> with open(\"tmp_dir/tab_preproc.pkl\", \"wb\") as dp:     pickle.dump(tab_preprocessor, dp) In\u00a0[18]: Copied! <pre>with open(\"tmp_dir/eary_stop.pkl\", \"wb\") as es:\n    pickle.dump(early_stopping, es)\n</pre> with open(\"tmp_dir/eary_stop.pkl\", \"wb\") as es:     pickle.dump(early_stopping, es) In\u00a0[19]: Copied! <pre>%%bash\n\nls tmp_dir/\n</pre> %%bash  ls tmp_dir/ <pre>adult_tabmlp_model_4.p\neary_stop.pkl\nhistory\nmodel_saved_option_1.pt\nmodel_saved_option_2.pt\nmodel_state_dict_saved_option_1.pt\nmodel_state_dict_saved_option_2.pt\ntab_preproc.pkl\n</pre> <p>And that is pretty much all you need to resume training or directly predict, let's see</p> In\u00a0[20]: Copied! <pre>test.head()\n</pre> test.head() Out[20]: age workclass fnlwgt education educational_num marital_status occupation relationship race gender capital_gain capital_loss hours_per_week native_country target 10103 43 Private 198282 HS-grad 9 Married-civ-spouse Craft-repair Husband White Male 0 0 40 United-States 1 31799 20 Private 228686 11th 7 Married-civ-spouse Other-service Husband White Male 0 0 40 United-States 0 19971 26 Private 291968 HS-grad 9 Married-civ-spouse Transport-moving Husband White Male 0 0 44 United-States 0 3039 48 Private 175958 Bachelors 13 Divorced Prof-specialty Not-in-family White Male 0 0 30 United-States 0 20725 18 Private 232024 11th 7 Never-married Machine-op-inspct Own-child White Male 0 0 55 United-States 0 In\u00a0[21]: Copied! <pre>with open(\"tmp_dir/tab_preproc.pkl\", \"rb\") as tp:\n    tab_preprocessor_new = pickle.load(tp)\n</pre> with open(\"tmp_dir/tab_preproc.pkl\", \"rb\") as tp:     tab_preprocessor_new = pickle.load(tp) In\u00a0[22]: Copied! <pre>X_test_tab = tab_preprocessor_new.transform(test)\ny_test = test.target\n</pre> X_test_tab = tab_preprocessor_new.transform(test) y_test = test.target In\u00a0[23]: Copied! <pre>tab_mlp_new = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    cat_embed_dropout=0.1,\n    continuous_cols=continuous_cols,\n    cont_norm_layer=\"layernorm\",\n    embed_continuous_method=\"standard\",\n    cont_embed_dim=8,\n    mlp_hidden_dims=[64, 32],\n    mlp_dropout=0.2,\n    mlp_activation=\"leaky_relu\",\n)\nnew_model = WideDeep(deeptabular=tab_mlp)\n</pre> tab_mlp_new = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     cat_embed_dropout=0.1,     continuous_cols=continuous_cols,     cont_norm_layer=\"layernorm\",     embed_continuous_method=\"standard\",     cont_embed_dim=8,     mlp_hidden_dims=[64, 32],     mlp_dropout=0.2,     mlp_activation=\"leaky_relu\", ) new_model = WideDeep(deeptabular=tab_mlp) In\u00a0[24]: Copied! <pre>new_model.load_state_dict(torch.load(\"tmp_dir/model_state_dict_saved_option_2.pt\"))\n</pre> new_model.load_state_dict(torch.load(\"tmp_dir/model_state_dict_saved_option_2.pt\")) Out[24]: <pre>&lt;All keys matched successfully&gt;</pre> In\u00a0[25]: Copied! <pre>trainer = Trainer(\n    model,\n    objective=\"binary\",\n)\n</pre> trainer = Trainer(     model,     objective=\"binary\", ) In\u00a0[26]: Copied! <pre>preds = trainer.predict(X_tab=X_test_tab, batch_size=32)\n</pre> preds = trainer.predict(X_tab=X_test_tab, batch_size=32) <pre>predict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:00&lt;00:00, 309.83it/s]\n</pre> In\u00a0[27]: Copied! <pre>from sklearn.metrics import accuracy_score\n</pre> from sklearn.metrics import accuracy_score In\u00a0[28]: Copied! <pre>accuracy_score(y_test, preds)\n</pre> accuracy_score(y_test, preds) Out[28]: <pre>0.8595701125895598</pre> In\u00a0[29]: Copied! <pre>shutil.rmtree(\"tmp_dir/\")\n</pre> shutil.rmtree(\"tmp_dir/\")"},{"location":"examples/05_save_and_load_model_and_artifacts.html#save-and-load-model-and-artifacts","title":"Save and load model and artifacts\u00b6","text":"<p>In this notebook I will show the different options to save and load a model, as well as some additional objects produced during training.</p> <p>On a given day, you train a model...</p>"},{"location":"examples/05_save_and_load_model_and_artifacts.html#save-model-option-1","title":"Save model: option 1\u00b6","text":"<p>save (and load) a model as you woud do with any other torch model</p>"},{"location":"examples/05_save_and_load_model_and_artifacts.html#save-model-option-2","title":"Save model: option 2\u00b6","text":"<p>use the <code>trainer</code>. The <code>trainer</code> will also save the training history and the learning rate history (if learning rate schedulers are used)</p>"},{"location":"examples/05_save_and_load_model_and_artifacts.html#save-preprocessors-and-callbacks","title":"Save preprocessors and callbacks\u00b6","text":"<p>...just pickle them</p>"},{"location":"examples/05_save_and_load_model_and_artifacts.html#run-new-experiment-prepare-new-dataset-load-model-and-predict","title":"Run New experiment: prepare new dataset, load model, and predict\u00b6","text":""},{"location":"examples/06_finetune_and_warmup.html","title":"06_finetune_and_warmup","text":"In\u00a0[1]: Copied! <pre>import numpy as np\nimport pandas as pd\nimport torch\n\nfrom pytorch_widedeep import Trainer\nfrom pytorch_widedeep.preprocessing import WidePreprocessor, TabPreprocessor\nfrom pytorch_widedeep.models import Wide, TabMlp, TabResnet, WideDeep\nfrom pytorch_widedeep.metrics import Accuracy\nfrom pytorch_widedeep.datasets import load_adult\n</pre> import numpy as np import pandas as pd import torch  from pytorch_widedeep import Trainer from pytorch_widedeep.preprocessing import WidePreprocessor, TabPreprocessor from pytorch_widedeep.models import Wide, TabMlp, TabResnet, WideDeep from pytorch_widedeep.metrics import Accuracy from pytorch_widedeep.datasets import load_adult <pre>/Users/javierrodriguezzaurin/.pyenv/versions/3.10.13/envs/widedeep310/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n  from .autonotebook import tqdm as notebook_tqdm\n</pre> In\u00a0[2]: Copied! <pre>df = load_adult(as_frame=True)\n# For convenience, we'll replace '-' with '_'\ndf.columns = [c.replace(\"-\", \"_\") for c in df.columns]\n# binary target\ndf[\"income_label\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int)\ndf.drop(\"income\", axis=1, inplace=True)\ndf.head()\n</pre> df = load_adult(as_frame=True) # For convenience, we'll replace '-' with '_' df.columns = [c.replace(\"-\", \"_\") for c in df.columns] # binary target df[\"income_label\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int) df.drop(\"income\", axis=1, inplace=True) df.head() Out[2]: age workclass fnlwgt education educational_num marital_status occupation relationship race gender capital_gain capital_loss hours_per_week native_country income_label 0 25 Private 226802 11th 7 Never-married Machine-op-inspct Own-child Black Male 0 0 40 United-States 0 1 38 Private 89814 HS-grad 9 Married-civ-spouse Farming-fishing Husband White Male 0 0 50 United-States 0 2 28 Local-gov 336951 Assoc-acdm 12 Married-civ-spouse Protective-serv Husband White Male 0 0 40 United-States 1 3 44 Private 160323 Some-college 10 Married-civ-spouse Machine-op-inspct Husband Black Male 7688 0 40 United-States 1 4 18 ? 103497 Some-college 10 Never-married ? Own-child White Female 0 0 30 United-States 0 In\u00a0[3]: Copied! <pre># Define wide, crossed and deep tabular columns\nwide_cols = [\n    \"workclass\",\n    \"education\",\n    \"marital_status\",\n    \"occupation\",\n    \"relationship\",\n    \"race\",\n    \"gender\",\n    \"native_country\",\n]\ncrossed_cols = [(\"education\", \"occupation\"), (\"native_country\", \"occupation\")]\ncat_embed_cols = [\n    \"workclass\",\n    \"education\",\n    \"marital_status\",\n    \"occupation\",\n    \"relationship\",\n    \"race\",\n    \"gender\",\n    \"capital_gain\",\n    \"capital_loss\",\n    \"native_country\",\n]\ncontinuous_cols = [\"age\", \"hours_per_week\"]\ntarget_col = \"income_label\"\ntarget = df[target_col].values\n</pre> # Define wide, crossed and deep tabular columns wide_cols = [     \"workclass\",     \"education\",     \"marital_status\",     \"occupation\",     \"relationship\",     \"race\",     \"gender\",     \"native_country\", ] crossed_cols = [(\"education\", \"occupation\"), (\"native_country\", \"occupation\")] cat_embed_cols = [     \"workclass\",     \"education\",     \"marital_status\",     \"occupation\",     \"relationship\",     \"race\",     \"gender\",     \"capital_gain\",     \"capital_loss\",     \"native_country\", ] continuous_cols = [\"age\", \"hours_per_week\"] target_col = \"income_label\" target = df[target_col].values In\u00a0[4]: Copied! <pre># TARGET\ntarget = df[target_col].values\n\n# WIDE\nwide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols)\nX_wide = wide_preprocessor.fit_transform(df)\n\n# DEEP\ntab_preprocessor = TabPreprocessor(\n    cat_embed_cols=cat_embed_cols, continuous_cols=continuous_cols\n)\nX_tab = tab_preprocessor.fit_transform(df)\n</pre> # TARGET target = df[target_col].values  # WIDE wide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols) X_wide = wide_preprocessor.fit_transform(df)  # DEEP tab_preprocessor = TabPreprocessor(     cat_embed_cols=cat_embed_cols, continuous_cols=continuous_cols ) X_tab = tab_preprocessor.fit_transform(df) <pre>/Users/javierrodriguezzaurin/Projects/pytorch-widedeep/pytorch_widedeep/preprocessing/tab_preprocessor.py:358: UserWarning: Continuous columns will not be normalised\n  warnings.warn(\"Continuous columns will not be normalised\")\n</pre> In\u00a0[5]: Copied! <pre>wide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1)\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    cat_embed_dropout=0.1,\n    continuous_cols=continuous_cols,\n    embed_continuous_method=\"standard\",\n    cont_embed_dim=8,\n    mlp_hidden_dims=[64, 32],\n    mlp_dropout=0.2,\n    mlp_activation=\"leaky_relu\",\n)\nmodel = WideDeep(wide=wide, deeptabular=tab_mlp)\n</pre> wide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1) tab_mlp = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     cat_embed_dropout=0.1,     continuous_cols=continuous_cols,     embed_continuous_method=\"standard\",     cont_embed_dim=8,     mlp_hidden_dims=[64, 32],     mlp_dropout=0.2,     mlp_activation=\"leaky_relu\", ) model = WideDeep(wide=wide, deeptabular=tab_mlp) In\u00a0[6]: Copied! <pre>trainer = Trainer(\n    model,\n    objective=\"binary\",\n    optimizers=torch.optim.Adam(model.parameters(), lr=0.01),\n    metrics=[Accuracy],\n)\n</pre> trainer = Trainer(     model,     objective=\"binary\",     optimizers=torch.optim.Adam(model.parameters(), lr=0.01),     metrics=[Accuracy], ) In\u00a0[7]: Copied! <pre>trainer.fit(\n    X_wide=X_wide, X_tab=X_tab, target=target, n_epochs=2, val_split=0.2, batch_size=256\n)\n</pre> trainer.fit(     X_wide=X_wide, X_tab=X_tab, target=target, n_epochs=2, val_split=0.2, batch_size=256 ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:02&lt;00:00, 74.26it/s, loss=0.399, metrics={'acc': 0.8163}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 39/39 [00:00&lt;00:00, 91.03it/s, loss=0.296, metrics={'acc': 0.8677}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 81.31it/s, loss=0.3, metrics={'acc': 0.8614}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 39/39 [00:00&lt;00:00, 106.45it/s, loss=0.285, metrics={'acc': 0.8721}]\n</pre> In\u00a0[8]: Copied! <pre>trainer.save(path=\"models_dir/\", save_state_dict=True, model_filename=\"model_1.pt\")\n</pre> trainer.save(path=\"models_dir/\", save_state_dict=True, model_filename=\"model_1.pt\") <p>Now time goes by...and we want to fine-tune the model to another, new dataset (for example, a dataset that is identical to the one you used to train the previous model but for another country).</p> <p>Here I will use the same dataset just for illustration purposes, but the flow would be identical to that new dataset</p> In\u00a0[9]: Copied! <pre>wide_1 = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1)\ntab_mlp_1 = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    cat_embed_dropout=0.1,\n    continuous_cols=continuous_cols,\n    embed_continuous_method=\"standard\",\n    cont_embed_dim=8,\n    mlp_hidden_dims=[64, 32],\n    mlp_dropout=0.2,\n    mlp_activation=\"leaky_relu\",\n)\nmodel_1 = WideDeep(wide=wide, deeptabular=tab_mlp)\n</pre> wide_1 = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1) tab_mlp_1 = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     cat_embed_dropout=0.1,     continuous_cols=continuous_cols,     embed_continuous_method=\"standard\",     cont_embed_dim=8,     mlp_hidden_dims=[64, 32],     mlp_dropout=0.2,     mlp_activation=\"leaky_relu\", ) model_1 = WideDeep(wide=wide, deeptabular=tab_mlp) In\u00a0[10]: Copied! <pre>model_1.load_state_dict(torch.load(\"models_dir/model_1.pt\"))\n</pre> model_1.load_state_dict(torch.load(\"models_dir/model_1.pt\")) Out[10]: <pre>&lt;All keys matched successfully&gt;</pre> In\u00a0[11]: Copied! <pre>trainer_1 = Trainer(model_1, objective=\"binary\", metrics=[Accuracy])\n</pre> trainer_1 = Trainer(model_1, objective=\"binary\", metrics=[Accuracy]) In\u00a0[12]: Copied! <pre>trainer_1.fit(\n    X_wide=X_wide,\n    X_tab=X_tab,\n    target=target,\n    n_epochs=2,\n    batch_size=256,\n    finetune=True,\n    finetune_epochs=2,\n)\n</pre> trainer_1.fit(     X_wide=X_wide,     X_tab=X_tab,     target=target,     n_epochs=2,     batch_size=256,     finetune=True,     finetune_epochs=2, ) <pre>Training wide for 2 epochs\n</pre> <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 191/191 [00:01&lt;00:00, 97.37it/s, loss=0.39, metrics={'acc': 0.8152}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 191/191 [00:01&lt;00:00, 104.04it/s, loss=0.359, metrics={'acc': 0.824}]\n</pre> <pre>Training deeptabular for 2 epochs\n</pre> <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 191/191 [00:02&lt;00:00, 83.83it/s, loss=0.297, metrics={'acc': 0.8365}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 191/191 [00:02&lt;00:00, 82.78it/s, loss=0.283, metrics={'acc': 0.8445}]\n</pre> <pre>Fine-tuning (or warmup) of individual components completed. Training the whole model for 2 epochs\n</pre> <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 191/191 [00:02&lt;00:00, 72.84it/s, loss=0.281, metrics={'acc': 0.8716}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 191/191 [00:02&lt;00:00, 77.46it/s, loss=0.273, metrics={'acc': 0.8744}]\n</pre> <p>Note that, as I describe above, in scenario 2, we can just use this to warm up models before they joined training begins:</p> In\u00a0[13]: Copied! <pre>wide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1)\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    cat_embed_dropout=0.1,\n    continuous_cols=continuous_cols,\n    embed_continuous_method=\"standard\",\n    cont_embed_dim=8,\n    mlp_hidden_dims=[64, 32],\n    mlp_dropout=0.2,\n    mlp_activation=\"leaky_relu\",\n)\nmodel = WideDeep(wide=wide, deeptabular=tab_mlp)\n</pre> wide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1) tab_mlp = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     cat_embed_dropout=0.1,     continuous_cols=continuous_cols,     embed_continuous_method=\"standard\",     cont_embed_dim=8,     mlp_hidden_dims=[64, 32],     mlp_dropout=0.2,     mlp_activation=\"leaky_relu\", ) model = WideDeep(wide=wide, deeptabular=tab_mlp) In\u00a0[14]: Copied! <pre>trainer_2 = Trainer(model, objective=\"binary\", metrics=[Accuracy])\n</pre> trainer_2 = Trainer(model, objective=\"binary\", metrics=[Accuracy]) In\u00a0[15]: Copied! <pre>trainer_2.fit(\n    X_wide=X_wide,\n    X_tab=X_tab,\n    target=target,\n    val_split=0.1,\n    warmup=True,\n    warmup_epochs=2,\n    n_epochs=2,\n    batch_size=256,\n)\n</pre> trainer_2.fit(     X_wide=X_wide,     X_tab=X_tab,     target=target,     val_split=0.1,     warmup=True,     warmup_epochs=2,     n_epochs=2,     batch_size=256, ) <pre>Training wide for 2 epochs\n</pre> <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:01&lt;00:00, 102.49it/s, loss=0.52, metrics={'acc': 0.7519}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:01&lt;00:00, 98.15it/s, loss=0.381, metrics={'acc': 0.7891}]\n</pre> <pre>Training deeptabular for 2 epochs\n</pre> <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:02&lt;00:00, 82.97it/s, loss=0.356, metrics={'acc': 0.8043}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:02&lt;00:00, 80.27it/s, loss=0.295, metrics={'acc': 0.8195}]\n</pre> <pre>Fine-tuning (or warmup) of individual components completed. Training the whole model for 2 epochs\n</pre> <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:02&lt;00:00, 77.27it/s, loss=0.291, metrics={'acc': 0.8667}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 20/20 [00:00&lt;00:00, 89.57it/s, loss=0.289, metrics={'acc': 0.8665}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:02&lt;00:00, 72.69it/s, loss=0.283, metrics={'acc': 0.8693}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 20/20 [00:00&lt;00:00, 91.81it/s, loss=0.284, metrics={'acc': 0.869}]\n</pre> <p>We need to explicitly indicate</p> <ol> <li><p>That we want fine-tune</p> </li> <li><p>The components that we want to individually fine-tune</p> </li> <li><p>In case of gradual fine-tuning, the routine (\"felbo\" or \"howard\")</p> </li> <li><p>The layers we want to fine-tune.</p> </li> </ol> <p>For example</p> In\u00a0[16]: Copied! <pre>wide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1)\ntab_resnet = TabResnet(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    cat_embed_dropout=0.1,\n    continuous_cols=continuous_cols,\n    blocks_dims=[200, 200, 200],\n)\nmodel = WideDeep(wide=wide, deeptabular=tab_resnet)\n</pre> wide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1) tab_resnet = TabResnet(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     cat_embed_dropout=0.1,     continuous_cols=continuous_cols,     blocks_dims=[200, 200, 200], ) model = WideDeep(wide=wide, deeptabular=tab_resnet) In\u00a0[17]: Copied! <pre>model\n</pre> model Out[17]: <pre>WideDeep(\n  (wide): Wide(\n    (wide_linear): Embedding(809, 1, padding_idx=0)\n  )\n  (deeptabular): Sequential(\n    (0): TabResnet(\n      (cat_embed): DiffSizeCatEmbeddings(\n        (embed_layers): ModuleDict(\n          (emb_layer_workclass): Embedding(10, 5, padding_idx=0)\n          (emb_layer_education): Embedding(17, 8, padding_idx=0)\n          (emb_layer_marital_status): Embedding(8, 5, padding_idx=0)\n          (emb_layer_occupation): Embedding(16, 7, padding_idx=0)\n          (emb_layer_relationship): Embedding(7, 4, padding_idx=0)\n          (emb_layer_race): Embedding(6, 4, padding_idx=0)\n          (emb_layer_gender): Embedding(3, 2, padding_idx=0)\n          (emb_layer_capital_gain): Embedding(124, 24, padding_idx=0)\n          (emb_layer_capital_loss): Embedding(100, 21, padding_idx=0)\n          (emb_layer_native_country): Embedding(43, 13, padding_idx=0)\n        )\n        (embedding_dropout): Dropout(p=0.1, inplace=False)\n      )\n      (cont_norm): Identity()\n      (encoder): DenseResnet(\n        (dense_resnet): Sequential(\n          (lin_inp): Linear(in_features=95, out_features=200, bias=False)\n          (bn_inp): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n          (block_0): BasicBlock(\n            (lin1): Linear(in_features=200, out_features=200, bias=False)\n            (bn1): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n            (leaky_relu): LeakyReLU(negative_slope=0.01, inplace=True)\n            (dp): Dropout(p=0.1, inplace=False)\n            (lin2): Linear(in_features=200, out_features=200, bias=False)\n            (bn2): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n          )\n          (block_1): BasicBlock(\n            (lin1): Linear(in_features=200, out_features=200, bias=False)\n            (bn1): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n            (leaky_relu): LeakyReLU(negative_slope=0.01, inplace=True)\n            (dp): Dropout(p=0.1, inplace=False)\n            (lin2): Linear(in_features=200, out_features=200, bias=False)\n            (bn2): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=200, out_features=1, bias=True)\n  )\n)</pre> <p>let's first train as usual</p> In\u00a0[18]: Copied! <pre>trainer_3 = Trainer(model, objective=\"binary\", metrics=[Accuracy])\n</pre> trainer_3 = Trainer(model, objective=\"binary\", metrics=[Accuracy]) In\u00a0[19]: Copied! <pre>trainer_3.fit(\n    X_wide=X_wide, X_tab=X_tab, target=target, val_split=0.1, n_epochs=2, batch_size=256\n)\n</pre> trainer_3.fit(     X_wide=X_wide, X_tab=X_tab, target=target, val_split=0.1, n_epochs=2, batch_size=256 ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:03&lt;00:00, 54.23it/s, loss=0.382, metrics={'acc': 0.8239}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 20/20 [00:00&lt;00:00, 84.72it/s, loss=0.331, metrics={'acc': 0.8526}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:03&lt;00:00, 54.35it/s, loss=0.33, metrics={'acc': 0.8465}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 20/20 [00:00&lt;00:00, 68.15it/s, loss=0.312, metrics={'acc': 0.8604}]\n</pre> In\u00a0[20]: Copied! <pre>trainer_3.save(path=\"models_dir\", save_state_dict=True, model_filename=\"model_3.pt\")\n</pre> trainer_3.save(path=\"models_dir\", save_state_dict=True, model_filename=\"model_3.pt\") <p>Now we are going to fine-tune the model components, and in the case of the <code>deeptabular</code> component, we will fine-tune the resnet-blocks and the linear layer but NOT the embeddings.</p> <p>For this, we need to access the model component's children: <code>deeptabular</code> $\\rightarrow$ <code>tab_resnet</code> $\\rightarrow$ <code>dense_resnet</code> $\\rightarrow$ <code>blocks</code></p> In\u00a0[21]: Copied! <pre>wide_3 = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1)\ntab_resnet_3 = TabResnet(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    cat_embed_dropout=0.1,\n    continuous_cols=continuous_cols,\n    blocks_dims=[200, 200, 200],\n)\nmodel_3 = WideDeep(wide=wide, deeptabular=tab_resnet)\n</pre> wide_3 = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1) tab_resnet_3 = TabResnet(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     cat_embed_dropout=0.1,     continuous_cols=continuous_cols,     blocks_dims=[200, 200, 200], ) model_3 = WideDeep(wide=wide, deeptabular=tab_resnet) In\u00a0[22]: Copied! <pre>model_3.load_state_dict(torch.load(\"models_dir/model_3.pt\"))\n</pre> model_3.load_state_dict(torch.load(\"models_dir/model_3.pt\")) Out[22]: <pre>&lt;All keys matched successfully&gt;</pre> In\u00a0[23]: Copied! <pre>model_3\n</pre> model_3 Out[23]: <pre>WideDeep(\n  (wide): Wide(\n    (wide_linear): Embedding(809, 1, padding_idx=0)\n  )\n  (deeptabular): Sequential(\n    (0): TabResnet(\n      (cat_embed): DiffSizeCatEmbeddings(\n        (embed_layers): ModuleDict(\n          (emb_layer_workclass): Embedding(10, 5, padding_idx=0)\n          (emb_layer_education): Embedding(17, 8, padding_idx=0)\n          (emb_layer_marital_status): Embedding(8, 5, padding_idx=0)\n          (emb_layer_occupation): Embedding(16, 7, padding_idx=0)\n          (emb_layer_relationship): Embedding(7, 4, padding_idx=0)\n          (emb_layer_race): Embedding(6, 4, padding_idx=0)\n          (emb_layer_gender): Embedding(3, 2, padding_idx=0)\n          (emb_layer_capital_gain): Embedding(124, 24, padding_idx=0)\n          (emb_layer_capital_loss): Embedding(100, 21, padding_idx=0)\n          (emb_layer_native_country): Embedding(43, 13, padding_idx=0)\n        )\n        (embedding_dropout): Dropout(p=0.1, inplace=False)\n      )\n      (cont_norm): Identity()\n      (encoder): DenseResnet(\n        (dense_resnet): Sequential(\n          (lin_inp): Linear(in_features=95, out_features=200, bias=False)\n          (bn_inp): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n          (block_0): BasicBlock(\n            (lin1): Linear(in_features=200, out_features=200, bias=False)\n            (bn1): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n            (leaky_relu): LeakyReLU(negative_slope=0.01, inplace=True)\n            (dp): Dropout(p=0.1, inplace=False)\n            (lin2): Linear(in_features=200, out_features=200, bias=False)\n            (bn2): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n          )\n          (block_1): BasicBlock(\n            (lin1): Linear(in_features=200, out_features=200, bias=False)\n            (bn1): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n            (leaky_relu): LeakyReLU(negative_slope=0.01, inplace=True)\n            (dp): Dropout(p=0.1, inplace=False)\n            (lin2): Linear(in_features=200, out_features=200, bias=False)\n            (bn2): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=200, out_features=1, bias=True)\n  )\n)</pre> In\u00a0[24]: Copied! <pre>tab_lin_layer = list(model_3.deeptabular.children())[1]\n</pre> tab_lin_layer = list(model_3.deeptabular.children())[1] In\u00a0[25]: Copied! <pre>tab_lin_layer\n</pre> tab_lin_layer Out[25]: <pre>Linear(in_features=200, out_features=1, bias=True)</pre> In\u00a0[26]: Copied! <pre>tab_deep_layers = []\nfor n1, c1 in model_3.deeptabular.named_children():\n    if (\n        n1 == \"0\"\n    ):  # 0 is the model component and 1 is always the prediction layer added by the `WideDeep` class\n        for n2, c2 in c1.named_children():\n            if n2 == \"encoder\":  # TabResnet\n                for _, c3 in c2.named_children():\n                    for n4, c4 in c3.named_children():  # dense_resnet\n                        if \"block\" in n4:\n                            tab_deep_layers.append((n4, c4))\n</pre> tab_deep_layers = [] for n1, c1 in model_3.deeptabular.named_children():     if (         n1 == \"0\"     ):  # 0 is the model component and 1 is always the prediction layer added by the `WideDeep` class         for n2, c2 in c1.named_children():             if n2 == \"encoder\":  # TabResnet                 for _, c3 in c2.named_children():                     for n4, c4 in c3.named_children():  # dense_resnet                         if \"block\" in n4:                             tab_deep_layers.append((n4, c4)) In\u00a0[27]: Copied! <pre>tab_deep_layers\n</pre> tab_deep_layers Out[27]: <pre>[('block_0',\n  BasicBlock(\n    (lin1): Linear(in_features=200, out_features=200, bias=False)\n    (bn1): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n    (leaky_relu): LeakyReLU(negative_slope=0.01, inplace=True)\n    (dp): Dropout(p=0.1, inplace=False)\n    (lin2): Linear(in_features=200, out_features=200, bias=False)\n    (bn2): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n  )),\n ('block_1',\n  BasicBlock(\n    (lin1): Linear(in_features=200, out_features=200, bias=False)\n    (bn1): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n    (leaky_relu): LeakyReLU(negative_slope=0.01, inplace=True)\n    (dp): Dropout(p=0.1, inplace=False)\n    (lin2): Linear(in_features=200, out_features=200, bias=False)\n    (bn2): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n  ))]</pre> <p>Now remember, we need to pass ONLY LAYERS (before I included the name for clarity) the layers in WARM UP ORDER, therefore:</p> In\u00a0[28]: Copied! <pre>tab_deep_layers = [el[1] for el in tab_deep_layers][::-1]\n</pre> tab_deep_layers = [el[1] for el in tab_deep_layers][::-1] In\u00a0[29]: Copied! <pre>tab_layers = [tab_lin_layer] + tab_deep_layers[::-1]\n</pre> tab_layers = [tab_lin_layer] + tab_deep_layers[::-1] In\u00a0[30]: Copied! <pre>tab_layers\n</pre> tab_layers Out[30]: <pre>[Linear(in_features=200, out_features=1, bias=True),\n BasicBlock(\n   (lin1): Linear(in_features=200, out_features=200, bias=False)\n   (bn1): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n   (leaky_relu): LeakyReLU(negative_slope=0.01, inplace=True)\n   (dp): Dropout(p=0.1, inplace=False)\n   (lin2): Linear(in_features=200, out_features=200, bias=False)\n   (bn2): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n ),\n BasicBlock(\n   (lin1): Linear(in_features=200, out_features=200, bias=False)\n   (bn1): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n   (leaky_relu): LeakyReLU(negative_slope=0.01, inplace=True)\n   (dp): Dropout(p=0.1, inplace=False)\n   (lin2): Linear(in_features=200, out_features=200, bias=False)\n   (bn2): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n )]</pre> <p>And now simply</p> In\u00a0[31]: Copied! <pre>trainer_4 = Trainer(model_3, objective=\"binary\", metrics=[Accuracy])\n</pre> trainer_4 = Trainer(model_3, objective=\"binary\", metrics=[Accuracy]) In\u00a0[32]: Copied! <pre>trainer_4.fit(\n    X_wide=X_wide,\n    X_tab=X_tab,\n    target=target,\n    val_split=0.1,\n    finetune=True,\n    finetune_epochs=2,\n    deeptabular_gradual=True,\n    deeptabular_layers=tab_layers,\n    deeptabular_max_lr=0.01,\n    n_epochs=2,\n    batch_size=256,\n)\n</pre> trainer_4.fit(     X_wide=X_wide,     X_tab=X_tab,     target=target,     val_split=0.1,     finetune=True,     finetune_epochs=2,     deeptabular_gradual=True,     deeptabular_layers=tab_layers,     deeptabular_max_lr=0.01,     n_epochs=2,     batch_size=256, ) <pre>Training wide for 2 epochs\n</pre> <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:01&lt;00:00, 95.17it/s, loss=0.504, metrics={'acc': 0.7523}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:01&lt;00:00, 99.83it/s, loss=0.384, metrics={'acc': 0.789}]\n</pre> <pre>Training deeptabular, layer 1 of 3\n</pre> <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:02&lt;00:00, 72.31it/s, loss=0.317, metrics={'acc': 0.8098}]\n</pre> <pre>Training deeptabular, layer 2 of 3\n</pre> <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:02&lt;00:00, 65.97it/s, loss=0.312, metrics={'acc': 0.8214}]\n</pre> <pre>Training deeptabular, layer 3 of 3\n</pre> <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:02&lt;00:00, 63.92it/s, loss=0.306, metrics={'acc': 0.8284}]\n</pre> <pre>Fine-tuning (or warmup) of individual components completed. Training the whole model for 2 epochs\n</pre> <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:03&lt;00:00, 57.26it/s, loss=0.292, metrics={'acc': 0.8664}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 20/20 [00:00&lt;00:00, 84.56it/s, loss=0.292, metrics={'acc': 0.8696}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:03&lt;00:00, 53.61it/s, loss=0.282, metrics={'acc': 0.8693}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 20/20 [00:00&lt;00:00, 80.59it/s, loss=0.289, metrics={'acc': 0.8719}]\n</pre> <p>Finally, there is one more use case I would like to consider. The case where we train only one component and we just want to fine-tune and stop the training afterwards, since there is no joined training. This is a simple as</p> In\u00a0[33]: Copied! <pre>tab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    cat_embed_dropout=0.1,\n    continuous_cols=continuous_cols,\n    mlp_hidden_dims=[64, 32],\n    mlp_dropout=0.2,\n    mlp_activation=\"leaky_relu\",\n)\nmodel = WideDeep(deeptabular=tab_mlp)\n</pre> tab_mlp = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     cat_embed_dropout=0.1,     continuous_cols=continuous_cols,     mlp_hidden_dims=[64, 32],     mlp_dropout=0.2,     mlp_activation=\"leaky_relu\", ) model = WideDeep(deeptabular=tab_mlp) In\u00a0[34]: Copied! <pre>trainer_5 = Trainer(\n    model,\n    objective=\"binary\",\n    optimizers=torch.optim.Adam(model.parameters(), lr=0.01),\n    metrics=[Accuracy],\n)\n</pre> trainer_5 = Trainer(     model,     objective=\"binary\",     optimizers=torch.optim.Adam(model.parameters(), lr=0.01),     metrics=[Accuracy], ) In\u00a0[35]: Copied! <pre>trainer_5.fit(\n    X_wide=X_wide, X_tab=X_tab, target=target, val_split=0.1, n_epochs=1, batch_size=256\n)\n</pre> trainer_5.fit(     X_wide=X_wide, X_tab=X_tab, target=target, val_split=0.1, n_epochs=1, batch_size=256 ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:02&lt;00:00, 73.69it/s, loss=0.365, metrics={'acc': 0.8331}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 20/20 [00:00&lt;00:00, 92.56it/s, loss=0.299, metrics={'acc': 0.8673}]\n</pre> In\u00a0[36]: Copied! <pre>trainer_5.save(path=\"models_dir\", save_state_dict=True, model_filename=\"model_5.pt\")\n</pre> trainer_5.save(path=\"models_dir\", save_state_dict=True, model_filename=\"model_5.pt\") In\u00a0[37]: Copied! <pre>tab_mlp_5 = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    cat_embed_dropout=0.1,\n    continuous_cols=continuous_cols,\n    mlp_hidden_dims=[64, 32],\n    mlp_dropout=0.2,\n    mlp_activation=\"leaky_relu\",\n)\nmodel_5 = WideDeep(deeptabular=tab_mlp_5)\n</pre> tab_mlp_5 = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     cat_embed_dropout=0.1,     continuous_cols=continuous_cols,     mlp_hidden_dims=[64, 32],     mlp_dropout=0.2,     mlp_activation=\"leaky_relu\", ) model_5 = WideDeep(deeptabular=tab_mlp_5) In\u00a0[38]: Copied! <pre>model_5.load_state_dict(torch.load(\"models_dir/model_5.pt\"))\n</pre> model_5.load_state_dict(torch.load(\"models_dir/model_5.pt\")) Out[38]: <pre>&lt;All keys matched successfully&gt;</pre> <p>...times go by...</p> In\u00a0[39]: Copied! <pre>trainer_6 = Trainer(\n    model_5,\n    objective=\"binary\",\n    optimizers=torch.optim.Adam(model.parameters(), lr=0.01),\n    metrics=[Accuracy],\n)\n</pre> trainer_6 = Trainer(     model_5,     objective=\"binary\",     optimizers=torch.optim.Adam(model.parameters(), lr=0.01),     metrics=[Accuracy], ) In\u00a0[40]: Copied! <pre>trainer_6.fit(\n    X_wide=X_wide,\n    X_tab=X_tab,\n    target=target,\n    val_split=0.1,\n    finetune=True,\n    finetune_epochs=2,\n    finetune_max_lr=0.01,\n    stop_after_finetuning=True,\n    batch_size=256,\n)\n</pre> trainer_6.fit(     X_wide=X_wide,     X_tab=X_tab,     target=target,     val_split=0.1,     finetune=True,     finetune_epochs=2,     finetune_max_lr=0.01,     stop_after_finetuning=True,     batch_size=256, ) <pre>Training deeptabular for 2 epochs\n</pre> <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:02&lt;00:00, 73.86it/s, loss=0.298, metrics={'acc': 0.8652}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:02&lt;00:00, 75.45it/s, loss=0.286, metrics={'acc': 0.8669}]\n</pre> <pre>Fine-tuning (or warmup) of individual components completed. Training the whole model for 1 epochs\n</pre> <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 172/172 [00:02&lt;00:00, 76.29it/s, loss=0.282, metrics={'acc': 0.8698}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 20/20 [00:00&lt;00:00, 84.93it/s, loss=0.281, metrics={'acc': 0.8749}]\n</pre> In\u00a0[42]: Copied! <pre>import shutil\n\nshutil.rmtree(\"models_dir/\")\nshutil.rmtree(\"model_weights/\")\n</pre> import shutil  shutil.rmtree(\"models_dir/\") shutil.rmtree(\"model_weights/\") In\u00a0[\u00a0]: Copied! <pre>\n</pre>"},{"location":"examples/06_finetune_and_warmup.html#the-finetunewarm-up-option","title":"The FineTune/Warm Up option\u00b6","text":"<p>Let's place ourselves in two possible scenarios.</p> <ol> <li><p>Let's assume we have run a model and we want to just transfer the learnings (you know...transfer-learning) to another dataset, or simply we have received new data and we do not want to start the training of each component from scratch. Simply, we want to load the pre-trained weights and fine-tune.</p> </li> <li><p>We just want to \"warm up\" individual model components individually before the joined training begins.</p> </li> </ol> <p>This can be done with the <code>finetune</code> set of parameters. There are 3 fine-tuning routines:</p> <ol> <li>Fine-tune all trainable layers at once with a triangular one-cycle learning rate (referred as slanted triangular learning rates in Howard &amp; Ruder 2018)</li> <li>Gradual fine-tuning inspired by the work of Felbo et al., 2017</li> <li>Gradual fine-tuning based on the work of Howard &amp; Ruder 2018</li> </ol> <p>Currently fine-tunning is only supported without a fully connected head, i.e. if <code>deephead=None</code>. In addition, <code>Felbo</code> and <code>Howard</code> routines only applied, of course, to the <code>deeptabular</code>, <code>deeptext</code> and <code>deepimage</code> models. The <code>wide</code> component can also be fine-tuned, but only in an \"all at once\" mode.</p>"},{"location":"examples/06_finetune_and_warmup.html#fine-tune-or-warm-up-all-at-once","title":"Fine-tune or warm-up all at once\u00b6","text":"<p>Here, the model components will be trained for <code>finetune_epochs</code> using a triangular one-cycle learning rate (slanted triangular learning rate) ranging from <code>finetune_max_lr/10</code> to <code>finetune_max_lr</code> (default is 0.01). 10% of the training steps are used to increase the learning rate which then decreases for the remaining 90%.</p> <p>Here all trainable layers are fine-tuned.</p> <p>Let's have a look to one example.</p>"},{"location":"examples/06_finetune_and_warmup.html#fine-tune-gradually-the-felbo-and-the-howard-routines","title":"Fine-tune Gradually: The \"felbo\"  and the \"howard\" routines\u00b6","text":"<p>The Felbo routine can be illustrated as follows:</p> <p> </p> <p>Figure 1. The figure can be described as follows: fine-tune (or train) the last layer for one epoch using a one cycle triangular learning rate. Then fine-tune the next deeper layer for one epoch, with a learning rate that is a factor of 2.5 lower than the previous learning rate (the 2.5 factor is fixed) while freezing the already warmed up layer(s). Repeat untill all individual layers are warmed. Then warm one last epoch with all warmed layers trainable. The vanishing color gradient in the figure attempts to illustrate the decreasing learning rate.</p> <p>Note that this is not identical to the Fine-Tunning routine described in Felbo et al, 2017, this is why I used the word 'inspired'.</p> <p>The Howard routine can be illustrated as follows:</p> <p> </p> <p>Figure 2. The figure can be described as follows: fine-tune (or train) the last layer for one epoch using a one cycle triangular learning rate. Then fine-tune the next deeper layer for one epoch, with a learning rate that is a factor of 2.5 lower than the previous learning rate (the 2.5 factor is fixed) while keeping the already warmed up layer(s) trainable. Repeat. The vanishing color gradient in the figure attempts to illustrate the decreasing learning rate.</p> <p>Note that I write \"fine-tune (or train) the last layer for one epoch [...]\". However, in practice the user will have to specify the order of the layers to be fine-tuned. This is another reason why I wrote that the fine-tune routines I have implemented are inspired by the work of Felbo and Howard and not identical to their implemenations.</p> <p>The <code>felbo</code> and <code>howard</code> routines can be accessed with via the <code>fine-tune</code> parameters.</p>"},{"location":"examples/07_custom_components.html","title":"07_custom_components","text":"In\u00a0[1]: Copied! <pre>import numpy as np\nimport pandas as pd\nimport os\nimport torch\n\nfrom torch import Tensor\nfrom pytorch_widedeep import Trainer\nfrom pytorch_widedeep.preprocessing import (\n    WidePreprocessor,\n    TabPreprocessor,\n    TextPreprocessor,\n    ImagePreprocessor,\n)\nfrom pytorch_widedeep.models import (\n    Wide,\n    TabMlp,\n    Vision,\n    BasicRNN,\n    WideDeep,\n)\nfrom pytorch_widedeep.losses import RMSELoss\nfrom pytorch_widedeep.initializers import *\nfrom pytorch_widedeep.callbacks import *\nfrom pytorch_widedeep.datasets import load_adult\n</pre> import numpy as np import pandas as pd import os import torch  from torch import Tensor from pytorch_widedeep import Trainer from pytorch_widedeep.preprocessing import (     WidePreprocessor,     TabPreprocessor,     TextPreprocessor,     ImagePreprocessor, ) from pytorch_widedeep.models import (     Wide,     TabMlp,     Vision,     BasicRNN,     WideDeep, ) from pytorch_widedeep.losses import RMSELoss from pytorch_widedeep.initializers import * from pytorch_widedeep.callbacks import * from pytorch_widedeep.datasets import load_adult <pre>/Users/javierrodriguezzaurin/.pyenv/versions/3.10.13/envs/widedeep310/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n  from .autonotebook import tqdm as notebook_tqdm\n</pre> In\u00a0[2]: Copied! <pre>df = pd.read_csv(\"../tmp_data/airbnb/airbnb_sample.csv\")\ndf.head()\n</pre> df = pd.read_csv(\"../tmp_data/airbnb/airbnb_sample.csv\") df.head() Out[2]: id host_id description host_listings_count host_identity_verified neighbourhood_cleansed latitude longitude is_location_exact property_type ... amenity_wide_entrance amenity_wide_entrance_for_guests amenity_wide_entryway amenity_wide_hallways amenity_wifi amenity_window_guards amenity_wine_cooler security_deposit extra_people yield 0 13913.jpg 54730 My bright double bedroom with a large window h... 4.0 f Islington 51.56802 -0.11121 t apartment ... 1 0 0 0 1 0 0 100.0 15.0 12.00 1 15400.jpg 60302 Lots of windows and light.  St Luke's Gardens ... 1.0 t Kensington and Chelsea 51.48796 -0.16898 t apartment ... 0 0 0 0 1 0 0 150.0 0.0 109.50 2 17402.jpg 67564 Open from June 2018 after a 3-year break, we a... 19.0 t Westminster 51.52098 -0.14002 t apartment ... 0 0 0 0 1 0 0 350.0 10.0 149.65 3 24328.jpg 41759 Artist house, bright high ceiling rooms, priva... 2.0 t Wandsworth 51.47298 -0.16376 t other ... 0 0 0 0 1 0 0 250.0 0.0 215.60 4 25023.jpg 102813 Large, all comforts, 2-bed flat; first floor; ... 1.0 f Wandsworth 51.44687 -0.21874 t apartment ... 0 0 0 0 1 0 0 250.0 11.0 79.35 <p>5 rows \u00d7 223 columns</p> In\u00a0[3]: Copied! <pre># There are a number of columns that are already binary. Therefore, no need to one hot encode them\ncrossed_cols = [(\"property_type\", \"room_type\")]\nalready_dummies = [c for c in df.columns if \"amenity\" in c] + [\"has_house_rules\"]\nwide_cols = [\n    \"is_location_exact\",\n    \"property_type\",\n    \"room_type\",\n    \"host_gender\",\n    \"instant_bookable\",\n] + already_dummies\n\ncat_embed_cols = [(c, 16) for c in df.columns if \"catg\" in c] + [\n    (\"neighbourhood_cleansed\", 64),\n    (\"cancellation_policy\", 16),\n]\ncontinuous_cols = [\"latitude\", \"longitude\", \"security_deposit\", \"extra_people\"]\n# it does not make sense to standarised Latitude and Longitude\nalready_standard = [\"latitude\", \"longitude\"]\n\n# text and image colnames\ntext_col = \"description\"\nimg_col = \"id\"\n\n# path to pretrained word embeddings and the images\nword_vectors_path = \"../tmp_data/glove.6B/glove.6B.100d.txt\"\nimg_path = \"../tmp_data/airbnb/property_picture\"\n\n# target\ntarget_col = \"yield\"\n</pre> # There are a number of columns that are already binary. Therefore, no need to one hot encode them crossed_cols = [(\"property_type\", \"room_type\")] already_dummies = [c for c in df.columns if \"amenity\" in c] + [\"has_house_rules\"] wide_cols = [     \"is_location_exact\",     \"property_type\",     \"room_type\",     \"host_gender\",     \"instant_bookable\", ] + already_dummies  cat_embed_cols = [(c, 16) for c in df.columns if \"catg\" in c] + [     (\"neighbourhood_cleansed\", 64),     (\"cancellation_policy\", 16), ] continuous_cols = [\"latitude\", \"longitude\", \"security_deposit\", \"extra_people\"] # it does not make sense to standarised Latitude and Longitude already_standard = [\"latitude\", \"longitude\"]  # text and image colnames text_col = \"description\" img_col = \"id\"  # path to pretrained word embeddings and the images word_vectors_path = \"../tmp_data/glove.6B/glove.6B.100d.txt\" img_path = \"../tmp_data/airbnb/property_picture\"  # target target_col = \"yield\" In\u00a0[4]: Copied! <pre>target = df[target_col].values\n</pre> target = df[target_col].values In\u00a0[5]: Copied! <pre>wide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols)\nX_wide = wide_preprocessor.fit_transform(df)\n\ntab_preprocessor = TabPreprocessor(\n    embed_cols=cat_embed_cols, continuous_cols=continuous_cols\n)\nX_tab = tab_preprocessor.fit_transform(df)\n\ntext_preprocessor = TextPreprocessor(\n    word_vectors_path=word_vectors_path, text_col=text_col\n)\nX_text = text_preprocessor.fit_transform(df)\n\nimage_processor = ImagePreprocessor(img_col=img_col, img_path=img_path)\nX_images = image_processor.fit_transform(df)\n</pre> wide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols) X_wide = wide_preprocessor.fit_transform(df)  tab_preprocessor = TabPreprocessor(     embed_cols=cat_embed_cols, continuous_cols=continuous_cols ) X_tab = tab_preprocessor.fit_transform(df)  text_preprocessor = TextPreprocessor(     word_vectors_path=word_vectors_path, text_col=text_col ) X_text = text_preprocessor.fit_transform(df)  image_processor = ImagePreprocessor(img_col=img_col, img_path=img_path) X_images = image_processor.fit_transform(df) <pre>/Users/javierrodriguezzaurin/Projects/pytorch-widedeep/pytorch_widedeep/preprocessing/tab_preprocessor.py:358: UserWarning: Continuous columns will not be normalised\n  warnings.warn(\"Continuous columns will not be normalised\")\n</pre> <pre>The vocabulary contains 2192 tokens\nIndexing word vectors...\nLoaded 400000 word vectors\nPreparing embeddings matrix...\n2175 words in the vocabulary had ../tmp_data/glove.6B/glove.6B.100d.txt vectors and appear more than 5 times\nReading Images from ../tmp_data/airbnb/property_picture\nResizing\n</pre> <pre>100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 1001/1001 [00:02&lt;00:00, 497.80it/s]\n</pre> <pre>Computing normalisation metrics\n</pre> <p>Now we are ready to build a wide and deep model. Three of the four components we will use are included in this package, and they will be combined with a custom <code>deeptext</code> component. Then the fit process will run with a custom loss function.</p> <p>Let's have a look</p> In\u00a0[6]: Copied! <pre># Linear model\nwide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1)\n\n# DeepDense: 2 Dense layers\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    cat_embed_dropout=0.1,\n    continuous_cols=continuous_cols,\n    mlp_hidden_dims=[128, 64],\n    mlp_dropout=0.1,\n)\n\n# Pretrained Resnet 18\nresnet = Vision(pretrained_model_name=\"resnet18\", n_trainable=0)\n</pre> # Linear model wide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1)  # DeepDense: 2 Dense layers tab_mlp = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     cat_embed_dropout=0.1,     continuous_cols=continuous_cols,     mlp_hidden_dims=[128, 64],     mlp_dropout=0.1, )  # Pretrained Resnet 18 resnet = Vision(pretrained_model_name=\"resnet18\", n_trainable=0) In\u00a0[7]: Copied! <pre>class MyDeepText(nn.Module):\n    def __init__(self, vocab_size, padding_idx=1, embed_dim=100, hidden_dim=64):\n        super(MyDeepText, self).__init__()\n\n        # word/token embeddings\n        self.word_embed = nn.Embedding(vocab_size, embed_dim, padding_idx=padding_idx)\n\n        # stack of RNNs\n        self.rnn = nn.GRU(\n            embed_dim,\n            hidden_dim,\n            num_layers=2,\n            bidirectional=True,\n            batch_first=True,\n        )\n\n        # Remember, this MUST be defined. If not WideDeep will through an error\n        self.output_dim = hidden_dim * 2\n\n    def forward(self, X):\n        embed = self.word_embed(X.long())\n        o, h = self.rnn(embed)\n        return torch.cat((h[-2], h[-1]), dim=1)\n</pre> class MyDeepText(nn.Module):     def __init__(self, vocab_size, padding_idx=1, embed_dim=100, hidden_dim=64):         super(MyDeepText, self).__init__()          # word/token embeddings         self.word_embed = nn.Embedding(vocab_size, embed_dim, padding_idx=padding_idx)          # stack of RNNs         self.rnn = nn.GRU(             embed_dim,             hidden_dim,             num_layers=2,             bidirectional=True,             batch_first=True,         )          # Remember, this MUST be defined. If not WideDeep will through an error         self.output_dim = hidden_dim * 2      def forward(self, X):         embed = self.word_embed(X.long())         o, h = self.rnn(embed)         return torch.cat((h[-2], h[-1]), dim=1) In\u00a0[8]: Copied! <pre>mydeeptext = MyDeepText(vocab_size=len(text_preprocessor.vocab.itos))\n</pre> mydeeptext = MyDeepText(vocab_size=len(text_preprocessor.vocab.itos)) In\u00a0[9]: Copied! <pre>model = WideDeep(wide=wide, deeptabular=tab_mlp, deeptext=mydeeptext, deepimage=resnet)\n</pre> model = WideDeep(wide=wide, deeptabular=tab_mlp, deeptext=mydeeptext, deepimage=resnet) In\u00a0[10]: Copied! <pre>class RMSELoss(nn.Module):\n    def __init__(self):\n        \"\"\"root mean squared error\"\"\"\n        super().__init__()\n        self.mse = nn.MSELoss()\n\n    def forward(self, input: Tensor, target: Tensor) -&gt; Tensor:\n        return torch.sqrt(self.mse(input, target))\n</pre> class RMSELoss(nn.Module):     def __init__(self):         \"\"\"root mean squared error\"\"\"         super().__init__()         self.mse = nn.MSELoss()      def forward(self, input: Tensor, target: Tensor) -&gt; Tensor:         return torch.sqrt(self.mse(input, target)) <p>and now we just instantiate the <code>Trainer</code> as usual. Needless to say, but this runs with 1000 random observations, so loss and metric values are meaningless. This is just an example</p> In\u00a0[11]: Copied! <pre>trainer = Trainer(model, objective=\"regression\", custom_loss_function=RMSELoss())\n</pre> trainer = Trainer(model, objective=\"regression\", custom_loss_function=RMSELoss()) In\u00a0[12]: Copied! <pre>trainer.fit(\n    X_wide=X_wide,\n    X_tab=X_tab,\n    X_text=X_text,\n    X_img=X_images,\n    target=target,\n    n_epochs=1,\n    batch_size=32,\n    val_split=0.2,\n)\n</pre> trainer.fit(     X_wide=X_wide,     X_tab=X_tab,     X_text=X_text,     X_img=X_images,     target=target,     n_epochs=1,     batch_size=32,     val_split=0.2, ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 25/25 [00:23&lt;00:00,  1.07it/s, loss=126]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 7/7 [00:05&lt;00:00,  1.24it/s, loss=97.4]\n</pre> <p>In addition to model components and loss functions, we can also use custom callbacks or custom metrics. The former need to be of type <code>Callback</code> and the latter need to be of type <code>Metric</code>. See:</p> <pre>pytorch-widedeep.callbacks\n</pre> <p>and</p> <pre>pytorch-widedeep.metrics\n</pre> <p>For this example let me use the adult dataset. Again, we first prepare the data as usual</p> In\u00a0[13]: Copied! <pre>df = load_adult(as_frame=True)\ndf.head()\n</pre> df = load_adult(as_frame=True) df.head() Out[13]: age workclass fnlwgt education educational-num marital-status occupation relationship race gender capital-gain capital-loss hours-per-week native-country income 0 25 Private 226802 11th 7 Never-married Machine-op-inspct Own-child Black Male 0 0 40 United-States &lt;=50K 1 38 Private 89814 HS-grad 9 Married-civ-spouse Farming-fishing Husband White Male 0 0 50 United-States &lt;=50K 2 28 Local-gov 336951 Assoc-acdm 12 Married-civ-spouse Protective-serv Husband White Male 0 0 40 United-States &gt;50K 3 44 Private 160323 Some-college 10 Married-civ-spouse Machine-op-inspct Husband Black Male 7688 0 40 United-States &gt;50K 4 18 ? 103497 Some-college 10 Never-married ? Own-child White Female 0 0 30 United-States &lt;=50K In\u00a0[14]: Copied! <pre># For convenience, we'll replace '-' with '_'\ndf.columns = [c.replace(\"-\", \"_\") for c in df.columns]\n# binary target\ndf[\"income_label\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int)\ndf.drop(\"income\", axis=1, inplace=True)\ndf.head()\n</pre> # For convenience, we'll replace '-' with '_' df.columns = [c.replace(\"-\", \"_\") for c in df.columns] # binary target df[\"income_label\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int) df.drop(\"income\", axis=1, inplace=True) df.head() Out[14]: age workclass fnlwgt education educational_num marital_status occupation relationship race gender capital_gain capital_loss hours_per_week native_country income_label 0 25 Private 226802 11th 7 Never-married Machine-op-inspct Own-child Black Male 0 0 40 United-States 0 1 38 Private 89814 HS-grad 9 Married-civ-spouse Farming-fishing Husband White Male 0 0 50 United-States 0 2 28 Local-gov 336951 Assoc-acdm 12 Married-civ-spouse Protective-serv Husband White Male 0 0 40 United-States 1 3 44 Private 160323 Some-college 10 Married-civ-spouse Machine-op-inspct Husband Black Male 7688 0 40 United-States 1 4 18 ? 103497 Some-college 10 Never-married ? Own-child White Female 0 0 30 United-States 0 In\u00a0[15]: Copied! <pre># Define wide, crossed and deep tabular columns\nwide_cols = [\n    \"workclass\",\n    \"education\",\n    \"marital_status\",\n    \"occupation\",\n    \"relationship\",\n    \"race\",\n    \"gender\",\n    \"native_country\",\n]\ncrossed_cols = [(\"education\", \"occupation\"), (\"native_country\", \"occupation\")]\ncat_embed_cols = [\n    \"workclass\",\n    \"education\",\n    \"marital_status\",\n    \"occupation\",\n    \"relationship\",\n    \"race\",\n    \"gender\",\n    \"capital_gain\",\n    \"capital_loss\",\n    \"native_country\",\n]\ncontinuous_cols = [\"age\", \"hours_per_week\"]\ntarget_col = \"income_label\"\ntarget = df[target_col].values\n</pre> # Define wide, crossed and deep tabular columns wide_cols = [     \"workclass\",     \"education\",     \"marital_status\",     \"occupation\",     \"relationship\",     \"race\",     \"gender\",     \"native_country\", ] crossed_cols = [(\"education\", \"occupation\"), (\"native_country\", \"occupation\")] cat_embed_cols = [     \"workclass\",     \"education\",     \"marital_status\",     \"occupation\",     \"relationship\",     \"race\",     \"gender\",     \"capital_gain\",     \"capital_loss\",     \"native_country\", ] continuous_cols = [\"age\", \"hours_per_week\"] target_col = \"income_label\" target = df[target_col].values In\u00a0[16]: Copied! <pre># wide\nwide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols)\nX_wide = wide_preprocessor.fit_transform(df)\n\n# deeptabular\ntab_preprocessor = TabPreprocessor(\n    embed_cols=cat_embed_cols, continuous_cols=continuous_cols\n)\nX_tab = tab_preprocessor.fit_transform(df)\n</pre> # wide wide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols) X_wide = wide_preprocessor.fit_transform(df)  # deeptabular tab_preprocessor = TabPreprocessor(     embed_cols=cat_embed_cols, continuous_cols=continuous_cols ) X_tab = tab_preprocessor.fit_transform(df) <pre>/Users/javierrodriguezzaurin/Projects/pytorch-widedeep/pytorch_widedeep/preprocessing/tab_preprocessor.py:358: UserWarning: Continuous columns will not be normalised\n  warnings.warn(\"Continuous columns will not be normalised\")\n</pre> In\u00a0[17]: Copied! <pre>wide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1)\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=continuous_cols,\n    mlp_hidden_dims=[128, 64],\n    mlp_dropout=0.2,\n    mlp_activation=\"leaky_relu\",\n)\nmodel = WideDeep(wide=wide, deeptabular=tab_mlp)\n</pre> wide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1) tab_mlp = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     continuous_cols=continuous_cols,     mlp_hidden_dims=[128, 64],     mlp_dropout=0.2,     mlp_activation=\"leaky_relu\", ) model = WideDeep(wide=wide, deeptabular=tab_mlp) In\u00a0[18]: Copied! <pre>from pytorch_widedeep.metrics import Metric\n</pre> from pytorch_widedeep.metrics import Metric In\u00a0[19]: Copied! <pre>class Accuracy(Metric):\n    def __init__(self, top_k: int = 1):\n        super(Accuracy, self).__init__()\n\n        self.top_k = top_k\n        self.correct_count = 0\n        self.total_count = 0\n\n        # \u00a0metric name needs to be defined\n        self._name = \"acc\"\n\n    def reset(self):\n        self.correct_count = 0\n        self.total_count = 0\n\n    def __call__(self, y_pred: Tensor, y_true: Tensor) -&gt; np.ndarray:\n        num_classes = y_pred.size(1)\n\n        if num_classes == 1:\n            y_pred = y_pred.round()\n            y_true = y_true\n        elif num_classes &gt; 1:\n            y_pred = y_pred.topk(self.top_k, 1)[1]\n            y_true = y_true.view(-1, 1).expand_as(y_pred)\n\n        self.correct_count += y_pred.eq(y_true).sum().item()\n        self.total_count += len(y_pred)\n        accuracy = float(self.correct_count) / float(self.total_count)\n        return np.array(accuracy)\n</pre> class Accuracy(Metric):     def __init__(self, top_k: int = 1):         super(Accuracy, self).__init__()          self.top_k = top_k         self.correct_count = 0         self.total_count = 0          # \u00a0metric name needs to be defined         self._name = \"acc\"      def reset(self):         self.correct_count = 0         self.total_count = 0      def __call__(self, y_pred: Tensor, y_true: Tensor) -&gt; np.ndarray:         num_classes = y_pred.size(1)          if num_classes == 1:             y_pred = y_pred.round()             y_true = y_true         elif num_classes &gt; 1:             y_pred = y_pred.topk(self.top_k, 1)[1]             y_true = y_true.view(-1, 1).expand_as(y_pred)          self.correct_count += y_pred.eq(y_true).sum().item()         self.total_count += len(y_pred)         accuracy = float(self.correct_count) / float(self.total_count)         return np.array(accuracy) In\u00a0[20]: Copied! <pre># have a look to the class\nfrom pytorch_widedeep.callbacks import Callback\n</pre> # have a look to the class from pytorch_widedeep.callbacks import Callback In\u00a0[21]: Copied! <pre>class SillyCallback(Callback):\n    def on_train_begin(self, logs=None):\n        # recordings will be the trainer object attributes\n        self.trainer.silly_callback = {}\n\n        self.trainer.silly_callback[\"beginning\"] = []\n        self.trainer.silly_callback[\"end\"] = []\n\n    def on_epoch_begin(self, epoch, logs=None):\n        self.trainer.silly_callback[\"beginning\"].append(epoch + 1)\n\n    def on_epoch_end(self, epoch, logs=None, metric=None):\n        self.trainer.silly_callback[\"end\"].append(epoch + 1)\n</pre> class SillyCallback(Callback):     def on_train_begin(self, logs=None):         # recordings will be the trainer object attributes         self.trainer.silly_callback = {}          self.trainer.silly_callback[\"beginning\"] = []         self.trainer.silly_callback[\"end\"] = []      def on_epoch_begin(self, epoch, logs=None):         self.trainer.silly_callback[\"beginning\"].append(epoch + 1)      def on_epoch_end(self, epoch, logs=None, metric=None):         self.trainer.silly_callback[\"end\"].append(epoch + 1) <p>and now, as usual:</p> In\u00a0[22]: Copied! <pre>trainer = Trainer(\n    model, objective=\"binary\", metrics=[Accuracy], callbacks=[SillyCallback]\n)\n</pre> trainer = Trainer(     model, objective=\"binary\", metrics=[Accuracy], callbacks=[SillyCallback] ) In\u00a0[23]: Copied! <pre>trainer.fit(\n    X_wide=X_wide, X_tab=X_tab, target=target, n_epochs=5, batch_size=64, val_split=0.2\n)\n</pre> trainer.fit(     X_wide=X_wide, X_tab=X_tab, target=target, n_epochs=5, batch_size=64, val_split=0.2 ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 611/611 [00:06&lt;00:00, 94.39it/s, loss=0.411, metrics={'acc': 0.814}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 121.91it/s, loss=0.327, metrics={'acc': 0.8449}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 611/611 [00:07&lt;00:00, 85.39it/s, loss=0.324, metrics={'acc': 0.8495}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 88.68it/s, loss=0.298, metrics={'acc': 0.8612}]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 611/611 [00:08&lt;00:00, 74.35it/s, loss=0.302, metrics={'acc': 0.8593}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 100.51it/s, loss=0.29, metrics={'acc': 0.8665}]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 611/611 [00:08&lt;00:00, 73.83it/s, loss=0.292, metrics={'acc': 0.8637}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 105.98it/s, loss=0.286, metrics={'acc': 0.8695}]\nepoch 5: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 611/611 [00:08&lt;00:00, 72.15it/s, loss=0.286, metrics={'acc': 0.866}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 92.27it/s, loss=0.284, metrics={'acc': 0.8698}]\n</pre> In\u00a0[24]: Copied! <pre>trainer.silly_callback\n</pre> trainer.silly_callback Out[24]: <pre>{'beginning': [1, 2, 3, 4, 5], 'end': [1, 2, 3, 4, 5]}</pre>"},{"location":"examples/07_custom_components.html#custom-components","title":"Custom components\u00b6","text":"<p>As I mentioned earlier in the example notebooks, and also in the <code>README</code>, it is possible to customise almost every component in <code>pytorch-widedeep</code>.</p> <p>Let's now go through a couple of simple examples to illustrate how that could be done.</p> <p>First let's load and process the data \"as usual\", let's start with a regression and the airbnb dataset.</p>"},{"location":"examples/07_custom_components.html#custom-deeptext","title":"Custom <code>deeptext</code>\u00b6","text":"<p>Standard Pytorch model</p>"},{"location":"examples/07_custom_components.html#custom-loss-function","title":"Custom loss function\u00b6","text":"<p>Loss functions must simply inherit pytorch's <code>nn.Module</code>. For example, let's say we want to use <code>RMSE</code> (note that this is already available in the package, but I will pass it here as a custom loss for illustration purposes)</p>"},{"location":"examples/07_custom_components.html#custom-metric","title":"Custom metric\u00b6","text":"<p>Let's say we want to use our own accuracy metric (again, this is already available in the package, but I will pass it here as a custom loss for illustration purposes).</p> <p>This could be done as:</p>"},{"location":"examples/07_custom_components.html#custom-callback","title":"Custom Callback\u00b6","text":"<p>Let's code a callback that records the current epoch at the beginning and the end of each epoch (silly, but you know, this is just an example)</p>"},{"location":"examples/08_custom_dataLoader_imbalanced_dataset.html","title":"08_custom_dataLoader_imbalanced_dataset","text":"<ul> <li>In this notebook we will use the higly imbalanced Protein Homology Dataset from KDD cup 2004</li> </ul> <pre><code>* The first element of each line is a BLOCK ID that denotes to which native sequence this example belongs. There is a unique BLOCK ID for each native sequence. BLOCK IDs are integers running from 1 to 303 (one for each native sequence, i.e. for each query). BLOCK IDs were assigned before the blocks were split into the train and test sets, so they do not run consecutively in either file.\n* The second element of each line is an EXAMPLE ID that uniquely describes the example. You will need this EXAMPLE ID and the BLOCK ID when you submit results.\n* The third element is the class of the example. Proteins that are homologous to the native sequence are denoted by 1, non-homologous proteins (i.e. decoys) by 0. Test examples have a \"?\" in this position.\n* All following elements are feature values. There are 74 feature values in each line. The features describe the match (e.g. the score of a sequence alignment) between the native protein sequence and the sequence that is tested for homology.\n</code></pre> In\u00a0[15]: Copied! <pre>from pytorch_widedeep import Trainer\nfrom pytorch_widedeep.preprocessing import TabPreprocessor\nfrom pytorch_widedeep.models import TabMlp, WideDeep\nfrom pytorch_widedeep.dataloaders import DataLoaderImbalanced\nfrom pytorch_widedeep.metrics import Accuracy, Precision\nfrom pytorch_widedeep.initializers import XavierNormal\nfrom pytorch_widedeep.datasets import load_bio_kdd04\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import classification_report\n\nimport time\nimport datetime\n\nimport warnings\n\nwarnings.filterwarnings(\"ignore\", category=DeprecationWarning)\n</pre> from pytorch_widedeep import Trainer from pytorch_widedeep.preprocessing import TabPreprocessor from pytorch_widedeep.models import TabMlp, WideDeep from pytorch_widedeep.dataloaders import DataLoaderImbalanced from pytorch_widedeep.metrics import Accuracy, Precision from pytorch_widedeep.initializers import XavierNormal from pytorch_widedeep.datasets import load_bio_kdd04  from sklearn.model_selection import train_test_split from sklearn.metrics import classification_report  import time import datetime  import warnings  warnings.filterwarnings(\"ignore\", category=DeprecationWarning) In\u00a0[10]: Copied! <pre>df = load_bio_kdd04(as_frame=True)\n# drop columns we won't need in this example\ndf.drop(columns=[\"EXAMPLE_ID\", \"BLOCK_ID\"], inplace=True)\n\ndf_train, df_valid = train_test_split(\n    df, test_size=0.2, stratify=df[\"target\"], random_state=1\n)\ndf_valid, df_test = train_test_split(\n    df_valid, test_size=0.5, stratify=df_valid[\"target\"], random_state=1\n)\n\ncontinuous_cols = df.drop(columns=[\"target\"]).columns.values.tolist()\n</pre> df = load_bio_kdd04(as_frame=True) # drop columns we won't need in this example df.drop(columns=[\"EXAMPLE_ID\", \"BLOCK_ID\"], inplace=True)  df_train, df_valid = train_test_split(     df, test_size=0.2, stratify=df[\"target\"], random_state=1 ) df_valid, df_test = train_test_split(     df_valid, test_size=0.5, stratify=df_valid[\"target\"], random_state=1 )  continuous_cols = df.drop(columns=[\"target\"]).columns.values.tolist() In\u00a0[11]: Copied! <pre># deeptabular\ntab_preprocessor = TabPreprocessor(continuous_cols=continuous_cols, scale=True)\nX_tab_train = tab_preprocessor.fit_transform(df_train)\nX_tab_valid = tab_preprocessor.transform(df_valid)\nX_tab_test = tab_preprocessor.transform(df_test)\n\n# target\ny_train = df_train[\"target\"].values\ny_valid = df_valid[\"target\"].values\ny_test = df_test[\"target\"].values\n</pre> # deeptabular tab_preprocessor = TabPreprocessor(continuous_cols=continuous_cols, scale=True) X_tab_train = tab_preprocessor.fit_transform(df_train) X_tab_valid = tab_preprocessor.transform(df_valid) X_tab_test = tab_preprocessor.transform(df_test)  # target y_train = df_train[\"target\"].values y_valid = df_valid[\"target\"].values y_test = df_test[\"target\"].values In\u00a0[12]: Copied! <pre>deeptabular = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    continuous_cols=tab_preprocessor.continuous_cols,\n    mlp_hidden_dims=[64, 32],\n)\nmodel = WideDeep(deeptabular=deeptabular)\nmodel\n</pre> deeptabular = TabMlp(     column_idx=tab_preprocessor.column_idx,     continuous_cols=tab_preprocessor.continuous_cols,     mlp_hidden_dims=[64, 32], ) model = WideDeep(deeptabular=deeptabular) model Out[12]: <pre>WideDeep(\n  (deeptabular): Sequential(\n    (0): TabMlp(\n      (cont_norm): Identity()\n      (encoder): MLP(\n        (mlp): Sequential(\n          (dense_layer_0): Sequential(\n            (0): Linear(in_features=74, out_features=64, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_1): Sequential(\n            (0): Linear(in_features=64, out_features=32, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=32, out_features=1, bias=True)\n  )\n)</pre> In\u00a0[13]: Copied! <pre>trainer = Trainer(\n    model,\n    objective=\"binary\",\n    metrics=[Accuracy(), Precision()],\n    verbose=1,\n)\n</pre> trainer = Trainer(     model,     objective=\"binary\",     metrics=[Accuracy(), Precision()],     verbose=1, ) In\u00a0[16]: Copied! <pre>train_dataloader = DataLoaderImbalanced(kwargs={\"oversample_mul\": 5})\n\n# in reality one should not really do this, but just to illustrate\neval_dataloader = DataLoaderImbalanced(kwargs={\"oversample_mul\": 5})\n\ntrainer.fit(\n    X_train={\"X_tab\": X_tab_train, \"target\": y_train},\n    X_val={\"X_tab\": X_tab_valid, \"target\": y_valid},\n    n_epochs=1,\n    batch_size=32,\n    train_dataloader=train_dataloader,\n    eval_dataloader=eval_dataloader,\n)\n\npreds = trainer.predict(X_tab=X_tab_test)\nprint(classification_report(df_test[\"target\"].to_list(), preds))\n</pre> train_dataloader = DataLoaderImbalanced(kwargs={\"oversample_mul\": 5})  # in reality one should not really do this, but just to illustrate eval_dataloader = DataLoaderImbalanced(kwargs={\"oversample_mul\": 5})  trainer.fit(     X_train={\"X_tab\": X_tab_train, \"target\": y_train},     X_val={\"X_tab\": X_tab_valid, \"target\": y_valid},     n_epochs=1,     batch_size=32,     train_dataloader=train_dataloader,     eval_dataloader=eval_dataloader, )  preds = trainer.predict(X_tab=X_tab_test) print(classification_report(df_test[\"target\"].to_list(), preds)) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 2074/2074 [00:10&lt;00:00, 194.90it/s, loss=0.115, metrics={'acc': 0.959, 'prec': 0.9637}] \nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 258/258 [00:00&lt;00:00, 330.04it/s, loss=0.189, metrics={'acc': 0.9147, 'prec': 0.9825}]\npredict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 456/456 [00:00&lt;00:00, 679.61it/s]\n</pre> <pre>              precision    recall  f1-score   support\n\n           0       1.00      0.98      0.99     14446\n           1       0.26      0.94      0.41       130\n\n    accuracy                           0.98     14576\n   macro avg       0.63      0.96      0.70     14576\nweighted avg       0.99      0.98      0.98     14576\n\n</pre>"},{"location":"examples/08_custom_dataLoader_imbalanced_dataset.html#custom-dataloader-for-imbalanced-dataset","title":"Custom DataLoader for Imbalanced dataset\u00b6","text":""},{"location":"examples/08_custom_dataLoader_imbalanced_dataset.html#initial-imports","title":"Initial imports\u00b6","text":""},{"location":"examples/09_extracting_embeddings.html","title":"09_extracting_embeddings","text":"In\u00a0[1]: Copied! <pre>import numpy as np\nimport pandas as pd\nimport torch\n\nfrom pytorch_widedeep.preprocessing import TabPreprocessor\nfrom pytorch_widedeep.training import Trainer\nfrom pytorch_widedeep.models import FTTransformer, WideDeep\nfrom pytorch_widedeep.metrics import Accuracy\nfrom pytorch_widedeep import Tab2Vec\nfrom pytorch_widedeep.datasets import load_adult\n</pre> import numpy as np import pandas as pd import torch  from pytorch_widedeep.preprocessing import TabPreprocessor from pytorch_widedeep.training import Trainer from pytorch_widedeep.models import FTTransformer, WideDeep from pytorch_widedeep.metrics import Accuracy from pytorch_widedeep import Tab2Vec from pytorch_widedeep.datasets import load_adult <pre>/Users/javierrodriguezzaurin/.pyenv/versions/3.10.15/envs/widedeep310/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n  from .autonotebook import tqdm as notebook_tqdm\n</pre> In\u00a0[2]: Copied! <pre>df = load_adult(as_frame=True)\ndf.head()\n</pre> df = load_adult(as_frame=True) df.head() Out[2]: age workclass fnlwgt education educational-num marital-status occupation relationship race gender capital-gain capital-loss hours-per-week native-country income 0 25 Private 226802 11th 7 Never-married Machine-op-inspct Own-child Black Male 0 0 40 United-States &lt;=50K 1 38 Private 89814 HS-grad 9 Married-civ-spouse Farming-fishing Husband White Male 0 0 50 United-States &lt;=50K 2 28 Local-gov 336951 Assoc-acdm 12 Married-civ-spouse Protective-serv Husband White Male 0 0 40 United-States &gt;50K 3 44 Private 160323 Some-college 10 Married-civ-spouse Machine-op-inspct Husband Black Male 7688 0 40 United-States &gt;50K 4 18 ? 103497 Some-college 10 Never-married ? Own-child White Female 0 0 30 United-States &lt;=50K In\u00a0[3]: Copied! <pre># For convenience, we'll replace '-' with '_'\ndf.columns = [c.replace(\"-\", \"_\") for c in df.columns]\n# binary target\ndf[\"target\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int)\ndf.drop([\"income\", \"educational_num\"], axis=1, inplace=True)\n\ndf.head()\n</pre> # For convenience, we'll replace '-' with '_' df.columns = [c.replace(\"-\", \"_\") for c in df.columns] # binary target df[\"target\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int) df.drop([\"income\", \"educational_num\"], axis=1, inplace=True)  df.head() Out[3]: age workclass fnlwgt education marital_status occupation relationship race gender capital_gain capital_loss hours_per_week native_country target 0 25 Private 226802 11th Never-married Machine-op-inspct Own-child Black Male 0 0 40 United-States 0 1 38 Private 89814 HS-grad Married-civ-spouse Farming-fishing Husband White Male 0 0 50 United-States 0 2 28 Local-gov 336951 Assoc-acdm Married-civ-spouse Protective-serv Husband White Male 0 0 40 United-States 1 3 44 Private 160323 Some-college Married-civ-spouse Machine-op-inspct Husband Black Male 7688 0 40 United-States 1 4 18 ? 103497 Some-college Never-married ? Own-child White Female 0 0 30 United-States 0 In\u00a0[4]: Copied! <pre>cat_cols, cont_cols = [], []\nfor col in df.columns:\n    # 50 is just a random number I choose here for this example\n    if df[col].dtype == \"O\" or df[col].nunique() &lt; 50 and col != \"target\":\n        cat_cols.append(col)\n    elif col != \"target\":\n        cont_cols.append(col)\ntarget_col = \"target\"\n</pre> cat_cols, cont_cols = [], [] for col in df.columns:     # 50 is just a random number I choose here for this example     if df[col].dtype == \"O\" or df[col].nunique() &lt; 50 and col != \"target\":         cat_cols.append(col)     elif col != \"target\":         cont_cols.append(col) target_col = \"target\" In\u00a0[5]: Copied! <pre>target = df[target_col].values\n\ntab_preprocessor = TabPreprocessor(\n    embed_cols=cat_cols, continuous_cols=cont_cols, for_transformer=True\n)\nX_tab = tab_preprocessor.fit_transform(df)\n</pre> target = df[target_col].values  tab_preprocessor = TabPreprocessor(     embed_cols=cat_cols, continuous_cols=cont_cols, for_transformer=True ) X_tab = tab_preprocessor.fit_transform(df) <pre>/Users/javierrodriguezzaurin/Projects/pytorch-widedeep/pytorch_widedeep/preprocessing/tab_preprocessor.py:364: UserWarning: Continuous columns will not be normalised\n  warnings.warn(\"Continuous columns will not be normalised\")\n</pre> In\u00a0[6]: Copied! <pre>ft_transformer = FTTransformer(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=tab_preprocessor.continuous_cols,\n    embed_continuous_method=\"standard\",\n    n_blocks=2,\n    n_heads=4,\n    input_dim=16,\n)\n</pre> ft_transformer = FTTransformer(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     continuous_cols=tab_preprocessor.continuous_cols,     embed_continuous_method=\"standard\",     n_blocks=2,     n_heads=4,     input_dim=16, ) In\u00a0[7]: Copied! <pre>model = WideDeep(deeptabular=ft_transformer)\ntrainer = Trainer(model, objective=\"binary\", metrics=[Accuracy])\ntrainer.fit(X_tab=X_tab, target=target, n_epochs=1, batch_size=256, val_split=0.2)\n</pre> model = WideDeep(deeptabular=ft_transformer) trainer = Trainer(model, objective=\"binary\", metrics=[Accuracy]) trainer.fit(X_tab=X_tab, target=target, n_epochs=1, batch_size=256, val_split=0.2) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:02&lt;00:00, 56.56it/s, loss=48.7, metrics={'acc': 0.7371}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 39/39 [00:00&lt;00:00, 129.23it/s, loss=7.44, metrics={'acc': 0.6928}]\n</pre> In\u00a0[8]: Copied! <pre>t2v = Tab2Vec(model=model, tab_preprocessor=tab_preprocessor, device=\"mps\")\n</pre> t2v = Tab2Vec(model=model, tab_preprocessor=tab_preprocessor, device=\"mps\") In\u00a0[9]: Copied! <pre># assuming is a test set with target col\nX_vec, y = t2v.transform(df.sample(100), target_col=\"target\")\n</pre> # assuming is a test set with target col X_vec, y = t2v.transform(df.sample(100), target_col=\"target\") In\u00a0[10]: Copied! <pre># X vec is the dataframe turned into the embeddings\nX_vec.shape\n</pre> # X vec is the dataframe turned into the embeddings X_vec.shape Out[10]: <pre>(100, 208)</pre> <p><code>208 = input_dim (16) * n_cols (13)</code></p> In\u00a0[11]: Copied! <pre># ...or if we don't have target col\nX_vec = t2v.transform(df.sample(100))\n</pre> # ...or if we don't have target col X_vec = t2v.transform(df.sample(100))"},{"location":"examples/09_extracting_embeddings.html#extracting-embeddings","title":"Extracting embeddings\u00b6","text":"<p>This notebook is a simple guide to extracting learned feature embeddings using Tab2Vec</p>"},{"location":"examples/10_3rd_party_integration-RayTune_WnB.html","title":"10_3rd_party_integration-RayTune_WnB","text":"In\u00a0[11]: Copied! <pre>from typing import Optional, Dict\nimport os\n\nimport numpy as np\nimport pandas as pd\nimport torch\nimport wandb\nfrom torch.optim import SGD, lr_scheduler\n\nfrom pytorch_widedeep import Trainer\nfrom pytorch_widedeep.preprocessing import TabPreprocessor\nfrom pytorch_widedeep.models import TabMlp, WideDeep\nfrom torchmetrics import F1Score as F1_torchmetrics\nfrom torchmetrics import Accuracy as Accuracy_torchmetrics\nfrom torchmetrics import Precision as Precision_torchmetrics\nfrom torchmetrics import Recall as Recall_torchmetrics\nfrom pytorch_widedeep.metrics import Accuracy, Recall, Precision, F1Score, R2Score\nfrom pytorch_widedeep.initializers import XavierNormal\nfrom pytorch_widedeep.callbacks import (\n    EarlyStopping,\n    ModelCheckpoint,\n    Callback,\n)\nfrom pytorch_widedeep.datasets import load_bio_kdd04\n\nfrom sklearn.model_selection import train_test_split\nimport warnings\n\nwarnings.filterwarnings(\"ignore\", category=DeprecationWarning)\n\nfrom ray import tune\nfrom ray.tune.schedulers import AsyncHyperBandScheduler\nfrom ray.tune import JupyterNotebookReporter\nfrom ray.air.integrations.wandb import WandbLoggerCallback\n\n# from ray.tune.integration.wandb import wandb_mixin\n\nimport tracemalloc\n\ntracemalloc.start()\n\n# increase displayed columns in jupyter notebook\npd.set_option(\"display.max_columns\", 200)\npd.set_option(\"display.max_rows\", 300)\n</pre> from typing import Optional, Dict import os  import numpy as np import pandas as pd import torch import wandb from torch.optim import SGD, lr_scheduler  from pytorch_widedeep import Trainer from pytorch_widedeep.preprocessing import TabPreprocessor from pytorch_widedeep.models import TabMlp, WideDeep from torchmetrics import F1Score as F1_torchmetrics from torchmetrics import Accuracy as Accuracy_torchmetrics from torchmetrics import Precision as Precision_torchmetrics from torchmetrics import Recall as Recall_torchmetrics from pytorch_widedeep.metrics import Accuracy, Recall, Precision, F1Score, R2Score from pytorch_widedeep.initializers import XavierNormal from pytorch_widedeep.callbacks import (     EarlyStopping,     ModelCheckpoint,     Callback, ) from pytorch_widedeep.datasets import load_bio_kdd04  from sklearn.model_selection import train_test_split import warnings  warnings.filterwarnings(\"ignore\", category=DeprecationWarning)  from ray import tune from ray.tune.schedulers import AsyncHyperBandScheduler from ray.tune import JupyterNotebookReporter from ray.air.integrations.wandb import WandbLoggerCallback  # from ray.tune.integration.wandb import wandb_mixin  import tracemalloc  tracemalloc.start()  # increase displayed columns in jupyter notebook pd.set_option(\"display.max_columns\", 200) pd.set_option(\"display.max_rows\", 300) In\u00a0[12]: Copied! <pre>class RayTuneReporter(Callback):\n    r\"\"\"Callback that allows reporting history and lr_history values to RayTune\n    during Hyperparameter tuning\n\n    Callbacks are passed as input parameters to the ``Trainer`` class. See\n    :class:`pytorch_widedeep.trainer.Trainer`\n\n    For examples see the examples folder at:\n\n        .. code-block:: bash\n\n            /examples/12_HyperParameter_tuning_w_RayTune.ipynb\n    \"\"\"\n\n    def on_epoch_end(\n        self, epoch: int, logs: Optional[Dict] = None, metric: Optional[float] = None\n    ):\n        report_dict = {}\n        for k, v in self.trainer.history.items():\n            report_dict.update({k: v[-1]})\n        if hasattr(self.trainer, \"lr_history\"):\n            for k, v in self.trainer.lr_history.items():\n                report_dict.update({k: v[-1]})\n        tune.report(report_dict)\n\n\nclass WnBReportBest(Callback):\n    r\"\"\"Callback that allows reporting best performance of a run to WnB\n    during Hyperparameter tuning. It is an adjusted pytorch_widedeep.callbacks.ModelCheckpoint\n    with added WnB and removed checkpoint saving.\n\n    Callbacks are passed as input parameters to the ``Trainer`` class.\n\n    Parameters\n    ----------\n    wb: obj\n        Weights&amp;Biases API interface to report single best result usable for\n        comparisson of multiple paramater combinations by, for example,\n        `parallel coordinates\n        &lt;https://docs.wandb.ai/ref/app/features/panels/parallel-coordinates&gt;`_.\n        E.g W&amp;B summary report `wandb.run.summary[\"best\"]`.\n    monitor: str, default=\"loss\"\n        quantity to monitor. Typically `'val_loss'` or metric name\n        (e.g. `'val_acc'`)\n    mode: str, default=\"auto\"\n        If ``save_best_only=True``, the decision to overwrite the current save\n        file is made based on either the maximization or the minimization of\n        the monitored quantity. For `'acc'`, this should be `'max'`, for\n        `'loss'` this should be `'min'`, etc. In `'auto'` mode, the\n        direction is automatically inferred from the name of the monitored\n        quantity.\n\n    \"\"\"\n\n    def __init__(\n        self,\n        wb: object,\n        monitor: str = \"val_loss\",\n        mode: str = \"auto\",\n    ):\n        super(WnBReportBest, self).__init__()\n\n        self.monitor = monitor\n        self.mode = mode\n        self.wb = wb\n\n        if self.mode not in [\"auto\", \"min\", \"max\"]:\n            warnings.warn(\n                \"WnBReportBest mode %s is unknown, \"\n                \"fallback to auto mode.\" % (self.mode),\n                RuntimeWarning,\n            )\n            self.mode = \"auto\"\n        if self.mode == \"min\":\n            self.monitor_op = np.less\n            self.best = np.Inf\n        elif self.mode == \"max\":\n            self.monitor_op = np.greater  # type: ignore[assignment]\n            self.best = -np.Inf\n        else:\n            if self._is_metric(self.monitor):\n                self.monitor_op = np.greater  # type: ignore[assignment]\n                self.best = -np.Inf\n            else:\n                self.monitor_op = np.less\n                self.best = np.Inf\n\n    def on_epoch_end(  # noqa: C901\n        self, epoch: int, logs: Optional[Dict] = None, metric: Optional[float] = None\n    ):\n        logs = logs or {}\n        current = logs.get(self.monitor)\n        if current is not None:\n            if self.monitor_op(current, self.best):\n                self.wb.run.summary[\"best\"] = current  # type: ignore[attr-defined]\n                self.best = current\n                self.best_epoch = epoch\n\n    @staticmethod\n    def _is_metric(monitor: str):\n        \"copied from pytorch_widedeep.callbacks\"\n        if any([s in monitor for s in [\"acc\", \"prec\", \"rec\", \"fscore\", \"f1\", \"f2\"]]):\n            return True\n        else:\n            return False\n</pre> class RayTuneReporter(Callback):     r\"\"\"Callback that allows reporting history and lr_history values to RayTune     during Hyperparameter tuning      Callbacks are passed as input parameters to the ``Trainer`` class. See     :class:`pytorch_widedeep.trainer.Trainer`      For examples see the examples folder at:          .. code-block:: bash              /examples/12_HyperParameter_tuning_w_RayTune.ipynb     \"\"\"      def on_epoch_end(         self, epoch: int, logs: Optional[Dict] = None, metric: Optional[float] = None     ):         report_dict = {}         for k, v in self.trainer.history.items():             report_dict.update({k: v[-1]})         if hasattr(self.trainer, \"lr_history\"):             for k, v in self.trainer.lr_history.items():                 report_dict.update({k: v[-1]})         tune.report(report_dict)   class WnBReportBest(Callback):     r\"\"\"Callback that allows reporting best performance of a run to WnB     during Hyperparameter tuning. It is an adjusted pytorch_widedeep.callbacks.ModelCheckpoint     with added WnB and removed checkpoint saving.      Callbacks are passed as input parameters to the ``Trainer`` class.      Parameters     ----------     wb: obj         Weights&amp;Biases API interface to report single best result usable for         comparisson of multiple paramater combinations by, for example,         `parallel coordinates         `_.         E.g W&amp;B summary report `wandb.run.summary[\"best\"]`.     monitor: str, default=\"loss\"         quantity to monitor. Typically `'val_loss'` or metric name         (e.g. `'val_acc'`)     mode: str, default=\"auto\"         If ``save_best_only=True``, the decision to overwrite the current save         file is made based on either the maximization or the minimization of         the monitored quantity. For `'acc'`, this should be `'max'`, for         `'loss'` this should be `'min'`, etc. In `'auto'` mode, the         direction is automatically inferred from the name of the monitored         quantity.      \"\"\"      def __init__(         self,         wb: object,         monitor: str = \"val_loss\",         mode: str = \"auto\",     ):         super(WnBReportBest, self).__init__()          self.monitor = monitor         self.mode = mode         self.wb = wb          if self.mode not in [\"auto\", \"min\", \"max\"]:             warnings.warn(                 \"WnBReportBest mode %s is unknown, \"                 \"fallback to auto mode.\" % (self.mode),                 RuntimeWarning,             )             self.mode = \"auto\"         if self.mode == \"min\":             self.monitor_op = np.less             self.best = np.Inf         elif self.mode == \"max\":             self.monitor_op = np.greater  # type: ignore[assignment]             self.best = -np.Inf         else:             if self._is_metric(self.monitor):                 self.monitor_op = np.greater  # type: ignore[assignment]                 self.best = -np.Inf             else:                 self.monitor_op = np.less                 self.best = np.Inf      def on_epoch_end(  # noqa: C901         self, epoch: int, logs: Optional[Dict] = None, metric: Optional[float] = None     ):         logs = logs or {}         current = logs.get(self.monitor)         if current is not None:             if self.monitor_op(current, self.best):                 self.wb.run.summary[\"best\"] = current  # type: ignore[attr-defined]                 self.best = current                 self.best_epoch = epoch      @staticmethod     def _is_metric(monitor: str):         \"copied from pytorch_widedeep.callbacks\"         if any([s in monitor for s in [\"acc\", \"prec\", \"rec\", \"fscore\", \"f1\", \"f2\"]]):             return True         else:             return False In\u00a0[13]: Copied! <pre>df = load_bio_kdd04(as_frame=True)\ndf.head()\n</pre> df = load_bio_kdd04(as_frame=True) df.head() Out[13]: EXAMPLE_ID BLOCK_ID target 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 0 279 261532 0 52.0 32.69 0.30 2.5 20.0 1256.8 -0.89 0.33 11.0 -55.0 267.2 0.52 0.05 -2.36 49.6 252.0 0.43 1.16 -2.06 -33.0 -123.2 1.60 -0.49 -6.06 65.0 296.1 -0.28 -0.26 -3.83 -22.6 -170.0 3.06 -1.05 -3.29 22.9 286.3 0.12 2.58 4.08 -33.0 -178.9 1.88 0.53 -7.0 -44.0 1987.0 -5.41 0.95 -4.0 -57.0 722.9 -3.26 -0.55 -7.5 125.5 1547.2 -0.36 1.12 9.0 -37.0 72.5 0.47 0.74 -11.0 -8.0 1595.1 -1.64 2.83 -2.0 -50.0 445.2 -0.35 0.26 0.76 1 279 261533 0 58.0 33.33 0.00 16.5 9.5 608.1 0.50 0.07 20.5 -52.5 521.6 -1.08 0.58 -0.02 -3.2 103.6 -0.95 0.23 -2.87 -25.9 -52.2 -0.21 0.87 -1.81 10.4 62.0 -0.28 -0.04 1.48 -17.6 -198.3 3.43 2.84 5.87 -16.9 72.6 -0.31 2.79 2.71 -33.5 -11.6 -1.11 4.01 5.0 -57.0 666.3 1.13 4.38 5.0 -64.0 39.3 1.07 -0.16 32.5 100.0 1893.7 -2.80 -0.22 2.5 -28.5 45.0 0.58 0.41 -19.0 -6.0 762.9 0.29 0.82 -3.0 -35.0 140.3 1.16 0.39 0.73 2 279 261534 0 77.0 27.27 -0.91 6.0 58.5 1623.6 -1.40 0.02 -6.5 -48.0 621.0 -1.20 0.14 -0.20 73.6 609.1 -0.44 -0.58 -0.04 -23.0 -27.4 -0.72 -1.04 -1.09 91.1 635.6 -0.88 0.24 0.59 -18.7 -7.2 -0.60 -2.82 -0.71 52.4 504.1 0.89 -0.67 -9.30 -20.8 -25.7 -0.77 -0.85 0.0 -20.0 2259.0 -0.94 1.15 -4.0 -44.0 -22.7 0.94 -0.98 -19.0 105.0 1267.9 1.03 1.27 11.0 -39.5 82.3 0.47 -0.19 -10.0 7.0 1491.8 0.32 -1.29 0.0 -34.0 658.2 -0.76 0.26 0.24 3 279 261535 0 41.0 27.91 -0.35 3.0 46.0 1921.6 -1.36 -0.47 -32.0 -51.5 560.9 -0.29 -0.10 -1.11 124.3 791.6 0.00 0.39 -1.85 -21.7 -44.9 -0.21 0.02 0.89 133.9 797.8 -0.08 1.06 -0.26 -16.4 -74.1 0.97 -0.80 -0.41 66.9 955.3 -1.90 1.28 -6.65 -28.1 47.5 -1.91 1.42 1.0 -30.0 1846.7 0.76 1.10 -4.0 -52.0 -53.9 1.71 -0.22 -12.0 97.5 1969.8 -1.70 0.16 -1.0 -32.5 255.9 -0.46 1.57 10.0 6.0 2047.7 -0.98 1.53 0.0 -49.0 554.2 -0.83 0.39 0.73 4 279 261536 0 50.0 28.00 -1.32 -9.0 12.0 464.8 0.88 0.19 8.0 -51.5 98.1 1.09 -0.33 -2.16 -3.9 102.7 0.39 -1.22 -3.39 -15.2 -42.2 -1.18 -1.11 -3.55 8.9 141.3 -0.16 -0.43 -4.15 -12.9 -13.4 -1.32 -0.98 -3.69 8.8 136.1 -0.30 4.13 1.89 -13.0 -18.7 -1.37 -0.93 0.0 -1.0 810.1 -2.29 6.72 1.0 -23.0 -29.7 0.58 -1.10 -18.5 33.5 206.8 1.84 -0.13 4.0 -29.0 30.1 0.80 -0.24 5.0 -14.0 479.5 0.68 -0.59 2.0 -36.0 -6.9 2.02 0.14 -0.23 In\u00a0[14]: Copied! <pre># imbalance of the classes\ndf[\"target\"].value_counts()\n</pre> # imbalance of the classes df[\"target\"].value_counts() Out[14]: <pre>target\n0    144455\n1      1296\nName: count, dtype: int64</pre> In\u00a0[15]: Copied! <pre># drop columns we won't need in this example\ndf.drop(columns=[\"EXAMPLE_ID\", \"BLOCK_ID\"], inplace=True)\n</pre> # drop columns we won't need in this example df.drop(columns=[\"EXAMPLE_ID\", \"BLOCK_ID\"], inplace=True) In\u00a0[16]: Copied! <pre>df_train, df_valid = train_test_split(\n    df, test_size=0.2, stratify=df[\"target\"], random_state=1\n)\ndf_valid, df_test = train_test_split(\n    df_valid, test_size=0.5, stratify=df_valid[\"target\"], random_state=1\n)\n</pre> df_train, df_valid = train_test_split(     df, test_size=0.2, stratify=df[\"target\"], random_state=1 ) df_valid, df_test = train_test_split(     df_valid, test_size=0.5, stratify=df_valid[\"target\"], random_state=1 ) In\u00a0[17]: Copied! <pre>continuous_cols = df.drop(columns=[\"target\"]).columns.values.tolist()\n</pre> continuous_cols = df.drop(columns=[\"target\"]).columns.values.tolist() In\u00a0[18]: Copied! <pre># deeptabular\ntab_preprocessor = TabPreprocessor(continuous_cols=continuous_cols, scale=True)\nX_tab_train = tab_preprocessor.fit_transform(df_train)\nX_tab_valid = tab_preprocessor.transform(df_valid)\nX_tab_test = tab_preprocessor.transform(df_test)\n\n# target\ny_train = df_train[\"target\"].values\ny_valid = df_valid[\"target\"].values\ny_test = df_test[\"target\"].values\n</pre> # deeptabular tab_preprocessor = TabPreprocessor(continuous_cols=continuous_cols, scale=True) X_tab_train = tab_preprocessor.fit_transform(df_train) X_tab_valid = tab_preprocessor.transform(df_valid) X_tab_test = tab_preprocessor.transform(df_test)  # target y_train = df_train[\"target\"].values y_valid = df_valid[\"target\"].values y_test = df_test[\"target\"].values In\u00a0[19]: Copied! <pre>input_layer = len(tab_preprocessor.continuous_cols)\noutput_layer = 1\nhidden_layers = np.linspace(\n    input_layer * 2, output_layer, 5, endpoint=False, dtype=int\n).tolist()\n</pre> input_layer = len(tab_preprocessor.continuous_cols) output_layer = 1 hidden_layers = np.linspace(     input_layer * 2, output_layer, 5, endpoint=False, dtype=int ).tolist() In\u00a0[20]: Copied! <pre>deeptabular = TabMlp(\n    mlp_hidden_dims=hidden_layers,\n    column_idx=tab_preprocessor.column_idx,\n    continuous_cols=tab_preprocessor.continuous_cols,\n)\nmodel = WideDeep(deeptabular=deeptabular)\nmodel\n</pre> deeptabular = TabMlp(     mlp_hidden_dims=hidden_layers,     column_idx=tab_preprocessor.column_idx,     continuous_cols=tab_preprocessor.continuous_cols, ) model = WideDeep(deeptabular=deeptabular) model Out[20]: <pre>WideDeep(\n  (deeptabular): Sequential(\n    (0): TabMlp(\n      (cont_norm): Identity()\n      (encoder): MLP(\n        (mlp): Sequential(\n          (dense_layer_0): Sequential(\n            (0): Linear(in_features=74, out_features=148, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_1): Sequential(\n            (0): Linear(in_features=148, out_features=118, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_2): Sequential(\n            (0): Linear(in_features=118, out_features=89, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_3): Sequential(\n            (0): Linear(in_features=89, out_features=59, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_4): Sequential(\n            (0): Linear(in_features=59, out_features=30, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=30, out_features=1, bias=True)\n  )\n)</pre> In\u00a0[24]: Copied! <pre># Metrics from torchmetrics\naccuracy = Accuracy_torchmetrics(average=None, num_classes=1, task=\"binary\")\nprecision = Precision_torchmetrics(average=\"micro\", num_classes=1, task=\"binary\")\nf1 = F1_torchmetrics(average=None, num_classes=1, task=\"binary\")\nrecall = Recall_torchmetrics(average=None, num_classes=1, task=\"binary\")\n</pre> # Metrics from torchmetrics accuracy = Accuracy_torchmetrics(average=None, num_classes=1, task=\"binary\") precision = Precision_torchmetrics(average=\"micro\", num_classes=1, task=\"binary\") f1 = F1_torchmetrics(average=None, num_classes=1, task=\"binary\") recall = Recall_torchmetrics(average=None, num_classes=1, task=\"binary\") <p>Note:</p> <p>Following cells includes usage of both <code>RayTuneReporter</code> and <code>WnBReportBest</code> callbacks. In case you want to use just <code>RayTuneReporter</code>, remove following:</p> <ul> <li>wandb from config</li> <li><code>WandbLoggerCallback</code></li> <li><code>WnBReportBest</code></li> <li><code>@wandb_mixin</code> decorator</li> </ul> <p>We do not see strong reason to use WnB without RayTune for a single paramater combination run, but it is possible:</p> <ul> <li>option01: define paramaters in config only for a single value <code>tune.grid_search([1000])</code> (single value RayTune run)</li> <li>option02: define WnB callback that reports currnet validation/training loss, metrics, etc. at the end of batch, ie. do not report to WnB at <code>epoch_end</code> as in <code>WnBReportBest</code> but at the <code>on_batch_end</code>, see <code>pytorch_widedeep.callbacks.Callback</code></li> </ul> In\u00a0[26]: Copied! <pre>config = {\n    \"batch_size\": tune.grid_search([1000, 5000]),\n    \"wandb\": {\n        \"project\": \"test\",\n        # \"api_key_file\": os.getcwd() + \"/wandb_api.key\",\n        \"api_key\": \"WNB_API_KEY\",\n    },\n}\n\n# Optimizers\ndeep_opt = SGD(model.deeptabular.parameters(), lr=0.1)\n# LR Scheduler\ndeep_sch = lr_scheduler.StepLR(deep_opt, step_size=3)\n\n\n@wandb_mixin\ndef training_function(config, X_train, X_val):\n    early_stopping = EarlyStopping()\n    model_checkpoint = ModelCheckpoint(save_best_only=True)\n    # Hyperparameters\n    batch_size = config[\"batch_size\"]\n    trainer = Trainer(\n        model,\n        objective=\"binary_focal_loss\",\n        callbacks=[\n            RayTuneReporter,\n            WnBReportBest(wb=wandb),\n            early_stopping,\n            model_checkpoint,\n        ],\n        lr_schedulers={\"deeptabular\": deep_sch},\n        initializers={\"deeptabular\": XavierNormal},\n        optimizers={\"deeptabular\": deep_opt},\n        metrics=[accuracy, precision, recall, f1],\n        verbose=0,\n    )\n\n    trainer.fit(X_train=X_train, X_val=X_val, n_epochs=5, batch_size=batch_size)\n\n\nX_train = {\"X_tab\": X_tab_train, \"target\": y_train}\nX_val = {\"X_tab\": X_tab_valid, \"target\": y_valid}\n\nasha_scheduler = AsyncHyperBandScheduler(\n    time_attr=\"training_iteration\",\n    metric=\"_metric/val_loss\",\n    mode=\"min\",\n    max_t=100,\n    grace_period=10,\n    reduction_factor=3,\n    brackets=1,\n)\n\nanalysis = tune.run(\n    tune.with_parameters(training_function, X_train=X_train, X_val=X_val),\n    resources_per_trial={\"cpu\": 1, \"gpu\": 0},\n    progress_reporter=JupyterNotebookReporter(overwrite=True),\n    scheduler=asha_scheduler,\n    config=config,\n    callbacks=[\n        WandbLoggerCallback(\n            project=config[\"wandb\"][\"project\"],\n            # api_key_file=config[\"wandb\"][\"api_key_file\"],\n            api_key=config[\"wandb\"][\"api_key\"],\n            log_config=True,\n        )\n    ],\n)\n</pre> config = {     \"batch_size\": tune.grid_search([1000, 5000]),     \"wandb\": {         \"project\": \"test\",         # \"api_key_file\": os.getcwd() + \"/wandb_api.key\",         \"api_key\": \"WNB_API_KEY\",     }, }  # Optimizers deep_opt = SGD(model.deeptabular.parameters(), lr=0.1) # LR Scheduler deep_sch = lr_scheduler.StepLR(deep_opt, step_size=3)   @wandb_mixin def training_function(config, X_train, X_val):     early_stopping = EarlyStopping()     model_checkpoint = ModelCheckpoint(save_best_only=True)     # Hyperparameters     batch_size = config[\"batch_size\"]     trainer = Trainer(         model,         objective=\"binary_focal_loss\",         callbacks=[             RayTuneReporter,             WnBReportBest(wb=wandb),             early_stopping,             model_checkpoint,         ],         lr_schedulers={\"deeptabular\": deep_sch},         initializers={\"deeptabular\": XavierNormal},         optimizers={\"deeptabular\": deep_opt},         metrics=[accuracy, precision, recall, f1],         verbose=0,     )      trainer.fit(X_train=X_train, X_val=X_val, n_epochs=5, batch_size=batch_size)   X_train = {\"X_tab\": X_tab_train, \"target\": y_train} X_val = {\"X_tab\": X_tab_valid, \"target\": y_valid}  asha_scheduler = AsyncHyperBandScheduler(     time_attr=\"training_iteration\",     metric=\"_metric/val_loss\",     mode=\"min\",     max_t=100,     grace_period=10,     reduction_factor=3,     brackets=1, )  analysis = tune.run(     tune.with_parameters(training_function, X_train=X_train, X_val=X_val),     resources_per_trial={\"cpu\": 1, \"gpu\": 0},     progress_reporter=JupyterNotebookReporter(overwrite=True),     scheduler=asha_scheduler,     config=config,     callbacks=[         WandbLoggerCallback(             project=config[\"wandb\"][\"project\"],             # api_key_file=config[\"wandb\"][\"api_key_file\"],             api_key=config[\"wandb\"][\"api_key\"],             log_config=True,         )     ], ) <pre>/Users/javierrodriguezzaurin/.pyenv/versions/3.10.13/lib/python3.10/tempfile.py:860: ResourceWarning: Implicitly cleaning up &lt;TemporaryDirectory '/var/folders/_2/lrjn1qn54c758tdtktr1bvkc0000gn/T/tmp60pfyl1kwandb'&gt;\n  _warnings.warn(warn_message, ResourceWarning)\n/Users/javierrodriguezzaurin/.pyenv/versions/3.10.13/lib/python3.10/tempfile.py:860: ResourceWarning: Implicitly cleaning up &lt;TemporaryDirectory '/var/folders/_2/lrjn1qn54c758tdtktr1bvkc0000gn/T/tmpnjv2rg1wwandb-artifacts'&gt;\n  _warnings.warn(warn_message, ResourceWarning)\n/Users/javierrodriguezzaurin/.pyenv/versions/3.10.13/lib/python3.10/tempfile.py:860: ResourceWarning: Implicitly cleaning up &lt;TemporaryDirectory '/var/folders/_2/lrjn1qn54c758tdtktr1bvkc0000gn/T/tmpgebu5k1kwandb-media'&gt;\n  _warnings.warn(warn_message, ResourceWarning)\n/Users/javierrodriguezzaurin/.pyenv/versions/3.10.13/lib/python3.10/tempfile.py:860: ResourceWarning: Implicitly cleaning up &lt;TemporaryDirectory '/var/folders/_2/lrjn1qn54c758tdtktr1bvkc0000gn/T/tmpxy9y2yriwandb-media'&gt;\n  _warnings.warn(warn_message, ResourceWarning)\n</pre> In\u00a0[14]: Copied! <pre>analysis.results\n</pre> analysis.results Out[14]: <pre>{'fc9a8_00000': {'_metric': {'train_loss': 0.006297602537127896,\n   'train_Accuracy': 0.9925042986869812,\n   'train_Precision': 0.9939393997192383,\n   'train_Recall': 0.15814851224422455,\n   'train_F1Score': 0.2728785574436188,\n   'val_loss': 0.005045663565397263,\n   'val_Accuracy': 0.9946483969688416,\n   'val_Precision': 1.0,\n   'val_Recall': 0.39534884691238403,\n   'val_F1Score': 0.5666667222976685},\n  'time_this_iter_s': 2.388202428817749,\n  'done': True,\n  'timesteps_total': None,\n  'episodes_total': None,\n  'training_iteration': 5,\n  'trial_id': 'fc9a8_00000',\n  'experiment_id': 'baad1d4f3d924b48b9ece1b9f26c80cc',\n  'date': '2022-07-31_14-06-51',\n  'timestamp': 1659276411,\n  'time_total_s': 12.656474113464355,\n  'pid': 1813,\n  'hostname': 'jupyter-5uperpalo',\n  'node_ip': '10.32.44.172',\n  'config': {'batch_size': 1000},\n  'time_since_restore': 12.656474113464355,\n  'timesteps_since_restore': 0,\n  'iterations_since_restore': 5,\n  'warmup_time': 0.8006253242492676,\n  'experiment_tag': '0_batch_size=1000'},\n 'fc9a8_00001': {'_metric': {'train_loss': 0.02519632239515583,\n   'train_Accuracy': 0.9910891652107239,\n   'train_Precision': 0.25,\n   'train_Recall': 0.0009643201483413577,\n   'train_F1Score': 0.0019212296465411782,\n   'val_loss': 0.02578434906899929,\n   'val_Accuracy': 0.9911492466926575,\n   'val_Precision': 0.0,\n   'val_Recall': 0.0,\n   'val_F1Score': 0.0},\n  'time_this_iter_s': 4.113586902618408,\n  'done': True,\n  'timesteps_total': None,\n  'episodes_total': None,\n  'training_iteration': 5,\n  'trial_id': 'fc9a8_00001',\n  'experiment_id': 'f2e54a6a5780429fbf0db0746853347e',\n  'date': '2022-07-31_14-06-56',\n  'timestamp': 1659276416,\n  'time_total_s': 12.926990509033203,\n  'pid': 1962,\n  'hostname': 'jupyter-5uperpalo',\n  'node_ip': '10.32.44.172',\n  'config': {'batch_size': 5000},\n  'time_since_restore': 12.926990509033203,\n  'timesteps_since_restore': 0,\n  'iterations_since_restore': 5,\n  'warmup_time': 0.9253025054931641,\n  'experiment_tag': '1_batch_size=5000'}}</pre> <p>Using Weights and Biases logging you can create parallel coordinates graphs that map parametr combinations to the best(lowest) loss achieved during the training of the networks</p> <p></p> <p>local visualization of raytune reults using tensorboard</p> In\u00a0[23]: Copied! <pre>%load_ext tensorboard\n%tensorboard --logdir ~/ray_results\n</pre> %load_ext tensorboard %tensorboard --logdir ~/ray_results"},{"location":"examples/10_3rd_party_integration-RayTune_WnB.html#3rd-party-integration-raytune-weights-biases","title":"3rd party integration - RayTune, Weights &amp; Biases\u00b6","text":"<p>This notebook provides guideline for integration of external library functions in the model training process through <code>Callback</code> objects, a popular concept of using objects as arguments for other objects.</p> <p>[DISCLAIMER]</p> <p>We show integration of RayTune (a hyperparameter tuning framework) and Weights &amp; Biases (ML projects experiment tracking and versioning solution) in the <code>pytorch_widedeep</code> model training process. We did not include <code>RayTuneReporter</code> and <code>WnBReportBest</code> in the library code to minimize the dependencies on other libraries that are not directly included in the model design and training.</p>"},{"location":"examples/10_3rd_party_integration-RayTune_WnB.html#initial-imports","title":"Initial imports\u00b6","text":""},{"location":"examples/10_3rd_party_integration-RayTune_WnB.html#preparing-the-data","title":"Preparing the data\u00b6","text":""},{"location":"examples/10_3rd_party_integration-RayTune_WnB.html#define-the-model","title":"Define the model\u00b6","text":""},{"location":"examples/11_auc_multiclass.html","title":"11_auc_multiclass","text":"In\u00a0[1]: Copied! <pre>import numpy as np\nimport pandas as pd\nfrom torch.optim import SGD, lr_scheduler\n\nfrom pytorch_widedeep import Trainer\nfrom pytorch_widedeep.preprocessing import TabPreprocessor\nfrom pytorch_widedeep.models import TabMlp, WideDeep\nfrom torchmetrics import AUROC\nfrom pytorch_widedeep.initializers import XavierNormal\nfrom pytorch_widedeep.datasets import load_ecoli\nfrom pytorch_widedeep.utils import LabelEncoder\n\nfrom sklearn.model_selection import train_test_split\n\n# increase displayed columns in jupyter notebook\npd.set_option(\"display.max_columns\", 200)\npd.set_option(\"display.max_rows\", 300)\n</pre> import numpy as np import pandas as pd from torch.optim import SGD, lr_scheduler  from pytorch_widedeep import Trainer from pytorch_widedeep.preprocessing import TabPreprocessor from pytorch_widedeep.models import TabMlp, WideDeep from torchmetrics import AUROC from pytorch_widedeep.initializers import XavierNormal from pytorch_widedeep.datasets import load_ecoli from pytorch_widedeep.utils import LabelEncoder  from sklearn.model_selection import train_test_split  # increase displayed columns in jupyter notebook pd.set_option(\"display.max_columns\", 200) pd.set_option(\"display.max_rows\", 300) In\u00a0[2]: Copied! <pre>df = load_ecoli(as_frame=True)\ndf.head()\n</pre> df = load_ecoli(as_frame=True) df.head() Out[2]: SequenceName mcg gvh lip chg aac alm1 alm2 class 0 AAT_ECOLI 0.49 0.29 0.48 0.5 0.56 0.24 0.35 cp 1 ACEA_ECOLI 0.07 0.40 0.48 0.5 0.54 0.35 0.44 cp 2 ACEK_ECOLI 0.56 0.40 0.48 0.5 0.49 0.37 0.46 cp 3 ACKA_ECOLI 0.59 0.49 0.48 0.5 0.52 0.45 0.36 cp 4 ADI_ECOLI 0.23 0.32 0.48 0.5 0.55 0.25 0.35 cp In\u00a0[3]: Copied! <pre># imbalance of the classes\ndf[\"class\"].value_counts()\n</pre> # imbalance of the classes df[\"class\"].value_counts() Out[3]: <pre>class\ncp     143\nim      77\npp      52\nimU     35\nom      20\nomL      5\nimS      2\nimL      2\nName: count, dtype: int64</pre> In\u00a0[4]: Copied! <pre>df = df.loc[~df[\"class\"].isin([\"omL\", \"imS\", \"imL\"])]\ndf.reset_index(inplace=True, drop=True)\n</pre> df = df.loc[~df[\"class\"].isin([\"omL\", \"imS\", \"imL\"])] df.reset_index(inplace=True, drop=True) In\u00a0[5]: Copied! <pre>encoder = LabelEncoder([\"class\"])\ndf_enc = encoder.fit_transform(df)\ndf_enc[\"class\"] = df_enc[\"class\"] - 1\n</pre> encoder = LabelEncoder([\"class\"]) df_enc = encoder.fit_transform(df) df_enc[\"class\"] = df_enc[\"class\"] - 1 In\u00a0[6]: Copied! <pre># drop columns we won't need in this example\ndf_enc = df_enc.drop(columns=[\"SequenceName\"])\n</pre> # drop columns we won't need in this example df_enc = df_enc.drop(columns=[\"SequenceName\"]) In\u00a0[7]: Copied! <pre>df_train, df_valid = train_test_split(\n    df_enc, test_size=0.2, stratify=df_enc[\"class\"], random_state=1\n)\ndf_valid, df_test = train_test_split(\n    df_valid, test_size=0.5, stratify=df_valid[\"class\"], random_state=1\n)\n</pre> df_train, df_valid = train_test_split(     df_enc, test_size=0.2, stratify=df_enc[\"class\"], random_state=1 ) df_valid, df_test = train_test_split(     df_valid, test_size=0.5, stratify=df_valid[\"class\"], random_state=1 ) In\u00a0[8]: Copied! <pre>continuous_cols = df_enc.drop(columns=[\"class\"]).columns.values.tolist()\n</pre> continuous_cols = df_enc.drop(columns=[\"class\"]).columns.values.tolist() In\u00a0[9]: Copied! <pre># deeptabular\ntab_preprocessor = TabPreprocessor(continuous_cols=continuous_cols, scale=True)\nX_tab_train = tab_preprocessor.fit_transform(df_train)\nX_tab_valid = tab_preprocessor.transform(df_valid)\nX_tab_test = tab_preprocessor.transform(df_test)\n\n# target\ny_train = df_train[\"class\"].values\ny_valid = df_valid[\"class\"].values\ny_test = df_test[\"class\"].values\n\nX_train = {\"X_tab\": X_tab_train, \"target\": y_train}\nX_val = {\"X_tab\": X_tab_valid, \"target\": y_valid}\n</pre> # deeptabular tab_preprocessor = TabPreprocessor(continuous_cols=continuous_cols, scale=True) X_tab_train = tab_preprocessor.fit_transform(df_train) X_tab_valid = tab_preprocessor.transform(df_valid) X_tab_test = tab_preprocessor.transform(df_test)  # target y_train = df_train[\"class\"].values y_valid = df_valid[\"class\"].values y_test = df_test[\"class\"].values  X_train = {\"X_tab\": X_tab_train, \"target\": y_train} X_val = {\"X_tab\": X_tab_valid, \"target\": y_valid} <pre>/Users/javierrodriguezzaurin/Projects/pytorch-widedeep/pytorch_widedeep/preprocessing/tab_preprocessor.py:295: DeprecationWarning: 'scale' and 'already_standard' will be deprecated in the next release. Please use 'cols_to_scale' instead\n  self._check_inputs(cat_embed_cols)\n</pre> In\u00a0[10]: Copied! <pre>deeptabular = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    continuous_cols=tab_preprocessor.continuous_cols,\n)\nmodel = WideDeep(deeptabular=deeptabular, pred_dim=df_enc[\"class\"].nunique())\nmodel\n</pre> deeptabular = TabMlp(     column_idx=tab_preprocessor.column_idx,     continuous_cols=tab_preprocessor.continuous_cols, ) model = WideDeep(deeptabular=deeptabular, pred_dim=df_enc[\"class\"].nunique()) model Out[10]: <pre>WideDeep(\n  (deeptabular): Sequential(\n    (0): TabMlp(\n      (cont_norm): Identity()\n      (encoder): MLP(\n        (mlp): Sequential(\n          (dense_layer_0): Sequential(\n            (0): Linear(in_features=7, out_features=200, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_1): Sequential(\n            (0): Linear(in_features=200, out_features=100, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=100, out_features=5, bias=True)\n  )\n)</pre> In\u00a0[11]: Copied! <pre>auroc = AUROC(num_classes=df_enc[\"class\"].nunique(), task=\"multiclass\")\n</pre> auroc = AUROC(num_classes=df_enc[\"class\"].nunique(), task=\"multiclass\") In\u00a0[12]: Copied! <pre># Optimizers\ndeep_opt = SGD(model.deeptabular.parameters(), lr=0.1)\n# LR Scheduler\ndeep_sch = lr_scheduler.StepLR(deep_opt, step_size=3)\n# Hyperparameters\ntrainer = Trainer(\n    model,\n    objective=\"multiclass_focal_loss\",\n    lr_schedulers={\"deeptabular\": deep_sch},\n    initializers={\"deeptabular\": XavierNormal},\n    optimizers={\"deeptabular\": deep_opt},\n    metrics=[auroc],\n)\n\ntrainer.fit(X_train=X_train, X_val=X_val, n_epochs=5, batch_size=50)\n</pre> # Optimizers deep_opt = SGD(model.deeptabular.parameters(), lr=0.1) # LR Scheduler deep_sch = lr_scheduler.StepLR(deep_opt, step_size=3) # Hyperparameters trainer = Trainer(     model,     objective=\"multiclass_focal_loss\",     lr_schedulers={\"deeptabular\": deep_sch},     initializers={\"deeptabular\": XavierNormal},     optimizers={\"deeptabular\": deep_opt},     metrics=[auroc], )  trainer.fit(X_train=X_train, X_val=X_val, n_epochs=5, batch_size=50) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 6/6 [00:00&lt;00:00, 54.59it/s, loss=0.109, metrics={'MulticlassAUROC': 0.314}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 1/1 [00:00&lt;00:00, 98.35it/s, loss=0.105, metrics={'MulticlassAUROC': 0.2558}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 6/6 [00:00&lt;00:00, 91.55it/s, loss=0.105, metrics={'MulticlassAUROC': 0.3546}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 1/1 [00:00&lt;00:00, 111.68it/s, loss=0.101, metrics={'MulticlassAUROC': 0.2737}]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 6/6 [00:00&lt;00:00, 62.55it/s, loss=0.1, metrics={'MulticlassAUROC': 0.3795}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 1/1 [00:00&lt;00:00, 108.51it/s, loss=0.0966, metrics={'MulticlassAUROC': 0.3053}]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 6/6 [00:00&lt;00:00, 99.35it/s, loss=0.0965, metrics={'MulticlassAUROC': 0.3809}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 1/1 [00:00&lt;00:00, 117.73it/s, loss=0.0962, metrics={'MulticlassAUROC': 0.3089}]\nepoch 5: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 6/6 [00:00&lt;00:00, 110.56it/s, loss=0.0967, metrics={'MulticlassAUROC': 0.3509}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 1/1 [00:00&lt;00:00, 127.35it/s, loss=0.0958, metrics={'MulticlassAUROC': 0.3089}]\n</pre> In\u00a0[\u00a0]: Copied! <pre>\n</pre>"},{"location":"examples/11_auc_multiclass.html#auc-multiclass-computation","title":"AUC multiclass computation\u00b6","text":""},{"location":"examples/11_auc_multiclass.html#initial-imports","title":"Initial imports\u00b6","text":""},{"location":"examples/11_auc_multiclass.html#preparing-the-data","title":"Preparing the data\u00b6","text":""},{"location":"examples/11_auc_multiclass.html#define-the-model","title":"Define the model\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html","title":"12_ZILNLoss_origkeras_vs_pytorch_widedeep","text":"In\u00a0[1]: Copied! <pre># @title Copyright 2019 The Lifetime Value Authors.\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     https://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n# ============================================================================\n</pre> # @title Copyright 2019 The Lifetime Value Authors. # Licensed under the Apache License, Version 2.0 (the \"License\"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # #     https://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an \"AS IS\" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ Run in Google Colab View source on GitHub In\u00a0[3]: Copied! <pre>import os\n\nimport numpy as np\nimport pandas as pd\nfrom scipy import stats\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport tensorflow as tf\nimport tensorflow_probability as tfp\nfrom typing import Sequence\n\n# install and import ltv\n!pip install -q git+https://github.com/google/lifetime_value\nimport lifetime_value as ltv\n</pre> import os  import numpy as np import pandas as pd from scipy import stats import matplotlib.pyplot as plt import seaborn as sns import tensorflow as tf import tensorflow_probability as tfp from typing import Sequence  # install and import ltv !pip install -q git+https://github.com/google/lifetime_value import lifetime_value as ltv In\u00a0[\u00a0]: Copied! <pre>tfd = tfp.distributions\n%config InlineBackend.figure_format='retina'\nsns.set_style(\"whitegrid\")\n</pre> tfd = tfp.distributions %config InlineBackend.figure_format='retina' sns.set_style(\"whitegrid\") In\u00a0[\u00a0]: Copied! <pre>MODEL = \"dnn\"\nLOSS = \"ziln\"  # @param { isTemplate: true, type: 'string'} ['mse', 'ziln']\nLEARNING_RATE = 0.001  # @param { isTemplate: true}\nVERSION = 0  # @param { isTemplate: true, type: 'integer'}\nOUTPUT_CSV_FOLDER = \"/tmp/lifetime-value/kdd_cup_98/result\"  # @param { isTemplate: true, type: 'string'}\n</pre> MODEL = \"dnn\" LOSS = \"ziln\"  # @param { isTemplate: true, type: 'string'} ['mse', 'ziln'] LEARNING_RATE = 0.001  # @param { isTemplate: true} VERSION = 0  # @param { isTemplate: true, type: 'integer'} OUTPUT_CSV_FOLDER = \"/tmp/lifetime-value/kdd_cup_98/result\"  # @param { isTemplate: true, type: 'string'} <p>Download kdd_cup_98 data to /tmp/lifetime-value/kdd_cup_98</p> In\u00a0[\u00a0]: Copied! <pre>%%bash\nmkdir -p /tmp/lifetime-value/kdd_cup_98\nwget https://kdd.ics.uci.edu/databases/kddcup98/epsilon_mirror/cup98lrn.zip -P /tmp/lifetime-value/kdd_cup_98/\nwget https://kdd.ics.uci.edu/databases/kddcup98/epsilon_mirror/cup98val.zip -P /tmp/lifetime-value/kdd_cup_98/\nwget https://kdd.ics.uci.edu/databases/kddcup98/epsilon_mirror/valtargt.txt -P /tmp/lifetime-value/kdd_cup_98/\ncd /tmp/lifetime-value/kdd_cup_98/\nunzip cup98lrn.zip\nunzip cup98val.zip\n</pre> %%bash mkdir -p /tmp/lifetime-value/kdd_cup_98 wget https://kdd.ics.uci.edu/databases/kddcup98/epsilon_mirror/cup98lrn.zip -P /tmp/lifetime-value/kdd_cup_98/ wget https://kdd.ics.uci.edu/databases/kddcup98/epsilon_mirror/cup98val.zip -P /tmp/lifetime-value/kdd_cup_98/ wget https://kdd.ics.uci.edu/databases/kddcup98/epsilon_mirror/valtargt.txt -P /tmp/lifetime-value/kdd_cup_98/ cd /tmp/lifetime-value/kdd_cup_98/ unzip cup98lrn.zip unzip cup98val.zip In\u00a0[\u00a0]: Copied! <pre>df_train = pd.read_csv(\"/tmp/lifetime-value/kdd_cup_98/cup98LRN.txt\")\nnum_train = df_train.shape[0]\ndf_eval = pd.read_csv(\"/tmp/lifetime-value/kdd_cup_98/cup98VAL.txt\")\ndf_eval_target = pd.read_csv(\"/tmp/lifetime-value/kdd_cup_98/valtargt.txt\")\ndf_eval = df_eval.merge(df_eval_target, on=\"CONTROLN\")\n</pre> df_train = pd.read_csv(\"/tmp/lifetime-value/kdd_cup_98/cup98LRN.txt\") num_train = df_train.shape[0] df_eval = pd.read_csv(\"/tmp/lifetime-value/kdd_cup_98/cup98VAL.txt\") df_eval_target = pd.read_csv(\"/tmp/lifetime-value/kdd_cup_98/valtargt.txt\") df_eval = df_eval.merge(df_eval_target, on=\"CONTROLN\") In\u00a0[\u00a0]: Copied! <pre>df = pd.concat([df_train, df_eval], axis=0, sort=True)\n</pre> df = pd.concat([df_train, df_eval], axis=0, sort=True) In\u00a0[\u00a0]: Copied! <pre>y = df[\"TARGET_D\"][:num_train]\n</pre> y = df[\"TARGET_D\"][:num_train] In\u00a0[\u00a0]: Copied! <pre>def plot_hist_log_scale(y):\n    max_val = y.max() + 1.0\n    ax = pd.Series(y).hist(\n        figsize=(8, 5), bins=10 ** np.linspace(0.0, np.log10(max_val), 20)\n    )\n\n    plt.xlabel(\"Donation ($)\")\n    plt.ylabel(\"Count\")\n    # plt.title('Histogram of LTV')\n    plt.xticks(rotation=\"horizontal\")\n    plt.legend(loc=\"upper left\")\n    ax.set_xscale(\"log\")\n    ax.grid(False)\n    # Hide the right and top spines\n    ax.spines[\"right\"].set_visible(False)\n    ax.spines[\"top\"].set_visible(False)\n    # Only show ticks on the left and bottom spines\n    ax.yaxis.set_ticks_position(\"left\")\n    ax.xaxis.set_ticks_position(\"bottom\")\n    plt.show()\n\n    fig = ax.get_figure()\n    output_file = tf.io.gfile.GFile(\n        \"/tmp/lifetime-value/kdd_cup_98/histogram_kdd98_log_scale.pdf\", \"wb\"\n    )\n    fig.savefig(output_file, bbox_inches=\"tight\", format=\"pdf\")\n</pre> def plot_hist_log_scale(y):     max_val = y.max() + 1.0     ax = pd.Series(y).hist(         figsize=(8, 5), bins=10 ** np.linspace(0.0, np.log10(max_val), 20)     )      plt.xlabel(\"Donation ($)\")     plt.ylabel(\"Count\")     # plt.title('Histogram of LTV')     plt.xticks(rotation=\"horizontal\")     plt.legend(loc=\"upper left\")     ax.set_xscale(\"log\")     ax.grid(False)     # Hide the right and top spines     ax.spines[\"right\"].set_visible(False)     ax.spines[\"top\"].set_visible(False)     # Only show ticks on the left and bottom spines     ax.yaxis.set_ticks_position(\"left\")     ax.xaxis.set_ticks_position(\"bottom\")     plt.show()      fig = ax.get_figure()     output_file = tf.io.gfile.GFile(         \"/tmp/lifetime-value/kdd_cup_98/histogram_kdd98_log_scale.pdf\", \"wb\"     )     fig.savefig(output_file, bbox_inches=\"tight\", format=\"pdf\") In\u00a0[\u00a0]: Copied! <pre>plot_hist_log_scale(y[y &gt; 0])\n</pre> plot_hist_log_scale(y[y &gt; 0]) In\u00a0[\u00a0]: Copied! <pre>VOCAB_FEATURES = [\n    \"ODATEDW\",  # date of donor's first gift (YYMM)\n    \"OSOURCE\",  # donor acquisition mailing list\n    \"TCODE\",  # donor title code\n    \"STATE\",\n    \"ZIP\",\n    \"DOMAIN\",  # urbanicity level and socio-economic status of the neighborhood\n    \"CLUSTER\",  # socio-economic status\n    \"GENDER\",\n    \"MAXADATE\",  # date of the most recent promotion received\n    \"MINRDATE\",\n    \"LASTDATE\",\n    \"FISTDATE\",\n    \"RFA_2A\",\n]\n</pre> VOCAB_FEATURES = [     \"ODATEDW\",  # date of donor's first gift (YYMM)     \"OSOURCE\",  # donor acquisition mailing list     \"TCODE\",  # donor title code     \"STATE\",     \"ZIP\",     \"DOMAIN\",  # urbanicity level and socio-economic status of the neighborhood     \"CLUSTER\",  # socio-economic status     \"GENDER\",     \"MAXADATE\",  # date of the most recent promotion received     \"MINRDATE\",     \"LASTDATE\",     \"FISTDATE\",     \"RFA_2A\", ] In\u00a0[\u00a0]: Copied! <pre>df[\"ODATEDW\"] = df[\"ODATEDW\"].astype(\"str\")\ndf[\"TCODE\"] = df[\"TCODE\"].apply(lambda x: \"{:03d}\".format(x // 1000 if x &gt; 1000 else x))\ndf[\"ZIP\"] = df[\"ZIP\"].str.slice(0, 5)\ndf[\"MAXADATE\"] = df[\"MAXADATE\"].astype(\"str\")\ndf[\"MINRDATE\"] = df[\"MINRDATE\"].astype(\"str\")\ndf[\"LASTDATE\"] = df[\"LASTDATE\"].astype(\"str\")\ndf[\"FISTDATE\"] = df[\"FISTDATE\"].astype(\"str\")\n</pre> df[\"ODATEDW\"] = df[\"ODATEDW\"].astype(\"str\") df[\"TCODE\"] = df[\"TCODE\"].apply(lambda x: \"{:03d}\".format(x // 1000 if x &gt; 1000 else x)) df[\"ZIP\"] = df[\"ZIP\"].str.slice(0, 5) df[\"MAXADATE\"] = df[\"MAXADATE\"].astype(\"str\") df[\"MINRDATE\"] = df[\"MINRDATE\"].astype(\"str\") df[\"LASTDATE\"] = df[\"LASTDATE\"].astype(\"str\") df[\"FISTDATE\"] = df[\"FISTDATE\"].astype(\"str\") In\u00a0[\u00a0]: Copied! <pre>def label_encoding(y, frequency_threshold=100):\n    value_counts = pd.value_counts(y)\n    categories = value_counts[value_counts &gt;= frequency_threshold].index.to_numpy()\n    # 0 indicates the unknown category.\n    return pd.Categorical(y, categories=categories).codes + 1\n</pre> def label_encoding(y, frequency_threshold=100):     value_counts = pd.value_counts(y)     categories = value_counts[value_counts &gt;= frequency_threshold].index.to_numpy()     # 0 indicates the unknown category.     return pd.Categorical(y, categories=categories).codes + 1 In\u00a0[\u00a0]: Copied! <pre>for key in VOCAB_FEATURES:\n    df[key] = label_encoding(df[key])\n</pre> for key in VOCAB_FEATURES:     df[key] = label_encoding(df[key]) In\u00a0[\u00a0]: Copied! <pre>MAIL_ORDER_RESPONSES = [\n    \"MBCRAFT\",\n    \"MBGARDEN\",\n    \"MBBOOKS\",\n    \"MBCOLECT\",\n    \"MAGFAML\",\n    \"MAGFEM\",\n    \"MAGMALE\",\n    \"PUBGARDN\",\n    \"PUBCULIN\",\n    \"PUBHLTH\",\n    \"PUBDOITY\",\n    \"PUBNEWFN\",\n    \"PUBPHOTO\",\n    \"PUBOPP\",\n    \"RFA_2F\",\n]\n</pre> MAIL_ORDER_RESPONSES = [     \"MBCRAFT\",     \"MBGARDEN\",     \"MBBOOKS\",     \"MBCOLECT\",     \"MAGFAML\",     \"MAGFEM\",     \"MAGMALE\",     \"PUBGARDN\",     \"PUBCULIN\",     \"PUBHLTH\",     \"PUBDOITY\",     \"PUBNEWFN\",     \"PUBPHOTO\",     \"PUBOPP\",     \"RFA_2F\", ] In\u00a0[\u00a0]: Copied! <pre>INDICATOR_FEATURES = [\n    \"AGE\",  # age decile, 0 indicates unknown\n    \"NUMCHLD\",\n    \"INCOME\",\n    \"WEALTH1\",\n    \"HIT\",\n] + MAIL_ORDER_RESPONSES\n</pre> INDICATOR_FEATURES = [     \"AGE\",  # age decile, 0 indicates unknown     \"NUMCHLD\",     \"INCOME\",     \"WEALTH1\",     \"HIT\", ] + MAIL_ORDER_RESPONSES In\u00a0[\u00a0]: Copied! <pre>df[\"AGE\"] = pd.qcut(df[\"AGE\"].values, 10).codes + 1\ndf[\"NUMCHLD\"] = df[\"NUMCHLD\"].apply(lambda x: 0 if np.isnan(x) else int(x))\ndf[\"INCOME\"] = df[\"INCOME\"].apply(lambda x: 0 if np.isnan(x) else int(x))\ndf[\"WEALTH1\"] = df[\"WEALTH1\"].apply(lambda x: 0 if np.isnan(x) else int(x) + 1)\ndf[\"HIT\"] = pd.qcut(df[\"HIT\"].values, q=50, duplicates=\"drop\").codes\n\nfor col in MAIL_ORDER_RESPONSES:\n    df[col] = pd.qcut(df[col].values, q=20, duplicates=\"drop\").codes + 1\n</pre> df[\"AGE\"] = pd.qcut(df[\"AGE\"].values, 10).codes + 1 df[\"NUMCHLD\"] = df[\"NUMCHLD\"].apply(lambda x: 0 if np.isnan(x) else int(x)) df[\"INCOME\"] = df[\"INCOME\"].apply(lambda x: 0 if np.isnan(x) else int(x)) df[\"WEALTH1\"] = df[\"WEALTH1\"].apply(lambda x: 0 if np.isnan(x) else int(x) + 1) df[\"HIT\"] = pd.qcut(df[\"HIT\"].values, q=50, duplicates=\"drop\").codes  for col in MAIL_ORDER_RESPONSES:     df[col] = pd.qcut(df[col].values, q=20, duplicates=\"drop\").codes + 1 In\u00a0[\u00a0]: Copied! <pre>NUMERIC_FEATURES = [\n    # binary\n    \"MAILCODE\",  # bad address\n    \"NOEXCH\",  # do not exchange\n    \"RECINHSE\",  # donor has given to PVA's in house program\n    \"RECP3\",  # donor has given to PVA's P3 program\n    \"RECPGVG\",  # planned giving record\n    \"RECSWEEP\",  # sweepstakes record\n    \"HOMEOWNR\",  # home owner\n    \"CHILD03\",\n    \"CHILD07\",\n    \"CHILD12\",\n    \"CHILD18\",\n    # continuous\n    \"CARDPROM\",\n    \"NUMPROM\",\n    \"CARDPM12\",\n    \"NUMPRM12\",\n    \"RAMNTALL\",\n    \"NGIFTALL\",\n    \"MINRAMNT\",\n    \"MAXRAMNT\",\n    \"LASTGIFT\",\n    \"AVGGIFT\",\n]\n</pre> NUMERIC_FEATURES = [     # binary     \"MAILCODE\",  # bad address     \"NOEXCH\",  # do not exchange     \"RECINHSE\",  # donor has given to PVA's in house program     \"RECP3\",  # donor has given to PVA's P3 program     \"RECPGVG\",  # planned giving record     \"RECSWEEP\",  # sweepstakes record     \"HOMEOWNR\",  # home owner     \"CHILD03\",     \"CHILD07\",     \"CHILD12\",     \"CHILD18\",     # continuous     \"CARDPROM\",     \"NUMPROM\",     \"CARDPM12\",     \"NUMPRM12\",     \"RAMNTALL\",     \"NGIFTALL\",     \"MINRAMNT\",     \"MAXRAMNT\",     \"LASTGIFT\",     \"AVGGIFT\", ] In\u00a0[\u00a0]: Copied! <pre>df[\"MAILCODE\"] = (df[\"MAILCODE\"] == \"B\").astype(\"float32\")\ndf[\"PVASTATE\"] = df[\"PVASTATE\"].isin([\"P\", \"E\"]).astype(\"float32\")\ndf[\"NOEXCH\"] = df[\"NOEXCH\"].isin([\"X\", \"1\"]).astype(\"float32\")\ndf[\"RECINHSE\"] = (df[\"RECINHSE\"] == \"X\").astype(\"float32\")\ndf[\"RECP3\"] = (df[\"RECP3\"] == \"X\").astype(\"float32\")\ndf[\"RECPGVG\"] = (df[\"RECPGVG\"] == \"X\").astype(\"float32\")\ndf[\"RECSWEEP\"] = (df[\"RECSWEEP\"] == \"X\").astype(\"float32\")\ndf[\"HOMEOWNR\"] = (df[\"HOMEOWNR\"] == \"H\").astype(\"float32\")\ndf[\"CHILD03\"] = df[\"CHILD03\"].isin([\"M\", \"F\", \"B\"]).astype(\"float32\")\ndf[\"CHILD07\"] = df[\"CHILD07\"].isin([\"M\", \"F\", \"B\"]).astype(\"float32\")\ndf[\"CHILD12\"] = df[\"CHILD12\"].isin([\"M\", \"F\", \"B\"]).astype(\"float32\")\ndf[\"CHILD18\"] = df[\"CHILD18\"].isin([\"M\", \"F\", \"B\"]).astype(\"float32\")\n\ndf[\"CARDPROM\"] = df[\"CARDPROM\"] / 100\ndf[\"NUMPROM\"] = df[\"NUMPROM\"] / 100\ndf[\"CARDPM12\"] = df[\"CARDPM12\"] / 100\ndf[\"NUMPRM12\"] = df[\"NUMPRM12\"] / 100\ndf[\"RAMNTALL\"] = np.log1p(df[\"RAMNTALL\"])\ndf[\"NGIFTALL\"] = np.log1p(df[\"NGIFTALL\"])\ndf[\"MINRAMNT\"] = np.log1p(df[\"MINRAMNT\"])\ndf[\"MAXRAMNT\"] = np.log1p(df[\"MAXRAMNT\"])\ndf[\"LASTGIFT\"] = np.log1p(df[\"LASTGIFT\"])\ndf[\"AVGGIFT\"] = np.log1p(df[\"AVGGIFT\"])\n</pre> df[\"MAILCODE\"] = (df[\"MAILCODE\"] == \"B\").astype(\"float32\") df[\"PVASTATE\"] = df[\"PVASTATE\"].isin([\"P\", \"E\"]).astype(\"float32\") df[\"NOEXCH\"] = df[\"NOEXCH\"].isin([\"X\", \"1\"]).astype(\"float32\") df[\"RECINHSE\"] = (df[\"RECINHSE\"] == \"X\").astype(\"float32\") df[\"RECP3\"] = (df[\"RECP3\"] == \"X\").astype(\"float32\") df[\"RECPGVG\"] = (df[\"RECPGVG\"] == \"X\").astype(\"float32\") df[\"RECSWEEP\"] = (df[\"RECSWEEP\"] == \"X\").astype(\"float32\") df[\"HOMEOWNR\"] = (df[\"HOMEOWNR\"] == \"H\").astype(\"float32\") df[\"CHILD03\"] = df[\"CHILD03\"].isin([\"M\", \"F\", \"B\"]).astype(\"float32\") df[\"CHILD07\"] = df[\"CHILD07\"].isin([\"M\", \"F\", \"B\"]).astype(\"float32\") df[\"CHILD12\"] = df[\"CHILD12\"].isin([\"M\", \"F\", \"B\"]).astype(\"float32\") df[\"CHILD18\"] = df[\"CHILD18\"].isin([\"M\", \"F\", \"B\"]).astype(\"float32\")  df[\"CARDPROM\"] = df[\"CARDPROM\"] / 100 df[\"NUMPROM\"] = df[\"NUMPROM\"] / 100 df[\"CARDPM12\"] = df[\"CARDPM12\"] / 100 df[\"NUMPRM12\"] = df[\"NUMPRM12\"] / 100 df[\"RAMNTALL\"] = np.log1p(df[\"RAMNTALL\"]) df[\"NGIFTALL\"] = np.log1p(df[\"NGIFTALL\"]) df[\"MINRAMNT\"] = np.log1p(df[\"MINRAMNT\"]) df[\"MAXRAMNT\"] = np.log1p(df[\"MAXRAMNT\"]) df[\"LASTGIFT\"] = np.log1p(df[\"LASTGIFT\"]) df[\"AVGGIFT\"] = np.log1p(df[\"AVGGIFT\"]) In\u00a0[\u00a0]: Copied! <pre>CATEGORICAL_FEATURES = VOCAB_FEATURES + INDICATOR_FEATURES\nALL_FEATURES = CATEGORICAL_FEATURES + NUMERIC_FEATURES\n</pre> CATEGORICAL_FEATURES = VOCAB_FEATURES + INDICATOR_FEATURES ALL_FEATURES = CATEGORICAL_FEATURES + NUMERIC_FEATURES In\u00a0[\u00a0]: Copied! <pre>def dnn_split(df):\n    df_train = df.iloc[:num_train]\n    df_eval = df.iloc[num_train:]\n\n    def feature_dict(df):\n        features = {k: v.values for k, v in dict(df[CATEGORICAL_FEATURES]).items()}\n        features[\"numeric\"] = df[NUMERIC_FEATURES].astype(\"float32\").values\n        return features\n\n    x_train, y_train = (\n        feature_dict(df_train),\n        df_train[\"TARGET_D\"].astype(\"float32\").values,\n    )\n    x_eval, y_eval = feature_dict(df_eval), df_eval[\"TARGET_D\"].astype(\"float32\").values\n\n    return x_train, x_eval, y_train, y_eval\n</pre> def dnn_split(df):     df_train = df.iloc[:num_train]     df_eval = df.iloc[num_train:]      def feature_dict(df):         features = {k: v.values for k, v in dict(df[CATEGORICAL_FEATURES]).items()}         features[\"numeric\"] = df[NUMERIC_FEATURES].astype(\"float32\").values         return features      x_train, y_train = (         feature_dict(df_train),         df_train[\"TARGET_D\"].astype(\"float32\").values,     )     x_eval, y_eval = feature_dict(df_eval), df_eval[\"TARGET_D\"].astype(\"float32\").values      return x_train, x_eval, y_train, y_eval In\u00a0[\u00a0]: Copied! <pre>def embedding_dim(x):\n    return int(x**0.25) + 1\n\n\ndef embedding_layer(vocab_size):\n    return tf.keras.Sequential(\n        [\n            tf.keras.layers.Embedding(\n                input_dim=vocab_size,\n                output_dim=embedding_dim(vocab_size),\n                input_length=1,\n            ),\n            tf.keras.layers.Flatten(),\n        ]\n    )\n\n\ndef dnn_model(output_units):\n    numeric_input = tf.keras.layers.Input(\n        shape=(len(NUMERIC_FEATURES),), name=\"numeric\"\n    )\n\n    embedding_inputs = [\n        tf.keras.layers.Input(shape=(1,), name=key, dtype=np.int64)\n        for key in CATEGORICAL_FEATURES\n    ]\n\n    embedding_outputs = [\n        embedding_layer(vocab_size=df[key].max() + 1)(input)\n        for key, input in zip(CATEGORICAL_FEATURES, embedding_inputs)\n    ]\n\n    deep_input = tf.keras.layers.concatenate([numeric_input] + embedding_outputs)\n    deep_model = tf.keras.Sequential(\n        [\n            tf.keras.layers.Dense(128, activation=\"relu\"),\n            tf.keras.layers.Dense(128, activation=\"relu\"),\n            tf.keras.layers.Dense(64, activation=\"relu\"),\n            tf.keras.layers.Dense(64, activation=\"relu\"),\n            tf.keras.layers.Dense(units=output_units),\n        ]\n    )\n    return tf.keras.Model(\n        inputs=[numeric_input] + embedding_inputs, outputs=deep_model(deep_input)\n    )\n</pre> def embedding_dim(x):     return int(x**0.25) + 1   def embedding_layer(vocab_size):     return tf.keras.Sequential(         [             tf.keras.layers.Embedding(                 input_dim=vocab_size,                 output_dim=embedding_dim(vocab_size),                 input_length=1,             ),             tf.keras.layers.Flatten(),         ]     )   def dnn_model(output_units):     numeric_input = tf.keras.layers.Input(         shape=(len(NUMERIC_FEATURES),), name=\"numeric\"     )      embedding_inputs = [         tf.keras.layers.Input(shape=(1,), name=key, dtype=np.int64)         for key in CATEGORICAL_FEATURES     ]      embedding_outputs = [         embedding_layer(vocab_size=df[key].max() + 1)(input)         for key, input in zip(CATEGORICAL_FEATURES, embedding_inputs)     ]      deep_input = tf.keras.layers.concatenate([numeric_input] + embedding_outputs)     deep_model = tf.keras.Sequential(         [             tf.keras.layers.Dense(128, activation=\"relu\"),             tf.keras.layers.Dense(128, activation=\"relu\"),             tf.keras.layers.Dense(64, activation=\"relu\"),             tf.keras.layers.Dense(64, activation=\"relu\"),             tf.keras.layers.Dense(units=output_units),         ]     )     return tf.keras.Model(         inputs=[numeric_input] + embedding_inputs, outputs=deep_model(deep_input)     ) In\u00a0[\u00a0]: Copied! <pre>if LOSS == \"mse\":\n    loss = tf.keras.losses.MeanSquaredError()\n    output_units = 1\n\nif LOSS == \"ziln\":\n    loss = ltv.zero_inflated_lognormal_loss\n    output_units = 3\n</pre> if LOSS == \"mse\":     loss = tf.keras.losses.MeanSquaredError()     output_units = 1  if LOSS == \"ziln\":     loss = ltv.zero_inflated_lognormal_loss     output_units = 3 In\u00a0[\u00a0]: Copied! <pre>x_train, x_eval, y_train, y_eval = dnn_split(df)\nmodel = dnn_model(output_units)\n</pre> x_train, x_eval, y_train, y_eval = dnn_split(df) model = dnn_model(output_units) In\u00a0[\u00a0]: Copied! <pre>model.compile(optimizer=tf.keras.optimizers.Nadam(lr=LEARNING_RATE), loss=loss)\n</pre> model.compile(optimizer=tf.keras.optimizers.Nadam(lr=LEARNING_RATE), loss=loss) In\u00a0[\u00a0]: Copied! <pre>callbacks = [\n    tf.keras.callbacks.ReduceLROnPlateau(monitor=\"val_loss\", min_lr=1e-6),\n    tf.keras.callbacks.EarlyStopping(monitor=\"val_loss\", patience=10),\n]\n</pre> callbacks = [     tf.keras.callbacks.ReduceLROnPlateau(monitor=\"val_loss\", min_lr=1e-6),     tf.keras.callbacks.EarlyStopping(monitor=\"val_loss\", patience=10), ] In\u00a0[\u00a0]: Copied! <pre>history = model.fit(\n    x=x_train,\n    y=y_train,\n    batch_size=2048,\n    epochs=200,\n    verbose=2,\n    callbacks=callbacks,\n    validation_data=(x_eval, y_eval),\n).history\n</pre> history = model.fit(     x=x_train,     y=y_train,     batch_size=2048,     epochs=200,     verbose=2,     callbacks=callbacks,     validation_data=(x_eval, y_eval), ).history In\u00a0[\u00a0]: Copied! <pre>pd.DataFrame(history)[[\"loss\", \"val_loss\"]].plot();\n</pre> pd.DataFrame(history)[[\"loss\", \"val_loss\"]].plot(); In\u00a0[\u00a0]: Copied! <pre>if LOSS == \"mse\":\n    y_pred = model.predict(x=x_eval, batch_size=1024).flatten()\n\nif LOSS == \"ziln\":\n    logits = model.predict(x=x_eval, batch_size=1024)\n    y_pred = ltv.zero_inflated_lognormal_pred(logits).numpy().flatten()\n</pre> if LOSS == \"mse\":     y_pred = model.predict(x=x_eval, batch_size=1024).flatten()  if LOSS == \"ziln\":     logits = model.predict(x=x_eval, batch_size=1024)     y_pred = ltv.zero_inflated_lognormal_pred(logits).numpy().flatten() In\u00a0[\u00a0]: Copied! <pre>from pytorch_widedeep.preprocessing import TabPreprocessor\nfrom pytorch_widedeep.training import Trainer\nfrom pytorch_widedeep.models import TabMlp, WideDeep\nfrom torch.optim.lr_scheduler import ReduceLROnPlateau\nfrom pytorch_widedeep.callbacks import EarlyStopping\nfrom torch.optim import NAdam\n</pre> from pytorch_widedeep.preprocessing import TabPreprocessor from pytorch_widedeep.training import Trainer from pytorch_widedeep.models import TabMlp, WideDeep from torch.optim.lr_scheduler import ReduceLROnPlateau from pytorch_widedeep.callbacks import EarlyStopping from torch.optim import NAdam In\u00a0[\u00a0]: Copied! <pre># CATEGORICAL_FEATURES\nNUMERICAL_FEATURES = [\"num\" + str(i) for i in range(21)]\nx_train_pyt_num = pd.DataFrame(columns=NUMERICAL_FEATURES, data=x_train[\"numeric\"])\nx_train_pyt_cat = pd.DataFrame(\n    {key: value for key, value in x_train.items() if key not in [\"numeric\"]}\n)\n\nx_eval_pyt_num = pd.DataFrame(columns=NUMERICAL_FEATURES, data=x_eval[\"numeric\"])\nx_eval_pyt_cat = pd.DataFrame(\n    {key: value for key, value in x_eval.items() if key not in [\"numeric\"]}\n)\n</pre> # CATEGORICAL_FEATURES NUMERICAL_FEATURES = [\"num\" + str(i) for i in range(21)] x_train_pyt_num = pd.DataFrame(columns=NUMERICAL_FEATURES, data=x_train[\"numeric\"]) x_train_pyt_cat = pd.DataFrame(     {key: value for key, value in x_train.items() if key not in [\"numeric\"]} )  x_eval_pyt_num = pd.DataFrame(columns=NUMERICAL_FEATURES, data=x_eval[\"numeric\"]) x_eval_pyt_cat = pd.DataFrame(     {key: value for key, value in x_eval.items() if key not in [\"numeric\"]} ) In\u00a0[\u00a0]: Copied! <pre>x_train_pyt = pd.concat([x_train_pyt_num, x_train_pyt_cat], axis=1)\nx_eval_pyt = pd.concat([x_eval_pyt_num, x_eval_pyt_cat], axis=1)\n</pre> x_train_pyt = pd.concat([x_train_pyt_num, x_train_pyt_cat], axis=1) x_eval_pyt = pd.concat([x_eval_pyt_num, x_eval_pyt_cat], axis=1) In\u00a0[\u00a0]: Copied! <pre>embed_input = [\n    (u, int(x_train_pyt[u].nunique() ** 0.25) + 1) for u in CATEGORICAL_FEATURES\n]\n</pre> embed_input = [     (u, int(x_train_pyt[u].nunique() ** 0.25) + 1) for u in CATEGORICAL_FEATURES ] In\u00a0[\u00a0]: Copied! <pre># deeptabular\ntab_preprocessor = TabPreprocessor(\n    embed_cols=embed_input,\n    continuous_cols=NUMERICAL_FEATURES,\n    shared_embed=False,\n    scale=False,\n)\nX_tab_train = tab_preprocessor.fit_transform(x_train_pyt)\nX_tab_valid = tab_preprocessor.transform(x_eval_pyt)\nX_tab_test = tab_preprocessor.transform(x_eval_pyt)\n\n# target\ny_train = y_train\ny_valid = y_eval\ny_test = y_train\n\nX_train = {\"X_tab\": X_tab_train, \"target\": y_train}\nX_val = {\"X_tab\": X_tab_valid, \"target\": y_valid}\nX_test = {\"X_tab\": X_tab_test}\n\ndeeptabular = TabMlp(\n    mlp_hidden_dims=[128, 128, 64, 64],\n    column_idx=tab_preprocessor.column_idx,\n    embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=tab_preprocessor.continuous_cols,\n)\n\nmodel = WideDeep(deeptabular=deeptabular, pred_dim=3)\n\ndeep_opt = NAdam(model.deeptabular.parameters(), lr=LEARNING_RATE)\ncallbacks = [EarlyStopping()]\ndeep_sch = ReduceLROnPlateau(deep_opt, min_lr=1e-6)\n\nobjective = \"ziln\"\n\ntrainer = Trainer(\n    model,\n    callbacks=callbacks,\n    lr_schedulers={\"deeptabular\": deep_sch},\n    objective=objective,\n    optimizers={\"deeptabular\": deep_opt},\n)\n\ntrainer.fit(\n    X_train=X_train,\n    X_val=X_val,\n    n_epochs=200,\n    batch_size=2048,\n)\n\ny_pred_pytorch = trainer.predict(X_test=X_test)\n</pre> # deeptabular tab_preprocessor = TabPreprocessor(     embed_cols=embed_input,     continuous_cols=NUMERICAL_FEATURES,     shared_embed=False,     scale=False, ) X_tab_train = tab_preprocessor.fit_transform(x_train_pyt) X_tab_valid = tab_preprocessor.transform(x_eval_pyt) X_tab_test = tab_preprocessor.transform(x_eval_pyt)  # target y_train = y_train y_valid = y_eval y_test = y_train  X_train = {\"X_tab\": X_tab_train, \"target\": y_train} X_val = {\"X_tab\": X_tab_valid, \"target\": y_valid} X_test = {\"X_tab\": X_tab_test}  deeptabular = TabMlp(     mlp_hidden_dims=[128, 128, 64, 64],     column_idx=tab_preprocessor.column_idx,     embed_input=tab_preprocessor.cat_embed_input,     continuous_cols=tab_preprocessor.continuous_cols, )  model = WideDeep(deeptabular=deeptabular, pred_dim=3)  deep_opt = NAdam(model.deeptabular.parameters(), lr=LEARNING_RATE) callbacks = [EarlyStopping()] deep_sch = ReduceLROnPlateau(deep_opt, min_lr=1e-6)  objective = \"ziln\"  trainer = Trainer(     model,     callbacks=callbacks,     lr_schedulers={\"deeptabular\": deep_sch},     objective=objective,     optimizers={\"deeptabular\": deep_opt}, )  trainer.fit(     X_train=X_train,     X_val=X_val,     n_epochs=200,     batch_size=2048, )  y_pred_pytorch = trainer.predict(X_test=X_test) In\u00a0[\u00a0]: Copied! <pre>pd.DataFrame(trainer.history)[[\"train_loss\", \"val_loss\"]].plot();\n</pre> pd.DataFrame(trainer.history)[[\"train_loss\", \"val_loss\"]].plot(); In\u00a0[\u00a0]: Copied! <pre>from sklearn.metrics import mean_squared_error\n\nmean_squared_error(y_pred, y_pred_pytorch)\n</pre> from sklearn.metrics import mean_squared_error  mean_squared_error(y_pred, y_pred_pytorch) In\u00a0[\u00a0]: Copied! <pre>unit_costs = [0.4, 0.5, 0.6, 0.68, 0.7, 0.8, 0.9, 1.0]\n</pre> unit_costs = [0.4, 0.5, 0.6, 0.68, 0.7, 0.8, 0.9, 1.0] In\u00a0[\u00a0]: Copied! <pre>num_mailed = [np.sum(y_pred &gt; v) for v in unit_costs]\nnum_mailed\n</pre> num_mailed = [np.sum(y_pred &gt; v) for v in unit_costs] num_mailed In\u00a0[\u00a0]: Copied! <pre>baseline_total_profit = np.sum(y_eval - 0.68)\nbaseline_total_profit\n</pre> baseline_total_profit = np.sum(y_eval - 0.68) baseline_total_profit In\u00a0[\u00a0]: Copied! <pre>total_profits = [np.sum(y_eval[y_pred &gt; v] - v) for v in unit_costs]\ntotal_profits\n</pre> total_profits = [np.sum(y_eval[y_pred &gt; v] - v) for v in unit_costs] total_profits In\u00a0[\u00a0]: Copied! <pre>gain = pd.DataFrame(\n    {\n        \"lorenz\": ltv.cumulative_true(y_eval, y_eval),\n        \"baseline\": ltv.cumulative_true(y_eval, x_eval[\"numeric\"][:, 19]),\n        \"model\": ltv.cumulative_true(y_eval, y_pred),\n    }\n)\n</pre> gain = pd.DataFrame(     {         \"lorenz\": ltv.cumulative_true(y_eval, y_eval),         \"baseline\": ltv.cumulative_true(y_eval, x_eval[\"numeric\"][:, 19]),         \"model\": ltv.cumulative_true(y_eval, y_pred),     } ) In\u00a0[\u00a0]: Copied! <pre>num_customers = np.float32(gain.shape[0])\ngain[\"cumulative_customer\"] = (np.arange(num_customers) + 1.0) / num_customers\n</pre> num_customers = np.float32(gain.shape[0]) gain[\"cumulative_customer\"] = (np.arange(num_customers) + 1.0) / num_customers In\u00a0[\u00a0]: Copied! <pre>ax = gain[\n    [\n        \"cumulative_customer\",\n        \"lorenz\",\n        \"baseline\",\n        \"model\",\n    ]\n].plot(x=\"cumulative_customer\", figsize=(8, 5), legend=True)\n\nax.legend([\"Groundtruth\", \"Baseline\", \"Model\"], loc=\"lower right\")\n\nax.set_xlabel(\"Cumulative Fraction of Customers\")\nax.set_xticks(np.arange(0, 1.1, 0.1))\nax.set_xlim((0, 1.0))\n\nax.set_ylabel(\"Cumulative Fraction of Total Lifetime Value\")\nax.set_yticks(np.arange(0, 1.1, 0.1))\nax.set_ylim((0, 1.05))\nax.set_title(\"Gain Chart\");\n</pre> ax = gain[     [         \"cumulative_customer\",         \"lorenz\",         \"baseline\",         \"model\",     ] ].plot(x=\"cumulative_customer\", figsize=(8, 5), legend=True)  ax.legend([\"Groundtruth\", \"Baseline\", \"Model\"], loc=\"lower right\")  ax.set_xlabel(\"Cumulative Fraction of Customers\") ax.set_xticks(np.arange(0, 1.1, 0.1)) ax.set_xlim((0, 1.0))  ax.set_ylabel(\"Cumulative Fraction of Total Lifetime Value\") ax.set_yticks(np.arange(0, 1.1, 0.1)) ax.set_ylim((0, 1.05)) ax.set_title(\"Gain Chart\"); In\u00a0[\u00a0]: Copied! <pre>gini = ltv.gini_from_gain(gain[[\"lorenz\", \"baseline\", \"model\"]])\ngini\n</pre> gini = ltv.gini_from_gain(gain[[\"lorenz\", \"baseline\", \"model\"]]) gini In\u00a0[\u00a0]: Copied! <pre>df_decile = ltv.decile_stats(y_eval, y_pred)\ndf_decile\n</pre> df_decile = ltv.decile_stats(y_eval, y_pred) df_decile In\u00a0[\u00a0]: Copied! <pre>ax = df_decile[[\"label_mean\", \"pred_mean\"]].plot.bar(rot=0)\n\nax.set_title(\"Decile Chart\")\nax.set_xlabel(\"Prediction bucket\")\nax.set_ylabel(\"Average bucket value\")\nax.legend([\"Label\", \"Prediction\"], loc=\"upper left\");\n</pre> ax = df_decile[[\"label_mean\", \"pred_mean\"]].plot.bar(rot=0)  ax.set_title(\"Decile Chart\") ax.set_xlabel(\"Prediction bucket\") ax.set_ylabel(\"Average bucket value\") ax.legend([\"Label\", \"Prediction\"], loc=\"upper left\"); In\u00a0[\u00a0]: Copied! <pre>def spearmanr(x1: Sequence[float], x2: Sequence[float]) -&gt; float:\n    \"\"\"Calculates spearmanr rank correlation coefficient.\n\n    See https://docs.scipy.org/doc/scipy/reference/stats.html.\n\n    Args:\n      x1: 1D array_like.\n      x2: 1D array_like.\n\n    Returns:\n      correlation: float.\n    \"\"\"\n    return stats.spearmanr(x1, x2, nan_policy=\"raise\")[0]\n\n\nspearman_corr = spearmanr(y_eval, y_pred)\nspearman_corr\n</pre> def spearmanr(x1: Sequence[float], x2: Sequence[float]) -&gt; float:     \"\"\"Calculates spearmanr rank correlation coefficient.      See https://docs.scipy.org/doc/scipy/reference/stats.html.      Args:       x1: 1D array_like.       x2: 1D array_like.      Returns:       correlation: float.     \"\"\"     return stats.spearmanr(x1, x2, nan_policy=\"raise\")[0]   spearman_corr = spearmanr(y_eval, y_pred) spearman_corr In\u00a0[\u00a0]: Copied! <pre>df_metrics = pd.DataFrame(\n    {\n        \"model\": MODEL,\n        \"loss_function\": LOSS,\n        \"train_loss\": history[\"loss\"][-1],\n        \"eval_loss\": history[\"val_loss\"][-1],\n        \"label_positive\": np.mean(y_eval &gt; 0),\n        \"label_mean\": y_eval.mean(),\n        \"pred_mean\": y_pred.mean(),\n        \"decile_mape\": df_decile[\"decile_mape\"].mean(),\n        \"baseline_gini\": gini[\"normalized\"][1],\n        \"gini\": gini[\"normalized\"][2],\n        \"spearman_corr\": spearman_corr,\n    },\n    index=[VERSION],\n)\n</pre> df_metrics = pd.DataFrame(     {         \"model\": MODEL,         \"loss_function\": LOSS,         \"train_loss\": history[\"loss\"][-1],         \"eval_loss\": history[\"val_loss\"][-1],         \"label_positive\": np.mean(y_eval &gt; 0),         \"label_mean\": y_eval.mean(),         \"pred_mean\": y_pred.mean(),         \"decile_mape\": df_decile[\"decile_mape\"].mean(),         \"baseline_gini\": gini[\"normalized\"][1],         \"gini\": gini[\"normalized\"][2],         \"spearman_corr\": spearman_corr,     },     index=[VERSION], ) In\u00a0[\u00a0]: Copied! <pre>for unit_cost, total_profit in zip(unit_costs, total_profits):\n    df_metrics[\"total_profit_{:02d}\".format(int(unit_cost * 100))] = total_profit\n</pre> for unit_cost, total_profit in zip(unit_costs, total_profits):     df_metrics[\"total_profit_{:02d}\".format(int(unit_cost * 100))] = total_profit In\u00a0[\u00a0]: Copied! <pre>df_metrics.T\n</pre> df_metrics.T In\u00a0[\u00a0]: Copied! <pre>output_path = OUTPUT_CSV_FOLDER\n</pre> output_path = OUTPUT_CSV_FOLDER In\u00a0[\u00a0]: Copied! <pre>if not os.path.isdir(output_path):\n    os.makedirs(output_path)\n</pre> if not os.path.isdir(output_path):     os.makedirs(output_path) In\u00a0[\u00a0]: Copied! <pre>output_file = os.path.join(\n    output_path, \"{}_regression_{}_{}.csv\".format(MODEL, LOSS, VERSION)\n)\n</pre> output_file = os.path.join(     output_path, \"{}_regression_{}_{}.csv\".format(MODEL, LOSS, VERSION) ) In\u00a0[\u00a0]: Copied! <pre>df_metrics.to_csv(output_file, index=False)\n</pre> df_metrics.to_csv(output_file, index=False)"},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#zilnloss","title":"ZILNLoss\u00b6","text":"<p>[DISCLAIMER]</p> <p>Purpose of this notebook is to check if ZILNloss implemented originaly Keras give same results in pytorch-widedeep implemenatation</p>"},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#kdd-cup-98-ltv-prediction","title":"KDD Cup 98 LTV Prediction\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#configs","title":"Configs\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#load-data","title":"Load data\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#label-distribution","title":"Label distribution\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#preprocess-features","title":"Preprocess features\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#vocab","title":"Vocab\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#indicator","title":"Indicator\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#numeric","title":"Numeric\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#all","title":"All\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#traineval-split","title":"Train/eval split\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#model","title":"Model\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#loss","title":"Loss\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#train","title":"Train\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#eval","title":"Eval\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#pytorch-widedeep-approach","title":"Pytorch-widedeep approach\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#appendix","title":"Appendix\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#total-profit","title":"Total Profit\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#gini-coefficient","title":"Gini Coefficient\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#calibration","title":"Calibration\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#rank-correlation","title":"Rank Correlation\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#all-metrics-together","title":"All metrics together\u00b6","text":""},{"location":"examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html#save","title":"Save\u00b6","text":""},{"location":"examples/13_model_uncertainty_prediction.html","title":"13_model_uncertainty_prediction","text":"<ul> <li>In this notebook we will use the higly imbalanced Protein Homology Dataset from KDD cup 2004</li> </ul> <pre><code>* The first element of each line is a BLOCK ID that denotes to which native sequence this example belongs. There is a unique BLOCK ID for each native sequence. BLOCK IDs are integers running from 1 to 303 (one for each native sequence, i.e. for each query). BLOCK IDs were assigned before the blocks were split into the train and test sets, so they do not run consecutively in either file.\n* The second element of each line is an EXAMPLE ID that uniquely describes the example. You will need this EXAMPLE ID and the BLOCK ID when you submit results.\n* The third element is the class of the example. Proteins that are homologous to the native sequence are denoted by 1, non-homologous proteins (i.e. decoys) by 0. Test examples have a \"?\" in this position.\n* All following elements are feature values. There are 74 feature values in each line. The features describe the match (e.g. the score of a sequence alignment) between the native protein sequence and the sequence that is tested for homology.\n</code></pre> In\u00a0[1]: Copied! <pre>import pandas as pd\nimport numpy as np\n\nfrom pytorch_widedeep import Trainer\nfrom pytorch_widedeep.preprocessing import TabPreprocessor\nfrom pytorch_widedeep.models import TabMlp, WideDeep\nfrom pytorch_widedeep.dataloaders import DataLoaderImbalanced\nfrom pytorch_widedeep.metrics import Accuracy, Recall, Precision, F1Score\nfrom pytorch_widedeep.initializers import XavierNormal\nfrom pytorch_widedeep.datasets import load_bio_kdd04\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import classification_report\n\nimport time\nimport datetime\n\nimport warnings\n\nwarnings.filterwarnings(\"ignore\", category=DeprecationWarning)\n</pre> import pandas as pd import numpy as np  from pytorch_widedeep import Trainer from pytorch_widedeep.preprocessing import TabPreprocessor from pytorch_widedeep.models import TabMlp, WideDeep from pytorch_widedeep.dataloaders import DataLoaderImbalanced from pytorch_widedeep.metrics import Accuracy, Recall, Precision, F1Score from pytorch_widedeep.initializers import XavierNormal from pytorch_widedeep.datasets import load_bio_kdd04  from sklearn.model_selection import train_test_split from sklearn.metrics import classification_report  import time import datetime  import warnings  warnings.filterwarnings(\"ignore\", category=DeprecationWarning) <pre>/Users/javierrodriguezzaurin/.pyenv/versions/3.10.15/envs/widedeep310/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n  from .autonotebook import tqdm as notebook_tqdm\n</pre> In\u00a0[2]: Copied! <pre>df = load_bio_kdd04(as_frame=True)\ndf.head()\n</pre> df = load_bio_kdd04(as_frame=True) df.head() Out[2]: EXAMPLE_ID BLOCK_ID target 4 5 6 7 8 9 10 ... 68 69 70 71 72 73 74 75 76 77 0 279 261532 0 52.0 32.69 0.30 2.5 20.0 1256.8 -0.89 ... -8.0 1595.1 -1.64 2.83 -2.0 -50.0 445.2 -0.35 0.26 0.76 1 279 261533 0 58.0 33.33 0.00 16.5 9.5 608.1 0.50 ... -6.0 762.9 0.29 0.82 -3.0 -35.0 140.3 1.16 0.39 0.73 2 279 261534 0 77.0 27.27 -0.91 6.0 58.5 1623.6 -1.40 ... 7.0 1491.8 0.32 -1.29 0.0 -34.0 658.2 -0.76 0.26 0.24 3 279 261535 0 41.0 27.91 -0.35 3.0 46.0 1921.6 -1.36 ... 6.0 2047.7 -0.98 1.53 0.0 -49.0 554.2 -0.83 0.39 0.73 4 279 261536 0 50.0 28.00 -1.32 -9.0 12.0 464.8 0.88 ... -14.0 479.5 0.68 -0.59 2.0 -36.0 -6.9 2.02 0.14 -0.23 <p>5 rows \u00d7 77 columns</p> In\u00a0[3]: Copied! <pre># drop columns we won't need in this example\ndf.drop(columns=[\"EXAMPLE_ID\", \"BLOCK_ID\"], inplace=True)\n</pre> # drop columns we won't need in this example df.drop(columns=[\"EXAMPLE_ID\", \"BLOCK_ID\"], inplace=True) In\u00a0[4]: Copied! <pre>df_train, df_valid = train_test_split(\n    df, test_size=0.2, stratify=df[\"target\"], random_state=1\n)\ndf_valid, df_test = train_test_split(\n    df_valid, test_size=0.5, stratify=df_valid[\"target\"], random_state=1\n)\n</pre> df_train, df_valid = train_test_split(     df, test_size=0.2, stratify=df[\"target\"], random_state=1 ) df_valid, df_test = train_test_split(     df_valid, test_size=0.5, stratify=df_valid[\"target\"], random_state=1 ) In\u00a0[5]: Copied! <pre>continuous_cols = df.drop(columns=[\"target\"]).columns.values.tolist()\n</pre> continuous_cols = df.drop(columns=[\"target\"]).columns.values.tolist() In\u00a0[6]: Copied! <pre># deeptabular\ntab_preprocessor = TabPreprocessor(continuous_cols=continuous_cols, scale=True)\nX_tab_train = tab_preprocessor.fit_transform(df_train)\nX_tab_valid = tab_preprocessor.transform(df_valid)\nX_tab_test = tab_preprocessor.transform(df_test)\n\n# target\ny_train = df_train[\"target\"].values\ny_valid = df_valid[\"target\"].values\ny_test = df_test[\"target\"].values\n</pre> # deeptabular tab_preprocessor = TabPreprocessor(continuous_cols=continuous_cols, scale=True) X_tab_train = tab_preprocessor.fit_transform(df_train) X_tab_valid = tab_preprocessor.transform(df_valid) X_tab_test = tab_preprocessor.transform(df_test)  # target y_train = df_train[\"target\"].values y_valid = df_valid[\"target\"].values y_test = df_test[\"target\"].values In\u00a0[7]: Copied! <pre>deeptabular = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    continuous_cols=tab_preprocessor.continuous_cols,\n    mlp_hidden_dims=[64, 32],\n)\nmodel = WideDeep(deeptabular=deeptabular, pred_dim=1)\nmodel\n</pre> deeptabular = TabMlp(     column_idx=tab_preprocessor.column_idx,     continuous_cols=tab_preprocessor.continuous_cols,     mlp_hidden_dims=[64, 32], ) model = WideDeep(deeptabular=deeptabular, pred_dim=1) model Out[7]: <pre>WideDeep(\n  (deeptabular): Sequential(\n    (0): TabMlp(\n      (cont_norm): Identity()\n      (encoder): MLP(\n        (mlp): Sequential(\n          (dense_layer_0): Sequential(\n            (0): Linear(in_features=74, out_features=64, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_1): Sequential(\n            (0): Linear(in_features=64, out_features=32, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=32, out_features=1, bias=True)\n  )\n)</pre> In\u00a0[8]: Copied! <pre>trainer = Trainer(\n    model,\n    objective=\"binary\",\n    metrics=[Accuracy(), Precision(), F1Score(), Recall()],\n    verbose=1,\n)\n</pre> trainer = Trainer(     model,     objective=\"binary\",     metrics=[Accuracy(), Precision(), F1Score(), Recall()],     verbose=1, ) In\u00a0[9]: Copied! <pre>start = time.time()\ntrainer.fit(\n    X_train={\"X_tab\": X_tab_train, \"target\": y_train},\n    X_val={\"X_tab\": X_tab_valid, \"target\": y_valid},\n    n_epochs=3,\n    batch_size=32,\n)\n</pre> start = time.time() trainer.fit(     X_train={\"X_tab\": X_tab_train, \"target\": y_train},     X_val={\"X_tab\": X_tab_valid, \"target\": y_valid},     n_epochs=3,     batch_size=32, ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 3644/3644 [00:20&lt;00:00, 175.39it/s, loss=0.0222, metrics={'acc': 0.9945, 'prec': 0.7565, 'f1': 0.6419, 'rec': 0.5574}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 456/456 [00:01&lt;00:00, 252.36it/s, loss=0.0125, metrics={'acc': 0.9969, 'prec': 0.92, 'f1': 0.8035, 'rec': 0.7132}]  \nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 3644/3644 [00:20&lt;00:00, 177.43it/s, loss=0.0119, metrics={'acc': 0.9968, 'prec': 0.9209, 'f1': 0.793, 'rec': 0.6962}] \nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 456/456 [00:01&lt;00:00, 255.61it/s, loss=0.0121, metrics={'acc': 0.997, 'prec': 0.8972, 'f1': 0.8136, 'rec': 0.7442}] \nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 3644/3644 [00:20&lt;00:00, 176.07it/s, loss=0.0103, metrics={'acc': 0.9973, 'prec': 0.9312, 'f1': 0.8351, 'rec': 0.757}] \nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 456/456 [00:01&lt;00:00, 259.70it/s, loss=0.0119, metrics={'acc': 0.997, 'prec': 0.8909, 'f1': 0.8201, 'rec': 0.7597}] \n</pre> In\u00a0[10]: Copied! <pre>pd.DataFrame(trainer.history)\n</pre> pd.DataFrame(trainer.history) Out[10]: train_loss train_acc train_prec train_f1 train_rec val_loss val_acc val_prec val_f1 val_rec 0 0.022229 0.994468 0.756545 0.641866 0.557377 0.012473 0.996913 0.920000 0.803493 0.713178 1 0.011912 0.996767 0.920918 0.792971 0.696239 0.012088 0.996981 0.897196 0.813559 0.744186 2 0.010341 0.997341 0.931198 0.835106 0.756991 0.011884 0.997050 0.890909 0.820084 0.759690 In\u00a0[11]: Copied! <pre>df_pred = trainer.predict(X_tab=X_tab_test)\nprint(classification_report(df_test[\"target\"].to_list(), df_pred))\nprint(\"Actual predicted values:\\n{}\".format(np.unique(df_pred, return_counts=True)))\n</pre> df_pred = trainer.predict(X_tab=X_tab_test) print(classification_report(df_test[\"target\"].to_list(), df_pred)) print(\"Actual predicted values:\\n{}\".format(np.unique(df_pred, return_counts=True))) <pre>predict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 456/456 [00:00&lt;00:00, 689.36it/s]\n</pre> <pre>              precision    recall  f1-score   support\n\n           0       1.00      1.00      1.00     14446\n           1       0.91      0.78      0.84       130\n\n    accuracy                           1.00     14576\n   macro avg       0.95      0.89      0.92     14576\nweighted avg       1.00      1.00      1.00     14576\n\nActual predicted values:\n(array([0, 1]), array([14465,   111]))\n</pre> In\u00a0[12]: Copied! <pre>df_pred_unc = trainer.predict_uncertainty(X_tab=X_tab_test, uncertainty_granularity=10)\nprint(classification_report(df_test[\"target\"].to_list(), df_pred))\nprint(\n    \"Actual predicted values:\\n{}\".format(\n        np.unique(df_pred_unc[:, -1], return_counts=True)\n    )\n)\n</pre> df_pred_unc = trainer.predict_uncertainty(X_tab=X_tab_test, uncertainty_granularity=10) print(classification_report(df_test[\"target\"].to_list(), df_pred)) print(     \"Actual predicted values:\\n{}\".format(         np.unique(df_pred_unc[:, -1], return_counts=True)     ) ) <pre>predict_UncertaintyIter: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 10/10 [00:05&lt;00:00,  1.86it/s]</pre> <pre>              precision    recall  f1-score   support\n\n           0       1.00      1.00      1.00     14446\n           1       0.91      0.78      0.84       130\n\n    accuracy                           1.00     14576\n   macro avg       0.95      0.89      0.92     14576\nweighted avg       1.00      1.00      1.00     14576\n\nActual predicted values:\n(array([0.]), array([14576]))\n</pre> <pre>\n</pre> In\u00a0[13]: Copied! <pre>df_pred_unc\n</pre> df_pred_unc Out[13]: <pre>array([[9.99999821e-01, 1.77245539e-07, 0.00000000e+00],\n       [1.00000000e+00, 8.29310925e-11, 0.00000000e+00],\n       [9.99995947e-01, 4.06420531e-06, 0.00000000e+00],\n       ...,\n       [9.99999940e-01, 3.85314713e-08, 0.00000000e+00],\n       [1.00000000e+00, 2.98146707e-09, 0.00000000e+00],\n       [1.00000000e+00, 1.21332046e-12, 0.00000000e+00]])</pre>"},{"location":"examples/13_model_uncertainty_prediction.html#model-uncertainty-prediction","title":"Model Uncertainty prediction\u00b6","text":"<p>Note:</p> <p>This notebook extends the \"Custom DataLoader for Imbalanced dataset\" notebook</p>"},{"location":"examples/13_model_uncertainty_prediction.html#initial-imports","title":"Initial imports\u00b6","text":""},{"location":"examples/13_model_uncertainty_prediction.html#preparing-the-data","title":"Preparing the data\u00b6","text":""},{"location":"examples/13_model_uncertainty_prediction.html#define-the-model","title":"Define the model\u00b6","text":""},{"location":"examples/13_model_uncertainty_prediction.html#normal-prediction","title":"\"Normal\" prediction\u00b6","text":""},{"location":"examples/13_model_uncertainty_prediction.html#prediction-using-uncertainty","title":"Prediction using uncertainty\u00b6","text":""},{"location":"examples/14_bayesian_models.html","title":"14_bayesian_models","text":"In\u00a0[1]: Copied! <pre>import numpy as np\nimport torch\nimport pandas as pd\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import accuracy_score\n\nfrom pytorch_widedeep.metrics import Accuracy\nfrom pytorch_widedeep.datasets import load_adult\nfrom pytorch_widedeep.callbacks import EarlyStopping, ModelCheckpoint\nfrom pytorch_widedeep.preprocessing import TabPreprocessor, WidePreprocessor\nfrom pytorch_widedeep.bayesian_models import BayesianWide, BayesianTabMlp\nfrom pytorch_widedeep.training.bayesian_trainer import BayesianTrainer\n</pre> import numpy as np import torch import pandas as pd  from sklearn.model_selection import train_test_split from sklearn.metrics import accuracy_score  from pytorch_widedeep.metrics import Accuracy from pytorch_widedeep.datasets import load_adult from pytorch_widedeep.callbacks import EarlyStopping, ModelCheckpoint from pytorch_widedeep.preprocessing import TabPreprocessor, WidePreprocessor from pytorch_widedeep.bayesian_models import BayesianWide, BayesianTabMlp from pytorch_widedeep.training.bayesian_trainer import BayesianTrainer <p>The first few things to do we know them very well, like with any other model described in any of the other notebooks</p> In\u00a0[2]: Copied! <pre>df = load_adult(as_frame=True)\ndf.columns = [c.replace(\"-\", \"_\") for c in df.columns]\ndf[\"age_buckets\"] = pd.cut(\n    df.age, bins=[16, 25, 30, 35, 40, 45, 50, 55, 60, 91], labels=np.arange(9)\n)\ndf[\"income_label\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int)\ndf.drop(\"income\", axis=1, inplace=True)\ndf.head()\n</pre> df = load_adult(as_frame=True) df.columns = [c.replace(\"-\", \"_\") for c in df.columns] df[\"age_buckets\"] = pd.cut(     df.age, bins=[16, 25, 30, 35, 40, 45, 50, 55, 60, 91], labels=np.arange(9) ) df[\"income_label\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int) df.drop(\"income\", axis=1, inplace=True) df.head() Out[2]: age workclass fnlwgt education educational_num marital_status occupation relationship race gender capital_gain capital_loss hours_per_week native_country age_buckets income_label 0 25 Private 226802 11th 7 Never-married Machine-op-inspct Own-child Black Male 0 0 40 United-States 0 0 1 38 Private 89814 HS-grad 9 Married-civ-spouse Farming-fishing Husband White Male 0 0 50 United-States 3 0 2 28 Local-gov 336951 Assoc-acdm 12 Married-civ-spouse Protective-serv Husband White Male 0 0 40 United-States 1 1 3 44 Private 160323 Some-college 10 Married-civ-spouse Machine-op-inspct Husband Black Male 7688 0 40 United-States 4 1 4 18 ? 103497 Some-college 10 Never-married ? Own-child White Female 0 0 30 United-States 0 0 In\u00a0[3]: Copied! <pre>train, test = train_test_split(df, test_size=0.2, stratify=df.income_label)\n</pre> train, test = train_test_split(df, test_size=0.2, stratify=df.income_label) In\u00a0[4]: Copied! <pre>wide_cols = [\n    \"age_buckets\",\n    \"education\",\n    \"relationship\",\n    \"workclass\",\n    \"occupation\",\n    \"native_country\",\n    \"gender\",\n]\ncrossed_cols = [(\"education\", \"occupation\"), (\"native_country\", \"occupation\")]\n\ncat_embed_cols = [\n    \"workclass\",\n    \"education\",\n    \"marital_status\",\n    \"occupation\",\n    \"relationship\",\n    \"race\",\n    \"gender\",\n    \"capital_gain\",\n    \"capital_loss\",\n    \"native_country\",\n]\ncontinuous_cols = [\"age\", \"hours_per_week\"]\n\ntarget = train[\"income_label\"].values\n</pre> wide_cols = [     \"age_buckets\",     \"education\",     \"relationship\",     \"workclass\",     \"occupation\",     \"native_country\",     \"gender\", ] crossed_cols = [(\"education\", \"occupation\"), (\"native_country\", \"occupation\")]  cat_embed_cols = [     \"workclass\",     \"education\",     \"marital_status\",     \"occupation\",     \"relationship\",     \"race\",     \"gender\",     \"capital_gain\",     \"capital_loss\",     \"native_country\", ] continuous_cols = [\"age\", \"hours_per_week\"]  target = train[\"income_label\"].values In\u00a0[5]: Copied! <pre>wide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols)\nX_tab = wide_preprocessor.fit_transform(train)\n</pre> wide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols) X_tab = wide_preprocessor.fit_transform(train) In\u00a0[6]: Copied! <pre>model = BayesianWide(\n    input_dim=np.unique(X_tab).shape[0],\n    prior_sigma_1=1.0,\n    prior_sigma_2=0.002,\n    prior_pi=0.8,\n    posterior_mu_init=0,\n    posterior_rho_init=-7.0,\n    pred_dim=1,  # here the models are NOT passed to a WideDeep constructor class so the output dim MUST be specified\n)\n</pre> model = BayesianWide(     input_dim=np.unique(X_tab).shape[0],     prior_sigma_1=1.0,     prior_sigma_2=0.002,     prior_pi=0.8,     posterior_mu_init=0,     posterior_rho_init=-7.0,     pred_dim=1,  # here the models are NOT passed to a WideDeep constructor class so the output dim MUST be specified ) In\u00a0[7]: Copied! <pre>trainer = BayesianTrainer(\n    model,\n    objective=\"binary\",\n    optimizer=torch.optim.Adam(model.parameters(), lr=0.01),\n    metrics=[Accuracy],\n)\n</pre> trainer = BayesianTrainer(     model,     objective=\"binary\",     optimizer=torch.optim.Adam(model.parameters(), lr=0.01),     metrics=[Accuracy], ) In\u00a0[8]: Copied! <pre>trainer.fit(\n    X_tab=X_tab,\n    target=target,\n    val_split=0.2,\n    n_epochs=2,\n    batch_size=256,\n)\n</pre> trainer.fit(     X_tab=X_tab,     target=target,     val_split=0.2,     n_epochs=2,     batch_size=256, ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 123/123 [00:00&lt;00:00, 124.32it/s, loss=163, metrics={'acc': 0.7813}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 31/31 [00:00&lt;00:00, 238.67it/s, loss=141, metrics={'acc': 0.8219}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 123/123 [00:00&lt;00:00, 132.81it/s, loss=140, metrics={'acc': 0.8285}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 31/31 [00:00&lt;00:00, 190.16it/s, loss=140, metrics={'acc': 0.8298}]\n</pre> In\u00a0[9]: Copied! <pre>tab_preprocessor = TabPreprocessor(\n    cat_embed_cols=cat_embed_cols, continuous_cols=continuous_cols\n)\nX_tab = tab_preprocessor.fit_transform(train)\n</pre> tab_preprocessor = TabPreprocessor(     cat_embed_cols=cat_embed_cols, continuous_cols=continuous_cols ) X_tab = tab_preprocessor.fit_transform(train) <pre>/Users/javierrodriguezzaurin/Projects/pytorch-widedeep/pytorch_widedeep/preprocessing/tab_preprocessor.py:358: UserWarning: Continuous columns will not be normalised\n  warnings.warn(\"Continuous columns will not be normalised\")\n</pre> In\u00a0[10]: Copied! <pre>model = BayesianTabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=continuous_cols,\n    #     embed_continuous_method = \"standard\",\n    #     cont_embed_activation=\"leaky_relu\",\n    #     cont_embed_dim = 8,\n    mlp_hidden_dims=[128, 64],\n    prior_sigma_1=1.0,\n    prior_sigma_2=0.002,\n    prior_pi=0.8,\n    posterior_mu_init=0,\n    posterior_rho_init=-7.0,\n    pred_dim=1,\n)\n</pre> model = BayesianTabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     continuous_cols=continuous_cols,     #     embed_continuous_method = \"standard\",     #     cont_embed_activation=\"leaky_relu\",     #     cont_embed_dim = 8,     mlp_hidden_dims=[128, 64],     prior_sigma_1=1.0,     prior_sigma_2=0.002,     prior_pi=0.8,     posterior_mu_init=0,     posterior_rho_init=-7.0,     pred_dim=1, ) In\u00a0[11]: Copied! <pre>trainer = BayesianTrainer(\n    model,\n    objective=\"binary\",\n    optimizer=torch.optim.Adam(model.parameters(), lr=0.01),\n    metrics=[Accuracy],\n)\n</pre> trainer = BayesianTrainer(     model,     objective=\"binary\",     optimizer=torch.optim.Adam(model.parameters(), lr=0.01),     metrics=[Accuracy], ) In\u00a0[12]: Copied! <pre>trainer.fit(\n    X_tab=X_tab,\n    target=target,\n    val_split=0.2,\n    n_epochs=2,\n    batch_size=256,\n)\n</pre> trainer.fit(     X_tab=X_tab,     target=target,     val_split=0.2,     n_epochs=2,     batch_size=256, ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 123/123 [00:04&lt;00:00, 28.74it/s, loss=2e+3, metrics={'acc': 0.8007}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 31/31 [00:00&lt;00:00, 136.89it/s, loss=1.75e+3, metrics={'acc': 0.8418}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 123/123 [00:04&lt;00:00, 29.41it/s, loss=1.73e+3, metrics={'acc': 0.8596}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 31/31 [00:00&lt;00:00, 143.87it/s, loss=1.71e+3, metrics={'acc': 0.8569}]\n</pre> <p>These models are powerful beyond the success metrics because they give us a sense of uncertainty as we predict. Let's have a look</p> In\u00a0[13]: Copied! <pre>X_tab_test = tab_preprocessor.transform(test)\n</pre> X_tab_test = tab_preprocessor.transform(test) In\u00a0[14]: Copied! <pre>preds = trainer.predict(X_tab_test, return_samples=True, n_samples=5)\n</pre> preds = trainer.predict(X_tab_test, return_samples=True, n_samples=5) <pre>predict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 39/39 [00:01&lt;00:00, 33.92it/s]\n</pre> In\u00a0[15]: Copied! <pre>preds.shape\n</pre> preds.shape Out[15]: <pre>(5, 9769)</pre> <p>as we can see the prediction have shape <code>(5, 9769)</code>, one set of predictions each time we have internally run predict (i.e. sample the network and predict, defined by the parameter <code>n_samples</code>). This gives us an idea of how certain the model is about a certain prediction.</p> <p>Similarly, we could obtain the probabilities</p> In\u00a0[16]: Copied! <pre>probs = trainer.predict_proba(X_tab_test, return_samples=True, n_samples=5)\n</pre> probs = trainer.predict_proba(X_tab_test, return_samples=True, n_samples=5) <pre>predict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 39/39 [00:01&lt;00:00, 32.79it/s]\n</pre> In\u00a0[17]: Copied! <pre>probs.shape\n</pre> probs.shape Out[17]: <pre>(5, 9769, 2)</pre> <p>And we could see how the model performs each time we sampled the network</p> In\u00a0[18]: Copied! <pre>for p in preds:\n    print(accuracy_score(p, test[\"income_label\"].values))\n</pre> for p in preds:     print(accuracy_score(p, test[\"income_label\"].values)) <pre>0.8559729757395844\n0.8564847988535162\n0.8567918927218753\n0.8562800696079435\n0.8558706111167981\n</pre> In\u00a0[\u00a0]: Copied! <pre>\n</pre>"},{"location":"examples/14_bayesian_models.html#the-bayesian-models","title":"The Bayesian Models\u00b6","text":"<p>Perhaps one of the most interesting functionality in the library is the access to full Bayesian models in almost exactly the same way one would use any of the other models in the library.</p> <p>Note however that the Bayesian models are ONLY available for tabular data and, at the moment, we do not support combining them to form a Wide and Deep model.</p> <p>The implementation in this library is based on the publication: Weight Uncertainty in Neural Networks, by Blundell et al., 2015. Code-wise, our implementation is inspired by a number of source:</p> <ol> <li>https://joshfeldman.net/WeightUncertainty/</li> <li>https://www.nitarshan.com/bayes-by-backprop/</li> <li>https://github.com/piEsposito/blitz-bayesian-deep-learning</li> <li>https://github.com/zackchase/mxnet-the-straight-dope/tree/master/chapter18_variational-methods-and-uncertainty</li> </ol> <p>The two Bayesian models available in the library are:</p> <ol> <li>BayesianWide: this is a linear model where the non-linearities are captured via crossed-columns</li> <li>BayesianMLP: this is a standard MLP that receives categorical embeddings and continuous cols (embedded or not) which are the passed through a series of dense layers. All parameters in the model are probabilistic.</li> </ol>"},{"location":"examples/14_bayesian_models.html#1-bayesianwide","title":"1. <code>BayesianWide</code>\u00b6","text":""},{"location":"examples/14_bayesian_models.html#2-bayesiantabmlp","title":"2. <code>BayesianTabMlp</code>\u00b6","text":""},{"location":"examples/15_Self_Supervised_Pretraning_pt1.html","title":"15_Self-Supervised Pre-Training pt 1","text":"In\u00a0[1]: Copied! <pre>import torch\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.model_selection import train_test_split\n\nfrom pytorch_widedeep import Trainer\nfrom pytorch_widedeep.models import TabMlp, WideDeep\nfrom pytorch_widedeep.metrics import Accuracy\nfrom pytorch_widedeep.datasets import load_adult\nfrom pytorch_widedeep.preprocessing import TabPreprocessor\nfrom pytorch_widedeep.self_supervised_training import EncoderDecoderTrainer\n</pre> import torch from sklearn.metrics import accuracy_score from sklearn.model_selection import train_test_split  from pytorch_widedeep import Trainer from pytorch_widedeep.models import TabMlp, WideDeep from pytorch_widedeep.metrics import Accuracy from pytorch_widedeep.datasets import load_adult from pytorch_widedeep.preprocessing import TabPreprocessor from pytorch_widedeep.self_supervised_training import EncoderDecoderTrainer In\u00a0[2]: Copied! <pre>df = load_adult(as_frame=True)\ndf.columns = [c.replace(\"-\", \"_\") for c in df.columns]\ndf[\"income_label\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int)\ndf.drop(\"income\", axis=1, inplace=True)\n</pre> df = load_adult(as_frame=True) df.columns = [c.replace(\"-\", \"_\") for c in df.columns] df[\"income_label\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int) df.drop(\"income\", axis=1, inplace=True) In\u00a0[3]: Copied! <pre># one could chose to use a validation set for early stopping, hyperparam\n# optimization, etc. This is just an example, so we simply use train/test\n# split\ndf_tr, df_te = train_test_split(df, test_size=0.2, stratify=df.income_label)\n</pre> # one could chose to use a validation set for early stopping, hyperparam # optimization, etc. This is just an example, so we simply use train/test # split df_tr, df_te = train_test_split(df, test_size=0.2, stratify=df.income_label) In\u00a0[4]: Copied! <pre>df_tr.head(2)\n</pre> df_tr.head(2) Out[4]: age workclass fnlwgt education educational_num marital_status occupation relationship race gender capital_gain capital_loss hours_per_week native_country income_label 9042 26 Local-gov 250551 HS-grad 9 Married-civ-spouse Craft-repair Own-child Black Male 0 0 40 United-States 0 25322 50 Private 34832 Bachelors 13 Married-civ-spouse Tech-support Husband White Male 15024 0 40 United-States 1 In\u00a0[5]: Copied! <pre># As always, we need to define which cols will be represented as embeddings\n# and which one will be continuous features\ncat_embed_cols = [\n    \"workclass\",\n    \"education\",\n    \"marital_status\",\n    \"occupation\",\n    \"relationship\",\n    \"race\",\n    \"gender\",\n    \"capital_gain\",\n    \"capital_loss\",\n    \"native_country\",\n]\ncontinuous_cols = [\"age\", \"hours_per_week\"]\ntarget_col = \"income_label\"\n</pre> # As always, we need to define which cols will be represented as embeddings # and which one will be continuous features cat_embed_cols = [     \"workclass\",     \"education\",     \"marital_status\",     \"occupation\",     \"relationship\",     \"race\",     \"gender\",     \"capital_gain\",     \"capital_loss\",     \"native_country\", ] continuous_cols = [\"age\", \"hours_per_week\"] target_col = \"income_label\" In\u00a0[6]: Copied! <pre># We prepare the data to be passed to the model\ntab_preprocessor = TabPreprocessor(\n    cat_embed_cols=cat_embed_cols, continuous_cols=continuous_cols\n)\nX_tab = tab_preprocessor.fit_transform(df_tr)\ntarget = df_tr[target_col].values\n</pre> # We prepare the data to be passed to the model tab_preprocessor = TabPreprocessor(     cat_embed_cols=cat_embed_cols, continuous_cols=continuous_cols ) X_tab = tab_preprocessor.fit_transform(df_tr) target = df_tr[target_col].values <pre>/Users/javierrodriguezzaurin/Projects/pytorch-widedeep/pytorch_widedeep/preprocessing/tab_preprocessor.py:358: UserWarning: Continuous columns will not be normalised\n  warnings.warn(\"Continuous columns will not be normalised\")\n</pre> In\u00a0[7]: Copied! <pre>X_tab[:5]\n</pre> X_tab[:5] Out[7]: <pre>array([[ 1,  1,  1,  1,  1,  1,  1,  1,  1,  1, 26, 40],\n       [ 2,  2,  1,  2,  2,  2,  1,  2,  1,  1, 50, 40],\n       [ 2,  1,  1,  3,  2,  2,  1,  1,  2,  1, 39, 46],\n       [ 2,  3,  2,  4,  1,  2,  2,  1,  1,  1, 17, 10],\n       [ 3,  4,  2,  1,  1,  2,  1,  1,  1,  1, 32, 20]])</pre> In\u00a0[8]: Copied! <pre># We define a model that will act as the encoder in the encoder/decoder\n# architecture. This could be any of: TabMlp, TabResnet or TabNet\ntab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=tab_preprocessor.continuous_cols,\n)\n</pre> # We define a model that will act as the encoder in the encoder/decoder # architecture. This could be any of: TabMlp, TabResnet or TabNet tab_mlp = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     continuous_cols=tab_preprocessor.continuous_cols, ) In\u00a0[9]: Copied! <pre>tab_mlp\n</pre> tab_mlp Out[9]: <pre>TabMlp(\n  (cat_embed): DiffSizeCatEmbeddings(\n    (embed_layers): ModuleDict(\n      (emb_layer_workclass): Embedding(10, 5, padding_idx=0)\n      (emb_layer_education): Embedding(17, 8, padding_idx=0)\n      (emb_layer_marital_status): Embedding(8, 5, padding_idx=0)\n      (emb_layer_occupation): Embedding(16, 7, padding_idx=0)\n      (emb_layer_relationship): Embedding(7, 4, padding_idx=0)\n      (emb_layer_race): Embedding(6, 4, padding_idx=0)\n      (emb_layer_gender): Embedding(3, 2, padding_idx=0)\n      (emb_layer_capital_gain): Embedding(124, 24, padding_idx=0)\n      (emb_layer_capital_loss): Embedding(98, 21, padding_idx=0)\n      (emb_layer_native_country): Embedding(42, 13, padding_idx=0)\n    )\n    (embedding_dropout): Dropout(p=0.0, inplace=False)\n  )\n  (cont_norm): Identity()\n  (encoder): MLP(\n    (mlp): Sequential(\n      (dense_layer_0): Sequential(\n        (0): Linear(in_features=95, out_features=200, bias=True)\n        (1): ReLU(inplace=True)\n        (2): Dropout(p=0.1, inplace=False)\n      )\n      (dense_layer_1): Sequential(\n        (0): Linear(in_features=200, out_features=100, bias=True)\n        (1): ReLU(inplace=True)\n        (2): Dropout(p=0.1, inplace=False)\n      )\n    )\n  )\n)</pre> In\u00a0[10]: Copied! <pre># If we do not pass a custom decoder, which is perfectly possible via the\n# decoder param,  the EncoderDecoderTrainer will automatically build a\n# decoder which will be the 'mirror' image of the encoder\nencoder_decoder_trainer = EncoderDecoderTrainer(encoder=tab_mlp)\n</pre> # If we do not pass a custom decoder, which is perfectly possible via the # decoder param,  the EncoderDecoderTrainer will automatically build a # decoder which will be the 'mirror' image of the encoder encoder_decoder_trainer = EncoderDecoderTrainer(encoder=tab_mlp) In\u00a0[11]: Copied! <pre># let's have a look to the encoder_decoder_model (aka ed_model)\nencoder_decoder_trainer.ed_model\n</pre> # let's have a look to the encoder_decoder_model (aka ed_model) encoder_decoder_trainer.ed_model Out[11]: <pre>EncoderDecoderModel(\n  (encoder): TabMlp(\n    (cat_embed): DiffSizeCatEmbeddings(\n      (embed_layers): ModuleDict(\n        (emb_layer_workclass): Embedding(10, 5, padding_idx=0)\n        (emb_layer_education): Embedding(17, 8, padding_idx=0)\n        (emb_layer_marital_status): Embedding(8, 5, padding_idx=0)\n        (emb_layer_occupation): Embedding(16, 7, padding_idx=0)\n        (emb_layer_relationship): Embedding(7, 4, padding_idx=0)\n        (emb_layer_race): Embedding(6, 4, padding_idx=0)\n        (emb_layer_gender): Embedding(3, 2, padding_idx=0)\n        (emb_layer_capital_gain): Embedding(124, 24, padding_idx=0)\n        (emb_layer_capital_loss): Embedding(98, 21, padding_idx=0)\n        (emb_layer_native_country): Embedding(42, 13, padding_idx=0)\n      )\n      (embedding_dropout): Dropout(p=0.0, inplace=False)\n    )\n    (cont_norm): Identity()\n    (encoder): MLP(\n      (mlp): Sequential(\n        (dense_layer_0): Sequential(\n          (0): Linear(in_features=95, out_features=200, bias=True)\n          (1): ReLU(inplace=True)\n          (2): Dropout(p=0.1, inplace=False)\n        )\n        (dense_layer_1): Sequential(\n          (0): Linear(in_features=200, out_features=100, bias=True)\n          (1): ReLU(inplace=True)\n          (2): Dropout(p=0.1, inplace=False)\n        )\n      )\n    )\n  )\n  (decoder): TabMlpDecoder(\n    (decoder): MLP(\n      (mlp): Sequential(\n        (dense_layer_0): Sequential(\n          (0): Linear(in_features=100, out_features=200, bias=True)\n          (1): ReLU(inplace=True)\n          (2): Dropout(p=0.1, inplace=False)\n        )\n        (dense_layer_1): Sequential(\n          (0): Linear(in_features=200, out_features=95, bias=True)\n          (1): ReLU(inplace=True)\n          (2): Dropout(p=0.1, inplace=False)\n        )\n      )\n    )\n  )\n  (masker): RandomObfuscator()\n)</pre> <p>Ignoring the <code>masker</code>, which just...well...masks, the <code>ed_model</code> consists of:</p> <ol> <li>An encoder model that is a <code>TabMlp</code> model that is in itself comprised by an Embedding layer (or rather a collection of them, referred as <code>cat_and_cont_embed</code>) and an encoder (a simple MLP, referred as <code>encoder</code>)</li> <li>A decoder which is just an \"inverted\" MLP (referred as <code>decoder</code>)</li> </ol> In\u00a0[12]: Copied! <pre># And we just...pretrain\nencoder_decoder_trainer.pretrain(X_tab, n_epochs=5, batch_size=256)\n</pre> # And we just...pretrain encoder_decoder_trainer.pretrain(X_tab, n_epochs=5, batch_size=256) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 82.90it/s, loss=4.07]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 89.87it/s, loss=3.09]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 92.86it/s, loss=2.53]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 91.24it/s, loss=2.09]\nepoch 5: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 91.38it/s, loss=1.78]\n</pre> <p>At this point we have two options, we could either save the model for later use or we could continue to supervised training. The latter is rather simple, after running:</p> <pre>encoder_decoder_trainer.pretrain(X_tab, n_epochs=5, batch_size=256)\n</pre> <p>you just have to</p> <pre>model = WideDeep(deeptabular=tab_mlp)\ntrainer = Trainer(model=model, objective=\"binary\", metrics=[Accuracy])\n\ntrainer.fit(X_tab=X_tab, target=target, n_epochs=5, batch_size=256)\n\n# And, you know...we get a test metric\nX_tab_te = tab_preprocessor.transform(df_te)\ntarget_te = df_te[target_col].values\n\npreds = trainer.predict(X_tab=X_tab_te)\ntest_acc = accuracy_score(target_te, preds)\n</pre> <p>Let's say that in any case, we are 'decent' scientists/people and we want to save the model:</p> In\u00a0[13]: Copied! <pre>encoder_decoder_trainer.save(\n    path=\"pretrained_weights\", model_filename=\"encoder_decoder_model.pt\"\n)\n</pre> encoder_decoder_trainer.save(     path=\"pretrained_weights\", model_filename=\"encoder_decoder_model.pt\" ) <p>some time has passed...</p> In\u00a0[14]: Copied! <pre>encoder_decoder_model = torch.load(\"pretrained_weights/encoder_decoder_model.pt\")\n</pre> encoder_decoder_model = torch.load(\"pretrained_weights/encoder_decoder_model.pt\") <p>Now, AND THIS IS IMPORTANT We have loaded the encoder AND the decoder. To proceed to the supervised training we ONLY need the encoder</p> In\u00a0[15]: Copied! <pre>pretrained_encoder = encoder_decoder_model.encoder\n</pre> pretrained_encoder = encoder_decoder_model.encoder In\u00a0[16]: Copied! <pre>pretrained_encoder\n</pre> pretrained_encoder Out[16]: <pre>TabMlp(\n  (cat_embed): DiffSizeCatEmbeddings(\n    (embed_layers): ModuleDict(\n      (emb_layer_workclass): Embedding(10, 5, padding_idx=0)\n      (emb_layer_education): Embedding(17, 8, padding_idx=0)\n      (emb_layer_marital_status): Embedding(8, 5, padding_idx=0)\n      (emb_layer_occupation): Embedding(16, 7, padding_idx=0)\n      (emb_layer_relationship): Embedding(7, 4, padding_idx=0)\n      (emb_layer_race): Embedding(6, 4, padding_idx=0)\n      (emb_layer_gender): Embedding(3, 2, padding_idx=0)\n      (emb_layer_capital_gain): Embedding(124, 24, padding_idx=0)\n      (emb_layer_capital_loss): Embedding(98, 21, padding_idx=0)\n      (emb_layer_native_country): Embedding(42, 13, padding_idx=0)\n    )\n    (embedding_dropout): Dropout(p=0.0, inplace=False)\n  )\n  (cont_norm): Identity()\n  (encoder): MLP(\n    (mlp): Sequential(\n      (dense_layer_0): Sequential(\n        (0): Linear(in_features=95, out_features=200, bias=True)\n        (1): ReLU(inplace=True)\n        (2): Dropout(p=0.1, inplace=False)\n      )\n      (dense_layer_1): Sequential(\n        (0): Linear(in_features=200, out_features=100, bias=True)\n        (1): ReLU(inplace=True)\n        (2): Dropout(p=0.1, inplace=False)\n      )\n    )\n  )\n)</pre> In\u00a0[17]: Copied! <pre># and as always, ANY supervised model in this library has to go throuth the WideDeep class:\nmodel = WideDeep(deeptabular=pretrained_encoder)\ntrainer = Trainer(model=model, objective=\"binary\", metrics=[Accuracy])\n\ntrainer.fit(X_tab=X_tab, target=target, n_epochs=5, batch_size=256)\n\nX_tab_te = tab_preprocessor.transform(df_te)\ntarget_te = df_te[target_col].values\n\npreds = trainer.predict(X_tab=X_tab_te)\ntest_acc = accuracy_score(target_te, preds)\nprint(test_acc)\n</pre> # and as always, ANY supervised model in this library has to go throuth the WideDeep class: model = WideDeep(deeptabular=pretrained_encoder) trainer = Trainer(model=model, objective=\"binary\", metrics=[Accuracy])  trainer.fit(X_tab=X_tab, target=target, n_epochs=5, batch_size=256)  X_tab_te = tab_preprocessor.transform(df_te) target_te = df_te[target_col].values  preds = trainer.predict(X_tab=X_tab_te) test_acc = accuracy_score(target_te, preds) print(test_acc) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 88.04it/s, loss=0.374, metrics={'acc': 0.8253}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 85.63it/s, loss=0.324, metrics={'acc': 0.8491}]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 87.56it/s, loss=0.301, metrics={'acc': 0.8608}]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:02&lt;00:00, 73.38it/s, loss=0.29, metrics={'acc': 0.8655}]\nepoch 5: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:01&lt;00:00, 78.68it/s, loss=0.284, metrics={'acc': 0.8686}]\npredict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 39/39 [00:00&lt;00:00, 173.02it/s]\n</pre> <pre>0.8730678677449074\n</pre> <p>As we mentioned before, we can also use a <code>TabResNet</code> or <code>TabNet</code> model and a custom decoder. Let's have a look:</p> In\u00a0[18]: Copied! <pre>from pytorch_widedeep.models import TabResnet as TabResnetEncoder, TabResnetDecoder\n</pre> from pytorch_widedeep.models import TabResnet as TabResnetEncoder, TabResnetDecoder In\u00a0[19]: Copied! <pre>resnet_encoder = TabResnetEncoder(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=continuous_cols,\n    blocks_dims=[200, 100, 100],\n)\n</pre> resnet_encoder = TabResnetEncoder(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     continuous_cols=continuous_cols,     blocks_dims=[200, 100, 100], ) <p>let's have a look to the model</p> In\u00a0[20]: Copied! <pre>resnet_encoder\n</pre> resnet_encoder Out[20]: <pre>TabResnet(\n  (cat_embed): DiffSizeCatEmbeddings(\n    (embed_layers): ModuleDict(\n      (emb_layer_workclass): Embedding(10, 5, padding_idx=0)\n      (emb_layer_education): Embedding(17, 8, padding_idx=0)\n      (emb_layer_marital_status): Embedding(8, 5, padding_idx=0)\n      (emb_layer_occupation): Embedding(16, 7, padding_idx=0)\n      (emb_layer_relationship): Embedding(7, 4, padding_idx=0)\n      (emb_layer_race): Embedding(6, 4, padding_idx=0)\n      (emb_layer_gender): Embedding(3, 2, padding_idx=0)\n      (emb_layer_capital_gain): Embedding(124, 24, padding_idx=0)\n      (emb_layer_capital_loss): Embedding(98, 21, padding_idx=0)\n      (emb_layer_native_country): Embedding(42, 13, padding_idx=0)\n    )\n    (embedding_dropout): Dropout(p=0.0, inplace=False)\n  )\n  (cont_norm): Identity()\n  (encoder): DenseResnet(\n    (dense_resnet): Sequential(\n      (lin_inp): Linear(in_features=95, out_features=200, bias=False)\n      (bn_inp): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n      (block_0): BasicBlock(\n        (resize): Sequential(\n          (0): Linear(in_features=200, out_features=100, bias=False)\n          (1): BatchNorm1d(100, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        )\n        (lin1): Linear(in_features=200, out_features=100, bias=False)\n        (bn1): BatchNorm1d(100, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (leaky_relu): LeakyReLU(negative_slope=0.01, inplace=True)\n        (dp): Dropout(p=0.1, inplace=False)\n        (lin2): Linear(in_features=100, out_features=100, bias=False)\n        (bn2): BatchNorm1d(100, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n      )\n      (block_1): BasicBlock(\n        (lin1): Linear(in_features=100, out_features=100, bias=False)\n        (bn1): BatchNorm1d(100, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (leaky_relu): LeakyReLU(negative_slope=0.01, inplace=True)\n        (dp): Dropout(p=0.1, inplace=False)\n        (lin2): Linear(in_features=100, out_features=100, bias=False)\n        (bn2): BatchNorm1d(100, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n      )\n    )\n  )\n)</pre> <p>As we can see, the tensor we are trying to reconstruct, the embeddings, is of size <code>94</code> (this number is stored in the attribute: <code>esnet_encoder.cat_and_cont_embed.output_dim</code>), with that information we could build or own decoder as:</p> In\u00a0[21]: Copied! <pre># for all possible params see the docs\nresnet_decoder = TabResnetDecoder(\n    embed_dim=resnet_encoder.cat_out_dim + resnet_encoder.cont_out_dim,\n    blocks_dims=[100, 100, 200],\n)\n</pre> # for all possible params see the docs resnet_decoder = TabResnetDecoder(     embed_dim=resnet_encoder.cat_out_dim + resnet_encoder.cont_out_dim,     blocks_dims=[100, 100, 200], ) In\u00a0[22]: Copied! <pre>resnet_decoder\n</pre> resnet_decoder Out[22]: <pre>TabResnetDecoder(\n  (decoder): DenseResnet(\n    (dense_resnet): Sequential(\n      (block_0): BasicBlock(\n        (lin1): Linear(in_features=100, out_features=100, bias=False)\n        (bn1): BatchNorm1d(100, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (leaky_relu): LeakyReLU(negative_slope=0.01, inplace=True)\n        (dp): Dropout(p=0.1, inplace=False)\n        (lin2): Linear(in_features=100, out_features=100, bias=False)\n        (bn2): BatchNorm1d(100, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n      )\n      (block_1): BasicBlock(\n        (resize): Sequential(\n          (0): Linear(in_features=100, out_features=200, bias=False)\n          (1): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        )\n        (lin1): Linear(in_features=100, out_features=200, bias=False)\n        (bn1): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n        (leaky_relu): LeakyReLU(negative_slope=0.01, inplace=True)\n        (dp): Dropout(p=0.1, inplace=False)\n        (lin2): Linear(in_features=200, out_features=200, bias=False)\n        (bn2): BatchNorm1d(200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n      )\n    )\n  )\n  (reconstruction_layer): Linear(in_features=200, out_features=95, bias=False)\n)</pre> <p>and now:</p> In\u00a0[23]: Copied! <pre>ec_trainer = EncoderDecoderTrainer(\n    encoder=resnet_encoder,\n    decoder=resnet_decoder,\n    masked_prob=0.2,\n)\nec_trainer.pretrain(X_tab, n_epochs=5, batch_size=256)\n</pre> ec_trainer = EncoderDecoderTrainer(     encoder=resnet_encoder,     decoder=resnet_decoder,     masked_prob=0.2, ) ec_trainer.pretrain(X_tab, n_epochs=5, batch_size=256) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:03&lt;00:00, 46.89it/s, loss=1.52]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:03&lt;00:00, 46.78it/s, loss=0.81]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:03&lt;00:00, 39.82it/s, loss=0.56]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:03&lt;00:00, 46.73it/s, loss=0.417]\nepoch 5: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:03&lt;00:00, 46.24it/s, loss=0.329]\n</pre> In\u00a0[24]: Copied! <pre># and as always, ANY supervised model in this library has to go throuth the WideDeep class:\nmodel = WideDeep(deeptabular=resnet_encoder)\ntrainer = Trainer(model=model, objective=\"binary\", metrics=[Accuracy])\n\ntrainer.fit(X_tab=X_tab, target=target, n_epochs=5, batch_size=256)\n\nX_tab_te = tab_preprocessor.transform(df_te)\ntarget_te = df_te[target_col].values\n\npreds = trainer.predict(X_tab=X_tab_te)\ntest_acc = accuracy_score(target_te, preds)\nprint(test_acc)\n</pre> # and as always, ANY supervised model in this library has to go throuth the WideDeep class: model = WideDeep(deeptabular=resnet_encoder) trainer = Trainer(model=model, objective=\"binary\", metrics=[Accuracy])  trainer.fit(X_tab=X_tab, target=target, n_epochs=5, batch_size=256)  X_tab_te = tab_preprocessor.transform(df_te) target_te = df_te[target_col].values  preds = trainer.predict(X_tab=X_tab_te) test_acc = accuracy_score(target_te, preds) print(test_acc) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:02&lt;00:00, 58.63it/s, loss=0.335, metrics={'acc': 0.8442}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:02&lt;00:00, 58.02it/s, loss=0.296, metrics={'acc': 0.864}]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:02&lt;00:00, 55.91it/s, loss=0.283, metrics={'acc': 0.8687}]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:02&lt;00:00, 55.00it/s, loss=0.276, metrics={'acc': 0.871}]\nepoch 5: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:02&lt;00:00, 51.95it/s, loss=0.272, metrics={'acc': 0.8732}]\npredict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 39/39 [00:00&lt;00:00, 120.15it/s]\n</pre> <pre>0.8725560446309756\n</pre> In\u00a0[\u00a0]: Copied! <pre>\n</pre>"},{"location":"examples/15_Self_Supervised_Pretraning_pt1.html#self-supervised-pretraining-for-tabular-data","title":"Self Supervised Pretraining for Tabular Data\u00b6","text":"<p>We have implemented two Self Supervised Pre-training routines that allow the user to pre-train all tabular models in the library with the exception of the TabPerceiver (which is a special monster).</p> <p>The two routines implemented are illustrated in the figures below. The 1st is from TabNet: Attentive Interpretable Tabular Learning and is designed for models that do not use transformer-based architectures, while the second is from SAINT: Improved Neural Networks for Tabular Data via Row Attention and Contrastive Pre-Training, and is designed for models that use transformer-based architectures.</p> <p> </p> <p>Fig 1. Figure 2 in their paper. I have included de original caption in case is useful, althought the Figure itself is pretty self explanatory</p> <p> </p> <p>Fig 2. Figure 1 in their paper. Here the caption is necessary \ud83d\ude0f</p> <p>It is beyond the scope of this notebook to explain in detail those implementations. Therefore, we strongly recommend the user to go and read the papers if this functionality is of interest to her/him.</p> <p>One thing is worth noticing however. As seen in Fig 1(the TabNet paper's Fig 2) the masking of the input features happens in the feature space. However, the implementation in this library is inspired by that at the dreamquark-ai repo, which is in itself inspired by the original implementation (by the way, at this point I will write it once again. All TabNet related things in this library are inspired when not directly based in the code in that repo, therefore, ALL CREDIT TO THE GUYS AT dreamquark-ai).</p> <p>In that implementation the masking happens in the embedding space, and currently does not mask the entire embedding (i.e. categorical feature). We decided to release as it is in this version and we will implement the exact same process described in the paper in future releases.</p> <p>Having said all of the above let's see how to use self supervision for tabular data with <code>pytorch-widedeep</code>. We will concentrate in this notebook on the 1st of the two approaches (the 'TabNet approach'). For details on the second approach please see <code>16_Self_Supervised_Pretraning_pt2</code>.</p>"},{"location":"examples/15_Self_Supervised_Pretraning_pt1.html#self-supervision-for-non-transformer-based-models","title":"Self Supervision for non-transformer-based models..\u00b6","text":"<p>...or in general, for models where the embeddigns can have all different dimensions. In this library, these are: <code>TabMlp</code>, <code>TabResNet</code> and <code>TabNet</code></p> <p>As shown in Figure, this is an encoder-encoder approach where we learn to predict values in the incoming data that have been masked. However, as I mentioned before, our implementation is a bit different, and the masking occurs in th embedding space.</p> <p>Nonetheless, the code below illustrates how to use this encoder-decoder approach with <code>pytorch-widedeep</code></p>"},{"location":"examples/15_Self_Supervised_Pretraning_pt2.html","title":"15_Self-Supervised Pre-Training pt 2","text":"In\u00a0[1]: Copied! <pre>import torch\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.model_selection import train_test_split\n\nfrom pytorch_widedeep import Trainer\nfrom pytorch_widedeep.models import WideDeep, FTTransformer\nfrom pytorch_widedeep.metrics import Accuracy\nfrom pytorch_widedeep.datasets import load_adult\nfrom pytorch_widedeep.preprocessing import TabPreprocessor\nfrom pytorch_widedeep.self_supervised_training import (\n    ContrastiveDenoisingTrainer,\n)\n</pre> import torch from sklearn.metrics import accuracy_score from sklearn.model_selection import train_test_split  from pytorch_widedeep import Trainer from pytorch_widedeep.models import WideDeep, FTTransformer from pytorch_widedeep.metrics import Accuracy from pytorch_widedeep.datasets import load_adult from pytorch_widedeep.preprocessing import TabPreprocessor from pytorch_widedeep.self_supervised_training import (     ContrastiveDenoisingTrainer, ) <pre>/Users/javierrodriguezzaurin/.pyenv/versions/3.10.15/envs/widedeep310/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n  from .autonotebook import tqdm as notebook_tqdm\n</pre> In\u00a0[2]: Copied! <pre>df = load_adult(as_frame=True)\ndf.columns = [c.replace(\"-\", \"_\") for c in df.columns]\ndf[\"income_label\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int)\ndf.drop(\"income\", axis=1, inplace=True)\n\n# one could chose to use a validation set for early stopping, hyperparam\n# optimization, etc. This is just an example, so we simply use train/test\n# split\ndf_tr, df_te = train_test_split(df, test_size=0.2, stratify=df.income_label)\n\ncat_embed_cols = [\n    \"workclass\",\n    \"education\",\n    \"marital_status\",\n    \"occupation\",\n    \"relationship\",\n    \"race\",\n    \"gender\",\n    \"capital_gain\",\n    \"capital_loss\",\n    \"native_country\",\n]\ncontinuous_cols = [\"age\", \"hours_per_week\"]\ntarget_col = \"income_label\"\n\ntab_preprocessor = TabPreprocessor(\n    cat_embed_cols=cat_embed_cols,\n    continuous_cols=continuous_cols,\n    with_attention=True,\n    with_cls_token=True,  # this is optional\n)\nX_tab = tab_preprocessor.fit_transform(df_tr)\ntarget = df_tr[target_col].values\n</pre> df = load_adult(as_frame=True) df.columns = [c.replace(\"-\", \"_\") for c in df.columns] df[\"income_label\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int) df.drop(\"income\", axis=1, inplace=True)  # one could chose to use a validation set for early stopping, hyperparam # optimization, etc. This is just an example, so we simply use train/test # split df_tr, df_te = train_test_split(df, test_size=0.2, stratify=df.income_label)  cat_embed_cols = [     \"workclass\",     \"education\",     \"marital_status\",     \"occupation\",     \"relationship\",     \"race\",     \"gender\",     \"capital_gain\",     \"capital_loss\",     \"native_country\", ] continuous_cols = [\"age\", \"hours_per_week\"] target_col = \"income_label\"  tab_preprocessor = TabPreprocessor(     cat_embed_cols=cat_embed_cols,     continuous_cols=continuous_cols,     with_attention=True,     with_cls_token=True,  # this is optional ) X_tab = tab_preprocessor.fit_transform(df_tr) target = df_tr[target_col].values <pre>/Users/javierrodriguezzaurin/Projects/pytorch-widedeep/pytorch_widedeep/preprocessing/tab_preprocessor.py:364: UserWarning: Continuous columns will not be normalised\n  warnings.warn(\"Continuous columns will not be normalised\")\n</pre> In\u00a0[3]: Copied! <pre>ft_transformer = FTTransformer(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=tab_preprocessor.continuous_cols,\n    embed_continuous_method=\"standard\",\n    input_dim=32,\n    kv_compression_factor=0.5,\n    n_blocks=3,\n    n_heads=4,\n)\n</pre> ft_transformer = FTTransformer(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     continuous_cols=tab_preprocessor.continuous_cols,     embed_continuous_method=\"standard\",     input_dim=32,     kv_compression_factor=0.5,     n_blocks=3,     n_heads=4, ) In\u00a0[4]: Copied! <pre># for a full list of the params for the the ContrastiveDenoisingTrainer (which are many) please see the docs.\n# Note that using these params involves some knowledge of the routine and the architecture of the model used\ncontrastive_denoising_trainer = ContrastiveDenoisingTrainer(\n    model=ft_transformer,\n    preprocessor=tab_preprocessor,\n)\ncontrastive_denoising_trainer.pretrain(X_tab, n_epochs=5, batch_size=256)\n</pre> # for a full list of the params for the the ContrastiveDenoisingTrainer (which are many) please see the docs. # Note that using these params involves some knowledge of the routine and the architecture of the model used contrastive_denoising_trainer = ContrastiveDenoisingTrainer(     model=ft_transformer,     preprocessor=tab_preprocessor, ) contrastive_denoising_trainer.pretrain(X_tab, n_epochs=5, batch_size=256) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:22&lt;00:00,  6.91it/s, loss=656]   \nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:07&lt;00:00, 20.84it/s, loss=141]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:07&lt;00:00, 20.75it/s, loss=137]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:07&lt;00:00, 21.14it/s, loss=135]\nepoch 5: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:07&lt;00:00, 21.05it/s, loss=134]\n</pre> In\u00a0[5]: Copied! <pre>contrastive_denoising_trainer.save(\n    path=\"pretrained_weights\", model_filename=\"contrastive_denoising_model.pt\"\n)\n</pre> contrastive_denoising_trainer.save(     path=\"pretrained_weights\", model_filename=\"contrastive_denoising_model.pt\" ) <p>some time has passed</p> In\u00a0[6]: Copied! <pre># some time has passed, we load the model with torch as usual:\ncontrastive_denoising_model = torch.load(\n    \"pretrained_weights/contrastive_denoising_model.pt\"\n)\n</pre> # some time has passed, we load the model with torch as usual: contrastive_denoising_model = torch.load(     \"pretrained_weights/contrastive_denoising_model.pt\" ) <pre>/var/folders/pd/_2wz_qt16yq1fk6jn_xtxqk40000gn/T/ipykernel_71058/884975850.py:2: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.\n  contrastive_denoising_model = torch.load(\n</pre> <p>NOW, AND THIS IS IMPORTANT! We have loaded the entire contrastive, denoising model. To proceed to the supervised training we ONLY need the attention-based model, which is the 'model' attribute of the trainer, let's have a look</p> In\u00a0[7]: Copied! <pre>contrastive_denoising_model.model\n</pre> contrastive_denoising_model.model Out[7]: <pre>FTTransformer(\n  (cat_embed): SameSizeCatEmbeddings(\n    (embed): Embedding(322, 32, padding_idx=0)\n    (dropout): Dropout(p=0.0, inplace=False)\n  )\n  (cont_norm): Identity()\n  (cont_embed): ContEmbeddings(\n    INFO: [ContLinear = weight(n_cont_cols, embed_dim) + bias(n_cont_cols, embed_dim)]\n    (linear): ContLinear(n_cont_cols=2, embed_dim=32, embed_dropout=0.0)\n    (dropout): Dropout(p=0.0, inplace=False)\n  )\n  (encoder): Sequential(\n    (fttransformer_block0): FTTransformerEncoder(\n      (attn): LinearAttentionLinformer(\n        (dropout): Dropout(p=0.2, inplace=False)\n        (qkv_proj): Linear(in_features=32, out_features=96, bias=False)\n        (out_proj): Linear(in_features=32, out_features=32, bias=False)\n      )\n      (ff): FeedForward(\n        (w_1): Linear(in_features=32, out_features=84, bias=True)\n        (w_2): Linear(in_features=42, out_features=32, bias=True)\n        (dropout): Dropout(p=0.1, inplace=False)\n        (activation): REGLU()\n      )\n      (attn_normadd): NormAdd(\n        (dropout): Dropout(p=0.2, inplace=False)\n        (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n      )\n      (ff_normadd): NormAdd(\n        (dropout): Dropout(p=0.1, inplace=False)\n        (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n      )\n    )\n    (fttransformer_block1): FTTransformerEncoder(\n      (attn): LinearAttentionLinformer(\n        (dropout): Dropout(p=0.2, inplace=False)\n        (qkv_proj): Linear(in_features=32, out_features=96, bias=False)\n        (out_proj): Linear(in_features=32, out_features=32, bias=False)\n      )\n      (ff): FeedForward(\n        (w_1): Linear(in_features=32, out_features=84, bias=True)\n        (w_2): Linear(in_features=42, out_features=32, bias=True)\n        (dropout): Dropout(p=0.1, inplace=False)\n        (activation): REGLU()\n      )\n      (attn_normadd): NormAdd(\n        (dropout): Dropout(p=0.2, inplace=False)\n        (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n      )\n      (ff_normadd): NormAdd(\n        (dropout): Dropout(p=0.1, inplace=False)\n        (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n      )\n    )\n    (fttransformer_block2): FTTransformerEncoder(\n      (attn): LinearAttentionLinformer(\n        (dropout): Dropout(p=0.2, inplace=False)\n        (qkv_proj): Linear(in_features=32, out_features=96, bias=False)\n        (out_proj): Linear(in_features=32, out_features=32, bias=False)\n      )\n      (ff): FeedForward(\n        (w_1): Linear(in_features=32, out_features=84, bias=True)\n        (w_2): Linear(in_features=42, out_features=32, bias=True)\n        (dropout): Dropout(p=0.1, inplace=False)\n        (activation): REGLU()\n      )\n      (attn_normadd): NormAdd(\n        (dropout): Dropout(p=0.2, inplace=False)\n        (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n      )\n      (ff_normadd): NormAdd(\n        (dropout): Dropout(p=0.1, inplace=False)\n        (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n      )\n    )\n  )\n)</pre> In\u00a0[8]: Copied! <pre>pretrained_model = contrastive_denoising_model.model\n</pre> pretrained_model = contrastive_denoising_model.model In\u00a0[9]: Copied! <pre># and as always, ANY supervised model in this library has to go throuth the WideDeep class:\nmodel = WideDeep(deeptabular=pretrained_model)\ntrainer = Trainer(model=model, objective=\"binary\", metrics=[Accuracy])\n\ntrainer.fit(X_tab=X_tab, target=target, n_epochs=5, batch_size=256)\n\n# And, you know...we get a test metric\nX_tab_te = tab_preprocessor.transform(df_te)\ntarget_te = df_te[target_col].values\n\npreds = trainer.predict(X_tab=X_tab_te)\ntest_acc = accuracy_score(target_te, preds)\nprint(test_acc)\n</pre> # and as always, ANY supervised model in this library has to go throuth the WideDeep class: model = WideDeep(deeptabular=pretrained_model) trainer = Trainer(model=model, objective=\"binary\", metrics=[Accuracy])  trainer.fit(X_tab=X_tab, target=target, n_epochs=5, batch_size=256)  # And, you know...we get a test metric X_tab_te = tab_preprocessor.transform(df_te) target_te = df_te[target_col].values  preds = trainer.predict(X_tab=X_tab_te) test_acc = accuracy_score(target_te, preds) print(test_acc) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:03&lt;00:00, 50.99it/s, loss=0.383, metrics={'acc': 0.8218}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:02&lt;00:00, 53.22it/s, loss=0.328, metrics={'acc': 0.8505}]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:02&lt;00:00, 54.50it/s, loss=0.309, metrics={'acc': 0.8588}]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:02&lt;00:00, 53.90it/s, loss=0.299, metrics={'acc': 0.8641}]\nepoch 5: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 153/153 [00:02&lt;00:00, 54.02it/s, loss=0.292, metrics={'acc': 0.8665}]\npredict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 39/39 [00:00&lt;00:00, 88.79it/s] </pre> <pre>0.8679496366055891\n</pre> <pre>\n</pre>"},{"location":"examples/15_Self_Supervised_Pretraning_pt2.html#self-supervised-pretraining-for-tabular-data","title":"Self Supervised Pretraining for Tabular Data\u00b6","text":"<p>We have implemented two Self Supervised Pre-training routines that allow the user to pre-train all tabular models in the library with the exception of the TabPerceiver (which is a special monster).</p> <p>The two routines implemented are illustrated in the figures below. The 1st is from TabNet: Attentive Interpretable Tabular Learning and is designed for models that do not use transformer-based architectures, while the second is from SAINT: Improved Neural Networks for Tabular Data via Row Attention and Contrastive Pre-Training, and is designed for models that use transformer-based architectures.</p> <p> </p> <p>Fig 1. Figure 2 in their paper. I have included de original caption in case is useful, althought the Figure itself is pretty self explanatory</p> <p> </p> <p>Fig 2. Figure 1 in their paper. Here the caption is necessary \ud83d\ude0f</p> <p>It is beyond the scope of this notebook to explain in detail those implementations. Therefore, we strongly recommend the user to go and read the papers if this functionality is of interest to her/him.</p> <p>One thing is worth noticing however. As seen in Fig 1(the TabNet paper's Fig 2) the masking of the input features happens in the feature space. However, the implementation in this library is inspired by that at the dreamquark-ai repo, which is in itself inspired by the original implementation (by the way, at this point I will write it once again. All TabNet related things in this library are inspired when not directly based in the code in that repo, therefore, ALL CREDIT TO THE GUYS AT dreamquark-ai).</p> <p>In that implementation the masking happens in the embedding space, and currently does not mask the entire embedding (i.e. categorical feature). We decided to release as it is in this version and we will implement the exact same process described in the paper in future releases.</p> <p>Having said all of the above let's see how to use self supervision for tabular data with <code>pytorch-widedeep</code>. We will concentrate in this notebook on the 2nd of the two approaches (the 'SAINT approach'). For details on the 1st approach (the 'TabNet' approach) please see <code>16_Self_Supervised_Pretraning_pt1</code>.</p>"},{"location":"examples/15_Self_Supervised_Pretraning_pt2.html#self-supervision-transformer-based-models","title":"Self Supervision transformer-based models..\u00b6","text":"<p>...or in general, for models where the embeddigns have all the same dimensions. In this library, these are:</p> <ul> <li>TabTransformer</li> <li>FTTransformer</li> <li>SAINT</li> <li>TabFastFormer</li> </ul> <p>Note that there is one additional Transformer-based model, the <code>TabPerceiver</code>, however this is a \"particular\" model and at the moment we do not support self supervision for it, but it will come.</p> <p>Let see at one example using the <code>FTTransformer</code>.</p>"},{"location":"examples/16_Usign_a_custom_hugging_face_model.html","title":"16_Usign-a-custom-hugging-face-model","text":"In\u00a0[1]: Copied! <pre>import numpy as np\nimport torch\nimport lightgbm as lgb\nfrom lightgbm import Dataset as lgbDataset\nfrom scipy.sparse import hstack, csr_matrix\nfrom sklearn.metrics import (\n    f1_score,\n    recall_score,\n    accuracy_score,\n    precision_score,\n    confusion_matrix,\n)\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.feature_extraction.text import TfidfVectorizer\n\nfrom torch import Tensor, nn\nfrom transformers import DistilBertModel, DistilBertTokenizer\nfrom pytorch_widedeep import Trainer\nfrom pytorch_widedeep.models import TabMlp, BasicRNN, WideDeep\nfrom pytorch_widedeep.metrics import F1Score, Accuracy\nfrom pytorch_widedeep.utils import Tokenizer, LabelEncoder\nfrom pytorch_widedeep.preprocessing import TextPreprocessor, TabPreprocessor\nfrom pytorch_widedeep.datasets import load_womens_ecommerce\nfrom pytorch_widedeep.utils.fastai_transforms import (\n    fix_html,\n    spec_add_spaces,\n    rm_useless_spaces,\n)\n</pre> import numpy as np import torch import lightgbm as lgb from lightgbm import Dataset as lgbDataset from scipy.sparse import hstack, csr_matrix from sklearn.metrics import (     f1_score,     recall_score,     accuracy_score,     precision_score,     confusion_matrix, ) from sklearn.model_selection import train_test_split from sklearn.feature_extraction.text import TfidfVectorizer  from torch import Tensor, nn from transformers import DistilBertModel, DistilBertTokenizer from pytorch_widedeep import Trainer from pytorch_widedeep.models import TabMlp, BasicRNN, WideDeep from pytorch_widedeep.metrics import F1Score, Accuracy from pytorch_widedeep.utils import Tokenizer, LabelEncoder from pytorch_widedeep.preprocessing import TextPreprocessor, TabPreprocessor from pytorch_widedeep.datasets import load_womens_ecommerce from pytorch_widedeep.utils.fastai_transforms import (     fix_html,     spec_add_spaces,     rm_useless_spaces, ) <p>Let's load the data and have a look:</p> In\u00a0[2]: Copied! <pre>df = load_womens_ecommerce(as_frame=True)\n\ndf.columns = [c.replace(\" \", \"_\").lower() for c in df.columns]\n\n# classes from [0,num_class)\ndf[\"rating\"] = (df[\"rating\"] - 1).astype(\"int64\")\n\n# group reviews with 1 and 2 scores into one class\ndf.loc[df.rating == 0, \"rating\"] = 1\n\n# and back again to [0,num_class)\ndf[\"rating\"] = (df[\"rating\"] - 1).astype(\"int64\")\n\n# drop short reviews\ndf = df[~df.review_text.isna()]\ndf[\"review_length\"] = df.review_text.apply(lambda x: len(x.split(\" \")))\ndf = df[df.review_length &gt;= 5]\ndf = df.drop(\"review_length\", axis=1).reset_index(drop=True)\n</pre> df = load_womens_ecommerce(as_frame=True)  df.columns = [c.replace(\" \", \"_\").lower() for c in df.columns]  # classes from [0,num_class) df[\"rating\"] = (df[\"rating\"] - 1).astype(\"int64\")  # group reviews with 1 and 2 scores into one class df.loc[df.rating == 0, \"rating\"] = 1  # and back again to [0,num_class) df[\"rating\"] = (df[\"rating\"] - 1).astype(\"int64\")  # drop short reviews df = df[~df.review_text.isna()] df[\"review_length\"] = df.review_text.apply(lambda x: len(x.split(\" \"))) df = df[df.review_length &gt;= 5] df = df.drop(\"review_length\", axis=1).reset_index(drop=True) In\u00a0[3]: Copied! <pre>df.head()\n</pre> df.head() Out[3]: clothing_id age title review_text rating recommended_ind positive_feedback_count division_name department_name class_name 0 767 33 None Absolutely wonderful - silky and sexy and comf... 2 1 0 Initmates Intimate Intimates 1 1080 34 None Love this dress!  it's sooo pretty.  i happene... 3 1 4 General Dresses Dresses 2 1077 60 Some major design flaws I had such high hopes for this dress and reall... 1 0 0 General Dresses Dresses 3 1049 50 My favorite buy! I love, love, love this jumpsuit. it's fun, fl... 3 1 0 General Petite Bottoms Pants 4 847 47 Flattering shirt This shirt is very flattering to all due to th... 3 1 6 General Tops Blouses <p>So, we will use the <code>review_text</code> column to predict the <code>rating</code>. Later on, we will try to combine it with some other columns (like <code>division_name</code> and <code>age</code>) see if these help.</p> <p>Let's first have a look to the distribution of ratings</p> In\u00a0[4]: Copied! <pre>df.rating.value_counts()\n</pre> df.rating.value_counts() Out[4]: <pre>rating\n3    12515\n2     4904\n1     2820\n0     2369\nName: count, dtype: int64</pre> <p>This shows that we could have perhaps grouped rating scores of 1, 2 and 3 into 1...but anyway, let's just move on with those 4 classes.</p> <p>We are not going to carry any hyperparameter optimization here, so, we will only need a train and a test set (i.e.  no need of a validation set for the example in this notebook)</p> In\u00a0[5]: Copied! <pre>train, test = train_test_split(df, train_size=0.8, random_state=1, stratify=df.rating)\n</pre> train, test = train_test_split(df, train_size=0.8, random_state=1, stratify=df.rating) <p>Let's see what we have to beat. What metrics would we obtain if we always predict the most common rating (3)?</p> In\u00a0[6]: Copied! <pre>most_common_pred = [train.rating.value_counts().index[0]] * len(test)\n\nmost_common_acc = accuracy_score(test.rating, most_common_pred)\nmost_common_f1 = f1_score(test.rating, most_common_pred, average=\"weighted\")\n</pre> most_common_pred = [train.rating.value_counts().index[0]] * len(test)  most_common_acc = accuracy_score(test.rating, most_common_pred) most_common_f1 = f1_score(test.rating, most_common_pred, average=\"weighted\") In\u00a0[7]: Copied! <pre>print(f\"Accuracy: {most_common_acc}. F1 Score: {most_common_f1}\")\n</pre> print(f\"Accuracy: {most_common_acc}. F1 Score: {most_common_f1}\") <pre>Accuracy: 0.553516143299425. F1 Score: 0.3944344218301668\n</pre> <p>ok, these are our \"baseline\" metrics.</p> <p>Let's start by using simply tf-idf + lightGBM</p> In\u00a0[8]: Copied! <pre># ?Tokenizer\n</pre> # ?Tokenizer In\u00a0[9]: Copied! <pre># this Tokenizer is part of our utils module but of course, any valid tokenizer can be used here.\n\n# When using notebooks there seems to be an issue related with multiprocessing (and sometimes tqdm)\n# that can only be solved by using only one CPU\ntok = Tokenizer(n_cpus=1)\ntok_reviews_tr = tok.process_all(train.review_text.tolist())\ntok_reviews_te = tok.process_all(test.review_text.tolist())\n</pre> # this Tokenizer is part of our utils module but of course, any valid tokenizer can be used here.  # When using notebooks there seems to be an issue related with multiprocessing (and sometimes tqdm) # that can only be solved by using only one CPU tok = Tokenizer(n_cpus=1) tok_reviews_tr = tok.process_all(train.review_text.tolist()) tok_reviews_te = tok.process_all(test.review_text.tolist()) In\u00a0[10]: Copied! <pre>vectorizer = TfidfVectorizer(\n    max_features=5000, preprocessor=lambda x: x, tokenizer=lambda x: x, min_df=5\n)\n\nX_text_tr = vectorizer.fit_transform(tok_reviews_tr)\nX_text_te = vectorizer.transform(tok_reviews_te)\n</pre> vectorizer = TfidfVectorizer(     max_features=5000, preprocessor=lambda x: x, tokenizer=lambda x: x, min_df=5 )  X_text_tr = vectorizer.fit_transform(tok_reviews_tr) X_text_te = vectorizer.transform(tok_reviews_te) <pre>/Users/javierrodriguezzaurin/.pyenv/versions/3.10.13/envs/widedeep310/lib/python3.10/site-packages/sklearn/feature_extraction/text.py:525: UserWarning: The parameter 'token_pattern' will not be used since 'tokenizer' is not None'\n  warnings.warn(\n</pre> In\u00a0[11]: Copied! <pre>X_text_tr\n</pre> X_text_tr Out[11]: <pre>&lt;18086x4566 sparse matrix of type '&lt;class 'numpy.float64'&gt;'\n\twith 884074 stored elements in Compressed Sparse Row format&gt;</pre> <p>We now move our matrices to lightGBM <code>Dataset</code> format</p> In\u00a0[12]: Copied! <pre>lgbtrain_text = lgbDataset(\n    X_text_tr,\n    train.rating.values,\n    free_raw_data=False,\n)\n\nlgbtest_text = lgbDataset(\n    X_text_te,\n    test.rating.values,\n    reference=lgbtrain_text,\n    free_raw_data=False,\n)\n</pre> lgbtrain_text = lgbDataset(     X_text_tr,     train.rating.values,     free_raw_data=False, )  lgbtest_text = lgbDataset(     X_text_te,     test.rating.values,     reference=lgbtrain_text,     free_raw_data=False, ) <p>and off we go. By the way, I think as we run the next cell, we should appreciate how fast lightGBM runs. Yes, the input is a sparse matrix, but still, trains on 18086x4566 in a matter of secs</p> In\u00a0[\u00a0]: Copied! <pre>lgb_text_model = lgb.train(\n    {\"objective\": \"multiclass\", \"num_classes\": 4},\n    lgbtrain_text,\n    valid_sets=[lgbtest_text, lgbtrain_text],\n    valid_names=[\"test\", \"train\"],\n)\n</pre> lgb_text_model = lgb.train(     {\"objective\": \"multiclass\", \"num_classes\": 4},     lgbtrain_text,     valid_sets=[lgbtest_text, lgbtrain_text],     valid_names=[\"test\", \"train\"], ) In\u00a0[14]: Copied! <pre>preds_text = lgb_text_model.predict(X_text_te)\npred_text_class = np.argmax(preds_text, 1)\n</pre> preds_text = lgb_text_model.predict(X_text_te) pred_text_class = np.argmax(preds_text, 1) In\u00a0[15]: Copied! <pre>acc_text = accuracy_score(lgbtest_text.label, pred_text_class)\nf1_text = f1_score(lgbtest_text.label, pred_text_class, average=\"weighted\")\ncm_text = confusion_matrix(lgbtest_text.label, pred_text_class)\n</pre> acc_text = accuracy_score(lgbtest_text.label, pred_text_class) f1_text = f1_score(lgbtest_text.label, pred_text_class, average=\"weighted\") cm_text = confusion_matrix(lgbtest_text.label, pred_text_class) In\u00a0[16]: Copied! <pre>print(f\"LightGBM Accuracy: {acc_text}. LightGBM F1 Score: {f1_text}\")\n</pre> print(f\"LightGBM Accuracy: {acc_text}. LightGBM F1 Score: {f1_text}\") <pre>LightGBM Accuracy: 0.6444051304732419. LightGBM F1 Score: 0.617154488246181\n</pre> In\u00a0[17]: Copied! <pre>print(f\"LightGBM Confusion Matrix: \\n {cm_text}\")\n</pre> print(f\"LightGBM Confusion Matrix: \\n {cm_text}\") <pre>LightGBM Confusion Matrix: \n [[ 199  135   61   79]\n [ 123  169  149  123]\n [  30   94  279  578]\n [  16   30  190 2267]]\n</pre> <p>Ok, so, with no hyperparameter optimization lightGBM gets an accuracy of 0.64 and a F1 score of 0.62. This is significantly better than predicting always the most popular.</p> <p>Let's see if in this implementation, some additional features, like <code>age</code> or <code>class_name</code> are of any help</p> In\u00a0[18]: Copied! <pre>tab_cols = [\n    \"age\",\n    \"division_name\",\n    \"department_name\",\n    \"class_name\",\n]\n\nfor tab_df in [train, test]:\n    for c in [\"division_name\", \"department_name\", \"class_name\"]:\n        tab_df[c] = tab_df[c].str.lower()\n        tab_df[c].fillna(\"missing\", inplace=True)\n</pre> tab_cols = [     \"age\",     \"division_name\",     \"department_name\",     \"class_name\", ]  for tab_df in [train, test]:     for c in [\"division_name\", \"department_name\", \"class_name\"]:         tab_df[c] = tab_df[c].str.lower()         tab_df[c].fillna(\"missing\", inplace=True) In\u00a0[19]: Copied! <pre># This is our LabelEncoder. A class that is designed to work with the models in this library but\n# can be used for general purposes\nle = LabelEncoder(columns_to_encode=[\"division_name\", \"department_name\", \"class_name\"])\ntrain_tab_le = le.fit_transform(train)\ntest_tab_le = le.transform(test)\n</pre> # This is our LabelEncoder. A class that is designed to work with the models in this library but # can be used for general purposes le = LabelEncoder(columns_to_encode=[\"division_name\", \"department_name\", \"class_name\"]) train_tab_le = le.fit_transform(train) test_tab_le = le.transform(test) In\u00a0[20]: Copied! <pre>train_tab_le.head()\n</pre> train_tab_le.head() Out[20]: clothing_id age title review_text rating recommended_ind positive_feedback_count division_name department_name class_name 4541 836 35 None Bought this on sale in my reg size- 10. im 5'9... 2 1 2 1 1 1 18573 1022 25 Look like \"mom jeans\" Maybe i just have the wrong body type for thes... 1 0 0 2 2 2 1058 815 39 Ig brought me here Love the way this top layers under my jackets ... 2 1 0 1 1 1 12132 984 47 Runs small especially the arms I love this jacket. it's the prettiest and mos... 3 1 0 1 3 3 20756 1051 42 True red, true beauty. These pants are gorgeous--the fabric has a sat... 3 1 0 2 2 4 <p>let's for example have a look to the encodings for the categorical feature <code>class_name</code></p> In\u00a0[21]: Copied! <pre>le.encoding_dict[\"class_name\"]\n</pre> le.encoding_dict[\"class_name\"] Out[21]: <pre>{'blouses': 1,\n 'jeans': 2,\n 'jackets': 3,\n 'pants': 4,\n 'knits': 5,\n 'dresses': 6,\n 'skirts': 7,\n 'sweaters': 8,\n 'fine gauge': 9,\n 'legwear': 10,\n 'lounge': 11,\n 'shorts': 12,\n 'outerwear': 13,\n 'intimates': 14,\n 'swim': 15,\n 'trend': 16,\n 'sleep': 17,\n 'layering': 18,\n 'missing': 19,\n 'casual bottoms': 20,\n 'chemises': 21}</pre> In\u00a0[22]: Copied! <pre># tabular training and test sets\nX_tab_tr = csr_matrix(train_tab_le[tab_cols].values)\nX_tab_te = csr_matrix(test_tab_le[tab_cols].values)\n\n# text + tabular training and test sets\nX_tab_text_tr = hstack((X_tab_tr, X_text_tr))\nX_tab_text_te = hstack((X_tab_te, X_text_te))\n</pre> # tabular training and test sets X_tab_tr = csr_matrix(train_tab_le[tab_cols].values) X_tab_te = csr_matrix(test_tab_le[tab_cols].values)  # text + tabular training and test sets X_tab_text_tr = hstack((X_tab_tr, X_text_tr)) X_tab_text_te = hstack((X_tab_te, X_text_te)) In\u00a0[23]: Copied! <pre>X_tab_tr\n</pre> X_tab_tr Out[23]: <pre>&lt;18086x4 sparse matrix of type '&lt;class 'numpy.int64'&gt;'\n\twith 72344 stored elements in Compressed Sparse Row format&gt;</pre> In\u00a0[24]: Copied! <pre>X_tab_text_tr\n</pre> X_tab_text_tr Out[24]: <pre>&lt;18086x4570 sparse matrix of type '&lt;class 'numpy.float64'&gt;'\n\twith 956418 stored elements in Compressed Sparse Row format&gt;</pre> In\u00a0[25]: Copied! <pre>lgbtrain_tab_text = lgbDataset(\n    X_tab_text_tr,\n    train.rating.values,\n    categorical_feature=[0, 1, 2, 3],\n    free_raw_data=False,\n)\n\nlgbtest_tab_text = lgbDataset(\n    X_tab_text_te,\n    test.rating.values,\n    reference=lgbtrain_tab_text,\n    free_raw_data=False,\n)\n</pre> lgbtrain_tab_text = lgbDataset(     X_tab_text_tr,     train.rating.values,     categorical_feature=[0, 1, 2, 3],     free_raw_data=False, )  lgbtest_tab_text = lgbDataset(     X_tab_text_te,     test.rating.values,     reference=lgbtrain_tab_text,     free_raw_data=False, ) In\u00a0[26]: Copied! <pre>lgb_tab_text_model = lgb.train(\n    {\"objective\": \"multiclass\", \"num_classes\": 4},\n    lgbtrain_tab_text,\n    valid_sets=[lgbtrain_tab_text, lgbtest_tab_text],\n    valid_names=[\"test\", \"train\"],\n    verbose_eval=False,\n)\n</pre> lgb_tab_text_model = lgb.train(     {\"objective\": \"multiclass\", \"num_classes\": 4},     lgbtrain_tab_text,     valid_sets=[lgbtrain_tab_text, lgbtest_tab_text],     valid_names=[\"test\", \"train\"],     verbose_eval=False, ) <pre>/opt/conda/envs/wd38/lib/python3.8/site-packages/lightgbm/basic.py:2065: UserWarning: Using categorical_feature in Dataset.\n  _log_warning('Using categorical_feature in Dataset.')\n/opt/conda/envs/wd38/lib/python3.8/site-packages/lightgbm/basic.py:2068: UserWarning: categorical_feature in Dataset is overridden.\nNew categorical_feature is [0, 1, 2, 3]\n  _log_warning('categorical_feature in Dataset is overridden.\\n'\n/opt/conda/envs/wd38/lib/python3.8/site-packages/lightgbm/engine.py:239: UserWarning: 'verbose_eval' argument is deprecated and will be removed in a future release of LightGBM. Pass 'log_evaluation()' callback via 'callbacks' argument instead.\n  _log_warning(\"'verbose_eval' argument is deprecated and will be removed in a future release of LightGBM. \"\n</pre> <pre>[LightGBM] [Warning] Auto-choosing col-wise multi-threading, the overhead of testing was 0.138280 seconds.\nYou can set `force_col_wise=true` to remove the overhead.\n[LightGBM] [Info] Total Bins 143432\n[LightGBM] [Info] Number of data points in the train set: 18086, number of used features: 2289\n[LightGBM] [Info] Start training from score -2.255919\n[LightGBM] [Info] Start training from score -2.081545\n[LightGBM] [Info] Start training from score -1.528281\n[LightGBM] [Info] Start training from score -0.591354\n</pre> <pre>/opt/conda/envs/wd38/lib/python3.8/site-packages/lightgbm/basic.py:1780: UserWarning: Overriding the parameters from Reference Dataset.\n  _log_warning('Overriding the parameters from Reference Dataset.')\n/opt/conda/envs/wd38/lib/python3.8/site-packages/lightgbm/basic.py:1513: UserWarning: categorical_column in param dict is overridden.\n  _log_warning(f'{cat_alias} in param dict is overridden.')\n</pre> In\u00a0[27]: Copied! <pre>preds_tab_text = lgb_tab_text_model.predict(X_tab_text_te)\npreds_tab_text_class = np.argmax(preds_tab_text, 1)\n\nacc_tab_text = accuracy_score(lgbtest_tab_text.label, preds_tab_text_class)\nf1_tab_text = f1_score(lgbtest_tab_text.label, preds_tab_text_class, average=\"weighted\")\ncm_tab_text = confusion_matrix(lgbtest_tab_text.label, preds_tab_text_class)\n</pre> preds_tab_text = lgb_tab_text_model.predict(X_tab_text_te) preds_tab_text_class = np.argmax(preds_tab_text, 1)  acc_tab_text = accuracy_score(lgbtest_tab_text.label, preds_tab_text_class) f1_tab_text = f1_score(lgbtest_tab_text.label, preds_tab_text_class, average=\"weighted\") cm_tab_text = confusion_matrix(lgbtest_tab_text.label, preds_tab_text_class) In\u00a0[28]: Copied! <pre>print(\n    f\"LightGBM text + tabular Accuracy: {acc_tab_text}. LightGBM text + tabular F1 Score: {f1_tab_text}\"\n)\n</pre> print(     f\"LightGBM text + tabular Accuracy: {acc_tab_text}. LightGBM text + tabular F1 Score: {f1_tab_text}\" ) <pre>LightGBM text + tabular Accuracy: 0.6382131800088456. LightGBM text + tabular F1 Score: 0.6080251307242649\n</pre> In\u00a0[29]: Copied! <pre>print(f\"LightGBM text + tabular Confusion Matrix:\\n {cm_tab_text}\")\n</pre> print(f\"LightGBM text + tabular Confusion Matrix:\\n {cm_tab_text}\") <pre>LightGBM text + tabular Confusion Matrix:\n [[ 193  123   68   90]\n [ 123  146  157  138]\n [  37   90  272  582]\n [  16   37  175 2275]]\n</pre> <p>So, in this set up, the addition tabular columns do not help performance.</p> In\u00a0[30]: Copied! <pre>text_preprocessor = TextPreprocessor(\n    text_col=\"review_text\", max_vocab=5000, min_freq=5, maxlen=90, n_cpus=1\n)\n\nwd_X_text_tr = text_preprocessor.fit_transform(train)\nwd_X_text_te = text_preprocessor.transform(test)\n</pre> text_preprocessor = TextPreprocessor(     text_col=\"review_text\", max_vocab=5000, min_freq=5, maxlen=90, n_cpus=1 )  wd_X_text_tr = text_preprocessor.fit_transform(train) wd_X_text_te = text_preprocessor.transform(test) <pre>The vocabulary contains 4328 tokens\n</pre> In\u00a0[31]: Copied! <pre>basic_rnn = BasicRNN(\n    vocab_size=len(text_preprocessor.vocab.itos),\n    embed_dim=300,\n    hidden_dim=64,\n    n_layers=3,\n    rnn_dropout=0.2,\n    head_hidden_dims=[32],\n)\n\n\nwd_text_model = WideDeep(deeptext=basic_rnn, pred_dim=4)\n</pre> basic_rnn = BasicRNN(     vocab_size=len(text_preprocessor.vocab.itos),     embed_dim=300,     hidden_dim=64,     n_layers=3,     rnn_dropout=0.2,     head_hidden_dims=[32], )   wd_text_model = WideDeep(deeptext=basic_rnn, pred_dim=4) In\u00a0[32]: Copied! <pre>wd_text_model\n</pre> wd_text_model Out[32]: <pre>WideDeep(\n  (deeptext): Sequential(\n    (0): BasicRNN(\n      (word_embed): Embedding(4328, 300, padding_idx=1)\n      (rnn): LSTM(300, 64, num_layers=3, batch_first=True, dropout=0.2)\n      (rnn_mlp): MLP(\n        (mlp): Sequential(\n          (dense_layer_0): Sequential(\n            (0): Linear(in_features=64, out_features=32, bias=True)\n            (1): ReLU(inplace=True)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=32, out_features=4, bias=True)\n  )\n)</pre> In\u00a0[33]: Copied! <pre>text_trainer = Trainer(\n    wd_text_model,\n    objective=\"multiclass\",\n    metrics=[Accuracy, F1Score(average=True)],\n    num_workers=0,  # As in the case of the tokenizer, in notebook I need to set this to 0 for the Trainer to work\n)\n</pre> text_trainer = Trainer(     wd_text_model,     objective=\"multiclass\",     metrics=[Accuracy, F1Score(average=True)],     num_workers=0,  # As in the case of the tokenizer, in notebook I need to set this to 0 for the Trainer to work ) In\u00a0[34]: Copied! <pre>text_trainer.fit(\n    X_text=wd_X_text_tr,\n    target=train.rating.values,\n    n_epochs=5,\n    batch_size=256,\n)\n</pre> text_trainer.fit(     X_text=wd_X_text_tr,     target=train.rating.values,     n_epochs=5,     batch_size=256, ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 71/71 [00:01&lt;00:00, 52.39it/s, loss=1.16, metrics={'acc': 0.5349, 'f1': 0.2011}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 71/71 [00:01&lt;00:00, 70.35it/s, loss=0.964, metrics={'acc': 0.5827, 'f1': 0.3005}]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 71/71 [00:01&lt;00:00, 70.33it/s, loss=0.845, metrics={'acc': 0.6252, 'f1': 0.4133}]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 71/71 [00:01&lt;00:00, 69.99it/s, loss=0.765, metrics={'acc': 0.6575, 'f1': 0.4875}]\nepoch 5: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 71/71 [00:01&lt;00:00, 69.55it/s, loss=0.709, metrics={'acc': 0.6879, 'f1': 0.5423}]\n</pre> In\u00a0[35]: Copied! <pre>wd_pred_text = text_trainer.predict_proba(X_text=wd_X_text_te)\nwd_pred_text_class = np.argmax(wd_pred_text, 1)\n</pre> wd_pred_text = text_trainer.predict_proba(X_text=wd_X_text_te) wd_pred_text_class = np.argmax(wd_pred_text, 1) <pre>predict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 18/18 [00:00&lt;00:00, 211.51it/s]\n</pre> In\u00a0[36]: Copied! <pre>wd_acc_text = accuracy_score(test.rating, wd_pred_text_class)\nwd_f1_text = f1_score(test.rating, wd_pred_text_class, average=\"weighted\")\nwd_cm_text = confusion_matrix(test.rating, wd_pred_text_class)\n</pre> wd_acc_text = accuracy_score(test.rating, wd_pred_text_class) wd_f1_text = f1_score(test.rating, wd_pred_text_class, average=\"weighted\") wd_cm_text = confusion_matrix(test.rating, wd_pred_text_class) In\u00a0[37]: Copied! <pre>print(f\"Basic RNN Accuracy: {wd_acc_text}. Basic RNN F1 Score: {wd_f1_text}\")\n</pre> print(f\"Basic RNN Accuracy: {wd_acc_text}. Basic RNN F1 Score: {wd_f1_text}\") <pre>Basic RNN Accuracy: 0.6076957098628926. Basic RNN F1 Score: 0.6017335854471788\n</pre> In\u00a0[38]: Copied! <pre>print(f\"Basic RNN Confusion Matrix:\\n {wd_cm_text}\")\n</pre> print(f\"Basic RNN Confusion Matrix:\\n {wd_cm_text}\") <pre>Basic RNN Confusion Matrix:\n [[ 327   76   62    9]\n [ 285  115  117   47]\n [ 131  122  315  413]\n [  42   69  401 1991]]\n</pre> <p>The performance is very similar to that of using simply tf-idf and lightgbm. Let see if adding tabular features helps when using <code>pytorch-widedeep</code></p> In\u00a0[39]: Copied! <pre># ?TabPreprocessor\n</pre> # ?TabPreprocessor In\u00a0[40]: Copied! <pre>tab_preprocessor = TabPreprocessor(cat_embed_cols=tab_cols)\n\nwd_X_tab_tr = tab_preprocessor.fit_transform(train)\nwd_X_tab_te = tab_preprocessor.transform(test)\n</pre> tab_preprocessor = TabPreprocessor(cat_embed_cols=tab_cols)  wd_X_tab_tr = tab_preprocessor.fit_transform(train) wd_X_tab_te = tab_preprocessor.transform(test) In\u00a0[41]: Copied! <pre># ?TabMlp\n</pre> # ?TabMlp In\u00a0[42]: Copied! <pre>tab_model = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    mlp_hidden_dims=[100, 50],\n)\n</pre> tab_model = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     mlp_hidden_dims=[100, 50], ) In\u00a0[43]: Copied! <pre>tab_model\n</pre> tab_model Out[43]: <pre>TabMlp(\n  (cat_and_cont_embed): DiffSizeCatAndContEmbeddings(\n    (cat_embed): DiffSizeCatEmbeddings(\n      (embed_layers): ModuleDict(\n        (emb_layer_age): Embedding(78, 18, padding_idx=0)\n        (emb_layer_division_name): Embedding(5, 3, padding_idx=0)\n        (emb_layer_department_name): Embedding(8, 5, padding_idx=0)\n        (emb_layer_class_name): Embedding(22, 9, padding_idx=0)\n      )\n      (embedding_dropout): Dropout(p=0.1, inplace=False)\n    )\n  )\n  (encoder): MLP(\n    (mlp): Sequential(\n      (dense_layer_0): Sequential(\n        (0): Dropout(p=0.1, inplace=False)\n        (1): Linear(in_features=35, out_features=100, bias=True)\n        (2): ReLU(inplace=True)\n      )\n      (dense_layer_1): Sequential(\n        (0): Dropout(p=0.1, inplace=False)\n        (1): Linear(in_features=100, out_features=50, bias=True)\n        (2): ReLU(inplace=True)\n      )\n    )\n  )\n)</pre> In\u00a0[44]: Copied! <pre>text_model = BasicRNN(\n    vocab_size=len(text_preprocessor.vocab.itos),\n    embed_dim=300,\n    hidden_dim=64,\n    n_layers=3,\n    rnn_dropout=0.2,\n    head_hidden_dims=[32],\n)\n</pre> text_model = BasicRNN(     vocab_size=len(text_preprocessor.vocab.itos),     embed_dim=300,     hidden_dim=64,     n_layers=3,     rnn_dropout=0.2,     head_hidden_dims=[32], ) In\u00a0[45]: Copied! <pre>wd_tab_and_text_model = WideDeep(deeptabular=tab_model, deeptext=text_model, pred_dim=4)\n</pre> wd_tab_and_text_model = WideDeep(deeptabular=tab_model, deeptext=text_model, pred_dim=4) In\u00a0[46]: Copied! <pre>wd_tab_and_text_model\n</pre> wd_tab_and_text_model Out[46]: <pre>WideDeep(\n  (deeptabular): Sequential(\n    (0): TabMlp(\n      (cat_and_cont_embed): DiffSizeCatAndContEmbeddings(\n        (cat_embed): DiffSizeCatEmbeddings(\n          (embed_layers): ModuleDict(\n            (emb_layer_age): Embedding(78, 18, padding_idx=0)\n            (emb_layer_division_name): Embedding(5, 3, padding_idx=0)\n            (emb_layer_department_name): Embedding(8, 5, padding_idx=0)\n            (emb_layer_class_name): Embedding(22, 9, padding_idx=0)\n          )\n          (embedding_dropout): Dropout(p=0.1, inplace=False)\n        )\n      )\n      (encoder): MLP(\n        (mlp): Sequential(\n          (dense_layer_0): Sequential(\n            (0): Dropout(p=0.1, inplace=False)\n            (1): Linear(in_features=35, out_features=100, bias=True)\n            (2): ReLU(inplace=True)\n          )\n          (dense_layer_1): Sequential(\n            (0): Dropout(p=0.1, inplace=False)\n            (1): Linear(in_features=100, out_features=50, bias=True)\n            (2): ReLU(inplace=True)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=50, out_features=4, bias=True)\n  )\n  (deeptext): Sequential(\n    (0): BasicRNN(\n      (word_embed): Embedding(4328, 300, padding_idx=1)\n      (rnn): LSTM(300, 64, num_layers=3, batch_first=True, dropout=0.2)\n      (rnn_mlp): MLP(\n        (mlp): Sequential(\n          (dense_layer_0): Sequential(\n            (0): Linear(in_features=64, out_features=32, bias=True)\n            (1): ReLU(inplace=True)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=32, out_features=4, bias=True)\n  )\n)</pre> In\u00a0[47]: Copied! <pre>tab_and_text_trainer = Trainer(\n    wd_tab_and_text_model,\n    objective=\"multiclass\",\n    metrics=[Accuracy, F1Score(average=True)],\n    num_workers=0,  # As in the case of the tokenizer, in notebook I need to set this to 0 for the Trainer to work\n)\n</pre> tab_and_text_trainer = Trainer(     wd_tab_and_text_model,     objective=\"multiclass\",     metrics=[Accuracy, F1Score(average=True)],     num_workers=0,  # As in the case of the tokenizer, in notebook I need to set this to 0 for the Trainer to work ) In\u00a0[48]: Copied! <pre>tab_and_text_trainer.fit(\n    X_tab=wd_X_tab_tr,\n    X_text=wd_X_text_tr,\n    target=train.rating.values,\n    n_epochs=5,\n    batch_size=256,\n)\n</pre> tab_and_text_trainer.fit(     X_tab=wd_X_tab_tr,     X_text=wd_X_text_tr,     target=train.rating.values,     n_epochs=5,     batch_size=256, ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 71/71 [00:01&lt;00:00, 52.04it/s, loss=1.13, metrics={'acc': 0.538, 'f1': 0.1911}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 71/71 [00:01&lt;00:00, 52.28it/s, loss=0.936, metrics={'acc': 0.5887, 'f1': 0.3507}]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 71/71 [00:01&lt;00:00, 52.26it/s, loss=0.825, metrics={'acc': 0.6394, 'f1': 0.4545}]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 71/71 [00:01&lt;00:00, 51.33it/s, loss=0.757, metrics={'acc': 0.6696, 'f1': 0.5214}]\nepoch 5: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 71/71 [00:01&lt;00:00, 50.39it/s, loss=0.702, metrics={'acc': 0.6963, 'f1': 0.5654}]\n</pre> In\u00a0[49]: Copied! <pre>wd_pred_tab_and_text = tab_and_text_trainer.predict_proba(\n    X_tab=wd_X_tab_te, X_text=wd_X_text_te\n)\nwd_pred_tab_and_text_class = np.argmax(wd_pred_tab_and_text, 1)\n</pre> wd_pred_tab_and_text = tab_and_text_trainer.predict_proba(     X_tab=wd_X_tab_te, X_text=wd_X_text_te ) wd_pred_tab_and_text_class = np.argmax(wd_pred_tab_and_text, 1) <pre>predict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 18/18 [00:00&lt;00:00, 136.94it/s]\n</pre> In\u00a0[50]: Copied! <pre>wd_acc_tab_and_text = accuracy_score(test.rating, wd_pred_tab_and_text_class)\nwd_f1_tab_and_text = f1_score(\n    test.rating, wd_pred_tab_and_text_class, average=\"weighted\"\n)\nwd_cm_tab_and_text = confusion_matrix(test.rating, wd_pred_tab_and_text_class)\n</pre> wd_acc_tab_and_text = accuracy_score(test.rating, wd_pred_tab_and_text_class) wd_f1_tab_and_text = f1_score(     test.rating, wd_pred_tab_and_text_class, average=\"weighted\" ) wd_cm_tab_and_text = confusion_matrix(test.rating, wd_pred_tab_and_text_class) In\u00a0[51]: Copied! <pre>print(\n    f\"Basic RNN + Tabular  Accuracy: {wd_acc_tab_and_text}. Basic RNN + TabularF1 Score: {wd_f1_tab_and_text}\"\n)\nprint(f\"Basic RNN + Tabular  Confusion Matrix:\\n {wd_cm_tab_and_text}\")\n</pre> print(     f\"Basic RNN + Tabular  Accuracy: {wd_acc_tab_and_text}. Basic RNN + TabularF1 Score: {wd_f1_tab_and_text}\" ) print(f\"Basic RNN + Tabular  Confusion Matrix:\\n {wd_cm_tab_and_text}\") <pre>Basic RNN + Tabular  Accuracy: 0.6333480760725343. Basic RNN + TabularF1 Score: 0.6332310089593208\nBasic RNN + Tabular  Confusion Matrix:\n [[ 267  132   65   10]\n [ 198  168  159   39]\n [  57  113  410  401]\n [  12   58  414 2019]]\n</pre> <p>We are going to \"manually\" code the Tokenizer and the model and see how they can be used as part of the process along with the <code>pytorch-widedeep</code> library.</p> <p>Tokenizer:</p> In\u00a0[52]: Copied! <pre>class BertTokenizer(object):\n    def __init__(\n        self,\n        pretrained_tokenizer=\"distilbert-base-uncased\",\n        do_lower_case=True,\n        max_length=90,\n    ):\n        super(BertTokenizer, self).__init__()\n        self.pretrained_tokenizer = pretrained_tokenizer\n        self.do_lower_case = do_lower_case\n        self.max_length = max_length\n\n    def fit(self, texts):\n        self.tokenizer = DistilBertTokenizer.from_pretrained(\n            self.pretrained_tokenizer, do_lower_case=self.do_lower_case\n        )\n\n        return self\n\n    def transform(self, texts):\n        input_ids = []\n        for text in texts:\n            encoded_sent = self.tokenizer.encode_plus(\n                text=self._pre_rules(text),\n                add_special_tokens=True,\n                max_length=self.max_length,\n                padding=\"max_length\",\n                truncation=True,\n            )\n\n            input_ids.append(encoded_sent.get(\"input_ids\"))\n        return np.stack(input_ids)\n\n    def fit_transform(self, texts):\n        return self.fit(texts).transform(texts)\n\n    @staticmethod\n    def _pre_rules(text):\n        return fix_html(rm_useless_spaces(spec_add_spaces(text)))\n</pre> class BertTokenizer(object):     def __init__(         self,         pretrained_tokenizer=\"distilbert-base-uncased\",         do_lower_case=True,         max_length=90,     ):         super(BertTokenizer, self).__init__()         self.pretrained_tokenizer = pretrained_tokenizer         self.do_lower_case = do_lower_case         self.max_length = max_length      def fit(self, texts):         self.tokenizer = DistilBertTokenizer.from_pretrained(             self.pretrained_tokenizer, do_lower_case=self.do_lower_case         )          return self      def transform(self, texts):         input_ids = []         for text in texts:             encoded_sent = self.tokenizer.encode_plus(                 text=self._pre_rules(text),                 add_special_tokens=True,                 max_length=self.max_length,                 padding=\"max_length\",                 truncation=True,             )              input_ids.append(encoded_sent.get(\"input_ids\"))         return np.stack(input_ids)      def fit_transform(self, texts):         return self.fit(texts).transform(texts)      @staticmethod     def _pre_rules(text):         return fix_html(rm_useless_spaces(spec_add_spaces(text))) <p>Model:</p> In\u00a0[53]: Copied! <pre>class BertModel(nn.Module):\n    def __init__(\n        self,\n        model_name: str = \"distilbert-base-uncased\",\n        freeze_bert: bool = False,\n    ):\n        super(BertModel, self).__init__()\n\n        self.bert = DistilBertModel.from_pretrained(\n            model_name,\n        )\n\n        if freeze_bert:\n            for param in self.bert.parameters():\n                param.requires_grad = False\n\n    def forward(self, X_inp: Tensor) -&gt; Tensor:\n        attn_mask = (X_inp != 0).type(torch.int8)\n        outputs = self.bert(input_ids=X_inp, attention_mask=attn_mask)\n        return outputs[0][:, 0, :]\n\n    @property\n    def output_dim(self) -&gt; int:\n        # This is THE ONLY requirement for any model to work with pytorch-widedeep. Must\n        # have a 'output_dim' property so the WideDeep class knows the incoming dims\n        # from the custom model. in this case, I hardcoded it\n        return 768\n</pre> class BertModel(nn.Module):     def __init__(         self,         model_name: str = \"distilbert-base-uncased\",         freeze_bert: bool = False,     ):         super(BertModel, self).__init__()          self.bert = DistilBertModel.from_pretrained(             model_name,         )          if freeze_bert:             for param in self.bert.parameters():                 param.requires_grad = False      def forward(self, X_inp: Tensor) -&gt; Tensor:         attn_mask = (X_inp != 0).type(torch.int8)         outputs = self.bert(input_ids=X_inp, attention_mask=attn_mask)         return outputs[0][:, 0, :]      @property     def output_dim(self) -&gt; int:         # This is THE ONLY requirement for any model to work with pytorch-widedeep. Must         # have a 'output_dim' property so the WideDeep class knows the incoming dims         # from the custom model. in this case, I hardcoded it         return 768 In\u00a0[54]: Copied! <pre>bert_tokenizer = BertTokenizer()\nX_bert_tr = bert_tokenizer.fit_transform(train[\"review_text\"].tolist())\nX_bert_te = bert_tokenizer.transform(test[\"review_text\"].tolist())\n</pre> bert_tokenizer = BertTokenizer() X_bert_tr = bert_tokenizer.fit_transform(train[\"review_text\"].tolist()) X_bert_te = bert_tokenizer.transform(test[\"review_text\"].tolist()) <p>As I mentioned a number of times in the documentation and examples, <code>pytorch-widedeep</code> is designed for flexibility. For any of the data modes (tabular, text and images) there are available components/models in the library. However, the user can choose to use any model they want with the only requirement that such model must have a <code>output_dim</code> property.</p> <p>With that in mind, the <code>BertModel</code> class defined above can be used by <code>pytorch-widedeep</code> as any other of the internal components. In other words, simply...pass it to the <code>WideDeep</code> class. In this case we are going to add a FC-head as part of the classifier.</p> In\u00a0[55]: Copied! <pre>bert_model = BertModel(freeze_bert=True)\nwd_bert_model = WideDeep(\n    deeptext=bert_model,\n    head_hidden_dims=[256, 128, 64],\n    pred_dim=4,\n)\n</pre> bert_model = BertModel(freeze_bert=True) wd_bert_model = WideDeep(     deeptext=bert_model,     head_hidden_dims=[256, 128, 64],     pred_dim=4, ) <pre>Some weights of the model checkpoint at distilbert-base-uncased were not used when initializing DistilBertModel: ['vocab_projector.bias', 'vocab_layer_norm.bias', 'vocab_transform.weight', 'vocab_transform.bias', 'vocab_projector.weight', 'vocab_layer_norm.weight']\n- This IS expected if you are initializing DistilBertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n- This IS NOT expected if you are initializing DistilBertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n</pre> In\u00a0[56]: Copied! <pre>wd_bert_model\n</pre> wd_bert_model Out[56]: <pre>WideDeep(\n  (deeptext): BertModel(\n    (bert): DistilBertModel(\n      (embeddings): Embeddings(\n        (word_embeddings): Embedding(30522, 768, padding_idx=0)\n        (position_embeddings): Embedding(512, 768)\n        (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n        (dropout): Dropout(p=0.1, inplace=False)\n      )\n      (transformer): Transformer(\n        (layer): ModuleList(\n          (0): TransformerBlock(\n            (attention): MultiHeadSelfAttention(\n              (dropout): Dropout(p=0.1, inplace=False)\n              (q_lin): Linear(in_features=768, out_features=768, bias=True)\n              (k_lin): Linear(in_features=768, out_features=768, bias=True)\n              (v_lin): Linear(in_features=768, out_features=768, bias=True)\n              (out_lin): Linear(in_features=768, out_features=768, bias=True)\n            )\n            (sa_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n            (ffn): FFN(\n              (dropout): Dropout(p=0.1, inplace=False)\n              (lin1): Linear(in_features=768, out_features=3072, bias=True)\n              (lin2): Linear(in_features=3072, out_features=768, bias=True)\n              (activation): GELUActivation()\n            )\n            (output_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n          )\n          (1): TransformerBlock(\n            (attention): MultiHeadSelfAttention(\n              (dropout): Dropout(p=0.1, inplace=False)\n              (q_lin): Linear(in_features=768, out_features=768, bias=True)\n              (k_lin): Linear(in_features=768, out_features=768, bias=True)\n              (v_lin): Linear(in_features=768, out_features=768, bias=True)\n              (out_lin): Linear(in_features=768, out_features=768, bias=True)\n            )\n            (sa_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n            (ffn): FFN(\n              (dropout): Dropout(p=0.1, inplace=False)\n              (lin1): Linear(in_features=768, out_features=3072, bias=True)\n              (lin2): Linear(in_features=3072, out_features=768, bias=True)\n              (activation): GELUActivation()\n            )\n            (output_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n          )\n          (2): TransformerBlock(\n            (attention): MultiHeadSelfAttention(\n              (dropout): Dropout(p=0.1, inplace=False)\n              (q_lin): Linear(in_features=768, out_features=768, bias=True)\n              (k_lin): Linear(in_features=768, out_features=768, bias=True)\n              (v_lin): Linear(in_features=768, out_features=768, bias=True)\n              (out_lin): Linear(in_features=768, out_features=768, bias=True)\n            )\n            (sa_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n            (ffn): FFN(\n              (dropout): Dropout(p=0.1, inplace=False)\n              (lin1): Linear(in_features=768, out_features=3072, bias=True)\n              (lin2): Linear(in_features=3072, out_features=768, bias=True)\n              (activation): GELUActivation()\n            )\n            (output_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n          )\n          (3): TransformerBlock(\n            (attention): MultiHeadSelfAttention(\n              (dropout): Dropout(p=0.1, inplace=False)\n              (q_lin): Linear(in_features=768, out_features=768, bias=True)\n              (k_lin): Linear(in_features=768, out_features=768, bias=True)\n              (v_lin): Linear(in_features=768, out_features=768, bias=True)\n              (out_lin): Linear(in_features=768, out_features=768, bias=True)\n            )\n            (sa_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n            (ffn): FFN(\n              (dropout): Dropout(p=0.1, inplace=False)\n              (lin1): Linear(in_features=768, out_features=3072, bias=True)\n              (lin2): Linear(in_features=3072, out_features=768, bias=True)\n              (activation): GELUActivation()\n            )\n            (output_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n          )\n          (4): TransformerBlock(\n            (attention): MultiHeadSelfAttention(\n              (dropout): Dropout(p=0.1, inplace=False)\n              (q_lin): Linear(in_features=768, out_features=768, bias=True)\n              (k_lin): Linear(in_features=768, out_features=768, bias=True)\n              (v_lin): Linear(in_features=768, out_features=768, bias=True)\n              (out_lin): Linear(in_features=768, out_features=768, bias=True)\n            )\n            (sa_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n            (ffn): FFN(\n              (dropout): Dropout(p=0.1, inplace=False)\n              (lin1): Linear(in_features=768, out_features=3072, bias=True)\n              (lin2): Linear(in_features=3072, out_features=768, bias=True)\n              (activation): GELUActivation()\n            )\n            (output_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n          )\n          (5): TransformerBlock(\n            (attention): MultiHeadSelfAttention(\n              (dropout): Dropout(p=0.1, inplace=False)\n              (q_lin): Linear(in_features=768, out_features=768, bias=True)\n              (k_lin): Linear(in_features=768, out_features=768, bias=True)\n              (v_lin): Linear(in_features=768, out_features=768, bias=True)\n              (out_lin): Linear(in_features=768, out_features=768, bias=True)\n            )\n            (sa_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n            (ffn): FFN(\n              (dropout): Dropout(p=0.1, inplace=False)\n              (lin1): Linear(in_features=768, out_features=3072, bias=True)\n              (lin2): Linear(in_features=3072, out_features=768, bias=True)\n              (activation): GELUActivation()\n            )\n            (output_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n          )\n        )\n      )\n    )\n  )\n  (deephead): Sequential(\n    (0): MLP(\n      (mlp): Sequential(\n        (dense_layer_0): Sequential(\n          (0): Linear(in_features=768, out_features=256, bias=True)\n          (1): ReLU(inplace=True)\n          (2): Dropout(p=0.1, inplace=False)\n        )\n        (dense_layer_1): Sequential(\n          (0): Linear(in_features=256, out_features=128, bias=True)\n          (1): ReLU(inplace=True)\n          (2): Dropout(p=0.1, inplace=False)\n        )\n        (dense_layer_2): Sequential(\n          (0): Linear(in_features=128, out_features=64, bias=True)\n          (1): ReLU(inplace=True)\n          (2): Dropout(p=0.1, inplace=False)\n        )\n      )\n    )\n    (1): Linear(in_features=64, out_features=4, bias=True)\n  )\n)</pre> In\u00a0[57]: Copied! <pre>wd_bert_trainer = Trainer(\n    wd_bert_model,\n    objective=\"multiclass\",\n    metrics=[Accuracy, F1Score(average=True)],\n    num_workers=0,  # As in the case of the tokenizer, in notebook I need to set this to 0 for the Trainer to work\n)\n\nwd_bert_trainer.fit(\n    X_text=X_bert_tr,\n    target=train.rating.values,\n    n_epochs=3,\n    batch_size=64,\n)\n</pre> wd_bert_trainer = Trainer(     wd_bert_model,     objective=\"multiclass\",     metrics=[Accuracy, F1Score(average=True)],     num_workers=0,  # As in the case of the tokenizer, in notebook I need to set this to 0 for the Trainer to work )  wd_bert_trainer.fit(     X_text=X_bert_tr,     target=train.rating.values,     n_epochs=3,     batch_size=64, ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 283/283 [00:14&lt;00:00, 19.68it/s, loss=0.968, metrics={'acc': 0.5879, 'f1': 0.3591}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 283/283 [00:14&lt;00:00, 19.63it/s, loss=0.884, metrics={'acc': 0.6178, 'f1': 0.4399}]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 283/283 [00:14&lt;00:00, 19.55it/s, loss=0.87, metrics={'acc': 0.6234, 'f1': 0.4527}]\n</pre> In\u00a0[58]: Copied! <pre>wd_bert_pred_text = wd_bert_trainer.predict_proba(X_text=X_bert_te)\nwd_bert_pred_text_class = np.argmax(wd_bert_pred_text, 1)\n</pre> wd_bert_pred_text = wd_bert_trainer.predict_proba(X_text=X_bert_te) wd_bert_pred_text_class = np.argmax(wd_bert_pred_text, 1) <pre>predict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 71/71 [00:03&lt;00:00, 21.97it/s]\n</pre> In\u00a0[59]: Copied! <pre>wd_bert_acc = accuracy_score(test.rating, wd_bert_pred_text_class)\nwd_bert_f1 = f1_score(test.rating, wd_bert_pred_text_class, average=\"weighted\")\nwd_bert_cm = confusion_matrix(test.rating, wd_bert_pred_text_class)\n</pre> wd_bert_acc = accuracy_score(test.rating, wd_bert_pred_text_class) wd_bert_f1 = f1_score(test.rating, wd_bert_pred_text_class, average=\"weighted\") wd_bert_cm = confusion_matrix(test.rating, wd_bert_pred_text_class) In\u00a0[60]: Copied! <pre>print(f\"Distilbert Accuracy: {wd_bert_acc}. Distilbert F1 Score: {wd_bert_f1}\")\nprint(f\"Distilbert Confusion Matrix:\\n {wd_bert_cm}\")\n</pre> print(f\"Distilbert Accuracy: {wd_bert_acc}. Distilbert F1 Score: {wd_bert_f1}\") print(f\"Distilbert Confusion Matrix:\\n {wd_bert_cm}\") <pre>Distilbert Accuracy: 0.6326846528084918. Distilbert F1 Score: 0.5796652991272998\nDistilbert Confusion Matrix:\n [[ 287   75   22   90]\n [ 197  136   62  169]\n [  68  119  123  671]\n [  40   64   84 2315]]\n</pre> <p>Now, adding a tabular model follows the exact same process as the one described in section 2.</p> In\u00a0[61]: Copied! <pre>tab_model = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    mlp_hidden_dims=[100, 50],\n)\n</pre> tab_model = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     mlp_hidden_dims=[100, 50], ) In\u00a0[62]: Copied! <pre>wd_tab_bert_model = WideDeep(\n    deeptabular=tab_model,\n    deeptext=bert_model,\n    head_hidden_dims=[256, 128, 64],\n    pred_dim=4,\n)\n</pre> wd_tab_bert_model = WideDeep(     deeptabular=tab_model,     deeptext=bert_model,     head_hidden_dims=[256, 128, 64],     pred_dim=4, ) In\u00a0[63]: Copied! <pre>wd_tab_bert_trainer = Trainer(\n    wd_tab_bert_model,\n    objective=\"multiclass\",\n    metrics=[Accuracy, F1Score(average=True)],\n    num_workers=0,  # As in the case of the tokenizer, in notebook I need to set this to 0 for the Trainer to work\n)\n</pre> wd_tab_bert_trainer = Trainer(     wd_tab_bert_model,     objective=\"multiclass\",     metrics=[Accuracy, F1Score(average=True)],     num_workers=0,  # As in the case of the tokenizer, in notebook I need to set this to 0 for the Trainer to work ) In\u00a0[64]: Copied! <pre>wd_tab_bert_trainer.fit(\n    X_tab=wd_X_tab_tr,\n    X_text=X_bert_tr,\n    target=train.rating.values,\n    n_epochs=3,\n    batch_size=64,\n)\n</pre> wd_tab_bert_trainer.fit(     X_tab=wd_X_tab_tr,     X_text=X_bert_tr,     target=train.rating.values,     n_epochs=3,     batch_size=64, ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 283/283 [00:15&lt;00:00, 18.15it/s, loss=0.974, metrics={'acc': 0.5838, 'f1': 0.3404}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 283/283 [00:15&lt;00:00, 18.38it/s, loss=0.885, metrics={'acc': 0.618, 'f1': 0.4378}]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 283/283 [00:15&lt;00:00, 18.40it/s, loss=0.868, metrics={'acc': 0.6252, 'f1': 0.4575}]\n</pre> In\u00a0[65]: Copied! <pre>wd_tab_bert_pred_text = wd_tab_bert_trainer.predict_proba(\n    X_tab=wd_X_tab_te, X_text=X_bert_te\n)\nwd_tab_bert_pred_text_class = np.argmax(wd_tab_bert_pred_text, 1)\n</pre> wd_tab_bert_pred_text = wd_tab_bert_trainer.predict_proba(     X_tab=wd_X_tab_te, X_text=X_bert_te ) wd_tab_bert_pred_text_class = np.argmax(wd_tab_bert_pred_text, 1) <pre>predict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 71/71 [00:03&lt;00:00, 21.32it/s]\n</pre> In\u00a0[66]: Copied! <pre>wd_tab_bert_acc = accuracy_score(test.rating, wd_tab_bert_pred_text_class)\nwd_tab_bert_f1 = f1_score(test.rating, wd_tab_bert_pred_text_class, average=\"weighted\")\nwd_tab_bert_cm = confusion_matrix(test.rating, wd_tab_bert_pred_text_class)\n</pre> wd_tab_bert_acc = accuracy_score(test.rating, wd_tab_bert_pred_text_class) wd_tab_bert_f1 = f1_score(test.rating, wd_tab_bert_pred_text_class, average=\"weighted\") wd_tab_bert_cm = confusion_matrix(test.rating, wd_tab_bert_pred_text_class) In\u00a0[67]: Copied! <pre>print(\n    f\"Distilbert + Tabular Accuracy: {wd_tab_bert_acc}. Distilbert+ Tabular F1 Score: {wd_tab_bert_f1}\"\n)\nprint(f\"Distilbert + Tabular Confusion Matrix:\\n {wd_tab_bert_cm}\")\n</pre> print(     f\"Distilbert + Tabular Accuracy: {wd_tab_bert_acc}. Distilbert+ Tabular F1 Score: {wd_tab_bert_f1}\" ) print(f\"Distilbert + Tabular Confusion Matrix:\\n {wd_tab_bert_cm}\") <pre>Distilbert + Tabular Accuracy: 0.6242812914639541. Distilbert+ Tabular F1 Score: 0.5508351761564895\nDistilbert + Tabular Confusion Matrix:\n [[ 297   56   11  110]\n [ 229   91   38  206]\n [  86   90   71  734]\n [  49   48   42 2364]]\n</pre>"},{"location":"examples/16_Usign_a_custom_hugging_face_model.html#using-a-hugginface-model","title":"Using a Hugginface model\u00b6","text":"<p>In this notebook we will show how to use an \"external\" Hugginface model along with any other model in the libray. In particular we will show how to combine it with a tabular DL model.</p> <p>Since we are here, we will also compare the performance of a few models on a text classification problem.</p> <p>The notebook will go as follows:</p> <ol> <li>Text classification using tf-idf + LightGBM</li> <li>Text classification using a basic RNN</li> <li>Text classification using Distilbert</li> </ol> <p>In all 3 cases we will add some tabular features to see if these help.</p> <p>In general, I would not pay much attention to the results since I have placed no effort in getting the best possible results (i.e. no hyperparameter optimization or trying different architectures, for example).</p> <p>Let's go</p>"},{"location":"examples/16_Usign_a_custom_hugging_face_model.html#1-text-classification-using-tf-idf-lightgbm","title":"1. Text classification using tf-idf + LightGBM\u00b6","text":""},{"location":"examples/16_Usign_a_custom_hugging_face_model.html#2-text-classification-using-pytorch-widedeeps-built-in-models-a-basic-rnn","title":"2. Text classification using pytorch-widedeep's built-in models (a basic RNN)\u00b6","text":"<p>Moving on now to fully using <code>pytorch-widedeep</code> in this dataset, let's have a look on how one could use a simple RNN to predict the ratings with the library.</p>"},{"location":"examples/16_Usign_a_custom_hugging_face_model.html#3-text-classification-using-a-hugginface-model-as-a-custom-model-in-pytorch-widedeeps","title":"3. Text classification using a Hugginface model as a custom model in pytorch-widedeep's\u00b6","text":""},{"location":"examples/17_feature_importance_via_attention_weights.html","title":"17_feature_importance_via_attention_weights","text":"In\u00a0[1]: Copied! <pre>import torch\n\nimport numpy as np\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import accuracy_score\n\n\nfrom pytorch_widedeep import Trainer\nfrom pytorch_widedeep.models import TabTransformer, ContextAttentionMLP, WideDeep\nfrom pytorch_widedeep.callbacks import EarlyStopping\nfrom pytorch_widedeep.metrics import Accuracy\nfrom pytorch_widedeep.datasets import load_adult\nfrom pytorch_widedeep.preprocessing import TabPreprocessor\n</pre> import torch  import numpy as np from sklearn.model_selection import train_test_split from sklearn.metrics import accuracy_score   from pytorch_widedeep import Trainer from pytorch_widedeep.models import TabTransformer, ContextAttentionMLP, WideDeep from pytorch_widedeep.callbacks import EarlyStopping from pytorch_widedeep.metrics import Accuracy from pytorch_widedeep.datasets import load_adult from pytorch_widedeep.preprocessing import TabPreprocessor In\u00a0[2]: Copied! <pre># use_cuda = torch.cuda.is_available()\ndf = load_adult(as_frame=True)\ndf.columns = [c.replace(\"-\", \"_\") for c in df.columns]\ndf[\"income_label\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int)\ndf.drop([\"income\", \"fnlwgt\", \"educational_num\"], axis=1, inplace=True)\ntarget_colname = \"income_label\"\n</pre> # use_cuda = torch.cuda.is_available() df = load_adult(as_frame=True) df.columns = [c.replace(\"-\", \"_\") for c in df.columns] df[\"income_label\"] = (df[\"income\"].apply(lambda x: \"&gt;50K\" in x)).astype(int) df.drop([\"income\", \"fnlwgt\", \"educational_num\"], axis=1, inplace=True) target_colname = \"income_label\" In\u00a0[3]: Copied! <pre>df.head()\n</pre> df.head() Out[3]: age workclass education marital_status occupation relationship race gender capital_gain capital_loss hours_per_week native_country income_label 0 25 Private 11th Never-married Machine-op-inspct Own-child Black Male 0 0 40 United-States 0 1 38 Private HS-grad Married-civ-spouse Farming-fishing Husband White Male 0 0 50 United-States 0 2 28 Local-gov Assoc-acdm Married-civ-spouse Protective-serv Husband White Male 0 0 40 United-States 1 3 44 Private Some-college Married-civ-spouse Machine-op-inspct Husband Black Male 7688 0 40 United-States 1 4 18 ? Some-college Never-married ? Own-child White Female 0 0 30 United-States 0 In\u00a0[4]: Copied! <pre>cat_embed_cols = []\nfor col in df.columns:\n    if df[col].dtype == \"O\" or df[col].nunique() &lt; 200 and col != target_colname:\n        cat_embed_cols.append(col)\n</pre> cat_embed_cols = [] for col in df.columns:     if df[col].dtype == \"O\" or df[col].nunique() &lt; 200 and col != target_colname:         cat_embed_cols.append(col) In\u00a0[5]: Copied! <pre># all cols will be categorical\nassert len(cat_embed_cols) == df.shape[1] - 1\n</pre> # all cols will be categorical assert len(cat_embed_cols) == df.shape[1] - 1 In\u00a0[6]: Copied! <pre>train, test = train_test_split(\n    df, test_size=0.1, random_state=1, stratify=df[[target_colname]]\n)\n</pre> train, test = train_test_split(     df, test_size=0.1, random_state=1, stratify=df[[target_colname]] ) In\u00a0[7]: Copied! <pre>tab_preprocessor = TabPreprocessor(cat_embed_cols=cat_embed_cols, with_attention=True)\n</pre> tab_preprocessor = TabPreprocessor(cat_embed_cols=cat_embed_cols, with_attention=True) In\u00a0[8]: Copied! <pre>X_tab_train = tab_preprocessor.fit_transform(train)\nX_tab_test = tab_preprocessor.transform(test)\ntarget = train[target_colname].values\n</pre> X_tab_train = tab_preprocessor.fit_transform(train) X_tab_test = tab_preprocessor.transform(test) target = train[target_colname].values In\u00a0[9]: Copied! <pre>tab_transformer = TabTransformer(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    embed_continuous_method=\"standard\",\n    input_dim=8,\n    n_heads=2,\n    n_blocks=1,\n    attn_dropout=0.1,\n    transformer_activation=\"relu\",\n)\n</pre> tab_transformer = TabTransformer(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     embed_continuous_method=\"standard\",     input_dim=8,     n_heads=2,     n_blocks=1,     attn_dropout=0.1,     transformer_activation=\"relu\", ) In\u00a0[10]: Copied! <pre>model = WideDeep(deeptabular=tab_transformer)\n</pre> model = WideDeep(deeptabular=tab_transformer) In\u00a0[11]: Copied! <pre>optimizer = torch.optim.Adam(model.parameters(), lr=0.01, weight_decay=0.0)\n</pre> optimizer = torch.optim.Adam(model.parameters(), lr=0.01, weight_decay=0.0) In\u00a0[12]: Copied! <pre>lr_scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(\n    optimizer,\n    threshold=0.001,\n    threshold_mode=\"abs\",\n    patience=10,\n)\n</pre> lr_scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(     optimizer,     threshold=0.001,     threshold_mode=\"abs\",     patience=10, ) In\u00a0[13]: Copied! <pre>early_stopping = EarlyStopping(\n    min_delta=0.001, patience=30, restore_best_weights=True, verbose=True\n)\n</pre> early_stopping = EarlyStopping(     min_delta=0.001, patience=30, restore_best_weights=True, verbose=True ) In\u00a0[14]: Copied! <pre>trainer = Trainer(\n    model,\n    objective=\"binary\",\n    optimizers=optimizer,\n    lr_schedulers=lr_scheduler,\n    reducelronplateau_criterion=\"loss\",\n    callbacks=[early_stopping],\n    metrics=[Accuracy],\n)\n</pre> trainer = Trainer(     model,     objective=\"binary\",     optimizers=optimizer,     lr_schedulers=lr_scheduler,     reducelronplateau_criterion=\"loss\",     callbacks=[early_stopping],     metrics=[Accuracy], ) <p>The feature importances will be computed after training, using a sample of the training dataset of size <code>feature_importance_sample_size</code></p> In\u00a0[15]: Copied! <pre>trainer.fit(\n    X_tab=X_tab_train,\n    target=target,\n    val_split=0.2,\n    n_epochs=100,\n    batch_size=128,\n    validation_freq=1,\n    feature_importance_sample_size=1000,\n)\n</pre> trainer.fit(     X_tab=X_tab_train,     target=target,     val_split=0.2,     n_epochs=100,     batch_size=128,     validation_freq=1,     feature_importance_sample_size=1000, ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 81.80it/s, loss=0.334, metrics={'acc': 0.847}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 111.34it/s, loss=0.294, metrics={'acc': 0.8669}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 83.02it/s, loss=0.293, metrics={'acc': 0.8656}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 124.03it/s, loss=0.283, metrics={'acc': 0.8678}]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 87.69it/s, loss=0.282, metrics={'acc': 0.8703}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 118.22it/s, loss=0.279, metrics={'acc': 0.8717}]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 79.24it/s, loss=0.277, metrics={'acc': 0.8718}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 115.29it/s, loss=0.277, metrics={'acc': 0.8731}]\nepoch 5: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 83.76it/s, loss=0.275, metrics={'acc': 0.8727}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 120.80it/s, loss=0.276, metrics={'acc': 0.8727}]\nepoch 6: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 82.78it/s, loss=0.273, metrics={'acc': 0.873}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 100.43it/s, loss=0.276, metrics={'acc': 0.871}]\nepoch 7: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 76.35it/s, loss=0.271, metrics={'acc': 0.8742}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 116.14it/s, loss=0.275, metrics={'acc': 0.8726}]\nepoch 8: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 74.29it/s, loss=0.271, metrics={'acc': 0.875}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 117.46it/s, loss=0.276, metrics={'acc': 0.8718}]\nepoch 9: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 77.10it/s, loss=0.27, metrics={'acc': 0.8761}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 105.49it/s, loss=0.275, metrics={'acc': 0.8728}]\nepoch 10: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 70.40it/s, loss=0.269, metrics={'acc': 0.8747}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 105.47it/s, loss=0.275, metrics={'acc': 0.8726}]\nepoch 11: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 72.83it/s, loss=0.268, metrics={'acc': 0.8742}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 105.03it/s, loss=0.274, metrics={'acc': 0.873}]\nepoch 12: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 71.86it/s, loss=0.267, metrics={'acc': 0.8743}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 106.61it/s, loss=0.274, metrics={'acc': 0.8734}]\nepoch 13: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 72.39it/s, loss=0.267, metrics={'acc': 0.876}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 108.05it/s, loss=0.275, metrics={'acc': 0.8717}]\nepoch 14: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 73.36it/s, loss=0.265, metrics={'acc': 0.8767}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 109.60it/s, loss=0.276, metrics={'acc': 0.8747}]\nepoch 15: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 73.34it/s, loss=0.264, metrics={'acc': 0.876}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 117.55it/s, loss=0.276, metrics={'acc': 0.8706}]\nepoch 16: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 75.35it/s, loss=0.264, metrics={'acc': 0.8777}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 122.08it/s, loss=0.275, metrics={'acc': 0.8753}]\nepoch 17: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 75.96it/s, loss=0.263, metrics={'acc': 0.877}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 120.83it/s, loss=0.277, metrics={'acc': 0.8739}]\nepoch 18: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 75.38it/s, loss=0.263, metrics={'acc': 0.8779}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 118.18it/s, loss=0.278, metrics={'acc': 0.8714}]\nepoch 19: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 75.61it/s, loss=0.261, metrics={'acc': 0.8784}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 119.19it/s, loss=0.278, metrics={'acc': 0.8712}]\nepoch 20: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 72.43it/s, loss=0.261, metrics={'acc': 0.8791}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 118.87it/s, loss=0.28, metrics={'acc': 0.873}]\nepoch 21: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 73.97it/s, loss=0.26, metrics={'acc': 0.8787}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 107.50it/s, loss=0.279, metrics={'acc': 0.8732}]\nepoch 22: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 71.76it/s, loss=0.253, metrics={'acc': 0.8816}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 108.11it/s, loss=0.279, metrics={'acc': 0.8707}]\nepoch 23: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 71.92it/s, loss=0.252, metrics={'acc': 0.8828}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 100.14it/s, loss=0.28, metrics={'acc': 0.8711}]\nepoch 24: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 75.04it/s, loss=0.252, metrics={'acc': 0.8829}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 98.36it/s, loss=0.28, metrics={'acc': 0.8708}]\nepoch 25: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 76.31it/s, loss=0.251, metrics={'acc': 0.883}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 122.97it/s, loss=0.281, metrics={'acc': 0.8709}]\nepoch 26: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 76.63it/s, loss=0.25, metrics={'acc': 0.8834}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 123.07it/s, loss=0.281, metrics={'acc': 0.8698}]\nepoch 27: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 78.37it/s, loss=0.251, metrics={'acc': 0.884}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 114.75it/s, loss=0.281, metrics={'acc': 0.87}]\nepoch 28: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 77.19it/s, loss=0.25, metrics={'acc': 0.883}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 117.99it/s, loss=0.282, metrics={'acc': 0.8699}]\nepoch 29: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 76.19it/s, loss=0.25, metrics={'acc': 0.8829}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 111.11it/s, loss=0.282, metrics={'acc': 0.8695}]\nepoch 30: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 77.57it/s, loss=0.249, metrics={'acc': 0.8839}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 115.64it/s, loss=0.283, metrics={'acc': 0.8689}]\nepoch 31: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 79.55it/s, loss=0.249, metrics={'acc': 0.8846}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 125.10it/s, loss=0.283, metrics={'acc': 0.869}]\nepoch 32: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 82.56it/s, loss=0.248, metrics={'acc': 0.8841}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 118.45it/s, loss=0.284, metrics={'acc': 0.8687}]\nepoch 33: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 81.06it/s, loss=0.248, metrics={'acc': 0.8848}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 129.26it/s, loss=0.284, metrics={'acc': 0.8689}]\nepoch 34: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 80.53it/s, loss=0.248, metrics={'acc': 0.8854}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 120.61it/s, loss=0.283, metrics={'acc': 0.869}]\nepoch 35: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 80.78it/s, loss=0.248, metrics={'acc': 0.8853}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 127.31it/s, loss=0.283, metrics={'acc': 0.8694}]\nepoch 36: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 82.51it/s, loss=0.248, metrics={'acc': 0.8863}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 125.94it/s, loss=0.283, metrics={'acc': 0.8693}]\nepoch 37: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 81.35it/s, loss=0.247, metrics={'acc': 0.8844}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 126.77it/s, loss=0.283, metrics={'acc': 0.8692}]\nepoch 38: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 80.62it/s, loss=0.248, metrics={'acc': 0.8837}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 116.62it/s, loss=0.283, metrics={'acc': 0.8692}]\nepoch 39: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 79.82it/s, loss=0.248, metrics={'acc': 0.8842}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 123.64it/s, loss=0.283, metrics={'acc': 0.8695}]\nepoch 40: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 76.86it/s, loss=0.247, metrics={'acc': 0.8855}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 120.34it/s, loss=0.283, metrics={'acc': 0.8692}]\n</pre> <pre>Best Epoch: 10. Best val_loss: 0.27451\nRestoring model weights from the end of the best epoch\n</pre> In\u00a0[16]: Copied! <pre>trainer.feature_importance\n</pre> trainer.feature_importance Out[16]: <pre>{'age': 0.09718182,\n 'workclass': 0.090637445,\n 'education': 0.08910798,\n 'marital_status': 0.08971319,\n 'occupation': 0.12546304,\n 'relationship': 0.086381145,\n 'race': 0.050686445,\n 'gender': 0.05116429,\n 'capital_gain': 0.08165918,\n 'capital_loss': 0.07702667,\n 'hours_per_week': 0.08205996,\n 'native_country': 0.07891885}</pre> In\u00a0[17]: Copied! <pre>preds = trainer.predict(X_tab=X_tab_test)\n</pre> preds = trainer.predict(X_tab=X_tab_test) <pre>predict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 39/39 [00:00&lt;00:00, 199.63it/s]\n</pre> In\u00a0[18]: Copied! <pre>accuracy_score(preds, test.income_label)\n</pre> accuracy_score(preds, test.income_label) Out[18]: <pre>0.8685772773797339</pre> In\u00a0[19]: Copied! <pre>test.reset_index(drop=True, inplace=True)\n</pre> test.reset_index(drop=True, inplace=True) In\u00a0[20]: Copied! <pre>test[test.income_label == 0].head(1)\n</pre> test[test.income_label == 0].head(1) Out[20]: age workclass education marital_status occupation relationship race gender capital_gain capital_loss hours_per_week native_country income_label 0 26 Private Some-college Never-married Exec-managerial Not-in-family White Male 0 0 60 United-States 0 In\u00a0[21]: Copied! <pre>test[test.income_label == 1].head(1)\n</pre> test[test.income_label == 1].head(1) Out[21]: age workclass education marital_status occupation relationship race gender capital_gain capital_loss hours_per_week native_country income_label 3 36 Local-gov Doctorate Married-civ-spouse Prof-specialty Husband White Male 0 1887 50 United-States 1 <p>To get the feature importance of a test dataset, simply use the <code>explain</code> method</p> In\u00a0[22]: Copied! <pre>feat_imp_per_sample = trainer.explain(X_tab_test, save_step_masks=False)\n</pre> feat_imp_per_sample = trainer.explain(X_tab_test, save_step_masks=False) In\u00a0[23]: Copied! <pre>list(test.iloc[0].index[np.argsort(-feat_imp_per_sample[0])])\n</pre> list(test.iloc[0].index[np.argsort(-feat_imp_per_sample[0])]) Out[23]: <pre>['hours_per_week',\n 'education',\n 'relationship',\n 'occupation',\n 'workclass',\n 'capital_gain',\n 'native_country',\n 'marital_status',\n 'capital_loss',\n 'age',\n 'race',\n 'gender']</pre> In\u00a0[24]: Copied! <pre>list(test.iloc[3].index[np.argsort(-feat_imp_per_sample[3])])\n</pre> list(test.iloc[3].index[np.argsort(-feat_imp_per_sample[3])]) Out[24]: <pre>['age',\n 'capital_loss',\n 'hours_per_week',\n 'marital_status',\n 'native_country',\n 'relationship',\n 'race',\n 'education',\n 'occupation',\n 'capital_gain',\n 'gender',\n 'workclass']</pre> <p>We could do the same with the <code>ContextAttentionMLP</code></p> In\u00a0[25]: Copied! <pre>context_attn_mlp = ContextAttentionMLP(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    cat_embed_dropout=0.0,\n    input_dim=16,\n    attn_dropout=0.1,\n    attn_activation=\"relu\",\n)\n</pre> context_attn_mlp = ContextAttentionMLP(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     cat_embed_dropout=0.0,     input_dim=16,     attn_dropout=0.1,     attn_activation=\"relu\", ) In\u00a0[26]: Copied! <pre>mlp_model = WideDeep(deeptabular=context_attn_mlp)\n</pre> mlp_model = WideDeep(deeptabular=context_attn_mlp) In\u00a0[27]: Copied! <pre>mlp_optimizer = torch.optim.Adam(mlp_model.parameters(), lr=0.01, weight_decay=0.0)\n</pre> mlp_optimizer = torch.optim.Adam(mlp_model.parameters(), lr=0.01, weight_decay=0.0) In\u00a0[28]: Copied! <pre>mlp_lr_scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(\n    mlp_optimizer,\n    threshold=0.001,\n    threshold_mode=\"abs\",\n    patience=10,\n)\n</pre> mlp_lr_scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(     mlp_optimizer,     threshold=0.001,     threshold_mode=\"abs\",     patience=10, ) In\u00a0[29]: Copied! <pre>mlp_early_stopping = EarlyStopping(\n    min_delta=0.001, patience=30, restore_best_weights=True, verbose=True\n)\n</pre> mlp_early_stopping = EarlyStopping(     min_delta=0.001, patience=30, restore_best_weights=True, verbose=True ) In\u00a0[30]: Copied! <pre>mlp_trainer = Trainer(\n    mlp_model,\n    objective=\"binary\",\n    optimizers=mlp_optimizer,\n    lr_schedulers=mlp_lr_scheduler,\n    reducelronplateau_criterion=\"loss\",\n    callbacks=[mlp_early_stopping],\n    metrics=[Accuracy],\n)\n</pre> mlp_trainer = Trainer(     mlp_model,     objective=\"binary\",     optimizers=mlp_optimizer,     lr_schedulers=mlp_lr_scheduler,     reducelronplateau_criterion=\"loss\",     callbacks=[mlp_early_stopping],     metrics=[Accuracy], ) In\u00a0[31]: Copied! <pre>mlp_trainer.fit(\n    X_tab=X_tab_train,\n    target=target,\n    val_split=0.2,\n    n_epochs=100,\n    batch_size=128,\n    validation_freq=1,\n    feature_importance_sample_size=1000,\n)\n</pre> mlp_trainer.fit(     X_tab=X_tab_train,     target=target,     val_split=0.2,     n_epochs=100,     batch_size=128,     validation_freq=1,     feature_importance_sample_size=1000, ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 73.11it/s, loss=0.405, metrics={'acc': 0.8094}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 119.26it/s, loss=0.309, metrics={'acc': 0.8583}]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 71.70it/s, loss=0.332, metrics={'acc': 0.8447}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 116.36it/s, loss=0.293, metrics={'acc': 0.8646}]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 78.42it/s, loss=0.319, metrics={'acc': 0.8505}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 118.05it/s, loss=0.293, metrics={'acc': 0.8654}]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 79.00it/s, loss=0.312, metrics={'acc': 0.8554}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 116.49it/s, loss=0.291, metrics={'acc': 0.8661}]\nepoch 5: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 79.50it/s, loss=0.308, metrics={'acc': 0.8583}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 120.48it/s, loss=0.287, metrics={'acc': 0.8669}]\nepoch 6: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 80.84it/s, loss=0.303, metrics={'acc': 0.8605}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 128.70it/s, loss=0.288, metrics={'acc': 0.8673}]\nepoch 7: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 79.93it/s, loss=0.301, metrics={'acc': 0.8597}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 121.47it/s, loss=0.298, metrics={'acc': 0.8628}]\nepoch 8: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 80.56it/s, loss=0.3, metrics={'acc': 0.8592}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 119.84it/s, loss=0.281, metrics={'acc': 0.8718}]\nepoch 9: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 79.17it/s, loss=0.298, metrics={'acc': 0.8619}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 126.32it/s, loss=0.28, metrics={'acc': 0.8716}]\nepoch 10: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 82.13it/s, loss=0.297, metrics={'acc': 0.8615}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 120.50it/s, loss=0.281, metrics={'acc': 0.8718}]\nepoch 11: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 82.54it/s, loss=0.293, metrics={'acc': 0.8641}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 122.57it/s, loss=0.284, metrics={'acc': 0.867}]\nepoch 12: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 80.92it/s, loss=0.293, metrics={'acc': 0.863}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 126.42it/s, loss=0.282, metrics={'acc': 0.8701}]\nepoch 13: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 79.61it/s, loss=0.293, metrics={'acc': 0.8635}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 117.56it/s, loss=0.276, metrics={'acc': 0.8719}]\nepoch 14: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 74.92it/s, loss=0.29, metrics={'acc': 0.8633}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 112.06it/s, loss=0.286, metrics={'acc': 0.8669}]\nepoch 15: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 76.83it/s, loss=0.291, metrics={'acc': 0.865}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 112.88it/s, loss=0.282, metrics={'acc': 0.8677}]\nepoch 16: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 76.74it/s, loss=0.29, metrics={'acc': 0.8653}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 120.85it/s, loss=0.285, metrics={'acc': 0.8672}]\nepoch 17: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 75.99it/s, loss=0.29, metrics={'acc': 0.865}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 113.53it/s, loss=0.282, metrics={'acc': 0.8681}]\nepoch 18: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 71.22it/s, loss=0.288, metrics={'acc': 0.8651}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 112.89it/s, loss=0.288, metrics={'acc': 0.8676}]\nepoch 19: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 75.85it/s, loss=0.29, metrics={'acc': 0.8661}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 114.26it/s, loss=0.284, metrics={'acc': 0.8662}]\nepoch 20: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 75.80it/s, loss=0.289, metrics={'acc': 0.8661}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 119.44it/s, loss=0.281, metrics={'acc': 0.8703}]\nepoch 21: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 75.72it/s, loss=0.29, metrics={'acc': 0.8661}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 113.04it/s, loss=0.285, metrics={'acc': 0.8648}]\nepoch 22: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 74.86it/s, loss=0.289, metrics={'acc': 0.8656}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 111.75it/s, loss=0.282, metrics={'acc': 0.8666}]\nepoch 23: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 76.68it/s, loss=0.289, metrics={'acc': 0.8668}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 115.86it/s, loss=0.282, metrics={'acc': 0.8724}]\nepoch 24: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 75.93it/s, loss=0.288, metrics={'acc': 0.8653}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 114.69it/s, loss=0.285, metrics={'acc': 0.8656}]\nepoch 25: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 79.99it/s, loss=0.284, metrics={'acc': 0.8671}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 118.27it/s, loss=0.277, metrics={'acc': 0.8707}]\nepoch 26: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 79.86it/s, loss=0.282, metrics={'acc': 0.8686}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 123.94it/s, loss=0.276, metrics={'acc': 0.8712}]\nepoch 27: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 77.86it/s, loss=0.283, metrics={'acc': 0.8691}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 114.11it/s, loss=0.277, metrics={'acc': 0.8716}]\nepoch 28: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 77.43it/s, loss=0.281, metrics={'acc': 0.8696}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 114.64it/s, loss=0.277, metrics={'acc': 0.8712}]\nepoch 29: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 77.39it/s, loss=0.281, metrics={'acc': 0.8696}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 115.83it/s, loss=0.277, metrics={'acc': 0.872}]\nepoch 30: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 74.93it/s, loss=0.28, metrics={'acc': 0.8706}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 113.07it/s, loss=0.275, metrics={'acc': 0.8714}]\nepoch 31: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 77.35it/s, loss=0.281, metrics={'acc': 0.8697}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 112.68it/s, loss=0.276, metrics={'acc': 0.872}]\nepoch 32: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 76.72it/s, loss=0.28, metrics={'acc': 0.8693}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 122.50it/s, loss=0.276, metrics={'acc': 0.8709}]\nepoch 33: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 76.68it/s, loss=0.28, metrics={'acc': 0.8716}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 110.07it/s, loss=0.277, metrics={'acc': 0.8709}]\nepoch 34: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 73.58it/s, loss=0.279, metrics={'acc': 0.8704}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 111.07it/s, loss=0.274, metrics={'acc': 0.8719}]\nepoch 35: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 74.03it/s, loss=0.28, metrics={'acc': 0.8687}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 110.15it/s, loss=0.276, metrics={'acc': 0.871}]\nepoch 36: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 73.11it/s, loss=0.279, metrics={'acc': 0.8706}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 111.10it/s, loss=0.278, metrics={'acc': 0.8705}]\nepoch 37: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 74.30it/s, loss=0.279, metrics={'acc': 0.869}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 110.75it/s, loss=0.279, metrics={'acc': 0.8702}]\nepoch 38: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 72.34it/s, loss=0.28, metrics={'acc': 0.8691}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 117.79it/s, loss=0.277, metrics={'acc': 0.8698}]\nepoch 39: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 75.49it/s, loss=0.279, metrics={'acc': 0.8694}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 115.25it/s, loss=0.279, metrics={'acc': 0.87}]\nepoch 40: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 75.51it/s, loss=0.28, metrics={'acc': 0.8694}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 110.90it/s, loss=0.277, metrics={'acc': 0.8694}]\nepoch 41: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 75.35it/s, loss=0.278, metrics={'acc': 0.8716}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 115.74it/s, loss=0.28, metrics={'acc': 0.8675}]\nepoch 42: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 76.35it/s, loss=0.279, metrics={'acc': 0.8695}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 122.76it/s, loss=0.277, metrics={'acc': 0.8699}]\nepoch 43: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:04&lt;00:00, 66.14it/s, loss=0.279, metrics={'acc': 0.8681}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 106.20it/s, loss=0.277, metrics={'acc': 0.8714}]\nepoch 44: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 73.37it/s, loss=0.279, metrics={'acc': 0.8704}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 115.18it/s, loss=0.277, metrics={'acc': 0.8716}]\nepoch 45: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 76.23it/s, loss=0.278, metrics={'acc': 0.8702}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 124.83it/s, loss=0.278, metrics={'acc': 0.8707}]\nepoch 46: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 77.12it/s, loss=0.278, metrics={'acc': 0.8704}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 126.62it/s, loss=0.279, metrics={'acc': 0.8693}]\nepoch 47: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 78.55it/s, loss=0.276, metrics={'acc': 0.8713}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 117.99it/s, loss=0.279, metrics={'acc': 0.8691}]\nepoch 48: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 77.25it/s, loss=0.278, metrics={'acc': 0.8719}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 124.52it/s, loss=0.278, metrics={'acc': 0.8695}]\nepoch 49: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 78.35it/s, loss=0.277, metrics={'acc': 0.8721}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 119.82it/s, loss=0.279, metrics={'acc': 0.8691}]\nepoch 50: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 77.15it/s, loss=0.277, metrics={'acc': 0.8717}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 122.62it/s, loss=0.278, metrics={'acc': 0.8699}]\nepoch 51: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 78.55it/s, loss=0.277, metrics={'acc': 0.8713}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 117.63it/s, loss=0.278, metrics={'acc': 0.87}]\nepoch 52: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 77.04it/s, loss=0.276, metrics={'acc': 0.8721}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 126.39it/s, loss=0.278, metrics={'acc': 0.8697}]\nepoch 53: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 78.15it/s, loss=0.277, metrics={'acc': 0.8721}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 127.56it/s, loss=0.278, metrics={'acc': 0.8699}]\nepoch 54: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 77.41it/s, loss=0.277, metrics={'acc': 0.8711}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 118.95it/s, loss=0.278, metrics={'acc': 0.8698}]\nepoch 55: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 76.35it/s, loss=0.277, metrics={'acc': 0.8718}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 126.90it/s, loss=0.278, metrics={'acc': 0.8699}]\nepoch 56: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 77.83it/s, loss=0.277, metrics={'acc': 0.8707}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 115.13it/s, loss=0.279, metrics={'acc': 0.8691}]\nepoch 57: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 75.20it/s, loss=0.277, metrics={'acc': 0.8722}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 123.16it/s, loss=0.279, metrics={'acc': 0.8691}]\nepoch 58: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 73.33it/s, loss=0.276, metrics={'acc': 0.871}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 123.37it/s, loss=0.278, metrics={'acc': 0.8691}]\nepoch 59: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 78.41it/s, loss=0.277, metrics={'acc': 0.8714}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 125.17it/s, loss=0.278, metrics={'acc': 0.8695}]\nepoch 60: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 79.26it/s, loss=0.276, metrics={'acc': 0.8721}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 120.60it/s, loss=0.278, metrics={'acc': 0.869}]\nepoch 61: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 75.88it/s, loss=0.278, metrics={'acc': 0.8703}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 124.47it/s, loss=0.278, metrics={'acc': 0.8692}]\nepoch 62: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 77.13it/s, loss=0.276, metrics={'acc': 0.8711}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 125.80it/s, loss=0.278, metrics={'acc': 0.8691}]\nepoch 63: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 77.20it/s, loss=0.277, metrics={'acc': 0.8715}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 118.50it/s, loss=0.278, metrics={'acc': 0.8695}]\nepoch 64: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 275/275 [00:03&lt;00:00, 78.11it/s, loss=0.276, metrics={'acc': 0.8719}]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 69/69 [00:00&lt;00:00, 114.52it/s, loss=0.278, metrics={'acc': 0.869}]\n</pre> <pre>Best Epoch: 34. Best val_loss: 0.27449\nRestoring model weights from the end of the best epoch\n</pre> In\u00a0[32]: Copied! <pre>mlp_trainer.feature_importance\n</pre> mlp_trainer.feature_importance Out[32]: <pre>{'age': 0.116632804,\n 'workclass': 0.050255153,\n 'education': 0.094621316,\n 'marital_status': 0.12328919,\n 'occupation': 0.107893184,\n 'relationship': 0.11747801,\n 'race': 0.054717205,\n 'gender': 0.07514235,\n 'capital_gain': 0.059732802,\n 'capital_loss': 0.06738944,\n 'hours_per_week': 0.0610674,\n 'native_country': 0.07178114}</pre> In\u00a0[33]: Copied! <pre>mlp_preds = mlp_trainer.predict(X_tab=X_tab_test)\n</pre> mlp_preds = mlp_trainer.predict(X_tab=X_tab_test) <pre>predict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 39/39 [00:00&lt;00:00, 212.38it/s]\n</pre> In\u00a0[34]: Copied! <pre>accuracy_score(mlp_preds, test.income_label)\n</pre> accuracy_score(mlp_preds, test.income_label) Out[34]: <pre>0.8726714431934494</pre>"},{"location":"examples/17_feature_importance_via_attention_weights.html#feature-importance-via-the-attention-weights","title":"Feature Importance via the attention weights\u00b6","text":"<p>I will start by saying that I consider this feature of the library purely experimental. First of all I think there are multiple ways one could address finding the features importances for these models. However, and more importantly, one has to bear in mind that even tree-based algorithms on the same dataset produce different feature importances. This is more \"dramatic\" if one uses different techniques, such as shap or feature permutation (see for example this and references therein). All this to say that, sometimes, feature importance is just a measure contained within the experiment run, and for the model used.</p> <p>With that in mind, each instantiation of a deep tabular model, that has millions of trainable parameters, will potentially produce a different set of feature importances, even if the model has the same architecture. Moreover, this effect will become more apparent if the dataset is relatively easy and there are dependent/related columns so that one could get to the same success metric with different parameters.</p> <p>In summary, feature importances are implemented in this librray for all attention-based models for tabular data, with the exception of the <code>TabPerceiver</code>. However this functionality has to be used and interpreted with care and consider of value within the 'universe' (or context) of the model with which these features were produced.</p> <p>Nonetheless, let's have a look to how one would access to the feature importances when using this library.</p>"},{"location":"examples/18_wide_and_deep_for_recsys_pt1.html","title":"18_wide_and_deep_for_recsys_pt1","text":"<p>The goal of this, and the companion (part 2) notebooks is to illustrate how one could use this library in the context of recommendation systems. In particular, this notebook and the scripts at the <code>wide_deep_for_recsys</code> dir are a response to this issue. Therefore, we will use the Kaggle notebook referred in that issue here.</p> <p>In order to keep the length of the notebook tractable, we will split this exercise in 2. In this first notebook we will prepare the data in almost the exact same way as it is done in the Kaggle notebook and also show how one could use <code>pytorch-widedeep</code> to build a model almost identical to the one in that notebook.</p> <p>In a second notebook, we will show how one could use this library to implement other models, still following the same problem formulation.</p> In\u00a0[1]: Copied! <pre>from pathlib import Path\nimport warnings\n\nimport pandas as pd\nfrom sklearn.model_selection import train_test_split\n\nfrom pytorch_widedeep.datasets import load_movielens100k\n</pre> from pathlib import Path import warnings  import pandas as pd from sklearn.model_selection import train_test_split  from pytorch_widedeep.datasets import load_movielens100k In\u00a0[2]: Copied! <pre>warnings.filterwarnings(\"ignore\")\n</pre> warnings.filterwarnings(\"ignore\") In\u00a0[3]: Copied! <pre>save_path = Path(\"prepared_data\")\nif not save_path.exists():\n    save_path.mkdir(parents=True, exist_ok=True)\n</pre> save_path = Path(\"prepared_data\") if not save_path.exists():     save_path.mkdir(parents=True, exist_ok=True) In\u00a0[4]: Copied! <pre>data, users, items = load_movielens100k(as_frame=True)\n</pre> data, users, items = load_movielens100k(as_frame=True) In\u00a0[5]: Copied! <pre># Alternatively, as specified in the docs: 'The last 19 fields are the genres' so:\n# list_of_genres = items.columns.tolist()[-19:]\nlist_of_genres = [\n    \"unknown\",\n    \"Action\",\n    \"Adventure\",\n    \"Animation\",\n    \"Children's\",\n    \"Comedy\",\n    \"Crime\",\n    \"Documentary\",\n    \"Drama\",\n    \"Fantasy\",\n    \"Film-Noir\",\n    \"Horror\",\n    \"Musical\",\n    \"Mystery\",\n    \"Romance\",\n    \"Sci-Fi\",\n    \"Thriller\",\n    \"War\",\n    \"Western\",\n]\n</pre> # Alternatively, as specified in the docs: 'The last 19 fields are the genres' so: # list_of_genres = items.columns.tolist()[-19:] list_of_genres = [     \"unknown\",     \"Action\",     \"Adventure\",     \"Animation\",     \"Children's\",     \"Comedy\",     \"Crime\",     \"Documentary\",     \"Drama\",     \"Fantasy\",     \"Film-Noir\",     \"Horror\",     \"Musical\",     \"Mystery\",     \"Romance\",     \"Sci-Fi\",     \"Thriller\",     \"War\",     \"Western\", ] <p>Let's first start by loading the interactions, user and item data</p> In\u00a0[6]: Copied! <pre>data.head()\n</pre> data.head() Out[6]: user_id movie_id rating timestamp 0 196 242 3 881250949 1 186 302 3 891717742 2 22 377 1 878887116 3 244 51 2 880606923 4 166 346 1 886397596 In\u00a0[7]: Copied! <pre>users.head()\n</pre> users.head() Out[7]: user_id age gender occupation zip_code 0 1 24 M technician 85711 1 2 53 F other 94043 2 3 23 M writer 32067 3 4 24 M technician 43537 4 5 33 F other 15213 In\u00a0[8]: Copied! <pre>items.head()\n</pre> items.head() Out[8]: movie_id movie_title release_date video_release_date IMDb_URL unknown Action Adventure Animation Children's ... Fantasy Film-Noir Horror Musical Mystery Romance Sci-Fi Thriller War Western 0 1 Toy Story (1995) 01-Jan-1995 NaN http://us.imdb.com/M/title-exact?Toy%20Story%2... 0 0 0 1 1 ... 0 0 0 0 0 0 0 0 0 0 1 2 GoldenEye (1995) 01-Jan-1995 NaN http://us.imdb.com/M/title-exact?GoldenEye%20(... 0 1 1 0 0 ... 0 0 0 0 0 0 0 1 0 0 2 3 Four Rooms (1995) 01-Jan-1995 NaN http://us.imdb.com/M/title-exact?Four%20Rooms%... 0 0 0 0 0 ... 0 0 0 0 0 0 0 1 0 0 3 4 Get Shorty (1995) 01-Jan-1995 NaN http://us.imdb.com/M/title-exact?Get%20Shorty%... 0 1 0 0 0 ... 0 0 0 0 0 0 0 0 0 0 4 5 Copycat (1995) 01-Jan-1995 NaN http://us.imdb.com/M/title-exact?Copycat%20(1995) 0 0 0 0 0 ... 0 0 0 0 0 0 0 1 0 0 <p>5 rows \u00d7 24 columns</p> In\u00a0[9]: Copied! <pre># adding a column with the number of movies watched per user\ndataset = data.sort_values([\"user_id\", \"timestamp\"]).reset_index(drop=True)\ndataset[\"one\"] = 1\ndataset[\"num_watched\"] = dataset.groupby(\"user_id\")[\"one\"].cumsum()\ndataset.drop(\"one\", axis=1, inplace=True)\ndataset.head()\n</pre> # adding a column with the number of movies watched per user dataset = data.sort_values([\"user_id\", \"timestamp\"]).reset_index(drop=True) dataset[\"one\"] = 1 dataset[\"num_watched\"] = dataset.groupby(\"user_id\")[\"one\"].cumsum() dataset.drop(\"one\", axis=1, inplace=True) dataset.head() Out[9]: user_id movie_id rating timestamp num_watched 0 1 168 5 874965478 1 1 1 172 5 874965478 2 2 1 165 5 874965518 3 3 1 156 4 874965556 4 4 1 196 5 874965677 5 In\u00a0[10]: Copied! <pre># adding a column with the mean rating at a point in time per user\ndataset[\"mean_rate\"] = (\n    dataset.groupby(\"user_id\")[\"rating\"].cumsum() / dataset[\"num_watched\"]\n)\ndataset.head()\n</pre> # adding a column with the mean rating at a point in time per user dataset[\"mean_rate\"] = (     dataset.groupby(\"user_id\")[\"rating\"].cumsum() / dataset[\"num_watched\"] ) dataset.head() Out[10]: user_id movie_id rating timestamp num_watched mean_rate 0 1 168 5 874965478 1 5.00 1 1 172 5 874965478 2 5.00 2 1 165 5 874965518 3 5.00 3 1 156 4 874965556 4 4.75 4 1 196 5 874965677 5 4.80 In\u00a0[11]: Copied! <pre>dataset[\"target\"] = dataset.groupby(\"user_id\")[\"movie_id\"].shift(-1)\n</pre> dataset[\"target\"] = dataset.groupby(\"user_id\")[\"movie_id\"].shift(-1) <p>Following the same processing used by the author in the before-mentioned Kaggle notebook, we build sequences of previous movies watched</p> In\u00a0[12]: Copied! <pre># Here the author builds the sequences\ndataset[\"prev_movies\"] = dataset[\"movie_id\"].apply(lambda x: str(x))\ndataset[\"prev_movies\"] = (\n    dataset.groupby(\"user_id\")[\"prev_movies\"]\n    .apply(lambda x: (x + \" \").cumsum().str.strip())\n    .reset_index(drop=True)\n)\ndataset[\"prev_movies\"] = dataset[\"prev_movies\"].apply(lambda x: x.split())\ndataset.head()\n</pre> # Here the author builds the sequences dataset[\"prev_movies\"] = dataset[\"movie_id\"].apply(lambda x: str(x)) dataset[\"prev_movies\"] = (     dataset.groupby(\"user_id\")[\"prev_movies\"]     .apply(lambda x: (x + \" \").cumsum().str.strip())     .reset_index(drop=True) ) dataset[\"prev_movies\"] = dataset[\"prev_movies\"].apply(lambda x: x.split()) dataset.head() Out[12]: user_id movie_id rating timestamp num_watched mean_rate target prev_movies 0 1 168 5 874965478 1 5.00 172.0 [168] 1 1 172 5 874965478 2 5.00 165.0 [168, 172] 2 1 165 5 874965518 3 5.00 156.0 [168, 172, 165] 3 1 156 4 874965556 4 4.75 196.0 [168, 172, 165, 156] 4 1 196 5 874965677 5 4.80 166.0 [168, 172, 165, 156, 196] <p>And now we add a <code>genre_rate</code> as the mean of all movies rated for a given genre per user</p> In\u00a0[13]: Copied! <pre>dataset = dataset.merge(items[[\"movie_id\"] + list_of_genres], on=\"movie_id\", how=\"left\")\nfor genre in list_of_genres:\n    dataset[f\"{genre}_rate\"] = dataset[genre] * dataset[\"rating\"]\n    dataset[genre] = dataset.groupby(\"user_id\")[genre].cumsum()\n    dataset[f\"{genre}_rate\"] = (\n        dataset.groupby(\"user_id\")[f\"{genre}_rate\"].cumsum() / dataset[genre]\n    )\ndataset[list_of_genres] = dataset[list_of_genres].apply(\n    lambda x: x / dataset[\"num_watched\"]\n)\ndataset.head()\n</pre> dataset = dataset.merge(items[[\"movie_id\"] + list_of_genres], on=\"movie_id\", how=\"left\") for genre in list_of_genres:     dataset[f\"{genre}_rate\"] = dataset[genre] * dataset[\"rating\"]     dataset[genre] = dataset.groupby(\"user_id\")[genre].cumsum()     dataset[f\"{genre}_rate\"] = (         dataset.groupby(\"user_id\")[f\"{genre}_rate\"].cumsum() / dataset[genre]     ) dataset[list_of_genres] = dataset[list_of_genres].apply(     lambda x: x / dataset[\"num_watched\"] ) dataset.head() Out[13]: user_id movie_id rating timestamp num_watched mean_rate target prev_movies unknown Action ... Fantasy_rate Film-Noir_rate Horror_rate Musical_rate Mystery_rate Romance_rate Sci-Fi_rate Thriller_rate War_rate Western_rate 0 1 168 5 874965478 1 5.00 172.0 [168] 0.0 0.000000 ... NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN 1 1 172 5 874965478 2 5.00 165.0 [168, 172] 0.0 0.500000 ... NaN NaN NaN NaN NaN 5.0 5.0 NaN 5.0 NaN 2 1 165 5 874965518 3 5.00 156.0 [168, 172, 165] 0.0 0.333333 ... NaN NaN NaN NaN NaN 5.0 5.0 NaN 5.0 NaN 3 1 156 4 874965556 4 4.75 196.0 [168, 172, 165, 156] 0.0 0.250000 ... NaN NaN NaN NaN NaN 5.0 5.0 4.0 5.0 NaN 4 1 196 5 874965677 5 4.80 166.0 [168, 172, 165, 156, 196] 0.0 0.200000 ... NaN NaN NaN NaN NaN 5.0 5.0 4.0 5.0 NaN <p>5 rows \u00d7 46 columns</p> <p>Adding user features</p> In\u00a0[14]: Copied! <pre>dataset = dataset.merge(users, on=\"user_id\", how=\"left\")\ndataset.head()\n</pre> dataset = dataset.merge(users, on=\"user_id\", how=\"left\") dataset.head() Out[14]: user_id movie_id rating timestamp num_watched mean_rate target prev_movies unknown Action ... Mystery_rate Romance_rate Sci-Fi_rate Thriller_rate War_rate Western_rate age gender occupation zip_code 0 1 168 5 874965478 1 5.00 172.0 [168] 0.0 0.000000 ... NaN NaN NaN NaN NaN NaN 24 M technician 85711 1 1 172 5 874965478 2 5.00 165.0 [168, 172] 0.0 0.500000 ... NaN 5.0 5.0 NaN 5.0 NaN 24 M technician 85711 2 1 165 5 874965518 3 5.00 156.0 [168, 172, 165] 0.0 0.333333 ... NaN 5.0 5.0 NaN 5.0 NaN 24 M technician 85711 3 1 156 4 874965556 4 4.75 196.0 [168, 172, 165, 156] 0.0 0.250000 ... NaN 5.0 5.0 4.0 5.0 NaN 24 M technician 85711 4 1 196 5 874965677 5 4.80 166.0 [168, 172, 165, 156, 196] 0.0 0.200000 ... NaN 5.0 5.0 4.0 5.0 NaN 24 M technician 85711 <p>5 rows \u00d7 50 columns</p> <p>Again, we use the same settings as those in the Kaggle notebook, but <code>COLD_START_TRESH</code> is pretty aggressive</p> In\u00a0[15]: Copied! <pre>COLD_START_TRESH = 5\n\nfiltred_data = dataset[\n    (dataset[\"num_watched\"] &gt;= COLD_START_TRESH) &amp; ~(dataset[\"target\"].isna())\n].sort_values(\"timestamp\")\ntrain_data, _test_data = train_test_split(filtred_data, test_size=0.2, shuffle=False)\nvalid_data, test_data = train_test_split(_test_data, test_size=0.5, shuffle=False)\n</pre> COLD_START_TRESH = 5  filtred_data = dataset[     (dataset[\"num_watched\"] &gt;= COLD_START_TRESH) &amp; ~(dataset[\"target\"].isna()) ].sort_values(\"timestamp\") train_data, _test_data = train_test_split(filtred_data, test_size=0.2, shuffle=False) valid_data, test_data = train_test_split(_test_data, test_size=0.5, shuffle=False) In\u00a0[16]: Copied! <pre>cols_to_drop = [\n    # \"rating\",\n    \"timestamp\",\n    \"num_watched\",\n]\n\ndf_train = train_data.drop(cols_to_drop, axis=1)\ndf_valid = valid_data.drop(cols_to_drop, axis=1)\ndf_test = test_data.drop(cols_to_drop, axis=1)\n\ndf_train.to_pickle(save_path / \"df_train.pkl\")\ndf_valid.to_pickle(save_path / \"df_valid.pkl\")\ndf_test.to_pickle(save_path / \"df_test.pkl\")\n</pre> cols_to_drop = [     # \"rating\",     \"timestamp\",     \"num_watched\", ]  df_train = train_data.drop(cols_to_drop, axis=1) df_valid = valid_data.drop(cols_to_drop, axis=1) df_test = test_data.drop(cols_to_drop, axis=1)  df_train.to_pickle(save_path / \"df_train.pkl\") df_valid.to_pickle(save_path / \"df_valid.pkl\") df_test.to_pickle(save_path / \"df_test.pkl\") <p>Let's now build a model that is nearly identical to the one use in the Kaggle notebook</p> In\u00a0[17]: Copied! <pre>import numpy as np\nimport torch\nfrom torch import nn\nfrom scipy.sparse import coo_matrix\n\nfrom pytorch_widedeep import Trainer\nfrom pytorch_widedeep.models import TabMlp, BasicRNN, WideDeep\nfrom pytorch_widedeep.preprocessing import TabPreprocessor\n</pre> import numpy as np import torch from torch import nn from scipy.sparse import coo_matrix  from pytorch_widedeep import Trainer from pytorch_widedeep.models import TabMlp, BasicRNN, WideDeep from pytorch_widedeep.preprocessing import TabPreprocessor In\u00a0[18]: Copied! <pre>device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n\nsave_path = Path(\"prepared_data\")\n\nPAD_IDX = 0\n</pre> device = \"cuda\" if torch.cuda.is_available() else \"cpu\"  save_path = Path(\"prepared_data\")  PAD_IDX = 0 <p>Let's use some of the functions the author of the kaggle's notebook uses to prepare the data</p> In\u00a0[19]: Copied! <pre>def get_coo_indexes(lil):\n    rows = []\n    cols = []\n    for i, el in enumerate(lil):\n        if type(el) != list:\n            el = [el]\n        for j in el:\n            rows.append(i)\n            cols.append(j)\n    return rows, cols\n\n\ndef get_sparse_features(series, shape):\n    coo_indexes = get_coo_indexes(series.tolist())\n    sparse_df = coo_matrix(\n        (np.ones(len(coo_indexes[0])), (coo_indexes[0], coo_indexes[1])), shape=shape\n    )\n    return sparse_df\n\n\ndef sparse_to_idx(data, pad_idx=-1):\n    indexes = data.nonzero()\n    indexes_df = pd.DataFrame()\n    indexes_df[\"rows\"] = indexes[0]\n    indexes_df[\"cols\"] = indexes[1]\n    mdf = indexes_df.groupby(\"rows\").apply(lambda x: x[\"cols\"].tolist())\n    max_len = mdf.apply(lambda x: len(x)).max()\n    return mdf.apply(lambda x: pd.Series(x + [pad_idx] * (max_len - len(x)))).values\n</pre> def get_coo_indexes(lil):     rows = []     cols = []     for i, el in enumerate(lil):         if type(el) != list:             el = [el]         for j in el:             rows.append(i)             cols.append(j)     return rows, cols   def get_sparse_features(series, shape):     coo_indexes = get_coo_indexes(series.tolist())     sparse_df = coo_matrix(         (np.ones(len(coo_indexes[0])), (coo_indexes[0], coo_indexes[1])), shape=shape     )     return sparse_df   def sparse_to_idx(data, pad_idx=-1):     indexes = data.nonzero()     indexes_df = pd.DataFrame()     indexes_df[\"rows\"] = indexes[0]     indexes_df[\"cols\"] = indexes[1]     mdf = indexes_df.groupby(\"rows\").apply(lambda x: x[\"cols\"].tolist())     max_len = mdf.apply(lambda x: len(x)).max()     return mdf.apply(lambda x: pd.Series(x + [pad_idx] * (max_len - len(x)))).values <p>For the time being, we will not use a validation set for hyperparameter optimization, and we will simply concatenate the validation and the test set in one test set. I simply splitted the data into train/valid/test in case the reader wants to actually do hyperparameter optimization (and because I know in the future I will).</p> <p>There is also another caveat worth mentioning, related to the indexing of the movies. To build the matrices of movies watched, we use the entire dataset. A more realistic (and correct) approach would be to use ONLY the movies that appear in the training set and consider <code>unknown</code> or <code>unseen</code> those in the testing set that have not been seen during training. Nonetheless, this will not affect the purposes of this notebook, which is to illustrate how one could use <code>pytorch-widedeep</code> to build a recommendation algorithm. However, if one wanted to explore the performance of different algorithms in a \"proper\" way, these \"details\" need to be accounted for.</p> In\u00a0[20]: Copied! <pre>df_test = pd.concat([df_valid, df_test], ignore_index=True)\n</pre> df_test = pd.concat([df_valid, df_test], ignore_index=True) In\u00a0[21]: Copied! <pre>id_cols = [\"user_id\", \"movie_id\"]\nmax_movie_index = max(df_train.movie_id.max(), df_test.movie_id.max())\n</pre> id_cols = [\"user_id\", \"movie_id\"] max_movie_index = max(df_train.movie_id.max(), df_test.movie_id.max()) In\u00a0[22]: Copied! <pre>X_train = df_train.drop(id_cols + [\"rating\", \"prev_movies\", \"target\"], axis=1)\ny_train = np.array(df_train.target.values, dtype=\"int64\")\ntrain_movies_watched = get_sparse_features(\n    df_train[\"prev_movies\"], (len(df_train), max_movie_index + 1)\n)\n\nX_test = df_test.drop(id_cols + [\"rating\", \"prev_movies\", \"target\"], axis=1)\ny_test = np.array(df_test.target.values, dtype=\"int64\")\ntest_movies_watched = get_sparse_features(\n    df_test[\"prev_movies\"], (len(df_test), max_movie_index + 1)\n)\n</pre> X_train = df_train.drop(id_cols + [\"rating\", \"prev_movies\", \"target\"], axis=1) y_train = np.array(df_train.target.values, dtype=\"int64\") train_movies_watched = get_sparse_features(     df_train[\"prev_movies\"], (len(df_train), max_movie_index + 1) )  X_test = df_test.drop(id_cols + [\"rating\", \"prev_movies\", \"target\"], axis=1) y_test = np.array(df_test.target.values, dtype=\"int64\") test_movies_watched = get_sparse_features(     df_test[\"prev_movies\"], (len(df_test), max_movie_index + 1) ) <p>let's have a look to the information in each dataset</p> In\u00a0[23]: Copied! <pre>X_train.head()\n</pre> X_train.head() Out[23]: mean_rate unknown Action Adventure Animation Children's Comedy Crime Documentary Drama ... Mystery_rate Romance_rate Sci-Fi_rate Thriller_rate War_rate Western_rate age gender occupation zip_code 25423 4.000000 0.0 0.400000 0.200000 0.0 0.0 0.400000 0.0 0.0 0.200000 ... NaN 4.0 4.0 4.000000 4.0 NaN 21 M student 48823 25425 4.000000 0.0 0.285714 0.142857 0.0 0.0 0.428571 0.0 0.0 0.285714 ... NaN 4.0 4.0 4.000000 4.0 NaN 21 M student 48823 25424 4.000000 0.0 0.333333 0.166667 0.0 0.0 0.333333 0.0 0.0 0.333333 ... NaN 4.0 4.0 4.000000 4.0 NaN 21 M student 48823 25426 3.875000 0.0 0.250000 0.125000 0.0 0.0 0.375000 0.0 0.0 0.250000 ... NaN 4.0 4.0 3.666667 4.0 NaN 21 M student 48823 25427 3.888889 0.0 0.222222 0.111111 0.0 0.0 0.333333 0.0 0.0 0.333333 ... NaN 4.0 4.0 3.666667 4.0 NaN 21 M student 48823 <p>5 rows \u00d7 43 columns</p> In\u00a0[24]: Copied! <pre>y_train\n</pre> y_train Out[24]: <pre>array([772, 288, 108, ..., 183, 432, 509])</pre> In\u00a0[25]: Copied! <pre>train_movies_watched\n</pre> train_movies_watched Out[25]: <pre>&lt;76228x1683 sparse matrix of type '&lt;class 'numpy.float64'&gt;'\n\twith 7957390 stored elements in COOrdinate format&gt;</pre> In\u00a0[26]: Copied! <pre>sorted(df_train.prev_movies.tolist()[0])\n</pre> sorted(df_train.prev_movies.tolist()[0]) Out[26]: <pre>['173', '185', '255', '286', '298']</pre> In\u00a0[27]: Copied! <pre>np.where(train_movies_watched.todense()[0])\n</pre> np.where(train_movies_watched.todense()[0]) Out[27]: <pre>(array([0, 0, 0, 0, 0]), array([173, 185, 255, 286, 298]))</pre> <p>And from now on is when the specifics related to this library start to appear. The only component that is going to be a bit different is the so-called tabular component, referred as <code>continuous</code> in the notebook.</p> <p>In the case of <code>pytorch-widedeep</code> we have the <code>TabPreprocessor</code> that allows for a lot of flexibility as to how we would like to process the tabular component of this Wide and Deep model. In other words, here our tabular component is a bit more elaborated than that in the notebook, just a bit...</p> In\u00a0[28]: Copied! <pre>cat_cols = [\"gender\", \"occupation\", \"zip_code\"]\ncont_cols = [c for c in X_train if c not in cat_cols]\ntab_preprocessor = TabPreprocessor(\n    cat_embed_cols=cat_cols,\n    continuous_cols=cont_cols,\n)\n</pre> cat_cols = [\"gender\", \"occupation\", \"zip_code\"] cont_cols = [c for c in X_train if c not in cat_cols] tab_preprocessor = TabPreprocessor(     cat_embed_cols=cat_cols,     continuous_cols=cont_cols, ) In\u00a0[29]: Copied! <pre>X_train_tab = tab_preprocessor.fit_transform(X_train.fillna(0))\nX_test_tab = tab_preprocessor.transform(X_test.fillna(0))\n</pre> X_train_tab = tab_preprocessor.fit_transform(X_train.fillna(0)) X_test_tab = tab_preprocessor.transform(X_test.fillna(0)) <p>Now, in the notebook, the author moves the sparse matrices to sparse tensors and then turns them into dense tensors. In reality, this is not neccessary, one could feed sparse tensors to <code>nn.Linear</code> layers in pytorch. Nonetheless, this is not the most efficient implementation and is the reason why in our library the wide, linear component is implemented as an embedding layer.</p> <p>Nonetheless, to reproduce the notebook the best we can and because currently the <code>Wide</code> model in <code>pytorch-widedeep</code> is not designed to receive sparse tensors (we might consider implementing this functionality), we will turn the sparse COO matrices into dense arrays. We will then code a fairly simple, custom <code>Wide</code> component.</p> In\u00a0[30]: Copied! <pre>X_train_wide = np.array(train_movies_watched.todense())\nX_test_wide = np.array(test_movies_watched.todense())\n</pre> X_train_wide = np.array(train_movies_watched.todense()) X_test_wide = np.array(test_movies_watched.todense()) <p>Finally, the author of the notebook uses a simple <code>Embedding</code> layer to encode the sequences of movies watched, the <code>prev_movies</code> columns. In my opinion, there is an element of information redundancy here. This is because the wide and text components have implicitely the same information, but in different form. Moreover, both of the models used for these two components ignore the sequential element in the data. Nonetheless, we want to reproduce the Kaggle notebook as close as possible, AND as one can explore later (by simply performing simple ablation studies), the wide component seems to carry most of the predictive power.</p> In\u00a0[31]: Copied! <pre>X_train_text = sparse_to_idx(train_movies_watched, pad_idx=PAD_IDX)\nX_test_text = sparse_to_idx(test_movies_watched, pad_idx=PAD_IDX)\n</pre> X_train_text = sparse_to_idx(train_movies_watched, pad_idx=PAD_IDX) X_test_text = sparse_to_idx(test_movies_watched, pad_idx=PAD_IDX) <p>Let's now build the models</p> In\u00a0[32]: Copied! <pre>class Wide(nn.Module):\n    def __init__(self, input_dim: int, pred_dim: int):\n        super().__init__()\n\n        self.input_dim = input_dim\n        self.pred_dim = pred_dim\n\n        # When I coded the library I never though that someone would want to code\n        # their own wide component. However, if you do, the wide component must have\n        # a 'wide_linear' attribute. In other words, the linear layer must be\n        # called 'wide_linear'\n        self.wide_linear = nn.Linear(input_dim, pred_dim)\n\n    def forward(self, X):\n        out = self.wide_linear(X.type(torch.float32))\n        return out\n\n\nwide = Wide(X_train_wide.shape[1], max_movie_index + 1)\n</pre> class Wide(nn.Module):     def __init__(self, input_dim: int, pred_dim: int):         super().__init__()          self.input_dim = input_dim         self.pred_dim = pred_dim          # When I coded the library I never though that someone would want to code         # their own wide component. However, if you do, the wide component must have         # a 'wide_linear' attribute. In other words, the linear layer must be         # called 'wide_linear'         self.wide_linear = nn.Linear(input_dim, pred_dim)      def forward(self, X):         out = self.wide_linear(X.type(torch.float32))         return out   wide = Wide(X_train_wide.shape[1], max_movie_index + 1) In\u00a0[33]: Copied! <pre>wide\n</pre> wide Out[33]: <pre>Wide(\n  (wide_linear): Linear(in_features=1683, out_features=1683, bias=True)\n)</pre> In\u00a0[34]: Copied! <pre>class SimpleEmbed(nn.Module):\n    def __init__(self, vocab_size: int, embed_dim: int, pad_idx: int):\n        super().__init__()\n\n        self.vocab_size = vocab_size\n        self.embed_dim = embed_dim\n        self.pad_idx = pad_idx\n\n        # The sequences of movies watched are simply embedded in the Kaggle\n        # notebook. No RNN, Transformer or any model is used\n        self.embed = nn.Embedding(vocab_size, embed_dim, padding_idx=pad_idx)\n\n    def forward(self, X):\n        embed = self.embed(X)\n        embed_mean = torch.mean(embed, dim=1)\n        return embed_mean\n\n    @property\n    def output_dim(self) -&gt; int:\n        # All deep components in a custom 'pytorch-widedeep' model must have\n        # an output_dim property\n        return self.embed_dim\n\n\n#  In the notebook the author uses simply embeddings\nsimple_embed = SimpleEmbed(max_movie_index + 1, 16, 0)\n</pre> class SimpleEmbed(nn.Module):     def __init__(self, vocab_size: int, embed_dim: int, pad_idx: int):         super().__init__()          self.vocab_size = vocab_size         self.embed_dim = embed_dim         self.pad_idx = pad_idx          # The sequences of movies watched are simply embedded in the Kaggle         # notebook. No RNN, Transformer or any model is used         self.embed = nn.Embedding(vocab_size, embed_dim, padding_idx=pad_idx)      def forward(self, X):         embed = self.embed(X)         embed_mean = torch.mean(embed, dim=1)         return embed_mean      @property     def output_dim(self) -&gt; int:         # All deep components in a custom 'pytorch-widedeep' model must have         # an output_dim property         return self.embed_dim   #  In the notebook the author uses simply embeddings simple_embed = SimpleEmbed(max_movie_index + 1, 16, 0) In\u00a0[35]: Copied! <pre>simple_embed\n</pre> simple_embed Out[35]: <pre>SimpleEmbed(\n  (embed): Embedding(1683, 16, padding_idx=0)\n)</pre> <p>Maybe one would like to use an RNN to account for the sequence nature of the problem. If that was the case it would be as easy as:</p> In\u00a0[36]: Copied! <pre>basic_rnn = BasicRNN(\n    vocab_size=max_movie_index + 1,\n    embed_dim=16,\n    hidden_dim=32,\n    n_layers=2,\n    rnn_type=\"gru\",\n)\n</pre> basic_rnn = BasicRNN(     vocab_size=max_movie_index + 1,     embed_dim=16,     hidden_dim=32,     n_layers=2,     rnn_type=\"gru\", ) <p>And finally, the tabular component, which is the notebook is simply a stak of linear + Rely layers. In our case we have an embedding layer before the linear layers to encode categorial and numerical cols</p> In\u00a0[37]: Copied! <pre>tab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=tab_preprocessor.continuous_cols,\n    cont_norm_layer=None,\n    mlp_hidden_dims=[1024, 512, 256],\n    mlp_activation=\"relu\",\n)\n</pre> tab_mlp = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     continuous_cols=tab_preprocessor.continuous_cols,     cont_norm_layer=None,     mlp_hidden_dims=[1024, 512, 256],     mlp_activation=\"relu\", ) In\u00a0[38]: Copied! <pre>tab_mlp\n</pre> tab_mlp Out[38]: <pre>TabMlp(\n  (cat_embed): DiffSizeCatEmbeddings(\n    (embed_layers): ModuleDict(\n      (emb_layer_gender): Embedding(3, 2, padding_idx=0)\n      (emb_layer_occupation): Embedding(22, 9, padding_idx=0)\n      (emb_layer_zip_code): Embedding(648, 60, padding_idx=0)\n    )\n    (embedding_dropout): Dropout(p=0.0, inplace=False)\n  )\n  (cont_norm): Identity()\n  (encoder): MLP(\n    (mlp): Sequential(\n      (dense_layer_0): Sequential(\n        (0): Linear(in_features=111, out_features=1024, bias=True)\n        (1): ReLU(inplace=True)\n        (2): Dropout(p=0.1, inplace=False)\n      )\n      (dense_layer_1): Sequential(\n        (0): Linear(in_features=1024, out_features=512, bias=True)\n        (1): ReLU(inplace=True)\n        (2): Dropout(p=0.1, inplace=False)\n      )\n      (dense_layer_2): Sequential(\n        (0): Linear(in_features=512, out_features=256, bias=True)\n        (1): ReLU(inplace=True)\n        (2): Dropout(p=0.1, inplace=False)\n      )\n    )\n  )\n)</pre> <p>Finally, we simply wrap up all models with the <code>WideDeep</code> 'collector' class and we are ready to train.</p> In\u00a0[39]: Copied! <pre>wide_deep_model = WideDeep(\n    wide=wide, deeptabular=tab_mlp, deeptext=simple_embed, pred_dim=max_movie_index + 1\n)\n</pre> wide_deep_model = WideDeep(     wide=wide, deeptabular=tab_mlp, deeptext=simple_embed, pred_dim=max_movie_index + 1 ) In\u00a0[40]: Copied! <pre>wide_deep_model\n</pre> wide_deep_model Out[40]: <pre>WideDeep(\n  (wide): Wide(\n    (wide_linear): Linear(in_features=1683, out_features=1683, bias=True)\n  )\n  (deeptabular): Sequential(\n    (0): TabMlp(\n      (cat_embed): DiffSizeCatEmbeddings(\n        (embed_layers): ModuleDict(\n          (emb_layer_gender): Embedding(3, 2, padding_idx=0)\n          (emb_layer_occupation): Embedding(22, 9, padding_idx=0)\n          (emb_layer_zip_code): Embedding(648, 60, padding_idx=0)\n        )\n        (embedding_dropout): Dropout(p=0.0, inplace=False)\n      )\n      (cont_norm): Identity()\n      (encoder): MLP(\n        (mlp): Sequential(\n          (dense_layer_0): Sequential(\n            (0): Linear(in_features=111, out_features=1024, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_1): Sequential(\n            (0): Linear(in_features=1024, out_features=512, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_2): Sequential(\n            (0): Linear(in_features=512, out_features=256, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=256, out_features=1683, bias=True)\n  )\n  (deeptext): Sequential(\n    (0): SimpleEmbed(\n      (embed): Embedding(1683, 16, padding_idx=0)\n    )\n    (1): Linear(in_features=16, out_features=1683, bias=True)\n  )\n)</pre> <p>Note that the main difference between this wide and deep model and the Wide and Deep model in the Kaggle notebook is that in that notebook, the author concatenates the embedings and the tabular features, then passes this concatenation through a stack of linear + Relu layers with a final output dim of 256. Then concatenates this output with the binary features and connects this concatenation with the final linear layer (so the final weights are of dim (batch_size, 256 + 1683)). Our implementation follows the notation of the original paper and instead of concatenating the tabular, text and wide components and then connect them to the output neurons, we first compute their output, and then add it (see here: https://arxiv.org/pdf/1606.07792.pdf, their Eq 3). Note that this is effectively the same, with the caveat that while in one case one initialises a big weight matrix \"at once\", in our implementation we initialise different matrices for different components. Anyway, let's give it a go.</p> In\u00a0[41]: Copied! <pre>trainer = Trainer(\n    model=wide_deep_model,\n    objective=\"multiclass\",\n    custom_loss_function=nn.CrossEntropyLoss(ignore_index=PAD_IDX),\n    optimizers=torch.optim.Adam(wide_deep_model.parameters(), lr=1e-3),\n)\n</pre> trainer = Trainer(     model=wide_deep_model,     objective=\"multiclass\",     custom_loss_function=nn.CrossEntropyLoss(ignore_index=PAD_IDX),     optimizers=torch.optim.Adam(wide_deep_model.parameters(), lr=1e-3), ) In\u00a0[42]: Copied! <pre>trainer.fit(\n    X_train={\n        \"X_wide\": X_train_wide,\n        \"X_tab\": X_train_tab,\n        \"X_text\": X_train_text,\n        \"target\": y_train,\n    },\n    X_val={\n        \"X_wide\": X_test_wide,\n        \"X_tab\": X_test_tab,\n        \"X_text\": X_test_text,\n        \"target\": y_test,\n    },\n    n_epochs=5,\n    batch_size=512,\n    shuffle=False,\n)\n</pre> trainer.fit(     X_train={         \"X_wide\": X_train_wide,         \"X_tab\": X_train_tab,         \"X_text\": X_train_text,         \"target\": y_train,     },     X_val={         \"X_wide\": X_test_wide,         \"X_tab\": X_test_tab,         \"X_text\": X_test_text,         \"target\": y_test,     },     n_epochs=5,     batch_size=512,     shuffle=False, ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 149/149 [00:19&lt;00:00,  7.66it/s, loss=6.66]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 38/38 [00:02&lt;00:00, 18.75it/s, loss=6.6]\nepoch 2: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 149/149 [00:21&lt;00:00,  6.95it/s, loss=5.97]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 38/38 [00:01&lt;00:00, 21.03it/s, loss=6.52]\nepoch 3: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 149/149 [00:19&lt;00:00,  7.51it/s, loss=5.65]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 38/38 [00:01&lt;00:00, 20.16it/s, loss=6.53]\nepoch 4: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 149/149 [00:23&lt;00:00,  6.29it/s, loss=5.41]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 38/38 [00:02&lt;00:00, 13.97it/s, loss=6.57]\nepoch 5: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 149/149 [00:19&lt;00:00,  7.58it/s, loss=5.2]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 38/38 [00:02&lt;00:00, 18.82it/s, loss=6.63]\n</pre> <p>Now one could continue to the 'compare' metrics section of the Kaggle notebook. However, for the purposes of illustrating how one could use <code>pytorch-widedeep</code> to build recommendation algorithms we consider this notebook completed and move onto part 2</p>"},{"location":"examples/18_wide_and_deep_for_recsys_pt1.html#problem-formulation","title":"Problem formulation\u00b6","text":"<p>In this particular exercise the problem is formulated as predicting the next movie that will be watched (in consequence the last interactions will be discarded)</p>"},{"location":"examples/18_wide_and_deep_for_recsys_pt2.html","title":"18_wide_and_deep_for_recsys_pt2","text":"<p>This is the second of the two notebooks where we aim to illustrate how one could use this library to build recommendation algorithms using the example in this Kaggle notebook as guidance. In the previous notebook we used <code>pytorch-widedeep</code> to build a model that replicated almost exactly that in the notebook. In this, shorter notebook we will show how one could use the library to explore other models, following the same problem formulation, this is: given a state of a user at a certain point in time having watched a series of movies, our goal is to predict which movie the user will watch next.</p> <p>Assuming that one has read (and run) the previous notebook, the required data will be stored in a local dir called <code>prepared_data</code>, so let's read it:</p> In\u00a0[1]: Copied! <pre>from pathlib import Path\n\nimport numpy as np\nimport torch\nimport pandas as pd\nfrom torch import nn\n\nfrom pytorch_widedeep import Trainer\nfrom pytorch_widedeep.utils import pad_sequences\nfrom pytorch_widedeep.models import TabMlp, WideDeep, Transformer\nfrom pytorch_widedeep.preprocessing import TabPreprocessor\n</pre> from pathlib import Path  import numpy as np import torch import pandas as pd from torch import nn  from pytorch_widedeep import Trainer from pytorch_widedeep.utils import pad_sequences from pytorch_widedeep.models import TabMlp, WideDeep, Transformer from pytorch_widedeep.preprocessing import TabPreprocessor In\u00a0[2]: Copied! <pre>save_path = Path(\"prepared_data\")\n\nPAD_IDX = 0\n\nid_cols = [\"user_id\", \"movie_id\"]\n\ndf_train = pd.read_pickle(save_path / \"df_train.pkl\")\ndf_valid = pd.read_pickle(save_path / \"df_valid.pkl\")\ndf_test = pd.read_pickle(save_path / \"df_test.pkl\")\n</pre> save_path = Path(\"prepared_data\")  PAD_IDX = 0  id_cols = [\"user_id\", \"movie_id\"]  df_train = pd.read_pickle(save_path / \"df_train.pkl\") df_valid = pd.read_pickle(save_path / \"df_valid.pkl\") df_test = pd.read_pickle(save_path / \"df_test.pkl\") <p>...remember that in the previous notebook we explained that we are not  going to use a validation set here (in a real-world example, or simply a more realistic example, one should always use it).</p> In\u00a0[3]: Copied! <pre>df_test = pd.concat([df_valid, df_test], ignore_index=True)\n</pre> df_test = pd.concat([df_valid, df_test], ignore_index=True) <p>Also remember that, in the previous notebook we discussed that the <code>'maxlen'</code> and <code>'max_movie_index'</code> parameters should be computed using only the train set. In particular, to properly do the tokenization, one would have to use ONLY train tokens and add a token for new 'unknown'/'unseen' movies in the test set. This can also be done with this library or manually, so I will leave it to the reader to implement that tokenzation appraoch.</p> In\u00a0[4]: Copied! <pre>maxlen = max(\n    df_train.prev_movies.apply(lambda x: len(x)).max(),\n    df_test.prev_movies.apply(lambda x: len(x)).max(),\n)\n\nmax_movie_index = max(df_train.movie_id.max(), df_test.movie_id.max())\n</pre> maxlen = max(     df_train.prev_movies.apply(lambda x: len(x)).max(),     df_test.prev_movies.apply(lambda x: len(x)).max(), )  max_movie_index = max(df_train.movie_id.max(), df_test.movie_id.max()) <p>From now one things are pretty simple, moreover bearing in mind that in this example we are not going to use a wide component since, in pple, one would believe that the information in that component is also 'carried' by the movie sequences (However in the previous notebook, if one performs ablation studies, these suggest that most of the prediction power comes from the linear, wide model).</p> <p>In the example here we are going to explore one (of many) possibilities. We are simply going to encode the triplet <code>(user, item, rating)</code> and use it as a <code>deeptabular</code> component and the sequences of previously watched movies as the <code>deeptext</code> component. For the <code>deeptext</code> component we are going to use a basic encoder-only transformer model.</p> <p>Let's start with the tabular data preparation</p> In\u00a0[5]: Copied! <pre>df_train_user_item = df_train[[\"user_id\", \"movie_id\", \"rating\"]]\ntrain_movies_sequences = df_train.prev_movies.apply(\n    lambda x: [int(el) for el in x]\n).to_list()\ny_train = df_train.target.values.astype(int)\n\ndf_test_user_item = df_train[[\"user_id\", \"movie_id\", \"rating\"]]\ntest_movies_sequences = df_test.prev_movies.apply(\n    lambda x: [int(el) for el in x]\n).to_list()\ny_test = df_test.target.values.astype(int)\n\ntab_preprocessor = tab_preprocessor = TabPreprocessor(\n    cat_embed_cols=[\"user_id\", \"movie_id\", \"rating\"],\n)\nX_train_tab = tab_preprocessor.fit_transform(df_train_user_item)\nX_test_tab = tab_preprocessor.transform(df_test_user_item)\n</pre> df_train_user_item = df_train[[\"user_id\", \"movie_id\", \"rating\"]] train_movies_sequences = df_train.prev_movies.apply(     lambda x: [int(el) for el in x] ).to_list() y_train = df_train.target.values.astype(int)  df_test_user_item = df_train[[\"user_id\", \"movie_id\", \"rating\"]] test_movies_sequences = df_test.prev_movies.apply(     lambda x: [int(el) for el in x] ).to_list() y_test = df_test.target.values.astype(int)  tab_preprocessor = tab_preprocessor = TabPreprocessor(     cat_embed_cols=[\"user_id\", \"movie_id\", \"rating\"], ) X_train_tab = tab_preprocessor.fit_transform(df_train_user_item) X_test_tab = tab_preprocessor.transform(df_test_user_item) <p>And not the text component, simply padding the sequences:</p> In\u00a0[6]: Copied! <pre>X_train_text = np.array(\n    [\n        pad_sequences(\n            s,\n            maxlen=maxlen,\n            pad_first=False,\n            pad_idx=PAD_IDX,\n        )\n        for s in train_movies_sequences\n    ]\n)\nX_test_text = np.array(\n    [\n        pad_sequences(\n            s,\n            maxlen=maxlen,\n            pad_first=False,\n            pad_idx=0,\n        )\n        for s in test_movies_sequences\n    ]\n)\n</pre> X_train_text = np.array(     [         pad_sequences(             s,             maxlen=maxlen,             pad_first=False,             pad_idx=PAD_IDX,         )         for s in train_movies_sequences     ] ) X_test_text = np.array(     [         pad_sequences(             s,             maxlen=maxlen,             pad_first=False,             pad_idx=0,         )         for s in test_movies_sequences     ] ) <p>We now define the model components and the wide and deep model.</p> In\u00a0[7]: Copied! <pre>tab_mlp = TabMlp(\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    mlp_hidden_dims=[1024, 512, 256],\n    mlp_activation=\"relu\",\n)\n\n# plenty of options here, see the docs\ntransformer = Transformer(\n    vocab_size=max_movie_index + 1,\n    embed_dim=32,\n    n_heads=2,\n    n_blocks=2,\n    seq_length=maxlen,\n)\n\nwide_deep_model = WideDeep(\n    deeptabular=tab_mlp, deeptext=transformer, pred_dim=max_movie_index + 1\n)\n</pre> tab_mlp = TabMlp(     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     mlp_hidden_dims=[1024, 512, 256],     mlp_activation=\"relu\", )  # plenty of options here, see the docs transformer = Transformer(     vocab_size=max_movie_index + 1,     embed_dim=32,     n_heads=2,     n_blocks=2,     seq_length=maxlen, )  wide_deep_model = WideDeep(     deeptabular=tab_mlp, deeptext=transformer, pred_dim=max_movie_index + 1 ) In\u00a0[8]: Copied! <pre>wide_deep_model\n</pre> wide_deep_model Out[8]: <pre>WideDeep(\n  (deeptabular): Sequential(\n    (0): TabMlp(\n      (cat_embed): DiffSizeCatEmbeddings(\n        (embed_layers): ModuleDict(\n          (emb_layer_user_id): Embedding(749, 65, padding_idx=0)\n          (emb_layer_movie_id): Embedding(1612, 100, padding_idx=0)\n          (emb_layer_rating): Embedding(6, 4, padding_idx=0)\n        )\n        (embedding_dropout): Dropout(p=0.0, inplace=False)\n      )\n      (encoder): MLP(\n        (mlp): Sequential(\n          (dense_layer_0): Sequential(\n            (0): Linear(in_features=169, out_features=1024, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_1): Sequential(\n            (0): Linear(in_features=1024, out_features=512, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_2): Sequential(\n            (0): Linear(in_features=512, out_features=256, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=256, out_features=1683, bias=True)\n  )\n  (deeptext): Sequential(\n    (0): Transformer(\n      (embedding): Embedding(1683, 32, padding_idx=0)\n      (pos_encoder): PositionalEncoding(\n        (dropout): Dropout(p=0.1, inplace=False)\n      )\n      (encoder): Sequential(\n        (transformer_block0): TransformerEncoder(\n          (attn): MultiHeadedAttention(\n            (dropout): Dropout(p=0.1, inplace=False)\n            (q_proj): Linear(in_features=32, out_features=32, bias=False)\n            (kv_proj): Linear(in_features=32, out_features=64, bias=False)\n            (out_proj): Linear(in_features=32, out_features=32, bias=False)\n          )\n          (ff): FeedForward(\n            (w_1): Linear(in_features=32, out_features=128, bias=True)\n            (w_2): Linear(in_features=128, out_features=32, bias=True)\n            (dropout): Dropout(p=0.1, inplace=False)\n            (activation): GELU(approximate='none')\n          )\n          (attn_addnorm): AddNorm(\n            (dropout): Dropout(p=0.1, inplace=False)\n            (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n          )\n          (ff_addnorm): AddNorm(\n            (dropout): Dropout(p=0.1, inplace=False)\n            (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n          )\n        )\n        (transformer_block1): TransformerEncoder(\n          (attn): MultiHeadedAttention(\n            (dropout): Dropout(p=0.1, inplace=False)\n            (q_proj): Linear(in_features=32, out_features=32, bias=False)\n            (kv_proj): Linear(in_features=32, out_features=64, bias=False)\n            (out_proj): Linear(in_features=32, out_features=32, bias=False)\n          )\n          (ff): FeedForward(\n            (w_1): Linear(in_features=32, out_features=128, bias=True)\n            (w_2): Linear(in_features=128, out_features=32, bias=True)\n            (dropout): Dropout(p=0.1, inplace=False)\n            (activation): GELU(approximate='none')\n          )\n          (attn_addnorm): AddNorm(\n            (dropout): Dropout(p=0.1, inplace=False)\n            (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n          )\n          (ff_addnorm): AddNorm(\n            (dropout): Dropout(p=0.1, inplace=False)\n            (ln): LayerNorm((32,), eps=1e-05, elementwise_affine=True)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=23552, out_features=1683, bias=True)\n  )\n)</pre> <p>And as in the previous notebook, let's train (you will need a GPU for this)</p> In\u00a0[\u00a0]: Copied! <pre>trainer = Trainer(\n    model=wide_deep_model,\n    objective=\"multiclass\",\n    custom_loss_function=nn.CrossEntropyLoss(ignore_index=PAD_IDX),\n    optimizers=torch.optim.Adam(wide_deep_model.parameters(), lr=1e-3),\n)\n\ntrainer.fit(\n    X_train={\n        \"X_tab\": X_train_tab,\n        \"X_text\": X_train_text,\n        \"target\": y_train,\n    },\n    X_val={\n        \"X_tab\": X_test_tab,\n        \"X_text\": X_test_text,\n        \"target\": y_test,\n    },\n    n_epochs=10,\n    batch_size=521,\n    shuffle=False,\n)\n</pre> trainer = Trainer(     model=wide_deep_model,     objective=\"multiclass\",     custom_loss_function=nn.CrossEntropyLoss(ignore_index=PAD_IDX),     optimizers=torch.optim.Adam(wide_deep_model.parameters(), lr=1e-3), )  trainer.fit(     X_train={         \"X_tab\": X_train_tab,         \"X_text\": X_train_text,         \"target\": y_train,     },     X_val={         \"X_tab\": X_test_tab,         \"X_text\": X_test_text,         \"target\": y_test,     },     n_epochs=10,     batch_size=521,     shuffle=False, ) <pre>epoch 1:   0%|                                                                                                         | 0/147 [00:34&lt;?, ?it/s]\n</pre> In\u00a0[\u00a0]: Copied! <pre>\n</pre>"},{"location":"examples/19_load_from_folder_functionality.html","title":"19_load_from_folder_functionality","text":"<p>In this notebook I want to illustrate how one can use our <code>[...]FromFolder</code> functionalities along with the <code>[...]ChunkPreProcessors</code> in those cases where the dataset is too bit to fit in memory.</p> <p>These functionalities in the library have been designed for the following scenarop</p> In\u00a0[1]: Copied! <pre>import numpy as np\nimport torch\nimport pandas as pd\nfrom torch.utils.data import DataLoader\n\nfrom pytorch_widedeep.models import TabMlp, Vision, BasicRNN, WideDeep\nfrom pytorch_widedeep.training import TrainerFromFolder\nfrom pytorch_widedeep.callbacks import EarlyStopping, ModelCheckpoint\nfrom pytorch_widedeep.preprocessing import (\n    TabPreprocessor,\n    TextPreprocessor,\n    ImagePreprocessor,\n    ChunkTabPreprocessor,\n    ChunkTextPreprocessor,\n)\nfrom pytorch_widedeep.load_from_folder import (\n    TabFromFolder,\n    TextFromFolder,\n    ImageFromFolder,\n    WideDeepDatasetFromFolder,\n)\n</pre> import numpy as np import torch import pandas as pd from torch.utils.data import DataLoader  from pytorch_widedeep.models import TabMlp, Vision, BasicRNN, WideDeep from pytorch_widedeep.training import TrainerFromFolder from pytorch_widedeep.callbacks import EarlyStopping, ModelCheckpoint from pytorch_widedeep.preprocessing import (     TabPreprocessor,     TextPreprocessor,     ImagePreprocessor,     ChunkTabPreprocessor,     ChunkTextPreprocessor, ) from pytorch_widedeep.load_from_folder import (     TabFromFolder,     TextFromFolder,     ImageFromFolder,     WideDeepDatasetFromFolder, ) In\u00a0[2]: Copied! <pre># in my case, I place the data in a folder I call tmp_data, let's see how it looks\nairbnb_data = pd.read_csv(\"../tmp_data/airbnb/airbnb_sample.csv\")\nairbnb_data.sample(5)\n</pre> # in my case, I place the data in a folder I call tmp_data, let's see how it looks airbnb_data = pd.read_csv(\"../tmp_data/airbnb/airbnb_sample.csv\") airbnb_data.sample(5) Out[2]: id host_id description host_listings_count host_identity_verified neighbourhood_cleansed latitude longitude is_location_exact property_type ... amenity_wide_entrance amenity_wide_entrance_for_guests amenity_wide_entryway amenity_wide_hallways amenity_wifi amenity_window_guards amenity_wine_cooler security_deposit extra_people yield 39 53242.jpg 247650 A lovely big bright bedroom in a 2 bedroom fla... 2.0 t Lambeth 51.47075 -0.12913 t apartment ... 0 0 0 0 1 0 0 250.0 5.0 9.75 214 236716.jpg 1241070 We offer a warm welcome in our quiet double ro... 1.0 t Hackney 51.56593 -0.07482 t other ... 0 0 0 0 1 0 0 200.0 10.0 76.50 400 346523.jpg 1756532 Available for you to rent is a cozy studio in ... 2.0 t Kensington and Chelsea 51.48311 -0.18428 t other ... 0 0 0 0 1 0 0 0.0 50.0 180.90 512 389627.jpg 1949299 This gorgeous studio flat is situated in the v... 1.0 t Westminster 51.51838 -0.14238 f apartment ... 0 0 0 0 1 0 0 250.0 25.0 276.90 504 388767.jpg 1945165 If you want to experience London at it's best ... 2.0 f Camden 51.54293 -0.14073 t apartment ... 0 0 0 0 1 0 0 150.0 10.0 591.10 <p>5 rows \u00d7 223 columns</p> In\u00a0[3]: Copied! <pre># for example\nfrom IPython.display import Image\n\npil_img = Image(filename=\"../tmp_data/airbnb/property_picture/272908.jpg\")\ndisplay(pil_img)\n</pre> # for example from IPython.display import Image  pil_img = Image(filename=\"../tmp_data/airbnb/property_picture/272908.jpg\") display(pil_img) In\u00a0[4]: Copied! <pre># And the description for the property that that picture belongs to is:\nairbnb_data[airbnb_data.id == \"272908.jpg\"].description.tolist()\n</pre> # And the description for the property that that picture belongs to is: airbnb_data[airbnb_data.id == \"272908.jpg\"].description.tolist() Out[4]: <pre>[\"Bright, sunny beautiful room that will give you the perfect base to explore all of London. Come and explore one of London's best neighbourhoods - Herne Hill! As mentioned in (Website hidden by Airbnb)   (Website hidden by Airbnb)  WiFi availability with a fully stocked and clean uplifting home. Lovely sunny, airy and big double bedroom on a leafy south-London street.    Note: This room comes with a reserved Off-Street parking spot! The room is on the first floor and boasts an enormous Super King bed, gorgeous wooden floors, tall ceilings and large windows which let in the sunshine almost all day. (Yoga May or meditation cushion available on request) The flat is bright and airy and big! So lots of space for all.  Location wise you are only 10 minutes walk to either Herne Hill or West Dulwich stations, both of which will take you to Victoria and the city within minutes. You can also hop on a bus right outside the flat that will take you to Brixton tube station within 8 minutes where you \"]</pre> <p>Ok, so we have tabular data where one column is <code>description</code> and another <code>id</code>, points towards the images stored in disk. Now, remember the following, because this will appear a few times in the notebook: our \"reference dataset\" is the tabular data.</p> <p>Therefore, since I want to illustrate a \"semi-realistic\" case, if we need to split the data into training, validation and test datasets, these datasets needs to be separetely stored in disk. In my case I have done this and in the <code>tmp_data/airbnb</code> dir I have the following:</p> <pre>../tmp_data/airbnb\n\u251c\u2500\u2500 airbnb_sample.csv\n\u251c\u2500\u2500 airbnb_sample_eval.csv\n\u251c\u2500\u2500 airbnb_sample_test.csv\n\u251c\u2500\u2500 airbnb_sample_train.csv\n\u2514\u2500\u2500 property_picture\n</pre> <p>Where <code>airbnb_sample.csv</code> is the full sample (1001 rows) and the <code>train</code>, <code>eval</code> and <code>test</code> set is the corresponding splits. In a realistic example, the full sample would be the 'gigantic' dataset and the rest the corresponding splits. One has to do this 'offline', prior to start the coding.</p> <p>Also, one thing that one needs to know is the number of total observations/rows, as well as the splits. In our case the train size is 800, and the eval and test sizes are 100 and 101 respectively.</p> <p>With all that info, let's star</p> In\u00a0[5]: Copied! <pre># path to the tabular data and the splits\ndata_path = \"../tmp_data/airbnb/\"\ntrain_fname = \"airbnb_sample_train.csv\"\neval_fname = \"airbnb_sample_eval.csv\"\ntest_fname = \"airbnb_sample_test.csv\"\n\n# split sizes\ntrain_size = 800\neval_size = 100\ntest_size = 101\n\n# number of chunks for the Chunk Preprocessors\nchunksize = 100\nn_chunks = int(np.ceil(train_size / chunksize))\n\n# path to the image dataset and name of the image col\nimg_path = \"../tmp_data/airbnb/property_picture/\"\nimg_col = \"id\"\n\n# name of the text col\ntext_col = \"description\"\n\n# mane of the target\ntarget_col = \"yield\"\n\n# definition of the categorical and continuous cols for the TabPreprocessor\ncat_embed_cols = [\n    \"host_listings_count\",\n    \"neighbourhood_cleansed\",\n    \"is_location_exact\",\n    \"property_type\",\n    \"room_type\",\n    \"accommodates\",\n    \"bathrooms\",\n    \"bedrooms\",\n    \"beds\",\n    \"guests_included\",\n    \"minimum_nights\",\n    \"instant_bookable\",\n    \"cancellation_policy\",\n    \"has_house_rules\",\n    \"host_gender\",\n    \"accommodates_catg\",\n    \"guests_included_catg\",\n    \"minimum_nights_catg\",\n    \"host_listings_count_catg\",\n    \"bathrooms_catg\",\n    \"bedrooms_catg\",\n    \"beds_catg\",\n    \"security_deposit\",\n    \"extra_people\",\n]\ncont_cols = [\"latitude\", \"longitude\"]\n</pre> # path to the tabular data and the splits data_path = \"../tmp_data/airbnb/\" train_fname = \"airbnb_sample_train.csv\" eval_fname = \"airbnb_sample_eval.csv\" test_fname = \"airbnb_sample_test.csv\"  # split sizes train_size = 800 eval_size = 100 test_size = 101  # number of chunks for the Chunk Preprocessors chunksize = 100 n_chunks = int(np.ceil(train_size / chunksize))  # path to the image dataset and name of the image col img_path = \"../tmp_data/airbnb/property_picture/\" img_col = \"id\"  # name of the text col text_col = \"description\"  # mane of the target target_col = \"yield\"  # definition of the categorical and continuous cols for the TabPreprocessor cat_embed_cols = [     \"host_listings_count\",     \"neighbourhood_cleansed\",     \"is_location_exact\",     \"property_type\",     \"room_type\",     \"accommodates\",     \"bathrooms\",     \"bedrooms\",     \"beds\",     \"guests_included\",     \"minimum_nights\",     \"instant_bookable\",     \"cancellation_policy\",     \"has_house_rules\",     \"host_gender\",     \"accommodates_catg\",     \"guests_included_catg\",     \"minimum_nights_catg\",     \"host_listings_count_catg\",     \"bathrooms_catg\",     \"bedrooms_catg\",     \"beds_catg\",     \"security_deposit\",     \"extra_people\", ] cont_cols = [\"latitude\", \"longitude\"] In\u00a0[6]: Copied! <pre>tab_preprocessor = TabPreprocessor(\n    embed_cols=cat_embed_cols,\n    continuous_cols=cont_cols,\n    default_embed_dim=8,\n    verbose=0,\n)\n\ntext_preprocessor = TextPreprocessor(\n    text_col=text_col,\n    n_cpus=1,\n)\n\nimg_preprocessor = ImagePreprocessor(\n    img_col=img_col,\n    img_path=img_path,\n)\n</pre> tab_preprocessor = TabPreprocessor(     embed_cols=cat_embed_cols,     continuous_cols=cont_cols,     default_embed_dim=8,     verbose=0, )  text_preprocessor = TextPreprocessor(     text_col=text_col,     n_cpus=1, )  img_preprocessor = ImagePreprocessor(     img_col=img_col,     img_path=img_path, ) In\u00a0[7]: Copied! <pre>tab_preprocessor.fit(airbnb_data)\ntext_preprocessor.fit(airbnb_data)\nimg_preprocessor.fit(airbnb_data)\n</pre> tab_preprocessor.fit(airbnb_data) text_preprocessor.fit(airbnb_data) img_preprocessor.fit(airbnb_data) <pre>The vocabulary contains 2192 tokens\n</pre> Out[7]: <pre>ImagePreprocessor(img_col=id, img_path=../tmp_data/airbnb/property_picture/, width=224, height=224, verbose=1)</pre> In\u00a0[8]: Copied! <pre>chunk_tab_preprocessor = ChunkTabPreprocessor(\n    embed_cols=cat_embed_cols,\n    continuous_cols=cont_cols,\n    n_chunks=n_chunks,\n    default_embed_dim=8,\n    verbose=0,\n)\n\nchunk_text_preprocessor = ChunkTextPreprocessor(\n    n_chunks=n_chunks,\n    text_col=text_col,\n    n_cpus=1,\n    verbose=0,\n)\n\nfor i, chunk in enumerate(\n    pd.read_csv(\"/\".join([data_path, train_fname]), chunksize=chunksize)\n):\n    print(f\"chunk in loop: {i + 1}\")\n    chunk_tab_preprocessor.fit(chunk)\n    chunk_text_preprocessor.fit(chunk)\n</pre> chunk_tab_preprocessor = ChunkTabPreprocessor(     embed_cols=cat_embed_cols,     continuous_cols=cont_cols,     n_chunks=n_chunks,     default_embed_dim=8,     verbose=0, )  chunk_text_preprocessor = ChunkTextPreprocessor(     n_chunks=n_chunks,     text_col=text_col,     n_cpus=1,     verbose=0, )  for i, chunk in enumerate(     pd.read_csv(\"/\".join([data_path, train_fname]), chunksize=chunksize) ):     print(f\"chunk in loop: {i + 1}\")     chunk_tab_preprocessor.fit(chunk)     chunk_text_preprocessor.fit(chunk) <pre>chunk in loop: 1\nchunk in loop: 2\nchunk in loop: 3\nchunk in loop: 4\nchunk in loop: 5\nchunk in loop: 6\nchunk in loop: 7\nchunk in loop: 8\n</pre> In\u00a0[9]: Copied! <pre>train_tab_folder = TabFromFolder(\n    fname=train_fname,\n    directory=data_path,\n    target_col=target_col,\n    preprocessor=tab_preprocessor,\n    text_col=text_col,\n    img_col=img_col,\n)\n\n# Note how we can use the `train_tab_folder` as reference so we don't have to\n# define all parameters again\neval_tab_folder = TabFromFolder(fname=eval_fname, reference=train_tab_folder)\n\n# Note that for the test set we can ignore the target as no metric will be\n# computed by the `predict` method\ntest_tab_folder = TabFromFolder(\n    fname=test_fname, reference=train_tab_folder, ignore_target=True\n)\n</pre> train_tab_folder = TabFromFolder(     fname=train_fname,     directory=data_path,     target_col=target_col,     preprocessor=tab_preprocessor,     text_col=text_col,     img_col=img_col, )  # Note how we can use the `train_tab_folder` as reference so we don't have to # define all parameters again eval_tab_folder = TabFromFolder(fname=eval_fname, reference=train_tab_folder)  # Note that for the test set we can ignore the target as no metric will be # computed by the `predict` method test_tab_folder = TabFromFolder(     fname=test_fname, reference=train_tab_folder, ignore_target=True ) In\u00a0[10]: Copied! <pre># for the text and image datasets we do not need to specify eval or test loaders\ntext_folder = TextFromFolder(preprocessor=text_preprocessor)\nimg_folder = ImageFromFolder(preprocessor=img_preprocessor)\n</pre> # for the text and image datasets we do not need to specify eval or test loaders text_folder = TextFromFolder(preprocessor=text_preprocessor) img_folder = ImageFromFolder(preprocessor=img_preprocessor) In\u00a0[11]: Copied! <pre>train_dataset_folder = WideDeepDatasetFromFolder(\n    n_samples=train_size,\n    tab_from_folder=train_tab_folder,\n    text_from_folder=text_folder,\n    img_from_folder=img_folder,\n)\n\n# Note that the eval and test loaders only need their corresponding\n# `TabFromFolder` classes. The rest of the parameters can be defined\n# via a `reference` `TabFromFolder` class\neval_dataset_folder = WideDeepDatasetFromFolder(\n    n_samples=eval_size,\n    tab_from_folder=eval_tab_folder,\n    reference=train_dataset_folder,\n)\n\ntest_dataset_folder = WideDeepDatasetFromFolder(\n    n_samples=test_size,\n    tab_from_folder=test_tab_folder,\n    reference=train_dataset_folder,\n)\n</pre> train_dataset_folder = WideDeepDatasetFromFolder(     n_samples=train_size,     tab_from_folder=train_tab_folder,     text_from_folder=text_folder,     img_from_folder=img_folder, )  # Note that the eval and test loaders only need their corresponding # `TabFromFolder` classes. The rest of the parameters can be defined # via a `reference` `TabFromFolder` class eval_dataset_folder = WideDeepDatasetFromFolder(     n_samples=eval_size,     tab_from_folder=eval_tab_folder,     reference=train_dataset_folder, )  test_dataset_folder = WideDeepDatasetFromFolder(     n_samples=test_size,     tab_from_folder=test_tab_folder,     reference=train_dataset_folder, ) In\u00a0[12]: Copied! <pre>train_loader = DataLoader(train_dataset_folder, batch_size=16, num_workers=1)\neval_loader = DataLoader(eval_dataset_folder, batch_size=16, num_workers=1)\ntest_loader = DataLoader(test_dataset_folder, batch_size=16, num_workers=1)\n</pre> train_loader = DataLoader(train_dataset_folder, batch_size=16, num_workers=1) eval_loader = DataLoader(eval_dataset_folder, batch_size=16, num_workers=1) test_loader = DataLoader(test_dataset_folder, batch_size=16, num_workers=1) <p>And from here on is business as usual:</p> In\u00a0[13]: Copied! <pre># for example\nbasic_rnn = BasicRNN(\n    vocab_size=len(text_preprocessor.vocab.itos),\n    embed_dim=32,\n    hidden_dim=64,\n    n_layers=2,\n)\n\ndeepimage = Vision()\n\ndeepdense = TabMlp(\n    mlp_hidden_dims=[32, 16],\n    column_idx=tab_preprocessor.column_idx,\n    cat_embed_input=tab_preprocessor.cat_embed_input,\n    continuous_cols=cont_cols,\n)\n\nmodel = WideDeep(\n    deeptabular=deepdense,\n    deeptext=basic_rnn,\n    deepimage=deepimage,\n)\n\nmodel\n</pre> # for example basic_rnn = BasicRNN(     vocab_size=len(text_preprocessor.vocab.itos),     embed_dim=32,     hidden_dim=64,     n_layers=2, )  deepimage = Vision()  deepdense = TabMlp(     mlp_hidden_dims=[32, 16],     column_idx=tab_preprocessor.column_idx,     cat_embed_input=tab_preprocessor.cat_embed_input,     continuous_cols=cont_cols, )  model = WideDeep(     deeptabular=deepdense,     deeptext=basic_rnn,     deepimage=deepimage, )  model Out[13]: <pre>WideDeep(\n  (deeptabular): Sequential(\n    (0): TabMlp(\n      (cat_embed): DiffSizeCatEmbeddings(\n        (embed_layers): ModuleDict(\n          (emb_layer_host_listings_count): Embedding(28, 10, padding_idx=0)\n          (emb_layer_neighbourhood_cleansed): Embedding(33, 11, padding_idx=0)\n          (emb_layer_is_location_exact): Embedding(3, 2, padding_idx=0)\n          (emb_layer_property_type): Embedding(4, 3, padding_idx=0)\n          (emb_layer_room_type): Embedding(4, 3, padding_idx=0)\n          (emb_layer_accommodates): Embedding(14, 7, padding_idx=0)\n          (emb_layer_bathrooms): Embedding(11, 6, padding_idx=0)\n          (emb_layer_bedrooms): Embedding(7, 4, padding_idx=0)\n          (emb_layer_beds): Embedding(11, 6, padding_idx=0)\n          (emb_layer_guests_included): Embedding(11, 6, padding_idx=0)\n          (emb_layer_minimum_nights): Embedding(25, 9, padding_idx=0)\n          (emb_layer_instant_bookable): Embedding(3, 2, padding_idx=0)\n          (emb_layer_cancellation_policy): Embedding(6, 4, padding_idx=0)\n          (emb_layer_has_house_rules): Embedding(3, 2, padding_idx=0)\n          (emb_layer_host_gender): Embedding(4, 3, padding_idx=0)\n          (emb_layer_accommodates_catg): Embedding(4, 3, padding_idx=0)\n          (emb_layer_guests_included_catg): Embedding(4, 3, padding_idx=0)\n          (emb_layer_minimum_nights_catg): Embedding(4, 3, padding_idx=0)\n          (emb_layer_host_listings_count_catg): Embedding(5, 3, padding_idx=0)\n          (emb_layer_bathrooms_catg): Embedding(4, 3, padding_idx=0)\n          (emb_layer_bedrooms_catg): Embedding(5, 3, padding_idx=0)\n          (emb_layer_beds_catg): Embedding(5, 3, padding_idx=0)\n          (emb_layer_security_deposit): Embedding(53, 15, padding_idx=0)\n          (emb_layer_extra_people): Embedding(39, 12, padding_idx=0)\n        )\n        (embedding_dropout): Dropout(p=0.0, inplace=False)\n      )\n      (cont_norm): Identity()\n      (encoder): MLP(\n        (mlp): Sequential(\n          (dense_layer_0): Sequential(\n            (0): Linear(in_features=128, out_features=32, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n          (dense_layer_1): Sequential(\n            (0): Linear(in_features=32, out_features=16, bias=True)\n            (1): ReLU(inplace=True)\n            (2): Dropout(p=0.1, inplace=False)\n          )\n        )\n      )\n    )\n    (1): Linear(in_features=16, out_features=1, bias=True)\n  )\n  (deeptext): Sequential(\n    (0): BasicRNN(\n      (word_embed): Embedding(2192, 32, padding_idx=1)\n      (rnn): LSTM(32, 64, num_layers=2, batch_first=True, dropout=0.1)\n      (rnn_mlp): Identity()\n    )\n    (1): Linear(in_features=64, out_features=1, bias=True)\n  )\n  (deepimage): Sequential(\n    (0): Vision(\n      (features): Sequential(\n        (conv_layer_0): Sequential(\n          (0): Conv2d(3, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False)\n          (1): BatchNorm2d(64, eps=1e-05, momentum=0.01, affine=True, track_running_stats=True)\n          (2): LeakyReLU(negative_slope=0.1, inplace=True)\n          (maxpool): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n        )\n        (conv_layer_1): Sequential(\n          (0): Conv2d(64, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n          (1): BatchNorm2d(128, eps=1e-05, momentum=0.01, affine=True, track_running_stats=True)\n          (2): LeakyReLU(negative_slope=0.1, inplace=True)\n        )\n        (conv_layer_2): Sequential(\n          (0): Conv2d(128, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n          (1): BatchNorm2d(256, eps=1e-05, momentum=0.01, affine=True, track_running_stats=True)\n          (2): LeakyReLU(negative_slope=0.1, inplace=True)\n        )\n        (conv_layer_3): Sequential(\n          (0): Conv2d(256, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n          (1): BatchNorm2d(512, eps=1e-05, momentum=0.01, affine=True, track_running_stats=True)\n          (2): LeakyReLU(negative_slope=0.1, inplace=True)\n          (adaptiveavgpool): AdaptiveAvgPool2d(output_size=(1, 1))\n        )\n      )\n    )\n    (1): Linear(in_features=512, out_features=1, bias=True)\n  )\n)</pre> In\u00a0[14]: Copied! <pre>trainer = TrainerFromFolder(\n    model,\n    objective=\"regression\",\n)\n\ntrainer.fit(\n    train_loader=train_loader,\n    eval_loader=eval_loader,\n)\n</pre> trainer = TrainerFromFolder(     model,     objective=\"regression\", )  trainer.fit(     train_loader=train_loader,     eval_loader=eval_loader, ) <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 50/50 [03:41&lt;00:00,  4.42s/it, loss=1.64e+4]\nvalid: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 7/7 [00:23&lt;00:00,  3.30s/it, loss=6.27e+3]\n</pre> In\u00a0[15]: Copied! <pre>preds = trainer.predict(test_loader=test_loader)\n</pre> preds = trainer.predict(test_loader=test_loader) <pre>predict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 7/7 [00:22&lt;00:00,  3.26s/it]\n</pre> <p>Note that in the case of predict you could also choose to do this</p> In\u00a0[16]: Copied! <pre>df_test = pd.read_csv(\"/\".join([data_path, test_fname]))\n</pre> df_test = pd.read_csv(\"/\".join([data_path, test_fname])) In\u00a0[17]: Copied! <pre># if the images for the test set fit in memory\nX_tab_test = chunk_tab_preprocessor.transform(df_test)\nX_text_test = chunk_text_preprocessor.transform(df_test)\nX_img_test = img_preprocessor.transform(df_test)\n</pre> # if the images for the test set fit in memory X_tab_test = chunk_tab_preprocessor.transform(df_test) X_text_test = chunk_text_preprocessor.transform(df_test) X_img_test = img_preprocessor.transform(df_test) <pre>Reading Images from ../tmp_data/airbnb/property_picture/\nResizing\n</pre> <pre>100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 101/101 [00:00&lt;00:00, 708.23it/s]</pre> <pre>Computing normalisation metrics\n</pre> <pre>\n</pre> In\u00a0[18]: Copied! <pre>preds = trainer.predict(\n    X_tab=X_tab_test, X_text=X_text_test, X_img=X_img_test, batch_size=32\n)\n</pre> preds = trainer.predict(     X_tab=X_tab_test, X_text=X_text_test, X_img=X_img_test, batch_size=32 ) <pre>predict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 4/4 [00:03&lt;00:00,  1.14it/s]\n</pre>"},{"location":"examples/19_load_from_folder_functionality.html#scenario","title":"Scenario\u00b6","text":"<p>We have a tabular dataset combined with images and text and either some, or all these datasets do not fit in memory. Note that the tabular dataset MUST ALWAYS be present as it is considered the rerefence. This is, if we have an image dataset, the tabular dataset must contain a column that points to the image file names as stored in disk. Similarly, if we have a text dataset, then the tabular dataset must contain a column with the texts themselves or a the file names of the text files as stored in disk.</p> <p>If you only have text and/or images and not a tabular component, I would suggest using other libraries (such as hugginface probably).</p> <p>Within this scenario, they are two possible scenarios that we will cover here:</p> <ol> <li><p>The tabular data itsel fits in memory and is only the images that do not: in this case you could use the 'standard' <code>Preprocessors</code> in the library and off you go, move directly to the <code>[...]FromFolder</code> functionalities</p> </li> <li><p>The tabular data is also very large and does not fit in memory, so we have to process it in chuncks. For this second case I have created the so called <code>Chunk[...]Preprocessor</code> (Wide, Tab and Text).</p> </li> </ol> <p>Note that at the moment ONLY csv format is allowed for the tabular file. More formats will be supported in the future.</p> <p>Let's see a complete example to illustrate how each of these cases would be addressed with the new functionalities in the library. For this example we will use a sample of the airbnb dataset</p> <p>The airbnb dataset, which you could get from here, is too big to be included in our datasets module (when including images). Therefore, what I did was, go there, download it, and use the download_images.py script to get the images and the airbnb_data_processing.py to process the data. I did this ages ago and I believe the format of the dataset might be different now. Nonetheless, I will show samples of the dataset as we go through so you can extrapolate the content of this notebook to your particular problem.</p> <p>In the future we will find better datasets\ud83d\ude42. Finally, note that here we are only using a small sample to illustrate the use, so PLEASE ignore the results, just focus on usage.</p>"},{"location":"examples/19_load_from_folder_functionality.html#setting-variables-and-constants","title":"Setting variables and constants\u00b6","text":""},{"location":"examples/19_load_from_folder_functionality.html#step-1-the-preprocessors","title":"Step 1: the preprocessors\u00b6","text":""},{"location":"examples/19_load_from_folder_functionality.html#scenario-1-only-the-images-do-not-fit-in-disk","title":"Scenario 1: only the images do not fit in disk\u00b6","text":"<p>In this case we can prepare the data in the 'standard' way</p>"},{"location":"examples/19_load_from_folder_functionality.html#scenario-2-the-tabular-data-is-also-huge","title":"Scenario 2: the tabular data is also huge\u00b6","text":"<p>Then we need to prepare it in chunks. Note that, unfortunately, the tabular and text preprocessors need to see the whole dataset once. This is because to process tabular or text data we need to encode values. For those encodings to be consistent they need to have seen the whole dataset. Alternatively, one could code a solution with some streaming encoder for both datasets. However, such implementation is not trivial for this library (and in general). I also don't think that having to see the whole data once is such a big limitation. Let's see how is done.</p> <p>Note that I have not mentioned the image dataset. This is because the processing of the image dataset does not require any form of encoding and in consequence can be done 'on the fly'. Therefore, no <code>ChunkImgPreprocessor</code> processor is needed.</p>"},{"location":"examples/19_load_from_folder_functionality.html#step-2-the-fromfolder-classes","title":"Step 2: the <code>[...]FromFolder</code> classes\u00b6","text":"<p>Once we have the preprocessors, we need to instantiate the classes that will enable us to load the data from their respective folders. From now on I am going to proceed with the <code>chunk_tab_preprocessor</code>, <code>chunk_text_preprocessor</code> and <code>img_preprocessor</code>, but the code would be identical if instead of the first two preprocessors we decided to use the <code>tab_preprocessor</code> and <code>text_preprocessor</code>.</p> <p>Once more, our reference datasets are the tabular datasets, which we have splitted in train, eval and test prior to start the coding. Therefore, we will eventually need a loader for each split</p>"},{"location":"examples/19_load_from_folder_functionality.html#step-3-pytorch-datasets-and-dataloaders","title":"Step 3: pytorch datasets and dataloaders\u00b6","text":"<p>From here in advance, is all very 'standard' if you are familiar with pytorch. One needs to define a class that inherits from the <code>Dataset</code> class in pytorch. Then this will be passed to a <code>DataLoader</code> class and we are ready to train. Our <code>Dataset</code> child class is <code>WideDeepDatasetFromFolder</code>. This class will use the tabular dataset and the corresponding text and image columns to load the adequate data in the batches</p> <p>Let's do it</p>"},{"location":"examples/19_load_from_folder_functionality.html#step-4-define-the-model","title":"Step 4: define the model\u00b6","text":""},{"location":"examples/19_load_from_folder_functionality.html#step-5-fit-and-predict","title":"Step 5: fit and predict\u00b6","text":""},{"location":"examples/20_Using_huggingface_within_widedeep.html","title":"20-Using-huggingface-within-widedeep","text":"<p>In this notebook we will show how to use Hugginface's tokenizers and models as they are integrated within the library. In notebook number 17 you can find examples on how to code your own, custom, Hugginface (hereafter HF) model and use it in combination of any other model in the library</p> In\u00a0[1]: Copied! <pre>import numpy as np\nimport pandas as pd\nfrom sklearn.metrics import f1_score, accuracy_score\nfrom sklearn.model_selection import train_test_split\n\nfrom pytorch_widedeep import Trainer\nfrom pytorch_widedeep.models import HFModel, WideDeep\nfrom pytorch_widedeep.metrics import F1Score, Accuracy\nfrom pytorch_widedeep.datasets import load_womens_ecommerce\nfrom pytorch_widedeep.preprocessing import HFPreprocessor\n</pre> import numpy as np import pandas as pd from sklearn.metrics import f1_score, accuracy_score from sklearn.model_selection import train_test_split  from pytorch_widedeep import Trainer from pytorch_widedeep.models import HFModel, WideDeep from pytorch_widedeep.metrics import F1Score, Accuracy from pytorch_widedeep.datasets import load_womens_ecommerce from pytorch_widedeep.preprocessing import HFPreprocessor <pre>/Users/javierrodriguezzaurin/.pyenv/versions/3.10.13/envs/widedeep310/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n  from .autonotebook import tqdm as notebook_tqdm\n</pre> In\u00a0[2]: Copied! <pre>df: pd.DataFrame = load_womens_ecommerce(as_frame=True)  # type: ignore\n</pre> df: pd.DataFrame = load_womens_ecommerce(as_frame=True)  # type: ignore In\u00a0[3]: Copied! <pre>df.shape\n</pre> df.shape Out[3]: <pre>(23486, 10)</pre> In\u00a0[4]: Copied! <pre>df.sample(3)\n</pre> df.sample(3) Out[4]: Clothing ID Age Title Review Text Rating Recommended IND Positive Feedback Count Division Name Department Name Class Name 7004 862 43 Cute and feminine Loved this sweater wrap and bought it in both ... 5 1 2 General Tops Knits 12508 975 66 Love it The linen fabric is elegantly thin feels and l... 5 1 3 General Jackets Jackets 10288 950 41 Perfect for fall This sweater is just as pictured. the fit is t... 5 1 0 General Tops Sweaters In\u00a0[5]: Copied! <pre># Let's do some mild preprocessing\ndf.columns = [c.replace(\" \", \"_\").lower() for c in df.columns]\n\n# classes from [0,num_class)\ndf[\"rating\"] = (df[\"rating\"] - 1).astype(\"int64\")\n\n# group reviews with 1 and 2 scores into one class\ndf.loc[df.rating == 0, \"rating\"] = 1\n\n# and back again to [0,num_class)\ndf[\"rating\"] = (df[\"rating\"] - 1).astype(\"int64\")\n</pre> # Let's do some mild preprocessing df.columns = [c.replace(\" \", \"_\").lower() for c in df.columns]  # classes from [0,num_class) df[\"rating\"] = (df[\"rating\"] - 1).astype(\"int64\")  # group reviews with 1 and 2 scores into one class df.loc[df.rating == 0, \"rating\"] = 1  # and back again to [0,num_class) df[\"rating\"] = (df[\"rating\"] - 1).astype(\"int64\") In\u00a0[6]: Copied! <pre># drop short reviews\ndf = df[~df.review_text.isna()]\ndf[\"review_length\"] = df.review_text.apply(lambda x: len(x.split(\" \")))\ndf = df[df.review_length &gt;= 5]\ndf = df.drop(\"review_length\", axis=1).reset_index(drop=True)\n</pre> # drop short reviews df = df[~df.review_text.isna()] df[\"review_length\"] = df.review_text.apply(lambda x: len(x.split(\" \"))) df = df[df.review_length &gt;= 5] df = df.drop(\"review_length\", axis=1).reset_index(drop=True) In\u00a0[7]: Copied! <pre>df.shape\n</pre> df.shape Out[7]: <pre>(22608, 10)</pre> In\u00a0[8]: Copied! <pre># if you run this on a CPU, you might want to subsample the dataset. With that in mind I am simply going to stratify-sample to the minimum category occurrence and then sample at random\n# If you run this on a GPU you can comment out the following two cells\ndf.rating.value_counts()\n</pre> # if you run this on a CPU, you might want to subsample the dataset. With that in mind I am simply going to stratify-sample to the minimum category occurrence and then sample at random # If you run this on a GPU you can comment out the following two cells df.rating.value_counts() Out[8]: <pre>rating\n3    12515\n2     4904\n1     2820\n0     2369\nName: count, dtype: int64</pre> In\u00a0[9]: Copied! <pre>df = (\n    df.groupby(\"rating\", group_keys=False)\n    .apply(lambda x: x.sample(min(len(x), 2369)))\n    .sample(1000)\n)\n</pre> df = (     df.groupby(\"rating\", group_keys=False)     .apply(lambda x: x.sample(min(len(x), 2369)))     .sample(1000) ) <pre>/var/folders/_2/lrjn1qn54c758tdtktr1bvkc0000gn/T/ipykernel_5886/895673206.py:3: DeprecationWarning: DataFrameGroupBy.apply operated on the grouping columns. This behavior is deprecated, and in a future version of pandas the grouping columns will be excluded from the operation. Either pass `include_groups=False` to exclude the groupings or explicitly select the grouping columns after groupby to silence this warning.\n  .apply(lambda x: x.sample(min(len(x), 2369)))\n</pre> In\u00a0[10]: Copied! <pre>train, test = train_test_split(df, train_size=0.8, random_state=1, stratify=df.rating)\n\n# possible model names currently supported in the library\nmodel_names = [\n    \"distilbert-base-uncased\",\n    \"bert-base-uncased\",\n    \"FacebookAI/roberta-base\",\n    \"albert-base-v2\",\n    \"google/electra-base-discriminator\",\n]\n\n# Let's choose one. The syntax is the same for all the models\nmodel_name = \"distilbert-base-uncased\"\n</pre> train, test = train_test_split(df, train_size=0.8, random_state=1, stratify=df.rating)  # possible model names currently supported in the library model_names = [     \"distilbert-base-uncased\",     \"bert-base-uncased\",     \"FacebookAI/roberta-base\",     \"albert-base-v2\",     \"google/electra-base-discriminator\", ]  # Let's choose one. The syntax is the same for all the models model_name = \"distilbert-base-uncased\" <p>Now we can use the <code>HFPreprocessor</code> class. As most things in this library, the integration with HF has been coded aiming for a flexible use. With this in mind, there are two ways one can use a <code>HFPreprocessor</code> class.</p> <ol> <li>Passing a <code>text_col</code> and <code>encode_params</code> as the class is instantiated and then using the <code>fit</code> and <code>transform</code> as with any other preprocessor in the library</li> <li>Without passing  <code>text_col</code> and <code>encode_params</code> as the class is instantiated and using the <code>encode</code> method of the <code>HFPreprocessor</code> which is simply a wrapper around the encode method of HF's tokenizers</li> </ol> <p>Let's have a look</p> In\u00a0[11]: Copied! <pre>tokenizer1 = HFPreprocessor(\n    model_name=model_name,\n    text_col=\"review_text\",\n    num_workers=1,\n    encode_params={\n        \"max_length\": 90,\n        \"padding\": \"max_length\",\n        \"truncation\": True,\n        \"add_special_tokens\": True,\n    },\n)\n\nX_text_tr1 = tokenizer1.fit_transform(train)\nX_text_te1 = tokenizer1.transform(test)\n</pre> tokenizer1 = HFPreprocessor(     model_name=model_name,     text_col=\"review_text\",     num_workers=1,     encode_params={         \"max_length\": 90,         \"padding\": \"max_length\",         \"truncation\": True,         \"add_special_tokens\": True,     }, )  X_text_tr1 = tokenizer1.fit_transform(train) X_text_te1 = tokenizer1.transform(test) In\u00a0[12]: Copied! <pre>tokenizer2 = HFPreprocessor(\n    model_name=model_name,\n    num_workers=1,\n)\n\nX_text_tr2 = tokenizer2.encode(\n    train.review_text.tolist(),\n    max_length=90,\n    padding=\"max_length\",\n    truncation=True,\n    add_special_tokens=True,\n)\nX_text_te2 = tokenizer2.encode(\n    test.review_text.tolist(),\n    max_length=90,\n    padding=\"max_length\",\n    truncation=True,\n    add_special_tokens=True,\n)\n</pre> tokenizer2 = HFPreprocessor(     model_name=model_name,     num_workers=1, )  X_text_tr2 = tokenizer2.encode(     train.review_text.tolist(),     max_length=90,     padding=\"max_length\",     truncation=True,     add_special_tokens=True, ) X_text_te2 = tokenizer2.encode(     test.review_text.tolist(),     max_length=90,     padding=\"max_length\",     truncation=True,     add_special_tokens=True, ) In\u00a0[13]: Copied! <pre>all(X_text_tr1[0] == X_text_tr2[0])\n</pre> all(X_text_tr1[0] == X_text_tr2[0]) Out[13]: <pre>True</pre> In\u00a0[14]: Copied! <pre># Now we define a model which is as easy as:\n# Note that this will instantiation will lead to NO parameter trainable in the HF model.\n# If you want to fine-tune the HF model, you can set the trainable parameters via the 'trainable_parameters' argument.\n# Alternatively, you can use a head (MLP) via the 'head'-related arguments (see the docs for more details)\nhf_model = HFModel(model_name=model_name)\n</pre> # Now we define a model which is as easy as: # Note that this will instantiation will lead to NO parameter trainable in the HF model. # If you want to fine-tune the HF model, you can set the trainable parameters via the 'trainable_parameters' argument. # Alternatively, you can use a head (MLP) via the 'head'-related arguments (see the docs for more details) hf_model = HFModel(model_name=model_name) In\u00a0[15]: Copied! <pre># And from here on is the same as any other WideDeep model\nmodel = WideDeep(\n    deeptext=hf_model,\n    pred_dim=4,\n)\n\ntrainer = Trainer(\n    model,\n    objective=\"multiclass\",\n    metrics=[Accuracy(), F1Score(average=True)],\n)\n\ntrainer.fit(\n    X_text=X_text_tr2,\n    target=train.rating.values,\n    n_epochs=1,\n    batch_size=64,\n)\n# If you run this on a CPU and you sampled the data, the metrics will not be better than a random guess. Remember, this is just a demo\n</pre> # And from here on is the same as any other WideDeep model model = WideDeep(     deeptext=hf_model,     pred_dim=4, )  trainer = Trainer(     model,     objective=\"multiclass\",     metrics=[Accuracy(), F1Score(average=True)], )  trainer.fit(     X_text=X_text_tr2,     target=train.rating.values,     n_epochs=1,     batch_size=64, ) # If you run this on a CPU and you sampled the data, the metrics will not be better than a random guess. Remember, this is just a demo <pre>epoch 1: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 13/13 [02:06&lt;00:00,  9.75s/it, loss=3.2, metrics={'acc': 0.235, 'f1': 0.2336}]  \n</pre> In\u00a0[17]: Copied! <pre>preds_text = trainer.predict_proba(X_text=X_text_te2)\npred_text_class = np.argmax(preds_text, 1)\n\nacc_text = accuracy_score(test.rating, pred_text_class)\nf1_text = f1_score(test.rating, pred_text_class, average=\"weighted\")\nprint(f\"Accuracy: {acc_text:.4f}\")\nprint(f\"F1: {f1_text:.4f}\")\n</pre> preds_text = trainer.predict_proba(X_text=X_text_te2) pred_text_class = np.argmax(preds_text, 1)  acc_text = accuracy_score(test.rating, pred_text_class) f1_text = f1_score(test.rating, pred_text_class, average=\"weighted\") print(f\"Accuracy: {acc_text:.4f}\") print(f\"F1: {f1_text:.4f}\") <pre>predict: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 4/4 [00:05&lt;00:00,  1.43s/it]</pre> <pre>Accuracy: 0.2500\nF1: 0.1000\n</pre> <pre>\n</pre>"},{"location":"pytorch-widedeep/bayesian_models.html","title":"The <code>bayesian models</code> module","text":"<p>This module contains the two Bayesian Models available in this library, namely the bayesian version of the <code>Wide</code> and <code>TabMlp</code> models, referred as <code>BayesianWide</code> and <code>BayesianTabMlp</code>. These models are very useful in scenarios where getting a measure of uncertainty is important.</p> <p>The models in this module are based on the publication: Weight Uncertainty in Neural Networks.</p>"},{"location":"pytorch-widedeep/bayesian_models.html#pytorch_widedeep.bayesian_models.tabular.bayesian_linear.bayesian_wide.BayesianWide","title":"BayesianWide","text":"<p>               Bases: <code>BaseBayesianModel</code></p> <p>Defines a <code>Wide</code> model. This is a linear model where the non-linearlities are captured via crossed-columns</p> <p>Parameters:</p> Name Type Description Default <code>input_dim</code> <code>int</code> <p>size of the Embedding layer. <code>input_dim</code> is the summation of all the individual values for all the features that go through the wide component. For example, if the wide component receives 2 features with 5 individual values each, <code>input_dim = 10</code></p> required <code>pred_dim</code> <code>int</code> <p>size of the ouput tensor containing the predictions</p> <code>1</code> <code>prior_sigma_1</code> <code>float</code> <p>The prior weight distribution is a scaled mixture of two Gaussian densities:</p> \\[    \\begin{aligned}    P(\\mathbf{w}) = \\prod_{i=j} \\pi N (\\mathbf{w}_j | 0, \\sigma_{1}^{2}) + (1 - \\pi) N (\\mathbf{w}_j | 0, \\sigma_{2}^{2})    \\end{aligned} \\] <p><code>prior_sigma_1</code> is the prior of the sigma parameter for the first of the two Gaussians that will be mixed to produce the prior weight distribution.</p> <code>1.0</code> <code>prior_sigma_2</code> <code>float</code> <p>Prior of the sigma parameter for the second of the two Gaussian distributions that will be mixed to produce the prior weight distribution</p> <code>0.002</code> <code>prior_pi</code> <code>float</code> <p>Scaling factor that will be used to mix the Gaussians to produce the prior weight distribution</p> <code>0.8</code> <code>posterior_mu_init</code> <code>float</code> <p>The posterior sample of the weights is defined as:</p> \\[    \\begin{aligned}    \\mathbf{w} &amp;= \\mu + log(1 + exp(\\rho))    \\end{aligned} \\] <p>where:</p> \\[    \\begin{aligned}    \\mathcal{N}(x\\vert \\mu, \\sigma) &amp;= \\frac{1}{\\sqrt{2\\pi}\\sigma}e^{-\\frac{(x-\\mu)^2}{2\\sigma^2}}\\\\    \\log{\\mathcal{N}(x\\vert \\mu, \\sigma)} &amp;= -\\log{\\sqrt{2\\pi}} -\\log{\\sigma} -\\frac{(x-\\mu)^2}{2\\sigma^2}\\\\    \\end{aligned} \\] <p>\\(\\mu\\) is initialised using a normal distributtion with mean <code>posterior_mu_init</code> and std equal to 0.1.</p> <code>0.0</code> <code>posterior_rho_init</code> <code>float</code> <p>As in the case of \\(\\mu\\), \\(\\rho\\) is initialised using a normal distributtion with mean <code>posterior_rho_init</code> and std equal to 0.1.</p> <code>-7.0</code> <p>Attributes:</p> Name Type Description <code>bayesian_wide_linear</code> <code>Module</code> <p>the linear layer that comprises the wide branch of the model</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.bayesian_models import BayesianWide\n&gt;&gt;&gt; X = torch.empty(4, 4).random_(6)\n&gt;&gt;&gt; wide = BayesianWide(input_dim=int(X.max().item()), pred_dim=1)\n&gt;&gt;&gt; out = wide(X)\n</code></pre> Source code in <code>pytorch_widedeep/bayesian_models/tabular/bayesian_linear/bayesian_wide.py</code> <pre><code>class BayesianWide(BaseBayesianModel):\n    r\"\"\"Defines a `Wide` model. This is a linear model where the\n    non-linearlities are captured via crossed-columns\n\n    Parameters\n    ----------\n    input_dim: int\n        size of the Embedding layer. `input_dim` is the summation of all the\n        individual values for all the features that go through the wide\n        component. For example, if the wide component receives 2 features with\n        5 individual values each, `input_dim = 10`\n    pred_dim: int\n        size of the ouput tensor containing the predictions\n    prior_sigma_1: float, default = 1.0\n        The prior weight distribution is a scaled mixture of two Gaussian\n        densities:\n\n        $$\n           \\begin{aligned}\n           P(\\mathbf{w}) = \\prod_{i=j} \\pi N (\\mathbf{w}_j | 0, \\sigma_{1}^{2}) + (1 - \\pi) N (\\mathbf{w}_j | 0, \\sigma_{2}^{2})\n           \\end{aligned}\n        $$\n\n        `prior_sigma_1` is the prior of the sigma parameter for the first of the two\n        Gaussians that will be mixed to produce the prior weight\n        distribution.\n    prior_sigma_2: float, default = 0.002\n        Prior of the sigma parameter for the second of the two Gaussian\n        distributions that will be mixed to produce the prior weight\n        distribution\n    prior_pi: float, default = 0.8\n        Scaling factor that will be used to mix the Gaussians to produce the\n        prior weight distribution\n    posterior_mu_init: float = 0.0\n        The posterior sample of the weights is defined as:\n\n        $$\n           \\begin{aligned}\n           \\mathbf{w} &amp;= \\mu + log(1 + exp(\\rho))\n           \\end{aligned}\n        $$\n\n        where:\n\n        $$\n           \\begin{aligned}\n           \\mathcal{N}(x\\vert \\mu, \\sigma) &amp;= \\frac{1}{\\sqrt{2\\pi}\\sigma}e^{-\\frac{(x-\\mu)^2}{2\\sigma^2}}\\\\\n           \\log{\\mathcal{N}(x\\vert \\mu, \\sigma)} &amp;= -\\log{\\sqrt{2\\pi}} -\\log{\\sigma} -\\frac{(x-\\mu)^2}{2\\sigma^2}\\\\\n           \\end{aligned}\n        $$\n\n        $\\mu$ is initialised using a normal distributtion with mean\n        `posterior_mu_init` and std equal to 0.1.\n    posterior_rho_init: float = -7.0\n        As in the case of $\\mu$, $\\rho$ is initialised using a\n        normal distributtion with mean `posterior_rho_init` and std equal to\n        0.1.\n\n    Attributes\n    -----------\n    bayesian_wide_linear: nn.Module\n        the linear layer that comprises the wide branch of the model\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.bayesian_models import BayesianWide\n    &gt;&gt;&gt; X = torch.empty(4, 4).random_(6)\n    &gt;&gt;&gt; wide = BayesianWide(input_dim=int(X.max().item()), pred_dim=1)\n    &gt;&gt;&gt; out = wide(X)\n    \"\"\"\n\n    def __init__(\n        self,\n        input_dim: int,\n        pred_dim: int = 1,\n        prior_sigma_1: float = 1.0,\n        prior_sigma_2: float = 0.002,\n        prior_pi: float = 0.8,\n        posterior_mu_init: float = 0.0,\n        posterior_rho_init: float = -7.0,\n    ):\n        super(BayesianWide, self).__init__()\n        #  Embeddings: val + 1 because 0 is reserved for padding/unseen cateogories.\n        self.bayesian_wide_linear = bnn.BayesianEmbedding(\n            n_embed=input_dim + 1,\n            embed_dim=pred_dim,\n            padding_idx=0,\n            prior_sigma_1=prior_sigma_1,\n            prior_sigma_2=prior_sigma_2,\n            prior_pi=prior_pi,\n            posterior_mu_init=posterior_mu_init,\n            posterior_rho_init=posterior_rho_init,\n        )\n        self.bias = nn.Parameter(torch.zeros(pred_dim))\n\n    def forward(self, X: Tensor) -&gt; Tensor:\n        out = self.bayesian_wide_linear(X.long()).sum(dim=1) + self.bias\n        return out\n</code></pre>"},{"location":"pytorch-widedeep/bayesian_models.html#pytorch_widedeep.bayesian_models.tabular.bayesian_mlp.bayesian_tab_mlp.BayesianTabMlp","title":"BayesianTabMlp","text":"<p>               Bases: <code>BaseBayesianModel</code></p> <p>Defines a <code>BayesianTabMlp</code> model.</p> <p>This class combines embedding representations of the categorical features with numerical (aka continuous) features, embedded or not. These are then passed through a series of probabilistic dense layers (i.e. a MLP).</p> <p>Parameters:</p> Name Type Description Default <code>column_idx</code> <code>Dict[str, int]</code> <p>Dict containing the index of the columns that will be passed through the <code>TabMlp</code> model. Required to slice the tensors. e.g. {'education': 0, 'relationship': 1, 'workclass': 2, ...}</p> required <code>cat_embed_input</code> <code>Optional[List[Tuple[str, int, int]]]</code> <p>List of Tuples with the column name, number of unique values and embedding dimension. e.g. [(education, 11, 32), ...]</p> <code>None</code> <code>cat_embed_activation</code> <code>Optional[str]</code> <p>Activation function for the categorical embeddings, if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> <code>None</code> <code>continuous_cols</code> <code>Optional[List[str]]</code> <p>List with the name of the numeric (aka continuous) columns</p> <code>None</code> <code>cont_norm_layer</code> <code>Optional[Literal[batchnorm, layernorm]]</code> <p>Type of normalization layer applied to the continuous features. Options are: 'layernorm', 'batchnorm' or None.</p> <code>None</code> <code>embed_continuous</code> <code>Optional[bool]</code> <p>Boolean indicating if the continuous columns will be embedded (i.e. passed each through a linear layer with or without activation)</p> <code>None</code> <code>cont_embed_dim</code> <code>Optional[int]</code> <p>Size of the continuous embeddings</p> <code>None</code> <code>cont_embed_dropout</code> <code>Optional[float]</code> <p>Dropout for the continuous embeddings</p> <code>None</code> <code>use_cont_bias</code> <code>Optional[bool]</code> <p>Boolean indicating if bias will be used for the continuous embeddings</p> <code>None</code> <code>cont_embed_activation</code> <code>Optional[str]</code> <p>Activation function for the continuous embeddings if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> <code>None</code> <code>mlp_hidden_dims</code> <code>List[int]</code> <p>List with the number of neurons per dense layer in the mlp.</p> <code>[200, 100]</code> <code>mlp_activation</code> <code>str</code> <p>Activation function for the dense layers of the MLP. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> <code>'leaky_relu'</code> <code>prior_sigma_1</code> <code>float</code> <p>The prior weight distribution is a scaled mixture of two Gaussian densities:</p> \\[    \\begin{aligned}    P(\\mathbf{w}) = \\prod_{i=j} \\pi N (\\mathbf{w}_j | 0, \\sigma_{1}^{2}) + (1 - \\pi) N (\\mathbf{w}_j | 0, \\sigma_{2}^{2})    \\end{aligned} \\] <p><code>prior_sigma_1</code> is the prior of the sigma parameter for the first of the two Gaussians that will be mixed to produce the prior weight distribution.</p> <code>1</code> <code>prior_sigma_2</code> <code>float</code> <p>Prior of the sigma parameter for the second of the two Gaussian distributions that will be mixed to produce the prior weight distribution for each Bayesian linear and embedding layer</p> <code>0.002</code> <code>prior_pi</code> <code>float</code> <p>Scaling factor that will be used to mix the Gaussians to produce the prior weight distribution ffor each Bayesian linear and embedding layer</p> <code>0.8</code> <code>posterior_mu_init</code> <code>float</code> <p>The posterior sample of the weights is defined as:</p> <p>$$    \\begin{aligned}    \\mathbf{w} &amp;= \\mu + log(1 + exp(\\rho))    \\end{aligned} $$ where:</p> \\[    \\begin{aligned}    \\mathcal{N}(x\\vert \\mu, \\sigma) &amp;= \\frac{1}{\\sqrt{2\\pi}\\sigma}e^{-\\frac{(x-\\mu)^2}{2\\sigma^2}}\\\\    \\log{\\mathcal{N}(x\\vert \\mu, \\sigma)} &amp;= -\\log{\\sqrt{2\\pi}} -\\log{\\sigma} -\\frac{(x-\\mu)^2}{2\\sigma^2}\\\\    \\end{aligned} \\] <p>\\(\\mu\\) is initialised using a normal distributtion with mean <code>posterior_mu_init</code> and std equal to 0.1.</p> <code>0.0</code> <code>posterior_rho_init</code> <code>float</code> <p>As in the case of \\(\\mu\\), \\(\\rho\\) is initialised using a normal distributtion with mean <code>posterior_rho_init</code> and std equal to 0.1.</p> <code>-7.0</code> <p>Attributes:</p> Name Type Description <code>bayesian_cat_and_cont_embed</code> <code>Module</code> <p>This is the module that processes the categorical and continuous columns</p> <code>bayesian_tab_mlp</code> <code>Sequential</code> <p>mlp model that will receive the concatenation of the embeddings and the continuous columns</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.bayesian_models import BayesianTabMlp\n&gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n&gt;&gt;&gt; colnames = ['a', 'b', 'c', 'd', 'e']\n&gt;&gt;&gt; cat_embed_input = [(u,i,j) for u,i,j in zip(colnames[:4], [4]*4, [8]*4)]\n&gt;&gt;&gt; column_idx = {k:v for v,k in enumerate(colnames)}\n&gt;&gt;&gt; model = BayesianTabMlp(mlp_hidden_dims=[8,4], column_idx=column_idx, cat_embed_input=cat_embed_input,\n... continuous_cols = ['e'])\n&gt;&gt;&gt; out = model(X_tab)\n</code></pre> Source code in <code>pytorch_widedeep/bayesian_models/tabular/bayesian_mlp/bayesian_tab_mlp.py</code> <pre><code>class BayesianTabMlp(BaseBayesianModel):\n    r\"\"\"Defines a `BayesianTabMlp` model.\n\n    This class combines embedding representations of the categorical features\n    with numerical (aka continuous) features, embedded or not. These are then\n    passed through a series of probabilistic dense layers (i.e. a MLP).\n\n    Parameters\n    ----------\n    column_idx: Dict\n        Dict containing the index of the columns that will be passed through\n        the `TabMlp` model. Required to slice the tensors. e.g. _{'education':\n        0, 'relationship': 1, 'workclass': 2, ...}_\n    cat_embed_input: List, Optional, default = None\n        List of Tuples with the column name, number of unique values and\n        embedding dimension. e.g. _[(education, 11, 32), ...]_\n    cat_embed_activation: Optional, str, default = None,\n        Activation function for the categorical embeddings, if any. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported\n    continuous_cols: List, Optional, default = None\n        List with the name of the numeric (aka continuous) columns\n    cont_norm_layer: str, default =  \"batchnorm\"\n        Type of normalization layer applied to the continuous features. Options\n        are: 'layernorm', 'batchnorm' or None.\n    embed_continuous: bool, default = False,\n        Boolean indicating if the continuous columns will be embedded\n        (i.e. passed each through a linear layer with or without activation)\n    cont_embed_dim: int, default = 32,\n        Size of the continuous embeddings\n    cont_embed_dropout: float, default = 0.1,\n        Dropout for the continuous embeddings\n    use_cont_bias: bool, default = True,\n        Boolean indicating if bias will be used for the continuous embeddings\n    cont_embed_activation: Optional, str, default = None,\n        Activation function for the continuous embeddings if any. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported\n    mlp_hidden_dims: List, default = [200, 100]\n        List with the number of neurons per dense layer in the mlp.\n    mlp_activation: str, default = \"relu\"\n        Activation function for the dense layers of the MLP. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported\n    prior_sigma_1: float, default = 1.0\n        The prior weight distribution is a scaled mixture of two Gaussian\n        densities:\n\n        $$\n           \\begin{aligned}\n           P(\\mathbf{w}) = \\prod_{i=j} \\pi N (\\mathbf{w}_j | 0, \\sigma_{1}^{2}) + (1 - \\pi) N (\\mathbf{w}_j | 0, \\sigma_{2}^{2})\n           \\end{aligned}\n        $$\n\n        `prior_sigma_1` is the prior of the sigma parameter for the first of the two\n        Gaussians that will be mixed to produce the prior weight\n        distribution.\n    prior_sigma_2: float, default = 0.002\n        Prior of the sigma parameter for the second of the two Gaussian\n        distributions that will be mixed to produce the prior weight\n        distribution for each Bayesian linear and embedding layer\n    prior_pi: float, default = 0.8\n        Scaling factor that will be used to mix the Gaussians to produce the\n        prior weight distribution ffor each Bayesian linear and embedding\n        layer\n    posterior_mu_init: float = 0.0\n        The posterior sample of the weights is defined as:\n\n        $$\n           \\begin{aligned}\n           \\mathbf{w} &amp;= \\mu + log(1 + exp(\\rho))\n           \\end{aligned}\n        $$\n        where:\n\n        $$\n           \\begin{aligned}\n           \\mathcal{N}(x\\vert \\mu, \\sigma) &amp;= \\frac{1}{\\sqrt{2\\pi}\\sigma}e^{-\\frac{(x-\\mu)^2}{2\\sigma^2}}\\\\\n           \\log{\\mathcal{N}(x\\vert \\mu, \\sigma)} &amp;= -\\log{\\sqrt{2\\pi}} -\\log{\\sigma} -\\frac{(x-\\mu)^2}{2\\sigma^2}\\\\\n           \\end{aligned}\n        $$\n\n        $\\mu$ is initialised using a normal distributtion with mean\n        `posterior_mu_init` and std equal to 0.1.\n    posterior_rho_init: float = -7.0\n        As in the case of $\\mu$, $\\rho$ is initialised using a\n        normal distributtion with mean `posterior_rho_init` and std equal to\n        0.1.\n\n    Attributes\n    ----------\n    bayesian_cat_and_cont_embed: nn.Module\n        This is the module that processes the categorical and continuous columns\n    bayesian_tab_mlp: nn.Sequential\n        mlp model that will receive the concatenation of the embeddings and\n        the continuous columns\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.bayesian_models import BayesianTabMlp\n    &gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n    &gt;&gt;&gt; colnames = ['a', 'b', 'c', 'd', 'e']\n    &gt;&gt;&gt; cat_embed_input = [(u,i,j) for u,i,j in zip(colnames[:4], [4]*4, [8]*4)]\n    &gt;&gt;&gt; column_idx = {k:v for v,k in enumerate(colnames)}\n    &gt;&gt;&gt; model = BayesianTabMlp(mlp_hidden_dims=[8,4], column_idx=column_idx, cat_embed_input=cat_embed_input,\n    ... continuous_cols = ['e'])\n    &gt;&gt;&gt; out = model(X_tab)\n    \"\"\"\n\n    def __init__(\n        self,\n        column_idx: Dict[str, int],\n        *,\n        cat_embed_input: Optional[List[Tuple[str, int, int]]] = None,\n        cat_embed_activation: Optional[str] = None,\n        continuous_cols: Optional[List[str]] = None,\n        embed_continuous: Optional[bool] = None,\n        cont_embed_dim: Optional[int] = None,\n        cont_embed_dropout: Optional[float] = None,\n        cont_embed_activation: Optional[str] = None,\n        use_cont_bias: Optional[bool] = None,\n        cont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\n        mlp_hidden_dims: List[int] = [200, 100],\n        mlp_activation: str = \"leaky_relu\",\n        prior_sigma_1: float = 1,\n        prior_sigma_2: float = 0.002,\n        prior_pi: float = 0.8,\n        posterior_mu_init: float = 0.0,\n        posterior_rho_init: float = -7.0,\n        pred_dim=1,  # Bayesian models will require their own trainer and need the output layer\n    ):\n        super(BayesianTabMlp, self).__init__()\n\n        self.column_idx = column_idx\n        self.cat_embed_input = cat_embed_input\n        self.cat_embed_activation = cat_embed_activation\n\n        self.continuous_cols = continuous_cols\n        self.cont_norm_layer = cont_norm_layer\n        self.embed_continuous = embed_continuous\n        self.cont_embed_dim = cont_embed_dim\n        self.cont_embed_dropout = cont_embed_dropout\n        self.use_cont_bias = use_cont_bias\n        self.cont_embed_activation = cont_embed_activation\n\n        self.mlp_hidden_dims = mlp_hidden_dims\n        self.mlp_activation = mlp_activation\n\n        self.prior_sigma_1 = prior_sigma_1\n        self.prior_sigma_2 = prior_sigma_2\n        self.prior_pi = prior_pi\n        self.posterior_mu_init = posterior_mu_init\n        self.posterior_rho_init = posterior_rho_init\n\n        self.pred_dim = pred_dim\n\n        allowed_activations = [\"relu\", \"leaky_relu\", \"tanh\", \"gelu\"]\n        if self.mlp_activation not in allowed_activations:\n            raise ValueError(\n                \"Currently, only the following activation functions are supported \"\n                \"for the Bayesian MLP's dense layers: {}. Got '{}' instead\".format(\n                    \", \".join(allowed_activations),\n                    self.mlp_activation,\n                )\n            )\n\n        # Categorical\n        if self.cat_embed_input is not None:\n            self.cat_embed = BayesianDiffSizeCatEmbeddings(\n                column_idx=self.column_idx,\n                embed_input=self.cat_embed_input,\n                prior_sigma_1=self.prior_sigma_1,\n                prior_sigma_2=self.prior_sigma_2,\n                prior_pi=self.prior_pi,\n                posterior_mu_init=self.posterior_mu_init,\n                posterior_rho_init=self.posterior_rho_init,\n                activation_fn=self.cat_embed_activation,\n            )\n            self.cat_out_dim = int(np.sum([embed[2] for embed in self.cat_embed_input]))\n        else:\n            self.cat_out_dim = 0\n\n        # Continuous\n        if self.continuous_cols is not None:\n            self.cont_idx = [self.column_idx[col] for col in self.continuous_cols]\n            if cont_norm_layer == \"layernorm\":\n                self.cont_norm: NormLayers = nn.LayerNorm(len(self.continuous_cols))\n            elif cont_norm_layer == \"batchnorm\":\n                self.cont_norm = nn.BatchNorm1d(len(self.continuous_cols))\n            else:\n                self.cont_norm = nn.Identity()\n            if self.embed_continuous:\n                assert self.cont_embed_dim is not None, (\n                    \"If 'embed_continuous' is True, 'cont_embed_dim' must be \"\n                    \"provided\"\n                )\n                self.cont_embed = BayesianContEmbeddings(\n                    n_cont_cols=len(self.continuous_cols),\n                    embed_dim=self.cont_embed_dim,\n                    prior_sigma_1=self.prior_sigma_1,\n                    prior_sigma_2=self.prior_sigma_2,\n                    prior_pi=self.prior_pi,\n                    posterior_mu_init=self.posterior_mu_init,\n                    posterior_rho_init=self.posterior_rho_init,\n                    use_bias=(\n                        False if self.use_cont_bias is None else self.use_cont_bias\n                    ),\n                    activation_fn=self.cont_embed_activation,\n                )\n                self.cont_out_dim = len(self.continuous_cols) * self.cont_embed_dim\n            else:\n                self.cont_out_dim = len(self.continuous_cols)\n        else:\n            self.cont_out_dim = 0\n\n        self.output_dim = self.cat_out_dim + self.cont_out_dim\n\n        mlp_hidden_dims = [self.output_dim] + mlp_hidden_dims + [pred_dim]\n        self.bayesian_tab_mlp = BayesianMLP(\n            mlp_hidden_dims,\n            mlp_activation,\n            True,  # use_bias\n            prior_sigma_1,\n            prior_sigma_2,\n            prior_pi,\n            posterior_mu_init,\n            posterior_rho_init,\n        )\n\n    def forward(self, X: Tensor) -&gt; Tensor:\n        x = self._get_embeddings(X)\n        x = self.bayesian_tab_mlp(x)\n        return x\n\n    def _get_embeddings(self, X: Tensor) -&gt; Tensor:\n        tensors_to_concat: List[Tensor] = []\n        if self.cat_embed_input is not None:\n            x_cat = self.cat_embed(X)\n            tensors_to_concat.append(x_cat)\n\n        if self.continuous_cols is not None:\n            x_cont = self.cont_norm((X[:, self.cont_idx].float()))\n            if self.embed_continuous:\n                x_cont = self.cont_embed(x_cont)\n                x_cont = einops.rearrange(x_cont, \"b s d -&gt; b (s d)\")\n            tensors_to_concat.append(x_cont)\n\n        x = torch.cat(tensors_to_concat, 1)\n\n        return x\n</code></pre>"},{"location":"pytorch-widedeep/bayesian_trainer.html","title":"Training Deep Learning Probabilistic Models","text":""},{"location":"pytorch-widedeep/bayesian_trainer.html#pytorch_widedeep.training.BayesianTrainer","title":"BayesianTrainer","text":"<p>               Bases: <code>BaseBayesianTrainer</code></p> <p>Class to set the of attributes that will be used during the training process.</p> <p>Both the Bayesian models and the Trainer in this repo are based on the paper: Weight Uncertainty in Neural Networks.</p> <p>Parameters:</p> Name Type Description Default <code>model</code> <code>BaseBayesianModel</code> <p>An object of class <code>BaseBayesianModel</code>. See the <code>Model Components</code> section here in the docs.</p> required <code>objective</code> <code>str</code> <p>Defines the objective, loss or cost function. Param aliases: <code>loss_function</code>, <code>loss_fn</code>, <code>loss</code>, <code>cost_function</code>, <code>cost_fn</code>, <code>cost</code> Possible values are: 'binary', 'multiclass', 'regression'</p> required <code>custom_loss_function</code> <code>Optional[Module]</code> <p>If none of the loss functions available suits the user, it is possible to pass a custom loss function. See for example <code>pytorch_widedeep.losses.FocalLoss</code> for the required structure of the object or the Examples folder in the repo.</p> <code>None</code> <code>optimizer</code> <code>Optional[Optimizer]</code> <p>An instance of Pytorch's <code>Optimizer</code> object(e.g. <code>torch.optim.Adam ()</code>). if no optimizer is passed it will default to <code>AdamW</code>.</p> <code>None</code> <code>lr_scheduler</code> <code>Optional[LRScheduler]</code> <p>An instance of Pytorch's <code>LRScheduler</code> object (e.g <code>torch.optim.lr_scheduler.StepLR(opt, step_size=5)</code>).</p> <code>None</code> <code>callbacks</code> <code>Optional[List[Callback]]</code> <p>List with <code>Callback</code> objects. The three callbacks available in <code>pytorch-widedeep</code> are: <code>LRHistory</code>, <code>ModelCheckpoint</code> and <code>EarlyStopping</code>. This can also be a custom callback. See <code>pytorch_widedeep.callbacks.Callback</code> or the Examples folder in the repo.</p> <code>None</code> <code>metrics</code> <code>Optional[Union[List[Metric], List[Metric]]]</code> <ul> <li>List of objects of type <code>Metric</code>. Metrics available are:   <code>Accuracy</code>, <code>Precision</code>, <code>Recall</code>, <code>FBetaScore</code>,   <code>F1Score</code> and <code>R2Score</code>. This can also be a custom metric as   long as it is an object of type <code>Metric</code>. See   <code>pytorch_widedeep.metrics.Metric</code> or the Examples folder in the repo</li> <li>List of objects of type <code>torchmetrics.Metric</code>. This can be any   metric from torchmetrics library Examples   classification-metrics&gt;<code>_. It can also be a torchmetric custom metric as   long as it is an object of type</code>Metric<code>.   See</code>the instructions</li> </ul> <code>None</code> <code>verbose</code> <code>int</code> <p>Setting it to 0 will print nothing during training.</p> <code>1</code> <code>seed</code> <code>int</code> <p>Random seed to be used internally for train_test_split</p> <code>1</code> <p>Other Parameters:</p> Name Type Description <code>**kwargs</code> <p>Other infrequently used arguments that can also be passed as kwargs are:</p> <ul> <li> <p>device: <code>str</code>     string indicating the device. One of 'cpu', 'gpu' or 'mps' if     run on a Mac with Apple silicon or AMD GPU(s)</p> </li> <li> <p>num_workers: <code>int</code>     number of workers to be used internally by the data loaders</p> </li> <li> <p>class_weight: <code>List[float]</code>     This is the <code>weight</code> or <code>pos_weight</code> parameter in     <code>CrossEntropyLoss</code> and <code>BCEWithLogitsLoss</code>, depending on whether</p> </li> <li> <p>reducelronplateau_criterion: <code>str</code>     This sets the criterion that will be used by the lr scheduler to     take a step: One of 'loss' or 'metric'. The ReduceLROnPlateau     learning rate is a bit particular.</p> </li> </ul> <p>Attributes:</p> Name Type Description <code>cyclic_lr</code> <code>bool</code> <p>Attribute that indicates if  the lr_scheduler is cyclic_lr (i.e. <code>CyclicLR</code> or <code>OneCycleLR</code>). See <code>Pytorch schedulers &lt;https://pytorch.org/docs/stable/optim.html&gt;</code>_.</p> Source code in <code>pytorch_widedeep/training/bayesian_trainer.py</code> <pre><code>class BayesianTrainer(BaseBayesianTrainer):\n    r\"\"\"Class to set the of attributes that will be used during the\n    training process.\n\n    Both the Bayesian models and the Trainer in this repo are based on the paper:\n    [Weight Uncertainty in Neural Networks](https://arxiv.org/pdf/1505.05424.pdf).\n\n    Parameters\n    ----------\n    model: `BaseBayesianModel`\n        An object of class `BaseBayesianModel`. See the `Model Components`\n        section here in the docs.\n    objective: str\n        Defines the objective, loss or cost function.&lt;br/&gt;\n        Param aliases: `loss_function`, `loss_fn`, `loss`,\n        `cost_function`, `cost_fn`, `cost`&lt;br/&gt;\n        Possible values are: _'binary'_, _'multiclass'_, _'regression'_\n    custom_loss_function: `nn.Module`, optional, default = None\n        If none of the loss functions available suits the user, it is possible\n        to pass a custom loss function. See for example\n        `pytorch_widedeep.losses.FocalLoss` for the required structure of the\n        object or the Examples folder in the repo.\n    optimizer: `Optimzer`, optional, default= None\n        An instance of Pytorch's `Optimizer` object(e.g. `torch.optim.Adam\n        ()`). if no optimizer is passed it will default to `AdamW`.\n    lr_scheduler: `LRScheduler`, optional, default=None\n        An instance of Pytorch's `LRScheduler` object\n        (e.g `torch.optim.lr_scheduler.StepLR(opt, step_size=5)`).\n    callbacks: List, optional, default=None\n        List with `Callback` objects. The three callbacks available in\n        `pytorch-widedeep` are: `LRHistory`, `ModelCheckpoint` and\n        `EarlyStopping`. This can also be a custom callback. See\n        `pytorch_widedeep.callbacks.Callback` or the Examples folder in the\n        repo.\n    metrics: List, optional, default=None\n        - List of objects of type `Metric`. Metrics available are:\n          `Accuracy`, `Precision`, `Recall`, `FBetaScore`,\n          `F1Score` and `R2Score`. This can also be a custom metric as\n          long as it is an object of type `Metric`. See\n          `pytorch_widedeep.metrics.Metric` or the Examples folder in the repo\n        - List of objects of type `torchmetrics.Metric`. This can be any\n          metric from torchmetrics library [Examples](https://lightning.ai/docs/torchmetrics)\n          classification-metrics&gt;`_. It can also be a torchmetric custom metric as\n          long as it is an object of type `Metric`.\n          See `the [instructions]((https://lightning.ai/docs/torchmetrics))\n    verbose: int, default=1\n        Setting it to 0 will print nothing during training.\n    seed: int, default=1\n        Random seed to be used internally for train_test_split\n\n    Other Parameters\n    ----------------\n    **kwargs: dict\n        Other infrequently used arguments that can also be passed as kwargs are:\n\n        - **device**: `str`&lt;br/&gt;\n            string indicating the device. One of _'cpu'_, _'gpu'_ or 'mps' if\n            run on a Mac with Apple silicon or AMD GPU(s)\n\n        - **num_workers**: `int`&lt;br/&gt;\n            number of workers to be used internally by the data loaders\n\n        - **class_weight**: `List[float]`&lt;br/&gt;\n            This is the `weight` or `pos_weight` parameter in\n            `CrossEntropyLoss` and `BCEWithLogitsLoss`, depending on whether\n\n        - **reducelronplateau_criterion**: `str`\n            This sets the criterion that will be used by the lr scheduler to\n            take a step: One of _'loss'_ or _'metric'_. The ReduceLROnPlateau\n            learning rate is a bit particular.\n\n    Attributes\n    ----------\n    cyclic_lr: bool\n        Attribute that indicates if  the lr_scheduler is cyclic_lr\n        (i.e. `CyclicLR` or `OneCycleLR`). See `Pytorch schedulers\n        &lt;https://pytorch.org/docs/stable/optim.html&gt;`_.\n    \"\"\"\n\n    @alias(  # noqa: C901\n        \"objective\",\n        [\"loss_function\", \"loss_fn\", \"loss\", \"cost_function\", \"cost_fn\", \"cost\"],\n    )\n    def __init__(\n        self,\n        model: BaseBayesianModel,\n        objective: str,\n        custom_loss_function: Optional[Module] = None,\n        optimizer: Optional[Optimizer] = None,\n        lr_scheduler: Optional[LRScheduler] = None,\n        callbacks: Optional[List[Callback]] = None,\n        metrics: Optional[Union[List[Metric], List[TorchMetric]]] = None,\n        verbose: int = 1,\n        seed: int = 1,\n        **kwargs,\n    ):\n        super().__init__(\n            model=model,\n            objective=objective,\n            custom_loss_function=custom_loss_function,\n            optimizer=optimizer,\n            lr_scheduler=lr_scheduler,\n            callbacks=callbacks,\n            metrics=metrics,\n            verbose=verbose,\n            seed=seed,\n            **kwargs,\n        )\n\n    def fit(  # noqa: C901\n        self,\n        X_tab: np.ndarray,\n        target: np.ndarray,\n        X_tab_val: Optional[np.ndarray] = None,\n        target_val: Optional[np.ndarray] = None,\n        val_split: Optional[float] = None,\n        n_epochs: int = 1,\n        validation_freq: int = 1,\n        batch_size: int = 32,\n        n_train_samples: int = 2,\n        n_val_samples: int = 2,\n    ):\n        r\"\"\"Fit method.\n\n        Parameters\n        ----------\n        X_tab: np.ndarray,\n            tabular dataset\n        target: np.ndarray\n            target values\n        X_tab_val: np.ndarray, Optional, default = None\n            validation data\n        target_val: np.ndarray, Optional, default = None\n            validation target values\n        val_split: float, Optional. default=None\n            An alterative to passing the validation set is to use a train/val\n            split fraction via `val_split`\n        n_epochs: int, default=1\n            number of epochs\n        validation_freq: int, default=1\n            epochs validation frequency\n        batch_size: int, default=32\n            batch size\n        n_train_samples: int, default=2\n            number of samples to average over during the training process.\n            See [Weight Uncertainty in Neural Networks](https://arxiv.org/pdf/1505.05424.pdf) for details.\n        n_val_samples: int, default=2\n            number of samples to average over during the validation process.\n            See [Weight Uncertainty in Neural Networks](https://arxiv.org/pdf/1505.05424.pdf) for details.\n        \"\"\"\n\n        self.batch_size = batch_size\n\n        train_set, eval_set = tabular_train_val_split(\n            self.seed, self.objective, X_tab, target, X_tab_val, target_val, val_split\n        )\n        train_loader = DataLoader(\n            dataset=train_set, batch_size=batch_size, num_workers=self.num_workers\n        )\n        train_steps = len(train_loader)\n\n        if eval_set is not None:\n            eval_loader = DataLoader(\n                dataset=eval_set,\n                batch_size=batch_size,\n                num_workers=self.num_workers,\n                shuffle=False,\n            )\n            eval_steps = len(eval_loader)\n\n        self.callback_container.on_train_begin(\n            {\n                \"batch_size\": batch_size,\n                \"train_steps\": train_steps,\n                \"n_epochs\": n_epochs,\n            }\n        )\n        for epoch in range(n_epochs):\n            epoch_logs: Dict[str, float] = {}\n            self.callback_container.on_epoch_begin(epoch, logs=epoch_logs)\n\n            self.train_running_loss = 0.0\n            with trange(train_steps, disable=self.verbose != 1) as t:\n                for batch_idx, (X, y) in zip(t, train_loader):\n                    t.set_description(\"epoch %i\" % (epoch + 1))\n                    train_score, train_loss = self._train_step(\n                        X, y, n_train_samples, train_steps, batch_idx\n                    )\n                    print_loss_and_metric(t, train_loss, train_score)\n                    self.callback_container.on_batch_end(batch=batch_idx)\n            epoch_logs = save_epoch_logs(epoch_logs, train_loss, train_score, \"train\")\n\n            on_epoch_end_metric = None\n            if eval_set is not None and epoch % validation_freq == (\n                validation_freq - 1\n            ):\n                self.callback_container.on_eval_begin()\n                self.valid_running_loss = 0.0\n                with trange(eval_steps, disable=self.verbose != 1) as v:\n                    for i, (X, y) in zip(v, eval_loader):\n                        v.set_description(\"valid\")\n                        val_score, val_loss = self._eval_step(\n                            X, y, n_val_samples, train_steps, i\n                        )\n                        print_loss_and_metric(v, val_loss, val_score)\n                epoch_logs = save_epoch_logs(epoch_logs, val_loss, val_score, \"val\")\n\n                if self.reducelronplateau:\n                    if self.reducelronplateau_criterion == \"loss\":\n                        on_epoch_end_metric = val_loss\n                    else:\n                        on_epoch_end_metric = val_score[\n                            self.reducelronplateau_criterion\n                        ]\n\n            self.callback_container.on_epoch_end(epoch, epoch_logs, on_epoch_end_metric)\n\n            if self.early_stop:\n                self.callback_container.on_train_end(epoch_logs)\n                break\n\n        self.callback_container.on_train_end(epoch_logs)\n        self._restore_best_weights()\n        self.model.train()\n\n    def predict(  # type: ignore[return]\n        self,\n        X_tab: np.ndarray,\n        n_samples: int = 5,\n        return_samples: bool = False,\n        batch_size: int = 256,\n    ) -&gt; np.ndarray:\n        r\"\"\"Returns the predictions\n\n        Parameters\n        ----------\n        X_tab: np.ndarray,\n            tabular dataset\n        n_samples: int, default=5\n            number of samples that will be either returned or averaged to\n            produce an overal prediction\n        return_samples: bool, default = False\n            Boolean indicating whether the n samples will be averaged or directly returned\n        batch_size: int, default = 256\n            batch size\n\n        Returns\n        -------\n        np.ndarray:\n            array with the predictions\n        \"\"\"\n\n        preds_l = self._predict(X_tab, n_samples, return_samples, batch_size)\n        preds = np.hstack(preds_l) if return_samples else np.vstack(preds_l)\n        axis = 2 if return_samples else 1\n\n        if self.objective == \"regression\":\n            return preds.squeeze(axis)\n        if self.objective == \"binary\":\n            return (preds.squeeze(axis) &gt; 0.5).astype(\"int\")\n        if self.objective == \"multiclass\":\n            return np.argmax(preds, axis)\n\n    def predict_proba(  # type: ignore[return]\n        self,\n        X_tab: np.ndarray,\n        n_samples: int = 5,\n        return_samples: bool = False,\n        batch_size: int = 256,\n    ) -&gt; np.ndarray:\n        r\"\"\"Returns the predicted probabilities\n\n        Parameters\n        ----------\n        X_tab: np.ndarray,\n            tabular dataset\n        n_samples: int, default=5\n            number of samples that will be either returned or averaged to\n            produce an overal prediction\n        return_samples: bool, default = False\n            Boolean indicating whether the n samples will be averaged or directly returned\n        batch_size: int, default = 256\n            batch size\n\n        Returns\n        -------\n        np.ndarray\n            array with the probabilities per class\n        \"\"\"\n        preds_l = self._predict(X_tab, n_samples, return_samples, batch_size)\n        preds = np.hstack(preds_l) if return_samples else np.vstack(preds_l)\n\n        if self.objective == \"binary\":\n            if return_samples:\n                preds = preds.squeeze(2)\n                probs = np.zeros([n_samples, preds.shape[1], 2])\n                for i in range(n_samples):\n                    probs[i, :, 0] = 1 - preds[i]\n                    probs[i, :, 1] = preds[i]\n            else:\n                preds = preds.squeeze(1)\n                probs = np.zeros([preds.shape[0], 2])\n                probs[:, 0] = 1 - preds\n                probs[:, 1] = preds\n            return probs\n        if self.objective == \"multiclass\":\n            return preds\n\n    def save(\n        self,\n        path: str,\n        save_state_dict: bool = False,\n        model_filename: str = \"bayesian_model.pt\",\n    ):\n        r\"\"\"Saves the model, training and evaluation history to disk\n\n        The `Trainer` class is built so that it 'just' trains a model. With\n        that in mind, all the torch related parameters (such as optimizers or\n        learning rate schedulers) have to be defined externally and then\n        passed to the `Trainer`. As a result, the `Trainer` does not\n        generate any attribute or additional data products that need to be\n        saved other than the `model` object itself, which can be saved as\n        any other torch model (e.g. `torch.save(model, path)`).\n\n        Parameters\n        ----------\n        path: str\n            path to the directory where the model and the feature importance\n            attribute will be saved.\n        save_state_dict: bool, default = False\n            Boolean indicating whether to save directly the model or the\n            model's state dictionary\n        model_filename: str, Optional, default = \"wd_model.pt\"\n            filename where the model weights will be store\n        \"\"\"\n\n        save_dir = Path(path)\n        history_dir = save_dir / \"history\"\n        history_dir.mkdir(exist_ok=True, parents=True)\n\n        # the trainer is run with the History Callback by default\n        with open(history_dir / \"train_eval_history.json\", \"w\") as teh:\n            json.dump(self.history, teh)  # type: ignore[attr-defined]\n\n        has_lr_history = any(\n            [clbk.__class__.__name__ == \"LRHistory\" for clbk in self.callbacks]\n        )\n        if self.lr_scheduler is not None and has_lr_history:\n            with open(history_dir / \"lr_history.json\", \"w\") as lrh:\n                json.dump(self.lr_history, lrh)  # type: ignore[attr-defined]\n\n        model_path = save_dir / model_filename\n        if save_state_dict:\n            torch.save(self.model.state_dict(), model_path)\n        else:\n            torch.save(self.model, model_path)\n\n    def _train_step(\n        self,\n        X_tab: Tensor,\n        target: Tensor,\n        n_samples: int,\n        n_batches: int,\n        batch_idx: int,\n    ):\n        self.model.train()\n\n        X = to_device(X_tab, self.device)\n        y = target.view(-1, 1).float() if self.objective != \"multiclass\" else target\n        y = to_device(y, self.device)\n\n        self.optimizer.zero_grad()\n        y_pred, loss = self.model.sample_elbo(X, y, self.loss_fn, n_samples, n_batches)  # type: ignore[arg-type]\n\n        y_pred = y_pred.mean(dim=0)\n        score = self._get_score(y_pred, y)\n\n        loss.backward()\n        self.optimizer.step()\n\n        self.train_running_loss += loss.item()\n        avg_loss = self.train_running_loss / (batch_idx + 1)\n\n        return score, avg_loss\n\n    def _eval_step(\n        self,\n        X_tab: Tensor,\n        target: Tensor,\n        n_samples: int,\n        n_batches: int,\n        batch_idx: int,\n    ):\n        self.model.eval()\n        with torch.no_grad():\n            X = to_device(X_tab, self.device)\n            y = target.view(-1, 1).float() if self.objective != \"multiclass\" else target\n            y = to_device(y, self.device)\n\n            y_pred, loss = self.model.sample_elbo(\n                X,  # type: ignore[arg-type]\n                y,\n                self.loss_fn,\n                n_samples,\n                n_batches,\n            )\n            y_pred = y_pred.mean(dim=0)\n            score = self._get_score(y_pred, y)\n\n            self.valid_running_loss += loss.item()\n            avg_loss = self.valid_running_loss / (batch_idx + 1)\n\n        return score, avg_loss\n\n    def _get_score(self, y_pred, y):\n        if self.metric is not None:\n            if self.objective == \"regression\":\n                score = self.metric(y_pred, y)\n            if self.objective == \"binary\":\n                score = self.metric(torch.sigmoid(y_pred), y)\n            if self.objective == \"multiclass\":\n                score = self.metric(F.softmax(y_pred, dim=1), y)\n            return score\n        else:\n            return None\n\n    def _predict(  # noqa: C901\n        self,\n        X_tab: np.ndarray,\n        n_samples: int = 5,\n        return_samples: bool = False,\n        batch_size: int = 256,\n    ) -&gt; List:\n        self.batch_size = batch_size\n\n        test_set = TensorDataset(torch.from_numpy(X_tab))\n        test_loader = DataLoader(\n            dataset=test_set,\n            batch_size=self.batch_size,\n            num_workers=self.num_workers,\n            shuffle=False,\n        )\n        test_steps = (len(test_loader.dataset) // test_loader.batch_size) + 1  # type: ignore[arg-type]\n\n        preds_l = []\n        with torch.no_grad():\n            with trange(test_steps, disable=self.verbose != 1) as tt:\n                for _, Xl in zip(tt, test_loader):\n                    tt.set_description(\"predict\")\n\n                    try:\n                        X = Xl[0].to(self.device)\n                    except TypeError:\n                        X = Xl[0].to(self.device, dtype=torch.float32)\n\n                    if return_samples:\n                        preds = torch.stack([self.model(X) for _ in range(n_samples)])\n                    else:\n                        self.model.eval()\n                        preds = self.model(X)\n\n                    if self.objective == \"binary\":\n                        preds = torch.sigmoid(preds)\n                    if self.objective == \"multiclass\":\n                        preds = (\n                            F.softmax(preds, dim=2)\n                            if return_samples\n                            else F.softmax(preds, dim=1)\n                        )\n\n                    preds_cpu = preds.cpu().data.numpy()\n                    preds_l.append(preds_cpu)\n\n        self.model.train()\n\n        return preds_l\n</code></pre>"},{"location":"pytorch-widedeep/bayesian_trainer.html#pytorch_widedeep.training.BayesianTrainer.fit","title":"fit","text":"<pre><code>fit(\n    X_tab,\n    target,\n    X_tab_val=None,\n    target_val=None,\n    val_split=None,\n    n_epochs=1,\n    validation_freq=1,\n    batch_size=32,\n    n_train_samples=2,\n    n_val_samples=2,\n)\n</code></pre> <p>Fit method.</p> <p>Parameters:</p> Name Type Description Default <code>X_tab</code> <code>ndarray</code> <p>tabular dataset</p> required <code>target</code> <code>ndarray</code> <p>target values</p> required <code>X_tab_val</code> <code>Optional[ndarray]</code> <p>validation data</p> <code>None</code> <code>target_val</code> <code>Optional[ndarray]</code> <p>validation target values</p> <code>None</code> <code>val_split</code> <code>Optional[float]</code> <p>An alterative to passing the validation set is to use a train/val split fraction via <code>val_split</code></p> <code>None</code> <code>n_epochs</code> <code>int</code> <p>number of epochs</p> <code>1</code> <code>validation_freq</code> <code>int</code> <p>epochs validation frequency</p> <code>1</code> <code>batch_size</code> <code>int</code> <p>batch size</p> <code>32</code> <code>n_train_samples</code> <code>int</code> <p>number of samples to average over during the training process. See Weight Uncertainty in Neural Networks for details.</p> <code>2</code> <code>n_val_samples</code> <code>int</code> <p>number of samples to average over during the validation process. See Weight Uncertainty in Neural Networks for details.</p> <code>2</code> Source code in <code>pytorch_widedeep/training/bayesian_trainer.py</code> <pre><code>def fit(  # noqa: C901\n    self,\n    X_tab: np.ndarray,\n    target: np.ndarray,\n    X_tab_val: Optional[np.ndarray] = None,\n    target_val: Optional[np.ndarray] = None,\n    val_split: Optional[float] = None,\n    n_epochs: int = 1,\n    validation_freq: int = 1,\n    batch_size: int = 32,\n    n_train_samples: int = 2,\n    n_val_samples: int = 2,\n):\n    r\"\"\"Fit method.\n\n    Parameters\n    ----------\n    X_tab: np.ndarray,\n        tabular dataset\n    target: np.ndarray\n        target values\n    X_tab_val: np.ndarray, Optional, default = None\n        validation data\n    target_val: np.ndarray, Optional, default = None\n        validation target values\n    val_split: float, Optional. default=None\n        An alterative to passing the validation set is to use a train/val\n        split fraction via `val_split`\n    n_epochs: int, default=1\n        number of epochs\n    validation_freq: int, default=1\n        epochs validation frequency\n    batch_size: int, default=32\n        batch size\n    n_train_samples: int, default=2\n        number of samples to average over during the training process.\n        See [Weight Uncertainty in Neural Networks](https://arxiv.org/pdf/1505.05424.pdf) for details.\n    n_val_samples: int, default=2\n        number of samples to average over during the validation process.\n        See [Weight Uncertainty in Neural Networks](https://arxiv.org/pdf/1505.05424.pdf) for details.\n    \"\"\"\n\n    self.batch_size = batch_size\n\n    train_set, eval_set = tabular_train_val_split(\n        self.seed, self.objective, X_tab, target, X_tab_val, target_val, val_split\n    )\n    train_loader = DataLoader(\n        dataset=train_set, batch_size=batch_size, num_workers=self.num_workers\n    )\n    train_steps = len(train_loader)\n\n    if eval_set is not None:\n        eval_loader = DataLoader(\n            dataset=eval_set,\n            batch_size=batch_size,\n            num_workers=self.num_workers,\n            shuffle=False,\n        )\n        eval_steps = len(eval_loader)\n\n    self.callback_container.on_train_begin(\n        {\n            \"batch_size\": batch_size,\n            \"train_steps\": train_steps,\n            \"n_epochs\": n_epochs,\n        }\n    )\n    for epoch in range(n_epochs):\n        epoch_logs: Dict[str, float] = {}\n        self.callback_container.on_epoch_begin(epoch, logs=epoch_logs)\n\n        self.train_running_loss = 0.0\n        with trange(train_steps, disable=self.verbose != 1) as t:\n            for batch_idx, (X, y) in zip(t, train_loader):\n                t.set_description(\"epoch %i\" % (epoch + 1))\n                train_score, train_loss = self._train_step(\n                    X, y, n_train_samples, train_steps, batch_idx\n                )\n                print_loss_and_metric(t, train_loss, train_score)\n                self.callback_container.on_batch_end(batch=batch_idx)\n        epoch_logs = save_epoch_logs(epoch_logs, train_loss, train_score, \"train\")\n\n        on_epoch_end_metric = None\n        if eval_set is not None and epoch % validation_freq == (\n            validation_freq - 1\n        ):\n            self.callback_container.on_eval_begin()\n            self.valid_running_loss = 0.0\n            with trange(eval_steps, disable=self.verbose != 1) as v:\n                for i, (X, y) in zip(v, eval_loader):\n                    v.set_description(\"valid\")\n                    val_score, val_loss = self._eval_step(\n                        X, y, n_val_samples, train_steps, i\n                    )\n                    print_loss_and_metric(v, val_loss, val_score)\n            epoch_logs = save_epoch_logs(epoch_logs, val_loss, val_score, \"val\")\n\n            if self.reducelronplateau:\n                if self.reducelronplateau_criterion == \"loss\":\n                    on_epoch_end_metric = val_loss\n                else:\n                    on_epoch_end_metric = val_score[\n                        self.reducelronplateau_criterion\n                    ]\n\n        self.callback_container.on_epoch_end(epoch, epoch_logs, on_epoch_end_metric)\n\n        if self.early_stop:\n            self.callback_container.on_train_end(epoch_logs)\n            break\n\n    self.callback_container.on_train_end(epoch_logs)\n    self._restore_best_weights()\n    self.model.train()\n</code></pre>"},{"location":"pytorch-widedeep/bayesian_trainer.html#pytorch_widedeep.training.BayesianTrainer.predict","title":"predict","text":"<pre><code>predict(\n    X_tab, n_samples=5, return_samples=False, batch_size=256\n)\n</code></pre> <p>Returns the predictions</p> <p>Parameters:</p> Name Type Description Default <code>X_tab</code> <code>ndarray</code> <p>tabular dataset</p> required <code>n_samples</code> <code>int</code> <p>number of samples that will be either returned or averaged to produce an overal prediction</p> <code>5</code> <code>return_samples</code> <code>bool</code> <p>Boolean indicating whether the n samples will be averaged or directly returned</p> <code>False</code> <code>batch_size</code> <code>int</code> <p>batch size</p> <code>256</code> <p>Returns:</p> Type Description <code>np.ndarray:</code> <p>array with the predictions</p> Source code in <code>pytorch_widedeep/training/bayesian_trainer.py</code> <pre><code>def predict(  # type: ignore[return]\n    self,\n    X_tab: np.ndarray,\n    n_samples: int = 5,\n    return_samples: bool = False,\n    batch_size: int = 256,\n) -&gt; np.ndarray:\n    r\"\"\"Returns the predictions\n\n    Parameters\n    ----------\n    X_tab: np.ndarray,\n        tabular dataset\n    n_samples: int, default=5\n        number of samples that will be either returned or averaged to\n        produce an overal prediction\n    return_samples: bool, default = False\n        Boolean indicating whether the n samples will be averaged or directly returned\n    batch_size: int, default = 256\n        batch size\n\n    Returns\n    -------\n    np.ndarray:\n        array with the predictions\n    \"\"\"\n\n    preds_l = self._predict(X_tab, n_samples, return_samples, batch_size)\n    preds = np.hstack(preds_l) if return_samples else np.vstack(preds_l)\n    axis = 2 if return_samples else 1\n\n    if self.objective == \"regression\":\n        return preds.squeeze(axis)\n    if self.objective == \"binary\":\n        return (preds.squeeze(axis) &gt; 0.5).astype(\"int\")\n    if self.objective == \"multiclass\":\n        return np.argmax(preds, axis)\n</code></pre>"},{"location":"pytorch-widedeep/bayesian_trainer.html#pytorch_widedeep.training.BayesianTrainer.predict_proba","title":"predict_proba","text":"<pre><code>predict_proba(\n    X_tab, n_samples=5, return_samples=False, batch_size=256\n)\n</code></pre> <p>Returns the predicted probabilities</p> <p>Parameters:</p> Name Type Description Default <code>X_tab</code> <code>ndarray</code> <p>tabular dataset</p> required <code>n_samples</code> <code>int</code> <p>number of samples that will be either returned or averaged to produce an overal prediction</p> <code>5</code> <code>return_samples</code> <code>bool</code> <p>Boolean indicating whether the n samples will be averaged or directly returned</p> <code>False</code> <code>batch_size</code> <code>int</code> <p>batch size</p> <code>256</code> <p>Returns:</p> Type Description <code>ndarray</code> <p>array with the probabilities per class</p> Source code in <code>pytorch_widedeep/training/bayesian_trainer.py</code> <pre><code>def predict_proba(  # type: ignore[return]\n    self,\n    X_tab: np.ndarray,\n    n_samples: int = 5,\n    return_samples: bool = False,\n    batch_size: int = 256,\n) -&gt; np.ndarray:\n    r\"\"\"Returns the predicted probabilities\n\n    Parameters\n    ----------\n    X_tab: np.ndarray,\n        tabular dataset\n    n_samples: int, default=5\n        number of samples that will be either returned or averaged to\n        produce an overal prediction\n    return_samples: bool, default = False\n        Boolean indicating whether the n samples will be averaged or directly returned\n    batch_size: int, default = 256\n        batch size\n\n    Returns\n    -------\n    np.ndarray\n        array with the probabilities per class\n    \"\"\"\n    preds_l = self._predict(X_tab, n_samples, return_samples, batch_size)\n    preds = np.hstack(preds_l) if return_samples else np.vstack(preds_l)\n\n    if self.objective == \"binary\":\n        if return_samples:\n            preds = preds.squeeze(2)\n            probs = np.zeros([n_samples, preds.shape[1], 2])\n            for i in range(n_samples):\n                probs[i, :, 0] = 1 - preds[i]\n                probs[i, :, 1] = preds[i]\n        else:\n            preds = preds.squeeze(1)\n            probs = np.zeros([preds.shape[0], 2])\n            probs[:, 0] = 1 - preds\n            probs[:, 1] = preds\n        return probs\n    if self.objective == \"multiclass\":\n        return preds\n</code></pre>"},{"location":"pytorch-widedeep/bayesian_trainer.html#pytorch_widedeep.training.BayesianTrainer.save","title":"save","text":"<pre><code>save(\n    path,\n    save_state_dict=False,\n    model_filename=\"bayesian_model.pt\",\n)\n</code></pre> <p>Saves the model, training and evaluation history to disk</p> <p>The <code>Trainer</code> class is built so that it 'just' trains a model. With that in mind, all the torch related parameters (such as optimizers or learning rate schedulers) have to be defined externally and then passed to the <code>Trainer</code>. As a result, the <code>Trainer</code> does not generate any attribute or additional data products that need to be saved other than the <code>model</code> object itself, which can be saved as any other torch model (e.g. <code>torch.save(model, path)</code>).</p> <p>Parameters:</p> Name Type Description Default <code>path</code> <code>str</code> <p>path to the directory where the model and the feature importance attribute will be saved.</p> required <code>save_state_dict</code> <code>bool</code> <p>Boolean indicating whether to save directly the model or the model's state dictionary</p> <code>False</code> <code>model_filename</code> <code>str</code> <p>filename where the model weights will be store</p> <code>'bayesian_model.pt'</code> Source code in <code>pytorch_widedeep/training/bayesian_trainer.py</code> <pre><code>def save(\n    self,\n    path: str,\n    save_state_dict: bool = False,\n    model_filename: str = \"bayesian_model.pt\",\n):\n    r\"\"\"Saves the model, training and evaluation history to disk\n\n    The `Trainer` class is built so that it 'just' trains a model. With\n    that in mind, all the torch related parameters (such as optimizers or\n    learning rate schedulers) have to be defined externally and then\n    passed to the `Trainer`. As a result, the `Trainer` does not\n    generate any attribute or additional data products that need to be\n    saved other than the `model` object itself, which can be saved as\n    any other torch model (e.g. `torch.save(model, path)`).\n\n    Parameters\n    ----------\n    path: str\n        path to the directory where the model and the feature importance\n        attribute will be saved.\n    save_state_dict: bool, default = False\n        Boolean indicating whether to save directly the model or the\n        model's state dictionary\n    model_filename: str, Optional, default = \"wd_model.pt\"\n        filename where the model weights will be store\n    \"\"\"\n\n    save_dir = Path(path)\n    history_dir = save_dir / \"history\"\n    history_dir.mkdir(exist_ok=True, parents=True)\n\n    # the trainer is run with the History Callback by default\n    with open(history_dir / \"train_eval_history.json\", \"w\") as teh:\n        json.dump(self.history, teh)  # type: ignore[attr-defined]\n\n    has_lr_history = any(\n        [clbk.__class__.__name__ == \"LRHistory\" for clbk in self.callbacks]\n    )\n    if self.lr_scheduler is not None and has_lr_history:\n        with open(history_dir / \"lr_history.json\", \"w\") as lrh:\n            json.dump(self.lr_history, lrh)  # type: ignore[attr-defined]\n\n    model_path = save_dir / model_filename\n    if save_state_dict:\n        torch.save(self.model.state_dict(), model_path)\n    else:\n        torch.save(self.model, model_path)\n</code></pre>"},{"location":"pytorch-widedeep/callbacks.html","title":"Callbacks","text":"<p>Here are the 4 callbacks available to the user in <code>pytorch-widedepp</code>: <code>LRHistory</code>, <code>ModelCheckpoint</code>, <code>EarlyStopping</code> and <code>RayTuneReporter</code>.</p> <p> NOTE: other callbacks , like <code>History</code>, run always  by default. In particular, the <code>History</code> callback saves the metrics in the  <code>history</code> attribute of the <code>Trainer</code>.</p>"},{"location":"pytorch-widedeep/callbacks.html#pytorch_widedeep.callbacks.LRHistory","title":"LRHistory","text":"<p>               Bases: <code>Callback</code></p> <p>Saves the learning rates during training in the <code>lr_history</code> attribute of the <code>Trainer</code>.</p> <p>Callbacks are passed as input parameters to the <code>Trainer</code> class. See <code>pytorch_widedeep.trainer.Trainer</code></p> <p>Parameters:</p> Name Type Description Default <code>n_epochs</code> <code>int</code> <p>number of training epochs</p> required <p>Examples:</p> <pre><code>&gt;&gt;&gt; from pytorch_widedeep.callbacks import LRHistory\n&gt;&gt;&gt; from pytorch_widedeep.models import TabMlp, Wide, WideDeep\n&gt;&gt;&gt; from pytorch_widedeep.training import Trainer\n&gt;&gt;&gt;\n&gt;&gt;&gt; embed_input = [(u, i, j) for u, i, j in zip([\"a\", \"b\", \"c\"][:4], [4] * 3, [8] * 3)]\n&gt;&gt;&gt; column_idx = {k: v for v, k in enumerate([\"a\", \"b\", \"c\"])}\n&gt;&gt;&gt; wide = Wide(10, 1)\n&gt;&gt;&gt; deep = TabMlp(mlp_hidden_dims=[8, 4], column_idx=column_idx, cat_embed_input=embed_input)\n&gt;&gt;&gt; model = WideDeep(wide, deep)\n&gt;&gt;&gt; trainer = Trainer(model, objective=\"regression\", callbacks=[LRHistory(n_epochs=10)])\n</code></pre> Source code in <code>pytorch_widedeep/callbacks.py</code> <pre><code>class LRHistory(Callback):\n    r\"\"\"Saves the learning rates during training in the `lr_history` attribute\n    of the `Trainer`.\n\n    Callbacks are passed as input parameters to the `Trainer` class. See\n    `pytorch_widedeep.trainer.Trainer`\n\n    Parameters\n    ----------\n    n_epochs: int\n        number of training epochs\n\n    Examples\n    --------\n    &gt;&gt;&gt; from pytorch_widedeep.callbacks import LRHistory\n    &gt;&gt;&gt; from pytorch_widedeep.models import TabMlp, Wide, WideDeep\n    &gt;&gt;&gt; from pytorch_widedeep.training import Trainer\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; embed_input = [(u, i, j) for u, i, j in zip([\"a\", \"b\", \"c\"][:4], [4] * 3, [8] * 3)]\n    &gt;&gt;&gt; column_idx = {k: v for v, k in enumerate([\"a\", \"b\", \"c\"])}\n    &gt;&gt;&gt; wide = Wide(10, 1)\n    &gt;&gt;&gt; deep = TabMlp(mlp_hidden_dims=[8, 4], column_idx=column_idx, cat_embed_input=embed_input)\n    &gt;&gt;&gt; model = WideDeep(wide, deep)\n    &gt;&gt;&gt; trainer = Trainer(model, objective=\"regression\", callbacks=[LRHistory(n_epochs=10)])\n    \"\"\"\n\n    def __init__(self, n_epochs: int):\n        super(LRHistory, self).__init__()\n        self.n_epochs = n_epochs\n\n    def on_epoch_begin(self, epoch: int, logs: Optional[Dict] = None):\n        if epoch == 0 and self.trainer.lr_scheduler is not None:\n            self.trainer.lr_history = {}\n            if self._multiple_scheduler(self.trainer.lr_scheduler):\n                self._save_group_lr_mulitple_scheduler(step_location=\"on_epoch_begin\")\n            else:\n                self._save_group_lr(self.trainer.optimizer)\n\n    def on_batch_end(self, batch: int, logs: Optional[Dict] = None):\n        if self.trainer.lr_scheduler is not None:\n            if self._multiple_scheduler(self.trainer.lr_scheduler):\n                self._save_group_lr_mulitple_scheduler(step_location=\"on_batch_end\")\n            elif self.trainer.cyclic_lr:\n                self._save_group_lr(self.trainer.optimizer)\n\n    def on_epoch_end(\n        self, epoch: int, logs: Optional[Dict] = None, metric: Optional[float] = None\n    ):\n        if epoch != (self.n_epochs - 1) and self.trainer.lr_scheduler is not None:\n            if self._multiple_scheduler(self.trainer.lr_scheduler):\n                self._save_group_lr_mulitple_scheduler(step_location=\"on_epoch_end\")\n            elif not self.trainer.cyclic_lr:\n                self._save_group_lr(self.trainer.optimizer)\n\n    def _save_group_lr_mulitple_scheduler(self, step_location: str):\n        for model_name, optimizer in self.trainer.optimizer._optimizers.items():\n            if isinstance(optimizer, list):\n                # then, if it has schedulers, we assume it has to have the\n                # same number of schedulers as optimizers\n                for i, opt in enumerate(optimizer):\n                    if (\n                        step_location == \"on_epoch_begin\"\n                        or (\n                            step_location == \"on_batch_end\"\n                            and self._has_cyclic_scheduler(model_name)\n                        )\n                        or (\n                            step_location == \"on_epoch_end\"\n                            and not self._has_cyclic_scheduler(model_name)\n                        )\n                    ):\n                        self._save_group_lr(opt, model_name, \"_\".join([\"opt\", str(i)]))\n                else:\n                    # do nothing\n                    pass\n            else:\n                if (\n                    step_location == \"on_epoch_begin\"\n                    or (\n                        step_location == \"on_batch_end\"\n                        and self._has_cyclic_scheduler(model_name)\n                    )\n                    or (\n                        step_location == \"on_epoch_end\"\n                        and not self._has_cyclic_scheduler(model_name)\n                    )\n                ):\n                    self._save_group_lr(optimizer, model_name)\n\n    def _save_group_lr(\n        self,\n        opt: Optimizer,\n        suffix: Optional[str] = None,\n        model_name: Optional[str] = None,\n    ):\n        suffix = suffix or \"\"\n        model_name = model_name or \"\"\n        for group_idx, group in enumerate(opt.param_groups):\n            group_name = (\"_\").join(\n                [x for x in [\"lr\", model_name, suffix, str(group_idx)] if x]\n            )\n            self.trainer.lr_history.setdefault(group_name, []).append(group[\"lr\"])\n\n    @staticmethod\n    def _multiple_scheduler(scheduler: LRScheduler) -&gt; bool:\n        return scheduler.__class__.__name__ == \"MultipleLRScheduler\"\n\n    def _has_cyclic_scheduler(self, model_name: str):\n        if model_name in self.trainer.lr_scheduler._schedulers:\n            if isinstance(self.trainer.lr_scheduler._schedulers[model_name], list):\n                return any(\n                    [\n                        self._is_cyclic(s)\n                        for s in self.trainer.lr_scheduler._schedulers[model_name]\n                    ]\n                )\n            else:\n                return self._is_cyclic(\n                    self.trainer.lr_scheduler._schedulers[model_name]\n                )\n\n    @staticmethod\n    def _is_cyclic(scheduler: LRScheduler) -&gt; bool:\n        return \"cycl\" in scheduler.__class__.__name__.lower()\n</code></pre>"},{"location":"pytorch-widedeep/callbacks.html#pytorch_widedeep.callbacks.ModelCheckpoint","title":"ModelCheckpoint","text":"<p>               Bases: <code>Callback</code></p> <p>Saves the model after every epoch.</p> <p>This class is almost identical to the corresponding keras class. Therefore, credit to the Keras Team.</p> <p>Callbacks are passed as input parameters to the <code>Trainer</code> class. See <code>pytorch_widedeep.trainer.Trainer</code></p> <p>Parameters:</p> Name Type Description Default <code>filepath</code> <code>Optional[str]</code> <p>Full path to save the output weights. It must contain only the root of the filenames. Epoch number and <code>.pt</code> extension (for pytorch) will be added. e.g. <code>filepath=\"path/to/output_weights/weights_out\"</code> And the saved files in that directory will be named: 'weights_out_1.pt', 'weights_out_2.pt', .... If set to <code>None</code> the class just report best metric and best_epoch.</p> <code>None</code> <code>monitor</code> <code>str</code> <p>quantity to monitor. Typically 'val_loss' or metric name (e.g. 'val_acc')</p> <code>'val_loss'</code> <code>min_delta</code> <code>float</code> <p>minimum change in the monitored quantity to qualify as an improvement, i.e. an absolute change of less than min_delta, will count as no improvement.</p> <code>0.0</code> <code>verbose</code> <code>int</code> <p>verbosity mode</p> <code>0</code> <code>save_best_only</code> <code>bool</code> <p>the latest best model according to the quantity monitored will not be overwritten.</p> <code>False</code> <code>mode</code> <code>str</code> <p>If <code>save_best_only=True</code>, the decision to overwrite the current save file is made based on either the maximization or the minimization of the monitored quantity. For 'acc', this should be 'max', for 'loss' this should be 'min', etc. In 'auto' mode, the direction is automatically inferred from the name of the monitored quantity.</p> <code>'auto'</code> <code>period</code> <code>int</code> <p>Interval (number of epochs) between checkpoints.</p> <code>1</code> <code>max_save</code> <code>int</code> <p>Maximum number of outputs to save. If -1 will save all outputs</p> <code>-1</code> <p>Attributes:</p> Name Type Description <code>best</code> <code>float</code> <p>best metric</p> <code>best_epoch</code> <code>int</code> <p>best epoch</p> <code>best_state_dict</code> <code>dict</code> <p>best model state dictionary. To restore model to its best state use <code>Trainer.model.load_state_dict (model_checkpoint.best_state_dict)</code> where <code>model_checkpoint</code> is an instance of the class <code>ModelCheckpoint</code>. See the Examples folder in the repo or the Examples section in this documentation for details</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; from pytorch_widedeep.callbacks import ModelCheckpoint\n&gt;&gt;&gt; from pytorch_widedeep.models import TabMlp, Wide, WideDeep\n&gt;&gt;&gt; from pytorch_widedeep.training import Trainer\n&gt;&gt;&gt;\n&gt;&gt;&gt; embed_input = [(u, i, j) for u, i, j in zip([\"a\", \"b\", \"c\"][:4], [4] * 3, [8] * 3)]\n&gt;&gt;&gt; column_idx = {k: v for v, k in enumerate([\"a\", \"b\", \"c\"])}\n&gt;&gt;&gt; wide = Wide(10, 1)\n&gt;&gt;&gt; deep = TabMlp(mlp_hidden_dims=[8, 4], column_idx=column_idx, cat_embed_input=embed_input)\n&gt;&gt;&gt; model = WideDeep(wide, deep)\n&gt;&gt;&gt; trainer = Trainer(model, objective=\"regression\", callbacks=[ModelCheckpoint(filepath='checkpoints/weights_out')])\n</code></pre> Source code in <code>pytorch_widedeep/callbacks.py</code> <pre><code>class ModelCheckpoint(Callback):\n    r\"\"\"Saves the model after every epoch.\n\n    This class is almost identical to the corresponding keras class.\n    Therefore, **credit** to the Keras Team.\n\n    Callbacks are passed as input parameters to the `Trainer` class. See\n    `pytorch_widedeep.trainer.Trainer`\n\n    Parameters\n    ----------\n    filepath: str, default=None\n        Full path to save the output weights. It must contain only the root of\n        the filenames. Epoch number and `.pt` extension (for pytorch) will be\n        added. e.g. `filepath=\"path/to/output_weights/weights_out\"` And the\n        saved files in that directory will be named:\n        _'weights_out_1.pt', 'weights_out_2.pt', ..._. If set to `None` the\n        class just report best metric and best_epoch.\n    monitor: str, default=\"loss\"\n        quantity to monitor. Typically _'val_loss'_ or metric name\n        (e.g. _'val_acc'_)\n    min_delta: float, default=0.\n        minimum change in the monitored quantity to qualify as an\n        improvement, i.e. an absolute change of less than min_delta, will\n        count as no improvement.\n    verbose:int, default=0\n        verbosity mode\n    save_best_only: bool, default=False,\n        the latest best model according to the quantity monitored will not be\n        overwritten.\n    mode: str, default=\"auto\"\n        If `save_best_only=True`, the decision to overwrite the current save\n        file is made based on either the maximization or the minimization of\n        the monitored quantity. For _'acc'_, this should be _'max'_, for\n        _'loss'_ this should be _'min'_, etc. In '_auto'_ mode, the\n        direction is automatically inferred from the name of the monitored\n        quantity.\n    period: int, default=1\n        Interval (number of epochs) between checkpoints.\n    max_save: int, default=-1\n        Maximum number of outputs to save. If -1 will save all outputs\n\n    Attributes\n    ----------\n    best: float\n        best metric\n    best_epoch: int\n        best epoch\n    best_state_dict: dict\n        best model state dictionary.&lt;br/&gt;\n        To restore model to its best state use `Trainer.model.load_state_dict\n        (model_checkpoint.best_state_dict)` where `model_checkpoint` is an\n        instance of the class `ModelCheckpoint`. See the Examples folder in\n        the repo or the Examples section in this documentation for details\n\n    Examples\n    --------\n    &gt;&gt;&gt; from pytorch_widedeep.callbacks import ModelCheckpoint\n    &gt;&gt;&gt; from pytorch_widedeep.models import TabMlp, Wide, WideDeep\n    &gt;&gt;&gt; from pytorch_widedeep.training import Trainer\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; embed_input = [(u, i, j) for u, i, j in zip([\"a\", \"b\", \"c\"][:4], [4] * 3, [8] * 3)]\n    &gt;&gt;&gt; column_idx = {k: v for v, k in enumerate([\"a\", \"b\", \"c\"])}\n    &gt;&gt;&gt; wide = Wide(10, 1)\n    &gt;&gt;&gt; deep = TabMlp(mlp_hidden_dims=[8, 4], column_idx=column_idx, cat_embed_input=embed_input)\n    &gt;&gt;&gt; model = WideDeep(wide, deep)\n    &gt;&gt;&gt; trainer = Trainer(model, objective=\"regression\", callbacks=[ModelCheckpoint(filepath='checkpoints/weights_out')])\n    \"\"\"\n\n    def __init__(\n        self,\n        filepath: Optional[str] = None,\n        monitor: str = \"val_loss\",\n        min_delta: float = 0.0,\n        verbose: int = 0,\n        save_best_only: bool = False,\n        mode: str = \"auto\",\n        period: int = 1,\n        max_save: int = -1,\n    ):\n        super(ModelCheckpoint, self).__init__()\n\n        self.filepath = filepath\n        self.monitor = monitor\n        self.min_delta = min_delta\n        self.verbose = verbose\n        self.save_best_only = save_best_only\n        self.mode = mode\n        self.period = period\n        self.max_save = max_save\n\n        self.epochs_since_last_save = 0\n\n        if self.filepath:\n            if len(self.filepath.split(\"/\")[:-1]) == 0:\n                raise ValueError(\n                    \"'filepath' must be the full path to save the output weights,\"\n                    \" including the root of the filenames. e.g. 'checkpoints/weights_out'\"\n                )\n\n            root_dir = (\"/\").join(self.filepath.split(\"/\")[:-1])\n            if not os.path.exists(root_dir):\n                os.makedirs(root_dir)\n\n        if self.max_save &gt; 0:\n            self.old_files: List[str] = []\n\n        if self.mode not in [\"auto\", \"min\", \"max\"]:\n            warnings.warn(\n                \"ModelCheckpoint mode %s is unknown, \"\n                \"fallback to auto mode.\" % (self.mode),\n                RuntimeWarning,\n            )\n            self.mode = \"auto\"\n        if self.mode == \"min\":\n            self.monitor_op = np.less\n            self.best = np.Inf\n        elif self.mode == \"max\":\n            self.monitor_op = np.greater  # type: ignore[assignment]\n            self.best = -np.Inf\n        else:\n            if _is_metric(self.monitor):\n                self.monitor_op = np.greater  # type: ignore[assignment]\n                self.best = -np.Inf\n            else:\n                self.monitor_op = np.less\n                self.best = np.Inf\n\n        if self.monitor_op == np.greater:\n            self.min_delta *= 1\n        else:\n            self.min_delta *= -1\n\n    def on_epoch_end(  # noqa: C901\n        self, epoch: int, logs: Optional[Dict] = None, metric: Optional[float] = None\n    ):\n        logs = logs or {}\n        self.epochs_since_last_save += 1\n        if self.epochs_since_last_save &gt;= self.period:\n            self.epochs_since_last_save = 0\n            if self.filepath:\n                filepath = \"{}_{}.p\".format(self.filepath, epoch + 1)\n            if self.save_best_only:\n                current = logs.get(self.monitor)\n                if current is None:\n                    warnings.warn(\n                        \"Can save best model only with %s available, \"\n                        \"skipping.\" % (self.monitor),\n                        RuntimeWarning,\n                    )\n                else:\n                    if self.monitor_op(current - self.min_delta, self.best):\n                        if self.verbose &gt; 0:\n                            if self.filepath:\n                                print(\n                                    f\"\\nEpoch {epoch + 1}: {self.monitor} improved from {self.best:.5f} to {current:.5f} \"\n                                    f\"Saving model to {filepath}\"\n                                )\n                            else:\n                                print(\n                                    f\"\\nEpoch {epoch + 1}: {self.monitor} improved from {self.best:.5f} to {current:.5f} \"\n                                )\n                        self.best = current\n                        self.best_epoch = epoch\n                        self.best_state_dict = copy.deepcopy(self.model.state_dict())\n                        if self.filepath:\n                            torch.save(self.best_state_dict, filepath)\n                            if self.max_save &gt; 0:\n                                if len(self.old_files) == self.max_save:\n                                    try:\n                                        os.remove(self.old_files[0])\n                                    except FileNotFoundError:\n                                        pass\n                                    self.old_files = self.old_files[1:]\n                                self.old_files.append(filepath)\n                    else:\n                        if self.verbose &gt; 0:\n                            print(\n                                f\"\\nEpoch {epoch + 1}: {self.monitor} did not improve from {self.best:.5f} \"\n                                f\" considering a 'min_delta' improvement of {self.min_delta:.5f}\"\n                            )\n            if not self.save_best_only and self.filepath:\n                if self.verbose &gt; 0:\n                    print(\"\\nEpoch %05d: saving model to %s\" % (epoch + 1, filepath))\n                torch.save(self.model.state_dict(), filepath)\n                if self.max_save &gt; 0:\n                    if len(self.old_files) == self.max_save:\n                        try:\n                            os.remove(self.old_files[0])\n                        except FileNotFoundError:\n                            pass\n                        self.old_files = self.old_files[1:]\n                    self.old_files.append(filepath)\n\n    def __getstate__(self):\n        d = self.__dict__\n        self_dict = {k: d[k] for k in d if k not in [\"trainer\", \"model\"]}\n        return self_dict\n\n    def __setstate__(self, state):\n        self.__dict__ = state\n</code></pre>"},{"location":"pytorch-widedeep/callbacks.html#pytorch_widedeep.callbacks.EarlyStopping","title":"EarlyStopping","text":"<p>               Bases: <code>Callback</code></p> <p>Stop training when a monitored quantity has stopped improving.</p> <p>This class is almost identical to the corresponding keras class. Therefore, credit to the Keras Team.</p> <p>Callbacks are passed as input parameters to the <code>Trainer</code> class. See <code>pytorch_widedeep.trainer.Trainer</code></p> <p>Parameters:</p> Name Type Description Default <code>monitor</code> <code>str</code> <p>Quantity to monitor. Typically 'val_loss' or metric name (e.g. 'val_acc')</p> <code>'val_loss'</code> <code>min_delta</code> <code>float</code> <p>minimum change in the monitored quantity to qualify as an improvement, i.e. an absolute change of less than min_delta, will count as no improvement.</p> <code>0.0</code> <code>patience</code> <code>int</code> <p>Number of epochs that produced the monitored quantity with no improvement after which training will be stopped.</p> <code>10</code> <code>verbose</code> <code>int</code> <p>verbosity mode.</p> <code>0</code> <code>mode</code> <code>str</code> <p>one of {'auto', 'min', 'max'}. In 'min' mode, training will stop when the quantity monitored has stopped decreasing; in 'max' mode it will stop when the quantity monitored has stopped increasing; in 'auto' mode, the direction is automatically inferred from the name of the monitored quantity.</p> <code>'auto'</code> <code>baseline</code> <code>Optional[float]</code> <p>Baseline value for the monitored quantity to reach. Training will stop if the model does not show improvement over the baseline.</p> <code>None</code> <code>restore_best_weights</code> <code>bool</code> <p>Whether to restore model weights from the epoch with the best value of the monitored quantity. If <code>False</code>, the model weights obtained at the last step of training are used.</p> <code>False</code> <p>Attributes:</p> Name Type Description <code>best</code> <code>float</code> <p>best metric</p> <code>stopped_epoch</code> <code>int</code> <p>epoch when the training stopped</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; from pytorch_widedeep.callbacks import EarlyStopping\n&gt;&gt;&gt; from pytorch_widedeep.models import TabMlp, Wide, WideDeep\n&gt;&gt;&gt; from pytorch_widedeep.training import Trainer\n&gt;&gt;&gt;\n&gt;&gt;&gt; embed_input = [(u, i, j) for u, i, j in zip([\"a\", \"b\", \"c\"][:4], [4] * 3, [8] * 3)]\n&gt;&gt;&gt; column_idx = {k: v for v, k in enumerate([\"a\", \"b\", \"c\"])}\n&gt;&gt;&gt; wide = Wide(10, 1)\n&gt;&gt;&gt; deep = TabMlp(mlp_hidden_dims=[8, 4], column_idx=column_idx, cat_embed_input=embed_input)\n&gt;&gt;&gt; model = WideDeep(wide, deep)\n&gt;&gt;&gt; trainer = Trainer(model, objective=\"regression\", callbacks=[EarlyStopping(patience=10)])\n</code></pre> Source code in <code>pytorch_widedeep/callbacks.py</code> <pre><code>class EarlyStopping(Callback):\n    r\"\"\"Stop training when a monitored quantity has stopped improving.\n\n    This class is almost identical to the corresponding keras class.\n    Therefore, **credit** to the Keras Team.\n\n    Callbacks are passed as input parameters to the `Trainer` class. See\n    `pytorch_widedeep.trainer.Trainer`\n\n    Parameters\n    -----------\n    monitor: str, default='val_loss'.\n        Quantity to monitor. Typically _'val_loss'_ or metric name\n        (e.g. _'val_acc'_)\n    min_delta: float, default=0.\n        minimum change in the monitored quantity to qualify as an\n        improvement, i.e. an absolute change of less than min_delta, will\n        count as no improvement.\n    patience: int, default=10.\n        Number of epochs that produced the monitored quantity with no\n        improvement after which training will be stopped.\n    verbose: int.\n        verbosity mode.\n    mode: str, default='auto'\n        one of _{'auto', 'min', 'max'}_. In _'min'_ mode, training will\n        stop when the quantity monitored has stopped decreasing; in _'max'_\n        mode it will stop when the quantity monitored has stopped increasing;\n        in _'auto'_ mode, the direction is automatically inferred from the\n        name of the monitored quantity.\n    baseline: float, Optional. default=None.\n        Baseline value for the monitored quantity to reach. Training will\n        stop if the model does not show improvement over the baseline.\n    restore_best_weights: bool, default=None\n        Whether to restore model weights from the epoch with the best\n        value of the monitored quantity. If `False`, the model weights\n        obtained at the last step of training are used.\n\n    Attributes\n    ----------\n    best: float\n        best metric\n    stopped_epoch: int\n        epoch when the training stopped\n\n    Examples\n    --------\n    &gt;&gt;&gt; from pytorch_widedeep.callbacks import EarlyStopping\n    &gt;&gt;&gt; from pytorch_widedeep.models import TabMlp, Wide, WideDeep\n    &gt;&gt;&gt; from pytorch_widedeep.training import Trainer\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; embed_input = [(u, i, j) for u, i, j in zip([\"a\", \"b\", \"c\"][:4], [4] * 3, [8] * 3)]\n    &gt;&gt;&gt; column_idx = {k: v for v, k in enumerate([\"a\", \"b\", \"c\"])}\n    &gt;&gt;&gt; wide = Wide(10, 1)\n    &gt;&gt;&gt; deep = TabMlp(mlp_hidden_dims=[8, 4], column_idx=column_idx, cat_embed_input=embed_input)\n    &gt;&gt;&gt; model = WideDeep(wide, deep)\n    &gt;&gt;&gt; trainer = Trainer(model, objective=\"regression\", callbacks=[EarlyStopping(patience=10)])\n    \"\"\"\n\n    def __init__(\n        self,\n        monitor: str = \"val_loss\",\n        min_delta: float = 0.0,\n        patience: int = 10,\n        verbose: int = 0,\n        mode: str = \"auto\",\n        baseline: Optional[float] = None,\n        restore_best_weights: bool = False,\n    ):\n        super(EarlyStopping, self).__init__()\n\n        self.monitor = monitor\n        self.min_delta = min_delta\n        self.patience = patience\n        self.verbose = verbose\n        self.mode = mode\n        self.baseline = baseline\n        self.restore_best_weights = restore_best_weights\n\n        self.wait = 0\n        self.stopped_epoch = 0\n        self.state_dict = None\n\n        if self.mode not in [\"auto\", \"min\", \"max\"]:\n            warnings.warn(\n                \"EarlyStopping mode %s is unknown, \"\n                \"fallback to auto mode.\" % self.mode,\n                RuntimeWarning,\n            )\n            self.mode = \"auto\"\n\n        if self.mode == \"min\":\n            self.monitor_op = np.less\n        elif self.mode == \"max\":\n            self.monitor_op = np.greater  # type: ignore[assignment]\n        else:\n            if _is_metric(self.monitor):\n                self.monitor_op = np.greater  # type: ignore[assignment]\n            else:\n                self.monitor_op = np.less\n\n        if self.monitor_op == np.greater:\n            self.min_delta *= 1\n        else:\n            self.min_delta *= -1\n\n    def on_train_begin(self, logs: Optional[Dict] = None):\n        # Allow instances to be re-used\n        self.wait = 0\n        self.stopped_epoch = 0\n        if self.baseline is not None:\n            self.best = self.baseline\n        else:\n            self.best = np.Inf if self.monitor_op == np.less else -np.Inf\n\n    def on_epoch_end(\n        self, epoch: int, logs: Optional[Dict] = None, metric: Optional[float] = None\n    ):\n        current = self.get_monitor_value(logs)\n        if current is None:\n            return\n\n        if self.monitor_op(current - self.min_delta, self.best):\n            self.best = current\n            self.wait = 0\n            self.best_epoch = epoch\n            if self.restore_best_weights:\n                self.state_dict = copy.deepcopy(self.model.state_dict())\n        else:\n            self.wait += 1\n            if self.wait &gt;= self.patience:\n                self.stopped_epoch = epoch\n                self.trainer.early_stop = True\n\n    def on_train_end(self, logs: Optional[Dict] = None):\n        if self.stopped_epoch &gt; 0 and self.verbose &gt; 0:\n            print(\n                f\"Best Epoch: {self.best_epoch + 1}. Best {self.monitor}: {self.best:.5f}\"\n            )\n        if self.restore_best_weights and self.state_dict is not None:\n            if self.verbose &gt; 0:\n                print(\"Restoring model weights from the end of the best epoch\")\n            self.model.load_state_dict(self.state_dict)\n\n    def get_monitor_value(self, logs):\n        monitor_value = logs.get(self.monitor)\n        if monitor_value is None:\n            warnings.warn(\n                \"Early stopping conditioned on metric `%s` \"\n                \"which is not available. Available metrics are: %s\"\n                % (self.monitor, \",\".join(list(logs.keys()))),\n                RuntimeWarning,\n            )\n        return monitor_value\n\n    def __getstate__(self):\n        d = self.__dict__\n        self_dict = {k: d[k] for k in d if k not in [\"trainer\", \"model\"]}\n        return self_dict\n\n    def __setstate__(self, state):\n        self.__dict__ = state\n</code></pre>"},{"location":"pytorch-widedeep/dataloaders.html","title":"Dataloaders","text":"<p> NOTE: This module should contain custom dataloaders  that the user might want to implement. At the moment <code>pytorch-widedeep</code>  offers one custom dataloader, <code>DataLoaderImbalanced</code>.</p>"},{"location":"pytorch-widedeep/dataloaders.html#pytorch_widedeep.dataloaders.DataLoaderImbalanced","title":"DataLoaderImbalanced","text":"<p>               Bases: <code>CustomDataLoader</code></p> <p>Class to load and shuffle batches with adjusted weights for imbalanced datasets. If the classes do not begin from 0 remapping is necessary. See here.</p> <p>Parameters:</p> Name Type Description Default <code>dataset</code> <code>Optional[WideDeepDataset]</code> <p>see <code>pytorch_widedeep.training._wd_dataset</code></p> <code>None</code> <p>Other Parameters:</p> Name Type Description <code>*args</code> <p>Positional arguments to be passed to the parent CustomDataLoader.</p> <code>**kwargs</code> <p>This can include any parameter that can be passed to the 'standard' pytorch DataLoader and that is not already explicitely passed to the class. In addition, the dictionary can also include the extra parameter <code>oversample_mul</code> which will multiply the number of samples of the minority class to be sampled by the <code>WeightedRandomSampler</code>.</p> <p>In other words, the <code>num_samples</code> param in <code>WeightedRandomSampler</code> will be defined as:</p> \\[ minority \\space class \\space count \\times number \\space of \\space classes \\times oversample\\_mul \\] Source code in <code>pytorch_widedeep/dataloaders.py</code> <pre><code>class DataLoaderImbalanced(CustomDataLoader):\n    r\"\"\"Class to load and shuffle batches with adjusted weights for imbalanced\n    datasets. If the classes do not begin from 0 remapping is necessary. See\n    [here](https://towardsdatascience.com/pytorch-tabular-multiclass-classification-9f8211a123ab).\n\n    Parameters\n    ----------\n    dataset: `WideDeepDataset`\n        see `pytorch_widedeep.training._wd_dataset`\n\n    Other Parameters\n    ----------------\n    *args: Any\n        Positional arguments to be passed to the parent CustomDataLoader.\n    **kwargs: Dict\n        This can include any parameter that can be passed to the _'standard'_\n        pytorch\n        [DataLoader](https://pytorch.org/docs/stable/data.html#torch.utils.data.DataLoader)\n        and that is not already explicitely passed to the class. In addition,\n        the dictionary can also include the extra parameter `oversample_mul` which\n        will multiply the number of samples of the minority class to be sampled by\n        the [`WeightedRandomSampler`](https://pytorch.org/docs/stable/data.html#torch.utils.data.WeightedRandomSampler).\n\n        In other words, the `num_samples` param in `WeightedRandomSampler` will be defined as:\n\n        $$\n        minority \\space class \\space count \\times number \\space of \\space classes \\times oversample\\_mul\n        $$\n    \"\"\"\n\n    def __init__(self, dataset: Optional[WideDeepDataset] = None, *args, **kwargs):\n        self.args = args\n        self.kwargs = kwargs\n\n        if dataset is not None:\n            self._setup_sampler(dataset)\n            super().__init__(dataset, *args, sampler=self.sampler, **kwargs)\n        else:\n            super().__init__()\n\n    def set_dataset(self, dataset: WideDeepDataset):\n        sampler = self._setup_sampler(dataset)\n        # update the kwargs with the new sampler\n        self.kwargs[\"sampler\"] = sampler\n        super().set_dataset(dataset)\n\n    def _setup_sampler(self, dataset: WideDeepDataset) -&gt; WeightedRandomSampler:\n        assert dataset.Y is not None, (\n            \"The 'dataset' instance of WideDeepDataset must contain a \"\n            \"target array 'Y'\"\n        )\n\n        oversample_mul = self.kwargs.pop(\"oversample_mul\", 1)\n        weights, minor_cls_cnt, num_clss = get_class_weights(dataset)\n        num_samples = int(minor_cls_cnt * num_clss * oversample_mul)\n        samples_weight = list(np.array([weights[i] for i in dataset.Y]))\n        sampler = WeightedRandomSampler(samples_weight, num_samples, replacement=True)\n        return sampler\n</code></pre>"},{"location":"pytorch-widedeep/load_from_folder.html","title":"The <code>load_from_folder</code> module","text":"<p>The <code>load_from_folder</code> module contains the classes that are necessary to load data from disk and these are inspired by the <code>ImageFolder</code> class in the <code>torchvision</code> library. This module is designed with one specific case in mind. Such case is the following: given a multi-modal dataset with tabular data, images and text, the images do not fit in memory, and therefore, they have to be loaded from disk. However, as any other functionality in this library, there is some flexibility and some additional cases can also be addressed using this module.</p> <p>For this module to be used, the datasets must be prepared in a certain way:</p> <ol> <li> <p>the tabular data must contain a column with the images names as stored in disk, including the extension (<code>.jpg</code>, <code>.png</code>, etc...).</p> </li> <li> <p>Regarding to the text dataset, the tabular data can contain a column with the texts themselves or the names of the files containing the texts as stored in disk.</p> </li> </ol> <p>The tabular data might or might not fit in disk itself. If it does not, please see the <code>ChunkPreprocessor</code> utilities at the <code>preprocessing</code> module and the examples folder in the repo, which illustrate such case. Finally note that only <code>csv</code> format is currently supported in that case (more formats might come soon).</p>"},{"location":"pytorch-widedeep/load_from_folder.html#pytorch_widedeep.load_from_folder.tabular.tabular_from_folder.TabFromFolder","title":"TabFromFolder","text":"<p>This class is used to load tabular data from disk. The current constrains are:</p> <ol> <li>The only file format supported right now is csv</li> <li>The csv file must contain headers</li> </ol> <p>For examples, please, see the examples folder in the repo.</p> <p>Parameters:</p> Name Type Description Default <code>fname</code> <code>str</code> <p>the name of the csv file</p> required <code>directory</code> <code>Optional[str]</code> <p>the path to the directory where the csv file is located. If None, a <code>TabFromFolder</code> reference object must be provided</p> <code>None</code> <code>target_col</code> <code>Optional[str]</code> <p>the name of the target column. If None, a <code>TabFromFolder</code> reference object must be provided</p> <code>None</code> <code>preprocessor</code> <code>Optional[TabularPreprocessor]</code> <p>a fitted <code>TabularPreprocessor</code> object. If None, a <code>TabFromFolder</code> reference object must be provided</p> <code>None</code> <code>text_col</code> <code>Optional[Union[str, List[str]]]</code> <p>the name of the column with the texts themselves or the names of the files that contain the text dataset. If None, either there is no text column or a <code>TabFromFolder</code> reference object must be provided</p> <code>None</code> <code>img_col</code> <code>Optional[Union[str, List[str]]]</code> <p>the name of the column with the the names of the images. If None, either there is no image column or a <code>TabFromFolder</code> reference object must be provided</p> <code>None</code> <code>ignore_target</code> <code>bool</code> <p>whether to ignore the target column. This is normally set to True when this class is used for a test dataset.</p> <code>False</code> <code>reference</code> <code>Optional[Any]</code> <p>a reference <code>TabFromFolder</code> object. If provided, the <code>TabFromFolder</code> object will be created using the attributes of the reference object. This is useful to instantiate a <code>TabFromFolder</code> object for evaluation or test purposes</p> <code>None</code> <code>verbose</code> <code>Optional[int]</code> <p>verbosity. If 0, no output will be printed during the process.</p> <code>1</code> Source code in <code>pytorch_widedeep/load_from_folder/tabular/tabular_from_folder.py</code> <pre><code>class TabFromFolder:\n    \"\"\"\n    This class is used to load tabular data from disk. The current constrains are:\n\n    1. The only file format supported right now is csv\n    2. The csv file must contain headers\n\n    For examples, please, see the examples folder in the repo.\n\n    Parameters\n    ----------\n    fname: str\n        the name of the csv file\n    directory: str, Optional, default = None\n        the path to the directory where the csv file is located. If None,\n        a `TabFromFolder` reference object must be provided\n    target_col: str, Optional, default = None\n        the name of the target column. If None, a `TabFromFolder` reference\n        object must be provided\n    preprocessor: `TabularPreprocessor`, Optional, default = None\n        a fitted `TabularPreprocessor` object. If None, a `TabFromFolder`\n        reference object must be provided\n    text_col: str, Optional, default = None\n        the name of the column with the texts themselves or the names of the\n        files that contain the text dataset. If None, either there is no text\n        column or a `TabFromFolder` reference object must be provided\n    img_col: str, Optional, default = None\n        the name of the column with the the names of the images. If None,\n        either there is no image column or a `TabFromFolder` reference object\n        must be provided\n    ignore_target: bool, default = False\n        whether to ignore the target column. This is normally set to True when\n        this class is used for a test dataset.\n    reference: `TabFromFolder`, Optional, default = None\n        a reference `TabFromFolder` object. If provided, the `TabFromFolder`\n        object will be created using the attributes of the reference object.\n        This is useful to instantiate a `TabFromFolder` object for evaluation\n        or test purposes\n    verbose: int, default = 1\n        verbosity. If 0, no output will be printed during the process.\n    \"\"\"\n\n    def __init__(\n        self,\n        fname: str,\n        directory: Optional[str] = None,\n        target_col: Optional[str] = None,\n        preprocessor: Optional[TabularPreprocessor] = None,\n        text_col: Optional[Union[str, List[str]]] = None,\n        img_col: Optional[Union[str, List[str]]] = None,\n        ignore_target: bool = False,\n        reference: Optional[Any] = None,  # is Type[\"TabFromFolder\"],\n        verbose: Optional[int] = 1,\n    ):\n        self.fname = fname\n        self.ignore_target = ignore_target\n        self.verbose = verbose\n\n        if reference is not None:\n            (\n                self.directory,\n                self.target_col,\n                self.preprocessor,\n                self.text_col,\n                self.img_col,\n            ) = self._set_from_reference(reference, preprocessor)\n        else:\n            assert (\n                directory is not None\n                and (target_col is not None and not ignore_target)\n                and preprocessor is not None\n            ), (\n                \"if no reference is provided, 'directory', 'target_col' and 'preprocessor' \"\n                \"must be provided\"\n            )\n\n            self.directory = directory\n            self.target_col = target_col\n            self.preprocessor = preprocessor\n            self.text_col = text_col\n            self.img_col = img_col\n\n        assert (\n            self.preprocessor.is_fitted\n        ), \"The preprocessor must be fitted before passing it to this class\"\n\n    def get_item(self, idx: int) -&gt; Tuple[  # noqa: C901\n        np.ndarray,\n        Optional[Union[str, List[str]]],\n        Optional[Union[str, List[str]]],\n        Optional[Union[int, float]],\n    ]:\n        \"\"\"\n        This method is used to retrieve a sample from the csv file\n\n        Parameters\n        ----------\n        idx: int\n            the index of the sample to retrieve\n\n        Returns\n        -------\n        Tuple\n            a tuple with the processed tabular data, the text data and/or the\n            image data, and the target variable\n        \"\"\"\n\n        path = os.path.join(self.directory, self.fname)\n\n        try:\n            if not hasattr(self, \"colnames\"):\n                self.colnames = pd.read_csv(path, nrows=0).columns.tolist()\n\n            # TO DO: we need to look into this as the treatment is different\n            # whether the csv contains headers or not. For the time being we\n            # will require that the csv file has headers\n\n            _sample = pd.read_csv(\n                path, skiprows=lambda x: x != idx + 1, header=None\n            ).values\n            sample = pd.DataFrame(_sample, columns=self.colnames)\n        except Exception:\n            raise ValueError(\"Currently only csv format is supported.\")\n\n        text_fnames_or_text: Optional[Union[str, List[str]]] = None\n        if self.text_col is not None:\n            if isinstance(self.text_col, list):\n                text_fnames_or_text = [\n                    sample[col].to_list()[0] for col in self.text_col\n                ]\n            else:\n                text_fnames_or_text = sample[self.text_col].to_list()[0]\n\n        img_fname: Optional[Union[str, List[str]]] = None\n        if self.img_col is not None:\n            if isinstance(self.img_col, list):\n                img_fname = [sample[col].to_list()[0] for col in self.img_col]\n            else:\n                img_fname = sample[self.img_col].to_list()[0]\n\n        processed_sample = self.preprocessor.transform_sample(sample)\n\n        if not self.ignore_target:\n            target = sample[self.target_col].to_list()[0]\n        else:\n            target = None\n\n        return processed_sample, text_fnames_or_text, img_fname, target\n\n    def _set_from_reference(\n        self,\n        reference: Any,  # is Type[\"TabFromFolder\"],\n        preprocessor: Optional[TabularPreprocessor],\n    ) -&gt; Tuple[\n        str,\n        str,\n        TabularPreprocessor,\n        Optional[Union[str, List[str]]],\n        Optional[Union[str, List[str]]],\n    ]:\n        (\n            directory,\n            target_col,\n            _preprocessor,\n            text_col,\n            img_col,\n        ) = self._get_from_reference(reference)\n\n        if preprocessor is not None:\n            preprocessor = preprocessor\n            if self.verbose:\n                UserWarning(\n                    \"The preprocessor from the reference object is overwritten \"\n                    \"by the provided preprocessor\"\n                )\n        else:\n            preprocessor = _preprocessor\n\n        return directory, target_col, preprocessor, text_col, img_col\n\n    @staticmethod\n    def _get_from_reference(\n        reference: Any,  # is Type[\"TabFromFolder\"],\n    ) -&gt; Tuple[str, str, TabularPreprocessor, Optional[str], Optional[str]]:\n        return (\n            reference.directory,\n            reference.target_col,\n            reference.preprocessor,\n            reference.text_col,\n            reference.img_col,\n        )\n\n    def __repr__(self) -&gt; str:  # noqa: C901\n        list_of_params: List[str] = []\n        if self.fname is not None:\n            list_of_params.append(\"fname={fname}\")\n        if self.directory is not None:\n            list_of_params.append(\"directory={directory}\")\n        if self.target_col is not None:\n            list_of_params.append(\"target_col={target_col}\")\n        if self.preprocessor is not None:\n            list_of_params.append(\n                f\"preprocessor={self.preprocessor.__class__.__name__}\"\n            )\n        if self.text_col is not None:\n            if isinstance(self.text_col, list):\n                list_of_params.append(\n                    f\"text_col={[text_col for text_col in self.text_col]}\"\n                )\n            else:\n                list_of_params.append(\"text_col={text_col}\")\n        if self.img_col is not None:\n            if isinstance(self.img_col, list):\n                list_of_params.append(\n                    f\"img_col={[img_col for img_col in self.img_col]}\"\n                )\n            else:\n                list_of_params.append(\"img_col={img_col}\")\n        if self.ignore_target is not None:\n            list_of_params.append(\"ignore_target={ignore_target}\")\n        if self.verbose is not None:\n            list_of_params.append(\"verbose={verbose}\")\n        all_params = \", \".join(list_of_params)\n        return f\"{self.__class__.__name__}({all_params.format(**self.__dict__)})\"\n</code></pre>"},{"location":"pytorch-widedeep/load_from_folder.html#pytorch_widedeep.load_from_folder.tabular.tabular_from_folder.TabFromFolder.get_item","title":"get_item","text":"<pre><code>get_item(idx)\n</code></pre> <p>This method is used to retrieve a sample from the csv file</p> <p>Parameters:</p> Name Type Description Default <code>idx</code> <code>int</code> <p>the index of the sample to retrieve</p> required <p>Returns:</p> Type Description <code>Tuple</code> <p>a tuple with the processed tabular data, the text data and/or the image data, and the target variable</p> Source code in <code>pytorch_widedeep/load_from_folder/tabular/tabular_from_folder.py</code> <pre><code>def get_item(self, idx: int) -&gt; Tuple[  # noqa: C901\n    np.ndarray,\n    Optional[Union[str, List[str]]],\n    Optional[Union[str, List[str]]],\n    Optional[Union[int, float]],\n]:\n    \"\"\"\n    This method is used to retrieve a sample from the csv file\n\n    Parameters\n    ----------\n    idx: int\n        the index of the sample to retrieve\n\n    Returns\n    -------\n    Tuple\n        a tuple with the processed tabular data, the text data and/or the\n        image data, and the target variable\n    \"\"\"\n\n    path = os.path.join(self.directory, self.fname)\n\n    try:\n        if not hasattr(self, \"colnames\"):\n            self.colnames = pd.read_csv(path, nrows=0).columns.tolist()\n\n        # TO DO: we need to look into this as the treatment is different\n        # whether the csv contains headers or not. For the time being we\n        # will require that the csv file has headers\n\n        _sample = pd.read_csv(\n            path, skiprows=lambda x: x != idx + 1, header=None\n        ).values\n        sample = pd.DataFrame(_sample, columns=self.colnames)\n    except Exception:\n        raise ValueError(\"Currently only csv format is supported.\")\n\n    text_fnames_or_text: Optional[Union[str, List[str]]] = None\n    if self.text_col is not None:\n        if isinstance(self.text_col, list):\n            text_fnames_or_text = [\n                sample[col].to_list()[0] for col in self.text_col\n            ]\n        else:\n            text_fnames_or_text = sample[self.text_col].to_list()[0]\n\n    img_fname: Optional[Union[str, List[str]]] = None\n    if self.img_col is not None:\n        if isinstance(self.img_col, list):\n            img_fname = [sample[col].to_list()[0] for col in self.img_col]\n        else:\n            img_fname = sample[self.img_col].to_list()[0]\n\n    processed_sample = self.preprocessor.transform_sample(sample)\n\n    if not self.ignore_target:\n        target = sample[self.target_col].to_list()[0]\n    else:\n        target = None\n\n    return processed_sample, text_fnames_or_text, img_fname, target\n</code></pre>"},{"location":"pytorch-widedeep/load_from_folder.html#pytorch_widedeep.load_from_folder.tabular.tabular_from_folder.WideFromFolder","title":"WideFromFolder","text":"<p>               Bases: <code>TabFromFolder</code></p> <p>This class is mostly identical to <code>TabFromFolder</code> but exists because we want to separate the treatment of the wide and the deep tabular components</p> <p>Parameters:</p> Name Type Description Default <code>fname</code> <code>str</code> <p>the name of the csv file</p> required <code>directory</code> <code>Optional[str]</code> <p>the path to the directory where the csv file is located. If None, a <code>WideFromFolder</code> reference object must be provided</p> <code>None</code> <code>target_col</code> <code>Optional[str]</code> <p>the name of the target column. If None, a <code>WideFromFolder</code> reference object must be provided</p> <code>None</code> <code>preprocessor</code> <code>Optional[TabularPreprocessor]</code> <p>a fitted <code>TabularPreprocessor</code> object. If None, a <code>WideFromFolder</code> reference object must be provided</p> <code>None</code> <code>text_col</code> <code>Optional[str]</code> <p>the name of the column with the texts themselves or the names of the files that contain the text dataset. If None, either there is no text column or a <code>WideFromFolder</code> reference object must be provided=</p> <code>None</code> <code>img_col</code> <code>Optional[str]</code> <p>the name of the column with the the names of the images. If None, either there is no image column or a <code>WideFromFolder</code> reference object must be provided</p> <code>None</code> <code>ignore_target</code> <code>bool</code> <p>whether to ignore the target column. This is normally used when this class is used for a test dataset.</p> <code>False</code> <code>reference</code> <code>Optional[Any]</code> <p>a reference <code>WideFromFolder</code> object. If provided, the <code>WideFromFolder</code> object will be created using the attributes of the reference object. This is useful to instantiate a <code>WideFromFolder</code> object for evaluation or test purposes</p> <code>None</code> <code>verbose</code> <code>int</code> <p>verbosity. If 0, no output will be printed during the process.</p> <code>1</code> Source code in <code>pytorch_widedeep/load_from_folder/tabular/tabular_from_folder.py</code> <pre><code>class WideFromFolder(TabFromFolder):\n    \"\"\"\n    This class is mostly identical to `TabFromFolder` but exists because we\n    want to separate the treatment of the wide and the deep tabular\n    components\n\n    Parameters\n    ----------\n    fname: str\n        the name of the csv file\n    directory: str, Optional, default = None\n        the path to the directory where the csv file is located. If None,\n        a `WideFromFolder` reference object must be provided\n    target_col: str, Optional, default = None\n        the name of the target column. If None, a `WideFromFolder` reference\n        object must be provided\n    preprocessor: `TabularPreprocessor`, Optional, default = None\n        a fitted `TabularPreprocessor` object. If None, a `WideFromFolder`\n        reference object must be provided\n    text_col: str, Optional, default = None\n        the name of the column with the texts themselves or the names of the\n        files that contain the text dataset. If None, either there is no text\n        column or a `WideFromFolder` reference object must be provided=\n    img_col: str, Optional, default = None\n        the name of the column with the the names of the images. If None,\n        either there is no image column or a `WideFromFolder` reference object\n        must be provided\n    ignore_target: bool, default = False\n        whether to ignore the target column. This is normally used when this\n        class is used for a test dataset.\n    reference: `WideFromFolder`, Optional, default = None\n        a reference `WideFromFolder` object. If provided, the `WideFromFolder`\n        object will be created using the attributes of the reference object.\n        This is useful to instantiate a `WideFromFolder` object for evaluation\n        or test purposes\n    verbose: int, default = 1\n        verbosity. If 0, no output will be printed during the process.\n    \"\"\"\n\n    def __init__(\n        self,\n        fname: str,\n        directory: Optional[str] = None,\n        target_col: Optional[str] = None,\n        preprocessor: Optional[TabularPreprocessor] = None,\n        text_col: Optional[str] = None,\n        img_col: Optional[str] = None,\n        ignore_target: bool = False,\n        reference: Optional[Any] = None,  # is Type[\"WideFromFolder\"],\n        verbose: int = 1,\n    ):\n        super(WideFromFolder, self).__init__(\n            fname=fname,\n            directory=directory,\n            target_col=target_col,\n            preprocessor=preprocessor,\n            text_col=text_col,\n            img_col=img_col,\n            reference=reference,\n            ignore_target=ignore_target,\n            verbose=verbose,\n        )\n</code></pre>"},{"location":"pytorch-widedeep/load_from_folder.html#pytorch_widedeep.load_from_folder.text.text_from_folder.TextFromFolder","title":"TextFromFolder","text":"<p>This class is used to load the text dataset (i.e. the text files) from a folder, or to retrieve the text given a texts column specified within the preprocessor object.</p> <p>For examples, please, see the examples folder in the repo.</p> <p>Parameters:</p> Name Type Description Default <code>preprocessor</code> <code>Union[TextPreprocessor, ChunkTextPreprocessor, HFPreprocessor, ChunkHFPreprocessor, List[TextPreprocessor], List[ChunkTextPreprocessor], List[HFPreprocessor], List[ChunkHFPreprocessor]]</code> <p>The preprocessor used to process the text. It must be fitted before using this class</p> required Source code in <code>pytorch_widedeep/load_from_folder/text/text_from_folder.py</code> <pre><code>class TextFromFolder:\n    \"\"\"\n    This class is used to load the text dataset (i.e. the text files) from a\n    folder, or to retrieve the text given a texts column specified within the\n    preprocessor object.\n\n    For examples, please, see the examples folder in the repo.\n\n    Parameters\n    ----------\n    preprocessor:\n        The preprocessor used to process the text. It must be fitted before using\n        this class\n    \"\"\"\n\n    def __init__(\n        self,\n        preprocessor: Union[\n            TextPreprocessor,\n            ChunkTextPreprocessor,\n            HFPreprocessor,\n            ChunkHFPreprocessor,\n            List[TextPreprocessor],\n            List[ChunkTextPreprocessor],\n            List[HFPreprocessor],\n            List[ChunkHFPreprocessor],\n        ],\n    ):\n        if isinstance(preprocessor, list):\n            for p in preprocessor:\n                assert (\n                    p.is_fitted\n                ), \"All preprocessors must be fitted before using this class\"\n        else:\n            assert (\n                preprocessor.is_fitted\n            ), \"The preprocessor must be fitted before using this class\"\n\n        self.preprocessor = preprocessor\n\n    def get_item(\n        self, text: Union[str, List[str]]\n    ) -&gt; Union[np.ndarray, List[np.ndarray]]:\n        \"\"\"\n        Given a text or a list of texts corresponding to different text\n        columns, this method will return the processed text or a list of\n        processed texts\n\n        Parameters\n        ----------\n        text: Union[str, List[str]]\n            The text or list of texts to be processed\n\n        Returns\n        -------\n        Union[np.ndarray, List[np.ndarray]]\n            The processed text or a list of processed texts\n        \"\"\"\n        if isinstance(self.preprocessor, list):\n            assert isinstance(text, list)\n            processed_sample: Union[np.ndarray, List[np.ndarray]] = [\n                self._preprocess_one_sample(t, self.preprocessor[i])\n                for i, t in enumerate(text)\n            ]\n        else:\n            assert isinstance(text, str)\n            processed_sample = self._preprocess_one_sample(text, self.preprocessor)\n\n        return processed_sample\n\n    def _preprocess_one_sample(\n        self,\n        text: str,\n        preprocessor: Union[\n            TextPreprocessor,\n            ChunkTextPreprocessor,\n            HFPreprocessor,\n            ChunkHFPreprocessor,\n        ],\n    ) -&gt; np.ndarray:\n        if (\n            isinstance(preprocessor, ChunkTextPreprocessor)\n            and preprocessor.root_dir is not None\n        ):\n            path = os.path.join(preprocessor.root_dir, text)\n\n            with open(path, \"r\") as f:\n                sample = f.read().replace(\"\\n\", \"\")\n        else:\n            sample = text\n\n        processed_sample = preprocessor.transform_sample(sample)\n\n        return processed_sample\n\n    def __repr__(self):\n        if isinstance(self.preprocessor, list):\n            return f\"{self.__class__.__name__}({[p.__class__.__name__ for p in self.preprocessor]})\"\n        else:\n            return f\"{self.__class__.__name__}({self.preprocessor.__class__.__name__})\"\n</code></pre>"},{"location":"pytorch-widedeep/load_from_folder.html#pytorch_widedeep.load_from_folder.text.text_from_folder.TextFromFolder.get_item","title":"get_item","text":"<pre><code>get_item(text)\n</code></pre> <p>Given a text or a list of texts corresponding to different text columns, this method will return the processed text or a list of processed texts</p> <p>Parameters:</p> Name Type Description Default <code>text</code> <code>Union[str, List[str]]</code> <p>The text or list of texts to be processed</p> required <p>Returns:</p> Type Description <code>Union[ndarray, List[ndarray]]</code> <p>The processed text or a list of processed texts</p> Source code in <code>pytorch_widedeep/load_from_folder/text/text_from_folder.py</code> <pre><code>def get_item(\n    self, text: Union[str, List[str]]\n) -&gt; Union[np.ndarray, List[np.ndarray]]:\n    \"\"\"\n    Given a text or a list of texts corresponding to different text\n    columns, this method will return the processed text or a list of\n    processed texts\n\n    Parameters\n    ----------\n    text: Union[str, List[str]]\n        The text or list of texts to be processed\n\n    Returns\n    -------\n    Union[np.ndarray, List[np.ndarray]]\n        The processed text or a list of processed texts\n    \"\"\"\n    if isinstance(self.preprocessor, list):\n        assert isinstance(text, list)\n        processed_sample: Union[np.ndarray, List[np.ndarray]] = [\n            self._preprocess_one_sample(t, self.preprocessor[i])\n            for i, t in enumerate(text)\n        ]\n    else:\n        assert isinstance(text, str)\n        processed_sample = self._preprocess_one_sample(text, self.preprocessor)\n\n    return processed_sample\n</code></pre>"},{"location":"pytorch-widedeep/load_from_folder.html#pytorch_widedeep.load_from_folder.image.image_from_folder.ImageFromFolder","title":"ImageFromFolder","text":"<p>This class is used to load the image dataset from disk. It is inspired by the <code>ImageFolder</code> class at the <code>torchvision</code> library. Here, we have simply adapted to work within the context of a Wide and Deep multi-modal model.</p> <p>For examples, please, see the examples folder in the repo.</p> <p>Parameters:</p> Name Type Description Default <code>directory</code> <code>Optional[Union[str, List[str]]]</code> <p>the path to the directory where the images are located. If None, a preprocessor must be provided.</p> <code>None</code> <code>preprocessor</code> <code>Optional[Union[ImagePreprocessor, List[ImagePreprocessor]]]</code> <p>a fitted <code>ImagePreprocessor</code> object.</p> <code>None</code> <code>loader</code> <code>Callable[[str], Any]</code> <p>a function to load a sample given its path.</p> <code>default_loader</code> <code>extensions</code> <code>Optional[Tuple[str, ...]]</code> <p>a tuple with the allowed extensions. If None, IMG_EXTENSIONS will be used where IMG_EXTENSIONS =\".jpg\", \".jpeg\", \".png\", \".ppm\", \".bmp\", \".pgm\", \".tif\", \".tiff\", \".webp\"</p> <code>None</code> <code>transforms</code> <code>Optional[Any]</code> <p>a <code>torchvision.transforms</code> object. If None, this class will simply return an array representation of the PIL Image</p> <code>None</code> Source code in <code>pytorch_widedeep/load_from_folder/image/image_from_folder.py</code> <pre><code>class ImageFromFolder:\n    \"\"\"\n    This class is used to load the image dataset from disk. It is inspired by\n    the `ImageFolder` class at the `torchvision` library. Here, we have\n    simply adapted to work within the context of a Wide and Deep multi-modal\n    model.\n\n    For examples, please, see the examples folder in the repo.\n\n    Parameters\n    ----------\n    directory: str, Optional, default = None\n        the path to the directory where the images are located. If None, a\n        preprocessor must be provided.\n    preprocessor: `ImagePreprocessor`, Optional, default = None\n        a fitted `ImagePreprocessor` object.\n    loader: Callable[[str], Any], Optional, default = default_loader\n        a function to load a sample given its path.\n    extensions: Tuple[str, ...], Optional, default = IMG_EXTENSIONS\n        a tuple with the allowed extensions. If None, IMG_EXTENSIONS will be\n        used where IMG_EXTENSIONS\n        =\".jpg\", \".jpeg\", \".png\", \".ppm\", \".bmp\", \".pgm\", \".tif\", \".tiff\", \".webp\"\n    transforms: Optional[Any], default = None\n        a `torchvision.transforms` object. If None, this class will simply\n        return an array representation of the PIL Image\n    \"\"\"\n\n    def __init__(\n        self,\n        directory: Optional[Union[str, List[str]]] = None,\n        preprocessor: Optional[\n            Union[ImagePreprocessor, List[ImagePreprocessor]]\n        ] = None,\n        loader: Callable[[str], Any] = default_loader,\n        extensions: Optional[Tuple[str, ...]] = None,\n        transforms: Optional[Any] = None,\n    ) -&gt; None:\n        assert (\n            directory is not None or preprocessor is not None\n        ), \"Either a directory or an instance of ImagePreprocessor(s) must be provided\"\n\n        if directory is not None and preprocessor is not None:  # pragma: no cover\n            error_msg = (\n                \"If both 'directory' and 'preprocessor' are provided, the 'img_path' \"\n                \"attribute of the 'preprocessor' must be the same as the 'directory'\"\n            )\n            if isinstance(directory, list):\n                assert isinstance(preprocessor, list)\n                assert len(directory) == len(preprocessor)\n                for d, p in zip(directory, preprocessor):\n                    assert d == p.img_path, error_msg\n            else:\n                assert isinstance(preprocessor, ImagePreprocessor)\n                assert directory == preprocessor.img_path, error_msg\n\n        if directory is not None:\n            self.directory = directory\n        else:\n            assert (\n                preprocessor is not None\n            ), \"Either a directory or an instance of ImagePreprocessor must be provided\"\n            if isinstance(preprocessor, list):\n                self.directory = [p.img_path for p in preprocessor]\n            else:\n                self.directory = preprocessor.img_path\n\n        self.preprocessor = preprocessor\n        self.loader = loader\n        self.extensions = extensions if extensions is not None else IMG_EXTENSIONS\n        self.transforms = transforms\n        if self.transforms:\n            self.transforms_names = [\n                tr.__class__.__name__ for tr in self.transforms.transforms\n            ]\n        else:\n            self.transforms_names = []\n\n            self.transpose = True\n\n    def get_item(\n        self, fname: Union[str, List[str]]\n    ) -&gt; Union[np.ndarray, List[np.ndarray]]:\n        \"\"\"\n        This method is used to load the image dataset(s) from disk.\n\n        Parameters\n        ----------\n        fname: Union[str, List[str]]\n            the name of the image file(s) to load. If a list is provided, the\n            method will return a list of numpy arrays. Each element in the list\n            corresponds to the image in the different image columns.\n\n        Returns\n        -------\n        Union[np.ndarray, List[np.ndarray]]\n            a numpy array or a list of numpy arrays representing the image(s)\n        \"\"\"\n\n        if isinstance(fname, list):\n            if not isinstance(self.directory, list):\n                _directory = [self.directory] * len(fname)\n            else:\n                _directory = self.directory\n            if self.preprocessor is not None:\n                assert isinstance(self.preprocessor, list)\n                processed_sample: Union[np.ndarray, List[np.ndarray]] = [\n                    self._preprocess_one_sample(f, d, p)\n                    for f, d, p in zip(fname, _directory, self.preprocessor)\n                ]\n            else:\n                processed_sample = [\n                    self._preprocess_one_sample(f, d) for f, d in zip(fname, _directory)\n                ]\n        else:\n            assert isinstance(self.directory, str)\n            if self.preprocessor is not None:\n                assert isinstance(self.preprocessor, ImagePreprocessor)\n                processed_sample = self._preprocess_one_sample(\n                    fname, self.directory, self.preprocessor\n                )\n            else:\n                processed_sample = self._preprocess_one_sample(fname, self.directory)\n\n        return processed_sample\n\n    def _preprocess_one_sample(\n        self,\n        fname: str,\n        directory: str,\n        preprocessor: Optional[ImagePreprocessor] = None,\n    ) -&gt; np.ndarray:\n        assert has_file_allowed_extension(fname, self.extensions)\n\n        path = os.path.join(directory, fname)\n        sample = self.loader(path)\n\n        assert isinstance(sample, (Image.Image, np.ndarray)), (  # pragma: no cover\n            \"The loader must return an instance of PIL.Image or np.ndarray, \"\n            f\"got {type(sample)} instead\"\n        )\n\n        if preprocessor is not None:\n            if not isinstance(sample, np.ndarray):\n                processed_sample = preprocessor.transform_sample(np.asarray(sample))\n            else:\n                processed_sample = preprocessor.transform_sample(sample)\n        else:\n            processed_sample = sample  # type: ignore\n\n        prepared_sample = self._prepare_sample(processed_sample)\n\n        return prepared_sample\n\n    def _prepare_sample(  # noqa: C901\n        self, processed_sample: Union[np.ndarray, Image.Image]\n    ) -&gt; np.ndarray:\n        # if an image dataset is used, make sure is in the right format to\n        # be ingested by the conv layers\n\n        if isinstance(processed_sample, Image.Image):\n            processed_sample = np.asarray(processed_sample)\n\n        # if int must be uint8\n        if \"int\" in str(processed_sample.dtype) and \"uint8\" != str(\n            processed_sample.dtype\n        ):\n            processed_sample = processed_sample.astype(\"uint8\")\n\n        # if float must be float32\n        if \"float\" in str(processed_sample.dtype) and \"float32\" != str(\n            processed_sample.dtype\n        ):\n            processed_sample = processed_sample.astype(\"float32\")\n\n        # if there are no transforms, or these do not include ToTensor()\n        # (weird or unexpected case, not sure is even possible) then we need\n        # to  replicate what ToTensor() does -&gt; transpose axis and normalize if\n        # necessary\n        if not self.transforms or \"ToTensor\" not in self.transforms_names:\n            if processed_sample.ndim == 2:\n                processed_sample = processed_sample[:, :, None]\n\n            processed_sample = processed_sample.transpose(2, 0, 1)\n\n            if \"int\" in str(processed_sample.dtype):\n                processed_sample = (processed_sample / processed_sample.max()).astype(\n                    \"float32\"\n                )\n        elif \"ToTensor\" in self.transforms_names:\n            # if ToTensor() is included, simply apply transforms\n            assert self.transforms_names[0] == \"ToTensor\", (\n                \"If ToTensor() is included in the transforms, it must be the \"\n                \"first transform in the list\"\n            )\n            processed_sample = self.transforms(processed_sample)\n        else:\n            # else apply transforms on the result of calling torch.tensor on\n            # processed_sample after all the previous manipulation\n            processed_sample = self.transforms(torch.tensor(processed_sample))\n\n        return processed_sample  # type: ignore\n\n    def __repr__(self) -&gt; str:\n        list_of_params: List[str] = []\n        if self.directory is not None:\n            list_of_params.append(\"directory={directory}\")\n        if self.preprocessor is not None:\n            list_of_params.append(\n                f\"preprocessor={self.preprocessor.__class__.__name__}\"\n            )\n        if self.loader is not None:\n            list_of_params.append(f\"loader={self.loader.__name__}\")\n        if self.extensions is not None:\n            list_of_params.append(\"extensions={extensions}\")\n        if self.transforms is not None:\n            list_of_params.append(f\"transforms={self.transforms_names}\")\n        all_params = \", \".join(list_of_params)\n        return f\"ImageFromFolder({all_params.format(**self.__dict__)})\"\n</code></pre>"},{"location":"pytorch-widedeep/load_from_folder.html#pytorch_widedeep.load_from_folder.image.image_from_folder.ImageFromFolder.get_item","title":"get_item","text":"<pre><code>get_item(fname)\n</code></pre> <p>This method is used to load the image dataset(s) from disk.</p> <p>Parameters:</p> Name Type Description Default <code>fname</code> <code>Union[str, List[str]]</code> <p>the name of the image file(s) to load. If a list is provided, the method will return a list of numpy arrays. Each element in the list corresponds to the image in the different image columns.</p> required <p>Returns:</p> Type Description <code>Union[ndarray, List[ndarray]]</code> <p>a numpy array or a list of numpy arrays representing the image(s)</p> Source code in <code>pytorch_widedeep/load_from_folder/image/image_from_folder.py</code> <pre><code>def get_item(\n    self, fname: Union[str, List[str]]\n) -&gt; Union[np.ndarray, List[np.ndarray]]:\n    \"\"\"\n    This method is used to load the image dataset(s) from disk.\n\n    Parameters\n    ----------\n    fname: Union[str, List[str]]\n        the name of the image file(s) to load. If a list is provided, the\n        method will return a list of numpy arrays. Each element in the list\n        corresponds to the image in the different image columns.\n\n    Returns\n    -------\n    Union[np.ndarray, List[np.ndarray]]\n        a numpy array or a list of numpy arrays representing the image(s)\n    \"\"\"\n\n    if isinstance(fname, list):\n        if not isinstance(self.directory, list):\n            _directory = [self.directory] * len(fname)\n        else:\n            _directory = self.directory\n        if self.preprocessor is not None:\n            assert isinstance(self.preprocessor, list)\n            processed_sample: Union[np.ndarray, List[np.ndarray]] = [\n                self._preprocess_one_sample(f, d, p)\n                for f, d, p in zip(fname, _directory, self.preprocessor)\n            ]\n        else:\n            processed_sample = [\n                self._preprocess_one_sample(f, d) for f, d in zip(fname, _directory)\n            ]\n    else:\n        assert isinstance(self.directory, str)\n        if self.preprocessor is not None:\n            assert isinstance(self.preprocessor, ImagePreprocessor)\n            processed_sample = self._preprocess_one_sample(\n                fname, self.directory, self.preprocessor\n            )\n        else:\n            processed_sample = self._preprocess_one_sample(fname, self.directory)\n\n    return processed_sample\n</code></pre>"},{"location":"pytorch-widedeep/load_from_folder.html#pytorch_widedeep.load_from_folder.wd_dataset_from_folder.WideDeepDatasetFromFolder","title":"WideDeepDatasetFromFolder","text":"<p>               Bases: <code>Dataset</code></p> <p>This class is the Dataset counterpart of the <code>WideDeepDataset</code> class.</p> <p>Given a reference tabular dataset, with columns that indicate the path to the images and to the text files or the texts themselves, it will use the <code>[...]FromFolder</code> classes to load the data consistently from disk per batch.</p> <p>For examples, please, see the examples folder in the repo.</p> <p>Parameters:</p> Name Type Description Default <code>n_samples</code> <code>int</code> <p>Number of samples in the dataset</p> required <code>tab_from_folder</code> <code>Optional[TabFromFolder]</code> <p>Instance of the <code>TabFromFolder</code> class</p> <code>None</code> <code>wide_from_folder</code> <code>Optional[WideFromFolder]</code> <p>Instance of the <code>WideFromFolder</code> class</p> <code>None</code> <code>text_from_folder</code> <code>Optional[TextFromFolder]</code> <p>Instance of the <code>TextFromFolder</code> class</p> <code>None</code> <code>img_from_folder</code> <code>Optional[ImageFromFolder]</code> <p>Instance of the <code>ImageFromFolder</code> class</p> <code>None</code> <code>reference</code> <code>Optional[Any]</code> <p>If <code>reference</code> not None, the <code>text_from_folder</code> and <code>img_from_folder</code> objects will be retrieved from the reference class. This is useful when we want to use a <code>WideDeepDatasetFromFolder</code> class used for a train dataset as a reference for the validation and test datasets. In this case, the <code>text_from_folder</code> and <code>img_from_folder</code> objects will be the same for all three datasets, so there is no need to create a new instance for each dataset.</p> <code>None</code> Source code in <code>pytorch_widedeep/load_from_folder/wd_dataset_from_folder.py</code> <pre><code>class WideDeepDatasetFromFolder(Dataset):\n    \"\"\"\n    This class is the Dataset counterpart of the `WideDeepDataset` class.\n\n    Given a reference tabular dataset, with columns that indicate the path to\n    the images and to the text files or the texts themselves, it will use the\n    `[...]FromFolder` classes to load the data consistently from disk per batch.\n\n    For examples, please, see the examples folder in the repo.\n\n    Parameters\n    ----------\n    n_samples: int\n        Number of samples in the dataset\n    tab_from_folder: TabFromFolder\n        Instance of the `TabFromFolder` class\n    wide_from_folder: Optional[WideFromFolder], default = None\n        Instance of the `WideFromFolder` class\n    text_from_folder: Optional[TextFromFolder], default = None\n        Instance of the `TextFromFolder` class\n    img_from_folder: Optional[ImageFromFolder], default = None\n        Instance of the `ImageFromFolder` class\n    reference: Type[\"WideDeepDatasetFromFolder\"], default = None\n        If `reference` not None, the `text_from_folder` and `img_from_folder`\n        objects will be retrieved from the reference class. This is useful\n        when we want to use a `WideDeepDatasetFromFolder` class used for a\n        train dataset as a reference for the validation and test datasets. In\n        this case, the `text_from_folder` and `img_from_folder` objects will\n        be the same for all three datasets, so there is no need to create a\n        new instance for each dataset.\n    \"\"\"\n\n    def __init__(\n        self,\n        n_samples: int,\n        tab_from_folder: Optional[TabFromFolder] = None,\n        wide_from_folder: Optional[WideFromFolder] = None,\n        text_from_folder: Optional[TextFromFolder] = None,\n        img_from_folder: Optional[ImageFromFolder] = None,\n        reference: Optional[Any] = None,  # is Type[\"WideDeepDatasetFromFolder\"],\n    ):\n        super(WideDeepDatasetFromFolder, self).__init__()\n\n        if tab_from_folder is None and wide_from_folder is None:\n            raise ValueError(\n                \"Either 'tab_from_folder' or 'wide_from_folder' must be not None\"\n            )\n\n        if reference is not None:\n            assert (\n                img_from_folder is None and text_from_folder is None\n            ), \"If reference is not None, 'img_from_folder' and 'text_from_folder' left as None\"\n            self.text_from_folder, self.img_from_folder = self._get_from_reference(\n                reference\n            )\n        else:\n            assert (\n                text_from_folder is not None and img_from_folder is not None\n            ), \"If reference is None, 'img_from_folder' and 'text_from_folder' must be not None\"\n            self.text_from_folder = text_from_folder\n            self.img_from_folder = img_from_folder\n\n        self.n_samples = n_samples\n        self.tab_from_folder = tab_from_folder\n        self.wide_from_folder = wide_from_folder\n\n    def __getitem__(self, idx: int):  # noqa: C901\n        x = (\n            Bunch()\n        )  # for consistency with WideDeepDataset, but this is just a Dict[str, Any]\n\n        if self.tab_from_folder is not None:\n            X_tab, text_fname_or_text, img_fname, y = self.tab_from_folder.get_item(\n                idx=idx\n            )\n            x.deeptabular = X_tab\n\n        if self.wide_from_folder is not None:\n            if self.tab_from_folder is None:\n                (\n                    X_wide,\n                    text_fname_or_text,\n                    img_fname,\n                    y,\n                ) = self.wide_from_folder.get_item(idx=idx)\n            else:\n                X_wide, _, _, _ = self.wide_from_folder.get_item(idx=idx)\n            x.wide = X_wide\n\n        if text_fname_or_text is not None:\n            # These assertions should never be raised, but just in case...\n            assert (\n                self.text_from_folder is not None\n            ), \"text_fname_or_text is not None but self.text_from_folder is None\"\n            X_text = self.text_from_folder.get_item(text_fname_or_text)\n            x.deeptext = X_text\n\n        if img_fname is not None:\n            assert (\n                self.img_from_folder is not None\n            ), \"img_fname is not None but self.img_from_folder is None\"\n            X_img = self.img_from_folder.get_item(img_fname)\n            x.deepimage = X_img\n\n        # We are aware that returning sometimes X and sometimes X, y is not\n        # the best practice, but is the easiest way at this stage\n        if y is not None:\n            return x, y\n        else:\n            return x\n\n    def __len__(self):\n        return self.n_samples\n\n    @staticmethod\n    def _get_from_reference(\n        reference: Type[\"WideDeepDatasetFromFolder\"],\n    ) -&gt; Tuple[Optional[TextFromFolder], Optional[ImageFromFolder]]:\n        return reference.text_from_folder, reference.img_from_folder\n\n    def __repr__(self) -&gt; str:\n        list_of_params: List[str] = []\n        list_of_params.append(\"n_samples={n_samples}\")\n        if self.tab_from_folder is not None:\n            list_of_params.append(\n                f\"tab_from_folder={self.tab_from_folder.__class__.__name__}\"\n            )\n        if self.wide_from_folder is not None:\n            list_of_params.append(\n                f\"wide_from_folder={self.wide_from_folder.__class__.__name__}\"\n            )\n        if self.text_from_folder is not None:\n            list_of_params.append(\n                f\"text_from_folder={self.text_from_folder.__class__.__name__}\"\n            )\n        if self.img_from_folder is not None:\n            list_of_params.append(\n                f\"img_from_folder={self.img_from_folder.__class__.__name__}\"\n            )\n        all_params = \", \".join(list_of_params)\n        return f\"WideDeepDatasetFromFolder({all_params.format(**self.__dict__)})\"\n</code></pre>"},{"location":"pytorch-widedeep/losses.html","title":"Losses","text":"<p><code>pytorch-widedeep</code> accepts a number of losses and objectives that can be passed to the <code>Trainer</code> class via the parameter <code>objective</code> (see <code>pytorch-widedeep.training.Trainer</code>). For most cases the loss function that <code>pytorch-widedeep</code> will use internally is already implemented in Pytorch.</p> <p>In addition, <code>pytorch-widedeep</code> implements a series of  \"custom\" loss functions. These are described below for completion since, as mentioned before, they are used internally by the <code>Trainer</code>. Of course, onen could always use them on their own and can be imported as:</p> <p><code>from pytorch_widedeep.losses import FocalLoss</code></p> <p> NOTE:  Losses in this module expect the predictions  and ground truth to have the same dimensions for regression and binary  classification problems \\((N_{samples}, 1)\\). In the case of multiclass  classification problems the ground truth is expected to be a 1D tensor with  the corresponding classes. See Examples below</p>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.MSELoss","title":"MSELoss","text":"<p>               Bases: <code>Module</code></p> <p>Mean square error loss</p> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>class MSELoss(nn.Module):\n    r\"\"\"Mean square error loss\"\"\"\n\n    # legacy code from when we used to support FDS-LDS and this class could\n    # taked the corresponding params. At this stage probably you want to use\n    # torch.nn.MSELoss\n    def __init__(self):\n        super().__init__()\n\n    def forward(\n        self,\n        input: Tensor,\n        target: Tensor,\n    ) -&gt; Tensor:\n        r\"\"\"\n        Parameters\n        ----------\n        input: Tensor\n            Input tensor with predictions\n        target: Tensor\n            Target tensor with the actual values\n\n        Examples\n        --------\n        &gt;&gt;&gt; import torch\n        &gt;&gt;&gt; from pytorch_widedeep.losses import MSELoss\n        &gt;&gt;&gt;\n        &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n        &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n        &gt;&gt;&gt; loss = MSELoss()(input, target)\n        \"\"\"\n        loss = (input - target) ** 2\n        return torch.mean(loss)\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.MSELoss.forward","title":"forward","text":"<pre><code>forward(input, target)\n</code></pre> <p>Parameters:</p> Name Type Description Default <code>input</code> <code>Tensor</code> <p>Input tensor with predictions</p> required <code>target</code> <code>Tensor</code> <p>Target tensor with the actual values</p> required <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.losses import MSELoss\n&gt;&gt;&gt;\n&gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n&gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n&gt;&gt;&gt; loss = MSELoss()(input, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(\n    self,\n    input: Tensor,\n    target: Tensor,\n) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    input: Tensor\n        Input tensor with predictions\n    target: Tensor\n        Target tensor with the actual values\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.losses import MSELoss\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n    &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n    &gt;&gt;&gt; loss = MSELoss()(input, target)\n    \"\"\"\n    loss = (input - target) ** 2\n    return torch.mean(loss)\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.MSLELoss","title":"MSLELoss","text":"<p>               Bases: <code>Module</code></p> <p>Mean square log error loss</p> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>class MSLELoss(nn.Module):\n    r\"\"\"Mean square log error loss\"\"\"\n\n    def __init__(self):\n        super().__init__()\n\n    def forward(\n        self,\n        input: Tensor,\n        target: Tensor,\n    ) -&gt; Tensor:\n        r\"\"\"\n        Parameters\n        ----------\n        input: Tensor\n            Input tensor with predictions (not probabilities)\n        target: Tensor\n            Target tensor with the actual classes\n\n        Examples\n        --------\n        &gt;&gt;&gt; import torch\n        &gt;&gt;&gt; from pytorch_widedeep.losses import MSLELoss\n        &gt;&gt;&gt;\n        &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n        &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n        &gt;&gt;&gt; loss = MSLELoss()(input, target)\n        \"\"\"\n        assert (\n            input.min() &gt;= 0\n        ), \"\"\"All input values must be &gt;=0, if your model is predicting\n            values &lt;0 try to enforce positive values by activation function\n            on last layer with `trainer.enforce_positive_output=True`\"\"\"\n        assert target.min() &gt;= 0, \"All target values must be &gt;=0\"\n\n        loss = (torch.log(input + 1) - torch.log(target + 1)) ** 2\n        return torch.mean(loss)\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.MSLELoss.forward","title":"forward","text":"<pre><code>forward(input, target)\n</code></pre> <p>Parameters:</p> Name Type Description Default <code>input</code> <code>Tensor</code> <p>Input tensor with predictions (not probabilities)</p> required <code>target</code> <code>Tensor</code> <p>Target tensor with the actual classes</p> required <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.losses import MSLELoss\n&gt;&gt;&gt;\n&gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n&gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n&gt;&gt;&gt; loss = MSLELoss()(input, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(\n    self,\n    input: Tensor,\n    target: Tensor,\n) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    input: Tensor\n        Input tensor with predictions (not probabilities)\n    target: Tensor\n        Target tensor with the actual classes\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.losses import MSLELoss\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n    &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n    &gt;&gt;&gt; loss = MSLELoss()(input, target)\n    \"\"\"\n    assert (\n        input.min() &gt;= 0\n    ), \"\"\"All input values must be &gt;=0, if your model is predicting\n        values &lt;0 try to enforce positive values by activation function\n        on last layer with `trainer.enforce_positive_output=True`\"\"\"\n    assert target.min() &gt;= 0, \"All target values must be &gt;=0\"\n\n    loss = (torch.log(input + 1) - torch.log(target + 1)) ** 2\n    return torch.mean(loss)\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.RMSELoss","title":"RMSELoss","text":"<p>               Bases: <code>Module</code></p> <p>Root mean square error loss</p> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>class RMSELoss(nn.Module):\n    r\"\"\"Root mean square error loss\"\"\"\n\n    # legacy code from when we used to support FDS-LDS and this class could\n    # taked the corresponding params. At this stage probably you want to use\n    # torch.sqrt(nn.MSELoss)\n    def __init__(self):\n        super().__init__()\n\n    def forward(self, input: Tensor, target: Tensor) -&gt; Tensor:\n        r\"\"\"\n        Parameters\n        ----------\n        input: Tensor\n            Input tensor with predictions (not probabilities)\n        target: Tensor\n            Target tensor with the actual classes\n\n        Examples\n        --------\n        &gt;&gt;&gt; import torch\n        &gt;&gt;&gt; from pytorch_widedeep.losses import RMSELoss\n        &gt;&gt;&gt;\n        &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n        &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n        &gt;&gt;&gt; loss = RMSELoss()(input, target)\n        \"\"\"\n        loss = (input - target) ** 2\n        return torch.sqrt(torch.mean(loss))\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.RMSELoss.forward","title":"forward","text":"<pre><code>forward(input, target)\n</code></pre> <p>Parameters:</p> Name Type Description Default <code>input</code> <code>Tensor</code> <p>Input tensor with predictions (not probabilities)</p> required <code>target</code> <code>Tensor</code> <p>Target tensor with the actual classes</p> required <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.losses import RMSELoss\n&gt;&gt;&gt;\n&gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n&gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n&gt;&gt;&gt; loss = RMSELoss()(input, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(self, input: Tensor, target: Tensor) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    input: Tensor\n        Input tensor with predictions (not probabilities)\n    target: Tensor\n        Target tensor with the actual classes\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.losses import RMSELoss\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n    &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n    &gt;&gt;&gt; loss = RMSELoss()(input, target)\n    \"\"\"\n    loss = (input - target) ** 2\n    return torch.sqrt(torch.mean(loss))\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.RMSLELoss","title":"RMSLELoss","text":"<p>               Bases: <code>Module</code></p> <p>Root mean square log error loss</p> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>class RMSLELoss(nn.Module):\n    r\"\"\"Root mean square log error loss\"\"\"\n\n    def __init__(self):\n        super().__init__()\n\n    def forward(self, input: Tensor, target: Tensor) -&gt; Tensor:\n        r\"\"\"\n        Parameters\n        ----------\n        input: Tensor\n            Input tensor with predictions (not probabilities)\n        target: Tensor\n            Target tensor with the actual classes\n\n        Examples\n        --------\n        &gt;&gt;&gt; import torch\n        &gt;&gt;&gt; from pytorch_widedeep.losses import RMSLELoss\n        &gt;&gt;&gt;\n        &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n        &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n        &gt;&gt;&gt; loss = RMSLELoss()(input, target)\n        \"\"\"\n        assert (\n            input.min() &gt;= 0\n        ), \"\"\"All input values must be &gt;=0, if your model is predicting\n            values &lt;0 try to enforce positive values by activation function\n            on last layer with `trainer.enforce_positive_output=True`\"\"\"\n        assert target.min() &gt;= 0, \"All target values must be &gt;=0\"\n\n        loss = (torch.log(input + 1) - torch.log(target + 1)) ** 2\n        return torch.sqrt(torch.mean(loss))\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.RMSLELoss.forward","title":"forward","text":"<pre><code>forward(input, target)\n</code></pre> <p>Parameters:</p> Name Type Description Default <code>input</code> <code>Tensor</code> <p>Input tensor with predictions (not probabilities)</p> required <code>target</code> <code>Tensor</code> <p>Target tensor with the actual classes</p> required <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.losses import RMSLELoss\n&gt;&gt;&gt;\n&gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n&gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n&gt;&gt;&gt; loss = RMSLELoss()(input, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(self, input: Tensor, target: Tensor) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    input: Tensor\n        Input tensor with predictions (not probabilities)\n    target: Tensor\n        Target tensor with the actual classes\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.losses import RMSLELoss\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n    &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n    &gt;&gt;&gt; loss = RMSLELoss()(input, target)\n    \"\"\"\n    assert (\n        input.min() &gt;= 0\n    ), \"\"\"All input values must be &gt;=0, if your model is predicting\n        values &lt;0 try to enforce positive values by activation function\n        on last layer with `trainer.enforce_positive_output=True`\"\"\"\n    assert target.min() &gt;= 0, \"All target values must be &gt;=0\"\n\n    loss = (torch.log(input + 1) - torch.log(target + 1)) ** 2\n    return torch.sqrt(torch.mean(loss))\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.QuantileLoss","title":"QuantileLoss","text":"<p>               Bases: <code>Module</code></p> <p>Quantile loss defined as:</p> \\[ Loss = max(q \\times (y-y_{pred}), (1-q) \\times (y_{pred}-y)) \\] <p>All credits go to the implementation at pytorch-forecasting.</p> <p>Parameters:</p> Name Type Description Default <code>quantiles</code> <code>List[float]</code> <p>List of quantiles</p> <code>[0.02, 0.1, 0.25, 0.5, 0.75, 0.9, 0.98]</code> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>class QuantileLoss(nn.Module):\n    r\"\"\"Quantile loss defined as:\n\n    $$\n    Loss = max(q \\times (y-y_{pred}), (1-q) \\times (y_{pred}-y))\n    $$\n\n    All credits go to the implementation at\n    [pytorch-forecasting](https://pytorch-forecasting.readthedocs.io/en/latest/_modules/pytorch_forecasting/metrics.html#QuantileLoss).\n\n    Parameters\n    ----------\n    quantiles: List, default = [0.02, 0.1, 0.25, 0.5, 0.75, 0.9, 0.98]\n        List of quantiles\n    \"\"\"\n\n    def __init__(\n        self,\n        quantiles: List[float] = [0.02, 0.1, 0.25, 0.5, 0.75, 0.9, 0.98],\n    ):\n        super().__init__()\n        self.quantiles = quantiles\n\n    def forward(self, input: Tensor, target: Tensor) -&gt; Tensor:\n        r\"\"\"\n        Parameters\n        ----------\n        input: Tensor\n            Input tensor with predictions\n        target: Tensor\n            Target tensor with the actual values\n\n        Examples\n        --------\n        &gt;&gt;&gt; import torch\n        &gt;&gt;&gt;\n        &gt;&gt;&gt; from pytorch_widedeep.losses import QuantileLoss\n        &gt;&gt;&gt;\n        &gt;&gt;&gt; # REGRESSION\n        &gt;&gt;&gt; target = torch.tensor([[0.6, 1.5]]).view(-1, 1)\n        &gt;&gt;&gt; input = torch.tensor([[.1, .2,], [.4, .5]])\n        &gt;&gt;&gt; qloss = QuantileLoss([0.25, 0.75])\n        &gt;&gt;&gt; loss = qloss(input, target)\n        \"\"\"\n\n        assert input.shape == torch.Size([target.shape[0], len(self.quantiles)]), (\n            \"The input and target have inconsistent shape. The dimension of the prediction \"\n            \"of the model that is using QuantileLoss must be equal to number of quantiles, \"\n            f\"i.e. {len(self.quantiles)}.\"\n        )\n        target = target.view(-1, 1).float()\n        losses = []\n        for i, q in enumerate(self.quantiles):\n            errors = target - input[..., i]\n            losses.append(torch.max((q - 1) * errors, q * errors).unsqueeze(-1))\n\n        loss = torch.cat(losses, dim=2)\n\n        return torch.mean(loss)\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.QuantileLoss.forward","title":"forward","text":"<pre><code>forward(input, target)\n</code></pre> <p>Parameters:</p> Name Type Description Default <code>input</code> <code>Tensor</code> <p>Input tensor with predictions</p> required <code>target</code> <code>Tensor</code> <p>Target tensor with the actual values</p> required <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt;\n&gt;&gt;&gt; from pytorch_widedeep.losses import QuantileLoss\n&gt;&gt;&gt;\n&gt;&gt;&gt; # REGRESSION\n&gt;&gt;&gt; target = torch.tensor([[0.6, 1.5]]).view(-1, 1)\n&gt;&gt;&gt; input = torch.tensor([[.1, .2,], [.4, .5]])\n&gt;&gt;&gt; qloss = QuantileLoss([0.25, 0.75])\n&gt;&gt;&gt; loss = qloss(input, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(self, input: Tensor, target: Tensor) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    input: Tensor\n        Input tensor with predictions\n    target: Tensor\n        Target tensor with the actual values\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; from pytorch_widedeep.losses import QuantileLoss\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; # REGRESSION\n    &gt;&gt;&gt; target = torch.tensor([[0.6, 1.5]]).view(-1, 1)\n    &gt;&gt;&gt; input = torch.tensor([[.1, .2,], [.4, .5]])\n    &gt;&gt;&gt; qloss = QuantileLoss([0.25, 0.75])\n    &gt;&gt;&gt; loss = qloss(input, target)\n    \"\"\"\n\n    assert input.shape == torch.Size([target.shape[0], len(self.quantiles)]), (\n        \"The input and target have inconsistent shape. The dimension of the prediction \"\n        \"of the model that is using QuantileLoss must be equal to number of quantiles, \"\n        f\"i.e. {len(self.quantiles)}.\"\n    )\n    target = target.view(-1, 1).float()\n    losses = []\n    for i, q in enumerate(self.quantiles):\n        errors = target - input[..., i]\n        losses.append(torch.max((q - 1) * errors, q * errors).unsqueeze(-1))\n\n    loss = torch.cat(losses, dim=2)\n\n    return torch.mean(loss)\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.FocalLoss","title":"FocalLoss","text":"<p>               Bases: <code>Module</code></p> <p>Implementation of the Focal loss for both binary and multiclass classification:</p> \\[ FL(p_t) = \\alpha (1 - p_t)^{\\gamma} log(p_t) \\] <p>where, for a case of a binary classification problem</p> \\[ \\begin{equation} p_t= \\begin{cases}p, &amp; \\text{if $y=1$}.\\\\1-p, &amp; \\text{otherwise}. \\end{cases} \\end{equation} \\] <p>Parameters:</p> Name Type Description Default <code>alpha</code> <code>float</code> <p>Focal Loss <code>alpha</code> parameter</p> <code>0.25</code> <code>gamma</code> <code>float</code> <p>Focal Loss <code>gamma</code> parameter</p> <code>1.0</code> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>class FocalLoss(nn.Module):\n    r\"\"\"Implementation of the [Focal loss](https://arxiv.org/pdf/1708.02002.pdf)\n    for both binary and multiclass classification:\n\n    $$\n    FL(p_t) = \\alpha (1 - p_t)^{\\gamma} log(p_t)\n    $$\n\n    where, for a case of a binary classification problem\n\n    $$\n    \\begin{equation} p_t= \\begin{cases}p, &amp; \\text{if $y=1$}.\\\\1-p, &amp; \\text{otherwise}. \\end{cases} \\end{equation}\n    $$\n\n    Parameters\n    ----------\n    alpha: float\n        Focal Loss `alpha` parameter\n    gamma: float\n        Focal Loss `gamma` parameter\n    \"\"\"\n\n    def __init__(self, alpha: float = 0.25, gamma: float = 1.0):\n        super().__init__()\n        self.alpha = alpha\n        self.gamma = gamma\n\n    def _get_weight(self, p: Tensor, t: Tensor) -&gt; Tensor:\n        pt = p * t + (1 - p) * (1 - t)  # type: ignore\n        w = self.alpha * t + (1 - self.alpha) * (1 - t)  # type: ignore\n        return (w * (1 - pt).pow(self.gamma)).detach()  # type: ignore\n\n    def forward(self, input: Tensor, target: Tensor) -&gt; Tensor:\n        r\"\"\"\n        Parameters\n        ----------\n        input: Tensor\n            Input tensor with predictions (not probabilities)\n        target: Tensor\n            Target tensor with the actual classes\n\n        Examples\n        --------\n        &gt;&gt;&gt; import torch\n        &gt;&gt;&gt;\n        &gt;&gt;&gt; from pytorch_widedeep.losses import FocalLoss\n        &gt;&gt;&gt;\n        &gt;&gt;&gt; # BINARY\n        &gt;&gt;&gt; target = torch.tensor([0, 1, 0, 1]).view(-1, 1)\n        &gt;&gt;&gt; input = torch.tensor([[0.6, 0.7, 0.3, 0.8]]).t()\n        &gt;&gt;&gt; loss = FocalLoss()(input, target)\n        &gt;&gt;&gt;\n        &gt;&gt;&gt; # MULTICLASS\n        &gt;&gt;&gt; target = torch.tensor([1, 0, 2]).view(-1, 1)\n        &gt;&gt;&gt; input = torch.tensor([[0.2, 0.5, 0.3], [0.8, 0.1, 0.1], [0.7, 0.2, 0.1]])\n        &gt;&gt;&gt; loss = FocalLoss()(input, target)\n        \"\"\"\n        input_prob = torch.sigmoid(input)\n        if input.size(1) == 1:\n            input_prob = torch.cat([1 - input_prob, input_prob], axis=1)  # type: ignore\n            num_class = 2\n        else:\n            num_class = input_prob.size(1)\n        binary_target = torch.eye(num_class)[target.squeeze().cpu().long()]\n        binary_target = binary_target.to(input.device)\n        binary_target = binary_target.contiguous()\n        weight = self._get_weight(input_prob, binary_target)\n\n        return F.binary_cross_entropy(\n            input_prob, binary_target, weight, reduction=\"mean\"\n        )\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.FocalLoss.forward","title":"forward","text":"<pre><code>forward(input, target)\n</code></pre> <p>Parameters:</p> Name Type Description Default <code>input</code> <code>Tensor</code> <p>Input tensor with predictions (not probabilities)</p> required <code>target</code> <code>Tensor</code> <p>Target tensor with the actual classes</p> required <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt;\n&gt;&gt;&gt; from pytorch_widedeep.losses import FocalLoss\n&gt;&gt;&gt;\n&gt;&gt;&gt; # BINARY\n&gt;&gt;&gt; target = torch.tensor([0, 1, 0, 1]).view(-1, 1)\n&gt;&gt;&gt; input = torch.tensor([[0.6, 0.7, 0.3, 0.8]]).t()\n&gt;&gt;&gt; loss = FocalLoss()(input, target)\n&gt;&gt;&gt;\n&gt;&gt;&gt; # MULTICLASS\n&gt;&gt;&gt; target = torch.tensor([1, 0, 2]).view(-1, 1)\n&gt;&gt;&gt; input = torch.tensor([[0.2, 0.5, 0.3], [0.8, 0.1, 0.1], [0.7, 0.2, 0.1]])\n&gt;&gt;&gt; loss = FocalLoss()(input, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(self, input: Tensor, target: Tensor) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    input: Tensor\n        Input tensor with predictions (not probabilities)\n    target: Tensor\n        Target tensor with the actual classes\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; from pytorch_widedeep.losses import FocalLoss\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; # BINARY\n    &gt;&gt;&gt; target = torch.tensor([0, 1, 0, 1]).view(-1, 1)\n    &gt;&gt;&gt; input = torch.tensor([[0.6, 0.7, 0.3, 0.8]]).t()\n    &gt;&gt;&gt; loss = FocalLoss()(input, target)\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; # MULTICLASS\n    &gt;&gt;&gt; target = torch.tensor([1, 0, 2]).view(-1, 1)\n    &gt;&gt;&gt; input = torch.tensor([[0.2, 0.5, 0.3], [0.8, 0.1, 0.1], [0.7, 0.2, 0.1]])\n    &gt;&gt;&gt; loss = FocalLoss()(input, target)\n    \"\"\"\n    input_prob = torch.sigmoid(input)\n    if input.size(1) == 1:\n        input_prob = torch.cat([1 - input_prob, input_prob], axis=1)  # type: ignore\n        num_class = 2\n    else:\n        num_class = input_prob.size(1)\n    binary_target = torch.eye(num_class)[target.squeeze().cpu().long()]\n    binary_target = binary_target.to(input.device)\n    binary_target = binary_target.contiguous()\n    weight = self._get_weight(input_prob, binary_target)\n\n    return F.binary_cross_entropy(\n        input_prob, binary_target, weight, reduction=\"mean\"\n    )\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.BayesianSELoss","title":"BayesianSELoss","text":"<p>               Bases: <code>Module</code></p> <p>Squared Loss (log Gaussian) for the case of a regression as specified in the original publication Weight Uncertainty in Neural Networks.</p> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>class BayesianSELoss(nn.Module):\n    r\"\"\"Squared Loss (log Gaussian) for the case of a regression as specified in\n    the original publication\n    [Weight Uncertainty in Neural Networks](https://arxiv.org/abs/1505.05424).\n    \"\"\"\n\n    def __init__(self):\n        super().__init__()\n\n    def forward(self, input: Tensor, target: Tensor) -&gt; Tensor:\n        r\"\"\"\n        Parameters\n        ----------\n        input: Tensor\n            Input tensor with predictions (not probabilities)\n        target: Tensor\n            Target tensor with the actual classes\n\n        Examples\n        --------\n        &gt;&gt;&gt; import torch\n        &gt;&gt;&gt; from pytorch_widedeep.losses import BayesianSELoss\n        &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n        &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n        &gt;&gt;&gt; loss = BayesianSELoss()(input, target)\n        \"\"\"\n        return (0.5 * (input - target) ** 2).sum()\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.BayesianSELoss.forward","title":"forward","text":"<pre><code>forward(input, target)\n</code></pre> <p>Parameters:</p> Name Type Description Default <code>input</code> <code>Tensor</code> <p>Input tensor with predictions (not probabilities)</p> required <code>target</code> <code>Tensor</code> <p>Target tensor with the actual classes</p> required <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.losses import BayesianSELoss\n&gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n&gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n&gt;&gt;&gt; loss = BayesianSELoss()(input, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(self, input: Tensor, target: Tensor) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    input: Tensor\n        Input tensor with predictions (not probabilities)\n    target: Tensor\n        Target tensor with the actual classes\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.losses import BayesianSELoss\n    &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n    &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n    &gt;&gt;&gt; loss = BayesianSELoss()(input, target)\n    \"\"\"\n    return (0.5 * (input - target) ** 2).sum()\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.TweedieLoss","title":"TweedieLoss","text":"<p>               Bases: <code>Module</code></p> <p>Tweedie loss for extremely unbalanced zero-inflated data</p> <p>All credits go to Wenbo Shi. See this post and the original publication for details.</p> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>class TweedieLoss(nn.Module):\n    \"\"\"\n    Tweedie loss for extremely unbalanced zero-inflated data\n\n    All credits go to Wenbo Shi. See\n    [this post](https://towardsdatascience.com/tweedie-loss-function-for-right-skewed-data-2c5ca470678f)\n    and the [original publication](https://arxiv.org/abs/1811.10192) for details.\n    \"\"\"\n\n    def __init__(self):\n        super().__init__()\n\n    def forward(\n        self,\n        input: Tensor,\n        target: Tensor,\n        p: float = 1.5,\n    ) -&gt; Tensor:\n        r\"\"\"\n        Parameters\n        ----------\n        input: Tensor\n            Input tensor with predictions\n        target: Tensor\n            Target tensor with the actual values\n        p: float, default = 1.5\n            the power to be used to compute the loss. See the original\n            publication for details\n\n        Examples\n        --------\n        &gt;&gt;&gt; import torch\n        &gt;&gt;&gt; from pytorch_widedeep.losses import TweedieLoss\n        &gt;&gt;&gt;\n        &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n        &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n        &gt;&gt;&gt; loss = TweedieLoss()(input, target)\n        \"\"\"\n\n        assert (\n            input.min() &gt; 0\n        ), \"\"\"All input values must be &gt;=0, if your model is predicting\n            values &lt;0 try to enforce positive values by activation function\n            on last layer with `trainer.enforce_positive_output=True`\"\"\"\n        assert target.min() &gt;= 0, \"All target values must be &gt;=0\"\n        loss = -target * torch.pow(input, 1 - p) / (1 - p) + torch.pow(input, 2 - p) / (\n            2 - p\n        )\n        return torch.mean(loss)\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.TweedieLoss.forward","title":"forward","text":"<pre><code>forward(input, target, p=1.5)\n</code></pre> <p>Parameters:</p> Name Type Description Default <code>input</code> <code>Tensor</code> <p>Input tensor with predictions</p> required <code>target</code> <code>Tensor</code> <p>Target tensor with the actual values</p> required <code>p</code> <code>float</code> <p>the power to be used to compute the loss. See the original publication for details</p> <code>1.5</code> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.losses import TweedieLoss\n&gt;&gt;&gt;\n&gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n&gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n&gt;&gt;&gt; loss = TweedieLoss()(input, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(\n    self,\n    input: Tensor,\n    target: Tensor,\n    p: float = 1.5,\n) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    input: Tensor\n        Input tensor with predictions\n    target: Tensor\n        Target tensor with the actual values\n    p: float, default = 1.5\n        the power to be used to compute the loss. See the original\n        publication for details\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.losses import TweedieLoss\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n    &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n    &gt;&gt;&gt; loss = TweedieLoss()(input, target)\n    \"\"\"\n\n    assert (\n        input.min() &gt; 0\n    ), \"\"\"All input values must be &gt;=0, if your model is predicting\n        values &lt;0 try to enforce positive values by activation function\n        on last layer with `trainer.enforce_positive_output=True`\"\"\"\n    assert target.min() &gt;= 0, \"All target values must be &gt;=0\"\n    loss = -target * torch.pow(input, 1 - p) / (1 - p) + torch.pow(input, 2 - p) / (\n        2 - p\n    )\n    return torch.mean(loss)\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.ZILNLoss","title":"ZILNLoss","text":"<p>               Bases: <code>Module</code></p> <p>Adjusted implementation of the Zero Inflated LogNormal Loss</p> <p>See A Deep Probabilistic Model for Customer Lifetime Value Prediction and the corresponding code.</p> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>class ZILNLoss(nn.Module):\n    r\"\"\"Adjusted implementation of the Zero Inflated LogNormal Loss\n\n    See [A Deep Probabilistic Model for Customer Lifetime Value Prediction](https://arxiv.org/pdf/1912.07753.pdf)\n    and the corresponding\n    [code](https://github.com/google/lifetime_value/blob/master/lifetime_value/zero_inflated_lognormal.py).\n    \"\"\"\n\n    def __init__(self):\n        super().__init__()\n\n    def forward(self, input: Tensor, target: Tensor) -&gt; Tensor:\n        r\"\"\"\n        Parameters\n        ----------\n        input: Tensor\n            Input tensor with predictions with spape (N,3), where N is the batch size\n        target: Tensor\n            Target tensor with the actual target values\n\n        Examples\n        --------\n        &gt;&gt;&gt; import torch\n        &gt;&gt;&gt; from pytorch_widedeep.losses import ZILNLoss\n        &gt;&gt;&gt;\n        &gt;&gt;&gt; target = torch.tensor([[0., 1.5]]).view(-1, 1)\n        &gt;&gt;&gt; input = torch.tensor([[.1, .2, .3], [.4, .5, .6]])\n        &gt;&gt;&gt; loss = ZILNLoss()(input, target)\n        \"\"\"\n        positive = target &gt; 0\n        positive = positive.float()\n\n        assert input.shape == torch.Size([target.shape[0], 3]), (\n            \"Wrong shape of the 'input' tensor. The pred_dim of the \"\n            \"model that is using ZILNLoss must be equal to 3.\"\n        )\n\n        positive_input = input[..., :1]\n\n        classification_loss = F.binary_cross_entropy_with_logits(\n            positive_input, positive, reduction=\"none\"\n        ).flatten()\n\n        loc = input[..., 1:2]\n\n        # when using max the two input tensors (input and other) have to be of\n        # the same type\n        max_input = F.softplus(input[..., 2:])\n        max_other = self.get_eps(max_input)\n        scale = torch.max(max_input, max_other)\n        safe_labels = positive * target + (1 - positive) * torch.ones_like(target)\n\n        regression_loss = -torch.mean(\n            positive\n            * torch.distributions.log_normal.LogNormal(loc=loc, scale=scale).log_prob(\n                safe_labels\n            ),\n            dim=-1,\n        )\n\n        return torch.mean(classification_loss + regression_loss)\n\n    @staticmethod\n    def get_eps(max_input: Tensor) -&gt; Tensor:\n        if max_input.device.type == \"mps\":\n            # For MPS, use float32 and then convert to the input type\n            eps = torch.finfo(torch.float32).eps\n            max_other = (\n                torch.sqrt(torch.tensor([eps], device=\"cpu\"))\n                .to(max_input.device)\n                .to(max_input.dtype)\n            )\n        else:\n            # For other devices, use the original approach\n            eps = torch.finfo(torch.double).eps\n            max_other = (\n                torch.sqrt(torch.tensor([eps])).to(max_input.device).to(max_input.dtype)\n            )\n\n        return max_other\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.ZILNLoss.forward","title":"forward","text":"<pre><code>forward(input, target)\n</code></pre> <p>Parameters:</p> Name Type Description Default <code>input</code> <code>Tensor</code> <p>Input tensor with predictions with spape (N,3), where N is the batch size</p> required <code>target</code> <code>Tensor</code> <p>Target tensor with the actual target values</p> required <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.losses import ZILNLoss\n&gt;&gt;&gt;\n&gt;&gt;&gt; target = torch.tensor([[0., 1.5]]).view(-1, 1)\n&gt;&gt;&gt; input = torch.tensor([[.1, .2, .3], [.4, .5, .6]])\n&gt;&gt;&gt; loss = ZILNLoss()(input, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(self, input: Tensor, target: Tensor) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    input: Tensor\n        Input tensor with predictions with spape (N,3), where N is the batch size\n    target: Tensor\n        Target tensor with the actual target values\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.losses import ZILNLoss\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; target = torch.tensor([[0., 1.5]]).view(-1, 1)\n    &gt;&gt;&gt; input = torch.tensor([[.1, .2, .3], [.4, .5, .6]])\n    &gt;&gt;&gt; loss = ZILNLoss()(input, target)\n    \"\"\"\n    positive = target &gt; 0\n    positive = positive.float()\n\n    assert input.shape == torch.Size([target.shape[0], 3]), (\n        \"Wrong shape of the 'input' tensor. The pred_dim of the \"\n        \"model that is using ZILNLoss must be equal to 3.\"\n    )\n\n    positive_input = input[..., :1]\n\n    classification_loss = F.binary_cross_entropy_with_logits(\n        positive_input, positive, reduction=\"none\"\n    ).flatten()\n\n    loc = input[..., 1:2]\n\n    # when using max the two input tensors (input and other) have to be of\n    # the same type\n    max_input = F.softplus(input[..., 2:])\n    max_other = self.get_eps(max_input)\n    scale = torch.max(max_input, max_other)\n    safe_labels = positive * target + (1 - positive) * torch.ones_like(target)\n\n    regression_loss = -torch.mean(\n        positive\n        * torch.distributions.log_normal.LogNormal(loc=loc, scale=scale).log_prob(\n            safe_labels\n        ),\n        dim=-1,\n    )\n\n    return torch.mean(classification_loss + regression_loss)\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.L1Loss","title":"L1Loss","text":"<p>               Bases: <code>Module</code></p> <p>L1 loss</p> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>class L1Loss(nn.Module):\n    r\"\"\"L1 loss\"\"\"\n\n    # legacy code from when we used to support FDS-LDS and this class could\n    # taked the corresponding params. At this stage probably you want to use\n    # torch.nn.L1Loss\n    def __init__(self):\n        super().__init__()\n\n    def forward(self, input: Tensor, target: Tensor) -&gt; Tensor:\n        r\"\"\"\n        Parameters\n        ----------\n        input: Tensor\n            Input tensor with predictions\n        target: Tensor\n            Target tensor with the actual values\n\n        Examples\n        --------\n        &gt;&gt;&gt; import torch\n        &gt;&gt;&gt;\n        &gt;&gt;&gt; from pytorch_widedeep.losses import L1Loss\n        &gt;&gt;&gt;\n        &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n        &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n        &gt;&gt;&gt; loss = L1Loss()(input, target)\n        \"\"\"\n        loss = F.l1_loss(input, target, reduction=\"none\")\n        return torch.mean(loss)\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.L1Loss.forward","title":"forward","text":"<pre><code>forward(input, target)\n</code></pre> <p>Parameters:</p> Name Type Description Default <code>input</code> <code>Tensor</code> <p>Input tensor with predictions</p> required <code>target</code> <code>Tensor</code> <p>Target tensor with the actual values</p> required <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt;\n&gt;&gt;&gt; from pytorch_widedeep.losses import L1Loss\n&gt;&gt;&gt;\n&gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n&gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n&gt;&gt;&gt; loss = L1Loss()(input, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(self, input: Tensor, target: Tensor) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    input: Tensor\n        Input tensor with predictions\n    target: Tensor\n        Target tensor with the actual values\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; from pytorch_widedeep.losses import L1Loss\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n    &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n    &gt;&gt;&gt; loss = L1Loss()(input, target)\n    \"\"\"\n    loss = F.l1_loss(input, target, reduction=\"none\")\n    return torch.mean(loss)\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.FocalR_L1Loss","title":"FocalR_L1Loss","text":"<p>               Bases: <code>Module</code></p> <p>Focal-R L1 loss</p> <p>Based on Delving into Deep Imbalanced Regression.</p> <p>Parameters:</p> Name Type Description Default <code>beta</code> <code>float</code> <p>Focal Loss <code>beta</code> parameter in their implementation</p> <code>0.2</code> <code>gamma</code> <code>float</code> <p>Focal Loss <code>gamma</code> parameter</p> <code>1.0</code> <code>activation_fn</code> <code>Literal[sigmoid, tanh]</code> <p>Activation function to be used during the computation of the loss. Possible values are 'sigmoid' and 'tanh'. See the original publication for details.</p> <code>'sigmoid'</code> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>class FocalR_L1Loss(nn.Module):\n    r\"\"\"Focal-R L1 loss\n\n    Based on [Delving into Deep Imbalanced Regression](https://arxiv.org/abs/2102.09554).\n\n    Parameters\n    ----------\n    beta: float\n        Focal Loss `beta` parameter in their implementation\n    gamma: float\n        Focal Loss `gamma` parameter\n    activation_fn: str, default = \"sigmoid\"\n        Activation function to be used during the computation of the loss.\n        Possible values are _'sigmoid'_ and _'tanh'_. See the original\n        publication for details.\n    \"\"\"\n\n    def __init__(\n        self,\n        beta: float = 0.2,\n        gamma: float = 1.0,\n        activation_fn: Literal[\"sigmoid\", \"tanh\"] = \"sigmoid\",\n    ):\n        super().__init__()\n        self.beta = beta\n        self.gamma = gamma\n        self.activation_fn = activation_fn\n\n    def forward(\n        self,\n        input: Tensor,\n        target: Tensor,\n    ) -&gt; Tensor:\n        r\"\"\"\n        Parameters\n        ----------\n        input: Tensor\n            Input tensor with predictions (not probabilities)\n        target: Tensor\n            Target tensor with the actual classes\n\n        Examples\n        --------\n        &gt;&gt;&gt; import torch\n        &gt;&gt;&gt;\n        &gt;&gt;&gt; from pytorch_widedeep.losses import FocalR_L1Loss\n        &gt;&gt;&gt;\n        &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n        &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n        &gt;&gt;&gt; loss = FocalR_L1Loss()(input, target)\n        \"\"\"\n        loss = F.l1_loss(input, target, reduction=\"none\")\n        if self.activation_fn == \"tanh\":\n            loss *= (torch.tanh(self.beta * torch.abs(input - target))) ** self.gamma\n        elif self.activation_fn == \"sigmoid\":\n            loss *= (\n                2 * torch.sigmoid(self.beta * torch.abs(input - target)) - 1\n            ) ** self.gamma\n        else:\n            ValueError(\n                \"Incorrect activation function value - must be in ['sigmoid', 'tanh']\"\n            )\n        return torch.mean(loss)\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.FocalR_L1Loss.forward","title":"forward","text":"<pre><code>forward(input, target)\n</code></pre> <p>Parameters:</p> Name Type Description Default <code>input</code> <code>Tensor</code> <p>Input tensor with predictions (not probabilities)</p> required <code>target</code> <code>Tensor</code> <p>Target tensor with the actual classes</p> required <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt;\n&gt;&gt;&gt; from pytorch_widedeep.losses import FocalR_L1Loss\n&gt;&gt;&gt;\n&gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n&gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n&gt;&gt;&gt; loss = FocalR_L1Loss()(input, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(\n    self,\n    input: Tensor,\n    target: Tensor,\n) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    input: Tensor\n        Input tensor with predictions (not probabilities)\n    target: Tensor\n        Target tensor with the actual classes\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; from pytorch_widedeep.losses import FocalR_L1Loss\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n    &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n    &gt;&gt;&gt; loss = FocalR_L1Loss()(input, target)\n    \"\"\"\n    loss = F.l1_loss(input, target, reduction=\"none\")\n    if self.activation_fn == \"tanh\":\n        loss *= (torch.tanh(self.beta * torch.abs(input - target))) ** self.gamma\n    elif self.activation_fn == \"sigmoid\":\n        loss *= (\n            2 * torch.sigmoid(self.beta * torch.abs(input - target)) - 1\n        ) ** self.gamma\n    else:\n        ValueError(\n            \"Incorrect activation function value - must be in ['sigmoid', 'tanh']\"\n        )\n    return torch.mean(loss)\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.FocalR_MSELoss","title":"FocalR_MSELoss","text":"<p>               Bases: <code>Module</code></p> <p>Focal-R MSE loss</p> <p>Based on Delving into Deep Imbalanced Regression.</p> <p>Parameters:</p> Name Type Description Default <code>beta</code> <code>float</code> <p>Focal Loss <code>beta</code> parameter in their implementation</p> <code>0.2</code> <code>gamma</code> <code>float</code> <p>Focal Loss <code>gamma</code> parameter</p> <code>1.0</code> <code>activation_fn</code> <code>Literal[sigmoid, tanh]</code> <p>Activation function to be used during the computation of the loss. Possible values are 'sigmoid' and 'tanh'. See the original publication for details.</p> <code>'sigmoid'</code> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>class FocalR_MSELoss(nn.Module):\n    r\"\"\"Focal-R MSE loss\n\n    Based on [Delving into Deep Imbalanced Regression](https://arxiv.org/abs/2102.09554).\n\n    Parameters\n    ----------\n    beta: float\n        Focal Loss `beta` parameter in their implementation\n    gamma: float\n        Focal Loss `gamma` parameter\n    activation_fn: str, default = \"sigmoid\"\n        Activation function to be used during the computation of the loss.\n        Possible values are _'sigmoid'_ and _'tanh'_. See the original\n        publication for details.\n    \"\"\"\n\n    def __init__(\n        self,\n        beta: float = 0.2,\n        gamma: float = 1.0,\n        activation_fn: Literal[\"sigmoid\", \"tanh\"] = \"sigmoid\",\n    ):\n        super().__init__()\n        self.beta = beta\n        self.gamma = gamma\n        self.activation_fn = activation_fn\n\n    def forward(\n        self,\n        input: Tensor,\n        target: Tensor,\n    ) -&gt; Tensor:\n        r\"\"\"\n        Parameters\n        ----------\n        input: Tensor\n            Input tensor with predictions (not probabilities)\n        target: Tensor\n            Target tensor with the actual classes\n\n        Examples\n        --------\n        &gt;&gt;&gt; import torch\n        &gt;&gt;&gt;\n        &gt;&gt;&gt; from pytorch_widedeep.losses import FocalR_MSELoss\n        &gt;&gt;&gt;\n        &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n        &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n        &gt;&gt;&gt; loss = FocalR_MSELoss()(input, target)\n        \"\"\"\n        loss = (input - target) ** 2\n        if self.activation_fn == \"tanh\":\n            loss *= (torch.tanh(self.beta * torch.abs(input - target))) ** self.gamma\n        elif self.activation_fn == \"sigmoid\":\n            loss *= (\n                2 * torch.sigmoid(self.beta * torch.abs((input - target) ** 2)) - 1\n            ) ** self.gamma\n        else:\n            ValueError(\n                \"Incorrect activation function value - must be in ['sigmoid', 'tanh']\"\n            )\n        return torch.mean(loss)\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.FocalR_MSELoss.forward","title":"forward","text":"<pre><code>forward(input, target)\n</code></pre> <p>Parameters:</p> Name Type Description Default <code>input</code> <code>Tensor</code> <p>Input tensor with predictions (not probabilities)</p> required <code>target</code> <code>Tensor</code> <p>Target tensor with the actual classes</p> required <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt;\n&gt;&gt;&gt; from pytorch_widedeep.losses import FocalR_MSELoss\n&gt;&gt;&gt;\n&gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n&gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n&gt;&gt;&gt; loss = FocalR_MSELoss()(input, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(\n    self,\n    input: Tensor,\n    target: Tensor,\n) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    input: Tensor\n        Input tensor with predictions (not probabilities)\n    target: Tensor\n        Target tensor with the actual classes\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; from pytorch_widedeep.losses import FocalR_MSELoss\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n    &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n    &gt;&gt;&gt; loss = FocalR_MSELoss()(input, target)\n    \"\"\"\n    loss = (input - target) ** 2\n    if self.activation_fn == \"tanh\":\n        loss *= (torch.tanh(self.beta * torch.abs(input - target))) ** self.gamma\n    elif self.activation_fn == \"sigmoid\":\n        loss *= (\n            2 * torch.sigmoid(self.beta * torch.abs((input - target) ** 2)) - 1\n        ) ** self.gamma\n    else:\n        ValueError(\n            \"Incorrect activation function value - must be in ['sigmoid', 'tanh']\"\n        )\n    return torch.mean(loss)\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.FocalR_RMSELoss","title":"FocalR_RMSELoss","text":"<p>               Bases: <code>Module</code></p> <p>Focal-R RMSE loss</p> <p>Based on Delving into Deep Imbalanced Regression.</p> <p>Parameters:</p> Name Type Description Default <code>beta</code> <code>float</code> <p>Focal Loss <code>beta</code> parameter in their implementation</p> <code>0.2</code> <code>gamma</code> <code>float</code> <p>Focal Loss <code>gamma</code> parameter</p> <code>1.0</code> <code>activation_fn</code> <code>Literal[sigmoid, tanh]</code> <p>Activation function to be used during the computation of the loss. Possible values are 'sigmoid' and 'tanh'. See the original publication for details.</p> <code>'sigmoid'</code> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>class FocalR_RMSELoss(nn.Module):\n    r\"\"\"Focal-R RMSE loss\n\n    Based on [Delving into Deep Imbalanced Regression](https://arxiv.org/abs/2102.09554).\n\n    Parameters\n    ----------\n    beta: float\n        Focal Loss `beta` parameter in their implementation\n    gamma: float\n        Focal Loss `gamma` parameter\n    activation_fn: str, default = \"sigmoid\"\n        Activation function to be used during the computation of the loss.\n        Possible values are _'sigmoid'_ and _'tanh'_. See the original\n        publication for details.\n    \"\"\"\n\n    def __init__(\n        self,\n        beta: float = 0.2,\n        gamma: float = 1.0,\n        activation_fn: Literal[\"sigmoid\", \"tanh\"] = \"sigmoid\",\n    ):\n        super().__init__()\n        self.beta = beta\n        self.gamma = gamma\n        self.activation_fn = activation_fn\n\n    def forward(\n        self,\n        input: Tensor,\n        target: Tensor,\n    ) -&gt; Tensor:\n        r\"\"\"\n        Parameters\n        ----------\n        input: Tensor\n            Input tensor with predictions (not probabilities)\n        target: Tensor\n            Target tensor with the actual classes\n\n        Examples\n        --------\n        &gt;&gt;&gt; import torch\n        &gt;&gt;&gt;\n        &gt;&gt;&gt; from pytorch_widedeep.losses import FocalR_RMSELoss\n        &gt;&gt;&gt;\n        &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n        &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n        &gt;&gt;&gt; loss = FocalR_RMSELoss()(input, target)\n        \"\"\"\n        loss = (input - target) ** 2\n        if self.activation_fn == \"tanh\":\n            loss *= (torch.tanh(self.beta * torch.abs(input - target))) ** self.gamma\n        elif self.activation_fn == \"sigmoid\":\n            loss *= (\n                2 * torch.sigmoid(self.beta * torch.abs((input - target) ** 2)) - 1\n            ) ** self.gamma\n        else:\n            ValueError(\n                \"Incorrect activation function value - must be in ['sigmoid', 'tanh']\"\n            )\n        return torch.sqrt(torch.mean(loss))\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.FocalR_RMSELoss.forward","title":"forward","text":"<pre><code>forward(input, target)\n</code></pre> <p>Parameters:</p> Name Type Description Default <code>input</code> <code>Tensor</code> <p>Input tensor with predictions (not probabilities)</p> required <code>target</code> <code>Tensor</code> <p>Target tensor with the actual classes</p> required <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt;\n&gt;&gt;&gt; from pytorch_widedeep.losses import FocalR_RMSELoss\n&gt;&gt;&gt;\n&gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n&gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n&gt;&gt;&gt; loss = FocalR_RMSELoss()(input, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(\n    self,\n    input: Tensor,\n    target: Tensor,\n) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    input: Tensor\n        Input tensor with predictions (not probabilities)\n    target: Tensor\n        Target tensor with the actual classes\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; from pytorch_widedeep.losses import FocalR_RMSELoss\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n    &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n    &gt;&gt;&gt; loss = FocalR_RMSELoss()(input, target)\n    \"\"\"\n    loss = (input - target) ** 2\n    if self.activation_fn == \"tanh\":\n        loss *= (torch.tanh(self.beta * torch.abs(input - target))) ** self.gamma\n    elif self.activation_fn == \"sigmoid\":\n        loss *= (\n            2 * torch.sigmoid(self.beta * torch.abs((input - target) ** 2)) - 1\n        ) ** self.gamma\n    else:\n        ValueError(\n            \"Incorrect activation function value - must be in ['sigmoid', 'tanh']\"\n        )\n    return torch.sqrt(torch.mean(loss))\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.HuberLoss","title":"HuberLoss","text":"<p>               Bases: <code>Module</code></p> <p>Hubbler Loss</p> <p>Based on Delving into Deep Imbalanced Regression.</p> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>class HuberLoss(nn.Module):\n    r\"\"\"Hubbler Loss\n\n    Based on [Delving into Deep Imbalanced Regression](https://arxiv.org/abs/2102.09554).\n    \"\"\"\n\n    def __init__(self, beta: float = 0.2):\n        super().__init__()\n        self.beta = beta\n\n    def forward(\n        self,\n        input: Tensor,\n        target: Tensor,\n    ) -&gt; Tensor:\n        r\"\"\"\n        Parameters\n        ----------\n        input: Tensor\n            Input tensor with predictions (not probabilities)\n        target: Tensor\n            Target tensor with the actual classes\n\n        Examples\n        --------\n        &gt;&gt;&gt; import torch\n        &gt;&gt;&gt;\n        &gt;&gt;&gt; from pytorch_widedeep.losses import HuberLoss\n        &gt;&gt;&gt;\n        &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n        &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n        &gt;&gt;&gt; loss = HuberLoss()(input, target)\n        \"\"\"\n        l1_loss = torch.abs(input - target)\n        cond = l1_loss &lt; self.beta\n        loss = torch.where(\n            cond, 0.5 * l1_loss**2 / self.beta, l1_loss - 0.5 * self.beta\n        )\n        return torch.mean(loss)\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.HuberLoss.forward","title":"forward","text":"<pre><code>forward(input, target)\n</code></pre> <p>Parameters:</p> Name Type Description Default <code>input</code> <code>Tensor</code> <p>Input tensor with predictions (not probabilities)</p> required <code>target</code> <code>Tensor</code> <p>Target tensor with the actual classes</p> required <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt;\n&gt;&gt;&gt; from pytorch_widedeep.losses import HuberLoss\n&gt;&gt;&gt;\n&gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n&gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n&gt;&gt;&gt; loss = HuberLoss()(input, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(\n    self,\n    input: Tensor,\n    target: Tensor,\n) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    input: Tensor\n        Input tensor with predictions (not probabilities)\n    target: Tensor\n        Target tensor with the actual classes\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; from pytorch_widedeep.losses import HuberLoss\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; target = torch.tensor([1, 1.2, 0, 2]).view(-1, 1)\n    &gt;&gt;&gt; input = torch.tensor([0.6, 0.7, 0.3, 0.8]).view(-1, 1)\n    &gt;&gt;&gt; loss = HuberLoss()(input, target)\n    \"\"\"\n    l1_loss = torch.abs(input - target)\n    cond = l1_loss &lt; self.beta\n    loss = torch.where(\n        cond, 0.5 * l1_loss**2 / self.beta, l1_loss - 0.5 * self.beta\n    )\n    return torch.mean(loss)\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.InfoNCELoss","title":"InfoNCELoss","text":"<p>               Bases: <code>Module</code></p> <p>InfoNCE Loss. Loss applied during the Contrastive Denoising Self Supervised Pre-training routine available in this library</p> <p> NOTE: This loss is in principle not exposed to  the user, as it is used internally in the library, but it is included  here for completion.</p> <p>See SAINT: Improved Neural Networks for Tabular Data via Row Attention and Contrastive Pre-Training and references therein</p> <p>Partially inspired by the code in this repo</p> <p>Parameters:</p> Name Type Description Default <code>temperature</code> <code>float</code> <p>The logits are divided by the temperature before computing the loss value</p> <code>0.1</code> <code>reduction</code> <code>str</code> <p>Loss reduction method</p> <code>'mean'</code> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>class InfoNCELoss(nn.Module):\n    r\"\"\"InfoNCE Loss. Loss applied during the Contrastive Denoising Self\n    Supervised Pre-training routine available in this library\n\n    :information_source: **NOTE**: This loss is in principle not exposed to\n     the user, as it is used internally in the library, but it is included\n     here for completion.\n\n    See [SAINT: Improved Neural Networks for Tabular Data via Row Attention\n    and Contrastive Pre-Training](https://arxiv.org/abs/2106.01342) and\n    references therein\n\n    Partially inspired by the code in this [repo](https://github.com/RElbers/info-nce-pytorch)\n\n    Parameters\n    ----------\n    temperature: float, default = 0.1\n        The logits are divided by the temperature before computing the loss value\n    reduction: str, default = \"mean\"\n        Loss reduction method\n    \"\"\"\n\n    def __init__(self, temperature: float = 0.1, reduction: str = \"mean\"):\n        super(InfoNCELoss, self).__init__()\n\n        self.temperature = temperature\n        self.reduction = reduction\n\n    def forward(self, g_projs: Tuple[Tensor, Tensor]) -&gt; Tensor:\n        r\"\"\"\n        Parameters\n        ----------\n        g_projs: Tuple\n            Tuple with the two tensors corresponding to the output of the two\n            projection heads, as described 'SAINT: Improved Neural Networks\n            for Tabular Data via Row Attention and Contrastive Pre-Training'.\n\n        Examples\n        --------\n        &gt;&gt;&gt; import torch\n        &gt;&gt;&gt; from pytorch_widedeep.losses import InfoNCELoss\n        &gt;&gt;&gt; g_projs = (torch.rand(3, 5, 16), torch.rand(3, 5, 16))\n        &gt;&gt;&gt; loss = InfoNCELoss()\n        &gt;&gt;&gt; res = loss(g_projs)\n        \"\"\"\n        z, z_ = g_projs[0], g_projs[1]\n\n        norm_z = F.normalize(z, dim=-1).flatten(1)\n        norm_z_ = F.normalize(z_, dim=-1).flatten(1)\n\n        logits = (norm_z @ norm_z_.t()) / self.temperature\n        logits_ = (norm_z_ @ norm_z.t()) / self.temperature\n\n        # the target/labels are the entries on the diagonal\n        target = torch.arange(len(norm_z), device=norm_z.device)\n\n        loss = F.cross_entropy(logits, target, reduction=self.reduction)\n        loss_ = F.cross_entropy(logits_, target, reduction=self.reduction)\n\n        return (loss + loss_) / 2.0\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.InfoNCELoss.forward","title":"forward","text":"<pre><code>forward(g_projs)\n</code></pre> <p>Parameters:</p> Name Type Description Default <code>g_projs</code> <code>Tuple[Tensor, Tensor]</code> <p>Tuple with the two tensors corresponding to the output of the two projection heads, as described 'SAINT: Improved Neural Networks for Tabular Data via Row Attention and Contrastive Pre-Training'.</p> required <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.losses import InfoNCELoss\n&gt;&gt;&gt; g_projs = (torch.rand(3, 5, 16), torch.rand(3, 5, 16))\n&gt;&gt;&gt; loss = InfoNCELoss()\n&gt;&gt;&gt; res = loss(g_projs)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(self, g_projs: Tuple[Tensor, Tensor]) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    g_projs: Tuple\n        Tuple with the two tensors corresponding to the output of the two\n        projection heads, as described 'SAINT: Improved Neural Networks\n        for Tabular Data via Row Attention and Contrastive Pre-Training'.\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.losses import InfoNCELoss\n    &gt;&gt;&gt; g_projs = (torch.rand(3, 5, 16), torch.rand(3, 5, 16))\n    &gt;&gt;&gt; loss = InfoNCELoss()\n    &gt;&gt;&gt; res = loss(g_projs)\n    \"\"\"\n    z, z_ = g_projs[0], g_projs[1]\n\n    norm_z = F.normalize(z, dim=-1).flatten(1)\n    norm_z_ = F.normalize(z_, dim=-1).flatten(1)\n\n    logits = (norm_z @ norm_z_.t()) / self.temperature\n    logits_ = (norm_z_ @ norm_z.t()) / self.temperature\n\n    # the target/labels are the entries on the diagonal\n    target = torch.arange(len(norm_z), device=norm_z.device)\n\n    loss = F.cross_entropy(logits, target, reduction=self.reduction)\n    loss_ = F.cross_entropy(logits_, target, reduction=self.reduction)\n\n    return (loss + loss_) / 2.0\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.DenoisingLoss","title":"DenoisingLoss","text":"<p>               Bases: <code>Module</code></p> <p>Denoising Loss. Loss applied during the Contrastive Denoising Self Supervised Pre-training routine available in this library</p> <p> NOTE: This loss is in principle not exposed to  the user, as it is used internally in the library, but it is included  here for completion.</p> <p>See SAINT: Improved Neural Networks for Tabular Data via Row Attention and Contrastive Pre-Training and references therein</p> <p>Parameters:</p> Name Type Description Default <code>lambda_cat</code> <code>float</code> <p>Multiplicative factor that will be applied to loss associated to the categorical features</p> <code>1.0</code> <code>lambda_cont</code> <code>float</code> <p>Multiplicative factor that will be applied to loss associated to the continuous features</p> <code>1.0</code> <code>reduction</code> <code>str</code> <p>Loss reduction method</p> <code>'mean'</code> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>class DenoisingLoss(nn.Module):\n    r\"\"\"Denoising Loss. Loss applied during the Contrastive Denoising Self\n    Supervised Pre-training routine available in this library\n\n    :information_source: **NOTE**: This loss is in principle not exposed to\n     the user, as it is used internally in the library, but it is included\n     here for completion.\n\n    See [SAINT: Improved Neural Networks for Tabular Data via Row Attention\n    and Contrastive Pre-Training](https://arxiv.org/abs/2106.01342) and\n    references therein\n\n    Parameters\n    ----------\n    lambda_cat: float, default = 1.\n        Multiplicative factor that will be applied to loss associated to the\n        categorical features\n    lambda_cont: float, default = 1.\n        Multiplicative factor that will be applied to loss associated to the\n        continuous features\n    reduction: str, default = \"mean\"\n        Loss reduction method\n    \"\"\"\n\n    def __init__(\n        self, lambda_cat: float = 1.0, lambda_cont: float = 1.0, reduction: str = \"mean\"\n    ):\n        super(DenoisingLoss, self).__init__()\n\n        self.lambda_cat = lambda_cat\n        self.lambda_cont = lambda_cont\n        self.reduction = reduction\n\n    def forward(\n        self,\n        x_cat_and_cat_: Optional[\n            Union[List[Tuple[Tensor, Tensor]], Tuple[Tensor, Tensor]]\n        ],\n        x_cont_and_cont_: Optional[\n            Union[List[Tuple[Tensor, Tensor]], Tuple[Tensor, Tensor]]\n        ],\n    ) -&gt; Tensor:\n        r\"\"\"\n        Parameters\n        ----------\n        x_cat_and_cat_: tuple of Tensors or lists of tuples\n            Tuple of tensors containing the raw input features and their\n            encodings, referred in the SAINT paper as $x$ and $x''$\n            respectively. If one denoising MLP is used per categorical\n            feature `x_cat_and_cat_` will be a list of tuples, one per\n            categorical feature\n        x_cont_and_cont_: tuple of Tensors or lists of tuples\n            same as `x_cat_and_cat_` but for continuous columns\n\n        Examples\n        --------\n        &gt;&gt;&gt; import torch\n        &gt;&gt;&gt; from pytorch_widedeep.losses import DenoisingLoss\n        &gt;&gt;&gt; x_cat_and_cat_ = (torch.empty(3).random_(3).long(), torch.randn(3, 3))\n        &gt;&gt;&gt; x_cont_and_cont_ = (torch.randn(3, 1), torch.randn(3, 1))\n        &gt;&gt;&gt; loss = DenoisingLoss()\n        &gt;&gt;&gt; res = loss(x_cat_and_cat_, x_cont_and_cont_)\n        \"\"\"\n\n        loss_cat = (\n            self._compute_cat_loss(x_cat_and_cat_)\n            if x_cat_and_cat_ is not None\n            else torch.tensor(0.0)\n        )\n        loss_cont = (\n            self._compute_cont_loss(x_cont_and_cont_)\n            if x_cont_and_cont_ is not None\n            else torch.tensor(0.0)\n        )\n\n        return self.lambda_cat * loss_cat + self.lambda_cont * loss_cont\n\n    def _compute_cat_loss(\n        self, x_cat_and_cat_: Union[List[Tuple[Tensor, Tensor]], Tuple[Tensor, Tensor]]\n    ) -&gt; Tensor:\n        loss_cat = torch.tensor(0.0, device=self._get_device(x_cat_and_cat_))\n        if isinstance(x_cat_and_cat_, list):\n            for x, x_ in x_cat_and_cat_:\n                loss_cat += F.cross_entropy(x_, x, reduction=self.reduction)\n        elif isinstance(x_cat_and_cat_, tuple):\n            x, x_ = x_cat_and_cat_\n            loss_cat += F.cross_entropy(x_, x, reduction=self.reduction)\n\n        return loss_cat\n\n    def _compute_cont_loss(self, x_cont_and_cont_) -&gt; Tensor:\n        loss_cont = torch.tensor(0.0, device=self._get_device(x_cont_and_cont_))\n        if isinstance(x_cont_and_cont_, list):\n            for x, x_ in x_cont_and_cont_:\n                loss_cont += F.mse_loss(x_, x, reduction=self.reduction)\n        elif isinstance(x_cont_and_cont_, tuple):\n            x, x_ = x_cont_and_cont_\n            loss_cont += F.mse_loss(x_, x, reduction=self.reduction)\n\n        return loss_cont\n\n    @staticmethod\n    def _get_device(\n        x_and_x_: Union[List[Tuple[Tensor, Tensor]], Tuple[Tensor, Tensor]]\n    ):\n        if isinstance(x_and_x_, tuple):\n            device = x_and_x_[0].device\n        elif isinstance(x_and_x_, list):\n            device = x_and_x_[0][0].device\n        return device\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.DenoisingLoss.forward","title":"forward","text":"<pre><code>forward(x_cat_and_cat_, x_cont_and_cont_)\n</code></pre> <p>Parameters:</p> Name Type Description Default <code>x_cat_and_cat_</code> <code>Optional[Union[List[Tuple[Tensor, Tensor]], Tuple[Tensor, Tensor]]]</code> <p>Tuple of tensors containing the raw input features and their encodings, referred in the SAINT paper as \\(x\\) and \\(x''\\) respectively. If one denoising MLP is used per categorical feature <code>x_cat_and_cat_</code> will be a list of tuples, one per categorical feature</p> required <code>x_cont_and_cont_</code> <code>Optional[Union[List[Tuple[Tensor, Tensor]], Tuple[Tensor, Tensor]]]</code> <p>same as <code>x_cat_and_cat_</code> but for continuous columns</p> required <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.losses import DenoisingLoss\n&gt;&gt;&gt; x_cat_and_cat_ = (torch.empty(3).random_(3).long(), torch.randn(3, 3))\n&gt;&gt;&gt; x_cont_and_cont_ = (torch.randn(3, 1), torch.randn(3, 1))\n&gt;&gt;&gt; loss = DenoisingLoss()\n&gt;&gt;&gt; res = loss(x_cat_and_cat_, x_cont_and_cont_)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(\n    self,\n    x_cat_and_cat_: Optional[\n        Union[List[Tuple[Tensor, Tensor]], Tuple[Tensor, Tensor]]\n    ],\n    x_cont_and_cont_: Optional[\n        Union[List[Tuple[Tensor, Tensor]], Tuple[Tensor, Tensor]]\n    ],\n) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    x_cat_and_cat_: tuple of Tensors or lists of tuples\n        Tuple of tensors containing the raw input features and their\n        encodings, referred in the SAINT paper as $x$ and $x''$\n        respectively. If one denoising MLP is used per categorical\n        feature `x_cat_and_cat_` will be a list of tuples, one per\n        categorical feature\n    x_cont_and_cont_: tuple of Tensors or lists of tuples\n        same as `x_cat_and_cat_` but for continuous columns\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.losses import DenoisingLoss\n    &gt;&gt;&gt; x_cat_and_cat_ = (torch.empty(3).random_(3).long(), torch.randn(3, 3))\n    &gt;&gt;&gt; x_cont_and_cont_ = (torch.randn(3, 1), torch.randn(3, 1))\n    &gt;&gt;&gt; loss = DenoisingLoss()\n    &gt;&gt;&gt; res = loss(x_cat_and_cat_, x_cont_and_cont_)\n    \"\"\"\n\n    loss_cat = (\n        self._compute_cat_loss(x_cat_and_cat_)\n        if x_cat_and_cat_ is not None\n        else torch.tensor(0.0)\n    )\n    loss_cont = (\n        self._compute_cont_loss(x_cont_and_cont_)\n        if x_cont_and_cont_ is not None\n        else torch.tensor(0.0)\n    )\n\n    return self.lambda_cat * loss_cat + self.lambda_cont * loss_cont\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.EncoderDecoderLoss","title":"EncoderDecoderLoss","text":"<p>               Bases: <code>Module</code></p> <p>'Standard' Encoder Decoder Loss. Loss applied during the Endoder-Decoder  Self-Supervised Pre-Training routine available in this library</p> <p> NOTE: This loss is in principle not exposed to  the user, as it is used internally in the library, but it is included  here for completion.</p> <p>The implementation of this lost is based on that at the tabnet repo, which is in itself an adaptation of that in the original paper TabNet: Attentive Interpretable Tabular Learning.</p> <p>Parameters:</p> Name Type Description Default <code>eps</code> <code>float</code> <p>Simply a small number to avoid dividing by zero</p> <code>1e-09</code> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>class EncoderDecoderLoss(nn.Module):\n    r\"\"\"'_Standard_' Encoder Decoder Loss. Loss applied during the Endoder-Decoder\n     Self-Supervised Pre-Training routine available in this library\n\n    :information_source: **NOTE**: This loss is in principle not exposed to\n     the user, as it is used internally in the library, but it is included\n     here for completion.\n\n    The implementation of this lost is based on that at the\n    [tabnet repo](https://github.com/dreamquark-ai/tabnet), which is in itself an\n    adaptation of that in the original paper [TabNet: Attentive\n    Interpretable Tabular Learning](https://arxiv.org/abs/1908.07442).\n\n    Parameters\n    ----------\n    eps: float\n        Simply a small number to avoid dividing by zero\n    \"\"\"\n\n    def __init__(self, eps: float = 1e-9):\n        super(EncoderDecoderLoss, self).__init__()\n        self.eps = eps\n\n    def forward(self, x_true: Tensor, x_pred: Tensor, mask: Tensor) -&gt; Tensor:\n        r\"\"\"\n        Parameters\n        ----------\n        x_true: Tensor\n            Embeddings of the input data\n        x_pred: Tensor\n            Reconstructed embeddings\n        mask: Tensor\n            Mask with 1s indicated that the reconstruction, and therefore the\n            loss, is based on those features.\n\n        Examples\n        --------\n        &gt;&gt;&gt; import torch\n        &gt;&gt;&gt; from pytorch_widedeep.losses import EncoderDecoderLoss\n        &gt;&gt;&gt; x_true = torch.rand(3, 3)\n        &gt;&gt;&gt; x_pred = torch.rand(3, 3)\n        &gt;&gt;&gt; mask = torch.empty(3, 3).random_(2)\n        &gt;&gt;&gt; loss = EncoderDecoderLoss()\n        &gt;&gt;&gt; res = loss(x_true, x_pred, mask)\n        \"\"\"\n\n        errors = x_pred - x_true\n\n        reconstruction_errors = torch.mul(errors, mask) ** 2\n\n        x_true_means = torch.mean(x_true, dim=0)\n        x_true_means[x_true_means == 0] = 1\n\n        x_true_stds = torch.std(x_true, dim=0) ** 2\n        x_true_stds[x_true_stds == 0] = x_true_means[x_true_stds == 0]\n\n        features_loss = torch.matmul(reconstruction_errors, 1 / x_true_stds)\n        nb_reconstructed_variables = torch.sum(mask, dim=1)\n        features_loss_norm = features_loss / (nb_reconstructed_variables + self.eps)\n\n        loss = torch.mean(features_loss_norm)\n\n        return loss\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses.EncoderDecoderLoss.forward","title":"forward","text":"<pre><code>forward(x_true, x_pred, mask)\n</code></pre> <p>Parameters:</p> Name Type Description Default <code>x_true</code> <code>Tensor</code> <p>Embeddings of the input data</p> required <code>x_pred</code> <code>Tensor</code> <p>Reconstructed embeddings</p> required <code>mask</code> <code>Tensor</code> <p>Mask with 1s indicated that the reconstruction, and therefore the loss, is based on those features.</p> required <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.losses import EncoderDecoderLoss\n&gt;&gt;&gt; x_true = torch.rand(3, 3)\n&gt;&gt;&gt; x_pred = torch.rand(3, 3)\n&gt;&gt;&gt; mask = torch.empty(3, 3).random_(2)\n&gt;&gt;&gt; loss = EncoderDecoderLoss()\n&gt;&gt;&gt; res = loss(x_true, x_pred, mask)\n</code></pre> Source code in <code>pytorch_widedeep/losses.py</code> <pre><code>def forward(self, x_true: Tensor, x_pred: Tensor, mask: Tensor) -&gt; Tensor:\n    r\"\"\"\n    Parameters\n    ----------\n    x_true: Tensor\n        Embeddings of the input data\n    x_pred: Tensor\n        Reconstructed embeddings\n    mask: Tensor\n        Mask with 1s indicated that the reconstruction, and therefore the\n        loss, is based on those features.\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.losses import EncoderDecoderLoss\n    &gt;&gt;&gt; x_true = torch.rand(3, 3)\n    &gt;&gt;&gt; x_pred = torch.rand(3, 3)\n    &gt;&gt;&gt; mask = torch.empty(3, 3).random_(2)\n    &gt;&gt;&gt; loss = EncoderDecoderLoss()\n    &gt;&gt;&gt; res = loss(x_true, x_pred, mask)\n    \"\"\"\n\n    errors = x_pred - x_true\n\n    reconstruction_errors = torch.mul(errors, mask) ** 2\n\n    x_true_means = torch.mean(x_true, dim=0)\n    x_true_means[x_true_means == 0] = 1\n\n    x_true_stds = torch.std(x_true, dim=0) ** 2\n    x_true_stds[x_true_stds == 0] = x_true_means[x_true_stds == 0]\n\n    features_loss = torch.matmul(reconstruction_errors, 1 / x_true_stds)\n    nb_reconstructed_variables = torch.sum(mask, dim=1)\n    features_loss_norm = features_loss / (nb_reconstructed_variables + self.eps)\n\n    loss = torch.mean(features_loss_norm)\n\n    return loss\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses_multitarget.MultiTargetRegressionLoss","title":"MultiTargetRegressionLoss","text":"<p>               Bases: <code>Module</code></p> <p>This class is a wrapper around the Pytorch MSELoss. It allows for multi-target regression problems. The user can provide a list of weights to apply to each target. The loss can be either the sum or the mean of the individual losses</p> <p>Parameters:</p> Name Type Description Default <code>weights</code> <code>Optional[List[float]]</code> <p>List of weights to apply to the loss associated to each target. The length of the list must match the number of targets. Alias: 'target_weights'</p> <code>None</code> <code>reduction</code> <code>Literal[mean, sum]</code> <p>Specifies the reduction to apply to the loss associated to each target: 'mean' | 'sum'. Note that this is NOT the same as the reduction in the MSELoss. This reduction is applied after the loss for each target has been computed. Alias: 'target_reduction'</p> <code>'mean'</code> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.losses_multitarget import MultiTargetRegressionLoss\n&gt;&gt;&gt; input = torch.randn(3, 2)\n&gt;&gt;&gt; target = torch.randn(3, 2)\n&gt;&gt;&gt; loss = MultiTargetRegressionLoss(weights=[0.5, 0.5], reduction=\"mean\")\n&gt;&gt;&gt; output = loss(input, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses_multitarget.py</code> <pre><code>class MultiTargetRegressionLoss(nn.Module):\n    \"\"\"\n    This class is a wrapper around the Pytorch MSELoss. It allows for multi-target\n    regression problems. The user can provide a list of weights to apply to each\n    target. The loss can be either the sum or the mean of the individual losses\n\n    Parameters\n    ----------\n    weights: Optional[List[float], default = None]\n        List of weights to apply to the loss associated to each target. The\n        length of the list must match the number of targets.\n        Alias: 'target_weights'\n    reduction: Literal[\"mean\", \"sum\"], default = \"mean\n        Specifies the reduction to apply to the loss associated to each\n        target: 'mean' | 'sum'. Note that this is NOT the same as the\n        reduction in the MSELoss. This reduction is applied after the loss\n        for each target has been computed. Alias: 'target_reduction'\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.losses_multitarget import MultiTargetRegressionLoss\n    &gt;&gt;&gt; input = torch.randn(3, 2)\n    &gt;&gt;&gt; target = torch.randn(3, 2)\n    &gt;&gt;&gt; loss = MultiTargetRegressionLoss(weights=[0.5, 0.5], reduction=\"mean\")\n    &gt;&gt;&gt; output = loss(input, target)\n    \"\"\"\n\n    @alias(\"reduction\", [\"target_reduction\"])\n    @alias(\"weights\", [\"target_weights\"])\n    def __init__(\n        self,\n        weights: Optional[List[float]] = None,\n        reduction: Literal[\"mean\", \"sum\"] = \"mean\",\n    ):\n        super(MultiTargetRegressionLoss, self).__init__()\n\n        self.weights = weights\n        self.reduction = reduction\n\n        if self.reduction not in [\"mean\", \"sum\"]:\n            raise ValueError(\"reduction must be either 'mean' or 'sum'\")\n\n    def forward(self, input: Tensor, target: Tensor) -&gt; Tensor:\n\n        assert input.size() == target.size()\n\n        if self.weights is not None:\n\n            assert len(self.weights) == input.size(1), (\n                \"The number of weights must match the number of targets. \"\n                f\"Got {len(self.weights)} weights and {input.size(1)} targets\"\n            )\n\n            loss = F.mse_loss(input, target, reduction=\"none\") * torch.tensor(\n                self.weights\n            ).to(input.device)\n        else:\n            loss = F.mse_loss(input, target, reduction=\"none\")\n\n        return loss.mean() if self.reduction == \"mean\" else loss.sum()\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses_multitarget.MultiTargetClassificationLoss","title":"MultiTargetClassificationLoss","text":"<p>               Bases: <code>Module</code></p> <p>This class is a wrapper around the Pytorch binary_cross_entropy_with_logits and cross_entropy losses. It allows for multi-target classification problems. The user can provide a list of weights to apply to each target. The loss can be either the sum or the mean of the individual losses</p> <p>Parameters:</p> Name Type Description Default <code>binary_config</code> <code>Optional[List[Union[int, Tuple[int, float]]]]</code> <p>List of integers with the index of the target for binary classification or tuples with two elements: the index of the targets or binary classification and the positive weight for binary classification</p> <code>None</code> <code>multiclass_config</code> <code>Optional[List[Union[Tuple[int, int], Tuple[int, int, List[float]]]]]</code> <p>List of tuples with two or three elements: the index of the target and the number of classes for multiclass classification, or a tuple with the index of the target, the number of classes and a list of weights to apply to each class (i.e. the 'weight' parameter in the cross_entropy loss)</p> <code>None</code> <code>weights</code> <code>Optional[List[float]]</code> <p>List of weights to apply to the loss associated to each target. The length of the list must match the number of targets. Alias: 'target_weights'</p> <code>None</code> <code>reduction</code> <code>Literal[mean, sum]</code> <p>Specifies the reduction to apply to the loss associated to each target: 'mean' | 'sum'. Note that this is NOT the same as the reduction in the cross_entropy loss or the binary_cross_entropy_with_logits. This reduction is applied after the loss for each target has been computed. Alias: 'target_reduction'</p> <code>'mean'</code> <code>binary_trick</code> <code>bool</code> <p>If True, each target will be considered independently and the loss will be computed as binary_cross_entropy_with_logits. This is a faster implementation. Note that the 'weights' parameter is not compatible with binary_trick=True. Also note that if binary_trick=True, the 'binary_config' must be a list of integers and the 'multiclass_config' must be a list of tuples with two integers: the index of the target and the number of classes. Finally, if binary_trick=True, the binary targets must be the first targets in the target tensor.</p> <p> NOTE: When using the binary_trick, the binary targets are   considered as 2 classes. Therefore, the pred_dim parametere of the   WideDeep class should be adjusted accordingly (adding 2 to per   binary target). For example, in a problem with a binary target and   a 4 class multiclassification target, the pred_dim should be 6.</p> <code>False</code> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.losses_multitarget import MultiTargetClassificationLoss\n&gt;&gt;&gt; input = torch.randn(5, 4)\n&gt;&gt;&gt; input_binary_trick = torch.randn(5, 5)\n&gt;&gt;&gt; target = torch.stack([torch.tensor([0, 1, 0, 1, 1]), torch.tensor([0, 1, 2, 0, 2])], 1)\n&gt;&gt;&gt; loss_1 = MultiTargetClassificationLoss(binary_config=[0], multiclass_config=[(1, 3)], reduction=\"mean\")\n&gt;&gt;&gt; output_1 = loss_1(input, target)\n&gt;&gt;&gt; loss_2 = MultiTargetClassificationLoss(binary_config=[(0, 0.5)], multiclass_config=[(1, 3, [1., 2., 3.])],\n... reduction=\"sum\", weights=[0.5, 0.5])\n&gt;&gt;&gt; output_2 = loss_2(input, target)\n&gt;&gt;&gt; loss_3 = MultiTargetClassificationLoss(binary_config=[0], multiclass_config=[(1, 3)], binary_trick=True)\n&gt;&gt;&gt; output_3 = loss_3(input_binary_trick, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses_multitarget.py</code> <pre><code>class MultiTargetClassificationLoss(nn.Module):\n    \"\"\"\n    This class is a wrapper around the Pytorch binary_cross_entropy_with_logits and\n    cross_entropy losses. It allows for multi-target classification problems. The\n    user can provide a list of weights to apply to each target. The loss can be\n    either the sum or the mean of the individual losses\n\n    Parameters\n    ----------\n    binary_config: Optional[List[int | Tuple[int, float]]], default = None\n        List of integers with the index of the target for binary\n        classification or tuples with two elements: the index of the targets\n        or binary classification and the positive weight for binary\n        classification\n    multiclass_config: Optional[Tuple[int, int] | Tuple[int, int, List[float]]], default = None\n        List of tuples with two or three elements: the index of the target and the\n        number of classes for multiclass classification, or a tuple with the index of\n        the target, the number of classes and a list of weights to apply to each class\n        (i.e. the 'weight' parameter in the cross_entropy loss)\n    weights: Optional[List[float], default = None]\n        List of weights to apply to the loss associated to each target. The\n        length of the list must match the number of targets.\n        Alias: 'target_weights'\n    reduction: Literal[\"mean\", \"sum\"], default = \"sum\n        Specifies the reduction to apply to the loss associated to each\n        target: 'mean' | 'sum'. Note that this is NOT the same as the\n        reduction in the cross_entropy loss or the\n        binary_cross_entropy_with_logits. This reduction is applied after the\n        loss for each target has been computed. Alias: 'target_reduction'\n    binary_trick: bool, default = False\n        If True, each target will be considered independently and the loss\n        will be computed as binary_cross_entropy_with_logits. This is a\n        faster implementation. Note that the 'weights' parameter is not\n        compatible with binary_trick=True. Also note that if\n        binary_trick=True, the 'binary_config' must be a list of integers and\n        the 'multiclass_config' must be a list of tuples with two integers:\n        the index of the target and the number of classes. Finally, if\n        binary_trick=True, the binary targets must be the first targets in\n        the target tensor.\n\n        :information_source: **NOTE**: When using the binary_trick, the binary targets are\n          considered as 2 classes. Therefore, the pred_dim parametere of the\n          WideDeep class should be adjusted accordingly (adding 2 to per\n          binary target). For example, in a problem with a binary target and\n          a 4 class multiclassification target, the pred_dim should be 6.\n\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.losses_multitarget import MultiTargetClassificationLoss\n    &gt;&gt;&gt; input = torch.randn(5, 4)\n    &gt;&gt;&gt; input_binary_trick = torch.randn(5, 5)\n    &gt;&gt;&gt; target = torch.stack([torch.tensor([0, 1, 0, 1, 1]), torch.tensor([0, 1, 2, 0, 2])], 1)\n    &gt;&gt;&gt; loss_1 = MultiTargetClassificationLoss(binary_config=[0], multiclass_config=[(1, 3)], reduction=\"mean\")\n    &gt;&gt;&gt; output_1 = loss_1(input, target)\n    &gt;&gt;&gt; loss_2 = MultiTargetClassificationLoss(binary_config=[(0, 0.5)], multiclass_config=[(1, 3, [1., 2., 3.])],\n    ... reduction=\"sum\", weights=[0.5, 0.5])\n    &gt;&gt;&gt; output_2 = loss_2(input, target)\n    &gt;&gt;&gt; loss_3 = MultiTargetClassificationLoss(binary_config=[0], multiclass_config=[(1, 3)], binary_trick=True)\n    &gt;&gt;&gt; output_3 = loss_3(input_binary_trick, target)\n    \"\"\"\n\n    @alias(\"reduction\", [\"target_reduction\"])\n    @alias(\"weights\", [\"target_weights\"])\n    def __init__(  # noqa: C901\n        self,\n        binary_config: Optional[List[Union[int, Tuple[int, float]]]] = None,\n        multiclass_config: Optional[\n            List[Union[Tuple[int, int], Tuple[int, int, List[float]]]]\n        ] = None,\n        weights: Optional[List[float]] = None,\n        reduction: Literal[\"mean\", \"sum\"] = \"mean\",\n        binary_trick: bool = False,\n    ):\n        super(MultiTargetClassificationLoss, self).__init__()\n\n        if reduction not in [\"mean\", \"sum\"]:\n            raise ValueError(\"reduction must be either 'mean' or 'sum'\")\n\n        self.binary_config = binary_config\n        self.multiclass_config = multiclass_config\n        self.weights = weights\n        self.reduction = reduction\n        self.binary_trick = binary_trick\n\n        if self.weights is not None:\n            if len(self.weights) != (\n                len(self.binary_config) if self.binary_config is not None else 0\n            ) + (\n                len(self.multiclass_config) if self.multiclass_config is not None else 0\n            ):\n                raise ValueError(\n                    \"The number of weights must match the number of binary and multiclass targets\"\n                )\n\n        if self.binary_trick:\n            self._check_inputs_with_binary_trick()\n            self._binary_config: List[int] = binary_config  # type: ignore[assignment]\n            self._multiclass_config: List[Tuple[int, int]] = self.multiclass_config  # type: ignore[assignment]\n        else:\n            self.binary_config_with_pos_weights = (\n                (self._set_binary_config_without_binary_trick())\n                if self.binary_config is not None\n                else None\n            )\n            self.multiclass_config_with_weights = (\n                (self._set_multiclass_config_without_binary_trick())\n                if self.multiclass_config is not None\n                else None\n            )\n\n    def forward(self, input: Tensor, target: Tensor) -&gt; Tensor:\n        if self.binary_trick:\n            return self._forward_binary_trick(input, target)\n        else:\n            return self._forward_without_binary_trick(input, target)\n\n    def _forward_binary_trick(self, input: Tensor, target: Tensor) -&gt; Tensor:\n        binary_target_tensors: List[Tensor] = []\n        if self._binary_config:\n            for idx in self._binary_config:\n                binary_target_tensors.append(\n                    torch.eye(2)[target[:, idx].long()].to(input.device)\n                )\n        if self._multiclass_config:\n            for idx, n_classes in self._multiclass_config:\n                binary_target_tensors.append(\n                    torch.eye(n_classes)[target[:, idx].long()].to(input.device)\n                )\n        binary_target = torch.cat(binary_target_tensors, 1)\n        return F.binary_cross_entropy_with_logits(input, binary_target)\n\n    def _forward_without_binary_trick(self, input: Tensor, target: Tensor) -&gt; Tensor:\n        losses: List[Tensor] = []\n        if self.binary_config_with_pos_weights:\n            for idx, bpos_weight in self.binary_config_with_pos_weights:\n                _loss = F.binary_cross_entropy_with_logits(\n                    input[:, idx],\n                    target[:, idx].float(),\n                    pos_weight=(\n                        torch.tensor(bpos_weight).to(input.device)\n                        if bpos_weight is not None\n                        else None\n                    ),\n                )\n                losses.append(_loss)\n        if self.multiclass_config_with_weights:\n            for idx, n_classes, mpos_weight in self.multiclass_config_with_weights:\n                _loss = F.cross_entropy(\n                    input[:, idx : idx + n_classes],\n                    target[:, idx].long(),\n                    weight=(\n                        torch.tensor(mpos_weight).to(input.device)\n                        if mpos_weight is not None\n                        else None\n                    ),\n                )\n                losses.append(_loss)\n\n            if self.weights is not None:\n                losses = [l * w for l, w in zip(losses, self.weights)]  # noqa: E741\n\n        return (\n            torch.stack(losses).sum()\n            if self.reduction == \"sum\"\n            else torch.stack(losses).mean()\n        )\n\n    def _check_inputs_with_binary_trick(self):\n        if self.binary_config is not None:\n            if any(isinstance(bc, tuple) for bc in self.binary_config):\n                raise ValueError(\n                    \"binary_trick=True is only compatible with binary_config as a list of integers\"\n                )\n\n        if self.multiclass_config is not None:\n            if not all(len(mc) == 2 for mc in self.multiclass_config):\n                raise ValueError(\n                    \"binary_trick=True is only compatible with multiclass_config as a list of \"\n                    \"tuples with two integers: the index of the target and the number of classes\"\n                )\n\n        if self.binary_config is not None and self.multiclass_config is not None:\n            last_binary_idx = (\n                self.binary_config[-1][0]\n                if isinstance(self.binary_config[-1], tuple)\n                else self.binary_config[-1]\n            )\n            if last_binary_idx &gt;= self.multiclass_config[0][0]:\n                raise ValueError(\n                    \"When using binary_trick=True, the binary targets must be the first targets\"\n                    \" in the target tensor\"\n                )\n\n    def _set_binary_config_without_binary_trick(\n        self,\n    ) -&gt; List[Tuple[int, Optional[float]]]:\n        binary_config_with_pos_weights: List[Tuple[int, Optional[float]]] = []\n        for bc in self.binary_config:\n            if isinstance(bc, tuple):\n                binary_config_with_pos_weights.append(bc)\n            else:\n                binary_config_with_pos_weights.append((bc, None))\n        return binary_config_with_pos_weights\n\n    def _set_multiclass_config_without_binary_trick(\n        self,\n    ) -&gt; List[Tuple[int, int, Optional[List[float]]]]:\n        multiclass_config_with_weights: List[Tuple[int, int, Optional[List[float]]]] = (\n            []\n        )\n        for mc in self.multiclass_config:\n            if len(mc) == 3:\n                multiclass_config_with_weights.append(mc)  # type: ignore[arg-type]\n            else:\n                multiclass_config_with_weights.append((mc[0], mc[1], None))\n        return multiclass_config_with_weights\n</code></pre>"},{"location":"pytorch-widedeep/losses.html#pytorch_widedeep.losses_multitarget.MutilTargetRegressionAndClassificationLoss","title":"MutilTargetRegressionAndClassificationLoss","text":"<p>               Bases: <code>Module</code></p> <p>This class is a wrapper around the MultiTargetRegressionLoss and the MultiTargetClassificationLoss. It allows for multi-target regression and classification problems. The user can provide a list of weights to apply to each target. The loss can be either the sum or the mean of the individual losses</p> <p>Parameters:</p> Name Type Description Default <code>regression_config</code> <code>List[int]</code> <p>List of integers with the indices of the regression targets</p> <code>[]</code> <code>binary_config</code> <code>Optional[List[Union[int, Tuple[int, float]]]]</code> <p>List of integers with the index of the target for binary classification or tuples with two elements: the index of the targets or binary classification and the positive weight for binary classification</p> <code>None</code> <code>multiclass_config</code> <code>Optional[List[Union[Tuple[int, int], Tuple[int, int, List[float]]]]]</code> <p>List of tuples with two or three elements: the index of the target and the number of classes for multiclass classification, or a tuple with the index of the target, the number of classes and a list of weights to apply to each class (i.e. the 'weight' parameter in the cross_entropy loss)</p> <code>None</code> <code>weights</code> <code>Optional[List[float]]</code> <p>List of weights to apply to the loss associated to each target. The length of the list must match the number of targets. Alias: 'target_weights'</p> <code>None</code> <code>reduction</code> <code>Literal[mean, sum]</code> <p>Specifies the reduction to apply to the output: 'mean' | 'sum'. Note that this is NOT the same as the reduction in the cross_entropy loss, the binary_cross_entropy_with_logits or the MSELoss. This reduction is applied after each target has been computed. Alias: 'target_reduction'</p> <code>'mean'</code> <code>binary_trick</code> <code>bool</code> <p>If True, each target will be considered independently and the loss will be computed as binary_cross_entropy_with_logits. This is a faster implementation. Note that the 'weights' parameter is not compatible with binary_trick=True. Also note that if binary_trick=True, the 'binary_config' must be a list of integers and the 'multiclass_config' must be a list of tuples with two integers: the index of the target and the number of classes. Finally, if binary_trick=True, the binary targets must be the first targets in the target tensor.</p> <p> NOTE: When using the binary_trick, the binary targets are   considered as 2 classes. Therefore, the pred_dim parametere of the   WideDeep class should be adjusted accordingly (adding 2 to per   binary target). For example, in a problem with a binary target and   a 4 class multiclassification target, the pred_dim should be 6.</p> <code>False</code> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.losses_multitarget import MutilTargetRegressionAndClassificationLoss\n&gt;&gt;&gt; input = torch.randn(5, 5)\n&gt;&gt;&gt; target = torch.stack([torch.randn(5), torch.tensor([0, 1, 0, 1, 1]), torch.tensor([0, 1, 2, 0, 2])], 1)\n&gt;&gt;&gt; loss = MutilTargetRegressionAndClassificationLoss(regression_config=[0], binary_config=[2],\n... multiclass_config=[(2, 3)], reduction=\"mean\")\n&gt;&gt;&gt; output = loss(input, target)\n</code></pre> Source code in <code>pytorch_widedeep/losses_multitarget.py</code> <pre><code>class MutilTargetRegressionAndClassificationLoss(nn.Module):\n    \"\"\"\n    This class is a wrapper around the MultiTargetRegressionLoss and the\n    MultiTargetClassificationLoss. It allows for multi-target regression and\n    classification problems. The user can provide a list of weights to apply to\n    each target. The loss can be either the sum or the mean of the individual losses\n\n    Parameters\n    ----------\n    regression_config: List[int], default = []\n        List of integers with the indices of the regression targets\n    binary_config: Optional[List[int | Tuple[int, float]]], default = None\n        List of integers with the index of the target for binary\n        classification or tuples with two elements: the index of the targets\n        or binary classification and the positive weight for binary\n        classification\n    multiclass_config: Optional[Tuple[int, int] | Tuple[int, int, List[float]]], default = None\n        List of tuples with two or three elements: the index of the target and the\n        number of classes for multiclass classification, or a tuple with the index of\n        the target, the number of classes and a list of weights to apply to each class\n        (i.e. the 'weight' parameter in the cross_entropy loss)\n    weights: Optional[List[float], default = None]\n        List of weights to apply to the loss associated to each target. The\n        length of the list must match the number of targets.\n        Alias: 'target_weights'\n    reduction: Literal[\"mean\", \"sum\"], default = \"sum\n        Specifies the reduction to apply to the output: 'mean' | 'sum'. Note\n        that this is NOT the same as the reduction in the cross_entropy loss,\n        the binary_cross_entropy_with_logits or the MSELoss. This reduction\n        is applied after each target has been computed. Alias: 'target_reduction'\n    binary_trick: bool, default = False\n        If True, each target will be considered independently and the loss\n        will be computed as binary_cross_entropy_with_logits. This is a\n        faster implementation. Note that the 'weights' parameter is not\n        compatible with binary_trick=True. Also note that if\n        binary_trick=True, the 'binary_config' must be a list of integers and\n        the 'multiclass_config' must be a list of tuples with two integers:\n        the index of the target and the number of classes. Finally, if\n        binary_trick=True, the binary targets must be the first targets in\n        the target tensor.\n\n        :information_source: **NOTE**: When using the binary_trick, the binary targets are\n          considered as 2 classes. Therefore, the pred_dim parametere of the\n          WideDeep class should be adjusted accordingly (adding 2 to per\n          binary target). For example, in a problem with a binary target and\n          a 4 class multiclassification target, the pred_dim should be 6.\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.losses_multitarget import MutilTargetRegressionAndClassificationLoss\n    &gt;&gt;&gt; input = torch.randn(5, 5)\n    &gt;&gt;&gt; target = torch.stack([torch.randn(5), torch.tensor([0, 1, 0, 1, 1]), torch.tensor([0, 1, 2, 0, 2])], 1)\n    &gt;&gt;&gt; loss = MutilTargetRegressionAndClassificationLoss(regression_config=[0], binary_config=[2],\n    ... multiclass_config=[(2, 3)], reduction=\"mean\")\n    &gt;&gt;&gt; output = loss(input, target)\n    \"\"\"\n\n    @alias(\"reduction\", [\"target_reduction\"])\n    @alias(\"weights\", [\"target_weights\"])\n    def __init__(  # noqa: C901\n        self,\n        regression_config: List[int] = [],\n        binary_config: Optional[List[Union[int, Tuple[int, float]]]] = None,\n        multiclass_config: Optional[\n            List[Union[Tuple[int, int], Tuple[int, int, List[float]]]]\n        ] = None,\n        weights: Optional[List[float]] = None,\n        reduction: Literal[\"mean\", \"sum\"] = \"mean\",\n        binary_trick: bool = False,\n    ):\n\n        super(MutilTargetRegressionAndClassificationLoss, self).__init__()\n\n        self.regression_config = regression_config\n\n        assert binary_config is not None or multiclass_config is not None, (\n            \"Either binary_config or multiclass_config must be provided. \"\n            \"Otherwise, use the MultiTargetRegressionLoss\"\n        )\n\n        if binary_trick:\n            self._check_inputs_with_binary_trick(\n                regression_config, binary_config, multiclass_config\n            )\n\n        if weights is not None:\n            if len(weights) != (\n                len(regression_config)\n                + (len(binary_config) if binary_config is not None else 0)\n                + (len(multiclass_config) if multiclass_config is not None else 0)\n            ):\n                raise ValueError(\n                    \"The number of weights must match the number of regression, binary and multiclass targets\"\n                )\n\n            self.weights_regression = self._prepare_weights_for_regression_targets(\n                weights, regression_config\n            )\n            self.weights_binary = self._prepare_weights_per_binary_targets(\n                weights, binary_config\n            )\n            self.weights_multiclass = self._prepare_weights_per_multiclass_targets(\n                weights, multiclass_config\n            )\n            self.weights = weights\n        else:\n            self.weights_regression = None\n            self.weights_binary = None\n            self.weights_multiclass = None\n\n        self.multi_target_regression_loss = MultiTargetRegressionLoss(\n            weights=self.weights_regression, reduction=reduction\n        )\n\n        self.multi_target_classification_loss = MultiTargetClassificationLoss(\n            binary_config=binary_config,\n            multiclass_config=multiclass_config,\n            weights=(\n                self.weights_binary + self.weights_multiclass\n                if self.weights_binary is not None\n                and self.weights_multiclass is not None\n                else (\n                    self.weights_binary\n                    if self.weights_binary is not None\n                    else self.weights_multiclass\n                )\n            ),\n            reduction=reduction,\n            binary_trick=binary_trick,\n        )\n\n    def forward(self, input: Tensor, target: Tensor) -&gt; Tensor:\n\n        regression_loss = self.multi_target_regression_loss(\n            input[:, self.regression_config],\n            target[:, self.regression_config],\n        )\n\n        if self.multi_target_classification_loss.binary_trick:\n            classification_loss = self.multi_target_classification_loss(\n                input[:, len(self.regression_config) :], target\n            )\n        else:\n            classification_loss = self.multi_target_classification_loss(input, target)\n\n        return regression_loss + classification_loss\n\n    def _check_inputs_with_binary_trick(\n        self,\n        regression_config: List[int],\n        binary_config: Optional[List[Union[int, Tuple[int, float]]]],\n        multiclass_config: Optional[\n            List[Union[Tuple[int, int], Tuple[int, int, List[float]]]]\n        ],\n    ) -&gt; None:\n\n        error_msg = \"When using binary_trick=True, the targets order must be: regression, binary and multiclass\"\n\n        first_regression_idx = regression_config[0]\n        last_regression_idx = regression_config[-1]\n        if first_regression_idx != 0:\n            raise ValueError(error_msg)\n\n        if binary_config is not None and multiclass_config is not None:\n            first_binary_idx = (\n                binary_config[0][0]\n                if isinstance(binary_config[0], tuple)\n                else binary_config[0]\n            )\n            last_binary_idx = (\n                binary_config[-1][0]\n                if isinstance(binary_config[-1], tuple)\n                else binary_config[-1]\n            )\n            first_multiclass_idx = multiclass_config[0][0]\n\n            if (first_binary_idx != last_regression_idx + 1) or (\n                last_binary_idx &gt;= first_multiclass_idx\n            ):\n                raise ValueError(error_msg)\n        elif binary_config is not None:\n            first_binary_idx = (\n                binary_config[0][0]\n                if isinstance(binary_config[0], tuple)\n                else binary_config[0]\n            )\n            if first_binary_idx != last_regression_idx + 1:\n                raise ValueError(error_msg)\n        elif multiclass_config is not None:\n            first_multiclass_idx = multiclass_config[0][0]\n            if first_multiclass_idx != last_regression_idx + 1:\n                raise ValueError(error_msg)\n        else:\n            raise ValueError(\n                \"Either binary_config or multiclass_config must be provided. \"\n                \"Otherwise, use the MultiTargetRegressionLoss\"\n            )\n\n    def _prepare_weights_for_regression_targets(\n        self,\n        weights: List[float],\n        regression_config: List[int],\n    ) -&gt; List[float]:\n\n        weights_regression = [\n            w for idx, w in enumerate(weights) if idx in regression_config\n        ]\n\n        return weights_regression\n\n    def _prepare_weights_per_binary_targets(\n        self,\n        weights: List[float],\n        binary_config: Optional[List[Union[int, Tuple[int, float]]]],\n    ) -&gt; Optional[List[float]]:\n\n        if binary_config is not None:\n            binary_idx: List[int] = []\n            for bc in binary_config:\n                if isinstance(bc, tuple):\n                    binary_idx.append(bc[0])\n                else:\n                    binary_idx.append(bc)\n            weights_binary = [w for idx, w in enumerate(weights) if idx in binary_idx]\n        else:\n            weights_binary = None\n\n        return weights_binary\n\n    def _prepare_weights_per_multiclass_targets(\n        self,\n        weights: List[float],\n        multiclass_config: Optional[\n            List[Union[Tuple[int, int], Tuple[int, int, List[float]]]]\n        ],\n    ) -&gt; Optional[List[float]]:\n\n        if multiclass_config is not None:\n            multiclass_idx: List[int] = [mc[0] for mc in multiclass_config]\n            weights_multiclass = [\n                w for idx, w in enumerate(weights) if idx in multiclass_idx\n            ]\n        else:\n            weights_multiclass = None\n\n        return weights_multiclass\n</code></pre>"},{"location":"pytorch-widedeep/metrics.html","title":"Metrics","text":"<p> NOTE: metrics in this module expect the predictions  and ground truth to have the same dimensions for regression and binary  classification problems: \\((N_{samples}, 1)\\). In the case of multiclass  classification problems the ground truth is expected to be a 1D tensor with  the corresponding classes. See Examples below</p> <p>We have added the possibility of using the metrics available at the torchmetrics library. Note that this library is still in its early versions and therefore this option should be used with caution. To use <code>torchmetrics</code> simply import them and use them as any of the <code>pytorch-widedeep</code> metrics described below.</p> <pre><code>from torchmetrics import Accuracy, Precision\n\naccuracy = Accuracy(average=None, num_classes=2)\nprecision = Precision(average='micro', num_classes=2)\n\ntrainer = Trainer(model, objective=\"binary\", metrics=[accuracy, precision])\n</code></pre> <p>A functioning example for <code>pytorch-widedeep</code> using <code>torchmetrics</code> can be found in the Examples folder</p> <p> NOTE: the forward method for all metrics in this  module takes two tensors, <code>y_pred</code> and <code>y_true</code> (in that order). Therefore,  we do not include the method in the documentation.</p>"},{"location":"pytorch-widedeep/metrics.html#pytorch_widedeep.metrics.Accuracy","title":"Accuracy","text":"<p>               Bases: <code>Metric</code></p> <p>Class to calculate the accuracy for both binary and categorical problems</p> <p>Parameters:</p> Name Type Description Default <code>top_k</code> <code>int</code> <p>Accuracy will be computed using the top k most likely classes in multiclass problems</p> <code>1</code> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt;\n&gt;&gt;&gt; from pytorch_widedeep.metrics import Accuracy\n&gt;&gt;&gt;\n&gt;&gt;&gt; acc = Accuracy()\n&gt;&gt;&gt; y_true = torch.tensor([0, 1, 0, 1]).view(-1, 1)\n&gt;&gt;&gt; y_pred = torch.tensor([[0.3, 0.2, 0.6, 0.7]]).view(-1, 1)\n&gt;&gt;&gt; acc(y_pred, y_true)\narray(0.5)\n&gt;&gt;&gt;\n&gt;&gt;&gt; acc = Accuracy(top_k=2)\n&gt;&gt;&gt; y_true = torch.tensor([0, 1, 2])\n&gt;&gt;&gt; y_pred = torch.tensor([[0.3, 0.5, 0.2], [0.1, 0.1, 0.8], [0.1, 0.5, 0.4]])\n&gt;&gt;&gt; acc(y_pred, y_true)\narray(0.66666667)\n</code></pre> Source code in <code>pytorch_widedeep/metrics.py</code> <pre><code>class Accuracy(Metric):\n    r\"\"\"Class to calculate the accuracy for both binary and categorical problems\n\n    Parameters\n    ----------\n    top_k: int, default = 1\n        Accuracy will be computed using the top k most likely classes in\n        multiclass problems\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; from pytorch_widedeep.metrics import Accuracy\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; acc = Accuracy()\n    &gt;&gt;&gt; y_true = torch.tensor([0, 1, 0, 1]).view(-1, 1)\n    &gt;&gt;&gt; y_pred = torch.tensor([[0.3, 0.2, 0.6, 0.7]]).view(-1, 1)\n    &gt;&gt;&gt; acc(y_pred, y_true)\n    array(0.5)\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; acc = Accuracy(top_k=2)\n    &gt;&gt;&gt; y_true = torch.tensor([0, 1, 2])\n    &gt;&gt;&gt; y_pred = torch.tensor([[0.3, 0.5, 0.2], [0.1, 0.1, 0.8], [0.1, 0.5, 0.4]])\n    &gt;&gt;&gt; acc(y_pred, y_true)\n    array(0.66666667)\n    \"\"\"\n\n    def __init__(self, top_k: int = 1):\n        super(Accuracy, self).__init__()\n\n        self.top_k = top_k\n        self.correct_count = 0\n        self.total_count = 0\n        self._name = \"acc\"\n\n    def reset(self):\n        \"\"\"\n        resets counters to 0\n        \"\"\"\n        self.correct_count = 0\n        self.total_count = 0\n\n    def __call__(self, y_pred: Tensor, y_true: Tensor) -&gt; np.ndarray:\n        num_classes = y_pred.size(1)\n\n        if num_classes == 1:\n            y_pred = y_pred.round()\n            y_true = y_true\n        elif num_classes &gt; 1:\n            y_pred = y_pred.topk(self.top_k, 1)[1]\n            y_true = y_true.view(-1, 1).expand_as(y_pred)\n\n        self.correct_count += y_pred.eq(y_true).sum().item()  # type: ignore[assignment]\n        self.total_count += len(y_pred)\n        accuracy = float(self.correct_count) / float(self.total_count)\n        return np.array(accuracy)\n</code></pre>"},{"location":"pytorch-widedeep/metrics.html#pytorch_widedeep.metrics.Accuracy.reset","title":"reset","text":"<pre><code>reset()\n</code></pre> <p>resets counters to 0</p> Source code in <code>pytorch_widedeep/metrics.py</code> <pre><code>def reset(self):\n    \"\"\"\n    resets counters to 0\n    \"\"\"\n    self.correct_count = 0\n    self.total_count = 0\n</code></pre>"},{"location":"pytorch-widedeep/metrics.html#pytorch_widedeep.metrics.Precision","title":"Precision","text":"<p>               Bases: <code>Metric</code></p> <p>Class to calculate the precision for both binary and categorical problems</p> <p>Parameters:</p> Name Type Description Default <code>average</code> <code>bool</code> <p>This applies only to multiclass problems. if <code>True</code> calculate precision for each label, and finds their unweighted mean.</p> <code>True</code> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt;\n&gt;&gt;&gt; from pytorch_widedeep.metrics import Precision\n&gt;&gt;&gt;\n&gt;&gt;&gt; prec = Precision()\n&gt;&gt;&gt; y_true = torch.tensor([0, 1, 0, 1]).view(-1, 1)\n&gt;&gt;&gt; y_pred = torch.tensor([[0.3, 0.2, 0.6, 0.7]]).view(-1, 1)\n&gt;&gt;&gt; prec(y_pred, y_true)\narray(0.5)\n&gt;&gt;&gt;\n&gt;&gt;&gt; prec = Precision(average=True)\n&gt;&gt;&gt; y_true = torch.tensor([0, 1, 2])\n&gt;&gt;&gt; y_pred = torch.tensor([[0.7, 0.1, 0.2], [0.1, 0.1, 0.8], [0.1, 0.5, 0.4]])\n&gt;&gt;&gt; prec(y_pred, y_true)\narray(0.33333334)\n</code></pre> Source code in <code>pytorch_widedeep/metrics.py</code> <pre><code>class Precision(Metric):\n    r\"\"\"Class to calculate the precision for both binary and categorical problems\n\n    Parameters\n    ----------\n    average: bool, default = True\n        This applies only to multiclass problems. if ``True`` calculate\n        precision for each label, and finds their unweighted mean.\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; from pytorch_widedeep.metrics import Precision\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; prec = Precision()\n    &gt;&gt;&gt; y_true = torch.tensor([0, 1, 0, 1]).view(-1, 1)\n    &gt;&gt;&gt; y_pred = torch.tensor([[0.3, 0.2, 0.6, 0.7]]).view(-1, 1)\n    &gt;&gt;&gt; prec(y_pred, y_true)\n    array(0.5)\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; prec = Precision(average=True)\n    &gt;&gt;&gt; y_true = torch.tensor([0, 1, 2])\n    &gt;&gt;&gt; y_pred = torch.tensor([[0.7, 0.1, 0.2], [0.1, 0.1, 0.8], [0.1, 0.5, 0.4]])\n    &gt;&gt;&gt; prec(y_pred, y_true)\n    array(0.33333334)\n    \"\"\"\n\n    def __init__(self, average: bool = True):\n        super(Precision, self).__init__()\n\n        self.average = average\n        self.true_positives = 0\n        self.all_positives = 0\n        self.eps = 1e-20\n        self._name = \"prec\"\n\n    def reset(self):\n        \"\"\"\n        resets counters to 0\n        \"\"\"\n        self.true_positives = 0\n        self.all_positives = 0\n\n    def __call__(self, y_pred: Tensor, y_true: Tensor) -&gt; np.ndarray:\n        num_class = y_pred.size(1)\n\n        if num_class == 1:\n            y_pred = y_pred.round()\n            y_true = y_true\n        elif num_class &gt; 1:\n            y_true = torch.eye(num_class)[y_true.squeeze().cpu().long()]\n            y_pred = y_pred.topk(1, 1)[1].view(-1)\n            y_pred = torch.eye(num_class)[y_pred.cpu().long()]\n\n        self.true_positives += (y_true * y_pred).sum(dim=0)  # type:ignore\n        self.all_positives += y_pred.sum(dim=0)  # type:ignore\n\n        precision = self.true_positives / (self.all_positives + self.eps)\n\n        if self.average:\n            return np.array(precision.mean().item())  # type:ignore\n        else:\n            return precision.detach().cpu().numpy()  # type: ignore[attr-defined]\n</code></pre>"},{"location":"pytorch-widedeep/metrics.html#pytorch_widedeep.metrics.Precision.reset","title":"reset","text":"<pre><code>reset()\n</code></pre> <p>resets counters to 0</p> Source code in <code>pytorch_widedeep/metrics.py</code> <pre><code>def reset(self):\n    \"\"\"\n    resets counters to 0\n    \"\"\"\n    self.true_positives = 0\n    self.all_positives = 0\n</code></pre>"},{"location":"pytorch-widedeep/metrics.html#pytorch_widedeep.metrics.Recall","title":"Recall","text":"<p>               Bases: <code>Metric</code></p> <p>Class to calculate the recall for both binary and categorical problems</p> <p>Parameters:</p> Name Type Description Default <code>average</code> <code>bool</code> <p>This applies only to multiclass problems. if <code>True</code> calculate recall for each label, and finds their unweighted mean.</p> <code>True</code> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt;\n&gt;&gt;&gt; from pytorch_widedeep.metrics import Recall\n&gt;&gt;&gt;\n&gt;&gt;&gt; rec = Recall()\n&gt;&gt;&gt; y_true = torch.tensor([0, 1, 0, 1]).view(-1, 1)\n&gt;&gt;&gt; y_pred = torch.tensor([[0.3, 0.2, 0.6, 0.7]]).view(-1, 1)\n&gt;&gt;&gt; rec(y_pred, y_true)\narray(0.5)\n&gt;&gt;&gt;\n&gt;&gt;&gt; rec = Recall(average=True)\n&gt;&gt;&gt; y_true = torch.tensor([0, 1, 2])\n&gt;&gt;&gt; y_pred = torch.tensor([[0.7, 0.1, 0.2], [0.1, 0.1, 0.8], [0.1, 0.5, 0.4]])\n&gt;&gt;&gt; rec(y_pred, y_true)\narray(0.33333334)\n</code></pre> Source code in <code>pytorch_widedeep/metrics.py</code> <pre><code>class Recall(Metric):\n    r\"\"\"Class to calculate the recall for both binary and categorical problems\n\n    Parameters\n    ----------\n    average: bool, default = True\n        This applies only to multiclass problems. if ``True`` calculate recall\n        for each label, and finds their unweighted mean.\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; from pytorch_widedeep.metrics import Recall\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; rec = Recall()\n    &gt;&gt;&gt; y_true = torch.tensor([0, 1, 0, 1]).view(-1, 1)\n    &gt;&gt;&gt; y_pred = torch.tensor([[0.3, 0.2, 0.6, 0.7]]).view(-1, 1)\n    &gt;&gt;&gt; rec(y_pred, y_true)\n    array(0.5)\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; rec = Recall(average=True)\n    &gt;&gt;&gt; y_true = torch.tensor([0, 1, 2])\n    &gt;&gt;&gt; y_pred = torch.tensor([[0.7, 0.1, 0.2], [0.1, 0.1, 0.8], [0.1, 0.5, 0.4]])\n    &gt;&gt;&gt; rec(y_pred, y_true)\n    array(0.33333334)\n    \"\"\"\n\n    def __init__(self, average: bool = True):\n        super(Recall, self).__init__()\n\n        self.average = average\n        self.true_positives = 0\n        self.actual_positives = 0\n        self.eps = 1e-20\n        self._name = \"rec\"\n\n    def reset(self):\n        \"\"\"\n        resets counters to 0\n        \"\"\"\n        self.true_positives = 0\n        self.actual_positives = 0\n\n    def __call__(self, y_pred: Tensor, y_true: Tensor) -&gt; np.ndarray:\n        num_class = y_pred.size(1)\n\n        if num_class == 1:\n            y_pred = y_pred.round()\n            y_true = y_true\n        elif num_class &gt; 1:\n            y_true = torch.eye(num_class)[y_true.squeeze().cpu().long()]\n            y_pred = y_pred.topk(1, 1)[1].view(-1)\n            y_pred = torch.eye(num_class)[y_pred.cpu().long()]\n\n        self.true_positives += (y_true * y_pred).sum(dim=0)  # type: ignore\n        self.actual_positives += y_true.sum(dim=0)  # type: ignore\n\n        recall = self.true_positives / (self.actual_positives + self.eps)\n\n        if self.average:\n            return np.array(recall.mean().item())  # type:ignore\n        else:\n            return recall.detach().cpu().numpy()  # type: ignore[attr-defined]\n</code></pre>"},{"location":"pytorch-widedeep/metrics.html#pytorch_widedeep.metrics.Recall.reset","title":"reset","text":"<pre><code>reset()\n</code></pre> <p>resets counters to 0</p> Source code in <code>pytorch_widedeep/metrics.py</code> <pre><code>def reset(self):\n    \"\"\"\n    resets counters to 0\n    \"\"\"\n    self.true_positives = 0\n    self.actual_positives = 0\n</code></pre>"},{"location":"pytorch-widedeep/metrics.html#pytorch_widedeep.metrics.FBetaScore","title":"FBetaScore","text":"<p>               Bases: <code>Metric</code></p> <p>Class to calculate the fbeta score for both binary and categorical problems</p> \\[ F_{\\beta} = ((1 + {\\beta}^2) * \\frac{(precision * recall)}{({\\beta}^2 * precision + recall)} \\] <p>Parameters:</p> Name Type Description Default <code>beta</code> <code>int</code> <p>Coefficient to control the balance between precision and recall</p> required <code>average</code> <code>bool</code> <p>This applies only to multiclass problems. if <code>True</code> calculate fbeta for each label, and find their unweighted mean.</p> <code>True</code> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt;\n&gt;&gt;&gt; from pytorch_widedeep.metrics import FBetaScore\n&gt;&gt;&gt;\n&gt;&gt;&gt; fbeta = FBetaScore(beta=2)\n&gt;&gt;&gt; y_true = torch.tensor([0, 1, 0, 1]).view(-1, 1)\n&gt;&gt;&gt; y_pred = torch.tensor([[0.3, 0.2, 0.6, 0.7]]).view(-1, 1)\n&gt;&gt;&gt; fbeta(y_pred, y_true)\narray(0.5)\n&gt;&gt;&gt;\n&gt;&gt;&gt; fbeta = FBetaScore(beta=2)\n&gt;&gt;&gt; y_true = torch.tensor([0, 1, 2])\n&gt;&gt;&gt; y_pred = torch.tensor([[0.7, 0.1, 0.2], [0.1, 0.1, 0.8], [0.1, 0.5, 0.4]])\n&gt;&gt;&gt; fbeta(y_pred, y_true)\narray(0.33333334)\n</code></pre> Source code in <code>pytorch_widedeep/metrics.py</code> <pre><code>class FBetaScore(Metric):\n    r\"\"\"Class to calculate the fbeta score for both binary and categorical problems\n\n    $$\n    F_{\\beta} = ((1 + {\\beta}^2) * \\frac{(precision * recall)}{({\\beta}^2 * precision + recall)}\n    $$\n\n    Parameters\n    ----------\n    beta: int\n        Coefficient to control the balance between precision and recall\n    average: bool, default = True\n        This applies only to multiclass problems. if ``True`` calculate fbeta\n        for each label, and find their unweighted mean.\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; from pytorch_widedeep.metrics import FBetaScore\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; fbeta = FBetaScore(beta=2)\n    &gt;&gt;&gt; y_true = torch.tensor([0, 1, 0, 1]).view(-1, 1)\n    &gt;&gt;&gt; y_pred = torch.tensor([[0.3, 0.2, 0.6, 0.7]]).view(-1, 1)\n    &gt;&gt;&gt; fbeta(y_pred, y_true)\n    array(0.5)\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; fbeta = FBetaScore(beta=2)\n    &gt;&gt;&gt; y_true = torch.tensor([0, 1, 2])\n    &gt;&gt;&gt; y_pred = torch.tensor([[0.7, 0.1, 0.2], [0.1, 0.1, 0.8], [0.1, 0.5, 0.4]])\n    &gt;&gt;&gt; fbeta(y_pred, y_true)\n    array(0.33333334)\n    \"\"\"\n\n    def __init__(self, beta: int, average: bool = True):\n        super(FBetaScore, self).__init__()\n\n        self.beta = beta\n        self.average = average\n        self.precision = Precision(average=False)\n        self.recall = Recall(average=False)\n        self.eps = 1e-20\n        self._name = \"\".join([\"f\", str(self.beta)])\n\n    def reset(self):\n        \"\"\"\n        resets precision and recall\n        \"\"\"\n        self.precision.reset()\n        self.recall.reset()\n\n    def __call__(self, y_pred: Tensor, y_true: Tensor) -&gt; np.ndarray:\n        prec = self.precision(y_pred, y_true)\n        rec = self.recall(y_pred, y_true)\n        beta2 = self.beta**2\n\n        fbeta = ((1 + beta2) * prec * rec) / (beta2 * prec + rec + self.eps)\n\n        if self.average:\n            return np.array(fbeta.mean().item())  # type: ignore[attr-defined]\n        else:\n            return fbeta\n</code></pre>"},{"location":"pytorch-widedeep/metrics.html#pytorch_widedeep.metrics.FBetaScore.reset","title":"reset","text":"<pre><code>reset()\n</code></pre> <p>resets precision and recall</p> Source code in <code>pytorch_widedeep/metrics.py</code> <pre><code>def reset(self):\n    \"\"\"\n    resets precision and recall\n    \"\"\"\n    self.precision.reset()\n    self.recall.reset()\n</code></pre>"},{"location":"pytorch-widedeep/metrics.html#pytorch_widedeep.metrics.F1Score","title":"F1Score","text":"<p>               Bases: <code>Metric</code></p> <p>Class to calculate the f1 score for both binary and categorical problems</p> <p>Parameters:</p> Name Type Description Default <code>average</code> <code>bool</code> <p>This applies only to multiclass problems. if <code>True</code> calculate f1 for each label, and find their unweighted mean.</p> <code>True</code> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt;\n&gt;&gt;&gt; from pytorch_widedeep.metrics import F1Score\n&gt;&gt;&gt;\n&gt;&gt;&gt; f1 = F1Score()\n&gt;&gt;&gt; y_true = torch.tensor([0, 1, 0, 1]).view(-1, 1)\n&gt;&gt;&gt; y_pred = torch.tensor([[0.3, 0.2, 0.6, 0.7]]).view(-1, 1)\n&gt;&gt;&gt; f1(y_pred, y_true)\narray(0.5)\n&gt;&gt;&gt;\n&gt;&gt;&gt; f1 = F1Score()\n&gt;&gt;&gt; y_true = torch.tensor([0, 1, 2])\n&gt;&gt;&gt; y_pred = torch.tensor([[0.7, 0.1, 0.2], [0.1, 0.1, 0.8], [0.1, 0.5, 0.4]])\n&gt;&gt;&gt; f1(y_pred, y_true)\narray(0.33333334)\n</code></pre> Source code in <code>pytorch_widedeep/metrics.py</code> <pre><code>class F1Score(Metric):\n    r\"\"\"Class to calculate the f1 score for both binary and categorical problems\n\n    Parameters\n    ----------\n    average: bool, default = True\n        This applies only to multiclass problems. if ``True`` calculate f1 for\n        each label, and find their unweighted mean.\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; from pytorch_widedeep.metrics import F1Score\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; f1 = F1Score()\n    &gt;&gt;&gt; y_true = torch.tensor([0, 1, 0, 1]).view(-1, 1)\n    &gt;&gt;&gt; y_pred = torch.tensor([[0.3, 0.2, 0.6, 0.7]]).view(-1, 1)\n    &gt;&gt;&gt; f1(y_pred, y_true)\n    array(0.5)\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; f1 = F1Score()\n    &gt;&gt;&gt; y_true = torch.tensor([0, 1, 2])\n    &gt;&gt;&gt; y_pred = torch.tensor([[0.7, 0.1, 0.2], [0.1, 0.1, 0.8], [0.1, 0.5, 0.4]])\n    &gt;&gt;&gt; f1(y_pred, y_true)\n    array(0.33333334)\n    \"\"\"\n\n    def __init__(self, average: bool = True):\n        super(F1Score, self).__init__()\n\n        self.average = average\n        self.f1 = FBetaScore(beta=1, average=self.average)\n        self._name = self.f1._name\n\n    def reset(self):\n        \"\"\"\n        resets counters to 0\n        \"\"\"\n        self.f1.reset()\n\n    def __call__(self, y_pred: Tensor, y_true: Tensor) -&gt; np.ndarray:\n        return self.f1(y_pred, y_true)\n</code></pre>"},{"location":"pytorch-widedeep/metrics.html#pytorch_widedeep.metrics.F1Score.reset","title":"reset","text":"<pre><code>reset()\n</code></pre> <p>resets counters to 0</p> Source code in <code>pytorch_widedeep/metrics.py</code> <pre><code>def reset(self):\n    \"\"\"\n    resets counters to 0\n    \"\"\"\n    self.f1.reset()\n</code></pre>"},{"location":"pytorch-widedeep/metrics.html#pytorch_widedeep.metrics.R2Score","title":"R2Score","text":"<p>               Bases: <code>Metric</code></p> <p>Calculates R-Squared, the coefficient of determination:</p> \\[ R^2 = 1 - \\frac{\\sum_{j=1}^n(y_j - \\hat{y_j})^2}{\\sum_{j=1}^n(y_j - \\bar{y})^2} \\] <p>where \\(\\hat{y_j}\\) is the ground truth, \\(y_j\\) is the predicted value and \\(\\bar{y}\\) is the mean of the ground truth.</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt;\n&gt;&gt;&gt; from pytorch_widedeep.metrics import R2Score\n&gt;&gt;&gt;\n&gt;&gt;&gt; r2 = R2Score()\n&gt;&gt;&gt; y_true = torch.tensor([3, -0.5, 2, 7]).view(-1, 1)\n&gt;&gt;&gt; y_pred = torch.tensor([2.5, 0.0, 2, 8]).view(-1, 1)\n&gt;&gt;&gt; r2(y_pred, y_true)\narray(0.94860814)\n</code></pre> Source code in <code>pytorch_widedeep/metrics.py</code> <pre><code>class R2Score(Metric):\n    r\"\"\"\n    Calculates R-Squared, the\n    [coefficient of determination](https://en.wikipedia.org/wiki/Coefficient_of_determination&gt;):\n\n    $$\n    R^2 = 1 - \\frac{\\sum_{j=1}^n(y_j - \\hat{y_j})^2}{\\sum_{j=1}^n(y_j - \\bar{y})^2}\n    $$\n\n    where $\\hat{y_j}$ is the ground truth, $y_j$ is the predicted value and\n    $\\bar{y}$ is the mean of the ground truth.\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; from pytorch_widedeep.metrics import R2Score\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; r2 = R2Score()\n    &gt;&gt;&gt; y_true = torch.tensor([3, -0.5, 2, 7]).view(-1, 1)\n    &gt;&gt;&gt; y_pred = torch.tensor([2.5, 0.0, 2, 8]).view(-1, 1)\n    &gt;&gt;&gt; r2(y_pred, y_true)\n    array(0.94860814)\n    \"\"\"\n\n    def __init__(self):\n        self.numerator = 0\n        self.denominator = 0\n        self.num_examples = 0\n        self.y_true_sum = 0\n\n        self._name = \"r2\"\n\n    def reset(self):\n        \"\"\"\n        resets counters to 0\n        \"\"\"\n        self.numerator = 0\n        self.denominator = 0\n        self.num_examples = 0\n        self.y_true_sum = 0\n\n    def __call__(self, y_pred: Tensor, y_true: Tensor) -&gt; np.ndarray:\n        self.numerator += ((y_pred - y_true) ** 2).sum().item()\n\n        self.num_examples += y_true.shape[0]\n        self.y_true_sum += y_true.sum().item()\n        y_true_avg = self.y_true_sum / self.num_examples\n        self.denominator += ((y_true - y_true_avg) ** 2).sum().item()\n        return np.array((1 - (self.numerator / self.denominator)))\n</code></pre>"},{"location":"pytorch-widedeep/metrics.html#pytorch_widedeep.metrics.R2Score.reset","title":"reset","text":"<pre><code>reset()\n</code></pre> <p>resets counters to 0</p> Source code in <code>pytorch_widedeep/metrics.py</code> <pre><code>def reset(self):\n    \"\"\"\n    resets counters to 0\n    \"\"\"\n    self.numerator = 0\n    self.denominator = 0\n    self.num_examples = 0\n    self.y_true_sum = 0\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html","title":"The <code>models</code> module","text":"<p>This module contains the models that can be used as the four main components that will comprise a Wide and Deep model (<code>wide</code>, <code>deeptabular</code>, <code>deeptext</code>, <code>deepimage</code>), as well as the <code>WideDeep</code> \"constructor\" class. Note that each of the four components can be used independently. It also contains all the documentation for the models that can be used for self-supervised pre-training with tabular data.</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.linear.wide.Wide","title":"Wide","text":"<p>               Bases: <code>Module</code></p> <p>Defines a <code>Wide</code> (linear) model where the non-linearities are captured via the so-called crossed-columns. This can be used as the <code>wide</code> component of a Wide &amp; Deep model.</p> <p>Parameters:</p> Name Type Description Default <code>input_dim</code> <code>int</code> <p>size of the Linear layer (implemented via an Embedding layer). <code>input_dim</code> is the summation of all the individual values for all the features that go through the wide model. For example, if the wide model receives 2 features with 5 individual values each, <code>input_dim = 10</code></p> required <code>pred_dim</code> <code>int</code> <p>size of the ouput tensor containing the predictions. Note that unlike all the other models, the wide model is connected directly to the output neuron(s) when used to build a Wide and Deep model. Therefore, it requires the <code>pred_dim</code> parameter.</p> <code>1</code> <p>Attributes:</p> Name Type Description <code>wide_linear</code> <code>Module</code> <p>the linear layer that comprises the wide branch of the model</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import Wide\n&gt;&gt;&gt; X = torch.empty(4, 4).random_(20)\n&gt;&gt;&gt; wide = Wide(input_dim=int(X.max().item()), pred_dim=1)\n&gt;&gt;&gt; out = wide(X)\n</code></pre> Source code in <code>pytorch_widedeep/models/tabular/linear/wide.py</code> <pre><code>class Wide(nn.Module):\n    r\"\"\"Defines a `Wide` (linear) model where the non-linearities are\n    captured via the so-called crossed-columns. This can be used as the\n    `wide` component of a Wide &amp; Deep model.\n\n    Parameters\n    -----------\n    input_dim: int\n        size of the Linear layer (implemented via an Embedding layer).\n        `input_dim` is the summation of all the individual values for all the\n        features that go through the wide model. For example, if the wide\n        model receives 2 features with 5 individual values each, `input_dim =\n        10`\n    pred_dim: int, default = 1\n        size of the ouput tensor containing the predictions. Note that unlike\n        all the other models, the wide model is connected directly to the\n        output neuron(s) when used to build a Wide and Deep model. Therefore,\n        it requires the `pred_dim` parameter.\n\n    Attributes\n    -----------\n    wide_linear: nn.Module\n        the linear layer that comprises the wide branch of the model\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.models import Wide\n    &gt;&gt;&gt; X = torch.empty(4, 4).random_(20)\n    &gt;&gt;&gt; wide = Wide(input_dim=int(X.max().item()), pred_dim=1)\n    &gt;&gt;&gt; out = wide(X)\n    \"\"\"\n\n    @alias(\"pred_dim\", [\"pred_size\", \"num_class\"])\n    def __init__(self, input_dim: int, pred_dim: int = 1):\n        super(Wide, self).__init__()\n\n        self.input_dim = input_dim\n        self.pred_dim = pred_dim\n\n        # Embeddings: val + 1 because 0 is reserved for padding/unseen cateogories.\n        self.wide_linear = nn.Embedding(input_dim + 1, pred_dim, padding_idx=0)\n        # (Sum(Embedding) + bias) is equivalent to (OneHotVector + Linear)\n        self.bias = nn.Parameter(torch.zeros(pred_dim))\n        self._reset_parameters()\n\n    def _reset_parameters(self) -&gt; None:\n        r\"\"\"initialize Embedding and bias like nn.Linear. See [original\n        implementation](https://pytorch.org/docs/stable/_modules/torch/nn/modules/linear.html#Linear).\n        \"\"\"\n        nn.init.kaiming_uniform_(self.wide_linear.weight, a=math.sqrt(5))\n        fan_in, _ = nn.init._calculate_fan_in_and_fan_out(self.wide_linear.weight)\n        bound = 1 / math.sqrt(fan_in)\n        nn.init.uniform_(self.bias, -bound, bound)\n\n    def forward(self, X: Tensor) -&gt; Tensor:\n        r\"\"\"Forward pass. Simply connecting the Embedding layer with the ouput\n        neuron(s)\"\"\"\n        out = self.wide_linear(X.long()).sum(dim=1) + self.bias\n        return out\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.linear.wide.Wide.forward","title":"forward","text":"<pre><code>forward(X)\n</code></pre> <p>Forward pass. Simply connecting the Embedding layer with the ouput neuron(s)</p> Source code in <code>pytorch_widedeep/models/tabular/linear/wide.py</code> <pre><code>def forward(self, X: Tensor) -&gt; Tensor:\n    r\"\"\"Forward pass. Simply connecting the Embedding layer with the ouput\n    neuron(s)\"\"\"\n    out = self.wide_linear(X.long()).sum(dim=1) + self.bias\n    return out\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.mlp.tab_mlp.TabMlp","title":"TabMlp","text":"<p>               Bases: <code>BaseTabularModelWithoutAttention</code></p> <p>Defines a <code>TabMlp</code> model that can be used as the <code>deeptabular</code> component of a Wide &amp; Deep model or independently by itself.</p> <p>This class combines embedding representations of the categorical features with numerical (aka continuous) features, embedded or not. These are then passed through a series of dense layers (i.e. a MLP).</p> <p>Most of the parameters for this class are <code>Optional</code> since the use of categorical or continuous is in fact optional (i.e. one can use categorical features only, continuous features only or both).</p> <p>Parameters:</p> Name Type Description Default <code>column_idx</code> <code>Dict[str, int]</code> <p>Dict containing the index of the columns that will be passed through the <code>TabMlp</code> model. Required to slice the tensors. e.g. {'education': 0, 'relationship': 1, 'workclass': 2, ...}.</p> required <code>cat_embed_input</code> <code>Optional[List[Tuple[str, int, int]]]</code> <p>List of Tuples with the column name, number of unique values and embedding dimension. e.g. [(education, 11, 32), ...]</p> <code>None</code> <code>cat_embed_dropout</code> <code>Optional[float]</code> <p>Categorical embeddings dropout. If <code>None</code>, it will default to 0.</p> <code>None</code> <code>use_cat_bias</code> <code>Optional[bool]</code> <p>Boolean indicating if bias will be used for the categorical embeddings. If <code>None</code>, it will default to 'False'.</p> <code>None</code> <code>cat_embed_activation</code> <code>Optional[str]</code> <p>Activation function for the categorical embeddings, if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> <code>None</code> <code>continuous_cols</code> <code>Optional[List[str]]</code> <p>List with the name of the numeric (aka continuous) columns</p> <code>None</code> <code>cont_norm_layer</code> <code>Optional[Literal[batchnorm, layernorm]]</code> <p>Type of normalization layer applied to the continuous features. Options are: 'layernorm' and 'batchnorm'. if <code>None</code>, no normalization layer will be used.</p> <code>None</code> <code>embed_continuous</code> <code>Optional[bool]</code> <p>Boolean indicating if the continuous columns will be embedded using one of the available methods: 'standard', 'periodic' or 'piecewise'. If <code>None</code>, it will default to 'False'. NOTE: This parameter is deprecated and it  will be removed in future releases. Please, use the  <code>embed_continuous_method</code> parameter instead.</p> <code>None</code> <code>embed_continuous_method</code> <code>Optional[Literal[standard, piecewise, periodic]]</code> <p>Method to use to embed the continuous features. Options are: 'standard', 'periodic' or 'piecewise'. The 'standard' embedding method is based on the FT-Transformer implementation presented in the paper: Revisiting Deep Learning Models for Tabular Data. The 'periodic' and_'piecewise'_ methods were presented in the paper: On Embeddings for Numerical Features in Tabular Deep Learning. Please, read the papers for details.</p> <code>None</code> <code>cont_embed_dim</code> <code>Optional[int]</code> <p>Size of the continuous embeddings. If the continuous columns are embedded, <code>cont_embed_dim</code> must be passed.</p> <code>None</code> <code>cont_embed_dropout</code> <code>Optional[float]</code> <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p> <code>None</code> <code>cont_embed_activation</code> <code>Optional[str]</code> <p>Activation function for the continuous embeddings if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported. If <code>None</code>, no activation function will be applied.</p> <code>None</code> <code>quantization_setup</code> <code>Optional[Dict[str, List[float]]]</code> <p>This parameter is used when the 'piecewise' method is used to embed the continuous cols. It is a dict where keys are the name of the continuous columns and values are lists with the boundaries for the quantization of the continuous_cols. See the examples for details. If If the 'piecewise' method is used, this parameter is required.</p> <code>None</code> <code>n_frequencies</code> <code>Optional[int]</code> <p>This is the so called 'k' in their paper On Embeddings for Numerical Features in Tabular Deep Learning, and is the number of 'frequencies' that will be used to represent each continuous column. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>sigma</code> <code>Optional[float]</code> <p>This is the sigma parameter in the paper mentioned when describing the previous parameters and it is used to initialise the 'frequency weights'. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>share_last_layer</code> <code>Optional[bool]</code> <p>This parameter is not present in the before mentioned paper but it is implemented in the official repo. If <code>True</code> the linear layer that turns the frequencies into embeddings will be shared across the continuous columns. If <code>False</code> a different linear layer will be used for each continuous column. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>full_embed_dropout</code> <code>Optional[bool]</code> <p>If <code>True</code>, the full embedding corresponding to a column will be masked out/dropout. If <code>None</code>, it will default to <code>False</code>.</p> <code>None</code> <code>mlp_hidden_dims</code> <code>List[int]</code> <p>List with the number of neurons per dense layer in the mlp.</p> <code>[200, 100]</code> <code>mlp_activation</code> <code>str</code> <p>Activation function for the dense layers of the MLP. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> <code>'relu'</code> <code>mlp_dropout</code> <code>Union[float, List[float]]</code> <p>float or List of floats with the dropout between the dense layers. e.g: [0.5,0.5]</p> <code>0.1</code> <code>mlp_batchnorm</code> <code>bool</code> <p>Boolean indicating whether or not batch normalization will be applied to the dense layers</p> <code>False</code> <code>mlp_batchnorm_last</code> <code>bool</code> <p>Boolean indicating whether or not batch normalization will be applied to the last of the dense layers</p> <code>False</code> <code>mlp_linear_first</code> <code>bool</code> <p>Boolean indicating the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code></p> <code>True</code> <p>Attributes:</p> Name Type Description <code>encoder</code> <code>Module</code> <p>mlp model that will receive the concatenation of the embeddings and the continuous columns</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import TabMlp\n&gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n&gt;&gt;&gt; colnames = [\"a\", \"b\", \"c\", \"d\", \"e\"]\n&gt;&gt;&gt; cat_embed_input = [(u, i, j) for u, i, j in zip(colnames[:4], [4] * 4, [8] * 4)]\n&gt;&gt;&gt; column_idx = {k: v for v, k in enumerate(colnames)}\n&gt;&gt;&gt; model = TabMlp(mlp_hidden_dims=[8, 4], column_idx=column_idx, cat_embed_input=cat_embed_input,\n... continuous_cols=[\"e\"])\n&gt;&gt;&gt; out = model(X_tab)\n</code></pre> Source code in <code>pytorch_widedeep/models/tabular/mlp/tab_mlp.py</code> <pre><code>class TabMlp(BaseTabularModelWithoutAttention):\n    r\"\"\"Defines a `TabMlp` model that can be used as the `deeptabular`\n    component of a Wide &amp; Deep model or independently by itself.\n\n    This class combines embedding representations of the categorical features\n    with numerical (aka continuous) features, embedded or not. These are then\n    passed through a series of dense layers (i.e. a MLP).\n\n    Most of the parameters for this class are `Optional` since the use of\n    categorical or continuous is in fact optional (i.e. one can use\n    categorical features only, continuous features only or both).\n\n    Parameters\n    ----------\n    column_idx: Dict\n        Dict containing the index of the columns that will be passed through\n        the `TabMlp` model. Required to slice the tensors. e.g. _{'education':\n        0, 'relationship': 1, 'workclass': 2, ...}_.\n    cat_embed_input: List, Optional, default = None\n        List of Tuples with the column name, number of unique values and\n        embedding dimension. e.g. _[(education, 11, 32), ...]_\n    cat_embed_dropout: float, Optional, default = None\n        Categorical embeddings dropout. If `None`, it will default\n        to 0.\n    use_cat_bias: bool, Optional, default = None,\n        Boolean indicating if bias will be used for the categorical embeddings.\n        If `None`, it will default to 'False'.\n    cat_embed_activation: Optional, str, default = None,\n        Activation function for the categorical embeddings, if any. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported\n    continuous_cols: List, Optional, default = None\n        List with the name of the numeric (aka continuous) columns\n    cont_norm_layer: str, Optional, default =  None\n        Type of normalization layer applied to the continuous features.\n        Options are: _'layernorm'_ and _'batchnorm'_. if `None`, no\n        normalization layer will be used.\n    embed_continuous: bool, Optional, default = None,\n        Boolean indicating if the continuous columns will be embedded using\n        one of the available methods: _'standard'_, _'periodic'_\n        or _'piecewise'_. If `None`, it will default to 'False'.&lt;br/&gt;\n        :information_source: **NOTE**: This parameter is deprecated and it\n         will be removed in future releases. Please, use the\n         `embed_continuous_method` parameter instead.\n    embed_continuous_method: Optional, str, default = None,\n        Method to use to embed the continuous features. Options are:\n        _'standard'_, _'periodic'_ or _'piecewise'_. The _'standard'_\n        embedding method is based on the FT-Transformer implementation\n        presented in the paper: [Revisiting Deep Learning Models for\n        Tabular Data](https://arxiv.org/abs/2106.11959v5). The _'periodic'_\n        and_'piecewise'_ methods were presented in the paper: [On Embeddings for\n        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556).\n        Please, read the papers for details.\n    cont_embed_dim: int, Optional, default = None,\n        Size of the continuous embeddings. If the continuous columns are\n        embedded, `cont_embed_dim` must be passed.\n    cont_embed_dropout: float, Optional, default = None,\n        Dropout for the continuous embeddings. If `None`, it will default to 0.0\n    cont_embed_activation: Optional, str, default = None,\n        Activation function for the continuous embeddings if any. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported.\n        If `None`, no activation function will be applied.\n    quantization_setup: Dict[str, List[float]], Optional, default = None,\n        This parameter is used when the _'piecewise'_ method is used to embed\n        the continuous cols. It is a dict where keys are the name of the continuous\n        columns and values are lists with the boundaries for the quantization\n        of the continuous_cols. See the examples for details. If\n        If the _'piecewise'_ method is used, this parameter is required.\n    n_frequencies: int, Optional, default = None,\n        This is the so called _'k'_ in their paper [On Embeddings for\n        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556),\n        and is the number of 'frequencies' that will be used to represent each\n        continuous column. See their Eq 2 in the paper for details. If\n        the _'periodic'_ method is used, this parameter is required.\n    sigma: float, Optional, default = None,\n        This is the sigma parameter in the paper mentioned when describing the\n        previous parameters and it is used to initialise the 'frequency\n        weights'. See their Eq 2 in the paper for details. If\n        the _'periodic'_ method is used, this parameter is required.\n    share_last_layer: bool, Optional, default = None,\n        This parameter is not present in the before mentioned paper but it is implemented in\n        the [official repo](https://github.com/yandex-research/rtdl-num-embeddings/tree/main).\n        If `True` the linear layer that turns the frequencies into embeddings\n        will be shared across the continuous columns. If `False` a different\n        linear layer will be used for each continuous column.\n        If the _'periodic'_ method is used, this parameter is required.\n    full_embed_dropout: bool, Optional, default = None,\n        If `True`, the full embedding corresponding to a column will be masked\n        out/dropout. If `None`, it will default to `False`.\n    mlp_hidden_dims: List, default = [200, 100]\n        List with the number of neurons per dense layer in the mlp.\n    mlp_activation: str, default = \"relu\"\n        Activation function for the dense layers of the MLP. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported\n    mlp_dropout: float or List, default = 0.1\n        float or List of floats with the dropout between the dense layers.\n        e.g: _[0.5,0.5]_\n    mlp_batchnorm: bool, default = False\n        Boolean indicating whether or not batch normalization will be applied\n        to the dense layers\n    mlp_batchnorm_last: bool, default = False\n        Boolean indicating whether or not batch normalization will be applied\n        to the last of the dense layers\n    mlp_linear_first: bool, default = False\n        Boolean indicating the order of the operations in the dense\n        layer. If `True: [LIN -&gt; ACT -&gt; BN -&gt; DP]`. If `False: [BN -&gt; DP -&gt;\n        LIN -&gt; ACT]`\n\n    Attributes\n    ----------\n    encoder: nn.Module\n        mlp model that will receive the concatenation of the embeddings and\n        the continuous columns\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.models import TabMlp\n    &gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n    &gt;&gt;&gt; colnames = [\"a\", \"b\", \"c\", \"d\", \"e\"]\n    &gt;&gt;&gt; cat_embed_input = [(u, i, j) for u, i, j in zip(colnames[:4], [4] * 4, [8] * 4)]\n    &gt;&gt;&gt; column_idx = {k: v for v, k in enumerate(colnames)}\n    &gt;&gt;&gt; model = TabMlp(mlp_hidden_dims=[8, 4], column_idx=column_idx, cat_embed_input=cat_embed_input,\n    ... continuous_cols=[\"e\"])\n    &gt;&gt;&gt; out = model(X_tab)\n    \"\"\"\n\n    def __init__(\n        self,\n        column_idx: Dict[str, int],\n        *,\n        cat_embed_input: Optional[List[Tuple[str, int, int]]] = None,\n        cat_embed_dropout: Optional[float] = None,\n        use_cat_bias: Optional[bool] = None,\n        cat_embed_activation: Optional[str] = None,\n        continuous_cols: Optional[List[str]] = None,\n        cont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\n        embed_continuous: Optional[bool] = None,\n        embed_continuous_method: Optional[\n            Literal[\"standard\", \"piecewise\", \"periodic\"]\n        ] = None,\n        cont_embed_dim: Optional[int] = None,\n        cont_embed_dropout: Optional[float] = None,\n        cont_embed_activation: Optional[str] = None,\n        quantization_setup: Optional[Dict[str, List[float]]] = None,\n        n_frequencies: Optional[int] = None,\n        sigma: Optional[float] = None,\n        share_last_layer: Optional[bool] = None,\n        full_embed_dropout: Optional[bool] = None,\n        mlp_hidden_dims: List[int] = [200, 100],\n        mlp_activation: str = \"relu\",\n        mlp_dropout: Union[float, List[float]] = 0.1,\n        mlp_batchnorm: bool = False,\n        mlp_batchnorm_last: bool = False,\n        mlp_linear_first: bool = True,\n    ):\n        super(TabMlp, self).__init__(\n            column_idx=column_idx,\n            cat_embed_input=cat_embed_input,\n            cat_embed_dropout=cat_embed_dropout,\n            use_cat_bias=use_cat_bias,\n            cat_embed_activation=cat_embed_activation,\n            continuous_cols=continuous_cols,\n            cont_norm_layer=cont_norm_layer,\n            embed_continuous=embed_continuous,\n            embed_continuous_method=embed_continuous_method,\n            cont_embed_dim=cont_embed_dim,\n            cont_embed_dropout=cont_embed_dropout,\n            cont_embed_activation=cont_embed_activation,\n            quantization_setup=quantization_setup,\n            n_frequencies=n_frequencies,\n            sigma=sigma,\n            share_last_layer=share_last_layer,\n            full_embed_dropout=full_embed_dropout,\n        )\n\n        self.mlp_hidden_dims = mlp_hidden_dims\n        self.mlp_activation = mlp_activation\n        self.mlp_dropout = mlp_dropout\n        self.mlp_batchnorm = mlp_batchnorm\n        self.mlp_batchnorm_last = mlp_batchnorm_last\n        self.mlp_linear_first = mlp_linear_first\n\n        # Embeddings are instantiated at the base model\n        # Mlp\n        mlp_input_dim = self.cat_out_dim + self.cont_out_dim\n        mlp_hidden_dims = [mlp_input_dim] + mlp_hidden_dims\n        self.encoder = MLP(\n            mlp_hidden_dims,\n            mlp_activation,\n            mlp_dropout,\n            mlp_batchnorm,\n            mlp_batchnorm_last,\n            mlp_linear_first,\n        )\n\n    def forward(self, X: Tensor) -&gt; Tensor:\n        x = self._get_embeddings(X)\n        return self.encoder(x)\n\n    @property\n    def output_dim(self) -&gt; int:\n        r\"\"\"The output dimension of the model. This is a required property\n        neccesary to build the `WideDeep` class\"\"\"\n        return self.mlp_hidden_dims[-1]\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.mlp.tab_mlp.TabMlp.output_dim","title":"output_dim  <code>property</code>","text":"<pre><code>output_dim\n</code></pre> <p>The output dimension of the model. This is a required property neccesary to build the <code>WideDeep</code> class</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.mlp.tab_mlp.TabMlpDecoder","title":"TabMlpDecoder","text":"<p>               Bases: <code>Module</code></p> <p>Companion decoder model for the <code>TabMlp</code> model (which can be considered an encoder itself).</p> <p>This class is designed to be used with the <code>EncoderDecoderTrainer</code> when using self-supervised pre-training (see the corresponding section in the docs). The <code>TabMlpDecoder</code> will receive the output from the MLP and 'reconstruct' the embeddings.</p> <p>Parameters:</p> Name Type Description Default <code>embed_dim</code> <code>int</code> <p>Size of the embeddings tensor that needs to be reconstructed.</p> required <code>mlp_hidden_dims</code> <code>List[int]</code> <p>List with the number of neurons per dense layer in the mlp.</p> <code>[100, 200]</code> <code>mlp_activation</code> <code>str</code> <p>Activation function for the dense layers of the MLP. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> <code>'relu'</code> <code>mlp_dropout</code> <code>Union[float, List[float]]</code> <p>float or List of floats with the dropout between the dense layers. e.g: [0.5,0.5]</p> <code>0.1</code> <code>mlp_batchnorm</code> <code>bool</code> <p>Boolean indicating whether or not batch normalization will be applied to the dense layers</p> <code>False</code> <code>mlp_batchnorm_last</code> <code>bool</code> <p>Boolean indicating whether or not batch normalization will be applied to the last of the dense layers</p> <code>False</code> <code>mlp_linear_first</code> <code>bool</code> <p>Boolean indicating the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code></p> <code>True</code> <p>Attributes:</p> Name Type Description <code>decoder</code> <code>Module</code> <p>mlp model that will receive the output of the encoder</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import TabMlpDecoder\n&gt;&gt;&gt; x_inp = torch.rand(3, 8)\n&gt;&gt;&gt; decoder = TabMlpDecoder(embed_dim=32, mlp_hidden_dims=[8,16])\n&gt;&gt;&gt; res = decoder(x_inp)\n&gt;&gt;&gt; res.shape\ntorch.Size([3, 32])\n</code></pre> Source code in <code>pytorch_widedeep/models/tabular/mlp/tab_mlp.py</code> <pre><code>class TabMlpDecoder(nn.Module):\n    r\"\"\"Companion decoder model for the `TabMlp` model (which can be considered\n    an encoder itself).\n\n    This class is designed to be used with the `EncoderDecoderTrainer` when\n    using self-supervised pre-training (see the corresponding section in the\n    docs). The `TabMlpDecoder` will receive the output from the MLP\n    and '_reconstruct_' the embeddings.\n\n    Parameters\n    ----------\n    embed_dim: int\n        Size of the embeddings tensor that needs to be reconstructed.\n    mlp_hidden_dims: List, default = [200, 100]\n        List with the number of neurons per dense layer in the mlp.\n    mlp_activation: str, default = \"relu\"\n        Activation function for the dense layers of the MLP. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported\n    mlp_dropout: float or List, default = 0.1\n        float or List of floats with the dropout between the dense layers.\n        e.g: _[0.5,0.5]_\n    mlp_batchnorm: bool, default = False\n        Boolean indicating whether or not batch normalization will be applied\n        to the dense layers\n    mlp_batchnorm_last: bool, default = False\n        Boolean indicating whether or not batch normalization will be applied\n        to the last of the dense layers\n    mlp_linear_first: bool, default = False\n        Boolean indicating the order of the operations in the dense\n        layer. If `True: [LIN -&gt; ACT -&gt; BN -&gt; DP]`. If `False: [BN -&gt; DP -&gt;\n        LIN -&gt; ACT]`\n\n    Attributes\n    ----------\n    decoder: nn.Module\n        mlp model that will receive the output of the encoder\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.models import TabMlpDecoder\n    &gt;&gt;&gt; x_inp = torch.rand(3, 8)\n    &gt;&gt;&gt; decoder = TabMlpDecoder(embed_dim=32, mlp_hidden_dims=[8,16])\n    &gt;&gt;&gt; res = decoder(x_inp)\n    &gt;&gt;&gt; res.shape\n    torch.Size([3, 32])\n    \"\"\"\n\n    def __init__(\n        self,\n        embed_dim: int,\n        mlp_hidden_dims: List[int] = [100, 200],\n        mlp_activation: str = \"relu\",\n        mlp_dropout: Union[float, List[float]] = 0.1,\n        mlp_batchnorm: bool = False,\n        mlp_batchnorm_last: bool = False,\n        mlp_linear_first: bool = True,\n    ):\n        super(TabMlpDecoder, self).__init__()\n\n        self.embed_dim = embed_dim\n\n        self.mlp_hidden_dims = mlp_hidden_dims\n        self.mlp_activation = mlp_activation\n        self.mlp_dropout = mlp_dropout\n        self.mlp_batchnorm = mlp_batchnorm\n        self.mlp_batchnorm_last = mlp_batchnorm_last\n        self.mlp_linear_first = mlp_linear_first\n\n        self.decoder = MLP(\n            mlp_hidden_dims + [self.embed_dim],\n            mlp_activation,\n            mlp_dropout,\n            mlp_batchnorm,\n            mlp_batchnorm_last,\n            mlp_linear_first,\n        )\n\n    def forward(self, X: Tensor) -&gt; Tensor:\n        return self.decoder(X)\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.resnet.tab_resnet.TabResnet","title":"TabResnet","text":"<p>               Bases: <code>BaseTabularModelWithoutAttention</code></p> <p>Defines a <code>TabResnet</code> model that can be used as the <code>deeptabular</code> component of a Wide &amp; Deep model or independently by itself.</p> <p>This class combines embedding representations of the categorical features with numerical (aka continuous) features, embedded or not. These are then passed through a series of Resnet blocks. See <code>pytorch_widedeep.models.tab_resnet._layers</code> for details on the structure of each block.</p> <p>Most of the parameters for this class are <code>Optional</code> since the use of categorical or continuous is in fact optional (i.e. one can use categorical features only, continuous features only or both).</p> <p>Parameters:</p> Name Type Description Default <code>column_idx</code> <code>Dict[str, int]</code> <p>Dict containing the index of the columns that will be passed through the <code>TabMlp</code> model. Required to slice the tensors. e.g. {'education': 0, 'relationship': 1, 'workclass': 2, ...}.</p> required <code>cat_embed_input</code> <code>Optional[List[Tuple[str, int, int]]]</code> <p>List of Tuples with the column name, number of unique values and embedding dimension. e.g. [(education, 11, 32), ...]</p> <code>None</code> <code>cat_embed_dropout</code> <code>Optional[float]</code> <p>Categorical embeddings dropout. If <code>None</code>, it will default to 0.</p> <code>None</code> <code>use_cat_bias</code> <code>Optional[bool]</code> <p>Boolean indicating if bias will be used for the categorical embeddings. If <code>None</code>, it will default to 'False'.</p> <code>None</code> <code>cat_embed_activation</code> <code>Optional[str]</code> <p>Activation function for the categorical embeddings, if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> <code>None</code> <code>continuous_cols</code> <code>Optional[List[str]]</code> <p>List with the name of the numeric (aka continuous) columns</p> <code>None</code> <code>cont_norm_layer</code> <code>Optional[Literal[batchnorm, layernorm]]</code> <p>Type of normalization layer applied to the continuous features. Options are: 'layernorm' and 'batchnorm'. if <code>None</code>, no normalization layer will be used.</p> <code>None</code> <code>embed_continuous</code> <code>Optional[bool]</code> <p>Boolean indicating if the continuous columns will be embedded using one of the available methods: 'standard', 'periodic' or 'piecewise'. If <code>None</code>, it will default to 'False'. NOTE: This parameter is deprecated and it  will be removed in future releases. Please, use the  <code>embed_continuous_method</code> parameter instead.</p> <code>None</code> <code>embed_continuous_method</code> <code>Optional[Literal[standard, piecewise, periodic]]</code> <p>Method to use to embed the continuous features. Options are: 'standard', 'periodic' or 'piecewise'. The 'standard' embedding method is based on the FT-Transformer implementation presented in the paper: Revisiting Deep Learning Models for Tabular Data. The 'periodic' and_'piecewise'_ methods were presented in the paper: On Embeddings for Numerical Features in Tabular Deep Learning. Please, read the papers for details.</p> <code>None</code> <code>cont_embed_dim</code> <code>Optional[int]</code> <p>Size of the continuous embeddings. If the continuous columns are embedded, <code>cont_embed_dim</code> must be passed.</p> <code>None</code> <code>cont_embed_dropout</code> <code>Optional[float]</code> <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p> <code>None</code> <code>cont_embed_activation</code> <code>Optional[str]</code> <p>Activation function for the continuous embeddings if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported. If <code>None</code>, no activation function will be applied.</p> <code>None</code> <code>quantization_setup</code> <code>Optional[Dict[str, List[float]]]</code> <p>This parameter is used when the 'piecewise' method is used to embed the continuous cols. It is a dict where keys are the name of the continuous columns and values are lists with the boundaries for the quantization of the continuous_cols. See the examples for details. If If the 'piecewise' method is used, this parameter is required.</p> <code>None</code> <code>n_frequencies</code> <code>Optional[int]</code> <p>This is the so called 'k' in their paper On Embeddings for Numerical Features in Tabular Deep Learning, and is the number of 'frequencies' that will be used to represent each continuous column. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>sigma</code> <code>Optional[float]</code> <p>This is the sigma parameter in the paper mentioned when describing the previous parameters and it is used to initialise the 'frequency weights'. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>share_last_layer</code> <code>Optional[bool]</code> <p>This parameter is not present in the before mentioned paper but it is implemented in the official repo. If <code>True</code> the linear layer that turns the frequencies into embeddings will be shared across the continuous columns. If <code>False</code> a different linear layer will be used for each continuous column. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>full_embed_dropout</code> <code>Optional[bool]</code> <p>If <code>True</code>, the full embedding corresponding to a column will be masked out/dropout. If <code>None</code>, it will default to <code>False</code>.</p> <code>None</code> <code>blocks_dims</code> <code>List[int]</code> <p>List of integers that define the input and output units of each block. For example: [200, 100, 100] will generate 2 blocks. The first will receive a tensor of size 200 and output a tensor of size 100, and the second will receive a tensor of size 100 and output a tensor of size 100. See <code>pytorch_widedeep.models.tab_resnet._layers</code> for details on the structure of each block.</p> <code>[200, 100, 100]</code> <code>blocks_dropout</code> <code>float</code> <p>Block's internal dropout.</p> <code>0.1</code> <code>simplify_blocks</code> <code>bool</code> <p>Boolean indicating if the simplest possible residual blocks (<code>X -&gt; [ [LIN, BN, ACT]  + X ]</code>) will be used instead of a standard one (<code>X -&gt; [ [LIN1, BN1, ACT1] -&gt; [LIN2, BN2]  + X ]</code>).</p> <code>False</code> <code>mlp_hidden_dims</code> <code>Optional[List[int]]</code> <p>List with the number of neurons per dense layer in the MLP. e.g: [64, 32]. If <code>None</code> the  output of the Resnet Blocks will be connected directly to the output neuron(s).</p> <code>None</code> <code>mlp_activation</code> <code>Optional[str]</code> <p>Activation function for the dense layers of the MLP. Currently 'tanh', 'relu', 'leaky'_relu' and _'gelu' are supported. If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to 'relu'.</p> <code>None</code> <code>mlp_dropout</code> <code>Optional[float]</code> <p>float with the dropout between the dense layers of the MLP. If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to 0.0.</p> <code>None</code> <code>mlp_batchnorm</code> <code>Optional[bool]</code> <p>Boolean indicating whether or not batch normalization will be applied to the dense layers If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to False.</p> <code>None</code> <code>mlp_batchnorm_last</code> <code>Optional[bool]</code> <p>Boolean indicating whether or not batch normalization will be applied to the last of the dense layers If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to False.</p> <code>None</code> <code>mlp_linear_first</code> <code>Optional[bool]</code> <p>Boolean indicating the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code> If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to <code>True</code>.</p> <code>None</code> <p>Attributes:</p> Name Type Description <code>encoder</code> <code>Module</code> <p>deep dense Resnet model that will receive the concatenation of the embeddings and the continuous columns</p> <code>mlp</code> <code>Module</code> <p>if <code>mlp_hidden_dims</code> is <code>True</code>, this attribute will be an mlp model that will receive the results of the concatenation of the embeddings and the continuous columns -- if present --.</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import TabResnet\n&gt;&gt;&gt; X_deep = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n&gt;&gt;&gt; colnames = ['a', 'b', 'c', 'd', 'e']\n&gt;&gt;&gt; cat_embed_input = [(u,i,j) for u,i,j in zip(colnames[:4], [4]*4, [8]*4)]\n&gt;&gt;&gt; column_idx = {k:v for v,k in enumerate(colnames)}\n&gt;&gt;&gt; model = TabResnet(blocks_dims=[16,4], column_idx=column_idx, cat_embed_input=cat_embed_input,\n... continuous_cols = ['e'])\n&gt;&gt;&gt; out = model(X_deep)\n</code></pre> Source code in <code>pytorch_widedeep/models/tabular/resnet/tab_resnet.py</code> <pre><code>class TabResnet(BaseTabularModelWithoutAttention):\n    r\"\"\"Defines a `TabResnet` model that can be used as the `deeptabular`\n    component of a Wide &amp; Deep model or independently by itself.\n\n    This class combines embedding representations of the categorical features\n    with numerical (aka continuous) features, embedded or not. These are then\n    passed through a series of Resnet blocks. See\n    `pytorch_widedeep.models.tab_resnet._layers` for details on the\n    structure of each block.\n\n    Most of the parameters for this class are `Optional` since the use of\n    categorical or continuous is in fact optional (i.e. one can use\n    categorical features only, continuous features only or both).\n\n    Parameters\n    ----------\n    column_idx: Dict\n        Dict containing the index of the columns that will be passed through\n        the `TabMlp` model. Required to slice the tensors. e.g. _{'education':\n        0, 'relationship': 1, 'workclass': 2, ...}_.\n    cat_embed_input: List, Optional, default = None\n        List of Tuples with the column name, number of unique values and\n        embedding dimension. e.g. _[(education, 11, 32), ...]_\n    cat_embed_dropout: float, Optional, default = None\n        Categorical embeddings dropout. If `None`, it will default\n        to 0.\n    use_cat_bias: bool, Optional, default = None,\n        Boolean indicating if bias will be used for the categorical embeddings.\n        If `None`, it will default to 'False'.\n    cat_embed_activation: Optional, str, default = None,\n        Activation function for the categorical embeddings, if any. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported\n    continuous_cols: List, Optional, default = None\n        List with the name of the numeric (aka continuous) columns\n    cont_norm_layer: str, Optional, default =  None\n        Type of normalization layer applied to the continuous features.\n        Options are: _'layernorm'_ and _'batchnorm'_. if `None`, no\n        normalization layer will be used.\n    embed_continuous: bool, Optional, default = None,\n        Boolean indicating if the continuous columns will be embedded using\n        one of the available methods: _'standard'_, _'periodic'_\n        or _'piecewise'_. If `None`, it will default to 'False'.&lt;br/&gt;\n        :information_source: **NOTE**: This parameter is deprecated and it\n         will be removed in future releases. Please, use the\n         `embed_continuous_method` parameter instead.\n    embed_continuous_method: Optional, str, default = None,\n        Method to use to embed the continuous features. Options are:\n        _'standard'_, _'periodic'_ or _'piecewise'_. The _'standard'_\n        embedding method is based on the FT-Transformer implementation\n        presented in the paper: [Revisiting Deep Learning Models for\n        Tabular Data](https://arxiv.org/abs/2106.11959v5). The _'periodic'_\n        and_'piecewise'_ methods were presented in the paper: [On Embeddings for\n        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556).\n        Please, read the papers for details.\n    cont_embed_dim: int, Optional, default = None,\n        Size of the continuous embeddings. If the continuous columns are\n        embedded, `cont_embed_dim` must be passed.\n    cont_embed_dropout: float, Optional, default = None,\n        Dropout for the continuous embeddings. If `None`, it will default to 0.0\n    cont_embed_activation: Optional, str, default = None,\n        Activation function for the continuous embeddings if any. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported.\n        If `None`, no activation function will be applied.\n    quantization_setup: Dict[str, List[float]], Optional, default = None,\n        This parameter is used when the _'piecewise'_ method is used to embed\n        the continuous cols. It is a dict where keys are the name of the continuous\n        columns and values are lists with the boundaries for the quantization\n        of the continuous_cols. See the examples for details. If\n        If the _'piecewise'_ method is used, this parameter is required.\n    n_frequencies: int, Optional, default = None,\n        This is the so called _'k'_ in their paper [On Embeddings for\n        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556),\n        and is the number of 'frequencies' that will be used to represent each\n        continuous column. See their Eq 2 in the paper for details. If\n        the _'periodic'_ method is used, this parameter is required.\n    sigma: float, Optional, default = None,\n        This is the sigma parameter in the paper mentioned when describing the\n        previous parameters and it is used to initialise the 'frequency\n        weights'. See their Eq 2 in the paper for details. If\n        the _'periodic'_ method is used, this parameter is required.\n    share_last_layer: bool, Optional, default = None,\n        This parameter is not present in the before mentioned paper but it is implemented in\n        the [official repo](https://github.com/yandex-research/rtdl-num-embeddings/tree/main).\n        If `True` the linear layer that turns the frequencies into embeddings\n        will be shared across the continuous columns. If `False` a different\n        linear layer will be used for each continuous column.\n        If the _'periodic'_ method is used, this parameter is required.\n    full_embed_dropout: bool, Optional, default = None,\n        If `True`, the full embedding corresponding to a column will be masked\n        out/dropout. If `None`, it will default to `False`.\n    blocks_dims: List, default = [200, 100, 100]\n        List of integers that define the input and output units of each block.\n        For example: _[200, 100, 100]_ will generate 2 blocks. The first will\n        receive a tensor of size 200 and output a tensor of size 100, and the\n        second will receive a tensor of size 100 and output a tensor of size\n        100. See `pytorch_widedeep.models.tab_resnet._layers` for\n        details on the structure of each block.\n    blocks_dropout: float, default =  0.1\n        Block's internal dropout.\n    simplify_blocks: bool, default = False,\n        Boolean indicating if the simplest possible residual blocks (`X -&gt; [\n        [LIN, BN, ACT]  + X ]`) will be used instead of a standard one\n        (`X -&gt; [ [LIN1, BN1, ACT1] -&gt; [LIN2, BN2]  + X ]`).\n    mlp_hidden_dims: List, Optional, default = None\n        List with the number of neurons per dense layer in the MLP. e.g:\n        _[64, 32]_. If `None` the  output of the Resnet Blocks will be\n        connected directly to the output neuron(s).\n    mlp_activation: str, Optional, default = None\n        Activation function for the dense layers of the MLP. Currently\n        _'tanh'_, _'relu'_, _'leaky'_relu' and _'gelu'_ are supported.\n        If 'mlp_hidden_dims' is not `None` and this parameter is `None`, it\n        will default to _'relu'_.\n    mlp_dropout: float, Optional, default = None\n        float with the dropout between the dense layers of the MLP.\n        If 'mlp_hidden_dims' is not `None` and this parameter is `None`, it\n        will default to 0.0.\n    mlp_batchnorm: bool, Optional, default = None\n        Boolean indicating whether or not batch normalization will be applied\n        to the dense layers\n        If 'mlp_hidden_dims' is not `None` and this parameter is `None`, it\n        will default to False.\n    mlp_batchnorm_last: bool, Optional, default = None\n        Boolean indicating whether or not batch normalization will be applied\n        to the last of the dense layers\n        If 'mlp_hidden_dims' is not `None` and this parameter is `None`, it\n        will default to False.\n    mlp_linear_first: bool, Optional, default = None\n        Boolean indicating the order of the operations in the dense\n        layer. If `True: [LIN -&gt; ACT -&gt; BN -&gt; DP]`. If `False: [BN -&gt; DP -&gt;\n        LIN -&gt; ACT]`\n        If 'mlp_hidden_dims' is not `None` and this parameter is `None`, it\n        will default to `True`.\n\n    Attributes\n    ----------\n    encoder: nn.Module\n        deep dense Resnet model that will receive the concatenation of the\n        embeddings and the continuous columns\n    mlp: nn.Module\n        if `mlp_hidden_dims` is `True`, this attribute will be an mlp\n        model that will receive the results of the concatenation of the\n        embeddings and the continuous columns -- if present --.\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.models import TabResnet\n    &gt;&gt;&gt; X_deep = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n    &gt;&gt;&gt; colnames = ['a', 'b', 'c', 'd', 'e']\n    &gt;&gt;&gt; cat_embed_input = [(u,i,j) for u,i,j in zip(colnames[:4], [4]*4, [8]*4)]\n    &gt;&gt;&gt; column_idx = {k:v for v,k in enumerate(colnames)}\n    &gt;&gt;&gt; model = TabResnet(blocks_dims=[16,4], column_idx=column_idx, cat_embed_input=cat_embed_input,\n    ... continuous_cols = ['e'])\n    &gt;&gt;&gt; out = model(X_deep)\n    \"\"\"\n\n    def __init__(\n        self,\n        column_idx: Dict[str, int],\n        *,\n        cat_embed_input: Optional[List[Tuple[str, int, int]]] = None,\n        cat_embed_dropout: Optional[float] = None,\n        use_cat_bias: Optional[bool] = None,\n        cat_embed_activation: Optional[str] = None,\n        continuous_cols: Optional[List[str]] = None,\n        cont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\n        embed_continuous: Optional[bool] = None,\n        embed_continuous_method: Optional[\n            Literal[\"standard\", \"piecewise\", \"periodic\"]\n        ] = None,\n        cont_embed_dim: Optional[int] = None,\n        cont_embed_dropout: Optional[float] = None,\n        cont_embed_activation: Optional[str] = None,\n        quantization_setup: Optional[Dict[str, List[float]]] = None,\n        n_frequencies: Optional[int] = None,\n        sigma: Optional[float] = None,\n        share_last_layer: Optional[bool] = None,\n        full_embed_dropout: Optional[bool] = None,\n        blocks_dims: List[int] = [200, 100, 100],\n        blocks_dropout: float = 0.1,\n        simplify_blocks: bool = False,\n        mlp_hidden_dims: Optional[List[int]] = None,\n        mlp_activation: Optional[str] = None,\n        mlp_dropout: Optional[float] = None,\n        mlp_batchnorm: Optional[bool] = None,\n        mlp_batchnorm_last: Optional[bool] = None,\n        mlp_linear_first: Optional[bool] = None,\n    ):\n        super(TabResnet, self).__init__(\n            column_idx=column_idx,\n            cat_embed_input=cat_embed_input,\n            cat_embed_dropout=cat_embed_dropout,\n            use_cat_bias=use_cat_bias,\n            cat_embed_activation=cat_embed_activation,\n            continuous_cols=continuous_cols,\n            cont_norm_layer=cont_norm_layer,\n            embed_continuous=embed_continuous,\n            embed_continuous_method=embed_continuous_method,\n            cont_embed_dim=cont_embed_dim,\n            cont_embed_dropout=cont_embed_dropout,\n            cont_embed_activation=cont_embed_activation,\n            quantization_setup=quantization_setup,\n            n_frequencies=n_frequencies,\n            sigma=sigma,\n            share_last_layer=share_last_layer,\n            full_embed_dropout=full_embed_dropout,\n        )\n\n        if len(blocks_dims) &lt; 2:\n            raise ValueError(\n                \"'blocks' must contain at least two elements, e.g. [256, 128]\"\n            )\n\n        self.blocks_dims = blocks_dims\n        self.blocks_dropout = blocks_dropout\n        self.simplify_blocks = simplify_blocks\n\n        self.mlp_hidden_dims = mlp_hidden_dims\n        self.mlp_activation = mlp_activation\n        self.mlp_dropout = mlp_dropout\n        self.mlp_batchnorm = mlp_batchnorm\n        self.mlp_batchnorm_last = mlp_batchnorm_last\n        self.mlp_linear_first = mlp_linear_first\n\n        # Embeddings are instantiated at the base model\n\n        # Resnet\n        dense_resnet_input_dim = self.cat_out_dim + self.cont_out_dim\n        self.encoder = DenseResnet(\n            dense_resnet_input_dim, blocks_dims, blocks_dropout, self.simplify_blocks\n        )\n\n        # Mlp: adding an MLP on top of the Resnet blocks is optional and\n        # therefore all related params are optional\n        if self.mlp_hidden_dims is not None:\n            self.mlp = MLP(\n                d_hidden=[self.blocks_dims[-1]] + self.mlp_hidden_dims,\n                activation=(\n                    \"relu\" if self.mlp_activation is None else self.mlp_activation\n                ),\n                dropout=0.0 if self.mlp_dropout is None else self.mlp_dropout,\n                batchnorm=False if self.mlp_batchnorm is None else self.mlp_batchnorm,\n                batchnorm_last=(\n                    False\n                    if self.mlp_batchnorm_last is None\n                    else self.mlp_batchnorm_last\n                ),\n                linear_first=(\n                    True if self.mlp_linear_first is None else self.mlp_linear_first\n                ),\n            )\n        else:\n            self.mlp = None\n\n    def forward(self, X: Tensor) -&gt; Tensor:\n        x = self._get_embeddings(X)\n        x = self.encoder(x)\n        if self.mlp is not None:\n            x = self.mlp(x)\n        return x\n\n    @property\n    def output_dim(self) -&gt; int:\n        r\"\"\"The output dimension of the model. This is a required property\n        neccesary to build the `WideDeep` class\n        \"\"\"\n        return (\n            self.mlp_hidden_dims[-1]\n            if self.mlp_hidden_dims is not None\n            else self.blocks_dims[-1]\n        )\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.resnet.tab_resnet.TabResnet.output_dim","title":"output_dim  <code>property</code>","text":"<pre><code>output_dim\n</code></pre> <p>The output dimension of the model. This is a required property neccesary to build the <code>WideDeep</code> class</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.resnet.tab_resnet.TabResnetDecoder","title":"TabResnetDecoder","text":"<p>               Bases: <code>Module</code></p> <p>Companion decoder model for the <code>TabResnet</code> model (which can be considered an encoder itself)</p> <p>This class is designed to be used with the <code>EncoderDecoderTrainer</code> when using self-supervised pre-training (see the corresponding section in the docs). This class will receive the output from the ResNet blocks or the MLP(if present) and 'reconstruct' the embeddings.</p> <p>Parameters:</p> Name Type Description Default <code>embed_dim</code> <code>int</code> <p>Size of the embeddings tensor to be reconstructed.</p> required <code>blocks_dims</code> <code>List[int]</code> <p>List of integers that define the input and output units of each block. For example: [200, 100, 100] will generate 2 blocks. The first will receive a tensor of size 200 and output a tensor of size 100, and the second will receive a tensor of size 100 and output a tensor of size 100. See <code>pytorch_widedeep.models.tab_resnet._layers</code> for details on the structure of each block.</p> <code>[100, 100, 200]</code> <code>blocks_dropout</code> <code>float</code> <p>Block's internal dropout.</p> <code>0.1</code> <code>simplify_blocks</code> <code>bool</code> <p>Boolean indicating if the simplest possible residual blocks (<code>X -&gt; [ [LIN, BN, ACT]  + X ]</code>) will be used instead of a standard one (<code>X -&gt; [ [LIN1, BN1, ACT1] -&gt; [LIN2, BN2]  + X ]</code>).</p> <code>False</code> <code>mlp_hidden_dims</code> <code>Optional[List[int]]</code> <p>List with the number of neurons per dense layer in the MLP. e.g: [64, 32]. If <code>None</code> the  output of the Resnet Blocks will be connected directly to the output neuron(s).</p> <code>None</code> <code>mlp_activation</code> <code>Optional[str]</code> <p>Activation function for the dense layers of the MLP. Currently 'tanh', 'relu', 'leaky'_relu' and _'gelu' are supported. If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to 'relu'.</p> <code>None</code> <code>mlp_dropout</code> <code>Optional[float]</code> <p>float with the dropout between the dense layers of the MLP. If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to 0.0.</p> <code>None</code> <code>mlp_batchnorm</code> <code>Optional[bool]</code> <p>Boolean indicating whether or not batch normalization will be applied to the dense layers If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to False.</p> <code>None</code> <code>mlp_batchnorm_last</code> <code>Optional[bool]</code> <p>Boolean indicating whether or not batch normalization will be applied to the last of the dense layers If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to False.</p> <code>None</code> <code>mlp_linear_first</code> <code>Optional[bool]</code> <p>Boolean indicating the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code> If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to <code>True</code>.</p> <code>None</code> <p>Attributes:</p> Name Type Description <code>decoder</code> <code>Module</code> <p>deep dense Resnet model that will receive the output of the encoder IF <code>mlp_hidden_dims</code> is None</p> <code>mlp</code> <code>Module</code> <p>if <code>mlp_hidden_dims</code> is not None, the overall decoder will consist in an MLP that will receive the output of the encoder followed by the deep dense Resnet.</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import TabResnetDecoder\n&gt;&gt;&gt; x_inp = torch.rand(3, 8)\n&gt;&gt;&gt; decoder = TabResnetDecoder(embed_dim=32, blocks_dims=[8, 16, 16])\n&gt;&gt;&gt; res = decoder(x_inp)\n&gt;&gt;&gt; res.shape\ntorch.Size([3, 32])\n</code></pre> Source code in <code>pytorch_widedeep/models/tabular/resnet/tab_resnet.py</code> <pre><code>class TabResnetDecoder(nn.Module):\n    r\"\"\"Companion decoder model for the `TabResnet` model (which can be\n    considered an encoder itself)\n\n    This class is designed to be used with the `EncoderDecoderTrainer` when\n    using self-supervised pre-training (see the corresponding section in the\n    docs). This class will receive the output from the ResNet blocks or the\n    MLP(if present) and '_reconstruct_' the embeddings.\n\n    Parameters\n    ----------\n    embed_dim: int\n        Size of the embeddings tensor to be reconstructed.\n    blocks_dims: List, default = [200, 100, 100]\n        List of integers that define the input and output units of each block.\n        For example: _[200, 100, 100]_ will generate 2 blocks. The first will\n        receive a tensor of size 200 and output a tensor of size 100, and the\n        second will receive a tensor of size 100 and output a tensor of size\n        100. See `pytorch_widedeep.models.tab_resnet._layers` for\n        details on the structure of each block.\n    blocks_dropout: float, default =  0.1\n        Block's internal dropout.\n    simplify_blocks: bool, default = False,\n        Boolean indicating if the simplest possible residual blocks (`X -&gt; [\n        [LIN, BN, ACT]  + X ]`) will be used instead of a standard one\n        (`X -&gt; [ [LIN1, BN1, ACT1] -&gt; [LIN2, BN2]  + X ]`).\n    mlp_hidden_dims: List, Optional, default = None\n        List with the number of neurons per dense layer in the MLP. e.g:\n        _[64, 32]_. If `None` the  output of the Resnet Blocks will be\n        connected directly to the output neuron(s).\n    mlp_activation: str, Optional, default = None\n        Activation function for the dense layers of the MLP. Currently\n        _'tanh'_, _'relu'_, _'leaky'_relu' and _'gelu'_ are supported.\n        If 'mlp_hidden_dims' is not `None` and this parameter is `None`, it\n        will default to _'relu'_.\n    mlp_dropout: float, Optional, default = None\n        float with the dropout between the dense layers of the MLP.\n        If 'mlp_hidden_dims' is not `None` and this parameter is `None`, it\n        will default to 0.0.\n    mlp_batchnorm: bool, Optional, default = None\n        Boolean indicating whether or not batch normalization will be applied\n        to the dense layers\n        If 'mlp_hidden_dims' is not `None` and this parameter is `None`, it\n        will default to False.\n    mlp_batchnorm_last: bool, Optional, default = None\n        Boolean indicating whether or not batch normalization will be applied\n        to the last of the dense layers\n        If 'mlp_hidden_dims' is not `None` and this parameter is `None`, it\n        will default to False.\n    mlp_linear_first: bool, Optional, default = None\n        Boolean indicating the order of the operations in the dense\n        layer. If `True: [LIN -&gt; ACT -&gt; BN -&gt; DP]`. If `False: [BN -&gt; DP -&gt;\n        LIN -&gt; ACT]`\n        If 'mlp_hidden_dims' is not `None` and this parameter is `None`, it\n        will default to `True`.\n\n    Attributes\n    ----------\n    decoder: nn.Module\n        deep dense Resnet model that will receive the output of the encoder IF\n        `mlp_hidden_dims` is None\n    mlp: nn.Module\n        if `mlp_hidden_dims` is not None, the overall decoder will consist\n        in an MLP that will receive the output of the encoder followed by the\n        deep dense Resnet.\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.models import TabResnetDecoder\n    &gt;&gt;&gt; x_inp = torch.rand(3, 8)\n    &gt;&gt;&gt; decoder = TabResnetDecoder(embed_dim=32, blocks_dims=[8, 16, 16])\n    &gt;&gt;&gt; res = decoder(x_inp)\n    &gt;&gt;&gt; res.shape\n    torch.Size([3, 32])\n    \"\"\"\n\n    def __init__(\n        self,\n        embed_dim: int,\n        blocks_dims: List[int] = [100, 100, 200],\n        blocks_dropout: float = 0.1,\n        simplify_blocks: bool = False,\n        mlp_hidden_dims: Optional[List[int]] = None,\n        mlp_activation: Optional[str] = None,\n        mlp_dropout: Optional[float] = None,\n        mlp_batchnorm: Optional[bool] = None,\n        mlp_batchnorm_last: Optional[bool] = None,\n        mlp_linear_first: Optional[bool] = None,\n    ):\n        super(TabResnetDecoder, self).__init__()\n\n        if len(blocks_dims) &lt; 2:\n            raise ValueError(\n                \"'blocks' must contain at least two elements, e.g. [256, 128]\"\n            )\n\n        self.embed_dim = embed_dim\n\n        self.blocks_dims = blocks_dims\n        self.blocks_dropout = blocks_dropout\n        self.simplify_blocks = simplify_blocks\n\n        self.mlp_hidden_dims = mlp_hidden_dims\n        self.mlp_activation = mlp_activation\n        self.mlp_dropout = mlp_dropout\n        self.mlp_batchnorm = mlp_batchnorm\n        self.mlp_batchnorm_last = mlp_batchnorm_last\n        self.mlp_linear_first = mlp_linear_first\n\n        if self.mlp_hidden_dims is not None:\n            self.mlp = MLP(\n                d_hidden=[self.mlp_first_hidden_dim] + self.mlp_hidden_dims,\n                activation=(\n                    \"relu\" if self.mlp_activation is None else self.mlp_activation\n                ),\n                dropout=0.0 if self.mlp_dropout is None else self.mlp_dropout,\n                batchnorm=False if self.mlp_batchnorm is None else self.mlp_batchnorm,\n                batchnorm_last=(\n                    False\n                    if self.mlp_batchnorm_last is None\n                    else self.mlp_batchnorm_last\n                ),\n                linear_first=(\n                    True if self.mlp_linear_first is None else self.mlp_linear_first\n                ),\n            )\n            self.decoder = DenseResnet(\n                self.mlp_hidden_dims[-1],\n                blocks_dims,\n                blocks_dropout,\n                self.simplify_blocks,\n            )\n        else:\n            self.mlp = None\n            self.decoder = DenseResnet(\n                blocks_dims[0], blocks_dims, blocks_dropout, self.simplify_blocks\n            )\n\n        self.reconstruction_layer = nn.Linear(blocks_dims[-1], embed_dim, bias=False)\n\n    def forward(self, X: Tensor) -&gt; Tensor:\n        x = self.mlp(X) if self.mlp is not None else X\n        return self.reconstruction_layer(self.decoder(x))\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.tabnet.tab_net.TabNet","title":"TabNet","text":"<p>               Bases: <code>BaseTabularModelWithoutAttention</code></p> <p>Defines a TabNet model that can be used as the <code>deeptabular</code> component of a Wide &amp; Deep model or independently by itself.</p> <p>The implementation in this library is fully based on that here by the dreamquark-ai team, simply adapted so that it can work within the <code>WideDeep</code> frame. Therefore, ALL CREDIT TO THE DREAMQUARK-AI TEAM.</p> <p>Parameters:</p> Name Type Description Default <code>column_idx</code> <code>Dict[str, int]</code> <p>Dict containing the index of the columns that will be passed through the <code>TabMlp</code> model. Required to slice the tensors. e.g. {'education': 0, 'relationship': 1, 'workclass': 2, ...}.</p> required <code>cat_embed_input</code> <code>Optional[List[Tuple[str, int, int]]]</code> <p>List of Tuples with the column name, number of unique values and embedding dimension. e.g. [(education, 11, 32), ...]</p> <code>None</code> <code>cat_embed_dropout</code> <code>Optional[float]</code> <p>Categorical embeddings dropout. If <code>None</code>, it will default to 0.</p> <code>None</code> <code>use_cat_bias</code> <code>Optional[bool]</code> <p>Boolean indicating if bias will be used for the categorical embeddings. If <code>None</code>, it will default to 'False'.</p> <code>None</code> <code>cat_embed_activation</code> <code>Optional[str]</code> <p>Activation function for the categorical embeddings, if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> <code>None</code> <code>continuous_cols</code> <code>Optional[List[str]]</code> <p>List with the name of the numeric (aka continuous) columns</p> <code>None</code> <code>cont_norm_layer</code> <code>Optional[Literal[batchnorm, layernorm]]</code> <p>Type of normalization layer applied to the continuous features. Options are: 'layernorm' and 'batchnorm'. if <code>None</code>, no normalization layer will be used.</p> <code>None</code> <code>embed_continuous</code> <code>Optional[bool]</code> <p>Boolean indicating if the continuous columns will be embedded using one of the available methods: 'standard', 'periodic' or 'piecewise'. If <code>None</code>, it will default to 'False'. NOTE: This parameter is deprecated and it  will be removed in future releases. Please, use the  <code>embed_continuous_method</code> parameter instead.</p> <code>None</code> <code>embed_continuous_method</code> <code>Optional[Literal[standard, piecewise, periodic]]</code> <p>Method to use to embed the continuous features. Options are: 'standard', 'periodic' or 'piecewise'. The 'standard' embedding method is based on the FT-Transformer implementation presented in the paper: Revisiting Deep Learning Models for Tabular Data. The 'periodic' and_'piecewise'_ methods were presented in the paper: On Embeddings for Numerical Features in Tabular Deep Learning. Please, read the papers for details.</p> <code>None</code> <code>cont_embed_dim</code> <code>Optional[int]</code> <p>Size of the continuous embeddings. If the continuous columns are embedded, <code>cont_embed_dim</code> must be passed.</p> <code>None</code> <code>cont_embed_dropout</code> <code>Optional[float]</code> <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p> <code>None</code> <code>cont_embed_activation</code> <code>Optional[str]</code> <p>Activation function for the continuous embeddings if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported. If <code>None</code>, no activation function will be applied.</p> <code>None</code> <code>quantization_setup</code> <code>Optional[Dict[str, List[float]]]</code> <p>This parameter is used when the 'piecewise' method is used to embed the continuous cols. It is a dict where keys are the name of the continuous columns and values are lists with the boundaries for the quantization of the continuous_cols. See the examples for details. If If the 'piecewise' method is used, this parameter is required.</p> <code>None</code> <code>n_frequencies</code> <code>Optional[int]</code> <p>This is the so called 'k' in their paper On Embeddings for Numerical Features in Tabular Deep Learning, and is the number of 'frequencies' that will be used to represent each continuous column. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>sigma</code> <code>Optional[float]</code> <p>This is the sigma parameter in the paper mentioned when describing the previous parameters and it is used to initialise the 'frequency weights'. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>share_last_layer</code> <code>Optional[bool]</code> <p>This parameter is not present in the before mentioned paper but it is implemented in the official repo. If <code>True</code> the linear layer that turns the frequencies into embeddings will be shared across the continuous columns. If <code>False</code> a different linear layer will be used for each continuous column. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>full_embed_dropout</code> <code>Optional[bool]</code> <p>If <code>True</code>, the full embedding corresponding to a column will be masked out/dropout. If <code>None</code>, it will default to <code>False</code>.</p> <code>None</code> <code>n_steps</code> <code>int</code> <p>number of decision steps. For a better understanding of the function of <code>n_steps</code> and the upcoming parameters, please see the paper.</p> <code>3</code> <code>step_dim</code> <code>int</code> <p>Step's output dimension. This is the output dimension that <code>WideDeep</code> will collect and connect to the output neuron(s).</p> <code>8</code> <code>attn_dim</code> <code>int</code> <p>Attention dimension</p> <code>8</code> <code>dropout</code> <code>float</code> <p>GLU block's internal dropout</p> <code>0.0</code> <code>n_glu_step_dependent</code> <code>int</code> <p>number of GLU Blocks (<code>[FC -&gt; BN -&gt; GLU]</code>) that are step dependent</p> <code>2</code> <code>n_glu_shared</code> <code>int</code> <p>number of GLU Blocks (<code>[FC -&gt; BN -&gt; GLU]</code>) that will be shared across decision steps</p> <code>2</code> <code>ghost_bn</code> <code>bool</code> <p>Boolean indicating if Ghost Batch Normalization will be used.</p> <code>True</code> <code>virtual_batch_size</code> <code>int</code> <p>Batch size when using Ghost Batch Normalization</p> <code>128</code> <code>momentum</code> <code>float</code> <p>Ghost Batch Normalization's momentum. The dreamquark-ai advises for very low values. However high values are used in the original publication. During our tests higher values lead to better results</p> <code>0.02</code> <code>gamma</code> <code>float</code> <p>Relaxation parameter in the paper. When gamma = 1, a feature is enforced to be used only at one decision step. As gamma increases, more flexibility is provided to use a feature at multiple decision steps</p> <code>1.3</code> <code>epsilon</code> <code>float</code> <p>Float to avoid log(0). Always keep low</p> <code>1e-15</code> <code>mask_type</code> <code>str</code> <p>Mask function to use. Either 'sparsemax' or 'entmax'</p> <code>'sparsemax'</code> <p>Attributes:</p> Name Type Description <code>encoder</code> <code>Module</code> <p>the TabNet encoder. For details see the original publication.</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n&gt;&gt;&gt; colnames = [\"a\", \"b\", \"c\", \"d\", \"e\"]\n&gt;&gt;&gt; cat_embed_input = [(u, i, j) for u, i, j in zip(colnames[:4], [4] * 4, [8] * 4)]\n&gt;&gt;&gt; column_idx = {k: v for v, k in enumerate(colnames)}\n&gt;&gt;&gt; model = TabNet(column_idx=column_idx, cat_embed_input=cat_embed_input, continuous_cols=[\"e\"])\n&gt;&gt;&gt; out = model(X_tab)\n</code></pre> Source code in <code>pytorch_widedeep/models/tabular/tabnet/tab_net.py</code> <pre><code>class TabNet(BaseTabularModelWithoutAttention):\n    r\"\"\"Defines a [TabNet model](https://arxiv.org/abs/1908.07442) that\n    can be used as the `deeptabular` component of a Wide &amp; Deep model or\n    independently by itself.\n\n    The implementation in this library is fully based on that\n    [here](https://github.com/dreamquark-ai/tabnet) by the dreamquark-ai team,\n    simply adapted so that it can work within the `WideDeep` frame.\n    Therefore, **ALL CREDIT TO THE DREAMQUARK-AI TEAM**.\n\n    Parameters\n    ----------\n    column_idx: Dict\n        Dict containing the index of the columns that will be passed through\n        the `TabMlp` model. Required to slice the tensors. e.g. _{'education':\n        0, 'relationship': 1, 'workclass': 2, ...}_.\n    cat_embed_input: List, Optional, default = None\n        List of Tuples with the column name, number of unique values and\n        embedding dimension. e.g. _[(education, 11, 32), ...]_\n    cat_embed_dropout: float, Optional, default = None\n        Categorical embeddings dropout. If `None`, it will default\n        to 0.\n    use_cat_bias: bool, Optional, default = None,\n        Boolean indicating if bias will be used for the categorical embeddings.\n        If `None`, it will default to 'False'.\n    cat_embed_activation: Optional, str, default = None,\n        Activation function for the categorical embeddings, if any. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported\n    continuous_cols: List, Optional, default = None\n        List with the name of the numeric (aka continuous) columns\n    cont_norm_layer: str, Optional, default =  None\n        Type of normalization layer applied to the continuous features.\n        Options are: _'layernorm'_ and _'batchnorm'_. if `None`, no\n        normalization layer will be used.\n    embed_continuous: bool, Optional, default = None,\n        Boolean indicating if the continuous columns will be embedded using\n        one of the available methods: _'standard'_, _'periodic'_\n        or _'piecewise'_. If `None`, it will default to 'False'.&lt;br/&gt;\n        :information_source: **NOTE**: This parameter is deprecated and it\n         will be removed in future releases. Please, use the\n         `embed_continuous_method` parameter instead.\n    embed_continuous_method: Optional, str, default = None,\n        Method to use to embed the continuous features. Options are:\n        _'standard'_, _'periodic'_ or _'piecewise'_. The _'standard'_\n        embedding method is based on the FT-Transformer implementation\n        presented in the paper: [Revisiting Deep Learning Models for\n        Tabular Data](https://arxiv.org/abs/2106.11959v5). The _'periodic'_\n        and_'piecewise'_ methods were presented in the paper: [On Embeddings for\n        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556).\n        Please, read the papers for details.\n    cont_embed_dim: int, Optional, default = None,\n        Size of the continuous embeddings. If the continuous columns are\n        embedded, `cont_embed_dim` must be passed.\n    cont_embed_dropout: float, Optional, default = None,\n        Dropout for the continuous embeddings. If `None`, it will default to 0.0\n    cont_embed_activation: Optional, str, default = None,\n        Activation function for the continuous embeddings if any. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported.\n        If `None`, no activation function will be applied.\n    quantization_setup: Dict[str, List[float]], Optional, default = None,\n        This parameter is used when the _'piecewise'_ method is used to embed\n        the continuous cols. It is a dict where keys are the name of the continuous\n        columns and values are lists with the boundaries for the quantization\n        of the continuous_cols. See the examples for details. If\n        If the _'piecewise'_ method is used, this parameter is required.\n    n_frequencies: int, Optional, default = None,\n        This is the so called _'k'_ in their paper [On Embeddings for\n        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556),\n        and is the number of 'frequencies' that will be used to represent each\n        continuous column. See their Eq 2 in the paper for details. If\n        the _'periodic'_ method is used, this parameter is required.\n    sigma: float, Optional, default = None,\n        This is the sigma parameter in the paper mentioned when describing the\n        previous parameters and it is used to initialise the 'frequency\n        weights'. See their Eq 2 in the paper for details. If\n        the _'periodic'_ method is used, this parameter is required.\n    share_last_layer: bool, Optional, default = None,\n        This parameter is not present in the before mentioned paper but it is implemented in\n        the [official repo](https://github.com/yandex-research/rtdl-num-embeddings/tree/main).\n        If `True` the linear layer that turns the frequencies into embeddings\n        will be shared across the continuous columns. If `False` a different\n        linear layer will be used for each continuous column.\n        If the _'periodic'_ method is used, this parameter is required.\n    full_embed_dropout: bool, Optional, default = None,\n        If `True`, the full embedding corresponding to a column will be masked\n        out/dropout. If `None`, it will default to `False`.\n    n_steps: int, default = 3\n        number of decision steps. For a better understanding of the function\n        of `n_steps` and the upcoming parameters, please see the\n        [paper](https://arxiv.org/abs/1908.07442).\n    step_dim: int, default = 8\n        Step's output dimension. This is the output dimension that\n        `WideDeep` will collect and connect to the output neuron(s).\n    attn_dim: int, default = 8\n        Attention dimension\n    dropout: float, default = 0.0\n        GLU block's internal dropout\n    n_glu_step_dependent: int, default = 2\n        number of GLU Blocks (`[FC -&gt; BN -&gt; GLU]`) that are step dependent\n    n_glu_shared: int, default = 2\n        number of GLU Blocks (`[FC -&gt; BN -&gt; GLU]`) that will be shared\n        across decision steps\n    ghost_bn: bool, default=True\n        Boolean indicating if [Ghost Batch Normalization](https://arxiv.org/abs/1705.08741)\n        will be used.\n    virtual_batch_size: int, default = 128\n        Batch size when using Ghost Batch Normalization\n    momentum: float, default = 0.02\n        Ghost Batch Normalization's momentum. The dreamquark-ai advises for\n        very low values. However high values are used in the original\n        publication. During our tests higher values lead to better results\n    gamma: float, default = 1.3\n        Relaxation parameter in the paper. When gamma = 1, a feature is\n        enforced to be used only at one decision step. As gamma increases,\n        more flexibility is provided to use a feature at multiple decision\n        steps\n    epsilon: float, default = 1e-15\n        Float to avoid log(0). Always keep low\n    mask_type: str, default = \"sparsemax\"\n        Mask function to use. Either _'sparsemax'_ or _'entmax'_\n\n    Attributes\n    ----------\n    encoder: nn.Module\n        the TabNet encoder. For details see the [original publication](https://arxiv.org/abs/1908.07442).\n\n    Examples\n    --------\n    &gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n    &gt;&gt;&gt; colnames = [\"a\", \"b\", \"c\", \"d\", \"e\"]\n    &gt;&gt;&gt; cat_embed_input = [(u, i, j) for u, i, j in zip(colnames[:4], [4] * 4, [8] * 4)]\n    &gt;&gt;&gt; column_idx = {k: v for v, k in enumerate(colnames)}\n    &gt;&gt;&gt; model = TabNet(column_idx=column_idx, cat_embed_input=cat_embed_input, continuous_cols=[\"e\"])\n    &gt;&gt;&gt; out = model(X_tab)\n    \"\"\"\n\n    def __init__(\n        self,\n        column_idx: Dict[str, int],\n        *,\n        cat_embed_input: Optional[List[Tuple[str, int, int]]] = None,\n        cat_embed_dropout: Optional[float] = None,\n        use_cat_bias: Optional[bool] = None,\n        cat_embed_activation: Optional[str] = None,\n        continuous_cols: Optional[List[str]] = None,\n        cont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\n        embed_continuous: Optional[bool] = None,\n        embed_continuous_method: Optional[\n            Literal[\"standard\", \"piecewise\", \"periodic\"]\n        ] = None,\n        cont_embed_dim: Optional[int] = None,\n        cont_embed_dropout: Optional[float] = None,\n        cont_embed_activation: Optional[str] = None,\n        quantization_setup: Optional[Dict[str, List[float]]] = None,\n        n_frequencies: Optional[int] = None,\n        sigma: Optional[float] = None,\n        share_last_layer: Optional[bool] = None,\n        full_embed_dropout: Optional[bool] = None,\n        n_steps: int = 3,\n        step_dim: int = 8,\n        attn_dim: int = 8,\n        dropout: float = 0.0,\n        n_glu_step_dependent: int = 2,\n        n_glu_shared: int = 2,\n        ghost_bn: bool = True,\n        virtual_batch_size: int = 128,\n        momentum: float = 0.02,\n        gamma: float = 1.3,\n        epsilon: float = 1e-15,\n        mask_type: str = \"sparsemax\",\n    ):\n        super(TabNet, self).__init__(\n            column_idx=column_idx,\n            cat_embed_input=cat_embed_input,\n            cat_embed_dropout=cat_embed_dropout,\n            use_cat_bias=use_cat_bias,\n            cat_embed_activation=cat_embed_activation,\n            continuous_cols=continuous_cols,\n            cont_norm_layer=cont_norm_layer,\n            embed_continuous=embed_continuous,\n            embed_continuous_method=embed_continuous_method,\n            cont_embed_dim=cont_embed_dim,\n            cont_embed_dropout=cont_embed_dropout,\n            cont_embed_activation=cont_embed_activation,\n            quantization_setup=quantization_setup,\n            n_frequencies=n_frequencies,\n            sigma=sigma,\n            share_last_layer=share_last_layer,\n            full_embed_dropout=full_embed_dropout,\n        )\n\n        self.n_steps = n_steps\n        self.step_dim = step_dim\n        self.attn_dim = attn_dim\n        self.dropout = dropout\n        self.n_glu_step_dependent = n_glu_step_dependent\n        self.n_glu_shared = n_glu_shared\n        self.ghost_bn = ghost_bn\n        self.virtual_batch_size = virtual_batch_size\n        self.momentum = momentum\n        self.gamma = gamma\n        self.epsilon = epsilon\n        self.mask_type = mask_type\n\n        # Embeddings are instantiated at the base model\n        self.embed_out_dim = self.cat_out_dim + self.cont_out_dim\n\n        # TabNet\n        self.encoder = TabNetEncoder(\n            self.embed_out_dim,\n            n_steps,\n            step_dim,\n            attn_dim,\n            dropout,\n            n_glu_step_dependent,\n            n_glu_shared,\n            ghost_bn,\n            virtual_batch_size,\n            momentum,\n            gamma,\n            epsilon,\n            mask_type,\n        )\n\n    def forward(\n        self, X: Tensor, prior: Optional[Tensor] = None\n    ) -&gt; Tuple[Tensor, Tensor]:\n        x = self._get_embeddings(X)\n        steps_output, M_loss = self.encoder(x, prior)\n        res = torch.sum(torch.stack(steps_output, dim=0), dim=0)\n        return (res, M_loss)\n\n    def forward_masks(self, X: Tensor) -&gt; Tuple[Tensor, Dict[int, Tensor]]:\n        x = self._get_embeddings(X)\n        return self.encoder.forward_masks(x)\n\n    @property\n    def output_dim(self) -&gt; int:\n        r\"\"\"The output dimension of the model. This is a required property\n        neccesary to build the `WideDeep` class\n        \"\"\"\n        return self.step_dim\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.tabnet.tab_net.TabNet.output_dim","title":"output_dim  <code>property</code>","text":"<pre><code>output_dim\n</code></pre> <p>The output dimension of the model. This is a required property neccesary to build the <code>WideDeep</code> class</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.tabnet.tab_net.TabNetDecoder","title":"TabNetDecoder","text":"<p>               Bases: <code>Module</code></p> <p>Companion decoder model for the <code>TabNet</code> model (which can be considered an encoder itself)</p> <p>This class is designed to be used with the <code>EncoderDecoderTrainer</code> when using self-supervised pre-training (see the corresponding section in the docs). This class will receive the output from the <code>TabNet</code> encoder (i.e. the output from the so called 'steps') and 'reconstruct' the embeddings.</p> <p>Parameters:</p> Name Type Description Default <code>embed_dim</code> <code>int</code> <p>Size of the embeddings tensor to be reconstructed.</p> required <code>n_steps</code> <code>int</code> <p>number of decision steps. For a better understanding of the function of <code>n_steps</code> and the upcoming parameters, please see the paper.</p> <code>3</code> <code>step_dim</code> <code>int</code> <p>Step's output dimension. This is the output dimension that <code>WideDeep</code> will collect and connect to the output neuron(s).</p> <code>8</code> <code>dropout</code> <code>float</code> <p>GLU block's internal dropout</p> <code>0.0</code> <code>n_glu_step_dependent</code> <code>int</code> <p>number of GLU Blocks (<code>[FC -&gt; BN -&gt; GLU]</code>) that are step dependent</p> <code>2</code> <code>n_glu_shared</code> <code>int</code> <p>number of GLU Blocks (<code>[FC -&gt; BN -&gt; GLU]</code>) that will be shared across decision steps</p> <code>2</code> <code>ghost_bn</code> <code>bool</code> <p>Boolean indicating if Ghost Batch Normalization will be used.</p> <code>True</code> <code>virtual_batch_size</code> <code>int</code> <p>Batch size when using Ghost Batch Normalization</p> <code>128</code> <code>momentum</code> <code>float</code> <p>Ghost Batch Normalization's momentum. The dreamquark-ai advises for very low values. However high values are used in the original publication. During our tests higher values lead to better results</p> <code>0.02</code> <p>Attributes:</p> Name Type Description <code>decoder</code> <code>Module</code> <p>decoder that will receive the output from the encoder's steps and will reconstruct the embeddings</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import TabNetDecoder\n&gt;&gt;&gt; x_inp = [torch.rand(3, 8), torch.rand(3, 8), torch.rand(3, 8)]\n&gt;&gt;&gt; decoder = TabNetDecoder(embed_dim=32, ghost_bn=False)\n&gt;&gt;&gt; res = decoder(x_inp)\n&gt;&gt;&gt; res.shape\ntorch.Size([3, 32])\n</code></pre> Source code in <code>pytorch_widedeep/models/tabular/tabnet/tab_net.py</code> <pre><code>class TabNetDecoder(nn.Module):\n    r\"\"\"Companion decoder model for the `TabNet` model (which can be\n    considered an encoder itself)\n\n    This class is designed to be used with the `EncoderDecoderTrainer` when\n    using self-supervised pre-training (see the corresponding section in the\n    docs). This class will receive the output from the `TabNet` encoder\n    (i.e. the output from the so called 'steps') and '_reconstruct_' the\n    embeddings.\n\n    Parameters\n    ----------\n    embed_dim: int\n        Size of the embeddings tensor to be reconstructed.\n    n_steps: int, default = 3\n        number of decision steps. For a better understanding of the function\n        of `n_steps` and the upcoming parameters, please see the\n        [paper](https://arxiv.org/abs/1908.07442).\n    step_dim: int, default = 8\n        Step's output dimension. This is the output dimension that\n        `WideDeep` will collect and connect to the output neuron(s).\n    dropout: float, default = 0.0\n        GLU block's internal dropout\n    n_glu_step_dependent: int, default = 2\n        number of GLU Blocks (`[FC -&gt; BN -&gt; GLU]`) that are step dependent\n    n_glu_shared: int, default = 2\n        number of GLU Blocks (`[FC -&gt; BN -&gt; GLU]`) that will be shared\n        across decision steps\n    ghost_bn: bool, default=True\n        Boolean indicating if [Ghost Batch Normalization](https://arxiv.org/abs/1705.08741)\n        will be used.\n    virtual_batch_size: int, default = 128\n        Batch size when using Ghost Batch Normalization\n    momentum: float, default = 0.02\n        Ghost Batch Normalization's momentum. The dreamquark-ai advises for\n        very low values. However high values are used in the original\n        publication. During our tests higher values lead to better results\n\n    Attributes\n    ----------\n    decoder: nn.Module\n        decoder that will receive the output from the encoder's steps and will\n        reconstruct the embeddings\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.models import TabNetDecoder\n    &gt;&gt;&gt; x_inp = [torch.rand(3, 8), torch.rand(3, 8), torch.rand(3, 8)]\n    &gt;&gt;&gt; decoder = TabNetDecoder(embed_dim=32, ghost_bn=False)\n    &gt;&gt;&gt; res = decoder(x_inp)\n    &gt;&gt;&gt; res.shape\n    torch.Size([3, 32])\n    \"\"\"\n\n    def __init__(\n        self,\n        embed_dim: int,\n        n_steps: int = 3,\n        step_dim: int = 8,\n        dropout: float = 0.0,\n        n_glu_step_dependent: int = 2,\n        n_glu_shared: int = 2,\n        ghost_bn: bool = True,\n        virtual_batch_size: int = 128,\n        momentum: float = 0.02,\n    ):\n        super(TabNetDecoder, self).__init__()\n\n        self.n_steps = n_steps\n        self.step_dim = step_dim\n        self.dropout = dropout\n        self.n_glu_step_dependent = n_glu_step_dependent\n        self.n_glu_shared = n_glu_shared\n        self.ghost_bn = ghost_bn\n        self.virtual_batch_size = virtual_batch_size\n        self.momentum = momentum\n\n        shared_layers = nn.ModuleList()\n        for i in range(n_glu_shared):\n            if i == 0:\n                shared_layers.append(nn.Linear(step_dim, 2 * step_dim, bias=False))\n            else:\n                shared_layers.append(nn.Linear(step_dim, 2 * step_dim, bias=False))\n\n        self.decoder = nn.ModuleList()\n        for step in range(n_steps):\n            transformer = FeatTransformer(\n                step_dim,\n                step_dim,\n                dropout,\n                shared_layers,\n                n_glu_step_dependent,\n                ghost_bn,\n                virtual_batch_size,\n                momentum=momentum,\n            )\n            self.decoder.append(transformer)\n\n        self.reconstruction_layer = nn.Linear(step_dim, embed_dim, bias=False)\n        initialize_non_glu(self.reconstruction_layer, step_dim, embed_dim)\n\n    def forward(self, X: List[Tensor]) -&gt; Tensor:\n        out = torch.tensor(0.0)\n        for i, x in enumerate(X):\n            x = self.decoder[i](x)\n            out = torch.add(out, x)\n        out = self.reconstruction_layer(out)\n        return out\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.mlp.context_attention_mlp.ContextAttentionMLP","title":"ContextAttentionMLP","text":"<p>               Bases: <code>BaseTabularModelWithAttention</code></p> <p>Defines a <code>ContextAttentionMLP</code> model that can be used as the <code>deeptabular</code> component of a Wide &amp; Deep model or independently by itself.</p> <p>This class combines embedding representations of the categorical features with numerical (aka continuous) features that are also embedded. These are then passed through a series of attention blocks. Each attention block is comprised by a <code>ContextAttentionEncoder</code>. Such encoder is in part inspired by the attention mechanism described in Hierarchical Attention Networks for Document Classification. See <code>pytorch_widedeep.models.tabular.mlp._attention_layers</code> for details.</p> <p>Most of the parameters for this class are <code>Optional</code> since the use of categorical or continuous is in fact optional (i.e. one can use categorical features only, continuous features only or both).</p> <p>Parameters:</p> Name Type Description Default <code>column_idx</code> <code>Dict[str, int]</code> <p>Dict containing the index of the columns that will be passed through the <code>TabMlp</code> model. Required to slice the tensors. e.g. {'education': 0, 'relationship': 1, 'workclass': 2, ...}.</p> required <code>cat_embed_input</code> <code>Optional[List[Tuple[str, int]]]</code> <p>List of Tuples with the column name and number of unique values and embedding dimension. e.g. [(education, 11), ...]</p> <code>None</code> <code>cat_embed_dropout</code> <code>Optional[float]</code> <p>Categorical embeddings dropout. If <code>None</code>, it will default to 0.</p> <code>None</code> <code>use_cat_bias</code> <code>Optional[bool]</code> <p>Boolean indicating if bias will be used for the categorical embeddings. If <code>None</code>, it will default to 'False'.</p> <code>None</code> <code>cat_embed_activation</code> <code>Optional[str]</code> <p>Activation function for the categorical embeddings, if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> <code>None</code> <code>shared_embed</code> <code>Optional[bool]</code> <p>Boolean indicating if the embeddings will be \"shared\". The idea behind <code>shared_embed</code> is described in the Appendix A in the TabTransformer paper: 'The goal of having column embedding is to enable the model to distinguish the classes in one column from those in the other columns'. In other words, the idea is to let the model learn which column is embedded at the time. See: <code>pytorch_widedeep.models.transformers._layers.SharedEmbeddings</code>.</p> <code>None</code> <code>add_shared_embed</code> <code>Optional[bool]</code> <p>The two embedding sharing strategies are: 1) add the shared embeddings to the column embeddings or 2) to replace the first <code>frac_shared_embed</code> with the shared embeddings. See <code>pytorch_widedeep.models.embeddings_layers.SharedEmbeddings</code> If 'None' is passed, it will default to 'False'.</p> <code>None</code> <code>frac_shared_embed</code> <code>Optional[float]</code> <p>The fraction of embeddings that will be shared (if <code>add_shared_embed = False</code>) by all the different categories for one particular column. If 'None' is passed, it will default to 0.0.</p> <code>None</code> <code>continuous_cols</code> <code>Optional[List[str]]</code> <p>List with the name of the numeric (aka continuous) columns</p> <code>None</code> <code>cont_norm_layer</code> <code>Optional[Literal[batchnorm, layernorm]]</code> <p>Type of normalization layer applied to the continuous features. Options are: 'layernorm' and 'batchnorm'. if <code>None</code>, no normalization layer will be used.</p> <code>None</code> <code>embed_continuous_method</code> <code>Optional[Literal[standard, piecewise, periodic]]</code> <p>Method to use to embed the continuous features. Options are: 'standard', 'periodic' or 'piecewise'. The 'standard' embedding method is based on the FT-Transformer implementation presented in the paper: Revisiting Deep Learning Models for Tabular Data. The 'periodic' and_'piecewise'_ methods were presented in the paper: On Embeddings for Numerical Features in Tabular Deep Learning. Please, read the papers for details.</p> <code>'standard'</code> <code>cont_embed_dropout</code> <code>Optional[float]</code> <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p> <code>None</code> <code>cont_embed_activation</code> <code>Optional[str]</code> <p>Activation function for the continuous embeddings if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported. If <code>None</code>, no activation function will be applied.</p> <code>None</code> <code>quantization_setup</code> <code>Optional[Dict[str, List[float]]]</code> <p>This parameter is used when the 'piecewise' method is used to embed the continuous cols. It is a dict where keys are the name of the continuous columns and values are lists with the boundaries for the quantization of the continuous_cols. See the examples for details. If If the 'piecewise' method is used, this parameter is required.</p> <code>None</code> <code>n_frequencies</code> <code>Optional[int]</code> <p>This is the so called 'k' in their paper On Embeddings for Numerical Features in Tabular Deep Learning, and is the number of 'frequencies' that will be used to represent each continuous column. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>sigma</code> <code>Optional[float]</code> <p>This is the sigma parameter in the paper mentioned when describing the previous parameters and it is used to initialise the 'frequency weights'. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>share_last_layer</code> <code>Optional[bool]</code> <p>This parameter is not present in the before mentioned paper but it is implemented in the official repo. If <code>True</code> the linear layer that turns the frequencies into embeddings will be shared across the continuous columns. If <code>False</code> a different linear layer will be used for each continuous column. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>full_embed_dropout</code> <code>Optional[bool]</code> <p>If <code>True</code>, the full embedding corresponding to a column will be masked out/dropout. If <code>None</code>, it will default to <code>False</code>.</p> <code>None</code> <code>input_dim</code> <code>int</code> <p>The so-called dimension of the model. Is the number of embeddings used to encode the categorical and/or continuous columns</p> <code>32</code> <code>attn_dropout</code> <code>float</code> <p>Dropout for each attention block</p> <code>0.2</code> <code>with_addnorm</code> <code>bool</code> <p>Boolean indicating if residual connections will be used in the attention blocks</p> <code>False</code> <code>attn_activation</code> <code>str</code> <p>String indicating the activation function to be applied to the dense layer in each attention encoder. 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported.</p> <code>'leaky_relu'</code> <code>n_blocks</code> <code>int</code> <p>Number of attention blocks</p> <code>3</code> <p>Attributes:</p> Name Type Description <code>encoder</code> <code>Module</code> <p>Sequence of attention encoders.</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import ContextAttentionMLP\n&gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n&gt;&gt;&gt; colnames = ['a', 'b', 'c', 'd', 'e']\n&gt;&gt;&gt; cat_embed_input = [(u,i,j) for u,i,j in zip(colnames[:4], [4]*4, [8]*4)]\n&gt;&gt;&gt; column_idx = {k:v for v,k in enumerate(colnames)}\n&gt;&gt;&gt; model = ContextAttentionMLP(column_idx=column_idx, cat_embed_input=cat_embed_input, continuous_cols = ['e'])\n&gt;&gt;&gt; out = model(X_tab)\n</code></pre> Source code in <code>pytorch_widedeep/models/tabular/mlp/context_attention_mlp.py</code> <pre><code>class ContextAttentionMLP(BaseTabularModelWithAttention):\n    r\"\"\"Defines a `ContextAttentionMLP` model that can be used as the\n    `deeptabular` component of a Wide &amp; Deep model or independently by\n    itself.\n\n    This class combines embedding representations of the categorical features\n    with numerical (aka continuous) features that are also embedded. These\n    are then passed through a series of attention blocks. Each attention\n    block is comprised by a `ContextAttentionEncoder`. Such encoder is in\n    part inspired by the attention mechanism described in\n    [Hierarchical Attention Networks for Document\n    Classification](https://www.cs.cmu.edu/~./hovy/papers/16HLT-hierarchical-attention-networks.pdf).\n    See `pytorch_widedeep.models.tabular.mlp._attention_layers` for details.\n\n    Most of the parameters for this class are `Optional` since the use of\n    categorical or continuous is in fact optional (i.e. one can use\n    categorical features only, continuous features only or both).\n\n    Parameters\n    ----------\n    column_idx: Dict\n        Dict containing the index of the columns that will be passed through\n        the `TabMlp` model. Required to slice the tensors. e.g. _{'education':\n        0, 'relationship': 1, 'workclass': 2, ...}_.\n    cat_embed_input: List, Optional, default = None\n        List of Tuples with the column name and number of unique values and\n        embedding dimension. e.g. _[(education, 11), ...]_\n    cat_embed_dropout: float, Optional, default = None\n        Categorical embeddings dropout. If `None`, it will default\n        to 0.\n    use_cat_bias: bool, Optional, default = None,\n        Boolean indicating if bias will be used for the categorical embeddings.\n        If `None`, it will default to 'False'.\n    cat_embed_activation: Optional, str, default = None,\n        Activation function for the categorical embeddings, if any. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported\n    shared_embed: bool, Optional, default = None\n        Boolean indicating if the embeddings will be \"shared\". The idea behind `shared_embed` is\n        described in the Appendix A in the [TabTransformer paper](https://arxiv.org/abs/2012.06678):\n        _'The goal of having column embedding is to enable the model to\n        distinguish the classes in one column from those in the other\n        columns'_. In other words, the idea is to let the model learn which\n        column is embedded at the time. See: `pytorch_widedeep.models.transformers._layers.SharedEmbeddings`.\n    add_shared_embed: bool, Optional, default = None\n        The two embedding sharing strategies are: 1) add the shared embeddings\n        to the column embeddings or 2) to replace the first\n        `frac_shared_embed` with the shared embeddings.\n        See `pytorch_widedeep.models.embeddings_layers.SharedEmbeddings`\n        If 'None' is passed, it will default to 'False'.\n    frac_shared_embed: float, Optional, default = None\n        The fraction of embeddings that will be shared (if `add_shared_embed\n        = False`) by all the different categories for one particular\n        column. If 'None' is passed, it will default to 0.0.\n    continuous_cols: List, Optional, default = None\n        List with the name of the numeric (aka continuous) columns\n    cont_norm_layer: str, Optional, default =  None\n        Type of normalization layer applied to the continuous features.\n        Options are: _'layernorm'_ and _'batchnorm'_. if `None`, no\n        normalization layer will be used.\n    embed_continuous_method: Optional, str, default = None,\n        Method to use to embed the continuous features. Options are:\n        _'standard'_, _'periodic'_ or _'piecewise'_. The _'standard'_\n        embedding method is based on the FT-Transformer implementation\n        presented in the paper: [Revisiting Deep Learning Models for\n        Tabular Data](https://arxiv.org/abs/2106.11959v5). The _'periodic'_\n        and_'piecewise'_ methods were presented in the paper: [On Embeddings for\n        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556).\n        Please, read the papers for details.\n    cont_embed_dropout: float, Optional, default = None,\n        Dropout for the continuous embeddings. If `None`, it will default to 0.0\n    cont_embed_activation: Optional, str, default = None,\n        Activation function for the continuous embeddings if any. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported.\n        If `None`, no activation function will be applied.\n    quantization_setup: Dict[str, List[float]], Optional, default = None,\n        This parameter is used when the _'piecewise'_ method is used to embed\n        the continuous cols. It is a dict where keys are the name of the continuous\n        columns and values are lists with the boundaries for the quantization\n        of the continuous_cols. See the examples for details. If\n        If the _'piecewise'_ method is used, this parameter is required.\n    n_frequencies: int, Optional, default = None,\n        This is the so called _'k'_ in their paper [On Embeddings for\n        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556),\n        and is the number of 'frequencies' that will be used to represent each\n        continuous column. See their Eq 2 in the paper for details. If\n        the _'periodic'_ method is used, this parameter is required.\n    sigma: float, Optional, default = None,\n        This is the sigma parameter in the paper mentioned when describing the\n        previous parameters and it is used to initialise the 'frequency\n        weights'. See their Eq 2 in the paper for details. If\n        the _'periodic'_ method is used, this parameter is required.\n    share_last_layer: bool, Optional, default = None,\n        This parameter is not present in the before mentioned paper but it is implemented in\n        the [official repo](https://github.com/yandex-research/rtdl-num-embeddings/tree/main).\n        If `True` the linear layer that turns the frequencies into embeddings\n        will be shared across the continuous columns. If `False` a different\n        linear layer will be used for each continuous column.\n        If the _'periodic'_ method is used, this parameter is required.\n    full_embed_dropout: bool, Optional, default = None,\n        If `True`, the full embedding corresponding to a column will be masked\n        out/dropout. If `None`, it will default to `False`.\n    input_dim: int, default = 32\n        The so-called *dimension of the model*. Is the number of embeddings\n        used to encode the categorical and/or continuous columns\n    attn_dropout: float, default = 0.2\n        Dropout for each attention block\n    with_addnorm: bool = False,\n        Boolean indicating if residual connections will be used in the\n        attention blocks\n    attn_activation: str, default = \"leaky_relu\"\n        String indicating the activation function to be applied to the dense\n        layer in each attention encoder. _'tanh'_, _'relu'_, _'leaky_relu'_\n        and _'gelu'_ are supported.\n    n_blocks: int, default = 3\n        Number of attention blocks\n\n    Attributes\n    ----------\n    encoder: nn.Module\n        Sequence of attention encoders.\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.models import ContextAttentionMLP\n    &gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n    &gt;&gt;&gt; colnames = ['a', 'b', 'c', 'd', 'e']\n    &gt;&gt;&gt; cat_embed_input = [(u,i,j) for u,i,j in zip(colnames[:4], [4]*4, [8]*4)]\n    &gt;&gt;&gt; column_idx = {k:v for v,k in enumerate(colnames)}\n    &gt;&gt;&gt; model = ContextAttentionMLP(column_idx=column_idx, cat_embed_input=cat_embed_input, continuous_cols = ['e'])\n    &gt;&gt;&gt; out = model(X_tab)\n    \"\"\"\n\n    def __init__(\n        self,\n        column_idx: Dict[str, int],\n        *,\n        cat_embed_input: Optional[List[Tuple[str, int]]] = None,\n        cat_embed_dropout: Optional[float] = None,\n        use_cat_bias: Optional[bool] = None,\n        cat_embed_activation: Optional[str] = None,\n        shared_embed: Optional[bool] = None,\n        add_shared_embed: Optional[bool] = None,\n        frac_shared_embed: Optional[float] = None,\n        continuous_cols: Optional[List[str]] = None,\n        cont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\n        embed_continuous_method: Optional[\n            Literal[\"standard\", \"piecewise\", \"periodic\"]\n        ] = \"standard\",\n        cont_embed_dropout: Optional[float] = None,\n        cont_embed_activation: Optional[str] = None,\n        quantization_setup: Optional[Dict[str, List[float]]] = None,\n        n_frequencies: Optional[int] = None,\n        sigma: Optional[float] = None,\n        share_last_layer: Optional[bool] = None,\n        full_embed_dropout: Optional[bool] = None,\n        input_dim: int = 32,\n        attn_dropout: float = 0.2,\n        with_addnorm: bool = False,\n        attn_activation: str = \"leaky_relu\",\n        n_blocks: int = 3,\n    ):\n        super(ContextAttentionMLP, self).__init__(\n            column_idx=column_idx,\n            cat_embed_input=cat_embed_input,\n            cat_embed_dropout=cat_embed_dropout,\n            use_cat_bias=use_cat_bias,\n            cat_embed_activation=cat_embed_activation,\n            shared_embed=shared_embed,\n            add_shared_embed=add_shared_embed,\n            frac_shared_embed=frac_shared_embed,\n            continuous_cols=continuous_cols,\n            cont_norm_layer=cont_norm_layer,\n            embed_continuous_method=embed_continuous_method,\n            cont_embed_dropout=cont_embed_dropout,\n            cont_embed_activation=cont_embed_activation,\n            input_dim=input_dim,\n            quantization_setup=quantization_setup,\n            n_frequencies=n_frequencies,\n            sigma=sigma,\n            share_last_layer=share_last_layer,\n            full_embed_dropout=full_embed_dropout,\n        )\n\n        self.attn_dropout = attn_dropout\n        self.with_addnorm = with_addnorm\n        self.attn_activation = attn_activation\n        self.n_blocks = n_blocks\n\n        self.with_cls_token = \"cls_token\" in column_idx\n        self.n_cat = len(cat_embed_input) if cat_embed_input is not None else 0\n        self.n_cont = len(continuous_cols) if continuous_cols is not None else 0\n\n        # Embeddings are instantiated at the base model\n        # Attention Blocks\n        self.encoder = nn.Sequential()\n        for i in range(n_blocks):\n            self.encoder.add_module(\n                \"attention_block\" + str(i),\n                ContextAttentionEncoder(\n                    input_dim,\n                    attn_dropout,\n                    with_addnorm,\n                    attn_activation,\n                ),\n            )\n\n    def forward(self, X: Tensor) -&gt; Tensor:\n        x = self._get_embeddings(X)\n        x = self.encoder(x)\n        if self.with_cls_token:\n            out = x[:, 0, :]\n        else:\n            out = x.flatten(1)\n        return out\n\n    @property\n    def output_dim(self) -&gt; int:\n        r\"\"\"The output dimension of the model. This is a required property\n        neccesary to build the `WideDeep` class\n        \"\"\"\n        return (\n            self.input_dim\n            if self.with_cls_token\n            else ((self.n_cat + self.n_cont) * self.input_dim)\n        )\n\n    @property\n    def attention_weights(self) -&gt; List[Tensor]:\n        r\"\"\"List with the attention weights per block\n\n        The shape of the attention weights is $(N, F)$, where $N$ is the batch\n        size and $F$ is the number of features/columns in the dataset\n        \"\"\"\n        return [blk.attn.attn_weights for blk in self.encoder]\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.mlp.context_attention_mlp.ContextAttentionMLP.output_dim","title":"output_dim  <code>property</code>","text":"<pre><code>output_dim\n</code></pre> <p>The output dimension of the model. This is a required property neccesary to build the <code>WideDeep</code> class</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.mlp.context_attention_mlp.ContextAttentionMLP.attention_weights","title":"attention_weights  <code>property</code>","text":"<pre><code>attention_weights\n</code></pre> <p>List with the attention weights per block</p> <p>The shape of the attention weights is \\((N, F)\\), where \\(N\\) is the batch size and \\(F\\) is the number of features/columns in the dataset</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.mlp.self_attention_mlp.SelfAttentionMLP","title":"SelfAttentionMLP","text":"<p>               Bases: <code>BaseTabularModelWithAttention</code></p> <p>Defines a <code>SelfAttentionMLP</code> model that can be used as the deeptabular component of a Wide &amp; Deep model or independently by itself.</p> <p>This class combines embedding representations of the categorical features with numerical (aka continuous) features that are also embedded. These are then passed through a series of attention blocks. Each attention block is comprised by what we would refer as a simplified <code>SelfAttentionEncoder</code>. See <code>pytorch_widedeep.models.tabular.mlp._attention_layers</code> for details. The reason to use a simplified version of self attention is because we observed that the 'standard' attention mechanism used in the TabTransformer has a notable tendency to overfit.</p> <p>In more detail, this model only uses Q and K (and not V). If we think about it as in terms of text (and intuitively), the Softmax(QK^T) is the attention mechanism that tells us how much, at each position in the input sentence, each word is represented or 'expressed'. We refer to that as 'attention weights'. These attention weighst are normally multiplied by a Value matrix to further strength the focus on the words that each word should be attending to (again, intuitively).</p> <p>In this implementation we skip this last multiplication and instead we multiply the attention weights directly by the input tensor. This is a simplification that we expect is beneficial in terms of avoiding overfitting for tabular data.</p> <p>Most of the parameters for this class are <code>Optional</code> since the use of categorical or continuous is in fact optional (i.e. one can use categorical features only, continuous features only or both).</p> <p>Parameters:</p> Name Type Description Default <code>column_idx</code> <code>Dict[str, int]</code> <p>Dict containing the index of the columns that will be passed through the <code>TabMlp</code> model. Required to slice the tensors. e.g. {'education': 0, 'relationship': 1, 'workclass': 2, ...}.</p> required <code>cat_embed_input</code> <code>Optional[List[Tuple[str, int]]]</code> <p>List of Tuples with the column name and number of unique values and embedding dimension. e.g. [(education, 11), ...]</p> <code>None</code> <code>cat_embed_dropout</code> <code>Optional[float]</code> <p>Categorical embeddings dropout. If <code>None</code>, it will default to 0.</p> <code>None</code> <code>use_cat_bias</code> <code>Optional[bool]</code> <p>Boolean indicating if bias will be used for the categorical embeddings. If <code>None</code>, it will default to 'False'.</p> <code>None</code> <code>cat_embed_activation</code> <code>Optional[str]</code> <p>Activation function for the categorical embeddings, if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> <code>None</code> <code>shared_embed</code> <code>Optional[bool]</code> <p>Boolean indicating if the embeddings will be \"shared\". The idea behind <code>shared_embed</code> is described in the Appendix A in the TabTransformer paper: 'The goal of having column embedding is to enable the model to distinguish the classes in one column from those in the other columns'. In other words, the idea is to let the model learn which column is embedded at the time. See: <code>pytorch_widedeep.models.transformers._layers.SharedEmbeddings</code>.</p> <code>None</code> <code>add_shared_embed</code> <code>Optional[bool]</code> <p>The two embedding sharing strategies are: 1) add the shared embeddings to the column embeddings or 2) to replace the first <code>frac_shared_embed</code> with the shared embeddings. See <code>pytorch_widedeep.models.embeddings_layers.SharedEmbeddings</code> If 'None' is passed, it will default to 'False'.</p> <code>None</code> <code>frac_shared_embed</code> <code>Optional[float]</code> <p>The fraction of embeddings that will be shared (if <code>add_shared_embed = False</code>) by all the different categories for one particular column. If 'None' is passed, it will default to 0.0.</p> <code>None</code> <code>continuous_cols</code> <code>Optional[List[str]]</code> <p>List with the name of the numeric (aka continuous) columns</p> <code>None</code> <code>cont_norm_layer</code> <code>Optional[Literal[batchnorm, layernorm]]</code> <p>Type of normalization layer applied to the continuous features. Options are: 'layernorm' and 'batchnorm'. if <code>None</code>, no normalization layer will be used.</p> <code>None</code> <code>embed_continuous_method</code> <code>Optional[Literal[standard, piecewise, periodic]]</code> <p>Method to use to embed the continuous features. Options are: 'standard', 'periodic' or 'piecewise'. The 'standard' embedding method is based on the FT-Transformer implementation presented in the paper: Revisiting Deep Learning Models for Tabular Data. The 'periodic' and_'piecewise'_ methods were presented in the paper: On Embeddings for Numerical Features in Tabular Deep Learning. Please, read the papers for details.</p> <code>'standard'</code> <code>cont_embed_dropout</code> <code>Optional[float]</code> <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p> <code>None</code> <code>cont_embed_activation</code> <code>Optional[str]</code> <p>Activation function for the continuous embeddings if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported. If <code>None</code>, no activation function will be applied.</p> <code>None</code> <code>quantization_setup</code> <code>Optional[Dict[str, List[float]]]</code> <p>This parameter is used when the 'piecewise' method is used to embed the continuous cols. It is a dict where keys are the name of the continuous columns and values are lists with the boundaries for the quantization of the continuous_cols. See the examples for details. If If the 'piecewise' method is used, this parameter is required.</p> <code>None</code> <code>n_frequencies</code> <code>Optional[int]</code> <p>This is the so called 'k' in their paper On Embeddings for Numerical Features in Tabular Deep Learning, and is the number of 'frequencies' that will be used to represent each continuous column. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>sigma</code> <code>Optional[float]</code> <p>This is the sigma parameter in the paper mentioned when describing the previous parameters and it is used to initialise the 'frequency weights'. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>share_last_layer</code> <code>Optional[bool]</code> <p>This parameter is not present in the before mentioned paper but it is implemented in the official repo. If <code>True</code> the linear layer that turns the frequencies into embeddings will be shared across the continuous columns. If <code>False</code> a different linear layer will be used for each continuous column. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>full_embed_dropout</code> <code>Optional[bool]</code> <p>If <code>True</code>, the full embedding corresponding to a column will be masked out/dropout. If <code>None</code>, it will default to <code>False</code>.</p> <code>None</code> <code>input_dim</code> <code>int</code> <p>The so-called dimension of the model. Is the number of embeddings used to encode the categorical and/or continuous columns</p> <code>32</code> <code>attn_dropout</code> <code>float</code> <p>Dropout for each attention block</p> <code>0.2</code> <code>n_heads</code> <code>int</code> <p>Number of attention heads per attention block.</p> <code>8</code> <code>use_bias</code> <code>bool</code> <p>Boolean indicating whether or not to use bias in the Q, K projection layers.</p> <code>False</code> <code>with_addnorm</code> <code>bool</code> <p>Boolean indicating if residual connections will be used in the attention blocks</p> <code>False</code> <code>attn_activation</code> <code>str</code> <p>String indicating the activation function to be applied to the dense layer in each attention encoder. 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported.</p> <code>'leaky_relu'</code> <code>n_blocks</code> <code>int</code> <p>Number of attention blocks</p> <code>3</code> <p>Attributes:</p> Name Type Description <code>cat_and_cont_embed</code> <code>Module</code> <p>This is the module that processes the categorical and continuous columns</p> <code>encoder</code> <code>Module</code> <p>Sequence of attention encoders.</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import SelfAttentionMLP\n&gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n&gt;&gt;&gt; colnames = ['a', 'b', 'c', 'd', 'e']\n&gt;&gt;&gt; cat_embed_input = [(u,i,j) for u,i,j in zip(colnames[:4], [4]*4, [8]*4)]\n&gt;&gt;&gt; column_idx = {k:v for v,k in enumerate(colnames)}\n&gt;&gt;&gt; model = SelfAttentionMLP(column_idx=column_idx, cat_embed_input=cat_embed_input, continuous_cols = ['e'])\n&gt;&gt;&gt; out = model(X_tab)\n</code></pre> Source code in <code>pytorch_widedeep/models/tabular/mlp/self_attention_mlp.py</code> <pre><code>class SelfAttentionMLP(BaseTabularModelWithAttention):\n    r\"\"\"Defines a `SelfAttentionMLP` model that can be used as the\n    deeptabular component of a Wide &amp; Deep model or independently by\n    itself.\n\n    This class combines embedding representations of the categorical features\n    with numerical (aka continuous) features that are also embedded. These\n    are then passed through a series of attention blocks. Each attention\n    block is comprised by what we would refer as a simplified\n    `SelfAttentionEncoder`. See\n    `pytorch_widedeep.models.tabular.mlp._attention_layers` for details. The\n    reason to use a simplified version of self attention is because we\n    observed that the '_standard_' attention mechanism used in the\n    TabTransformer has a notable tendency to overfit.\n\n    In more detail, this model only uses Q and K (and not V). If we think\n    about it as in terms of text (and intuitively), the Softmax(QK^T) is the\n    attention mechanism that tells us how much, at each position in the input\n    sentence, each word is represented or 'expressed'. We refer to that\n    as 'attention weights'. These attention weighst are normally multiplied\n    by a Value matrix to further strength the focus on the words that each\n    word should be attending to (again, intuitively).\n\n    In this implementation we skip this last multiplication and instead we\n    multiply the attention weights directly by the input tensor. This is a\n    simplification that we expect is beneficial in terms of avoiding\n    overfitting for tabular data.\n\n    Most of the parameters for this class are `Optional` since the use of\n    categorical or continuous is in fact optional (i.e. one can use\n    categorical features only, continuous features only or both).\n\n    Parameters\n    ----------\n    column_idx: Dict\n        Dict containing the index of the columns that will be passed through\n        the `TabMlp` model. Required to slice the tensors. e.g. _{'education':\n        0, 'relationship': 1, 'workclass': 2, ...}_.\n    cat_embed_input: List, Optional, default = None\n        List of Tuples with the column name and number of unique values and\n        embedding dimension. e.g. _[(education, 11), ...]_\n    cat_embed_dropout: float, Optional, default = None\n        Categorical embeddings dropout. If `None`, it will default\n        to 0.\n    use_cat_bias: bool, Optional, default = None,\n        Boolean indicating if bias will be used for the categorical embeddings.\n        If `None`, it will default to 'False'.\n    cat_embed_activation: Optional, str, default = None,\n        Activation function for the categorical embeddings, if any. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported\n    shared_embed: bool, Optional, default = None\n        Boolean indicating if the embeddings will be \"shared\". The idea behind `shared_embed` is\n        described in the Appendix A in the [TabTransformer paper](https://arxiv.org/abs/2012.06678):\n        _'The goal of having column embedding is to enable the model to\n        distinguish the classes in one column from those in the other\n        columns'_. In other words, the idea is to let the model learn which\n        column is embedded at the time. See: `pytorch_widedeep.models.transformers._layers.SharedEmbeddings`.\n    add_shared_embed: bool, Optional, default = None\n        The two embedding sharing strategies are: 1) add the shared embeddings\n        to the column embeddings or 2) to replace the first\n        `frac_shared_embed` with the shared embeddings.\n        See `pytorch_widedeep.models.embeddings_layers.SharedEmbeddings`\n        If 'None' is passed, it will default to 'False'.\n    frac_shared_embed: float, Optional, default = None\n        The fraction of embeddings that will be shared (if `add_shared_embed\n        = False`) by all the different categories for one particular\n        column. If 'None' is passed, it will default to 0.0.\n    continuous_cols: List, Optional, default = None\n        List with the name of the numeric (aka continuous) columns\n    cont_norm_layer: str, Optional, default =  None\n        Type of normalization layer applied to the continuous features.\n        Options are: _'layernorm'_ and _'batchnorm'_. if `None`, no\n        normalization layer will be used.\n    embed_continuous_method: Optional, str, default = None,\n        Method to use to embed the continuous features. Options are:\n        _'standard'_, _'periodic'_ or _'piecewise'_. The _'standard'_\n        embedding method is based on the FT-Transformer implementation\n        presented in the paper: [Revisiting Deep Learning Models for\n        Tabular Data](https://arxiv.org/abs/2106.11959v5). The _'periodic'_\n        and_'piecewise'_ methods were presented in the paper: [On Embeddings for\n        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556).\n        Please, read the papers for details.\n    cont_embed_dropout: float, Optional, default = None,\n        Dropout for the continuous embeddings. If `None`, it will default to 0.0\n    cont_embed_activation: Optional, str, default = None,\n        Activation function for the continuous embeddings if any. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported.\n        If `None`, no activation function will be applied.\n    quantization_setup: Dict[str, List[float]], Optional, default = None,\n        This parameter is used when the _'piecewise'_ method is used to embed\n        the continuous cols. It is a dict where keys are the name of the continuous\n        columns and values are lists with the boundaries for the quantization\n        of the continuous_cols. See the examples for details. If\n        If the _'piecewise'_ method is used, this parameter is required.\n    n_frequencies: int, Optional, default = None,\n        This is the so called _'k'_ in their paper [On Embeddings for\n        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556),\n        and is the number of 'frequencies' that will be used to represent each\n        continuous column. See their Eq 2 in the paper for details. If\n        the _'periodic'_ method is used, this parameter is required.\n    sigma: float, Optional, default = None,\n        This is the sigma parameter in the paper mentioned when describing the\n        previous parameters and it is used to initialise the 'frequency\n        weights'. See their Eq 2 in the paper for details. If\n        the _'periodic'_ method is used, this parameter is required.\n    share_last_layer: bool, Optional, default = None,\n        This parameter is not present in the before mentioned paper but it is implemented in\n        the [official repo](https://github.com/yandex-research/rtdl-num-embeddings/tree/main).\n        If `True` the linear layer that turns the frequencies into embeddings\n        will be shared across the continuous columns. If `False` a different\n        linear layer will be used for each continuous column.\n        If the _'periodic'_ method is used, this parameter is required.\n    full_embed_dropout: bool, Optional, default = None,\n        If `True`, the full embedding corresponding to a column will be masked\n        out/dropout. If `None`, it will default to `False`.\n    input_dim: int, default = 32\n        The so-called *dimension of the model*. Is the number of\n        embeddings used to encode the categorical and/or continuous columns\n    attn_dropout: float, default = 0.2\n        Dropout for each attention block\n    n_heads: int, default = 8\n        Number of attention heads per attention block.\n    use_bias: bool, default = False\n        Boolean indicating whether or not to use bias in the Q, K projection\n        layers.\n    with_addnorm: bool = False,\n        Boolean indicating if residual connections will be used in the attention blocks\n    attn_activation: str, default = \"leaky_relu\"\n        String indicating the activation function to be applied to the dense\n        layer in each attention encoder. _'tanh'_, _'relu'_, _'leaky_relu'_\n        and _'gelu'_ are supported.\n    n_blocks: int, default = 3\n        Number of attention blocks\n\n    Attributes\n    ----------\n    cat_and_cont_embed: nn.Module\n        This is the module that processes the categorical and continuous columns\n    encoder: nn.Module\n        Sequence of attention encoders.\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.models import SelfAttentionMLP\n    &gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n    &gt;&gt;&gt; colnames = ['a', 'b', 'c', 'd', 'e']\n    &gt;&gt;&gt; cat_embed_input = [(u,i,j) for u,i,j in zip(colnames[:4], [4]*4, [8]*4)]\n    &gt;&gt;&gt; column_idx = {k:v for v,k in enumerate(colnames)}\n    &gt;&gt;&gt; model = SelfAttentionMLP(column_idx=column_idx, cat_embed_input=cat_embed_input, continuous_cols = ['e'])\n    &gt;&gt;&gt; out = model(X_tab)\n    \"\"\"\n\n    def __init__(\n        self,\n        column_idx: Dict[str, int],\n        *,\n        cat_embed_input: Optional[List[Tuple[str, int]]] = None,\n        cat_embed_dropout: Optional[float] = None,\n        use_cat_bias: Optional[bool] = None,\n        cat_embed_activation: Optional[str] = None,\n        shared_embed: Optional[bool] = None,\n        add_shared_embed: Optional[bool] = None,\n        frac_shared_embed: Optional[float] = None,\n        continuous_cols: Optional[List[str]] = None,\n        cont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\n        embed_continuous_method: Optional[\n            Literal[\"standard\", \"piecewise\", \"periodic\"]\n        ] = \"standard\",\n        cont_embed_dropout: Optional[float] = None,\n        cont_embed_activation: Optional[str] = None,\n        quantization_setup: Optional[Dict[str, List[float]]] = None,\n        n_frequencies: Optional[int] = None,\n        sigma: Optional[float] = None,\n        share_last_layer: Optional[bool] = None,\n        full_embed_dropout: Optional[bool] = None,\n        input_dim: int = 32,\n        attn_dropout: float = 0.2,\n        n_heads: int = 8,\n        use_bias: bool = False,\n        with_addnorm: bool = False,\n        attn_activation: str = \"leaky_relu\",\n        n_blocks: int = 3,\n    ):\n        super(SelfAttentionMLP, self).__init__(\n            column_idx=column_idx,\n            input_dim=input_dim,\n            cat_embed_input=cat_embed_input,\n            cat_embed_dropout=cat_embed_dropout,\n            use_cat_bias=use_cat_bias,\n            cat_embed_activation=cat_embed_activation,\n            shared_embed=shared_embed,\n            add_shared_embed=add_shared_embed,\n            frac_shared_embed=frac_shared_embed,\n            continuous_cols=continuous_cols,\n            cont_norm_layer=cont_norm_layer,\n            embed_continuous_method=embed_continuous_method,\n            cont_embed_dropout=cont_embed_dropout,\n            cont_embed_activation=cont_embed_activation,\n            quantization_setup=quantization_setup,\n            n_frequencies=n_frequencies,\n            sigma=sigma,\n            share_last_layer=share_last_layer,\n            full_embed_dropout=full_embed_dropout,\n        )\n\n        self.attn_dropout = attn_dropout\n        self.n_heads = n_heads\n        self.use_bias = use_bias\n        self.with_addnorm = with_addnorm\n        self.attn_activation = attn_activation\n        self.n_blocks = n_blocks\n\n        self.with_cls_token = \"cls_token\" in column_idx\n        self.n_cat = len(cat_embed_input) if cat_embed_input is not None else 0\n        self.n_cont = len(continuous_cols) if continuous_cols is not None else 0\n\n        # Embeddings are instantiated at the base model\n        # Attention Blocks\n        self.encoder = nn.Sequential()\n        for i in range(n_blocks):\n            self.encoder.add_module(\n                \"attention_block\" + str(i),\n                SelfAttentionEncoder(\n                    input_dim,\n                    attn_dropout,\n                    use_bias,\n                    n_heads,\n                    with_addnorm,\n                    attn_activation,\n                ),\n            )\n\n    def forward(self, X: Tensor) -&gt; Tensor:\n        x = self._get_embeddings(X)\n        x = self.encoder(x)\n        if self.with_cls_token:\n            out = x[:, 0, :]\n        else:\n            out = x.flatten(1)\n        return out\n\n    @property\n    def output_dim(self) -&gt; int:\n        r\"\"\"The output dimension of the model. This is a required property\n        neccesary to build the WideDeep class\n        \"\"\"\n        return (\n            self.input_dim\n            if self.with_cls_token\n            else ((self.n_cat + self.n_cont) * self.input_dim)\n        )\n\n    @property\n    def attention_weights(self) -&gt; List[Tensor]:\n        r\"\"\"List with the attention weights per block\n\n        The shape of the attention weights is $(N, H, F, F)$, where $N$ is the\n        batch size, $H$ is the number of attention heads and $F$ is the\n        number of features/columns in the dataset\n        \"\"\"\n        return [blk.attn.attn_weights for blk in self.encoder]\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.mlp.self_attention_mlp.SelfAttentionMLP.output_dim","title":"output_dim  <code>property</code>","text":"<pre><code>output_dim\n</code></pre> <p>The output dimension of the model. This is a required property neccesary to build the WideDeep class</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.mlp.self_attention_mlp.SelfAttentionMLP.attention_weights","title":"attention_weights  <code>property</code>","text":"<pre><code>attention_weights\n</code></pre> <p>List with the attention weights per block</p> <p>The shape of the attention weights is \\((N, H, F, F)\\), where \\(N\\) is the batch size, \\(H\\) is the number of attention heads and \\(F\\) is the number of features/columns in the dataset</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.transformers.tab_transformer.TabTransformer","title":"TabTransformer","text":"<p>               Bases: <code>BaseTabularModelWithAttention</code></p> <p>Defines our adptation of the TabTransformer model that can be used as the <code>deeptabular</code> component of a Wide &amp; Deep model or independently by itself.</p> <p>Most of the parameters for this class are <code>Optional</code> since the use of categorical or continuous is in fact optional (i.e. one can use categorical features only, continuous features only or both).</p> <p> NOTE: This is an enhanced adaptation of the model described in the paper. It can be considered as the flagship of our transformer family of models for tabular data and offers mutiple, additional features relative to the original publication(and some other models in the library)</p> <p>Parameters:</p> Name Type Description Default <code>column_idx</code> <code>Dict[str, int]</code> <p>Dict containing the index of the columns that will be passed through the <code>TabMlp</code> model. Required to slice the tensors. e.g. {'education': 0, 'relationship': 1, 'workclass': 2, ...}.</p> required <code>cat_embed_input</code> <code>Optional[List[Tuple[str, int]]]</code> <p>List of Tuples with the column name and number of unique values and embedding dimension. e.g. [(education, 11), ...]</p> <code>None</code> <code>cat_embed_dropout</code> <code>Optional[float]</code> <p>Categorical embeddings dropout. If <code>None</code>, it will default to 0.</p> <code>None</code> <code>use_cat_bias</code> <code>Optional[bool]</code> <p>Boolean indicating if bias will be used for the categorical embeddings. If <code>None</code>, it will default to 'False'.</p> <code>None</code> <code>cat_embed_activation</code> <code>Optional[str]</code> <p>Activation function for the categorical embeddings, if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> <code>None</code> <code>shared_embed</code> <code>Optional[bool]</code> <p>Boolean indicating if the embeddings will be \"shared\". The idea behind <code>shared_embed</code> is described in the Appendix A in the TabTransformer paper: 'The goal of having column embedding is to enable the model to distinguish the classes in one column from those in the other columns'. In other words, the idea is to let the model learn which column is embedded at the time. See: <code>pytorch_widedeep.models.transformers._layers.SharedEmbeddings</code>.</p> <code>None</code> <code>add_shared_embed</code> <code>Optional[bool]</code> <p>The two embedding sharing strategies are: 1) add the shared embeddings to the column embeddings or 2) to replace the first <code>frac_shared_embed</code> with the shared embeddings. See <code>pytorch_widedeep.models.embeddings_layers.SharedEmbeddings</code> If 'None' is passed, it will default to 'False'.</p> <code>None</code> <code>frac_shared_embed</code> <code>Optional[float]</code> <p>The fraction of embeddings that will be shared (if <code>add_shared_embed = False</code>) by all the different categories for one particular column. If 'None' is passed, it will default to 0.0.</p> <code>None</code> <code>continuous_cols</code> <code>Optional[List[str]]</code> <p>List with the name of the numeric (aka continuous) columns</p> <code>None</code> <code>cont_norm_layer</code> <code>Optional[Literal[batchnorm, layernorm]]</code> <p>Type of normalization layer applied to the continuous features. Options are: 'layernorm' and 'batchnorm'. if <code>None</code>, no normalization layer will be used.</p> <code>None</code> <code>embed_continuous_method</code> <code>Optional[Literal[standard, piecewise, periodic]]</code> <p>Method to use to embed the continuous features. Options are: 'standard', 'periodic' or 'piecewise'. The 'standard' embedding method is based on the FT-Transformer implementation presented in the paper: Revisiting Deep Learning Models for Tabular Data. The 'periodic' and_'piecewise'_ methods were presented in the paper: On Embeddings for Numerical Features in Tabular Deep Learning. Please, read the papers for details.</p> <code>None</code> <code>cont_embed_dropout</code> <code>Optional[float]</code> <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p> <code>None</code> <code>cont_embed_activation</code> <code>Optional[str]</code> <p>Activation function for the continuous embeddings if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported. If <code>None</code>, no activation function will be applied.</p> <code>None</code> <code>quantization_setup</code> <code>Optional[Dict[str, List[float]]]</code> <p>This parameter is used when the 'piecewise' method is used to embed the continuous cols. It is a dict where keys are the name of the continuous columns and values are lists with the boundaries for the quantization of the continuous_cols. See the examples for details. If If the 'piecewise' method is used, this parameter is required.</p> <code>None</code> <code>n_frequencies</code> <code>Optional[int]</code> <p>This is the so called 'k' in their paper On Embeddings for Numerical Features in Tabular Deep Learning, and is the number of 'frequencies' that will be used to represent each continuous column. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>sigma</code> <code>Optional[float]</code> <p>This is the sigma parameter in the paper mentioned when describing the previous parameters and it is used to initialise the 'frequency weights'. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>share_last_layer</code> <code>Optional[bool]</code> <p>This parameter is not present in the before mentioned paper but it is implemented in the official repo. If <code>True</code> the linear layer that turns the frequencies into embeddings will be shared across the continuous columns. If <code>False</code> a different linear layer will be used for each continuous column. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>full_embed_dropout</code> <code>Optional[bool]</code> <p>If <code>True</code>, the full embedding corresponding to a column will be masked out/dropout. If <code>None</code>, it will default to <code>False</code>.</p> <code>None</code> <code>input_dim</code> <code>int</code> <p>The so-called dimension of the model. Is the number of embeddings used to encode the categorical and/or continuous columns</p> <code>32</code> <code>n_heads</code> <code>int</code> <p>Number of attention heads per Transformer block</p> <code>8</code> <code>use_qkv_bias</code> <code>bool</code> <p>Boolean indicating whether or not to use bias in the Q, K, and V projection layers.</p> <code>False</code> <code>n_blocks</code> <code>int</code> <p>Number of Transformer blocks</p> <code>4</code> <code>attn_dropout</code> <code>float</code> <p>Dropout that will be applied to the Multi-Head Attention layers</p> <code>0.2</code> <code>ff_dropout</code> <code>float</code> <p>Dropout that will be applied to the FeedForward network</p> <code>0.1</code> <code>ff_factor</code> <code>int</code> <p>Multiplicative factor applied to the first layer of the FF network in each Transformer block, This is normally set to 4.</p> <code>4</code> <code>transformer_activation</code> <code>str</code> <p>Transformer Encoder activation function. 'tanh', 'relu', 'leaky_relu', 'gelu', 'geglu' and 'reglu' are supported</p> <code>'gelu'</code> <code>use_linear_attention</code> <code>bool</code> <p>Boolean indicating if Linear Attention (from Transformers are RNNs: Fast Autoregressive Transformers with Linear Attention) will be used. The inclusing of this mode of attention is inspired by this post, where the Uber team finds that this attention mechanism leads to the best results for their tabular data.</p> <code>False</code> <code>use_flash_attention</code> <code>bool</code> <p>Boolean indicating if Flash Attention will be used. </p> <code>False</code> <code>mlp_hidden_dims</code> <code>Optional[List[int]]</code> <p>List with the number of neurons per dense layer in the MLP. e.g: [64, 32]. If not provided no MLP on top of the final Transformer block will be used.</p> <code>None</code> <code>mlp_activation</code> <code>str</code> <p>Activation function for the dense layers of the MLP. Currently 'tanh', 'relu', 'leaky'_relu' and _'gelu' are supported. If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to 'relu'.</p> <code>'relu'</code> <code>mlp_dropout</code> <code>float</code> <p>float with the dropout between the dense layers of the MLP. If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to 0.0.</p> <code>0.1</code> <code>mlp_batchnorm</code> <code>bool</code> <p>Boolean indicating whether or not batch normalization will be applied to the dense layers If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to False.</p> <code>False</code> <code>mlp_batchnorm_last</code> <code>bool</code> <p>Boolean indicating whether or not batch normalization will be applied to the last of the dense layers If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to False.</p> <code>False</code> <code>mlp_linear_first</code> <code>bool</code> <p>Boolean indicating the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code> If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to <code>True</code>.</p> <code>True</code> <p>Attributes:</p> Name Type Description <code>encoder</code> <code>Module</code> <p>Sequence of Transformer blocks</p> <code>mlp</code> <code>Module</code> <p>MLP component in the model</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import TabTransformer\n&gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n&gt;&gt;&gt; colnames = ['a', 'b', 'c', 'd', 'e']\n&gt;&gt;&gt; cat_embed_input = [(u,i) for u,i in zip(colnames[:4], [4]*4)]\n&gt;&gt;&gt; continuous_cols = ['e']\n&gt;&gt;&gt; column_idx = {k:v for v,k in enumerate(colnames)}\n&gt;&gt;&gt; model = TabTransformer(column_idx=column_idx, cat_embed_input=cat_embed_input, continuous_cols=continuous_cols)\n&gt;&gt;&gt; out = model(X_tab)\n</code></pre> Source code in <code>pytorch_widedeep/models/tabular/transformers/tab_transformer.py</code> <pre><code>class TabTransformer(BaseTabularModelWithAttention):\n    r\"\"\"Defines our adptation of the\n    [TabTransformer model](https://arxiv.org/abs/2012.06678)\n    that can be used as the `deeptabular` component of a\n    Wide &amp; Deep model or independently by itself.\n\n    Most of the parameters for this class are `Optional` since the use of\n    categorical or continuous is in fact optional (i.e. one can use\n    categorical features only, continuous features only or both).\n\n    :information_source: **NOTE**:\n    This is an enhanced adaptation of the model described in the paper. It can\n    be considered as the flagship of our transformer family of models for\n    tabular data and offers mutiple, additional features relative to the\n    original publication(and some other models in the library)\n\n    Parameters\n    ----------\n    column_idx: Dict\n        Dict containing the index of the columns that will be passed through\n        the `TabMlp` model. Required to slice the tensors. e.g. _{'education':\n        0, 'relationship': 1, 'workclass': 2, ...}_.\n    cat_embed_input: List, Optional, default = None\n        List of Tuples with the column name and number of unique values and\n        embedding dimension. e.g. _[(education, 11), ...]_\n    cat_embed_dropout: float, Optional, default = None\n        Categorical embeddings dropout. If `None`, it will default\n        to 0.\n    use_cat_bias: bool, Optional, default = None,\n        Boolean indicating if bias will be used for the categorical embeddings.\n        If `None`, it will default to 'False'.\n    cat_embed_activation: Optional, str, default = None,\n        Activation function for the categorical embeddings, if any. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported\n    shared_embed: bool, Optional, default = None\n        Boolean indicating if the embeddings will be \"shared\". The idea behind `shared_embed` is\n        described in the Appendix A in the [TabTransformer paper](https://arxiv.org/abs/2012.06678):\n        _'The goal of having column embedding is to enable the model to\n        distinguish the classes in one column from those in the other\n        columns'_. In other words, the idea is to let the model learn which\n        column is embedded at the time. See: `pytorch_widedeep.models.transformers._layers.SharedEmbeddings`.\n    add_shared_embed: bool, Optional, default = None\n        The two embedding sharing strategies are: 1) add the shared embeddings\n        to the column embeddings or 2) to replace the first\n        `frac_shared_embed` with the shared embeddings.\n        See `pytorch_widedeep.models.embeddings_layers.SharedEmbeddings`\n        If 'None' is passed, it will default to 'False'.\n    frac_shared_embed: float, Optional, default = None\n        The fraction of embeddings that will be shared (if `add_shared_embed\n        = False`) by all the different categories for one particular\n        column. If 'None' is passed, it will default to 0.0.\n    continuous_cols: List, Optional, default = None\n        List with the name of the numeric (aka continuous) columns\n    cont_norm_layer: str, Optional, default =  None\n        Type of normalization layer applied to the continuous features.\n        Options are: _'layernorm'_ and _'batchnorm'_. if `None`, no\n        normalization layer will be used.\n    embed_continuous_method: Optional, str, default = None,\n        Method to use to embed the continuous features. Options are:\n        _'standard'_, _'periodic'_ or _'piecewise'_. The _'standard'_\n        embedding method is based on the FT-Transformer implementation\n        presented in the paper: [Revisiting Deep Learning Models for\n        Tabular Data](https://arxiv.org/abs/2106.11959v5). The _'periodic'_\n        and_'piecewise'_ methods were presented in the paper: [On Embeddings for\n        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556).\n        Please, read the papers for details.\n    cont_embed_dropout: float, Optional, default = None,\n        Dropout for the continuous embeddings. If `None`, it will default to 0.0\n    cont_embed_activation: Optional, str, default = None,\n        Activation function for the continuous embeddings if any. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported.\n        If `None`, no activation function will be applied.\n    quantization_setup: Dict[str, List[float]], Optional, default = None,\n        This parameter is used when the _'piecewise'_ method is used to embed\n        the continuous cols. It is a dict where keys are the name of the continuous\n        columns and values are lists with the boundaries for the quantization\n        of the continuous_cols. See the examples for details. If\n        If the _'piecewise'_ method is used, this parameter is required.\n    n_frequencies: int, Optional, default = None,\n        This is the so called _'k'_ in their paper [On Embeddings for\n        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556),\n        and is the number of 'frequencies' that will be used to represent each\n        continuous column. See their Eq 2 in the paper for details. If\n        the _'periodic'_ method is used, this parameter is required.\n    sigma: float, Optional, default = None,\n        This is the sigma parameter in the paper mentioned when describing the\n        previous parameters and it is used to initialise the 'frequency\n        weights'. See their Eq 2 in the paper for details. If\n        the _'periodic'_ method is used, this parameter is required.\n    share_last_layer: bool, Optional, default = None,\n        This parameter is not present in the before mentioned paper but it is implemented in\n        the [official repo](https://github.com/yandex-research/rtdl-num-embeddings/tree/main).\n        If `True` the linear layer that turns the frequencies into embeddings\n        will be shared across the continuous columns. If `False` a different\n        linear layer will be used for each continuous column.\n        If the _'periodic'_ method is used, this parameter is required.\n    full_embed_dropout: bool, Optional, default = None,\n        If `True`, the full embedding corresponding to a column will be masked\n        out/dropout. If `None`, it will default to `False`.\n    input_dim: int, default = 32\n        The so-called *dimension of the model*. Is the number of\n        embeddings used to encode the categorical and/or continuous columns\n    n_heads: int, default = 8\n        Number of attention heads per Transformer block\n    use_qkv_bias: bool, default = False\n        Boolean indicating whether or not to use bias in the Q, K, and V\n        projection layers.\n    n_blocks: int, default = 4\n        Number of Transformer blocks\n    attn_dropout: float, default = 0.2\n        Dropout that will be applied to the Multi-Head Attention layers\n    ff_dropout: float, default = 0.1\n        Dropout that will be applied to the FeedForward network\n    ff_factor: float, default = 4\n        Multiplicative factor applied to the first layer of the FF network in\n        each Transformer block, This is normally set to 4.\n    transformer_activation: str, default = \"gelu\"\n        Transformer Encoder activation function. _'tanh'_, _'relu'_,\n        _'leaky_relu'_, _'gelu'_, _'geglu'_ and _'reglu'_ are supported\n    use_linear_attention: Boolean, default = False,\n        Boolean indicating if Linear Attention (from [Transformers are RNNs:\n        Fast Autoregressive Transformers with Linear Attention](https://arxiv.org/abs/2006.16236))\n        will be used. The inclusing of this mode of attention is inspired by\n        [this post](https://www.uber.com/en-GB/blog/deepeta-how-uber-predicts-arrival-times/),\n        where the Uber team finds that this attention mechanism leads to the\n        best results for their tabular data.\n    use_flash_attention: Boolean, default = False,\n        Boolean indicating if\n        [Flash Attention](https://pytorch.org/docs/stable/generated/torch.nn.functional.scaled_dot_product_attention.html)\n        will be used. &lt;br/&gt;\n    mlp_hidden_dims: List, Optional, default = None\n        List with the number of neurons per dense layer in the MLP. e.g:\n        _[64, 32]_. If not provided no MLP on top of the final\n        Transformer block will be used.\n    mlp_activation: str, Optional, default = None\n        Activation function for the dense layers of the MLP. Currently\n        _'tanh'_, _'relu'_, _'leaky'_relu' and _'gelu'_ are supported.\n        If 'mlp_hidden_dims' is not `None` and this parameter is `None`, it\n        will default to _'relu'_.\n    mlp_dropout: float, Optional, default = None\n        float with the dropout between the dense layers of the MLP.\n        If 'mlp_hidden_dims' is not `None` and this parameter is `None`, it\n        will default to 0.0.\n    mlp_batchnorm: bool, Optional, default = None\n        Boolean indicating whether or not batch normalization will be applied\n        to the dense layers\n        If 'mlp_hidden_dims' is not `None` and this parameter is `None`, it\n        will default to False.\n    mlp_batchnorm_last: bool, Optional, default = None\n        Boolean indicating whether or not batch normalization will be applied\n        to the last of the dense layers\n        If 'mlp_hidden_dims' is not `None` and this parameter is `None`, it\n        will default to False.\n    mlp_linear_first: bool, Optional, default = None\n        Boolean indicating the order of the operations in the dense\n        layer. If `True: [LIN -&gt; ACT -&gt; BN -&gt; DP]`. If `False: [BN -&gt; DP -&gt;\n        LIN -&gt; ACT]`\n        If 'mlp_hidden_dims' is not `None` and this parameter is `None`, it\n        will default to `True`.\n\n    Attributes\n    ----------\n    encoder: nn.Module\n        Sequence of Transformer blocks\n    mlp: nn.Module\n        MLP component in the model\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.models import TabTransformer\n    &gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n    &gt;&gt;&gt; colnames = ['a', 'b', 'c', 'd', 'e']\n    &gt;&gt;&gt; cat_embed_input = [(u,i) for u,i in zip(colnames[:4], [4]*4)]\n    &gt;&gt;&gt; continuous_cols = ['e']\n    &gt;&gt;&gt; column_idx = {k:v for v,k in enumerate(colnames)}\n    &gt;&gt;&gt; model = TabTransformer(column_idx=column_idx, cat_embed_input=cat_embed_input, continuous_cols=continuous_cols)\n    &gt;&gt;&gt; out = model(X_tab)\n    \"\"\"\n\n    def __init__(\n        self,\n        column_idx: Dict[str, int],\n        *,\n        cat_embed_input: Optional[List[Tuple[str, int]]] = None,\n        cat_embed_dropout: Optional[float] = None,\n        use_cat_bias: Optional[bool] = None,\n        cat_embed_activation: Optional[str] = None,\n        shared_embed: Optional[bool] = None,\n        add_shared_embed: Optional[bool] = None,\n        frac_shared_embed: Optional[float] = None,\n        continuous_cols: Optional[List[str]] = None,\n        cont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\n        embed_continuous: Optional[bool] = None,\n        embed_continuous_method: Optional[\n            Literal[\"standard\", \"piecewise\", \"periodic\"]\n        ] = None,\n        cont_embed_dropout: Optional[float] = None,\n        cont_embed_activation: Optional[str] = None,\n        quantization_setup: Optional[Dict[str, List[float]]] = None,\n        n_frequencies: Optional[int] = None,\n        sigma: Optional[float] = None,\n        share_last_layer: Optional[bool] = None,\n        full_embed_dropout: Optional[bool] = None,\n        input_dim: int = 32,\n        n_heads: int = 8,\n        use_qkv_bias: bool = False,\n        n_blocks: int = 4,\n        attn_dropout: float = 0.2,\n        ff_dropout: float = 0.1,\n        ff_factor: int = 4,\n        transformer_activation: str = \"gelu\",\n        use_linear_attention: bool = False,\n        use_flash_attention: bool = False,\n        mlp_hidden_dims: Optional[List[int]] = None,\n        mlp_activation: str = \"relu\",\n        mlp_dropout: float = 0.1,\n        mlp_batchnorm: bool = False,\n        mlp_batchnorm_last: bool = False,\n        mlp_linear_first: bool = True,\n    ):\n        super(TabTransformer, self).__init__(\n            column_idx=column_idx,\n            cat_embed_input=cat_embed_input,\n            cat_embed_dropout=cat_embed_dropout,\n            use_cat_bias=use_cat_bias,\n            cat_embed_activation=cat_embed_activation,\n            shared_embed=shared_embed,\n            add_shared_embed=add_shared_embed,\n            frac_shared_embed=frac_shared_embed,\n            continuous_cols=continuous_cols,\n            cont_norm_layer=cont_norm_layer,\n            # UGLY hack to be able to use the base class __init__ method\n            embed_continuous_method=(\n                \"standard\"\n                if embed_continuous_method is None\n                else embed_continuous_method\n            ),\n            cont_embed_dropout=cont_embed_dropout,\n            cont_embed_activation=cont_embed_activation,\n            quantization_setup=quantization_setup,\n            n_frequencies=n_frequencies,\n            sigma=sigma,\n            share_last_layer=share_last_layer,\n            input_dim=input_dim,\n            full_embed_dropout=full_embed_dropout,\n        )\n\n        # we overwrite the embed_continuous_method parameter that was set\n        # to 'standard' if embed_continuous_method is None. In other words,\n        # this will already have the right value if the user provided a value\n        # for embed_continuous_method, otherwise will be \"standard\" when it\n        # should be None\n        self.embed_continuous_method = embed_continuous_method\n\n        if embed_continuous is not None:\n            self.embed_continuous = embed_continuous\n            if embed_continuous and embed_continuous_method is None:\n                raise ValueError(\n                    \"If 'embed_continuous' is True, 'embed_continuous_method' must be \"\n                    \"one of 'standard', 'piecewise' or 'periodic'.\"\n                )\n        elif self.embed_continuous_method is not None:\n            self.embed_continuous = True\n        else:\n            self.embed_continuous = False\n\n        self.n_heads = n_heads\n        self.use_qkv_bias = use_qkv_bias\n        self.n_blocks = n_blocks\n        self.attn_dropout = attn_dropout\n        self.ff_dropout = ff_dropout\n        self.transformer_activation = transformer_activation\n        self.use_linear_attention = use_linear_attention\n        self.use_flash_attention = use_flash_attention\n        self.ff_factor = ff_factor\n\n        self.mlp_hidden_dims = mlp_hidden_dims\n        self.mlp_activation = mlp_activation\n        self.mlp_dropout = mlp_dropout\n        self.mlp_batchnorm = mlp_batchnorm\n        self.mlp_batchnorm_last = mlp_batchnorm_last\n        self.mlp_linear_first = mlp_linear_first\n\n        self.with_cls_token = \"cls_token\" in column_idx\n        self.n_cat = len(cat_embed_input) if cat_embed_input is not None else 0\n        self.n_cont = len(continuous_cols) if continuous_cols is not None else 0\n\n        if self.n_cont and not self.n_cat and not self.embed_continuous:\n            raise ValueError(\n                \"If only continuous features are used 'embed_continuous' must be set to 'True'\"\n            )\n\n        # Embeddings are instantiated at the base model\n        # Transformer blocks\n        self.encoder = nn.Sequential()\n        for i in range(n_blocks):\n            self.encoder.add_module(\n                \"transformer_block\" + str(i),\n                TransformerEncoder(\n                    input_dim,\n                    n_heads,\n                    use_qkv_bias,\n                    attn_dropout,\n                    ff_dropout,\n                    ff_factor,\n                    transformer_activation,\n                    use_linear_attention,\n                    use_flash_attention,\n                ),\n            )\n\n        self.mlp_first_hidden_dim = self._mlp_first_hidden_dim()\n\n        if self.mlp_hidden_dims is not None:\n            self.mlp = MLP(\n                d_hidden=[self.mlp_first_hidden_dim] + self.mlp_hidden_dims,\n                activation=(\n                    \"relu\" if self.mlp_activation is None else self.mlp_activation\n                ),\n                dropout=0.0 if self.mlp_dropout is None else self.mlp_dropout,\n                batchnorm=False if self.mlp_batchnorm is None else self.mlp_batchnorm,\n                batchnorm_last=(\n                    False\n                    if self.mlp_batchnorm_last is None\n                    else self.mlp_batchnorm_last\n                ),\n                linear_first=(\n                    False if self.mlp_linear_first is None else self.mlp_linear_first\n                ),\n            )\n        else:\n            self.mlp = None\n\n    def forward(self, X: Tensor) -&gt; Tensor:\n        x_embed, x_cont_not_embed = self._get_embeddings_tt(X)\n        x = self.encoder(x_embed)\n\n        if self.with_cls_token:\n            x = x[:, 0, :]\n        else:\n            x = x.flatten(1)\n\n        if x_cont_not_embed is not None:\n            x = torch.cat([x, x_cont_not_embed], 1)\n\n        if self.mlp is not None:\n            x = self.mlp(x)\n        return x\n\n    def _mlp_first_hidden_dim(self) -&gt; int:\n        if self.with_cls_token:\n            if self.embed_continuous:\n                attn_output_dim = self.input_dim\n            else:\n                attn_output_dim = self.input_dim + self.n_cont\n        elif self.embed_continuous:\n            attn_output_dim = (self.n_cat + self.n_cont) * self.input_dim\n        else:\n            attn_output_dim = self.n_cat * self.input_dim + self.n_cont\n\n        return attn_output_dim\n\n    @property\n    def output_dim(self) -&gt; int:\n        r\"\"\"The output dimension of the model. This is a required property\n        neccesary to build the `WideDeep` class\n        \"\"\"\n        return (\n            self.mlp_hidden_dims[-1]\n            if self.mlp_hidden_dims is not None\n            else self.mlp_first_hidden_dim\n        )\n\n    @property\n    def attention_weights(self) -&gt; List[Tensor]:\n        r\"\"\"List with the attention weights per block\n\n        The shape of the attention weights is $(N, H, F, F)$, where $N$ is the\n        batch size, $H$ is the number of attention heads and $F$ is the\n        number of features/columns in the dataset\n\n        :information_source: **NOTE**:\n        if flash attention or linear attention\n        are used, no attention weights are saved during the training process\n        and calling this property will throw a ValueError\n        \"\"\"\n        if self.use_flash_attention or self.use_linear_attention:\n            raise ValueError(\n                \"Extraction of the attention weights is not supported for \"\n                \"linear or flash attention\"\n            )\n\n        return [blk.attn.attn_weights for blk in self.encoder]\n\n    def _get_embeddings_tt(self, X: Tensor) -&gt; Tuple[Tensor, Optional[Tensor]]:  # type: ignore[override]\n        if self.n_cont and self.embed_continuous and self.n_cat:\n            # there are continuous and categorical features and the continuous\n            # features are embedded\n            x_embed = self._get_embeddings(X)\n            x_cont_not_embed = None\n        elif self.n_cont and self.embed_continuous:\n            # there are only continuous features and they are embedded\n            x_embed = self.cont_norm((X[:, self.cont_idx].float()))\n            x_embed = self.cont_embed(x_embed)\n            x_cont_not_embed = None\n        elif self.n_cont and self.n_cat:\n            # there are continuous and categorical features but the continuous\n            # features are not embedded\n            x_embed = self.cat_embed(X)\n            x_cont_not_embed = self.cont_norm((X[:, self.cont_idx].float()))\n        elif self.n_cat:\n            # there are only categorical features\n            x_embed = self.cat_embed(X)\n            x_cont_not_embed = None\n        else:\n            raise ValueError(\n                \"Using only continuous features that are not embedded is \"\n                \"is not supported in this implementation. Please set 'embed_continuous_method' \"\n                \"to one of ['standard', 'piecewise', 'periodic']\"\n            )\n        return x_embed, x_cont_not_embed\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.transformers.tab_transformer.TabTransformer.output_dim","title":"output_dim  <code>property</code>","text":"<pre><code>output_dim\n</code></pre> <p>The output dimension of the model. This is a required property neccesary to build the <code>WideDeep</code> class</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.transformers.tab_transformer.TabTransformer.attention_weights","title":"attention_weights  <code>property</code>","text":"<pre><code>attention_weights\n</code></pre> <p>List with the attention weights per block</p> <p>The shape of the attention weights is \\((N, H, F, F)\\), where \\(N\\) is the batch size, \\(H\\) is the number of attention heads and \\(F\\) is the number of features/columns in the dataset</p> <p> NOTE: if flash attention or linear attention are used, no attention weights are saved during the training process and calling this property will throw a ValueError</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.transformers.saint.SAINT","title":"SAINT","text":"<p>               Bases: <code>BaseTabularModelWithAttention</code></p> <p>Defines a SAINT model that can be used as the <code>deeptabular</code> component of a Wide &amp; Deep model or independently by itself.</p> <p>Most of the parameters for this class are <code>Optional</code> since the use of categorical or continuous is in fact optional (i.e. one can use categorical features only, continuous features only or both).</p> <p> NOTE: This is an slightly modified and enhanced  version of the model described in the paper,</p> <p>Parameters:</p> Name Type Description Default <code>column_idx</code> <code>Dict[str, int]</code> <p>Dict containing the index of the columns that will be passed through the <code>TabMlp</code> model. Required to slice the tensors. e.g. {'education': 0, 'relationship': 1, 'workclass': 2, ...}.</p> required <code>cat_embed_input</code> <code>Optional[List[Tuple[str, int]]]</code> <p>List of Tuples with the column name and number of unique values and embedding dimension. e.g. [(education, 11), ...]</p> <code>None</code> <code>cat_embed_dropout</code> <code>Optional[float]</code> <p>Categorical embeddings dropout. If <code>None</code>, it will default to 0.</p> <code>None</code> <code>use_cat_bias</code> <code>Optional[bool]</code> <p>Boolean indicating if bias will be used for the categorical embeddings. If <code>None</code>, it will default to 'False'.</p> <code>None</code> <code>cat_embed_activation</code> <code>Optional[str]</code> <p>Activation function for the categorical embeddings, if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> <code>None</code> <code>shared_embed</code> <code>Optional[bool]</code> <p>Boolean indicating if the embeddings will be \"shared\". The idea behind <code>shared_embed</code> is described in the Appendix A in the TabTransformer paper: 'The goal of having column embedding is to enable the model to distinguish the classes in one column from those in the other columns'. In other words, the idea is to let the model learn which column is embedded at the time. See: <code>pytorch_widedeep.models.transformers._layers.SharedEmbeddings</code>.</p> <code>None</code> <code>add_shared_embed</code> <code>Optional[bool]</code> <p>The two embedding sharing strategies are: 1) add the shared embeddings to the column embeddings or 2) to replace the first <code>frac_shared_embed</code> with the shared embeddings. See <code>pytorch_widedeep.models.embeddings_layers.SharedEmbeddings</code> If 'None' is passed, it will default to 'False'.</p> <code>None</code> <code>frac_shared_embed</code> <code>Optional[float]</code> <p>The fraction of embeddings that will be shared (if <code>add_shared_embed = False</code>) by all the different categories for one particular column. If 'None' is passed, it will default to 0.0.</p> <code>None</code> <code>continuous_cols</code> <code>Optional[List[str]]</code> <p>List with the name of the numeric (aka continuous) columns</p> <code>None</code> <code>cont_norm_layer</code> <code>Optional[Literal[batchnorm, layernorm]]</code> <p>Type of normalization layer applied to the continuous features. Options are: 'layernorm' and 'batchnorm'. if <code>None</code>, no normalization layer will be used.</p> <code>None</code> <code>embed_continuous_method</code> <code>Optional[Literal[standard, piecewise, periodic]]</code> <p>Method to use to embed the continuous features. Options are: 'standard', 'periodic' or 'piecewise'. The 'standard' embedding method is based on the FT-Transformer implementation presented in the paper: Revisiting Deep Learning Models for Tabular Data. The 'periodic' and_'piecewise'_ methods were presented in the paper: On Embeddings for Numerical Features in Tabular Deep Learning. Please, read the papers for details.</p> <code>'standard'</code> <code>cont_embed_dropout</code> <code>Optional[float]</code> <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p> <code>None</code> <code>cont_embed_activation</code> <code>Optional[str]</code> <p>Activation function for the continuous embeddings if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported. If <code>None</code>, no activation function will be applied.</p> <code>None</code> <code>quantization_setup</code> <code>Optional[Dict[str, List[float]]]</code> <p>This parameter is used when the 'piecewise' method is used to embed the continuous cols. It is a dict where keys are the name of the continuous columns and values are lists with the boundaries for the quantization of the continuous_cols. See the examples for details. If If the 'piecewise' method is used, this parameter is required.</p> <code>None</code> <code>n_frequencies</code> <code>Optional[int]</code> <p>This is the so called 'k' in their paper On Embeddings for Numerical Features in Tabular Deep Learning, and is the number of 'frequencies' that will be used to represent each continuous column. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>sigma</code> <code>Optional[float]</code> <p>This is the sigma parameter in the paper mentioned when describing the previous parameters and it is used to initialise the 'frequency weights'. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>share_last_layer</code> <code>Optional[bool]</code> <p>This parameter is not present in the before mentioned paper but it is implemented in the official repo. If <code>True</code> the linear layer that turns the frequencies into embeddings will be shared across the continuous columns. If <code>False</code> a different linear layer will be used for each continuous column. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>full_embed_dropout</code> <code>Optional[bool]</code> <p>If <code>True</code>, the full embedding corresponding to a column will be masked out/dropout. If <code>None</code>, it will default to <code>False</code>.</p> <code>None</code> <code>input_dim</code> <code>int</code> <p>The so-called dimension of the model. Is the number of embeddings used to encode the categorical and/or continuous columns</p> <code>32</code> <code>n_heads</code> <code>int</code> <p>Number of attention heads per Transformer block</p> <code>8</code> <code>use_qkv_bias</code> <code>bool</code> <p>Boolean indicating whether or not to use bias in the Q, K, and V projection layers</p> <code>False</code> <code>n_blocks</code> <code>int</code> <p>Number of SAINT-Transformer blocks.</p> <code>2</code> <code>attn_dropout</code> <code>float</code> <p>Dropout that will be applied to the Multi-Head Attention column and row layers</p> <code>0.1</code> <code>ff_dropout</code> <code>float</code> <p>Dropout that will be applied to the FeedForward network</p> <code>0.2</code> <code>ff_factor</code> <code>int</code> <p>Multiplicative factor applied to the first layer of the FF network in each Transformer block, This is normally set to 4.</p> <code>4</code> <code>transformer_activation</code> <code>str</code> <p>Transformer Encoder activation function. 'tanh', 'relu', 'leaky_relu', 'gelu', 'geglu' and 'reglu' are supported</p> <code>'gelu'</code> <code>mlp_hidden_dims</code> <code>Optional[List[int]]</code> <p>List with the number of neurons per dense layer in the MLP. e.g: [64, 32]. If not provided no MLP on top of the final Transformer block will be used.</p> <code>None</code> <code>mlp_activation</code> <code>Optional[str]</code> <p>Activation function for the dense layers of the MLP. Currently 'tanh', 'relu', 'leaky'_relu' and _'gelu' are supported. If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to 'relu'.</p> <code>None</code> <code>mlp_dropout</code> <code>Optional[float]</code> <p>float with the dropout between the dense layers of the MLP. If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to 0.0.</p> <code>None</code> <code>mlp_batchnorm</code> <code>Optional[bool]</code> <p>Boolean indicating whether or not batch normalization will be applied to the dense layers If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to False.</p> <code>None</code> <code>mlp_batchnorm_last</code> <code>Optional[bool]</code> <p>Boolean indicating whether or not batch normalization will be applied to the last of the dense layers If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to False.</p> <code>None</code> <code>mlp_linear_first</code> <code>Optional[bool]</code> <p>Boolean indicating the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code> If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to <code>True</code>.</p> <code>None</code> <p>Attributes:</p> Name Type Description <code>encoder</code> <code>Module</code> <p>Sequence of SAINT-Transformer blocks</p> <code>mlp</code> <code>Module</code> <p>MLP component in the model</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import SAINT\n&gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n&gt;&gt;&gt; colnames = ['a', 'b', 'c', 'd', 'e']\n&gt;&gt;&gt; cat_embed_input = [(u,i) for u,i in zip(colnames[:4], [4]*4)]\n&gt;&gt;&gt; continuous_cols = ['e']\n&gt;&gt;&gt; column_idx = {k:v for v,k in enumerate(colnames)}\n&gt;&gt;&gt; model = SAINT(column_idx=column_idx, cat_embed_input=cat_embed_input, continuous_cols=continuous_cols)\n&gt;&gt;&gt; out = model(X_tab)\n</code></pre> Source code in <code>pytorch_widedeep/models/tabular/transformers/saint.py</code> <pre><code>class SAINT(BaseTabularModelWithAttention):\n    r\"\"\"Defines a [SAINT model](https://arxiv.org/abs/2106.01342) that\n    can be used as the `deeptabular` component of a Wide &amp; Deep model or\n    independently by itself.\n\n    Most of the parameters for this class are `Optional` since the use of\n    categorical or continuous is in fact optional (i.e. one can use\n    categorical features only, continuous features only or both).\n\n    :information_source: **NOTE**: This is an slightly modified and enhanced\n     version of the model described in the paper,\n\n    Parameters\n    ----------\n    column_idx: Dict\n        Dict containing the index of the columns that will be passed through\n        the `TabMlp` model. Required to slice the tensors. e.g. _{'education':\n        0, 'relationship': 1, 'workclass': 2, ...}_.\n    cat_embed_input: List, Optional, default = None\n        List of Tuples with the column name and number of unique values and\n        embedding dimension. e.g. _[(education, 11), ...]_\n    cat_embed_dropout: float, Optional, default = None\n        Categorical embeddings dropout. If `None`, it will default\n        to 0.\n    use_cat_bias: bool, Optional, default = None,\n        Boolean indicating if bias will be used for the categorical embeddings.\n        If `None`, it will default to 'False'.\n    cat_embed_activation: Optional, str, default = None,\n        Activation function for the categorical embeddings, if any. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported\n    shared_embed: bool, Optional, default = None\n        Boolean indicating if the embeddings will be \"shared\". The idea behind `shared_embed` is\n        described in the Appendix A in the [TabTransformer paper](https://arxiv.org/abs/2012.06678):\n        _'The goal of having column embedding is to enable the model to\n        distinguish the classes in one column from those in the other\n        columns'_. In other words, the idea is to let the model learn which\n        column is embedded at the time. See: `pytorch_widedeep.models.transformers._layers.SharedEmbeddings`.\n    add_shared_embed: bool, Optional, default = None\n        The two embedding sharing strategies are: 1) add the shared embeddings\n        to the column embeddings or 2) to replace the first\n        `frac_shared_embed` with the shared embeddings.\n        See `pytorch_widedeep.models.embeddings_layers.SharedEmbeddings`\n        If 'None' is passed, it will default to 'False'.\n    frac_shared_embed: float, Optional, default = None\n        The fraction of embeddings that will be shared (if `add_shared_embed\n        = False`) by all the different categories for one particular\n        column. If 'None' is passed, it will default to 0.0.\n    continuous_cols: List, Optional, default = None\n        List with the name of the numeric (aka continuous) columns\n    cont_norm_layer: str, Optional, default =  None\n        Type of normalization layer applied to the continuous features.\n        Options are: _'layernorm'_ and _'batchnorm'_. if `None`, no\n        normalization layer will be used.\n    embed_continuous_method: Optional, str, default = None,\n        Method to use to embed the continuous features. Options are:\n        _'standard'_, _'periodic'_ or _'piecewise'_. The _'standard'_\n        embedding method is based on the FT-Transformer implementation\n        presented in the paper: [Revisiting Deep Learning Models for\n        Tabular Data](https://arxiv.org/abs/2106.11959v5). The _'periodic'_\n        and_'piecewise'_ methods were presented in the paper: [On Embeddings for\n        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556).\n        Please, read the papers for details.\n    cont_embed_dropout: float, Optional, default = None,\n        Dropout for the continuous embeddings. If `None`, it will default to 0.0\n    cont_embed_activation: Optional, str, default = None,\n        Activation function for the continuous embeddings if any. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported.\n        If `None`, no activation function will be applied.\n    quantization_setup: Dict[str, List[float]], Optional, default = None,\n        This parameter is used when the _'piecewise'_ method is used to embed\n        the continuous cols. It is a dict where keys are the name of the continuous\n        columns and values are lists with the boundaries for the quantization\n        of the continuous_cols. See the examples for details. If\n        If the _'piecewise'_ method is used, this parameter is required.\n    n_frequencies: int, Optional, default = None,\n        This is the so called _'k'_ in their paper [On Embeddings for\n        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556),\n        and is the number of 'frequencies' that will be used to represent each\n        continuous column. See their Eq 2 in the paper for details. If\n        the _'periodic'_ method is used, this parameter is required.\n    sigma: float, Optional, default = None,\n        This is the sigma parameter in the paper mentioned when describing the\n        previous parameters and it is used to initialise the 'frequency\n        weights'. See their Eq 2 in the paper for details. If\n        the _'periodic'_ method is used, this parameter is required.\n    share_last_layer: bool, Optional, default = None,\n        This parameter is not present in the before mentioned paper but it is implemented in\n        the [official repo](https://github.com/yandex-research/rtdl-num-embeddings/tree/main).\n        If `True` the linear layer that turns the frequencies into embeddings\n        will be shared across the continuous columns. If `False` a different\n        linear layer will be used for each continuous column.\n        If the _'periodic'_ method is used, this parameter is required.\n    full_embed_dropout: bool, Optional, default = None,\n        If `True`, the full embedding corresponding to a column will be masked\n        out/dropout. If `None`, it will default to `False`.\n    input_dim: int, default = 32\n        The so-called *dimension of the model*. Is the number of\n        embeddings used to encode the categorical and/or continuous columns\n    n_heads: int, default = 8\n        Number of attention heads per Transformer block\n    use_qkv_bias: bool, default = False\n        Boolean indicating whether or not to use bias in the Q, K, and V\n        projection layers\n    n_blocks: int, default = 2\n        Number of SAINT-Transformer blocks.\n    attn_dropout: float, default = 0.2\n        Dropout that will be applied to the Multi-Head Attention column and\n        row layers\n    ff_dropout: float, default = 0.1\n        Dropout that will be applied to the FeedForward network\n    ff_factor: float, default = 4\n        Multiplicative factor applied to the first layer of the FF network in\n        each Transformer block, This is normally set to 4.\n    transformer_activation: str, default = \"gelu\"\n        Transformer Encoder activation function. _'tanh'_, _'relu'_,\n        _'leaky_relu'_, _'gelu'_, _'geglu'_ and _'reglu'_ are supported\n    mlp_hidden_dims: List, Optional, default = None\n        List with the number of neurons per dense layer in the MLP. e.g:\n        _[64, 32]_. If not provided no MLP on top of the final\n        Transformer block will be used.\n    mlp_activation: str, Optional, default = None\n        Activation function for the dense layers of the MLP. Currently\n        _'tanh'_, _'relu'_, _'leaky'_relu' and _'gelu'_ are supported.\n        If 'mlp_hidden_dims' is not `None` and this parameter is `None`, it\n        will default to _'relu'_.\n    mlp_dropout: float, Optional, default = None\n        float with the dropout between the dense layers of the MLP.\n        If 'mlp_hidden_dims' is not `None` and this parameter is `None`, it\n        will default to 0.0.\n    mlp_batchnorm: bool, Optional, default = None\n        Boolean indicating whether or not batch normalization will be applied\n        to the dense layers\n        If 'mlp_hidden_dims' is not `None` and this parameter is `None`, it\n        will default to False.\n    mlp_batchnorm_last: bool, Optional, default = None\n        Boolean indicating whether or not batch normalization will be applied\n        to the last of the dense layers\n        If 'mlp_hidden_dims' is not `None` and this parameter is `None`, it\n        will default to False.\n    mlp_linear_first: bool, Optional, default = None\n        Boolean indicating the order of the operations in the dense\n        layer. If `True: [LIN -&gt; ACT -&gt; BN -&gt; DP]`. If `False: [BN -&gt; DP -&gt;\n        LIN -&gt; ACT]`\n        If 'mlp_hidden_dims' is not `None` and this parameter is `None`, it\n        will default to `True`.\n\n    Attributes\n    ----------\n    encoder: nn.Module\n        Sequence of SAINT-Transformer blocks\n    mlp: nn.Module\n        MLP component in the model\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.models import SAINT\n    &gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n    &gt;&gt;&gt; colnames = ['a', 'b', 'c', 'd', 'e']\n    &gt;&gt;&gt; cat_embed_input = [(u,i) for u,i in zip(colnames[:4], [4]*4)]\n    &gt;&gt;&gt; continuous_cols = ['e']\n    &gt;&gt;&gt; column_idx = {k:v for v,k in enumerate(colnames)}\n    &gt;&gt;&gt; model = SAINT(column_idx=column_idx, cat_embed_input=cat_embed_input, continuous_cols=continuous_cols)\n    &gt;&gt;&gt; out = model(X_tab)\n    \"\"\"\n\n    def __init__(\n        self,\n        column_idx: Dict[str, int],\n        *,\n        cat_embed_input: Optional[List[Tuple[str, int]]] = None,\n        cat_embed_dropout: Optional[float] = None,\n        use_cat_bias: Optional[bool] = None,\n        cat_embed_activation: Optional[str] = None,\n        shared_embed: Optional[bool] = None,\n        add_shared_embed: Optional[bool] = None,\n        frac_shared_embed: Optional[float] = None,\n        continuous_cols: Optional[List[str]] = None,\n        cont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\n        embed_continuous_method: Optional[\n            Literal[\"standard\", \"piecewise\", \"periodic\"]\n        ] = \"standard\",\n        cont_embed_dropout: Optional[float] = None,\n        cont_embed_activation: Optional[str] = None,\n        quantization_setup: Optional[Dict[str, List[float]]] = None,\n        n_frequencies: Optional[int] = None,\n        sigma: Optional[float] = None,\n        share_last_layer: Optional[bool] = None,\n        full_embed_dropout: Optional[bool] = None,\n        input_dim: int = 32,\n        use_qkv_bias: bool = False,\n        n_heads: int = 8,\n        n_blocks: int = 2,\n        attn_dropout: float = 0.1,\n        ff_dropout: float = 0.2,\n        ff_factor: int = 4,\n        transformer_activation: str = \"gelu\",\n        mlp_hidden_dims: Optional[List[int]] = None,\n        mlp_activation: Optional[str] = None,\n        mlp_dropout: Optional[float] = None,\n        mlp_batchnorm: Optional[bool] = None,\n        mlp_batchnorm_last: Optional[bool] = None,\n        mlp_linear_first: Optional[bool] = None,\n    ):\n        super(SAINT, self).__init__(\n            column_idx=column_idx,\n            cat_embed_input=cat_embed_input,\n            cat_embed_dropout=cat_embed_dropout,\n            use_cat_bias=use_cat_bias,\n            cat_embed_activation=cat_embed_activation,\n            shared_embed=shared_embed,\n            add_shared_embed=add_shared_embed,\n            frac_shared_embed=frac_shared_embed,\n            continuous_cols=continuous_cols,\n            cont_norm_layer=cont_norm_layer,\n            embed_continuous_method=embed_continuous_method,\n            cont_embed_dropout=cont_embed_dropout,\n            cont_embed_activation=cont_embed_activation,\n            input_dim=input_dim,\n            quantization_setup=quantization_setup,\n            n_frequencies=n_frequencies,\n            sigma=sigma,\n            share_last_layer=share_last_layer,\n            full_embed_dropout=full_embed_dropout,\n        )\n\n        self.use_qkv_bias = use_qkv_bias\n        self.n_heads = n_heads\n        self.n_blocks = n_blocks\n        self.attn_dropout = attn_dropout\n        self.ff_dropout = ff_dropout\n        self.ff_factor = ff_factor\n        self.transformer_activation = transformer_activation\n\n        self.mlp_hidden_dims = mlp_hidden_dims\n        self.mlp_activation = mlp_activation\n        self.mlp_dropout = mlp_dropout\n        self.mlp_batchnorm = mlp_batchnorm\n        self.mlp_batchnorm_last = mlp_batchnorm_last\n        self.mlp_linear_first = mlp_linear_first\n\n        self.with_cls_token = \"cls_token\" in column_idx\n        self.n_cat = len(cat_embed_input) if cat_embed_input is not None else 0\n        self.n_cont = len(continuous_cols) if continuous_cols is not None else 0\n        self.n_feats = self.n_cat + self.n_cont\n\n        # Embeddings are instantiated at the base model\n        # Transformer blocks\n        self.encoder = nn.Sequential()\n        for i in range(n_blocks):\n            self.encoder.add_module(\n                \"saint_block\" + str(i),\n                SaintEncoder(\n                    input_dim,\n                    n_heads,\n                    use_qkv_bias,\n                    attn_dropout,\n                    ff_dropout,\n                    ff_factor,\n                    transformer_activation,\n                    self.n_feats,\n                ),\n            )\n\n        self.mlp_first_hidden_dim = (\n            self.input_dim if self.with_cls_token else (self.n_feats * self.input_dim)\n        )\n\n        # Mlp: adding an MLP on top of the Resnet blocks is optional and\n        # therefore all related params are optional\n        if self.mlp_hidden_dims is not None:\n            self.mlp = MLP(\n                d_hidden=[self.mlp_first_hidden_dim] + self.mlp_hidden_dims,\n                activation=(\n                    \"relu\" if self.mlp_activation is None else self.mlp_activation\n                ),\n                dropout=0.0 if self.mlp_dropout is None else self.mlp_dropout,\n                batchnorm=False if self.mlp_batchnorm is None else self.mlp_batchnorm,\n                batchnorm_last=(\n                    False\n                    if self.mlp_batchnorm_last is None\n                    else self.mlp_batchnorm_last\n                ),\n                linear_first=(\n                    False if self.mlp_linear_first is None else self.mlp_linear_first\n                ),\n            )\n        else:\n            self.mlp = None\n\n    def forward(self, X: Tensor) -&gt; Tensor:\n        x = self._get_embeddings(X)\n        x = self.encoder(x)\n        if self.with_cls_token:\n            x = x[:, 0, :]\n        else:\n            x = x.flatten(1)\n        if self.mlp is not None:\n            x = self.mlp(x)\n        return x\n\n    @property\n    def output_dim(self) -&gt; int:\n        r\"\"\"The output dimension of the model. This is a required property\n        neccesary to build the `WideDeep` class\n        \"\"\"\n        return (\n            self.mlp_hidden_dims[-1]\n            if self.mlp_hidden_dims is not None\n            else self.mlp_first_hidden_dim\n        )\n\n    @property\n    def attention_weights(self) -&gt; List[Tuple[Tensor, Tensor]]:\n        r\"\"\"List with the attention weights. Each element of the list is a tuple\n        where the first and the second elements are the column and row\n        attention weights respectively\n\n        The shape of the attention weights is:\n\n        - column attention: $(N, H, F, F)$\n\n        - row attention: $(1, H, N, N)$\n\n        where $N$ is the batch size, $H$ is the number of heads and $F$ is the\n        number of features/columns in the dataset\n        \"\"\"\n        attention_weights = []\n        for blk in self.encoder:\n            attention_weights.append(\n                (blk.col_attn.attn_weights, blk.row_attn.attn_weights)\n            )\n        return attention_weights\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.transformers.saint.SAINT.output_dim","title":"output_dim  <code>property</code>","text":"<pre><code>output_dim\n</code></pre> <p>The output dimension of the model. This is a required property neccesary to build the <code>WideDeep</code> class</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.transformers.saint.SAINT.attention_weights","title":"attention_weights  <code>property</code>","text":"<pre><code>attention_weights\n</code></pre> <p>List with the attention weights. Each element of the list is a tuple where the first and the second elements are the column and row attention weights respectively</p> <p>The shape of the attention weights is:</p> <ul> <li> <p>column attention: \\((N, H, F, F)\\)</p> </li> <li> <p>row attention: \\((1, H, N, N)\\)</p> </li> </ul> <p>where \\(N\\) is the batch size, \\(H\\) is the number of heads and \\(F\\) is the number of features/columns in the dataset</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.transformers.ft_transformer.FTTransformer","title":"FTTransformer","text":"<p>               Bases: <code>BaseTabularModelWithAttention</code></p> <p>Defines a FTTransformer model that can be used as the <code>deeptabular</code> component of a Wide &amp; Deep model or independently by itself.</p> <p>Most of the parameters for this class are <code>Optional</code> since the use of categorical or continuous is in fact optional (i.e. one can use categorical features only, continuous features only or both).</p> <p>Parameters:</p> Name Type Description Default <code>column_idx</code> <code>Dict[str, int]</code> <p>Dict containing the index of the columns that will be passed through the <code>TabMlp</code> model. Required to slice the tensors. e.g. {'education': 0, 'relationship': 1, 'workclass': 2, ...}.</p> required <code>cat_embed_input</code> <code>Optional[List[Tuple[str, int]]]</code> <p>List of Tuples with the column name and number of unique values and embedding dimension. e.g. [(education, 11), ...]</p> <code>None</code> <code>cat_embed_dropout</code> <code>Optional[float]</code> <p>Categorical embeddings dropout. If <code>None</code>, it will default to 0.</p> <code>None</code> <code>use_cat_bias</code> <code>Optional[bool]</code> <p>Boolean indicating if bias will be used for the categorical embeddings. If <code>None</code>, it will default to 'False'.</p> <code>None</code> <code>cat_embed_activation</code> <code>Optional[str]</code> <p>Activation function for the categorical embeddings, if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> <code>None</code> <code>shared_embed</code> <code>Optional[bool]</code> <p>Boolean indicating if the embeddings will be \"shared\". The idea behind <code>shared_embed</code> is described in the Appendix A in the TabTransformer paper: 'The goal of having column embedding is to enable the model to distinguish the classes in one column from those in the other columns'. In other words, the idea is to let the model learn which column is embedded at the time. See: <code>pytorch_widedeep.models.transformers._layers.SharedEmbeddings</code>.</p> <code>None</code> <code>add_shared_embed</code> <code>Optional[bool]</code> <p>The two embedding sharing strategies are: 1) add the shared embeddings to the column embeddings or 2) to replace the first <code>frac_shared_embed</code> with the shared embeddings. See <code>pytorch_widedeep.models.embeddings_layers.SharedEmbeddings</code> If 'None' is passed, it will default to 'False'.</p> <code>None</code> <code>frac_shared_embed</code> <code>Optional[float]</code> <p>The fraction of embeddings that will be shared (if <code>add_shared_embed = False</code>) by all the different categories for one particular column. If 'None' is passed, it will default to 0.0.</p> <code>None</code> <code>continuous_cols</code> <code>Optional[List[str]]</code> <p>List with the name of the numeric (aka continuous) columns</p> <code>None</code> <code>cont_norm_layer</code> <code>Optional[Literal[batchnorm, layernorm]]</code> <p>Type of normalization layer applied to the continuous features. Options are: 'layernorm' and 'batchnorm'. if <code>None</code>, no normalization layer will be used.</p> <code>None</code> <code>embed_continuous_method</code> <code>Optional[Literal[standard, piecewise, periodic]]</code> <p>Method to use to embed the continuous features. Options are: 'standard', 'periodic' or 'piecewise'. The 'standard' embedding method is based on the FT-Transformer implementation presented in the paper: Revisiting Deep Learning Models for Tabular Data. The 'periodic' and_'piecewise'_ methods were presented in the paper: On Embeddings for Numerical Features in Tabular Deep Learning. Please, read the papers for details.</p> <code>'standard'</code> <code>cont_embed_dropout</code> <code>Optional[float]</code> <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p> <code>None</code> <code>cont_embed_activation</code> <code>Optional[str]</code> <p>Activation function for the continuous embeddings if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported. If <code>None</code>, no activation function will be applied.</p> <code>None</code> <code>quantization_setup</code> <code>Optional[Dict[str, List[float]]]</code> <p>This parameter is used when the 'piecewise' method is used to embed the continuous cols. It is a dict where keys are the name of the continuous columns and values are lists with the boundaries for the quantization of the continuous_cols. See the examples for details. If If the 'piecewise' method is used, this parameter is required.</p> <code>None</code> <code>n_frequencies</code> <code>Optional[int]</code> <p>This is the so called 'k' in their paper On Embeddings for Numerical Features in Tabular Deep Learning, and is the number of 'frequencies' that will be used to represent each continuous column. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>sigma</code> <code>Optional[float]</code> <p>This is the sigma parameter in the paper mentioned when describing the previous parameters and it is used to initialise the 'frequency weights'. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>share_last_layer</code> <code>Optional[bool]</code> <p>This parameter is not present in the before mentioned paper but it is implemented in the official repo. If <code>True</code> the linear layer that turns the frequencies into embeddings will be shared across the continuous columns. If <code>False</code> a different linear layer will be used for each continuous column. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>full_embed_dropout</code> <code>Optional[bool]</code> <p>If <code>True</code>, the full embedding corresponding to a column will be masked out/dropout. If <code>None</code>, it will default to <code>False</code>.</p> <code>None</code> <code>input_dim</code> <code>int</code> <p>The so-called dimension of the model. Is the number of embeddings used to encode the categorical and/or continuous columns.</p> <code>64</code> <code>kv_compression_factor</code> <code>float</code> <p>By default, the FTTransformer uses Linear Attention (See Linformer: Self-Attention with Linear Complexity ). The compression factor that will be used to reduce the input sequence length. If we denote the resulting sequence length as \\(k = int(kv_{compression \\space factor} \\times s)\\) where \\(s\\) is the input sequence length.</p> <code>0.5</code> <code>kv_sharing</code> <code>bool</code> <p>Boolean indicating if the \\(E\\) and \\(F\\) projection matrices will share weights.  See Linformer: Self-Attention with Linear Complexity for details</p> <code>False</code> <code>n_heads</code> <code>int</code> <p>Number of attention heads per FTTransformer block</p> <code>8</code> <code>use_qkv_bias</code> <code>bool</code> <p>Boolean indicating whether or not to use bias in the Q, K, and V projection layers</p> <code>False</code> <code>n_blocks</code> <code>int</code> <p>Number of FTTransformer blocks</p> <code>4</code> <code>attn_dropout</code> <code>float</code> <p>Dropout that will be applied to the Linear-Attention layers</p> <code>0.2</code> <code>ff_dropout</code> <code>float</code> <p>Dropout that will be applied to the FeedForward network</p> <code>0.1</code> <code>ff_factor</code> <code>float</code> <p>Multiplicative factor applied to the first layer of the FF network in each Transformer block, This is normally set to 4, but they use 4/3 in the paper.</p> <code>1.33</code> <code>transformer_activation</code> <code>str</code> <p>Transformer Encoder activation function. 'tanh', 'relu', 'leaky_relu', 'gelu', 'geglu' and 'reglu' are supported</p> <code>'reglu'</code> <code>mlp_hidden_dims</code> <code>Optional[List[int]]</code> <p>List with the number of neurons per dense layer in the MLP. e.g: [64, 32]. If not provided no MLP on top of the final FTTransformer block will be used.</p> <code>None</code> <code>mlp_activation</code> <code>Optional[str]</code> <p>Activation function for the dense layers of the MLP. Currently 'tanh', 'relu', 'leaky'_relu' and _'gelu' are supported. If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to 'relu'.</p> <code>None</code> <code>mlp_dropout</code> <code>Optional[float]</code> <p>float with the dropout between the dense layers of the MLP. If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to 0.0.</p> <code>None</code> <code>mlp_batchnorm</code> <code>Optional[bool]</code> <p>Boolean indicating whether or not batch normalization will be applied to the dense layers If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to False.</p> <code>None</code> <code>mlp_batchnorm_last</code> <code>Optional[bool]</code> <p>Boolean indicating whether or not batch normalization will be applied to the last of the dense layers If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to False.</p> <code>None</code> <code>mlp_linear_first</code> <code>Optional[bool]</code> <p>Boolean indicating the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code> If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to <code>True</code>.</p> <code>None</code> <p>Attributes:</p> Name Type Description <code>encoder</code> <code>Module</code> <p>Sequence of FTTransformer blocks</p> <code>mlp</code> <code>Module</code> <p>MLP component in the model</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import FTTransformer\n&gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n&gt;&gt;&gt; colnames = ['a', 'b', 'c', 'd', 'e']\n&gt;&gt;&gt; cat_embed_input = [(u,i) for u,i in zip(colnames[:4], [4]*4)]\n&gt;&gt;&gt; continuous_cols = ['e']\n&gt;&gt;&gt; column_idx = {k:v for v,k in enumerate(colnames)}\n&gt;&gt;&gt; model = FTTransformer(column_idx=column_idx, cat_embed_input=cat_embed_input, continuous_cols=continuous_cols)\n&gt;&gt;&gt; out = model(X_tab)\n</code></pre> Source code in <code>pytorch_widedeep/models/tabular/transformers/ft_transformer.py</code> <pre><code>class FTTransformer(BaseTabularModelWithAttention):\n    r\"\"\"Defines a [FTTransformer model](https://arxiv.org/abs/2106.11959) that\n    can be used as the `deeptabular` component of a Wide &amp; Deep model or\n    independently by itself.\n\n    Most of the parameters for this class are `Optional` since the use of\n    categorical or continuous is in fact optional (i.e. one can use\n    categorical features only, continuous features only or both).\n\n    Parameters\n    ----------\n    column_idx: Dict\n        Dict containing the index of the columns that will be passed through\n        the `TabMlp` model. Required to slice the tensors. e.g. _{'education':\n        0, 'relationship': 1, 'workclass': 2, ...}_.\n    cat_embed_input: List, Optional, default = None\n        List of Tuples with the column name and number of unique values and\n        embedding dimension. e.g. _[(education, 11), ...]_\n    cat_embed_dropout: float, Optional, default = None\n        Categorical embeddings dropout. If `None`, it will default\n        to 0.\n    use_cat_bias: bool, Optional, default = None,\n        Boolean indicating if bias will be used for the categorical embeddings.\n        If `None`, it will default to 'False'.\n    cat_embed_activation: Optional, str, default = None,\n        Activation function for the categorical embeddings, if any. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported\n    shared_embed: bool, Optional, default = None\n        Boolean indicating if the embeddings will be \"shared\". The idea behind `shared_embed` is\n        described in the Appendix A in the [TabTransformer paper](https://arxiv.org/abs/2012.06678):\n        _'The goal of having column embedding is to enable the model to\n        distinguish the classes in one column from those in the other\n        columns'_. In other words, the idea is to let the model learn which\n        column is embedded at the time. See: `pytorch_widedeep.models.transformers._layers.SharedEmbeddings`.\n    add_shared_embed: bool, Optional, default = None\n        The two embedding sharing strategies are: 1) add the shared embeddings\n        to the column embeddings or 2) to replace the first\n        `frac_shared_embed` with the shared embeddings.\n        See `pytorch_widedeep.models.embeddings_layers.SharedEmbeddings`\n        If 'None' is passed, it will default to 'False'.\n    frac_shared_embed: float, Optional, default = None\n        The fraction of embeddings that will be shared (if `add_shared_embed\n        = False`) by all the different categories for one particular\n        column. If 'None' is passed, it will default to 0.0.\n    continuous_cols: List, Optional, default = None\n        List with the name of the numeric (aka continuous) columns\n    cont_norm_layer: str, Optional, default =  None\n        Type of normalization layer applied to the continuous features.\n        Options are: _'layernorm'_ and _'batchnorm'_. if `None`, no\n        normalization layer will be used.\n    embed_continuous_method: Optional, str, default = None,\n        Method to use to embed the continuous features. Options are:\n        _'standard'_, _'periodic'_ or _'piecewise'_. The _'standard'_\n        embedding method is based on the FT-Transformer implementation\n        presented in the paper: [Revisiting Deep Learning Models for\n        Tabular Data](https://arxiv.org/abs/2106.11959v5). The _'periodic'_\n        and_'piecewise'_ methods were presented in the paper: [On Embeddings for\n        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556).\n        Please, read the papers for details.\n    cont_embed_dropout: float, Optional, default = None,\n        Dropout for the continuous embeddings. If `None`, it will default to 0.0\n    cont_embed_activation: Optional, str, default = None,\n        Activation function for the continuous embeddings if any. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported.\n        If `None`, no activation function will be applied.\n    quantization_setup: Dict[str, List[float]], Optional, default = None,\n        This parameter is used when the _'piecewise'_ method is used to embed\n        the continuous cols. It is a dict where keys are the name of the continuous\n        columns and values are lists with the boundaries for the quantization\n        of the continuous_cols. See the examples for details. If\n        If the _'piecewise'_ method is used, this parameter is required.\n    n_frequencies: int, Optional, default = None,\n        This is the so called _'k'_ in their paper [On Embeddings for\n        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556),\n        and is the number of 'frequencies' that will be used to represent each\n        continuous column. See their Eq 2 in the paper for details. If\n        the _'periodic'_ method is used, this parameter is required.\n    sigma: float, Optional, default = None,\n        This is the sigma parameter in the paper mentioned when describing the\n        previous parameters and it is used to initialise the 'frequency\n        weights'. See their Eq 2 in the paper for details. If\n        the _'periodic'_ method is used, this parameter is required.\n    share_last_layer: bool, Optional, default = None,\n        This parameter is not present in the before mentioned paper but it is implemented in\n        the [official repo](https://github.com/yandex-research/rtdl-num-embeddings/tree/main).\n        If `True` the linear layer that turns the frequencies into embeddings\n        will be shared across the continuous columns. If `False` a different\n        linear layer will be used for each continuous column.\n        If the _'periodic'_ method is used, this parameter is required.\n    full_embed_dropout: bool, Optional, default = None,\n        If `True`, the full embedding corresponding to a column will be masked\n        out/dropout. If `None`, it will default to `False`.\n    input_dim: int, default = 64\n        The so-called *dimension of the model*. Is the number of embeddings used to encode\n        the categorical and/or continuous columns.\n    kv_compression_factor: int, default = 0.5\n        By default, the FTTransformer uses Linear Attention\n        (See [Linformer: Self-Attention with Linear Complexity](https://arxiv.org/abs/2006.04768&gt;) ).\n        The compression factor that will be used to reduce the input sequence\n        length. If we denote the resulting sequence length as\n        $k = int(kv_{compression \\space factor} \\times s)$\n        where $s$ is the input sequence length.\n    kv_sharing: bool, default = False\n        Boolean indicating if the $E$ and $F$ projection matrices\n        will share weights.  See [Linformer: Self-Attention with Linear\n        Complexity](https://arxiv.org/abs/2006.04768) for details\n    n_heads: int, default = 8\n        Number of attention heads per FTTransformer block\n    use_qkv_bias: bool, default = False\n        Boolean indicating whether or not to use bias in the Q, K, and V\n        projection layers\n    n_blocks: int, default = 4\n        Number of FTTransformer blocks\n    attn_dropout: float, default = 0.2\n        Dropout that will be applied to the Linear-Attention layers\n    ff_dropout: float, default = 0.1\n        Dropout that will be applied to the FeedForward network\n    ff_factor: float, default = 4 / 3\n        Multiplicative factor applied to the first layer of the FF network in\n        each Transformer block, This is normally set to 4, but they use 4/3\n        in the paper.\n    transformer_activation: str, default = \"gelu\"\n        Transformer Encoder activation function. _'tanh'_, _'relu'_,\n        _'leaky_relu'_, _'gelu'_, _'geglu'_ and _'reglu'_ are supported\n    mlp_hidden_dims: List, Optional, default = None\n        List with the number of neurons per dense layer in the MLP. e.g:\n        _[64, 32]_. If not provided no MLP on top of the final\n        FTTransformer block will be used.\n    mlp_activation: str, Optional, default = None\n        Activation function for the dense layers of the MLP. Currently\n        _'tanh'_, _'relu'_, _'leaky'_relu' and _'gelu'_ are supported.\n        If 'mlp_hidden_dims' is not `None` and this parameter is `None`, it\n        will default to _'relu'_.\n    mlp_dropout: float, Optional, default = None\n        float with the dropout between the dense layers of the MLP.\n        If 'mlp_hidden_dims' is not `None` and this parameter is `None`, it\n        will default to 0.0.\n    mlp_batchnorm: bool, Optional, default = None\n        Boolean indicating whether or not batch normalization will be applied\n        to the dense layers\n        If 'mlp_hidden_dims' is not `None` and this parameter is `None`, it\n        will default to False.\n    mlp_batchnorm_last: bool, Optional, default = None\n        Boolean indicating whether or not batch normalization will be applied\n        to the last of the dense layers\n        If 'mlp_hidden_dims' is not `None` and this parameter is `None`, it\n        will default to False.\n    mlp_linear_first: bool, Optional, default = None\n        Boolean indicating the order of the operations in the dense\n        layer. If `True: [LIN -&gt; ACT -&gt; BN -&gt; DP]`. If `False: [BN -&gt; DP -&gt;\n        LIN -&gt; ACT]`\n        If 'mlp_hidden_dims' is not `None` and this parameter is `None`, it\n        will default to `True`.\n\n    Attributes\n    ----------\n    encoder: nn.Module\n        Sequence of FTTransformer blocks\n    mlp: nn.Module\n        MLP component in the model\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.models import FTTransformer\n    &gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n    &gt;&gt;&gt; colnames = ['a', 'b', 'c', 'd', 'e']\n    &gt;&gt;&gt; cat_embed_input = [(u,i) for u,i in zip(colnames[:4], [4]*4)]\n    &gt;&gt;&gt; continuous_cols = ['e']\n    &gt;&gt;&gt; column_idx = {k:v for v,k in enumerate(colnames)}\n    &gt;&gt;&gt; model = FTTransformer(column_idx=column_idx, cat_embed_input=cat_embed_input, continuous_cols=continuous_cols)\n    &gt;&gt;&gt; out = model(X_tab)\n    \"\"\"\n\n    def __init__(\n        self,\n        column_idx: Dict[str, int],\n        *,\n        cat_embed_input: Optional[List[Tuple[str, int]]] = None,\n        cat_embed_dropout: Optional[float] = None,\n        use_cat_bias: Optional[bool] = None,\n        cat_embed_activation: Optional[str] = None,\n        shared_embed: Optional[bool] = None,\n        add_shared_embed: Optional[bool] = None,\n        frac_shared_embed: Optional[float] = None,\n        continuous_cols: Optional[List[str]] = None,\n        cont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\n        embed_continuous_method: Optional[\n            Literal[\"standard\", \"piecewise\", \"periodic\"]\n        ] = \"standard\",\n        cont_embed_dropout: Optional[float] = None,\n        cont_embed_activation: Optional[str] = None,\n        quantization_setup: Optional[Dict[str, List[float]]] = None,\n        n_frequencies: Optional[int] = None,\n        sigma: Optional[float] = None,\n        share_last_layer: Optional[bool] = None,\n        full_embed_dropout: Optional[bool] = None,\n        input_dim: int = 64,\n        kv_compression_factor: float = 0.5,\n        kv_sharing: bool = False,\n        use_qkv_bias: bool = False,\n        n_heads: int = 8,\n        n_blocks: int = 4,\n        attn_dropout: float = 0.2,\n        ff_dropout: float = 0.1,\n        ff_factor: float = 1.33,\n        transformer_activation: str = \"reglu\",\n        mlp_hidden_dims: Optional[List[int]] = None,\n        mlp_activation: Optional[str] = None,\n        mlp_dropout: Optional[float] = None,\n        mlp_batchnorm: Optional[bool] = None,\n        mlp_batchnorm_last: Optional[bool] = None,\n        mlp_linear_first: Optional[bool] = None,\n    ):\n        super(FTTransformer, self).__init__(\n            column_idx=column_idx,\n            cat_embed_input=cat_embed_input,\n            cat_embed_dropout=cat_embed_dropout,\n            use_cat_bias=use_cat_bias,\n            cat_embed_activation=cat_embed_activation,\n            shared_embed=shared_embed,\n            add_shared_embed=add_shared_embed,\n            frac_shared_embed=frac_shared_embed,\n            continuous_cols=continuous_cols,\n            cont_norm_layer=cont_norm_layer,\n            embed_continuous_method=embed_continuous_method,\n            cont_embed_dropout=cont_embed_dropout,\n            cont_embed_activation=cont_embed_activation,\n            input_dim=input_dim,\n            quantization_setup=quantization_setup,\n            n_frequencies=n_frequencies,\n            sigma=sigma,\n            share_last_layer=share_last_layer,\n            full_embed_dropout=full_embed_dropout,\n        )\n\n        self.kv_compression_factor = kv_compression_factor\n        self.kv_sharing = kv_sharing\n        self.use_qkv_bias = use_qkv_bias\n        self.n_heads = n_heads\n        self.n_blocks = n_blocks\n        self.attn_dropout = attn_dropout\n        self.ff_dropout = ff_dropout\n        self.ff_factor = ff_factor\n        self.transformer_activation = transformer_activation\n\n        self.mlp_hidden_dims = mlp_hidden_dims\n        self.mlp_activation = mlp_activation\n        self.mlp_dropout = mlp_dropout\n        self.mlp_batchnorm = mlp_batchnorm\n        self.mlp_batchnorm_last = mlp_batchnorm_last\n        self.mlp_linear_first = mlp_linear_first\n\n        self.with_cls_token = \"cls_token\" in column_idx\n        self.n_cat = len(cat_embed_input) if cat_embed_input is not None else 0\n        self.n_cont = len(continuous_cols) if continuous_cols is not None else 0\n        self.n_feats = self.n_cat + self.n_cont\n\n        # Embeddings are instantiated at the base model\n        # Transformer blocks\n        is_first = True\n        self.encoder = nn.Sequential()\n        for i in range(n_blocks):\n            self.encoder.add_module(\n                \"fttransformer_block\" + str(i),\n                FTTransformerEncoder(\n                    input_dim,\n                    self.n_feats,\n                    n_heads,\n                    use_qkv_bias,\n                    attn_dropout,\n                    ff_dropout,\n                    ff_factor,\n                    kv_compression_factor,\n                    kv_sharing,\n                    transformer_activation,\n                    is_first,\n                ),\n            )\n            is_first = False\n\n        self.mlp_first_hidden_dim = (\n            self.input_dim if self.with_cls_token else (self.n_feats * self.input_dim)\n        )\n\n        # Mlp: adding an MLP on top of the Resnet blocks is optional and\n        # therefore all related params are optional\n        if self.mlp_hidden_dims is not None:\n            self.mlp = MLP(\n                d_hidden=[self.mlp_first_hidden_dim] + self.mlp_hidden_dims,\n                activation=(\n                    \"relu\" if self.mlp_activation is None else self.mlp_activation\n                ),\n                dropout=0.0 if self.mlp_dropout is None else self.mlp_dropout,\n                batchnorm=False if self.mlp_batchnorm is None else self.mlp_batchnorm,\n                batchnorm_last=(\n                    False\n                    if self.mlp_batchnorm_last is None\n                    else self.mlp_batchnorm_last\n                ),\n                linear_first=(\n                    False if self.mlp_linear_first is None else self.mlp_linear_first\n                ),\n            )\n        else:\n            self.mlp = None\n\n    def forward(self, X: Tensor) -&gt; Tensor:\n        x = self._get_embeddings(X)\n        x = self.encoder(x)\n        if self.with_cls_token:\n            x = x[:, 0, :]\n        else:\n            x = x.flatten(1)\n        if self.mlp is not None:\n            x = self.mlp(x)\n        return x\n\n    @property\n    def output_dim(self) -&gt; int:\n        r\"\"\"The output dimension of the model. This is a required property\n        neccesary to build the `WideDeep` class\n        \"\"\"\n        return (\n            self.mlp_hidden_dims[-1]\n            if self.mlp_hidden_dims is not None\n            else self.mlp_first_hidden_dim\n        )\n\n    @property\n    def attention_weights(self) -&gt; List[Tensor]:\n        r\"\"\"List with the attention weights per block\n\n        The shape of the attention weights is: $(N, H, F, k)$, where $N$ is\n        the batch size, $H$ is the number of attention heads, $F$ is the\n        number of features/columns and $k$ is the reduced sequence length or\n        dimension, i.e. $k = int(kv_{compression \\space factor} \\times s)$\n        \"\"\"\n        return [blk.attn.attn_weights for blk in self.encoder]\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.transformers.ft_transformer.FTTransformer.output_dim","title":"output_dim  <code>property</code>","text":"<pre><code>output_dim\n</code></pre> <p>The output dimension of the model. This is a required property neccesary to build the <code>WideDeep</code> class</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.transformers.ft_transformer.FTTransformer.attention_weights","title":"attention_weights  <code>property</code>","text":"<pre><code>attention_weights\n</code></pre> <p>List with the attention weights per block</p> <p>The shape of the attention weights is: \\((N, H, F, k)\\), where \\(N\\) is the batch size, \\(H\\) is the number of attention heads, \\(F\\) is the number of features/columns and \\(k\\) is the reduced sequence length or dimension, i.e. \\(k = int(kv_{compression \\space factor} \\times s)\\)</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.transformers.tab_perceiver.TabPerceiver","title":"TabPerceiver","text":"<p>               Bases: <code>BaseTabularModelWithAttention</code></p> <p>Defines an adaptation of a Perceiver  that can be used as the <code>deeptabular</code> component of a Wide &amp; Deep model  or independently by itself.</p> <p>Most of the parameters for this class are <code>Optional</code> since the use of categorical or continuous is in fact optional (i.e. one can use categorical features only, continuous features only or both).</p> <p> NOTE: while there are scientific publications for  the <code>TabTransformer</code>, <code>SAINT</code> and <code>FTTransformer</code>, the <code>TabPerceiver</code>  and the <code>TabFastFormer</code> are our own adaptations of the  Perceiver and the  FastFormer for tabular data.</p> <p>Parameters:</p> Name Type Description Default <code>column_idx</code> <code>Dict[str, int]</code> <p>Dict containing the index of the columns that will be passed through the <code>TabMlp</code> model. Required to slice the tensors. e.g. {'education': 0, 'relationship': 1, 'workclass': 2, ...}.</p> required <code>cat_embed_input</code> <code>Optional[List[Tuple[str, int]]]</code> <p>List of Tuples with the column name and number of unique values and embedding dimension. e.g. [(education, 11), ...]</p> <code>None</code> <code>cat_embed_dropout</code> <code>Optional[float]</code> <p>Categorical embeddings dropout. If <code>None</code>, it will default to 0.</p> <code>None</code> <code>use_cat_bias</code> <code>Optional[bool]</code> <p>Boolean indicating if bias will be used for the categorical embeddings. If <code>None</code>, it will default to 'False'.</p> <code>None</code> <code>cat_embed_activation</code> <code>Optional[str]</code> <p>Activation function for the categorical embeddings, if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> <code>None</code> <code>shared_embed</code> <code>Optional[bool]</code> <p>Boolean indicating if the embeddings will be \"shared\". The idea behind <code>shared_embed</code> is described in the Appendix A in the TabTransformer paper: 'The goal of having column embedding is to enable the model to distinguish the classes in one column from those in the other columns'. In other words, the idea is to let the model learn which column is embedded at the time. See: <code>pytorch_widedeep.models.transformers._layers.SharedEmbeddings</code>.</p> <code>None</code> <code>add_shared_embed</code> <code>Optional[bool]</code> <p>The two embedding sharing strategies are: 1) add the shared embeddings to the column embeddings or 2) to replace the first <code>frac_shared_embed</code> with the shared embeddings. See <code>pytorch_widedeep.models.embeddings_layers.SharedEmbeddings</code> If 'None' is passed, it will default to 'False'.</p> <code>None</code> <code>frac_shared_embed</code> <code>Optional[float]</code> <p>The fraction of embeddings that will be shared (if <code>add_shared_embed = False</code>) by all the different categories for one particular column. If 'None' is passed, it will default to 0.0.</p> <code>None</code> <code>continuous_cols</code> <code>Optional[List[str]]</code> <p>List with the name of the numeric (aka continuous) columns</p> <code>None</code> <code>cont_norm_layer</code> <code>Optional[Literal[batchnorm, layernorm]]</code> <p>Type of normalization layer applied to the continuous features. Options are: 'layernorm' and 'batchnorm'. if <code>None</code>, no normalization layer will be used.</p> <code>None</code> <code>embed_continuous_method</code> <code>Optional[Literal[standard, piecewise, periodic]]</code> <p>Method to use to embed the continuous features. Options are: 'standard', 'periodic' or 'piecewise'. The 'standard' embedding method is based on the FT-Transformer implementation presented in the paper: Revisiting Deep Learning Models for Tabular Data. The 'periodic' and_'piecewise'_ methods were presented in the paper: On Embeddings for Numerical Features in Tabular Deep Learning. Please, read the papers for details.</p> <code>'standard'</code> <code>cont_embed_dropout</code> <code>Optional[float]</code> <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p> <code>None</code> <code>cont_embed_activation</code> <code>Optional[str]</code> <p>Activation function for the continuous embeddings if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported. If <code>None</code>, no activation function will be applied.</p> <code>None</code> <code>quantization_setup</code> <code>Optional[Dict[str, List[float]]]</code> <p>This parameter is used when the 'piecewise' method is used to embed the continuous cols. It is a dict where keys are the name of the continuous columns and values are lists with the boundaries for the quantization of the continuous_cols. See the examples for details. If If the 'piecewise' method is used, this parameter is required.</p> <code>None</code> <code>n_frequencies</code> <code>Optional[int]</code> <p>This is the so called 'k' in their paper On Embeddings for Numerical Features in Tabular Deep Learning, and is the number of 'frequencies' that will be used to represent each continuous column. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>sigma</code> <code>Optional[float]</code> <p>This is the sigma parameter in the paper mentioned when describing the previous parameters and it is used to initialise the 'frequency weights'. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>share_last_layer</code> <code>Optional[bool]</code> <p>This parameter is not present in the before mentioned paper but it is implemented in the official repo. If <code>True</code> the linear layer that turns the frequencies into embeddings will be shared across the continuous columns. If <code>False</code> a different linear layer will be used for each continuous column. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>full_embed_dropout</code> <code>Optional[bool]</code> <p>If <code>True</code>, the full embedding corresponding to a column will be masked out/dropout. If <code>None</code>, it will default to <code>False</code>.</p> <code>None</code> <code>input_dim</code> <code>int</code> <p>The so-called dimension of the model. Is the number of embeddings used to encode the categorical and/or continuous columns.</p> <code>32</code> <code>n_cross_attns</code> <code>int</code> <p>Number of times each perceiver block will cross attend to the input data (i.e. number of cross attention components per perceiver block). This should normally be 1. However, in the paper they describe some architectures (normally computer vision-related problems) where the Perceiver attends multiple times to the input array. Therefore, maybe multiple cross attention to the input array is also useful in some cases for tabular data  .</p> <code>1</code> <code>n_cross_attn_heads</code> <code>int</code> <p>Number of attention heads for the cross attention component</p> <code>4</code> <code>n_latents</code> <code>int</code> <p>Number of latents. This is the \\(N\\) parameter in the paper. As indicated in the paper, this number should be significantly lower than \\(M\\) (the number of columns in the dataset). Setting \\(N\\) closer to \\(M\\) defies the main purpose of the Perceiver, which is to overcome the transformer quadratic bottleneck</p> <code>16</code> <code>latent_dim</code> <code>int</code> <p>Latent dimension.</p> <code>128</code> <code>n_latent_heads</code> <code>int</code> <p>Number of attention heads per Latent Transformer</p> <code>4</code> <code>n_latent_blocks</code> <code>int</code> <p>Number of transformer encoder blocks (normalised MHA + normalised FF) per Latent Transformer</p> <code>4</code> <code>n_perceiver_blocks</code> <code>int</code> <p>Number of Perceiver blocks defined as [Cross Attention + Latent Transformer]</p> <code>4</code> <code>share_weights</code> <code>bool</code> <p>Boolean indicating if the weights will be shared between Perceiver blocks</p> <code>False</code> <code>attn_dropout</code> <code>float</code> <p>Dropout that will be applied to the Multi-Head Attention layers</p> <code>0.1</code> <code>ff_dropout</code> <code>float</code> <p>Dropout that will be applied to the FeedForward network</p> <code>0.1</code> <code>ff_factor</code> <code>int</code> <p>Multiplicative factor applied to the first layer of the FF network in each Transformer block, This is normally set to 4.</p> <code>4</code> <code>transformer_activation</code> <code>str</code> <p>Transformer Encoder activation function. 'tanh', 'relu', 'leaky_relu', 'gelu', 'geglu' and 'reglu' are supported</p> <code>'geglu'</code> <code>mlp_hidden_dims</code> <code>Optional[List[int]]</code> <p>List with the number of neurons per dense layer in the MLP. e.g: [64, 32]. If not provided no MLP on top of the final Transformer block will be used.</p> <code>None</code> <code>mlp_activation</code> <code>Optional[str]</code> <p>Activation function for the dense layers of the MLP. Currently 'tanh', 'relu', 'leaky'_relu' and _'gelu' are supported. If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to 'relu'.</p> <code>None</code> <code>mlp_dropout</code> <code>Optional[float]</code> <p>float with the dropout between the dense layers of the MLP. If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to 0.0.</p> <code>None</code> <code>mlp_batchnorm</code> <code>Optional[bool]</code> <p>Boolean indicating whether or not batch normalization will be applied to the dense layers If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to False.</p> <code>None</code> <code>mlp_batchnorm_last</code> <code>Optional[bool]</code> <p>Boolean indicating whether or not batch normalization will be applied to the last of the dense layers If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to False.</p> <code>None</code> <code>mlp_linear_first</code> <code>Optional[bool]</code> <p>Boolean indicating the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code> If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to <code>True</code>.</p> <code>None</code> <p>Attributes:</p> Name Type Description <code>encoder</code> <code>ModuleDict</code> <p>ModuleDict with the Perceiver blocks</p> <code>latents</code> <code>Parameter</code> <p>Latents that will be used for prediction</p> <code>mlp</code> <code>Module</code> <p>MLP component in the model</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import TabPerceiver\n&gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n&gt;&gt;&gt; colnames = ['a', 'b', 'c', 'd', 'e']\n&gt;&gt;&gt; cat_embed_input = [(u,i) for u,i in zip(colnames[:4], [4]*4)]\n&gt;&gt;&gt; continuous_cols = ['e']\n&gt;&gt;&gt; column_idx = {k:v for v,k in enumerate(colnames)}\n&gt;&gt;&gt; model = TabPerceiver(column_idx=column_idx, cat_embed_input=cat_embed_input,\n... continuous_cols=continuous_cols, n_latents=2, latent_dim=16,\n... n_perceiver_blocks=2)\n&gt;&gt;&gt; out = model(X_tab)\n</code></pre> Source code in <code>pytorch_widedeep/models/tabular/transformers/tab_perceiver.py</code> <pre><code>class TabPerceiver(BaseTabularModelWithAttention):\n    r\"\"\"Defines an adaptation of a [Perceiver](https://arxiv.org/abs/2103.03206)\n     that can be used as the `deeptabular` component of a Wide &amp; Deep model\n     or independently by itself.\n\n    Most of the parameters for this class are `Optional` since the use of\n    categorical or continuous is in fact optional (i.e. one can use\n    categorical features only, continuous features only or both).\n\n    :information_source: **NOTE**: while there are scientific publications for\n     the `TabTransformer`, `SAINT` and `FTTransformer`, the `TabPerceiver`\n     and the `TabFastFormer` are our own adaptations of the\n     [Perceiver](https://arxiv.org/abs/2103.03206) and the\n     [FastFormer](https://arxiv.org/abs/2108.09084) for tabular data.\n\n    Parameters\n    ----------\n    column_idx: Dict\n        Dict containing the index of the columns that will be passed through\n        the `TabMlp` model. Required to slice the tensors. e.g. _{'education':\n        0, 'relationship': 1, 'workclass': 2, ...}_.\n    cat_embed_input: List, Optional, default = None\n        List of Tuples with the column name and number of unique values and\n        embedding dimension. e.g. _[(education, 11), ...]_\n    cat_embed_dropout: float, Optional, default = None\n        Categorical embeddings dropout. If `None`, it will default\n        to 0.\n    use_cat_bias: bool, Optional, default = None,\n        Boolean indicating if bias will be used for the categorical embeddings.\n        If `None`, it will default to 'False'.\n    cat_embed_activation: Optional, str, default = None,\n        Activation function for the categorical embeddings, if any. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported\n    shared_embed: bool, Optional, default = None\n        Boolean indicating if the embeddings will be \"shared\". The idea behind `shared_embed` is\n        described in the Appendix A in the [TabTransformer paper](https://arxiv.org/abs/2012.06678):\n        _'The goal of having column embedding is to enable the model to\n        distinguish the classes in one column from those in the other\n        columns'_. In other words, the idea is to let the model learn which\n        column is embedded at the time. See: `pytorch_widedeep.models.transformers._layers.SharedEmbeddings`.\n    add_shared_embed: bool, Optional, default = None\n        The two embedding sharing strategies are: 1) add the shared embeddings\n        to the column embeddings or 2) to replace the first\n        `frac_shared_embed` with the shared embeddings.\n        See `pytorch_widedeep.models.embeddings_layers.SharedEmbeddings`\n        If 'None' is passed, it will default to 'False'.\n    frac_shared_embed: float, Optional, default = None\n        The fraction of embeddings that will be shared (if `add_shared_embed\n        = False`) by all the different categories for one particular\n        column. If 'None' is passed, it will default to 0.0.\n    continuous_cols: List, Optional, default = None\n        List with the name of the numeric (aka continuous) columns\n    cont_norm_layer: str, Optional, default =  None\n        Type of normalization layer applied to the continuous features.\n        Options are: _'layernorm'_ and _'batchnorm'_. if `None`, no\n        normalization layer will be used.\n    embed_continuous_method: Optional, str, default = None,\n        Method to use to embed the continuous features. Options are:\n        _'standard'_, _'periodic'_ or _'piecewise'_. The _'standard'_\n        embedding method is based on the FT-Transformer implementation\n        presented in the paper: [Revisiting Deep Learning Models for\n        Tabular Data](https://arxiv.org/abs/2106.11959v5). The _'periodic'_\n        and_'piecewise'_ methods were presented in the paper: [On Embeddings for\n        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556).\n        Please, read the papers for details.\n    cont_embed_dropout: float, Optional, default = None,\n        Dropout for the continuous embeddings. If `None`, it will default to 0.0\n    cont_embed_activation: Optional, str, default = None,\n        Activation function for the continuous embeddings if any. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported.\n        If `None`, no activation function will be applied.\n    quantization_setup: Dict[str, List[float]], Optional, default = None,\n        This parameter is used when the _'piecewise'_ method is used to embed\n        the continuous cols. It is a dict where keys are the name of the continuous\n        columns and values are lists with the boundaries for the quantization\n        of the continuous_cols. See the examples for details. If\n        If the _'piecewise'_ method is used, this parameter is required.\n    n_frequencies: int, Optional, default = None,\n        This is the so called _'k'_ in their paper [On Embeddings for\n        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556),\n        and is the number of 'frequencies' that will be used to represent each\n        continuous column. See their Eq 2 in the paper for details. If\n        the _'periodic'_ method is used, this parameter is required.\n    sigma: float, Optional, default = None,\n        This is the sigma parameter in the paper mentioned when describing the\n        previous parameters and it is used to initialise the 'frequency\n        weights'. See their Eq 2 in the paper for details. If\n        the _'periodic'_ method is used, this parameter is required.\n    share_last_layer: bool, Optional, default = None,\n        This parameter is not present in the before mentioned paper but it is implemented in\n        the [official repo](https://github.com/yandex-research/rtdl-num-embeddings/tree/main).\n        If `True` the linear layer that turns the frequencies into embeddings\n        will be shared across the continuous columns. If `False` a different\n        linear layer will be used for each continuous column.\n        If the _'periodic'_ method is used, this parameter is required.\n    full_embed_dropout: bool, Optional, default = None,\n        If `True`, the full embedding corresponding to a column will be masked\n        out/dropout. If `None`, it will default to `False`.\n    input_dim: int, default = 32\n        The so-called *dimension of the model*. Is the number of embeddings\n        used to encode the categorical and/or continuous columns.\n    n_cross_attns: int, default = 1\n        Number of times each perceiver block will cross attend to the input\n        data (i.e. number of cross attention components per perceiver block).\n        This should normally be 1. However, in the paper they describe some\n        architectures (normally computer vision-related problems) where the\n        Perceiver attends multiple times to the input array. Therefore, maybe\n        multiple cross attention to the input array is also useful in some\n        cases for tabular data :shrug: .\n    n_cross_attn_heads: int, default = 4\n        Number of attention heads for the cross attention component\n    n_latents: int, default = 16\n        Number of latents. This is the $N$ parameter in the paper. As\n        indicated in the paper, this number should be significantly lower\n        than $M$ (the number of columns in the dataset). Setting $N$ closer\n        to $M$ defies the main purpose of the Perceiver, which is to overcome\n        the transformer quadratic bottleneck\n    latent_dim: int, default = 128\n        Latent dimension.\n    n_latent_heads: int, default = 4\n        Number of attention heads per Latent Transformer\n    n_latent_blocks: int, default = 4\n        Number of transformer encoder blocks (normalised MHA + normalised FF)\n        per Latent Transformer\n    n_perceiver_blocks: int, default = 4\n        Number of Perceiver blocks defined as [Cross Attention + Latent\n        Transformer]\n    share_weights: Boolean, default = False\n        Boolean indicating if the weights will be shared between Perceiver\n        blocks\n    attn_dropout: float, default = 0.2\n        Dropout that will be applied to the Multi-Head Attention layers\n    ff_dropout: float, default = 0.1\n        Dropout that will be applied to the FeedForward network\n    ff_factor: float, default = 4\n        Multiplicative factor applied to the first layer of the FF network in\n        each Transformer block, This is normally set to 4.\n    transformer_activation: str, default = \"gelu\"\n        Transformer Encoder activation function. _'tanh'_, _'relu'_,\n        _'leaky_relu'_, _'gelu'_, _'geglu'_ and _'reglu'_ are supported\n    mlp_hidden_dims: List, Optional, default = None\n        List with the number of neurons per dense layer in the MLP. e.g:\n        _[64, 32]_. If not provided no MLP on top of the final\n        Transformer block will be used.\n    mlp_activation: str, Optional, default = None\n        Activation function for the dense layers of the MLP. Currently\n        _'tanh'_, _'relu'_, _'leaky'_relu' and _'gelu'_ are supported.\n        If 'mlp_hidden_dims' is not `None` and this parameter is `None`, it\n        will default to _'relu'_.\n    mlp_dropout: float, Optional, default = None\n        float with the dropout between the dense layers of the MLP.\n        If 'mlp_hidden_dims' is not `None` and this parameter is `None`, it\n        will default to 0.0.\n    mlp_batchnorm: bool, Optional, default = None\n        Boolean indicating whether or not batch normalization will be applied\n        to the dense layers\n        If 'mlp_hidden_dims' is not `None` and this parameter is `None`, it\n        will default to False.\n    mlp_batchnorm_last: bool, Optional, default = None\n        Boolean indicating whether or not batch normalization will be applied\n        to the last of the dense layers\n        If 'mlp_hidden_dims' is not `None` and this parameter is `None`, it\n        will default to False.\n    mlp_linear_first: bool, Optional, default = None\n        Boolean indicating the order of the operations in the dense\n        layer. If `True: [LIN -&gt; ACT -&gt; BN -&gt; DP]`. If `False: [BN -&gt; DP -&gt;\n        LIN -&gt; ACT]`\n        If 'mlp_hidden_dims' is not `None` and this parameter is `None`, it\n        will default to `True`.\n\n    Attributes\n    ----------\n    encoder: nn.ModuleDict\n        ModuleDict with the Perceiver blocks\n    latents: nn.Parameter\n        Latents that will be used for prediction\n    mlp: nn.Module\n        MLP component in the model\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.models import TabPerceiver\n    &gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n    &gt;&gt;&gt; colnames = ['a', 'b', 'c', 'd', 'e']\n    &gt;&gt;&gt; cat_embed_input = [(u,i) for u,i in zip(colnames[:4], [4]*4)]\n    &gt;&gt;&gt; continuous_cols = ['e']\n    &gt;&gt;&gt; column_idx = {k:v for v,k in enumerate(colnames)}\n    &gt;&gt;&gt; model = TabPerceiver(column_idx=column_idx, cat_embed_input=cat_embed_input,\n    ... continuous_cols=continuous_cols, n_latents=2, latent_dim=16,\n    ... n_perceiver_blocks=2)\n    &gt;&gt;&gt; out = model(X_tab)\n    \"\"\"\n\n    def __init__(\n        self,\n        column_idx: Dict[str, int],\n        *,\n        cat_embed_input: Optional[List[Tuple[str, int]]] = None,\n        cat_embed_dropout: Optional[float] = None,\n        use_cat_bias: Optional[bool] = None,\n        cat_embed_activation: Optional[str] = None,\n        shared_embed: Optional[bool] = None,\n        add_shared_embed: Optional[bool] = None,\n        frac_shared_embed: Optional[float] = None,\n        continuous_cols: Optional[List[str]] = None,\n        cont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\n        embed_continuous_method: Optional[\n            Literal[\"standard\", \"piecewise\", \"periodic\"]\n        ] = \"standard\",\n        cont_embed_dropout: Optional[float] = None,\n        cont_embed_activation: Optional[str] = None,\n        quantization_setup: Optional[Dict[str, List[float]]] = None,\n        n_frequencies: Optional[int] = None,\n        sigma: Optional[float] = None,\n        share_last_layer: Optional[bool] = None,\n        full_embed_dropout: Optional[bool] = None,\n        input_dim: int = 32,\n        n_cross_attns: int = 1,\n        n_cross_attn_heads: int = 4,\n        n_latents: int = 16,\n        latent_dim: int = 128,\n        n_latent_heads: int = 4,\n        n_latent_blocks: int = 4,\n        n_perceiver_blocks: int = 4,\n        share_weights: bool = False,\n        attn_dropout: float = 0.1,\n        ff_dropout: float = 0.1,\n        ff_factor: int = 4,\n        transformer_activation: str = \"geglu\",\n        mlp_hidden_dims: Optional[List[int]] = None,\n        mlp_activation: Optional[str] = None,\n        mlp_dropout: Optional[float] = None,\n        mlp_batchnorm: Optional[bool] = None,\n        mlp_batchnorm_last: Optional[bool] = None,\n        mlp_linear_first: Optional[bool] = None,\n    ):\n        super(TabPerceiver, self).__init__(\n            column_idx=column_idx,\n            cat_embed_input=cat_embed_input,\n            cat_embed_dropout=cat_embed_dropout,\n            use_cat_bias=use_cat_bias,\n            cat_embed_activation=cat_embed_activation,\n            shared_embed=shared_embed,\n            add_shared_embed=add_shared_embed,\n            frac_shared_embed=frac_shared_embed,\n            continuous_cols=continuous_cols,\n            cont_norm_layer=cont_norm_layer,\n            embed_continuous_method=embed_continuous_method,\n            cont_embed_dropout=cont_embed_dropout,\n            cont_embed_activation=cont_embed_activation,\n            input_dim=input_dim,\n            quantization_setup=quantization_setup,\n            n_frequencies=n_frequencies,\n            sigma=sigma,\n            share_last_layer=share_last_layer,\n            full_embed_dropout=full_embed_dropout,\n        )\n\n        self.n_cross_attns = n_cross_attns\n        self.n_cross_attn_heads = n_cross_attn_heads\n        self.n_latents = n_latents\n        self.latent_dim = latent_dim\n        self.n_latent_heads = n_latent_heads\n        self.n_latent_blocks = n_latent_blocks\n        self.n_perceiver_blocks = n_perceiver_blocks\n        self.share_weights = share_weights\n        self.attn_dropout = attn_dropout\n        self.ff_dropout = ff_dropout\n        self.ff_factor = ff_factor\n        self.transformer_activation = transformer_activation\n\n        self.mlp_hidden_dims = mlp_hidden_dims\n        self.mlp_activation = mlp_activation\n        self.mlp_dropout = mlp_dropout\n        self.mlp_batchnorm = mlp_batchnorm\n        self.mlp_batchnorm_last = mlp_batchnorm_last\n        self.mlp_linear_first = mlp_linear_first\n\n        # Embeddings are instantiated at the base model\n        # Transformer blocks\n        self.latents = nn.init.trunc_normal_(\n            nn.Parameter(torch.empty(n_latents, latent_dim))\n        )\n\n        self.encoder = nn.ModuleDict()\n        first_perceiver_block = self._build_perceiver_block()\n        self.encoder[\"perceiver_block0\"] = first_perceiver_block\n\n        if share_weights:\n            for n in range(1, n_perceiver_blocks):\n                self.encoder[\"perceiver_block\" + str(n)] = first_perceiver_block\n        else:\n            for n in range(1, n_perceiver_blocks):\n                self.encoder[\"perceiver_block\" + str(n)] = self._build_perceiver_block()\n\n        self.mlp_first_hidden_dim = self.latent_dim\n\n        # Mlp\n        if self.mlp_hidden_dims is not None:\n            self.mlp = MLP(\n                d_hidden=[self.mlp_first_hidden_dim] + self.mlp_hidden_dims,\n                activation=(\n                    \"relu\" if self.mlp_activation is None else self.mlp_activation\n                ),\n                dropout=0.0 if self.mlp_dropout is None else self.mlp_dropout,\n                batchnorm=False if self.mlp_batchnorm is None else self.mlp_batchnorm,\n                batchnorm_last=(\n                    False\n                    if self.mlp_batchnorm_last is None\n                    else self.mlp_batchnorm_last\n                ),\n                linear_first=(\n                    False if self.mlp_linear_first is None else self.mlp_linear_first\n                ),\n            )\n        else:\n            self.mlp = None\n\n    def forward(self, X: Tensor) -&gt; Tensor:\n        x_emb = self._get_embeddings(X)\n\n        x = einops.repeat(self.latents, \"n d -&gt; b n d\", b=X.shape[0])\n\n        for n in range(self.n_perceiver_blocks):\n            cross_attns = self.encoder[\"perceiver_block\" + str(n)][\"cross_attns\"]\n            latent_transformer = self.encoder[\"perceiver_block\" + str(n)][\n                \"latent_transformer\"\n            ]\n            for cross_attn in cross_attns:\n                x = cross_attn(x, x_emb)\n            x = latent_transformer(x)\n\n        # average along the latent index axis\n        x = x.mean(dim=1)\n\n        if self.mlp is not None:\n            x = self.mlp(x)\n\n        return x\n\n    @property\n    def output_dim(self) -&gt; int:\n        r\"\"\"The output dimension of the model. This is a required property\n        neccesary to build the `WideDeep` class\n        \"\"\"\n        return (\n            self.mlp_hidden_dims[-1]\n            if self.mlp_hidden_dims is not None\n            else self.mlp_first_hidden_dim\n        )\n\n    @property\n    def attention_weights(self) -&gt; List:\n        r\"\"\"List with the attention weights. If the weights are not shared\n        between perceiver blocks each element of the list will be a list\n        itself containing the Cross Attention and Latent Transformer\n        attention weights respectively\n\n        The shape of the attention weights is:\n\n        - Cross Attention: $(N, C, L, F)$\n\n        - Latent Attention: $(N, T, L, L)$\n\n        WHere $N$ is the batch size, $C$ is the number of Cross Attention\n        heads, $L$ is the number of Latents, $F$ is the number of\n        features/columns in the dataset and $T$ is the number of Latent\n        Attention heads\n        \"\"\"\n        if self.share_weights:\n            cross_attns = self.encoder[\"perceiver_block0\"][\"cross_attns\"]\n            latent_transformer = self.encoder[\"perceiver_block0\"][\"latent_transformer\"]\n            attention_weights = self._extract_attn_weights(\n                cross_attns, latent_transformer\n            )\n        else:\n            attention_weights = []\n            for n in range(self.n_perceiver_blocks):\n                cross_attns = self.encoder[\"perceiver_block\" + str(n)][\"cross_attns\"]\n                latent_transformer = self.encoder[\"perceiver_block\" + str(n)][\n                    \"latent_transformer\"\n                ]\n                attention_weights.append(\n                    self._extract_attn_weights(cross_attns, latent_transformer)\n                )\n        return attention_weights\n\n    def _build_perceiver_block(self) -&gt; nn.ModuleDict:\n        perceiver_block = nn.ModuleDict()\n\n        # Cross Attention\n        cross_attns = nn.ModuleList()\n        for _ in range(self.n_cross_attns):\n            cross_attns.append(\n                PerceiverEncoder(\n                    self.input_dim,\n                    self.n_cross_attn_heads,\n                    False,  # use_bias\n                    self.attn_dropout,\n                    self.ff_dropout,\n                    self.ff_factor,\n                    self.transformer_activation,\n                    self.latent_dim,  # q_dim,\n                ),\n            )\n        perceiver_block[\"cross_attns\"] = cross_attns\n\n        # Latent Transformer\n        latent_transformer = nn.Sequential()\n        for i in range(self.n_latent_blocks):\n            latent_transformer.add_module(\n                \"latent_block\" + str(i),\n                PerceiverEncoder(\n                    self.latent_dim,  # input_dim\n                    self.n_latent_heads,\n                    False,  # use_bias\n                    self.attn_dropout,\n                    self.ff_dropout,\n                    self.ff_factor,\n                    self.transformer_activation,\n                ),\n            )\n        perceiver_block[\"latent_transformer\"] = latent_transformer\n\n        return perceiver_block\n\n    @staticmethod\n    def _extract_attn_weights(cross_attns, latent_transformer) -&gt; List:\n        attention_weights = []\n        for cross_attn in cross_attns:\n            attention_weights.append(cross_attn.attn.attn_weights)\n        for latent_block in latent_transformer:\n            attention_weights.append(latent_block.attn.attn_weights)\n        return attention_weights\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.transformers.tab_perceiver.TabPerceiver.output_dim","title":"output_dim  <code>property</code>","text":"<pre><code>output_dim\n</code></pre> <p>The output dimension of the model. This is a required property neccesary to build the <code>WideDeep</code> class</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.transformers.tab_perceiver.TabPerceiver.attention_weights","title":"attention_weights  <code>property</code>","text":"<pre><code>attention_weights\n</code></pre> <p>List with the attention weights. If the weights are not shared between perceiver blocks each element of the list will be a list itself containing the Cross Attention and Latent Transformer attention weights respectively</p> <p>The shape of the attention weights is:</p> <ul> <li> <p>Cross Attention: \\((N, C, L, F)\\)</p> </li> <li> <p>Latent Attention: \\((N, T, L, L)\\)</p> </li> </ul> <p>WHere \\(N\\) is the batch size, \\(C\\) is the number of Cross Attention heads, \\(L\\) is the number of Latents, \\(F\\) is the number of features/columns in the dataset and \\(T\\) is the number of Latent Attention heads</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.transformers.tab_fastformer.TabFastFormer","title":"TabFastFormer","text":"<p>               Bases: <code>BaseTabularModelWithAttention</code></p> <p>Defines an adaptation of a FastFormer that can be used as the <code>deeptabular</code> component of a Wide &amp; Deep model or independently by itself.</p> <p>Most of the parameters for this class are <code>Optional</code> since the use of categorical or continuous is in fact optional (i.e. one can use categorical features only, continuous features only or both).</p> <p> NOTE: while there are scientific publications for  the <code>TabTransformer</code>, <code>SAINT</code> and <code>FTTransformer</code>, the <code>TabPerceiver</code>  and the <code>TabFastFormer</code> are our own adaptations of the  Perceiver and the  FastFormer for tabular data.</p> <p>Parameters:</p> Name Type Description Default <code>column_idx</code> <code>Dict[str, int]</code> <p>Dict containing the index of the columns that will be passed through the <code>TabMlp</code> model. Required to slice the tensors. e.g. {'education': 0, 'relationship': 1, 'workclass': 2, ...}.</p> required <code>cat_embed_input</code> <code>Optional[List[Tuple[str, int]]]</code> <p>List of Tuples with the column name and number of unique values and embedding dimension. e.g. [(education, 11), ...]</p> <code>None</code> <code>cat_embed_dropout</code> <code>Optional[float]</code> <p>Categorical embeddings dropout. If <code>None</code>, it will default to 0.</p> <code>None</code> <code>use_cat_bias</code> <code>Optional[bool]</code> <p>Boolean indicating if bias will be used for the categorical embeddings. If <code>None</code>, it will default to 'False'.</p> <code>None</code> <code>cat_embed_activation</code> <code>Optional[str]</code> <p>Activation function for the categorical embeddings, if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> <code>None</code> <code>shared_embed</code> <code>Optional[bool]</code> <p>Boolean indicating if the embeddings will be \"shared\". The idea behind <code>shared_embed</code> is described in the Appendix A in the TabTransformer paper: 'The goal of having column embedding is to enable the model to distinguish the classes in one column from those in the other columns'. In other words, the idea is to let the model learn which column is embedded at the time. See: <code>pytorch_widedeep.models.transformers._layers.SharedEmbeddings</code>.</p> <code>None</code> <code>add_shared_embed</code> <code>Optional[bool]</code> <p>The two embedding sharing strategies are: 1) add the shared embeddings to the column embeddings or 2) to replace the first <code>frac_shared_embed</code> with the shared embeddings. See <code>pytorch_widedeep.models.embeddings_layers.SharedEmbeddings</code> If 'None' is passed, it will default to 'False'.</p> <code>None</code> <code>frac_shared_embed</code> <code>Optional[float]</code> <p>The fraction of embeddings that will be shared (if <code>add_shared_embed = False</code>) by all the different categories for one particular column. If 'None' is passed, it will default to 0.0.</p> <code>None</code> <code>continuous_cols</code> <code>Optional[List[str]]</code> <p>List with the name of the numeric (aka continuous) columns</p> <code>None</code> <code>cont_norm_layer</code> <code>Optional[Literal[batchnorm, layernorm]]</code> <p>Type of normalization layer applied to the continuous features. Options are: 'layernorm' and 'batchnorm'. if <code>None</code>, no normalization layer will be used.</p> <code>None</code> <code>embed_continuous_method</code> <code>Optional[Literal[standard, piecewise, periodic]]</code> <p>Method to use to embed the continuous features. Options are: 'standard', 'periodic' or 'piecewise'. The 'standard' embedding method is based on the FT-Transformer implementation presented in the paper: Revisiting Deep Learning Models for Tabular Data. The 'periodic' and_'piecewise'_ methods were presented in the paper: On Embeddings for Numerical Features in Tabular Deep Learning. Please, read the papers for details.</p> <code>'standard'</code> <code>cont_embed_dropout</code> <code>Optional[float]</code> <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p> <code>None</code> <code>cont_embed_activation</code> <code>Optional[str]</code> <p>Activation function for the continuous embeddings if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported. If <code>None</code>, no activation function will be applied.</p> <code>None</code> <code>quantization_setup</code> <code>Optional[Dict[str, List[float]]]</code> <p>This parameter is used when the 'piecewise' method is used to embed the continuous cols. It is a dict where keys are the name of the continuous columns and values are lists with the boundaries for the quantization of the continuous_cols. See the examples for details. If If the 'piecewise' method is used, this parameter is required.</p> <code>None</code> <code>n_frequencies</code> <code>Optional[int]</code> <p>This is the so called 'k' in their paper On Embeddings for Numerical Features in Tabular Deep Learning, and is the number of 'frequencies' that will be used to represent each continuous column. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>sigma</code> <code>Optional[float]</code> <p>This is the sigma parameter in the paper mentioned when describing the previous parameters and it is used to initialise the 'frequency weights'. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>share_last_layer</code> <code>Optional[bool]</code> <p>This parameter is not present in the before mentioned paper but it is implemented in the official repo. If <code>True</code> the linear layer that turns the frequencies into embeddings will be shared across the continuous columns. If <code>False</code> a different linear layer will be used for each continuous column. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>full_embed_dropout</code> <code>Optional[bool]</code> <p>If <code>True</code>, the full embedding corresponding to a column will be masked out/dropout. If <code>None</code>, it will default to <code>False</code>.</p> <code>None</code> <code>input_dim</code> <code>int</code> <p>The so-called dimension of the model. Is the number of embeddings used to encode the categorical and/or continuous columns</p> <code>32</code> <code>n_heads</code> <code>int</code> <p>Number of attention heads per FastFormer block</p> <code>8</code> <code>use_bias</code> <code>bool</code> <p>Boolean indicating whether or not to use bias in the Q, K, and V projection layers</p> <code>False</code> <code>n_blocks</code> <code>int</code> <p>Number of FastFormer blocks</p> <code>4</code> <code>attn_dropout</code> <code>float</code> <p>Dropout that will be applied to the Additive Attention layers</p> <code>0.1</code> <code>ff_dropout</code> <code>float</code> <p>Dropout that will be applied to the FeedForward network</p> <code>0.2</code> <code>ff_factor</code> <code>int</code> <p>Multiplicative factor applied to the first layer of the FF network in each Transformer block, This is normally set to 4.</p> <code>4</code> <code>share_qv_weights</code> <code>bool</code> <p>Following the paper, this is a boolean indicating if the Value (\\(V\\)) and the Query (\\(Q\\)) transformation parameters will be shared.</p> <code>False</code> <code>share_weights</code> <code>bool</code> <p>In addition to sharing the \\(V\\) and \\(Q\\) transformation parameters, the parameters across different Fastformer layers can also be shared. Please, see <code>pytorch_widedeep/models/tabular/transformers/tab_fastformer.py</code> for details</p> <code>False</code> <code>transformer_activation</code> <code>str</code> <p>Transformer Encoder activation function. 'tanh', 'relu', 'leaky_relu', 'gelu', 'geglu' and 'reglu' are supported</p> <code>'relu'</code> <code>mlp_hidden_dims</code> <code>Optional[List[int]]</code> <p>MLP hidden dimensions. If not provided no MLP on top of the final FTTransformer block will be used</p> <code>None</code> <code>mlp_hidden_dims</code> <code>Optional[List[int]]</code> <p>List with the number of neurons per dense layer in the MLP. e.g: [64, 32]. If not provided no MLP on top of the final Transformer block will be used.</p> <code>None</code> <code>mlp_activation</code> <code>Optional[str]</code> <p>Activation function for the dense layers of the MLP. Currently 'tanh', 'relu', 'leaky'_relu' and _'gelu' are supported. If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to 'relu'.</p> <code>None</code> <code>mlp_dropout</code> <code>Optional[float]</code> <p>float with the dropout between the dense layers of the MLP. If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to 0.0.</p> <code>None</code> <code>mlp_batchnorm</code> <code>Optional[bool]</code> <p>Boolean indicating whether or not batch normalization will be applied to the dense layers If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to False.</p> <code>None</code> <code>mlp_batchnorm_last</code> <code>Optional[bool]</code> <p>Boolean indicating whether or not batch normalization will be applied to the last of the dense layers If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to False.</p> <code>None</code> <code>mlp_linear_first</code> <code>Optional[bool]</code> <p>Boolean indicating the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code> If 'mlp_hidden_dims' is not <code>None</code> and this parameter is <code>None</code>, it will default to <code>True</code>.</p> <code>None</code> <p>Attributes:</p> Name Type Description <code>encoder</code> <code>Module</code> <p>Sequence of FasFormer blocks.</p> <code>mlp</code> <code>Module</code> <p>MLP component in the model</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import TabFastFormer\n&gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n&gt;&gt;&gt; colnames = ['a', 'b', 'c', 'd', 'e']\n&gt;&gt;&gt; cat_embed_input = [(u,i) for u,i in zip(colnames[:4], [4]*4)]\n&gt;&gt;&gt; continuous_cols = ['e']\n&gt;&gt;&gt; column_idx = {k:v for v,k in enumerate(colnames)}\n&gt;&gt;&gt; model = TabFastFormer(column_idx=column_idx, cat_embed_input=cat_embed_input, continuous_cols=continuous_cols)\n&gt;&gt;&gt; out = model(X_tab)\n</code></pre> Source code in <code>pytorch_widedeep/models/tabular/transformers/tab_fastformer.py</code> <pre><code>class TabFastFormer(BaseTabularModelWithAttention):\n    r\"\"\"Defines an adaptation of a [FastFormer](https://arxiv.org/abs/2108.09084)\n    that can be used as the `deeptabular` component of a Wide &amp; Deep model\n    or independently by itself.\n\n    Most of the parameters for this class are `Optional` since the use of\n    categorical or continuous is in fact optional (i.e. one can use\n    categorical features only, continuous features only or both).\n\n    :information_source: **NOTE**: while there are scientific publications for\n     the `TabTransformer`, `SAINT` and `FTTransformer`, the `TabPerceiver`\n     and the `TabFastFormer` are our own adaptations of the\n     [Perceiver](https://arxiv.org/abs/2103.03206) and the\n     [FastFormer](https://arxiv.org/abs/2108.09084) for tabular data.\n\n    Parameters\n    ----------\n    column_idx: Dict\n        Dict containing the index of the columns that will be passed through\n        the `TabMlp` model. Required to slice the tensors. e.g. _{'education':\n        0, 'relationship': 1, 'workclass': 2, ...}_.\n    cat_embed_input: List, Optional, default = None\n        List of Tuples with the column name and number of unique values and\n        embedding dimension. e.g. _[(education, 11), ...]_\n    cat_embed_dropout: float, Optional, default = None\n        Categorical embeddings dropout. If `None`, it will default\n        to 0.\n    use_cat_bias: bool, Optional, default = None,\n        Boolean indicating if bias will be used for the categorical embeddings.\n        If `None`, it will default to 'False'.\n    cat_embed_activation: Optional, str, default = None,\n        Activation function for the categorical embeddings, if any. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported\n    shared_embed: bool, Optional, default = None\n        Boolean indicating if the embeddings will be \"shared\". The idea behind `shared_embed` is\n        described in the Appendix A in the [TabTransformer paper](https://arxiv.org/abs/2012.06678):\n        _'The goal of having column embedding is to enable the model to\n        distinguish the classes in one column from those in the other\n        columns'_. In other words, the idea is to let the model learn which\n        column is embedded at the time. See: `pytorch_widedeep.models.transformers._layers.SharedEmbeddings`.\n    add_shared_embed: bool, Optional, default = None\n        The two embedding sharing strategies are: 1) add the shared embeddings\n        to the column embeddings or 2) to replace the first\n        `frac_shared_embed` with the shared embeddings.\n        See `pytorch_widedeep.models.embeddings_layers.SharedEmbeddings`\n        If 'None' is passed, it will default to 'False'.\n    frac_shared_embed: float, Optional, default = None\n        The fraction of embeddings that will be shared (if `add_shared_embed\n        = False`) by all the different categories for one particular\n        column. If 'None' is passed, it will default to 0.0.\n    continuous_cols: List, Optional, default = None\n        List with the name of the numeric (aka continuous) columns\n    cont_norm_layer: str, Optional, default =  None\n        Type of normalization layer applied to the continuous features.\n        Options are: _'layernorm'_ and _'batchnorm'_. if `None`, no\n        normalization layer will be used.\n    embed_continuous_method: Optional, str, default = None,\n        Method to use to embed the continuous features. Options are:\n        _'standard'_, _'periodic'_ or _'piecewise'_. The _'standard'_\n        embedding method is based on the FT-Transformer implementation\n        presented in the paper: [Revisiting Deep Learning Models for\n        Tabular Data](https://arxiv.org/abs/2106.11959v5). The _'periodic'_\n        and_'piecewise'_ methods were presented in the paper: [On Embeddings for\n        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556).\n        Please, read the papers for details.\n    cont_embed_dropout: float, Optional, default = None,\n        Dropout for the continuous embeddings. If `None`, it will default to 0.0\n    cont_embed_activation: Optional, str, default = None,\n        Activation function for the continuous embeddings if any. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported.\n        If `None`, no activation function will be applied.\n    quantization_setup: Dict[str, List[float]], Optional, default = None,\n        This parameter is used when the _'piecewise'_ method is used to embed\n        the continuous cols. It is a dict where keys are the name of the continuous\n        columns and values are lists with the boundaries for the quantization\n        of the continuous_cols. See the examples for details. If\n        If the _'piecewise'_ method is used, this parameter is required.\n    n_frequencies: int, Optional, default = None,\n        This is the so called _'k'_ in their paper [On Embeddings for\n        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556),\n        and is the number of 'frequencies' that will be used to represent each\n        continuous column. See their Eq 2 in the paper for details. If\n        the _'periodic'_ method is used, this parameter is required.\n    sigma: float, Optional, default = None,\n        This is the sigma parameter in the paper mentioned when describing the\n        previous parameters and it is used to initialise the 'frequency\n        weights'. See their Eq 2 in the paper for details. If\n        the _'periodic'_ method is used, this parameter is required.\n    share_last_layer: bool, Optional, default = None,\n        This parameter is not present in the before mentioned paper but it is implemented in\n        the [official repo](https://github.com/yandex-research/rtdl-num-embeddings/tree/main).\n        If `True` the linear layer that turns the frequencies into embeddings\n        will be shared across the continuous columns. If `False` a different\n        linear layer will be used for each continuous column.\n        If the _'periodic'_ method is used, this parameter is required.\n    full_embed_dropout: bool, Optional, default = None,\n        If `True`, the full embedding corresponding to a column will be masked\n        out/dropout. If `None`, it will default to `False`.\n    input_dim: int, default = 32\n        The so-called *dimension of the model*. Is the number of\n        embeddings used to encode the categorical and/or continuous columns\n    n_heads: int, default = 8\n        Number of attention heads per FastFormer block\n    use_bias: bool, default = False\n        Boolean indicating whether or not to use bias in the Q, K, and V\n        projection layers\n    n_blocks: int, default = 4\n        Number of FastFormer blocks\n    attn_dropout: float, default = 0.2\n        Dropout that will be applied to the Additive Attention layers\n    ff_dropout: float, default = 0.1\n        Dropout that will be applied to the FeedForward network\n    ff_factor: float, default = 4\n        Multiplicative factor applied to the first layer of the FF network in\n        each Transformer block, This is normally set to 4.\n    share_qv_weights: bool, default = False\n        Following the paper, this is a boolean indicating if the Value ($V$) and\n        the Query ($Q$) transformation parameters will be shared.\n    share_weights: bool, default = False\n        In addition to sharing the $V$ and $Q$ transformation parameters, the\n        parameters across different Fastformer layers can also be shared.\n        Please, see\n        `pytorch_widedeep/models/tabular/transformers/tab_fastformer.py` for\n        details\n    transformer_activation: str, default = \"gelu\"\n        Transformer Encoder activation function. _'tanh'_, _'relu'_,\n        _'leaky_relu'_, _'gelu'_, _'geglu'_ and _'reglu'_ are supported\n    mlp_hidden_dims: List, Optional, default = None\n        MLP hidden dimensions. If not provided no MLP on top of the final\n        FTTransformer block will be used\n    mlp_hidden_dims: List, Optional, default = None\n        List with the number of neurons per dense layer in the MLP. e.g:\n        _[64, 32]_. If not provided no MLP on top of the final\n        Transformer block will be used.\n    mlp_activation: str, Optional, default = None\n        Activation function for the dense layers of the MLP. Currently\n        _'tanh'_, _'relu'_, _'leaky'_relu' and _'gelu'_ are supported.\n        If 'mlp_hidden_dims' is not `None` and this parameter is `None`, it\n        will default to _'relu'_.\n    mlp_dropout: float, Optional, default = None\n        float with the dropout between the dense layers of the MLP.\n        If 'mlp_hidden_dims' is not `None` and this parameter is `None`, it\n        will default to 0.0.\n    mlp_batchnorm: bool, Optional, default = None\n        Boolean indicating whether or not batch normalization will be applied\n        to the dense layers\n        If 'mlp_hidden_dims' is not `None` and this parameter is `None`, it\n        will default to False.\n    mlp_batchnorm_last: bool, Optional, default = None\n        Boolean indicating whether or not batch normalization will be applied\n        to the last of the dense layers\n        If 'mlp_hidden_dims' is not `None` and this parameter is `None`, it\n        will default to False.\n    mlp_linear_first: bool, Optional, default = None\n        Boolean indicating the order of the operations in the dense\n        layer. If `True: [LIN -&gt; ACT -&gt; BN -&gt; DP]`. If `False: [BN -&gt; DP -&gt;\n        LIN -&gt; ACT]`\n        If 'mlp_hidden_dims' is not `None` and this parameter is `None`, it\n        will default to `True`.\n\n    Attributes\n    ----------\n    encoder: nn.Module\n        Sequence of FasFormer blocks.\n    mlp: nn.Module\n        MLP component in the model\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.models import TabFastFormer\n    &gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n    &gt;&gt;&gt; colnames = ['a', 'b', 'c', 'd', 'e']\n    &gt;&gt;&gt; cat_embed_input = [(u,i) for u,i in zip(colnames[:4], [4]*4)]\n    &gt;&gt;&gt; continuous_cols = ['e']\n    &gt;&gt;&gt; column_idx = {k:v for v,k in enumerate(colnames)}\n    &gt;&gt;&gt; model = TabFastFormer(column_idx=column_idx, cat_embed_input=cat_embed_input, continuous_cols=continuous_cols)\n    &gt;&gt;&gt; out = model(X_tab)\n    \"\"\"\n\n    def __init__(\n        self,\n        column_idx: Dict[str, int],\n        *,\n        cat_embed_input: Optional[List[Tuple[str, int]]] = None,\n        cat_embed_dropout: Optional[float] = None,\n        use_cat_bias: Optional[bool] = None,\n        cat_embed_activation: Optional[str] = None,\n        shared_embed: Optional[bool] = None,\n        add_shared_embed: Optional[bool] = None,\n        frac_shared_embed: Optional[float] = None,\n        continuous_cols: Optional[List[str]] = None,\n        cont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\n        embed_continuous_method: Optional[\n            Literal[\"standard\", \"piecewise\", \"periodic\"]\n        ] = \"standard\",\n        cont_embed_dropout: Optional[float] = None,\n        cont_embed_activation: Optional[str] = None,\n        quantization_setup: Optional[Dict[str, List[float]]] = None,\n        n_frequencies: Optional[int] = None,\n        sigma: Optional[float] = None,\n        share_last_layer: Optional[bool] = None,\n        full_embed_dropout: Optional[bool] = None,\n        input_dim: int = 32,\n        n_heads: int = 8,\n        use_bias: bool = False,\n        n_blocks: int = 4,\n        attn_dropout: float = 0.1,\n        ff_dropout: float = 0.2,\n        ff_factor: int = 4,\n        share_qv_weights: bool = False,\n        share_weights: bool = False,\n        transformer_activation: str = \"relu\",\n        mlp_hidden_dims: Optional[List[int]] = None,\n        mlp_activation: Optional[str] = None,\n        mlp_dropout: Optional[float] = None,\n        mlp_batchnorm: Optional[bool] = None,\n        mlp_batchnorm_last: Optional[bool] = None,\n        mlp_linear_first: Optional[bool] = None,\n    ):\n        super(TabFastFormer, self).__init__(\n            column_idx=column_idx,\n            cat_embed_input=cat_embed_input,\n            cat_embed_dropout=cat_embed_dropout,\n            use_cat_bias=use_cat_bias,\n            cat_embed_activation=cat_embed_activation,\n            shared_embed=shared_embed,\n            add_shared_embed=add_shared_embed,\n            frac_shared_embed=frac_shared_embed,\n            continuous_cols=continuous_cols,\n            cont_norm_layer=cont_norm_layer,\n            embed_continuous_method=embed_continuous_method,\n            cont_embed_dropout=cont_embed_dropout,\n            cont_embed_activation=cont_embed_activation,\n            input_dim=input_dim,\n            quantization_setup=quantization_setup,\n            n_frequencies=n_frequencies,\n            sigma=sigma,\n            share_last_layer=share_last_layer,\n            full_embed_dropout=full_embed_dropout,\n        )\n\n        self.n_heads = n_heads\n        self.use_bias = use_bias\n        self.n_blocks = n_blocks\n        self.attn_dropout = attn_dropout\n        self.ff_dropout = ff_dropout\n        self.ff_factor = ff_factor\n        self.share_qv_weights = share_qv_weights\n        self.share_weights = share_weights\n        self.transformer_activation = transformer_activation\n\n        self.mlp_hidden_dims = mlp_hidden_dims\n        self.mlp_activation = mlp_activation\n        self.mlp_dropout = mlp_dropout\n        self.mlp_batchnorm = mlp_batchnorm\n        self.mlp_batchnorm_last = mlp_batchnorm_last\n        self.mlp_linear_first = mlp_linear_first\n\n        self.with_cls_token = \"cls_token\" in column_idx\n        self.n_cat = len(cat_embed_input) if cat_embed_input is not None else 0\n        self.n_cont = len(continuous_cols) if continuous_cols is not None else 0\n        self.n_feats = self.n_cat + self.n_cont\n\n        # Embeddings are instantiated at the base model\n        # Transformer blocks\n        self.encoder = nn.Sequential()\n        first_fastformer_block = FastFormerEncoder(\n            input_dim,\n            n_heads,\n            use_bias,\n            attn_dropout,\n            ff_dropout,\n            ff_factor,\n            share_qv_weights,\n            transformer_activation,\n        )\n        self.encoder.add_module(\"fastformer_block0\", first_fastformer_block)\n        for i in range(1, n_blocks):\n            if share_weights:\n                self.encoder.add_module(\n                    \"fastformer_block\" + str(i), first_fastformer_block\n                )\n            else:\n                self.encoder.add_module(\n                    \"fastformer_block\" + str(i),\n                    FastFormerEncoder(\n                        input_dim,\n                        n_heads,\n                        use_bias,\n                        attn_dropout,\n                        ff_dropout,\n                        ff_factor,\n                        share_qv_weights,\n                        transformer_activation,\n                    ),\n                )\n\n        self.mlp_first_hidden_dim = (\n            self.input_dim if self.with_cls_token else (self.n_feats * self.input_dim)\n        )\n\n        # Mlp: adding an MLP on top of the Resnet blocks is optional and\n        # therefore all related params are optional\n        if self.mlp_hidden_dims is not None:\n            self.mlp = MLP(\n                d_hidden=[self.mlp_first_hidden_dim] + self.mlp_hidden_dims,\n                activation=(\n                    \"relu\" if self.mlp_activation is None else self.mlp_activation\n                ),\n                dropout=0.0 if self.mlp_dropout is None else self.mlp_dropout,\n                batchnorm=False if self.mlp_batchnorm is None else self.mlp_batchnorm,\n                batchnorm_last=(\n                    False\n                    if self.mlp_batchnorm_last is None\n                    else self.mlp_batchnorm_last\n                ),\n                linear_first=(\n                    False if self.mlp_linear_first is None else self.mlp_linear_first\n                ),\n            )\n        else:\n            self.mlp = None\n\n    def forward(self, X: Tensor) -&gt; Tensor:\n        x = self._get_embeddings(X)\n        x = self.encoder(x)\n        if self.with_cls_token:\n            x = x[:, 0, :]\n        else:\n            x = x.flatten(1)\n        if self.mlp is not None:\n            x = self.mlp(x)\n        return x\n\n    @property\n    def output_dim(self) -&gt; int:\n        r\"\"\"The output dimension of the model. This is a required property\n        neccesary to build the `WideDeep` class\n        \"\"\"\n        return (\n            self.mlp_hidden_dims[-1]\n            if self.mlp_hidden_dims is not None\n            else self.mlp_first_hidden_dim\n        )\n\n    @property\n    def attention_weights(self) -&gt; List[Tuple[Tensor, Tensor]]:\n        r\"\"\"List with the attention weights. Each element of the list is a\n        tuple where the first and second elements are the $\\alpha$\n        and $\\beta$ attention weights in the paper.\n\n        The shape of the attention weights is $(N, H, F)$ where $N$ is the\n        batch size, $H$ is the number of attention heads and $F$ is the\n        number of features/columns in the dataset\n        \"\"\"\n        if self.share_weights:\n            attention_weights = [self.encoder[0].attn.attn_weight]\n        else:\n            attention_weights = [blk.attn.attn_weights for blk in self.encoder]\n        return attention_weights\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.transformers.tab_fastformer.TabFastFormer.output_dim","title":"output_dim  <code>property</code>","text":"<pre><code>output_dim\n</code></pre> <p>The output dimension of the model. This is a required property neccesary to build the <code>WideDeep</code> class</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.tabular.transformers.tab_fastformer.TabFastFormer.attention_weights","title":"attention_weights  <code>property</code>","text":"<pre><code>attention_weights\n</code></pre> <p>List with the attention weights. Each element of the list is a tuple where the first and second elements are the \\(\\alpha\\) and \\(\\beta\\) attention weights in the paper.</p> <p>The shape of the attention weights is \\((N, H, F)\\) where \\(N\\) is the batch size, \\(H\\) is the number of attention heads and \\(F\\) is the number of features/columns in the dataset</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.text.rnns.basic_rnn.BasicRNN","title":"BasicRNN","text":"<p>               Bases: <code>BaseWDModelComponent</code></p> <p>Standard text classifier/regressor comprised by a stack of RNNs (LSTMs or GRUs) that can be used as the <code>deeptext</code> component of a Wide &amp; Deep model or independently by itself.</p> <p>In addition, there is the option to add a Fully Connected (FC) set of dense layers on top of the stack of RNNs</p> <p>Parameters:</p> Name Type Description Default <code>vocab_size</code> <code>int</code> <p>Number of words in the vocabulary</p> required <code>embed_dim</code> <code>Optional[int]</code> <p>Dimension of the word embeddings if non-pretained word vectors are used</p> <code>None</code> <code>embed_matrix</code> <code>Optional[ndarray]</code> <p>Pretrained word embeddings</p> <code>None</code> <code>embed_trainable</code> <code>bool</code> <p>Boolean indicating if the pretrained embeddings are trainable</p> <code>True</code> <code>rnn_type</code> <code>Literal[lstm, gru]</code> <p>String indicating the type of RNN to use. One of 'lstm' or 'gru'</p> <code>'lstm'</code> <code>hidden_dim</code> <code>int</code> <p>Hidden dim of the RNN</p> <code>64</code> <code>n_layers</code> <code>int</code> <p>Number of recurrent layers</p> <code>3</code> <code>rnn_dropout</code> <code>float</code> <p>Dropout for each RNN layer except the last layer</p> <code>0.0</code> <code>bidirectional</code> <code>bool</code> <p>Boolean indicating whether the staked RNNs are bidirectional</p> <code>False</code> <code>use_hidden_state</code> <code>bool</code> <p>Boolean indicating whether to use the final hidden state or the RNN's output as predicting features. Typically the former is used.</p> <code>True</code> <code>padding_idx</code> <code>int</code> <p>index of the padding token in the padded-tokenised sequences. The <code>TextPreprocessor</code> class within this library uses fastai's tokenizer where the token index 0 is reserved for the 'unknown' word token. Therefore, the default value is set to 1.</p> <code>1</code> <code>head_hidden_dims</code> <code>Optional[List[int]]</code> <p>List with the sizes of the dense layers in the head e.g: [128, 64]</p> <code>None</code> <code>head_activation</code> <code>str</code> <p>Activation function for the dense layers in the head. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> <code>'relu'</code> <code>head_dropout</code> <code>Optional[float]</code> <p>Dropout of the dense layers in the head</p> <code>None</code> <code>head_batchnorm</code> <code>bool</code> <p>Boolean indicating whether or not to include batch normalization in the dense layers that form the 'rnn_mlp'</p> <code>False</code> <code>head_batchnorm_last</code> <code>bool</code> <p>Boolean indicating whether or not to apply batch normalization to the last of the dense layers in the head</p> <code>False</code> <code>head_linear_first</code> <code>bool</code> <p>Boolean indicating whether the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code></p> <code>False</code> <p>Attributes:</p> Name Type Description <code>word_embed</code> <code>Module</code> <p>word embedding matrix</p> <code>rnn</code> <code>Module</code> <p>Stack of RNNs</p> <code>rnn_mlp</code> <code>Module</code> <p>Stack of dense layers on top of the RNN. This will only exists if <code>head_layers_dim</code> is not None</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import BasicRNN\n&gt;&gt;&gt; X_text = torch.cat((torch.zeros([5,1]), torch.empty(5, 4).random_(1,4)), axis=1)\n&gt;&gt;&gt; model = BasicRNN(vocab_size=4, hidden_dim=4, n_layers=2, padding_idx=0, embed_dim=4)\n&gt;&gt;&gt; out = model(X_text)\n</code></pre> Source code in <code>pytorch_widedeep/models/text/rnns/basic_rnn.py</code> <pre><code>class BasicRNN(BaseWDModelComponent):\n    r\"\"\"Standard text classifier/regressor comprised by a stack of RNNs\n    (LSTMs or GRUs) that can be used as the `deeptext` component of a Wide &amp;\n    Deep model or independently by itself.\n\n    In addition, there is the option to add a Fully Connected (FC) set of\n    dense layers on top of the stack of RNNs\n\n    Parameters\n    ----------\n    vocab_size: int\n        Number of words in the vocabulary\n    embed_dim: int, Optional, default = None\n        Dimension of the word embeddings if non-pretained word vectors are\n        used\n    embed_matrix: np.ndarray, Optional, default = None\n        Pretrained word embeddings\n    embed_trainable: bool, default = True\n        Boolean indicating if the pretrained embeddings are trainable\n    rnn_type: str, default = 'lstm'\n        String indicating the type of RNN to use. One of _'lstm'_ or _'gru'_\n    hidden_dim: int, default = 64\n        Hidden dim of the RNN\n    n_layers: int, default = 3\n        Number of recurrent layers\n    rnn_dropout: float, default = 0.1\n        Dropout for each RNN layer except the last layer\n    bidirectional: bool, default = True\n        Boolean indicating whether the staked RNNs are bidirectional\n    use_hidden_state: str, default = True\n        Boolean indicating whether to use the final hidden state or the RNN's\n        output as predicting features. Typically the former is used.\n    padding_idx: int, default = 1\n        index of the padding token in the padded-tokenised sequences. The\n        `TextPreprocessor` class within this library uses fastai's tokenizer\n        where the token index 0 is reserved for the _'unknown'_ word token.\n        Therefore, the default value is set to 1.\n    head_hidden_dims: List, Optional, default = None\n        List with the sizes of the dense layers in the head e.g: _[128, 64]_\n    head_activation: str, default = \"relu\"\n        Activation function for the dense layers in the head. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported\n    head_dropout: float, Optional, default = None\n        Dropout of the dense layers in the head\n    head_batchnorm: bool, default = False\n        Boolean indicating whether or not to include batch normalization in\n        the dense layers that form the _'rnn_mlp'_\n    head_batchnorm_last: bool, default = False\n        Boolean indicating whether or not to apply batch normalization to the\n        last of the dense layers in the head\n    head_linear_first: bool, default = False\n        Boolean indicating whether the order of the operations in the dense\n        layer. If `True: [LIN -&gt; ACT -&gt; BN -&gt; DP]`. If `False: [BN -&gt; DP -&gt;\n        LIN -&gt; ACT]`\n\n    Attributes\n    ----------\n    word_embed: nn.Module\n        word embedding matrix\n    rnn: nn.Module\n        Stack of RNNs\n    rnn_mlp: nn.Module\n        Stack of dense layers on top of the RNN. This will only exists if\n        `head_layers_dim` is not None\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.models import BasicRNN\n    &gt;&gt;&gt; X_text = torch.cat((torch.zeros([5,1]), torch.empty(5, 4).random_(1,4)), axis=1)\n    &gt;&gt;&gt; model = BasicRNN(vocab_size=4, hidden_dim=4, n_layers=2, padding_idx=0, embed_dim=4)\n    &gt;&gt;&gt; out = model(X_text)\n    \"\"\"\n\n    def __init__(\n        self,\n        vocab_size: int,\n        embed_dim: Optional[int] = None,\n        embed_matrix: Optional[np.ndarray] = None,\n        embed_trainable: bool = True,\n        rnn_type: Literal[\"lstm\", \"gru\"] = \"lstm\",\n        hidden_dim: int = 64,\n        n_layers: int = 3,\n        rnn_dropout: float = 0.0,\n        bidirectional: bool = False,\n        use_hidden_state: bool = True,\n        padding_idx: int = 1,\n        head_hidden_dims: Optional[List[int]] = None,\n        head_activation: str = \"relu\",\n        head_dropout: Optional[float] = None,\n        head_batchnorm: bool = False,\n        head_batchnorm_last: bool = False,\n        head_linear_first: bool = False,\n    ):\n        super(BasicRNN, self).__init__()\n\n        if embed_dim is None and embed_matrix is None:\n            raise ValueError(\n                \"If no 'embed_matrix' is passed, the embedding dimension must\"\n                \"be specified with 'embed_dim'\"\n            )\n\n        if rnn_type.lower() not in [\"lstm\", \"gru\"]:\n            raise ValueError(\n                f\"'rnn_type' must be 'lstm' or 'gru', got {rnn_type} instead\"\n            )\n\n        if (\n            embed_dim is not None\n            and embed_matrix is not None\n            and not embed_dim == embed_matrix.shape[1]\n        ):\n            warnings.warn(\n                \"the input embedding dimension {} and the dimension of the \"\n                \"pretrained embeddings {} do not match. The pretrained embeddings \"\n                \"dimension ({}) will be used\".format(\n                    embed_dim, embed_matrix.shape[1], embed_matrix.shape[1]\n                ),\n                UserWarning,\n            )\n\n        self.vocab_size = vocab_size\n        self.embed_trainable = embed_trainable\n        self.embed_dim = embed_dim\n\n        self.rnn_type = rnn_type\n        self.hidden_dim = hidden_dim\n        self.n_layers = n_layers\n        self.rnn_dropout = rnn_dropout\n        self.bidirectional = bidirectional\n        self.use_hidden_state = use_hidden_state\n        self.padding_idx = padding_idx\n\n        self.head_hidden_dims = head_hidden_dims\n        self.head_activation = head_activation\n        self.head_dropout = head_dropout\n        self.head_batchnorm = head_batchnorm\n        self.head_batchnorm_last = head_batchnorm_last\n        self.head_linear_first = head_linear_first\n\n        # Embeddings\n        if embed_matrix is not None:\n            self.word_embed, self.embed_dim = self._set_embeddings(embed_matrix)\n        else:\n            assert self.embed_dim is not None\n            self.word_embed = nn.Embedding(\n                self.vocab_size, self.embed_dim, padding_idx=self.padding_idx\n            )\n\n        # RNN\n        rnn_params = {\n            \"input_size\": self.embed_dim,\n            \"hidden_size\": hidden_dim,\n            \"num_layers\": n_layers,\n            \"bidirectional\": bidirectional,\n            \"dropout\": rnn_dropout,\n            \"batch_first\": True,\n        }\n        if self.rnn_type.lower() == \"lstm\":\n            self.rnn: Union[nn.LSTM, nn.GRU] = nn.LSTM(**rnn_params)\n        elif self.rnn_type.lower() == \"gru\":\n            self.rnn = nn.GRU(**rnn_params)\n        else:\n            raise ValueError(\n                f\"'rnn_type' must be 'lstm' or 'gru', got {self.rnn_type} instead\"\n            )\n\n        self.rnn_output_dim = hidden_dim * 2 if bidirectional else hidden_dim\n\n        # FC-Head (Mlp)\n        if self.head_hidden_dims is not None:\n            head_hidden_dims = [self.rnn_output_dim] + self.head_hidden_dims\n            self.rnn_mlp: Union[MLP, nn.Identity] = MLP(\n                head_hidden_dims,\n                head_activation,\n                head_dropout,\n                head_batchnorm,\n                head_batchnorm_last,\n                head_linear_first,\n            )\n        else:\n            # simple hack to add readability in the forward pass\n            self.rnn_mlp = nn.Identity()\n\n    def forward(self, X: Tensor) -&gt; Tensor:\n        embed = self.word_embed(X.long())\n\n        if self.rnn_type.lower() == \"lstm\":\n            o, (h, c) = self.rnn(embed)\n        elif self.rnn_type.lower() == \"gru\":\n            o, h = self.rnn(embed)\n        else:\n            raise ValueError(\n                f\"'rnn_type' must be 'lstm' or 'gru', got {self.rnn_type} instead\"\n            )\n\n        processed_outputs = self._process_rnn_outputs(o, h)\n\n        return self.rnn_mlp(processed_outputs)\n\n    @property\n    def output_dim(self) -&gt; int:\n        r\"\"\"The output dimension of the model. This is a required property\n        neccesary to build the `WideDeep` class\n        \"\"\"\n        return (\n            self.head_hidden_dims[-1]\n            if self.head_hidden_dims is not None\n            else self.rnn_output_dim\n        )\n\n    def _set_embeddings(\n        self, embed_matrix: Optional[np.ndarray] = None\n    ) -&gt; Tuple[nn.Module, int]:\n        if embed_matrix is not None:\n            assert (\n                embed_matrix.dtype == \"float32\"\n            ), \"'embed_matrix' must be of dtype 'float32', got dtype '{}'\".format(\n                str(embed_matrix.dtype)\n            )\n            word_embed = nn.Embedding(\n                self.vocab_size, embed_matrix.shape[1], padding_idx=self.padding_idx\n            )\n            if self.embed_trainable:\n                word_embed.weight = nn.Parameter(\n                    torch.tensor(embed_matrix), requires_grad=True\n                )\n            else:\n                word_embed.weight = nn.Parameter(\n                    torch.tensor(embed_matrix), requires_grad=False\n                )\n            embed_dim = embed_matrix.shape[1]\n        else:\n            assert self.embed_dim is not None\n            word_embed = nn.Embedding(\n                self.vocab_size, self.embed_dim, padding_idx=self.padding_idx\n            )\n            embed_dim = self.embed_dim\n\n        return word_embed, embed_dim\n\n    def _process_rnn_outputs(self, output: Tensor, hidden: Tensor) -&gt; Tensor:\n        output = output.permute(1, 0, 2)\n        if self.bidirectional:\n            processed_outputs = (\n                torch.cat((hidden[-2], hidden[-1]), dim=1)\n                if self.use_hidden_state\n                else output[-1]\n            )\n        else:\n            processed_outputs = hidden[-1] if self.use_hidden_state else output[-1]\n\n        return processed_outputs\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.text.rnns.basic_rnn.BasicRNN.output_dim","title":"output_dim  <code>property</code>","text":"<pre><code>output_dim\n</code></pre> <p>The output dimension of the model. This is a required property neccesary to build the <code>WideDeep</code> class</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.text.rnns.attentive_rnn.AttentiveRNN","title":"AttentiveRNN","text":"<p>               Bases: <code>BasicRNN</code></p> <p>Text classifier/regressor comprised by a stack of RNNs (LSTMs or GRUs) plus an attention layer. This model can be used as the <code>deeptext</code> component of a Wide &amp; Deep model or independently by itself.</p> <p>In addition, there is the option to add a Fully Connected (FC) set of dense layers on top of attention layer</p> <p>Parameters:</p> Name Type Description Default <code>vocab_size</code> <code>int</code> <p>Number of words in the vocabulary</p> required <code>embed_dim</code> <code>Optional[int]</code> <p>Dimension of the word embeddings if non-pretained word vectors are used</p> <code>None</code> <code>embed_matrix</code> <code>Optional[ndarray]</code> <p>Pretrained word embeddings</p> <code>None</code> <code>embed_trainable</code> <code>bool</code> <p>Boolean indicating if the pretrained embeddings are trainable</p> <code>True</code> <code>rnn_type</code> <code>Literal[lstm, gru]</code> <p>String indicating the type of RNN to use. One of 'lstm' or 'gru'</p> <code>'lstm'</code> <code>hidden_dim</code> <code>int</code> <p>Hidden dim of the RNN</p> <code>64</code> <code>n_layers</code> <code>int</code> <p>Number of recurrent layers</p> <code>3</code> <code>rnn_dropout</code> <code>float</code> <p>Dropout for each RNN layer except the last layer</p> <code>0.1</code> <code>bidirectional</code> <code>bool</code> <p>Boolean indicating whether the staked RNNs are bidirectional</p> <code>False</code> <code>use_hidden_state</code> <code>bool</code> <p>Boolean indicating whether to use the final hidden state or the RNN's output as predicting features. Typically the former is used.</p> <code>True</code> <code>padding_idx</code> <code>int</code> <p>index of the padding token in the padded-tokenised sequences. The <code>TextPreprocessor</code> class within this library uses fastai's tokenizer where the token index 0 is reserved for the 'unknown' word token. Therefore, the default value is set to 1.</p> <code>1</code> <code>attn_concatenate</code> <code>bool</code> <p>Boolean indicating if the input to the attention mechanism will be the output of the RNN or the output of the RNN concatenated with the last hidden state.</p> <code>True</code> <code>attn_dropout</code> <code>float</code> <p>Internal dropout for the attention mechanism</p> <code>0.1</code> <code>head_hidden_dims</code> <code>Optional[List[int]]</code> <p>List with the sizes of the dense layers in the head e.g: [128, 64]</p> <code>None</code> <code>head_activation</code> <code>str</code> <p>Activation function for the dense layers in the head. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> <code>'relu'</code> <code>head_dropout</code> <code>Optional[float]</code> <p>Dropout of the dense layers in the head</p> <code>None</code> <code>head_batchnorm</code> <code>bool</code> <p>Boolean indicating whether or not to include batch normalization in the dense layers that form the 'rnn_mlp'</p> <code>False</code> <code>head_batchnorm_last</code> <code>bool</code> <p>Boolean indicating whether or not to apply batch normalization to the last of the dense layers in the head</p> <code>False</code> <code>head_linear_first</code> <code>bool</code> <p>Boolean indicating whether the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code></p> <code>False</code> <p>Attributes:</p> Name Type Description <code>word_embed</code> <code>Module</code> <p>word embedding matrix</p> <code>rnn</code> <code>Module</code> <p>Stack of RNNs</p> <code>rnn_mlp</code> <code>Module</code> <p>Stack of dense layers on top of the RNN. This will only exists if <code>head_layers_dim</code> is not <code>None</code></p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import AttentiveRNN\n&gt;&gt;&gt; X_text = torch.cat((torch.zeros([5,1]), torch.empty(5, 4).random_(1,4)), axis=1)\n&gt;&gt;&gt; model = AttentiveRNN(vocab_size=4, hidden_dim=4, n_layers=2, padding_idx=0, embed_dim=4)\n&gt;&gt;&gt; out = model(X_text)\n</code></pre> Source code in <code>pytorch_widedeep/models/text/rnns/attentive_rnn.py</code> <pre><code>class AttentiveRNN(BasicRNN):\n    r\"\"\"Text classifier/regressor comprised by a stack of RNNs\n    (LSTMs or GRUs) plus an attention layer. This model can be used as the\n    `deeptext` component of a Wide &amp; Deep model or independently by\n    itself.\n\n    In addition, there is the option to add a Fully Connected (FC) set of dense\n    layers on top of attention layer\n\n    Parameters\n    ----------\n    vocab_size: int\n        Number of words in the vocabulary\n    embed_dim: int, Optional, default = None\n        Dimension of the word embeddings if non-pretained word vectors are\n        used\n    embed_matrix: np.ndarray, Optional, default = None\n        Pretrained word embeddings\n    embed_trainable: bool, default = True\n        Boolean indicating if the pretrained embeddings are trainable\n    rnn_type: str, default = 'lstm'\n        String indicating the type of RNN to use. One of _'lstm'_ or _'gru'_\n    hidden_dim: int, default = 64\n        Hidden dim of the RNN\n    n_layers: int, default = 3\n        Number of recurrent layers\n    rnn_dropout: float, default = 0.1\n        Dropout for each RNN layer except the last layer\n    bidirectional: bool, default = True\n        Boolean indicating whether the staked RNNs are bidirectional\n    use_hidden_state: str, default = True\n        Boolean indicating whether to use the final hidden state or the RNN's\n        output as predicting features. Typically the former is used.\n    padding_idx: int, default = 1\n        index of the padding token in the padded-tokenised sequences. The\n        `TextPreprocessor` class within this library uses fastai's\n        tokenizer where the token index 0 is reserved for the _'unknown'_\n        word token. Therefore, the default value is set to 1.\n    attn_concatenate: bool, default = True\n        Boolean indicating if the input to the attention mechanism will be the\n        output of the RNN or the output of the RNN concatenated with the last\n        hidden state.\n    attn_dropout: float, default = 0.1\n        Internal dropout for the attention mechanism\n    head_hidden_dims: List, Optional, default = None\n        List with the sizes of the dense layers in the head e.g: _[128, 64]_\n    head_activation: str, default = \"relu\"\n        Activation function for the dense layers in the head. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported\n    head_dropout: float, Optional, default = None\n        Dropout of the dense layers in the head\n    head_batchnorm: bool, default = False\n        Boolean indicating whether or not to include batch normalization in\n        the dense layers that form the _'rnn_mlp'_\n    head_batchnorm_last: bool, default = False\n        Boolean indicating whether or not to apply batch normalization to the\n        last of the dense layers in the head\n    head_linear_first: bool, default = False\n        Boolean indicating whether the order of the operations in the dense\n        layer. If `True: [LIN -&gt; ACT -&gt; BN -&gt; DP]`. If `False: [BN -&gt; DP -&gt;\n        LIN -&gt; ACT]`\n\n    Attributes\n    ----------\n    word_embed: nn.Module\n        word embedding matrix\n    rnn: nn.Module\n        Stack of RNNs\n    rnn_mlp: nn.Module\n        Stack of dense layers on top of the RNN. This will only exists if\n        `head_layers_dim` is not `None`\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.models import AttentiveRNN\n    &gt;&gt;&gt; X_text = torch.cat((torch.zeros([5,1]), torch.empty(5, 4).random_(1,4)), axis=1)\n    &gt;&gt;&gt; model = AttentiveRNN(vocab_size=4, hidden_dim=4, n_layers=2, padding_idx=0, embed_dim=4)\n    &gt;&gt;&gt; out = model(X_text)\n    \"\"\"\n\n    def __init__(\n        self,\n        vocab_size: int,\n        embed_dim: Optional[int] = None,\n        embed_matrix: Optional[np.ndarray] = None,\n        embed_trainable: bool = True,\n        rnn_type: Literal[\"lstm\", \"gru\"] = \"lstm\",\n        hidden_dim: int = 64,\n        n_layers: int = 3,\n        rnn_dropout: float = 0.1,\n        bidirectional: bool = False,\n        use_hidden_state: bool = True,\n        padding_idx: int = 1,\n        attn_concatenate: bool = True,\n        attn_dropout: float = 0.1,\n        head_hidden_dims: Optional[List[int]] = None,\n        head_activation: str = \"relu\",\n        head_dropout: Optional[float] = None,\n        head_batchnorm: bool = False,\n        head_batchnorm_last: bool = False,\n        head_linear_first: bool = False,\n    ):\n        super(AttentiveRNN, self).__init__(\n            vocab_size=vocab_size,\n            embed_dim=embed_dim,\n            embed_matrix=embed_matrix,\n            embed_trainable=embed_trainable,\n            rnn_type=rnn_type,\n            hidden_dim=hidden_dim,\n            n_layers=n_layers,\n            rnn_dropout=rnn_dropout,\n            bidirectional=bidirectional,\n            use_hidden_state=use_hidden_state,\n            padding_idx=padding_idx,\n            head_hidden_dims=head_hidden_dims,\n            head_activation=head_activation,\n            head_dropout=head_dropout,\n            head_batchnorm=head_batchnorm,\n            head_batchnorm_last=head_batchnorm_last,\n            head_linear_first=head_linear_first,\n        )\n\n        # Embeddings and RNN defined in the BasicRNN inherited class\n\n        # Attention\n        self.attn_concatenate = attn_concatenate\n        self.attn_dropout = attn_dropout\n\n        if bidirectional and attn_concatenate:\n            self.rnn_output_dim = hidden_dim * 4\n        elif bidirectional or attn_concatenate:\n            self.rnn_output_dim = hidden_dim * 2\n        else:\n            self.rnn_output_dim = hidden_dim\n        self.attn = ContextAttention(\n            self.rnn_output_dim, attn_dropout, sum_along_seq=True\n        )\n\n        # FC-Head (Mlp)\n        if self.head_hidden_dims is not None:\n            head_hidden_dims = [self.rnn_output_dim] + self.head_hidden_dims\n            self.rnn_mlp = MLP(\n                head_hidden_dims,\n                head_activation,\n                head_dropout,\n                head_batchnorm,\n                head_batchnorm_last,\n                head_linear_first,\n            )\n\n    def _process_rnn_outputs(self, output: Tensor, hidden: Tensor) -&gt; Tensor:\n        if self.attn_concatenate:\n            if self.bidirectional:\n                bi_hidden = torch.cat((hidden[-2], hidden[-1]), dim=1)\n                attn_inp = torch.cat(\n                    [output, bi_hidden.unsqueeze(1).expand_as(output)], dim=2\n                )\n            else:\n                attn_inp = torch.cat(\n                    [output, hidden[-1].unsqueeze(1).expand_as(output)], dim=2\n                )\n        else:\n            attn_inp = output\n\n        return self.attn(attn_inp)\n\n    @property\n    def attention_weights(self) -&gt; List:\n        r\"\"\"List with the attention weights\n\n        The shape of the attention weights is $(N, S)$, where $N$ is the batch\n        size and $S$ is the length of the sequence\n        \"\"\"\n        return self.attn.attn_weights\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.text.rnns.attentive_rnn.AttentiveRNN.attention_weights","title":"attention_weights  <code>property</code>","text":"<pre><code>attention_weights\n</code></pre> <p>List with the attention weights</p> <p>The shape of the attention weights is \\((N, S)\\), where \\(N\\) is the batch size and \\(S\\) is the length of the sequence</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.text.rnns.stacked_attentive_rnn.StackedAttentiveRNN","title":"StackedAttentiveRNN","text":"<p>               Bases: <code>BaseWDModelComponent</code></p> <p>Text classifier/regressor comprised by a stack of blocks: <code>[RNN + Attention]</code>. This can be used as the <code>deeptext</code> component of a Wide &amp; Deep model or independently by itself.</p> <p>In addition, there is the option to add a Fully Connected (FC) set of dense layers on top of the attentiob blocks</p> <p>Parameters:</p> Name Type Description Default <code>vocab_size</code> <code>int</code> <p>Number of words in the vocabulary</p> required <code>embed_dim</code> <code>Optional[int]</code> <p>Dimension of the word embeddings if non-pretained word vectors are used</p> <code>None</code> <code>embed_matrix</code> <code>Optional[ndarray]</code> <p>Pretrained word embeddings</p> <code>None</code> <code>embed_trainable</code> <code>bool</code> <p>Boolean indicating if the pretrained embeddings are trainable</p> <code>True</code> <code>rnn_type</code> <code>Literal[lstm, gru]</code> <p>String indicating the type of RNN to use. One of 'lstm' or 'gru'</p> <code>'lstm'</code> <code>hidden_dim</code> <code>int</code> <p>Hidden dim of the RNN</p> <code>64</code> <code>bidirectional</code> <code>bool</code> <p>Boolean indicating whether the staked RNNs are bidirectional</p> <code>False</code> <code>padding_idx</code> <code>int</code> <p>index of the padding token in the padded-tokenised sequences. The <code>TextPreprocessor</code> class within this library uses fastai's tokenizer where the token index 0 is reserved for the 'unknown' word token. Therefore, the default value is set to 1.</p> <code>1</code> <code>n_blocks</code> <code>int</code> <p>Number of attention blocks. Each block is comprised by an RNN and a Context Attention Encoder</p> <code>3</code> <code>attn_concatenate</code> <code>bool</code> <p>Boolean indicating if the input to the attention mechanism will be the output of the RNN or the output of the RNN concatenated with the last hidden state or simply</p> <code>False</code> <code>attn_dropout</code> <code>float</code> <p>Internal dropout for the attention mechanism</p> <code>0.1</code> <code>with_addnorm</code> <code>bool</code> <p>Boolean indicating if the output of each block will be added to the input and normalised</p> <code>False</code> <code>head_hidden_dims</code> <code>Optional[List[int]]</code> <p>List with the sizes of the dense layers in the head e.g: [128, 64]</p> <code>None</code> <code>head_activation</code> <code>str</code> <p>Activation function for the dense layers in the head. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> <code>'relu'</code> <code>head_dropout</code> <code>Optional[float]</code> <p>Dropout of the dense layers in the head</p> <code>None</code> <code>head_batchnorm</code> <code>bool</code> <p>Boolean indicating whether or not to include batch normalization in the dense layers that form the 'rnn_mlp'</p> <code>False</code> <code>head_batchnorm_last</code> <code>bool</code> <p>Boolean indicating whether or not to apply batch normalization to the last of the dense layers in the head</p> <code>False</code> <code>head_linear_first</code> <code>bool</code> <p>Boolean indicating whether the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code></p> <code>False</code> <p>Attributes:</p> Name Type Description <code>word_embed</code> <code>Module</code> <p>word embedding matrix</p> <code>rnn</code> <code>Module</code> <p>Stack of RNNs</p> <code>rnn_mlp</code> <code>Module</code> <p>Stack of dense layers on top of the RNN. This will only exists if <code>head_layers_dim</code> is not <code>None</code></p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import StackedAttentiveRNN\n&gt;&gt;&gt; X_text = torch.cat((torch.zeros([5,1]), torch.empty(5, 4).random_(1,4)), axis=1)\n&gt;&gt;&gt; model = StackedAttentiveRNN(vocab_size=4, hidden_dim=4, padding_idx=0, embed_dim=4)\n&gt;&gt;&gt; out = model(X_text)\n</code></pre> Source code in <code>pytorch_widedeep/models/text/rnns/stacked_attentive_rnn.py</code> <pre><code>class StackedAttentiveRNN(BaseWDModelComponent):\n    r\"\"\"Text classifier/regressor comprised by a stack of blocks:\n    `[RNN + Attention]`. This can be used as the `deeptext` component of a\n    Wide &amp; Deep model or independently by itself.\n\n    In addition, there is the option to add a Fully Connected (FC) set of\n    dense layers on top of the attentiob blocks\n\n    Parameters\n    ----------\n    vocab_size: int\n        Number of words in the vocabulary\n    embed_dim: int, Optional, default = None\n        Dimension of the word embeddings if non-pretained word vectors are\n        used\n    embed_matrix: np.ndarray, Optional, default = None\n        Pretrained word embeddings\n    embed_trainable: bool, default = True\n        Boolean indicating if the pretrained embeddings are trainable\n    rnn_type: str, default = 'lstm'\n        String indicating the type of RNN to use. One of 'lstm' or 'gru'\n    hidden_dim: int, default = 64\n        Hidden dim of the RNN\n    bidirectional: bool, default = True\n        Boolean indicating whether the staked RNNs are bidirectional\n    padding_idx: int, default = 1\n        index of the padding token in the padded-tokenised sequences. The\n        `TextPreprocessor` class within this library uses fastai's\n        tokenizer where the token index 0 is reserved for the _'unknown'_\n        word token. Therefore, the default value is set to 1.\n    n_blocks: int, default = 3\n        Number of attention blocks. Each block is comprised by an RNN and a\n        Context Attention Encoder\n    attn_concatenate: bool, default = True\n        Boolean indicating if the input to the attention mechanism will be the\n        output of the RNN or the output of the RNN concatenated with the last\n        hidden state or simply\n    attn_dropout: float, default = 0.1\n        Internal dropout for the attention mechanism\n    with_addnorm: bool, default = False\n        Boolean indicating if the output of each block will be added to the\n        input and normalised\n    head_hidden_dims: List, Optional, default = None\n        List with the sizes of the dense layers in the head e.g: [128, 64]\n    head_activation: str, default = \"relu\"\n        Activation function for the dense layers in the head. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported\n    head_dropout: float, Optional, default = None\n        Dropout of the dense layers in the head\n    head_batchnorm: bool, default = False\n        Boolean indicating whether or not to include batch normalization in\n        the dense layers that form the _'rnn_mlp'_\n    head_batchnorm_last: bool, default = False\n        Boolean indicating whether or not to apply batch normalization to the\n        last of the dense layers in the head\n    head_linear_first: bool, default = False\n        Boolean indicating whether the order of the operations in the dense\n        layer. If `True: [LIN -&gt; ACT -&gt; BN -&gt; DP]`. If `False: [BN -&gt; DP -&gt;\n        LIN -&gt; ACT]`\n\n    Attributes\n    ----------\n    word_embed: nn.Module\n        word embedding matrix\n    rnn: nn.Module\n        Stack of RNNs\n    rnn_mlp: nn.Module\n        Stack of dense layers on top of the RNN. This will only exists if\n        `head_layers_dim` is not `None`\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.models import StackedAttentiveRNN\n    &gt;&gt;&gt; X_text = torch.cat((torch.zeros([5,1]), torch.empty(5, 4).random_(1,4)), axis=1)\n    &gt;&gt;&gt; model = StackedAttentiveRNN(vocab_size=4, hidden_dim=4, padding_idx=0, embed_dim=4)\n    &gt;&gt;&gt; out = model(X_text)\n    \"\"\"\n\n    def __init__(\n        self,\n        vocab_size: int,\n        embed_dim: Optional[int] = None,\n        embed_matrix: Optional[np.ndarray] = None,\n        embed_trainable: bool = True,\n        rnn_type: Literal[\"lstm\", \"gru\"] = \"lstm\",\n        hidden_dim: int = 64,\n        bidirectional: bool = False,\n        padding_idx: int = 1,\n        n_blocks: int = 3,\n        attn_concatenate: bool = False,\n        attn_dropout: float = 0.1,\n        with_addnorm: bool = False,\n        head_hidden_dims: Optional[List[int]] = None,\n        head_activation: str = \"relu\",\n        head_dropout: Optional[float] = None,\n        head_batchnorm: bool = False,\n        head_batchnorm_last: bool = False,\n        head_linear_first: bool = False,\n    ):\n        super(StackedAttentiveRNN, self).__init__()\n\n        if (\n            embed_dim is not None\n            and embed_matrix is not None\n            and not embed_dim == embed_matrix.shape[1]\n        ):\n            warnings.warn(\n                \"the input embedding dimension {} and the dimension of the \"\n                \"pretrained embeddings {} do not match. The pretrained embeddings \"\n                \"dimension ({}) will be used\".format(\n                    embed_dim, embed_matrix.shape[1], embed_matrix.shape[1]\n                ),\n                UserWarning,\n            )\n\n        if rnn_type.lower() not in [\"lstm\", \"gru\"]:\n            raise ValueError(\n                f\"'rnn_type' must be 'lstm' or 'gru', got {rnn_type} instead\"\n            )\n\n        self.vocab_size = vocab_size\n        self.embed_trainable = embed_trainable\n        self.embed_dim = embed_dim\n\n        self.rnn_type = rnn_type\n        self.hidden_dim = hidden_dim\n        self.bidirectional = bidirectional\n        self.padding_idx = padding_idx\n\n        self.n_blocks = n_blocks\n        self.attn_concatenate = attn_concatenate\n        self.attn_dropout = attn_dropout\n        self.with_addnorm = with_addnorm\n\n        self.head_hidden_dims = head_hidden_dims\n        self.head_activation = head_activation\n        self.head_dropout = head_dropout\n        self.head_batchnorm = head_batchnorm\n        self.head_batchnorm_last = head_batchnorm_last\n        self.head_linear_first = head_linear_first\n\n        # Embeddings\n        self.word_embed, self.embed_dim = self._set_embeddings(embed_matrix)\n\n        # Linear Projection: if embed_dim is different that the input of the\n        # attention blocks we add a linear projection\n        if bidirectional and attn_concatenate:\n            self.rnn_output_dim = hidden_dim * 4\n        elif bidirectional or attn_concatenate:\n            self.rnn_output_dim = hidden_dim * 2\n        else:\n            self.rnn_output_dim = hidden_dim\n\n        if self.rnn_output_dim != self.embed_dim:\n            self.embed_proj: Union[nn.Linear, nn.Identity] = nn.Linear(\n                self.embed_dim, self.rnn_output_dim\n            )\n        else:\n            self.embed_proj = nn.Identity()\n\n        # RNN\n        rnn_params = {\n            \"input_size\": self.rnn_output_dim,\n            \"hidden_size\": hidden_dim,\n            \"bidirectional\": bidirectional,\n            \"batch_first\": True,\n        }\n        if self.rnn_type.lower() == \"lstm\":\n            self.rnn: Union[nn.LSTM, nn.GRU] = nn.LSTM(**rnn_params)\n        elif self.rnn_type.lower() == \"gru\":\n            self.rnn = nn.GRU(**rnn_params)\n\n        # FC-Head (Mlp)\n        self.attention_blks = nn.ModuleList()\n        for i in range(n_blocks):\n            self.attention_blks.append(\n                ContextAttentionEncoder(\n                    self.rnn,\n                    self.rnn_output_dim,\n                    attn_dropout,\n                    attn_concatenate,\n                    with_addnorm=with_addnorm if i != n_blocks - 1 else False,\n                    sum_along_seq=i == n_blocks - 1,\n                )\n            )\n\n        # Mlp\n        if self.head_hidden_dims is not None:\n            head_hidden_dims = [self.rnn_output_dim] + self.head_hidden_dims\n            self.rnn_mlp: Union[MLP, nn.Identity] = MLP(\n                head_hidden_dims,\n                head_activation,\n                head_dropout,\n                head_batchnorm,\n                head_batchnorm_last,\n                head_linear_first,\n            )\n        else:\n            # simple hack to add readability in the forward pass\n            self.rnn_mlp = nn.Identity()\n\n    def forward(self, X: Tensor) -&gt; Tensor:  # type: ignore\n        x = self.embed_proj(self.word_embed(X.long()))\n\n        h = nn.init.zeros_(\n            torch.Tensor(2 if self.bidirectional else 1, X.shape[0], self.hidden_dim)\n        ).to(x.device)\n        if self.rnn_type == \"lstm\":\n            c = nn.init.zeros_(\n                torch.Tensor(\n                    2 if self.bidirectional else 1, X.shape[0], self.hidden_dim\n                )\n            ).to(x.device)\n        else:\n            c = None\n\n        for blk in self.attention_blks:\n            x, h, c = blk(x, h, c)\n\n        return self.rnn_mlp(x)\n\n    @property\n    def output_dim(self) -&gt; int:\n        r\"\"\"The output dimension of the model. This is a required property\n        neccesary to build the `WideDeep` class\n        \"\"\"\n        return (\n            self.head_hidden_dims[-1]\n            if self.head_hidden_dims is not None\n            else self.rnn_output_dim\n        )\n\n    @property\n    def attention_weights(self) -&gt; List:\n        r\"\"\"List with the attention weights per block\n\n        The shape of the attention weights is $(N, S)$ Where $N$ is the batch\n        size and $S$ is the length of the sequence\n        \"\"\"\n        return [blk.attn.attn_weights for blk in self.attention_blks]\n\n    def _set_embeddings(\n        self, embed_matrix: Union[Any, np.ndarray]\n    ) -&gt; Tuple[nn.Module, int]:\n        if isinstance(embed_matrix, np.ndarray):\n            assert (\n                embed_matrix.dtype == \"float32\"\n            ), \"'embed_matrix' must be of dtype 'float32', got dtype '{}'\".format(\n                str(embed_matrix.dtype)\n            )\n            word_embed = nn.Embedding(\n                self.vocab_size, embed_matrix.shape[1], padding_idx=self.padding_idx\n            )\n            if self.embed_trainable:\n                word_embed.weight = nn.Parameter(\n                    torch.tensor(embed_matrix), requires_grad=True\n                )\n            else:\n                word_embed.weight = nn.Parameter(\n                    torch.tensor(embed_matrix), requires_grad=False\n                )\n            embed_dim = embed_matrix.shape[1]\n        else:\n            assert self.embed_dim is not None\n            word_embed = nn.Embedding(\n                self.vocab_size, self.embed_dim, padding_idx=self.padding_idx\n            )\n            embed_dim = self.embed_dim\n\n        return word_embed, embed_dim\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.text.rnns.stacked_attentive_rnn.StackedAttentiveRNN.output_dim","title":"output_dim  <code>property</code>","text":"<pre><code>output_dim\n</code></pre> <p>The output dimension of the model. This is a required property neccesary to build the <code>WideDeep</code> class</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.text.rnns.stacked_attentive_rnn.StackedAttentiveRNN.attention_weights","title":"attention_weights  <code>property</code>","text":"<pre><code>attention_weights\n</code></pre> <p>List with the attention weights per block</p> <p>The shape of the attention weights is \\((N, S)\\) Where \\(N\\) is the batch size and \\(S\\) is the length of the sequence</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.text.huggingface_transformers.hf_model.HFModel","title":"HFModel","text":"<p>               Bases: <code>BaseWDModelComponent</code></p> <p>This class is a wrapper around the Hugging Face transformers library. It can be used as the text component of a Wide &amp; Deep model or independently by itself.</p> <p>At the moment only models from the families BERT, RoBERTa, DistilBERT, ALBERT and ELECTRA are supported. This is because this library is designed to address classification and regression tasks and these are the most 'popular' encoder-only models, which have proved to be those that work best for these tasks.</p> <p>Parameters:</p> Name Type Description Default <code>model_name</code> <code>str</code> <p>The model name from the transformers library e.g. 'bert-base-uncased'. Currently supported models are those from the families: BERT, RoBERTa, DistilBERT, ALBERT and ELECTRA.</p> required <code>use_cls_token</code> <code>bool</code> <p>Boolean indicating whether to use the [CLS] token or the mean of the sequence of hidden states as the sentence embedding</p> <code>True</code> <code>trainable_parameters</code> <code>Optional[List[str]]</code> <p>List with the names of the model parameters that will be trained. If None, none of the parameters will be trainable</p> <code>None</code> <code>head_hidden_dims</code> <code>Optional[List[int]]</code> <p>List with the sizes of the dense layers in the head e.g: [128, 64]</p> <code>None</code> <code>head_activation</code> <code>str</code> <p>Activation function for the dense layers in the head. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> <code>'relu'</code> <code>head_dropout</code> <code>Optional[float]</code> <p>Dropout of the dense layers in the head</p> <code>None</code> <code>head_batchnorm</code> <code>bool</code> <p>Boolean indicating whether or not to include batch normalization in the dense layers that form the head</p> <code>False</code> <code>head_batchnorm_last</code> <code>bool</code> <p>Boolean indicating whether or not to apply batch normalization to the last of the dense layers in the head</p> <code>False</code> <code>head_linear_first</code> <code>bool</code> <p>Boolean indicating whether the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code></p> <code>False</code> <code>verbose</code> <code>bool</code> <p>If True, it will print information about the model</p> <code>False</code> <code>**kwargs</code> <p>Additional kwargs to be passed to the model</p> <code>{}</code> <p>Attributes:</p> Name Type Description <code>head</code> <code>Module</code> <p>Stack of dense layers on top of the transformer. This will only exists if <code>head_layers_dim</code> is not None</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import HFModel\n&gt;&gt;&gt; X_text = torch.cat((torch.zeros([5,1]), torch.empty(5, 4).random_(1,4)), axis=1).long()\n&gt;&gt;&gt; model = HFModel(model_name='bert-base-uncased')\n&gt;&gt;&gt; out = model(X_text)\n</code></pre> Source code in <code>pytorch_widedeep/models/text/huggingface_transformers/hf_model.py</code> <pre><code>class HFModel(BaseWDModelComponent):\n    \"\"\"This class is a wrapper around the Hugging Face transformers library. It\n    can be used as the text component of a Wide &amp; Deep model or independently\n    by itself.\n\n    At the moment only models from the families BERT, RoBERTa, DistilBERT,\n    ALBERT and ELECTRA are supported. This is because this library is\n    designed to address classification and regression tasks and these are the\n    most 'popular' encoder-only models, which have proved to be those that\n    work best for these tasks.\n\n    Parameters\n    ----------\n    model_name: str\n        The model name from the transformers library e.g. 'bert-base-uncased'.\n        Currently supported models are those from the families: BERT, RoBERTa,\n        DistilBERT, ALBERT and ELECTRA.\n    use_cls_token: bool, default = True\n        Boolean indicating whether to use the [CLS] token or the mean of the\n        sequence of hidden states as the sentence embedding\n    trainable_parameters: List, Optional, default = None\n        List with the names of the model parameters that will be trained. If\n        None, none of the parameters will be trainable\n    head_hidden_dims: List, Optional, default = None\n        List with the sizes of the dense layers in the head e.g: _[128, 64]_\n    head_activation: str, default = \"relu\"\n        Activation function for the dense layers in the head. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported\n    head_dropout: float, Optional, default = None\n        Dropout of the dense layers in the head\n    head_batchnorm: bool, default = False\n        Boolean indicating whether or not to include batch normalization in the\n        dense layers that form the head\n    head_batchnorm_last: bool, default = False\n        Boolean indicating whether or not to apply batch normalization to the\n        last of the dense layers in the head\n    head_linear_first: bool, default = False\n        Boolean indicating whether the order of the operations in the dense\n        layer. If `True: [LIN -&gt; ACT -&gt; BN -&gt; DP]`. If `False: [BN -&gt; DP -&gt;\n        LIN -&gt; ACT]`\n    verbose: bool, default = False\n        If True, it will print information about the model\n    **kwargs\n        Additional kwargs to be passed to the model\n\n    Attributes\n    ----------\n    head: nn.Module\n        Stack of dense layers on top of the transformer. This will only exists\n        if `head_layers_dim` is not None\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.models import HFModel\n    &gt;&gt;&gt; X_text = torch.cat((torch.zeros([5,1]), torch.empty(5, 4).random_(1,4)), axis=1).long()\n    &gt;&gt;&gt; model = HFModel(model_name='bert-base-uncased')\n    &gt;&gt;&gt; out = model(X_text)\n    \"\"\"\n\n    @alias(\"use_cls_token\", [\"use_special_token\"])\n    def __init__(\n        self,\n        model_name: str,\n        use_cls_token: bool = True,\n        trainable_parameters: Optional[List[str]] = None,\n        head_hidden_dims: Optional[List[int]] = None,\n        head_activation: str = \"relu\",\n        head_dropout: Optional[float] = None,\n        head_batchnorm: bool = False,\n        head_batchnorm_last: bool = False,\n        head_linear_first: bool = False,\n        verbose: bool = False,\n        **kwargs,\n    ):\n        super().__init__()\n\n        # TO DO: add warning regarging ELECTRA as ELECTRA does not have a cls\n        # token.  Research what happens with ELECTRA\n        self.model_name = model_name\n        self.use_cls_token = use_cls_token\n        self.trainable_parameters = trainable_parameters\n        self.head_hidden_dims = head_hidden_dims\n        self.head_activation = head_activation\n        self.head_dropout = head_dropout\n        self.head_batchnorm = head_batchnorm\n        self.head_batchnorm_last = head_batchnorm_last\n        self.head_linear_first = head_linear_first\n        self.verbose = verbose\n        self.kwargs = kwargs\n\n        if self.verbose and self.use_cls_token:\n            warnings.warn(\n                \"The model will use the [CLS] token. Make sure the tokenizer \"\n                \"was run with add_special_tokens=True\",\n                UserWarning,\n            )\n\n        self.model_class = get_model_class(model_name)\n\n        self.config, self.model = get_config_and_model(self.model_name)\n\n        self.output_attention_weights = kwargs.get(\"output_attentions\", False)\n\n        if self.trainable_parameters is not None:\n            for n, p in self.model.named_parameters():\n                p.requires_grad = any([tl in n for tl in self.trainable_parameters])\n\n        # FC-Head (Mlp). Note that the FC head will always be trainable\n        if self.head_hidden_dims is not None:\n            head_hidden_dims = [self.config.hidden_size] + self.head_hidden_dims\n            self.head = MLP(\n                head_hidden_dims,\n                head_activation,\n                head_dropout,\n                head_batchnorm,\n                head_batchnorm_last,\n                head_linear_first,\n            )\n\n    def forward(self, X: Tensor) -&gt; Tensor:\n\n        # this is inefficient since the attention mask is returned by the\n        # tokenizer, but all models in this library use a forward pass that\n        # takes ONLY an input tensor. A fix will be addressed in a future\n        attn_mask = (X != 0).type(torch.int8)\n\n        output = self.model(input_ids=X, attention_mask=attn_mask, **self.kwargs)\n\n        if self.output_attention_weights:\n            # TO CONSIDER: attention weights as a returned object and not an\n            # attribute\n            self.attn_weights = output[\"attentions\"]\n\n        if self.use_cls_token:\n            output = output[0][:, 0, :]\n        else:\n            # Here one can choose to flatten, but unless the sequence length\n            # is very small, flatten will result in a very large output\n            # tensor.\n            output = output[0].mean(dim=1)\n\n        if self.head_hidden_dims is not None:\n            output = self.head(output)\n\n        return output\n\n    @property\n    def output_dim(self) -&gt; int:\n        return (\n            self.head_hidden_dims[-1]\n            if self.head_hidden_dims is not None\n            else self.config.hidden_size\n        )\n\n    @property\n    def attention_weight(self) -&gt; Tensor:\n        r\"\"\"Returns the attention weights if the model was created with the\n        output_attention_weights=True argument. If not, it will raise an\n        AttributeError.\n\n        The shape of the attention weights is $(N, H, F, F)$, where $N$ is the\n        batch size, $H$ is the number of attention heads and $F$ is the\n        sequence length.\n        \"\"\"\n        if not self.output_attention_weights:\n            raise AttributeError(\n                \"The output_attention_weights attribute was not set to True when creating the model object \"\n                \"Please pass an output_attention_weights=True argument when creating the HFModel object\"\n            )\n        return self.attn_weights\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.text.huggingface_transformers.hf_model.HFModel.attention_weight","title":"attention_weight  <code>property</code>","text":"<pre><code>attention_weight\n</code></pre> <p>Returns the attention weights if the model was created with the output_attention_weights=True argument. If not, it will raise an AttributeError.</p> <p>The shape of the attention weights is \\((N, H, F, F)\\), where \\(N\\) is the batch size, \\(H\\) is the number of attention heads and \\(F\\) is the sequence length.</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.image.vision.Vision","title":"Vision","text":"<p>               Bases: <code>BaseWDModelComponent</code></p> <p>Defines a standard image classifier/regressor using a pretrained network or a sequence of convolution layers that can be used as the <code>deepimage</code> component of a Wide &amp; Deep model or independently by itself.</p> <p> NOTE: this class represents the integration  between <code>pytorch-widedeep</code> and <code>torchvision</code>. New architectures will be  available as they are added to <code>torchvision</code>. In a distant future we aim  to bring transformer-based architectures as well. However, simple  CNN-based architectures (and even MLP-based) seem to produce SoTA  results. For the time being, we describe below the options available  through this class</p> <p>Parameters:</p> Name Type Description Default <code>pretrained_model_setup</code> <code>Union[str, Dict[str, Union[str, WeightsEnum]]]</code> <p>Name of the pretrained model. Should be a variant of the following architectures: 'resnet', 'shufflenet', 'resnext', 'wide_resnet', 'regnet', 'densenet', 'mobilenetv3', 'mobilenetv2', 'mnasnet', 'efficientnet' and 'squeezenet'. if <code>pretrained_model_setup = None</code> a basic, fully trainable CNN will be used. Alternatively, since Torchvision 0.13 one can use pretrained models with different weigths. Therefore, <code>pretrained_model_setup</code> can also be dictionary with the name of the model and the weights (e.g. <code>{'resnet50': ResNet50_Weights.DEFAULT}</code> or <code>{'resnet50': \"IMAGENET1K_V2\"}</code>).  Aliased as <code>pretrained_model_name</code>.</p> <code>None</code> <code>n_trainable</code> <code>Optional[int]</code> <p>Number of trainable layers starting from the layer closer to the output neuron(s). Note that this number DOES NOT take into account the so-called 'head' which is ALWAYS trainable. If <code>trainable_params</code> is not None this parameter will be ignored</p> <code>None</code> <code>trainable_params</code> <code>Optional[List[str]]</code> <p>List of strings containing the names (or substring within the name) of the parameters that will be trained. For example, if we use a 'resnet18' pretrained model and we set <code>trainable_params = ['layer4']</code> only the parameters of 'layer4' of the network (and the head, as mentioned before) will be trained. Note that setting this or the previous parameter involves some knowledge of the architecture used.</p> <code>None</code> <code>channel_sizes</code> <code>List[int]</code> <p>List of integers with the channel sizes of a CNN in case we choose not to use a pretrained model</p> <code>[64, 128, 256, 512]</code> <code>kernel_sizes</code> <code>Union[int, List[int]]</code> <p>List of integers with the kernel sizes of a CNN in case we choose not to use a pretrained model. Must be of length equal to <code>len(channel_sizes) - 1</code>.</p> <code>[7, 3, 3, 3]</code> <code>strides</code> <code>Union[int, List[int]]</code> <p>List of integers with the stride sizes of a CNN in case we choose not to use a pretrained model. Must be of length equal to <code>len(channel_sizes) - 1</code>.</p> <code>[2, 1, 1, 1]</code> <code>head_hidden_dims</code> <code>Optional[List[int]]</code> <p>List with the number of neurons per dense layer in the head. e.g: [64,32]</p> <code>None</code> <code>head_activation</code> <code>str</code> <p>Activation function for the dense layers in the head. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> <code>'relu'</code> <code>head_dropout</code> <code>Union[float, List[float]]</code> <p>float indicating the dropout between the dense layers.</p> <code>0.1</code> <code>head_batchnorm</code> <code>bool</code> <p>Boolean indicating whether or not batch normalization will be applied to the dense layers</p> <code>False</code> <code>head_batchnorm_last</code> <code>bool</code> <p>Boolean indicating whether or not batch normalization will be applied to the last of the dense layers</p> <code>False</code> <code>head_linear_first</code> <code>bool</code> <p>Boolean indicating the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code></p> <code>False</code> <p>Attributes:</p> Name Type Description <code>features</code> <code>Module</code> <p>The pretrained model or Standard CNN plus the optional head</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models import Vision\n&gt;&gt;&gt; X_img = torch.rand((2,3,224,224))\n&gt;&gt;&gt; model = Vision(channel_sizes=[64, 128], kernel_sizes = [3, 3], strides=[1, 1], head_hidden_dims=[32, 8])\n&gt;&gt;&gt; out = model(X_img)\n</code></pre> Source code in <code>pytorch_widedeep/models/image/vision.py</code> <pre><code>class Vision(BaseWDModelComponent):\n    r\"\"\"Defines a standard image classifier/regressor using a pretrained\n    network or a sequence of convolution layers that can be used as the\n    `deepimage` component of a Wide &amp; Deep model or independently by\n    itself.\n\n    :information_source: **NOTE**: this class represents the integration\n     between `pytorch-widedeep` and `torchvision`. New architectures will be\n     available as they are added to `torchvision`. In a distant future we aim\n     to bring transformer-based architectures as well. However, simple\n     CNN-based architectures (and even MLP-based) seem to produce SoTA\n     results. For the time being, we describe below the options available\n     through this class\n\n    Parameters\n    ----------\n    pretrained_model_setup: Optional, str or dict, default = None\n        Name of the pretrained model. Should be a variant of the following\n        architectures: _'resnet'_, _'shufflenet'_, _'resnext'_,\n        _'wide_resnet'_, _'regnet'_, _'densenet'_, _'mobilenetv3'_,\n        _'mobilenetv2'_, _'mnasnet'_, _'efficientnet'_ and _'squeezenet'_. if\n        `pretrained_model_setup = None` a basic, fully trainable CNN will be\n        used. Alternatively, since Torchvision 0.13 one can use pretrained\n        models with different weigths. Therefore, `pretrained_model_setup` can\n        also be dictionary with the name of the model and the weights (e.g.\n        `{'resnet50': ResNet50_Weights.DEFAULT}` or\n        `{'resnet50': \"IMAGENET1K_V2\"}`). &lt;br/&gt; Aliased as `pretrained_model_name`.\n    n_trainable: Optional, int, default = None\n        Number of trainable layers starting from the layer closer to the\n        output neuron(s). Note that this number DOES NOT take into account\n        the so-called _'head'_ which is ALWAYS trainable. If\n        `trainable_params` is not None this parameter will be ignored\n    trainable_params: Optional, list, default = None\n        List of strings containing the names (or substring within the name) of\n        the parameters that will be trained. For example, if we use a\n        _'resnet18'_ pretrained model and we set `trainable_params =\n        ['layer4']` only the parameters of _'layer4'_ of the network\n        (and the head, as mentioned before) will be trained. Note that\n        setting this or the previous parameter involves some knowledge of\n        the architecture used.\n    channel_sizes: list, default = [64, 128, 256, 512]\n        List of integers with the channel sizes of a CNN in case we choose not\n        to use a pretrained model\n    kernel_sizes: list or int, default = 3\n        List of integers with the kernel sizes of a CNN in case we choose not\n        to use a pretrained model. Must be of length equal to `len(channel_sizes) - 1`.\n    strides: list or int, default = 1\n        List of integers with the stride sizes of a CNN in case we choose not\n        to use a pretrained model. Must be of length equal to `len(channel_sizes) - 1`.\n    head_hidden_dims: Optional, list, default = None\n        List with the number of neurons per dense layer in the head. e.g: _[64,32]_\n    head_activation: str, default = \"relu\"\n        Activation function for the dense layers in the head. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported\n    head_dropout: float, default = 0.1\n        float indicating the dropout between the dense layers.\n    head_batchnorm: bool, default = False\n        Boolean indicating whether or not batch normalization will be applied\n        to the dense layers\n    head_batchnorm_last: bool, default = False\n        Boolean indicating whether or not batch normalization will be applied\n        to the last of the dense layers\n    head_linear_first: bool, default = False\n        Boolean indicating the order of the operations in the dense\n        layer. If `True: [LIN -&gt; ACT -&gt; BN -&gt; DP]`. If `False: [BN -&gt; DP -&gt;\n        LIN -&gt; ACT]`\n\n    Attributes\n    ----------\n    features: nn.Module\n        The pretrained model or Standard CNN plus the optional head\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.models import Vision\n    &gt;&gt;&gt; X_img = torch.rand((2,3,224,224))\n    &gt;&gt;&gt; model = Vision(channel_sizes=[64, 128], kernel_sizes = [3, 3], strides=[1, 1], head_hidden_dims=[32, 8])\n    &gt;&gt;&gt; out = model(X_img)\n    \"\"\"\n\n    @alias(\"pretrained_model_setup\", [\"pretrained_model_name\"])\n    def __init__(\n        self,\n        pretrained_model_setup: Union[str, Dict[str, Union[str, WeightsEnum]]] = None,\n        n_trainable: Optional[int] = None,\n        trainable_params: Optional[List[str]] = None,\n        channel_sizes: List[int] = [64, 128, 256, 512],\n        kernel_sizes: Union[int, List[int]] = [7, 3, 3, 3],\n        strides: Union[int, List[int]] = [2, 1, 1, 1],\n        head_hidden_dims: Optional[List[int]] = None,\n        head_activation: str = \"relu\",\n        head_dropout: Union[float, List[float]] = 0.1,\n        head_batchnorm: bool = False,\n        head_batchnorm_last: bool = False,\n        head_linear_first: bool = False,\n    ):\n        super(Vision, self).__init__()\n\n        self._check_pretrained_model_setup(\n            pretrained_model_setup, n_trainable, trainable_params\n        )\n\n        self.pretrained_model_setup = pretrained_model_setup\n        self.n_trainable = n_trainable\n        self.trainable_params = trainable_params\n        self.channel_sizes = channel_sizes\n        self.kernel_sizes = kernel_sizes\n        self.strides = strides\n        self.head_hidden_dims = head_hidden_dims\n        self.head_activation = head_activation\n        self.head_dropout = head_dropout\n        self.head_batchnorm = head_batchnorm\n        self.head_batchnorm_last = head_batchnorm_last\n        self.head_linear_first = head_linear_first\n\n        self.features, self.backbone_output_dim = self._get_features()\n\n        if pretrained_model_setup is not None:\n            self._freeze(self.features)\n\n        if self.head_hidden_dims is not None:\n            head_hidden_dims = [self.backbone_output_dim] + self.head_hidden_dims\n            self.vision_mlp = MLP(\n                head_hidden_dims,\n                self.head_activation,\n                self.head_dropout,\n                self.head_batchnorm,\n                self.head_batchnorm_last,\n                self.head_linear_first,\n            )\n\n    def forward(self, X: Tensor) -&gt; Tensor:\n        x = self.features(X)\n\n        if len(x.shape) &gt; 2:\n            if x.shape[2] &gt; 1:\n                x = nn.functional.adaptive_avg_pool2d(x, (1, 1))\n            x = torch.flatten(x, 1)\n\n        if self.head_hidden_dims is not None:\n            x = self.vision_mlp(x)\n\n        return x\n\n    @property\n    def output_dim(self) -&gt; int:\n        r\"\"\"The output dimension of the model. This is a required property\n        neccesary to build the `WideDeep` class\n        \"\"\"\n        return (\n            self.head_hidden_dims[-1]\n            if self.head_hidden_dims is not None\n            else self.backbone_output_dim\n        )\n\n    def _get_features(self) -&gt; Tuple[nn.Module, int]:\n        if self.pretrained_model_setup is not None:\n            if isinstance(self.pretrained_model_setup, str):\n                if self.pretrained_model_setup in allowed_pretrained_models.keys():\n                    model = allowed_pretrained_models[self.pretrained_model_setup]\n                    pretrained_model = torchvision.models.__dict__[model](\n                        weights=torchvision.models.get_model_weights(model).DEFAULT\n                    )\n                    warnings.warn(\n                        f\"{self.pretrained_model_setup} defaulting to {model}\",\n                        UserWarning,\n                    )\n                else:\n                    pretrained_model = torchvision.models.__dict__[\n                        self.pretrained_model_setup\n                    ](weights=\"IMAGENET1K_V1\")\n\n            elif isinstance(self.pretrained_model_setup, Dict):\n                model_name = next(iter(self.pretrained_model_setup))\n                model_weights = self.pretrained_model_setup[model_name]\n\n                if model_name in allowed_pretrained_models.keys():\n                    model_name = allowed_pretrained_models[model_name]\n\n                pretrained_model = torchvision.models.__dict__[model_name](\n                    weights=model_weights\n                )\n            output_dim: int = self.get_backbone_output_dim(pretrained_model)\n            features = nn.Sequential(*(list(pretrained_model.children())[:-1]))\n\n        else:\n            features = self._basic_cnn()\n            output_dim = self.channel_sizes[-1]\n\n        return features, output_dim\n\n    def _basic_cnn(self):\n        channel_sizes = [3] + self.channel_sizes\n        kernel_sizes = (\n            [self.kernel_sizes] * len(self.channel_sizes)\n            if isinstance(self.kernel_sizes, int)\n            else self.kernel_sizes\n        )\n        strides = (\n            [self.strides] * len(self.channel_sizes)\n            if isinstance(self.strides, int)\n            else self.strides\n        )\n\n        BasicCNN = nn.Sequential()\n        for i in range(1, len(channel_sizes)):\n            BasicCNN.add_module(\n                \"conv_layer_{}\".format(i - 1),\n                conv_layer(\n                    channel_sizes[i - 1],\n                    channel_sizes[i],\n                    kernel_sizes[i - 1],\n                    strides[i - 1],\n                    maxpool=i == 1,\n                    adaptiveavgpool=i == len(channel_sizes) - 1,\n                ),\n            )\n        return BasicCNN\n\n    def _freeze(self, features):\n        if self.trainable_params is not None:\n            for name, param in features.named_parameters():\n                for tl in self.trainable_params:\n                    param.requires_grad = tl in name\n        elif self.n_trainable is not None:\n            for i, (name, param) in enumerate(\n                reversed(list(features.named_parameters()))\n            ):\n                param.requires_grad = i &lt; self.n_trainable\n        else:\n            warnings.warn(\n                \"Both 'trainable_params' and 'n_trainable' are 'None' and the entire network will be trained\",\n                UserWarning,\n            )\n\n    @staticmethod\n    def get_backbone_output_dim(features):\n        try:\n            return features.fc.in_features\n        except AttributeError:\n            try:\n                features.classifier.__dict__[\"_modules\"][\"0\"].in_features\n            except AttributeError:\n                try:\n                    return features.classifier.__dict__[\"_modules\"][\"1\"].in_features\n                except AttributeError:\n                    return features.classifier.__dict__[\"_modules\"][\"1\"].in_channels\n\n    @staticmethod\n    def _check_pretrained_model_setup(\n        pretrained_model_setup, n_trainable, trainable_params\n    ):\n        if pretrained_model_setup is not None:\n            if isinstance(pretrained_model_setup, str):\n                pretrained_model_name = pretrained_model_setup\n            elif isinstance(pretrained_model_setup, Dict):\n                pretrained_model_name = list(pretrained_model_setup.keys())[0]\n        else:\n            pretrained_model_name = None\n\n        if pretrained_model_name is not None:\n            valid_pretrained_model_name = any(\n                [name in pretrained_model_name for name in allowed_pretrained_models]\n            )\n            if not valid_pretrained_model_name:\n                raise ValueError(\n                    f\"{pretrained_model_setup} is not among the allowed pretrained models.\"\n                    f\" These are {allowed_pretrained_models.keys()}. Please choose a variant of these architectures\"\n                )\n            if n_trainable is not None and trainable_params is not None:\n                raise UserWarning(\n                    \"Both 'n_trainable' and 'trainable_params' are not None. 'trainable_params' will be used\"\n                )\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.image.vision.Vision.output_dim","title":"output_dim  <code>property</code>","text":"<pre><code>output_dim\n</code></pre> <p>The output dimension of the model. This is a required property neccesary to build the <code>WideDeep</code> class</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.model_fusion.ModelFuser","title":"ModelFuser","text":"<p>               Bases: <code>BaseWDModelComponent</code></p> <p>This class is a wrapper around a list of models that are associated to the different text and/or image columns (and datasets) The class is designed to 'fuse' the models using a variety of methods.</p> <p>Parameters:</p> Name Type Description Default <code>models</code> <code>List[BaseWDModelComponent]</code> <p>List of models whose outputs will be fused</p> required <code>fusion_method</code> <code>Union[Literal[concatenate, mean, max, sum, mult, dot, head], List[Literal[concatenate, mean, max, sum, mult, head]]]</code> <p>Method to fuse the output of the models. It can be one of ['concatenate', 'mean', 'max', 'sum', 'mult', 'dot', 'head'] or a list of those, but 'dot'. If a list is provided the output of the models will be fused using all the methods in the list and the final output will be the concatenation of the outputs of each method</p> required <code>projection_method</code> <code>Optional[Literal[min, max, mean]]</code> <p>If the fusion_method is not 'concatenate', this parameter will determine how to project the output of the models to a common dimension. It can be one of ['min', 'max', 'mean']. Default is None</p> <code>None</code> <code>custom_head</code> <code>Optional[Union[BaseWDModelComponent, Module]]</code> <p>Custom head to be used to fuse the output of the models. If provided, this will take precedence over head_hidden_dims. Also, if provided, 'projection_method' will be ignored.</p> <code>None</code> <code>head_hidden_dims</code> <code>Optional[List[int]]</code> <p>List with the number of neurons per layer in the custom head. If custom_head is provided, this parameter will be ignored</p> <code>None</code> <code>head_activation</code> <code>Optional[str]</code> <p>Activation function to be used in the custom head. Default is None</p> <code>None</code> <code>head_dropout</code> <code>Optional[float]</code> <p>Dropout to be used in the custom head. Default is None</p> <code>None</code> <code>head_batchnorm</code> <code>Optional[bool]</code> <p>Whether to use batchnorm in the custom head. Default is None</p> <code>None</code> <code>head_batchnorm_last</code> <code>Optional[bool]</code> <p>Whether or not batch normalization will be applied to the last of the dense layers</p> <code>None</code> <code>head_linear_first</code> <code>Optional[bool]</code> <p>Boolean indicating the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code></p> <code>None</code> <p>Attributes:</p> Name Type Description <code>head</code> <code>Module or BaseWDModelComponent</code> <p>Custom head to be used to fuse the output of the models. If custom_head is provided, this will take precedence over head_hidden_dims</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; from pytorch_widedeep.preprocessing import TextPreprocessor\n&gt;&gt;&gt; from pytorch_widedeep.models import BasicRNN, ModelFuser\n&gt;&gt;&gt; import torch\n&gt;&gt;&gt; import pandas as pd\n&gt;&gt;&gt;\n&gt;&gt;&gt; df = pd.DataFrame({'text_col1': ['hello world', 'this is a test'],\n... 'text_col2': ['goodbye world', 'this is another test']})\n&gt;&gt;&gt; text_preprocessor_1 = TextPreprocessor(\n...     text_col=\"text_col1\",\n...     max_vocab=10,\n...     min_freq=1,\n...     maxlen=5,\n...     n_cpus=1,\n...     verbose=0)\n&gt;&gt;&gt; text_preprocessor_2 = TextPreprocessor(\n...     text_col=\"text_col2\",\n...     max_vocab=10,\n...     min_freq=1,\n...     maxlen=5,\n...     n_cpus=1,\n...     verbose=0)\n&gt;&gt;&gt; X_text1 = text_preprocessor_1.fit_transform(df)\n&gt;&gt;&gt; X_text2 = text_preprocessor_2.fit_transform(df)\n&gt;&gt;&gt; X_text1_tnsr = torch.from_numpy(X_text1)\n&gt;&gt;&gt; X_text2_tnsr = torch.from_numpy(X_text2)\n&gt;&gt;&gt; rnn1 = BasicRNN(\n...     vocab_size=len(text_preprocessor_1.vocab.itos),\n...     embed_dim=4,\n...     hidden_dim=4,\n...     n_layers=1,\n...     bidirectional=False)\n&gt;&gt;&gt; rnn2 = BasicRNN(\n...     vocab_size=len(text_preprocessor_2.vocab.itos),\n...     embed_dim=4,\n...     hidden_dim=4,\n...     n_layers=1,\n...     bidirectional=False)\n&gt;&gt;&gt; fused_model = ModelFuser(models=[rnn1, rnn2], fusion_method='concatenate')\n&gt;&gt;&gt; out = fused_model([X_text1_tnsr, X_text2_tnsr])\n</code></pre> Source code in <code>pytorch_widedeep/models/model_fusion.py</code> <pre><code>class ModelFuser(BaseWDModelComponent):\n    \"\"\"\n    This class is a wrapper around a list of models that are associated to the\n    different text and/or image columns (and datasets) The class is designed\n    to 'fuse' the models using a variety of methods.\n\n    Parameters\n    ----------\n    models: List[BaseWDModelComponent]\n        List of models whose outputs will be fused\n    fusion_method: Union[str, List[str]]\n        Method to fuse the output of the models. It can be one of\n        ['concatenate', 'mean', 'max', 'sum', 'mult', 'dot', 'head'] or a\n        list of those, but 'dot'. If a list is provided the output of the\n        models will be fused using all the methods in the list and the final\n        output will be the concatenation of the outputs of each method\n    projection_method: Optional[str]\n        If the fusion_method is not 'concatenate', this parameter will\n        determine how to project the output of the models to a common\n        dimension. It can be one of ['min', 'max', 'mean']. Default is None\n    custom_head: Optional[BaseWDModelComponent | nn.Module]\n        Custom head to be used to fuse the output of the models. If provided,\n        this will take precedence over head_hidden_dims. Also, if\n        provided, 'projection_method' will be ignored.\n    head_hidden_dims: Optional[List[int]]\n        List with the number of neurons per layer in the custom head. If\n        custom_head is provided, this parameter will be ignored\n    head_activation: Optional[str]\n        Activation function to be used in the custom head. Default is None\n    head_dropout: Optional[float]\n        Dropout to be used in the custom head. Default is None\n    head_batchnorm: Optional[bool]\n        Whether to use batchnorm in the custom head. Default is None\n    head_batchnorm_last: Optional[bool]\n        Whether or not batch normalization will be applied to the last of the\n        dense layers\n    head_linear_first: Optional[bool]\n        Boolean indicating the order of the operations in the dense\n        layer. If `True: [LIN -&gt; ACT -&gt; BN -&gt; DP]`. If `False: [BN -&gt; DP -&gt;\n        LIN -&gt; ACT]`\n\n    Attributes\n    ----------\n    head: nn.Module or BaseWDModelComponent\n        Custom head to be used to fuse the output of the models. If\n        custom_head is provided, this will take precedence over\n        head_hidden_dims\n\n    Examples\n    --------\n    &gt;&gt;&gt; from pytorch_widedeep.preprocessing import TextPreprocessor\n    &gt;&gt;&gt; from pytorch_widedeep.models import BasicRNN, ModelFuser\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; import pandas as pd\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; df = pd.DataFrame({'text_col1': ['hello world', 'this is a test'],\n    ... 'text_col2': ['goodbye world', 'this is another test']})\n    &gt;&gt;&gt; text_preprocessor_1 = TextPreprocessor(\n    ...     text_col=\"text_col1\",\n    ...     max_vocab=10,\n    ...     min_freq=1,\n    ...     maxlen=5,\n    ...     n_cpus=1,\n    ...     verbose=0)\n    &gt;&gt;&gt; text_preprocessor_2 = TextPreprocessor(\n    ...     text_col=\"text_col2\",\n    ...     max_vocab=10,\n    ...     min_freq=1,\n    ...     maxlen=5,\n    ...     n_cpus=1,\n    ...     verbose=0)\n    &gt;&gt;&gt; X_text1 = text_preprocessor_1.fit_transform(df)\n    &gt;&gt;&gt; X_text2 = text_preprocessor_2.fit_transform(df)\n    &gt;&gt;&gt; X_text1_tnsr = torch.from_numpy(X_text1)\n    &gt;&gt;&gt; X_text2_tnsr = torch.from_numpy(X_text2)\n    &gt;&gt;&gt; rnn1 = BasicRNN(\n    ...     vocab_size=len(text_preprocessor_1.vocab.itos),\n    ...     embed_dim=4,\n    ...     hidden_dim=4,\n    ...     n_layers=1,\n    ...     bidirectional=False)\n    &gt;&gt;&gt; rnn2 = BasicRNN(\n    ...     vocab_size=len(text_preprocessor_2.vocab.itos),\n    ...     embed_dim=4,\n    ...     hidden_dim=4,\n    ...     n_layers=1,\n    ...     bidirectional=False)\n    &gt;&gt;&gt; fused_model = ModelFuser(models=[rnn1, rnn2], fusion_method='concatenate')\n    &gt;&gt;&gt; out = fused_model([X_text1_tnsr, X_text2_tnsr])\n    \"\"\"\n\n    def __init__(\n        self,\n        models: List[BaseWDModelComponent],\n        *,\n        fusion_method: Union[\n            Literal[\n                \"concatenate\",\n                \"mean\",\n                \"max\",\n                \"sum\",\n                \"mult\",\n                \"dot\",\n                \"head\",\n            ],\n            List[Literal[\"concatenate\", \"mean\", \"max\", \"sum\", \"mult\", \"head\"]],\n        ],\n        projection_method: Optional[Literal[\"min\", \"max\", \"mean\"]] = None,\n        custom_head: Optional[Union[BaseWDModelComponent, nn.Module]] = None,\n        head_hidden_dims: Optional[List[int]] = None,\n        head_activation: Optional[str] = None,\n        head_dropout: Optional[float] = None,\n        head_batchnorm: Optional[bool] = None,\n        head_batchnorm_last: Optional[bool] = None,\n        head_linear_first: Optional[bool] = None,\n    ) -&gt; None:\n        super(ModelFuser, self).__init__()\n\n        self.models = nn.ModuleList(models)\n        self.fusion_method = fusion_method\n        self.projection_method = projection_method\n\n        self.all_output_dim_equal = all(\n            model.output_dim == self.models[0].output_dim for model in self.models\n        )\n\n        self.check_input_parameters()\n\n        if self.fusion_method == \"head\":\n            assert (\n                head_hidden_dims is not None or custom_head is not None\n            ), \"When using 'head' as fusion_method, either head_hidden_dims or custom_head must be provided\"\n            if custom_head is not None:\n                # custom_head takes precedence over head_hidden_dims (in case\n                # both are provided)\n                assert hasattr(\n                    custom_head, \"output_dim\"\n                ), \"custom_head must have an 'output_dim' property\"\n                self.head: Union[BaseWDModelComponent, nn.Module] = custom_head\n            else:\n                assert head_hidden_dims is not None\n                self.head_hidden_dims = head_hidden_dims\n                self.head_activation = head_activation\n                self.head_dropout = head_dropout\n                self.head_batchnorm = head_batchnorm\n                self.head_batchnorm_last = head_batchnorm_last\n                self.head_linear_first = head_linear_first\n\n                self.head = MLP(\n                    d_hidden=[sum([model.output_dim for model in self.models])]\n                    + self.head_hidden_dims,\n                    activation=(\n                        \"relu\" if self.head_activation is None else self.head_activation\n                    ),\n                    dropout=0.0 if self.head_dropout is None else self.head_dropout,\n                    batchnorm=(\n                        False if self.head_batchnorm is None else self.head_batchnorm\n                    ),\n                    batchnorm_last=(\n                        False\n                        if self.head_batchnorm_last is None\n                        else self.head_batchnorm_last\n                    ),\n                    linear_first=(\n                        True\n                        if self.head_linear_first is None\n                        else self.head_linear_first\n                    ),\n                )\n\n    def forward(self, X: List[Tensor]) -&gt; Tensor:  # noqa: C901\n        if self.fusion_method == \"head\":\n            return self._head_fusion(X)\n        elif self.fusion_method == \"dot\":\n            return self._dot_fusion(X)\n        else:\n            return self._other_fusions(X)\n\n    def _head_fusion(self, X: List[Tensor]) -&gt; Tensor:\n        return self.head(torch.cat([model(x) for model, x in zip(self.models, X)], -1))\n\n    def _dot_fusion(self, X: List[Tensor]) -&gt; Tensor:\n        assert (\n            len(X) == 2\n        ), \"When using 'dot' as fusion_method, only two models can be fused\"\n        outputs = [model(x) for model, x in zip(self.models, X)]\n        return torch.bmm(outputs[1].unsqueeze(1), outputs[0].unsqueeze(2)).view(-1, 1)\n\n    def _other_fusions(self, X: List[Tensor]) -&gt; Tensor:\n        fusion_methods = (\n            [self.fusion_method]\n            if isinstance(self.fusion_method, str)\n            else self.fusion_method\n        )\n        fused_outputs = [self._apply_fusion_method(fm, X) for fm in fusion_methods]  # type: ignore[attr-defined]\n        return (\n            fused_outputs[0] if len(fused_outputs) == 1 else torch.cat(fused_outputs, 1)\n        )\n\n    def _apply_fusion_method(self, fusion_method: str, X: List[Tensor]) -&gt; Tensor:\n        if fusion_method == \"concatenate\":\n            return torch.cat([model(x) for model, x in zip(self.models, X)], 1)\n\n        model_outputs = [model(x) for model, x in zip(self.models, X)]\n        projections = self._project(model_outputs)\n        stacked = torch.stack(projections, -1)\n\n        if fusion_method == \"mean\":\n            return torch.mean(stacked, -1)\n        elif fusion_method == \"max\":\n            return torch.max(stacked, -1)[0]\n        elif fusion_method == \"min\":\n            return torch.min(stacked, -1)[0]\n        elif fusion_method == \"sum\":\n            return torch.sum(stacked, -1)\n        elif fusion_method == \"mult\":\n            return torch.prod(stacked, -1)\n        else:\n            raise ValueError(f\"Unsupported fusion method: {fusion_method}\")\n\n    def _project(self, X: List[Tensor]) -&gt; List[Tensor]:\n        r\"\"\"Projects the output of the models to a common dimension.\"\"\"\n\n        if self.all_output_dim_equal and self.projection_method is None:\n            return X\n\n        output_dims = [model.output_dim for model in self.models]\n\n        if self.projection_method == \"min\":\n            proj_dim = min(output_dims)\n            idx = output_dims.index(proj_dim)\n        elif self.projection_method == \"max\":\n            proj_dim = max(output_dims)\n            idx = output_dims.index(proj_dim)\n        elif self.projection_method == \"mean\":\n            proj_dim = int(sum(output_dims) / len(output_dims))\n            idx = None\n        else:\n            raise ValueError(\"projection_method must be one of ['min', 'max', 'mean']\")\n\n        x_proj: List[Tensor] = []\n        for i, x in enumerate(X):\n            if i == idx:\n                x_proj.append(x)\n            else:\n                x_proj.append(\n                    nn.Linear(output_dims[i], proj_dim, bias=False, device=x.device)(x)\n                )\n\n        return x_proj\n\n    @property\n    def output_dim(self) -&gt; int:\n        r\"\"\"Returns the output dimension of the model.\"\"\"\n        if self.fusion_method == \"head\":\n            output_dim = (\n                self.head_hidden_dims[-1]\n                if hasattr(self, \"head_hidden_dims\")\n                else self.head.output_dim\n            )\n        elif self.fusion_method == \"dot\":\n            output_dim = 1\n        else:\n            output_dim = 0\n            if isinstance(self.fusion_method, str):\n                fusion_methods = [self.fusion_method]\n            else:\n                fusion_methods = self.fusion_method  # type: ignore\n            for fm in fusion_methods:\n                if fm == \"concatenate\":\n                    output_dim += sum([model.output_dim for model in self.models])\n                elif self.projection_method == \"mean\":\n                    output_dim += int(\n                        sum([model.output_dim for model in self.models])\n                        / len(self.models)\n                    )\n                elif self.projection_method == \"min\":\n                    output_dim += min([model.output_dim for model in self.models])\n                elif self.projection_method == \"max\":\n                    output_dim += max([model.output_dim for model in self.models])\n                elif self.all_output_dim_equal:\n                    output_dim += self.models[0].output_dim\n                else:\n                    raise ValueError(\n                        \"projection_method must be one of ['min', 'max', 'mean']\"\n                    )\n\n        return output_dim\n\n    def check_input_parameters(self):\n        self._validate_tabnet()\n        self._validate_fusion_methods()\n        self._validate_projection_requirements()\n        self._validate_head_dot_exclusivity()\n\n    def _validate_tabnet(self):\n        if any(isinstance(model, TabNet) for model in self.models):\n            raise ValueError(\n                \"TabNet is not supported in ModelFuser. \"\n                \"Please, use another model for tabular data\"\n            )\n\n    def _validate_fusion_methods(self):\n        valid_methods = [\n            \"concatenate\",\n            \"min\",\n            \"max\",\n            \"mean\",\n            \"sum\",\n            \"mult\",\n            \"dot\",\n            \"head\",\n        ]\n\n        if isinstance(self.fusion_method, str):\n            if self.fusion_method not in valid_methods:\n                raise ValueError(\n                    \"fusion_method must be one of ['concatenate', 'mean', 'max', 'sum', 'mult', 'dot', 'head'] \"\n                    \"or a list of any those but 'dot'\"\n                )\n        else:\n            if not all(fm in valid_methods for fm in self.fusion_method):\n                raise ValueError(\n                    \"fusion_method must be one of ['concatenate', 'mean', 'max', 'sum', 'mult', 'dot', 'head'] \"\n                    \"or a list of those but 'dot'\"\n                )\n\n    def _validate_projection_requirements(self):\n        needs_projection = not self.all_output_dim_equal\n        has_size_dependent_method = False\n\n        if isinstance(self.fusion_method, str):\n            has_size_dependent_method = self.fusion_method in [\"min\", \"max\", \"mean\"]\n        else:\n            has_size_dependent_method = all(\n                fm in [\"min\", \"max\", \"mean\"] for fm in self.fusion_method\n            )\n\n        if has_size_dependent_method and needs_projection:\n            if self.projection_method is None:\n                raise ValueError(\n                    \"If 'fusion_method' is not 'concatenate' or 'head', \"\n                    \"and the output dimensions of the models are not equal, \"\n                    \"'projection_method' must be provided\"\n                )\n            elif self.projection_method not in [\"min\", \"max\", \"mean\"]:\n                raise ValueError(\n                    \"projection_method must be one of ['min', 'max', 'mean']\"\n                )\n\n    def _validate_head_dot_exclusivity(self):\n        if isinstance(self.fusion_method, list):\n            if any(method in [\"head\", \"dot\"] for method in self.fusion_method):\n                raise ValueError(\n                    \"When using 'head' or 'dot' as fusion_method, no other method should be provided\"\n                )\n\n    def __repr__(self):\n        if self.projection_method is not None:\n            proj = f\"{self.projection_method}\"\n        else:\n            proj = \"\"\n\n        return f\"Fusion method: {self.fusion_method}. Projection method: {proj}\\nFused Models:\\n{self.models}\"\n</code></pre>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.model_fusion.ModelFuser.output_dim","title":"output_dim  <code>property</code>","text":"<pre><code>output_dim\n</code></pre> <p>Returns the output dimension of the model.</p>"},{"location":"pytorch-widedeep/model_components.html#pytorch_widedeep.models.wide_deep.WideDeep","title":"WideDeep","text":"<p>               Bases: <code>Module</code></p> <p>Main collector class that combines all <code>wide</code>, <code>deeptabular</code> <code>deeptext</code> and <code>deepimage</code> models.</p> <p>Note that all models described so far in this library must be passed to the <code>WideDeep</code> class once constructed. This is because the models output the last layer before the prediction layer. Such prediction layer is added by the <code>WideDeep</code> class as it collects the components for every data mode.</p> <p>There are two options to combine these models that correspond to the two main architectures that <code>pytorch-widedeep</code> can build.</p> <ul> <li> <p>Directly connecting the output of the model components to an ouput neuron(s).</p> </li> <li> <p>Adding a <code>Fully-Connected Head</code> (FC-Head) on top of the deep models.   This FC-Head will combine the output form the <code>deeptabular</code>, <code>deeptext</code> and   <code>deepimage</code> and will be then connected to the output neuron(s).</p> </li> </ul> <p>Parameters:</p> Name Type Description Default <code>wide</code> <code>Optional[Module]</code> <p><code>Wide</code> model. This is a linear model where the non-linearities are captured via crossed-columns.</p> <code>None</code> <code>deeptabular</code> <code>Optional[Union[BaseWDModelComponent, List[BaseWDModelComponent]]]</code> <p>Currently this library implements a number of possible architectures for the <code>deeptabular</code> component. See the documenation of the package. Note that <code>deeptabular</code> can be a list of models. This is useful when using multiple tabular inputs (e.g. for example in the context of a two-tower model for recommendation systems)</p> <code>None</code> <code>deeptext</code> <code>Optional[Union[BaseWDModelComponent, List[BaseWDModelComponent]]]</code> <p>Currently this library implements a number of possible architectures for the <code>deeptext</code> component. See the documenation of the package. Note that <code>deeptext</code> can be a list of models. This is useful when using multiple text inputs.</p> <code>None</code> <code>deepimage</code> <code>Optional[Union[BaseWDModelComponent, List[BaseWDModelComponent]]]</code> <p>Currently this library uses <code>torchvision</code> and implements a number of possible architectures for the <code>deepimage</code> component. See the documenation of the package. Note that <code>deepimage</code> can be a list of models. This is useful when using multiple image inputs.</p> <code>None</code> <code>deephead</code> <code>Optional[BaseWDModelComponent]</code> <p>Alternatively, the user can pass a custom model that will receive the output of the deep component. If <code>deephead</code> is not None all the previous fc-head parameters will be ignored</p> <code>None</code> <code>head_hidden_dims</code> <code>Optional[List[int]]</code> <p>List with the sizes of the dense layers in the head e.g: [128, 64]</p> <code>None</code> <code>head_activation</code> <code>str</code> <p>Activation function for the dense layers in the head. Currently <code>'tanh'</code>, <code>'relu'</code>, <code>'leaky_relu'</code> and <code>'gelu'</code> are supported</p> <code>'relu'</code> <code>head_dropout</code> <code>float</code> <p>Dropout of the dense layers in the head</p> <code>0.1</code> <code>head_batchnorm</code> <code>bool</code> <p>Boolean indicating whether or not to include batch normalization in the dense layers that form the <code>'rnn_mlp'</code></p> <code>False</code> <code>head_batchnorm_last</code> <code>bool</code> <p>Boolean indicating whether or not to apply batch normalization to the last of the dense layers in the head</p> <code>False</code> <code>head_linear_first</code> <code>bool</code> <p>Boolean indicating whether the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code></p> <code>True</code> <code>enforce_positive</code> <code>bool</code> <p>Boolean indicating if the output from the final layer must be positive. This is important if you are using loss functions with non-negative input restrictions, e.g. RMSLE, or if you know your predictions are bounded in between 0 and inf</p> <code>False</code> <code>enforce_positive_activation</code> <code>str</code> <p>Activation function to enforce that the final layer has a positive output. <code>'softplus'</code> or <code>'relu'</code> are supported.</p> <code>'softplus'</code> <code>pred_dim</code> <code>int</code> <p>Size of the final wide and deep output layer containing the predictions. <code>1</code> for regression and binary classification or number of classes for multiclass classification.</p> <code>1</code> <p>Examples:</p> <pre><code>&gt;&gt;&gt; from pytorch_widedeep.models import TabResnet, Vision, BasicRNN, Wide, WideDeep\n&gt;&gt;&gt; embed_input = [(u, i, j) for u, i, j in zip([\"a\", \"b\", \"c\"][:4], [4] * 3, [8] * 3)]\n&gt;&gt;&gt; column_idx = {k: v for v, k in enumerate([\"a\", \"b\", \"c\"])}\n&gt;&gt;&gt; wide = Wide(10, 1)\n&gt;&gt;&gt; deeptabular = TabResnet(blocks_dims=[8, 4], column_idx=column_idx, cat_embed_input=embed_input)\n&gt;&gt;&gt; deeptext = BasicRNN(vocab_size=10, embed_dim=4, padding_idx=0)\n&gt;&gt;&gt; deepimage = Vision()\n&gt;&gt;&gt; model = WideDeep(wide=wide, deeptabular=deeptabular, deeptext=deeptext, deepimage=deepimage)\n</code></pre> <p> NOTE: It is possible to use custom components to  build Wide &amp; Deep models. Simply, build them and pass them as the  corresponding parameters. Note that the custom models MUST return a last  layer of activations(i.e. not the final prediction) so that  these  activations are collected by <code>WideDeep</code> and combined accordingly. In  addition, the models MUST also contain an attribute <code>output_dim</code> with  the size of these last layers of activations. See for example  <code>pytorch_widedeep.models.tab_mlp.TabMlp</code></p> Source code in <code>pytorch_widedeep/models/wide_deep.py</code> <pre><code>class WideDeep(nn.Module):\n    r\"\"\"Main collector class that combines all `wide`, `deeptabular`\n    `deeptext` and `deepimage` models.\n\n    Note that all models described so far in this library must be passed to\n    the `WideDeep` class once constructed. This is because the models output\n    the last layer before the prediction layer. Such prediction layer is\n    added by the `WideDeep` class as it collects the components for every\n    data mode.\n\n    There are two options to combine these models that correspond to the\n    two main architectures that `pytorch-widedeep` can build.\n\n    - Directly connecting the output of the model components to an ouput neuron(s).\n\n    - Adding a `Fully-Connected Head` (FC-Head) on top of the deep models.\n      This FC-Head will combine the output form the `deeptabular`, `deeptext` and\n      `deepimage` and will be then connected to the output neuron(s).\n\n    Parameters\n    ----------\n    wide: nn.Module, Optional, default = None\n        `Wide` model. This is a linear model where the non-linearities are\n        captured via crossed-columns.\n    deeptabular: BaseWDModelComponent, Optional, default = None\n        Currently this library implements a number of possible architectures\n        for the `deeptabular` component. See the documenation of the\n        package. Note that `deeptabular` can be a list of models. This is\n        useful when using multiple tabular inputs (e.g. for example in the\n        context of a two-tower model for recommendation systems)\n    deeptext: BaseWDModelComponent | List[BaseWDModelComponent], Optional, default = None\n        Currently this library implements a number of possible architectures\n        for the `deeptext` component. See the documenation of the\n        package. Note that `deeptext` can be a list of models. This is useful\n        when using multiple text inputs.\n    deepimage: BaseWDModelComponent | List[BaseWDModelComponent], Optional, default = None\n        Currently this library uses `torchvision` and implements a number of\n        possible architectures for the `deepimage` component. See the\n        documenation of the package. Note that `deepimage` can be a list of\n        models. This is useful when using multiple image inputs.\n    deephead: BaseWDModelComponent, Optional, default = None\n        Alternatively, the user can pass a custom model that will receive the\n        output of the deep component. If `deephead` is not None all the\n        previous fc-head parameters will be ignored\n    head_hidden_dims: List, Optional, default = None\n        List with the sizes of the dense layers in the head e.g: [128, 64]\n    head_activation: str, default = \"relu\"\n        Activation function for the dense layers in the head. Currently\n        `'tanh'`, `'relu'`, `'leaky_relu'` and `'gelu'` are supported\n    head_dropout: float, Optional, default = None\n        Dropout of the dense layers in the head\n    head_batchnorm: bool, default = False\n        Boolean indicating whether or not to include batch normalization in\n        the dense layers that form the `'rnn_mlp'`\n    head_batchnorm_last: bool, default = False\n        Boolean indicating whether or not to apply batch normalization to the\n        last of the dense layers in the head\n    head_linear_first: bool, default = False\n        Boolean indicating whether the order of the operations in the dense\n        layer. If `True: [LIN -&gt; ACT -&gt; BN -&gt; DP]`. If `False: [BN -&gt; DP -&gt;\n        LIN -&gt; ACT]`\n    enforce_positive: bool, default = False\n        Boolean indicating if the output from the final layer must be\n        positive. This is important if you are using loss functions with\n        non-negative input restrictions, e.g. RMSLE, or if you know your\n        predictions are bounded in between 0 and inf\n    enforce_positive_activation: str, default = \"softplus\"\n        Activation function to enforce that the final layer has a positive\n        output. `'softplus'` or `'relu'` are supported.\n    pred_dim: int, default = 1\n        Size of the final wide and deep output layer containing the\n        predictions. `1` for regression and binary classification or number\n        of classes for multiclass classification.\n\n\n    Examples\n    --------\n\n    &gt;&gt;&gt; from pytorch_widedeep.models import TabResnet, Vision, BasicRNN, Wide, WideDeep\n    &gt;&gt;&gt; embed_input = [(u, i, j) for u, i, j in zip([\"a\", \"b\", \"c\"][:4], [4] * 3, [8] * 3)]\n    &gt;&gt;&gt; column_idx = {k: v for v, k in enumerate([\"a\", \"b\", \"c\"])}\n    &gt;&gt;&gt; wide = Wide(10, 1)\n    &gt;&gt;&gt; deeptabular = TabResnet(blocks_dims=[8, 4], column_idx=column_idx, cat_embed_input=embed_input)\n    &gt;&gt;&gt; deeptext = BasicRNN(vocab_size=10, embed_dim=4, padding_idx=0)\n    &gt;&gt;&gt; deepimage = Vision()\n    &gt;&gt;&gt; model = WideDeep(wide=wide, deeptabular=deeptabular, deeptext=deeptext, deepimage=deepimage)\n\n\n    :information_source: **NOTE**: It is possible to use custom components to\n     build Wide &amp; Deep models. Simply, build them and pass them as the\n     corresponding parameters. Note that the custom models MUST return a last\n     layer of activations(i.e. not the final prediction) so that  these\n     activations are collected by `WideDeep` and combined accordingly. In\n     addition, the models MUST also contain an attribute `output_dim` with\n     the size of these last layers of activations. See for example\n     `pytorch_widedeep.models.tab_mlp.TabMlp`\n    \"\"\"\n\n    @alias(  # noqa: C901\n        \"pred_dim\",\n        [\"num_class\", \"pred_size\"],\n    )\n    def __init__(\n        self,\n        wide: Optional[nn.Module] = None,\n        deeptabular: Optional[\n            Union[BaseWDModelComponent, List[BaseWDModelComponent]]\n        ] = None,\n        deeptext: Optional[\n            Union[BaseWDModelComponent, List[BaseWDModelComponent]]\n        ] = None,\n        deepimage: Optional[\n            Union[BaseWDModelComponent, List[BaseWDModelComponent]]\n        ] = None,\n        deephead: Optional[BaseWDModelComponent] = None,\n        head_hidden_dims: Optional[List[int]] = None,\n        head_activation: str = \"relu\",\n        head_dropout: float = 0.1,\n        head_batchnorm: bool = False,\n        head_batchnorm_last: bool = False,\n        head_linear_first: bool = True,\n        enforce_positive: bool = False,\n        enforce_positive_activation: str = \"softplus\",\n        pred_dim: int = 1,\n    ):\n        super(WideDeep, self).__init__()\n\n        self._check_inputs(\n            wide,\n            deeptabular,\n            deeptext,\n            deepimage,\n            deephead,\n            head_hidden_dims,\n            pred_dim,\n        )\n\n        # this attribute will be eventually over-written by the Trainer's\n        # device. Acts here as a 'placeholder'.\n        self.wd_device: Optional[str] = None\n\n        # required as attribute just in case we pass a deephead\n        self.pred_dim = pred_dim\n\n        self.enforce_positive = enforce_positive\n\n        # better to set this attribute already here\n        if isinstance(deeptabular, list):\n            self.is_tabnet = False\n        else:\n            self.is_tabnet = deeptabular.__class__.__name__ == \"TabNet\"\n\n        # The main 5 components of the wide and deep assemble: wide,\n        # deeptabular, deeptext, deepimage and deephead\n        self.with_deephead = deephead is not None or head_hidden_dims is not None\n        if deephead is None and head_hidden_dims is not None:\n            self.deephead = self._build_deephead(\n                deeptabular,\n                deeptext,\n                deepimage,\n                head_hidden_dims,\n                head_activation,\n                head_dropout,\n                head_batchnorm,\n                head_batchnorm_last,\n                head_linear_first,\n            )\n        elif deephead is not None:\n            self.deephead = nn.Sequential(\n                deephead, nn.Linear(deephead.output_dim, self.pred_dim)\n            )\n        else:\n            # for consistency with other components we default to None\n            self.deephead = None\n\n        self.wide = wide\n        self.deeptabular, self.deeptext, self.deepimage = self._set_model_components(\n            deeptabular, deeptext, deepimage\n        )\n\n        if self.enforce_positive:\n            self.enf_pos = get_activation_fn(enforce_positive_activation)\n\n    def forward(\n        self,\n        X: Dict[str, Union[Tensor, List[Tensor]]],\n        y: Optional[Tensor] = None,\n    ) -&gt; Union[Tensor, Tuple[Tensor, Tensor]]:\n\n        wide_out = self._forward_wide(X)\n        if self.with_deephead:\n            deep = self._forward_deephead(X, wide_out)\n        else:\n            deep = self._forward_deep(X, wide_out)\n\n        if self.enforce_positive:\n            return self.enf_pos(deep)\n        else:\n            return deep\n\n    def _build_deephead(\n        self,\n        deeptabular: Optional[Union[BaseWDModelComponent, List[BaseWDModelComponent]]],\n        deeptext: Optional[Union[BaseWDModelComponent, List[BaseWDModelComponent]]],\n        deepimage: Optional[Union[BaseWDModelComponent, List[BaseWDModelComponent]]],\n        head_hidden_dims: List[int],\n        head_activation: str,\n        head_dropout: float,\n        head_batchnorm: bool,\n        head_batchnorm_last: bool,\n        head_linear_first: bool,\n    ) -&gt; nn.Sequential:\n        deep_dim = 0\n        if deeptabular is not None:\n            if isinstance(deeptabular, list):\n                for dt in deeptabular:\n                    deep_dim += dt.output_dim\n            else:\n                deep_dim += deeptabular.output_dim\n        if deeptext is not None:\n            if isinstance(deeptext, list):\n                for dt in deeptext:\n                    deep_dim += dt.output_dim\n            else:\n                deep_dim += deeptext.output_dim\n        if deepimage is not None:\n            if isinstance(deepimage, list):\n                for di in deepimage:\n                    deep_dim += di.output_dim\n            else:\n                deep_dim += deepimage.output_dim\n\n        head_hidden_dims = [deep_dim] + head_hidden_dims\n        deephead = nn.Sequential(\n            MLP(\n                head_hidden_dims,\n                head_activation,\n                head_dropout,\n                head_batchnorm,\n                head_batchnorm_last,\n                head_linear_first,\n            ),\n            nn.Linear(head_hidden_dims[-1], self.pred_dim),\n        )\n\n        return deephead\n\n    def _set_model_components(  # noqa: C901\n        self,\n        deeptabular: Optional[Union[BaseWDModelComponent, List[BaseWDModelComponent]]],\n        deeptext: Optional[Union[BaseWDModelComponent, List[BaseWDModelComponent]]],\n        deepimage: Optional[Union[BaseWDModelComponent, List[BaseWDModelComponent]]],\n    ) -&gt; Tuple[\n        Optional[Union[nn.ModuleList, WDModel]],\n        Optional[Union[nn.ModuleList, WDModel]],\n        Optional[Union[nn.ModuleList, WDModel]],\n    ]:\n        if deeptabular is not None:\n            deeptabular_ = self._set_model_component(deeptabular, is_deeptabular=True)\n        else:\n            deeptabular_ = None\n\n        if deeptext is not None:\n            deeptext_ = self._set_model_component(deeptext)\n        else:\n            deeptext_ = None\n\n        if deepimage is not None:\n            deepimage_ = self._set_model_component(deepimage)\n        else:\n            deepimage_ = None\n\n        return deeptabular_, deeptext_, deepimage_\n\n    def _forward_wide(self, X: Dict[str, Union[Tensor, List[Tensor]]]) -&gt; Tensor:\n        if self.wide is not None:\n            out = self.wide(X[\"wide\"])\n        else:\n            first_model_mode = list(X.keys())[0]\n            if isinstance(X[first_model_mode], list):\n                batch_size = X[first_model_mode][0].size(0)\n            else:\n                batch_size = X[first_model_mode].size(0)  # type: ignore[union-attr]\n            out = torch.zeros(batch_size, self.pred_dim).to(self.wd_device)\n\n        return out\n\n    def _forward_deep(\n        self, X: Dict[str, Union[Tensor, List[Tensor]]], wide_out: Tensor\n    ) -&gt; Union[Tensor, Tuple[Tensor, Tensor]]:\n        if self.deeptabular is not None:\n            if self.is_tabnet:\n                tab_out, M_loss = self.deeptabular(X[\"deeptabular\"])\n                wide_out.add_(tab_out)\n            else:\n                wide_out = self._forward_component(\n                    X, self.deeptabular, \"deeptabular\", wide_out\n                )\n\n        if self.deeptext is not None:\n            wide_out = self._forward_component(X, self.deeptext, \"deeptext\", wide_out)\n\n        if self.deepimage is not None:\n            wide_out = self._forward_component(X, self.deepimage, \"deepimage\", wide_out)\n\n        if self.is_tabnet:\n            res: Union[Tensor, Tuple[Tensor, Tensor]] = (wide_out, M_loss)\n        else:\n            res = wide_out\n\n        return res\n\n    def _forward_deephead(\n        self, X: Dict[str, Union[Tensor, List[Tensor]]], wide_out: Tensor\n    ) -&gt; Union[Tensor, Tuple[Tensor, Tensor]]:\n        deepside = torch.FloatTensor().to(self.wd_device)\n\n        if self.deeptabular is not None:\n            if self.is_tabnet:\n                deepside, M_loss = self.deeptabular(X[\"deeptabular\"])\n            else:\n                deepside = self._forward_component_with_head(\n                    X, self.deeptabular, \"deeptabular\", deepside\n                )\n\n        if self.deeptext is not None:\n            deepside = self._forward_component_with_head(\n                X, self.deeptext, \"deeptext\", deepside\n            )\n\n        if self.deepimage is not None:\n            deepside = self._forward_component_with_head(\n                X, self.deepimage, \"deepimage\", deepside\n            )\n\n        # assertion to avoid type issues\n        assert self.deephead is not None\n        deepside_out = self.deephead(deepside)\n\n        if self.is_tabnet:\n            res: Union[Tensor, Tuple[Tensor, Tensor]] = (\n                wide_out.add_(deepside_out),\n                M_loss,\n            )\n        else:\n            res = wide_out.add_(deepside_out)\n\n        return res\n\n    def _forward_component(\n        self,\n        X: Dict[str, Union[Tensor, List[Tensor]]],\n        component: Union[nn.ModuleList, WDModel],\n        component_type: Literal[\"deeptabular\", \"deeptext\", \"deepimage\"],\n        wide_out: Tensor,\n    ) -&gt; Tensor:\n        if isinstance(component, nn.ModuleList):\n            component_out = torch.add(  # type: ignore[call-overload]\n                *[cp(X[component_type][i]) for i, cp in enumerate(component)]\n            )\n        else:\n            component_out = component(X[component_type])\n\n        return wide_out.add_(component_out)\n\n    def _forward_component_with_head(\n        self,\n        X: Dict[str, Union[Tensor, List[Tensor]]],\n        component: Union[nn.ModuleList, WDModel],\n        component_type: Literal[\"deeptabular\", \"deeptext\", \"deepimage\"],\n        deepside: Tensor,\n    ) -&gt; Tensor:\n        if isinstance(component, nn.ModuleList):\n            component_out = torch.cat(  # type: ignore[call-overload]\n                [cp(X[component_type][i]) for i, cp in enumerate(component)], axis=1\n            )\n        else:\n            component_out = component(X[component_type])\n\n        return torch.cat([deepside, component_out], axis=1)  # type: ignore[call-overload]\n\n    def _set_model_component(\n        self,\n        component: Union[BaseWDModelComponent, List[BaseWDModelComponent]],\n        is_deeptabular: bool = False,\n    ) -&gt; Union[nn.ModuleList, WDModel]:\n        if isinstance(component, list):\n            component_: Optional[Union[nn.ModuleList, WDModel]] = nn.ModuleList()\n            for cp in component:\n                if self.with_deephead or cp.output_dim == 1:\n                    component_.append(cp)\n                else:\n                    component_.append(\n                        nn.Sequential(cp, nn.Linear(cp.output_dim, self.pred_dim))\n                    )\n        elif self.with_deephead or component.output_dim == 1:\n            component_ = component\n        elif is_deeptabular and self.is_tabnet:\n            component_ = nn.Sequential(\n                component, TabNetPredLayer(component.output_dim, self.pred_dim)\n            )\n        else:\n            component_ = nn.Sequential(\n                component, nn.Linear(component.output_dim, self.pred_dim)\n            )\n\n        return component_\n\n    @staticmethod  # noqa: C901\n    def _check_inputs(  # noqa: C901\n        wide,\n        deeptabular,\n        deeptext,\n        deepimage,\n        deephead,\n        head_hidden_dims,\n        pred_dim,\n    ):\n        if wide is not None:\n            assert wide.wide_linear.weight.size(1) == pred_dim, (\n                \"the 'pred_dim' of the wide component ({}) must be equal to the 'pred_dim' \"\n                \"of the deep component and the overall model itself ({})\".format(\n                    wide.wide_linear.weight.size(1), pred_dim\n                )\n            )\n\n        if deeptabular is not None:\n            err_msg = (\n                \"deeptabular model must have an 'output_dim' attribute or property.\"\n            )\n            if isinstance(deeptabular, list):\n                all_have_output_dim = all(\n                    hasattr(dt, \"output_dim\") for dt in deeptabular\n                )\n                if not all_have_output_dim:\n                    raise AttributeError(err_msg)\n            else:\n                if not hasattr(deeptabular, \"output_dim\"):\n                    raise AttributeError(err_msg)\n                # the following assertion is thought for those cases where we\n                # use fusion with 'dot product' so that the output_dim will\n                # be 1 and the pred_dim is not 1\n                if deeptabular.output_dim == 1:\n                    assert pred_dim == 1, \"If 'output_dim' is 1, 'pred_dim' must be 1\"\n\n        if deeptabular is not None:\n            is_tabnet = False\n            if isinstance(deeptabular, list):\n                is_any_tabnet = any(\n                    dt.__class__.__name__ == \"TabNet\" for dt in deeptabular\n                )\n                if is_any_tabnet:\n                    raise ValueError(\n                        \"Currently TabNet is not supported as a component of a multiple \"\n                        \"tabular component model.\"\n                    )\n            else:\n                is_tabnet = deeptabular.__class__.__name__ == \"TabNet\"\n            has_wide_text_or_image = (\n                wide is not None or deeptext is not None or deepimage is not None\n            )\n            if is_tabnet and has_wide_text_or_image:\n                warnings.warn(\n                    \"'WideDeep' is a model comprised by multiple components and the 'deeptabular'\"\n                    \" component is 'TabNet'. We recommend using 'TabNet' in isolation.\"\n                    \" The reasons are: i)'TabNet' uses sparse regularization which partially losses\"\n                    \" its purpose when used in combination with other components.\"\n                    \" If you still want to use a multiple component model with 'TabNet',\"\n                    \" consider setting 'lambda_sparse' to 0 during training. ii) The feature\"\n                    \" importances will be computed only for TabNet but the model will comprise multiple\"\n                    \" components. Therefore, such importances will partially lose their 'meaning'.\",\n                    UserWarning,\n                )\n\n        if deeptext is not None:\n            err_msg = \"deeptext model must have an 'output_dim' attribute or property.\"\n            if isinstance(deeptext, list):\n                all_have_output_dim = all(hasattr(dt, \"output_dim\") for dt in deeptext)\n                if not all_have_output_dim:\n                    raise AttributeError(err_msg)\n            else:\n                if not hasattr(deeptext, \"output_dim\"):\n                    raise AttributeError(err_msg)\n                if deeptext.output_dim == 1:\n                    assert pred_dim == 1, \"If 'output_dim' is 1, 'pred_dim' must be 1\"\n\n        if deepimage is not None:\n            err_msg = \"deepimage model must have an 'output_dim' attribute or property.\"\n            if isinstance(deepimage, list):\n                all_have_output_dim = all(hasattr(di, \"output_dim\") for di in deepimage)\n                if not all_have_output_dim:\n                    raise AttributeError(err_msg)\n            else:\n                if not hasattr(deepimage, \"output_dim\"):\n                    raise AttributeError(err_msg)\n                if deepimage.output_dim == 1:\n                    assert pred_dim == 1, \"If 'output_dim' is 1, 'pred_dim' must be 1\"\n\n        if deephead is not None and head_hidden_dims is not None:\n            raise ValueError(\n                \"both 'deephead' and 'head_hidden_dims' are not None. Use one of the other, but not both\"\n            )\n        if (\n            head_hidden_dims is not None\n            and not deeptabular\n            and not deeptext\n            and not deepimage\n        ):\n            raise ValueError(\n                \"if 'head_hidden_dims' is not None, at least one deep component must be used\"\n            )\n\n        if deephead is not None:\n            if not hasattr(deephead, \"output_dim\"):\n                raise AttributeError(\n                    \"As any other custom model passed to 'WideDeep', 'deephead' must have an \"\n                    \"'output_dim' attribute or property. \"\n                )\n            deephead_inp_feat = next(deephead.parameters()).size(1)\n            output_dim = 0\n            if deeptabular is not None:\n                if isinstance(deeptabular, list):\n                    for dt in deeptabular:\n                        output_dim += dt.output_dim\n                else:\n                    output_dim += deeptabular.output_dim\n            if deeptext is not None:\n                if isinstance(deeptext, list):\n                    for dt in deeptext:\n                        output_dim += dt.output_dim\n                else:\n                    output_dim += deeptext.output_dim\n            if deepimage is not None:\n                if isinstance(deepimage, list):\n                    for di in deepimage:\n                        output_dim += di.output_dim\n                else:\n                    output_dim += deepimage.output_dim\n            if deephead_inp_feat != output_dim:\n                warnings.warn(\n                    \"A custom 'deephead' is used and it seems that the input features \"\n                    \"do not match the output of the deep components\",\n                    UserWarning,\n                )\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html","title":"The <code>preprocessing</code> module","text":"<p>This module contains the classes that are used to prepare the data before being passed to the models. There is one Preprocessor per data mode or model component (<code>wide</code>, <code>deeptabular</code>, <code>deepimage</code> and <code>deeptext</code>) with the exception of the <code>deeptext</code> component. In this case, two processors are available: one for the case when no Hugging Face model is used (<code>TextPreprocessor</code>) and another one when a Hugging Face model is used (<code>HFPreprocessor</code>).</p>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.wide_preprocessor.WidePreprocessor","title":"WidePreprocessor","text":"<p>               Bases: <code>BasePreprocessor</code></p> <p>Preprocessor to prepare the wide input dataset</p> <p>This Preprocessor prepares the data for the wide, linear component. This linear model is implemented via an Embedding layer that is connected to the output neuron. <code>WidePreprocessor</code> numerically encodes all the unique values of all categorical columns <code>wide_cols + crossed_cols</code>. See the Example below.</p> <p>Parameters:</p> Name Type Description Default <code>wide_cols</code> <code>List[str]</code> <p>List of strings with the name of the columns that will label encoded and passed through the <code>wide</code> component</p> required <code>crossed_cols</code> <code>Optional[List[Tuple[str, str]]]</code> <p>List of Tuples with the name of the columns that will be <code>'crossed'</code> and then label encoded. e.g. [('education', 'occupation'), ...]. For binary features, a cross-product transformation is 1 if and only if the constituent features are all 1, and 0 otherwise.</p> <code>None</code> <p>Attributes:</p> Name Type Description <code>wide_crossed_cols</code> <code>List</code> <p>List with the names of all columns that will be label encoded</p> <code>encoding_dict</code> <code>Dict</code> <p>Dictionary where the keys are the result of pasting <code>colname + '_' + column value</code> and the values are the corresponding mapped integer.</p> <code>inverse_encoding_dict</code> <code>Dict</code> <p>the inverse encoding dictionary</p> <code>wide_dim</code> <code>int</code> <p>Dimension of the wide model (i.e. dim of the linear layer)</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import pandas as pd\n&gt;&gt;&gt; from pytorch_widedeep.preprocessing import WidePreprocessor\n&gt;&gt;&gt; df = pd.DataFrame({'color': ['r', 'b', 'g'], 'size': ['s', 'n', 'l']})\n&gt;&gt;&gt; wide_cols = ['color']\n&gt;&gt;&gt; crossed_cols = [('color', 'size')]\n&gt;&gt;&gt; wide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols)\n&gt;&gt;&gt; X_wide = wide_preprocessor.fit_transform(df)\n&gt;&gt;&gt; X_wide\narray([[1, 4],\n       [2, 5],\n       [3, 6]])\n&gt;&gt;&gt; wide_preprocessor.encoding_dict\n{'color_r': 1, 'color_b': 2, 'color_g': 3, 'color_size_r-s': 4, 'color_size_b-n': 5, 'color_size_g-l': 6}\n&gt;&gt;&gt; wide_preprocessor.inverse_transform(X_wide)\n  color color_size\n0     r        r-s\n1     b        b-n\n2     g        g-l\n</code></pre> Source code in <code>pytorch_widedeep/preprocessing/wide_preprocessor.py</code> <pre><code>class WidePreprocessor(BasePreprocessor):\n    r\"\"\"Preprocessor to prepare the wide input dataset\n\n    This Preprocessor prepares the data for the wide, linear component.\n    This linear model is implemented via an Embedding layer that is\n    connected to the output neuron. `WidePreprocessor` numerically\n    encodes all the unique values of all categorical columns `wide_cols +\n    crossed_cols`. See the Example below.\n\n    Parameters\n    ----------\n    wide_cols: List\n        List of strings with the name of the columns that will label\n        encoded and passed through the `wide` component\n    crossed_cols: List, default = None\n        List of Tuples with the name of the columns that will be `'crossed'`\n        and then label encoded. e.g. _[('education', 'occupation'), ...]_. For\n        binary features, a cross-product transformation is 1 if and only if\n        the constituent features are all 1, and 0 otherwise.\n\n    Attributes\n    ----------\n    wide_crossed_cols: List\n        List with the names of all columns that will be label encoded\n    encoding_dict: Dict\n        Dictionary where the keys are the result of pasting `colname + '_' +\n        column value` and the values are the corresponding mapped integer.\n    inverse_encoding_dict: Dict\n        the inverse encoding dictionary\n    wide_dim: int\n        Dimension of the wide model (i.e. dim of the linear layer)\n\n    Examples\n    --------\n    &gt;&gt;&gt; import pandas as pd\n    &gt;&gt;&gt; from pytorch_widedeep.preprocessing import WidePreprocessor\n    &gt;&gt;&gt; df = pd.DataFrame({'color': ['r', 'b', 'g'], 'size': ['s', 'n', 'l']})\n    &gt;&gt;&gt; wide_cols = ['color']\n    &gt;&gt;&gt; crossed_cols = [('color', 'size')]\n    &gt;&gt;&gt; wide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols)\n    &gt;&gt;&gt; X_wide = wide_preprocessor.fit_transform(df)\n    &gt;&gt;&gt; X_wide\n    array([[1, 4],\n           [2, 5],\n           [3, 6]])\n    &gt;&gt;&gt; wide_preprocessor.encoding_dict\n    {'color_r': 1, 'color_b': 2, 'color_g': 3, 'color_size_r-s': 4, 'color_size_b-n': 5, 'color_size_g-l': 6}\n    &gt;&gt;&gt; wide_preprocessor.inverse_transform(X_wide)\n      color color_size\n    0     r        r-s\n    1     b        b-n\n    2     g        g-l\n    \"\"\"\n\n    def __init__(\n        self, wide_cols: List[str], crossed_cols: Optional[List[Tuple[str, str]]] = None\n    ):\n        super(WidePreprocessor, self).__init__()\n\n        self.wide_cols = wide_cols\n        self.crossed_cols = crossed_cols\n\n        self.is_fitted = False\n\n    def fit(self, df: pd.DataFrame) -&gt; \"WidePreprocessor\":\n        r\"\"\"Fits the Preprocessor and creates required attributes\n\n        Parameters\n        ----------\n        df: pd.DataFrame\n            Input pandas dataframe\n\n        Returns\n        -------\n        WidePreprocessor\n            `WidePreprocessor` fitted object\n        \"\"\"\n        df_wide = self._prepare_wide(df)\n        self.wide_crossed_cols = df_wide.columns.tolist()\n        glob_feature_list = self._make_global_feature_list(\n            df_wide[self.wide_crossed_cols]\n        )\n        # leave 0 for padding/\"unseen\" categories\n        self.encoding_dict = {v: i + 1 for i, v in enumerate(glob_feature_list)}\n        self.wide_dim = len(self.encoding_dict)\n        self.inverse_encoding_dict = {k: v for v, k in self.encoding_dict.items()}\n        self.inverse_encoding_dict[0] = \"unseen\"\n\n        self.is_fitted = True\n\n        return self\n\n    def transform(self, df: pd.DataFrame) -&gt; np.ndarray:\n        r\"\"\"\n        Parameters\n        ----------\n        df: pd.DataFrame\n            Input pandas dataframe\n\n        Returns\n        -------\n        np.ndarray\n            transformed input dataframe\n        \"\"\"\n        check_is_fitted(self, attributes=[\"encoding_dict\"])\n        df_wide = self._prepare_wide(df)\n        encoded = np.zeros([len(df_wide), len(self.wide_crossed_cols)])\n        for col_i, col in enumerate(self.wide_crossed_cols):\n            encoded[:, col_i] = df_wide[col].apply(\n                lambda x: (\n                    self.encoding_dict[col + \"_\" + str(x)]\n                    if col + \"_\" + str(x) in self.encoding_dict\n                    else 0\n                )\n            )\n        return encoded.astype(\"int64\")\n\n    def transform_sample(self, df: pd.DataFrame) -&gt; np.ndarray:\n        return self.transform(df)[0]\n\n    def inverse_transform(self, encoded: np.ndarray) -&gt; pd.DataFrame:\n        r\"\"\"Takes as input the output from the `transform` method and it will\n        return the original values.\n\n        Parameters\n        ----------\n        encoded: np.ndarray\n            numpy array with the encoded values that are the output from the\n            `transform` method\n\n        Returns\n        -------\n        pd.DataFrame\n            Pandas dataframe with the original values\n        \"\"\"\n        decoded = pd.DataFrame(encoded, columns=self.wide_crossed_cols)\n\n        if pd.__version__ &gt;= \"2.1.0\":\n            decoded = decoded.map(lambda x: self.inverse_encoding_dict[x])\n        else:\n            decoded = decoded.applymap(lambda x: self.inverse_encoding_dict[x])\n\n        for col in decoded.columns:\n            rm_str = \"\".join([col, \"_\"])\n            decoded[col] = decoded[col].apply(lambda x: x.replace(rm_str, \"\"))\n        return decoded\n\n    def fit_transform(self, df: pd.DataFrame) -&gt; np.ndarray:\n        \"\"\"Combines `fit` and `transform`\n\n        Parameters\n        ----------\n        df: pd.DataFrame\n            Input pandas dataframe\n\n        Returns\n        -------\n        np.ndarray\n            transformed input dataframe\n        \"\"\"\n        return self.fit(df).transform(df)\n\n    def _make_global_feature_list(self, df: pd.DataFrame) -&gt; List:\n        glob_feature_list = []\n        for column in df.columns:\n            glob_feature_list += self._make_column_feature_list(df[column])\n        return glob_feature_list\n\n    def _make_column_feature_list(self, s: pd.Series) -&gt; List:\n        return [s.name + \"_\" + str(x) for x in s.unique()]\n\n    def _cross_cols(self, df: pd.DataFrame):\n        df_cc = df.copy()\n        crossed_colnames = []\n        for cols in self.crossed_cols:\n            for c in cols:\n                df_cc[c] = df_cc[c].astype(\"str\")\n            colname = \"_\".join(cols)\n            df_cc[colname] = df_cc[list(cols)].apply(lambda x: \"-\".join(x), axis=1)\n            crossed_colnames.append(colname)\n        return df_cc[crossed_colnames]\n\n    def _prepare_wide(self, df: pd.DataFrame):\n        if self.crossed_cols is not None:\n            df_cc = self._cross_cols(df)\n            return pd.concat([df[self.wide_cols], df_cc], axis=1)\n        else:\n            return df.copy()[self.wide_cols]\n\n    def __repr__(self) -&gt; str:\n        list_of_params: List[str] = [\"wide_cols={wide_cols}\"]\n        if self.crossed_cols is not None:\n            list_of_params.append(\"crossed_cols={crossed_cols}\")\n        all_params = \", \".join(list_of_params)\n        return f\"WidePreprocessor({all_params.format(**self.__dict__)})\"\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.wide_preprocessor.WidePreprocessor.fit","title":"fit","text":"<pre><code>fit(df)\n</code></pre> <p>Fits the Preprocessor and creates required attributes</p> <p>Parameters:</p> Name Type Description Default <code>df</code> <code>DataFrame</code> <p>Input pandas dataframe</p> required <p>Returns:</p> Type Description <code>WidePreprocessor</code> <p><code>WidePreprocessor</code> fitted object</p> Source code in <code>pytorch_widedeep/preprocessing/wide_preprocessor.py</code> <pre><code>def fit(self, df: pd.DataFrame) -&gt; \"WidePreprocessor\":\n    r\"\"\"Fits the Preprocessor and creates required attributes\n\n    Parameters\n    ----------\n    df: pd.DataFrame\n        Input pandas dataframe\n\n    Returns\n    -------\n    WidePreprocessor\n        `WidePreprocessor` fitted object\n    \"\"\"\n    df_wide = self._prepare_wide(df)\n    self.wide_crossed_cols = df_wide.columns.tolist()\n    glob_feature_list = self._make_global_feature_list(\n        df_wide[self.wide_crossed_cols]\n    )\n    # leave 0 for padding/\"unseen\" categories\n    self.encoding_dict = {v: i + 1 for i, v in enumerate(glob_feature_list)}\n    self.wide_dim = len(self.encoding_dict)\n    self.inverse_encoding_dict = {k: v for v, k in self.encoding_dict.items()}\n    self.inverse_encoding_dict[0] = \"unseen\"\n\n    self.is_fitted = True\n\n    return self\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.wide_preprocessor.WidePreprocessor.transform","title":"transform","text":"<pre><code>transform(df)\n</code></pre> <p>Parameters:</p> Name Type Description Default <code>df</code> <code>DataFrame</code> <p>Input pandas dataframe</p> required <p>Returns:</p> Type Description <code>ndarray</code> <p>transformed input dataframe</p> Source code in <code>pytorch_widedeep/preprocessing/wide_preprocessor.py</code> <pre><code>def transform(self, df: pd.DataFrame) -&gt; np.ndarray:\n    r\"\"\"\n    Parameters\n    ----------\n    df: pd.DataFrame\n        Input pandas dataframe\n\n    Returns\n    -------\n    np.ndarray\n        transformed input dataframe\n    \"\"\"\n    check_is_fitted(self, attributes=[\"encoding_dict\"])\n    df_wide = self._prepare_wide(df)\n    encoded = np.zeros([len(df_wide), len(self.wide_crossed_cols)])\n    for col_i, col in enumerate(self.wide_crossed_cols):\n        encoded[:, col_i] = df_wide[col].apply(\n            lambda x: (\n                self.encoding_dict[col + \"_\" + str(x)]\n                if col + \"_\" + str(x) in self.encoding_dict\n                else 0\n            )\n        )\n    return encoded.astype(\"int64\")\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.wide_preprocessor.WidePreprocessor.inverse_transform","title":"inverse_transform","text":"<pre><code>inverse_transform(encoded)\n</code></pre> <p>Takes as input the output from the <code>transform</code> method and it will return the original values.</p> <p>Parameters:</p> Name Type Description Default <code>encoded</code> <code>ndarray</code> <p>numpy array with the encoded values that are the output from the <code>transform</code> method</p> required <p>Returns:</p> Type Description <code>DataFrame</code> <p>Pandas dataframe with the original values</p> Source code in <code>pytorch_widedeep/preprocessing/wide_preprocessor.py</code> <pre><code>def inverse_transform(self, encoded: np.ndarray) -&gt; pd.DataFrame:\n    r\"\"\"Takes as input the output from the `transform` method and it will\n    return the original values.\n\n    Parameters\n    ----------\n    encoded: np.ndarray\n        numpy array with the encoded values that are the output from the\n        `transform` method\n\n    Returns\n    -------\n    pd.DataFrame\n        Pandas dataframe with the original values\n    \"\"\"\n    decoded = pd.DataFrame(encoded, columns=self.wide_crossed_cols)\n\n    if pd.__version__ &gt;= \"2.1.0\":\n        decoded = decoded.map(lambda x: self.inverse_encoding_dict[x])\n    else:\n        decoded = decoded.applymap(lambda x: self.inverse_encoding_dict[x])\n\n    for col in decoded.columns:\n        rm_str = \"\".join([col, \"_\"])\n        decoded[col] = decoded[col].apply(lambda x: x.replace(rm_str, \"\"))\n    return decoded\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.wide_preprocessor.WidePreprocessor.fit_transform","title":"fit_transform","text":"<pre><code>fit_transform(df)\n</code></pre> <p>Combines <code>fit</code> and <code>transform</code></p> <p>Parameters:</p> Name Type Description Default <code>df</code> <code>DataFrame</code> <p>Input pandas dataframe</p> required <p>Returns:</p> Type Description <code>ndarray</code> <p>transformed input dataframe</p> Source code in <code>pytorch_widedeep/preprocessing/wide_preprocessor.py</code> <pre><code>def fit_transform(self, df: pd.DataFrame) -&gt; np.ndarray:\n    \"\"\"Combines `fit` and `transform`\n\n    Parameters\n    ----------\n    df: pd.DataFrame\n        Input pandas dataframe\n\n    Returns\n    -------\n    np.ndarray\n        transformed input dataframe\n    \"\"\"\n    return self.fit(df).transform(df)\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor","title":"TabPreprocessor","text":"<p>               Bases: <code>BasePreprocessor</code></p> <p>Preprocessor to prepare the <code>deeptabular</code> component input dataset</p> <p>Parameters:</p> Name Type Description Default <code>cat_embed_cols</code> <code>Optional[Union[List[str], List[Tuple[str, int]]]]</code> <p>List containing the name of the categorical columns that will be represented by embeddings (e.g. ['education', 'relationship', ...]) or a Tuple with the name and the embedding dimension (e.g.: [ ('education',32), ('relationship',16), ...]). Param alias: <code>embed_cols</code></p> <code>None</code> <code>continuous_cols</code> <code>Optional[List[str]]</code> <p>List with the name of the continuous cols</p> <code>None</code> <code>quantization_setup</code> <code>Optional[Union[int, Dict[str, Union[int, List[float]]]]]</code> <p>Continuous columns can be turned into categorical via <code>pd.cut</code>. If <code>quantization_setup</code> is an <code>int</code>, all continuous columns will be quantized using this value as the number of bins. Alternatively, a dictionary where the keys are the column names to quantize and the values are the either integers indicating the number of bins or a list of scalars indicating the bin edges can also be used. Param alias: <code>cols_and_bins</code></p> <code>None</code> <code>cols_to_scale</code> <code>Optional[Union[List[str], str]]</code> <p>List with the names of the columns that will be standarised via sklearn's <code>StandardScaler</code>. It can also be the string <code>'all'</code> in which case all the continuous cols will be scaled.</p> <code>None</code> <code>auto_embed_dim</code> <code>bool</code> <p>Boolean indicating whether the embedding dimensions will be automatically defined via rule of thumb. See <code>embedding_rule</code> below.</p> <code>True</code> <code>embedding_rule</code> <code>Literal[google, fastai_old, fastai_new]</code> <p>If <code>auto_embed_dim=True</code>, this is the choice of embedding rule of thumb. Choices are:</p> <ul> <li> <p>fastai_new: \\(min(600, round(1.6 \\times n_{cat}^{0.56}))\\)</p> </li> <li> <p>fastai_old: \\(min(50, (n_{cat}//{2})+1)\\)</p> </li> <li> <p>google: \\(min(600, round(n_{cat}^{0.24}))\\)</p> </li> </ul> <code>'fastai_new'</code> <code>default_embed_dim</code> <code>int</code> <p>Dimension for the embeddings if the embedding dimension is not provided in the <code>cat_embed_cols</code> parameter and <code>auto_embed_dim</code> is set to <code>False</code>.</p> <code>16</code> <code>with_attention</code> <code>bool</code> <p>Boolean indicating whether the preprocessed data will be passed to an attention-based model (more precisely a model where all embeddings must have the same dimensions). If <code>True</code>, the param <code>cat_embed_cols</code> must just be a list containing just the categorical column names: e.g. ['education', 'relationship', ...]. This is because they will all be  encoded using embeddings of the same dim, which will be specified  later when the model is defined.  Param alias:  <code>for_transformer</code>, <code>for_matrix_factorization</code>, <code>for_mf</code></p> <code>False</code> <code>with_cls_token</code> <code>bool</code> <p>Boolean indicating if a <code>'[CLS]'</code> token will be added to the dataset when using attention-based models. The final hidden state corresponding to this token is used as the aggregated representation for classification and regression tasks. If not, the categorical and/or continuous embeddings will be concatenated before being passed to the final MLP (if present).</p> <code>False</code> <code>shared_embed</code> <code>bool</code> <p>Boolean indicating if the embeddings will be \"shared\" when using attention-based models. The idea behind <code>shared_embed</code> is described in the Appendix A in the TabTransformer paper: 'The goal of having column embedding is to enable the model to distinguish the classes in one column from those in the other columns'. In other words, the idea is to let the model learn which column is embedded at the time. See: <code>pytorch_widedeep.models.transformers._layers.SharedEmbeddings</code>.</p> <code>False</code> <code>verbose</code> <code>int</code> <code>1</code> <code>scale</code> <code>bool</code> <p> note: this arg will be removed in upcoming  releases. Please use <code>cols_to_scale</code> instead.  Bool indicating  whether or not to scale/standarise continuous cols. It is important  to emphasize that all the DL models for tabular data in the library  also include the possibility of normalising the input continuous  features via a <code>BatchNorm</code> or a <code>LayerNorm</code>.  Param alias:  <code>scale_cont_cols</code>.</p> <code>False</code> <code>already_standard</code> <code>Optional[List[str]]</code> <p> note: this arg will be removed in upcoming  releases. Please use <code>cols_to_scale</code> instead.  List with the  name of the continuous cols that do not need to be  scaled/standarised.</p> <code>None</code> <p>Other Parameters:</p> Name Type Description <code>**kwargs</code> <p><code>pd.cut</code> and <code>StandardScaler</code> related args</p> <p>Attributes:</p> Name Type Description <code>cat_cols</code> <code>List[str]</code> <p>List containing the names of the categorical columns after processing.</p> <code>cols_and_bins</code> <code>Optional[Dict[str, Union[int, List[float]]]]</code> <p>Dictionary containing the quantization setup after processing if quantization is requested. This is derived from the quantization_setup parameter.</p> <code>quant_args</code> <code>Dict</code> <p>Dictionary containing arguments passed to pandas.cut() function for quantization.</p> <code>scale_args</code> <code>Dict</code> <p>Dictionary containing arguments passed to StandardScaler.</p> <code>label_encoder</code> <code>LabelEncoder</code> <p>Instance of LabelEncoder used to encode categorical variables. See <code>pytorch_widedeep.utils.dense_utils.LabelEncoder</code>.</p> <code>cat_embed_input</code> <code>List</code> <p>List of tuples with the column name, number of individual values for that column and, if <code>with_attention</code> is set to <code>False</code>, the corresponding embeddings dim, e.g. [('education', 16, 10), ('relationship', 6, 8), ...].</p> <code>standardize_cols</code> <code>List</code> <p>List of the columns that will be standardized.</p> <code>scaler</code> <code>StandardScaler</code> <p>An instance of <code>sklearn.preprocessing.StandardScaler</code> if standardization is requested.</p> <code>column_idx</code> <code>Dict</code> <p>Dictionary where keys are column names and values are column indexes. This is necessary to slice tensors.</p> <code>quantizer</code> <code>Quantizer</code> <p>An instance of <code>Quantizer</code> if quantization is requested.</p> <code>is_fitted</code> <code>bool</code> <p>Boolean indicating if the preprocessor has been fitted.</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import pandas as pd\n&gt;&gt;&gt; import numpy as np\n&gt;&gt;&gt; from pytorch_widedeep.preprocessing import TabPreprocessor\n&gt;&gt;&gt; df = pd.DataFrame({'color': ['r', 'b', 'g'], 'size': ['s', 'n', 'l'], 'age': [25, 40, 55]})\n&gt;&gt;&gt; cat_embed_cols = [('color',5), ('size',5)]\n&gt;&gt;&gt; cont_cols = ['age']\n&gt;&gt;&gt; deep_preprocessor = TabPreprocessor(cat_embed_cols=cat_embed_cols, continuous_cols=cont_cols)\n&gt;&gt;&gt; X_tab = deep_preprocessor.fit_transform(df)\n&gt;&gt;&gt; deep_preprocessor.cat_embed_cols\n[('color', 5), ('size', 5)]\n&gt;&gt;&gt; deep_preprocessor.column_idx\n{'color': 0, 'size': 1, 'age': 2}\n&gt;&gt;&gt; cont_df = pd.DataFrame({\"col1\": np.random.rand(10), \"col2\": np.random.rand(10) + 1})\n&gt;&gt;&gt; cont_cols = [\"col1\", \"col2\"]\n&gt;&gt;&gt; tab_preprocessor = TabPreprocessor(continuous_cols=cont_cols, quantization_setup=3)\n&gt;&gt;&gt; ft_cont_df = tab_preprocessor.fit_transform(cont_df)\n&gt;&gt;&gt; # or...\n&gt;&gt;&gt; quantization_setup = {'col1': [0., 0.4, 1.], 'col2': [1., 1.4, 2.]}\n&gt;&gt;&gt; tab_preprocessor2 = TabPreprocessor(continuous_cols=cont_cols, quantization_setup=quantization_setup)\n&gt;&gt;&gt; ft_cont_df2 = tab_preprocessor2.fit_transform(cont_df)\n</code></pre> Source code in <code>pytorch_widedeep/preprocessing/tab_preprocessor.py</code> <pre><code>class TabPreprocessor(BasePreprocessor):\n    r\"\"\"Preprocessor to prepare the `deeptabular` component input dataset\n\n    Parameters\n    ----------\n    cat_embed_cols: List, default = None\n        List containing the name of the categorical columns that will be\n        represented by embeddings (e.g. _['education', 'relationship', ...]_) or\n        a Tuple with the name and the embedding dimension (e.g.: _[\n        ('education',32), ('relationship',16), ...]_).&lt;br/&gt; Param alias: `embed_cols`\n    continuous_cols: List, default = None\n        List with the name of the continuous cols\n    quantization_setup: int or Dict, default = None\n        Continuous columns can be turned into categorical via `pd.cut`. If\n        `quantization_setup` is an `int`, all continuous columns will be\n        quantized using this value as the number of bins. Alternatively, a\n        dictionary where the keys are the column names to quantize and the\n        values are the either integers indicating the number of bins or a\n        list of scalars indicating the bin edges can also be used.&lt;br/&gt;\n        Param alias: `cols_and_bins`\n    cols_to_scale: List or str, default = None,\n        List with the names of the columns that will be standarised via\n        sklearn's `StandardScaler`. It can also be the string `'all'` in\n        which case all the continuous cols will be scaled.\n    auto_embed_dim: bool, default = True\n        Boolean indicating whether the embedding dimensions will be\n        automatically defined via rule of thumb. See `embedding_rule`\n        below.\n    embedding_rule: str, default = 'fastai_new'\n        If `auto_embed_dim=True`, this is the choice of embedding rule of\n        thumb. Choices are:\n\n        - _fastai_new_: $min(600, round(1.6 \\times n_{cat}^{0.56}))$\n\n        - _fastai_old_: $min(50, (n_{cat}//{2})+1)$\n\n        - _google_: $min(600, round(n_{cat}^{0.24}))$\n    default_embed_dim: int, default=16\n        Dimension for the embeddings if the embedding dimension is not\n        provided in the `cat_embed_cols` parameter and `auto_embed_dim` is\n        set to `False`.\n    with_attention: bool, default = False\n        Boolean indicating whether the preprocessed data will be passed to an\n        attention-based model (more precisely a model where all embeddings\n        must have the same dimensions). If `True`, the param `cat_embed_cols`\n        must just be a list containing just the categorical column names:\n        e.g.\n        _['education', 'relationship', ...]_. This is because they will all be\n         encoded using embeddings of the same dim, which will be specified\n         later when the model is defined. &lt;br/&gt; Param alias:\n         `for_transformer`, `for_matrix_factorization`, `for_mf`\n    with_cls_token: bool, default = False\n        Boolean indicating if a `'[CLS]'` token will be added to the dataset\n        when using attention-based models. The final hidden state\n        corresponding to this token is used as the aggregated representation\n        for classification and regression tasks. If not, the categorical\n        and/or continuous embeddings will be concatenated before being passed\n        to the final MLP (if present).\n    shared_embed: bool, default = False\n        Boolean indicating if the embeddings will be \"shared\" when using\n        attention-based models. The idea behind `shared_embed` is\n        described in the Appendix A in the [TabTransformer paper](https://arxiv.org/abs/2012.06678):\n        _'The goal of having column embedding is to enable the model to\n        distinguish the classes in one column from those in the other\n        columns'_. In other words, the idea is to let the model learn which\n        column is embedded at the time. See: `pytorch_widedeep.models.transformers._layers.SharedEmbeddings`.\n    verbose: int, default = 1\n    scale: bool, default = False\n        :information_source: **note**: this arg will be removed in upcoming\n         releases. Please use `cols_to_scale` instead. &lt;br/&gt; Bool indicating\n         whether or not to scale/standarise continuous cols. It is important\n         to emphasize that all the DL models for tabular data in the library\n         also include the possibility of normalising the input continuous\n         features via a `BatchNorm` or a `LayerNorm`. &lt;br/&gt; Param alias:\n         `scale_cont_cols`.\n    already_standard: List, default = None\n        :information_source: **note**: this arg will be removed in upcoming\n         releases. Please use `cols_to_scale` instead. &lt;br/&gt; List with the\n         name of the continuous cols that do not need to be\n         scaled/standarised.\n\n    Other Parameters\n    ----------------\n    **kwargs: dict\n        `pd.cut` and `StandardScaler` related args\n\n    Attributes\n    ----------\n    cat_cols: List[str]\n        List containing the names of the categorical columns after processing.\n    cols_and_bins: Optional[Dict[str, Union[int, List[float]]]]\n        Dictionary containing the quantization setup after processing if\n        quantization is requested. This is derived from the\n        quantization_setup parameter.\n    quant_args: Dict\n        Dictionary containing arguments passed to pandas.cut() function for quantization.\n    scale_args: Dict\n        Dictionary containing arguments passed to StandardScaler.\n    label_encoder: LabelEncoder\n        Instance of LabelEncoder used to encode categorical variables.\n        See `pytorch_widedeep.utils.dense_utils.LabelEncoder`.\n    cat_embed_input: List\n        List of tuples with the column name, number of individual values for\n        that column and, if `with_attention` is set to `False`, the\n        corresponding embeddings dim, e.g. _[('education', 16, 10),\n        ('relationship', 6, 8), ...]_.\n    standardize_cols: List\n        List of the columns that will be standardized.\n    scaler: StandardScaler\n        An instance of `sklearn.preprocessing.StandardScaler` if standardization\n        is requested.\n    column_idx: Dict\n        Dictionary where keys are column names and values are column indexes.\n        This is necessary to slice tensors.\n    quantizer: Quantizer\n        An instance of `Quantizer` if quantization is requested.\n    is_fitted: bool\n        Boolean indicating if the preprocessor has been fitted.\n\n    Examples\n    --------\n    &gt;&gt;&gt; import pandas as pd\n    &gt;&gt;&gt; import numpy as np\n    &gt;&gt;&gt; from pytorch_widedeep.preprocessing import TabPreprocessor\n    &gt;&gt;&gt; df = pd.DataFrame({'color': ['r', 'b', 'g'], 'size': ['s', 'n', 'l'], 'age': [25, 40, 55]})\n    &gt;&gt;&gt; cat_embed_cols = [('color',5), ('size',5)]\n    &gt;&gt;&gt; cont_cols = ['age']\n    &gt;&gt;&gt; deep_preprocessor = TabPreprocessor(cat_embed_cols=cat_embed_cols, continuous_cols=cont_cols)\n    &gt;&gt;&gt; X_tab = deep_preprocessor.fit_transform(df)\n    &gt;&gt;&gt; deep_preprocessor.cat_embed_cols\n    [('color', 5), ('size', 5)]\n    &gt;&gt;&gt; deep_preprocessor.column_idx\n    {'color': 0, 'size': 1, 'age': 2}\n    &gt;&gt;&gt; cont_df = pd.DataFrame({\"col1\": np.random.rand(10), \"col2\": np.random.rand(10) + 1})\n    &gt;&gt;&gt; cont_cols = [\"col1\", \"col2\"]\n    &gt;&gt;&gt; tab_preprocessor = TabPreprocessor(continuous_cols=cont_cols, quantization_setup=3)\n    &gt;&gt;&gt; ft_cont_df = tab_preprocessor.fit_transform(cont_df)\n    &gt;&gt;&gt; # or...\n    &gt;&gt;&gt; quantization_setup = {'col1': [0., 0.4, 1.], 'col2': [1., 1.4, 2.]}\n    &gt;&gt;&gt; tab_preprocessor2 = TabPreprocessor(continuous_cols=cont_cols, quantization_setup=quantization_setup)\n    &gt;&gt;&gt; ft_cont_df2 = tab_preprocessor2.fit_transform(cont_df)\n    \"\"\"\n\n    @alias(\"with_attention\", [\"for_transformer\", \"for_matrix_factorization\", \"for_mf\"])\n    @alias(\"cat_embed_cols\", [\"embed_cols\"])\n    @alias(\"scale\", [\"scale_cont_cols\"])\n    @alias(\"quantization_setup\", [\"cols_and_bins\"])\n    def __init__(\n        self,\n        cat_embed_cols: Optional[Union[List[str], List[Tuple[str, int]]]] = None,\n        continuous_cols: Optional[List[str]] = None,\n        quantization_setup: Optional[\n            Union[int, Dict[str, Union[int, List[float]]]]\n        ] = None,\n        cols_to_scale: Optional[Union[List[str], str]] = None,\n        auto_embed_dim: bool = True,\n        embedding_rule: Literal[\"google\", \"fastai_old\", \"fastai_new\"] = \"fastai_new\",\n        default_embed_dim: int = 16,\n        with_attention: bool = False,\n        with_cls_token: bool = False,\n        shared_embed: bool = False,\n        verbose: int = 1,\n        *,\n        scale: bool = False,\n        already_standard: Optional[List[str]] = None,\n        **kwargs,\n    ):\n        super(TabPreprocessor, self).__init__()\n\n        self.continuous_cols = continuous_cols\n        self.quantization_setup = quantization_setup\n        self.cols_to_scale = cols_to_scale\n        self.scale = scale\n        self.already_standard = already_standard\n        self.auto_embed_dim = auto_embed_dim\n        self.embedding_rule = embedding_rule\n        self.default_embed_dim = default_embed_dim\n        self.with_attention = with_attention\n        self.with_cls_token = with_cls_token\n        self.shared_embed = shared_embed\n        self.verbose = verbose\n\n        self.quant_args = {\n            k: v for k, v in kwargs.items() if k in pd.cut.__code__.co_varnames\n        }\n        self.scale_args = {\n            k: v for k, v in kwargs.items() if k in StandardScaler().get_params()\n        }\n\n        self._check_inputs(cat_embed_cols)\n\n        if with_cls_token:\n            self.cat_embed_cols = (\n                [\"cls_token\"] + cat_embed_cols  # type: ignore[operator]\n                if cat_embed_cols is not None\n                else [\"cls_token\"]\n            )\n        else:\n            self.cat_embed_cols = cat_embed_cols  # type: ignore[assignment]\n\n        self.is_fitted = False\n\n    def fit(self, df: pd.DataFrame) -&gt; BasePreprocessor:  # noqa: C901\n        \"\"\"Fits the Preprocessor and creates required attributes\n\n        Parameters\n        ----------\n        df: pd.DataFrame\n            Input pandas dataframe\n\n        Returns\n        -------\n        TabPreprocessor\n            `TabPreprocessor` fitted object\n        \"\"\"\n\n        df_adj = self._insert_cls_token(df) if self.with_cls_token else df.copy()\n\n        self.column_idx: Dict[str, int] = {}\n\n        # Categorical embeddings logic\n        if self.cat_embed_cols is not None or self.quantization_setup is not None:\n            self.cat_embed_input: List[Union[Tuple[str, int], Tuple[str, int, int]]] = (\n                []\n            )\n\n        if self.cat_embed_cols is not None:\n            df_cat, cat_embed_dim = self._prepare_categorical(df_adj)\n\n            self.label_encoder = LabelEncoder(\n                columns_to_encode=df_cat.columns.tolist(),\n                shared_embed=self.shared_embed,\n                with_attention=self.with_attention,\n            )\n            self.label_encoder.fit(df_cat)\n\n            for k, v in self.label_encoder.encoding_dict.items():\n                if self.with_attention:\n                    self.cat_embed_input.append((k, len(v)))\n                else:\n                    self.cat_embed_input.append((k, len(v), cat_embed_dim[k]))\n\n            self.column_idx.update({k: v for v, k in enumerate(df_cat.columns)})\n\n        # Continuous columns logic\n        if self.continuous_cols is not None:\n            df_cont, cont_embed_dim = self._prepare_continuous(df_adj)\n\n            # Standardization logic\n            if self.standardize_cols is not None:\n                self.scaler = StandardScaler(**self.scale_args).fit(\n                    df_cont[self.standardize_cols].values\n                )\n            elif self.verbose:\n                warnings.warn(\"Continuous columns will not be normalised\")\n\n            # Quantization logic\n            if self.cols_and_bins is not None:\n                # we do not run 'Quantizer.fit' here since in the wild case\n                # someone wants standardization and quantization for the same\n                # columns, the Quantizer will run on the scaled data\n                self.quantizer = Quantizer(self.cols_and_bins, **self.quant_args)\n\n                if self.with_attention:\n                    for col, n_cat, _ in cont_embed_dim:\n                        self.cat_embed_input.append((col, n_cat))\n                else:\n                    self.cat_embed_input.extend(cont_embed_dim)\n\n            self.column_idx.update(\n                {k: v + len(self.column_idx) for v, k in enumerate(df_cont)}\n            )\n\n        self.is_fitted = True\n\n        return self\n\n    def transform(self, df: pd.DataFrame) -&gt; np.ndarray:  # noqa: C901\n        \"\"\"Returns the processed `dataframe` as a np.ndarray\n\n        Parameters\n        ----------\n        df: pd.DataFrame\n            Input pandas dataframe\n\n        Returns\n        -------\n        np.ndarray\n            transformed input dataframe\n        \"\"\"\n        check_is_fitted(self, condition=self.is_fitted)\n\n        df_adj = self._insert_cls_token(df) if self.with_cls_token else df.copy()\n\n        if self.cat_embed_cols is not None:\n            df_cat = df_adj[self.cat_cols]\n            df_cat = self.label_encoder.transform(df_cat)\n        if self.continuous_cols is not None:\n            df_cont = df_adj[self.continuous_cols]\n            # Standardization logic\n            if self.standardize_cols:\n                df_cont[self.standardize_cols] = self.scaler.transform(\n                    df_cont[self.standardize_cols].values\n                )\n            # Quantization logic\n            if self.cols_and_bins is not None:\n                # Adjustment so I don't have to override the method\n                # in 'ChunkTabPreprocessor'\n                if self.quantizer.is_fitted:\n                    df_cont = self.quantizer.transform(df_cont)\n                else:\n                    df_cont = self.quantizer.fit_transform(df_cont)\n        try:\n            df_deep = pd.concat([df_cat, df_cont], axis=1)\n        except NameError:\n            try:\n                df_deep = df_cat.copy()\n            except NameError:\n                df_deep = df_cont.copy()\n\n        return df_deep.values\n\n    def transform_sample(self, df: pd.DataFrame) -&gt; np.ndarray:\n        return self.transform(df).astype(\"float\")[0]\n\n    def inverse_transform(self, encoded: np.ndarray) -&gt; pd.DataFrame:  # noqa: C901\n        r\"\"\"Takes as input the output from the `transform` method and it will\n        return the original values.\n\n        Parameters\n        ----------\n        encoded: np.ndarray\n            array with the output of the `transform` method\n\n        Returns\n        -------\n        pd.DataFrame\n            Pandas dataframe with the original values\n        \"\"\"\n        decoded = pd.DataFrame(encoded, columns=list(self.column_idx.keys()))\n        # embeddings back to original category\n        if self.cat_embed_cols is not None:\n            decoded = self.label_encoder.inverse_transform(decoded)\n        if self.continuous_cols is not None:\n            # quantized cols to the mid point\n            if self.cols_and_bins is not None:\n                if self.verbose:\n                    print(\n                        \"Note that quantized cols will be turned into the mid point of \"\n                        \"the corresponding bin\"\n                    )\n                for k, v in self.quantizer.inversed_bins.items():\n                    decoded[k] = decoded[k].map(v)\n            # continuous_cols back to non-standarised\n            try:\n                decoded[self.standardize_cols] = self.scaler.inverse_transform(\n                    decoded[self.standardize_cols]\n                )\n            except Exception:  # KeyError:\n                pass\n\n        if \"cls_token\" in decoded.columns:\n            decoded.drop(\"cls_token\", axis=1, inplace=True)\n\n        return decoded\n\n    def fit_transform(self, df: pd.DataFrame) -&gt; np.ndarray:\n        \"\"\"Combines `fit` and `transform`\n\n        Parameters\n        ----------\n        df: pd.DataFrame\n            Input pandas dataframe\n\n        Returns\n        -------\n        np.ndarray\n            transformed input dataframe\n        \"\"\"\n        return self.fit(df).transform(df)\n\n    def _insert_cls_token(self, df: pd.DataFrame) -&gt; pd.DataFrame:\n        df_cls = df.copy()\n        df_cls.insert(loc=0, column=\"cls_token\", value=\"[CLS]\")\n        return df_cls\n\n    def _prepare_categorical(\n        self, df: pd.DataFrame\n    ) -&gt; Tuple[pd.DataFrame, Dict[str, int]]:\n        if isinstance(self.cat_embed_cols[0], tuple):\n            self.cat_cols: List[str] = [emb[0] for emb in self.cat_embed_cols]\n            cat_embed_dim: Dict[str, int] = dict(self.cat_embed_cols)  # type: ignore\n        else:\n            self.cat_cols = self.cat_embed_cols  # type: ignore[assignment]\n            if self.auto_embed_dim:\n                assert isinstance(self.cat_embed_cols[0], str), (\n                    \"If 'auto_embed_dim' is 'True' and 'with_attention' is 'False', \"\n                    \"'cat_embed_cols' must be a list of strings with the columns to \"\n                    \"be encoded as embeddings.\"\n                )\n                n_cats: Dict[str, int] = {col: df[col].nunique() for col in self.cat_embed_cols}  # type: ignore[misc]\n                cat_embed_dim = {\n                    col: embed_sz_rule(n_cat, self.embedding_rule)  # type: ignore[misc]\n                    for col, n_cat in n_cats.items()\n                }\n            else:\n                cat_embed_dim = {\n                    e: self.default_embed_dim for e in self.cat_embed_cols  # type: ignore[misc]\n                }  # type: ignore\n        return df[self.cat_cols], cat_embed_dim\n\n    def _prepare_continuous(\n        self, df: pd.DataFrame\n    ) -&gt; Tuple[pd.DataFrame, Optional[List[Tuple[str, int, int]]]]:\n        # Standardization logic\n        if self.cols_to_scale is not None:\n            self.standardize_cols = (\n                self.cols_to_scale\n                if self.cols_to_scale != \"all\"\n                else self.continuous_cols\n            )\n        elif self.scale:\n            if self.already_standard is not None:\n                self.standardize_cols = [\n                    c for c in self.continuous_cols if c not in self.already_standard\n                ]\n            else:\n                self.standardize_cols = self.continuous_cols\n        else:\n            self.standardize_cols = None\n\n        # Quantization logic\n        if self.quantization_setup is not None:\n            # the quantized columns are then treated as categorical\n            quant_cont_embed_input: Optional[List[Tuple[str, int, int]]] = []\n            if isinstance(self.quantization_setup, int):\n                self.cols_and_bins: Optional[Dict[str, Union[int, List[float]]]] = {}\n                for col in self.continuous_cols:\n                    self.cols_and_bins[col] = self.quantization_setup\n                    quant_cont_embed_input.append(\n                        (\n                            col,\n                            self.quantization_setup,\n                            (\n                                embed_sz_rule(\n                                    self.quantization_setup + 1, self.embedding_rule  # type: ignore[arg-type]\n                                )\n                                if self.auto_embed_dim\n                                else self.default_embed_dim\n                            ),\n                        )\n                    )\n            else:\n                for col, val in self.quantization_setup.items():\n                    if isinstance(val, int):\n                        quant_cont_embed_input.append(\n                            (\n                                col,\n                                val,\n                                (\n                                    embed_sz_rule(\n                                        val + 1, self.embedding_rule  # type: ignore[arg-type]\n                                    )\n                                    if self.auto_embed_dim\n                                    else self.default_embed_dim\n                                ),\n                            )\n                        )\n                    else:\n                        quant_cont_embed_input.append(\n                            (\n                                col,\n                                len(val) - 1,\n                                (\n                                    embed_sz_rule(len(val), self.embedding_rule)  # type: ignore[arg-type]\n                                    if self.auto_embed_dim\n                                    else self.default_embed_dim\n                                ),\n                            )\n                        )\n\n                self.cols_and_bins = self.quantization_setup.copy()\n        else:\n            self.cols_and_bins = None\n            quant_cont_embed_input = None\n\n        return df[self.continuous_cols], quant_cont_embed_input\n\n    def _check_inputs(self, cat_embed_cols):  # noqa: C901\n        if self.scale or self.already_standard is not None:\n            warnings.warn(\n                \"'scale' and 'already_standard' will be deprecated in the next release. \"\n                \"Please use 'cols_to_scale' instead\",\n                DeprecationWarning,\n                stacklevel=2,\n            )\n\n        if self.scale:\n            if self.already_standard is not None:\n                standardize_cols = [\n                    c for c in self.continuous_cols if c not in self.already_standard\n                ]\n            else:\n                standardize_cols = self.continuous_cols\n        elif self.cols_to_scale is not None:\n            standardize_cols = self.cols_to_scale\n        else:\n            standardize_cols = None\n\n        if standardize_cols is not None:\n            if isinstance(self.quantization_setup, int):\n                cols_to_quantize_and_standardize = [\n                    c for c in standardize_cols if c in self.continuous_cols\n                ]\n            elif isinstance(self.quantization_setup, dict):\n                cols_to_quantize_and_standardize = [\n                    c for c in standardize_cols if c in self.quantization_setup\n                ]\n            else:\n                cols_to_quantize_and_standardize = None\n            if cols_to_quantize_and_standardize is not None:\n                warnings.warn(\n                    f\"the following columns: {cols_to_quantize_and_standardize} will be first scaled\"\n                    \" using a StandardScaler and then quantized. Make sure this is what you really want\"\n                )\n\n        if self.with_cls_token and not self.with_attention:\n            warnings.warn(\n                \"If 'with_cls_token' is set to 'True', 'with_attention' will be automatically \"\n                \"to 'True' if is 'False'\",\n                UserWarning,\n            )\n            self.with_attention = True\n\n        if (cat_embed_cols is None) and (self.continuous_cols is None):\n            raise ValueError(\n                \"'cat_embed_cols' and 'continuous_cols' are 'None'. Please, define at least one of the two.\"\n            )\n\n        if (\n            cat_embed_cols is not None\n            and self.continuous_cols is not None\n            and len(np.intersect1d(cat_embed_cols, self.continuous_cols)) &gt; 0\n        ):\n            overlapping_cols = list(\n                np.intersect1d(cat_embed_cols, self.continuous_cols)\n            )\n            raise ValueError(\n                \"Currently passing columns as both categorical and continuum is not supported.\"\n                \" Please, choose one or the other for the following columns: {}\".format(\n                    \", \".join(overlapping_cols)\n                )\n            )\n        transformer_error_message = (\n            \"If with_attention is 'True' cat_embed_cols must be a list \"\n            \" of strings with the columns to be encoded as embeddings.\"\n        )\n        if (\n            self.with_attention\n            and cat_embed_cols is not None\n            and isinstance(cat_embed_cols[0], tuple)\n        ):\n            raise ValueError(transformer_error_message)\n\n    def __repr__(self) -&gt; str:  # noqa: C901\n        list_of_params: List[str] = []\n        if self.cat_embed_cols is not None:\n            list_of_params.append(\"cat_embed_cols={cat_embed_cols}\")\n        if self.continuous_cols is not None:\n            list_of_params.append(\"continuous_cols={continuous_cols}\")\n        if self.quantization_setup is not None:\n            list_of_params.append(\"quantization_setup={quantization_setup}\")\n        if self.cols_to_scale is not None:\n            list_of_params.append(\"cols_to_scale={cols_to_scale}\")\n        if not self.auto_embed_dim:\n            list_of_params.append(\"auto_embed_dim={auto_embed_dim}\")\n        if self.embedding_rule != \"fastai_new\":\n            list_of_params.append(\"embedding_rule='{embedding_rule}'\")\n        if self.default_embed_dim != 16:\n            list_of_params.append(\"default_embed_dim={default_embed_dim}\")\n        if self.with_attention:\n            list_of_params.append(\"with_attention={with_attention}\")\n        if self.with_cls_token:\n            list_of_params.append(\"with_cls_token={with_cls_token}\")\n        if self.shared_embed:\n            list_of_params.append(\"shared_embed={shared_embed}\")\n        if self.verbose != 1:\n            list_of_params.append(\"verbose={verbose}\")\n        if self.scale:\n            list_of_params.append(\"scale={scale}\")\n        if self.already_standard is not None:\n            list_of_params.append(\"already_standard={already_standard}\")\n        if len(self.quant_args) &gt; 0:\n            list_of_params.append(\n                \", \".join([f\"{k}\" + \"=\" + f\"{v}\" for k, v in self.quant_args.items()])\n            )\n        if len(self.scale_args) &gt; 0:\n            list_of_params.append(\n                \", \".join([f\"{k}\" + \"=\" + f\"{v}\" for k, v in self.scale_args.items()])\n            )\n        all_params = \", \".join(list_of_params)\n        return f\"TabPreprocessor({all_params.format(**self.__dict__)})\"\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor.fit","title":"fit","text":"<pre><code>fit(df)\n</code></pre> <p>Fits the Preprocessor and creates required attributes</p> <p>Parameters:</p> Name Type Description Default <code>df</code> <code>DataFrame</code> <p>Input pandas dataframe</p> required <p>Returns:</p> Type Description <code>TabPreprocessor</code> <p><code>TabPreprocessor</code> fitted object</p> Source code in <code>pytorch_widedeep/preprocessing/tab_preprocessor.py</code> <pre><code>def fit(self, df: pd.DataFrame) -&gt; BasePreprocessor:  # noqa: C901\n    \"\"\"Fits the Preprocessor and creates required attributes\n\n    Parameters\n    ----------\n    df: pd.DataFrame\n        Input pandas dataframe\n\n    Returns\n    -------\n    TabPreprocessor\n        `TabPreprocessor` fitted object\n    \"\"\"\n\n    df_adj = self._insert_cls_token(df) if self.with_cls_token else df.copy()\n\n    self.column_idx: Dict[str, int] = {}\n\n    # Categorical embeddings logic\n    if self.cat_embed_cols is not None or self.quantization_setup is not None:\n        self.cat_embed_input: List[Union[Tuple[str, int], Tuple[str, int, int]]] = (\n            []\n        )\n\n    if self.cat_embed_cols is not None:\n        df_cat, cat_embed_dim = self._prepare_categorical(df_adj)\n\n        self.label_encoder = LabelEncoder(\n            columns_to_encode=df_cat.columns.tolist(),\n            shared_embed=self.shared_embed,\n            with_attention=self.with_attention,\n        )\n        self.label_encoder.fit(df_cat)\n\n        for k, v in self.label_encoder.encoding_dict.items():\n            if self.with_attention:\n                self.cat_embed_input.append((k, len(v)))\n            else:\n                self.cat_embed_input.append((k, len(v), cat_embed_dim[k]))\n\n        self.column_idx.update({k: v for v, k in enumerate(df_cat.columns)})\n\n    # Continuous columns logic\n    if self.continuous_cols is not None:\n        df_cont, cont_embed_dim = self._prepare_continuous(df_adj)\n\n        # Standardization logic\n        if self.standardize_cols is not None:\n            self.scaler = StandardScaler(**self.scale_args).fit(\n                df_cont[self.standardize_cols].values\n            )\n        elif self.verbose:\n            warnings.warn(\"Continuous columns will not be normalised\")\n\n        # Quantization logic\n        if self.cols_and_bins is not None:\n            # we do not run 'Quantizer.fit' here since in the wild case\n            # someone wants standardization and quantization for the same\n            # columns, the Quantizer will run on the scaled data\n            self.quantizer = Quantizer(self.cols_and_bins, **self.quant_args)\n\n            if self.with_attention:\n                for col, n_cat, _ in cont_embed_dim:\n                    self.cat_embed_input.append((col, n_cat))\n            else:\n                self.cat_embed_input.extend(cont_embed_dim)\n\n        self.column_idx.update(\n            {k: v + len(self.column_idx) for v, k in enumerate(df_cont)}\n        )\n\n    self.is_fitted = True\n\n    return self\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor.transform","title":"transform","text":"<pre><code>transform(df)\n</code></pre> <p>Returns the processed <code>dataframe</code> as a np.ndarray</p> <p>Parameters:</p> Name Type Description Default <code>df</code> <code>DataFrame</code> <p>Input pandas dataframe</p> required <p>Returns:</p> Type Description <code>ndarray</code> <p>transformed input dataframe</p> Source code in <code>pytorch_widedeep/preprocessing/tab_preprocessor.py</code> <pre><code>def transform(self, df: pd.DataFrame) -&gt; np.ndarray:  # noqa: C901\n    \"\"\"Returns the processed `dataframe` as a np.ndarray\n\n    Parameters\n    ----------\n    df: pd.DataFrame\n        Input pandas dataframe\n\n    Returns\n    -------\n    np.ndarray\n        transformed input dataframe\n    \"\"\"\n    check_is_fitted(self, condition=self.is_fitted)\n\n    df_adj = self._insert_cls_token(df) if self.with_cls_token else df.copy()\n\n    if self.cat_embed_cols is not None:\n        df_cat = df_adj[self.cat_cols]\n        df_cat = self.label_encoder.transform(df_cat)\n    if self.continuous_cols is not None:\n        df_cont = df_adj[self.continuous_cols]\n        # Standardization logic\n        if self.standardize_cols:\n            df_cont[self.standardize_cols] = self.scaler.transform(\n                df_cont[self.standardize_cols].values\n            )\n        # Quantization logic\n        if self.cols_and_bins is not None:\n            # Adjustment so I don't have to override the method\n            # in 'ChunkTabPreprocessor'\n            if self.quantizer.is_fitted:\n                df_cont = self.quantizer.transform(df_cont)\n            else:\n                df_cont = self.quantizer.fit_transform(df_cont)\n    try:\n        df_deep = pd.concat([df_cat, df_cont], axis=1)\n    except NameError:\n        try:\n            df_deep = df_cat.copy()\n        except NameError:\n            df_deep = df_cont.copy()\n\n    return df_deep.values\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor.inverse_transform","title":"inverse_transform","text":"<pre><code>inverse_transform(encoded)\n</code></pre> <p>Takes as input the output from the <code>transform</code> method and it will return the original values.</p> <p>Parameters:</p> Name Type Description Default <code>encoded</code> <code>ndarray</code> <p>array with the output of the <code>transform</code> method</p> required <p>Returns:</p> Type Description <code>DataFrame</code> <p>Pandas dataframe with the original values</p> Source code in <code>pytorch_widedeep/preprocessing/tab_preprocessor.py</code> <pre><code>def inverse_transform(self, encoded: np.ndarray) -&gt; pd.DataFrame:  # noqa: C901\n    r\"\"\"Takes as input the output from the `transform` method and it will\n    return the original values.\n\n    Parameters\n    ----------\n    encoded: np.ndarray\n        array with the output of the `transform` method\n\n    Returns\n    -------\n    pd.DataFrame\n        Pandas dataframe with the original values\n    \"\"\"\n    decoded = pd.DataFrame(encoded, columns=list(self.column_idx.keys()))\n    # embeddings back to original category\n    if self.cat_embed_cols is not None:\n        decoded = self.label_encoder.inverse_transform(decoded)\n    if self.continuous_cols is not None:\n        # quantized cols to the mid point\n        if self.cols_and_bins is not None:\n            if self.verbose:\n                print(\n                    \"Note that quantized cols will be turned into the mid point of \"\n                    \"the corresponding bin\"\n                )\n            for k, v in self.quantizer.inversed_bins.items():\n                decoded[k] = decoded[k].map(v)\n        # continuous_cols back to non-standarised\n        try:\n            decoded[self.standardize_cols] = self.scaler.inverse_transform(\n                decoded[self.standardize_cols]\n            )\n        except Exception:  # KeyError:\n            pass\n\n    if \"cls_token\" in decoded.columns:\n        decoded.drop(\"cls_token\", axis=1, inplace=True)\n\n    return decoded\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor.fit_transform","title":"fit_transform","text":"<pre><code>fit_transform(df)\n</code></pre> <p>Combines <code>fit</code> and <code>transform</code></p> <p>Parameters:</p> Name Type Description Default <code>df</code> <code>DataFrame</code> <p>Input pandas dataframe</p> required <p>Returns:</p> Type Description <code>ndarray</code> <p>transformed input dataframe</p> Source code in <code>pytorch_widedeep/preprocessing/tab_preprocessor.py</code> <pre><code>def fit_transform(self, df: pd.DataFrame) -&gt; np.ndarray:\n    \"\"\"Combines `fit` and `transform`\n\n    Parameters\n    ----------\n    df: pd.DataFrame\n        Input pandas dataframe\n\n    Returns\n    -------\n    np.ndarray\n        transformed input dataframe\n    \"\"\"\n    return self.fit(df).transform(df)\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.tab_preprocessor.Quantizer","title":"Quantizer","text":"<p>Helper class to perform the quantization of continuous columns. It is included in this docs for completion, since depending on the value of the parameter <code>'quantization_setup'</code> of the <code>TabPreprocessor</code> class, that class might have an attribute of type <code>Quantizer</code>. However, this class is designed to always run internally within the <code>TabPreprocessor</code> class.</p> <p>Parameters:</p> Name Type Description Default <code>quantization_setup</code> <code>Dict[str, Union[int, List[float]]]</code> <p>Dictionary where the keys are the column names to quantize and the values are the either integers indicating the number of bins or a list of scalars indicating the bin edges.</p> required Source code in <code>pytorch_widedeep/preprocessing/tab_preprocessor.py</code> <pre><code>class Quantizer:\n    \"\"\"Helper class to perform the quantization of continuous columns. It is\n    included in this docs for completion, since depending on the value of the\n    parameter `'quantization_setup'` of the `TabPreprocessor` class, that\n    class might have an attribute of type `Quantizer`. However, this class is\n    designed to always run internally within the `TabPreprocessor` class.\n\n    Parameters\n    ----------\n    quantization_setup: Dict, default = None\n        Dictionary where the keys are the column names to quantize and the\n        values are the either integers indicating the number of bins or a\n        list of scalars indicating the bin edges.\n    \"\"\"\n\n    def __init__(\n        self,\n        quantization_setup: Dict[str, Union[int, List[float]]],\n        **kwargs,\n    ):\n        self.quantization_setup = quantization_setup\n        self.quant_args = kwargs\n\n        self.is_fitted = False\n\n    def fit(self, df: pd.DataFrame) -&gt; \"Quantizer\":\n        self.bins: Dict[str, List[float]] = {}\n        for col, bins in self.quantization_setup.items():\n            _, self.bins[col] = pd.cut(\n                df[col], bins, retbins=True, labels=False, **self.quant_args\n            )\n\n        self.inversed_bins: Dict[str, Dict[int, float]] = {}\n        for col, bins in self.bins.items():\n            self.inversed_bins[col] = {\n                k: v\n                for k, v in list(\n                    zip(\n                        range(1, len(bins) + 1),\n                        [(a + b) / 2.0 for a, b in zip(bins, bins[1:])],\n                    )\n                )\n            }\n            # 0\n            self.inversed_bins[col][0] = np.nan\n\n        self.is_fitted = True\n\n        return self\n\n    def transform(self, df: pd.DataFrame) -&gt; pd.DataFrame:\n        check_is_fitted(self, condition=self.is_fitted)\n\n        dfc = df.copy()\n        for col, bins in self.bins.items():\n            dfc[col] = pd.cut(dfc[col], bins, labels=False, **self.quant_args)\n            # 0 will be left for numbers outside the bins, i.e. smaller than\n            # the smaller boundary or larger than the largest boundary\n            dfc[col] = dfc[col] + 1\n            dfc[col] = dfc[col].fillna(0)\n            dfc[col] = dfc[col].astype(int)\n\n        return dfc\n\n    def fit_transform(self, df: pd.DataFrame) -&gt; pd.DataFrame:\n        return self.fit(df).transform(df)\n\n    def __repr__(self) -&gt; str:\n        return f\"Quantizer(quantization_setup={self.quantization_setup})\"\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor","title":"TextPreprocessor","text":"<p>               Bases: <code>BasePreprocessor</code></p> <p>Preprocessor to prepare the <code>deeptext</code> input dataset</p> <p>Parameters:</p> Name Type Description Default <code>text_col</code> <code>str</code> <p>column in the input dataframe containing the texts</p> required <code>max_vocab</code> <code>int</code> <p>Maximum number of tokens in the vocabulary</p> <code>30000</code> <code>min_freq</code> <code>int</code> <p>Minimum frequency for a token to be part of the vocabulary</p> <code>5</code> <code>maxlen</code> <code>int</code> <p>Maximum length of the tokenized sequences</p> <code>80</code> <code>pad_first</code> <code>bool</code> <p>Indicates whether the padding index will be added at the beginning or the end of the sequences</p> <code>True</code> <code>pad_idx</code> <code>int</code> <p>padding index. Fastai's Tokenizer leaves 0 for the 'unknown' token.</p> <code>1</code> <code>already_processed</code> <code>Optional[bool]</code> <p>Boolean indicating if the sequence of elements is already processed or prepared. If this is the case, this Preprocessor will simply tokenize and pad the sequence. </p> <pre><code>Param aliases: `not_text`. &lt;br/&gt;\n</code></pre> <p>This parameter is thought for those cases where the input sequences are already fully processed or are directly not text (e.g. IDs)</p> <code>False</code> <code>word_vectors_path</code> <code>Optional[str]</code> <p>Path to the pretrained word vectors</p> <code>None</code> <code>n_cpus</code> <code>Optional[int]</code> <p>number of CPUs to used during the tokenization process</p> <code>None</code> <code>verbose</code> <code>int</code> <p>Enable verbose output.</p> <code>1</code> <p>Attributes:</p> Name Type Description <code>vocab</code> <code>Vocab</code> <p>an instance of <code>pytorch_widedeep.utils.fastai_transforms.Vocab</code></p> <code>embedding_matrix</code> <code>ndarray</code> <p>Array with the pretrained embeddings</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import pandas as pd\n&gt;&gt;&gt; from pytorch_widedeep.preprocessing import TextPreprocessor\n&gt;&gt;&gt; df_train = pd.DataFrame({'text_column': [\"life is like a box of chocolates\",\n... \"You never know what you're gonna get\"]})\n&gt;&gt;&gt; text_preprocessor = TextPreprocessor(text_col='text_column', max_vocab=25, min_freq=1, maxlen=10)\n&gt;&gt;&gt; text_preprocessor.fit_transform(df_train)\nThe vocabulary contains 24 tokens\narray([[ 1,  1,  1,  1, 10, 11, 12, 13, 14, 15],\n       [ 5,  9, 16, 17, 18,  9, 19, 20, 21, 22]], dtype=int32)\n&gt;&gt;&gt; df_te = pd.DataFrame({'text_column': ['you never know what is in the box']})\n&gt;&gt;&gt; text_preprocessor.transform(df_te)\narray([[ 1,  1,  9, 16, 17, 18, 11,  0,  0, 13]], dtype=int32)\n</code></pre> Source code in <code>pytorch_widedeep/preprocessing/text_preprocessor.py</code> <pre><code>class TextPreprocessor(BasePreprocessor):\n    r\"\"\"Preprocessor to prepare the ``deeptext`` input dataset\n\n    Parameters\n    ----------\n    text_col: str\n        column in the input dataframe containing the texts\n    max_vocab: int, default=30000\n        Maximum number of tokens in the vocabulary\n    min_freq: int, default=5\n        Minimum frequency for a token to be part of the vocabulary\n    maxlen: int, default=80\n        Maximum length of the tokenized sequences\n    pad_first: bool,  default = True\n        Indicates whether the padding index will be added at the beginning or the\n        end of the sequences\n    pad_idx: int, default = 1\n        padding index. Fastai's Tokenizer leaves 0 for the 'unknown' token.\n    already_processed: bool, Optional, default = False\n        Boolean indicating if the sequence of elements is already processed or\n        prepared. If this is the case, this Preprocessor will simply tokenize\n        and pad the sequence. &lt;br/&gt;\n\n            Param aliases: `not_text`. &lt;br/&gt;\n\n        This parameter is thought for those cases where the input sequences\n        are already fully processed or are directly not text (e.g. IDs)\n    word_vectors_path: str, Optional\n        Path to the pretrained word vectors\n    n_cpus: int, Optional, default = None\n        number of CPUs to used during the tokenization process\n    verbose: int, default 1\n        Enable verbose output.\n\n    Attributes\n    ----------\n    vocab: Vocab\n        an instance of `pytorch_widedeep.utils.fastai_transforms.Vocab`\n    embedding_matrix: np.ndarray\n        Array with the pretrained embeddings\n\n    Examples\n    ---------\n    &gt;&gt;&gt; import pandas as pd\n    &gt;&gt;&gt; from pytorch_widedeep.preprocessing import TextPreprocessor\n    &gt;&gt;&gt; df_train = pd.DataFrame({'text_column': [\"life is like a box of chocolates\",\n    ... \"You never know what you're gonna get\"]})\n    &gt;&gt;&gt; text_preprocessor = TextPreprocessor(text_col='text_column', max_vocab=25, min_freq=1, maxlen=10)\n    &gt;&gt;&gt; text_preprocessor.fit_transform(df_train)\n    The vocabulary contains 24 tokens\n    array([[ 1,  1,  1,  1, 10, 11, 12, 13, 14, 15],\n           [ 5,  9, 16, 17, 18,  9, 19, 20, 21, 22]], dtype=int32)\n    &gt;&gt;&gt; df_te = pd.DataFrame({'text_column': ['you never know what is in the box']})\n    &gt;&gt;&gt; text_preprocessor.transform(df_te)\n    array([[ 1,  1,  9, 16, 17, 18, 11,  0,  0, 13]], dtype=int32)\n    \"\"\"\n\n    @alias(\"already_processed\", [\"not_text\"])\n    def __init__(\n        self,\n        text_col: str,\n        max_vocab: int = 30000,\n        min_freq: int = 5,\n        maxlen: int = 80,\n        pad_first: bool = True,\n        pad_idx: int = 1,\n        already_processed: Optional[bool] = False,\n        word_vectors_path: Optional[str] = None,\n        n_cpus: Optional[int] = None,\n        verbose: int = 1,\n    ):\n        super(TextPreprocessor, self).__init__()\n\n        self.text_col = text_col\n        self.max_vocab = max_vocab\n        self.min_freq = min_freq\n        self.maxlen = maxlen\n        self.pad_first = pad_first\n        self.pad_idx = pad_idx\n        self.already_processed = already_processed\n        self.word_vectors_path = word_vectors_path\n        self.verbose = verbose\n        self.n_cpus = n_cpus if n_cpus is not None else os.cpu_count()\n\n        self.is_fitted = False\n\n    def fit(self, df: pd.DataFrame) -&gt; BasePreprocessor:\n        \"\"\"Builds the vocabulary\n\n        Parameters\n        ----------\n        df: pd.DataFrame\n            Input pandas dataframe\n\n        Returns\n        -------\n        TextPreprocessor\n            `TextPreprocessor` fitted object\n        \"\"\"\n        texts = self._read_texts(df)\n\n        tokens = get_texts(texts, self.already_processed, self.n_cpus)\n\n        self.vocab: TVocab = Vocab(\n            max_vocab=self.max_vocab,\n            min_freq=self.min_freq,\n            pad_idx=self.pad_idx,\n        ).fit(\n            tokens,\n        )\n\n        if self.verbose:\n            print(\"The vocabulary contains {} tokens\".format(len(self.vocab.stoi)))\n        if self.word_vectors_path is not None:\n            self.embedding_matrix = build_embeddings_matrix(\n                self.vocab, self.word_vectors_path, self.min_freq\n            )\n\n        self.is_fitted = True\n\n        return self\n\n    def transform(self, df: pd.DataFrame) -&gt; np.ndarray:\n        \"\"\"Returns the padded, _'numericalised'_ sequences\n\n        Parameters\n        ----------\n        df: pd.DataFrame\n            Input pandas dataframe\n\n        Returns\n        -------\n        np.ndarray\n            Padded, _'numericalised'_ sequences\n        \"\"\"\n        check_is_fitted(self, attributes=[\"vocab\"])\n        texts = self._read_texts(df)\n        tokens = get_texts(texts, self.already_processed, self.n_cpus)\n        return self._pad_sequences(tokens)\n\n    def transform_sample(self, text: str) -&gt; np.ndarray:\n        \"\"\"Returns the padded, _'numericalised'_ sequence\n\n        Parameters\n        ----------\n        text: str\n            text to be tokenized and padded\n\n        Returns\n        -------\n        np.ndarray\n            Padded, _'numericalised'_ sequence\n        \"\"\"\n        check_is_fitted(self, attributes=[\"vocab\"])\n        tokens = get_texts([text], self.already_processed, self.n_cpus)\n        return self._pad_sequences(tokens)[0]\n\n    def fit_transform(self, df: pd.DataFrame) -&gt; np.ndarray:\n        \"\"\"Combines `fit` and `transform`\n\n        Parameters\n        ----------\n        df: pd.DataFrame\n            Input pandas dataframe\n\n        Returns\n        -------\n        np.ndarray\n            Padded, _'numericalised'_ sequences\n        \"\"\"\n        return self.fit(df).transform(df)\n\n    def inverse_transform(self, padded_seq: np.ndarray) -&gt; pd.DataFrame:\n        \"\"\"Returns the original text plus the added 'special' tokens\n\n        Parameters\n        ----------\n        padded_seq: np.ndarray\n            array with the output of the `transform` method\n\n        Returns\n        -------\n        pd.DataFrame\n            Pandas dataframe with the original text plus the added 'special' tokens\n        \"\"\"\n        texts = [self.vocab.inverse_transform(num) for num in padded_seq]\n        return pd.DataFrame({self.text_col: texts})\n\n    def _pad_sequences(self, tokens: List[List[str]]) -&gt; np.ndarray:\n        sequences = [self.vocab.transform(t) for t in tokens]\n        padded_seq = np.array(\n            [\n                pad_sequences(\n                    s,\n                    maxlen=self.maxlen,\n                    pad_first=self.pad_first,\n                    pad_idx=self.pad_idx,\n                )\n                for s in sequences\n            ]\n        )\n        return padded_seq\n\n    def _read_texts(\n        self, df: pd.DataFrame, root_dir: Optional[str] = None\n    ) -&gt; List[str]:\n        if root_dir is not None:\n            if not os.path.exists(root_dir):\n                raise ValueError(\n                    \"root_dir does not exist. Please create it before fitting the preprocessor\"\n                )\n            texts_fnames = df[self.text_col].tolist()\n            texts: List[str] = []\n            for texts_fname in texts_fnames:\n                with open(os.path.join(root_dir, texts_fname), \"r\") as f:\n                    texts.append(f.read().replace(\"\\n\", \"\"))\n        else:\n            texts = df[self.text_col].tolist()\n\n        return texts\n\n    def _load_vocab(self, vocab: TVocab) -&gt; None:\n        self.vocab = vocab\n\n    def __repr__(self) -&gt; str:\n        list_of_params: List[str] = [\"text_col={text_col}\"]\n        list_of_params.append(\"max_vocab={max_vocab}\")\n        list_of_params.append(\"min_freq={min_freq}\")\n        list_of_params.append(\"maxlen={maxlen}\")\n        list_of_params.append(\"pad_first={pad_first}\")\n        list_of_params.append(\"pad_idx={pad_idx}\")\n        list_of_params.append(\"already_processed={already_processed}\")\n        if self.word_vectors_path is not None:\n            list_of_params.append(\"word_vectors_path={word_vectors_path}\")\n        if self.n_cpus is not None:\n            list_of_params.append(\"n_cpus={n_cpus}\")\n        if self.verbose is not None:\n            list_of_params.append(\"verbose={verbose}\")\n        all_params = \", \".join(list_of_params)\n        return f\"TextPreprocessor({all_params.format(**self.__dict__)})\"\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor.fit","title":"fit","text":"<pre><code>fit(df)\n</code></pre> <p>Builds the vocabulary</p> <p>Parameters:</p> Name Type Description Default <code>df</code> <code>DataFrame</code> <p>Input pandas dataframe</p> required <p>Returns:</p> Type Description <code>TextPreprocessor</code> <p><code>TextPreprocessor</code> fitted object</p> Source code in <code>pytorch_widedeep/preprocessing/text_preprocessor.py</code> <pre><code>def fit(self, df: pd.DataFrame) -&gt; BasePreprocessor:\n    \"\"\"Builds the vocabulary\n\n    Parameters\n    ----------\n    df: pd.DataFrame\n        Input pandas dataframe\n\n    Returns\n    -------\n    TextPreprocessor\n        `TextPreprocessor` fitted object\n    \"\"\"\n    texts = self._read_texts(df)\n\n    tokens = get_texts(texts, self.already_processed, self.n_cpus)\n\n    self.vocab: TVocab = Vocab(\n        max_vocab=self.max_vocab,\n        min_freq=self.min_freq,\n        pad_idx=self.pad_idx,\n    ).fit(\n        tokens,\n    )\n\n    if self.verbose:\n        print(\"The vocabulary contains {} tokens\".format(len(self.vocab.stoi)))\n    if self.word_vectors_path is not None:\n        self.embedding_matrix = build_embeddings_matrix(\n            self.vocab, self.word_vectors_path, self.min_freq\n        )\n\n    self.is_fitted = True\n\n    return self\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor.transform","title":"transform","text":"<pre><code>transform(df)\n</code></pre> <p>Returns the padded, 'numericalised' sequences</p> <p>Parameters:</p> Name Type Description Default <code>df</code> <code>DataFrame</code> <p>Input pandas dataframe</p> required <p>Returns:</p> Type Description <code>ndarray</code> <p>Padded, 'numericalised' sequences</p> Source code in <code>pytorch_widedeep/preprocessing/text_preprocessor.py</code> <pre><code>def transform(self, df: pd.DataFrame) -&gt; np.ndarray:\n    \"\"\"Returns the padded, _'numericalised'_ sequences\n\n    Parameters\n    ----------\n    df: pd.DataFrame\n        Input pandas dataframe\n\n    Returns\n    -------\n    np.ndarray\n        Padded, _'numericalised'_ sequences\n    \"\"\"\n    check_is_fitted(self, attributes=[\"vocab\"])\n    texts = self._read_texts(df)\n    tokens = get_texts(texts, self.already_processed, self.n_cpus)\n    return self._pad_sequences(tokens)\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor.transform_sample","title":"transform_sample","text":"<pre><code>transform_sample(text)\n</code></pre> <p>Returns the padded, 'numericalised' sequence</p> <p>Parameters:</p> Name Type Description Default <code>text</code> <code>str</code> <p>text to be tokenized and padded</p> required <p>Returns:</p> Type Description <code>ndarray</code> <p>Padded, 'numericalised' sequence</p> Source code in <code>pytorch_widedeep/preprocessing/text_preprocessor.py</code> <pre><code>def transform_sample(self, text: str) -&gt; np.ndarray:\n    \"\"\"Returns the padded, _'numericalised'_ sequence\n\n    Parameters\n    ----------\n    text: str\n        text to be tokenized and padded\n\n    Returns\n    -------\n    np.ndarray\n        Padded, _'numericalised'_ sequence\n    \"\"\"\n    check_is_fitted(self, attributes=[\"vocab\"])\n    tokens = get_texts([text], self.already_processed, self.n_cpus)\n    return self._pad_sequences(tokens)[0]\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor.fit_transform","title":"fit_transform","text":"<pre><code>fit_transform(df)\n</code></pre> <p>Combines <code>fit</code> and <code>transform</code></p> <p>Parameters:</p> Name Type Description Default <code>df</code> <code>DataFrame</code> <p>Input pandas dataframe</p> required <p>Returns:</p> Type Description <code>ndarray</code> <p>Padded, 'numericalised' sequences</p> Source code in <code>pytorch_widedeep/preprocessing/text_preprocessor.py</code> <pre><code>def fit_transform(self, df: pd.DataFrame) -&gt; np.ndarray:\n    \"\"\"Combines `fit` and `transform`\n\n    Parameters\n    ----------\n    df: pd.DataFrame\n        Input pandas dataframe\n\n    Returns\n    -------\n    np.ndarray\n        Padded, _'numericalised'_ sequences\n    \"\"\"\n    return self.fit(df).transform(df)\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor.inverse_transform","title":"inverse_transform","text":"<pre><code>inverse_transform(padded_seq)\n</code></pre> <p>Returns the original text plus the added 'special' tokens</p> <p>Parameters:</p> Name Type Description Default <code>padded_seq</code> <code>ndarray</code> <p>array with the output of the <code>transform</code> method</p> required <p>Returns:</p> Type Description <code>DataFrame</code> <p>Pandas dataframe with the original text plus the added 'special' tokens</p> Source code in <code>pytorch_widedeep/preprocessing/text_preprocessor.py</code> <pre><code>def inverse_transform(self, padded_seq: np.ndarray) -&gt; pd.DataFrame:\n    \"\"\"Returns the original text plus the added 'special' tokens\n\n    Parameters\n    ----------\n    padded_seq: np.ndarray\n        array with the output of the `transform` method\n\n    Returns\n    -------\n    pd.DataFrame\n        Pandas dataframe with the original text plus the added 'special' tokens\n    \"\"\"\n    texts = [self.vocab.inverse_transform(num) for num in padded_seq]\n    return pd.DataFrame({self.text_col: texts})\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor","title":"HFPreprocessor","text":"<p>               Bases: <code>BasePreprocessor</code></p> <p>Text processor to prepare the <code>deeptext</code> input dataset that is a wrapper around HuggingFace's tokenizers.</p> <p>Following the main phylosophy of the <code>pytorch-widedeep</code> library, this class is designed to be as flexible as possible. Therefore, it is coded so that the user can use it as one would use any HuggingFace tokenizers, or following the API call 'protocol' of the rest of the library.</p> <p>Parameters:</p> Name Type Description Default <code>model_name</code> <code>str</code> <p>The model name from the transformers library e.g. 'bert-base-uncased'. Currently supported models are those from the families: BERT, RoBERTa, DistilBERT, ALBERT and ELECTRA.</p> required <code>use_fast_tokenizer</code> <code>bool</code> <p>Whether to use the fast tokenizer from HuggingFace or not</p> <code>False</code> <code>text_col</code> <code>Optional[str]</code> <p>The column in the input dataframe containing the text data. If this tokenizer is used via the <code>fit</code> and <code>transform</code> methods, this argument is mandatory. If the tokenizer is used via the <code>encode</code> method, this argument is not needed since the input text is passed directly to the <code>encode</code> method.</p> <code>None</code> <code>num_workers</code> <code>Optional[int]</code> <p>Number of workers to use when preprocessing the text data. If not None, and <code>use_fast_tokenizer</code> is False, the text data will be preprocessed in parallel using the number of workers specified. If <code>use_fast_tokenizer</code> is True, this argument is ignored.</p> <code>None</code> <code>preprocessing_rules</code> <code>Optional[List[Callable[[str], str]]]</code> <p>A list of functions to be applied to the text data before encoding. This can be useful to clean the text data before encoding. For example, removing html tags, special characters, etc.</p> <code>None</code> <code>tokenizer_params</code> <code>Optional[Dict[str, Any]]</code> <p>Additional parameters to be passed to the HuggingFace's <code>PreTrainedTokenizer</code>. Parameters to the <code>PreTrainedTokenizer</code> can also be passed via the <code>**kwargs</code> argument</p> <code>None</code> <code>encode_params</code> <code>Optional[Dict[str, Any]]</code> <p>Additional parameters to be passed to the <code>batch_encode_plus</code> method of the HuggingFace's <code>PreTrainedTokenizer</code>. If the <code>fit</code> and <code>transform</code> methods are used, the <code>encode_params</code> dict parameter is mandatory. If the <code>encode</code> method is used, this parameter is not needed since the input text is passed directly to the <code>encode</code> method.</p> <code>None</code> <code>**kwargs</code> <p>Additional kwargs to be passed to the model, in particular to the <code>PreTrainedTokenizer</code> class.</p> <code>{}</code> <p>Attributes:</p> Name Type Description <code>is_fitted</code> <code>bool</code> <p>Boolean indicating if the preprocessor has been fitted. This is a HuggingFacea tokenizer, so it is always considered fitted and this attribute is manually set to True internally. This parameter exists for consistency with the rest of the library and because is needed for some functionality in the library.</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import pandas as pd\n&gt;&gt;&gt; from pytorch_widedeep.preprocessing import HFPreprocessor\n&gt;&gt;&gt; df = pd.DataFrame({\"text\": [\"this is the first text\", \"this is the second text\"]})\n&gt;&gt;&gt; hf_processor_1 = HFPreprocessor(model_name=\"bert-base-uncased\", text_col=\"text\")\n&gt;&gt;&gt; X_text_1 = hf_processor_1.fit_transform(df)\n&gt;&gt;&gt; texts = [\"this is a new text\", \"this is another text\"]\n&gt;&gt;&gt; hf_processor_2 = HFPreprocessor(model_name=\"bert-base-uncased\")\n&gt;&gt;&gt; X_text_2 = hf_processor_2.encode(texts, max_length=10, padding=\"max_length\", truncation=True)\n</code></pre> Source code in <code>pytorch_widedeep/preprocessing/hf_preprocessor.py</code> <pre><code>class HFPreprocessor(BasePreprocessor):\n    \"\"\"Text processor to prepare the ``deeptext`` input dataset that is a\n    wrapper around HuggingFace's tokenizers.\n\n    Following the main phylosophy of the `pytorch-widedeep` library, this\n    class is designed to be as flexible as possible. Therefore, it is coded\n    so that the user can use it as one would use any HuggingFace tokenizers,\n    or following the API call 'protocol' of the rest of the library.\n\n    Parameters\n    ----------\n    model_name: str\n        The model name from the transformers library e.g. _'bert-base-uncased'_.\n        Currently supported models are those from the families: BERT, RoBERTa,\n        DistilBERT, ALBERT and ELECTRA.\n    use_fast_tokenizer: bool, default = False\n        Whether to use the fast tokenizer from HuggingFace or not\n    text_col: Optional[str], default = None\n        The column in the input dataframe containing the text data. If this\n        tokenizer is used via the `fit` and `transform` methods, this\n        argument is mandatory. If the tokenizer is used via the `encode`\n        method, this argument is not needed since the input text is passed\n        directly to the `encode` method.\n    num_workers: Optional[int], default = None\n        Number of workers to use when preprocessing the text data. If not\n        None, and `use_fast_tokenizer` is False, the text data will be\n        preprocessed in parallel using the number of workers specified. If\n        `use_fast_tokenizer` is True, this argument is ignored.\n    preprocessing_rules: Optional[List[Callable[[str], str]]], default = None\n        A list of functions to be applied to the text data before encoding.\n        This can be useful to clean the text data before encoding. For\n        example, removing html tags, special characters, etc.\n    tokenizer_params: Optional[Dict[str, Any]], default = None\n        Additional parameters to be passed to the HuggingFace's\n        `PreTrainedTokenizer`. Parameters to the `PreTrainedTokenizer`\n        can also be passed via the `**kwargs` argument\n    encode_params: Optional[Dict[str, Any]], default = None\n        Additional parameters to be passed to the `batch_encode_plus` method\n        of the HuggingFace's `PreTrainedTokenizer`. If the `fit` and `transform`\n        methods are used, the `encode_params` dict parameter is mandatory. If\n        the `encode` method is used, this parameter is not needed since the\n        input text is passed directly to the `encode` method.\n    **kwargs\n        Additional kwargs to be passed to the model, in particular to the\n        `PreTrainedTokenizer` class.\n\n    Attributes\n    ----------\n    is_fitted: bool\n        Boolean indicating if the preprocessor has been fitted. This is a\n        HuggingFacea tokenizer, so it is always considered fitted and this\n        attribute is manually set to True internally. This parameter exists\n        for consistency with the rest of the library and because is needed\n        for some functionality in the library.\n\n    Examples\n    --------\n    &gt;&gt;&gt; import pandas as pd\n    &gt;&gt;&gt; from pytorch_widedeep.preprocessing import HFPreprocessor\n    &gt;&gt;&gt; df = pd.DataFrame({\"text\": [\"this is the first text\", \"this is the second text\"]})\n    &gt;&gt;&gt; hf_processor_1 = HFPreprocessor(model_name=\"bert-base-uncased\", text_col=\"text\")\n    &gt;&gt;&gt; X_text_1 = hf_processor_1.fit_transform(df)\n    &gt;&gt;&gt; texts = [\"this is a new text\", \"this is another text\"]\n    &gt;&gt;&gt; hf_processor_2 = HFPreprocessor(model_name=\"bert-base-uncased\")\n    &gt;&gt;&gt; X_text_2 = hf_processor_2.encode(texts, max_length=10, padding=\"max_length\", truncation=True)\n    \"\"\"\n\n    def __init__(\n        self,\n        model_name: str,\n        *,\n        use_fast_tokenizer: bool = False,\n        text_col: Optional[str] = None,\n        root_dir: Optional[str] = None,\n        num_workers: Optional[int] = None,\n        preprocessing_rules: Optional[List[Callable[[str], str]]] = None,\n        tokenizer_params: Optional[Dict[str, Any]] = None,\n        encode_params: Optional[Dict[str, Any]] = None,\n        **kwargs,\n    ):\n        self.model_name = model_name\n        self.use_fast_tokenizer = use_fast_tokenizer\n        self.text_col = text_col\n        self.root_dir = root_dir\n        self.num_workers = num_workers\n        self.preprocessing_rules = preprocessing_rules\n        self.tokenizer_params = tokenizer_params if tokenizer_params is not None else {}\n        self.encode_params = encode_params if encode_params is not None else {}\n\n        self._multiprocessing = (\n            num_workers is not None and num_workers &gt; 1 and not use_fast_tokenizer\n        )\n\n        if kwargs:\n            self.tokenizer_params.update(kwargs)\n\n        self.tokenizer = get_tokenizer(\n            model_name=self.model_name,\n            use_fast_tokenizer=self.use_fast_tokenizer,\n            **self.tokenizer_params,\n        )\n\n        # A HuggingFace tokenizer is already trained, since we need this\n        # attribute elsewhere in the library, we simply set it to True\n        self.is_fitted = True\n\n    def encode(self, texts: List[str], **kwargs) -&gt; npt.NDArray[np.int64]:\n        \"\"\"\n        Encodes a list of texts. The method is a wrapper around the\n        `batch_encode_plus` method of the HuggingFace's tokenizer.\n\n        if 'use_fast_tokenizer' is True, the method will use the `batch_encode_plus`\n\n        Parameters\n        ----------\n        texts: List[str]\n            List of texts to be encoded\n        **kwargs\n            Additional parameters to be passed to the `batch_encode_plus` method\n            of the HuggingFace's tokenizer. If the 'encode_params' dict was passed\n            when instantiating the class, that dictionaly will be updated with\n            the kwargs passed here.\n\n        Returns\n        -------\n        np.array\n            The encoded texts\n        \"\"\"\n        if kwargs:\n            self.encode_params.update(kwargs)\n\n        if self.preprocessing_rules:\n            if self._multiprocessing:\n                texts = self._process_text_parallel(texts)\n            else:\n                texts = [self._preprocess_text(text) for text in texts]\n\n        if self._multiprocessing:\n            input_ids = self._encode_paralell(texts, **self.encode_params)\n        else:\n            encoded_texts = self.tokenizer.batch_encode_plus(\n                texts,\n                **self.encode_params,\n            )\n            input_ids = encoded_texts.get(\"input_ids\")\n\n        self.is_fitted = True\n\n        try:\n            output = np.array(input_ids)\n        except ValueError:\n            warnings.warn(\n                \"Padding and Truncating parameters were not passed and all input arrays \"\n                \"do not have the same shape. Padding to the longest sequence. \"\n                \"Padding will be done with the index of the pad token for the model\",\n                UserWarning,\n            )\n            max_len = max([len(ids) for ids in input_ids])\n            output = np.array(\n                [\n                    np.pad(ids, (self.tokenizer.pad_token_id, max_len - len(ids)))\n                    for ids in input_ids\n                ]\n            )\n\n        return output\n\n    def decode(\n        self, input_ids: npt.NDArray[np.int64], skip_special_tokens: bool\n    ) -&gt; List[str]:\n        \"\"\"\n        Decodes a list of input_ids. The method is a wrapper around the\n        `convert_ids_to_tokens` and `convert_tokens_to_string` methods of the\n        HuggingFace's tokenizer.\n\n        Parameters\n        ----------\n        input_ids: npt.NDArray[np.int64]\n            The input_ids to be decoded\n        skip_special_tokens: bool\n            Whether to skip the special tokens or not\n\n        Returns\n        -------\n        List[str]\n            The decoded texts\n        \"\"\"\n        texts = [\n            self.tokenizer.convert_tokens_to_string(\n                self.tokenizer.convert_ids_to_tokens(input_ids[i], skip_special_tokens)\n            )\n            for i in range(input_ids.shape[0])\n        ]\n        return texts\n\n    def fit(self, df: pd.DataFrame) -&gt; \"HFPreprocessor\":\n        \"\"\"\n        This method is included for consistency with the rest of the library\n        in general and with the `BasePreprocessor` in particular. HuggingFace's\n        tokenizers and models are already trained. Therefore, the 'fit' method\n        here does nothing other than checking that the 'text_col' parameter is\n        not `None`.\n\n        Parameters\n        ----------\n        df: pd.DataFrame\n            The dataframe containing the text data in the column specified by\n            the 'text_col' parameter\n        \"\"\"\n        if self.text_col is None:\n            raise ValueError(\n                \"'text_col' is None. Please specify the column name containing the text data\"\n                \" if you want to use the 'fit' method\"\n            )\n        return self\n\n    def transform(self, df: pd.DataFrame) -&gt; npt.NDArray[np.int64]:\n        \"\"\"\n        Encodes the text data in the input dataframe. This method simply\n        calls the `encode` method under the hood. Similar to the `fit` method,\n        this method is included for consistency with the rest of the library\n        in general and with the `BasePreprocessor` in particular.\n\n        Parameters\n        ----------\n        df: pd.DataFrame\n            The dataframe containing the text data in the column specified by\n            the 'text_col' parameter\n\n        Returns\n        -------\n        np.array\n            The encoded texts\n        \"\"\"\n        if self.text_col is None:\n            raise ValueError(\n                \"'text_col' is None. Please specify the column name containing the text data\"\n                \" if you want to use the 'fit' method\"\n            )\n\n        texts = self._read_texts(df, self.root_dir)\n\n        return self.encode(texts)\n\n    def transform_sample(self, text: str) -&gt; npt.NDArray[np.int64]:\n        \"\"\"\n        Encodes a single text sample.\n\n        Parameters\n        ----------\n        text: str\n            The text sample to be encoded\n\n        Returns\n        -------\n        np.array\n            The encoded text\n        \"\"\"\n\n        if not self.is_fitted:\n            raise ValueError(\n                \"The `encode` (or `fit`) method must be called before calling `transform_sample`\"\n            )\n        return self.encode([text])[0]\n\n    def fit_transform(self, df: pd.DataFrame) -&gt; npt.NDArray[np.int64]:\n        \"\"\"\n        Encodes the text data in the input dataframe.\n\n        Parameters\n        ----------\n        df: pd.DataFrame\n            The dataframe containing the text data in the column specified by\n            the 'text_col' parameter\n\n        Returns\n        -------\n        np.array\n            The encoded texts\n        \"\"\"\n        return self.fit(df).transform(df)\n\n    def inverse_transform(\n        self, input_ids: npt.NDArray[np.int64], skip_special_tokens: bool\n    ) -&gt; List[str]:\n        \"\"\"\n        Decodes a list of input_ids. The method simply calls the `decode` method\n        under the hood.\n\n        Parameters\n        ----------\n        input_ids: npt.NDArray[np.int64]\n            The input_ids to be decoded\n        skip_special_tokens: bool\n            Whether to skip the special tokens or not\n\n        Returns\n        -------\n        List[str]\n            The decoded texts\n        \"\"\"\n        return self.decode(input_ids, skip_special_tokens)\n\n    def _process_text_parallel(self, texts: List[str]) -&gt; List[str]:\n        num_processes = (\n            self.num_workers if self.num_workers is not None else os.cpu_count()\n        )\n        with ProcessPoolExecutor(max_workers=num_processes) as executor:\n            processed_texts = list(executor.map(self._preprocess_text, texts))\n        return processed_texts\n\n    def _preprocess_text(self, text: str) -&gt; str:\n        for rule in self.preprocessing_rules:\n            text = rule(text)\n        return text\n\n    def _encode_paralell(self, texts: List[str], **kwargs) -&gt; List[List[int]]:\n        num_processes = (\n            self.num_workers if self.num_workers is not None else os.cpu_count()\n        )\n        ptokenizer = partial(self.tokenizer.encode, **kwargs)\n        with ProcessPoolExecutor(max_workers=num_processes) as executor:\n            encoded_texts = list(executor.map(ptokenizer, texts))\n        return encoded_texts\n\n    def _read_texts(\n        self, df: pd.DataFrame, root_dir: Optional[str] = None\n    ) -&gt; List[str]:\n        if root_dir is not None:\n            if not os.path.exists(root_dir):\n                raise ValueError(\n                    \"root_dir does not exist. Please create it before fitting the preprocessor\"\n                )\n            texts_fnames = df[self.text_col].tolist()\n            texts: List[str] = []\n            for texts_fname in texts_fnames:\n                with open(os.path.join(root_dir, texts_fname), \"r\") as f:\n                    texts.append(f.read().replace(\"\\n\", \"\"))\n        else:\n            texts = df[self.text_col].tolist()\n\n        return texts\n\n    def __repr__(self):\n        return (\n            f\"HFPreprocessor(text_col={self.text_col}, model_name={self.model_name}, \"\n            f\"use_fast_tokenizer={self.use_fast_tokenizer}, num_workers={self.num_workers}, \"\n            f\"preprocessing_rules={self.preprocessing_rules}, tokenizer_params={self.tokenizer_params}, \"\n            f\"encode_params={self.encode_params})\"\n        )\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.encode","title":"encode","text":"<pre><code>encode(texts, **kwargs)\n</code></pre> <p>Encodes a list of texts. The method is a wrapper around the <code>batch_encode_plus</code> method of the HuggingFace's tokenizer.</p> <p>if 'use_fast_tokenizer' is True, the method will use the <code>batch_encode_plus</code></p> <p>Parameters:</p> Name Type Description Default <code>texts</code> <code>List[str]</code> <p>List of texts to be encoded</p> required <code>**kwargs</code> <p>Additional parameters to be passed to the <code>batch_encode_plus</code> method of the HuggingFace's tokenizer. If the 'encode_params' dict was passed when instantiating the class, that dictionaly will be updated with the kwargs passed here.</p> <code>{}</code> <p>Returns:</p> Type Description <code>array</code> <p>The encoded texts</p> Source code in <code>pytorch_widedeep/preprocessing/hf_preprocessor.py</code> <pre><code>def encode(self, texts: List[str], **kwargs) -&gt; npt.NDArray[np.int64]:\n    \"\"\"\n    Encodes a list of texts. The method is a wrapper around the\n    `batch_encode_plus` method of the HuggingFace's tokenizer.\n\n    if 'use_fast_tokenizer' is True, the method will use the `batch_encode_plus`\n\n    Parameters\n    ----------\n    texts: List[str]\n        List of texts to be encoded\n    **kwargs\n        Additional parameters to be passed to the `batch_encode_plus` method\n        of the HuggingFace's tokenizer. If the 'encode_params' dict was passed\n        when instantiating the class, that dictionaly will be updated with\n        the kwargs passed here.\n\n    Returns\n    -------\n    np.array\n        The encoded texts\n    \"\"\"\n    if kwargs:\n        self.encode_params.update(kwargs)\n\n    if self.preprocessing_rules:\n        if self._multiprocessing:\n            texts = self._process_text_parallel(texts)\n        else:\n            texts = [self._preprocess_text(text) for text in texts]\n\n    if self._multiprocessing:\n        input_ids = self._encode_paralell(texts, **self.encode_params)\n    else:\n        encoded_texts = self.tokenizer.batch_encode_plus(\n            texts,\n            **self.encode_params,\n        )\n        input_ids = encoded_texts.get(\"input_ids\")\n\n    self.is_fitted = True\n\n    try:\n        output = np.array(input_ids)\n    except ValueError:\n        warnings.warn(\n            \"Padding and Truncating parameters were not passed and all input arrays \"\n            \"do not have the same shape. Padding to the longest sequence. \"\n            \"Padding will be done with the index of the pad token for the model\",\n            UserWarning,\n        )\n        max_len = max([len(ids) for ids in input_ids])\n        output = np.array(\n            [\n                np.pad(ids, (self.tokenizer.pad_token_id, max_len - len(ids)))\n                for ids in input_ids\n            ]\n        )\n\n    return output\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.decode","title":"decode","text":"<pre><code>decode(input_ids, skip_special_tokens)\n</code></pre> <p>Decodes a list of input_ids. The method is a wrapper around the <code>convert_ids_to_tokens</code> and <code>convert_tokens_to_string</code> methods of the HuggingFace's tokenizer.</p> <p>Parameters:</p> Name Type Description Default <code>input_ids</code> <code>NDArray[int64]</code> <p>The input_ids to be decoded</p> required <code>skip_special_tokens</code> <code>bool</code> <p>Whether to skip the special tokens or not</p> required <p>Returns:</p> Type Description <code>List[str]</code> <p>The decoded texts</p> Source code in <code>pytorch_widedeep/preprocessing/hf_preprocessor.py</code> <pre><code>def decode(\n    self, input_ids: npt.NDArray[np.int64], skip_special_tokens: bool\n) -&gt; List[str]:\n    \"\"\"\n    Decodes a list of input_ids. The method is a wrapper around the\n    `convert_ids_to_tokens` and `convert_tokens_to_string` methods of the\n    HuggingFace's tokenizer.\n\n    Parameters\n    ----------\n    input_ids: npt.NDArray[np.int64]\n        The input_ids to be decoded\n    skip_special_tokens: bool\n        Whether to skip the special tokens or not\n\n    Returns\n    -------\n    List[str]\n        The decoded texts\n    \"\"\"\n    texts = [\n        self.tokenizer.convert_tokens_to_string(\n            self.tokenizer.convert_ids_to_tokens(input_ids[i], skip_special_tokens)\n        )\n        for i in range(input_ids.shape[0])\n    ]\n    return texts\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.fit","title":"fit","text":"<pre><code>fit(df)\n</code></pre> <p>This method is included for consistency with the rest of the library in general and with the <code>BasePreprocessor</code> in particular. HuggingFace's tokenizers and models are already trained. Therefore, the 'fit' method here does nothing other than checking that the 'text_col' parameter is not <code>None</code>.</p> <p>Parameters:</p> Name Type Description Default <code>df</code> <code>DataFrame</code> <p>The dataframe containing the text data in the column specified by the 'text_col' parameter</p> required Source code in <code>pytorch_widedeep/preprocessing/hf_preprocessor.py</code> <pre><code>def fit(self, df: pd.DataFrame) -&gt; \"HFPreprocessor\":\n    \"\"\"\n    This method is included for consistency with the rest of the library\n    in general and with the `BasePreprocessor` in particular. HuggingFace's\n    tokenizers and models are already trained. Therefore, the 'fit' method\n    here does nothing other than checking that the 'text_col' parameter is\n    not `None`.\n\n    Parameters\n    ----------\n    df: pd.DataFrame\n        The dataframe containing the text data in the column specified by\n        the 'text_col' parameter\n    \"\"\"\n    if self.text_col is None:\n        raise ValueError(\n            \"'text_col' is None. Please specify the column name containing the text data\"\n            \" if you want to use the 'fit' method\"\n        )\n    return self\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.transform","title":"transform","text":"<pre><code>transform(df)\n</code></pre> <p>Encodes the text data in the input dataframe. This method simply calls the <code>encode</code> method under the hood. Similar to the <code>fit</code> method, this method is included for consistency with the rest of the library in general and with the <code>BasePreprocessor</code> in particular.</p> <p>Parameters:</p> Name Type Description Default <code>df</code> <code>DataFrame</code> <p>The dataframe containing the text data in the column specified by the 'text_col' parameter</p> required <p>Returns:</p> Type Description <code>array</code> <p>The encoded texts</p> Source code in <code>pytorch_widedeep/preprocessing/hf_preprocessor.py</code> <pre><code>def transform(self, df: pd.DataFrame) -&gt; npt.NDArray[np.int64]:\n    \"\"\"\n    Encodes the text data in the input dataframe. This method simply\n    calls the `encode` method under the hood. Similar to the `fit` method,\n    this method is included for consistency with the rest of the library\n    in general and with the `BasePreprocessor` in particular.\n\n    Parameters\n    ----------\n    df: pd.DataFrame\n        The dataframe containing the text data in the column specified by\n        the 'text_col' parameter\n\n    Returns\n    -------\n    np.array\n        The encoded texts\n    \"\"\"\n    if self.text_col is None:\n        raise ValueError(\n            \"'text_col' is None. Please specify the column name containing the text data\"\n            \" if you want to use the 'fit' method\"\n        )\n\n    texts = self._read_texts(df, self.root_dir)\n\n    return self.encode(texts)\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.transform_sample","title":"transform_sample","text":"<pre><code>transform_sample(text)\n</code></pre> <p>Encodes a single text sample.</p> <p>Parameters:</p> Name Type Description Default <code>text</code> <code>str</code> <p>The text sample to be encoded</p> required <p>Returns:</p> Type Description <code>array</code> <p>The encoded text</p> Source code in <code>pytorch_widedeep/preprocessing/hf_preprocessor.py</code> <pre><code>def transform_sample(self, text: str) -&gt; npt.NDArray[np.int64]:\n    \"\"\"\n    Encodes a single text sample.\n\n    Parameters\n    ----------\n    text: str\n        The text sample to be encoded\n\n    Returns\n    -------\n    np.array\n        The encoded text\n    \"\"\"\n\n    if not self.is_fitted:\n        raise ValueError(\n            \"The `encode` (or `fit`) method must be called before calling `transform_sample`\"\n        )\n    return self.encode([text])[0]\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.fit_transform","title":"fit_transform","text":"<pre><code>fit_transform(df)\n</code></pre> <p>Encodes the text data in the input dataframe.</p> <p>Parameters:</p> Name Type Description Default <code>df</code> <code>DataFrame</code> <p>The dataframe containing the text data in the column specified by the 'text_col' parameter</p> required <p>Returns:</p> Type Description <code>array</code> <p>The encoded texts</p> Source code in <code>pytorch_widedeep/preprocessing/hf_preprocessor.py</code> <pre><code>def fit_transform(self, df: pd.DataFrame) -&gt; npt.NDArray[np.int64]:\n    \"\"\"\n    Encodes the text data in the input dataframe.\n\n    Parameters\n    ----------\n    df: pd.DataFrame\n        The dataframe containing the text data in the column specified by\n        the 'text_col' parameter\n\n    Returns\n    -------\n    np.array\n        The encoded texts\n    \"\"\"\n    return self.fit(df).transform(df)\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.inverse_transform","title":"inverse_transform","text":"<pre><code>inverse_transform(input_ids, skip_special_tokens)\n</code></pre> <p>Decodes a list of input_ids. The method simply calls the <code>decode</code> method under the hood.</p> <p>Parameters:</p> Name Type Description Default <code>input_ids</code> <code>NDArray[int64]</code> <p>The input_ids to be decoded</p> required <code>skip_special_tokens</code> <code>bool</code> <p>Whether to skip the special tokens or not</p> required <p>Returns:</p> Type Description <code>List[str]</code> <p>The decoded texts</p> Source code in <code>pytorch_widedeep/preprocessing/hf_preprocessor.py</code> <pre><code>def inverse_transform(\n    self, input_ids: npt.NDArray[np.int64], skip_special_tokens: bool\n) -&gt; List[str]:\n    \"\"\"\n    Decodes a list of input_ids. The method simply calls the `decode` method\n    under the hood.\n\n    Parameters\n    ----------\n    input_ids: npt.NDArray[np.int64]\n        The input_ids to be decoded\n    skip_special_tokens: bool\n        Whether to skip the special tokens or not\n\n    Returns\n    -------\n    List[str]\n        The decoded texts\n    \"\"\"\n    return self.decode(input_ids, skip_special_tokens)\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.image_preprocessor.ImagePreprocessor","title":"ImagePreprocessor","text":"<p>               Bases: <code>BasePreprocessor</code></p> <p>Preprocessor to prepare the <code>deepimage</code> input dataset.</p> <p>The Preprocessing consists simply on resizing according to their aspect ratio</p> <p>Parameters:</p> Name Type Description Default <code>img_col</code> <code>str</code> <p>name of the column with the images filenames</p> required <code>img_path</code> <code>str</code> <p>path to the dicrectory where the images are stored</p> required <code>width</code> <code>int</code> <p>width of the resulting processed image.</p> <code>224</code> <code>height</code> <code>int</code> <p>width of the resulting processed image.</p> <code>224</code> <code>verbose</code> <code>int</code> <p>Enable verbose output.</p> <code>1</code> <p>Attributes:</p> Name Type Description <code>aap</code> <code>AspectAwarePreprocessor</code> <p>an instance of <code>pytorch_widedeep.utils.image_utils.AspectAwarePreprocessor</code></p> <code>spp</code> <code>SimplePreprocessor</code> <p>an instance of <code>pytorch_widedeep.utils.image_utils.SimplePreprocessor</code></p> <code>normalise_metrics</code> <code>Dict</code> <p>Dict containing the normalisation metrics of the image dataset, i.e. mean and std for the R, G and B channels</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import pandas as pd\n&gt;&gt;&gt;\n&gt;&gt;&gt; from pytorch_widedeep.preprocessing import ImagePreprocessor\n&gt;&gt;&gt;\n&gt;&gt;&gt; path_to_image1 = 'tests/test_data_utils/images/galaxy1.png'\n&gt;&gt;&gt; path_to_image2 = 'tests/test_data_utils/images/galaxy2.png'\n&gt;&gt;&gt;\n&gt;&gt;&gt; df_train = pd.DataFrame({'images_column': [path_to_image1]})\n&gt;&gt;&gt; df_test = pd.DataFrame({'images_column': [path_to_image2]})\n&gt;&gt;&gt; img_preprocessor = ImagePreprocessor(img_col='images_column', img_path='.', verbose=0)\n&gt;&gt;&gt; resized_images = img_preprocessor.fit_transform(df_train)\n&gt;&gt;&gt; new_resized_images = img_preprocessor.transform(df_train)\n</code></pre> <p> NOTE: Normalising metrics will only be computed when the <code>fit_transform</code> method is run. Running <code>transform</code> only will not change the computed metrics and running <code>fit</code> only simply instantiates the resizing functions.</p> Source code in <code>pytorch_widedeep/preprocessing/image_preprocessor.py</code> <pre><code>class ImagePreprocessor(BasePreprocessor):\n    r\"\"\"Preprocessor to prepare the ``deepimage`` input dataset.\n\n    The Preprocessing consists simply on resizing according to their\n    aspect ratio\n\n    Parameters\n    ----------\n    img_col: str\n        name of the column with the images filenames\n    img_path: str\n        path to the dicrectory where the images are stored\n    width: int, default=224\n        width of the resulting processed image.\n    height: int, default=224\n        width of the resulting processed image.\n    verbose: int, default 1\n        Enable verbose output.\n\n    Attributes\n    ----------\n    aap: AspectAwarePreprocessor\n        an instance of `pytorch_widedeep.utils.image_utils.AspectAwarePreprocessor`\n    spp: SimplePreprocessor\n        an instance of `pytorch_widedeep.utils.image_utils.SimplePreprocessor`\n    normalise_metrics: Dict\n        Dict containing the normalisation metrics of the image dataset, i.e.\n        mean and std for the R, G and B channels\n\n    Examples\n    --------\n    &gt;&gt;&gt; import pandas as pd\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; from pytorch_widedeep.preprocessing import ImagePreprocessor\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; path_to_image1 = 'tests/test_data_utils/images/galaxy1.png'\n    &gt;&gt;&gt; path_to_image2 = 'tests/test_data_utils/images/galaxy2.png'\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; df_train = pd.DataFrame({'images_column': [path_to_image1]})\n    &gt;&gt;&gt; df_test = pd.DataFrame({'images_column': [path_to_image2]})\n    &gt;&gt;&gt; img_preprocessor = ImagePreprocessor(img_col='images_column', img_path='.', verbose=0)\n    &gt;&gt;&gt; resized_images = img_preprocessor.fit_transform(df_train)\n    &gt;&gt;&gt; new_resized_images = img_preprocessor.transform(df_train)\n\n    :information_source: **NOTE**:\n    Normalising metrics will only be computed when the ``fit_transform``\n    method is run. Running ``transform`` only will not change the computed\n    metrics and running ``fit`` only simply instantiates the resizing\n    functions.\n    \"\"\"\n\n    def __init__(\n        self,\n        img_col: str,\n        img_path: str,\n        width: int = 224,\n        height: int = 224,\n        verbose: int = 1,\n    ):\n        super(ImagePreprocessor, self).__init__()\n\n        self.img_col = img_col\n        self.img_path = img_path\n        self.width = width\n        self.height = height\n        self.verbose = verbose\n\n        self.aap = AspectAwarePreprocessor(self.width, self.height)\n        self.spp = SimplePreprocessor(self.width, self.height)\n\n        self.compute_normalising_computed = False\n\n    def fit(self, df: pd.DataFrame) -&gt; BasePreprocessor:\n        # simply include this method to comply with the BasePreprocessor and\n        # for consistency with the rest of the preprocessors\n        return self\n\n    def transform_sample(self, img: np.ndarray) -&gt; np.ndarray:\n        aspect_r = img.shape[0] / img.shape[1]\n        if aspect_r != 1.0:\n            resized_img = self.aap.preprocess(img)\n        else:\n            resized_img = self.spp.preprocess(img)\n        return resized_img\n\n    def transform(self, df: pd.DataFrame) -&gt; np.ndarray:\n        \"\"\"Resizes the images to the input height and width.\n\n\n        Parameters\n        ----------\n        df: pd.DataFrame\n            Input pandas dataframe with the `img_col`\n\n        Returns\n        -------\n        np.ndarray\n            Resized images to the input height and width\n        \"\"\"\n        image_list = df[self.img_col].tolist()\n        if self.verbose:\n            print(\"Reading Images from {}\".format(self.img_path))\n        imgs = [cv2.imread(\"/\".join([self.img_path, img])) for img in image_list]\n\n        # finding images with different height and width\n        aspect = [(im.shape[0], im.shape[1]) for im in imgs]\n        aspect_r = [a[0] / a[1] for a in aspect]\n        diff_idx = [i for i, r in enumerate(aspect_r) if r != 1.0]\n\n        if self.verbose:\n            print(\"Resizing\")\n        resized_imgs = []\n        for i, img in tqdm(enumerate(imgs), total=len(imgs), disable=self.verbose != 1):\n            if i in diff_idx:\n                resized_imgs.append(self.aap.preprocess(img))\n            else:\n                # if aspect ratio is 1:1, no need for AspectAwarePreprocessor\n                resized_imgs.append(self.spp.preprocess(img))\n\n        if not self.compute_normalising_computed:\n            if self.verbose:\n                print(\"Computing normalisation metrics\")\n            # mean and std deviation will only be computed when the fit method\n            # is called\n            mean_R, mean_G, mean_B = [], [], []\n            std_R, std_G, std_B = [], [], []\n            for rsz_img in resized_imgs:\n                (mean_b, mean_g, mean_r), (std_b, std_g, std_r) = cv2.meanStdDev(\n                    rsz_img\n                )\n                mean_R.append(mean_r)\n                mean_G.append(mean_g)\n                mean_B.append(mean_b)\n                std_R.append(std_r)\n                std_G.append(std_g)\n                std_B.append(std_b)\n            self.normalise_metrics = dict(\n                mean={\n                    \"R\": np.mean(mean_R) / 255.0,\n                    \"G\": np.mean(mean_G) / 255.0,\n                    \"B\": np.mean(mean_B) / 255.0,\n                },\n                std={\n                    \"R\": np.mean(std_R) / 255.0,\n                    \"G\": np.mean(std_G) / 255.0,\n                    \"B\": np.mean(std_B) / 255.0,\n                },\n            )\n            self.compute_normalising_computed = True\n        return np.asarray(resized_imgs)\n\n    def fit_transform(self, df: pd.DataFrame) -&gt; np.ndarray:\n        \"\"\"Combines `fit` and `transform`\n\n        Parameters\n        ----------\n        df: pd.DataFrame\n            Input pandas dataframe\n\n        Returns\n        -------\n        np.ndarray\n            Resized images to the input height and width\n        \"\"\"\n        return self.fit(df).transform(df)\n\n    def inverse_transform(self, transformed_image):\n        raise NotImplementedError(\n            \"'inverse_transform' method is not implemented for 'ImagePreprocessor'\"\n        )\n\n    def __repr__(self) -&gt; str:\n        list_of_params: List[str] = []\n        list_of_params.append(\"img_col={img_col}\")\n        list_of_params.append(\"img_path={img_path}\")\n        list_of_params.append(\"width={width}\")\n        list_of_params.append(\"height={height}\")\n        list_of_params.append(\"verbose={verbose}\")\n        all_params = \", \".join(list_of_params)\n        return f\"ImagePreprocessor({all_params.format(**self.__dict__)})\"\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.image_preprocessor.ImagePreprocessor.transform","title":"transform","text":"<pre><code>transform(df)\n</code></pre> <p>Resizes the images to the input height and width.</p> <p>Parameters:</p> Name Type Description Default <code>df</code> <code>DataFrame</code> <p>Input pandas dataframe with the <code>img_col</code></p> required <p>Returns:</p> Type Description <code>ndarray</code> <p>Resized images to the input height and width</p> Source code in <code>pytorch_widedeep/preprocessing/image_preprocessor.py</code> <pre><code>def transform(self, df: pd.DataFrame) -&gt; np.ndarray:\n    \"\"\"Resizes the images to the input height and width.\n\n\n    Parameters\n    ----------\n    df: pd.DataFrame\n        Input pandas dataframe with the `img_col`\n\n    Returns\n    -------\n    np.ndarray\n        Resized images to the input height and width\n    \"\"\"\n    image_list = df[self.img_col].tolist()\n    if self.verbose:\n        print(\"Reading Images from {}\".format(self.img_path))\n    imgs = [cv2.imread(\"/\".join([self.img_path, img])) for img in image_list]\n\n    # finding images with different height and width\n    aspect = [(im.shape[0], im.shape[1]) for im in imgs]\n    aspect_r = [a[0] / a[1] for a in aspect]\n    diff_idx = [i for i, r in enumerate(aspect_r) if r != 1.0]\n\n    if self.verbose:\n        print(\"Resizing\")\n    resized_imgs = []\n    for i, img in tqdm(enumerate(imgs), total=len(imgs), disable=self.verbose != 1):\n        if i in diff_idx:\n            resized_imgs.append(self.aap.preprocess(img))\n        else:\n            # if aspect ratio is 1:1, no need for AspectAwarePreprocessor\n            resized_imgs.append(self.spp.preprocess(img))\n\n    if not self.compute_normalising_computed:\n        if self.verbose:\n            print(\"Computing normalisation metrics\")\n        # mean and std deviation will only be computed when the fit method\n        # is called\n        mean_R, mean_G, mean_B = [], [], []\n        std_R, std_G, std_B = [], [], []\n        for rsz_img in resized_imgs:\n            (mean_b, mean_g, mean_r), (std_b, std_g, std_r) = cv2.meanStdDev(\n                rsz_img\n            )\n            mean_R.append(mean_r)\n            mean_G.append(mean_g)\n            mean_B.append(mean_b)\n            std_R.append(std_r)\n            std_G.append(std_g)\n            std_B.append(std_b)\n        self.normalise_metrics = dict(\n            mean={\n                \"R\": np.mean(mean_R) / 255.0,\n                \"G\": np.mean(mean_G) / 255.0,\n                \"B\": np.mean(mean_B) / 255.0,\n            },\n            std={\n                \"R\": np.mean(std_R) / 255.0,\n                \"G\": np.mean(std_G) / 255.0,\n                \"B\": np.mean(std_B) / 255.0,\n            },\n        )\n        self.compute_normalising_computed = True\n    return np.asarray(resized_imgs)\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.image_preprocessor.ImagePreprocessor.fit_transform","title":"fit_transform","text":"<pre><code>fit_transform(df)\n</code></pre> <p>Combines <code>fit</code> and <code>transform</code></p> <p>Parameters:</p> Name Type Description Default <code>df</code> <code>DataFrame</code> <p>Input pandas dataframe</p> required <p>Returns:</p> Type Description <code>ndarray</code> <p>Resized images to the input height and width</p> Source code in <code>pytorch_widedeep/preprocessing/image_preprocessor.py</code> <pre><code>def fit_transform(self, df: pd.DataFrame) -&gt; np.ndarray:\n    \"\"\"Combines `fit` and `transform`\n\n    Parameters\n    ----------\n    df: pd.DataFrame\n        Input pandas dataframe\n\n    Returns\n    -------\n    np.ndarray\n        Resized images to the input height and width\n    \"\"\"\n    return self.fit(df).transform(df)\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.din_preprocessor.DINPreprocessor","title":"DINPreprocessor","text":"<p>               Bases: <code>BasePreprocessor</code></p> <p>Preprocessor for Deep Interest Network (DIN) models.</p> <p>This preprocessor handles the preparation of data for DIN models, including sequence building, label encoding, and handling of various types of input columns (categorical, continuous, and sequential).</p> <p>Parameters:</p> Name Type Description Default <code>user_id_col</code> <code>str</code> <p>Name of the column containing user IDs.</p> required <code>item_embed_col</code> <code>Union[str, Tuple[str, int]]</code> <p>Name of the column containing item IDs to be embedded, or a tuple of (column_name, embedding_dim). If embedding_dim is not provided, it will be automatically calculated using a rule of thumb specified by <code>embedding_rule</code>. Aliased as <code>item_id_col</code>.</p> required <code>target_col</code> <code>str</code> <p>Name of the column containing the target variable.</p> required <code>max_seq_length</code> <code>int</code> <p>Maximum length of sequences to be created.</p> required <code>action_col</code> <code>Optional[str]</code> <p>Name of the column containing user actions (if applicable), for example a rating, or purchased/not-purchased, etc). This 'action_col' can also be the same as the 'target_col'.</p> <code>None</code> <code>other_seq_embed_cols</code> <code>Optional[List[str] | List[Tuple[str, int]]]</code> <p>List of other columns to be treated as sequences. Each element in the list can be a string with the name of the column or a tuple with the name of the column and the embedding dimension.</p> <code>None</code> <code>cat_embed_cols</code> <code>Optional[Union[List[str], List[Tuple[str, int]]]]</code> <p>This and the following parameters are identical to those used in the TabPreprocessor. List containing the name of the categorical columns that will be represented by embeddings (e.g. ['education', 'relationship', ...]) or a Tuple with the name and the embedding dimension (e.g.: [ ('education',32), ('relationship',16), ...])</p> <code>None</code> <code>continuous_cols</code> <code>Optional[List[str]]</code> <p>List with the name of the continuous cols</p> <code>None</code> <code>quantization_setup</code> <code>Optional[Union[int, Dict[str, Union[int, List[float]]]]]</code> <p>Continuous columns can be turned into categorical via <code>pd.cut</code>. If <code>quantization_setup</code> is an <code>int</code>, all continuous columns will be quantized using this value as the number of bins. Alternatively, a dictionary where the keys are the column names to quantize and the values are the either integers indicating the number of bins or a list of scalars indicating the bin edges can also be used.</p> <code>None</code> <code>cols_to_scale</code> <code>Optional[Union[List[str], str]]</code> <p>List with the names of the columns that will be standarised via sklearn's <code>StandardScaler</code>. It can also be the string <code>'all'</code> in which case all the continuous cols will be scaled.</p> <code>None</code> <code>auto_embed_dim</code> <code>bool</code> <p>Boolean indicating whether the embedding dimensions will be automatically defined via rule of thumb. See <code>embedding_rule</code> below.</p> <code>True</code> <code>embedding_rule</code> <code>Literal['google', 'fastai_old', 'fastai_new']</code> <p>If <code>auto_embed_dim=True</code>, this is the choice of embedding rule of thumb. Choices are:</p> <ul> <li> <p>fastai_new: \\(min(600, round(1.6     imes n_{cat}^{0.56}))\\)</p> </li> <li> <p>fastai_old: \\(min(50, (n_{cat}//{2})+1)\\)</p> </li> <li> <p>google: \\(min(600, round(n_{cat}^{0.24}))\\)</p> </li> </ul> <code>'fastai_new'</code> <code>default_embed_dim</code> <code>int</code> <p>Dimension for the embeddings if the embedding dimension is not provided in the <code>cat_embed_cols</code> parameter and <code>auto_embed_dim</code> is set to <code>False</code>.</p> <code>16</code> <code>verbose</code> <code>int</code> <p>Verbosity level.</p> <code>1</code> <code>scale</code> <code>bool</code> <p> note: this arg will be removed in upcoming  releases. Please use <code>cols_to_scale</code> instead.  Bool indicating  whether or not to scale/standarise continuous cols. It is important  to emphasize that all the DL models for tabular data in the library  also include the possibility of normalising the input continuous  features via a <code>BatchNorm</code> or a <code>LayerNorm</code>.  Param alias:  <code>scale_cont_cols</code>.</p> <code>False</code> <code>already_standard</code> <code>Optional[List[str]]</code> <p> note: this arg will be removed in upcoming  releases. Please use <code>cols_to_scale</code> instead.  List with the  name of the continuous cols that do not need to be  scaled/standarised.</p> <code>None</code> <p>Other Parameters:</p> Name Type Description <code>**kwargs</code> <p><code>pd.cut</code> and <code>StandardScaler</code> related args</p> <p>Attributes:</p> Name Type Description <code>is_fitted</code> <code>bool</code> <p>Whether the preprocessor has been fitted.</p> <code>has_standard_tab_data</code> <code>bool</code> <p>Whether the data includes standard tabular data.</p> <code>tab_preprocessor</code> <code>TabPreprocessor</code> <p>Preprocessor for standard tabular data (if applicable).</p> <code>din_columns_idx</code> <code>Dict[str, int]</code> <p>Dictionary mapping column names to their indices in the processed data.</p> <code>item_le</code> <code>LabelEncoder</code> <p>Label encoder for item IDs.</p> <code>n_items</code> <code>int</code> <p>Number of unique items.</p> <code>user_behaviour_config</code> <code>Tuple[List[str], int, int]</code> <p>Configuration for user behavior sequences.</p> <code>action_le</code> <code>LabelEncoder</code> <p>Label encoder for action column (if applicable).</p> <code>n_actions</code> <code>int</code> <p>Number of unique actions (if applicable).</p> <code>action_seq_config</code> <code>Tuple[List[str], int]</code> <p>Configuration for action sequences (if applicable).</p> <code>other_seq_le</code> <code>LabelEncoder</code> <p>Label encoder for other sequence columns (if applicable).</p> <code>n_other_seq_cols</code> <code>Dict[str, int]</code> <p>Number of unique values in each other sequence column.</p> <code>other_seq_config</code> <code>List[Tuple[List[str], int, int]]</code> <p>Configuration for other sequence columns.</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import pandas as pd\n&gt;&gt;&gt; from pytorch_widedeep.preprocessing import DINPreprocessor\n&gt;&gt;&gt; data = {\n...    'user_id': [1, 1, 1, 2, 2, 2, 3, 3, 3],\n...    'item_id': [101, 102, 103, 101, 103, 104, 102, 103, 104],\n...    'timestamp': [1, 2, 3, 1, 2, 3, 1, 2, 3],\n...    'category': ['A', 'B', 'A', 'B', 'A', 'C', 'B', 'A', 'C'],\n...    'price': [10.5, 15.0, 12.0, 10.5, 12.0, 20.0, 15.0, 12.0, 20.0],\n...    'rating': [0, 1, 0, 1, 0, 1, 0, 1, 0]\n... }\n&gt;&gt;&gt; df = pd.DataFrame(data)\n&gt;&gt;&gt; din_preprocessor = DINPreprocessor(\n...     user_id_col='user_id',\n...     item_embed_col='item_id',\n...     target_col='rating',\n...     max_seq_length=2,\n...     action_col='rating',\n...     other_seq_embed_cols=['category'],\n...     cat_embed_cols=['user_id'],\n...     continuous_cols=['price'],\n...     cols_to_scale=['price']\n... )\n&gt;&gt;&gt; X, y = din_preprocessor.fit_transform(df)\n</code></pre> Source code in <code>pytorch_widedeep/preprocessing/din_preprocessor.py</code> <pre><code>class DINPreprocessor(BasePreprocessor):\n    \"\"\"\n    Preprocessor for Deep Interest Network (DIN) models.\n\n    This preprocessor handles the preparation of data for DIN models,\n    including sequence building, label encoding, and handling of various\n    types of input columns (categorical, continuous, and sequential).\n\n    Parameters\n    ----------\n    user_id_col : str\n        Name of the column containing user IDs.\n    item_embed_col : Union[str, Tuple[str, int]]\n        Name of the column containing item IDs to be embedded, or a tuple of\n        (column_name, embedding_dim). If embedding_dim is not provided, it\n        will be automatically calculated using a rule of thumb specified by\n        `embedding_rule`. Aliased as `item_id_col`.\n    target_col : str\n        Name of the column containing the target variable.\n    max_seq_length : int\n        Maximum length of sequences to be created.\n    action_col : Optional[str], default=None\n        Name of the column containing user actions (if applicable), for\n        example a rating, or purchased/not-purchased, etc). This 'action_col'\n        can also be the same as the 'target_col'.\n    other_seq_embed_cols : Optional[List[str] | List[Tuple[str, int]]], default=None\n        List of other columns to be treated as sequences. Each element in the\n        list can be a string with the name of the column or a tuple with the\n        name of the column and the embedding dimension.\n\n    cat_embed_cols : Optional[Union[List[str], List[Tuple[str, int]]]], default=None\n        This and the following parameters are identical to those used in the\n        TabPreprocessor.&lt;br&gt;\n        List containing the name of the categorical columns that will be\n        represented by embeddings (e.g. _['education', 'relationship', ...]_) or\n        a Tuple with the name and the embedding dimension (e.g.: _[\n        ('education',32), ('relationship',16), ...]_)\n    continuous_cols: List, default = None\n        List with the name of the continuous cols\n    quantization_setup: int or Dict, default = None\n        Continuous columns can be turned into categorical via `pd.cut`. If\n        `quantization_setup` is an `int`, all continuous columns will be\n        quantized using this value as the number of bins. Alternatively, a\n        dictionary where the keys are the column names to quantize and the\n        values are the either integers indicating the number of bins or a\n        list of scalars indicating the bin edges can also be used.\n    cols_to_scale: List or str, default = None,\n        List with the names of the columns that will be standarised via\n        sklearn's `StandardScaler`. It can also be the string `'all'` in\n        which case all the continuous cols will be scaled.\n    auto_embed_dim: bool, default = True\n        Boolean indicating whether the embedding dimensions will be\n        automatically defined via rule of thumb. See `embedding_rule`\n        below.\n    embedding_rule: str, default = 'fastai_new'\n        If `auto_embed_dim=True`, this is the choice of embedding rule of\n        thumb. Choices are:\n\n        - _fastai_new_: $min(600, round(1.6 \\times n_{cat}^{0.56}))$\n\n        - _fastai_old_: $min(50, (n_{cat}//{2})+1)$\n\n        - _google_: $min(600, round(n_{cat}^{0.24}))$\n    default_embed_dim: int, default=16\n        Dimension for the embeddings if the embedding dimension is not\n        provided in the `cat_embed_cols` parameter and `auto_embed_dim` is\n        set to `False`.\n    verbose : int, default=1\n        Verbosity level.\n    scale: bool, default = False\n        :information_source: **note**: this arg will be removed in upcoming\n         releases. Please use `cols_to_scale` instead. &lt;br/&gt; Bool indicating\n         whether or not to scale/standarise continuous cols. It is important\n         to emphasize that all the DL models for tabular data in the library\n         also include the possibility of normalising the input continuous\n         features via a `BatchNorm` or a `LayerNorm`. &lt;br/&gt; Param alias:\n         `scale_cont_cols`.\n    already_standard: List, default = None\n        :information_source: **note**: this arg will be removed in upcoming\n         releases. Please use `cols_to_scale` instead. &lt;br/&gt; List with the\n         name of the continuous cols that do not need to be\n         scaled/standarised.\n\n    Other Parameters\n    ----------------\n    **kwargs: dict\n        `pd.cut` and `StandardScaler` related args\n\n    Attributes\n    ----------\n    is_fitted : bool\n        Whether the preprocessor has been fitted.\n    has_standard_tab_data : bool\n        Whether the data includes standard tabular data.\n    tab_preprocessor : TabPreprocessor\n        Preprocessor for standard tabular data (if applicable).\n    din_columns_idx : Dict[str, int]\n        Dictionary mapping column names to their indices in the processed data.\n    item_le : LabelEncoder\n        Label encoder for item IDs.\n    n_items : int\n        Number of unique items.\n    user_behaviour_config : Tuple[List[str], int, int]\n        Configuration for user behavior sequences.\n    action_le : LabelEncoder\n        Label encoder for action column (if applicable).\n    n_actions : int\n        Number of unique actions (if applicable).\n    action_seq_config : Tuple[List[str], int]\n        Configuration for action sequences (if applicable).\n    other_seq_le : LabelEncoder\n        Label encoder for other sequence columns (if applicable).\n    n_other_seq_cols : Dict[str, int]\n        Number of unique values in each other sequence column.\n    other_seq_config : List[Tuple[List[str], int, int]]\n        Configuration for other sequence columns.\n\n    Examples\n    --------\n    &gt;&gt;&gt; import pandas as pd\n    &gt;&gt;&gt; from pytorch_widedeep.preprocessing import DINPreprocessor\n    &gt;&gt;&gt; data = {\n    ...    'user_id': [1, 1, 1, 2, 2, 2, 3, 3, 3],\n    ...    'item_id': [101, 102, 103, 101, 103, 104, 102, 103, 104],\n    ...    'timestamp': [1, 2, 3, 1, 2, 3, 1, 2, 3],\n    ...    'category': ['A', 'B', 'A', 'B', 'A', 'C', 'B', 'A', 'C'],\n    ...    'price': [10.5, 15.0, 12.0, 10.5, 12.0, 20.0, 15.0, 12.0, 20.0],\n    ...    'rating': [0, 1, 0, 1, 0, 1, 0, 1, 0]\n    ... }\n    &gt;&gt;&gt; df = pd.DataFrame(data)\n    &gt;&gt;&gt; din_preprocessor = DINPreprocessor(\n    ...     user_id_col='user_id',\n    ...     item_embed_col='item_id',\n    ...     target_col='rating',\n    ...     max_seq_length=2,\n    ...     action_col='rating',\n    ...     other_seq_embed_cols=['category'],\n    ...     cat_embed_cols=['user_id'],\n    ...     continuous_cols=['price'],\n    ...     cols_to_scale=['price']\n    ... )\n    &gt;&gt;&gt; X, y = din_preprocessor.fit_transform(df)\n    \"\"\"\n\n    @alias(\"item_embed_col\", [\"item_id_col\"])\n    def __init__(\n        self,\n        *,\n        user_id_col: str,\n        item_embed_col: Union[str, Tuple[str, int]],\n        target_col: str,\n        max_seq_length: int,\n        action_col: Optional[str] = None,\n        other_seq_embed_cols: Optional[Union[List[str], List[Tuple[str, int]]]] = None,\n        cat_embed_cols: Optional[Union[List[str], List[Tuple[str, int]]]] = None,\n        continuous_cols: Optional[List[str]] = None,\n        quantization_setup: Optional[\n            Union[int, Dict[str, Union[int, List[float]]]]\n        ] = None,\n        cols_to_scale: Optional[Union[List[str], str]] = None,\n        auto_embed_dim: bool = True,\n        embedding_rule: Literal[\"google\", \"fastai_old\", \"fastai_new\"] = \"fastai_new\",\n        default_embed_dim: int = 16,\n        verbose: int = 1,\n        scale: bool = False,\n        already_standard: Optional[List[str]] = None,\n        **kwargs,\n    ):\n        self.user_id_col = user_id_col\n        self.item_embed_col = item_embed_col\n        self.max_seq_length = max_seq_length\n        self.target_col = target_col if target_col is not None else \"target\"\n        self.action_col = action_col\n        self.other_seq_embed_cols = other_seq_embed_cols\n        self.cat_embed_cols = cat_embed_cols\n        self.continuous_cols = continuous_cols\n        self.quantization_setup = quantization_setup\n        self.cols_to_scale = cols_to_scale\n        self.auto_embed_dim = auto_embed_dim\n        self.embedding_rule = embedding_rule\n        self.default_embed_dim = default_embed_dim\n        self.verbose = verbose\n        self.scale = scale\n        self.already_standard = already_standard\n        self.kwargs = kwargs\n\n        self.has_standard_tab_data = bool(self.cat_embed_cols or self.continuous_cols)\n        if self.has_standard_tab_data:\n            self.tab_preprocessor = TabPreprocessor(\n                cat_embed_cols=self.cat_embed_cols,\n                continuous_cols=self.continuous_cols,\n                quantization_setup=self.quantization_setup,\n                cols_to_scale=self.cols_to_scale,\n                auto_embed_dim=self.auto_embed_dim,\n                embedding_rule=self.embedding_rule,\n                default_embed_dim=self.default_embed_dim,\n                verbose=self.verbose,\n                scale=self.scale,\n                already_standard=self.already_standard,\n                **self.kwargs,\n            )\n\n        self.is_fitted = False\n\n    def fit(self, df: pd.DataFrame) -&gt; \"DINPreprocessor\":\n        if self.has_standard_tab_data:\n            self.tab_preprocessor.fit(df)\n            self.din_columns_idx = {\n                col: i for i, col in enumerate(self.tab_preprocessor.column_idx.keys())\n            }\n        else:\n            self.din_columns_idx = {}\n\n        self._fit_label_encoders(df)\n        self.is_fitted = True\n        return self\n\n    def transform(self, df: pd.DataFrame) -&gt; Tuple[np.ndarray, np.ndarray]:\n        _df = self._pre_transform(df)\n        df_w_sequences = self._build_sequences(_df)\n        X_all = self._concatenate_features(df_w_sequences)\n        return X_all, np.array(df_w_sequences[self.target_col].tolist())\n\n    def fit_transform(self, df: pd.DataFrame) -&gt; Tuple[np.ndarray, np.ndarray]:\n        self.fit(df)\n        return self.transform(df)\n\n    def inverse_transform(self, X: np.ndarray) -&gt; pd.DataFrame:\n        raise NotImplementedError(\n            \"inverse_transform is not implemented for this preprocessor\"\n        )\n\n    def _fit_label_encoders(self, df: pd.DataFrame) -&gt; None:\n        self.item_le = LabelEncoder(columns_to_encode=[self._get_item_col()])\n        self.item_le.fit(df)\n        self.n_items = max(self.item_le.encoding_dict[self._get_item_col()].values())\n        user_behaviour_embed_size = self._get_embed_size(\n            self.item_embed_col, self.n_items\n        )\n        self.user_behaviour_config = (\n            [f\"item_{i+1}\" for i in range(self.max_seq_length)],\n            self.n_items,\n            user_behaviour_embed_size,\n        )\n\n        self._update_din_columns_idx(\"item\", self.max_seq_length)\n        self.din_columns_idx[\"target_item\"] = len(self.din_columns_idx)\n\n        if self.action_col:\n            self._fit_action_label_encoder(df)\n        if self.other_seq_embed_cols:\n            self._fit_other_seq_label_encoders(df)\n\n    def _fit_action_label_encoder(self, df: pd.DataFrame) -&gt; None:\n        self.action_le = LabelEncoder(columns_to_encode=[self.action_col])\n        self.action_le.fit(df)\n        self.n_actions = len(self.action_le.encoding_dict[self.action_col])\n        self.action_seq_config = (\n            [f\"action_{i+1}\" for i in range(self.max_seq_length)],\n            self.n_actions,\n        )\n        self._update_din_columns_idx(\"action\", self.max_seq_length)\n\n    def _other_seq_cols_float_warning(self, df: pd.DataFrame) -&gt; None:\n        other_seq_cols = [\n            col[0] if isinstance(col, tuple) else col\n            for col in self.other_seq_embed_cols\n        ]\n        for col in other_seq_cols:\n            if df[col].dtype == float:\n                warnings.warn(\n                    f\"{col} is a float column. It will be converted to integers. \"\n                    \"If this is not what you want, please convert it beforehand.\",\n                    UserWarning,\n                )\n\n    def _convert_other_seq_cols_to_int(self, df: pd.DataFrame) -&gt; None:\n        other_seq_cols = [\n            col[0] if isinstance(col, tuple) else col\n            for col in self.other_seq_embed_cols\n        ]\n        for col in other_seq_cols:\n            if df[col].dtype == float:\n                df[col] = df[col].astype(int)\n\n    def _fit_other_seq_label_encoders(self, df: pd.DataFrame) -&gt; None:\n        self._other_seq_cols_float_warning(df)\n        self._convert_other_seq_cols_to_int(df)\n        self.other_seq_le = LabelEncoder(\n            columns_to_encode=[\n                col[0] if isinstance(col, tuple) else col\n                for col in self.other_seq_embed_cols\n            ]\n        )\n        self.other_seq_le.fit(df)\n        other_seq_cols = [\n            col[0] if isinstance(col, tuple) else col\n            for col in self.other_seq_embed_cols\n        ]\n        self.n_other_seq_cols = {\n            col: max(self.other_seq_le.encoding_dict[col].values())\n            for col in other_seq_cols\n        }\n        other_seq_embed_sizes = {\n            col: self._get_embed_size(col, self.n_other_seq_cols[col])\n            for col in other_seq_cols\n        }\n        self.other_seq_config = [\n            (\n                self._get_seq_col_names(col),\n                self.n_other_seq_cols[col],\n                other_seq_embed_sizes[col],\n            )\n            for col in other_seq_cols\n        ]\n        self._update_din_columns_idx_for_other_seq(other_seq_cols)\n\n    def _get_item_col(self) -&gt; str:\n        return (\n            self.item_embed_col[0]\n            if isinstance(self.item_embed_col, tuple)\n            else self.item_embed_col\n        )\n\n    def _get_embed_size(self, col: Union[str, Tuple[str, int]], n_unique: int) -&gt; int:\n        return col[1] if isinstance(col, tuple) else embed_sz_rule(n_unique)\n\n    def _get_seq_col_names(self, col: str) -&gt; List[str]:\n        return [f\"{col}_{i+1}\" for i in range(self.max_seq_length)]\n\n    def _update_din_columns_idx(self, prefix: str, length: int) -&gt; None:\n        current_len = len(self.din_columns_idx)\n        self.din_columns_idx.update(\n            {f\"{prefix}_{i+1}\": i + current_len for i in range(length)}\n        )\n\n    def _update_din_columns_idx_for_other_seq(self, other_seq_cols: List[str]) -&gt; None:\n        current_len = len(self.din_columns_idx)\n        for col in other_seq_cols:\n            self.din_columns_idx.update(\n                {f\"{col}_{i+1}\": i + current_len for i in range(self.max_seq_length)}\n            )\n            self.din_columns_idx[f\"target_{col}\"] = len(self.din_columns_idx)\n            current_len = len(self.din_columns_idx)\n\n    def _pre_transform(self, df: pd.DataFrame) -&gt; pd.DataFrame:\n        check_is_fitted(self, attributes=[\"din_columns_idx\"])\n        df = self.item_le.transform(df)\n        if self.action_col:\n            df = self.action_le.transform(df)\n        if self.other_seq_embed_cols:\n            df = self.other_seq_le.transform(df)\n        return df\n\n    def _build_sequences(self, df: pd.DataFrame) -&gt; pd.DataFrame:\n        # df.columns.tolist() -&gt; to avoid annoying pandas warning\n        return (\n            df.groupby(self.user_id_col)[df.columns.tolist()]  # type: ignore[index]\n            .apply(self._group_sequences)\n            .reset_index(drop=True)\n        )\n\n    def _group_sequences(self, group: pd.DataFrame) -&gt; pd.DataFrame:\n        item_col = self._get_item_col()\n        items = group[item_col].tolist()\n        targets = group[self.target_col].tolist()\n        drop_cols = [item_col, self.target_col]\n\n        # sequences cannot be built with user, item pairs with only one\n        # interaction\n        if len(items) &lt;= 1:\n            return pd.DataFrame()\n\n        sequences = [\n            self._create_sequence(group, items, targets, i, drop_cols)\n            for i in range(max(1, len(items) - self.max_seq_length))\n        ]\n\n        seq_df = pd.DataFrame(sequences)\n        non_seq_cols = group.drop_duplicates([self.user_id_col]).drop(drop_cols, axis=1)\n        return pd.merge(seq_df, non_seq_cols, on=self.user_id_col)\n\n    def _create_sequence(\n        self,\n        group: pd.DataFrame,\n        items: List[int],\n        targets: List[int],\n        i: int,\n        drop_cols: List[str],\n    ) -&gt; Dict[str, Union[str, List[int]]]:\n        end_idx = min(i + self.max_seq_length, len(items) - 1)\n        item_sequences = items[i:end_idx]\n        target_item = items[end_idx]\n        target = targets[end_idx]\n\n        sequence = {\n            self.user_id_col: group.name,\n            \"items_sequence\": item_sequences,\n            \"target_item\": target_item,\n        }\n\n        if self.action_col:\n            sequence.update(\n                self._create_action_sequence(group, targets, i, target, drop_cols)\n            )\n        else:\n            sequence[self.target_col] = target\n            drop_cols.append(self.target_col)\n\n        if self.other_seq_embed_cols:\n            sequence.update(self._create_other_seq_sequence(group, i, drop_cols))\n\n        return sequence\n\n    def _create_action_sequence(\n        self,\n        group: pd.DataFrame,\n        targets: List[int],\n        i: int,\n        target: int,\n        drop_cols: List[str],\n    ) -&gt; Dict[str, Union[int, List[int]]]:\n        if self.action_col != self.target_col:\n            actions = group[self.action_col].tolist()\n            action_sequences = (\n                actions[i : i + self.max_seq_length]\n                if i + self.max_seq_length &lt; len(actions)\n                else actions[i:-1]\n            )\n            drop_cols.append(self.action_col)\n        else:\n            target -= 1  # the 'transform' method adds 1 as it saves 0 for padding\n            action_sequences = (\n                targets[i : i + self.max_seq_length]\n                if i + self.max_seq_length &lt; len(targets)\n                else targets[i:-1]\n            )\n\n        return {self.target_col: target, \"actions_sequence\": action_sequences}\n\n    def _create_other_seq_sequence(\n        self, group: pd.DataFrame, i: int, drop_cols: List[str]\n    ) -&gt; Dict[str, Union[int, List[int]]]:\n        other_seq_cols = [\n            col[0] if isinstance(col, tuple) else col\n            for col in self.other_seq_embed_cols\n        ]\n        drop_cols += other_seq_cols\n        other_seqs = {col: group[col].tolist() for col in other_seq_cols}\n\n        other_seqs_sequences: Dict[str, Union[int, List[int]]] = {}\n        for col in other_seq_cols:\n            other_seqs_sequences[col] = (\n                other_seqs[col][i : i + self.max_seq_length]\n                if i + self.max_seq_length &lt; len(other_seqs[col])\n                else other_seqs[col][i:-1]\n            )\n\n        sequence: Dict[str, Union[int, List[int]]] = {}\n        for col in other_seq_cols:\n            sequence[f\"{col}_sequence\"] = other_seqs_sequences[col]\n            sequence[f\"target_{col}\"] = (\n                other_seqs[col][i + self.max_seq_length]\n                if i + self.max_seq_length &lt; len(other_seqs[col])\n                else other_seqs[col][-1]\n            )\n\n        return sequence\n\n    def _concatenate_features(self, df_w_sequences: pd.DataFrame) -&gt; np.ndarray:\n        user_behaviour_seq = df_w_sequences[\"items_sequence\"].tolist()\n        X_user_behaviour = np.vstack(\n            [\n                pad_sequences(seq, self.max_seq_length, pad_idx=0)\n                for seq in user_behaviour_seq\n            ]\n        )\n        X_target_item = np.array(df_w_sequences[\"target_item\"].tolist()).reshape(-1, 1)\n        X_all = np.concatenate([X_user_behaviour, X_target_item], axis=1)\n\n        if self.has_standard_tab_data:\n            X_tab = self.tab_preprocessor.transform(df_w_sequences)\n            X_all = np.concatenate([X_tab, X_all], axis=1)\n\n        if self.action_col:\n            action_seq = df_w_sequences[\"actions_sequence\"].tolist()\n            X_actions = np.vstack(\n                [\n                    pad_sequences(seq, self.max_seq_length, pad_idx=0)\n                    for seq in action_seq\n                ]\n            )\n            X_all = np.concatenate([X_all, X_actions], axis=1)\n\n        if self.other_seq_embed_cols:\n            X_all = self._concatenate_other_seq_features(df_w_sequences, X_all)\n\n        assert len(self.din_columns_idx) == X_all.shape[1], (\n            f\"Something went wrong. The number of columns in the final array \"\n            f\"({X_all.shape[1]}) is different from the number of columns in \"\n            f\"self.din_columns_idx ({len(self.din_columns_idx)})\"\n        )\n\n        return X_all\n\n    def _concatenate_other_seq_features(\n        self, df_w_sequences: pd.DataFrame, X_all: np.ndarray\n    ) -&gt; np.ndarray:\n        other_seq_cols = [\n            col[0] if isinstance(col, tuple) else col\n            for col in self.other_seq_embed_cols\n        ]\n        other_seq = {\n            col: df_w_sequences[f\"{col}_sequence\"].tolist() for col in other_seq_cols\n        }\n        other_seq_target = {\n            col: df_w_sequences[f\"target_{col}\"].tolist() for col in other_seq_cols\n        }\n        X_other_seq_arrays = []\n\n        for col in other_seq_cols:\n            X_other_seq_arrays.append(\n                np.vstack(\n                    [\n                        pad_sequences(s, self.max_seq_length, pad_idx=0)\n                        for s in other_seq[col]\n                    ]\n                )\n            )\n            X_other_seq_arrays.append(np.array(other_seq_target[col]).reshape(-1, 1))\n\n        return np.concatenate([X_all] + X_other_seq_arrays, axis=1)\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#chunked-versions","title":"Chunked versions","text":"<p>Chunked versions of the preprocessors are also available. These are useful when the data is too big to fit in memory. See also the <code>load_from_folder</code> module in the library and the corresponding section here in the documentation.</p> <p>Note that there is not a <code>ChunkImagePreprocessor</code>. This is because the processing of the images will occur inside the <code>ImageFromFolder</code> class in the <code>load_from_folder</code> module.</p>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.wide_preprocessor.ChunkWidePreprocessor","title":"ChunkWidePreprocessor","text":"<p>               Bases: <code>WidePreprocessor</code></p> <p>Preprocessor to prepare the wide input dataset</p> <p>This Preprocessor prepares the data for the wide, linear component. This linear model is implemented via an Embedding layer that is connected to the output neuron. <code>ChunkWidePreprocessor</code> numerically encodes all the unique values of all categorical columns <code>wide_cols + crossed_cols</code>. See the Example below.</p> <p>Parameters:</p> Name Type Description Default <code>wide_cols</code> <code>List[str]</code> <p>List of strings with the name of the columns that will label encoded and passed through the <code>wide</code> component</p> required <code>crossed_cols</code> <code>Optional[List[Tuple[str, str]]]</code> <p>List of Tuples with the name of the columns that will be <code>'crossed'</code> and then label encoded. e.g. [('education', 'occupation'), ...]. For binary features, a cross-product transformation is 1 if and only if the constituent features are all 1, and 0 otherwise.</p> <code>None</code> <p>Attributes:</p> Name Type Description <code>wide_crossed_cols</code> <code>List</code> <p>List with the names of all columns that will be label encoded</p> <code>encoding_dict</code> <code>Dict</code> <p>Dictionary where the keys are the result of pasting <code>colname + '_' + column value</code> and the values are the corresponding mapped integer.</p> <code>inverse_encoding_dict</code> <code>Dict</code> <p>the inverse encoding dictionary</p> <code>wide_dim</code> <code>int</code> <p>Dimension of the wide model (i.e. dim of the linear layer)</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import pandas as pd\n&gt;&gt;&gt; from pytorch_widedeep.preprocessing import ChunkWidePreprocessor\n&gt;&gt;&gt; chunk = pd.DataFrame({'color': ['r', 'b', 'g'], 'size': ['s', 'n', 'l']})\n&gt;&gt;&gt; wide_cols = ['color']\n&gt;&gt;&gt; crossed_cols = [('color', 'size')]\n&gt;&gt;&gt; chunk_wide_preprocessor = ChunkWidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols,\n... n_chunks=1)\n&gt;&gt;&gt; X_wide = chunk_wide_preprocessor.fit_transform(chunk)\n</code></pre> Source code in <code>pytorch_widedeep/preprocessing/wide_preprocessor.py</code> <pre><code>class ChunkWidePreprocessor(WidePreprocessor):\n    r\"\"\"Preprocessor to prepare the wide input dataset\n\n    This Preprocessor prepares the data for the wide, linear component.\n    This linear model is implemented via an Embedding layer that is\n    connected to the output neuron. `ChunkWidePreprocessor` numerically\n    encodes all the unique values of all categorical columns `wide_cols +\n    crossed_cols`. See the Example below.\n\n    Parameters\n    ----------\n    wide_cols: List\n        List of strings with the name of the columns that will label\n        encoded and passed through the `wide` component\n    crossed_cols: List, default = None\n        List of Tuples with the name of the columns that will be `'crossed'`\n        and then label encoded. e.g. _[('education', 'occupation'), ...]_. For\n        binary features, a cross-product transformation is 1 if and only if\n        the constituent features are all 1, and 0 otherwise.\n\n    Attributes\n    ----------\n    wide_crossed_cols: List\n        List with the names of all columns that will be label encoded\n    encoding_dict: Dict\n        Dictionary where the keys are the result of pasting `colname + '_' +\n        column value` and the values are the corresponding mapped integer.\n    inverse_encoding_dict: Dict\n        the inverse encoding dictionary\n    wide_dim: int\n        Dimension of the wide model (i.e. dim of the linear layer)\n\n    Examples\n    --------\n    &gt;&gt;&gt; import pandas as pd\n    &gt;&gt;&gt; from pytorch_widedeep.preprocessing import ChunkWidePreprocessor\n    &gt;&gt;&gt; chunk = pd.DataFrame({'color': ['r', 'b', 'g'], 'size': ['s', 'n', 'l']})\n    &gt;&gt;&gt; wide_cols = ['color']\n    &gt;&gt;&gt; crossed_cols = [('color', 'size')]\n    &gt;&gt;&gt; chunk_wide_preprocessor = ChunkWidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols,\n    ... n_chunks=1)\n    &gt;&gt;&gt; X_wide = chunk_wide_preprocessor.fit_transform(chunk)\n    \"\"\"\n\n    def __init__(\n        self,\n        wide_cols: List[str],\n        n_chunks: int,\n        crossed_cols: Optional[List[Tuple[str, str]]] = None,\n    ):\n        super(ChunkWidePreprocessor, self).__init__(wide_cols, crossed_cols)\n\n        self.n_chunks = n_chunks\n\n        self.chunk_counter = 0\n\n        self.is_fitted = False\n\n    def partial_fit(self, chunk: pd.DataFrame) -&gt; \"ChunkWidePreprocessor\":\n        r\"\"\"Fits the Preprocessor and creates required attributes\n\n        Parameters\n        ----------\n        chunk: pd.DataFrame\n            Input pandas dataframe\n\n        Returns\n        -------\n        ChunkWidePreprocessor\n            `ChunkWidePreprocessor` fitted object\n        \"\"\"\n        df_wide = self._prepare_wide(chunk)\n        self.wide_crossed_cols = df_wide.columns.tolist()\n\n        if self.chunk_counter == 0:\n            self.glob_feature_set = set(\n                self._make_global_feature_list(df_wide[self.wide_crossed_cols])\n            )\n        else:\n            self.glob_feature_set.update(\n                self._make_global_feature_list(df_wide[self.wide_crossed_cols])\n            )\n\n        self.chunk_counter += 1\n\n        if self.chunk_counter == self.n_chunks:\n            self.encoding_dict = {v: i + 1 for i, v in enumerate(self.glob_feature_set)}\n            self.wide_dim = len(self.encoding_dict)\n            self.inverse_encoding_dict = {k: v for v, k in self.encoding_dict.items()}\n            self.inverse_encoding_dict[0] = \"unseen\"\n\n            self.is_fitted = True\n\n        return self\n\n    def fit(self, df: pd.DataFrame) -&gt; \"ChunkWidePreprocessor\":\n        \"\"\"\n        Runs `partial_fit`. This is just to override the fit method in the base\n        class. This class is not designed or thought to run fit\n        \"\"\"\n        return self.partial_fit(df)\n\n    def __repr__(self) -&gt; str:\n        list_of_params: List[str] = [\"wide_cols={wide_cols}\"]\n        list_of_params.append(\"n_chunks={n_chunks}\")\n        if self.crossed_cols is not None:\n            list_of_params.append(\"crossed_cols={crossed_cols}\")\n        all_params = \", \".join(list_of_params)\n        return f\"WidePreprocessor({all_params.format(**self.__dict__)})\"\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.wide_preprocessor.ChunkWidePreprocessor.partial_fit","title":"partial_fit","text":"<pre><code>partial_fit(chunk)\n</code></pre> <p>Fits the Preprocessor and creates required attributes</p> <p>Parameters:</p> Name Type Description Default <code>chunk</code> <code>DataFrame</code> <p>Input pandas dataframe</p> required <p>Returns:</p> Type Description <code>ChunkWidePreprocessor</code> <p><code>ChunkWidePreprocessor</code> fitted object</p> Source code in <code>pytorch_widedeep/preprocessing/wide_preprocessor.py</code> <pre><code>def partial_fit(self, chunk: pd.DataFrame) -&gt; \"ChunkWidePreprocessor\":\n    r\"\"\"Fits the Preprocessor and creates required attributes\n\n    Parameters\n    ----------\n    chunk: pd.DataFrame\n        Input pandas dataframe\n\n    Returns\n    -------\n    ChunkWidePreprocessor\n        `ChunkWidePreprocessor` fitted object\n    \"\"\"\n    df_wide = self._prepare_wide(chunk)\n    self.wide_crossed_cols = df_wide.columns.tolist()\n\n    if self.chunk_counter == 0:\n        self.glob_feature_set = set(\n            self._make_global_feature_list(df_wide[self.wide_crossed_cols])\n        )\n    else:\n        self.glob_feature_set.update(\n            self._make_global_feature_list(df_wide[self.wide_crossed_cols])\n        )\n\n    self.chunk_counter += 1\n\n    if self.chunk_counter == self.n_chunks:\n        self.encoding_dict = {v: i + 1 for i, v in enumerate(self.glob_feature_set)}\n        self.wide_dim = len(self.encoding_dict)\n        self.inverse_encoding_dict = {k: v for v, k in self.encoding_dict.items()}\n        self.inverse_encoding_dict[0] = \"unseen\"\n\n        self.is_fitted = True\n\n    return self\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.wide_preprocessor.ChunkWidePreprocessor.fit","title":"fit","text":"<pre><code>fit(df)\n</code></pre> <p>Runs <code>partial_fit</code>. This is just to override the fit method in the base class. This class is not designed or thought to run fit</p> Source code in <code>pytorch_widedeep/preprocessing/wide_preprocessor.py</code> <pre><code>def fit(self, df: pd.DataFrame) -&gt; \"ChunkWidePreprocessor\":\n    \"\"\"\n    Runs `partial_fit`. This is just to override the fit method in the base\n    class. This class is not designed or thought to run fit\n    \"\"\"\n    return self.partial_fit(df)\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.tab_preprocessor.ChunkTabPreprocessor","title":"ChunkTabPreprocessor","text":"<p>               Bases: <code>TabPreprocessor</code></p> <p>Preprocessor to prepare the <code>deeptabular</code> component input dataset</p> <p>Parameters:</p> Name Type Description Default <code>n_chunks</code> <code>int</code> <p>Number of chunks that the tabular dataset is divided by.</p> required <code>cat_embed_cols</code> <code>Optional[Union[List[str], List[Tuple[str, int]]]]</code> <p>List containing the name of the categorical columns that will be represented by embeddings (e.g. ['education', 'relationship', ...]) or a Tuple with the name and the embedding dimension (e.g.: [ ('education',32), ('relationship',16), ...]).  Param alias: <code>embed_cols</code></p> <code>None</code> <code>continuous_cols</code> <code>Optional[List[str]]</code> <p>List with the name of the continuous cols</p> <code>None</code> <code>cols_and_bins</code> <code>Optional[Dict[str, List[float]]]</code> <p>Continuous columns can be turned into categorical via <code>pd.cut</code>. 'cols_and_bins' is dictionary where the keys are the column names to quantize and the values are a list of scalars indicating the bin edges.  Param alias: <code>quantization_setup</code></p> <code>None</code> <code>cols_to_scale</code> <code>Optional[Union[List[str], str]]</code> <p>List with the names of the columns that will be standarised via sklearn's <code>StandardScaler</code></p> <code>None</code> <code>default_embed_dim</code> <code>int</code> <p>Dimension for the embeddings if the embed_dim is not provided in the <code>cat_embed_cols</code> parameter and <code>auto_embed_dim</code> is set to <code>False</code>.</p> <code>16</code> <code>with_attention</code> <code>bool</code> <p>Boolean indicating whether the preprocessed data will be passed to an attention-based model (more precisely a model where all embeddings must have the same dimensions). If <code>True</code>, the param <code>cat_embed_cols</code> must just be a list containing just the categorical column names: e.g. ['education', 'relationship', ...]. This is because they will all be  encoded using embeddings of the same dim, which will be specified  later when the model is defined.  Param alias:  <code>for_transformer</code></p> <code>False</code> <code>with_cls_token</code> <code>bool</code> <p>Boolean indicating if a <code>'[CLS]'</code> token will be added to the dataset when using attention-based models. The final hidden state corresponding to this token is used as the aggregated representation for classification and regression tasks. If not, the categorical (and continuous embeddings if present) will be concatenated before being passed to the final MLP (if present).</p> <code>False</code> <code>shared_embed</code> <code>bool</code> <p>Boolean indicating if the embeddings will be \"shared\" when using attention-based models. The idea behind <code>shared_embed</code> is described in the Appendix A in the TabTransformer paper: 'The goal of having column embedding is to enable the model to distinguish the classes in one column from those in the other columns'. In other words, the idea is to let the model learn which column is embedded at the time. See: <code>pytorch_widedeep.models.transformers._layers.SharedEmbeddings</code>.</p> <code>False</code> <code>verbose</code> <code>int</code> <code>1</code> <code>scale</code> <code>bool</code> <p> note: this arg will be removed in upcoming  releases. Please use <code>cols_to_scale</code> instead.  Bool indicating  whether or not to scale/standarise continuous cols. It is important  to emphasize that all the DL models for tabular data in the library  also include the possibility of normalising the input continuous  features via a <code>BatchNorm</code> or a <code>LayerNorm</code>.  Param alias:  <code>scale_cont_cols</code>.</p> <code>False</code> <code>already_standard</code> <code>Optional[List[str]]</code> <p> note: this arg will be removed in upcoming  releases. Please use <code>cols_to_scale</code> instead.  List with the  name of the continuous cols that do not need to be  scaled/standarised.</p> <code>None</code> <p>Other Parameters:</p> Name Type Description <code>**kwargs</code> <p><code>pd.cut</code> and <code>StandardScaler</code> related args</p> <p>Attributes:</p> Name Type Description <code>cat_cols</code> <code>List[str]</code> <p>List containing the names of the categorical columns after processing.</p> <code>cols_and_bins</code> <code>Optional[Dict[str, Union[int, List[float]]]]</code> <p>Dictionary containing the quantization setup after processing if quantization is requested. This is derived from the quantization_setup parameter.</p> <code>quant_args</code> <code>Dict</code> <p>Dictionary containing arguments passed to pandas.cut() function for quantization.</p> <code>scale_args</code> <code>Dict</code> <p>Dictionary containing arguments passed to StandardScaler.</p> <code>label_encoder</code> <code>LabelEncoder</code> <p>Instance of LabelEncoder used to encode categorical variables. See <code>pytorch_widedeep.utils.dense_utils.LabelEncoder</code>.</p> <code>cat_embed_input</code> <code>List</code> <p>List of tuples with the column name, number of individual values for that column and, if <code>with_attention</code> is set to <code>False</code>, the corresponding embeddings dim, e.g. [('education', 16, 10), ('relationship', 6, 8), ...].</p> <code>standardize_cols</code> <code>List</code> <p>List of the columns that will be standardized.</p> <code>scaler</code> <code>StandardScaler</code> <p>An instance of <code>sklearn.preprocessing.StandardScaler</code> if standardization is requested.</p> <code>column_idx</code> <code>Dict</code> <p>Dictionary where keys are column names and values are column indexes. This is necessary to slice tensors.</p> <code>quantizer</code> <code>Quantizer</code> <p>An instance of <code>Quantizer</code> if quantization is requested.</p> <code>is_fitted</code> <code>bool</code> <p>Boolean indicating if the preprocessor has been fitted.</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import pandas as pd\n&gt;&gt;&gt; import numpy as np\n&gt;&gt;&gt; from pytorch_widedeep.preprocessing import ChunkTabPreprocessor\n&gt;&gt;&gt; np.random.seed(42)\n&gt;&gt;&gt; chunk_df = pd.DataFrame({'cat_col': np.random.choice(['A', 'B', 'C'], size=8),\n... 'cont_col': np.random.uniform(1, 100, size=8)})\n&gt;&gt;&gt; cat_embed_cols = [('cat_col',4)]\n&gt;&gt;&gt; cont_cols = ['cont_col']\n&gt;&gt;&gt; tab_preprocessor = ChunkTabPreprocessor(\n... n_chunks=1, cat_embed_cols=cat_embed_cols, continuous_cols=cont_cols\n... )\n&gt;&gt;&gt; X_tab = tab_preprocessor.fit_transform(chunk_df)\n&gt;&gt;&gt; tab_preprocessor.cat_embed_cols\n[('cat_col', 4)]\n&gt;&gt;&gt; tab_preprocessor.column_idx\n{'cat_col': 0, 'cont_col': 1}\n</code></pre> Source code in <code>pytorch_widedeep/preprocessing/tab_preprocessor.py</code> <pre><code>class ChunkTabPreprocessor(TabPreprocessor):\n    r\"\"\"Preprocessor to prepare the `deeptabular` component input dataset\n\n    Parameters\n    ----------\n    n_chunks: int\n        Number of chunks that the tabular dataset is divided by.\n    cat_embed_cols: List, default = None\n        List containing the name of the categorical columns that will be\n        represented by embeddings (e.g. _['education', 'relationship', ...]_) or\n        a Tuple with the name and the embedding dimension (e.g.: _[\n        ('education',32), ('relationship',16), ...]_). &lt;br/&gt; Param alias:\n        `embed_cols`\n    continuous_cols: List, default = None\n        List with the name of the continuous cols\n    cols_and_bins: Dict, default = None\n        Continuous columns can be turned into categorical via\n        `pd.cut`. 'cols_and_bins' is dictionary where the keys are the column\n        names to quantize and the values are a list of scalars indicating the\n        bin edges. &lt;br/&gt; Param alias: `quantization_setup`\n    cols_to_scale: List, default = None,\n        List with the names of the columns that will be standarised via\n        sklearn's `StandardScaler`\n    default_embed_dim: int, default=16\n        Dimension for the embeddings if the embed_dim is not provided in the\n        `cat_embed_cols` parameter and `auto_embed_dim` is set to\n        `False`.\n    with_attention: bool, default = False\n        Boolean indicating whether the preprocessed data will be passed to an\n        attention-based model (more precisely a model where all embeddings\n        must have the same dimensions). If `True`, the param `cat_embed_cols`\n        must just be a list containing just the categorical column names:\n        e.g.\n        _['education', 'relationship', ...]_. This is because they will all be\n         encoded using embeddings of the same dim, which will be specified\n         later when the model is defined. &lt;br/&gt; Param alias:\n         `for_transformer`\n    with_cls_token: bool, default = False\n        Boolean indicating if a `'[CLS]'` token will be added to the dataset\n        when using attention-based models. The final hidden state\n        corresponding to this token is used as the aggregated representation\n        for classification and regression tasks. If not, the categorical\n        (and continuous embeddings if present) will be concatenated before\n        being passed to the final MLP (if present).\n    shared_embed: bool, default = False\n        Boolean indicating if the embeddings will be \"shared\" when using\n        attention-based models. The idea behind `shared_embed` is\n        described in the Appendix A in the [TabTransformer paper](https://arxiv.org/abs/2012.06678):\n        _'The goal of having column embedding is to enable the model to\n        distinguish the classes in one column from those in the other\n        columns'_. In other words, the idea is to let the model learn which\n        column is embedded at the time. See: `pytorch_widedeep.models.transformers._layers.SharedEmbeddings`.\n    verbose: int, default = 1\n    scale: bool, default = False\n        :information_source: **note**: this arg will be removed in upcoming\n         releases. Please use `cols_to_scale` instead. &lt;br/&gt; Bool indicating\n         whether or not to scale/standarise continuous cols. It is important\n         to emphasize that all the DL models for tabular data in the library\n         also include the possibility of normalising the input continuous\n         features via a `BatchNorm` or a `LayerNorm`. &lt;br/&gt; Param alias:\n         `scale_cont_cols`.\n    already_standard: List, default = None\n        :information_source: **note**: this arg will be removed in upcoming\n         releases. Please use `cols_to_scale` instead. &lt;br/&gt; List with the\n         name of the continuous cols that do not need to be\n         scaled/standarised.\n\n    Other Parameters\n    ----------------\n    **kwargs: dict\n        `pd.cut` and `StandardScaler` related args\n\n    Attributes\n    ----------\n    cat_cols: List[str]\n        List containing the names of the categorical columns after processing.\n    cols_and_bins: Optional[Dict[str, Union[int, List[float]]]]\n        Dictionary containing the quantization setup after processing if\n        quantization is requested. This is derived from the\n        quantization_setup parameter.\n    quant_args: Dict\n        Dictionary containing arguments passed to pandas.cut() function for quantization.\n    scale_args: Dict\n        Dictionary containing arguments passed to StandardScaler.\n    label_encoder: LabelEncoder\n        Instance of LabelEncoder used to encode categorical variables.\n        See `pytorch_widedeep.utils.dense_utils.LabelEncoder`.\n    cat_embed_input: List\n        List of tuples with the column name, number of individual values for\n        that column and, if `with_attention` is set to `False`, the\n        corresponding embeddings dim, e.g. _[('education', 16, 10),\n        ('relationship', 6, 8), ...]_.\n    standardize_cols: List\n        List of the columns that will be standardized.\n    scaler: StandardScaler\n        An instance of `sklearn.preprocessing.StandardScaler` if standardization\n        is requested.\n    column_idx: Dict\n        Dictionary where keys are column names and values are column indexes.\n        This is necessary to slice tensors.\n    quantizer: Quantizer\n        An instance of `Quantizer` if quantization is requested.\n    is_fitted: bool\n        Boolean indicating if the preprocessor has been fitted.\n\n    Examples\n    --------\n    &gt;&gt;&gt; import pandas as pd\n    &gt;&gt;&gt; import numpy as np\n    &gt;&gt;&gt; from pytorch_widedeep.preprocessing import ChunkTabPreprocessor\n    &gt;&gt;&gt; np.random.seed(42)\n    &gt;&gt;&gt; chunk_df = pd.DataFrame({'cat_col': np.random.choice(['A', 'B', 'C'], size=8),\n    ... 'cont_col': np.random.uniform(1, 100, size=8)})\n    &gt;&gt;&gt; cat_embed_cols = [('cat_col',4)]\n    &gt;&gt;&gt; cont_cols = ['cont_col']\n    &gt;&gt;&gt; tab_preprocessor = ChunkTabPreprocessor(\n    ... n_chunks=1, cat_embed_cols=cat_embed_cols, continuous_cols=cont_cols\n    ... )\n    &gt;&gt;&gt; X_tab = tab_preprocessor.fit_transform(chunk_df)\n    &gt;&gt;&gt; tab_preprocessor.cat_embed_cols\n    [('cat_col', 4)]\n    &gt;&gt;&gt; tab_preprocessor.column_idx\n    {'cat_col': 0, 'cont_col': 1}\n    \"\"\"\n\n    @alias(\"with_attention\", [\"for_transformer\"])\n    @alias(\"cat_embed_cols\", [\"embed_cols\"])\n    @alias(\"scale\", [\"scale_cont_cols\"])\n    @alias(\"cols_and_bins\", [\"quantization_setup\"])\n    def __init__(\n        self,\n        n_chunks: int,\n        cat_embed_cols: Optional[Union[List[str], List[Tuple[str, int]]]] = None,\n        continuous_cols: Optional[List[str]] = None,\n        cols_and_bins: Optional[Dict[str, List[float]]] = None,\n        cols_to_scale: Optional[Union[List[str], str]] = None,\n        default_embed_dim: int = 16,\n        with_attention: bool = False,\n        with_cls_token: bool = False,\n        shared_embed: bool = False,\n        verbose: int = 1,\n        *,\n        scale: bool = False,\n        already_standard: Optional[List[str]] = None,\n        **kwargs,\n    ):\n        super(ChunkTabPreprocessor, self).__init__(\n            cat_embed_cols=cat_embed_cols,\n            continuous_cols=continuous_cols,\n            quantization_setup=None,\n            cols_to_scale=cols_to_scale,\n            auto_embed_dim=False,\n            embedding_rule=\"google\",  # does not matter, irrelevant\n            default_embed_dim=default_embed_dim,\n            with_attention=with_attention,\n            with_cls_token=with_cls_token,\n            shared_embed=shared_embed,\n            verbose=verbose,\n            scale=scale,\n            already_standard=already_standard,\n            **kwargs,\n        )\n\n        self.n_chunks = n_chunks\n        self.chunk_counter = 0\n\n        self.cols_and_bins = cols_and_bins  # type: ignore[assignment]\n        if self.cols_and_bins is not None:\n            self.quantizer = Quantizer(self.cols_and_bins, **self.quant_args)\n\n        self.embed_prepared = False\n        self.continuous_prepared = False\n\n    def partial_fit(self, df: pd.DataFrame) -&gt; \"ChunkTabPreprocessor\":  # noqa: C901\n        # df here, and throughout the class, is a chunk of the original df\n        self.chunk_counter += 1\n\n        df_adj = self._insert_cls_token(df) if self.with_cls_token else df.copy()\n\n        self.column_idx: Dict[str, int] = {}\n\n        # Categorical embeddings logic\n        if self.cat_embed_cols is not None:\n            if not self.embed_prepared:\n                df_cat, self.cat_embed_dim = self._prepare_categorical(df_adj)\n                self.label_encoder = LabelEncoder(\n                    columns_to_encode=df_cat.columns.tolist(),\n                    shared_embed=self.shared_embed,\n                    with_attention=self.with_attention,\n                )\n                self.label_encoder.partial_fit(df_cat)\n            else:\n                df_cat = df_adj[self.cat_cols]\n                self.label_encoder.partial_fit(df_cat)\n\n            self.column_idx.update({k: v for v, k in enumerate(df_cat.columns)})\n\n        # Continuous columns logic\n        if self.continuous_cols is not None:\n            if not self.continuous_prepared:\n                df_cont, self.cont_embed_dim = self._prepare_continuous(df_adj)\n            else:\n                df_cont = df[self.continuous_cols]\n\n            self.column_idx.update(\n                {k: v + len(self.column_idx) for v, k in enumerate(df_cont.columns)}\n            )\n\n            if self.standardize_cols is not None:\n                self.scaler.partial_fit(df_cont[self.standardize_cols].values)\n\n        if self.chunk_counter == self.n_chunks:\n            if self.cat_embed_cols is not None or self.cols_and_bins is not None:\n                self.cat_embed_input: List[\n                    Union[Tuple[str, int], Tuple[str, int, int]]\n                ] = []\n\n            if self.cat_embed_cols is not None:\n                for k, v in self.label_encoder.encoding_dict.items():\n                    if self.with_attention:\n                        self.cat_embed_input.append((k, len(v)))\n                    else:\n                        self.cat_embed_input.append((k, len(v), self.cat_embed_dim[k]))\n\n            if self.cols_and_bins is not None:\n                assert self.cont_embed_dim is not None  # just to make mypy happy\n                if self.with_attention:\n                    for col, n_cat, _ in self.cont_embed_dim:\n                        self.cat_embed_input.append((col, n_cat))\n                else:\n                    self.cat_embed_input.extend(self.cont_embed_dim)\n\n            self.is_fitted = True\n\n        return self\n\n    def fit(self, df: pd.DataFrame) -&gt; \"ChunkTabPreprocessor\":\n        # just to override the fit method in the base class. This class is not\n        # designed or thought to run fit\n        return self.partial_fit(df)\n\n    def _prepare_categorical(\n        self, df: pd.DataFrame\n    ) -&gt; Tuple[pd.DataFrame, Dict[str, int]]:\n        # When dealing with chunks we will NOT support the option of\n        # automatically define embeddings as this implies going through the\n        # entire dataset\n        if isinstance(self.cat_embed_cols[0], tuple):\n            self.cat_cols: List[str] = [emb[0] for emb in self.cat_embed_cols]\n            cat_embed_dim: Dict[str, int] = dict(self.cat_embed_cols)  # type: ignore\n        else:\n            self.cat_cols = self.cat_embed_cols  # type: ignore[assignment]\n            cat_embed_dim = {\n                e: self.default_embed_dim for e in self.cat_embed_cols  # type: ignore[misc]\n            }  # type: ignore\n\n        self.embed_prepared = True\n\n        return df[self.cat_cols], cat_embed_dim\n\n    def _prepare_continuous(\n        self, df: pd.DataFrame\n    ) -&gt; Tuple[pd.DataFrame, Optional[List[Tuple[str, int, int]]]]:\n        # Standardization logic\n        if self.cols_to_scale is not None:\n            self.standardize_cols = (\n                self.cols_to_scale\n                if self.cols_to_scale != \"all\"\n                else self.continuous_cols\n            )\n        elif self.scale:\n            if self.already_standard is not None:\n                self.standardize_cols = [\n                    c\n                    for c in self.continuous_cols  # type: ignore[misc]\n                    if c not in self.already_standard\n                ]\n            else:\n                self.standardize_cols = self.continuous_cols\n        else:\n            self.standardize_cols = None\n\n        if self.standardize_cols is not None:\n            self.scaler = StandardScaler(**self.scale_args)\n        elif self.verbose:\n            warnings.warn(\"Continuous columns will not be normalised\")\n\n        # Quantization logic\n        if self.cols_and_bins is not None:\n            # the quantized columns are then treated as categorical\n            quant_cont_embed_input: Optional[List[Tuple[str, int, int]]] = []\n            for col, val in self.cols_and_bins.items():\n                if isinstance(val, int):\n                    quant_cont_embed_input.append(\n                        (\n                            col,\n                            val,\n                            self.default_embed_dim,\n                        )\n                    )\n                else:\n                    quant_cont_embed_input.append(\n                        (\n                            col,\n                            len(val) - 1,\n                            self.default_embed_dim,\n                        )\n                    )\n        else:\n            quant_cont_embed_input = None\n\n        self.continuous_prepared = True\n\n        return df[self.continuous_cols], quant_cont_embed_input\n\n    def __repr__(self) -&gt; str:  # noqa: C901\n        list_of_params: List[str] = []\n        if self.n_chunks is not None:\n            list_of_params.append(\"n_chunks={n_chunks}\")\n        if self.cat_embed_cols is not None:\n            list_of_params.append(\"cat_embed_cols={cat_embed_cols}\")\n        if self.continuous_cols is not None:\n            list_of_params.append(\"continuous_cols={continuous_cols}\")\n        if self.cols_and_bins is not None:\n            list_of_params.append(\"cols_and_bins={cols_and_bins}\")\n        if self.cols_to_scale is not None:\n            list_of_params.append(\"cols_to_scale={cols_to_scale}\")\n        if self.default_embed_dim != 16:\n            list_of_params.append(\"default_embed_dim={default_embed_dim}\")\n        if self.with_attention:\n            list_of_params.append(\"with_attention={with_attention}\")\n        if self.with_cls_token:\n            list_of_params.append(\"with_cls_token={with_cls_token}\")\n        if self.shared_embed:\n            list_of_params.append(\"shared_embed={shared_embed}\")\n        if self.verbose != 1:\n            list_of_params.append(\"verbose={verbose}\")\n        if self.scale:\n            list_of_params.append(\"scale={scale}\")\n        if self.already_standard is not None:\n            list_of_params.append(\"already_standard={already_standard}\")\n        if len(self.quant_args) &gt; 0:\n            list_of_params.append(\n                \", \".join([f\"{k}\" + \"=\" + f\"{v}\" for k, v in self.quant_args.items()])\n            )\n        if len(self.scale_args) &gt; 0:\n            list_of_params.append(\n                \", \".join([f\"{k}\" + \"=\" + f\"{v}\" for k, v in self.scale_args.items()])\n            )\n        all_params = \", \".join(list_of_params)\n        return f\"ChunkTabPreprocessor({all_params.format(**self.__dict__)})\"\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.text_preprocessor.ChunkTextPreprocessor","title":"ChunkTextPreprocessor","text":"<p>               Bases: <code>TextPreprocessor</code></p> <p>Preprocessor to prepare the <code>deeptext</code> input dataset</p> <p>Parameters:</p> Name Type Description Default <code>text_col</code> <code>str</code> <p>column in the input dataframe containing either the texts or the filenames where the text documents are stored</p> required <code>n_chunks</code> <code>int</code> <p>Number of chunks that the text dataset is divided by.</p> required <code>root_dir</code> <code>Optional[str]</code> <p>If 'text_col' contains the filenames with the text documents, this is the path to the directory where those documents are stored.</p> <code>None</code> <code>max_vocab</code> <code>int</code> <p>Maximum number of tokens in the vocabulary</p> <code>30000</code> <code>min_freq</code> <code>int</code> <p>Minimum frequency for a token to be part of the vocabulary</p> <code>5</code> <code>maxlen</code> <code>int</code> <p>Maximum length of the tokenized sequences</p> <code>80</code> <code>pad_first</code> <code>bool</code> <p>Indicates whether the padding index will be added at the beginning or the end of the sequences</p> <code>True</code> <code>pad_idx</code> <code>int</code> <p>padding index. Fastai's Tokenizer leaves 0 for the 'unknown' token.</p> <code>1</code> <code>word_vectors_path</code> <code>Optional[str]</code> <p>Path to the pretrained word vectors</p> <code>None</code> <code>n_cpus</code> <code>Optional[int]</code> <p>number of CPUs to used during the tokenization process</p> <code>None</code> <code>verbose</code> <code>int</code> <p>Enable verbose output.</p> <code>1</code> <p>Attributes:</p> Name Type Description <code>vocab</code> <code>Vocab</code> <p>an instance of <code>pytorch_widedeep.utils.fastai_transforms.ChunkVocab</code></p> <code>embedding_matrix</code> <code>ndarray</code> <p>Array with the pretrained embeddings if <code>word_vectors_path</code> is not None</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import pandas as pd\n&gt;&gt;&gt; from pytorch_widedeep.preprocessing import ChunkTextPreprocessor\n&gt;&gt;&gt; chunk_df = pd.DataFrame({'text_column': [\"life is like a box of chocolates\",\n... \"You never know what you're gonna get\"]})\n&gt;&gt;&gt; chunk_text_preprocessor = ChunkTextPreprocessor(text_col='text_column', n_chunks=1,\n... max_vocab=25, min_freq=1, maxlen=10, verbose=0, n_cpus=1)\n&gt;&gt;&gt; processed_chunk = chunk_text_preprocessor.fit_transform(chunk_df)\n</code></pre> Source code in <code>pytorch_widedeep/preprocessing/text_preprocessor.py</code> <pre><code>class ChunkTextPreprocessor(TextPreprocessor):\n    r\"\"\"Preprocessor to prepare the ``deeptext`` input dataset\n\n    Parameters\n    ----------\n    text_col: str\n        column in the input dataframe containing either the texts or the\n        filenames where the text documents are stored\n    n_chunks: int\n        Number of chunks that the text dataset is divided by.\n    root_dir: str, Optional, default = None\n        If 'text_col' contains the filenames with the text documents, this is\n        the path to the directory where those documents are stored.\n    max_vocab: int, default=30000\n        Maximum number of tokens in the vocabulary\n    min_freq: int, default=5\n        Minimum frequency for a token to be part of the vocabulary\n    maxlen: int, default=80\n        Maximum length of the tokenized sequences\n    pad_first: bool,  default = True\n        Indicates whether the padding index will be added at the beginning or the\n        end of the sequences\n    pad_idx: int, default = 1\n        padding index. Fastai's Tokenizer leaves 0 for the 'unknown' token.\n    word_vectors_path: str, Optional\n        Path to the pretrained word vectors\n    n_cpus: int, Optional, default = None\n        number of CPUs to used during the tokenization process\n    verbose: int, default 1\n        Enable verbose output.\n\n    Attributes\n    ----------\n    vocab: Vocab\n        an instance of `pytorch_widedeep.utils.fastai_transforms.ChunkVocab`\n    embedding_matrix: np.ndarray\n        Array with the pretrained embeddings if `word_vectors_path` is not None\n\n    Examples\n    ---------\n    &gt;&gt;&gt; import pandas as pd\n    &gt;&gt;&gt; from pytorch_widedeep.preprocessing import ChunkTextPreprocessor\n    &gt;&gt;&gt; chunk_df = pd.DataFrame({'text_column': [\"life is like a box of chocolates\",\n    ... \"You never know what you're gonna get\"]})\n    &gt;&gt;&gt; chunk_text_preprocessor = ChunkTextPreprocessor(text_col='text_column', n_chunks=1,\n    ... max_vocab=25, min_freq=1, maxlen=10, verbose=0, n_cpus=1)\n    &gt;&gt;&gt; processed_chunk = chunk_text_preprocessor.fit_transform(chunk_df)\n    \"\"\"\n\n    def __init__(\n        self,\n        text_col: str,\n        n_chunks: int,\n        root_dir: Optional[str] = None,\n        max_vocab: int = 30000,\n        min_freq: int = 5,\n        maxlen: int = 80,\n        pad_first: bool = True,\n        pad_idx: int = 1,\n        already_processed: Optional[bool] = False,\n        word_vectors_path: Optional[str] = None,\n        n_cpus: Optional[int] = None,\n        verbose: int = 1,\n    ):\n        super(ChunkTextPreprocessor, self).__init__(\n            text_col=text_col,\n            max_vocab=max_vocab,\n            min_freq=min_freq,\n            maxlen=maxlen,\n            pad_first=pad_first,\n            pad_idx=pad_idx,\n            already_processed=already_processed,\n            word_vectors_path=word_vectors_path,\n            n_cpus=n_cpus,\n            verbose=verbose,\n        )\n\n        self.n_chunks = n_chunks\n        self.root_dir = root_dir\n\n        self.chunk_counter = 0\n\n        self.is_fitted = False\n\n    def partial_fit(self, df: pd.DataFrame) -&gt; \"ChunkTextPreprocessor\":\n        # df is a chunk of the original dataframe\n        self.chunk_counter += 1\n\n        texts = self._read_texts(df, self.root_dir)\n\n        tokens = get_texts(texts, self.already_processed, self.n_cpus)\n\n        if not hasattr(self, \"vocab\"):\n            self.vocab = ChunkVocab(\n                max_vocab=self.max_vocab,\n                min_freq=self.min_freq,\n                pad_idx=self.pad_idx,\n                n_chunks=self.n_chunks,\n            )\n\n        self.vocab.fit(tokens)\n\n        if self.chunk_counter == self.n_chunks:\n            if self.verbose:\n                print(\"The vocabulary contains {} tokens\".format(len(self.vocab.stoi)))\n            if self.word_vectors_path is not None:\n                self.embedding_matrix = build_embeddings_matrix(\n                    self.vocab, self.word_vectors_path, self.min_freq\n                )\n\n            self.is_fitted = True\n\n        return self\n\n    def fit(self, df: pd.DataFrame) -&gt; \"ChunkTextPreprocessor\":\n        # df is a chunk of the original dataframe\n        return self.partial_fit(df)\n\n    def __repr__(self) -&gt; str:\n        list_of_params: List[str] = [\"text_col='{text_col}'\"]\n        if self.n_chunks is not None:\n            list_of_params.append(\"n_chunks={n_chunks}\")\n        if self.root_dir is not None:\n            list_of_params.append(\"root_dir={root_dir}\")\n        list_of_params.append(\"max_vocab={max_vocab}\")\n        list_of_params.append(\"min_freq={min_freq}\")\n        list_of_params.append(\"maxlen={maxlen}\")\n        list_of_params.append(\"pad_first={pad_first}\")\n        list_of_params.append(\"pad_idx={pad_idx}\")\n        if self.word_vectors_path is not None:\n            list_of_params.append(\"word_vectors_path={word_vectors_path}\")\n        if self.n_cpus is not None:\n            list_of_params.append(\"n_cpus={n_cpus}\")\n        list_of_params.append(\"verbose={verbose}\")\n        all_params = \", \".join(list_of_params)\n        return f\"ChunkTextPreprocessor({all_params.format(**self.__dict__)})\"\n</code></pre>"},{"location":"pytorch-widedeep/preprocessing.html#pytorch_widedeep.preprocessing.hf_preprocessor.ChunkHFPreprocessor","title":"ChunkHFPreprocessor","text":"<p>               Bases: <code>HFPreprocessor</code></p> <p>Text processor to prepare the <code>deeptext</code> input dataset that is a wrapper around HuggingFace's tokenizers.</p> <p>Hugginface Tokenizer's are already 'trained'. Therefore, unlike the <code>ChunkTextPreprocessor</code> this is mostly identical to the <code>HFPreprocessor</code> with the only difference that the class needs a 'text_col' parameter to be passed. Also the parameter <code>encode_params</code> is not really optional when using this class. It must be passed containing at least the 'max_length' encoding parameter. This is because we need to ensure that  all sequences have the same length when encoding in chunks.</p> <p>Parameters:</p> Name Type Description Default <code>model_name</code> <code>str</code> <p>The model name from the transformers library e.g. 'bert-base-uncased'. Currently supported models are those from the families: BERT, RoBERTa, DistilBERT, ALBERT and ELECTRA.</p> required <code>text_col</code> <code>str</code> <p>The column in the input dataframe containing the text data. When using the <code>ChunkHFPreprocessor</code> the <code>text_col</code> parameter is mandatory.</p> required <code>root_dir</code> <code>Optional[str]</code> <p>The root directory where the text files are located. This is only needed if the text data is stored in text files. If the text data is stored in a column in the input dataframe, this parameter is not needed.</p> <code>None</code> <code>use_fast_tokenizer</code> <code>bool</code> <p>Whether to use the fast tokenizer from HuggingFace or not</p> <code>True</code> <code>num_workers</code> <code>Optional[int]</code> <p>Number of workers to use when preprocessing the text data. If not None, and <code>use_fast_tokenizer</code> is False, the text data will be preprocessed in parallel using the number of workers specified. If <code>use_fast_tokenizer</code> is True, this argument is ignored.</p> <code>None</code> <code>preprocessing_rules</code> <code>Optional[List[Callable[[str], str]]]</code> <p>A list of functions to be applied to the text data before encoding. This can be useful to clean the text data before encoding. For example, removing html tags, special characters, etc.</p> <code>None</code> <code>tokenizer_params</code> <code>Optional[Dict[str, Any]]</code> <p>Additional parameters to be passed to the HuggingFace's <code>PreTrainedTokenizer</code>.</p> <code>None</code> <code>encode_params</code> <code>Optional[Dict[str, Any]]</code> <p>Additional parameters to be passed to the <code>batch_encode_plus</code> method of the HuggingFace's <code>PreTrainedTokenizer</code>. In the case of the <code>ChunkHFPreprocessor</code>, this parameter is not really <code>Optional</code>. It must be passed containing at least the 'max_length' encoding parameter</p> <code>None</code> <p>Attributes:</p> Name Type Description <code>is_fitted</code> <code>bool</code> <p>Boolean indicating if the preprocessor has been fitted. This is a HuggingFacea tokenizer, so it is always considered fitted and this attribute is manually set to True internally. This parameter exists for consistency with the rest of the library and because is needed for some functionality in the library.</p> Source code in <code>pytorch_widedeep/preprocessing/hf_preprocessor.py</code> <pre><code>class ChunkHFPreprocessor(HFPreprocessor):\n    \"\"\"Text processor to prepare the ``deeptext`` input dataset that is a\n    wrapper around HuggingFace's tokenizers.\n\n    Hugginface Tokenizer's are already 'trained'. Therefore, unlike the\n    `ChunkTextPreprocessor` this is mostly identical to the `HFPreprocessor`\n    with the only difference that the class needs a 'text_col' parameter to\n    be passed. Also the parameter `encode_params` is not really optional when\n    using this class. It must be passed containing at least the\n    'max_length' encoding parameter. This is because we need to ensure that\n     all sequences have the same length when encoding in chunks.\n\n    Parameters\n    ----------\n    model_name: str\n        The model name from the transformers library e.g. _'bert-base-uncased'_.\n        Currently supported models are those from the families: BERT, RoBERTa,\n        DistilBERT, ALBERT and ELECTRA.\n    text_col: str, default = None\n        The column in the input dataframe containing the text data. When using\n        the `ChunkHFPreprocessor` the `text_col` parameter is mandatory.\n    root_dir: Optional[str], default = None\n        The root directory where the text files are located. This is only\n        needed if the text data is stored in text files. If the text data is\n        stored in a column in the input dataframe, this parameter is not\n        needed.\n    use_fast_tokenizer: bool, default = False\n        Whether to use the fast tokenizer from HuggingFace or not\n    num_workers: Optional[int], default = None\n        Number of workers to use when preprocessing the text data. If not\n        None, and `use_fast_tokenizer` is False, the text data will be\n        preprocessed in parallel using the number of workers specified. If\n        `use_fast_tokenizer` is True, this argument is ignored.\n    preprocessing_rules: Optional[List[Callable[[str], str]]], default = None\n        A list of functions to be applied to the text data before encoding.\n        This can be useful to clean the text data before encoding. For\n        example, removing html tags, special characters, etc.\n    tokenizer_params: Optional[Dict[str, Any]], default = None\n        Additional parameters to be passed to the HuggingFace's\n        `PreTrainedTokenizer`.\n    encode_params: Optional[Dict[str, Any]], default = None\n        Additional parameters to be passed to the `batch_encode_plus` method\n        of the HuggingFace's `PreTrainedTokenizer`. In the case of the\n        `ChunkHFPreprocessor`, this parameter is not really `Optional`. It\n        must be passed containing at least the 'max_length' encoding\n        parameter\n\n    Attributes\n    ----------\n    is_fitted: bool\n        Boolean indicating if the preprocessor has been fitted. This is a\n        HuggingFacea tokenizer, so it is always considered fitted and this\n        attribute is manually set to True internally. This parameter exists\n        for consistency with the rest of the library and because is needed\n        for some functionality in the library.\n\n    \"\"\"\n\n    def __init__(\n        self,\n        model_name: str,\n        *,\n        text_col: str,\n        root_dir: Optional[str] = None,\n        use_fast_tokenizer: bool = True,\n        num_workers: Optional[int] = None,\n        preprocessing_rules: Optional[List[Callable[[str], str]]] = None,\n        tokenizer_params: Optional[Dict[str, Any]] = None,\n        encode_params: Optional[Dict[str, Any]] = None,\n    ):\n        super().__init__(\n            model_name=model_name,\n            use_fast_tokenizer=use_fast_tokenizer,\n            text_col=text_col,\n            num_workers=num_workers,\n            preprocessing_rules=preprocessing_rules,\n            tokenizer_params=tokenizer_params,\n            encode_params=encode_params,\n        )\n\n        self.root_dir = root_dir\n\n        # when using in chunks encode_params is not really optional. I will\n        # review types in due time\n        if self.encode_params is None:\n            raise ValueError(\n                \"The 'encode_params' dict must be passed to the ChunkHFTokenizer \"\n                \"containing at least the 'max_length' encoding parameter\"\n            )\n\n        if \"padding\" not in self.encode_params or not self.encode_params[\"padding\"]:\n            self.encode_params[\"padding\"] = True\n\n        if (\n            \"truncation\" not in self.encode_params\n            or not self.encode_params[\"truncation\"]\n        ):\n            self.encode_params[\"truncation\"] = True\n\n    def __repr__(self):\n        return (\n            f\"ChunkHFPreprocessor(text_col={self.text_col}, model_name={self.model_name}, \"\n            f\"use_fast_tokenizer={self.use_fast_tokenizer}, num_workers={self.num_workers}, \"\n            f\"preprocessing_rules={self.preprocessing_rules}, tokenizer_params={self.tokenizer_params}, \"\n            f\"encode_params={self.encode_params}, root_dir={self.root_dir})\"\n        )\n</code></pre>"},{"location":"pytorch-widedeep/self_supervised_pretraining.html","title":"Self Supervised Pre-training for tabular data","text":"<p>In this library we have implemented two methods or routines that allow the user to use self-suerpvised pre-training for all tabular models in the library with the exception of the <code>TabPerceiver</code> (this is a particular model and self-supervised pre-training requires some adjustments that will be implemented in future versions). Please see the examples folder in the repo or the examples section in the docs for details on how to use self-supervised pre-training with this library.</p> <p>The two routines implemented are illustrated in the figures below. The first is from TabNet: Attentive Interpretable Tabular Learning. It is a 'standard' encoder-decoder architecture and and is designed here for models that do not use transformer-based architectures (or when the embeddings can all have different dimensions). The second is from SAINT: Improved Neural Networks for Tabular Data via Row Attention and Contrastive Pre-Training, it is based on Contrastive and Denoising learning and is designed for models that use transformer-based architectures (or when the embeddings all need to have the same dimension):</p> <p> </p> <p>Figure 1. Figure 2 in their paper. The caption of the original paper is included in case it is useful.</p> <p> </p> <p>Figure 2. Figure 1 in their paper. The caption of the original paper is included in case it is useful.</p> <p>Note that the self-supervised pre-trainers described below focus, of course, on the self-supervised pre-training phase, i.e. the left side in Figure 1 and the upper part in Figure 2. When combined with the <code>Trainer</code> described earlier in the documenation, one can reproduce the full process illustrated in the figures above.</p> <p>Also Note that it is beyond the scope of this docs to explain in detail these routines. In addition, to fully utilise the self-supervised trainers implemented in this library a minimum understanding of the processes as described in the papers is required. Therefore, we strongly encourage the users to have a look to the papers.</p>"},{"location":"pytorch-widedeep/self_supervised_pretraining.html#pytorch_widedeep.self_supervised_training.EncoderDecoderTrainer","title":"EncoderDecoderTrainer","text":"<p>               Bases: <code>BaseEncoderDecoderTrainer</code></p> <p>This class implements an Encoder-Decoder self-supervised 'routine' inspired by TabNet: Attentive Interpretable Tabular Learning. See Figure 1 above.</p> <p>Parameters:</p> Name Type Description Default <code>encoder</code> <code>ModelWithoutAttention</code> <p>An instance of a <code>TabMlp</code>, <code>TabResNet</code> or <code>TabNet</code> model</p> required <code>decoder</code> <code>Optional[DecoderWithoutAttention]</code> <p>An instance of  a <code>TabMlpDecoder</code>, <code>TabResNetDecoder</code> or <code>TabNetDecoder</code> model. if <code>None</code> the decoder will be automatically built as a 'simetric' model to the Encoder</p> <code>None</code> <code>masked_prob</code> <code>float</code> <p>Indicates the fraction of elements in the embedding tensor that will be masked and hence used for reconstruction</p> <code>0.2</code> <code>optimizer</code> <code>Optional[Optimizer]</code> <p>An instance of Pytorch's <code>Optimizer</code> object (e.g. <code>torch.optim.Adam ()</code>). if no optimizer is passed it will default to <code>AdamW</code>.</p> <code>None</code> <code>lr_scheduler</code> <code>Optional[LRScheduler]</code> <p>An instance of Pytorch's <code>LRScheduler</code> object (e.g <code>torch.optim.lr_scheduler.StepLR(opt, step_size=5)</code>).</p> <code>None</code> <code>callbacks</code> <code>Optional[List[Callback]]</code> <p>List with <code>Callback</code> objects. The three callbacks available in <code>pytorch-widedeep</code> are: <code>LRHistory</code>, <code>ModelCheckpoint</code> and <code>EarlyStopping</code>. This can also be a custom callback. See <code>pytorch_widedeep.callbacks.Callback</code> or the Examples folder in the repo.</p> <code>None</code> <code>verbose</code> <code>int</code> <p>Setting it to 0 will print nothing during training.</p> <code>1</code> <code>seed</code> <code>int</code> <p>Random seed to be used internally for train_test_split</p> <code>1</code> <p>Other Parameters:</p> Name Type Description <code>**kwargs</code> <p>Other infrequently used arguments that can also be passed as kwargs are:</p> <ul> <li> <p>device: <code>str</code>     string indicating the device. One of 'cpu' or 'gpu'</p> </li> <li> <p>num_workers: <code>int</code>     number of workers to be used internally by the data loaders</p> </li> <li> <p>reducelronplateau_criterion: <code>str</code>     This sets the criterion that will be used by the lr scheduler to     take a step: One of 'loss' or 'metric'. The ReduceLROnPlateau     learning rate is a bit particular.</p> </li> </ul> Source code in <code>pytorch_widedeep/self_supervised_training/encoder_decoder_trainer.py</code> <pre><code>class EncoderDecoderTrainer(BaseEncoderDecoderTrainer):\n    r\"\"\"This class implements an Encoder-Decoder self-supervised 'routine'\n    inspired by\n    [TabNet: Attentive Interpretable Tabular Learning](https://arxiv.org/abs/1908.07442).\n    See Figure 1 above.\n\n    Parameters\n    ----------\n    encoder: ModelWithoutAttention,\n        An instance of a `TabMlp`, `TabResNet` or `TabNet` model\n    decoder: Optional[DecoderWithoutAttention] = None,\n        An instance of  a `TabMlpDecoder`, `TabResNetDecoder` or\n        `TabNetDecoder` model. if `None` the decoder will be automatically\n        built as a '_simetric_' model to the Encoder\n    masked_prob: float = 0.2,\n        Indicates the fraction of elements in the embedding tensor that will\n        be masked and hence used for reconstruction\n    optimizer: Optional[Optimizer] = None,\n        An instance of Pytorch's `Optimizer` object (e.g. `torch.optim.Adam\n        ()`). if no optimizer is passed it will default to `AdamW`.\n    lr_scheduler: Optional[LRScheduler] = None,\n        An instance of Pytorch's `LRScheduler` object\n        (e.g `torch.optim.lr_scheduler.StepLR(opt, step_size=5)`).\n    callbacks: Optional[List[Callback]] = None,\n        List with `Callback` objects. The three callbacks available in\n        `pytorch-widedeep` are: `LRHistory`, `ModelCheckpoint` and\n        `EarlyStopping`. This can also be a custom callback. See\n        `pytorch_widedeep.callbacks.Callback` or the Examples folder in the\n        repo.\n    verbose: int, default=1\n        Setting it to 0 will print nothing during training.\n    seed: int, default=1\n        Random seed to be used internally for train_test_split\n\n    Other Parameters\n    ----------------\n    **kwargs: dict\n        Other infrequently used arguments that can also be passed as kwargs are:\n\n        - **device**: `str`&lt;br/&gt;\n            string indicating the device. One of _'cpu'_ or _'gpu'_\n\n        - **num_workers**: `int`&lt;br/&gt;\n            number of workers to be used internally by the data loaders\n\n        - **reducelronplateau_criterion**: `str`\n            This sets the criterion that will be used by the lr scheduler to\n            take a step: One of _'loss'_ or _'metric'_. The ReduceLROnPlateau\n            learning rate is a bit particular.\n\n    \"\"\"\n\n    def __init__(\n        self,\n        encoder: ModelWithoutAttention,\n        decoder: Optional[DecoderWithoutAttention] = None,\n        masked_prob: float = 0.2,\n        optimizer: Optional[Optimizer] = None,\n        lr_scheduler: Optional[LRScheduler] = None,\n        callbacks: Optional[List[Callback]] = None,\n        verbose: int = 1,\n        seed: int = 1,\n        **kwargs,\n    ):\n        super().__init__(\n            encoder=encoder,\n            decoder=decoder,\n            masked_prob=masked_prob,\n            optimizer=optimizer,\n            lr_scheduler=lr_scheduler,\n            callbacks=callbacks,\n            verbose=verbose,\n            seed=seed,\n            **kwargs,\n        )\n\n    def pretrain(\n        self,\n        X_tab: np.ndarray,\n        X_tab_val: Optional[np.ndarray] = None,\n        val_split: Optional[float] = None,\n        validation_freq: int = 1,\n        n_epochs: int = 1,\n        batch_size: int = 32,\n    ):\n        r\"\"\"Pretrain method. Can also be called using `.fit(&lt;same_args&gt;)`\n\n        Parameters\n        ----------\n        X_tab: np.ndarray,\n            tabular dataset\n        X_tab_val: np.ndarray, Optional, default = None\n            validation data\n        val_split: float, Optional. default=None\n            An alterative to passing the validation set is to use a train/val\n            split fraction via `val_split`\n        validation_freq: int, default=1\n            epochs validation frequency\n        n_epochs: int, default=1\n            number of epochs\n        batch_size: int, default=32\n            batch size\n        \"\"\"\n\n        self.batch_size = batch_size\n\n        train_set, eval_set = self._train_eval_split(X_tab, X_tab_val, val_split)\n        train_loader = DataLoader(\n            dataset=train_set, batch_size=batch_size, num_workers=self.num_workers\n        )\n        train_steps = len(train_loader)\n        if eval_set is not None:\n            eval_loader = DataLoader(\n                dataset=eval_set,\n                batch_size=batch_size,\n                num_workers=self.num_workers,\n                shuffle=False,\n            )\n            eval_steps = len(eval_loader)\n\n        self.callback_container.on_train_begin(\n            {\n                \"batch_size\": batch_size,\n                \"train_steps\": train_steps,\n                \"n_epochs\": n_epochs,\n            }\n        )\n        for epoch in range(n_epochs):\n            epoch_logs: Dict[str, float] = {}\n            self.callback_container.on_epoch_begin(epoch, logs=epoch_logs)\n\n            self.train_running_loss = 0.0\n            with trange(train_steps, disable=self.verbose != 1) as t:\n                for batch_idx, X in zip(t, train_loader):\n                    t.set_description(\"epoch %i\" % (epoch + 1))\n                    train_loss = self._train_step(X[0], batch_idx)\n                    self.callback_container.on_batch_end(batch=batch_idx)\n                    print_loss_and_metric(t, train_loss)\n\n            epoch_logs = save_epoch_logs(epoch_logs, train_loss, None, \"train\")\n\n            on_epoch_end_metric = None\n            if eval_set is not None and epoch % validation_freq == (\n                validation_freq - 1\n            ):\n                self.callback_container.on_eval_begin()\n                self.valid_running_loss = 0.0\n                with trange(eval_steps, disable=self.verbose != 1) as v:\n                    for batch_idx, X in zip(v, eval_loader):\n                        v.set_description(\"valid\")\n                        val_loss = self._eval_step(X[0], batch_idx)\n                        print_loss_and_metric(v, val_loss)\n                epoch_logs = save_epoch_logs(epoch_logs, val_loss, None, \"val\")\n                on_epoch_end_metric = val_loss\n            else:\n                if self.reducelronplateau:\n                    raise NotImplementedError(\n                        \"ReduceLROnPlateau scheduler can be used only with validation data.\"\n                    )\n\n            self.callback_container.on_epoch_end(epoch, epoch_logs, on_epoch_end_metric)\n\n            if self.early_stop:\n                self.callback_container.on_train_end(epoch_logs)\n                break\n\n        self.callback_container.on_train_end(epoch_logs)\n        self._restore_best_weights()\n        self.ed_model.train()\n\n    def fit(\n        self,\n        X_tab: np.ndarray,\n        X_tab_val: Optional[np.ndarray] = None,\n        val_split: Optional[float] = None,\n        validation_freq: int = 1,\n        n_epochs: int = 1,\n        batch_size: int = 32,\n    ):\n        return self.pretrain(\n            X_tab, X_tab_val, val_split, validation_freq, n_epochs, batch_size\n        )\n\n    def explain(self, X_tab: np.ndarray, save_step_masks: bool = False):\n        raise NotImplementedError(\n            \"The 'explain' is currently not implemented for Self Supervised Pretraining\"\n        )\n\n    def _train_step(self, X_tab: Tensor, batch_idx: int) -&gt; float:\n        X = to_device(X_tab, self.device)\n\n        self.optimizer.zero_grad()\n        x_embed, x_embed_rec, mask = self.ed_model(X)\n        loss = self.loss_fn(x_embed, x_embed_rec, mask)\n        loss.backward()\n        self.optimizer.step()\n\n        self.train_running_loss += loss.item()\n        avg_loss = self.train_running_loss / (batch_idx + 1)\n\n        return avg_loss\n\n    def _eval_step(self, X_tab: Tensor, batch_idx: int) -&gt; float:\n        self.ed_model.eval()\n\n        with torch.no_grad():\n            X = to_device(X_tab, self.device)\n\n            x_embed, x_embed_rec, mask = self.ed_model(X)\n            loss = self.loss_fn(x_embed, x_embed_rec, mask)\n\n            self.valid_running_loss += loss.item()\n            avg_loss = self.valid_running_loss / (batch_idx + 1)\n\n        return avg_loss\n\n    def _train_eval_split(\n        self,\n        X: np.ndarray,\n        X_tab_val: Optional[np.ndarray] = None,\n        val_split: Optional[float] = None,\n    ) -&gt; Tuple[TensorDataset, Optional[TensorDataset]]:\n        if X_tab_val is not None:\n            train_set = TensorDataset(torch.from_numpy(X))\n            eval_set = TensorDataset(torch.from_numpy(X_tab_val))\n        elif val_split is not None:\n            X_tr, X_tab_val = train_test_split(  # type: ignore\n                X, test_size=val_split, random_state=self.seed\n            )\n            train_set = TensorDataset(torch.from_numpy(X_tr))\n            eval_set = TensorDataset(torch.from_numpy(X_tab_val))\n        else:\n            train_set = TensorDataset(torch.from_numpy(X))\n            eval_set = None\n\n        return train_set, eval_set\n</code></pre>"},{"location":"pytorch-widedeep/self_supervised_pretraining.html#pytorch_widedeep.self_supervised_training.EncoderDecoderTrainer.pretrain","title":"pretrain","text":"<pre><code>pretrain(\n    X_tab,\n    X_tab_val=None,\n    val_split=None,\n    validation_freq=1,\n    n_epochs=1,\n    batch_size=32,\n)\n</code></pre> <p>Pretrain method. Can also be called using <code>.fit(&lt;same_args&gt;)</code></p> <p>Parameters:</p> Name Type Description Default <code>X_tab</code> <code>ndarray</code> <p>tabular dataset</p> required <code>X_tab_val</code> <code>Optional[ndarray]</code> <p>validation data</p> <code>None</code> <code>val_split</code> <code>Optional[float]</code> <p>An alterative to passing the validation set is to use a train/val split fraction via <code>val_split</code></p> <code>None</code> <code>validation_freq</code> <code>int</code> <p>epochs validation frequency</p> <code>1</code> <code>n_epochs</code> <code>int</code> <p>number of epochs</p> <code>1</code> <code>batch_size</code> <code>int</code> <p>batch size</p> <code>32</code> Source code in <code>pytorch_widedeep/self_supervised_training/encoder_decoder_trainer.py</code> <pre><code>def pretrain(\n    self,\n    X_tab: np.ndarray,\n    X_tab_val: Optional[np.ndarray] = None,\n    val_split: Optional[float] = None,\n    validation_freq: int = 1,\n    n_epochs: int = 1,\n    batch_size: int = 32,\n):\n    r\"\"\"Pretrain method. Can also be called using `.fit(&lt;same_args&gt;)`\n\n    Parameters\n    ----------\n    X_tab: np.ndarray,\n        tabular dataset\n    X_tab_val: np.ndarray, Optional, default = None\n        validation data\n    val_split: float, Optional. default=None\n        An alterative to passing the validation set is to use a train/val\n        split fraction via `val_split`\n    validation_freq: int, default=1\n        epochs validation frequency\n    n_epochs: int, default=1\n        number of epochs\n    batch_size: int, default=32\n        batch size\n    \"\"\"\n\n    self.batch_size = batch_size\n\n    train_set, eval_set = self._train_eval_split(X_tab, X_tab_val, val_split)\n    train_loader = DataLoader(\n        dataset=train_set, batch_size=batch_size, num_workers=self.num_workers\n    )\n    train_steps = len(train_loader)\n    if eval_set is not None:\n        eval_loader = DataLoader(\n            dataset=eval_set,\n            batch_size=batch_size,\n            num_workers=self.num_workers,\n            shuffle=False,\n        )\n        eval_steps = len(eval_loader)\n\n    self.callback_container.on_train_begin(\n        {\n            \"batch_size\": batch_size,\n            \"train_steps\": train_steps,\n            \"n_epochs\": n_epochs,\n        }\n    )\n    for epoch in range(n_epochs):\n        epoch_logs: Dict[str, float] = {}\n        self.callback_container.on_epoch_begin(epoch, logs=epoch_logs)\n\n        self.train_running_loss = 0.0\n        with trange(train_steps, disable=self.verbose != 1) as t:\n            for batch_idx, X in zip(t, train_loader):\n                t.set_description(\"epoch %i\" % (epoch + 1))\n                train_loss = self._train_step(X[0], batch_idx)\n                self.callback_container.on_batch_end(batch=batch_idx)\n                print_loss_and_metric(t, train_loss)\n\n        epoch_logs = save_epoch_logs(epoch_logs, train_loss, None, \"train\")\n\n        on_epoch_end_metric = None\n        if eval_set is not None and epoch % validation_freq == (\n            validation_freq - 1\n        ):\n            self.callback_container.on_eval_begin()\n            self.valid_running_loss = 0.0\n            with trange(eval_steps, disable=self.verbose != 1) as v:\n                for batch_idx, X in zip(v, eval_loader):\n                    v.set_description(\"valid\")\n                    val_loss = self._eval_step(X[0], batch_idx)\n                    print_loss_and_metric(v, val_loss)\n            epoch_logs = save_epoch_logs(epoch_logs, val_loss, None, \"val\")\n            on_epoch_end_metric = val_loss\n        else:\n            if self.reducelronplateau:\n                raise NotImplementedError(\n                    \"ReduceLROnPlateau scheduler can be used only with validation data.\"\n                )\n\n        self.callback_container.on_epoch_end(epoch, epoch_logs, on_epoch_end_metric)\n\n        if self.early_stop:\n            self.callback_container.on_train_end(epoch_logs)\n            break\n\n    self.callback_container.on_train_end(epoch_logs)\n    self._restore_best_weights()\n    self.ed_model.train()\n</code></pre>"},{"location":"pytorch-widedeep/self_supervised_pretraining.html#pytorch_widedeep.self_supervised_training.ContrastiveDenoisingTrainer","title":"ContrastiveDenoisingTrainer","text":"<p>               Bases: <code>BaseContrastiveDenoisingTrainer</code></p> <p>This class trains a Contrastive, Denoising Self Supervised 'routine' that is based on the one described in SAINT: Improved Neural Networks for Tabular Data via Row Attention and Contrastive Pre-Training, their Figure 1.</p> <p>Parameters:</p> Name Type Description Default <code>model</code> <code>ModelWithAttention</code> <p>An instance of a <code>TabTransformer</code>, <code>SAINT</code>, <code>FTTransformer</code>, <code>TabFastFormer</code>, <code>TabPerceiver</code>, <code>ContextAttentionMLP</code> and <code>SelfAttentionMLP</code>.</p> required <code>preprocessor</code> <code>TabPreprocessor</code> <p>A fitted <code>TabPreprocessor</code> object. See <code>pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor</code></p> required <code>optimizer</code> <code>Optional[Optimizer]</code> <p>An instance of Pytorch's <code>Optimizer</code> object (e.g. <code>torch.optim.Adam ()</code>). if no optimizer is passed it will default to <code>AdamW</code>.</p> <code>None</code> <code>lr_scheduler</code> <code>Optional[LRScheduler]</code> <p>An instance of Pytorch's <code>LRScheduler</code> object (e.g <code>torch.optim.lr_scheduler.StepLR(opt, step_size=5)</code>).</p> <code>None</code> <code>callbacks</code> <code>Optional[List[Callback]]</code> <p>List with <code>Callback</code> objects. The three callbacks available in <code>pytorch-widedeep</code> are: <code>LRHistory</code>, <code>ModelCheckpoint</code> and <code>EarlyStopping</code>. This can also be a custom callback. See <code>pytorch_widedeep.callbacks.Callback</code> or the Examples folder in the repo.</p> <code>None</code> <code>loss_type</code> <code>Literal[contrastive, denoising, both]</code> <p>One of 'contrastive', 'denoising' or 'both'. See SAINT: Improved Neural Networks for Tabular Data via Row Attention and Contrastive Pre-Training, their figure (1) and their equation (5).</p> <code>'both'</code> <code>projection_head1_dims</code> <code>Optional[List[int]]</code> <p>The projection heads are simply MLPs. This parameter is a list of integers with the dimensions of the MLP hidden layers. See the paper for details. Note that setting up this parameter requires some knowledge of the architecture one is using. For example, if we are representing the features with embeddings of dim 32 (i.e. the so called dimension of the model is 32), then the first dimension of the projection head must be 32 (e.g. [32, 16])</p> <code>None</code> <code>projection_head2_dims</code> <code>Optional[List[int]]</code> <p>Same as 'projection_head1_dims' for the second head</p> <code>None</code> <code>projection_heads_activation</code> <code>str</code> <p>Activation function for the projection heads</p> <code>'relu'</code> <code>cat_mlp_type</code> <code>Literal[single, multiple]</code> <p>If 'denoising' loss is used, one can choose two types of 'stacked' MLPs to process the output from the transformer-based encoder that receives 'corrupted' (cut-mixed and mixed-up) features. These are 'single' or 'multiple'. The former approach will apply a single MLP to all the categorical features while the latter will use one MLP per categorical feature</p> <code>'multiple'</code> <code>cont_mlp_type</code> <code>Literal[single, multiple]</code> <p>Same as 'cat_mlp_type' but for the continuous features</p> <code>'multiple'</code> <code>denoise_mlps_activation</code> <code>str</code> <p>activation function for the so called 'denoising mlps'.</p> <code>'relu'</code> <code>verbose</code> <code>int</code> <p>Setting it to 0 will print nothing during training.</p> <code>1</code> <code>seed</code> <code>int</code> <p>Random seed to be used internally for train_test_split</p> <code>1</code> <p>Other Parameters:</p> Name Type Description <code>**kwargs</code> <p>Other infrequently used arguments that can also be passed as kwargs are:</p> <ul> <li> <p>device: <code>str</code>     string indicating the device. One of 'cpu' or 'gpu'</p> </li> <li> <p>num_workers: <code>int</code>     number of workers to be used internally by the data loaders</p> </li> <li> <p>reducelronplateau_criterion: <code>str</code>     This sets the criterion that will be used by the lr scheduler to     take a step: One of 'loss' or 'metric'. The ReduceLROnPlateau     learning rate is a bit particular.</p> </li> </ul> Source code in <code>pytorch_widedeep/self_supervised_training/contrastive_denoising_trainer.py</code> <pre><code>class ContrastiveDenoisingTrainer(BaseContrastiveDenoisingTrainer):\n    r\"\"\"This class trains a Contrastive, Denoising Self Supervised 'routine' that\n    is based on the one described in\n    [SAINT: Improved Neural Networks for Tabular Data via Row Attention and\n    Contrastive Pre-Training](https://arxiv.org/abs/2106.01342), their Figure 1.\n\n    Parameters\n    ----------\n    model: ModelWithAttention,\n        An instance of a `TabTransformer`, `SAINT`, `FTTransformer`,\n        `TabFastFormer`, `TabPerceiver`, `ContextAttentionMLP` and\n        `SelfAttentionMLP`.\n    preprocessor: `TabPreprocessor`\n        A fitted `TabPreprocessor` object. See\n        `pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor`\n    optimizer: Optional[Optimizer] = None,\n        An instance of Pytorch's `Optimizer` object (e.g. `torch.optim.Adam\n        ()`). if no optimizer is passed it will default to `AdamW`.\n    lr_scheduler: Optional[LRScheduler] = None,\n        An instance of Pytorch's `LRScheduler` object\n        (e.g `torch.optim.lr_scheduler.StepLR(opt, step_size=5)`).\n    callbacks: Optional[List[Callback]] = None,\n        List with `Callback` objects. The three callbacks available in\n        `pytorch-widedeep` are: `LRHistory`, `ModelCheckpoint` and\n        `EarlyStopping`. This can also be a custom callback. See\n        `pytorch_widedeep.callbacks.Callback` or the Examples folder in the\n        repo.\n    loss_type: str, default = \"both\"\n        One of '_contrastive_', '_denoising_' or '_both_'. See [SAINT: Improved\n        Neural Networks for Tabular Data via Row Attention and Contrastive\n        Pre-Training](https://arxiv.org/abs/2203.05556), their figure (1)\n        and their equation (5).\n    projection_head1_dims: list, Optional, default = None\n        The projection heads are simply MLPs. This parameter is a list\n        of integers with the dimensions of the MLP hidden layers. See the\n        [paper](https://arxiv.org/abs/2203.05556) for details. Note that\n        setting up this parameter requires some knowledge of the architecture\n        one is using. For example, if we are representing the features with\n        embeddings of dim 32 (i.e. the so called dimension of the model is\n        32), then the first dimension of the projection head must be 32 (e.g.\n        [32, 16])\n    projection_head2_dims: list, Optional, default = None\n        Same as '_projection_head1_dims_' for the second head\n    projection_heads_activation: str, default = \"relu\"\n        Activation function for the projection heads\n    cat_mlp_type: str, default = \"multiple\"\n        If '_denoising_' loss is used, one can choose two types of 'stacked'\n        MLPs to process the output from the transformer-based encoder that\n        receives 'corrupted' (cut-mixed and mixed-up) features. These\n        are '_single_' or '_multiple_'. The former approach will apply a single\n        MLP to all the categorical features while the latter will use one MLP\n        per categorical feature\n    cont_mlp_type: str, default = \"multiple\"\n        Same as 'cat_mlp_type' but for the continuous features\n    denoise_mlps_activation: str, default = \"relu\"\n        activation function for the so called 'denoising mlps'.\n    verbose: int, default=1\n        Setting it to 0 will print nothing during training.\n    seed: int, default=1\n        Random seed to be used internally for train_test_split\n\n    Other Parameters\n    ----------------\n    **kwargs: dict\n        Other infrequently used arguments that can also be passed as kwargs are:\n\n        - **device**: `str`&lt;br/&gt;\n            string indicating the device. One of _'cpu'_ or _'gpu'_\n\n        - **num_workers**: `int`&lt;br/&gt;\n            number of workers to be used internally by the data loaders\n\n        - **reducelronplateau_criterion**: `str`\n            This sets the criterion that will be used by the lr scheduler to\n            take a step: One of _'loss'_ or _'metric'_. The ReduceLROnPlateau\n            learning rate is a bit particular.\n\n    \"\"\"\n\n    def __init__(\n        self,\n        model: ModelWithAttention,\n        preprocessor: TabPreprocessor,\n        optimizer: Optional[Optimizer] = None,\n        lr_scheduler: Optional[LRScheduler] = None,\n        callbacks: Optional[List[Callback]] = None,\n        loss_type: Literal[\"contrastive\", \"denoising\", \"both\"] = \"both\",\n        projection_head1_dims: Optional[List[int]] = None,\n        projection_head2_dims: Optional[List[int]] = None,\n        projection_heads_activation: str = \"relu\",\n        cat_mlp_type: Literal[\"single\", \"multiple\"] = \"multiple\",\n        cont_mlp_type: Literal[\"single\", \"multiple\"] = \"multiple\",\n        denoise_mlps_activation: str = \"relu\",\n        verbose: int = 1,\n        seed: int = 1,\n        **kwargs,\n    ):\n        super().__init__(\n            model=model,\n            preprocessor=preprocessor,\n            loss_type=loss_type,\n            optimizer=optimizer,\n            lr_scheduler=lr_scheduler,\n            callbacks=callbacks,\n            projection_head1_dims=projection_head1_dims,\n            projection_head2_dims=projection_head2_dims,\n            projection_heads_activation=projection_heads_activation,\n            cat_mlp_type=cat_mlp_type,\n            cont_mlp_type=cont_mlp_type,\n            denoise_mlps_activation=denoise_mlps_activation,\n            verbose=verbose,\n            seed=seed,\n            **kwargs,\n        )\n\n    def pretrain(\n        self,\n        X_tab: np.ndarray,\n        X_tab_val: Optional[np.ndarray] = None,\n        val_split: Optional[float] = None,\n        validation_freq: int = 1,\n        n_epochs: int = 1,\n        batch_size: int = 32,\n    ):\n        r\"\"\"Pretrain method. Can also be called using `.fit(&lt;same_args&gt;)`\n\n        Parameters\n        ----------\n        X_tab: np.ndarray,\n            tabular dataset\n        X_tab_val: np.ndarray, Optional, default = None\n            validation data. Note that, although it is possible to use\n            contrastive-denoising training with a validation set, such set\n            must include feature values that are _all_ seen in the training\n            set in the case of the categorical columns. This is because the\n            values of the columns themselves will be used as targets when\n            computing the loss. Therefore, if a new category is present in\n            the validation set that was not seen in training this will\n            effectively be like trying to predict a new, never seen category\n            (and Pytorch will throw an error)\n        val_split: float, Optional. default=None\n            An alterative to passing the validation set is to use a train/val\n            split fraction via `val_split`\n        validation_freq: int, default=1\n            epochs validation frequency\n        n_epochs: int, default=1\n            number of epochs\n        batch_size: int, default=32\n            batch size\n        \"\"\"\n\n        self.batch_size = batch_size\n\n        train_set, eval_set = self._train_eval_split(X_tab, X_tab_val, val_split)\n        train_loader = DataLoader(\n            dataset=train_set, batch_size=batch_size, num_workers=self.num_workers\n        )\n        train_steps = len(train_loader)\n        if eval_set is not None:\n            eval_loader = DataLoader(\n                dataset=eval_set,\n                batch_size=batch_size,\n                num_workers=self.num_workers,\n                shuffle=False,\n            )\n            eval_steps = len(eval_loader)\n\n        self.callback_container.on_train_begin(\n            {\n                \"batch_size\": batch_size,\n                \"train_steps\": train_steps,\n                \"n_epochs\": n_epochs,\n            }\n        )\n        for epoch in range(n_epochs):\n            epoch_logs: Dict[str, float] = {}\n            self.callback_container.on_epoch_begin(epoch, logs=epoch_logs)\n\n            self.train_running_loss = 0.0\n            with trange(train_steps, disable=self.verbose != 1) as t:\n                for batch_idx, X in zip(t, train_loader):\n                    t.set_description(\"epoch %i\" % (epoch + 1))\n                    train_loss = self._train_step(X[0], batch_idx)\n                    self.callback_container.on_batch_end(batch=batch_idx)\n                    print_loss_and_metric(t, train_loss)\n\n            epoch_logs = save_epoch_logs(epoch_logs, train_loss, None, \"train\")\n\n            on_epoch_end_metric = None\n            if eval_set is not None and epoch % validation_freq == (\n                validation_freq - 1\n            ):\n                self.callback_container.on_eval_begin()\n                self.valid_running_loss = 0.0\n                with trange(eval_steps, disable=self.verbose != 1) as v:\n                    for batch_idx, X in zip(v, eval_loader):\n                        v.set_description(\"valid\")\n                        val_loss = self._eval_step(X[0], batch_idx)\n                        print_loss_and_metric(v, val_loss)\n                epoch_logs = save_epoch_logs(epoch_logs, val_loss, None, \"val\")\n                on_epoch_end_metric = val_loss\n            else:\n                if self.reducelronplateau:\n                    raise NotImplementedError(\n                        \"ReduceLROnPlateau scheduler can be used only with validation data.\"\n                    )\n\n            self.callback_container.on_epoch_end(epoch, epoch_logs, on_epoch_end_metric)\n\n            if self.early_stop:\n                self.callback_container.on_train_end(epoch_logs)\n                break\n\n        self.callback_container.on_train_end(epoch_logs)\n        self._restore_best_weights()\n        self.cd_model.train()\n\n    def fit(\n        self,\n        X_tab: np.ndarray,\n        X_tab_val: Optional[np.ndarray] = None,\n        val_split: Optional[float] = None,\n        validation_freq: int = 1,\n        n_epochs: int = 1,\n        batch_size: int = 32,\n    ):\n        return self.pretrain(\n            X_tab, X_tab_val, val_split, validation_freq, n_epochs, batch_size\n        )\n\n    def _train_step(self, X_tab: Tensor, batch_idx: int) -&gt; float:\n        X = to_device(X_tab, self.device)\n\n        self.optimizer.zero_grad()\n        g_projs, cat_x_and_x_, cont_x_and_x_ = self.cd_model(X)\n        loss = self._compute_loss(g_projs, cat_x_and_x_, cont_x_and_x_)\n        loss.backward()\n        self.optimizer.step()\n\n        self.train_running_loss += loss.item()\n        avg_loss = self.train_running_loss / (batch_idx + 1)\n\n        return avg_loss\n\n    def _eval_step(self, X_tab: Tensor, batch_idx: int) -&gt; float:\n        self.cd_model.eval()\n\n        with torch.no_grad():\n            X = to_device(X_tab, self.device)\n\n            g_projs, cat_x_and_x_, cont_x_and_x_ = self.cd_model(X)\n            loss = self._compute_loss(g_projs, cat_x_and_x_, cont_x_and_x_)\n\n            self.valid_running_loss += loss.item()\n            avg_loss = self.valid_running_loss / (batch_idx + 1)\n\n        return avg_loss\n\n    def _train_eval_split(\n        self,\n        X: np.ndarray,\n        X_tab_val: Optional[np.ndarray] = None,\n        val_split: Optional[float] = None,\n    ) -&gt; Tuple[TensorDataset, Optional[TensorDataset]]:\n        if X_tab_val is not None:\n            train_set = TensorDataset(torch.from_numpy(X))\n            eval_set = TensorDataset(torch.from_numpy(X_tab_val))\n        elif val_split is not None:\n            X_tr, X_tab_val = train_test_split(  # type: ignore\n                X, test_size=val_split, random_state=self.seed\n            )\n            train_set = TensorDataset(torch.from_numpy(X_tr))\n            eval_set = TensorDataset(torch.from_numpy(X_tab_val))\n        else:\n            train_set = TensorDataset(torch.from_numpy(X))\n            eval_set = None\n\n        return train_set, eval_set\n</code></pre>"},{"location":"pytorch-widedeep/self_supervised_pretraining.html#pytorch_widedeep.self_supervised_training.ContrastiveDenoisingTrainer.pretrain","title":"pretrain","text":"<pre><code>pretrain(\n    X_tab,\n    X_tab_val=None,\n    val_split=None,\n    validation_freq=1,\n    n_epochs=1,\n    batch_size=32,\n)\n</code></pre> <p>Pretrain method. Can also be called using <code>.fit(&lt;same_args&gt;)</code></p> <p>Parameters:</p> Name Type Description Default <code>X_tab</code> <code>ndarray</code> <p>tabular dataset</p> required <code>X_tab_val</code> <code>Optional[ndarray]</code> <p>validation data. Note that, although it is possible to use contrastive-denoising training with a validation set, such set must include feature values that are all seen in the training set in the case of the categorical columns. This is because the values of the columns themselves will be used as targets when computing the loss. Therefore, if a new category is present in the validation set that was not seen in training this will effectively be like trying to predict a new, never seen category (and Pytorch will throw an error)</p> <code>None</code> <code>val_split</code> <code>Optional[float]</code> <p>An alterative to passing the validation set is to use a train/val split fraction via <code>val_split</code></p> <code>None</code> <code>validation_freq</code> <code>int</code> <p>epochs validation frequency</p> <code>1</code> <code>n_epochs</code> <code>int</code> <p>number of epochs</p> <code>1</code> <code>batch_size</code> <code>int</code> <p>batch size</p> <code>32</code> Source code in <code>pytorch_widedeep/self_supervised_training/contrastive_denoising_trainer.py</code> <pre><code>def pretrain(\n    self,\n    X_tab: np.ndarray,\n    X_tab_val: Optional[np.ndarray] = None,\n    val_split: Optional[float] = None,\n    validation_freq: int = 1,\n    n_epochs: int = 1,\n    batch_size: int = 32,\n):\n    r\"\"\"Pretrain method. Can also be called using `.fit(&lt;same_args&gt;)`\n\n    Parameters\n    ----------\n    X_tab: np.ndarray,\n        tabular dataset\n    X_tab_val: np.ndarray, Optional, default = None\n        validation data. Note that, although it is possible to use\n        contrastive-denoising training with a validation set, such set\n        must include feature values that are _all_ seen in the training\n        set in the case of the categorical columns. This is because the\n        values of the columns themselves will be used as targets when\n        computing the loss. Therefore, if a new category is present in\n        the validation set that was not seen in training this will\n        effectively be like trying to predict a new, never seen category\n        (and Pytorch will throw an error)\n    val_split: float, Optional. default=None\n        An alterative to passing the validation set is to use a train/val\n        split fraction via `val_split`\n    validation_freq: int, default=1\n        epochs validation frequency\n    n_epochs: int, default=1\n        number of epochs\n    batch_size: int, default=32\n        batch size\n    \"\"\"\n\n    self.batch_size = batch_size\n\n    train_set, eval_set = self._train_eval_split(X_tab, X_tab_val, val_split)\n    train_loader = DataLoader(\n        dataset=train_set, batch_size=batch_size, num_workers=self.num_workers\n    )\n    train_steps = len(train_loader)\n    if eval_set is not None:\n        eval_loader = DataLoader(\n            dataset=eval_set,\n            batch_size=batch_size,\n            num_workers=self.num_workers,\n            shuffle=False,\n        )\n        eval_steps = len(eval_loader)\n\n    self.callback_container.on_train_begin(\n        {\n            \"batch_size\": batch_size,\n            \"train_steps\": train_steps,\n            \"n_epochs\": n_epochs,\n        }\n    )\n    for epoch in range(n_epochs):\n        epoch_logs: Dict[str, float] = {}\n        self.callback_container.on_epoch_begin(epoch, logs=epoch_logs)\n\n        self.train_running_loss = 0.0\n        with trange(train_steps, disable=self.verbose != 1) as t:\n            for batch_idx, X in zip(t, train_loader):\n                t.set_description(\"epoch %i\" % (epoch + 1))\n                train_loss = self._train_step(X[0], batch_idx)\n                self.callback_container.on_batch_end(batch=batch_idx)\n                print_loss_and_metric(t, train_loss)\n\n        epoch_logs = save_epoch_logs(epoch_logs, train_loss, None, \"train\")\n\n        on_epoch_end_metric = None\n        if eval_set is not None and epoch % validation_freq == (\n            validation_freq - 1\n        ):\n            self.callback_container.on_eval_begin()\n            self.valid_running_loss = 0.0\n            with trange(eval_steps, disable=self.verbose != 1) as v:\n                for batch_idx, X in zip(v, eval_loader):\n                    v.set_description(\"valid\")\n                    val_loss = self._eval_step(X[0], batch_idx)\n                    print_loss_and_metric(v, val_loss)\n            epoch_logs = save_epoch_logs(epoch_logs, val_loss, None, \"val\")\n            on_epoch_end_metric = val_loss\n        else:\n            if self.reducelronplateau:\n                raise NotImplementedError(\n                    \"ReduceLROnPlateau scheduler can be used only with validation data.\"\n                )\n\n        self.callback_container.on_epoch_end(epoch, epoch_logs, on_epoch_end_metric)\n\n        if self.early_stop:\n            self.callback_container.on_train_end(epoch_logs)\n            break\n\n    self.callback_container.on_train_end(epoch_logs)\n    self._restore_best_weights()\n    self.cd_model.train()\n</code></pre>"},{"location":"pytorch-widedeep/tab2vec.html","title":"Tab2Vec","text":""},{"location":"pytorch-widedeep/tab2vec.html#pytorch_widedeep.tab2vec.Tab2Vec","title":"Tab2Vec","text":"<p>Class to transform an input dataframe into vectorized form.</p> <p>This class will take an input dataframe in the form of the dataframe used for training, and it will turn it into a vectorised form based on the processing applied by the model to the categorical and continuous columns.</p> <p> NOTE: Currently this class is only implemented  for the deeptabular component. Therefore, if the input dataframe has a  text column or a column with the path to images, these will be ignored.  We will be adding these functionalities in future versions</p> <p>Parameters:</p> Name Type Description Default <code>model</code> <code>Union[WideDeep, BayesianWide, BayesianTabMlp]</code> <p><code>WideDeep</code>, <code>BayesianWide</code> or <code>BayesianTabMlp</code> model. Must be trained.</p> required <code>tab_preprocessor</code> <code>TabPreprocessor</code> <p><code>TabPreprocessor</code> object. Must be fitted.</p> required <code>return_dataframe</code> <code>bool</code> <p>Boolean indicating of the returned object(s) will be array(s) or pandas dataframe(s)</p> <code>False</code> <p>Attributes:</p> Name Type Description <code>vectorizer</code> <code>Module</code> <p>Torch module with the categorical and continuous encoding process</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import string\n&gt;&gt;&gt; from random import choices\n&gt;&gt;&gt; import numpy as np\n&gt;&gt;&gt; import pandas as pd\n&gt;&gt;&gt; from pytorch_widedeep import Tab2Vec\n&gt;&gt;&gt; from pytorch_widedeep.models import TabMlp, WideDeep\n&gt;&gt;&gt; from pytorch_widedeep.preprocessing import TabPreprocessor\n&gt;&gt;&gt;\n&gt;&gt;&gt; colnames = list(string.ascii_lowercase)[:4]\n&gt;&gt;&gt; cat_col1_vals = [\"a\", \"b\", \"c\"]\n&gt;&gt;&gt; cat_col2_vals = [\"d\", \"e\", \"f\"]\n&gt;&gt;&gt;\n&gt;&gt;&gt; # Create the toy input dataframe and a toy dataframe to be vectorised\n&gt;&gt;&gt; cat_inp = [np.array(choices(c, k=5)) for c in [cat_col1_vals, cat_col2_vals]]\n&gt;&gt;&gt; cont_inp = [np.round(np.random.rand(5), 2) for _ in range(2)]\n&gt;&gt;&gt; df_inp = pd.DataFrame(np.vstack(cat_inp + cont_inp).transpose(), columns=colnames)\n&gt;&gt;&gt; cat_t2v = [np.array(choices(c, k=5)) for c in [cat_col1_vals, cat_col2_vals]]\n&gt;&gt;&gt; cont_t2v = [np.round(np.random.rand(5), 2) for _ in range(2)]\n&gt;&gt;&gt; df_t2v = pd.DataFrame(np.vstack(cat_t2v + cont_t2v).transpose(), columns=colnames)\n&gt;&gt;&gt;\n&gt;&gt;&gt; # fit the TabPreprocessor\n&gt;&gt;&gt; embed_cols = [(\"a\", 2), (\"b\", 4)]\n&gt;&gt;&gt; cont_cols = [\"c\", \"d\"]\n&gt;&gt;&gt; tab_preprocessor = TabPreprocessor(cat_embed_cols=embed_cols, continuous_cols=cont_cols)\n&gt;&gt;&gt; X_tab = tab_preprocessor.fit_transform(df_inp)\n&gt;&gt;&gt;\n&gt;&gt;&gt; # define the model (and let's assume we train it)\n&gt;&gt;&gt; tabmlp = TabMlp(\n... column_idx=tab_preprocessor.column_idx,\n... cat_embed_input=tab_preprocessor.cat_embed_input,\n... continuous_cols=tab_preprocessor.continuous_cols,\n... mlp_hidden_dims=[8, 4])\n&gt;&gt;&gt; model = WideDeep(deeptabular=tabmlp)\n&gt;&gt;&gt; # ...train the model...\n&gt;&gt;&gt;\n&gt;&gt;&gt; # vectorise the dataframe\n&gt;&gt;&gt; t2v = Tab2Vec(tab_preprocessor, model, device=\"cpu\")\n&gt;&gt;&gt; X_vec = t2v.transform(df_t2v)\n</code></pre> Source code in <code>pytorch_widedeep/tab2vec.py</code> <pre><code>class Tab2Vec:\n    r\"\"\"Class to transform an input dataframe into vectorized form.\n\n    This class will take an input dataframe in the form of the dataframe used\n    for training, and it will turn it into a vectorised form based on the\n    processing applied by the model to the categorical and continuous\n    columns.\n\n    :information_source: **NOTE**: Currently this class is only implemented\n     for the deeptabular component. Therefore, if the input dataframe has a\n     text column or a column with the path to images, these will be ignored.\n     We will be adding these functionalities in future versions\n\n    Parameters\n    ----------\n    model: `WideDeep`, `BayesianWide` or `BayesianTabMlp`\n        `WideDeep`, `BayesianWide` or `BayesianTabMlp` model. Must be trained.\n    tab_preprocessor: `TabPreprocessor`\n        `TabPreprocessor` object. Must be fitted.\n    return_dataframe: bool\n        Boolean indicating of the returned object(s) will be array(s) or\n        pandas dataframe(s)\n\n    Attributes\n    ----------\n    vectorizer: nn.Module\n        Torch module with the categorical and continuous encoding process\n\n    Examples\n    --------\n    &gt;&gt;&gt; import string\n    &gt;&gt;&gt; from random import choices\n    &gt;&gt;&gt; import numpy as np\n    &gt;&gt;&gt; import pandas as pd\n    &gt;&gt;&gt; from pytorch_widedeep import Tab2Vec\n    &gt;&gt;&gt; from pytorch_widedeep.models import TabMlp, WideDeep\n    &gt;&gt;&gt; from pytorch_widedeep.preprocessing import TabPreprocessor\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; colnames = list(string.ascii_lowercase)[:4]\n    &gt;&gt;&gt; cat_col1_vals = [\"a\", \"b\", \"c\"]\n    &gt;&gt;&gt; cat_col2_vals = [\"d\", \"e\", \"f\"]\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; # Create the toy input dataframe and a toy dataframe to be vectorised\n    &gt;&gt;&gt; cat_inp = [np.array(choices(c, k=5)) for c in [cat_col1_vals, cat_col2_vals]]\n    &gt;&gt;&gt; cont_inp = [np.round(np.random.rand(5), 2) for _ in range(2)]\n    &gt;&gt;&gt; df_inp = pd.DataFrame(np.vstack(cat_inp + cont_inp).transpose(), columns=colnames)\n    &gt;&gt;&gt; cat_t2v = [np.array(choices(c, k=5)) for c in [cat_col1_vals, cat_col2_vals]]\n    &gt;&gt;&gt; cont_t2v = [np.round(np.random.rand(5), 2) for _ in range(2)]\n    &gt;&gt;&gt; df_t2v = pd.DataFrame(np.vstack(cat_t2v + cont_t2v).transpose(), columns=colnames)\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; # fit the TabPreprocessor\n    &gt;&gt;&gt; embed_cols = [(\"a\", 2), (\"b\", 4)]\n    &gt;&gt;&gt; cont_cols = [\"c\", \"d\"]\n    &gt;&gt;&gt; tab_preprocessor = TabPreprocessor(cat_embed_cols=embed_cols, continuous_cols=cont_cols)\n    &gt;&gt;&gt; X_tab = tab_preprocessor.fit_transform(df_inp)\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; # define the model (and let's assume we train it)\n    &gt;&gt;&gt; tabmlp = TabMlp(\n    ... column_idx=tab_preprocessor.column_idx,\n    ... cat_embed_input=tab_preprocessor.cat_embed_input,\n    ... continuous_cols=tab_preprocessor.continuous_cols,\n    ... mlp_hidden_dims=[8, 4])\n    &gt;&gt;&gt; model = WideDeep(deeptabular=tabmlp)\n    &gt;&gt;&gt; # ...train the model...\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; # vectorise the dataframe\n    &gt;&gt;&gt; t2v = Tab2Vec(tab_preprocessor, model, device=\"cpu\")\n    &gt;&gt;&gt; X_vec = t2v.transform(df_t2v)\n    \"\"\"\n\n    def __init__(\n        self,\n        tab_preprocessor: TabPreprocessor,\n        model: Union[WideDeep, BayesianWide, BayesianTabMlp],\n        device: Union[str, torch.device],\n        return_dataframe: bool = False,\n        verbose: bool = False,\n    ):\n        super(Tab2Vec, self).__init__()\n\n        self._check_inputs(tab_preprocessor, model, verbose)\n\n        if isinstance(device, str):\n            self.device = torch.device(device)\n        else:\n            self.device = device\n\n        self.tab_preprocessor = tab_preprocessor\n        self.return_dataframe = return_dataframe\n        self.verbose = verbose\n\n        self.vectorizer = self._set_vectorizer(model)\n\n        self._set_dim_attributes(tab_preprocessor, model)\n\n    def fit(self, df: pd.DataFrame, target_col: Optional[str] = None) -&gt; \"Tab2Vec\":\n        r\"\"\"This is an empty method i.e. Returns the unchanged object itself. Is\n        only included for consistency in case `Tab2Vec` is used as part of a\n        Pipeline\n\n        Parameters\n        ----------\n        df: pd.DataFrame\n            DataFrame to be vectorised, i.e. the categorical and continuous\n            columns will be encoded based on the processing applied within\n            the model\n        target_col: str, Optional\n            Column name of the target_col variable. If `None` only the array of\n            predictors will be returned\n\n        Returns\n        -------\n        Tab2Vec\n        \"\"\"\n\n        return self\n\n    def transform(\n        self,\n        df: pd.DataFrame,\n        target_col: Optional[str] = None,\n    ) -&gt; Union[\n        np.ndarray,\n        Tuple[np.ndarray, np.ndarray],\n        pd.DataFrame,\n        Tuple[pd.DataFrame, pd.Series],\n    ]:\n        r\"\"\"Transforms the input dataframe into vectorized form. If a target\n        column name is passed the target values will be returned separately\n        in their corresponding type (np.ndarray or pd.DataFrame)\n\n        Parameters\n        ----------\n        df: pd.DataFrame\n            DataFrame to be vectorised, i.e. the categorical and continuous\n            columns will be encoded based on the processing applied within\n            the model\n        target_col: str, Optional\n            Column name of the target_col variable. If `None` only the array of\n            predictors will be returned\n\n        Returns\n        -------\n        Union[np.ndarray, Tuple[np.ndarray, np.ndarray], pd.DataFrame, Tuple[pd.DataFrame, pd.Series]\n            Returns eiter a numpy array with the vectorised values, or a Tuple\n            of numpy arrays with the vectorised values and the target. The\n            same applies to dataframes in case we choose to set\n            `return_dataframe = True`\n        \"\"\"\n\n        X_tab = self.tab_preprocessor.transform(df)\n\n        if self.device.type == \"mps\":\n            X = torch.from_numpy(X_tab.astype(\"float32\")).to(self.device)\n        else:\n            X = torch.from_numpy(X_tab.astype(\"float\")).to(self.device)\n\n        with torch.no_grad():\n            if self.is_tab_transformer:\n                x_vec, x_cont_not_embed = self.vectorizer(X)\n            else:\n                x_vec = self.vectorizer(X)\n                x_cont_not_embed = None\n\n        if self.tab_preprocessor.with_cls_token:\n            x_vec = x_vec[:, 1:, :]\n\n        if self.tab_preprocessor.with_attention:\n            x_vec = einops.rearrange(x_vec, \"s c e -&gt; s (c e)\")\n\n        if x_cont_not_embed is not None:\n            x_vec = torch.cat([x_vec, x_cont_not_embed], 1).detach().cpu().numpy()\n        else:\n            x_vec = x_vec.detach().cpu().numpy()\n\n        if self.return_dataframe:\n            new_colnames = self._new_colnames()\n            if target_col:\n                return pd.DataFrame(data=x_vec, columns=new_colnames), df[[target_col]]\n            else:\n                return pd.DataFrame(data=x_vec, columns=new_colnames)\n        else:\n            if target_col:\n                return x_vec, df[target_col].values\n            else:\n                return x_vec\n\n    def fit_transform(\n        self, df: pd.DataFrame, target_col: Optional[str] = None\n    ) -&gt; Union[\n        np.ndarray,\n        Tuple[np.ndarray, np.ndarray],\n        pd.DataFrame,\n        Tuple[pd.DataFrame, pd.Series],\n    ]:\n        r\"\"\"Combines `fit` and `transform`\"\"\"\n        return self.fit(df, target_col).transform(df, target_col)\n\n    def _new_colnames(self) -&gt; List[str]:\n        if self.tab_preprocessor.with_attention:\n            return self._new_colnames_with_attention()\n        else:\n            return self._new_colnames_without_attention()\n\n    def _new_colnames_with_attention(\n        self,\n    ) -&gt; List[str]:\n        cat_cols: List[str] = (\n            [\n                ei[0]\n                for ei in self.tab_preprocessor.cat_embed_input\n                if ei[0] != \"cls_token\"\n            ]\n            if self.tab_preprocessor.cat_embed_input is not None\n            else []\n        )\n\n        cont_cols: List[str] = (\n            self.tab_preprocessor.continuous_cols\n            if self.tab_preprocessor.continuous_cols is not None\n            else []\n        )\n\n        if self.are_cont_embed:\n            all_embed_cols = cat_cols + cont_cols\n        else:\n            all_embed_cols = cat_cols\n\n        new_colnames = []\n        for colname in all_embed_cols:\n            assert self.input_dim is not None\n            new_colnames.extend(\n                [\n                    \"_\".join([colname, \"embed\", str(i + 1)])\n                    for i in range(self.input_dim)\n                ]\n            )\n\n        if not self.are_cont_embed:\n            new_colnames += cont_cols\n\n        return new_colnames\n\n    def _new_colnames_without_attention(\n        self,\n    ) -&gt; List[str]:\n        new_colnames: List[str] = []\n\n        if self.tab_preprocessor.cat_embed_input is not None:\n            for colname, _, embedding_dim in self.tab_preprocessor.cat_embed_input:  # type: ignore[misc]\n                new_colnames.extend(\n                    [\n                        \"_\".join([colname, \"embed\", str(i + 1)])\n                        for i in range(embedding_dim)\n                    ]\n                )\n\n        if self.tab_preprocessor.continuous_cols is not None:\n            if self.are_cont_embed:\n                for colname in self.tab_preprocessor.continuous_cols:\n                    assert self.cont_embed_dim is not None\n                    new_colnames.extend(\n                        [\n                            \"_\".join([colname, \"embed\", str(i + 1)])\n                            for i in range(self.cont_embed_dim)\n                        ]\n                    )\n            else:\n                new_colnames += self.tab_preprocessor.continuous_cols\n\n        return new_colnames\n\n    def _check_inputs(\n        self,\n        tab_preprocessor: TabPreprocessor,\n        model: Union[WideDeep, BayesianWide, BayesianTabMlp],\n        verbose: bool,\n    ) -&gt; None:\n        if not isinstance(model, BaseBayesianModel):\n            if verbose:\n                if model.deepimage is not None or model.deeptext is not None:\n                    warnings.warn(\n                        \"Currently 'Tab2Vec' is only implemented for the 'deeptabular' component.\"\n                    )\n\n            if model.deeptabular is None:\n                raise RuntimeError(\n                    \"Currently 'Tab2Vec' is only implemented for the 'deeptabular' component.\"\n                )\n            if not tab_preprocessor.is_fitted:\n                raise RuntimeError(\n                    \"The 'tab_preprocessor' must be fitted before is passed to 'Tab2Vec'\"\n                )\n\n    def _set_vectorizer(\n        self, model: Union[WideDeep, BayesianWide, BayesianTabMlp]\n    ) -&gt; Callable:\n        if isinstance(model, BaseBayesianModel):\n            vectorizer = model._get_embeddings\n            self.is_tab_transformer = False\n        elif model.deeptabular[0].__class__.__name__.lower() == \"tabtransformer\":  # type: ignore\n            self.is_tab_transformer = True\n            vectorizer = model.deeptabular[0]._get_embeddings_tt  # type: ignore\n        else:\n            self.is_tab_transformer = False\n            vectorizer = model.deeptabular[0]._get_embeddings  # type: ignore\n        return vectorizer\n\n    def _set_dim_attributes(\n        self,\n        tab_preprocessor: TabPreprocessor,\n        model: Union[WideDeep, BayesianWide, BayesianTabMlp],\n    ) -&gt; None:\n        if isinstance(model, BaseBayesianModel):\n            self.are_cont_embed: bool = (\n                model.embed_continuous if model.embed_continuous else False\n            )\n            self.input_dim: Optional[int] = None\n            self.cont_embed_dim: Optional[int] = model.cont_embed_dim\n        else:\n            self.are_cont_embed = (\n                model.deeptabular[0].embed_continuous_method is not None  # type: ignore\n            )\n            if tab_preprocessor.with_attention:\n                self.input_dim = model.deeptabular[0].input_dim  # type: ignore\n                self.cont_embed_dim = None\n            else:\n                self.input_dim = None\n                self.cont_embed_dim = model.deeptabular[0].cont_embed_dim  # type: ignore\n</code></pre>"},{"location":"pytorch-widedeep/tab2vec.html#pytorch_widedeep.tab2vec.Tab2Vec.fit","title":"fit","text":"<pre><code>fit(df, target_col=None)\n</code></pre> <p>This is an empty method i.e. Returns the unchanged object itself. Is only included for consistency in case <code>Tab2Vec</code> is used as part of a Pipeline</p> <p>Parameters:</p> Name Type Description Default <code>df</code> <code>DataFrame</code> <p>DataFrame to be vectorised, i.e. the categorical and continuous columns will be encoded based on the processing applied within the model</p> required <code>target_col</code> <code>Optional[str]</code> <p>Column name of the target_col variable. If <code>None</code> only the array of predictors will be returned</p> <code>None</code> <p>Returns:</p> Type Description <code>Tab2Vec</code> Source code in <code>pytorch_widedeep/tab2vec.py</code> <pre><code>def fit(self, df: pd.DataFrame, target_col: Optional[str] = None) -&gt; \"Tab2Vec\":\n    r\"\"\"This is an empty method i.e. Returns the unchanged object itself. Is\n    only included for consistency in case `Tab2Vec` is used as part of a\n    Pipeline\n\n    Parameters\n    ----------\n    df: pd.DataFrame\n        DataFrame to be vectorised, i.e. the categorical and continuous\n        columns will be encoded based on the processing applied within\n        the model\n    target_col: str, Optional\n        Column name of the target_col variable. If `None` only the array of\n        predictors will be returned\n\n    Returns\n    -------\n    Tab2Vec\n    \"\"\"\n\n    return self\n</code></pre>"},{"location":"pytorch-widedeep/tab2vec.html#pytorch_widedeep.tab2vec.Tab2Vec.transform","title":"transform","text":"<pre><code>transform(df, target_col=None)\n</code></pre> <p>Transforms the input dataframe into vectorized form. If a target column name is passed the target values will be returned separately in their corresponding type (np.ndarray or pd.DataFrame)</p> <p>Parameters:</p> Name Type Description Default <code>df</code> <code>DataFrame</code> <p>DataFrame to be vectorised, i.e. the categorical and continuous columns will be encoded based on the processing applied within the model</p> required <code>target_col</code> <code>Optional[str]</code> <p>Column name of the target_col variable. If <code>None</code> only the array of predictors will be returned</p> <code>None</code> <p>Returns:</p> Type Description <code>Union[np.ndarray, Tuple[np.ndarray, np.ndarray], pd.DataFrame, Tuple[pd.DataFrame, pd.Series]</code> <p>Returns eiter a numpy array with the vectorised values, or a Tuple of numpy arrays with the vectorised values and the target. The same applies to dataframes in case we choose to set <code>return_dataframe = True</code></p> Source code in <code>pytorch_widedeep/tab2vec.py</code> <pre><code>def transform(\n    self,\n    df: pd.DataFrame,\n    target_col: Optional[str] = None,\n) -&gt; Union[\n    np.ndarray,\n    Tuple[np.ndarray, np.ndarray],\n    pd.DataFrame,\n    Tuple[pd.DataFrame, pd.Series],\n]:\n    r\"\"\"Transforms the input dataframe into vectorized form. If a target\n    column name is passed the target values will be returned separately\n    in their corresponding type (np.ndarray or pd.DataFrame)\n\n    Parameters\n    ----------\n    df: pd.DataFrame\n        DataFrame to be vectorised, i.e. the categorical and continuous\n        columns will be encoded based on the processing applied within\n        the model\n    target_col: str, Optional\n        Column name of the target_col variable. If `None` only the array of\n        predictors will be returned\n\n    Returns\n    -------\n    Union[np.ndarray, Tuple[np.ndarray, np.ndarray], pd.DataFrame, Tuple[pd.DataFrame, pd.Series]\n        Returns eiter a numpy array with the vectorised values, or a Tuple\n        of numpy arrays with the vectorised values and the target. The\n        same applies to dataframes in case we choose to set\n        `return_dataframe = True`\n    \"\"\"\n\n    X_tab = self.tab_preprocessor.transform(df)\n\n    if self.device.type == \"mps\":\n        X = torch.from_numpy(X_tab.astype(\"float32\")).to(self.device)\n    else:\n        X = torch.from_numpy(X_tab.astype(\"float\")).to(self.device)\n\n    with torch.no_grad():\n        if self.is_tab_transformer:\n            x_vec, x_cont_not_embed = self.vectorizer(X)\n        else:\n            x_vec = self.vectorizer(X)\n            x_cont_not_embed = None\n\n    if self.tab_preprocessor.with_cls_token:\n        x_vec = x_vec[:, 1:, :]\n\n    if self.tab_preprocessor.with_attention:\n        x_vec = einops.rearrange(x_vec, \"s c e -&gt; s (c e)\")\n\n    if x_cont_not_embed is not None:\n        x_vec = torch.cat([x_vec, x_cont_not_embed], 1).detach().cpu().numpy()\n    else:\n        x_vec = x_vec.detach().cpu().numpy()\n\n    if self.return_dataframe:\n        new_colnames = self._new_colnames()\n        if target_col:\n            return pd.DataFrame(data=x_vec, columns=new_colnames), df[[target_col]]\n        else:\n            return pd.DataFrame(data=x_vec, columns=new_colnames)\n    else:\n        if target_col:\n            return x_vec, df[target_col].values\n        else:\n            return x_vec\n</code></pre>"},{"location":"pytorch-widedeep/tab2vec.html#pytorch_widedeep.tab2vec.Tab2Vec.fit_transform","title":"fit_transform","text":"<pre><code>fit_transform(df, target_col=None)\n</code></pre> <p>Combines <code>fit</code> and <code>transform</code></p> Source code in <code>pytorch_widedeep/tab2vec.py</code> <pre><code>def fit_transform(\n    self, df: pd.DataFrame, target_col: Optional[str] = None\n) -&gt; Union[\n    np.ndarray,\n    Tuple[np.ndarray, np.ndarray],\n    pd.DataFrame,\n    Tuple[pd.DataFrame, pd.Series],\n]:\n    r\"\"\"Combines `fit` and `transform`\"\"\"\n    return self.fit(df, target_col).transform(df, target_col)\n</code></pre>"},{"location":"pytorch-widedeep/the_rec_module.html","title":"The <code>rec</code> module","text":"<p>This module contains models are that specifically designed for recommendation systems. While the rest of the models can be accessed from the <code>pytorch_widedeep.models</code> module, models in this module need to be specifically imported from the <code>rec</code> module, e.g.:</p> <pre><code>from pytorch_widedeep.models.rec import DeepFactorizationMachine\n</code></pre> <p>The list of models here is not meant to be exhaustive, but it includes some common architectures such as factorization machines, field aware factorization machines or extreme factorization machines. More models will be added in the future.</p>"},{"location":"pytorch-widedeep/the_rec_module.html#pytorch_widedeep.models.rec.deepfm.DeepFactorizationMachine","title":"DeepFactorizationMachine","text":"<p>               Bases: <code>BaseTabularModelWithAttention</code></p> <p>Deep Factorization Machine (DeepFM) for recommendation systems, which is an adaptation of 'Factorization Machines' by Steffen Rendle. Presented in 'DeepFM: A Factorization-Machine based Neural Network for CTR Prediction' by Huifeng Guo, Ruiming Tang, Yunming Yey, Zhenguo Li, Xiuqiang He. 2017.</p> <p>The implementation in this library takes advantage of all the functionalities available to encode categorical and continuous features. The model can be used with only the factorization machine</p> <p>Note that this class implements only the 'Deep' component of the model described in the paper. The linear component is not implemented 'internally' and, if one wants to include it, it can be easily added using the 'wide' (aka linear) component available in this library. See the examples in the examples folder.</p> <p>Parameters:</p> Name Type Description Default <code>column_idx</code> <code>Dict[str, int]</code> <p>Dictionary mapping column names to their corresponding index.</p> required <code>num_factors</code> <code>int</code> <p>Number of factors for the factorization machine.</p> required <code>reduce_sum</code> <code>bool</code> <p>Whether to reduce the sum in the factorization machine output.</p> <code>True</code> <code>cat_embed_input</code> <code>Optional[List[Tuple[str, int]]]</code> <p>List of tuples with categorical column names and number of unique values.</p> <code>None</code> <code>cat_embed_dropout</code> <code>Optional[float]</code> <p>Categorical embeddings dropout. If <code>None</code>, it will default to 0.</p> <code>None</code> <code>use_cat_bias</code> <code>Optional[bool]</code> <p>Boolean indicating if bias will be used for the categorical embeddings. If <code>None</code>, it will default to 'False'.</p> <code>None</code> <code>cat_embed_activation</code> <code>Optional[str]</code> <p>Activation function for the categorical embeddings, if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> <code>None</code> <code>continuous_cols</code> <code>Optional[List[str]]</code> <p>List with the name of the numeric (aka continuous) columns</p> <code>None</code> <code>cont_norm_layer</code> <code>Optional[Literal['batchnorm', 'layernorm']]</code> <p>Type of normalization layer applied to the continuous features. Options are: 'layernorm' and 'batchnorm'. if <code>None</code>, no normalization layer will be used.</p> <code>None</code> <code>embed_continuous_method</code> <code>Optional[Literal['piecewise', 'periodic', 'standard']]</code> <p>Method to use to embed the continuous features. Options are: 'standard', 'periodic' or 'piecewise'. The 'standard' embedding method is based on the FT-Transformer implementation presented in the paper: Revisiting Deep Learning Models for Tabular Data. The 'periodic' and_'piecewise'_ methods were presented in the paper: On Embeddings for Numerical Features in Tabular Deep Learning. Please, read the papers for details.</p> <code>'standard'</code> <code>cont_embed_dropout</code> <code>Optional[float]</code> <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p> <code>None</code> <code>cont_embed_activation</code> <code>Optional[str]</code> <p>Activation function for the continuous embeddings if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported. If <code>None</code>, no activation function will be applied.</p> <code>None</code> <code>quantization_setup</code> <code>Optional[Dict[str, List[float]]]</code> <p>This parameter is used when the 'piecewise' method is used to embed the continuous cols. It is a dict where keys are the name of the continuous columns and values are lists with the boundaries for the quantization of the continuous_cols. See the examples for details. If If the 'piecewise' method is used, this parameter is required.</p> <code>None</code> <code>n_frequencies</code> <code>Optional[int]</code> <p>This is the so called 'k' in their paper On Embeddings for Numerical Features in Tabular Deep Learning, and is the number of 'frequencies' that will be used to represent each continuous column. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>sigma</code> <code>Optional[float]</code> <p>This is the sigma parameter in the paper mentioned when describing the previous parameters and it is used to initialise the 'frequency weights'. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>share_last_layer</code> <code>Optional[bool]</code> <p>This parameter is not present in the before mentioned paper but it is implemented in the official repo. If <code>True</code> the linear layer that turns the frequencies into embeddings will be shared across the continuous columns. If <code>False</code> a different linear layer will be used for each continuous column. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>full_embed_dropout</code> <code>Optional[bool]</code> <p>If <code>True</code>, the full embedding corresponding to a column will be masked out/dropout. If <code>None</code>, it will default to <code>False</code>.</p> <code>None</code> <code>mlp_hidden_dims</code> <code>Optional[List[int]]</code> <p>List with the number of neurons per dense layer in the mlp.</p> <code>None</code> <code>mlp_activation</code> <code>Optional[str]</code> <p>Activation function for the dense layers of the MLP. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> <code>None</code> <code>mlp_dropout</code> <code>Optional[float]</code> <p>float or List of floats with the dropout between the dense layers. e.g: [0.5,0.5]</p> <code>None</code> <code>mlp_batchnorm</code> <code>Optional[bool]</code> <p>Boolean indicating whether or not batch normalization will be applied to the dense layers</p> <code>None</code> <code>mlp_batchnorm_last</code> <code>Optional[bool]</code> <p>Boolean indicating whether or not batch normalization will be applied to the last of the dense layers</p> <code>None</code> <code>mlp_linear_first</code> <code>Optional[bool]</code> <p>Boolean indicating the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code></p> <code>None</code> <p>Attributes:</p> Name Type Description <code>mlp</code> <code>MLP</code> <p>MLP component of the model if the mlp_hidden_dims parameter is not None. If None, the model will only return the output of the factorization machine.</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; from typing import Dict, List, Tuple\n&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from torch import Tensor\n&gt;&gt;&gt; from pytorch_widedeep.models.rec import DeepFactorizationMachine\n&gt;&gt;&gt; X = torch.randint(0, 10, (16, 2))\n&gt;&gt;&gt; column_idx: Dict[str, int] = {\"col1\": 0, \"col2\": 1}\n&gt;&gt;&gt; cat_embed_input: List[Tuple[str, int]] = [(\"col1\", 10), (\"col2\", 10)]\n&gt;&gt;&gt; fm = DeepFactorizationMachine(\n...     column_idx=column_idx,\n...     num_factors=8,\n...     cat_embed_input=cat_embed_input,\n...     mlp_hidden_dims=[16, 8]\n... )\n&gt;&gt;&gt; out = fm(X)\n</code></pre> Source code in <code>pytorch_widedeep/models/rec/deepfm.py</code> <pre><code>class DeepFactorizationMachine(BaseTabularModelWithAttention):\n    \"\"\"\n    Deep Factorization Machine (DeepFM) for recommendation systems, which is\n    an adaptation of 'Factorization Machines' by Steffen Rendle. Presented\n    in 'DeepFM: A Factorization-Machine based Neural Network for CTR\n    Prediction' by Huifeng Guo, Ruiming Tang, Yunming Yey, Zhenguo Li,\n    Xiuqiang He. 2017.\n\n    The implementation in this library takes advantage of all the\n    functionalities available to encode categorical and continuous features.\n    The model can be used with only the factorization machine\n\n    Note that this class implements only the 'Deep' component of the model\n    described in the paper. The linear component is not\n    implemented 'internally' and, if one wants to include it, it can be\n    easily added using the 'wide' (aka linear) component available in this\n    library. See the examples in the examples folder.\n\n    Parameters\n    ----------\n    column_idx : Dict[str, int]\n        Dictionary mapping column names to their corresponding index.\n    num_factors : int\n        Number of factors for the factorization machine.\n    reduce_sum : bool, default=True\n        Whether to reduce the sum in the factorization machine output.\n    cat_embed_input : Optional[List[Tuple[str, int]]], default=None\n        List of tuples with categorical column names and number of unique values.\n    cat_embed_dropout : Optional[float], default=None\n        Categorical embeddings dropout. If `None`, it will default\n        to 0.\n    use_cat_bias : Optional[bool], default=None\n        Boolean indicating if bias will be used for the categorical embeddings.\n        If `None`, it will default to 'False'.\n    cat_embed_activation : Optional[str], default=None\n        Activation function for the categorical embeddings, if any. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported\n    continuous_cols : Optional[List[str]], default=None\n        List with the name of the numeric (aka continuous) columns\n    cont_norm_layer : Optional[Literal[\"batchnorm\", \"layernorm\"]], default=None\n        Type of normalization layer applied to the continuous features.\n        Options are: _'layernorm'_ and _'batchnorm'_. if `None`, no\n        normalization layer will be used.\n    embed_continuous_method: Optional[Literal[\"piecewise\", \"periodic\", \"standard\"]], default=\"standard\"\n        Method to use to embed the continuous features. Options are:\n        _'standard'_, _'periodic'_ or _'piecewise'_. The _'standard'_\n        embedding method is based on the FT-Transformer implementation\n        presented in the paper: [Revisiting Deep Learning Models for\n        Tabular Data](https://arxiv.org/abs/2106.11959v5). The _'periodic'_\n        and_'piecewise'_ methods were presented in the paper: [On Embeddings for\n        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556).\n        Please, read the papers for details.\n    cont_embed_dropout : Optional[float], default=None\n        Dropout for the continuous embeddings. If `None`, it will default to 0.0\n    cont_embed_activation : Optional[str], default=None\n        Activation function for the continuous embeddings if any. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported.\n        If `None`, no activation function will be applied.\n    quantization_setup : Optional[Dict[str, List[float]]], default=None\n        This parameter is used when the _'piecewise'_ method is used to embed\n        the continuous cols. It is a dict where keys are the name of the continuous\n        columns and values are lists with the boundaries for the quantization\n        of the continuous_cols. See the examples for details. If\n        If the _'piecewise'_ method is used, this parameter is required.\n    n_frequencies : Optional[int], default=None\n        This is the so called _'k'_ in their paper [On Embeddings for\n        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556),\n        and is the number of 'frequencies' that will be used to represent each\n        continuous column. See their Eq 2 in the paper for details. If\n        the _'periodic'_ method is used, this parameter is required.\n    sigma : Optional[float], default=None\n        This is the sigma parameter in the paper mentioned when describing the\n        previous parameters and it is used to initialise the 'frequency\n        weights'. See their Eq 2 in the paper for details. If\n        the _'periodic'_ method is used, this parameter is required.\n    share_last_layer : Optional[bool], default=None\n        This parameter is not present in the before mentioned paper but it is implemented in\n        the [official repo](https://github.com/yandex-research/rtdl-num-embeddings/tree/main).\n        If `True` the linear layer that turns the frequencies into embeddings\n        will be shared across the continuous columns. If `False` a different\n        linear layer will be used for each continuous column.\n        If the _'periodic'_ method is used, this parameter is required.\n    full_embed_dropout: bool, Optional, default = None,\n        If `True`, the full embedding corresponding to a column will be masked\n        out/dropout. If `None`, it will default to `False`.\n    mlp_hidden_dims: List, default = [200, 100]\n        List with the number of neurons per dense layer in the mlp.\n    mlp_activation: str, default = \"relu\"\n        Activation function for the dense layers of the MLP. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported\n    mlp_dropout: float or List, default = 0.1\n        float or List of floats with the dropout between the dense layers.\n        e.g: _[0.5,0.5]_\n    mlp_batchnorm: bool, default = False\n        Boolean indicating whether or not batch normalization will be applied\n        to the dense layers\n    mlp_batchnorm_last: bool, default = False\n        Boolean indicating whether or not batch normalization will be applied\n        to the last of the dense layers\n    mlp_linear_first: bool, default = False\n        Boolean indicating the order of the operations in the dense\n        layer. If `True: [LIN -&gt; ACT -&gt; BN -&gt; DP]`. If `False: [BN -&gt; DP -&gt;\n        LIN -&gt; ACT]`\n\n    Attributes\n    ----------\n    mlp: MLP\n        MLP component of the model if the mlp_hidden_dims parameter is not\n        None. If None, the model will only return the output of the\n        factorization machine.\n\n    Examples\n    --------\n    &gt;&gt;&gt; from typing import Dict, List, Tuple\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from torch import Tensor\n    &gt;&gt;&gt; from pytorch_widedeep.models.rec import DeepFactorizationMachine\n    &gt;&gt;&gt; X = torch.randint(0, 10, (16, 2))\n    &gt;&gt;&gt; column_idx: Dict[str, int] = {\"col1\": 0, \"col2\": 1}\n    &gt;&gt;&gt; cat_embed_input: List[Tuple[str, int]] = [(\"col1\", 10), (\"col2\", 10)]\n    &gt;&gt;&gt; fm = DeepFactorizationMachine(\n    ...     column_idx=column_idx,\n    ...     num_factors=8,\n    ...     cat_embed_input=cat_embed_input,\n    ...     mlp_hidden_dims=[16, 8]\n    ... )\n    &gt;&gt;&gt; out = fm(X)\n    \"\"\"\n\n    def __init__(\n        self,\n        *,\n        column_idx: Dict[str, int],\n        num_factors: int,\n        reduce_sum: bool = True,\n        cat_embed_input: Optional[List[Tuple[str, int]]] = None,\n        cat_embed_dropout: Optional[float] = None,\n        use_cat_bias: Optional[bool] = None,\n        cat_embed_activation: Optional[str] = None,\n        continuous_cols: Optional[List[str]] = None,\n        cont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\n        embed_continuous_method: Optional[\n            Literal[\"piecewise\", \"periodic\", \"standard\"]\n        ] = \"standard\",\n        cont_embed_dropout: Optional[float] = None,\n        cont_embed_activation: Optional[str] = None,\n        quantization_setup: Optional[Dict[str, List[float]]] = None,\n        n_frequencies: Optional[int] = None,\n        sigma: Optional[float] = None,\n        share_last_layer: Optional[bool] = None,\n        full_embed_dropout: Optional[bool] = None,\n        mlp_hidden_dims: Optional[List[int]] = None,\n        mlp_activation: Optional[str] = None,\n        mlp_dropout: Optional[float] = None,\n        mlp_batchnorm: Optional[bool] = None,\n        mlp_batchnorm_last: Optional[bool] = None,\n        mlp_linear_first: Optional[bool] = None,\n    ):\n        super(DeepFactorizationMachine, self).__init__(\n            column_idx=column_idx,\n            input_dim=num_factors,\n            cat_embed_input=cat_embed_input,\n            cat_embed_dropout=cat_embed_dropout,\n            use_cat_bias=use_cat_bias,\n            cat_embed_activation=cat_embed_activation,\n            shared_embed=False,\n            add_shared_embed=None,\n            frac_shared_embed=None,\n            continuous_cols=continuous_cols,\n            cont_norm_layer=cont_norm_layer,\n            embed_continuous_method=embed_continuous_method,\n            cont_embed_dropout=cont_embed_dropout,\n            cont_embed_activation=cont_embed_activation,\n            quantization_setup=quantization_setup,\n            n_frequencies=n_frequencies,\n            sigma=sigma,\n            share_last_layer=share_last_layer,\n            full_embed_dropout=full_embed_dropout,\n        )\n\n        self.reduce_sum = reduce_sum\n\n        self.mlp_hidden_dims = mlp_hidden_dims\n        self.mlp_activation = mlp_activation\n        self.mlp_dropout = mlp_dropout\n        self.mlp_batchnorm = mlp_batchnorm\n        self.mlp_batchnorm_last = mlp_batchnorm_last\n        self.mlp_linear_first = mlp_linear_first\n\n        if self.mlp_hidden_dims is not None:\n\n            if self.mlp_hidden_dims[-1] != self.input_dim:\n                d_hidden = (\n                    [self.input_dim * len(self.column_idx)]\n                    + self.mlp_hidden_dims\n                    + [self.input_dim]\n                )\n            else:\n                d_hidden = [\n                    self.input_dim * len(self.column_idx)\n                ] + self.mlp_hidden_dims\n\n            self.mlp = MLP(\n                d_hidden=d_hidden,\n                activation=(\n                    \"relu\" if self.mlp_activation is None else self.mlp_activation\n                ),\n                dropout=0.0 if self.mlp_dropout is None else self.mlp_dropout,\n                batchnorm=False if self.mlp_batchnorm is None else self.mlp_batchnorm,\n                batchnorm_last=(\n                    False\n                    if self.mlp_batchnorm_last is None\n                    else self.mlp_batchnorm_last\n                ),\n                linear_first=(\n                    False if self.mlp_linear_first is None else self.mlp_linear_first\n                ),\n            )\n        else:\n            self.mlp = None\n\n    def forward(self, X: Tensor) -&gt; Tensor:\n        embed = self._get_embeddings(X)\n        fm_output = factorization_machine(embed, self.reduce_sum)\n\n        if self.mlp is None:\n            return fm_output\n\n        mlp_input = embed.view(embed.size(0), -1)\n        mlp_output = self.mlp(mlp_input)\n\n        if self.reduce_sum:\n            mlp_output = mlp_output.sum(1, keepdim=True)\n\n        return fm_output + mlp_output\n\n    @property\n    def output_dim(self) -&gt; int:\n        if self.reduce_sum:\n            return 1\n        else:\n            return self.input_dim\n</code></pre>"},{"location":"pytorch-widedeep/the_rec_module.html#pytorch_widedeep.models.rec.deepffm.DeepFieldAwareFactorizationMachine","title":"DeepFieldAwareFactorizationMachine","text":"<p>               Bases: <code>BaseTabularModelWithAttention</code></p> <p>Deep Field Aware Factorization Machine (DeepFFM) for recommendation systems. Adaptation of the paper 'Field-aware Factorization Machines in a Real-world Online Advertising System', Juan et al. 2017.</p> <p>This class implements only the 'Deep' component of the model described in the paper. The linear component is not implemented 'internally' and, if one wants to include it, it can be easily added using the 'wide'/linear component in this library. See the examples in the examples folder.</p> <p>Note that in this case, only categorical features are accepted. This is because the embeddings of each feature will be learned using all other features. Therefore these embeddings have to be all of the same nature. This does not occur if we mix categorical and continuous features.</p> <p>Parameters:</p> Name Type Description Default <code>column_idx</code> <code>Dict[str, int]</code> <p>Dictionary mapping column names to their corresponding index.</p> required <code>num_factors</code> <code>int</code> <p>Number of factors for the factorization machine.</p> required <code>reduce_sum</code> <code>bool</code> <p>Whether to reduce the sum in the factorization machine output.</p> <code>True</code> <code>cat_embed_input</code> <code>Optional[List[Tuple[str, int]]]</code> <p>List of tuples with categorical column names and number of unique values.</p> <code>None</code> <code>cat_embed_dropout</code> <code>Optional[float]</code> <p>Categorical embeddings dropout. If <code>None</code>, it will default to 0.</p> <code>None</code> <code>use_cat_bias</code> <code>Optional[bool]</code> <p>Boolean indicating if bias will be used for the categorical embeddings. If <code>None</code>, it will default to 'False'.</p> <code>None</code> <code>cat_embed_activation</code> <code>Optional[str]</code> <p>Activation function for the categorical embeddings, if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> <code>None</code> <code>mlp_hidden_dims</code> <code>Optional[List[int]]</code> <p>List with the number of neurons per dense layer in the mlp.</p> <code>None</code> <code>mlp_activation</code> <code>Optional[str]</code> <p>Activation function for the dense layers of the MLP. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> <code>None</code> <code>mlp_dropout</code> <code>Optional[float]</code> <p>float or List of floats with the dropout between the dense layers. e.g: [0.5,0.5]</p> <code>None</code> <code>mlp_batchnorm</code> <code>Optional[bool]</code> <p>Boolean indicating whether or not batch normalization will be applied to the dense layers</p> <code>None</code> <code>mlp_batchnorm_last</code> <code>Optional[bool]</code> <p>Boolean indicating whether or not batch normalization will be applied to the last of the dense layers</p> <code>None</code> <code>mlp_linear_first</code> <code>Optional[bool]</code> <p>Boolean indicating the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code></p> <code>None</code> <p>Attributes:</p> Name Type Description <code>n_features</code> <code>int</code> <p>Number of unique features/columns</p> <code>n_tokens</code> <code>int</code> <p>Number of unique values (tokens) in the full dataset (corpus)</p> <code>encoders</code> <code>ModuleList</code> <p>List of <code>BaseTabularModelWithAttention</code> instances. One per categorical column</p> <code>mlp</code> <code>Module</code> <p>Multi-layer perceptron. If <code>None</code> the output will be the output of the factorization machine (i.e. the sum of the interactions)</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from torch import Tensor\n&gt;&gt;&gt; from typing import Dict, List, Tuple\n&gt;&gt;&gt; from pytorch_widedeep.models.rec import DeepFieldAwareFactorizationMachine\n&gt;&gt;&gt; X = torch.randint(0, 10, (16, 2))\n&gt;&gt;&gt; column_idx: Dict[str, int] = {\"col1\": 0, \"col2\": 1}\n&gt;&gt;&gt; cat_embed_input: List[Tuple[str, int]] = [(\"col1\", 10), (\"col2\", 10)]\n&gt;&gt;&gt; ffm = DeepFieldAwareFactorizationMachine(\n...     column_idx=column_idx,\n...     num_factors=4,\n...     cat_embed_input=cat_embed_input,\n...     mlp_hidden_dims=[16, 8]\n... )\n&gt;&gt;&gt; output = ffm(X)\n</code></pre> Source code in <code>pytorch_widedeep/models/rec/deepffm.py</code> <pre><code>class DeepFieldAwareFactorizationMachine(BaseTabularModelWithAttention):\n    \"\"\"\n    Deep Field Aware Factorization Machine (DeepFFM) for recommendation\n    systems. Adaptation of the paper 'Field-aware Factorization Machines in a\n    Real-world Online Advertising System', Juan et al. 2017.\n\n    This class implements only the 'Deep' component of the model described in\n    the paper. The linear component is not implemented 'internally' and, if\n    one wants to include it, it can be easily added using the 'wide'/linear\n    component in this library. See the examples in the examples folder.\n\n    Note that in this case, only categorical features are accepted. This is\n    because the embeddings of each feature will be learned using all other\n    features. Therefore these embeddings have to be all of the same nature.\n    This does not occur if we mix categorical and continuous features.\n\n    Parameters\n    ----------\n    column_idx : Dict[str, int]\n        Dictionary mapping column names to their corresponding index.\n    num_factors : int\n        Number of factors for the factorization machine.\n    reduce_sum : bool, default=True\n        Whether to reduce the sum in the factorization machine output.\n    cat_embed_input : Optional[List[Tuple[str, int]]], default=None\n        List of tuples with categorical column names and number of unique values.\n    cat_embed_dropout : Optional[float], default=None\n        Categorical embeddings dropout. If `None`, it will default\n        to 0.\n    use_cat_bias : Optional[bool], default=None\n        Boolean indicating if bias will be used for the categorical embeddings.\n        If `None`, it will default to 'False'.\n    cat_embed_activation : Optional[str], default=None\n        Activation function for the categorical embeddings, if any. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported\n    mlp_hidden_dims: List, default = [200, 100]\n        List with the number of neurons per dense layer in the mlp.\n    mlp_activation: str, default = \"relu\"\n        Activation function for the dense layers of the MLP. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported\n    mlp_dropout: float or List, default = 0.1\n        float or List of floats with the dropout between the dense layers.\n        e.g: _[0.5,0.5]_\n    mlp_batchnorm: bool, default = False\n        Boolean indicating whether or not batch normalization will be applied\n        to the dense layers\n    mlp_batchnorm_last: bool, default = False\n        Boolean indicating whether or not batch normalization will be applied\n        to the last of the dense layers\n    mlp_linear_first: bool, default = False\n        Boolean indicating the order of the operations in the dense\n        layer. If `True: [LIN -&gt; ACT -&gt; BN -&gt; DP]`. If `False: [BN -&gt; DP -&gt;\n        LIN -&gt; ACT]`\n\n    Attributes\n    ----------\n    n_features: int\n        Number of unique features/columns\n    n_tokens: int\n        Number of unique values (tokens) in the full dataset (corpus)\n    encoders: nn.ModuleList\n        List of `BaseTabularModelWithAttention` instances. One per categorical\n        column\n    mlp: nn.Module\n        Multi-layer perceptron. If `None` the output will be the output of the\n        factorization machine (i.e. the sum of the interactions)\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from torch import Tensor\n    &gt;&gt;&gt; from typing import Dict, List, Tuple\n    &gt;&gt;&gt; from pytorch_widedeep.models.rec import DeepFieldAwareFactorizationMachine\n    &gt;&gt;&gt; X = torch.randint(0, 10, (16, 2))\n    &gt;&gt;&gt; column_idx: Dict[str, int] = {\"col1\": 0, \"col2\": 1}\n    &gt;&gt;&gt; cat_embed_input: List[Tuple[str, int]] = [(\"col1\", 10), (\"col2\", 10)]\n    &gt;&gt;&gt; ffm = DeepFieldAwareFactorizationMachine(\n    ...     column_idx=column_idx,\n    ...     num_factors=4,\n    ...     cat_embed_input=cat_embed_input,\n    ...     mlp_hidden_dims=[16, 8]\n    ... )\n    &gt;&gt;&gt; output = ffm(X)\n    \"\"\"\n\n    def __init__(\n        self,\n        *,\n        column_idx: Dict[str, int],\n        num_factors: int,\n        cat_embed_input: List[Tuple[str, int]],\n        reduce_sum: bool = True,\n        cat_embed_dropout: Optional[float] = None,\n        use_cat_bias: Optional[bool] = None,\n        cat_embed_activation: Optional[str] = None,\n        mlp_hidden_dims: Optional[List[int]] = None,\n        mlp_activation: Optional[str] = None,\n        mlp_dropout: Optional[float] = None,\n        mlp_batchnorm: Optional[bool] = None,\n        mlp_batchnorm_last: Optional[bool] = None,\n        mlp_linear_first: Optional[bool] = None,\n    ):\n        super(DeepFieldAwareFactorizationMachine, self).__init__(\n            column_idx=column_idx,\n            input_dim=num_factors,\n            cat_embed_input=cat_embed_input,\n            cat_embed_dropout=cat_embed_dropout,\n            use_cat_bias=use_cat_bias,\n            cat_embed_activation=cat_embed_activation,\n            shared_embed=False,\n            add_shared_embed=None,\n            frac_shared_embed=None,\n            continuous_cols=None,\n            cont_norm_layer=None,\n            embed_continuous_method=None,\n            cont_embed_dropout=None,\n            cont_embed_activation=None,\n            quantization_setup=None,\n            n_frequencies=None,\n            sigma=None,\n            share_last_layer=None,\n            full_embed_dropout=None,\n        )\n\n        self.reduce_sum = reduce_sum\n\n        self.mlp_hidden_dims = mlp_hidden_dims\n        self.mlp_activation = mlp_activation\n        self.mlp_dropout = mlp_dropout\n        self.mlp_batchnorm = mlp_batchnorm\n        self.mlp_batchnorm_last = mlp_batchnorm_last\n        self.mlp_linear_first = mlp_linear_first\n\n        self.n_features = len(self.column_idx)\n        self.n_tokens = sum([ei[1] for ei in cat_embed_input])\n\n        self.encoders = nn.ModuleList(\n            [\n                BaseTabularModelWithAttention(**config)\n                for config in self._get_encoder_configs()\n            ]\n        )\n\n        if self.mlp_hidden_dims is not None:\n            d_hidden = [\n                self.n_features * (self.n_features - 1) // 2 * num_factors\n            ] + self.mlp_hidden_dims\n            self.mlp = MLP(\n                d_hidden=d_hidden,\n                activation=(\n                    \"relu\" if self.mlp_activation is None else self.mlp_activation\n                ),\n                dropout=0.0 if self.mlp_dropout is None else self.mlp_dropout,\n                batchnorm=False if self.mlp_batchnorm is None else self.mlp_batchnorm,\n                batchnorm_last=(\n                    False\n                    if self.mlp_batchnorm_last is None\n                    else self.mlp_batchnorm_last\n                ),\n                linear_first=(\n                    False if self.mlp_linear_first is None else self.mlp_linear_first\n                ),\n            )\n        else:\n            self.mlp = None\n\n    def forward(self, X: Tensor) -&gt; Tensor:\n\n        interactions_l: List[Tensor] = []\n        for i in range(len(self.column_idx)):\n            for j in range(i + 1, len(self.column_idx)):\n                # the syntax [i] and [j] is to keep the shape of the tensors\n                # as they are sliced within '_get_embeddings'. This will\n                # return a tensor of shape (b, 1, embed_dim). Then it has to\n                # be squeezed to (b, embed_dim)  before multiplied\n                embed_i = self.encoders[i]._get_embeddings(X[:, [i]]).squeeze(1)\n                embed_j = self.encoders[j]._get_embeddings(X[:, [j]]).squeeze(1)\n                interactions_l.append(embed_i * embed_j)\n\n        interactions = torch.cat(interactions_l, dim=1)\n\n        if self.mlp is not None:\n            interactions = interactions.view(X.size(0), -1)\n            deep_out = self.mlp(interactions)\n        else:\n            deep_out = interactions\n\n        if self.reduce_sum:\n            deep_out = deep_out.sum(dim=1, keepdim=True)\n\n        return deep_out\n\n    def _get_encoder_configs(self) -&gt; List[Dict[str, Any]]:\n        config: List[Dict[str, Any]] = []\n        for col, _ in self.column_idx.items():\n            cat_embed_input = [(col, self.n_tokens)]\n            _config = {\n                \"column_idx\": {col: 0},\n                \"input_dim\": self.input_dim,\n                \"cat_embed_input\": cat_embed_input,\n                \"cat_embed_dropout\": self.cat_embed_dropout,\n                \"use_cat_bias\": self.use_cat_bias,\n                \"cat_embed_activation\": self.cat_embed_activation,\n                \"shared_embed\": None,\n                \"add_shared_embed\": None,\n                \"frac_shared_embed\": None,\n                \"continuous_cols\": None,\n                \"cont_norm_layer\": None,\n                \"embed_continuous_method\": None,\n                \"cont_embed_dropout\": None,\n                \"cont_embed_activation\": None,\n                \"quantization_setup\": None,\n                \"n_frequencies\": None,\n                \"sigma\": None,\n                \"share_last_layer\": None,\n                \"full_embed_dropout\": None,\n            }\n\n            config.append(_config)\n\n        return config\n\n    @property\n    def output_dim(self) -&gt; int:\n        if self.reduce_sum:\n            return 1\n        elif self.mlp_hidden_dims is not None:\n            return self.mlp_hidden_dims[-1]\n        else:\n            return self.n_features * (self.n_features - 1) // 2 * self.input_dim\n</code></pre>"},{"location":"pytorch-widedeep/the_rec_module.html#pytorch_widedeep.models.rec.din.DeepInterestNetwork","title":"DeepInterestNetwork","text":"<p>               Bases: <code>BaseWDModelComponent</code></p> <p>Adaptation of the Deep Interest Network (DIN) for recommendation systems as described in the paper: 'Deep Interest Network for Click-Through Rate Prediction' by Guorui Zhou et al. 2018.</p> <p>Note that all the categorical- and continuous-related parameters refer to the categorical and continuous columns that are not part of the sequential columns and will be treated as standard tabular data.</p> <p>This model requires some specific data preparation that allows for quite a lot of flexibility. Therefore, I have included a preprocessor (<code>DINPreprocessor</code>) in the preprocessing module that will take care of the data preparation.</p> <p>Parameters:</p> Name Type Description Default <code>column_idx</code> <code>Dict[str, int]</code> <p>Dictionary mapping column names to their corresponding index.</p> required <code>target_item_col</code> <code>str</code> <p>Name of the target item column. Note that this is not the target column. This algorithm relies on a sequence representation of interactions. The target item would be the next item in the sequence of interactions (e.g. item 6<sup>th</sup> in a sequence of 5 items), and our goal is to predict a given action on it.</p> <code>'target_item'</code> <code>user_behavior_confiq</code> <code>Tuple[List[str], int, int]</code> <p>Configuration for user behavior sequence columns. Tuple containing: - List of column names that correspond to the user behavior sequence - Number of unique feature values (n_tokens) - Embedding dimension Example: <code>([\"item_1\", \"item_2\", \"item_3\"], 5, 8)</code></p> required <code>action_seq_config</code> <code>Optional[Tuple[List[str], int]]</code> <p>Configuration for a so-called action sequence columns (for example a rating, or purchased/not-purchased, etc). Tuple containing: - List of column names - Number of unique feature values (n_tokens) This action will always be learned as a 1d embedding and will be combined with the user behaviour. For example, imagine that the action is purchased/not-purchased. then per item in the user behaviour sequence there will be a binary action to learn 0/1. Such action will be represented by a float number that will multiply the corresponding item embedding in the user behaviour sequence. Example: <code>([\"rating_1\", \"rating_2\", \"rating_3\"], 5)</code> Internally, the embedding dimension will be set to 1</p> <code>None</code> <code>other_seq_cols_confiq</code> <code>Optional[List[Tuple[List[str], int, int]]]</code> <p>Configuration for other sequential columns. List of tuples containing: - List of column names that correspond to the sequential column - Number of unique feature values (n_tokens) - Embedding dimension Example: <code>[([\"seq1_col1\", \"seq1_col2\"], 5, 8), ([\"seq2_col1\", \"seq2_col2\"], 5, 8)]</code></p> <code>None</code> <code>attention_unit_activation</code> <code>Literal['prelu', 'dice']</code> <p>Activation function to use in the attention unit.</p> <code>\"prelu\"</code> <code>cat_embed_input</code> <code>Optional[List[Tuple[str, int, int]]]</code> <p>Configuration for other columns. List of tuples containing: - Column name - Number of unique feature values (n_tokens) - Embedding dimension</p> <p>Note: From here in advance the remaining parameters are related to the categorical and continuous columns that are not part of the sequential columns and will be treated as standard tabular data.</p> <code>None</code> <code>cat_embed_dropout</code> <code>Optional[float]</code> <p>Categorical embeddings dropout. If <code>None</code>, it will default to 0.</p> <code>None</code> <code>use_cat_bias</code> <code>Optional[bool]</code> <p>Boolean indicating if bias will be used for the categorical embeddings. If <code>None</code>, it will default to 'False'.</p> <code>None</code> <code>cat_embed_activation</code> <code>Optional[str]</code> <p>Activation function for the categorical embeddings, if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> <code>None</code> <code>continuous_cols</code> <code>Optional[List[str]]</code> <p>List with the name of the numeric (aka continuous) columns</p> <code>None</code> <code>cont_norm_layer</code> <code>Optional[Literal['batchnorm', 'layernorm']]</code> <p>Type of normalization layer applied to the continuous features. Options are: 'layernorm' and 'batchnorm'. if <code>None</code>, no normalization layer will be used.</p> <code>None</code> <code>embed_continuous</code> <code>Optional[bool]</code> <p>Boolean indicating if the continuous columns will be embedded using one of the available methods: 'standard', 'periodic' or 'piecewise'. If <code>None</code>, it will default to 'False'. NOTE: This parameter is deprecated and it  will be removed in future releases. Please, use the  <code>embed_continuous_method</code> parameter instead.</p> <code>None</code> <code>embed_continuous_method</code> <code>Optional[Literal['piecewise', 'periodic', 'standard']]</code> <p>Method to use to embed the continuous features. Options are: 'standard', 'periodic' or 'piecewise'. The 'standard' embedding method is based on the FT-Transformer implementation presented in the paper: Revisiting Deep Learning Models for Tabular Data. The 'periodic' and_'piecewise'_ methods were presented in the paper: On Embeddings for Numerical Features in Tabular Deep Learning. Please, read the papers for details.</p> <code>None</code> <code>cont_embed_dim</code> <code>Optional[int]</code> <p>Size of the continuous embeddings. If the continuous columns are embedded, <code>cont_embed_dim</code> must be passed.</p> <code>None</code> <code>cont_embed_dropout</code> <code>Optional[float]</code> <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p> <code>None</code> <code>cont_embed_activation</code> <code>Optional[str]</code> <p>Activation function for the continuous embeddings if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported. If <code>None</code>, no activation function will be applied.</p> <code>None</code> <code>quantization_setup</code> <code>Optional[Dict[str, List[float]]]</code> <p>This parameter is used when the 'piecewise' method is used to embed the continuous cols. It is a dict where keys are the name of the continuous columns and values are lists with the boundaries for the quantization of the continuous_cols. See the examples for details. If If the 'piecewise' method is used, this parameter is required.</p> <code>None</code> <code>n_frequencies</code> <code>Optional[int]</code> <p>This is the so called 'k' in their paper On Embeddings for Numerical Features in Tabular Deep Learning, and is the number of 'frequencies' that will be used to represent each continuous column. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>sigma</code> <code>Optional[float]</code> <p>This is the sigma parameter in the paper mentioned when describing the previous parameters and it is used to initialise the 'frequency weights'. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>share_last_layer</code> <code>Optional[bool]</code> <p>This parameter is not present in the before mentioned paper but it is implemented in the official repo. If <code>True</code> the linear layer that turns the frequencies into embeddings will be shared across the continuous columns. If <code>False</code> a different linear layer will be used for each continuous column. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>full_embed_dropout</code> <code>Optional[bool]</code> <p>If <code>True</code>, the full embedding corresponding to a column will be masked out/dropout. If <code>None</code>, it will default to <code>False</code>.</p> <code>None</code> <code>mlp_hidden_dims</code> <code>Optional[List[int]]</code> <p>List with the number of neurons per dense layer in the mlp.</p> <code>None</code> <code>mlp_activation</code> <code>Optional[str]</code> <p>Activation function for the dense layers of the MLP. Currently 'tanh', 'relu', 'leaky_relu', 'gelu' and 'preglu' are   supported</p> <code>None</code> <code>mlp_dropout</code> <code>Optional[float]</code> <p>float or List of floats with the dropout between the dense layers. e.g: [0.5,0.5]</p> <code>None</code> <code>mlp_batchnorm</code> <code>Optional[bool]</code> <p>Boolean indicating whether or not batch normalization will be applied to the dense layers</p> <code>None</code> <code>mlp_batchnorm_last</code> <code>Optional[bool]</code> <p>Boolean indicating whether or not batch normalization will be applied to the last of the dense layers</p> <code>None</code> <code>mlp_linear_first</code> <code>Optional[bool]</code> <p>Boolean indicating the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code></p> <code>None</code> <p>Attributes:</p> Name Type Description <code>user_behavior_indexes</code> <code>List[int]</code> <p>List with the indexes of the user behavior columns</p> <code>user_behavior_embed</code> <code>BaseTabularModelWithAttention</code> <p>Embedding layer for the user</p> <code>action_seq_indexes</code> <code>List[int]</code> <p>List with the indexes of the rating sequence columns if the action_seq_config parameter is not None</p> <code>action_embed</code> <code>BaseTabularModelWithAttention</code> <p>Embedding layer for the rating sequence columns if the action_seq_config parameter is not None</p> <code>other_seq_cols_indexes</code> <code>Dict[str, List[int]]</code> <p>Dictionary with the indexes of the other sequential columns if the other_seq_cols_confiq parameter is not None</p> <code>other_seq_cols_embed</code> <code>ModuleDict</code> <p>Dictionary with the embedding layers for the other sequential columns if the other_seq_cols_confiq parameter is not None</p> <code>other_cols_idx</code> <code>List[int]</code> <p>List with the indexes of the other columns if the other_cols_config parameter is not None</p> <code>other_col_embed</code> <code>BaseTabularModel</code> <p>Embedding layer for the other columns if the other_cols_config parameter is not None</p> <code>mlp</code> <code>Optional[MLP]</code> <p>MLP component of the model. If None, no MLP will be used. This should almost always be not None.</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; import numpy as np\n&gt;&gt;&gt; from torch import Tensor\n&gt;&gt;&gt; from typing import Dict, List, Tuple\n&gt;&gt;&gt; from pytorch_widedeep.models.rec import DeepInterestNetwork\n&gt;&gt;&gt; np_seed = np.random.seed(42)\n&gt;&gt;&gt; torch_seed = torch.manual_seed(42)\n&gt;&gt;&gt; num_users = 10\n&gt;&gt;&gt; num_items = 5\n&gt;&gt;&gt; num_contexts = 3\n&gt;&gt;&gt; seq_length = 3\n&gt;&gt;&gt; num_samples = 10\n&gt;&gt;&gt; user_ids = np.random.randint(0, num_users, num_samples)\n&gt;&gt;&gt; target_item_ids = np.random.randint(0, num_items, num_samples)\n&gt;&gt;&gt; context_ids = np.random.randint(0, num_contexts, num_samples)\n&gt;&gt;&gt; user_behavior = np.array(\n...     [\n...         np.random.choice(num_items, seq_length, replace=False)\n...         for _ in range(num_samples)\n...     ]\n... )\n&gt;&gt;&gt; X_arr = np.column_stack((user_ids, target_item_ids, context_ids, user_behavior))\n&gt;&gt;&gt; X = torch.tensor(X_arr, dtype=torch.long)\n&gt;&gt;&gt; column_idx: Dict[str, int] = {\n...     \"user_id\": 0,\n...     \"target_item\": 1,\n...     \"context\": 2,\n...     \"item_1\": 3,\n...     \"item_2\": 4,\n...     \"item_3\": 5,\n... }\n&gt;&gt;&gt; user_behavior_config: Tuple[List[str], int, int] = (\n...     [\"item_1\", \"item_2\", \"item_3\"],\n...     num_items,\n...     8,\n... )\n&gt;&gt;&gt; cat_embed_input: List[Tuple[str, int, int]] = [\n...     (\"user_id\", num_users, 8),\n...     (\"context\", num_contexts, 4),\n... ]\n&gt;&gt;&gt; model = DeepInterestNetwork(\n...     column_idx=column_idx,\n...     target_item_col=\"target_item\",\n...     user_behavior_confiq=user_behavior_config,\n...     cat_embed_input=cat_embed_input,\n...     mlp_hidden_dims=[16, 8],\n... )\n&gt;&gt;&gt; output = model(X)\n</code></pre> Source code in <code>pytorch_widedeep/models/rec/din.py</code> <pre><code>class DeepInterestNetwork(BaseWDModelComponent):\n    \"\"\"\n    Adaptation of the Deep Interest Network (DIN) for recommendation systems\n    as described in the paper: 'Deep Interest Network for Click-Through Rate\n    Prediction' by Guorui Zhou et al. 2018.\n\n    Note that all the categorical- and continuous-related parameters refer to\n    the categorical and continuous columns that are not part of the\n    sequential columns and will be treated as standard tabular data.\n\n    This model requires some specific data preparation that allows for quite a\n    lot of flexibility. Therefore, I have included a preprocessor\n    (`DINPreprocessor`) in the preprocessing module that will take care of\n    the data preparation.\n\n    Parameters\n    ----------\n    column_idx : Dict[str, int]\n        Dictionary mapping column names to their corresponding index.\n    target_item_col : str\n        Name of the target item column. Note that this is not the target\n        column. This algorithm relies on a sequence representation of\n        interactions. The target item would be the next item in the sequence\n        of interactions (e.g. item 6th in a sequence of 5 items), and our\n        goal is to predict a given action on it.\n    user_behavior_confiq : Tuple[List[str], int, int]\n        Configuration for user behavior sequence columns. Tuple containing:\n        - List of column names that correspond to the user behavior sequence&lt;br/&gt;\n        - Number of unique feature values (n_tokens)&lt;br/&gt;\n        - Embedding dimension&lt;br/&gt;\n        Example: `([\"item_1\", \"item_2\", \"item_3\"], 5, 8)`\n    action_seq_config : Optional[Tuple[List[str], int]], default=None\n        Configuration for a so-called action sequence columns (for example a\n        rating, or purchased/not-purchased, etc). Tuple containing:&lt;br/&gt;\n        - List of column names&lt;br/&gt;\n        - Number of unique feature values (n_tokens)&lt;br/&gt;\n        This action will **always** be learned as a 1d embedding and will be\n        combined with the user behaviour. For example, imagine that the\n        action is purchased/not-purchased. then per item in the user\n        behaviour sequence there will be a binary action to learn 0/1. Such\n        action will be represented by a float number that will multiply the\n        corresponding item embedding in the user behaviour sequence.&lt;br/&gt;\n        Example: `([\"rating_1\", \"rating_2\", \"rating_3\"], 5)`&lt;br/&gt;\n        Internally, the embedding dimension will be set to 1\n    other_seq_cols_confiq : Optional[List[Tuple[List[str], int, int]]], default=None\n        Configuration for other sequential columns. List of tuples containing:&lt;br/&gt;\n        - List of column names that correspond to the sequential column&lt;br/&gt;\n        - Number of unique feature values (n_tokens)&lt;br/&gt;\n        - Embedding dimension&lt;br/&gt;\n        Example: `[([\"seq1_col1\", \"seq1_col2\"], 5, 8), ([\"seq2_col1\", \"seq2_col2\"], 5, 8)]`\n    attention_unit_activation : Literal[\"prelu\", \"dice\"], default=\"prelu\"\n        Activation function to use in the attention unit.\n    cat_embed_input : Optional[List[Tuple[str, int, int]]], default=None\n        Configuration for other columns. List of tuples containing:&lt;br/&gt;\n        - Column name&lt;br/&gt;\n        - Number of unique feature values (n_tokens)&lt;br/&gt;\n        - Embedding dimension&lt;br/&gt;\n\n        **Note**: From here in advance the remaining parameters are related to\n        the categorical and continuous columns that are not part of the\n        sequential columns and will be treated as standard tabular data.\n\n    cat_embed_dropout : Optional[float], default=None\n        Categorical embeddings dropout. If `None`, it will default\n        to 0.\n    use_cat_bias : Optional[bool], default=None\n        Boolean indicating if bias will be used for the categorical embeddings.\n        If `None`, it will default to 'False'.\n    cat_embed_activation : Optional[str], default=None\n        Activation function for the categorical embeddings, if any. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported\n    continuous_cols : Optional[List[str]], default=None\n        List with the name of the numeric (aka continuous) columns\n    cont_norm_layer : Optional[Literal[\"batchnorm\", \"layernorm\"]], default=None\n        Type of normalization layer applied to the continuous features.\n        Options are: _'layernorm'_ and _'batchnorm'_. if `None`, no\n        normalization layer will be used.\n    embed_continuous : Optional[bool], default=None\n        Boolean indicating if the continuous columns will be embedded using\n        one of the available methods: _'standard'_, _'periodic'_\n        or _'piecewise'_. If `None`, it will default to 'False'.&lt;br/&gt;\n        :information_source: **NOTE**: This parameter is deprecated and it\n         will be removed in future releases. Please, use the\n         `embed_continuous_method` parameter instead.\n    embed_continuous_method : Optional[Literal[\"piecewise\", \"periodic\", \"standard\"]], default=None\n        Method to use to embed the continuous features. Options are:\n        _'standard'_, _'periodic'_ or _'piecewise'_. The _'standard'_\n        embedding method is based on the FT-Transformer implementation\n        presented in the paper: [Revisiting Deep Learning Models for\n        Tabular Data](https://arxiv.org/abs/2106.11959v5). The _'periodic'_\n        and_'piecewise'_ methods were presented in the paper: [On Embeddings for\n        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556).\n        Please, read the papers for details.\n    cont_embed_dim : Optional[int], default=None\n        Size of the continuous embeddings. If the continuous columns are\n        embedded, `cont_embed_dim` must be passed.\n    cont_embed_dropout : Optional[float], default=None\n        Dropout for the continuous embeddings. If `None`, it will default to 0.0\n    cont_embed_activation : Optional[str], default=None\n        Activation function for the continuous embeddings if any. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported.\n        If `None`, no activation function will be applied.\n    quantization_setup : Optional[Dict[str, List[float]]], default=None\n        This parameter is used when the _'piecewise'_ method is used to embed\n        the continuous cols. It is a dict where keys are the name of the continuous\n        columns and values are lists with the boundaries for the quantization\n        of the continuous_cols. See the examples for details. If\n        If the _'piecewise'_ method is used, this parameter is required.\n    n_frequencies : Optional[int], default=None\n        This is the so called _'k'_ in their paper [On Embeddings for\n        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556),\n        and is the number of 'frequencies' that will be used to represent each\n        continuous column. See their Eq 2 in the paper for details. If\n        the _'periodic'_ method is used, this parameter is required.\n    sigma : Optional[float], default=None\n        This is the sigma parameter in the paper mentioned when describing the\n        previous parameters and it is used to initialise the 'frequency\n        weights'. See their Eq 2 in the paper for details. If\n        the _'periodic'_ method is used, this parameter is required.\n    share_last_layer : Optional[bool], default=None\n        This parameter is not present in the before mentioned paper but it is implemented in\n        the [official repo](https://github.com/yandex-research/rtdl-num-embeddings/tree/main).\n        If `True` the linear layer that turns the frequencies into embeddings\n        will be shared across the continuous columns. If `False` a different\n        linear layer will be used for each continuous column.\n        If the _'periodic'_ method is used, this parameter is required.\n    full_embed_dropout: bool, Optional, default = None,\n        If `True`, the full embedding corresponding to a column will be masked\n        out/dropout. If `None`, it will default to `False`.\n    mlp_hidden_dims: List, default = [200, 100]\n        List with the number of neurons per dense layer in the mlp.\n    mlp_activation: str, default = \"relu\"\n        Activation function for the dense layers of the MLP. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_, _'gelu'_ and _'preglu'_ are\n          supported\n    mlp_dropout: float or List, default = 0.1\n        float or List of floats with the dropout between the dense layers.\n        e.g: _[0.5,0.5]_\n    mlp_batchnorm: bool, default = False\n        Boolean indicating whether or not batch normalization will be applied\n        to the dense layers\n    mlp_batchnorm_last: bool, default = False\n        Boolean indicating whether or not batch normalization will be applied\n        to the last of the dense layers\n    mlp_linear_first: bool, default = False\n        Boolean indicating the order of the operations in the dense\n        layer. If `True: [LIN -&gt; ACT -&gt; BN -&gt; DP]`. If `False: [BN -&gt; DP -&gt;\n        LIN -&gt; ACT]`\n\n    Attributes\n    ----------\n    user_behavior_indexes: List[int]\n        List with the indexes of the user behavior columns\n    user_behavior_embed: BaseTabularModelWithAttention\n        Embedding layer for the user\n    action_seq_indexes: List[int]\n        List with the indexes of the rating sequence columns if the\n        action_seq_config parameter is not None\n    action_embed: BaseTabularModelWithAttention\n        Embedding layer for the rating sequence columns if the\n        action_seq_config parameter is not None\n    other_seq_cols_indexes: Dict[str, List[int]]\n        Dictionary with the indexes of the other sequential columns if the\n        other_seq_cols_confiq parameter is not None\n    other_seq_cols_embed: nn.ModuleDict\n        Dictionary with the embedding layers for the other sequential columns\n        if the other_seq_cols_confiq parameter is not None\n    other_cols_idx: List[int]\n        List with the indexes of the other columns if the other_cols_config\n        parameter is not None\n    other_col_embed: BaseTabularModel\n        Embedding layer for the other columns if the other_cols_config\n        parameter is not None\n    mlp: Optional[MLP]\n        MLP component of the model. If None, no MLP will be used. This should\n        almost always be not None.\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; import numpy as np\n    &gt;&gt;&gt; from torch import Tensor\n    &gt;&gt;&gt; from typing import Dict, List, Tuple\n    &gt;&gt;&gt; from pytorch_widedeep.models.rec import DeepInterestNetwork\n    &gt;&gt;&gt; np_seed = np.random.seed(42)\n    &gt;&gt;&gt; torch_seed = torch.manual_seed(42)\n    &gt;&gt;&gt; num_users = 10\n    &gt;&gt;&gt; num_items = 5\n    &gt;&gt;&gt; num_contexts = 3\n    &gt;&gt;&gt; seq_length = 3\n    &gt;&gt;&gt; num_samples = 10\n    &gt;&gt;&gt; user_ids = np.random.randint(0, num_users, num_samples)\n    &gt;&gt;&gt; target_item_ids = np.random.randint(0, num_items, num_samples)\n    &gt;&gt;&gt; context_ids = np.random.randint(0, num_contexts, num_samples)\n    &gt;&gt;&gt; user_behavior = np.array(\n    ...     [\n    ...         np.random.choice(num_items, seq_length, replace=False)\n    ...         for _ in range(num_samples)\n    ...     ]\n    ... )\n    &gt;&gt;&gt; X_arr = np.column_stack((user_ids, target_item_ids, context_ids, user_behavior))\n    &gt;&gt;&gt; X = torch.tensor(X_arr, dtype=torch.long)\n    &gt;&gt;&gt; column_idx: Dict[str, int] = {\n    ...     \"user_id\": 0,\n    ...     \"target_item\": 1,\n    ...     \"context\": 2,\n    ...     \"item_1\": 3,\n    ...     \"item_2\": 4,\n    ...     \"item_3\": 5,\n    ... }\n    &gt;&gt;&gt; user_behavior_config: Tuple[List[str], int, int] = (\n    ...     [\"item_1\", \"item_2\", \"item_3\"],\n    ...     num_items,\n    ...     8,\n    ... )\n    &gt;&gt;&gt; cat_embed_input: List[Tuple[str, int, int]] = [\n    ...     (\"user_id\", num_users, 8),\n    ...     (\"context\", num_contexts, 4),\n    ... ]\n    &gt;&gt;&gt; model = DeepInterestNetwork(\n    ...     column_idx=column_idx,\n    ...     target_item_col=\"target_item\",\n    ...     user_behavior_confiq=user_behavior_config,\n    ...     cat_embed_input=cat_embed_input,\n    ...     mlp_hidden_dims=[16, 8],\n    ... )\n    &gt;&gt;&gt; output = model(X)\n    \"\"\"\n\n    def __init__(\n        self,\n        *,\n        column_idx: Dict[str, int],\n        user_behavior_confiq: Tuple[List[str], int, int],\n        target_item_col: str = \"target_item\",\n        action_seq_config: Optional[Tuple[List[str], int]] = None,\n        other_seq_cols_confiq: Optional[List[Tuple[List[str], int, int]]] = None,\n        attention_unit_activation: Literal[\"prelu\", \"dice\"] = \"prelu\",\n        cat_embed_input: Optional[List[Tuple[str, int, int]]] = None,\n        cat_embed_dropout: Optional[float] = None,\n        use_cat_bias: Optional[bool] = None,\n        cat_embed_activation: Optional[str] = None,\n        continuous_cols: Optional[List[str]] = None,\n        cont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\n        embed_continuous: Optional[bool] = None,\n        embed_continuous_method: Optional[\n            Literal[\"standard\", \"piecewise\", \"periodic\"]\n        ] = None,\n        cont_embed_dim: Optional[int] = None,\n        cont_embed_dropout: Optional[float] = None,\n        cont_embed_activation: Optional[str] = None,\n        quantization_setup: Optional[Dict[str, List[float]]] = None,\n        n_frequencies: Optional[int] = None,\n        sigma: Optional[float] = None,\n        share_last_layer: Optional[bool] = None,\n        full_embed_dropout: Optional[bool] = None,\n        mlp_hidden_dims: Optional[List[int]] = None,\n        mlp_activation: Optional[str] = None,\n        mlp_dropout: Optional[float] = None,\n        mlp_batchnorm: Optional[bool] = None,\n        mlp_batchnorm_last: Optional[bool] = None,\n        mlp_linear_first: Optional[bool] = None,\n    ):\n        super(DeepInterestNetwork, self).__init__()\n\n        self.column_idx = {\n            k: v for k, v in sorted(column_idx.items(), key=lambda x: x[1])\n        }\n\n        self.column_idx = column_idx\n        self.target_item_col = target_item_col\n        self.user_behavior_confiq = user_behavior_confiq\n        self.action_seq_config = (\n            (action_seq_config[0], action_seq_config[1], 1)\n            if action_seq_config is not None\n            else None\n        )\n        self.other_seq_cols_confiq = other_seq_cols_confiq\n        self.cat_embed_input = cat_embed_input\n        self.attention_unit_activation = attention_unit_activation\n\n        self.cat_embed_dropout = cat_embed_dropout\n        self.use_cat_bias = use_cat_bias\n        self.cat_embed_activation = cat_embed_activation\n        self.continuous_cols = continuous_cols\n        self.cont_norm_layer = cont_norm_layer\n        self.embed_continuous = embed_continuous\n        self.embed_continuous_method = embed_continuous_method\n        self.cont_embed_dim = cont_embed_dim\n        self.cont_embed_dropout = cont_embed_dropout\n        self.cont_embed_activation = cont_embed_activation\n        self.quantization_setup = quantization_setup\n        self.n_frequencies = n_frequencies\n        self.sigma = sigma\n        self.share_last_layer = share_last_layer\n        self.full_embed_dropout = full_embed_dropout\n\n        self.mlp_hidden_dims = mlp_hidden_dims\n        self.mlp_activation = mlp_activation\n        self.mlp_dropout = mlp_dropout\n        self.mlp_batchnorm = mlp_batchnorm\n        self.mlp_batchnorm_last = mlp_batchnorm_last\n        self.mlp_linear_first = mlp_linear_first\n\n        self.target_item_idx = self.column_idx[target_item_col]\n\n        self.user_behavior_indexes, self.user_behavior_embed = (\n            self.set_user_behavior_indexes_and_embed(user_behavior_confiq)\n        )\n        self.user_behavior_dim = user_behavior_confiq[2]\n\n        if self.action_seq_config is not None:\n            self.action_seq_indexes, self.action_embed = (\n                self._set_rating_indexes_and_embed(self.action_seq_config)\n            )\n        else:\n            self.action_embed = None\n\n        if self.other_seq_cols_confiq is not None:\n            (\n                self.other_seq_cols_indexes,\n                self.other_seq_cols_embed,\n                self.other_seq_dim,\n            ) = self._set_other_seq_cols_indexes_embed_and_dim(\n                self.other_seq_cols_confiq\n            )\n        else:\n            self.other_seq_cols_embed = None\n            self.other_seq_dim = 0\n\n        if self.cat_embed_input is not None or self.continuous_cols is not None:\n            self.other_cols_idx, self.other_col_embed, self.other_cols_dim = (\n                self._set_other_cols_idx_embed_and_dim(\n                    self.cat_embed_input, self.continuous_cols\n                )\n            )\n        else:\n            self.other_col_embed = None\n            self.other_cols_dim = 0\n\n        self.attention = ActivationUnit(\n            user_behavior_confiq[2], attention_unit_activation\n        )\n\n        if self.mlp_hidden_dims is not None:\n            mlp_input_dim = (\n                self.user_behavior_dim * 2 + self.other_seq_dim + self.other_cols_dim\n            )\n            self.mlp = MLP(\n                d_hidden=[mlp_input_dim] + self.mlp_hidden_dims,\n                activation=(\n                    \"relu\" if self.mlp_activation is None else self.mlp_activation\n                ),\n                dropout=0.0 if self.mlp_dropout is None else self.mlp_dropout,\n                batchnorm=False if self.mlp_batchnorm is None else self.mlp_batchnorm,\n                batchnorm_last=(\n                    False\n                    if self.mlp_batchnorm_last is None\n                    else self.mlp_batchnorm_last\n                ),\n                linear_first=(\n                    False if self.mlp_linear_first is None else self.mlp_linear_first\n                ),\n            )\n        else:\n            self.mlp = None\n\n    def forward(self, X: Tensor) -&gt; Tensor:\n\n        X_target_item = X[:, [self.target_item_idx]]\n        item_embed = self.user_behavior_embed.cat_embed.embed(X_target_item.long())\n\n        X_user_behavior = X[:, self.user_behavior_indexes]\n        user_behavior_embed = self.user_behavior_embed._get_embeddings(X_user_behavior)\n        # 0 is the padding index\n        mask = (X_user_behavior != 0).float().to(X.device)\n\n        if self.action_embed is not None:\n            X_rating = X[:, self.action_seq_indexes]\n            action_embed = self.action_embed._get_embeddings(X_rating)\n            user_behavior_embed = user_behavior_embed * action_embed\n\n        attention_scores = self.attention(item_embed, user_behavior_embed)\n        attention_scores = attention_scores * mask\n        user_interest = (attention_scores.unsqueeze(-1) * user_behavior_embed).sum(1)\n\n        deep_out = torch.cat([item_embed.squeeze(1), user_interest], dim=1)\n\n        if self.other_seq_cols_embed is not None:\n            X_other_seq: Dict[str, Tensor] = {\n                col: X[:, idx] for col, idx in self.other_seq_cols_indexes.items()\n            }\n            other_seq_embed = torch.cat(\n                [\n                    self.other_seq_cols_embed[col]._get_embeddings(X_other_seq[col])\n                    for col in self.other_seq_cols_indexes.keys()\n                ],\n                dim=-1,\n            ).sum(1)\n            deep_out = torch.cat([deep_out, other_seq_embed], dim=1)\n\n        if self.other_col_embed is not None:\n            X_other_cols = X[:, self.other_cols_idx]\n            other_cols_embed = self.other_col_embed._get_embeddings(X_other_cols)\n            deep_out = torch.cat([deep_out, other_cols_embed], dim=1)\n\n        if self.mlp is not None:\n            deep_out = self.mlp(deep_out)\n\n        return deep_out\n\n    @staticmethod\n    def _get_seq_cols_embed_confiq(tup: Tuple[List[str], int, int]) -&gt; Dict[str, Any]:\n        # tup[0] is the list of columns\n        # tup[1] is the number of unique feat value or \"n_tokens\"\n        # tup[2] is the embedding dimension\n\n        # Once sliced, the indexes will go from 0 to len(tup[0]) it is assumed\n        # that the columns in tup[0] are ordered of appearance in the input\n        # data\n        column_idx = {col: i for i, col in enumerate(tup[0])}\n\n        # This is a hack so that I can use any BaseTabularModelWithAttention.\n        # For this model to work 'cat_embed_input' is normally a List of\n        # Tuples where the first element is the column name and the second is\n        # the number of unique values for that column. That second elements\n        # is added internally to compute what one could call \"n_tokens\". Here\n        # I'm passing that value as the second element of the first tuple\n        # and then adding 0s for the rest of the columns\n        cat_embed_input = [(tup[0][0], tup[1])] + [(col, 0) for col in tup[0][1:]]\n\n        input_dim = tup[2]\n\n        col_config = {\n            \"column_idx\": column_idx,\n            \"input_dim\": input_dim,\n            \"cat_embed_input\": cat_embed_input,\n            \"cat_embed_dropout\": None,\n            \"use_cat_bias\": None,\n            \"cat_embed_activation\": None,\n            \"shared_embed\": None,\n            \"add_shared_embed\": None,\n            \"frac_shared_embed\": None,\n            \"continuous_cols\": None,\n            \"cont_norm_layer\": None,\n            \"embed_continuous_method\": None,\n            \"cont_embed_dropout\": None,\n            \"cont_embed_activation\": None,\n            \"quantization_setup\": None,\n            \"n_frequencies\": None,\n            \"sigma\": None,\n            \"share_last_layer\": None,\n            \"full_embed_dropout\": None,\n        }\n\n        return col_config\n\n    def _get_other_cols_embed_config(\n        self,\n        cat_embed_input: Optional[List[Tuple[str, int, int]]],\n        continuous_cols: Optional[List[str]],\n        column_idx: Dict[str, int],\n    ) -&gt; Dict[str, Any]:\n\n        cols_config = {\n            \"column_idx\": {col: i for i, col in enumerate(column_idx.keys())},\n            \"cat_embed_input\": cat_embed_input,\n            \"cat_embed_dropout\": self.cat_embed_dropout,\n            \"use_cat_bias\": self.use_cat_bias,\n            \"cat_embed_activation\": self.cat_embed_activation,\n            \"continuous_cols\": continuous_cols,\n            \"cont_norm_layer\": self.cont_norm_layer,\n            \"embed_continuous\": self.embed_continuous,\n            \"embed_continuous_method\": self.embed_continuous_method,\n            \"cont_embed_dim\": self.cont_embed_dim,\n            \"cont_embed_dropout\": self.cont_embed_dropout,\n            \"cont_embed_activation\": self.cont_embed_activation,\n            \"quantization_setup\": self.quantization_setup,\n            \"n_frequencies\": self.n_frequencies,\n            \"sigma\": self.sigma,\n            \"share_last_layer\": self.share_last_layer,\n            \"full_embed_dropout\": self.full_embed_dropout,\n        }\n\n        return cols_config\n\n    def set_user_behavior_indexes_and_embed(\n        self, user_behavior_confiq: Tuple[List[str], int, int]\n    ) -&gt; Tuple[List[int], BaseTabularModelWithAttention]:\n        user_behavior_indexes = [\n            self.column_idx[col] for col in user_behavior_confiq[0]\n        ]\n        user_behavior_embed = BaseTabularModelWithAttention(\n            **self._get_seq_cols_embed_confiq(user_behavior_confiq)\n        )\n        return user_behavior_indexes, user_behavior_embed\n\n    def _set_rating_indexes_and_embed(\n        self, action_seq_config: Tuple[List[str], int, int]\n    ) -&gt; Tuple[List[int], BaseTabularModelWithAttention]:\n        action_seq_indexes = [self.column_idx[col] for col in action_seq_config[0]]\n        action_embed = BaseTabularModelWithAttention(\n            **self._get_seq_cols_embed_confiq(action_seq_config)\n        )\n        return action_seq_indexes, action_embed\n\n    def _set_other_seq_cols_indexes_embed_and_dim(\n        self, other_seq_cols_confiq: List[Tuple[List[str], int, int]]\n    ) -&gt; Tuple[Dict[str, List[int]], nn.ModuleDict, int]:\n        other_seq_cols_indexes: Dict[str, List[int]] = {}\n        for i, el in enumerate(other_seq_cols_confiq):\n            key = f\"seq_{i}\"\n            idxs = [self.column_idx[col] for col in el[0]]\n            other_seq_cols_indexes[key] = idxs\n        other_seq_cols_config = {\n            f\"seq_{i}\": self._get_seq_cols_embed_confiq(el)\n            for i, el in enumerate(other_seq_cols_confiq)\n        }\n        other_seq_cols_embed = nn.ModuleDict(\n            {\n                key: BaseTabularModelWithAttention(**config)\n                for key, config in other_seq_cols_config.items()\n            }\n        )\n        other_seq_dim = sum([el[2] for el in other_seq_cols_confiq])\n\n        return other_seq_cols_indexes, other_seq_cols_embed, other_seq_dim\n\n    def _set_other_cols_idx_embed_and_dim(\n        self,\n        cat_embed_input: Optional[List[Tuple[str, int, int]]],\n        continuous_cols: Optional[List[str]],\n    ) -&gt; Tuple[List[int], BaseTabularModelWithoutAttention, int]:\n        other_cols_idx: Dict[str, int] = {}\n        if cat_embed_input is not None:\n            other_cols_idx = {\n                col: self.column_idx[col] for col in [el[0] for el in cat_embed_input]\n            }\n        if continuous_cols is not None:\n            other_cols_idx.update(\n                {col: self.column_idx[col] for col in continuous_cols}\n            )\n\n        sorted_other_cols_idx = {\n            k: v for k, v in sorted(other_cols_idx.items(), key=lambda x: x[1])\n        }\n\n        other_col_embed = BaseTabularModelWithoutAttention(\n            **self._get_other_cols_embed_config(\n                cat_embed_input, continuous_cols, sorted_other_cols_idx\n            )\n        )\n\n        other_cols_dim = other_col_embed.output_dim\n\n        return list(other_cols_idx.values()), other_col_embed, other_cols_dim\n\n    @property\n    def output_dim(self) -&gt; int:\n        if self.mlp_hidden_dims is not None:\n            return self.mlp_hidden_dims[-1]\n        else:\n            return self.user_behavior_dim * 2 + self.other_seq_dim + self.other_cols_dim\n</code></pre>"},{"location":"pytorch-widedeep/the_rec_module.html#pytorch_widedeep.models.rec.xdeepfm.ExtremeDeepFactorizationMachine","title":"ExtremeDeepFactorizationMachine","text":"<p>               Bases: <code>BaseTabularModelWithAttention</code></p> <p>Adaptation of 'xDeepFM implementation: xDeepFM: Combining Explicit and Implicit Feature Interactions for Recommender Systems' by Jianxun Lian, Xiaohuan Zhou, Fuzheng Zhang, Zhongxia Chen, Xing Xie, Guangzhong Sun and Enhong Chen, 2018</p> <p>The implementation in this library takes advantage of all the functionalities available to encode categorical and continuous features. The model can be used with only the factorization machine</p> <p>Note that this class implements only the 'Deep' component of the model described in the paper. The linear component is not implemented 'internally' and, if one wants to include it, it can be easily added using the 'wide'/linear component in this library. See the examples in the examples folder.</p> <p>Parameters:</p> Name Type Description Default <code>column_idx</code> <code>Dict[str, int]</code> <p>Dictionary mapping column names to their corresponding index.</p> required <code>input_dim</code> <code>int</code> <p>Embedding input dimensions</p> required <code>reduce_sum</code> <code>bool</code> <p>Whether to reduce the sum in the factorization machine output.</p> <code>True</code> <code>cin_layer_dims</code> <code>List[int]</code> <p>List with the number of units per CIN layer. e.g: [128, 64]</p> required <code>cat_embed_input</code> <code>Optional[List[Tuple[str, int]]]</code> <p>List of tuples with categorical column names and number of unique values.</p> <code>None</code> <code>cat_embed_dropout</code> <code>Optional[float]</code> <p>Categorical embeddings dropout. If <code>None</code>, it will default to 0.</p> <code>None</code> <code>use_cat_bias</code> <code>Optional[bool]</code> <p>Boolean indicating if bias will be used for the categorical embeddings. If <code>None</code>, it will default to 'False'.</p> <code>None</code> <code>cat_embed_activation</code> <code>Optional[str]</code> <p>Activation function for the categorical embeddings, if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> <code>None</code> <code>continuous_cols</code> <code>Optional[List[str]]</code> <p>List with the name of the numeric (aka continuous) columns</p> <code>None</code> <code>cont_norm_layer</code> <code>Optional[Literal['batchnorm', 'layernorm']]</code> <p>Type of normalization layer applied to the continuous features. Options are: 'layernorm' and 'batchnorm'. if <code>None</code>, no normalization layer will be used.</p> <code>None</code> <code>embed_continuous_method</code> <code>Optional[Literal['piecewise', 'periodic', 'standard']]</code> <p>Method to use to embed the continuous features. Options are: 'standard', 'periodic' or 'piecewise'. The 'standard' embedding method is based on the FT-Transformer implementation presented in the paper: Revisiting Deep Learning Models for Tabular Data. The 'periodic' and_'piecewise'_ methods were presented in the paper: On Embeddings for Numerical Features in Tabular Deep Learning. Please, read the papers for details.</p> <code>'standard'</code> <code>cont_embed_dropout</code> <code>Optional[float]</code> <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p> <code>None</code> <code>cont_embed_activation</code> <code>Optional[str]</code> <p>Activation function for the continuous embeddings if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported. If <code>None</code>, no activation function will be applied.</p> <code>None</code> <code>quantization_setup</code> <code>Optional[Dict[str, List[float]]]</code> <p>This parameter is used when the 'piecewise' method is used to embed the continuous cols. It is a dict where keys are the name of the continuous columns and values are lists with the boundaries for the quantization of the continuous_cols. See the examples for details. If If the 'piecewise' method is used, this parameter is required.</p> <code>None</code> <code>n_frequencies</code> <code>Optional[int]</code> <p>This is the so called 'k' in their paper On Embeddings for Numerical Features in Tabular Deep Learning, and is the number of 'frequencies' that will be used to represent each continuous column. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>sigma</code> <code>Optional[float]</code> <p>This is the sigma parameter in the paper mentioned when describing the previous parameters and it is used to initialise the 'frequency weights'. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>share_last_layer</code> <code>Optional[bool]</code> <p>This parameter is not present in the before mentioned paper but it is implemented in the official repo. If <code>True</code> the linear layer that turns the frequencies into embeddings will be shared across the continuous columns. If <code>False</code> a different linear layer will be used for each continuous column. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>full_embed_dropout</code> <code>Optional[bool]</code> <p>If <code>True</code>, the full embedding corresponding to a column will be masked out/dropout. If <code>None</code>, it will default to <code>False</code>.</p> <code>None</code> <code>mlp_hidden_dims</code> <code>Optional[List[int]]</code> <p>List with the number of neurons per dense layer in the mlp.</p> <code>None</code> <code>mlp_activation</code> <code>Optional[str]</code> <p>Activation function for the dense layers of the MLP. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> <code>None</code> <code>mlp_dropout</code> <code>Optional[float]</code> <p>float or List of floats with the dropout between the dense layers. e.g: [0.5,0.5]</p> <code>None</code> <code>mlp_batchnorm</code> <code>Optional[bool]</code> <p>Boolean indicating whether or not batch normalization will be applied to the dense layers</p> <code>None</code> <code>mlp_batchnorm_last</code> <code>Optional[bool]</code> <p>Boolean indicating whether or not batch normalization will be applied to the last of the dense layers</p> <code>None</code> <code>mlp_linear_first</code> <code>Optional[bool]</code> <p>Boolean indicating the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code></p> <code>None</code> <p>Attributes:</p> Name Type Description <code>n_features</code> <code>int</code> <p>Number of unique features/columns</p> <code>cin</code> <code>CompressedInteractionNetwork</code> <p>Instance of the <code>CompressedInteractionNetwork</code> class</p> <code>mlp</code> <code>MLP</code> <p>Instance of the <code>MLP</code> class if <code>mlp_hidden_dims</code> is not None. If None, the model will return directly the output of the <code>CIN</code></p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models.rec import ExtremeDeepFactorizationMachine\n&gt;&gt;&gt; X_tab = torch.randint(0, 10, (16, 2))\n&gt;&gt;&gt; column_idx = {\"col1\": 0, \"col2\": 1}\n&gt;&gt;&gt; cat_embed_input = [(\"col1\", 10), (\"col2\", 10)]\n&gt;&gt;&gt; xdeepfm = ExtremeDeepFactorizationMachine(\n...     column_idx=column_idx,\n...     input_dim=4,\n...     cin_layer_dims=[8, 16],\n...     cat_embed_input=cat_embed_input,\n...     mlp_hidden_dims=[16, 8]\n... )\n&gt;&gt;&gt; output = xdeepfm(X_tab)\n</code></pre> Source code in <code>pytorch_widedeep/models/rec/xdeepfm.py</code> <pre><code>class ExtremeDeepFactorizationMachine(BaseTabularModelWithAttention):\n    \"\"\"\n    Adaptation of 'xDeepFM implementation: xDeepFM: Combining Explicit and\n    Implicit Feature Interactions for Recommender Systems' by Jianxun Lian,\n    Xiaohuan Zhou, Fuzheng Zhang, Zhongxia Chen, Xing Xie, Guangzhong Sun and\n    Enhong Chen, 2018\n\n    The implementation in this library takes advantage of all the\n    functionalities available to encode categorical and continuous features.\n    The model can be used with only the factorization machine\n\n    Note that this class implements only the 'Deep' component of the model\n    described in the paper. The linear component is not\n    implemented 'internally' and, if one wants to include it, it can be\n    easily added using the 'wide'/linear component in this library. See the\n    examples in the examples folder.\n\n    Parameters\n    ----------\n    column_idx : Dict[str, int]\n        Dictionary mapping column names to their corresponding index.\n    input_dim : int\n        Embedding input dimensions\n    reduce_sum : bool, default=True\n        Whether to reduce the sum in the factorization machine output.\n    cin_layer_dims : List[int]\n        List with the number of units per CIN layer. e.g: _[128, 64]_\n    cat_embed_input : Optional[List[Tuple[str, int]]], default=None\n        List of tuples with categorical column names and number of unique values.\n    cat_embed_dropout : Optional[float], default=None\n        Categorical embeddings dropout. If `None`, it will default\n        to 0.\n    use_cat_bias : Optional[bool], default=None\n        Boolean indicating if bias will be used for the categorical embeddings.\n        If `None`, it will default to 'False'.\n    cat_embed_activation : Optional[str], default=None\n        Activation function for the categorical embeddings, if any. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported\n    continuous_cols : Optional[List[str]], default=None\n        List with the name of the numeric (aka continuous) columns\n    cont_norm_layer : Optional[Literal[\"batchnorm\", \"layernorm\"]], default=None\n        Type of normalization layer applied to the continuous features.\n        Options are: _'layernorm'_ and _'batchnorm'_. if `None`, no\n        normalization layer will be used.\n    embed_continuous_method: Optional[Literal[\"piecewise\", \"periodic\", \"standard\"]], default=\"standard\"\n        Method to use to embed the continuous features. Options are:\n        _'standard'_, _'periodic'_ or _'piecewise'_. The _'standard'_\n        embedding method is based on the FT-Transformer implementation\n        presented in the paper: [Revisiting Deep Learning Models for\n        Tabular Data](https://arxiv.org/abs/2106.11959v5). The _'periodic'_\n        and_'piecewise'_ methods were presented in the paper: [On Embeddings for\n        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556).\n        Please, read the papers for details.\n    cont_embed_dropout : Optional[float], default=None\n        Dropout for the continuous embeddings. If `None`, it will default to 0.0\n    cont_embed_activation : Optional[str], default=None\n        Activation function for the continuous embeddings if any. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported.\n        If `None`, no activation function will be applied.\n    quantization_setup : Optional[Dict[str, List[float]]], default=None\n        This parameter is used when the _'piecewise'_ method is used to embed\n        the continuous cols. It is a dict where keys are the name of the continuous\n        columns and values are lists with the boundaries for the quantization\n        of the continuous_cols. See the examples for details. If\n        If the _'piecewise'_ method is used, this parameter is required.\n    n_frequencies : Optional[int], default=None\n        This is the so called _'k'_ in their paper [On Embeddings for\n        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556),\n        and is the number of 'frequencies' that will be used to represent each\n        continuous column. See their Eq 2 in the paper for details. If\n        the _'periodic'_ method is used, this parameter is required.\n    sigma : Optional[float], default=None\n        This is the sigma parameter in the paper mentioned when describing the\n        previous parameters and it is used to initialise the 'frequency\n        weights'. See their Eq 2 in the paper for details. If\n        the _'periodic'_ method is used, this parameter is required.\n    share_last_layer : Optional[bool], default=None\n        This parameter is not present in the before mentioned paper but it is implemented in\n        the [official repo](https://github.com/yandex-research/rtdl-num-embeddings/tree/main).\n        If `True` the linear layer that turns the frequencies into embeddings\n        will be shared across the continuous columns. If `False` a different\n        linear layer will be used for each continuous column.\n        If the _'periodic'_ method is used, this parameter is required.\n    full_embed_dropout: bool, Optional, default = None,\n        If `True`, the full embedding corresponding to a column will be masked\n        out/dropout. If `None`, it will default to `False`.\n    mlp_hidden_dims: List, default = [200, 100]\n        List with the number of neurons per dense layer in the mlp.\n    mlp_activation: str, default = \"relu\"\n        Activation function for the dense layers of the MLP. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported\n    mlp_dropout: float or List, default = 0.1\n        float or List of floats with the dropout between the dense layers.\n        e.g: _[0.5,0.5]_\n    mlp_batchnorm: bool, default = False\n        Boolean indicating whether or not batch normalization will be applied\n        to the dense layers\n    mlp_batchnorm_last: bool, default = False\n        Boolean indicating whether or not batch normalization will be applied\n        to the last of the dense layers\n    mlp_linear_first: bool, default = False\n        Boolean indicating the order of the operations in the dense\n        layer. If `True: [LIN -&gt; ACT -&gt; BN -&gt; DP]`. If `False: [BN -&gt; DP -&gt;\n        LIN -&gt; ACT]`\n\n    Attributes\n    ----------\n    n_features: int\n        Number of unique features/columns\n    cin: CompressedInteractionNetwork\n        Instance of the `CompressedInteractionNetwork` class\n    mlp: MLP\n        Instance of the `MLP` class if `mlp_hidden_dims` is not None. If None,\n        the model will return directly the output of the `CIN`\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.models.rec import ExtremeDeepFactorizationMachine\n    &gt;&gt;&gt; X_tab = torch.randint(0, 10, (16, 2))\n    &gt;&gt;&gt; column_idx = {\"col1\": 0, \"col2\": 1}\n    &gt;&gt;&gt; cat_embed_input = [(\"col1\", 10), (\"col2\", 10)]\n    &gt;&gt;&gt; xdeepfm = ExtremeDeepFactorizationMachine(\n    ...     column_idx=column_idx,\n    ...     input_dim=4,\n    ...     cin_layer_dims=[8, 16],\n    ...     cat_embed_input=cat_embed_input,\n    ...     mlp_hidden_dims=[16, 8]\n    ... )\n    &gt;&gt;&gt; output = xdeepfm(X_tab)\n    \"\"\"\n\n    def __init__(\n        self,\n        *,\n        column_idx: Dict[str, int],\n        input_dim: int,\n        reduce_sum: bool = True,\n        cin_layer_dims: List[int],\n        cat_embed_input: List[Tuple[str, int]],\n        cat_embed_dropout: Optional[float] = None,\n        use_cat_bias: Optional[bool] = None,\n        cat_embed_activation: Optional[str] = None,\n        continuous_cols: Optional[List[str]] = None,\n        cont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\n        embed_continuous_method: Optional[\n            Literal[\"piecewise\", \"periodic\", \"standard\"]\n        ] = \"standard\",\n        cont_embed_dropout: Optional[float] = None,\n        cont_embed_activation: Optional[str] = None,\n        quantization_setup: Optional[Dict[str, List[float]]] = None,\n        n_frequencies: Optional[int] = None,\n        sigma: Optional[float] = None,\n        share_last_layer: Optional[bool] = None,\n        full_embed_dropout: Optional[bool] = None,\n        mlp_hidden_dims: Optional[List[int]] = None,\n        mlp_activation: Optional[str] = None,\n        mlp_dropout: Optional[float] = None,\n        mlp_batchnorm: Optional[bool] = None,\n        mlp_batchnorm_last: Optional[bool] = None,\n        mlp_linear_first: Optional[bool] = None,\n    ):\n        super(ExtremeDeepFactorizationMachine, self).__init__(\n            column_idx=column_idx,\n            input_dim=input_dim,\n            cat_embed_input=cat_embed_input,\n            cat_embed_dropout=cat_embed_dropout,\n            use_cat_bias=use_cat_bias,\n            cat_embed_activation=cat_embed_activation,\n            shared_embed=False,\n            add_shared_embed=None,\n            frac_shared_embed=None,\n            continuous_cols=continuous_cols,\n            cont_norm_layer=cont_norm_layer,\n            embed_continuous_method=embed_continuous_method,\n            cont_embed_dropout=cont_embed_dropout,\n            cont_embed_activation=cont_embed_activation,\n            quantization_setup=quantization_setup,\n            n_frequencies=n_frequencies,\n            sigma=sigma,\n            share_last_layer=share_last_layer,\n            full_embed_dropout=full_embed_dropout,\n        )\n\n        self.mlp_hidden_dims = mlp_hidden_dims\n        self.mlp_activation = mlp_activation\n        self.mlp_dropout = mlp_dropout\n        self.mlp_batchnorm = mlp_batchnorm\n        self.mlp_batchnorm_last = mlp_batchnorm_last\n        self.mlp_linear_first = mlp_linear_first\n\n        self.reduce_sum = reduce_sum\n        self.cin_layer_dims = cin_layer_dims\n\n        self.n_features = len(self.column_idx)\n\n        self.cin = CompressedInteractionNetwork(\n            num_cols=self.n_features, cin_layer_dims=self.cin_layer_dims\n        )\n\n        if self.mlp_hidden_dims is not None:\n            if (\n                self.mlp_hidden_dims[-1] != sum(self.cin_layer_dims)\n                and not self.reduce_sum\n            ):\n                d_hidden = (\n                    [sum(self.cin_layer_dims)]\n                    + self.mlp_hidden_dims\n                    + [sum(self.cin_layer_dims)]\n                )\n            else:\n                d_hidden = [sum(self.cin_layer_dims)] + self.mlp_hidden_dims\n\n            self.mlp = MLP(\n                d_hidden=d_hidden,\n                activation=(\n                    \"relu\" if self.mlp_activation is None else self.mlp_activation\n                ),\n                dropout=0.0 if self.mlp_dropout is None else self.mlp_dropout,\n                batchnorm=False if self.mlp_batchnorm is None else self.mlp_batchnorm,\n                batchnorm_last=(\n                    False\n                    if self.mlp_batchnorm_last is None\n                    else self.mlp_batchnorm_last\n                ),\n                linear_first=(\n                    False if self.mlp_linear_first is None else self.mlp_linear_first\n                ),\n            )\n        else:\n            self.mlp = None\n\n    def forward(self, X: Tensor) -&gt; Tensor:\n\n        embeddings = self._get_embeddings(X)\n        cin_out = self.cin(embeddings)\n\n        if self.mlp is None:\n            if self.reduce_sum:\n                return cin_out.sum(dim=1, keepdim=True)\n            return cin_out\n\n        mlp_out = self.mlp(cin_out)\n\n        if self.reduce_sum:\n            cin_out = cin_out.sum(dim=1, keepdim=True)\n            mlp_out = mlp_out.sum(dim=1, keepdim=True)\n\n        return mlp_out + cin_out\n\n    @property\n    def output_dim(self):\n        if self.reduce_sum:\n            return 1\n        else:\n            return sum(self.cin_layer_dims)\n</code></pre>"},{"location":"pytorch-widedeep/the_rec_module.html#pytorch_widedeep.models.rec.autoint.AutoInt","title":"AutoInt","text":"<p>               Bases: <code>BaseTabularModelWithAttention</code></p> <p>Defines an <code>AutoInt</code> model that can be used as the <code>deeptabular</code> component of a Wide &amp; Deep model or independently by itself.</p> <p>This class implements the AutoInt: Automatic Feature Interaction Learning via Self-Attentive Neural Networks architecture, which learns feature interactions through multi-head self-attention networks.</p> <p>Parameters:</p> Name Type Description Default <code>column_idx</code> <code>Dict[str, int]</code> <p>Dict containing the index of the columns that will be passed through the <code>AutoInt</code> model. Required to slice the tensors. e.g. {'education': 0, 'relationship': 1, 'workclass': 2, ...}.</p> required <code>input_dim</code> <code>int</code> <p>Dimension of the input embeddings</p> required <code>num_heads</code> <code>int</code> <p>Number of attention heads</p> <code>4</code> <code>num_layers</code> <code>int</code> <p>Number of interacting layers (attention + residual)</p> <code>2</code> <code>reduction</code> <code>Literal['mean', 'cat']</code> <p>How to reduce the output of the attention layers. Options are: 'mean': mean of attention outputs 'cat': concatenation of attention outputs</p> <code>'mean'</code> <code>cat_embed_input</code> <code>Optional[List[Tuple[str, int]]]</code> <p>List of Tuples with the column name, number of unique values and embedding dimension. e.g. [(education, 11, 32), ...]</p> <code>None</code> <code>cat_embed_dropout</code> <code>Optional[float]</code> <p>Categorical embeddings dropout. If <code>None</code>, it will default to 0.</p> <code>None</code> <code>use_cat_bias</code> <code>Optional[bool]</code> <p>Boolean indicating if bias will be used for the categorical embeddings</p> <code>None</code> <code>cat_embed_activation</code> <code>Optional[str]</code> <p>Activation function for the categorical embeddings</p> <code>None</code> <code>continuous_cols</code> <code>Optional[List[str]]</code> <p>List with the name of the numeric (aka continuous) columns</p> <code>None</code> <code>cont_norm_layer</code> <code>Optional[Literal['batchnorm', 'layernorm']]</code> <p>Type of normalization layer applied to the continuous features. Options are: 'layernorm' and 'batchnorm'. if <code>None</code>, no normalization layer will be used.</p> <code>None</code> <code>embed_continuous_method</code> <code>Optional[Literal['standard', 'piecewise', 'periodic']]</code> <p>Method to use to embed the continuous features. Options are: 'standard', 'periodic' or 'piecewise'. The 'standard' embedding method is based on the FT-Transformer implementation presented in the paper: Revisiting Deep Learning Models for Tabular Data. The 'periodic' and_'piecewise'_ methods were presented in the paper: On Embeddings for Numerical Features in Tabular Deep Learning. Please, read the papers for details.</p> <code>None</code> <code>cont_embed_dropout</code> <code>Optional[float]</code> <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p> <code>None</code> <code>cont_embed_activation</code> <code>Optional[str]</code> <p>Activation function for the continuous embeddings if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported. If <code>None</code>, no activation function will be applied.</p> <code>None</code> <code>quantization_setup</code> <code>Optional[Dict[str, List[float]]]</code> <p>This parameter is used when the 'piecewise' method is used to embed the continuous cols. It is a dict where keys are the name of the continuous columns and values are lists with the boundaries for the quantization of the continuous_cols. See the examples for details. If If the 'piecewise' method is used, this parameter is required.</p> <code>None</code> <code>n_frequencies</code> <code>Optional[int]</code> <p>This is the so called 'k' in their paper On Embeddings for Numerical Features in Tabular Deep Learning, and is the number of 'frequencies' that will be used to represent each continuous column. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>sigma</code> <code>Optional[float]</code> <p>This is the sigma parameter in the paper mentioned when describing the previous parameters and it is used to initialise the 'frequency weights'. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>share_last_layer</code> <code>Optional[bool]</code> <p>This parameter is not present in the before mentioned paper but it is implemented in the official repo. If <code>True</code> the linear layer that turns the frequencies into embeddings will be shared across the continuous columns. If <code>False</code> a different linear layer will be used for each continuous column. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>full_embed_dropout</code> <code>Optional[bool]</code> <p>If <code>True</code>, the full embedding corresponding to a column will be masked out/dropout. If <code>None</code>, it will default to <code>False</code>.</p> <code>None</code> <p>Attributes:</p> Name Type Description <code>attention_layers</code> <code>ModuleList</code> <p>List of multi-head attention layers</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models.rec import AutoInt\n&gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n&gt;&gt;&gt; colnames = [\"a\", \"b\", \"c\", \"d\", \"e\"]\n&gt;&gt;&gt; cat_embed_input = [(u, i, j) for u, i, j in zip(colnames[:4], [4] * 4, [8] * 4)]\n&gt;&gt;&gt; column_idx = {k: v for v, k in enumerate(colnames)}\n&gt;&gt;&gt; model = AutoInt(\n...     column_idx=column_idx,\n...     input_dim=32,\n...     cat_embed_input=cat_embed_input,\n...     continuous_cols=[\"e\"],\n...     embed_continuous_method=\"standard\",\n...     num_heads=4,\n...     num_layers=2\n... )\n&gt;&gt;&gt; out = model(X_tab)\n</code></pre> Source code in <code>pytorch_widedeep/models/rec/autoint.py</code> <pre><code>class AutoInt(BaseTabularModelWithAttention):\n    r\"\"\"Defines an `AutoInt` model that can be used as the `deeptabular` component\n    of a Wide &amp; Deep model or independently by itself.\n\n    This class implements the [AutoInt: Automatic Feature Interaction Learning via Self-Attentive\n    Neural Networks](https://arxiv.org/abs/1810.11921) architecture, which learns feature\n    interactions through multi-head self-attention networks.\n\n    Parameters\n    ----------\n    column_idx: Dict\n        Dict containing the index of the columns that will be passed through\n        the `AutoInt` model. Required to slice the tensors. e.g.\n        _{'education': 0, 'relationship': 1, 'workclass': 2, ...}_.\n    input_dim: int\n        Dimension of the input embeddings\n    num_heads: int, default = 4\n        Number of attention heads\n    num_layers: int, default = 2\n        Number of interacting layers (attention + residual)\n    reduction: str, default = \"mean\"\n        How to reduce the output of the attention layers. Options are:\n        _'mean'_: mean of attention outputs\n        _'cat'_: concatenation of attention outputs\n    cat_embed_input: List, Optional, default = None\n        List of Tuples with the column name, number of unique values and\n        embedding dimension. e.g. _[(education, 11, 32), ...]_\n    cat_embed_dropout: float, Optional, default = None\n        Categorical embeddings dropout. If `None`, it will default to 0.\n    use_cat_bias: bool, Optional, default = None,\n        Boolean indicating if bias will be used for the categorical embeddings\n    cat_embed_activation: Optional, str, default = None,\n        Activation function for the categorical embeddings\n    continuous_cols : Optional[List[str]], default=None\n        List with the name of the numeric (aka continuous) columns\n    cont_norm_layer : Optional[Literal[\"batchnorm\", \"layernorm\"]], default=None\n        Type of normalization layer applied to the continuous features.\n        Options are: _'layernorm'_ and _'batchnorm'_. if `None`, no\n        normalization layer will be used.\n    embed_continuous_method: Optional[Literal[\"piecewise\", \"periodic\", \"standard\"]], default=\"standard\"\n        Method to use to embed the continuous features. Options are:\n        _'standard'_, _'periodic'_ or _'piecewise'_. The _'standard'_\n        embedding method is based on the FT-Transformer implementation\n        presented in the paper: [Revisiting Deep Learning Models for\n        Tabular Data](https://arxiv.org/abs/2106.11959v5). The _'periodic'_\n        and_'piecewise'_ methods were presented in the paper: [On Embeddings for\n        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556).\n        Please, read the papers for details.\n    cont_embed_dropout : Optional[float], default=None\n        Dropout for the continuous embeddings. If `None`, it will default to 0.0\n    cont_embed_activation : Optional[str], default=None\n        Activation function for the continuous embeddings if any. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported.\n        If `None`, no activation function will be applied.\n    quantization_setup : Optional[Dict[str, List[float]]], default=None\n        This parameter is used when the _'piecewise'_ method is used to embed\n        the continuous cols. It is a dict where keys are the name of the continuous\n        columns and values are lists with the boundaries for the quantization\n        of the continuous_cols. See the examples for details. If\n        If the _'piecewise'_ method is used, this parameter is required.\n    n_frequencies : Optional[int], default=None\n        This is the so called _'k'_ in their paper [On Embeddings for\n        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556),\n        and is the number of 'frequencies' that will be used to represent each\n        continuous column. See their Eq 2 in the paper for details. If\n        the _'periodic'_ method is used, this parameter is required.\n    sigma : Optional[float], default=None\n        This is the sigma parameter in the paper mentioned when describing the\n        previous parameters and it is used to initialise the 'frequency\n        weights'. See their Eq 2 in the paper for details. If\n        the _'periodic'_ method is used, this parameter is required.\n    share_last_layer : Optional[bool], default=None\n        This parameter is not present in the before mentioned paper but it is implemented in\n        the [official repo](https://github.com/yandex-research/rtdl-num-embeddings/tree/main).\n        If `True` the linear layer that turns the frequencies into embeddings\n        will be shared across the continuous columns. If `False` a different\n        linear layer will be used for each continuous column.\n        If the _'periodic'_ method is used, this parameter is required.\n    full_embed_dropout: bool, Optional, default = None,\n        If `True`, the full embedding corresponding to a column will be masked\n        out/dropout. If `None`, it will default to `False`.\n\n    Attributes\n    ----------\n    attention_layers: nn.ModuleList\n        List of multi-head attention layers\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.models.rec import AutoInt\n    &gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n    &gt;&gt;&gt; colnames = [\"a\", \"b\", \"c\", \"d\", \"e\"]\n    &gt;&gt;&gt; cat_embed_input = [(u, i, j) for u, i, j in zip(colnames[:4], [4] * 4, [8] * 4)]\n    &gt;&gt;&gt; column_idx = {k: v for v, k in enumerate(colnames)}\n    &gt;&gt;&gt; model = AutoInt(\n    ...     column_idx=column_idx,\n    ...     input_dim=32,\n    ...     cat_embed_input=cat_embed_input,\n    ...     continuous_cols=[\"e\"],\n    ...     embed_continuous_method=\"standard\",\n    ...     num_heads=4,\n    ...     num_layers=2\n    ... )\n    &gt;&gt;&gt; out = model(X_tab)\n    \"\"\"\n\n    def __init__(\n        self,\n        *,\n        column_idx: Dict[str, int],\n        input_dim: int,\n        num_heads: int = 4,\n        num_layers: int = 2,\n        reduction: Literal[\"mean\", \"cat\"] = \"mean\",\n        cat_embed_input: Optional[List[Tuple[str, int]]] = None,\n        cat_embed_dropout: Optional[float] = None,\n        use_cat_bias: Optional[bool] = None,\n        cat_embed_activation: Optional[str] = None,\n        continuous_cols: Optional[List[str]] = None,\n        cont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\n        embed_continuous_method: Optional[\n            Literal[\"standard\", \"piecewise\", \"periodic\"]\n        ] = None,\n        cont_embed_dropout: Optional[float] = None,\n        cont_embed_activation: Optional[str] = None,\n        quantization_setup: Optional[Dict[str, List[float]]] = None,\n        n_frequencies: Optional[int] = None,\n        sigma: Optional[float] = None,\n        share_last_layer: Optional[bool] = None,\n        full_embed_dropout: Optional[bool] = None,\n    ):\n        super(AutoInt, self).__init__(\n            column_idx=column_idx,\n            input_dim=input_dim,\n            cat_embed_input=cat_embed_input,\n            cat_embed_dropout=cat_embed_dropout,\n            use_cat_bias=use_cat_bias,\n            cat_embed_activation=cat_embed_activation,\n            shared_embed=None,\n            add_shared_embed=None,\n            frac_shared_embed=None,\n            continuous_cols=continuous_cols,\n            cont_norm_layer=cont_norm_layer,\n            embed_continuous_method=embed_continuous_method,\n            cont_embed_dropout=cont_embed_dropout,\n            cont_embed_activation=cont_embed_activation,\n            quantization_setup=quantization_setup,\n            n_frequencies=n_frequencies,\n            sigma=sigma,\n            share_last_layer=share_last_layer,\n            full_embed_dropout=full_embed_dropout,\n        )\n\n        self.num_heads = num_heads\n        self.num_layers = num_layers\n        self.reduction = reduction\n\n        self.attention_layers = nn.ModuleList(\n            [\n                nn.MultiheadAttention(\n                    embed_dim=input_dim,\n                    num_heads=num_heads,\n                    batch_first=True,\n                )\n                for _ in range(num_layers)\n            ]\n        )\n\n    def forward(self, X: torch.Tensor) -&gt; torch.Tensor:\n        x = self._get_embeddings(X)\n\n        for layer in self.attention_layers:\n            attn_output, _ = layer(x, x, x)\n            x = attn_output + x\n\n        if self.reduction == \"mean\":\n            out = x.mean(dim=1)\n        else:\n            out = x.reshape(x.size(0), -1)\n\n        return out\n\n    @property\n    def output_dim(self) -&gt; int:\n        if self.reduction == \"mean\":\n            return self.input_dim\n        else:\n            return self.input_dim * len(self.column_idx)\n</code></pre>"},{"location":"pytorch-widedeep/the_rec_module.html#pytorch_widedeep.models.rec.autoint_plus.AutoIntPlus","title":"AutoIntPlus","text":"<p>               Bases: <code>BaseTabularModelWithAttention</code></p> <p>Defines an <code>AutoIntPlus</code> model that can be used as the <code>deeptabular</code> component of a Wide &amp; Deep model or independently by itself.</p> <p>This class implements an enhanced version of the AutoInt architecture, adding a parallel or stacked deep network and an optional gating mechanism to control the contribution of the attention-based and MLP branches.</p> <p>Parameters:</p> Name Type Description Default <code>column_idx</code> <code>Dict[str, int]</code> <p>Dict containing the index of the columns that will be passed through the model. Required to slice the tensors. e.g. {'education': 0, 'relationship': 1, 'workclass': 2, ...}.</p> required <code>input_dim</code> <code>int</code> <p>Dimension of the input embeddings</p> required <code>num_heads</code> <code>int</code> <p>Number of attention heads</p> <code>4</code> <code>num_layers</code> <code>int</code> <p>Number of interacting layers (attention + residual)</p> <code>2</code> <code>reduction</code> <code>Literal['mean', 'cat']</code> <p>How to reduce the output of the attention layers. Options are: 'mean': mean of attention outputs 'cat': concatenation of attention outputs</p> <code>'mean'</code> <code>structure</code> <code>Literal['stacked', 'parallel']</code> <p>Structure of the model. Either 'parallel' or 'stacked'. If 'parallel', the output will be the concatenation of the attention and deep networks. If 'stacked', the attention output will be fed into the deep network.</p> <code>'parallel'</code> <code>gated</code> <code>bool</code> <p>If True and structure is 'parallel', uses a gating mechanism to combine the attention and deep networks. Note: requires reduction='mean'.</p> <code>True</code> <code>cat_embed_input</code> <code>Optional[List[Tuple[str, int]]]</code> <p>List of Tuples with the column name, number of unique values and embedding dimension. e.g. [(education, 11, 32), ...]</p> <code>None</code> <code>cat_embed_dropout</code> <code>Optional[float]</code> <p>Categorical embeddings dropout. If <code>None</code>, it will default to 0.</p> <code>None</code> <code>use_cat_bias</code> <code>Optional[bool]</code> <p>Boolean indicating if bias will be used for the categorical embeddings</p> <code>None</code> <code>cat_embed_activation</code> <code>Optional[str]</code> <p>Activation function for the categorical embeddings</p> <code>None</code> <code>continuous_cols</code> <code>Optional[List[str]]</code> <p>List with the name of the numeric (aka continuous) columns</p> <code>None</code> <code>cont_norm_layer</code> <code>Optional[Literal['batchnorm', 'layernorm']]</code> <p>Type of normalization layer applied to the continuous features. Options are: 'layernorm' and 'batchnorm'. if <code>None</code>, no normalization layer will be used.</p> <code>None</code> <code>embed_continuous_method</code> <code>Optional[Literal['standard', 'piecewise', 'periodic']]</code> <p>Method to use to embed the continuous features. Options are: 'standard', 'periodic' or 'piecewise'. The 'standard' embedding method is based on the FT-Transformer implementation presented in the paper: Revisiting Deep Learning Models for Tabular Data. The 'periodic' and_'piecewise'_ methods were presented in the paper: On Embeddings for Numerical Features in Tabular Deep Learning. Please, read the papers for details.</p> <code>None</code> <code>cont_embed_dropout</code> <code>Optional[float]</code> <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p> <code>None</code> <code>cont_embed_activation</code> <code>Optional[str]</code> <p>Activation function for the continuous embeddings if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported. If <code>None</code>, no activation function will be applied.</p> <code>None</code> <code>quantization_setup</code> <code>Optional[Dict[str, List[float]]]</code> <p>This parameter is used when the 'piecewise' method is used to embed the continuous cols. It is a dict where keys are the name of the continuous columns and values are lists with the boundaries for the quantization of the continuous_cols. See the examples for details. If If the 'piecewise' method is used, this parameter is required.</p> <code>None</code> <code>n_frequencies</code> <code>Optional[int]</code> <p>This is the so called 'k' in their paper On Embeddings for Numerical Features in Tabular Deep Learning, and is the number of 'frequencies' that will be used to represent each continuous column. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>sigma</code> <code>Optional[float]</code> <p>This is the sigma parameter in the paper mentioned when describing the previous parameters and it is used to initialise the 'frequency weights'. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>share_last_layer</code> <code>Optional[bool]</code> <p>This parameter is not present in the before mentioned paper but it is implemented in the official repo. If <code>True</code> the linear layer that turns the frequencies into embeddings will be shared across the continuous columns. If <code>False</code> a different linear layer will be used for each continuous column. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>full_embed_dropout</code> <code>Optional[bool]</code> <p>If <code>True</code>, the full embedding corresponding to a column will be masked out/dropout. If <code>None</code>, it will default to <code>False</code>.</p> <code>None</code> <code>mlp_hidden_dims</code> <code>List[int]</code> <p>List with the number of neurons per dense layer in the mlp.</p> <code>[100, 100]</code> <code>mlp_activation</code> <code>str</code> <p>Activation function for the dense layers of the MLP. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> <code>'relu'</code> <code>mlp_dropout</code> <code>Union[float, List[float]]</code> <p>float or List of floats with the dropout between the dense layers. e.g: [0.5,0.5]</p> <code>0.1</code> <code>mlp_batchnorm</code> <code>bool</code> <p>Boolean indicating whether or not batch normalization will be applied to the dense layers</p> <code>False</code> <code>mlp_batchnorm_last</code> <code>bool</code> <p>Boolean indicating whether or not batch normalization will be applied to the last of the dense layers</p> <code>False</code> <code>mlp_linear_first</code> <code>bool</code> <p>Boolean indicating the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code></p> <code>True</code> <p>Attributes:</p> Name Type Description <code>attention_layers</code> <code>ModuleList</code> <p>List of multi-head attention layers</p> <code>deep_network</code> <code>Module</code> <p>The deep network component (MLP)</p> <code>gate</code> <code>(Module, optional)</code> <p>The gating network (if gated=True)</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models.rec import AutoIntPlus\n&gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n&gt;&gt;&gt; colnames = [\"a\", \"b\", \"c\", \"d\", \"e\"]\n&gt;&gt;&gt; cat_embed_input = [(u, i, j) for u, i, j in zip(colnames[:4], [4] * 4, [8] * 4)]\n&gt;&gt;&gt; column_idx = {k: v for v, k in enumerate(colnames)}\n&gt;&gt;&gt; model = AutoIntPlus(\n...     column_idx=column_idx,\n...     input_dim=32,\n...     cat_embed_input=cat_embed_input,\n...     continuous_cols=[\"e\"],\n...     embed_continuous_method=\"standard\",\n...     num_heads=4,\n...     num_layers=2,\n...     structure=\"parallel\",\n...     gated=True,\n...     mlp_hidden_dims=[64, 32]\n... )\n&gt;&gt;&gt; out = model(X_tab)\n</code></pre> Source code in <code>pytorch_widedeep/models/rec/autoint_plus.py</code> <pre><code>class AutoIntPlus(BaseTabularModelWithAttention):\n    r\"\"\"Defines an `AutoIntPlus` model that can be used as the `deeptabular` component\n    of a Wide &amp; Deep model or independently by itself.\n\n    This class implements an enhanced version of the [AutoInt](https://arxiv.org/abs/1810.11921)\n    architecture, adding a parallel or stacked deep network and an optional gating mechanism\n    to control the contribution of the attention-based and MLP branches.\n\n    Parameters\n    ----------\n    column_idx: Dict\n        Dict containing the index of the columns that will be passed through\n        the model. Required to slice the tensors. e.g.\n        _{'education': 0, 'relationship': 1, 'workclass': 2, ...}_.\n    input_dim: int\n        Dimension of the input embeddings\n    num_heads: int, default = 4\n        Number of attention heads\n    num_layers: int, default = 2\n        Number of interacting layers (attention + residual)\n    reduction: str, default = \"mean\"\n        How to reduce the output of the attention layers. Options are:\n        _'mean'_: mean of attention outputs\n        _'cat'_: concatenation of attention outputs\n    structure: str, default = \"parallel\"\n        Structure of the model. Either _'parallel'_ or _'stacked'_. If _'parallel'_,\n        the output will be the concatenation of the attention and deep networks.\n        If _'stacked'_, the attention output will be fed into the deep network.\n    gated: bool, default = True\n        If True and structure is 'parallel', uses a gating mechanism to combine\n        the attention and deep networks. Note: requires reduction='mean'.\n    cat_embed_input: List, Optional, default = None\n        List of Tuples with the column name, number of unique values and\n        embedding dimension. e.g. _[(education, 11, 32), ...]_\n    cat_embed_dropout: float, Optional, default = None\n        Categorical embeddings dropout. If `None`, it will default to 0.\n    use_cat_bias: bool, Optional, default = None,\n        Boolean indicating if bias will be used for the categorical embeddings\n    cat_embed_activation: Optional, str, default = None,\n        Activation function for the categorical embeddings\n    continuous_cols : Optional[List[str]], default=None\n        List with the name of the numeric (aka continuous) columns\n    cont_norm_layer : Optional[Literal[\"batchnorm\", \"layernorm\"]], default=None\n        Type of normalization layer applied to the continuous features.\n        Options are: _'layernorm'_ and _'batchnorm'_. if `None`, no\n        normalization layer will be used.\n    embed_continuous_method: Optional[Literal[\"piecewise\", \"periodic\", \"standard\"]], default=\"standard\"\n        Method to use to embed the continuous features. Options are:\n        _'standard'_, _'periodic'_ or _'piecewise'_. The _'standard'_\n        embedding method is based on the FT-Transformer implementation\n        presented in the paper: [Revisiting Deep Learning Models for\n        Tabular Data](https://arxiv.org/abs/2106.11959v5). The _'periodic'_\n        and_'piecewise'_ methods were presented in the paper: [On Embeddings for\n        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556).\n        Please, read the papers for details.\n    cont_embed_dropout : Optional[float], default=None\n        Dropout for the continuous embeddings. If `None`, it will default to 0.0\n    cont_embed_activation : Optional[str], default=None\n        Activation function for the continuous embeddings if any. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported.\n        If `None`, no activation function will be applied.\n    quantization_setup : Optional[Dict[str, List[float]]], default=None\n        This parameter is used when the _'piecewise'_ method is used to embed\n        the continuous cols. It is a dict where keys are the name of the continuous\n        columns and values are lists with the boundaries for the quantization\n        of the continuous_cols. See the examples for details. If\n        If the _'piecewise'_ method is used, this parameter is required.\n    n_frequencies : Optional[int], default=None\n        This is the so called _'k'_ in their paper [On Embeddings for\n        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556),\n        and is the number of 'frequencies' that will be used to represent each\n        continuous column. See their Eq 2 in the paper for details. If\n        the _'periodic'_ method is used, this parameter is required.\n    sigma : Optional[float], default=None\n        This is the sigma parameter in the paper mentioned when describing the\n        previous parameters and it is used to initialise the 'frequency\n        weights'. See their Eq 2 in the paper for details. If\n        the _'periodic'_ method is used, this parameter is required.\n    share_last_layer : Optional[bool], default=None\n        This parameter is not present in the before mentioned paper but it is implemented in\n        the [official repo](https://github.com/yandex-research/rtdl-num-embeddings/tree/main).\n        If `True` the linear layer that turns the frequencies into embeddings\n        will be shared across the continuous columns. If `False` a different\n        linear layer will be used for each continuous column.\n        If the _'periodic'_ method is used, this parameter is required.\n    full_embed_dropout: bool, Optional, default = None,\n        If `True`, the full embedding corresponding to a column will be masked\n        out/dropout. If `None`, it will default to `False`.\n    mlp_hidden_dims: List, default = [200, 100]\n        List with the number of neurons per dense layer in the mlp.\n    mlp_activation: str, default = \"relu\"\n        Activation function for the dense layers of the MLP. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported\n    mlp_dropout: float or List, default = 0.1\n        float or List of floats with the dropout between the dense layers.\n        e.g: _[0.5,0.5]_\n    mlp_batchnorm: bool, default = False\n        Boolean indicating whether or not batch normalization will be applied\n        to the dense layers\n    mlp_batchnorm_last: bool, default = False\n        Boolean indicating whether or not batch normalization will be applied\n        to the last of the dense layers\n    mlp_linear_first: bool, default = False\n        Boolean indicating the order of the operations in the dense\n        layer. If `True: [LIN -&gt; ACT -&gt; BN -&gt; DP]`. If `False: [BN -&gt; DP -&gt;\n        LIN -&gt; ACT]`\n\n    Attributes\n    ----------\n    attention_layers: nn.ModuleList\n        List of multi-head attention layers\n    deep_network: nn.Module\n        The deep network component (MLP)\n    gate: nn.Module, optional\n        The gating network (if gated=True)\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.models.rec import AutoIntPlus\n    &gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n    &gt;&gt;&gt; colnames = [\"a\", \"b\", \"c\", \"d\", \"e\"]\n    &gt;&gt;&gt; cat_embed_input = [(u, i, j) for u, i, j in zip(colnames[:4], [4] * 4, [8] * 4)]\n    &gt;&gt;&gt; column_idx = {k: v for v, k in enumerate(colnames)}\n    &gt;&gt;&gt; model = AutoIntPlus(\n    ...     column_idx=column_idx,\n    ...     input_dim=32,\n    ...     cat_embed_input=cat_embed_input,\n    ...     continuous_cols=[\"e\"],\n    ...     embed_continuous_method=\"standard\",\n    ...     num_heads=4,\n    ...     num_layers=2,\n    ...     structure=\"parallel\",\n    ...     gated=True,\n    ...     mlp_hidden_dims=[64, 32]\n    ... )\n    &gt;&gt;&gt; out = model(X_tab)\n    \"\"\"\n\n    def __init__(\n        self,\n        *,\n        column_idx: Dict[str, int],\n        input_dim: int,\n        num_heads: int = 4,\n        num_layers: int = 2,\n        reduction: Literal[\"mean\", \"cat\"] = \"mean\",\n        structure: Literal[\"stacked\", \"parallel\"] = \"parallel\",\n        gated: bool = True,\n        cat_embed_input: Optional[List[Tuple[str, int]]] = None,\n        cat_embed_dropout: Optional[float] = None,\n        use_cat_bias: Optional[bool] = None,\n        cat_embed_activation: Optional[str] = None,\n        continuous_cols: Optional[List[str]] = None,\n        cont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\n        embed_continuous_method: Optional[\n            Literal[\"standard\", \"piecewise\", \"periodic\"]\n        ] = None,\n        cont_embed_dropout: Optional[float] = None,\n        cont_embed_activation: Optional[str] = None,\n        quantization_setup: Optional[Dict[str, List[float]]] = None,\n        n_frequencies: Optional[int] = None,\n        sigma: Optional[float] = None,\n        share_last_layer: Optional[bool] = None,\n        full_embed_dropout: Optional[bool] = None,\n        mlp_hidden_dims: List[int] = [100, 100],\n        mlp_activation: str = \"relu\",\n        mlp_dropout: Union[float, List[float]] = 0.1,\n        mlp_batchnorm: bool = False,\n        mlp_batchnorm_last: bool = False,\n        mlp_linear_first: bool = True,\n    ):\n        super(AutoIntPlus, self).__init__(\n            column_idx=column_idx,\n            input_dim=input_dim,\n            cat_embed_input=cat_embed_input,\n            cat_embed_dropout=cat_embed_dropout,\n            use_cat_bias=use_cat_bias,\n            cat_embed_activation=cat_embed_activation,\n            shared_embed=None,\n            add_shared_embed=None,\n            frac_shared_embed=None,\n            continuous_cols=continuous_cols,\n            cont_norm_layer=cont_norm_layer,\n            embed_continuous_method=embed_continuous_method,\n            cont_embed_dropout=cont_embed_dropout,\n            cont_embed_activation=cont_embed_activation,\n            quantization_setup=quantization_setup,\n            n_frequencies=n_frequencies,\n            sigma=sigma,\n            share_last_layer=share_last_layer,\n            full_embed_dropout=full_embed_dropout,\n        )\n\n        self.input_dim = input_dim\n        self.num_heads = num_heads\n        self.num_layers = num_layers\n        self.reduction = reduction\n        self.structure = structure\n        self.gated = gated\n\n        self.mlp_hidden_dims = mlp_hidden_dims\n        self.mlp_activation = mlp_activation\n        self.mlp_dropout = mlp_dropout\n        self.mlp_batchnorm = mlp_batchnorm\n        self.mlp_batchnorm_last = mlp_batchnorm_last\n        self.mlp_linear_first = mlp_linear_first\n\n        self.attention_layers = nn.ModuleList(\n            [\n                nn.MultiheadAttention(\n                    embed_dim=self.input_dim,\n                    num_heads=self.num_heads,\n                    batch_first=True,\n                )\n                for _ in range(num_layers)\n            ]\n        )\n\n        if self.gated:\n            self.gate = self._build_gate()\n\n        deep_network_inp_dim = self._get_deep_network_input_dim()\n        self.deep_network = MLP(\n            [deep_network_inp_dim] + self.mlp_hidden_dims,\n            self.mlp_activation,\n            self.mlp_dropout,\n            self.mlp_batchnorm,\n            self.mlp_batchnorm_last,\n            self.mlp_linear_first,\n        )\n\n    def forward(self, X: torch.Tensor) -&gt; torch.Tensor:\n        x = self._get_embeddings(X)\n        attn_output = self._apply_attention_layers(x)\n        reduced_attn_output = self._reduce_attention_output(attn_output)\n        if self.structure == \"parallel\":\n            return self._parallel_output(reduced_attn_output, x)\n        else:  # structure == \"stacked\"\n            return self.deep_network(reduced_attn_output)\n\n    def _get_deep_network_input_dim(self) -&gt; int:\n        if self.structure == \"parallel\":\n            return self.input_dim * len(self.column_idx)\n        elif self.reduction == \"mean\":  # structure == \"stacked\"\n            return self.input_dim\n        else:\n            return self.input_dim * len(self.column_idx)\n\n    def _build_gate(self) -&gt; nn.Linear:\n        self._setup_gating()\n        return nn.Linear(self.input_dim * 2, self.input_dim)\n\n    def _setup_gating(self):\n        if self.structure == \"stacked\":\n            raise ValueError(\n                \"Gating is not supported for stacked structure. Set `gated=False`.\"\n            )\n        if self.reduction != \"mean\":\n            raise ValueError(\n                \"When using a gated structure, the reduction must be 'mean'.\"\n            )\n        if self.mlp_hidden_dims[-1] != self.input_dim:\n            self.mlp_hidden_dims = self.mlp_hidden_dims + [self.input_dim]\n            UserWarning(\n                \"When using a gated structure, the last hidden layer of \"\n                \"the MLP must have the same dimension as the input. \"\n                \"The last hidden layer has been set to the input dimension.\"\n            )\n\n    def _apply_attention_layers(self, x: torch.Tensor) -&gt; torch.Tensor:\n        attn_output = x\n        for layer in self.attention_layers:\n            layer_out, _ = layer(attn_output, attn_output, attn_output)\n            attn_output = layer_out + attn_output\n        return attn_output\n\n    def _reduce_attention_output(self, attn_output: torch.Tensor) -&gt; torch.Tensor:\n        return (\n            attn_output.mean(dim=1)\n            if self.reduction == \"mean\"\n            else attn_output.flatten(1)\n        )\n\n    def _parallel_output(\n        self, attn_output: torch.Tensor, x: torch.Tensor\n    ) -&gt; torch.Tensor:\n        deep_output = self.deep_network(x.reshape(x.size(0), -1))\n\n        if not self.gated:\n            return torch.cat([attn_output, deep_output], dim=1)\n\n        combined_features = torch.cat([attn_output, deep_output], dim=-1)\n        gate_weights = torch.sigmoid(self.gate(combined_features))\n        return gate_weights * attn_output + (1 - gate_weights) * deep_output\n\n    @property\n    def output_dim(self) -&gt; int:\n        if self.structure == \"parallel\":\n            if self.gated:\n                return self.input_dim\n            else:\n                if self.reduction == \"mean\":\n                    return self.mlp_hidden_dims[-1] + self.input_dim\n                else:\n                    return self.mlp_hidden_dims[-1] + self.input_dim * len(\n                        self.column_idx\n                    )\n        else:\n            return self.mlp_hidden_dims[-1]\n</code></pre>"},{"location":"pytorch-widedeep/the_rec_module.html#pytorch_widedeep.models.rec.basic_transformer.Transformer","title":"Transformer","text":"<p>               Bases: <code>Module</code></p> <p>Basic Encoder-Only Transformer Model for sequence classification/regression. As all other models in the library this model can be used as the <code>deeptext</code> component of a Wide &amp; Deep model or independently by itself.</p> <p> NOTE: This model is introduced in the context of recommendation systems and thought for sequences of any nature (e.g. items). It can, of course, still be used for text.</p> <p>Parameters:</p> Name Type Description Default <code>vocab_size</code> <code>int</code> <p>Number of words in the vocabulary</p> required <code>input_dim</code> <code>int</code> <p>Dimension of the token embeddings</p> <p>Param aliases: <code>embed_dim</code>, <code>d_model</code>. </p> required <code>seq_length</code> <code>int</code> <p>Input sequence length</p> required <code>n_heads</code> <code>int</code> <p>Number of attention heads per Transformer block</p> required <code>n_blocks</code> <code>int</code> <p>Number of Transformer blocks</p> required <code>attn_dropout</code> <code>float</code> <p>Dropout that will be applied to the Multi-Head Attention layers</p> <code>0.1</code> <code>ff_dropout</code> <code>float</code> <p>Dropout that will be applied to the FeedForward network</p> <code>0.1</code> <code>ff_factor</code> <code>int</code> <p>Multiplicative factor applied to the first layer of the FF network in each Transformer block, This is normally set to 4.</p> <code>4</code> <code>activation</code> <code>str</code> <p>Transformer Encoder activation function. 'tanh', 'relu', 'leaky_relu', 'gelu', 'geglu' and 'reglu' are supported</p> <code>'gelu'</code> <code>padding_idx</code> <code>int</code> <p>index of the padding token in the padded-tokenised sequences.</p> <code>0</code> <code>with_cls_token</code> <code>bool</code> <p>Boolean indicating if a <code>'[CLS]'</code> token is included in the tokenized sequences. If present, the final hidden state corresponding to this token is used as the aggregated representation for classification and regression tasks. NOTE: if included in the tokenized sequences it must be inserted as the first token in the sequences.</p> <code>False</code> <code>with_pos_encoding</code> <code>bool</code> <p>Boolean indicating if positional encoding will be used</p> <code>True</code> <code>pos_encoding_dropout</code> <code>float</code> <p>Positional encoding dropout</p> <code>0.1</code> <code>pos_encoder</code> <code>Optional[Module]</code> <p>This model uses by default a standard positional encoding approach. However, any custom positional encoder can also be used and pass to the Transformer model via the 'pos_encoder' parameter</p> <code>None</code> <p>Attributes:</p> Name Type Description <code>embedding</code> <code>Module</code> <p>Standard token embedding layer</p> <code>pos_encoder</code> <code>Module</code> <p>Positional Encoder</p> <code>encoder</code> <code>Module</code> <p>Sequence of Transformer blocks</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models.rec import Transformer\n&gt;&gt;&gt; X = torch.cat((torch.zeros([5,1]), torch.empty(5, 4).random_(1,4)), axis=1)\n&gt;&gt;&gt; model = Transformer(vocab_size=4, seq_length=5, input_dim=8, n_heads=1, n_blocks=1)\n&gt;&gt;&gt; out = model(X)\n</code></pre> Source code in <code>pytorch_widedeep/models/rec/basic_transformer.py</code> <pre><code>class Transformer(nn.Module):\n    r\"\"\"\n    Basic Encoder-Only Transformer Model for sequence\n    classification/regression. As all other models in the library this model\n    can be used as the `deeptext` component of a Wide &amp; Deep model or\n    independently by itself.\n\n    :information_source: **NOTE**:\n    This model is introduced in the context of recommendation systems and\n    thought for sequences of any nature (e.g. items). It can, of course,\n    still be used for text.\n\n    Parameters\n    ----------\n    vocab_size: int\n        Number of words in the vocabulary\n    input_dim: int\n        Dimension of the token embeddings\n\n        Param aliases: `embed_dim`, `d_model`. &lt;br/&gt;\n    seq_length: int, Optional, default = None\n        Input sequence length\n    n_heads: int, default = 8\n        Number of attention heads per Transformer block\n    n_blocks: int, default = 4\n        Number of Transformer blocks\n    attn_dropout: float, default = 0.2\n        Dropout that will be applied to the Multi-Head Attention layers\n    ff_dropout: float, default = 0.1\n        Dropout that will be applied to the FeedForward network\n    ff_factor: float, default = 4\n        Multiplicative factor applied to the first layer of the FF network in\n        each Transformer block, This is normally set to 4.\n    activation: str, default = \"gelu\"\n        Transformer Encoder activation function. _'tanh'_, _'relu'_,\n        _'leaky_relu'_, _'gelu'_, _'geglu'_ and _'reglu'_ are supported\n    padding_idx: int, default = 0\n        index of the padding token in the padded-tokenised sequences.\n    with_cls_token: bool, default = False\n        Boolean indicating if a `'[CLS]'` token is included in the tokenized\n        sequences. If present, the final hidden state corresponding to this\n        token is used as the aggregated representation for classification and\n        regression tasks. **NOTE**: if included in the tokenized sequences it\n        must be inserted as the first token in the sequences.\n    with_pos_encoding: bool, default = True\n        Boolean indicating if positional encoding will be used\n    pos_encoding_dropout: float, default = 0.1\n        Positional encoding dropout\n    pos_encoder: nn.Module, Optional, default = None\n        This model uses by default a standard positional encoding approach.\n        However, any custom positional encoder can also be used and pass to\n        the Transformer model via the 'pos_encoder' parameter\n\n    Attributes\n    ----------\n    embedding: nn.Module\n        Standard token embedding layer\n    pos_encoder: nn.Module\n        Positional Encoder\n    encoder: nn.Module\n        Sequence of Transformer blocks\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.models.rec import Transformer\n    &gt;&gt;&gt; X = torch.cat((torch.zeros([5,1]), torch.empty(5, 4).random_(1,4)), axis=1)\n    &gt;&gt;&gt; model = Transformer(vocab_size=4, seq_length=5, input_dim=8, n_heads=1, n_blocks=1)\n    &gt;&gt;&gt; out = model(X)\n    \"\"\"\n\n    @alias(\"input_dim\", [\"embed_dim\", \"d_model\"])\n    @alias(\"seq_length\", [\"max_length\", \"maxlen\"])\n    def __init__(\n        self,\n        *,\n        vocab_size: int,\n        seq_length: int,\n        input_dim: int,\n        n_heads: int,\n        n_blocks: int,\n        attn_dropout: float = 0.1,\n        ff_dropout: float = 0.1,\n        ff_factor: int = 4,\n        activation: str = \"gelu\",\n        use_linear_attention: bool = False,\n        use_flash_attention: bool = False,\n        padding_idx: int = 0,\n        with_cls_token: bool = False,\n        with_pos_encoding: bool = True,\n        pos_encoding_dropout: float = 0.1,\n        pos_encoder: Optional[nn.Module] = None,\n    ):\n        super().__init__()\n\n        self.input_dim = input_dim\n        self.seq_length = seq_length\n        self.n_heads = n_heads\n        self.n_blocks = n_blocks\n        self.attn_dropout = attn_dropout\n        self.ff_dropout = ff_dropout\n        self.ff_factor = ff_factor\n        self.activation = activation\n        self.use_linear_attention = use_linear_attention\n        self.use_flash_attention = use_flash_attention\n        self.padding_idx = padding_idx\n        self.with_cls_token = with_cls_token\n        self.with_pos_encoding = with_pos_encoding\n        self.pos_encoding_dropout = pos_encoding_dropout\n\n        self.embedding = nn.Embedding(\n            vocab_size, input_dim, padding_idx=self.padding_idx\n        )\n\n        if with_pos_encoding:\n            if pos_encoder is not None:\n                self.pos_encoder: Union[nn.Module, nn.Identity, PositionalEncoding] = (\n                    pos_encoder\n                )\n            else:\n                self.pos_encoder = PositionalEncoding(\n                    input_dim, pos_encoding_dropout, seq_length\n                )\n        else:\n            self.pos_encoder = nn.Identity()\n\n        self.encoder = nn.Sequential()\n        for i in range(n_blocks):\n            self.encoder.add_module(\n                \"transformer_block\" + str(i),\n                TransformerEncoder(\n                    input_dim,\n                    n_heads,\n                    False,  # use_qkv_bias\n                    attn_dropout,\n                    ff_dropout,\n                    ff_factor,\n                    activation,\n                    use_linear_attention,\n                    use_flash_attention,\n                ),\n            )\n\n    def forward(self, X: Tensor) -&gt; Tensor:\n        x = self.embedding(X.long())\n        x = self.pos_encoder(x)\n        x = self.encoder(x)\n        if self.with_cls_token:\n            x = x[:, 0, :]\n        else:\n            x = x.flatten(1)\n        return x\n\n    @property\n    def output_dim(self) -&gt; int:\n        if self.with_cls_token:\n            output_dim = self.input_dim\n        else:\n            output_dim = self.input_dim * self.seq_length\n        return output_dim\n</code></pre>"},{"location":"pytorch-widedeep/the_rec_module.html#pytorch_widedeep.models.rec.dcn.DeepCrossNetwork","title":"DeepCrossNetwork","text":"<p>               Bases: <code>BaseTabularModelWithoutAttention</code></p> <p>Defines a <code>DeepCrossNetwork</code> model that can be used as the <code>deeptabular</code> component of a Wide &amp; Deep model or independently by itself.</p> <p>This class implements the Deep &amp; Cross Network for Ad Click Predictions architecture, which automatically combines features to generate feature interactions in an explicit fashion and at each layer.</p> <p>The cross layer implements the following equation:</p> \\[x_{l+1} = x_0 \\odot (W_l x_l + b_l) + x_l\\] <p>where:</p> <ul> <li>\\(\\odot\\) represents element-wise multiplication</li> <li>\\(x_l\\), \\(x_{l+1}\\) are the outputs from the \\(l^{th}\\) and \\((l+1)^{th}\\) cross layers</li> <li>\\(W_l\\), \\(b_l\\) are the weight and bias parameters</li> </ul> <p>Parameters:</p> Name Type Description Default <code>column_idx</code> <code>Dict[str, int]</code> <p>Dict containing the index of the columns that will be passed through the <code>DeepCrossNetwork</code> model. Required to slice the tensors. e.g. {'education': 0, 'relationship': 1, 'workclass': 2, ...}.</p> required <code>n_cross_layers</code> <code>int</code> <p>Number of cross layers in the cross network</p> <code>3</code> <code>cat_embed_input</code> <code>Optional[List[Tuple[str, int, int]]]</code> <p>List of Tuples with the column name, number of unique values and embedding dimension. e.g. [(education, 11, 32), ...]</p> <code>None</code> <code>cat_embed_dropout</code> <code>Optional[float]</code> <p>Categorical embeddings dropout. If <code>None</code>, it will default to 0.</p> <code>None</code> <code>use_cat_bias</code> <code>Optional[bool]</code> <p>Boolean indicating if bias will be used for the categorical embeddings</p> <code>None</code> <code>cat_embed_activation</code> <code>Optional[str]</code> <p>Activation function for the categorical embeddings</p> <code>None</code> <code>continuous_cols</code> <code>Optional[List[str]]</code> <p>List with the name of the numeric (aka continuous) columns</p> <code>None</code> <code>cont_norm_layer</code> <code>Optional[Literal['batchnorm', 'layernorm']]</code> <p>Type of normalization layer applied to the continuous features. Options are: 'layernorm' and 'batchnorm'. if <code>None</code>, no normalization layer will be used.</p> <code>None</code> <code>embed_continuous_method</code> <code>Optional[Literal['standard', 'piecewise', 'periodic']]</code> <p>Method to use to embed the continuous features. Options are: 'standard', 'periodic' or 'piecewise'. The 'standard' embedding method is based on the FT-Transformer implementation presented in the paper: Revisiting Deep Learning Models for Tabular Data. The 'periodic' and_'piecewise'_ methods were presented in the paper: On Embeddings for Numerical Features in Tabular Deep Learning. Please, read the papers for details.</p> <code>None</code> <code>cont_embed_dropout</code> <code>Optional[float]</code> <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p> <code>None</code> <code>cont_embed_activation</code> <code>Optional[str]</code> <p>Activation function for the continuous embeddings if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported. If <code>None</code>, no activation function will be applied.</p> <code>None</code> <code>quantization_setup</code> <code>Optional[Dict[str, List[float]]]</code> <p>This parameter is used when the 'piecewise' method is used to embed the continuous cols. It is a dict where keys are the name of the continuous columns and values are lists with the boundaries for the quantization of the continuous_cols. See the examples for details. If If the 'piecewise' method is used, this parameter is required.</p> <code>None</code> <code>n_frequencies</code> <code>Optional[int]</code> <p>This is the so called 'k' in their paper On Embeddings for Numerical Features in Tabular Deep Learning, and is the number of 'frequencies' that will be used to represent each continuous column. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>sigma</code> <code>Optional[float]</code> <p>This is the sigma parameter in the paper mentioned when describing the previous parameters and it is used to initialise the 'frequency weights'. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>share_last_layer</code> <code>Optional[bool]</code> <p>This parameter is not present in the before mentioned paper but it is implemented in the official repo. If <code>True</code> the linear layer that turns the frequencies into embeddings will be shared across the continuous columns. If <code>False</code> a different linear layer will be used for each continuous column. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>full_embed_dropout</code> <code>Optional[bool]</code> <p>If <code>True</code>, the full embedding corresponding to a column will be masked out/dropout. If <code>None</code>, it will default to <code>False</code>.</p> <code>None</code> <code>mlp_hidden_dims</code> <code>List[int]</code> <p>List with the number of neurons per dense layer in the mlp.</p> <code>[200, 100]</code> <code>mlp_activation</code> <code>str</code> <p>Activation function for the dense layers of the MLP. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> <code>'relu'</code> <code>mlp_dropout</code> <code>Union[float, List[float]]</code> <p>float or List of floats with the dropout between the dense layers. e.g: [0.5,0.5]</p> <code>0.1</code> <code>mlp_batchnorm</code> <code>bool</code> <p>Boolean indicating whether or not batch normalization will be applied to the dense layers</p> <code>False</code> <code>mlp_batchnorm_last</code> <code>bool</code> <p>Boolean indicating whether or not batch normalization will be applied to the last of the dense layers</p> <code>False</code> <code>mlp_linear_first</code> <code>bool</code> <p>Boolean indicating the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code></p> <code>True</code> <p>Attributes:</p> Name Type Description <code>cross_network</code> <code>Module</code> <p>The cross network component</p> <code>deep_network</code> <code>Module</code> <p>The deep network component (MLP)</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models.rec import DeepCrossNetwork\n&gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n&gt;&gt;&gt; colnames = [\"a\", \"b\", \"c\", \"d\", \"e\"]\n&gt;&gt;&gt; cat_embed_input = [(u, i, j) for u, i, j in zip(colnames[:4], [4] * 4, [8] * 4)]\n&gt;&gt;&gt; column_idx = {k: v for v, k in enumerate(colnames)}\n&gt;&gt;&gt; model = DeepCrossNetwork(\n...     column_idx=column_idx,\n...     cat_embed_input=cat_embed_input,\n...     continuous_cols=[\"e\"],\n...     n_cross_layers=2,\n...     mlp_hidden_dims=[16, 8]\n... )\n&gt;&gt;&gt; out = model(X_tab)\n</code></pre> Source code in <code>pytorch_widedeep/models/rec/dcn.py</code> <pre><code>class DeepCrossNetwork(BaseTabularModelWithoutAttention):\n    r\"\"\"Defines a `DeepCrossNetwork` model that can be used as the `deeptabular`\n    component of a Wide &amp; Deep model or independently by itself.\n\n    This class implements the [Deep &amp; Cross Network for Ad Click Predictions](https://arxiv.org/abs/1708.05123)\n    architecture, which automatically combines features to generate feature interactions\n    in an explicit fashion and at each layer.\n\n    The cross layer implements the following equation:\n\n    $$x_{l+1} = x_0 \\odot (W_l x_l + b_l) + x_l$$\n\n    where:\n\n    * $\\odot$ represents element-wise multiplication\n    * $x_l$, $x_{l+1}$ are the outputs from the $l^{th}$ and $(l+1)^{th}$ cross layers\n    * $W_l$, $b_l$ are the weight and bias parameters\n\n    Parameters\n    ----------\n    column_idx: Dict\n        Dict containing the index of the columns that will be passed through\n        the `DeepCrossNetwork` model. Required to slice the tensors. e.g.\n        _{'education': 0, 'relationship': 1, 'workclass': 2, ...}_.\n    n_cross_layers: int, default = 3\n        Number of cross layers in the cross network\n    cat_embed_input: List, Optional, default = None\n        List of Tuples with the column name, number of unique values and\n        embedding dimension. e.g. _[(education, 11, 32), ...]_\n    cat_embed_dropout: float, Optional, default = None\n        Categorical embeddings dropout. If `None`, it will default to 0.\n    use_cat_bias: bool, Optional, default = None,\n        Boolean indicating if bias will be used for the categorical embeddings\n    cat_embed_activation: Optional, str, default = None,\n        Activation function for the categorical embeddings\n    continuous_cols : Optional[List[str]], default=None\n        List with the name of the numeric (aka continuous) columns\n    cont_norm_layer : Optional[Literal[\"batchnorm\", \"layernorm\"]], default=None\n        Type of normalization layer applied to the continuous features.\n        Options are: _'layernorm'_ and _'batchnorm'_. if `None`, no\n        normalization layer will be used.\n    embed_continuous_method: Optional[Literal[\"piecewise\", \"periodic\", \"standard\"]], default=\"standard\"\n        Method to use to embed the continuous features. Options are:\n        _'standard'_, _'periodic'_ or _'piecewise'_. The _'standard'_\n        embedding method is based on the FT-Transformer implementation\n        presented in the paper: [Revisiting Deep Learning Models for\n        Tabular Data](https://arxiv.org/abs/2106.11959v5). The _'periodic'_\n        and_'piecewise'_ methods were presented in the paper: [On Embeddings for\n        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556).\n        Please, read the papers for details.\n    cont_embed_dropout : Optional[float], default=None\n        Dropout for the continuous embeddings. If `None`, it will default to 0.0\n    cont_embed_activation : Optional[str], default=None\n        Activation function for the continuous embeddings if any. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported.\n        If `None`, no activation function will be applied.\n    quantization_setup : Optional[Dict[str, List[float]]], default=None\n        This parameter is used when the _'piecewise'_ method is used to embed\n        the continuous cols. It is a dict where keys are the name of the continuous\n        columns and values are lists with the boundaries for the quantization\n        of the continuous_cols. See the examples for details. If\n        If the _'piecewise'_ method is used, this parameter is required.\n    n_frequencies : Optional[int], default=None\n        This is the so called _'k'_ in their paper [On Embeddings for\n        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556),\n        and is the number of 'frequencies' that will be used to represent each\n        continuous column. See their Eq 2 in the paper for details. If\n        the _'periodic'_ method is used, this parameter is required.\n    sigma : Optional[float], default=None\n        This is the sigma parameter in the paper mentioned when describing the\n        previous parameters and it is used to initialise the 'frequency\n        weights'. See their Eq 2 in the paper for details. If\n        the _'periodic'_ method is used, this parameter is required.\n    share_last_layer : Optional[bool], default=None\n        This parameter is not present in the before mentioned paper but it is implemented in\n        the [official repo](https://github.com/yandex-research/rtdl-num-embeddings/tree/main).\n        If `True` the linear layer that turns the frequencies into embeddings\n        will be shared across the continuous columns. If `False` a different\n        linear layer will be used for each continuous column.\n        If the _'periodic'_ method is used, this parameter is required.\n    full_embed_dropout: bool, Optional, default = None,\n        If `True`, the full embedding corresponding to a column will be masked\n        out/dropout. If `None`, it will default to `False`.\n    mlp_hidden_dims: List, default = [200, 100]\n        List with the number of neurons per dense layer in the mlp.\n    mlp_activation: str, default = \"relu\"\n        Activation function for the dense layers of the MLP. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported\n    mlp_dropout: float or List, default = 0.1\n        float or List of floats with the dropout between the dense layers.\n        e.g: _[0.5,0.5]_\n    mlp_batchnorm: bool, default = False\n        Boolean indicating whether or not batch normalization will be applied\n        to the dense layers\n    mlp_batchnorm_last: bool, default = False\n        Boolean indicating whether or not batch normalization will be applied\n        to the last of the dense layers\n    mlp_linear_first: bool, default = False\n        Boolean indicating the order of the operations in the dense\n        layer. If `True: [LIN -&gt; ACT -&gt; BN -&gt; DP]`. If `False: [BN -&gt; DP -&gt;\n        LIN -&gt; ACT]`\n\n    Attributes\n    ----------\n    cross_network: nn.Module\n        The cross network component\n    deep_network: nn.Module\n        The deep network component (MLP)\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.models.rec import DeepCrossNetwork\n    &gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n    &gt;&gt;&gt; colnames = [\"a\", \"b\", \"c\", \"d\", \"e\"]\n    &gt;&gt;&gt; cat_embed_input = [(u, i, j) for u, i, j in zip(colnames[:4], [4] * 4, [8] * 4)]\n    &gt;&gt;&gt; column_idx = {k: v for v, k in enumerate(colnames)}\n    &gt;&gt;&gt; model = DeepCrossNetwork(\n    ...     column_idx=column_idx,\n    ...     cat_embed_input=cat_embed_input,\n    ...     continuous_cols=[\"e\"],\n    ...     n_cross_layers=2,\n    ...     mlp_hidden_dims=[16, 8]\n    ... )\n    &gt;&gt;&gt; out = model(X_tab)\n    \"\"\"\n\n    def __init__(\n        self,\n        *,\n        column_idx: Dict[str, int],\n        n_cross_layers: int = 3,\n        cat_embed_input: Optional[List[Tuple[str, int, int]]] = None,\n        cat_embed_dropout: Optional[float] = None,\n        use_cat_bias: Optional[bool] = None,\n        cat_embed_activation: Optional[str] = None,\n        continuous_cols: Optional[List[str]] = None,\n        cont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\n        embed_continuous: Optional[bool] = None,\n        embed_continuous_method: Optional[\n            Literal[\"standard\", \"piecewise\", \"periodic\"]\n        ] = None,\n        cont_embed_dim: Optional[int] = None,\n        cont_embed_dropout: Optional[float] = None,\n        cont_embed_activation: Optional[str] = None,\n        quantization_setup: Optional[Dict[str, List[float]]] = None,\n        n_frequencies: Optional[int] = None,\n        sigma: Optional[float] = None,\n        share_last_layer: Optional[bool] = None,\n        full_embed_dropout: Optional[bool] = None,\n        mlp_hidden_dims: List[int] = [200, 100],\n        mlp_activation: str = \"relu\",\n        mlp_dropout: Union[float, List[float]] = 0.1,\n        mlp_batchnorm: bool = False,\n        mlp_batchnorm_last: bool = False,\n        mlp_linear_first: bool = True,\n    ):\n        super(DeepCrossNetwork, self).__init__(\n            column_idx=column_idx,\n            cat_embed_input=cat_embed_input,\n            cat_embed_dropout=cat_embed_dropout,\n            use_cat_bias=use_cat_bias,\n            cat_embed_activation=cat_embed_activation,\n            continuous_cols=continuous_cols,\n            cont_norm_layer=cont_norm_layer,\n            embed_continuous=embed_continuous,\n            embed_continuous_method=embed_continuous_method,\n            cont_embed_dim=cont_embed_dim,\n            cont_embed_dropout=cont_embed_dropout,\n            cont_embed_activation=cont_embed_activation,\n            quantization_setup=quantization_setup,\n            n_frequencies=n_frequencies,\n            sigma=sigma,\n            share_last_layer=share_last_layer,\n            full_embed_dropout=full_embed_dropout,\n        )\n\n        self.mlp_hidden_dims = mlp_hidden_dims\n        self.mlp_activation = mlp_activation\n        self.mlp_dropout = mlp_dropout\n        self.mlp_batchnorm = mlp_batchnorm\n        self.mlp_batchnorm_last = mlp_batchnorm_last\n        self.mlp_linear_first = mlp_linear_first\n\n        embeddings_output_dim = self.cat_out_dim + self.cont_out_dim\n        self.deep_network = MLP(\n            [embeddings_output_dim] + self.mlp_hidden_dims,\n            self.mlp_activation,\n            self.mlp_dropout,\n            self.mlp_batchnorm,\n            self.mlp_batchnorm_last,\n            self.mlp_linear_first,\n        )\n        self.cross_network = CrossNetwork(embeddings_output_dim, n_cross_layers)\n\n    def forward(self, X: torch.Tensor) -&gt; torch.Tensor:\n        x = self._get_embeddings(X)\n        cross_output = self.cross_network(x)\n        deep_output = self.deep_network(x)\n        return torch.cat([cross_output, deep_output], dim=1)\n\n    @property\n    def output_dim(self) -&gt; int:\n        return self.mlp_hidden_dims[-1] + self.cat_out_dim + self.cont_out_dim\n</code></pre>"},{"location":"pytorch-widedeep/the_rec_module.html#pytorch_widedeep.models.rec.dcnv2.DeepCrossNetworkV2","title":"DeepCrossNetworkV2","text":"<p>               Bases: <code>BaseTabularModelWithoutAttention</code></p> <p>Defines a <code>DeepCrossNetworkV2</code> model that can be used as the <code>deeptabular</code> component of a Wide &amp; Deep model or independently by itself.</p> <p>This class implements the DCN V2: Improved Deep &amp; Cross Network and Practical Lessons for Web-scale Learning to Rank Systems architecture, which enhances the original DCN by introducing a more expressive cross network that uses multiple experts and matrix decomposition techniques to improve model capacity while maintaining computational efficiency.</p> <p>The cross layer implements the following equation:</p> \\[E_i(x_l) = x_0 \\odot (U_l^i \\cdot g(C_l^i \\cdot g(V_l^iT x_l)) + b_l)\\] <p>where:</p> <ul> <li>\\(\\odot\\) represents element-wise multiplication</li> <li>\\(U_l^i\\), \\(C_l^i\\), \\(V_l^i\\) are the decomposed weight matrices for expert \\(i\\) at layer \\(l\\)</li> <li>\\(g\\) is the activation function (ReLU)</li> <li>\\(b_l\\) is the bias term</li> </ul> <p>Parameters:</p> Name Type Description Default <code>column_idx</code> <code>Dict[str, int]</code> <p>Dict containing the index of the columns that will be passed through the <code>DeepCrossNetworkV2</code> model. Required to slice the tensors. e.g. {'education': 0, 'relationship': 1, 'workclass': 2, ...}.</p> required <code>num_cross_layers</code> <code>int</code> <p>Number of cross layers in the cross network</p> <code>2</code> <code>low_rank</code> <code>Optional[int]</code> <p>Rank of the weight matrix decomposition. If None, full-rank weights are used</p> <code>None</code> <code>num_experts</code> <code>int</code> <p>Number of expert networks in mixture of experts</p> <code>2</code> <code>expert_dropout</code> <code>float</code> <p>Dropout rate for expert outputs</p> <code>0.0</code> <code>structure</code> <code>Literal['stacked', 'parallel']</code> <p>Structure of the model. Either 'parallel' or 'stacked'. If 'parallel', the output will be the concatenation of the cross network and deep network outputs. If 'stacked', the cross network output will be fed into the deep network.</p> <code>'parallel'</code> <code>cat_embed_input</code> <code>Optional[List[Tuple[str, int, int]]]</code> <p>List of Tuples with the column name, number of unique values and embedding dimension. e.g. [(education, 11, 32), ...]</p> <code>None</code> <code>cat_embed_dropout</code> <code>Optional[float]</code> <p>Categorical embeddings dropout. If <code>None</code>, it will default to 0.</p> <code>None</code> <code>use_cat_bias</code> <code>Optional[bool]</code> <p>Boolean indicating if bias will be used for the categorical embeddings</p> <code>None</code> <code>cat_embed_activation</code> <code>Optional[str]</code> <p>Activation function for the categorical embeddings</p> <code>None</code> <code>continuous_cols</code> <code>Optional[List[str]]</code> <p>List with the name of the numeric (aka continuous) columns</p> <code>None</code> <code>cont_norm_layer</code> <code>Optional[Literal['batchnorm', 'layernorm']]</code> <p>Type of normalization layer applied to the continuous features. Options are: 'layernorm' and 'batchnorm'. if <code>None</code>, no normalization layer will be used.</p> <code>None</code> <code>embed_continuous_method</code> <code>Optional[Literal['standard', 'piecewise', 'periodic']]</code> <p>Method to use to embed the continuous features. Options are: 'standard', 'periodic' or 'piecewise'. The 'standard' embedding method is based on the FT-Transformer implementation presented in the paper: Revisiting Deep Learning Models for Tabular Data. The 'periodic' and_'piecewise'_ methods were presented in the paper: On Embeddings for Numerical Features in Tabular Deep Learning. Please, read the papers for details.</p> <code>None</code> <code>cont_embed_dropout</code> <code>Optional[float]</code> <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p> <code>None</code> <code>cont_embed_activation</code> <code>Optional[str]</code> <p>Activation function for the continuous embeddings if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported. If <code>None</code>, no activation function will be applied.</p> <code>None</code> <code>quantization_setup</code> <code>Optional[Dict[str, List[float]]]</code> <p>This parameter is used when the 'piecewise' method is used to embed the continuous cols. It is a dict where keys are the name of the continuous columns and values are lists with the boundaries for the quantization of the continuous_cols. See the examples for details. If If the 'piecewise' method is used, this parameter is required.</p> <code>None</code> <code>n_frequencies</code> <code>Optional[int]</code> <p>This is the so called 'k' in their paper On Embeddings for Numerical Features in Tabular Deep Learning, and is the number of 'frequencies' that will be used to represent each continuous column. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>sigma</code> <code>Optional[float]</code> <p>This is the sigma parameter in the paper mentioned when describing the previous parameters and it is used to initialise the 'frequency weights'. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>share_last_layer</code> <code>Optional[bool]</code> <p>This parameter is not present in the before mentioned paper but it is implemented in the official repo. If <code>True</code> the linear layer that turns the frequencies into embeddings will be shared across the continuous columns. If <code>False</code> a different linear layer will be used for each continuous column. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>full_embed_dropout</code> <code>Optional[bool]</code> <p>If <code>True</code>, the full embedding corresponding to a column will be masked out/dropout. If <code>None</code>, it will default to <code>False</code>.</p> <code>None</code> <code>mlp_hidden_dims</code> <code>List[int]</code> <p>List with the number of neurons per dense layer in the mlp.</p> <code>[200, 100]</code> <code>mlp_activation</code> <code>str</code> <p>Activation function for the dense layers of the MLP. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> <code>'relu'</code> <code>mlp_dropout</code> <code>Union[float, List[float]]</code> <p>float or List of floats with the dropout between the dense layers. e.g: [0.5,0.5]</p> <code>0.1</code> <code>mlp_batchnorm</code> <code>bool</code> <p>Boolean indicating whether or not batch normalization will be applied to the dense layers</p> <code>False</code> <code>mlp_batchnorm_last</code> <code>bool</code> <p>Boolean indicating whether or not batch normalization will be applied to the last of the dense layers</p> <code>False</code> <code>mlp_linear_first</code> <code>bool</code> <p>Boolean indicating the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code></p> <code>True</code> <p>Attributes:</p> Name Type Description <code>cross_network</code> <code>Module</code> <p>The cross network component with mixture of experts</p> <code>deep_network</code> <code>Module</code> <p>The deep network component (MLP)</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models.rec import DeepCrossNetworkV2\n&gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n&gt;&gt;&gt; colnames = [\"a\", \"b\", \"c\", \"d\", \"e\"]\n&gt;&gt;&gt; cat_embed_input = [(u, i, j) for u, i, j in zip(colnames[:4], [4] * 4, [8] * 4)]\n&gt;&gt;&gt; column_idx = {k: v for v, k in enumerate(colnames)}\n&gt;&gt;&gt; model = DeepCrossNetworkV2(\n...     column_idx=column_idx,\n...     cat_embed_input=cat_embed_input,\n...     continuous_cols=[\"e\"],\n...     num_cross_layers=2,\n...     low_rank=32,\n...     num_experts=4,\n...     mlp_hidden_dims=[16, 8]\n... )\n&gt;&gt;&gt; out = model(X_tab)\n</code></pre> Source code in <code>pytorch_widedeep/models/rec/dcnv2.py</code> <pre><code>class DeepCrossNetworkV2(BaseTabularModelWithoutAttention):\n    r\"\"\"Defines a `DeepCrossNetworkV2` model that can be used as the `deeptabular`\n    component of a Wide &amp; Deep model or independently by itself.\n\n    This class implements the\n    [DCN V2: Improved Deep &amp; Cross Network and Practical Lessons for Web-scale Learning to Rank Systems](https://arxiv.org/pdf/2008.13535)\n    architecture, which enhances the original DCN by introducing a more expressive cross\n    network that uses multiple experts and matrix decomposition techniques to improve\n    model capacity while maintaining computational efficiency.\n\n    The cross layer implements the following equation:\n\n    $$E_i(x_l) = x_0 \\odot (U_l^i \\cdot g(C_l^i \\cdot g(V_l^iT x_l)) + b_l)$$\n\n    where:\n\n    * $\\odot$ represents element-wise multiplication\n    * $U_l^i$, $C_l^i$, $V_l^i$ are the decomposed weight matrices for expert $i$ at layer $l$\n    * $g$ is the activation function (ReLU)\n    * $b_l$ is the bias term\n\n    Parameters\n    ----------\n    column_idx: Dict\n        Dict containing the index of the columns that will be passed through\n        the `DeepCrossNetworkV2` model. Required to slice the tensors. e.g.\n        _{'education': 0, 'relationship': 1, 'workclass': 2, ...}_.\n    num_cross_layers: int, default = 2\n        Number of cross layers in the cross network\n    low_rank: int, Optional, default = None\n        Rank of the weight matrix decomposition. If None, full-rank weights are used\n    num_experts: int, default = 2\n        Number of expert networks in mixture of experts\n    expert_dropout: float, default = 0.0\n        Dropout rate for expert outputs\n    structure: str, default = \"parallel\"\n        Structure of the model. Either _'parallel'_ or _'stacked'_. If _'parallel'_,\n        the output will be the concatenation of the cross network and deep network\n        outputs. If _'stacked'_, the cross network output will be fed into the deep\n        network.\n    cat_embed_input: List, Optional, default = None\n        List of Tuples with the column name, number of unique values and\n        embedding dimension. e.g. _[(education, 11, 32), ...]_\n    cat_embed_dropout: float, Optional, default = None\n        Categorical embeddings dropout. If `None`, it will default to 0.\n    use_cat_bias: bool, Optional, default = None,\n        Boolean indicating if bias will be used for the categorical embeddings\n    cat_embed_activation: Optional, str, default = None,\n        Activation function for the categorical embeddings\n    continuous_cols : Optional[List[str]], default=None\n        List with the name of the numeric (aka continuous) columns\n    cont_norm_layer : Optional[Literal[\"batchnorm\", \"layernorm\"]], default=None\n        Type of normalization layer applied to the continuous features.\n        Options are: _'layernorm'_ and _'batchnorm'_. if `None`, no\n        normalization layer will be used.\n    embed_continuous_method: Optional[Literal[\"piecewise\", \"periodic\", \"standard\"]], default=\"standard\"\n        Method to use to embed the continuous features. Options are:\n        _'standard'_, _'periodic'_ or _'piecewise'_. The _'standard'_\n        embedding method is based on the FT-Transformer implementation\n        presented in the paper: [Revisiting Deep Learning Models for\n        Tabular Data](https://arxiv.org/abs/2106.11959v5). The _'periodic'_\n        and_'piecewise'_ methods were presented in the paper: [On Embeddings for\n        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556).\n        Please, read the papers for details.\n    cont_embed_dropout : Optional[float], default=None\n        Dropout for the continuous embeddings. If `None`, it will default to 0.0\n    cont_embed_activation : Optional[str], default=None\n        Activation function for the continuous embeddings if any. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported.\n        If `None`, no activation function will be applied.\n    quantization_setup : Optional[Dict[str, List[float]]], default=None\n        This parameter is used when the _'piecewise'_ method is used to embed\n        the continuous cols. It is a dict where keys are the name of the continuous\n        columns and values are lists with the boundaries for the quantization\n        of the continuous_cols. See the examples for details. If\n        If the _'piecewise'_ method is used, this parameter is required.\n    n_frequencies : Optional[int], default=None\n        This is the so called _'k'_ in their paper [On Embeddings for\n        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556),\n        and is the number of 'frequencies' that will be used to represent each\n        continuous column. See their Eq 2 in the paper for details. If\n        the _'periodic'_ method is used, this parameter is required.\n    sigma : Optional[float], default=None\n        This is the sigma parameter in the paper mentioned when describing the\n        previous parameters and it is used to initialise the 'frequency\n        weights'. See their Eq 2 in the paper for details. If\n        the _'periodic'_ method is used, this parameter is required.\n    share_last_layer : Optional[bool], default=None\n        This parameter is not present in the before mentioned paper but it is implemented in\n        the [official repo](https://github.com/yandex-research/rtdl-num-embeddings/tree/main).\n        If `True` the linear layer that turns the frequencies into embeddings\n        will be shared across the continuous columns. If `False` a different\n        linear layer will be used for each continuous column.\n        If the _'periodic'_ method is used, this parameter is required.\n    full_embed_dropout: bool, Optional, default = None,\n        If `True`, the full embedding corresponding to a column will be masked\n        out/dropout. If `None`, it will default to `False`.\n    mlp_hidden_dims: List, default = [200, 100]\n        List with the number of neurons per dense layer in the mlp.\n    mlp_activation: str, default = \"relu\"\n        Activation function for the dense layers of the MLP. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported\n    mlp_dropout: float or List, default = 0.1\n        float or List of floats with the dropout between the dense layers.\n        e.g: _[0.5,0.5]_\n    mlp_batchnorm: bool, default = False\n        Boolean indicating whether or not batch normalization will be applied\n        to the dense layers\n    mlp_batchnorm_last: bool, default = False\n        Boolean indicating whether or not batch normalization will be applied\n        to the last of the dense layers\n    mlp_linear_first: bool, default = False\n        Boolean indicating the order of the operations in the dense\n        layer. If `True: [LIN -&gt; ACT -&gt; BN -&gt; DP]`. If `False: [BN -&gt; DP -&gt;\n        LIN -&gt; ACT]`\n    Attributes\n    ----------\n    cross_network: nn.Module\n        The cross network component with mixture of experts\n    deep_network: nn.Module\n        The deep network component (MLP)\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.models.rec import DeepCrossNetworkV2\n    &gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n    &gt;&gt;&gt; colnames = [\"a\", \"b\", \"c\", \"d\", \"e\"]\n    &gt;&gt;&gt; cat_embed_input = [(u, i, j) for u, i, j in zip(colnames[:4], [4] * 4, [8] * 4)]\n    &gt;&gt;&gt; column_idx = {k: v for v, k in enumerate(colnames)}\n    &gt;&gt;&gt; model = DeepCrossNetworkV2(\n    ...     column_idx=column_idx,\n    ...     cat_embed_input=cat_embed_input,\n    ...     continuous_cols=[\"e\"],\n    ...     num_cross_layers=2,\n    ...     low_rank=32,\n    ...     num_experts=4,\n    ...     mlp_hidden_dims=[16, 8]\n    ... )\n    &gt;&gt;&gt; out = model(X_tab)\n    \"\"\"\n\n    def __init__(\n        self,\n        *,\n        column_idx: Dict[str, int],\n        num_cross_layers: int = 2,\n        low_rank: Optional[int] = None,\n        num_experts: int = 2,\n        expert_dropout: float = 0.0,\n        structure: Literal[\"stacked\", \"parallel\"] = \"parallel\",\n        cat_embed_input: Optional[List[Tuple[str, int, int]]] = None,\n        cat_embed_dropout: Optional[float] = None,\n        use_cat_bias: Optional[bool] = None,\n        cat_embed_activation: Optional[str] = None,\n        continuous_cols: Optional[List[str]] = None,\n        cont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\n        embed_continuous: Optional[bool] = None,\n        embed_continuous_method: Optional[\n            Literal[\"standard\", \"piecewise\", \"periodic\"]\n        ] = None,\n        cont_embed_dim: Optional[int] = None,\n        cont_embed_dropout: Optional[float] = None,\n        cont_embed_activation: Optional[str] = None,\n        quantization_setup: Optional[Dict[str, List[float]]] = None,\n        n_frequencies: Optional[int] = None,\n        sigma: Optional[float] = None,\n        share_last_layer: Optional[bool] = None,\n        full_embed_dropout: Optional[bool] = None,\n        mlp_hidden_dims: List[int] = [200, 100],\n        mlp_activation: str = \"relu\",\n        mlp_dropout: Union[float, List[float]] = 0.1,\n        mlp_batchnorm: bool = False,\n        mlp_batchnorm_last: bool = False,\n        mlp_linear_first: bool = True,\n    ):\n        super(DeepCrossNetworkV2, self).__init__(\n            column_idx=column_idx,\n            cat_embed_input=cat_embed_input,\n            cat_embed_dropout=cat_embed_dropout,\n            use_cat_bias=use_cat_bias,\n            cat_embed_activation=cat_embed_activation,\n            continuous_cols=continuous_cols,\n            cont_norm_layer=cont_norm_layer,\n            embed_continuous=embed_continuous,\n            embed_continuous_method=embed_continuous_method,\n            cont_embed_dim=cont_embed_dim,\n            cont_embed_dropout=cont_embed_dropout,\n            cont_embed_activation=cont_embed_activation,\n            quantization_setup=quantization_setup,\n            n_frequencies=n_frequencies,\n            sigma=sigma,\n            share_last_layer=share_last_layer,\n            full_embed_dropout=full_embed_dropout,\n        )\n\n        self.num_cross_layers = num_cross_layers\n        self.num_experts = num_experts\n        self.low_rank = low_rank\n        self.expert_dropout = expert_dropout\n        self.structure = structure\n\n        self.mlp_hidden_dims = mlp_hidden_dims\n        self.mlp_activation = mlp_activation\n        self.mlp_dropout = mlp_dropout\n        self.mlp_batchnorm = mlp_batchnorm\n        self.mlp_batchnorm_last = mlp_batchnorm_last\n        self.mlp_linear_first = mlp_linear_first\n\n        embeddings_output_dim = self.cat_out_dim + self.cont_out_dim\n        self.deep_network = MLP(\n            [embeddings_output_dim] + self.mlp_hidden_dims,\n            self.mlp_activation,\n            self.mlp_dropout,\n            self.mlp_batchnorm,\n            self.mlp_batchnorm_last,\n            self.mlp_linear_first,\n        )\n\n        self.cross_network = CrossNetworkV2(\n            input_dim=embeddings_output_dim,\n            num_layers=self.num_cross_layers,\n            low_rank=self.low_rank,\n            num_experts=self.num_experts,\n            expert_dropout=self.expert_dropout,\n        )\n\n    def forward(self, X: torch.Tensor) -&gt; torch.Tensor:\n        x = self._get_embeddings(X)\n        cross_output = self.cross_network(x)\n\n        if self.structure == \"stacked\":\n            deep_output = self.deep_network(cross_output)\n            return deep_output\n        else:  # parallel\n            deep_output = self.deep_network(x)\n            return torch.cat([cross_output, deep_output], dim=1)\n\n    @property\n    def output_dim(self) -&gt; int:\n        if self.structure == \"stacked\":\n            return self.mlp_hidden_dims[-1]\n        else:  # parallel\n            return self.mlp_hidden_dims[-1] + self.cat_out_dim + self.cont_out_dim\n</code></pre>"},{"location":"pytorch-widedeep/the_rec_module.html#pytorch_widedeep.models.rec.gdcn.GatedDeepCrossNetwork","title":"GatedDeepCrossNetwork","text":"<p>               Bases: <code>BaseTabularModelWithoutAttention</code></p> <p>Defines a <code>GatedDeepCrossNetwork</code> model that can be used as the <code>deeptabular</code> component of a Wide &amp; Deep model or independently by itself.</p> <p>This class implements the Gated Deep &amp; Cross Network (GDCN) architecture as described in the paper Towards Deeper, Lighter and Interpretable Cross Network for CTR Prediction. The GDCN enhances the original DCN by introducing a gating mechanism in the cross network. The gating mechanism controls feature interactions by learning which interactions are more important.</p> <p>The cross layer implements the following equation:</p> \\[c_{i+1} = c_0 \\odot (W^c \\times c_i + b) \\odot \\sigma(W^g \\times c_i) + c_i\\] <p>where:</p> <ul> <li>\\(\\odot\\) represents element-wise multiplication</li> <li>\\(W^c\\) and \\(W^g\\) are the cross and gate weight matrices respectively</li> <li>\\(\\sigma\\) is the sigmoid activation function</li> </ul> <p>Parameters:</p> Name Type Description Default <code>column_idx</code> <code>Dict[str, int]</code> <p>Dict containing the index of the columns that will be passed through the <code>GatedDeepCrossNetwork</code> model. Required to slice the tensors. e.g. {'education': 0, 'relationship': 1, 'workclass': 2, ...}.</p> required <code>num_cross_layers</code> <code>int</code> <p>Number of cross layers in the cross network</p> <code>3</code> <code>structure</code> <code>Literal['stacked', 'parallel']</code> <p>Structure of the model. Either 'parallel' or 'stacked'. If 'parallel', the output will be the concatenation of the cross network and deep network outputs. If 'stacked', the cross network output will be fed into the deep network.</p> <code>'parallel'</code> <code>cat_embed_input</code> <code>Optional[List[Tuple[str, int, int]]]</code> <p>List of Tuples with the column name, number of unique values and embedding dimension. e.g. [(education, 11, 32), ...]</p> <code>None</code> <code>cat_embed_dropout</code> <code>Optional[float]</code> <p>Categorical embeddings dropout. If <code>None</code>, it will default to 0.</p> <code>None</code> <code>use_cat_bias</code> <code>Optional[bool]</code> <p>Boolean indicating if bias will be used for the categorical embeddings. If <code>None</code>, it will default to 'False'.</p> <code>None</code> <code>cat_embed_activation</code> <code>Optional[str]</code> <p>Activation function for the categorical embeddings, if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> <code>None</code> <code>continuous_cols</code> <code>Optional[List[str]]</code> <p>List with the name of the numeric (aka continuous) columns</p> <code>None</code> <code>cont_norm_layer</code> <code>Optional[Literal['batchnorm', 'layernorm']]</code> <p>Type of normalization layer applied to the continuous features. Options are: 'layernorm' and 'batchnorm'. if <code>None</code>, no normalization layer will be used.</p> <code>None</code> <code>embed_continuous_method</code> <code>Optional[Literal['standard', 'piecewise', 'periodic']]</code> <p>Method to use to embed the continuous features. Options are: 'standard', 'periodic' or 'piecewise'. The 'standard' embedding method is based on the FT-Transformer implementation presented in the paper: Revisiting Deep Learning Models for Tabular Data. The 'periodic' and_'piecewise'_ methods were presented in the paper: On Embeddings for Numerical Features in Tabular Deep Learning. Please, read the papers for details.</p> <code>None</code> <code>cont_embed_dropout</code> <code>Optional[float]</code> <p>Dropout for the continuous embeddings. If <code>None</code>, it will default to 0.0</p> <code>None</code> <code>cont_embed_activation</code> <code>Optional[str]</code> <p>Activation function for the continuous embeddings if any. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported. If <code>None</code>, no activation function will be applied.</p> <code>None</code> <code>quantization_setup</code> <code>Optional[Dict[str, List[float]]]</code> <p>This parameter is used when the 'piecewise' method is used to embed the continuous cols. It is a dict where keys are the name of the continuous columns and values are lists with the boundaries for the quantization of the continuous_cols. See the examples for details. If If the 'piecewise' method is used, this parameter is required.</p> <code>None</code> <code>n_frequencies</code> <code>Optional[int]</code> <p>This is the so called 'k' in their paper On Embeddings for Numerical Features in Tabular Deep Learning, and is the number of 'frequencies' that will be used to represent each continuous column. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>sigma</code> <code>Optional[float]</code> <p>This is the sigma parameter in the paper mentioned when describing the previous parameters and it is used to initialise the 'frequency weights'. See their Eq 2 in the paper for details. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>share_last_layer</code> <code>Optional[bool]</code> <p>This parameter is not present in the before mentioned paper but it is implemented in the official repo. If <code>True</code> the linear layer that turns the frequencies into embeddings will be shared across the continuous columns. If <code>False</code> a different linear layer will be used for each continuous column. If the 'periodic' method is used, this parameter is required.</p> <code>None</code> <code>full_embed_dropout</code> <code>Optional[bool]</code> <p>If <code>True</code>, the full embedding corresponding to a column will be masked out/dropout. If <code>None</code>, it will default to <code>False</code>.</p> <code>None</code> <code>mlp_hidden_dims</code> <code>List[int]</code> <p>List with the number of neurons per dense layer in the mlp.</p> <code>[200, 100]</code> <code>mlp_activation</code> <code>str</code> <p>Activation function for the dense layers of the MLP. Currently 'tanh', 'relu', 'leaky_relu' and 'gelu' are supported</p> <code>'relu'</code> <code>mlp_dropout</code> <code>Union[float, List[float]]</code> <p>float or List of floats with the dropout between the dense layers. e.g: [0.5,0.5]</p> <code>0.1</code> <code>mlp_batchnorm</code> <code>bool</code> <p>Boolean indicating whether or not batch normalization will be applied to the dense layers</p> <code>False</code> <code>mlp_batchnorm_last</code> <code>bool</code> <p>Boolean indicating whether or not batch normalization will be applied to the last of the dense layers</p> <code>False</code> <code>mlp_linear_first</code> <code>bool</code> <p>Boolean indicating the order of the operations in the dense layer. If <code>True: [LIN -&gt; ACT -&gt; BN -&gt; DP]</code>. If <code>False: [BN -&gt; DP -&gt; LIN -&gt; ACT]</code></p> <code>True</code> <p>Attributes:</p> Name Type Description <code>cross_network</code> <code>Module</code> <p>The gated cross network component</p> <code>deep_network</code> <code>Module</code> <p>The deep network component (MLP)</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from pytorch_widedeep.models.rec import GatedDeepCrossNetwork\n&gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n&gt;&gt;&gt; colnames = [\"a\", \"b\", \"c\", \"d\", \"e\"]\n&gt;&gt;&gt; cat_embed_input = [(u, i, j) for u, i, j in zip(colnames[:4], [4] * 4, [8] * 4)]\n&gt;&gt;&gt; column_idx = {k: v for v, k in enumerate(colnames)}\n&gt;&gt;&gt; model = GatedDeepCrossNetwork(\n...     column_idx=column_idx,\n...     cat_embed_input=cat_embed_input,\n...     continuous_cols=[\"e\"],\n...     num_cross_layers=2,\n...     mlp_hidden_dims=[16, 8]\n... )\n&gt;&gt;&gt; out = model(X_tab)\n</code></pre> Source code in <code>pytorch_widedeep/models/rec/gdcn.py</code> <pre><code>class GatedDeepCrossNetwork(BaseTabularModelWithoutAttention):\n    r\"\"\"Defines a `GatedDeepCrossNetwork` model that can be used as the `deeptabular`\n    component of a Wide &amp; Deep model or independently by itself.\n\n    This class implements the Gated Deep &amp; Cross Network (GDCN) architecture\n    as described in the paper\n    [Towards Deeper, Lighter and Interpretable Cross Network for CTR Prediction](https://arxiv.org/pdf/2311.04635).\n    The GDCN enhances the original DCN by introducing a gating mechanism in\n    the cross network. The gating mechanism controls feature interactions by\n    learning which interactions are more important.\n\n    The cross layer implements the following equation:\n\n    $$c_{i+1} = c_0 \\odot (W^c \\times c_i + b) \\odot \\sigma(W^g \\times c_i) + c_i$$\n\n    where:\n\n    * $\\odot$ represents element-wise multiplication\n    * $W^c$ and $W^g$ are the cross and gate weight matrices respectively\n    * $\\sigma$ is the sigmoid activation function\n\n    Parameters\n    ----------\n    column_idx: Dict\n        Dict containing the index of the columns that will be passed through\n        the `GatedDeepCrossNetwork` model. Required to slice the tensors. e.g.\n        _{'education': 0, 'relationship': 1, 'workclass': 2, ...}_.\n    num_cross_layers: int, default = 3\n        Number of cross layers in the cross network\n    structure: str, default = \"parallel\"\n        Structure of the model. Either _'parallel'_ or _'stacked'_. If _'parallel'_,\n        the output will be the concatenation of the cross network and deep network\n        outputs. If _'stacked'_, the cross network output will be fed into the deep\n        network.\n    cat_embed_input: List, Optional, default = None\n        List of Tuples with the column name, number of unique values and\n        embedding dimension. e.g. _[(education, 11, 32), ...]_\n    cat_embed_dropout: float, Optional, default = None\n        Categorical embeddings dropout. If `None`, it will default to 0.\n    use_cat_bias: bool, Optional, default = None,\n        Boolean indicating if bias will be used for the categorical embeddings.\n        If `None`, it will default to 'False'.\n    cat_embed_activation: Optional, str, default = None,\n        Activation function for the categorical embeddings, if any. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported\n    continuous_cols : Optional[List[str]], default=None\n        List with the name of the numeric (aka continuous) columns\n    cont_norm_layer : Optional[Literal[\"batchnorm\", \"layernorm\"]], default=None\n        Type of normalization layer applied to the continuous features.\n        Options are: _'layernorm'_ and _'batchnorm'_. if `None`, no\n        normalization layer will be used.\n    embed_continuous_method: Optional[Literal[\"piecewise\", \"periodic\", \"standard\"]], default=\"standard\"\n        Method to use to embed the continuous features. Options are:\n        _'standard'_, _'periodic'_ or _'piecewise'_. The _'standard'_\n        embedding method is based on the FT-Transformer implementation\n        presented in the paper: [Revisiting Deep Learning Models for\n        Tabular Data](https://arxiv.org/abs/2106.11959v5). The _'periodic'_\n        and_'piecewise'_ methods were presented in the paper: [On Embeddings for\n        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556).\n        Please, read the papers for details.\n    cont_embed_dropout : Optional[float], default=None\n        Dropout for the continuous embeddings. If `None`, it will default to 0.0\n    cont_embed_activation : Optional[str], default=None\n        Activation function for the continuous embeddings if any. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported.\n        If `None`, no activation function will be applied.\n    quantization_setup : Optional[Dict[str, List[float]]], default=None\n        This parameter is used when the _'piecewise'_ method is used to embed\n        the continuous cols. It is a dict where keys are the name of the continuous\n        columns and values are lists with the boundaries for the quantization\n        of the continuous_cols. See the examples for details. If\n        If the _'piecewise'_ method is used, this parameter is required.\n    n_frequencies : Optional[int], default=None\n        This is the so called _'k'_ in their paper [On Embeddings for\n        Numerical Features in Tabular Deep Learning](https://arxiv.org/abs/2203.05556),\n        and is the number of 'frequencies' that will be used to represent each\n        continuous column. See their Eq 2 in the paper for details. If\n        the _'periodic'_ method is used, this parameter is required.\n    sigma : Optional[float], default=None\n        This is the sigma parameter in the paper mentioned when describing the\n        previous parameters and it is used to initialise the 'frequency\n        weights'. See their Eq 2 in the paper for details. If\n        the _'periodic'_ method is used, this parameter is required.\n    share_last_layer : Optional[bool], default=None\n        This parameter is not present in the before mentioned paper but it is implemented in\n        the [official repo](https://github.com/yandex-research/rtdl-num-embeddings/tree/main).\n        If `True` the linear layer that turns the frequencies into embeddings\n        will be shared across the continuous columns. If `False` a different\n        linear layer will be used for each continuous column.\n        If the _'periodic'_ method is used, this parameter is required.\n    full_embed_dropout: bool, Optional, default = None,\n        If `True`, the full embedding corresponding to a column will be masked\n        out/dropout. If `None`, it will default to `False`.\n    mlp_hidden_dims: List, default = [200, 100]\n        List with the number of neurons per dense layer in the mlp.\n    mlp_activation: str, default = \"relu\"\n        Activation function for the dense layers of the MLP. Currently\n        _'tanh'_, _'relu'_, _'leaky_relu'_ and _'gelu'_ are supported\n    mlp_dropout: float or List, default = 0.1\n        float or List of floats with the dropout between the dense layers.\n        e.g: _[0.5,0.5]_\n    mlp_batchnorm: bool, default = False\n        Boolean indicating whether or not batch normalization will be applied\n        to the dense layers\n    mlp_batchnorm_last: bool, default = False\n        Boolean indicating whether or not batch normalization will be applied\n        to the last of the dense layers\n    mlp_linear_first: bool, default = False\n        Boolean indicating the order of the operations in the dense\n        layer. If `True: [LIN -&gt; ACT -&gt; BN -&gt; DP]`. If `False: [BN -&gt; DP -&gt;\n        LIN -&gt; ACT]`\n\n    Attributes\n    ----------\n    cross_network: nn.Module\n        The gated cross network component\n    deep_network: nn.Module\n        The deep network component (MLP)\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from pytorch_widedeep.models.rec import GatedDeepCrossNetwork\n    &gt;&gt;&gt; X_tab = torch.cat((torch.empty(5, 4).random_(4), torch.rand(5, 1)), axis=1)\n    &gt;&gt;&gt; colnames = [\"a\", \"b\", \"c\", \"d\", \"e\"]\n    &gt;&gt;&gt; cat_embed_input = [(u, i, j) for u, i, j in zip(colnames[:4], [4] * 4, [8] * 4)]\n    &gt;&gt;&gt; column_idx = {k: v for v, k in enumerate(colnames)}\n    &gt;&gt;&gt; model = GatedDeepCrossNetwork(\n    ...     column_idx=column_idx,\n    ...     cat_embed_input=cat_embed_input,\n    ...     continuous_cols=[\"e\"],\n    ...     num_cross_layers=2,\n    ...     mlp_hidden_dims=[16, 8]\n    ... )\n    &gt;&gt;&gt; out = model(X_tab)\n    \"\"\"\n\n    def __init__(\n        self,\n        *,\n        column_idx: Dict[str, int],\n        num_cross_layers: int = 3,\n        structure: Literal[\"stacked\", \"parallel\"] = \"parallel\",\n        cat_embed_input: Optional[List[Tuple[str, int, int]]] = None,\n        cat_embed_dropout: Optional[float] = None,\n        use_cat_bias: Optional[bool] = None,\n        cat_embed_activation: Optional[str] = None,\n        continuous_cols: Optional[List[str]] = None,\n        cont_norm_layer: Optional[Literal[\"batchnorm\", \"layernorm\"]] = None,\n        embed_continuous: Optional[bool] = None,\n        embed_continuous_method: Optional[\n            Literal[\"standard\", \"piecewise\", \"periodic\"]\n        ] = None,\n        cont_embed_dim: Optional[int] = None,\n        cont_embed_dropout: Optional[float] = None,\n        cont_embed_activation: Optional[str] = None,\n        quantization_setup: Optional[Dict[str, List[float]]] = None,\n        n_frequencies: Optional[int] = None,\n        sigma: Optional[float] = None,\n        share_last_layer: Optional[bool] = None,\n        full_embed_dropout: Optional[bool] = None,\n        mlp_hidden_dims: List[int] = [200, 100],\n        mlp_activation: str = \"relu\",\n        mlp_dropout: Union[float, List[float]] = 0.1,\n        mlp_batchnorm: bool = False,\n        mlp_batchnorm_last: bool = False,\n        mlp_linear_first: bool = True,\n    ):\n        super(GatedDeepCrossNetwork, self).__init__(\n            column_idx=column_idx,\n            cat_embed_input=cat_embed_input,\n            cat_embed_dropout=cat_embed_dropout,\n            use_cat_bias=use_cat_bias,\n            cat_embed_activation=cat_embed_activation,\n            continuous_cols=continuous_cols,\n            cont_norm_layer=cont_norm_layer,\n            embed_continuous=embed_continuous,\n            embed_continuous_method=embed_continuous_method,\n            cont_embed_dim=cont_embed_dim,\n            cont_embed_dropout=cont_embed_dropout,\n            cont_embed_activation=cont_embed_activation,\n            quantization_setup=quantization_setup,\n            n_frequencies=n_frequencies,\n            sigma=sigma,\n            share_last_layer=share_last_layer,\n            full_embed_dropout=full_embed_dropout,\n        )\n\n        self.num_cross_layers = num_cross_layers\n        self.structure = structure\n\n        self.mlp_hidden_dims = mlp_hidden_dims\n        self.mlp_activation = mlp_activation\n        self.mlp_dropout = mlp_dropout\n        self.mlp_batchnorm = mlp_batchnorm\n        self.mlp_batchnorm_last = mlp_batchnorm_last\n        self.mlp_linear_first = mlp_linear_first\n\n        embeddings_output_dim = self.cat_out_dim + self.cont_out_dim\n        self.deep_network = MLP(\n            [embeddings_output_dim] + self.mlp_hidden_dims,\n            self.mlp_activation,\n            self.mlp_dropout,\n            self.mlp_batchnorm,\n            self.mlp_batchnorm_last,\n            self.mlp_linear_first,\n        )\n\n        self.cross_network = GatedCrossNetwork(embeddings_output_dim, num_cross_layers)\n\n    def forward(self, X: torch.Tensor) -&gt; torch.Tensor:\n        x = self._get_embeddings(X)\n        cross_output = self.cross_network(x)\n\n        if self.structure == \"stacked\":\n            deep_output = self.deep_network(cross_output)\n            return deep_output\n        else:  # parallel\n            deep_output = self.deep_network(x)\n            return torch.cat([cross_output, deep_output], dim=1)\n\n    @property\n    def output_dim(self) -&gt; int:\n        if self.structure == \"stacked\":\n            return self.mlp_hidden_dims[-1]\n        else:  # parallel\n            return self.mlp_hidden_dims[-1] + self.cat_out_dim + self.cont_out_dim\n</code></pre>"},{"location":"pytorch-widedeep/trainer.html","title":"Training multimodal Deep Learning Models","text":"<p>Here is the documentation for the <code>Trainer</code> class, that will do all the heavy lifting.</p> <p>Trainer is also available from <code>pytorch-widedeep</code> directly, for example, one could do:</p> <pre><code>    from pytorch-widedeep.training import Trainer\n</code></pre> <p>or also:</p> <pre><code>    from pytorch-widedeep import Trainer\n</code></pre>"},{"location":"pytorch-widedeep/trainer.html#pytorch_widedeep.training.Trainer","title":"Trainer","text":"<p>               Bases: <code>BaseTrainer</code></p> <p>Class to set the of attributes that will be used during the training process.</p> <p>Parameters:</p> Name Type Description Default <code>model</code> <code>WideDeep</code> <p>An object of class <code>WideDeep</code></p> required <code>objective</code> <code>str</code> <p>Defines the objective, loss or cost function. </p> <p>Param aliases: <code>loss_function</code>, <code>loss_fn</code>, <code>loss</code>, <code>cost_function</code>, <code>cost_fn</code>, <code>cost</code>. </p> <p>Possible values are:</p> <ul> <li> <p><code>binary</code>, aliases: <code>logistic</code>, <code>binary_logloss</code>, <code>binary_cross_entropy</code></p> </li> <li> <p><code>binary_focal_loss</code></p> </li> <li> <p><code>multiclass</code>, aliases: <code>multi_logloss</code>, <code>cross_entropy</code>, <code>categorical_cross_entropy</code>,</p> </li> <li> <p><code>multiclass_focal_loss</code></p> </li> <li> <p><code>regression</code>, aliases: <code>mse</code>, <code>l2</code>, <code>mean_squared_error</code></p> </li> <li> <p><code>mean_absolute_error</code>, aliases: <code>mae</code>, <code>l1</code></p> </li> <li> <p><code>mean_squared_log_error</code>, aliases: <code>msle</code></p> </li> <li> <p><code>root_mean_squared_error</code>, aliases:  <code>rmse</code></p> </li> <li> <p><code>root_mean_squared_log_error</code>, aliases: <code>rmsle</code></p> </li> <li> <p><code>zero_inflated_lognormal</code>, aliases: <code>ziln</code></p> </li> <li> <p><code>quantile</code></p> </li> <li> <p><code>tweedie</code></p> </li> <li> <p><code>multitarget</code>, aliases: <code>multi_target</code></p> </li> </ul> <p>NOTE: For <code>multitarget</code> a custom loss function must be passed</p> required <code>custom_loss_function</code> <code>Optional[Module]</code> <p>It is possible to pass a custom loss function. See for example <code>pytorch_widedeep.losses.FocalLoss</code> for the required structure of the object or the Examples section in this documentation or in the repo. Note that if <code>custom_loss_function</code> is not <code>None</code>, <code>objective</code> must be 'binary', 'multiclass' or 'regression', consistent with the loss function</p> <code>None</code> <code>optimizers</code> <code>Optional[Union[Optimizer, Dict[str, Union[Optimizer, List[Optimizer]]]]]</code> <ul> <li>An instance of Pytorch's <code>Optimizer</code> object   (e.g. <code>torch.optim.Adam()</code>) or</li> <li>a dictionary where there keys are the model components (i.e.   'wide', 'deeptabular', 'deeptext', 'deepimage'   and/or 'deephead')  and the values are the corresponding   optimizers or list of optimizers if multiple models are used for   the given data mode (e.g. two text columns/models for the deeptext   component). If multiple optimizers are used the   dictionary MUST contain an optimizer per model component.</li> </ul> <p>if no optimizers are passed it will default to <code>Adam</code> for all model components</p> <code>None</code> <code>lr_schedulers</code> <code>Optional[Union[LRScheduler, Dict[str, Union[LRScheduler, List[LRScheduler]]]]]</code> <ul> <li>An instance of Pytorch's <code>LRScheduler</code> object (e.g   <code>torch.optim.lr_scheduler.StepLR(opt, step_size=5)</code>) or</li> <li>a dictionary where there keys are the model componenst (i.e. 'wide',   'deeptabular', 'deeptext', 'deepimage' and/or 'deephead') and the   values are the corresponding learning rate schedulers or list of     learning rate schedulers if multiple models are used for the given     data mode (e.g. two text columns/models for the deeptext component).</li> </ul> <code>None</code> <code>initializers</code> <code>Optional[Union[Initializer, Dict[str, Union[Initializer, List[Initializer]]]]]</code> <ul> <li>An instance of an <code>Initializer</code> object see <code>pytorch-widedeep.initializers</code> or</li> <li>a dictionary where there keys are the model components (i.e. 'wide',   'deeptabular', 'deeptext', 'deepimage' and/or 'deephead')   and the values are the corresponding initializers or list of     initializers if multiple models are used for the given data mode (e.g.     two text columns/models for the deeptext component).</li> </ul> <code>None</code> <code>transforms</code> <code>Optional[List[Transforms]]</code> <p>List with <code>torchvision.transforms</code> to be applied to the image component of the model (i.e. <code>deepimage</code>) See torchvision transforms.</p> <code>None</code> <code>callbacks</code> <code>Optional[List[Callback]]</code> <p>List with <code>Callback</code> objects. The three callbacks available in <code>pytorch-widedeep</code> are: <code>LRHistory</code>, <code>ModelCheckpoint</code> and <code>EarlyStopping</code>. The <code>History</code> and the <code>LRShedulerCallback</code> callbacks are used by default. This can also be a custom callback as long as the object of type <code>Callback</code>. See <code>pytorch_widedeep.callbacks.Callback</code> or the examples folder in the repo.</p> <code>None</code> <code>metrics</code> <code>Optional[Union[List[Metric], List[Metric]]]</code> <ul> <li>List of objects of type <code>Metric</code>. Metrics available are:   <code>Accuracy</code>, <code>Precision</code>, <code>Recall</code>, <code>FBetaScore</code>,   <code>F1Score</code> and <code>R2Score</code>. This can also be a custom metric as long   as it is an object of type <code>Metric</code>. See   <code>pytorch_widedeep.metrics.Metric</code> or the examples folder in the   repo</li> <li>List of objects of type <code>torchmetrics.Metric</code>. This can be any   metric from torchmetrics library   Examples.   This can also be a custom metric as long as   it is an object of type <code>Metric</code>. See   the instructions.</li> </ul> <code>None</code> <code>verbose</code> <code>int</code> <p>Verbosity level. If set to 0 nothing will be printed during training</p> <code>1</code> <code>seed</code> <code>int</code> <p>Random seed to be used internally for train/test split</p> <code>1</code> <p>Other Parameters:</p> Name Type Description <code>**kwargs</code> <p>Other infrequently used arguments that can also be passed as kwargs are:</p> <ul> <li> <p>device: <code>str</code>     string indicating the device. One of 'cpu', 'gpu' or 'mps' if     run on a Mac with Apple silicon or AMD GPU(s)</p> </li> <li> <p>num_workers: <code>int</code>     number of workers to be used internally by the data loaders</p> </li> <li> <p>lambda_sparse: <code>float</code>     lambda sparse parameter in case the <code>deeptabular</code> component is <code>TabNet</code></p> </li> <li> <p>class_weight: <code>List[float]</code>     This is the <code>weight</code> or <code>pos_weight</code> parameter in     <code>CrossEntropyLoss</code> and <code>BCEWithLogitsLoss</code>, depending on whether</p> </li> <li>reducelronplateau_criterion: <code>str</code>     This sets the criterion that will be used by the lr scheduler to     take a step: One of 'loss' or 'metric'. The ReduceLROnPlateau     learning rate is a bit particular.</li> </ul> <p>Attributes:</p> Name Type Description <code>cyclic_lr</code> <code>bool</code> <p>Attribute that indicates if any of the lr_schedulers is cyclic_lr (i.e. <code>CyclicLR</code> or <code>OneCycleLR</code>). See Pytorch schedulers.</p> <code>feature_importance</code> <code>dict</code> <p>dict where the keys are the column names and the values are the corresponding feature importances. This attribute will only exist if the <code>deeptabular</code> component is a Tabnet model.</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import torch\n&gt;&gt;&gt; from torchvision.transforms import ToTensor\n&gt;&gt;&gt;\n&gt;&gt;&gt; # wide deep imports\n&gt;&gt;&gt; from pytorch_widedeep.callbacks import EarlyStopping, LRHistory\n&gt;&gt;&gt; from pytorch_widedeep.initializers import KaimingNormal, KaimingUniform, Normal, Uniform\n&gt;&gt;&gt; from pytorch_widedeep.models import TabResnet, Vision, BasicRNN, Wide, WideDeep\n&gt;&gt;&gt; from pytorch_widedeep import Trainer\n&gt;&gt;&gt;\n&gt;&gt;&gt; embed_input = [(u, i, j) for u, i, j in zip([\"a\", \"b\", \"c\"][:4], [4] * 3, [8] * 3)]\n&gt;&gt;&gt; column_idx = {k: v for v, k in enumerate([\"a\", \"b\", \"c\"])}\n&gt;&gt;&gt; wide = Wide(10, 1)\n&gt;&gt;&gt;\n&gt;&gt;&gt; # build the model\n&gt;&gt;&gt; deeptabular = TabResnet(blocks_dims=[8, 4], column_idx=column_idx, cat_embed_input=embed_input)\n&gt;&gt;&gt; deeptext = BasicRNN(vocab_size=10, embed_dim=4, padding_idx=0)\n&gt;&gt;&gt; deepimage = Vision()\n&gt;&gt;&gt; model = WideDeep(wide=wide, deeptabular=deeptabular, deeptext=deeptext, deepimage=deepimage)\n&gt;&gt;&gt;\n&gt;&gt;&gt; # set optimizers and schedulers\n&gt;&gt;&gt; wide_opt = torch.optim.Adam(model.wide.parameters())\n&gt;&gt;&gt; deep_opt = torch.optim.AdamW(model.deeptabular.parameters())\n&gt;&gt;&gt; text_opt = torch.optim.Adam(model.deeptext.parameters())\n&gt;&gt;&gt; img_opt = torch.optim.AdamW(model.deepimage.parameters())\n&gt;&gt;&gt;\n&gt;&gt;&gt; wide_sch = torch.optim.lr_scheduler.StepLR(wide_opt, step_size=5)\n&gt;&gt;&gt; deep_sch = torch.optim.lr_scheduler.StepLR(deep_opt, step_size=3)\n&gt;&gt;&gt; text_sch = torch.optim.lr_scheduler.StepLR(text_opt, step_size=5)\n&gt;&gt;&gt; img_sch = torch.optim.lr_scheduler.StepLR(img_opt, step_size=3)\n&gt;&gt;&gt;\n&gt;&gt;&gt; optimizers = {\"wide\": wide_opt, \"deeptabular\": deep_opt, \"deeptext\": text_opt, \"deepimage\": img_opt}\n&gt;&gt;&gt; schedulers = {\"wide\": wide_sch, \"deeptabular\": deep_sch, \"deeptext\": text_sch, \"deepimage\": img_sch}\n&gt;&gt;&gt;\n&gt;&gt;&gt; # set initializers and callbacks\n&gt;&gt;&gt; initializers = {\"wide\": Uniform, \"deeptabular\": Normal, \"deeptext\": KaimingNormal, \"deepimage\": KaimingUniform}\n&gt;&gt;&gt; transforms = [ToTensor]\n&gt;&gt;&gt; callbacks = [LRHistory(n_epochs=4), EarlyStopping]\n&gt;&gt;&gt;\n&gt;&gt;&gt; # set the trainer\n&gt;&gt;&gt; trainer = Trainer(model, objective=\"regression\", initializers=initializers, optimizers=optimizers,\n... lr_schedulers=schedulers, callbacks=callbacks, transforms=transforms)\n</code></pre> Source code in <code>pytorch_widedeep/training/trainer.py</code> <pre><code>class Trainer(BaseTrainer):\n    r\"\"\"Class to set the of attributes that will be used during the\n    training process.\n\n    Parameters\n    ----------\n    model: `WideDeep`\n        An object of class `WideDeep`\n    objective: str\n        Defines the objective, loss or cost function. &lt;br/&gt;\n\n        Param aliases: `loss_function`, `loss_fn`, `loss`,\n        `cost_function`, `cost_fn`, `cost`. &lt;br/&gt;\n\n        Possible values are:\n\n        - `binary`, aliases: `logistic`, `binary_logloss`, `binary_cross_entropy`\n\n        - `binary_focal_loss`\n\n        - `multiclass`, aliases: `multi_logloss`, `cross_entropy`, `categorical_cross_entropy`,\n\n        - `multiclass_focal_loss`\n\n        - `regression`, aliases: `mse`, `l2`, `mean_squared_error`\n\n        - `mean_absolute_error`, aliases: `mae`, `l1`\n\n        - `mean_squared_log_error`, aliases: `msle`\n\n        - `root_mean_squared_error`, aliases:  `rmse`\n\n        - `root_mean_squared_log_error`, aliases: `rmsle`\n\n        - `zero_inflated_lognormal`, aliases: `ziln`\n\n        - `quantile`\n\n        - `tweedie`\n\n        - `multitarget`, aliases: `multi_target`\n\n        **NOTE**: For `multitarget` a custom loss function must be passed\n    custom_loss_function: `nn.Module`. Optional, default = None\n        It is possible to pass a custom loss function. See for example\n        `pytorch_widedeep.losses.FocalLoss` for the required structure of the\n        object or the Examples section in this documentation or in the repo.\n        Note that if `custom_loss_function` is not `None`, `objective` must\n        be _'binary'_, _'multiclass'_ or _'regression'_, consistent with the\n        loss function\n    optimizers: `Optimizer` or dict. Optional, default=None\n        - An instance of Pytorch's `Optimizer` object\n          (e.g. `torch.optim.Adam()`) or\n        - a dictionary where there keys are the model components (i.e.\n          _'wide'_, _'deeptabular'_, _'deeptext'_, _'deepimage'_\n          and/or _'deephead'_)  and the values are the corresponding\n          optimizers or list of optimizers if multiple models are used for\n          the given data mode (e.g. two text columns/models for the deeptext\n          component). If multiple optimizers are used the\n          dictionary **MUST** contain an optimizer per model component.\n\n        if no optimizers are passed it will default to `Adam` for all\n        model components\n    lr_schedulers: `LRScheduler` or dict. Optional, default=None\n        - An instance of Pytorch's `LRScheduler` object (e.g\n          `torch.optim.lr_scheduler.StepLR(opt, step_size=5)`) or\n        - a dictionary where there keys are the model componenst (i.e. _'wide'_,\n          _'deeptabular'_, _'deeptext'_, _'deepimage'_ and/or _'deephead'_) and the\n          values are the corresponding learning rate schedulers or list of\n            learning rate schedulers if multiple models are used for the given\n            data mode (e.g. two text columns/models for the deeptext component).\n    initializers: `Initializer` or dict. Optional, default=None\n        - An instance of an `Initializer` object see `pytorch-widedeep.initializers` or\n        - a dictionary where there keys are the model components (i.e. _'wide'_,\n          _'deeptabular'_, _'deeptext'_, _'deepimage'_ and/or _'deephead'_)\n          and the values are the corresponding initializers or list of\n            initializers if multiple models are used for the given data mode (e.g.\n            two text columns/models for the deeptext component).\n    transforms: List. Optional, default=None\n        List with `torchvision.transforms` to be applied to the image\n        component of the model (i.e. `deepimage`) See\n        [torchvision transforms](https://pytorch.org/docs/stable/torchvision/transforms.html).\n    callbacks: List. Optional, default=None\n        List with `Callback` objects. The three callbacks available in\n        `pytorch-widedeep` are: `LRHistory`, `ModelCheckpoint` and\n        `EarlyStopping`. The `History` and the `LRShedulerCallback` callbacks\n        are used by default. This can also be a custom callback as long as\n        the object of type `Callback`. See\n        `pytorch_widedeep.callbacks.Callback` or the examples folder in the\n        repo.\n    metrics: List. Optional, default=None\n        - List of objects of type `Metric`. Metrics available are:\n          `Accuracy`, `Precision`, `Recall`, `FBetaScore`,\n          `F1Score` and `R2Score`. This can also be a custom metric as long\n          as it is an object of type `Metric`. See\n          `pytorch_widedeep.metrics.Metric` or the examples folder in the\n          repo\n        - List of objects of type `torchmetrics.Metric`. This can be any\n          metric from torchmetrics library\n          [Examples](https://torchmetrics.readthedocs.io/en/latest/).\n          This can also be a custom metric as long as\n          it is an object of type `Metric`. See\n          [the instructions](https://torchmetrics.readthedocs.io/en/latest/).\n    verbose: int, default=1\n        Verbosity level. If set to 0 nothing will be printed during training\n    seed: int, default=1\n        Random seed to be used internally for train/test split\n\n    Other Parameters\n    ----------------\n    **kwargs: dict\n        Other infrequently used arguments that can also be passed as kwargs are:\n\n        - **device**: `str`&lt;br/&gt;\n            string indicating the device. One of _'cpu'_, _'gpu'_ or 'mps' if\n            run on a Mac with Apple silicon or AMD GPU(s)\n\n        - **num_workers**: `int`&lt;br/&gt;\n            number of workers to be used internally by the data loaders\n\n        - **lambda_sparse**: `float`&lt;br/&gt;\n            lambda sparse parameter in case the `deeptabular` component is `TabNet`\n\n        - **class_weight**: `List[float]`&lt;br/&gt;\n            This is the `weight` or `pos_weight` parameter in\n            `CrossEntropyLoss` and `BCEWithLogitsLoss`, depending on whether\n        - **reducelronplateau_criterion**: `str`\n            This sets the criterion that will be used by the lr scheduler to\n            take a step: One of _'loss'_ or _'metric'_. The ReduceLROnPlateau\n            learning rate is a bit particular.\n\n    Attributes\n    ----------\n    cyclic_lr: bool\n        Attribute that indicates if any of the lr_schedulers is cyclic_lr\n        (i.e. `CyclicLR` or\n        `OneCycleLR`). See [Pytorch schedulers](https://pytorch.org/docs/stable/optim.html).\n    feature_importance: dict\n        dict where the keys are the column names and the values are the\n        corresponding feature importances. This attribute will only exist\n        if the `deeptabular` component is a Tabnet model.\n\n    Examples\n    --------\n    &gt;&gt;&gt; import torch\n    &gt;&gt;&gt; from torchvision.transforms import ToTensor\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; # wide deep imports\n    &gt;&gt;&gt; from pytorch_widedeep.callbacks import EarlyStopping, LRHistory\n    &gt;&gt;&gt; from pytorch_widedeep.initializers import KaimingNormal, KaimingUniform, Normal, Uniform\n    &gt;&gt;&gt; from pytorch_widedeep.models import TabResnet, Vision, BasicRNN, Wide, WideDeep\n    &gt;&gt;&gt; from pytorch_widedeep import Trainer\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; embed_input = [(u, i, j) for u, i, j in zip([\"a\", \"b\", \"c\"][:4], [4] * 3, [8] * 3)]\n    &gt;&gt;&gt; column_idx = {k: v for v, k in enumerate([\"a\", \"b\", \"c\"])}\n    &gt;&gt;&gt; wide = Wide(10, 1)\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; # build the model\n    &gt;&gt;&gt; deeptabular = TabResnet(blocks_dims=[8, 4], column_idx=column_idx, cat_embed_input=embed_input)\n    &gt;&gt;&gt; deeptext = BasicRNN(vocab_size=10, embed_dim=4, padding_idx=0)\n    &gt;&gt;&gt; deepimage = Vision()\n    &gt;&gt;&gt; model = WideDeep(wide=wide, deeptabular=deeptabular, deeptext=deeptext, deepimage=deepimage)\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; # set optimizers and schedulers\n    &gt;&gt;&gt; wide_opt = torch.optim.Adam(model.wide.parameters())\n    &gt;&gt;&gt; deep_opt = torch.optim.AdamW(model.deeptabular.parameters())\n    &gt;&gt;&gt; text_opt = torch.optim.Adam(model.deeptext.parameters())\n    &gt;&gt;&gt; img_opt = torch.optim.AdamW(model.deepimage.parameters())\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; wide_sch = torch.optim.lr_scheduler.StepLR(wide_opt, step_size=5)\n    &gt;&gt;&gt; deep_sch = torch.optim.lr_scheduler.StepLR(deep_opt, step_size=3)\n    &gt;&gt;&gt; text_sch = torch.optim.lr_scheduler.StepLR(text_opt, step_size=5)\n    &gt;&gt;&gt; img_sch = torch.optim.lr_scheduler.StepLR(img_opt, step_size=3)\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; optimizers = {\"wide\": wide_opt, \"deeptabular\": deep_opt, \"deeptext\": text_opt, \"deepimage\": img_opt}\n    &gt;&gt;&gt; schedulers = {\"wide\": wide_sch, \"deeptabular\": deep_sch, \"deeptext\": text_sch, \"deepimage\": img_sch}\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; # set initializers and callbacks\n    &gt;&gt;&gt; initializers = {\"wide\": Uniform, \"deeptabular\": Normal, \"deeptext\": KaimingNormal, \"deepimage\": KaimingUniform}\n    &gt;&gt;&gt; transforms = [ToTensor]\n    &gt;&gt;&gt; callbacks = [LRHistory(n_epochs=4), EarlyStopping]\n    &gt;&gt;&gt;\n    &gt;&gt;&gt; # set the trainer\n    &gt;&gt;&gt; trainer = Trainer(model, objective=\"regression\", initializers=initializers, optimizers=optimizers,\n    ... lr_schedulers=schedulers, callbacks=callbacks, transforms=transforms)\n    \"\"\"\n\n    @alias(  # noqa: C901\n        \"objective\",\n        [\"loss_function\", \"loss_fn\", \"loss\", \"cost_function\", \"cost_fn\", \"cost\"],\n    )\n    @alias(\"metrics\", [\"train_metrics\"])\n    def __init__(\n        self,\n        model: WideDeep,\n        objective: str,\n        custom_loss_function: Optional[nn.Module] = None,\n        optimizers: Optional[\n            Union[Optimizer, Dict[str, Union[Optimizer, List[Optimizer]]]]\n        ] = None,\n        lr_schedulers: Optional[\n            Union[LRScheduler, Dict[str, Union[LRScheduler, List[LRScheduler]]]]\n        ] = None,\n        initializers: Optional[\n            Union[Initializer, Dict[str, Union[Initializer, List[Initializer]]]]\n        ] = None,\n        transforms: Optional[List[Transforms]] = None,\n        callbacks: Optional[List[Callback]] = None,\n        metrics: Optional[Union[List[Metric], List[TorchMetric]]] = None,\n        eval_metrics: Optional[Union[List[Metric], List[TorchMetric]]] = None,\n        verbose: int = 1,\n        seed: int = 1,\n        **kwargs,\n    ):\n        super().__init__(\n            model=model,\n            objective=objective,\n            custom_loss_function=custom_loss_function,\n            optimizers=optimizers,\n            lr_schedulers=lr_schedulers,\n            initializers=initializers,\n            transforms=transforms,\n            callbacks=callbacks,\n            metrics=metrics,\n            eval_metrics=eval_metrics,\n            verbose=verbose,\n            seed=seed,\n            **kwargs,\n        )\n\n    @alias(\"finetune\", [\"warmup\"])\n    @alias(\"train_dataloader\", [\"custom_dataloader\"])\n    @alias(\"eval_dataloader\", [\"custom_eval_dataloader\"])\n    def fit(  # noqa: C901\n        self,\n        X_wide: Optional[np.ndarray] = None,\n        X_tab: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n        X_text: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n        X_img: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n        X_train: Optional[Dict[str, Union[np.ndarray, List[np.ndarray]]]] = None,\n        X_val: Optional[Dict[str, Union[np.ndarray, List[np.ndarray]]]] = None,\n        val_split: Optional[float] = None,\n        target: Optional[np.ndarray] = None,\n        n_epochs: int = 1,\n        validation_freq: int = 1,\n        batch_size: int = 32,\n        train_dataloader: Optional[CustomDataLoader] = None,\n        eval_dataloader: Optional[CustomDataLoader] = None,\n        feature_importance_sample_size: Optional[int] = None,\n        finetune: bool = False,\n        **kwargs,\n    ):\n        r\"\"\"Fit method.\n\n        The input datasets can be passed either directly via numpy arrays\n        (`X_wide`, `X_tab`, `X_text` or `X_img`) or alternatively, in\n        dictionaries (`X_train` or `X_val`).\n\n        Parameters\n        ----------\n        X_wide: np.ndarray, Optional. default=None\n            Input for the `wide` model component.\n            See `pytorch_widedeep.preprocessing.WidePreprocessor`\n        X_tab: np.ndarray, Optional. default=None\n            Input for the `deeptabular` model component.\n            See `pytorch_widedeep.preprocessing.TabPreprocessor`. If multiple\n            tabular models are used for different columns, this should be a\n            list of numpy arrays\n        X_text: Union[np.ndarray, List[np.ndarray]], Optional. default=None\n            Input for the `deeptext` model component.\n            See `pytorch_widedeep.preprocessing.TextPreprocessor`.\n            If multiple text columns/models are used, this should be a list of\n            numpy arrays\n        X_img: np.ndarray, Optional. default=None\n            Input for the `deepimage` model component.\n            See `pytorch_widedeep.preprocessing.ImagePreprocessor`.\n            If multiple image columns/models are used, this should be a list of\n            numpy arrays\n        X_train: Dict, Optional. default=None\n            The training dataset can also be passed in a dictionary. Keys are\n            _'X_wide'_, _'X_tab'_, _'X_text'_, _'X_img'_ and _'target'_. Values\n            are the corresponding matrices. Note that of multiple text or image\n            columns/models are used, the corresponding values should be lists\n            of numpy arrays\n        X_val: Dict, Optional. default=None\n            The validation dataset can also be passed in a dictionary. Keys\n            are _'X_wide'_, _'X_tab'_, _'X_text'_, _'X_img'_ and _'target'_.\n            Values are the corresponding matrices. Note that of multiple text\n            or image columns/models are used, the corresponding values should\n            be lists of numpy arrays\n        val_split: float, Optional. default=None\n            train/val split fraction\n        target: np.ndarray, Optional. default=None\n            target values\n        n_epochs: int, default=1\n            number of epochs\n        validation_freq: int, default=1\n            epochs validation frequency\n        batch_size: int, default=32\n            batch size\n        custom_dataloader: `DataLoader`, Optional, default=None\n            object of class `torch.utils.data.DataLoader`. Available\n            predefined dataloaders are in `pytorch-widedeep.dataloaders`.If\n            `None`, a standard torch `DataLoader` is used.\n        finetune: bool, default=False\n            fine-tune individual model components. This functionality can also\n            be used to 'warm-up' (and hence the alias `warmup`) individual\n            components before the joined training starts, and hence its\n            alias. See the Examples folder in the repo for more details\n\n            `pytorch_widedeep` implements 3 fine-tune routines.\n\n            - fine-tune all trainable layers at once. This routine is\n              inspired by the work of Howard &amp; Sebastian Ruder 2018 in their\n              [ULMfit paper](https://arxiv.org/abs/1801.06146). Using a\n              Slanted Triangular learing (see\n              [Leslie N. Smith paper](https://arxiv.org/pdf/1506.01186.pdf) ) ,\n              the process is the following: *i*) the learning rate will\n              gradually increase for 10% of the training steps from max_lr/10\n              to max_lr. *ii*) It will then gradually decrease to max_lr/10\n              for the remaining 90% of the steps. The optimizer used in the\n              process is `Adam`.\n\n            and two gradual fine-tune routines, where only certain layers are\n            trained at a time.\n\n            - The so called `Felbo` gradual fine-tune rourine, based on the the\n              Felbo et al., 2017 [DeepEmoji paper](https://arxiv.org/abs/1708.00524).\n            - The `Howard` routine based on the work of Howard &amp; Sebastian Ruder 2018 in their\n              [ULMfit paper](https://arxiv.org/abs/1801.06146&gt;).\n\n            For details on how these routines work, please see the Examples\n            section in this documentation and the Examples folder in the repo. &lt;br/&gt;\n            Param Alias: `warmup`\n\n        Other Parameters\n        ----------------\n        **kwargs:\n            Other keyword arguments are:\n\n            - **DataLoader related parameters**:&lt;br/&gt;\n                For example,  `sampler`, `batch_sampler`, `collate_fn`, etc.\n                Please, see the pytorch\n                [DataLoader docs](https://pytorch.org/docs/stable/data.html#torch.utils.data.DataLoader)\n                for details.\n\n            - **Finetune related parameters**:&lt;br/&gt;\n                see the source code at `pytorch_widedeep._finetune`. Namely, these are:\n\n                - `finetune_epochs` (`int`):\n                    number of epochs use for fine tuning\n                - `finetune_max_lr` (`float`):\n                   max lr during fine tuning\n                - `routine` (`str`):\n                   one of _'howard'_ or _'felbo'_\n                - `deeptabular_gradual` (`bool`):\n                   boolean indicating if the `deeptabular` component will be fine tuned gradually\n                - `deeptabular_layers` (`Optional[Union[List[nn.Module], List[List[nn.Module]]]]`):\n                   List of pytorch modules indicating the layers of the\n                   `deeptabular` that will be fine tuned\n                - `deeptabular_max_lr` (`Union[float, List[float]]`):\n                   max lr for the `deeptabular` componet during fine tuning\n                - `deeptext_gradual` (`bool`):\n                   same as `deeptabular_gradual` but for the `deeptext` component\n                - `deeptext_layers` (`Optional[Union[List[nn.Module], List[List[nn.Module]]]]`):\n                   same as `deeptabular_gradual` but for the `deeptext` component.\n                   If there are multiple text columns/models, this should be a list of lists\n                - `deeptext_max_lr` (`Union[float, List[float]]`):\n                   same as `deeptabular_gradual` but for the `deeptext` component\n                   If there are multiple text columns/models, this should be a list of floats\n                - `deepimage_gradual` (`bool`):\n                   same as `deeptext_layers` but for the `deepimage` component\n                - `deepimage_layers` (`Optional[Union[List[nn.Module], List[List[nn.Module]]]]`):\n                   same as `deeptext_layers` but for the `deepimage` component\n                - `deepimage_max_lr` (`Union[float, List[float]]`):\n                    same as `deeptext_layers` but for the `deepimage` component\n\n        Examples\n        --------\n\n        For a series of comprehensive examples on how to use the `fit` method, please see the\n        [Examples](https://github.com/jrzaurin/pytorch-widedeep/tree/master/examples)\n        folder in the repo\n        \"\"\"\n        dataloader_args, finetune_args = self._extract_kwargs(kwargs)\n        self.batch_size = batch_size\n        train_set, eval_set = wd_train_val_split(\n            self.seed,\n            self.method,  # type: ignore\n            X_wide,\n            X_tab,\n            X_text,\n            X_img,\n            X_train,\n            X_val,\n            val_split,\n            target,\n            self.transforms,\n        )\n        train_loader = self._set_dataloader(\n            train_dataloader, train_set, batch_size, dataloader_args\n        )\n        eval_loader = (\n            self._set_dataloader(eval_dataloader, eval_set, batch_size, dataloader_args)\n            if eval_set is not None\n            else None\n        )\n\n        if finetune:\n            self.with_finetuning: bool = True\n            self._do_finetune(train_loader, **finetune_args)\n            if self.verbose:\n                print(\n                    \"Fine-tuning (or warmup) of individual components completed. \"\n                    \"Training the whole model for {} epochs\".format(n_epochs)\n                )\n        else:\n            self.with_finetuning = False\n\n        self.callback_container.on_train_begin(\n            {\n                \"batch_size\": batch_size,\n                \"train_steps\": len(train_loader),\n                \"n_epochs\": n_epochs,\n            }\n        )\n        for epoch in range(n_epochs):\n            epoch_logs = self._train_epoch(train_loader, epoch)\n            if eval_loader is not None and epoch % validation_freq == (\n                validation_freq - 1\n            ):\n                epoch_logs, on_epoch_end_metric = self._eval_epoch(\n                    eval_loader, epoch_logs\n                )\n            else:\n                on_epoch_end_metric = None\n                if self.reducelronplateau:\n                    raise NotImplementedError(\n                        \"ReduceLROnPlateau scheduler can be used only with validation data.\"\n                    )\n            self.callback_container.on_epoch_end(epoch, epoch_logs, on_epoch_end_metric)\n            if self.early_stop:\n                # self.callback_container.on_train_end(epoch_logs)\n                break\n\n        self.callback_container.on_train_end(epoch_logs)\n        if feature_importance_sample_size is not None:\n            self.feature_importance = FeatureImportance(\n                self.device, feature_importance_sample_size\n            ).feature_importance(train_loader, self.model)\n        self._restore_best_weights()\n        self.model.train()\n\n    def predict(  # type: ignore[override, return]\n        self,\n        X_wide: Optional[np.ndarray] = None,\n        X_tab: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n        X_text: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n        X_img: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n        X_test: Optional[Dict[str, Union[np.ndarray, List[np.ndarray]]]] = None,\n        batch_size: Optional[int] = None,\n    ) -&gt; np.ndarray:\n        r\"\"\"Returns the predictions\n\n        The input datasets can be passed either directly via numpy arrays\n        (`X_wide`, `X_tab`, `X_text` or `X_img`) or alternatively, in\n        a dictionary (`X_test`)\n\n\n        Parameters\n        ----------\n        X_wide: np.ndarray, Optional. default=None\n            Input for the `wide` model component.\n            See `pytorch_widedeep.preprocessing.WidePreprocessor`\n        X_tab: np.ndarray or List[np.ndarray], Optional. default=None\n            Input for the `deeptabular` model component.\n            See `pytorch_widedeep.preprocessing.TabPreprocessor`\n        X_text: np.ndarray or List[np.ndarray], Optional. default=None\n            Input for the `deeptext` model component.\n            See `pytorch_widedeep.preprocessing.TextPreprocessor`\n        X_img: np.ndarray or List[np.ndarray], Optional. default=None\n            Input for the `deepimage` model component.\n            See `pytorch_widedeep.preprocessing.ImagePreprocessor`\n        X_test: Dict, Optional. default=None\n            The test dataset can also be passed in a dictionary. Keys are\n            `X_wide`, _'X_tab'_, _'X_text'_, _'X_img'_ and _'target'_. Values\n            are the corresponding matrices.\n        batch_size: int, default = 256\n            If a trainer is used to predict after having trained a model, the\n            `batch_size` needs to be defined as it will not be defined as\n            the `Trainer` is instantiated\n\n        Returns\n        -------\n        np.ndarray:\n            array with the predictions\n        \"\"\"\n        preds_l = self._predict(X_wide, X_tab, X_text, X_img, X_test, batch_size)\n        if self.method == \"regression\":\n            return np.vstack(preds_l).squeeze(1)\n        if self.method == \"binary\":\n            preds = np.vstack(preds_l).squeeze(1)\n            return (preds &gt; 0.5).astype(\"int\")\n        if self.method == \"qregression\":\n            return np.vstack(preds_l)\n        if self.method == \"multiclass\":\n            preds = np.vstack(preds_l)\n            return np.argmax(preds, 1)  # type: ignore[return-value]\n        if self.method == \"multitarget\":\n            if self.loss_fn.__class__.__name__ in [\n                \"MultiTargetClassificationLoss\",\n                \"MutilTargetRegressionAndClassificationLoss\",\n            ]:\n                raise ValueError(\n                    \"MultiTargetClassificationLoss and MutilTargetRegressionAndClassificationLoss \"\n                    \"are not supported by predict method. Please use predict_proba method instead.\"\n                )\n            return np.vstack(preds_l)\n\n    def predict_uncertainty(  # type: ignore[return]\n        self,\n        X_wide: Optional[np.ndarray] = None,\n        X_tab: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n        X_text: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n        X_img: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n        X_test: Optional[Dict[str, Union[np.ndarray, List[np.ndarray]]]] = None,\n        batch_size: Optional[int] = None,\n        uncertainty_granularity: int = 1000,\n    ) -&gt; np.ndarray:\n        r\"\"\"Returns the predicted ucnertainty of the model for the test dataset\n        using a Monte Carlo method during which dropout layers are activated\n        in the evaluation/prediction phase and each sample is predicted N\n        times (`uncertainty_granularity` times).\n\n        This is based on\n        [Dropout as a Bayesian Approximation: Representing\n        Model Uncertainty in Deep Learning](https://arxiv.org/abs/1506.02142?context=stat).\n\n        Parameters\n        ----------\n        X_wide: np.ndarray, Optional. default=None\n            Input for the `wide` model component.\n            See `pytorch_widedeep.preprocessing.WidePreprocessor`\n        X_tab: np.ndarray or List[np.ndarray], Optional. default=None\n            Input for the `deeptabular` model component.\n            See `pytorch_widedeep.preprocessing.TabPreprocessor`\n        X_text: np.ndarray or List[np.ndarray], Optional. default=None\n            Input for the `deeptext` model component.\n            See `pytorch_widedeep.preprocessing.TextPreprocessor`\n        X_img: np.ndarray or List[np.ndarray], Optional. default=None\n            Input for the `deepimage` model component.\n            See `pytorch_widedeep.preprocessing.ImagePreprocessor`\n        X_test: Dict, Optional. default=None\n            The test dataset can also be passed in a dictionary. Keys are\n            _'X_wide'_, _'X_tab'_, _'X_text'_, _'X_img'_ and _'target'_. Values\n            are the corresponding matrices.\n        batch_size: int, default = 256\n            If a trainer is used to predict after having trained a model, the\n            `batch_size` needs to be defined as it will not be defined as\n            the `Trainer` is instantiated\n        uncertainty_granularity: int default = 1000\n            number of times the model does prediction for each sample\n\n        Returns\n        -------\n        np.ndarray:\n            - if `method = regression`, it will return an array with `(max, min, mean, stdev)`\n              values for each sample.\n            - if `method = binary` it will return an array with\n              `(mean_cls_0_prob, mean_cls_1_prob, predicted_cls)` for each sample.\n            - if `method = multiclass` it will return an array with\n              `(mean_cls_0_prob, mean_cls_1_prob, mean_cls_2_prob, ... , predicted_cls)`\n              values for each sample.\n\n        \"\"\"\n        preds_l = self._predict(\n            X_wide,\n            X_tab,\n            X_text,\n            X_img,\n            X_test,\n            batch_size,\n            uncertainty_granularity,\n            uncertainty=True,\n        )\n        preds = np.vstack(preds_l)\n        samples_num = int(preds.shape[0] / uncertainty_granularity)\n        if self.method == \"regression\":\n            preds = preds.squeeze(1)\n            preds = preds.reshape((uncertainty_granularity, samples_num))\n            return np.array(\n                (\n                    preds.max(axis=0),\n                    preds.min(axis=0),\n                    preds.mean(axis=0),\n                    preds.std(axis=0),\n                )\n            ).T\n        if self.method == \"qregression\":\n            raise ValueError(\n                \"Currently predict_uncertainty is not supported for qregression method\"\n            )\n        if self.method == \"binary\":\n            preds = preds.squeeze(1)\n            preds = preds.reshape((uncertainty_granularity, samples_num))\n            preds = preds.mean(axis=0)\n            probs = np.zeros([preds.shape[0], 3])\n            probs[:, 0] = 1 - preds\n            probs[:, 1] = preds\n            return probs\n        if self.method == \"multiclass\":\n            preds = preds.reshape(uncertainty_granularity, samples_num, preds.shape[1])\n            preds = preds.mean(axis=0)\n            preds = np.hstack((preds, np.vstack(np.argmax(preds, 1))))\n            return preds\n\n        if self.method == \"multitarget\":\n            raise ValueError(\n                \"Currently predict_uncertainty is not supported for multitarget method\"\n            )\n\n    def predict_proba(  # type: ignore[override, return]  # noqa: C901\n        self,\n        X_wide: Optional[np.ndarray] = None,\n        X_tab: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n        X_text: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n        X_img: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n        X_test: Optional[Dict[str, Union[np.ndarray, List[np.ndarray]]]] = None,\n        batch_size: Optional[int] = None,\n    ) -&gt; np.ndarray:\n        r\"\"\"Returns the predicted probabilities for the test dataset for  binary\n        and multiclass methods\n\n        The input datasets can be passed either directly via numpy arrays\n        (`X_wide`, `X_tab`, `X_text` or `X_img`) or alternatively, in\n        a dictionary (`X_test`)\n\n        Parameters\n        ----------\n        X_wide: np.ndarray, Optional. default=None\n            Input for the `wide` model component.\n            See `pytorch_widedeep.preprocessing.WidePreprocessor`\n        X_tab: np.ndarray or List[np.ndarray], Optional. default=None\n            Input for the `deeptabular` model component.\n            See `pytorch_widedeep.preprocessing.TabPreprocessor`\n        X_text: np.ndarray or List[np.ndarray], Optional. default=None\n            Input for the `deeptext` model component.\n            See `pytorch_widedeep.preprocessing.TextPreprocessor`\n        X_img: np.ndarray or List[np.ndarray], Optional. default=None\n            Input for the `deepimage` model component.\n            See `pytorch_widedeep.preprocessing.ImagePreprocessor`\n        X_test: Dict, Optional. default=None\n            The test dataset can also be passed in a dictionary. Keys are\n            `X_wide`, _'X_tab'_, _'X_text'_, _'X_img'_ and _'target'_. Values\n            are the corresponding matrices.\n        batch_size: int, default = 256\n            If a trainer is used to predict after having trained a model, the\n            `batch_size` needs to be defined as it will not be defined as\n            the `Trainer` is instantiated\n\n        Returns\n        -------\n        np.ndarray\n            array with the probabilities per class\n        \"\"\"\n\n        preds_l = self._predict(X_wide, X_tab, X_text, X_img, X_test, batch_size)\n        if self.method == \"binary\":\n            preds = np.vstack(preds_l).squeeze(1)\n            probs = np.zeros([preds.shape[0], 2])\n            probs[:, 0] = 1 - preds\n            probs[:, 1] = preds\n            return probs\n        if self.method == \"multiclass\":\n            return np.vstack(preds_l)\n        if self.method == \"multitarget\":\n            return np.vstack(preds_l)\n\n    def explain(self, X_tab: np.ndarray, save_step_masks: Optional[bool] = None):\n        # TO DO: Add docs to this, to the feat imp parameter and the all\n        # related classes\n        explainer = Explainer(self.device)\n\n        res = explainer.explain(\n            self.model, X_tab, self.num_workers, self.batch_size, save_step_masks\n        )\n\n        return res\n\n    def save(\n        self,\n        path: str,\n        save_state_dict: bool = False,\n        save_optimizer: bool = False,\n        model_filename: str = \"wd_model.pt\",\n    ):\n        r\"\"\"Saves the model, training and evaluation history, and the\n        `feature_importance` attribute (if the `deeptabular` component is a\n        Tabnet model) to disk\n\n        The `Trainer` class is built so that it 'just' trains a model. With\n        that in mind, all the torch related parameters (such as optimizers,\n        learning rate schedulers, initializers, etc) have to be defined\n        externally and then passed to the `Trainer`. As a result, the\n        `Trainer` does not generate any attribute or additional data\n        products that need to be saved other than the `model` object itself,\n        which can be saved as any other torch model (e.g. `torch.save(model,\n        path)`).\n\n        The exception is Tabnet. If the `deeptabular` component is a Tabnet\n        model, an attribute (a dict) called `feature_importance` will be\n        created at the end of the training process. Therefore, a `save`\n        method was created that will save the feature importance dictionary\n        to a json file and, since we are here, the model weights, training\n        history and learning rate history.\n\n        Parameters\n        ----------\n        path: str\n            path to the directory where the model and the feature importance\n            attribute will be saved.\n        save_state_dict: bool, default = False\n            Boolean indicating whether to save directly the model\n            (and optimizer) or the model's (and optimizer's) state\n            dictionary\n        save_optimizer: bool, default = False\n            Boolean indicating whether to save the optimizer\n        model_filename: str, Optional, default = \"wd_model.pt\"\n            filename where the model weights will be store\n        \"\"\"\n\n        self._save_history(path)\n\n        self._save_model_and_optimizer(\n            path, save_state_dict, save_optimizer, model_filename\n        )\n\n        if self.model.is_tabnet:\n            with open(Path(path) / \"feature_importance.json\", \"w\") as fi:\n                json.dump(self.feature_importance, fi)\n\n    def _set_dataloader(\n        self,\n        dataloader: Optional[CustomDataLoader],\n        dataset: WideDeepDataset,\n        batch_size: int,\n        dataloader_args: Dict[str, Any],\n    ) -&gt; Union[DataLoader, CustomDataLoader]:\n        if dataloader is not None:\n            dataloader.set_dataset(dataset)\n            return dataloader\n        else:\n            # var name 'loader' to avoid reassigment and type errors\n            loader = DataLoader(\n                dataset=dataset,\n                batch_size=batch_size,\n                num_workers=self.num_workers,\n                **dataloader_args,\n            )\n        return loader\n\n    @alias(\"n_epochs\", [\"finetune_epochs\", \"warmup_epochs\"])\n    @alias(\"max_lr\", [\"finetune_max_lr\", \"warmup_max_lr\"])\n    def _do_finetune(\n        self,\n        loader: DataLoader,\n        n_epochs: int = 5,\n        max_lr: float = 0.01,\n        routine: Literal[\"howard\", \"felbo\"] = \"howard\",\n        deeptabular_gradual: bool = False,\n        deeptabular_layers: Optional[\n            Union[List[nn.Module], List[List[nn.Module]]]\n        ] = None,\n        deeptabular_max_lr: Union[float, List[float]] = 0.01,\n        deeptext_gradual: bool = False,\n        deeptext_layers: Optional[Union[List[nn.Module], List[List[nn.Module]]]] = None,\n        deeptext_max_lr: Union[float, List[float]] = 0.01,\n        deepimage_gradual: bool = False,\n        deepimage_layers: Optional[\n            Union[List[nn.Module], List[List[nn.Module]]]\n        ] = None,\n        deepimage_max_lr: Union[float, List[float]] = 0.01,\n    ):\n        r\"\"\"\n        Simple wrap-up to individually fine-tune model components\n        \"\"\"\n        if self.model.deephead is not None:\n            raise ValueError(\n                \"Currently warming up is only supported without a fully connected 'DeepHead'\"\n            )\n\n        finetuner = FineTune(self.loss_fn, self.metric, self.method, self.verbose)  # type: ignore[arg-type]\n        if self.model.wide:\n            finetuner.finetune_all(self.model.wide, \"wide\", loader, n_epochs, max_lr)\n\n        if self.model.deeptabular:\n            if deeptabular_gradual:\n                assert (\n                    deeptabular_layers is not None\n                ), \"deeptabular_layers must be passed if deeptabular_gradual=True\"\n                finetuner.finetune_gradual(\n                    self.model.deeptabular,\n                    \"deeptabular\",\n                    loader,\n                    deeptabular_max_lr,\n                    deeptabular_layers,\n                    routine,\n                )\n            else:\n                finetuner.finetune_all(\n                    self.model.deeptabular, \"deeptabular\", loader, n_epochs, max_lr\n                )\n\n        if self.model.deeptext:\n            if deeptext_gradual:\n                assert (\n                    deeptext_layers is not None\n                ), \"deeptext_layers must be passed if deeptext_gradual=True\"\n                finetuner.finetune_gradual(\n                    self.model.deeptext,\n                    \"deeptext\",\n                    loader,\n                    deeptext_max_lr,\n                    deeptext_layers,\n                    routine,\n                )\n            else:\n                finetuner.finetune_all(\n                    self.model.deeptext, \"deeptext\", loader, n_epochs, max_lr\n                )\n\n        if self.model.deepimage:\n            if deepimage_gradual:\n                assert (\n                    deepimage_layers is not None\n                ), \"deepimage_layers must be passed if deepimage_gradual=True\"\n                finetuner.finetune_gradual(\n                    self.model.deepimage,\n                    \"deepimage\",\n                    loader,\n                    deepimage_max_lr,\n                    deepimage_layers,\n                    routine,\n                )\n            else:\n                finetuner.finetune_all(\n                    self.model.deepimage, \"deepimage\", loader, n_epochs, max_lr\n                )\n\n    def _train_epoch(\n        self,\n        train_loader: DataLoader,\n        epoch: int,\n    ):\n        epoch_logs: Dict[str, float] = {}\n        self.callback_container.on_epoch_begin(epoch, logs=epoch_logs)\n        self.train_running_loss = 0.0\n\n        train_steps = len(train_loader)\n        with trange(train_steps, disable=self.verbose != 1) as t:\n            for batch_idx, (data, targett) in zip(t, train_loader):\n                t.set_description(\"epoch %i\" % (epoch + 1))\n                train_score, train_loss = self._train_step(data, targett, batch_idx)\n                print_loss_and_metric(t, train_loss, train_score)\n                self.callback_container.on_batch_end(batch=batch_idx)\n\n        epoch_logs = save_epoch_logs(epoch_logs, train_loss, train_score, \"train\")\n\n        return epoch_logs\n\n    def _train_step(\n        self,\n        data: Dict[str, Union[Tensor, List[Tensor]]],\n        target: Tensor,\n        batch_idx: int,\n    ):\n\n        self.model.train()\n\n        X: Dict[str, Union[Tensor, List[Tensor]]] = {}\n        for k, v in data.items():\n            if isinstance(v, list):\n                X[k] = [to_device(i, self.device) for i in v]\n            else:\n                X[k] = to_device(v, self.device)\n        y = (\n            target.view(-1, 1).float()\n            if self.method not in [\"multiclass\", \"qregression\", \"multitarget\"]\n            else target\n        )\n        y = to_device(y, self.device)\n\n        self.optimizer.zero_grad()\n\n        y_pred = self.model(X)\n\n        if self.model.is_tabnet:\n            loss = self.loss_fn(y_pred[0], y) - self.lambda_sparse * y_pred[1]\n            score = self._get_score(y_pred[0], y, is_train=True)\n        else:\n            loss = self.loss_fn(y_pred, y)\n            score = self._get_score(y_pred, y, is_train=True)\n\n        loss.backward()\n        self.optimizer.step()\n\n        self.train_running_loss += loss.item()\n        avg_loss = self.train_running_loss / (batch_idx + 1)\n\n        return score, avg_loss\n\n    def _eval_epoch(\n        self,\n        eval_loader: DataLoader,\n        epoch_logs: Dict[str, float],\n    ) -&gt; Tuple[Dict[str, float], Optional[float]]:\n        self.callback_container.on_eval_begin()\n        self.valid_running_loss = 0.0\n\n        eval_steps = len(eval_loader)\n        with trange(eval_steps, disable=self.verbose != 1) as v:\n            for i, (data, targett) in zip(v, eval_loader):\n                v.set_description(\"valid\")\n                val_score, val_loss = self._eval_step(data, targett, i)\n                print_loss_and_metric(v, val_loss, val_score)\n\n        epoch_logs = save_epoch_logs(epoch_logs, val_loss, val_score, \"val\")\n\n        if self.reducelronplateau:\n            if self.reducelronplateau_criterion == \"loss\":\n                on_epoch_end_metric = val_loss\n            else:\n                on_epoch_end_metric = val_score[self.reducelronplateau_criterion]\n        else:\n            on_epoch_end_metric = None\n\n        return epoch_logs, on_epoch_end_metric\n\n    def _eval_step(\n        self,\n        data: Dict[str, Union[Tensor, List[Tensor]]],\n        target: Tensor,\n        batch_idx: int,\n    ):\n        self.model.eval()\n        with torch.no_grad():\n            X: Dict[str, Union[Tensor, List[Tensor]]] = {}\n            for k, v in data.items():\n                if isinstance(v, list):\n                    X[k] = [to_device(i, self.device) for i in v]\n                else:\n                    X[k] = to_device(v, self.device)\n            y = (\n                target.view(-1, 1).float()\n                if self.method not in [\"multiclass\", \"qregression\", \"multitarget\"]\n                else target\n            )\n            y = to_device(y, self.device)\n\n            y_pred = self.model(X)\n            if self.model.is_tabnet:\n                loss = self.loss_fn(y_pred[0], y) - self.lambda_sparse * y_pred[1]\n                score = self._get_score(y_pred[0], y, is_train=False)\n            else:\n                score = self._get_score(y_pred, y, is_train=False)\n                loss = self.loss_fn(y_pred, y)\n\n            self.valid_running_loss += loss.item()\n            avg_loss = self.valid_running_loss / (batch_idx + 1)\n\n        self.model.train()\n        return score, avg_loss\n\n    def _get_score(\n        self, y_pred: Tensor, y: Tensor, is_train: bool\n    ) -&gt; Optional[Dict[str, float]]:\n\n        score = None\n        metric = None\n\n        if self.metric and not self.eval_metric:\n            metric = self.metric\n        elif self.metric and self.eval_metric:\n            metric = self.metric if is_train else self.eval_metric\n        elif not self.metric and self.eval_metric:\n            metric = None if is_train else self.eval_metric\n\n        if metric is not None:\n            if self.method == \"regression\":\n                score = metric(y_pred, y)\n            if self.method == \"binary\":\n                score = metric(torch.sigmoid(y_pred), y)\n            if self.method == \"qregression\":\n                score = metric(y_pred, y)\n            if self.method == \"multiclass\":\n                score = metric(F.softmax(y_pred, dim=1), y)\n        return score\n\n    def _predict(  # type: ignore[override, return]  # noqa: C901\n        self,\n        X_wide: Optional[np.ndarray] = None,\n        X_tab: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n        X_text: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n        X_img: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n        X_test: Optional[Dict[str, Union[np.ndarray, List[np.ndarray]]]] = None,\n        batch_size: Optional[int] = None,\n        uncertainty_granularity: int = 1000,\n        uncertainty: bool = False,\n    ) -&gt; List:\n        r\"\"\"Private method to avoid code repetition in predict and\n        predict_proba. For parameter information, please, see the .predict()\n        method documentation\n        \"\"\"\n        if X_test is not None:\n            test_set = WideDeepDataset(**X_test)  # type: ignore[arg-type]\n        else:\n            load_dict: Dict[str, Union[np.ndarray, List[np.ndarray]]] = {}\n            if X_wide is not None:\n                load_dict = {\"X_wide\": X_wide}\n            if X_tab is not None:\n                load_dict.update({\"X_tab\": X_tab})\n            if X_text is not None:\n                load_dict.update({\"X_text\": X_text})\n            if X_img is not None:\n                load_dict.update({\"X_img\": X_img})\n            test_set = WideDeepDataset(**load_dict)  # type: ignore[arg-type]\n\n        if not hasattr(self, \"batch_size\"):\n            assert batch_size is not None, (\n                \"'batch_size' has not be previosly set in this Trainer and must be\"\n                \" specified via the 'batch_size' argument in this predict call\"\n            )\n            self.batch_size = batch_size\n\n        test_loader = DataLoader(\n            dataset=test_set,\n            batch_size=self.batch_size,\n            num_workers=self.num_workers,\n            shuffle=False,\n        )\n        test_steps = (len(test_loader.dataset) // test_loader.batch_size) + 1  # type: ignore[arg-type]\n\n        self.model.eval()\n        preds_l = []\n\n        if uncertainty:\n            for m in self.model.modules():\n                if m.__class__.__name__.startswith(\"Dropout\"):\n                    m.train()\n            prediction_iters = uncertainty_granularity\n        else:\n            prediction_iters = 1\n\n        with torch.no_grad():\n            with trange(uncertainty_granularity, disable=uncertainty is False) as t:\n                for _, _ in zip(t, range(prediction_iters)):\n                    t.set_description(\"predict_UncertaintyIter\")\n\n                    with trange(\n                        test_steps, disable=self.verbose != 1 or uncertainty is True\n                    ) as tt:\n                        for _, data in zip(tt, test_loader):\n                            tt.set_description(\"predict\")\n                            X: Dict[str, Union[Tensor, List[Tensor]]] = {}\n                            for k, v in data.items():\n                                if isinstance(v, list):\n                                    X[k] = [to_device(i, self.device) for i in v]\n                                else:\n                                    X[k] = to_device(v, self.device)\n                            preds = (\n                                self.model(X)\n                                if not self.model.is_tabnet\n                                else self.model(X)[0]\n                            )\n                            if self.method == \"binary\":\n                                preds = torch.sigmoid(preds)\n                            if self.method == \"multiclass\":\n                                preds = F.softmax(preds, dim=1)\n                            if self.method == \"regression\" and isinstance(\n                                self.loss_fn, ZILNLoss\n                            ):\n                                preds = self._predict_ziln(preds)\n                            preds = preds.cpu().data.numpy()\n                            preds_l.append(preds)\n        self.model.train()\n        return preds_l\n\n    @staticmethod\n    def _predict_ziln(preds: Tensor) -&gt; Tensor:\n        # Legacy implementation. It will be removed in future versions\n        \"\"\"Calculates predicted mean of zero inflated lognormal logits.\n\n        Adjusted implementaion of `code\n        &lt;https://github.com/google/lifetime_value/blob/master/lifetime_value/zero_inflated_lognormal.py&gt;`\n\n        Arguments:\n            preds: [batch_size, 3] tensor of logits.\n        Returns:\n            ziln_preds: [batch_size, 1] tensor of predicted mean.\n        \"\"\"\n        positive_probs = torch.sigmoid(preds[..., :1])\n        loc = preds[..., 1:2]\n        scale = F.softplus(preds[..., 2:])\n        ziln_preds = positive_probs * torch.exp(loc + 0.5 * torch.square(scale))\n        return ziln_preds\n\n    @staticmethod\n    def _extract_kwargs(kwargs):\n        dataloader_params = [\n            \"shuffle\",\n            \"sampler\",\n            \"batch_sampler\",\n            \"num_workers\",\n            \"collate_fn\",\n            \"pin_memory\",\n            \"drop_last\",\n            \"timeout\",\n            \"worker_init_fn\",\n            \"generator\",\n            \"prefetch_factor\",\n            \"persistent_workers\",\n            \"oversample_mul\",\n        ]\n        finetune_params = [\n            \"n_epochs\",\n            \"finetune_epochs\",\n            \"warmup_epochs\",\n            \"max_lr\",\n            \"finetune_max_lr\",\n            \"warmup_max_lr\",\n            \"routine\",\n            \"deeptabular_gradual\",\n            \"deeptabular_layers\",\n            \"deeptabular_max_lr\",\n            \"deeptext_gradual\",\n            \"deeptext_layers\",\n            \"deeptext_max_lr\",\n            \"deepimage_gradual\",\n            \"deepimage_layers\",\n            \"deepimage_max_lr\",\n        ]\n\n        dataloader_args, finetune_args = {}, {}\n        for k, v in kwargs.items():\n            if k in dataloader_params:\n                dataloader_args[k] = v\n            if k in finetune_params:\n                finetune_args[k] = v\n\n        return dataloader_args, finetune_args\n</code></pre>"},{"location":"pytorch-widedeep/trainer.html#pytorch_widedeep.training.Trainer.fit","title":"fit","text":"<pre><code>fit(\n    X_wide=None,\n    X_tab=None,\n    X_text=None,\n    X_img=None,\n    X_train=None,\n    X_val=None,\n    val_split=None,\n    target=None,\n    n_epochs=1,\n    validation_freq=1,\n    batch_size=32,\n    train_dataloader=None,\n    eval_dataloader=None,\n    feature_importance_sample_size=None,\n    finetune=False,\n    **kwargs\n)\n</code></pre> <p>Fit method.</p> <p>The input datasets can be passed either directly via numpy arrays (<code>X_wide</code>, <code>X_tab</code>, <code>X_text</code> or <code>X_img</code>) or alternatively, in dictionaries (<code>X_train</code> or <code>X_val</code>).</p> <p>Parameters:</p> Name Type Description Default <code>X_wide</code> <code>Optional[ndarray]</code> <p>Input for the <code>wide</code> model component. See <code>pytorch_widedeep.preprocessing.WidePreprocessor</code></p> <code>None</code> <code>X_tab</code> <code>Optional[Union[ndarray, List[ndarray]]]</code> <p>Input for the <code>deeptabular</code> model component. See <code>pytorch_widedeep.preprocessing.TabPreprocessor</code>. If multiple tabular models are used for different columns, this should be a list of numpy arrays</p> <code>None</code> <code>X_text</code> <code>Optional[Union[ndarray, List[ndarray]]]</code> <p>Input for the <code>deeptext</code> model component. See <code>pytorch_widedeep.preprocessing.TextPreprocessor</code>. If multiple text columns/models are used, this should be a list of numpy arrays</p> <code>None</code> <code>X_img</code> <code>Optional[Union[ndarray, List[ndarray]]]</code> <p>Input for the <code>deepimage</code> model component. See <code>pytorch_widedeep.preprocessing.ImagePreprocessor</code>. If multiple image columns/models are used, this should be a list of numpy arrays</p> <code>None</code> <code>X_train</code> <code>Optional[Dict[str, Union[ndarray, List[ndarray]]]]</code> <p>The training dataset can also be passed in a dictionary. Keys are 'X_wide', 'X_tab', 'X_text', 'X_img' and 'target'. Values are the corresponding matrices. Note that of multiple text or image columns/models are used, the corresponding values should be lists of numpy arrays</p> <code>None</code> <code>X_val</code> <code>Optional[Dict[str, Union[ndarray, List[ndarray]]]]</code> <p>The validation dataset can also be passed in a dictionary. Keys are 'X_wide', 'X_tab', 'X_text', 'X_img' and 'target'. Values are the corresponding matrices. Note that of multiple text or image columns/models are used, the corresponding values should be lists of numpy arrays</p> <code>None</code> <code>val_split</code> <code>Optional[float]</code> <p>train/val split fraction</p> <code>None</code> <code>target</code> <code>Optional[ndarray]</code> <p>target values</p> <code>None</code> <code>n_epochs</code> <code>int</code> <p>number of epochs</p> <code>1</code> <code>validation_freq</code> <code>int</code> <p>epochs validation frequency</p> <code>1</code> <code>batch_size</code> <code>int</code> <p>batch size</p> <code>32</code> <code>custom_dataloader</code> <p>object of class <code>torch.utils.data.DataLoader</code>. Available predefined dataloaders are in <code>pytorch-widedeep.dataloaders</code>.If <code>None</code>, a standard torch <code>DataLoader</code> is used.</p> required <code>finetune</code> <code>bool</code> <p>fine-tune individual model components. This functionality can also be used to 'warm-up' (and hence the alias <code>warmup</code>) individual components before the joined training starts, and hence its alias. See the Examples folder in the repo for more details</p> <p><code>pytorch_widedeep</code> implements 3 fine-tune routines.</p> <ul> <li>fine-tune all trainable layers at once. This routine is   inspired by the work of Howard &amp; Sebastian Ruder 2018 in their   ULMfit paper. Using a   Slanted Triangular learing (see   Leslie N. Smith paper ) ,   the process is the following: i) the learning rate will   gradually increase for 10% of the training steps from max_lr/10   to max_lr. ii) It will then gradually decrease to max_lr/10   for the remaining 90% of the steps. The optimizer used in the   process is <code>Adam</code>.</li> </ul> <p>and two gradual fine-tune routines, where only certain layers are trained at a time.</p> <ul> <li>The so called <code>Felbo</code> gradual fine-tune rourine, based on the the   Felbo et al., 2017 DeepEmoji paper.</li> <li>The <code>Howard</code> routine based on the work of Howard &amp; Sebastian Ruder 2018 in their   ULMfit paper.</li> </ul> <p>For details on how these routines work, please see the Examples section in this documentation and the Examples folder in the repo.  Param Alias: <code>warmup</code></p> <code>False</code> <p>Other Parameters:</p> Name Type Description <code>**kwargs</code> <p>Other keyword arguments are:</p> <ul> <li> <p>DataLoader related parameters:     For example,  <code>sampler</code>, <code>batch_sampler</code>, <code>collate_fn</code>, etc.     Please, see the pytorch     DataLoader docs     for details.</p> </li> <li> <p>Finetune related parameters:     see the source code at <code>pytorch_widedeep._finetune</code>. Namely, these are:</p> <ul> <li><code>finetune_epochs</code> (<code>int</code>):     number of epochs use for fine tuning</li> <li><code>finetune_max_lr</code> (<code>float</code>):    max lr during fine tuning</li> <li><code>routine</code> (<code>str</code>):    one of 'howard' or 'felbo'</li> <li><code>deeptabular_gradual</code> (<code>bool</code>):    boolean indicating if the <code>deeptabular</code> component will be fine tuned gradually</li> <li><code>deeptabular_layers</code> (<code>Optional[Union[List[nn.Module], List[List[nn.Module]]]]</code>):    List of pytorch modules indicating the layers of the    <code>deeptabular</code> that will be fine tuned</li> <li><code>deeptabular_max_lr</code> (<code>Union[float, List[float]]</code>):    max lr for the <code>deeptabular</code> componet during fine tuning</li> <li><code>deeptext_gradual</code> (<code>bool</code>):    same as <code>deeptabular_gradual</code> but for the <code>deeptext</code> component</li> <li><code>deeptext_layers</code> (<code>Optional[Union[List[nn.Module], List[List[nn.Module]]]]</code>):    same as <code>deeptabular_gradual</code> but for the <code>deeptext</code> component.    If there are multiple text columns/models, this should be a list of lists</li> <li><code>deeptext_max_lr</code> (<code>Union[float, List[float]]</code>):    same as <code>deeptabular_gradual</code> but for the <code>deeptext</code> component    If there are multiple text columns/models, this should be a list of floats</li> <li><code>deepimage_gradual</code> (<code>bool</code>):    same as <code>deeptext_layers</code> but for the <code>deepimage</code> component</li> <li><code>deepimage_layers</code> (<code>Optional[Union[List[nn.Module], List[List[nn.Module]]]]</code>):    same as <code>deeptext_layers</code> but for the <code>deepimage</code> component</li> <li><code>deepimage_max_lr</code> (<code>Union[float, List[float]]</code>):     same as <code>deeptext_layers</code> but for the <code>deepimage</code> component</li> </ul> </li> </ul> <p>Examples:</p> <p>For a series of comprehensive examples on how to use the <code>fit</code> method, please see the Examples folder in the repo</p> Source code in <code>pytorch_widedeep/training/trainer.py</code> <pre><code>@alias(\"finetune\", [\"warmup\"])\n@alias(\"train_dataloader\", [\"custom_dataloader\"])\n@alias(\"eval_dataloader\", [\"custom_eval_dataloader\"])\ndef fit(  # noqa: C901\n    self,\n    X_wide: Optional[np.ndarray] = None,\n    X_tab: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n    X_text: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n    X_img: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n    X_train: Optional[Dict[str, Union[np.ndarray, List[np.ndarray]]]] = None,\n    X_val: Optional[Dict[str, Union[np.ndarray, List[np.ndarray]]]] = None,\n    val_split: Optional[float] = None,\n    target: Optional[np.ndarray] = None,\n    n_epochs: int = 1,\n    validation_freq: int = 1,\n    batch_size: int = 32,\n    train_dataloader: Optional[CustomDataLoader] = None,\n    eval_dataloader: Optional[CustomDataLoader] = None,\n    feature_importance_sample_size: Optional[int] = None,\n    finetune: bool = False,\n    **kwargs,\n):\n    r\"\"\"Fit method.\n\n    The input datasets can be passed either directly via numpy arrays\n    (`X_wide`, `X_tab`, `X_text` or `X_img`) or alternatively, in\n    dictionaries (`X_train` or `X_val`).\n\n    Parameters\n    ----------\n    X_wide: np.ndarray, Optional. default=None\n        Input for the `wide` model component.\n        See `pytorch_widedeep.preprocessing.WidePreprocessor`\n    X_tab: np.ndarray, Optional. default=None\n        Input for the `deeptabular` model component.\n        See `pytorch_widedeep.preprocessing.TabPreprocessor`. If multiple\n        tabular models are used for different columns, this should be a\n        list of numpy arrays\n    X_text: Union[np.ndarray, List[np.ndarray]], Optional. default=None\n        Input for the `deeptext` model component.\n        See `pytorch_widedeep.preprocessing.TextPreprocessor`.\n        If multiple text columns/models are used, this should be a list of\n        numpy arrays\n    X_img: np.ndarray, Optional. default=None\n        Input for the `deepimage` model component.\n        See `pytorch_widedeep.preprocessing.ImagePreprocessor`.\n        If multiple image columns/models are used, this should be a list of\n        numpy arrays\n    X_train: Dict, Optional. default=None\n        The training dataset can also be passed in a dictionary. Keys are\n        _'X_wide'_, _'X_tab'_, _'X_text'_, _'X_img'_ and _'target'_. Values\n        are the corresponding matrices. Note that of multiple text or image\n        columns/models are used, the corresponding values should be lists\n        of numpy arrays\n    X_val: Dict, Optional. default=None\n        The validation dataset can also be passed in a dictionary. Keys\n        are _'X_wide'_, _'X_tab'_, _'X_text'_, _'X_img'_ and _'target'_.\n        Values are the corresponding matrices. Note that of multiple text\n        or image columns/models are used, the corresponding values should\n        be lists of numpy arrays\n    val_split: float, Optional. default=None\n        train/val split fraction\n    target: np.ndarray, Optional. default=None\n        target values\n    n_epochs: int, default=1\n        number of epochs\n    validation_freq: int, default=1\n        epochs validation frequency\n    batch_size: int, default=32\n        batch size\n    custom_dataloader: `DataLoader`, Optional, default=None\n        object of class `torch.utils.data.DataLoader`. Available\n        predefined dataloaders are in `pytorch-widedeep.dataloaders`.If\n        `None`, a standard torch `DataLoader` is used.\n    finetune: bool, default=False\n        fine-tune individual model components. This functionality can also\n        be used to 'warm-up' (and hence the alias `warmup`) individual\n        components before the joined training starts, and hence its\n        alias. See the Examples folder in the repo for more details\n\n        `pytorch_widedeep` implements 3 fine-tune routines.\n\n        - fine-tune all trainable layers at once. This routine is\n          inspired by the work of Howard &amp; Sebastian Ruder 2018 in their\n          [ULMfit paper](https://arxiv.org/abs/1801.06146). Using a\n          Slanted Triangular learing (see\n          [Leslie N. Smith paper](https://arxiv.org/pdf/1506.01186.pdf) ) ,\n          the process is the following: *i*) the learning rate will\n          gradually increase for 10% of the training steps from max_lr/10\n          to max_lr. *ii*) It will then gradually decrease to max_lr/10\n          for the remaining 90% of the steps. The optimizer used in the\n          process is `Adam`.\n\n        and two gradual fine-tune routines, where only certain layers are\n        trained at a time.\n\n        - The so called `Felbo` gradual fine-tune rourine, based on the the\n          Felbo et al., 2017 [DeepEmoji paper](https://arxiv.org/abs/1708.00524).\n        - The `Howard` routine based on the work of Howard &amp; Sebastian Ruder 2018 in their\n          [ULMfit paper](https://arxiv.org/abs/1801.06146&gt;).\n\n        For details on how these routines work, please see the Examples\n        section in this documentation and the Examples folder in the repo. &lt;br/&gt;\n        Param Alias: `warmup`\n\n    Other Parameters\n    ----------------\n    **kwargs:\n        Other keyword arguments are:\n\n        - **DataLoader related parameters**:&lt;br/&gt;\n            For example,  `sampler`, `batch_sampler`, `collate_fn`, etc.\n            Please, see the pytorch\n            [DataLoader docs](https://pytorch.org/docs/stable/data.html#torch.utils.data.DataLoader)\n            for details.\n\n        - **Finetune related parameters**:&lt;br/&gt;\n            see the source code at `pytorch_widedeep._finetune`. Namely, these are:\n\n            - `finetune_epochs` (`int`):\n                number of epochs use for fine tuning\n            - `finetune_max_lr` (`float`):\n               max lr during fine tuning\n            - `routine` (`str`):\n               one of _'howard'_ or _'felbo'_\n            - `deeptabular_gradual` (`bool`):\n               boolean indicating if the `deeptabular` component will be fine tuned gradually\n            - `deeptabular_layers` (`Optional[Union[List[nn.Module], List[List[nn.Module]]]]`):\n               List of pytorch modules indicating the layers of the\n               `deeptabular` that will be fine tuned\n            - `deeptabular_max_lr` (`Union[float, List[float]]`):\n               max lr for the `deeptabular` componet during fine tuning\n            - `deeptext_gradual` (`bool`):\n               same as `deeptabular_gradual` but for the `deeptext` component\n            - `deeptext_layers` (`Optional[Union[List[nn.Module], List[List[nn.Module]]]]`):\n               same as `deeptabular_gradual` but for the `deeptext` component.\n               If there are multiple text columns/models, this should be a list of lists\n            - `deeptext_max_lr` (`Union[float, List[float]]`):\n               same as `deeptabular_gradual` but for the `deeptext` component\n               If there are multiple text columns/models, this should be a list of floats\n            - `deepimage_gradual` (`bool`):\n               same as `deeptext_layers` but for the `deepimage` component\n            - `deepimage_layers` (`Optional[Union[List[nn.Module], List[List[nn.Module]]]]`):\n               same as `deeptext_layers` but for the `deepimage` component\n            - `deepimage_max_lr` (`Union[float, List[float]]`):\n                same as `deeptext_layers` but for the `deepimage` component\n\n    Examples\n    --------\n\n    For a series of comprehensive examples on how to use the `fit` method, please see the\n    [Examples](https://github.com/jrzaurin/pytorch-widedeep/tree/master/examples)\n    folder in the repo\n    \"\"\"\n    dataloader_args, finetune_args = self._extract_kwargs(kwargs)\n    self.batch_size = batch_size\n    train_set, eval_set = wd_train_val_split(\n        self.seed,\n        self.method,  # type: ignore\n        X_wide,\n        X_tab,\n        X_text,\n        X_img,\n        X_train,\n        X_val,\n        val_split,\n        target,\n        self.transforms,\n    )\n    train_loader = self._set_dataloader(\n        train_dataloader, train_set, batch_size, dataloader_args\n    )\n    eval_loader = (\n        self._set_dataloader(eval_dataloader, eval_set, batch_size, dataloader_args)\n        if eval_set is not None\n        else None\n    )\n\n    if finetune:\n        self.with_finetuning: bool = True\n        self._do_finetune(train_loader, **finetune_args)\n        if self.verbose:\n            print(\n                \"Fine-tuning (or warmup) of individual components completed. \"\n                \"Training the whole model for {} epochs\".format(n_epochs)\n            )\n    else:\n        self.with_finetuning = False\n\n    self.callback_container.on_train_begin(\n        {\n            \"batch_size\": batch_size,\n            \"train_steps\": len(train_loader),\n            \"n_epochs\": n_epochs,\n        }\n    )\n    for epoch in range(n_epochs):\n        epoch_logs = self._train_epoch(train_loader, epoch)\n        if eval_loader is not None and epoch % validation_freq == (\n            validation_freq - 1\n        ):\n            epoch_logs, on_epoch_end_metric = self._eval_epoch(\n                eval_loader, epoch_logs\n            )\n        else:\n            on_epoch_end_metric = None\n            if self.reducelronplateau:\n                raise NotImplementedError(\n                    \"ReduceLROnPlateau scheduler can be used only with validation data.\"\n                )\n        self.callback_container.on_epoch_end(epoch, epoch_logs, on_epoch_end_metric)\n        if self.early_stop:\n            # self.callback_container.on_train_end(epoch_logs)\n            break\n\n    self.callback_container.on_train_end(epoch_logs)\n    if feature_importance_sample_size is not None:\n        self.feature_importance = FeatureImportance(\n            self.device, feature_importance_sample_size\n        ).feature_importance(train_loader, self.model)\n    self._restore_best_weights()\n    self.model.train()\n</code></pre>"},{"location":"pytorch-widedeep/trainer.html#pytorch_widedeep.training.Trainer.predict","title":"predict","text":"<pre><code>predict(\n    X_wide=None,\n    X_tab=None,\n    X_text=None,\n    X_img=None,\n    X_test=None,\n    batch_size=None,\n)\n</code></pre> <p>Returns the predictions</p> <p>The input datasets can be passed either directly via numpy arrays (<code>X_wide</code>, <code>X_tab</code>, <code>X_text</code> or <code>X_img</code>) or alternatively, in a dictionary (<code>X_test</code>)</p> <p>Parameters:</p> Name Type Description Default <code>X_wide</code> <code>Optional[ndarray]</code> <p>Input for the <code>wide</code> model component. See <code>pytorch_widedeep.preprocessing.WidePreprocessor</code></p> <code>None</code> <code>X_tab</code> <code>Optional[Union[ndarray, List[ndarray]]]</code> <p>Input for the <code>deeptabular</code> model component. See <code>pytorch_widedeep.preprocessing.TabPreprocessor</code></p> <code>None</code> <code>X_text</code> <code>Optional[Union[ndarray, List[ndarray]]]</code> <p>Input for the <code>deeptext</code> model component. See <code>pytorch_widedeep.preprocessing.TextPreprocessor</code></p> <code>None</code> <code>X_img</code> <code>Optional[Union[ndarray, List[ndarray]]]</code> <p>Input for the <code>deepimage</code> model component. See <code>pytorch_widedeep.preprocessing.ImagePreprocessor</code></p> <code>None</code> <code>X_test</code> <code>Optional[Dict[str, Union[ndarray, List[ndarray]]]]</code> <p>The test dataset can also be passed in a dictionary. Keys are <code>X_wide</code>, 'X_tab', 'X_text', 'X_img' and 'target'. Values are the corresponding matrices.</p> <code>None</code> <code>batch_size</code> <code>Optional[int]</code> <p>If a trainer is used to predict after having trained a model, the <code>batch_size</code> needs to be defined as it will not be defined as the <code>Trainer</code> is instantiated</p> <code>None</code> <p>Returns:</p> Type Description <code>np.ndarray:</code> <p>array with the predictions</p> Source code in <code>pytorch_widedeep/training/trainer.py</code> <pre><code>def predict(  # type: ignore[override, return]\n    self,\n    X_wide: Optional[np.ndarray] = None,\n    X_tab: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n    X_text: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n    X_img: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n    X_test: Optional[Dict[str, Union[np.ndarray, List[np.ndarray]]]] = None,\n    batch_size: Optional[int] = None,\n) -&gt; np.ndarray:\n    r\"\"\"Returns the predictions\n\n    The input datasets can be passed either directly via numpy arrays\n    (`X_wide`, `X_tab`, `X_text` or `X_img`) or alternatively, in\n    a dictionary (`X_test`)\n\n\n    Parameters\n    ----------\n    X_wide: np.ndarray, Optional. default=None\n        Input for the `wide` model component.\n        See `pytorch_widedeep.preprocessing.WidePreprocessor`\n    X_tab: np.ndarray or List[np.ndarray], Optional. default=None\n        Input for the `deeptabular` model component.\n        See `pytorch_widedeep.preprocessing.TabPreprocessor`\n    X_text: np.ndarray or List[np.ndarray], Optional. default=None\n        Input for the `deeptext` model component.\n        See `pytorch_widedeep.preprocessing.TextPreprocessor`\n    X_img: np.ndarray or List[np.ndarray], Optional. default=None\n        Input for the `deepimage` model component.\n        See `pytorch_widedeep.preprocessing.ImagePreprocessor`\n    X_test: Dict, Optional. default=None\n        The test dataset can also be passed in a dictionary. Keys are\n        `X_wide`, _'X_tab'_, _'X_text'_, _'X_img'_ and _'target'_. Values\n        are the corresponding matrices.\n    batch_size: int, default = 256\n        If a trainer is used to predict after having trained a model, the\n        `batch_size` needs to be defined as it will not be defined as\n        the `Trainer` is instantiated\n\n    Returns\n    -------\n    np.ndarray:\n        array with the predictions\n    \"\"\"\n    preds_l = self._predict(X_wide, X_tab, X_text, X_img, X_test, batch_size)\n    if self.method == \"regression\":\n        return np.vstack(preds_l).squeeze(1)\n    if self.method == \"binary\":\n        preds = np.vstack(preds_l).squeeze(1)\n        return (preds &gt; 0.5).astype(\"int\")\n    if self.method == \"qregression\":\n        return np.vstack(preds_l)\n    if self.method == \"multiclass\":\n        preds = np.vstack(preds_l)\n        return np.argmax(preds, 1)  # type: ignore[return-value]\n    if self.method == \"multitarget\":\n        if self.loss_fn.__class__.__name__ in [\n            \"MultiTargetClassificationLoss\",\n            \"MutilTargetRegressionAndClassificationLoss\",\n        ]:\n            raise ValueError(\n                \"MultiTargetClassificationLoss and MutilTargetRegressionAndClassificationLoss \"\n                \"are not supported by predict method. Please use predict_proba method instead.\"\n            )\n        return np.vstack(preds_l)\n</code></pre>"},{"location":"pytorch-widedeep/trainer.html#pytorch_widedeep.training.Trainer.predict_uncertainty","title":"predict_uncertainty","text":"<pre><code>predict_uncertainty(\n    X_wide=None,\n    X_tab=None,\n    X_text=None,\n    X_img=None,\n    X_test=None,\n    batch_size=None,\n    uncertainty_granularity=1000,\n)\n</code></pre> <p>Returns the predicted ucnertainty of the model for the test dataset using a Monte Carlo method during which dropout layers are activated in the evaluation/prediction phase and each sample is predicted N times (<code>uncertainty_granularity</code> times).</p> <p>This is based on Dropout as a Bayesian Approximation: Representing Model Uncertainty in Deep Learning.</p> <p>Parameters:</p> Name Type Description Default <code>X_wide</code> <code>Optional[ndarray]</code> <p>Input for the <code>wide</code> model component. See <code>pytorch_widedeep.preprocessing.WidePreprocessor</code></p> <code>None</code> <code>X_tab</code> <code>Optional[Union[ndarray, List[ndarray]]]</code> <p>Input for the <code>deeptabular</code> model component. See <code>pytorch_widedeep.preprocessing.TabPreprocessor</code></p> <code>None</code> <code>X_text</code> <code>Optional[Union[ndarray, List[ndarray]]]</code> <p>Input for the <code>deeptext</code> model component. See <code>pytorch_widedeep.preprocessing.TextPreprocessor</code></p> <code>None</code> <code>X_img</code> <code>Optional[Union[ndarray, List[ndarray]]]</code> <p>Input for the <code>deepimage</code> model component. See <code>pytorch_widedeep.preprocessing.ImagePreprocessor</code></p> <code>None</code> <code>X_test</code> <code>Optional[Dict[str, Union[ndarray, List[ndarray]]]]</code> <p>The test dataset can also be passed in a dictionary. Keys are 'X_wide', 'X_tab', 'X_text', 'X_img' and 'target'. Values are the corresponding matrices.</p> <code>None</code> <code>batch_size</code> <code>Optional[int]</code> <p>If a trainer is used to predict after having trained a model, the <code>batch_size</code> needs to be defined as it will not be defined as the <code>Trainer</code> is instantiated</p> <code>None</code> <code>uncertainty_granularity</code> <code>int</code> <p>number of times the model does prediction for each sample</p> <code>1000</code> <p>Returns:</p> Type Description <code>np.ndarray:</code> <ul> <li>if <code>method = regression</code>, it will return an array with <code>(max, min, mean, stdev)</code>   values for each sample.</li> <li>if <code>method = binary</code> it will return an array with   <code>(mean_cls_0_prob, mean_cls_1_prob, predicted_cls)</code> for each sample.</li> <li>if <code>method = multiclass</code> it will return an array with   <code>(mean_cls_0_prob, mean_cls_1_prob, mean_cls_2_prob, ... , predicted_cls)</code>   values for each sample.</li> </ul> Source code in <code>pytorch_widedeep/training/trainer.py</code> <pre><code>def predict_uncertainty(  # type: ignore[return]\n    self,\n    X_wide: Optional[np.ndarray] = None,\n    X_tab: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n    X_text: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n    X_img: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n    X_test: Optional[Dict[str, Union[np.ndarray, List[np.ndarray]]]] = None,\n    batch_size: Optional[int] = None,\n    uncertainty_granularity: int = 1000,\n) -&gt; np.ndarray:\n    r\"\"\"Returns the predicted ucnertainty of the model for the test dataset\n    using a Monte Carlo method during which dropout layers are activated\n    in the evaluation/prediction phase and each sample is predicted N\n    times (`uncertainty_granularity` times).\n\n    This is based on\n    [Dropout as a Bayesian Approximation: Representing\n    Model Uncertainty in Deep Learning](https://arxiv.org/abs/1506.02142?context=stat).\n\n    Parameters\n    ----------\n    X_wide: np.ndarray, Optional. default=None\n        Input for the `wide` model component.\n        See `pytorch_widedeep.preprocessing.WidePreprocessor`\n    X_tab: np.ndarray or List[np.ndarray], Optional. default=None\n        Input for the `deeptabular` model component.\n        See `pytorch_widedeep.preprocessing.TabPreprocessor`\n    X_text: np.ndarray or List[np.ndarray], Optional. default=None\n        Input for the `deeptext` model component.\n        See `pytorch_widedeep.preprocessing.TextPreprocessor`\n    X_img: np.ndarray or List[np.ndarray], Optional. default=None\n        Input for the `deepimage` model component.\n        See `pytorch_widedeep.preprocessing.ImagePreprocessor`\n    X_test: Dict, Optional. default=None\n        The test dataset can also be passed in a dictionary. Keys are\n        _'X_wide'_, _'X_tab'_, _'X_text'_, _'X_img'_ and _'target'_. Values\n        are the corresponding matrices.\n    batch_size: int, default = 256\n        If a trainer is used to predict after having trained a model, the\n        `batch_size` needs to be defined as it will not be defined as\n        the `Trainer` is instantiated\n    uncertainty_granularity: int default = 1000\n        number of times the model does prediction for each sample\n\n    Returns\n    -------\n    np.ndarray:\n        - if `method = regression`, it will return an array with `(max, min, mean, stdev)`\n          values for each sample.\n        - if `method = binary` it will return an array with\n          `(mean_cls_0_prob, mean_cls_1_prob, predicted_cls)` for each sample.\n        - if `method = multiclass` it will return an array with\n          `(mean_cls_0_prob, mean_cls_1_prob, mean_cls_2_prob, ... , predicted_cls)`\n          values for each sample.\n\n    \"\"\"\n    preds_l = self._predict(\n        X_wide,\n        X_tab,\n        X_text,\n        X_img,\n        X_test,\n        batch_size,\n        uncertainty_granularity,\n        uncertainty=True,\n    )\n    preds = np.vstack(preds_l)\n    samples_num = int(preds.shape[0] / uncertainty_granularity)\n    if self.method == \"regression\":\n        preds = preds.squeeze(1)\n        preds = preds.reshape((uncertainty_granularity, samples_num))\n        return np.array(\n            (\n                preds.max(axis=0),\n                preds.min(axis=0),\n                preds.mean(axis=0),\n                preds.std(axis=0),\n            )\n        ).T\n    if self.method == \"qregression\":\n        raise ValueError(\n            \"Currently predict_uncertainty is not supported for qregression method\"\n        )\n    if self.method == \"binary\":\n        preds = preds.squeeze(1)\n        preds = preds.reshape((uncertainty_granularity, samples_num))\n        preds = preds.mean(axis=0)\n        probs = np.zeros([preds.shape[0], 3])\n        probs[:, 0] = 1 - preds\n        probs[:, 1] = preds\n        return probs\n    if self.method == \"multiclass\":\n        preds = preds.reshape(uncertainty_granularity, samples_num, preds.shape[1])\n        preds = preds.mean(axis=0)\n        preds = np.hstack((preds, np.vstack(np.argmax(preds, 1))))\n        return preds\n\n    if self.method == \"multitarget\":\n        raise ValueError(\n            \"Currently predict_uncertainty is not supported for multitarget method\"\n        )\n</code></pre>"},{"location":"pytorch-widedeep/trainer.html#pytorch_widedeep.training.Trainer.predict_proba","title":"predict_proba","text":"<pre><code>predict_proba(\n    X_wide=None,\n    X_tab=None,\n    X_text=None,\n    X_img=None,\n    X_test=None,\n    batch_size=None,\n)\n</code></pre> <p>Returns the predicted probabilities for the test dataset for  binary and multiclass methods</p> <p>The input datasets can be passed either directly via numpy arrays (<code>X_wide</code>, <code>X_tab</code>, <code>X_text</code> or <code>X_img</code>) or alternatively, in a dictionary (<code>X_test</code>)</p> <p>Parameters:</p> Name Type Description Default <code>X_wide</code> <code>Optional[ndarray]</code> <p>Input for the <code>wide</code> model component. See <code>pytorch_widedeep.preprocessing.WidePreprocessor</code></p> <code>None</code> <code>X_tab</code> <code>Optional[Union[ndarray, List[ndarray]]]</code> <p>Input for the <code>deeptabular</code> model component. See <code>pytorch_widedeep.preprocessing.TabPreprocessor</code></p> <code>None</code> <code>X_text</code> <code>Optional[Union[ndarray, List[ndarray]]]</code> <p>Input for the <code>deeptext</code> model component. See <code>pytorch_widedeep.preprocessing.TextPreprocessor</code></p> <code>None</code> <code>X_img</code> <code>Optional[Union[ndarray, List[ndarray]]]</code> <p>Input for the <code>deepimage</code> model component. See <code>pytorch_widedeep.preprocessing.ImagePreprocessor</code></p> <code>None</code> <code>X_test</code> <code>Optional[Dict[str, Union[ndarray, List[ndarray]]]]</code> <p>The test dataset can also be passed in a dictionary. Keys are <code>X_wide</code>, 'X_tab', 'X_text', 'X_img' and 'target'. Values are the corresponding matrices.</p> <code>None</code> <code>batch_size</code> <code>Optional[int]</code> <p>If a trainer is used to predict after having trained a model, the <code>batch_size</code> needs to be defined as it will not be defined as the <code>Trainer</code> is instantiated</p> <code>None</code> <p>Returns:</p> Type Description <code>ndarray</code> <p>array with the probabilities per class</p> Source code in <code>pytorch_widedeep/training/trainer.py</code> <pre><code>def predict_proba(  # type: ignore[override, return]  # noqa: C901\n    self,\n    X_wide: Optional[np.ndarray] = None,\n    X_tab: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n    X_text: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n    X_img: Optional[Union[np.ndarray, List[np.ndarray]]] = None,\n    X_test: Optional[Dict[str, Union[np.ndarray, List[np.ndarray]]]] = None,\n    batch_size: Optional[int] = None,\n) -&gt; np.ndarray:\n    r\"\"\"Returns the predicted probabilities for the test dataset for  binary\n    and multiclass methods\n\n    The input datasets can be passed either directly via numpy arrays\n    (`X_wide`, `X_tab`, `X_text` or `X_img`) or alternatively, in\n    a dictionary (`X_test`)\n\n    Parameters\n    ----------\n    X_wide: np.ndarray, Optional. default=None\n        Input for the `wide` model component.\n        See `pytorch_widedeep.preprocessing.WidePreprocessor`\n    X_tab: np.ndarray or List[np.ndarray], Optional. default=None\n        Input for the `deeptabular` model component.\n        See `pytorch_widedeep.preprocessing.TabPreprocessor`\n    X_text: np.ndarray or List[np.ndarray], Optional. default=None\n        Input for the `deeptext` model component.\n        See `pytorch_widedeep.preprocessing.TextPreprocessor`\n    X_img: np.ndarray or List[np.ndarray], Optional. default=None\n        Input for the `deepimage` model component.\n        See `pytorch_widedeep.preprocessing.ImagePreprocessor`\n    X_test: Dict, Optional. default=None\n        The test dataset can also be passed in a dictionary. Keys are\n        `X_wide`, _'X_tab'_, _'X_text'_, _'X_img'_ and _'target'_. Values\n        are the corresponding matrices.\n    batch_size: int, default = 256\n        If a trainer is used to predict after having trained a model, the\n        `batch_size` needs to be defined as it will not be defined as\n        the `Trainer` is instantiated\n\n    Returns\n    -------\n    np.ndarray\n        array with the probabilities per class\n    \"\"\"\n\n    preds_l = self._predict(X_wide, X_tab, X_text, X_img, X_test, batch_size)\n    if self.method == \"binary\":\n        preds = np.vstack(preds_l).squeeze(1)\n        probs = np.zeros([preds.shape[0], 2])\n        probs[:, 0] = 1 - preds\n        probs[:, 1] = preds\n        return probs\n    if self.method == \"multiclass\":\n        return np.vstack(preds_l)\n    if self.method == \"multitarget\":\n        return np.vstack(preds_l)\n</code></pre>"},{"location":"pytorch-widedeep/trainer.html#pytorch_widedeep.training.Trainer.save","title":"save","text":"<pre><code>save(\n    path,\n    save_state_dict=False,\n    save_optimizer=False,\n    model_filename=\"wd_model.pt\",\n)\n</code></pre> <p>Saves the model, training and evaluation history, and the <code>feature_importance</code> attribute (if the <code>deeptabular</code> component is a Tabnet model) to disk</p> <p>The <code>Trainer</code> class is built so that it 'just' trains a model. With that in mind, all the torch related parameters (such as optimizers, learning rate schedulers, initializers, etc) have to be defined externally and then passed to the <code>Trainer</code>. As a result, the <code>Trainer</code> does not generate any attribute or additional data products that need to be saved other than the <code>model</code> object itself, which can be saved as any other torch model (e.g. <code>torch.save(model, path)</code>).</p> <p>The exception is Tabnet. If the <code>deeptabular</code> component is a Tabnet model, an attribute (a dict) called <code>feature_importance</code> will be created at the end of the training process. Therefore, a <code>save</code> method was created that will save the feature importance dictionary to a json file and, since we are here, the model weights, training history and learning rate history.</p> <p>Parameters:</p> Name Type Description Default <code>path</code> <code>str</code> <p>path to the directory where the model and the feature importance attribute will be saved.</p> required <code>save_state_dict</code> <code>bool</code> <p>Boolean indicating whether to save directly the model (and optimizer) or the model's (and optimizer's) state dictionary</p> <code>False</code> <code>save_optimizer</code> <code>bool</code> <p>Boolean indicating whether to save the optimizer</p> <code>False</code> <code>model_filename</code> <code>str</code> <p>filename where the model weights will be store</p> <code>'wd_model.pt'</code> Source code in <code>pytorch_widedeep/training/trainer.py</code> <pre><code>def save(\n    self,\n    path: str,\n    save_state_dict: bool = False,\n    save_optimizer: bool = False,\n    model_filename: str = \"wd_model.pt\",\n):\n    r\"\"\"Saves the model, training and evaluation history, and the\n    `feature_importance` attribute (if the `deeptabular` component is a\n    Tabnet model) to disk\n\n    The `Trainer` class is built so that it 'just' trains a model. With\n    that in mind, all the torch related parameters (such as optimizers,\n    learning rate schedulers, initializers, etc) have to be defined\n    externally and then passed to the `Trainer`. As a result, the\n    `Trainer` does not generate any attribute or additional data\n    products that need to be saved other than the `model` object itself,\n    which can be saved as any other torch model (e.g. `torch.save(model,\n    path)`).\n\n    The exception is Tabnet. If the `deeptabular` component is a Tabnet\n    model, an attribute (a dict) called `feature_importance` will be\n    created at the end of the training process. Therefore, a `save`\n    method was created that will save the feature importance dictionary\n    to a json file and, since we are here, the model weights, training\n    history and learning rate history.\n\n    Parameters\n    ----------\n    path: str\n        path to the directory where the model and the feature importance\n        attribute will be saved.\n    save_state_dict: bool, default = False\n        Boolean indicating whether to save directly the model\n        (and optimizer) or the model's (and optimizer's) state\n        dictionary\n    save_optimizer: bool, default = False\n        Boolean indicating whether to save the optimizer\n    model_filename: str, Optional, default = \"wd_model.pt\"\n        filename where the model weights will be store\n    \"\"\"\n\n    self._save_history(path)\n\n    self._save_model_and_optimizer(\n        path, save_state_dict, save_optimizer, model_filename\n    )\n\n    if self.model.is_tabnet:\n        with open(Path(path) / \"feature_importance.json\", \"w\") as fi:\n            json.dump(self.feature_importance, fi)\n</code></pre>"},{"location":"pytorch-widedeep/utils/index.html","title":"The <code>utils</code> module","text":"<p>These are a series of utilities that might be useful for a number of preprocessing tasks, even not directly related to <code>pytorch-widedeep</code>. All the classes and functions discussed here are available directly from the <code>utils</code> module. For example, the <code>LabelEncoder</code> within the <code>deeptabular_utils</code> submodule can be imported as:</p> <pre><code>from pytorch_widedeep.utils import LabelEncoder\n</code></pre> <p>These are classes and functions that are internally used in the library. We include them here in case the user finds them useful for other purposes.</p>"},{"location":"pytorch-widedeep/utils/deeptabular_utils.html","title":"deeptabular utils","text":""},{"location":"pytorch-widedeep/utils/deeptabular_utils.html#pytorch_widedeep.utils.deeptabular_utils.LabelEncoder","title":"LabelEncoder","text":"<p>Label Encode categorical values for multiple columns at once</p> <p> NOTE: LabelEncoder reserves 0 for <code>unseen</code> new categories. This is convenient when defining the embedding layers, since we can just set padding idx to 0.</p> <p>Parameters:</p> Name Type Description Default <code>columns_to_encode</code> <code>Optional[List[str]]</code> <p>List of strings containing the names of the columns to encode. If <code>None</code> all columns of type <code>object</code> in the dataframe will be label encoded.</p> <code>None</code> <code>with_attention</code> <code>bool</code> <p>Boolean indicating whether the preprocessed data will be passed to an attention-based model. Aliased as <code>for_transformer</code>.</p> <code>False</code> <code>shared_embed</code> <code>bool</code> <p>Boolean indicating if the embeddings will be \"shared\" when using attention-based models. The idea behind <code>shared_embed</code> is described in the Appendix A in the TabTransformer paper: 'The goal of having column embedding is to enable the model to distinguish the classes in one column from those in the other columns'. In other words, the idea is to let the model learn which column is embedded at the time. See: <code>pytorch_widedeep.models.transformers._layers.SharedEmbeddings</code>.</p> <code>False</code> <p>Attributes:</p> Name Type Description <code>encoding_dict</code> <code>Dict</code> <p>Dictionary containing the encoding mappings in the format, e.g. :  <code>{'colname1': {'cat1': 1, 'cat2': 2, ...}, 'colname2': {'cat1': 1, 'cat2': 2, ...}, ...}</code></p> <code>inverse_encoding_dict</code> <code>Dict</code> <p>Dictionary containing the inverse encoding mappings in the format, e.g. :  <code>{'colname1': {1: 'cat1', 2: 'cat2', ...}, 'colname2': {1: 'cat1', 2: 'cat2', ...}, ...}</code></p> Source code in <code>pytorch_widedeep/utils/deeptabular_utils.py</code> <pre><code>class LabelEncoder:\n    r\"\"\"Label Encode categorical values for multiple columns at once\n\n    :information_source: **NOTE**:\n    LabelEncoder reserves 0 for `unseen` new categories. This is convenient\n    when defining the embedding layers, since we can just set padding idx to 0.\n\n    Parameters\n    ----------\n    columns_to_encode: list, Optional, default = None\n        List of strings containing the names of the columns to encode. If\n        `None` all columns of type `object` in the dataframe will be label\n        encoded.\n    with_attention: bool, default = False\n        Boolean indicating whether the preprocessed data will be passed to an\n        attention-based model. Aliased as `for_transformer`.\n    shared_embed: bool, default = False\n        Boolean indicating if the embeddings will be \"_shared_\" when using\n        attention-based models. The idea behind `shared_embed` is described\n        in the Appendix A in the [TabTransformer paper](https://arxiv.org/abs/2012.06678):\n        '_The goal of having column embedding is to enable the model to\n        distinguish the classes in one column from those in the\n        other columns_'. In other words, the idea is to let the model learn\n        which column is embedded at the time. See: `pytorch_widedeep.models.transformers._layers.SharedEmbeddings`.\n\n    Attributes\n    ----------\n    encoding_dict : Dict\n        Dictionary containing the encoding mappings in the format, e.g. : &lt;br/&gt;\n        `{'colname1': {'cat1': 1, 'cat2': 2, ...}, 'colname2': {'cat1': 1, 'cat2': 2, ...}, ...}`\n    inverse_encoding_dict : Dict\n        Dictionary containing the inverse encoding mappings in the format, e.g. : &lt;br/&gt;\n        `{'colname1': {1: 'cat1', 2: 'cat2', ...}, 'colname2': {1: 'cat1', 2: 'cat2', ...}, ...}`\n\n    \"\"\"\n\n    @alias(\"with_attention\", [\"for_transformer\"])\n    def __init__(\n        self,\n        columns_to_encode: Optional[List[str]] = None,\n        with_attention: bool = False,\n        shared_embed: bool = False,\n    ):\n        self.columns_to_encode = columns_to_encode\n\n        self.shared_embed = shared_embed\n        self.with_attention = with_attention\n\n        self.reset_embed_idx = not self.with_attention or self.shared_embed\n\n    def partial_fit(self, df: pd.DataFrame) -&gt; \"LabelEncoder\":  # noqa: C901\n        \"\"\"Main method. Creates encoding attributes.\n\n        Returns\n        -------\n        LabelEncoder\n            `LabelEncoder` fitted object\n        \"\"\"\n        # here df is a chunk of the data. this is meant to be run when the\n        # data is large and we pass a chunk at a time. Therefore, we do not\n        # copy the input chunk as mutating a chunk is ok\n        if self.columns_to_encode is None:\n            self.columns_to_encode = list(df.select_dtypes(include=[\"object\"]).columns)\n        else:\n            # sanity check to make sure all categorical columns are in an adequate\n            # format\n            for col in self.columns_to_encode:\n                df[col] = df[col].astype(\"O\")\n\n        unique_column_vals: Dict[str, List[str]] = {}\n        for c in self.columns_to_encode:\n            unique_column_vals[c] = df[c].unique().tolist()\n\n        if not hasattr(self, \"encoding_dict\"):\n            # we run the method 'partial_fit' for the 1st time\n            self.encoding_dict: Dict[str, Dict[str, int]] = {}\n            if \"cls_token\" in unique_column_vals and self.shared_embed:\n                self.encoding_dict[\"cls_token\"] = {\"[CLS]\": 0}\n                del unique_column_vals[\"cls_token\"]\n\n            # leave 0 for padding/\"unseen\" categories. Also we need an\n            # attribute to keep track of the encoding in case we use\n            # attention and we do not re-start the index/counter\n            self.cum_idx: int = 1\n            for k, v in unique_column_vals.items():\n                self.encoding_dict[k] = {o: i + self.cum_idx for i, o in enumerate(v)}\n                self.cum_idx = 1 if self.reset_embed_idx else self.cum_idx + len(v)\n        else:\n            # the 'partial_fit' method has already run.\n            # \"cls_token\" will have been added already\n            if \"cls_token\" in unique_column_vals and self.shared_embed:\n                del unique_column_vals[\"cls_token\"]\n\n            # Classes in the new df/chunk of the dataset that have not been seen\n            # before\n            unseen_classes: Dict[str, List[str]] = {}\n            for c in self.columns_to_encode:\n                unseen_classes[c] = list(\n                    np.setdiff1d(\n                        unique_column_vals[c], list(self.encoding_dict[c].keys())\n                    )\n                )\n\n            # leave 0 for padding/\"unseen\" categories\n            for k, v in unique_column_vals.items():\n                # if we use attention we need to start encoding from the\n                # last 'overall' encoding index. Otherwise, we use the max\n                # encoding index per categorical col\n                _idx = (\n                    max(self.encoding_dict[k].values()) + 1\n                    if self.reset_embed_idx\n                    else self.cum_idx\n                )\n                if len(unseen_classes[k]) != 0:\n                    for i, o in enumerate(unseen_classes[k]):\n                        if o not in self.encoding_dict[k]:\n                            self.encoding_dict[k][o] = i + _idx\n                    # if self.reset_embed_idx is True it will be 1 anyway\n                    self.cum_idx = (\n                        1\n                        if self.reset_embed_idx\n                        else self.cum_idx + len(unseen_classes[k])\n                    )\n\n        return self\n\n    def fit(self, df: pd.DataFrame) -&gt; \"LabelEncoder\":\n        \"\"\"Simply runs the `partial_fit` method when the data fits in memory\n\n        Returns\n        -------\n        LabelEncoder\n            `LabelEncoder` fitted object\n        \"\"\"\n        # this is meant to be run when the data fits in memory and therefore,\n        # we do not want to mutate the original df, so we copy it\n        self.partial_fit(df.copy())\n\n        self.inverse_encoding_dict = self.create_inverse_encoding_dict()\n\n        return self\n\n    def transform(self, df: pd.DataFrame) -&gt; pd.DataFrame:\n        \"\"\"Label Encoded the categories in `columns_to_encode`\n\n        Returns\n        -------\n        pd.DataFrame\n            label-encoded dataframe\n        \"\"\"\n        try:\n            self.encoding_dict\n        except AttributeError:\n            raise NotFittedError(\n                \"This LabelEncoder instance is not fitted yet. \"\n                \"Call 'fit' with appropriate arguments before using this LabelEncoder.\"\n            )\n\n        df_inp = df.copy()\n        # sanity check to make sure all categorical columns are in an adequate\n        # format\n        for col in self.columns_to_encode:  # type: ignore\n            df_inp[col] = df_inp[col].astype(\"O\")\n\n        for k, v in self.encoding_dict.items():\n            df_inp[k] = df_inp[k].apply(lambda x: v[x] if x in v.keys() else 0)\n\n        return df_inp\n\n    def fit_transform(self, df: pd.DataFrame) -&gt; pd.DataFrame:\n        \"\"\"Combines `fit` and `transform`\n\n        Examples\n        --------\n\n        &gt;&gt;&gt; import pandas as pd\n        &gt;&gt;&gt; from pytorch_widedeep.utils import LabelEncoder\n        &gt;&gt;&gt; df = pd.DataFrame({'col1': [1,2,3], 'col2': ['me', 'you', 'him']})\n        &gt;&gt;&gt; columns_to_encode = ['col2']\n        &gt;&gt;&gt; encoder = LabelEncoder(columns_to_encode)\n        &gt;&gt;&gt; encoder.fit_transform(df)\n           col1  col2\n        0     1     1\n        1     2     2\n        2     3     3\n        &gt;&gt;&gt; encoder.encoding_dict\n        {'col2': {'me': 1, 'you': 2, 'him': 3}}\n\n        Returns\n        -------\n        pd.DataFrame\n            label-encoded dataframe\n        \"\"\"\n        return self.fit(df).transform(df)\n\n    def create_inverse_encoding_dict(self) -&gt; Dict[str, Dict[int, str]]:\n        inverse_encoding_dict = dict()\n        for c in self.encoding_dict:\n            inverse_encoding_dict[c] = {v: k for k, v in self.encoding_dict[c].items()}\n            inverse_encoding_dict[c][0] = \"unseen\"\n        return inverse_encoding_dict\n\n    def inverse_transform(self, df: pd.DataFrame) -&gt; pd.DataFrame:\n        \"\"\"Returns the original categories\n\n        Examples\n        --------\n\n        &gt;&gt;&gt; import pandas as pd\n        &gt;&gt;&gt; from pytorch_widedeep.utils import LabelEncoder\n        &gt;&gt;&gt; df = pd.DataFrame({'col1': [1,2,3], 'col2': ['me', 'you', 'him']})\n        &gt;&gt;&gt; columns_to_encode = ['col2']\n        &gt;&gt;&gt; encoder = LabelEncoder(columns_to_encode)\n        &gt;&gt;&gt; df_enc = encoder.fit_transform(df)\n        &gt;&gt;&gt; encoder.inverse_transform(df_enc)\n           col1 col2\n        0     1   me\n        1     2  you\n        2     3  him\n\n        Returns\n        -------\n        pd.DataFrame\n            DataFrame with original categories\n        \"\"\"\n\n        if not hasattr(self, \"inverse_encoding_dict\"):\n            self.inverse_encoding_dict = self.create_inverse_encoding_dict()\n\n        for k, v in self.inverse_encoding_dict.items():\n            df[k] = df[k].apply(lambda x: v[x])\n\n        return df\n\n    def __repr__(self) -&gt; str:\n        list_of_params: List[str] = []\n        if self.columns_to_encode is not None:\n            list_of_params.append(\"columns_to_encode={columns_to_encode}\")\n        if self.with_attention:\n            list_of_params.append(\"with_attention={with_attention}\")\n        if self.shared_embed:\n            list_of_params.append(\"shared_embed={shared_embed}\")\n        all_params = \", \".join(list_of_params)\n        return f\"LabelEncoder({all_params.format(**self.__dict__)})\"\n</code></pre>"},{"location":"pytorch-widedeep/utils/deeptabular_utils.html#pytorch_widedeep.utils.deeptabular_utils.LabelEncoder.partial_fit","title":"partial_fit","text":"<pre><code>partial_fit(df)\n</code></pre> <p>Main method. Creates encoding attributes.</p> <p>Returns:</p> Type Description <code>LabelEncoder</code> <p><code>LabelEncoder</code> fitted object</p> Source code in <code>pytorch_widedeep/utils/deeptabular_utils.py</code> <pre><code>def partial_fit(self, df: pd.DataFrame) -&gt; \"LabelEncoder\":  # noqa: C901\n    \"\"\"Main method. Creates encoding attributes.\n\n    Returns\n    -------\n    LabelEncoder\n        `LabelEncoder` fitted object\n    \"\"\"\n    # here df is a chunk of the data. this is meant to be run when the\n    # data is large and we pass a chunk at a time. Therefore, we do not\n    # copy the input chunk as mutating a chunk is ok\n    if self.columns_to_encode is None:\n        self.columns_to_encode = list(df.select_dtypes(include=[\"object\"]).columns)\n    else:\n        # sanity check to make sure all categorical columns are in an adequate\n        # format\n        for col in self.columns_to_encode:\n            df[col] = df[col].astype(\"O\")\n\n    unique_column_vals: Dict[str, List[str]] = {}\n    for c in self.columns_to_encode:\n        unique_column_vals[c] = df[c].unique().tolist()\n\n    if not hasattr(self, \"encoding_dict\"):\n        # we run the method 'partial_fit' for the 1st time\n        self.encoding_dict: Dict[str, Dict[str, int]] = {}\n        if \"cls_token\" in unique_column_vals and self.shared_embed:\n            self.encoding_dict[\"cls_token\"] = {\"[CLS]\": 0}\n            del unique_column_vals[\"cls_token\"]\n\n        # leave 0 for padding/\"unseen\" categories. Also we need an\n        # attribute to keep track of the encoding in case we use\n        # attention and we do not re-start the index/counter\n        self.cum_idx: int = 1\n        for k, v in unique_column_vals.items():\n            self.encoding_dict[k] = {o: i + self.cum_idx for i, o in enumerate(v)}\n            self.cum_idx = 1 if self.reset_embed_idx else self.cum_idx + len(v)\n    else:\n        # the 'partial_fit' method has already run.\n        # \"cls_token\" will have been added already\n        if \"cls_token\" in unique_column_vals and self.shared_embed:\n            del unique_column_vals[\"cls_token\"]\n\n        # Classes in the new df/chunk of the dataset that have not been seen\n        # before\n        unseen_classes: Dict[str, List[str]] = {}\n        for c in self.columns_to_encode:\n            unseen_classes[c] = list(\n                np.setdiff1d(\n                    unique_column_vals[c], list(self.encoding_dict[c].keys())\n                )\n            )\n\n        # leave 0 for padding/\"unseen\" categories\n        for k, v in unique_column_vals.items():\n            # if we use attention we need to start encoding from the\n            # last 'overall' encoding index. Otherwise, we use the max\n            # encoding index per categorical col\n            _idx = (\n                max(self.encoding_dict[k].values()) + 1\n                if self.reset_embed_idx\n                else self.cum_idx\n            )\n            if len(unseen_classes[k]) != 0:\n                for i, o in enumerate(unseen_classes[k]):\n                    if o not in self.encoding_dict[k]:\n                        self.encoding_dict[k][o] = i + _idx\n                # if self.reset_embed_idx is True it will be 1 anyway\n                self.cum_idx = (\n                    1\n                    if self.reset_embed_idx\n                    else self.cum_idx + len(unseen_classes[k])\n                )\n\n    return self\n</code></pre>"},{"location":"pytorch-widedeep/utils/deeptabular_utils.html#pytorch_widedeep.utils.deeptabular_utils.LabelEncoder.fit","title":"fit","text":"<pre><code>fit(df)\n</code></pre> <p>Simply runs the <code>partial_fit</code> method when the data fits in memory</p> <p>Returns:</p> Type Description <code>LabelEncoder</code> <p><code>LabelEncoder</code> fitted object</p> Source code in <code>pytorch_widedeep/utils/deeptabular_utils.py</code> <pre><code>def fit(self, df: pd.DataFrame) -&gt; \"LabelEncoder\":\n    \"\"\"Simply runs the `partial_fit` method when the data fits in memory\n\n    Returns\n    -------\n    LabelEncoder\n        `LabelEncoder` fitted object\n    \"\"\"\n    # this is meant to be run when the data fits in memory and therefore,\n    # we do not want to mutate the original df, so we copy it\n    self.partial_fit(df.copy())\n\n    self.inverse_encoding_dict = self.create_inverse_encoding_dict()\n\n    return self\n</code></pre>"},{"location":"pytorch-widedeep/utils/deeptabular_utils.html#pytorch_widedeep.utils.deeptabular_utils.LabelEncoder.transform","title":"transform","text":"<pre><code>transform(df)\n</code></pre> <p>Label Encoded the categories in <code>columns_to_encode</code></p> <p>Returns:</p> Type Description <code>DataFrame</code> <p>label-encoded dataframe</p> Source code in <code>pytorch_widedeep/utils/deeptabular_utils.py</code> <pre><code>def transform(self, df: pd.DataFrame) -&gt; pd.DataFrame:\n    \"\"\"Label Encoded the categories in `columns_to_encode`\n\n    Returns\n    -------\n    pd.DataFrame\n        label-encoded dataframe\n    \"\"\"\n    try:\n        self.encoding_dict\n    except AttributeError:\n        raise NotFittedError(\n            \"This LabelEncoder instance is not fitted yet. \"\n            \"Call 'fit' with appropriate arguments before using this LabelEncoder.\"\n        )\n\n    df_inp = df.copy()\n    # sanity check to make sure all categorical columns are in an adequate\n    # format\n    for col in self.columns_to_encode:  # type: ignore\n        df_inp[col] = df_inp[col].astype(\"O\")\n\n    for k, v in self.encoding_dict.items():\n        df_inp[k] = df_inp[k].apply(lambda x: v[x] if x in v.keys() else 0)\n\n    return df_inp\n</code></pre>"},{"location":"pytorch-widedeep/utils/deeptabular_utils.html#pytorch_widedeep.utils.deeptabular_utils.LabelEncoder.fit_transform","title":"fit_transform","text":"<pre><code>fit_transform(df)\n</code></pre> <p>Combines <code>fit</code> and <code>transform</code></p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import pandas as pd\n&gt;&gt;&gt; from pytorch_widedeep.utils import LabelEncoder\n&gt;&gt;&gt; df = pd.DataFrame({'col1': [1,2,3], 'col2': ['me', 'you', 'him']})\n&gt;&gt;&gt; columns_to_encode = ['col2']\n&gt;&gt;&gt; encoder = LabelEncoder(columns_to_encode)\n&gt;&gt;&gt; encoder.fit_transform(df)\n   col1  col2\n0     1     1\n1     2     2\n2     3     3\n&gt;&gt;&gt; encoder.encoding_dict\n{'col2': {'me': 1, 'you': 2, 'him': 3}}\n</code></pre> <p>Returns:</p> Type Description <code>DataFrame</code> <p>label-encoded dataframe</p> Source code in <code>pytorch_widedeep/utils/deeptabular_utils.py</code> <pre><code>def fit_transform(self, df: pd.DataFrame) -&gt; pd.DataFrame:\n    \"\"\"Combines `fit` and `transform`\n\n    Examples\n    --------\n\n    &gt;&gt;&gt; import pandas as pd\n    &gt;&gt;&gt; from pytorch_widedeep.utils import LabelEncoder\n    &gt;&gt;&gt; df = pd.DataFrame({'col1': [1,2,3], 'col2': ['me', 'you', 'him']})\n    &gt;&gt;&gt; columns_to_encode = ['col2']\n    &gt;&gt;&gt; encoder = LabelEncoder(columns_to_encode)\n    &gt;&gt;&gt; encoder.fit_transform(df)\n       col1  col2\n    0     1     1\n    1     2     2\n    2     3     3\n    &gt;&gt;&gt; encoder.encoding_dict\n    {'col2': {'me': 1, 'you': 2, 'him': 3}}\n\n    Returns\n    -------\n    pd.DataFrame\n        label-encoded dataframe\n    \"\"\"\n    return self.fit(df).transform(df)\n</code></pre>"},{"location":"pytorch-widedeep/utils/deeptabular_utils.html#pytorch_widedeep.utils.deeptabular_utils.LabelEncoder.inverse_transform","title":"inverse_transform","text":"<pre><code>inverse_transform(df)\n</code></pre> <p>Returns the original categories</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; import pandas as pd\n&gt;&gt;&gt; from pytorch_widedeep.utils import LabelEncoder\n&gt;&gt;&gt; df = pd.DataFrame({'col1': [1,2,3], 'col2': ['me', 'you', 'him']})\n&gt;&gt;&gt; columns_to_encode = ['col2']\n&gt;&gt;&gt; encoder = LabelEncoder(columns_to_encode)\n&gt;&gt;&gt; df_enc = encoder.fit_transform(df)\n&gt;&gt;&gt; encoder.inverse_transform(df_enc)\n   col1 col2\n0     1   me\n1     2  you\n2     3  him\n</code></pre> <p>Returns:</p> Type Description <code>DataFrame</code> <p>DataFrame with original categories</p> Source code in <code>pytorch_widedeep/utils/deeptabular_utils.py</code> <pre><code>def inverse_transform(self, df: pd.DataFrame) -&gt; pd.DataFrame:\n    \"\"\"Returns the original categories\n\n    Examples\n    --------\n\n    &gt;&gt;&gt; import pandas as pd\n    &gt;&gt;&gt; from pytorch_widedeep.utils import LabelEncoder\n    &gt;&gt;&gt; df = pd.DataFrame({'col1': [1,2,3], 'col2': ['me', 'you', 'him']})\n    &gt;&gt;&gt; columns_to_encode = ['col2']\n    &gt;&gt;&gt; encoder = LabelEncoder(columns_to_encode)\n    &gt;&gt;&gt; df_enc = encoder.fit_transform(df)\n    &gt;&gt;&gt; encoder.inverse_transform(df_enc)\n       col1 col2\n    0     1   me\n    1     2  you\n    2     3  him\n\n    Returns\n    -------\n    pd.DataFrame\n        DataFrame with original categories\n    \"\"\"\n\n    if not hasattr(self, \"inverse_encoding_dict\"):\n        self.inverse_encoding_dict = self.create_inverse_encoding_dict()\n\n    for k, v in self.inverse_encoding_dict.items():\n        df[k] = df[k].apply(lambda x: v[x])\n\n    return df\n</code></pre>"},{"location":"pytorch-widedeep/utils/fastai_transforms.html","title":"Fastai transforms","text":"<p>We directly copied and pasted part of the <code>transforms.py</code> module from the <code>fastai</code> library (from an old version). The reason to do such a thing is because <code>pytorch_widedeep</code> only needs the <code>Tokenizer</code> and the <code>Vocab</code> classes there. This way we avoid extra dependencies. Credit for all the code in the <code>fastai_transforms</code> module in this <code>pytorch-widedeep</code> package goes to Jeremy Howard and the <code>fastai</code> team. I only include the documentation here for completion, but I strongly advise the user to read the <code>fastai</code> documentation.</p>"},{"location":"pytorch-widedeep/utils/fastai_transforms.html#pytorch_widedeep.utils.fastai_transforms.Tokenizer","title":"Tokenizer","text":"<p>Class to combine a series of rules and a tokenizer function to tokenize text with multiprocessing.</p> <p>Setting some of the parameters of this class require perhaps some familiarity with the source code.</p> <p>Parameters:</p> Name Type Description Default <code>tok_func</code> <code>Callable</code> <p>Tokenizer Object. See <code>pytorch_widedeep.utils.fastai_transforms.SpacyTokenizer</code></p> <code>SpacyTokenizer</code> <code>lang</code> <code>str</code> <p>Text's Language</p> <code>'en'</code> <code>pre_rules</code> <code>Optional[ListRules]</code> <p>Custom type: <code>Collection[Callable[[str], str]]</code>. These are <code>Callable</code> objects that will be applied to the text (str) directly as <code>rule(tok)</code> before being tokenized.</p> <code>None</code> <code>post_rules</code> <code>Optional[ListRules]</code> <p>Custom type: <code>Collection[Callable[[str], str]]</code>. These are <code>Callable</code> objects that will be applied to the tokens as <code>rule(tokens)</code> after the text has been tokenized.</p> <code>None</code> <code>special_cases</code> <code>Optional[Collection[str]]</code> <p>special cases to be added to the tokenizer via <code>Spacy</code>'s <code>add_special_case</code> method</p> <code>None</code> <code>n_cpus</code> <code>Optional[int]</code> <p>number of CPUs to used during the tokenization process</p> <code>None</code> Source code in <code>pytorch_widedeep/utils/fastai_transforms.py</code> <pre><code>class Tokenizer:\n    r\"\"\"Class to combine a series of rules and a tokenizer function to tokenize\n    text with multiprocessing.\n\n    Setting some of the parameters of this class require perhaps some\n    familiarity with the source code.\n\n    Parameters\n    ----------\n    tok_func: Callable, default = ``SpacyTokenizer``\n        Tokenizer Object. See `pytorch_widedeep.utils.fastai_transforms.SpacyTokenizer`\n    lang: str, default = \"en\"\n        Text's Language\n    pre_rules: ListRules, Optional, default = None\n        Custom type: ``Collection[Callable[[str], str]]``. These are\n        `Callable` objects that will be applied to the text (str) directly as\n        `rule(tok)` before being tokenized.\n    post_rules: ListRules, Optional, default = None\n        Custom type: ``Collection[Callable[[str], str]]``. These are\n        `Callable` objects that will be applied to the tokens as\n        `rule(tokens)` after the text has been tokenized.\n    special_cases: Collection, Optional, default= None\n        special cases to be added to the tokenizer via ``Spacy``'s\n        ``add_special_case`` method\n    n_cpus: int, Optional, default = None\n        number of CPUs to used during the tokenization process\n    \"\"\"\n\n    def __init__(\n        self,\n        tok_func: Callable = SpacyTokenizer,\n        lang: str = \"en\",\n        pre_rules: Optional[ListRules] = None,\n        post_rules: Optional[ListRules] = None,\n        special_cases: Optional[Collection[str]] = None,\n        n_cpus: Optional[int] = None,\n    ):\n        self.tok_func, self.lang, self.special_cases = tok_func, lang, special_cases\n        self.pre_rules = ifnone(pre_rules, defaults.text_pre_rules)\n        self.post_rules = ifnone(post_rules, defaults.text_post_rules)\n        self.special_cases = (\n            special_cases if special_cases is not None else defaults.text_spec_tok\n        )\n        self.n_cpus = ifnone(n_cpus, defaults.cpus)\n\n    def __repr__(self) -&gt; str:\n        res = f\"Tokenizer {self.tok_func.__name__} in {self.lang} with the following rules:\\n\"\n        for rule in self.pre_rules:\n            res += f\" - {rule.__name__}\\n\"\n        for rule in self.post_rules:\n            res += f\" - {rule.__name__}\\n\"\n        return res\n\n    def process_text(self, t: str, tok: BaseTokenizer) -&gt; List[str]:\n        r\"\"\"Process and tokenize one text ``t`` with tokenizer ``tok``.\n\n        Parameters\n        ----------\n        t: str\n            text to be processed and tokenized\n        tok: ``BaseTokenizer``\n            Instance of `BaseTokenizer`. See\n            `pytorch_widedeep.utils.fastai_transforms.BaseTokenizer`\n\n        Returns\n        -------\n        List[str]\n            List of tokens\n        \"\"\"\n        for rule in self.pre_rules:\n            t = rule(t)\n        toks = tok.tokenizer(t)\n        for rule in self.post_rules:\n            toks = rule(toks)\n        return toks\n\n    def _process_all_1(self, texts: Collection[str]) -&gt; List[List[str]]:\n        \"\"\"Process a list of ``texts`` in one process.\"\"\"\n\n        tok = self.tok_func(self.lang)\n        if self.special_cases:\n            tok.add_special_cases(self.special_cases)\n        return [self.process_text(str(t), tok) for t in texts]\n\n    def process_all(self, texts: Collection[str]) -&gt; List[List[str]]:\n        r\"\"\"Process a list of texts. Parallel execution of ``process_text``.\n\n        Examples\n        --------\n        &gt;&gt;&gt; from pytorch_widedeep.utils import Tokenizer\n        &gt;&gt;&gt; texts = ['Machine learning is great', 'but building stuff is even better']\n        &gt;&gt;&gt; tok = Tokenizer()\n        &gt;&gt;&gt; tok.process_all(texts)\n        [['xxmaj', 'machine', 'learning', 'is', 'great'], ['but', 'building', 'stuff', 'is', 'even', 'better']]\n\n        :information_source: **NOTE**:\n        Note the token ``TK_MAJ`` (`xxmaj`), used to indicate the\n        next word begins with a capital in the original text. For more\n        details of special tokens please see the [``fastai`` docs](https://docs.fast.ai/text.core.html#Tokenizing).\n\n        Returns\n        -------\n        List[List[str]]\n            List containing lists of tokens. One list per \"_document_\"\n\n        \"\"\"\n\n        if self.n_cpus &lt;= 1:\n            return self._process_all_1(texts)\n\n        with Pool(self.n_cpus) as p:\n            partitioned_texts = partition_by_cores(texts, self.n_cpus)\n            results = p.map(self._process_all_1, partitioned_texts)\n            res = sum(results, [])\n        return res\n</code></pre>"},{"location":"pytorch-widedeep/utils/fastai_transforms.html#pytorch_widedeep.utils.fastai_transforms.Tokenizer.process_text","title":"process_text","text":"<pre><code>process_text(t, tok)\n</code></pre> <p>Process and tokenize one text <code>t</code> with tokenizer <code>tok</code>.</p> <p>Parameters:</p> Name Type Description Default <code>t</code> <code>str</code> <p>text to be processed and tokenized</p> required <code>tok</code> <code>BaseTokenizer</code> <p>Instance of <code>BaseTokenizer</code>. See <code>pytorch_widedeep.utils.fastai_transforms.BaseTokenizer</code></p> required <p>Returns:</p> Type Description <code>List[str]</code> <p>List of tokens</p> Source code in <code>pytorch_widedeep/utils/fastai_transforms.py</code> <pre><code>def process_text(self, t: str, tok: BaseTokenizer) -&gt; List[str]:\n    r\"\"\"Process and tokenize one text ``t`` with tokenizer ``tok``.\n\n    Parameters\n    ----------\n    t: str\n        text to be processed and tokenized\n    tok: ``BaseTokenizer``\n        Instance of `BaseTokenizer`. See\n        `pytorch_widedeep.utils.fastai_transforms.BaseTokenizer`\n\n    Returns\n    -------\n    List[str]\n        List of tokens\n    \"\"\"\n    for rule in self.pre_rules:\n        t = rule(t)\n    toks = tok.tokenizer(t)\n    for rule in self.post_rules:\n        toks = rule(toks)\n    return toks\n</code></pre>"},{"location":"pytorch-widedeep/utils/fastai_transforms.html#pytorch_widedeep.utils.fastai_transforms.Tokenizer.process_all","title":"process_all","text":"<pre><code>process_all(texts)\n</code></pre> <p>Process a list of texts. Parallel execution of <code>process_text</code>.</p> <p>Examples:</p> <pre><code>&gt;&gt;&gt; from pytorch_widedeep.utils import Tokenizer\n&gt;&gt;&gt; texts = ['Machine learning is great', 'but building stuff is even better']\n&gt;&gt;&gt; tok = Tokenizer()\n&gt;&gt;&gt; tok.process_all(texts)\n[['xxmaj', 'machine', 'learning', 'is', 'great'], ['but', 'building', 'stuff', 'is', 'even', 'better']]\n</code></pre> <p> NOTE: Note the token <code>TK_MAJ</code> (<code>xxmaj</code>), used to indicate the next word begins with a capital in the original text. For more details of special tokens please see the <code>fastai</code> docs.</p> <p>Returns:</p> Type Description <code>List[List[str]]</code> <p>List containing lists of tokens. One list per \"document\"</p> Source code in <code>pytorch_widedeep/utils/fastai_transforms.py</code> <pre><code>def process_all(self, texts: Collection[str]) -&gt; List[List[str]]:\n    r\"\"\"Process a list of texts. Parallel execution of ``process_text``.\n\n    Examples\n    --------\n    &gt;&gt;&gt; from pytorch_widedeep.utils import Tokenizer\n    &gt;&gt;&gt; texts = ['Machine learning is great', 'but building stuff is even better']\n    &gt;&gt;&gt; tok = Tokenizer()\n    &gt;&gt;&gt; tok.process_all(texts)\n    [['xxmaj', 'machine', 'learning', 'is', 'great'], ['but', 'building', 'stuff', 'is', 'even', 'better']]\n\n    :information_source: **NOTE**:\n    Note the token ``TK_MAJ`` (`xxmaj`), used to indicate the\n    next word begins with a capital in the original text. For more\n    details of special tokens please see the [``fastai`` docs](https://docs.fast.ai/text.core.html#Tokenizing).\n\n    Returns\n    -------\n    List[List[str]]\n        List containing lists of tokens. One list per \"_document_\"\n\n    \"\"\"\n\n    if self.n_cpus &lt;= 1:\n        return self._process_all_1(texts)\n\n    with Pool(self.n_cpus) as p:\n        partitioned_texts = partition_by_cores(texts, self.n_cpus)\n        results = p.map(self._process_all_1, partitioned_texts)\n        res = sum(results, [])\n    return res\n</code></pre>"},{"location":"pytorch-widedeep/utils/fastai_transforms.html#pytorch_widedeep.utils.fastai_transforms.Vocab","title":"Vocab","text":"<p>Contains the correspondence between numbers and tokens.</p> <p>Parameters:</p> Name Type Description Default <code>max_vocab</code> <code>int</code> <p>maximum vocabulary size</p> required <code>min_freq</code> <code>int</code> <p>minimum frequency for a token to be considereds</p> required <code>pad_idx</code> <code>Optional[int]</code> <p>padding index. If <code>None</code>, Fastai's Tokenizer leaves the 0 index for the unknown token ('xxunk') and defaults to 1 for the padding token ('xxpad').</p> <code>None</code> <p>Attributes:</p> Name Type Description <code>itos</code> <code>Collection</code> <p><code>index to str</code>. Collection of strings that are the tokens of the vocabulary</p> <code>stoi</code> <code>defaultdict</code> <p><code>str to index</code>. Dictionary containing the tokens of the vocabulary and their corresponding index</p> Source code in <code>pytorch_widedeep/utils/fastai_transforms.py</code> <pre><code>class Vocab:\n    r\"\"\"Contains the correspondence between numbers and tokens.\n\n    Parameters\n    ----------\n    max_vocab: int\n        maximum vocabulary size\n    min_freq: int\n        minimum frequency for a token to be considereds\n    pad_idx: int, Optional, default = None\n        padding index. If `None`, Fastai's Tokenizer leaves the 0 index\n        for the unknown token (_'xxunk'_) and defaults to 1 for the padding\n        token (_'xxpad'_).\n\n    Attributes\n    ----------\n    itos: Collection\n        `index to str`. Collection of strings that are the tokens of the\n        vocabulary\n    stoi: defaultdict\n        `str to index`. Dictionary containing the tokens of the vocabulary and\n        their corresponding index\n    \"\"\"\n\n    def __init__(\n        self,\n        max_vocab: int,\n        min_freq: int,\n        pad_idx: Optional[int] = None,\n        special_cases: Optional[Collection[str]] = None,\n    ):\n        self.max_vocab = max_vocab\n        self.min_freq = min_freq\n        self.pad_idx = pad_idx\n        self.special_cases = (\n            special_cases if special_cases is not None else defaults.text_spec_tok\n        )\n\n    def create(\n        self,\n        tokens: Tokens,\n    ) -&gt; \"Vocab\":\n        r\"\"\"Create a vocabulary object from a set of tokens.\n\n        Parameters\n        ----------\n        tokens: Tokens\n            Custom type: ``Collection[Collection[str]]``  see\n            `pytorch_widedeep.wdtypes`. Collection of collection of\n            strings (e.g. list of tokenized sentences)\n\n        Examples\n        --------\n        &gt;&gt;&gt; from pytorch_widedeep.utils import Tokenizer, Vocab\n        &gt;&gt;&gt; texts = ['Machine learning is great', 'but building stuff is even better']\n        &gt;&gt;&gt; tokens = Tokenizer().process_all(texts)\n        &gt;&gt;&gt; vocab = Vocab(max_vocab=18, min_freq=1).create(tokens)\n        &gt;&gt;&gt; vocab.numericalize(['machine', 'learning', 'is', 'great'])\n        [10, 11, 9, 12]\n        &gt;&gt;&gt; vocab.textify([10, 11, 9, 12])\n        'machine learning is great'\n\n        :information_source: **NOTE**:\n        Note the many special tokens that ``fastai``'s' tokenizer adds. These\n        are particularly useful when building Language models and/or in\n        classification/Regression tasks. Please see the [``fastai`` docs](https://docs.fast.ai/text.core.html#Tokenizing).\n\n        Returns\n        -------\n        Vocab\n            An instance of a `Vocab` object\n        \"\"\"\n\n        freq = Counter(p for o in tokens for p in o)\n        itos = [o for o, c in freq.most_common(self.max_vocab) if c &gt;= self.min_freq]\n        for o in reversed(self.special_cases):  # type: ignore[arg-type]\n            if o in itos:\n                itos.remove(o)\n            itos.insert(0, o)\n\n        if self.pad_idx is not None and self.pad_idx != 1:\n            itos.remove(PAD)\n            itos.insert(self.pad_idx, PAD)\n            # get the new 'xxunk' index\n            xxunk_idx = np.where([el == \"xxunk\" for el in itos])[0][0]\n        else:\n            xxunk_idx = 0\n\n        itos = itos[: self.max_vocab]\n        if (\n            len(itos) &lt; self.max_vocab\n        ):  # Make sure vocab size is a multiple of 8 for fast mixed precision training\n            while len(itos) % 8 != 0:\n                itos.append(\"xxfake\")\n\n        self.itos = itos\n        self.stoi = defaultdict(\n            lambda: xxunk_idx, {v: k for k, v in enumerate(self.itos)}\n        )\n\n        return self\n\n    def fit(\n        self,\n        tokens: Tokens,\n    ) -&gt; \"Vocab\":\n        \"\"\"\n        Calls the `create` method. I simply want to honor fast ai naming, but\n        for consistency with the rest of the library I am including a fit method\n        \"\"\"\n        return self.create(tokens)\n\n    def numericalize(self, t: Collection[str]) -&gt; List[int]:\n        \"\"\"Convert a list of tokens ``t`` to their ids.\n\n        Returns\n        -------\n        List[int]\n            List of '_numericalsed_' tokens\n        \"\"\"\n        return [self.stoi[w] for w in t]\n\n    def transform(self, t: Collection[str]) -&gt; List[int]:\n        \"\"\"\n        Calls the `numericalize` method. I simply want to honor fast ai naming,\n        but for consistency with the rest of the library I am including a\n        transform method\n        \"\"\"\n        return self.numericalize(t)\n\n    def textify(self, nums: Collection[int], sep=\" \") -&gt; Union[str, List[str]]:\n        \"\"\"Convert a list of ``nums`` (or indexes) to their tokens.\n\n        Returns\n        -------\n        List[str]\n            List of tokens\n        \"\"\"\n        return (\n            sep.join([self.itos[i] for i in nums])\n            if sep is not None\n            else [self.itos[i] for i in nums]\n        )\n\n    def inverse_transform(\n        self, nums: Collection[int], sep=\" \"\n    ) -&gt; Union[str, List[str]]:\n        \"\"\"\n        Calls the `textify` method. I simply want to honor fast ai naming, but\n        for consistency with the rest of the library I am including an\n        inverse_transform method\n        \"\"\"\n        # I simply want to honor fast ai naming, but for consistency with the\n        # rest of the library I am including an inverse_transform method\n        return self.textify(nums, sep)\n\n    def __getstate__(self):\n        return {\"itos\": self.itos}\n\n    def __setstate__(self, state: dict):\n        self.itos = state[\"itos\"]\n        self.stoi = defaultdict(int, {v: k for k, v in enumerate(self.itos)})\n</code></pre>"},{"location":"pytorch-widedeep/utils/fastai_transforms.html#pytorch_widedeep.utils.fastai_transforms.Vocab.create","title":"create","text":"<pre><code>create(tokens)\n</code></pre> <p>Create a vocabulary object from a set of tokens.</p> <p>Parameters:</p> Name Type Description Default <code>tokens</code> <code>Tokens</code> <p>Custom type: <code>Collection[Collection[str]]</code>  see <code>pytorch_widedeep.wdtypes</code>. Collection of collection of strings (e.g. list of tokenized sentences)</p> required <p>Examples:</p> <pre><code>&gt;&gt;&gt; from pytorch_widedeep.utils import Tokenizer, Vocab\n&gt;&gt;&gt; texts = ['Machine learning is great', 'but building stuff is even better']\n&gt;&gt;&gt; tokens = Tokenizer().process_all(texts)\n&gt;&gt;&gt; vocab = Vocab(max_vocab=18, min_freq=1).create(tokens)\n&gt;&gt;&gt; vocab.numericalize(['machine', 'learning', 'is', 'great'])\n[10, 11, 9, 12]\n&gt;&gt;&gt; vocab.textify([10, 11, 9, 12])\n'machine learning is great'\n</code></pre> <p> NOTE: Note the many special tokens that <code>fastai</code>'s' tokenizer adds. These are particularly useful when building Language models and/or in classification/Regression tasks. Please see the <code>fastai</code> docs.</p> <p>Returns:</p> Type Description <code>Vocab</code> <p>An instance of a <code>Vocab</code> object</p> Source code in <code>pytorch_widedeep/utils/fastai_transforms.py</code> <pre><code>def create(\n    self,\n    tokens: Tokens,\n) -&gt; \"Vocab\":\n    r\"\"\"Create a vocabulary object from a set of tokens.\n\n    Parameters\n    ----------\n    tokens: Tokens\n        Custom type: ``Collection[Collection[str]]``  see\n        `pytorch_widedeep.wdtypes`. Collection of collection of\n        strings (e.g. list of tokenized sentences)\n\n    Examples\n    --------\n    &gt;&gt;&gt; from pytorch_widedeep.utils import Tokenizer, Vocab\n    &gt;&gt;&gt; texts = ['Machine learning is great', 'but building stuff is even better']\n    &gt;&gt;&gt; tokens = Tokenizer().process_all(texts)\n    &gt;&gt;&gt; vocab = Vocab(max_vocab=18, min_freq=1).create(tokens)\n    &gt;&gt;&gt; vocab.numericalize(['machine', 'learning', 'is', 'great'])\n    [10, 11, 9, 12]\n    &gt;&gt;&gt; vocab.textify([10, 11, 9, 12])\n    'machine learning is great'\n\n    :information_source: **NOTE**:\n    Note the many special tokens that ``fastai``'s' tokenizer adds. These\n    are particularly useful when building Language models and/or in\n    classification/Regression tasks. Please see the [``fastai`` docs](https://docs.fast.ai/text.core.html#Tokenizing).\n\n    Returns\n    -------\n    Vocab\n        An instance of a `Vocab` object\n    \"\"\"\n\n    freq = Counter(p for o in tokens for p in o)\n    itos = [o for o, c in freq.most_common(self.max_vocab) if c &gt;= self.min_freq]\n    for o in reversed(self.special_cases):  # type: ignore[arg-type]\n        if o in itos:\n            itos.remove(o)\n        itos.insert(0, o)\n\n    if self.pad_idx is not None and self.pad_idx != 1:\n        itos.remove(PAD)\n        itos.insert(self.pad_idx, PAD)\n        # get the new 'xxunk' index\n        xxunk_idx = np.where([el == \"xxunk\" for el in itos])[0][0]\n    else:\n        xxunk_idx = 0\n\n    itos = itos[: self.max_vocab]\n    if (\n        len(itos) &lt; self.max_vocab\n    ):  # Make sure vocab size is a multiple of 8 for fast mixed precision training\n        while len(itos) % 8 != 0:\n            itos.append(\"xxfake\")\n\n    self.itos = itos\n    self.stoi = defaultdict(\n        lambda: xxunk_idx, {v: k for k, v in enumerate(self.itos)}\n    )\n\n    return self\n</code></pre>"},{"location":"pytorch-widedeep/utils/fastai_transforms.html#pytorch_widedeep.utils.fastai_transforms.Vocab.fit","title":"fit","text":"<pre><code>fit(tokens)\n</code></pre> <p>Calls the <code>create</code> method. I simply want to honor fast ai naming, but for consistency with the rest of the library I am including a fit method</p> Source code in <code>pytorch_widedeep/utils/fastai_transforms.py</code> <pre><code>def fit(\n    self,\n    tokens: Tokens,\n) -&gt; \"Vocab\":\n    \"\"\"\n    Calls the `create` method. I simply want to honor fast ai naming, but\n    for consistency with the rest of the library I am including a fit method\n    \"\"\"\n    return self.create(tokens)\n</code></pre>"},{"location":"pytorch-widedeep/utils/fastai_transforms.html#pytorch_widedeep.utils.fastai_transforms.Vocab.numericalize","title":"numericalize","text":"<pre><code>numericalize(t)\n</code></pre> <p>Convert a list of tokens <code>t</code> to their ids.</p> <p>Returns:</p> Type Description <code>List[int]</code> <p>List of 'numericalsed' tokens</p> Source code in <code>pytorch_widedeep/utils/fastai_transforms.py</code> <pre><code>def numericalize(self, t: Collection[str]) -&gt; List[int]:\n    \"\"\"Convert a list of tokens ``t`` to their ids.\n\n    Returns\n    -------\n    List[int]\n        List of '_numericalsed_' tokens\n    \"\"\"\n    return [self.stoi[w] for w in t]\n</code></pre>"},{"location":"pytorch-widedeep/utils/fastai_transforms.html#pytorch_widedeep.utils.fastai_transforms.Vocab.transform","title":"transform","text":"<pre><code>transform(t)\n</code></pre> <p>Calls the <code>numericalize</code> method. I simply want to honor fast ai naming, but for consistency with the rest of the library I am including a transform method</p> Source code in <code>pytorch_widedeep/utils/fastai_transforms.py</code> <pre><code>def transform(self, t: Collection[str]) -&gt; List[int]:\n    \"\"\"\n    Calls the `numericalize` method. I simply want to honor fast ai naming,\n    but for consistency with the rest of the library I am including a\n    transform method\n    \"\"\"\n    return self.numericalize(t)\n</code></pre>"},{"location":"pytorch-widedeep/utils/fastai_transforms.html#pytorch_widedeep.utils.fastai_transforms.Vocab.textify","title":"textify","text":"<pre><code>textify(nums, sep=' ')\n</code></pre> <p>Convert a list of <code>nums</code> (or indexes) to their tokens.</p> <p>Returns:</p> Type Description <code>List[str]</code> <p>List of tokens</p> Source code in <code>pytorch_widedeep/utils/fastai_transforms.py</code> <pre><code>def textify(self, nums: Collection[int], sep=\" \") -&gt; Union[str, List[str]]:\n    \"\"\"Convert a list of ``nums`` (or indexes) to their tokens.\n\n    Returns\n    -------\n    List[str]\n        List of tokens\n    \"\"\"\n    return (\n        sep.join([self.itos[i] for i in nums])\n        if sep is not None\n        else [self.itos[i] for i in nums]\n    )\n</code></pre>"},{"location":"pytorch-widedeep/utils/fastai_transforms.html#pytorch_widedeep.utils.fastai_transforms.Vocab.inverse_transform","title":"inverse_transform","text":"<pre><code>inverse_transform(nums, sep=' ')\n</code></pre> <p>Calls the <code>textify</code> method. I simply want to honor fast ai naming, but for consistency with the rest of the library I am including an inverse_transform method</p> Source code in <code>pytorch_widedeep/utils/fastai_transforms.py</code> <pre><code>def inverse_transform(\n    self, nums: Collection[int], sep=\" \"\n) -&gt; Union[str, List[str]]:\n    \"\"\"\n    Calls the `textify` method. I simply want to honor fast ai naming, but\n    for consistency with the rest of the library I am including an\n    inverse_transform method\n    \"\"\"\n    # I simply want to honor fast ai naming, but for consistency with the\n    # rest of the library I am including an inverse_transform method\n    return self.textify(nums, sep)\n</code></pre>"},{"location":"pytorch-widedeep/utils/image_utils.html","title":"Image utils","text":"<p>SimplePreprocessor and AspectAwarePreprocessor are directly taked from the great series of Books `Deep Learning for Computer Vision by Adrian. Therefore, all credit for the code in the <code>image_utils</code> module goes to Adrian Rosebrock.</p>"},{"location":"pytorch-widedeep/utils/image_utils.html#pytorch_widedeep.utils.image_utils.AspectAwarePreprocessor","title":"AspectAwarePreprocessor","text":"<p>Class to resize an image to a certain width and height taking into account the image aspect ratio</p> <p>Parameters:</p> Name Type Description Default <code>width</code> <code>int</code> <p>output width</p> required <code>height</code> <code>int</code> <p>output height</p> required <code>inter</code> <p><code>opencv</code> interpolation method. See <code>opencv</code> <code>InterpolationFlags</code>.</p> <code>INTER_AREA</code> Source code in <code>pytorch_widedeep/utils/image_utils.py</code> <pre><code>class AspectAwarePreprocessor:\n    r\"\"\"Class to resize an image to a certain width and height taking into account\n    the image aspect ratio\n\n    Parameters\n    ----------\n    width: int\n        output width\n    height: int\n        output height\n    inter: interpolation method,  default = ``cv2.INTER_AREA``\n        ``opencv`` interpolation method. See ``opencv``\n        `InterpolationFlags`.\n    \"\"\"\n\n    def __init__(self, width: int, height: int, inter=cv2.INTER_AREA):\n        self.width = width\n        self.height = height\n        self.inter = inter\n\n    def preprocess(self, image: np.ndarray) -&gt; np.ndarray:\n        r\"\"\"Returns the resized input image taking into account the image aspect ratio\n\n        Parameters\n        ----------\n        image: np.ndarray\n            Input image to be resized\n\n        Examples\n        --------\n        &gt;&gt;&gt; import cv2\n        &gt;&gt;&gt; from pytorch_widedeep.utils import AspectAwarePreprocessor\n        &gt;&gt;&gt; img = cv2.imread(\"tests/test_data_utils/images/galaxy1.png\")\n        &gt;&gt;&gt; img.shape\n        (694, 890, 3)\n        &gt;&gt;&gt; app = AspectAwarePreprocessor(width=224, height=224)\n        &gt;&gt;&gt; resized_img = app.preprocess(img)\n        &gt;&gt;&gt; resized_img.shape\n        (224, 224, 3)\n\n        Returns\n        -------\n        np.ndarray\n            Resized image according to its original image aspect ratio\n        \"\"\"\n        (h, w) = image.shape[:2]\n        dW = 0\n        dH = 0\n\n        if w &lt; h:\n            image = resize(image, width=self.width, inter=self.inter)\n            dH = int((image.shape[0] - self.height) / 2.0)\n        else:\n            image = resize(image, height=self.height, inter=self.inter)\n            dW = int((image.shape[1] - self.width) / 2.0)\n\n        (h, w) = image.shape[:2]\n        image = image[dH : h - dH, dW : w - dW]\n\n        resized_image = cv2.resize(\n            image, (self.width, self.height), interpolation=self.inter\n        )\n\n        return resized_image\n</code></pre>"},{"location":"pytorch-widedeep/utils/image_utils.html#pytorch_widedeep.utils.image_utils.AspectAwarePreprocessor.preprocess","title":"preprocess","text":"<pre><code>preprocess(image)\n</code></pre> <p>Returns the resized input image taking into account the image aspect ratio</p> <p>Parameters:</p> Name Type Description Default <code>image</code> <code>ndarray</code> <p>Input image to be resized</p> required <p>Examples:</p> <pre><code>&gt;&gt;&gt; import cv2\n&gt;&gt;&gt; from pytorch_widedeep.utils import AspectAwarePreprocessor\n&gt;&gt;&gt; img = cv2.imread(\"tests/test_data_utils/images/galaxy1.png\")\n&gt;&gt;&gt; img.shape\n(694, 890, 3)\n&gt;&gt;&gt; app = AspectAwarePreprocessor(width=224, height=224)\n&gt;&gt;&gt; resized_img = app.preprocess(img)\n&gt;&gt;&gt; resized_img.shape\n(224, 224, 3)\n</code></pre> <p>Returns:</p> Type Description <code>ndarray</code> <p>Resized image according to its original image aspect ratio</p> Source code in <code>pytorch_widedeep/utils/image_utils.py</code> <pre><code>def preprocess(self, image: np.ndarray) -&gt; np.ndarray:\n    r\"\"\"Returns the resized input image taking into account the image aspect ratio\n\n    Parameters\n    ----------\n    image: np.ndarray\n        Input image to be resized\n\n    Examples\n    --------\n    &gt;&gt;&gt; import cv2\n    &gt;&gt;&gt; from pytorch_widedeep.utils import AspectAwarePreprocessor\n    &gt;&gt;&gt; img = cv2.imread(\"tests/test_data_utils/images/galaxy1.png\")\n    &gt;&gt;&gt; img.shape\n    (694, 890, 3)\n    &gt;&gt;&gt; app = AspectAwarePreprocessor(width=224, height=224)\n    &gt;&gt;&gt; resized_img = app.preprocess(img)\n    &gt;&gt;&gt; resized_img.shape\n    (224, 224, 3)\n\n    Returns\n    -------\n    np.ndarray\n        Resized image according to its original image aspect ratio\n    \"\"\"\n    (h, w) = image.shape[:2]\n    dW = 0\n    dH = 0\n\n    if w &lt; h:\n        image = resize(image, width=self.width, inter=self.inter)\n        dH = int((image.shape[0] - self.height) / 2.0)\n    else:\n        image = resize(image, height=self.height, inter=self.inter)\n        dW = int((image.shape[1] - self.width) / 2.0)\n\n    (h, w) = image.shape[:2]\n    image = image[dH : h - dH, dW : w - dW]\n\n    resized_image = cv2.resize(\n        image, (self.width, self.height), interpolation=self.inter\n    )\n\n    return resized_image\n</code></pre>"},{"location":"pytorch-widedeep/utils/image_utils.html#pytorch_widedeep.utils.image_utils.SimplePreprocessor","title":"SimplePreprocessor","text":"<p>Class to resize an image to a certain width and height</p> <p>Parameters:</p> Name Type Description Default <code>width</code> <code>int</code> <p>output width</p> required <code>height</code> <code>int</code> <p>output height</p> required <code>inter</code> <p><code>opencv</code> interpolation method. See <code>opencv</code> <code>InterpolationFlags</code>.</p> <code>INTER_AREA</code> Source code in <code>pytorch_widedeep/utils/image_utils.py</code> <pre><code>class SimplePreprocessor:\n    r\"\"\"Class to resize an image to a certain width and height\n\n    Parameters\n    ----------\n    width: int\n        output width\n    height: int\n        output height\n    inter: interpolation method, default = ``cv2.INTER_AREA``\n        ``opencv`` interpolation method. See ``opencv``\n        `InterpolationFlags`.\n    \"\"\"\n\n    def __init__(self, width: int, height: int, inter=cv2.INTER_AREA):\n        self.width = width\n        self.height = height\n        self.inter = inter\n\n    def preprocess(self, image: np.ndarray) -&gt; np.ndarray:\n        r\"\"\"Returns the resized input image\n\n        Parameters\n        ----------\n        image: np.ndarray\n            Input image to be resized\n\n        Returns\n        -------\n        np.ndarray\n            Resized image\n\n        \"\"\"\n        resized_image = cv2.resize(\n            image, (self.width, self.height), interpolation=self.inter\n        )\n\n        return resized_image\n</code></pre>"},{"location":"pytorch-widedeep/utils/image_utils.html#pytorch_widedeep.utils.image_utils.SimplePreprocessor.preprocess","title":"preprocess","text":"<pre><code>preprocess(image)\n</code></pre> <p>Returns the resized input image</p> <p>Parameters:</p> Name Type Description Default <code>image</code> <code>ndarray</code> <p>Input image to be resized</p> required <p>Returns:</p> Type Description <code>ndarray</code> <p>Resized image</p> Source code in <code>pytorch_widedeep/utils/image_utils.py</code> <pre><code>def preprocess(self, image: np.ndarray) -&gt; np.ndarray:\n    r\"\"\"Returns the resized input image\n\n    Parameters\n    ----------\n    image: np.ndarray\n        Input image to be resized\n\n    Returns\n    -------\n    np.ndarray\n        Resized image\n\n    \"\"\"\n    resized_image = cv2.resize(\n        image, (self.width, self.height), interpolation=self.inter\n    )\n\n    return resized_image\n</code></pre>"},{"location":"pytorch-widedeep/utils/text_utils.html","title":"Text utils","text":"<p>Collection of helper function that facilitate processing text.</p>"},{"location":"pytorch-widedeep/utils/text_utils.html#pytorch_widedeep.utils.text_utils.simple_preprocess","title":"simple_preprocess","text":"<pre><code>simple_preprocess(\n    doc, lower=False, deacc=False, min_len=2, max_len=15\n)\n</code></pre> <p>This is <code>Gensim</code>'s <code>simple_preprocess</code> with a <code>lower</code> param to indicate wether or not to lower case all the token in the doc</p> <p>For more information see: <code>Gensim</code> utils module</p> <p>Parameters:</p> Name Type Description Default <code>doc</code> <code>str</code> <p>Input document.</p> required <code>lower</code> <code>bool</code> <p>Lower case tokens in the input doc</p> <code>False</code> <code>deacc</code> <code>bool</code> <p>Remove accent marks from tokens using <code>Gensim</code>'s <code>deaccent</code></p> <code>False</code> <code>min_len</code> <code>int</code> <p>Minimum length of token (inclusive). Shorter tokens are discarded.</p> <code>2</code> <code>max_len</code> <code>int</code> <p>Maximum length of token in result (inclusive). Longer tokens are discarded.</p> <code>15</code> <p>Examples:</p> <pre><code>&gt;&gt;&gt; from pytorch_widedeep.utils import simple_preprocess\n&gt;&gt;&gt; simple_preprocess('Machine learning is great')\n['Machine', 'learning', 'is', 'great']\n</code></pre> <p>Returns:</p> Type Description <code>List[str]</code> <p>List with the processed tokens</p> Source code in <code>pytorch_widedeep/utils/text_utils.py</code> <pre><code>def simple_preprocess(\n    doc: str,\n    lower: bool = False,\n    deacc: bool = False,\n    min_len: int = 2,\n    max_len: int = 15,\n) -&gt; List[str]:\n    r\"\"\"\n    This is `Gensim`'s `simple_preprocess` with a `lower` param to\n    indicate wether or not to lower case all the token in the doc\n\n    For more information see: `Gensim` [utils module](https://radimrehurek.com/gensim/utils.html)\n\n    Parameters\n    ----------\n    doc: str\n        Input document.\n    lower: bool, default = False\n        Lower case tokens in the input doc\n    deacc: bool, default = False\n        Remove accent marks from tokens using `Gensim`'s `deaccent`\n    min_len: int, default = 2\n        Minimum length of token (inclusive). Shorter tokens are discarded.\n    max_len: int, default = 15\n        Maximum length of token in result (inclusive). Longer tokens are discarded.\n\n    Examples\n    --------\n    &gt;&gt;&gt; from pytorch_widedeep.utils import simple_preprocess\n    &gt;&gt;&gt; simple_preprocess('Machine learning is great')\n    ['Machine', 'learning', 'is', 'great']\n\n    Returns\n    -------\n    List[str]\n        List with the processed tokens\n    \"\"\"\n    tokens = [\n        token\n        for token in tokenize(doc, lower=lower, deacc=deacc, errors=\"ignore\")\n        if min_len &lt;= len(token) &lt;= max_len and not token.startswith(\"_\")\n    ]\n    return tokens\n</code></pre>"},{"location":"pytorch-widedeep/utils/text_utils.html#pytorch_widedeep.utils.text_utils.get_texts","title":"get_texts","text":"<pre><code>get_texts(texts, already_processed=False, n_cpus=None)\n</code></pre> <p>Tokenization using <code>Fastai</code>'s <code>Tokenizer</code> because it does a series of very convenients things during the tokenization process</p> <p>See <code>pytorch_widedeep.utils.fastai_utils.Tokenizer</code></p> <p> NOTE: <code>get_texts</code> uses <code>pytorch_widedeep.utils.fastai_transforms.Tokenizer</code>. Such tokenizer uses a series of convenient processing steps, including the  addition of some special tokens, such as <code>TK_MAJ</code> (<code>xxmaj</code>), used to indicate the next word begins with a capital in the original text. For more details of special tokens please see the <code>fastai</code> `docs</p> <p>Parameters:</p> Name Type Description Default <code>texts</code> <code>List[str]</code> <p>List of str with the texts (or documents). One str per document</p> required <code>already_processed</code> <code>Optional[bool]</code> <p>Boolean indicating if the text is already processed and we simply want to tokenize it. This parameter is thought for those cases where the input sequences might not be text (but IDs, or anything else) and we just want to tokenize it</p> <code>False</code> <code>n_cpus</code> <code>Optional[int]</code> <p>number of CPUs to used during the tokenization process</p> <code>None</code> <p>Examples:</p> <pre><code>&gt;&gt;&gt; from pytorch_widedeep.utils import get_texts\n&gt;&gt;&gt; texts = ['Machine learning is great', 'but building stuff is even better']\n&gt;&gt;&gt; get_texts(texts)\n[['xxmaj', 'machine', 'learning', 'is', 'great'], ['but', 'building', 'stuff', 'is', 'even', 'better']]\n</code></pre> <p>Returns:</p> Type Description <code>List[List[str]]</code> <p>List of lists, one list per 'document' containing its corresponding tokens</p> Source code in <code>pytorch_widedeep/utils/text_utils.py</code> <pre><code>def get_texts(\n    texts: List[str],\n    already_processed: Optional[bool] = False,\n    n_cpus: Optional[int] = None,\n) -&gt; List[List[str]]:\n    r\"\"\"Tokenization using `Fastai`'s `Tokenizer` because it does a\n    series of very convenients things during the tokenization process\n\n    See `pytorch_widedeep.utils.fastai_utils.Tokenizer`\n\n    :information_source: **NOTE**:\n    `get_texts` uses `pytorch_widedeep.utils.fastai_transforms.Tokenizer`.\n    Such tokenizer uses a series of convenient processing steps, including\n    the  addition of some special tokens, such as `TK_MAJ` (`xxmaj`), used to\n    indicate the next word begins with a capital in the original text. For more\n    details of special tokens please see the [`fastai` `docs](https://docs.fast.ai/text.core.html#Tokenizing)\n\n    Parameters\n    ----------\n    texts: List\n        List of str with the texts (or documents). One str per document\n    already_processed: bool, Optional, default = False\n        Boolean indicating if the text is already processed and we simply want\n        to tokenize it. This parameter is thought for those cases where the\n        input sequences might not be text (but IDs, or anything else) and we\n        just want to tokenize it\n    n_cpus: int, Optional, default = None\n        number of CPUs to used during the tokenization process\n\n    Examples\n    --------\n    &gt;&gt;&gt; from pytorch_widedeep.utils import get_texts\n    &gt;&gt;&gt; texts = ['Machine learning is great', 'but building stuff is even better']\n    &gt;&gt;&gt; get_texts(texts)\n    [['xxmaj', 'machine', 'learning', 'is', 'great'], ['but', 'building', 'stuff', 'is', 'even', 'better']]\n\n    Returns\n    -------\n    List[List[str]]\n        List of lists, one list per '_document_' containing its corresponding tokens\n    \"\"\"\n\n    num_cpus = n_cpus if n_cpus is not None else os.cpu_count()\n\n    if not already_processed:\n        processed_texts = [\" \".join(simple_preprocess(t)) for t in texts]\n    else:\n        processed_texts = texts\n    tok = Tokenizer(n_cpus=num_cpus).process_all(processed_texts)\n    return tok\n</code></pre>"},{"location":"pytorch-widedeep/utils/text_utils.html#pytorch_widedeep.utils.text_utils.pad_sequences","title":"pad_sequences","text":"<pre><code>pad_sequences(seq, maxlen, pad_first=True, pad_idx=1)\n</code></pre> <p>Given a List of tokenized and <code>numericalised</code> sequences it will return padded sequences according to the input parameters.</p> <p>Parameters:</p> Name Type Description Default <code>seq</code> <code>List[int]</code> <p>List of int with the <code>numericalised</code> tokens</p> required <code>maxlen</code> <code>int</code> <p>Maximum length of the padded sequences</p> required <code>pad_first</code> <code>bool</code> <p>Indicates whether the padding index will be added at the beginning or the end of the sequences</p> <code>True</code> <code>pad_idx</code> <code>int</code> <p>padding index. Fastai's Tokenizer leaves 0 for the 'unknown' token.</p> <code>1</code> <p>Examples:</p> <pre><code>&gt;&gt;&gt; from pytorch_widedeep.utils import pad_sequences\n&gt;&gt;&gt; seq = [1,2,3]\n&gt;&gt;&gt; pad_sequences(seq, maxlen=5, pad_idx=0)\narray([0, 0, 1, 2, 3], dtype=int32)\n</code></pre> <p>Returns:</p> Type Description <code>ndarray</code> <p>numpy array with the padded sequences</p> Source code in <code>pytorch_widedeep/utils/text_utils.py</code> <pre><code>def pad_sequences(\n    seq: List[int], maxlen: int, pad_first: bool = True, pad_idx: int = 1\n) -&gt; np.ndarray:\n    r\"\"\"\n    Given a List of tokenized and `numericalised` sequences it will return\n    padded sequences according to the input parameters.\n\n    Parameters\n    ----------\n    seq: List\n        List of int with the `numericalised` tokens\n    maxlen: int\n        Maximum length of the padded sequences\n    pad_first: bool,  default = True\n        Indicates whether the padding index will be added at the beginning or the\n        end of the sequences\n    pad_idx: int, default = 1\n        padding index. Fastai's Tokenizer leaves 0 for the 'unknown' token.\n\n    Examples\n    --------\n    &gt;&gt;&gt; from pytorch_widedeep.utils import pad_sequences\n    &gt;&gt;&gt; seq = [1,2,3]\n    &gt;&gt;&gt; pad_sequences(seq, maxlen=5, pad_idx=0)\n    array([0, 0, 1, 2, 3], dtype=int32)\n\n    Returns\n    -------\n    np.ndarray\n        numpy array with the padded sequences\n    \"\"\"\n    if len(seq) == 0:\n        return np.zeros(maxlen, dtype=\"int32\") + pad_idx\n    elif len(seq) &gt;= maxlen:\n        res = np.array(seq[-maxlen:]).astype(\"int32\")\n        return res\n    else:\n        res = np.zeros(maxlen, dtype=\"int32\") + pad_idx\n        if pad_first:\n            res[-len(seq) :] = seq\n        else:\n            res[: len(seq) :] = seq\n        return res\n</code></pre>"},{"location":"pytorch-widedeep/utils/text_utils.html#pytorch_widedeep.utils.text_utils.build_embeddings_matrix","title":"build_embeddings_matrix","text":"<pre><code>build_embeddings_matrix(\n    vocab, word_vectors_path, min_freq, verbose=1\n)\n</code></pre> <p>Build the embedding matrix using pretrained word vectors.</p> <p>Returns pretrained word embeddings. If a word in our vocabulary is not among the pretrained embeddings it will be assigned the mean pretrained word-embeddings vector</p> <p>Parameters:</p> Name Type Description Default <code>vocab</code> <code>Union[Vocab, ChunkVocab]</code> <p>see <code>pytorch_widedeep.utils.fastai_utils.Vocab</code></p> required <code>word_vectors_path</code> <code>str</code> <p>path to the pretrained word embeddings</p> required <code>min_freq</code> <code>int</code> <p>minimum frequency required for a word to be in the vocabulary</p> required <code>verbose</code> <code>int</code> <p>level of verbosity. Set to 0 for no verbosity</p> <code>1</code> <p>Returns:</p> Type Description <code>ndarray</code> <p>Pretrained word embeddings</p> Source code in <code>pytorch_widedeep/utils/text_utils.py</code> <pre><code>def build_embeddings_matrix(\n    vocab: Union[Vocab, ChunkVocab],\n    word_vectors_path: str,\n    min_freq: int,\n    verbose: int = 1,\n) -&gt; np.ndarray:  # pragma: no cover\n    r\"\"\"Build the embedding matrix using pretrained word vectors.\n\n    Returns pretrained word embeddings. If a word in our vocabulary is not\n    among the pretrained embeddings it will be assigned the mean pretrained\n    word-embeddings vector\n\n    Parameters\n    ----------\n    vocab: Vocab\n        see `pytorch_widedeep.utils.fastai_utils.Vocab`\n    word_vectors_path: str\n        path to the pretrained word embeddings\n    min_freq: int\n        minimum frequency required for a word to be in the vocabulary\n    verbose: int,  default=1\n        level of verbosity. Set to 0 for no verbosity\n\n    Returns\n    -------\n    np.ndarray\n        Pretrained word embeddings\n    \"\"\"\n    if not os.path.isfile(word_vectors_path):\n        raise FileNotFoundError(\"{} not found\".format(word_vectors_path))\n    if verbose:\n        print(\"Indexing word vectors...\")\n\n    embeddings_index = {}\n    f = open(word_vectors_path)\n    for line in f:\n        values = line.split()\n        word = values[0]\n        coefs = np.asarray(values[1:], dtype=\"float32\")\n        embeddings_index[word] = coefs\n    f.close()\n\n    if verbose:\n        print(\"Loaded {} word vectors\".format(len(embeddings_index)))\n        print(\"Preparing embeddings matrix...\")\n\n    mean_word_vector = np.mean(list(embeddings_index.values()), axis=0)  # type: ignore[arg-type]\n    embedding_dim = len(list(embeddings_index.values())[0])\n    num_words = len(vocab.itos)\n    embedding_matrix = np.zeros((num_words, embedding_dim))\n    found_words = 0\n    for i, word in enumerate(vocab.itos):\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[i] = embedding_vector\n            found_words += 1\n        else:\n            embedding_matrix[i] = mean_word_vector\n\n    if verbose:\n        print(\n            \"{} words in the vocabulary had {} vectors and appear more than {} times\".format(\n                found_words, word_vectors_path, min_freq\n            )\n        )\n\n    return embedding_matrix.astype(\"float32\")\n</code></pre>"}]}
\ No newline at end of file
diff --git a/mkdocs/site/sitemap.xml b/mkdocs/site/sitemap.xml
index 51b090e4..7dc40a71 100644
--- a/mkdocs/site/sitemap.xml
+++ b/mkdocs/site/sitemap.xml
@@ -2,178 +2,178 @@
 <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/index.html</loc>
-         <lastmod>2024-11-01</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/contributing.html</loc>
-         <lastmod>2024-11-01</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/installation.html</loc>
-         <lastmod>2024-11-01</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/quick_start.html</loc>
-         <lastmod>2024-11-01</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/examples/01_preprocessors_and_utils.html</loc>
-         <lastmod>2024-11-01</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/examples/02_model_components.html</loc>
-         <lastmod>2024-11-01</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/examples/03_binary_classification_with_defaults.html</loc>
-         <lastmod>2024-11-01</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/examples/04_regression_with_images_and_text.html</loc>
-         <lastmod>2024-11-01</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/examples/05_save_and_load_model_and_artifacts.html</loc>
-         <lastmod>2024-11-01</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/examples/06_finetune_and_warmup.html</loc>
-         <lastmod>2024-11-01</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/examples/07_custom_components.html</loc>
-         <lastmod>2024-11-01</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/examples/08_custom_dataLoader_imbalanced_dataset.html</loc>
-         <lastmod>2024-11-01</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/examples/09_extracting_embeddings.html</loc>
-         <lastmod>2024-11-01</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/examples/10_3rd_party_integration-RayTune_WnB.html</loc>
-         <lastmod>2024-11-01</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/examples/11_auc_multiclass.html</loc>
-         <lastmod>2024-11-01</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html</loc>
-         <lastmod>2024-11-01</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/examples/13_model_uncertainty_prediction.html</loc>
-         <lastmod>2024-11-01</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/examples/14_bayesian_models.html</loc>
-         <lastmod>2024-11-01</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/examples/15_Self_Supervised_Pretraning_pt1.html</loc>
-         <lastmod>2024-11-01</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/examples/15_Self_Supervised_Pretraning_pt2.html</loc>
-         <lastmod>2024-11-01</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/examples/16_Usign_a_custom_hugging_face_model.html</loc>
-         <lastmod>2024-11-01</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/examples/17_feature_importance_via_attention_weights.html</loc>
-         <lastmod>2024-11-01</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/examples/18_wide_and_deep_for_recsys_pt1.html</loc>
-         <lastmod>2024-11-01</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/examples/18_wide_and_deep_for_recsys_pt2.html</loc>
-         <lastmod>2024-11-01</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/examples/19_load_from_folder_functionality.html</loc>
-         <lastmod>2024-11-01</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/examples/20_Using_huggingface_within_widedeep.html</loc>
-         <lastmod>2024-11-01</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/pytorch-widedeep/bayesian_models.html</loc>
-         <lastmod>2024-11-01</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/pytorch-widedeep/bayesian_trainer.html</loc>
-         <lastmod>2024-11-01</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/pytorch-widedeep/callbacks.html</loc>
-         <lastmod>2024-11-01</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/pytorch-widedeep/dataloaders.html</loc>
-         <lastmod>2024-11-01</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/pytorch-widedeep/load_from_folder.html</loc>
-         <lastmod>2024-11-01</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/pytorch-widedeep/losses.html</loc>
-         <lastmod>2024-11-01</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/pytorch-widedeep/metrics.html</loc>
-         <lastmod>2024-11-01</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/pytorch-widedeep/model_components.html</loc>
-         <lastmod>2024-11-01</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/pytorch-widedeep/preprocessing.html</loc>
-         <lastmod>2024-11-01</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/pytorch-widedeep/self_supervised_pretraining.html</loc>
-         <lastmod>2024-11-01</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/pytorch-widedeep/tab2vec.html</loc>
-         <lastmod>2024-11-01</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/pytorch-widedeep/the_rec_module.html</loc>
-         <lastmod>2024-11-01</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/pytorch-widedeep/trainer.html</loc>
-         <lastmod>2024-11-01</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/pytorch-widedeep/utils/index.html</loc>
-         <lastmod>2024-11-01</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/pytorch-widedeep/utils/deeptabular_utils.html</loc>
-         <lastmod>2024-11-01</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/pytorch-widedeep/utils/fastai_transforms.html</loc>
-         <lastmod>2024-11-01</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/pytorch-widedeep/utils/image_utils.html</loc>
-         <lastmod>2024-11-01</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
     <url>
          <loc>https://pytorch-widedeep.readthedocs.io/pytorch-widedeep/utils/text_utils.html</loc>
-         <lastmod>2024-11-01</lastmod>
+         <lastmod>2024-11-06</lastmod>
     </url>
 </urlset>
\ No newline at end of file
diff --git a/mkdocs/site/sitemap.xml.gz b/mkdocs/site/sitemap.xml.gz
index cd6a171e0efd519ae7446666c59c2683e87f0e6a..4b6cf602ba00c33c30d60424693b75c9d363545f 100644
GIT binary patch
literal 796
zcmV+%1LOQ3iwFpSu_|W*|8r?{Wo=<_E_iKh0M(gIZ`&{ohVT6qMed1{^m}MiV21$%
zieXq=4A_NWWZD)gOP(kv&A%Vzr0vk6yKF!>Ut%kc^g!fGzECc&A6s|0qR2M*tI=#S
z9UYQ4p|rlf8r|Q08lR7@7YCPpbWG_`UiN%7YEtSx6vcYIp0G{SV#kvZ>w--q4_xd9
zi|OQIv^Y4FD^IY!8?H+i%p%XiGEKKhA)02qwk4HB-6Rs0si86$p4d=WU((~GNv*pq
z^6!412068%T+FBQ<MC`Zo}S7pyRq%M#q+{{gJ6PB(JuQW+p=F0>p7uwm?U2NCHdJ~
z^8i8<(|$qd5!=oY7t<MZk-8WRF^9;2z66PxV=aJlkh#f$32hfV`J{&I2$t4k+<=jF
z*jCmIDF$ns8Yrp4-syoohDddkAU_k@7HirDJkg_?gggPp6%AM}V7ZHaeiYT%3XPsU
zJ%!48O1<CWt}(WKrv>f|OwTE_3Wan2gjQn08woN+$;Kr*^oGh|R63^?+6xeMM9Dj8
zWI*kb%G~L-Z02kVN3n!X#IXVElZbq%LgR0^xyzaP-G9`AIs@ztv{DxBP%^b}&Ed!A
zn=dzk8A7!614U$5G3;i5=NXjn9_@-^FLH?qCF+~p#Y=1UZ*g8Ik70=$VvBx@t0vqh
za7(U&+rFc?vMd$nYa~&)&jRn#>_6e0YvDYFd$zR)d@AcrU)PyYnl)`>uLboCD#Fx9
zl77<#(T*qxRu%zMk`6g|BeHd)rdQ6+hw3x<i~J%~A&QeTykSkL`bSDNiF&bhlvR`*
zt%A$8qmmk$!;a3TjS|lJGz&`fv(wQaqior&SJH}Je_yDo^zYM2@f5=zmEjq5Zi(hW
z+qPvN?gn?RjqJ6I)!{LdwmoabI#{EQO4ZrU8@!jZvAv&SzG35R4*A6#c7r*zULR?~
z<$Og(8&yNuqvV#=JJJTHvQrHQ!y5ivc!w<CO0xA1qlQE5cPJ}a3T-wcp2aA)Iw^<a
aVa2<c&#eCi-tfVc7QX-&9E&j68vp<%NQlq?

literal 796
zcmV+%1LOQ3iwFn+8YE`||8r?{Wo=<_E_iKh0M(gMYuhjo$KUrUO5PL4X}h+gO<@m%
zK^bG~80_V^$Y)zrmOP!DG~a$FCv8Vd_p(85eu=F((huGJ@BS#4*Y}+}T2o{j{MDqK
z%_m3XO{lDIt|s4ZKTOXj*ULATLv&2(NM81QHEC1o-xbAXvzf6?)M3xF5SxNcD-T@k
z2aEaaVzPX5Bv+nbcQ;;_E|_JWg=Lz4n?f}0bYm;3i27M1tWrx=Fg&xNu)d=ES(`d{
zS>)gSJ`Hl}LbY7X7bnxQoX$&mWk0rCw|HFmZxBrIDcaSLWLpkPVm&8x4wJ;|up~bR
zYwkd3Vmd4c-DB4~;$mJxAE}SQ5OatO=qr$zIo1NW0GXQ{n9%jXlTT{Mj$vgz#w{3G
zhiz@mm}0Q8sfCJa9Go856NuDA3GySM?XaO;z!Tl8Nyt-RT+@i<0#^Iz=SNYEt<mV&
z)3;DtPigQw+zrNV=(WI|ff+c3PN8tlAJ8gHcq2ikDA~9|hu%;%j!Ng$LVE$CjwpF2
zjSQ$;QI$Krmdz~Za2zY>MI2kOK8eW3Dm49q+uNL(-~D?ns3qWFpp&v_$C9aqYXLt#
z-h8?V%n+h&?kFO|nqfZ!JkFqm_jq3v2a!umC{f?$E?!x4c#HEyc>*ik5?k~;Ts7f7
zg|Fml_&W3y*OsN?e2ydv_gUb5D*qGCg%-}Y@QrQb0UyeGJ2XvZlx9u4*lR&OgPJf6
zk)+@BL9`<Zg0)4!l%zwB-iU0|s_B*G`B;5Mf018=Iz(}DhPSLKRsTq-CQ&bTj<SxD
zqjhlEcGOZsbJ)?@v{k~nm}fzWe)c*VWt1(u^-5aN^Y05)mHu@)DV}23qcS{$&aKeg
zY1_8!!`<l4wUIrSu{u0v(za)vSO;s=QK>rHd4czIHg@n+%ol8&%^^RT!+tb}*6Slp
zxLT~qXrpQ=dz9R=21nZ9RCcQIU|7SS3$KvnTS>OTVbpMl{R(9*OQFqX#IqRXPABDf
aJgj&Z^O^O(z#Bi9(()H#HlZBY8vp=oqmCc|

diff --git a/mkdocs/sources/examples/08_custom_dataLoader_imbalanced_dataset.ipynb b/mkdocs/sources/examples/08_custom_dataLoader_imbalanced_dataset.ipynb
index ced37a9d..7d662a39 100644
--- a/mkdocs/sources/examples/08_custom_dataLoader_imbalanced_dataset.ipynb
+++ b/mkdocs/sources/examples/08_custom_dataLoader_imbalanced_dataset.ipynb
@@ -33,34 +33,16 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": 15,
    "id": "5d189eca",
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "/Users/javierrodriguezzaurin/.pyenv/versions/3.10.13/envs/widedeep310/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
-      "  from .autonotebook import tqdm as notebook_tqdm\n"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
-    "import numpy as np\n",
-    "import pandas as pd\n",
-    "import torch\n",
-    "from torch.optim import SGD, lr_scheduler\n",
-    "\n",
     "from pytorch_widedeep import Trainer\n",
     "from pytorch_widedeep.preprocessing import TabPreprocessor\n",
     "from pytorch_widedeep.models import TabMlp, WideDeep\n",
-    "from pytorch_widedeep.dataloaders import DataLoaderImbalanced, DataLoaderDefault\n",
-    "from torchmetrics import F1Score as F1_torchmetrics\n",
-    "from torchmetrics import Accuracy as Accuracy_torchmetrics\n",
-    "from torchmetrics import Precision as Precision_torchmetrics\n",
-    "from torchmetrics import Recall as Recall_torchmetrics\n",
-    "from pytorch_widedeep.metrics import Accuracy, Recall, Precision, F1Score, R2Score\n",
+    "from pytorch_widedeep.dataloaders import DataLoaderImbalanced\n",
+    "from pytorch_widedeep.metrics import Accuracy, Precision\n",
     "from pytorch_widedeep.initializers import XavierNormal\n",
     "from pytorch_widedeep.datasets import load_bio_kdd04\n",
     "\n",
@@ -72,16 +54,12 @@
     "\n",
     "import warnings\n",
     "\n",
-    "warnings.filterwarnings(\"ignore\", category=DeprecationWarning)\n",
-    "\n",
-    "# increase displayed columns in jupyter notebook\n",
-    "pd.set_option(\"display.max_columns\", 200)\n",
-    "pd.set_option(\"display.max_rows\", 300)"
+    "warnings.filterwarnings(\"ignore\", category=DeprecationWarning)"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": 10,
    "id": "e72ebad3",
    "metadata": {},
    "outputs": [],
@@ -102,7 +80,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": 11,
    "id": "81cf3284",
    "metadata": {},
    "outputs": [],
@@ -121,7 +99,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": 12,
    "id": "d202b17c",
    "metadata": {},
    "outputs": [
@@ -135,55 +113,33 @@
        "      (encoder): MLP(\n",
        "        (mlp): Sequential(\n",
        "          (dense_layer_0): Sequential(\n",
-       "            (0): Linear(in_features=74, out_features=148, bias=True)\n",
+       "            (0): Linear(in_features=74, out_features=64, bias=True)\n",
        "            (1): ReLU(inplace=True)\n",
        "            (2): Dropout(p=0.1, inplace=False)\n",
        "          )\n",
        "          (dense_layer_1): Sequential(\n",
-       "            (0): Linear(in_features=148, out_features=118, bias=True)\n",
-       "            (1): ReLU(inplace=True)\n",
-       "            (2): Dropout(p=0.1, inplace=False)\n",
-       "          )\n",
-       "          (dense_layer_2): Sequential(\n",
-       "            (0): Linear(in_features=118, out_features=89, bias=True)\n",
-       "            (1): ReLU(inplace=True)\n",
-       "            (2): Dropout(p=0.1, inplace=False)\n",
-       "          )\n",
-       "          (dense_layer_3): Sequential(\n",
-       "            (0): Linear(in_features=89, out_features=59, bias=True)\n",
-       "            (1): ReLU(inplace=True)\n",
-       "            (2): Dropout(p=0.1, inplace=False)\n",
-       "          )\n",
-       "          (dense_layer_4): Sequential(\n",
-       "            (0): Linear(in_features=59, out_features=30, bias=True)\n",
+       "            (0): Linear(in_features=64, out_features=32, bias=True)\n",
        "            (1): ReLU(inplace=True)\n",
        "            (2): Dropout(p=0.1, inplace=False)\n",
        "          )\n",
        "        )\n",
        "      )\n",
        "    )\n",
-       "    (1): Linear(in_features=30, out_features=1, bias=True)\n",
+       "    (1): Linear(in_features=32, out_features=1, bias=True)\n",
        "  )\n",
        ")"
       ]
      },
-     "execution_count": 4,
+     "execution_count": 12,
      "metadata": {},
      "output_type": "execute_result"
     }
    ],
    "source": [
-    "# Define the model\n",
-    "input_layer = len(tab_preprocessor.continuous_cols)\n",
-    "output_layer = 1\n",
-    "hidden_layers = np.linspace(\n",
-    "    input_layer * 2, output_layer, 5, endpoint=False, dtype=int\n",
-    ").tolist()\n",
-    "\n",
     "deeptabular = TabMlp(\n",
     "    column_idx=tab_preprocessor.column_idx,\n",
     "    continuous_cols=tab_preprocessor.continuous_cols,\n",
-    "    mlp_hidden_dims=hidden_layers,\n",
+    "    mlp_hidden_dims=[64, 32],\n",
     ")\n",
     "model = WideDeep(deeptabular=deeptabular)\n",
     "model"
@@ -191,39 +147,22 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": 13,
    "id": "c6262897",
    "metadata": {},
    "outputs": [],
    "source": [
-    "# Metrics from pytorch-widedeep\n",
-    "accuracy = Accuracy(top_k=2)\n",
-    "precision = Precision(average=False)\n",
-    "\n",
-    "# # Metrics from torchmetrics\n",
-    "# accuracy = Accuracy_torchmetrics(average=None, num_classes=1)\n",
-    "# precision = Precision_torchmetrics(average=\"micro\", num_classes=1)\n",
-    "\n",
-    "# Optimizers\n",
-    "deep_opt = SGD(model.deeptabular.parameters(), lr=0.1)\n",
-    "\n",
-    "# LR Scheduler\n",
-    "deep_sch = lr_scheduler.StepLR(deep_opt, step_size=3)\n",
-    "\n",
     "trainer = Trainer(\n",
     "    model,\n",
     "    objective=\"binary\",\n",
-    "    lr_schedulers={\"deeptabular\": deep_sch},\n",
-    "    initializers={\"deeptabular\": XavierNormal},\n",
-    "    optimizers={\"deeptabular\": deep_opt},\n",
-    "    metrics=[accuracy, precision],\n",
+    "    metrics=[Accuracy(), Precision()],\n",
     "    verbose=1,\n",
     ")"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 6,
+   "execution_count": 16,
    "id": "3c34b76f",
    "metadata": {},
    "outputs": [
@@ -231,22 +170,9 @@
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "epoch 1: 100%|███████████████████████████████████████| 325/325 [00:02<00:00, 153.99it/s, loss=0.163, metrics={'acc': 0.9363, 'prec': [0.9358]}]\n",
-      "valid: 100%|███████████████████████████████████████████| 456/456 [00:02<00:00, 205.93it/s, loss=0.1, metrics={'acc': 0.9501, 'prec': [0.1447]}]\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Training time[s]: 0:00:04\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "predict: 100%|██████████████████████████████████████████████████████████████████████████████████████████████| 456/456 [00:01<00:00, 368.16it/s]\n"
+      "epoch 1: 100%|██████████| 2074/2074 [00:10<00:00, 194.90it/s, loss=0.115, metrics={'acc': 0.959, 'prec': 0.9637}] \n",
+      "valid: 100%|██████████| 258/258 [00:00<00:00, 330.04it/s, loss=0.189, metrics={'acc': 0.9147, 'prec': 0.9825}]\n",
+      "predict: 100%|██████████| 456/456 [00:00<00:00, 679.61it/s]\n"
      ]
     },
     {
@@ -255,39 +181,33 @@
      "text": [
       "              precision    recall  f1-score   support\n",
       "\n",
-      "           0       1.00      0.95      0.97     14446\n",
-      "           1       0.15      0.95      0.25       130\n",
+      "           0       1.00      0.98      0.99     14446\n",
+      "           1       0.26      0.94      0.41       130\n",
       "\n",
-      "    accuracy                           0.95     14576\n",
-      "   macro avg       0.57      0.95      0.61     14576\n",
-      "weighted avg       0.99      0.95      0.97     14576\n",
-      "\n",
-      "Actual predicted values:\n",
-      "(array([0, 1]), array([13736,   840]))\n"
+      "    accuracy                           0.98     14576\n",
+      "   macro avg       0.63      0.96      0.70     14576\n",
+      "weighted avg       0.99      0.98      0.98     14576\n",
+      "\n"
      ]
     }
    ],
    "source": [
-    "start = time.time()\n",
+    "train_dataloader = DataLoaderImbalanced(kwargs={\"oversample_mul\": 5})\n",
+    "\n",
+    "# in reality one should not really do this, but just to illustrate\n",
+    "eval_dataloader = DataLoaderImbalanced(kwargs={\"oversample_mul\": 5})\n",
+    "\n",
     "trainer.fit(\n",
     "    X_train={\"X_tab\": X_tab_train, \"target\": y_train},\n",
     "    X_val={\"X_tab\": X_tab_valid, \"target\": y_valid},\n",
     "    n_epochs=1,\n",
     "    batch_size=32,\n",
-    "    custom_dataloader=DataLoaderImbalanced,\n",
-    "    oversample_mul=5,\n",
+    "    train_dataloader=train_dataloader,\n",
+    "    eval_dataloader=eval_dataloader,\n",
     ")\n",
-    "print(\n",
-    "    \"Training time[s]: {}\".format(\n",
-    "        datetime.timedelta(seconds=round(time.time() - start))\n",
-    "    )\n",
-    ")\n",
-    "\n",
-    "pd.DataFrame(trainer.history)\n",
     "\n",
-    "df_pred = trainer.predict(X_tab=X_tab_test)\n",
-    "print(classification_report(df_test[\"target\"].to_list(), df_pred))\n",
-    "print(\"Actual predicted values:\\n{}\".format(np.unique(df_pred, return_counts=True)))"
+    "preds = trainer.predict(X_tab=X_tab_test)\n",
+    "print(classification_report(df_test[\"target\"].to_list(), preds))"
    ]
   }
  ],
@@ -307,7 +227,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.10.13"
+   "version": "3.10.15"
   }
  },
  "nbformat": 4,
diff --git a/mkdocs/sources/examples/09_extracting_embeddings.ipynb b/mkdocs/sources/examples/09_extracting_embeddings.ipynb
index e26646da..73dcd457 100644
--- a/mkdocs/sources/examples/09_extracting_embeddings.ipynb
+++ b/mkdocs/sources/examples/09_extracting_embeddings.ipynb
@@ -13,7 +13,16 @@
    "cell_type": "code",
    "execution_count": 1,
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/Users/javierrodriguezzaurin/.pyenv/versions/3.10.15/envs/widedeep310/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
+      "  from .autonotebook import tqdm as notebook_tqdm\n"
+     ]
+    }
+   ],
    "source": [
     "import numpy as np\n",
     "import pandas as pd\n",
@@ -393,7 +402,7 @@
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "/Users/javierrodriguezzaurin/Projects/pytorch-widedeep/pytorch_widedeep/preprocessing/tab_preprocessor.py:358: UserWarning: Continuous columns will not be normalised\n",
+      "/Users/javierrodriguezzaurin/Projects/pytorch-widedeep/pytorch_widedeep/preprocessing/tab_preprocessor.py:364: UserWarning: Continuous columns will not be normalised\n",
       "  warnings.warn(\"Continuous columns will not be normalised\")\n"
      ]
     }
@@ -411,16 +420,7 @@
    "cell_type": "code",
    "execution_count": 6,
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "/Users/javierrodriguezzaurin/Projects/pytorch-widedeep/pytorch_widedeep/utils/general_utils.py:12: DeprecationWarning: The 'embed_continuous' parameter is deprecated and will be removed in the next release. Please use 'embed_continuous_method' instead See the documentation for more details.\n",
-      "  return func(*args, **kwargs)\n"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
     "ft_transformer = FTTransformer(\n",
     "    column_idx=tab_preprocessor.column_idx,\n",
@@ -442,8 +442,8 @@
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "epoch 1: 100%|█████████████████████████████████████████████████████████████| 153/153 [00:03<00:00, 41.47it/s, loss=221, metrics={'acc': 0.686}]\n",
-      "valid: 100%|█████████████████████████████████████████████████████████████████| 39/39 [00:00<00:00, 86.24it/s, loss=9.28, metrics={'acc': 0.76}]\n"
+      "epoch 1: 100%|██████████| 153/153 [00:02<00:00, 56.56it/s, loss=48.7, metrics={'acc': 0.7371}]\n",
+      "valid: 100%|██████████| 39/39 [00:00<00:00, 129.23it/s, loss=7.44, metrics={'acc': 0.6928}]\n"
      ]
     }
    ],
@@ -459,7 +459,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "t2v = Tab2Vec(model=model, tab_preprocessor=tab_preprocessor)"
+    "t2v = Tab2Vec(model=model, tab_preprocessor=tab_preprocessor, device=\"mps\")"
    ]
   },
   {
@@ -527,7 +527,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.10.13"
+   "version": "3.10.15"
   },
   "vscode": {
    "interpreter": {
diff --git a/mkdocs/sources/examples/13_model_uncertainty_prediction.ipynb b/mkdocs/sources/examples/13_model_uncertainty_prediction.ipynb
index 0d450d2d..59ccce20 100644
--- a/mkdocs/sources/examples/13_model_uncertainty_prediction.ipynb
+++ b/mkdocs/sources/examples/13_model_uncertainty_prediction.ipynb
@@ -40,22 +40,25 @@
    "execution_count": 1,
    "id": "9c4e5042",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/Users/javierrodriguezzaurin/.pyenv/versions/3.10.15/envs/widedeep310/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
+      "  from .autonotebook import tqdm as notebook_tqdm\n"
+     ]
+    }
+   ],
    "source": [
-    "import numpy as np\n",
     "import pandas as pd\n",
-    "import torch\n",
-    "from torch.optim import SGD, lr_scheduler\n",
+    "import numpy as np\n",
     "\n",
     "from pytorch_widedeep import Trainer\n",
     "from pytorch_widedeep.preprocessing import TabPreprocessor\n",
     "from pytorch_widedeep.models import TabMlp, WideDeep\n",
-    "from pytorch_widedeep.dataloaders import DataLoaderImbalanced, DataLoaderDefault\n",
-    "from torchmetrics import F1Score as F1_torchmetrics\n",
-    "from torchmetrics import Accuracy as Accuracy_torchmetrics\n",
-    "from torchmetrics import Precision as Precision_torchmetrics\n",
-    "from torchmetrics import Recall as Recall_torchmetrics\n",
-    "from pytorch_widedeep.metrics import Accuracy, Recall, Precision, F1Score, R2Score\n",
+    "from pytorch_widedeep.dataloaders import DataLoaderImbalanced\n",
+    "from pytorch_widedeep.metrics import Accuracy, Recall, Precision, F1Score\n",
     "from pytorch_widedeep.initializers import XavierNormal\n",
     "from pytorch_widedeep.datasets import load_bio_kdd04\n",
     "\n",
@@ -67,11 +70,7 @@
     "\n",
     "import warnings\n",
     "\n",
-    "warnings.filterwarnings(\"ignore\", category=DeprecationWarning)\n",
-    "\n",
-    "# increase displayed columns in jupyter notebook\n",
-    "pd.set_option(\"display.max_columns\", 200)\n",
-    "pd.set_option(\"display.max_rows\", 300)"
+    "warnings.filterwarnings(\"ignore\", category=DeprecationWarning)"
    ]
   },
   {
@@ -111,63 +110,7 @@
        "      <th>8</th>\n",
        "      <th>9</th>\n",
        "      <th>10</th>\n",
-       "      <th>11</th>\n",
-       "      <th>12</th>\n",
-       "      <th>13</th>\n",
-       "      <th>14</th>\n",
-       "      <th>15</th>\n",
-       "      <th>16</th>\n",
-       "      <th>17</th>\n",
-       "      <th>18</th>\n",
-       "      <th>19</th>\n",
-       "      <th>20</th>\n",
-       "      <th>21</th>\n",
-       "      <th>22</th>\n",
-       "      <th>23</th>\n",
-       "      <th>24</th>\n",
-       "      <th>25</th>\n",
-       "      <th>26</th>\n",
-       "      <th>27</th>\n",
-       "      <th>28</th>\n",
-       "      <th>29</th>\n",
-       "      <th>30</th>\n",
-       "      <th>31</th>\n",
-       "      <th>32</th>\n",
-       "      <th>33</th>\n",
-       "      <th>34</th>\n",
-       "      <th>35</th>\n",
-       "      <th>36</th>\n",
-       "      <th>37</th>\n",
-       "      <th>38</th>\n",
-       "      <th>39</th>\n",
-       "      <th>40</th>\n",
-       "      <th>41</th>\n",
-       "      <th>42</th>\n",
-       "      <th>43</th>\n",
-       "      <th>44</th>\n",
-       "      <th>45</th>\n",
-       "      <th>46</th>\n",
-       "      <th>47</th>\n",
-       "      <th>48</th>\n",
-       "      <th>49</th>\n",
-       "      <th>50</th>\n",
-       "      <th>51</th>\n",
-       "      <th>52</th>\n",
-       "      <th>53</th>\n",
-       "      <th>54</th>\n",
-       "      <th>55</th>\n",
-       "      <th>56</th>\n",
-       "      <th>57</th>\n",
-       "      <th>58</th>\n",
-       "      <th>59</th>\n",
-       "      <th>60</th>\n",
-       "      <th>61</th>\n",
-       "      <th>62</th>\n",
-       "      <th>63</th>\n",
-       "      <th>64</th>\n",
-       "      <th>65</th>\n",
-       "      <th>66</th>\n",
-       "      <th>67</th>\n",
+       "      <th>...</th>\n",
        "      <th>68</th>\n",
        "      <th>69</th>\n",
        "      <th>70</th>\n",
@@ -193,63 +136,7 @@
        "      <td>20.0</td>\n",
        "      <td>1256.8</td>\n",
        "      <td>-0.89</td>\n",
-       "      <td>0.33</td>\n",
-       "      <td>11.0</td>\n",
-       "      <td>-55.0</td>\n",
-       "      <td>267.2</td>\n",
-       "      <td>0.52</td>\n",
-       "      <td>0.05</td>\n",
-       "      <td>-2.36</td>\n",
-       "      <td>49.6</td>\n",
-       "      <td>252.0</td>\n",
-       "      <td>0.43</td>\n",
-       "      <td>1.16</td>\n",
-       "      <td>-2.06</td>\n",
-       "      <td>-33.0</td>\n",
-       "      <td>-123.2</td>\n",
-       "      <td>1.60</td>\n",
-       "      <td>-0.49</td>\n",
-       "      <td>-6.06</td>\n",
-       "      <td>65.0</td>\n",
-       "      <td>296.1</td>\n",
-       "      <td>-0.28</td>\n",
-       "      <td>-0.26</td>\n",
-       "      <td>-3.83</td>\n",
-       "      <td>-22.6</td>\n",
-       "      <td>-170.0</td>\n",
-       "      <td>3.06</td>\n",
-       "      <td>-1.05</td>\n",
-       "      <td>-3.29</td>\n",
-       "      <td>22.9</td>\n",
-       "      <td>286.3</td>\n",
-       "      <td>0.12</td>\n",
-       "      <td>2.58</td>\n",
-       "      <td>4.08</td>\n",
-       "      <td>-33.0</td>\n",
-       "      <td>-178.9</td>\n",
-       "      <td>1.88</td>\n",
-       "      <td>0.53</td>\n",
-       "      <td>-7.0</td>\n",
-       "      <td>-44.0</td>\n",
-       "      <td>1987.0</td>\n",
-       "      <td>-5.41</td>\n",
-       "      <td>0.95</td>\n",
-       "      <td>-4.0</td>\n",
-       "      <td>-57.0</td>\n",
-       "      <td>722.9</td>\n",
-       "      <td>-3.26</td>\n",
-       "      <td>-0.55</td>\n",
-       "      <td>-7.5</td>\n",
-       "      <td>125.5</td>\n",
-       "      <td>1547.2</td>\n",
-       "      <td>-0.36</td>\n",
-       "      <td>1.12</td>\n",
-       "      <td>9.0</td>\n",
-       "      <td>-37.0</td>\n",
-       "      <td>72.5</td>\n",
-       "      <td>0.47</td>\n",
-       "      <td>0.74</td>\n",
-       "      <td>-11.0</td>\n",
+       "      <td>...</td>\n",
        "      <td>-8.0</td>\n",
        "      <td>1595.1</td>\n",
        "      <td>-1.64</td>\n",
@@ -273,63 +160,7 @@
        "      <td>9.5</td>\n",
        "      <td>608.1</td>\n",
        "      <td>0.50</td>\n",
-       "      <td>0.07</td>\n",
-       "      <td>20.5</td>\n",
-       "      <td>-52.5</td>\n",
-       "      <td>521.6</td>\n",
-       "      <td>-1.08</td>\n",
-       "      <td>0.58</td>\n",
-       "      <td>-0.02</td>\n",
-       "      <td>-3.2</td>\n",
-       "      <td>103.6</td>\n",
-       "      <td>-0.95</td>\n",
-       "      <td>0.23</td>\n",
-       "      <td>-2.87</td>\n",
-       "      <td>-25.9</td>\n",
-       "      <td>-52.2</td>\n",
-       "      <td>-0.21</td>\n",
-       "      <td>0.87</td>\n",
-       "      <td>-1.81</td>\n",
-       "      <td>10.4</td>\n",
-       "      <td>62.0</td>\n",
-       "      <td>-0.28</td>\n",
-       "      <td>-0.04</td>\n",
-       "      <td>1.48</td>\n",
-       "      <td>-17.6</td>\n",
-       "      <td>-198.3</td>\n",
-       "      <td>3.43</td>\n",
-       "      <td>2.84</td>\n",
-       "      <td>5.87</td>\n",
-       "      <td>-16.9</td>\n",
-       "      <td>72.6</td>\n",
-       "      <td>-0.31</td>\n",
-       "      <td>2.79</td>\n",
-       "      <td>2.71</td>\n",
-       "      <td>-33.5</td>\n",
-       "      <td>-11.6</td>\n",
-       "      <td>-1.11</td>\n",
-       "      <td>4.01</td>\n",
-       "      <td>5.0</td>\n",
-       "      <td>-57.0</td>\n",
-       "      <td>666.3</td>\n",
-       "      <td>1.13</td>\n",
-       "      <td>4.38</td>\n",
-       "      <td>5.0</td>\n",
-       "      <td>-64.0</td>\n",
-       "      <td>39.3</td>\n",
-       "      <td>1.07</td>\n",
-       "      <td>-0.16</td>\n",
-       "      <td>32.5</td>\n",
-       "      <td>100.0</td>\n",
-       "      <td>1893.7</td>\n",
-       "      <td>-2.80</td>\n",
-       "      <td>-0.22</td>\n",
-       "      <td>2.5</td>\n",
-       "      <td>-28.5</td>\n",
-       "      <td>45.0</td>\n",
-       "      <td>0.58</td>\n",
-       "      <td>0.41</td>\n",
-       "      <td>-19.0</td>\n",
+       "      <td>...</td>\n",
        "      <td>-6.0</td>\n",
        "      <td>762.9</td>\n",
        "      <td>0.29</td>\n",
@@ -353,63 +184,7 @@
        "      <td>58.5</td>\n",
        "      <td>1623.6</td>\n",
        "      <td>-1.40</td>\n",
-       "      <td>0.02</td>\n",
-       "      <td>-6.5</td>\n",
-       "      <td>-48.0</td>\n",
-       "      <td>621.0</td>\n",
-       "      <td>-1.20</td>\n",
-       "      <td>0.14</td>\n",
-       "      <td>-0.20</td>\n",
-       "      <td>73.6</td>\n",
-       "      <td>609.1</td>\n",
-       "      <td>-0.44</td>\n",
-       "      <td>-0.58</td>\n",
-       "      <td>-0.04</td>\n",
-       "      <td>-23.0</td>\n",
-       "      <td>-27.4</td>\n",
-       "      <td>-0.72</td>\n",
-       "      <td>-1.04</td>\n",
-       "      <td>-1.09</td>\n",
-       "      <td>91.1</td>\n",
-       "      <td>635.6</td>\n",
-       "      <td>-0.88</td>\n",
-       "      <td>0.24</td>\n",
-       "      <td>0.59</td>\n",
-       "      <td>-18.7</td>\n",
-       "      <td>-7.2</td>\n",
-       "      <td>-0.60</td>\n",
-       "      <td>-2.82</td>\n",
-       "      <td>-0.71</td>\n",
-       "      <td>52.4</td>\n",
-       "      <td>504.1</td>\n",
-       "      <td>0.89</td>\n",
-       "      <td>-0.67</td>\n",
-       "      <td>-9.30</td>\n",
-       "      <td>-20.8</td>\n",
-       "      <td>-25.7</td>\n",
-       "      <td>-0.77</td>\n",
-       "      <td>-0.85</td>\n",
-       "      <td>0.0</td>\n",
-       "      <td>-20.0</td>\n",
-       "      <td>2259.0</td>\n",
-       "      <td>-0.94</td>\n",
-       "      <td>1.15</td>\n",
-       "      <td>-4.0</td>\n",
-       "      <td>-44.0</td>\n",
-       "      <td>-22.7</td>\n",
-       "      <td>0.94</td>\n",
-       "      <td>-0.98</td>\n",
-       "      <td>-19.0</td>\n",
-       "      <td>105.0</td>\n",
-       "      <td>1267.9</td>\n",
-       "      <td>1.03</td>\n",
-       "      <td>1.27</td>\n",
-       "      <td>11.0</td>\n",
-       "      <td>-39.5</td>\n",
-       "      <td>82.3</td>\n",
-       "      <td>0.47</td>\n",
-       "      <td>-0.19</td>\n",
-       "      <td>-10.0</td>\n",
+       "      <td>...</td>\n",
        "      <td>7.0</td>\n",
        "      <td>1491.8</td>\n",
        "      <td>0.32</td>\n",
@@ -433,63 +208,7 @@
        "      <td>46.0</td>\n",
        "      <td>1921.6</td>\n",
        "      <td>-1.36</td>\n",
-       "      <td>-0.47</td>\n",
-       "      <td>-32.0</td>\n",
-       "      <td>-51.5</td>\n",
-       "      <td>560.9</td>\n",
-       "      <td>-0.29</td>\n",
-       "      <td>-0.10</td>\n",
-       "      <td>-1.11</td>\n",
-       "      <td>124.3</td>\n",
-       "      <td>791.6</td>\n",
-       "      <td>0.00</td>\n",
-       "      <td>0.39</td>\n",
-       "      <td>-1.85</td>\n",
-       "      <td>-21.7</td>\n",
-       "      <td>-44.9</td>\n",
-       "      <td>-0.21</td>\n",
-       "      <td>0.02</td>\n",
-       "      <td>0.89</td>\n",
-       "      <td>133.9</td>\n",
-       "      <td>797.8</td>\n",
-       "      <td>-0.08</td>\n",
-       "      <td>1.06</td>\n",
-       "      <td>-0.26</td>\n",
-       "      <td>-16.4</td>\n",
-       "      <td>-74.1</td>\n",
-       "      <td>0.97</td>\n",
-       "      <td>-0.80</td>\n",
-       "      <td>-0.41</td>\n",
-       "      <td>66.9</td>\n",
-       "      <td>955.3</td>\n",
-       "      <td>-1.90</td>\n",
-       "      <td>1.28</td>\n",
-       "      <td>-6.65</td>\n",
-       "      <td>-28.1</td>\n",
-       "      <td>47.5</td>\n",
-       "      <td>-1.91</td>\n",
-       "      <td>1.42</td>\n",
-       "      <td>1.0</td>\n",
-       "      <td>-30.0</td>\n",
-       "      <td>1846.7</td>\n",
-       "      <td>0.76</td>\n",
-       "      <td>1.10</td>\n",
-       "      <td>-4.0</td>\n",
-       "      <td>-52.0</td>\n",
-       "      <td>-53.9</td>\n",
-       "      <td>1.71</td>\n",
-       "      <td>-0.22</td>\n",
-       "      <td>-12.0</td>\n",
-       "      <td>97.5</td>\n",
-       "      <td>1969.8</td>\n",
-       "      <td>-1.70</td>\n",
-       "      <td>0.16</td>\n",
-       "      <td>-1.0</td>\n",
-       "      <td>-32.5</td>\n",
-       "      <td>255.9</td>\n",
-       "      <td>-0.46</td>\n",
-       "      <td>1.57</td>\n",
-       "      <td>10.0</td>\n",
+       "      <td>...</td>\n",
        "      <td>6.0</td>\n",
        "      <td>2047.7</td>\n",
        "      <td>-0.98</td>\n",
@@ -513,63 +232,7 @@
        "      <td>12.0</td>\n",
        "      <td>464.8</td>\n",
        "      <td>0.88</td>\n",
-       "      <td>0.19</td>\n",
-       "      <td>8.0</td>\n",
-       "      <td>-51.5</td>\n",
-       "      <td>98.1</td>\n",
-       "      <td>1.09</td>\n",
-       "      <td>-0.33</td>\n",
-       "      <td>-2.16</td>\n",
-       "      <td>-3.9</td>\n",
-       "      <td>102.7</td>\n",
-       "      <td>0.39</td>\n",
-       "      <td>-1.22</td>\n",
-       "      <td>-3.39</td>\n",
-       "      <td>-15.2</td>\n",
-       "      <td>-42.2</td>\n",
-       "      <td>-1.18</td>\n",
-       "      <td>-1.11</td>\n",
-       "      <td>-3.55</td>\n",
-       "      <td>8.9</td>\n",
-       "      <td>141.3</td>\n",
-       "      <td>-0.16</td>\n",
-       "      <td>-0.43</td>\n",
-       "      <td>-4.15</td>\n",
-       "      <td>-12.9</td>\n",
-       "      <td>-13.4</td>\n",
-       "      <td>-1.32</td>\n",
-       "      <td>-0.98</td>\n",
-       "      <td>-3.69</td>\n",
-       "      <td>8.8</td>\n",
-       "      <td>136.1</td>\n",
-       "      <td>-0.30</td>\n",
-       "      <td>4.13</td>\n",
-       "      <td>1.89</td>\n",
-       "      <td>-13.0</td>\n",
-       "      <td>-18.7</td>\n",
-       "      <td>-1.37</td>\n",
-       "      <td>-0.93</td>\n",
-       "      <td>0.0</td>\n",
-       "      <td>-1.0</td>\n",
-       "      <td>810.1</td>\n",
-       "      <td>-2.29</td>\n",
-       "      <td>6.72</td>\n",
-       "      <td>1.0</td>\n",
-       "      <td>-23.0</td>\n",
-       "      <td>-29.7</td>\n",
-       "      <td>0.58</td>\n",
-       "      <td>-1.10</td>\n",
-       "      <td>-18.5</td>\n",
-       "      <td>33.5</td>\n",
-       "      <td>206.8</td>\n",
-       "      <td>1.84</td>\n",
-       "      <td>-0.13</td>\n",
-       "      <td>4.0</td>\n",
-       "      <td>-29.0</td>\n",
-       "      <td>30.1</td>\n",
-       "      <td>0.80</td>\n",
-       "      <td>-0.24</td>\n",
-       "      <td>5.0</td>\n",
+       "      <td>...</td>\n",
        "      <td>-14.0</td>\n",
        "      <td>479.5</td>\n",
        "      <td>0.68</td>\n",
@@ -583,6 +246,7 @@
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
+       "<p>5 rows × 77 columns</p>\n",
        "</div>"
       ],
       "text/plain": [
@@ -593,47 +257,14 @@
        "3         279    261535       0  41.0  27.91 -0.35   3.0  46.0  1921.6 -1.36   \n",
        "4         279    261536       0  50.0  28.00 -1.32  -9.0  12.0   464.8  0.88   \n",
        "\n",
-       "     11    12    13     14    15    16    17     18     19    20    21    22  \\\n",
-       "0  0.33  11.0 -55.0  267.2  0.52  0.05 -2.36   49.6  252.0  0.43  1.16 -2.06   \n",
-       "1  0.07  20.5 -52.5  521.6 -1.08  0.58 -0.02   -3.2  103.6 -0.95  0.23 -2.87   \n",
-       "2  0.02  -6.5 -48.0  621.0 -1.20  0.14 -0.20   73.6  609.1 -0.44 -0.58 -0.04   \n",
-       "3 -0.47 -32.0 -51.5  560.9 -0.29 -0.10 -1.11  124.3  791.6  0.00  0.39 -1.85   \n",
-       "4  0.19   8.0 -51.5   98.1  1.09 -0.33 -2.16   -3.9  102.7  0.39 -1.22 -3.39   \n",
-       "\n",
-       "     23     24    25    26    27     28     29    30    31    32    33     34  \\\n",
-       "0 -33.0 -123.2  1.60 -0.49 -6.06   65.0  296.1 -0.28 -0.26 -3.83 -22.6 -170.0   \n",
-       "1 -25.9  -52.2 -0.21  0.87 -1.81   10.4   62.0 -0.28 -0.04  1.48 -17.6 -198.3   \n",
-       "2 -23.0  -27.4 -0.72 -1.04 -1.09   91.1  635.6 -0.88  0.24  0.59 -18.7   -7.2   \n",
-       "3 -21.7  -44.9 -0.21  0.02  0.89  133.9  797.8 -0.08  1.06 -0.26 -16.4  -74.1   \n",
-       "4 -15.2  -42.2 -1.18 -1.11 -3.55    8.9  141.3 -0.16 -0.43 -4.15 -12.9  -13.4   \n",
-       "\n",
-       "     35    36    37    38     39    40    41    42    43     44    45    46  \\\n",
-       "0  3.06 -1.05 -3.29  22.9  286.3  0.12  2.58  4.08 -33.0 -178.9  1.88  0.53   \n",
-       "1  3.43  2.84  5.87 -16.9   72.6 -0.31  2.79  2.71 -33.5  -11.6 -1.11  4.01   \n",
-       "2 -0.60 -2.82 -0.71  52.4  504.1  0.89 -0.67 -9.30 -20.8  -25.7 -0.77 -0.85   \n",
-       "3  0.97 -0.80 -0.41  66.9  955.3 -1.90  1.28 -6.65 -28.1   47.5 -1.91  1.42   \n",
-       "4 -1.32 -0.98 -3.69   8.8  136.1 -0.30  4.13  1.89 -13.0  -18.7 -1.37 -0.93   \n",
+       "   ...    68      69    70    71   72    73     74    75    76    77  \n",
+       "0  ...  -8.0  1595.1 -1.64  2.83 -2.0 -50.0  445.2 -0.35  0.26  0.76  \n",
+       "1  ...  -6.0   762.9  0.29  0.82 -3.0 -35.0  140.3  1.16  0.39  0.73  \n",
+       "2  ...   7.0  1491.8  0.32 -1.29  0.0 -34.0  658.2 -0.76  0.26  0.24  \n",
+       "3  ...   6.0  2047.7 -0.98  1.53  0.0 -49.0  554.2 -0.83  0.39  0.73  \n",
+       "4  ... -14.0   479.5  0.68 -0.59  2.0 -36.0   -6.9  2.02  0.14 -0.23  \n",
        "\n",
-       "    47    48      49    50    51   52    53     54    55    56    57     58  \\\n",
-       "0 -7.0 -44.0  1987.0 -5.41  0.95 -4.0 -57.0  722.9 -3.26 -0.55  -7.5  125.5   \n",
-       "1  5.0 -57.0   666.3  1.13  4.38  5.0 -64.0   39.3  1.07 -0.16  32.5  100.0   \n",
-       "2  0.0 -20.0  2259.0 -0.94  1.15 -4.0 -44.0  -22.7  0.94 -0.98 -19.0  105.0   \n",
-       "3  1.0 -30.0  1846.7  0.76  1.10 -4.0 -52.0  -53.9  1.71 -0.22 -12.0   97.5   \n",
-       "4  0.0  -1.0   810.1 -2.29  6.72  1.0 -23.0  -29.7  0.58 -1.10 -18.5   33.5   \n",
-       "\n",
-       "       59    60    61    62    63     64    65    66    67    68      69  \\\n",
-       "0  1547.2 -0.36  1.12   9.0 -37.0   72.5  0.47  0.74 -11.0  -8.0  1595.1   \n",
-       "1  1893.7 -2.80 -0.22   2.5 -28.5   45.0  0.58  0.41 -19.0  -6.0   762.9   \n",
-       "2  1267.9  1.03  1.27  11.0 -39.5   82.3  0.47 -0.19 -10.0   7.0  1491.8   \n",
-       "3  1969.8 -1.70  0.16  -1.0 -32.5  255.9 -0.46  1.57  10.0   6.0  2047.7   \n",
-       "4   206.8  1.84 -0.13   4.0 -29.0   30.1  0.80 -0.24   5.0 -14.0   479.5   \n",
-       "\n",
-       "     70    71   72    73     74    75    76    77  \n",
-       "0 -1.64  2.83 -2.0 -50.0  445.2 -0.35  0.26  0.76  \n",
-       "1  0.29  0.82 -3.0 -35.0  140.3  1.16  0.39  0.73  \n",
-       "2  0.32 -1.29  0.0 -34.0  658.2 -0.76  0.26  0.24  \n",
-       "3 -0.98  1.53  0.0 -49.0  554.2 -0.83  0.39  0.73  \n",
-       "4  0.68 -0.59  2.0 -36.0   -6.9  2.02  0.14 -0.23  "
+       "[5 rows x 77 columns]"
       ]
      },
      "execution_count": 2,
@@ -649,31 +280,6 @@
   {
    "cell_type": "code",
    "execution_count": 3,
-   "id": "d818e9f2",
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "target\n",
-       "0    144455\n",
-       "1      1296\n",
-       "Name: count, dtype: int64"
-      ]
-     },
-     "execution_count": 3,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "# imbalance of the classes\n",
-    "df[\"target\"].value_counts()"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 4,
    "id": "38cdc663",
    "metadata": {},
    "outputs": [],
@@ -684,7 +290,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": 4,
    "id": "039147fc",
    "metadata": {},
    "outputs": [],
@@ -707,7 +313,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 6,
+   "execution_count": 5,
    "id": "95358c9c",
    "metadata": {},
    "outputs": [],
@@ -717,7 +323,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 7,
+   "execution_count": 6,
    "id": "4bf475a3",
    "metadata": {},
    "outputs": [],
@@ -744,21 +350,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 8,
-   "id": "439c350e",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "input_layer = len(tab_preprocessor.continuous_cols)\n",
-    "output_layer = 1\n",
-    "hidden_layers = np.linspace(\n",
-    "    input_layer * 2, output_layer, 5, endpoint=False, dtype=int\n",
-    ").tolist()"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 9,
+   "execution_count": 7,
    "id": "c8334b9b",
    "metadata": {},
    "outputs": [
@@ -772,48 +364,33 @@
        "      (encoder): MLP(\n",
        "        (mlp): Sequential(\n",
        "          (dense_layer_0): Sequential(\n",
-       "            (0): Linear(in_features=74, out_features=148, bias=True)\n",
+       "            (0): Linear(in_features=74, out_features=64, bias=True)\n",
        "            (1): ReLU(inplace=True)\n",
        "            (2): Dropout(p=0.1, inplace=False)\n",
        "          )\n",
        "          (dense_layer_1): Sequential(\n",
-       "            (0): Linear(in_features=148, out_features=118, bias=True)\n",
-       "            (1): ReLU(inplace=True)\n",
-       "            (2): Dropout(p=0.1, inplace=False)\n",
-       "          )\n",
-       "          (dense_layer_2): Sequential(\n",
-       "            (0): Linear(in_features=118, out_features=89, bias=True)\n",
-       "            (1): ReLU(inplace=True)\n",
-       "            (2): Dropout(p=0.1, inplace=False)\n",
-       "          )\n",
-       "          (dense_layer_3): Sequential(\n",
-       "            (0): Linear(in_features=89, out_features=59, bias=True)\n",
-       "            (1): ReLU(inplace=True)\n",
-       "            (2): Dropout(p=0.1, inplace=False)\n",
-       "          )\n",
-       "          (dense_layer_4): Sequential(\n",
-       "            (0): Linear(in_features=59, out_features=30, bias=True)\n",
+       "            (0): Linear(in_features=64, out_features=32, bias=True)\n",
        "            (1): ReLU(inplace=True)\n",
        "            (2): Dropout(p=0.1, inplace=False)\n",
        "          )\n",
        "        )\n",
        "      )\n",
        "    )\n",
-       "    (1): Linear(in_features=30, out_features=1, bias=True)\n",
+       "    (1): Linear(in_features=32, out_features=1, bias=True)\n",
        "  )\n",
        ")"
       ]
      },
-     "execution_count": 9,
+     "execution_count": 7,
      "metadata": {},
      "output_type": "execute_result"
     }
    ],
    "source": [
     "deeptabular = TabMlp(\n",
-    "    mlp_hidden_dims=hidden_layers,\n",
     "    column_idx=tab_preprocessor.column_idx,\n",
     "    continuous_cols=tab_preprocessor.continuous_cols,\n",
+    "    mlp_hidden_dims=[64, 32],\n",
     ")\n",
     "model = WideDeep(deeptabular=deeptabular, pred_dim=1)\n",
     "model"
@@ -821,58 +398,22 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 10,
-   "id": "eabe6cc2",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# # Metrics from torchmetrics\n",
-    "# accuracy = Accuracy_torchmetrics(average=None, num_classes=1)\n",
-    "# precision = Precision_torchmetrics(average=\"micro\", num_classes=1)\n",
-    "# f1 = F1_torchmetrics(average=None, num_classes=1)\n",
-    "# recall = Recall_torchmetrics(average=None, num_classes=1)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 11,
-   "id": "54b0bd74",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# Metrics from pytorch-widedeep\n",
-    "accuracy = Accuracy(top_k=2)\n",
-    "precision = Precision(average=False)\n",
-    "recall = Recall(average=True)\n",
-    "f1 = F1Score(average=False)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 12,
+   "execution_count": 8,
    "id": "64561e0d",
    "metadata": {},
    "outputs": [],
    "source": [
-    "# Optimizers\n",
-    "deep_opt = SGD(model.deeptabular.parameters(), lr=0.1)\n",
-    "# LR Scheduler\n",
-    "deep_sch = lr_scheduler.StepLR(deep_opt, step_size=3)\n",
-    "\n",
     "trainer = Trainer(\n",
     "    model,\n",
     "    objective=\"binary\",\n",
-    "    lr_schedulers={\"deeptabular\": deep_sch},\n",
-    "    initializers={\"deeptabular\": XavierNormal},\n",
-    "    optimizers={\"deeptabular\": deep_opt},\n",
-    "    metrics=[accuracy, precision, recall, f1],\n",
+    "    metrics=[Accuracy(), Precision(), F1Score(), Recall()],\n",
     "    verbose=1,\n",
     ")"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 13,
+   "execution_count": 9,
    "id": "edf1284f",
    "metadata": {},
    "outputs": [
@@ -880,26 +421,12 @@
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "epoch 1: 100%|████████| 208/208 [00:01<00:00, 130.75it/s, loss=0.187, metrics={'acc': 0.9214, 'prec': [0.9149], 'rec': 0.9318, 'f1': [0.9233]}]\n",
-      "valid: 100%|████████████| 292/292 [00:01<00:00, 173.71it/s, loss=0.106, metrics={'acc': 0.9499, 'prec': [0.1435], 'rec': 0.938, 'f1': [0.249]}]\n",
-      "epoch 2: 100%|████████| 208/208 [00:01<00:00, 139.68it/s, loss=0.109, metrics={'acc': 0.9559, 'prec': [0.9537], 'rec': 0.9572, 'f1': [0.9554]}]\n",
-      "valid: 100%|█████████| 292/292 [00:01<00:00, 177.53it/s, loss=0.0888, metrics={'acc': 0.9602, 'prec': [0.1755], 'rec': 0.9457, 'f1': [0.2961]}]\n",
-      "epoch 3: 100%|█████████| 208/208 [00:01<00:00, 141.63it/s, loss=0.08, metrics={'acc': 0.9706, 'prec': [0.9648], 'rec': 0.9766, 'f1': [0.9707]}]\n",
-      "valid: 100%|█████████| 292/292 [00:01<00:00, 165.36it/s, loss=0.0969, metrics={'acc': 0.9564, 'prec': [0.1636], 'rec': 0.9535, 'f1': [0.2792]}]"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Training time[s]: 0:00:10\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "\n"
+      "epoch 1: 100%|██████████| 3644/3644 [00:20<00:00, 175.39it/s, loss=0.0222, metrics={'acc': 0.9945, 'prec': 0.7565, 'f1': 0.6419, 'rec': 0.5574}]\n",
+      "valid: 100%|██████████| 456/456 [00:01<00:00, 252.36it/s, loss=0.0125, metrics={'acc': 0.9969, 'prec': 0.92, 'f1': 0.8035, 'rec': 0.7132}]  \n",
+      "epoch 2: 100%|██████████| 3644/3644 [00:20<00:00, 177.43it/s, loss=0.0119, metrics={'acc': 0.9968, 'prec': 0.9209, 'f1': 0.793, 'rec': 0.6962}] \n",
+      "valid: 100%|██████████| 456/456 [00:01<00:00, 255.61it/s, loss=0.0121, metrics={'acc': 0.997, 'prec': 0.8972, 'f1': 0.8136, 'rec': 0.7442}] \n",
+      "epoch 3: 100%|██████████| 3644/3644 [00:20<00:00, 176.07it/s, loss=0.0103, metrics={'acc': 0.9973, 'prec': 0.9312, 'f1': 0.8351, 'rec': 0.757}] \n",
+      "valid: 100%|██████████| 456/456 [00:01<00:00, 259.70it/s, loss=0.0119, metrics={'acc': 0.997, 'prec': 0.8909, 'f1': 0.8201, 'rec': 0.7597}] \n"
      ]
     }
    ],
@@ -909,20 +436,13 @@
     "    X_train={\"X_tab\": X_tab_train, \"target\": y_train},\n",
     "    X_val={\"X_tab\": X_tab_valid, \"target\": y_valid},\n",
     "    n_epochs=3,\n",
-    "    batch_size=50,\n",
-    "    custom_dataloader=DataLoaderImbalanced,\n",
-    "    oversample_mul=5,\n",
-    ")\n",
-    "print(\n",
-    "    \"Training time[s]: {}\".format(\n",
-    "        datetime.timedelta(seconds=round(time.time() - start))\n",
-    "    )\n",
+    "    batch_size=32,\n",
     ")"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 14,
+   "execution_count": 10,
    "id": "5bbc47ca",
    "metadata": {},
    "outputs": [
@@ -950,77 +470,72 @@
        "      <th>train_loss</th>\n",
        "      <th>train_acc</th>\n",
        "      <th>train_prec</th>\n",
-       "      <th>train_rec</th>\n",
        "      <th>train_f1</th>\n",
+       "      <th>train_rec</th>\n",
        "      <th>val_loss</th>\n",
        "      <th>val_acc</th>\n",
        "      <th>val_prec</th>\n",
-       "      <th>val_rec</th>\n",
        "      <th>val_f1</th>\n",
+       "      <th>val_rec</th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
        "      <th>0</th>\n",
-       "      <td>0.186707</td>\n",
-       "      <td>0.921408</td>\n",
-       "      <td>[0.9149412512779236]</td>\n",
-       "      <td>0.931801</td>\n",
-       "      <td>[0.9232940673828125]</td>\n",
-       "      <td>0.106023</td>\n",
-       "      <td>0.949914</td>\n",
-       "      <td>[0.14353498816490173]</td>\n",
-       "      <td>0.937984</td>\n",
-       "      <td>[0.24897116422653198]</td>\n",
+       "      <td>0.022229</td>\n",
+       "      <td>0.994468</td>\n",
+       "      <td>0.756545</td>\n",
+       "      <td>0.641866</td>\n",
+       "      <td>0.557377</td>\n",
+       "      <td>0.012473</td>\n",
+       "      <td>0.996913</td>\n",
+       "      <td>0.920000</td>\n",
+       "      <td>0.803493</td>\n",
+       "      <td>0.713178</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>1</th>\n",
-       "      <td>0.109498</td>\n",
-       "      <td>0.955931</td>\n",
-       "      <td>[0.9536514282226562]</td>\n",
-       "      <td>0.957193</td>\n",
-       "      <td>[0.9554190039634705]</td>\n",
-       "      <td>0.088787</td>\n",
-       "      <td>0.960206</td>\n",
-       "      <td>[0.17553956806659698]</td>\n",
-       "      <td>0.945736</td>\n",
-       "      <td>[0.29611650109291077]</td>\n",
+       "      <td>0.011912</td>\n",
+       "      <td>0.996767</td>\n",
+       "      <td>0.920918</td>\n",
+       "      <td>0.792971</td>\n",
+       "      <td>0.696239</td>\n",
+       "      <td>0.012088</td>\n",
+       "      <td>0.996981</td>\n",
+       "      <td>0.897196</td>\n",
+       "      <td>0.813559</td>\n",
+       "      <td>0.744186</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>2</th>\n",
-       "      <td>0.079979</td>\n",
-       "      <td>0.970588</td>\n",
-       "      <td>[0.9648183584213257]</td>\n",
-       "      <td>0.976582</td>\n",
-       "      <td>[0.9706646203994751]</td>\n",
-       "      <td>0.096858</td>\n",
-       "      <td>0.956432</td>\n",
-       "      <td>[0.1635638326406479]</td>\n",
-       "      <td>0.953488</td>\n",
-       "      <td>[0.279228150844574]</td>\n",
+       "      <td>0.010341</td>\n",
+       "      <td>0.997341</td>\n",
+       "      <td>0.931198</td>\n",
+       "      <td>0.835106</td>\n",
+       "      <td>0.756991</td>\n",
+       "      <td>0.011884</td>\n",
+       "      <td>0.997050</td>\n",
+       "      <td>0.890909</td>\n",
+       "      <td>0.820084</td>\n",
+       "      <td>0.759690</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "</div>"
       ],
       "text/plain": [
-       "   train_loss  train_acc            train_prec  train_rec  \\\n",
-       "0    0.186707   0.921408  [0.9149412512779236]   0.931801   \n",
-       "1    0.109498   0.955931  [0.9536514282226562]   0.957193   \n",
-       "2    0.079979   0.970588  [0.9648183584213257]   0.976582   \n",
-       "\n",
-       "               train_f1  val_loss   val_acc               val_prec   val_rec  \\\n",
-       "0  [0.9232940673828125]  0.106023  0.949914  [0.14353498816490173]  0.937984   \n",
-       "1  [0.9554190039634705]  0.088787  0.960206  [0.17553956806659698]  0.945736   \n",
-       "2  [0.9706646203994751]  0.096858  0.956432   [0.1635638326406479]  0.953488   \n",
+       "   train_loss  train_acc  train_prec  train_f1  train_rec  val_loss   val_acc  \\\n",
+       "0    0.022229   0.994468    0.756545  0.641866   0.557377  0.012473  0.996913   \n",
+       "1    0.011912   0.996767    0.920918  0.792971   0.696239  0.012088  0.996981   \n",
+       "2    0.010341   0.997341    0.931198  0.835106   0.756991  0.011884  0.997050   \n",
        "\n",
-       "                  val_f1  \n",
-       "0  [0.24897116422653198]  \n",
-       "1  [0.29611650109291077]  \n",
-       "2    [0.279228150844574]  "
+       "   val_prec    val_f1   val_rec  \n",
+       "0  0.920000  0.803493  0.713178  \n",
+       "1  0.897196  0.813559  0.744186  \n",
+       "2  0.890909  0.820084  0.759690  "
       ]
      },
-     "execution_count": 14,
+     "execution_count": 10,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -1039,7 +554,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 15,
+   "execution_count": 11,
    "id": "2fc67e9b",
    "metadata": {
     "scrolled": true
@@ -1049,7 +564,7 @@
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "predict: 100%|██████████████████████████████████████████████████████████████████████████████████████████████| 292/292 [00:00<00:00, 346.55it/s]\n"
+      "predict: 100%|██████████| 456/456 [00:00<00:00, 689.36it/s]\n"
      ]
     },
     {
@@ -1058,15 +573,15 @@
      "text": [
       "              precision    recall  f1-score   support\n",
       "\n",
-      "           0       1.00      0.96      0.98     14446\n",
-      "           1       0.17      0.95      0.29       130\n",
+      "           0       1.00      1.00      1.00     14446\n",
+      "           1       0.91      0.78      0.84       130\n",
       "\n",
-      "    accuracy                           0.96     14576\n",
-      "   macro avg       0.58      0.95      0.63     14576\n",
-      "weighted avg       0.99      0.96      0.97     14576\n",
+      "    accuracy                           1.00     14576\n",
+      "   macro avg       0.95      0.89      0.92     14576\n",
+      "weighted avg       1.00      1.00      1.00     14576\n",
       "\n",
       "Actual predicted values:\n",
-      "(array([0, 1]), array([13845,   731]))\n"
+      "(array([0, 1]), array([14465,   111]))\n"
      ]
     }
    ],
@@ -1086,7 +601,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 16,
+   "execution_count": 12,
    "id": "c7317302",
    "metadata": {},
    "outputs": [
@@ -1094,7 +609,7 @@
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "predict_UncertaintyIter: 100%|█████████████████████████████████████████████████████████████████████████████████| 10/10 [00:03<00:00,  3.25it/s]"
+      "predict_UncertaintyIter: 100%|██████████| 10/10 [00:05<00:00,  1.86it/s]"
      ]
     },
     {
@@ -1103,12 +618,12 @@
      "text": [
       "              precision    recall  f1-score   support\n",
       "\n",
-      "           0       1.00      0.96      0.98     14446\n",
-      "           1       0.17      0.95      0.29       130\n",
+      "           0       1.00      1.00      1.00     14446\n",
+      "           1       0.91      0.78      0.84       130\n",
       "\n",
-      "    accuracy                           0.96     14576\n",
-      "   macro avg       0.58      0.95      0.63     14576\n",
-      "weighted avg       0.99      0.96      0.97     14576\n",
+      "    accuracy                           1.00     14576\n",
+      "   macro avg       0.95      0.89      0.92     14576\n",
+      "weighted avg       1.00      1.00      1.00     14576\n",
       "\n",
       "Actual predicted values:\n",
       "(array([0.]), array([14576]))\n"
@@ -1134,23 +649,23 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 17,
+   "execution_count": 13,
    "id": "f63ca87a",
    "metadata": {},
    "outputs": [
     {
      "data": {
       "text/plain": [
-       "array([[9.98401165e-01, 1.59881881e-03, 0.00000000e+00],\n",
-       "       [9.99941409e-01, 5.85634953e-05, 0.00000000e+00],\n",
-       "       [9.97351170e-01, 2.64881272e-03, 0.00000000e+00],\n",
+       "array([[9.99999821e-01, 1.77245539e-07, 0.00000000e+00],\n",
+       "       [1.00000000e+00, 8.29310925e-11, 0.00000000e+00],\n",
+       "       [9.99995947e-01, 4.06420531e-06, 0.00000000e+00],\n",
        "       ...,\n",
-       "       [9.99494374e-01, 5.05603210e-04, 0.00000000e+00],\n",
-       "       [9.99981642e-01, 1.83574630e-05, 0.00000000e+00],\n",
-       "       [9.99996483e-01, 3.52600046e-06, 0.00000000e+00]])"
+       "       [9.99999940e-01, 3.85314713e-08, 0.00000000e+00],\n",
+       "       [1.00000000e+00, 2.98146707e-09, 0.00000000e+00],\n",
+       "       [1.00000000e+00, 1.21332046e-12, 0.00000000e+00]])"
       ]
      },
-     "execution_count": 17,
+     "execution_count": 13,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -1179,7 +694,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.10.13"
+   "version": "3.10.15"
   },
   "toc": {
    "base_numbering": 1,
diff --git a/mkdocs/sources/examples/15_Self_Supervised_Pretraning_pt2.ipynb b/mkdocs/sources/examples/15_Self_Supervised_Pretraning_pt2.ipynb
index 78d8f884..b39f1f78 100644
--- a/mkdocs/sources/examples/15_Self_Supervised_Pretraning_pt2.ipynb
+++ b/mkdocs/sources/examples/15_Self_Supervised_Pretraning_pt2.ipynb
@@ -58,7 +58,16 @@
    "execution_count": 1,
    "id": "e5b94ab2",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/Users/javierrodriguezzaurin/.pyenv/versions/3.10.15/envs/widedeep310/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
+      "  from .autonotebook import tqdm as notebook_tqdm\n"
+     ]
+    }
+   ],
    "source": [
     "import torch\n",
     "from sklearn.metrics import accuracy_score\n",
@@ -84,7 +93,7 @@
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "/Users/javierrodriguezzaurin/Projects/pytorch-widedeep/pytorch_widedeep/preprocessing/tab_preprocessor.py:358: UserWarning: Continuous columns will not be normalised\n",
+      "/Users/javierrodriguezzaurin/Projects/pytorch-widedeep/pytorch_widedeep/preprocessing/tab_preprocessor.py:364: UserWarning: Continuous columns will not be normalised\n",
       "  warnings.warn(\"Continuous columns will not be normalised\")\n"
      ]
     }
@@ -154,11 +163,11 @@
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "epoch 1: 100%|█████████████████████████████████████████████████████████████████████████████████████| 153/153 [00:13<00:00, 11.73it/s, loss=579]\n",
-      "epoch 2: 100%|█████████████████████████████████████████████████████████████████████████████████████| 153/153 [00:12<00:00, 12.56it/s, loss=143]\n",
-      "epoch 3: 100%|█████████████████████████████████████████████████████████████████████████████████████| 153/153 [00:12<00:00, 12.49it/s, loss=141]\n",
-      "epoch 4: 100%|█████████████████████████████████████████████████████████████████████████████████████| 153/153 [00:11<00:00, 12.77it/s, loss=138]\n",
-      "epoch 5: 100%|█████████████████████████████████████████████████████████████████████████████████████| 153/153 [00:11<00:00, 13.29it/s, loss=137]\n"
+      "epoch 1: 100%|██████████| 153/153 [00:22<00:00,  6.91it/s, loss=656]   \n",
+      "epoch 2: 100%|██████████| 153/153 [00:07<00:00, 20.84it/s, loss=141]\n",
+      "epoch 3: 100%|██████████| 153/153 [00:07<00:00, 20.75it/s, loss=137]\n",
+      "epoch 4: 100%|██████████| 153/153 [00:07<00:00, 21.14it/s, loss=135]\n",
+      "epoch 5: 100%|██████████| 153/153 [00:07<00:00, 21.05it/s, loss=134]\n"
      ]
     }
    ],
@@ -197,7 +206,16 @@
    "execution_count": 6,
    "id": "b60be9fd",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/var/folders/pd/_2wz_qt16yq1fk6jn_xtxqk40000gn/T/ipykernel_71058/884975850.py:2: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.\n",
+      "  contrastive_denoising_model = torch.load(\n"
+     ]
+    }
+   ],
    "source": [
     "# some time has passed, we load the model with torch as usual:\n",
     "contrastive_denoising_model = torch.load(\n",
@@ -224,7 +242,7 @@
       "text/plain": [
        "FTTransformer(\n",
        "  (cat_embed): SameSizeCatEmbeddings(\n",
-       "    (embed): Embedding(323, 32, padding_idx=0)\n",
+       "    (embed): Embedding(322, 32, padding_idx=0)\n",
        "    (dropout): Dropout(p=0.0, inplace=False)\n",
        "  )\n",
        "  (cont_norm): Identity()\n",
@@ -330,19 +348,19 @@
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "epoch 1: 100%|██████████████████████████████████████████████████████████| 153/153 [00:05<00:00, 27.19it/s, loss=0.383, metrics={'acc': 0.8176}]\n",
-      "epoch 2: 100%|██████████████████████████████████████████████████████████| 153/153 [00:05<00:00, 26.08it/s, loss=0.325, metrics={'acc': 0.8502}]\n",
-      "epoch 3: 100%|██████████████████████████████████████████████████████████| 153/153 [00:05<00:00, 26.56it/s, loss=0.306, metrics={'acc': 0.8601}]\n",
-      "epoch 4: 100%|██████████████████████████████████████████████████████████| 153/153 [00:05<00:00, 27.41it/s, loss=0.295, metrics={'acc': 0.8641}]\n",
-      "epoch 5: 100%|██████████████████████████████████████████████████████████| 153/153 [00:06<00:00, 24.70it/s, loss=0.289, metrics={'acc': 0.8656}]\n",
-      "predict: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████| 39/39 [00:00<00:00, 97.26it/s]"
+      "epoch 1: 100%|██████████| 153/153 [00:03<00:00, 50.99it/s, loss=0.383, metrics={'acc': 0.8218}]\n",
+      "epoch 2: 100%|██████████| 153/153 [00:02<00:00, 53.22it/s, loss=0.328, metrics={'acc': 0.8505}]\n",
+      "epoch 3: 100%|██████████| 153/153 [00:02<00:00, 54.50it/s, loss=0.309, metrics={'acc': 0.8588}]\n",
+      "epoch 4: 100%|██████████| 153/153 [00:02<00:00, 53.90it/s, loss=0.299, metrics={'acc': 0.8641}]\n",
+      "epoch 5: 100%|██████████| 153/153 [00:02<00:00, 54.02it/s, loss=0.292, metrics={'acc': 0.8665}]\n",
+      "predict: 100%|██████████| 39/39 [00:00<00:00, 88.79it/s] "
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "0.8695874705701709\n"
+      "0.8679496366055891\n"
      ]
     },
     {
@@ -368,14 +386,6 @@
     "test_acc = accuracy_score(target_te, preds)\n",
     "print(test_acc)"
    ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "2f61e849-6c90-4b51-9e91-67007025222d",
-   "metadata": {},
-   "outputs": [],
-   "source": []
   }
  ],
  "metadata": {
@@ -394,7 +404,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.10.13"
+   "version": "3.10.15"
   }
  },
  "nbformat": 4,
diff --git a/pytorch_widedeep/utils/fastai_transforms.py b/pytorch_widedeep/utils/fastai_transforms.py
index a19cef30..d3ee62aa 100644
--- a/pytorch_widedeep/utils/fastai_transforms.py
+++ b/pytorch_widedeep/utils/fastai_transforms.py
@@ -12,7 +12,7 @@
 import re
 import html
 from collections import Counter, defaultdict
-from concurrent.futures import ProcessPoolExecutor
+from multiprocessing import Pool
 
 import numpy as np
 import spacy
@@ -333,10 +333,12 @@ def process_all(self, texts: Collection[str]) -> List[List[str]]:
 
         if self.n_cpus <= 1:
             return self._process_all_1(texts)
-        with ProcessPoolExecutor(self.n_cpus) as e:
-            return sum(
-                e.map(self._process_all_1, partition_by_cores(texts, self.n_cpus)), []
-            )
+
+        with Pool(self.n_cpus) as p:
+            partitioned_texts = partition_by_cores(texts, self.n_cpus)
+            results = p.map(self._process_all_1, partitioned_texts)
+            res = sum(results, [])
+        return res
 
 
 class Vocab:
diff --git a/pytorch_widedeep/version.py b/pytorch_widedeep/version.py
index dc79f8f0..81162ee4 100644
--- a/pytorch_widedeep/version.py
+++ b/pytorch_widedeep/version.py
@@ -1 +1 @@
-__version__ = "1.6.4"
+__version__ = "1.6.5"

From 278c72f72873414f026e4905dbcbf710719b020e Mon Sep 17 00:00:00 2001
From: Javier <jrzaurin@gmail.com>
Date: Wed, 6 Nov 2024 15:02:28 +0000
Subject: [PATCH 24/24] removed tmp file added by mistake

---
 examples/scripts/tmp.py | 64 -----------------------------------------
 1 file changed, 64 deletions(-)
 delete mode 100644 examples/scripts/tmp.py

diff --git a/examples/scripts/tmp.py b/examples/scripts/tmp.py
deleted file mode 100644
index 970b4c38..00000000
--- a/examples/scripts/tmp.py
+++ /dev/null
@@ -1,64 +0,0 @@
-import numpy as np
-import pandas as pd
-
-from pytorch_widedeep import Trainer
-from pytorch_widedeep.models import TabMlp, WideDeep
-from pytorch_widedeep.metrics import Accuracy, Precision
-from pytorch_widedeep.datasets import load_adult
-from pytorch_widedeep.preprocessing import TabPreprocessor
-
-if __name__ == "__main__":
-    df: pd.DataFrame = load_adult(as_frame=True)
-    df.columns = [c.replace("-", "_") for c in df.columns]
-    df["age_buckets"] = pd.cut(
-        df.age, bins=[16, 25, 30, 35, 40, 45, 50, 55, 60, 91], labels=np.arange(9)
-    )
-    df["income_label"] = (df["income"].apply(lambda x: ">50K" in x)).astype(int)
-    df.drop("income", axis=1, inplace=True)
-
-    cat_embed_cols = [
-        "workclass",
-        "education",
-        "marital_status",
-        "occupation",
-        "relationship",
-        "race",
-        "gender",
-        "capital_gain",
-        "capital_loss",
-        "native_country",
-    ]
-    continuous_cols = ["age", "hours_per_week"]
-    target = "income_label"
-    target = df[target].values
-
-    tab_preprocessor = TabPreprocessor(
-        cat_embed_cols=cat_embed_cols,
-        continuous_cols=continuous_cols,
-        scale=True,  # type: ignore[arg-type]
-    )
-    X_tab = tab_preprocessor.fit_transform(df)
-
-    tab_mlp = TabMlp(
-        column_idx=tab_preprocessor.column_idx,
-        cat_embed_input=tab_preprocessor.cat_embed_input,
-        continuous_cols=continuous_cols,
-        mlp_hidden_dims=[200, 100],
-        mlp_dropout=[0.2, 0.2],
-    )
-
-    model = WideDeep(deeptabular=tab_mlp)
-
-    trainer = Trainer(
-        model,
-        objective="binary",
-        metrics=[Accuracy, Precision],
-    )
-
-    trainer.fit(
-        X_tab=X_tab,
-        target=target,
-        n_epochs=2,
-        batch_size=256,
-        val_split=0.2,
-    )